diff --git a/.dir-locals.el b/.dir-locals.el
new file mode 100644
index 0000000..3ac0cfc
--- /dev/null
+++ b/.dir-locals.el
@@ -0,0 +1,2 @@
+((c-mode . ((c-file-style . "stroustrup")
+	    (indent-tabs-mode . nil))))
diff --git a/.gitignore b/.gitignore
index e32a584..88ec249 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
 /config-target.*
 /config.status
 /config-temp
+/trace-events-all
 /trace/generated-tracers.h
 /trace/generated-tracers.c
 /trace/generated-tracers-dtrace.h
@@ -17,12 +18,15 @@
 /trace/generated-tcg-tracers.h
 /trace/generated-ust-provider.h
 /trace/generated-ust.c
-/libcacard/trace/generated-tracers.c
+/ui/shader/texture-blit-frag.h
+/ui/shader/texture-blit-vert.h
 *-timestamp
 /*-softmmu
 /*-darwin-user
 /*-linux-user
 /*-bsd-user
+/ivshmem-client
+/ivshmem-server
 /libdis*
 /libuser
 /linux-headers/asm
@@ -32,19 +36,14 @@
 /qapi-visit.[ch]
 /qapi-event.[ch]
 /qmp-commands.h
+/qmp-introspect.[ch]
 /qmp-marshal.c
 /qemu-doc.html
 /qemu-tech.html
 /qemu-doc.info
 /qemu-tech.info
-/qemu.1
-/qemu.pod
-/qemu-img.1
-/qemu-img.pod
 /qemu-img
 /qemu-nbd
-/qemu-nbd.8
-/qemu-nbd.pod
 /qemu-options.def
 /qemu-options.texi
 /qemu-img-cmds.texi
@@ -53,16 +52,17 @@
 /qemu-ga
 /qemu-bridge-helper
 /qemu-monitor.texi
+/qemu-monitor-info.texi
 /qmp-commands.txt
 /vscclient
 /fsdev/virtfs-proxy-helper
-/fsdev/virtfs-proxy-helper.1
-/fsdev/virtfs-proxy-helper.pod
+*.[1-9]
 *.a
 *.aux
 *.cp
 *.dvi
 *.exe
+*.msi
 *.dll
 *.so
 *.mo
@@ -70,6 +70,7 @@
 *.ky
 *.log
 *.pdf
+*.pod
 *.cps
 *.fns
 *.kys
@@ -94,6 +95,10 @@
 /pc-bios/optionrom/linuxboot.bin
 /pc-bios/optionrom/linuxboot.raw
 /pc-bios/optionrom/linuxboot.img
+/pc-bios/optionrom/linuxboot_dma.asm
+/pc-bios/optionrom/linuxboot_dma.bin
+/pc-bios/optionrom/linuxboot_dma.raw
+/pc-bios/optionrom/linuxboot_dma.img
 /pc-bios/optionrom/multiboot.asm
 /pc-bios/optionrom/multiboot.bin
 /pc-bios/optionrom/multiboot.raw
@@ -108,4 +113,5 @@
 cscope.*
 tags
 TAGS
+docker-src.*
 *~
diff --git a/.travis.yml b/.travis.yml
index ad66e5b..f30b10e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,100 +1,101 @@
+sudo: false
 language: c
 python:
   - "2.4"
 compiler:
   - gcc
   - clang
+cache: ccache
+addons:
+  apt:
+    packages:
+      - libaio-dev
+      - libattr1-dev
+      - libbrlapi-dev
+      - libcap-ng-dev
+      - libgnutls-dev
+      - libgtk-3-dev
+      - libiscsi-dev
+      - liblttng-ust-dev
+      - libnfs-dev
+      - libncurses5-dev
+      - libnss3-dev
+      - libpixman-1-dev
+      - libpng12-dev
+      - librados-dev
+      - libsdl1.2-dev
+      - libseccomp-dev
+      - libspice-protocol-dev
+      - libspice-server-dev
+      - libssh2-1-dev
+      - liburcu-dev
+      - libusb-1.0-0-dev
+      - libvte-2.90-dev
+      - sparse
+      - uuid-dev
+
+# The channel name "irc.oftc.net#qemu" is encrypted against qemu/qemu
+# to prevent IRC notifications from forks. This was created using:
+# $ travis encrypt -r "qemu/qemu" "irc.oftc.net#qemu"
 notifications:
   irc:
     channels:
-      - "irc.oftc.net#qemu"
+      - secure: "F7GDRgjuOo5IUyRLqSkmDL7kvdU4UcH3Lm/W2db2JnDHTGCqgEdaYEYKciyCLZ57vOTsTsOgesN8iUT7hNHBd1KWKjZe9KDTZWppWRYVwAwQMzVeSOsbbU4tRoJ6Pp+3qhH1Z0eGYR9ZgKYAoTumDFgSAYRp4IscKS8jkoedOqM="
     on_success: change
     on_failure: always
 env:
   global:
-    - TEST_CMD=""
-    - EXTRA_CONFIG=""
-    # Development packages, EXTRA_PKGS saved for additional builds
-    - CORE_PKGS="libusb-1.0-0-dev libiscsi-dev librados-dev libncurses5-dev"
-    - NET_PKGS="libseccomp-dev libgnutls-dev libssh2-1-dev  libspice-server-dev libspice-protocol-dev libnss3-dev"
-    - GUI_PKGS="libgtk-3-dev libvte-2.90-dev libsdl1.2-dev libpng12-dev libpixman-1-dev"
-    - EXTRA_PKGS=""
+    - TEST_CMD="make check"
   matrix:
-    # Group major targets together with their linux-user counterparts
-    - TARGETS=alpha-softmmu,alpha-linux-user
-    - TARGETS=arm-softmmu,arm-linux-user,armeb-linux-user,aarch64-softmmu,aarch64-linux-user
-    - TARGETS=cris-softmmu,cris-linux-user
-    - TARGETS=i386-softmmu,i386-linux-user,x86_64-softmmu,x86_64-linux-user
-    - TARGETS=m68k-softmmu,m68k-linux-user
-    - TARGETS=microblaze-softmmu,microblazeel-softmmu,microblaze-linux-user,microblazeel-linux-user
-    - TARGETS=mips-softmmu,mips64-softmmu,mips64el-softmmu,mipsel-softmmu
-    - TARGETS=mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,mipsn32-linux-user,mipsn32el-linux-user
-    - TARGETS=or32-softmmu,or32-linux-user
-    - TARGETS=ppc-softmmu,ppc64-softmmu,ppcemb-softmmu,ppc-linux-user,ppc64-linux-user,ppc64abi32-linux-user,ppc64le-linux-user
-    - TARGETS=s390x-softmmu,s390x-linux-user
-    - TARGETS=sh4-softmmu,sh4eb-softmmu,sh4-linux-user sh4eb-linux-user
-    - TARGETS=sparc-softmmu,sparc64-softmmu,sparc-linux-user,sparc32plus-linux-user,sparc64-linux-user
-    - TARGETS=unicore32-softmmu,unicore32-linux-user
-    # Group remaining softmmu only targets into one build
-    - TARGETS=lm32-softmmu,moxie-softmmu,tricore-softmmu,xtensa-softmmu,xtensaeb-softmmu
+    - CONFIG=""
+    - CONFIG="--enable-debug --enable-debug-tcg --enable-trace-backends=log"
+    - CONFIG="--disable-linux-aio --disable-cap-ng --disable-attr --disable-brlapi --disable-uuid --disable-libusb"
+    - CONFIG="--enable-modules"
+    - CONFIG="--with-coroutine=ucontext"
+    - CONFIG="--with-coroutine=sigaltstack"
 git:
   # we want to do this ourselves
   submodules: false
 before_install:
+  - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew update ; fi
+  - if [ "$TRAVIS_OS_NAME" == "osx" ]; then brew install libffi gettext glib pixman ; fi
   - wget -O - http://people.linaro.org/~alex.bennee/qemu-submodule-git-seed.tar.xz | tar -xvJ
   - git submodule update --init --recursive
-  - sudo apt-get update -qq
-  - sudo apt-get install -qq ${CORE_PKGS} ${NET_PKGS} ${GUI_PKGS} ${EXTRA_PKGS}
 before_script:
-  - ./configure --target-list=${TARGETS} --enable-debug-tcg ${EXTRA_CONFIG}
+  - ./configure ${CONFIG}
 script:
-  - make -j2 && ${TEST_CMD}
+  - make -j3 && ${TEST_CMD}
 matrix:
-  # We manually include a number of additional build for non-standard bits
   include:
-    # Make check target (we only do this once)
-    - env:
-        - TARGETS=alpha-softmmu,arm-softmmu,aarch64-softmmu,cris-softmmu,
-                  i386-softmmu,x86_64-softmmu,m68k-softmmu,microblaze-softmmu,
-                  microblazeel-softmmu,mips-softmmu,mips64-softmmu,
-                  mips64el-softmmu,mipsel-softmmu,or32-softmmu,ppc-softmmu,
-                  ppc64-softmmu,ppcemb-softmmu,s390x-softmmu,sh4-softmmu,
-                  sh4eb-softmmu,sparc-softmmu,sparc64-softmmu,
-                  unicore32-softmmu,unicore32-linux-user,
-                  lm32-softmmu,moxie-softmmu,tricore-softmmu,xtensa-softmmu,
-                  xtensaeb-softmmu
-          TEST_CMD="make check"
+    # gprof/gcov are GCC features
+    - env: CONFIG="--enable-gprof --enable-gcov --disable-pie"
       compiler: gcc
-    # Debug related options
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_CONFIG="--enable-debug"
+    # We manually include builds which we disable "make check" for
+    - env: CONFIG="--enable-debug --enable-tcg-interpreter"
+           TEST_CMD=""
       compiler: gcc
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_CONFIG="--enable-debug --enable-tcg-interpreter"
+    - env: CONFIG="--enable-trace-backends=simple"
+           TEST_CMD=""
       compiler: gcc
-    # All the extra -dev packages
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_PKGS="libaio-dev libcap-ng-dev libattr1-dev libbrlapi-dev uuid-dev libusb-1.0.0-dev"
+    - env: CONFIG="--enable-trace-backends=ftrace"
+           TEST_CMD=""
       compiler: gcc
-    # Currently configure doesn't force --disable-pie
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_CONFIG="--enable-gprof --enable-gcov --disable-pie"
+    - env: CONFIG="--enable-trace-backends=ust"
+           TEST_CMD=""
       compiler: gcc
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_PKGS="sparse"
-           EXTRA_CONFIG="--enable-sparse"
+    - env: CONFIG="--with-coroutine=gthread"
+           TEST_CMD=""
       compiler: gcc
-    # All the trace backends (apart from dtrace)
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_CONFIG="--enable-trace-backends=stderr"
+    - env: CONFIG=""
+      os: osx
+      compiler: clang
+    - env: CONFIG=""
+      sudo: required
+      addons:
+      dist: trusty
       compiler: gcc
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_CONFIG="--enable-trace-backends=simple"
-      compiler: gcc
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-           EXTRA_CONFIG="--enable-trace-backends=ftrace"
-      compiler: gcc
-    - env: TARGETS=i386-softmmu,x86_64-softmmu
-          EXTRA_PKGS="liblttng-ust-dev liburcu-dev"
-          EXTRA_CONFIG="--enable-trace-backends=ust"
-      compiler: gcc
+      before_install:
+        - sudo apt-get update -qq
+        - sudo apt-get build-dep -qq qemu
+        - wget -O - http://people.linaro.org/~alex.bennee/qemu-submodule-git-seed.tar.xz | tar -xvJ
+        - git submodule update --init --recursive
diff --git a/CODING_STYLE b/CODING_STYLE
index d46cfa5..e7fde15 100644
--- a/CODING_STYLE
+++ b/CODING_STYLE
@@ -31,7 +31,11 @@
 
 2. Line width
 
-Lines are 80 characters; not longer.
+Lines should be 80 characters; try not to make them longer.
+
+Sometimes it is hard to do, especially when dealing with QEMU subsystems
+that use long function or symbol names.  Even in that case, do not make
+lines much longer than 80 characters.
 
 Rationale:
  - Some people like to tile their 24" screens with a 6x4 matrix of 80x24
@@ -39,6 +43,8 @@
    let them keep doing it.
  - Code and especially patches is much more readable if limited to a sane
    line length.  Eighty is traditional.
+ - The four-space indentation makes the most common excuse ("But look
+   at all that white space on the left!") moot.
  - It is the QEMU coding style.
 
 3. Naming
@@ -87,10 +93,15 @@
 
 5. Declarations
 
-Mixed declarations (interleaving statements and declarations within blocks)
-are not allowed; declarations should be at the beginning of blocks.  In other
-words, the code should not generate warnings if using GCC's
--Wdeclaration-after-statement option.
+Mixed declarations (interleaving statements and declarations within
+blocks) are generally not allowed; declarations should be at the beginning
+of blocks.
+
+Every now and then, an exception is made for declarations inside a
+#ifdef or #ifndef block: if the code looks nicer, such declarations can
+be placed at the top of the block even if there are statements above.
+On the other hand, however, it's often best to move that #ifdef/#ifndef
+block to a separate function altogether.
 
 6. Conditional statements
 
diff --git a/HACKING b/HACKING
index 12fbc8a..20a9101 100644
--- a/HACKING
+++ b/HACKING
@@ -157,3 +157,62 @@
  * you may assume that integers are 2s complement representation
  * you may assume that right shift of a signed integer duplicates
    the sign bit (ie it is an arithmetic shift, not a logical shift)
+
+In addition, QEMU assumes that the compiler does not use the latitude
+given in C99 and C11 to treat aspects of signed '<<' as undefined, as
+documented in the GNU Compiler Collection manual starting at version 4.0.
+
+7. Error handling and reporting
+
+7.1 Reporting errors to the human user
+
+Do not use printf(), fprintf() or monitor_printf().  Instead, use
+error_report() or error_vreport() from error-report.h.  This ensures the
+error is reported in the right place (current monitor or stderr), and in
+a uniform format.
+
+Use error_printf() & friends to print additional information.
+
+error_report() prints the current location.  In certain common cases
+like command line parsing, the current location is tracked
+automatically.  To manipulate it manually, use the loc_*() from
+error-report.h.
+
+7.2 Propagating errors
+
+An error can't always be reported to the user right where it's detected,
+but often needs to be propagated up the call chain to a place that can
+handle it.  This can be done in various ways.
+
+The most flexible one is Error objects.  See error.h for usage
+information.
+
+Use the simplest suitable method to communicate success / failure to
+callers.  Stick to common methods: non-negative on success / -1 on
+error, non-negative / -errno, non-null / null, or Error objects.
+
+Example: when a function returns a non-null pointer on success, and it
+can fail only in one way (as far as the caller is concerned), returning
+null on failure is just fine, and certainly simpler and a lot easier on
+the eyes than propagating an Error object through an Error ** parameter.
+
+Example: when a function's callers need to report details on failure
+only the function really knows, use Error **, and set suitable errors.
+
+Do not report an error to the user when you're also returning an error
+for somebody else to handle.  Leave the reporting to the place that
+consumes the error returned.
+
+7.3 Handling errors
+
+Calling exit() is fine when handling configuration errors during
+startup.  It's problematic during normal operation.  In particular,
+monitor commands should never exit().
+
+Do not call exit() or abort() to handle an error that can be triggered
+by the guest (e.g., some unimplemented corner case in guest code
+translation or device emulation).  Guests should not be able to
+terminate QEMU.
+
+Note that &error_fatal is just another way to exit(1), and &error_abort
+is just another way to abort().
diff --git a/LICENSE b/LICENSE
index da70e94..0e0b4b9 100644
--- a/LICENSE
+++ b/LICENSE
@@ -11,7 +11,7 @@
 
 As of July 2013, contributions under version 2 of the GNU General Public
 License (and no later version) are only accepted for the following files
-or directories: bsd-user/, linux-user/, hw/misc/vfio.c, hw/xen/xen_pt*.
+or directories: bsd-user/, linux-user/, hw/vfio/, hw/xen/xen_pt*.
 
 3) The Tiny Code Generator (TCG) is released under the BSD license
    (see license headers in files).
diff --git a/MAINTAINERS b/MAINTAINERS
index bcb69e8..b6fb84e 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -50,28 +50,46 @@
 
 General Project Administration
 ------------------------------
-M: Anthony Liguori <aliguori@amazon.com>
 M: Peter Maydell <peter.maydell@linaro.org>
 
+All patches CC here
+L: qemu-devel@nongnu.org
+F: *
+F: */
+
 Responsible Disclosure, Reporting Security Issues
 ------------------------------
 W: http://wiki.qemu.org/SecurityProcess
 M: Michael S. Tsirkin <mst@redhat.com>
-M: Anthony Liguori <aliguori@amazon.com>
 L: secalert@redhat.com
 
 Guest CPU cores (TCG):
 ----------------------
 Overall
 L: qemu-devel@nongnu.org
-S: Odd fixes
+M: Paolo Bonzini <pbonzini@redhat.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+M: Richard Henderson <rth@twiddle.net>
+S: Maintained
 F: cpu-exec.c
+F: cpu-exec-common.c
+F: cpus.c
 F: cputlb.c
+F: exec.c
 F: softmmu_template.h
-F: translate-all.c
-F: include/exec/cpu_ldst.h
-F: include/exec/cpu_ldst_template.h
+F: translate-all.*
+F: translate-common.c
+F: include/exec/cpu*.h
+F: include/exec/exec-all.h
 F: include/exec/helper*.h
+F: include/exec/tb-hash.h
+
+FPU emulation
+M: Aurelien Jarno <aurelien@aurel32.net>
+M: Peter Maydell <peter.maydell@linaro.org>
+S: Odd Fixes
+F: fpu/
+F: include/fpu/
 
 Alpha
 M: Richard Henderson <rth@twiddle.net>
@@ -79,13 +97,18 @@
 F: target-alpha/
 F: hw/alpha/
 F: tests/tcg/alpha/
+F: disas/alpha.c
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: target-arm/
 F: hw/arm/
 F: hw/cpu/a*mpcore.c
+F: disas/arm.c
+F: disas/arm-a64.cc
+F: disas/libvixl/
 
 CRIS
 M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
@@ -93,13 +116,18 @@
 F: target-cris/
 F: hw/cris/
 F: tests/tcg/cris/
+F: disas/cris.c
 
 LM32
 M: Michael Walle <michael@walle.cc>
 S: Maintained
 F: target-lm32/
+F: disas/lm32.c
 F: hw/lm32/
-F: hw/char/lm32_*
+F: hw/*/lm32_*
+F: hw/*/milkymist-*
+F: include/hw/char/lm32_juart.h
+F: include/hw/lm32/
 F: tests/tcg/lm32/
 
 M68K
@@ -112,6 +140,7 @@
 S: Maintained
 F: target-microblaze/
 F: hw/microblaze/
+F: disas/microblaze.c
 
 MIPS
 M: Aurelien Jarno <aurelien@aurel32.net>
@@ -120,11 +149,13 @@
 F: target-mips/
 F: hw/mips/
 F: tests/tcg/mips/
+F: disas/mips.c
 
 Moxie
 M: Anthony Green <green@moxielogic.com>
 S: Maintained
 F: target-moxie/
+F: disas/moxie.c
 
 OpenRISC
 M: Jia Liu <proljc@gmail.com>
@@ -134,11 +165,13 @@
 F: tests/tcg/openrisc/
 
 PowerPC
+M: David Gibson <david@gibson.dropbear.id.au>
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
 S: Maintained
 F: target-ppc/
 F: hw/ppc/
+F: disas/ppc.c
 
 S390
 M: Richard Henderson <rth@twiddle.net>
@@ -146,19 +179,23 @@
 S: Maintained
 F: target-s390x/
 F: hw/s390x/
+F: disas/s390.c
 
 SH4
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Odd Fixes
 F: target-sh4/
 F: hw/sh4/
+F: disas/sh4.c
 
 SPARC
-M: Blue Swirl <blauwirbel@gmail.com>
+M: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk>
+M: Artyom Tarasenko <atar4qemu@gmail.com>
 S: Maintained
 F: target-sparc/
 F: hw/sparc/
 F: hw/sparc64/
+F: disas/sparc.c
 
 UniCore32
 M: Guan Xuetao <gxt@mprc.pku.edu.cn>
@@ -169,9 +206,11 @@
 X86
 M: Paolo Bonzini <pbonzini@redhat.com>
 M: Richard Henderson <rth@twiddle.net>
-S: Odd Fixes
+M: Eduardo Habkost <ehabkost@redhat.com>
+S: Maintained
 F: target-i386/
 F: hw/i386/
+F: disas/i386.c
 
 Xtensa
 M: Max Filippov <jcmvbkbc@gmail.com>
@@ -196,9 +235,11 @@
 S: Supported
 F: kvm-*
 F: */kvm.*
+F: include/sysemu/kvm*.h
 
 ARM
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: target-arm/kvm.c
 
@@ -218,9 +259,14 @@
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
 F: target-s390x/kvm.c
+F: target-s390x/ioinst.[ch]
+F: target-s390x/machine.c
 F: hw/intc/s390_flic.c
 F: hw/intc/s390_flic_kvm.c
 F: include/hw/s390x/s390_flic.h
+F: gdb-xml/s390*.xml
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 X86
 M: Paolo Bonzini <pbonzini@redhat.com>
@@ -233,11 +279,21 @@
 ----------------------
 
 X86
-M: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
+M: Stefano Stabellini <sstabellini@kernel.org>
+M: Anthony Perard <anthony.perard@citrix.com>
 L: xen-devel@lists.xensource.com
 S: Supported
 F: xen-*
 F: */xen*
+F: hw/char/xen_console.c
+F: hw/display/xenfb.c
+F: hw/net/xen_nic.c
+F: hw/block/xen_*
+F: hw/xen/
+F: hw/xenpv/
+F: hw/i386/xen/
+F: include/hw/xen/
+F: include/sysemu/xen-mapcache.h
 
 Hosts:
 ------
@@ -258,99 +314,166 @@
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: *win32*
+F: qemu.nsi
 
 ARM Machines
 ------------
 Allwinner-a10
-M: Li Guang <lig.fnst@cn.fujitsu.com>
+M: Beniamino Galvani <b.galvani@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
-F: hw/*/allwinner-a10*
-F: include/hw/*/allwinner-a10*
+F: hw/*/allwinner*
+F: include/hw/*/allwinner*
 F: hw/arm/cubieboard.c
 
+ARM PrimeCell
+M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/char/pl011.c
+F: hw/display/pl110*
+F: hw/dma/pl080.c
+F: hw/dma/pl330.c
+F: hw/gpio/pl061.c
+F: hw/input/pl050.c
+F: hw/intc/pl190.c
+F: hw/sd/pl181.c
+F: hw/timer/pl031.c
+F: include/hw/arm/primecell.h
+
+ARM cores
+M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/intc/arm*
+F: hw/intc/gic_internal.h
+F: hw/misc/a9scu.c
+F: hw/misc/arm11scu.c
+F: hw/timer/a9gtimer*
+F: hw/timer/arm_*
+F: include/hw/arm/arm.h
+F: include/hw/intc/arm*
+F: include/hw/misc/a9scu.h
+F: include/hw/misc/arm11scu.h
+F: include/hw/timer/a9gtimer.h
+F: include/hw/timer/arm_mptimer.h
+
 Exynos
-M: Evgeny Voevodin <e.voevodin@samsung.com>
-M: Maksim Kozlov <m.kozlov@samsung.com>
 M: Igor Mitsyanko <i.mitsyanko@gmail.com>
-M: Dmitry Solodkiy <d.solodkiy@samsung.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/exynos*
+F: include/hw/arm/exynos4210.h
 
 Calxeda Highbank
 M: Rob Herring <robh@kernel.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/highbank.c
 F: hw/net/xgmac.c
 
 Canon DIGIC
 M: Antony Pavlov <antonynpavlov@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: include/hw/arm/digic.h
 F: hw/*/digic*
 
 Gumstix
 L: qemu-devel@nongnu.org
+L: qemu-arm@nongnu.org
 S: Orphan
 F: hw/arm/gumstix.c
 
 i.MX31
 M: Peter Chubb <peter.chubb@nicta.com.au>
+L: qemu-arm@nongnu.org
 S: Odd fixes
 F: hw/*/imx*
 F: hw/arm/kzm.c
+F: include/hw/arm/fsl-imx31.h
 
 Integrator CP
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/integratorcp.c
 
-Mainstone
-L: qemu-devel@nongnu.org
-S: Orphan
-F: hw/arm/mainstone.c
-
 Musicpal
 M: Jan Kiszka <jan.kiszka@web.de>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/musicpal.c
 
 nSeries
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/nseries.c
 
 Palm
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/palm.c
 
 Real View
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/arm/realview*
+F: hw/intc/realview_gic.c
+F: include/hw/intc/realview_gic.h
 
-Spitz
+PXA2XX
 M: Andrzej Zaborowski <balrogg@gmail.com>
+L: qemu-arm@nongnu.org
 S: Maintained
+F: hw/arm/mainstone.c
 F: hw/arm/spitz.c
+F: hw/arm/tosa.c
+F: hw/arm/z2.c
+F: hw/*/pxa2xx*
+F: include/hw/arm/pxa.h
 
 Stellaris
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/stellaris*
 
 Versatile PB
 M: Peter Maydell <peter.maydell@linaro.org>
+L: qemu-arm@nongnu.org
 S: Maintained
 F: hw/*/versatile*
 
 Xilinx Zynq
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+M: Alistair Francis <alistair.francis@xilinx.com>
+L: qemu-arm@nongnu.org
 S: Maintained
-F: hw/arm/xilinx_zynq.c
-F: hw/misc/zynq_slcr.c
+F: hw/*/xilinx_*
 F: hw/*/cadence_*
-F: hw/ssi/xilinx_spips.c
+F: hw/misc/zynq_slcr.c
+F: include/hw/xilinx.h
+X: hw/ssi/xilinx_*
+
+Xilinx ZynqMP
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/*/xlnx*.c
+F: include/hw/*/xlnx*.c
+
+ARM ACPI Subsystem
+M: Shannon Zhao <zhaoshenglong@huawei.com>
+M: Shannon Zhao <shannon.zhao@linaro.org>
+L: qemu-arm@nongnu.org
+S: Maintained
+F: hw/arm/virt-acpi-build.c
+F: include/hw/arm/virt-acpi-build.h
 
 CRIS Machines
 -------------
@@ -394,7 +517,7 @@
 F: hw/microblaze/petalogix_s3adsp1800_mmu.c
 
 petalogix_ml605
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
 S: Maintained
 F: hw/microblaze/petalogix_ml605_mmu.c
 
@@ -475,7 +598,7 @@
 F: hw/misc/macio/
 
 PReP
-M: Andreas Färber <andreas.faerber@web.de>
+L: qemu-devel@nongnu.org
 L: qemu-ppc@nongnu.org
 S: Odd Fixes
 F: hw/ppc/prep.c
@@ -483,6 +606,7 @@
 F: hw/isa/pc87312.[hc]
 
 sPAPR
+M: David Gibson <david@gibson.dropbear.id.au>
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
 S: Supported
@@ -514,11 +638,13 @@
 --------------
 Sun4m
 M: Blue Swirl <blauwirbel@gmail.com>
+M: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk>
 S: Maintained
 F: hw/sparc/sun4m.c
 
 Sun4u
 M: Blue Swirl <blauwirbel@gmail.com>
+M: Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk>
 S: Maintained
 F: hw/sparc64/sun4u.c
 
@@ -530,23 +656,18 @@
 
 S390 Machines
 -------------
-S390 Virtio
-M: Alexander Graf <agraf@suse.de>
-S: Maintained
-F: hw/s390x/s390-*.c
-
 S390 Virtio-ccw
 M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Christian Borntraeger <borntraeger@de.ibm.com>
 M: Alexander Graf <agraf@suse.de>
 S: Supported
-F: hw/s390x/s390-virtio-ccw.c
-F: hw/s390x/css.[hc]
-F: hw/s390x/sclp*.[hc]
-F: hw/s390x/ipl*.[hc]
+F: hw/char/sclp*.[hc]
+F: hw/s390x/
 F: include/hw/s390x/
 F: pc-bios/s390-ccw/
-T: git git://github.com/cohuck/qemu virtio-ccw-upstr
+F: hw/watchdog/wdt_diag288.c
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
 
 UniCore32 Machines
 -------------
@@ -559,7 +680,6 @@
 X86 Machines
 ------------
 PC
-M: Anthony Liguori <aliguori@amazon.com>
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
 F: include/hw/i386/
@@ -576,7 +696,32 @@
 F: hw/acpi/ich9.c
 F: include/hw/acpi/ich9.h
 F: include/hw/acpi/piix.h
+F: hw/misc/sga.c
 
+PC Chipset
+M: Michael S. Tsirkin <mst@redhat.com>
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Support
+F: hw/char/debugcon.c
+F: hw/char/parallel.c
+F: hw/char/serial*
+F: hw/dma/i8257*
+F: hw/i2c/pm_smbus.c
+F: hw/intc/apic*
+F: hw/intc/ioapic*
+F: hw/intc/i8259*
+F: hw/misc/debugexit.c
+F: hw/misc/pc-testdev.c
+F: hw/timer/hpet*
+F: hw/timer/i8254*
+F: hw/timer/mc146818rtc*
+
+Machine core
+M: Eduardo Habkost <ehabkost@redhat.com>
+M: Marcel Apfelbaum <marcel@redhat.com>
+S: Supported
+F: hw/core/machine.c
+F: include/hw/boards.h
 
 Xtensa Machines
 ---------------
@@ -593,24 +738,69 @@
 
 Devices
 -------
+EDU
+M: Jiri Slaby <jslaby@suse.cz>
+S: Maintained
+F: hw/misc/edu.c
+
 IDE
-M: Kevin Wolf <kwolf@redhat.com>
-M: Stefan Hajnoczi <stefanha@redhat.com>
-S: Odd Fixes
+M: John Snow <jsnow@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
 F: include/hw/ide.h
 F: hw/ide/
+F: hw/block/block.c
+F: hw/block/cdrom.c
+F: hw/block/hd-geometry.c
+F: tests/ide-test.c
+F: tests/ahci-test.c
+F: tests/libqos/ahci*
+T: git git://github.com/jnsnow/qemu.git ide
+
+Floppy
+M: John Snow <jsnow@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: hw/block/fdc.c
+F: include/hw/block/fdc.h
+F: tests/fdc-test.c
+T: git git://github.com/jnsnow/qemu.git ide
 
 OMAP
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Maintained
 F: hw/*/omap*
+F: include/hw/arm/omap.h
+
+IPack
+M: Alberto Garcia <berto@igalia.com>
+S: Odd Fixes
+F: hw/char/ipoctal232.c
+F: hw/ipack/
 
 PCI
 M: Michael S. Tsirkin <mst@redhat.com>
+M: Marcel Apfelbaum <marcel@redhat.com>
 S: Supported
 F: include/hw/pci/*
+F: hw/misc/pci-testdev.c
 F: hw/pci/*
+F: hw/pci-bridge/*
+
+ACPI/SMBIOS
+M: Michael S. Tsirkin <mst@redhat.com>
+M: Igor Mammedov <imammedo@redhat.com>
+S: Supported
+F: include/hw/acpi/*
+F: include/hw/smbios/*
+F: hw/mem/*
 F: hw/acpi/*
+F: hw/smbios/*
+F: hw/i386/acpi-build.[hc]
+F: hw/i386/*dsl
+F: hw/arm/virt-acpi-build.c
+F: include/hw/arm/virt-acpi-build.h
+F: scripts/acpi*py
 
 ppc4xx
 M: Alexander Graf <agraf@suse.de>
@@ -624,6 +814,19 @@
 L: qemu-ppc@nongnu.org
 S: Supported
 F: hw/ppc/e500*
+F: hw/pci-host/ppce500.c
+F: hw/net/fsl_etsec/
+
+Character devices
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Odd Fixes
+F: hw/char/
+
+Network devices
+M: Jason Wang <jasowang@redhat.com>
+S: Odd Fixes
+F: hw/net/
+T: git git://github.com/jasowang/qemu.git net
 
 SCSI
 M: Paolo Bonzini <pbonzini@redhat.com>
@@ -637,16 +840,27 @@
 F: hw/scsi/lsi53c895a.c
 
 SSI
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 S: Maintained
 F: hw/ssi/*
 F: hw/block/m25p80.c
+X: hw/ssi/xilinx_*
+
+Xilinx SPI
+M: Alistair Francis <alistair.francis@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
+S: Maintained
+F: hw/ssi/xilinx_*
 
 USB
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: hw/usb/*
 F: tests/usb-*-test.c
+F: docs/usb2.txt
+F: docs/usb-storage.txt
+F: include/hw/usb.h
+F: include/hw/usb/
 
 USB (serial adapter)
 M: Gerd Hoffmann <kraxel@redhat.com>
@@ -657,7 +871,8 @@
 VFIO
 M: Alex Williamson <alex.williamson@redhat.com>
 S: Supported
-F: hw/misc/vfio.c
+F: hw/vfio/*
+F: include/hw/vfio/
 
 vhost
 M: Michael S. Tsirkin <mst@redhat.com>
@@ -665,56 +880,81 @@
 F: hw/*/*vhost*
 
 virtio
-M: Anthony Liguori <aliguori@amazon.com>
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
 F: hw/*/virtio*
+F: net/vhost-user.c
+F: include/hw/virtio/
 
 virtio-9p
 M: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+M: Greg Kurz <groug@kaod.org>
 S: Supported
 F: hw/9pfs/
 F: fsdev/
 F: tests/virtio-9p-test.c
 T: git git://github.com/kvaneesh/QEMU.git
+T: git git://github.com/gkurz/qemu.git 9p-next
 
 virtio-blk
-M: Kevin Wolf <kwolf@redhat.com>
 M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
 S: Supported
 F: hw/block/virtio-blk.c
+F: hw/block/dataplane/*
+F: hw/virtio/dataplane/*
+T: git git://github.com/stefanha/qemu.git block
 
 virtio-ccw
 M: Cornelia Huck <cornelia.huck@de.ibm.com>
 M: Christian Borntraeger <borntraeger@de.ibm.com>
 S: Supported
 F: hw/s390x/virtio-ccw.[hc]
-T: git git://github.com/cohuck/qemu virtio-ccw-upstr
+T: git git://github.com/cohuck/qemu.git s390-next
+T: git git://github.com/borntraeger/qemu.git s390-next
+
+virtio-input
+M: Gerd Hoffmann <kraxel@redhat.com>
+S: Maintained
+F: hw/input/virtio-input*.c
+F: include/hw/virtio/virtio-input.h
 
 virtio-serial
 M: Amit Shah <amit.shah@redhat.com>
 S: Supported
 F: hw/char/virtio-serial-bus.c
 F: hw/char/virtio-console.c
+F: include/hw/virtio/virtio-serial.h
+
+virtio-rng
+M: Amit Shah <amit.shah@redhat.com>
+S: Supported
+F: hw/virtio/virtio-rng.c
+F: include/hw/virtio/virtio-rng.h
+F: include/sysemu/rng*.h
+F: backends/rng*.c
 
 nvme
 M: Keith Busch <keith.busch@intel.com>
+L: qemu-block@nongnu.org
 S: Supported
 F: hw/block/nvme*
 F: tests/nvme-test.c
 
 megasas
 M: Hannes Reinecke <hare@suse.de>
+L: qemu-block@nongnu.org
 S: Supported
 F: hw/scsi/megasas.c
 F: hw/scsi/mfi.h
 
-Xilinx EDK
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
-M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+Network packet abstractions
+M: Dmitry Fleytman <dmitry@daynix.com>
 S: Maintained
-F: hw/*/xilinx_*
-F: include/hw/xilinx.h
+F: include/net/eth.h
+F: net/eth.c
+F: hw/net/net_rx_pkt*
+F: hw/net/net_tx_pkt*
 
 Vmware
 M: Dmitry Fleytman <dmitry@daynix.com>
@@ -722,10 +962,31 @@
 F: hw/net/vmxnet*
 F: hw/scsi/vmw_pvscsi*
 
+Rocker
+M: Jiri Pirko <jiri@resnulli.us>
+S: Maintained
+F: hw/net/rocker/
+
+NVDIMM
+M: Xiao Guangrong <guangrong.xiao@linux.intel.com>
+S: Maintained
+F: hw/acpi/nvdimm.c
+F: hw/mem/nvdimm.c
+F: include/hw/mem/nvdimm.h
+
+e1000x
+M: Dmitry Fleytman <dmitry@daynix.com>
+S: Maintained
+F: hw/net/e1000x*
+
+e1000e
+M: Dmitry Fleytman <dmitry@daynix.com>
+S: Maintained
+F: hw/net/e1000e*
+
 Subsystems
 ----------
 Audio
-M: Vassili Karpov (malc) <av1474@comtv.ru>
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: audio/
@@ -734,24 +995,53 @@
 F: tests/es1370-test.c
 F: tests/intel-hda-test.c
 
-Block
+Block layer core
 M: Kevin Wolf <kwolf@redhat.com>
-M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Max Reitz <mreitz@redhat.com>
+L: qemu-block@nongnu.org
 S: Supported
-F: async.c
-F: aio-*.c
 F: block*
 F: block/
 F: hw/block/
+F: include/block/
 F: qemu-img*
 F: qemu-io*
-F: tests/image-fuzzer/
 F: tests/qemu-iotests/
 T: git git://repo.or.cz/qemu/kevin.git block
+
+Block I/O path
+M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Fam Zheng <famz@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: async.c
+F: aio-*.c
+F: block/io.c
+F: migration/block*
+F: include/block/aio.h
 T: git git://github.com/stefanha/qemu.git block
 
-Character Devices
-M: Anthony Liguori <aliguori@amazon.com>
+Block Jobs
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: blockjob.c
+F: include/block/blockjob.h
+F: block/backup.c
+F: block/commit.c
+F: block/stream.c
+F: block/mirror.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
+
+Block QAPI, monitor, command line
+M: Markus Armbruster <armbru@redhat.com>
+S: Supported
+F: blockdev.c
+F: block/qapi.c
+F: qapi/block*.json
+T: git git://repo.or.cz/qemu/armbru.git block-next
+
+Character device backends
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: qemu-char.c
@@ -763,12 +1053,16 @@
 S: Maintained
 F: backends/baum.c
 
+Coverity model
+M: Markus Armbruster <armbru@redhat.com>
+S: Supported
+F: scripts/coverity-model.c
+
 CPU
-M: Andreas Färber <afaerber@suse.de>
+L: qemu-devel@nongnu.org
 S: Supported
 F: qom/cpu.c
 F: include/qom/cpu.h
-F: target-i386/cpu.c
 
 ICC Bus
 M: Igor Mammedov <imammedo@redhat.com>
@@ -777,10 +1071,19 @@
 F: hw/cpu/icc_bus.c
 
 Device Tree
-M: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+M: Peter Crosthwaite <crosthwaite.peter@gmail.com>
 M: Alexander Graf <agraf@suse.de>
 S: Maintained
-F: device_tree.[ch]
+F: device_tree.c
+F: include/sysemu/device_tree.h
+
+Error reporting
+M: Markus Armbruster <armbru@redhat.com>
+S: Supported
+F: include/qapi/error.h
+F: include/qemu/error-report.h
+F: util/error.c
+F: util/qemu-error.c
 
 GDB stub
 L: qemu-devel@nongnu.org
@@ -794,6 +1097,7 @@
 F: include/exec/ioport.h
 F: ioport.c
 F: include/exec/memory.h
+F: include/exec/ram_addr.h
 F: memory.c
 F: include/exec/memory-internal.h
 F: exec.c
@@ -802,24 +1106,23 @@
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Supported
 F: include/ui/qemu-spice.h
+F: include/ui/spice-display.h
 F: ui/spice-*.c
 F: audio/spiceaudio.c
 F: hw/display/qxl*
 
 Graphics
-M: Anthony Liguori <aliguori@amazon.com>
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Odd Fixes
 F: ui/
+F: include/ui/
 
 Cocoa graphics
-M: Andreas Färber <andreas.faerber@web.de>
 M: Peter Maydell <peter.maydell@linaro.org>
 S: Odd Fixes
 F: ui/cocoa.m
 
 Main loop
-M: Anthony Liguori <aliguori@amazon.com>
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Maintained
 F: cpus.c
@@ -835,12 +1138,12 @@
 F: hmp-commands.hx
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
-Network device layer
-M: Anthony Liguori <aliguori@amazon.com>
-M: Stefan Hajnoczi <stefanha@redhat.com>
+Network device backends
+M: Jason Wang <jasowang@redhat.com>
 S: Maintained
 F: net/
-T: git git://github.com/stefanha/qemu.git net
+F: include/net/
+T: git git://github.com/jasowang/qemu.git net
 
 Netmap network backend
 M: Luigi Rizzo <rizzo@iet.unipi.it>
@@ -853,31 +1156,62 @@
 Network Block Device (NBD)
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Odd Fixes
-F: block/nbd.c
-F: nbd.*
+F: block/nbd*
+F: nbd/
+F: include/block/nbd*
 F: qemu-nbd.c
 T: git git://github.com/bonzini/qemu.git nbd-next
 
-QAPI
-M: Luiz Capitulino <lcapitulino@redhat.com>
-M: Michael Roth <mdroth@linux.vnet.ibm.com>
+NUMA
+M: Eduardo Habkost <ehabkost@redhat.com>
 S: Maintained
+F: numa.c
+F: include/sysemu/numa.h
+T: git git://github.com/ehabkost/qemu.git numa
+
+Host Memory Backends
+M: Eduardo Habkost <ehabkost@redhat.com>
+M: Igor Mammedov <imammedo@redhat.com>
+S: Maintained
+F: backends/hostmem*.c
+F: include/sysemu/hostmem.h
+
+QAPI
+M: Markus Armbruster <armbru@redhat.com>
+M: Michael Roth <mdroth@linux.vnet.ibm.com>
+S: Supported
 F: qapi/
+X: qapi/*.json
+F: include/qapi/
+X: include/qapi/qmp/
+F: include/qapi/qmp/dispatch.h
 F: tests/qapi-schema/
-T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
+F: tests/test-*-visitor.c
+F: tests/test-qmp-*.c
+F: scripts/qapi*
+F: docs/qapi*
+T: git git://repo.or.cz/qemu/armbru.git qapi-next
 
 QAPI Schema
 M: Eric Blake <eblake@redhat.com>
-M: Luiz Capitulino <lcapitulino@redhat.com>
 M: Markus Armbruster <armbru@redhat.com>
 S: Supported
 F: qapi-schema.json
-T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
+F: qapi/*.json
+T: git git://repo.or.cz/qemu/armbru.git qapi-next
 
 QObject
 M: Luiz Capitulino <lcapitulino@redhat.com>
 S: Maintained
 F: qobject/
+F: include/qapi/qmp/
+X: include/qapi/qmp/dispatch.h
+F: tests/check-qdict.c
+F: tests/check-qfloat.c
+F: tests/check-qint.c
+F: tests/check-qjson.c
+F: tests/check-qlist.c
+F: tests/check-qstring.c
 T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
 
 QEMU Guest Agent
@@ -887,7 +1221,6 @@
 T: git git://github.com/mdroth/qemu.git qga
 
 QOM
-M: Anthony Liguori <aliguori@amazon.com>
 M: Andreas Färber <afaerber@suse.de>
 S: Supported
 T: git git://github.com/afaerber/qemu-cpu.git qom-next
@@ -895,21 +1228,33 @@
 X: include/qom/cpu.h
 F: qom/
 X: qom/cpu.c
+F: tests/check-qom-interface.c
+F: tests/check-qom-proplist.c
 F: tests/qom-test.c
 
 QMP
-M: Luiz Capitulino <lcapitulino@redhat.com>
-S: Maintained
+M: Markus Armbruster <armbru@redhat.com>
+S: Supported
 F: qmp.c
 F: monitor.c
 F: qmp-commands.hx
-F: QMP/
-T: git git://repo.or.cz/qemu/qmp-unstable.git queue/qmp
+F: docs/*qmp-*
+F: scripts/qmp/
+T: git git://repo.or.cz/qemu/armbru.git qapi-next
+
+Register API
+M: Alistair Francis <alistair.francis@xilinx.com>
+S: Maintained
+F: hw/core/register.c
+F: include/hw/register.h
 
 SLIRP
+M: Samuel Thibault <samuel.thibault@ens-lyon.org>
 M: Jan Kiszka <jan.kiszka@siemens.com>
 S: Maintained
 F: slirp/
+F: net/slirp.c
+F: include/net/slirp.h
 T: git git://git.kiszka.org/qemu.git queues/slirp
 
 Tracing
@@ -922,18 +1267,18 @@
 T: git git://github.com/stefanha/qemu.git tracing
 
 Checkpatch
-M: Blue Swirl <blauwirbel@gmail.com>
 S: Odd Fixes
 F: scripts/checkpatch.pl
 
 Migration
 M: Juan Quintela <quintela@redhat.com>
+M: Amit Shah <amit.shah@redhat.com>
 S: Maintained
 F: include/migration/
-F: migration*
-F: savevm.c
-F: arch_init.c
-F: vmstate.c
+F: migration/
+F: scripts/vmstate-static-checker.py
+F: tests/vmstate-static-checker-data/
+F: docs/migration.txt
 
 Seccomp
 M: Eduardo Otubo <eduardo.otubo@profitbricks.com>
@@ -941,6 +1286,50 @@
 F: qemu-seccomp.c
 F: include/sysemu/seccomp.h
 
+Cryptography
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Maintained
+F: crypto/
+F: include/crypto/
+F: tests/test-crypto-*
+
+Coroutines
+M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Kevin Wolf <kwolf@redhat.com>
+F: util/*coroutine*
+F: include/qemu/coroutine*
+F: tests/test-coroutine.c
+
+Buffers
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Odd fixes
+F: util/buffer.c
+F: include/qemu/buffer.h
+
+I/O Channels
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Maintained
+F: io/
+F: include/io/
+F: tests/test-io-*
+
+Sockets
+M: Daniel P. Berrange <berrange@redhat.com>
+M: Gerd Hoffmann <kraxel@redhat.com>
+M: Paolo Bonzini <pbonzini@redhat.com>
+S: Maintained
+F: include/qemu/sockets.h
+F: util/qemu-sockets.c
+
+Throttling infrastructure
+M: Alberto Garcia <berto@igalia.com>
+S: Supported
+F: block/throttle-groups.c
+F: include/block/throttle-groups.h
+F: include/qemu/throttle.h
+F: util/throttle.c
+L: qemu-block@nongnu.org
+
 Usermode Emulation
 ------------------
 Overall
@@ -950,8 +1339,7 @@
 F: user-exec.c
 
 BSD user
-M: Blue Swirl <blauwirbel@gmail.com>
-S: Maintained
+S: Orphan
 F: bsd-user/
 
 Linux user
@@ -970,54 +1358,60 @@
 M: Claudio Fontana <claudio.fontana@huawei.com>
 M: Claudio Fontana <claudio.fontana@gmail.com>
 S: Maintained
+L: qemu-arm@nongnu.org
 F: tcg/aarch64/
+F: disas/arm-a64.cc
+F: disas/libvixl/
 
 ARM target
 M: Andrzej Zaborowski <balrogg@gmail.com>
 S: Maintained
+L: qemu-arm@nongnu.org
 F: tcg/arm/
+F: disas/arm.c
 
 i386 target
 L: qemu-devel@nongnu.org
 S: Maintained
 F: tcg/i386/
+F: disas/i386.c
 
 IA64 target
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Maintained
 F: tcg/ia64/
+F: disas/ia64.c
 
 MIPS target
 M: Aurelien Jarno <aurelien@aurel32.net>
 S: Maintained
 F: tcg/mips/
+F: disas/mips.c
 
 PPC
 M: Vassili Karpov (malc) <av1474@comtv.ru>
 S: Maintained
 F: tcg/ppc/
-
-PPC64 target
-M: Vassili Karpov (malc) <av1474@comtv.ru>
-S: Maintained
-F: tcg/ppc64/
+F: disas/ppc.c
 
 S390 target
 M: Alexander Graf <agraf@suse.de>
 M: Richard Henderson <rth@twiddle.net>
 S: Maintained
 F: tcg/s390/
+F: disas/s390.c
 
 SPARC target
-M: Blue Swirl <blauwirbel@gmail.com>
-S: Maintained
+S: Odd Fixes
 F: tcg/sparc/
+F: disas/sparc.c
 
 TCI target
 M: Stefan Weil <sw@weilnetz.de>
 S: Maintained
 F: tcg/tci/
 F: tci.c
+F: disas/tci.c
 
 Stable branches
 ---------------
@@ -1028,9 +1422,8 @@
 
 Stable 0.15
 L: qemu-stable@nongnu.org
-M: Andreas Färber <afaerber@suse.de>
 T: git git://git.qemu-project.org/qemu-stable-0.15.git
-S: Supported
+S: Orphan
 
 Stable 0.14
 L: qemu-stable@nongnu.org
@@ -1046,28 +1439,38 @@
 -------------
 VMDK
 M: Fam Zheng <famz@redhat.com>
+L: qemu-block@nongnu.org
 S: Supported
 F: block/vmdk.c
 
 RBD
-M: Josh Durgin <josh.durgin@inktank.com>
+M: Josh Durgin <jdurgin@redhat.com>
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
 S: Supported
 F: block/rbd.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
 
 Sheepdog
 M: Hitoshi Mitake <mitake.hitoshi@lab.ntt.co.jp>
 M: Liu Yuan <namei.unix@gmail.com>
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
 L: sheepdog@lists.wpkg.org
 S: Supported
 F: block/sheepdog.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
 
 VHDX
 M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
 S: Supported
 F: block/vhdx*
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
 
 VDI
 M: Stefan Weil <sw@weilnetz.de>
+L: qemu-block@nongnu.org
 S: Maintained
 F: block/vdi.c
 
@@ -1075,26 +1478,169 @@
 M: Ronnie Sahlberg <ronniesahlberg@gmail.com>
 M: Paolo Bonzini <pbonzini@redhat.com>
 M: Peter Lieven <pl@kamp.de>
+L: qemu-block@nongnu.org
 S: Supported
 F: block/iscsi.c
 
 NFS
+M: Jeff Cody <jcody@redhat.com>
 M: Peter Lieven <pl@kamp.de>
+L: qemu-block@nongnu.org
 S: Maintained
 F: block/nfs.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
 
 SSH
 M: Richard W.M. Jones <rjones@redhat.com>
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
 S: Supported
 F: block/ssh.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
 
 ARCHIPELAGO
-M: Chrysostomos Nanakos <cnanakos@grnet.gr>
 M: Chrysostomos Nanakos <chris@include.gr>
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
 S: Maintained
 F: block/archipelago.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
+
+CURL
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/curl.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
+
+GLUSTER
+M: Jeff Cody <jcody@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/gluster.c
+T: git git://github.com/codyprime/qemu-kvm-jtc.git block
+
+Null Block Driver
+M: Fam Zheng <famz@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/null.c
 
 Bootdevice
 M: Gonglei <arei.gonglei@huawei.com>
 S: Maintained
 F: bootdevice.c
+
+Quorum
+M: Alberto Garcia <berto@igalia.com>
+S: Supported
+F: block/quorum.c
+L: qemu-block@nongnu.org
+
+blkverify
+M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/blkverify.c
+
+bochs
+M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/bochs.c
+
+cloop
+M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/cloop.c
+
+dmg
+M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/dmg.c
+
+parallels
+M: Stefan Hajnoczi <stefanha@redhat.com>
+M: Denis V. Lunev <den@openvz.org>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/parallels.c
+F: docs/specs/parallels.txt
+
+qed
+M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/qed.c
+
+raw
+M: Kevin Wolf <kwolf@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/linux-aio.c
+F: block/raw-aio.h
+F: block/raw-posix.c
+F: block/raw-win32.c
+F: block/raw_bsd.c
+F: block/win32-aio.c
+
+qcow2
+M: Kevin Wolf <kwolf@redhat.com>
+M: Max Reitz <mreitz@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/qcow2*
+
+qcow
+M: Kevin Wolf <kwolf@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/qcow.c
+
+blkdebug
+M: Kevin Wolf <kwolf@redhat.com>
+M: Max Reitz <mreitz@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/blkdebug.c
+
+vpc
+M: Kevin Wolf <kwolf@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/vpc.c
+
+vvfat
+M: Kevin Wolf <kwolf@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: block/vvfat.c
+
+Image format fuzzer
+M: Stefan Hajnoczi <stefanha@redhat.com>
+L: qemu-block@nongnu.org
+S: Supported
+F: tests/image-fuzzer/
+
+Build and test automation
+-------------------------
+M: Alex Bennée <alex.bennee@linaro.org>
+L: qemu-devel@nongnu.org
+S: Supported
+F: .travis.yml
+
+Documentation
+-------------
+Build system architecture
+M: Daniel P. Berrange <berrange@redhat.com>
+S: Odd Fixes
+F: docs/build-system.txt
+
+Docker testing
+--------------
+Docker based testing framework and cases
+M: Fam Zheng <famz@redhat.com>
+S: Maintained
+F: tests/docker/
diff --git a/Makefile b/Makefile
index f505202..50b4b3a 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,11 @@
 # Always point to the root of the build tree (needs GNU make).
 BUILD_DIR=$(CURDIR)
 
+# Before including a proper config-host.mak, assume we are in the source tree
+SRC_PATH=.
+
+UNCHECKED_GOALS := %clean TAGS cscope ctags docker docker-%
+
 # All following code might depend on configuration variables
 ifneq ($(wildcard config-host.mak),)
 # Put the all: rule here so that config-host.mak can contain dependencies.
@@ -25,8 +30,7 @@
 -include config-all-devices.mak
 -include config-all-disas.mak
 
-include $(SRC_PATH)/rules.mak
-config-host.mak: $(SRC_PATH)/configure
+config-host.mak: $(SRC_PATH)/configure $(SRC_PATH)/pc-bios
 	@echo $@ is out-of-date, running configure
 	@# TODO: The next lines include code which supports a smooth
 	@# transition from old configurations without config.status.
@@ -38,15 +42,19 @@
 	fi
 else
 config-host.mak:
-ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
+ifneq ($(filter-out $(UNCHECKED_GOALS),$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 	@echo "Please call configure before running make!"
 	@exit 1
 endif
 endif
 
-GENERATED_HEADERS = config-host.h qemu-options.def
+include $(SRC_PATH)/rules.mak
+
+GENERATED_HEADERS = qemu-version.h config-host.h qemu-options.def
 GENERATED_HEADERS += qmp-commands.h qapi-types.h qapi-visit.h qapi-event.h
 GENERATED_SOURCES += qmp-marshal.c qapi-types.c qapi-visit.c qapi-event.c
+GENERATED_HEADERS += qmp-introspect.h
+GENERATED_SOURCES += qmp-introspect.c
 
 GENERATED_HEADERS += trace/generated-events.h
 GENERATED_SOURCES += trace/generated-events.c
@@ -74,7 +82,7 @@
 configure: ;
 
 .PHONY: all clean cscope distclean dvi html info install install-doc \
-	pdf recurse-all speed test dist
+	pdf recurse-all speed test dist msi FORCE
 
 $(call set-vpath, $(SRC_PATH))
 
@@ -83,7 +91,8 @@
 HELPERS-$(CONFIG_LINUX) = qemu-bridge-helper$(EXESUF)
 
 ifdef BUILD_DOCS
-DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qmp-commands.txt
+DOCS=qemu-doc.html qemu-tech.html qemu.1 qemu-img.1 qemu-nbd.8 qemu-ga.8
+DOCS+=qmp-commands.txt
 ifdef CONFIG_VIRTFS
 DOCS+=fsdev/virtfs-proxy-helper.1
 endif
@@ -108,9 +117,10 @@
 
 -include $(SUBDIR_DEVICES_MAK_DEP)
 
-%/config-devices.mak: default-configs/%.mak
-	$(call quiet-command,$(SHELL) $(SRC_PATH)/scripts/make_device_config.sh $@ $<, "  GEN   $@")
-	@if test -f $@; then \
+%/config-devices.mak: default-configs/%.mak $(SRC_PATH)/scripts/make_device_config.sh
+	$(call quiet-command, \
+            $(SHELL) $(SRC_PATH)/scripts/make_device_config.sh $< $*-config-devices.mak.d $@ > $@.tmp, "  GEN   $@.tmp")
+	$(call quiet-command, if test -f $@; then \
 	  if cmp -s $@.old $@; then \
 	    mv $@.tmp $@; \
 	    cp -p $@ $@.old; \
@@ -126,7 +136,7 @@
 	 else \
 	  mv $@.tmp $@; \
 	  cp -p $@ $@.old; \
-	 fi
+	 fi, "  GEN   $@");
 
 defconfig:
 	rm -f config-all-devices.mak $(SUBDIR_DEVICES_MAK)
@@ -139,30 +149,55 @@
                 stub-obj-y \
                 util-obj-y \
                 qga-obj-y \
+                ivshmem-client-obj-y \
+                ivshmem-server-obj-y \
                 qga-vss-dll-obj-y \
                 block-obj-y \
                 block-obj-m \
+                crypto-obj-y \
+                crypto-aes-obj-y \
+                qom-obj-y \
+                io-obj-y \
                 common-obj-y \
                 common-obj-m)
 
 ifneq ($(wildcard config-host.mak),)
-include $(SRC_PATH)/tests/Makefile
-endif
-ifeq ($(CONFIG_SMARTCARD_NSS),y)
-include $(SRC_PATH)/libcacard/Makefile
+include $(SRC_PATH)/tests/Makefile.include
 endif
 
 all: $(DOCS) $(TOOLS) $(HELPERS-y) recurse-all modules
 
+qemu-version.h: FORCE
+	$(call quiet-command, \
+		(cd $(SRC_PATH); \
+		printf '#define QEMU_PKGVERSION '; \
+		if test -n "$(PKGVERSION)"; then \
+			printf '"$(PKGVERSION)"\n'; \
+		else \
+			if test -d .git; then \
+				printf '" ('; \
+				git describe --match 'v*' 2>/dev/null | tr -d '\n'; \
+				if ! git diff-index --quiet HEAD &>/dev/null; then \
+					printf -- '-dirty'; \
+				fi; \
+				printf ')"\n'; \
+			else \
+				printf '""\n'; \
+			fi; \
+		fi) > $@.tmp)
+	$(call quiet-command, cmp -s $@ $@.tmp || mv $@.tmp $@)
+
 config-host.h: config-host.h-timestamp
 config-host.h-timestamp: config-host.mak
-qemu-options.def: $(SRC_PATH)/qemu-options.hx
+qemu-options.def: $(SRC_PATH)/qemu-options.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $@")
 
 SUBDIR_RULES=$(patsubst %,subdir-%, $(TARGET_DIRS))
 SOFTMMU_SUBDIR_RULES=$(filter %-softmmu,$(SUBDIR_RULES))
 
 $(SOFTMMU_SUBDIR_RULES): $(block-obj-y)
+$(SOFTMMU_SUBDIR_RULES): $(crypto-obj-y)
+$(SOFTMMU_SUBDIR_RULES): $(io-obj-y)
 $(SOFTMMU_SUBDIR_RULES): config-all-devices.mak
 
 subdir-%:
@@ -187,19 +222,20 @@
 dtc/%:
 	mkdir -p $@
 
-$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y)
+$(SUBDIR_RULES): libqemuutil.a libqemustub.a $(common-obj-y) $(qom-obj-y) $(crypto-aes-obj-$(CONFIG_USER_ONLY))
 
 ROMSUBDIR_RULES=$(patsubst %,romsubdir-%, $(ROMS))
+# Only keep -O and -g cflags
 romsubdir-%:
-	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C pc-bios/$* V="$(V)" TARGET_DIR="$*/",)
+	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C pc-bios/$* V="$(V)" TARGET_DIR="$*/" CFLAGS="$(filter -O% -g%,$(CFLAGS))",)
 
 ALL_SUBDIRS=$(TARGET_DIRS) $(patsubst %,pc-bios/%, $(ROMS))
 
 recurse-all: $(SUBDIR_RULES) $(ROMSUBDIR_RULES)
 
-$(BUILD_DIR)/version.o: $(SRC_PATH)/version.rc $(BUILD_DIR)/config-host.h | $(BUILD_DIR)/version.lo
+$(BUILD_DIR)/version.o: $(SRC_PATH)/version.rc config-host.h | $(BUILD_DIR)/version.lo
 	$(call quiet-command,$(WINDRES) -I$(BUILD_DIR) -o $@ $<,"  RC    version.o")
-$(BUILD_DIR)/version.lo: $(SRC_PATH)/version.rc $(BUILD_DIR)/config-host.h
+$(BUILD_DIR)/version.lo: $(SRC_PATH)/version.rc config-host.h
 	$(call quiet-command,$(WINDRES) -I$(BUILD_DIR) -o $@ $<,"  RC    version.lo")
 
 Makefile: $(version-obj-y) $(version-lobj-y)
@@ -217,16 +253,16 @@
 
 qemu-img.o: qemu-img-cmds.h
 
-qemu-img$(EXESUF): qemu-img.o $(block-obj-y) libqemuutil.a libqemustub.a
-qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) libqemuutil.a libqemustub.a
-qemu-io$(EXESUF): qemu-io.o $(block-obj-y) libqemuutil.a libqemustub.a
+qemu-img$(EXESUF): qemu-img.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-nbd$(EXESUF): qemu-nbd.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
+qemu-io$(EXESUF): qemu-io.o $(block-obj-y) $(crypto-obj-y) $(io-obj-y) $(qom-obj-y) libqemuutil.a libqemustub.a
 
-qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
+qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o libqemuutil.a libqemustub.a
 
-fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o libqemuutil.a libqemustub.a
+fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/9p-marshal.o fsdev/9p-iov-marshal.o libqemuutil.a libqemustub.a
 fsdev/virtfs-proxy-helper$(EXESUF): LIBS += -lcap
 
-qemu-img-cmds.h: $(SRC_PATH)/qemu-img-cmds.hx
+qemu-img-cmds.h: $(SRC_PATH)/qemu-img-cmds.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $@")
 
 qemu-ga$(EXESUF): LIBS = $(LIBS_QGA)
@@ -239,42 +275,49 @@
 qga/qapi-generated/qga-qapi-types.c qga/qapi-generated/qga-qapi-types.h :\
 $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
-		$(gen-out-type) -o qga/qapi-generated -p "qga-" -i $<, \
+		$(gen-out-type) -o qga/qapi-generated -p "qga-" $<, \
 		"  GEN   $@")
 qga/qapi-generated/qga-qapi-visit.c qga/qapi-generated/qga-qapi-visit.h :\
 $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
-		$(gen-out-type) -o qga/qapi-generated -p "qga-" -i $<, \
+		$(gen-out-type) -o qga/qapi-generated -p "qga-" $<, \
 		"  GEN   $@")
 qga/qapi-generated/qga-qmp-commands.h qga/qapi-generated/qga-qmp-marshal.c :\
 $(SRC_PATH)/qga/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
-		$(gen-out-type) -o qga/qapi-generated -p "qga-" -i $<, \
+		$(gen-out-type) -o qga/qapi-generated -p "qga-" $<, \
 		"  GEN   $@")
 
 qapi-modules = $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/qapi/common.json \
                $(SRC_PATH)/qapi/block.json $(SRC_PATH)/qapi/block-core.json \
-               $(SRC_PATH)/qapi/event.json
+               $(SRC_PATH)/qapi/event.json $(SRC_PATH)/qapi/introspect.json \
+               $(SRC_PATH)/qapi/crypto.json $(SRC_PATH)/qapi/rocker.json \
+               $(SRC_PATH)/qapi/trace.json
 
 qapi-types.c qapi-types.h :\
 $(qapi-modules) $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
-		$(gen-out-type) -o "." -b -i $<, \
+		$(gen-out-type) -o "." -b $<, \
 		"  GEN   $@")
 qapi-visit.c qapi-visit.h :\
 $(qapi-modules) $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
-		$(gen-out-type) -o "." -b -i $<, \
+		$(gen-out-type) -o "." -b $<, \
 		"  GEN   $@")
 qapi-event.c qapi-event.h :\
 $(qapi-modules) $(SRC_PATH)/scripts/qapi-event.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-event.py \
-		$(gen-out-type) -o "." -b -i $<, \
+		$(gen-out-type) -o "." $<, \
 		"  GEN   $@")
 qmp-commands.h qmp-marshal.c :\
 $(qapi-modules) $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
-		$(gen-out-type) -o "." -m -i $<, \
+		$(gen-out-type) -o "." -m $<, \
+		"  GEN   $@")
+qmp-introspect.h qmp-introspect.c :\
+$(qapi-modules) $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-introspect.py \
+		$(gen-out-type) -o "." $<, \
 		"  GEN   $@")
 
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
@@ -283,15 +326,44 @@
 qemu-ga$(EXESUF): $(qga-obj-y) libqemuutil.a libqemustub.a
 	$(call LINK, $^)
 
+ifdef QEMU_GA_MSI_ENABLED
+QEMU_GA_MSI=qemu-ga-$(ARCH).msi
+
+msi: $(QEMU_GA_MSI)
+
+$(QEMU_GA_MSI): qemu-ga.exe $(QGA_VSS_PROVIDER)
+
+$(QEMU_GA_MSI): config-host.mak
+
+$(QEMU_GA_MSI):  $(SRC_PATH)/qga/installer/qemu-ga.wxs
+	$(call quiet-command,QEMU_GA_VERSION="$(QEMU_GA_VERSION)" QEMU_GA_MANUFACTURER="$(QEMU_GA_MANUFACTURER)" QEMU_GA_DISTRO="$(QEMU_GA_DISTRO)" BUILD_DIR="$(BUILD_DIR)" \
+	wixl -o $@ $(QEMU_GA_MSI_ARCH) $(QEMU_GA_MSI_WITH_VSS) $(QEMU_GA_MSI_MINGW_DLL_PATH) $<, "  WIXL  $@")
+else
+msi:
+	@echo "MSI build not configured or dependency resolution failed (reconfigure with --enable-guest-agent-msi option)"
+endif
+
+ifneq ($(EXESUF),)
+.PHONY: qemu-ga
+qemu-ga: qemu-ga$(EXESUF) $(QGA_VSS_PROVIDER) $(QEMU_GA_MSI)
+endif
+
+ivshmem-client$(EXESUF): $(ivshmem-client-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
+ivshmem-server$(EXESUF): $(ivshmem-server-obj-y) libqemuutil.a libqemustub.a
+	$(call LINK, $^)
+
 clean:
 # avoid old build problems by removing potentially incorrect old files
 	rm -f config.mak op-i386.h opc-i386.h gen-op-i386.h op-arm.h opc-arm.h gen-op-arm.h
 	rm -f qemu-options.def
+	rm -f *.msi
 	find . \( -name '*.l[oa]' -o -name '*.so' -o -name '*.dll' -o -name '*.mo' -o -name '*.[oda]' \) -type f -exec rm {} +
 	rm -f $(filter-out %.tlb,$(TOOLS)) $(HELPERS-y) qemu-ga TAGS cscope.* *.pod *~ */*~
 	rm -f fsdev/*.pod
 	rm -rf .libs */.libs
 	rm -f qemu-img-cmds.h
+	rm -f ui/shader/*-vert.h ui/shader/*-frag.h
 	@# May not be present in GENERATED_HEADERS
 	rm -f trace/generated-tracers-dtrace.dtrace*
 	rm -f trace/generated-tracers-dtrace.h*
@@ -303,6 +375,7 @@
 	if test -d $$d; then $(MAKE) -C $$d $@ || exit 1; fi; \
 	rm -f $$d/qemu-options.def; \
         done
+	rm -f $(SUBDIR_DEVICES_MAK) config-all-devices.mak
 
 VERSION ?= $(shell cat VERSION)
 
@@ -312,9 +385,9 @@
 	$(SRC_PATH)/scripts/make-release "$(SRC_PATH)" "$(patsubst qemu-%.tar.bz2,%,$@)"
 
 distclean: clean
-	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi
-	rm -f config-all-devices.mak config-all-disas.mak
-	rm -f po/*.mo
+	rm -f config-host.mak config-host.h* config-host.ld $(DOCS) qemu-options.texi qemu-img-cmds.texi qemu-monitor.texi qemu-monitor-info.texi
+	rm -f config-all-devices.mak config-all-disas.mak config.status
+	rm -f po/*.mo tests/qemu-iotests/common.env
 	rm -f roms/seabios/config.mak roms/vgabios/config.mak
 	rm -f qemu-doc.info qemu-doc.aux qemu-doc.cp qemu-doc.cps qemu-doc.dvi
 	rm -f qemu-doc.fn qemu-doc.fns qemu-doc.info qemu-doc.ky qemu-doc.kys
@@ -327,8 +400,8 @@
 	rm -rf $$d || exit 1 ; \
         done
 	rm -Rf .sdk
-	if test -f pixman/config.log; then make -C pixman distclean; fi
-	if test -f dtc/version_gen.h; then make $(DTC_MAKE_ARGS) clean; fi
+	if test -f pixman/config.log; then $(MAKE) -C pixman distclean; fi
+	if test -f dtc/version_gen.h; then $(MAKE) $(DTC_MAKE_ARGS) clean; fi
 
 KEYMAPS=da     en-gb  et  fr     fr-ch  is  lt  modifiers  no  pt-br  sv \
 ar      de     en-us  fi  fr-be  hr     it  lv  nl         pl  ru     th \
@@ -337,17 +410,17 @@
 
 ifdef INSTALL_BLOBS
 BLOBS=bios.bin bios-256k.bin sgabios.bin vgabios.bin vgabios-cirrus.bin \
-vgabios-stdvga.bin vgabios-vmware.bin vgabios-qxl.bin \
-acpi-dsdt.aml q35-acpi-dsdt.aml \
+vgabios-stdvga.bin vgabios-vmware.bin vgabios-qxl.bin vgabios-virtio.bin \
+acpi-dsdt.aml \
 ppc_rom.bin openbios-sparc32 openbios-sparc64 openbios-ppc QEMU,tcx.bin QEMU,cgthree.bin \
 pxe-e1000.rom pxe-eepro100.rom pxe-ne2k_pci.rom \
 pxe-pcnet.rom pxe-rtl8139.rom pxe-virtio.rom \
 efi-e1000.rom efi-eepro100.rom efi-ne2k_pci.rom \
 efi-pcnet.rom efi-rtl8139.rom efi-virtio.rom \
+efi-e1000e.rom efi-vmxnet3.rom \
 qemu-icon.bmp qemu_logo_no_text.svg \
 bamboo.dtb petalogix-s3adsp1800.dtb petalogix-ml605.dtb \
-multiboot.bin linuxboot.bin kvmvapic.bin \
-s390-zipl.rom \
+multiboot.bin linuxboot.bin linuxboot_dma.bin kvmvapic.bin \
 s390-ccw.img \
 spapr-rtas.bin slof.bin \
 palcode-clipper \
@@ -368,6 +441,9 @@
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man8"
 	$(INSTALL_DATA) qemu-nbd.8 "$(DESTDIR)$(mandir)/man8"
 endif
+ifneq (,$(findstring qemu-ga,$(TOOLS)))
+	$(INSTALL_DATA) qemu-ga.8 "$(DESTDIR)$(mandir)/man8"
+endif
 endif
 ifdef CONFIG_VIRTFS
 	$(INSTALL_DIR) "$(DESTDIR)$(mandir)/man1"
@@ -384,16 +460,11 @@
 endif
 endif
 
-install-confdir:
-	$(INSTALL_DIR) "$(DESTDIR)$(qemu_confdir)"
 
-install-sysconfig: install-datadir install-confdir
-	$(INSTALL_DATA) $(SRC_PATH)/sysconfigs/target/target-x86_64.conf "$(DESTDIR)$(qemu_confdir)"
-
-install: all $(if $(BUILD_DOCS),install-doc) install-sysconfig \
+install: all $(if $(BUILD_DOCS),install-doc) \
 install-datadir install-localstatedir
 ifneq ($(TOOLS),)
-	$(call install-prog,$(TOOLS),$(DESTDIR)$(bindir))
+	$(call install-prog,$(subst qemu-ga,qemu-ga$(EXESUF),$(TOOLS)),$(DESTDIR)$(bindir))
 endif
 ifneq ($(CONFIG_MODULES),)
 	$(INSTALL_DIR) "$(DESTDIR)$(qemu_moddir)"
@@ -418,7 +489,7 @@
 	set -e; for x in $(KEYMAPS); do \
 		$(INSTALL_DATA) $(SRC_PATH)/pc-bios/keymaps/$$x "$(DESTDIR)$(qemu_datadir)/keymaps"; \
 	done
-	$(INSTALL_DATA) $(SRC_PATH)/trace-events "$(DESTDIR)$(qemu_datadir)/trace-events"
+	$(INSTALL_DATA) $(BUILD_DIR)/trace-events-all "$(DESTDIR)$(qemu_datadir)/trace-events-all"
 	for d in $(TARGET_DIRS); do \
 	$(MAKE) $(SUBDIR_MAKEFLAGS) TARGET_DIR=$$d/ -C $$d $@ || exit 1 ; \
         done
@@ -427,15 +498,36 @@
 test speed: all
 	$(MAKE) -C tests/tcg $@
 
+.PHONY: ctags
+ctags:
+	rm -f tags
+	find "$(SRC_PATH)" -name '*.[hc]' -exec ctags --append {} +
+
 .PHONY: TAGS
 TAGS:
-	rm -f $@
+	rm -f TAGS
 	find "$(SRC_PATH)" -name '*.[hc]' -exec etags --append {} +
 
 cscope:
-	rm -f ./cscope.*
-	find "$(SRC_PATH)" -name "*.[chsS]" -print | sed 's,^\./,,' > ./cscope.files
-	cscope -b
+	rm -f "$(SRC_PATH)"/cscope.*
+	find "$(SRC_PATH)/" -name "*.[chsS]" -print | sed 's,^\./,,' > "$(SRC_PATH)/cscope.files"
+	cscope -b -i"$(SRC_PATH)/cscope.files"
+
+# opengl shader programs
+ui/shader/%-vert.h: $(SRC_PATH)/ui/shader/%.vert $(SRC_PATH)/scripts/shaderinclude.pl
+	@mkdir -p $(dir $@)
+	$(call quiet-command,\
+		perl $(SRC_PATH)/scripts/shaderinclude.pl $< > $@,\
+		"  VERT  $@")
+
+ui/shader/%-frag.h: $(SRC_PATH)/ui/shader/%.frag $(SRC_PATH)/scripts/shaderinclude.pl
+	@mkdir -p $(dir $@)
+	$(call quiet-command,\
+		perl $(SRC_PATH)/scripts/shaderinclude.pl $< > $@,\
+		"  FRAG  $@")
+
+ui/console-gl.o: $(SRC_PATH)/ui/console-gl.c \
+	ui/shader/texture-blit-vert.h ui/shader/texture-blit-frag.h
 
 # documentation
 MAKEINFO=makeinfo
@@ -454,25 +546,29 @@
 %.pdf: %.texi
 	$(call quiet-command,texi2pdf $(TEXIFLAG) -I . $<,"  GEN   $@")
 
-qemu-options.texi: $(SRC_PATH)/qemu-options.hx
+qemu-options.texi: $(SRC_PATH)/qemu-options.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
-qemu-monitor.texi: $(SRC_PATH)/hmp-commands.hx
+qemu-monitor.texi: $(SRC_PATH)/hmp-commands.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
-qmp-commands.txt: $(SRC_PATH)/qmp-commands.hx
+qemu-monitor-info.texi: $(SRC_PATH)/hmp-commands-info.hx $(SRC_PATH)/scripts/hxtool
+	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
+
+qmp-commands.txt: $(SRC_PATH)/qmp-commands.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -q < $< > $@,"  GEN   $@")
 
-qemu-img-cmds.texi: $(SRC_PATH)/qemu-img-cmds.hx
+qemu-img-cmds.texi: $(SRC_PATH)/qemu-img-cmds.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -t < $< > $@,"  GEN   $@")
 
-qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi
+qemu.1: qemu-doc.texi qemu-options.texi qemu-monitor.texi qemu-monitor-info.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu.pod && \
 	  $(POD2MAN) --section=1 --center=" " --release=" " qemu.pod > $@, \
 	  "  GEN   $@")
+qemu.1: qemu-option-trace.texi
 
-qemu-img.1: qemu-img.texi qemu-img-cmds.texi
+qemu-img.1: qemu-img.texi qemu-option-trace.texi qemu-img-cmds.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu-img.pod && \
 	  $(POD2MAN) --section=1 --center=" " --release=" " qemu-img.pod > $@, \
@@ -484,20 +580,27 @@
 	  $(POD2MAN) --section=1 --center=" " --release=" " fsdev/virtfs-proxy-helper.pod > $@, \
 	  "  GEN   $@")
 
-qemu-nbd.8: qemu-nbd.texi
+qemu-nbd.8: qemu-nbd.texi qemu-option-trace.texi
 	$(call quiet-command, \
 	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu-nbd.pod && \
 	  $(POD2MAN) --section=8 --center=" " --release=" " qemu-nbd.pod > $@, \
 	  "  GEN   $@")
 
+qemu-ga.8: qemu-ga.texi
+	$(call quiet-command, \
+	  perl -Ww -- $(SRC_PATH)/scripts/texi2pod.pl $< qemu-ga.pod && \
+	  $(POD2MAN) --section=8 --center=" " --release=" " qemu-ga.pod > $@, \
+	  "  GEN   $@")
+
 dvi: qemu-doc.dvi qemu-tech.dvi
 html: qemu-doc.html qemu-tech.html
 info: qemu-doc.info qemu-tech.info
 pdf: qemu-doc.pdf qemu-tech.pdf
 
 qemu-doc.dvi qemu-doc.html qemu-doc.info qemu-doc.pdf: \
-	qemu-img.texi qemu-nbd.texi qemu-options.texi \
-	qemu-monitor.texi qemu-img-cmds.texi
+	qemu-img.texi qemu-nbd.texi qemu-options.texi qemu-option-trace.texi \
+	qemu-monitor.texi qemu-img-cmds.texi qemu-ga.texi \
+	qemu-monitor-info.texi
 
 ifdef CONFIG_WIN32
 
@@ -522,7 +625,7 @@
 INSTDIR=/tmp/qemu-nsis
 
 $(INSTALLER): $(SRC_PATH)/qemu.nsi
-	make install prefix=${INSTDIR}
+	$(MAKE) install prefix=${INSTDIR}
 ifdef SIGNCODE
 	(cd ${INSTDIR}; \
          for i in *.exe; do \
@@ -547,6 +650,7 @@
                 $(if $(DLL_PATH),-DDLLDIR="$(DLL_PATH)") \
                 -DSRCDIR="$(SRC_PATH)" \
                 -DOUTFILE="$(INSTALLER)" \
+                -DDISPLAYVERSION="$(VERSION)" \
                 $(SRC_PATH)/qemu.nsi
 	rm -r ${INSTDIR}
 ifdef SIGNCODE
@@ -556,10 +660,12 @@
 
 # Add a dependency on the generated files, so that they are always
 # rebuilt before other object files
-ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
+ifneq ($(filter-out $(UNCHECKED_GOALS),$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 Makefile: $(GENERATED_HEADERS)
 endif
 
 # Include automatically generated dependency files
 # Dependencies in Makefile.objs files come from our recursive subdir rules
 -include $(wildcard *.d tests/*.d)
+
+include $(SRC_PATH)/tests/docker/Makefile.include
diff --git a/Makefile.objs b/Makefile.objs
index 18fd35c..297600a 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -1,37 +1,38 @@
 #######################################################################
 # Common libraries for tools and emulators
-stub-obj-y = stubs/
-util-obj-y = util/ qobject/ qapi/ qapi-types.o qapi-visit.o qapi-event.o
+stub-obj-y = stubs/ crypto/
+util-obj-y = util/ qobject/ qapi/
+util-obj-y += qmp-introspect.o qapi-types.o qapi-visit.o qapi-event.o
 
 #######################################################################
 # block-obj-y is code used by both qemu system emulation and qemu-img
 
 block-obj-y = async.o thread-pool.o
-block-obj-y += nbd.o block.o blockjob.o
+block-obj-y += nbd/
+block-obj-y += block.o blockjob.o
 block-obj-y += main-loop.o iohandler.o qemu-timer.o
 block-obj-$(CONFIG_POSIX) += aio-posix.o
 block-obj-$(CONFIG_WIN32) += aio-win32.o
 block-obj-y += block/
 block-obj-y += qemu-io-cmds.o
 
-block-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
-block-obj-y += qemu-coroutine-sleep.o
-block-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
-
 block-obj-m = block/
 
+#######################################################################
+# crypto-obj-y is code used by both qemu system emulation and qemu-img
 
-######################################################################
-# smartcard
+crypto-obj-y = crypto/
+crypto-aes-obj-y = crypto/
 
-libcacard-y += libcacard/cac.o libcacard/event.o
-libcacard-y += libcacard/vcard.o libcacard/vreader.o
-libcacard-y += libcacard/vcard_emul_nss.o
-libcacard-y += libcacard/vcard_emul_type.o
-libcacard-y += libcacard/card_7816.o
-libcacard-y += libcacard/vcardt.o
-libcacard/vcard_emul_nss.o-cflags := $(NSS_CFLAGS)
-libcacard/vcard_emul_nss.o-libs := $(NSS_LIBS)
+#######################################################################
+# qom-obj-y is code used by both qemu system emulation and qemu-img
+
+qom-obj-y = qom/
+
+#######################################################################
+# io-obj-y is code used by both qemu system emulation and qemu-img
+
+io-obj-y = io/
 
 ######################################################################
 # Target independent part of system emulation. The long term path is to
@@ -48,15 +49,9 @@
 
 common-obj-$(CONFIG_LINUX) += fsdev/
 
-common-obj-y += migration.o migration-tcp.o
-common-obj-y += vmstate.o
-common-obj-y += qemu-file.o qemu-file-unix.o qemu-file-stdio.o
-common-obj-$(CONFIG_RDMA) += migration-rdma.o
+common-obj-y += migration/
 common-obj-y += qemu-char.o #aio.o
-common-obj-y += block-migration.o
-common-obj-y += page_cache.o xbzrle.o
-
-common-obj-$(CONFIG_POSIX) += migration-exec.o migration-unix.o migration-fd.o
+common-obj-y += page_cache.o
 
 common-obj-$(CONFIG_SPICE) += spice-qemu-char.o
 
@@ -64,6 +59,8 @@
 common-obj-y += hw/
 common-obj-y += accel.o
 
+common-obj-y += replay/
+
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
 bt-host.o-cflags := $(BLUEZ_CFLAGS)
@@ -71,6 +68,7 @@
 common-obj-y += dma-helpers.o
 common-obj-y += vl.o
 vl.o-cflags := $(GPROF_CFLAGS) $(SDL_CFLAGS)
+vl.o-cflags += -DALLOW_CONFIG_ANDROID
 common-obj-y += tpm.o
 
 common-obj-$(CONFIG_SLIRP) += slirp/
@@ -79,18 +77,18 @@
 
 common-obj-$(CONFIG_SECCOMP) += qemu-seccomp.o
 
-common-obj-$(CONFIG_SMARTCARD_NSS) += $(libcacard-y)
+common-obj-$(CONFIG_FDT) += device_tree.o
 
 ######################################################################
 # qapi
 
 common-obj-y += qmp-marshal.o
+common-obj-y += qmp-introspect.o
 common-obj-y += qmp.o hmp.o
 endif
 
 #######################################################################
 # Target-independent parts used in system and user emulation
-common-obj-y += qemu-log.o
 common-obj-y += tcg-runtime.o
 common-obj-y += hw/
 common-obj-y += qom/
@@ -113,3 +111,52 @@
 # by libqemuutil.a.  These should be moved to a separate .json schema.
 qga-obj-y = qga/
 qga-vss-dll-obj-y = qga/
+
+######################################################################
+# contrib
+ivshmem-client-obj-y = contrib/ivshmem-client/
+ivshmem-server-obj-y = contrib/ivshmem-server/
+
+
+######################################################################
+trace-events-y = trace-events
+trace-events-y += util/trace-events
+trace-events-y += crypto/trace-events
+trace-events-y += io/trace-events
+trace-events-y += migration/trace-events
+trace-events-y += block/trace-events
+trace-events-y += hw/block/trace-events
+trace-events-y += hw/char/trace-events
+trace-events-y += hw/intc/trace-events
+trace-events-y += hw/net/trace-events
+trace-events-y += hw/virtio/trace-events
+trace-events-y += hw/audio/trace-events
+trace-events-y += hw/misc/trace-events
+trace-events-y += hw/usb/trace-events
+trace-events-y += hw/scsi/trace-events
+trace-events-y += hw/nvram/trace-events
+trace-events-y += hw/display/trace-events
+trace-events-y += hw/input/trace-events
+trace-events-y += hw/timer/trace-events
+trace-events-y += hw/dma/trace-events
+trace-events-y += hw/sparc/trace-events
+trace-events-y += hw/sd/trace-events
+trace-events-y += hw/isa/trace-events
+trace-events-y += hw/i386/trace-events
+trace-events-y += hw/9pfs/trace-events
+trace-events-y += hw/ppc/trace-events
+trace-events-y += hw/pci/trace-events
+trace-events-y += hw/s390x/trace-events
+trace-events-y += hw/vfio/trace-events
+trace-events-y += hw/acpi/trace-events
+trace-events-y += hw/arm/trace-events
+trace-events-y += hw/alpha/trace-events
+trace-events-y += ui/trace-events
+trace-events-y += audio/trace-events
+trace-events-y += net/trace-events
+trace-events-y += target-i386/trace-events
+trace-events-y += target-sparc/trace-events
+trace-events-y += target-s390x/trace-events
+trace-events-y += target-ppc/trace-events
+trace-events-y += qom/trace-events
+trace-events-y += linux-user/trace-events
diff --git a/Makefile.target b/Makefile.target
index 523602b..9bd4c2e 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -1,11 +1,13 @@
 # -*- Mode: makefile -*-
 
+BUILD_DIR?=$(CURDIR)/..
+
 include ../config-host.mak
 include config-target.mak
 include config-devices.mak
 include $(SRC_PATH)/rules.mak
 
-$(call set-vpath, $(SRC_PATH))
+$(call set-vpath, $(SRC_PATH):$(BUILD_DIR))
 ifdef CONFIG_LINUX
 QEMU_CFLAGS += -I../linux-headers
 endif
@@ -46,7 +48,7 @@
 TARGET_TYPE=system
 endif
 
-$(QEMU_PROG).stp-installed: $(SRC_PATH)/trace-events
+$(QEMU_PROG).stp-installed: $(BUILD_DIR)/trace-events-all
 	$(call quiet-command,$(TRACETOOL) \
 		--format=stap \
 		--backends=$(TRACE_BACKENDS) \
@@ -55,7 +57,7 @@
 		--target-type=$(TARGET_TYPE) \
 		< $< > $@,"  GEN   $(TARGET_DIR)$(QEMU_PROG).stp-installed")
 
-$(QEMU_PROG).stp: $(SRC_PATH)/trace-events
+$(QEMU_PROG).stp: $(BUILD_DIR)/trace-events-all
 	$(call quiet-command,$(TRACETOOL) \
 		--format=stap \
 		--backends=$(TRACE_BACKENDS) \
@@ -64,7 +66,7 @@
 		--target-type=$(TARGET_TYPE) \
 		< $< > $@,"  GEN   $(TARGET_DIR)$(QEMU_PROG).stp")
 
-$(QEMU_PROG)-simpletrace.stp: $(SRC_PATH)/trace-events
+$(QEMU_PROG)-simpletrace.stp: $(BUILD_DIR)/trace-events-all
 	$(call quiet-command,$(TRACETOOL) \
 		--format=simpletrace-stap \
 		--backends=$(TRACE_BACKENDS) \
@@ -83,8 +85,11 @@
 #########################################################
 # cpu emulator library
 obj-y = exec.o translate-all.o cpu-exec.o
+obj-y += translate-common.o
+obj-y += cpu-exec-common.o
 obj-y += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o
 obj-$(CONFIG_TCG_INTERPRETER) += tci.o
+obj-y += tcg/tcg-common.o
 obj-$(CONFIG_TCG_INTERPRETER) += disas/tci.o
 obj-y += fpu/softfloat.o
 obj-y += target-$(TARGET_BASE_ARCH)/
@@ -103,7 +108,9 @@
 
 ifdef CONFIG_LINUX_USER
 
-QEMU_CFLAGS+=-I$(SRC_PATH)/linux-user/$(TARGET_ABI_DIR) -I$(SRC_PATH)/linux-user
+QEMU_CFLAGS+=-I$(SRC_PATH)/linux-user/$(TARGET_ABI_DIR) \
+             -I$(SRC_PATH)/linux-user/host/$(ARCH) \
+             -I$(SRC_PATH)/linux-user
 
 obj-y += linux-user/
 obj-y += gdbstub.o thunk.o user-exec.o
@@ -129,12 +136,12 @@
 obj-y += arch_init.o cpus.o monitor.o gdbstub.o balloon.o ioport.o numa.o
 obj-y += qtest.o bootdevice.o
 obj-y += hw/
-obj-$(CONFIG_FDT) += device_tree.o
 obj-$(CONFIG_KVM) += kvm-all.o
-obj-y += memory.o savevm.o cputlb.o
+obj-y += memory.o cputlb.o
 obj-y += memory_mapping.o
 obj-y += dump.o
-LIBS+=$(libs_softmmu)
+obj-y += migration/ram.o migration/savevm.o
+LIBS := $(libs_softmmu) $(LIBS)
 
 # xen support
 obj-$(CONFIG_XEN) += xen-common.o
@@ -142,14 +149,16 @@
 obj-$(call lnot,$(CONFIG_XEN)) += xen-common-stub.o
 obj-$(call lnot,$(CONFIG_XEN_I386)) += xen-hvm-stub.o
 
-# HAX support
-ifdef CONFIG_WIN32
-obj-$(CONFIG_HAX) += target-i386/hax-all.o target-i386/hax-slot.o target-i386/hax-windows.o
+# HAX support, only when targetting i386 or x86_64
+ifeq (y,$(CONFIG_HAX))
+ifneq (,$(filter i386 x86_64,$(TARGET_NAME))
+obj-y += target-i386/hax-all.o target-i386/hax-slot.o
+obj-$(CONFIG_WIN32) += target-i386/hax-windows.o
+obj-$(CONFIG_DARWIN) += target-i386/hax-darwin.o
+else
+obj-y += hax-stub.o
 endif
-ifdef CONFIG_DARWIN
-obj-$(CONFIG_HAX) += target-i386/hax-all.o target-i386/hax-slot.o target-i386/hax-darwin.o
-endif
-obj-$(call lnot,$(CONFIG_HAX)) += hax-stub.o
+endif # CONFIG_HAX
 
 # Hardware support
 ifeq ($(TARGET_NAME), sparc64)
@@ -158,7 +167,7 @@
 obj-y += hw/$(TARGET_BASE_ARCH)/
 endif
 
-GENERATED_HEADERS += hmp-commands.h qmp-commands-old.h
+GENERATED_HEADERS += hmp-commands.h hmp-commands-info.h qmp-commands-old.h
 
 endif # CONFIG_SOFTMMU
 
@@ -177,24 +186,41 @@
 dummy := $(call unnest-vars,.., \
                block-obj-y \
                block-obj-m \
+               crypto-obj-y \
+               crypto-aes-obj-y \
+               qom-obj-y \
+               io-obj-y \
                common-obj-y \
                common-obj-m)
 target-obj-y := $(target-obj-y-save)
 all-obj-y += $(common-obj-y)
 all-obj-y += $(target-obj-y)
+all-obj-y += $(qom-obj-y)
 all-obj-$(CONFIG_SOFTMMU) += $(block-obj-y)
+all-obj-$(CONFIG_USER_ONLY) += $(crypto-aes-obj-y)
+all-obj-$(CONFIG_SOFTMMU) += $(crypto-obj-y)
+all-obj-$(CONFIG_SOFTMMU) += $(io-obj-y)
+
+$(QEMU_PROG_BUILD): config-devices.mak
 
 # build either PROG or PROGW
 $(QEMU_PROG_BUILD): $(all-obj-y) ../libqemuutil.a ../libqemustub.a
-	$(call LINK,$^)
+	$(call LINK, $(filter-out %.mak, $^))
+ifdef CONFIG_DARWIN
+	$(call quiet-command,Rez -append $(SRC_PATH)/pc-bios/qemu.rsrc -o $@,"  REZ   $(TARGET_DIR)$@")
+	$(call quiet-command,SetFile -a C $@,"  SETFILE $(TARGET_DIR)$@")
+endif
 
 gdbstub-xml.c: $(TARGET_XML_FILES) $(SRC_PATH)/scripts/feature_to_c.sh
 	$(call quiet-command,rm -f $@ && $(SHELL) $(SRC_PATH)/scripts/feature_to_c.sh $@ $(TARGET_XML_FILES),"  GEN   $(TARGET_DIR)$@")
 
-hmp-commands.h: $(SRC_PATH)/hmp-commands.hx
+hmp-commands.h: $(SRC_PATH)/hmp-commands.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
 
-qmp-commands-old.h: $(SRC_PATH)/qmp-commands.hx
+hmp-commands-info.h: $(SRC_PATH)/hmp-commands-info.hx $(SRC_PATH)/scripts/hxtool
+	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
+
+qmp-commands-old.h: $(SRC_PATH)/qmp-commands.hx $(SRC_PATH)/scripts/hxtool
 	$(call quiet-command,sh $(SRC_PATH)/scripts/hxtool -h < $< > $@,"  GEN   $(TARGET_DIR)$@")
 
 clean:
diff --git a/README b/README
index c7c990d..f38193f 100644
--- a/README
+++ b/README
@@ -1,3 +1,107 @@
-Read the documentation in qemu-doc.html or on http://wiki.qemu-project.org
+         QEMU README
+         ===========
 
-- QEMU team
+QEMU is a generic and open source machine & userspace emulator and
+virtualizer.
+
+QEMU is capable of emulating a complete machine in software without any
+need for hardware virtualization support. By using dynamic translation,
+it achieves very good performance. QEMU can also integrate with the Xen
+and KVM hypervisors to provide emulated hardware while allowing the
+hypervisor to manage the CPU. With hypervisor support, QEMU can achieve
+near native performance for CPUs. When QEMU emulates CPUs directly it is
+capable of running operating systems made for one machine (e.g. an ARMv7
+board) on a different machine (e.g. an x86_64 PC board).
+
+QEMU is also capable of providing userspace API virtualization for Linux
+and BSD kernel interfaces. This allows binaries compiled against one
+architecture ABI (e.g. the Linux PPC64 ABI) to be run on a host using a
+different architecture ABI (e.g. the Linux x86_64 ABI). This does not
+involve any hardware emulation, simply CPU and syscall emulation.
+
+QEMU aims to fit into a variety of use cases. It can be invoked directly
+by users wishing to have full control over its behaviour and settings.
+It also aims to facilitate integration into higher level management
+layers, by providing a stable command line interface and monitor API.
+It is commonly invoked indirectly via the libvirt library when using
+open source applications such as oVirt, OpenStack and virt-manager.
+
+QEMU as a whole is released under the GNU General Public License,
+version 2. For full licensing details, consult the LICENSE file.
+
+
+Building
+========
+
+QEMU is multi-platform software intended to be buildable on all modern
+Linux platforms, OS-X, Win32 (via the Mingw64 toolchain) and a variety
+of other UNIX targets. The simple steps to build QEMU are:
+
+  mkdir build
+  cd build
+  ../configure
+  make
+
+Complete details of the process for building and configuring QEMU for
+all supported host platforms can be found in the qemu-tech.html file.
+Additional information can also be found online via the QEMU website:
+
+  http://qemu-project.org/Hosts/Linux
+  http://qemu-project.org/Hosts/W32
+
+
+Submitting patches
+==================
+
+The QEMU source code is maintained under the GIT version control system.
+
+   git clone git://git.qemu-project.org/qemu.git
+
+When submitting patches, the preferred approach is to use 'git
+format-patch' and/or 'git send-email' to format & send the mail to the
+qemu-devel@nongnu.org mailing list. All patches submitted must contain
+a 'Signed-off-by' line from the author. Patches should follow the
+guidelines set out in the HACKING and CODING_STYLE files.
+
+Additional information on submitting patches can be found online via
+the QEMU website
+
+  http://qemu-project.org/Contribute/SubmitAPatch
+  http://qemu-project.org/Contribute/TrivialPatches
+
+
+Bug reporting
+=============
+
+The QEMU project uses Launchpad as its primary upstream bug tracker. Bugs
+found when running code built from QEMU git or upstream released sources
+should be reported via:
+
+  https://bugs.launchpad.net/qemu/
+
+If using QEMU via an operating system vendor pre-built binary package, it
+is preferable to report bugs to the vendor's own bug tracker first. If
+the bug is also known to affect latest upstream code, it can also be
+reported via launchpad.
+
+For additional information on bug reporting consult:
+
+  http://qemu-project.org/Contribute/ReportABug
+
+
+Contact
+=======
+
+The QEMU community can be contacted in a number of ways, with the two
+main methods being email and IRC
+
+ - qemu-devel@nongnu.org
+   http://lists.nongnu.org/mailman/listinfo/qemu-devel
+ - #qemu on irc.oftc.net
+
+Information on additional methods of contacting the community can be
+found online via the QEMU website:
+
+  http://qemu-project.org/Contribute/StartHere
+
+-- End
diff --git a/VERSION b/VERSION
index ccbccc3..24ba9a3 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.0
+2.7.0
diff --git a/accel.c b/accel.c
index 8dc588f..6745813 100644
--- a/accel.c
+++ b/accel.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/accel.h"
 #include "hw/boards.h"
 #include "qemu-common.h"
@@ -121,14 +122,13 @@
         if (!init_failed) {
             fprintf(stderr, "No accelerator found!\n");
         }
-        return 1;
+        return -1;
     }
 
     if (init_failed) {
         fprintf(stderr, "Back to %s accelerator.\n", acc->name);
     }
-
-    return !accel_initialised;
+    return 0;
 }
 
 
diff --git a/aio-posix.c b/aio-posix.c
index d3ac06e..43162a9 100644
--- a/aio-posix.c
+++ b/aio-posix.c
@@ -13,10 +13,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/block.h"
 #include "qemu/queue.h"
 #include "qemu/sockets.h"
+#ifdef CONFIG_EPOLL_CREATE1
+#include <sys/epoll.h>
+#endif
 
 struct AioHandler
 {
@@ -24,11 +28,167 @@
     IOHandler *io_read;
     IOHandler *io_write;
     int deleted;
-    int pollfds_idx;
     void *opaque;
+    bool is_external;
     QLIST_ENTRY(AioHandler) node;
 };
 
+#ifdef CONFIG_EPOLL_CREATE1
+
+/* The fd number threashold to switch to epoll */
+#define EPOLL_ENABLE_THRESHOLD 64
+
+static void aio_epoll_disable(AioContext *ctx)
+{
+    ctx->epoll_available = false;
+    if (!ctx->epoll_enabled) {
+        return;
+    }
+    ctx->epoll_enabled = false;
+    close(ctx->epollfd);
+}
+
+static inline int epoll_events_from_pfd(int pfd_events)
+{
+    return (pfd_events & G_IO_IN ? EPOLLIN : 0) |
+           (pfd_events & G_IO_OUT ? EPOLLOUT : 0) |
+           (pfd_events & G_IO_HUP ? EPOLLHUP : 0) |
+           (pfd_events & G_IO_ERR ? EPOLLERR : 0);
+}
+
+static bool aio_epoll_try_enable(AioContext *ctx)
+{
+    AioHandler *node;
+    struct epoll_event event;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        int r;
+        if (node->deleted || !node->pfd.events) {
+            continue;
+        }
+        event.events = epoll_events_from_pfd(node->pfd.events);
+        event.data.ptr = node;
+        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
+        if (r) {
+            return false;
+        }
+    }
+    ctx->epoll_enabled = true;
+    return true;
+}
+
+static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
+{
+    struct epoll_event event;
+    int r;
+
+    if (!ctx->epoll_enabled) {
+        return;
+    }
+    if (!node->pfd.events) {
+        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_DEL, node->pfd.fd, &event);
+        if (r) {
+            aio_epoll_disable(ctx);
+        }
+    } else {
+        event.data.ptr = node;
+        event.events = epoll_events_from_pfd(node->pfd.events);
+        if (is_new) {
+            r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
+            if (r) {
+                aio_epoll_disable(ctx);
+            }
+        } else {
+            r = epoll_ctl(ctx->epollfd, EPOLL_CTL_MOD, node->pfd.fd, &event);
+            if (r) {
+                aio_epoll_disable(ctx);
+            }
+        }
+    }
+}
+
+static int aio_epoll(AioContext *ctx, GPollFD *pfds,
+                     unsigned npfd, int64_t timeout)
+{
+    AioHandler *node;
+    int i, ret = 0;
+    struct epoll_event events[128];
+
+    assert(npfd == 1);
+    assert(pfds[0].fd == ctx->epollfd);
+    if (timeout > 0) {
+        ret = qemu_poll_ns(pfds, npfd, timeout);
+    }
+    if (timeout <= 0 || ret > 0) {
+        ret = epoll_wait(ctx->epollfd, events,
+                         sizeof(events) / sizeof(events[0]),
+                         timeout);
+        if (ret <= 0) {
+            goto out;
+        }
+        for (i = 0; i < ret; i++) {
+            int ev = events[i].events;
+            node = events[i].data.ptr;
+            node->pfd.revents = (ev & EPOLLIN ? G_IO_IN : 0) |
+                (ev & EPOLLOUT ? G_IO_OUT : 0) |
+                (ev & EPOLLHUP ? G_IO_HUP : 0) |
+                (ev & EPOLLERR ? G_IO_ERR : 0);
+        }
+    }
+out:
+    return ret;
+}
+
+static bool aio_epoll_enabled(AioContext *ctx)
+{
+    /* Fall back to ppoll when external clients are disabled. */
+    return !aio_external_disabled(ctx) && ctx->epoll_enabled;
+}
+
+static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
+                                 unsigned npfd, int64_t timeout)
+{
+    if (!ctx->epoll_available) {
+        return false;
+    }
+    if (aio_epoll_enabled(ctx)) {
+        return true;
+    }
+    if (npfd >= EPOLL_ENABLE_THRESHOLD) {
+        if (aio_epoll_try_enable(ctx)) {
+            return true;
+        } else {
+            aio_epoll_disable(ctx);
+        }
+    }
+    return false;
+}
+
+#else
+
+static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
+{
+}
+
+static int aio_epoll(AioContext *ctx, GPollFD *pfds,
+                     unsigned npfd, int64_t timeout)
+{
+    assert(false);
+}
+
+static bool aio_epoll_enabled(AioContext *ctx)
+{
+    return false;
+}
+
+static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
+                          unsigned npfd, int64_t timeout)
+{
+    return false;
+}
+
+#endif
+
 static AioHandler *find_aio_handler(AioContext *ctx, int fd)
 {
     AioHandler *node;
@@ -44,11 +204,14 @@
 
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque)
 {
     AioHandler *node;
+    bool is_new = false;
+    bool deleted = false;
 
     node = find_aio_handler(ctx, fd);
 
@@ -67,37 +230,43 @@
                  * releasing the walking_handlers lock.
                  */
                 QLIST_REMOVE(node, node);
-                g_free(node);
+                deleted = true;
             }
         }
     } else {
         if (node == NULL) {
             /* Alloc and insert if it's not already there */
-            node = g_malloc0(sizeof(AioHandler));
+            node = g_new0(AioHandler, 1);
             node->pfd.fd = fd;
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
 
             g_source_add_poll(&ctx->source, &node->pfd);
+            is_new = true;
         }
         /* Update handler with latest information */
         node->io_read = io_read;
         node->io_write = io_write;
         node->opaque = opaque;
-        node->pollfds_idx = -1;
+        node->is_external = is_external;
 
         node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
         node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
     }
 
+    aio_epoll_update(ctx, node, is_new);
     aio_notify(ctx);
+    if (deleted) {
+        g_free(node);
+    }
 }
 
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *notifier,
+                            bool is_external,
                             EventNotifierHandler *io_read)
 {
     aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
-                       (IOHandler *)io_read, NULL, notifier);
+                       is_external, (IOHandler *)io_read, NULL, notifier);
 }
 
 bool aio_prepare(AioContext *ctx)
@@ -113,10 +282,12 @@
         int revents;
 
         revents = node->pfd.revents & node->pfd.events;
-        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
+        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read &&
+            aio_node_check(ctx, node->is_external)) {
             return true;
         }
-        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
+        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write &&
+            aio_node_check(ctx, node->is_external)) {
             return true;
         }
     }
@@ -154,6 +325,7 @@
 
         if (!node->deleted &&
             (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
+            aio_node_check(ctx, node->is_external) &&
             node->io_read) {
             node->io_read(node->opaque);
 
@@ -164,6 +336,7 @@
         }
         if (!node->deleted &&
             (revents & (G_IO_OUT | G_IO_ERR)) &&
+            aio_node_check(ctx, node->is_external) &&
             node->io_write) {
             node->io_write(node->opaque);
             progress = true;
@@ -186,69 +359,142 @@
     return progress;
 }
 
+/* These thread-local variables are used only in a small part of aio_poll
+ * around the call to the poll() system call.  In particular they are not
+ * used while aio_poll is performing callbacks, which makes it much easier
+ * to think about reentrancy!
+ *
+ * Stack-allocated arrays would be perfect but they have size limitations;
+ * heap allocation is expensive enough that we want to reuse arrays across
+ * calls to aio_poll().  And because poll() has to be called without holding
+ * any lock, the arrays cannot be stored in AioContext.  Thread-local data
+ * has none of the disadvantages of these three options.
+ */
+static __thread GPollFD *pollfds;
+static __thread AioHandler **nodes;
+static __thread unsigned npfd, nalloc;
+static __thread Notifier pollfds_cleanup_notifier;
+
+static void pollfds_cleanup(Notifier *n, void *unused)
+{
+    g_assert(npfd == 0);
+    g_free(pollfds);
+    g_free(nodes);
+    nalloc = 0;
+}
+
+static void add_pollfd(AioHandler *node)
+{
+    if (npfd == nalloc) {
+        if (nalloc == 0) {
+            pollfds_cleanup_notifier.notify = pollfds_cleanup;
+            qemu_thread_atexit_add(&pollfds_cleanup_notifier);
+            nalloc = 8;
+        } else {
+            g_assert(nalloc <= INT_MAX);
+            nalloc *= 2;
+        }
+        pollfds = g_renew(GPollFD, pollfds, nalloc);
+        nodes = g_renew(AioHandler *, nodes, nalloc);
+    }
+    nodes[npfd] = node;
+    pollfds[npfd] = (GPollFD) {
+        .fd = node->pfd.fd,
+        .events = node->pfd.events,
+    };
+    npfd++;
+}
+
 bool aio_poll(AioContext *ctx, bool blocking)
 {
     AioHandler *node;
-    bool was_dispatching;
-    int ret;
+    int i, ret;
     bool progress;
+    int64_t timeout;
 
-    was_dispatching = ctx->dispatching;
+    aio_context_acquire(ctx);
     progress = false;
 
     /* aio_notify can avoid the expensive event_notifier_set if
      * everything (file descriptors, bottom halves, timers) will
      * be re-evaluated before the next blocking poll().  This is
      * already true when aio_poll is called with blocking == false;
-     * if blocking == true, it is only true after poll() returns.
-     *
-     * If we're in a nested event loop, ctx->dispatching might be true.
-     * In that case we can restore it just before returning, but we
-     * have to clear it now.
+     * if blocking == true, it is only true after poll() returns,
+     * so disable the optimization now.
      */
-    aio_set_dispatching(ctx, !blocking);
+    if (blocking) {
+        atomic_add(&ctx->notify_me, 2);
+    }
 
     ctx->walking_handlers++;
 
-    g_array_set_size(ctx->pollfds, 0);
+    assert(npfd == 0);
 
     /* fill pollfds */
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        node->pollfds_idx = -1;
-        if (!node->deleted && node->pfd.events) {
-            GPollFD pfd = {
-                .fd = node->pfd.fd,
-                .events = node->pfd.events,
-            };
-            node->pollfds_idx = ctx->pollfds->len;
-            g_array_append_val(ctx->pollfds, pfd);
+        if (!node->deleted && node->pfd.events
+            && !aio_epoll_enabled(ctx)
+            && aio_node_check(ctx, node->is_external)) {
+            add_pollfd(node);
         }
     }
 
-    ctx->walking_handlers--;
+    timeout = blocking ? aio_compute_timeout(ctx) : 0;
 
     /* wait until next event */
-    ret = qemu_poll_ns((GPollFD *)ctx->pollfds->data,
-                         ctx->pollfds->len,
-                         blocking ? aio_compute_timeout(ctx) : 0);
+    if (timeout) {
+        aio_context_release(ctx);
+    }
+    if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) {
+        AioHandler epoll_handler;
+
+        epoll_handler.pfd.fd = ctx->epollfd;
+        epoll_handler.pfd.events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
+        npfd = 0;
+        add_pollfd(&epoll_handler);
+        ret = aio_epoll(ctx, pollfds, npfd, timeout);
+    } else  {
+        ret = qemu_poll_ns(pollfds, npfd, timeout);
+    }
+    if (blocking) {
+        atomic_sub(&ctx->notify_me, 2);
+    }
+    if (timeout) {
+        aio_context_acquire(ctx);
+    }
+
+    aio_notify_accept(ctx);
 
     /* if we have any readable fds, dispatch event */
     if (ret > 0) {
-        QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-            if (node->pollfds_idx != -1) {
-                GPollFD *pfd = &g_array_index(ctx->pollfds, GPollFD,
-                                              node->pollfds_idx);
-                node->pfd.revents = pfd->revents;
-            }
+        for (i = 0; i < npfd; i++) {
+            nodes[i]->pfd.revents = pollfds[i].revents;
         }
     }
 
+    npfd = 0;
+    ctx->walking_handlers--;
+
     /* Run dispatch even if there were no readable fds to run timers */
-    aio_set_dispatching(ctx, true);
     if (aio_dispatch(ctx)) {
         progress = true;
     }
 
-    aio_set_dispatching(ctx, was_dispatching);
+    aio_context_release(ctx);
+
     return progress;
 }
+
+void aio_context_setup(AioContext *ctx)
+{
+#ifdef CONFIG_EPOLL_CREATE1
+    assert(!ctx->epollfd);
+    ctx->epollfd = epoll_create1(EPOLL_CLOEXEC);
+    if (ctx->epollfd == -1) {
+        fprintf(stderr, "Failed to create epoll instance: %s", strerror(errno));
+        ctx->epoll_available = false;
+    } else {
+        ctx->epoll_available = true;
+    }
+#endif
+}
diff --git a/aio-win32.c b/aio-win32.c
index d81313b..c8c249e 100644
--- a/aio-win32.c
+++ b/aio-win32.c
@@ -15,6 +15,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/block.h"
 #include "qemu/queue.h"
@@ -28,11 +29,13 @@
     GPollFD pfd;
     int deleted;
     void *opaque;
+    bool is_external;
     QLIST_ENTRY(AioHandler) node;
 };
 
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque)
@@ -67,7 +70,7 @@
 
         if (node == NULL) {
             /* Alloc and insert if it's not already there */
-            node = g_malloc0(sizeof(AioHandler));
+            node = g_new0(AioHandler, 1);
             node->pfd.fd = fd;
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
         }
@@ -86,6 +89,7 @@
         node->opaque = opaque;
         node->io_read = io_read;
         node->io_write = io_write;
+        node->is_external = is_external;
 
         event = event_notifier_get_handle(&ctx->notifier);
         WSAEventSelect(node->pfd.fd, event,
@@ -98,6 +102,7 @@
 
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *e,
+                            bool is_external,
                             EventNotifierHandler *io_notify)
 {
     AioHandler *node;
@@ -129,10 +134,11 @@
     } else {
         if (node == NULL) {
             /* Alloc and insert if it's not already there */
-            node = g_malloc0(sizeof(AioHandler));
+            node = g_new0(AioHandler, 1);
             node->e = e;
             node->pfd.fd = (uintptr_t)event_notifier_get_handle(e);
             node->pfd.events = G_IO_IN;
+            node->is_external = is_external;
             QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
 
             g_source_add_poll(&ctx->source, &node->pfd);
@@ -279,36 +285,33 @@
 {
     AioHandler *node;
     HANDLE events[MAXIMUM_WAIT_OBJECTS + 1];
-    bool was_dispatching, progress, have_select_revents, first;
+    bool progress, have_select_revents, first;
     int count;
     int timeout;
 
-    have_select_revents = aio_prepare(ctx);
-    if (have_select_revents) {
-        blocking = false;
-    }
-
-    was_dispatching = ctx->dispatching;
+    aio_context_acquire(ctx);
     progress = false;
 
     /* aio_notify can avoid the expensive event_notifier_set if
      * everything (file descriptors, bottom halves, timers) will
      * be re-evaluated before the next blocking poll().  This is
      * already true when aio_poll is called with blocking == false;
-     * if blocking == true, it is only true after poll() returns.
-     *
-     * If we're in a nested event loop, ctx->dispatching might be true.
-     * In that case we can restore it just before returning, but we
-     * have to clear it now.
+     * if blocking == true, it is only true after poll() returns,
+     * so disable the optimization now.
      */
-    aio_set_dispatching(ctx, !blocking);
+    if (blocking) {
+        atomic_add(&ctx->notify_me, 2);
+    }
+
+    have_select_revents = aio_prepare(ctx);
 
     ctx->walking_handlers++;
 
     /* fill fd sets */
     count = 0;
     QLIST_FOREACH(node, &ctx->aio_handlers, node) {
-        if (!node->deleted && node->io_notify) {
+        if (!node->deleted && node->io_notify
+            && aio_node_check(ctx, node->is_external)) {
             events[count++] = event_notifier_get_handle(node->e);
         }
     }
@@ -316,20 +319,36 @@
     ctx->walking_handlers--;
     first = true;
 
-    /* wait until next event */
-    while (count > 0) {
+    /* ctx->notifier is always registered.  */
+    assert(count > 0);
+
+    /* Multiple iterations, all of them non-blocking except the first,
+     * may be necessary to process all pending events.  After the first
+     * WaitForMultipleObjects call ctx->notify_me will be decremented.
+     */
+    do {
         HANDLE event;
         int ret;
 
-        timeout = blocking
+        timeout = blocking && !have_select_revents
             ? qemu_timeout_ns_to_ms(aio_compute_timeout(ctx)) : 0;
-        ret = WaitForMultipleObjects(count, events, FALSE, timeout);
-        aio_set_dispatching(ctx, true);
-
-        if (first && aio_bh_poll(ctx)) {
-            progress = true;
+        if (timeout) {
+            aio_context_release(ctx);
         }
-        first = false;
+        ret = WaitForMultipleObjects(count, events, FALSE, timeout);
+        if (blocking) {
+            assert(first);
+            atomic_sub(&ctx->notify_me, 2);
+        }
+        if (timeout) {
+            aio_context_acquire(ctx);
+        }
+
+        if (first) {
+            aio_notify_accept(ctx);
+            progress |= aio_bh_poll(ctx);
+            first = false;
+        }
 
         /* if we have any signaled events, dispatch event */
         event = NULL;
@@ -344,10 +363,14 @@
         blocking = false;
 
         progress |= aio_dispatch_handlers(ctx, event);
-    }
+    } while (count > 0);
 
     progress |= timerlistgroup_run_timers(&ctx->tlg);
 
-    aio_set_dispatching(ctx, was_dispatching);
+    aio_context_release(ctx);
     return progress;
 }
+
+void aio_context_setup(AioContext *ctx)
+{
+}
diff --git a/android-commands.h b/android-commands.h
deleted file mode 100644
index e612bb0..0000000
--- a/android-commands.h
+++ /dev/null
@@ -1,522 +0,0 @@
-
-/* hand-written for now; consider .hx autogen */
-
-static mon_cmd_t android_redir_cmds[] = {
-    {
-        .name = "list",
-        .args_type = "",
-        .params = "",
-        .help = "list current redirections",
-        .mhandler.cmd = android_console_redir_list,
-    },
-    {
-        .name = "add",
-        .args_type = "arg:s",
-        .params = "[tcp|udp]:hostport:guestport",
-        .help = "add new redirection",
-        .mhandler.cmd = android_console_redir_add,
-    },
-    {
-        .name = "del",
-        .args_type = "arg:s",
-        .params = "[tcp|udp]:hostport",
-        .help = "remove existing redirection",
-        .mhandler.cmd = android_console_redir_del,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_power_cmds[] = {
-    {
-        .name = "display",
-        .args_type = "",
-        .params = "",
-        .help = "display battery and charger state",
-        .mhandler.cmd = android_console_power_display,
-    },
-    {
-        .name = "ac",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "set AC charging state",
-        .mhandler.cmd = android_console_power_ac,
-    },
-    {
-        .name = "status",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "set battery status",
-        .mhandler.cmd = android_console_power_status,
-    },
-    {
-        .name = "present",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "set battery present state",
-        .mhandler.cmd = android_console_power_present,
-    },
-    {
-        .name = "health",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "set battery health state",
-        .mhandler.cmd = android_console_power_health,
-    },
-    {
-        .name = "capacity",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "set battery capacity state",
-        .mhandler.cmd = android_console_power_capacity,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_event_cmds[] = {
-    {
-        .name = "types",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "list all <type> aliases",
-        .mhandler.cmd = android_console_event_types,
-    },
-    {
-        .name = "codes",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "list all <code> aliases for a given <type>",
-        .mhandler.cmd = android_console_event_codes,
-    },
-    {
-        .name = "send",
-        .args_type  = "arg:s?",
-        .params = "",
-        .help = "send a series of events to the kernel",
-        .mhandler.cmd = android_console_event_send,
-    },
-    {
-        .name = "text",
-        .args_type  = "arg:S?",
-        .params = "",
-        .help = "simulate keystrokes from a given text",
-        .mhandler.cmd = android_console_event_text,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_avd_snapshot_cmds[] = {
-    {
-        .name = "list",
-        .args_type = "",
-        .params = "",
-        .help = "'avd snapshot list' will show a list of all state snapshots "
-                "that can be loaded",
-        .mhandler.cmd = android_console_avd_snapshot_list,
-    },
-    {
-        .name = "save",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "'avd snapshot save <name>' will save the current (run-time) "
-                "state to a snapshot with the given name",
-        .mhandler.cmd = android_console_avd_snapshot_save,
-    },
-    {
-        .name = "load",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "'avd snapshot load <name>' will load the state snapshot of "
-                "the given name",
-        .mhandler.cmd = android_console_avd_snapshot_load,
-    },
-    {
-        .name = "del",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "'avd snapshot del <name>' will delete the state snapshot with "
-                "the given name",
-        .mhandler.cmd = android_console_avd_snapshot_del,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_avd_cmds[] = {
-    {
-        .name = "stop",
-        .args_type = "",
-        .params = "",
-        .help = "stop the virtual device",
-        .mhandler.cmd = android_console_avd_stop,
-    },
-    {
-        .name = "start",
-        .args_type = "",
-        .params = "",
-        .help = "start/restart the virtual device",
-        .mhandler.cmd = android_console_avd_start,
-    },
-    {
-        .name = "status",
-        .args_type = "",
-        .params = "",
-        .help = "query virtual device status",
-        .mhandler.cmd = android_console_avd_status,
-    },
-    {
-        .name = "name",
-        .args_type = "",
-        .params = "",
-        .help = "query virtual device name",
-        .mhandler.cmd = android_console_avd_name,
-    },
-    {
-        .name = "snapshot",
-        .args_type = "item:s",
-        .params = "",
-        .help = "state snapshot commands",
-        .mhandler.cmd = android_console_avd_snapshot,
-        .sub_cmds.static_table = android_avd_snapshot_cmds,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_finger_cmds[] = {
-    {
-        .name = "touch",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "touch fingerprint sensor with <fingerid>",
-        .mhandler.cmd = android_console_finger_touch,
-    },
-    {
-        .name = "remove",
-        .args_type = "",
-        .params = "",
-        .help = "remove finger from the fingerprint sensor",
-        .mhandler.cmd = android_console_finger_remove,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_geo_cmds[] = {
-    {
-        .name = "nmea",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "send a GPS NMEA sentence\n"
-            "'geo nema <sentence>' sends an NMEA 0183 sentence to the emulated device, as\n"
-            "if it came from an emulated GPS modem. <sentence> must begin with '$GP'. Only\n"
-            "'$GPGGA' and '$GPRCM' sentences are supported at the moment.\n",
-        .mhandler.cmd = android_console_geo_nmea,
-    },
-    {
-        .name = "fix",
-        .args_type = "arg:S?",
-        .params = "",
-        .help = "send a simple GPS fix\n"
-            "'geo fix <longitude> <latitude> [<altitude> [<satellites>]]'\n"
-            " allows you to send a simple GPS fix to the emulated system.\n"
-            " The parameters are:\n\n"
-            "  <longitude>   longitude, in decimal degrees\n"
-            "  <latitude>    latitude, in decimal degrees\n"
-            "  <altitude>    optional altitude in meters\n"
-            "  <satellites>  number of satellites being tracked (1-12)\n"
-            "\n",
-        .mhandler.cmd = android_console_geo_fix,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_sms_cmds[] = {
-    {
-        .name = "send",
-        .args_type = "arg:S?",
-        .params = "",
-        .help = "send inbound SMS text message\n"
-            "'sms send <phonenumber> <message>' allows you to simulate a new inbound sms message\n",
-        .mhandler.cmd = android_console_sms_send,
-    },
-    {
-        .name = "pdu",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "send inbound SMS PDU\n"
-            "'sms pdu <hexstring>' allows you to simulate a new inbound sms PDU\n"
-             "(used internally when one emulator sends SMS messages to another instance).\n"
-             "you probably don't want to play with this at all\n"
-             "\n",
-        .mhandler.cmd = android_console_sms_pdu,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_cdma_cmds[] = {
-    {
-        .name = "ssource",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "set the current CDMA subscription source\n"
-            "'cdma ssource <ssource>' allows you to specify where to read the subscription from\n",
-        .mhandler.cmd = android_console_cdma_ssource,
-    },
-    {
-        .name = "prl_version",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "dump the current PRL version\n"
-        "'cdma prl_version <version>' allows you to dump the current PRL version\n",
-        .mhandler.cmd = android_console_cdma_prl_version,
-    },
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_gsm_cmds[] = {
-    {
-        .name = "list",
-        .args_type = "",
-        .params = "",
-        .help = "list current phone calls\n"
-            "'gsm list' lists all inbound and outbound calls and their state\n",
-        .mhandler.cmd = android_console_gsm_list,
-    },
-    {
-        .name = "call",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "create inbound phone call\n"
-            "'gsm call <phonenumber>' allows you to simulate a new inbound call\n",
-        .mhandler.cmd = android_console_gsm_call,
-    },
-    {
-        .name = "busy",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "close waiting outbound call as busy\n"
-            "'gsm busy <phonenumber>' closes an outbound call, reporting\n"
-    "the remote phone as busy. only possible if the call is 'waiting'.\n",
-        .mhandler.cmd = android_console_gsm_busy,
-    },
-    {
-        .name = "hold",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "change the state of an outbound call to 'held'\n"
-            "'gsm hold <remoteNumber>' change the state of a call to 'held'. this is only possible\n"
-    "if the call in the 'waiting' or 'active' state\n",
-        .mhandler.cmd = android_console_gsm_hold,
-    },
-    {
-        .name = "accept",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "change the state of an outbound call to 'active'\n"
-            "'gsm accept <remoteNumber>' change the state of a call to 'active'. this is only possible\n"
-    "if the call is in the 'waiting' or 'held' state\n",
-        .mhandler.cmd = android_console_gsm_accept,
-    },
-    {
-        .name = "cancel",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "disconnect an inbound or outbound phone call\n"
-            "'gsm cancel <phonenumber>' allows you to simulate the end of an inbound or outbound call\n",
-        .mhandler.cmd = android_console_gsm_cancel,
-    },
-    {
-        .name = "data",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "modify data connection state"
-            "'gsm data <state>' allows you to modify the data connection state\n",
-        .mhandler.cmd = android_console_gsm_data,
-    },
-    {
-        .name = "voice",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "modify voice connection state"
-            "'gsm voice <state>' allows you to modify the voice connection state\n",
-        .mhandler.cmd = android_console_gsm_voice,
-    },
-    {
-        .name = "status",
-        .args_type = "",
-        .params = "",
-        .help = "display GSM status"
-            "'gsm status' displays the current state of the GSM emulation\n",
-        .mhandler.cmd = android_console_gsm_status,
-    },
-    {
-        .name = "signal",
-        .args_type = "arg:S?",
-        .params = "",
-        .help = "sets the rssi and ber"
-                "signal <rssi> [<ber>]' changes the reported strength and error rate on next (15s) update.\n"
-                "rssi range is 0..31 and 99 for unknown\n"
-                "ber range is 0..7 percent and 99 for unknown\n",
-        .mhandler.cmd = android_console_gsm_signal,
-    },
-    {
-        .name = "signal-profile",
-        .args_type = "arg:S?",
-        .params = "",
-        .help = "sets the signal strength profile"
-            "signal-profile <strength>' changes the reported strength on next (15s) update.\n"
-            "strength range is 0..4\n",
-        .mhandler.cmd = android_console_gsm_signal_profile,
-    },
-    { NULL, NULL, },
-};
-
-#ifdef _WIN32
-// NOTE: HELP_COMMAND is defined by winuser.h which conflicts with this.
-#undef HELP_COMMAND
-#endif
-
-#define HELP_COMMAND                                                  \
-    {                                                                 \
-        .name = "help|h|?", .args_type = "helptext:S?", .params = "", \
-        .help = "print a list of commands",                           \
-        .mhandler.cmd = android_console_help,                         \
-    }
-
-#define QUIT_COMMAND                                                          \
-    {                                                                         \
-        .name = "quit|exit", .args_type = "", .params = "",                   \
-        .help = "quit control session", .mhandler.cmd = android_console_quit, \
-    }
-
-#define AVD_HELP "control virtual device execution"
-
-static mon_cmd_t android_cmds[] = {
-    HELP_COMMAND,
-    {
-        .name = "crash",
-        .args_type = "",
-        .params = "",
-        .help = "crash the emulator instance",
-        .mhandler.cmd = android_console_crash,
-    },
-    {
-        .name = "kill",
-        .args_type = "",
-        .params = "",
-        .help = "kill the emulator instance",
-        .mhandler.cmd = android_console_kill,
-    },
-    QUIT_COMMAND,
-    {
-        .name = "redir",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "manage port redirections",
-        .mhandler.cmd = android_console_redir,
-        .sub_cmds.static_table = android_redir_cmds,
-    },
-    {   .name = "power",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "power related commands",
-        .mhandler.cmd = android_console_power,
-        .sub_cmds.static_table = android_power_cmds,
-    },
-    {   .name = "event",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "simulate hardware events",
-        .mhandler.cmd = android_console_event,
-        .sub_cmds.static_table = android_event_cmds,
-    },
-    {   .name = "avd",
-        .args_type = "item:s?",
-        .params = "",
-        .help = AVD_HELP,
-        .mhandler.cmd = android_console_avd,
-        .sub_cmds.static_table = android_avd_cmds,
-    },
-    {   .name = "finger",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "manage emulator fingerprint",
-        .mhandler.cmd = android_console_finger,
-        .sub_cmds.static_table = android_finger_cmds,
-    },
-    {   .name = "geo",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "Geo-location commands",
-        .mhandler.cmd = android_console_geo,
-        .sub_cmds.static_table = android_geo_cmds,
-    },
-    {   .name = "sms",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "SMS related commands",
-        .mhandler.cmd = android_console_sms,
-        .sub_cmds.static_table = android_sms_cmds,
-    },
-    {   .name = "cdma",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "CDMA related commands",
-        .mhandler.cmd = android_console_cdma,
-        .sub_cmds.static_table = android_cdma_cmds,
-    },
-        {   .name = "gsm",
-        .args_type = "item:s?",
-        .params = "",
-        .help = "GSM related commands",
-        .mhandler.cmd = android_console_gsm,
-        .sub_cmds.static_table = android_gsm_cmds,
-    },
-    {
-        .name = "rotate",
-        .args_type = "",
-        .params = "",
-        .help = "rotate the screen by 90 degrees",
-        .mhandler.cmd = android_console_rotate_screen,
-    },
-
-    { NULL, NULL, },
-};
-
-static mon_cmd_t android_preauth_avd_cmds[] = {
-    {
-        .name = "name",
-        .args_type = "",
-        .params = "",
-        .help = "query virtual device name",
-        .mhandler.cmd = android_console_avd_name,
-    },
-    {
-        NULL,
-        NULL,
-    },
-};
-
-/* "preauth" commands are the set of commands that are legal before
-* authentication.  "avd name is special cased here because it is needed by
-* older versions of Android Studio */
-static mon_cmd_t android_preauth_cmds[] = {
-    HELP_COMMAND,
-    {
-        .name = "auth",
-        .args_type = "arg:s?",
-        .params = "",
-        .help = "use 'auth <auth_token>' to get extended functionality",
-        .mhandler.cmd = android_console_auth,
-    },
-    {
-        .name = "avd",
-        .args_type = "item:s?",
-        .params = "",
-        .help = AVD_HELP,
-        .mhandler.cmd = android_console_avd_preauth,
-        .sub_cmds.static_table = android_preauth_avd_cmds,
-    },
-    QUIT_COMMAND,
-    {
-        NULL,
-        NULL,
-    },
-};
diff --git a/android-qemu2-glue/android_qemud.cpp b/android-qemu2-glue/android_qemud.cpp
index 26e4313..c6e32bc 100644
--- a/android-qemu2-glue/android_qemud.cpp
+++ b/android-qemu2-glue/android_qemud.cpp
@@ -16,8 +16,10 @@
 #include "android-qemu2-glue/utils/stream.h"
 
 extern "C" {
-    #include "qemu-common.h"
-    #include "migration/vmstate.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qom/object.h"
+#include "migration/vmstate.h"
 }
 
 /* Version number of snapshots code. Increment whenever the data saved
diff --git a/android-qemu2-glue/base/async/Looper.cpp b/android-qemu2-glue/base/async/Looper.cpp
index 3aa29a6..643047f 100644
--- a/android-qemu2-glue/base/async/Looper.cpp
+++ b/android-qemu2-glue/base/async/Looper.cpp
@@ -19,6 +19,7 @@
 #include "android/utils/stream.h"
 
 extern "C" {
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "sysemu/char.h"
diff --git a/android-qemu2-glue/base/files/QemuFileStream.cpp b/android-qemu2-glue/base/files/QemuFileStream.cpp
index 32ac51e..33cb007 100644
--- a/android-qemu2-glue/base/files/QemuFileStream.cpp
+++ b/android-qemu2-glue/base/files/QemuFileStream.cpp
@@ -14,6 +14,7 @@
 #include "android/base/Log.h"
 
 extern "C" {
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "migration/qemu-file.h"
 }
diff --git a/android-qemu2-glue/build/Makefile.qemu2-glue.mk b/android-qemu2-glue/build/Makefile.qemu2-glue.mk
index 330fb92..649c1a0 100644
--- a/android-qemu2-glue/build/Makefile.qemu2-glue.mk
+++ b/android-qemu2-glue/build/Makefile.qemu2-glue.mk
@@ -1,45 +1,55 @@
 # A static library containing the android-emu glue code
 
-$(call start-emulator-library,libqemu2_glue)
+QEMU2_GLUE_INCLUDES := $(ANDROID_EMU_INCLUDES)
+
+$(call start-emulator-library,libqemu2-glue)
 
 LOCAL_CFLAGS += $(QEMU2_CFLAGS)
 
 LOCAL_C_INCLUDES += \
     $(QEMU2_INCLUDES) \
+    $(QEMU2_GLUE_INCLUDES) \
     $(LOCAL_PATH)/slirp \
-    $(LOCAL_PATH)/tcg \
 
 LOCAL_SRC_FILES := \
     android_qemud.cpp \
-    display.cpp \
-    looper-qemu.cpp \
-    net-android.cpp \
-    qemu-battery-agent-impl.c \
-    qemu-clipboard-agent-impl.cpp \
-    qemu-cellular-agent-impl.c \
-    qemu-display-agent-impl.cpp \
-    qemu-finger-agent-impl.c \
-    qemu-location-agent-impl.c \
-    qemu-net-agent-impl.c \
-    qemu-sensors-agent-impl.c \
-    qemu-setup.cpp \
-    qemu-telephony-agent-impl.c \
-    qemu-user-event-agent-impl.c \
-    qemu-vm-operations-impl.c \
-    qemu-window-agent-impl.c \
-    utils/stream.cpp \
     base/async/Looper.cpp \
     base/files/QemuFileStream.cpp \
+    display.cpp \
     emulation/android_pipe_device.cpp \
     emulation/charpipe.c \
     emulation/CharSerialLine.cpp \
     emulation/goldfish_sync.cpp \
     emulation/serial_line.cpp \
     emulation/VmLock.cpp \
+    looper-qemu.cpp \
+    net-android.cpp \
+    qemu-battery-agent-impl.c \
+    qemu-cellular-agent-impl.c \
+    qemu-clipboard-agent-impl.cpp \
+    qemu-display-agent-impl.cpp \
+    qemu-finger-agent-impl.c \
+    qemu-location-agent-impl.c \
+    qemu-net-agent-impl.c \
+    qemu-sensors-agent-impl.c \
+    qemu-setup.cpp \
+    qemu-setup-dns-servers.cpp \
+    qemu-telephony-agent-impl.c \
+    qemu-user-event-agent-impl.c \
+    qemu-vm-operations-impl.c \
+    qemu-window-agent-impl.c \
     telephony/modem_init.c \
+    utils/stream.cpp \
 
 LOCAL_SRC_FILES := $(LOCAL_SRC_FILES:%=android-qemu2-glue/%)
 
 $(call end-emulator-library)
 
-QEMU2_GLUE_STATIC_LIBRARIES := libqemu2_glue
+QEMU2_GLUE_STATIC_LIBRARIES := \
+    libqemu2-glue \
+    emulator-libui \
+    $(EMULATOR_LIBUI_STATIC_LIBRARIES)
+
+QEMU2_GLUE_LDFLAGS := $(EMULATOR_LIBUI_LDFLAGS)
+
+QEMU2_GLUE_LDLIBS := $(EMULATOR_LIBUI_LDLIBS)
diff --git a/android-qemu2-glue/build/Makefile.qemu2-sources.mk b/android-qemu2-glue/build/Makefile.qemu2-sources.mk
index 09bd237..94060c2 100644
--- a/android-qemu2-glue/build/Makefile.qemu2-sources.mk
+++ b/android-qemu2-glue/build/Makefile.qemu2-sources.mk
@@ -15,17 +15,20 @@
     backends/rng.c \
     backends/testdev.c \
     backends/tpm.c \
-    block-migration.c \
     block.c \
     block/accounting.c \
     block/backup.c \
     block/blkdebug.c \
+    block/blkreplay.c \
     block/blkverify.c \
     block/block-backend.c \
     block/bochs.c \
     block/cloop.c \
     block/commit.c \
+    block/crypto.c \
+    block/dirty-bitmap.c \
     block/dmg.c \
+    block/io.c \
     block/mirror.c \
     block/nbd-client.c \
     block/nbd.c \
@@ -44,23 +47,52 @@
     block/qed-l2-cache.c \
     block/qed-table.c \
     block/qed.c \
+    block/quorum.c \
     block/raw_bsd.c \
     block/sheepdog.c \
     block/snapshot.c \
     block/stream.c \
+    block/throttle-groups.c \
     block/vdi.c \
     block/vmdk.c \
     block/vpc.c \
     block/vvfat.c \
+    block/write-threshold.c \
     blockdev-nbd.c \
     blockdev.c \
     blockjob.c \
     bt-host.c \
     bt-vhci.c \
+    crypto/aes.c \
+    crypto/afsplit.c \
+    crypto/block-luks.c \
+    crypto/block-qcow.c \
+    crypto/block.c \
+    crypto/cipher.c \
+    crypto/desrfb.c \
+    crypto/hash-glib.c \
+    crypto/hash.c \
+    crypto/init.c \
+    crypto/ivgen-essiv.c \
+    crypto/ivgen-plain.c \
+    crypto/ivgen-plain64.c \
+    crypto/ivgen.c \
+    crypto/pbkdf.c \
+    crypto/random-platform.c \
+    crypto/secret.c \
+    crypto/tlscreds.c \
+    crypto/tlscredsanon.c \
+    crypto/tlscredsx509.c \
+    crypto/tlssession.c \
+    crypto/xts.c \
     device-hotplug.c \
+    device_tree.c \
     disas/i386.c \
     dma-helpers.c \
     hmp.c \
+    hw/acpi/acpi_interface.c \
+    hw/acpi/aml-build.c \
+    hw/acpi/bios-linker-loader.c \
     hw/audio/ac97.c \
     hw/audio/es1370.c \
     hw/audio/hda-codec.c \
@@ -81,19 +113,19 @@
     hw/char/serial-pci.c \
     hw/char/serial.c \
     hw/char/virtio-console.c \
-    hw/char/virtio-pingpong.c \
+    hw/core/bus.c \
     hw/core/fw-path-provider.c \
     hw/core/hotplug.c \
     hw/core/irq.c \
     hw/core/loader.c \
     hw/core/machine.c \
-    hw/core/nmi.c \
     hw/core/null-machine.c \
-    hw/core/platform-bus.c \
     hw/core/qdev-properties-system.c \
     hw/core/qdev-properties.c \
     hw/core/qdev.c \
+    hw/core/register.c \
     hw/core/sysbus.c \
+    hw/display/vga-pci.c \
     hw/i2c/core.c \
     hw/i2c/smbus.c \
     hw/i2c/smbus_eeprom.c \
@@ -105,26 +137,33 @@
     hw/ide/qdev.c \
     hw/input/hid.c \
     hw/input/ps2.c \
+    hw/input/virtio-input-hid.c \
+    hw/input/virtio-input.c \
     hw/ipack/ipack.c \
     hw/ipack/tpci200.c \
     hw/isa/isa-bus.c \
     hw/misc/pci-testdev.c \
     hw/net/e1000.c \
+    hw/net/e1000e.c \
+    hw/net/e1000e_core.c \
+    hw/net/e1000x_common.c \
     hw/net/eepro100.c \
     hw/net/ne2000.c \
+    hw/net/net_rx_pkt.c \
+    hw/net/net_tx_pkt.c \
     hw/net/pcnet-pci.c \
     hw/net/pcnet.c \
+    hw/net/rocker/rocker.c \
+    hw/net/rocker/rocker_desc.c \
+    hw/net/rocker/rocker_fp.c \
+    hw/net/rocker/rocker_of_dpa.c \
+    hw/net/rocker/rocker_world.c \
     hw/net/rtl8139.c \
     hw/net/vmxnet3.c \
-    hw/net/vmxnet_rx_pkt.c \
-    hw/net/vmxnet_tx_pkt.c \
     hw/nvram/eeprom93xx.c \
     hw/nvram/fw_cfg.c \
-    hw/pci-bridge/i82801b11.c \
-    hw/pci-bridge/ioh3420.c \
     hw/pci-bridge/pci_bridge_dev.c \
-    hw/pci-bridge/xio3130_downstream.c \
-    hw/pci-bridge/xio3130_upstream.c \
+    hw/pci-bridge/pci_expander_bridge.c \
     hw/pci-host/pam.c \
     hw/pci/msi.c \
     hw/pci/msix.c \
@@ -142,12 +181,17 @@
     hw/scsi/esp.c \
     hw/scsi/lsi53c895a.c \
     hw/scsi/megasas.c \
+    hw/scsi/mptconfig.c \
+    hw/scsi/mptendian.c \
+    hw/scsi/mptsas.c \
     hw/scsi/scsi-bus.c \
     hw/scsi/scsi-disk.c \
     hw/scsi/scsi-generic.c \
     hw/scsi/vmw_pvscsi.c \
+    hw/sd/core.c \
+    hw/sd/sd.c \
+    hw/sd/sdhci.c \
     hw/usb/bus.c \
-    hw/usb/ccid-card-passthru.c \
     hw/usb/combined-packet.c \
     hw/usb/core.c \
     hw/usb/desc-msos.c \
@@ -163,29 +207,52 @@
     hw/usb/dev-uas.c \
     hw/usb/dev-wacom.c \
     hw/usb/hcd-ehci-pci.c \
-    hw/usb/hcd-ehci-sysbus.c \
     hw/usb/hcd-ehci.c \
     hw/usb/hcd-ohci.c \
     hw/usb/hcd-uhci.c \
     hw/usb/hcd-xhci.c \
     hw/usb/host-stub.c \
     hw/usb/libhw.c \
-    hw/virtio/dataplane/vring.c \
     hw/virtio/virtio-bus.c \
     hw/virtio/virtio-mmio.c \
     hw/virtio/virtio-pci.c \
     hw/virtio/virtio-rng.c \
     hw/watchdog/watchdog.c \
     hw/watchdog/wdt_i6300esb.c \
+    io/channel-buffer.c \
+    io/channel-command.c \
+    io/channel-file.c \
+    io/channel-socket.c \
+    io/channel-tls.c \
+    io/channel-util.c \
+    io/channel-watch.c \
+    io/channel-websock.c \
+    io/channel.c \
+    io/task.c \
     iohandler.c \
     iothread.c \
     main-loop.c \
-    migration-tcp.c \
-    migration.c \
-    nbd.c \
+    migration/block.c \
+    migration/exec.c \
+    migration/fd.c \
+    migration/migration.c \
+    migration/postcopy-ram.c \
+    migration/qemu-file-channel.c \
+    migration/qemu-file.c \
+    migration/qjson.c \
+    migration/socket.c \
+    migration/tls.c \
+    migration/vmstate.c \
+    migration/xbzrle.c \
+    nbd/client.c \
+    nbd/common.c \
+    nbd/server.c \
     net/checksum.c \
     net/dump.c \
     net/eth.c \
+    net/filter-buffer.c \
+    net/filter-mirror.c \
+    net/filter.c \
     net/hub.c \
     net/net.c \
     net/queue.c \
@@ -195,15 +262,7 @@
     page_cache.c \
     qdev-monitor.c \
     qemu-char.c \
-    qemu-coroutine-io.c \
-    qemu-coroutine-lock.c \
-    qemu-coroutine-sleep.c \
-    qemu-coroutine.c \
-    qemu-file-stdio.c \
-    qemu-file-unix.c \
-    qemu-file.c \
     qemu-io-cmds.c \
-    qemu-log.c \
     qemu-timer.c \
     qmp.c \
     qom/container.c \
@@ -211,16 +270,27 @@
     qom/object.c \
     qom/object_interfaces.c \
     qom/qom-qobject.c \
+    replay/replay-char.c \
+    replay/replay-events.c \
+    replay/replay-input.c \
+    replay/replay-internal.c \
+    replay/replay-time.c \
+    replay/replay.c \
     slirp/arp_table.c \
     slirp/bootp.c \
     slirp/cksum.c \
+    slirp/dhcpv6.c \
     slirp/dnssearch.c \
     slirp/if.c \
+    slirp/ip6_icmp.c \
+    slirp/ip6_input.c \
+    slirp/ip6_output.c \
     slirp/ip_icmp.c \
     slirp/ip_input.c \
     slirp/ip_output.c \
     slirp/mbuf.c \
     slirp/misc.c \
+    slirp/ndp_table.c \
     slirp/sbuf.c \
     slirp/slirp.c \
     slirp/socket.c \
@@ -230,38 +300,34 @@
     slirp/tcp_timer.c \
     slirp/tftp.c \
     slirp/udp.c \
+    slirp/udp6.c \
     tcg-runtime.c \
     thread-pool.c \
     tpm.c \
     ui/console.c \
     ui/cursor.c \
-    ui/d3des.c \
     ui/input-keymap.c \
     ui/input-legacy.c \
     ui/input.c \
     ui/keymaps.c \
     ui/qemu-pixman.c \
+    ui/vnc-auth-vencrypt.c \
     ui/vnc-enc-hextile.c \
     ui/vnc-enc-tight.c \
     ui/vnc-enc-zlib.c \
     ui/vnc-enc-zrle.c \
     ui/vnc-jobs.c \
     ui/vnc-palette.c \
+    ui/vnc-ws.c \
     ui/vnc.c \
     ui/x_keymap.c \
-    vmstate.c \
-    xbzrle.c \
 
 QEMU2_COMMON_SOURCES_darwin-x86_64 := \
     aio-posix.c \
     audio/coreaudio.c \
     backends/rng-random.c \
     block/raw-posix.c \
-    coroutine-sigaltstack.c \
     hw/usb/dev-mtp.c \
-    migration-exec.c \
-    migration-fd.c \
-    migration-unix.c \
     net/tap-bsd.c \
     net/tap.c \
     net/vhost-user.c \
@@ -274,38 +340,35 @@
     backends/hostmem-file.c \
     backends/rng-random.c \
     block/raw-posix.c \
-    coroutine-ucontext.c \
     fsdev/qemu-fsdev-dummy.c \
     fsdev/qemu-fsdev-opts.c \
+    hw/input/virtio-input-host.c \
     hw/tpm/tpm_passthrough.c \
+    hw/tpm/tpm_util.c \
     hw/usb/dev-mtp.c \
-    migration-exec.c \
-    migration-fd.c \
-    migration-unix.c \
     net/tap-linux.c \
     net/tap.c \
     net/vhost-user.c \
     os-posix.c \
+    ui/input-linux.c \
 
 QEMU2_COMMON_SOURCES_windows-x86 := \
     aio-win32.c \
     audio/audio_win_int.c \
+    audio/dsoundaudio.c \
     audio/winaudio.c \
-    audio/winwaveaudio.c \
     block/raw-win32.c \
     block/win32-aio.c \
-    coroutine-win32.c \
     net/tap-win32.c \
     os-win32.c \
 
 QEMU2_COMMON_SOURCES_windows-x86_64 := \
     aio-win32.c \
     audio/audio_win_int.c \
+    audio/dsoundaudio.c \
     audio/winaudio.c \
-    audio/winwaveaudio.c \
     block/raw-win32.c \
     block/win32-aio.c \
-    coroutine-win32.c \
     net/tap-win32.c \
     os-win32.c \
 
@@ -313,10 +376,10 @@
     arch_init.c \
     balloon.c \
     bootdevice.c \
+    cpu-exec-common.c \
     cpu-exec.c \
     cpus.c \
     cputlb.c \
-    device_tree.c \
     disas.c \
     dump.c \
     exec.c \
@@ -325,7 +388,13 @@
     hw/block/dataplane/virtio-blk.c \
     hw/block/virtio-blk.c \
     hw/char/virtio-serial-bus.c \
+    hw/core/nmi.c \
+    hw/cpu/core.c \
     hw/display/vga.c \
+    hw/display/virtio-gpu-3d.c \
+    hw/display/virtio-gpu-pci.c \
+    hw/display/virtio-gpu.c \
+    hw/misc/edu.c \
     hw/net/vhost_net.c \
     hw/net/virtio-net.c \
     hw/scsi/virtio-scsi-dataplane.c \
@@ -335,23 +404,34 @@
     ioport.c \
     memory.c \
     memory_mapping.c \
+    migration/ram.c \
+    migration/savevm.c \
     monitor.c \
     numa.c \
     qtest.c \
-    savevm.c \
     tcg/optimize.c \
-    tcg/tcg.c \
+    tcg/tcg-common.c \
     tcg/tcg-op.c \
+    tcg/tcg.c \
+    trace/control-target.c \
     translate-all.c \
-    vl.c \
+    translate-common.c \
     xen-common-stub.c \
     xen-hvm-stub.c \
 
 QEMU2_TARGET_aarch64_SOURCES := \
+    disas/arm-a64.cc \
     disas/arm.c \
-    hax-stub.c \
+    disas/libvixl/vixl/a64/decoder-a64.cc \
+    disas/libvixl/vixl/a64/disasm-a64.cc \
+    disas/libvixl/vixl/a64/instructions-a64.cc \
+    disas/libvixl/vixl/compiler-intrinsics.cc \
+    disas/libvixl/vixl/utils.cc \
     hw/arm/allwinner-a10.c \
     hw/arm/armv7m.c \
+    hw/arm/ast2400.c \
+    hw/arm/bcm2835_peripherals.c \
+    hw/arm/bcm2836.c \
     hw/arm/boot.c \
     hw/arm/collie.c \
     hw/arm/cubieboard.c \
@@ -359,29 +439,42 @@
     hw/arm/digic_boards.c \
     hw/arm/exynos4210.c \
     hw/arm/exynos4_boards.c \
+    hw/arm/fsl-imx25.c \
+    hw/arm/fsl-imx31.c \
+    hw/arm/fsl-imx6.c \
     hw/arm/gumstix.c \
     hw/arm/highbank.c \
+    hw/arm/imx25_pdk.c \
     hw/arm/integratorcp.c \
     hw/arm/kzm.c \
     hw/arm/mainstone.c \
     hw/arm/musicpal.c \
+    hw/arm/netduino2.c \
     hw/arm/nseries.c \
     hw/arm/omap1.c \
     hw/arm/omap2.c \
     hw/arm/omap_sx1.c \
     hw/arm/palm.c \
+    hw/arm/palmetto-bmc.c \
     hw/arm/pxa2xx.c \
     hw/arm/pxa2xx_gpio.c \
     hw/arm/pxa2xx_pic.c \
+    hw/arm/raspi.c \
     hw/arm/realview.c \
+    hw/arm/sabrelite.c \
     hw/arm/spitz.c \
     hw/arm/stellaris.c \
+    hw/arm/stm32f205_soc.c \
     hw/arm/strongarm.c \
+    hw/arm/sysbus-fdt.c \
     hw/arm/tosa.c \
     hw/arm/versatilepb.c \
     hw/arm/vexpress.c \
+    hw/arm/virt-acpi-build.c \
     hw/arm/virt.c \
     hw/arm/xilinx_zynq.c \
+    hw/arm/xlnx-ep108.c \
+    hw/arm/xlnx-zynqmp.c \
     hw/arm/z2.c \
     hw/audio/lm4549.c \
     hw/audio/marvell_88w8618.c \
@@ -392,20 +485,26 @@
     hw/block/nand.c \
     hw/block/onenand.c \
     hw/block/pflash_cfi02.c \
+    hw/char/bcm2835_aux.c \
     hw/char/cadence_uart.c \
     hw/char/digic-uart.c \
     hw/char/exynos4210_uart.c \
     hw/char/imx_serial.c \
     hw/char/omap_uart.c \
     hw/char/pl011.c \
+    hw/char/stm32f2xx_usart.c \
+    hw/core/platform-bus.c \
     hw/core/ptimer.c \
     hw/cpu/a15mpcore.c \
     hw/cpu/a9mpcore.c \
     hw/cpu/arm11mpcore.c \
     hw/cpu/realview_mpcore.c \
     hw/display/ads7846.c \
+    hw/display/bcm2835_fb.c \
     hw/display/blizzard.c \
+    hw/display/dpcd.c \
     hw/display/exynos4210_fimd.c \
+    hw/display/framebuffer.c \
     hw/display/omap_dss.c \
     hw/display/omap_lcdc.c \
     hw/display/pl110.c \
@@ -413,17 +512,26 @@
     hw/display/ssd0303.c \
     hw/display/ssd0323.c \
     hw/display/tc6393xb.c \
+    hw/display/xlnx_dp.c \
+    hw/dma/bcm2835_dma.c \
     hw/dma/omap_dma.c \
     hw/dma/pl080.c \
     hw/dma/pl330.c \
     hw/dma/pxa2xx_dma.c \
     hw/dma/soc_dma.c \
+    hw/dma/xlnx-zynq-devcfg.c \
+    hw/dma/xlnx_dpdma.c \
+    hw/gpio/gpio_key.c \
+    hw/gpio/imx_gpio.c \
     hw/gpio/max7310.c \
     hw/gpio/omap_gpio.c \
     hw/gpio/pl061.c \
     hw/gpio/zaurus.c \
+    hw/i2c/aspeed_i2c.c \
     hw/i2c/bitbang_i2c.c \
     hw/i2c/exynos4210_i2c.c \
+    hw/i2c/i2c-ddc.c \
+    hw/i2c/imx_i2c.c \
     hw/i2c/omap_i2c.c \
     hw/i2c/versatile_i2c.c \
     hw/ide/microdrive.c \
@@ -436,7 +544,16 @@
     hw/intc/allwinner-a10-pic.c \
     hw/intc/arm_gic.c \
     hw/intc/arm_gic_common.c \
+    hw/intc/arm_gicv2m.c \
+    hw/intc/arm_gicv3.c \
+    hw/intc/arm_gicv3_common.c \
+    hw/intc/arm_gicv3_cpuif.c \
+    hw/intc/arm_gicv3_dist.c \
+    hw/intc/arm_gicv3_redist.c \
     hw/intc/armv7m_nvic.c \
+    hw/intc/aspeed_vic.c \
+    hw/intc/bcm2835_ic.c \
+    hw/intc/bcm2836_control.c \
     hw/intc/exynos4210_combiner.c \
     hw/intc/exynos4210_gic.c \
     hw/intc/imx_avic.c \
@@ -448,8 +565,16 @@
     hw/misc/arm_integrator_debug.c \
     hw/misc/arm_l2x0.c \
     hw/misc/arm_sysctl.c \
+    hw/misc/aspeed_scu.c \
+    hw/misc/auxbus.c \
+    hw/misc/bcm2835_mbox.c \
+    hw/misc/bcm2835_property.c \
     hw/misc/cbus.c \
     hw/misc/exynos4210_pmu.c \
+    hw/misc/imx25_ccm.c \
+    hw/misc/imx31_ccm.c \
+    hw/misc/imx6_ccm.c \
+    hw/misc/imx6_src.c \
     hw/misc/imx_ccm.c \
     hw/misc/max111x.c \
     hw/misc/mst_fpga.c \
@@ -458,22 +583,31 @@
     hw/misc/omap_l4.c \
     hw/misc/omap_sdrc.c \
     hw/misc/omap_tap.c \
+    hw/misc/stm32f2xx_syscfg.c \
     hw/misc/tmp105.c \
+    hw/misc/zynq-xadc.c \
     hw/misc/zynq_slcr.c \
     hw/net/allwinner_emac.c \
     hw/net/cadence_gem.c \
+    hw/net/imx_fec.c \
     hw/net/lan9118.c \
     hw/net/smc91c111.c \
     hw/net/stellaris_enet.c \
     hw/net/xgmac.c \
+    hw/pci-bridge/i82801b11.c \
+    hw/pci-bridge/ioh3420.c \
+    hw/pci-bridge/xio3130_downstream.c \
+    hw/pci-bridge/xio3130_upstream.c \
+    hw/pci-host/gpex.c \
     hw/pci-host/versatile.c \
     hw/pcmcia/pxa2xx.c \
     hw/sd/omap_mmc.c \
     hw/sd/pl181.c \
     hw/sd/pxa2xx_mmci.c \
-    hw/sd/sd.c \
-    hw/sd/sdhci.c \
     hw/sd/ssi-sd.c \
+    hw/smbios/smbios.c \
+    hw/ssi/aspeed_smc.c \
+    hw/ssi/imx_spi.c \
     hw/ssi/omap_spi.c \
     hw/ssi/pl022.c \
     hw/ssi/ssi.c \
@@ -482,6 +616,7 @@
     hw/timer/allwinner-a10-pit.c \
     hw/timer/arm_mptimer.c \
     hw/timer/arm_timer.c \
+    hw/timer/aspeed_timer.c \
     hw/timer/cadence_ttc.c \
     hw/timer/digic-timer.c \
     hw/timer/ds1338.c \
@@ -494,10 +629,14 @@
     hw/timer/omap_synctimer.c \
     hw/timer/pl031.c \
     hw/timer/pxa2xx_timer.c \
-    hw/timer/tusb6010.c \
+    hw/timer/stm32f2xx_timer.c \
     hw/timer/twl92230.c \
+    hw/usb/hcd-ehci-sysbus.c \
     hw/usb/hcd-musb.c \
+    hw/usb/tusb6010.c \
     kvm-stub.c \
+    target-arm/arch_dump.c \
+    target-arm/arm-powerctl.c \
     target-arm/arm-semi.c \
     target-arm/cpu.c \
     target-arm/cpu64.c \
@@ -509,6 +648,7 @@
     target-arm/iwmmxt_helper.c \
     target-arm/kvm-stub.c \
     target-arm/machine.c \
+    target-arm/monitor.c \
     target-arm/neon_helper.c \
     target-arm/op_helper.c \
     target-arm/psci.c \
@@ -516,10 +656,18 @@
     target-arm/translate.c \
 
 QEMU2_TARGET_arm_SOURCES := \
+    disas/arm-a64.cc \
     disas/arm.c \
-    hax-stub.c \
+    disas/libvixl/vixl/a64/decoder-a64.cc \
+    disas/libvixl/vixl/a64/disasm-a64.cc \
+    disas/libvixl/vixl/a64/instructions-a64.cc \
+    disas/libvixl/vixl/compiler-intrinsics.cc \
+    disas/libvixl/vixl/utils.cc \
     hw/arm/allwinner-a10.c \
     hw/arm/armv7m.c \
+    hw/arm/ast2400.c \
+    hw/arm/bcm2835_peripherals.c \
+    hw/arm/bcm2836.c \
     hw/arm/boot.c \
     hw/arm/collie.c \
     hw/arm/cubieboard.c \
@@ -527,27 +675,38 @@
     hw/arm/digic_boards.c \
     hw/arm/exynos4210.c \
     hw/arm/exynos4_boards.c \
+    hw/arm/fsl-imx25.c \
+    hw/arm/fsl-imx31.c \
+    hw/arm/fsl-imx6.c \
     hw/arm/gumstix.c \
     hw/arm/highbank.c \
+    hw/arm/imx25_pdk.c \
     hw/arm/integratorcp.c \
     hw/arm/kzm.c \
     hw/arm/mainstone.c \
     hw/arm/musicpal.c \
+    hw/arm/netduino2.c \
     hw/arm/nseries.c \
     hw/arm/omap1.c \
     hw/arm/omap2.c \
     hw/arm/omap_sx1.c \
     hw/arm/palm.c \
+    hw/arm/palmetto-bmc.c \
     hw/arm/pxa2xx.c \
     hw/arm/pxa2xx_gpio.c \
     hw/arm/pxa2xx_pic.c \
+    hw/arm/raspi.c \
     hw/arm/realview.c \
+    hw/arm/sabrelite.c \
     hw/arm/spitz.c \
     hw/arm/stellaris.c \
+    hw/arm/stm32f205_soc.c \
     hw/arm/strongarm.c \
+    hw/arm/sysbus-fdt.c \
     hw/arm/tosa.c \
     hw/arm/versatilepb.c \
     hw/arm/vexpress.c \
+    hw/arm/virt-acpi-build.c \
     hw/arm/virt.c \
     hw/arm/xilinx_zynq.c \
     hw/arm/z2.c \
@@ -560,20 +719,25 @@
     hw/block/nand.c \
     hw/block/onenand.c \
     hw/block/pflash_cfi02.c \
+    hw/char/bcm2835_aux.c \
     hw/char/cadence_uart.c \
     hw/char/digic-uart.c \
     hw/char/exynos4210_uart.c \
     hw/char/imx_serial.c \
     hw/char/omap_uart.c \
     hw/char/pl011.c \
+    hw/char/stm32f2xx_usart.c \
+    hw/core/platform-bus.c \
     hw/core/ptimer.c \
     hw/cpu/a15mpcore.c \
     hw/cpu/a9mpcore.c \
     hw/cpu/arm11mpcore.c \
     hw/cpu/realview_mpcore.c \
     hw/display/ads7846.c \
+    hw/display/bcm2835_fb.c \
     hw/display/blizzard.c \
     hw/display/exynos4210_fimd.c \
+    hw/display/framebuffer.c \
     hw/display/omap_dss.c \
     hw/display/omap_lcdc.c \
     hw/display/pl110.c \
@@ -581,17 +745,23 @@
     hw/display/ssd0303.c \
     hw/display/ssd0323.c \
     hw/display/tc6393xb.c \
+    hw/dma/bcm2835_dma.c \
     hw/dma/omap_dma.c \
     hw/dma/pl080.c \
     hw/dma/pl330.c \
     hw/dma/pxa2xx_dma.c \
     hw/dma/soc_dma.c \
+    hw/dma/xlnx-zynq-devcfg.c \
+    hw/gpio/gpio_key.c \
+    hw/gpio/imx_gpio.c \
     hw/gpio/max7310.c \
     hw/gpio/omap_gpio.c \
     hw/gpio/pl061.c \
     hw/gpio/zaurus.c \
+    hw/i2c/aspeed_i2c.c \
     hw/i2c/bitbang_i2c.c \
     hw/i2c/exynos4210_i2c.c \
+    hw/i2c/imx_i2c.c \
     hw/i2c/omap_i2c.c \
     hw/i2c/versatile_i2c.c \
     hw/ide/microdrive.c \
@@ -604,7 +774,16 @@
     hw/intc/allwinner-a10-pic.c \
     hw/intc/arm_gic.c \
     hw/intc/arm_gic_common.c \
+    hw/intc/arm_gicv2m.c \
+    hw/intc/arm_gicv3.c \
+    hw/intc/arm_gicv3_common.c \
+    hw/intc/arm_gicv3_cpuif.c \
+    hw/intc/arm_gicv3_dist.c \
+    hw/intc/arm_gicv3_redist.c \
     hw/intc/armv7m_nvic.c \
+    hw/intc/aspeed_vic.c \
+    hw/intc/bcm2835_ic.c \
+    hw/intc/bcm2836_control.c \
     hw/intc/exynos4210_combiner.c \
     hw/intc/exynos4210_gic.c \
     hw/intc/imx_avic.c \
@@ -616,8 +795,15 @@
     hw/misc/arm_integrator_debug.c \
     hw/misc/arm_l2x0.c \
     hw/misc/arm_sysctl.c \
+    hw/misc/aspeed_scu.c \
+    hw/misc/bcm2835_mbox.c \
+    hw/misc/bcm2835_property.c \
     hw/misc/cbus.c \
     hw/misc/exynos4210_pmu.c \
+    hw/misc/imx25_ccm.c \
+    hw/misc/imx31_ccm.c \
+    hw/misc/imx6_ccm.c \
+    hw/misc/imx6_src.c \
     hw/misc/imx_ccm.c \
     hw/misc/max111x.c \
     hw/misc/mst_fpga.c \
@@ -626,22 +812,31 @@
     hw/misc/omap_l4.c \
     hw/misc/omap_sdrc.c \
     hw/misc/omap_tap.c \
+    hw/misc/stm32f2xx_syscfg.c \
     hw/misc/tmp105.c \
+    hw/misc/zynq-xadc.c \
     hw/misc/zynq_slcr.c \
     hw/net/allwinner_emac.c \
     hw/net/cadence_gem.c \
+    hw/net/imx_fec.c \
     hw/net/lan9118.c \
     hw/net/smc91c111.c \
     hw/net/stellaris_enet.c \
     hw/net/xgmac.c \
+    hw/pci-bridge/i82801b11.c \
+    hw/pci-bridge/ioh3420.c \
+    hw/pci-bridge/xio3130_downstream.c \
+    hw/pci-bridge/xio3130_upstream.c \
+    hw/pci-host/gpex.c \
     hw/pci-host/versatile.c \
     hw/pcmcia/pxa2xx.c \
     hw/sd/omap_mmc.c \
     hw/sd/pl181.c \
     hw/sd/pxa2xx_mmci.c \
-    hw/sd/sd.c \
-    hw/sd/sdhci.c \
     hw/sd/ssi-sd.c \
+    hw/smbios/smbios.c \
+    hw/ssi/aspeed_smc.c \
+    hw/ssi/imx_spi.c \
     hw/ssi/omap_spi.c \
     hw/ssi/pl022.c \
     hw/ssi/ssi.c \
@@ -650,6 +845,7 @@
     hw/timer/allwinner-a10-pit.c \
     hw/timer/arm_mptimer.c \
     hw/timer/arm_timer.c \
+    hw/timer/aspeed_timer.c \
     hw/timer/cadence_ttc.c \
     hw/timer/digic-timer.c \
     hw/timer/ds1338.c \
@@ -662,10 +858,14 @@
     hw/timer/omap_synctimer.c \
     hw/timer/pl031.c \
     hw/timer/pxa2xx_timer.c \
-    hw/timer/tusb6010.c \
+    hw/timer/stm32f2xx_timer.c \
     hw/timer/twl92230.c \
+    hw/usb/hcd-ehci-sysbus.c \
     hw/usb/hcd-musb.c \
+    hw/usb/tusb6010.c \
     kvm-stub.c \
+    target-arm/arch_dump.c \
+    target-arm/arm-powerctl.c \
     target-arm/arm-semi.c \
     target-arm/cpu.c \
     target-arm/crypto_helper.c \
@@ -674,19 +874,24 @@
     target-arm/iwmmxt_helper.c \
     target-arm/kvm-stub.c \
     target-arm/machine.c \
+    target-arm/monitor.c \
     target-arm/neon_helper.c \
     target-arm/op_helper.c \
     target-arm/psci.c \
     target-arm/translate.c \
 
 QEMU2_TARGET_i386_SOURCES := \
-    hw/acpi/acpi_interface.c \
     hw/acpi/core.c \
+    hw/acpi/cpu.c \
     hw/acpi/cpu_hotplug.c \
     hw/acpi/ich9.c \
+    hw/acpi/ipmi.c \
     hw/acpi/memory_hotplug.c \
+    hw/acpi/memory_hotplug_acpi_table.c \
+    hw/acpi/nvdimm.c \
     hw/acpi/pcihp.c \
     hw/acpi/piix4.c \
+    hw/acpi/tco.c \
     hw/audio/adlib.c \
     hw/audio/cs4231a.c \
     hw/audio/fmopl.c \
@@ -698,24 +903,21 @@
     hw/block/fdc.c \
     hw/char/debugcon.c \
     hw/char/parallel.c \
-    hw/cpu/icc_bus.c \
     hw/display/cirrus_vga.c \
     hw/display/vga-isa.c \
-    hw/display/vga-pci.c \
+    hw/display/virtio-vga.c \
     hw/display/vmware_vga.c \
     hw/dma/i8257.c \
     hw/i2c/pm_smbus.c \
     hw/i2c/smbus_ich9.c \
-    hw/i386/acpi-build.c \
-    hw/i386/bios-linker-loader.c \
     hw/i386/intel_iommu.c \
     hw/i386/kvmvapic.c \
     hw/i386/multiboot.c \
     hw/i386/pc.c \
-    hw/i386/pc_piix.c \
     hw/i386/pc_q35.c \
     hw/i386/pc_sysfw.c \
-    hw/i386/smbios.c \
+    hw/i386/pci-assign-load-rom.c \
+    hw/i386/x86-iommu.c \
     hw/ide/isa.c \
     hw/ide/piix.c \
     hw/input/pckbd.c \
@@ -726,8 +928,14 @@
     hw/intc/i8259_common.c \
     hw/intc/ioapic.c \
     hw/intc/ioapic_common.c \
+    hw/ipmi/ipmi.c \
+    hw/ipmi/ipmi_bmc_extern.c \
+    hw/ipmi/ipmi_bmc_sim.c \
+    hw/ipmi/isa_ipmi_bt.c \
+    hw/ipmi/isa_ipmi_kcs.c \
     hw/isa/apm.c \
     hw/isa/lpc_ich9.c \
+    hw/mem/nvdimm.c \
     hw/mem/pc-dimm.c \
     hw/misc/applesmc.c \
     hw/misc/debugexit.c \
@@ -736,9 +944,14 @@
     hw/misc/sga.c \
     hw/misc/vmport.c \
     hw/net/ne2000-isa.c \
+    hw/pci-bridge/i82801b11.c \
+    hw/pci-bridge/ioh3420.c \
+    hw/pci-bridge/xio3130_downstream.c \
+    hw/pci-bridge/xio3130_upstream.c \
     hw/pci-host/piix.c \
     hw/pci-host/q35.c \
-    hw/pci/pci-hotplug-old.c \
+    hw/smbios/smbios.c \
+    hw/smbios/smbios_type_38.c \
     hw/timer/hpet.c \
     hw/timer/i8254.c \
     hw/timer/i8254_common.c \
@@ -747,6 +960,7 @@
     hw/watchdog/wdt_ib700.c \
     target-i386/arch_dump.c \
     target-i386/arch_memory_mapping.c \
+    target-i386/bpt_helper.c \
     target-i386/cc_helper.c \
     target-i386/cpu.c \
     target-i386/excp_helper.c \
@@ -757,6 +971,8 @@
     target-i386/machine.c \
     target-i386/mem_helper.c \
     target-i386/misc_helper.c \
+    target-i386/monitor.c \
+    target-i386/mpx_helper.c \
     target-i386/seg_helper.c \
     target-i386/smm_helper.c \
     target-i386/svm_helper.c \
@@ -764,12 +980,11 @@
 
 QEMU2_TARGET_mips64el_SOURCES := \
     disas/mips.c \
-    hax-stub.c \
-    hw/acpi/acpi_interface.c \
     hw/acpi/core.c \
+    hw/acpi/cpu.c \
     hw/acpi/cpu_hotplug.c \
-    hw/acpi/ich9.c \
     hw/acpi/memory_hotplug.c \
+    hw/acpi/memory_hotplug_acpi_table.c \
     hw/acpi/pcihp.c \
     hw/acpi/piix4.c \
     hw/audio/adlib.c \
@@ -788,7 +1003,6 @@
     hw/display/jazz_led.c \
     hw/display/vga-isa-mm.c \
     hw/display/vga-isa.c \
-    hw/display/vga-pci.c \
     hw/display/vmware_vga.c \
     hw/dma/i8257.c \
     hw/dma/rc4030.c \
@@ -800,10 +1014,12 @@
     hw/input/pckbd.c \
     hw/intc/i8259.c \
     hw/intc/i8259_common.c \
+    hw/intc/mips_gic.c \
     hw/isa/apm.c \
     hw/isa/piix4.c \
     hw/isa/vt82c686.c \
     hw/mips/addr.c \
+    hw/mips/cps.c \
     hw/mips/cputimer.c \
     hw/mips/gt64xxx_pci.c \
     hw/mips/mips_fulong2e.c \
@@ -812,6 +1028,9 @@
     hw/mips/mips_malta.c \
     hw/mips/mips_mipssim.c \
     hw/mips/mips_r4k.c \
+    hw/misc/mips_cmgcr.c \
+    hw/misc/mips_cpc.c \
+    hw/misc/mips_itu.c \
     hw/misc/pc-testdev.c \
     hw/net/dp8393x.c \
     hw/net/mipsnet.c \
@@ -821,6 +1040,7 @@
     hw/timer/i8254.c \
     hw/timer/i8254_common.c \
     hw/timer/mc146818rtc.c \
+    hw/timer/mips_gictimer.c \
     kvm-stub.c \
     target-mips/cpu.c \
     target-mips/dsp_helper.c \
@@ -828,18 +1048,18 @@
     target-mips/helper.c \
     target-mips/lmi_helper.c \
     target-mips/machine.c \
+    target-mips/mips-semi.c \
     target-mips/msa_helper.c \
     target-mips/op_helper.c \
     target-mips/translate.c \
 
 QEMU2_TARGET_mipsel_SOURCES := \
     disas/mips.c \
-    hax-stub.c \
-    hw/acpi/acpi_interface.c \
     hw/acpi/core.c \
+    hw/acpi/cpu.c \
     hw/acpi/cpu_hotplug.c \
-    hw/acpi/ich9.c \
     hw/acpi/memory_hotplug.c \
+    hw/acpi/memory_hotplug_acpi_table.c \
     hw/acpi/pcihp.c \
     hw/acpi/piix4.c \
     hw/audio/adlib.c \
@@ -854,14 +1074,10 @@
     hw/char/parallel.c \
     hw/core/empty_slot.c \
     hw/display/cirrus_vga.c \
-    hw/display/g364fb.c \
-    hw/display/jazz_led.c \
     hw/display/vga-isa-mm.c \
     hw/display/vga-isa.c \
-    hw/display/vga-pci.c \
     hw/display/vmware_vga.c \
     hw/dma/i8257.c \
-    hw/dma/rc4030.c \
     hw/i2c/pm_smbus.c \
     hw/i2c/smbus_ich9.c \
     hw/ide/isa.c \
@@ -869,24 +1085,27 @@
     hw/input/pckbd.c \
     hw/intc/i8259.c \
     hw/intc/i8259_common.c \
+    hw/intc/mips_gic.c \
     hw/isa/apm.c \
     hw/isa/piix4.c \
     hw/mips/addr.c \
+    hw/mips/cps.c \
     hw/mips/cputimer.c \
     hw/mips/gt64xxx_pci.c \
     hw/mips/mips_int.c \
-    hw/mips/mips_jazz.c \
     hw/mips/mips_malta.c \
     hw/mips/mips_mipssim.c \
     hw/mips/mips_r4k.c \
+    hw/misc/mips_cmgcr.c \
+    hw/misc/mips_cpc.c \
+    hw/misc/mips_itu.c \
     hw/misc/pc-testdev.c \
-    hw/net/dp8393x.c \
     hw/net/mipsnet.c \
     hw/net/ne2000-isa.c \
-    hw/nvram/ds1225y.c \
     hw/timer/i8254.c \
     hw/timer/i8254_common.c \
     hw/timer/mc146818rtc.c \
+    hw/timer/mips_gictimer.c \
     kvm-stub.c \
     target-mips/cpu.c \
     target-mips/dsp_helper.c \
@@ -894,18 +1113,23 @@
     target-mips/helper.c \
     target-mips/lmi_helper.c \
     target-mips/machine.c \
+    target-mips/mips-semi.c \
     target-mips/msa_helper.c \
     target-mips/op_helper.c \
     target-mips/translate.c \
 
 QEMU2_TARGET_x86_64_SOURCES := \
-    hw/acpi/acpi_interface.c \
     hw/acpi/core.c \
+    hw/acpi/cpu.c \
     hw/acpi/cpu_hotplug.c \
     hw/acpi/ich9.c \
+    hw/acpi/ipmi.c \
     hw/acpi/memory_hotplug.c \
+    hw/acpi/memory_hotplug_acpi_table.c \
+    hw/acpi/nvdimm.c \
     hw/acpi/pcihp.c \
     hw/acpi/piix4.c \
+    hw/acpi/tco.c \
     hw/audio/adlib.c \
     hw/audio/cs4231a.c \
     hw/audio/fmopl.c \
@@ -917,24 +1141,21 @@
     hw/block/fdc.c \
     hw/char/debugcon.c \
     hw/char/parallel.c \
-    hw/cpu/icc_bus.c \
     hw/display/cirrus_vga.c \
     hw/display/vga-isa.c \
-    hw/display/vga-pci.c \
+    hw/display/virtio-vga.c \
     hw/display/vmware_vga.c \
     hw/dma/i8257.c \
     hw/i2c/pm_smbus.c \
     hw/i2c/smbus_ich9.c \
-    hw/i386/acpi-build.c \
-    hw/i386/bios-linker-loader.c \
     hw/i386/intel_iommu.c \
     hw/i386/kvmvapic.c \
     hw/i386/multiboot.c \
     hw/i386/pc.c \
-    hw/i386/pc_piix.c \
     hw/i386/pc_q35.c \
     hw/i386/pc_sysfw.c \
-    hw/i386/smbios.c \
+    hw/i386/pci-assign-load-rom.c \
+    hw/i386/x86-iommu.c \
     hw/ide/isa.c \
     hw/ide/piix.c \
     hw/input/pckbd.c \
@@ -945,8 +1166,14 @@
     hw/intc/i8259_common.c \
     hw/intc/ioapic.c \
     hw/intc/ioapic_common.c \
+    hw/ipmi/ipmi.c \
+    hw/ipmi/ipmi_bmc_extern.c \
+    hw/ipmi/ipmi_bmc_sim.c \
+    hw/ipmi/isa_ipmi_bt.c \
+    hw/ipmi/isa_ipmi_kcs.c \
     hw/isa/apm.c \
     hw/isa/lpc_ich9.c \
+    hw/mem/nvdimm.c \
     hw/mem/pc-dimm.c \
     hw/misc/applesmc.c \
     hw/misc/debugexit.c \
@@ -955,9 +1182,14 @@
     hw/misc/sga.c \
     hw/misc/vmport.c \
     hw/net/ne2000-isa.c \
+    hw/pci-bridge/i82801b11.c \
+    hw/pci-bridge/ioh3420.c \
+    hw/pci-bridge/xio3130_downstream.c \
+    hw/pci-bridge/xio3130_upstream.c \
     hw/pci-host/piix.c \
     hw/pci-host/q35.c \
-    hw/pci/pci-hotplug-old.c \
+    hw/smbios/smbios.c \
+    hw/smbios/smbios_type_38.c \
     hw/timer/hpet.c \
     hw/timer/i8254.c \
     hw/timer/i8254_common.c \
@@ -966,6 +1198,7 @@
     hw/watchdog/wdt_ib700.c \
     target-i386/arch_dump.c \
     target-i386/arch_memory_mapping.c \
+    target-i386/bpt_helper.c \
     target-i386/cc_helper.c \
     target-i386/cpu.c \
     target-i386/excp_helper.c \
@@ -976,6 +1209,8 @@
     target-i386/machine.c \
     target-i386/mem_helper.c \
     target-i386/misc_helper.c \
+    target-i386/monitor.c \
+    target-i386/mpx_helper.c \
     target-i386/seg_helper.c \
     target-i386/smm_helper.c \
     target-i386/svm_helper.c \
@@ -984,8 +1219,15 @@
 QEMU2_TARGET_aarch64_SOURCES_darwin-x86_64 := \
 
 QEMU2_TARGET_aarch64_SOURCES_linux-x86_64 := \
-    hw/misc/vfio.c \
+    hw/misc/ivshmem.c \
     hw/scsi/vhost-scsi.c \
+    hw/vfio/amd-xgbe.c \
+    hw/vfio/calxeda-xgmac.c \
+    hw/vfio/common.c \
+    hw/vfio/pci-quirks.c \
+    hw/vfio/pci.c \
+    hw/vfio/platform.c \
+    hw/vfio/spapr.c \
     hw/virtio/vhost-backend.c \
     hw/virtio/vhost-user.c \
     hw/virtio/vhost.c \
@@ -997,8 +1239,15 @@
 QEMU2_TARGET_arm_SOURCES_darwin-x86_64 := \
 
 QEMU2_TARGET_arm_SOURCES_linux-x86_64 := \
-    hw/misc/vfio.c \
+    hw/misc/ivshmem.c \
     hw/scsi/vhost-scsi.c \
+    hw/vfio/amd-xgbe.c \
+    hw/vfio/calxeda-xgmac.c \
+    hw/vfio/common.c \
+    hw/vfio/pci-quirks.c \
+    hw/vfio/pci.c \
+    hw/vfio/platform.c \
+    hw/vfio/spapr.c \
     hw/virtio/vhost-backend.c \
     hw/virtio/vhost-user.c \
     hw/virtio/vhost.c \
@@ -1009,47 +1258,52 @@
 
 QEMU2_TARGET_i386_SOURCES_darwin-x86_64 := \
     kvm-stub.c \
-    target-i386/hax-all.c \
-    target-i386/hax-darwin.c \
-    target-i386/hax-slot.c \
     target-i386/kvm-stub.c \
 
 QEMU2_TARGET_i386_SOURCES_linux-x86_64 := \
-    hax-stub.c \
     hw/i386/kvm/apic.c \
     hw/i386/kvm/clock.c \
     hw/i386/kvm/i8254.c \
     hw/i386/kvm/i8259.c \
     hw/i386/kvm/ioapic.c \
     hw/i386/kvm/pci-assign.c \
+    hw/misc/hyperv_testdev.c \
     hw/misc/ivshmem.c \
-    hw/misc/vfio.c \
     hw/scsi/vhost-scsi.c \
+    hw/vfio/amd-xgbe.c \
+    hw/vfio/calxeda-xgmac.c \
+    hw/vfio/common.c \
+    hw/vfio/pci-quirks.c \
+    hw/vfio/pci.c \
+    hw/vfio/platform.c \
+    hw/vfio/spapr.c \
     hw/virtio/vhost-backend.c \
     hw/virtio/vhost-user.c \
     hw/virtio/vhost.c \
     kvm-all.c \
+    target-i386/hyperv.c \
     target-i386/kvm.c \
 
 QEMU2_TARGET_i386_SOURCES_windows-x86 := \
     kvm-stub.c \
-    target-i386/hax-all.c \
-    target-i386/hax-slot.c \
-    target-i386/hax-windows.c \
     target-i386/kvm-stub.c \
 
 QEMU2_TARGET_i386_SOURCES_windows-x86_64 := \
     kvm-stub.c \
-    target-i386/hax-all.c \
-    target-i386/hax-slot.c \
-    target-i386/hax-windows.c \
     target-i386/kvm-stub.c \
 
 QEMU2_TARGET_mips64el_SOURCES_darwin-x86_64 := \
 
 QEMU2_TARGET_mips64el_SOURCES_linux-x86_64 := \
-    hw/misc/vfio.c \
+    hw/misc/ivshmem.c \
     hw/scsi/vhost-scsi.c \
+    hw/vfio/amd-xgbe.c \
+    hw/vfio/calxeda-xgmac.c \
+    hw/vfio/common.c \
+    hw/vfio/pci-quirks.c \
+    hw/vfio/pci.c \
+    hw/vfio/platform.c \
+    hw/vfio/spapr.c \
     hw/virtio/vhost-backend.c \
     hw/virtio/vhost-user.c \
     hw/virtio/vhost.c \
@@ -1061,8 +1315,15 @@
 QEMU2_TARGET_mipsel_SOURCES_darwin-x86_64 := \
 
 QEMU2_TARGET_mipsel_SOURCES_linux-x86_64 := \
-    hw/misc/vfio.c \
+    hw/misc/ivshmem.c \
     hw/scsi/vhost-scsi.c \
+    hw/vfio/amd-xgbe.c \
+    hw/vfio/calxeda-xgmac.c \
+    hw/vfio/common.c \
+    hw/vfio/pci-quirks.c \
+    hw/vfio/pci.c \
+    hw/vfio/platform.c \
+    hw/vfio/spapr.c \
     hw/virtio/vhost-backend.c \
     hw/virtio/vhost-user.c \
     hw/virtio/vhost.c \
@@ -1073,39 +1334,37 @@
 
 QEMU2_TARGET_x86_64_SOURCES_darwin-x86_64 := \
     kvm-stub.c \
-    target-i386/hax-all.c \
-    target-i386/hax-darwin.c \
-    target-i386/hax-slot.c \
     target-i386/kvm-stub.c \
 
 QEMU2_TARGET_x86_64_SOURCES_linux-x86_64 := \
-    hax-stub.c \
     hw/i386/kvm/apic.c \
     hw/i386/kvm/clock.c \
     hw/i386/kvm/i8254.c \
     hw/i386/kvm/i8259.c \
     hw/i386/kvm/ioapic.c \
     hw/i386/kvm/pci-assign.c \
+    hw/misc/hyperv_testdev.c \
     hw/misc/ivshmem.c \
-    hw/misc/vfio.c \
     hw/scsi/vhost-scsi.c \
+    hw/vfio/amd-xgbe.c \
+    hw/vfio/calxeda-xgmac.c \
+    hw/vfio/common.c \
+    hw/vfio/pci-quirks.c \
+    hw/vfio/pci.c \
+    hw/vfio/platform.c \
+    hw/vfio/spapr.c \
     hw/virtio/vhost-backend.c \
     hw/virtio/vhost-user.c \
     hw/virtio/vhost.c \
     kvm-all.c \
+    target-i386/hyperv.c \
     target-i386/kvm.c \
 
 QEMU2_TARGET_x86_64_SOURCES_windows-x86 := \
     kvm-stub.c \
-    target-i386/hax-all.c \
-    target-i386/hax-slot.c \
-    target-i386/hax-windows.c \
     target-i386/kvm-stub.c \
 
 QEMU2_TARGET_x86_64_SOURCES_windows-x86_64 := \
     kvm-stub.c \
-    target-i386/hax-all.c \
-    target-i386/hax-slot.c \
-    target-i386/hax-windows.c \
     target-i386/kvm-stub.c \
 
diff --git a/android-qemu2-glue/build/Makefile.qemu2-target.mk b/android-qemu2-glue/build/Makefile.qemu2-target.mk
index 621c87c..2c45668 100644
--- a/android-qemu2-glue/build/Makefile.qemu2-target.mk
+++ b/android-qemu2-glue/build/Makefile.qemu2-target.mk
@@ -36,13 +36,14 @@
 # If $(QEMU2_TARGET_CPU) is $1, then return $2, or the empty string otherwise.
 qemu2-if-target-arch = $(if $(filter $1,$(QEMU2_TARGET_CPU)),$2)
 
-$(call start-emulator-program,qemu-system-$(QEMU2_TARGET_SYSTEM))
+# A library that contains all QEMU2 target-specific sources, excluding
+# anything implemented by the glue code.
 
-LOCAL_CFLAGS += \
+QEMU2_SYSTEM_CFLAGS := \
     $(QEMU2_CFLAGS) \
-    $(EMULATOR_LIBUI_CFLAGS)
+    -DNEED_CPU_H \
 
-LOCAL_C_INCLUDES += \
+QEMU2_SYSTEM_INCLUDES := \
     $(QEMU2_INCLUDES) \
     $(QEMU2_DEPS_TOP_DIR)/include \
     $(call qemu2-if-linux,$(LOCAL_PATH)/linux-headers) \
@@ -50,15 +51,31 @@
     $(LOCAL_PATH)/target-$(QEMU2_TARGET_TARGET) \
     $(LOCAL_PATH)/tcg \
     $(LOCAL_PATH)/tcg/i386 \
-    $(LOCAL_PATH)/target-$(QEMU2_TARGET_TARGET) \
-    $(ANDROID_EMU_INCLUDES) \
-    $(LIBCURL_INCLUDES) \
-    $(EMULATOR_LIBUI_INCLUDES) \
 
-LOCAL_CFLAGS += -DNEED_CPU_H
+QEMU2_SYSTEM_LDFLAGS := $(QEMU2_DEPS_LDFLAGS)
+
+QEMU2_SYSTEM_LDLIBS := \
+    $(QEMU2_GLIB_LDLIBS) \
+    $(QEMU2_PIXMAN_LDLIBS) \
+    $(CXX_STD_LIB) \
+    -lfdt \
+    $(call qemu2-if-windows, -lvfw32 -ldxguid) \
+    $(call qemu2-if-linux, -lpulse) \
+
+QEMU2_SYSTEM_STATIC_LIBRARIES := \
+    emulator-zlib
+
+$(call start-emulator-library,libqemu2-system-$(QEMU2_TARGET_SYSTEM))
+
+LOCAL_CFLAGS += \
+    $(QEMU2_SYSTEM_CFLAGS) \
+    -DPOISON_CONFIG_ANDROID \
+
+LOCAL_C_INCLUDES += \
+    $(QEMU2_SYSTEM_INCLUDES) \
+    $(call qemu2-if-target,arm arm64,$(LOCAL_PATH)/disas/libvixl) \
 
 LOCAL_SRC_FILES += \
-    android-qemu2-glue/main.cpp \
     $(QEMU2_TARGET_SOURCES) \
     $(QEMU2_TARGET_$(QEMU2_TARGET_CPU)_SOURCES) \
     $(QEMU2_TARGET_$(QEMU2_TARGET_CPU)_SOURCES_$(BUILD_TARGET_TAG))
@@ -66,14 +83,16 @@
 LOCAL_SRC_FILES += \
     hw/audio/goldfish_audio.c \
     hw/char/goldfish_tty.c \
-    hw/display/framebuffer.c \
     hw/display/goldfish_fb.c \
+    $(call qemu2-if-target,arm arm64,, \
+        hw/display/framebuffer.c \
+        ) \
     hw/input/goldfish_events.c \
     hw/intc/goldfish_pic.c \
-    hw/misc/goldfish_pipe.c \
-    hw/misc/goldfish_battery.c \
-    hw/misc/goldfish_sync.c \
     hw/timer/goldfish_timer.c \
+    hw/misc/goldfish_battery.c \
+    hw/misc/goldfish_pipe.c \
+    hw/misc/goldfish_sync.c \
     $(call qemu2-if-target,arm arm64,\
         hw/arm/ranchu.c) \
     $(call qemu2-if-target,mips mips64,\
@@ -88,40 +107,141 @@
 endif
 
 LOCAL_SRC_FILES += \
+    stubs/target-get-monitor-def.c \
     $(call qemu2-if-target,arm arm64 mips mips64,\
-        stubs/dump.c \
-        stubs/pci-drive-hot-add.c \
         stubs/qmp_pc_dimm_device_list.c \
         ) \
+    $(call qemu2-if-target,x86 x86_64,, \
+        stubs/pc_madt_cpu_entry.c \
+        stubs/smbios_type_38.c \
+        stubs/target-monitor-defs.c \
+        ) \
     $(call qemu2-if-target,mips mips64, \
-        stubs/arch-query-cpu-def.c)
+        stubs/dump.c \
+        stubs/arch-query-cpu-def.c \
+        ) \
+    $(call qemu2-if-linux,, \
+        stubs/vhost.c \
+        ) \
+
+# HAX support.
+HAX_COMMON_SOURCES := \
+    target-i386/hax-all.c \
+    target-i386/hax-slot.c \
+
+LOCAL_SRC_FILES += \
+    $(call qemu2-if-target,x86 x86_64, \
+        $(call qemu2-if-windows, \
+            $(HAX_COMMON_SOURCES) \
+            target-i386/hax-windows.c) \
+        $(call qemu2-if-darwin, \
+            $(HAX_COMMON_SOURCES) \
+            target-i386/hax-darwin.c) \
+	$(call qemu2-if-linux, \
+	    hax-stub.c) \
+    , \
+        hax-stub.c \
+    ) \
 
 LOCAL_PREBUILTS_OBJ_FILES += \
     $(call qemu2-if-windows,$(QEMU2_AUTO_GENERATED_DIR)/version.o)
 
+$(call end-emulator-library)
+
+# The upstream version of QEMU2, without any Android-specific hacks.
+# This uses the regular SDL2 UI backend.
+
+$(call start-emulator-program,qemu-upstream-$(QEMU2_TARGET_SYSTEM))
+
 LOCAL_WHOLE_STATIC_LIBRARIES += \
-    libqemu2_common \
-    libqemu2_glue \
-    $(call qemu2-if-target,arm arm64, libqemu2_common_aarch64) \
+    libqemu2-system-$(QEMU2_TARGET_SYSTEM) \
+    libqemu2-common \
 
 LOCAL_STATIC_LIBRARIES += \
-    emulator-libui \
-    $(EMULATOR_LIBUI_STATIC_LIBRARIES) \
+    $(QEMU2_SYSTEM_STATIC_LIBRARIES) \
+
+LOCAL_CFLAGS += \
+    $(QEMU2_SYSTEM_CFLAGS) \
+
+LOCAL_C_INCLUDES += \
+    $(QEMU2_SYSTEM_INCLUDES) \
+    $(QEMU2_SDL2_INCLUDES) \
+
+LOCAL_SRC_FILES += \
+    $(call qemu2-if-target,x86 x86_64, \
+        hw/i386/acpi-build.c \
+        hw/i386/pc_piix.c \
+        ) \
+    $(call qemu2-if-windows, \
+        stubs/win32-stubs.c \
+        ) \
+    ui/sdl2.c \
+    ui/sdl2-input.c \
+    ui/sdl2-2d.c \
+    vl.c \
+
+LOCAL_LDFLAGS += $(QEMU2_SYSTEM_LDFLAGS)
+
+LOCAL_LDLIBS += \
+    $(QEMU2_SYSTEM_LDLIBS) \
+    $(QEMU2_SDL2_LDLIBS) \
+
+LOCAL_INSTALL_DIR := qemu/$(BUILD_TARGET_TAG)
+
+$(call end-emulator-program)
+
+
+# The emulator-specific version of QEMU2, with CONFIG_ANDROID defined at
+# compile-time.
+
+$(call start-emulator-program,qemu-system-$(QEMU2_TARGET_SYSTEM))
+
+LOCAL_WHOLE_STATIC_LIBRARIES += \
+    libqemu2-glue \
+    libqemu2-system-$(QEMU2_TARGET_SYSTEM) \
+    libqemu2-common \
+
+LOCAL_STATIC_LIBRARIES += \
+    $(QEMU2_SYSTEM_STATIC_LIBRARIES) \
+    $(QEMU2_GLUE_STATIC_LIBRARIES) \
     $(ANDROID_EMU_STATIC_LIBRARIES) \
 
+LOCAL_CFLAGS += \
+    $(QEMU2_SYSTEM_CFLAGS) \
+    -DCONFIG_ANDROID \
+
+LOCAL_C_INCLUDES += \
+    $(QEMU2_SYSTEM_INCLUDES) \
+    $(QEMU2_GLUE_INCLUDES) \
+    $(ANDROID_EMU_INCLUDES) \
+
+# For now, use stubs/sdl-null.c as an empty/fake SDL UI backend.
+# TODO: Use the glue code to use the Qt-based UI instead.
+LOCAL_SRC_FILES += \
+    android-qemu2-glue/main.cpp \
+    $(call qemu2-if-target,x86 x86_64, \
+        hw/i386/acpi-build.c \
+        hw/i386/pc_piix.c \
+        ) \
+    $(call qemu2-if-windows, \
+        android-qemu2-glue/stubs/win32-stubs.c \
+        ) \
+    vl.c \
+
+LOCAL_LDFLAGS += $(QEMU2_SYSTEM_LDFLAGS) $(QEMU2_GLUE_LDFLAGS)
+LOCAL_LDLIBS += $(QEMU2_SYSTEM_LDLIBS) $(QEMU2_GLUE_LDLIBS)
+
 LOCAL_LDFLAGS += \
     $(QEMU2_DEPS_LDFLAGS) \
-    $(EMULATOR_LIBUI_LDFLAGS)
 
 LOCAL_LDLIBS += \
     $(QEMU2_GLIB_LDLIBS) \
     $(QEMU2_PIXMAN_LDLIBS) \
     $(CXX_STD_LIB) \
     -lfdt \
-    $(call qemu2-if-windows, -lvfw32) \
+    $(call qemu2-if-windows, -lvfw32 -ldxguid) \
     $(call qemu2-if-linux, -lpulse) \
     $(ANDROID_EMU_LDLIBS) \
-    $(EMULATOR_LIBUI_LDLIBS) \
 
 LOCAL_INSTALL_DIR := qemu/$(BUILD_TARGET_TAG)
 
diff --git a/android-qemu2-glue/build/Makefile.qemu2.mk b/android-qemu2-glue/build/Makefile.qemu2.mk
index 481f665..8934608 100644
--- a/android-qemu2-glue/build/Makefile.qemu2.mk
+++ b/android-qemu2-glue/build/Makefile.qemu2.mk
@@ -56,6 +56,47 @@
 QEMU2_PIXMAN_INCLUDES := $(QEMU2_DEPS_TOP_DIR)/include/pixman-1
 QEMU2_PIXMAN_LDLIBS := -lpixman-1
 
+QEMU2_SDL2_INCLUDES := $(QEMU2_DEPS_TOP_DIR)/include/SDL2
+QEMU2_SDL2_LDLIBS := \
+    $(call qemu2-if-windows, \
+        -lmingw32 \
+    ) \
+    -lSDL2 \
+    $(call qemu2-if-darwin,, \
+        -lSDL2main \
+    ) \
+    $(call qemu2-if-windows, \
+        -limm32 \
+        -ldinput8 \
+        -ldxguid \
+        -ldxerr8 \
+        -luser32 \
+        -lgdi32 \
+        -lwinmm \
+        -lole32 \
+        -loleaut32 \
+        -lshell32 \
+        -lversion \
+        -luuid \
+    , \
+        -ldl \
+    ) \
+
+ifeq (darwin,$(BUILD_TARGET_OS))
+# NOTE: Because the following contain commas, we cannot use qemu2-if-darwin!
+QEMU2_SDL2_LDLIBS += \
+    -Wl,-framework,OpenGL \
+    -Wl,-framework,ForceFeedback \
+    -lobjc \
+    -Wl,-framework,Cocoa \
+    -Wl,-framework,Carbon \
+    -Wl,-framework,IOKit \
+    -Wl,-framework,CoreAudio \
+    -Wl,-framework,AudioToolbox \
+    -Wl,-framework,AudioUnit \
+
+endif
+
 # Ensure config-host.h can be found properly.
 QEMU2_INCLUDES := $(LOCAL_PATH)/android-qemu2-glue/config/$(BUILD_TARGET_TAG)
 
@@ -64,7 +105,6 @@
     $(LOCAL_PATH) \
     $(LOCAL_PATH)/include \
     $(QEMU2_AUTO_GENERATED_DIR) \
-    $(ANDROID_EMU_INCLUDES) \
 
 QEMU2_INCLUDES += $(QEMU2_GLIB_INCLUDES) $(QEMU2_PIXMAN_INCLUDES)
 
@@ -79,23 +119,33 @@
     $(LIBCURL_CFLAGS) \
     -D_GNU_SOURCE \
     -D_FILE_OFFSET_BITS=64 \
-    -DCONFIG_ANDROID \
     $(call qemu2-if-darwin, -Wno-initializer-overrides) \
 
-include $(LOCAL_PATH)/android-qemu2-glue/build/Makefile.qemu2-glue.mk
+QEMU2_CFLAGS += \
+    -Wno-unused-function \
+    $(call qemu2-if-darwin, \
+        -Wno-unused-value \
+        -Wno-parentheses-equality \
+        -Wno-self-assign \
+        , \
+        -Wno-unused-variable \
+        -Wno-unused-but-set-variable \
+        -Wno-maybe-uninitialized \
+        ) \
+    -UNDEBUG \
+
+#include $(LOCAL_PATH)/android-qemu2-glue/build/Makefile.qemu2-glue.mk
 
 #include $(LOCAL_PATH)/android-qemu2-glue/build/Makefile.qemu2-qt.mk
 
 include $(LOCAL_PATH)/android-qemu2-glue/build/Makefile.qemu2-sources.mk
 
-# Custom fixes.
-QEMU2_COMMON_SOURCES += \
-    stubs/kvm.c
-
 # A static library containing target-independent code
-$(call start-emulator-library,libqemu2_common)
+$(call start-emulator-library,libqemu2-common)
 
-LOCAL_CFLAGS += $(QEMU2_CFLAGS)
+LOCAL_CFLAGS += \
+    $(QEMU2_CFLAGS) \
+    -DPOISON_CONFIG_ANDROID \
 
 LOCAL_C_INCLUDES += \
     $(QEMU2_INCLUDES) \
@@ -111,117 +161,112 @@
     $(QEMU2_AUTO_GENERATED_DIR)/qapi-event.c \
     $(QEMU2_AUTO_GENERATED_DIR)/qapi-types.c \
     $(QEMU2_AUTO_GENERATED_DIR)/qapi-visit.c \
+    $(QEMU2_AUTO_GENERATED_DIR)/qmp-introspect.c \
     $(QEMU2_AUTO_GENERATED_DIR)/qmp-marshal.c \
     $(QEMU2_AUTO_GENERATED_DIR)/trace/generated-events.c \
     $(QEMU2_AUTO_GENERATED_DIR)/trace/generated-tracers.c \
 
 # Stuff from libqemuutil, order follows util/Makefile.objs
 LOCAL_SRC_FILES += \
-    qapi/qapi-visit-core.c \
+    crypto/pbkdf-stub.c \
+    qapi/opts-visitor.c \
+    qapi/qapi-clone-visitor.c \
     qapi/qapi-dealloc-visitor.c \
+    qapi/qapi-util.c \
+    qapi/qapi-visit-core.c \
+    qapi/qmp-dispatch.c \
+    qapi/qmp-event.c \
     qapi/qmp-input-visitor.c \
     qapi/qmp-output-visitor.c \
     qapi/qmp-registry.c \
-    qapi/qmp-dispatch.c \
     qapi/string-input-visitor.c \
     qapi/string-output-visitor.c \
-    qapi/opts-visitor.c \
-    qapi/qmp-event.c \
-    qapi/qapi-util.c \
-    qobject/qint.c \
-    qobject/qstring.c \
-    qobject/qdict.c \
-    qobject/qlist.c \
-    qobject/qfloat.c \
-    qobject/qbool.c \
-    qobject/qjson.c \
     qobject/json-lexer.c \
-    qobject/json-streamer.c \
     qobject/json-parser.c \
-    qobject/qerror.c \
+    qobject/json-streamer.c \
+    qobject/qbool.c \
+    qobject/qdict.c \
+    qobject/qfloat.c \
+    qobject/qint.c \
+    qobject/qjson.c \
+    qobject/qlist.c \
+    qobject/qnull.c \
+    qobject/qobject.c \
+    qobject/qstring.c \
     trace/control.c \
     trace/qmp.c \
-    util/osdep.c \
-    util/cutils.c \
-    util/unicode.c \
-    util/qemu-timer-common.c \
-    $(call qemu2-if-windows, \
-        util/oslib-win32.c \
-        util/qemu-thread-win32.c \
-        util/event_notifier-win32.c \
-        ) \
-    $(call qemu2-if-posix, \
-        util/oslib-posix.c \
-        util/qemu-thread-posix.c \
-        util/event_notifier-posix.c \
-        util/qemu-openpty.c \
-        ) \
-    util/envlist.c \
-    util/path.c \
-    util/module.c \
-    $(call qemu2-if-build-target-arch,x86, util/host-utils.c) \
+    util/abort.c \
+    util/acl.c \
+    util/base64.c \
     util/bitmap.c \
     util/bitops.c \
-    util/hbitmap.c \
-    util/fifo8.c \
-    util/acl.c \
+    util/buffer.c \
+    util/crc32c.c \
+    util/cutils.c \
+    util/envlist.c \
     util/error.c \
-    util/qemu-error.c \
-    $(call qemu2-if-posix, \
-        util/compatfd.c \
-        ) \
+    util/fifo8.c \
+    util/getauxval.c \
+    util/hexdump.c \
+    util/hbitmap.c \
     util/id.c \
     util/iov.c \
-    util/aes.c \
-    util/qemu-config.c \
-    util/qemu-sockets.c \
-    util/uri.c \
+    util/log.c \
+    util/module.c \
     util/notify.c \
+    util/osdep.c \
+    util/path.c \
+    util/qdist.c \
+    util/qemu-config.c \
+    util/qemu-coroutine.c \
+    util/qemu-coroutine-io.c \
+    util/qemu-coroutine-lock.c \
+    util/qemu-coroutine-sleep.c \
+    util/qemu-error.c \
     util/qemu-option.c \
     util/qemu-progress.c \
-    util/hexdump.c \
-    util/crc32c.c \
-    util/throttle.c \
-    util/getauxval.c \
+    util/qemu-sockets.c \
+    util/qemu-timer-common.c \
+    util/qht.c \
+    util/range.c \
+    util/rcu.c \
     util/readline.c \
     util/rfifolock.c \
+    util/timed-average.c \
+    util/throttle.c \
+    util/unicode.c \
+    util/uri.c \
     $(call qemu2-if-windows, \
-        util/shared-library-win32.c \
+        util/coroutine-win32.c \
+        util/event_notifier-win32.c \
+        util/oslib-win32.c \
+        util/qemu-thread-win32.c \
+        ) \
+    $(call qemu2-if-linux, \
+        util/coroutine-ucontext.c \
+        util/memfd.c \
+        ) \
+    $(call qemu2-if-darwin, \
+        util/coroutine-sigaltstack.c \
         ) \
     $(call qemu2-if-posix, \
-        util/shared-library-posix.c \
+        util/event_notifier-posix.c \
+        util/mmap-alloc.c \
+        util/oslib-posix.c \
+        util/qemu-openpty.c \
+        util/qemu-thread-posix.c \
+        ) \
+    $(call qemu2-if-build-target-arch,x86, util/host-utils.c) \
+    $(call qemu2-if-posix, \
+        util/compatfd.c \
         ) \
 
 $(call gen-hw-config-defs)
 QEMU2_INCLUDES += $(QEMU_HW_CONFIG_DEFS_INCLUDES)
 
-LOCAL_SRC_FILES += \
-    $(QEMU2_GLUE_SOURCES)
-
 $(call end-emulator-library)
 
-# Special case, the following sources are only used by the arm64
-# target but cannot be part of libqemu2_aarch64 because they need
-# to be compiled without NEED_CPU_H
-$(call start-emulator-library,libqemu2_common_aarch64)
-
-LOCAL_CPP_EXTENSION := .cc
-
-LOCAL_C_INCLUDES += \
-    $(QEMU2_INCLUDES) \
-    $(LOCAL_PATH)/target-arm \
-    $(LOCAL_PATH)/disas/libvixl
-
-LOCAL_CFLAGS += $(QEMU2_CFLAGS)
-
-LOCAL_SRC_FILES += \
-    disas/arm-a64.cc \
-    disas/libvixl/a64/decoder-a64.cc \
-    disas/libvixl/a64/disasm-a64.cc \
-    disas/libvixl/a64/instructions-a64.cc \
-    disas/libvixl/utils.cc \
-
-$(call end-emulator-library)
+include $(LOCAL_PATH)/android-qemu2-glue/build/Makefile.qemu2-glue.mk
 
 QEMU2_TARGET := x86
 include $(LOCAL_PATH)/android-qemu2-glue/build/Makefile.qemu2-target.mk
diff --git a/android-qemu2-glue/build/configure.sh b/android-qemu2-glue/build/configure.sh
index c258d55..1dbe87f 100644
--- a/android-qemu2-glue/build/configure.sh
+++ b/android-qemu2-glue/build/configure.sh
@@ -7,6 +7,10 @@
 
 QEMU2_AUTOGENERATED_DIR=$OUT_DIR/build/qemu2-qapi-auto-generated
 
+replace_with_if_different () {
+    cmp -s "$1" "$2" || mv "$2" "$1"
+}
+
 probe_prebuilts_dir "QEMU2 Dependencies" \
     QEMU2_DEPS_PREBUILTS_DIR \
     qemu-android-deps
@@ -15,23 +19,76 @@
 
 python $QEMU2_TOP_DIR/scripts/qapi-types.py \
     --builtins \
-    -i $QEMU2_TOP_DIR/qapi-schema.json \
-    -o $QEMU2_AUTOGENERATED_DIR
+    -o $QEMU2_AUTOGENERATED_DIR \
+    $QEMU2_TOP_DIR/qapi-schema.json
 
 python $QEMU2_TOP_DIR/scripts/qapi-visit.py \
     --builtins \
-    -i $QEMU2_TOP_DIR/qapi-schema.json \
-    -o $QEMU2_AUTOGENERATED_DIR
+    -o $QEMU2_AUTOGENERATED_DIR \
+    $QEMU2_TOP_DIR/qapi-schema.json
 
 python $QEMU2_TOP_DIR/scripts/qapi-event.py \
-    --builtins \
-    -i $QEMU2_TOP_DIR/qapi-schema.json \
-    -o $QEMU2_AUTOGENERATED_DIR
+    -o $QEMU2_AUTOGENERATED_DIR \
+    $QEMU2_TOP_DIR/qapi-schema.json
+
+python $QEMU2_TOP_DIR/scripts/qapi-introspect.py \
+    -o $QEMU2_AUTOGENERATED_DIR \
+    $QEMU2_TOP_DIR/qapi-schema.json
 
 python $QEMU2_TOP_DIR/scripts/qapi-commands.py \
     --middle \
-    -i $QEMU2_TOP_DIR/qapi-schema.json \
-    -o $QEMU2_AUTOGENERATED_DIR
+    -o $QEMU2_AUTOGENERATED_DIR \
+    $QEMU2_TOP_DIR/qapi-schema.json
+
+TRACE_EVENT_FILES="\
+    trace-events \
+    util/trace-events \
+    crypto/trace-events \
+    io/trace-events \
+    migration/trace-events \
+    block/trace-events \
+    hw/block/trace-events \
+    hw/char/trace-events \
+    hw/intc/trace-events \
+    hw/net/trace-events \
+    hw/virtio/trace-events \
+    hw/audio/trace-events \
+    hw/misc/trace-events \
+    hw/usb/trace-events \
+    hw/scsi/trace-events \
+    hw/nvram/trace-events \
+    hw/display/trace-events \
+    hw/input/trace-events \
+    hw/timer/trace-events \
+    hw/dma/trace-events \
+    hw/sparc/trace-events \
+    hw/sd/trace-events \
+    hw/isa/trace-events \
+    hw/i386/trace-events \
+    hw/9pfs/trace-events \
+    hw/ppc/trace-events \
+    hw/pci/trace-events \
+    hw/s390x/trace-events \
+    hw/vfio/trace-events \
+    hw/acpi/trace-events \
+    hw/arm/trace-events \
+    hw/alpha/trace-events \
+    ui/trace-events \
+    audio/trace-events \
+    net/trace-events \
+    target-i386/trace-events \
+    target-sparc/trace-events \
+    target-s390x/trace-events \
+    target-ppc/trace-events \
+    qom/trace-events \
+    linux-user/trace-events \
+"
+
+TRACE_EVENTS_ALL=$QEMU2_AUTOGENERATED_DIR/trace-events-all
+rm -f "$TRACE_EVENTS_ALL" && touch "$TRACE_EVENTS_ALL"
+for TRACE_EVENT_FILE in $TRACE_EVENT_FILES; do
+    (cat "$QEMU2_TOP_DIR/$TRACE_EVENT_FILE" >> $TRACE_EVENTS_ALL)
+done
 
 mkdir -p "$QEMU2_AUTOGENERATED_DIR"/trace
 
@@ -39,60 +96,69 @@
     --backends=nop \
     --format=h \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-tracers.h
 
 python $QEMU2_TOP_DIR/scripts/tracetool.py \
     --backends=nop \
     --format=c \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-tracers.c
 
 python $QEMU2_TOP_DIR/scripts/tracetool.py \
     --backends=nop \
     --format=events-h \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-events.h
 
 python $QEMU2_TOP_DIR/scripts/tracetool.py \
     --backends=nop \
     --format=events-c \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-events.c
 
 python $QEMU2_TOP_DIR/scripts/tracetool.py \
     --backends=nop \
     --format=tcg-helper-h \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-helpers.h
 
 python $QEMU2_TOP_DIR/scripts/tracetool.py \
     --backends=nop \
     --format=tcg-helper-wrapper-h \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-helpers-wrappers.h
 
 python $QEMU2_TOP_DIR/scripts/tracetool.py \
     --backends=nop \
     --format=tcg-h \
     --target-type system \
-    < $QEMU2_TOP_DIR/trace-events \
+    < $TRACE_EVENTS_ALL \
     > $QEMU2_AUTOGENERATED_DIR/trace/generated-tcg-tracers.h
 
 bash $QEMU2_TOP_DIR/scripts/hxtool -h \
     < $QEMU2_TOP_DIR/qemu-options.hx \
     > $QEMU2_AUTOGENERATED_DIR/qemu-options.def
 
+replace_with_if_different \
+    "$QEMU2_TOP_DIR/qemu-options.def" \
+    $QEMU2_AUTOGENERATED_DIR/qemu-options.def
+
+
 bash $QEMU2_TOP_DIR/scripts/hxtool -h \
     < $QEMU2_TOP_DIR/hmp-commands.hx \
     > $QEMU2_AUTOGENERATED_DIR/hmp-commands.h
 
 bash $QEMU2_TOP_DIR/scripts/hxtool -h \
+    < $QEMU2_TOP_DIR/hmp-commands-info.hx \
+    > $QEMU2_AUTOGENERATED_DIR/hmp-commands-info.h
+
+bash $QEMU2_TOP_DIR/scripts/hxtool -h \
     < $QEMU2_TOP_DIR/qmp-commands.hx \
     > $QEMU2_AUTOGENERATED_DIR/qmp-commands-old.h
 
@@ -100,13 +166,6 @@
     < $QEMU2_TOP_DIR/qemu-img-cmds.hx \
     > $QEMU2_AUTOGENERATED_DIR/qemu-img-cmds.h
 
-HEX_FILES=$(find $QEMU2_TOP_DIR/hw/i386/ -name "*.hex.generated" | \
-    sed -e 's|'$QEMU2_TOP_DIR/'||g')
-mkdir -p $QEMU2_AUTOGENERATED_DIR/hw/i386
-for HEX_FILES in $HEX_FILES; do
-    cp "$QEMU2_TOP_DIR/$HEX_FILES" "$QEMU2_AUTOGENERATED_DIR/${HEX_FILES%.generated}"
-done
-
 rm -f $QEMU2_AUTOGENERATED_DIR/gdbstub-xml-arm64.c
 bash $QEMU2_TOP_DIR/scripts/feature_to_c.sh \
     $QEMU2_AUTOGENERATED_DIR/gdbstub-xml-arm64.c \
@@ -131,6 +190,22 @@
         $QEMU2_TOP_DIR/version.rc
 fi
 
+# Generate qemu-version.h from Git history.
+QEMU_VERSION_H=$QEMU2_AUTOGENERATED_DIR/qemu-version.h
+QEMU_VERSION_H_TMP=$QEMU_VERSION_H.tmp
+rm -f "$QEMU_VERSION_H"
+if [ -d "$QEMU2_TOP_DIR/.git" ]; then
+    QEMU_VERSION=$(cd "$QEMU2_TOP_DIR" && git describe --match 'v*' 2>/dev/null | tr -d '\n')
+else
+    QEMU_VERSION=$(date "+%Y-%m-%d")
+fi
+
+echo "QEMU2    : Version [$QEMU_VERSION]"
+
+printf "#define QEMU_PKGVERSION \"(android-%s)\"\n" "$QEMU_VERSION" > $QEMU_VERSION_H_TMP
+replace_with_if_different "$QEMU_VERSION_H" "$QEMU_VERSION_H_TMP"
+rm -f "$QEMU_VERSION_TMP_H"
+
 # Work-around for a QEMU2 bug:
 # $QEMU2/linux-headers/linux/kvm.h includes <asm/kvm.h>
 # but $QEMU2/linux-headers/asm/ doesn't exist. It is supposed
@@ -139,4 +214,5 @@
 # The end result is that the <asm/kvm.h> from the host system
 # or toolchain sysroot is being included, which ends up in a
 # conflict. Work around it by creating a symlink here
+rm -f $QEMU2_AUTOGENERATED_DIR/asm
 ln -sf $QEMU2_TOP_DIR/linux-headers/asm-x86 $QEMU2_AUTOGENERATED_DIR/asm
diff --git a/android-qemu2-glue/config/darwin-x86_64/config-host.h b/android-qemu2-glue/config/darwin-x86_64/config-host.h
index 72dc9eb..1a7cdee 100644
--- a/android-qemu2-glue/config/darwin-x86_64/config-host.h
+++ b/android-qemu2-glue/config/darwin-x86_64/config-host.h
@@ -1,11 +1,11 @@
 /* Automatically generated by create_config - do not modify */
-#define CONFIG_QEMU_CONFDIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/etc/qemu"
-#define CONFIG_QEMU_DATADIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/share/qemu"
-#define CONFIG_QEMU_DOCDIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/share/doc/qemu"
-#define CONFIG_QEMU_MODDIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/lib/qemu"
-#define CONFIG_QEMU_LOCALSTATEDIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/var"
-#define CONFIG_QEMU_HELPERDIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/libexec"
-#define CONFIG_QEMU_LOCALEDIR "/Volumes/Android/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/darwin-x86_64/share/locale"
+#define CONFIG_QEMU_CONFDIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/etc/qemu"
+#define CONFIG_QEMU_DATADIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/share/qemu"
+#define CONFIG_QEMU_DOCDIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/share/doc/qemu"
+#define CONFIG_QEMU_MODDIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/lib/qemu"
+#define CONFIG_QEMU_LOCALSTATEDIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/var"
+#define CONFIG_QEMU_HELPERDIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/libexec"
+#define CONFIG_QEMU_LOCALEDIR "/tmp/qemu-android-build/prebuilts/qemu-android-deps/darwin-x86_64/share/locale"
 #define HOST_X86_64 1
 #define CONFIG_POSIX 1
 #define CONFIG_DARWIN 1
@@ -22,23 +22,23 @@
 #define CONFIG_VNC 1
 #define CONFIG_FNMATCH 1
 #define CONFIG_UUID 1
-#define QEMU_VERSION "2.2.0"
-#define QEMU_PKGVERSION ""
+#define QEMU_VERSION "2.7.0"
 #define CONFIG_SDL 1
 #define CONFIG_SDLABI 2.0
+#define CONFIG_HAS_GLIB_SUBPROCESS_TESTS 1
+#define CONFIG_TLS_PRIORITY "NORMAL"
+#define HAVE_IFADDRS_H 1
 #define CONFIG_IOVEC 1
 #define CONFIG_FDT 1
 #define CONFIG_MADVISE 1
 #define CONFIG_POSIX_MADVISE 1
 #define CONFIG_BSD 1
-#define CONFIG_ZERO_MALLOC 1
 #define CONFIG_QOM_CAST_DEBUG 1
 #define CONFIG_COROUTINE_BACKEND sigaltstack
 #define CONFIG_COROUTINE_POOL 1
 #define CONFIG_CPUID_H 1
 #define CONFIG_INT128 1
-#define CONFIG_TPM $(CONFIG_SOFTMMU)
-#define CONFIG_TRACE_NOP 1
+#define CONFIG_TPM 1
+#define CONFIG_TRACE_LOG 1
 #define CONFIG_TRACE_FILE trace
 #define HOST_DSOSUF ".so"
-#define CONFIG_ANDROID 1
diff --git a/android-qemu2-glue/config/linux-x86_64/config-host.h b/android-qemu2-glue/config/linux-x86_64/config-host.h
index 739c90a..1cbccdf 100644
--- a/android-qemu2-glue/config/linux-x86_64/config-host.h
+++ b/android-qemu2-glue/config/linux-x86_64/config-host.h
@@ -22,8 +22,7 @@
     NULL
 #define CONFIG_VNC 1
 #define CONFIG_FNMATCH 1
-#define QEMU_VERSION "2.2.0"
-#define QEMU_PKGVERSION ""
+#define QEMU_VERSION "2.7.0"
 #define CONFIG_SDL 1
 #define CONFIG_SDLABI 2.0
 #define CONFIG_UTIMENSAT 1
@@ -40,12 +39,14 @@
 #define CONFIG_PRCTL_PR_SET_TIMERSLACK 1
 #define CONFIG_EPOLL 1
 #define CONFIG_EPOLL_CREATE1 1
-#define CONFIG_EPOLL_PWAIT 1
 #define CONFIG_SENDFILE 1
 #define CONFIG_TIMERFD 1
 #define CONFIG_INOTIFY 1
 #define CONFIG_INOTIFY1 1
 #define CONFIG_BYTESWAP_H 1
+#define CONFIG_HAS_GLIB_SUBPROCESS_TESTS 1
+#define CONFIG_TLS_PRIORITY "NORMAL"
+#define HAVE_IFADDRS_H 1
 #define CONFIG_VHOST_SCSI 1
 #define CONFIG_IOVEC 1
 #define CONFIG_PREADV 1
@@ -54,9 +55,6 @@
 #define CONFIG_FDATASYNC 1
 #define CONFIG_MADVISE 1
 #define CONFIG_POSIX_MADVISE 1
-#define CONFIG_SIGEV_THREAD_ID 1
-#define CONFIG_GLX 1
-#define CONFIG_ZERO_MALLOC 1
 #define CONFIG_QOM_CAST_DEBUG 1
 #define CONFIG_COROUTINE_BACKEND ucontext
 #define CONFIG_COROUTINE_POOL 1
@@ -65,9 +63,9 @@
 #define CONFIG_HAS_ENVIRON 1
 #define CONFIG_CPUID_H 1
 #define CONFIG_INT128 1
-#define CONFIG_TPM $(CONFIG_SOFTMMU)
+#define CONFIG_TPM 1
 #define CONFIG_TPM_PASSTHROUGH 1
-#define CONFIG_TRACE_NOP 1
+#define CONFIG_TRACE_LOG 1
 #define CONFIG_TRACE_FILE trace
+#define CONFIG_IASL iasl
 #define HOST_DSOSUF ".so"
-#define CONFIG_ANDROID 1
diff --git a/android-qemu2-glue/config/target-mips/config-target.h b/android-qemu2-glue/config/target-mips/config-target.h
index f7c37f0..e0a14fb 100644
--- a/android-qemu2-glue/config/target-mips/config-target.h
+++ b/android-qemu2-glue/config/target-mips/config-target.h
@@ -2,6 +2,7 @@
 #define TARGET_ABI_MIPSO32 1
 #define TARGET_MIPS 1
 #define TARGET_NAME "mipsel"
+#define TARGET_MIPS 1
 #define CONFIG_SOFTMMU 1
 #define CONFIG_I386_DIS 1
 #define CONFIG_MIPS_DIS 1
diff --git a/android-qemu2-glue/config/windows-x86/config-host.h b/android-qemu2-glue/config/windows-x86/config-host.h
index 3bc3a6e..476bdb9 100644
--- a/android-qemu2-glue/config/windows-x86/config-host.h
+++ b/android-qemu2-glue/config/windows-x86/config-host.h
@@ -7,35 +7,35 @@
 #define CONFIG_QEMU_LOCALEDIR "/opt2/digit/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/windows-x86/locale"
 #define HOST_I386 1
 #define CONFIG_WIN32 1
-#define CONFIG_FILEVERSION 2,2,0,0
-#define CONFIG_PRODUCTVERSION 2,2,0,0
+#define CONFIG_FILEVERSION 2,7,0,0
+#define CONFIG_PRODUCTVERSION 2,7,0,0
 #define CONFIG_SLIRP 1
 #define CONFIG_SMBD_COMMAND "/usr/sbin/smbd"
 #define CONFIG_AUDIO_DRIVERS \
     &winaudio_audio_driver,\
-    &winwave_audio_driver,\
+    &dsound_audio_driver,\
 
 #define CONFIG_WINAUDIO 1
-#define CONFIG_WINWAVE 1
+#define CONFIG_DSOUND 1
 #define CONFIG_AUDIO_WIN_INT 1
 #define CONFIG_BDRV_RW_WHITELIST\
     NULL
 #define CONFIG_BDRV_RO_WHITELIST\
     NULL
 #define CONFIG_VNC 1
-#define QEMU_VERSION "2.2.0"
-#define QEMU_PKGVERSION ""
+#define QEMU_VERSION "2.7.0"
 #define CONFIG_SDL 1
 #define CONFIG_SDLABI 2.0
+#define CONFIG_HAS_GLIB_SUBPROCESS_TESTS 1
+#define CONFIG_TLS_PRIORITY "NORMAL"
 #define CONFIG_FDT 1
-#define CONFIG_ZERO_MALLOC 1
 #define CONFIG_QOM_CAST_DEBUG 1
 #define CONFIG_COROUTINE_BACKEND win32
 #define CONFIG_COROUTINE_POOL 1
 #define CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE 1
 #define CONFIG_CPUID_H 1
-#define CONFIG_TPM $(CONFIG_SOFTMMU)
-#define CONFIG_TRACE_NOP 1
+#define CONFIG_TPM 1
+#define CONFIG_TRACE_LOG 1
 #define CONFIG_TRACE_FILE trace
+#define CONFIG_IASL iasl
 #define HOST_DSOSUF ".dll"
-#define CONFIG_ANDROID 1
diff --git a/android-qemu2-glue/config/windows-x86_64/config-host.h b/android-qemu2-glue/config/windows-x86_64/config-host.h
index a3df876..ff2870d 100644
--- a/android-qemu2-glue/config/windows-x86_64/config-host.h
+++ b/android-qemu2-glue/config/windows-x86_64/config-host.h
@@ -7,36 +7,36 @@
 #define CONFIG_QEMU_LOCALEDIR "/opt2/digit/repo/studio-dev/prebuilts/android-emulator-build/qemu-android-deps/windows-x86_64/locale"
 #define HOST_X86_64 1
 #define CONFIG_WIN32 1
-#define CONFIG_FILEVERSION 2,2,0,0
-#define CONFIG_PRODUCTVERSION 2,2,0,0
+#define CONFIG_FILEVERSION 2,7,0,0
+#define CONFIG_PRODUCTVERSION 2,7,0,0
 #define CONFIG_SLIRP 1
 #define CONFIG_SMBD_COMMAND "/usr/sbin/smbd"
 #define CONFIG_AUDIO_DRIVERS \
     &winaudio_audio_driver,\
-    &winwave_audio_driver,\
+    &dsound_audio_driver,\
 
 #define CONFIG_WINAUDIO 1
-#define CONFIG_WINWAVE 1
+#define CONFIG_DSOUND 1
 #define CONFIG_AUDIO_WIN_INT 1
 #define CONFIG_BDRV_RW_WHITELIST\
     NULL
 #define CONFIG_BDRV_RO_WHITELIST\
     NULL
 #define CONFIG_VNC 1
-#define QEMU_VERSION "2.2.0"
-#define QEMU_PKGVERSION ""
+#define QEMU_VERSION "2.7.0"
 #define CONFIG_SDL 1
 #define CONFIG_SDLABI 2.0
+#define CONFIG_HAS_GLIB_SUBPROCESS_TESTS 1
+#define CONFIG_TLS_PRIORITY "NORMAL"
 #define CONFIG_FDT 1
-#define CONFIG_ZERO_MALLOC 1
 #define CONFIG_QOM_CAST_DEBUG 1
 #define CONFIG_COROUTINE_BACKEND win32
 #define CONFIG_COROUTINE_POOL 1
 #define CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE 1
 #define CONFIG_CPUID_H 1
 #define CONFIG_INT128 1
-#define CONFIG_TPM $(CONFIG_SOFTMMU)
-#define CONFIG_TRACE_NOP 1
+#define CONFIG_TPM 1
+#define CONFIG_TRACE_LOG 1
 #define CONFIG_TRACE_FILE trace
+#define CONFIG_IASL iasl
 #define HOST_DSOSUF ".dll"
-#define CONFIG_ANDROID 1
diff --git a/android-qemu2-glue/display.cpp b/android-qemu2-glue/display.cpp
index 99a9683..0f2a055 100644
--- a/android-qemu2-glue/display.cpp
+++ b/android-qemu2-glue/display.cpp
@@ -19,7 +19,8 @@
 #include "android/emulator-window.h"
 
 extern "C" {
-    #include "ui/console.h"
+#include "qemu/osdep.h"
+#include "ui/console.h"
 }
 
 namespace {
@@ -149,6 +150,10 @@
     return true;
 }
 
+extern "C" void sdl_display_early_init(int opengl) {
+    (void)opengl;
+}
+
 extern "C" bool sdl_display_init(DisplayState* ds,
                                  int full_screen,
                                  int no_frame) {
diff --git a/android-qemu2-glue/display.h b/android-qemu2-glue/display.h
index 98fc9c7..9eb7148 100644
--- a/android-qemu2-glue/display.h
+++ b/android-qemu2-glue/display.h
@@ -18,8 +18,9 @@
 
 ANDROID_BEGIN_HEADER
 
-#include "ui/console.h"
-#include "../qemu/android/framebuffer.h"
+#include "qemu/osdep.h"
+#include "qemu/typedefs.h"
+#include "android/framebuffer.h"
 
 bool android_display_init(DisplayState* ds, QFrameBuffer* qfbuff);
 
diff --git a/android-qemu2-glue/emulation/CharSerialLine.cpp b/android-qemu2-glue/emulation/CharSerialLine.cpp
index 6a958f6..e3e4063 100644
--- a/android-qemu2-glue/emulation/CharSerialLine.cpp
+++ b/android-qemu2-glue/emulation/CharSerialLine.cpp
@@ -11,7 +11,10 @@
 
 #include "android-qemu2-glue/emulation/CharSerialLine.h"
 
+#include <type_traits>
+
 extern "C" {
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 }
 
diff --git a/android-qemu2-glue/emulation/VmLock.cpp b/android-qemu2-glue/emulation/VmLock.cpp
index ece6ab3..9bc0ce3 100644
--- a/android-qemu2-glue/emulation/VmLock.cpp
+++ b/android-qemu2-glue/emulation/VmLock.cpp
@@ -14,24 +14,15 @@
 
 #include "android-qemu2-glue/emulation/VmLock.h"
 
-#include "android/base/Log.h"
+#include <type_traits>
 
 extern "C" {
+#include "qemu/osdep.h"
 #include "qemu/main-loop.h"
 }  // extern "C"
 
 namespace qemu2 {
 
-// TECHNICAL NOTE:
-//
-// This implementation needs to protect against recursive lock() calls
-// which can happen because some code in AndroidEmu calls it, without
-// knowing whether it's running in the thread that holds the BQL or not.
-//
-// What we want is to ensure that qemu_mutex_lock_iothread() is always true
-// when we leave ::lock(), and that it will be false when ::unlock() has
-// been called as often as ::lock() was.
-
 void VmLock::lock() {
     qemu_mutex_lock_iothread();
 }
@@ -41,7 +32,7 @@
 }
 
 bool VmLock::isLockedBySelf() const {
-    return qemu_mutex_check_iothread();
+    return qemu_mutex_iothread_locked();
 }
 
 }  // namespace qemu2
diff --git a/android-qemu2-glue/emulation/charpipe.c b/android-qemu2-glue/emulation/charpipe.c
index c03f8ac..1d4b59e 100644
--- a/android-qemu2-glue/emulation/charpipe.c
+++ b/android-qemu2-glue/emulation/charpipe.c
@@ -11,11 +11,12 @@
 */
 #include "android/utils/debug.h"
 #include "android/utils/cbuffer.h"
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 
-#define  xxDEBUG
+#define DEBUG 0
 
-#ifdef DEBUG
+#if DEBUG
 #  include <stdio.h>
 #  define  D(...)   ( fprintf( stderr, __VA_ARGS__ ), fprintf(stderr, "\n") )
 #else
@@ -441,7 +442,7 @@
 
 
 void
-charpipe_poll( void )
+qemu_charpipe_poll( void )
 {
     CharPipeState*  cp     = _s_charpipes;
     CharPipeState*  cp_end = cp + MAX_CHAR_PIPES;
diff --git a/android-qemu2-glue/emulation/charpipe.h b/android-qemu2-glue/emulation/charpipe.h
index 7a62487..2ab24e9 100644
--- a/android-qemu2-glue/emulation/charpipe.h
+++ b/android-qemu2-glue/emulation/charpipe.h
@@ -11,6 +11,7 @@
 */
 #pragma once
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "android/utils/compiler.h"
 
@@ -20,16 +21,17 @@
  * QEMU components. For Android, this is used to connect an emulated serial port
  * with the android modem
  */
-extern int  qemu_chr_open_charpipe( CharDriverState* *pfirst, CharDriverState* *psecond );
+extern int qemu_chr_open_charpipe(CharDriverState* *pfirst,
+                                  CharDriverState* *psecond);
 
 /* create a buffering character driver for a given endpoint. The result will buffer
  * anything that is sent to it but cannot be sent to the endpoint immediately.
  * On the other hand, if the endpoint calls can_read() or read(), these calls
  * are passed immediately to the can_read() or read() handlers of the result.
  */
-extern CharDriverState*  qemu_chr_open_buffer( CharDriverState*  endpoint );
+extern CharDriverState *qemu_chr_open_buffer(CharDriverState*  endpoint);
 
 /* must be called from the main event loop to poll all charpipes */
-extern void charpipe_poll( void );
+extern void qemu_charpipe_poll(void);
 
 ANDROID_END_HEADER
diff --git a/android-qemu2-glue/main.cpp b/android-qemu2-glue/main.cpp
index 3735019..0a18f06 100755
--- a/android-qemu2-glue/main.cpp
+++ b/android-qemu2-glue/main.cpp
@@ -448,6 +448,7 @@
     if (opts->skip_adb_auth) {
         args[n++] = "-skip-adb-auth";
     }
+
     /** SNAPSHOT STORAGE HANDLING */
 
     /* If we have a valid snapshot storage path */
@@ -709,12 +710,6 @@
     args[n++] = "-cpu";
     args[n++] = kTarget.qemuCpu;
 
-    // Set env var to "on" for Intel PMU if the feature is enabled.
-    // cpu.c will then read that.
-    if (android::featurecontrol::isEnabled(android::featurecontrol::IntelPerformanceMonitoringUnit)) {
-        System::get()->envSet("ANDROID_EMU_FEATURE_IntelPerformanceMonitoringUnit", "on");
-    }
-
 #if defined(TARGET_X86_64) || defined(TARGET_I386)
     char* accel_status = NULL;
     CpuAccelMode accel_mode = ACCEL_AUTO;
diff --git a/android-qemu2-glue/net-android.cpp b/android-qemu2-glue/net-android.cpp
index d5fe8b3..913e6b5 100644
--- a/android-qemu2-glue/net-android.cpp
+++ b/android-qemu2-glue/net-android.cpp
@@ -1,4 +1,4 @@
-// Copyright 2015 The Android Open Source Project
+// Copyright 2016 The Android Open Source Project
 //
 // This software is licensed under the terms of the GNU General Public
 // License version 2, as published by the Free Software Foundation, and
@@ -9,7 +9,8 @@
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU General Public License for more details.
 
-#include "net-android.h"
+#include "android-qemu2-glue/net-android.h"
+
 #include "android/android.h"
 #include "android/network/constants.h"
 #include "android/network/globals.h"
@@ -17,48 +18,48 @@
 #include "android/shaper.h"
 
 extern "C" {
-#include "net/net.h"
+#include "qemu/osdep.h"
+#include "net/slirp.h"
 }
 
-#if defined(CONFIG_SLIRP)
-static void* s_slirp_state = nullptr;
-static void* s_net_client_state = nullptr;
-static Slirp* s_slirp = nullptr;
+static void* s_opaque = nullptr;
 
-static void
-android_net_delay_in_cb(void* data, size_t size, void* opaque)
+void android_qemu_init_slirp_shapers(void)
 {
-    slirp_input(s_slirp, static_cast<const uint8_t*>(data), size);
-}
+    android_net_delay_in = netdelay_create(
+            [](void* data, size_t size, void* opaque) {
+                net_slirp_receive_raw(s_opaque,
+                                      static_cast<const uint8_t*>(data),
+                                      static_cast<int>(size));
+            });
 
-static void
-android_net_shaper_in_cb(void* data, size_t size, void* opaque)
-{
-    netdelay_send_aux(android_net_delay_in, data, size, opaque);
-}
+    android_net_shaper_in = netshaper_create(1,
+            [](void* data, size_t size, void* opaque) {
+                netdelay_send_aux(android_net_delay_in, data, size, opaque);
+            });
 
-static void
-android_net_shaper_out_cb(void* data, size_t size, void* opaque)
-{
-    qemu_send_packet(static_cast<NetClientState*>(s_net_client_state),
-                     static_cast<const uint8_t*>(data),
-                     size);
-}
-
-void
-slirp_init_shapers(void* slirp_state, void* net_client_state, Slirp* slirp)
-{
-    s_slirp_state = slirp_state;
-    s_net_client_state = net_client_state;
-    s_slirp = slirp;
-    android_net_delay_in = netdelay_create(android_net_delay_in_cb);
-    android_net_shaper_in = netshaper_create(1, android_net_shaper_in_cb);
-    android_net_shaper_out = netshaper_create(1, android_net_shaper_out_cb);
+    android_net_shaper_out = netshaper_create(1,
+            [](void* data, size_t size, void* opaque) {
+                net_slirp_output_raw(opaque,
+                                     static_cast<const uint8_t*>(data),
+                                     static_cast<int>(size));
+            });
 
     netdelay_set_latency(android_net_delay_in, android_net_min_latency,
                          android_net_max_latency);
+
     netshaper_set_rate(android_net_shaper_out, android_net_download_speed);
     netshaper_set_rate(android_net_shaper_in, android_net_upload_speed);
-}
-#endif  // CONFIG_SLIRP
 
+    s_opaque = net_slirp_set_shapers(
+            android_net_shaper_out,
+            [](void* opaque, const void* data, int len) {
+                netshaper_send_aux(static_cast<NetShaper>(opaque),
+                                   (char*)data, len, s_opaque);
+            },
+            android_net_shaper_in,
+            [](void* opaque, const void* data, int len) {
+                netshaper_send_aux(static_cast<NetShaper>(opaque),
+                                   (void*)data, len, s_opaque);
+            });
+}
diff --git a/android-qemu2-glue/net-android.h b/android-qemu2-glue/net-android.h
index 693d87e..83add05 100644
--- a/android-qemu2-glue/net-android.h
+++ b/android-qemu2-glue/net-android.h
@@ -1,4 +1,4 @@
-// Copyright 2015 The Android Open Source Project
+// Copyright 2016 The Android Open Source Project
 //
 // This software is licensed under the terms of the GNU General Public
 // License version 2, as published by the Free Software Foundation, and
@@ -11,17 +11,10 @@
 
 #pragma once
 
-#include "config-host.h"
 #include "android/utils/compiler.h"
 
 ANDROID_BEGIN_HEADER
 
-#if defined(CONFIG_SLIRP)
-#include "libslirp.h"
-
-void slirp_init_shapers(void* slirp_state,
-                        void* net_client_state,
-                        Slirp *slirp);
-#endif  // CONFIG_SLIRP
+void android_qemu_init_slirp_shapers(void);
 
 ANDROID_END_HEADER
diff --git a/android-qemu2-glue/qemu-display-agent-impl.cpp b/android-qemu2-glue/qemu-display-agent-impl.cpp
index 8559da5..8167372 100644
--- a/android-qemu2-glue/qemu-display-agent-impl.cpp
+++ b/android-qemu2-glue/qemu-display-agent-impl.cpp
@@ -15,9 +15,12 @@
 #include "android-qemu2-glue/qemu-control-impl.h"
 
 extern "C" {
-    #include "ui/console.h"
+#include "qemu/osdep.h"
+#include "ui/console.h"
 }
 
+#include <string.h>
+
 static void getFrameBuffer(int* w, int* h, int* lineSize, int* bytesPerPixel,
                            uint8_t** frameBufferData) {
     // find the first graphics console
@@ -56,36 +59,51 @@
 }
 
 namespace {
-    struct dul_data {
-        AndroidDisplayUpdateCallback callback;
-        void* opaque;
-    };
-}
 
-static void on_display_update(DisplayUpdateListener* dul,
-        int x, int y, int w, int h) {
-    dul_data* data = static_cast<dul_data*>(dul->opaque);
-    data->callback(data->opaque, x, y, w, h);
-}
+struct AndroidDisplayChangeListener : public DisplayChangeListener {
+    AndroidDisplayChangeListener(AndroidDisplayUpdateCallback callback,
+                                 void* opaque) {
+        memset(this, 0, sizeof(*this));
+        mCallback = callback;
+        mOpaque = opaque;
+        this->ops = &kOps;
+        register_displaychangelistener(this);
+    }
+
+    ~AndroidDisplayChangeListener() {
+        unregister_displaychangelistener(this);
+    }
+
+    AndroidDisplayUpdateCallback mCallback;
+    void* mOpaque;
+
+    static void onDisplayUpdate(DisplayChangeListener* dcl,
+                                int x, int y, int w, int h) {
+        auto adcl = reinterpret_cast<AndroidDisplayChangeListener*>(dcl);
+        adcl->mCallback(adcl->mOpaque, x, y, w, h);
+    }
+
+    static const DisplayChangeListenerOps kOps;
+};
+
+// static
+const DisplayChangeListenerOps AndroidDisplayChangeListener::kOps = {
+    .dpy_name = "qemu2 display",
+    .dpy_refresh = nullptr,
+    .dpy_gfx_update = &onDisplayUpdate,
+};
+
+}  // namespace
 
 static void registerUpdateListener(AndroidDisplayUpdateCallback callback,
                                    void* opaque) {
-    const auto listener = new DisplayUpdateListener();
-    *listener = DisplayUpdateListener();
-    listener->dpy_gfx_update = &on_display_update;
-
-    auto data = new dul_data();
-    data->callback = callback;
-    data->opaque = opaque;
-    listener->opaque = data;
-
-    register_displayupdatelistener(listener);
+    static AndroidDisplayChangeListener* s_listener =
+            new AndroidDisplayChangeListener(callback, opaque);
 }
 
-
 static const QAndroidDisplayAgent displayAgent = {
-        .getFrameBuffer = &getFrameBuffer,
-        .registerUpdateListener = &registerUpdateListener
+    .getFrameBuffer = &getFrameBuffer,
+    .registerUpdateListener = &registerUpdateListener
 };
 
 const QAndroidDisplayAgent* const gQAndroidDisplayAgent = &displayAgent;
diff --git a/android-qemu2-glue/qemu-net-agent-impl.c b/android-qemu2-glue/qemu-net-agent-impl.c
index 3bb1e72..eac5b77 100644
--- a/android-qemu2-glue/qemu-net-agent-impl.c
+++ b/android-qemu2-glue/qemu-net-agent-impl.c
@@ -15,6 +15,7 @@
 #include "android/emulation/control/net_agent.h"
 
 #include "android/utils/sockets.h"
+#include "qemu/osdep.h"
 #include "qemu/sockets.h"
 #include "net/slirp.h"
 
@@ -22,21 +23,20 @@
 #include "libslirp.h"
 
 static bool isSlirpInited() {
-    return net_slirp_is_inited() != 0;
+    return net_slirp_state() != NULL;
 }
 
 static bool slirpRedir(bool isUdp, int hostPort,
                       uint32_t guestAddr, int guestPort) {
     struct in_addr host = { .s_addr = htonl(SOCK_ADDRESS_INET_LOOPBACK) };
     struct in_addr guest = { .s_addr = 0 };
-    return slirp_add_hostfwd(net_slirp_lookup(NULL, NULL, NULL), isUdp,
-                             host, hostPort, guest, guestPort) == 0;
+    return slirp_add_hostfwd(net_slirp_state(), isUdp, host, hostPort, guest,
+                             guestPort) == 0;
 }
 
 bool slirpUnredir(bool isUdp, int hostPort) {
     struct in_addr host = { .s_addr = htonl(SOCK_ADDRESS_INET_LOOPBACK) };
-    return slirp_remove_hostfwd(net_slirp_lookup(NULL, NULL, NULL), isUdp,
-                                host, hostPort) == 0;
+    return slirp_remove_hostfwd(net_slirp_state(), isUdp, host, hostPort) == 0;
 }
 
 
diff --git a/android-qemu2-glue/qemu-setup-dns-servers.cpp b/android-qemu2-glue/qemu-setup-dns-servers.cpp
new file mode 100644
index 0000000..108e53d
--- /dev/null
+++ b/android-qemu2-glue/qemu-setup-dns-servers.cpp
@@ -0,0 +1,175 @@
+// Copyright 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "android-qemu2-glue/qemu-setup.h"
+
+#include "android/base/Log.h"
+#include "android/utils/debug.h"
+
+#ifdef _WIN32
+// This includes must happen before qemu/osdep.h to avoid compiler
+// errors regarding FD_SETSIZE being redefined.
+#include "android/base/sockets/Winsock.h"
+#include "android/base/sockets/SocketErrors.h"
+#else
+#include <sys/socket.h>
+#include <netdb.h>
+#endif
+
+extern "C" {
+#include "qemu/osdep.h"
+#include "net/slirp.h"
+#include "slirp/libslirp.h"
+}  // extern "C"
+
+#include <vector>
+
+#include <errno.h>
+
+#define MAX_DNS_SERVERS 8
+
+static int s_num_dns_server_addresses = 0;
+static sockaddr_storage s_dns_server_addresses[MAX_DNS_SERVERS] = {};
+
+// Resolve host name |hostName| into a list of sockaddr_storage.
+// On success, return true and append the names to |*out|. On failure
+// return false, leave |*out| untouched, and sets errno.
+static bool resolveHostNameToList(
+        const char* hostName, std::vector<sockaddr_storage>* out) {
+    addrinfo* res = nullptr;
+    addrinfo hints = {};
+    hints.ai_family = AF_UNSPEC;
+    int ret = ::getaddrinfo(hostName, nullptr, &hints, &res);
+    if (ret != 0) {
+        // Handle errors.
+        int err = 0;
+        switch (ret) {
+            case EAI_AGAIN:  // server is down
+            case EAI_FAIL:   // server is sick
+                err = EHOSTDOWN;
+                break;
+/* NOTE that in x86_64-w64-mingw32 both EAI_NODATA and EAI_NONAME are the same */
+#if defined(EAI_NODATA) && (EAI_NODATA != EAI_NONAME)
+            case EAI_NODATA:
+#endif
+            case EAI_NONAME:
+                err = ENOENT;
+                break;
+
+            case EAI_MEMORY:
+                err = ENOMEM;
+                break;
+
+            default:
+                err = EINVAL;
+        }
+        errno = err;
+        return false;
+    }
+
+    int count = 0;
+
+    for (auto r = res; r != nullptr; r = r->ai_next) {
+        sockaddr_storage addr = {};
+        switch (r->ai_family) {
+            case AF_INET:
+                *(struct sockaddr_in *)&addr =
+                        *(const struct sockaddr_in *)r->ai_addr;
+                break;
+
+            case AF_INET6:
+                *(struct sockaddr_in6 *)&addr =
+                        *(const struct sockaddr_in6 *)r->ai_addr;
+                break;
+            default:
+                continue;
+        }
+        out->emplace_back(std::move(addr));
+        count++;
+    }
+    ::freeaddrinfo(res);
+
+    return (count > 0);
+}
+
+bool qemu_android_emulation_setup_dns_servers(const char* dns_servers,
+                                              int* pcount4, int* pcount6) {
+    CHECK(net_slirp_state() != nullptr) << "slirp stack should be inited!";
+
+    if (!dns_servers || !dns_servers[0]) {
+        // Empty list, use the default behaviour.
+        return 0;
+    }
+
+    std::vector<sockaddr_storage> server_addresses;
+
+    // Separate individual DNS server names, then resolve each one of them
+    // into one or more IP addresses. Support both IPv4 and IPv6 at the same
+    // time.
+    const char* p = dns_servers;
+    while (*p) {
+        const char* next_p;
+        const char* comma = strchr(p, ',');
+        if (!comma) {
+            comma = p + strlen(p);
+            next_p = comma;
+        } else {
+            next_p = comma + 1;
+        }
+        while (p < comma && *p == ' ') p++;
+        while (p < comma && comma[-1] == ' ') comma--;
+
+        if (comma > p) {
+            // Extract single server name.
+            std::string server(p, comma - p);
+            if (!resolveHostNameToList(server.c_str(), &server_addresses)) {
+                dwarning("Ignoring ivalid DNS address: [%s]\n", server.c_str());
+            }
+        }
+        p = next_p;
+    }
+
+    int count = static_cast<int>(server_addresses.size());
+    if (!count) {
+        return 0;
+    }
+
+    // Save it for qemu_android_emulator_init_slirp().
+    s_num_dns_server_addresses = count;
+    memcpy(s_dns_server_addresses, &server_addresses[0],
+           count * sizeof(server_addresses[0]));
+
+    // Count number of IPv4 and IPv6 DNS servers.
+    int count4 = 0;
+    int count6 = 0;
+    for (const auto& item : server_addresses) {
+        if (item.ss_family == AF_INET) {
+            count4 += 1;
+        } else if (item.ss_family == AF_INET6) {
+            count6 += 1;
+        }
+    }
+
+    *pcount4 = count4;
+    *pcount6 = count6;
+
+    return true;
+}
+
+void qemu_android_emulation_init_slirp(void) {
+    slirp_init_custom_dns_servers(static_cast<Slirp*>(net_slirp_state()),
+                                  s_dns_server_addresses,
+                                  s_num_dns_server_addresses);
+}
+
diff --git a/android-qemu2-glue/qemu-setup.cpp b/android-qemu2-glue/qemu-setup.cpp
index 0bbdb77..bdb0af8 100644
--- a/android-qemu2-glue/qemu-setup.cpp
+++ b/android-qemu2-glue/qemu-setup.cpp
@@ -18,27 +18,36 @@
 #include "android/base/Log.h"
 #include "android/console.h"
 #include "android-qemu2-glue/emulation/android_pipe_device.h"
-#include "android-qemu2-glue/emulation/VmLock.h"
-#include "android-qemu2-glue/qemu-control-impl.h"
+#include "android-qemu2-glue/emulation/charpipe.h"
 #include "android-qemu2-glue/emulation/goldfish_sync.h"
+#include "android-qemu2-glue/emulation/VmLock.h"
+#include "android-qemu2-glue/looper-qemu.h"
+#include "android-qemu2-glue/android_qemud.h"
+#include "android-qemu2-glue/net-android.h"
+#include "android-qemu2-glue/qemu-control-impl.h"
 
 extern "C" {
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "qemu/main-loop.h"
+#include "qemu/thread.h"
 }  // extern "C"
 
 using android::VmLock;
 
-bool qemu_android_emulation_setup() {
-    static const AndroidConsoleAgents consoleAgents = {
-            gQAndroidBatteryAgent,
-            gQAndroidFingerAgent,
-            gQAndroidLocationAgent,
-            gQAndroidTelephonyAgent,
-            gQAndroidUserEventAgent,
-            gQAndroidVmOperations,
-            gQAndroidNetAgent
-    };
+bool qemu_android_emulation_early_setup() {
+    // Ensure that the looper is set for the main thread and for any
+    // future thread created by QEMU.
+    qemu_looper_setForThread();
+    qemu_thread_register_setup_callback(qemu_looper_setForThread);
 
+    // Ensure charpipes i/o are handled properly.
+    main_loop_register_poll_callback(qemu_charpipe_poll);
+
+    // Register qemud-related snapshot callbacks.
+    android_qemu2_qemud_init();
+
+    // Ensure the VmLock implementation is setup.
     VmLock* vmLock = new qemu2::VmLock();
     VmLock* prevVmLock = VmLock::set(vmLock);
     CHECK(prevVmLock == nullptr) << "Another VmLock was already installed!";
@@ -48,9 +57,30 @@
         return false;
     }
 
+    // Initialize host sync service.
     if (!qemu_android_sync_init(vmLock)) {
         return false;
     }
 
+    return true;
+}
+
+bool qemu_android_emulation_setup() {
+    android_qemu_init_slirp_shapers();
+
+    // Initialize UI/console agents.
+    static const AndroidConsoleAgents consoleAgents = {
+            gQAndroidBatteryAgent,
+            gQAndroidFingerAgent,
+            gQAndroidLocationAgent,
+            gQAndroidTelephonyAgent,
+            gQAndroidUserEventAgent,
+            gQAndroidVmOperations,
+            gQAndroidNetAgent,
+    };
+
     return android_emulation_setup(&consoleAgents);
 }
+
+void qemu_android_emulation_teardown() {
+}
diff --git a/android-qemu2-glue/qemu-setup.h b/android-qemu2-glue/qemu-setup.h
index 1be84b4..aaf2f0d 100644
--- a/android-qemu2-glue/qemu-setup.h
+++ b/android-qemu2-glue/qemu-setup.h
@@ -20,6 +20,32 @@
 
 ANDROID_BEGIN_HEADER
 
+/* Call this function at the start of the QEMU main() function to perform
+ * early setup of Android emulation. This will ensure the glue will inject
+ * all relevant callbacks into QEMU2. As well as setup the looper for the
+ * main thread. Return true on success, false otherwise. */
+extern bool qemu_android_emulation_early_setup(void);
+
+/* Call this function to setup a list of custom DNS servers to be used
+ * by the network stack. |dns_servers| must be the content of the
+ * -dns-server option, i.e. a comma-separated list of DNS server addresses.
+ * On success, return true and set |*count4| and |*count6| to the number
+ * of IPv4 and IPv6 IP addresses, respectively. Return false on failure. */
+extern bool qemu_android_emulation_setup_dns_servers(const char* dns_servers,
+                                                     int* count4,
+                                                     int* count6);
+
+/* Call this function after the slirp stack has been initialized, typically
+ * by calling net_init_clients() in vl.c, to inject Android-specific features
+ * (e.g. custom DNS server list) into the network stack. */
+extern void qemu_android_emulation_init_slirp(void);
+
+/* Call this function after the QEMU main() function has inited the
+ * machine, but before it has started it. */
 extern bool qemu_android_emulation_setup(void);
 
+/* Call this function at the end of the QEMU main() function, just
+ * after the main loop has returned due to a machine exit. */
+extern void qemu_android_emulation_teardown(void);
+
 ANDROID_END_HEADER
diff --git a/android-qemu2-glue/qemu-user-event-agent-impl.c b/android-qemu2-glue/qemu-user-event-agent-impl.c
index a19f1e1..c43e68c 100644
--- a/android-qemu2-glue/qemu-user-event-agent-impl.c
+++ b/android-qemu2-glue/qemu-user-event-agent-impl.c
@@ -13,6 +13,8 @@
 
 #include "android/multitouch-screen.h"
 #include "android/utils/debug.h"
+
+#include "qemu/osdep.h"
 #include "hw/input/goldfish_events.h"
 #include "ui/console.h"
 
diff --git a/android-qemu2-glue/qemu-vm-operations-impl.c b/android-qemu2-glue/qemu-vm-operations-impl.c
index 8cf1587..2f745ba 100644
--- a/android-qemu2-glue/qemu-vm-operations-impl.c
+++ b/android-qemu2-glue/qemu-vm-operations-impl.c
@@ -16,8 +16,8 @@
 
 #include "android/emulation/control/callbacks.h"
 #include "android/emulation/control/vm_operations.h"
-//#include "cpu.h"
-//#include "monitor/monitor.h"
+
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 
 #include <stdlib.h>
diff --git a/android-qemu2-glue/qemu-window-agent-impl.c b/android-qemu2-glue/qemu-window-agent-impl.c
index 9687956..82b4996 100644
--- a/android-qemu2-glue/qemu-window-agent-impl.c
+++ b/android-qemu2-glue/qemu-window-agent-impl.c
@@ -21,4 +21,3 @@
 
 const QAndroidEmulatorWindowAgent* const gQAndroidEmulatorWindowAgent =
     &sQAndroidEmulatorWindowAgent;
-
diff --git a/android-qemu2-glue/scripts/gen-qemu2-sources-mk.py b/android-qemu2-glue/scripts/gen-qemu2-sources-mk.py
index 0404ba8..a951a0a 100755
--- a/android-qemu2-glue/scripts/gen-qemu2-sources-mk.py
+++ b/android-qemu2-glue/scripts/gen-qemu2-sources-mk.py
@@ -26,29 +26,28 @@
 LINK_QEMU_PREFIX = 'LINK-qemu-system-'
 IGNORED_OBJECTS = [
     '../audio/sdlaudio.o',
-    '../disas/arm-a64.o',
-    '../disas/libvixl/a64/decoder-a64.o',
-    '../disas/libvixl/a64/disasm-a64.o',
-    '../disas/libvixl/a64/instructions-a64.o',
-    '../disas/libvixl/utils.o',
     'gdbstub-xml.o',
-    '../hw/display/framebuffer.o',
-    'hw/misc/android_boot_properties.o',
-    'hw/misc/android_pipe_opengles.o',
+    'hw/i386/acpi-build.o',
+    'hw/i386/pc_piix.o',
+    '../qmp-introspect.o',
     '../qmp-marshal.o',
     'trace/generated-helpers.o',
     '../ui/sdl_zoom.o',
     '../ui/sdl.o',
     '../ui/sdl2.o',
+    '../ui/sdl2-2d.o',
+    '../ui/sdl2-input.o',
+    '../vl.o',
     '/version.o',   # something from the Windows build
     ]
 
 CC_OBJECTS = [
     'disas/arm-a64.o',
-    'disas/libvixl/a64/decoder-a64.o',
-    'disas/libvixl/a64/disasm-a64.o',
-    'disas/libvixl/a64/instructions-a64.o',
-    'disas/libvixl/utils.o',
+    'disas/libvixl/vixl/a64/decoder-a64.o',
+    'disas/libvixl/vixl/a64/disasm-a64.o',
+    'disas/libvixl/vixl/a64/instructions-a64.o',
+    'disas/libvixl/vixl/compiler-intrinsics.o',
+    'disas/libvixl/vixl/utils.o',
     ]
 
 # objects which have to be moved to *TARGET files,
diff --git a/android-qemu2-glue/stubs/win32-stubs.c b/android-qemu2-glue/stubs/win32-stubs.c
new file mode 100644
index 0000000..aecdc29
--- /dev/null
+++ b/android-qemu2-glue/stubs/win32-stubs.c
@@ -0,0 +1,54 @@
+#include "qemu/osdep.h"
+
+#include "android/utils/win32_unicode.h"
+
+HANDLE win32CreateFile(
+        LPCTSTR               lpFileName,
+        DWORD                 dwDesiredAccess,
+        DWORD                 dwShareMode,
+        LPSECURITY_ATTRIBUTES lpSecurityAttributes,
+        DWORD                 dwCreationDisposition,
+        DWORD                 dwFlagsAndAttributes,
+        HANDLE                hTemplateFile)
+{
+    HANDLE result = INVALID_HANDLE_VALUE;
+    wchar_t* wide_name = win32_utf8_to_utf16_str(lpFileName);
+    if (wide_name != NULL) {
+        result = CreateFileW(wide_name, dwDesiredAccess,
+                             dwShareMode, lpSecurityAttributes,
+                             dwCreationDisposition, dwFlagsAndAttributes,
+                             hTemplateFile);
+        free(wide_name);
+    }
+    return result;
+}
+
+DWORD win32GetCurrentDirectory(
+        DWORD  nBufferLength,
+        LPTSTR lpBuffer)
+{
+    wchar_t wide_buffer[MAX_PATH];
+    DWORD ret = GetCurrentDirectoryW(MAX_PATH, wide_buffer);
+    if (ret == 0 || ret > MAX_PATH) {
+        return ret;
+    }
+    int ret2 = win32_utf16_to_utf8_buf(wide_buffer, lpBuffer, nBufferLength);
+    return (ret2 < 0 || (DWORD)ret2 > nBufferLength) ? 0 : (DWORD)ret2;
+}
+
+DWORD win32GetModuleFileName(
+        HMODULE hModule,
+        LPTSTR  lpFilename,
+        DWORD   nSize)
+{
+    wchar_t wide_buffer[MAX_PATH];
+    if (!GetModuleFileNameW(hModule, wide_buffer, MAX_PATH)) {
+        return 0;
+    }
+
+    int ret = win32_utf16_to_utf8_buf(wide_buffer, lpFilename, nSize);
+    if (ret < 0 || ret >= nSize) {
+        return 0;
+    }
+    return (DWORD)ret;
+}
diff --git a/android-qemu2-glue/telephony/modem_init.c b/android-qemu2-glue/telephony/modem_init.c
index 0377c25..0e397a6 100644
--- a/android-qemu2-glue/telephony/modem_init.c
+++ b/android-qemu2-glue/telephony/modem_init.c
@@ -14,8 +14,11 @@
 #include "android/telephony/modem_driver.h"
 #include "android-qemu2-glue/utils/stream.h"
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 
+#include <assert.h>
+
 #define MODEM_DEV_STATE_SAVE_VERSION 1
 
 static void modem_state_save(QEMUFile* file, void* opaque)
@@ -41,13 +44,13 @@
 void qemu_android_modem_init(int base_port) {
     android_modem_init(base_port);
 
-    if (android_modem_serial_line != NULL) {
-        register_savevm(NULL,
-                        "android_modem",
-                        0,
-                        MODEM_DEV_STATE_SAVE_VERSION,
-                        modem_state_save,
-                        modem_state_load,
-                        android_modem);
-    }
+    assert(android_modem_serial_line != NULL);
+
+    register_savevm(NULL,
+                    "android_modem",
+                    0,
+                    MODEM_DEV_STATE_SAVE_VERSION,
+                    modem_state_save,
+                    modem_state_load,
+                    android_modem);
 }
diff --git a/arch_init.c b/arch_init.c
index b94d23f..1d09f32 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -21,45 +21,19 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdint.h>
-#include <stdarg.h>
-#include <stdlib.h>
-#ifndef _WIN32
-#include <sys/types.h>
-#include <sys/mman.h>
-#endif
-#include "config.h"
-#include "monitor/monitor.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
-#include "qemu/bitops.h"
-#include "qemu/bitmap.h"
 #include "sysemu/arch_init.h"
-#include "audio/audio.h"
-#include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
 #include "hw/audio/audio.h"
-#include "sysemu/kvm.h"
-#include "migration/migration.h"
-#include "hw/i386/smbios.h"
-#include "exec/address-spaces.h"
-#include "hw/audio/pcspk.h"
-#include "migration/page_cache.h"
+#include "hw/smbios/smbios.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
 #include "qmp-commands.h"
-#include "trace.h"
-#include "exec/cpu-all.h"
-#include "exec/ram_addr.h"
 #include "hw/acpi/acpi.h"
-#include "qemu/host-utils.h"
-
-#ifdef DEBUG_ARCH_INIT
-#define DPRINTF(fmt, ...) \
-    do { fprintf(stdout, "arch_init: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
+#include "qemu/help_option.h"
 
 #ifdef TARGET_SPARC
 int graphic_width = 1024;
@@ -109,23 +83,6 @@
 #endif
 
 const uint32_t arch_type = QEMU_ARCH;
-static bool mig_throttle_on;
-static int dirty_rate_high_cnt;
-static void check_guest_throttling(void);
-
-static uint64_t bitmap_sync_count;
-
-/***********************************************************/
-/* ram save/restore */
-
-#define RAM_SAVE_FLAG_FULL     0x01 /* Obsolete, not used anymore */
-#define RAM_SAVE_FLAG_COMPRESS 0x02
-#define RAM_SAVE_FLAG_MEM_SIZE 0x04
-#define RAM_SAVE_FLAG_PAGE     0x08
-#define RAM_SAVE_FLAG_EOS      0x10
-#define RAM_SAVE_FLAG_CONTINUE 0x20
-#define RAM_SAVE_FLAG_XBZRLE   0x40
-/* 0x80 is reserved in migration.h start with 0x100 next */
 
 static struct defconfig_file {
     const char *filename;
@@ -133,12 +90,9 @@
     bool userconfig;
 } default_config_files[] = {
     { CONFIG_QEMU_CONFDIR "/qemu.conf",                   true },
-    { CONFIG_QEMU_CONFDIR "/target-" TARGET_NAME ".conf", true },
     { NULL }, /* end of list */
 };
 
-static const uint8_t ZERO_TARGET_PAGE[TARGET_PAGE_SIZE];
-
 int qemu_read_default_config_files(bool userconfig)
 {
     int ret;
@@ -157,1026 +111,6 @@
     return 0;
 }
 
-static inline bool is_zero_range(uint8_t *p, uint64_t size)
-{
-    return buffer_find_nonzero_offset(p, size) == size;
-}
-
-/* struct contains XBZRLE cache and a static page
-   used by the compression */
-static struct {
-    /* buffer used for XBZRLE encoding */
-    uint8_t *encoded_buf;
-    /* buffer for storing page content */
-    uint8_t *current_buf;
-    /* Cache for XBZRLE, Protected by lock. */
-    PageCache *cache;
-    QemuMutex lock;
-} XBZRLE;
-
-/* buffer used for XBZRLE decoding */
-static uint8_t *xbzrle_decoded_buf;
-
-static void XBZRLE_cache_lock(void)
-{
-    if (migrate_use_xbzrle())
-        qemu_mutex_lock(&XBZRLE.lock);
-}
-
-static void XBZRLE_cache_unlock(void)
-{
-    if (migrate_use_xbzrle())
-        qemu_mutex_unlock(&XBZRLE.lock);
-}
-
-/*
- * called from qmp_migrate_set_cache_size in main thread, possibly while
- * a migration is in progress.
- * A running migration maybe using the cache and might finish during this
- * call, hence changes to the cache are protected by XBZRLE.lock().
- */
-int64_t xbzrle_cache_resize(int64_t new_size)
-{
-    PageCache *new_cache;
-    int64_t ret;
-
-    if (new_size < TARGET_PAGE_SIZE) {
-        return -1;
-    }
-
-    XBZRLE_cache_lock();
-
-    if (XBZRLE.cache != NULL) {
-        if (pow2floor(new_size) == migrate_xbzrle_cache_size()) {
-            goto out_new_size;
-        }
-        new_cache = cache_init(new_size / TARGET_PAGE_SIZE,
-                                        TARGET_PAGE_SIZE);
-        if (!new_cache) {
-            error_report("Error creating cache");
-            ret = -1;
-            goto out;
-        }
-
-        cache_fini(XBZRLE.cache);
-        XBZRLE.cache = new_cache;
-    }
-
-out_new_size:
-    ret = pow2floor(new_size);
-out:
-    XBZRLE_cache_unlock();
-    return ret;
-}
-
-/* accounting for migration statistics */
-typedef struct AccountingInfo {
-    uint64_t dup_pages;
-    uint64_t skipped_pages;
-    uint64_t norm_pages;
-    uint64_t iterations;
-    uint64_t xbzrle_bytes;
-    uint64_t xbzrle_pages;
-    uint64_t xbzrle_cache_miss;
-    double xbzrle_cache_miss_rate;
-    uint64_t xbzrle_overflows;
-} AccountingInfo;
-
-static AccountingInfo acct_info;
-
-static void acct_clear(void)
-{
-    memset(&acct_info, 0, sizeof(acct_info));
-}
-
-uint64_t dup_mig_bytes_transferred(void)
-{
-    return acct_info.dup_pages * TARGET_PAGE_SIZE;
-}
-
-uint64_t dup_mig_pages_transferred(void)
-{
-    return acct_info.dup_pages;
-}
-
-uint64_t skipped_mig_bytes_transferred(void)
-{
-    return acct_info.skipped_pages * TARGET_PAGE_SIZE;
-}
-
-uint64_t skipped_mig_pages_transferred(void)
-{
-    return acct_info.skipped_pages;
-}
-
-uint64_t norm_mig_bytes_transferred(void)
-{
-    return acct_info.norm_pages * TARGET_PAGE_SIZE;
-}
-
-uint64_t norm_mig_pages_transferred(void)
-{
-    return acct_info.norm_pages;
-}
-
-uint64_t xbzrle_mig_bytes_transferred(void)
-{
-    return acct_info.xbzrle_bytes;
-}
-
-uint64_t xbzrle_mig_pages_transferred(void)
-{
-    return acct_info.xbzrle_pages;
-}
-
-uint64_t xbzrle_mig_pages_cache_miss(void)
-{
-    return acct_info.xbzrle_cache_miss;
-}
-
-double xbzrle_mig_cache_miss_rate(void)
-{
-    return acct_info.xbzrle_cache_miss_rate;
-}
-
-uint64_t xbzrle_mig_pages_overflow(void)
-{
-    return acct_info.xbzrle_overflows;
-}
-
-static size_t save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
-                             int cont, int flag)
-{
-    size_t size;
-
-    qemu_put_be64(f, offset | cont | flag);
-    size = 8;
-
-    if (!cont) {
-        qemu_put_byte(f, strlen(block->idstr));
-        qemu_put_buffer(f, (uint8_t *)block->idstr,
-                        strlen(block->idstr));
-        size += 1 + strlen(block->idstr);
-    }
-    return size;
-}
-
-/* This is the last block that we have visited serching for dirty pages
- */
-static RAMBlock *last_seen_block;
-/* This is the last block from where we have sent data */
-static RAMBlock *last_sent_block;
-static ram_addr_t last_offset;
-static unsigned long *migration_bitmap;
-static uint64_t migration_dirty_pages;
-static uint32_t last_version;
-static bool ram_bulk_stage;
-
-/* Update the xbzrle cache to reflect a page that's been sent as all 0.
- * The important thing is that a stale (not-yet-0'd) page be replaced
- * by the new data.
- * As a bonus, if the page wasn't in the cache it gets added so that
- * when a small write is made into the 0'd page it gets XBZRLE sent
- */
-static void xbzrle_cache_zero_page(ram_addr_t current_addr)
-{
-    if (ram_bulk_stage || !migrate_use_xbzrle()) {
-        return;
-    }
-
-    /* We don't care if this fails to allocate a new cache page
-     * as long as it updated an old one */
-    cache_insert(XBZRLE.cache, current_addr, ZERO_TARGET_PAGE);
-}
-
-#define ENCODING_FLAG_XBZRLE 0x1
-
-static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data,
-                            ram_addr_t current_addr, RAMBlock *block,
-                            ram_addr_t offset, int cont, bool last_stage)
-{
-    int encoded_len = 0, bytes_sent = -1;
-    uint8_t *prev_cached_page;
-
-    if (!cache_is_cached(XBZRLE.cache, current_addr)) {
-        acct_info.xbzrle_cache_miss++;
-        if (!last_stage) {
-            if (cache_insert(XBZRLE.cache, current_addr, *current_data) == -1) {
-                return -1;
-            } else {
-                /* update *current_data when the page has been
-                   inserted into cache */
-                *current_data = get_cached_data(XBZRLE.cache, current_addr);
-            }
-        }
-        return -1;
-    }
-
-    prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
-
-    /* save current buffer into memory */
-    memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
-
-    /* XBZRLE encoding (if there is no overflow) */
-    encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
-                                       TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
-                                       TARGET_PAGE_SIZE);
-    if (encoded_len == 0) {
-        DPRINTF("Skipping unmodified page\n");
-        return 0;
-    } else if (encoded_len == -1) {
-        DPRINTF("Overflow\n");
-        acct_info.xbzrle_overflows++;
-        /* update data in the cache */
-        if (!last_stage) {
-            memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
-            *current_data = prev_cached_page;
-        }
-        return -1;
-    }
-
-    /* we need to update the data in the cache, in order to get the same data */
-    if (!last_stage) {
-        memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
-    }
-
-    /* Send XBZRLE based compressed page */
-    bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_XBZRLE);
-    qemu_put_byte(f, ENCODING_FLAG_XBZRLE);
-    qemu_put_be16(f, encoded_len);
-    qemu_put_buffer(f, XBZRLE.encoded_buf, encoded_len);
-    bytes_sent += encoded_len + 1 + 2;
-    acct_info.xbzrle_pages++;
-    acct_info.xbzrle_bytes += bytes_sent;
-
-    return bytes_sent;
-}
-
-static inline
-ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr,
-                                                 ram_addr_t start)
-{
-    unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS;
-    unsigned long nr = base + (start >> TARGET_PAGE_BITS);
-    uint64_t mr_size = TARGET_PAGE_ALIGN(memory_region_size(mr));
-    unsigned long size = base + (mr_size >> TARGET_PAGE_BITS);
-
-    unsigned long next;
-
-    if (ram_bulk_stage && nr > base) {
-        next = nr + 1;
-    } else {
-        next = find_next_bit(migration_bitmap, size, nr);
-    }
-
-    if (next < size) {
-        clear_bit(next, migration_bitmap);
-        migration_dirty_pages--;
-    }
-    return (next - base) << TARGET_PAGE_BITS;
-}
-
-static inline bool migration_bitmap_set_dirty(ram_addr_t addr)
-{
-    bool ret;
-    int nr = addr >> TARGET_PAGE_BITS;
-
-    ret = test_and_set_bit(nr, migration_bitmap);
-
-    if (!ret) {
-        migration_dirty_pages++;
-    }
-    return ret;
-}
-
-static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
-{
-    ram_addr_t addr;
-    unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
-
-    /* start address is aligned at the start of a word? */
-    if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
-        int k;
-        int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
-        unsigned long *src = ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION];
-
-        for (k = page; k < page + nr; k++) {
-            if (src[k]) {
-                unsigned long new_dirty;
-                new_dirty = ~migration_bitmap[k];
-                migration_bitmap[k] |= src[k];
-                new_dirty &= src[k];
-                migration_dirty_pages += ctpopl(new_dirty);
-                src[k] = 0;
-            }
-        }
-    } else {
-        for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
-            if (cpu_physical_memory_get_dirty(start + addr,
-                                              TARGET_PAGE_SIZE,
-                                              DIRTY_MEMORY_MIGRATION)) {
-                cpu_physical_memory_reset_dirty(start + addr,
-                                                TARGET_PAGE_SIZE,
-                                                DIRTY_MEMORY_MIGRATION);
-                migration_bitmap_set_dirty(start + addr);
-            }
-        }
-    }
-}
-
-
-/* Needs iothread lock! */
-/* Fix me: there are too many global variables used in migration process. */
-static int64_t start_time;
-static int64_t bytes_xfer_prev;
-static int64_t num_dirty_pages_period;
-
-static void migration_bitmap_sync_init(void)
-{
-    start_time = 0;
-    bytes_xfer_prev = 0;
-    num_dirty_pages_period = 0;
-}
-
-static void migration_bitmap_sync(void)
-{
-    RAMBlock *block;
-    uint64_t num_dirty_pages_init = migration_dirty_pages;
-    MigrationState *s = migrate_get_current();
-    int64_t end_time;
-    int64_t bytes_xfer_now;
-    static uint64_t xbzrle_cache_miss_prev;
-    static uint64_t iterations_prev;
-
-    bitmap_sync_count++;
-
-    if (!bytes_xfer_prev) {
-        bytes_xfer_prev = ram_bytes_transferred();
-    }
-
-    if (!start_time) {
-        start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-    }
-
-    trace_migration_bitmap_sync_start();
-    address_space_sync_dirty_bitmap(&address_space_memory);
-
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        migration_bitmap_sync_range(block->mr->ram_addr, block->length);
-    }
-    trace_migration_bitmap_sync_end(migration_dirty_pages
-                                    - num_dirty_pages_init);
-    num_dirty_pages_period += migration_dirty_pages - num_dirty_pages_init;
-    end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-
-    /* more than 1 second = 1000 millisecons */
-    if (end_time > start_time + 1000) {
-        if (migrate_auto_converge()) {
-            /* The following detection logic can be refined later. For now:
-               Check to see if the dirtied bytes is 50% more than the approx.
-               amount of bytes that just got transferred since the last time we
-               were in this routine. If that happens >N times (for now N==4)
-               we turn on the throttle down logic */
-            bytes_xfer_now = ram_bytes_transferred();
-            if (s->dirty_pages_rate &&
-               (num_dirty_pages_period * TARGET_PAGE_SIZE >
-                   (bytes_xfer_now - bytes_xfer_prev)/2) &&
-               (dirty_rate_high_cnt++ > 4)) {
-                    trace_migration_throttle();
-                    mig_throttle_on = true;
-                    dirty_rate_high_cnt = 0;
-             }
-             bytes_xfer_prev = bytes_xfer_now;
-        } else {
-             mig_throttle_on = false;
-        }
-        if (migrate_use_xbzrle()) {
-            if (iterations_prev != 0) {
-                acct_info.xbzrle_cache_miss_rate =
-                   (double)(acct_info.xbzrle_cache_miss -
-                            xbzrle_cache_miss_prev) /
-                   (acct_info.iterations - iterations_prev);
-            }
-            iterations_prev = acct_info.iterations;
-            xbzrle_cache_miss_prev = acct_info.xbzrle_cache_miss;
-        }
-        s->dirty_pages_rate = num_dirty_pages_period * 1000
-            / (end_time - start_time);
-        s->dirty_bytes_rate = s->dirty_pages_rate * TARGET_PAGE_SIZE;
-        start_time = end_time;
-        num_dirty_pages_period = 0;
-        s->dirty_sync_count = bitmap_sync_count;
-    }
-}
-
-/*
- * ram_save_page: Send the given page to the stream
- *
- * Returns: Number of bytes written.
- */
-static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset,
-                         bool last_stage)
-{
-    int bytes_sent;
-    int cont;
-    ram_addr_t current_addr;
-    MemoryRegion *mr = block->mr;
-    uint8_t *p;
-    int ret;
-    bool send_async = true;
-
-    cont = (block == last_sent_block) ? RAM_SAVE_FLAG_CONTINUE : 0;
-
-    p = memory_region_get_ram_ptr(mr) + offset;
-
-    /* In doubt sent page as normal */
-    bytes_sent = -1;
-    ret = ram_control_save_page(f, block->offset,
-                           offset, TARGET_PAGE_SIZE, &bytes_sent);
-
-    XBZRLE_cache_lock();
-
-    current_addr = block->offset + offset;
-    if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
-        if (ret != RAM_SAVE_CONTROL_DELAYED) {
-            if (bytes_sent > 0) {
-                acct_info.norm_pages++;
-            } else if (bytes_sent == 0) {
-                acct_info.dup_pages++;
-            }
-        }
-    } else if (is_zero_range(p, TARGET_PAGE_SIZE)) {
-        acct_info.dup_pages++;
-        bytes_sent = save_block_hdr(f, block, offset, cont,
-                                    RAM_SAVE_FLAG_COMPRESS);
-        qemu_put_byte(f, 0);
-        bytes_sent++;
-        /* Must let xbzrle know, otherwise a previous (now 0'd) cached
-         * page would be stale
-         */
-        xbzrle_cache_zero_page(current_addr);
-    } else if (!ram_bulk_stage && migrate_use_xbzrle()) {
-        bytes_sent = save_xbzrle_page(f, &p, current_addr, block,
-                                      offset, cont, last_stage);
-        if (!last_stage) {
-            /* Can't send this cached data async, since the cache page
-             * might get updated before it gets to the wire
-             */
-            send_async = false;
-        }
-    }
-
-    /* XBZRLE overflow or normal page */
-    if (bytes_sent == -1) {
-        bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE);
-        if (send_async) {
-            qemu_put_buffer_async(f, p, TARGET_PAGE_SIZE);
-        } else {
-            qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
-        }
-        bytes_sent += TARGET_PAGE_SIZE;
-        acct_info.norm_pages++;
-    }
-
-    XBZRLE_cache_unlock();
-
-    return bytes_sent;
-}
-
-/*
- * ram_find_and_save_block: Finds a page to send and sends it to f
- *
- * Returns:  The number of bytes written.
- *           0 means no dirty pages
- */
-
-static int ram_find_and_save_block(QEMUFile *f, bool last_stage)
-{
-    RAMBlock *block = last_seen_block;
-    ram_addr_t offset = last_offset;
-    bool complete_round = false;
-    int bytes_sent = 0;
-    MemoryRegion *mr;
-
-    if (!block)
-        block = QTAILQ_FIRST(&ram_list.blocks);
-
-    while (true) {
-        mr = block->mr;
-        offset = migration_bitmap_find_and_reset_dirty(mr, offset);
-        if (complete_round && block == last_seen_block &&
-            offset >= last_offset) {
-            break;
-        }
-        if (offset >= block->length) {
-            offset = 0;
-            block = QTAILQ_NEXT(block, next);
-            if (!block) {
-                block = QTAILQ_FIRST(&ram_list.blocks);
-                complete_round = true;
-                ram_bulk_stage = false;
-            }
-        } else {
-            bytes_sent = ram_save_page(f, block, offset, last_stage);
-
-            /* if page is unmodified, continue to the next */
-            if (bytes_sent > 0) {
-                last_sent_block = block;
-                break;
-            }
-        }
-    }
-    last_seen_block = block;
-    last_offset = offset;
-
-    return bytes_sent;
-}
-
-static uint64_t bytes_transferred;
-
-void acct_update_position(QEMUFile *f, size_t size, bool zero)
-{
-    uint64_t pages = size / TARGET_PAGE_SIZE;
-    if (zero) {
-        acct_info.dup_pages += pages;
-    } else {
-        acct_info.norm_pages += pages;
-        bytes_transferred += size;
-        qemu_update_position(f, size);
-    }
-}
-
-static ram_addr_t ram_save_remaining(void)
-{
-    return migration_dirty_pages;
-}
-
-uint64_t ram_bytes_remaining(void)
-{
-    return ram_save_remaining() * TARGET_PAGE_SIZE;
-}
-
-uint64_t ram_bytes_transferred(void)
-{
-    return bytes_transferred;
-}
-
-uint64_t ram_bytes_total(void)
-{
-    RAMBlock *block;
-    uint64_t total = 0;
-
-    QTAILQ_FOREACH(block, &ram_list.blocks, next)
-        total += block->length;
-
-    return total;
-}
-
-void free_xbzrle_decoded_buf(void)
-{
-    g_free(xbzrle_decoded_buf);
-    xbzrle_decoded_buf = NULL;
-}
-
-static void migration_end(void)
-{
-    if (migration_bitmap) {
-        memory_global_dirty_log_stop();
-        g_free(migration_bitmap);
-        migration_bitmap = NULL;
-    }
-
-    XBZRLE_cache_lock();
-    if (XBZRLE.cache) {
-        cache_fini(XBZRLE.cache);
-        g_free(XBZRLE.encoded_buf);
-        g_free(XBZRLE.current_buf);
-        XBZRLE.cache = NULL;
-        XBZRLE.encoded_buf = NULL;
-        XBZRLE.current_buf = NULL;
-    }
-    XBZRLE_cache_unlock();
-}
-
-static void ram_migration_cancel(void *opaque)
-{
-    migration_end();
-}
-
-static void reset_ram_globals(void)
-{
-    last_seen_block = NULL;
-    last_sent_block = NULL;
-    last_offset = 0;
-    last_version = ram_list.version;
-    ram_bulk_stage = true;
-}
-
-#define MAX_WAIT 50 /* ms, half buffered_file limit */
-
-static int ram_save_setup(QEMUFile *f, void *opaque)
-{
-    RAMBlock *block;
-    int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
-
-    mig_throttle_on = false;
-    dirty_rate_high_cnt = 0;
-    bitmap_sync_count = 0;
-    migration_bitmap_sync_init();
-
-    if (migrate_use_xbzrle()) {
-        XBZRLE_cache_lock();
-        XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
-                                  TARGET_PAGE_SIZE,
-                                  TARGET_PAGE_SIZE);
-        if (!XBZRLE.cache) {
-            XBZRLE_cache_unlock();
-            error_report("Error creating cache");
-            return -1;
-        }
-        XBZRLE_cache_unlock();
-
-        /* We prefer not to abort if there is no memory */
-        XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
-        if (!XBZRLE.encoded_buf) {
-            error_report("Error allocating encoded_buf");
-            return -1;
-        }
-
-        XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
-        if (!XBZRLE.current_buf) {
-            error_report("Error allocating current_buf");
-            g_free(XBZRLE.encoded_buf);
-            XBZRLE.encoded_buf = NULL;
-            return -1;
-        }
-
-        acct_clear();
-    }
-
-    qemu_mutex_lock_iothread();
-    qemu_mutex_lock_ramlist();
-    bytes_transferred = 0;
-    reset_ram_globals();
-
-    ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
-    migration_bitmap = bitmap_new(ram_bitmap_pages);
-    bitmap_set(migration_bitmap, 0, ram_bitmap_pages);
-
-    /*
-     * Count the total number of pages used by ram blocks not including any
-     * gaps due to alignment or unplugs.
-     */
-    migration_dirty_pages = 0;
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        uint64_t block_pages;
-
-        block_pages = block->length >> TARGET_PAGE_BITS;
-        migration_dirty_pages += block_pages;
-    }
-
-    memory_global_dirty_log_start();
-    migration_bitmap_sync();
-    qemu_mutex_unlock_iothread();
-
-    qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
-
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        qemu_put_byte(f, strlen(block->idstr));
-        qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
-        qemu_put_be64(f, block->length);
-    }
-
-    qemu_mutex_unlock_ramlist();
-
-    ram_control_before_iterate(f, RAM_CONTROL_SETUP);
-    ram_control_after_iterate(f, RAM_CONTROL_SETUP);
-
-    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
-
-    return 0;
-}
-
-static int ram_save_iterate(QEMUFile *f, void *opaque)
-{
-    int ret;
-    int i;
-    int64_t t0;
-    int total_sent = 0;
-
-    qemu_mutex_lock_ramlist();
-
-    if (ram_list.version != last_version) {
-        reset_ram_globals();
-    }
-
-    ram_control_before_iterate(f, RAM_CONTROL_ROUND);
-
-    t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-    i = 0;
-    while ((ret = qemu_file_rate_limit(f)) == 0) {
-        int bytes_sent;
-
-        bytes_sent = ram_find_and_save_block(f, false);
-        /* no more blocks to sent */
-        if (bytes_sent == 0) {
-            break;
-        }
-        total_sent += bytes_sent;
-        acct_info.iterations++;
-        check_guest_throttling();
-        /* we want to check in the 1st loop, just in case it was the 1st time
-           and we had to sync the dirty bitmap.
-           qemu_get_clock_ns() is a bit expensive, so we only check each some
-           iterations
-        */
-        if ((i & 63) == 0) {
-            uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
-            if (t1 > MAX_WAIT) {
-                DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
-                        t1, i);
-                break;
-            }
-        }
-        i++;
-    }
-
-    qemu_mutex_unlock_ramlist();
-
-    /*
-     * Must occur before EOS (or any QEMUFile operation)
-     * because of RDMA protocol.
-     */
-    ram_control_after_iterate(f, RAM_CONTROL_ROUND);
-
-    bytes_transferred += total_sent;
-
-    /*
-     * Do not count these 8 bytes into total_sent, so that we can
-     * return 0 if no page had been dirtied.
-     */
-    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
-    bytes_transferred += 8;
-
-    ret = qemu_file_get_error(f);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return total_sent;
-}
-
-static int ram_save_complete(QEMUFile *f, void *opaque)
-{
-    qemu_mutex_lock_ramlist();
-    migration_bitmap_sync();
-
-    ram_control_before_iterate(f, RAM_CONTROL_FINISH);
-
-    /* try transferring iterative blocks of memory */
-
-    /* flush all remaining blocks regardless of rate limiting */
-    while (true) {
-        int bytes_sent;
-
-        bytes_sent = ram_find_and_save_block(f, true);
-        /* no more blocks to sent */
-        if (bytes_sent == 0) {
-            break;
-        }
-        bytes_transferred += bytes_sent;
-    }
-
-    ram_control_after_iterate(f, RAM_CONTROL_FINISH);
-    migration_end();
-
-    qemu_mutex_unlock_ramlist();
-    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
-
-    return 0;
-}
-
-static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
-{
-    uint64_t remaining_size;
-
-    remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
-
-    if (remaining_size < max_size) {
-        qemu_mutex_lock_iothread();
-        migration_bitmap_sync();
-        qemu_mutex_unlock_iothread();
-        remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
-    }
-    return remaining_size;
-}
-
-static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
-{
-    unsigned int xh_len;
-    int xh_flags;
-
-    if (!xbzrle_decoded_buf) {
-        xbzrle_decoded_buf = g_malloc(TARGET_PAGE_SIZE);
-    }
-
-    /* extract RLE header */
-    xh_flags = qemu_get_byte(f);
-    xh_len = qemu_get_be16(f);
-
-    if (xh_flags != ENCODING_FLAG_XBZRLE) {
-        error_report("Failed to load XBZRLE page - wrong compression!");
-        return -1;
-    }
-
-    if (xh_len > TARGET_PAGE_SIZE) {
-        error_report("Failed to load XBZRLE page - len overflow!");
-        return -1;
-    }
-    /* load data and decode */
-    qemu_get_buffer(f, xbzrle_decoded_buf, xh_len);
-
-    /* decode RLE */
-    if (xbzrle_decode_buffer(xbzrle_decoded_buf, xh_len, host,
-                             TARGET_PAGE_SIZE) == -1) {
-        error_report("Failed to load XBZRLE page - decode error!");
-        return -1;
-    }
-
-    return 0;
-}
-
-static inline void *host_from_stream_offset(QEMUFile *f,
-                                            ram_addr_t offset,
-                                            int flags)
-{
-    static RAMBlock *block = NULL;
-    char id[256];
-    uint8_t len;
-
-    if (flags & RAM_SAVE_FLAG_CONTINUE) {
-        if (!block || block->length <= offset) {
-            error_report("Ack, bad migration stream!");
-            return NULL;
-        }
-
-        return memory_region_get_ram_ptr(block->mr) + offset;
-    }
-
-    len = qemu_get_byte(f);
-    qemu_get_buffer(f, (uint8_t *)id, len);
-    id[len] = 0;
-
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (!strncmp(id, block->idstr, sizeof(id)) && block->length > offset) {
-            return memory_region_get_ram_ptr(block->mr) + offset;
-        }
-    }
-
-    error_report("Can't find block %s!", id);
-    return NULL;
-}
-
-/*
- * If a page (or a whole RDMA chunk) has been
- * determined to be zero, then zap it.
- */
-void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
-{
-    if (ch != 0 || !is_zero_range(host, size)) {
-        memset(host, ch, size);
-    }
-}
-
-static int ram_load(QEMUFile *f, void *opaque, int version_id)
-{
-    int flags = 0, ret = 0;
-    static uint64_t seq_iter;
-
-    seq_iter++;
-
-    if (version_id != 4) {
-        ret = -EINVAL;
-    }
-
-    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
-        ram_addr_t addr, total_ram_bytes;
-        void *host;
-        uint8_t ch;
-
-        addr = qemu_get_be64(f);
-        flags = addr & ~TARGET_PAGE_MASK;
-        addr &= TARGET_PAGE_MASK;
-
-        switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
-        case RAM_SAVE_FLAG_MEM_SIZE:
-            /* Synchronize RAM block list */
-            total_ram_bytes = addr;
-            while (!ret && total_ram_bytes) {
-                RAMBlock *block;
-                uint8_t len;
-                char id[256];
-                ram_addr_t length;
-
-                len = qemu_get_byte(f);
-                qemu_get_buffer(f, (uint8_t *)id, len);
-                id[len] = 0;
-                length = qemu_get_be64(f);
-
-                QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-                    if (!strncmp(id, block->idstr, sizeof(id))) {
-                        if (block->length != length) {
-                            error_report("Length mismatch: %s: 0x" RAM_ADDR_FMT
-                                         " in != 0x" RAM_ADDR_FMT, id, length,
-                                         block->length);
-                            ret =  -EINVAL;
-                        }
-                        break;
-                    }
-                }
-
-                if (!block) {
-                    error_report("Unknown ramblock \"%s\", cannot "
-                                 "accept migration", id);
-                    ret = -EINVAL;
-                }
-
-                total_ram_bytes -= length;
-            }
-            break;
-        case RAM_SAVE_FLAG_COMPRESS:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
-
-            ch = qemu_get_byte(f);
-            ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
-            break;
-        case RAM_SAVE_FLAG_PAGE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
-
-            qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
-            break;
-        case RAM_SAVE_FLAG_XBZRLE:
-            host = host_from_stream_offset(f, addr, flags);
-            if (!host) {
-                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
-
-            if (load_xbzrle(f, addr, host) < 0) {
-                error_report("Failed to decompress XBZRLE page at "
-                             RAM_ADDR_FMT, addr);
-                ret = -EINVAL;
-                break;
-            }
-            break;
-        case RAM_SAVE_FLAG_EOS:
-            /* normal exit */
-            break;
-        default:
-            if (flags & RAM_SAVE_FLAG_HOOK) {
-                ram_control_load_hook(f, flags);
-            } else {
-                error_report("Unknown combination of migration flags: %#x",
-                             flags);
-                ret = -EINVAL;
-            }
-        }
-        if (!ret) {
-            ret = qemu_file_get_error(f);
-        }
-    }
-
-    DPRINTF("Completed load of VM with exit code %d seq iteration "
-            "%" PRIu64 "\n", ret, seq_iter);
-    return ret;
-}
-
-static SaveVMHandlers savevm_ram_handlers = {
-    .save_live_setup = ram_save_setup,
-    .save_live_iterate = ram_save_iterate,
-    .save_live_complete = ram_save_complete,
-    .save_live_pending = ram_save_pending,
-    .load_state = ram_load,
-    .cancel = ram_migration_cancel,
-};
-
-void ram_mig_init(void)
-{
-    qemu_mutex_init(&XBZRLE.lock);
-    register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
-}
-
 struct soundhw {
     const char *name;
     const char *descr;
@@ -1329,9 +263,7 @@
 
     acpi_table_add(opts, &err);
     if (err) {
-        error_report("Wrong acpi table provided: %s",
-                     error_get_pretty(err));
-        error_free(err);
+        error_reportf_err(err, "Wrong acpi table provided: ");
         exit(1);
     }
 #endif
@@ -1344,13 +276,6 @@
 #endif
 }
 
-void cpudef_init(void)
-{
-#if defined(cpudef_setup)
-    cpudef_setup(); /* parse cpu definitions in target config file */
-#endif
-}
-
 int kvm_available(void)
 {
 #ifdef CONFIG_KVM
@@ -1386,52 +311,3 @@
 
     return info;
 }
-
-/* Stub function that's gets run on the vcpu when its brought out of the
-   VM to run inside qemu via async_run_on_cpu()*/
-static void mig_sleep_cpu(void *opq)
-{
-    qemu_mutex_unlock_iothread();
-    g_usleep(30*1000);
-    qemu_mutex_lock_iothread();
-}
-
-/* To reduce the dirty rate explicitly disallow the VCPUs from spending
-   much time in the VM. The migration thread will try to catchup.
-   Workload will experience a performance drop.
-*/
-static void mig_throttle_guest_down(void)
-{
-    CPUState *cpu;
-
-    qemu_mutex_lock_iothread();
-    CPU_FOREACH(cpu) {
-        async_run_on_cpu(cpu, mig_sleep_cpu, NULL);
-    }
-    qemu_mutex_unlock_iothread();
-}
-
-static void check_guest_throttling(void)
-{
-    static int64_t t0;
-    int64_t        t1;
-
-    if (!mig_throttle_on) {
-        return;
-    }
-
-    if (!t0)  {
-        t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        return;
-    }
-
-    t1 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-
-    /* If it has been more than 40 ms since the last time the guest
-     * was throttled then do it again.
-     */
-    if (40 < (t1-t0)/1000000) {
-        mig_throttle_guest_down();
-        t0 = t1;
-    }
-}
diff --git a/async.c b/async.c
index 6e1b282..3bca9b0 100644
--- a/async.c
+++ b/async.c
@@ -22,11 +22,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/aio.h"
 #include "block/thread-pool.h"
 #include "qemu/main-loop.h"
 #include "qemu/atomic.h"
+#include "block/raw-aio.h"
 
 /***********************************************************/
 /* bottom halves (can be seen as timers which expire ASAP) */
@@ -44,10 +47,12 @@
 QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque)
 {
     QEMUBH *bh;
-    bh = g_malloc0(sizeof(QEMUBH));
-    bh->ctx = ctx;
-    bh->cb = cb;
-    bh->opaque = opaque;
+    bh = g_new(QEMUBH, 1);
+    *bh = (QEMUBH){
+        .ctx = ctx,
+        .cb = cb,
+        .opaque = opaque,
+    };
     qemu_mutex_lock(&ctx->bh_lock);
     bh->next = ctx->first_bh;
     /* Make sure that the members are ready before putting bh into list */
@@ -57,6 +62,11 @@
     return bh;
 }
 
+void aio_bh_call(QEMUBH *bh)
+{
+    bh->cb(bh->opaque);
+}
+
 /* Multiple occurrences of aio_bh_poll cannot be called concurrently */
 int aio_bh_poll(AioContext *ctx)
 {
@@ -70,16 +80,19 @@
         /* Make sure that fetching bh happens before accessing its members */
         smp_read_barrier_depends();
         next = bh->next;
-        if (!bh->deleted && bh->scheduled) {
-            bh->scheduled = 0;
-            /* Paired with write barrier in bh schedule to ensure reading for
-             * idle & callbacks coming after bh's scheduling.
-             */
-            smp_rmb();
-            if (!bh->idle)
+        /* The atomic_xchg is paired with the one in qemu_bh_schedule.  The
+         * implicit memory barrier ensures that the callback sees all writes
+         * done by the scheduling thread.  It also ensures that the scheduling
+         * thread sees the zero before bh->cb has run, and thus will call
+         * aio_notify again if necessary.
+         */
+        if (!bh->deleted && atomic_xchg(&bh->scheduled, 0)) {
+            /* Idle BHs and the notify BH don't count as progress */
+            if (!bh->idle && bh != ctx->notify_dummy_bh) {
                 ret = 1;
+            }
             bh->idle = 0;
-            bh->cb(bh->opaque);
+            aio_bh_call(bh);
         }
     }
 
@@ -106,33 +119,28 @@
 
 void qemu_bh_schedule_idle(QEMUBH *bh)
 {
-    if (bh->scheduled)
-        return;
     bh->idle = 1;
     /* Make sure that idle & any writes needed by the callback are done
      * before the locations are read in the aio_bh_poll.
      */
-    smp_wmb();
-    bh->scheduled = 1;
+    atomic_mb_set(&bh->scheduled, 1);
 }
 
 void qemu_bh_schedule(QEMUBH *bh)
 {
     AioContext *ctx;
 
-    if (bh->scheduled)
-        return;
     ctx = bh->ctx;
     bh->idle = 0;
-    /* Make sure that:
+    /* The memory barrier implicit in atomic_xchg makes sure that:
      * 1. idle & any writes needed by the callback are done before the
      *    locations are read in the aio_bh_poll.
      * 2. ctx is loaded before scheduled is set and the callback has a chance
      *    to execute.
      */
-    smp_mb();
-    bh->scheduled = 1;
-    aio_notify(ctx);
+    if (atomic_xchg(&bh->scheduled, 1) == 0) {
+        aio_notify(ctx);
+    }
 }
 
 
@@ -186,6 +194,8 @@
 {
     AioContext *ctx = (AioContext *) source;
 
+    atomic_or(&ctx->notify_me, 1);
+
     /* We assume there is no timeout already supplied */
     *timeout = qemu_timeout_ns_to_ms(aio_compute_timeout(ctx));
 
@@ -202,10 +212,13 @@
     AioContext *ctx = (AioContext *) source;
     QEMUBH *bh;
 
+    atomic_and(&ctx->notify_me, ~1);
+    aio_notify_accept(ctx);
+
     for (bh = ctx->first_bh; bh; bh = bh->next) {
         if (!bh->deleted && bh->scheduled) {
             return true;
-	}
+        }
     }
     return aio_pending(ctx) || (timerlistgroup_deadline_ns(&ctx->tlg) == 0);
 }
@@ -227,12 +240,33 @@
 {
     AioContext *ctx = (AioContext *) source;
 
+    qemu_bh_delete(ctx->notify_dummy_bh);
     thread_pool_free(ctx->thread_pool);
-    aio_set_event_notifier(ctx, &ctx->notifier, NULL);
+
+#ifdef CONFIG_LINUX_AIO
+    if (ctx->linux_aio) {
+        laio_detach_aio_context(ctx->linux_aio, ctx);
+        laio_cleanup(ctx->linux_aio);
+        ctx->linux_aio = NULL;
+    }
+#endif
+
+    qemu_mutex_lock(&ctx->bh_lock);
+    while (ctx->first_bh) {
+        QEMUBH *next = ctx->first_bh->next;
+
+        /* qemu_bh_delete() must have been called on BHs in this AioContext */
+        assert(ctx->first_bh->deleted);
+
+        g_free(ctx->first_bh);
+        ctx->first_bh = next;
+    }
+    qemu_mutex_unlock(&ctx->bh_lock);
+
+    aio_set_event_notifier(ctx, &ctx->notifier, false, NULL);
     event_notifier_cleanup(&ctx->notifier);
     rfifolock_destroy(&ctx->lock);
     qemu_mutex_destroy(&ctx->bh_lock);
-    g_array_free(ctx->pollfds, TRUE);
     timerlistgroup_deinit(&ctx->tlg);
 }
 
@@ -257,24 +291,33 @@
     return ctx->thread_pool;
 }
 
-void aio_set_dispatching(AioContext *ctx, bool dispatching)
+#ifdef CONFIG_LINUX_AIO
+LinuxAioState *aio_get_linux_aio(AioContext *ctx)
 {
-    ctx->dispatching = dispatching;
-    if (!dispatching) {
-        /* Write ctx->dispatching before reading e.g. bh->scheduled.
-         * Optimization: this is only needed when we're entering the "unsafe"
-         * phase where other threads must call event_notifier_set.
-         */
-        smp_mb();
+    if (!ctx->linux_aio) {
+        ctx->linux_aio = laio_init();
+        laio_attach_aio_context(ctx->linux_aio, ctx);
     }
+    return ctx->linux_aio;
 }
+#endif
 
 void aio_notify(AioContext *ctx)
 {
-    /* Write e.g. bh->scheduled before reading ctx->dispatching.  */
+    /* Write e.g. bh->scheduled before reading ctx->notify_me.  Pairs
+     * with atomic_or in aio_ctx_prepare or atomic_add in aio_poll.
+     */
     smp_mb();
-    if (!ctx->dispatching) {
+    if (ctx->notify_me) {
         event_notifier_set(&ctx->notifier);
+        atomic_mb_set(&ctx->notified, true);
+    }
+}
+
+void aio_notify_accept(AioContext *ctx)
+{
+    if (atomic_xchg(&ctx->notified, false)) {
+        event_notifier_test_and_clear(&ctx->notifier);
     }
 }
 
@@ -285,31 +328,53 @@
 
 static void aio_rfifolock_cb(void *opaque)
 {
+    AioContext *ctx = opaque;
+
     /* Kick owner thread in case they are blocked in aio_poll() */
-    aio_notify(opaque);
+    qemu_bh_schedule(ctx->notify_dummy_bh);
+}
+
+static void notify_dummy_bh(void *opaque)
+{
+    /* Do nothing, we were invoked just to force the event loop to iterate */
+}
+
+static void event_notifier_dummy_cb(EventNotifier *e)
+{
 }
 
 AioContext *aio_context_new(Error **errp)
 {
     int ret;
     AioContext *ctx;
+
     ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
+    aio_context_setup(ctx);
+
     ret = event_notifier_init(&ctx->notifier, false);
     if (ret < 0) {
-        g_source_destroy(&ctx->source);
         error_setg_errno(errp, -ret, "Failed to initialize event notifier");
-        return NULL;
+        goto fail;
     }
+    g_source_set_can_recurse(&ctx->source, true);
     aio_set_event_notifier(ctx, &ctx->notifier,
+                           false,
                            (EventNotifierHandler *)
-                           event_notifier_test_and_clear);
-    ctx->pollfds = g_array_new(FALSE, FALSE, sizeof(GPollFD));
+                           event_notifier_dummy_cb);
+#ifdef CONFIG_LINUX_AIO
+    ctx->linux_aio = NULL;
+#endif
     ctx->thread_pool = NULL;
     qemu_mutex_init(&ctx->bh_lock);
     rfifolock_init(&ctx->lock, aio_rfifolock_cb, ctx);
     timerlistgroup_init(&ctx->tlg, aio_timerlist_notify, ctx);
 
+    ctx->notify_dummy_bh = aio_bh_new(ctx, notify_dummy_bh, NULL);
+
     return ctx;
+fail:
+    g_source_destroy(&ctx->source);
+    return NULL;
 }
 
 void aio_context_ref(AioContext *ctx)
diff --git a/audio/Makefile.objs b/audio/Makefile.objs
index b495cb4..fb8e7a3 100644
--- a/audio/Makefile.objs
+++ b/audio/Makefile.objs
@@ -5,14 +5,10 @@
 common-obj-$(CONFIG_COREAUDIO) += coreaudio.o
 common-obj-$(CONFIG_ALSA) += alsaaudio.o
 common-obj-$(CONFIG_DSOUND) += dsoundaudio.o
-common-obj-$(CONFIG_FMOD) += fmodaudio.o
-common-obj-$(CONFIG_ESD) += esdaudio.o
 common-obj-$(CONFIG_PA) += paaudio.o
-common-obj-$(CONFIG_WINWAVE) += winwaveaudio.o
-common-obj-$(CONFIG_WINAUDIO) += winaudio.o
 common-obj-$(CONFIG_AUDIO_PT_INT) += audio_pt_int.o
 common-obj-$(CONFIG_AUDIO_WIN_INT) += audio_win_int.o
 common-obj-y += wavcapture.o
+common-obj-$(CONFIG_WINAUDIO) += winaudio.o
 
-$(obj)/audio.o $(obj)/fmodaudio.o: QEMU_CFLAGS += $(FMOD_CFLAGS)
 sdlaudio.o-cflags := $(SDL_CFLAGS)
diff --git a/audio/alsaaudio.c b/audio/alsaaudio.c
index 74ead97..3652a7b 100644
--- a/audio/alsaaudio.c
+++ b/audio/alsaaudio.c
@@ -21,10 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <alsa/asoundlib.h>
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 #include "audio.h"
+#include "trace.h"
 
 #if QEMU_GNUC_PREREQ(4, 3)
 #pragma GCC diagnostic ignored "-Waddress"
@@ -33,9 +35,28 @@
 #define AUDIO_CAP "alsa"
 #include "audio_int.h"
 
+typedef struct ALSAConf {
+    int size_in_usec_in;
+    int size_in_usec_out;
+    const char *pcm_name_in;
+    const char *pcm_name_out;
+    unsigned int buffer_size_in;
+    unsigned int period_size_in;
+    unsigned int buffer_size_out;
+    unsigned int period_size_out;
+    unsigned int threshold;
+
+    int buffer_size_in_overridden;
+    int period_size_in_overridden;
+
+    int buffer_size_out_overridden;
+    int period_size_out_overridden;
+} ALSAConf;
+
 struct pollhlp {
     snd_pcm_t *handle;
     struct pollfd *pfds;
+    ALSAConf *conf;
     int count;
     int mask;
 };
@@ -56,30 +77,6 @@
     struct pollhlp pollhlp;
 } ALSAVoiceIn;
 
-static struct {
-    int size_in_usec_in;
-    int size_in_usec_out;
-    const char *pcm_name_in;
-    const char *pcm_name_out;
-    unsigned int buffer_size_in;
-    unsigned int period_size_in;
-    unsigned int buffer_size_out;
-    unsigned int period_size_out;
-    unsigned int threshold;
-
-    int buffer_size_in_overridden;
-    int period_size_in_overridden;
-
-    int buffer_size_out_overridden;
-    int period_size_out_overridden;
-    int verbose;
-} conf = {
-    .buffer_size_out = 4096,
-    .period_size_out = 1024,
-    .pcm_name_out = "default",
-    .pcm_name_in = "default",
-};
-
 struct alsa_params_req {
     int freq;
     snd_pcm_format_t fmt;
@@ -205,9 +202,7 @@
     }
 
     if (!(revents & hlp->mask)) {
-        if (conf.verbose) {
-            dolog ("revents = %d\n", revents);
-        }
+        trace_alsa_revents(revents);
         return;
     }
 
@@ -266,31 +261,14 @@
 
     for (i = 0; i < count; ++i) {
         if (pfds[i].events & POLLIN) {
-            err = qemu_set_fd_handler (pfds[i].fd, alsa_poll_handler,
-                                       NULL, hlp);
+            qemu_set_fd_handler (pfds[i].fd, alsa_poll_handler, NULL, hlp);
         }
         if (pfds[i].events & POLLOUT) {
-            if (conf.verbose) {
-                dolog ("POLLOUT %d %d\n", i, pfds[i].fd);
-            }
-            err = qemu_set_fd_handler (pfds[i].fd, NULL,
-                                       alsa_poll_handler, hlp);
+            trace_alsa_pollout(i, pfds[i].fd);
+            qemu_set_fd_handler (pfds[i].fd, NULL, alsa_poll_handler, hlp);
         }
-        if (conf.verbose) {
-            dolog ("Set handler events=%#x index=%d fd=%d err=%d\n",
-                   pfds[i].events, i, pfds[i].fd, err);
-        }
+        trace_alsa_set_handler(pfds[i].events, i, pfds[i].fd, err);
 
-        if (err) {
-            dolog ("Failed to set handler events=%#x index=%d fd=%d err=%d\n",
-                   pfds[i].events, i, pfds[i].fd, err);
-
-            while (i--) {
-                qemu_set_fd_handler (pfds[i].fd, NULL, NULL, NULL);
-            }
-            g_free (pfds);
-            return -1;
-        }
     }
     hlp->pfds = pfds;
     hlp->count = count;
@@ -476,14 +454,15 @@
 }
 
 static int alsa_open (int in, struct alsa_params_req *req,
-                      struct alsa_params_obt *obt, snd_pcm_t **handlep)
+                      struct alsa_params_obt *obt, snd_pcm_t **handlep,
+                      ALSAConf *conf)
 {
     snd_pcm_t *handle;
     snd_pcm_hw_params_t *hw_params;
     int err;
     int size_in_usec;
     unsigned int freq, nchannels;
-    const char *pcm_name = in ? conf.pcm_name_in : conf.pcm_name_out;
+    const char *pcm_name = in ? conf->pcm_name_in : conf->pcm_name_out;
     snd_pcm_uframes_t obt_buffer_size;
     const char *typ = in ? "ADC" : "DAC";
     snd_pcm_format_t obtfmt;
@@ -522,7 +501,7 @@
     }
 
     err = snd_pcm_hw_params_set_format (handle, hw_params, req->fmt);
-    if (err < 0 && conf.verbose) {
+    if (err < 0) {
         alsa_logerr2 (err, typ, "Failed to set format %d\n", req->fmt);
     }
 
@@ -654,7 +633,7 @@
         goto err;
     }
 
-    if (!in && conf.threshold) {
+    if (!in && conf->threshold) {
         snd_pcm_uframes_t threshold;
         int bytes_per_sec;
 
@@ -676,7 +655,7 @@
             break;
         }
 
-        threshold = (conf.threshold * bytes_per_sec) / 1000;
+        threshold = (conf->threshold * bytes_per_sec) / 1000;
         alsa_set_threshold (handle, threshold);
     }
 
@@ -686,10 +665,9 @@
 
     *handlep = handle;
 
-    if (conf.verbose &&
-        (obtfmt != req->fmt ||
+    if (obtfmt != req->fmt ||
          obt->nchannels != req->nchannels ||
-         obt->freq != req->freq)) {
+         obt->freq != req->freq) {
         dolog ("Audio parameters for %s\n", typ);
         alsa_dump_info (req, obt, obtfmt);
     }
@@ -743,9 +721,7 @@
             if (written <= 0) {
                 switch (written) {
                 case 0:
-                    if (conf.verbose) {
-                        dolog ("Failed to write %d frames (wrote zero)\n", len);
-                    }
+                    trace_alsa_wrote_zero(len);
                     return;
 
                 case -EPIPE:
@@ -754,9 +730,7 @@
                                      len);
                         return;
                     }
-                    if (conf.verbose) {
-                        dolog ("Recovering from playback xrun\n");
-                    }
+                    trace_alsa_xrun_out();
                     continue;
 
                 case -ESTRPIPE:
@@ -767,9 +741,7 @@
                                      len);
                         return;
                     }
-                    if (conf.verbose) {
-                        dolog ("Resuming suspended output stream\n");
-                    }
+                    trace_alsa_resume_out();
                     continue;
 
                 case -EAGAIN:
@@ -819,25 +791,27 @@
     alsa->pcm_buf = NULL;
 }
 
-static int alsa_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int alsa_init_out(HWVoiceOut *hw, struct audsettings *as,
+                         void *drv_opaque)
 {
     ALSAVoiceOut *alsa = (ALSAVoiceOut *) hw;
     struct alsa_params_req req;
     struct alsa_params_obt obt;
     snd_pcm_t *handle;
     struct audsettings obt_as;
+    ALSAConf *conf = drv_opaque;
 
     req.fmt = aud_to_alsafmt (as->fmt, as->endianness);
     req.freq = as->freq;
     req.nchannels = as->nchannels;
-    req.period_size = conf.period_size_out;
-    req.buffer_size = conf.buffer_size_out;
-    req.size_in_usec = conf.size_in_usec_out;
+    req.period_size = conf->period_size_out;
+    req.buffer_size = conf->buffer_size_out;
+    req.size_in_usec = conf->size_in_usec_out;
     req.override_mask =
-        (conf.period_size_out_overridden ? 1 : 0) |
-        (conf.buffer_size_out_overridden ? 2 : 0);
+        (conf->period_size_out_overridden ? 1 : 0) |
+        (conf->buffer_size_out_overridden ? 2 : 0);
 
-    if (alsa_open (0, &req, &obt, &handle)) {
+    if (alsa_open (0, &req, &obt, &handle, conf)) {
         return -1;
     }
 
@@ -858,6 +832,7 @@
     }
 
     alsa->handle = handle;
+    alsa->pollhlp.conf = conf;
     return 0;
 }
 
@@ -928,25 +903,26 @@
     return -1;
 }
 
-static int alsa_init_in (HWVoiceIn *hw, struct audsettings *as)
+static int alsa_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
 {
     ALSAVoiceIn *alsa = (ALSAVoiceIn *) hw;
     struct alsa_params_req req;
     struct alsa_params_obt obt;
     snd_pcm_t *handle;
     struct audsettings obt_as;
+    ALSAConf *conf = drv_opaque;
 
     req.fmt = aud_to_alsafmt (as->fmt, as->endianness);
     req.freq = as->freq;
     req.nchannels = as->nchannels;
-    req.period_size = conf.period_size_in;
-    req.buffer_size = conf.buffer_size_in;
-    req.size_in_usec = conf.size_in_usec_in;
+    req.period_size = conf->period_size_in;
+    req.buffer_size = conf->buffer_size_in;
+    req.size_in_usec = conf->size_in_usec_in;
     req.override_mask =
-        (conf.period_size_in_overridden ? 1 : 0) |
-        (conf.buffer_size_in_overridden ? 2 : 0);
+        (conf->period_size_in_overridden ? 1 : 0) |
+        (conf->buffer_size_in_overridden ? 2 : 0);
 
-    if (alsa_open (1, &req, &obt, &handle)) {
+    if (alsa_open (1, &req, &obt, &handle, conf)) {
         return -1;
     }
 
@@ -967,6 +943,7 @@
     }
 
     alsa->handle = handle;
+    alsa->pollhlp.conf = conf;
     return 0;
 }
 
@@ -1022,14 +999,10 @@
                 dolog ("Failed to resume suspended input stream\n");
                 return 0;
             }
-            if (conf.verbose) {
-                dolog ("Resuming suspended input stream\n");
-            }
+            trace_alsa_resume_in();
             break;
         default:
-            if (conf.verbose) {
-                dolog ("No frames available and ALSA state is %d\n", state);
-            }
+            trace_alsa_no_frames(state);
             return 0;
         }
     }
@@ -1064,9 +1037,7 @@
             if (nread <= 0) {
                 switch (nread) {
                 case 0:
-                    if (conf.verbose) {
-                        dolog ("Failed to read %ld frames (read zero)\n", len);
-                    }
+                    trace_alsa_read_zero(len);
                     goto exit;
 
                 case -EPIPE:
@@ -1074,9 +1045,7 @@
                         alsa_logerr (nread, "Failed to read %ld frames\n", len);
                         goto exit;
                     }
-                    if (conf.verbose) {
-                        dolog ("Recovering from capture xrun\n");
-                    }
+                    trace_alsa_xrun_in();
                     continue;
 
                 case -EAGAIN:
@@ -1148,82 +1117,85 @@
     return -1;
 }
 
+static ALSAConf glob_conf = {
+    .buffer_size_out = 4096,
+    .period_size_out = 1024,
+    .pcm_name_out = "default",
+    .pcm_name_in = "default",
+};
+
 static void *alsa_audio_init (void)
 {
-    return &conf;
+    ALSAConf *conf = g_malloc(sizeof(ALSAConf));
+    *conf = glob_conf;
+    return conf;
 }
 
 static void alsa_audio_fini (void *opaque)
 {
-    (void) opaque;
+    g_free(opaque);
 }
 
 static struct audio_option alsa_options[] = {
     {
         .name        = "DAC_SIZE_IN_USEC",
         .tag         = AUD_OPT_BOOL,
-        .valp        = &conf.size_in_usec_out,
+        .valp        = &glob_conf.size_in_usec_out,
         .descr       = "DAC period/buffer size in microseconds (otherwise in frames)"
     },
     {
         .name        = "DAC_PERIOD_SIZE",
         .tag         = AUD_OPT_INT,
-        .valp        = &conf.period_size_out,
+        .valp        = &glob_conf.period_size_out,
         .descr       = "DAC period size (0 to go with system default)",
-        .overriddenp = &conf.period_size_out_overridden
+        .overriddenp = &glob_conf.period_size_out_overridden
     },
     {
         .name        = "DAC_BUFFER_SIZE",
         .tag         = AUD_OPT_INT,
-        .valp        = &conf.buffer_size_out,
+        .valp        = &glob_conf.buffer_size_out,
         .descr       = "DAC buffer size (0 to go with system default)",
-        .overriddenp = &conf.buffer_size_out_overridden
+        .overriddenp = &glob_conf.buffer_size_out_overridden
     },
     {
         .name        = "ADC_SIZE_IN_USEC",
         .tag         = AUD_OPT_BOOL,
-        .valp        = &conf.size_in_usec_in,
+        .valp        = &glob_conf.size_in_usec_in,
         .descr       =
         "ADC period/buffer size in microseconds (otherwise in frames)"
     },
     {
         .name        = "ADC_PERIOD_SIZE",
         .tag         = AUD_OPT_INT,
-        .valp        = &conf.period_size_in,
+        .valp        = &glob_conf.period_size_in,
         .descr       = "ADC period size (0 to go with system default)",
-        .overriddenp = &conf.period_size_in_overridden
+        .overriddenp = &glob_conf.period_size_in_overridden
     },
     {
         .name        = "ADC_BUFFER_SIZE",
         .tag         = AUD_OPT_INT,
-        .valp        = &conf.buffer_size_in,
+        .valp        = &glob_conf.buffer_size_in,
         .descr       = "ADC buffer size (0 to go with system default)",
-        .overriddenp = &conf.buffer_size_in_overridden
+        .overriddenp = &glob_conf.buffer_size_in_overridden
     },
     {
         .name        = "THRESHOLD",
         .tag         = AUD_OPT_INT,
-        .valp        = &conf.threshold,
+        .valp        = &glob_conf.threshold,
         .descr       = "(undocumented)"
     },
     {
         .name        = "DAC_DEV",
         .tag         = AUD_OPT_STR,
-        .valp        = &conf.pcm_name_out,
+        .valp        = &glob_conf.pcm_name_out,
         .descr       = "DAC device name (for instance dmix)"
     },
     {
         .name        = "ADC_DEV",
         .tag         = AUD_OPT_STR,
-        .valp        = &conf.pcm_name_in,
+        .valp        = &glob_conf.pcm_name_in,
         .descr       = "ADC device name"
     },
-    {
-        .name        = "VERBOSE",
-        .tag         = AUD_OPT_BOOL,
-        .valp        = &conf.verbose,
-        .descr       = "Behave in a more verbose way"
-    },
     { /* End of list */ }
 };
 
diff --git a/audio/audio.c b/audio/audio.c
index 9d018e9..c845a44 100644
--- a/audio/audio.c
+++ b/audio/audio.c
@@ -21,16 +21,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio.h"
 #include "monitor/monitor.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
 #define AUDIO_CAP "audio"
 #include "audio_int.h"
 
-/* #define DEBUG_PLIVE */
 /* #define DEBUG_LIVE */
 /* #define DEBUG_OUT */
 /* #define DEBUG_CAPTURE */
@@ -66,8 +67,6 @@
         int hertz;
         int64_t ticks;
     } period;
-    int plive;
-    int log_to_monitor;
     int try_poll_in;
     int try_poll_out;
 } conf = {
@@ -96,8 +95,6 @@
     },
 
     .period = { .hertz = 100 },
-    .plive = 0,
-    .log_to_monitor = 0,
     .try_poll_in = 1,
     .try_poll_out = 1,
 };
@@ -331,20 +328,11 @@
 
 void AUD_vlog (const char *cap, const char *fmt, va_list ap)
 {
-    if (conf.log_to_monitor) {
-        if (cap) {
-            monitor_printf(default_mon, "%s: ", cap);
-        }
-
-        monitor_vprintf(default_mon, fmt, ap);
+    if (cap) {
+        fprintf(stderr, "%s: ", cap);
     }
-    else {
-        if (cap) {
-            fprintf (stderr, "%s: ", cap);
-        }
 
-        vfprintf (stderr, fmt, ap);
-    }
+    vfprintf(stderr, fmt, ap);
 }
 
 void AUD_log (const char *cap, const char *fmt, ...)
@@ -1143,8 +1131,6 @@
  */
 int AUD_write (SWVoiceOut *sw, void *buf, int size)
 {
-    int bytes;
-
     if (!sw) {
         /* XXX: Consider options */
         return size;
@@ -1155,14 +1141,11 @@
         return 0;
     }
 
-    bytes = sw->hw->pcm_ops->write (sw, buf, size);
-    return bytes;
+    return sw->hw->pcm_ops->write(sw, buf, size);
 }
 
 int AUD_read (SWVoiceIn *sw, void *buf, int size)
 {
-    int bytes;
-
     if (!sw) {
         /* XXX: Consider options */
         return size;
@@ -1173,8 +1156,7 @@
         return 0;
     }
 
-    bytes = sw->hw->pcm_ops->read (sw, buf, size);
-    return bytes;
+    return sw->hw->pcm_ops->read(sw, buf, size);
 }
 
 int AUD_get_buffer_size_out (SWVoiceOut *sw)
@@ -1454,9 +1436,6 @@
             while (sw) {
                 sw1 = sw->entries.le_next;
                 if (!sw->active && !sw->callback.fn) {
-#ifdef DEBUG_PLIVE
-                    dolog ("Finishing with old voice\n");
-#endif
                     audio_close_out (sw);
                 }
                 sw = sw1;
@@ -1648,18 +1627,6 @@
         .valp  = &conf.period.hertz,
         .descr = "Timer period in HZ (0 - use lowest possible)"
     },
-    {
-        .name  = "PLIVE",
-        .tag   = AUD_OPT_BOOL,
-        .valp  = &conf.plive,
-        .descr = "(undocumented)"
-    },
-    {
-        .name  = "LOG_TO_MONITOR",
-        .tag   = AUD_OPT_BOOL,
-        .valp  = &conf.log_to_monitor,
-        .descr = "Print logging messages to monitor instead of stderr"
-    },
     { /* End of list */ }
 };
 
@@ -1772,13 +1739,21 @@
     audio_reset_timer (s);
 }
 
-static void audio_atexit (void)
+static bool is_cleaning_up;
+
+bool audio_is_cleaning_up(void)
+{
+    return is_cleaning_up;
+}
+
+void audio_cleanup(void)
 {
     AudioState *s = &glob_audio_state;
-    HWVoiceOut *hwo = NULL;
-    HWVoiceIn *hwi = NULL;
+    HWVoiceOut *hwo, *hwon;
+    HWVoiceIn *hwi, *hwin;
 
-    while ((hwo = audio_pcm_hw_find_any_out (hwo))) {
+    is_cleaning_up = true;
+    QLIST_FOREACH_SAFE(hwo, &glob_audio_state.hw_head_out, entries, hwon) {
         SWVoiceCap *sc;
 
         if (hwo->enabled) {
@@ -1794,17 +1769,20 @@
                 cb->ops.destroy (cb->opaque);
             }
         }
+        QLIST_REMOVE(hwo, entries);
     }
 
-    while ((hwi = audio_pcm_hw_find_any_in (hwi))) {
+    QLIST_FOREACH_SAFE(hwi, &glob_audio_state.hw_head_in, entries, hwin) {
         if (hwi->enabled) {
             hwi->pcm_ops->ctl_in (hwi, VOICE_DISABLE);
         }
         hwi->pcm_ops->fini_in (hwi);
+        QLIST_REMOVE(hwi, entries);
     }
 
     if (s->drv) {
         s->drv->fini (s->drv_opaque);
+        s->drv = NULL;
     }
 }
 
@@ -1832,12 +1810,9 @@
     QLIST_INIT (&s->hw_head_out);
     QLIST_INIT (&s->hw_head_in);
     QLIST_INIT (&s->cap_head);
-    atexit (audio_atexit);
+    atexit(audio_cleanup);
 
     s->ts = timer_new_ns(QEMU_CLOCK_VIRTUAL, audio_timer, s);
-    if (!s->ts) {
-        hw_error("Could not create audio timer\n");
-    }
 
     audio_process_options ("AUDIO", audio_options);
 
@@ -1888,12 +1863,8 @@
 
     if (!done) {
         done = !audio_driver_init (s, &no_audio_driver);
-        if (!done) {
-            hw_error("Could not initialize audio subsystem\n");
-        }
-        else {
-            dolog ("warning: Using timer based audio emulation\n");
-        }
+        assert(done);
+        dolog("warning: Using timer based audio emulation\n");
     }
 
     if (conf.period.hertz <= 0) {
@@ -1904,8 +1875,7 @@
         }
         conf.period.ticks = 1;
     } else {
-        conf.period.ticks =
-            muldiv64 (1, get_ticks_per_sec (), conf.period.hertz);
+        conf.period.ticks = NANOSECONDS_PER_SECOND / conf.period.hertz;
     }
 
     e = qemu_add_vm_change_state_handler (audio_vm_change_state_handler, s);
@@ -2007,8 +1977,7 @@
         QLIST_INSERT_HEAD (&s->cap_head, cap, entries);
         QLIST_INSERT_HEAD (&cap->cb_head, cb, entries);
 
-        hw = NULL;
-        while ((hw = audio_pcm_hw_find_any_out (hw))) {
+        QLIST_FOREACH(hw, &glob_audio_state.hw_head_out, entries) {
             audio_attach_capture (hw);
         }
         return cap;
diff --git a/audio/audio.h b/audio/audio.h
index e7ea397..c3c5198 100644
--- a/audio/audio.h
+++ b/audio/audio.h
@@ -21,10 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef QEMU_AUDIO_H
 #define QEMU_AUDIO_H
 
-#include "config-host.h"
 #include "qemu/queue.h"
 
 typedef void (*audio_callback_fn) (void *opaque, int avail);
@@ -163,4 +163,7 @@
 int wav_start_capture (CaptureState *s, const char *path, int freq,
                        int bits, int nchannels);
 
-#endif  /* audio.h */
+bool audio_is_cleaning_up(void);
+void audio_cleanup(void);
+
+#endif /* QEMU_AUDIO_H */
diff --git a/audio/audio_int.h b/audio/audio_int.h
index 031eb6e..6c6236a 100644
--- a/audio/audio_int.h
+++ b/audio/audio_int.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef QEMU_AUDIO_INT_H
 #define QEMU_AUDIO_INT_H
 
@@ -156,13 +157,13 @@
 };
 
 struct audio_pcm_ops {
-    int  (*init_out)(HWVoiceOut *hw, struct audsettings *as);
+    int  (*init_out)(HWVoiceOut *hw, struct audsettings *as, void *drv_opaque);
     void (*fini_out)(HWVoiceOut *hw);
     int  (*run_out) (HWVoiceOut *hw, int live);
     int  (*write)   (SWVoiceOut *sw, void *buf, int size);
     int  (*ctl_out) (HWVoiceOut *hw, int cmd, ...);
 
-    int  (*init_in) (HWVoiceIn *hw, struct audsettings *as);
+    int  (*init_in) (HWVoiceIn *hw, struct audsettings *as, void *drv_opaque);
     void (*fini_in) (HWVoiceIn *hw);
     int  (*run_in)  (HWVoiceIn *hw);
     int  (*read)    (SWVoiceIn *sw, void *buf, int size);
@@ -206,14 +207,11 @@
 extern struct audio_driver oss_audio_driver;
 extern struct audio_driver sdl_audio_driver;
 extern struct audio_driver wav_audio_driver;
-extern struct audio_driver fmod_audio_driver;
 extern struct audio_driver alsa_audio_driver;
 extern struct audio_driver coreaudio_audio_driver;
 extern struct audio_driver dsound_audio_driver;
-extern struct audio_driver esd_audio_driver;
 extern struct audio_driver pa_audio_driver;
 extern struct audio_driver spice_audio_driver;
-extern struct audio_driver winwave_audio_driver;
 extern struct audio_driver winaudio_audio_driver;
 extern const struct mixeng_volume nominal_volume;
 
@@ -261,4 +259,4 @@
 #define AUDIO_FUNC __FILE__ ":" AUDIO_STRINGIFY (__LINE__)
 #endif
 
-#endif /* audio_int.h */
+#endif /* QEMU_AUDIO_INT_H */
diff --git a/audio/audio_pt_int.c b/audio/audio_pt_int.c
index 9a9c306..21ff9c5 100644
--- a/audio/audio_pt_int.c
+++ b/audio/audio_pt_int.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
diff --git a/audio/audio_pt_int.h b/audio/audio_pt_int.h
index 0dfff76..4c0c15b 100644
--- a/audio/audio_pt_int.h
+++ b/audio/audio_pt_int.h
@@ -19,4 +19,4 @@
 int audio_pt_unlock_and_signal (struct audio_pt *, const char *);
 int audio_pt_join (struct audio_pt *, void **, const char *);
 
-#endif /* audio_pt_int.h */
+#endif /* QEMU_AUDIO_PT_INT_H */
diff --git a/audio/audio_template.h b/audio/audio_template.h
index 8173188..99b27b2 100644
--- a/audio/audio_template.h
+++ b/audio/audio_template.h
@@ -191,9 +191,9 @@
         audio_detach_capture (hw);
 #endif
         QLIST_REMOVE (hw, entries);
+        glue (hw->pcm_ops->fini_, TYPE) (hw);
         glue (s->nb_hw_voices_, TYPE) += 1;
         glue (audio_pcm_hw_free_resources_ ,TYPE) (hw);
-        glue (hw->pcm_ops->fini_, TYPE) (hw);
         g_free (hw);
         *hwp = NULL;
     }
@@ -262,7 +262,7 @@
 #ifdef DAC
     QLIST_INIT (&hw->cap_head);
 #endif
-    if (glue (hw->pcm_ops->init_, TYPE) (hw, as)) {
+    if (glue (hw->pcm_ops->init_, TYPE) (hw, as, s->drv_opaque)) {
         goto err0;
     }
 
@@ -398,10 +398,6 @@
     )
 {
     AudioState *s = &glob_audio_state;
-#ifdef DAC
-    int live = 0;
-    SW *old_sw = NULL;
-#endif
 
     if (audio_bug (AUDIO_FUNC, !card || !name || !callback_fn || !as)) {
         dolog ("card=%p name=%p callback_fn=%p as=%p\n",
@@ -426,29 +422,6 @@
         return sw;
     }
 
-#ifdef DAC
-    if (conf.plive && sw && (!sw->active && !sw->empty)) {
-        live = sw->total_hw_samples_mixed;
-
-#ifdef DEBUG_PLIVE
-        dolog ("Replacing voice %s with %d live samples\n", SW_NAME (sw), live);
-        dolog ("Old %s freq %d, bits %d, channels %d\n",
-               SW_NAME (sw), sw->info.freq, sw->info.bits, sw->info.nchannels);
-        dolog ("New %s freq %d, bits %d, channels %d\n",
-               name,
-               as->freq,
-               (as->fmt == AUD_FMT_S16 || as->fmt == AUD_FMT_U16) ? 16 : 8,
-               as->nchannels);
-#endif
-
-        if (live) {
-            old_sw = sw;
-            old_sw->callback.fn = NULL;
-            sw = NULL;
-        }
-    }
-#endif
-
     if (!glue (conf.fixed_, TYPE).enabled && sw) {
         glue (AUD_close_, TYPE) (card, sw);
         sw = NULL;
@@ -481,20 +454,6 @@
     sw->callback.fn = callback_fn;
     sw->callback.opaque = callback_opaque;
 
-#ifdef DAC
-    if (live) {
-        int mixed =
-            (live << old_sw->info.shift)
-            * old_sw->info.bytes_per_second
-            / sw->info.bytes_per_second;
-
-#ifdef DEBUG_PLIVE
-        dolog ("Silence will be mixed %d\n", mixed);
-#endif
-        sw->total_hw_samples_mixed += mixed;
-    }
-#endif
-
 #ifdef DEBUG_AUDIO
     dolog ("%s\n", name);
     audio_pcm_print_info ("hw", &sw->hw->info);
diff --git a/audio/audio_win_int.c b/audio/audio_win_int.c
index e132405..6900008 100644
--- a/audio/audio_win_int.c
+++ b/audio/audio_win_int.c
@@ -1,5 +1,6 @@
 /* public domain */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 #define AUDIO_CAP "win-int"
diff --git a/audio/coreaudio.c b/audio/coreaudio.c
index 8304a38..9ef76c4 100644
--- a/audio/coreaudio.c
+++ b/audio/coreaudio.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <CoreAudio/CoreAudio.h>
-#include <string.h>             /* strerror */
 #include <pthread.h>            /* pthread_X */
 
 #include "qemu-common.h"
@@ -33,33 +33,290 @@
 #define AUDIO_CAP "coreaudio"
 #include "audio_int.h"
 
-#if 0
-#  define  D(...)  fprintf(stderr, __VA_ARGS__)
-#else
-#  define  D(...)  ((void)0)
+#ifndef MAC_OS_X_VERSION_10_6
+#define MAC_OS_X_VERSION_10_6 1060
 #endif
 
-struct {
-    int out_buffer_frames;
-    int out_nbuffers;
-    int in_buffer_frames;
-    int in_nbuffers;
-    int isAtexit;
-} conf = {
-    .out_buffer_frames = 512,
-    .out_nbuffers = 4,
-    .in_buffer_frames = 512,
-    .in_nbuffers = 4,
-    .isAtexit = 0
-};
+typedef struct {
+    int buffer_frames;
+    int nbuffers;
+} CoreaudioConf;
 
-/***************************************************************************************/
-/***************************************************************************************/
-/***                                                                                 ***/
-/***       U T I L I T Y   R O U T I N E S                                           ***/
-/***                                                                                 ***/
-/***************************************************************************************/
-/***************************************************************************************/
+typedef struct coreaudioVoiceBase {
+    pthread_mutex_t mutex;
+    AudioDeviceID deviceID;
+    UInt32 audioDevicePropertyBufferFrameSize;
+    AudioStreamBasicDescription streamBasicDescription;
+    AudioDeviceIOProcID ioprocid;
+    Boolean isInput;
+    int live;
+    int decr;
+    int pos;
+} coreaudioVoiceBase;
+
+typedef struct coreaudioVoiceOut {
+    HWVoiceOut hw;
+    coreaudioVoiceBase core;
+} coreaudioVoiceOut;
+
+typedef struct coreaudioVoiceIn {
+    HWVoiceIn hw;
+    coreaudioVoiceBase core;
+} coreaudioVoiceIn;
+
+// Set to true when atexit() is running.
+static bool gIsAtExit = false;
+
+static void coreaudio_atexit(void) {
+    gIsAtExit = true;
+}
+
+#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6
+/* The APIs used here only become available from 10.6 */
+
+static OSStatus coreaudio_get_voice(AudioDeviceID *id, Boolean isInput)
+{
+    UInt32 size = sizeof(*id);
+    AudioObjectPropertyAddress addr = {
+        isInput ? kAudioHardwarePropertyDefaultInputDevice
+                : kAudioHardwarePropertyDefaultOutputDevice,
+        kAudioObjectPropertyScopeGlobal,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(kAudioObjectSystemObject,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      id);
+}
+
+static OSStatus coreaudio_get_framesizerange(AudioDeviceID id,
+                                             AudioValueRange *framerange,
+                                             Boolean isInput)
+{
+    UInt32 size = sizeof(*framerange);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyBufferFrameSizeRange,
+        isInput ? kAudioDevicePropertyScopeInput
+                : kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      framerange);
+}
+
+static OSStatus coreaudio_get_framesize(AudioDeviceID id,
+                                        UInt32 *framesize,
+                                        Boolean isInput)
+{
+    UInt32 size = sizeof(*framesize);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyBufferFrameSize,
+        isInput ? kAudioDevicePropertyScopeInput
+                : kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      framesize);
+}
+
+static OSStatus coreaudio_set_framesize(AudioDeviceID id,
+                                        UInt32 *framesize,
+                                        Boolean isInput)
+{
+    UInt32 size = sizeof(*framesize);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyBufferFrameSize,
+        isInput ? kAudioDevicePropertyScopeInput
+                : kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectSetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      size,
+                                      framesize);
+}
+
+static OSStatus coreaudio_get_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d,
+                                           Boolean isInput)
+{
+    UInt32 size = sizeof(*d);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyStreamFormat,
+        isInput ? kAudioDevicePropertyScopeInput
+                : kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      d);
+}
+
+static OSStatus coreaudio_set_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d,
+                                           Boolean isInput)
+{
+    UInt32 size = sizeof(*d);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyStreamFormat,
+        isInput ? kAudioDevicePropertyScopeInput
+                : kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectSetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      size,
+                                      d);
+}
+
+static OSStatus coreaudio_get_isrunning(AudioDeviceID id,
+                                        UInt32 *result,
+                                        Boolean isInput)
+{
+    UInt32 size = sizeof(*result);
+    AudioObjectPropertyAddress addr = {
+        kAudioDevicePropertyDeviceIsRunning,
+        isInput ? kAudioDevicePropertyScopeInput
+                : kAudioDevicePropertyScopeOutput,
+        kAudioObjectPropertyElementMaster
+    };
+
+    return AudioObjectGetPropertyData(id,
+                                      &addr,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      result);
+}
+#else
+/* Legacy versions of functions using deprecated APIs */
+
+static OSStatus coreaudio_get_voice(AudioDeviceID *id, Boolean isInput)
+{
+    UInt32 size = sizeof(*id);
+
+    return AudioHardwareGetProperty(
+        isInput ? kAudioHardwarePropertyDefaultInputDevice
+                : kAudioHardwarePropertyDefaultOutputDevice,
+        &size,
+        id);
+}
+
+static OSStatus coreaudio_get_framesizerange(AudioDeviceID id,
+                                             AudioValueRange *framerange,
+                                             Boolean isInput)
+{
+    UInt32 size = sizeof(*framerange);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        isInput,
+        kAudioDevicePropertyBufferFrameSizeRange,
+        &size,
+        framerange);
+}
+
+static OSStatus coreaudio_get_framesize(AudioDeviceID id,
+                                        UInt32 *framesize,
+                                        Boolean isInput)
+{
+    UInt32 size = sizeof(*framesize);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        isInput,
+        kAudioDevicePropertyBufferFrameSize,
+        &size,
+        framesize);
+}
+
+static OSStatus coreaudio_set_framesize(AudioDeviceID id,
+                                        UInt32 *framesize,
+                                        Boolean isInput)
+{
+    UInt32 size = sizeof(*framesize);
+
+    return AudioDeviceSetProperty(
+        id,
+        NULL,
+        0,
+        isInput,
+        kAudioDevicePropertyBufferFrameSize,
+        size,
+        framesize);
+}
+
+static OSStatus coreaudio_get_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d,
+                                           Boolean isInput)
+{
+    UInt32 size = sizeof(*d);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        isInput,
+        kAudioDevicePropertyStreamFormat,
+        &size,
+        d);
+}
+
+static OSStatus coreaudio_set_streamformat(AudioDeviceID id,
+                                           AudioStreamBasicDescription *d,
+                                           Boolean isInput)
+{
+    UInt32 size = sizeof(*d);
+
+    return AudioDeviceSetProperty(
+        id,
+        0,
+        0,
+        isInput,
+        kAudioDevicePropertyStreamFormat,
+        size,
+        d);
+}
+
+static OSStatus coreaudio_get_isrunning(AudioDeviceID id,
+                                        UInt32 *result,
+                                        Boolean isInput)
+{
+    UInt32 size = sizeof(*result);
+
+    return AudioDeviceGetProperty(
+        id,
+        0,
+        isInput,
+        kAudioDevicePropertyDeviceIsRunning,
+        &size,
+        result);
+}
+#endif
 
 static void coreaudio_logstatus (OSStatus status)
 {
@@ -151,34 +408,11 @@
     coreaudio_logstatus (status);
 }
 
-/***************************************************************************************/
-/***************************************************************************************/
-/***                                                                                 ***/
-/***       S H A R E D   I N / O U T   V O I C E                                     ***/
-/***                                                                                 ***/
-/***************************************************************************************/
-/***************************************************************************************/
-
-typedef struct coreAudioVoice {
-    pthread_mutex_t              mutex;
-    AudioDeviceID                deviceID;
-    Boolean                      isInput;
-    UInt32                       bufferFrameSize;
-    AudioStreamBasicDescription  streamBasicDescription;
-    AudioDeviceIOProc            ioproc;
-    int                          live;
-    int                          decr;
-    int                          pos;
-} coreaudioVoice;
-
-static inline UInt32 coreaudio_voice_isPlaying (coreaudioVoice *core)
+static inline UInt32 isPlaying (AudioDeviceID deviceID, Boolean isInput)
 {
     OSStatus status;
     UInt32 result = 0;
-    UInt32 propertySize = sizeof(core->deviceID);
-    status = AudioDeviceGetProperty(
-        core->deviceID, 0, core->isInput,
-        kAudioDevicePropertyDeviceIsRunning, &propertySize, &result);
+    status = coreaudio_get_isrunning(deviceID, &result, isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr(status,
                          "Could not determine whether Device is playing\n");
@@ -186,12 +420,7 @@
     return result;
 }
 
-static void coreaudio_atexit (void)
-{
-    conf.isAtexit = 1;
-}
-
-static int coreaudio_voice_lock (coreaudioVoice *core, const char *fn_name)
+static int coreaudio_lock (coreaudioVoiceBase *core, const char *fn_name)
 {
     int err;
 
@@ -204,7 +433,7 @@
     return 0;
 }
 
-static int coreaudio_voice_unlock (coreaudioVoice *core, const char *fn_name)
+static int coreaudio_unlock (coreaudioVoiceBase *core, const char *fn_name)
 {
     int err;
 
@@ -217,78 +446,20 @@
     return 0;
 }
 
-static int coreaudio_voice_ctl (coreaudioVoice *core, int cmd)
-{
-    OSStatus status;
-
-    switch (cmd) {
-    case VOICE_ENABLE:
-        /* start playback */
-        D("%s: %s started\n", __FUNCTION__, core->isInput ? "input" : "output");
-        if (!coreaudio_voice_isPlaying(core)) {
-            status = AudioDeviceStart(core->deviceID, core->ioproc);
-            if (status != kAudioHardwareNoError) {
-                coreaudio_logerr (status, "Could not resume playback\n");
-            }
-        }
-        break;
-
-    case VOICE_DISABLE:
-        /* stop playback */
-        D("%s: %s stopped\n", __FUNCTION__, core->isInput ? "input" : "output");
-        if (!conf.isAtexit) {
-            if (coreaudio_voice_isPlaying(core)) {
-                status = AudioDeviceStop(core->deviceID, core->ioproc);
-                if (status != kAudioHardwareNoError) {
-                    coreaudio_logerr (status, "Could not pause playback\n");
-                }
-            }
-        }
-        break;
-    }
-    return 0;
-}
-
-static void coreaudio_voice_fini (coreaudioVoice *core)
+static int coreaudio_init_base(coreaudioVoiceBase *core,
+                               struct audsettings *as,
+                               void *drv_opaque,
+                               AudioDeviceIOProc ioproc,
+                               void *hw,
+                               Boolean isInput)
 {
     OSStatus status;
     int err;
-
-    pthread_mutex_lock(&core->mutex);
-
-    /* stop playback */
-    coreaudio_voice_ctl(core, VOICE_DISABLE);
-
-    /* remove callback */
-    status = AudioDeviceRemoveIOProc(core->deviceID, core->ioproc);
-    if (status != kAudioHardwareNoError) {
-        coreaudio_logerr (status, "Could not remove IOProc\n");
-    }
-
-    core->deviceID = kAudioDeviceUnknown;
-
-    pthread_mutex_unlock(&core->mutex);
-
-    /* destroy mutex */
-    err = pthread_mutex_destroy(&core->mutex);
-    if (err) {
-        dolog("Could not destroy mutex\nReason: %s\n", strerror (err));
-    }
-}
-
-
-static int coreaudio_voice_init (coreaudioVoice *core, struct audsettings *as,
-                                 int frameSize, AudioDeviceIOProc ioproc,
-                                 void *hw, int input)
-{
-    OSStatus  status;
-    UInt32    propertySize;
-    int       err;
-    int       bits = 8;
+    const char *typ = isInput ? "record" : "playback";
     AudioValueRange frameRange;
-    const char*  typ = input ? "input" : "playback";
+    CoreaudioConf *conf = drv_opaque;
 
-    core->isInput = input ? true : false;
+    core->isInput = isInput;
 
     /* create mutex */
     err = pthread_mutex_init(&core->mutex, NULL);
@@ -297,22 +468,9 @@
         return -1;
     }
 
-    if (as->fmt == AUD_FMT_S16 || as->fmt == AUD_FMT_U16) {
-        bits = 16;
-    }
+    // audio_pcm_init_info (&hw->info, as);
 
-    // TODO: audio_pcm_init_info (&hw->info, as);
-    /* open default output device */
-    /* note: we use DefaultSystemOutputDevice because DefaultOutputDevice seems to
-     * always link to the internal speakers, and not the ones selected through system properties
-     * go figure...
-     */
-    propertySize = sizeof(core->deviceID);
-    status = AudioHardwareGetProperty(
-        input ? kAudioHardwarePropertyDefaultInputDevice :
-                kAudioHardwarePropertyDefaultSystemOutputDevice,
-        &propertySize,
-        &core->deviceID);
+    status = coreaudio_get_voice(&core->deviceID, isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get default %s device\n", typ);
@@ -324,74 +482,52 @@
     }
 
     /* get minimum and maximum buffer frame sizes */
-    propertySize = sizeof(frameRange);
-    status = AudioDeviceGetProperty(
-        core->deviceID,
-        0,
-        core->isInput,
-        kAudioDevicePropertyBufferFrameSizeRange,
-        &propertySize,
-        &frameRange);
+    status = coreaudio_get_framesizerange(core->deviceID,
+                                          &frameRange, isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get device buffer frame range\n");
         return -1;
     }
 
-    if (frameRange.mMinimum > frameSize) {
-        core->bufferFrameSize = (UInt32) frameRange.mMinimum;
-        dolog ("warning: Upsizing Output Buffer Frames to %f\n", frameRange.mMinimum);
+    if (frameRange.mMinimum > conf->buffer_frames) {
+        core->audioDevicePropertyBufferFrameSize = (UInt32) frameRange.mMinimum;
+        dolog ("warning: Upsizing Buffer Frames to %f\n", frameRange.mMinimum);
     }
-    else if (frameRange.mMaximum < frameSize) {
-        core->bufferFrameSize = (UInt32) frameRange.mMaximum;
-        dolog ("warning: Downsizing Output Buffer Frames to %f\n", frameRange.mMaximum);
+    else if (frameRange.mMaximum < conf->buffer_frames) {
+        core->audioDevicePropertyBufferFrameSize = (UInt32) frameRange.mMaximum;
+        dolog ("warning: Downsizing Buffer Frames to %f\n", frameRange.mMaximum);
     }
     else {
-        core->bufferFrameSize = frameSize;
+        core->audioDevicePropertyBufferFrameSize = conf->buffer_frames;
     }
 
     /* set Buffer Frame Size */
-    propertySize = sizeof(core->bufferFrameSize);
-    status = AudioDeviceSetProperty(
-        core->deviceID,
-        NULL,
-        0,
-        core->isInput,
-        kAudioDevicePropertyBufferFrameSize,
-        propertySize,
-        &core->bufferFrameSize);
+    status = coreaudio_set_framesize(core->deviceID,
+                                     &core->audioDevicePropertyBufferFrameSize,
+                                     isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not set device buffer frame size %" PRIu32 "\n",
-                           (uint32_t)core->bufferFrameSize);
+                           (uint32_t)core->audioDevicePropertyBufferFrameSize);
         return -1;
     }
 
     /* get Buffer Frame Size */
-    propertySize = sizeof(core->bufferFrameSize);
-    status = AudioDeviceGetProperty(
-        core->deviceID,
-        0,
-        core->isInput,
-        kAudioDevicePropertyBufferFrameSize,
-        &propertySize,
-        &core->bufferFrameSize);
+    status = coreaudio_get_framesize(core->deviceID,
+                                     &core->audioDevicePropertyBufferFrameSize,
+                                     isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get device buffer frame size\n");
         return -1;
     }
-    // TODO: hw->samples = *pNBuffers * core->bufferFrameSize;
+    // hw->samples = conf->nbuffers * core->audioDevicePropertyBufferFrameSize;
 
     /* get StreamFormat */
-    propertySize = sizeof(core->streamBasicDescription);
-    status = AudioDeviceGetProperty(
-        core->deviceID,
-        0,
-        core->isInput,
-        kAudioDevicePropertyStreamFormat,
-        &propertySize,
-        &core->streamBasicDescription);
+    status = coreaudio_get_streamformat(core->deviceID,
+                                        &core->streamBasicDescription,
+                                        isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ,
                            "Could not get Device Stream properties\n");
@@ -401,15 +537,9 @@
 
     /* set Samplerate */
     core->streamBasicDescription.mSampleRate = (Float64) as->freq;
-    propertySize = sizeof(core->streamBasicDescription);
-    status = AudioDeviceSetProperty(
-        core->deviceID,
-        0,
-        0,
-        core->isInput,
-        kAudioDevicePropertyStreamFormat,
-        propertySize,
-        &core->streamBasicDescription);
+    status = coreaudio_set_streamformat(core->deviceID,
+                                        &core->streamBasicDescription,
+                                        isInput);
     if (status != kAudioHardwareNoError) {
         coreaudio_logerr2 (status, typ, "Could not set samplerate %d\n",
                            as->freq);
@@ -418,20 +548,23 @@
     }
 
     /* set Callback */
-    core->ioproc = ioproc;
-    status = AudioDeviceAddIOProc(core->deviceID, ioproc, hw);
-    if (status != kAudioHardwareNoError) {
+    core->ioprocid = NULL;
+    status = AudioDeviceCreateIOProcID(core->deviceID,
+                                       ioproc,
+                                       hw,
+                                       &core->ioprocid);
+    if (status != kAudioHardwareNoError || core->ioprocid == NULL) {
         coreaudio_logerr2 (status, typ, "Could not set IOProc\n");
         core->deviceID = kAudioDeviceUnknown;
         return -1;
     }
 
     /* start Playback */
-    if (!input && !coreaudio_voice_isPlaying(core)) {
-        status = AudioDeviceStart(core->deviceID, core->ioproc);
+    if (!isPlaying(core->deviceID, isInput)) {
+        status = AudioDeviceStart(core->deviceID, core->ioprocid);
         if (status != kAudioHardwareNoError) {
             coreaudio_logerr2 (status, typ, "Could not start playback\n");
-            AudioDeviceRemoveIOProc(core->deviceID, core->ioproc);
+            AudioDeviceDestroyIOProcID(core->deviceID, core->ioprocid);
             core->deviceID = kAudioDeviceUnknown;
             return -1;
         }
@@ -440,29 +573,76 @@
     return 0;
 }
 
+static void coreaudio_fini_base (coreaudioVoiceBase *core)
+{
+    OSStatus status;
+    int err;
 
-/***************************************************************************************/
-/***************************************************************************************/
-/***                                                                                 ***/
-/***       O U T P U T   V O I C E                                                   ***/
-/***                                                                                 ***/
-/***************************************************************************************/
-/***************************************************************************************/
+    if (!audio_is_cleaning_up()) {
+        /* stop playback */
+        if (isPlaying(core->deviceID, core->isInput)) {
+            status = AudioDeviceStop(core->deviceID, core->ioprocid);
+            if (status != kAudioHardwareNoError) {
+                coreaudio_logerr (status, "Could not stop %s\n",
+                                  core->isInput ? "recording" : "playback");
+            }
+        }
 
-typedef struct coreaudioVoiceOut {
-    HWVoiceOut                   hw;
-    coreaudioVoice               core[1];
-} coreaudioVoiceOut;
+        /* remove callback */
+        status = AudioDeviceDestroyIOProcID(core->deviceID,
+                                            core->ioprocid);
+        if (status != kAudioHardwareNoError) {
+            coreaudio_logerr (status, "Could not remove IOProc\n");
+        }
+    }
+    core->deviceID = kAudioDeviceUnknown;
 
-#define  CORE_OUT(hw)  ((coreaudioVoiceOut*)(hw))->core
+    /* destroy mutex */
+    err = pthread_mutex_destroy(&core->mutex);
+    if (err) {
+        dolog("Could not destroy mutex\nReason: %s\n", strerror (err));
+    }
+}
 
+static int coreaudio_ctl_base (coreaudioVoiceBase *core, int cmd)
+{
+    OSStatus status;
+
+    switch (cmd) {
+    case VOICE_ENABLE:
+        /* start playback */
+        if (!isPlaying(core->deviceID, core->isInput)) {
+            status = AudioDeviceStart(core->deviceID, core->ioprocid);
+            if (status != kAudioHardwareNoError) {
+                coreaudio_logerr (status, "Could not resume %s\n",
+                                  core->isInput ? "recording" : "playback");
+            }
+        }
+        break;
+
+    case VOICE_DISABLE:
+        /* stop playback */
+        if (!audio_is_cleaning_up()) {
+            if (isPlaying(core->deviceID, core->isInput)) {
+                status = AudioDeviceStop(core->deviceID,
+                                         core->ioprocid);
+                if (status != kAudioHardwareNoError) {
+                    coreaudio_logerr (status, "Could not pause %s\n",
+                                      core->isInput ? "recording" : "playback");
+                }
+            }
+        }
+        break;
+    }
+    return 0;
+}
 
 static int coreaudio_run_out (HWVoiceOut *hw, int live)
 {
     int decr;
-    coreaudioVoice *core = CORE_OUT(hw);
+    coreaudioVoiceBase *core = &((coreaudioVoiceOut *)hw)->core;
 
-    if (conf.isAtexit || coreaudio_voice_lock (core, "coreaudio_run_out")) {
+    if (coreaudio_lock (core, "coreaudio_run_out")) {
         return 0;
     }
 
@@ -479,12 +659,12 @@
     core->live = live - decr;
     hw->rpos = core->pos;
 
-    coreaudio_voice_unlock (core, "coreaudio_run_out");
+    coreaudio_unlock (core, "coreaudio_run_out");
     return decr;
 }
 
 /* callback to feed audiooutput buffer */
-static OSStatus audioOutDeviceIOProc(
+static OSStatus audioOutputDeviceIOProc(
     AudioDeviceID inDevice,
     const AudioTimeStamp* inNow,
     const AudioBufferList* inInputData,
@@ -496,7 +676,7 @@
     UInt32 frame, frameCount;
     float *out = outOutputData->mBuffers[0].mData;
     HWVoiceOut *hw = hwptr;
-    coreaudioVoice *core = CORE_OUT(hw);
+    coreaudioVoiceBase *core = &((coreaudioVoiceOut *) hw)->core;
     int rpos, live;
     struct st_sample *src;
 #ifndef FLOAT_MIXENG
@@ -507,18 +687,18 @@
 #endif
 #endif
 
-    if (coreaudio_voice_lock (core, "audioDeviceIOProc")) {
+    if (coreaudio_lock (core, "audioOutputDeviceIOProc")) {
         inInputTime = 0;
         return 0;
     }
 
-    frameCount = core->bufferFrameSize;
+    frameCount = core->audioDevicePropertyBufferFrameSize;
     live = core->live;
 
     /* if there are not enough samples, set signal and return */
-    if (live < (int)frameCount) {
+    if (live < frameCount) {
         inInputTime = 0;
-        coreaudio_voice_unlock (core, "audioDeviceIOProc(empty)");
+        coreaudio_unlock (core, "audioOutputDeviceIOProc(empty)");
         return 0;
     }
 
@@ -545,7 +725,7 @@
     core->decr += frameCount;
     core->pos = rpos;
 
-    coreaudio_voice_unlock (core, "audioDeviceIOProc");
+    coreaudio_unlock (core, "audioOutputDeviceIOProc");
     return 0;
 }
 
@@ -554,75 +734,54 @@
     return audio_pcm_sw_write (sw, buf, len);
 }
 
-static int coreaudio_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int coreaudio_init_out(HWVoiceOut *hw, struct audsettings *as,
+                              void *drv_opaque)
 {
-    coreaudioVoice *core = CORE_OUT(hw);
-    int err;
+    coreaudioVoiceBase *core = &((coreaudioVoiceOut *) hw)->core;
+    CoreaudioConf *conf = drv_opaque;
 
     audio_pcm_init_info (&hw->info, as);
 
-    err = coreaudio_voice_init (core, as, conf.out_buffer_frames, audioOutDeviceIOProc, hw, 0);
-    if (err < 0)
-        return err;
+    if (coreaudio_init_base(core, as, drv_opaque,
+                            audioOutputDeviceIOProc, hw, false) < 0) {
+        return -1;
+    }
 
-    hw->samples = core->bufferFrameSize * conf.out_nbuffers;
+    hw->samples = conf->nbuffers * core->audioDevicePropertyBufferFrameSize;
+
     return 0;
 }
-
 static void coreaudio_fini_out (HWVoiceOut *hw)
 {
-    coreaudioVoice *core = CORE_OUT(hw);
-
-    coreaudio_voice_fini (core);
+    coreaudioVoiceBase *core = &((coreaudioVoiceOut *) hw)->core;
+    coreaudio_fini_base(core);
 }
 
-static int
-coreaudio_ctl_out (HWVoiceOut *hw, int cmd, ...)
+static int coreaudio_ctl_out (HWVoiceOut *hw, int cmd, ...)
 {
-    coreaudioVoice *core = CORE_OUT(hw);
-
-    return coreaudio_voice_ctl (core, cmd);
+    coreaudioVoiceBase *core = &((coreaudioVoiceOut *) hw)->core;
+    return coreaudio_ctl_base(core, cmd);
 }
 
-/***************************************************************************************/
-/***************************************************************************************/
-/***                                                                                 ***/
-/***       I N P U T   V O I C E                                                     ***/
-/***                                                                                 ***/
-/***************************************************************************************/
-/***************************************************************************************/
-
-
-
-typedef struct coreaudioVoiceIn {
-    HWVoiceIn        hw;
-    coreaudioVoice   core[1];
-} coreaudioVoiceIn;
-
-#define  CORE_IN(hw)  ((coreaudioVoiceIn *) (hw))->core
-
-
 static int coreaudio_run_in (HWVoiceIn *hw)
 {
     int decr;
 
-    coreaudioVoice *core = CORE_IN(hw);
+    coreaudioVoiceBase *core = &((coreaudioVoiceIn *)hw)->core;
 
-    if (conf.isAtexit || coreaudio_voice_lock (core, "coreaudio_run_in")) {
+    if (gIsAtExit || coreaudio_lock (core, "coreaudio_run_in")) {
         return 0;
     }
-    D("%s: core.decr=%d core.pos=%d\n", __FUNCTION__, core->decr, core->pos);
     decr        = core->decr;
     core->decr -= decr;
     hw->wpos    = core->pos;
 
-    coreaudio_voice_unlock (core, "coreaudio_run_in");
+    coreaudio_unlock (core, "coreaudio_run_in");
     return decr;
 }
 
-
-/* callback to feed audiooutput buffer */
-static OSStatus audioInDeviceIOProc(
+/* callback to feed audioinput buffer */
+static OSStatus audioInputDeviceIOProc(
     AudioDeviceID inDevice,
     const AudioTimeStamp* inNow,
     const AudioBufferList* inInputData,
@@ -634,7 +793,7 @@
     UInt32 frame, frameCount;
     float *in = inInputData->mBuffers[0].mData;
     HWVoiceIn *hw = hwptr;
-    coreaudioVoice *core = CORE_IN(hw);
+    coreaudioVoiceBase *core = &((coreaudioVoiceIn *)hw)->core;
     int wpos, avail;
     struct st_sample *dst;
 #ifndef FLOAT_MIXENG
@@ -645,21 +804,18 @@
 #endif
 #endif
 
-    if (coreaudio_voice_lock (core, "audioDeviceIOProc")) {
+    if (coreaudio_lock (core, "audioInputDeviceIOProc")) {
         inInputTime = 0;
         return 0;
     }
 
-    frameCount = core->bufferFrameSize;
+    frameCount = core->audioDevicePropertyBufferFrameSize;
     avail      = hw->samples - hw->total_samples_captured - core->decr;
 
-    D("%s: enter avail=%d core.decr=%d core.pos=%d hw.samples=%d hw.total_samples_captured=%d frameCount=%d\n",
-      __FUNCTION__, avail, core->decr, core->pos, hw->samples, hw->total_samples_captured, (int)frameCount);
-
     /* if there are not enough samples, set signal and return */
     if (avail < (int)frameCount) {
         inInputTime = 0;
-        coreaudio_voice_unlock (core, "audioDeviceIOProc(empty)");
+        coreaudio_unlock (core, "audioInputDeviceIOProc(empty)");
         return 0;
     }
 
@@ -686,84 +842,75 @@
     core->decr += frameCount;
     core->pos   = wpos;
 
-    D("exit: core.decr=%d core.pos=%d\n", core->decr, core->pos);
-    coreaudio_voice_unlock (core, "audioDeviceIOProc");
+    coreaudio_unlock (core, "audioInputDeviceIOProc");
     return 0;
 }
 
 static int coreaudio_read (SWVoiceIn *sw, void *buf, int len)
 {
-    int  result = audio_pcm_sw_read (sw, buf, len);
-    D("%s: audio_pcm_sw_read(%d) returned %d\n", __FUNCTION__, len, result);
-    return result;
+    return audio_pcm_sw_read (sw, buf, len);
 }
 
-static int coreaudio_init_in (HWVoiceIn *hw, struct audsettings *as)
+static int coreaudio_init_in(HWVoiceIn *hw, struct audsettings *as,
+                             void *drv_opaque)
 {
-    coreaudioVoice*  core = CORE_IN(hw);
-    int              err;
+    coreaudioVoiceBase *core = &((coreaudioVoiceIn *) hw)->core;
+    CoreaudioConf *conf = drv_opaque;
 
     audio_pcm_init_info (&hw->info, as);
 
-    err = coreaudio_voice_init (core, as, conf.in_buffer_frames, audioInDeviceIOProc, hw, 1);
-    if (err < 0) {
-        return err;
+    if (coreaudio_init_base(core, as, drv_opaque,
+                            audioInputDeviceIOProc, hw, true) < 0) {
+        return -1;
     }
 
-    hw->samples = core->bufferFrameSize * conf.in_nbuffers;
+    hw->samples = conf->nbuffers * core->audioDevicePropertyBufferFrameSize;
+
     return 0;
 }
-
 static void coreaudio_fini_in (HWVoiceIn *hw)
 {
-
-    coreaudioVoice*  core = CORE_IN(hw);
-
-    coreaudio_voice_fini(core);
+    coreaudioVoiceBase *core = &((coreaudioVoiceIn *) hw)->core;
+    coreaudio_fini_base(core);
 }
 
 static int coreaudio_ctl_in (HWVoiceIn *hw, int cmd, ...)
 {
-    coreaudioVoice*  core = CORE_IN(hw);
-
-    return coreaudio_voice_ctl(core, cmd);
+    coreaudioVoiceBase *core = &((coreaudioVoiceIn *) hw)->core;
+    return coreaudio_ctl_base(core, cmd);
 }
 
+static CoreaudioConf glob_conf = {
+    .buffer_frames = 512,
+    .nbuffers = 4,
+};
+
 static void *coreaudio_audio_init (void)
 {
+    CoreaudioConf *conf = g_malloc(sizeof(CoreaudioConf));
+    *conf = glob_conf;
+
     atexit(coreaudio_atexit);
-    return &coreaudio_audio_init;
+    return conf;
 }
 
 static void coreaudio_audio_fini (void *opaque)
 {
-    (void) opaque;
+    g_free(opaque);
 }
 
 static struct audio_option coreaudio_options[] = {
     {
-        .name  = "OUT_BUFFER_SIZE",
+        .name  = "BUFFER_SIZE",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.out_buffer_frames,
-        .descr = "Size of the output buffer in frames"
+        .valp  = &glob_conf.buffer_frames,
+        .descr = "Size of the buffer in frames"
     },
     {
-        .name  = "OUT_BUFFER_COUNT",
+        .name  = "BUFFER_COUNT",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.out_nbuffers,
-        .descr = "Number of output buffers"
-    },
-    {
-        .name  = "IN_BUFFER_SIZE",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.in_buffer_frames,
-        .descr = "Size of the input buffer in frames"
-    },
-    {
-        .name  = "IN_BUFFER_COUNT",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.in_nbuffers,
-        .descr = "Number of input buffers"
+        .valp  = &glob_conf.nbuffers,
+        .descr = "Number of buffers"
     },
     { /* End of list */ }
 };
diff --git a/audio/dsound_template.h b/audio/dsound_template.h
index 8b37d16..b439f33 100644
--- a/audio/dsound_template.h
+++ b/audio/dsound_template.h
@@ -67,11 +67,11 @@
     LPVOID *p2p,
     DWORD *blen1p,
     DWORD *blen2p,
-    int entire
+    int entire,
+    dsound *s
     )
 {
     HRESULT hr;
-    int i;
     LPVOID p1 = NULL, p2 = NULL;
     DWORD blen1 = 0, blen2 = 0;
     DWORD flag;
@@ -81,37 +81,18 @@
 #else
     flag = entire ? DSBLOCK_ENTIREBUFFER : 0;
 #endif
-    for (i = 0; i < conf.lock_retries; ++i) {
-        hr = glue (IFACE, _Lock) (
-            buf,
-            pos,
-            len,
-            &p1,
-            &blen1,
-            &p2,
-            &blen2,
-            flag
-            );
+    hr = glue(IFACE, _Lock)(buf, pos, len, &p1, &blen1, &p2, &blen2, flag);
 
-        if (FAILED (hr)) {
+    if (FAILED (hr)) {
 #ifndef DSBTYPE_IN
-            if (hr == DSERR_BUFFERLOST) {
-                if (glue (dsound_restore_, TYPE) (buf)) {
-                    dsound_logerr (hr, "Could not lock " NAME "\n");
-                    goto fail;
-                }
-                continue;
+        if (hr == DSERR_BUFFERLOST) {
+            if (glue (dsound_restore_, TYPE) (buf, s)) {
+                dsound_logerr (hr, "Could not lock " NAME "\n");
             }
-#endif
-            dsound_logerr (hr, "Could not lock " NAME "\n");
             goto fail;
         }
-
-        break;
-    }
-
-    if (i == conf.lock_retries) {
-        dolog ("%d attempts to lock " NAME " failed\n", i);
+#endif
+        dsound_logerr (hr, "Could not lock " NAME "\n");
         goto fail;
     }
 
@@ -174,16 +155,19 @@
 }
 
 #ifdef DSBTYPE_IN
-static int dsound_init_in (HWVoiceIn *hw, struct audsettings *as)
+static int dsound_init_in(HWVoiceIn *hw, struct audsettings *as,
+                          void *drv_opaque)
 #else
-static int dsound_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int dsound_init_out(HWVoiceOut *hw, struct audsettings *as,
+                           void *drv_opaque)
 #endif
 {
     int err;
     HRESULT hr;
-    dsound *s = &glob_dsound;
+    dsound *s = drv_opaque;
     WAVEFORMATEX wfx;
     struct audsettings obt_as;
+    DSoundConf *conf = &s->conf;
 #ifdef DSBTYPE_IN
     const char *typ = "ADC";
     DSoundVoiceIn *ds = (DSoundVoiceIn *) hw;
@@ -210,7 +194,7 @@
     bd.dwSize = sizeof (bd);
     bd.lpwfxFormat = &wfx;
 #ifdef DSBTYPE_IN
-    bd.dwBufferBytes = conf.bufsize_in;
+    bd.dwBufferBytes = conf->bufsize_in;
     hr = IDirectSoundCapture_CreateCaptureBuffer (
         s->dsound_capture,
         &bd,
@@ -219,7 +203,7 @@
         );
 #else
     bd.dwFlags = DSBCAPS_STICKYFOCUS | DSBCAPS_GETCURRENTPOSITION2;
-    bd.dwBufferBytes = conf.bufsize_out;
+    bd.dwBufferBytes = conf->bufsize_out;
     hr = IDirectSound_CreateSoundBuffer (
         s->dsound,
         &bd,
@@ -269,6 +253,7 @@
             );
     }
     hw->samples = bc.dwBufferBytes >> hw->info.shift;
+    ds->s = s;
 
 #ifdef DEBUG_DSOUND
     dolog ("caps %ld, desc %ld\n",
diff --git a/audio/dsoundaudio.c b/audio/dsoundaudio.c
index e2d89fd..516846e 100644
--- a/audio/dsoundaudio.c
+++ b/audio/dsoundaudio.c
@@ -26,6 +26,7 @@
  * SEAL 1.07 by Carlos 'pel' Hasan was used as documentation
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
@@ -41,42 +42,25 @@
 
 /* #define DEBUG_DSOUND */
 
-static struct {
-    int lock_retries;
-    int restore_retries;
-    int getstatus_retries;
-    int set_primary;
+typedef struct {
     int bufsize_in;
     int bufsize_out;
-    struct audsettings settings;
     int latency_millis;
-} conf = {
-    .lock_retries       = 1,
-    .restore_retries    = 1,
-    .getstatus_retries  = 1,
-    .set_primary        = 0,
-    .bufsize_in         = 16384,
-    .bufsize_out        = 16384,
-    .settings.freq      = 44100,
-    .settings.nchannels = 2,
-    .settings.fmt       = AUD_FMT_S16,
-    .latency_millis     = 10
-};
+} DSoundConf;
 
 typedef struct {
     LPDIRECTSOUND dsound;
     LPDIRECTSOUNDCAPTURE dsound_capture;
-    LPDIRECTSOUNDBUFFER dsound_primary_buffer;
     struct audsettings settings;
+    DSoundConf conf;
 } dsound;
 
-static dsound glob_dsound;
-
 typedef struct {
     HWVoiceOut hw;
     LPDIRECTSOUNDBUFFER dsound_buffer;
     DWORD old_pos;
     int first_time;
+    dsound *s;
 #ifdef DEBUG_DSOUND
     DWORD old_ppos;
     DWORD played;
@@ -88,6 +72,7 @@
     HWVoiceIn hw;
     int first_time;
     LPDIRECTSOUNDCAPTUREBUFFER dsound_capture_buffer;
+    dsound *s;
 } DSoundVoiceIn;
 
 static void dsound_log_hresult (HRESULT hr)
@@ -281,29 +266,17 @@
 }
 #endif
 
-static int dsound_restore_out (LPDIRECTSOUNDBUFFER dsb)
+static int dsound_restore_out (LPDIRECTSOUNDBUFFER dsb, dsound *s)
 {
     HRESULT hr;
-    int i;
 
-    for (i = 0; i < conf.restore_retries; ++i) {
-        hr = IDirectSoundBuffer_Restore (dsb);
+    hr = IDirectSoundBuffer_Restore (dsb);
 
-        switch (hr) {
-        case DS_OK:
-            return 0;
-
-        case DSERR_BUFFERLOST:
-            continue;
-
-        default:
-            dsound_logerr (hr, "Could not restore playback buffer\n");
-            return -1;
-        }
+    if (hr != DS_OK) {
+        dsound_logerr (hr, "Could not restore playback buffer\n");
+        return -1;
     }
-
-    dolog ("%d attempts to restore playback buffer failed\n", i);
-    return -1;
+    return 0;
 }
 
 #include "dsound_template.h"
@@ -311,25 +284,20 @@
 #include "dsound_template.h"
 #undef DSBTYPE_IN
 
-static int dsound_get_status_out (LPDIRECTSOUNDBUFFER dsb, DWORD *statusp)
+static int dsound_get_status_out (LPDIRECTSOUNDBUFFER dsb, DWORD *statusp,
+                                  dsound *s)
 {
     HRESULT hr;
-    int i;
 
-    for (i = 0; i < conf.getstatus_retries; ++i) {
-        hr = IDirectSoundBuffer_GetStatus (dsb, statusp);
-        if (FAILED (hr)) {
-            dsound_logerr (hr, "Could not get playback buffer status\n");
-            return -1;
-        }
+    hr = IDirectSoundBuffer_GetStatus (dsb, statusp);
+    if (FAILED (hr)) {
+        dsound_logerr (hr, "Could not get playback buffer status\n");
+        return -1;
+    }
 
-        if (*statusp & DSERR_BUFFERLOST) {
-            if (dsound_restore_out (dsb)) {
-                return -1;
-            }
-            continue;
-        }
-        break;
+    if (*statusp & DSERR_BUFFERLOST) {
+        dsound_restore_out(dsb, s);
+        return -1;
     }
 
     return 0;
@@ -376,7 +344,8 @@
     hw->rpos = pos % hw->samples;
 }
 
-static void dsound_clear_sample (HWVoiceOut *hw, LPDIRECTSOUNDBUFFER dsb)
+static void dsound_clear_sample (HWVoiceOut *hw, LPDIRECTSOUNDBUFFER dsb,
+                                 dsound *s)
 {
     int err;
     LPVOID p1, p2;
@@ -389,7 +358,8 @@
         hw->samples << hw->info.shift,
         &p1, &p2,
         &blen1, &blen2,
-        1
+        1,
+        s
         );
     if (err) {
         return;
@@ -415,25 +385,9 @@
     dsound_unlock_out (dsb, p1, p2, blen1, blen2);
 }
 
-static void dsound_close (dsound *s)
-{
-    HRESULT hr;
-
-    if (s->dsound_primary_buffer) {
-        hr = IDirectSoundBuffer_Release (s->dsound_primary_buffer);
-        if (FAILED (hr)) {
-            dsound_logerr (hr, "Could not release primary buffer\n");
-        }
-        s->dsound_primary_buffer = NULL;
-    }
-}
-
 static int dsound_open (dsound *s)
 {
-    int err;
     HRESULT hr;
-    WAVEFORMATEX wfx;
-    DSBUFFERDESC dsbd;
     HWND hwnd;
 
     hwnd = GetForegroundWindow ();
@@ -449,63 +403,7 @@
         return -1;
     }
 
-    if (!conf.set_primary) {
-        return 0;
-    }
-
-    err = waveformat_from_audio_settings (&wfx, &conf.settings);
-    if (err) {
-        return -1;
-    }
-
-    memset (&dsbd, 0, sizeof (dsbd));
-    dsbd.dwSize = sizeof (dsbd);
-    dsbd.dwFlags = DSBCAPS_PRIMARYBUFFER;
-    dsbd.dwBufferBytes = 0;
-    dsbd.lpwfxFormat = NULL;
-
-    hr = IDirectSound_CreateSoundBuffer (
-        s->dsound,
-        &dsbd,
-        &s->dsound_primary_buffer,
-        NULL
-        );
-    if (FAILED (hr)) {
-        dsound_logerr (hr, "Could not create primary playback buffer\n");
-        return -1;
-    }
-
-    hr = IDirectSoundBuffer_SetFormat (s->dsound_primary_buffer, &wfx);
-    if (FAILED (hr)) {
-        dsound_logerr (hr, "Could not set primary playback buffer format\n");
-    }
-
-    hr = IDirectSoundBuffer_GetFormat (
-        s->dsound_primary_buffer,
-        &wfx,
-        sizeof (wfx),
-        NULL
-        );
-    if (FAILED (hr)) {
-        dsound_logerr (hr, "Could not get primary playback buffer format\n");
-        goto fail0;
-    }
-
-#ifdef DEBUG_DSOUND
-    dolog ("Primary\n");
-    print_wave_format (&wfx);
-#endif
-
-    err = waveformat_to_audio_settings (&wfx, &s->settings);
-    if (err) {
-        goto fail0;
-    }
-
     return 0;
-
- fail0:
-    dsound_close (s);
-    return -1;
 }
 
 static int dsound_ctl_out (HWVoiceOut *hw, int cmd, ...)
@@ -514,6 +412,7 @@
     DWORD status;
     DSoundVoiceOut *ds = (DSoundVoiceOut *) hw;
     LPDIRECTSOUNDBUFFER dsb = ds->dsound_buffer;
+    dsound *s = ds->s;
 
     if (!dsb) {
         dolog ("Attempt to control voice without a buffer\n");
@@ -522,7 +421,7 @@
 
     switch (cmd) {
     case VOICE_ENABLE:
-        if (dsound_get_status_out (dsb, &status)) {
+        if (dsound_get_status_out (dsb, &status, s)) {
             return -1;
         }
 
@@ -531,7 +430,7 @@
             return 0;
         }
 
-        dsound_clear_sample (hw, dsb);
+        dsound_clear_sample (hw, dsb, s);
 
         hr = IDirectSoundBuffer_Play (dsb, 0, 0, DSBPLAY_LOOPING);
         if (FAILED (hr)) {
@@ -541,7 +440,7 @@
         break;
 
     case VOICE_DISABLE:
-        if (dsound_get_status_out (dsb, &status)) {
+        if (dsound_get_status_out (dsb, &status, s)) {
             return -1;
         }
 
@@ -578,6 +477,8 @@
     DWORD wpos, ppos, old_pos;
     LPVOID p1, p2;
     int bufsize;
+    dsound *s = ds->s;
+    DSoundConf *conf = &s->conf;
 
     if (!dsb) {
         dolog ("Attempt to run empty with playback buffer\n");
@@ -600,14 +501,14 @@
     len = live << hwshift;
 
     if (ds->first_time) {
-        if (conf.latency_millis) {
+        if (conf->latency_millis) {
             DWORD cur_blat;
 
             cur_blat = audio_ring_dist (wpos, ppos, bufsize);
             ds->first_time = 0;
             old_pos = wpos;
             old_pos +=
-                millis_to_bytes (&hw->info, conf.latency_millis) - cur_blat;
+                millis_to_bytes (&hw->info, conf->latency_millis) - cur_blat;
             old_pos %= bufsize;
             old_pos &= ~hw->info.align;
         }
@@ -663,7 +564,8 @@
         len,
         &p1, &p2,
         &blen1, &blen2,
-        0
+        0,
+        s
         );
     if (err) {
         return 0;
@@ -766,6 +668,7 @@
     DWORD cpos, rpos;
     LPVOID p1, p2;
     int hwshift;
+    dsound *s = ds->s;
 
     if (!dscb) {
         dolog ("Attempt to run without capture buffer\n");
@@ -820,7 +723,8 @@
         &p2,
         &blen1,
         &blen2,
-        0
+        0,
+        s
         );
     if (err) {
         return 0;
@@ -843,12 +747,19 @@
     return decr;
 }
 
+static DSoundConf glob_conf = {
+    .bufsize_in         = 16384,
+    .bufsize_out        = 16384,
+    .latency_millis     = 10
+};
+
 static void dsound_audio_fini (void *opaque)
 {
     HRESULT hr;
     dsound *s = opaque;
 
     if (!s->dsound) {
+        g_free(s);
         return;
     }
 
@@ -859,6 +770,7 @@
     s->dsound = NULL;
 
     if (!s->dsound_capture) {
+        g_free(s);
         return;
     }
 
@@ -867,17 +779,21 @@
         dsound_logerr (hr, "Could not release DirectSoundCapture\n");
     }
     s->dsound_capture = NULL;
+
+    g_free(s);
 }
 
 static void *dsound_audio_init (void)
 {
     int err;
     HRESULT hr;
-    dsound *s = &glob_dsound;
+    dsound *s = g_malloc0(sizeof(dsound));
 
+    s->conf = glob_conf;
     hr = CoInitialize (NULL);
     if (FAILED (hr)) {
         dsound_logerr (hr, "Could not initialize COM\n");
+        g_free(s);
         return NULL;
     }
 
@@ -890,6 +806,7 @@
         );
     if (FAILED (hr)) {
         dsound_logerr (hr, "Could not create DirectSound instance\n");
+        g_free(s);
         return NULL;
     }
 
@@ -901,7 +818,7 @@
         if (FAILED (hr)) {
             dsound_logerr (hr, "Could not release DirectSound\n");
         }
-        s->dsound = NULL;
+        g_free(s);
         return NULL;
     }
 
@@ -939,63 +856,21 @@
 
 static struct audio_option dsound_options[] = {
     {
-        .name  = "LOCK_RETRIES",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.lock_retries,
-        .descr = "Number of times to attempt locking the buffer"
-    },
-    {
-        .name  = "RESTOURE_RETRIES",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.restore_retries,
-        .descr = "Number of times to attempt restoring the buffer"
-    },
-    {
-        .name  = "GETSTATUS_RETRIES",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.getstatus_retries,
-        .descr = "Number of times to attempt getting status of the buffer"
-    },
-    {
-        .name  = "SET_PRIMARY",
-        .tag   = AUD_OPT_BOOL,
-        .valp  = &conf.set_primary,
-        .descr = "Set the parameters of primary buffer"
-    },
-    {
         .name  = "LATENCY_MILLIS",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.latency_millis,
+        .valp  = &glob_conf.latency_millis,
         .descr = "(undocumented)"
     },
     {
-        .name  = "PRIMARY_FREQ",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.settings.freq,
-        .descr = "Primary buffer frequency"
-    },
-    {
-        .name  = "PRIMARY_CHANNELS",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.settings.nchannels,
-        .descr = "Primary buffer number of channels (1 - mono, 2 - stereo)"
-    },
-    {
-        .name  = "PRIMARY_FMT",
-        .tag   = AUD_OPT_FMT,
-        .valp  = &conf.settings.fmt,
-        .descr = "Primary buffer format"
-    },
-    {
         .name  = "BUFSIZE_OUT",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.bufsize_out,
+        .valp  = &glob_conf.bufsize_out,
         .descr = "(undocumented)"
     },
     {
         .name  = "BUFSIZE_IN",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.bufsize_in,
+        .valp  = &glob_conf.bufsize_in,
         .descr = "(undocumented)"
     },
     { /* End of list */ }
diff --git a/audio/esdaudio.c b/audio/esdaudio.c
deleted file mode 100644
index eea9cce..0000000
--- a/audio/esdaudio.c
+++ /dev/null
@@ -1,557 +0,0 @@
-/*
- * QEMU ESD audio driver
- *
- * Copyright (c) 2006 Frederick Reeve (brushed up by malc)
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include <esd.h>
-#include "qemu-common.h"
-#include "audio.h"
-
-#define AUDIO_CAP "esd"
-#include "audio_int.h"
-#include "audio_pt_int.h"
-
-typedef struct {
-    HWVoiceOut hw;
-    int done;
-    int live;
-    int decr;
-    int rpos;
-    void *pcm_buf;
-    int fd;
-    struct audio_pt pt;
-} ESDVoiceOut;
-
-typedef struct {
-    HWVoiceIn hw;
-    int done;
-    int dead;
-    int incr;
-    int wpos;
-    void *pcm_buf;
-    int fd;
-    struct audio_pt pt;
-} ESDVoiceIn;
-
-static struct {
-    int samples;
-    int divisor;
-    char *dac_host;
-    char *adc_host;
-} conf = {
-    .samples = 1024,
-    .divisor = 2,
-};
-
-static void GCC_FMT_ATTR (2, 3) qesd_logerr (int err, const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start (ap, fmt);
-    AUD_vlog (AUDIO_CAP, fmt, ap);
-    va_end (ap);
-
-    AUD_log (AUDIO_CAP, "Reason: %s\n", strerror (err));
-}
-
-/* playback */
-static void *qesd_thread_out (void *arg)
-{
-    ESDVoiceOut *esd = arg;
-    HWVoiceOut *hw = &esd->hw;
-    int threshold;
-
-    threshold = conf.divisor ? hw->samples / conf.divisor : 0;
-
-    if (audio_pt_lock (&esd->pt, AUDIO_FUNC)) {
-        return NULL;
-    }
-
-    for (;;) {
-        int decr, to_mix, rpos;
-
-        for (;;) {
-            if (esd->done) {
-                goto exit;
-            }
-
-            if (esd->live > threshold) {
-                break;
-            }
-
-            if (audio_pt_wait (&esd->pt, AUDIO_FUNC)) {
-                goto exit;
-            }
-        }
-
-        decr = to_mix = esd->live;
-        rpos = hw->rpos;
-
-        if (audio_pt_unlock (&esd->pt, AUDIO_FUNC)) {
-            return NULL;
-        }
-
-        while (to_mix) {
-            ssize_t written;
-            int chunk = audio_MIN (to_mix, hw->samples - rpos);
-            struct st_sample *src = hw->mix_buf + rpos;
-
-            hw->clip (esd->pcm_buf, src, chunk);
-
-        again:
-            written = write (esd->fd, esd->pcm_buf, chunk << hw->info.shift);
-            if (written == -1) {
-                if (errno == EINTR || errno == EAGAIN) {
-                    goto again;
-                }
-                qesd_logerr (errno, "write failed\n");
-                return NULL;
-            }
-
-            if (written != chunk << hw->info.shift) {
-                int wsamples = written >> hw->info.shift;
-                int wbytes = wsamples << hw->info.shift;
-                if (wbytes != written) {
-                    dolog ("warning: Misaligned write %d (requested %zd), "
-                           "alignment %d\n",
-                           wbytes, written, hw->info.align + 1);
-                }
-                to_mix -= wsamples;
-                rpos = (rpos + wsamples) % hw->samples;
-                break;
-            }
-
-            rpos = (rpos + chunk) % hw->samples;
-            to_mix -= chunk;
-        }
-
-        if (audio_pt_lock (&esd->pt, AUDIO_FUNC)) {
-            return NULL;
-        }
-
-        esd->rpos = rpos;
-        esd->live -= decr;
-        esd->decr += decr;
-    }
-
- exit:
-    audio_pt_unlock (&esd->pt, AUDIO_FUNC);
-    return NULL;
-}
-
-static int qesd_run_out (HWVoiceOut *hw, int live)
-{
-    int decr;
-    ESDVoiceOut *esd = (ESDVoiceOut *) hw;
-
-    if (audio_pt_lock (&esd->pt, AUDIO_FUNC)) {
-        return 0;
-    }
-
-    decr = audio_MIN (live, esd->decr);
-    esd->decr -= decr;
-    esd->live = live - decr;
-    hw->rpos = esd->rpos;
-    if (esd->live > 0) {
-        audio_pt_unlock_and_signal (&esd->pt, AUDIO_FUNC);
-    }
-    else {
-        audio_pt_unlock (&esd->pt, AUDIO_FUNC);
-    }
-    return decr;
-}
-
-static int qesd_write (SWVoiceOut *sw, void *buf, int len)
-{
-    return audio_pcm_sw_write (sw, buf, len);
-}
-
-static int qesd_init_out (HWVoiceOut *hw, struct audsettings *as)
-{
-    ESDVoiceOut *esd = (ESDVoiceOut *) hw;
-    struct audsettings obt_as = *as;
-    int esdfmt = ESD_STREAM | ESD_PLAY;
-
-    esdfmt |= (as->nchannels == 2) ? ESD_STEREO : ESD_MONO;
-    switch (as->fmt) {
-    case AUD_FMT_S8:
-    case AUD_FMT_U8:
-        esdfmt |= ESD_BITS8;
-        obt_as.fmt = AUD_FMT_U8;
-        break;
-
-    case AUD_FMT_S32:
-    case AUD_FMT_U32:
-        dolog ("Will use 16 instead of 32 bit samples\n");
-        /* fall through */
-    case AUD_FMT_S16:
-    case AUD_FMT_U16:
-    deffmt:
-        esdfmt |= ESD_BITS16;
-        obt_as.fmt = AUD_FMT_S16;
-        break;
-
-    default:
-        dolog ("Internal logic error: Bad audio format %d\n", as->fmt);
-        goto deffmt;
-
-    }
-    obt_as.endianness = AUDIO_HOST_ENDIANNESS;
-
-    audio_pcm_init_info (&hw->info, &obt_as);
-
-    hw->samples = conf.samples;
-    esd->pcm_buf = audio_calloc (AUDIO_FUNC, hw->samples, 1 << hw->info.shift);
-    if (!esd->pcm_buf) {
-        dolog ("Could not allocate buffer (%d bytes)\n",
-               hw->samples << hw->info.shift);
-        return -1;
-    }
-
-    esd->fd = esd_play_stream (esdfmt, as->freq, conf.dac_host, NULL);
-    if (esd->fd < 0) {
-        qesd_logerr (errno, "esd_play_stream failed\n");
-        goto fail1;
-    }
-
-    if (audio_pt_init (&esd->pt, qesd_thread_out, esd, AUDIO_CAP, AUDIO_FUNC)) {
-        goto fail2;
-    }
-
-    return 0;
-
- fail2:
-    if (close (esd->fd)) {
-        qesd_logerr (errno, "%s: close on esd socket(%d) failed\n",
-                     AUDIO_FUNC, esd->fd);
-    }
-    esd->fd = -1;
-
- fail1:
-    g_free (esd->pcm_buf);
-    esd->pcm_buf = NULL;
-    return -1;
-}
-
-static void qesd_fini_out (HWVoiceOut *hw)
-{
-    void *ret;
-    ESDVoiceOut *esd = (ESDVoiceOut *) hw;
-
-    audio_pt_lock (&esd->pt, AUDIO_FUNC);
-    esd->done = 1;
-    audio_pt_unlock_and_signal (&esd->pt, AUDIO_FUNC);
-    audio_pt_join (&esd->pt, &ret, AUDIO_FUNC);
-
-    if (esd->fd >= 0) {
-        if (close (esd->fd)) {
-            qesd_logerr (errno, "failed to close esd socket\n");
-        }
-        esd->fd = -1;
-    }
-
-    audio_pt_fini (&esd->pt, AUDIO_FUNC);
-
-    g_free (esd->pcm_buf);
-    esd->pcm_buf = NULL;
-}
-
-static int qesd_ctl_out (HWVoiceOut *hw, int cmd, ...)
-{
-    (void) hw;
-    (void) cmd;
-    return 0;
-}
-
-/* capture */
-static void *qesd_thread_in (void *arg)
-{
-    ESDVoiceIn *esd = arg;
-    HWVoiceIn *hw = &esd->hw;
-    int threshold;
-
-    threshold = conf.divisor ? hw->samples / conf.divisor : 0;
-
-    if (audio_pt_lock (&esd->pt, AUDIO_FUNC)) {
-        return NULL;
-    }
-
-    for (;;) {
-        int incr, to_grab, wpos;
-
-        for (;;) {
-            if (esd->done) {
-                goto exit;
-            }
-
-            if (esd->dead > threshold) {
-                break;
-            }
-
-            if (audio_pt_wait (&esd->pt, AUDIO_FUNC)) {
-                goto exit;
-            }
-        }
-
-        incr = to_grab = esd->dead;
-        wpos = hw->wpos;
-
-        if (audio_pt_unlock (&esd->pt, AUDIO_FUNC)) {
-            return NULL;
-        }
-
-        while (to_grab) {
-            ssize_t nread;
-            int chunk = audio_MIN (to_grab, hw->samples - wpos);
-            void *buf = advance (esd->pcm_buf, wpos);
-
-        again:
-            nread = read (esd->fd, buf, chunk << hw->info.shift);
-            if (nread == -1) {
-                if (errno == EINTR || errno == EAGAIN) {
-                    goto again;
-                }
-                qesd_logerr (errno, "read failed\n");
-                return NULL;
-            }
-
-            if (nread != chunk << hw->info.shift) {
-                int rsamples = nread >> hw->info.shift;
-                int rbytes = rsamples << hw->info.shift;
-                if (rbytes != nread) {
-                    dolog ("warning: Misaligned write %d (requested %zd), "
-                           "alignment %d\n",
-                           rbytes, nread, hw->info.align + 1);
-                }
-                to_grab -= rsamples;
-                wpos = (wpos + rsamples) % hw->samples;
-                break;
-            }
-
-            hw->conv (hw->conv_buf + wpos, buf, nread >> hw->info.shift);
-            wpos = (wpos + chunk) % hw->samples;
-            to_grab -= chunk;
-        }
-
-        if (audio_pt_lock (&esd->pt, AUDIO_FUNC)) {
-            return NULL;
-        }
-
-        esd->wpos = wpos;
-        esd->dead -= incr;
-        esd->incr += incr;
-    }
-
- exit:
-    audio_pt_unlock (&esd->pt, AUDIO_FUNC);
-    return NULL;
-}
-
-static int qesd_run_in (HWVoiceIn *hw)
-{
-    int live, incr, dead;
-    ESDVoiceIn *esd = (ESDVoiceIn *) hw;
-
-    if (audio_pt_lock (&esd->pt, AUDIO_FUNC)) {
-        return 0;
-    }
-
-    live = audio_pcm_hw_get_live_in (hw);
-    dead = hw->samples - live;
-    incr = audio_MIN (dead, esd->incr);
-    esd->incr -= incr;
-    esd->dead = dead - incr;
-    hw->wpos = esd->wpos;
-    if (esd->dead > 0) {
-        audio_pt_unlock_and_signal (&esd->pt, AUDIO_FUNC);
-    }
-    else {
-        audio_pt_unlock (&esd->pt, AUDIO_FUNC);
-    }
-    return incr;
-}
-
-static int qesd_read (SWVoiceIn *sw, void *buf, int len)
-{
-    return audio_pcm_sw_read (sw, buf, len);
-}
-
-static int qesd_init_in (HWVoiceIn *hw, struct audsettings *as)
-{
-    ESDVoiceIn *esd = (ESDVoiceIn *) hw;
-    struct audsettings obt_as = *as;
-    int esdfmt = ESD_STREAM | ESD_RECORD;
-
-    esdfmt |= (as->nchannels == 2) ? ESD_STEREO : ESD_MONO;
-    switch (as->fmt) {
-    case AUD_FMT_S8:
-    case AUD_FMT_U8:
-        esdfmt |= ESD_BITS8;
-        obt_as.fmt = AUD_FMT_U8;
-        break;
-
-    case AUD_FMT_S16:
-    case AUD_FMT_U16:
-        esdfmt |= ESD_BITS16;
-        obt_as.fmt = AUD_FMT_S16;
-        break;
-
-    case AUD_FMT_S32:
-    case AUD_FMT_U32:
-        dolog ("Will use 16 instead of 32 bit samples\n");
-        esdfmt |= ESD_BITS16;
-        obt_as.fmt = AUD_FMT_S16;
-        break;
-    }
-    obt_as.endianness = AUDIO_HOST_ENDIANNESS;
-
-    audio_pcm_init_info (&hw->info, &obt_as);
-
-    hw->samples = conf.samples;
-    esd->pcm_buf = audio_calloc (AUDIO_FUNC, hw->samples, 1 << hw->info.shift);
-    if (!esd->pcm_buf) {
-        dolog ("Could not allocate buffer (%d bytes)\n",
-               hw->samples << hw->info.shift);
-        return -1;
-    }
-
-    esd->fd = esd_record_stream (esdfmt, as->freq, conf.adc_host, NULL);
-    if (esd->fd < 0) {
-        qesd_logerr (errno, "esd_record_stream failed\n");
-        goto fail1;
-    }
-
-    if (audio_pt_init (&esd->pt, qesd_thread_in, esd, AUDIO_CAP, AUDIO_FUNC)) {
-        goto fail2;
-    }
-
-    return 0;
-
- fail2:
-    if (close (esd->fd)) {
-        qesd_logerr (errno, "%s: close on esd socket(%d) failed\n",
-                     AUDIO_FUNC, esd->fd);
-    }
-    esd->fd = -1;
-
- fail1:
-    g_free (esd->pcm_buf);
-    esd->pcm_buf = NULL;
-    return -1;
-}
-
-static void qesd_fini_in (HWVoiceIn *hw)
-{
-    void *ret;
-    ESDVoiceIn *esd = (ESDVoiceIn *) hw;
-
-    audio_pt_lock (&esd->pt, AUDIO_FUNC);
-    esd->done = 1;
-    audio_pt_unlock_and_signal (&esd->pt, AUDIO_FUNC);
-    audio_pt_join (&esd->pt, &ret, AUDIO_FUNC);
-
-    if (esd->fd >= 0) {
-        if (close (esd->fd)) {
-            qesd_logerr (errno, "failed to close esd socket\n");
-        }
-        esd->fd = -1;
-    }
-
-    audio_pt_fini (&esd->pt, AUDIO_FUNC);
-
-    g_free (esd->pcm_buf);
-    esd->pcm_buf = NULL;
-}
-
-static int qesd_ctl_in (HWVoiceIn *hw, int cmd, ...)
-{
-    (void) hw;
-    (void) cmd;
-    return 0;
-}
-
-/* common */
-static void *qesd_audio_init (void)
-{
-    return &conf;
-}
-
-static void qesd_audio_fini (void *opaque)
-{
-    (void) opaque;
-    ldebug ("esd_fini");
-}
-
-struct audio_option qesd_options[] = {
-    {
-        .name  = "SAMPLES",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.samples,
-        .descr = "buffer size in samples"
-    },
-    {
-        .name  = "DIVISOR",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.divisor,
-        .descr = "threshold divisor"
-    },
-    {
-        .name  = "DAC_HOST",
-        .tag   = AUD_OPT_STR,
-        .valp  = &conf.dac_host,
-        .descr = "playback host"
-    },
-    {
-        .name  = "ADC_HOST",
-        .tag   = AUD_OPT_STR,
-        .valp  = &conf.adc_host,
-        .descr = "capture host"
-    },
-    { /* End of list */ }
-};
-
-static struct audio_pcm_ops qesd_pcm_ops = {
-    .init_out = qesd_init_out,
-    .fini_out = qesd_fini_out,
-    .run_out  = qesd_run_out,
-    .write    = qesd_write,
-    .ctl_out  = qesd_ctl_out,
-
-    .init_in  = qesd_init_in,
-    .fini_in  = qesd_fini_in,
-    .run_in   = qesd_run_in,
-    .read     = qesd_read,
-    .ctl_in   = qesd_ctl_in,
-};
-
-struct audio_driver esd_audio_driver = {
-    .name           = "esd",
-    .descr          = "http://en.wikipedia.org/wiki/Esound",
-    .options        = qesd_options,
-    .init           = qesd_audio_init,
-    .fini           = qesd_audio_fini,
-    .pcm_ops        = &qesd_pcm_ops,
-    .can_be_default = 0,
-    .max_voices_out = INT_MAX,
-    .max_voices_in  = INT_MAX,
-    .voice_size_out = sizeof (ESDVoiceOut),
-    .voice_size_in  = sizeof (ESDVoiceIn)
-};
diff --git a/audio/fmodaudio.c b/audio/fmodaudio.c
deleted file mode 100644
index fabf84d..0000000
--- a/audio/fmodaudio.c
+++ /dev/null
@@ -1,685 +0,0 @@
-/*
- * QEMU FMOD audio driver
- *
- * Copyright (c) 2004-2005 Vassili Karpov (malc)
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include <fmod.h>
-#include <fmod_errors.h>
-#include "qemu-common.h"
-#include "audio.h"
-
-#define AUDIO_CAP "fmod"
-#include "audio_int.h"
-
-typedef struct FMODVoiceOut {
-    HWVoiceOut hw;
-    unsigned int old_pos;
-    FSOUND_SAMPLE *fmod_sample;
-    int channel;
-} FMODVoiceOut;
-
-typedef struct FMODVoiceIn {
-    HWVoiceIn hw;
-    FSOUND_SAMPLE *fmod_sample;
-} FMODVoiceIn;
-
-static struct {
-    const char *drvname;
-    int nb_samples;
-    int freq;
-    int nb_channels;
-    int bufsize;
-    int broken_adc;
-} conf = {
-    .nb_samples  = 2048 * 2,
-    .freq        = 44100,
-    .nb_channels = 2,
-};
-
-static void GCC_FMT_ATTR (1, 2) fmod_logerr (const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start (ap, fmt);
-    AUD_vlog (AUDIO_CAP, fmt, ap);
-    va_end (ap);
-
-    AUD_log (AUDIO_CAP, "Reason: %s\n",
-             FMOD_ErrorString (FSOUND_GetError ()));
-}
-
-static void GCC_FMT_ATTR (2, 3) fmod_logerr2 (
-    const char *typ,
-    const char *fmt,
-    ...
-    )
-{
-    va_list ap;
-
-    AUD_log (AUDIO_CAP, "Could not initialize %s\n", typ);
-
-    va_start (ap, fmt);
-    AUD_vlog (AUDIO_CAP, fmt, ap);
-    va_end (ap);
-
-    AUD_log (AUDIO_CAP, "Reason: %s\n",
-             FMOD_ErrorString (FSOUND_GetError ()));
-}
-
-static int fmod_write (SWVoiceOut *sw, void *buf, int len)
-{
-    return audio_pcm_sw_write (sw, buf, len);
-}
-
-static void fmod_clear_sample (FMODVoiceOut *fmd)
-{
-    HWVoiceOut *hw = &fmd->hw;
-    int status;
-    void *p1 = 0, *p2 = 0;
-    unsigned int len1 = 0, len2 = 0;
-
-    status = FSOUND_Sample_Lock (
-        fmd->fmod_sample,
-        0,
-        hw->samples << hw->info.shift,
-        &p1,
-        &p2,
-        &len1,
-        &len2
-        );
-
-    if (!status) {
-        fmod_logerr ("Failed to lock sample\n");
-        return;
-    }
-
-    if ((len1 & hw->info.align) || (len2 & hw->info.align)) {
-        dolog ("Lock returned misaligned length %d, %d, alignment %d\n",
-               len1, len2, hw->info.align + 1);
-        goto fail;
-    }
-
-    if ((len1 + len2) - (hw->samples << hw->info.shift)) {
-        dolog ("Lock returned incomplete length %d, %d\n",
-               len1 + len2, hw->samples << hw->info.shift);
-        goto fail;
-    }
-
-    audio_pcm_info_clear_buf (&hw->info, p1, hw->samples);
-
- fail:
-    status = FSOUND_Sample_Unlock (fmd->fmod_sample, p1, p2, len1, len2);
-    if (!status) {
-        fmod_logerr ("Failed to unlock sample\n");
-    }
-}
-
-static void fmod_write_sample (HWVoiceOut *hw, uint8_t *dst, int dst_len)
-{
-    int src_len1 = dst_len;
-    int src_len2 = 0;
-    int pos = hw->rpos + dst_len;
-    struct st_sample *src1 = hw->mix_buf + hw->rpos;
-    struct st_sample *src2 = NULL;
-
-    if (pos > hw->samples) {
-        src_len1 = hw->samples - hw->rpos;
-        src2 = hw->mix_buf;
-        src_len2 = dst_len - src_len1;
-        pos = src_len2;
-    }
-
-    if (src_len1) {
-        hw->clip (dst, src1, src_len1);
-    }
-
-    if (src_len2) {
-        dst = advance (dst, src_len1 << hw->info.shift);
-        hw->clip (dst, src2, src_len2);
-    }
-
-    hw->rpos = pos % hw->samples;
-}
-
-static int fmod_unlock_sample (FSOUND_SAMPLE *sample, void *p1, void *p2,
-                               unsigned int blen1, unsigned int blen2)
-{
-    int status = FSOUND_Sample_Unlock (sample, p1, p2, blen1, blen2);
-    if (!status) {
-        fmod_logerr ("Failed to unlock sample\n");
-        return -1;
-    }
-    return 0;
-}
-
-static int fmod_lock_sample (
-    FSOUND_SAMPLE *sample,
-    struct audio_pcm_info *info,
-    int pos,
-    int len,
-    void **p1,
-    void **p2,
-    unsigned int *blen1,
-    unsigned int *blen2
-    )
-{
-    int status;
-
-    status = FSOUND_Sample_Lock (
-        sample,
-        pos << info->shift,
-        len << info->shift,
-        p1,
-        p2,
-        blen1,
-        blen2
-        );
-
-    if (!status) {
-        fmod_logerr ("Failed to lock sample\n");
-        return -1;
-    }
-
-    if ((*blen1 & info->align) || (*blen2 & info->align)) {
-        dolog ("Lock returned misaligned length %d, %d, alignment %d\n",
-               *blen1, *blen2, info->align + 1);
-
-        fmod_unlock_sample (sample, *p1, *p2, *blen1, *blen2);
-
-        *p1 = NULL - 1;
-        *p2 = NULL - 1;
-        *blen1 = ~0U;
-        *blen2 = ~0U;
-        return -1;
-    }
-
-    if (!*p1 && *blen1) {
-        dolog ("warning: !p1 && blen1=%d\n", *blen1);
-        *blen1 = 0;
-    }
-
-    if (!p2 && *blen2) {
-        dolog ("warning: !p2 && blen2=%d\n", *blen2);
-        *blen2 = 0;
-    }
-
-    return 0;
-}
-
-static int fmod_run_out (HWVoiceOut *hw, int live)
-{
-    FMODVoiceOut *fmd = (FMODVoiceOut *) hw;
-    int decr;
-    void *p1 = 0, *p2 = 0;
-    unsigned int blen1 = 0, blen2 = 0;
-    unsigned int len1 = 0, len2 = 0;
-
-    if (!hw->pending_disable) {
-        return 0;
-    }
-
-    decr = live;
-
-    if (fmd->channel >= 0) {
-        int len = decr;
-        int old_pos = fmd->old_pos;
-        int ppos = FSOUND_GetCurrentPosition (fmd->channel);
-
-        if (ppos == old_pos || !ppos) {
-            return 0;
-        }
-
-        if ((old_pos < ppos) && ((old_pos + len) > ppos)) {
-            len = ppos - old_pos;
-        }
-        else {
-            if ((old_pos > ppos) && ((old_pos + len) > (ppos + hw->samples))) {
-                len = hw->samples - old_pos + ppos;
-            }
-        }
-        decr = len;
-
-        if (audio_bug (AUDIO_FUNC, decr < 0)) {
-            dolog ("decr=%d live=%d ppos=%d old_pos=%d len=%d\n",
-                   decr, live, ppos, old_pos, len);
-            return 0;
-        }
-    }
-
-
-    if (!decr) {
-        return 0;
-    }
-
-    if (fmod_lock_sample (fmd->fmod_sample, &fmd->hw.info,
-                          fmd->old_pos, decr,
-                          &p1, &p2,
-                          &blen1, &blen2)) {
-        return 0;
-    }
-
-    len1 = blen1 >> hw->info.shift;
-    len2 = blen2 >> hw->info.shift;
-    ldebug ("%p %p %d %d %d %d\n", p1, p2, len1, len2, blen1, blen2);
-    decr = len1 + len2;
-
-    if (p1 && len1) {
-        fmod_write_sample (hw, p1, len1);
-    }
-
-    if (p2 && len2) {
-        fmod_write_sample (hw, p2, len2);
-    }
-
-    fmod_unlock_sample (fmd->fmod_sample, p1, p2, blen1, blen2);
-
-    fmd->old_pos = (fmd->old_pos + decr) % hw->samples;
-    return decr;
-}
-
-static int aud_to_fmodfmt (audfmt_e fmt, int stereo)
-{
-    int mode = FSOUND_LOOP_NORMAL;
-
-    switch (fmt) {
-    case AUD_FMT_S8:
-        mode |= FSOUND_SIGNED | FSOUND_8BITS;
-        break;
-
-    case AUD_FMT_U8:
-        mode |= FSOUND_UNSIGNED | FSOUND_8BITS;
-        break;
-
-    case AUD_FMT_S16:
-        mode |= FSOUND_SIGNED | FSOUND_16BITS;
-        break;
-
-    case AUD_FMT_U16:
-        mode |= FSOUND_UNSIGNED | FSOUND_16BITS;
-        break;
-
-    default:
-        dolog ("Internal logic error: Bad audio format %d\n", fmt);
-#ifdef DEBUG_FMOD
-        abort ();
-#endif
-        mode |= FSOUND_8BITS;
-    }
-    mode |= stereo ? FSOUND_STEREO : FSOUND_MONO;
-    return mode;
-}
-
-static void fmod_fini_out (HWVoiceOut *hw)
-{
-    FMODVoiceOut *fmd = (FMODVoiceOut *) hw;
-
-    if (fmd->fmod_sample) {
-        FSOUND_Sample_Free (fmd->fmod_sample);
-        fmd->fmod_sample = 0;
-
-        if (fmd->channel >= 0) {
-            FSOUND_StopSound (fmd->channel);
-        }
-    }
-}
-
-static int fmod_init_out (HWVoiceOut *hw, struct audsettings *as)
-{
-    int mode, channel;
-    FMODVoiceOut *fmd = (FMODVoiceOut *) hw;
-    struct audsettings obt_as = *as;
-
-    mode = aud_to_fmodfmt (as->fmt, as->nchannels == 2 ? 1 : 0);
-    fmd->fmod_sample = FSOUND_Sample_Alloc (
-        FSOUND_FREE,            /* index */
-        conf.nb_samples,        /* length */
-        mode,                   /* mode */
-        as->freq,               /* freq */
-        255,                    /* volume */
-        128,                    /* pan */
-        255                     /* priority */
-        );
-
-    if (!fmd->fmod_sample) {
-        fmod_logerr2 ("DAC", "Failed to allocate FMOD sample\n");
-        return -1;
-    }
-
-    channel = FSOUND_PlaySoundEx (FSOUND_FREE, fmd->fmod_sample, 0, 1);
-    if (channel < 0) {
-        fmod_logerr2 ("DAC", "Failed to start playing sound\n");
-        FSOUND_Sample_Free (fmd->fmod_sample);
-        return -1;
-    }
-    fmd->channel = channel;
-
-    /* FMOD always operates on little endian frames? */
-    obt_as.endianness = 0;
-    audio_pcm_init_info (&hw->info, &obt_as);
-    hw->samples = conf.nb_samples;
-    return 0;
-}
-
-static int fmod_ctl_out (HWVoiceOut *hw, int cmd, ...)
-{
-    int status;
-    FMODVoiceOut *fmd = (FMODVoiceOut *) hw;
-
-    switch (cmd) {
-    case VOICE_ENABLE:
-        fmod_clear_sample (fmd);
-        status = FSOUND_SetPaused (fmd->channel, 0);
-        if (!status) {
-            fmod_logerr ("Failed to resume channel %d\n", fmd->channel);
-        }
-        break;
-
-    case VOICE_DISABLE:
-        status = FSOUND_SetPaused (fmd->channel, 1);
-        if (!status) {
-            fmod_logerr ("Failed to pause channel %d\n", fmd->channel);
-        }
-        break;
-    }
-    return 0;
-}
-
-static int fmod_init_in (HWVoiceIn *hw, struct audsettings *as)
-{
-    int mode;
-    FMODVoiceIn *fmd = (FMODVoiceIn *) hw;
-    struct audsettings obt_as = *as;
-
-    if (conf.broken_adc) {
-        return -1;
-    }
-
-    mode = aud_to_fmodfmt (as->fmt, as->nchannels == 2 ? 1 : 0);
-    fmd->fmod_sample = FSOUND_Sample_Alloc (
-        FSOUND_FREE,            /* index */
-        conf.nb_samples,        /* length */
-        mode,                   /* mode */
-        as->freq,               /* freq */
-        255,                    /* volume */
-        128,                    /* pan */
-        255                     /* priority */
-        );
-
-    if (!fmd->fmod_sample) {
-        fmod_logerr2 ("ADC", "Failed to allocate FMOD sample\n");
-        return -1;
-    }
-
-    /* FMOD always operates on little endian frames? */
-    obt_as.endianness = 0;
-    audio_pcm_init_info (&hw->info, &obt_as);
-    hw->samples = conf.nb_samples;
-    return 0;
-}
-
-static void fmod_fini_in (HWVoiceIn *hw)
-{
-    FMODVoiceIn *fmd = (FMODVoiceIn *) hw;
-
-    if (fmd->fmod_sample) {
-        FSOUND_Record_Stop ();
-        FSOUND_Sample_Free (fmd->fmod_sample);
-        fmd->fmod_sample = 0;
-    }
-}
-
-static int fmod_run_in (HWVoiceIn *hw)
-{
-    FMODVoiceIn *fmd = (FMODVoiceIn *) hw;
-    int hwshift = hw->info.shift;
-    int live, dead, new_pos, len;
-    unsigned int blen1 = 0, blen2 = 0;
-    unsigned int len1, len2;
-    unsigned int decr;
-    void *p1, *p2;
-
-    live = audio_pcm_hw_get_live_in (hw);
-    dead = hw->samples - live;
-    if (!dead) {
-        return 0;
-    }
-
-    new_pos = FSOUND_Record_GetPosition ();
-    if (new_pos < 0) {
-        fmod_logerr ("Could not get recording position\n");
-        return 0;
-    }
-
-    len = audio_ring_dist (new_pos,  hw->wpos, hw->samples);
-    if (!len) {
-        return 0;
-    }
-    len = audio_MIN (len, dead);
-
-    if (fmod_lock_sample (fmd->fmod_sample, &fmd->hw.info,
-                          hw->wpos, len,
-                          &p1, &p2,
-                          &blen1, &blen2)) {
-        return 0;
-    }
-
-    len1 = blen1 >> hwshift;
-    len2 = blen2 >> hwshift;
-    decr = len1 + len2;
-
-    if (p1 && blen1) {
-        hw->conv (hw->conv_buf + hw->wpos, p1, len1);
-    }
-    if (p2 && len2) {
-        hw->conv (hw->conv_buf, p2, len2);
-    }
-
-    fmod_unlock_sample (fmd->fmod_sample, p1, p2, blen1, blen2);
-    hw->wpos = (hw->wpos + decr) % hw->samples;
-    return decr;
-}
-
-static struct {
-    const char *name;
-    int type;
-} drvtab[] = {
-    { .name = "none",   .type = FSOUND_OUTPUT_NOSOUND },
-#ifdef _WIN32
-    { .name = "winmm",  .type = FSOUND_OUTPUT_WINMM   },
-    { .name = "dsound", .type = FSOUND_OUTPUT_DSOUND  },
-    { .name = "a3d",    .type = FSOUND_OUTPUT_A3D     },
-    { .name = "asio",   .type = FSOUND_OUTPUT_ASIO    },
-#endif
-#ifdef __linux__
-    { .name = "oss",    .type = FSOUND_OUTPUT_OSS     },
-    { .name = "alsa",   .type = FSOUND_OUTPUT_ALSA    },
-    { .name = "esd",    .type = FSOUND_OUTPUT_ESD     },
-#endif
-#ifdef __APPLE__
-    { .name = "mac",    .type = FSOUND_OUTPUT_MAC     },
-#endif
-#if 0
-    { .name = "xbox",   .type = FSOUND_OUTPUT_XBOX    },
-    { .name = "ps2",    .type = FSOUND_OUTPUT_PS2     },
-    { .name = "gcube",  .type = FSOUND_OUTPUT_GC      },
-#endif
-    { .name = "none-realtime", .type = FSOUND_OUTPUT_NOSOUND_NONREALTIME }
-};
-
-static void *fmod_audio_init (void)
-{
-    size_t i;
-    double ver;
-    int status;
-    int output_type = -1;
-    const char *drv = conf.drvname;
-
-    ver = FSOUND_GetVersion ();
-    if (ver < FMOD_VERSION) {
-        dolog ("Wrong FMOD version %f, need at least %f\n", ver, FMOD_VERSION);
-        return NULL;
-    }
-
-#ifdef __linux__
-    if (ver < 3.75) {
-        dolog ("FMOD before 3.75 has bug preventing ADC from working\n"
-               "ADC will be disabled.\n");
-        conf.broken_adc = 1;
-    }
-#endif
-
-    if (drv) {
-        int found = 0;
-        for (i = 0; i < ARRAY_SIZE (drvtab); i++) {
-            if (!strcmp (drv, drvtab[i].name)) {
-                output_type = drvtab[i].type;
-                found = 1;
-                break;
-            }
-        }
-        if (!found) {
-            dolog ("Unknown FMOD driver `%s'\n", drv);
-            dolog ("Valid drivers:\n");
-            for (i = 0; i < ARRAY_SIZE (drvtab); i++) {
-                dolog ("  %s\n", drvtab[i].name);
-            }
-        }
-    }
-
-    if (output_type != -1) {
-        status = FSOUND_SetOutput (output_type);
-        if (!status) {
-            fmod_logerr ("FSOUND_SetOutput(%d) failed\n", output_type);
-            return NULL;
-        }
-    }
-
-    if (conf.bufsize) {
-        status = FSOUND_SetBufferSize (conf.bufsize);
-        if (!status) {
-            fmod_logerr ("FSOUND_SetBufferSize (%d) failed\n", conf.bufsize);
-        }
-    }
-
-    status = FSOUND_Init (conf.freq, conf.nb_channels, 0);
-    if (!status) {
-        fmod_logerr ("FSOUND_Init failed\n");
-        return NULL;
-    }
-
-    return &conf;
-}
-
-static int fmod_read (SWVoiceIn *sw, void *buf, int size)
-{
-    return audio_pcm_sw_read (sw, buf, size);
-}
-
-static int fmod_ctl_in (HWVoiceIn *hw, int cmd, ...)
-{
-    int status;
-    FMODVoiceIn *fmd = (FMODVoiceIn *) hw;
-
-    switch (cmd) {
-    case VOICE_ENABLE:
-        status = FSOUND_Record_StartSample (fmd->fmod_sample, 1);
-        if (!status) {
-            fmod_logerr ("Failed to start recording\n");
-        }
-        break;
-
-    case VOICE_DISABLE:
-        status = FSOUND_Record_Stop ();
-        if (!status) {
-            fmod_logerr ("Failed to stop recording\n");
-        }
-        break;
-    }
-    return 0;
-}
-
-static void fmod_audio_fini (void *opaque)
-{
-    (void) opaque;
-    FSOUND_Close ();
-}
-
-static struct audio_option fmod_options[] = {
-    {
-        .name  = "DRV",
-        .tag   = AUD_OPT_STR,
-        .valp  = &conf.drvname,
-        .descr = "FMOD driver"
-    },
-    {
-        .name  = "FREQ",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.freq,
-        .descr = "Default frequency"
-    },
-    {
-        .name  = "SAMPLES",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.nb_samples,
-        .descr = "Buffer size in samples"
-    },
-    {
-        .name  = "CHANNELS",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.nb_channels,
-        .descr = "Number of default channels (1 - mono, 2 - stereo)"
-    },
-    {
-        .name  = "BUFSIZE",
-        .tag   = AUD_OPT_INT,
-        .valp  = &conf.bufsize,
-        .descr = "(undocumented)"
-    },
-    { /* End of list */ }
-};
-
-static struct audio_pcm_ops fmod_pcm_ops = {
-    .init_out = fmod_init_out,
-    .fini_out = fmod_fini_out,
-    .run_out  = fmod_run_out,
-    .write    = fmod_write,
-    .ctl_out  = fmod_ctl_out,
-
-    .init_in  = fmod_init_in,
-    .fini_in  = fmod_fini_in,
-    .run_in   = fmod_run_in,
-    .read     = fmod_read,
-    .ctl_in   = fmod_ctl_in
-};
-
-struct audio_driver fmod_audio_driver = {
-    .name           = "fmod",
-    .descr          = "FMOD 3.xx http://www.fmod.org",
-    .options        = fmod_options,
-    .init           = fmod_audio_init,
-    .fini           = fmod_audio_fini,
-    .pcm_ops        = &fmod_pcm_ops,
-    .can_be_default = 1,
-    .max_voices_out = INT_MAX,
-    .max_voices_in  = INT_MAX,
-    .voice_size_out = sizeof (FMODVoiceOut),
-    .voice_size_in  = sizeof (FMODVoiceIn)
-};
diff --git a/audio/mixeng.c b/audio/mixeng.c
index 0e4976f..66c0328 100644
--- a/audio/mixeng.c
+++ b/audio/mixeng.c
@@ -22,7 +22,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/bswap.h"
 #include "audio.h"
 
 #define AUDIO_CAP "mixeng"
@@ -269,7 +271,7 @@
  * August 21, 1998
  * Copyright 1998 Fabrice Bellard.
  *
- * [Rewrote completly the code of Lance Norskog And Sundry
+ * [Rewrote completely the code of Lance Norskog And Sundry
  * Contributors with a more efficient algorithm.]
  *
  * This source code is freely redistributable and may be used for
diff --git a/audio/mixeng.h b/audio/mixeng.h
index 9de443b..b53a5ef 100644
--- a/audio/mixeng.h
+++ b/audio/mixeng.h
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef QEMU_MIXENG_H
 #define QEMU_MIXENG_H
 
@@ -48,4 +49,4 @@
 void mixeng_clear (struct st_sample *buf, int len);
 void mixeng_volume (struct st_sample *buf, int len, struct mixeng_volume *vol);
 
-#endif  /* mixeng.h */
+#endif /* QEMU_MIXENG_H */
diff --git a/audio/noaudio.c b/audio/noaudio.c
index cb38662..9ca9eaf 100644
--- a/audio/noaudio.c
+++ b/audio/noaudio.c
@@ -21,7 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/host-utils.h"
 #include "audio.h"
 #include "qemu/timer.h"
 
@@ -48,8 +50,8 @@
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     ticks = now - no->old_ticks;
-    bytes = muldiv64 (ticks, hw->info.bytes_per_second, get_ticks_per_sec ());
-    bytes = audio_MIN (bytes, INT_MAX);
+    bytes = muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
+    bytes = audio_MIN(bytes, INT_MAX);
     samples = bytes >> hw->info.shift;
 
     no->old_ticks = now;
@@ -60,10 +62,10 @@
 
 static int no_write (SWVoiceOut *sw, void *buf, int len)
 {
-    return audio_pcm_sw_write (sw, buf, len);
+    return audio_pcm_sw_write(sw, buf, len);
 }
 
-static int no_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int no_init_out(HWVoiceOut *hw, struct audsettings *as, void *drv_opaque)
 {
     audio_pcm_init_info (&hw->info, as);
     hw->samples = 1024;
@@ -82,7 +84,7 @@
     return 0;
 }
 
-static int no_init_in (HWVoiceIn *hw, struct audsettings *as)
+static int no_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
 {
     audio_pcm_init_info (&hw->info, as);
     hw->samples = 1024;
@@ -105,7 +107,7 @@
         int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
         int64_t ticks = now - no->old_ticks;
         int64_t bytes =
-            muldiv64 (ticks, hw->info.bytes_per_second, get_ticks_per_sec ());
+            muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
 
         no->old_ticks = now;
         bytes = audio_MIN (bytes, INT_MAX);
diff --git a/audio/ossaudio.c b/audio/ossaudio.c
index 4db2ca6..0edd7ea 100644
--- a/audio/ossaudio.c
+++ b/audio/ossaudio.c
@@ -21,15 +21,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/soundcard.h>
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 #include "qemu/host-utils.h"
 #include "audio.h"
+#include "trace.h"
 
 #define AUDIO_CAP "oss"
 #include "audio_int.h"
@@ -38,6 +37,16 @@
 #define USE_DSP_POLICY
 #endif
 
+typedef struct OSSConf {
+    int try_mmap;
+    int nfrags;
+    int fragsize;
+    const char *devpath_out;
+    const char *devpath_in;
+    int exclusive;
+    int policy;
+} OSSConf;
+
 typedef struct OSSVoiceOut {
     HWVoiceOut hw;
     void *pcm_buf;
@@ -47,6 +56,7 @@
     int fragsize;
     int mmapped;
     int pending;
+    OSSConf *conf;
 } OSSVoiceOut;
 
 typedef struct OSSVoiceIn {
@@ -55,28 +65,9 @@
     int fd;
     int nfrags;
     int fragsize;
+    OSSConf *conf;
 } OSSVoiceIn;
 
-static struct {
-    int try_mmap;
-    int nfrags;
-    int fragsize;
-    const char *devpath_out;
-    const char *devpath_in;
-    int debug;
-    int exclusive;
-    int policy;
-} conf = {
-    .try_mmap = 0,
-    .nfrags = 4,
-    .fragsize = 4096,
-    .devpath_out = "/dev/dsp",
-    .devpath_in = "/dev/dsp",
-    .debug = 0,
-    .exclusive = 0,
-    .policy = 5
-};
-
 struct oss_params {
     int freq;
     audfmt_e fmt;
@@ -138,18 +129,18 @@
     audio_run ("oss_poll_in");
 }
 
-static int oss_poll_out (HWVoiceOut *hw)
+static void oss_poll_out (HWVoiceOut *hw)
 {
     OSSVoiceOut *oss = (OSSVoiceOut *) hw;
 
-    return qemu_set_fd_handler (oss->fd, NULL, oss_helper_poll_out, NULL);
+    qemu_set_fd_handler (oss->fd, NULL, oss_helper_poll_out, NULL);
 }
 
-static int oss_poll_in (HWVoiceIn *hw)
+static void oss_poll_in (HWVoiceIn *hw)
 {
     OSSVoiceIn *oss = (OSSVoiceIn *) hw;
 
-    return qemu_set_fd_handler (oss->fd, oss_helper_poll_in, NULL, NULL);
+    qemu_set_fd_handler (oss->fd, oss_helper_poll_in, NULL, NULL);
 }
 
 static int oss_write (SWVoiceOut *sw, void *buf, int len)
@@ -272,18 +263,18 @@
 #endif
 
 static int oss_open (int in, struct oss_params *req,
-                     struct oss_params *obt, int *pfd)
+                     struct oss_params *obt, int *pfd, OSSConf* conf)
 {
     int fd;
-    int oflags = conf.exclusive ? O_EXCL : 0;
+    int oflags = conf->exclusive ? O_EXCL : 0;
     audio_buf_info abinfo;
     int fmt, freq, nchannels;
     int setfragment = 1;
-    const char *dspname = in ? conf.devpath_in : conf.devpath_out;
+    const char *dspname = in ? conf->devpath_in : conf->devpath_out;
     const char *typ = in ? "ADC" : "DAC";
 
     /* Kludge needed to have working mmap on Linux */
-    oflags |= conf.try_mmap ? O_RDWR : (in ? O_RDONLY : O_WRONLY);
+    oflags |= conf->try_mmap ? O_RDWR : (in ? O_RDONLY : O_WRONLY);
 
     fd = open (dspname, oflags | O_NONBLOCK);
     if (-1 == fd) {
@@ -317,20 +308,18 @@
     }
 
 #ifdef USE_DSP_POLICY
-    if (conf.policy >= 0) {
+    if (conf->policy >= 0) {
         int version;
 
         if (!oss_get_version (fd, &version, typ)) {
-            if (conf.debug) {
-                dolog ("OSS version = %#x\n", version);
-            }
+            trace_oss_version(version);
 
             if (version >= 0x040000) {
-                int policy = conf.policy;
+                int policy = conf->policy;
                 if (ioctl (fd, SNDCTL_DSP_POLICY, &policy)) {
                     oss_logerr2 (errno, typ,
                                  "Failed to set timing policy to %d\n",
-                                 conf.policy);
+                                 conf->policy);
                     goto err;
                 }
                 setfragment = 0;
@@ -458,19 +447,12 @@
         }
 
         if (abinfo.bytes > bufsize) {
-            if (conf.debug) {
-                dolog ("warning: Invalid available size, size=%d bufsize=%d\n"
-                       "please report your OS/audio hw to av1474@comtv.ru\n",
-                       abinfo.bytes, bufsize);
-            }
+            trace_oss_invalid_available_size(abinfo.bytes, bufsize);
             abinfo.bytes = bufsize;
         }
 
         if (abinfo.bytes < 0) {
-            if (conf.debug) {
-                dolog ("warning: Invalid available size, size=%d bufsize=%d\n",
-                       abinfo.bytes, bufsize);
-            }
+            trace_oss_invalid_available_size(abinfo.bytes, bufsize);
             return 0;
         }
 
@@ -510,7 +492,8 @@
     }
 }
 
-static int oss_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int oss_init_out(HWVoiceOut *hw, struct audsettings *as,
+                        void *drv_opaque)
 {
     OSSVoiceOut *oss = (OSSVoiceOut *) hw;
     struct oss_params req, obt;
@@ -519,16 +502,17 @@
     int fd;
     audfmt_e effective_fmt;
     struct audsettings obt_as;
+    OSSConf *conf = drv_opaque;
 
     oss->fd = -1;
 
     req.fmt = aud_to_ossfmt (as->fmt, as->endianness);
     req.freq = as->freq;
     req.nchannels = as->nchannels;
-    req.fragsize = conf.fragsize;
-    req.nfrags = conf.nfrags;
+    req.fragsize = conf->fragsize;
+    req.nfrags = conf->nfrags;
 
-    if (oss_open (0, &req, &obt, &fd)) {
+    if (oss_open (0, &req, &obt, &fd, conf)) {
         return -1;
     }
 
@@ -555,7 +539,7 @@
     hw->samples = (obt.nfrags * obt.fragsize) >> hw->info.shift;
 
     oss->mmapped = 0;
-    if (conf.try_mmap) {
+    if (conf->try_mmap) {
         oss->pcm_buf = mmap (
             NULL,
             hw->samples << hw->info.shift,
@@ -615,6 +599,7 @@
     }
 
     oss->fd = fd;
+    oss->conf = conf;
     return 0;
 }
 
@@ -634,7 +619,8 @@
             va_end (ap);
 
             ldebug ("enabling voice\n");
-            if (poll_mode && oss_poll_out (hw)) {
+            if (poll_mode) {
+                oss_poll_out (hw);
                 poll_mode = 0;
             }
             hw->poll_mode = poll_mode;
@@ -676,7 +662,7 @@
     return 0;
 }
 
-static int oss_init_in (HWVoiceIn *hw, struct audsettings *as)
+static int oss_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
 {
     OSSVoiceIn *oss = (OSSVoiceIn *) hw;
     struct oss_params req, obt;
@@ -685,15 +671,16 @@
     int fd;
     audfmt_e effective_fmt;
     struct audsettings obt_as;
+    OSSConf *conf = drv_opaque;
 
     oss->fd = -1;
 
     req.fmt = aud_to_ossfmt (as->fmt, as->endianness);
     req.freq = as->freq;
     req.nchannels = as->nchannels;
-    req.fragsize = conf.fragsize;
-    req.nfrags = conf.nfrags;
-    if (oss_open (1, &req, &obt, &fd)) {
+    req.fragsize = conf->fragsize;
+    req.nfrags = conf->nfrags;
+    if (oss_open (1, &req, &obt, &fd, conf)) {
         return -1;
     }
 
@@ -727,6 +714,7 @@
     }
 
     oss->fd = fd;
+    oss->conf = conf;
     return 0;
 }
 
@@ -828,7 +816,8 @@
             poll_mode = va_arg (ap, int);
             va_end (ap);
 
-            if (poll_mode && oss_poll_in (hw)) {
+            if (poll_mode) {
+                oss_poll_in (hw);
                 poll_mode = 0;
             }
             hw->poll_mode = poll_mode;
@@ -845,71 +834,79 @@
     return 0;
 }
 
+static OSSConf glob_conf = {
+    .try_mmap = 0,
+    .nfrags = 4,
+    .fragsize = 4096,
+    .devpath_out = "/dev/dsp",
+    .devpath_in = "/dev/dsp",
+    .exclusive = 0,
+    .policy = 5
+};
+
 static void *oss_audio_init (void)
 {
-    if (access(conf.devpath_in, R_OK | W_OK) < 0 ||
-        access(conf.devpath_out, R_OK | W_OK) < 0) {
+    OSSConf *conf = g_malloc(sizeof(OSSConf));
+    *conf = glob_conf;
+
+    if (access(conf->devpath_in, R_OK | W_OK) < 0 ||
+        access(conf->devpath_out, R_OK | W_OK) < 0) {
+        g_free(conf);
         return NULL;
     }
-    return &conf;
+    return conf;
 }
 
 static void oss_audio_fini (void *opaque)
 {
-    (void) opaque;
+    g_free(opaque);
 }
 
 static struct audio_option oss_options[] = {
     {
         .name  = "FRAGSIZE",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.fragsize,
+        .valp  = &glob_conf.fragsize,
         .descr = "Fragment size in bytes"
     },
     {
         .name  = "NFRAGS",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.nfrags,
+        .valp  = &glob_conf.nfrags,
         .descr = "Number of fragments"
     },
     {
         .name  = "MMAP",
         .tag   = AUD_OPT_BOOL,
-        .valp  = &conf.try_mmap,
+        .valp  = &glob_conf.try_mmap,
         .descr = "Try using memory mapped access"
     },
     {
         .name  = "DAC_DEV",
         .tag   = AUD_OPT_STR,
-        .valp  = &conf.devpath_out,
+        .valp  = &glob_conf.devpath_out,
         .descr = "Path to DAC device"
     },
     {
         .name  = "ADC_DEV",
         .tag   = AUD_OPT_STR,
-        .valp  = &conf.devpath_in,
+        .valp  = &glob_conf.devpath_in,
         .descr = "Path to ADC device"
     },
     {
         .name  = "EXCLUSIVE",
         .tag   = AUD_OPT_BOOL,
-        .valp  = &conf.exclusive,
-        .descr = "Open device in exclusive mode (vmix wont work)"
+        .valp  = &glob_conf.exclusive,
+        .descr = "Open device in exclusive mode (vmix won't work)"
     },
 #ifdef USE_DSP_POLICY
     {
         .name  = "POLICY",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.policy,
+        .valp  = &glob_conf.policy,
         .descr = "Set the timing policy of the device, -1 to use fragment mode",
     },
 #endif
-    {
-        .name  = "DEBUG",
-        .tag   = AUD_OPT_BOOL,
-        .valp  = &conf.debug,
-        .descr = "Turn on some debugging messages"
-    },
     { /* End of list */ }
 };
 
diff --git a/audio/paaudio.c b/audio/paaudio.c
index 887a5db..b69dc2d 100644
--- a/audio/paaudio.c
+++ b/audio/paaudio.c
@@ -1,4 +1,5 @@
 /* public domain */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "audio.h"
 
@@ -9,6 +10,19 @@
 #include "audio_pt_int.h"
 
 typedef struct {
+    int samples;
+    char *server;
+    char *sink;
+    char *source;
+} PAConf;
+
+typedef struct {
+    PAConf conf;
+    pa_threaded_mainloop *mainloop;
+    pa_context *context;
+} paaudio;
+
+typedef struct {
     HWVoiceOut hw;
     int done;
     int live;
@@ -17,6 +31,7 @@
     pa_stream *stream;
     void *pcm_buf;
     struct audio_pt pt;
+    paaudio *g;
 } PAVoiceOut;
 
 typedef struct {
@@ -30,20 +45,10 @@
     struct audio_pt pt;
     const void *read_data;
     size_t read_index, read_length;
+    paaudio *g;
 } PAVoiceIn;
 
-typedef struct {
-    int samples;
-    char *server;
-    char *sink;
-    char *source;
-    pa_threaded_mainloop *mainloop;
-    pa_context *context;
-} paaudio;
-
-static paaudio glob_paaudio = {
-    .samples = 4096,
-};
+static void qpa_audio_fini(void *opaque);
 
 static void GCC_FMT_ATTR (2, 3) qpa_logerr (int err, const char *fmt, ...)
 {
@@ -106,7 +111,7 @@
 
 static int qpa_simple_read (PAVoiceIn *p, void *data, size_t length, int *rerror)
 {
-    paaudio *g = &glob_paaudio;
+    paaudio *g = p->g;
 
     pa_threaded_mainloop_lock (g->mainloop);
 
@@ -160,7 +165,7 @@
 
 static int qpa_simple_write (PAVoiceOut *p, const void *data, size_t length, int *rerror)
 {
-    paaudio *g = &glob_paaudio;
+    paaudio *g = p->g;
 
     pa_threaded_mainloop_lock (g->mainloop);
 
@@ -222,7 +227,7 @@
             }
         }
 
-        decr = to_mix = audio_MIN (pa->live, glob_paaudio.samples >> 2);
+        decr = to_mix = audio_MIN (pa->live, pa->g->conf.samples >> 2);
         rpos = pa->rpos;
 
         if (audio_pt_unlock (&pa->pt, AUDIO_FUNC)) {
@@ -314,7 +319,7 @@
             }
         }
 
-        incr = to_grab = audio_MIN (pa->dead, glob_paaudio.samples >> 2);
+        incr = to_grab = audio_MIN (pa->dead, pa->g->conf.samples >> 2);
         wpos = pa->wpos;
 
         if (audio_pt_unlock (&pa->pt, AUDIO_FUNC)) {
@@ -430,7 +435,7 @@
 
 static void context_state_cb (pa_context *c, void *userdata)
 {
-    paaudio *g = &glob_paaudio;
+    paaudio *g = userdata;
 
     switch (pa_context_get_state(c)) {
     case PA_CONTEXT_READY:
@@ -449,7 +454,7 @@
 
 static void stream_state_cb (pa_stream *s, void * userdata)
 {
-    paaudio *g = &glob_paaudio;
+    paaudio *g = userdata;
 
     switch (pa_stream_get_state (s)) {
 
@@ -467,23 +472,21 @@
 
 static void stream_request_cb (pa_stream *s, size_t length, void *userdata)
 {
-    paaudio *g = &glob_paaudio;
+    paaudio *g = userdata;
 
     pa_threaded_mainloop_signal (g->mainloop, 0);
 }
 
 static pa_stream *qpa_simple_new (
-        const char *server,
+        paaudio *g,
         const char *name,
         pa_stream_direction_t dir,
         const char *dev,
-        const char *stream_name,
         const pa_sample_spec *ss,
         const pa_channel_map *map,
         const pa_buffer_attr *attr,
         int *rerror)
 {
-    paaudio *g = &glob_paaudio;
     int r;
     pa_stream *stream;
 
@@ -535,13 +538,15 @@
     return NULL;
 }
 
-static int qpa_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int qpa_init_out(HWVoiceOut *hw, struct audsettings *as,
+                        void *drv_opaque)
 {
     int error;
-    static pa_sample_spec ss;
-    static pa_buffer_attr ba;
+    pa_sample_spec ss;
+    pa_buffer_attr ba;
     struct audsettings obt_as = *as;
     PAVoiceOut *pa = (PAVoiceOut *) hw;
+    paaudio *g = pa->g = drv_opaque;
 
     ss.format = audfmt_to_pa (as->fmt, as->endianness);
     ss.channels = as->nchannels;
@@ -559,11 +564,10 @@
     obt_as.fmt = pa_to_audfmt (ss.format, &obt_as.endianness);
 
     pa->stream = qpa_simple_new (
-        glob_paaudio.server,
+        g,
         "qemu",
         PA_STREAM_PLAYBACK,
-        glob_paaudio.sink,
-        "pcm.playback",
+        g->conf.sink,
         &ss,
         NULL,                   /* channel map */
         &ba,                    /* buffering attributes */
@@ -575,7 +579,7 @@
     }
 
     audio_pcm_init_info (&hw->info, &obt_as);
-    hw->samples = glob_paaudio.samples;
+    hw->samples = g->conf.samples;
     pa->pcm_buf = audio_calloc (AUDIO_FUNC, hw->samples, 1 << hw->info.shift);
     pa->rpos = hw->rpos;
     if (!pa->pcm_buf) {
@@ -602,13 +606,14 @@
     return -1;
 }
 
-static int qpa_init_in (HWVoiceIn *hw, struct audsettings *as)
+static int qpa_init_in(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
 {
     int error;
-    static pa_sample_spec ss;
-    static pa_buffer_attr ba;
+    pa_sample_spec ss;
+    pa_buffer_attr ba;
     struct audsettings obt_as = *as;
     PAVoiceIn *pa = (PAVoiceIn *) hw;
+    paaudio *g = pa->g = drv_opaque;
 
     ss.format = audfmt_to_pa (as->fmt, as->endianness);
     ss.channels = as->nchannels;
@@ -624,14 +629,13 @@
     obt_as.fmt = pa_to_audfmt (ss.format, &obt_as.endianness);
 
     pa->stream = qpa_simple_new (
-        glob_paaudio.server,
+        g,
         "qemu",
         PA_STREAM_RECORD,
-        glob_paaudio.source,
-        "pcm.capture",
+        g->conf.source,
         &ss,
         NULL,                   /* channel map */
-        &ba,                   /* buffering attributes */
+        &ba,                    /* buffering attributes */
         &error
         );
     if (!pa->stream) {
@@ -640,7 +644,7 @@
     }
 
     audio_pcm_init_info (&hw->info, &obt_as);
-    hw->samples = glob_paaudio.samples;
+    hw->samples = g->conf.samples;
     pa->pcm_buf = audio_calloc (AUDIO_FUNC, hw->samples, 1 << hw->info.shift);
     pa->wpos = hw->wpos;
     if (!pa->pcm_buf) {
@@ -713,7 +717,7 @@
     PAVoiceOut *pa = (PAVoiceOut *) hw;
     pa_operation *op;
     pa_cvolume v;
-    paaudio *g = &glob_paaudio;
+    paaudio *g = pa->g;
 
 #ifdef PA_CHECK_VERSION    /* macro is present in 0.9.16+ */
     pa_cvolume_init (&v);  /* function is present in 0.9.13+ */
@@ -765,7 +769,7 @@
     PAVoiceIn *pa = (PAVoiceIn *) hw;
     pa_operation *op;
     pa_cvolume v;
-    paaudio *g = &glob_paaudio;
+    paaudio *g = pa->g;
 
 #ifdef PA_CHECK_VERSION
     pa_cvolume_init (&v);
@@ -837,23 +841,31 @@
 }
 
 /* common */
+static PAConf glob_conf = {
+    .samples = 4096,
+};
+
 static void *qpa_audio_init (void)
 {
-    paaudio *g = &glob_paaudio;
+    paaudio *g = g_malloc(sizeof(paaudio));
+    g->conf = glob_conf;
+    g->mainloop = NULL;
+    g->context = NULL;
 
     g->mainloop = pa_threaded_mainloop_new ();
     if (!g->mainloop) {
         goto fail;
     }
 
-    g->context = pa_context_new (pa_threaded_mainloop_get_api (g->mainloop), glob_paaudio.server);
+    g->context = pa_context_new (pa_threaded_mainloop_get_api (g->mainloop),
+                                 g->conf.server);
     if (!g->context) {
         goto fail;
     }
 
     pa_context_set_state_callback (g->context, context_state_cb, g);
 
-    if (pa_context_connect (g->context, glob_paaudio.server, 0, NULL) < 0) {
+    if (pa_context_connect (g->context, g->conf.server, 0, NULL) < 0) {
         qpa_logerr (pa_context_errno (g->context),
                     "pa_context_connect() failed\n");
         goto fail;
@@ -886,12 +898,13 @@
 
     pa_threaded_mainloop_unlock (g->mainloop);
 
-    return &glob_paaudio;
+    return g;
 
 unlock_and_fail:
     pa_threaded_mainloop_unlock (g->mainloop);
 fail:
     AUD_log (AUDIO_CAP, "Failed to initialize PA context");
+    qpa_audio_fini(g);
     return NULL;
 }
 
@@ -906,39 +919,38 @@
     if (g->context) {
         pa_context_disconnect (g->context);
         pa_context_unref (g->context);
-        g->context = NULL;
     }
 
     if (g->mainloop) {
         pa_threaded_mainloop_free (g->mainloop);
     }
 
-    g->mainloop = NULL;
+    g_free(g);
 }
 
 struct audio_option qpa_options[] = {
     {
         .name  = "SAMPLES",
         .tag   = AUD_OPT_INT,
-        .valp  = &glob_paaudio.samples,
+        .valp  = &glob_conf.samples,
         .descr = "buffer size in samples"
     },
     {
         .name  = "SERVER",
         .tag   = AUD_OPT_STR,
-        .valp  = &glob_paaudio.server,
+        .valp  = &glob_conf.server,
         .descr = "server address"
     },
     {
         .name  = "SINK",
         .tag   = AUD_OPT_STR,
-        .valp  = &glob_paaudio.sink,
+        .valp  = &glob_conf.sink,
         .descr = "sink device name"
     },
     {
         .name  = "SOURCE",
         .tag   = AUD_OPT_STR,
-        .valp  = &glob_paaudio.source,
+        .valp  = &glob_conf.source,
         .descr = "source device name"
     },
     { /* End of list */ }
diff --git a/audio/sdlaudio.c b/audio/sdlaudio.c
index d24daa5..db69fe1 100644
--- a/audio/sdlaudio.c
+++ b/audio/sdlaudio.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <SDL.h>
 #include <SDL_thread.h>
 #include "qemu-common.h"
@@ -55,6 +56,7 @@
     SDL_mutex *mutex;
     SDL_sem *sem;
     int initialized;
+    bool driver_created;
 } glob_sdl;
 typedef struct SDLAudioState SDLAudioState;
 
@@ -332,7 +334,8 @@
     sdl_close (&glob_sdl);
 }
 
-static int sdl_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int sdl_init_out(HWVoiceOut *hw, struct audsettings *as,
+                        void *drv_opaque)
 {
     SDLVoiceOut *sdl = (SDLVoiceOut *) hw;
     SDLAudioState *s = &glob_sdl;
@@ -392,6 +395,10 @@
 static void *sdl_audio_init (void)
 {
     SDLAudioState *s = &glob_sdl;
+    if (s->driver_created) {
+        sdl_logerr("Can't create multiple sdl backends\n");
+        return NULL;
+    }
 
     if (SDL_InitSubSystem (SDL_INIT_AUDIO)) {
         sdl_logerr ("SDL failed to initialize audio subsystem\n");
@@ -413,6 +420,7 @@
         return NULL;
     }
 
+    s->driver_created = true;
     return s;
 }
 
@@ -423,6 +431,7 @@
     SDL_DestroySemaphore (s->sem);
     SDL_DestroyMutex (s->mutex);
     SDL_QuitSubSystem (SDL_INIT_AUDIO);
+    s->driver_created = false;
 }
 
 static struct audio_option sdl_options[] = {
diff --git a/audio/spiceaudio.c b/audio/spiceaudio.c
index 7b79bed..5580e76 100644
--- a/audio/spiceaudio.c
+++ b/audio/spiceaudio.c
@@ -17,7 +17,10 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qemu/host-utils.h"
+#include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "ui/qemu-spice.h"
 
@@ -102,11 +105,11 @@
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     ticks = now - rate->start_ticks;
-    bytes = muldiv64 (ticks, info->bytes_per_second, get_ticks_per_sec ());
+    bytes = muldiv64(ticks, info->bytes_per_second, NANOSECONDS_PER_SECOND);
     samples = (bytes - rate->bytes_sent) >> info->shift;
     if (samples < 0 || samples > 65536) {
         error_report("Resetting rate control (%" PRId64 " samples)", samples);
-        rate_start (rate);
+        rate_start(rate);
         samples = 0;
     }
     rate->bytes_sent += samples << info->shift;
@@ -115,7 +118,8 @@
 
 /* playback */
 
-static int line_out_init (HWVoiceOut *hw, struct audsettings *as)
+static int line_out_init(HWVoiceOut *hw, struct audsettings *as,
+                         void *drv_opaque)
 {
     SpiceVoiceOut *out = container_of (hw, SpiceVoiceOut, hw);
     struct audsettings settings;
@@ -243,7 +247,7 @@
 
 /* record */
 
-static int line_in_init (HWVoiceIn *hw, struct audsettings *as)
+static int line_in_init(HWVoiceIn *hw, struct audsettings *as, void *drv_opaque)
 {
     SpiceVoiceIn *in = container_of (hw, SpiceVoiceIn, hw);
     struct audsettings settings;
diff --git a/audio/trace-events b/audio/trace-events
new file mode 100644
index 0000000..5173590
--- /dev/null
+++ b/audio/trace-events
@@ -0,0 +1,17 @@
+# See docs/tracing.txt for syntax documentation.
+
+# audio/alsaaudio.c
+alsa_revents(int revents) "revents = %d"
+alsa_pollout(int i, int fd) "i = %d fd = %d"
+alsa_set_handler(int events, int index, int fd, int err) "events=%#x index=%d fd=%d err=%d"
+alsa_wrote_zero(int len) "Failed to write %d frames (wrote zero)"
+alsa_read_zero(long len) "Failed to read %ld frames (read zero)"
+alsa_xrun_out(void) "Recovering from playback xrun"
+alsa_xrun_in(void) "Recovering from capture xrun"
+alsa_resume_out(void) "Resuming suspended output stream"
+alsa_resume_in(void) "Resuming suspended input stream"
+alsa_no_frames(int state) "No frames available and ALSA state is %d"
+
+# audio/ossaudio.c
+oss_version(int version) "OSS version = %#x"
+oss_invalid_available_size(int size, int bufsize) "Invalid available size, size=%d bufsize=%d"
diff --git a/audio/wavaudio.c b/audio/wavaudio.c
index 6846a1a..341eec3 100644
--- a/audio/wavaudio.c
+++ b/audio/wavaudio.c
@@ -21,7 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw/hw.h"
+#include "qemu/osdep.h"
+#include "qemu/host-utils.h"
 #include "qemu/timer.h"
 #include "audio.h"
 
@@ -36,15 +37,10 @@
     int total_samples;
 } WAVVoiceOut;
 
-static struct {
+typedef struct {
     struct audsettings settings;
     const char *wav_path;
-} conf = {
-    .settings.freq      = 44100,
-    .settings.nchannels = 2,
-    .settings.fmt       = AUD_FMT_S16,
-    .wav_path           = "qemu.wav"
-};
+} WAVConf;
 
 static int wav_run_out (HWVoiceOut *hw, int live)
 {
@@ -55,7 +51,7 @@
     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     int64_t ticks = now - wav->old_ticks;
     int64_t bytes =
-        muldiv64 (ticks, hw->info.bytes_per_second, get_ticks_per_sec ());
+        muldiv64(ticks, hw->info.bytes_per_second, NANOSECONDS_PER_SECOND);
 
     if (bytes > INT_MAX) {
         samples = INT_MAX >> hw->info.shift;
@@ -105,7 +101,8 @@
     }
 }
 
-static int wav_init_out (HWVoiceOut *hw, struct audsettings *as)
+static int wav_init_out(HWVoiceOut *hw, struct audsettings *as,
+                        void *drv_opaque)
 {
     WAVVoiceOut *wav = (WAVVoiceOut *) hw;
     int bits16 = 0, stereo = 0;
@@ -115,9 +112,8 @@
         0x02, 0x00, 0x44, 0xac, 0x00, 0x00, 0x10, 0xb1, 0x02, 0x00, 0x04,
         0x00, 0x10, 0x00, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00
     };
-    struct audsettings wav_as = conf.settings;
-
-    (void) as;
+    WAVConf *conf = drv_opaque;
+    struct audsettings wav_as = conf->settings;
 
     stereo = wav_as.nchannels == 2;
     switch (wav_as.fmt) {
@@ -155,10 +151,10 @@
     le_store (hdr + 28, hw->info.freq << (bits16 + stereo), 4);
     le_store (hdr + 32, 1 << (bits16 + stereo), 2);
 
-    wav->f = fopen (conf.wav_path, "wb");
+    wav->f = fopen (conf->wav_path, "wb");
     if (!wav->f) {
         dolog ("Failed to open wave file `%s'\nReason: %s\n",
-               conf.wav_path, strerror (errno));
+               conf->wav_path, strerror (errno));
         g_free (wav->pcm_buf);
         wav->pcm_buf = NULL;
         return -1;
@@ -226,40 +222,49 @@
     return 0;
 }
 
+static WAVConf glob_conf = {
+    .settings.freq      = 44100,
+    .settings.nchannels = 2,
+    .settings.fmt       = AUD_FMT_S16,
+    .wav_path           = "qemu.wav"
+};
+
 static void *wav_audio_init (void)
 {
-    return &conf;
+    WAVConf *conf = g_malloc(sizeof(WAVConf));
+    *conf = glob_conf;
+    return conf;
 }
 
 static void wav_audio_fini (void *opaque)
 {
-    (void) opaque;
     ldebug ("wav_fini");
+    g_free(opaque);
 }
 
 static struct audio_option wav_options[] = {
     {
         .name  = "FREQUENCY",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.settings.freq,
+        .valp  = &glob_conf.settings.freq,
         .descr = "Frequency"
     },
     {
         .name  = "FORMAT",
         .tag   = AUD_OPT_FMT,
-        .valp  = &conf.settings.fmt,
+        .valp  = &glob_conf.settings.fmt,
         .descr = "Format"
     },
     {
         .name  = "DAC_FIXED_CHANNELS",
         .tag   = AUD_OPT_INT,
-        .valp  = &conf.settings.nchannels,
+        .valp  = &glob_conf.settings.nchannels,
         .descr = "Number of channels (1 - mono, 2 - stereo)"
     },
     {
         .name  = "PATH",
         .tag   = AUD_OPT_STR,
-        .valp  = &conf.wav_path,
+        .valp  = &glob_conf.wav_path,
         .descr = "Path to wave file"
     },
     { /* End of list */ }
diff --git a/audio/wavcapture.c b/audio/wavcapture.c
index 6f6d792..8bfb9e7 100644
--- a/audio/wavcapture.c
+++ b/audio/wavcapture.c
@@ -1,5 +1,7 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "monitor/monitor.h"
+#include "qemu/error-report.h"
 #include "audio.h"
 
 typedef struct {
diff --git a/audio/winaudio.c b/audio/winaudio.c
index 6aec129..63f923f 100644
--- a/audio/winaudio.c
+++ b/audio/winaudio.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 #include "audio.h"
@@ -161,7 +162,7 @@
 
 
 static int
-winaudio_out_init (HWVoiceOut *hw, struct audsettings *as)
+winaudio_out_init (HWVoiceOut *hw, struct audsettings *as, void* drv_opaque)
 {
     WinAudioOut*   s = (WinAudioOut*) hw;
     MMRESULT       result;
@@ -296,14 +297,6 @@
             played       += wav_samples;
             s->write_pos += wav_bytes;
             if (s->write_pos == s->write_size) {
-#if DEBUG
-                int64_t  now  = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - start_time;
-                int64_t  diff = now - last_time;
-
-                D("run_out: (%7.3f:%7d):waveOutWrite buffer:%d\n",
-                   now/1e9, (now-last_time)/1e9, s->write_index);
-                last_time = now;
-#endif
                 waveOutWrite( s->waveout, wav_buffer, sizeof(*wav_buffer) );
                 s->write_pos    = 0;
                 s->write_index += 1;
@@ -407,7 +400,7 @@
 
 
 static int
-winaudio_in_init (HWVoiceIn *hw, struct audsettings *as)
+winaudio_in_init (HWVoiceIn *hw, struct audsettings *as, void* drv_opaque)
 {
     WinAudioIn*   s = (WinAudioIn*) hw;
     MMRESULT       result;
diff --git a/audio/winwaveaudio.c b/audio/winwaveaudio.c
deleted file mode 100644
index 8dbd145..0000000
--- a/audio/winwaveaudio.c
+++ /dev/null
@@ -1,717 +0,0 @@
-/* public domain */
-
-#include "qemu-common.h"
-#include "sysemu/sysemu.h"
-#include "audio.h"
-
-#define AUDIO_CAP "winwave"
-#include "audio_int.h"
-
-#include <windows.h>
-#include <mmsystem.h>
-
-#include "audio_win_int.h"
-
-static struct {
-    int dac_headers;
-    int dac_samples;
-    int adc_headers;
-    int adc_samples;
-} conf = {
-    .dac_headers = 4,
-    .dac_samples = 1024,
-    .adc_headers = 4,
-    .adc_samples = 1024
-};
-
-typedef struct {
-    HWVoiceOut hw;
-    HWAVEOUT hwo;
-    WAVEHDR *hdrs;
-    HANDLE event;
-    void *pcm_buf;
-    int avail;
-    int pending;
-    int curhdr;
-    int paused;
-    CRITICAL_SECTION crit_sect;
-} WaveVoiceOut;
-
-typedef struct {
-    HWVoiceIn hw;
-    HWAVEIN hwi;
-    WAVEHDR *hdrs;
-    HANDLE event;
-    void *pcm_buf;
-    int curhdr;
-    int paused;
-    int rpos;
-    int avail;
-    CRITICAL_SECTION crit_sect;
-} WaveVoiceIn;
-
-static void winwave_log_mmresult (MMRESULT mr)
-{
-    const char *str = "BUG";
-
-    switch (mr) {
-    case MMSYSERR_NOERROR:
-        str = "Success";
-        break;
-
-    case MMSYSERR_INVALHANDLE:
-        str = "Specified device handle is invalid";
-        break;
-
-    case MMSYSERR_BADDEVICEID:
-        str = "Specified device id is out of range";
-        break;
-
-    case MMSYSERR_NODRIVER:
-        str = "No device driver is present";
-        break;
-
-    case MMSYSERR_NOMEM:
-        str = "Unable to allocate or lock memory";
-        break;
-
-    case WAVERR_SYNC:
-        str = "Device is synchronous but waveOutOpen was called "
-            "without using the WINWAVE_ALLOWSYNC flag";
-        break;
-
-    case WAVERR_UNPREPARED:
-        str = "The data block pointed to by the pwh parameter "
-            "hasn't been prepared";
-        break;
-
-    case WAVERR_STILLPLAYING:
-        str = "There are still buffers in the queue";
-        break;
-
-    default:
-        dolog ("Reason: Unknown (MMRESULT %#x)\n", mr);
-        return;
-    }
-
-    dolog ("Reason: %s\n", str);
-}
-
-static void GCC_FMT_ATTR (2, 3) winwave_logerr (
-    MMRESULT mr,
-    const char *fmt,
-    ...
-    )
-{
-    va_list ap;
-
-    va_start (ap, fmt);
-    AUD_vlog (AUDIO_CAP, fmt, ap);
-    va_end (ap);
-
-    AUD_log (NULL, " failed\n");
-    winwave_log_mmresult (mr);
-}
-
-static void winwave_anal_close_out (WaveVoiceOut *wave)
-{
-    MMRESULT mr;
-
-    mr = waveOutClose (wave->hwo);
-    if (mr != MMSYSERR_NOERROR) {
-        winwave_logerr (mr, "waveOutClose");
-    }
-    wave->hwo = NULL;
-}
-
-static void CALLBACK winwave_callback_out (
-    HWAVEOUT hwo,
-    UINT msg,
-    DWORD_PTR dwInstance,
-    DWORD_PTR dwParam1,
-    DWORD_PTR dwParam2
-    )
-{
-    WaveVoiceOut *wave = (WaveVoiceOut *) dwInstance;
-
-    switch (msg) {
-    case WOM_DONE:
-        {
-            WAVEHDR *h = (WAVEHDR *) dwParam1;
-            if (!h->dwUser) {
-                h->dwUser = 1;
-                EnterCriticalSection (&wave->crit_sect);
-                {
-                    wave->avail += conf.dac_samples;
-                }
-                LeaveCriticalSection (&wave->crit_sect);
-                if (wave->hw.poll_mode) {
-                    if (!SetEvent (wave->event)) {
-                        dolog ("DAC SetEvent failed %lx\n", GetLastError ());
-                    }
-                }
-            }
-        }
-        break;
-
-    case WOM_CLOSE:
-    case WOM_OPEN:
-        break;
-
-    default:
-        dolog ("unknown wave out callback msg %x\n", msg);
-    }
-}
-
-static int winwave_init_out (HWVoiceOut *hw, struct audsettings *as)
-{
-    int i;
-    int err;
-    MMRESULT mr;
-    WAVEFORMATEX wfx;
-    WaveVoiceOut *wave;
-
-    wave = (WaveVoiceOut *) hw;
-
-    InitializeCriticalSection (&wave->crit_sect);
-
-    err = waveformat_from_audio_settings (&wfx, as);
-    if (err) {
-        goto err0;
-    }
-
-    mr = waveOutOpen (&wave->hwo, WAVE_MAPPER, &wfx,
-                      (DWORD_PTR) winwave_callback_out,
-                      (DWORD_PTR) wave, CALLBACK_FUNCTION);
-    if (mr != MMSYSERR_NOERROR) {
-        winwave_logerr (mr, "waveOutOpen");
-        goto err1;
-    }
-
-    wave->hdrs = audio_calloc (AUDIO_FUNC, conf.dac_headers,
-                               sizeof (*wave->hdrs));
-    if (!wave->hdrs) {
-        goto err2;
-    }
-
-    audio_pcm_init_info (&hw->info, as);
-    hw->samples = conf.dac_samples * conf.dac_headers;
-    wave->avail = hw->samples;
-
-    wave->pcm_buf = audio_calloc (AUDIO_FUNC, conf.dac_samples,
-                                  conf.dac_headers << hw->info.shift);
-    if (!wave->pcm_buf) {
-        goto err3;
-    }
-
-    for (i = 0; i < conf.dac_headers; ++i) {
-        WAVEHDR *h = &wave->hdrs[i];
-
-        h->dwUser = 0;
-        h->dwBufferLength = conf.dac_samples << hw->info.shift;
-        h->lpData = advance (wave->pcm_buf, i * h->dwBufferLength);
-        h->dwFlags = 0;
-
-        mr = waveOutPrepareHeader (wave->hwo, h, sizeof (*h));
-        if (mr != MMSYSERR_NOERROR) {
-            winwave_logerr (mr, "waveOutPrepareHeader(%d)", i);
-            goto err4;
-        }
-    }
-
-    return 0;
-
- err4:
-    g_free (wave->pcm_buf);
- err3:
-    g_free (wave->hdrs);
- err2:
-    winwave_anal_close_out (wave);
- err1:
- err0:
-    return -1;
-}
-
-static int winwave_write (SWVoiceOut *sw, void *buf, int len)
-{
-    return audio_pcm_sw_write (sw, buf, len);
-}
-
-static int winwave_run_out (HWVoiceOut *hw, int live)
-{
-    WaveVoiceOut *wave = (WaveVoiceOut *) hw;
-    int decr;
-    int doreset;
-
-    EnterCriticalSection (&wave->crit_sect);
-    {
-        decr = audio_MIN (live, wave->avail);
-        decr = audio_pcm_hw_clip_out (hw, wave->pcm_buf, decr, wave->pending);
-        wave->pending += decr;
-        wave->avail -= decr;
-    }
-    LeaveCriticalSection (&wave->crit_sect);
-
-    doreset = hw->poll_mode && (wave->pending >= conf.dac_samples);
-    if (doreset && !ResetEvent (wave->event)) {
-        dolog ("DAC ResetEvent failed %lx\n", GetLastError ());
-    }
-
-    while (wave->pending >= conf.dac_samples) {
-        MMRESULT mr;
-        WAVEHDR *h = &wave->hdrs[wave->curhdr];
-
-        h->dwUser = 0;
-        mr = waveOutWrite (wave->hwo, h, sizeof (*h));
-        if (mr != MMSYSERR_NOERROR) {
-            winwave_logerr (mr, "waveOutWrite(%d)", wave->curhdr);
-            break;
-        }
-
-        wave->pending -= conf.dac_samples;
-        wave->curhdr = (wave->curhdr + 1) % conf.dac_headers;
-    }
-
-    return decr;
-}
-
-static void winwave_poll (void *opaque)
-{
-    (void) opaque;
-    audio_run ("winwave_poll");
-}
-
-static void winwave_fini_out (HWVoiceOut *hw)
-{
-    int i;
-    MMRESULT mr;
-    WaveVoiceOut *wave = (WaveVoiceOut *) hw;
-
-    mr = waveOutReset (wave->hwo);
-    if (mr != MMSYSERR_NOERROR) {
-        winwave_logerr (mr, "waveOutReset");
-    }
-
-    for (i = 0; i < conf.dac_headers; ++i) {
-        mr = waveOutUnprepareHeader (wave->hwo, &wave->hdrs[i],
-                                     sizeof (wave->hdrs[i]));
-        if (mr != MMSYSERR_NOERROR) {
-            winwave_logerr (mr, "waveOutUnprepareHeader(%d)", i);
-        }
-    }
-
-    winwave_anal_close_out (wave);
-
-    if (wave->event) {
-        qemu_del_wait_object (wave->event, winwave_poll, wave);
-        if (!CloseHandle (wave->event)) {
-            dolog ("DAC CloseHandle failed %lx\n", GetLastError ());
-        }
-        wave->event = NULL;
-    }
-
-    g_free (wave->pcm_buf);
-    wave->pcm_buf = NULL;
-
-    g_free (wave->hdrs);
-    wave->hdrs = NULL;
-}
-
-static int winwave_ctl_out (HWVoiceOut *hw, int cmd, ...)
-{
-    MMRESULT mr;
-    WaveVoiceOut *wave = (WaveVoiceOut *) hw;
-
-    switch (cmd) {
-    case VOICE_ENABLE:
-        {
-            va_list ap;
-            int poll_mode;
-
-            va_start (ap, cmd);
-            poll_mode = va_arg (ap, int);
-            va_end (ap);
-
-            if (poll_mode && !wave->event) {
-                wave->event = CreateEvent (NULL, TRUE, TRUE, NULL);
-                if (!wave->event) {
-                    dolog ("DAC CreateEvent: %lx, poll mode will be disabled\n",
-                           GetLastError ());
-                }
-            }
-
-            if (wave->event) {
-                int ret;
-
-                ret = qemu_add_wait_object (wave->event, winwave_poll, wave);
-                hw->poll_mode = (ret == 0);
-            }
-            else {
-                hw->poll_mode = 0;
-            }
-            wave->paused = 0;
-        }
-        return 0;
-
-    case VOICE_DISABLE:
-        if (!wave->paused) {
-            mr = waveOutReset (wave->hwo);
-            if (mr != MMSYSERR_NOERROR) {
-                winwave_logerr (mr, "waveOutReset");
-            }
-            else {
-                wave->paused = 1;
-            }
-        }
-        if (wave->event) {
-            qemu_del_wait_object (wave->event, winwave_poll, wave);
-        }
-        return 0;
-    }
-    return -1;
-}
-
-static void winwave_anal_close_in (WaveVoiceIn *wave)
-{
-    MMRESULT mr;
-
-    mr = waveInClose (wave->hwi);
-    if (mr != MMSYSERR_NOERROR) {
-        winwave_logerr (mr, "waveInClose");
-    }
-    wave->hwi = NULL;
-}
-
-static void CALLBACK winwave_callback_in (
-    HWAVEIN *hwi,
-    UINT msg,
-    DWORD_PTR dwInstance,
-    DWORD_PTR dwParam1,
-    DWORD_PTR dwParam2
-    )
-{
-    WaveVoiceIn *wave = (WaveVoiceIn *) dwInstance;
-
-    switch (msg) {
-    case WIM_DATA:
-        {
-            WAVEHDR *h = (WAVEHDR *) dwParam1;
-            if (!h->dwUser) {
-                h->dwUser = 1;
-                EnterCriticalSection (&wave->crit_sect);
-                {
-                    wave->avail += conf.adc_samples;
-                }
-                LeaveCriticalSection (&wave->crit_sect);
-                if (wave->hw.poll_mode) {
-                    if (!SetEvent (wave->event)) {
-                        dolog ("ADC SetEvent failed %lx\n", GetLastError ());
-                    }
-                }
-            }
-        }
-        break;
-
-    case WIM_CLOSE:
-    case WIM_OPEN:
-        break;
-
-    default:
-        dolog ("unknown wave in callback msg %x\n", msg);
-    }
-}
-
-static void winwave_add_buffers (WaveVoiceIn *wave, int samples)
-{
-    int doreset;
-
-    doreset = wave->hw.poll_mode && (samples >= conf.adc_samples);
-    if (doreset && !ResetEvent (wave->event)) {
-        dolog ("ADC ResetEvent failed %lx\n", GetLastError ());
-    }
-
-    while (samples >= conf.adc_samples) {
-        MMRESULT mr;
-        WAVEHDR *h = &wave->hdrs[wave->curhdr];
-
-        h->dwUser = 0;
-        mr = waveInAddBuffer (wave->hwi, h, sizeof (*h));
-        if (mr != MMSYSERR_NOERROR) {
-            winwave_logerr (mr, "waveInAddBuffer(%d)", wave->curhdr);
-        }
-        wave->curhdr = (wave->curhdr + 1) % conf.adc_headers;
-        samples -= conf.adc_samples;
-    }
-}
-
-static int winwave_init_in (HWVoiceIn *hw, struct audsettings *as)
-{
-    int i;
-    int err;
-    MMRESULT mr;
-    WAVEFORMATEX wfx;
-    WaveVoiceIn *wave;
-
-    wave = (WaveVoiceIn *) hw;
-
-    InitializeCriticalSection (&wave->crit_sect);
-
-    err = waveformat_from_audio_settings (&wfx, as);
-    if (err) {
-        goto err0;
-    }
-
-    mr = waveInOpen (&wave->hwi, WAVE_MAPPER, &wfx,
-                     (DWORD_PTR) winwave_callback_in,
-                     (DWORD_PTR) wave, CALLBACK_FUNCTION);
-    if (mr != MMSYSERR_NOERROR) {
-        winwave_logerr (mr, "waveInOpen");
-        goto err1;
-    }
-
-    wave->hdrs = audio_calloc (AUDIO_FUNC, conf.dac_headers,
-                               sizeof (*wave->hdrs));
-    if (!wave->hdrs) {
-        goto err2;
-    }
-
-    audio_pcm_init_info (&hw->info, as);
-    hw->samples = conf.adc_samples * conf.adc_headers;
-    wave->avail = 0;
-
-    wave->pcm_buf = audio_calloc (AUDIO_FUNC, conf.adc_samples,
-                                  conf.adc_headers << hw->info.shift);
-    if (!wave->pcm_buf) {
-        goto err3;
-    }
-
-    for (i = 0; i < conf.adc_headers; ++i) {
-        WAVEHDR *h = &wave->hdrs[i];
-
-        h->dwUser = 0;
-        h->dwBufferLength = conf.adc_samples << hw->info.shift;
-        h->lpData = advance (wave->pcm_buf, i * h->dwBufferLength);
-        h->dwFlags = 0;
-
-        mr = waveInPrepareHeader (wave->hwi, h, sizeof (*h));
-        if (mr != MMSYSERR_NOERROR) {
-            winwave_logerr (mr, "waveInPrepareHeader(%d)", i);
-            goto err4;
-        }
-    }
-
-    wave->paused = 1;
-    winwave_add_buffers (wave, hw->samples);
-    return 0;
-
- err4:
-    g_free (wave->pcm_buf);
- err3:
-    g_free (wave->hdrs);
- err2:
-    winwave_anal_close_in (wave);
- err1:
- err0:
-    return -1;
-}
-
-static void winwave_fini_in (HWVoiceIn *hw)
-{
-    int i;
-    MMRESULT mr;
-    WaveVoiceIn *wave = (WaveVoiceIn *) hw;
-
-    mr = waveInReset (wave->hwi);
-    if (mr != MMSYSERR_NOERROR) {
-        winwave_logerr (mr, "waveInReset");
-    }
-
-    for (i = 0; i < conf.adc_headers; ++i) {
-        mr = waveInUnprepareHeader (wave->hwi, &wave->hdrs[i],
-                                     sizeof (wave->hdrs[i]));
-        if (mr != MMSYSERR_NOERROR) {
-            winwave_logerr (mr, "waveInUnprepareHeader(%d)", i);
-        }
-    }
-
-    winwave_anal_close_in (wave);
-
-    if (wave->event) {
-        qemu_del_wait_object (wave->event, winwave_poll, wave);
-        if (!CloseHandle (wave->event)) {
-            dolog ("ADC CloseHandle failed %lx\n", GetLastError ());
-        }
-        wave->event = NULL;
-    }
-
-    g_free (wave->pcm_buf);
-    wave->pcm_buf = NULL;
-
-    g_free (wave->hdrs);
-    wave->hdrs = NULL;
-}
-
-static int winwave_run_in (HWVoiceIn *hw)
-{
-    WaveVoiceIn *wave = (WaveVoiceIn *) hw;
-    int live = audio_pcm_hw_get_live_in (hw);
-    int dead = hw->samples - live;
-    int decr, ret;
-
-    if (!dead) {
-        return 0;
-    }
-
-    EnterCriticalSection (&wave->crit_sect);
-    {
-        decr = audio_MIN (dead, wave->avail);
-        wave->avail -= decr;
-    }
-    LeaveCriticalSection (&wave->crit_sect);
-
-    ret = decr;
-    while (decr) {
-        int left = hw->samples - hw->wpos;
-        int conv = audio_MIN (left, decr);
-        hw->conv (hw->conv_buf + hw->wpos,
-                  advance (wave->pcm_buf, wave->rpos << hw->info.shift),
-                  conv);
-
-        wave->rpos = (wave->rpos + conv) % hw->samples;
-        hw->wpos = (hw->wpos + conv) % hw->samples;
-        decr -= conv;
-    }
-
-    winwave_add_buffers (wave, ret);
-    return ret;
-}
-
-static int winwave_read (SWVoiceIn *sw, void *buf, int size)
-{
-    return audio_pcm_sw_read (sw, buf, size);
-}
-
-static int winwave_ctl_in (HWVoiceIn *hw, int cmd, ...)
-{
-    MMRESULT mr;
-    WaveVoiceIn *wave = (WaveVoiceIn *) hw;
-
-    switch (cmd) {
-    case VOICE_ENABLE:
-        {
-            va_list ap;
-            int poll_mode;
-
-            va_start (ap, cmd);
-            poll_mode = va_arg (ap, int);
-            va_end (ap);
-
-            if (poll_mode && !wave->event) {
-                wave->event = CreateEvent (NULL, TRUE, TRUE, NULL);
-                if (!wave->event) {
-                    dolog ("ADC CreateEvent: %lx, poll mode will be disabled\n",
-                           GetLastError ());
-                }
-            }
-
-            if (wave->event) {
-                int ret;
-
-                ret = qemu_add_wait_object (wave->event, winwave_poll, wave);
-                hw->poll_mode = (ret == 0);
-            }
-            else {
-                hw->poll_mode = 0;
-            }
-            if (wave->paused) {
-                mr = waveInStart (wave->hwi);
-                if (mr != MMSYSERR_NOERROR) {
-                    winwave_logerr (mr, "waveInStart");
-                }
-                wave->paused = 0;
-            }
-        }
-        return 0;
-
-    case VOICE_DISABLE:
-        if (!wave->paused) {
-            mr = waveInStop (wave->hwi);
-            if (mr != MMSYSERR_NOERROR) {
-                winwave_logerr (mr, "waveInStop");
-            }
-            else {
-                wave->paused = 1;
-            }
-        }
-        if (wave->event) {
-            qemu_del_wait_object (wave->event, winwave_poll, wave);
-        }
-        return 0;
-    }
-    return 0;
-}
-
-static void *winwave_audio_init (void)
-{
-    return &conf;
-}
-
-static void winwave_audio_fini (void *opaque)
-{
-    (void) opaque;
-}
-
-static struct audio_option winwave_options[] = {
-    {
-        .name        = "DAC_HEADERS",
-        .tag         = AUD_OPT_INT,
-        .valp        = &conf.dac_headers,
-        .descr       = "DAC number of headers",
-    },
-    {
-        .name        = "DAC_SAMPLES",
-        .tag         = AUD_OPT_INT,
-        .valp        = &conf.dac_samples,
-        .descr       = "DAC number of samples per header",
-    },
-    {
-        .name        = "ADC_HEADERS",
-        .tag         = AUD_OPT_INT,
-        .valp        = &conf.adc_headers,
-        .descr       = "ADC number of headers",
-    },
-    {
-        .name        = "ADC_SAMPLES",
-        .tag         = AUD_OPT_INT,
-        .valp        = &conf.adc_samples,
-        .descr       = "ADC number of samples per header",
-    },
-    { /* End of list */ }
-};
-
-static struct audio_pcm_ops winwave_pcm_ops = {
-    .init_out = winwave_init_out,
-    .fini_out = winwave_fini_out,
-    .run_out  = winwave_run_out,
-    .write    = winwave_write,
-    .ctl_out  = winwave_ctl_out,
-    .init_in  = winwave_init_in,
-    .fini_in  = winwave_fini_in,
-    .run_in   = winwave_run_in,
-    .read     = winwave_read,
-    .ctl_in   = winwave_ctl_in
-};
-
-struct audio_driver winwave_audio_driver = {
-    .name           = "winwave",
-    .descr          = "Windows Waveform Audio http://msdn.microsoft.com",
-    .options        = winwave_options,
-    .init           = winwave_audio_init,
-    .fini           = winwave_audio_fini,
-    .pcm_ops        = &winwave_pcm_ops,
-    .can_be_default = 1,
-    .max_voices_out = INT_MAX,
-    .max_voices_in  = INT_MAX,
-    .voice_size_out = sizeof (WaveVoiceOut),
-    .voice_size_in  = sizeof (WaveVoiceIn)
-};
diff --git a/backends/baum.c b/backends/baum.c
index a69aaff..c537141 100644
--- a/backends/baum.c
+++ b/backends/baum.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 #include "qemu/timer.h"
@@ -303,7 +305,7 @@
                 return 0;
             cur++;
         }
-        DPRINTF("Dropped %d bytes!\n", cur - buf);
+        DPRINTF("Dropped %td bytes!\n", cur - buf);
     }
 
 #define EAT(c) do {\
@@ -335,7 +337,7 @@
 
         /* Allow 100ms to complete the DisplayData packet */
         timer_mod(baum->cellCount_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                       get_ticks_per_sec() / 10);
+                       NANOSECONDS_PER_SECOND / 10);
         for (i = 0; i < baum->x * baum->y ; i++) {
             EAT(c);
             cells[i] = c;
@@ -561,8 +563,12 @@
     g_free(baum);
 }
 
-CharDriverState *chr_baum_init(void)
+static CharDriverState *chr_baum_init(const char *id,
+                                      ChardevBackend *backend,
+                                      ChardevReturn *ret,
+                                      Error **errp)
 {
+    ChardevCommon *common = backend->u.braille.data;
     BaumDriverState *baum;
     CharDriverState *chr;
     brlapi_handle_t *handle;
@@ -573,8 +579,12 @@
 #endif
     int tty;
 
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     baum = g_malloc0(sizeof(BaumDriverState));
-    baum->chr = chr = qemu_chr_alloc();
+    baum->chr = chr;
 
     chr->opaque = baum;
     chr->chr_write = baum_write;
@@ -586,14 +596,16 @@
 
     baum->brlapi_fd = brlapi__openConnection(handle, NULL, NULL);
     if (baum->brlapi_fd == -1) {
-        brlapi_perror("baum_init: brlapi_openConnection");
+        error_setg(errp, "brlapi__openConnection: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail_handle;
     }
 
     baum->cellCount_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, baum_cellCount_timer_cb, baum);
 
     if (brlapi__getDisplaySize(handle, &baum->x, &baum->y) == -1) {
-        brlapi_perror("baum_init: brlapi_getDisplaySize");
+        error_setg(errp, "brlapi__getDisplaySize: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail;
     }
 
@@ -609,7 +621,8 @@
         tty = BRLAPI_TTY_DEFAULT;
 
     if (brlapi__enterTtyMode(handle, tty, NULL) == -1) {
-        brlapi_perror("baum_init: brlapi_enterTtyMode");
+        error_setg(errp, "brlapi__enterTtyMode: %s",
+                   brlapi_strerror(brlapi_error_location()));
         goto fail;
     }
 
@@ -629,7 +642,8 @@
 
 static void register_types(void)
 {
-    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL);
+    register_char_driver("braille", CHARDEV_BACKEND_KIND_BRAILLE, NULL,
+                         chr_baum_init);
 }
 
 type_init(register_types);
diff --git a/backends/hostmem-file.c b/backends/hostmem-file.c
index 5179994..5c4b808 100644
--- a/backends/hostmem-file.c
+++ b/backends/hostmem-file.c
@@ -9,6 +9,8 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "sysemu/hostmem.h"
 #include "sysemu/sysemu.h"
@@ -43,18 +45,21 @@
         return;
     }
     if (!fb->mem_path) {
-        error_setg(errp, "mem_path property not set");
+        error_setg(errp, "mem-path property not set");
         return;
     }
 #ifndef CONFIG_LINUX
     error_setg(errp, "-mem-path not supported on this host");
 #else
     if (!memory_region_size(&backend->mr)) {
+        gchar *path;
         backend->force_prealloc = mem_prealloc;
+        path = object_get_canonical_path(OBJECT(backend));
         memory_region_init_ram_from_file(&backend->mr, OBJECT(backend),
-                                 object_get_canonical_path(OBJECT(backend)),
+                                 path,
                                  backend->size, fb->share,
                                  fb->mem_path, errp);
+        g_free(path);
     }
 #endif
 }
@@ -83,9 +88,7 @@
         error_setg(errp, "cannot change property value");
         return;
     }
-    if (fb->mem_path) {
-        g_free(fb->mem_path);
-    }
+    g_free(fb->mem_path);
     fb->mem_path = g_strdup(str);
 }
 
@@ -118,11 +121,19 @@
                             set_mem_path, NULL);
 }
 
+static void file_backend_instance_finalize(Object *o)
+{
+    HostMemoryBackendFile *fb = MEMORY_BACKEND_FILE(o);
+
+    g_free(fb->mem_path);
+}
+
 static const TypeInfo file_backend_info = {
     .name = TYPE_MEMORY_BACKEND_FILE,
     .parent = TYPE_MEMORY_BACKEND,
     .class_init = file_backend_class_init,
     .instance_init = file_backend_instance_init,
+    .instance_finalize = file_backend_instance_finalize,
     .instance_size = sizeof(HostMemoryBackendFile),
 };
 
diff --git a/backends/hostmem-ram.c b/backends/hostmem-ram.c
index a67a134..04a7ac3 100644
--- a/backends/hostmem-ram.c
+++ b/backends/hostmem-ram.c
@@ -9,7 +9,9 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "sysemu/hostmem.h"
+#include "qapi/error.h"
 #include "qom/object_interfaces.h"
 
 #define TYPE_MEMORY_BACKEND_RAM "memory-backend-ram"
diff --git a/backends/hostmem.c b/backends/hostmem.c
index 99e8f99..b7a208d 100644
--- a/backends/hostmem.c
+++ b/backends/hostmem.c
@@ -9,11 +9,13 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "sysemu/hostmem.h"
+#include "hw/boards.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "qapi-types.h"
 #include "qapi-visit.h"
-#include "qapi/qmp/qerror.h"
 #include "qemu/config-file.h"
 #include "qom/object_interfaces.h"
 
@@ -26,18 +28,18 @@
 #endif
 
 static void
-host_memory_backend_get_size(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+host_memory_backend_get_size(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     uint64_t value = backend->size;
 
-    visit_type_size(v, &value, name, errp);
+    visit_type_size(v, name, &value, errp);
 }
 
 static void
-host_memory_backend_set_size(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+host_memory_backend_set_size(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     Error *local_err = NULL;
@@ -48,7 +50,7 @@
         goto out;
     }
 
-    visit_type_size(v, &value, name, &local_err);
+    visit_type_size(v, name, &value, &local_err);
     if (local_err) {
         goto out;
     }
@@ -62,9 +64,17 @@
     error_propagate(errp, local_err);
 }
 
+static uint16List **host_memory_append_node(uint16List **node,
+                                            unsigned long value)
+{
+     *node = g_malloc0(sizeof(**node));
+     (*node)->value = value;
+     return &(*node)->next;
+}
+
 static void
-host_memory_backend_get_host_nodes(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+host_memory_backend_get_host_nodes(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     uint16List *host_nodes = NULL;
@@ -72,13 +82,12 @@
     unsigned long value;
 
     value = find_first_bit(backend->host_nodes, MAX_NODES);
-    if (value == MAX_NODES) {
-        return;
-    }
 
-    *node = g_malloc0(sizeof(**node));
-    (*node)->value = value;
-    node = &(*node)->next;
+    node = host_memory_append_node(node, value);
+
+    if (value == MAX_NODES) {
+        goto out;
+    }
 
     do {
         value = find_next_bit(backend->host_nodes, MAX_NODES, value + 1);
@@ -86,23 +95,22 @@
             break;
         }
 
-        *node = g_malloc0(sizeof(**node));
-        (*node)->value = value;
-        node = &(*node)->next;
+        node = host_memory_append_node(node, value);
     } while (true);
 
-    visit_type_uint16List(v, &host_nodes, name, errp);
+out:
+    visit_type_uint16List(v, name, &host_nodes, errp);
 }
 
 static void
-host_memory_backend_set_host_nodes(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+host_memory_backend_set_host_nodes(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
 #ifdef CONFIG_NUMA
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
     uint16List *l = NULL;
 
-    visit_type_uint16List(v, &l, name, errp);
+    visit_type_uint16List(v, name, &l, errp);
 
     while (l) {
         bitmap_set(backend->host_nodes, l->value, 1);
@@ -113,24 +121,17 @@
 #endif
 }
 
-static void
-host_memory_backend_get_policy(Object *obj, Visitor *v, void *opaque,
-                               const char *name, Error **errp)
+static int
+host_memory_backend_get_policy(Object *obj, Error **errp G_GNUC_UNUSED)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
-    int policy = backend->policy;
-
-    visit_type_enum(v, &policy, HostMemPolicy_lookup, NULL, name, errp);
+    return backend->policy;
 }
 
 static void
-host_memory_backend_set_policy(Object *obj, Visitor *v, void *opaque,
-                               const char *name, Error **errp)
+host_memory_backend_set_policy(Object *obj, int policy, Error **errp)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
-    int policy;
-
-    visit_type_enum(v, &policy, HostMemPolicy_lookup, NULL, name, errp);
     backend->policy = policy;
 
 #ifndef CONFIG_NUMA
@@ -202,6 +203,7 @@
 static void host_memory_backend_set_prealloc(Object *obj, bool value,
                                              Error **errp)
 {
+    Error *local_err = NULL;
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
 
     if (backend->force_prealloc) {
@@ -222,7 +224,11 @@
         void *ptr = memory_region_get_ram_ptr(&backend->mr);
         uint64_t sz = memory_region_size(&backend->mr);
 
-        os_mem_prealloc(fd, ptr, sz);
+        os_mem_prealloc(fd, ptr, sz, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
         backend->prealloc = true;
     }
 }
@@ -230,11 +236,10 @@
 static void host_memory_backend_init(Object *obj)
 {
     HostMemoryBackend *backend = MEMORY_BACKEND(obj);
+    MachineState *machine = MACHINE(qdev_get_machine());
 
-    backend->merge = qemu_opt_get_bool(qemu_get_machine_opts(),
-                                       "mem-merge", true);
-    backend->dump = qemu_opt_get_bool(qemu_get_machine_opts(),
-                                      "dump-guest-core", true);
+    backend->merge = machine_mem_merge(machine);
+    backend->dump = machine_dump_guest_core(machine);
     backend->prealloc = mem_prealloc;
 
     object_property_add_bool(obj, "merge",
@@ -252,9 +257,10 @@
     object_property_add(obj, "host-nodes", "int",
                         host_memory_backend_get_host_nodes,
                         host_memory_backend_set_host_nodes, NULL, NULL, NULL);
-    object_property_add(obj, "policy", "str",
-                        host_memory_backend_get_policy,
-                        host_memory_backend_set_policy, NULL, NULL, NULL);
+    object_property_add_enum(obj, "policy", "HostMemPolicy",
+                             HostMemPolicy_lookup,
+                             host_memory_backend_get_policy,
+                             host_memory_backend_set_policy, NULL);
 }
 
 MemoryRegion *
@@ -263,6 +269,16 @@
     return memory_region_size(&backend->mr) ? &backend->mr : NULL;
 }
 
+void host_memory_backend_set_mapped(HostMemoryBackend *backend, bool mapped)
+{
+    backend->is_mapped = mapped;
+}
+
+bool host_memory_backend_is_mapped(HostMemoryBackend *backend)
+{
+    return backend->is_mapped;
+}
+
 static void
 host_memory_backend_memory_complete(UserCreatable *uc, Error **errp)
 {
@@ -275,8 +291,7 @@
     if (bc->alloc) {
         bc->alloc(backend, &local_err);
         if (local_err) {
-            error_propagate(errp, local_err);
-            return;
+            goto out;
         }
 
         ptr = memory_region_get_ram_ptr(&backend->mr);
@@ -320,9 +335,11 @@
         assert(maxnode <= MAX_NODES);
         if (mbind(ptr, sz, backend->policy,
                   maxnode ? backend->host_nodes : NULL, maxnode + 1, flags)) {
-            error_setg_errno(errp, errno,
-                             "cannot bind memory to host NUMA nodes");
-            return;
+            if (backend->policy != MPOL_DEFAULT || errno != ENOSYS) {
+                error_setg_errno(errp, errno,
+                                 "cannot bind memory to host NUMA nodes");
+                return;
+            }
         }
 #endif
         /* Preallocate memory after the NUMA policy has been instantiated.
@@ -330,9 +347,25 @@
          * specified NUMA policy in place.
          */
         if (backend->prealloc) {
-            os_mem_prealloc(memory_region_get_fd(&backend->mr), ptr, sz);
+            os_mem_prealloc(memory_region_get_fd(&backend->mr), ptr, sz,
+                            &local_err);
+            if (local_err) {
+                goto out;
+            }
         }
     }
+out:
+    error_propagate(errp, local_err);
+}
+
+static bool
+host_memory_backend_can_be_deleted(UserCreatable *uc, Error **errp)
+{
+    if (host_memory_backend_is_mapped(MEMORY_BACKEND(uc))) {
+        return false;
+    } else {
+        return true;
+    }
 }
 
 static void
@@ -341,6 +374,7 @@
     UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
 
     ucc->complete = host_memory_backend_memory_complete;
+    ucc->can_be_deleted = host_memory_backend_can_be_deleted;
 }
 
 static const TypeInfo host_memory_backend_info = {
diff --git a/backends/msmouse.c b/backends/msmouse.c
index 0119110..aeb9055 100644
--- a/backends/msmouse.c
+++ b/backends/msmouse.c
@@ -21,20 +21,55 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 #include "ui/console.h"
+#include "ui/input.h"
 
 #define MSMOUSE_LO6(n) ((n) & 0x3f)
 #define MSMOUSE_HI2(n) (((n) & 0xc0) >> 6)
 
-static void msmouse_event(void *opaque,
-                          int dx, int dy, int dz, int buttons_state)
-{
-    CharDriverState *chr = (CharDriverState *)opaque;
+typedef struct {
+    CharDriverState *chr;
+    QemuInputHandlerState *hs;
+    int axis[INPUT_AXIS__MAX];
+    bool btns[INPUT_BUTTON__MAX];
+    bool btnc[INPUT_BUTTON__MAX];
+    uint8_t outbuf[32];
+    int outlen;
+} MouseState;
 
+static void msmouse_chr_accept_input(CharDriverState *chr)
+{
+    MouseState *mouse = chr->opaque;
+    int len;
+
+    len = qemu_chr_be_can_write(chr);
+    if (len > mouse->outlen) {
+        len = mouse->outlen;
+    }
+    if (!len) {
+        return;
+    }
+
+    qemu_chr_be_write(chr, mouse->outbuf, len);
+    mouse->outlen -= len;
+    if (mouse->outlen) {
+        memmove(mouse->outbuf, mouse->outbuf + len, mouse->outlen);
+    }
+}
+
+static void msmouse_queue_event(MouseState *mouse)
+{
     unsigned char bytes[4] = { 0x40, 0x00, 0x00, 0x00 };
+    int dx, dy, count = 3;
+
+    dx = mouse->axis[INPUT_AXIS_X];
+    mouse->axis[INPUT_AXIS_X] = 0;
+
+    dy = mouse->axis[INPUT_AXIS_Y];
+    mouse->axis[INPUT_AXIS_Y] = 0;
 
     /* Movement deltas */
     bytes[0] |= (MSMOUSE_HI2(dy) << 2) | MSMOUSE_HI2(dx);
@@ -42,14 +77,54 @@
     bytes[2] |= MSMOUSE_LO6(dy);
 
     /* Buttons */
-    bytes[0] |= (buttons_state & 0x01 ? 0x20 : 0x00);
-    bytes[0] |= (buttons_state & 0x02 ? 0x10 : 0x00);
-    bytes[3] |= (buttons_state & 0x04 ? 0x20 : 0x00);
+    bytes[0] |= (mouse->btns[INPUT_BUTTON_LEFT]   ? 0x20 : 0x00);
+    bytes[0] |= (mouse->btns[INPUT_BUTTON_RIGHT]  ? 0x10 : 0x00);
+    if (mouse->btns[INPUT_BUTTON_MIDDLE] ||
+        mouse->btnc[INPUT_BUTTON_MIDDLE]) {
+        bytes[3] |= (mouse->btns[INPUT_BUTTON_MIDDLE] ? 0x20 : 0x00);
+        mouse->btnc[INPUT_BUTTON_MIDDLE] = false;
+        count = 4;
+    }
 
-    /* We always send the packet of, so that we do not have to keep track
-       of previous state of the middle button. This can potentially confuse
-       some very old drivers for two button mice though. */
-    qemu_chr_be_write(chr, bytes, 4);
+    if (mouse->outlen <= sizeof(mouse->outbuf) - count) {
+        memcpy(mouse->outbuf + mouse->outlen, bytes, count);
+        mouse->outlen += count;
+    } else {
+        /* queue full -> drop event */
+    }
+}
+
+static void msmouse_input_event(DeviceState *dev, QemuConsole *src,
+                                InputEvent *evt)
+{
+    MouseState *mouse = (MouseState *)dev;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
+
+    switch (evt->type) {
+    case INPUT_EVENT_KIND_REL:
+        move = evt->u.rel.data;
+        mouse->axis[move->axis] += move->value;
+        break;
+
+    case INPUT_EVENT_KIND_BTN:
+        btn = evt->u.btn.data;
+        mouse->btns[btn->button] = btn->down;
+        mouse->btnc[btn->button] = true;
+        break;
+
+    default:
+        /* keep gcc happy */
+        break;
+    }
+}
+
+static void msmouse_input_sync(DeviceState *dev)
+{
+    MouseState *mouse = (MouseState *)dev;
+
+    msmouse_queue_event(mouse);
+    msmouse_chr_accept_input(mouse->chr);
 }
 
 static int msmouse_chr_write (struct CharDriverState *s, const uint8_t *buf, int len)
@@ -60,26 +135,49 @@
 
 static void msmouse_chr_close (struct CharDriverState *chr)
 {
-    g_free (chr);
+    MouseState *mouse = chr->opaque;
+
+    qemu_input_handler_unregister(mouse->hs);
+    g_free(mouse);
+    g_free(chr);
 }
 
-CharDriverState *qemu_chr_open_msmouse(void)
+static QemuInputHandler msmouse_handler = {
+    .name  = "QEMU Microsoft Mouse",
+    .mask  = INPUT_EVENT_MASK_BTN | INPUT_EVENT_MASK_REL,
+    .event = msmouse_input_event,
+    .sync  = msmouse_input_sync,
+};
+
+static CharDriverState *qemu_chr_open_msmouse(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevCommon *common = backend->u.msmouse.data;
+    MouseState *mouse;
     CharDriverState *chr;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
     chr->chr_write = msmouse_chr_write;
     chr->chr_close = msmouse_chr_close;
+    chr->chr_accept_input = msmouse_chr_accept_input;
     chr->explicit_be_open = true;
 
-    qemu_add_mouse_event_handler(msmouse_event, chr, 0, "QEMU Microsoft Mouse");
+    mouse = g_new0(MouseState, 1);
+    mouse->hs = qemu_input_handler_register((DeviceState *)mouse,
+                                            &msmouse_handler);
+
+    mouse->chr = chr;
+    chr->opaque = mouse;
 
     return chr;
 }
 
 static void register_types(void)
 {
-    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL);
+    register_char_driver("msmouse", CHARDEV_BACKEND_KIND_MSMOUSE, NULL,
+                         qemu_chr_open_msmouse);
 }
 
 type_init(register_types);
diff --git a/backends/rng-egd.c b/backends/rng-egd.c
index 2962795..7a1b924 100644
--- a/backends/rng-egd.c
+++ b/backends/rng-egd.c
@@ -10,8 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/rng.h"
 #include "sysemu/char.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "hw/qdev.h" /* just for DEFINE_PROP_CHR */
 
@@ -24,33 +26,12 @@
 
     CharDriverState *chr;
     char *chr_name;
-
-    GSList *requests;
 } RngEgd;
 
-typedef struct RngRequest
-{
-    EntropyReceiveFunc *receive_entropy;
-    uint8_t *data;
-    void *opaque;
-    size_t offset;
-    size_t size;
-} RngRequest;
-
-static void rng_egd_request_entropy(RngBackend *b, size_t size,
-                                    EntropyReceiveFunc *receive_entropy,
-                                    void *opaque)
+static void rng_egd_request_entropy(RngBackend *b, RngRequest *req)
 {
     RngEgd *s = RNG_EGD(b);
-    RngRequest *req;
-
-    req = g_malloc(sizeof(*req));
-
-    req->offset = 0;
-    req->size = size;
-    req->receive_entropy = receive_entropy;
-    req->opaque = opaque;
-    req->data = g_malloc(req->size);
+    size_t size = req->size;
 
     while (size > 0) {
         uint8_t header[2];
@@ -64,24 +45,15 @@
 
         size -= len;
     }
-
-    s->requests = g_slist_append(s->requests, req);
-}
-
-static void rng_egd_free_request(RngRequest *req)
-{
-    g_free(req->data);
-    g_free(req);
 }
 
 static int rng_egd_chr_can_read(void *opaque)
 {
     RngEgd *s = RNG_EGD(opaque);
-    GSList *i;
+    RngRequest *req;
     int size = 0;
 
-    for (i = s->requests; i; i = i->next) {
-        RngRequest *req = i->data;
+    QSIMPLEQ_FOREACH(req, &s->parent.requests, next) {
         size += req->size - req->offset;
     }
 
@@ -93,8 +65,8 @@
     RngEgd *s = RNG_EGD(opaque);
     size_t buf_offset = 0;
 
-    while (size > 0 && s->requests) {
-        RngRequest *req = s->requests->data;
+    while (size > 0 && !QSIMPLEQ_EMPTY(&s->parent.requests)) {
+        RngRequest *req = QSIMPLEQ_FIRST(&s->parent.requests);
         int len = MIN(size, req->size - req->offset);
 
         memcpy(req->data + req->offset, buf + buf_offset, len);
@@ -103,56 +75,32 @@
         size -= len;
 
         if (req->offset == req->size) {
-            s->requests = g_slist_remove_link(s->requests, s->requests);
-
             req->receive_entropy(req->opaque, req->data, req->size);
 
-            rng_egd_free_request(req);
+            rng_backend_finalize_request(&s->parent, req);
         }
     }
 }
 
-static void rng_egd_free_requests(RngEgd *s)
-{
-    GSList *i;
-
-    for (i = s->requests; i; i = i->next) {
-        rng_egd_free_request(i->data);
-    }
-
-    g_slist_free(s->requests);
-    s->requests = NULL;
-}
-
-static void rng_egd_cancel_requests(RngBackend *b)
-{
-    RngEgd *s = RNG_EGD(b);
-
-    /* We simply delete the list of pending requests.  If there is data in the 
-     * queue waiting to be read, this is okay, because there will always be
-     * more data than we requested originally
-     */
-    rng_egd_free_requests(s);
-}
-
 static void rng_egd_opened(RngBackend *b, Error **errp)
 {
     RngEgd *s = RNG_EGD(b);
 
     if (s->chr_name == NULL) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  "chardev", "a valid character device");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "chardev", "a valid character device");
         return;
     }
 
     s->chr = qemu_chr_find(s->chr_name);
     if (s->chr == NULL) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, s->chr_name);
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", s->chr_name);
         return;
     }
 
     if (qemu_chr_fe_claim(s->chr) != 0) {
-        error_set(errp, QERR_DEVICE_IN_USE, s->chr_name);
+        error_setg(errp, QERR_DEVICE_IN_USE, s->chr_name);
         return;
     }
 
@@ -167,7 +115,7 @@
     RngEgd *s = RNG_EGD(b);
 
     if (b->opened) {
-        error_set(errp, QERR_PERMISSION_DENIED);
+        error_setg(errp, QERR_PERMISSION_DENIED);
     } else {
         g_free(s->chr_name);
         s->chr_name = g_strdup(value);
@@ -202,8 +150,6 @@
     }
 
     g_free(s->chr_name);
-
-    rng_egd_free_requests(s);
 }
 
 static void rng_egd_class_init(ObjectClass *klass, void *data)
@@ -211,7 +157,6 @@
     RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
 
     rbc->request_entropy = rng_egd_request_entropy;
-    rbc->cancel_requests = rng_egd_cancel_requests;
     rbc->opened = rng_egd_opened;
 }
 
diff --git a/backends/rng-random.c b/backends/rng-random.c
index 601d9dc..e2a49b0 100644
--- a/backends/rng-random.c
+++ b/backends/rng-random.c
@@ -10,21 +10,19 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/rng-random.h"
 #include "sysemu/rng.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/main-loop.h"
 
-struct RndRandom
+struct RngRandom
 {
     RngBackend parent;
 
     int fd;
     char *filename;
-
-    EntropyReceiveFunc *receive_func;
-    void *opaque;
-    size_t size;
 };
 
 /**
@@ -36,46 +34,45 @@
 
 static void entropy_available(void *opaque)
 {
-    RndRandom *s = RNG_RANDOM(opaque);
-    uint8_t buffer[s->size];
-    ssize_t len;
+    RngRandom *s = RNG_RANDOM(opaque);
 
-    len = read(s->fd, buffer, s->size);
-    if (len < 0 && errno == EAGAIN) {
-        return;
+    while (!QSIMPLEQ_EMPTY(&s->parent.requests)) {
+        RngRequest *req = QSIMPLEQ_FIRST(&s->parent.requests);
+        ssize_t len;
+
+        len = read(s->fd, req->data, req->size);
+        if (len < 0 && errno == EAGAIN) {
+            return;
+        }
+        g_assert(len != -1);
+
+        req->receive_entropy(req->opaque, req->data, len);
+
+        rng_backend_finalize_request(&s->parent, req);
     }
-    g_assert(len != -1);
 
-    s->receive_func(s->opaque, buffer, len);
-    s->receive_func = NULL;
-
+    /* We've drained all requests, the fd handler can be reset. */
     qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
 }
 
-static void rng_random_request_entropy(RngBackend *b, size_t size,
-                                        EntropyReceiveFunc *receive_entropy,
-                                        void *opaque)
+static void rng_random_request_entropy(RngBackend *b, RngRequest *req)
 {
-    RndRandom *s = RNG_RANDOM(b);
+    RngRandom *s = RNG_RANDOM(b);
 
-    if (s->receive_func) {
-        s->receive_func(s->opaque, NULL, 0);
+    if (QSIMPLEQ_EMPTY(&s->parent.requests)) {
+        /* If there are no pending requests yet, we need to
+         * install our fd handler. */
+        qemu_set_fd_handler(s->fd, entropy_available, NULL, s);
     }
-
-    s->receive_func = receive_entropy;
-    s->opaque = opaque;
-    s->size = size;
-
-    qemu_set_fd_handler(s->fd, entropy_available, NULL, s);
 }
 
 static void rng_random_opened(RngBackend *b, Error **errp)
 {
-    RndRandom *s = RNG_RANDOM(b);
+    RngRandom *s = RNG_RANDOM(b);
 
     if (s->filename == NULL) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  "filename", "a valid filename");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "filename", "a valid filename");
     } else {
         s->fd = qemu_open(s->filename, O_RDONLY | O_NONBLOCK);
         if (s->fd == -1) {
@@ -86,23 +83,19 @@
 
 static char *rng_random_get_filename(Object *obj, Error **errp)
 {
-    RndRandom *s = RNG_RANDOM(obj);
+    RngRandom *s = RNG_RANDOM(obj);
 
-    if (s->filename) {
-        return g_strdup(s->filename);
-    }
-
-    return NULL;
+    return g_strdup(s->filename);
 }
 
 static void rng_random_set_filename(Object *obj, const char *filename,
                                  Error **errp)
 {
     RngBackend *b = RNG_BACKEND(obj);
-    RndRandom *s = RNG_RANDOM(obj);
+    RngRandom *s = RNG_RANDOM(obj);
 
     if (b->opened) {
-        error_set(errp, QERR_PERMISSION_DENIED);
+        error_setg(errp, QERR_PERMISSION_DENIED);
         return;
     }
 
@@ -112,7 +105,7 @@
 
 static void rng_random_init(Object *obj)
 {
-    RndRandom *s = RNG_RANDOM(obj);
+    RngRandom *s = RNG_RANDOM(obj);
 
     object_property_add_str(obj, "filename",
                             rng_random_get_filename,
@@ -125,7 +118,7 @@
 
 static void rng_random_finalize(Object *obj)
 {
-    RndRandom *s = RNG_RANDOM(obj);
+    RngRandom *s = RNG_RANDOM(obj);
 
     if (s->fd != -1) {
         qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
@@ -146,7 +139,7 @@
 static const TypeInfo rng_random_info = {
     .name = TYPE_RNG_RANDOM,
     .parent = TYPE_RNG_BACKEND,
-    .instance_size = sizeof(RndRandom),
+    .instance_size = sizeof(RngRandom),
     .class_init = rng_random_class_init,
     .instance_init = rng_random_init,
     .instance_finalize = rng_random_finalize,
diff --git a/backends/rng.c b/backends/rng.c
index 0f2fc11..398ebe4 100644
--- a/backends/rng.c
+++ b/backends/rng.c
@@ -10,7 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/rng.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qom/object_interfaces.h"
 
@@ -19,18 +21,20 @@
                                  void *opaque)
 {
     RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+    RngRequest *req;
 
     if (k->request_entropy) {
-        k->request_entropy(s, size, receive_entropy, opaque);
-    }
-}
+        req = g_malloc(sizeof(*req));
 
-void rng_backend_cancel_requests(RngBackend *s)
-{
-    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+        req->offset = 0;
+        req->size = size;
+        req->receive_entropy = receive_entropy;
+        req->opaque = opaque;
+        req->data = g_malloc(req->size);
 
-    if (k->cancel_requests) {
-        k->cancel_requests(s);
+        k->request_entropy(s, req);
+
+        QSIMPLEQ_INSERT_TAIL(&s->requests, req, next);
     }
 }
 
@@ -57,7 +61,7 @@
     }
 
     if (!value && s->opened) {
-        error_set(errp, QERR_PERMISSION_DENIED);
+        error_setg(errp, QERR_PERMISSION_DENIED);
         return;
     }
 
@@ -72,14 +76,48 @@
     s->opened = true;
 }
 
+static void rng_backend_free_request(RngRequest *req)
+{
+    g_free(req->data);
+    g_free(req);
+}
+
+static void rng_backend_free_requests(RngBackend *s)
+{
+    RngRequest *req, *next;
+
+    QSIMPLEQ_FOREACH_SAFE(req, &s->requests, next, next) {
+        rng_backend_free_request(req);
+    }
+
+    QSIMPLEQ_INIT(&s->requests);
+}
+
+void rng_backend_finalize_request(RngBackend *s, RngRequest *req)
+{
+    QSIMPLEQ_REMOVE(&s->requests, req, RngRequest, next);
+    rng_backend_free_request(req);
+}
+
 static void rng_backend_init(Object *obj)
 {
+    RngBackend *s = RNG_BACKEND(obj);
+
+    QSIMPLEQ_INIT(&s->requests);
+
     object_property_add_bool(obj, "opened",
                              rng_backend_prop_get_opened,
                              rng_backend_prop_set_opened,
                              NULL);
 }
 
+static void rng_backend_finalize(Object *obj)
+{
+    RngBackend *s = RNG_BACKEND(obj);
+
+    rng_backend_free_requests(s);
+}
+
 static void rng_backend_class_init(ObjectClass *oc, void *data)
 {
     UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
@@ -92,6 +130,7 @@
     .parent = TYPE_OBJECT,
     .instance_size = sizeof(RngBackend),
     .instance_init = rng_backend_init,
+    .instance_finalize = rng_backend_finalize,
     .class_size = sizeof(RngBackendClass),
     .class_init = rng_backend_class_init,
     .abstract = true,
diff --git a/backends/testdev.c b/backends/testdev.c
index eba396a..3ab1c90 100644
--- a/backends/testdev.c
+++ b/backends/testdev.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 
@@ -108,13 +109,16 @@
     g_free(testdev);
 }
 
-CharDriverState *chr_testdev_init(void)
+static CharDriverState *chr_testdev_init(const char *id,
+                                         ChardevBackend *backend,
+                                         ChardevReturn *ret,
+                                         Error **errp)
 {
     TestdevCharState *testdev;
     CharDriverState *chr;
 
-    testdev = g_malloc0(sizeof(TestdevCharState));
-    testdev->chr = chr = g_malloc0(sizeof(CharDriverState));
+    testdev = g_new0(TestdevCharState, 1);
+    testdev->chr = chr = g_new0(CharDriverState, 1);
 
     chr->opaque = testdev;
     chr->chr_write = testdev_write;
@@ -125,7 +129,8 @@
 
 static void register_types(void)
 {
-    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL);
+    register_char_driver("testdev", CHARDEV_BACKEND_KIND_TESTDEV, NULL,
+                         chr_testdev_init);
 }
 
 type_init(register_types);
diff --git a/backends/tpm.c b/backends/tpm.c
index 01860c4..536f262 100644
--- a/backends/tpm.c
+++ b/backends/tpm.c
@@ -12,7 +12,9 @@
  * Based on backends/rng.c by Anthony Liguori
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/tpm_backend.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/tpm.h"
 #include "qemu/thread.h"
@@ -36,7 +38,7 @@
 {
     TPMBackendClass *k = TPM_BACKEND_GET_CLASS(s);
 
-    return k->ops->destroy(s);
+    k->ops->destroy(s);
 }
 
 int tpm_backend_init(TPMBackend *s, TPMState *state,
@@ -96,6 +98,20 @@
     return k->ops->get_tpm_established_flag(s);
 }
 
+int tpm_backend_reset_tpm_established_flag(TPMBackend *s, uint8_t locty)
+{
+    TPMBackendClass *k = TPM_BACKEND_GET_CLASS(s);
+
+    return k->ops->reset_tpm_established_flag(s, locty);
+}
+
+TPMVersion tpm_backend_get_tpm_version(TPMBackend *s)
+{
+    TPMBackendClass *k = TPM_BACKEND_GET_CLASS(s);
+
+    return k->ops->get_tpm_version(s);
+}
+
 static bool tpm_backend_prop_get_opened(Object *obj, Error **errp)
 {
     TPMBackend *s = TPM_BACKEND(obj);
@@ -119,7 +135,7 @@
     }
 
     if (!value && s->opened) {
-        error_set(errp, QERR_PERMISSION_DENIED);
+        error_setg(errp, QERR_PERMISSION_DENIED);
         return;
     }
 
@@ -165,17 +181,6 @@
     }
 }
 
-void tpm_backend_thread_tpm_reset(TPMBackendThread *tbt,
-                                  GFunc func, gpointer user_data)
-{
-    if (!tbt->pool) {
-        tpm_backend_thread_create(tbt, func, user_data);
-    } else {
-        g_thread_pool_push(tbt->pool, (gpointer)TPM_BACKEND_CMD_TPM_RESET,
-                           NULL);
-    }
-}
-
 static const TypeInfo tpm_backend_info = {
     .name = TYPE_TPM_BACKEND,
     .parent = TYPE_OBJECT,
diff --git a/balloon.c b/balloon.c
index b70da4f..f2ef50c 100644
--- a/balloon.c
+++ b/balloon.c
@@ -24,17 +24,45 @@
  * THE SOFTWARE.
  */
 
-#include "monitor/monitor.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "exec/cpu-common.h"
 #include "sysemu/kvm.h"
 #include "sysemu/balloon.h"
 #include "trace.h"
 #include "qmp-commands.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qjson.h"
 
 static QEMUBalloonEvent *balloon_event_fn;
 static QEMUBalloonStatus *balloon_stat_fn;
 static void *balloon_opaque;
+static bool balloon_inhibited;
+
+bool qemu_balloon_is_inhibited(void)
+{
+    return balloon_inhibited;
+}
+
+void qemu_balloon_inhibit(bool state)
+{
+    balloon_inhibited = state;
+}
+
+static bool have_balloon(Error **errp)
+{
+    if (kvm_enabled() && !kvm_has_sync_mmu()) {
+        error_set(errp, ERROR_CLASS_KVM_MISSING_CAP,
+                  "Using KVM without synchronous MMU, balloon unavailable");
+        return false;
+    }
+    if (!balloon_event_fn) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_ACTIVE,
+                  "No balloon device has been activated");
+        return false;
+    }
+    return true;
+}
 
 int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
                              QEMUBalloonStatus *stat_func, void *opaque)
@@ -43,7 +71,6 @@
         /* We're already registered one balloon handler.  How many can
          * a guest really have?
          */
-        error_report("Another balloon device already registered");
         return -1;
     }
     balloon_event_fn = event_func;
@@ -62,58 +89,30 @@
     balloon_opaque = NULL;
 }
 
-static int qemu_balloon(ram_addr_t target)
-{
-    if (!balloon_event_fn) {
-        return 0;
-    }
-    trace_balloon_event(balloon_opaque, target);
-    balloon_event_fn(balloon_opaque, target);
-    return 1;
-}
-
-static int qemu_balloon_status(BalloonInfo *info)
-{
-    if (!balloon_stat_fn) {
-        return 0;
-    }
-    balloon_stat_fn(balloon_opaque, info);
-    return 1;
-}
-
 BalloonInfo *qmp_query_balloon(Error **errp)
 {
     BalloonInfo *info;
 
-    if (kvm_enabled() && !kvm_has_sync_mmu()) {
-        error_set(errp, QERR_KVM_MISSING_CAP, "synchronous MMU", "balloon");
+    if (!have_balloon(errp)) {
         return NULL;
     }
 
     info = g_malloc0(sizeof(*info));
-
-    if (qemu_balloon_status(info) == 0) {
-        error_set(errp, QERR_DEVICE_NOT_ACTIVE, "balloon");
-        qapi_free_BalloonInfo(info);
-        return NULL;
-    }
-
+    balloon_stat_fn(balloon_opaque, info);
     return info;
 }
 
-void qmp_balloon(int64_t value, Error **errp)
+void qmp_balloon(int64_t target, Error **errp)
 {
-    if (kvm_enabled() && !kvm_has_sync_mmu()) {
-        error_set(errp, QERR_KVM_MISSING_CAP, "synchronous MMU", "balloon");
+    if (!have_balloon(errp)) {
         return;
     }
 
-    if (value <= 0) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "target", "a size");
+    if (target <= 0) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "target", "a size");
         return;
     }
-    
-    if (qemu_balloon(value) == 0) {
-        error_set(errp, QERR_DEVICE_NOT_ACTIVE, "balloon");
-    }
+
+    trace_balloon_event(balloon_opaque, target);
+    balloon_event_fn(balloon_opaque, target);
 }
diff --git a/block-migration.c b/block-migration.c
deleted file mode 100644
index 08db01a..0000000
--- a/block-migration.c
+++ /dev/null
@@ -1,884 +0,0 @@
-/*
- * QEMU live block migration
- *
- * Copyright IBM, Corp. 2009
- *
- * Authors:
- *  Liran Schour   <lirans@il.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "block/block.h"
-#include "qemu/error-report.h"
-#include "qemu/main-loop.h"
-#include "hw/hw.h"
-#include "qemu/queue.h"
-#include "qemu/timer.h"
-#include "migration/block.h"
-#include "migration/migration.h"
-#include "sysemu/blockdev.h"
-#include <assert.h>
-
-#define BLOCK_SIZE                       (1 << 20)
-#define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
-
-#define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
-#define BLK_MIG_FLAG_EOS                0x02
-#define BLK_MIG_FLAG_PROGRESS           0x04
-#define BLK_MIG_FLAG_ZERO_BLOCK         0x08
-
-#define MAX_IS_ALLOCATED_SEARCH 65536
-
-//#define DEBUG_BLK_MIGRATION
-
-#ifdef DEBUG_BLK_MIGRATION
-#define DPRINTF(fmt, ...) \
-    do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-typedef struct BlkMigDevState {
-    /* Written during setup phase.  Can be read without a lock.  */
-    BlockDriverState *bs;
-    int shared_base;
-    int64_t total_sectors;
-    QSIMPLEQ_ENTRY(BlkMigDevState) entry;
-
-    /* Only used by migration thread.  Does not need a lock.  */
-    int bulk_completed;
-    int64_t cur_sector;
-    int64_t cur_dirty;
-
-    /* Protected by block migration lock.  */
-    unsigned long *aio_bitmap;
-    int64_t completed_sectors;
-    BdrvDirtyBitmap *dirty_bitmap;
-    Error *blocker;
-} BlkMigDevState;
-
-typedef struct BlkMigBlock {
-    /* Only used by migration thread.  */
-    uint8_t *buf;
-    BlkMigDevState *bmds;
-    int64_t sector;
-    int nr_sectors;
-    struct iovec iov;
-    QEMUIOVector qiov;
-    BlockAIOCB *aiocb;
-
-    /* Protected by block migration lock.  */
-    int ret;
-    QSIMPLEQ_ENTRY(BlkMigBlock) entry;
-} BlkMigBlock;
-
-typedef struct BlkMigState {
-    /* Written during setup phase.  Can be read without a lock.  */
-    int blk_enable;
-    int shared_base;
-    QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
-    int64_t total_sector_sum;
-    bool zero_blocks;
-
-    /* Protected by lock.  */
-    QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
-    int submitted;
-    int read_done;
-
-    /* Only used by migration thread.  Does not need a lock.  */
-    int transferred;
-    int prev_progress;
-    int bulk_completed;
-
-    /* Lock must be taken _inside_ the iothread lock.  */
-    QemuMutex lock;
-} BlkMigState;
-
-static BlkMigState block_mig_state;
-
-static void blk_mig_lock(void)
-{
-    qemu_mutex_lock(&block_mig_state.lock);
-}
-
-static void blk_mig_unlock(void)
-{
-    qemu_mutex_unlock(&block_mig_state.lock);
-}
-
-/* Must run outside of the iothread lock during the bulk phase,
- * or the VM will stall.
- */
-
-static void blk_send(QEMUFile *f, BlkMigBlock * blk)
-{
-    int len;
-    uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
-
-    if (block_mig_state.zero_blocks &&
-        buffer_is_zero(blk->buf, BLOCK_SIZE)) {
-        flags |= BLK_MIG_FLAG_ZERO_BLOCK;
-    }
-
-    /* sector number and flags */
-    qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
-                     | flags);
-
-    /* device name */
-    len = strlen(bdrv_get_device_name(blk->bmds->bs));
-    qemu_put_byte(f, len);
-    qemu_put_buffer(f, (uint8_t *)bdrv_get_device_name(blk->bmds->bs), len);
-
-    /* if a block is zero we need to flush here since the network
-     * bandwidth is now a lot higher than the storage device bandwidth.
-     * thus if we queue zero blocks we slow down the migration */
-    if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
-        qemu_fflush(f);
-        return;
-    }
-
-    qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
-}
-
-int blk_mig_active(void)
-{
-    return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
-}
-
-uint64_t blk_mig_bytes_transferred(void)
-{
-    BlkMigDevState *bmds;
-    uint64_t sum = 0;
-
-    blk_mig_lock();
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        sum += bmds->completed_sectors;
-    }
-    blk_mig_unlock();
-    return sum << BDRV_SECTOR_BITS;
-}
-
-uint64_t blk_mig_bytes_remaining(void)
-{
-    return blk_mig_bytes_total() - blk_mig_bytes_transferred();
-}
-
-uint64_t blk_mig_bytes_total(void)
-{
-    BlkMigDevState *bmds;
-    uint64_t sum = 0;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        sum += bmds->total_sectors;
-    }
-    return sum << BDRV_SECTOR_BITS;
-}
-
-
-/* Called with migration lock held.  */
-
-static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
-{
-    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    if (sector < bdrv_nb_sectors(bmds->bs)) {
-        return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
-            (1UL << (chunk % (sizeof(unsigned long) * 8))));
-    } else {
-        return 0;
-    }
-}
-
-/* Called with migration lock held.  */
-
-static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
-                             int nb_sectors, int set)
-{
-    int64_t start, end;
-    unsigned long val, idx, bit;
-
-    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
-    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    for (; start <= end; start++) {
-        idx = start / (sizeof(unsigned long) * 8);
-        bit = start % (sizeof(unsigned long) * 8);
-        val = bmds->aio_bitmap[idx];
-        if (set) {
-            val |= 1UL << bit;
-        } else {
-            val &= ~(1UL << bit);
-        }
-        bmds->aio_bitmap[idx] = val;
-    }
-}
-
-static void alloc_aio_bitmap(BlkMigDevState *bmds)
-{
-    BlockDriverState *bs = bmds->bs;
-    int64_t bitmap_size;
-
-    bitmap_size = bdrv_nb_sectors(bs) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
-    bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
-
-    bmds->aio_bitmap = g_malloc0(bitmap_size);
-}
-
-/* Never hold migration lock when yielding to the main loop!  */
-
-static void blk_mig_read_cb(void *opaque, int ret)
-{
-    BlkMigBlock *blk = opaque;
-
-    blk_mig_lock();
-    blk->ret = ret;
-
-    QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
-    bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
-
-    block_mig_state.submitted--;
-    block_mig_state.read_done++;
-    assert(block_mig_state.submitted >= 0);
-    blk_mig_unlock();
-}
-
-/* Called with no lock taken.  */
-
-static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
-{
-    int64_t total_sectors = bmds->total_sectors;
-    int64_t cur_sector = bmds->cur_sector;
-    BlockDriverState *bs = bmds->bs;
-    BlkMigBlock *blk;
-    int nr_sectors;
-
-    if (bmds->shared_base) {
-        qemu_mutex_lock_iothread();
-        while (cur_sector < total_sectors &&
-               !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH,
-                                  &nr_sectors)) {
-            cur_sector += nr_sectors;
-        }
-        qemu_mutex_unlock_iothread();
-    }
-
-    if (cur_sector >= total_sectors) {
-        bmds->cur_sector = bmds->completed_sectors = total_sectors;
-        return 1;
-    }
-
-    bmds->completed_sectors = cur_sector;
-
-    cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
-
-    /* we are going to transfer a full block even if it is not allocated */
-    nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
-
-    if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
-        nr_sectors = total_sectors - cur_sector;
-    }
-
-    blk = g_new(BlkMigBlock, 1);
-    blk->buf = g_malloc(BLOCK_SIZE);
-    blk->bmds = bmds;
-    blk->sector = cur_sector;
-    blk->nr_sectors = nr_sectors;
-
-    blk->iov.iov_base = blk->buf;
-    blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
-    qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
-
-    blk_mig_lock();
-    block_mig_state.submitted++;
-    blk_mig_unlock();
-
-    qemu_mutex_lock_iothread();
-    blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov,
-                                nr_sectors, blk_mig_read_cb, blk);
-
-    bdrv_reset_dirty(bs, cur_sector, nr_sectors);
-    qemu_mutex_unlock_iothread();
-
-    bmds->cur_sector = cur_sector + nr_sectors;
-    return (bmds->cur_sector >= total_sectors);
-}
-
-/* Called with iothread lock taken.  */
-
-static int set_dirty_tracking(void)
-{
-    BlkMigDevState *bmds;
-    int ret;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE,
-                                                      NULL);
-        if (!bmds->dirty_bitmap) {
-            ret = -errno;
-            goto fail;
-        }
-    }
-    return 0;
-
-fail:
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        if (bmds->dirty_bitmap) {
-            bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
-        }
-    }
-    return ret;
-}
-
-static void unset_dirty_tracking(void)
-{
-    BlkMigDevState *bmds;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap);
-    }
-}
-
-static void init_blk_migration(QEMUFile *f)
-{
-    BlockDriverState *bs;
-    BlkMigDevState *bmds;
-    int64_t sectors;
-
-    block_mig_state.submitted = 0;
-    block_mig_state.read_done = 0;
-    block_mig_state.transferred = 0;
-    block_mig_state.total_sector_sum = 0;
-    block_mig_state.prev_progress = -1;
-    block_mig_state.bulk_completed = 0;
-    block_mig_state.zero_blocks = migrate_zero_blocks();
-
-    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-        if (bdrv_is_read_only(bs)) {
-            continue;
-        }
-
-        sectors = bdrv_nb_sectors(bs);
-        if (sectors <= 0) {
-            return;
-        }
-
-        bmds = g_new0(BlkMigDevState, 1);
-        bmds->bs = bs;
-        bmds->bulk_completed = 0;
-        bmds->total_sectors = sectors;
-        bmds->completed_sectors = 0;
-        bmds->shared_base = block_mig_state.shared_base;
-        alloc_aio_bitmap(bmds);
-        error_setg(&bmds->blocker, "block device is in use by migration");
-        bdrv_op_block_all(bs, bmds->blocker);
-        bdrv_ref(bs);
-
-        block_mig_state.total_sector_sum += sectors;
-
-        if (bmds->shared_base) {
-            DPRINTF("Start migration for %s with shared base image\n",
-                    bdrv_get_device_name(bs));
-        } else {
-            DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
-        }
-
-        QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
-    }
-}
-
-/* Called with no lock taken.  */
-
-static int blk_mig_save_bulked_block(QEMUFile *f)
-{
-    int64_t completed_sector_sum = 0;
-    BlkMigDevState *bmds;
-    int progress;
-    int ret = 0;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        if (bmds->bulk_completed == 0) {
-            if (mig_save_device_bulk(f, bmds) == 1) {
-                /* completed bulk section for this device */
-                bmds->bulk_completed = 1;
-            }
-            completed_sector_sum += bmds->completed_sectors;
-            ret = 1;
-            break;
-        } else {
-            completed_sector_sum += bmds->completed_sectors;
-        }
-    }
-
-    if (block_mig_state.total_sector_sum != 0) {
-        progress = completed_sector_sum * 100 /
-                   block_mig_state.total_sector_sum;
-    } else {
-        progress = 100;
-    }
-    if (progress != block_mig_state.prev_progress) {
-        block_mig_state.prev_progress = progress;
-        qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
-                         | BLK_MIG_FLAG_PROGRESS);
-        DPRINTF("Completed %d %%\r", progress);
-    }
-
-    return ret;
-}
-
-static void blk_mig_reset_dirty_cursor(void)
-{
-    BlkMigDevState *bmds;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        bmds->cur_dirty = 0;
-    }
-}
-
-/* Called with iothread lock taken.  */
-
-static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
-                                 int is_async)
-{
-    BlkMigBlock *blk;
-    int64_t total_sectors = bmds->total_sectors;
-    int64_t sector;
-    int nr_sectors;
-    int ret = -EIO;
-
-    for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
-        blk_mig_lock();
-        if (bmds_aio_inflight(bmds, sector)) {
-            blk_mig_unlock();
-            bdrv_drain_all();
-        } else {
-            blk_mig_unlock();
-        }
-        if (bdrv_get_dirty(bmds->bs, bmds->dirty_bitmap, sector)) {
-
-            if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
-                nr_sectors = total_sectors - sector;
-            } else {
-                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
-            }
-            blk = g_new(BlkMigBlock, 1);
-            blk->buf = g_malloc(BLOCK_SIZE);
-            blk->bmds = bmds;
-            blk->sector = sector;
-            blk->nr_sectors = nr_sectors;
-
-            if (is_async) {
-                blk->iov.iov_base = blk->buf;
-                blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
-                qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
-
-                blk->aiocb = bdrv_aio_readv(bmds->bs, sector, &blk->qiov,
-                                            nr_sectors, blk_mig_read_cb, blk);
-
-                blk_mig_lock();
-                block_mig_state.submitted++;
-                bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
-                blk_mig_unlock();
-            } else {
-                ret = bdrv_read(bmds->bs, sector, blk->buf, nr_sectors);
-                if (ret < 0) {
-                    goto error;
-                }
-                blk_send(f, blk);
-
-                g_free(blk->buf);
-                g_free(blk);
-            }
-
-            bdrv_reset_dirty(bmds->bs, sector, nr_sectors);
-            break;
-        }
-        sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
-        bmds->cur_dirty = sector;
-    }
-
-    return (bmds->cur_dirty >= bmds->total_sectors);
-
-error:
-    DPRINTF("Error reading sector %" PRId64 "\n", sector);
-    g_free(blk->buf);
-    g_free(blk);
-    return ret;
-}
-
-/* Called with iothread lock taken.
- *
- * return value:
- * 0: too much data for max_downtime
- * 1: few enough data for max_downtime
-*/
-static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
-{
-    BlkMigDevState *bmds;
-    int ret = 1;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        ret = mig_save_device_dirty(f, bmds, is_async);
-        if (ret <= 0) {
-            break;
-        }
-    }
-
-    return ret;
-}
-
-/* Called with no locks taken.  */
-
-static int flush_blks(QEMUFile *f)
-{
-    BlkMigBlock *blk;
-    int ret = 0;
-
-    DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
-            __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
-            block_mig_state.transferred);
-
-    blk_mig_lock();
-    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
-        if (qemu_file_rate_limit(f)) {
-            break;
-        }
-        if (blk->ret < 0) {
-            ret = blk->ret;
-            break;
-        }
-
-        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
-        blk_mig_unlock();
-        blk_send(f, blk);
-        blk_mig_lock();
-
-        g_free(blk->buf);
-        g_free(blk);
-
-        block_mig_state.read_done--;
-        block_mig_state.transferred++;
-        assert(block_mig_state.read_done >= 0);
-    }
-    blk_mig_unlock();
-
-    DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
-            block_mig_state.submitted, block_mig_state.read_done,
-            block_mig_state.transferred);
-    return ret;
-}
-
-/* Called with iothread lock taken.  */
-
-static int64_t get_remaining_dirty(void)
-{
-    BlkMigDevState *bmds;
-    int64_t dirty = 0;
-
-    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        dirty += bdrv_get_dirty_count(bmds->bs, bmds->dirty_bitmap);
-    }
-
-    return dirty << BDRV_SECTOR_BITS;
-}
-
-/* Called with iothread lock taken.  */
-
-static void blk_mig_cleanup(void)
-{
-    BlkMigDevState *bmds;
-    BlkMigBlock *blk;
-
-    bdrv_drain_all();
-
-    unset_dirty_tracking();
-
-    blk_mig_lock();
-    while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
-        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
-        bdrv_op_unblock_all(bmds->bs, bmds->blocker);
-        error_free(bmds->blocker);
-        bdrv_unref(bmds->bs);
-        g_free(bmds->aio_bitmap);
-        g_free(bmds);
-    }
-
-    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
-        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
-        g_free(blk->buf);
-        g_free(blk);
-    }
-    blk_mig_unlock();
-}
-
-static void block_migration_cancel(void *opaque)
-{
-    blk_mig_cleanup();
-}
-
-static int block_save_setup(QEMUFile *f, void *opaque)
-{
-    int ret;
-
-    DPRINTF("Enter save live setup submitted %d transferred %d\n",
-            block_mig_state.submitted, block_mig_state.transferred);
-
-    qemu_mutex_lock_iothread();
-    init_blk_migration(f);
-
-    /* start track dirty blocks */
-    ret = set_dirty_tracking();
-
-    if (ret) {
-        qemu_mutex_unlock_iothread();
-        return ret;
-    }
-
-    qemu_mutex_unlock_iothread();
-
-    ret = flush_blks(f);
-    blk_mig_reset_dirty_cursor();
-    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
-
-    return ret;
-}
-
-static int block_save_iterate(QEMUFile *f, void *opaque)
-{
-    int ret;
-    int64_t last_ftell = qemu_ftell(f);
-
-    DPRINTF("Enter save live iterate submitted %d transferred %d\n",
-            block_mig_state.submitted, block_mig_state.transferred);
-
-    ret = flush_blks(f);
-    if (ret) {
-        return ret;
-    }
-
-    blk_mig_reset_dirty_cursor();
-
-    /* control the rate of transfer */
-    blk_mig_lock();
-    while ((block_mig_state.submitted +
-            block_mig_state.read_done) * BLOCK_SIZE <
-           qemu_file_get_rate_limit(f)) {
-        blk_mig_unlock();
-        if (block_mig_state.bulk_completed == 0) {
-            /* first finish the bulk phase */
-            if (blk_mig_save_bulked_block(f) == 0) {
-                /* finished saving bulk on all devices */
-                block_mig_state.bulk_completed = 1;
-            }
-            ret = 0;
-        } else {
-            /* Always called with iothread lock taken for
-             * simplicity, block_save_complete also calls it.
-             */
-            qemu_mutex_lock_iothread();
-            ret = blk_mig_save_dirty_block(f, 1);
-            qemu_mutex_unlock_iothread();
-        }
-        if (ret < 0) {
-            return ret;
-        }
-        blk_mig_lock();
-        if (ret != 0) {
-            /* no more dirty blocks */
-            break;
-        }
-    }
-    blk_mig_unlock();
-
-    ret = flush_blks(f);
-    if (ret) {
-        return ret;
-    }
-
-    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
-    return qemu_ftell(f) - last_ftell;
-}
-
-/* Called with iothread lock taken.  */
-
-static int block_save_complete(QEMUFile *f, void *opaque)
-{
-    int ret;
-
-    DPRINTF("Enter save live complete submitted %d transferred %d\n",
-            block_mig_state.submitted, block_mig_state.transferred);
-
-    ret = flush_blks(f);
-    if (ret) {
-        return ret;
-    }
-
-    blk_mig_reset_dirty_cursor();
-
-    /* we know for sure that save bulk is completed and
-       all async read completed */
-    blk_mig_lock();
-    assert(block_mig_state.submitted == 0);
-    blk_mig_unlock();
-
-    do {
-        ret = blk_mig_save_dirty_block(f, 0);
-        if (ret < 0) {
-            return ret;
-        }
-    } while (ret == 0);
-
-    /* report completion */
-    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
-
-    DPRINTF("Block migration completed\n");
-
-    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
-
-    blk_mig_cleanup();
-    return 0;
-}
-
-static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size)
-{
-    /* Estimate pending number of bytes to send */
-    uint64_t pending;
-
-    qemu_mutex_lock_iothread();
-    blk_mig_lock();
-    pending = get_remaining_dirty() +
-                       block_mig_state.submitted * BLOCK_SIZE +
-                       block_mig_state.read_done * BLOCK_SIZE;
-
-    /* Report at least one block pending during bulk phase */
-    if (pending == 0 && !block_mig_state.bulk_completed) {
-        pending = BLOCK_SIZE;
-    }
-    blk_mig_unlock();
-    qemu_mutex_unlock_iothread();
-
-    DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
-    return pending;
-}
-
-static int block_load(QEMUFile *f, void *opaque, int version_id)
-{
-    static int banner_printed;
-    int len, flags;
-    char device_name[256];
-    int64_t addr;
-    BlockDriverState *bs, *bs_prev = NULL;
-    uint8_t *buf;
-    int64_t total_sectors = 0;
-    int nr_sectors;
-    int ret;
-
-    do {
-        addr = qemu_get_be64(f);
-
-        flags = addr & ~BDRV_SECTOR_MASK;
-        addr >>= BDRV_SECTOR_BITS;
-
-        if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
-            /* get device name */
-            len = qemu_get_byte(f);
-            qemu_get_buffer(f, (uint8_t *)device_name, len);
-            device_name[len] = '\0';
-
-            bs = bdrv_find(device_name);
-            if (!bs) {
-                fprintf(stderr, "Error unknown block device %s\n",
-                        device_name);
-                return -EINVAL;
-            }
-
-            if (bs != bs_prev) {
-                bs_prev = bs;
-                total_sectors = bdrv_nb_sectors(bs);
-                if (total_sectors <= 0) {
-                    error_report("Error getting length of block device %s",
-                                 device_name);
-                    return -EINVAL;
-                }
-            }
-
-            if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
-                nr_sectors = total_sectors - addr;
-            } else {
-                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
-            }
-
-            if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
-                ret = bdrv_write_zeroes(bs, addr, nr_sectors,
-                                        BDRV_REQ_MAY_UNMAP);
-            } else {
-                buf = g_malloc(BLOCK_SIZE);
-                qemu_get_buffer(f, buf, BLOCK_SIZE);
-                ret = bdrv_write(bs, addr, buf, nr_sectors);
-                g_free(buf);
-            }
-
-            if (ret < 0) {
-                return ret;
-            }
-        } else if (flags & BLK_MIG_FLAG_PROGRESS) {
-            if (!banner_printed) {
-                printf("Receiving block device images\n");
-                banner_printed = 1;
-            }
-            printf("Completed %d %%%c", (int)addr,
-                   (addr == 100) ? '\n' : '\r');
-            fflush(stdout);
-        } else if (!(flags & BLK_MIG_FLAG_EOS)) {
-            fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
-            return -EINVAL;
-        }
-        ret = qemu_file_get_error(f);
-        if (ret != 0) {
-            return ret;
-        }
-    } while (!(flags & BLK_MIG_FLAG_EOS));
-
-    return 0;
-}
-
-static void block_set_params(const MigrationParams *params, void *opaque)
-{
-    block_mig_state.blk_enable = params->blk;
-    block_mig_state.shared_base = params->shared;
-
-    /* shared base means that blk_enable = 1 */
-    block_mig_state.blk_enable |= params->shared;
-}
-
-static bool block_is_active(void *opaque)
-{
-    return block_mig_state.blk_enable == 1;
-}
-
-static SaveVMHandlers savevm_block_handlers = {
-    .set_params = block_set_params,
-    .save_live_setup = block_save_setup,
-    .save_live_iterate = block_save_iterate,
-    .save_live_complete = block_save_complete,
-    .save_live_pending = block_save_pending,
-    .load_state = block_load,
-    .cancel = block_migration_cancel,
-    .is_active = block_is_active,
-};
-
-void blk_mig_init(void)
-{
-    QSIMPLEQ_INIT(&block_mig_state.bmds_list);
-    QSIMPLEQ_INIT(&block_mig_state.blk_list);
-    qemu_mutex_init(&block_mig_state.lock);
-
-    register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
-                         &block_mig_state);
-}
diff --git a/block.c b/block.c
index 457afd4..8efa999 100644
--- a/block.c
+++ b/block.c
@@ -21,25 +21,27 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qemu/error-report.h"
 #include "qemu/module.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qjson.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/qapi.h"
 #include "qmp-commands.h"
 #include "qemu/timer.h"
 #include "qapi-event.h"
+#include "qemu/cutils.h"
+#include "qemu/id.h"
 
 #ifdef CONFIG_BSD
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <sys/ioctl.h>
 #include <sys/queue.h>
 #ifndef __DragonFly__
@@ -51,52 +53,24 @@
 #include <windows.h>
 #endif
 
-struct BdrvDirtyBitmap {
-    HBitmap *bitmap;
-    QLIST_ENTRY(BdrvDirtyBitmap) list;
-};
-
 #define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
 
-static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque);
-static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque);
-static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov);
-static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov);
-static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-    BdrvRequestFlags flags);
-static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-    BdrvRequestFlags flags);
-static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
-                                         int64_t sector_num,
-                                         QEMUIOVector *qiov,
-                                         int nb_sectors,
-                                         BdrvRequestFlags flags,
-                                         BlockCompletionFunc *cb,
-                                         void *opaque,
-                                         bool is_write);
-static void coroutine_fn bdrv_co_do_rw(void *opaque);
-static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
-
-static QTAILQ_HEAD(, BlockDriverState) bdrv_states =
-    QTAILQ_HEAD_INITIALIZER(bdrv_states);
-
 static QTAILQ_HEAD(, BlockDriverState) graph_bdrv_states =
     QTAILQ_HEAD_INITIALIZER(graph_bdrv_states);
 
+static QTAILQ_HEAD(, BlockDriverState) all_bdrv_states =
+    QTAILQ_HEAD_INITIALIZER(all_bdrv_states);
+
 static QLIST_HEAD(, BlockDriver) bdrv_drivers =
     QLIST_HEAD_INITIALIZER(bdrv_drivers);
 
+static BlockDriverState *bdrv_open_inherit(const char *filename,
+                                           const char *reference,
+                                           QDict *options, int flags,
+                                           BlockDriverState *parent,
+                                           const BdrvChildRole *child_role,
+                                           Error **errp);
+
 /* If non-zero, use only whitelisted block drivers */
 static int use_bdrv_whitelist;
 
@@ -120,116 +94,28 @@
 }
 #endif
 
-/* throttling disk I/O limits */
-void bdrv_set_io_limits(BlockDriverState *bs,
-                        ThrottleConfig *cfg)
-{
-    int i;
-
-    throttle_config(&bs->throttle_state, cfg);
-
-    for (i = 0; i < 2; i++) {
-        qemu_co_enter_next(&bs->throttled_reqs[i]);
-    }
-}
-
-/* this function drain all the throttled IOs */
-static bool bdrv_start_throttled_reqs(BlockDriverState *bs)
-{
-    bool drained = false;
-    bool enabled = bs->io_limits_enabled;
-    int i;
-
-    bs->io_limits_enabled = false;
-
-    for (i = 0; i < 2; i++) {
-        while (qemu_co_enter_next(&bs->throttled_reqs[i])) {
-            drained = true;
-        }
-    }
-
-    bs->io_limits_enabled = enabled;
-
-    return drained;
-}
-
-void bdrv_io_limits_disable(BlockDriverState *bs)
-{
-    bs->io_limits_enabled = false;
-
-    bdrv_start_throttled_reqs(bs);
-
-    throttle_destroy(&bs->throttle_state);
-}
-
-static void bdrv_throttle_read_timer_cb(void *opaque)
-{
-    BlockDriverState *bs = opaque;
-    qemu_co_enter_next(&bs->throttled_reqs[0]);
-}
-
-static void bdrv_throttle_write_timer_cb(void *opaque)
-{
-    BlockDriverState *bs = opaque;
-    qemu_co_enter_next(&bs->throttled_reqs[1]);
-}
-
-/* should be called before bdrv_set_io_limits if a limit is set */
-void bdrv_io_limits_enable(BlockDriverState *bs)
-{
-    assert(!bs->io_limits_enabled);
-    throttle_init(&bs->throttle_state,
-                  bdrv_get_aio_context(bs),
-                  QEMU_CLOCK_VIRTUAL,
-                  bdrv_throttle_read_timer_cb,
-                  bdrv_throttle_write_timer_cb,
-                  bs);
-    bs->io_limits_enabled = true;
-}
-
-/* This function makes an IO wait if needed
- *
- * @nb_sectors: the number of sectors of the IO
- * @is_write:   is the IO a write
- */
-static void bdrv_io_limits_intercept(BlockDriverState *bs,
-                                     unsigned int bytes,
-                                     bool is_write)
-{
-    /* does this io must wait */
-    bool must_wait = throttle_schedule_timer(&bs->throttle_state, is_write);
-
-    /* if must wait or any request of this type throttled queue the IO */
-    if (must_wait ||
-        !qemu_co_queue_empty(&bs->throttled_reqs[is_write])) {
-        qemu_co_queue_wait(&bs->throttled_reqs[is_write]);
-    }
-
-    /* the IO will be executed, do the accounting */
-    throttle_account(&bs->throttle_state, is_write, bytes);
-
-
-    /* if the next request must wait -> do nothing */
-    if (throttle_schedule_timer(&bs->throttle_state, is_write)) {
-        return;
-    }
-
-    /* else queue next request for execution */
-    qemu_co_queue_next(&bs->throttled_reqs[is_write]);
-}
-
 size_t bdrv_opt_mem_align(BlockDriverState *bs)
 {
     if (!bs || !bs->drv) {
-        /* 4k should be on the safe side */
-        return 4096;
+        /* page size or 4k (hdd sector size) should be on the safe side */
+        return MAX(4096, getpagesize());
     }
 
     return bs->bl.opt_mem_alignment;
 }
 
+size_t bdrv_min_mem_align(BlockDriverState *bs)
+{
+    if (!bs || !bs->drv) {
+        /* page size or 4k (hdd sector size) should be on the safe side */
+        return MAX(4096, getpagesize());
+    }
+
+    return bs->bl.min_mem_alignment;
+}
+
 /* check if the path starts with "<protocol>:" */
-static int path_has_protocol(const char *path)
+int path_has_protocol(const char *path)
 {
     const char *p;
 
@@ -246,7 +132,7 @@
     return *p == ':';
 }
 
-static int path_check_absolute(const char *path)
+int path_is_absolute(const char *path)
 {
 #ifdef _WIN32
     /* specific case for names like: "\\.\d:" */
@@ -271,7 +157,7 @@
 
     if (dest_size <= 0)
         return;
-    if (path_check_absolute(filename)) {
+    if (path_is_absolute(filename)) {
         pstrcpy(dest, dest_size, filename);
     } else {
         p = strchr(base_path, ':');
@@ -303,43 +189,37 @@
     }
 }
 
-void bdrv_get_full_backing_filename(BlockDriverState *bs, char *dest, size_t sz)
+void bdrv_get_full_backing_filename_from_filename(const char *backed,
+                                                  const char *backing,
+                                                  char *dest, size_t sz,
+                                                  Error **errp)
 {
-    if (bs->backing_file[0] == '\0' || path_has_protocol(bs->backing_file)) {
-        pstrcpy(dest, sz, bs->backing_file);
+    if (backing[0] == '\0' || path_has_protocol(backing) ||
+        path_is_absolute(backing))
+    {
+        pstrcpy(dest, sz, backing);
+    } else if (backed[0] == '\0' || strstart(backed, "json:", NULL)) {
+        error_setg(errp, "Cannot use relative backing file names for '%s'",
+                   backed);
     } else {
-        path_combine(dest, sz, bs->filename, bs->backing_file);
+        path_combine(dest, sz, backed, backing);
     }
 }
 
+void bdrv_get_full_backing_filename(BlockDriverState *bs, char *dest, size_t sz,
+                                    Error **errp)
+{
+    char *backed = bs->exact_filename[0] ? bs->exact_filename : bs->filename;
+
+    bdrv_get_full_backing_filename_from_filename(backed, bs->backing_file,
+                                                 dest, sz, errp);
+}
+
 void bdrv_register(BlockDriver *bdrv)
 {
-    /* Block drivers without coroutine functions need emulation */
-    if (!bdrv->bdrv_co_readv) {
-        bdrv->bdrv_co_readv = bdrv_co_readv_em;
-        bdrv->bdrv_co_writev = bdrv_co_writev_em;
-
-        /* bdrv_co_readv_em()/brdv_co_writev_em() work in terms of aio, so if
-         * the block driver lacks aio we need to emulate that too.
-         */
-        if (!bdrv->bdrv_aio_readv) {
-            /* add AIO emulation layer */
-            bdrv->bdrv_aio_readv = bdrv_aio_readv_em;
-            bdrv->bdrv_aio_writev = bdrv_aio_writev_em;
-        }
-    }
-
     QLIST_INSERT_HEAD(&bdrv_drivers, bdrv, list);
 }
 
-BlockDriverState *bdrv_new_root(void)
-{
-    BlockDriverState *bs = bdrv_new();
-
-    QTAILQ_INSERT_TAIL(&bdrv_states, bs, device_list);
-    return bs;
-}
-
 BlockDriverState *bdrv_new(void)
 {
     BlockDriverState *bs;
@@ -350,20 +230,15 @@
     for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
         QLIST_INIT(&bs->op_blockers[i]);
     }
-    bdrv_iostatus_disable(bs);
-    notifier_list_init(&bs->close_notifiers);
     notifier_with_return_list_init(&bs->before_write_notifiers);
-    qemu_co_queue_init(&bs->throttled_reqs[0]);
-    qemu_co_queue_init(&bs->throttled_reqs[1]);
     bs->refcnt = 1;
     bs->aio_context = qemu_get_aio_context();
 
-    return bs;
-}
+    qemu_co_queue_init(&bs->flush_queue);
 
-void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
-{
-    notifier_list_add(&bs->close_notifiers, notify);
+    QTAILQ_INSERT_TAIL(&all_bdrv_states, bs, bs_list);
+
+    return bs;
 }
 
 BlockDriver *bdrv_find_format(const char *format_name)
@@ -406,11 +281,9 @@
     return 0;
 }
 
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool read_only)
+bool bdrv_uses_whitelist(void)
 {
-    BlockDriver *drv = bdrv_find_format(format_name);
-    return drv && bdrv_is_whitelisted(drv, read_only) ? drv : NULL;
+    return use_bdrv_whitelist;
 }
 
 typedef struct CreateCo {
@@ -430,9 +303,7 @@
     assert(cco->drv);
 
     ret = cco->drv->bdrv_create(cco->filename, cco->opts, &local_err);
-    if (local_err) {
-        error_propagate(&cco->err, local_err);
-    }
+    error_propagate(&cco->err, local_err);
     cco->ret = ret;
 }
 
@@ -460,8 +331,8 @@
         /* Fast-path if already in coroutine context */
         bdrv_create_co_entry(&cco);
     } else {
-        co = qemu_coroutine_create(bdrv_create_co_entry);
-        qemu_coroutine_enter(co, &cco);
+        co = qemu_coroutine_create(bdrv_create_co_entry, &cco);
+        qemu_coroutine_enter(co);
         while (cco.ret == NOT_DONE) {
             aio_poll(qemu_get_aio_context(), true);
         }
@@ -487,65 +358,48 @@
     Error *local_err = NULL;
     int ret;
 
-    drv = bdrv_find_protocol(filename, true);
+    drv = bdrv_find_protocol(filename, true, errp);
     if (drv == NULL) {
-        error_setg(errp, "Could not find protocol for file '%s'", filename);
         return -ENOENT;
     }
 
     ret = bdrv_create(drv, filename, opts, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
     return ret;
 }
 
-void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
+/**
+ * Try to get @bs's logical and physical block size.
+ * On success, store them in @bsz struct and return 0.
+ * On failure return -errno.
+ * @bs must not be empty.
+ */
+int bdrv_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
 {
     BlockDriver *drv = bs->drv;
-    Error *local_err = NULL;
 
-    memset(&bs->bl, 0, sizeof(bs->bl));
-
-    if (!drv) {
-        return;
+    if (drv && drv->bdrv_probe_blocksizes) {
+        return drv->bdrv_probe_blocksizes(bs, bsz);
     }
 
-    /* Take some limits from the children as a default */
-    if (bs->file) {
-        bdrv_refresh_limits(bs->file, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return;
-        }
-        bs->bl.opt_transfer_length = bs->file->bl.opt_transfer_length;
-        bs->bl.max_transfer_length = bs->file->bl.max_transfer_length;
-        bs->bl.opt_mem_alignment = bs->file->bl.opt_mem_alignment;
-    } else {
-        bs->bl.opt_mem_alignment = 512;
+    return -ENOTSUP;
+}
+
+/**
+ * Try to get @bs's geometry (cyls, heads, sectors).
+ * On success, store them in @geo struct and return 0.
+ * On failure return -errno.
+ * @bs must not be empty.
+ */
+int bdrv_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (drv && drv->bdrv_probe_geometry) {
+        return drv->bdrv_probe_geometry(bs, geo);
     }
 
-    if (bs->backing_hd) {
-        bdrv_refresh_limits(bs->backing_hd, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return;
-        }
-        bs->bl.opt_transfer_length =
-            MAX(bs->bl.opt_transfer_length,
-                bs->backing_hd->bl.opt_transfer_length);
-        bs->bl.max_transfer_length =
-            MIN_NON_ZERO(bs->bl.max_transfer_length,
-                         bs->backing_hd->bl.max_transfer_length);
-        bs->bl.opt_mem_alignment =
-            MAX(bs->bl.opt_mem_alignment,
-                bs->backing_hd->bl.opt_mem_alignment);
-    }
-
-    /* Then let the driver override it */
-    if (drv->bdrv_refresh_limits) {
-        drv->bdrv_refresh_limits(bs, errp);
-    }
+    return -ENOTSUP;
 }
 
 /*
@@ -607,7 +461,8 @@
 }
 
 BlockDriver *bdrv_find_protocol(const char *filename,
-                                bool allow_protocol_prefix)
+                                bool allow_protocol_prefix,
+                                Error **errp)
 {
     BlockDriver *drv1;
     char protocol[128];
@@ -629,7 +484,7 @@
     }
 
     if (!path_has_protocol(filename) || !allow_protocol_prefix) {
-        return bdrv_find_format("file");
+        return &bdrv_file;
     }
 
     p = strchr(filename, ':');
@@ -645,29 +500,59 @@
             return drv1;
         }
     }
+
+    error_setg(errp, "Unknown protocol '%s'", protocol);
     return NULL;
 }
 
-static int find_image_format(BlockDriverState *bs, const char *filename,
+/*
+ * Guess image format by probing its contents.
+ * This is not a good idea when your image is raw (CVE-2008-2004), but
+ * we do it anyway for backward compatibility.
+ *
+ * @buf         contains the image's first @buf_size bytes.
+ * @buf_size    is the buffer size in bytes (generally BLOCK_PROBE_BUF_SIZE,
+ *              but can be smaller if the image file is smaller)
+ * @filename    is its filename.
+ *
+ * For all block drivers, call the bdrv_probe() method to get its
+ * probing score.
+ * Return the first block driver with the highest probing score.
+ */
+BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,
+                            const char *filename)
+{
+    int score_max = 0, score;
+    BlockDriver *drv = NULL, *d;
+
+    QLIST_FOREACH(d, &bdrv_drivers, list) {
+        if (d->bdrv_probe) {
+            score = d->bdrv_probe(buf, buf_size, filename);
+            if (score > score_max) {
+                score_max = score;
+                drv = d;
+            }
+        }
+    }
+
+    return drv;
+}
+
+static int find_image_format(BdrvChild *file, const char *filename,
                              BlockDriver **pdrv, Error **errp)
 {
-    int score, score_max;
-    BlockDriver *drv1, *drv;
-    uint8_t buf[2048];
+    BlockDriverState *bs = file->bs;
+    BlockDriver *drv;
+    uint8_t buf[BLOCK_PROBE_BUF_SIZE];
     int ret = 0;
 
     /* Return the raw BlockDriver * to scsi-generic devices or empty drives */
-    if (bs->sg || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
-        drv = bdrv_find_format("raw");
-        if (!drv) {
-            error_setg(errp, "Could not find raw image format");
-            ret = -ENOENT;
-        }
-        *pdrv = drv;
+    if (bdrv_is_sg(bs) || !bdrv_is_inserted(bs) || bdrv_getlength(bs) == 0) {
+        *pdrv = &bdrv_raw;
         return ret;
     }
 
-    ret = bdrv_pread(bs, 0, buf, sizeof(buf));
+    ret = bdrv_pread(file, 0, buf, sizeof(buf));
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not read image for determining its "
                          "format");
@@ -675,17 +560,7 @@
         return ret;
     }
 
-    score_max = 0;
-    drv = NULL;
-    QLIST_FOREACH(drv1, &bdrv_drivers, list) {
-        if (drv1->bdrv_probe) {
-            score = drv1->bdrv_probe(buf, ret, filename);
-            if (score > score_max) {
-                score_max = score;
-                drv = drv1;
-            }
-        }
-    }
+    drv = bdrv_probe_all(buf, ret, filename);
     if (!drv) {
         error_setg(errp, "Could not determine image format: No compatible "
                    "driver found");
@@ -704,7 +579,7 @@
     BlockDriver *drv = bs->drv;
 
     /* Do not attempt drv->bdrv_getlength() on scsi-generic devices */
-    if (bs->sg)
+    if (bdrv_is_sg(bs))
         return 0;
 
     /* query actual device if possible, otherwise just trust the hint */
@@ -721,6 +596,20 @@
 }
 
 /**
+ * Combines a QDict of new block driver @options with any missing options taken
+ * from @old_options, so that leaving out an option defaults to its old value.
+ */
+static void bdrv_join_options(BlockDriverState *bs, QDict *options,
+                              QDict *old_options)
+{
+    if (bs->drv && bs->drv->bdrv_join_options) {
+        bs->drv->bdrv_join_options(options, old_options);
+    } else {
+        qdict_join(options, old_options, false);
+    }
+}
+
+/**
  * Set open flags for a given discard mode
  *
  * Return 0 on success, -1 if the discard mode was invalid.
@@ -745,21 +634,23 @@
  *
  * Return 0 on success, -1 if the cache mode was invalid.
  */
-int bdrv_parse_cache_flags(const char *mode, int *flags)
+int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough)
 {
     *flags &= ~BDRV_O_CACHE_MASK;
 
     if (!strcmp(mode, "off") || !strcmp(mode, "none")) {
-        *flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+        *writethrough = false;
+        *flags |= BDRV_O_NOCACHE;
     } else if (!strcmp(mode, "directsync")) {
+        *writethrough = true;
         *flags |= BDRV_O_NOCACHE;
     } else if (!strcmp(mode, "writeback")) {
-        *flags |= BDRV_O_CACHE_WB;
+        *writethrough = false;
     } else if (!strcmp(mode, "unsafe")) {
-        *flags |= BDRV_O_CACHE_WB;
+        *writethrough = false;
         *flags |= BDRV_O_NO_FLUSH;
     } else if (!strcmp(mode, "writethrough")) {
-        /* this is the default */
+        *writethrough = true;
     } else {
         return -1;
     }
@@ -767,69 +658,124 @@
     return 0;
 }
 
-/**
- * The copy-on-read flag is actually a reference count so multiple users may
- * use the feature without worrying about clobbering its previous state.
- * Copy-on-read stays enabled until all users have called to disable it.
- */
-void bdrv_enable_copy_on_read(BlockDriverState *bs)
+static void bdrv_child_cb_drained_begin(BdrvChild *child)
 {
-    bs->copy_on_read++;
+    BlockDriverState *bs = child->opaque;
+    bdrv_drained_begin(bs);
 }
 
-void bdrv_disable_copy_on_read(BlockDriverState *bs)
+static void bdrv_child_cb_drained_end(BdrvChild *child)
 {
-    assert(bs->copy_on_read > 0);
-    bs->copy_on_read--;
+    BlockDriverState *bs = child->opaque;
+    bdrv_drained_end(bs);
 }
 
 /*
- * Returns the flags that a temporary snapshot should get, based on the
- * originally requested flags (the originally requested image will have flags
- * like a backing file)
+ * Returns the options and flags that a temporary snapshot should get, based on
+ * the originally requested flags (the originally requested image will have
+ * flags like a backing file)
  */
-static int bdrv_temp_snapshot_flags(int flags)
+static void bdrv_temp_snapshot_options(int *child_flags, QDict *child_options,
+                                       int parent_flags, QDict *parent_options)
 {
-    return (flags & ~BDRV_O_SNAPSHOT) | BDRV_O_TEMPORARY;
+    *child_flags = (parent_flags & ~BDRV_O_SNAPSHOT) | BDRV_O_TEMPORARY;
+
+    /* For temporary files, unconditional cache=unsafe is fine */
+    qdict_set_default_str(child_options, BDRV_OPT_CACHE_DIRECT, "off");
+    qdict_set_default_str(child_options, BDRV_OPT_CACHE_NO_FLUSH, "on");
+
+    /* aio=native doesn't work for cache.direct=off, so disable it for the
+     * temporary snapshot */
+    *child_flags &= ~BDRV_O_NATIVE_AIO;
 }
 
 /*
- * Returns the flags that bs->file should get, based on the given flags for
- * the parent BDS
+ * Returns the options and flags that bs->file should get if a protocol driver
+ * is expected, based on the given options and flags for the parent BDS
  */
-static int bdrv_inherited_flags(int flags)
+static void bdrv_inherited_options(int *child_flags, QDict *child_options,
+                                   int parent_flags, QDict *parent_options)
 {
+    int flags = parent_flags;
+
     /* Enable protocol handling, disable format probing for bs->file */
     flags |= BDRV_O_PROTOCOL;
 
+    /* If the cache mode isn't explicitly set, inherit direct and no-flush from
+     * the parent. */
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_DIRECT);
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_NO_FLUSH);
+
     /* Our block drivers take care to send flushes and respect unmap policy,
-     * so we can enable both unconditionally on lower layers. */
-    flags |= BDRV_O_CACHE_WB | BDRV_O_UNMAP;
+     * so we can default to enable both on lower layers regardless of the
+     * corresponding parent options. */
+    flags |= BDRV_O_UNMAP;
 
     /* Clear flags that only apply to the top layer */
-    flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING | BDRV_O_COPY_ON_READ);
+    flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING | BDRV_O_COPY_ON_READ |
+               BDRV_O_NO_IO);
 
-    return flags;
+    *child_flags = flags;
 }
 
+const BdrvChildRole child_file = {
+    .inherit_options = bdrv_inherited_options,
+    .drained_begin   = bdrv_child_cb_drained_begin,
+    .drained_end     = bdrv_child_cb_drained_end,
+};
+
 /*
- * Returns the flags that bs->backing_hd should get, based on the given flags
- * for the parent BDS
+ * Returns the options and flags that bs->file should get if the use of formats
+ * (and not only protocols) is permitted for it, based on the given options and
+ * flags for the parent BDS
  */
-static int bdrv_backing_flags(int flags)
+static void bdrv_inherited_fmt_options(int *child_flags, QDict *child_options,
+                                       int parent_flags, QDict *parent_options)
 {
+    child_file.inherit_options(child_flags, child_options,
+                               parent_flags, parent_options);
+
+    *child_flags &= ~(BDRV_O_PROTOCOL | BDRV_O_NO_IO);
+}
+
+const BdrvChildRole child_format = {
+    .inherit_options = bdrv_inherited_fmt_options,
+    .drained_begin   = bdrv_child_cb_drained_begin,
+    .drained_end     = bdrv_child_cb_drained_end,
+};
+
+/*
+ * Returns the options and flags that bs->backing should get, based on the
+ * given options and flags for the parent BDS
+ */
+static void bdrv_backing_options(int *child_flags, QDict *child_options,
+                                 int parent_flags, QDict *parent_options)
+{
+    int flags = parent_flags;
+
+    /* The cache mode is inherited unmodified for backing files; except WCE,
+     * which is only applied on the top level (BlockBackend) */
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_DIRECT);
+    qdict_copy_default(child_options, parent_options, BDRV_OPT_CACHE_NO_FLUSH);
+
     /* backing files always opened read-only */
     flags &= ~(BDRV_O_RDWR | BDRV_O_COPY_ON_READ);
 
     /* snapshot=on is handled on the top layer */
     flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_TEMPORARY);
 
-    return flags;
+    *child_flags = flags;
 }
 
+static const BdrvChildRole child_backing = {
+    .inherit_options = bdrv_backing_options,
+    .drained_begin   = bdrv_child_cb_drained_begin,
+    .drained_end     = bdrv_child_cb_drained_end,
+};
+
 static int bdrv_open_flags(BlockDriverState *bs, int flags)
 {
-    int open_flags = flags | BDRV_O_CACHE_WB;
+    int open_flags = flags;
 
     /*
      * Clear flags that are internal to the block layer before opening the
@@ -847,16 +793,46 @@
     return open_flags;
 }
 
+static void update_flags_from_options(int *flags, QemuOpts *opts)
+{
+    *flags &= ~BDRV_O_CACHE_MASK;
+
+    assert(qemu_opt_find(opts, BDRV_OPT_CACHE_NO_FLUSH));
+    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_NO_FLUSH, false)) {
+        *flags |= BDRV_O_NO_FLUSH;
+    }
+
+    assert(qemu_opt_find(opts, BDRV_OPT_CACHE_DIRECT));
+    if (qemu_opt_get_bool(opts, BDRV_OPT_CACHE_DIRECT, false)) {
+        *flags |= BDRV_O_NOCACHE;
+    }
+}
+
+static void update_options_from_flags(QDict *options, int flags)
+{
+    if (!qdict_haskey(options, BDRV_OPT_CACHE_DIRECT)) {
+        qdict_put(options, BDRV_OPT_CACHE_DIRECT,
+                  qbool_from_bool(flags & BDRV_O_NOCACHE));
+    }
+    if (!qdict_haskey(options, BDRV_OPT_CACHE_NO_FLUSH)) {
+        qdict_put(options, BDRV_OPT_CACHE_NO_FLUSH,
+                  qbool_from_bool(flags & BDRV_O_NO_FLUSH));
+    }
+}
+
 static void bdrv_assign_node_name(BlockDriverState *bs,
                                   const char *node_name,
                                   Error **errp)
 {
-    if (!node_name) {
-        return;
-    }
+    char *gen_node_name = NULL;
 
-    /* Check for empty string or invalid characters */
-    if (!id_wellformed(node_name)) {
+    if (!node_name) {
+        node_name = gen_node_name = id_generate(ID_BLOCK);
+    } else if (!id_wellformed(node_name)) {
+        /*
+         * Check for empty string or invalid characters, but not if it is
+         * generated (generated names use characters not available to the user)
+         */
         error_setg(errp, "Invalid node name");
         return;
     }
@@ -865,39 +841,83 @@
     if (blk_by_name(node_name)) {
         error_setg(errp, "node-name=%s is conflicting with a device id",
                    node_name);
-        return;
+        goto out;
     }
 
     /* takes care of avoiding duplicates node names */
     if (bdrv_find_node(node_name)) {
         error_setg(errp, "Duplicate node name");
-        return;
+        goto out;
     }
 
     /* copy node name into the bs and insert it into the graph list */
     pstrcpy(bs->node_name, sizeof(bs->node_name), node_name);
     QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs, node_list);
+out:
+    g_free(gen_node_name);
 }
 
+static QemuOptsList bdrv_runtime_opts = {
+    .name = "bdrv_common",
+    .head = QTAILQ_HEAD_INITIALIZER(bdrv_runtime_opts.head),
+    .desc = {
+        {
+            .name = "node-name",
+            .type = QEMU_OPT_STRING,
+            .help = "Node name of the block device node",
+        },
+        {
+            .name = "driver",
+            .type = QEMU_OPT_STRING,
+            .help = "Block driver to use for the node",
+        },
+        {
+            .name = BDRV_OPT_CACHE_DIRECT,
+            .type = QEMU_OPT_BOOL,
+            .help = "Bypass software writeback cache on the host",
+        },
+        {
+            .name = BDRV_OPT_CACHE_NO_FLUSH,
+            .type = QEMU_OPT_BOOL,
+            .help = "Ignore flush requests",
+        },
+        { /* end of list */ }
+    },
+};
+
 /*
  * Common part for opening disk images and files
  *
  * Removes all processed options from *options.
  */
-static int bdrv_open_common(BlockDriverState *bs, BlockDriverState *file,
-    QDict *options, int flags, BlockDriver *drv, Error **errp)
+static int bdrv_open_common(BlockDriverState *bs, BdrvChild *file,
+                            QDict *options, Error **errp)
 {
     int ret, open_flags;
     const char *filename;
+    const char *driver_name = NULL;
     const char *node_name = NULL;
+    QemuOpts *opts;
+    BlockDriver *drv;
     Error *local_err = NULL;
 
-    assert(drv != NULL);
     assert(bs->file == NULL);
     assert(options != NULL && bs->options != options);
 
+    opts = qemu_opts_create(&bdrv_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail_opts;
+    }
+
+    driver_name = qemu_opt_get(opts, "driver");
+    drv = bdrv_find_format(driver_name);
+    assert(drv != NULL);
+
     if (file != NULL) {
-        filename = file->filename;
+        filename = file->bs->filename;
     } else {
         filename = qdict_get_try_str(options, "filename");
     }
@@ -905,34 +925,22 @@
     if (drv->bdrv_needs_filename && !filename) {
         error_setg(errp, "The '%s' block driver requires a file name",
                    drv->format_name);
-        return -EINVAL;
+        ret = -EINVAL;
+        goto fail_opts;
     }
 
-    trace_bdrv_open_common(bs, filename ?: "", flags, drv->format_name);
+    trace_bdrv_open_common(bs, filename ?: "", bs->open_flags,
+                           drv->format_name);
 
-    node_name = qdict_get_try_str(options, "node-name");
+    node_name = qemu_opt_get(opts, "node-name");
     bdrv_assign_node_name(bs, node_name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return -EINVAL;
-    }
-    qdict_del(options, "node-name");
-
-    /* bdrv_open() with directly using a protocol as drv. This layer is already
-     * opened, so assign it to bs (while file becomes a closed BlockDriverState)
-     * and return immediately. */
-    if (file != NULL && drv->bdrv_file_open) {
-        bdrv_swap(file, bs);
-        return 0;
+        ret = -EINVAL;
+        goto fail_opts;
     }
 
-    bs->open_flags = flags;
-    bs->guest_block_size = 512;
-    bs->request_alignment = 512;
-    bs->zero_beyond_eof = true;
-    open_flags = bdrv_open_flags(bs, flags);
-    bs->read_only = !(open_flags & BDRV_O_RDWR);
-    bs->growable = !!(flags & BDRV_O_PROTOCOL);
+    bs->read_only = !(bs->open_flags & BDRV_O_RDWR);
 
     if (use_bdrv_whitelist && !bdrv_is_whitelisted(drv, bs->read_only)) {
         error_setg(errp,
@@ -940,16 +948,18 @@
                         ? "Driver '%s' can only be used for read-only devices"
                         : "Driver '%s' is not whitelisted",
                    drv->format_name);
-        return -ENOTSUP;
+        ret = -ENOTSUP;
+        goto fail_opts;
     }
 
     assert(bs->copy_on_read == 0); /* bdrv_new() and bdrv_close() make it so */
-    if (flags & BDRV_O_COPY_ON_READ) {
+    if (bs->open_flags & BDRV_O_COPY_ON_READ) {
         if (!bs->read_only) {
             bdrv_enable_copy_on_read(bs);
         } else {
             error_setg(errp, "Can't use copy-on-read on read-only device");
-            return -EINVAL;
+            ret = -EINVAL;
+            goto fail_opts;
         }
     }
 
@@ -963,9 +973,11 @@
     bs->drv = drv;
     bs->opaque = g_malloc0(drv->instance_size);
 
-    bs->enable_write_cache = !!(flags & BDRV_O_CACHE_WB);
+    /* Apply cache mode options */
+    update_flags_from_options(&bs->open_flags, opts);
 
     /* Open the image, either directly or using a protocol */
+    open_flags = bdrv_open_flags(bs, bs->open_flags);
     if (drv->bdrv_file_open) {
         assert(file == NULL);
         assert(!drv->bdrv_needs_filename || filename != NULL);
@@ -1006,7 +1018,10 @@
     }
 
     assert(bdrv_opt_mem_align(bs) != 0);
-    assert((bs->request_alignment != 0) || bs->sg);
+    assert(bdrv_min_mem_align(bs) != 0);
+    assert(is_power_of_2(bs->bl.request_alignment));
+
+    qemu_opts_del(opts);
     return 0;
 
 free_and_fail:
@@ -1014,6 +1029,8 @@
     g_free(bs->opaque);
     bs->opaque = NULL;
     bs->drv = NULL;
+fail_opts:
+    qemu_opts_del(opts);
     return ret;
 }
 
@@ -1045,34 +1062,66 @@
     return options;
 }
 
-/*
- * Fills in default options for opening images and converts the legacy
- * filename/flags pair to option QDict entries.
- */
-static int bdrv_fill_options(QDict **options, const char **pfilename, int flags,
-                             BlockDriver *drv, Error **errp)
+static void parse_json_protocol(QDict *options, const char **pfilename,
+                                Error **errp)
 {
-    const char *filename = *pfilename;
-    const char *drvname;
-    bool protocol = flags & BDRV_O_PROTOCOL;
-    bool parse_filename = false;
+    QDict *json_options;
     Error *local_err = NULL;
 
     /* Parse json: pseudo-protocol */
-    if (filename && g_str_has_prefix(filename, "json:")) {
-        QDict *json_options = parse_json_filename(filename, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return -EINVAL;
-        }
-
-        /* Options given in the filename have lower priority than options
-         * specified directly */
-        qdict_join(*options, json_options, false);
-        QDECREF(json_options);
-        *pfilename = filename = NULL;
+    if (!*pfilename || !g_str_has_prefix(*pfilename, "json:")) {
+        return;
     }
 
+    json_options = parse_json_filename(*pfilename, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Options given in the filename have lower priority than options
+     * specified directly */
+    qdict_join(options, json_options, false);
+    QDECREF(json_options);
+    *pfilename = NULL;
+}
+
+/*
+ * Fills in default options for opening images and converts the legacy
+ * filename/flags pair to option QDict entries.
+ * The BDRV_O_PROTOCOL flag in *flags will be set or cleared accordingly if a
+ * block driver has been specified explicitly.
+ */
+static int bdrv_fill_options(QDict **options, const char *filename,
+                             int *flags, Error **errp)
+{
+    const char *drvname;
+    bool protocol = *flags & BDRV_O_PROTOCOL;
+    bool parse_filename = false;
+    BlockDriver *drv = NULL;
+    Error *local_err = NULL;
+
+    drvname = qdict_get_try_str(*options, "driver");
+    if (drvname) {
+        drv = bdrv_find_format(drvname);
+        if (!drv) {
+            error_setg(errp, "Unknown driver '%s'", drvname);
+            return -ENOENT;
+        }
+        /* If the user has explicitly specified the driver, this choice should
+         * override the BDRV_O_PROTOCOL flag */
+        protocol = drv->bdrv_file_open;
+    }
+
+    if (protocol) {
+        *flags |= BDRV_O_PROTOCOL;
+    } else {
+        *flags &= ~BDRV_O_PROTOCOL;
+    }
+
+    /* Translate cache options from flags into options */
+    update_options_from_flags(*options, *flags);
+
     /* Fetch the file name from the options QDict if necessary */
     if (protocol && filename) {
         if (!qdict_haskey(*options, "filename")) {
@@ -1087,36 +1136,19 @@
 
     /* Find the right block driver */
     filename = qdict_get_try_str(*options, "filename");
-    drvname = qdict_get_try_str(*options, "driver");
 
-    if (drv) {
-        if (drvname) {
-            error_setg(errp, "Driver specified twice");
-            return -EINVAL;
-        }
-        drvname = drv->format_name;
-        qdict_put(*options, "driver", qstring_from_str(drvname));
-    } else {
-        if (!drvname && protocol) {
-            if (filename) {
-                drv = bdrv_find_protocol(filename, parse_filename);
-                if (!drv) {
-                    error_setg(errp, "Unknown protocol");
-                    return -EINVAL;
-                }
-
-                drvname = drv->format_name;
-                qdict_put(*options, "driver", qstring_from_str(drvname));
-            } else {
-                error_setg(errp, "Must specify either driver or file");
+    if (!drvname && protocol) {
+        if (filename) {
+            drv = bdrv_find_protocol(filename, parse_filename, errp);
+            if (!drv) {
                 return -EINVAL;
             }
-        } else if (drvname) {
-            drv = bdrv_find_format(drvname);
-            if (!drv) {
-                error_setg(errp, "Unknown driver '%s'", drvname);
-                return -ENOENT;
-            }
+
+            drvname = drv->format_name;
+            qdict_put(*options, "driver", qstring_from_str(drvname));
+        } else {
+            error_setg(errp, "Must specify either driver or file");
+            return -EINVAL;
         }
     }
 
@@ -1138,32 +1170,147 @@
     return 0;
 }
 
-void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd)
+static void bdrv_replace_child(BdrvChild *child, BlockDriverState *new_bs)
 {
+    BlockDriverState *old_bs = child->bs;
 
-    if (bs->backing_hd) {
-        assert(bs->backing_blocker);
-        bdrv_op_unblock_all(bs->backing_hd, bs->backing_blocker);
-    } else if (backing_hd) {
-        error_setg(&bs->backing_blocker,
-                   "device is used as backing hd of '%s'",
-                   bdrv_get_device_name(bs));
+    if (old_bs) {
+        if (old_bs->quiesce_counter && child->role->drained_end) {
+            child->role->drained_end(child);
+        }
+        QLIST_REMOVE(child, next_parent);
     }
 
-    bs->backing_hd = backing_hd;
+    child->bs = new_bs;
+
+    if (new_bs) {
+        QLIST_INSERT_HEAD(&new_bs->parents, child, next_parent);
+        if (new_bs->quiesce_counter && child->role->drained_begin) {
+            child->role->drained_begin(child);
+        }
+    }
+}
+
+BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs,
+                                  const char *child_name,
+                                  const BdrvChildRole *child_role,
+                                  void *opaque)
+{
+    BdrvChild *child = g_new(BdrvChild, 1);
+    *child = (BdrvChild) {
+        .bs     = NULL,
+        .name   = g_strdup(child_name),
+        .role   = child_role,
+        .opaque = opaque,
+    };
+
+    bdrv_replace_child(child, child_bs);
+
+    return child;
+}
+
+BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs,
+                             BlockDriverState *child_bs,
+                             const char *child_name,
+                             const BdrvChildRole *child_role)
+{
+    BdrvChild *child = bdrv_root_attach_child(child_bs, child_name, child_role,
+                                              parent_bs);
+    QLIST_INSERT_HEAD(&parent_bs->children, child, next);
+    return child;
+}
+
+static void bdrv_detach_child(BdrvChild *child)
+{
+    if (child->next.le_prev) {
+        QLIST_REMOVE(child, next);
+        child->next.le_prev = NULL;
+    }
+
+    bdrv_replace_child(child, NULL);
+
+    g_free(child->name);
+    g_free(child);
+}
+
+void bdrv_root_unref_child(BdrvChild *child)
+{
+    BlockDriverState *child_bs;
+
+    child_bs = child->bs;
+    bdrv_detach_child(child);
+    bdrv_unref(child_bs);
+}
+
+void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child)
+{
+    if (child == NULL) {
+        return;
+    }
+
+    if (child->bs->inherits_from == parent) {
+        child->bs->inherits_from = NULL;
+    }
+
+    bdrv_root_unref_child(child);
+}
+
+
+static void bdrv_parent_cb_change_media(BlockDriverState *bs, bool load)
+{
+    BdrvChild *c;
+    QLIST_FOREACH(c, &bs->parents, next_parent) {
+        if (c->role->change_media) {
+            c->role->change_media(c, load);
+        }
+    }
+}
+
+static void bdrv_parent_cb_resize(BlockDriverState *bs)
+{
+    BdrvChild *c;
+    QLIST_FOREACH(c, &bs->parents, next_parent) {
+        if (c->role->resize) {
+            c->role->resize(c);
+        }
+    }
+}
+
+/*
+ * Sets the backing file link of a BDS. A new reference is created; callers
+ * which don't need their own reference any more must call bdrv_unref().
+ */
+void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd)
+{
+    if (backing_hd) {
+        bdrv_ref(backing_hd);
+    }
+
+    if (bs->backing) {
+        assert(bs->backing_blocker);
+        bdrv_op_unblock_all(bs->backing->bs, bs->backing_blocker);
+        bdrv_unref_child(bs, bs->backing);
+    } else if (backing_hd) {
+        error_setg(&bs->backing_blocker,
+                   "node is used as backing hd of '%s'",
+                   bdrv_get_device_or_node_name(bs));
+    }
+
     if (!backing_hd) {
         error_free(bs->backing_blocker);
         bs->backing_blocker = NULL;
+        bs->backing = NULL;
         goto out;
     }
+    bs->backing = bdrv_attach_child(bs, backing_hd, "backing", &child_backing);
     bs->open_flags &= ~BDRV_O_NO_BACKING;
     pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_hd->filename);
     pstrcpy(bs->backing_format, sizeof(bs->backing_format),
             backing_hd->drv ? backing_hd->drv->format_name : "");
 
-    bdrv_op_block_all(bs->backing_hd, bs->backing_blocker);
+    bdrv_op_block_all(backing_hd, bs->backing_blocker);
     /* Otherwise we won't be able to commit due to check in bdrv_commit */
-    bdrv_op_unblock(bs->backing_hd, BLOCK_OP_TYPE_COMMIT,
+    bdrv_op_unblock(backing_hd, BLOCK_OP_TYPE_COMMIT_TARGET,
                     bs->backing_blocker);
 out:
     bdrv_refresh_limits(bs, NULL);
@@ -1172,37 +1319,56 @@
 /*
  * Opens the backing file for a BlockDriverState if not yet open
  *
- * options is a QDict of options to pass to the block drivers, or NULL for an
- * empty set of options. The reference to the QDict is transferred to this
- * function (even on failure), so if the caller intends to reuse the dictionary,
- * it needs to use QINCREF() before calling bdrv_file_open.
+ * bdref_key specifies the key for the image's BlockdevRef in the options QDict.
+ * That QDict has to be flattened; therefore, if the BlockdevRef is a QDict
+ * itself, all options starting with "${bdref_key}." are considered part of the
+ * BlockdevRef.
+ *
+ * TODO Can this be unified with bdrv_open_image()?
  */
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp)
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options,
+                           const char *bdref_key, Error **errp)
 {
     char *backing_filename = g_malloc0(PATH_MAX);
+    char *bdref_key_dot;
+    const char *reference = NULL;
     int ret = 0;
-    BlockDriver *back_drv = NULL;
     BlockDriverState *backing_hd;
+    QDict *options;
+    QDict *tmp_parent_options = NULL;
     Error *local_err = NULL;
 
-    if (bs->backing_hd != NULL) {
-        QDECREF(options);
+    if (bs->backing != NULL) {
         goto free_exit;
     }
 
     /* NULL means an empty set of options */
-    if (options == NULL) {
-        options = qdict_new();
+    if (parent_options == NULL) {
+        tmp_parent_options = qdict_new();
+        parent_options = tmp_parent_options;
     }
 
     bs->open_flags &= ~BDRV_O_NO_BACKING;
-    if (qdict_haskey(options, "file.filename")) {
+
+    bdref_key_dot = g_strdup_printf("%s.", bdref_key);
+    qdict_extract_subqdict(parent_options, &options, bdref_key_dot);
+    g_free(bdref_key_dot);
+
+    reference = qdict_get_try_str(parent_options, bdref_key);
+    if (reference || qdict_haskey(options, "file.filename")) {
         backing_filename[0] = '\0';
     } else if (bs->backing_file[0] == '\0' && qdict_size(options) == 0) {
         QDECREF(options);
         goto free_exit;
     } else {
-        bdrv_get_full_backing_filename(bs, backing_filename, PATH_MAX);
+        bdrv_get_full_backing_filename(bs, backing_filename, PATH_MAX,
+                                       &local_err);
+        if (local_err) {
+            ret = -EINVAL;
+            error_propagate(errp, local_err);
+            QDECREF(options);
+            goto free_exit;
+        }
     }
 
     if (!bs->drv || !bs->drv->supports_backing) {
@@ -1212,29 +1378,30 @@
         goto free_exit;
     }
 
-    backing_hd = bdrv_new();
-
-    if (bs->backing_format[0] != '\0') {
-        back_drv = bdrv_find_format(bs->backing_format);
+    if (bs->backing_format[0] != '\0' && !qdict_haskey(options, "driver")) {
+        qdict_put(options, "driver", qstring_from_str(bs->backing_format));
     }
 
-    assert(bs->backing_hd == NULL);
-    ret = bdrv_open(&backing_hd,
-                    *backing_filename ? backing_filename : NULL, NULL, options,
-                    bdrv_backing_flags(bs->open_flags), back_drv, &local_err);
-    if (ret < 0) {
-        bdrv_unref(backing_hd);
-        backing_hd = NULL;
+    backing_hd = bdrv_open_inherit(*backing_filename ? backing_filename : NULL,
+                                   reference, options, 0, bs, &child_backing,
+                                   errp);
+    if (!backing_hd) {
         bs->open_flags |= BDRV_O_NO_BACKING;
-        error_setg(errp, "Could not open backing file: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+        error_prepend(errp, "Could not open backing file: ");
+        ret = -EINVAL;
         goto free_exit;
     }
+
+    /* Hook up the backing file link; drop our reference, bs owns the
+     * backing_hd reference now */
     bdrv_set_backing_hd(bs, backing_hd);
+    bdrv_unref(backing_hd);
+
+    qdict_del(parent_options, bdref_key);
 
 free_exit:
     g_free(backing_filename);
+    QDECREF(tmp_parent_options);
     return ret;
 }
 
@@ -1243,7 +1410,7 @@
  * device's options.
  *
  * If allow_none is true, no image will be opened if filename is false and no
- * BlockdevRef is given. *pbs will remain unchanged and 0 will be returned.
+ * BlockdevRef is given. NULL will be returned, but errp remains unset.
  *
  * bdrev_key specifies the key for the image's BlockdevRef in the options QDict.
  * That QDict has to be flattened; therefore, if the BlockdevRef is a QDict
@@ -1251,20 +1418,20 @@
  * BlockdevRef.
  *
  * The BlockdevRef will be removed from the options QDict.
- *
- * To conform with the behavior of bdrv_open(), *pbs has to be NULL.
  */
-int bdrv_open_image(BlockDriverState **pbs, const char *filename,
-                    QDict *options, const char *bdref_key, int flags,
-                    bool allow_none, Error **errp)
+BdrvChild *bdrv_open_child(const char *filename,
+                           QDict *options, const char *bdref_key,
+                           BlockDriverState* parent,
+                           const BdrvChildRole *child_role,
+                           bool allow_none, Error **errp)
 {
+    BdrvChild *c = NULL;
+    BlockDriverState *bs;
     QDict *image_options;
-    int ret;
     char *bdref_key_dot;
     const char *reference;
 
-    assert(pbs);
-    assert(*pbs == NULL);
+    assert(child_role != NULL);
 
     bdref_key_dot = g_strdup_printf("%s.", bdref_key);
     qdict_extract_subqdict(options, &image_options, bdref_key_dot);
@@ -1272,34 +1439,37 @@
 
     reference = qdict_get_try_str(options, bdref_key);
     if (!filename && !reference && !qdict_size(image_options)) {
-        if (allow_none) {
-            ret = 0;
-        } else {
+        if (!allow_none) {
             error_setg(errp, "A block device must be specified for \"%s\"",
                        bdref_key);
-            ret = -EINVAL;
         }
         QDECREF(image_options);
         goto done;
     }
 
-    ret = bdrv_open(pbs, filename, reference, image_options, flags, NULL, errp);
+    bs = bdrv_open_inherit(filename, reference, image_options, 0,
+                           parent, child_role, errp);
+    if (!bs) {
+        goto done;
+    }
+
+    c = bdrv_attach_child(parent, bs, bdref_key, child_role);
 
 done:
     qdict_del(options, bdref_key);
-    return ret;
+    return c;
 }
 
-int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp)
+static BlockDriverState *bdrv_append_temp_snapshot(BlockDriverState *bs,
+                                                   int flags,
+                                                   QDict *snapshot_options,
+                                                   Error **errp)
 {
     /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
     char *tmp_filename = g_malloc0(PATH_MAX + 1);
     int64_t total_size;
-    BlockDriver *bdrv_qcow2;
     QemuOpts *opts = NULL;
-    QDict *snapshot_options;
     BlockDriverState *bs_snapshot;
-    Error *local_err;
     int ret;
 
     /* if snapshot, we create a temporary backing file and open it
@@ -1308,7 +1478,6 @@
     /* Get the required size from the image */
     total_size = bdrv_getlength(bs);
     if (total_size < 0) {
-        ret = total_size;
         error_setg_errno(errp, -total_size, "Could not get image size");
         goto out;
     }
@@ -1320,41 +1489,45 @@
         goto out;
     }
 
-    bdrv_qcow2 = bdrv_find_format("qcow2");
-    opts = qemu_opts_create(bdrv_qcow2->create_opts, NULL, 0,
+    opts = qemu_opts_create(bdrv_qcow2.create_opts, NULL, 0,
                             &error_abort);
-    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, total_size);
-    ret = bdrv_create(bdrv_qcow2, tmp_filename, opts, &local_err);
+    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, total_size, &error_abort);
+    ret = bdrv_create(&bdrv_qcow2, tmp_filename, opts, errp);
     qemu_opts_del(opts);
     if (ret < 0) {
-        error_setg_errno(errp, -ret, "Could not create temporary overlay "
-                         "'%s': %s", tmp_filename,
-                         error_get_pretty(local_err));
-        error_free(local_err);
+        error_prepend(errp, "Could not create temporary overlay '%s': ",
+                      tmp_filename);
         goto out;
     }
 
-    /* Prepare a new options QDict for the temporary file */
-    snapshot_options = qdict_new();
+    /* Prepare options QDict for the temporary file */
     qdict_put(snapshot_options, "file.driver",
               qstring_from_str("file"));
     qdict_put(snapshot_options, "file.filename",
               qstring_from_str(tmp_filename));
+    qdict_put(snapshot_options, "driver",
+              qstring_from_str("qcow2"));
 
-    bs_snapshot = bdrv_new();
-
-    ret = bdrv_open(&bs_snapshot, NULL, NULL, snapshot_options,
-                    flags, bdrv_qcow2, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
+    bs_snapshot = bdrv_open(NULL, NULL, snapshot_options, flags, errp);
+    snapshot_options = NULL;
+    if (!bs_snapshot) {
+        ret = -EINVAL;
         goto out;
     }
 
+    /* bdrv_append() consumes a strong reference to bs_snapshot (i.e. it will
+     * call bdrv_unref() on it), so in order to be able to return one, we have
+     * to increase bs_snapshot's refcount here */
+    bdrv_ref(bs_snapshot);
     bdrv_append(bs_snapshot, bs);
 
-out:
     g_free(tmp_filename);
-    return ret;
+    return bs_snapshot;
+
+out:
+    QDECREF(snapshot_options);
+    g_free(tmp_filename);
+    return NULL;
 }
 
 /*
@@ -1372,81 +1545,92 @@
  * should be opened. If specified, neither options nor a filename may be given,
  * nor can an existing BDS be reused (that is, *pbs has to be NULL).
  */
-int bdrv_open(BlockDriverState **pbs, const char *filename,
-              const char *reference, QDict *options, int flags,
-              BlockDriver *drv, Error **errp)
+static BlockDriverState *bdrv_open_inherit(const char *filename,
+                                           const char *reference,
+                                           QDict *options, int flags,
+                                           BlockDriverState *parent,
+                                           const BdrvChildRole *child_role,
+                                           Error **errp)
 {
     int ret;
-    BlockDriverState *file = NULL, *bs;
+    BdrvChild *file = NULL;
+    BlockDriverState *bs;
+    BlockDriver *drv = NULL;
     const char *drvname;
+    const char *backing;
     Error *local_err = NULL;
+    QDict *snapshot_options = NULL;
     int snapshot_flags = 0;
 
-    assert(pbs);
+    assert(!child_role || !flags);
+    assert(!child_role == !parent);
 
     if (reference) {
         bool options_non_empty = options ? qdict_size(options) : false;
         QDECREF(options);
 
-        if (*pbs) {
-            error_setg(errp, "Cannot reuse an existing BDS when referencing "
-                       "another block device");
-            return -EINVAL;
-        }
-
         if (filename || options_non_empty) {
             error_setg(errp, "Cannot reference an existing block device with "
                        "additional options or a new filename");
-            return -EINVAL;
+            return NULL;
         }
 
         bs = bdrv_lookup_bs(reference, reference, errp);
         if (!bs) {
-            return -ENODEV;
+            return NULL;
         }
+
         bdrv_ref(bs);
-        *pbs = bs;
-        return 0;
+        return bs;
     }
 
-    if (*pbs) {
-        bs = *pbs;
-    } else {
-        bs = bdrv_new();
-    }
+    bs = bdrv_new();
 
     /* NULL means an empty set of options */
     if (options == NULL) {
         options = qdict_new();
     }
 
-    ret = bdrv_fill_options(&options, &filename, flags, drv, &local_err);
+    /* json: syntax counts as explicit options, as if in the QDict */
+    parse_json_protocol(options, &filename, &local_err);
     if (local_err) {
         goto fail;
     }
 
+    bs->explicit_options = qdict_clone_shallow(options);
+
+    if (child_role) {
+        bs->inherits_from = parent;
+        child_role->inherit_options(&flags, options,
+                                    parent->open_flags, parent->options);
+    }
+
+    ret = bdrv_fill_options(&options, filename, &flags, &local_err);
+    if (local_err) {
+        goto fail;
+    }
+
+    bs->open_flags = flags;
+    bs->options = options;
+    options = qdict_clone_shallow(options);
+
     /* Find the right image format driver */
-    drv = NULL;
     drvname = qdict_get_try_str(options, "driver");
     if (drvname) {
         drv = bdrv_find_format(drvname);
-        qdict_del(options, "driver");
         if (!drv) {
             error_setg(errp, "Unknown driver: '%s'", drvname);
-            ret = -EINVAL;
             goto fail;
         }
     }
 
     assert(drvname || !(flags & BDRV_O_PROTOCOL));
-    if (drv && !drv->bdrv_file_open) {
-        /* If the user explicitly wants a format driver here, we'll need to add
-         * another layer for the protocol in bs->file */
-        flags &= ~BDRV_O_PROTOCOL;
-    }
 
-    bs->options = options;
-    options = qdict_clone_shallow(options);
+    backing = qdict_get_try_str(options, "backing");
+    if (backing && *backing == '\0') {
+        flags |= BDRV_O_NO_BACKING;
+        qdict_del(options, "backing");
+    }
 
     /* Open image file without format layer */
     if ((flags & BDRV_O_PROTOCOL) == 0) {
@@ -1454,48 +1638,66 @@
             flags |= BDRV_O_ALLOW_RDWR;
         }
         if (flags & BDRV_O_SNAPSHOT) {
-            snapshot_flags = bdrv_temp_snapshot_flags(flags);
-            flags = bdrv_backing_flags(flags);
+            snapshot_options = qdict_new();
+            bdrv_temp_snapshot_options(&snapshot_flags, snapshot_options,
+                                       flags, options);
+            bdrv_backing_options(&flags, options, flags, options);
         }
 
-        assert(file == NULL);
-        ret = bdrv_open_image(&file, filename, options, "file",
-                              bdrv_inherited_flags(flags),
-                              true, &local_err);
-        if (ret < 0) {
+        bs->open_flags = flags;
+
+        file = bdrv_open_child(filename, options, "file", bs,
+                               &child_file, true, &local_err);
+        if (local_err) {
             goto fail;
         }
     }
 
     /* Image format probing */
+    bs->probed = !drv;
     if (!drv && file) {
         ret = find_image_format(file, filename, &drv, &local_err);
         if (ret < 0) {
             goto fail;
         }
+        /*
+         * This option update would logically belong in bdrv_fill_options(),
+         * but we first need to open bs->file for the probing to work, while
+         * opening bs->file already requires the (mostly) final set of options
+         * so that cache mode etc. can be inherited.
+         *
+         * Adding the driver later is somewhat ugly, but it's not an option
+         * that would ever be inherited, so it's correct. We just need to make
+         * sure to update both bs->options (which has the full effective
+         * options for bs) and options (which has file.* already removed).
+         */
+        qdict_put(bs->options, "driver", qstring_from_str(drv->format_name));
+        qdict_put(options, "driver", qstring_from_str(drv->format_name));
     } else if (!drv) {
         error_setg(errp, "Must specify either driver or file");
-        ret = -EINVAL;
         goto fail;
     }
 
+    /* BDRV_O_PROTOCOL must be set iff a protocol BDS is about to be created */
+    assert(!!(flags & BDRV_O_PROTOCOL) == !!drv->bdrv_file_open);
+    /* file must be NULL if a protocol BDS is about to be created
+     * (the inverse results in an error message from bdrv_open_common()) */
+    assert(!(flags & BDRV_O_PROTOCOL) || !file);
+
     /* Open the image */
-    ret = bdrv_open_common(bs, file, options, flags, drv, &local_err);
+    ret = bdrv_open_common(bs, file, options, &local_err);
     if (ret < 0) {
         goto fail;
     }
 
     if (file && (bs->file != file)) {
-        bdrv_unref(file);
+        bdrv_unref_child(bs, file);
         file = NULL;
     }
 
     /* If there is a backing file, use it */
     if ((flags & BDRV_O_NO_BACKING) == 0) {
-        QDict *backing_options;
-
-        qdict_extract_subqdict(options, &backing_options, "backing.");
-        ret = bdrv_open_backing_file(bs, backing_options, &local_err);
+        ret = bdrv_open_backing_file(bs, options, "backing", &local_err);
         if (ret < 0) {
             goto close_and_fail;
         }
@@ -1503,15 +1705,6 @@
 
     bdrv_refresh_filename(bs);
 
-    /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
-     * temporary snapshot afterwards. */
-    if (snapshot_flags) {
-        ret = bdrv_append_temp_snapshot(bs, snapshot_flags, &local_err);
-        if (local_err) {
-            goto close_and_fail;
-        }
-    }
-
     /* Check if any unknown options were used */
     if (options && (qdict_size(options) != 0)) {
         const QDictEntry *entry = qdict_first(options);
@@ -1519,62 +1712,72 @@
             error_setg(errp, "Block protocol '%s' doesn't support the option "
                        "'%s'", drv->format_name, entry->key);
         } else {
-            error_setg(errp, "Block format '%s' used by device '%s' doesn't "
-                       "support the option '%s'", drv->format_name,
-                       bdrv_get_device_name(bs), entry->key);
+            error_setg(errp,
+                       "Block format '%s' does not support the option '%s'",
+                       drv->format_name, entry->key);
         }
 
-        ret = -EINVAL;
         goto close_and_fail;
     }
 
     if (!bdrv_key_required(bs)) {
-        if (bs->blk) {
-            blk_dev_change_media_cb(bs->blk, true);
-        }
+        bdrv_parent_cb_change_media(bs, true);
     } else if (!runstate_check(RUN_STATE_PRELAUNCH)
                && !runstate_check(RUN_STATE_INMIGRATE)
                && !runstate_check(RUN_STATE_PAUSED)) { /* HACK */
         error_setg(errp,
                    "Guest must be stopped for opening of encrypted image");
-        ret = -EBUSY;
         goto close_and_fail;
     }
 
     QDECREF(options);
-    *pbs = bs;
-    return 0;
+
+    /* For snapshot=on, create a temporary qcow2 overlay. bs points to the
+     * temporary snapshot afterwards. */
+    if (snapshot_flags) {
+        BlockDriverState *snapshot_bs;
+        snapshot_bs = bdrv_append_temp_snapshot(bs, snapshot_flags,
+                                                snapshot_options, &local_err);
+        snapshot_options = NULL;
+        if (local_err) {
+            goto close_and_fail;
+        }
+        /* We are not going to return bs but the overlay on top of it
+         * (snapshot_bs); thus, we have to drop the strong reference to bs
+         * (which we obtained by calling bdrv_new()). bs will not be deleted,
+         * though, because the overlay still has a reference to it. */
+        bdrv_unref(bs);
+        bs = snapshot_bs;
+    }
+
+    return bs;
 
 fail:
     if (file != NULL) {
-        bdrv_unref(file);
+        bdrv_unref_child(bs, file);
     }
+    QDECREF(snapshot_options);
+    QDECREF(bs->explicit_options);
     QDECREF(bs->options);
     QDECREF(options);
     bs->options = NULL;
-    if (!*pbs) {
-        /* If *pbs is NULL, a new BDS has been created in this function and
-           needs to be freed now. Otherwise, it does not need to be closed,
-           since it has not really been opened yet. */
-        bdrv_unref(bs);
-    }
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    return ret;
+    bdrv_unref(bs);
+    error_propagate(errp, local_err);
+    return NULL;
 
 close_and_fail:
-    /* See fail path, but now the BDS has to be always closed */
-    if (*pbs) {
-        bdrv_close(bs);
-    } else {
-        bdrv_unref(bs);
-    }
+    bdrv_unref(bs);
+    QDECREF(snapshot_options);
     QDECREF(options);
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    return ret;
+    error_propagate(errp, local_err);
+    return NULL;
+}
+
+BlockDriverState *bdrv_open(const char *filename, const char *reference,
+                            QDict *options, int flags, Error **errp)
+{
+    return bdrv_open_inherit(filename, reference, options, flags, NULL,
+                             NULL, errp);
 }
 
 typedef struct BlockReopenQueueEntry {
@@ -1595,39 +1798,116 @@
  *
  * bs is the BlockDriverState to add to the reopen queue.
  *
+ * options contains the changed options for the associated bs
+ * (the BlockReopenQueue takes ownership)
+ *
  * flags contains the open flags for the associated bs
  *
  * returns a pointer to bs_queue, which is either the newly allocated
  * bs_queue, or the existing bs_queue being used.
  *
  */
-BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags)
+static BlockReopenQueue *bdrv_reopen_queue_child(BlockReopenQueue *bs_queue,
+                                                 BlockDriverState *bs,
+                                                 QDict *options,
+                                                 int flags,
+                                                 const BdrvChildRole *role,
+                                                 QDict *parent_options,
+                                                 int parent_flags)
 {
     assert(bs != NULL);
 
     BlockReopenQueueEntry *bs_entry;
+    BdrvChild *child;
+    QDict *old_options, *explicit_options;
+
     if (bs_queue == NULL) {
         bs_queue = g_new0(BlockReopenQueue, 1);
         QSIMPLEQ_INIT(bs_queue);
     }
 
+    if (!options) {
+        options = qdict_new();
+    }
+
+    /*
+     * Precedence of options:
+     * 1. Explicitly passed in options (highest)
+     * 2. Set in flags (only for top level)
+     * 3. Retained from explicitly set options of bs
+     * 4. Inherited from parent node
+     * 5. Retained from effective options of bs
+     */
+
+    if (!parent_options) {
+        /*
+         * Any setting represented by flags is always updated. If the
+         * corresponding QDict option is set, it takes precedence. Otherwise
+         * the flag is translated into a QDict option. The old setting of bs is
+         * not considered.
+         */
+        update_options_from_flags(options, flags);
+    }
+
+    /* Old explicitly set values (don't overwrite by inherited value) */
+    old_options = qdict_clone_shallow(bs->explicit_options);
+    bdrv_join_options(bs, options, old_options);
+    QDECREF(old_options);
+
+    explicit_options = qdict_clone_shallow(options);
+
+    /* Inherit from parent node */
+    if (parent_options) {
+        assert(!flags);
+        role->inherit_options(&flags, options, parent_flags, parent_options);
+    }
+
+    /* Old values are used for options that aren't set yet */
+    old_options = qdict_clone_shallow(bs->options);
+    bdrv_join_options(bs, options, old_options);
+    QDECREF(old_options);
+
     /* bdrv_open() masks this flag out */
     flags &= ~BDRV_O_PROTOCOL;
 
-    if (bs->file) {
-        bdrv_reopen_queue(bs_queue, bs->file, bdrv_inherited_flags(flags));
+    QLIST_FOREACH(child, &bs->children, next) {
+        QDict *new_child_options;
+        char *child_key_dot;
+
+        /* reopen can only change the options of block devices that were
+         * implicitly created and inherited options. For other (referenced)
+         * block devices, a syntax like "backing.foo" results in an error. */
+        if (child->bs->inherits_from != bs) {
+            continue;
+        }
+
+        child_key_dot = g_strdup_printf("%s.", child->name);
+        qdict_extract_subqdict(options, &new_child_options, child_key_dot);
+        g_free(child_key_dot);
+
+        bdrv_reopen_queue_child(bs_queue, child->bs, new_child_options, 0,
+                                child->role, options, flags);
     }
 
     bs_entry = g_new0(BlockReopenQueueEntry, 1);
     QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
 
     bs_entry->state.bs = bs;
+    bs_entry->state.options = options;
+    bs_entry->state.explicit_options = explicit_options;
     bs_entry->state.flags = flags;
 
     return bs_queue;
 }
 
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs,
+                                    QDict *options, int flags)
+{
+    return bdrv_reopen_queue_child(bs_queue, bs, options, flags,
+                                   NULL, NULL, 0);
+}
+
 /*
  * Reopen multiple BlockDriverStates atomically & transactionally.
  *
@@ -1674,7 +1954,10 @@
     QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
         if (ret && bs_entry->prepared) {
             bdrv_reopen_abort(&bs_entry->state);
+        } else if (ret) {
+            QDECREF(bs_entry->state.explicit_options);
         }
+        QDECREF(bs_entry->state.options);
         g_free(bs_entry);
     }
     g_free(bs_queue);
@@ -1687,7 +1970,7 @@
 {
     int ret = -1;
     Error *local_err = NULL;
-    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, NULL, bdrv_flags);
 
     ret = bdrv_reopen_multiple(queue, &local_err);
     if (local_err != NULL) {
@@ -1720,25 +2003,49 @@
     int ret = -1;
     Error *local_err = NULL;
     BlockDriver *drv;
+    QemuOpts *opts;
+    const char *value;
 
     assert(reopen_state != NULL);
     assert(reopen_state->bs->drv != NULL);
     drv = reopen_state->bs->drv;
 
+    /* Process generic block layer options */
+    opts = qemu_opts_create(&bdrv_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, reopen_state->options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto error;
+    }
+
+    update_flags_from_options(&reopen_state->flags, opts);
+
+    /* node-name and driver must be unchanged. Put them back into the QDict, so
+     * that they are checked at the end of this function. */
+    value = qemu_opt_get(opts, "node-name");
+    if (value) {
+        qdict_put(reopen_state->options, "node-name", qstring_from_str(value));
+    }
+
+    value = qemu_opt_get(opts, "driver");
+    if (value) {
+        qdict_put(reopen_state->options, "driver", qstring_from_str(value));
+    }
+
     /* if we are to stay read-only, do not allow permission change
      * to r/w */
     if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
         reopen_state->flags & BDRV_O_RDWR) {
-        error_set(errp, QERR_DEVICE_IS_READ_ONLY,
-                  bdrv_get_device_name(reopen_state->bs));
+        error_setg(errp, "Node '%s' is read only",
+                   bdrv_get_device_or_node_name(reopen_state->bs));
         goto error;
     }
 
 
     ret = bdrv_flush(reopen_state->bs);
     if (ret) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
-                  strerror(-ret));
+        error_setg_errno(errp, -ret, "Error flushing drive");
         goto error;
     }
 
@@ -1756,16 +2063,37 @@
     } else {
         /* It is currently mandatory to have a bdrv_reopen_prepare()
          * handler for each supported drv. */
-        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-                  drv->format_name, bdrv_get_device_name(reopen_state->bs),
-                 "reopening of file");
+        error_setg(errp, "Block format '%s' used by node '%s' "
+                   "does not support reopening files", drv->format_name,
+                   bdrv_get_device_or_node_name(reopen_state->bs));
         ret = -1;
         goto error;
     }
 
+    /* Options that are not handled are only okay if they are unchanged
+     * compared to the old state. It is expected that some options are only
+     * used for the initial open, but not reopen (e.g. filename) */
+    if (qdict_size(reopen_state->options)) {
+        const QDictEntry *entry = qdict_first(reopen_state->options);
+
+        do {
+            QString *new_obj = qobject_to_qstring(entry->value);
+            const char *new = qstring_get_str(new_obj);
+            const char *old = qdict_get_try_str(reopen_state->bs->options,
+                                                entry->key);
+
+            if (!old || strcmp(new, old)) {
+                error_setg(errp, "Cannot change the option '%s'", entry->key);
+                ret = -EINVAL;
+                goto error;
+            }
+        } while ((entry = qdict_next(reopen_state->options, entry)));
+    }
+
     ret = 0;
 
 error:
+    qemu_opts_del(opts);
     return ret;
 }
 
@@ -1788,9 +2116,10 @@
     }
 
     /* set BDS specific flags now */
+    QDECREF(reopen_state->bs->explicit_options);
+
+    reopen_state->bs->explicit_options   = reopen_state->explicit_options;
     reopen_state->bs->open_flags         = reopen_state->flags;
-    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
-                                              BDRV_O_CACHE_WB);
     reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
 
     bdrv_refresh_limits(reopen_state->bs, NULL);
@@ -1811,295 +2140,111 @@
     if (drv->bdrv_reopen_abort) {
         drv->bdrv_reopen_abort(reopen_state);
     }
+
+    QDECREF(reopen_state->explicit_options);
 }
 
 
-void bdrv_close(BlockDriverState *bs)
+static void bdrv_close(BlockDriverState *bs)
 {
     BdrvAioNotifier *ban, *ban_next;
 
-    if (bs->job) {
-        block_job_cancel_sync(bs->job);
-    }
-    bdrv_drain_all(); /* complete I/O */
+    assert(!bs->job);
+    assert(!bs->refcnt);
+
+    bdrv_drained_begin(bs); /* complete I/O */
     bdrv_flush(bs);
-    bdrv_drain_all(); /* in case flush left pending I/O */
-    notifier_list_notify(&bs->close_notifiers, bs);
+    bdrv_drain(bs); /* in case flush left pending I/O */
+
+    bdrv_release_named_dirty_bitmaps(bs);
+    assert(QLIST_EMPTY(&bs->dirty_bitmaps));
 
     if (bs->drv) {
-        if (bs->backing_hd) {
-            BlockDriverState *backing_hd = bs->backing_hd;
-            bdrv_set_backing_hd(bs, NULL);
-            bdrv_unref(backing_hd);
-        }
+        BdrvChild *child, *next;
+
         bs->drv->bdrv_close(bs);
+        bs->drv = NULL;
+
+        bdrv_set_backing_hd(bs, NULL);
+
+        if (bs->file != NULL) {
+            bdrv_unref_child(bs, bs->file);
+            bs->file = NULL;
+        }
+
+        QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
+            /* TODO Remove bdrv_unref() from drivers' close function and use
+             * bdrv_unref_child() here */
+            if (child->bs->inherits_from == bs) {
+                child->bs->inherits_from = NULL;
+            }
+            bdrv_detach_child(child);
+        }
+
         g_free(bs->opaque);
         bs->opaque = NULL;
-        bs->drv = NULL;
         bs->copy_on_read = 0;
         bs->backing_file[0] = '\0';
         bs->backing_format[0] = '\0';
         bs->total_sectors = 0;
-        bs->encrypted = 0;
-        bs->valid_key = 0;
-        bs->sg = 0;
-        bs->growable = 0;
-        bs->zero_beyond_eof = false;
+        bs->encrypted = false;
+        bs->valid_key = false;
+        bs->sg = false;
         QDECREF(bs->options);
+        QDECREF(bs->explicit_options);
         bs->options = NULL;
         QDECREF(bs->full_open_options);
         bs->full_open_options = NULL;
-
-        if (bs->file != NULL) {
-            bdrv_unref(bs->file);
-            bs->file = NULL;
-        }
-    }
-
-    if (bs->blk) {
-        blk_dev_change_media_cb(bs->blk, false);
-    }
-
-    /*throttling disk I/O limits*/
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_disable(bs);
     }
 
     QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_next) {
         g_free(ban);
     }
     QLIST_INIT(&bs->aio_notifiers);
+    bdrv_drained_end(bs);
 }
 
 void bdrv_close_all(void)
 {
-    BlockDriverState *bs;
+    block_job_cancel_sync_all();
 
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
+    /* Drop references from requests still in flight, such as canceled block
+     * jobs whose AIO context has not been polled yet */
+    bdrv_drain_all();
 
-        aio_context_acquire(aio_context);
-        bdrv_close(bs);
-        aio_context_release(aio_context);
-    }
+    blk_remove_all_bs();
+    blockdev_close_all_bdrv_states();
+
+    assert(QTAILQ_EMPTY(&all_bdrv_states));
 }
 
-/* Check if any requests are in-flight (including throttled requests) */
-static bool bdrv_requests_pending(BlockDriverState *bs)
+static void change_parent_backing_link(BlockDriverState *from,
+                                       BlockDriverState *to)
 {
-    if (!QLIST_EMPTY(&bs->tracked_requests)) {
-        return true;
-    }
-    if (!qemu_co_queue_empty(&bs->throttled_reqs[0])) {
-        return true;
-    }
-    if (!qemu_co_queue_empty(&bs->throttled_reqs[1])) {
-        return true;
-    }
-    if (bs->file && bdrv_requests_pending(bs->file)) {
-        return true;
-    }
-    if (bs->backing_hd && bdrv_requests_pending(bs->backing_hd)) {
-        return true;
-    }
-    return false;
-}
+    BdrvChild *c, *next, *to_c;
 
-static bool bdrv_drain_one(BlockDriverState *bs)
-{
-    bool bs_busy;
-
-    bdrv_flush_io_queue(bs);
-    bdrv_start_throttled_reqs(bs);
-    bs_busy = bdrv_requests_pending(bs);
-    bs_busy |= aio_poll(bdrv_get_aio_context(bs), bs_busy);
-    return bs_busy;
-}
-
-/*
- * Wait for pending requests to complete on a single BlockDriverState subtree
- *
- * See the warning in bdrv_drain_all().  This function can only be called if
- * you are sure nothing can generate I/O because you have op blockers
- * installed.
- *
- * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
- * AioContext.
- */
-void bdrv_drain(BlockDriverState *bs)
-{
-    while (bdrv_drain_one(bs)) {
-        /* Keep iterating */
-    }
-}
-
-/*
- * Wait for pending requests to complete across all BlockDriverStates
- *
- * This function does not flush data to disk, use bdrv_flush_all() for that
- * after calling this function.
- *
- * Note that completion of an asynchronous I/O operation can trigger any
- * number of other I/O operations on other devices---for example a coroutine
- * can be arbitrarily complex and a constant flow of I/O can come until the
- * coroutine is complete.  Because of this, it is not possible to have a
- * function to drain a single device's I/O queue.
- */
-void bdrv_drain_all(void)
-{
-    /* Always run first iteration so any pending completion BHs run */
-    bool busy = true;
-    BlockDriverState *bs;
-
-    while (busy) {
-        busy = false;
-
-        QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-            AioContext *aio_context = bdrv_get_aio_context(bs);
-
-            aio_context_acquire(aio_context);
-            busy |= bdrv_drain_one(bs);
-            aio_context_release(aio_context);
+    QLIST_FOREACH_SAFE(c, &from->parents, next_parent, next) {
+        if (c->role == &child_backing) {
+            /* @from is generally not allowed to be a backing file, except for
+             * when @to is the overlay. In that case, @from may not be replaced
+             * by @to as @to's backing node. */
+            QLIST_FOREACH(to_c, &to->children, next) {
+                if (to_c == c) {
+                    break;
+                }
+            }
+            if (to_c) {
+                continue;
+            }
         }
+
+        assert(c->role != &child_backing);
+        bdrv_ref(to);
+        bdrv_replace_child(c, to);
+        bdrv_unref(from);
     }
 }
 
-/* make a BlockDriverState anonymous by removing from bdrv_state and
- * graph_bdrv_state list.
-   Also, NULL terminate the device_name to prevent double remove */
-void bdrv_make_anon(BlockDriverState *bs)
-{
-    /*
-     * Take care to remove bs from bdrv_states only when it's actually
-     * in it.  Note that bs->device_list.tqe_prev is initially null,
-     * and gets set to non-null by QTAILQ_INSERT_TAIL().  Establish
-     * the useful invariant "bs in bdrv_states iff bs->tqe_prev" by
-     * resetting it to null on remove.
-     */
-    if (bs->device_list.tqe_prev) {
-        QTAILQ_REMOVE(&bdrv_states, bs, device_list);
-        bs->device_list.tqe_prev = NULL;
-    }
-    if (bs->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs, node_list);
-    }
-    bs->node_name[0] = '\0';
-}
-
-static void bdrv_rebind(BlockDriverState *bs)
-{
-    if (bs->drv && bs->drv->bdrv_rebind) {
-        bs->drv->bdrv_rebind(bs);
-    }
-}
-
-static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
-                                     BlockDriverState *bs_src)
-{
-    /* move some fields that need to stay attached to the device */
-
-    /* dev info */
-    bs_dest->guest_block_size   = bs_src->guest_block_size;
-    bs_dest->copy_on_read       = bs_src->copy_on_read;
-
-    bs_dest->enable_write_cache = bs_src->enable_write_cache;
-
-    /* i/o throttled req */
-    memcpy(&bs_dest->throttle_state,
-           &bs_src->throttle_state,
-           sizeof(ThrottleState));
-    bs_dest->throttled_reqs[0]  = bs_src->throttled_reqs[0];
-    bs_dest->throttled_reqs[1]  = bs_src->throttled_reqs[1];
-    bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
-
-    /* r/w error */
-    bs_dest->on_read_error      = bs_src->on_read_error;
-    bs_dest->on_write_error     = bs_src->on_write_error;
-
-    /* i/o status */
-    bs_dest->iostatus_enabled   = bs_src->iostatus_enabled;
-    bs_dest->iostatus           = bs_src->iostatus;
-
-    /* dirty bitmap */
-    bs_dest->dirty_bitmaps      = bs_src->dirty_bitmaps;
-
-    /* reference count */
-    bs_dest->refcnt             = bs_src->refcnt;
-
-    /* job */
-    bs_dest->job                = bs_src->job;
-
-    /* keep the same entry in bdrv_states */
-    bs_dest->device_list = bs_src->device_list;
-    bs_dest->blk = bs_src->blk;
-
-    memcpy(bs_dest->op_blockers, bs_src->op_blockers,
-           sizeof(bs_dest->op_blockers));
-}
-
-/*
- * Swap bs contents for two image chains while they are live,
- * while keeping required fields on the BlockDriverState that is
- * actually attached to a device.
- *
- * This will modify the BlockDriverState fields, and swap contents
- * between bs_new and bs_old. Both bs_new and bs_old are modified.
- *
- * bs_new must not be attached to a BlockBackend.
- *
- * This function does not create any image files.
- */
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
-{
-    BlockDriverState tmp;
-
-    /* The code needs to swap the node_name but simply swapping node_list won't
-     * work so first remove the nodes from the graph list, do the swap then
-     * insert them back if needed.
-     */
-    if (bs_new->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs_new, node_list);
-    }
-    if (bs_old->node_name[0] != '\0') {
-        QTAILQ_REMOVE(&graph_bdrv_states, bs_old, node_list);
-    }
-
-    /* bs_new must be unattached and shouldn't have anything fancy enabled */
-    assert(!bs_new->blk);
-    assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
-    assert(bs_new->job == NULL);
-    assert(bs_new->io_limits_enabled == false);
-    assert(!throttle_have_timer(&bs_new->throttle_state));
-
-    tmp = *bs_new;
-    *bs_new = *bs_old;
-    *bs_old = tmp;
-
-    /* there are some fields that should not be swapped, move them back */
-    bdrv_move_feature_fields(&tmp, bs_old);
-    bdrv_move_feature_fields(bs_old, bs_new);
-    bdrv_move_feature_fields(bs_new, &tmp);
-
-    /* bs_new must remain unattached */
-    assert(!bs_new->blk);
-
-    /* Check a few fields that should remain attached to the device */
-    assert(bs_new->job == NULL);
-    assert(bs_new->io_limits_enabled == false);
-    assert(!throttle_have_timer(&bs_new->throttle_state));
-
-    /* insert the nodes back into the graph node list if needed */
-    if (bs_new->node_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs_new, node_list);
-    }
-    if (bs_old->node_name[0] != '\0') {
-        QTAILQ_INSERT_TAIL(&graph_bdrv_states, bs_old, node_list);
-    }
-
-    bdrv_rebind(bs_new);
-    bdrv_rebind(bs_old);
-}
-
 /*
  * Add new bs contents at the top of an image chain while the chain is
  * live, while keeping required fields on the top layer.
@@ -2110,14 +2255,38 @@
  * bs_new must not be attached to a BlockBackend.
  *
  * This function does not create any image files.
+ *
+ * bdrv_append() takes ownership of a bs_new reference and unrefs it because
+ * that's what the callers commonly need. bs_new will be referenced by the old
+ * parents of bs_top after bdrv_append() returns. If the caller needs to keep a
+ * reference of its own, it must call bdrv_ref().
  */
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top)
 {
-    bdrv_swap(bs_new, bs_top);
+    assert(!bdrv_requests_pending(bs_top));
+    assert(!bdrv_requests_pending(bs_new));
 
-    /* The contents of 'tmp' will become bs_top, as we are
-     * swapping bs_new and bs_top contents. */
-    bdrv_set_backing_hd(bs_top, bs_new);
+    bdrv_ref(bs_top);
+
+    change_parent_backing_link(bs_top, bs_new);
+    bdrv_set_backing_hd(bs_new, bs_top);
+    bdrv_unref(bs_top);
+
+    /* bs_new is now referenced by its new parents, we don't need the
+     * additional reference any more. */
+    bdrv_unref(bs_new);
+}
+
+void bdrv_replace_in_backing_chain(BlockDriverState *old, BlockDriverState *new)
+{
+    assert(!bdrv_requests_pending(old));
+    assert(!bdrv_requests_pending(new));
+
+    bdrv_ref(old);
+
+    change_parent_backing_link(old, new);
+
+    bdrv_unref(old);
 }
 
 static void bdrv_delete(BlockDriverState *bs)
@@ -2125,12 +2294,14 @@
     assert(!bs->job);
     assert(bdrv_op_blocker_is_empty(bs));
     assert(!bs->refcnt);
-    assert(QLIST_EMPTY(&bs->dirty_bitmaps));
 
     bdrv_close(bs);
 
     /* remove from list, if necessary */
-    bdrv_make_anon(bs);
+    if (bs->node_name[0] != '\0') {
+        QTAILQ_REMOVE(&graph_bdrv_states, bs, node_list);
+    }
+    QTAILQ_REMOVE(&all_bdrv_states, bs, bs_list);
 
     g_free(bs);
 }
@@ -2155,285 +2326,6 @@
     return bs->drv->bdrv_check(bs, res, fix);
 }
 
-#define COMMIT_BUF_SECTORS 2048
-
-/* commit COW file into the raw image */
-int bdrv_commit(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    int64_t sector, total_sectors, length, backing_length;
-    int n, ro, open_flags;
-    int ret = 0;
-    uint8_t *buf = NULL;
-    char filename[PATH_MAX];
-
-    if (!drv)
-        return -ENOMEDIUM;
-
-    if (!bs->backing_hd) {
-        return -ENOTSUP;
-    }
-
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT, NULL) ||
-        bdrv_op_is_blocked(bs->backing_hd, BLOCK_OP_TYPE_COMMIT, NULL)) {
-        return -EBUSY;
-    }
-
-    ro = bs->backing_hd->read_only;
-    /* Use pstrcpy (not strncpy): filename must be NUL-terminated. */
-    pstrcpy(filename, sizeof(filename), bs->backing_hd->filename);
-    open_flags =  bs->backing_hd->open_flags;
-
-    if (ro) {
-        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
-            return -EACCES;
-        }
-    }
-
-    length = bdrv_getlength(bs);
-    if (length < 0) {
-        ret = length;
-        goto ro_cleanup;
-    }
-
-    backing_length = bdrv_getlength(bs->backing_hd);
-    if (backing_length < 0) {
-        ret = backing_length;
-        goto ro_cleanup;
-    }
-
-    /* If our top snapshot is larger than the backing file image,
-     * grow the backing file image if possible.  If not possible,
-     * we must return an error */
-    if (length > backing_length) {
-        ret = bdrv_truncate(bs->backing_hd, length);
-        if (ret < 0) {
-            goto ro_cleanup;
-        }
-    }
-
-    total_sectors = length >> BDRV_SECTOR_BITS;
-
-    /* qemu_try_blockalign() for bs will choose an alignment that works for
-     * bs->backing_hd as well, so no need to compare the alignment manually. */
-    buf = qemu_try_blockalign(bs, COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
-    if (buf == NULL) {
-        ret = -ENOMEM;
-        goto ro_cleanup;
-    }
-
-    for (sector = 0; sector < total_sectors; sector += n) {
-        ret = bdrv_is_allocated(bs, sector, COMMIT_BUF_SECTORS, &n);
-        if (ret < 0) {
-            goto ro_cleanup;
-        }
-        if (ret) {
-            ret = bdrv_read(bs, sector, buf, n);
-            if (ret < 0) {
-                goto ro_cleanup;
-            }
-
-            ret = bdrv_write(bs->backing_hd, sector, buf, n);
-            if (ret < 0) {
-                goto ro_cleanup;
-            }
-        }
-    }
-
-    if (drv->bdrv_make_empty) {
-        ret = drv->bdrv_make_empty(bs);
-        if (ret < 0) {
-            goto ro_cleanup;
-        }
-        bdrv_flush(bs);
-    }
-
-    /*
-     * Make sure all data we wrote to the backing device is actually
-     * stable on disk.
-     */
-    if (bs->backing_hd) {
-        bdrv_flush(bs->backing_hd);
-    }
-
-    ret = 0;
-ro_cleanup:
-    qemu_vfree(buf);
-
-    if (ro) {
-        /* ignoring error return here */
-        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
-    }
-
-    return ret;
-}
-
-int bdrv_commit_all(void)
-{
-    BlockDriverState *bs;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-
-        aio_context_acquire(aio_context);
-        if (bs->drv && bs->backing_hd) {
-            int ret = bdrv_commit(bs);
-            if (ret < 0) {
-                aio_context_release(aio_context);
-                return ret;
-            }
-        }
-        aio_context_release(aio_context);
-    }
-    return 0;
-}
-
-/**
- * Remove an active request from the tracked requests list
- *
- * This function should be called when a tracked request is completing.
- */
-static void tracked_request_end(BdrvTrackedRequest *req)
-{
-    if (req->serialising) {
-        req->bs->serialising_in_flight--;
-    }
-
-    QLIST_REMOVE(req, list);
-    qemu_co_queue_restart_all(&req->wait_queue);
-}
-
-/**
- * Add an active request to the tracked requests list
- */
-static void tracked_request_begin(BdrvTrackedRequest *req,
-                                  BlockDriverState *bs,
-                                  int64_t offset,
-                                  unsigned int bytes, bool is_write)
-{
-    *req = (BdrvTrackedRequest){
-        .bs = bs,
-        .offset         = offset,
-        .bytes          = bytes,
-        .is_write       = is_write,
-        .co             = qemu_coroutine_self(),
-        .serialising    = false,
-        .overlap_offset = offset,
-        .overlap_bytes  = bytes,
-    };
-
-    qemu_co_queue_init(&req->wait_queue);
-
-    QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
-}
-
-static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
-{
-    int64_t overlap_offset = req->offset & ~(align - 1);
-    unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
-                               - overlap_offset;
-
-    if (!req->serialising) {
-        req->bs->serialising_in_flight++;
-        req->serialising = true;
-    }
-
-    req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
-    req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
-}
-
-/**
- * Round a region to cluster boundaries
- */
-void bdrv_round_to_clusters(BlockDriverState *bs,
-                            int64_t sector_num, int nb_sectors,
-                            int64_t *cluster_sector_num,
-                            int *cluster_nb_sectors)
-{
-    BlockDriverInfo bdi;
-
-    if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
-        *cluster_sector_num = sector_num;
-        *cluster_nb_sectors = nb_sectors;
-    } else {
-        int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
-        *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
-        *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
-                                            nb_sectors, c);
-    }
-}
-
-static int bdrv_get_cluster_size(BlockDriverState *bs)
-{
-    BlockDriverInfo bdi;
-    int ret;
-
-    ret = bdrv_get_info(bs, &bdi);
-    if (ret < 0 || bdi.cluster_size == 0) {
-        return bs->request_alignment;
-    } else {
-        return bdi.cluster_size;
-    }
-}
-
-static bool tracked_request_overlaps(BdrvTrackedRequest *req,
-                                     int64_t offset, unsigned int bytes)
-{
-    /*        aaaa   bbbb */
-    if (offset >= req->overlap_offset + req->overlap_bytes) {
-        return false;
-    }
-    /* bbbb   aaaa        */
-    if (req->overlap_offset >= offset + bytes) {
-        return false;
-    }
-    return true;
-}
-
-static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
-{
-    BlockDriverState *bs = self->bs;
-    BdrvTrackedRequest *req;
-    bool retry;
-    bool waited = false;
-
-    if (!bs->serialising_in_flight) {
-        return false;
-    }
-
-    do {
-        retry = false;
-        QLIST_FOREACH(req, &bs->tracked_requests, list) {
-            if (req == self || (!req->serialising && !self->serialising)) {
-                continue;
-            }
-            if (tracked_request_overlaps(req, self->overlap_offset,
-                                         self->overlap_bytes))
-            {
-                /* Hitting this means there was a reentrant request, for
-                 * example, a block driver issuing nested requests.  This must
-                 * never happen since it means deadlock.
-                 */
-                assert(qemu_coroutine_self() != req->co);
-
-                /* If the request is already (indirectly) waiting for us, or
-                 * will wait for us as soon as it wakes up, then just go on
-                 * (instead of producing a deadlock in the former case). */
-                if (!req->waiting_for) {
-                    self->waiting_for = req;
-                    qemu_co_queue_wait(&req->wait_queue);
-                    self->waiting_for = NULL;
-                    retry = true;
-                    waited = true;
-                    break;
-                }
-            }
-        }
-    } while (retry);
-
-    return waited;
-}
-
 /*
  * Return values:
  * 0        - success
@@ -2479,8 +2371,8 @@
 BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
                                     BlockDriverState *bs)
 {
-    while (active && bs != active->backing_hd) {
-        active = active->backing_hd;
+    while (active && bs != backing_bs(active)) {
+        active = backing_bs(active);
     }
 
     return active;
@@ -2492,12 +2384,6 @@
     return bdrv_find_overlay(bs, NULL);
 }
 
-typedef struct BlkIntermediateStates {
-    BlockDriverState *bs;
-    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
-} BlkIntermediateStates;
-
-
 /*
  * Drops images above 'base' up to and including 'top', and sets the image
  * above 'top' to have base as its backing file.
@@ -2530,15 +2416,9 @@
 int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
                            BlockDriverState *base, const char *backing_file_str)
 {
-    BlockDriverState *intermediate;
-    BlockDriverState *base_bs = NULL;
     BlockDriverState *new_top_bs = NULL;
-    BlkIntermediateStates *intermediate_state, *next;
     int ret = -EIO;
 
-    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
-    QSIMPLEQ_INIT(&states_to_delete);
-
     if (!top->drv || !base->drv) {
         goto exit;
     }
@@ -2550,909 +2430,32 @@
         goto exit;
     }
 
-    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+    /* special case of new_top_bs->backing->bs already pointing to base - nothing
      * to do, no intermediate images */
-    if (new_top_bs->backing_hd == base) {
+    if (backing_bs(new_top_bs) == base) {
         ret = 0;
         goto exit;
     }
 
-    intermediate = top;
-
-    /* now we will go down through the list, and add each BDS we find
-     * into our deletion queue, until we hit the 'base'
-     */
-    while (intermediate) {
-        intermediate_state = g_new0(BlkIntermediateStates, 1);
-        intermediate_state->bs = intermediate;
-        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
-
-        if (intermediate->backing_hd == base) {
-            base_bs = intermediate->backing_hd;
-            break;
-        }
-        intermediate = intermediate->backing_hd;
-    }
-    if (base_bs == NULL) {
-        /* something went wrong, we did not end at the base. safely
-         * unravel everything, and exit with error */
+    /* Make sure that base is in the backing chain of top */
+    if (!bdrv_chain_contains(top, base)) {
         goto exit;
     }
 
     /* success - we can delete the intermediate states, and link top->base */
-    backing_file_str = backing_file_str ? backing_file_str : base_bs->filename;
+    backing_file_str = backing_file_str ? backing_file_str : base->filename;
     ret = bdrv_change_backing_file(new_top_bs, backing_file_str,
-                                   base_bs->drv ? base_bs->drv->format_name : "");
+                                   base->drv ? base->drv->format_name : "");
     if (ret) {
         goto exit;
     }
-    bdrv_set_backing_hd(new_top_bs, base_bs);
+    bdrv_set_backing_hd(new_top_bs, base);
 
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        /* so that bdrv_close() does not recursively close the chain */
-        bdrv_set_backing_hd(intermediate_state->bs, NULL);
-        bdrv_unref(intermediate_state->bs);
-    }
     ret = 0;
-
 exit:
-    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
-        g_free(intermediate_state);
-    }
     return ret;
 }
 
-
-static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
-                                   size_t size)
-{
-    int64_t len;
-
-    if (size > INT_MAX) {
-        return -EIO;
-    }
-
-    if (!bdrv_is_inserted(bs))
-        return -ENOMEDIUM;
-
-    if (bs->growable)
-        return 0;
-
-    len = bdrv_getlength(bs);
-
-    if (offset < 0)
-        return -EIO;
-
-    if ((offset > len) || (len - offset < size))
-        return -EIO;
-
-    return 0;
-}
-
-static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
-                              int nb_sectors)
-{
-    if (nb_sectors < 0 || nb_sectors > INT_MAX / BDRV_SECTOR_SIZE) {
-        return -EIO;
-    }
-
-    return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
-                                   nb_sectors * BDRV_SECTOR_SIZE);
-}
-
-typedef struct RwCo {
-    BlockDriverState *bs;
-    int64_t offset;
-    QEMUIOVector *qiov;
-    bool is_write;
-    int ret;
-    BdrvRequestFlags flags;
-} RwCo;
-
-static void coroutine_fn bdrv_rw_co_entry(void *opaque)
-{
-    RwCo *rwco = opaque;
-
-    if (!rwco->is_write) {
-        rwco->ret = bdrv_co_do_preadv(rwco->bs, rwco->offset,
-                                      rwco->qiov->size, rwco->qiov,
-                                      rwco->flags);
-    } else {
-        rwco->ret = bdrv_co_do_pwritev(rwco->bs, rwco->offset,
-                                       rwco->qiov->size, rwco->qiov,
-                                       rwco->flags);
-    }
-}
-
-/*
- * Process a vectored synchronous request using coroutines
- */
-static int bdrv_prwv_co(BlockDriverState *bs, int64_t offset,
-                        QEMUIOVector *qiov, bool is_write,
-                        BdrvRequestFlags flags)
-{
-    Coroutine *co;
-    RwCo rwco = {
-        .bs = bs,
-        .offset = offset,
-        .qiov = qiov,
-        .is_write = is_write,
-        .ret = NOT_DONE,
-        .flags = flags,
-    };
-
-    /**
-     * In sync call context, when the vcpu is blocked, this throttling timer
-     * will not fire; so the I/O throttling function has to be disabled here
-     * if it has been enabled.
-     */
-    if (bs->io_limits_enabled) {
-        fprintf(stderr, "Disabling I/O throttling on '%s' due "
-                        "to synchronous I/O.\n", bdrv_get_device_name(bs));
-        bdrv_io_limits_disable(bs);
-    }
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_rw_co_entry(&rwco);
-    } else {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-
-        co = qemu_coroutine_create(bdrv_rw_co_entry);
-        qemu_coroutine_enter(co, &rwco);
-        while (rwco.ret == NOT_DONE) {
-            aio_poll(aio_context, true);
-        }
-    }
-    return rwco.ret;
-}
-
-/*
- * Process a synchronous request using coroutines
- */
-static int bdrv_rw_co(BlockDriverState *bs, int64_t sector_num, uint8_t *buf,
-                      int nb_sectors, bool is_write, BdrvRequestFlags flags)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base = (void *)buf,
-        .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
-    };
-
-    if (nb_sectors < 0 || nb_sectors > INT_MAX / BDRV_SECTOR_SIZE) {
-        return -EINVAL;
-    }
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    return bdrv_prwv_co(bs, sector_num << BDRV_SECTOR_BITS,
-                        &qiov, is_write, flags);
-}
-
-/* return < 0 if error. See bdrv_write() for the return codes */
-int bdrv_read(BlockDriverState *bs, int64_t sector_num,
-              uint8_t *buf, int nb_sectors)
-{
-    return bdrv_rw_co(bs, sector_num, buf, nb_sectors, false, 0);
-}
-
-/* Just like bdrv_read(), but with I/O throttling temporarily disabled */
-int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
-                          uint8_t *buf, int nb_sectors)
-{
-    bool enabled;
-    int ret;
-
-    enabled = bs->io_limits_enabled;
-    bs->io_limits_enabled = false;
-    ret = bdrv_read(bs, sector_num, buf, nb_sectors);
-    bs->io_limits_enabled = enabled;
-    return ret;
-}
-
-/* Return < 0 if error. Important errors are:
-  -EIO         generic I/O error (may happen for all errors)
-  -ENOMEDIUM   No media inserted.
-  -EINVAL      Invalid sector number or nb_sectors
-  -EACCES      Trying to write a read-only device
-*/
-int bdrv_write(BlockDriverState *bs, int64_t sector_num,
-               const uint8_t *buf, int nb_sectors)
-{
-    return bdrv_rw_co(bs, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
-}
-
-int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-                      int nb_sectors, BdrvRequestFlags flags)
-{
-    return bdrv_rw_co(bs, sector_num, NULL, nb_sectors, true,
-                      BDRV_REQ_ZERO_WRITE | flags);
-}
-
-/*
- * Completely zero out a block device with the help of bdrv_write_zeroes.
- * The operation is sped up by checking the block status and only writing
- * zeroes to the device if they currently do not return zeroes. Optional
- * flags are passed through to bdrv_write_zeroes (e.g. BDRV_REQ_MAY_UNMAP).
- *
- * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
- */
-int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags)
-{
-    int64_t target_sectors, ret, nb_sectors, sector_num = 0;
-    int n;
-
-    target_sectors = bdrv_nb_sectors(bs);
-    if (target_sectors < 0) {
-        return target_sectors;
-    }
-
-    for (;;) {
-        nb_sectors = target_sectors - sector_num;
-        if (nb_sectors <= 0) {
-            return 0;
-        }
-        if (nb_sectors > INT_MAX / BDRV_SECTOR_SIZE) {
-            nb_sectors = INT_MAX / BDRV_SECTOR_SIZE;
-        }
-        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n);
-        if (ret < 0) {
-            error_report("error getting block status at sector %" PRId64 ": %s",
-                         sector_num, strerror(-ret));
-            return ret;
-        }
-        if (ret & BDRV_BLOCK_ZERO) {
-            sector_num += n;
-            continue;
-        }
-        ret = bdrv_write_zeroes(bs, sector_num, n, flags);
-        if (ret < 0) {
-            error_report("error writing zeroes at sector %" PRId64 ": %s",
-                         sector_num, strerror(-ret));
-            return ret;
-        }
-        sector_num += n;
-    }
-}
-
-int bdrv_pread(BlockDriverState *bs, int64_t offset, void *buf, int bytes)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base = (void *)buf,
-        .iov_len = bytes,
-    };
-    int ret;
-
-    if (bytes < 0) {
-        return -EINVAL;
-    }
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    ret = bdrv_prwv_co(bs, offset, &qiov, false, 0);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return bytes;
-}
-
-int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov)
-{
-    int ret;
-
-    ret = bdrv_prwv_co(bs, offset, qiov, true, 0);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return qiov->size;
-}
-
-int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
-                const void *buf, int bytes)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base   = (void *) buf,
-        .iov_len    = bytes,
-    };
-
-    if (bytes < 0) {
-        return -EINVAL;
-    }
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    return bdrv_pwritev(bs, offset, &qiov);
-}
-
-/*
- * Writes to the file and ensures that no writes are reordered across this
- * request (acts as a barrier)
- *
- * Returns 0 on success, -errno in error cases.
- */
-int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
-    const void *buf, int count)
-{
-    int ret;
-
-    ret = bdrv_pwrite(bs, offset, buf, count);
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* No flush needed for cache modes that already do it */
-    if (bs->enable_write_cache) {
-        bdrv_flush(bs);
-    }
-
-    return 0;
-}
-
-static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
-{
-    /* Perform I/O through a temporary buffer so that users who scribble over
-     * their read buffer while the operation is in progress do not end up
-     * modifying the image file.  This is critical for zero-copy guest I/O
-     * where anything might happen inside guest memory.
-     */
-    void *bounce_buffer;
-
-    BlockDriver *drv = bs->drv;
-    struct iovec iov;
-    QEMUIOVector bounce_qiov;
-    int64_t cluster_sector_num;
-    int cluster_nb_sectors;
-    size_t skip_bytes;
-    int ret;
-
-    /* Cover entire cluster so no additional backing file I/O is required when
-     * allocating cluster in the image file.
-     */
-    bdrv_round_to_clusters(bs, sector_num, nb_sectors,
-                           &cluster_sector_num, &cluster_nb_sectors);
-
-    trace_bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors,
-                                   cluster_sector_num, cluster_nb_sectors);
-
-    iov.iov_len = cluster_nb_sectors * BDRV_SECTOR_SIZE;
-    iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
-    if (bounce_buffer == NULL) {
-        ret = -ENOMEM;
-        goto err;
-    }
-
-    qemu_iovec_init_external(&bounce_qiov, &iov, 1);
-
-    ret = drv->bdrv_co_readv(bs, cluster_sector_num, cluster_nb_sectors,
-                             &bounce_qiov);
-    if (ret < 0) {
-        goto err;
-    }
-
-    if (drv->bdrv_co_write_zeroes &&
-        buffer_is_zero(bounce_buffer, iov.iov_len)) {
-        ret = bdrv_co_do_write_zeroes(bs, cluster_sector_num,
-                                      cluster_nb_sectors, 0);
-    } else {
-        /* This does not change the data on the disk, it is not necessary
-         * to flush even in cache=writethrough mode.
-         */
-        ret = drv->bdrv_co_writev(bs, cluster_sector_num, cluster_nb_sectors,
-                                  &bounce_qiov);
-    }
-
-    if (ret < 0) {
-        /* It might be okay to ignore write errors for guest requests.  If this
-         * is a deliberate copy-on-read then we don't want to ignore the error.
-         * Simply report it in all cases.
-         */
-        goto err;
-    }
-
-    skip_bytes = (sector_num - cluster_sector_num) * BDRV_SECTOR_SIZE;
-    qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes,
-                        nb_sectors * BDRV_SECTOR_SIZE);
-
-err:
-    qemu_vfree(bounce_buffer);
-    return ret;
-}
-
-/*
- * Forwards an already correctly aligned request to the BlockDriver. This
- * handles copy on read and zeroing after EOF; any other features must be
- * implemented by the caller.
- */
-static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
-    BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
-    int64_t align, QEMUIOVector *qiov, int flags)
-{
-    BlockDriver *drv = bs->drv;
-    int ret;
-
-    int64_t sector_num = offset >> BDRV_SECTOR_BITS;
-    unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
-
-    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
-    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
-    assert(!qiov || bytes == qiov->size);
-
-    /* Handle Copy on Read and associated serialisation */
-    if (flags & BDRV_REQ_COPY_ON_READ) {
-        /* If we touch the same cluster it counts as an overlap.  This
-         * guarantees that allocating writes will be serialized and not race
-         * with each other for the same cluster.  For example, in copy-on-read
-         * it ensures that the CoR read and write operations are atomic and
-         * guest writes cannot interleave between them. */
-        mark_request_serialising(req, bdrv_get_cluster_size(bs));
-    }
-
-    wait_serialising_requests(req);
-
-    if (flags & BDRV_REQ_COPY_ON_READ) {
-        int pnum;
-
-        ret = bdrv_is_allocated(bs, sector_num, nb_sectors, &pnum);
-        if (ret < 0) {
-            goto out;
-        }
-
-        if (!ret || pnum != nb_sectors) {
-            ret = bdrv_co_do_copy_on_readv(bs, sector_num, nb_sectors, qiov);
-            goto out;
-        }
-    }
-
-    /* Forward the request to the BlockDriver */
-    if (!(bs->zero_beyond_eof && bs->growable)) {
-        ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
-    } else {
-        /* Read zeros after EOF of growable BDSes */
-        int64_t total_sectors, max_nb_sectors;
-
-        total_sectors = bdrv_nb_sectors(bs);
-        if (total_sectors < 0) {
-            ret = total_sectors;
-            goto out;
-        }
-
-        max_nb_sectors = ROUND_UP(MAX(0, total_sectors - sector_num),
-                                  align >> BDRV_SECTOR_BITS);
-        if (max_nb_sectors > 0) {
-            QEMUIOVector local_qiov;
-            size_t local_sectors;
-
-            max_nb_sectors = MIN(max_nb_sectors, SIZE_MAX / BDRV_SECTOR_BITS);
-            local_sectors = MIN(max_nb_sectors, nb_sectors);
-
-            qemu_iovec_init(&local_qiov, qiov->niov);
-            qemu_iovec_concat(&local_qiov, qiov, 0,
-                              local_sectors * BDRV_SECTOR_SIZE);
-
-            ret = drv->bdrv_co_readv(bs, sector_num, local_sectors,
-                                     &local_qiov);
-
-            qemu_iovec_destroy(&local_qiov);
-        } else {
-            ret = 0;
-        }
-
-        /* Reading beyond end of file is supposed to produce zeroes */
-        if (ret == 0 && total_sectors < sector_num + nb_sectors) {
-            uint64_t offset = MAX(0, total_sectors - sector_num);
-            uint64_t bytes = (sector_num + nb_sectors - offset) *
-                              BDRV_SECTOR_SIZE;
-            qemu_iovec_memset(qiov, offset * BDRV_SECTOR_SIZE, 0, bytes);
-        }
-    }
-
-out:
-    return ret;
-}
-
-/*
- * Handle a read request in coroutine context
- */
-static int coroutine_fn bdrv_co_do_preadv(BlockDriverState *bs,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-    BdrvRequestFlags flags)
-{
-    BlockDriver *drv = bs->drv;
-    BdrvTrackedRequest req;
-
-    /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
-    uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
-    uint8_t *head_buf = NULL;
-    uint8_t *tail_buf = NULL;
-    QEMUIOVector local_qiov;
-    bool use_local_qiov = false;
-    int ret;
-
-    if (!drv) {
-        return -ENOMEDIUM;
-    }
-    if (bdrv_check_byte_request(bs, offset, bytes)) {
-        return -EIO;
-    }
-
-    if (bs->copy_on_read) {
-        flags |= BDRV_REQ_COPY_ON_READ;
-    }
-
-    /* throttling disk I/O */
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_intercept(bs, bytes, false);
-    }
-
-    /* Align read if necessary by padding qiov */
-    if (offset & (align - 1)) {
-        head_buf = qemu_blockalign(bs, align);
-        qemu_iovec_init(&local_qiov, qiov->niov + 2);
-        qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
-        qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
-        use_local_qiov = true;
-
-        bytes += offset & (align - 1);
-        offset = offset & ~(align - 1);
-    }
-
-    if ((offset + bytes) & (align - 1)) {
-        if (!use_local_qiov) {
-            qemu_iovec_init(&local_qiov, qiov->niov + 1);
-            qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
-            use_local_qiov = true;
-        }
-        tail_buf = qemu_blockalign(bs, align);
-        qemu_iovec_add(&local_qiov, tail_buf,
-                       align - ((offset + bytes) & (align - 1)));
-
-        bytes = ROUND_UP(bytes, align);
-    }
-
-    tracked_request_begin(&req, bs, offset, bytes, false);
-    ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
-                              use_local_qiov ? &local_qiov : qiov,
-                              flags);
-    tracked_request_end(&req);
-
-    if (use_local_qiov) {
-        qemu_iovec_destroy(&local_qiov);
-        qemu_vfree(head_buf);
-        qemu_vfree(tail_buf);
-    }
-
-    return ret;
-}
-
-static int coroutine_fn bdrv_co_do_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
-    BdrvRequestFlags flags)
-{
-    if (nb_sectors < 0 || nb_sectors > (UINT_MAX >> BDRV_SECTOR_BITS)) {
-        return -EINVAL;
-    }
-
-    return bdrv_co_do_preadv(bs, sector_num << BDRV_SECTOR_BITS,
-                             nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
-}
-
-int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov)
-{
-    trace_bdrv_co_readv(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov, 0);
-}
-
-int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
-{
-    trace_bdrv_co_copy_on_readv(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_readv(bs, sector_num, nb_sectors, qiov,
-                            BDRV_REQ_COPY_ON_READ);
-}
-
-/* if no limit is specified in the BlockLimits use a default
- * of 32768 512-byte sectors (16 MiB) per request.
- */
-#define MAX_WRITE_ZEROES_DEFAULT 32768
-
-static int coroutine_fn bdrv_co_do_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
-{
-    BlockDriver *drv = bs->drv;
-    QEMUIOVector qiov;
-    struct iovec iov = {0};
-    int ret = 0;
-
-    int max_write_zeroes = bs->bl.max_write_zeroes ?
-                           bs->bl.max_write_zeroes : MAX_WRITE_ZEROES_DEFAULT;
-
-    while (nb_sectors > 0 && !ret) {
-        int num = nb_sectors;
-
-        /* Align request.  Block drivers can expect the "bulk" of the request
-         * to be aligned.
-         */
-        if (bs->bl.write_zeroes_alignment
-            && num > bs->bl.write_zeroes_alignment) {
-            if (sector_num % bs->bl.write_zeroes_alignment != 0) {
-                /* Make a small request up to the first aligned sector.  */
-                num = bs->bl.write_zeroes_alignment;
-                num -= sector_num % bs->bl.write_zeroes_alignment;
-            } else if ((sector_num + num) % bs->bl.write_zeroes_alignment != 0) {
-                /* Shorten the request to the last aligned sector.  num cannot
-                 * underflow because num > bs->bl.write_zeroes_alignment.
-                 */
-                num -= (sector_num + num) % bs->bl.write_zeroes_alignment;
-            }
-        }
-
-        /* limit request size */
-        if (num > max_write_zeroes) {
-            num = max_write_zeroes;
-        }
-
-        ret = -ENOTSUP;
-        /* First try the efficient write zeroes operation */
-        if (drv->bdrv_co_write_zeroes) {
-            ret = drv->bdrv_co_write_zeroes(bs, sector_num, num, flags);
-        }
-
-        if (ret == -ENOTSUP) {
-            /* Fall back to bounce buffer if write zeroes is unsupported */
-            iov.iov_len = num * BDRV_SECTOR_SIZE;
-            if (iov.iov_base == NULL) {
-                iov.iov_base = qemu_try_blockalign(bs, num * BDRV_SECTOR_SIZE);
-                if (iov.iov_base == NULL) {
-                    ret = -ENOMEM;
-                    goto fail;
-                }
-                memset(iov.iov_base, 0, num * BDRV_SECTOR_SIZE);
-            }
-            qemu_iovec_init_external(&qiov, &iov, 1);
-
-            ret = drv->bdrv_co_writev(bs, sector_num, num, &qiov);
-
-            /* Keep bounce buffer around if it is big enough for all
-             * all future requests.
-             */
-            if (num < max_write_zeroes) {
-                qemu_vfree(iov.iov_base);
-                iov.iov_base = NULL;
-            }
-        }
-
-        sector_num += num;
-        nb_sectors -= num;
-    }
-
-fail:
-    qemu_vfree(iov.iov_base);
-    return ret;
-}
-
-/*
- * Forwards an already correctly aligned write request to the BlockDriver.
- */
-static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
-    BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
-    QEMUIOVector *qiov, int flags)
-{
-    BlockDriver *drv = bs->drv;
-    bool waited;
-    int ret;
-
-    int64_t sector_num = offset >> BDRV_SECTOR_BITS;
-    unsigned int nb_sectors = bytes >> BDRV_SECTOR_BITS;
-
-    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
-    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
-    assert(!qiov || bytes == qiov->size);
-
-    waited = wait_serialising_requests(req);
-    assert(!waited || !req->serialising);
-    assert(req->overlap_offset <= offset);
-    assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
-    (void)waited;
-
-    ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
-
-    if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
-        !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
-        qemu_iovec_is_zero(qiov)) {
-        flags |= BDRV_REQ_ZERO_WRITE;
-        if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
-            flags |= BDRV_REQ_MAY_UNMAP;
-        }
-    }
-
-    if (ret < 0) {
-        /* Do nothing, write notifier decided to fail this request */
-    } else if (flags & BDRV_REQ_ZERO_WRITE) {
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_ZERO);
-        ret = bdrv_co_do_write_zeroes(bs, sector_num, nb_sectors, flags);
-    } else {
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV);
-        ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
-    }
-    BLKDBG_EVENT(bs, BLKDBG_PWRITEV_DONE);
-
-    if (ret == 0 && !bs->enable_write_cache) {
-        ret = bdrv_co_flush(bs);
-    }
-
-    bdrv_set_dirty(bs, sector_num, nb_sectors);
-
-    block_acct_highest_sector(&bs->stats, sector_num, nb_sectors);
-
-    if (bs->growable && ret >= 0) {
-        bs->total_sectors = MAX(bs->total_sectors, sector_num + nb_sectors);
-    }
-
-    return ret;
-}
-
-/*
- * Handle a write request in coroutine context
- */
-static int coroutine_fn bdrv_co_do_pwritev(BlockDriverState *bs,
-    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
-    BdrvRequestFlags flags)
-{
-    BdrvTrackedRequest req;
-    /* TODO Lift BDRV_SECTOR_SIZE restriction in BlockDriver interface */
-    uint64_t align = MAX(BDRV_SECTOR_SIZE, bs->request_alignment);
-    uint8_t *head_buf = NULL;
-    uint8_t *tail_buf = NULL;
-    QEMUIOVector local_qiov;
-    bool use_local_qiov = false;
-    int ret;
-
-    if (!bs->drv) {
-        return -ENOMEDIUM;
-    }
-    if (bs->read_only) {
-        return -EACCES;
-    }
-    if (bdrv_check_byte_request(bs, offset, bytes)) {
-        return -EIO;
-    }
-
-    /* throttling disk I/O */
-    if (bs->io_limits_enabled) {
-        bdrv_io_limits_intercept(bs, bytes, true);
-    }
-
-    /*
-     * Align write if necessary by performing a read-modify-write cycle.
-     * Pad qiov with the read parts and be sure to have a tracked request not
-     * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
-     */
-    tracked_request_begin(&req, bs, offset, bytes, true);
-
-    if (offset & (align - 1)) {
-        QEMUIOVector head_qiov;
-        struct iovec head_iov;
-
-        mark_request_serialising(&req, align);
-        wait_serialising_requests(&req);
-
-        head_buf = qemu_blockalign(bs, align);
-        head_iov = (struct iovec) {
-            .iov_base   = head_buf,
-            .iov_len    = align,
-        };
-        qemu_iovec_init_external(&head_qiov, &head_iov, 1);
-
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_HEAD);
-        ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
-                                  align, &head_qiov, 0);
-        if (ret < 0) {
-            goto fail;
-        }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
-
-        qemu_iovec_init(&local_qiov, qiov->niov + 2);
-        qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
-        qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
-        use_local_qiov = true;
-
-        bytes += offset & (align - 1);
-        offset = offset & ~(align - 1);
-    }
-
-    if ((offset + bytes) & (align - 1)) {
-        QEMUIOVector tail_qiov;
-        struct iovec tail_iov;
-        size_t tail_bytes;
-        bool waited;
-
-        mark_request_serialising(&req, align);
-        waited = wait_serialising_requests(&req);
-        assert(!waited || !use_local_qiov);
-        (void)waited;
-
-        tail_buf = qemu_blockalign(bs, align);
-        tail_iov = (struct iovec) {
-            .iov_base   = tail_buf,
-            .iov_len    = align,
-        };
-        qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
-
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_TAIL);
-        ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
-                                  align, &tail_qiov, 0);
-        if (ret < 0) {
-            goto fail;
-        }
-        BLKDBG_EVENT(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
-
-        if (!use_local_qiov) {
-            qemu_iovec_init(&local_qiov, qiov->niov + 1);
-            qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
-            use_local_qiov = true;
-        }
-
-        tail_bytes = (offset + bytes) & (align - 1);
-        qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
-
-        bytes = ROUND_UP(bytes, align);
-    }
-
-    ret = bdrv_aligned_pwritev(bs, &req, offset, bytes,
-                               use_local_qiov ? &local_qiov : qiov,
-                               flags);
-
-fail:
-    tracked_request_end(&req);
-
-    if (use_local_qiov) {
-        qemu_iovec_destroy(&local_qiov);
-    }
-    qemu_vfree(head_buf);
-    qemu_vfree(tail_buf);
-
-    return ret;
-}
-
-static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
-    BdrvRequestFlags flags)
-{
-    if (nb_sectors < 0 || nb_sectors > (INT_MAX >> BDRV_SECTOR_BITS)) {
-        return -EINVAL;
-    }
-
-    return bdrv_co_do_pwritev(bs, sector_num << BDRV_SECTOR_BITS,
-                              nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
-}
-
-int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov)
-{
-    trace_bdrv_co_writev(bs, sector_num, nb_sectors);
-
-    return bdrv_co_do_writev(bs, sector_num, nb_sectors, qiov, 0);
-}
-
-int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs,
-                                      int64_t sector_num, int nb_sectors,
-                                      BdrvRequestFlags flags)
-{
-    trace_bdrv_co_write_zeroes(bs, sector_num, nb_sectors, flags);
-
-    if (!(bs->open_flags & BDRV_O_UNMAP)) {
-        flags &= ~BDRV_REQ_MAY_UNMAP;
-    }
-
-    return bdrv_co_do_writev(bs, sector_num, nb_sectors, NULL,
-                             BDRV_REQ_ZERO_WRITE | flags);
-}
-
 /**
  * Truncate file to 'offset' bytes (needed only for file protocols)
  */
@@ -3470,9 +2473,9 @@
     ret = drv->bdrv_truncate(bs, offset);
     if (ret == 0) {
         ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
-        if (bs->blk) {
-            blk_dev_resize_cb(bs->blk);
-        }
+        bdrv_dirty_bitmap_truncate(bs);
+        bdrv_parent_cb_resize(bs);
+        ++bs->write_gen;
     }
     return ret;
 }
@@ -3491,7 +2494,7 @@
         return drv->bdrv_get_allocated_file_size(bs);
     }
     if (bs->file) {
-        return bdrv_get_allocated_file_size(bs->file);
+        return bdrv_get_allocated_file_size(bs->file->bs);
     }
     return -ENOTSUP;
 }
@@ -3523,6 +2526,7 @@
 {
     int64_t ret = bdrv_nb_sectors(bs);
 
+    ret = ret > INT64_MAX / BDRV_SECTOR_SIZE ? -EFBIG : ret;
     return ret < 0 ? ret : ret * BDRV_SECTOR_SIZE;
 }
 
@@ -3534,130 +2538,39 @@
     *nb_sectors_ptr = nb_sectors < 0 ? 0 : nb_sectors;
 }
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error)
-{
-    bs->on_read_error = on_read_error;
-    bs->on_write_error = on_write_error;
-}
-
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
-{
-    return is_read ? bs->on_read_error : bs->on_write_error;
-}
-
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
-{
-    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
-
-    switch (on_err) {
-    case BLOCKDEV_ON_ERROR_ENOSPC:
-        return (error == ENOSPC) ?
-               BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_STOP:
-        return BLOCK_ERROR_ACTION_STOP;
-    case BLOCKDEV_ON_ERROR_REPORT:
-        return BLOCK_ERROR_ACTION_REPORT;
-    case BLOCKDEV_ON_ERROR_IGNORE:
-        return BLOCK_ERROR_ACTION_IGNORE;
-    default:
-        abort();
-    }
-}
-
-static void send_qmp_error_event(BlockDriverState *bs,
-                                 BlockErrorAction action,
-                                 bool is_read, int error)
-{
-    IoOperationType optype;
-
-    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
-    qapi_event_send_block_io_error(bdrv_get_device_name(bs), optype, action,
-                                   bdrv_iostatus_is_enabled(bs),
-                                   error == ENOSPC, strerror(error),
-                                   &error_abort);
-}
-
-/* This is done by device models because, while the block layer knows
- * about the error, it does not know whether an operation comes from
- * the device or the block layer (from a job, for example).
- */
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error)
-{
-    assert(error >= 0);
-
-    if (action == BLOCK_ERROR_ACTION_STOP) {
-        /* First set the iostatus, so that "info block" returns an iostatus
-         * that matches the events raised so far (an additional error iostatus
-         * is fine, but not a lost one).
-         */
-        bdrv_iostatus_set_err(bs, error);
-
-        /* Then raise the request to stop the VM and the event.
-         * qemu_system_vmstop_request_prepare has two effects.  First,
-         * it ensures that the STOP event always comes after the
-         * BLOCK_IO_ERROR event.  Second, it ensures that even if management
-         * can observe the STOP event and do a "cont" before the STOP
-         * event is issued, the VM will not stop.  In this case, vm_start()
-         * also ensures that the STOP/RESUME pair of events is emitted.
-         */
-        qemu_system_vmstop_request_prepare();
-        send_qmp_error_event(bs, action, is_read, error);
-        qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
-    } else {
-        send_qmp_error_event(bs, action, is_read, error);
-    }
-}
-
-int bdrv_is_read_only(BlockDriverState *bs)
+bool bdrv_is_read_only(BlockDriverState *bs)
 {
     return bs->read_only;
 }
 
-int bdrv_is_sg(BlockDriverState *bs)
+bool bdrv_is_sg(BlockDriverState *bs)
 {
     return bs->sg;
 }
 
-int bdrv_enable_write_cache(BlockDriverState *bs)
+bool bdrv_is_encrypted(BlockDriverState *bs)
 {
-    return bs->enable_write_cache;
-}
-
-void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
-{
-    bs->enable_write_cache = wce;
-
-    /* so a reopen() will preserve wce */
-    if (wce) {
-        bs->open_flags |= BDRV_O_CACHE_WB;
-    } else {
-        bs->open_flags &= ~BDRV_O_CACHE_WB;
+    if (bs->backing && bs->backing->bs->encrypted) {
+        return true;
     }
-}
-
-int bdrv_is_encrypted(BlockDriverState *bs)
-{
-    if (bs->backing_hd && bs->backing_hd->encrypted)
-        return 1;
     return bs->encrypted;
 }
 
-int bdrv_key_required(BlockDriverState *bs)
+bool bdrv_key_required(BlockDriverState *bs)
 {
-    BlockDriverState *backing_hd = bs->backing_hd;
+    BdrvChild *backing = bs->backing;
 
-    if (backing_hd && backing_hd->encrypted && !backing_hd->valid_key)
-        return 1;
+    if (backing && backing->bs->encrypted && !backing->bs->valid_key) {
+        return true;
+    }
     return (bs->encrypted && !bs->valid_key);
 }
 
 int bdrv_set_key(BlockDriverState *bs, const char *key)
 {
     int ret;
-    if (bs->backing_hd && bs->backing_hd->encrypted) {
-        ret = bdrv_set_key(bs->backing_hd, key);
+    if (bs->backing && bs->backing->bs->encrypted) {
+        ret = bdrv_set_key(bs->backing->bs, key);
         if (ret < 0)
             return ret;
         if (!bs->encrypted)
@@ -3670,17 +2583,45 @@
     }
     ret = bs->drv->bdrv_set_key(bs, key);
     if (ret < 0) {
-        bs->valid_key = 0;
+        bs->valid_key = false;
     } else if (!bs->valid_key) {
-        bs->valid_key = 1;
-        if (bs->blk) {
-            /* call the change callback now, we skipped it on open */
-            blk_dev_change_media_cb(bs->blk, true);
-        }
+        /* call the change callback now, we skipped it on open */
+        bs->valid_key = true;
+        bdrv_parent_cb_change_media(bs, true);
     }
     return ret;
 }
 
+/*
+ * Provide an encryption key for @bs.
+ * If @key is non-null:
+ *     If @bs is not encrypted, fail.
+ *     Else if the key is invalid, fail.
+ *     Else set @bs's key to @key, replacing the existing key, if any.
+ * If @key is null:
+ *     If @bs is encrypted and still lacks a key, fail.
+ *     Else do nothing.
+ * On failure, store an error object through @errp if non-null.
+ */
+void bdrv_add_key(BlockDriverState *bs, const char *key, Error **errp)
+{
+    if (key) {
+        if (!bdrv_is_encrypted(bs)) {
+            error_setg(errp, "Node '%s' is not encrypted",
+                      bdrv_get_device_or_node_name(bs));
+        } else if (bdrv_set_key(bs, key) < 0) {
+            error_setg(errp, QERR_INVALID_PASSWORD);
+        }
+    } else {
+        if (bdrv_key_required(bs)) {
+            error_set(errp, ERROR_CLASS_DEVICE_ENCRYPTED,
+                      "'%s' (%s) is encrypted",
+                      bdrv_get_device_or_node_name(bs),
+                      bdrv_get_encrypted_filename(bs));
+        }
+    }
+}
+
 const char *bdrv_get_format_name(BlockDriverState *bs)
 {
     return bs->drv ? bs->drv->format_name : NULL;
@@ -3723,15 +2664,6 @@
     g_free(formats);
 }
 
-/* This function is to find block backend bs */
-/* TODO convert callers to blk_by_name(), then remove */
-BlockDriverState *bdrv_find(const char *name)
-{
-    BlockBackend *blk = blk_by_name(name);
-
-    return blk ? blk_bs(blk) : NULL;
-}
-
 /* This function is to find a node in the bs graph */
 BlockDriverState *bdrv_find_node(const char *node_name)
 {
@@ -3748,15 +2680,20 @@
 }
 
 /* Put this QMP function here so it can access the static graph_bdrv_states. */
-BlockDeviceInfoList *bdrv_named_nodes_list(void)
+BlockDeviceInfoList *bdrv_named_nodes_list(Error **errp)
 {
     BlockDeviceInfoList *list, *entry;
     BlockDriverState *bs;
 
     list = NULL;
     QTAILQ_FOREACH(bs, &graph_bdrv_states, node_list) {
+        BlockDeviceInfo *info = bdrv_block_device_info(NULL, bs, errp);
+        if (!info) {
+            qapi_free_BlockDeviceInfoList(list);
+            return NULL;
+        }
         entry = g_malloc0(sizeof(*entry));
-        entry->value = bdrv_block_device_info(bs);
+        entry->value = info;
         entry->next = list;
         list = entry;
     }
@@ -3775,7 +2712,12 @@
         blk = blk_by_name(device);
 
         if (blk) {
-            return blk_bs(blk);
+            bs = blk_bs(blk);
+            if (!bs) {
+                error_setg(errp, "Device '%s' has no medium", device);
+            }
+
+            return bs;
         }
     }
 
@@ -3798,24 +2740,56 @@
 bool bdrv_chain_contains(BlockDriverState *top, BlockDriverState *base)
 {
     while (top && top != base) {
-        top = top->backing_hd;
+        top = backing_bs(top);
     }
 
     return top != NULL;
 }
 
-BlockDriverState *bdrv_next(BlockDriverState *bs)
+BlockDriverState *bdrv_next_node(BlockDriverState *bs)
 {
     if (!bs) {
-        return QTAILQ_FIRST(&bdrv_states);
+        return QTAILQ_FIRST(&graph_bdrv_states);
     }
-    return QTAILQ_NEXT(bs, device_list);
+    return QTAILQ_NEXT(bs, node_list);
+}
+
+const char *bdrv_get_node_name(const BlockDriverState *bs)
+{
+    return bs->node_name;
+}
+
+const char *bdrv_get_parent_name(const BlockDriverState *bs)
+{
+    BdrvChild *c;
+    const char *name;
+
+    /* If multiple parents have a name, just pick the first one. */
+    QLIST_FOREACH(c, &bs->parents, next_parent) {
+        if (c->role->get_name) {
+            name = c->role->get_name(c);
+            if (name && *name) {
+                return name;
+            }
+        }
+    }
+
+    return NULL;
 }
 
 /* TODO check what callers really want: bs->node_name or blk_name() */
 const char *bdrv_get_device_name(const BlockDriverState *bs)
 {
-    return bs->blk ? blk_name(bs->blk) : "";
+    return bdrv_get_parent_name(bs) ?: "";
+}
+
+/* This can be used to identify nodes that might not have a device
+ * name associated. Since node and device names live in the same
+ * namespace, the result is unambiguous. The exception is if both are
+ * absent, then this returns an empty (non-null) string. */
+const char *bdrv_get_device_or_node_name(const BlockDriverState *bs)
+{
+    return bdrv_get_parent_name(bs) ?: bs->node_name;
 }
 
 int bdrv_get_flags(BlockDriverState *bs)
@@ -3823,26 +2797,6 @@
     return bs->open_flags;
 }
 
-int bdrv_flush_all(void)
-{
-    BlockDriverState *bs;
-    int result = 0;
-
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-        int ret;
-
-        aio_context_acquire(aio_context);
-        ret = bdrv_flush(bs);
-        if (ret < 0 && !result) {
-            result = ret;
-        }
-        aio_context_release(aio_context);
-    }
-
-    return result;
-}
-
 int bdrv_has_zero_init_1(BlockDriverState *bs)
 {
     return 1;
@@ -3854,7 +2808,7 @@
 
     /* If BS is a copy on write image, it is initialized to
        the contents of the base image, which may not be zeroes.  */
-    if (bs->backing_hd) {
+    if (bs->backing) {
         return 0;
     }
     if (bs->drv->bdrv_has_zero_init) {
@@ -3869,7 +2823,7 @@
 {
     BlockDriverInfo bdi;
 
-    if (bs->backing_hd) {
+    if (bs->backing) {
         return false;
     }
 
@@ -3884,7 +2838,7 @@
 {
     BlockDriverInfo bdi;
 
-    if (bs->backing_hd || !(bs->open_flags & BDRV_O_UNMAP)) {
+    if (!(bs->open_flags & BDRV_O_UNMAP)) {
         return false;
     }
 
@@ -3895,225 +2849,9 @@
     return false;
 }
 
-typedef struct BdrvCoGetBlockStatusData {
-    BlockDriverState *bs;
-    BlockDriverState *base;
-    int64_t sector_num;
-    int nb_sectors;
-    int *pnum;
-    int64_t ret;
-    bool done;
-} BdrvCoGetBlockStatusData;
-
-/*
- * Returns the allocation status of the specified sectors.
- * Drivers not implementing the functionality are assumed to not support
- * backing files, hence all their sectors are reported as allocated.
- *
- * If 'sector_num' is beyond the end of the disk image the return value is 0
- * and 'pnum' is set to 0.
- *
- * 'pnum' is set to the number of sectors (including and immediately following
- * the specified sector) that are known to be in the same
- * allocated/unallocated state.
- *
- * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
- * beyond the end of the disk image it will be clamped.
- */
-static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
-                                                     int64_t sector_num,
-                                                     int nb_sectors, int *pnum)
-{
-    int64_t total_sectors;
-    int64_t n;
-    int64_t ret, ret2;
-
-    total_sectors = bdrv_nb_sectors(bs);
-    if (total_sectors < 0) {
-        return total_sectors;
-    }
-
-    if (sector_num >= total_sectors) {
-        *pnum = 0;
-        return 0;
-    }
-
-    n = total_sectors - sector_num;
-    if (n < nb_sectors) {
-        nb_sectors = n;
-    }
-
-    if (!bs->drv->bdrv_co_get_block_status) {
-        *pnum = nb_sectors;
-        ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
-        if (bs->drv->protocol_name) {
-            ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
-        }
-        return ret;
-    }
-
-    ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum);
-    if (ret < 0) {
-        *pnum = 0;
-        return ret;
-    }
-
-    if (ret & BDRV_BLOCK_RAW) {
-        assert(ret & BDRV_BLOCK_OFFSET_VALID);
-        return bdrv_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
-                                     *pnum, pnum);
-    }
-
-    if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
-        ret |= BDRV_BLOCK_ALLOCATED;
-    }
-
-    if (!(ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO)) {
-        if (bdrv_unallocated_blocks_are_zero(bs)) {
-            ret |= BDRV_BLOCK_ZERO;
-        } else if (bs->backing_hd) {
-            BlockDriverState *bs2 = bs->backing_hd;
-            int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
-            if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
-                ret |= BDRV_BLOCK_ZERO;
-            }
-        }
-    }
-
-    if (bs->file &&
-        (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
-        (ret & BDRV_BLOCK_OFFSET_VALID)) {
-        int file_pnum;
-
-        ret2 = bdrv_co_get_block_status(bs->file, ret >> BDRV_SECTOR_BITS,
-                                        *pnum, &file_pnum);
-        if (ret2 >= 0) {
-            /* Ignore errors.  This is just providing extra information, it
-             * is useful but not necessary.
-             */
-            if (!file_pnum) {
-                /* !file_pnum indicates an offset at or beyond the EOF; it is
-                 * perfectly valid for the format block driver to point to such
-                 * offsets, so catch it and mark everything as zero */
-                ret |= BDRV_BLOCK_ZERO;
-            } else {
-                /* Limit request to the range reported by the protocol driver */
-                *pnum = file_pnum;
-                ret |= (ret2 & BDRV_BLOCK_ZERO);
-            }
-        }
-    }
-
-    return ret;
-}
-
-/* Coroutine wrapper for bdrv_get_block_status() */
-static void coroutine_fn bdrv_get_block_status_co_entry(void *opaque)
-{
-    BdrvCoGetBlockStatusData *data = opaque;
-    BlockDriverState *bs = data->bs;
-
-    data->ret = bdrv_co_get_block_status(bs, data->sector_num, data->nb_sectors,
-                                         data->pnum);
-    data->done = true;
-}
-
-/*
- * Synchronous wrapper around bdrv_co_get_block_status().
- *
- * See bdrv_co_get_block_status() for details.
- */
-int64_t bdrv_get_block_status(BlockDriverState *bs, int64_t sector_num,
-                              int nb_sectors, int *pnum)
-{
-    Coroutine *co;
-    BdrvCoGetBlockStatusData data = {
-        .bs = bs,
-        .sector_num = sector_num,
-        .nb_sectors = nb_sectors,
-        .pnum = pnum,
-        .done = false,
-    };
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_get_block_status_co_entry(&data);
-    } else {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-
-        co = qemu_coroutine_create(bdrv_get_block_status_co_entry);
-        qemu_coroutine_enter(co, &data);
-        while (!data.done) {
-            aio_poll(aio_context, true);
-        }
-    }
-    return data.ret;
-}
-
-int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
-                                   int nb_sectors, int *pnum)
-{
-    int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum);
-    if (ret < 0) {
-        return ret;
-    }
-    return !!(ret & BDRV_BLOCK_ALLOCATED);
-}
-
-/*
- * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
- *
- * Return true if the given sector is allocated in any image between
- * BASE and TOP (inclusive).  BASE can be NULL to check if the given
- * sector is allocated in any image of the chain.  Return false otherwise.
- *
- * 'pnum' is set to the number of sectors (including and immediately following
- *  the specified sector) that are known to be in the same
- *  allocated/unallocated state.
- *
- */
-int bdrv_is_allocated_above(BlockDriverState *top,
-                            BlockDriverState *base,
-                            int64_t sector_num,
-                            int nb_sectors, int *pnum)
-{
-    BlockDriverState *intermediate;
-    int ret, n = nb_sectors;
-
-    intermediate = top;
-    while (intermediate && intermediate != base) {
-        int pnum_inter;
-        ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
-                                &pnum_inter);
-        if (ret < 0) {
-            return ret;
-        } else if (ret) {
-            *pnum = pnum_inter;
-            return 1;
-        }
-
-        /*
-         * [sector_num, nb_sectors] is unallocated on top but intermediate
-         * might have
-         *
-         * [sector_num+x, nr_sectors] allocated.
-         */
-        if (n > pnum_inter &&
-            (intermediate == top ||
-             sector_num + pnum_inter < intermediate->total_sectors)) {
-            n = pnum_inter;
-        }
-
-        intermediate = intermediate->backing_hd;
-    }
-
-    *pnum = n;
-    return 0;
-}
-
 const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
 {
-    if (bs->backing_hd && bs->backing_hd->encrypted)
+    if (bs->backing && bs->backing->bs->encrypted)
         return bs->backing_file;
     else if (bs->encrypted)
         return bs->filename;
@@ -4127,22 +2865,6 @@
     pstrcpy(filename, filename_size, bs->backing_file);
 }
 
-int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
-                          const uint8_t *buf, int nb_sectors)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv)
-        return -ENOMEDIUM;
-    if (!drv->bdrv_write_compressed)
-        return -ENOTSUP;
-    if (bdrv_check_request(bs, sector_num, nb_sectors))
-        return -EIO;
-
-    assert(QLIST_EMPTY(&bs->dirty_bitmaps));
-
-    return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
-}
-
 int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
     BlockDriver *drv = bs->drv;
@@ -4163,48 +2885,7 @@
     return NULL;
 }
 
-int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
-                      int64_t pos, int size)
-{
-    QEMUIOVector qiov;
-    struct iovec iov = {
-        .iov_base   = (void *) buf,
-        .iov_len    = size,
-    };
-
-    qemu_iovec_init_external(&qiov, &iov, 1);
-    return bdrv_writev_vmstate(bs, &qiov, pos);
-}
-
-int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (!drv) {
-        return -ENOMEDIUM;
-    } else if (drv->bdrv_save_vmstate) {
-        return drv->bdrv_save_vmstate(bs, qiov, pos);
-    } else if (bs->file) {
-        return bdrv_writev_vmstate(bs->file, qiov, pos);
-    }
-
-    return -ENOTSUP;
-}
-
-int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
-                      int64_t pos, int size)
-{
-    BlockDriver *drv = bs->drv;
-    if (!drv)
-        return -ENOMEDIUM;
-    if (drv->bdrv_load_vmstate)
-        return drv->bdrv_load_vmstate(bs, buf, pos, size);
-    if (bs->file)
-        return bdrv_load_vmstate(bs->file, buf, pos, size);
-    return -ENOTSUP;
-}
-
-void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event)
+void bdrv_debug_event(BlockDriverState *bs, BlkdebugEvent event)
 {
     if (!bs || !bs->drv || !bs->drv->bdrv_debug_event) {
         return;
@@ -4217,7 +2898,7 @@
                           const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_breakpoint) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_breakpoint) {
@@ -4230,7 +2911,7 @@
 int bdrv_debug_remove_breakpoint(BlockDriverState *bs, const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_remove_breakpoint) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_remove_breakpoint) {
@@ -4243,7 +2924,7 @@
 int bdrv_debug_resume(BlockDriverState *bs, const char *tag)
 {
     while (bs && (!bs->drv || !bs->drv->bdrv_debug_resume)) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_resume) {
@@ -4256,7 +2937,7 @@
 bool bdrv_debug_is_suspended(BlockDriverState *bs, const char *tag)
 {
     while (bs && bs->drv && !bs->drv->bdrv_debug_is_suspended) {
-        bs = bs->file;
+        bs = bs->file ? bs->file->bs : NULL;
     }
 
     if (bs && bs->drv && bs->drv->bdrv_debug_is_suspended) {
@@ -4295,13 +2976,13 @@
 
     is_protocol = path_has_protocol(backing_file);
 
-    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
+    for (curr_bs = bs; curr_bs->backing; curr_bs = curr_bs->backing->bs) {
 
         /* If either of the filename paths is actually a protocol, then
          * compare unmodified paths; otherwise make paths relative */
         if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
             if (strcmp(backing_file, curr_bs->backing_file) == 0) {
-                retval = curr_bs->backing_hd;
+                retval = curr_bs->backing->bs;
                 break;
             }
         } else {
@@ -4325,7 +3006,7 @@
             }
 
             if (strcmp(backing_file_full, filename_full) == 0) {
-                retval = curr_bs->backing_hd;
+                retval = curr_bs->backing->bs;
                 break;
             }
         }
@@ -4343,455 +3024,11 @@
         return 0;
     }
 
-    if (!bs->backing_hd) {
+    if (!bs->backing) {
         return 0;
     }
 
-    return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
-}
-
-/**************************************************************/
-/* async I/Os */
-
-BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
-                           QEMUIOVector *qiov, int nb_sectors,
-                           BlockCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_readv(bs, sector_num, nb_sectors, opaque);
-
-    return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
-                                 cb, opaque, false);
-}
-
-BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
-                            QEMUIOVector *qiov, int nb_sectors,
-                            BlockCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_writev(bs, sector_num, nb_sectors, opaque);
-
-    return bdrv_co_aio_rw_vector(bs, sector_num, qiov, nb_sectors, 0,
-                                 cb, opaque, true);
-}
-
-BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, BdrvRequestFlags flags,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_write_zeroes(bs, sector_num, nb_sectors, flags, opaque);
-
-    return bdrv_co_aio_rw_vector(bs, sector_num, NULL, nb_sectors,
-                                 BDRV_REQ_ZERO_WRITE | flags,
-                                 cb, opaque, true);
-}
-
-
-typedef struct MultiwriteCB {
-    int error;
-    int num_requests;
-    int num_callbacks;
-    struct {
-        BlockCompletionFunc *cb;
-        void *opaque;
-        QEMUIOVector *free_qiov;
-    } callbacks[];
-} MultiwriteCB;
-
-static void multiwrite_user_cb(MultiwriteCB *mcb)
-{
-    int i;
-
-    for (i = 0; i < mcb->num_callbacks; i++) {
-        mcb->callbacks[i].cb(mcb->callbacks[i].opaque, mcb->error);
-        if (mcb->callbacks[i].free_qiov) {
-            qemu_iovec_destroy(mcb->callbacks[i].free_qiov);
-        }
-        g_free(mcb->callbacks[i].free_qiov);
-    }
-}
-
-static void multiwrite_cb(void *opaque, int ret)
-{
-    MultiwriteCB *mcb = opaque;
-
-    trace_multiwrite_cb(mcb, ret);
-
-    if (ret < 0 && !mcb->error) {
-        mcb->error = ret;
-    }
-
-    mcb->num_requests--;
-    if (mcb->num_requests == 0) {
-        multiwrite_user_cb(mcb);
-        g_free(mcb);
-    }
-}
-
-static int multiwrite_req_compare(const void *a, const void *b)
-{
-    const BlockRequest *req1 = a, *req2 = b;
-
-    /*
-     * Note that we can't simply subtract req2->sector from req1->sector
-     * here as that could overflow the return value.
-     */
-    if (req1->sector > req2->sector) {
-        return 1;
-    } else if (req1->sector < req2->sector) {
-        return -1;
-    } else {
-        return 0;
-    }
-}
-
-/*
- * Takes a bunch of requests and tries to merge them. Returns the number of
- * requests that remain after merging.
- */
-static int multiwrite_merge(BlockDriverState *bs, BlockRequest *reqs,
-    int num_reqs, MultiwriteCB *mcb)
-{
-    int i, outidx;
-
-    // Sort requests by start sector
-    qsort(reqs, num_reqs, sizeof(*reqs), &multiwrite_req_compare);
-
-    // Check if adjacent requests touch the same clusters. If so, combine them,
-    // filling up gaps with zero sectors.
-    outidx = 0;
-    for (i = 1; i < num_reqs; i++) {
-        int merge = 0;
-        int64_t oldreq_last = reqs[outidx].sector + reqs[outidx].nb_sectors;
-
-        // Handle exactly sequential writes and overlapping writes.
-        if (reqs[i].sector <= oldreq_last) {
-            merge = 1;
-        }
-
-        if (reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1 > IOV_MAX) {
-            merge = 0;
-        }
-
-        if (bs->bl.max_transfer_length && reqs[outidx].nb_sectors +
-            reqs[i].nb_sectors > bs->bl.max_transfer_length) {
-            merge = 0;
-        }
-
-        if (merge) {
-            size_t size;
-            QEMUIOVector *qiov = g_malloc0(sizeof(*qiov));
-            qemu_iovec_init(qiov,
-                reqs[outidx].qiov->niov + reqs[i].qiov->niov + 1);
-
-            // Add the first request to the merged one. If the requests are
-            // overlapping, drop the last sectors of the first request.
-            size = (reqs[i].sector - reqs[outidx].sector) << 9;
-            qemu_iovec_concat(qiov, reqs[outidx].qiov, 0, size);
-
-            // We should need to add any zeros between the two requests
-            assert (reqs[i].sector <= oldreq_last);
-
-            // Add the second request
-            qemu_iovec_concat(qiov, reqs[i].qiov, 0, reqs[i].qiov->size);
-
-            // Add tail of first request, if necessary
-            if (qiov->size < reqs[outidx].qiov->size) {
-                qemu_iovec_concat(qiov, reqs[outidx].qiov, qiov->size,
-                                  reqs[outidx].qiov->size - qiov->size);
-            }
-
-            reqs[outidx].nb_sectors = qiov->size >> 9;
-            reqs[outidx].qiov = qiov;
-
-            mcb->callbacks[i].free_qiov = reqs[outidx].qiov;
-        } else {
-            outidx++;
-            reqs[outidx].sector     = reqs[i].sector;
-            reqs[outidx].nb_sectors = reqs[i].nb_sectors;
-            reqs[outidx].qiov       = reqs[i].qiov;
-        }
-    }
-
-    return outidx + 1;
-}
-
-/*
- * Submit multiple AIO write requests at once.
- *
- * On success, the function returns 0 and all requests in the reqs array have
- * been submitted. In error case this function returns -1, and any of the
- * requests may or may not be submitted yet. In particular, this means that the
- * callback will be called for some of the requests, for others it won't. The
- * caller must check the error field of the BlockRequest to wait for the right
- * callbacks (if error != 0, no callback will be called).
- *
- * The implementation may modify the contents of the reqs array, e.g. to merge
- * requests. However, the fields opaque and error are left unmodified as they
- * are used to signal failure for a single request to the caller.
- */
-int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
-{
-    MultiwriteCB *mcb;
-    int i;
-
-    /* don't submit writes if we don't have a medium */
-    if (bs->drv == NULL) {
-        for (i = 0; i < num_reqs; i++) {
-            reqs[i].error = -ENOMEDIUM;
-        }
-        return -1;
-    }
-
-    if (num_reqs == 0) {
-        return 0;
-    }
-
-    // Create MultiwriteCB structure
-    mcb = g_malloc0(sizeof(*mcb) + num_reqs * sizeof(*mcb->callbacks));
-    mcb->num_requests = 0;
-    mcb->num_callbacks = num_reqs;
-
-    for (i = 0; i < num_reqs; i++) {
-        mcb->callbacks[i].cb = reqs[i].cb;
-        mcb->callbacks[i].opaque = reqs[i].opaque;
-    }
-
-    // Check for mergable requests
-    num_reqs = multiwrite_merge(bs, reqs, num_reqs, mcb);
-
-    trace_bdrv_aio_multiwrite(mcb, mcb->num_callbacks, num_reqs);
-
-    /* Run the aio requests. */
-    mcb->num_requests = num_reqs;
-    for (i = 0; i < num_reqs; i++) {
-        bdrv_co_aio_rw_vector(bs, reqs[i].sector, reqs[i].qiov,
-                              reqs[i].nb_sectors, reqs[i].flags,
-                              multiwrite_cb, mcb,
-                              true);
-    }
-
-    return 0;
-}
-
-void bdrv_aio_cancel(BlockAIOCB *acb)
-{
-    qemu_aio_ref(acb);
-    bdrv_aio_cancel_async(acb);
-    while (acb->refcnt > 1) {
-        if (acb->aiocb_info->get_aio_context) {
-            aio_poll(acb->aiocb_info->get_aio_context(acb), true);
-        } else if (acb->bs) {
-            aio_poll(bdrv_get_aio_context(acb->bs), true);
-        } else {
-            abort();
-        }
-    }
-    qemu_aio_unref(acb);
-}
-
-/* Async version of aio cancel. The caller is not blocked if the acb implements
- * cancel_async, otherwise we do nothing and let the request normally complete.
- * In either case the completion callback must be called. */
-void bdrv_aio_cancel_async(BlockAIOCB *acb)
-{
-    if (acb->aiocb_info->cancel_async) {
-        acb->aiocb_info->cancel_async(acb);
-    }
-}
-
-/**************************************************************/
-/* async block device emulation */
-
-typedef struct BlockAIOCBSync {
-    BlockAIOCB common;
-    QEMUBH *bh;
-    int ret;
-    /* vector translation state */
-    QEMUIOVector *qiov;
-    uint8_t *bounce;
-    int is_write;
-} BlockAIOCBSync;
-
-static const AIOCBInfo bdrv_em_aiocb_info = {
-    .aiocb_size         = sizeof(BlockAIOCBSync),
-};
-
-static void bdrv_aio_bh_cb(void *opaque)
-{
-    BlockAIOCBSync *acb = opaque;
-
-    if (!acb->is_write && acb->ret >= 0) {
-        qemu_iovec_from_buf(acb->qiov, 0, acb->bounce, acb->qiov->size);
-    }
-    qemu_vfree(acb->bounce);
-    acb->common.cb(acb->common.opaque, acb->ret);
-    qemu_bh_delete(acb->bh);
-    acb->bh = NULL;
-    qemu_aio_unref(acb);
-}
-
-static BlockAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
-                                      int64_t sector_num,
-                                      QEMUIOVector *qiov,
-                                      int nb_sectors,
-                                      BlockCompletionFunc *cb,
-                                      void *opaque,
-                                      int is_write)
-
-{
-    BlockAIOCBSync *acb;
-
-    acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
-    acb->is_write = is_write;
-    acb->qiov = qiov;
-    acb->bounce = qemu_try_blockalign(bs, qiov->size);
-    acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_aio_bh_cb, acb);
-
-    if (acb->bounce == NULL) {
-        acb->ret = -ENOMEM;
-    } else if (is_write) {
-        qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
-        acb->ret = bs->drv->bdrv_write(bs, sector_num, acb->bounce, nb_sectors);
-    } else {
-        acb->ret = bs->drv->bdrv_read(bs, sector_num, acb->bounce, nb_sectors);
-    }
-
-    qemu_bh_schedule(acb->bh);
-
-    return &acb->common;
-}
-
-static BlockAIOCB *bdrv_aio_readv_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
-}
-
-static BlockAIOCB *bdrv_aio_writev_em(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    return bdrv_aio_rw_vector(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
-}
-
-
-typedef struct BlockAIOCBCoroutine {
-    BlockAIOCB common;
-    BlockRequest req;
-    bool is_write;
-    bool *done;
-    QEMUBH* bh;
-} BlockAIOCBCoroutine;
-
-static const AIOCBInfo bdrv_em_co_aiocb_info = {
-    .aiocb_size         = sizeof(BlockAIOCBCoroutine),
-};
-
-static void bdrv_co_em_bh(void *opaque)
-{
-    BlockAIOCBCoroutine *acb = opaque;
-
-    acb->common.cb(acb->common.opaque, acb->req.error);
-
-    qemu_bh_delete(acb->bh);
-    qemu_aio_unref(acb);
-}
-
-/* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
-static void coroutine_fn bdrv_co_do_rw(void *opaque)
-{
-    BlockAIOCBCoroutine *acb = opaque;
-    BlockDriverState *bs = acb->common.bs;
-
-    if (!acb->is_write) {
-        acb->req.error = bdrv_co_do_readv(bs, acb->req.sector,
-            acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
-    } else {
-        acb->req.error = bdrv_co_do_writev(bs, acb->req.sector,
-            acb->req.nb_sectors, acb->req.qiov, acb->req.flags);
-    }
-
-    acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
-    qemu_bh_schedule(acb->bh);
-}
-
-static BlockAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
-                                         int64_t sector_num,
-                                         QEMUIOVector *qiov,
-                                         int nb_sectors,
-                                         BdrvRequestFlags flags,
-                                         BlockCompletionFunc *cb,
-                                         void *opaque,
-                                         bool is_write)
-{
-    Coroutine *co;
-    BlockAIOCBCoroutine *acb;
-
-    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-    acb->req.sector = sector_num;
-    acb->req.nb_sectors = nb_sectors;
-    acb->req.qiov = qiov;
-    acb->req.flags = flags;
-    acb->is_write = is_write;
-
-    co = qemu_coroutine_create(bdrv_co_do_rw);
-    qemu_coroutine_enter(co, acb);
-
-    return &acb->common;
-}
-
-static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
-{
-    BlockAIOCBCoroutine *acb = opaque;
-    BlockDriverState *bs = acb->common.bs;
-
-    acb->req.error = bdrv_co_flush(bs);
-    acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
-    qemu_bh_schedule(acb->bh);
-}
-
-BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    trace_bdrv_aio_flush(bs, opaque);
-
-    Coroutine *co;
-    BlockAIOCBCoroutine *acb;
-
-    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-
-    co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
-    qemu_coroutine_enter(co, acb);
-
-    return &acb->common;
-}
-
-static void coroutine_fn bdrv_aio_discard_co_entry(void *opaque)
-{
-    BlockAIOCBCoroutine *acb = opaque;
-    BlockDriverState *bs = acb->common.bs;
-
-    acb->req.error = bdrv_co_discard(bs, acb->req.sector, acb->req.nb_sectors);
-    acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
-    qemu_bh_schedule(acb->bh);
-}
-
-BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    Coroutine *co;
-    BlockAIOCBCoroutine *acb;
-
-    trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
-
-    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
-    acb->req.sector = sector_num;
-    acb->req.nb_sectors = nb_sectors;
-    co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
-    qemu_coroutine_enter(co, acb);
-
-    return &acb->common;
+    return 1 + bdrv_get_backing_file_depth(bs->backing->bs);
 }
 
 void bdrv_init(void)
@@ -4805,163 +3042,9 @@
     bdrv_init();
 }
 
-void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
-                   BlockCompletionFunc *cb, void *opaque)
-{
-    BlockAIOCB *acb;
-
-    acb = g_slice_alloc(aiocb_info->aiocb_size);
-    acb->aiocb_info = aiocb_info;
-    acb->bs = bs;
-    acb->cb = cb;
-    acb->opaque = opaque;
-    acb->refcnt = 1;
-    return acb;
-}
-
-void qemu_aio_ref(void *p)
-{
-    BlockAIOCB *acb = p;
-    acb->refcnt++;
-}
-
-void qemu_aio_unref(void *p)
-{
-    BlockAIOCB *acb = p;
-    assert(acb->refcnt > 0);
-    if (--acb->refcnt == 0) {
-        g_slice_free1(acb->aiocb_info->aiocb_size, acb);
-    }
-}
-
-/**************************************************************/
-/* Coroutine block device emulation */
-
-typedef struct CoroutineIOCompletion {
-    Coroutine *coroutine;
-    int ret;
-} CoroutineIOCompletion;
-
-static void bdrv_co_io_em_complete(void *opaque, int ret)
-{
-    CoroutineIOCompletion *co = opaque;
-
-    co->ret = ret;
-    qemu_coroutine_enter(co->coroutine, NULL);
-}
-
-static int coroutine_fn bdrv_co_io_em(BlockDriverState *bs, int64_t sector_num,
-                                      int nb_sectors, QEMUIOVector *iov,
-                                      bool is_write)
-{
-    CoroutineIOCompletion co = {
-        .coroutine = qemu_coroutine_self(),
-    };
-    BlockAIOCB *acb;
-
-    if (is_write) {
-        acb = bs->drv->bdrv_aio_writev(bs, sector_num, iov, nb_sectors,
-                                       bdrv_co_io_em_complete, &co);
-    } else {
-        acb = bs->drv->bdrv_aio_readv(bs, sector_num, iov, nb_sectors,
-                                      bdrv_co_io_em_complete, &co);
-    }
-
-    trace_bdrv_co_io_em(bs, sector_num, nb_sectors, is_write, acb);
-    if (!acb) {
-        return -EIO;
-    }
-    qemu_coroutine_yield();
-
-    return co.ret;
-}
-
-static int coroutine_fn bdrv_co_readv_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov)
-{
-    return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, false);
-}
-
-static int coroutine_fn bdrv_co_writev_em(BlockDriverState *bs,
-                                         int64_t sector_num, int nb_sectors,
-                                         QEMUIOVector *iov)
-{
-    return bdrv_co_io_em(bs, sector_num, nb_sectors, iov, true);
-}
-
-static void coroutine_fn bdrv_flush_co_entry(void *opaque)
-{
-    RwCo *rwco = opaque;
-
-    rwco->ret = bdrv_co_flush(rwco->bs);
-}
-
-int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
-{
-    int ret;
-
-    if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-        return 0;
-    }
-
-    /* Write back cached data to the OS even with cache=unsafe */
-    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
-    if (bs->drv->bdrv_co_flush_to_os) {
-        ret = bs->drv->bdrv_co_flush_to_os(bs);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    /* But don't actually force it to the disk with cache=unsafe */
-    if (bs->open_flags & BDRV_O_NO_FLUSH) {
-        goto flush_parent;
-    }
-
-    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
-    if (bs->drv->bdrv_co_flush_to_disk) {
-        ret = bs->drv->bdrv_co_flush_to_disk(bs);
-    } else if (bs->drv->bdrv_aio_flush) {
-        BlockAIOCB *acb;
-        CoroutineIOCompletion co = {
-            .coroutine = qemu_coroutine_self(),
-        };
-
-        acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
-        if (acb == NULL) {
-            ret = -EIO;
-        } else {
-            qemu_coroutine_yield();
-            ret = co.ret;
-        }
-    } else {
-        /*
-         * Some block drivers always operate in either writethrough or unsafe
-         * mode and don't support bdrv_flush therefore. Usually qemu doesn't
-         * know how the server works (because the behaviour is hardcoded or
-         * depends on server-side configuration), so we can't ensure that
-         * everything is safe on disk. Returning an error doesn't work because
-         * that would break guests even if the server operates in writethrough
-         * mode.
-         *
-         * Let's hope the user knows what he's doing.
-         */
-        ret = 0;
-    }
-    if (ret < 0) {
-        return ret;
-    }
-
-    /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
-     * in the case of cache=unsafe, so there are no useless flushes.
-     */
-flush_parent:
-    return bdrv_co_flush(bs->file);
-}
-
 void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp)
 {
+    BdrvChild *child;
     Error *local_err = NULL;
     int ret;
 
@@ -4969,23 +3052,32 @@
         return;
     }
 
-    if (!(bs->open_flags & BDRV_O_INCOMING)) {
+    if (!(bs->open_flags & BDRV_O_INACTIVE)) {
         return;
     }
-    bs->open_flags &= ~BDRV_O_INCOMING;
+    bs->open_flags &= ~BDRV_O_INACTIVE;
 
     if (bs->drv->bdrv_invalidate_cache) {
         bs->drv->bdrv_invalidate_cache(bs, &local_err);
-    } else if (bs->file) {
-        bdrv_invalidate_cache(bs->file, &local_err);
+        if (local_err) {
+            bs->open_flags |= BDRV_O_INACTIVE;
+            error_propagate(errp, local_err);
+            return;
+        }
     }
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_invalidate_cache(child->bs, &local_err);
+        if (local_err) {
+            bs->open_flags |= BDRV_O_INACTIVE;
+            error_propagate(errp, local_err);
+            return;
+        }
     }
 
     ret = refresh_total_sectors(bs, bs->total_sectors);
     if (ret < 0) {
+        bs->open_flags |= BDRV_O_INACTIVE;
         error_setg_errno(errp, -ret, "Could not refresh total sector count");
         return;
     }
@@ -4995,8 +3087,9 @@
 {
     BlockDriverState *bs;
     Error *local_err = NULL;
+    BdrvNextIterator it;
 
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
         AioContext *aio_context = bdrv_get_aio_context(bs);
 
         aio_context_acquire(aio_context);
@@ -5009,143 +3102,62 @@
     }
 }
 
-int bdrv_flush(BlockDriverState *bs)
+static int bdrv_inactivate_recurse(BlockDriverState *bs,
+                                   bool setting_flag)
 {
-    Coroutine *co;
-    RwCo rwco = {
-        .bs = bs,
-        .ret = NOT_DONE,
-    };
-
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_flush_co_entry(&rwco);
-    } else {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
-
-        co = qemu_coroutine_create(bdrv_flush_co_entry);
-        qemu_coroutine_enter(co, &rwco);
-        while (rwco.ret == NOT_DONE) {
-            aio_poll(aio_context, true);
-        }
-    }
-
-    return rwco.ret;
-}
-
-typedef struct DiscardCo {
-    BlockDriverState *bs;
-    int64_t sector_num;
-    int nb_sectors;
+    BdrvChild *child;
     int ret;
-} DiscardCo;
-static void coroutine_fn bdrv_discard_co_entry(void *opaque)
-{
-    DiscardCo *rwco = opaque;
 
-    rwco->ret = bdrv_co_discard(rwco->bs, rwco->sector_num, rwco->nb_sectors);
-}
-
-/* if no limit is specified in the BlockLimits use a default
- * of 32768 512-byte sectors (16 MiB) per request.
- */
-#define MAX_DISCARD_DEFAULT 32768
-
-int coroutine_fn bdrv_co_discard(BlockDriverState *bs, int64_t sector_num,
-                                 int nb_sectors)
-{
-    int max_discard;
-
-    if (!bs->drv) {
-        return -ENOMEDIUM;
-    } else if (bdrv_check_request(bs, sector_num, nb_sectors)) {
-        return -EIO;
-    } else if (bs->read_only) {
-        return -EROFS;
-    }
-
-    bdrv_reset_dirty(bs, sector_num, nb_sectors);
-
-    /* Do nothing if disabled.  */
-    if (!(bs->open_flags & BDRV_O_UNMAP)) {
-        return 0;
-    }
-
-    if (!bs->drv->bdrv_co_discard && !bs->drv->bdrv_aio_discard) {
-        return 0;
-    }
-
-    max_discard = bs->bl.max_discard ?  bs->bl.max_discard : MAX_DISCARD_DEFAULT;
-    while (nb_sectors > 0) {
-        int ret;
-        int num = nb_sectors;
-
-        /* align request */
-        if (bs->bl.discard_alignment &&
-            num >= bs->bl.discard_alignment &&
-            sector_num % bs->bl.discard_alignment) {
-            if (num > bs->bl.discard_alignment) {
-                num = bs->bl.discard_alignment;
-            }
-            num -= sector_num % bs->bl.discard_alignment;
-        }
-
-        /* limit request size */
-        if (num > max_discard) {
-            num = max_discard;
-        }
-
-        if (bs->drv->bdrv_co_discard) {
-            ret = bs->drv->bdrv_co_discard(bs, sector_num, num);
-        } else {
-            BlockAIOCB *acb;
-            CoroutineIOCompletion co = {
-                .coroutine = qemu_coroutine_self(),
-            };
-
-            acb = bs->drv->bdrv_aio_discard(bs, sector_num, nb_sectors,
-                                            bdrv_co_io_em_complete, &co);
-            if (acb == NULL) {
-                return -EIO;
-            } else {
-                qemu_coroutine_yield();
-                ret = co.ret;
-            }
-        }
-        if (ret && ret != -ENOTSUP) {
+    if (!setting_flag && bs->drv->bdrv_inactivate) {
+        ret = bs->drv->bdrv_inactivate(bs);
+        if (ret < 0) {
             return ret;
         }
+    }
 
-        sector_num += num;
-        nb_sectors -= num;
+    QLIST_FOREACH(child, &bs->children, next) {
+        ret = bdrv_inactivate_recurse(child->bs, setting_flag);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    if (setting_flag) {
+        bs->open_flags |= BDRV_O_INACTIVE;
     }
     return 0;
 }
 
-int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors)
+int bdrv_inactivate_all(void)
 {
-    Coroutine *co;
-    DiscardCo rwco = {
-        .bs = bs,
-        .sector_num = sector_num,
-        .nb_sectors = nb_sectors,
-        .ret = NOT_DONE,
-    };
+    BlockDriverState *bs = NULL;
+    BdrvNextIterator it;
+    int ret = 0;
+    int pass;
 
-    if (qemu_in_coroutine()) {
-        /* Fast-path if already in coroutine context */
-        bdrv_discard_co_entry(&rwco);
-    } else {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        aio_context_acquire(bdrv_get_aio_context(bs));
+    }
 
-        co = qemu_coroutine_create(bdrv_discard_co_entry);
-        qemu_coroutine_enter(co, &rwco);
-        while (rwco.ret == NOT_DONE) {
-            aio_poll(aio_context, true);
+    /* We do two passes of inactivation. The first pass calls to drivers'
+     * .bdrv_inactivate callbacks recursively so all cache is flushed to disk;
+     * the second pass sets the BDRV_O_INACTIVE flag so that no further write
+     * is allowed. */
+    for (pass = 0; pass < 2; pass++) {
+        for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+            ret = bdrv_inactivate_recurse(bs, pass);
+            if (ret < 0) {
+                goto out;
+            }
         }
     }
 
-    return rwco.ret;
+out:
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        aio_context_release(bdrv_get_aio_context(bs));
+    }
+
+    return ret;
 }
 
 /**************************************************************/
@@ -5154,15 +3166,23 @@
 /**
  * Return TRUE if the media is present
  */
-int bdrv_is_inserted(BlockDriverState *bs)
+bool bdrv_is_inserted(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
+    BdrvChild *child;
 
-    if (!drv)
-        return 0;
-    if (!drv->bdrv_is_inserted)
-        return 1;
-    return drv->bdrv_is_inserted(bs);
+    if (!drv) {
+        return false;
+    }
+    if (drv->bdrv_is_inserted) {
+        return drv->bdrv_is_inserted(bs);
+    }
+    QLIST_FOREACH(child, &bs->children, next) {
+        if (!bdrv_is_inserted(child->bs)) {
+            return false;
+        }
+    }
+    return true;
 }
 
 /**
@@ -5213,179 +3233,6 @@
     }
 }
 
-/* needed for generic scsi interface */
-
-int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (drv && drv->bdrv_ioctl)
-        return drv->bdrv_ioctl(bs, req, buf);
-    return -ENOTSUP;
-}
-
-BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
-        unsigned long int req, void *buf,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    BlockDriver *drv = bs->drv;
-
-    if (drv && drv->bdrv_aio_ioctl)
-        return drv->bdrv_aio_ioctl(bs, req, buf, cb, opaque);
-    return NULL;
-}
-
-void bdrv_set_guest_block_size(BlockDriverState *bs, int align)
-{
-    bs->guest_block_size = align;
-}
-
-void *qemu_blockalign(BlockDriverState *bs, size_t size)
-{
-    return qemu_memalign(bdrv_opt_mem_align(bs), size);
-}
-
-void *qemu_blockalign0(BlockDriverState *bs, size_t size)
-{
-    return memset(qemu_blockalign(bs, size), 0, size);
-}
-
-void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
-{
-    size_t align = bdrv_opt_mem_align(bs);
-
-    /* Ensure that NULL is never returned on success */
-    assert(align > 0);
-    if (size == 0) {
-        size = align;
-    }
-
-    return qemu_try_memalign(align, size);
-}
-
-void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
-{
-    void *mem = qemu_try_blockalign(bs, size);
-
-    if (mem) {
-        memset(mem, 0, size);
-    }
-
-    return mem;
-}
-
-/*
- * Check if all memory in this vector is sector aligned.
- */
-bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
-{
-    int i;
-    size_t alignment = bdrv_opt_mem_align(bs);
-
-    for (i = 0; i < qiov->niov; i++) {
-        if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
-            return false;
-        }
-        if (qiov->iov[i].iov_len % alignment) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
-BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs, int granularity,
-                                          Error **errp)
-{
-    int64_t bitmap_size;
-    BdrvDirtyBitmap *bitmap;
-
-    assert((granularity & (granularity - 1)) == 0);
-
-    granularity >>= BDRV_SECTOR_BITS;
-    assert(granularity);
-    bitmap_size = bdrv_nb_sectors(bs);
-    if (bitmap_size < 0) {
-        error_setg_errno(errp, -bitmap_size, "could not get length of device");
-        errno = -bitmap_size;
-        return NULL;
-    }
-    bitmap = g_new0(BdrvDirtyBitmap, 1);
-    bitmap->bitmap = hbitmap_alloc(bitmap_size, ffs(granularity) - 1);
-    QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);
-    return bitmap;
-}
-
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
-{
-    BdrvDirtyBitmap *bm, *next;
-    QLIST_FOREACH_SAFE(bm, &bs->dirty_bitmaps, list, next) {
-        if (bm == bitmap) {
-            QLIST_REMOVE(bitmap, list);
-            hbitmap_free(bitmap->bitmap);
-            g_free(bitmap);
-            return;
-        }
-    }
-}
-
-BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
-{
-    BdrvDirtyBitmap *bm;
-    BlockDirtyInfoList *list = NULL;
-    BlockDirtyInfoList **plist = &list;
-
-    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
-        BlockDirtyInfo *info = g_new0(BlockDirtyInfo, 1);
-        BlockDirtyInfoList *entry = g_new0(BlockDirtyInfoList, 1);
-        info->count = bdrv_get_dirty_count(bs, bm);
-        info->granularity =
-            ((int64_t) BDRV_SECTOR_SIZE << hbitmap_granularity(bm->bitmap));
-        entry->value = info;
-        *plist = entry;
-        plist = &entry->next;
-    }
-
-    return list;
-}
-
-int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap, int64_t sector)
-{
-    if (bitmap) {
-        return hbitmap_get(bitmap->bitmap, sector);
-    } else {
-        return 0;
-    }
-}
-
-void bdrv_dirty_iter_init(BlockDriverState *bs,
-                          BdrvDirtyBitmap *bitmap, HBitmapIter *hbi)
-{
-    hbitmap_iter_init(hbi, bitmap->bitmap, 0);
-}
-
-void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
-                    int nr_sectors)
-{
-    BdrvDirtyBitmap *bitmap;
-    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
-        hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
-    }
-}
-
-void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors)
-{
-    BdrvDirtyBitmap *bitmap;
-    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
-        hbitmap_reset(bitmap->bitmap, cur_sector, nr_sectors);
-    }
-}
-
-int64_t bdrv_get_dirty_count(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
-{
-    return hbitmap_count(bitmap->bitmap);
-}
-
 /* Get a reference to bs */
 void bdrv_ref(BlockDriverState *bs)
 {
@@ -5418,9 +3265,9 @@
     if (!QLIST_EMPTY(&bs->op_blockers[op])) {
         blocker = QLIST_FIRST(&bs->op_blockers[op]);
         if (errp) {
-            error_setg(errp, "Device '%s' is busy: %s",
-                       bdrv_get_device_name(bs),
-                       error_get_pretty(blocker->reason));
+            *errp = error_copy(blocker->reason);
+            error_prepend(errp, "Node '%s' is busy: ",
+                          bdrv_get_device_or_node_name(bs));
         }
         return true;
     }
@@ -5477,46 +3324,6 @@
     return true;
 }
 
-void bdrv_iostatus_enable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = true;
-    bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-}
-
-/* The I/O status is only enabled if the drive explicitly
- * enables it _and_ the VM is configured to stop on errors */
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
-{
-    return (bs->iostatus_enabled &&
-           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
-            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
-            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
-}
-
-void bdrv_iostatus_disable(BlockDriverState *bs)
-{
-    bs->iostatus_enabled = false;
-}
-
-void bdrv_iostatus_reset(BlockDriverState *bs)
-{
-    if (bdrv_iostatus_is_enabled(bs)) {
-        bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
-        if (bs->job) {
-            block_job_iostatus_reset(bs->job);
-        }
-    }
-}
-
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
-{
-    assert(bdrv_iostatus_is_enabled(bs));
-    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-        bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
-                                         BLOCK_DEVICE_IO_STATUS_FAILED;
-    }
-}
-
 void bdrv_img_create(const char *filename, const char *fmt,
                      const char *base_filename, const char *base_fmt,
                      char *options, uint64_t img_size, int flags,
@@ -5527,7 +3334,6 @@
     const char *backing_fmt, *backing_file;
     int64_t size;
     BlockDriver *drv, *proto_drv;
-    BlockDriver *backing_drv = NULL;
     Error *local_err = NULL;
     int ret = 0;
 
@@ -5538,9 +3344,20 @@
         return;
     }
 
-    proto_drv = bdrv_find_protocol(filename, true);
+    proto_drv = bdrv_find_protocol(filename, true, errp);
     if (!proto_drv) {
-        error_setg(errp, "Unknown protocol '%s'", filename);
+        return;
+    }
+
+    if (!drv->create_opts) {
+        error_setg(errp, "Format driver '%s' does not support image creation",
+                   drv->format_name);
+        return;
+    }
+
+    if (!proto_drv->create_opts) {
+        error_setg(errp, "Protocol driver '%s' does not support image creation",
+                   proto_drv->format_name);
         return;
     }
 
@@ -5549,18 +3366,22 @@
 
     /* Create parameter list with default values */
     opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
-    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, img_size);
+    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, img_size, &error_abort);
 
     /* Parse -o options */
     if (options) {
-        if (qemu_opts_do_parse(opts, options, NULL) != 0) {
+        qemu_opts_do_parse(opts, options, NULL, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
+            local_err = NULL;
             error_setg(errp, "Invalid options for file format '%s'", fmt);
             goto out;
         }
     }
 
     if (base_filename) {
-        if (qemu_opt_set(opts, BLOCK_OPT_BACKING_FILE, base_filename)) {
+        qemu_opt_set(opts, BLOCK_OPT_BACKING_FILE, base_filename, &local_err);
+        if (local_err) {
             error_setg(errp, "Backing file not supported for file format '%s'",
                        fmt);
             goto out;
@@ -5568,7 +3389,8 @@
     }
 
     if (base_fmt) {
-        if (qemu_opt_set(opts, BLOCK_OPT_BACKING_FMT, base_fmt)) {
+        qemu_opt_set(opts, BLOCK_OPT_BACKING_FMT, base_fmt, &local_err);
+        if (local_err) {
             error_setg(errp, "Backing file format not supported for file "
                              "format '%s'", fmt);
             goto out;
@@ -5585,14 +3407,6 @@
     }
 
     backing_fmt = qemu_opt_get(opts, BLOCK_OPT_BACKING_FMT);
-    if (backing_fmt) {
-        backing_drv = bdrv_find_format(backing_fmt);
-        if (!backing_drv) {
-            error_setg(errp, "Unknown backing file format '%s'",
-                       backing_fmt);
-            goto out;
-        }
-    }
 
     // The size for the image must always be specified, with one exception:
     // If we are using a backing file, we can obtain the size from there
@@ -5600,17 +3414,33 @@
     if (size == -1) {
         if (backing_file) {
             BlockDriverState *bs;
+            char *full_backing = g_new0(char, PATH_MAX);
             int64_t size;
             int back_flags;
+            QDict *backing_options = NULL;
+
+            bdrv_get_full_backing_filename_from_filename(filename, backing_file,
+                                                         full_backing, PATH_MAX,
+                                                         &local_err);
+            if (local_err) {
+                g_free(full_backing);
+                goto out;
+            }
 
             /* backing files always opened read-only */
-            back_flags =
-                flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
+            back_flags = flags;
+            back_flags &= ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
 
-            bs = NULL;
-            ret = bdrv_open(&bs, backing_file, NULL, NULL, back_flags,
-                            backing_drv, &local_err);
-            if (ret < 0) {
+            if (backing_fmt) {
+                backing_options = qdict_new();
+                qdict_put(backing_options, "driver",
+                          qstring_from_str(backing_fmt));
+            }
+
+            bs = bdrv_open(full_backing, NULL, backing_options, back_flags,
+                           &local_err);
+            g_free(full_backing);
+            if (!bs) {
                 goto out;
             }
             size = bdrv_getlength(bs);
@@ -5621,7 +3451,7 @@
                 goto out;
             }
 
-            qemu_opt_set_number(opts, BLOCK_OPT_SIZE, size);
+            qemu_opt_set_number(opts, BLOCK_OPT_SIZE, size, &error_abort);
 
             bdrv_unref(bs);
         } else {
@@ -5632,7 +3462,7 @@
 
     if (!quiet) {
         printf("Formatting '%s', fmt=%s ", filename, fmt);
-        qemu_opts_print(opts);
+        qemu_opts_print(opts, " ");
         puts("");
     }
 
@@ -5655,9 +3485,7 @@
 out:
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
 AioContext *bdrv_get_aio_context(BlockDriverState *bs)
@@ -5665,29 +3493,40 @@
     return bs->aio_context;
 }
 
+static void bdrv_do_remove_aio_context_notifier(BdrvAioNotifier *ban)
+{
+    QLIST_REMOVE(ban, list);
+    g_free(ban);
+}
+
 void bdrv_detach_aio_context(BlockDriverState *bs)
 {
-    BdrvAioNotifier *baf;
+    BdrvAioNotifier *baf, *baf_tmp;
+    BdrvChild *child;
 
     if (!bs->drv) {
         return;
     }
 
-    QLIST_FOREACH(baf, &bs->aio_notifiers, list) {
-        baf->detach_aio_context(baf->opaque);
+    assert(!bs->walking_aio_notifiers);
+    bs->walking_aio_notifiers = true;
+    QLIST_FOREACH_SAFE(baf, &bs->aio_notifiers, list, baf_tmp) {
+        if (baf->deleted) {
+            bdrv_do_remove_aio_context_notifier(baf);
+        } else {
+            baf->detach_aio_context(baf->opaque);
+        }
     }
+    /* Never mind iterating again to check for ->deleted.  bdrv_close() will
+     * remove remaining aio notifiers if we aren't called again.
+     */
+    bs->walking_aio_notifiers = false;
 
-    if (bs->io_limits_enabled) {
-        throttle_detach_aio_context(&bs->throttle_state);
-    }
     if (bs->drv->bdrv_detach_aio_context) {
         bs->drv->bdrv_detach_aio_context(bs);
     }
-    if (bs->file) {
-        bdrv_detach_aio_context(bs->file);
-    }
-    if (bs->backing_hd) {
-        bdrv_detach_aio_context(bs->backing_hd);
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_detach_aio_context(child->bs);
     }
 
     bs->aio_context = NULL;
@@ -5696,7 +3535,8 @@
 void bdrv_attach_aio_context(BlockDriverState *bs,
                              AioContext *new_context)
 {
-    BdrvAioNotifier *ban;
+    BdrvAioNotifier *ban, *ban_tmp;
+    BdrvChild *child;
 
     if (!bs->drv) {
         return;
@@ -5704,27 +3544,28 @@
 
     bs->aio_context = new_context;
 
-    if (bs->backing_hd) {
-        bdrv_attach_aio_context(bs->backing_hd, new_context);
-    }
-    if (bs->file) {
-        bdrv_attach_aio_context(bs->file, new_context);
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_attach_aio_context(child->bs, new_context);
     }
     if (bs->drv->bdrv_attach_aio_context) {
         bs->drv->bdrv_attach_aio_context(bs, new_context);
     }
-    if (bs->io_limits_enabled) {
-        throttle_attach_aio_context(&bs->throttle_state, new_context);
-    }
 
-    QLIST_FOREACH(ban, &bs->aio_notifiers, list) {
-        ban->attached_aio_context(new_context, ban->opaque);
+    assert(!bs->walking_aio_notifiers);
+    bs->walking_aio_notifiers = true;
+    QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_tmp) {
+        if (ban->deleted) {
+            bdrv_do_remove_aio_context_notifier(ban);
+        } else {
+            ban->attached_aio_context(new_context, ban->opaque);
+        }
     }
+    bs->walking_aio_notifiers = false;
 }
 
 void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context)
 {
-    bdrv_drain_all(); /* ensure there are no in-flight requests */
+    bdrv_drain(bs); /* ensure there are no in-flight requests */
 
     bdrv_detach_aio_context(bs);
 
@@ -5761,11 +3602,14 @@
     QLIST_FOREACH_SAFE(ban, &bs->aio_notifiers, list, ban_next) {
         if (ban->attached_aio_context == attached_aio_context &&
             ban->detach_aio_context   == detach_aio_context   &&
-            ban->opaque               == opaque)
+            ban->opaque               == opaque               &&
+            ban->deleted              == false)
         {
-            QLIST_REMOVE(ban, list);
-            g_free(ban);
-
+            if (bs->walking_aio_notifiers) {
+                ban->deleted = true;
+            } else {
+                bdrv_do_remove_aio_context_notifier(ban);
+            }
             return;
         }
     }
@@ -5773,19 +3617,13 @@
     abort();
 }
 
-void bdrv_add_before_write_notifier(BlockDriverState *bs,
-                                    NotifierWithReturn *notifier)
-{
-    notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
-}
-
 int bdrv_amend_options(BlockDriverState *bs, QemuOpts *opts,
-                       BlockDriverAmendStatusCB *status_cb)
+                       BlockDriverAmendStatusCB *status_cb, void *cb_opaque)
 {
     if (!bs->drv->bdrv_amend_options) {
         return -ENOTSUP;
     }
-    return bs->drv->bdrv_amend_options(bs, opts, status_cb);
+    return bs->drv->bdrv_amend_options(bs, opts, status_cb, cb_opaque);
 }
 
 /* This function will be called by the bdrv_recurse_is_first_non_filter method
@@ -5828,9 +3666,10 @@
 bool bdrv_is_first_non_filter(BlockDriverState *candidate)
 {
     BlockDriverState *bs;
+    BdrvNextIterator it;
 
     /* walk down the bs forest recursively */
-    QTAILQ_FOREACH(bs, &bdrv_states, device_list) {
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
         bool perm;
 
         /* try to recurse in this top level bs */
@@ -5845,7 +3684,8 @@
     return false;
 }
 
-BlockDriverState *check_to_replace_node(const char *node_name, Error **errp)
+BlockDriverState *check_to_replace_node(BlockDriverState *parent_bs,
+                                        const char *node_name, Error **errp)
 {
     BlockDriverState *to_replace_bs = bdrv_find_node(node_name);
     AioContext *aio_context;
@@ -5868,7 +3708,7 @@
      * Another benefit is that this tests exclude backing files which are
      * blocked by the backing blockers.
      */
-    if (!bdrv_is_first_non_filter(to_replace_bs)) {
+    if (!bdrv_recurse_is_first_non_filter(parent_bs, to_replace_bs)) {
         error_setg(errp, "Only top most non filter can be replaced");
         to_replace_bs = NULL;
         goto out;
@@ -5879,53 +3719,42 @@
     return to_replace_bs;
 }
 
-void bdrv_io_plug(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (drv && drv->bdrv_io_plug) {
-        drv->bdrv_io_plug(bs);
-    } else if (bs->file) {
-        bdrv_io_plug(bs->file);
-    }
-}
-
-void bdrv_io_unplug(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (drv && drv->bdrv_io_unplug) {
-        drv->bdrv_io_unplug(bs);
-    } else if (bs->file) {
-        bdrv_io_unplug(bs->file);
-    }
-}
-
-void bdrv_flush_io_queue(BlockDriverState *bs)
-{
-    BlockDriver *drv = bs->drv;
-    if (drv && drv->bdrv_flush_io_queue) {
-        drv->bdrv_flush_io_queue(bs);
-    } else if (bs->file) {
-        bdrv_flush_io_queue(bs->file);
-    }
-}
-
 static bool append_open_options(QDict *d, BlockDriverState *bs)
 {
     const QDictEntry *entry;
+    QemuOptDesc *desc;
+    BdrvChild *child;
     bool found_any = false;
+    const char *p;
 
     for (entry = qdict_first(bs->options); entry;
          entry = qdict_next(bs->options, entry))
     {
-        /* Only take options for this level and exclude all non-driver-specific
-         * options */
-        if (!strchr(qdict_entry_key(entry), '.') &&
-            strcmp(qdict_entry_key(entry), "node-name"))
-        {
-            qobject_incref(qdict_entry_value(entry));
-            qdict_put_obj(d, qdict_entry_key(entry), qdict_entry_value(entry));
-            found_any = true;
+        /* Exclude options for children */
+        QLIST_FOREACH(child, &bs->children, next) {
+            if (strstart(qdict_entry_key(entry), child->name, &p)
+                && (!*p || *p == '.'))
+            {
+                break;
+            }
         }
+        if (child) {
+            continue;
+        }
+
+        /* And exclude all non-driver-specific options */
+        for (desc = bdrv_runtime_opts.desc; desc->name; desc++) {
+            if (!strcmp(qdict_entry_key(entry), desc->name)) {
+                break;
+            }
+        }
+        if (desc->name) {
+            continue;
+        }
+
+        qobject_incref(qdict_entry_value(entry));
+        qdict_put_obj(d, qdict_entry_key(entry), qdict_entry_value(entry));
+        found_any = true;
     }
 
     return found_any;
@@ -5955,7 +3784,7 @@
     /* This BDS's file name will most probably depend on its file's name, so
      * refresh that first */
     if (bs->file) {
-        bdrv_refresh_filename(bs->file);
+        bdrv_refresh_filename(bs->file->bs);
     }
 
     if (drv->bdrv_refresh_filename) {
@@ -5967,7 +3796,10 @@
             bs->full_open_options = NULL;
         }
 
-        drv->bdrv_refresh_filename(bs);
+        opts = qdict_new();
+        append_open_options(opts, bs);
+        drv->bdrv_refresh_filename(bs, opts);
+        QDECREF(opts);
     } else if (bs->file) {
         /* Try to reconstruct valid information from the underlying file */
         bool has_open_options;
@@ -5983,19 +3815,20 @@
 
         /* If no specific options have been given for this BDS, the filename of
          * the underlying file should suffice for this one as well */
-        if (bs->file->exact_filename[0] && !has_open_options) {
-            strcpy(bs->exact_filename, bs->file->exact_filename);
+        if (bs->file->bs->exact_filename[0] && !has_open_options) {
+            strcpy(bs->exact_filename, bs->file->bs->exact_filename);
         }
         /* Reconstructing the full options QDict is simple for most format block
          * drivers, as long as the full options are known for the underlying
          * file BDS. The full options QDict of that file BDS should somehow
          * contain a representation of the filename, therefore the following
          * suffices without querying the (exact_)filename of this BDS. */
-        if (bs->file->full_open_options) {
+        if (bs->file->bs->full_open_options) {
             qdict_put_obj(opts, "driver",
                           QOBJECT(qstring_from_str(drv->format_name)));
-            QINCREF(bs->file->full_open_options);
-            qdict_put_obj(opts, "file", QOBJECT(bs->file->full_open_options));
+            QINCREF(bs->file->bs->full_open_options);
+            qdict_put_obj(opts, "file",
+                          QOBJECT(bs->file->bs->full_open_options));
 
             bs->full_open_options = opts;
         } else {
@@ -6039,13 +3872,51 @@
     }
 }
 
-/* This accessor function purpose is to allow the device models to access the
- * BlockAcctStats structure embedded inside a BlockDriverState without being
- * aware of the BlockDriverState structure layout.
- * It will go away when the BlockAcctStats structure will be moved inside
- * the device models.
+/*
+ * Hot add/remove a BDS's child. So the user can take a child offline when
+ * it is broken and take a new child online
  */
-BlockAcctStats *bdrv_get_stats(BlockDriverState *bs)
+void bdrv_add_child(BlockDriverState *parent_bs, BlockDriverState *child_bs,
+                    Error **errp)
 {
-    return &bs->stats;
+
+    if (!parent_bs->drv || !parent_bs->drv->bdrv_add_child) {
+        error_setg(errp, "The node %s does not support adding a child",
+                   bdrv_get_device_or_node_name(parent_bs));
+        return;
+    }
+
+    if (!QLIST_EMPTY(&child_bs->parents)) {
+        error_setg(errp, "The node %s already has a parent",
+                   child_bs->node_name);
+        return;
+    }
+
+    parent_bs->drv->bdrv_add_child(parent_bs, child_bs, errp);
+}
+
+void bdrv_del_child(BlockDriverState *parent_bs, BdrvChild *child, Error **errp)
+{
+    BdrvChild *tmp;
+
+    if (!parent_bs->drv || !parent_bs->drv->bdrv_del_child) {
+        error_setg(errp, "The node %s does not support removing a child",
+                   bdrv_get_device_or_node_name(parent_bs));
+        return;
+    }
+
+    QLIST_FOREACH(tmp, &parent_bs->children, next) {
+        if (tmp == child) {
+            break;
+        }
+    }
+
+    if (!tmp) {
+        error_setg(errp, "The node %s does not have a child named %s",
+                   bdrv_get_device_or_node_name(parent_bs),
+                   bdrv_get_device_or_node_name(child->bs));
+        return;
+    }
+
+    parent_bs->drv->bdrv_del_child(parent_bs, child, errp);
 }
diff --git a/block/Makefile.objs b/block/Makefile.objs
index 04b0e43..2593a2f 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -1,15 +1,16 @@
-block-obj-y += raw_bsd.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat.o
+block-obj-y += raw_bsd.o qcow.o vdi.o vmdk.o cloop.o bochs.o vpc.o vvfat.o
 block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o
 block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o
 block-obj-y += qed-check.o
 block-obj-$(CONFIG_VHDX) += vhdx.o vhdx-endian.o vhdx-log.o
-block-obj-$(CONFIG_QUORUM) += quorum.o
-block-obj-y += parallels.o blkdebug.o blkverify.o
+block-obj-y += quorum.o
+block-obj-y += parallels.o blkdebug.o blkverify.o blkreplay.o
 block-obj-y += block-backend.o snapshot.o qapi.o
 block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
 block-obj-$(CONFIG_POSIX) += raw-posix.o
 block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
-block-obj-y += null.o mirror.o
+block-obj-y += null.o mirror.o commit.o io.o
+block-obj-y += throttle-groups.o
 
 block-obj-y += nbd.o nbd-client.o sheepdog.o
 block-obj-$(CONFIG_LIBISCSI) += iscsi.o
@@ -19,10 +20,12 @@
 block-obj-$(CONFIG_GLUSTERFS) += gluster.o
 block-obj-$(CONFIG_ARCHIPELAGO) += archipelago.o
 block-obj-$(CONFIG_LIBSSH2) += ssh.o
-block-obj-y += accounting.o
+block-obj-y += accounting.o dirty-bitmap.o
+block-obj-y += write-threshold.o
+
+block-obj-y += crypto.o
 
 common-obj-y += stream.o
-common-obj-y += commit.o
 common-obj-y += backup.o
 
 iscsi.o-cflags     := $(LIBISCSI_CFLAGS)
@@ -36,5 +39,7 @@
 ssh.o-cflags       := $(LIBSSH2_CFLAGS)
 ssh.o-libs         := $(LIBSSH2_LIBS)
 archipelago.o-libs := $(ARCHIPELAGO_LIBS)
+block-obj-m        += dmg.o
+dmg.o-libs         := $(BZIP2_LIBS)
 qcow.o-libs        := -lz
 linux-aio.o-libs   := -laio
diff --git a/block/accounting.c b/block/accounting.c
index edbb1cc..3f457c4 100644
--- a/block/accounting.c
+++ b/block/accounting.c
@@ -2,6 +2,7 @@
  * QEMU System Emulator block accounting
  *
  * Copyright (c) 2011 Christoph Hellwig
+ * Copyright (c) 2015 Igalia, S.L.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,8 +23,58 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/accounting.h"
 #include "block/block_int.h"
+#include "qemu/timer.h"
+#include "sysemu/qtest.h"
+
+static QEMUClockType clock_type = QEMU_CLOCK_REALTIME;
+static const int qtest_latency_ns = NANOSECONDS_PER_SECOND / 1000;
+
+void block_acct_init(BlockAcctStats *stats, bool account_invalid,
+                     bool account_failed)
+{
+    stats->account_invalid = account_invalid;
+    stats->account_failed = account_failed;
+
+    if (qtest_enabled()) {
+        clock_type = QEMU_CLOCK_VIRTUAL;
+    }
+}
+
+void block_acct_cleanup(BlockAcctStats *stats)
+{
+    BlockAcctTimedStats *s, *next;
+    QSLIST_FOREACH_SAFE(s, &stats->intervals, entries, next) {
+        g_free(s);
+    }
+}
+
+void block_acct_add_interval(BlockAcctStats *stats, unsigned interval_length)
+{
+    BlockAcctTimedStats *s;
+    unsigned i;
+
+    s = g_new0(BlockAcctTimedStats, 1);
+    s->interval_length = interval_length;
+    QSLIST_INSERT_HEAD(&stats->intervals, s, entries);
+
+    for (i = 0; i < BLOCK_MAX_IOTYPE; i++) {
+        timed_average_init(&s->latency[i], clock_type,
+                           (uint64_t) interval_length * NANOSECONDS_PER_SECOND);
+    }
+}
+
+BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
+                                              BlockAcctTimedStats *s)
+{
+    if (s == NULL) {
+        return QSLIST_FIRST(&stats->intervals);
+    } else {
+        return QSLIST_NEXT(s, entries);
+    }
+}
 
 void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
                       int64_t bytes, enum BlockAcctType type)
@@ -31,24 +82,92 @@
     assert(type < BLOCK_MAX_IOTYPE);
 
     cookie->bytes = bytes;
-    cookie->start_time_ns = get_clock();
+    cookie->start_time_ns = qemu_clock_get_ns(clock_type);
     cookie->type = type;
 }
 
 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie)
 {
+    BlockAcctTimedStats *s;
+    int64_t time_ns = qemu_clock_get_ns(clock_type);
+    int64_t latency_ns = time_ns - cookie->start_time_ns;
+
+    if (qtest_enabled()) {
+        latency_ns = qtest_latency_ns;
+    }
+
     assert(cookie->type < BLOCK_MAX_IOTYPE);
 
     stats->nr_bytes[cookie->type] += cookie->bytes;
     stats->nr_ops[cookie->type]++;
-    stats->total_time_ns[cookie->type] += get_clock() - cookie->start_time_ns;
+    stats->total_time_ns[cookie->type] += latency_ns;
+    stats->last_access_time_ns = time_ns;
+
+    QSLIST_FOREACH(s, &stats->intervals, entries) {
+        timed_average_account(&s->latency[cookie->type], latency_ns);
+    }
 }
 
-
-void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
-                               unsigned int nb_sectors)
+void block_acct_failed(BlockAcctStats *stats, BlockAcctCookie *cookie)
 {
-    if (stats->wr_highest_sector < sector_num + nb_sectors - 1) {
-        stats->wr_highest_sector = sector_num + nb_sectors - 1;
+    assert(cookie->type < BLOCK_MAX_IOTYPE);
+
+    stats->failed_ops[cookie->type]++;
+
+    if (stats->account_failed) {
+        BlockAcctTimedStats *s;
+        int64_t time_ns = qemu_clock_get_ns(clock_type);
+        int64_t latency_ns = time_ns - cookie->start_time_ns;
+
+        if (qtest_enabled()) {
+            latency_ns = qtest_latency_ns;
+        }
+
+        stats->total_time_ns[cookie->type] += latency_ns;
+        stats->last_access_time_ns = time_ns;
+
+        QSLIST_FOREACH(s, &stats->intervals, entries) {
+            timed_average_account(&s->latency[cookie->type], latency_ns);
+        }
     }
 }
+
+void block_acct_invalid(BlockAcctStats *stats, enum BlockAcctType type)
+{
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    /* block_acct_done() and block_acct_failed() update
+     * total_time_ns[], but this one does not. The reason is that
+     * invalid requests are accounted during their submission,
+     * therefore there's no actual I/O involved. */
+
+    stats->invalid_ops[type]++;
+
+    if (stats->account_invalid) {
+        stats->last_access_time_ns = qemu_clock_get_ns(clock_type);
+    }
+}
+
+void block_acct_merge_done(BlockAcctStats *stats, enum BlockAcctType type,
+                      int num_requests)
+{
+    assert(type < BLOCK_MAX_IOTYPE);
+    stats->merged[type] += num_requests;
+}
+
+int64_t block_acct_idle_time_ns(BlockAcctStats *stats)
+{
+    return qemu_clock_get_ns(clock_type) - stats->last_access_time_ns;
+}
+
+double block_acct_queue_depth(BlockAcctTimedStats *stats,
+                              enum BlockAcctType type)
+{
+    uint64_t sum, elapsed;
+
+    assert(type < BLOCK_MAX_IOTYPE);
+
+    sum = timed_average_sum(&stats->latency[type], &elapsed);
+
+    return (double) sum / elapsed;
+}
diff --git a/block/archipelago.c b/block/archipelago.c
index a8114b5..37b8aca 100644
--- a/block/archipelago.c
+++ b/block/archipelago.c
@@ -50,7 +50,8 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "block/block_int.h"
 #include "qemu/error-report.h"
 #include "qemu/thread.h"
@@ -59,7 +60,6 @@
 #include "qapi/qmp/qjson.h"
 #include "qemu/atomic.h"
 
-#include <inttypes.h>
 #include <xseg/xseg.h>
 #include <xseg/protocol.h>
 
@@ -291,7 +291,7 @@
 
     ret = qemu_archipelago_xseg_init(s);
     if (ret < 0) {
-        error_report("Cannot initialize XSEG. Aborting...\n");
+        error_report("Cannot initialize XSEG. Aborting...");
         goto err_exit;
     }
 
@@ -645,7 +645,7 @@
 
     target = xseg_get_target(xseg, req);
     if (!target) {
-        error_setg(errp, "Cannot get XSEG target.\n");
+        error_setg(errp, "Cannot get XSEG target.");
         goto err_exit;
     }
     memcpy(target, volname, targetlen);
@@ -889,7 +889,7 @@
     return &aio_cb->common;
 
 err_exit:
-    error_report("qemu_archipelago_aio_rw(): I/O Error\n");
+    error_report("qemu_archipelago_aio_rw(): I/O Error");
     qemu_aio_unref(aio_cb);
     return NULL;
 }
@@ -974,11 +974,9 @@
 
 static int64_t qemu_archipelago_getlength(BlockDriverState *bs)
 {
-    int64_t ret;
     BDRVArchipelagoState *s = bs->opaque;
 
-    ret = archipelago_volume_info(s);
-    return ret;
+    return archipelago_volume_info(s);
 }
 
 static int qemu_archipelago_truncate(BlockDriverState *bs, int64_t offset)
diff --git a/block/backup.c b/block/backup.c
index 792e655..2c05323 100644
--- a/block/backup.c
+++ b/block/backup.c
@@ -11,20 +11,20 @@
  *
  */
 
-#include <stdio.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "trace.h"
 #include "block/block.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "qemu/cutils.h"
+#include "sysemu/block-backend.h"
+#include "qemu/bitmap.h"
 
-#define BACKUP_CLUSTER_BITS 16
-#define BACKUP_CLUSTER_SIZE (1 << BACKUP_CLUSTER_BITS)
-#define BACKUP_SECTORS_PER_CLUSTER (BACKUP_CLUSTER_SIZE / BDRV_SECTOR_SIZE)
-
+#define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16)
 #define SLICE_TIME 100000000ULL /* ns */
 
 typedef struct CowRequest {
@@ -36,17 +36,27 @@
 
 typedef struct BackupBlockJob {
     BlockJob common;
-    BlockDriverState *target;
+    BlockBackend *target;
+    /* bitmap for sync=incremental */
+    BdrvDirtyBitmap *sync_bitmap;
     MirrorSyncMode sync_mode;
     RateLimit limit;
     BlockdevOnError on_source_error;
     BlockdevOnError on_target_error;
     CoRwlock flush_rwlock;
     uint64_t sectors_read;
-    HBitmap *bitmap;
+    unsigned long *done_bitmap;
+    int64_t cluster_size;
+    NotifierWithReturn before_write;
     QLIST_HEAD(, CowRequest) inflight_reqs;
 } BackupBlockJob;
 
+/* Size of a cluster in sectors, instead of bytes. */
+static inline int64_t cluster_size_sectors(BackupBlockJob *job)
+{
+  return job->cluster_size / BDRV_SECTOR_SIZE;
+}
+
 /* See if in-flight requests overlap and wait for them to complete */
 static void coroutine_fn wait_for_overlapping_requests(BackupBlockJob *job,
                                                        int64_t start,
@@ -84,23 +94,25 @@
     qemu_co_queue_restart_all(&req->wait_queue);
 }
 
-static int coroutine_fn backup_do_cow(BlockDriverState *bs,
+static int coroutine_fn backup_do_cow(BackupBlockJob *job,
                                       int64_t sector_num, int nb_sectors,
-                                      bool *error_is_read)
+                                      bool *error_is_read,
+                                      bool is_write_notifier)
 {
-    BackupBlockJob *job = (BackupBlockJob *)bs->job;
+    BlockBackend *blk = job->common.blk;
     CowRequest cow_request;
     struct iovec iov;
     QEMUIOVector bounce_qiov;
     void *bounce_buffer = NULL;
     int ret = 0;
+    int64_t sectors_per_cluster = cluster_size_sectors(job);
     int64_t start, end;
     int n;
 
     qemu_co_rwlock_rdlock(&job->flush_rwlock);
 
-    start = sector_num / BACKUP_SECTORS_PER_CLUSTER;
-    end = DIV_ROUND_UP(sector_num + nb_sectors, BACKUP_SECTORS_PER_CLUSTER);
+    start = sector_num / sectors_per_cluster;
+    end = DIV_ROUND_UP(sector_num + nb_sectors, sectors_per_cluster);
 
     trace_backup_do_cow_enter(job, start, sector_num, nb_sectors);
 
@@ -108,26 +120,27 @@
     cow_request_begin(&cow_request, job, start, end);
 
     for (; start < end; start++) {
-        if (hbitmap_get(job->bitmap, start)) {
+        if (test_bit(start, job->done_bitmap)) {
             trace_backup_do_cow_skip(job, start);
             continue; /* already copied */
         }
 
         trace_backup_do_cow_process(job, start);
 
-        n = MIN(BACKUP_SECTORS_PER_CLUSTER,
+        n = MIN(sectors_per_cluster,
                 job->common.len / BDRV_SECTOR_SIZE -
-                start * BACKUP_SECTORS_PER_CLUSTER);
+                start * sectors_per_cluster);
 
         if (!bounce_buffer) {
-            bounce_buffer = qemu_blockalign(bs, BACKUP_CLUSTER_SIZE);
+            bounce_buffer = blk_blockalign(blk, job->cluster_size);
         }
         iov.iov_base = bounce_buffer;
         iov.iov_len = n * BDRV_SECTOR_SIZE;
         qemu_iovec_init_external(&bounce_qiov, &iov, 1);
 
-        ret = bdrv_co_readv(bs, start * BACKUP_SECTORS_PER_CLUSTER, n,
-                            &bounce_qiov);
+        ret = blk_co_preadv(blk, start * job->cluster_size,
+                            bounce_qiov.size, &bounce_qiov,
+                            is_write_notifier ? BDRV_REQ_NO_SERIALISING : 0);
         if (ret < 0) {
             trace_backup_do_cow_read_fail(job, start, ret);
             if (error_is_read) {
@@ -137,13 +150,11 @@
         }
 
         if (buffer_is_zero(iov.iov_base, iov.iov_len)) {
-            ret = bdrv_co_write_zeroes(job->target,
-                                       start * BACKUP_SECTORS_PER_CLUSTER,
-                                       n, BDRV_REQ_MAY_UNMAP);
+            ret = blk_co_pwrite_zeroes(job->target, start * job->cluster_size,
+                                       bounce_qiov.size, BDRV_REQ_MAY_UNMAP);
         } else {
-            ret = bdrv_co_writev(job->target,
-                                 start * BACKUP_SECTORS_PER_CLUSTER, n,
-                                 &bounce_qiov);
+            ret = blk_co_pwritev(job->target, start * job->cluster_size,
+                                 bounce_qiov.size, &bounce_qiov, 0);
         }
         if (ret < 0) {
             trace_backup_do_cow_write_fail(job, start, ret);
@@ -153,7 +164,7 @@
             goto out;
         }
 
-        hbitmap_set(job->bitmap, start, 1);
+        set_bit(start, job->done_bitmap);
 
         /* Publish progress, guest I/O counts as progress too.  Note that the
          * offset field is an opaque progress value, it is not a disk offset.
@@ -180,14 +191,16 @@
         NotifierWithReturn *notifier,
         void *opaque)
 {
+    BackupBlockJob *job = container_of(notifier, BackupBlockJob, before_write);
     BdrvTrackedRequest *req = opaque;
     int64_t sector_num = req->offset >> BDRV_SECTOR_BITS;
     int nb_sectors = req->bytes >> BDRV_SECTOR_BITS;
 
+    assert(req->bs == blk_bs(job->common.blk));
     assert((req->offset & (BDRV_SECTOR_SIZE - 1)) == 0);
     assert((req->bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
 
-    return backup_do_cow(req->bs, sector_num, nb_sectors, NULL);
+    return backup_do_cow(job, sector_num, nb_sectors, NULL, true);
 }
 
 static void backup_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -195,35 +208,69 @@
     BackupBlockJob *s = container_of(job, BackupBlockJob, common);
 
     if (speed < 0) {
-        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        error_setg(errp, QERR_INVALID_PARAMETER, "speed");
         return;
     }
     ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
 }
 
-static void backup_iostatus_reset(BlockJob *job)
+static void backup_cleanup_sync_bitmap(BackupBlockJob *job, int ret)
+{
+    BdrvDirtyBitmap *bm;
+    BlockDriverState *bs = blk_bs(job->common.blk);
+
+    if (ret < 0 || block_job_is_cancelled(&job->common)) {
+        /* Merge the successor back into the parent, delete nothing. */
+        bm = bdrv_reclaim_dirty_bitmap(bs, job->sync_bitmap, NULL);
+        assert(bm);
+    } else {
+        /* Everything is fine, delete this bitmap and install the backup. */
+        bm = bdrv_dirty_bitmap_abdicate(bs, job->sync_bitmap, NULL);
+        assert(bm);
+    }
+}
+
+static void backup_commit(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    if (s->sync_bitmap) {
+        backup_cleanup_sync_bitmap(s, 0);
+    }
+}
+
+static void backup_abort(BlockJob *job)
+{
+    BackupBlockJob *s = container_of(job, BackupBlockJob, common);
+    if (s->sync_bitmap) {
+        backup_cleanup_sync_bitmap(s, -1);
+    }
+}
+
+static void backup_attached_aio_context(BlockJob *job, AioContext *aio_context)
 {
     BackupBlockJob *s = container_of(job, BackupBlockJob, common);
 
-    bdrv_iostatus_reset(s->target);
+    blk_set_aio_context(s->target, aio_context);
 }
 
 static const BlockJobDriver backup_job_driver = {
-    .instance_size  = sizeof(BackupBlockJob),
-    .job_type       = BLOCK_JOB_TYPE_BACKUP,
-    .set_speed      = backup_set_speed,
-    .iostatus_reset = backup_iostatus_reset,
+    .instance_size          = sizeof(BackupBlockJob),
+    .job_type               = BLOCK_JOB_TYPE_BACKUP,
+    .set_speed              = backup_set_speed,
+    .commit                 = backup_commit,
+    .abort                  = backup_abort,
+    .attached_aio_context   = backup_attached_aio_context,
 };
 
 static BlockErrorAction backup_error_action(BackupBlockJob *job,
                                             bool read, int error)
 {
     if (read) {
-        return block_job_error_action(&job->common, job->common.bs,
-                                      job->on_source_error, true, error);
+        return block_job_error_action(&job->common, job->on_source_error,
+                                      true, error);
     } else {
-        return block_job_error_action(&job->common, job->target,
-                                      job->on_target_error, false, error);
+        return block_job_error_action(&job->common, job->on_target_error,
+                                      false, error);
     }
 }
 
@@ -236,70 +283,132 @@
     BackupBlockJob *s = container_of(job, BackupBlockJob, common);
     BackupCompleteData *data = opaque;
 
-    bdrv_unref(s->target);
+    blk_unref(s->target);
 
     block_job_completed(job, data->ret);
     g_free(data);
 }
 
+static bool coroutine_fn yield_and_check(BackupBlockJob *job)
+{
+    if (block_job_is_cancelled(&job->common)) {
+        return true;
+    }
+
+    /* we need to yield so that bdrv_drain_all() returns.
+     * (without, VM does not reboot)
+     */
+    if (job->common.speed) {
+        uint64_t delay_ns = ratelimit_calculate_delay(&job->limit,
+                                                      job->sectors_read);
+        job->sectors_read = 0;
+        block_job_sleep_ns(&job->common, QEMU_CLOCK_REALTIME, delay_ns);
+    } else {
+        block_job_sleep_ns(&job->common, QEMU_CLOCK_REALTIME, 0);
+    }
+
+    if (block_job_is_cancelled(&job->common)) {
+        return true;
+    }
+
+    return false;
+}
+
+static int coroutine_fn backup_run_incremental(BackupBlockJob *job)
+{
+    bool error_is_read;
+    int ret = 0;
+    int clusters_per_iter;
+    uint32_t granularity;
+    int64_t sector;
+    int64_t cluster;
+    int64_t end;
+    int64_t last_cluster = -1;
+    int64_t sectors_per_cluster = cluster_size_sectors(job);
+    HBitmapIter hbi;
+
+    granularity = bdrv_dirty_bitmap_granularity(job->sync_bitmap);
+    clusters_per_iter = MAX((granularity / job->cluster_size), 1);
+    bdrv_dirty_iter_init(job->sync_bitmap, &hbi);
+
+    /* Find the next dirty sector(s) */
+    while ((sector = hbitmap_iter_next(&hbi)) != -1) {
+        cluster = sector / sectors_per_cluster;
+
+        /* Fake progress updates for any clusters we skipped */
+        if (cluster != last_cluster + 1) {
+            job->common.offset += ((cluster - last_cluster - 1) *
+                                   job->cluster_size);
+        }
+
+        for (end = cluster + clusters_per_iter; cluster < end; cluster++) {
+            do {
+                if (yield_and_check(job)) {
+                    return ret;
+                }
+                ret = backup_do_cow(job, cluster * sectors_per_cluster,
+                                    sectors_per_cluster, &error_is_read,
+                                    false);
+                if ((ret < 0) &&
+                    backup_error_action(job, error_is_read, -ret) ==
+                    BLOCK_ERROR_ACTION_REPORT) {
+                    return ret;
+                }
+            } while (ret < 0);
+        }
+
+        /* If the bitmap granularity is smaller than the backup granularity,
+         * we need to advance the iterator pointer to the next cluster. */
+        if (granularity < job->cluster_size) {
+            bdrv_set_dirty_iter(&hbi, cluster * sectors_per_cluster);
+        }
+
+        last_cluster = cluster - 1;
+    }
+
+    /* Play some final catchup with the progress meter */
+    end = DIV_ROUND_UP(job->common.len, job->cluster_size);
+    if (last_cluster + 1 < end) {
+        job->common.offset += ((end - last_cluster - 1) * job->cluster_size);
+    }
+
+    return ret;
+}
+
 static void coroutine_fn backup_run(void *opaque)
 {
     BackupBlockJob *job = opaque;
     BackupCompleteData *data;
-    BlockDriverState *bs = job->common.bs;
-    BlockDriverState *target = job->target;
-    BlockdevOnError on_target_error = job->on_target_error;
-    NotifierWithReturn before_write = {
-        .notify = backup_before_write_notify,
-    };
+    BlockDriverState *bs = blk_bs(job->common.blk);
+    BlockBackend *target = job->target;
     int64_t start, end;
+    int64_t sectors_per_cluster = cluster_size_sectors(job);
     int ret = 0;
 
     QLIST_INIT(&job->inflight_reqs);
     qemu_co_rwlock_init(&job->flush_rwlock);
 
     start = 0;
-    end = DIV_ROUND_UP(job->common.len / BDRV_SECTOR_SIZE,
-                       BACKUP_SECTORS_PER_CLUSTER);
+    end = DIV_ROUND_UP(job->common.len, job->cluster_size);
 
-    job->bitmap = hbitmap_alloc(end, 0);
+    job->done_bitmap = bitmap_new(end);
 
-    bdrv_set_enable_write_cache(target, true);
-    bdrv_set_on_error(target, on_target_error, on_target_error);
-    bdrv_iostatus_enable(target);
-
-    bdrv_add_before_write_notifier(bs, &before_write);
+    job->before_write.notify = backup_before_write_notify;
+    bdrv_add_before_write_notifier(bs, &job->before_write);
 
     if (job->sync_mode == MIRROR_SYNC_MODE_NONE) {
         while (!block_job_is_cancelled(&job->common)) {
             /* Yield until the job is cancelled.  We just let our before_write
              * notify callback service CoW requests. */
-            job->common.busy = false;
-            qemu_coroutine_yield();
-            job->common.busy = true;
+            block_job_yield(&job->common);
         }
+    } else if (job->sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
+        ret = backup_run_incremental(job);
     } else {
         /* Both FULL and TOP SYNC_MODE's require copying.. */
         for (; start < end; start++) {
             bool error_is_read;
-
-            if (block_job_is_cancelled(&job->common)) {
-                break;
-            }
-
-            /* we need to yield so that qemu_aio_flush() returns.
-             * (without, VM does not reboot)
-             */
-            if (job->common.speed) {
-                uint64_t delay_ns = ratelimit_calculate_delay(
-                        &job->limit, job->sectors_read);
-                job->sectors_read = 0;
-                block_job_sleep_ns(&job->common, QEMU_CLOCK_REALTIME, delay_ns);
-            } else {
-                block_job_sleep_ns(&job->common, QEMU_CLOCK_REALTIME, 0);
-            }
-
-            if (block_job_is_cancelled(&job->common)) {
+            if (yield_and_check(job)) {
                 break;
             }
 
@@ -310,7 +419,7 @@
                 /* Check to see if these blocks are already in the
                  * backing file. */
 
-                for (i = 0; i < BACKUP_SECTORS_PER_CLUSTER;) {
+                for (i = 0; i < sectors_per_cluster;) {
                     /* bdrv_is_allocated() only returns true/false based
                      * on the first set of sectors it comes across that
                      * are are all in the same state.
@@ -319,8 +428,8 @@
                      * needed but at some point that is always the case. */
                     alloced =
                         bdrv_is_allocated(bs,
-                                start * BACKUP_SECTORS_PER_CLUSTER + i,
-                                BACKUP_SECTORS_PER_CLUSTER - i, &n);
+                                start * sectors_per_cluster + i,
+                                sectors_per_cluster - i, &n);
                     i += n;
 
                     if (alloced == 1 || n == 0) {
@@ -335,8 +444,8 @@
                 }
             }
             /* FULL sync mode we copy the whole drive. */
-            ret = backup_do_cow(bs, start * BACKUP_SECTORS_PER_CLUSTER,
-                    BACKUP_SECTORS_PER_CLUSTER, &error_is_read);
+            ret = backup_do_cow(job, start * sectors_per_cluster,
+                                sectors_per_cluster, &error_is_read, false);
             if (ret < 0) {
                 /* Depending on error action, fail now or retry cluster */
                 BlockErrorAction action =
@@ -351,38 +460,77 @@
         }
     }
 
-    notifier_with_return_remove(&before_write);
+    notifier_with_return_remove(&job->before_write);
 
     /* wait until pending backup_do_cow() calls have completed */
     qemu_co_rwlock_wrlock(&job->flush_rwlock);
     qemu_co_rwlock_unlock(&job->flush_rwlock);
+    g_free(job->done_bitmap);
 
-    hbitmap_free(job->bitmap);
-
-    bdrv_iostatus_disable(target);
+    bdrv_op_unblock_all(blk_bs(target), job->common.blocker);
 
     data = g_malloc(sizeof(*data));
     data->ret = ret;
     block_job_defer_to_main_loop(&job->common, backup_complete, data);
 }
 
-void backup_start(BlockDriverState *bs, BlockDriverState *target,
-                  int64_t speed, MirrorSyncMode sync_mode,
+void backup_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *target, int64_t speed,
+                  MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockCompletionFunc *cb, void *opaque,
-                  Error **errp)
+                  BlockJobTxn *txn, Error **errp)
 {
     int64_t len;
+    BlockDriverInfo bdi;
+    BackupBlockJob *job = NULL;
+    int ret;
 
     assert(bs);
     assert(target);
-    assert(cb);
 
-    if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
-         on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
-        error_set(errp, QERR_INVALID_PARAMETER, "on-source-error");
+    if (bs == target) {
+        error_setg(errp, "Source and target cannot be the same");
+        return;
+    }
+
+    if (!bdrv_is_inserted(bs)) {
+        error_setg(errp, "Device is not inserted: %s",
+                   bdrv_get_device_name(bs));
+        return;
+    }
+
+    if (!bdrv_is_inserted(target)) {
+        error_setg(errp, "Device is not inserted: %s",
+                   bdrv_get_device_name(target));
+        return;
+    }
+
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
+        return;
+    }
+
+    if (bdrv_op_is_blocked(target, BLOCK_OP_TYPE_BACKUP_TARGET, errp)) {
+        return;
+    }
+
+    if (sync_mode == MIRROR_SYNC_MODE_INCREMENTAL) {
+        if (!sync_bitmap) {
+            error_setg(errp, "must provide a valid bitmap name for "
+                             "\"incremental\" sync mode");
+            return;
+        }
+
+        /* Create a new bitmap, and freeze/disable this one. */
+        if (bdrv_dirty_bitmap_create_successor(bs, sync_bitmap, errp) < 0) {
+            return;
+        }
+    } else if (sync_bitmap) {
+        error_setg(errp,
+                   "a sync_bitmap was provided to backup_run, "
+                   "but received an incompatible sync_mode (%s)",
+                   MirrorSyncMode_lookup[sync_mode]);
         return;
     }
 
@@ -390,20 +538,55 @@
     if (len < 0) {
         error_setg_errno(errp, -len, "unable to get length for '%s'",
                          bdrv_get_device_name(bs));
-        return;
+        goto error;
     }
 
-    BackupBlockJob *job = block_job_create(&backup_job_driver, bs, speed,
-                                           cb, opaque, errp);
+    job = block_job_create(job_id, &backup_job_driver, bs, speed,
+                           cb, opaque, errp);
     if (!job) {
-        return;
+        goto error;
     }
 
+    job->target = blk_new();
+    blk_insert_bs(job->target, target);
+
     job->on_source_error = on_source_error;
     job->on_target_error = on_target_error;
-    job->target = target;
     job->sync_mode = sync_mode;
+    job->sync_bitmap = sync_mode == MIRROR_SYNC_MODE_INCREMENTAL ?
+                       sync_bitmap : NULL;
+
+    /* If there is no backing file on the target, we cannot rely on COW if our
+     * backup cluster size is smaller than the target cluster size. Even for
+     * targets with a backing file, try to avoid COW if possible. */
+    ret = bdrv_get_info(target, &bdi);
+    if (ret < 0 && !target->backing) {
+        error_setg_errno(errp, -ret,
+            "Couldn't determine the cluster size of the target image, "
+            "which has no backing file");
+        error_append_hint(errp,
+            "Aborting, since this may create an unusable destination image\n");
+        goto error;
+    } else if (ret < 0 && target->backing) {
+        /* Not fatal; just trudge on ahead. */
+        job->cluster_size = BACKUP_CLUSTER_SIZE_DEFAULT;
+    } else {
+        job->cluster_size = MAX(BACKUP_CLUSTER_SIZE_DEFAULT, bdi.cluster_size);
+    }
+
+    bdrv_op_block_all(target, job->common.blocker);
     job->common.len = len;
-    job->common.co = qemu_coroutine_create(backup_run);
-    qemu_coroutine_enter(job->common.co, job);
+    job->common.co = qemu_coroutine_create(backup_run, job);
+    block_job_txn_add_job(txn, &job->common);
+    qemu_coroutine_enter(job->common.co);
+    return;
+
+ error:
+    if (sync_bitmap) {
+        bdrv_reclaim_dirty_bitmap(bs, sync_bitmap, NULL);
+    }
+    if (job) {
+        blk_unref(job->target);
+        block_job_unref(&job->common);
+    }
 }
diff --git a/block/blkdebug.c b/block/blkdebug.c
index 862d93b..d5db166 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -22,7 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/config-file.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
@@ -30,12 +32,17 @@
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qstring.h"
+#include "sysemu/qtest.h"
 
 typedef struct BDRVBlkdebugState {
     int state;
     int new_state;
+    int align;
 
-    QLIST_HEAD(, BlkdebugRule) rules[BLKDBG_EVENT_MAX];
+    /* For blkdebug_refresh_filename() */
+    char *config_file;
+
+    QLIST_HEAD(, BlkdebugRule) rules[BLKDBG__MAX];
     QSIMPLEQ_HEAD(, BlkdebugRule) active_rules;
     QLIST_HEAD(, BlkdebugSuspendedReq) suspended_reqs;
 } BDRVBlkdebugState;
@@ -63,7 +70,7 @@
 };
 
 typedef struct BlkdebugRule {
-    BlkDebugEvent event;
+    BlkdebugEvent event;
     int action;
     int state;
     union {
@@ -142,69 +149,12 @@
     NULL
 };
 
-static const char *event_names[BLKDBG_EVENT_MAX] = {
-    [BLKDBG_L1_UPDATE]                      = "l1_update",
-    [BLKDBG_L1_GROW_ALLOC_TABLE]            = "l1_grow.alloc_table",
-    [BLKDBG_L1_GROW_WRITE_TABLE]            = "l1_grow.write_table",
-    [BLKDBG_L1_GROW_ACTIVATE_TABLE]         = "l1_grow.activate_table",
-
-    [BLKDBG_L2_LOAD]                        = "l2_load",
-    [BLKDBG_L2_UPDATE]                      = "l2_update",
-    [BLKDBG_L2_UPDATE_COMPRESSED]           = "l2_update_compressed",
-    [BLKDBG_L2_ALLOC_COW_READ]              = "l2_alloc.cow_read",
-    [BLKDBG_L2_ALLOC_WRITE]                 = "l2_alloc.write",
-
-    [BLKDBG_READ_AIO]                       = "read_aio",
-    [BLKDBG_READ_BACKING_AIO]               = "read_backing_aio",
-    [BLKDBG_READ_COMPRESSED]                = "read_compressed",
-
-    [BLKDBG_WRITE_AIO]                      = "write_aio",
-    [BLKDBG_WRITE_COMPRESSED]               = "write_compressed",
-
-    [BLKDBG_VMSTATE_LOAD]                   = "vmstate_load",
-    [BLKDBG_VMSTATE_SAVE]                   = "vmstate_save",
-
-    [BLKDBG_COW_READ]                       = "cow_read",
-    [BLKDBG_COW_WRITE]                      = "cow_write",
-
-    [BLKDBG_REFTABLE_LOAD]                  = "reftable_load",
-    [BLKDBG_REFTABLE_GROW]                  = "reftable_grow",
-    [BLKDBG_REFTABLE_UPDATE]                = "reftable_update",
-
-    [BLKDBG_REFBLOCK_LOAD]                  = "refblock_load",
-    [BLKDBG_REFBLOCK_UPDATE]                = "refblock_update",
-    [BLKDBG_REFBLOCK_UPDATE_PART]           = "refblock_update_part",
-    [BLKDBG_REFBLOCK_ALLOC]                 = "refblock_alloc",
-    [BLKDBG_REFBLOCK_ALLOC_HOOKUP]          = "refblock_alloc.hookup",
-    [BLKDBG_REFBLOCK_ALLOC_WRITE]           = "refblock_alloc.write",
-    [BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS]    = "refblock_alloc.write_blocks",
-    [BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE]     = "refblock_alloc.write_table",
-    [BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE]    = "refblock_alloc.switch_table",
-
-    [BLKDBG_CLUSTER_ALLOC]                  = "cluster_alloc",
-    [BLKDBG_CLUSTER_ALLOC_BYTES]            = "cluster_alloc_bytes",
-    [BLKDBG_CLUSTER_FREE]                   = "cluster_free",
-
-    [BLKDBG_FLUSH_TO_OS]                    = "flush_to_os",
-    [BLKDBG_FLUSH_TO_DISK]                  = "flush_to_disk",
-
-    [BLKDBG_PWRITEV_RMW_HEAD]               = "pwritev_rmw.head",
-    [BLKDBG_PWRITEV_RMW_AFTER_HEAD]         = "pwritev_rmw.after_head",
-    [BLKDBG_PWRITEV_RMW_TAIL]               = "pwritev_rmw.tail",
-    [BLKDBG_PWRITEV_RMW_AFTER_TAIL]         = "pwritev_rmw.after_tail",
-    [BLKDBG_PWRITEV]                        = "pwritev",
-    [BLKDBG_PWRITEV_ZERO]                   = "pwritev_zero",
-    [BLKDBG_PWRITEV_DONE]                   = "pwritev_done",
-
-    [BLKDBG_EMPTY_IMAGE_PREPARE]            = "empty_image_prepare",
-};
-
-static int get_event_by_name(const char *name, BlkDebugEvent *event)
+static int get_event_by_name(const char *name, BlkdebugEvent *event)
 {
     int i;
 
-    for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
-        if (!strcmp(event_names[i], name)) {
+    for (i = 0; i < BLKDBG__MAX; i++) {
+        if (!strcmp(BlkdebugEvent_lookup[i], name)) {
             *event = i;
             return 0;
         }
@@ -216,24 +166,23 @@
 struct add_rule_data {
     BDRVBlkdebugState *s;
     int action;
-    Error **errp;
 };
 
-static int add_rule(QemuOpts *opts, void *opaque)
+static int add_rule(void *opaque, QemuOpts *opts, Error **errp)
 {
     struct add_rule_data *d = opaque;
     BDRVBlkdebugState *s = d->s;
     const char* event_name;
-    BlkDebugEvent event;
+    BlkdebugEvent event;
     struct BlkdebugRule *rule;
 
     /* Find the right event for the rule */
     event_name = qemu_opt_get(opts, "event");
     if (!event_name) {
-        error_setg(d->errp, "Missing event name for rule");
+        error_setg(errp, "Missing event name for rule");
         return -1;
     } else if (get_event_by_name(event_name, &event) < 0) {
-        error_setg(d->errp, "Invalid event name \"%s\"", event_name);
+        error_setg(errp, "Invalid event name \"%s\"", event_name);
         return -1;
     }
 
@@ -319,8 +268,7 @@
 
     d.s = s;
     d.action = ACTION_INJECT_ERROR;
-    d.errp = &local_err;
-    qemu_opts_foreach(&inject_error_opts, add_rule, &d, 1);
+    qemu_opts_foreach(&inject_error_opts, add_rule, &d, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         ret = -EINVAL;
@@ -328,7 +276,7 @@
     }
 
     d.action = ACTION_SET_STATE;
-    qemu_opts_foreach(&set_state_opts, add_rule, &d, 1);
+    qemu_opts_foreach(&set_state_opts, add_rule, &d, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         ret = -EINVAL;
@@ -406,7 +354,6 @@
     BDRVBlkdebugState *s = bs->opaque;
     QemuOpts *opts;
     Error *local_err = NULL;
-    const char *config;
     uint64_t align;
     int ret;
 
@@ -419,8 +366,8 @@
     }
 
     /* Read rules from config file or command line options */
-    config = qemu_opt_get(opts, "config");
-    ret = read_config(s, config, options, errp);
+    s->config_file = g_strdup(qemu_opt_get(opts, "config"));
+    ret = read_config(s, s->config_file, options, errp);
     if (ret) {
         goto out;
     }
@@ -428,20 +375,20 @@
     /* Set initial state */
     s->state = 1;
 
-    /* Open the backing file */
-    assert(bs->file == NULL);
-    ret = bdrv_open_image(&bs->file, qemu_opt_get(opts, "x-image"), options, "image",
-                          flags | BDRV_O_PROTOCOL, false, &local_err);
-    if (ret < 0) {
+    /* Open the image file */
+    bs->file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options, "image",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
         goto out;
     }
 
     /* Set request alignment */
-    align = qemu_opt_get_size(opts, "align", bs->request_alignment);
-    if (align > 0 && align < INT_MAX && !(align & (align - 1))) {
-        bs->request_alignment = align;
-    } else {
+    align = qemu_opt_get_size(opts, "align", 0);
+    if (align < INT_MAX && is_power_of_2(align)) {
+        s->align = align;
+    } else if (align) {
         error_setg(errp, "Invalid alignment");
         ret = -EINVAL;
         goto fail_unref;
@@ -451,8 +398,11 @@
     goto out;
 
 fail_unref:
-    bdrv_unref(bs->file);
+    bdrv_unref_child(bs, bs->file);
 out:
+    if (ret < 0) {
+        g_free(s->config_file);
+    }
     qemu_opts_del(opts);
     return ret;
 }
@@ -472,12 +422,14 @@
     int error = rule->options.inject.error;
     struct BlkdebugAIOCB *acb;
     QEMUBH *bh;
+    bool immediately = rule->options.inject.immediately;
 
     if (rule->options.inject.once) {
-        QSIMPLEQ_INIT(&s->active_rules);
+        QSIMPLEQ_REMOVE(&s->active_rules, rule, BlkdebugRule, active_next);
+        remove_rule(rule);
     }
 
-    if (rule->options.inject.immediately) {
+    if (immediately) {
         return NULL;
     }
 
@@ -510,7 +462,8 @@
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_readv(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
+    return bdrv_aio_readv(bs->file, sector_num, qiov, nb_sectors,
+                          cb, opaque);
 }
 
 static BlockAIOCB *blkdebug_aio_writev(BlockDriverState *bs,
@@ -532,7 +485,8 @@
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors, cb, opaque);
+    return bdrv_aio_writev(bs->file, sector_num, qiov, nb_sectors,
+                           cb, opaque);
 }
 
 static BlockAIOCB *blkdebug_aio_flush(BlockDriverState *bs,
@@ -551,7 +505,7 @@
         return inject_error(bs, cb, opaque, rule);
     }
 
-    return bdrv_aio_flush(bs->file, cb, opaque);
+    return bdrv_aio_flush(bs->file->bs, cb, opaque);
 }
 
 
@@ -561,11 +515,13 @@
     BlkdebugRule *rule, *next;
     int i;
 
-    for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
+    for (i = 0; i < BLKDBG__MAX; i++) {
         QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) {
             remove_rule(rule);
         }
     }
+
+    g_free(s->config_file);
 }
 
 static void suspend_request(BlockDriverState *bs, BlkdebugRule *rule)
@@ -581,9 +537,13 @@
     remove_rule(rule);
     QLIST_INSERT_HEAD(&s->suspended_reqs, &r, next);
 
-    printf("blkdebug: Suspended request '%s'\n", r.tag);
+    if (!qtest_enabled()) {
+        printf("blkdebug: Suspended request '%s'\n", r.tag);
+    }
     qemu_coroutine_yield();
-    printf("blkdebug: Resuming request '%s'\n", r.tag);
+    if (!qtest_enabled()) {
+        printf("blkdebug: Resuming request '%s'\n", r.tag);
+    }
 
     QLIST_REMOVE(&r, next);
     g_free(r.tag);
@@ -620,13 +580,13 @@
     return injected;
 }
 
-static void blkdebug_debug_event(BlockDriverState *bs, BlkDebugEvent event)
+static void blkdebug_debug_event(BlockDriverState *bs, BlkdebugEvent event)
 {
     BDRVBlkdebugState *s = bs->opaque;
     struct BlkdebugRule *rule, *next;
     bool injected;
 
-    assert((int)event >= 0 && event < BLKDBG_EVENT_MAX);
+    assert((int)event >= 0 && event < BLKDBG__MAX);
 
     injected = false;
     s->new_state = s->state;
@@ -641,7 +601,7 @@
 {
     BDRVBlkdebugState *s = bs->opaque;
     struct BlkdebugRule *rule;
-    BlkDebugEvent blkdebug_event;
+    BlkdebugEvent blkdebug_event;
 
     if (get_event_by_name(event, &blkdebug_event) < 0) {
         return -ENOENT;
@@ -668,7 +628,7 @@
 
     QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, next) {
         if (!strcmp(r->tag, tag)) {
-            qemu_coroutine_enter(r->co, NULL);
+            qemu_coroutine_enter(r->co);
             return 0;
         }
     }
@@ -683,7 +643,7 @@
     BlkdebugRule *rule, *next;
     int i, ret = -ENOENT;
 
-    for (i = 0; i < BLKDBG_EVENT_MAX; i++) {
+    for (i = 0; i < BLKDBG__MAX; i++) {
         QLIST_FOREACH_SAFE(rule, &s->rules[i], next, next) {
             if (rule->action == ACTION_SUSPEND &&
                 !strcmp(rule->options.suspend.tag, tag)) {
@@ -694,7 +654,7 @@
     }
     QLIST_FOREACH_SAFE(r, &s->suspended_reqs, next, r_next) {
         if (!strcmp(r->tag, tag)) {
-            qemu_coroutine_enter(r->co, NULL);
+            qemu_coroutine_enter(r->co);
             ret = 0;
         }
     }
@@ -716,101 +676,73 @@
 
 static int64_t blkdebug_getlength(BlockDriverState *bs)
 {
-    return bdrv_getlength(bs->file);
+    return bdrv_getlength(bs->file->bs);
 }
 
-static void blkdebug_refresh_filename(BlockDriverState *bs)
+static int blkdebug_truncate(BlockDriverState *bs, int64_t offset)
+{
+    return bdrv_truncate(bs->file->bs, offset);
+}
+
+static void blkdebug_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     BDRVBlkdebugState *s = bs->opaque;
-    struct BlkdebugRule *rule;
     QDict *opts;
-    QList *inject_error_list = NULL, *set_state_list = NULL;
-    QList *suspend_list = NULL;
-    int event;
+    const QDictEntry *e;
+    bool force_json = false;
 
-    if (!bs->file->full_open_options) {
+    for (e = qdict_first(options); e; e = qdict_next(options, e)) {
+        if (strcmp(qdict_entry_key(e), "config") &&
+            strcmp(qdict_entry_key(e), "x-image"))
+        {
+            force_json = true;
+            break;
+        }
+    }
+
+    if (force_json && !bs->file->bs->full_open_options) {
         /* The config file cannot be recreated, so creating a plain filename
          * is impossible */
         return;
     }
 
+    if (!force_json && bs->file->bs->exact_filename[0]) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename),
+                 "blkdebug:%s:%s", s->config_file ?: "",
+                 bs->file->bs->exact_filename);
+    }
+
     opts = qdict_new();
     qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkdebug")));
 
-    QINCREF(bs->file->full_open_options);
-    qdict_put_obj(opts, "image", QOBJECT(bs->file->full_open_options));
+    QINCREF(bs->file->bs->full_open_options);
+    qdict_put_obj(opts, "image", QOBJECT(bs->file->bs->full_open_options));
 
-    for (event = 0; event < BLKDBG_EVENT_MAX; event++) {
-        QLIST_FOREACH(rule, &s->rules[event], next) {
-            if (rule->action == ACTION_INJECT_ERROR) {
-                QDict *inject_error = qdict_new();
-
-                qdict_put_obj(inject_error, "event", QOBJECT(qstring_from_str(
-                              BlkdebugEvent_lookup[rule->event])));
-                qdict_put_obj(inject_error, "state",
-                              QOBJECT(qint_from_int(rule->state)));
-                qdict_put_obj(inject_error, "errno", QOBJECT(qint_from_int(
-                              rule->options.inject.error)));
-                qdict_put_obj(inject_error, "sector", QOBJECT(qint_from_int(
-                              rule->options.inject.sector)));
-                qdict_put_obj(inject_error, "once", QOBJECT(qbool_from_int(
-                              rule->options.inject.once)));
-                qdict_put_obj(inject_error, "immediately",
-                              QOBJECT(qbool_from_int(
-                              rule->options.inject.immediately)));
-
-                if (!inject_error_list) {
-                    inject_error_list = qlist_new();
-                }
-
-                qlist_append_obj(inject_error_list, QOBJECT(inject_error));
-            } else if (rule->action == ACTION_SET_STATE) {
-                QDict *set_state = qdict_new();
-
-                qdict_put_obj(set_state, "event", QOBJECT(qstring_from_str(
-                              BlkdebugEvent_lookup[rule->event])));
-                qdict_put_obj(set_state, "state",
-                              QOBJECT(qint_from_int(rule->state)));
-                qdict_put_obj(set_state, "new_state", QOBJECT(qint_from_int(
-                              rule->options.set_state.new_state)));
-
-                if (!set_state_list) {
-                    set_state_list = qlist_new();
-                }
-
-                qlist_append_obj(set_state_list, QOBJECT(set_state));
-            } else if (rule->action == ACTION_SUSPEND) {
-                QDict *suspend = qdict_new();
-
-                qdict_put_obj(suspend, "event", QOBJECT(qstring_from_str(
-                              BlkdebugEvent_lookup[rule->event])));
-                qdict_put_obj(suspend, "state",
-                              QOBJECT(qint_from_int(rule->state)));
-                qdict_put_obj(suspend, "tag", QOBJECT(qstring_from_str(
-                              rule->options.suspend.tag)));
-
-                if (!suspend_list) {
-                    suspend_list = qlist_new();
-                }
-
-                qlist_append_obj(suspend_list, QOBJECT(suspend));
-            }
+    for (e = qdict_first(options); e; e = qdict_next(options, e)) {
+        if (strcmp(qdict_entry_key(e), "x-image")) {
+            qobject_incref(qdict_entry_value(e));
+            qdict_put_obj(opts, qdict_entry_key(e), qdict_entry_value(e));
         }
     }
 
-    if (inject_error_list) {
-        qdict_put_obj(opts, "inject-error", QOBJECT(inject_error_list));
-    }
-    if (set_state_list) {
-        qdict_put_obj(opts, "set-state", QOBJECT(set_state_list));
-    }
-    if (suspend_list) {
-        qdict_put_obj(opts, "suspend", QOBJECT(suspend_list));
-    }
-
     bs->full_open_options = opts;
 }
 
+static void blkdebug_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    BDRVBlkdebugState *s = bs->opaque;
+
+    if (s->align) {
+        bs->bl.request_alignment = s->align;
+    }
+}
+
+static int blkdebug_reopen_prepare(BDRVReopenState *reopen_state,
+                                   BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static BlockDriver bdrv_blkdebug = {
     .format_name            = "blkdebug",
     .protocol_name          = "blkdebug",
@@ -819,8 +751,11 @@
     .bdrv_parse_filename    = blkdebug_parse_filename,
     .bdrv_file_open         = blkdebug_open,
     .bdrv_close             = blkdebug_close,
+    .bdrv_reopen_prepare    = blkdebug_reopen_prepare,
     .bdrv_getlength         = blkdebug_getlength,
+    .bdrv_truncate          = blkdebug_truncate,
     .bdrv_refresh_filename  = blkdebug_refresh_filename,
+    .bdrv_refresh_limits    = blkdebug_refresh_limits,
 
     .bdrv_aio_readv         = blkdebug_aio_readv,
     .bdrv_aio_writev        = blkdebug_aio_writev,
diff --git a/block/blkreplay.c b/block/blkreplay.c
new file mode 100755
index 0000000..30f9d5f
--- /dev/null
+++ b/block/blkreplay.c
@@ -0,0 +1,160 @@
+/*
+ * Block protocol for record/replay
+ *
+ * Copyright (c) 2010-2016 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "block/block_int.h"
+#include "sysemu/replay.h"
+#include "qapi/error.h"
+
+typedef struct Request {
+    Coroutine *co;
+    QEMUBH *bh;
+} Request;
+
+/* Next request id.
+   This counter is global, because requests from different
+   block devices should not get overlapping ids. */
+static uint64_t request_id;
+
+static int blkreplay_open(BlockDriverState *bs, QDict *options, int flags,
+                          Error **errp)
+{
+    Error *local_err = NULL;
+    int ret;
+
+    /* Open the image file */
+    bs->file = bdrv_open_child(NULL, options, "image",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    ret = 0;
+fail:
+    if (ret < 0) {
+        bdrv_unref_child(bs, bs->file);
+    }
+    return ret;
+}
+
+static void blkreplay_close(BlockDriverState *bs)
+{
+}
+
+static int64_t blkreplay_getlength(BlockDriverState *bs)
+{
+    return bdrv_getlength(bs->file->bs);
+}
+
+/* This bh is used for synchronization of return from coroutines.
+   It continues yielded coroutine which then finishes its execution.
+   BH is called adjusted to some replay checkpoint, therefore
+   record and replay will always finish coroutines deterministically.
+*/
+static void blkreplay_bh_cb(void *opaque)
+{
+    Request *req = opaque;
+    qemu_coroutine_enter(req->co);
+    qemu_bh_delete(req->bh);
+    g_free(req);
+}
+
+static void block_request_create(uint64_t reqid, BlockDriverState *bs,
+                                 Coroutine *co)
+{
+    Request *req = g_new(Request, 1);
+    *req = (Request) {
+        .co = co,
+        .bh = aio_bh_new(bdrv_get_aio_context(bs), blkreplay_bh_cb, req),
+    };
+    replay_block_event(req->bh, reqid);
+}
+
+static int coroutine_fn blkreplay_co_preadv(BlockDriverState *bs,
+    uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_preadv(bs->file, offset, bytes, qiov, flags);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_pwritev(BlockDriverState *bs,
+    uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_pwrite_zeroes(BlockDriverState *bs,
+    int64_t offset, int count, BdrvRequestFlags flags)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_pwrite_zeroes(bs->file, offset, count, flags);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_pdiscard(BlockDriverState *bs,
+                                              int64_t offset, int count)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_pdiscard(bs->file->bs, offset, count);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static int coroutine_fn blkreplay_co_flush(BlockDriverState *bs)
+{
+    uint64_t reqid = request_id++;
+    int ret = bdrv_co_flush(bs->file->bs);
+    block_request_create(reqid, bs, qemu_coroutine_self());
+    qemu_coroutine_yield();
+
+    return ret;
+}
+
+static BlockDriver bdrv_blkreplay = {
+    .format_name            = "blkreplay",
+    .protocol_name          = "blkreplay",
+    .instance_size          = 0,
+
+    .bdrv_file_open         = blkreplay_open,
+    .bdrv_close             = blkreplay_close,
+    .bdrv_getlength         = blkreplay_getlength,
+
+    .bdrv_co_preadv         = blkreplay_co_preadv,
+    .bdrv_co_pwritev        = blkreplay_co_pwritev,
+
+    .bdrv_co_pwrite_zeroes  = blkreplay_co_pwrite_zeroes,
+    .bdrv_co_pdiscard       = blkreplay_co_pdiscard,
+    .bdrv_co_flush          = blkreplay_co_flush,
+};
+
+static void bdrv_blkreplay_init(void)
+{
+    bdrv_register(&bdrv_blkreplay);
+}
+
+block_init(bdrv_blkreplay_init);
diff --git a/block/blkverify.c b/block/blkverify.c
index 438dff8..da62d75 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -7,14 +7,16 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdarg.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "block/block_int.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qstring.h"
+#include "qemu/cutils.h"
 
 typedef struct {
-    BlockDriverState *test_file;
+    BdrvChild *test_file;
 } BDRVBlkverifyState;
 
 typedef struct BlkverifyAIOCB BlkverifyAIOCB;
@@ -123,26 +125,29 @@
     }
 
     /* Open the raw file */
-    assert(bs->file == NULL);
-    ret = bdrv_open_image(&bs->file, qemu_opt_get(opts, "x-raw"), options,
-                          "raw", flags | BDRV_O_PROTOCOL, false, &local_err);
-    if (ret < 0) {
+    bs->file = bdrv_open_child(qemu_opt_get(opts, "x-raw"), options, "raw",
+                               bs, &child_file, false, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
         goto fail;
     }
 
     /* Open the test file */
-    assert(s->test_file == NULL);
-    ret = bdrv_open_image(&s->test_file, qemu_opt_get(opts, "x-image"), options,
-                          "test", flags, false, &local_err);
-    if (ret < 0) {
+    s->test_file = bdrv_open_child(qemu_opt_get(opts, "x-image"), options,
+                                   "test", bs, &child_format, false,
+                                   &local_err);
+    if (local_err) {
+        ret = -EINVAL;
         error_propagate(errp, local_err);
-        s->test_file = NULL;
         goto fail;
     }
 
     ret = 0;
 fail:
+    if (ret < 0) {
+        bdrv_unref_child(bs, bs->file);
+    }
     qemu_opts_del(opts);
     return ret;
 }
@@ -151,7 +156,7 @@
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_unref(s->test_file);
+    bdrv_unref_child(bs, s->test_file);
     s->test_file = NULL;
 }
 
@@ -159,7 +164,7 @@
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    return bdrv_getlength(s->test_file);
+    return bdrv_getlength(s->test_file->bs);
 }
 
 static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
@@ -238,7 +243,7 @@
                                             nb_sectors, cb, opaque);
 
     acb->verify = blkverify_verify_readv;
-    acb->buf = qemu_blockalign(bs->file, qiov->size);
+    acb->buf = qemu_blockalign(bs->file->bs, qiov->size);
     qemu_iovec_init(&acb->raw_qiov, acb->qiov->niov);
     qemu_iovec_clone(&acb->raw_qiov, qiov, acb->buf);
 
@@ -271,7 +276,7 @@
     BDRVBlkverifyState *s = bs->opaque;
 
     /* Only flush test file, the raw file is not important */
-    return bdrv_aio_flush(s->test_file, cb, opaque);
+    return bdrv_aio_flush(s->test_file->bs, cb, opaque);
 }
 
 static bool blkverify_recurse_is_first_non_filter(BlockDriverState *bs,
@@ -279,54 +284,44 @@
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bool perm = bdrv_recurse_is_first_non_filter(bs->file, candidate);
+    bool perm = bdrv_recurse_is_first_non_filter(bs->file->bs, candidate);
 
     if (perm) {
         return true;
     }
 
-    return bdrv_recurse_is_first_non_filter(s->test_file, candidate);
+    return bdrv_recurse_is_first_non_filter(s->test_file->bs, candidate);
 }
 
-/* Propagate AioContext changes to ->test_file */
-static void blkverify_detach_aio_context(BlockDriverState *bs)
+static void blkverify_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     BDRVBlkverifyState *s = bs->opaque;
 
-    bdrv_detach_aio_context(s->test_file);
-}
+    /* bs->file->bs has already been refreshed */
+    bdrv_refresh_filename(s->test_file->bs);
 
-static void blkverify_attach_aio_context(BlockDriverState *bs,
-                                         AioContext *new_context)
-{
-    BDRVBlkverifyState *s = bs->opaque;
-
-    bdrv_attach_aio_context(s->test_file, new_context);
-}
-
-static void blkverify_refresh_filename(BlockDriverState *bs)
-{
-    BDRVBlkverifyState *s = bs->opaque;
-
-    /* bs->file has already been refreshed */
-    bdrv_refresh_filename(s->test_file);
-
-    if (bs->file->full_open_options && s->test_file->full_open_options) {
+    if (bs->file->bs->full_open_options
+        && s->test_file->bs->full_open_options)
+    {
         QDict *opts = qdict_new();
         qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("blkverify")));
 
-        QINCREF(bs->file->full_open_options);
-        qdict_put_obj(opts, "raw", QOBJECT(bs->file->full_open_options));
-        QINCREF(s->test_file->full_open_options);
-        qdict_put_obj(opts, "test", QOBJECT(s->test_file->full_open_options));
+        QINCREF(bs->file->bs->full_open_options);
+        qdict_put_obj(opts, "raw", QOBJECT(bs->file->bs->full_open_options));
+        QINCREF(s->test_file->bs->full_open_options);
+        qdict_put_obj(opts, "test",
+                      QOBJECT(s->test_file->bs->full_open_options));
 
         bs->full_open_options = opts;
     }
 
-    if (bs->file->exact_filename[0] && s->test_file->exact_filename[0]) {
+    if (bs->file->bs->exact_filename[0]
+        && s->test_file->bs->exact_filename[0])
+    {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
                  "blkverify:%s:%s",
-                 bs->file->exact_filename, s->test_file->exact_filename);
+                 bs->file->bs->exact_filename,
+                 s->test_file->bs->exact_filename);
     }
 }
 
@@ -345,9 +340,6 @@
     .bdrv_aio_writev                  = blkverify_aio_writev,
     .bdrv_aio_flush                   = blkverify_aio_flush,
 
-    .bdrv_attach_aio_context          = blkverify_attach_aio_context,
-    .bdrv_detach_aio_context          = blkverify_detach_aio_context,
-
     .is_filter                        = true,
     .bdrv_recurse_is_first_non_filter = blkverify_recurse_is_first_non_filter,
 };
diff --git a/block/block-backend.c b/block/block-backend.c
index d0692b1..effa038 100644
--- a/block/block-backend.c
+++ b/block/block-backend.c
@@ -1,7 +1,7 @@
 /*
  * QEMU Block backends
  *
- * Copyright (C) 2014 Red Hat, Inc.
+ * Copyright (C) 2014-2016 Red Hat, Inc.
  *
  * Authors:
  *  Markus Armbruster <armbru@redhat.com>,
@@ -10,103 +10,177 @@
  * or later.  See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
+#include "block/blockjob.h"
+#include "block/throttle-groups.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/sysemu.h"
 #include "qapi-event.h"
+#include "qemu/id.h"
+#include "trace.h"
 
 /* Number of coroutines to reserve per attached device model */
 #define COROUTINE_POOL_RESERVATION 64
 
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+
+static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb);
+
 struct BlockBackend {
     char *name;
     int refcnt;
-    BlockDriverState *bs;
+    BdrvChild *root;
     DriveInfo *legacy_dinfo;    /* null unless created by drive_new() */
-    QTAILQ_ENTRY(BlockBackend) link; /* for blk_backends */
+    QTAILQ_ENTRY(BlockBackend) link;         /* for block_backends */
+    QTAILQ_ENTRY(BlockBackend) monitor_link; /* for monitor_block_backends */
+    BlockBackendPublic public;
 
     void *dev;                  /* attached device model, if any */
     /* TODO change to DeviceState when all users are qdevified */
     const BlockDevOps *dev_ops;
     void *dev_opaque;
+
+    /* the block size for which the guest device expects atomicity */
+    int guest_block_size;
+
+    /* If the BDS tree is removed, some of its options are stored here (which
+     * can be used to restore those options in the new BDS on insert) */
+    BlockBackendRootState root_state;
+
+    bool enable_write_cache;
+
+    /* I/O stats (display with "info blockstats"). */
+    BlockAcctStats stats;
+
+    BlockdevOnError on_read_error, on_write_error;
+    bool iostatus_enabled;
+    BlockDeviceIoStatus iostatus;
+
+    bool allow_write_beyond_eof;
+
+    NotifierList remove_bs_notifiers, insert_bs_notifiers;
+};
+
+typedef struct BlockBackendAIOCB {
+    BlockAIOCB common;
+    QEMUBH *bh;
+    BlockBackend *blk;
+    int ret;
+} BlockBackendAIOCB;
+
+static const AIOCBInfo block_backend_aiocb_info = {
+    .get_aio_context = blk_aiocb_get_aio_context,
+    .aiocb_size = sizeof(BlockBackendAIOCB),
 };
 
 static void drive_info_del(DriveInfo *dinfo);
+static BlockBackend *bdrv_first_blk(BlockDriverState *bs);
 
-/* All the BlockBackends (except for hidden ones) */
-static QTAILQ_HEAD(, BlockBackend) blk_backends =
-    QTAILQ_HEAD_INITIALIZER(blk_backends);
+/* All BlockBackends */
+static QTAILQ_HEAD(, BlockBackend) block_backends =
+    QTAILQ_HEAD_INITIALIZER(block_backends);
+
+/* All BlockBackends referenced by the monitor and which are iterated through by
+ * blk_next() */
+static QTAILQ_HEAD(, BlockBackend) monitor_block_backends =
+    QTAILQ_HEAD_INITIALIZER(monitor_block_backends);
+
+static void blk_root_inherit_options(int *child_flags, QDict *child_options,
+                                     int parent_flags, QDict *parent_options)
+{
+    /* We're not supposed to call this function for root nodes */
+    abort();
+}
+static void blk_root_drained_begin(BdrvChild *child);
+static void blk_root_drained_end(BdrvChild *child);
+
+static void blk_root_change_media(BdrvChild *child, bool load);
+static void blk_root_resize(BdrvChild *child);
+
+static const char *blk_root_get_name(BdrvChild *child)
+{
+    return blk_name(child->opaque);
+}
+
+static const BdrvChildRole child_root = {
+    .inherit_options    = blk_root_inherit_options,
+
+    .change_media       = blk_root_change_media,
+    .resize             = blk_root_resize,
+    .get_name           = blk_root_get_name,
+
+    .drained_begin      = blk_root_drained_begin,
+    .drained_end        = blk_root_drained_end,
+};
 
 /*
- * Create a new BlockBackend with @name, with a reference count of one.
- * @name must not be null or empty.
- * Fail if a BlockBackend with this name already exists.
+ * Create a new BlockBackend with a reference count of one.
  * Store an error through @errp on failure, unless it's null.
  * Return the new BlockBackend on success, null on failure.
  */
-BlockBackend *blk_new(const char *name, Error **errp)
+BlockBackend *blk_new(void)
 {
     BlockBackend *blk;
 
-    assert(name && name[0]);
-    if (!id_wellformed(name)) {
-        error_setg(errp, "Invalid device name");
-        return NULL;
-    }
-    if (blk_by_name(name)) {
-        error_setg(errp, "Device with id '%s' already exists", name);
-        return NULL;
-    }
-    if (bdrv_find_node(name)) {
-        error_setg(errp,
-                   "Device name '%s' conflicts with an existing node name",
-                   name);
-        return NULL;
-    }
-
     blk = g_new0(BlockBackend, 1);
-    blk->name = g_strdup(name);
     blk->refcnt = 1;
-    QTAILQ_INSERT_TAIL(&blk_backends, blk, link);
+    blk_set_enable_write_cache(blk, true);
+
+    qemu_co_queue_init(&blk->public.throttled_reqs[0]);
+    qemu_co_queue_init(&blk->public.throttled_reqs[1]);
+
+    notifier_list_init(&blk->remove_bs_notifiers);
+    notifier_list_init(&blk->insert_bs_notifiers);
+
+    QTAILQ_INSERT_TAIL(&block_backends, blk, link);
     return blk;
 }
 
 /*
- * Create a new BlockBackend with a new BlockDriverState attached.
- * Otherwise just like blk_new(), which see.
+ * Creates a new BlockBackend, opens a new BlockDriverState, and connects both.
+ *
+ * Just as with bdrv_open(), after having called this function the reference to
+ * @options belongs to the block layer (even on failure).
+ *
+ * TODO: Remove @filename and @flags; it should be possible to specify a whole
+ * BDS tree just by specifying the @options QDict (or @reference,
+ * alternatively). At the time of adding this function, this is not possible,
+ * though, so callers of this function have to be able to specify @filename and
+ * @flags.
  */
-BlockBackend *blk_new_with_bs(const char *name, Error **errp)
+BlockBackend *blk_new_open(const char *filename, const char *reference,
+                           QDict *options, int flags, Error **errp)
 {
     BlockBackend *blk;
     BlockDriverState *bs;
 
-    blk = blk_new(name, errp);
-    if (!blk) {
+    blk = blk_new();
+    bs = bdrv_open(filename, reference, options, flags, errp);
+    if (!bs) {
+        blk_unref(blk);
         return NULL;
     }
 
-    bs = bdrv_new_root();
-    blk->bs = bs;
-    bs->blk = blk;
+    blk->root = bdrv_root_attach_child(bs, "root", &child_root, blk);
+
     return blk;
 }
 
 static void blk_delete(BlockBackend *blk)
 {
     assert(!blk->refcnt);
+    assert(!blk->name);
     assert(!blk->dev);
-    if (blk->bs) {
-        assert(blk->bs->blk == blk);
-        blk->bs->blk = NULL;
-        bdrv_unref(blk->bs);
-        blk->bs = NULL;
+    if (blk->root) {
+        blk_remove_bs(blk);
     }
-    /* Avoid double-remove after blk_hide_on_behalf_of_do_drive_del() */
-    if (blk->name[0]) {
-        QTAILQ_REMOVE(&blk_backends, blk, link);
-    }
-    g_free(blk->name);
+    assert(QLIST_EMPTY(&blk->remove_bs_notifiers.notifiers));
+    assert(QLIST_EMPTY(&blk->insert_bs_notifiers.notifiers));
+    QTAILQ_REMOVE(&block_backends, blk, link);
     drive_info_del(blk->legacy_dinfo);
+    block_acct_cleanup(&blk->stats);
     g_free(blk);
 }
 
@@ -120,6 +194,11 @@
     g_free(dinfo);
 }
 
+int blk_get_refcnt(BlockBackend *blk)
+{
+    return blk ? blk->refcnt : 0;
+}
+
 /*
  * Increment @blk's reference count.
  * @blk must not be null.
@@ -145,7 +224,32 @@
 }
 
 /*
- * Return the BlockBackend after @blk.
+ * Behaves similarly to blk_next() but iterates over all BlockBackends, even the
+ * ones which are hidden (i.e. are not referenced by the monitor).
+ */
+static BlockBackend *blk_all_next(BlockBackend *blk)
+{
+    return blk ? QTAILQ_NEXT(blk, link)
+               : QTAILQ_FIRST(&block_backends);
+}
+
+void blk_remove_all_bs(void)
+{
+    BlockBackend *blk = NULL;
+
+    while ((blk = blk_all_next(blk)) != NULL) {
+        AioContext *ctx = blk_get_aio_context(blk);
+
+        aio_context_acquire(ctx);
+        if (blk->root) {
+            blk_remove_bs(blk);
+        }
+        aio_context_release(ctx);
+    }
+}
+
+/*
+ * Return the monitor-owned BlockBackend after @blk.
  * If @blk is null, return the first one.
  * Else, return @blk's next sibling, which may be null.
  *
@@ -156,17 +260,108 @@
  */
 BlockBackend *blk_next(BlockBackend *blk)
 {
-    return blk ? QTAILQ_NEXT(blk, link) : QTAILQ_FIRST(&blk_backends);
+    return blk ? QTAILQ_NEXT(blk, monitor_link)
+               : QTAILQ_FIRST(&monitor_block_backends);
+}
+
+/* Iterates over all top-level BlockDriverStates, i.e. BDSs that are owned by
+ * the monitor or attached to a BlockBackend */
+BlockDriverState *bdrv_next(BdrvNextIterator *it)
+{
+    BlockDriverState *bs;
+
+    /* First, return all root nodes of BlockBackends. In order to avoid
+     * returning a BDS twice when multiple BBs refer to it, we only return it
+     * if the BB is the first one in the parent list of the BDS. */
+    if (it->phase == BDRV_NEXT_BACKEND_ROOTS) {
+        do {
+            it->blk = blk_all_next(it->blk);
+            bs = it->blk ? blk_bs(it->blk) : NULL;
+        } while (it->blk && (bs == NULL || bdrv_first_blk(bs) != it->blk));
+
+        if (bs) {
+            return bs;
+        }
+        it->phase = BDRV_NEXT_MONITOR_OWNED;
+    }
+
+    /* Then return the monitor-owned BDSes without a BB attached. Ignore all
+     * BDSes that are attached to a BlockBackend here; they have been handled
+     * by the above block already */
+    do {
+        it->bs = bdrv_next_monitor_owned(it->bs);
+        bs = it->bs;
+    } while (bs && bdrv_has_blk(bs));
+
+    return bs;
+}
+
+BlockDriverState *bdrv_first(BdrvNextIterator *it)
+{
+    *it = (BdrvNextIterator) {
+        .phase = BDRV_NEXT_BACKEND_ROOTS,
+    };
+
+    return bdrv_next(it);
+}
+
+/*
+ * Add a BlockBackend into the list of backends referenced by the monitor, with
+ * the given @name acting as the handle for the monitor.
+ * Strictly for use by blockdev.c.
+ *
+ * @name must not be null or empty.
+ *
+ * Returns true on success and false on failure. In the latter case, an Error
+ * object is returned through @errp.
+ */
+bool monitor_add_blk(BlockBackend *blk, const char *name, Error **errp)
+{
+    assert(!blk->name);
+    assert(name && name[0]);
+
+    if (!id_wellformed(name)) {
+        error_setg(errp, "Invalid device name");
+        return false;
+    }
+    if (blk_by_name(name)) {
+        error_setg(errp, "Device with id '%s' already exists", name);
+        return false;
+    }
+    if (bdrv_find_node(name)) {
+        error_setg(errp,
+                   "Device name '%s' conflicts with an existing node name",
+                   name);
+        return false;
+    }
+
+    blk->name = g_strdup(name);
+    QTAILQ_INSERT_TAIL(&monitor_block_backends, blk, monitor_link);
+    return true;
+}
+
+/*
+ * Remove a BlockBackend from the list of backends referenced by the monitor.
+ * Strictly for use by blockdev.c.
+ */
+void monitor_remove_blk(BlockBackend *blk)
+{
+    if (!blk->name) {
+        return;
+    }
+
+    QTAILQ_REMOVE(&monitor_block_backends, blk, monitor_link);
+    g_free(blk->name);
+    blk->name = NULL;
 }
 
 /*
  * Return @blk's name, a non-null string.
- * Wart: the name is empty iff @blk has been hidden with
- * blk_hide_on_behalf_of_do_drive_del().
+ * Returns an empty string iff @blk is not referenced by the monitor.
  */
 const char *blk_name(BlockBackend *blk)
 {
-    return blk->name;
+    return blk->name ?: "";
 }
 
 /*
@@ -175,10 +370,10 @@
  */
 BlockBackend *blk_by_name(const char *name)
 {
-    BlockBackend *blk;
+    BlockBackend *blk = NULL;
 
     assert(name);
-    QTAILQ_FOREACH(blk, &blk_backends, link) {
+    while ((blk = blk_next(blk)) != NULL) {
         if (!strcmp(name, blk->name)) {
             return blk;
         }
@@ -191,7 +386,27 @@
  */
 BlockDriverState *blk_bs(BlockBackend *blk)
 {
-    return blk->bs;
+    return blk->root ? blk->root->bs : NULL;
+}
+
+static BlockBackend *bdrv_first_blk(BlockDriverState *bs)
+{
+    BdrvChild *child;
+    QLIST_FOREACH(child, &bs->parents, next_parent) {
+        if (child->role == &child_root) {
+            return child->opaque;
+        }
+    }
+
+    return NULL;
+}
+
+/*
+ * Returns true if @bs has an associated BlockBackend.
+ */
+bool bdrv_has_blk(BlockDriverState *bs)
+{
+    return bdrv_first_blk(bs) != NULL;
 }
 
 /*
@@ -219,9 +434,9 @@
  */
 BlockBackend *blk_by_legacy_dinfo(DriveInfo *dinfo)
 {
-    BlockBackend *blk;
+    BlockBackend *blk = NULL;
 
-    QTAILQ_FOREACH(blk, &blk_backends, link) {
+    while ((blk = blk_next(blk)) != NULL) {
         if (blk->legacy_dinfo == dinfo) {
             return blk;
         }
@@ -230,20 +445,49 @@
 }
 
 /*
- * Hide @blk.
- * @blk must not have been hidden already.
- * Make attached BlockDriverState, if any, anonymous.
- * Once hidden, @blk is invisible to all functions that don't receive
- * it as argument.  For example, blk_by_name() won't return it.
- * Strictly for use by do_drive_del().
- * TODO get rid of it!
+ * Returns a pointer to the publicly accessible fields of @blk.
  */
-void blk_hide_on_behalf_of_do_drive_del(BlockBackend *blk)
+BlockBackendPublic *blk_get_public(BlockBackend *blk)
 {
-    QTAILQ_REMOVE(&blk_backends, blk, link);
-    blk->name[0] = 0;
-    if (blk->bs) {
-        bdrv_make_anon(blk->bs);
+    return &blk->public;
+}
+
+/*
+ * Returns a BlockBackend given the associated @public fields.
+ */
+BlockBackend *blk_by_public(BlockBackendPublic *public)
+{
+    return container_of(public, BlockBackend, public);
+}
+
+/*
+ * Disassociates the currently associated BlockDriverState from @blk.
+ */
+void blk_remove_bs(BlockBackend *blk)
+{
+    notifier_list_notify(&blk->remove_bs_notifiers, blk);
+    if (blk->public.throttle_state) {
+        throttle_timers_detach_aio_context(&blk->public.throttle_timers);
+    }
+
+    blk_update_root_state(blk);
+
+    bdrv_root_unref_child(blk->root);
+    blk->root = NULL;
+}
+
+/*
+ * Associates a new BlockDriverState with @blk.
+ */
+void blk_insert_bs(BlockBackend *blk, BlockDriverState *bs)
+{
+    bdrv_ref(bs);
+    blk->root = bdrv_root_attach_child(bs, "root", &child_root, blk);
+
+    notifier_list_notify(&blk->insert_bs_notifiers, blk);
+    if (blk->public.throttle_state) {
+        throttle_timers_attach_aio_context(
+            &blk->public.throttle_timers, bdrv_get_aio_context(bs));
     }
 }
 
@@ -259,10 +503,7 @@
     }
     blk_ref(blk);
     blk->dev = dev;
-    bdrv_iostatus_reset(blk->bs);
-
-    /* We're expecting I/O from the device so bump up coroutine pool size */
-    qemu_coroutine_adjust_pool_size(COROUTINE_POOL_RESERVATION);
+    blk_iostatus_reset(blk);
     return 0;
 }
 
@@ -289,8 +530,7 @@
     blk->dev = NULL;
     blk->dev_ops = NULL;
     blk->dev_opaque = NULL;
-    bdrv_set_guest_block_size(blk->bs, 512);
-    qemu_coroutine_adjust_pool_size(-COROUTINE_POOL_RESERVATION);
+    blk->guest_block_size = 512;
     blk_unref(blk);
 }
 
@@ -324,22 +564,24 @@
 void blk_dev_change_media_cb(BlockBackend *blk, bool load)
 {
     if (blk->dev_ops && blk->dev_ops->change_media_cb) {
-        bool tray_was_closed = !blk_dev_is_tray_open(blk);
+        bool tray_was_open, tray_is_open;
 
+        tray_was_open = blk_dev_is_tray_open(blk);
         blk->dev_ops->change_media_cb(blk->dev_opaque, load);
-        if (tray_was_closed) {
-            /* tray open */
-            qapi_event_send_device_tray_moved(blk_name(blk),
-                                              true, &error_abort);
-        }
-        if (load) {
-            /* tray close */
-            qapi_event_send_device_tray_moved(blk_name(blk),
-                                              false, &error_abort);
+        tray_is_open = blk_dev_is_tray_open(blk);
+
+        if (tray_was_open != tray_is_open) {
+            qapi_event_send_device_tray_moved(blk_name(blk), tray_is_open,
+                                              &error_abort);
         }
     }
 }
 
+static void blk_root_change_media(BdrvChild *child, bool load)
+{
+    blk_dev_change_media_cb(child->opaque, load);
+}
+
 /*
  * Does @blk's attached device model have removable media?
  * %true if no device model is attached.
@@ -350,6 +592,14 @@
 }
 
 /*
+ * Does @blk's attached device model have a tray?
+ */
+bool blk_dev_has_tray(BlockBackend *blk)
+{
+    return blk->dev_ops && blk->dev_ops->is_tray_open;
+}
+
+/*
  * Notify @blk's attached device model of a media eject request.
  * If @force is true, the medium is about to be yanked out forcefully.
  */
@@ -365,7 +615,7 @@
  */
 bool blk_dev_is_tray_open(BlockBackend *blk)
 {
-    if (blk->dev_ops && blk->dev_ops->is_tray_open) {
+    if (blk_dev_has_tray(blk)) {
         return blk->dev_ops->is_tray_open(blk->dev_opaque);
     }
     return false;
@@ -386,8 +636,10 @@
 /*
  * Notify @blk's attached device model of a backend size change.
  */
-void blk_dev_resize_cb(BlockBackend *blk)
+static void blk_root_resize(BdrvChild *child)
 {
+    BlockBackend *blk = child->opaque;
+
     if (blk->dev_ops && blk->dev_ops->resize_cb) {
         blk->dev_ops->resize_cb(blk->dev_opaque);
     }
@@ -395,80 +647,434 @@
 
 void blk_iostatus_enable(BlockBackend *blk)
 {
-    bdrv_iostatus_enable(blk->bs);
+    blk->iostatus_enabled = true;
+    blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
 }
 
-int blk_read(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
-             int nb_sectors)
+/* The I/O status is only enabled if the drive explicitly
+ * enables it _and_ the VM is configured to stop on errors */
+bool blk_iostatus_is_enabled(const BlockBackend *blk)
 {
-    return bdrv_read(blk->bs, sector_num, buf, nb_sectors);
+    return (blk->iostatus_enabled &&
+           (blk->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
+            blk->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
+            blk->on_read_error == BLOCKDEV_ON_ERROR_STOP));
 }
 
-int blk_read_unthrottled(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
-                         int nb_sectors)
+BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk)
 {
-    return bdrv_read_unthrottled(blk->bs, sector_num, buf, nb_sectors);
+    return blk->iostatus;
 }
 
-int blk_write(BlockBackend *blk, int64_t sector_num, const uint8_t *buf,
-              int nb_sectors)
+void blk_iostatus_disable(BlockBackend *blk)
 {
-    return bdrv_write(blk->bs, sector_num, buf, nb_sectors);
+    blk->iostatus_enabled = false;
 }
 
-BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
-                                 int nb_sectors, BdrvRequestFlags flags,
-                                 BlockCompletionFunc *cb, void *opaque)
+void blk_iostatus_reset(BlockBackend *blk)
 {
-    return bdrv_aio_write_zeroes(blk->bs, sector_num, nb_sectors, flags,
-                                 cb, opaque);
+    if (blk_iostatus_is_enabled(blk)) {
+        BlockDriverState *bs = blk_bs(blk);
+        blk->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+        if (bs && bs->job) {
+            block_job_iostatus_reset(bs->job);
+        }
+    }
+}
+
+void blk_iostatus_set_err(BlockBackend *blk, int error)
+{
+    assert(blk_iostatus_is_enabled(blk));
+    if (blk->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+        blk->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
+                                          BLOCK_DEVICE_IO_STATUS_FAILED;
+    }
+}
+
+void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow)
+{
+    blk->allow_write_beyond_eof = allow;
+}
+
+static int blk_check_byte_request(BlockBackend *blk, int64_t offset,
+                                  size_t size)
+{
+    int64_t len;
+
+    if (size > INT_MAX) {
+        return -EIO;
+    }
+
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    if (offset < 0) {
+        return -EIO;
+    }
+
+    if (!blk->allow_write_beyond_eof) {
+        len = blk_getlength(blk);
+        if (len < 0) {
+            return len;
+        }
+
+        if (offset > len || len - offset < size) {
+            return -EIO;
+        }
+    }
+
+    return 0;
+}
+
+static int blk_check_request(BlockBackend *blk, int64_t sector_num,
+                             int nb_sectors)
+{
+    if (sector_num < 0 || sector_num > INT64_MAX / BDRV_SECTOR_SIZE) {
+        return -EIO;
+    }
+
+    if (nb_sectors < 0 || nb_sectors > INT_MAX / BDRV_SECTOR_SIZE) {
+        return -EIO;
+    }
+
+    return blk_check_byte_request(blk, sector_num * BDRV_SECTOR_SIZE,
+                                  nb_sectors * BDRV_SECTOR_SIZE);
+}
+
+int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
+                               unsigned int bytes, QEMUIOVector *qiov,
+                               BdrvRequestFlags flags)
+{
+    int ret;
+
+    trace_blk_co_preadv(blk, blk_bs(blk), offset, bytes, flags);
+
+    ret = blk_check_byte_request(blk, offset, bytes);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* throttling disk I/O */
+    if (blk->public.throttle_state) {
+        throttle_group_co_io_limits_intercept(blk, bytes, false);
+    }
+
+    return bdrv_co_preadv(blk->root, offset, bytes, qiov, flags);
+}
+
+int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
+                                unsigned int bytes, QEMUIOVector *qiov,
+                                BdrvRequestFlags flags)
+{
+    int ret;
+
+    trace_blk_co_pwritev(blk, blk_bs(blk), offset, bytes, flags);
+
+    ret = blk_check_byte_request(blk, offset, bytes);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* throttling disk I/O */
+    if (blk->public.throttle_state) {
+        throttle_group_co_io_limits_intercept(blk, bytes, true);
+    }
+
+    if (!blk->enable_write_cache) {
+        flags |= BDRV_REQ_FUA;
+    }
+
+    return bdrv_co_pwritev(blk->root, offset, bytes, qiov, flags);
+}
+
+typedef struct BlkRwCo {
+    BlockBackend *blk;
+    int64_t offset;
+    QEMUIOVector *qiov;
+    int ret;
+    BdrvRequestFlags flags;
+} BlkRwCo;
+
+static void blk_read_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, rwco->qiov->size,
+                              rwco->qiov, rwco->flags);
+}
+
+static void blk_write_entry(void *opaque)
+{
+    BlkRwCo *rwco = opaque;
+
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, rwco->qiov->size,
+                               rwco->qiov, rwco->flags);
+}
+
+static int blk_prw(BlockBackend *blk, int64_t offset, uint8_t *buf,
+                   int64_t bytes, CoroutineEntry co_entry,
+                   BdrvRequestFlags flags)
+{
+    AioContext *aio_context;
+    QEMUIOVector qiov;
+    struct iovec iov;
+    Coroutine *co;
+    BlkRwCo rwco;
+
+    iov = (struct iovec) {
+        .iov_base = buf,
+        .iov_len = bytes,
+    };
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    rwco = (BlkRwCo) {
+        .blk    = blk,
+        .offset = offset,
+        .qiov   = &qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+
+    co = qemu_coroutine_create(co_entry, &rwco);
+    qemu_coroutine_enter(co);
+
+    aio_context = blk_get_aio_context(blk);
+    while (rwco.ret == NOT_DONE) {
+        aio_poll(aio_context, true);
+    }
+
+    return rwco.ret;
+}
+
+int blk_pread_unthrottled(BlockBackend *blk, int64_t offset, uint8_t *buf,
+                          int count)
+{
+    int ret;
+
+    ret = blk_check_byte_request(blk, offset, count);
+    if (ret < 0) {
+        return ret;
+    }
+
+    blk_root_drained_begin(blk->root);
+    ret = blk_pread(blk, offset, buf, count);
+    blk_root_drained_end(blk->root);
+    return ret;
+}
+
+int blk_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                      int count, BdrvRequestFlags flags)
+{
+    return blk_prw(blk, offset, NULL, count, blk_write_entry,
+                   flags | BDRV_REQ_ZERO_WRITE);
+}
+
+int blk_make_zero(BlockBackend *blk, BdrvRequestFlags flags)
+{
+    return bdrv_make_zero(blk->root, flags);
+}
+
+static void error_callback_bh(void *opaque)
+{
+    struct BlockBackendAIOCB *acb = opaque;
+    qemu_bh_delete(acb->bh);
+    acb->common.cb(acb->common.opaque, acb->ret);
+    qemu_aio_unref(acb);
+}
+
+BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque, int ret)
+{
+    struct BlockBackendAIOCB *acb;
+    QEMUBH *bh;
+
+    acb = blk_aio_get(&block_backend_aiocb_info, blk, cb, opaque);
+    acb->blk = blk;
+    acb->ret = ret;
+
+    bh = aio_bh_new(blk_get_aio_context(blk), error_callback_bh, acb);
+    acb->bh = bh;
+    qemu_bh_schedule(bh);
+
+    return &acb->common;
+}
+
+typedef struct BlkAioEmAIOCB {
+    BlockAIOCB common;
+    BlkRwCo rwco;
+    int bytes;
+    bool has_returned;
+    QEMUBH* bh;
+} BlkAioEmAIOCB;
+
+static const AIOCBInfo blk_aio_em_aiocb_info = {
+    .aiocb_size         = sizeof(BlkAioEmAIOCB),
+};
+
+static void blk_aio_complete(BlkAioEmAIOCB *acb)
+{
+    if (acb->bh) {
+        assert(acb->has_returned);
+        qemu_bh_delete(acb->bh);
+    }
+    if (acb->has_returned) {
+        acb->common.cb(acb->common.opaque, acb->rwco.ret);
+        qemu_aio_unref(acb);
+    }
+}
+
+static void blk_aio_complete_bh(void *opaque)
+{
+    blk_aio_complete(opaque);
+}
+
+static BlockAIOCB *blk_aio_prwv(BlockBackend *blk, int64_t offset, int bytes,
+                                QEMUIOVector *qiov, CoroutineEntry co_entry,
+                                BdrvRequestFlags flags,
+                                BlockCompletionFunc *cb, void *opaque)
+{
+    BlkAioEmAIOCB *acb;
+    Coroutine *co;
+
+    acb = blk_aio_get(&blk_aio_em_aiocb_info, blk, cb, opaque);
+    acb->rwco = (BlkRwCo) {
+        .blk    = blk,
+        .offset = offset,
+        .qiov   = qiov,
+        .flags  = flags,
+        .ret    = NOT_DONE,
+    };
+    acb->bytes = bytes;
+    acb->bh = NULL;
+    acb->has_returned = false;
+
+    co = qemu_coroutine_create(co_entry, acb);
+    qemu_coroutine_enter(co);
+
+    acb->has_returned = true;
+    if (acb->rwco.ret != NOT_DONE) {
+        acb->bh = aio_bh_new(blk_get_aio_context(blk), blk_aio_complete_bh, acb);
+        qemu_bh_schedule(acb->bh);
+    }
+
+    return &acb->common;
+}
+
+static void blk_aio_read_entry(void *opaque)
+{
+    BlkAioEmAIOCB *acb = opaque;
+    BlkRwCo *rwco = &acb->rwco;
+
+    assert(rwco->qiov->size == acb->bytes);
+    rwco->ret = blk_co_preadv(rwco->blk, rwco->offset, acb->bytes,
+                              rwco->qiov, rwco->flags);
+    blk_aio_complete(acb);
+}
+
+static void blk_aio_write_entry(void *opaque)
+{
+    BlkAioEmAIOCB *acb = opaque;
+    BlkRwCo *rwco = &acb->rwco;
+
+    assert(!rwco->qiov || rwco->qiov->size == acb->bytes);
+    rwco->ret = blk_co_pwritev(rwco->blk, rwco->offset, acb->bytes,
+                               rwco->qiov, rwco->flags);
+    blk_aio_complete(acb);
+}
+
+BlockAIOCB *blk_aio_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                                  int count, BdrvRequestFlags flags,
+                                  BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_prwv(blk, offset, count, NULL, blk_aio_write_entry,
+                        flags | BDRV_REQ_ZERO_WRITE, cb, opaque);
 }
 
 int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count)
 {
-    return bdrv_pread(blk->bs, offset, buf, count);
+    int ret = blk_prw(blk, offset, buf, count, blk_read_entry, 0);
+    if (ret < 0) {
+        return ret;
+    }
+    return count;
 }
 
-int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count)
+int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count,
+               BdrvRequestFlags flags)
 {
-    return bdrv_pwrite(blk->bs, offset, buf, count);
+    int ret = blk_prw(blk, offset, (void *) buf, count, blk_write_entry,
+                      flags);
+    if (ret < 0) {
+        return ret;
+    }
+    return count;
 }
 
 int64_t blk_getlength(BlockBackend *blk)
 {
-    return bdrv_getlength(blk->bs);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_getlength(blk_bs(blk));
 }
 
 void blk_get_geometry(BlockBackend *blk, uint64_t *nb_sectors_ptr)
 {
-    bdrv_get_geometry(blk->bs, nb_sectors_ptr);
+    if (!blk_bs(blk)) {
+        *nb_sectors_ptr = 0;
+    } else {
+        bdrv_get_geometry(blk_bs(blk), nb_sectors_ptr);
+    }
 }
 
-BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
-                          QEMUIOVector *iov, int nb_sectors,
-                          BlockCompletionFunc *cb, void *opaque)
+int64_t blk_nb_sectors(BlockBackend *blk)
 {
-    return bdrv_aio_readv(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_nb_sectors(blk_bs(blk));
 }
 
-BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
-                           QEMUIOVector *iov, int nb_sectors,
+BlockAIOCB *blk_aio_preadv(BlockBackend *blk, int64_t offset,
+                           QEMUIOVector *qiov, BdrvRequestFlags flags,
                            BlockCompletionFunc *cb, void *opaque)
 {
-    return bdrv_aio_writev(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
+    return blk_aio_prwv(blk, offset, qiov->size, qiov,
+                        blk_aio_read_entry, flags, cb, opaque);
+}
+
+BlockAIOCB *blk_aio_pwritev(BlockBackend *blk, int64_t offset,
+                            QEMUIOVector *qiov, BdrvRequestFlags flags,
+                            BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_prwv(blk, offset, qiov->size, qiov,
+                        blk_aio_write_entry, flags, cb, opaque);
 }
 
 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
                           BlockCompletionFunc *cb, void *opaque)
 {
-    return bdrv_aio_flush(blk->bs, cb, opaque);
+    if (!blk_is_available(blk)) {
+        return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
+    }
+
+    return bdrv_aio_flush(blk_bs(blk), cb, opaque);
 }
 
-BlockAIOCB *blk_aio_discard(BlockBackend *blk,
-                            int64_t sector_num, int nb_sectors,
-                            BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *blk_aio_pdiscard(BlockBackend *blk,
+                             int64_t offset, int count,
+                             BlockCompletionFunc *cb, void *opaque)
 {
-    return bdrv_aio_discard(blk->bs, sector_num, nb_sectors, cb, opaque);
+    int ret = blk_check_byte_request(blk, offset, count);
+    if (ret < 0) {
+        return blk_abort_aio_request(blk, cb, opaque, ret);
+    }
+
+    return bdrv_aio_pdiscard(blk_bs(blk), offset, count, cb, opaque);
 }
 
 void blk_aio_cancel(BlockAIOCB *acb)
@@ -481,30 +1087,58 @@
     bdrv_aio_cancel_async(acb);
 }
 
-int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs)
-{
-    return bdrv_aio_multiwrite(blk->bs, reqs, num_reqs);
-}
-
 int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
 {
-    return bdrv_ioctl(blk->bs, req, buf);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_ioctl(blk_bs(blk), req, buf);
 }
 
 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
                           BlockCompletionFunc *cb, void *opaque)
 {
-    return bdrv_aio_ioctl(blk->bs, req, buf, cb, opaque);
+    if (!blk_is_available(blk)) {
+        return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
+    }
+
+    return bdrv_aio_ioctl(blk_bs(blk), req, buf, cb, opaque);
+}
+
+int blk_co_pdiscard(BlockBackend *blk, int64_t offset, int count)
+{
+    int ret = blk_check_byte_request(blk, offset, count);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return bdrv_co_pdiscard(blk_bs(blk), offset, count);
+}
+
+int blk_co_flush(BlockBackend *blk)
+{
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_co_flush(blk_bs(blk));
 }
 
 int blk_flush(BlockBackend *blk)
 {
-    return bdrv_flush(blk->bs);
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_flush(blk_bs(blk));
 }
 
-int blk_flush_all(void)
+void blk_drain(BlockBackend *blk)
 {
-    return bdrv_flush_all();
+    if (blk_bs(blk)) {
+        bdrv_drain(blk_bs(blk));
+    }
 }
 
 void blk_drain_all(void)
@@ -512,116 +1146,329 @@
     bdrv_drain_all();
 }
 
+void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
+                      BlockdevOnError on_write_error)
+{
+    blk->on_read_error = on_read_error;
+    blk->on_write_error = on_write_error;
+}
+
 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read)
 {
-    return bdrv_get_on_error(blk->bs, is_read);
+    return is_read ? blk->on_read_error : blk->on_write_error;
 }
 
 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
                                       int error)
 {
-    return bdrv_get_error_action(blk->bs, is_read, error);
+    BlockdevOnError on_err = blk_get_on_error(blk, is_read);
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        return (error == ENOSPC) ?
+               BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_STOP:
+        return BLOCK_ERROR_ACTION_STOP;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        return BLOCK_ERROR_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        return BLOCK_ERROR_ACTION_IGNORE;
+    case BLOCKDEV_ON_ERROR_AUTO:
+    default:
+        abort();
+    }
 }
 
+static void send_qmp_error_event(BlockBackend *blk,
+                                 BlockErrorAction action,
+                                 bool is_read, int error)
+{
+    IoOperationType optype;
+
+    optype = is_read ? IO_OPERATION_TYPE_READ : IO_OPERATION_TYPE_WRITE;
+    qapi_event_send_block_io_error(blk_name(blk), optype, action,
+                                   blk_iostatus_is_enabled(blk),
+                                   error == ENOSPC, strerror(error),
+                                   &error_abort);
+}
+
+/* This is done by device models because, while the block layer knows
+ * about the error, it does not know whether an operation comes from
+ * the device or the block layer (from a job, for example).
+ */
 void blk_error_action(BlockBackend *blk, BlockErrorAction action,
                       bool is_read, int error)
 {
-    bdrv_error_action(blk->bs, action, is_read, error);
+    assert(error >= 0);
+
+    if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* First set the iostatus, so that "info block" returns an iostatus
+         * that matches the events raised so far (an additional error iostatus
+         * is fine, but not a lost one).
+         */
+        blk_iostatus_set_err(blk, error);
+
+        /* Then raise the request to stop the VM and the event.
+         * qemu_system_vmstop_request_prepare has two effects.  First,
+         * it ensures that the STOP event always comes after the
+         * BLOCK_IO_ERROR event.  Second, it ensures that even if management
+         * can observe the STOP event and do a "cont" before the STOP
+         * event is issued, the VM will not stop.  In this case, vm_start()
+         * also ensures that the STOP/RESUME pair of events is emitted.
+         */
+        qemu_system_vmstop_request_prepare();
+        send_qmp_error_event(blk, action, is_read, error);
+        qemu_system_vmstop_request(RUN_STATE_IO_ERROR);
+    } else {
+        send_qmp_error_event(blk, action, is_read, error);
+    }
 }
 
 int blk_is_read_only(BlockBackend *blk)
 {
-    return bdrv_is_read_only(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bdrv_is_read_only(bs);
+    } else {
+        return blk->root_state.read_only;
+    }
 }
 
 int blk_is_sg(BlockBackend *blk)
 {
-    return bdrv_is_sg(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (!bs) {
+        return 0;
+    }
+
+    return bdrv_is_sg(bs);
 }
 
 int blk_enable_write_cache(BlockBackend *blk)
 {
-    return bdrv_enable_write_cache(blk->bs);
+    return blk->enable_write_cache;
 }
 
 void blk_set_enable_write_cache(BlockBackend *blk, bool wce)
 {
-    bdrv_set_enable_write_cache(blk->bs, wce);
+    blk->enable_write_cache = wce;
 }
 
-int blk_is_inserted(BlockBackend *blk)
+void blk_invalidate_cache(BlockBackend *blk, Error **errp)
 {
-    return bdrv_is_inserted(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (!bs) {
+        error_setg(errp, "Device '%s' has no medium", blk->name);
+        return;
+    }
+
+    bdrv_invalidate_cache(bs, errp);
+}
+
+bool blk_is_inserted(BlockBackend *blk)
+{
+    BlockDriverState *bs = blk_bs(blk);
+
+    return bs && bdrv_is_inserted(bs);
+}
+
+bool blk_is_available(BlockBackend *blk)
+{
+    return blk_is_inserted(blk) && !blk_dev_is_tray_open(blk);
 }
 
 void blk_lock_medium(BlockBackend *blk, bool locked)
 {
-    bdrv_lock_medium(blk->bs, locked);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_lock_medium(bs, locked);
+    }
 }
 
 void blk_eject(BlockBackend *blk, bool eject_flag)
 {
-    bdrv_eject(blk->bs, eject_flag);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_eject(bs, eject_flag);
+    }
 }
 
 int blk_get_flags(BlockBackend *blk)
 {
-    return bdrv_get_flags(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bdrv_get_flags(bs);
+    } else {
+        return blk->root_state.open_flags;
+    }
+}
+
+/* Returns the maximum transfer length, in bytes; guaranteed nonzero */
+uint32_t blk_get_max_transfer(BlockBackend *blk)
+{
+    BlockDriverState *bs = blk_bs(blk);
+    uint32_t max = 0;
+
+    if (bs) {
+        max = bs->bl.max_transfer;
+    }
+    return MIN_NON_ZERO(max, INT_MAX);
+}
+
+int blk_get_max_iov(BlockBackend *blk)
+{
+    return blk->root->bs->bl.max_iov;
 }
 
 void blk_set_guest_block_size(BlockBackend *blk, int align)
 {
-    bdrv_set_guest_block_size(blk->bs, align);
+    blk->guest_block_size = align;
+}
+
+void *blk_try_blockalign(BlockBackend *blk, size_t size)
+{
+    return qemu_try_blockalign(blk ? blk_bs(blk) : NULL, size);
 }
 
 void *blk_blockalign(BlockBackend *blk, size_t size)
 {
-    return qemu_blockalign(blk ? blk->bs : NULL, size);
+    return qemu_blockalign(blk ? blk_bs(blk) : NULL, size);
 }
 
 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp)
 {
-    return bdrv_op_is_blocked(blk->bs, op, errp);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (!bs) {
+        return false;
+    }
+
+    return bdrv_op_is_blocked(bs, op, errp);
 }
 
 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason)
 {
-    bdrv_op_unblock(blk->bs, op, reason);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_op_unblock(bs, op, reason);
+    }
 }
 
 void blk_op_block_all(BlockBackend *blk, Error *reason)
 {
-    bdrv_op_block_all(blk->bs, reason);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_op_block_all(bs, reason);
+    }
 }
 
 void blk_op_unblock_all(BlockBackend *blk, Error *reason)
 {
-    bdrv_op_unblock_all(blk->bs, reason);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_op_unblock_all(bs, reason);
+    }
 }
 
 AioContext *blk_get_aio_context(BlockBackend *blk)
 {
-    return bdrv_get_aio_context(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        return bdrv_get_aio_context(bs);
+    } else {
+        return qemu_get_aio_context();
+    }
+}
+
+static AioContext *blk_aiocb_get_aio_context(BlockAIOCB *acb)
+{
+    BlockBackendAIOCB *blk_acb = DO_UPCAST(BlockBackendAIOCB, common, acb);
+    return blk_get_aio_context(blk_acb->blk);
 }
 
 void blk_set_aio_context(BlockBackend *blk, AioContext *new_context)
 {
-    bdrv_set_aio_context(blk->bs, new_context);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        if (blk->public.throttle_state) {
+            throttle_timers_detach_aio_context(&blk->public.throttle_timers);
+        }
+        bdrv_set_aio_context(bs, new_context);
+        if (blk->public.throttle_state) {
+            throttle_timers_attach_aio_context(&blk->public.throttle_timers,
+                                               new_context);
+        }
+    }
+}
+
+void blk_add_aio_context_notifier(BlockBackend *blk,
+        void (*attached_aio_context)(AioContext *new_context, void *opaque),
+        void (*detach_aio_context)(void *opaque), void *opaque)
+{
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_add_aio_context_notifier(bs, attached_aio_context,
+                                      detach_aio_context, opaque);
+    }
+}
+
+void blk_remove_aio_context_notifier(BlockBackend *blk,
+                                     void (*attached_aio_context)(AioContext *,
+                                                                  void *),
+                                     void (*detach_aio_context)(void *),
+                                     void *opaque)
+{
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_remove_aio_context_notifier(bs, attached_aio_context,
+                                         detach_aio_context, opaque);
+    }
+}
+
+void blk_add_remove_bs_notifier(BlockBackend *blk, Notifier *notify)
+{
+    notifier_list_add(&blk->remove_bs_notifiers, notify);
+}
+
+void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify)
+{
+    notifier_list_add(&blk->insert_bs_notifiers, notify);
 }
 
 void blk_io_plug(BlockBackend *blk)
 {
-    bdrv_io_plug(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_io_plug(bs);
+    }
 }
 
 void blk_io_unplug(BlockBackend *blk)
 {
-    bdrv_io_unplug(blk->bs);
+    BlockDriverState *bs = blk_bs(blk);
+
+    if (bs) {
+        bdrv_io_unplug(bs);
+    }
 }
 
 BlockAcctStats *blk_get_stats(BlockBackend *blk)
 {
-    return bdrv_get_stats(blk->bs);
+    return &blk->stats;
 }
 
 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
@@ -629,3 +1476,231 @@
 {
     return qemu_aio_get(aiocb_info, blk_bs(blk), cb, opaque);
 }
+
+int coroutine_fn blk_co_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                                      int count, BdrvRequestFlags flags)
+{
+    return blk_co_pwritev(blk, offset, count, NULL,
+                          flags | BDRV_REQ_ZERO_WRITE);
+}
+
+int blk_write_compressed(BlockBackend *blk, int64_t sector_num,
+                         const uint8_t *buf, int nb_sectors)
+{
+    int ret = blk_check_request(blk, sector_num, nb_sectors);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return bdrv_write_compressed(blk_bs(blk), sector_num, buf, nb_sectors);
+}
+
+int blk_truncate(BlockBackend *blk, int64_t offset)
+{
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_truncate(blk_bs(blk), offset);
+}
+
+int blk_pdiscard(BlockBackend *blk, int64_t offset, int count)
+{
+    int ret = blk_check_byte_request(blk, offset, count);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return bdrv_pdiscard(blk_bs(blk), offset, count);
+}
+
+int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
+                     int64_t pos, int size)
+{
+    int ret;
+
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    ret = bdrv_save_vmstate(blk_bs(blk), buf, pos, size);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret == size && !blk->enable_write_cache) {
+        ret = bdrv_flush(blk_bs(blk));
+    }
+
+    return ret < 0 ? ret : size;
+}
+
+int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size)
+{
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_load_vmstate(blk_bs(blk), buf, pos, size);
+}
+
+int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz)
+{
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_probe_blocksizes(blk_bs(blk), bsz);
+}
+
+int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo)
+{
+    if (!blk_is_available(blk)) {
+        return -ENOMEDIUM;
+    }
+
+    return bdrv_probe_geometry(blk_bs(blk), geo);
+}
+
+/*
+ * Updates the BlockBackendRootState object with data from the currently
+ * attached BlockDriverState.
+ */
+void blk_update_root_state(BlockBackend *blk)
+{
+    assert(blk->root);
+
+    blk->root_state.open_flags    = blk->root->bs->open_flags;
+    blk->root_state.read_only     = blk->root->bs->read_only;
+    blk->root_state.detect_zeroes = blk->root->bs->detect_zeroes;
+}
+
+/*
+ * Applies the information in the root state to the given BlockDriverState. This
+ * does not include the flags which have to be specified for bdrv_open(), use
+ * blk_get_open_flags_from_root_state() to inquire them.
+ */
+void blk_apply_root_state(BlockBackend *blk, BlockDriverState *bs)
+{
+    bs->detect_zeroes = blk->root_state.detect_zeroes;
+}
+
+/*
+ * Returns the flags to be used for bdrv_open() of a BlockDriverState which is
+ * supposed to inherit the root state.
+ */
+int blk_get_open_flags_from_root_state(BlockBackend *blk)
+{
+    int bs_flags;
+
+    bs_flags = blk->root_state.read_only ? 0 : BDRV_O_RDWR;
+    bs_flags |= blk->root_state.open_flags & ~BDRV_O_RDWR;
+
+    return bs_flags;
+}
+
+BlockBackendRootState *blk_get_root_state(BlockBackend *blk)
+{
+    return &blk->root_state;
+}
+
+int blk_commit_all(void)
+{
+    BlockBackend *blk = NULL;
+
+    while ((blk = blk_all_next(blk)) != NULL) {
+        AioContext *aio_context = blk_get_aio_context(blk);
+
+        aio_context_acquire(aio_context);
+        if (blk_is_inserted(blk) && blk->root->bs->backing) {
+            int ret = bdrv_commit(blk->root->bs);
+            if (ret < 0) {
+                aio_context_release(aio_context);
+                return ret;
+            }
+        }
+        aio_context_release(aio_context);
+    }
+    return 0;
+}
+
+int blk_flush_all(void)
+{
+    BlockBackend *blk = NULL;
+    int result = 0;
+
+    while ((blk = blk_all_next(blk)) != NULL) {
+        AioContext *aio_context = blk_get_aio_context(blk);
+        int ret;
+
+        aio_context_acquire(aio_context);
+        if (blk_is_inserted(blk)) {
+            ret = blk_flush(blk);
+            if (ret < 0 && !result) {
+                result = ret;
+            }
+        }
+        aio_context_release(aio_context);
+    }
+
+    return result;
+}
+
+
+/* throttling disk I/O limits */
+void blk_set_io_limits(BlockBackend *blk, ThrottleConfig *cfg)
+{
+    throttle_group_config(blk, cfg);
+}
+
+void blk_io_limits_disable(BlockBackend *blk)
+{
+    assert(blk->public.throttle_state);
+    bdrv_drained_begin(blk_bs(blk));
+    throttle_group_unregister_blk(blk);
+    bdrv_drained_end(blk_bs(blk));
+}
+
+/* should be called before blk_set_io_limits if a limit is set */
+void blk_io_limits_enable(BlockBackend *blk, const char *group)
+{
+    assert(!blk->public.throttle_state);
+    throttle_group_register_blk(blk, group);
+}
+
+void blk_io_limits_update_group(BlockBackend *blk, const char *group)
+{
+    /* this BB is not part of any group */
+    if (!blk->public.throttle_state) {
+        return;
+    }
+
+    /* this BB is a part of the same group than the one we want */
+    if (!g_strcmp0(throttle_group_get_name(blk), group)) {
+        return;
+    }
+
+    /* need to change the group this bs belong to */
+    blk_io_limits_disable(blk);
+    blk_io_limits_enable(blk, group);
+}
+
+static void blk_root_drained_begin(BdrvChild *child)
+{
+    BlockBackend *blk = child->opaque;
+
+    /* Note that blk->root may not be accessible here yet if we are just
+     * attaching to a BlockDriverState that is drained. Use child instead. */
+
+    if (blk->public.io_limits_disabled++ == 0) {
+        throttle_group_restart_blk(blk);
+    }
+}
+
+static void blk_root_drained_end(BdrvChild *child)
+{
+    BlockBackend *blk = child->opaque;
+
+    assert(blk->public.io_limits_disabled);
+    --blk->public.io_limits_disabled;
+}
diff --git a/block/bochs.c b/block/bochs.c
index 199ac2b..8c9652e 100644
--- a/block/bochs.c
+++ b/block/bochs.c
@@ -22,9 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 
 /**************************************************************/
 
@@ -101,7 +104,7 @@
     struct bochs_header bochs;
     int ret;
 
-    bs->read_only = 1; // no write support yet
+    bs->read_only = true; /* no write support yet */
 
     ret = bdrv_pread(bs->file, 0, &bochs, sizeof(bochs));
     if (ret < 0) {
@@ -185,6 +188,11 @@
     return ret;
 }
 
+static void bochs_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    bs->bl.request_alignment = BDRV_SECTOR_SIZE; /* No sub-sector I/O */
+}
+
 static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
 {
     BDRVBochsState *s = bs->opaque;
@@ -219,38 +227,52 @@
     return bitmap_offset + (512 * (s->bitmap_blocks + extent_offset));
 }
 
-static int bochs_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+bochs_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+                QEMUIOVector *qiov, int flags)
 {
+    BDRVBochsState *s = bs->opaque;
+    uint64_t sector_num = offset >> BDRV_SECTOR_BITS;
+    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
+    uint64_t bytes_done = 0;
+    QEMUIOVector local_qiov;
     int ret;
 
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+
+    qemu_iovec_init(&local_qiov, qiov->niov);
+    qemu_co_mutex_lock(&s->lock);
+
     while (nb_sectors > 0) {
         int64_t block_offset = seek_to_sector(bs, sector_num);
         if (block_offset < 0) {
-            return block_offset;
-        } else if (block_offset > 0) {
-            ret = bdrv_pread(bs->file, block_offset, buf, 512);
+            ret = block_offset;
+            goto fail;
+        }
+
+        qemu_iovec_reset(&local_qiov);
+        qemu_iovec_concat(&local_qiov, qiov, bytes_done, 512);
+
+        if (block_offset > 0) {
+            ret = bdrv_co_preadv(bs->file, block_offset, 512,
+                                 &local_qiov, 0);
             if (ret < 0) {
-                return ret;
+                goto fail;
             }
         } else {
-            memset(buf, 0, 512);
+            qemu_iovec_memset(&local_qiov, 0, 0, 512);
         }
         nb_sectors--;
         sector_num++;
-        buf += 512;
+        bytes_done += 512;
     }
-    return 0;
-}
 
-static coroutine_fn int bochs_co_read(BlockDriverState *bs, int64_t sector_num,
-                                      uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVBochsState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = bochs_read(bs, sector_num, buf, nb_sectors);
+    ret = 0;
+fail:
     qemu_co_mutex_unlock(&s->lock);
+    qemu_iovec_destroy(&local_qiov);
+
     return ret;
 }
 
@@ -265,7 +287,8 @@
     .instance_size	= sizeof(BDRVBochsState),
     .bdrv_probe		= bochs_probe,
     .bdrv_open		= bochs_open,
-    .bdrv_read          = bochs_co_read,
+    .bdrv_refresh_limits = bochs_refresh_limits,
+    .bdrv_co_preadv = bochs_co_preadv,
     .bdrv_close		= bochs_close,
 };
 
diff --git a/block/cloop.c b/block/cloop.c
index f328be0..7b75f7e 100644
--- a/block/cloop.c
+++ b/block/cloop.c
@@ -21,9 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 #include <zlib.h>
 
 /* Maximum compressed block size */
@@ -63,7 +66,7 @@
     uint32_t offsets_size, max_compressed_block_size = 1, i;
     int ret;
 
-    bs->read_only = 1;
+    bs->read_only = true;
 
     /* read header */
     ret = bdrv_pread(bs->file, 128, &s->block_size, 4);
@@ -195,6 +198,11 @@
     return ret;
 }
 
+static void cloop_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    bs->bl.request_alignment = BDRV_SECTOR_SIZE; /* No sub-sector I/O */
+}
+
 static inline int cloop_read_block(BlockDriverState *bs, int block_num)
 {
     BDRVCloopState *s = bs->opaque;
@@ -203,8 +211,8 @@
         int ret;
         uint32_t bytes = s->offsets[block_num + 1] - s->offsets[block_num];
 
-        ret = bdrv_pread(bs->file, s->offsets[block_num], s->compressed_block,
-                         bytes);
+        ret = bdrv_pread(bs->file, s->offsets[block_num],
+                         s->compressed_block, bytes);
         if (ret != bytes) {
             return -1;
         }
@@ -227,33 +235,38 @@
     return 0;
 }
 
-static int cloop_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+cloop_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+                QEMUIOVector *qiov, int flags)
 {
     BDRVCloopState *s = bs->opaque;
-    int i;
+    uint64_t sector_num = offset >> BDRV_SECTOR_BITS;
+    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
+    int ret, i;
+
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+
+    qemu_co_mutex_lock(&s->lock);
 
     for (i = 0; i < nb_sectors; i++) {
+        void *data;
         uint32_t sector_offset_in_block =
             ((sector_num + i) % s->sectors_per_block),
             block_num = (sector_num + i) / s->sectors_per_block;
         if (cloop_read_block(bs, block_num) != 0) {
-            return -1;
+            ret = -EIO;
+            goto fail;
         }
-        memcpy(buf + i * 512,
-            s->uncompressed_block + sector_offset_in_block * 512, 512);
-    }
-    return 0;
-}
 
-static coroutine_fn int cloop_co_read(BlockDriverState *bs, int64_t sector_num,
-                                      uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVCloopState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = cloop_read(bs, sector_num, buf, nb_sectors);
+        data = s->uncompressed_block + sector_offset_in_block * 512;
+        qemu_iovec_from_buf(qiov, i * 512, data, 512);
+    }
+
+    ret = 0;
+fail:
     qemu_co_mutex_unlock(&s->lock);
+
     return ret;
 }
 
@@ -271,7 +284,8 @@
     .instance_size  = sizeof(BDRVCloopState),
     .bdrv_probe     = cloop_probe,
     .bdrv_open      = cloop_open,
-    .bdrv_read      = cloop_co_read,
+    .bdrv_refresh_limits = cloop_refresh_limits,
+    .bdrv_co_preadv = cloop_co_preadv,
     .bdrv_close     = cloop_close,
 };
 
diff --git a/block/commit.c b/block/commit.c
index cfa2bbe..553e18d 100644
--- a/block/commit.c
+++ b/block/commit.c
@@ -12,10 +12,14 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 enum {
     /*
@@ -32,28 +36,36 @@
     BlockJob common;
     RateLimit limit;
     BlockDriverState *active;
-    BlockDriverState *top;
-    BlockDriverState *base;
+    BlockBackend *top;
+    BlockBackend *base;
     BlockdevOnError on_error;
     int base_flags;
     int orig_overlay_flags;
     char *backing_file_str;
 } CommitBlockJob;
 
-static int coroutine_fn commit_populate(BlockDriverState *bs,
-                                        BlockDriverState *base,
+static int coroutine_fn commit_populate(BlockBackend *bs, BlockBackend *base,
                                         int64_t sector_num, int nb_sectors,
                                         void *buf)
 {
     int ret = 0;
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base = buf,
+        .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
+    };
 
-    ret = bdrv_read(bs, sector_num, buf, nb_sectors);
-    if (ret) {
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    ret = blk_co_preadv(bs, sector_num * BDRV_SECTOR_SIZE,
+                        qiov.size, &qiov, 0);
+    if (ret < 0) {
         return ret;
     }
 
-    ret = bdrv_write(base, sector_num, buf, nb_sectors);
-    if (ret) {
+    ret = blk_co_pwritev(base, sector_num * BDRV_SECTOR_SIZE,
+                         qiov.size, &qiov, 0);
+    if (ret < 0) {
         return ret;
     }
 
@@ -69,8 +81,8 @@
     CommitBlockJob *s = container_of(job, CommitBlockJob, common);
     CommitCompleteData *data = opaque;
     BlockDriverState *active = s->active;
-    BlockDriverState *top = s->top;
-    BlockDriverState *base = s->base;
+    BlockDriverState *top = blk_bs(s->top);
+    BlockDriverState *base = blk_bs(s->base);
     BlockDriverState *overlay_bs;
     int ret = data->ret;
 
@@ -90,6 +102,8 @@
         bdrv_reopen(overlay_bs, s->orig_overlay_flags, NULL);
     }
     g_free(s->backing_file_str);
+    blk_unref(s->top);
+    blk_unref(s->base);
     block_job_completed(&s->common, ret);
     g_free(data);
 }
@@ -98,42 +112,39 @@
 {
     CommitBlockJob *s = opaque;
     CommitCompleteData *data;
-    BlockDriverState *top = s->top;
-    BlockDriverState *base = s->base;
     int64_t sector_num, end;
+    uint64_t delay_ns = 0;
     int ret = 0;
     int n = 0;
     void *buf = NULL;
     int bytes_written = 0;
     int64_t base_len;
 
-    ret = s->common.len = bdrv_getlength(top);
+    ret = s->common.len = blk_getlength(s->top);
 
 
     if (s->common.len < 0) {
         goto out;
     }
 
-    ret = base_len = bdrv_getlength(base);
+    ret = base_len = blk_getlength(s->base);
     if (base_len < 0) {
         goto out;
     }
 
     if (base_len < s->common.len) {
-        ret = bdrv_truncate(base, s->common.len);
+        ret = blk_truncate(s->base, s->common.len);
         if (ret) {
             goto out;
         }
     }
 
     end = s->common.len >> BDRV_SECTOR_BITS;
-    buf = qemu_blockalign(top, COMMIT_BUFFER_SIZE);
+    buf = blk_blockalign(s->top, COMMIT_BUFFER_SIZE);
 
     for (sector_num = 0; sector_num < end; sector_num += n) {
-        uint64_t delay_ns = 0;
         bool copy;
 
-wait:
         /* Note that even when no rate limit is applied we need to yield
          * with no pending I/O here so that bdrv_drain_all() returns.
          */
@@ -142,25 +153,20 @@
             break;
         }
         /* Copy if allocated above the base */
-        ret = bdrv_is_allocated_above(top, base, sector_num,
+        ret = bdrv_is_allocated_above(blk_bs(s->top), blk_bs(s->base),
+                                      sector_num,
                                       COMMIT_BUFFER_SIZE / BDRV_SECTOR_SIZE,
                                       &n);
         copy = (ret == 1);
         trace_commit_one_iteration(s, sector_num, n, ret);
         if (copy) {
-            if (s->common.speed) {
-                delay_ns = ratelimit_calculate_delay(&s->limit, n);
-                if (delay_ns > 0) {
-                    goto wait;
-                }
-            }
-            ret = commit_populate(top, base, sector_num, n, buf);
+            ret = commit_populate(s->top, s->base, sector_num, n, buf);
             bytes_written += n * BDRV_SECTOR_SIZE;
         }
         if (ret < 0) {
-            if (s->on_error == BLOCKDEV_ON_ERROR_STOP ||
-                s->on_error == BLOCKDEV_ON_ERROR_REPORT||
-                (s->on_error == BLOCKDEV_ON_ERROR_ENOSPC && ret == -ENOSPC)) {
+            BlockErrorAction action =
+                block_job_error_action(&s->common, false, s->on_error, -ret);
+            if (action == BLOCK_ERROR_ACTION_REPORT) {
                 goto out;
             } else {
                 n = 0;
@@ -169,6 +175,10 @@
         }
         /* Publish progress */
         s->common.offset += n * BDRV_SECTOR_SIZE;
+
+        if (copy && s->common.speed) {
+            delay_ns = ratelimit_calculate_delay(&s->limit, n);
+        }
     }
 
     ret = 0;
@@ -186,7 +196,7 @@
     CommitBlockJob *s = container_of(job, CommitBlockJob, common);
 
     if (speed < 0) {
-        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        error_setg(errp, QERR_INVALID_PARAMETER, "speed");
         return;
     }
     ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
@@ -198,8 +208,8 @@
     .set_speed     = commit_set_speed,
 };
 
-void commit_start(BlockDriverState *bs, BlockDriverState *base,
-                  BlockDriverState *top, int64_t speed,
+void commit_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *base, BlockDriverState *top, int64_t speed,
                   BlockdevOnError on_error, BlockCompletionFunc *cb,
                   void *opaque, const char *backing_file_str, Error **errp)
 {
@@ -210,13 +220,6 @@
     BlockDriverState *overlay_bs;
     Error *local_err = NULL;
 
-    if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
-         on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
-        error_setg(errp, "Invalid parameter combination");
-        return;
-    }
-
     assert(top != bs);
     if (top == base) {
         error_setg(errp, "Invalid files for merge: top and base are the same");
@@ -230,34 +233,40 @@
         return;
     }
 
+    s = block_job_create(job_id, &commit_job_driver, bs, speed,
+                         cb, opaque, errp);
+    if (!s) {
+        return;
+    }
+
     orig_base_flags    = bdrv_get_flags(base);
     orig_overlay_flags = bdrv_get_flags(overlay_bs);
 
     /* convert base & overlay_bs to r/w, if necessary */
-    if (!(orig_base_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, base,
-                                         orig_base_flags | BDRV_O_RDWR);
-    }
     if (!(orig_overlay_flags & BDRV_O_RDWR)) {
-        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs,
+        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs, NULL,
                                          orig_overlay_flags | BDRV_O_RDWR);
     }
+    if (!(orig_base_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, base, NULL,
+                                         orig_base_flags | BDRV_O_RDWR);
+    }
     if (reopen_queue) {
         bdrv_reopen_multiple(reopen_queue, &local_err);
         if (local_err != NULL) {
             error_propagate(errp, local_err);
+            block_job_unref(&s->common);
             return;
         }
     }
 
 
-    s = block_job_create(&commit_job_driver, bs, speed, cb, opaque, errp);
-    if (!s) {
-        return;
-    }
+    s->base = blk_new();
+    blk_insert_bs(s->base, base);
 
-    s->base   = base;
-    s->top    = top;
+    s->top = blk_new();
+    blk_insert_bs(s->top, top);
+
     s->active = bs;
 
     s->base_flags          = orig_base_flags;
@@ -266,8 +275,129 @@
     s->backing_file_str = g_strdup(backing_file_str);
 
     s->on_error = on_error;
-    s->common.co = qemu_coroutine_create(commit_run);
+    s->common.co = qemu_coroutine_create(commit_run, s);
 
     trace_commit_start(bs, base, top, s, s->common.co, opaque);
-    qemu_coroutine_enter(s->common.co, s);
+    qemu_coroutine_enter(s->common.co);
+}
+
+
+#define COMMIT_BUF_SECTORS 2048
+
+/* commit COW file into the raw image */
+int bdrv_commit(BlockDriverState *bs)
+{
+    BlockBackend *src, *backing;
+    BlockDriver *drv = bs->drv;
+    int64_t sector, total_sectors, length, backing_length;
+    int n, ro, open_flags;
+    int ret = 0;
+    uint8_t *buf = NULL;
+
+    if (!drv)
+        return -ENOMEDIUM;
+
+    if (!bs->backing) {
+        return -ENOTSUP;
+    }
+
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT_SOURCE, NULL) ||
+        bdrv_op_is_blocked(bs->backing->bs, BLOCK_OP_TYPE_COMMIT_TARGET, NULL)) {
+        return -EBUSY;
+    }
+
+    ro = bs->backing->bs->read_only;
+    open_flags =  bs->backing->bs->open_flags;
+
+    if (ro) {
+        if (bdrv_reopen(bs->backing->bs, open_flags | BDRV_O_RDWR, NULL)) {
+            return -EACCES;
+        }
+    }
+
+    src = blk_new();
+    blk_insert_bs(src, bs);
+
+    backing = blk_new();
+    blk_insert_bs(backing, bs->backing->bs);
+
+    length = blk_getlength(src);
+    if (length < 0) {
+        ret = length;
+        goto ro_cleanup;
+    }
+
+    backing_length = blk_getlength(backing);
+    if (backing_length < 0) {
+        ret = backing_length;
+        goto ro_cleanup;
+    }
+
+    /* If our top snapshot is larger than the backing file image,
+     * grow the backing file image if possible.  If not possible,
+     * we must return an error */
+    if (length > backing_length) {
+        ret = blk_truncate(backing, length);
+        if (ret < 0) {
+            goto ro_cleanup;
+        }
+    }
+
+    total_sectors = length >> BDRV_SECTOR_BITS;
+
+    /* blk_try_blockalign() for src will choose an alignment that works for
+     * backing as well, so no need to compare the alignment manually. */
+    buf = blk_try_blockalign(src, COMMIT_BUF_SECTORS * BDRV_SECTOR_SIZE);
+    if (buf == NULL) {
+        ret = -ENOMEM;
+        goto ro_cleanup;
+    }
+
+    for (sector = 0; sector < total_sectors; sector += n) {
+        ret = bdrv_is_allocated(bs, sector, COMMIT_BUF_SECTORS, &n);
+        if (ret < 0) {
+            goto ro_cleanup;
+        }
+        if (ret) {
+            ret = blk_pread(src, sector * BDRV_SECTOR_SIZE, buf,
+                            n * BDRV_SECTOR_SIZE);
+            if (ret < 0) {
+                goto ro_cleanup;
+            }
+
+            ret = blk_pwrite(backing, sector * BDRV_SECTOR_SIZE, buf,
+                             n * BDRV_SECTOR_SIZE, 0);
+            if (ret < 0) {
+                goto ro_cleanup;
+            }
+        }
+    }
+
+    if (drv->bdrv_make_empty) {
+        ret = drv->bdrv_make_empty(bs);
+        if (ret < 0) {
+            goto ro_cleanup;
+        }
+        blk_flush(src);
+    }
+
+    /*
+     * Make sure all data we wrote to the backing device is actually
+     * stable on disk.
+     */
+    blk_flush(backing);
+
+    ret = 0;
+ro_cleanup:
+    qemu_vfree(buf);
+
+    blk_unref(src);
+    blk_unref(backing);
+
+    if (ro) {
+        /* ignoring error return here */
+        bdrv_reopen(bs->backing->bs, open_flags & ~BDRV_O_RDWR, NULL);
+    }
+
+    return ret;
 }
diff --git a/block/crypto.c b/block/crypto.c
new file mode 100644
index 0000000..7f61e12
--- /dev/null
+++ b/block/crypto.c
@@ -0,0 +1,635 @@
+/*
+ * QEMU block full disk encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "crypto/block.h"
+#include "qapi/opts-visitor.h"
+#include "qapi-visit.h"
+#include "qapi/error.h"
+
+#define BLOCK_CRYPTO_OPT_LUKS_KEY_SECRET "key-secret"
+#define BLOCK_CRYPTO_OPT_LUKS_CIPHER_ALG "cipher-alg"
+#define BLOCK_CRYPTO_OPT_LUKS_CIPHER_MODE "cipher-mode"
+#define BLOCK_CRYPTO_OPT_LUKS_IVGEN_ALG "ivgen-alg"
+#define BLOCK_CRYPTO_OPT_LUKS_IVGEN_HASH_ALG "ivgen-hash-alg"
+#define BLOCK_CRYPTO_OPT_LUKS_HASH_ALG "hash-alg"
+
+typedef struct BlockCrypto BlockCrypto;
+
+struct BlockCrypto {
+    QCryptoBlock *block;
+};
+
+
+static int block_crypto_probe_generic(QCryptoBlockFormat format,
+                                      const uint8_t *buf,
+                                      int buf_size,
+                                      const char *filename)
+{
+    if (qcrypto_block_has_format(format, buf, buf_size)) {
+        return 100;
+    } else {
+        return 0;
+    }
+}
+
+
+static ssize_t block_crypto_read_func(QCryptoBlock *block,
+                                      size_t offset,
+                                      uint8_t *buf,
+                                      size_t buflen,
+                                      Error **errp,
+                                      void *opaque)
+{
+    BlockDriverState *bs = opaque;
+    ssize_t ret;
+
+    ret = bdrv_pread(bs->file, offset, buf, buflen);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not read encryption header");
+        return ret;
+    }
+    return ret;
+}
+
+
+struct BlockCryptoCreateData {
+    const char *filename;
+    QemuOpts *opts;
+    BlockBackend *blk;
+    uint64_t size;
+};
+
+
+static ssize_t block_crypto_write_func(QCryptoBlock *block,
+                                       size_t offset,
+                                       const uint8_t *buf,
+                                       size_t buflen,
+                                       Error **errp,
+                                       void *opaque)
+{
+    struct BlockCryptoCreateData *data = opaque;
+    ssize_t ret;
+
+    ret = blk_pwrite(data->blk, offset, buf, buflen, 0);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not write encryption header");
+        return ret;
+    }
+    return ret;
+}
+
+
+static ssize_t block_crypto_init_func(QCryptoBlock *block,
+                                      size_t headerlen,
+                                      Error **errp,
+                                      void *opaque)
+{
+    struct BlockCryptoCreateData *data = opaque;
+    int ret;
+
+    /* User provided size should reflect amount of space made
+     * available to the guest, so we must take account of that
+     * which will be used by the crypto header
+     */
+    data->size += headerlen;
+
+    qemu_opt_set_number(data->opts, BLOCK_OPT_SIZE, data->size, &error_abort);
+    ret = bdrv_create_file(data->filename, data->opts, errp);
+    if (ret < 0) {
+        return -1;
+    }
+
+    data->blk = blk_new_open(data->filename, NULL, NULL,
+                             BDRV_O_RDWR | BDRV_O_PROTOCOL, errp);
+    if (!data->blk) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static QemuOptsList block_crypto_runtime_opts_luks = {
+    .name = "crypto",
+    .head = QTAILQ_HEAD_INITIALIZER(block_crypto_runtime_opts_luks.head),
+    .desc = {
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_KEY_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the secret that provides the encryption key",
+        },
+        { /* end of list */ }
+    },
+};
+
+
+static QemuOptsList block_crypto_create_opts_luks = {
+    .name = "crypto",
+    .head = QTAILQ_HEAD_INITIALIZER(block_crypto_create_opts_luks.head),
+    .desc = {
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Virtual disk size"
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_KEY_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the secret that provides the encryption key",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_CIPHER_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of encryption cipher algorithm",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_CIPHER_MODE,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of encryption cipher mode",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_IVGEN_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of IV generator algorithm",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_IVGEN_HASH_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of IV generator hash algorithm",
+        },
+        {
+            .name = BLOCK_CRYPTO_OPT_LUKS_HASH_ALG,
+            .type = QEMU_OPT_STRING,
+            .help = "Name of encryption hash algorithm",
+        },
+        { /* end of list */ }
+    },
+};
+
+
+static QCryptoBlockOpenOptions *
+block_crypto_open_opts_init(QCryptoBlockFormat format,
+                            QemuOpts *opts,
+                            Error **errp)
+{
+    Visitor *v;
+    QCryptoBlockOpenOptions *ret = NULL;
+    Error *local_err = NULL;
+
+    ret = g_new0(QCryptoBlockOpenOptions, 1);
+    ret->format = format;
+
+    v = opts_visitor_new(opts);
+
+    visit_start_struct(v, NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    switch (format) {
+    case Q_CRYPTO_BLOCK_FORMAT_LUKS:
+        visit_type_QCryptoBlockOptionsLUKS_members(
+            v, &ret->u.luks, &local_err);
+        break;
+
+    default:
+        error_setg(&local_err, "Unsupported block format %d", format);
+        break;
+    }
+    if (!local_err) {
+        visit_check_struct(v, &local_err);
+    }
+
+    visit_end_struct(v, NULL);
+
+ out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+        qapi_free_QCryptoBlockOpenOptions(ret);
+        ret = NULL;
+    }
+    visit_free(v);
+    return ret;
+}
+
+
+static QCryptoBlockCreateOptions *
+block_crypto_create_opts_init(QCryptoBlockFormat format,
+                              QemuOpts *opts,
+                              Error **errp)
+{
+    Visitor *v;
+    QCryptoBlockCreateOptions *ret = NULL;
+    Error *local_err = NULL;
+
+    ret = g_new0(QCryptoBlockCreateOptions, 1);
+    ret->format = format;
+
+    v = opts_visitor_new(opts);
+
+    visit_start_struct(v, NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    switch (format) {
+    case Q_CRYPTO_BLOCK_FORMAT_LUKS:
+        visit_type_QCryptoBlockCreateOptionsLUKS_members(
+            v, &ret->u.luks, &local_err);
+        break;
+
+    default:
+        error_setg(&local_err, "Unsupported block format %d", format);
+        break;
+    }
+    if (!local_err) {
+        visit_check_struct(v, &local_err);
+    }
+
+    visit_end_struct(v, NULL);
+
+ out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+        qapi_free_QCryptoBlockCreateOptions(ret);
+        ret = NULL;
+    }
+    visit_free(v);
+    return ret;
+}
+
+
+static int block_crypto_open_generic(QCryptoBlockFormat format,
+                                     QemuOptsList *opts_spec,
+                                     BlockDriverState *bs,
+                                     QDict *options,
+                                     int flags,
+                                     Error **errp)
+{
+    BlockCrypto *crypto = bs->opaque;
+    QemuOpts *opts = NULL;
+    Error *local_err = NULL;
+    int ret = -EINVAL;
+    QCryptoBlockOpenOptions *open_opts = NULL;
+    unsigned int cflags = 0;
+
+    opts = qemu_opts_create(opts_spec, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto cleanup;
+    }
+
+    open_opts = block_crypto_open_opts_init(format, opts, errp);
+    if (!open_opts) {
+        goto cleanup;
+    }
+
+    if (flags & BDRV_O_NO_IO) {
+        cflags |= QCRYPTO_BLOCK_OPEN_NO_IO;
+    }
+    crypto->block = qcrypto_block_open(open_opts,
+                                       block_crypto_read_func,
+                                       bs,
+                                       cflags,
+                                       errp);
+
+    if (!crypto->block) {
+        ret = -EIO;
+        goto cleanup;
+    }
+
+    bs->encrypted = true;
+    bs->valid_key = true;
+
+    ret = 0;
+ cleanup:
+    qapi_free_QCryptoBlockOpenOptions(open_opts);
+    return ret;
+}
+
+
+static int block_crypto_create_generic(QCryptoBlockFormat format,
+                                       const char *filename,
+                                       QemuOpts *opts,
+                                       Error **errp)
+{
+    int ret = -EINVAL;
+    QCryptoBlockCreateOptions *create_opts = NULL;
+    QCryptoBlock *crypto = NULL;
+    struct BlockCryptoCreateData data = {
+        .size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                         BDRV_SECTOR_SIZE),
+        .opts = opts,
+        .filename = filename,
+    };
+
+    create_opts = block_crypto_create_opts_init(format, opts, errp);
+    if (!create_opts) {
+        return -1;
+    }
+
+    crypto = qcrypto_block_create(create_opts,
+                                  block_crypto_init_func,
+                                  block_crypto_write_func,
+                                  &data,
+                                  errp);
+
+    if (!crypto) {
+        ret = -EIO;
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    qcrypto_block_free(crypto);
+    blk_unref(data.blk);
+    qapi_free_QCryptoBlockCreateOptions(create_opts);
+    return ret;
+}
+
+static int block_crypto_truncate(BlockDriverState *bs, int64_t offset)
+{
+    BlockCrypto *crypto = bs->opaque;
+    size_t payload_offset =
+        qcrypto_block_get_payload_offset(crypto->block);
+
+    offset += payload_offset;
+
+    return bdrv_truncate(bs->file->bs, offset);
+}
+
+static void block_crypto_close(BlockDriverState *bs)
+{
+    BlockCrypto *crypto = bs->opaque;
+    qcrypto_block_free(crypto->block);
+}
+
+
+#define BLOCK_CRYPTO_MAX_SECTORS 32
+
+static coroutine_fn int
+block_crypto_co_readv(BlockDriverState *bs, int64_t sector_num,
+                      int remaining_sectors, QEMUIOVector *qiov)
+{
+    BlockCrypto *crypto = bs->opaque;
+    int cur_nr_sectors; /* number of sectors in current iteration */
+    uint64_t bytes_done = 0;
+    uint8_t *cipher_data = NULL;
+    QEMUIOVector hd_qiov;
+    int ret = 0;
+    size_t payload_offset =
+        qcrypto_block_get_payload_offset(crypto->block) / 512;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    /* Bounce buffer so we have a linear mem region for
+     * entire sector. XXX optimize so we avoid bounce
+     * buffer in case that qiov->niov == 1
+     */
+    cipher_data =
+        qemu_try_blockalign(bs->file->bs, MIN(BLOCK_CRYPTO_MAX_SECTORS * 512,
+                                              qiov->size));
+    if (cipher_data == NULL) {
+        ret = -ENOMEM;
+        goto cleanup;
+    }
+
+    while (remaining_sectors) {
+        cur_nr_sectors = remaining_sectors;
+
+        if (cur_nr_sectors > BLOCK_CRYPTO_MAX_SECTORS) {
+            cur_nr_sectors = BLOCK_CRYPTO_MAX_SECTORS;
+        }
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_add(&hd_qiov, cipher_data, cur_nr_sectors * 512);
+
+        ret = bdrv_co_readv(bs->file,
+                            payload_offset + sector_num,
+                            cur_nr_sectors, &hd_qiov);
+        if (ret < 0) {
+            goto cleanup;
+        }
+
+        if (qcrypto_block_decrypt(crypto->block,
+                                  sector_num,
+                                  cipher_data, cur_nr_sectors * 512,
+                                  NULL) < 0) {
+            ret = -EIO;
+            goto cleanup;
+        }
+
+        qemu_iovec_from_buf(qiov, bytes_done,
+                            cipher_data, cur_nr_sectors * 512);
+
+        remaining_sectors -= cur_nr_sectors;
+        sector_num += cur_nr_sectors;
+        bytes_done += cur_nr_sectors * 512;
+    }
+
+ cleanup:
+    qemu_iovec_destroy(&hd_qiov);
+    qemu_vfree(cipher_data);
+
+    return ret;
+}
+
+
+static coroutine_fn int
+block_crypto_co_writev(BlockDriverState *bs, int64_t sector_num,
+                       int remaining_sectors, QEMUIOVector *qiov)
+{
+    BlockCrypto *crypto = bs->opaque;
+    int cur_nr_sectors; /* number of sectors in current iteration */
+    uint64_t bytes_done = 0;
+    uint8_t *cipher_data = NULL;
+    QEMUIOVector hd_qiov;
+    int ret = 0;
+    size_t payload_offset =
+        qcrypto_block_get_payload_offset(crypto->block) / 512;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    /* Bounce buffer so we have a linear mem region for
+     * entire sector. XXX optimize so we avoid bounce
+     * buffer in case that qiov->niov == 1
+     */
+    cipher_data =
+        qemu_try_blockalign(bs->file->bs, MIN(BLOCK_CRYPTO_MAX_SECTORS * 512,
+                                              qiov->size));
+    if (cipher_data == NULL) {
+        ret = -ENOMEM;
+        goto cleanup;
+    }
+
+    while (remaining_sectors) {
+        cur_nr_sectors = remaining_sectors;
+
+        if (cur_nr_sectors > BLOCK_CRYPTO_MAX_SECTORS) {
+            cur_nr_sectors = BLOCK_CRYPTO_MAX_SECTORS;
+        }
+
+        qemu_iovec_to_buf(qiov, bytes_done,
+                          cipher_data, cur_nr_sectors * 512);
+
+        if (qcrypto_block_encrypt(crypto->block,
+                                  sector_num,
+                                  cipher_data, cur_nr_sectors * 512,
+                                  NULL) < 0) {
+            ret = -EIO;
+            goto cleanup;
+        }
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_add(&hd_qiov, cipher_data, cur_nr_sectors * 512);
+
+        ret = bdrv_co_writev(bs->file,
+                             payload_offset + sector_num,
+                             cur_nr_sectors, &hd_qiov);
+        if (ret < 0) {
+            goto cleanup;
+        }
+
+        remaining_sectors -= cur_nr_sectors;
+        sector_num += cur_nr_sectors;
+        bytes_done += cur_nr_sectors * 512;
+    }
+
+ cleanup:
+    qemu_iovec_destroy(&hd_qiov);
+    qemu_vfree(cipher_data);
+
+    return ret;
+}
+
+
+static int64_t block_crypto_getlength(BlockDriverState *bs)
+{
+    BlockCrypto *crypto = bs->opaque;
+    int64_t len = bdrv_getlength(bs->file->bs);
+
+    ssize_t offset = qcrypto_block_get_payload_offset(crypto->block);
+
+    len -= offset;
+
+    return len;
+}
+
+
+static int block_crypto_probe_luks(const uint8_t *buf,
+                                   int buf_size,
+                                   const char *filename) {
+    return block_crypto_probe_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+                                      buf, buf_size, filename);
+}
+
+static int block_crypto_open_luks(BlockDriverState *bs,
+                                  QDict *options,
+                                  int flags,
+                                  Error **errp)
+{
+    return block_crypto_open_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+                                     &block_crypto_runtime_opts_luks,
+                                     bs, options, flags, errp);
+}
+
+static int block_crypto_create_luks(const char *filename,
+                                    QemuOpts *opts,
+                                    Error **errp)
+{
+    return block_crypto_create_generic(Q_CRYPTO_BLOCK_FORMAT_LUKS,
+                                       filename, opts, errp);
+}
+
+static int block_crypto_get_info_luks(BlockDriverState *bs,
+                                      BlockDriverInfo *bdi)
+{
+    BlockDriverInfo subbdi;
+    int ret;
+
+    ret = bdrv_get_info(bs->file->bs, &subbdi);
+    if (ret != 0) {
+        return ret;
+    }
+
+    bdi->unallocated_blocks_are_zero = false;
+    bdi->can_write_zeroes_with_unmap = false;
+    bdi->cluster_size = subbdi.cluster_size;
+
+    return 0;
+}
+
+static ImageInfoSpecific *
+block_crypto_get_specific_info_luks(BlockDriverState *bs)
+{
+    BlockCrypto *crypto = bs->opaque;
+    ImageInfoSpecific *spec_info;
+    QCryptoBlockInfo *info;
+
+    info = qcrypto_block_get_info(crypto->block, NULL);
+    if (!info) {
+        return NULL;
+    }
+    if (info->format != Q_CRYPTO_BLOCK_FORMAT_LUKS) {
+        qapi_free_QCryptoBlockInfo(info);
+        return NULL;
+    }
+
+    spec_info = g_new(ImageInfoSpecific, 1);
+    spec_info->type = IMAGE_INFO_SPECIFIC_KIND_LUKS;
+    spec_info->u.luks.data = g_new(QCryptoBlockInfoLUKS, 1);
+    *spec_info->u.luks.data = info->u.luks;
+
+    /* Blank out pointers we've just stolen to avoid double free */
+    memset(&info->u.luks, 0, sizeof(info->u.luks));
+
+    qapi_free_QCryptoBlockInfo(info);
+
+    return spec_info;
+}
+
+BlockDriver bdrv_crypto_luks = {
+    .format_name        = "luks",
+    .instance_size      = sizeof(BlockCrypto),
+    .bdrv_probe         = block_crypto_probe_luks,
+    .bdrv_open          = block_crypto_open_luks,
+    .bdrv_close         = block_crypto_close,
+    .bdrv_create        = block_crypto_create_luks,
+    .bdrv_truncate      = block_crypto_truncate,
+    .create_opts        = &block_crypto_create_opts_luks,
+
+    .bdrv_co_readv      = block_crypto_co_readv,
+    .bdrv_co_writev     = block_crypto_co_writev,
+    .bdrv_getlength     = block_crypto_getlength,
+    .bdrv_get_info      = block_crypto_get_info_luks,
+    .bdrv_get_specific_info = block_crypto_get_specific_info_luks,
+};
+
+static void block_crypto_init(void)
+{
+    bdrv_register(&bdrv_crypto_luks);
+}
+
+block_init(block_crypto_init);
diff --git a/block/curl.c b/block/curl.c
index bbee3ca..426fb4d 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -21,19 +21,31 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "block/block_int.h"
 #include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qstring.h"
+#include "crypto/secret.h"
 #include <curl/curl.h>
+#include "qemu/cutils.h"
 
 // #define DEBUG_CURL
 // #define DEBUG_VERBOSE
 
 #ifdef DEBUG_CURL
-#define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
+#define DEBUG_CURL_PRINT 1
 #else
-#define DPRINTF(fmt, ...) do { } while (0)
+#define DEBUG_CURL_PRINT 0
 #endif
+#define DPRINTF(fmt, ...)                                            \
+    do {                                                             \
+        if (DEBUG_CURL_PRINT) {                                      \
+            fprintf(stderr, fmt, ## __VA_ARGS__);                    \
+        }                                                            \
+    } while (0)
 
 #if LIBCURL_VERSION_NUM >= 0x071000
 /* The multi interface timer callback was introduced in 7.16.0 */
@@ -75,6 +87,10 @@
 #define CURL_BLOCK_OPT_SSLVERIFY "sslverify"
 #define CURL_BLOCK_OPT_TIMEOUT "timeout"
 #define CURL_BLOCK_OPT_COOKIE    "cookie"
+#define CURL_BLOCK_OPT_USERNAME "username"
+#define CURL_BLOCK_OPT_PASSWORD_SECRET "password-secret"
+#define CURL_BLOCK_OPT_PROXY_USERNAME "proxy-username"
+#define CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET "proxy-password-secret"
 
 struct BDRVCURLState;
 
@@ -117,6 +133,10 @@
     char *cookie;
     bool accept_range;
     AioContext *aio_context;
+    char *username;
+    char *password;
+    char *proxyusername;
+    char *proxypassword;
 } BDRVCURLState;
 
 static void curl_clean_state(CURLState *s);
@@ -149,21 +169,23 @@
     state->sock_fd = fd;
     s = state->s;
 
-    DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
+    DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, (int)fd);
     switch (action) {
         case CURL_POLL_IN:
-            aio_set_fd_handler(s->aio_context, fd, curl_multi_read,
-                               NULL, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               curl_multi_read, NULL, state);
             break;
         case CURL_POLL_OUT:
-            aio_set_fd_handler(s->aio_context, fd, NULL, curl_multi_do, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               NULL, curl_multi_do, state);
             break;
         case CURL_POLL_INOUT:
-            aio_set_fd_handler(s->aio_context, fd, curl_multi_read,
-                               curl_multi_do, state);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               curl_multi_read, curl_multi_do, state);
             break;
         case CURL_POLL_REMOVE:
-            aio_set_fd_handler(s->aio_context, fd, NULL, NULL, NULL);
+            aio_set_fd_handler(s->aio_context, fd, false,
+                               NULL, NULL, NULL);
             break;
     }
 
@@ -297,6 +319,18 @@
             /* ACBs for successful messages get completed in curl_read_cb */
             if (msg->data.result != CURLE_OK) {
                 int i;
+                static int errcount = 100;
+
+                /* Don't lose the original error message from curl, since
+                 * it contains extra data.
+                 */
+                if (errcount > 0) {
+                    error_report("curl: %s", state->errmsg);
+                    if (--errcount == 0) {
+                        error_report("curl: further errors suppressed");
+                    }
+                }
+
                 for (i = 0; i < CURL_NUM_ACB; i++) {
                     CURLAIOCB *acb = state->acb[i];
 
@@ -304,7 +338,7 @@
                         continue;
                     }
 
-                    acb->common.cb(acb->common.opaque, -EIO);
+                    acb->common.cb(acb->common.opaque, -EPROTO);
                     qemu_aio_unref(acb);
                     state->acb[i] = NULL;
                 }
@@ -402,6 +436,21 @@
         curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
         curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 
+        if (s->username) {
+            curl_easy_setopt(state->curl, CURLOPT_USERNAME, s->username);
+        }
+        if (s->password) {
+            curl_easy_setopt(state->curl, CURLOPT_PASSWORD, s->password);
+        }
+        if (s->proxyusername) {
+            curl_easy_setopt(state->curl,
+                             CURLOPT_PROXYUSERNAME, s->proxyusername);
+        }
+        if (s->proxypassword) {
+            curl_easy_setopt(state->curl,
+                             CURLOPT_PROXYPASSWORD, s->proxypassword);
+        }
+
         /* Restrict supported protocols to avoid security issues in the more
          * obscure protocols.  For example, do not allow POP3/SMTP/IMAP see
          * CVE-2013-0249.
@@ -508,10 +557,31 @@
             .type = QEMU_OPT_STRING,
             .help = "Pass the cookie or list of cookies with each request"
         },
+        {
+            .name = CURL_BLOCK_OPT_USERNAME,
+            .type = QEMU_OPT_STRING,
+            .help = "Username for HTTP auth"
+        },
+        {
+            .name = CURL_BLOCK_OPT_PASSWORD_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret used as password for HTTP auth",
+        },
+        {
+            .name = CURL_BLOCK_OPT_PROXY_USERNAME,
+            .type = QEMU_OPT_STRING,
+            .help = "Username for HTTP proxy auth"
+        },
+        {
+            .name = CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET,
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret used as password for HTTP proxy auth",
+        },
         { /* end of list */ }
     },
 };
 
+
 static int curl_open(BlockDriverState *bs, QDict *options, int flags,
                      Error **errp)
 {
@@ -522,6 +592,7 @@
     const char *file;
     const char *cookie;
     double d;
+    const char *secretid;
 
     static int inited = 0;
 
@@ -563,6 +634,26 @@
         goto out_noclean;
     }
 
+    s->username = g_strdup(qemu_opt_get(opts, CURL_BLOCK_OPT_USERNAME));
+    secretid = qemu_opt_get(opts, CURL_BLOCK_OPT_PASSWORD_SECRET);
+
+    if (secretid) {
+        s->password = qcrypto_secret_lookup_as_utf8(secretid, errp);
+        if (!s->password) {
+            goto out_noclean;
+        }
+    }
+
+    s->proxyusername = g_strdup(
+        qemu_opt_get(opts, CURL_BLOCK_OPT_PROXY_USERNAME));
+    secretid = qemu_opt_get(opts, CURL_BLOCK_OPT_PROXY_PASSWORD_SECRET);
+    if (secretid) {
+        s->proxypassword = qcrypto_secret_lookup_as_utf8(secretid, errp);
+        if (!s->proxypassword) {
+            goto out_noclean;
+        }
+    }
+
     if (!inited) {
         curl_global_init(CURL_GLOBAL_ALL);
         inited = 1;
diff --git a/block/dirty-bitmap.c b/block/dirty-bitmap.c
new file mode 100644
index 0000000..f2bfdcf
--- /dev/null
+++ b/block/dirty-bitmap.c
@@ -0,0 +1,387 @@
+/*
+ * Block Dirty Bitmap
+ *
+ * Copyright (c) 2016 Red Hat. Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "trace.h"
+#include "block/block_int.h"
+#include "block/blockjob.h"
+
+/**
+ * A BdrvDirtyBitmap can be in three possible states:
+ * (1) successor is NULL and disabled is false: full r/w mode
+ * (2) successor is NULL and disabled is true: read only mode ("disabled")
+ * (3) successor is set: frozen mode.
+ *     A frozen bitmap cannot be renamed, deleted, anonymized, cleared, set,
+ *     or enabled. A frozen bitmap can only abdicate() or reclaim().
+ */
+struct BdrvDirtyBitmap {
+    HBitmap *bitmap;            /* Dirty sector bitmap implementation */
+    BdrvDirtyBitmap *successor; /* Anonymous child; implies frozen status */
+    char *name;                 /* Optional non-empty unique ID */
+    int64_t size;               /* Size of the bitmap (Number of sectors) */
+    bool disabled;              /* Bitmap is read-only */
+    QLIST_ENTRY(BdrvDirtyBitmap) list;
+};
+
+BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs, const char *name)
+{
+    BdrvDirtyBitmap *bm;
+
+    assert(name);
+    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
+        if (bm->name && !strcmp(name, bm->name)) {
+            return bm;
+        }
+    }
+    return NULL;
+}
+
+void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap)
+{
+    assert(!bdrv_dirty_bitmap_frozen(bitmap));
+    g_free(bitmap->name);
+    bitmap->name = NULL;
+}
+
+BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
+                                          uint32_t granularity,
+                                          const char *name,
+                                          Error **errp)
+{
+    int64_t bitmap_size;
+    BdrvDirtyBitmap *bitmap;
+    uint32_t sector_granularity;
+
+    assert((granularity & (granularity - 1)) == 0);
+
+    if (name && bdrv_find_dirty_bitmap(bs, name)) {
+        error_setg(errp, "Bitmap already exists: %s", name);
+        return NULL;
+    }
+    sector_granularity = granularity >> BDRV_SECTOR_BITS;
+    assert(sector_granularity);
+    bitmap_size = bdrv_nb_sectors(bs);
+    if (bitmap_size < 0) {
+        error_setg_errno(errp, -bitmap_size, "could not get length of device");
+        errno = -bitmap_size;
+        return NULL;
+    }
+    bitmap = g_new0(BdrvDirtyBitmap, 1);
+    bitmap->bitmap = hbitmap_alloc(bitmap_size, ctz32(sector_granularity));
+    bitmap->size = bitmap_size;
+    bitmap->name = g_strdup(name);
+    bitmap->disabled = false;
+    QLIST_INSERT_HEAD(&bs->dirty_bitmaps, bitmap, list);
+    return bitmap;
+}
+
+bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap)
+{
+    return bitmap->successor;
+}
+
+bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap)
+{
+    return !(bitmap->disabled || bitmap->successor);
+}
+
+DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap)
+{
+    if (bdrv_dirty_bitmap_frozen(bitmap)) {
+        return DIRTY_BITMAP_STATUS_FROZEN;
+    } else if (!bdrv_dirty_bitmap_enabled(bitmap)) {
+        return DIRTY_BITMAP_STATUS_DISABLED;
+    } else {
+        return DIRTY_BITMAP_STATUS_ACTIVE;
+    }
+}
+
+/**
+ * Create a successor bitmap destined to replace this bitmap after an operation.
+ * Requires that the bitmap is not frozen and has no successor.
+ */
+int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
+                                       BdrvDirtyBitmap *bitmap, Error **errp)
+{
+    uint64_t granularity;
+    BdrvDirtyBitmap *child;
+
+    if (bdrv_dirty_bitmap_frozen(bitmap)) {
+        error_setg(errp, "Cannot create a successor for a bitmap that is "
+                   "currently frozen");
+        return -1;
+    }
+    assert(!bitmap->successor);
+
+    /* Create an anonymous successor */
+    granularity = bdrv_dirty_bitmap_granularity(bitmap);
+    child = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
+    if (!child) {
+        return -1;
+    }
+
+    /* Successor will be on or off based on our current state. */
+    child->disabled = bitmap->disabled;
+
+    /* Install the successor and freeze the parent */
+    bitmap->successor = child;
+    return 0;
+}
+
+/**
+ * For a bitmap with a successor, yield our name to the successor,
+ * delete the old bitmap, and return a handle to the new bitmap.
+ */
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
+                                            BdrvDirtyBitmap *bitmap,
+                                            Error **errp)
+{
+    char *name;
+    BdrvDirtyBitmap *successor = bitmap->successor;
+
+    if (successor == NULL) {
+        error_setg(errp, "Cannot relinquish control if "
+                   "there's no successor present");
+        return NULL;
+    }
+
+    name = bitmap->name;
+    bitmap->name = NULL;
+    successor->name = name;
+    bitmap->successor = NULL;
+    bdrv_release_dirty_bitmap(bs, bitmap);
+
+    return successor;
+}
+
+/**
+ * In cases of failure where we can no longer safely delete the parent,
+ * we may wish to re-join the parent and child/successor.
+ * The merged parent will be un-frozen, but not explicitly re-enabled.
+ */
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
+                                           BdrvDirtyBitmap *parent,
+                                           Error **errp)
+{
+    BdrvDirtyBitmap *successor = parent->successor;
+
+    if (!successor) {
+        error_setg(errp, "Cannot reclaim a successor when none is present");
+        return NULL;
+    }
+
+    if (!hbitmap_merge(parent->bitmap, successor->bitmap)) {
+        error_setg(errp, "Merging of parent and successor bitmap failed");
+        return NULL;
+    }
+    bdrv_release_dirty_bitmap(bs, successor);
+    parent->successor = NULL;
+
+    return parent;
+}
+
+/**
+ * Truncates _all_ bitmaps attached to a BDS.
+ */
+void bdrv_dirty_bitmap_truncate(BlockDriverState *bs)
+{
+    BdrvDirtyBitmap *bitmap;
+    uint64_t size = bdrv_nb_sectors(bs);
+
+    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
+        assert(!bdrv_dirty_bitmap_frozen(bitmap));
+        hbitmap_truncate(bitmap->bitmap, size);
+        bitmap->size = size;
+    }
+}
+
+static void bdrv_do_release_matching_dirty_bitmap(BlockDriverState *bs,
+                                                  BdrvDirtyBitmap *bitmap,
+                                                  bool only_named)
+{
+    BdrvDirtyBitmap *bm, *next;
+    QLIST_FOREACH_SAFE(bm, &bs->dirty_bitmaps, list, next) {
+        if ((!bitmap || bm == bitmap) && (!only_named || bm->name)) {
+            assert(!bdrv_dirty_bitmap_frozen(bm));
+            QLIST_REMOVE(bm, list);
+            hbitmap_free(bm->bitmap);
+            g_free(bm->name);
+            g_free(bm);
+
+            if (bitmap) {
+                return;
+            }
+        }
+    }
+}
+
+void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap)
+{
+    bdrv_do_release_matching_dirty_bitmap(bs, bitmap, false);
+}
+
+/**
+ * Release all named dirty bitmaps attached to a BDS (for use in bdrv_close()).
+ * There must not be any frozen bitmaps attached.
+ */
+void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs)
+{
+    bdrv_do_release_matching_dirty_bitmap(bs, NULL, true);
+}
+
+void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+{
+    assert(!bdrv_dirty_bitmap_frozen(bitmap));
+    bitmap->disabled = true;
+}
+
+void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap)
+{
+    assert(!bdrv_dirty_bitmap_frozen(bitmap));
+    bitmap->disabled = false;
+}
+
+BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs)
+{
+    BdrvDirtyBitmap *bm;
+    BlockDirtyInfoList *list = NULL;
+    BlockDirtyInfoList **plist = &list;
+
+    QLIST_FOREACH(bm, &bs->dirty_bitmaps, list) {
+        BlockDirtyInfo *info = g_new0(BlockDirtyInfo, 1);
+        BlockDirtyInfoList *entry = g_new0(BlockDirtyInfoList, 1);
+        info->count = bdrv_get_dirty_count(bm);
+        info->granularity = bdrv_dirty_bitmap_granularity(bm);
+        info->has_name = !!bm->name;
+        info->name = g_strdup(bm->name);
+        info->status = bdrv_dirty_bitmap_status(bm);
+        entry->value = info;
+        *plist = entry;
+        plist = &entry->next;
+    }
+
+    return list;
+}
+
+int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap,
+                   int64_t sector)
+{
+    if (bitmap) {
+        return hbitmap_get(bitmap->bitmap, sector);
+    } else {
+        return 0;
+    }
+}
+
+/**
+ * Chooses a default granularity based on the existing cluster size,
+ * but clamped between [4K, 64K]. Defaults to 64K in the case that there
+ * is no cluster size information available.
+ */
+uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs)
+{
+    BlockDriverInfo bdi;
+    uint32_t granularity;
+
+    if (bdrv_get_info(bs, &bdi) >= 0 && bdi.cluster_size > 0) {
+        granularity = MAX(4096, bdi.cluster_size);
+        granularity = MIN(65536, granularity);
+    } else {
+        granularity = 65536;
+    }
+
+    return granularity;
+}
+
+uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap)
+{
+    return BDRV_SECTOR_SIZE << hbitmap_granularity(bitmap->bitmap);
+}
+
+void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, HBitmapIter *hbi)
+{
+    hbitmap_iter_init(hbi, bitmap->bitmap, 0);
+}
+
+void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                           int64_t cur_sector, int64_t nr_sectors)
+{
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
+}
+
+void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                             int64_t cur_sector, int64_t nr_sectors)
+{
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    hbitmap_reset(bitmap->bitmap, cur_sector, nr_sectors);
+}
+
+void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out)
+{
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    if (!out) {
+        hbitmap_reset_all(bitmap->bitmap);
+    } else {
+        HBitmap *backup = bitmap->bitmap;
+        bitmap->bitmap = hbitmap_alloc(bitmap->size,
+                                       hbitmap_granularity(backup));
+        *out = backup;
+    }
+}
+
+void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in)
+{
+    HBitmap *tmp = bitmap->bitmap;
+    assert(bdrv_dirty_bitmap_enabled(bitmap));
+    bitmap->bitmap = in;
+    hbitmap_free(tmp);
+}
+
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
+                    int64_t nr_sectors)
+{
+    BdrvDirtyBitmap *bitmap;
+    QLIST_FOREACH(bitmap, &bs->dirty_bitmaps, list) {
+        if (!bdrv_dirty_bitmap_enabled(bitmap)) {
+            continue;
+        }
+        hbitmap_set(bitmap->bitmap, cur_sector, nr_sectors);
+    }
+}
+
+/**
+ * Advance an HBitmapIter to an arbitrary offset.
+ */
+void bdrv_set_dirty_iter(HBitmapIter *hbi, int64_t offset)
+{
+    assert(hbi->hb);
+    hbitmap_iter_init(hbi, hbi->hb, offset);
+}
+
+int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap)
+{
+    return hbitmap_count(bitmap->bitmap);
+}
diff --git a/block/dmg.c b/block/dmg.c
index e455886..b0ed89b 100644
--- a/block/dmg.c
+++ b/block/dmg.c
@@ -21,11 +21,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "qemu/bswap.h"
+#include "qemu/error-report.h"
 #include "qemu/module.h"
 #include <zlib.h>
+#ifdef CONFIG_BZIP2
+#include <bzlib.h>
+#endif
 
 enum {
     /* Limit chunk sizes to prevent unreasonable amounts of memory being used
@@ -55,6 +61,9 @@
     uint8_t *compressed_chunk;
     uint8_t *uncompressed_chunk;
     z_stream zstream;
+#ifdef CONFIG_BZIP2
+    bz_stream bzstream;
+#endif
 } BDRVDMGState;
 
 static int dmg_probe(const uint8_t *buf, int buf_size, const char *filename)
@@ -100,6 +109,16 @@
     return 0;
 }
 
+static inline uint64_t buff_read_uint64(const uint8_t *buffer, int64_t offset)
+{
+    return be64_to_cpu(*(uint64_t *)&buffer[offset]);
+}
+
+static inline uint32_t buff_read_uint32(const uint8_t *buffer, int64_t offset)
+{
+    return be32_to_cpu(*(uint32_t *)&buffer[offset]);
+}
+
 /* Increase max chunk sizes, if necessary.  This function is used to calculate
  * the buffer sizes needed for compressed/uncompressed chunk I/O.
  */
@@ -112,6 +131,7 @@
 
     switch (s->types[chunk]) {
     case 0x80000005: /* zlib compressed */
+    case 0x80000006: /* bzip2 compressed */
         compressed_size = s->lengths[chunk];
         uncompressed_sectors = s->sectorcounts[chunk];
         break;
@@ -119,7 +139,9 @@
         uncompressed_sectors = (s->lengths[chunk] + 511) / 512;
         break;
     case 2: /* zero */
-        uncompressed_sectors = s->sectorcounts[chunk];
+        /* as the all-zeroes block may be large, it is treated specially: the
+         * sector is not copied from a large buffer, a simple memset is used
+         * instead. Therefore uncompressed_sectors does not need to be set. */
         break;
     }
 
@@ -131,163 +153,374 @@
     }
 }
 
-static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
-                    Error **errp)
+static int64_t dmg_find_koly_offset(BdrvChild *file, Error **errp)
+{
+    BlockDriverState *file_bs = file->bs;
+    int64_t length;
+    int64_t offset = 0;
+    uint8_t buffer[515];
+    int i, ret;
+
+    /* bdrv_getlength returns a multiple of block size (512), rounded up. Since
+     * dmg images can have odd sizes, try to look for the "koly" magic which
+     * marks the begin of the UDIF trailer (512 bytes). This magic can be found
+     * in the last 511 bytes of the second-last sector or the first 4 bytes of
+     * the last sector (search space: 515 bytes) */
+    length = bdrv_getlength(file_bs);
+    if (length < 0) {
+        error_setg_errno(errp, -length,
+            "Failed to get file size while reading UDIF trailer");
+        return length;
+    } else if (length < 512) {
+        error_setg(errp, "dmg file must be at least 512 bytes long");
+        return -EINVAL;
+    }
+    if (length > 511 + 512) {
+        offset = length - 511 - 512;
+    }
+    length = length < 515 ? length : 515;
+    ret = bdrv_pread(file, offset, buffer, length);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Failed while reading UDIF trailer");
+        return ret;
+    }
+    for (i = 0; i < length - 3; i++) {
+        if (buffer[i] == 'k' && buffer[i+1] == 'o' &&
+            buffer[i+2] == 'l' && buffer[i+3] == 'y') {
+            return offset + i;
+        }
+    }
+    error_setg(errp, "Could not locate UDIF trailer in dmg file");
+    return -EINVAL;
+}
+
+/* used when building the sector table */
+typedef struct DmgHeaderState {
+    /* used internally by dmg_read_mish_block to remember offsets of blocks
+     * across calls */
+    uint64_t data_fork_offset;
+    /* exported for dmg_open */
+    uint32_t max_compressed_size;
+    uint32_t max_sectors_per_chunk;
+} DmgHeaderState;
+
+static bool dmg_is_known_block_type(uint32_t entry_type)
+{
+    switch (entry_type) {
+    case 0x00000001:    /* uncompressed */
+    case 0x00000002:    /* zeroes */
+    case 0x80000005:    /* zlib */
+#ifdef CONFIG_BZIP2
+    case 0x80000006:    /* bzip2 */
+#endif
+        return true;
+    default:
+        return false;
+    }
+}
+
+static int dmg_read_mish_block(BDRVDMGState *s, DmgHeaderState *ds,
+                               uint8_t *buffer, uint32_t count)
+{
+    uint32_t type, i;
+    int ret;
+    size_t new_size;
+    uint32_t chunk_count;
+    int64_t offset = 0;
+    uint64_t data_offset;
+    uint64_t in_offset = ds->data_fork_offset;
+    uint64_t out_offset;
+
+    type = buff_read_uint32(buffer, offset);
+    /* skip data that is not a valid MISH block (invalid magic or too small) */
+    if (type != 0x6d697368 || count < 244) {
+        /* assume success for now */
+        return 0;
+    }
+
+    /* chunk offsets are relative to this sector number */
+    out_offset = buff_read_uint64(buffer, offset + 8);
+
+    /* location in data fork for (compressed) blob (in bytes) */
+    data_offset = buff_read_uint64(buffer, offset + 0x18);
+    in_offset += data_offset;
+
+    /* move to begin of chunk entries */
+    offset += 204;
+
+    chunk_count = (count - 204) / 40;
+    new_size = sizeof(uint64_t) * (s->n_chunks + chunk_count);
+    s->types = g_realloc(s->types, new_size / 2);
+    s->offsets = g_realloc(s->offsets, new_size);
+    s->lengths = g_realloc(s->lengths, new_size);
+    s->sectors = g_realloc(s->sectors, new_size);
+    s->sectorcounts = g_realloc(s->sectorcounts, new_size);
+
+    for (i = s->n_chunks; i < s->n_chunks + chunk_count; i++) {
+        s->types[i] = buff_read_uint32(buffer, offset);
+        if (!dmg_is_known_block_type(s->types[i])) {
+            chunk_count--;
+            i--;
+            offset += 40;
+            continue;
+        }
+
+        /* sector number */
+        s->sectors[i] = buff_read_uint64(buffer, offset + 8);
+        s->sectors[i] += out_offset;
+
+        /* sector count */
+        s->sectorcounts[i] = buff_read_uint64(buffer, offset + 0x10);
+
+        /* all-zeroes sector (type 2) does not need to be "uncompressed" and can
+         * therefore be unbounded. */
+        if (s->types[i] != 2 && s->sectorcounts[i] > DMG_SECTORCOUNTS_MAX) {
+            error_report("sector count %" PRIu64 " for chunk %" PRIu32
+                         " is larger than max (%u)",
+                         s->sectorcounts[i], i, DMG_SECTORCOUNTS_MAX);
+            ret = -EINVAL;
+            goto fail;
+        }
+
+        /* offset in (compressed) data fork */
+        s->offsets[i] = buff_read_uint64(buffer, offset + 0x18);
+        s->offsets[i] += in_offset;
+
+        /* length in (compressed) data fork */
+        s->lengths[i] = buff_read_uint64(buffer, offset + 0x20);
+
+        if (s->lengths[i] > DMG_LENGTHS_MAX) {
+            error_report("length %" PRIu64 " for chunk %" PRIu32
+                         " is larger than max (%u)",
+                         s->lengths[i], i, DMG_LENGTHS_MAX);
+            ret = -EINVAL;
+            goto fail;
+        }
+
+        update_max_chunk_size(s, i, &ds->max_compressed_size,
+                              &ds->max_sectors_per_chunk);
+        offset += 40;
+    }
+    s->n_chunks += chunk_count;
+    return 0;
+
+fail:
+    return ret;
+}
+
+static int dmg_read_resource_fork(BlockDriverState *bs, DmgHeaderState *ds,
+                                  uint64_t info_begin, uint64_t info_length)
 {
     BDRVDMGState *s = bs->opaque;
-    uint64_t info_begin, info_end, last_in_offset, last_out_offset;
-    uint32_t count, tmp;
-    uint32_t max_compressed_size = 1, max_sectors_per_chunk = 1, i;
-    int64_t offset;
     int ret;
+    uint32_t count, rsrc_data_offset;
+    uint8_t *buffer = NULL;
+    uint64_t info_end;
+    uint64_t offset;
 
-    bs->read_only = 1;
-    s->n_chunks = 0;
-    s->offsets = s->lengths = s->sectors = s->sectorcounts = NULL;
-
-    /* read offset of info blocks */
-    offset = bdrv_getlength(bs->file);
-    if (offset < 0) {
-        ret = offset;
-        goto fail;
-    }
-    offset -= 0x1d8;
-
-    ret = read_uint64(bs, offset, &info_begin);
+    /* read offset from begin of resource fork (info_begin) to resource data */
+    ret = read_uint32(bs, info_begin, &rsrc_data_offset);
     if (ret < 0) {
         goto fail;
-    } else if (info_begin == 0) {
+    } else if (rsrc_data_offset > info_length) {
         ret = -EINVAL;
         goto fail;
     }
 
-    ret = read_uint32(bs, info_begin, &tmp);
+    /* read length of resource data */
+    ret = read_uint32(bs, info_begin + 8, &count);
     if (ret < 0) {
         goto fail;
-    } else if (tmp != 0x100) {
+    } else if (count == 0 || rsrc_data_offset + count > info_length) {
         ret = -EINVAL;
         goto fail;
     }
 
-    ret = read_uint32(bs, info_begin + 4, &count);
-    if (ret < 0) {
-        goto fail;
-    } else if (count == 0) {
-        ret = -EINVAL;
-        goto fail;
-    }
-    info_end = info_begin + count;
+    /* begin of resource data (consisting of one or more resources) */
+    offset = info_begin + rsrc_data_offset;
 
-    offset = info_begin + 0x100;
+    /* end of resource data (there is possibly a following resource map
+     * which will be ignored). */
+    info_end = offset + count;
 
-    /* read offsets */
-    last_in_offset = last_out_offset = 0;
+    /* read offsets (mish blocks) from one or more resources in resource data */
     while (offset < info_end) {
-        uint32_t type;
-
+        /* size of following resource */
         ret = read_uint32(bs, offset, &count);
         if (ret < 0) {
             goto fail;
-        } else if (count == 0) {
+        } else if (count == 0 || count > info_end - offset) {
             ret = -EINVAL;
             goto fail;
         }
         offset += 4;
 
-        ret = read_uint32(bs, offset, &type);
+        buffer = g_realloc(buffer, count);
+        ret = bdrv_pread(bs->file, offset, buffer, count);
         if (ret < 0) {
             goto fail;
         }
 
-        if (type == 0x6d697368 && count >= 244) {
-            size_t new_size;
-            uint32_t chunk_count;
-
-            offset += 4;
-            offset += 200;
-
-            chunk_count = (count - 204) / 40;
-            new_size = sizeof(uint64_t) * (s->n_chunks + chunk_count);
-            s->types = g_realloc(s->types, new_size / 2);
-            s->offsets = g_realloc(s->offsets, new_size);
-            s->lengths = g_realloc(s->lengths, new_size);
-            s->sectors = g_realloc(s->sectors, new_size);
-            s->sectorcounts = g_realloc(s->sectorcounts, new_size);
-
-            for (i = s->n_chunks; i < s->n_chunks + chunk_count; i++) {
-                ret = read_uint32(bs, offset, &s->types[i]);
-                if (ret < 0) {
-                    goto fail;
-                }
-                offset += 4;
-                if (s->types[i] != 0x80000005 && s->types[i] != 1 &&
-                    s->types[i] != 2) {
-                    if (s->types[i] == 0xffffffff && i > 0) {
-                        last_in_offset = s->offsets[i - 1] + s->lengths[i - 1];
-                        last_out_offset = s->sectors[i - 1] +
-                                          s->sectorcounts[i - 1];
-                    }
-                    chunk_count--;
-                    i--;
-                    offset += 36;
-                    continue;
-                }
-                offset += 4;
-
-                ret = read_uint64(bs, offset, &s->sectors[i]);
-                if (ret < 0) {
-                    goto fail;
-                }
-                s->sectors[i] += last_out_offset;
-                offset += 8;
-
-                ret = read_uint64(bs, offset, &s->sectorcounts[i]);
-                if (ret < 0) {
-                    goto fail;
-                }
-                offset += 8;
-
-                if (s->sectorcounts[i] > DMG_SECTORCOUNTS_MAX) {
-                    error_report("sector count %" PRIu64 " for chunk %" PRIu32
-                                 " is larger than max (%u)",
-                                 s->sectorcounts[i], i, DMG_SECTORCOUNTS_MAX);
-                    ret = -EINVAL;
-                    goto fail;
-                }
-
-                ret = read_uint64(bs, offset, &s->offsets[i]);
-                if (ret < 0) {
-                    goto fail;
-                }
-                s->offsets[i] += last_in_offset;
-                offset += 8;
-
-                ret = read_uint64(bs, offset, &s->lengths[i]);
-                if (ret < 0) {
-                    goto fail;
-                }
-                offset += 8;
-
-                if (s->lengths[i] > DMG_LENGTHS_MAX) {
-                    error_report("length %" PRIu64 " for chunk %" PRIu32
-                                 " is larger than max (%u)",
-                                 s->lengths[i], i, DMG_LENGTHS_MAX);
-                    ret = -EINVAL;
-                    goto fail;
-                }
-
-                update_max_chunk_size(s, i, &max_compressed_size,
-                                      &max_sectors_per_chunk);
-            }
-            s->n_chunks += chunk_count;
+        ret = dmg_read_mish_block(s, ds, buffer, count);
+        if (ret < 0) {
+            goto fail;
         }
+        /* advance offset by size of resource */
+        offset += count;
+    }
+    ret = 0;
+
+fail:
+    g_free(buffer);
+    return ret;
+}
+
+static int dmg_read_plist_xml(BlockDriverState *bs, DmgHeaderState *ds,
+                              uint64_t info_begin, uint64_t info_length)
+{
+    BDRVDMGState *s = bs->opaque;
+    int ret;
+    uint8_t *buffer = NULL;
+    char *data_begin, *data_end;
+
+    /* Have at least some length to avoid NULL for g_malloc. Attempt to set a
+     * safe upper cap on the data length. A test sample had a XML length of
+     * about 1 MiB. */
+    if (info_length == 0 || info_length > 16 * 1024 * 1024) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    buffer = g_malloc(info_length + 1);
+    buffer[info_length] = '\0';
+    ret = bdrv_pread(bs->file, info_begin, buffer, info_length);
+    if (ret != info_length) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* look for <data>...</data>. The data is 284 (0x11c) bytes after base64
+     * decode. The actual data element has 431 (0x1af) bytes which includes tabs
+     * and line feeds. */
+    data_end = (char *)buffer;
+    while ((data_begin = strstr(data_end, "<data>")) != NULL) {
+        guchar *mish;
+        gsize out_len = 0;
+
+        data_begin += 6;
+        data_end = strstr(data_begin, "</data>");
+        /* malformed XML? */
+        if (data_end == NULL) {
+            ret = -EINVAL;
+            goto fail;
+        }
+        *data_end++ = '\0';
+        mish = g_base64_decode(data_begin, &out_len);
+        ret = dmg_read_mish_block(s, ds, mish, (uint32_t)out_len);
+        g_free(mish);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+    ret = 0;
+
+fail:
+    g_free(buffer);
+    return ret;
+}
+
+static int dmg_open(BlockDriverState *bs, QDict *options, int flags,
+                    Error **errp)
+{
+    BDRVDMGState *s = bs->opaque;
+    DmgHeaderState ds;
+    uint64_t rsrc_fork_offset, rsrc_fork_length;
+    uint64_t plist_xml_offset, plist_xml_length;
+    int64_t offset;
+    int ret;
+
+    bs->read_only = true;
+
+    s->n_chunks = 0;
+    s->offsets = s->lengths = s->sectors = s->sectorcounts = NULL;
+    /* used by dmg_read_mish_block to keep track of the current I/O position */
+    ds.data_fork_offset = 0;
+    ds.max_compressed_size = 1;
+    ds.max_sectors_per_chunk = 1;
+
+    /* locate the UDIF trailer */
+    offset = dmg_find_koly_offset(bs->file, errp);
+    if (offset < 0) {
+        ret = offset;
+        goto fail;
+    }
+
+    /* offset of data fork (DataForkOffset) */
+    ret = read_uint64(bs, offset + 0x18, &ds.data_fork_offset);
+    if (ret < 0) {
+        goto fail;
+    } else if (ds.data_fork_offset > offset) {
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* offset of resource fork (RsrcForkOffset) */
+    ret = read_uint64(bs, offset + 0x28, &rsrc_fork_offset);
+    if (ret < 0) {
+        goto fail;
+    }
+    ret = read_uint64(bs, offset + 0x30, &rsrc_fork_length);
+    if (ret < 0) {
+        goto fail;
+    }
+    if (rsrc_fork_offset >= offset ||
+        rsrc_fork_length > offset - rsrc_fork_offset) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    /* offset of property list (XMLOffset) */
+    ret = read_uint64(bs, offset + 0xd8, &plist_xml_offset);
+    if (ret < 0) {
+        goto fail;
+    }
+    ret = read_uint64(bs, offset + 0xe0, &plist_xml_length);
+    if (ret < 0) {
+        goto fail;
+    }
+    if (plist_xml_offset >= offset ||
+        plist_xml_length > offset - plist_xml_offset) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    ret = read_uint64(bs, offset + 0x1ec, (uint64_t *)&bs->total_sectors);
+    if (ret < 0) {
+        goto fail;
+    }
+    if (bs->total_sectors < 0) {
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (rsrc_fork_length != 0) {
+        ret = dmg_read_resource_fork(bs, &ds,
+                                     rsrc_fork_offset, rsrc_fork_length);
+        if (ret < 0) {
+            goto fail;
+        }
+    } else if (plist_xml_length != 0) {
+        ret = dmg_read_plist_xml(bs, &ds, plist_xml_offset, plist_xml_length);
+        if (ret < 0) {
+            goto fail;
+        }
+    } else {
+        ret = -EINVAL;
+        goto fail;
     }
 
     /* initialize zlib engine */
-    s->compressed_chunk = qemu_try_blockalign(bs->file,
-                                              max_compressed_size + 1);
-    s->uncompressed_chunk = qemu_try_blockalign(bs->file,
-                                                512 * max_sectors_per_chunk);
+    s->compressed_chunk = qemu_try_blockalign(bs->file->bs,
+                                              ds.max_compressed_size + 1);
+    s->uncompressed_chunk = qemu_try_blockalign(bs->file->bs,
+                                                512 * ds.max_sectors_per_chunk);
     if (s->compressed_chunk == NULL || s->uncompressed_chunk == NULL) {
         ret = -ENOMEM;
         goto fail;
@@ -314,6 +547,11 @@
     return ret;
 }
 
+static void dmg_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    bs->bl.request_alignment = BDRV_SECTOR_SIZE; /* No sub-sector I/O */
+}
+
 static inline int is_sector_in_chunk(BDRVDMGState* s,
                 uint32_t chunk_num, uint64_t sector_num)
 {
@@ -349,13 +587,16 @@
     if (!is_sector_in_chunk(s, s->current_chunk, sector_num)) {
         int ret;
         uint32_t chunk = search_chunk(s, sector_num);
+#ifdef CONFIG_BZIP2
+        uint64_t total_out;
+#endif
 
         if (chunk >= s->n_chunks) {
             return -1;
         }
 
         s->current_chunk = s->n_chunks;
-        switch (s->types[chunk]) {
+        switch (s->types[chunk]) { /* block entry type */
         case 0x80000005: { /* zlib compressed */
             /* we need to buffer, because only the chunk as whole can be
              * inflated. */
@@ -379,6 +620,34 @@
                 return -1;
             }
             break; }
+#ifdef CONFIG_BZIP2
+        case 0x80000006: /* bzip2 compressed */
+            /* we need to buffer, because only the chunk as whole can be
+             * inflated. */
+            ret = bdrv_pread(bs->file, s->offsets[chunk],
+                             s->compressed_chunk, s->lengths[chunk]);
+            if (ret != s->lengths[chunk]) {
+                return -1;
+            }
+
+            ret = BZ2_bzDecompressInit(&s->bzstream, 0, 0);
+            if (ret != BZ_OK) {
+                return -1;
+            }
+            s->bzstream.next_in = (char *)s->compressed_chunk;
+            s->bzstream.avail_in = (unsigned int) s->lengths[chunk];
+            s->bzstream.next_out = (char *)s->uncompressed_chunk;
+            s->bzstream.avail_out = (unsigned int) 512 * s->sectorcounts[chunk];
+            ret = BZ2_bzDecompress(&s->bzstream);
+            total_out = ((uint64_t)s->bzstream.total_out_hi32 << 32) +
+                        s->bzstream.total_out_lo32;
+            BZ2_bzDecompressEnd(&s->bzstream);
+            if (ret != BZ_STREAM_END ||
+                total_out != 512 * s->sectorcounts[chunk]) {
+                return -1;
+            }
+            break;
+#endif /* CONFIG_BZIP2 */
         case 1: /* copy */
             ret = bdrv_pread(bs->file, s->offsets[chunk],
                              s->uncompressed_chunk, s->lengths[chunk]);
@@ -387,7 +656,8 @@
             }
             break;
         case 2: /* zero */
-            memset(s->uncompressed_chunk, 0, 512 * s->sectorcounts[chunk]);
+            /* see dmg_read, it is treated specially. No buffer needs to be
+             * pre-filled, the zeroes can be set directly. */
             break;
         }
         s->current_chunk = chunk;
@@ -395,31 +665,42 @@
     return 0;
 }
 
-static int dmg_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+dmg_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+              QEMUIOVector *qiov, int flags)
 {
     BDRVDMGState *s = bs->opaque;
-    int i;
+    uint64_t sector_num = offset >> BDRV_SECTOR_BITS;
+    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
+    int ret, i;
+
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+
+    qemu_co_mutex_lock(&s->lock);
 
     for (i = 0; i < nb_sectors; i++) {
         uint32_t sector_offset_in_chunk;
+        void *data;
+
         if (dmg_read_chunk(bs, sector_num + i) != 0) {
-            return -1;
+            ret = -EIO;
+            goto fail;
+        }
+        /* Special case: current chunk is all zeroes. Do not perform a memcpy as
+         * s->uncompressed_chunk may be too small to cover the large all-zeroes
+         * section. dmg_read_chunk is called to find s->current_chunk */
+        if (s->types[s->current_chunk] == 2) { /* all zeroes block entry */
+            qemu_iovec_memset(qiov, i * 512, 0, 512);
+            continue;
         }
         sector_offset_in_chunk = sector_num + i - s->sectors[s->current_chunk];
-        memcpy(buf + i * 512,
-               s->uncompressed_chunk + sector_offset_in_chunk * 512, 512);
+        data = s->uncompressed_chunk + sector_offset_in_chunk * 512;
+        qemu_iovec_from_buf(qiov, i * 512, data, 512);
     }
-    return 0;
-}
 
-static coroutine_fn int dmg_co_read(BlockDriverState *bs, int64_t sector_num,
-                                    uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVDMGState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = dmg_read(bs, sector_num, buf, nb_sectors);
+    ret = 0;
+fail:
     qemu_co_mutex_unlock(&s->lock);
     return ret;
 }
@@ -444,7 +725,8 @@
     .instance_size  = sizeof(BDRVDMGState),
     .bdrv_probe     = dmg_probe,
     .bdrv_open      = dmg_open,
-    .bdrv_read      = dmg_co_read,
+    .bdrv_refresh_limits = dmg_refresh_limits,
+    .bdrv_co_preadv = dmg_co_preadv,
     .bdrv_close     = dmg_close,
 };
 
diff --git a/block/gluster.c b/block/gluster.c
index 1eb3a8c..01b479f 100644
--- a/block/gluster.c
+++ b/block/gluster.c
@@ -7,9 +7,31 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include <glusterfs/api/glfs.h>
 #include "block/block_int.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu/uri.h"
+#include "qemu/error-report.h"
+
+#define GLUSTER_OPT_FILENAME        "filename"
+#define GLUSTER_OPT_VOLUME          "volume"
+#define GLUSTER_OPT_PATH            "path"
+#define GLUSTER_OPT_TYPE            "type"
+#define GLUSTER_OPT_SERVER_PATTERN  "server."
+#define GLUSTER_OPT_HOST            "host"
+#define GLUSTER_OPT_PORT            "port"
+#define GLUSTER_OPT_TO              "to"
+#define GLUSTER_OPT_IPV4            "ipv4"
+#define GLUSTER_OPT_IPV6            "ipv6"
+#define GLUSTER_OPT_SOCKET          "socket"
+#define GLUSTER_OPT_DEBUG           "debug"
+#define GLUSTER_DEFAULT_PORT        24007
+#define GLUSTER_DEBUG_DEFAULT       4
+#define GLUSTER_DEBUG_MAX           9
+
+#define GERR_INDEX_HINT "hint: check in 'server' array index '%d'\n"
 
 typedef struct GlusterAIOCB {
     int64_t size;
@@ -22,28 +44,145 @@
 typedef struct BDRVGlusterState {
     struct glfs *glfs;
     struct glfs_fd *fd;
+    bool supports_seek_data;
+    int debug_level;
 } BDRVGlusterState;
 
-typedef struct GlusterConf {
-    char *server;
-    int port;
-    char *volname;
-    char *image;
-    char *transport;
-} GlusterConf;
+typedef struct BDRVGlusterReopenState {
+    struct glfs *glfs;
+    struct glfs_fd *fd;
+} BDRVGlusterReopenState;
 
-static void qemu_gluster_gconf_free(GlusterConf *gconf)
-{
-    if (gconf) {
-        g_free(gconf->server);
-        g_free(gconf->volname);
-        g_free(gconf->image);
-        g_free(gconf->transport);
-        g_free(gconf);
+
+static QemuOptsList qemu_gluster_create_opts = {
+    .name = "qemu-gluster-create-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts.head),
+    .desc = {
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Virtual disk size"
+        },
+        {
+            .name = BLOCK_OPT_PREALLOC,
+            .type = QEMU_OPT_STRING,
+            .help = "Preallocation mode (allowed values: off, full)"
+        },
+        {
+            .name = GLUSTER_OPT_DEBUG,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Gluster log level, valid range is 0-9",
+        },
+        { /* end of list */ }
     }
-}
+};
 
-static int parse_volume_options(GlusterConf *gconf, char *path)
+static QemuOptsList runtime_opts = {
+    .name = "gluster",
+    .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
+    .desc = {
+        {
+            .name = GLUSTER_OPT_FILENAME,
+            .type = QEMU_OPT_STRING,
+            .help = "URL to the gluster image",
+        },
+        {
+            .name = GLUSTER_OPT_DEBUG,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Gluster log level, valid range is 0-9",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList runtime_json_opts = {
+    .name = "gluster_json",
+    .head = QTAILQ_HEAD_INITIALIZER(runtime_json_opts.head),
+    .desc = {
+        {
+            .name = GLUSTER_OPT_VOLUME,
+            .type = QEMU_OPT_STRING,
+            .help = "name of gluster volume where VM image resides",
+        },
+        {
+            .name = GLUSTER_OPT_PATH,
+            .type = QEMU_OPT_STRING,
+            .help = "absolute path to image file in gluster volume",
+        },
+        {
+            .name = GLUSTER_OPT_DEBUG,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Gluster log level, valid range is 0-9",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList runtime_type_opts = {
+    .name = "gluster_type",
+    .head = QTAILQ_HEAD_INITIALIZER(runtime_type_opts.head),
+    .desc = {
+        {
+            .name = GLUSTER_OPT_TYPE,
+            .type = QEMU_OPT_STRING,
+            .help = "tcp|unix",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList runtime_unix_opts = {
+    .name = "gluster_unix",
+    .head = QTAILQ_HEAD_INITIALIZER(runtime_unix_opts.head),
+    .desc = {
+        {
+            .name = GLUSTER_OPT_SOCKET,
+            .type = QEMU_OPT_STRING,
+            .help = "socket file path)",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList runtime_tcp_opts = {
+    .name = "gluster_tcp",
+    .head = QTAILQ_HEAD_INITIALIZER(runtime_tcp_opts.head),
+    .desc = {
+        {
+            .name = GLUSTER_OPT_TYPE,
+            .type = QEMU_OPT_STRING,
+            .help = "tcp|unix",
+        },
+        {
+            .name = GLUSTER_OPT_HOST,
+            .type = QEMU_OPT_STRING,
+            .help = "host address (hostname/ipv4/ipv6 addresses)",
+        },
+        {
+            .name = GLUSTER_OPT_PORT,
+            .type = QEMU_OPT_NUMBER,
+            .help = "port number on which glusterd is listening (default 24007)",
+        },
+        {
+            .name = "to",
+            .type = QEMU_OPT_NUMBER,
+            .help = "max port number, not supported by gluster",
+        },
+        {
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+            .help = "ipv4 bool value, not supported by gluster",
+        },
+        {
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+            .help = "ipv6 bool value, not supported by gluster",
+        },
+        { /* end of list */ }
+    },
+};
+
+static int parse_volume_options(BlockdevOptionsGluster *gconf, char *path)
 {
     char *p, *q;
 
@@ -57,31 +196,29 @@
     if (*p == '\0') {
         return -EINVAL;
     }
-    gconf->volname = g_strndup(q, p - q);
+    gconf->volume = g_strndup(q, p - q);
 
-    /* image */
+    /* path */
     p += strspn(p, "/");
     if (*p == '\0') {
         return -EINVAL;
     }
-    gconf->image = g_strdup(p);
+    gconf->path = g_strdup(p);
     return 0;
 }
 
 /*
- * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
+ * file=gluster[+transport]://[host[:port]]/volume/path[?socket=...]
  *
  * 'gluster' is the protocol.
  *
  * 'transport' specifies the transport type used to connect to gluster
  * management daemon (glusterd). Valid transport types are
- * tcp, unix and rdma. If a transport type isn't specified, then tcp
- * type is assumed.
+ * tcp or unix. If a transport type isn't specified, then tcp type is assumed.
  *
- * 'server' specifies the server where the volume file specification for
- * the given volume resides. This can be either hostname, ipv4 address
- * or ipv6 address. ipv6 address needs to be within square brackets [ ].
- * If transport type is 'unix', then 'server' field should not be specified.
+ * 'host' specifies the host where the volume file specification for
+ * the given volume resides. This can be either hostname or ipv4 address.
+ * If transport type is 'unix', then 'host' field should not be specified.
  * The 'socket' field needs to be populated with the path to unix domain
  * socket.
  *
@@ -90,23 +227,22 @@
  * default port. If the transport type is unix, then 'port' should not be
  * specified.
  *
- * 'volname' is the name of the gluster volume which contains the VM image.
+ * 'volume' is the name of the gluster volume which contains the VM image.
  *
- * 'image' is the path to the actual VM image that resides on gluster volume.
+ * 'path' is the path to the actual VM image that resides on gluster volume.
  *
  * Examples:
  *
  * file=gluster://1.2.3.4/testvol/a.img
  * file=gluster+tcp://1.2.3.4/testvol/a.img
  * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
- * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
- * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
- * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
+ * file=gluster+tcp://host.domain.com:24007/testvol/dir/a.img
  * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
- * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
  */
-static int qemu_gluster_parseuri(GlusterConf *gconf, const char *filename)
+static int qemu_gluster_parse_uri(BlockdevOptionsGluster *gconf,
+                                  const char *filename)
 {
+    GlusterServer *gsconf;
     URI *uri;
     QueryParams *qp = NULL;
     bool is_unix = false;
@@ -117,16 +253,21 @@
         return -EINVAL;
     }
 
+    gconf->server = g_new0(GlusterServerList, 1);
+    gconf->server->value = gsconf = g_new0(GlusterServer, 1);
+
     /* transport */
     if (!uri->scheme || !strcmp(uri->scheme, "gluster")) {
-        gconf->transport = g_strdup("tcp");
+        gsconf->type = GLUSTER_TRANSPORT_TCP;
     } else if (!strcmp(uri->scheme, "gluster+tcp")) {
-        gconf->transport = g_strdup("tcp");
+        gsconf->type = GLUSTER_TRANSPORT_TCP;
     } else if (!strcmp(uri->scheme, "gluster+unix")) {
-        gconf->transport = g_strdup("unix");
+        gsconf->type = GLUSTER_TRANSPORT_UNIX;
         is_unix = true;
     } else if (!strcmp(uri->scheme, "gluster+rdma")) {
-        gconf->transport = g_strdup("rdma");
+        gsconf->type = GLUSTER_TRANSPORT_TCP;
+        error_report("Warning: rdma feature is not supported, falling "
+                     "back to tcp");
     } else {
         ret = -EINVAL;
         goto out;
@@ -152,10 +293,14 @@
             ret = -EINVAL;
             goto out;
         }
-        gconf->server = g_strdup(qp->p[0].value);
+        gsconf->u.q_unix.path = g_strdup(qp->p[0].value);
     } else {
-        gconf->server = g_strdup(uri->server ? uri->server : "localhost");
-        gconf->port = uri->port;
+        gsconf->u.tcp.host = g_strdup(uri->server ? uri->server : "localhost");
+        if (uri->port) {
+            gsconf->u.tcp.port = g_strdup_printf("%d", uri->port);
+        } else {
+            gsconf->u.tcp.port = g_strdup_printf("%d", GLUSTER_DEFAULT_PORT);
+        }
     }
 
 out:
@@ -166,52 +311,62 @@
     return ret;
 }
 
-static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename,
-                                      Error **errp)
+static struct glfs *qemu_gluster_glfs_init(BlockdevOptionsGluster *gconf,
+                                           Error **errp)
 {
-    struct glfs *glfs = NULL;
+    struct glfs *glfs;
     int ret;
     int old_errno;
+    GlusterServerList *server;
 
-    ret = qemu_gluster_parseuri(gconf, filename);
-    if (ret < 0) {
-        error_setg(errp, "Usage: file=gluster[+transport]://[server[:port]]/"
-                   "volname/image[?socket=...]");
-        errno = -ret;
-        goto out;
-    }
-
-    glfs = glfs_new(gconf->volname);
+    glfs = glfs_new(gconf->volume);
     if (!glfs) {
         goto out;
     }
 
-    ret = glfs_set_volfile_server(glfs, gconf->transport, gconf->server,
-            gconf->port);
-    if (ret < 0) {
-        goto out;
+    for (server = gconf->server; server; server = server->next) {
+        if (server->value->type  == GLUSTER_TRANSPORT_UNIX) {
+            ret = glfs_set_volfile_server(glfs,
+                                   GlusterTransport_lookup[server->value->type],
+                                   server->value->u.q_unix.path, 0);
+        } else {
+            ret = glfs_set_volfile_server(glfs,
+                                   GlusterTransport_lookup[server->value->type],
+                                   server->value->u.tcp.host,
+                                   atoi(server->value->u.tcp.port));
+        }
+
+        if (ret < 0) {
+            goto out;
+        }
     }
 
-    /*
-     * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
-     * GlusterFS makes GF_LOG_* macros available to libgfapi users.
-     */
-    ret = glfs_set_logging(glfs, "-", 4);
+    ret = glfs_set_logging(glfs, "-", gconf->debug_level);
     if (ret < 0) {
         goto out;
     }
 
     ret = glfs_init(glfs);
     if (ret) {
-        error_setg_errno(errp, errno,
-                         "Gluster connection failed for server=%s port=%d "
-                         "volume=%s image=%s transport=%s", gconf->server,
-                         gconf->port, gconf->volname, gconf->image,
-                         gconf->transport);
+        error_setg(errp, "Gluster connection for volume %s, path %s failed"
+                         " to connect", gconf->volume, gconf->path);
+        for (server = gconf->server; server; server = server->next) {
+            if (server->value->type  == GLUSTER_TRANSPORT_UNIX) {
+                error_append_hint(errp, "hint: failed on socket %s ",
+                                  server->value->u.q_unix.path);
+            } else {
+                error_append_hint(errp, "hint: failed on host %s and port %s ",
+                                  server->value->u.tcp.host,
+                                  server->value->u.tcp.port);
+            }
+        }
+
+        error_append_hint(errp, "Please refer to gluster logs for more info\n");
 
         /* glfs_init sometimes doesn't set errno although docs suggest that */
-        if (errno == 0)
+        if (errno == 0) {
             errno = EINVAL;
+        }
 
         goto out;
     }
@@ -226,13 +381,233 @@
     return NULL;
 }
 
+static int qapi_enum_parse(const char *opt)
+{
+    int i;
+
+    if (!opt) {
+        return GLUSTER_TRANSPORT__MAX;
+    }
+
+    for (i = 0; i < GLUSTER_TRANSPORT__MAX; i++) {
+        if (!strcmp(opt, GlusterTransport_lookup[i])) {
+            return i;
+        }
+    }
+
+    return i;
+}
+
+/*
+ * Convert the json formatted command line into qapi.
+*/
+static int qemu_gluster_parse_json(BlockdevOptionsGluster *gconf,
+                                  QDict *options, Error **errp)
+{
+    QemuOpts *opts;
+    GlusterServer *gsconf;
+    GlusterServerList *curr = NULL;
+    QDict *backing_options = NULL;
+    Error *local_err = NULL;
+    char *str = NULL;
+    const char *ptr;
+    size_t num_servers;
+    int i;
+
+    /* create opts info from runtime_json_opts list */
+    opts = qemu_opts_create(&runtime_json_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    num_servers = qdict_array_entries(options, GLUSTER_OPT_SERVER_PATTERN);
+    if (num_servers < 1) {
+        error_setg(&local_err, QERR_MISSING_PARAMETER, "server");
+        goto out;
+    }
+
+    ptr = qemu_opt_get(opts, GLUSTER_OPT_VOLUME);
+    if (!ptr) {
+        error_setg(&local_err, QERR_MISSING_PARAMETER, GLUSTER_OPT_VOLUME);
+        goto out;
+    }
+    gconf->volume = g_strdup(ptr);
+
+    ptr = qemu_opt_get(opts, GLUSTER_OPT_PATH);
+    if (!ptr) {
+        error_setg(&local_err, QERR_MISSING_PARAMETER, GLUSTER_OPT_PATH);
+        goto out;
+    }
+    gconf->path = g_strdup(ptr);
+    qemu_opts_del(opts);
+
+    for (i = 0; i < num_servers; i++) {
+        str = g_strdup_printf(GLUSTER_OPT_SERVER_PATTERN"%d.", i);
+        qdict_extract_subqdict(options, &backing_options, str);
+
+        /* create opts info from runtime_type_opts list */
+        opts = qemu_opts_create(&runtime_type_opts, NULL, 0, &error_abort);
+        qemu_opts_absorb_qdict(opts, backing_options, &local_err);
+        if (local_err) {
+            goto out;
+        }
+
+        ptr = qemu_opt_get(opts, GLUSTER_OPT_TYPE);
+        gsconf = g_new0(GlusterServer, 1);
+        gsconf->type = qapi_enum_parse(ptr);
+        if (!ptr) {
+            error_setg(&local_err, QERR_MISSING_PARAMETER, GLUSTER_OPT_TYPE);
+            error_append_hint(&local_err, GERR_INDEX_HINT, i);
+            goto out;
+
+        }
+        if (gsconf->type == GLUSTER_TRANSPORT__MAX) {
+            error_setg(&local_err, QERR_INVALID_PARAMETER_VALUE,
+                       GLUSTER_OPT_TYPE, "tcp or unix");
+            error_append_hint(&local_err, GERR_INDEX_HINT, i);
+            goto out;
+        }
+        qemu_opts_del(opts);
+
+        if (gsconf->type == GLUSTER_TRANSPORT_TCP) {
+            /* create opts info from runtime_tcp_opts list */
+            opts = qemu_opts_create(&runtime_tcp_opts, NULL, 0, &error_abort);
+            qemu_opts_absorb_qdict(opts, backing_options, &local_err);
+            if (local_err) {
+                goto out;
+            }
+
+            ptr = qemu_opt_get(opts, GLUSTER_OPT_HOST);
+            if (!ptr) {
+                error_setg(&local_err, QERR_MISSING_PARAMETER,
+                           GLUSTER_OPT_HOST);
+                error_append_hint(&local_err, GERR_INDEX_HINT, i);
+                goto out;
+            }
+            gsconf->u.tcp.host = g_strdup(ptr);
+            ptr = qemu_opt_get(opts, GLUSTER_OPT_PORT);
+            if (!ptr) {
+                error_setg(&local_err, QERR_MISSING_PARAMETER,
+                           GLUSTER_OPT_PORT);
+                error_append_hint(&local_err, GERR_INDEX_HINT, i);
+                goto out;
+            }
+            gsconf->u.tcp.port = g_strdup(ptr);
+
+            /* defend for unsupported fields in InetSocketAddress,
+             * i.e. @ipv4, @ipv6  and @to
+             */
+            ptr = qemu_opt_get(opts, GLUSTER_OPT_TO);
+            if (ptr) {
+                gsconf->u.tcp.has_to = true;
+            }
+            ptr = qemu_opt_get(opts, GLUSTER_OPT_IPV4);
+            if (ptr) {
+                gsconf->u.tcp.has_ipv4 = true;
+            }
+            ptr = qemu_opt_get(opts, GLUSTER_OPT_IPV6);
+            if (ptr) {
+                gsconf->u.tcp.has_ipv6 = true;
+            }
+            if (gsconf->u.tcp.has_to) {
+                error_setg(&local_err, "Parameter 'to' not supported");
+                goto out;
+            }
+            if (gsconf->u.tcp.has_ipv4 || gsconf->u.tcp.has_ipv6) {
+                error_setg(&local_err, "Parameters 'ipv4/ipv6' not supported");
+                goto out;
+            }
+            qemu_opts_del(opts);
+        } else {
+            /* create opts info from runtime_unix_opts list */
+            opts = qemu_opts_create(&runtime_unix_opts, NULL, 0, &error_abort);
+            qemu_opts_absorb_qdict(opts, backing_options, &local_err);
+            if (local_err) {
+                goto out;
+            }
+
+            ptr = qemu_opt_get(opts, GLUSTER_OPT_SOCKET);
+            if (!ptr) {
+                error_setg(&local_err, QERR_MISSING_PARAMETER,
+                           GLUSTER_OPT_SOCKET);
+                error_append_hint(&local_err, GERR_INDEX_HINT, i);
+                goto out;
+            }
+            gsconf->u.q_unix.path = g_strdup(ptr);
+            qemu_opts_del(opts);
+        }
+
+        if (gconf->server == NULL) {
+            gconf->server = g_new0(GlusterServerList, 1);
+            gconf->server->value = gsconf;
+            curr = gconf->server;
+        } else {
+            curr->next = g_new0(GlusterServerList, 1);
+            curr->next->value = gsconf;
+            curr = curr->next;
+        }
+
+        qdict_del(backing_options, str);
+        g_free(str);
+        str = NULL;
+    }
+
+    return 0;
+
+out:
+    error_propagate(errp, local_err);
+    qemu_opts_del(opts);
+    if (str) {
+        qdict_del(backing_options, str);
+        g_free(str);
+    }
+    errno = EINVAL;
+    return -errno;
+}
+
+static struct glfs *qemu_gluster_init(BlockdevOptionsGluster *gconf,
+                                      const char *filename,
+                                      QDict *options, Error **errp)
+{
+    int ret;
+    if (filename) {
+        ret = qemu_gluster_parse_uri(gconf, filename);
+        if (ret < 0) {
+            error_setg(errp, "invalid URI");
+            error_append_hint(errp, "Usage: file=gluster[+transport]://"
+                                    "[host[:port]]/volume/path[?socket=...]\n");
+            errno = -ret;
+            return NULL;
+        }
+    } else {
+        ret = qemu_gluster_parse_json(gconf, options, errp);
+        if (ret < 0) {
+            error_append_hint(errp, "Usage: "
+                             "-drive driver=qcow2,file.driver=gluster,"
+                             "file.volume=testvol,file.path=/path/a.qcow2"
+                             "[,file.debug=9],file.server.0.type=tcp,"
+                             "file.server.0.host=1.2.3.4,"
+                             "file.server.0.port=24007,"
+                             "file.server.1.transport=unix,"
+                             "file.server.1.socket=/var/run/glusterd.socket ..."
+                             "\n");
+            errno = -ret;
+            return NULL;
+        }
+
+    }
+
+    return qemu_gluster_glfs_init(gconf, errp);
+}
+
 static void qemu_gluster_complete_aio(void *opaque)
 {
     GlusterAIOCB *acb = (GlusterAIOCB *)opaque;
 
     qemu_bh_delete(acb->bh);
     acb->bh = NULL;
-    qemu_coroutine_enter(acb->coroutine, NULL);
+    qemu_coroutine_enter(acb->coroutine);
 }
 
 /*
@@ -245,7 +620,7 @@
     if (!ret || ret == acb->size) {
         acb->ret = 0; /* Success */
     } else if (ret < 0) {
-        acb->ret = ret; /* Read/Write failed */
+        acb->ret = -errno; /* Read/Write failed */
     } else {
         acb->ret = -EIO; /* Partial read/write - fail it */
     }
@@ -254,20 +629,6 @@
     qemu_bh_schedule(acb->bh);
 }
 
-/* TODO Convert to fine grained options */
-static QemuOptsList runtime_opts = {
-    .name = "gluster",
-    .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
-    .desc = {
-        {
-            .name = "filename",
-            .type = QEMU_OPT_STRING,
-            .help = "URL to the gluster image",
-        },
-        { /* end of list */ }
-    },
-};
-
 static void qemu_gluster_parse_flags(int bdrv_flags, int *open_flags)
 {
     assert(open_flags != NULL);
@@ -285,13 +646,35 @@
     }
 }
 
+/*
+ * Do SEEK_DATA/HOLE to detect if it is functional. Older broken versions of
+ * gfapi incorrectly return the current offset when SEEK_DATA/HOLE is used.
+ * - Corrected versions return -1 and set errno to EINVAL.
+ * - Versions that support SEEK_DATA/HOLE correctly, will return -1 and set
+ *   errno to ENXIO when SEEK_DATA is called with a position of EOF.
+ */
+static bool qemu_gluster_test_seek(struct glfs_fd *fd)
+{
+    off_t ret, eof;
+
+    eof = glfs_lseek(fd, 0, SEEK_END);
+    if (eof < 0) {
+        /* this should never occur */
+        return false;
+    }
+
+    /* this should always fail with ENXIO if SEEK_DATA is supported */
+    ret = glfs_lseek(fd, eof, SEEK_DATA);
+    return (ret < 0) && (errno == ENXIO);
+}
+
 static int qemu_gluster_open(BlockDriverState *bs,  QDict *options,
                              int bdrv_flags, Error **errp)
 {
     BDRVGlusterState *s = bs->opaque;
     int open_flags = 0;
     int ret = 0;
-    GlusterConf *gconf = g_new0(GlusterConf, 1);
+    BlockdevOptionsGluster *gconf = NULL;
     QemuOpts *opts;
     Error *local_err = NULL;
     const char *filename;
@@ -304,24 +687,54 @@
         goto out;
     }
 
-    filename = qemu_opt_get(opts, "filename");
+    filename = qemu_opt_get(opts, GLUSTER_OPT_FILENAME);
 
-    s->glfs = qemu_gluster_init(gconf, filename, errp);
+    s->debug_level = qemu_opt_get_number(opts, GLUSTER_OPT_DEBUG,
+                                         GLUSTER_DEBUG_DEFAULT);
+    if (s->debug_level < 0) {
+        s->debug_level = 0;
+    } else if (s->debug_level > GLUSTER_DEBUG_MAX) {
+        s->debug_level = GLUSTER_DEBUG_MAX;
+    }
+
+    gconf = g_new0(BlockdevOptionsGluster, 1);
+    gconf->debug_level = s->debug_level;
+    gconf->has_debug_level = true;
+    s->glfs = qemu_gluster_init(gconf, filename, options, errp);
     if (!s->glfs) {
         ret = -errno;
         goto out;
     }
 
+#ifdef CONFIG_GLUSTERFS_XLATOR_OPT
+    /* Without this, if fsync fails for a recoverable reason (for instance,
+     * ENOSPC), gluster will dump its cache, preventing retries.  This means
+     * almost certain data loss.  Not all gluster versions support the
+     * 'resync-failed-syncs-after-fsync' key value, but there is no way to
+     * discover during runtime if it is supported (this api returns success for
+     * unknown key/value pairs) */
+    ret = glfs_set_xlator_option(s->glfs, "*-write-behind",
+                                          "resync-failed-syncs-after-fsync",
+                                          "on");
+    if (ret < 0) {
+        error_setg_errno(errp, errno, "Unable to set xlator key/value pair");
+        ret = -errno;
+        goto out;
+    }
+#endif
+
     qemu_gluster_parse_flags(bdrv_flags, &open_flags);
 
-    s->fd = glfs_open(s->glfs, gconf->image, open_flags);
+    s->fd = glfs_open(s->glfs, gconf->path, open_flags);
     if (!s->fd) {
         ret = -errno;
     }
 
+    s->supports_seek_data = qemu_gluster_test_seek(s->fd);
+
 out:
     qemu_opts_del(opts);
-    qemu_gluster_gconf_free(gconf);
+    qapi_free_BlockdevOptionsGluster(gconf);
     if (!ret) {
         return ret;
     }
@@ -334,37 +747,45 @@
     return ret;
 }
 
-typedef struct BDRVGlusterReopenState {
-    struct glfs *glfs;
-    struct glfs_fd *fd;
-} BDRVGlusterReopenState;
-
-
 static int qemu_gluster_reopen_prepare(BDRVReopenState *state,
                                        BlockReopenQueue *queue, Error **errp)
 {
     int ret = 0;
+    BDRVGlusterState *s;
     BDRVGlusterReopenState *reop_s;
-    GlusterConf *gconf = NULL;
+    BlockdevOptionsGluster *gconf;
     int open_flags = 0;
 
     assert(state != NULL);
     assert(state->bs != NULL);
 
+    s = state->bs->opaque;
+
     state->opaque = g_new0(BDRVGlusterReopenState, 1);
     reop_s = state->opaque;
 
     qemu_gluster_parse_flags(state->flags, &open_flags);
 
-    gconf = g_new0(GlusterConf, 1);
-
-    reop_s->glfs = qemu_gluster_init(gconf, state->bs->filename, errp);
+    gconf = g_new0(BlockdevOptionsGluster, 1);
+    gconf->debug_level = s->debug_level;
+    gconf->has_debug_level = true;
+    reop_s->glfs = qemu_gluster_init(gconf, state->bs->filename, NULL, errp);
     if (reop_s->glfs == NULL) {
         ret = -errno;
         goto exit;
     }
 
-    reop_s->fd = glfs_open(reop_s->glfs, gconf->image, open_flags);
+#ifdef CONFIG_GLUSTERFS_XLATOR_OPT
+    ret = glfs_set_xlator_option(reop_s->glfs, "*-write-behind",
+                                 "resync-failed-syncs-after-fsync", "on");
+    if (ret < 0) {
+        error_setg_errno(errp, errno, "Unable to set xlator key/value pair");
+        ret = -errno;
+        goto exit;
+    }
+#endif
+
+    reop_s->fd = glfs_open(reop_s->glfs, gconf->path, open_flags);
     if (reop_s->fd == NULL) {
         /* reops->glfs will be cleaned up in _abort */
         ret = -errno;
@@ -373,7 +794,7 @@
 
 exit:
     /* state->opaque will be freed in either the _abort or _commit */
-    qemu_gluster_gconf_free(gconf);
+    qapi_free_BlockdevOptionsGluster(gconf);
     return ret;
 }
 
@@ -425,32 +846,27 @@
 }
 
 #ifdef CONFIG_GLUSTERFS_ZEROFILL
-static coroutine_fn int qemu_gluster_co_write_zeroes(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
+static coroutine_fn int qemu_gluster_co_pwrite_zeroes(BlockDriverState *bs,
+                                                      int64_t offset,
+                                                      int size,
+                                                      BdrvRequestFlags flags)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
-    off_t size = nb_sectors * BDRV_SECTOR_SIZE;
-    off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = size;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = size;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_zerofill_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
+    ret = glfs_zerofill_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 static inline bool gluster_supports_zerofill(void)
@@ -459,7 +875,7 @@
 }
 
 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
-        int64_t size)
+                                        int64_t size)
 {
     return glfs_zerofill(fd, offset, size);
 }
@@ -471,7 +887,7 @@
 }
 
 static inline int qemu_gluster_zerofill(struct glfs_fd *fd, int64_t offset,
-        int64_t size)
+                                        int64_t size)
 {
     return 0;
 }
@@ -480,15 +896,25 @@
 static int qemu_gluster_create(const char *filename,
                                QemuOpts *opts, Error **errp)
 {
+    BlockdevOptionsGluster *gconf;
     struct glfs *glfs;
     struct glfs_fd *fd;
     int ret = 0;
     int prealloc = 0;
     int64_t total_size = 0;
     char *tmp = NULL;
-    GlusterConf *gconf = g_new0(GlusterConf, 1);
 
-    glfs = qemu_gluster_init(gconf, filename, errp);
+    gconf = g_new0(BlockdevOptionsGluster, 1);
+    gconf->debug_level = qemu_opt_get_number_del(opts, GLUSTER_OPT_DEBUG,
+                                                 GLUSTER_DEBUG_DEFAULT);
+    if (gconf->debug_level < 0) {
+        gconf->debug_level = 0;
+    } else if (gconf->debug_level > GLUSTER_DEBUG_MAX) {
+        gconf->debug_level = GLUSTER_DEBUG_MAX;
+    }
+    gconf->has_debug_level = true;
+
+    glfs = qemu_gluster_init(gconf, filename, NULL, errp);
     if (!glfs) {
         ret = -errno;
         goto out;
@@ -500,19 +926,17 @@
     tmp = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     if (!tmp || !strcmp(tmp, "off")) {
         prealloc = 0;
-    } else if (!strcmp(tmp, "full") &&
-               gluster_supports_zerofill()) {
+    } else if (!strcmp(tmp, "full") && gluster_supports_zerofill()) {
         prealloc = 1;
     } else {
         error_setg(errp, "Invalid preallocation mode: '%s'"
-            " or GlusterFS doesn't support zerofill API",
-            tmp);
+                         " or GlusterFS doesn't support zerofill API", tmp);
         ret = -EINVAL;
         goto out;
     }
 
-    fd = glfs_creat(glfs, gconf->image,
-        O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
+    fd = glfs_creat(glfs, gconf->path,
+                    O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
     if (!fd) {
         ret = -errno;
     } else {
@@ -530,7 +954,7 @@
     }
 out:
     g_free(tmp);
-    qemu_gluster_gconf_free(gconf);
+    qapi_free_BlockdevOptionsGluster(gconf);
     if (glfs) {
         glfs_fini(glfs);
     }
@@ -538,38 +962,34 @@
 }
 
 static coroutine_fn int qemu_gluster_co_rw(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int write)
+                                           int64_t sector_num, int nb_sectors,
+                                           QEMUIOVector *qiov, int write)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
     size_t size = nb_sectors * BDRV_SECTOR_SIZE;
     off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = size;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = size;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
     if (write) {
         ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
-            &gluster_finish_aiocb, acb);
+                                 gluster_finish_aiocb, &acb);
     } else {
         ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
-            &gluster_finish_aiocb, acb);
+                                gluster_finish_aiocb, &acb);
     }
 
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 
 static int qemu_gluster_truncate(BlockDriverState *bs, int64_t offset)
@@ -586,69 +1006,96 @@
 }
 
 static coroutine_fn int qemu_gluster_co_readv(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+                                              int64_t sector_num,
+                                              int nb_sectors,
+                                              QEMUIOVector *qiov)
 {
     return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 0);
 }
 
 static coroutine_fn int qemu_gluster_co_writev(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+                                               int64_t sector_num,
+                                               int nb_sectors,
+                                               QEMUIOVector *qiov)
 {
     return qemu_gluster_co_rw(bs, sector_num, nb_sectors, qiov, 1);
 }
 
+static void qemu_gluster_close(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+
+    if (s->fd) {
+        glfs_close(s->fd);
+        s->fd = NULL;
+    }
+    glfs_fini(s->glfs);
+}
+
 static coroutine_fn int qemu_gluster_co_flush_to_disk(BlockDriverState *bs)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
 
-    acb->size = 0;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = 0;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb);
+    ret = glfs_fsync_async(s->fd, gluster_finish_aiocb, &acb);
     if (ret < 0) {
         ret = -errno;
-        goto out;
+        goto error;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
+    if (acb.ret < 0) {
+        ret = acb.ret;
+        goto error;
+    }
 
-out:
-    g_slice_free(GlusterAIOCB, acb);
+    return acb.ret;
+
+error:
+    /* Some versions of Gluster (3.5.6 -> 3.5.8?) will not retain its cache
+     * after a fsync failure, so we have no way of allowing the guest to safely
+     * continue.  Gluster versions prior to 3.5.6 don't retain the cache
+     * either, but will invalidate the fd on error, so this is again our only
+     * option.
+     *
+     * The 'resync-failed-syncs-after-fsync' xlator option for the
+     * write-behind cache will cause later gluster versions to retain its
+     * cache after error, so long as the fd remains open.  However, we
+     * currently have no way of knowing if this option is supported.
+     *
+     * TODO: Once gluster provides a way for us to determine if the option
+     * is supported, bypass the closure and setting drv to NULL.  */
+    qemu_gluster_close(bs);
+    bs->drv = NULL;
     return ret;
 }
 
 #ifdef CONFIG_GLUSTERFS_DISCARD
-static coroutine_fn int qemu_gluster_co_discard(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors)
+static coroutine_fn int qemu_gluster_co_pdiscard(BlockDriverState *bs,
+                                                 int64_t offset, int size)
 {
     int ret;
-    GlusterAIOCB *acb = g_slice_new(GlusterAIOCB);
+    GlusterAIOCB acb;
     BDRVGlusterState *s = bs->opaque;
-    size_t size = nb_sectors * BDRV_SECTOR_SIZE;
-    off_t offset = sector_num * BDRV_SECTOR_SIZE;
 
-    acb->size = 0;
-    acb->ret = 0;
-    acb->coroutine = qemu_coroutine_self();
-    acb->aio_context = bdrv_get_aio_context(bs);
+    acb.size = 0;
+    acb.ret = 0;
+    acb.coroutine = qemu_coroutine_self();
+    acb.aio_context = bdrv_get_aio_context(bs);
 
-    ret = glfs_discard_async(s->fd, offset, size, &gluster_finish_aiocb, acb);
+    ret = glfs_discard_async(s->fd, offset, size, gluster_finish_aiocb, &acb);
     if (ret < 0) {
-        ret = -errno;
-        goto out;
+        return -errno;
     }
 
     qemu_coroutine_yield();
-    ret = acb->ret;
-
-out:
-    g_slice_free(GlusterAIOCB, acb);
-    return ret;
+    return acb.ret;
 }
 #endif
 
@@ -679,46 +1126,170 @@
     }
 }
 
-static void qemu_gluster_close(BlockDriverState *bs)
-{
-    BDRVGlusterState *s = bs->opaque;
-
-    if (s->fd) {
-        glfs_close(s->fd);
-        s->fd = NULL;
-    }
-    glfs_fini(s->glfs);
-}
-
 static int qemu_gluster_has_zero_init(BlockDriverState *bs)
 {
     /* GlusterFS volume could be backed by a block device */
     return 0;
 }
 
-static QemuOptsList qemu_gluster_create_opts = {
-    .name = "qemu-gluster-create-opts",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_gluster_create_opts.head),
-    .desc = {
-        {
-            .name = BLOCK_OPT_SIZE,
-            .type = QEMU_OPT_SIZE,
-            .help = "Virtual disk size"
-        },
-        {
-            .name = BLOCK_OPT_PREALLOC,
-            .type = QEMU_OPT_STRING,
-            .help = "Preallocation mode (allowed values: off, full)"
-        },
-        { /* end of list */ }
+/*
+ * Find allocation range in @bs around offset @start.
+ * May change underlying file descriptor's file offset.
+ * If @start is not in a hole, store @start in @data, and the
+ * beginning of the next hole in @hole, and return 0.
+ * If @start is in a non-trailing hole, store @start in @hole and the
+ * beginning of the next non-hole in @data, and return 0.
+ * If @start is in a trailing hole or beyond EOF, return -ENXIO.
+ * If we can't find out, return a negative errno other than -ENXIO.
+ *
+ * (Shamefully copied from raw-posix.c, only miniscule adaptions.)
+ */
+static int find_allocation(BlockDriverState *bs, off_t start,
+                           off_t *data, off_t *hole)
+{
+    BDRVGlusterState *s = bs->opaque;
+    off_t offs;
+
+    if (!s->supports_seek_data) {
+        return -ENOTSUP;
     }
-};
+
+    /*
+     * SEEK_DATA cases:
+     * D1. offs == start: start is in data
+     * D2. offs > start: start is in a hole, next data at offs
+     * D3. offs < 0, errno = ENXIO: either start is in a trailing hole
+     *                              or start is beyond EOF
+     *     If the latter happens, the file has been truncated behind
+     *     our back since we opened it.  All bets are off then.
+     *     Treating like a trailing hole is simplest.
+     * D4. offs < 0, errno != ENXIO: we learned nothing
+     */
+    offs = glfs_lseek(s->fd, start, SEEK_DATA);
+    if (offs < 0) {
+        return -errno;          /* D3 or D4 */
+    }
+    assert(offs >= start);
+
+    if (offs > start) {
+        /* D2: in hole, next data at offs */
+        *hole = start;
+        *data = offs;
+        return 0;
+    }
+
+    /* D1: in data, end not yet known */
+
+    /*
+     * SEEK_HOLE cases:
+     * H1. offs == start: start is in a hole
+     *     If this happens here, a hole has been dug behind our back
+     *     since the previous lseek().
+     * H2. offs > start: either start is in data, next hole at offs,
+     *                   or start is in trailing hole, EOF at offs
+     *     Linux treats trailing holes like any other hole: offs ==
+     *     start.  Solaris seeks to EOF instead: offs > start (blech).
+     *     If that happens here, a hole has been dug behind our back
+     *     since the previous lseek().
+     * H3. offs < 0, errno = ENXIO: start is beyond EOF
+     *     If this happens, the file has been truncated behind our
+     *     back since we opened it.  Treat it like a trailing hole.
+     * H4. offs < 0, errno != ENXIO: we learned nothing
+     *     Pretend we know nothing at all, i.e. "forget" about D1.
+     */
+    offs = glfs_lseek(s->fd, start, SEEK_HOLE);
+    if (offs < 0) {
+        return -errno;          /* D1 and (H3 or H4) */
+    }
+    assert(offs >= start);
+
+    if (offs > start) {
+        /*
+         * D1 and H2: either in data, next hole at offs, or it was in
+         * data but is now in a trailing hole.  In the latter case,
+         * all bets are off.  Treating it as if it there was data all
+         * the way to EOF is safe, so simply do that.
+         */
+        *data = start;
+        *hole = offs;
+        return 0;
+    }
+
+    /* D1 and H1 */
+    return -EBUSY;
+}
+
+/*
+ * Returns the allocation status of the specified sectors.
+ *
+ * If 'sector_num' is beyond the end of the disk image the return value is 0
+ * and 'pnum' is set to 0.
+ *
+ * 'pnum' is set to the number of sectors (including and immediately following
+ * the specified sector) that are known to be in the same
+ * allocated/unallocated state.
+ *
+ * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
+ * beyond the end of the disk image it will be clamped.
+ *
+ * (Based on raw_co_get_block_status() from raw-posix.c.)
+ */
+static int64_t coroutine_fn qemu_gluster_co_get_block_status(
+        BlockDriverState *bs, int64_t sector_num, int nb_sectors, int *pnum,
+        BlockDriverState **file)
+{
+    BDRVGlusterState *s = bs->opaque;
+    off_t start, data = 0, hole = 0;
+    int64_t total_size;
+    int ret = -EINVAL;
+
+    if (!s->fd) {
+        return ret;
+    }
+
+    start = sector_num * BDRV_SECTOR_SIZE;
+    total_size = bdrv_getlength(bs);
+    if (total_size < 0) {
+        return total_size;
+    } else if (start >= total_size) {
+        *pnum = 0;
+        return 0;
+    } else if (start + nb_sectors * BDRV_SECTOR_SIZE > total_size) {
+        nb_sectors = DIV_ROUND_UP(total_size - start, BDRV_SECTOR_SIZE);
+    }
+
+    ret = find_allocation(bs, start, &data, &hole);
+    if (ret == -ENXIO) {
+        /* Trailing hole */
+        *pnum = nb_sectors;
+        ret = BDRV_BLOCK_ZERO;
+    } else if (ret < 0) {
+        /* No info available, so pretend there are no holes */
+        *pnum = nb_sectors;
+        ret = BDRV_BLOCK_DATA;
+    } else if (data == start) {
+        /* On a data extent, compute sectors to the end of the extent,
+         * possibly including a partial sector at EOF. */
+        *pnum = MIN(nb_sectors, DIV_ROUND_UP(hole - start, BDRV_SECTOR_SIZE));
+        ret = BDRV_BLOCK_DATA;
+    } else {
+        /* On a hole, compute sectors to the beginning of the next extent.  */
+        assert(hole == start);
+        *pnum = MIN(nb_sectors, (data - start) / BDRV_SECTOR_SIZE);
+        ret = BDRV_BLOCK_ZERO;
+    }
+
+    *file = bs;
+
+    return ret | BDRV_BLOCK_OFFSET_VALID | start;
+}
+
 
 static BlockDriver bdrv_gluster = {
     .format_name                  = "gluster",
     .protocol_name                = "gluster",
     .instance_size                = sizeof(BDRVGlusterState),
-    .bdrv_needs_filename          = true,
+    .bdrv_needs_filename          = false,
     .bdrv_file_open               = qemu_gluster_open,
     .bdrv_reopen_prepare          = qemu_gluster_reopen_prepare,
     .bdrv_reopen_commit           = qemu_gluster_reopen_commit,
@@ -733,11 +1304,12 @@
     .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
     .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
-    .bdrv_co_discard              = qemu_gluster_co_discard,
+    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
 #ifdef CONFIG_GLUSTERFS_ZEROFILL
-    .bdrv_co_write_zeroes         = qemu_gluster_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes        = qemu_gluster_co_pwrite_zeroes,
 #endif
+    .bdrv_co_get_block_status     = qemu_gluster_co_get_block_status,
     .create_opts                  = &qemu_gluster_create_opts,
 };
 
@@ -745,7 +1317,7 @@
     .format_name                  = "gluster",
     .protocol_name                = "gluster+tcp",
     .instance_size                = sizeof(BDRVGlusterState),
-    .bdrv_needs_filename          = true,
+    .bdrv_needs_filename          = false,
     .bdrv_file_open               = qemu_gluster_open,
     .bdrv_reopen_prepare          = qemu_gluster_reopen_prepare,
     .bdrv_reopen_commit           = qemu_gluster_reopen_commit,
@@ -760,11 +1332,12 @@
     .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
     .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
-    .bdrv_co_discard              = qemu_gluster_co_discard,
+    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
 #ifdef CONFIG_GLUSTERFS_ZEROFILL
-    .bdrv_co_write_zeroes         = qemu_gluster_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes        = qemu_gluster_co_pwrite_zeroes,
 #endif
+    .bdrv_co_get_block_status     = qemu_gluster_co_get_block_status,
     .create_opts                  = &qemu_gluster_create_opts,
 };
 
@@ -787,14 +1360,21 @@
     .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
     .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
-    .bdrv_co_discard              = qemu_gluster_co_discard,
+    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
 #ifdef CONFIG_GLUSTERFS_ZEROFILL
-    .bdrv_co_write_zeroes         = qemu_gluster_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes        = qemu_gluster_co_pwrite_zeroes,
 #endif
+    .bdrv_co_get_block_status     = qemu_gluster_co_get_block_status,
     .create_opts                  = &qemu_gluster_create_opts,
 };
 
+/* rdma is deprecated (actually never supported for volfile fetch).
+ * Let's maintain it for the protocol compatibility, to make sure things
+ * won't break immediately. For now, gluster+rdma will fall back to gluster+tcp
+ * protocol with a warning.
+ * TODO: remove gluster+rdma interface support
+ */
 static BlockDriver bdrv_gluster_rdma = {
     .format_name                  = "gluster",
     .protocol_name                = "gluster+rdma",
@@ -814,11 +1394,12 @@
     .bdrv_co_flush_to_disk        = qemu_gluster_co_flush_to_disk,
     .bdrv_has_zero_init           = qemu_gluster_has_zero_init,
 #ifdef CONFIG_GLUSTERFS_DISCARD
-    .bdrv_co_discard              = qemu_gluster_co_discard,
+    .bdrv_co_pdiscard             = qemu_gluster_co_pdiscard,
 #endif
 #ifdef CONFIG_GLUSTERFS_ZEROFILL
-    .bdrv_co_write_zeroes         = qemu_gluster_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes        = qemu_gluster_co_pwrite_zeroes,
 #endif
+    .bdrv_co_get_block_status     = qemu_gluster_co_get_block_status,
     .create_opts                  = &qemu_gluster_create_opts,
 };
 
diff --git a/block/io.c b/block/io.c
new file mode 100644
index 0000000..420944d
--- /dev/null
+++ b/block/io.c
@@ -0,0 +1,2736 @@
+/*
+ * Block layer I/O functions
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "trace.h"
+#include "sysemu/block-backend.h"
+#include "block/blockjob.h"
+#include "block/block_int.h"
+#include "qemu/cutils.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
+
+#define NOT_DONE 0x7fffffff /* used while emulated sync operation in progress */
+
+static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
+                                          int64_t offset,
+                                          QEMUIOVector *qiov,
+                                          BdrvRequestFlags flags,
+                                          BlockCompletionFunc *cb,
+                                          void *opaque,
+                                          bool is_write);
+static void coroutine_fn bdrv_co_do_rw(void *opaque);
+static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
+    int64_t offset, int count, BdrvRequestFlags flags);
+
+static void bdrv_parent_drained_begin(BlockDriverState *bs)
+{
+    BdrvChild *c;
+
+    QLIST_FOREACH(c, &bs->parents, next_parent) {
+        if (c->role->drained_begin) {
+            c->role->drained_begin(c);
+        }
+    }
+}
+
+static void bdrv_parent_drained_end(BlockDriverState *bs)
+{
+    BdrvChild *c;
+
+    QLIST_FOREACH(c, &bs->parents, next_parent) {
+        if (c->role->drained_end) {
+            c->role->drained_end(c);
+        }
+    }
+}
+
+static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
+{
+    dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
+    dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
+    dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
+                                 src->opt_mem_alignment);
+    dst->min_mem_alignment = MAX(dst->min_mem_alignment,
+                                 src->min_mem_alignment);
+    dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
+}
+
+void bdrv_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    BlockDriver *drv = bs->drv;
+    Error *local_err = NULL;
+
+    memset(&bs->bl, 0, sizeof(bs->bl));
+
+    if (!drv) {
+        return;
+    }
+
+    /* Default alignment based on whether driver has byte interface */
+    bs->bl.request_alignment = drv->bdrv_co_preadv ? 1 : 512;
+
+    /* Take some limits from the children as a default */
+    if (bs->file) {
+        bdrv_refresh_limits(bs->file->bs, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+        bdrv_merge_limits(&bs->bl, &bs->file->bs->bl);
+    } else {
+        bs->bl.min_mem_alignment = 512;
+        bs->bl.opt_mem_alignment = getpagesize();
+
+        /* Safe default since most protocols use readv()/writev()/etc */
+        bs->bl.max_iov = IOV_MAX;
+    }
+
+    if (bs->backing) {
+        bdrv_refresh_limits(bs->backing->bs, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+        bdrv_merge_limits(&bs->bl, &bs->backing->bs->bl);
+    }
+
+    /* Then let the driver override it */
+    if (drv->bdrv_refresh_limits) {
+        drv->bdrv_refresh_limits(bs, errp);
+    }
+}
+
+/**
+ * The copy-on-read flag is actually a reference count so multiple users may
+ * use the feature without worrying about clobbering its previous state.
+ * Copy-on-read stays enabled until all users have called to disable it.
+ */
+void bdrv_enable_copy_on_read(BlockDriverState *bs)
+{
+    bs->copy_on_read++;
+}
+
+void bdrv_disable_copy_on_read(BlockDriverState *bs)
+{
+    assert(bs->copy_on_read > 0);
+    bs->copy_on_read--;
+}
+
+/* Check if any requests are in-flight (including throttled requests) */
+bool bdrv_requests_pending(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    if (!QLIST_EMPTY(&bs->tracked_requests)) {
+        return true;
+    }
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        if (bdrv_requests_pending(child->bs)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void bdrv_drain_recurse(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    if (bs->drv && bs->drv->bdrv_drain) {
+        bs->drv->bdrv_drain(bs);
+    }
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_drain_recurse(child->bs);
+    }
+}
+
+typedef struct {
+    Coroutine *co;
+    BlockDriverState *bs;
+    QEMUBH *bh;
+    bool done;
+} BdrvCoDrainData;
+
+static void bdrv_drain_poll(BlockDriverState *bs)
+{
+    bool busy = true;
+
+    while (busy) {
+        /* Keep iterating */
+        busy = bdrv_requests_pending(bs);
+        busy |= aio_poll(bdrv_get_aio_context(bs), busy);
+    }
+}
+
+static void bdrv_co_drain_bh_cb(void *opaque)
+{
+    BdrvCoDrainData *data = opaque;
+    Coroutine *co = data->co;
+
+    qemu_bh_delete(data->bh);
+    bdrv_drain_poll(data->bs);
+    data->done = true;
+    qemu_coroutine_enter(co);
+}
+
+static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs)
+{
+    BdrvCoDrainData data;
+
+    /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
+     * other coroutines run if they were queued from
+     * qemu_co_queue_run_restart(). */
+
+    assert(qemu_in_coroutine());
+    data = (BdrvCoDrainData) {
+        .co = qemu_coroutine_self(),
+        .bs = bs,
+        .done = false,
+        .bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_drain_bh_cb, &data),
+    };
+    qemu_bh_schedule(data.bh);
+
+    qemu_coroutine_yield();
+    /* If we are resumed from some other event (such as an aio completion or a
+     * timer callback), it is a bug in the caller that should be fixed. */
+    assert(data.done);
+}
+
+void bdrv_drained_begin(BlockDriverState *bs)
+{
+    if (!bs->quiesce_counter++) {
+        aio_disable_external(bdrv_get_aio_context(bs));
+        bdrv_parent_drained_begin(bs);
+    }
+
+    bdrv_io_unplugged_begin(bs);
+    bdrv_drain_recurse(bs);
+    if (qemu_in_coroutine()) {
+        bdrv_co_yield_to_drain(bs);
+    } else {
+        bdrv_drain_poll(bs);
+    }
+    bdrv_io_unplugged_end(bs);
+}
+
+void bdrv_drained_end(BlockDriverState *bs)
+{
+    assert(bs->quiesce_counter > 0);
+    if (--bs->quiesce_counter > 0) {
+        return;
+    }
+
+    bdrv_parent_drained_end(bs);
+    aio_enable_external(bdrv_get_aio_context(bs));
+}
+
+/*
+ * Wait for pending requests to complete on a single BlockDriverState subtree,
+ * and suspend block driver's internal I/O until next request arrives.
+ *
+ * Note that unlike bdrv_drain_all(), the caller must hold the BlockDriverState
+ * AioContext.
+ *
+ * Only this BlockDriverState's AioContext is run, so in-flight requests must
+ * not depend on events in other AioContexts.  In that case, use
+ * bdrv_drain_all() instead.
+ */
+void coroutine_fn bdrv_co_drain(BlockDriverState *bs)
+{
+    assert(qemu_in_coroutine());
+    bdrv_drained_begin(bs);
+    bdrv_drained_end(bs);
+}
+
+void bdrv_drain(BlockDriverState *bs)
+{
+    bdrv_drained_begin(bs);
+    bdrv_drained_end(bs);
+}
+
+/*
+ * Wait for pending requests to complete across all BlockDriverStates
+ *
+ * This function does not flush data to disk, use bdrv_flush_all() for that
+ * after calling this function.
+ */
+void bdrv_drain_all(void)
+{
+    /* Always run first iteration so any pending completion BHs run */
+    bool busy = true;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+    BlockJob *job = NULL;
+    GSList *aio_ctxs = NULL, *ctx;
+
+    while ((job = block_job_next(job))) {
+        AioContext *aio_context = blk_get_aio_context(job->blk);
+
+        aio_context_acquire(aio_context);
+        block_job_pause(job);
+        aio_context_release(aio_context);
+    }
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(aio_context);
+        bdrv_parent_drained_begin(bs);
+        bdrv_io_unplugged_begin(bs);
+        bdrv_drain_recurse(bs);
+        aio_context_release(aio_context);
+
+        if (!g_slist_find(aio_ctxs, aio_context)) {
+            aio_ctxs = g_slist_prepend(aio_ctxs, aio_context);
+        }
+    }
+
+    /* Note that completion of an asynchronous I/O operation can trigger any
+     * number of other I/O operations on other devices---for example a
+     * coroutine can submit an I/O request to another device in response to
+     * request completion.  Therefore we must keep looping until there was no
+     * more activity rather than simply draining each device independently.
+     */
+    while (busy) {
+        busy = false;
+
+        for (ctx = aio_ctxs; ctx != NULL; ctx = ctx->next) {
+            AioContext *aio_context = ctx->data;
+
+            aio_context_acquire(aio_context);
+            for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+                if (aio_context == bdrv_get_aio_context(bs)) {
+                    if (bdrv_requests_pending(bs)) {
+                        busy = true;
+                        aio_poll(aio_context, busy);
+                    }
+                }
+            }
+            busy |= aio_poll(aio_context, false);
+            aio_context_release(aio_context);
+        }
+    }
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(aio_context);
+        bdrv_io_unplugged_end(bs);
+        bdrv_parent_drained_end(bs);
+        aio_context_release(aio_context);
+    }
+    g_slist_free(aio_ctxs);
+
+    job = NULL;
+    while ((job = block_job_next(job))) {
+        AioContext *aio_context = blk_get_aio_context(job->blk);
+
+        aio_context_acquire(aio_context);
+        block_job_resume(job);
+        aio_context_release(aio_context);
+    }
+}
+
+/**
+ * Remove an active request from the tracked requests list
+ *
+ * This function should be called when a tracked request is completing.
+ */
+static void tracked_request_end(BdrvTrackedRequest *req)
+{
+    if (req->serialising) {
+        req->bs->serialising_in_flight--;
+    }
+
+    QLIST_REMOVE(req, list);
+    qemu_co_queue_restart_all(&req->wait_queue);
+}
+
+/**
+ * Add an active request to the tracked requests list
+ */
+static void tracked_request_begin(BdrvTrackedRequest *req,
+                                  BlockDriverState *bs,
+                                  int64_t offset,
+                                  unsigned int bytes,
+                                  enum BdrvTrackedRequestType type)
+{
+    *req = (BdrvTrackedRequest){
+        .bs = bs,
+        .offset         = offset,
+        .bytes          = bytes,
+        .type           = type,
+        .co             = qemu_coroutine_self(),
+        .serialising    = false,
+        .overlap_offset = offset,
+        .overlap_bytes  = bytes,
+    };
+
+    qemu_co_queue_init(&req->wait_queue);
+
+    QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
+}
+
+static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
+{
+    int64_t overlap_offset = req->offset & ~(align - 1);
+    unsigned int overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
+                               - overlap_offset;
+
+    if (!req->serialising) {
+        req->bs->serialising_in_flight++;
+        req->serialising = true;
+    }
+
+    req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
+    req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
+}
+
+/**
+ * Round a region to cluster boundaries (sector-based)
+ */
+void bdrv_round_sectors_to_clusters(BlockDriverState *bs,
+                                    int64_t sector_num, int nb_sectors,
+                                    int64_t *cluster_sector_num,
+                                    int *cluster_nb_sectors)
+{
+    BlockDriverInfo bdi;
+
+    if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
+        *cluster_sector_num = sector_num;
+        *cluster_nb_sectors = nb_sectors;
+    } else {
+        int64_t c = bdi.cluster_size / BDRV_SECTOR_SIZE;
+        *cluster_sector_num = QEMU_ALIGN_DOWN(sector_num, c);
+        *cluster_nb_sectors = QEMU_ALIGN_UP(sector_num - *cluster_sector_num +
+                                            nb_sectors, c);
+    }
+}
+
+/**
+ * Round a region to cluster boundaries
+ */
+void bdrv_round_to_clusters(BlockDriverState *bs,
+                            int64_t offset, unsigned int bytes,
+                            int64_t *cluster_offset,
+                            unsigned int *cluster_bytes)
+{
+    BlockDriverInfo bdi;
+
+    if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
+        *cluster_offset = offset;
+        *cluster_bytes = bytes;
+    } else {
+        int64_t c = bdi.cluster_size;
+        *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
+        *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
+    }
+}
+
+static int bdrv_get_cluster_size(BlockDriverState *bs)
+{
+    BlockDriverInfo bdi;
+    int ret;
+
+    ret = bdrv_get_info(bs, &bdi);
+    if (ret < 0 || bdi.cluster_size == 0) {
+        return bs->bl.request_alignment;
+    } else {
+        return bdi.cluster_size;
+    }
+}
+
+static bool tracked_request_overlaps(BdrvTrackedRequest *req,
+                                     int64_t offset, unsigned int bytes)
+{
+    /*        aaaa   bbbb */
+    if (offset >= req->overlap_offset + req->overlap_bytes) {
+        return false;
+    }
+    /* bbbb   aaaa        */
+    if (req->overlap_offset >= offset + bytes) {
+        return false;
+    }
+    return true;
+}
+
+static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
+{
+    BlockDriverState *bs = self->bs;
+    BdrvTrackedRequest *req;
+    bool retry;
+    bool waited = false;
+
+    if (!bs->serialising_in_flight) {
+        return false;
+    }
+
+    do {
+        retry = false;
+        QLIST_FOREACH(req, &bs->tracked_requests, list) {
+            if (req == self || (!req->serialising && !self->serialising)) {
+                continue;
+            }
+            if (tracked_request_overlaps(req, self->overlap_offset,
+                                         self->overlap_bytes))
+            {
+                /* Hitting this means there was a reentrant request, for
+                 * example, a block driver issuing nested requests.  This must
+                 * never happen since it means deadlock.
+                 */
+                assert(qemu_coroutine_self() != req->co);
+
+                /* If the request is already (indirectly) waiting for us, or
+                 * will wait for us as soon as it wakes up, then just go on
+                 * (instead of producing a deadlock in the former case). */
+                if (!req->waiting_for) {
+                    self->waiting_for = req;
+                    qemu_co_queue_wait(&req->wait_queue);
+                    self->waiting_for = NULL;
+                    retry = true;
+                    waited = true;
+                    break;
+                }
+            }
+        }
+    } while (retry);
+
+    return waited;
+}
+
+static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
+                                   size_t size)
+{
+    if (size > BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS) {
+        return -EIO;
+    }
+
+    if (!bdrv_is_inserted(bs)) {
+        return -ENOMEDIUM;
+    }
+
+    if (offset < 0) {
+        return -EIO;
+    }
+
+    return 0;
+}
+
+static int bdrv_check_request(BlockDriverState *bs, int64_t sector_num,
+                              int nb_sectors)
+{
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return -EIO;
+    }
+
+    return bdrv_check_byte_request(bs, sector_num * BDRV_SECTOR_SIZE,
+                                   nb_sectors * BDRV_SECTOR_SIZE);
+}
+
+typedef struct RwCo {
+    BdrvChild *child;
+    int64_t offset;
+    QEMUIOVector *qiov;
+    bool is_write;
+    int ret;
+    BdrvRequestFlags flags;
+} RwCo;
+
+static void coroutine_fn bdrv_rw_co_entry(void *opaque)
+{
+    RwCo *rwco = opaque;
+
+    if (!rwco->is_write) {
+        rwco->ret = bdrv_co_preadv(rwco->child, rwco->offset,
+                                   rwco->qiov->size, rwco->qiov,
+                                   rwco->flags);
+    } else {
+        rwco->ret = bdrv_co_pwritev(rwco->child, rwco->offset,
+                                    rwco->qiov->size, rwco->qiov,
+                                    rwco->flags);
+    }
+}
+
+/*
+ * Process a vectored synchronous request using coroutines
+ */
+static int bdrv_prwv_co(BdrvChild *child, int64_t offset,
+                        QEMUIOVector *qiov, bool is_write,
+                        BdrvRequestFlags flags)
+{
+    Coroutine *co;
+    RwCo rwco = {
+        .child = child,
+        .offset = offset,
+        .qiov = qiov,
+        .is_write = is_write,
+        .ret = NOT_DONE,
+        .flags = flags,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_rw_co_entry(&rwco);
+    } else {
+        AioContext *aio_context = bdrv_get_aio_context(child->bs);
+
+        co = qemu_coroutine_create(bdrv_rw_co_entry, &rwco);
+        qemu_coroutine_enter(co);
+        while (rwco.ret == NOT_DONE) {
+            aio_poll(aio_context, true);
+        }
+    }
+    return rwco.ret;
+}
+
+/*
+ * Process a synchronous request using coroutines
+ */
+static int bdrv_rw_co(BdrvChild *child, int64_t sector_num, uint8_t *buf,
+                      int nb_sectors, bool is_write, BdrvRequestFlags flags)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = nb_sectors * BDRV_SECTOR_SIZE,
+    };
+
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return -EINVAL;
+    }
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_prwv_co(child, sector_num << BDRV_SECTOR_BITS,
+                        &qiov, is_write, flags);
+}
+
+/* return < 0 if error. See bdrv_write() for the return codes */
+int bdrv_read(BdrvChild *child, int64_t sector_num,
+              uint8_t *buf, int nb_sectors)
+{
+    return bdrv_rw_co(child, sector_num, buf, nb_sectors, false, 0);
+}
+
+/* Return < 0 if error. Important errors are:
+  -EIO         generic I/O error (may happen for all errors)
+  -ENOMEDIUM   No media inserted.
+  -EINVAL      Invalid sector number or nb_sectors
+  -EACCES      Trying to write a read-only device
+*/
+int bdrv_write(BdrvChild *child, int64_t sector_num,
+               const uint8_t *buf, int nb_sectors)
+{
+    return bdrv_rw_co(child, sector_num, (uint8_t *)buf, nb_sectors, true, 0);
+}
+
+int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
+                       int count, BdrvRequestFlags flags)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base = NULL,
+        .iov_len = count,
+    };
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_prwv_co(child, offset, &qiov, true,
+                        BDRV_REQ_ZERO_WRITE | flags);
+}
+
+/*
+ * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
+ * The operation is sped up by checking the block status and only writing
+ * zeroes to the device if they currently do not return zeroes. Optional
+ * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
+ * BDRV_REQ_FUA).
+ *
+ * Returns < 0 on error, 0 on success. For error codes see bdrv_write().
+ */
+int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
+{
+    int64_t target_sectors, ret, nb_sectors, sector_num = 0;
+    BlockDriverState *bs = child->bs;
+    BlockDriverState *file;
+    int n;
+
+    target_sectors = bdrv_nb_sectors(bs);
+    if (target_sectors < 0) {
+        return target_sectors;
+    }
+
+    for (;;) {
+        nb_sectors = MIN(target_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
+        if (nb_sectors <= 0) {
+            return 0;
+        }
+        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &n, &file);
+        if (ret < 0) {
+            error_report("error getting block status at sector %" PRId64 ": %s",
+                         sector_num, strerror(-ret));
+            return ret;
+        }
+        if (ret & BDRV_BLOCK_ZERO) {
+            sector_num += n;
+            continue;
+        }
+        ret = bdrv_pwrite_zeroes(child, sector_num << BDRV_SECTOR_BITS,
+                                 n << BDRV_SECTOR_BITS, flags);
+        if (ret < 0) {
+            error_report("error writing zeroes at sector %" PRId64 ": %s",
+                         sector_num, strerror(-ret));
+            return ret;
+        }
+        sector_num += n;
+    }
+}
+
+int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
+{
+    int ret;
+
+    ret = bdrv_prwv_co(child, offset, qiov, false, 0);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return qiov->size;
+}
+
+int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = bytes,
+    };
+
+    if (bytes < 0) {
+        return -EINVAL;
+    }
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_preadv(child, offset, &qiov);
+}
+
+int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov)
+{
+    int ret;
+
+    ret = bdrv_prwv_co(child, offset, qiov, true, 0);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return qiov->size;
+}
+
+int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base   = (void *) buf,
+        .iov_len    = bytes,
+    };
+
+    if (bytes < 0) {
+        return -EINVAL;
+    }
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    return bdrv_pwritev(child, offset, &qiov);
+}
+
+/*
+ * Writes to the file and ensures that no writes are reordered across this
+ * request (acts as a barrier)
+ *
+ * Returns 0 on success, -errno in error cases.
+ */
+int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
+                     const void *buf, int count)
+{
+    int ret;
+
+    ret = bdrv_pwrite(child, offset, buf, count);
+    if (ret < 0) {
+        return ret;
+    }
+
+    ret = bdrv_flush(child->bs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return 0;
+}
+
+typedef struct CoroutineIOCompletion {
+    Coroutine *coroutine;
+    int ret;
+} CoroutineIOCompletion;
+
+static void bdrv_co_io_em_complete(void *opaque, int ret)
+{
+    CoroutineIOCompletion *co = opaque;
+
+    co->ret = ret;
+    qemu_coroutine_enter(co->coroutine);
+}
+
+static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
+                                           uint64_t offset, uint64_t bytes,
+                                           QEMUIOVector *qiov, int flags)
+{
+    BlockDriver *drv = bs->drv;
+    int64_t sector_num;
+    unsigned int nb_sectors;
+
+    assert(!(flags & ~BDRV_REQ_MASK));
+
+    if (drv->bdrv_co_preadv) {
+        return drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
+    }
+
+    sector_num = offset >> BDRV_SECTOR_BITS;
+    nb_sectors = bytes >> BDRV_SECTOR_BITS;
+
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
+
+    if (drv->bdrv_co_readv) {
+        return drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
+    } else {
+        BlockAIOCB *acb;
+        CoroutineIOCompletion co = {
+            .coroutine = qemu_coroutine_self(),
+        };
+
+        acb = bs->drv->bdrv_aio_readv(bs, sector_num, qiov, nb_sectors,
+                                      bdrv_co_io_em_complete, &co);
+        if (acb == NULL) {
+            return -EIO;
+        } else {
+            qemu_coroutine_yield();
+            return co.ret;
+        }
+    }
+}
+
+static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
+                                            uint64_t offset, uint64_t bytes,
+                                            QEMUIOVector *qiov, int flags)
+{
+    BlockDriver *drv = bs->drv;
+    int64_t sector_num;
+    unsigned int nb_sectors;
+    int ret;
+
+    assert(!(flags & ~BDRV_REQ_MASK));
+
+    if (drv->bdrv_co_pwritev) {
+        ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
+                                   flags & bs->supported_write_flags);
+        flags &= ~bs->supported_write_flags;
+        goto emulate_flags;
+    }
+
+    sector_num = offset >> BDRV_SECTOR_BITS;
+    nb_sectors = bytes >> BDRV_SECTOR_BITS;
+
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes >> BDRV_SECTOR_BITS) <= BDRV_REQUEST_MAX_SECTORS);
+
+    if (drv->bdrv_co_writev_flags) {
+        ret = drv->bdrv_co_writev_flags(bs, sector_num, nb_sectors, qiov,
+                                        flags & bs->supported_write_flags);
+        flags &= ~bs->supported_write_flags;
+    } else if (drv->bdrv_co_writev) {
+        assert(!bs->supported_write_flags);
+        ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov);
+    } else {
+        BlockAIOCB *acb;
+        CoroutineIOCompletion co = {
+            .coroutine = qemu_coroutine_self(),
+        };
+
+        acb = bs->drv->bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
+                                       bdrv_co_io_em_complete, &co);
+        if (acb == NULL) {
+            ret = -EIO;
+        } else {
+            qemu_coroutine_yield();
+            ret = co.ret;
+        }
+    }
+
+emulate_flags:
+    if (ret == 0 && (flags & BDRV_REQ_FUA)) {
+        ret = bdrv_co_flush(bs);
+    }
+
+    return ret;
+}
+
+static int coroutine_fn bdrv_co_do_copy_on_readv(BlockDriverState *bs,
+        int64_t offset, unsigned int bytes, QEMUIOVector *qiov)
+{
+    /* Perform I/O through a temporary buffer so that users who scribble over
+     * their read buffer while the operation is in progress do not end up
+     * modifying the image file.  This is critical for zero-copy guest I/O
+     * where anything might happen inside guest memory.
+     */
+    void *bounce_buffer;
+
+    BlockDriver *drv = bs->drv;
+    struct iovec iov;
+    QEMUIOVector bounce_qiov;
+    int64_t cluster_offset;
+    unsigned int cluster_bytes;
+    size_t skip_bytes;
+    int ret;
+
+    /* Cover entire cluster so no additional backing file I/O is required when
+     * allocating cluster in the image file.
+     */
+    bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
+
+    trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
+                                   cluster_offset, cluster_bytes);
+
+    iov.iov_len = cluster_bytes;
+    iov.iov_base = bounce_buffer = qemu_try_blockalign(bs, iov.iov_len);
+    if (bounce_buffer == NULL) {
+        ret = -ENOMEM;
+        goto err;
+    }
+
+    qemu_iovec_init_external(&bounce_qiov, &iov, 1);
+
+    ret = bdrv_driver_preadv(bs, cluster_offset, cluster_bytes,
+                             &bounce_qiov, 0);
+    if (ret < 0) {
+        goto err;
+    }
+
+    if (drv->bdrv_co_pwrite_zeroes &&
+        buffer_is_zero(bounce_buffer, iov.iov_len)) {
+        /* FIXME: Should we (perhaps conditionally) be setting
+         * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
+         * that still correctly reads as zero? */
+        ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, cluster_bytes, 0);
+    } else {
+        /* This does not change the data on the disk, it is not necessary
+         * to flush even in cache=writethrough mode.
+         */
+        ret = bdrv_driver_pwritev(bs, cluster_offset, cluster_bytes,
+                                  &bounce_qiov, 0);
+    }
+
+    if (ret < 0) {
+        /* It might be okay to ignore write errors for guest requests.  If this
+         * is a deliberate copy-on-read then we don't want to ignore the error.
+         * Simply report it in all cases.
+         */
+        goto err;
+    }
+
+    skip_bytes = offset - cluster_offset;
+    qemu_iovec_from_buf(qiov, 0, bounce_buffer + skip_bytes, bytes);
+
+err:
+    qemu_vfree(bounce_buffer);
+    return ret;
+}
+
+/*
+ * Forwards an already correctly aligned request to the BlockDriver. This
+ * handles copy on read, zeroing after EOF, and fragmentation of large
+ * reads; any other features must be implemented by the caller.
+ */
+static int coroutine_fn bdrv_aligned_preadv(BlockDriverState *bs,
+    BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
+    int64_t align, QEMUIOVector *qiov, int flags)
+{
+    int64_t total_bytes, max_bytes;
+    int ret = 0;
+    uint64_t bytes_remaining = bytes;
+    int max_transfer;
+
+    assert(is_power_of_2(align));
+    assert((offset & (align - 1)) == 0);
+    assert((bytes & (align - 1)) == 0);
+    assert(!qiov || bytes == qiov->size);
+    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
+    max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
+                                   align);
+
+    /* TODO: We would need a per-BDS .supported_read_flags and
+     * potential fallback support, if we ever implement any read flags
+     * to pass through to drivers.  For now, there aren't any
+     * passthrough flags.  */
+    assert(!(flags & ~(BDRV_REQ_NO_SERIALISING | BDRV_REQ_COPY_ON_READ)));
+
+    /* Handle Copy on Read and associated serialisation */
+    if (flags & BDRV_REQ_COPY_ON_READ) {
+        /* If we touch the same cluster it counts as an overlap.  This
+         * guarantees that allocating writes will be serialized and not race
+         * with each other for the same cluster.  For example, in copy-on-read
+         * it ensures that the CoR read and write operations are atomic and
+         * guest writes cannot interleave between them. */
+        mark_request_serialising(req, bdrv_get_cluster_size(bs));
+    }
+
+    if (!(flags & BDRV_REQ_NO_SERIALISING)) {
+        wait_serialising_requests(req);
+    }
+
+    if (flags & BDRV_REQ_COPY_ON_READ) {
+        int64_t start_sector = offset >> BDRV_SECTOR_BITS;
+        int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
+        unsigned int nb_sectors = end_sector - start_sector;
+        int pnum;
+
+        ret = bdrv_is_allocated(bs, start_sector, nb_sectors, &pnum);
+        if (ret < 0) {
+            goto out;
+        }
+
+        if (!ret || pnum != nb_sectors) {
+            ret = bdrv_co_do_copy_on_readv(bs, offset, bytes, qiov);
+            goto out;
+        }
+    }
+
+    /* Forward the request to the BlockDriver, possibly fragmenting it */
+    total_bytes = bdrv_getlength(bs);
+    if (total_bytes < 0) {
+        ret = total_bytes;
+        goto out;
+    }
+
+    max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
+    if (bytes <= max_bytes && bytes <= max_transfer) {
+        ret = bdrv_driver_preadv(bs, offset, bytes, qiov, 0);
+        goto out;
+    }
+
+    while (bytes_remaining) {
+        int num;
+
+        if (max_bytes) {
+            QEMUIOVector local_qiov;
+
+            num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
+            assert(num);
+            qemu_iovec_init(&local_qiov, qiov->niov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
+
+            ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
+                                     num, &local_qiov, 0);
+            max_bytes -= num;
+            qemu_iovec_destroy(&local_qiov);
+        } else {
+            num = bytes_remaining;
+            ret = qemu_iovec_memset(qiov, bytes - bytes_remaining, 0,
+                                    bytes_remaining);
+        }
+        if (ret < 0) {
+            goto out;
+        }
+        bytes_remaining -= num;
+    }
+
+out:
+    return ret < 0 ? ret : 0;
+}
+
+/*
+ * Handle a read request in coroutine context
+ */
+int coroutine_fn bdrv_co_preadv(BdrvChild *child,
+    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
+    BdrvRequestFlags flags)
+{
+    BlockDriverState *bs = child->bs;
+    BlockDriver *drv = bs->drv;
+    BdrvTrackedRequest req;
+
+    uint64_t align = bs->bl.request_alignment;
+    uint8_t *head_buf = NULL;
+    uint8_t *tail_buf = NULL;
+    QEMUIOVector local_qiov;
+    bool use_local_qiov = false;
+    int ret;
+
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+
+    ret = bdrv_check_byte_request(bs, offset, bytes);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* Don't do copy-on-read if we read data before write operation */
+    if (bs->copy_on_read && !(flags & BDRV_REQ_NO_SERIALISING)) {
+        flags |= BDRV_REQ_COPY_ON_READ;
+    }
+
+    /* Align read if necessary by padding qiov */
+    if (offset & (align - 1)) {
+        head_buf = qemu_blockalign(bs, align);
+        qemu_iovec_init(&local_qiov, qiov->niov + 2);
+        qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
+        qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
+        use_local_qiov = true;
+
+        bytes += offset & (align - 1);
+        offset = offset & ~(align - 1);
+    }
+
+    if ((offset + bytes) & (align - 1)) {
+        if (!use_local_qiov) {
+            qemu_iovec_init(&local_qiov, qiov->niov + 1);
+            qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
+            use_local_qiov = true;
+        }
+        tail_buf = qemu_blockalign(bs, align);
+        qemu_iovec_add(&local_qiov, tail_buf,
+                       align - ((offset + bytes) & (align - 1)));
+
+        bytes = ROUND_UP(bytes, align);
+    }
+
+    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
+    ret = bdrv_aligned_preadv(bs, &req, offset, bytes, align,
+                              use_local_qiov ? &local_qiov : qiov,
+                              flags);
+    tracked_request_end(&req);
+
+    if (use_local_qiov) {
+        qemu_iovec_destroy(&local_qiov);
+        qemu_vfree(head_buf);
+        qemu_vfree(tail_buf);
+    }
+
+    return ret;
+}
+
+static int coroutine_fn bdrv_co_do_readv(BdrvChild *child,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags)
+{
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return -EINVAL;
+    }
+
+    return bdrv_co_preadv(child, sector_num << BDRV_SECTOR_BITS,
+                          nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
+}
+
+int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
+                               int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_readv(child->bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_readv(child, sector_num, nb_sectors, qiov, 0);
+}
+
+/* Maximum buffer for write zeroes fallback, in bytes */
+#define MAX_WRITE_ZEROES_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
+
+static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
+    int64_t offset, int count, BdrvRequestFlags flags)
+{
+    BlockDriver *drv = bs->drv;
+    QEMUIOVector qiov;
+    struct iovec iov = {0};
+    int ret = 0;
+    bool need_flush = false;
+    int head = 0;
+    int tail = 0;
+
+    int max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes, INT_MAX);
+    int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
+                        bs->bl.request_alignment);
+
+    assert(alignment % bs->bl.request_alignment == 0);
+    head = offset % alignment;
+    tail = (offset + count) % alignment;
+    max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
+    assert(max_write_zeroes >= bs->bl.request_alignment);
+
+    while (count > 0 && !ret) {
+        int num = count;
+
+        /* Align request.  Block drivers can expect the "bulk" of the request
+         * to be aligned, and that unaligned requests do not cross cluster
+         * boundaries.
+         */
+        if (head) {
+            /* Make a small request up to the first aligned sector.  */
+            num = MIN(count, alignment - head);
+            head = 0;
+        } else if (tail && num > alignment) {
+            /* Shorten the request to the last aligned sector.  */
+            num -= tail;
+        }
+
+        /* limit request size */
+        if (num > max_write_zeroes) {
+            num = max_write_zeroes;
+        }
+
+        ret = -ENOTSUP;
+        /* First try the efficient write zeroes operation */
+        if (drv->bdrv_co_pwrite_zeroes) {
+            ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
+                                             flags & bs->supported_zero_flags);
+            if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
+                !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
+                need_flush = true;
+            }
+        } else {
+            assert(!bs->supported_zero_flags);
+        }
+
+        if (ret == -ENOTSUP) {
+            /* Fall back to bounce buffer if write zeroes is unsupported */
+            int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
+                                            MAX_WRITE_ZEROES_BOUNCE_BUFFER);
+            BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
+
+            if ((flags & BDRV_REQ_FUA) &&
+                !(bs->supported_write_flags & BDRV_REQ_FUA)) {
+                /* No need for bdrv_driver_pwrite() to do a fallback
+                 * flush on each chunk; use just one at the end */
+                write_flags &= ~BDRV_REQ_FUA;
+                need_flush = true;
+            }
+            num = MIN(num, max_transfer);
+            iov.iov_len = num;
+            if (iov.iov_base == NULL) {
+                iov.iov_base = qemu_try_blockalign(bs, num);
+                if (iov.iov_base == NULL) {
+                    ret = -ENOMEM;
+                    goto fail;
+                }
+                memset(iov.iov_base, 0, num);
+            }
+            qemu_iovec_init_external(&qiov, &iov, 1);
+
+            ret = bdrv_driver_pwritev(bs, offset, num, &qiov, write_flags);
+
+            /* Keep bounce buffer around if it is big enough for all
+             * all future requests.
+             */
+            if (num < max_transfer) {
+                qemu_vfree(iov.iov_base);
+                iov.iov_base = NULL;
+            }
+        }
+
+        offset += num;
+        count -= num;
+    }
+
+fail:
+    if (ret == 0 && need_flush) {
+        ret = bdrv_co_flush(bs);
+    }
+    qemu_vfree(iov.iov_base);
+    return ret;
+}
+
+/*
+ * Forwards an already correctly aligned write request to the BlockDriver,
+ * after possibly fragmenting it.
+ */
+static int coroutine_fn bdrv_aligned_pwritev(BlockDriverState *bs,
+    BdrvTrackedRequest *req, int64_t offset, unsigned int bytes,
+    int64_t align, QEMUIOVector *qiov, int flags)
+{
+    BlockDriver *drv = bs->drv;
+    bool waited;
+    int ret;
+
+    int64_t start_sector = offset >> BDRV_SECTOR_BITS;
+    int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
+    uint64_t bytes_remaining = bytes;
+    int max_transfer;
+
+    assert(is_power_of_2(align));
+    assert((offset & (align - 1)) == 0);
+    assert((bytes & (align - 1)) == 0);
+    assert(!qiov || bytes == qiov->size);
+    assert((bs->open_flags & BDRV_O_NO_IO) == 0);
+    assert(!(flags & ~BDRV_REQ_MASK));
+    max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
+                                   align);
+
+    waited = wait_serialising_requests(req);
+    assert(!waited || !req->serialising);
+    assert(req->overlap_offset <= offset);
+    assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
+
+    ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
+
+    if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
+        !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
+        qemu_iovec_is_zero(qiov)) {
+        flags |= BDRV_REQ_ZERO_WRITE;
+        if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
+            flags |= BDRV_REQ_MAY_UNMAP;
+        }
+    }
+
+    if (ret < 0) {
+        /* Do nothing, write notifier decided to fail this request */
+    } else if (flags & BDRV_REQ_ZERO_WRITE) {
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
+        ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
+    } else if (bytes <= max_transfer) {
+        bdrv_debug_event(bs, BLKDBG_PWRITEV);
+        ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, flags);
+    } else {
+        bdrv_debug_event(bs, BLKDBG_PWRITEV);
+        while (bytes_remaining) {
+            int num = MIN(bytes_remaining, max_transfer);
+            QEMUIOVector local_qiov;
+            int local_flags = flags;
+
+            assert(num);
+            if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
+                !(bs->supported_write_flags & BDRV_REQ_FUA)) {
+                /* If FUA is going to be emulated by flush, we only
+                 * need to flush on the last iteration */
+                local_flags &= ~BDRV_REQ_FUA;
+            }
+            qemu_iovec_init(&local_qiov, qiov->niov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes - bytes_remaining, num);
+
+            ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
+                                      num, &local_qiov, local_flags);
+            qemu_iovec_destroy(&local_qiov);
+            if (ret < 0) {
+                break;
+            }
+            bytes_remaining -= num;
+        }
+    }
+    bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
+
+    ++bs->write_gen;
+    bdrv_set_dirty(bs, start_sector, end_sector - start_sector);
+
+    if (bs->wr_highest_offset < offset + bytes) {
+        bs->wr_highest_offset = offset + bytes;
+    }
+
+    if (ret >= 0) {
+        bs->total_sectors = MAX(bs->total_sectors, end_sector);
+        ret = 0;
+    }
+
+    return ret;
+}
+
+static int coroutine_fn bdrv_co_do_zero_pwritev(BlockDriverState *bs,
+                                                int64_t offset,
+                                                unsigned int bytes,
+                                                BdrvRequestFlags flags,
+                                                BdrvTrackedRequest *req)
+{
+    uint8_t *buf = NULL;
+    QEMUIOVector local_qiov;
+    struct iovec iov;
+    uint64_t align = bs->bl.request_alignment;
+    unsigned int head_padding_bytes, tail_padding_bytes;
+    int ret = 0;
+
+    head_padding_bytes = offset & (align - 1);
+    tail_padding_bytes = align - ((offset + bytes) & (align - 1));
+
+
+    assert(flags & BDRV_REQ_ZERO_WRITE);
+    if (head_padding_bytes || tail_padding_bytes) {
+        buf = qemu_blockalign(bs, align);
+        iov = (struct iovec) {
+            .iov_base   = buf,
+            .iov_len    = align,
+        };
+        qemu_iovec_init_external(&local_qiov, &iov, 1);
+    }
+    if (head_padding_bytes) {
+        uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
+
+        /* RMW the unaligned part before head. */
+        mark_request_serialising(req, align);
+        wait_serialising_requests(req);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
+        ret = bdrv_aligned_preadv(bs, req, offset & ~(align - 1), align,
+                                  align, &local_qiov, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+
+        memset(buf + head_padding_bytes, 0, zero_bytes);
+        ret = bdrv_aligned_pwritev(bs, req, offset & ~(align - 1), align,
+                                   align, &local_qiov,
+                                   flags & ~BDRV_REQ_ZERO_WRITE);
+        if (ret < 0) {
+            goto fail;
+        }
+        offset += zero_bytes;
+        bytes -= zero_bytes;
+    }
+
+    assert(!bytes || (offset & (align - 1)) == 0);
+    if (bytes >= align) {
+        /* Write the aligned part in the middle. */
+        uint64_t aligned_bytes = bytes & ~(align - 1);
+        ret = bdrv_aligned_pwritev(bs, req, offset, aligned_bytes, align,
+                                   NULL, flags);
+        if (ret < 0) {
+            goto fail;
+        }
+        bytes -= aligned_bytes;
+        offset += aligned_bytes;
+    }
+
+    assert(!bytes || (offset & (align - 1)) == 0);
+    if (bytes) {
+        assert(align == tail_padding_bytes + bytes);
+        /* RMW the unaligned part after tail. */
+        mark_request_serialising(req, align);
+        wait_serialising_requests(req);
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
+        ret = bdrv_aligned_preadv(bs, req, offset, align,
+                                  align, &local_qiov, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+
+        memset(buf, 0, bytes);
+        ret = bdrv_aligned_pwritev(bs, req, offset, align, align,
+                                   &local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
+    }
+fail:
+    qemu_vfree(buf);
+    return ret;
+
+}
+
+/*
+ * Handle a write request in coroutine context
+ */
+int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
+    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
+    BdrvRequestFlags flags)
+{
+    BlockDriverState *bs = child->bs;
+    BdrvTrackedRequest req;
+    uint64_t align = bs->bl.request_alignment;
+    uint8_t *head_buf = NULL;
+    uint8_t *tail_buf = NULL;
+    QEMUIOVector local_qiov;
+    bool use_local_qiov = false;
+    int ret;
+
+    if (!bs->drv) {
+        return -ENOMEDIUM;
+    }
+    if (bs->read_only) {
+        return -EPERM;
+    }
+    assert(!(bs->open_flags & BDRV_O_INACTIVE));
+
+    ret = bdrv_check_byte_request(bs, offset, bytes);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /*
+     * Align write if necessary by performing a read-modify-write cycle.
+     * Pad qiov with the read parts and be sure to have a tracked request not
+     * only for bdrv_aligned_pwritev, but also for the reads of the RMW cycle.
+     */
+    tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
+
+    if (!qiov) {
+        ret = bdrv_co_do_zero_pwritev(bs, offset, bytes, flags, &req);
+        goto out;
+    }
+
+    if (offset & (align - 1)) {
+        QEMUIOVector head_qiov;
+        struct iovec head_iov;
+
+        mark_request_serialising(&req, align);
+        wait_serialising_requests(&req);
+
+        head_buf = qemu_blockalign(bs, align);
+        head_iov = (struct iovec) {
+            .iov_base   = head_buf,
+            .iov_len    = align,
+        };
+        qemu_iovec_init_external(&head_qiov, &head_iov, 1);
+
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
+        ret = bdrv_aligned_preadv(bs, &req, offset & ~(align - 1), align,
+                                  align, &head_qiov, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+
+        qemu_iovec_init(&local_qiov, qiov->niov + 2);
+        qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
+        qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
+        use_local_qiov = true;
+
+        bytes += offset & (align - 1);
+        offset = offset & ~(align - 1);
+
+        /* We have read the tail already if the request is smaller
+         * than one aligned block.
+         */
+        if (bytes < align) {
+            qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes);
+            bytes = align;
+        }
+    }
+
+    if ((offset + bytes) & (align - 1)) {
+        QEMUIOVector tail_qiov;
+        struct iovec tail_iov;
+        size_t tail_bytes;
+        bool waited;
+
+        mark_request_serialising(&req, align);
+        waited = wait_serialising_requests(&req);
+        assert(!waited || !use_local_qiov);
+
+        tail_buf = qemu_blockalign(bs, align);
+        tail_iov = (struct iovec) {
+            .iov_base   = tail_buf,
+            .iov_len    = align,
+        };
+        qemu_iovec_init_external(&tail_qiov, &tail_iov, 1);
+
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
+        ret = bdrv_aligned_preadv(bs, &req, (offset + bytes) & ~(align - 1), align,
+                                  align, &tail_qiov, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+        bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+
+        if (!use_local_qiov) {
+            qemu_iovec_init(&local_qiov, qiov->niov + 1);
+            qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
+            use_local_qiov = true;
+        }
+
+        tail_bytes = (offset + bytes) & (align - 1);
+        qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
+
+        bytes = ROUND_UP(bytes, align);
+    }
+
+    ret = bdrv_aligned_pwritev(bs, &req, offset, bytes, align,
+                               use_local_qiov ? &local_qiov : qiov,
+                               flags);
+
+fail:
+
+    if (use_local_qiov) {
+        qemu_iovec_destroy(&local_qiov);
+    }
+    qemu_vfree(head_buf);
+    qemu_vfree(tail_buf);
+out:
+    tracked_request_end(&req);
+    return ret;
+}
+
+static int coroutine_fn bdrv_co_do_writev(BdrvChild *child,
+    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov,
+    BdrvRequestFlags flags)
+{
+    if (nb_sectors < 0 || nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return -EINVAL;
+    }
+
+    return bdrv_co_pwritev(child, sector_num << BDRV_SECTOR_BITS,
+                           nb_sectors << BDRV_SECTOR_BITS, qiov, flags);
+}
+
+int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
+    int nb_sectors, QEMUIOVector *qiov)
+{
+    trace_bdrv_co_writev(child->bs, sector_num, nb_sectors);
+
+    return bdrv_co_do_writev(child, sector_num, nb_sectors, qiov, 0);
+}
+
+int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
+                                       int count, BdrvRequestFlags flags)
+{
+    trace_bdrv_co_pwrite_zeroes(child->bs, offset, count, flags);
+
+    if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
+        flags &= ~BDRV_REQ_MAY_UNMAP;
+    }
+
+    return bdrv_co_pwritev(child, offset, count, NULL,
+                           BDRV_REQ_ZERO_WRITE | flags);
+}
+
+typedef struct BdrvCoGetBlockStatusData {
+    BlockDriverState *bs;
+    BlockDriverState *base;
+    BlockDriverState **file;
+    int64_t sector_num;
+    int nb_sectors;
+    int *pnum;
+    int64_t ret;
+    bool done;
+} BdrvCoGetBlockStatusData;
+
+/*
+ * Returns the allocation status of the specified sectors.
+ * Drivers not implementing the functionality are assumed to not support
+ * backing files, hence all their sectors are reported as allocated.
+ *
+ * If 'sector_num' is beyond the end of the disk image the return value is 0
+ * and 'pnum' is set to 0.
+ *
+ * 'pnum' is set to the number of sectors (including and immediately following
+ * the specified sector) that are known to be in the same
+ * allocated/unallocated state.
+ *
+ * 'nb_sectors' is the max value 'pnum' should be set to.  If nb_sectors goes
+ * beyond the end of the disk image it will be clamped.
+ *
+ * If returned value is positive and BDRV_BLOCK_OFFSET_VALID bit is set, 'file'
+ * points to the BDS which the sector range is allocated in.
+ */
+static int64_t coroutine_fn bdrv_co_get_block_status(BlockDriverState *bs,
+                                                     int64_t sector_num,
+                                                     int nb_sectors, int *pnum,
+                                                     BlockDriverState **file)
+{
+    int64_t total_sectors;
+    int64_t n;
+    int64_t ret, ret2;
+
+    total_sectors = bdrv_nb_sectors(bs);
+    if (total_sectors < 0) {
+        return total_sectors;
+    }
+
+    if (sector_num >= total_sectors) {
+        *pnum = 0;
+        return 0;
+    }
+
+    n = total_sectors - sector_num;
+    if (n < nb_sectors) {
+        nb_sectors = n;
+    }
+
+    if (!bs->drv->bdrv_co_get_block_status) {
+        *pnum = nb_sectors;
+        ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
+        if (bs->drv->protocol_name) {
+            ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
+        }
+        return ret;
+    }
+
+    *file = NULL;
+    ret = bs->drv->bdrv_co_get_block_status(bs, sector_num, nb_sectors, pnum,
+                                            file);
+    if (ret < 0) {
+        *pnum = 0;
+        return ret;
+    }
+
+    if (ret & BDRV_BLOCK_RAW) {
+        assert(ret & BDRV_BLOCK_OFFSET_VALID);
+        return bdrv_get_block_status(bs->file->bs, ret >> BDRV_SECTOR_BITS,
+                                     *pnum, pnum, file);
+    }
+
+    if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
+        ret |= BDRV_BLOCK_ALLOCATED;
+    } else {
+        if (bdrv_unallocated_blocks_are_zero(bs)) {
+            ret |= BDRV_BLOCK_ZERO;
+        } else if (bs->backing) {
+            BlockDriverState *bs2 = bs->backing->bs;
+            int64_t nb_sectors2 = bdrv_nb_sectors(bs2);
+            if (nb_sectors2 >= 0 && sector_num >= nb_sectors2) {
+                ret |= BDRV_BLOCK_ZERO;
+            }
+        }
+    }
+
+    if (*file && *file != bs &&
+        (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
+        (ret & BDRV_BLOCK_OFFSET_VALID)) {
+        BlockDriverState *file2;
+        int file_pnum;
+
+        ret2 = bdrv_co_get_block_status(*file, ret >> BDRV_SECTOR_BITS,
+                                        *pnum, &file_pnum, &file2);
+        if (ret2 >= 0) {
+            /* Ignore errors.  This is just providing extra information, it
+             * is useful but not necessary.
+             */
+            if (!file_pnum) {
+                /* !file_pnum indicates an offset at or beyond the EOF; it is
+                 * perfectly valid for the format block driver to point to such
+                 * offsets, so catch it and mark everything as zero */
+                ret |= BDRV_BLOCK_ZERO;
+            } else {
+                /* Limit request to the range reported by the protocol driver */
+                *pnum = file_pnum;
+                ret |= (ret2 & BDRV_BLOCK_ZERO);
+            }
+        }
+    }
+
+    return ret;
+}
+
+static int64_t coroutine_fn bdrv_co_get_block_status_above(BlockDriverState *bs,
+        BlockDriverState *base,
+        int64_t sector_num,
+        int nb_sectors,
+        int *pnum,
+        BlockDriverState **file)
+{
+    BlockDriverState *p;
+    int64_t ret = 0;
+
+    assert(bs != base);
+    for (p = bs; p != base; p = backing_bs(p)) {
+        ret = bdrv_co_get_block_status(p, sector_num, nb_sectors, pnum, file);
+        if (ret < 0 || ret & BDRV_BLOCK_ALLOCATED) {
+            break;
+        }
+        /* [sector_num, pnum] unallocated on this layer, which could be only
+         * the first part of [sector_num, nb_sectors].  */
+        nb_sectors = MIN(nb_sectors, *pnum);
+    }
+    return ret;
+}
+
+/* Coroutine wrapper for bdrv_get_block_status_above() */
+static void coroutine_fn bdrv_get_block_status_above_co_entry(void *opaque)
+{
+    BdrvCoGetBlockStatusData *data = opaque;
+
+    data->ret = bdrv_co_get_block_status_above(data->bs, data->base,
+                                               data->sector_num,
+                                               data->nb_sectors,
+                                               data->pnum,
+                                               data->file);
+    data->done = true;
+}
+
+/*
+ * Synchronous wrapper around bdrv_co_get_block_status_above().
+ *
+ * See bdrv_co_get_block_status_above() for details.
+ */
+int64_t bdrv_get_block_status_above(BlockDriverState *bs,
+                                    BlockDriverState *base,
+                                    int64_t sector_num,
+                                    int nb_sectors, int *pnum,
+                                    BlockDriverState **file)
+{
+    Coroutine *co;
+    BdrvCoGetBlockStatusData data = {
+        .bs = bs,
+        .base = base,
+        .file = file,
+        .sector_num = sector_num,
+        .nb_sectors = nb_sectors,
+        .pnum = pnum,
+        .done = false,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_get_block_status_above_co_entry(&data);
+    } else {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        co = qemu_coroutine_create(bdrv_get_block_status_above_co_entry,
+                                   &data);
+        qemu_coroutine_enter(co);
+        while (!data.done) {
+            aio_poll(aio_context, true);
+        }
+    }
+    return data.ret;
+}
+
+int64_t bdrv_get_block_status(BlockDriverState *bs,
+                              int64_t sector_num,
+                              int nb_sectors, int *pnum,
+                              BlockDriverState **file)
+{
+    return bdrv_get_block_status_above(bs, backing_bs(bs),
+                                       sector_num, nb_sectors, pnum, file);
+}
+
+int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num,
+                                   int nb_sectors, int *pnum)
+{
+    BlockDriverState *file;
+    int64_t ret = bdrv_get_block_status(bs, sector_num, nb_sectors, pnum,
+                                        &file);
+    if (ret < 0) {
+        return ret;
+    }
+    return !!(ret & BDRV_BLOCK_ALLOCATED);
+}
+
+/*
+ * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
+ *
+ * Return true if the given sector is allocated in any image between
+ * BASE and TOP (inclusive).  BASE can be NULL to check if the given
+ * sector is allocated in any image of the chain.  Return false otherwise.
+ *
+ * 'pnum' is set to the number of sectors (including and immediately following
+ *  the specified sector) that are known to be in the same
+ *  allocated/unallocated state.
+ *
+ */
+int bdrv_is_allocated_above(BlockDriverState *top,
+                            BlockDriverState *base,
+                            int64_t sector_num,
+                            int nb_sectors, int *pnum)
+{
+    BlockDriverState *intermediate;
+    int ret, n = nb_sectors;
+
+    intermediate = top;
+    while (intermediate && intermediate != base) {
+        int pnum_inter;
+        ret = bdrv_is_allocated(intermediate, sector_num, nb_sectors,
+                                &pnum_inter);
+        if (ret < 0) {
+            return ret;
+        } else if (ret) {
+            *pnum = pnum_inter;
+            return 1;
+        }
+
+        /*
+         * [sector_num, nb_sectors] is unallocated on top but intermediate
+         * might have
+         *
+         * [sector_num+x, nr_sectors] allocated.
+         */
+        if (n > pnum_inter &&
+            (intermediate == top ||
+             sector_num + pnum_inter < intermediate->total_sectors)) {
+            n = pnum_inter;
+        }
+
+        intermediate = backing_bs(intermediate);
+    }
+
+    *pnum = n;
+    return 0;
+}
+
+int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
+                          const uint8_t *buf, int nb_sectors)
+{
+    BlockDriver *drv = bs->drv;
+    int ret;
+
+    if (!drv) {
+        return -ENOMEDIUM;
+    }
+    if (!drv->bdrv_write_compressed) {
+        return -ENOTSUP;
+    }
+    ret = bdrv_check_request(bs, sector_num, nb_sectors);
+    if (ret < 0) {
+        return ret;
+    }
+
+    assert(QLIST_EMPTY(&bs->dirty_bitmaps));
+
+    return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
+}
+
+typedef struct BdrvVmstateCo {
+    BlockDriverState    *bs;
+    QEMUIOVector        *qiov;
+    int64_t             pos;
+    bool                is_read;
+    int                 ret;
+} BdrvVmstateCo;
+
+static int coroutine_fn
+bdrv_co_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
+                   bool is_read)
+{
+    BlockDriver *drv = bs->drv;
+
+    if (!drv) {
+        return -ENOMEDIUM;
+    } else if (drv->bdrv_load_vmstate) {
+        return is_read ? drv->bdrv_load_vmstate(bs, qiov, pos)
+                       : drv->bdrv_save_vmstate(bs, qiov, pos);
+    } else if (bs->file) {
+        return bdrv_co_rw_vmstate(bs->file->bs, qiov, pos, is_read);
+    }
+
+    return -ENOTSUP;
+}
+
+static void coroutine_fn bdrv_co_rw_vmstate_entry(void *opaque)
+{
+    BdrvVmstateCo *co = opaque;
+    co->ret = bdrv_co_rw_vmstate(co->bs, co->qiov, co->pos, co->is_read);
+}
+
+static inline int
+bdrv_rw_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos,
+                bool is_read)
+{
+    if (qemu_in_coroutine()) {
+        return bdrv_co_rw_vmstate(bs, qiov, pos, is_read);
+    } else {
+        BdrvVmstateCo data = {
+            .bs         = bs,
+            .qiov       = qiov,
+            .pos        = pos,
+            .is_read    = is_read,
+            .ret        = -EINPROGRESS,
+        };
+        Coroutine *co = qemu_coroutine_create(bdrv_co_rw_vmstate_entry, &data);
+
+        qemu_coroutine_enter(co);
+        while (data.ret == -EINPROGRESS) {
+            aio_poll(bdrv_get_aio_context(bs), true);
+        }
+        return data.ret;
+    }
+}
+
+int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
+                      int64_t pos, int size)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base   = (void *) buf,
+        .iov_len    = size,
+    };
+    int ret;
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    ret = bdrv_writev_vmstate(bs, &qiov, pos);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return size;
+}
+
+int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
+{
+    return bdrv_rw_vmstate(bs, qiov, pos, false);
+}
+
+int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
+                      int64_t pos, int size)
+{
+    QEMUIOVector qiov;
+    struct iovec iov = {
+        .iov_base   = buf,
+        .iov_len    = size,
+    };
+    int ret;
+
+    qemu_iovec_init_external(&qiov, &iov, 1);
+    ret = bdrv_readv_vmstate(bs, &qiov, pos);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return size;
+}
+
+int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
+{
+    return bdrv_rw_vmstate(bs, qiov, pos, true);
+}
+
+/**************************************************************/
+/* async I/Os */
+
+BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num,
+                           QEMUIOVector *qiov, int nb_sectors,
+                           BlockCompletionFunc *cb, void *opaque)
+{
+    trace_bdrv_aio_readv(child->bs, sector_num, nb_sectors, opaque);
+
+    assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
+    return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
+                                  0, cb, opaque, false);
+}
+
+BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num,
+                            QEMUIOVector *qiov, int nb_sectors,
+                            BlockCompletionFunc *cb, void *opaque)
+{
+    trace_bdrv_aio_writev(child->bs, sector_num, nb_sectors, opaque);
+
+    assert(nb_sectors << BDRV_SECTOR_BITS == qiov->size);
+    return bdrv_co_aio_prw_vector(child, sector_num << BDRV_SECTOR_BITS, qiov,
+                                  0, cb, opaque, true);
+}
+
+void bdrv_aio_cancel(BlockAIOCB *acb)
+{
+    qemu_aio_ref(acb);
+    bdrv_aio_cancel_async(acb);
+    while (acb->refcnt > 1) {
+        if (acb->aiocb_info->get_aio_context) {
+            aio_poll(acb->aiocb_info->get_aio_context(acb), true);
+        } else if (acb->bs) {
+            aio_poll(bdrv_get_aio_context(acb->bs), true);
+        } else {
+            abort();
+        }
+    }
+    qemu_aio_unref(acb);
+}
+
+/* Async version of aio cancel. The caller is not blocked if the acb implements
+ * cancel_async, otherwise we do nothing and let the request normally complete.
+ * In either case the completion callback must be called. */
+void bdrv_aio_cancel_async(BlockAIOCB *acb)
+{
+    if (acb->aiocb_info->cancel_async) {
+        acb->aiocb_info->cancel_async(acb);
+    }
+}
+
+/**************************************************************/
+/* async block device emulation */
+
+typedef struct BlockRequest {
+    union {
+        /* Used during read, write, trim */
+        struct {
+            int64_t offset;
+            int bytes;
+            int flags;
+            QEMUIOVector *qiov;
+        };
+        /* Used during ioctl */
+        struct {
+            int req;
+            void *buf;
+        };
+    };
+    BlockCompletionFunc *cb;
+    void *opaque;
+
+    int error;
+} BlockRequest;
+
+typedef struct BlockAIOCBCoroutine {
+    BlockAIOCB common;
+    BdrvChild *child;
+    BlockRequest req;
+    bool is_write;
+    bool need_bh;
+    bool *done;
+    QEMUBH* bh;
+} BlockAIOCBCoroutine;
+
+static const AIOCBInfo bdrv_em_co_aiocb_info = {
+    .aiocb_size         = sizeof(BlockAIOCBCoroutine),
+};
+
+static void bdrv_co_complete(BlockAIOCBCoroutine *acb)
+{
+    if (!acb->need_bh) {
+        acb->common.cb(acb->common.opaque, acb->req.error);
+        qemu_aio_unref(acb);
+    }
+}
+
+static void bdrv_co_em_bh(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+
+    assert(!acb->need_bh);
+    qemu_bh_delete(acb->bh);
+    bdrv_co_complete(acb);
+}
+
+static void bdrv_co_maybe_schedule_bh(BlockAIOCBCoroutine *acb)
+{
+    acb->need_bh = false;
+    if (acb->req.error != -EINPROGRESS) {
+        BlockDriverState *bs = acb->common.bs;
+
+        acb->bh = aio_bh_new(bdrv_get_aio_context(bs), bdrv_co_em_bh, acb);
+        qemu_bh_schedule(acb->bh);
+    }
+}
+
+/* Invoke bdrv_co_do_readv/bdrv_co_do_writev */
+static void coroutine_fn bdrv_co_do_rw(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+
+    if (!acb->is_write) {
+        acb->req.error = bdrv_co_preadv(acb->child, acb->req.offset,
+            acb->req.qiov->size, acb->req.qiov, acb->req.flags);
+    } else {
+        acb->req.error = bdrv_co_pwritev(acb->child, acb->req.offset,
+            acb->req.qiov->size, acb->req.qiov, acb->req.flags);
+    }
+
+    bdrv_co_complete(acb);
+}
+
+static BlockAIOCB *bdrv_co_aio_prw_vector(BdrvChild *child,
+                                          int64_t offset,
+                                          QEMUIOVector *qiov,
+                                          BdrvRequestFlags flags,
+                                          BlockCompletionFunc *cb,
+                                          void *opaque,
+                                          bool is_write)
+{
+    Coroutine *co;
+    BlockAIOCBCoroutine *acb;
+
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, child->bs, cb, opaque);
+    acb->child = child;
+    acb->need_bh = true;
+    acb->req.error = -EINPROGRESS;
+    acb->req.offset = offset;
+    acb->req.qiov = qiov;
+    acb->req.flags = flags;
+    acb->is_write = is_write;
+
+    co = qemu_coroutine_create(bdrv_co_do_rw, acb);
+    qemu_coroutine_enter(co);
+
+    bdrv_co_maybe_schedule_bh(acb);
+    return &acb->common;
+}
+
+static void coroutine_fn bdrv_aio_flush_co_entry(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+    BlockDriverState *bs = acb->common.bs;
+
+    acb->req.error = bdrv_co_flush(bs);
+    bdrv_co_complete(acb);
+}
+
+BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
+        BlockCompletionFunc *cb, void *opaque)
+{
+    trace_bdrv_aio_flush(bs, opaque);
+
+    Coroutine *co;
+    BlockAIOCBCoroutine *acb;
+
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->need_bh = true;
+    acb->req.error = -EINPROGRESS;
+
+    co = qemu_coroutine_create(bdrv_aio_flush_co_entry, acb);
+    qemu_coroutine_enter(co);
+
+    bdrv_co_maybe_schedule_bh(acb);
+    return &acb->common;
+}
+
+static void coroutine_fn bdrv_aio_pdiscard_co_entry(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+    BlockDriverState *bs = acb->common.bs;
+
+    acb->req.error = bdrv_co_pdiscard(bs, acb->req.offset, acb->req.bytes);
+    bdrv_co_complete(acb);
+}
+
+BlockAIOCB *bdrv_aio_pdiscard(BlockDriverState *bs, int64_t offset, int count,
+                              BlockCompletionFunc *cb, void *opaque)
+{
+    Coroutine *co;
+    BlockAIOCBCoroutine *acb;
+
+    trace_bdrv_aio_pdiscard(bs, offset, count, opaque);
+
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
+    acb->need_bh = true;
+    acb->req.error = -EINPROGRESS;
+    acb->req.offset = offset;
+    acb->req.bytes = count;
+    co = qemu_coroutine_create(bdrv_aio_pdiscard_co_entry, acb);
+    qemu_coroutine_enter(co);
+
+    bdrv_co_maybe_schedule_bh(acb);
+    return &acb->common;
+}
+
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
+                   BlockCompletionFunc *cb, void *opaque)
+{
+    BlockAIOCB *acb;
+
+    acb = g_malloc(aiocb_info->aiocb_size);
+    acb->aiocb_info = aiocb_info;
+    acb->bs = bs;
+    acb->cb = cb;
+    acb->opaque = opaque;
+    acb->refcnt = 1;
+    return acb;
+}
+
+void qemu_aio_ref(void *p)
+{
+    BlockAIOCB *acb = p;
+    acb->refcnt++;
+}
+
+void qemu_aio_unref(void *p)
+{
+    BlockAIOCB *acb = p;
+    assert(acb->refcnt > 0);
+    if (--acb->refcnt == 0) {
+        g_free(acb);
+    }
+}
+
+/**************************************************************/
+/* Coroutine block device emulation */
+
+typedef struct FlushCo {
+    BlockDriverState *bs;
+    int ret;
+} FlushCo;
+
+
+static void coroutine_fn bdrv_flush_co_entry(void *opaque)
+{
+    FlushCo *rwco = opaque;
+
+    rwco->ret = bdrv_co_flush(rwco->bs);
+}
+
+int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
+{
+    int ret;
+    BdrvTrackedRequest req;
+
+    if (!bs || !bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
+        bdrv_is_sg(bs)) {
+        return 0;
+    }
+
+    tracked_request_begin(&req, bs, 0, 0, BDRV_TRACKED_FLUSH);
+
+    int current_gen = bs->write_gen;
+
+    /* Wait until any previous flushes are completed */
+    while (bs->active_flush_req != NULL) {
+        qemu_co_queue_wait(&bs->flush_queue);
+    }
+
+    bs->active_flush_req = &req;
+
+    /* Write back all layers by calling one driver function */
+    if (bs->drv->bdrv_co_flush) {
+        ret = bs->drv->bdrv_co_flush(bs);
+        goto out;
+    }
+
+    /* Write back cached data to the OS even with cache=unsafe */
+    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_OS);
+    if (bs->drv->bdrv_co_flush_to_os) {
+        ret = bs->drv->bdrv_co_flush_to_os(bs);
+        if (ret < 0) {
+            goto out;
+        }
+    }
+
+    /* But don't actually force it to the disk with cache=unsafe */
+    if (bs->open_flags & BDRV_O_NO_FLUSH) {
+        goto flush_parent;
+    }
+
+    /* Check if we really need to flush anything */
+    if (bs->flushed_gen == current_gen) {
+        goto flush_parent;
+    }
+
+    BLKDBG_EVENT(bs->file, BLKDBG_FLUSH_TO_DISK);
+    if (bs->drv->bdrv_co_flush_to_disk) {
+        ret = bs->drv->bdrv_co_flush_to_disk(bs);
+    } else if (bs->drv->bdrv_aio_flush) {
+        BlockAIOCB *acb;
+        CoroutineIOCompletion co = {
+            .coroutine = qemu_coroutine_self(),
+        };
+
+        acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
+        if (acb == NULL) {
+            ret = -EIO;
+        } else {
+            qemu_coroutine_yield();
+            ret = co.ret;
+        }
+    } else {
+        /*
+         * Some block drivers always operate in either writethrough or unsafe
+         * mode and don't support bdrv_flush therefore. Usually qemu doesn't
+         * know how the server works (because the behaviour is hardcoded or
+         * depends on server-side configuration), so we can't ensure that
+         * everything is safe on disk. Returning an error doesn't work because
+         * that would break guests even if the server operates in writethrough
+         * mode.
+         *
+         * Let's hope the user knows what he's doing.
+         */
+        ret = 0;
+    }
+
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* Now flush the underlying protocol.  It will also have BDRV_O_NO_FLUSH
+     * in the case of cache=unsafe, so there are no useless flushes.
+     */
+flush_parent:
+    ret = bs->file ? bdrv_co_flush(bs->file->bs) : 0;
+out:
+    /* Notify any pending flushes that we have completed */
+    bs->flushed_gen = current_gen;
+    bs->active_flush_req = NULL;
+    /* Return value is ignored - it's ok if wait queue is empty */
+    qemu_co_queue_next(&bs->flush_queue);
+
+    tracked_request_end(&req);
+    return ret;
+}
+
+int bdrv_flush(BlockDriverState *bs)
+{
+    Coroutine *co;
+    FlushCo flush_co = {
+        .bs = bs,
+        .ret = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_flush_co_entry(&flush_co);
+    } else {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        co = qemu_coroutine_create(bdrv_flush_co_entry, &flush_co);
+        qemu_coroutine_enter(co);
+        while (flush_co.ret == NOT_DONE) {
+            aio_poll(aio_context, true);
+        }
+    }
+
+    return flush_co.ret;
+}
+
+typedef struct DiscardCo {
+    BlockDriverState *bs;
+    int64_t offset;
+    int count;
+    int ret;
+} DiscardCo;
+static void coroutine_fn bdrv_pdiscard_co_entry(void *opaque)
+{
+    DiscardCo *rwco = opaque;
+
+    rwco->ret = bdrv_co_pdiscard(rwco->bs, rwco->offset, rwco->count);
+}
+
+int coroutine_fn bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset,
+                                  int count)
+{
+    BdrvTrackedRequest req;
+    int max_pdiscard, ret;
+    int head, align;
+
+    if (!bs->drv) {
+        return -ENOMEDIUM;
+    }
+
+    ret = bdrv_check_byte_request(bs, offset, count);
+    if (ret < 0) {
+        return ret;
+    } else if (bs->read_only) {
+        return -EPERM;
+    }
+    assert(!(bs->open_flags & BDRV_O_INACTIVE));
+
+    /* Do nothing if disabled.  */
+    if (!(bs->open_flags & BDRV_O_UNMAP)) {
+        return 0;
+    }
+
+    if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
+        return 0;
+    }
+
+    /* Discard is advisory, so ignore any unaligned head or tail */
+    align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
+    assert(align % bs->bl.request_alignment == 0);
+    head = offset % align;
+    if (head) {
+        head = MIN(count, align - head);
+        count -= head;
+        offset += head;
+    }
+    count = QEMU_ALIGN_DOWN(count, align);
+    if (!count) {
+        return 0;
+    }
+
+    tracked_request_begin(&req, bs, offset, count, BDRV_TRACKED_DISCARD);
+
+    ret = notifier_with_return_list_notify(&bs->before_write_notifiers, &req);
+    if (ret < 0) {
+        goto out;
+    }
+
+    max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT_MAX),
+                                   align);
+    assert(max_pdiscard);
+
+    while (count > 0) {
+        int ret;
+        int num = MIN(count, max_pdiscard);
+
+        if (bs->drv->bdrv_co_pdiscard) {
+            ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
+        } else {
+            BlockAIOCB *acb;
+            CoroutineIOCompletion co = {
+                .coroutine = qemu_coroutine_self(),
+            };
+
+            acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
+                                             bdrv_co_io_em_complete, &co);
+            if (acb == NULL) {
+                ret = -EIO;
+                goto out;
+            } else {
+                qemu_coroutine_yield();
+                ret = co.ret;
+            }
+        }
+        if (ret && ret != -ENOTSUP) {
+            goto out;
+        }
+
+        offset += num;
+        count -= num;
+    }
+    ret = 0;
+out:
+    ++bs->write_gen;
+    bdrv_set_dirty(bs, req.offset >> BDRV_SECTOR_BITS,
+                   req.bytes >> BDRV_SECTOR_BITS);
+    tracked_request_end(&req);
+    return ret;
+}
+
+int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count)
+{
+    Coroutine *co;
+    DiscardCo rwco = {
+        .bs = bs,
+        .offset = offset,
+        .count = count,
+        .ret = NOT_DONE,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_pdiscard_co_entry(&rwco);
+    } else {
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+
+        co = qemu_coroutine_create(bdrv_pdiscard_co_entry, &rwco);
+        qemu_coroutine_enter(co);
+        while (rwco.ret == NOT_DONE) {
+            aio_poll(aio_context, true);
+        }
+    }
+
+    return rwco.ret;
+}
+
+static int bdrv_co_do_ioctl(BlockDriverState *bs, int req, void *buf)
+{
+    BlockDriver *drv = bs->drv;
+    BdrvTrackedRequest tracked_req;
+    CoroutineIOCompletion co = {
+        .coroutine = qemu_coroutine_self(),
+    };
+    BlockAIOCB *acb;
+
+    tracked_request_begin(&tracked_req, bs, 0, 0, BDRV_TRACKED_IOCTL);
+    if (!drv || !drv->bdrv_aio_ioctl) {
+        co.ret = -ENOTSUP;
+        goto out;
+    }
+
+    acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
+    if (!acb) {
+        co.ret = -ENOTSUP;
+        goto out;
+    }
+    qemu_coroutine_yield();
+out:
+    tracked_request_end(&tracked_req);
+    return co.ret;
+}
+
+typedef struct {
+    BlockDriverState *bs;
+    int req;
+    void *buf;
+    int ret;
+} BdrvIoctlCoData;
+
+static void coroutine_fn bdrv_co_ioctl_entry(void *opaque)
+{
+    BdrvIoctlCoData *data = opaque;
+    data->ret = bdrv_co_do_ioctl(data->bs, data->req, data->buf);
+}
+
+/* needed for generic scsi interface */
+int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
+{
+    BdrvIoctlCoData data = {
+        .bs = bs,
+        .req = req,
+        .buf = buf,
+        .ret = -EINPROGRESS,
+    };
+
+    if (qemu_in_coroutine()) {
+        /* Fast-path if already in coroutine context */
+        bdrv_co_ioctl_entry(&data);
+    } else {
+        Coroutine *co = qemu_coroutine_create(bdrv_co_ioctl_entry, &data);
+
+        qemu_coroutine_enter(co);
+        while (data.ret == -EINPROGRESS) {
+            aio_poll(bdrv_get_aio_context(bs), true);
+        }
+    }
+    return data.ret;
+}
+
+static void coroutine_fn bdrv_co_aio_ioctl_entry(void *opaque)
+{
+    BlockAIOCBCoroutine *acb = opaque;
+    acb->req.error = bdrv_co_do_ioctl(acb->common.bs,
+                                      acb->req.req, acb->req.buf);
+    bdrv_co_complete(acb);
+}
+
+BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
+        unsigned long int req, void *buf,
+        BlockCompletionFunc *cb, void *opaque)
+{
+    BlockAIOCBCoroutine *acb = qemu_aio_get(&bdrv_em_co_aiocb_info,
+                                            bs, cb, opaque);
+    Coroutine *co;
+
+    acb->need_bh = true;
+    acb->req.error = -EINPROGRESS;
+    acb->req.req = req;
+    acb->req.buf = buf;
+    co = qemu_coroutine_create(bdrv_co_aio_ioctl_entry, acb);
+    qemu_coroutine_enter(co);
+
+    bdrv_co_maybe_schedule_bh(acb);
+    return &acb->common;
+}
+
+void *qemu_blockalign(BlockDriverState *bs, size_t size)
+{
+    return qemu_memalign(bdrv_opt_mem_align(bs), size);
+}
+
+void *qemu_blockalign0(BlockDriverState *bs, size_t size)
+{
+    return memset(qemu_blockalign(bs, size), 0, size);
+}
+
+void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
+{
+    size_t align = bdrv_opt_mem_align(bs);
+
+    /* Ensure that NULL is never returned on success */
+    assert(align > 0);
+    if (size == 0) {
+        size = align;
+    }
+
+    return qemu_try_memalign(align, size);
+}
+
+void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
+{
+    void *mem = qemu_try_blockalign(bs, size);
+
+    if (mem) {
+        memset(mem, 0, size);
+    }
+
+    return mem;
+}
+
+/*
+ * Check if all memory in this vector is sector aligned.
+ */
+bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
+{
+    int i;
+    size_t alignment = bdrv_min_mem_align(bs);
+
+    for (i = 0; i < qiov->niov; i++) {
+        if ((uintptr_t) qiov->iov[i].iov_base % alignment) {
+            return false;
+        }
+        if (qiov->iov[i].iov_len % alignment) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+void bdrv_add_before_write_notifier(BlockDriverState *bs,
+                                    NotifierWithReturn *notifier)
+{
+    notifier_with_return_list_add(&bs->before_write_notifiers, notifier);
+}
+
+void bdrv_io_plug(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_io_plug(child->bs);
+    }
+
+    if (bs->io_plugged++ == 0 && bs->io_plug_disabled == 0) {
+        BlockDriver *drv = bs->drv;
+        if (drv && drv->bdrv_io_plug) {
+            drv->bdrv_io_plug(bs);
+        }
+    }
+}
+
+void bdrv_io_unplug(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    assert(bs->io_plugged);
+    if (--bs->io_plugged == 0 && bs->io_plug_disabled == 0) {
+        BlockDriver *drv = bs->drv;
+        if (drv && drv->bdrv_io_unplug) {
+            drv->bdrv_io_unplug(bs);
+        }
+    }
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_io_unplug(child->bs);
+    }
+}
+
+void bdrv_io_unplugged_begin(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    if (bs->io_plug_disabled++ == 0 && bs->io_plugged > 0) {
+        BlockDriver *drv = bs->drv;
+        if (drv && drv->bdrv_io_unplug) {
+            drv->bdrv_io_unplug(bs);
+        }
+    }
+
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_io_unplugged_begin(child->bs);
+    }
+}
+
+void bdrv_io_unplugged_end(BlockDriverState *bs)
+{
+    BdrvChild *child;
+
+    assert(bs->io_plug_disabled);
+    QLIST_FOREACH(child, &bs->children, next) {
+        bdrv_io_unplugged_end(child->bs);
+    }
+
+    if (--bs->io_plug_disabled == 0 && bs->io_plugged > 0) {
+        BlockDriver *drv = bs->drv;
+        if (drv && drv->bdrv_io_plug) {
+            drv->bdrv_io_plug(bs);
+        }
+    }
+}
diff --git a/block/iscsi.c b/block/iscsi.c
index ed375fc..95ce9e1 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -2,7 +2,7 @@
  * QEMU Block driver for iSCSI images
  *
  * Copyright (c) 2010-2011 Ronnie Sahlberg <ronniesahlberg@gmail.com>
- * Copyright (c) 2012-2014 Peter Lieven <pl@kamp.de>
+ * Copyright (c) 2012-2016 Peter Lieven <pl@kamp.de>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include <poll.h>
 #include <math.h>
@@ -38,13 +38,14 @@
 #include "qemu/iov.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
+#include "qapi/qmp/qstring.h"
+#include "crypto/secret.h"
 
 #include <iscsi/iscsi.h>
 #include <iscsi/scsi-lowlevel.h>
 
 #ifdef __linux__
 #include <scsi/sg.h>
-#include <block/scsi.h>
 #endif
 
 typedef struct IscsiLun {
@@ -56,15 +57,35 @@
     uint64_t num_blocks;
     int events;
     QEMUTimer *nop_timer;
-    uint8_t lbpme;
-    uint8_t lbprz;
-    uint8_t has_write_same;
+    QEMUTimer *event_timer;
     struct scsi_inquiry_logical_block_provisioning lbp;
     struct scsi_inquiry_block_limits bl;
     unsigned char *zeroblock;
-    unsigned long *allocationmap;
+    /* The allocmap tracks which clusters (pages) on the iSCSI target are
+     * allocated and which are not. In case a target returns zeros for
+     * unallocated pages (iscsilun->lprz) we can directly return zeros instead
+     * of reading zeros over the wire if a read request falls within an
+     * unallocated block. As there are 3 possible states we need 2 bitmaps to
+     * track. allocmap_valid keeps track if QEMU's information about a page is
+     * valid. allocmap tracks if a page is allocated or not. In case QEMU has no
+     * valid information about a page the corresponding allocmap entry should be
+     * switched to unallocated as well to force a new lookup of the allocation
+     * status as lookups are generally skipped if a page is suspect to be
+     * allocated. If a iSCSI target is opened with cache.direct = on the
+     * allocmap_valid does not exist turning all cached information invalid so
+     * that a fresh lookup is made for any page even if allocmap entry returns
+     * it's unallocated. */
+    unsigned long *allocmap;
+    unsigned long *allocmap_valid;
+    long allocmap_size;
     int cluster_sectors;
     bool use_16_for_rw;
+    bool write_protected;
+    bool lbpme;
+    bool lbprz;
+    bool dpofua;
+    bool has_write_same;
+    bool request_timed_out;
 } IscsiLun;
 
 typedef struct IscsiTask {
@@ -77,6 +98,7 @@
     QEMUBH *bh;
     IscsiLun *iscsilun;
     QEMUTimer retry_timer;
+    int err_code;
 } IscsiTask;
 
 typedef struct IscsiAIOCB {
@@ -89,15 +111,18 @@
     int status;
     int64_t sector_num;
     int nb_sectors;
+    int ret;
 #ifdef __linux__
     sg_io_hdr_t *ioh;
 #endif
 } IscsiAIOCB;
 
+/* libiscsi uses time_t so its enough to process events every second */
+#define EVENT_INTERVAL 1000
 #define NOP_INTERVAL 5000
 #define MAX_NOP_FAILURES 3
 #define ISCSI_CMD_RETRIES ARRAY_SIZE(iscsi_retry_times)
-static const unsigned iscsi_retry_times[] = {8, 32, 128, 512, 2048};
+static const unsigned iscsi_retry_times[] = {8, 32, 128, 512, 2048, 8192, 32768};
 
 /* this threshold is a trade-off knob to choose between
  * the potential additional overhead of an extra GET_LBA_STATUS request
@@ -143,7 +168,7 @@
     struct IscsiTask *iTask = opaque;
     iTask->complete = 1;
     qemu_bh_delete(iTask->bh);
-    qemu_coroutine_enter(iTask->co, NULL);
+    qemu_coroutine_enter(iTask->co);
 }
 
 static void iscsi_retry_timer_expired(void *opaque)
@@ -151,7 +176,7 @@
     struct IscsiTask *iTask = opaque;
     iTask->complete = 1;
     if (iTask->co) {
-        qemu_coroutine_enter(iTask->co, NULL);
+        qemu_coroutine_enter(iTask->co);
     }
 }
 
@@ -160,6 +185,70 @@
     return -mean * log((double)rand() / RAND_MAX);
 }
 
+/* SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST was introduced in
+ * libiscsi 1.10.0, together with other constants we need.  Use it as
+ * a hint that we have to define them ourselves if needed, to keep the
+ * minimum required libiscsi version at 1.9.0.  We use an ASCQ macro for
+ * the test because SCSI_STATUS_* is an enum.
+ *
+ * To guard against future changes where SCSI_SENSE_ASCQ_* also becomes
+ * an enum, check against the LIBISCSI_API_VERSION macro, which was
+ * introduced in 1.11.0.  If it is present, there is no need to define
+ * anything.
+ */
+#if !defined(SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST) && \
+    !defined(LIBISCSI_API_VERSION)
+#define SCSI_STATUS_TASK_SET_FULL                          0x28
+#define SCSI_STATUS_TIMEOUT                                0x0f000002
+#define SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST    0x2600
+#define SCSI_SENSE_ASCQ_PARAMETER_LIST_LENGTH_ERROR        0x1a00
+#endif
+
+static int iscsi_translate_sense(struct scsi_sense *sense)
+{
+    int ret;
+
+    switch (sense->key) {
+    case SCSI_SENSE_NOT_READY:
+        return -EBUSY;
+    case SCSI_SENSE_DATA_PROTECTION:
+        return -EACCES;
+    case SCSI_SENSE_COMMAND_ABORTED:
+        return -ECANCELED;
+    case SCSI_SENSE_ILLEGAL_REQUEST:
+        /* Parse ASCQ */
+        break;
+    default:
+        return -EIO;
+    }
+    switch (sense->ascq) {
+    case SCSI_SENSE_ASCQ_PARAMETER_LIST_LENGTH_ERROR:
+    case SCSI_SENSE_ASCQ_INVALID_OPERATION_CODE:
+    case SCSI_SENSE_ASCQ_INVALID_FIELD_IN_CDB:
+    case SCSI_SENSE_ASCQ_INVALID_FIELD_IN_PARAMETER_LIST:
+        ret = -EINVAL;
+        break;
+    case SCSI_SENSE_ASCQ_LBA_OUT_OF_RANGE:
+        ret = -ENOSPC;
+        break;
+    case SCSI_SENSE_ASCQ_LOGICAL_UNIT_NOT_SUPPORTED:
+        ret = -ENOTSUP;
+        break;
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT:
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT_TRAY_CLOSED:
+    case SCSI_SENSE_ASCQ_MEDIUM_NOT_PRESENT_TRAY_OPEN:
+        ret = -ENOMEDIUM;
+        break;
+    case SCSI_SENSE_ASCQ_WRITE_PROTECTED:
+        ret = -EACCES;
+        break;
+    default:
+        ret = -EIO;
+        break;
+    }
+    return ret;
+}
+
 static void
 iscsi_co_generic_cb(struct iscsi_context *iscsi, int status,
                         void *command_data, void *opaque)
@@ -180,10 +269,19 @@
                 iTask->do_retry = 1;
                 goto out;
             }
-            if (status == SCSI_STATUS_BUSY) {
+            if (status == SCSI_STATUS_BUSY ||
+                status == SCSI_STATUS_TIMEOUT ||
+                status == SCSI_STATUS_TASK_SET_FULL) {
                 unsigned retry_time =
                     exp_random(iscsi_retry_times[iTask->retries - 1]);
-                error_report("iSCSI Busy (retry #%u in %u ms): %s",
+                if (status == SCSI_STATUS_TIMEOUT) {
+                    /* make sure the request is rescheduled AFTER the
+                     * reconnect is initiated */
+                    retry_time = EVENT_INTERVAL * 2;
+                    iTask->iscsilun->request_timed_out = true;
+                }
+                error_report("iSCSI Busy/TaskSetFull/TimeOut"
+                             " (retry #%u in %u ms): %s",
                              iTask->retries, retry_time,
                              iscsi_get_error(iscsi));
                 aio_timer_init(iTask->iscsilun->aio_context,
@@ -195,6 +293,7 @@
                 return;
             }
         }
+        iTask->err_code = iscsi_translate_sense(&task->sense);
         error_report("iSCSI Failure: %s", iscsi_get_error(iscsi));
     }
 
@@ -255,21 +354,36 @@
 iscsi_set_events(IscsiLun *iscsilun)
 {
     struct iscsi_context *iscsi = iscsilun->iscsi;
-    int ev;
+    int ev = iscsi_which_events(iscsi);
 
-    /* We always register a read handler.  */
-    ev = POLLIN;
-    ev |= iscsi_which_events(iscsi);
     if (ev != iscsilun->events) {
-        aio_set_fd_handler(iscsilun->aio_context,
-                           iscsi_get_fd(iscsi),
-                           iscsi_process_read,
+        aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsi),
+                           false,
+                           (ev & POLLIN) ? iscsi_process_read : NULL,
                            (ev & POLLOUT) ? iscsi_process_write : NULL,
                            iscsilun);
+        iscsilun->events = ev;
+    }
+}
 
+static void iscsi_timed_check_events(void *opaque)
+{
+    IscsiLun *iscsilun = opaque;
+
+    /* check for timed out requests */
+    iscsi_service(iscsilun->iscsi, 0);
+
+    if (iscsilun->request_timed_out) {
+        iscsilun->request_timed_out = false;
+        iscsi_reconnect(iscsilun->iscsi);
     }
 
-    iscsilun->events = ev;
+    /* newer versions of libiscsi may return zero events. Ensure we are able
+     * to return to service once this situation changes. */
+    iscsi_set_events(iscsilun);
+
+    timer_mod(iscsilun->event_timer,
+              qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + EVENT_INTERVAL);
 }
 
 static void
@@ -302,70 +416,178 @@
     return sector * BDRV_SECTOR_SIZE / iscsilun->block_size;
 }
 
-static bool is_request_lun_aligned(int64_t sector_num, int nb_sectors,
-                                      IscsiLun *iscsilun)
+static bool is_byte_request_lun_aligned(int64_t offset, int count,
+                                        IscsiLun *iscsilun)
 {
-    if ((sector_num * BDRV_SECTOR_SIZE) % iscsilun->block_size ||
-        (nb_sectors * BDRV_SECTOR_SIZE) % iscsilun->block_size) {
-            error_report("iSCSI misaligned request: "
-                         "iscsilun->block_size %u, sector_num %" PRIi64
-                         ", nb_sectors %d",
-                         iscsilun->block_size, sector_num, nb_sectors);
-            return 0;
+    if (offset % iscsilun->block_size || count % iscsilun->block_size) {
+        error_report("iSCSI misaligned request: "
+                     "iscsilun->block_size %u, offset %" PRIi64
+                     ", count %d",
+                     iscsilun->block_size, offset, count);
+        return false;
     }
-    return 1;
+    return true;
 }
 
-static unsigned long *iscsi_allocationmap_init(IscsiLun *iscsilun)
+static bool is_sector_request_lun_aligned(int64_t sector_num, int nb_sectors,
+                                          IscsiLun *iscsilun)
 {
-    return bitmap_try_new(DIV_ROUND_UP(sector_lun2qemu(iscsilun->num_blocks,
-                                                       iscsilun),
-                                       iscsilun->cluster_sectors));
+    assert(nb_sectors <= BDRV_REQUEST_MAX_SECTORS);
+    return is_byte_request_lun_aligned(sector_num << BDRV_SECTOR_BITS,
+                                       nb_sectors << BDRV_SECTOR_BITS,
+                                       iscsilun);
 }
 
-static void iscsi_allocationmap_set(IscsiLun *iscsilun, int64_t sector_num,
-                                    int nb_sectors)
+static void iscsi_allocmap_free(IscsiLun *iscsilun)
 {
-    if (iscsilun->allocationmap == NULL) {
+    g_free(iscsilun->allocmap);
+    g_free(iscsilun->allocmap_valid);
+    iscsilun->allocmap = NULL;
+    iscsilun->allocmap_valid = NULL;
+}
+
+
+static int iscsi_allocmap_init(IscsiLun *iscsilun, int open_flags)
+{
+    iscsi_allocmap_free(iscsilun);
+
+    iscsilun->allocmap_size =
+        DIV_ROUND_UP(sector_lun2qemu(iscsilun->num_blocks, iscsilun),
+                     iscsilun->cluster_sectors);
+
+    iscsilun->allocmap = bitmap_try_new(iscsilun->allocmap_size);
+    if (!iscsilun->allocmap) {
+        return -ENOMEM;
+    }
+
+    if (open_flags & BDRV_O_NOCACHE) {
+        /* in case that cache.direct = on all allocmap entries are
+         * treated as invalid to force a relookup of the block
+         * status on every read request */
+        return 0;
+    }
+
+    iscsilun->allocmap_valid = bitmap_try_new(iscsilun->allocmap_size);
+    if (!iscsilun->allocmap_valid) {
+        /* if we are under memory pressure free the allocmap as well */
+        iscsi_allocmap_free(iscsilun);
+        return -ENOMEM;
+    }
+
+    return 0;
+}
+
+static void
+iscsi_allocmap_update(IscsiLun *iscsilun, int64_t sector_num,
+                      int nb_sectors, bool allocated, bool valid)
+{
+    int64_t cl_num_expanded, nb_cls_expanded, cl_num_shrunk, nb_cls_shrunk;
+
+    if (iscsilun->allocmap == NULL) {
         return;
     }
-    bitmap_set(iscsilun->allocationmap,
-               sector_num / iscsilun->cluster_sectors,
-               DIV_ROUND_UP(nb_sectors, iscsilun->cluster_sectors));
-}
+    /* expand to entirely contain all affected clusters */
+    cl_num_expanded = sector_num / iscsilun->cluster_sectors;
+    nb_cls_expanded = DIV_ROUND_UP(sector_num + nb_sectors,
+                                   iscsilun->cluster_sectors) - cl_num_expanded;
+    /* shrink to touch only completely contained clusters */
+    cl_num_shrunk = DIV_ROUND_UP(sector_num, iscsilun->cluster_sectors);
+    nb_cls_shrunk = (sector_num + nb_sectors) / iscsilun->cluster_sectors
+                      - cl_num_shrunk;
+    if (allocated) {
+        bitmap_set(iscsilun->allocmap, cl_num_expanded, nb_cls_expanded);
+    } else {
+        bitmap_clear(iscsilun->allocmap, cl_num_shrunk, nb_cls_shrunk);
+    }
 
-static void iscsi_allocationmap_clear(IscsiLun *iscsilun, int64_t sector_num,
-                                      int nb_sectors)
-{
-    int64_t cluster_num, nb_clusters;
-    if (iscsilun->allocationmap == NULL) {
+    if (iscsilun->allocmap_valid == NULL) {
         return;
     }
-    cluster_num = DIV_ROUND_UP(sector_num, iscsilun->cluster_sectors);
-    nb_clusters = (sector_num + nb_sectors) / iscsilun->cluster_sectors
-                  - cluster_num;
-    if (nb_clusters > 0) {
-        bitmap_clear(iscsilun->allocationmap, cluster_num, nb_clusters);
+    if (valid) {
+        bitmap_set(iscsilun->allocmap_valid, cl_num_shrunk, nb_cls_shrunk);
+    } else {
+        bitmap_clear(iscsilun->allocmap_valid, cl_num_expanded,
+                     nb_cls_expanded);
     }
 }
 
-static int coroutine_fn iscsi_co_writev(BlockDriverState *bs,
-                                        int64_t sector_num, int nb_sectors,
-                                        QEMUIOVector *iov)
+static void
+iscsi_allocmap_set_allocated(IscsiLun *iscsilun, int64_t sector_num,
+                             int nb_sectors)
+{
+    iscsi_allocmap_update(iscsilun, sector_num, nb_sectors, true, true);
+}
+
+static void
+iscsi_allocmap_set_unallocated(IscsiLun *iscsilun, int64_t sector_num,
+                               int nb_sectors)
+{
+    /* Note: if cache.direct=on the fifth argument to iscsi_allocmap_update
+     * is ignored, so this will in effect be an iscsi_allocmap_set_invalid.
+     */
+    iscsi_allocmap_update(iscsilun, sector_num, nb_sectors, false, true);
+}
+
+static void iscsi_allocmap_set_invalid(IscsiLun *iscsilun, int64_t sector_num,
+                                       int nb_sectors)
+{
+    iscsi_allocmap_update(iscsilun, sector_num, nb_sectors, false, false);
+}
+
+static void iscsi_allocmap_invalidate(IscsiLun *iscsilun)
+{
+    if (iscsilun->allocmap) {
+        bitmap_zero(iscsilun->allocmap, iscsilun->allocmap_size);
+    }
+    if (iscsilun->allocmap_valid) {
+        bitmap_zero(iscsilun->allocmap_valid, iscsilun->allocmap_size);
+    }
+}
+
+static inline bool
+iscsi_allocmap_is_allocated(IscsiLun *iscsilun, int64_t sector_num,
+                            int nb_sectors)
+{
+    unsigned long size;
+    if (iscsilun->allocmap == NULL) {
+        return true;
+    }
+    size = DIV_ROUND_UP(sector_num + nb_sectors, iscsilun->cluster_sectors);
+    return !(find_next_bit(iscsilun->allocmap, size,
+                           sector_num / iscsilun->cluster_sectors) == size);
+}
+
+static inline bool iscsi_allocmap_is_valid(IscsiLun *iscsilun,
+                                           int64_t sector_num, int nb_sectors)
+{
+    unsigned long size;
+    if (iscsilun->allocmap_valid == NULL) {
+        return false;
+    }
+    size = DIV_ROUND_UP(sector_num + nb_sectors, iscsilun->cluster_sectors);
+    return (find_next_zero_bit(iscsilun->allocmap_valid, size,
+                               sector_num / iscsilun->cluster_sectors) == size);
+}
+
+static int coroutine_fn
+iscsi_co_writev_flags(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
+                      QEMUIOVector *iov, int flags)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct IscsiTask iTask;
     uint64_t lba;
     uint32_t num_sectors;
+    bool fua = flags & BDRV_REQ_FUA;
 
-    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
+    if (fua) {
+        assert(iscsilun->dpofua);
+    }
+    if (!is_sector_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
         return -EINVAL;
     }
 
-    if (bs->bl.max_transfer_length && nb_sectors > bs->bl.max_transfer_length) {
-        error_report("iSCSI Error: Write of %d sectors exceeds max_xfer_len "
-                     "of %d sectors", nb_sectors, bs->bl.max_transfer_length);
-        return -EINVAL;
+    if (bs->bl.max_transfer) {
+        assert(nb_sectors << BDRV_SECTOR_BITS <= bs->bl.max_transfer);
     }
 
     lba = sector_qemu2lun(sector_num, iscsilun);
@@ -375,12 +597,12 @@
     if (iscsilun->use_16_for_rw) {
         iTask.task = iscsi_write16_task(iscsilun->iscsi, iscsilun->lun, lba,
                                         NULL, num_sectors * iscsilun->block_size,
-                                        iscsilun->block_size, 0, 0, 0, 0, 0,
+                                        iscsilun->block_size, 0, 0, fua, 0, 0,
                                         iscsi_co_generic_cb, &iTask);
     } else {
         iTask.task = iscsi_write10_task(iscsilun->iscsi, iscsilun->lun, lba,
                                         NULL, num_sectors * iscsilun->block_size,
-                                        iscsilun->block_size, 0, 0, 0, 0, 0,
+                                        iscsilun->block_size, 0, 0, fua, 0, 0,
                                         iscsi_co_generic_cb, &iTask);
     }
     if (iTask.task == NULL) {
@@ -404,30 +626,21 @@
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        iscsi_allocmap_set_invalid(iscsilun, sector_num, nb_sectors);
+        return iTask.err_code;
     }
 
-    iscsi_allocationmap_set(iscsilun, sector_num, nb_sectors);
+    iscsi_allocmap_set_allocated(iscsilun, sector_num, nb_sectors);
 
     return 0;
 }
 
 
-static bool iscsi_allocationmap_is_allocated(IscsiLun *iscsilun,
-                                             int64_t sector_num, int nb_sectors)
-{
-    unsigned long size;
-    if (iscsilun->allocationmap == NULL) {
-        return true;
-    }
-    size = DIV_ROUND_UP(sector_num + nb_sectors, iscsilun->cluster_sectors);
-    return !(find_next_bit(iscsilun->allocationmap, size,
-                           sector_num / iscsilun->cluster_sectors) == size);
-}
 
 static int64_t coroutine_fn iscsi_co_get_block_status(BlockDriverState *bs,
                                                   int64_t sector_num,
-                                                  int nb_sectors, int *pnum)
+                                                  int nb_sectors, int *pnum,
+                                                  BlockDriverState **file)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct scsi_get_lba_status *lbas = NULL;
@@ -437,7 +650,7 @@
 
     iscsi_co_init_iscsitask(iscsilun, &iTask);
 
-    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
+    if (!is_sector_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
         ret = -EINVAL;
         goto out;
     }
@@ -448,7 +661,7 @@
     *pnum = nb_sectors;
 
     /* LUN does not support logical block provisioning */
-    if (iscsilun->lbpme == 0) {
+    if (!iscsilun->lbpme) {
         goto out;
     }
 
@@ -507,9 +720,9 @@
     }
 
     if (ret & BDRV_BLOCK_ZERO) {
-        iscsi_allocationmap_clear(iscsilun, sector_num, *pnum);
+        iscsi_allocmap_set_unallocated(iscsilun, sector_num, *pnum);
     } else {
-        iscsi_allocationmap_set(iscsilun, sector_num, *pnum);
+        iscsi_allocmap_set_allocated(iscsilun, sector_num, *pnum);
     }
 
     if (*pnum > nb_sectors) {
@@ -519,6 +732,9 @@
     if (iTask.task != NULL) {
         scsi_free_scsi_task(iTask.task);
     }
+    if (ret > 0 && ret & BDRV_BLOCK_OFFSET_VALID) {
+        *file = bs;
+    }
     return ret;
 }
 
@@ -531,25 +747,40 @@
     uint64_t lba;
     uint32_t num_sectors;
 
-    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
+    if (!is_sector_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
         return -EINVAL;
     }
 
-    if (bs->bl.max_transfer_length && nb_sectors > bs->bl.max_transfer_length) {
-        error_report("iSCSI Error: Read of %d sectors exceeds max_xfer_len "
-                     "of %d sectors", nb_sectors, bs->bl.max_transfer_length);
-        return -EINVAL;
+    if (bs->bl.max_transfer) {
+        assert(nb_sectors << BDRV_SECTOR_BITS <= bs->bl.max_transfer);
     }
 
-    if (iscsilun->lbprz && nb_sectors >= ISCSI_CHECKALLOC_THRES &&
-        !iscsi_allocationmap_is_allocated(iscsilun, sector_num, nb_sectors)) {
-        int64_t ret;
+    /* if cache.direct is off and we have a valid entry in our allocation map
+     * we can skip checking the block status and directly return zeroes if
+     * the request falls within an unallocated area */
+    if (iscsi_allocmap_is_valid(iscsilun, sector_num, nb_sectors) &&
+        !iscsi_allocmap_is_allocated(iscsilun, sector_num, nb_sectors)) {
+            qemu_iovec_memset(iov, 0, 0x00, iov->size);
+            return 0;
+    }
+
+    if (nb_sectors >= ISCSI_CHECKALLOC_THRES &&
+        !iscsi_allocmap_is_valid(iscsilun, sector_num, nb_sectors) &&
+        !iscsi_allocmap_is_allocated(iscsilun, sector_num, nb_sectors)) {
         int pnum;
-        ret = iscsi_co_get_block_status(bs, sector_num, INT_MAX, &pnum);
+        BlockDriverState *file;
+        /* check the block status from the beginning of the cluster
+         * containing the start sector */
+        int64_t ret = iscsi_co_get_block_status(bs,
+                          sector_num - sector_num % iscsilun->cluster_sectors,
+                          BDRV_REQUEST_MAX_SECTORS, &pnum, &file);
         if (ret < 0) {
             return ret;
         }
-        if (ret & BDRV_BLOCK_ZERO && pnum >= nb_sectors) {
+        /* if the whole request falls into an unallocated area we can avoid
+         * to read and directly return zeroes instead */
+        if (ret & BDRV_BLOCK_ZERO &&
+            pnum >= nb_sectors + sector_num % iscsilun->cluster_sectors) {
             qemu_iovec_memset(iov, 0, 0x00, iov->size);
             return 0;
         }
@@ -593,7 +824,7 @@
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     return 0;
@@ -604,12 +835,7 @@
     IscsiLun *iscsilun = bs->opaque;
     struct IscsiTask iTask;
 
-    if (bs->sg) {
-        return 0;
-    }
-
     iscsi_co_init_iscsitask(iscsilun, &iTask);
-
 retry:
     if (iscsi_synchronizecache10_task(iscsilun->iscsi, iscsilun->lun, 0, 0, 0,
                                       0, iscsi_co_generic_cb, &iTask) == NULL) {
@@ -632,7 +858,7 @@
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
     return 0;
@@ -652,12 +878,13 @@
     if (status < 0) {
         error_report("Failed to ioctl(SG_IO) to iSCSI lun. %s",
                      iscsi_get_error(iscsi));
-        acb->status = -EIO;
+        acb->status = iscsi_translate_sense(&acb->task->sense);
     }
 
     acb->ioh->driver_status = 0;
     acb->ioh->host_status   = 0;
     acb->ioh->resid         = 0;
+    acb->ioh->status        = status;
 
 #define SG_ERR_DRIVER_SENSE    0x08
 
@@ -675,6 +902,38 @@
     iscsi_schedule_bh(acb);
 }
 
+static void iscsi_ioctl_bh_completion(void *opaque)
+{
+    IscsiAIOCB *acb = opaque;
+
+    qemu_bh_delete(acb->bh);
+    acb->common.cb(acb->common.opaque, acb->ret);
+    qemu_aio_unref(acb);
+}
+
+static void iscsi_ioctl_handle_emulated(IscsiAIOCB *acb, int req, void *buf)
+{
+    BlockDriverState *bs = acb->common.bs;
+    IscsiLun *iscsilun = bs->opaque;
+    int ret = 0;
+
+    switch (req) {
+    case SG_GET_VERSION_NUM:
+        *(int *)buf = 30000;
+        break;
+    case SG_GET_SCSI_ID:
+        ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
+        break;
+    default:
+        ret = -EINVAL;
+    }
+    assert(!acb->bh);
+    acb->bh = aio_bh_new(bdrv_get_aio_context(bs),
+                         iscsi_ioctl_bh_completion, acb);
+    acb->ret = ret;
+    qemu_bh_schedule(acb->bh);
+}
+
 static BlockAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque)
@@ -684,8 +943,6 @@
     struct iscsi_data data;
     IscsiAIOCB *acb;
 
-    assert(req == SG_IO);
-
     acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
@@ -694,6 +951,18 @@
     acb->buf         = NULL;
     acb->ioh         = buf;
 
+    if (req != SG_IO) {
+        iscsi_ioctl_handle_emulated(acb, req, buf);
+        return &acb->common;
+    }
+
+    if (acb->ioh->cmd_len > SCSI_CDB_MAX_SIZE) {
+        error_report("iSCSI: ioctl error CDB exceeds max size (%d > %d)",
+                     acb->ioh->cmd_len, SCSI_CDB_MAX_SIZE);
+        qemu_aio_unref(acb);
+        return NULL;
+    }
+
     acb->task = malloc(sizeof(struct scsi_task));
     if (acb->task == NULL) {
         error_report("iSCSI: Failed to allocate task for scsi command. %s",
@@ -758,38 +1027,6 @@
     return &acb->common;
 }
 
-static void ioctl_cb(void *opaque, int status)
-{
-    int *p_status = opaque;
-    *p_status = status;
-}
-
-static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    IscsiLun *iscsilun = bs->opaque;
-    int status;
-
-    switch (req) {
-    case SG_GET_VERSION_NUM:
-        *(int *)buf = 30000;
-        break;
-    case SG_GET_SCSI_ID:
-        ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
-        break;
-    case SG_IO:
-        status = -EINPROGRESS;
-        iscsi_aio_ioctl(bs, req, buf, ioctl_cb, &status);
-
-        while (status == -EINPROGRESS) {
-            aio_poll(iscsilun->aio_context, true);
-        }
-
-        return 0;
-    default:
-        return -1;
-    }
-    return 0;
-}
 #endif
 
 static int64_t
@@ -805,29 +1042,26 @@
 }
 
 static int
-coroutine_fn iscsi_co_discard(BlockDriverState *bs, int64_t sector_num,
-                                   int nb_sectors)
+coroutine_fn iscsi_co_pdiscard(BlockDriverState *bs, int64_t offset, int count)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct IscsiTask iTask;
     struct unmap_list list;
 
-    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
-        return -EINVAL;
-    }
+    assert(is_byte_request_lun_aligned(offset, count, iscsilun));
 
     if (!iscsilun->lbp.lbpu) {
         /* UNMAP is not supported by the target */
         return 0;
     }
 
-    list.lba = sector_qemu2lun(sector_num, iscsilun);
-    list.num = sector_qemu2lun(nb_sectors, iscsilun);
+    list.lba = offset / iscsilun->block_size;
+    list.num = count / iscsilun->block_size;
 
     iscsi_co_init_iscsitask(iscsilun, &iTask);
 retry:
     if (iscsi_unmap_task(iscsilun->iscsi, iscsilun->lun, 0, 0, &list, 1,
-                     iscsi_co_generic_cb, &iTask) == NULL) {
+                         iscsi_co_generic_cb, &iTask) == NULL) {
         return -ENOMEM;
     }
 
@@ -854,17 +1088,18 @@
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        return iTask.err_code;
     }
 
-    iscsi_allocationmap_clear(iscsilun, sector_num, nb_sectors);
+    iscsi_allocmap_set_invalid(iscsilun, offset >> BDRV_SECTOR_BITS,
+                               count >> BDRV_SECTOR_BITS);
 
     return 0;
 }
 
 static int
-coroutine_fn iscsi_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-                                   int nb_sectors, BdrvRequestFlags flags)
+coroutine_fn iscsi_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset,
+                                    int count, BdrvRequestFlags flags)
 {
     IscsiLun *iscsilun = bs->opaque;
     struct IscsiTask iTask;
@@ -872,8 +1107,8 @@
     uint32_t nb_blocks;
     bool use_16_for_ws = iscsilun->use_16_for_rw;
 
-    if (!is_request_lun_aligned(sector_num, nb_sectors, iscsilun)) {
-        return -EINVAL;
+    if (!is_byte_request_lun_aligned(offset, count, iscsilun)) {
+        return -ENOTSUP;
     }
 
     if (flags & BDRV_REQ_MAY_UNMAP) {
@@ -894,8 +1129,8 @@
         return -ENOTSUP;
     }
 
-    lba = sector_qemu2lun(sector_num, iscsilun);
-    nb_blocks = sector_qemu2lun(nb_sectors, iscsilun);
+    lba = offset / iscsilun->block_size;
+    nb_blocks = count / iscsilun->block_size;
 
     if (iscsilun->zeroblock == NULL) {
         iscsilun->zeroblock = g_try_malloc0(iscsilun->block_size);
@@ -947,13 +1182,17 @@
     }
 
     if (iTask.status != SCSI_STATUS_GOOD) {
-        return -EIO;
+        iscsi_allocmap_set_invalid(iscsilun, offset >> BDRV_SECTOR_BITS,
+                                   count >> BDRV_SECTOR_BITS);
+        return iTask.err_code;
     }
 
     if (flags & BDRV_REQ_MAY_UNMAP) {
-        iscsi_allocationmap_clear(iscsilun, sector_num, nb_sectors);
+        iscsi_allocmap_set_invalid(iscsilun, offset >> BDRV_SECTOR_BITS,
+                                   count >> BDRV_SECTOR_BITS);
     } else {
-        iscsi_allocationmap_set(iscsilun, sector_num, nb_sectors);
+        iscsi_allocmap_set_allocated(iscsilun, offset >> BDRV_SECTOR_BITS,
+                                     count >> BDRV_SECTOR_BITS);
     }
 
     return 0;
@@ -966,6 +1205,8 @@
     QemuOpts *opts;
     const char *user = NULL;
     const char *password = NULL;
+    const char *secretid;
+    char *secret = NULL;
 
     list = qemu_find_opts("iscsi");
     if (!list) {
@@ -985,8 +1226,20 @@
         return;
     }
 
+    secretid = qemu_opt_get(opts, "password-secret");
     password = qemu_opt_get(opts, "password");
-    if (!password) {
+    if (secretid && password) {
+        error_setg(errp, "'password' and 'password-secret' properties are "
+                   "mutually exclusive");
+        return;
+    }
+    if (secretid) {
+        secret = qcrypto_secret_lookup_as_utf8(secretid, errp);
+        if (!secret) {
+            return;
+        }
+        password = secret;
+    } else if (!password) {
         error_setg(errp, "CHAP username specified but no password was given");
         return;
     }
@@ -994,6 +1247,8 @@
     if (iscsi_set_initiator_username_pwd(iscsi, user, password)) {
         error_setg(errp, "Failed to set initiator username and password");
     }
+
+    g_free(secret);
 }
 
 static void parse_header_digest(struct iscsi_context *iscsi, const char *target,
@@ -1068,16 +1323,37 @@
     return iscsi_name;
 }
 
+static int parse_timeout(const char *target)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    const char *timeout;
+
+    list = qemu_find_opts("iscsi");
+    if (list) {
+        opts = qemu_opts_find(list, target);
+        if (!opts) {
+            opts = QTAILQ_FIRST(&list->head);
+        }
+        if (opts) {
+            timeout = qemu_opt_get(opts, "timeout");
+            if (timeout) {
+                return atoi(timeout);
+            }
+        }
+    }
+
+    return 0;
+}
+
 static void iscsi_nop_timed_event(void *opaque)
 {
     IscsiLun *iscsilun = opaque;
 
-    if (iscsi_get_nops_in_flight(iscsilun->iscsi) > MAX_NOP_FAILURES) {
+    if (iscsi_get_nops_in_flight(iscsilun->iscsi) >= MAX_NOP_FAILURES) {
         error_report("iSCSI: NOP timeout. Reconnecting...");
-        iscsi_reconnect(iscsilun->iscsi);
-    }
-
-    if (iscsi_nop_out_async(iscsilun->iscsi, NULL, NULL, 0, NULL) != 0) {
+        iscsilun->request_timed_out = true;
+    } else if (iscsi_nop_out_async(iscsilun->iscsi, NULL, NULL, 0, NULL) != 0) {
         error_report("iSCSI: failed to sent NOP-Out. Disabling NOP messages.");
         return;
     }
@@ -1109,12 +1385,17 @@
                 } else {
                     iscsilun->block_size = rc16->block_length;
                     iscsilun->num_blocks = rc16->returned_lba + 1;
-                    iscsilun->lbpme = rc16->lbpme;
-                    iscsilun->lbprz = rc16->lbprz;
+                    iscsilun->lbpme = !!rc16->lbpme;
+                    iscsilun->lbprz = !!rc16->lbprz;
                     iscsilun->use_16_for_rw = (rc16->returned_lba > 0xffffffff);
                 }
+                break;
             }
-            break;
+            if (task != NULL && task->status == SCSI_STATUS_CHECK_CONDITION
+                && task->sense.key == SCSI_SENSE_UNIT_ATTENTION) {
+                break;
+            }
+            /* Fall through and try READ CAPACITY(10) instead.  */
         case TYPE_ROM:
             task = iscsi_readcapacity10_sync(iscsilun->iscsi, iscsilun->lun, 0, 0);
             if (task != NULL && task->status == SCSI_STATUS_GOOD) {
@@ -1140,7 +1421,11 @@
              && retries-- > 0);
 
     if (task == NULL || task->status != SCSI_STATUS_GOOD) {
-        error_setg(errp, "iSCSI: failed to send readcapacity10 command.");
+        error_setg(errp, "iSCSI: failed to send readcapacity10/16 command");
+    } else if (!iscsilun->block_size ||
+               iscsilun->block_size % BDRV_SECTOR_SIZE) {
+        error_setg(errp, "iSCSI: the target returned an invalid "
+                   "block size of %d.", iscsilun->block_size);
     }
     if (task) {
         scsi_free_scsi_task(task);
@@ -1203,9 +1488,8 @@
 {
     IscsiLun *iscsilun = bs->opaque;
 
-    aio_set_fd_handler(iscsilun->aio_context,
-                       iscsi_get_fd(iscsilun->iscsi),
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(iscsilun->aio_context, iscsi_get_fd(iscsilun->iscsi),
+                       false, NULL, NULL, NULL);
     iscsilun->events = 0;
 
     if (iscsilun->nop_timer) {
@@ -1213,6 +1497,11 @@
         timer_free(iscsilun->nop_timer);
         iscsilun->nop_timer = NULL;
     }
+    if (iscsilun->event_timer) {
+        timer_del(iscsilun->event_timer);
+        timer_free(iscsilun->event_timer);
+        iscsilun->event_timer = NULL;
+    }
 }
 
 static void iscsi_attach_aio_context(BlockDriverState *bs,
@@ -1229,13 +1518,22 @@
                                         iscsi_nop_timed_event, iscsilun);
     timer_mod(iscsilun->nop_timer,
               qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + NOP_INTERVAL);
+
+    /* Set up a timer for periodic calls to iscsi_set_events and to
+     * scan for command timeout */
+    iscsilun->event_timer = aio_timer_new(iscsilun->aio_context,
+                                          QEMU_CLOCK_REALTIME, SCALE_MS,
+                                          iscsi_timed_check_events, iscsilun);
+    timer_mod(iscsilun->event_timer,
+              qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + EVENT_INTERVAL);
 }
 
-static bool iscsi_is_write_protected(IscsiLun *iscsilun)
+static void iscsi_modesense_sync(IscsiLun *iscsilun)
 {
     struct scsi_task *task;
     struct scsi_mode_sense *ms = NULL;
-    bool wrprotected = false;
+    iscsilun->write_protected = false;
+    iscsilun->dpofua = false;
 
     task = iscsi_modesense6_sync(iscsilun->iscsi, iscsilun->lun,
                                  1, SCSI_MODESENSE_PC_CURRENT,
@@ -1256,22 +1554,18 @@
                      iscsi_get_error(iscsilun->iscsi));
         goto out;
     }
-    wrprotected = ms->device_specific_parameter & 0x80;
+    iscsilun->write_protected = ms->device_specific_parameter & 0x80;
+    iscsilun->dpofua          = ms->device_specific_parameter & 0x10;
 
 out:
     if (task) {
         scsi_free_scsi_task(task);
     }
-    return wrprotected;
 }
 
 /*
  * We support iscsi url's on the form
  * iscsi://[<username>%<password>@]<host>[:<port>]/<targetname>/<lun>
- *
- * Note: flags are currently not used by iscsi_open.  If this function
- * is changed such that flags are used, please examine iscsi_reopen_prepare()
- * to see if needs to be changed as well.
  */
 static int iscsi_open(BlockDriverState *bs, QDict *options, int flags,
                       Error **errp)
@@ -1286,14 +1580,7 @@
     QemuOpts *opts;
     Error *local_err = NULL;
     const char *filename;
-    int i, ret;
-
-    if ((BDRV_SECTOR_SIZE % 512) != 0) {
-        error_setg(errp, "iSCSI: Invalid BDRV_SECTOR_SIZE. "
-                   "BDRV_SECTOR_SIZE(%lld) is not a multiple "
-                   "of 512", BDRV_SECTOR_SIZE);
-        return -EINVAL;
-    }
+    int i, ret = 0, timeout = 0;
 
     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
     qemu_opts_absorb_qdict(opts, options, &local_err);
@@ -1329,7 +1616,7 @@
         goto out;
     }
 
-    if (iscsi_url->user != NULL) {
+    if (iscsi_url->user[0] != '\0') {
         ret = iscsi_set_initiator_username_pwd(iscsi, iscsi_url->user,
                                               iscsi_url->passwd);
         if (ret != 0) {
@@ -1363,6 +1650,16 @@
         goto out;
     }
 
+    /* timeout handling is broken in libiscsi before 1.15.0 */
+    timeout = parse_timeout(iscsi_url->target);
+#if defined(LIBISCSI_API_VERSION) && LIBISCSI_API_VERSION >= 20150621
+    iscsi_set_timeout(iscsi, timeout);
+#else
+    if (timeout) {
+        error_report("iSCSI: ignoring timeout value for libiscsi <1.15.0");
+    }
+#endif
+
     if (iscsi_full_connect_sync(iscsi, iscsi_url->portal, iscsi_url->lun) != 0) {
         error_setg(errp, "iSCSI: Failed to connect to LUN : %s",
             iscsi_get_error(iscsi));
@@ -1385,9 +1682,15 @@
     scsi_free_scsi_task(task);
     task = NULL;
 
+    iscsi_modesense_sync(iscsilun);
+    if (iscsilun->dpofua) {
+        bs->supported_write_flags = BDRV_REQ_FUA;
+    }
+    bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;
+
     /* Check the write protect flag of the LUN if we want to write */
     if (iscsilun->type == TYPE_DISK && (flags & BDRV_O_RDWR) &&
-        iscsi_is_write_protected(iscsilun)) {
+        iscsilun->write_protected) {
         error_setg(errp, "Cannot open a write protected LUN as read-write");
         ret = -EACCES;
         goto out;
@@ -1400,14 +1703,13 @@
         goto out;
     }
     bs->total_sectors = sector_lun2qemu(iscsilun->num_blocks, iscsilun);
-    bs->request_alignment = iscsilun->block_size;
 
     /* We don't have any emulation for devices other than disks and CD-ROMs, so
      * this must be sg ioctl compatible. We force it to be sg, otherwise qemu
      * will try to read from the device to guess the image format.
      */
     if (iscsilun->type != TYPE_DISK && iscsilun->type != TYPE_ROM) {
-        bs->sg = 1;
+        bs->sg = true;
     }
 
     task = iscsi_do_inquiry(iscsilun->iscsi, iscsilun->lun, 1,
@@ -1462,11 +1764,8 @@
         iscsilun->bl.opt_unmap_gran * iscsilun->block_size <= 16 * 1024 * 1024) {
         iscsilun->cluster_sectors = (iscsilun->bl.opt_unmap_gran *
                                      iscsilun->block_size) >> BDRV_SECTOR_BITS;
-        if (iscsilun->lbprz && !(bs->open_flags & BDRV_O_NOCACHE)) {
-            iscsilun->allocationmap = iscsi_allocationmap_init(iscsilun);
-            if (iscsilun->allocationmap == NULL) {
-                ret = -ENOMEM;
-            }
+        if (iscsilun->lbprz) {
+            ret = iscsi_allocmap_init(iscsilun, bs->open_flags);
         }
     }
 
@@ -1482,6 +1781,9 @@
 
     if (ret) {
         if (iscsi != NULL) {
+            if (iscsi_is_logged_in(iscsi)) {
+                iscsi_logout_sync(iscsi);
+            }
             iscsi_destroy_context(iscsi);
         }
         memset(iscsilun, 0, sizeof(IscsiLun));
@@ -1495,62 +1797,85 @@
     struct iscsi_context *iscsi = iscsilun->iscsi;
 
     iscsi_detach_aio_context(bs);
+    if (iscsi_is_logged_in(iscsi)) {
+        iscsi_logout_sync(iscsi);
+    }
     iscsi_destroy_context(iscsi);
     g_free(iscsilun->zeroblock);
-    g_free(iscsilun->allocationmap);
+    iscsi_allocmap_free(iscsilun);
     memset(iscsilun, 0, sizeof(IscsiLun));
 }
 
-static int sector_limits_lun2qemu(int64_t sector, IscsiLun *iscsilun)
-{
-    return MIN(sector_lun2qemu(sector, iscsilun), INT_MAX / 2 + 1);
-}
-
 static void iscsi_refresh_limits(BlockDriverState *bs, Error **errp)
 {
     /* We don't actually refresh here, but just return data queried in
      * iscsi_open(): iscsi targets don't change their limits. */
 
     IscsiLun *iscsilun = bs->opaque;
-    uint32_t max_xfer_len = iscsilun->use_16_for_rw ? 0xffffffff : 0xffff;
+    uint64_t max_xfer_len = iscsilun->use_16_for_rw ? 0xffffffff : 0xffff;
+
+    bs->bl.request_alignment = iscsilun->block_size;
 
     if (iscsilun->bl.max_xfer_len) {
         max_xfer_len = MIN(max_xfer_len, iscsilun->bl.max_xfer_len);
     }
 
-    bs->bl.max_transfer_length = sector_limits_lun2qemu(max_xfer_len, iscsilun);
+    if (max_xfer_len * iscsilun->block_size < INT_MAX) {
+        bs->bl.max_transfer = max_xfer_len * iscsilun->block_size;
+    }
 
     if (iscsilun->lbp.lbpu) {
-        if (iscsilun->bl.max_unmap < 0xffffffff) {
-            bs->bl.max_discard =
-                sector_limits_lun2qemu(iscsilun->bl.max_unmap, iscsilun);
+        if (iscsilun->bl.max_unmap < 0xffffffff / iscsilun->block_size) {
+            bs->bl.max_pdiscard =
+                iscsilun->bl.max_unmap * iscsilun->block_size;
         }
-        bs->bl.discard_alignment =
-            sector_limits_lun2qemu(iscsilun->bl.opt_unmap_gran, iscsilun);
+        bs->bl.pdiscard_alignment =
+            iscsilun->bl.opt_unmap_gran * iscsilun->block_size;
+    } else {
+        bs->bl.pdiscard_alignment = iscsilun->block_size;
     }
 
-    if (iscsilun->bl.max_ws_len < 0xffffffff) {
-        bs->bl.max_write_zeroes =
-            sector_limits_lun2qemu(iscsilun->bl.max_ws_len, iscsilun);
+    if (iscsilun->bl.max_ws_len < 0xffffffff / iscsilun->block_size) {
+        bs->bl.max_pwrite_zeroes =
+            iscsilun->bl.max_ws_len * iscsilun->block_size;
     }
     if (iscsilun->lbp.lbpws) {
-        bs->bl.write_zeroes_alignment =
-            sector_limits_lun2qemu(iscsilun->bl.opt_unmap_gran, iscsilun);
+        bs->bl.pwrite_zeroes_alignment =
+            iscsilun->bl.opt_unmap_gran * iscsilun->block_size;
+    } else {
+        bs->bl.pwrite_zeroes_alignment = iscsilun->block_size;
     }
-    bs->bl.opt_transfer_length =
-        sector_limits_lun2qemu(iscsilun->bl.opt_xfer_len, iscsilun);
+    if (iscsilun->bl.opt_xfer_len &&
+        iscsilun->bl.opt_xfer_len < INT_MAX / iscsilun->block_size) {
+        bs->bl.opt_transfer = pow2floor(iscsilun->bl.opt_xfer_len *
+                                        iscsilun->block_size);
+    }
 }
 
-/* Since iscsi_open() ignores bdrv_flags, there is nothing to do here in
- * prepare.  Note that this will not re-establish a connection with an iSCSI
- * target - it is effectively a NOP.  */
+/* Note that this will not re-establish a connection with an iSCSI target - it
+ * is effectively a NOP.  */
 static int iscsi_reopen_prepare(BDRVReopenState *state,
                                 BlockReopenQueue *queue, Error **errp)
 {
-    /* NOP */
+    IscsiLun *iscsilun = state->bs->opaque;
+
+    if (state->flags & BDRV_O_RDWR && iscsilun->write_protected) {
+        error_setg(errp, "Cannot open a write protected LUN as read-write");
+        return -EACCES;
+    }
     return 0;
 }
 
+static void iscsi_reopen_commit(BDRVReopenState *reopen_state)
+{
+    IscsiLun *iscsilun = reopen_state->bs->opaque;
+
+    /* the cache.direct status might have changed */
+    if (iscsilun->allocmap != NULL) {
+        iscsi_allocmap_init(iscsilun, reopen_state->flags);
+    }
+}
+
 static int iscsi_truncate(BlockDriverState *bs, int64_t offset)
 {
     IscsiLun *iscsilun = bs->opaque;
@@ -1570,9 +1895,8 @@
         return -EINVAL;
     }
 
-    if (iscsilun->allocationmap != NULL) {
-        g_free(iscsilun->allocationmap);
-        iscsilun->allocationmap = iscsi_allocationmap_init(iscsilun);
+    if (iscsilun->allocmap != NULL) {
+        iscsi_allocmap_init(iscsilun, bs->open_flags);
     }
 
     return 0;
@@ -1626,12 +1950,19 @@
 static int iscsi_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
     IscsiLun *iscsilun = bs->opaque;
-    bdi->unallocated_blocks_are_zero = !!iscsilun->lbprz;
+    bdi->unallocated_blocks_are_zero = iscsilun->lbprz;
     bdi->can_write_zeroes_with_unmap = iscsilun->lbprz && iscsilun->lbp.lbpws;
     bdi->cluster_size = iscsilun->cluster_sectors * BDRV_SECTOR_SIZE;
     return 0;
 }
 
+static void iscsi_invalidate_cache(BlockDriverState *bs,
+                                   Error **errp)
+{
+    IscsiLun *iscsilun = bs->opaque;
+    iscsi_allocmap_invalidate(iscsilun);
+}
+
 static QemuOptsList iscsi_create_opts = {
     .name = "iscsi-create-opts",
     .head = QTAILQ_HEAD_INITIALIZER(iscsi_create_opts.head),
@@ -1655,7 +1986,9 @@
     .bdrv_close      = iscsi_close,
     .bdrv_create     = iscsi_create,
     .create_opts     = &iscsi_create_opts,
-    .bdrv_reopen_prepare  = iscsi_reopen_prepare,
+    .bdrv_reopen_prepare   = iscsi_reopen_prepare,
+    .bdrv_reopen_commit    = iscsi_reopen_commit,
+    .bdrv_invalidate_cache = iscsi_invalidate_cache,
 
     .bdrv_getlength  = iscsi_getlength,
     .bdrv_get_info   = iscsi_get_info,
@@ -1663,14 +1996,13 @@
     .bdrv_refresh_limits = iscsi_refresh_limits,
 
     .bdrv_co_get_block_status = iscsi_co_get_block_status,
-    .bdrv_co_discard      = iscsi_co_discard,
-    .bdrv_co_write_zeroes = iscsi_co_write_zeroes,
+    .bdrv_co_pdiscard      = iscsi_co_pdiscard,
+    .bdrv_co_pwrite_zeroes = iscsi_co_pwrite_zeroes,
     .bdrv_co_readv         = iscsi_co_readv,
-    .bdrv_co_writev        = iscsi_co_writev,
+    .bdrv_co_writev_flags  = iscsi_co_writev_flags,
     .bdrv_co_flush_to_disk = iscsi_co_flush,
 
 #ifdef __linux__
-    .bdrv_ioctl       = iscsi_ioctl,
     .bdrv_aio_ioctl   = iscsi_aio_ioctl,
 #endif
 
@@ -1691,6 +2023,11 @@
             .type = QEMU_OPT_STRING,
             .help = "password for CHAP authentication to target",
         },{
+            .name = "password-secret",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the secret providing password for CHAP "
+                    "authentication to target",
+        },{
             .name = "header-digest",
             .type = QEMU_OPT_STRING,
             .help = "HeaderDigest setting. "
@@ -1699,6 +2036,10 @@
             .name = "initiator-name",
             .type = QEMU_OPT_STRING,
             .help = "Initiator iqn name to use when connecting",
+        },{
+            .name = "timeout",
+            .type = QEMU_OPT_NUMBER,
+            .help = "Request timeout in seconds (default 0 = no timeout)",
         },
         { /* end of list */ }
     },
diff --git a/block/linux-aio.c b/block/linux-aio.c
index d92513b..e906abe 100644
--- a/block/linux-aio.c
+++ b/block/linux-aio.c
@@ -7,11 +7,14 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/aio.h"
 #include "qemu/queue.h"
+#include "block/block.h"
 #include "block/raw-aio.h"
 #include "qemu/event_notifier.h"
+#include "qemu/coroutine.h"
 
 #include <libaio.h>
 
@@ -25,27 +28,29 @@
  */
 #define MAX_EVENTS 128
 
-#define MAX_QUEUED_IO  128
-
 struct qemu_laiocb {
     BlockAIOCB common;
-    struct qemu_laio_state *ctx;
+    Coroutine *co;
+    LinuxAioState *ctx;
     struct iocb iocb;
     ssize_t ret;
     size_t nbytes;
     QEMUIOVector *qiov;
     bool is_read;
-    QLIST_ENTRY(qemu_laiocb) node;
+    QSIMPLEQ_ENTRY(qemu_laiocb) next;
 };
 
 typedef struct {
-    struct iocb *iocbs[MAX_QUEUED_IO];
     int plugged;
-    unsigned int size;
-    unsigned int idx;
+    unsigned int in_queue;
+    unsigned int in_flight;
+    bool blocked;
+    QSIMPLEQ_HEAD(, qemu_laiocb) pending;
 } LaioQueue;
 
-struct qemu_laio_state {
+struct LinuxAioState {
+    AioContext *aio_context;
+
     io_context_t ctx;
     EventNotifier e;
 
@@ -59,6 +64,8 @@
     int event_max;
 };
 
+static void ioq_submit(LinuxAioState *s);
+
 static inline ssize_t io_event_ret(struct io_event *ev)
 {
     return (ssize_t)(((uint64_t)ev->res2 << 32) | ev->res);
@@ -67,8 +74,7 @@
 /*
  * Completes an AIO request (calls the callback and frees the ACB).
  */
-static void qemu_laio_process_completion(struct qemu_laio_state *s,
-    struct qemu_laiocb *laiocb)
+static void qemu_laio_process_completion(struct qemu_laiocb *laiocb)
 {
     int ret;
 
@@ -82,13 +88,18 @@
                 qemu_iovec_memset(laiocb->qiov, ret, 0,
                     laiocb->qiov->size - ret);
             } else {
-                ret = -EINVAL;
+                ret = -ENOSPC;
             }
         }
     }
-    laiocb->common.cb(laiocb->common.opaque, ret);
 
-    qemu_aio_unref(laiocb);
+    laiocb->ret = ret;
+    if (laiocb->co) {
+        qemu_coroutine_enter(laiocb->co);
+    } else {
+        laiocb->common.cb(laiocb->common.opaque, ret);
+        qemu_aio_unref(laiocb);
+    }
 }
 
 /* The completion BH fetches completed I/O requests and invokes their
@@ -96,7 +107,7 @@
  *
  * The function is somewhat tricky because it supports nested event loops, for
  * example when a request callback invokes aio_poll().  In order to do this,
- * the completion events array and index are kept in qemu_laio_state.  The BH
+ * the completion events array and index are kept in LinuxAioState.  The BH
  * reschedules itself as long as there are completions pending so it will
  * either be called again in a nested event loop or will be called after all
  * events have been completed.  When there are no events left to complete, the
@@ -104,7 +115,7 @@
  */
 static void qemu_laio_completion_bh(void *opaque)
 {
-    struct qemu_laio_state *s = opaque;
+    LinuxAioState *s = opaque;
 
     /* Fetch more completion events when empty */
     if (s->event_idx == s->event_max) {
@@ -119,6 +130,7 @@
             s->event_max = 0;
             return; /* no more events */
         }
+        s->io_q.in_flight -= s->event_max;
     }
 
     /* Reschedule so nested event loops see currently pending completions */
@@ -133,16 +145,22 @@
         laiocb->ret = io_event_ret(&s->events[s->event_idx]);
         s->event_idx++;
 
-        qemu_laio_process_completion(s, laiocb);
+        qemu_laio_process_completion(laiocb);
     }
+
+    if (!s->io_q.plugged && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
+        ioq_submit(s);
+    }
+
+    qemu_bh_cancel(s->completion_bh);
 }
 
 static void qemu_laio_completion_cb(EventNotifier *e)
 {
-    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
+    LinuxAioState *s = container_of(e, LinuxAioState, e);
 
     if (event_notifier_test_and_clear(&s->e)) {
-        qemu_bh_schedule(s->completion_bh);
+        qemu_laio_completion_bh(s);
     }
 }
 
@@ -172,94 +190,74 @@
 
 static void ioq_init(LaioQueue *io_q)
 {
-    io_q->size = MAX_QUEUED_IO;
-    io_q->idx = 0;
+    QSIMPLEQ_INIT(&io_q->pending);
     io_q->plugged = 0;
+    io_q->in_queue = 0;
+    io_q->in_flight = 0;
+    io_q->blocked = false;
 }
 
-static int ioq_submit(struct qemu_laio_state *s)
+static void ioq_submit(LinuxAioState *s)
 {
-    int ret, i = 0;
-    int len = s->io_q.idx;
+    int ret, len;
+    struct qemu_laiocb *aiocb;
+    struct iocb *iocbs[MAX_EVENTS];
+    QSIMPLEQ_HEAD(, qemu_laiocb) completed;
 
     do {
-        ret = io_submit(s->ctx, len, s->io_q.iocbs);
-    } while (i++ < 3 && ret == -EAGAIN);
+        if (s->io_q.in_flight >= MAX_EVENTS) {
+            break;
+        }
+        len = 0;
+        QSIMPLEQ_FOREACH(aiocb, &s->io_q.pending, next) {
+            iocbs[len++] = &aiocb->iocb;
+            if (s->io_q.in_flight + len >= MAX_EVENTS) {
+                break;
+            }
+        }
 
-    /* empty io queue */
-    s->io_q.idx = 0;
+        ret = io_submit(s->ctx, len, iocbs);
+        if (ret == -EAGAIN) {
+            break;
+        }
+        if (ret < 0) {
+            /* Fail the first request, retry the rest */
+            aiocb = QSIMPLEQ_FIRST(&s->io_q.pending);
+            QSIMPLEQ_REMOVE_HEAD(&s->io_q.pending, next);
+            s->io_q.in_queue--;
+            aiocb->ret = ret;
+            qemu_laio_process_completion(aiocb);
+            continue;
+        }
 
-    if (ret < 0) {
-        i = 0;
-    } else {
-        i = ret;
-    }
-
-    for (; i < len; i++) {
-        struct qemu_laiocb *laiocb =
-            container_of(s->io_q.iocbs[i], struct qemu_laiocb, iocb);
-
-        laiocb->ret = (ret < 0) ? ret : -EIO;
-        qemu_laio_process_completion(s, laiocb);
-    }
-    return ret;
+        s->io_q.in_flight += ret;
+        s->io_q.in_queue  -= ret;
+        aiocb = container_of(iocbs[ret - 1], struct qemu_laiocb, iocb);
+        QSIMPLEQ_SPLIT_AFTER(&s->io_q.pending, aiocb, next, &completed);
+    } while (ret == len && !QSIMPLEQ_EMPTY(&s->io_q.pending));
+    s->io_q.blocked = (s->io_q.in_queue > 0);
 }
 
-static void ioq_enqueue(struct qemu_laio_state *s, struct iocb *iocb)
+void laio_io_plug(BlockDriverState *bs, LinuxAioState *s)
 {
-    unsigned int idx = s->io_q.idx;
+    s->io_q.plugged++;
+}
 
-    s->io_q.iocbs[idx++] = iocb;
-    s->io_q.idx = idx;
-
-    /* submit immediately if queue is full */
-    if (idx == s->io_q.size) {
+void laio_io_unplug(BlockDriverState *bs, LinuxAioState *s)
+{
+    assert(s->io_q.plugged);
+    if (--s->io_q.plugged == 0 &&
+        !s->io_q.blocked && !QSIMPLEQ_EMPTY(&s->io_q.pending)) {
         ioq_submit(s);
     }
 }
 
-void laio_io_plug(BlockDriverState *bs, void *aio_ctx)
+static int laio_do_submit(int fd, struct qemu_laiocb *laiocb, off_t offset,
+                          int type)
 {
-    struct qemu_laio_state *s = aio_ctx;
-
-    s->io_q.plugged++;
-}
-
-int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug)
-{
-    struct qemu_laio_state *s = aio_ctx;
-    int ret = 0;
-
-    assert(s->io_q.plugged > 0 || !unplug);
-
-    if (unplug && --s->io_q.plugged > 0) {
-        return 0;
-    }
-
-    if (s->io_q.idx > 0) {
-        ret = ioq_submit(s);
-    }
-
-    return ret;
-}
-
-BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque, int type)
-{
-    struct qemu_laio_state *s = aio_ctx;
-    struct qemu_laiocb *laiocb;
-    struct iocb *iocbs;
-    off_t offset = sector_num * 512;
-
-    laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque);
-    laiocb->nbytes = nb_sectors * 512;
-    laiocb->ctx = s;
-    laiocb->ret = -EINPROGRESS;
-    laiocb->is_read = (type == QEMU_AIO_READ);
-    laiocb->qiov = qiov;
-
-    iocbs = &laiocb->iocb;
+    LinuxAioState *s = laiocb->ctx;
+    struct iocb *iocbs = &laiocb->iocb;
+    QEMUIOVector *qiov = laiocb->qiov;
 
     switch (type) {
     case QEMU_AIO_WRITE:
@@ -272,43 +270,83 @@
     default:
         fprintf(stderr, "%s: invalid AIO request type 0x%x.\n",
                         __func__, type);
-        goto out_free_aiocb;
+        return -EIO;
     }
     io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
 
-    if (!s->io_q.plugged) {
-        if (io_submit(s->ctx, 1, &iocbs) < 0) {
-            goto out_free_aiocb;
-        }
-    } else {
-        ioq_enqueue(s, iocbs);
+    QSIMPLEQ_INSERT_TAIL(&s->io_q.pending, laiocb, next);
+    s->io_q.in_queue++;
+    if (!s->io_q.blocked &&
+        (!s->io_q.plugged ||
+         s->io_q.in_flight + s->io_q.in_queue >= MAX_EVENTS)) {
+        ioq_submit(s);
     }
-    return &laiocb->common;
 
-out_free_aiocb:
-    qemu_aio_unref(laiocb);
-    return NULL;
+    return 0;
 }
 
-void laio_detach_aio_context(void *s_, AioContext *old_context)
+int coroutine_fn laio_co_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
+                                uint64_t offset, QEMUIOVector *qiov, int type)
 {
-    struct qemu_laio_state *s = s_;
+    int ret;
+    struct qemu_laiocb laiocb = {
+        .co         = qemu_coroutine_self(),
+        .nbytes     = qiov->size,
+        .ctx        = s,
+        .is_read    = (type == QEMU_AIO_READ),
+        .qiov       = qiov,
+    };
 
-    aio_set_event_notifier(old_context, &s->e, NULL);
+    ret = laio_do_submit(fd, &laiocb, offset, type);
+    if (ret < 0) {
+        return ret;
+    }
+
+    qemu_coroutine_yield();
+    return laiocb.ret;
+}
+
+BlockAIOCB *laio_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockCompletionFunc *cb, void *opaque, int type)
+{
+    struct qemu_laiocb *laiocb;
+    off_t offset = sector_num * BDRV_SECTOR_SIZE;
+    int ret;
+
+    laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque);
+    laiocb->nbytes = nb_sectors * BDRV_SECTOR_SIZE;
+    laiocb->ctx = s;
+    laiocb->ret = -EINPROGRESS;
+    laiocb->is_read = (type == QEMU_AIO_READ);
+    laiocb->qiov = qiov;
+
+    ret = laio_do_submit(fd, laiocb, offset, type);
+    if (ret < 0) {
+        qemu_aio_unref(laiocb);
+        return NULL;
+    }
+
+    return &laiocb->common;
+}
+
+void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context)
+{
+    aio_set_event_notifier(old_context, &s->e, false, NULL);
     qemu_bh_delete(s->completion_bh);
 }
 
-void laio_attach_aio_context(void *s_, AioContext *new_context)
+void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context)
 {
-    struct qemu_laio_state *s = s_;
-
+    s->aio_context = new_context;
     s->completion_bh = aio_bh_new(new_context, qemu_laio_completion_bh, s);
-    aio_set_event_notifier(new_context, &s->e, qemu_laio_completion_cb);
+    aio_set_event_notifier(new_context, &s->e, false,
+                           qemu_laio_completion_cb);
 }
 
-void *laio_init(void)
+LinuxAioState *laio_init(void)
 {
-    struct qemu_laio_state *s;
+    LinuxAioState *s;
 
     s = g_malloc0(sizeof(*s));
     if (event_notifier_init(&s->e, false) < 0) {
@@ -330,10 +368,8 @@
     return NULL;
 }
 
-void laio_cleanup(void *s_)
+void laio_cleanup(LinuxAioState *s)
 {
-    struct qemu_laio_state *s = s_;
-
     event_notifier_cleanup(&s->e);
 
     if (io_destroy(s->ctx) != 0) {
diff --git a/block/mirror.c b/block/mirror.c
index 2c6dd2a..e0b3f41 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -11,14 +11,21 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
 #include "qemu/bitmap.h"
 
 #define SLICE_TIME    100000000ULL /* ns */
 #define MAX_IN_FLIGHT 16
+#define MAX_IO_SECTORS ((1 << 20) >> BDRV_SECTOR_BITS) /* 1 Mb */
+#define DEFAULT_MIRROR_BUF_SIZE \
+    (MAX_IN_FLIGHT * MAX_IO_SECTORS * BDRV_SECTOR_SIZE)
 
 /* The mirroring buffer is a list of granularity-sized chunks.
  * Free chunks are organized in a list.
@@ -30,7 +37,7 @@
 typedef struct MirrorBlockJob {
     BlockJob common;
     RateLimit limit;
-    BlockDriverState *target;
+    BlockBackend *target;
     BlockDriverState *base;
     /* The name of the graph node to replace */
     char *replaces;
@@ -39,10 +46,10 @@
     /* Used to block operations on the drive-mirror-replace target */
     Error *replace_blocker;
     bool is_none_mode;
+    BlockMirrorBackingMode backing_mode;
     BlockdevOnError on_source_error, on_target_error;
     bool synced;
     bool should_complete;
-    int64_t sector_num;
     int64_t granularity;
     size_t buf_size;
     int64_t bdev_length;
@@ -53,10 +60,15 @@
     QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
     int buf_free_count;
 
+    uint64_t last_pause_ns;
     unsigned long *in_flight_bitmap;
     int in_flight;
-    int sectors_in_flight;
+    int64_t sectors_in_flight;
     int ret;
+    bool unmap;
+    bool waiting_for_io;
+    int target_cluster_sectors;
+    int max_iov;
 } MirrorBlockJob;
 
 typedef struct MirrorOp {
@@ -71,11 +83,11 @@
 {
     s->synced = false;
     if (read) {
-        return block_job_error_action(&s->common, s->common.bs,
-                                      s->on_source_error, true, error);
+        return block_job_error_action(&s->common, s->on_source_error,
+                                      true, error);
     } else {
-        return block_job_error_action(&s->common, s->target,
-                                      s->on_target_error, false, error);
+        return block_job_error_action(&s->common, s->on_target_error,
+                                      false, error);
     }
 }
 
@@ -99,7 +111,7 @@
 
     sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
     chunk_num = op->sector_num / sectors_per_chunk;
-    nb_chunks = op->nb_sectors / sectors_per_chunk;
+    nb_chunks = DIV_ROUND_UP(op->nb_sectors, sectors_per_chunk);
     bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
     if (ret >= 0) {
         if (s->cow_bitmap) {
@@ -109,14 +121,10 @@
     }
 
     qemu_iovec_destroy(&op->qiov);
-    g_slice_free(MirrorOp, op);
+    g_free(op);
 
-    /* Enter coroutine when it is not sleeping.  The coroutine sleeps to
-     * rate-limit itself.  The coroutine will eventually resume since there is
-     * a sleep timeout so don't wake it early.
-     */
-    if (s->common.busy) {
-        qemu_coroutine_enter(s->common.co, NULL);
+    if (s->waiting_for_io) {
+        qemu_coroutine_enter(s->common.co);
     }
 }
 
@@ -125,10 +133,9 @@
     MirrorOp *op = opaque;
     MirrorBlockJob *s = op->s;
     if (ret < 0) {
-        BlockDriverState *source = s->common.bs;
         BlockErrorAction action;
 
-        bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
+        bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors);
         action = mirror_error_action(s, false, -ret);
         if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
             s->ret = ret;
@@ -142,10 +149,9 @@
     MirrorOp *op = opaque;
     MirrorBlockJob *s = op->s;
     if (ret < 0) {
-        BlockDriverState *source = s->common.bs;
         BlockErrorAction action;
 
-        bdrv_set_dirty(source, op->sector_num, op->nb_sectors);
+        bdrv_set_dirty_bitmap(s->dirty_bitmap, op->sector_num, op->nb_sectors);
         action = mirror_error_action(s, true, -ret);
         if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
             s->ret = ret;
@@ -154,110 +160,108 @@
         mirror_iteration_done(op, ret);
         return;
     }
-    bdrv_aio_writev(s->target, op->sector_num, &op->qiov, op->nb_sectors,
-                    mirror_write_complete, op);
+    blk_aio_pwritev(s->target, op->sector_num * BDRV_SECTOR_SIZE, &op->qiov,
+                    0, mirror_write_complete, op);
 }
 
-static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
+static inline void mirror_clip_sectors(MirrorBlockJob *s,
+                                       int64_t sector_num,
+                                       int *nb_sectors)
 {
-    BlockDriverState *source = s->common.bs;
-    int nb_sectors, sectors_per_chunk, nb_chunks;
-    int64_t end, sector_num, next_chunk, next_sector, hbitmap_next_sector;
-    uint64_t delay_ns = 0;
+    *nb_sectors = MIN(*nb_sectors,
+                      s->bdev_length / BDRV_SECTOR_SIZE - sector_num);
+}
+
+/* Round sector_num and/or nb_sectors to target cluster if COW is needed, and
+ * return the offset of the adjusted tail sector against original. */
+static int mirror_cow_align(MirrorBlockJob *s,
+                            int64_t *sector_num,
+                            int *nb_sectors)
+{
+    bool need_cow;
+    int ret = 0;
+    int chunk_sectors = s->granularity >> BDRV_SECTOR_BITS;
+    int64_t align_sector_num = *sector_num;
+    int align_nb_sectors = *nb_sectors;
+    int max_sectors = chunk_sectors * s->max_iov;
+
+    need_cow = !test_bit(*sector_num / chunk_sectors, s->cow_bitmap);
+    need_cow |= !test_bit((*sector_num + *nb_sectors - 1) / chunk_sectors,
+                          s->cow_bitmap);
+    if (need_cow) {
+        bdrv_round_sectors_to_clusters(blk_bs(s->target), *sector_num,
+                                       *nb_sectors, &align_sector_num,
+                                       &align_nb_sectors);
+    }
+
+    if (align_nb_sectors > max_sectors) {
+        align_nb_sectors = max_sectors;
+        if (need_cow) {
+            align_nb_sectors = QEMU_ALIGN_DOWN(align_nb_sectors,
+                                               s->target_cluster_sectors);
+        }
+    }
+    /* Clipping may result in align_nb_sectors unaligned to chunk boundary, but
+     * that doesn't matter because it's already the end of source image. */
+    mirror_clip_sectors(s, align_sector_num, &align_nb_sectors);
+
+    ret = align_sector_num + align_nb_sectors - (*sector_num + *nb_sectors);
+    *sector_num = align_sector_num;
+    *nb_sectors = align_nb_sectors;
+    assert(ret >= 0);
+    return ret;
+}
+
+static inline void mirror_wait_for_io(MirrorBlockJob *s)
+{
+    assert(!s->waiting_for_io);
+    s->waiting_for_io = true;
+    qemu_coroutine_yield();
+    s->waiting_for_io = false;
+}
+
+/* Submit async read while handling COW.
+ * Returns: The number of sectors copied after and including sector_num,
+ *          excluding any sectors copied prior to sector_num due to alignment.
+ *          This will be nb_sectors if no alignment is necessary, or
+ *          (new_end - sector_num) if tail is rounded up or down due to
+ *          alignment or buffer limit.
+ */
+static int mirror_do_read(MirrorBlockJob *s, int64_t sector_num,
+                          int nb_sectors)
+{
+    BlockBackend *source = s->common.blk;
+    int sectors_per_chunk, nb_chunks;
+    int ret;
     MirrorOp *op;
+    int max_sectors;
 
-    s->sector_num = hbitmap_iter_next(&s->hbi);
-    if (s->sector_num < 0) {
-        bdrv_dirty_iter_init(source, s->dirty_bitmap, &s->hbi);
-        s->sector_num = hbitmap_iter_next(&s->hbi);
-        trace_mirror_restart_iter(s,
-                                  bdrv_get_dirty_count(source, s->dirty_bitmap));
-        assert(s->sector_num >= 0);
-    }
-
-    hbitmap_next_sector = s->sector_num;
-    sector_num = s->sector_num;
     sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
-    end = s->bdev_length / BDRV_SECTOR_SIZE;
+    max_sectors = sectors_per_chunk * s->max_iov;
 
-    /* Extend the QEMUIOVector to include all adjacent blocks that will
-     * be copied in this operation.
-     *
-     * We have to do this if we have no backing file yet in the destination,
-     * and the cluster size is very large.  Then we need to do COW ourselves.
-     * The first time a cluster is copied, copy it entirely.  Note that,
-     * because both the granularity and the cluster size are powers of two,
-     * the number of sectors to copy cannot exceed one cluster.
-     *
-     * We also want to extend the QEMUIOVector to include more adjacent
-     * dirty blocks if possible, to limit the number of I/O operations and
-     * run efficiently even with a small granularity.
-     */
-    nb_chunks = 0;
-    nb_sectors = 0;
-    next_sector = sector_num;
-    next_chunk = sector_num / sectors_per_chunk;
+    /* We can only handle as much as buf_size at a time. */
+    nb_sectors = MIN(s->buf_size >> BDRV_SECTOR_BITS, nb_sectors);
+    nb_sectors = MIN(max_sectors, nb_sectors);
+    assert(nb_sectors);
+    ret = nb_sectors;
 
-    /* Wait for I/O to this cluster (from a previous iteration) to be done.  */
-    while (test_bit(next_chunk, s->in_flight_bitmap)) {
-        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
-        qemu_coroutine_yield();
+    if (s->cow_bitmap) {
+        ret += mirror_cow_align(s, &sector_num, &nb_sectors);
     }
+    assert(nb_sectors << BDRV_SECTOR_BITS <= s->buf_size);
+    /* The sector range must meet granularity because:
+     * 1) Caller passes in aligned values;
+     * 2) mirror_cow_align is used only when target cluster is larger. */
+    assert(!(sector_num % sectors_per_chunk));
+    nb_chunks = DIV_ROUND_UP(nb_sectors, sectors_per_chunk);
 
-    do {
-        int added_sectors, added_chunks;
-
-        if (!bdrv_get_dirty(source, s->dirty_bitmap, next_sector) ||
-            test_bit(next_chunk, s->in_flight_bitmap)) {
-            assert(nb_sectors > 0);
-            break;
-        }
-
-        added_sectors = sectors_per_chunk;
-        if (s->cow_bitmap && !test_bit(next_chunk, s->cow_bitmap)) {
-            bdrv_round_to_clusters(s->target,
-                                   next_sector, added_sectors,
-                                   &next_sector, &added_sectors);
-
-            /* On the first iteration, the rounding may make us copy
-             * sectors before the first dirty one.
-             */
-            if (next_sector < sector_num) {
-                assert(nb_sectors == 0);
-                sector_num = next_sector;
-                next_chunk = next_sector / sectors_per_chunk;
-            }
-        }
-
-        added_sectors = MIN(added_sectors, end - (sector_num + nb_sectors));
-        added_chunks = (added_sectors + sectors_per_chunk - 1) / sectors_per_chunk;
-
-        /* When doing COW, it may happen that there is not enough space for
-         * a full cluster.  Wait if that is the case.
-         */
-        while (nb_chunks == 0 && s->buf_free_count < added_chunks) {
-            trace_mirror_yield_buf_busy(s, nb_chunks, s->in_flight);
-            qemu_coroutine_yield();
-        }
-        if (s->buf_free_count < nb_chunks + added_chunks) {
-            trace_mirror_break_buf_busy(s, nb_chunks, s->in_flight);
-            break;
-        }
-
-        /* We have enough free space to copy these sectors.  */
-        bitmap_set(s->in_flight_bitmap, next_chunk, added_chunks);
-
-        nb_sectors += added_sectors;
-        nb_chunks += added_chunks;
-        next_sector += added_sectors;
-        next_chunk += added_chunks;
-        if (!s->synced && s->common.speed) {
-            delay_ns = ratelimit_calculate_delay(&s->limit, added_sectors);
-        }
-    } while (delay_ns == 0 && next_sector < end);
+    while (s->buf_free_count < nb_chunks) {
+        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
+        mirror_wait_for_io(s);
+    }
 
     /* Allocate a MirrorOp that is used as an AIO callback.  */
-    op = g_slice_new(MirrorOp);
+    op = g_new(MirrorOp, 1);
     op->s = s;
     op->sector_num = sector_num;
     op->nb_sectors = nb_sectors;
@@ -266,34 +270,185 @@
      * from s->buf_free.
      */
     qemu_iovec_init(&op->qiov, nb_chunks);
-    next_sector = sector_num;
     while (nb_chunks-- > 0) {
         MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
-        size_t remaining = (nb_sectors * BDRV_SECTOR_SIZE) - op->qiov.size;
+        size_t remaining = nb_sectors * BDRV_SECTOR_SIZE - op->qiov.size;
 
         QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
         s->buf_free_count--;
         qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
-
-        /* Advance the HBitmapIter in parallel, so that we do not examine
-         * the same sector twice.
-         */
-        if (next_sector > hbitmap_next_sector
-            && bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) {
-            hbitmap_next_sector = hbitmap_iter_next(&s->hbi);
-        }
-
-        next_sector += sectors_per_chunk;
     }
 
-    bdrv_reset_dirty(source, sector_num, nb_sectors);
-
     /* Copy the dirty cluster.  */
     s->in_flight++;
     s->sectors_in_flight += nb_sectors;
     trace_mirror_one_iteration(s, sector_num, nb_sectors);
-    bdrv_aio_readv(source, sector_num, &op->qiov, nb_sectors,
+
+    blk_aio_preadv(source, sector_num * BDRV_SECTOR_SIZE, &op->qiov, 0,
                    mirror_read_complete, op);
+    return ret;
+}
+
+static void mirror_do_zero_or_discard(MirrorBlockJob *s,
+                                      int64_t sector_num,
+                                      int nb_sectors,
+                                      bool is_discard)
+{
+    MirrorOp *op;
+
+    /* Allocate a MirrorOp that is used as an AIO callback. The qiov is zeroed
+     * so the freeing in mirror_iteration_done is nop. */
+    op = g_new0(MirrorOp, 1);
+    op->s = s;
+    op->sector_num = sector_num;
+    op->nb_sectors = nb_sectors;
+
+    s->in_flight++;
+    s->sectors_in_flight += nb_sectors;
+    if (is_discard) {
+        blk_aio_pdiscard(s->target, sector_num << BDRV_SECTOR_BITS,
+                         op->nb_sectors << BDRV_SECTOR_BITS,
+                         mirror_write_complete, op);
+    } else {
+        blk_aio_pwrite_zeroes(s->target, sector_num * BDRV_SECTOR_SIZE,
+                              op->nb_sectors * BDRV_SECTOR_SIZE,
+                              s->unmap ? BDRV_REQ_MAY_UNMAP : 0,
+                              mirror_write_complete, op);
+    }
+}
+
+static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
+{
+    BlockDriverState *source = blk_bs(s->common.blk);
+    int64_t sector_num, first_chunk;
+    uint64_t delay_ns = 0;
+    /* At least the first dirty chunk is mirrored in one iteration. */
+    int nb_chunks = 1;
+    int64_t end = s->bdev_length / BDRV_SECTOR_SIZE;
+    int sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
+    bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
+    int max_io_sectors = MAX((s->buf_size >> BDRV_SECTOR_BITS) / MAX_IN_FLIGHT,
+                             MAX_IO_SECTORS);
+
+    sector_num = hbitmap_iter_next(&s->hbi);
+    if (sector_num < 0) {
+        bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi);
+        sector_num = hbitmap_iter_next(&s->hbi);
+        trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
+        assert(sector_num >= 0);
+    }
+
+    first_chunk = sector_num / sectors_per_chunk;
+    while (test_bit(first_chunk, s->in_flight_bitmap)) {
+        trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
+        mirror_wait_for_io(s);
+    }
+
+    block_job_pause_point(&s->common);
+
+    /* Find the number of consective dirty chunks following the first dirty
+     * one, and wait for in flight requests in them. */
+    while (nb_chunks * sectors_per_chunk < (s->buf_size >> BDRV_SECTOR_BITS)) {
+        int64_t hbitmap_next;
+        int64_t next_sector = sector_num + nb_chunks * sectors_per_chunk;
+        int64_t next_chunk = next_sector / sectors_per_chunk;
+        if (next_sector >= end ||
+            !bdrv_get_dirty(source, s->dirty_bitmap, next_sector)) {
+            break;
+        }
+        if (test_bit(next_chunk, s->in_flight_bitmap)) {
+            break;
+        }
+
+        hbitmap_next = hbitmap_iter_next(&s->hbi);
+        if (hbitmap_next > next_sector || hbitmap_next < 0) {
+            /* The bitmap iterator's cache is stale, refresh it */
+            bdrv_set_dirty_iter(&s->hbi, next_sector);
+            hbitmap_next = hbitmap_iter_next(&s->hbi);
+        }
+        assert(hbitmap_next == next_sector);
+        nb_chunks++;
+    }
+
+    /* Clear dirty bits before querying the block status, because
+     * calling bdrv_get_block_status_above could yield - if some blocks are
+     * marked dirty in this window, we need to know.
+     */
+    bdrv_reset_dirty_bitmap(s->dirty_bitmap, sector_num,
+                            nb_chunks * sectors_per_chunk);
+    bitmap_set(s->in_flight_bitmap, sector_num / sectors_per_chunk, nb_chunks);
+    while (nb_chunks > 0 && sector_num < end) {
+        int ret;
+        int io_sectors, io_sectors_acct;
+        BlockDriverState *file;
+        enum MirrorMethod {
+            MIRROR_METHOD_COPY,
+            MIRROR_METHOD_ZERO,
+            MIRROR_METHOD_DISCARD
+        } mirror_method = MIRROR_METHOD_COPY;
+
+        assert(!(sector_num % sectors_per_chunk));
+        ret = bdrv_get_block_status_above(source, NULL, sector_num,
+                                          nb_chunks * sectors_per_chunk,
+                                          &io_sectors, &file);
+        if (ret < 0) {
+            io_sectors = MIN(nb_chunks * sectors_per_chunk, max_io_sectors);
+        } else if (ret & BDRV_BLOCK_DATA) {
+            io_sectors = MIN(io_sectors, max_io_sectors);
+        }
+
+        io_sectors -= io_sectors % sectors_per_chunk;
+        if (io_sectors < sectors_per_chunk) {
+            io_sectors = sectors_per_chunk;
+        } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
+            int64_t target_sector_num;
+            int target_nb_sectors;
+            bdrv_round_sectors_to_clusters(blk_bs(s->target), sector_num,
+                                           io_sectors,  &target_sector_num,
+                                           &target_nb_sectors);
+            if (target_sector_num == sector_num &&
+                target_nb_sectors == io_sectors) {
+                mirror_method = ret & BDRV_BLOCK_ZERO ?
+                                    MIRROR_METHOD_ZERO :
+                                    MIRROR_METHOD_DISCARD;
+            }
+        }
+
+        while (s->in_flight >= MAX_IN_FLIGHT) {
+            trace_mirror_yield_in_flight(s, sector_num, s->in_flight);
+            mirror_wait_for_io(s);
+        }
+
+        if (s->ret < 0) {
+            return 0;
+        }
+
+        mirror_clip_sectors(s, sector_num, &io_sectors);
+        switch (mirror_method) {
+        case MIRROR_METHOD_COPY:
+            io_sectors = mirror_do_read(s, sector_num, io_sectors);
+            io_sectors_acct = io_sectors;
+            break;
+        case MIRROR_METHOD_ZERO:
+        case MIRROR_METHOD_DISCARD:
+            mirror_do_zero_or_discard(s, sector_num, io_sectors,
+                                      mirror_method == MIRROR_METHOD_DISCARD);
+            if (write_zeroes_ok) {
+                io_sectors_acct = 0;
+            } else {
+                io_sectors_acct = io_sectors;
+            }
+            break;
+        default:
+            abort();
+        }
+        assert(io_sectors);
+        sector_num += io_sectors;
+        nb_chunks -= DIV_ROUND_UP(io_sectors, sectors_per_chunk);
+        if (s->common.speed) {
+            delay_ns = ratelimit_calculate_delay(&s->limit, io_sectors_acct);
+        }
+    }
     return delay_ns;
 }
 
@@ -317,7 +472,7 @@
 static void mirror_drain(MirrorBlockJob *s)
 {
     while (s->in_flight > 0) {
-        qemu_coroutine_yield();
+        mirror_wait_for_io(s);
     }
 }
 
@@ -330,6 +485,12 @@
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
     MirrorExitData *data = opaque;
     AioContext *replace_aio_context = NULL;
+    BlockDriverState *src = blk_bs(s->common.blk);
+    BlockDriverState *target_bs = blk_bs(s->target);
+
+    /* Make sure that the source BDS doesn't go away before we called
+     * block_job_completed(). */
+    bdrv_ref(src);
 
     if (s->to_replace) {
         replace_aio_context = bdrv_get_aio_context(s->to_replace);
@@ -337,21 +498,24 @@
     }
 
     if (s->should_complete && data->ret == 0) {
-        BlockDriverState *to_replace = s->common.bs;
+        BlockDriverState *to_replace = src;
         if (s->to_replace) {
             to_replace = s->to_replace;
         }
-        if (bdrv_get_flags(s->target) != bdrv_get_flags(to_replace)) {
-            bdrv_reopen(s->target, bdrv_get_flags(to_replace), NULL);
+
+        if (bdrv_get_flags(target_bs) != bdrv_get_flags(to_replace)) {
+            bdrv_reopen(target_bs, bdrv_get_flags(to_replace), NULL);
         }
-        bdrv_swap(s->target, to_replace);
-        if (s->common.driver->job_type == BLOCK_JOB_TYPE_COMMIT) {
-            /* drop the bs loop chain formed by the swap: break the loop then
-             * trigger the unref from the top one */
-            BlockDriverState *p = s->base->backing_hd;
-            bdrv_set_backing_hd(s->base, NULL);
-            bdrv_unref(p);
-        }
+
+        /* The mirror job has no requests in flight any more, but we need to
+         * drain potential other users of the BDS before changing the graph. */
+        bdrv_drained_begin(target_bs);
+        bdrv_replace_in_backing_chain(to_replace, target_bs);
+        bdrv_drained_end(target_bs);
+
+        /* We just changed the BDS the job BB refers to */
+        blk_remove_bs(job->blk);
+        blk_insert_bs(job->blk, src);
     }
     if (s->to_replace) {
         bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
@@ -362,22 +526,103 @@
         aio_context_release(replace_aio_context);
     }
     g_free(s->replaces);
-    bdrv_unref(s->target);
+    bdrv_op_unblock_all(target_bs, s->common.blocker);
+    blk_unref(s->target);
     block_job_completed(&s->common, data->ret);
     g_free(data);
+    bdrv_drained_end(src);
+    bdrv_unref(src);
+}
+
+static void mirror_throttle(MirrorBlockJob *s)
+{
+    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+
+    if (now - s->last_pause_ns > SLICE_TIME) {
+        s->last_pause_ns = now;
+        block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, 0);
+    } else {
+        block_job_pause_point(&s->common);
+    }
+}
+
+static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
+{
+    int64_t sector_num, end;
+    BlockDriverState *base = s->base;
+    BlockDriverState *bs = blk_bs(s->common.blk);
+    BlockDriverState *target_bs = blk_bs(s->target);
+    int ret, n;
+
+    end = s->bdev_length / BDRV_SECTOR_SIZE;
+
+    if (base == NULL && !bdrv_has_zero_init(target_bs)) {
+        if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
+            bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, end);
+            return 0;
+        }
+
+        for (sector_num = 0; sector_num < end; ) {
+            int nb_sectors = MIN(end - sector_num,
+                QEMU_ALIGN_DOWN(INT_MAX, s->granularity) >> BDRV_SECTOR_BITS);
+
+            mirror_throttle(s);
+
+            if (block_job_is_cancelled(&s->common)) {
+                return 0;
+            }
+
+            if (s->in_flight >= MAX_IN_FLIGHT) {
+                trace_mirror_yield(s, s->in_flight, s->buf_free_count, -1);
+                mirror_wait_for_io(s);
+                continue;
+            }
+
+            mirror_do_zero_or_discard(s, sector_num, nb_sectors, false);
+            sector_num += nb_sectors;
+        }
+
+        mirror_drain(s);
+    }
+
+    /* First part, loop on the sectors and initialize the dirty bitmap.  */
+    for (sector_num = 0; sector_num < end; ) {
+        /* Just to make sure we are not exceeding int limit. */
+        int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS,
+                             end - sector_num);
+
+        mirror_throttle(s);
+
+        if (block_job_is_cancelled(&s->common)) {
+            return 0;
+        }
+
+        ret = bdrv_is_allocated_above(bs, base, sector_num, nb_sectors, &n);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assert(n > 0);
+        if (ret == 1) {
+            bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n);
+        }
+        sector_num += n;
+    }
+    return 0;
 }
 
 static void coroutine_fn mirror_run(void *opaque)
 {
     MirrorBlockJob *s = opaque;
     MirrorExitData *data;
-    BlockDriverState *bs = s->common.bs;
-    int64_t sector_num, end, sectors_per_chunk, length;
-    uint64_t last_pause_ns;
+    BlockDriverState *bs = blk_bs(s->common.blk);
+    BlockDriverState *target_bs = blk_bs(s->target);
+    int64_t length;
     BlockDriverInfo bdi;
-    char backing_filename[1024];
+    char backing_filename[2]; /* we only need 2 characters because we are only
+                                 checking for a NULL string */
     int ret = 0;
-    int n;
+    int target_cluster_size = BDRV_SECTOR_SIZE;
 
     if (block_job_is_cancelled(&s->common)) {
         goto immediate_exit;
@@ -405,56 +650,39 @@
      * the destination do COW.  Instead, we copy sectors around the
      * dirty data if needed.  We need a bitmap to do that.
      */
-    bdrv_get_backing_filename(s->target, backing_filename,
+    bdrv_get_backing_filename(target_bs, backing_filename,
                               sizeof(backing_filename));
-    if (backing_filename[0] && !s->target->backing_hd) {
-        ret = bdrv_get_info(s->target, &bdi);
-        if (ret < 0) {
-            goto immediate_exit;
-        }
-        if (s->granularity < bdi.cluster_size) {
-            s->buf_size = MAX(s->buf_size, bdi.cluster_size);
-            s->cow_bitmap = bitmap_new(length);
-        }
+    if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
+        target_cluster_size = bdi.cluster_size;
     }
+    if (backing_filename[0] && !target_bs->backing
+        && s->granularity < target_cluster_size) {
+        s->buf_size = MAX(s->buf_size, target_cluster_size);
+        s->cow_bitmap = bitmap_new(length);
+    }
+    s->target_cluster_sectors = target_cluster_size >> BDRV_SECTOR_BITS;
+    s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
 
-    end = s->bdev_length / BDRV_SECTOR_SIZE;
     s->buf = qemu_try_blockalign(bs, s->buf_size);
     if (s->buf == NULL) {
         ret = -ENOMEM;
         goto immediate_exit;
     }
 
-    sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
     mirror_free_init(s);
 
+    s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
     if (!s->is_none_mode) {
-        /* First part, loop on the sectors and initialize the dirty bitmap.  */
-        BlockDriverState *base = s->base;
-        for (sector_num = 0; sector_num < end; ) {
-            int64_t next = (sector_num | (sectors_per_chunk - 1)) + 1;
-            ret = bdrv_is_allocated_above(bs, base,
-                                          sector_num, next - sector_num, &n);
-
-            if (ret < 0) {
-                goto immediate_exit;
-            }
-
-            assert(n > 0);
-            if (ret == 1) {
-                bdrv_set_dirty(bs, sector_num, n);
-                sector_num = next;
-            } else {
-                sector_num += n;
-            }
+        ret = mirror_dirty_init(s);
+        if (ret < 0 || block_job_is_cancelled(&s->common)) {
+            goto immediate_exit;
         }
     }
 
-    bdrv_dirty_iter_init(bs, s->dirty_bitmap, &s->hbi);
-    last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    bdrv_dirty_iter_init(s->dirty_bitmap, &s->hbi);
     for (;;) {
         uint64_t delay_ns = 0;
-        int64_t cnt;
+        int64_t cnt, delta;
         bool should_complete;
 
         if (s->ret < 0) {
@@ -462,7 +690,9 @@
             goto immediate_exit;
         }
 
-        cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap);
+        block_job_pause_point(&s->common);
+
+        cnt = bdrv_get_dirty_count(s->dirty_bitmap);
         /* s->common.offset contains the number of bytes already processed so
          * far, cnt is the number of dirty sectors remaining and
          * s->sectors_in_flight is the number of sectors currently being
@@ -471,29 +701,27 @@
                         (cnt + s->sectors_in_flight) * BDRV_SECTOR_SIZE;
 
         /* Note that even when no rate limit is applied we need to yield
-         * periodically with no pending I/O so that qemu_aio_flush() returns.
+         * periodically with no pending I/O so that bdrv_drain_all() returns.
          * We do so every SLICE_TIME nanoseconds, or when there is an error,
          * or when the source is clean, whichever comes first.
          */
-        if (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - last_pause_ns < SLICE_TIME &&
+        delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
+        if (delta < SLICE_TIME &&
             s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-            if (s->in_flight == MAX_IN_FLIGHT || s->buf_free_count == 0 ||
+            if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
                 (cnt == 0 && s->in_flight > 0)) {
                 trace_mirror_yield(s, s->in_flight, s->buf_free_count, cnt);
-                qemu_coroutine_yield();
+                mirror_wait_for_io(s);
                 continue;
             } else if (cnt != 0) {
                 delay_ns = mirror_iteration(s);
-                if (delay_ns == 0) {
-                    continue;
-                }
             }
         }
 
         should_complete = false;
         if (s->in_flight == 0 && cnt == 0) {
             trace_mirror_before_flush(s);
-            ret = bdrv_flush(s->target);
+            ret = blk_flush(s->target);
             if (ret < 0) {
                 if (mirror_error_action(s, false, -ret) ==
                     BLOCK_ERROR_ACTION_REPORT) {
@@ -512,7 +740,7 @@
 
                 should_complete = s->should_complete ||
                     block_job_is_cancelled(&s->common);
-                cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap);
+                cnt = bdrv_get_dirty_count(s->dirty_bitmap);
             }
         }
 
@@ -526,8 +754,8 @@
              * mirror_populate runs.
              */
             trace_mirror_before_drain(s, cnt);
-            bdrv_drain(bs);
-            cnt = bdrv_get_dirty_count(bs, s->dirty_bitmap);
+            bdrv_co_drain(bs);
+            cnt = bdrv_get_dirty_count(s->dirty_bitmap);
         }
 
         ret = 0;
@@ -548,7 +776,7 @@
             s->common.cancelled = false;
             break;
         }
-        last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+        s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
     }
 
 immediate_exit:
@@ -566,10 +794,12 @@
     g_free(s->cow_bitmap);
     g_free(s->in_flight_bitmap);
     bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
-    bdrv_iostatus_disable(s->target);
 
     data = g_malloc(sizeof(*data));
     data->ret = ret;
+    /* Before we switch to target in mirror_exit, make sure data doesn't
+     * change. */
+    bdrv_drained_begin(bs);
     block_job_defer_to_main_loop(&s->common, mirror_exit, data);
 }
 
@@ -578,43 +808,43 @@
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
 
     if (speed < 0) {
-        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        error_setg(errp, QERR_INVALID_PARAMETER, "speed");
         return;
     }
     ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
 }
 
-static void mirror_iostatus_reset(BlockJob *job)
-{
-    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
-
-    bdrv_iostatus_reset(s->target);
-}
-
 static void mirror_complete(BlockJob *job, Error **errp)
 {
     MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
-    Error *local_err = NULL;
-    int ret;
+    BlockDriverState *src, *target;
 
-    ret = bdrv_open_backing_file(s->target, NULL, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
-        return;
-    }
+    src = blk_bs(job->blk);
+    target = blk_bs(s->target);
+
     if (!s->synced) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_READY,
-                  bdrv_get_device_name(job->bs));
+        error_setg(errp, "The active block job '%s' cannot be completed",
+                   job->id);
         return;
     }
 
-    /* check the target bs is not blocked and block all operations on it */
+    if (s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
+        int ret;
+
+        assert(!target->backing);
+        ret = bdrv_open_backing_file(target, NULL, "backing", errp);
+        if (ret < 0) {
+            return;
+        }
+    }
+
+    /* block all operations on to_replace bs */
     if (s->replaces) {
         AioContext *replace_aio_context;
 
-        s->to_replace = check_to_replace_node(s->replaces, &local_err);
+        s->to_replace = bdrv_find_node(s->replaces);
         if (!s->to_replace) {
-            error_propagate(errp, local_err);
+            error_setg(errp, "Node name '%s' not found", s->replaces);
             return;
         }
 
@@ -629,33 +859,60 @@
         aio_context_release(replace_aio_context);
     }
 
+    if (s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
+        BlockDriverState *backing = s->is_none_mode ? src : s->base;
+        if (backing_bs(target) != backing) {
+            bdrv_set_backing_hd(target, backing);
+        }
+    }
+
     s->should_complete = true;
-    block_job_resume(job);
+    block_job_enter(&s->common);
+}
+
+/* There is no matching mirror_resume() because mirror_run() will begin
+ * iterating again when the job is resumed.
+ */
+static void coroutine_fn mirror_pause(BlockJob *job)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    mirror_drain(s);
+}
+
+static void mirror_attached_aio_context(BlockJob *job, AioContext *new_context)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    blk_set_aio_context(s->target, new_context);
 }
 
 static const BlockJobDriver mirror_job_driver = {
-    .instance_size = sizeof(MirrorBlockJob),
-    .job_type      = BLOCK_JOB_TYPE_MIRROR,
-    .set_speed     = mirror_set_speed,
-    .iostatus_reset= mirror_iostatus_reset,
-    .complete      = mirror_complete,
+    .instance_size          = sizeof(MirrorBlockJob),
+    .job_type               = BLOCK_JOB_TYPE_MIRROR,
+    .set_speed              = mirror_set_speed,
+    .complete               = mirror_complete,
+    .pause                  = mirror_pause,
+    .attached_aio_context   = mirror_attached_aio_context,
 };
 
 static const BlockJobDriver commit_active_job_driver = {
-    .instance_size = sizeof(MirrorBlockJob),
-    .job_type      = BLOCK_JOB_TYPE_COMMIT,
-    .set_speed     = mirror_set_speed,
-    .iostatus_reset
-                   = mirror_iostatus_reset,
-    .complete      = mirror_complete,
+    .instance_size          = sizeof(MirrorBlockJob),
+    .job_type               = BLOCK_JOB_TYPE_COMMIT,
+    .set_speed              = mirror_set_speed,
+    .complete               = mirror_complete,
+    .pause                  = mirror_pause,
+    .attached_aio_context   = mirror_attached_aio_context,
 };
 
-static void mirror_start_job(BlockDriverState *bs, BlockDriverState *target,
-                             const char *replaces,
-                             int64_t speed, int64_t granularity,
+static void mirror_start_job(const char *job_id, BlockDriverState *bs,
+                             BlockDriverState *target, const char *replaces,
+                             int64_t speed, uint32_t granularity,
                              int64_t buf_size,
+                             BlockMirrorBackingMode backing_mode,
                              BlockdevOnError on_source_error,
                              BlockdevOnError on_target_error,
+                             bool unmap,
                              BlockCompletionFunc *cb,
                              void *opaque, Error **errp,
                              const BlockJobDriver *driver,
@@ -664,74 +921,80 @@
     MirrorBlockJob *s;
 
     if (granularity == 0) {
-        /* Choose the default granularity based on the target file's cluster
-         * size, clamped between 4k and 64k.  */
-        BlockDriverInfo bdi;
-        if (bdrv_get_info(target, &bdi) >= 0 && bdi.cluster_size != 0) {
-            granularity = MAX(4096, bdi.cluster_size);
-            granularity = MIN(65536, granularity);
-        } else {
-            granularity = 65536;
-        }
+        granularity = bdrv_get_default_bitmap_granularity(target);
     }
 
     assert ((granularity & (granularity - 1)) == 0);
 
-    if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
-         on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
-        error_set(errp, QERR_INVALID_PARAMETER, "on-source-error");
+    if (buf_size < 0) {
+        error_setg(errp, "Invalid parameter 'buf-size'");
         return;
     }
 
+    if (buf_size == 0) {
+        buf_size = DEFAULT_MIRROR_BUF_SIZE;
+    }
 
-    s = block_job_create(driver, bs, speed, cb, opaque, errp);
+    s = block_job_create(job_id, driver, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
     }
 
+    s->target = blk_new();
+    blk_insert_bs(s->target, target);
+
     s->replaces = g_strdup(replaces);
     s->on_source_error = on_source_error;
     s->on_target_error = on_target_error;
-    s->target = target;
     s->is_none_mode = is_none_mode;
+    s->backing_mode = backing_mode;
     s->base = base;
     s->granularity = granularity;
-    s->buf_size = MAX(buf_size, granularity);
+    s->buf_size = ROUND_UP(buf_size, granularity);
+    s->unmap = unmap;
 
-    s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, errp);
+    s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
     if (!s->dirty_bitmap) {
+        g_free(s->replaces);
+        blk_unref(s->target);
+        block_job_unref(&s->common);
         return;
     }
-    bdrv_set_enable_write_cache(s->target, true);
-    bdrv_set_on_error(s->target, on_target_error, on_target_error);
-    bdrv_iostatus_enable(s->target);
-    s->common.co = qemu_coroutine_create(mirror_run);
+
+    bdrv_op_block_all(target, s->common.blocker);
+
+    s->common.co = qemu_coroutine_create(mirror_run, s);
     trace_mirror_start(bs, s, s->common.co, opaque);
-    qemu_coroutine_enter(s->common.co, s);
+    qemu_coroutine_enter(s->common.co);
 }
 
-void mirror_start(BlockDriverState *bs, BlockDriverState *target,
-                  const char *replaces,
-                  int64_t speed, int64_t granularity, int64_t buf_size,
-                  MirrorSyncMode mode, BlockdevOnError on_source_error,
+void mirror_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *target, const char *replaces,
+                  int64_t speed, uint32_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
+                  BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
+                  bool unmap,
                   BlockCompletionFunc *cb,
                   void *opaque, Error **errp)
 {
     bool is_none_mode;
     BlockDriverState *base;
 
+    if (mode == MIRROR_SYNC_MODE_INCREMENTAL) {
+        error_setg(errp, "Sync mode 'incremental' not supported");
+        return;
+    }
     is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
-    base = mode == MIRROR_SYNC_MODE_TOP ? bs->backing_hd : NULL;
-    mirror_start_job(bs, target, replaces,
-                     speed, granularity, buf_size,
-                     on_source_error, on_target_error, cb, opaque, errp,
+    base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
+    mirror_start_job(job_id, bs, target, replaces,
+                     speed, granularity, buf_size, backing_mode,
+                     on_source_error, on_target_error, unmap, cb, opaque, errp,
                      &mirror_job_driver, is_none_mode, base);
 }
 
-void commit_active_start(BlockDriverState *bs, BlockDriverState *base,
-                         int64_t speed,
+void commit_active_start(const char *job_id, BlockDriverState *bs,
+                         BlockDriverState *base, int64_t speed,
                          BlockdevOnError on_error,
                          BlockCompletionFunc *cb,
                          void *opaque, Error **errp)
@@ -772,9 +1035,9 @@
         }
     }
 
-    bdrv_ref(base);
-    mirror_start_job(bs, base, NULL, speed, 0, 0,
-                     on_error, on_error, cb, opaque, &local_err,
+    mirror_start_job(job_id, bs, base, NULL, speed, 0, 0,
+                     MIRROR_LEAVE_BACKING_CHAIN,
+                     on_error, on_error, false, cb, opaque, &local_err,
                      &commit_active_job_driver, false, base);
     if (local_err) {
         error_propagate(errp, local_err);
diff --git a/block/nbd-client.c b/block/nbd-client.c
index 6e1c97c..2cf3237 100644
--- a/block/nbd-client.c
+++ b/block/nbd-client.c
@@ -26,8 +26,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "nbd-client.h"
-#include "qemu/sockets.h"
 
 #define HANDLE_TO_INDEX(bs, handle) ((handle) ^ ((uint64_t)(intptr_t)bs))
 #define INDEX_TO_HANDLE(bs, index)  ((index)  ^ ((uint64_t)(intptr_t)bs))
@@ -38,34 +38,49 @@
 
     for (i = 0; i < MAX_NBD_REQUESTS; i++) {
         if (s->recv_coroutine[i]) {
-            qemu_coroutine_enter(s->recv_coroutine[i], NULL);
+            qemu_coroutine_enter(s->recv_coroutine[i]);
         }
     }
 }
 
-static void nbd_teardown_connection(NbdClientSession *client)
+static void nbd_teardown_connection(BlockDriverState *bs)
 {
+    NbdClientSession *client = nbd_get_client_session(bs);
+
+    if (!client->ioc) { /* Already closed */
+        return;
+    }
+
     /* finish any pending coroutines */
-    shutdown(client->sock, 2);
+    qio_channel_shutdown(client->ioc,
+                         QIO_CHANNEL_SHUTDOWN_BOTH,
+                         NULL);
     nbd_recv_coroutines_enter_all(client);
 
-    nbd_client_session_detach_aio_context(client);
-    closesocket(client->sock);
-    client->sock = -1;
+    nbd_client_detach_aio_context(bs);
+    object_unref(OBJECT(client->sioc));
+    client->sioc = NULL;
+    object_unref(OBJECT(client->ioc));
+    client->ioc = NULL;
 }
 
 static void nbd_reply_ready(void *opaque)
 {
-    NbdClientSession *s = opaque;
+    BlockDriverState *bs = opaque;
+    NbdClientSession *s = nbd_get_client_session(bs);
     uint64_t i;
     int ret;
 
+    if (!s->ioc) { /* Already closed */
+        return;
+    }
+
     if (s->reply.handle == 0) {
         /* No reply already in flight.  Fetch a header.  It is possible
          * that another thread has done the same thing in parallel, so
          * the socket is not readable anymore.
          */
-        ret = nbd_receive_reply(s->sock, &s->reply);
+        ret = nbd_receive_reply(s->ioc, &s->reply);
         if (ret == -EAGAIN) {
             return;
         }
@@ -84,60 +99,77 @@
     }
 
     if (s->recv_coroutine[i]) {
-        qemu_coroutine_enter(s->recv_coroutine[i], NULL);
+        qemu_coroutine_enter(s->recv_coroutine[i]);
         return;
     }
 
 fail:
-    nbd_teardown_connection(s);
+    nbd_teardown_connection(bs);
 }
 
 static void nbd_restart_write(void *opaque)
 {
-    NbdClientSession *s = opaque;
+    BlockDriverState *bs = opaque;
 
-    qemu_coroutine_enter(s->send_coroutine, NULL);
+    qemu_coroutine_enter(nbd_get_client_session(bs)->send_coroutine);
 }
 
-static int nbd_co_send_request(NbdClientSession *s,
-    struct nbd_request *request,
-    QEMUIOVector *qiov, int offset)
+static int nbd_co_send_request(BlockDriverState *bs,
+                               struct nbd_request *request,
+                               QEMUIOVector *qiov)
 {
+    NbdClientSession *s = nbd_get_client_session(bs);
     AioContext *aio_context;
-    int rc, ret;
+    int rc, ret, i;
 
     qemu_co_mutex_lock(&s->send_mutex);
-    s->send_coroutine = qemu_coroutine_self();
-    aio_context = bdrv_get_aio_context(s->bs);
-    aio_set_fd_handler(aio_context, s->sock,
-                       nbd_reply_ready, nbd_restart_write, s);
-    if (qiov) {
-        if (!s->is_unix) {
-            socket_set_cork(s->sock, 1);
+
+    for (i = 0; i < MAX_NBD_REQUESTS; i++) {
+        if (s->recv_coroutine[i] == NULL) {
+            s->recv_coroutine[i] = qemu_coroutine_self();
+            break;
         }
-        rc = nbd_send_request(s->sock, request);
+    }
+
+    g_assert(qemu_in_coroutine());
+    assert(i < MAX_NBD_REQUESTS);
+    request->handle = INDEX_TO_HANDLE(s, i);
+
+    if (!s->ioc) {
+        qemu_co_mutex_unlock(&s->send_mutex);
+        return -EPIPE;
+    }
+
+    s->send_coroutine = qemu_coroutine_self();
+    aio_context = bdrv_get_aio_context(bs);
+
+    aio_set_fd_handler(aio_context, s->sioc->fd, false,
+                       nbd_reply_ready, nbd_restart_write, bs);
+    if (qiov) {
+        qio_channel_set_cork(s->ioc, true);
+        rc = nbd_send_request(s->ioc, request);
         if (rc >= 0) {
-            ret = qemu_co_sendv(s->sock, qiov->iov, qiov->niov,
-                                offset, request->len);
+            ret = nbd_wr_syncv(s->ioc, qiov->iov, qiov->niov, request->len,
+                               false);
             if (ret != request->len) {
                 rc = -EIO;
             }
         }
-        if (!s->is_unix) {
-            socket_set_cork(s->sock, 0);
-        }
+        qio_channel_set_cork(s->ioc, false);
     } else {
-        rc = nbd_send_request(s->sock, request);
+        rc = nbd_send_request(s->ioc, request);
     }
-    aio_set_fd_handler(aio_context, s->sock, nbd_reply_ready, NULL, s);
+    aio_set_fd_handler(aio_context, s->sioc->fd, false,
+                       nbd_reply_ready, NULL, bs);
     s->send_coroutine = NULL;
     qemu_co_mutex_unlock(&s->send_mutex);
     return rc;
 }
 
 static void nbd_co_receive_reply(NbdClientSession *s,
-    struct nbd_request *request, struct nbd_reply *reply,
-    QEMUIOVector *qiov, int offset)
+                                 struct nbd_request *request,
+                                 struct nbd_reply *reply,
+                                 QEMUIOVector *qiov)
 {
     int ret;
 
@@ -145,12 +177,13 @@
      * peek at the next reply and avoid yielding if it's ours?  */
     qemu_coroutine_yield();
     *reply = s->reply;
-    if (reply->handle != request->handle) {
+    if (reply->handle != request->handle ||
+        !s->ioc) {
         reply->error = EIO;
     } else {
         if (qiov && reply->error == 0) {
-            ret = qemu_co_recvv(s->sock, qiov->iov, qiov->niov,
-                                offset, request->len);
+            ret = nbd_wr_syncv(s->ioc, qiov->iov, qiov->niov, request->len,
+                               true);
             if (ret != request->len) {
                 reply->error = EIO;
             }
@@ -164,8 +197,6 @@
 static void nbd_coroutine_start(NbdClientSession *s,
    struct nbd_request *request)
 {
-    int i;
-
     /* Poor man semaphore.  The free_sema is locked when no other request
      * can be accepted, and unlocked after receiving one reply.  */
     if (s->in_flight >= MAX_NBD_REQUESTS - 1) {
@@ -174,15 +205,7 @@
     }
     s->in_flight++;
 
-    for (i = 0; i < MAX_NBD_REQUESTS; i++) {
-        if (s->recv_coroutine[i] == NULL) {
-            s->recv_coroutine[i] = qemu_coroutine_self();
-            break;
-        }
-    }
-
-    assert(i < MAX_NBD_REQUESTS);
-    request->handle = INDEX_TO_HANDLE(s, i);
+    /* s->recv_coroutine[i] is set as soon as we get the send_lock.  */
 }
 
 static void nbd_coroutine_end(NbdClientSession *s,
@@ -195,98 +218,65 @@
     }
 }
 
-static int nbd_co_readv_1(NbdClientSession *client, int64_t sector_num,
-                          int nb_sectors, QEMUIOVector *qiov,
-                          int offset)
+int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset,
+                         uint64_t bytes, QEMUIOVector *qiov, int flags)
 {
-    struct nbd_request request = { .type = NBD_CMD_READ };
+    NbdClientSession *client = nbd_get_client_session(bs);
+    struct nbd_request request = {
+        .type = NBD_CMD_READ,
+        .from = offset,
+        .len = bytes,
+    };
     struct nbd_reply reply;
     ssize_t ret;
 
-    request.from = sector_num * 512;
-    request.len = nb_sectors * 512;
+    assert(bytes <= NBD_MAX_BUFFER_SIZE);
+    assert(!flags);
 
     nbd_coroutine_start(client, &request);
-    ret = nbd_co_send_request(client, &request, NULL, 0);
+    ret = nbd_co_send_request(bs, &request, NULL);
     if (ret < 0) {
         reply.error = -ret;
     } else {
-        nbd_co_receive_reply(client, &request, &reply, qiov, offset);
+        nbd_co_receive_reply(client, &request, &reply, qiov);
     }
     nbd_coroutine_end(client, &request);
     return -reply.error;
-
 }
 
-static int nbd_co_writev_1(NbdClientSession *client, int64_t sector_num,
-                           int nb_sectors, QEMUIOVector *qiov,
-                           int offset)
+int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                          uint64_t bytes, QEMUIOVector *qiov, int flags)
 {
-    struct nbd_request request = { .type = NBD_CMD_WRITE };
+    NbdClientSession *client = nbd_get_client_session(bs);
+    struct nbd_request request = {
+        .type = NBD_CMD_WRITE,
+        .from = offset,
+        .len = bytes,
+    };
     struct nbd_reply reply;
     ssize_t ret;
 
-    if (!bdrv_enable_write_cache(client->bs) &&
-        (client->nbdflags & NBD_FLAG_SEND_FUA)) {
+    if (flags & BDRV_REQ_FUA) {
+        assert(client->nbdflags & NBD_FLAG_SEND_FUA);
         request.type |= NBD_CMD_FLAG_FUA;
     }
 
-    request.from = sector_num * 512;
-    request.len = nb_sectors * 512;
+    assert(bytes <= NBD_MAX_BUFFER_SIZE);
 
     nbd_coroutine_start(client, &request);
-    ret = nbd_co_send_request(client, &request, qiov, offset);
+    ret = nbd_co_send_request(bs, &request, qiov);
     if (ret < 0) {
         reply.error = -ret;
     } else {
-        nbd_co_receive_reply(client, &request, &reply, NULL, 0);
+        nbd_co_receive_reply(client, &request, &reply, NULL);
     }
     nbd_coroutine_end(client, &request);
     return -reply.error;
 }
 
-/* qemu-nbd has a limit of slightly less than 1M per request.  Try to
- * remain aligned to 4K. */
-#define NBD_MAX_SECTORS 2040
-
-int nbd_client_session_co_readv(NbdClientSession *client, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov)
+int nbd_client_co_flush(BlockDriverState *bs)
 {
-    int offset = 0;
-    int ret;
-    while (nb_sectors > NBD_MAX_SECTORS) {
-        ret = nbd_co_readv_1(client, sector_num,
-                             NBD_MAX_SECTORS, qiov, offset);
-        if (ret < 0) {
-            return ret;
-        }
-        offset += NBD_MAX_SECTORS * 512;
-        sector_num += NBD_MAX_SECTORS;
-        nb_sectors -= NBD_MAX_SECTORS;
-    }
-    return nbd_co_readv_1(client, sector_num, nb_sectors, qiov, offset);
-}
-
-int nbd_client_session_co_writev(NbdClientSession *client, int64_t sector_num,
-                                 int nb_sectors, QEMUIOVector *qiov)
-{
-    int offset = 0;
-    int ret;
-    while (nb_sectors > NBD_MAX_SECTORS) {
-        ret = nbd_co_writev_1(client, sector_num,
-                              NBD_MAX_SECTORS, qiov, offset);
-        if (ret < 0) {
-            return ret;
-        }
-        offset += NBD_MAX_SECTORS * 512;
-        sector_num += NBD_MAX_SECTORS;
-        nb_sectors -= NBD_MAX_SECTORS;
-    }
-    return nbd_co_writev_1(client, sector_num, nb_sectors, qiov, offset);
-}
-
-int nbd_client_session_co_flush(NbdClientSession *client)
-{
+    NbdClientSession *client = nbd_get_client_session(bs);
     struct nbd_request request = { .type = NBD_CMD_FLUSH };
     struct nbd_reply reply;
     ssize_t ret;
@@ -295,109 +285,121 @@
         return 0;
     }
 
-    if (client->nbdflags & NBD_FLAG_SEND_FUA) {
-        request.type |= NBD_CMD_FLAG_FUA;
-    }
-
     request.from = 0;
     request.len = 0;
 
     nbd_coroutine_start(client, &request);
-    ret = nbd_co_send_request(client, &request, NULL, 0);
+    ret = nbd_co_send_request(bs, &request, NULL);
     if (ret < 0) {
         reply.error = -ret;
     } else {
-        nbd_co_receive_reply(client, &request, &reply, NULL, 0);
+        nbd_co_receive_reply(client, &request, &reply, NULL);
     }
     nbd_coroutine_end(client, &request);
     return -reply.error;
 }
 
-int nbd_client_session_co_discard(NbdClientSession *client, int64_t sector_num,
-    int nb_sectors)
+int nbd_client_co_pdiscard(BlockDriverState *bs, int64_t offset, int count)
 {
-    struct nbd_request request = { .type = NBD_CMD_TRIM };
+    NbdClientSession *client = nbd_get_client_session(bs);
+    struct nbd_request request = {
+        .type = NBD_CMD_TRIM,
+        .from = offset,
+        .len = count,
+    };
     struct nbd_reply reply;
     ssize_t ret;
 
     if (!(client->nbdflags & NBD_FLAG_SEND_TRIM)) {
         return 0;
     }
-    request.from = sector_num * 512;
-    request.len = nb_sectors * 512;
 
     nbd_coroutine_start(client, &request);
-    ret = nbd_co_send_request(client, &request, NULL, 0);
+    ret = nbd_co_send_request(bs, &request, NULL);
     if (ret < 0) {
         reply.error = -ret;
     } else {
-        nbd_co_receive_reply(client, &request, &reply, NULL, 0);
+        nbd_co_receive_reply(client, &request, &reply, NULL);
     }
     nbd_coroutine_end(client, &request);
     return -reply.error;
 
 }
 
-void nbd_client_session_detach_aio_context(NbdClientSession *client)
+void nbd_client_detach_aio_context(BlockDriverState *bs)
 {
-    aio_set_fd_handler(bdrv_get_aio_context(client->bs), client->sock,
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs),
+                       nbd_get_client_session(bs)->sioc->fd,
+                       false, NULL, NULL, NULL);
 }
 
-void nbd_client_session_attach_aio_context(NbdClientSession *client,
-                                           AioContext *new_context)
+void nbd_client_attach_aio_context(BlockDriverState *bs,
+                                   AioContext *new_context)
 {
-    aio_set_fd_handler(new_context, client->sock,
-                       nbd_reply_ready, NULL, client);
+    aio_set_fd_handler(new_context, nbd_get_client_session(bs)->sioc->fd,
+                       false, nbd_reply_ready, NULL, bs);
 }
 
-void nbd_client_session_close(NbdClientSession *client)
+void nbd_client_close(BlockDriverState *bs)
 {
+    NbdClientSession *client = nbd_get_client_session(bs);
     struct nbd_request request = {
         .type = NBD_CMD_DISC,
         .from = 0,
         .len = 0
     };
 
-    if (!client->bs) {
-        return;
-    }
-    if (client->sock == -1) {
+    if (client->ioc == NULL) {
         return;
     }
 
-    nbd_send_request(client->sock, &request);
+    nbd_send_request(client->ioc, &request);
 
-    nbd_teardown_connection(client);
-    client->bs = NULL;
+    nbd_teardown_connection(bs);
 }
 
-int nbd_client_session_init(NbdClientSession *client, BlockDriverState *bs,
-    int sock, const char *export)
+int nbd_client_init(BlockDriverState *bs,
+                    QIOChannelSocket *sioc,
+                    const char *export,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *hostname,
+                    Error **errp)
 {
+    NbdClientSession *client = nbd_get_client_session(bs);
     int ret;
 
     /* NBD handshake */
     logout("session init %s\n", export);
-    qemu_set_block(sock);
-    ret = nbd_receive_negotiate(sock, export,
-                                &client->nbdflags, &client->size,
-                                &client->blocksize);
+    qio_channel_set_blocking(QIO_CHANNEL(sioc), true, NULL);
+
+    ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), export,
+                                &client->nbdflags,
+                                tlscreds, hostname,
+                                &client->ioc,
+                                &client->size, errp);
     if (ret < 0) {
         logout("Failed to negotiate with the NBD server\n");
-        closesocket(sock);
         return ret;
     }
+    if (client->nbdflags & NBD_FLAG_SEND_FUA) {
+        bs->supported_write_flags = BDRV_REQ_FUA;
+    }
 
     qemu_co_mutex_init(&client->send_mutex);
     qemu_co_mutex_init(&client->free_sema);
-    client->bs = bs;
-    client->sock = sock;
+    client->sioc = sioc;
+    object_ref(OBJECT(client->sioc));
+
+    if (!client->ioc) {
+        client->ioc = QIO_CHANNEL(sioc);
+        object_ref(OBJECT(client->ioc));
+    }
 
     /* Now that we're connected, set the socket to be non-blocking and
      * kick the reply mechanism.  */
-    qemu_set_nonblock(sock);
-    nbd_client_session_attach_aio_context(client, bdrv_get_aio_context(bs));
+    qio_channel_set_blocking(QIO_CHANNEL(sioc), false, NULL);
+
+    nbd_client_attach_aio_context(bs, bdrv_get_aio_context(bs));
 
     logout("Established connection with NBD server\n");
     return 0;
diff --git a/block/nbd-client.h b/block/nbd-client.h
index cd478f3..044aca4 100644
--- a/block/nbd-client.h
+++ b/block/nbd-client.h
@@ -4,6 +4,7 @@
 #include "qemu-common.h"
 #include "block/nbd.h"
 #include "block/block_int.h"
+#include "io/channel-socket.h"
 
 /* #define DEBUG_NBD */
 
@@ -17,10 +18,10 @@
 #define MAX_NBD_REQUESTS    16
 
 typedef struct NbdClientSession {
-    int sock;
-    uint32_t nbdflags;
+    QIOChannelSocket *sioc; /* The master data channel */
+    QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
+    uint16_t nbdflags;
     off_t size;
-    size_t blocksize;
 
     CoMutex send_mutex;
     CoMutex free_sema;
@@ -31,24 +32,27 @@
     struct nbd_reply reply;
 
     bool is_unix;
-
-    BlockDriverState *bs;
 } NbdClientSession;
 
-int nbd_client_session_init(NbdClientSession *client, BlockDriverState *bs,
-                            int sock, const char *export_name);
-void nbd_client_session_close(NbdClientSession *client);
+NbdClientSession *nbd_get_client_session(BlockDriverState *bs);
 
-int nbd_client_session_co_discard(NbdClientSession *client, int64_t sector_num,
-                                  int nb_sectors);
-int nbd_client_session_co_flush(NbdClientSession *client);
-int nbd_client_session_co_writev(NbdClientSession *client, int64_t sector_num,
-                                 int nb_sectors, QEMUIOVector *qiov);
-int nbd_client_session_co_readv(NbdClientSession *client, int64_t sector_num,
-                                int nb_sectors, QEMUIOVector *qiov);
+int nbd_client_init(BlockDriverState *bs,
+                    QIOChannelSocket *sock,
+                    const char *export_name,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *hostname,
+                    Error **errp);
+void nbd_client_close(BlockDriverState *bs);
 
-void nbd_client_session_detach_aio_context(NbdClientSession *client);
-void nbd_client_session_attach_aio_context(NbdClientSession *client,
-                                           AioContext *new_context);
+int nbd_client_co_pdiscard(BlockDriverState *bs, int64_t offset, int count);
+int nbd_client_co_flush(BlockDriverState *bs);
+int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                          uint64_t bytes, QEMUIOVector *qiov, int flags);
+int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset,
+                         uint64_t bytes, QEMUIOVector *qiov, int flags);
+
+void nbd_client_detach_aio_context(BlockDriverState *bs);
+void nbd_client_attach_aio_context(BlockDriverState *bs,
+                                   AioContext *new_context);
 
 #endif /* NBD_CLIENT_H */
diff --git a/block/nbd.c b/block/nbd.c
index 04cc845..6bc06d6 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -26,24 +26,25 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/nbd-client.h"
+#include "qapi/error.h"
 #include "qemu/uri.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
-#include "qemu/sockets.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qstring.h"
-
-#include <sys/types.h>
-#include <unistd.h>
+#include "qemu/cutils.h"
 
 #define EN_OPTSTR ":exportname="
 
 typedef struct BDRVNBDState {
     NbdClientSession client;
-    QemuOpts *socket_opts;
+
+    /* For nbd_refresh_filename() */
+    char *path, *host, *port, *export, *tlscredsid;
 } BDRVNBDState;
 
 static int nbd_parse_uri(const char *filename, QDict *options)
@@ -190,132 +191,232 @@
     g_free(file);
 }
 
-static void nbd_config(BDRVNBDState *s, QDict *options, char **export,
-                       Error **errp)
+static SocketAddress *nbd_config(BDRVNBDState *s, QemuOpts *opts, Error **errp)
 {
-    Error *local_err = NULL;
+    SocketAddress *saddr;
 
-    if (qdict_haskey(options, "path") == qdict_haskey(options, "host")) {
-        if (qdict_haskey(options, "path")) {
+    s->path = g_strdup(qemu_opt_get(opts, "path"));
+    s->host = g_strdup(qemu_opt_get(opts, "host"));
+
+    if (!s->path == !s->host) {
+        if (s->path) {
             error_setg(errp, "path and host may not be used at the same time.");
         } else {
             error_setg(errp, "one of path and host must be specified.");
         }
-        return;
+        return NULL;
     }
 
-    s->client.is_unix = qdict_haskey(options, "path");
-    s->socket_opts = qemu_opts_create(&socket_optslist, NULL, 0,
-                                      &error_abort);
+    saddr = g_new0(SocketAddress, 1);
 
-    qemu_opts_absorb_qdict(s->socket_opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    if (!qemu_opt_get(s->socket_opts, "port")) {
-        qemu_opt_set_number(s->socket_opts, "port", NBD_DEFAULT_PORT);
-    }
-
-    *export = g_strdup(qdict_get_try_str(options, "export"));
-    if (*export) {
-        qdict_del(options, "export");
-    }
-}
-
-static int nbd_establish_connection(BlockDriverState *bs, Error **errp)
-{
-    BDRVNBDState *s = bs->opaque;
-    int sock;
-
-    if (s->client.is_unix) {
-        sock = unix_connect_opts(s->socket_opts, errp, NULL, NULL);
+    if (s->path) {
+        UnixSocketAddress *q_unix;
+        saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+        q_unix = saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+        q_unix->path = g_strdup(s->path);
     } else {
-        sock = inet_connect_opts(s->socket_opts, errp, NULL, NULL);
-        if (sock >= 0) {
-            socket_set_nodelay(sock);
+        InetSocketAddress *inet;
+
+        s->port = g_strdup(qemu_opt_get(opts, "port"));
+
+        saddr->type = SOCKET_ADDRESS_KIND_INET;
+        inet = saddr->u.inet.data = g_new0(InetSocketAddress, 1);
+        inet->host = g_strdup(s->host);
+        inet->port = g_strdup(s->port);
+        if (!inet->port) {
+            inet->port = g_strdup_printf("%d", NBD_DEFAULT_PORT);
         }
     }
 
-    /* Failed to establish connection */
-    if (sock < 0) {
-        logout("Failed to establish connection to NBD server\n");
-        return -errno;
+    s->client.is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
+
+    s->export = g_strdup(qemu_opt_get(opts, "export"));
+
+    return saddr;
+}
+
+NbdClientSession *nbd_get_client_session(BlockDriverState *bs)
+{
+    BDRVNBDState *s = bs->opaque;
+    return &s->client;
+}
+
+static QIOChannelSocket *nbd_establish_connection(SocketAddress *saddr,
+                                                  Error **errp)
+{
+    QIOChannelSocket *sioc;
+    Error *local_err = NULL;
+
+    sioc = qio_channel_socket_new();
+
+    qio_channel_socket_connect_sync(sioc,
+                                    saddr,
+                                    &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return NULL;
     }
 
-    return sock;
+    qio_channel_set_delay(QIO_CHANNEL(sioc), false);
+
+    return sioc;
 }
 
+
+static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp)
+{
+    Object *obj;
+    QCryptoTLSCreds *creds;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), id);
+    if (!obj) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   id);
+        return NULL;
+    }
+    creds = (QCryptoTLSCreds *)
+        object_dynamic_cast(obj, TYPE_QCRYPTO_TLS_CREDS);
+    if (!creds) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   id);
+        return NULL;
+    }
+
+    if (creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+        error_setg(errp,
+                   "Expecting TLS credentials with a client endpoint");
+        return NULL;
+    }
+    object_ref(obj);
+    return creds;
+}
+
+
+static QemuOptsList nbd_runtime_opts = {
+    .name = "nbd",
+    .head = QTAILQ_HEAD_INITIALIZER(nbd_runtime_opts.head),
+    .desc = {
+        {
+            .name = "host",
+            .type = QEMU_OPT_STRING,
+            .help = "TCP host to connect to",
+        },
+        {
+            .name = "port",
+            .type = QEMU_OPT_STRING,
+            .help = "TCP port to connect to",
+        },
+        {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+            .help = "Unix socket path to connect to",
+        },
+        {
+            .name = "export",
+            .type = QEMU_OPT_STRING,
+            .help = "Name of the NBD export to open",
+        },
+        {
+            .name = "tls-creds",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of the TLS credentials to use",
+        },
+    },
+};
+
 static int nbd_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
     BDRVNBDState *s = bs->opaque;
-    char *export = NULL;
-    int result, sock;
+    QemuOpts *opts = NULL;
     Error *local_err = NULL;
+    QIOChannelSocket *sioc = NULL;
+    SocketAddress *saddr = NULL;
+    QCryptoTLSCreds *tlscreds = NULL;
+    const char *hostname = NULL;
+    int ret = -EINVAL;
 
-    /* Pop the config into our state object. Exit if invalid. */
-    nbd_config(s, options, &export, &local_err);
+    opts = qemu_opts_create(&nbd_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return -EINVAL;
+        goto error;
+    }
+
+    /* Pop the config into our state object. Exit if invalid. */
+    saddr = nbd_config(s, opts, errp);
+    if (!saddr) {
+        goto error;
+    }
+
+    s->tlscredsid = g_strdup(qemu_opt_get(opts, "tls-creds"));
+    if (s->tlscredsid) {
+        tlscreds = nbd_get_tls_creds(s->tlscredsid, errp);
+        if (!tlscreds) {
+            goto error;
+        }
+
+        if (saddr->type != SOCKET_ADDRESS_KIND_INET) {
+            error_setg(errp, "TLS only supported over IP sockets");
+            goto error;
+        }
+        hostname = saddr->u.inet.data->host;
     }
 
     /* establish TCP connection, return error if it fails
      * TODO: Configurable retry-until-timeout behaviour.
      */
-    sock = nbd_establish_connection(bs, errp);
-    if (sock < 0) {
-        return sock;
+    sioc = nbd_establish_connection(saddr, errp);
+    if (!sioc) {
+        ret = -ECONNREFUSED;
+        goto error;
     }
 
     /* NBD handshake */
-    result = nbd_client_session_init(&s->client, bs, sock, export);
-    g_free(export);
-    return result;
-}
-
-static int nbd_co_readv(BlockDriverState *bs, int64_t sector_num,
-                        int nb_sectors, QEMUIOVector *qiov)
-{
-    BDRVNBDState *s = bs->opaque;
-
-    return nbd_client_session_co_readv(&s->client, sector_num,
-                                       nb_sectors, qiov);
-}
-
-static int nbd_co_writev(BlockDriverState *bs, int64_t sector_num,
-                         int nb_sectors, QEMUIOVector *qiov)
-{
-    BDRVNBDState *s = bs->opaque;
-
-    return nbd_client_session_co_writev(&s->client, sector_num,
-                                        nb_sectors, qiov);
+    ret = nbd_client_init(bs, sioc, s->export,
+                          tlscreds, hostname, errp);
+ error:
+    if (sioc) {
+        object_unref(OBJECT(sioc));
+    }
+    if (tlscreds) {
+        object_unref(OBJECT(tlscreds));
+    }
+    if (ret < 0) {
+        g_free(s->path);
+        g_free(s->host);
+        g_free(s->port);
+        g_free(s->export);
+        g_free(s->tlscredsid);
+    }
+    qapi_free_SocketAddress(saddr);
+    qemu_opts_del(opts);
+    return ret;
 }
 
 static int nbd_co_flush(BlockDriverState *bs)
 {
-    BDRVNBDState *s = bs->opaque;
-
-    return nbd_client_session_co_flush(&s->client);
+    return nbd_client_co_flush(bs);
 }
 
-static int nbd_co_discard(BlockDriverState *bs, int64_t sector_num,
-                          int nb_sectors)
+static void nbd_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    BDRVNBDState *s = bs->opaque;
-
-    return nbd_client_session_co_discard(&s->client, sector_num,
-                                         nb_sectors);
+    bs->bl.max_pdiscard = NBD_MAX_BUFFER_SIZE;
+    bs->bl.max_transfer = NBD_MAX_BUFFER_SIZE;
 }
 
 static void nbd_close(BlockDriverState *bs)
 {
     BDRVNBDState *s = bs->opaque;
 
-    qemu_opts_del(s->socket_opts);
-    nbd_client_session_close(&s->client);
+    nbd_client_close(bs);
+
+    g_free(s->path);
+    g_free(s->host);
+    g_free(s->port);
+    g_free(s->export);
+    g_free(s->tlscredsid);
 }
 
 static int64_t nbd_getlength(BlockDriverState *bs)
@@ -327,59 +428,56 @@
 
 static void nbd_detach_aio_context(BlockDriverState *bs)
 {
-    BDRVNBDState *s = bs->opaque;
-
-    nbd_client_session_detach_aio_context(&s->client);
+    nbd_client_detach_aio_context(bs);
 }
 
 static void nbd_attach_aio_context(BlockDriverState *bs,
                                    AioContext *new_context)
 {
-    BDRVNBDState *s = bs->opaque;
-
-    nbd_client_session_attach_aio_context(&s->client, new_context);
+    nbd_client_attach_aio_context(bs, new_context);
 }
 
-static void nbd_refresh_filename(BlockDriverState *bs)
+static void nbd_refresh_filename(BlockDriverState *bs, QDict *options)
 {
+    BDRVNBDState *s = bs->opaque;
     QDict *opts = qdict_new();
-    const char *path   = qdict_get_try_str(bs->options, "path");
-    const char *host   = qdict_get_try_str(bs->options, "host");
-    const char *port   = qdict_get_try_str(bs->options, "port");
-    const char *export = qdict_get_try_str(bs->options, "export");
 
     qdict_put_obj(opts, "driver", QOBJECT(qstring_from_str("nbd")));
 
-    if (path && export) {
+    if (s->path && s->export) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
-                 "nbd+unix:///%s?socket=%s", export, path);
-    } else if (path && !export) {
+                 "nbd+unix:///%s?socket=%s", s->export, s->path);
+    } else if (s->path && !s->export) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
-                 "nbd+unix://?socket=%s", path);
-    } else if (!path && export && port) {
+                 "nbd+unix://?socket=%s", s->path);
+    } else if (!s->path && s->export && s->port) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
-                 "nbd://%s:%s/%s", host, port, export);
-    } else if (!path && export && !port) {
+                 "nbd://%s:%s/%s", s->host, s->port, s->export);
+    } else if (!s->path && s->export && !s->port) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
-                 "nbd://%s/%s", host, export);
-    } else if (!path && !export && port) {
+                 "nbd://%s/%s", s->host, s->export);
+    } else if (!s->path && !s->export && s->port) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
-                 "nbd://%s:%s", host, port);
-    } else if (!path && !export && !port) {
+                 "nbd://%s:%s", s->host, s->port);
+    } else if (!s->path && !s->export && !s->port) {
         snprintf(bs->exact_filename, sizeof(bs->exact_filename),
-                 "nbd://%s", host);
+                 "nbd://%s", s->host);
     }
 
-    if (path) {
-        qdict_put_obj(opts, "path", QOBJECT(qstring_from_str(path)));
-    } else if (port) {
-        qdict_put_obj(opts, "host", QOBJECT(qstring_from_str(host)));
-        qdict_put_obj(opts, "port", QOBJECT(qstring_from_str(port)));
+    if (s->path) {
+        qdict_put_obj(opts, "path", QOBJECT(qstring_from_str(s->path)));
+    } else if (s->port) {
+        qdict_put_obj(opts, "host", QOBJECT(qstring_from_str(s->host)));
+        qdict_put_obj(opts, "port", QOBJECT(qstring_from_str(s->port)));
     } else {
-        qdict_put_obj(opts, "host", QOBJECT(qstring_from_str(host)));
+        qdict_put_obj(opts, "host", QOBJECT(qstring_from_str(s->host)));
     }
-    if (export) {
-        qdict_put_obj(opts, "export", QOBJECT(qstring_from_str(export)));
+    if (s->export) {
+        qdict_put_obj(opts, "export", QOBJECT(qstring_from_str(s->export)));
+    }
+    if (s->tlscredsid) {
+        qdict_put_obj(opts, "tls-creds",
+                      QOBJECT(qstring_from_str(s->tlscredsid)));
     }
 
     bs->full_open_options = opts;
@@ -391,11 +489,12 @@
     .instance_size              = sizeof(BDRVNBDState),
     .bdrv_parse_filename        = nbd_parse_filename,
     .bdrv_file_open             = nbd_open,
-    .bdrv_co_readv              = nbd_co_readv,
-    .bdrv_co_writev             = nbd_co_writev,
+    .bdrv_co_preadv             = nbd_client_co_preadv,
+    .bdrv_co_pwritev            = nbd_client_co_pwritev,
     .bdrv_close                 = nbd_close,
     .bdrv_co_flush_to_os        = nbd_co_flush,
-    .bdrv_co_discard            = nbd_co_discard,
+    .bdrv_co_pdiscard           = nbd_client_co_pdiscard,
+    .bdrv_refresh_limits        = nbd_refresh_limits,
     .bdrv_getlength             = nbd_getlength,
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
@@ -408,11 +507,12 @@
     .instance_size              = sizeof(BDRVNBDState),
     .bdrv_parse_filename        = nbd_parse_filename,
     .bdrv_file_open             = nbd_open,
-    .bdrv_co_readv              = nbd_co_readv,
-    .bdrv_co_writev             = nbd_co_writev,
+    .bdrv_co_preadv             = nbd_client_co_preadv,
+    .bdrv_co_pwritev            = nbd_client_co_pwritev,
     .bdrv_close                 = nbd_close,
     .bdrv_co_flush_to_os        = nbd_co_flush,
-    .bdrv_co_discard            = nbd_co_discard,
+    .bdrv_co_pdiscard           = nbd_client_co_pdiscard,
+    .bdrv_refresh_limits        = nbd_refresh_limits,
     .bdrv_getlength             = nbd_getlength,
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
@@ -425,11 +525,12 @@
     .instance_size              = sizeof(BDRVNBDState),
     .bdrv_parse_filename        = nbd_parse_filename,
     .bdrv_file_open             = nbd_open,
-    .bdrv_co_readv              = nbd_co_readv,
-    .bdrv_co_writev             = nbd_co_writev,
+    .bdrv_co_preadv             = nbd_client_co_preadv,
+    .bdrv_co_pwritev            = nbd_client_co_pwritev,
     .bdrv_close                 = nbd_close,
     .bdrv_co_flush_to_os        = nbd_co_flush,
-    .bdrv_co_discard            = nbd_co_discard,
+    .bdrv_co_pdiscard           = nbd_client_co_pdiscard,
+    .bdrv_refresh_limits        = nbd_refresh_limits,
     .bdrv_getlength             = nbd_getlength,
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
diff --git a/block/nfs.c b/block/nfs.c
index c76e368..8602a44 100644
--- a/block/nfs.c
+++ b/block/nfs.c
@@ -1,7 +1,7 @@
 /*
  * QEMU Block driver for native access to files on NFS shares
  *
- * Copyright (c) 2014 Peter Lieven <pl@kamp.de>
+ * Copyright (c) 2014-2016 Peter Lieven <pl@kamp.de>
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,25 +22,33 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include <poll.h>
 #include "qemu-common.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 #include "trace.h"
 #include "qemu/iov.h"
 #include "qemu/uri.h"
+#include "qemu/cutils.h"
 #include "sysemu/sysemu.h"
 #include <nfsc/libnfs.h>
 
+#define QEMU_NFS_MAX_READAHEAD_SIZE 1048576
+#define QEMU_NFS_MAX_PAGECACHE_SIZE (8388608 / NFS_BLKSIZE)
+#define QEMU_NFS_MAX_DEBUG_LEVEL 2
+
 typedef struct NFSClient {
     struct nfs_context *context;
     struct nfsfh *fh;
     int events;
     bool has_zero_init;
     AioContext *aio_context;
+    blkcnt_t st_blocks;
+    bool cache_used;
 } NFSClient;
 
 typedef struct NFSRPC {
@@ -60,11 +68,10 @@
 {
     int ev = nfs_which_events(client->context);
     if (ev != client->events) {
-        aio_set_fd_handler(client->aio_context,
-                           nfs_get_fd(client->context),
+        aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                           false,
                            (ev & POLLIN) ? nfs_process_read : NULL,
-                           (ev & POLLOUT) ? nfs_process_write : NULL,
-                           client);
+                           (ev & POLLOUT) ? nfs_process_write : NULL, client);
 
     }
     client->events = ev;
@@ -97,7 +104,7 @@
     NFSRPC *task = opaque;
     task->complete = 1;
     qemu_bh_delete(task->bh);
-    qemu_coroutine_enter(task->co, NULL);
+    qemu_coroutine_enter(task->co);
 }
 
 static void
@@ -239,9 +246,8 @@
 {
     NFSClient *client = bs->opaque;
 
-    aio_set_fd_handler(client->aio_context,
-                       nfs_get_fd(client->context),
-                       NULL, NULL, NULL);
+    aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                       false, NULL, NULL, NULL);
     client->events = 0;
 }
 
@@ -260,9 +266,8 @@
         if (client->fh) {
             nfs_close(client->context, client->fh);
         }
-        aio_set_fd_handler(client->aio_context,
-                           nfs_get_fd(client->context),
-                           NULL, NULL, NULL);
+        aio_set_fd_handler(client->aio_context, nfs_get_fd(client->context),
+                           false, NULL, NULL, NULL);
         nfs_destroy_context(client->context);
     }
     memset(client, 0, sizeof(NFSClient));
@@ -275,7 +280,7 @@
 }
 
 static int64_t nfs_client_open(NFSClient *client, const char *filename,
-                               int flags, Error **errp)
+                               int flags, Error **errp, int open_flags)
 {
     int ret = -EINVAL, i;
     struct stat st;
@@ -327,7 +332,49 @@
             nfs_set_tcp_syncnt(client->context, val);
 #ifdef LIBNFS_FEATURE_READAHEAD
         } else if (!strcmp(qp->p[i].name, "readahead")) {
+            if (open_flags & BDRV_O_NOCACHE) {
+                error_setg(errp, "Cannot enable NFS readahead "
+                                 "if cache.direct = on");
+                goto fail;
+            }
+            if (val > QEMU_NFS_MAX_READAHEAD_SIZE) {
+                error_report("NFS Warning: Truncating NFS readahead"
+                             " size to %d", QEMU_NFS_MAX_READAHEAD_SIZE);
+                val = QEMU_NFS_MAX_READAHEAD_SIZE;
+            }
             nfs_set_readahead(client->context, val);
+#ifdef LIBNFS_FEATURE_PAGECACHE
+            nfs_set_pagecache_ttl(client->context, 0);
+#endif
+            client->cache_used = true;
+#endif
+#ifdef LIBNFS_FEATURE_PAGECACHE
+            nfs_set_pagecache_ttl(client->context, 0);
+        } else if (!strcmp(qp->p[i].name, "pagecache")) {
+            if (open_flags & BDRV_O_NOCACHE) {
+                error_setg(errp, "Cannot enable NFS pagecache "
+                                 "if cache.direct = on");
+                goto fail;
+            }
+            if (val > QEMU_NFS_MAX_PAGECACHE_SIZE) {
+                error_report("NFS Warning: Truncating NFS pagecache"
+                             " size to %d pages", QEMU_NFS_MAX_PAGECACHE_SIZE);
+                val = QEMU_NFS_MAX_PAGECACHE_SIZE;
+            }
+            nfs_set_pagecache(client->context, val);
+            nfs_set_pagecache_ttl(client->context, 0);
+            client->cache_used = true;
+#endif
+#ifdef LIBNFS_FEATURE_DEBUG
+        } else if (!strcmp(qp->p[i].name, "debug")) {
+            /* limit the maximum debug level to avoid potential flooding
+             * of our log files. */
+            if (val > QEMU_NFS_MAX_DEBUG_LEVEL) {
+                error_report("NFS Warning: Limiting NFS debug level"
+                             " to %d", QEMU_NFS_MAX_DEBUG_LEVEL);
+                val = QEMU_NFS_MAX_DEBUG_LEVEL;
+            }
+            nfs_set_debug(client->context, val);
 #endif
         } else {
             error_setg(errp, "Unknown NFS parameter name: %s",
@@ -367,6 +414,7 @@
     }
 
     ret = DIV_ROUND_UP(st.st_size, BDRV_SECTOR_SIZE);
+    client->st_blocks = st.st_blocks;
     client->has_zero_init = S_ISREG(st.st_mode);
     goto out;
 fail:
@@ -398,7 +446,7 @@
     }
     ret = nfs_client_open(client, qemu_opt_get(opts, "filename"),
                           (flags & BDRV_O_RDWR) ? O_RDWR : O_RDONLY,
-                          errp);
+                          errp, bs->open_flags);
     if (ret < 0) {
         goto out;
     }
@@ -409,6 +457,19 @@
     return ret;
 }
 
+static QemuOptsList nfs_create_opts = {
+    .name = "nfs-create-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(nfs_create_opts.head),
+    .desc = {
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Virtual disk size"
+        },
+        { /* end of list */ }
+    }
+};
+
 static int nfs_file_create(const char *url, QemuOpts *opts, Error **errp)
 {
     int ret = 0;
@@ -421,7 +482,7 @@
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
                           BDRV_SECTOR_SIZE);
 
-    ret = nfs_client_open(client, url, O_CREAT, errp);
+    ret = nfs_client_open(client, url, O_CREAT, errp, 0);
     if (ret < 0) {
         goto out;
     }
@@ -444,6 +505,11 @@
     NFSRPC task = {0};
     struct stat st;
 
+    if (bdrv_is_read_only(bs) &&
+        !(bs->open_flags & BDRV_O_NOCACHE)) {
+        return client->st_blocks * 512;
+    }
+
     task.st = &st;
     if (nfs_fstat_async(client->context, client->fh, nfs_co_generic_cb,
                         &task) != 0) {
@@ -455,7 +521,7 @@
         aio_poll(client->aio_context, true);
     }
 
-    return (task.ret < 0 ? task.ret : st.st_blocks * st.st_blksize);
+    return (task.ret < 0 ? task.ret : st.st_blocks * 512);
 }
 
 static int nfs_file_truncate(BlockDriverState *bs, int64_t offset)
@@ -464,12 +530,57 @@
     return nfs_ftruncate(client->context, client->fh, offset);
 }
 
+/* Note that this will not re-establish a connection with the NFS server
+ * - it is effectively a NOP.  */
+static int nfs_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    NFSClient *client = state->bs->opaque;
+    struct stat st;
+    int ret = 0;
+
+    if (state->flags & BDRV_O_RDWR && bdrv_is_read_only(state->bs)) {
+        error_setg(errp, "Cannot open a read-only mount as read-write");
+        return -EACCES;
+    }
+
+    if ((state->flags & BDRV_O_NOCACHE) && client->cache_used) {
+        error_setg(errp, "Cannot disable cache if libnfs readahead or"
+                         " pagecache is enabled");
+        return -EINVAL;
+    }
+
+    /* Update cache for read-only reopens */
+    if (!(state->flags & BDRV_O_RDWR)) {
+        ret = nfs_fstat(client->context, client->fh, &st);
+        if (ret < 0) {
+            error_setg(errp, "Failed to fstat file: %s",
+                       nfs_get_error(client->context));
+            return ret;
+        }
+        client->st_blocks = st.st_blocks;
+    }
+
+    return 0;
+}
+
+#ifdef LIBNFS_FEATURE_PAGECACHE
+static void nfs_invalidate_cache(BlockDriverState *bs,
+                                 Error **errp)
+{
+    NFSClient *client = bs->opaque;
+    nfs_pagecache_invalidate(client->context, client->fh);
+}
+#endif
+
 static BlockDriver bdrv_nfs = {
     .format_name                    = "nfs",
     .protocol_name                  = "nfs",
 
     .instance_size                  = sizeof(NFSClient),
     .bdrv_needs_filename            = true,
+    .create_opts                    = &nfs_create_opts,
+
     .bdrv_has_zero_init             = nfs_has_zero_init,
     .bdrv_get_allocated_file_size   = nfs_get_allocated_file_size,
     .bdrv_truncate                  = nfs_file_truncate,
@@ -477,6 +588,7 @@
     .bdrv_file_open                 = nfs_file_open,
     .bdrv_close                     = nfs_file_close,
     .bdrv_create                    = nfs_file_create,
+    .bdrv_reopen_prepare            = nfs_reopen_prepare,
 
     .bdrv_co_readv                  = nfs_co_readv,
     .bdrv_co_writev                 = nfs_co_writev,
@@ -484,6 +596,10 @@
 
     .bdrv_detach_aio_context        = nfs_detach_aio_context,
     .bdrv_attach_aio_context        = nfs_attach_aio_context,
+
+#ifdef LIBNFS_FEATURE_PAGECACHE
+    .bdrv_invalidate_cache          = nfs_invalidate_cache,
+#endif
 };
 
 static void nfs_block_init(void)
diff --git a/block/null.c b/block/null.c
index ec2bd27..b511010 100644
--- a/block/null.c
+++ b/block/null.c
@@ -10,10 +10,19 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
 #include "block/block_int.h"
 
+#define NULL_OPT_LATENCY "latency-ns"
+#define NULL_OPT_ZEROES  "read-zeroes"
+
 typedef struct {
     int64_t length;
+    int64_t latency_ns;
+    bool read_zeroes;
 } BDRVNullState;
 
 static QemuOptsList runtime_opts = {
@@ -30,6 +39,17 @@
             .type = QEMU_OPT_SIZE,
             .help = "size of the null block",
         },
+        {
+            .name = NULL_OPT_LATENCY,
+            .type = QEMU_OPT_NUMBER,
+            .help = "nanoseconds (approximated) to wait "
+                    "before completing request",
+        },
+        {
+            .name = NULL_OPT_ZEROES,
+            .type = QEMU_OPT_BOOL,
+            .help = "return zeroes when read",
+        },
         { /* end of list */ }
     },
 };
@@ -39,13 +59,21 @@
 {
     QemuOpts *opts;
     BDRVNullState *s = bs->opaque;
+    int ret = 0;
 
     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
     qemu_opts_absorb_qdict(opts, options, &error_abort);
     s->length =
         qemu_opt_get_size(opts, BLOCK_OPT_SIZE, 1 << 30);
+    s->latency_ns =
+        qemu_opt_get_number(opts, NULL_OPT_LATENCY, 0);
+    if (s->latency_ns < 0) {
+        error_setg(errp, "latency-ns is invalid");
+        ret = -EINVAL;
+    }
+    s->read_zeroes = qemu_opt_get_bool(opts, NULL_OPT_ZEROES, false);
     qemu_opts_del(opts);
-    return 0;
+    return ret;
 }
 
 static void null_close(BlockDriverState *bs)
@@ -58,28 +86,46 @@
     return s->length;
 }
 
+static coroutine_fn int null_co_common(BlockDriverState *bs)
+{
+    BDRVNullState *s = bs->opaque;
+
+    if (s->latency_ns) {
+        co_aio_sleep_ns(bdrv_get_aio_context(bs), QEMU_CLOCK_REALTIME,
+                        s->latency_ns);
+    }
+    return 0;
+}
+
 static coroutine_fn int null_co_readv(BlockDriverState *bs,
                                       int64_t sector_num, int nb_sectors,
                                       QEMUIOVector *qiov)
 {
-    return 0;
+    BDRVNullState *s = bs->opaque;
+
+    if (s->read_zeroes) {
+        qemu_iovec_memset(qiov, 0, 0, nb_sectors * BDRV_SECTOR_SIZE);
+    }
+
+    return null_co_common(bs);
 }
 
 static coroutine_fn int null_co_writev(BlockDriverState *bs,
                                        int64_t sector_num, int nb_sectors,
                                        QEMUIOVector *qiov)
 {
-    return 0;
+    return null_co_common(bs);
 }
 
 static coroutine_fn int null_co_flush(BlockDriverState *bs)
 {
-    return 0;
+    return null_co_common(bs);
 }
 
 typedef struct {
     BlockAIOCB common;
     QEMUBH *bh;
+    QEMUTimer timer;
 } NullAIOCB;
 
 static const AIOCBInfo null_aiocb_info = {
@@ -94,15 +140,33 @@
     qemu_aio_unref(acb);
 }
 
+static void null_timer_cb(void *opaque)
+{
+    NullAIOCB *acb = opaque;
+    acb->common.cb(acb->common.opaque, 0);
+    timer_deinit(&acb->timer);
+    qemu_aio_unref(acb);
+}
+
 static inline BlockAIOCB *null_aio_common(BlockDriverState *bs,
                                           BlockCompletionFunc *cb,
                                           void *opaque)
 {
     NullAIOCB *acb;
+    BDRVNullState *s = bs->opaque;
 
     acb = qemu_aio_get(&null_aiocb_info, bs, cb, opaque);
-    acb->bh = aio_bh_new(bdrv_get_aio_context(bs), null_bh_cb, acb);
-    qemu_bh_schedule(acb->bh);
+    /* Only emulate latency after vcpu is running. */
+    if (s->latency_ns) {
+        aio_timer_init(bdrv_get_aio_context(bs), &acb->timer,
+                       QEMU_CLOCK_REALTIME, SCALE_NS,
+                       null_timer_cb, acb);
+        timer_mod_ns(&acb->timer,
+                     qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + s->latency_ns);
+    } else {
+        acb->bh = aio_bh_new(bdrv_get_aio_context(bs), null_bh_cb, acb);
+        qemu_bh_schedule(acb->bh);
+    }
     return &acb->common;
 }
 
@@ -112,6 +176,12 @@
                                   BlockCompletionFunc *cb,
                                   void *opaque)
 {
+    BDRVNullState *s = bs->opaque;
+
+    if (s->read_zeroes) {
+        qemu_iovec_memset(qiov, 0, 0, nb_sectors * BDRV_SECTOR_SIZE);
+    }
+
     return null_aio_common(bs, cb, opaque);
 }
 
@@ -131,6 +201,44 @@
     return null_aio_common(bs, cb, opaque);
 }
 
+static int null_reopen_prepare(BDRVReopenState *reopen_state,
+                               BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
+static int64_t coroutine_fn null_co_get_block_status(BlockDriverState *bs,
+                                                     int64_t sector_num,
+                                                     int nb_sectors, int *pnum,
+                                                     BlockDriverState **file)
+{
+    BDRVNullState *s = bs->opaque;
+    off_t start = sector_num * BDRV_SECTOR_SIZE;
+
+    *pnum = nb_sectors;
+    *file = bs;
+
+    if (s->read_zeroes) {
+        return BDRV_BLOCK_OFFSET_VALID | start | BDRV_BLOCK_ZERO;
+    } else {
+        return BDRV_BLOCK_OFFSET_VALID | start;
+    }
+}
+
+static void null_refresh_filename(BlockDriverState *bs, QDict *opts)
+{
+    QINCREF(opts);
+    qdict_del(opts, "filename");
+
+    if (!qdict_size(opts)) {
+        snprintf(bs->exact_filename, sizeof(bs->exact_filename), "%s://",
+                 bs->drv->format_name);
+    }
+
+    qdict_put(opts, "driver", qstring_from_str(bs->drv->format_name));
+    bs->full_open_options = opts;
+}
+
 static BlockDriver bdrv_null_co = {
     .format_name            = "null-co",
     .protocol_name          = "null-co",
@@ -143,6 +251,11 @@
     .bdrv_co_readv          = null_co_readv,
     .bdrv_co_writev         = null_co_writev,
     .bdrv_co_flush_to_disk  = null_co_flush,
+    .bdrv_reopen_prepare    = null_reopen_prepare,
+
+    .bdrv_co_get_block_status   = null_co_get_block_status,
+
+    .bdrv_refresh_filename  = null_refresh_filename,
 };
 
 static BlockDriver bdrv_null_aio = {
@@ -157,6 +270,11 @@
     .bdrv_aio_readv         = null_aio_readv,
     .bdrv_aio_writev        = null_aio_writev,
     .bdrv_aio_flush         = null_aio_flush,
+    .bdrv_reopen_prepare    = null_reopen_prepare,
+
+    .bdrv_co_get_block_status   = null_co_get_block_status,
+
+    .bdrv_refresh_filename  = null_refresh_filename,
 };
 
 static void bdrv_null_init(void)
diff --git a/block/parallels.c b/block/parallels.c
index 4f9cd8d..2ccefa7 100644
--- a/block/parallels.c
+++ b/block/parallels.c
@@ -2,8 +2,12 @@
  * Block driver for Parallels disk image format
  *
  * Copyright (c) 2007 Alex Beregszaszi
+ * Copyright (c) 2015 Denis V. Lunev <den@openvz.org>
  *
- * This code is based on comparing different disk images created by Parallels.
+ * This code was originally based on comparing different disk images created
+ * by Parallels. Currently it is based on opened OpenVZ sources
+ * available at
+ *     http://git.openvz.org/?p=ploop;a=summary
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,66 +27,559 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
+#include "qemu/bitmap.h"
+#include "qapi/util.h"
 
 /**************************************************************/
 
 #define HEADER_MAGIC "WithoutFreeSpace"
 #define HEADER_MAGIC2 "WithouFreSpacExt"
 #define HEADER_VERSION 2
-#define HEADER_SIZE 64
+#define HEADER_INUSE_MAGIC  (0x746F6E59)
+#define MAX_PARALLELS_IMAGE_FACTOR (1ull << 32)
+
+#define DEFAULT_CLUSTER_SIZE 1048576        /* 1 MiB */
+
 
 // always little-endian
-struct parallels_header {
+typedef struct ParallelsHeader {
     char magic[16]; // "WithoutFreeSpace"
     uint32_t version;
     uint32_t heads;
     uint32_t cylinders;
     uint32_t tracks;
-    uint32_t catalog_entries;
+    uint32_t bat_entries;
     uint64_t nb_sectors;
     uint32_t inuse;
     uint32_t data_off;
     char padding[12];
-} QEMU_PACKED;
+} QEMU_PACKED ParallelsHeader;
+
+
+typedef enum ParallelsPreallocMode {
+    PRL_PREALLOC_MODE_FALLOCATE = 0,
+    PRL_PREALLOC_MODE_TRUNCATE = 1,
+    PRL_PREALLOC_MODE__MAX = 2,
+} ParallelsPreallocMode;
+
+static const char *prealloc_mode_lookup[] = {
+    "falloc",
+    "truncate",
+    NULL,
+};
+
 
 typedef struct BDRVParallelsState {
+    /** Locking is conservative, the lock protects
+     *   - image file extending (truncate, fallocate)
+     *   - any access to block allocation table
+     */
     CoMutex lock;
 
-    uint32_t *catalog_bitmap;
-    unsigned int catalog_size;
+    ParallelsHeader *header;
+    uint32_t header_size;
+    bool header_unclean;
+
+    unsigned long *bat_dirty_bmap;
+    unsigned int  bat_dirty_block;
+
+    uint32_t *bat_bitmap;
+    unsigned int bat_size;
+
+    int64_t  data_end;
+    uint64_t prealloc_size;
+    ParallelsPreallocMode prealloc_mode;
 
     unsigned int tracks;
 
     unsigned int off_multiplier;
 } BDRVParallelsState;
 
-static int parallels_probe(const uint8_t *buf, int buf_size, const char *filename)
+
+#define PARALLELS_OPT_PREALLOC_MODE     "prealloc-mode"
+#define PARALLELS_OPT_PREALLOC_SIZE     "prealloc-size"
+
+static QemuOptsList parallels_runtime_opts = {
+    .name = "parallels",
+    .head = QTAILQ_HEAD_INITIALIZER(parallels_runtime_opts.head),
+    .desc = {
+        {
+            .name = PARALLELS_OPT_PREALLOC_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Preallocation size on image expansion",
+            .def_value_str = "128MiB",
+        },
+        {
+            .name = PARALLELS_OPT_PREALLOC_MODE,
+            .type = QEMU_OPT_STRING,
+            .help = "Preallocation mode on image expansion "
+                    "(allowed values: falloc, truncate)",
+            .def_value_str = "falloc",
+        },
+        { /* end of list */ },
+    },
+};
+
+
+static int64_t bat2sect(BDRVParallelsState *s, uint32_t idx)
 {
-    const struct parallels_header *ph = (const void *)buf;
+    return (uint64_t)le32_to_cpu(s->bat_bitmap[idx]) * s->off_multiplier;
+}
 
-    if (buf_size < HEADER_SIZE)
+static uint32_t bat_entry_off(uint32_t idx)
+{
+    return sizeof(ParallelsHeader) + sizeof(uint32_t) * idx;
+}
+
+static int64_t seek_to_sector(BDRVParallelsState *s, int64_t sector_num)
+{
+    uint32_t index, offset;
+
+    index = sector_num / s->tracks;
+    offset = sector_num % s->tracks;
+
+    /* not allocated */
+    if ((index >= s->bat_size) || (s->bat_bitmap[index] == 0)) {
+        return -1;
+    }
+    return bat2sect(s, index) + offset;
+}
+
+static int cluster_remainder(BDRVParallelsState *s, int64_t sector_num,
+        int nb_sectors)
+{
+    int ret = s->tracks - sector_num % s->tracks;
+    return MIN(nb_sectors, ret);
+}
+
+static int64_t block_status(BDRVParallelsState *s, int64_t sector_num,
+                            int nb_sectors, int *pnum)
+{
+    int64_t start_off = -2, prev_end_off = -2;
+
+    *pnum = 0;
+    while (nb_sectors > 0 || start_off == -2) {
+        int64_t offset = seek_to_sector(s, sector_num);
+        int to_end;
+
+        if (start_off == -2) {
+            start_off = offset;
+            prev_end_off = offset;
+        } else if (offset != prev_end_off) {
+            break;
+        }
+
+        to_end = cluster_remainder(s, sector_num, nb_sectors);
+        nb_sectors -= to_end;
+        sector_num += to_end;
+        *pnum += to_end;
+
+        if (offset > 0) {
+            prev_end_off += to_end;
+        }
+    }
+    return start_off;
+}
+
+static int64_t allocate_clusters(BlockDriverState *bs, int64_t sector_num,
+                                 int nb_sectors, int *pnum)
+{
+    BDRVParallelsState *s = bs->opaque;
+    uint32_t idx, to_allocate, i;
+    int64_t pos, space;
+
+    pos = block_status(s, sector_num, nb_sectors, pnum);
+    if (pos > 0) {
+        return pos;
+    }
+
+    idx = sector_num / s->tracks;
+    if (idx >= s->bat_size) {
+        return -EINVAL;
+    }
+
+    to_allocate = DIV_ROUND_UP(sector_num + *pnum, s->tracks) - idx;
+    space = to_allocate * s->tracks;
+    if (s->data_end + space > bdrv_getlength(bs->file->bs) >> BDRV_SECTOR_BITS) {
+        int ret;
+        space += s->prealloc_size;
+        if (s->prealloc_mode == PRL_PREALLOC_MODE_FALLOCATE) {
+            ret = bdrv_pwrite_zeroes(bs->file,
+                                     s->data_end << BDRV_SECTOR_BITS,
+                                     space << BDRV_SECTOR_BITS, 0);
+        } else {
+            ret = bdrv_truncate(bs->file->bs,
+                                (s->data_end + space) << BDRV_SECTOR_BITS);
+        }
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    for (i = 0; i < to_allocate; i++) {
+        s->bat_bitmap[idx + i] = cpu_to_le32(s->data_end / s->off_multiplier);
+        s->data_end += s->tracks;
+        bitmap_set(s->bat_dirty_bmap,
+                   bat_entry_off(idx + i) / s->bat_dirty_block, 1);
+    }
+
+    return bat2sect(s, idx) + sector_num % s->tracks;
+}
+
+
+static coroutine_fn int parallels_co_flush_to_os(BlockDriverState *bs)
+{
+    BDRVParallelsState *s = bs->opaque;
+    unsigned long size = DIV_ROUND_UP(s->header_size, s->bat_dirty_block);
+    unsigned long bit;
+
+    qemu_co_mutex_lock(&s->lock);
+
+    bit = find_first_bit(s->bat_dirty_bmap, size);
+    while (bit < size) {
+        uint32_t off = bit * s->bat_dirty_block;
+        uint32_t to_write = s->bat_dirty_block;
+        int ret;
+
+        if (off + to_write > s->header_size) {
+            to_write = s->header_size - off;
+        }
+        ret = bdrv_pwrite(bs->file, off, (uint8_t *)s->header + off,
+                          to_write);
+        if (ret < 0) {
+            qemu_co_mutex_unlock(&s->lock);
+            return ret;
+        }
+        bit = find_next_bit(s->bat_dirty_bmap, size, bit + 1);
+    }
+    bitmap_zero(s->bat_dirty_bmap, size);
+
+    qemu_co_mutex_unlock(&s->lock);
+    return 0;
+}
+
+
+static int64_t coroutine_fn parallels_co_get_block_status(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
+{
+    BDRVParallelsState *s = bs->opaque;
+    int64_t offset;
+
+    qemu_co_mutex_lock(&s->lock);
+    offset = block_status(s, sector_num, nb_sectors, pnum);
+    qemu_co_mutex_unlock(&s->lock);
+
+    if (offset < 0) {
         return 0;
+    }
 
-    if ((!memcmp(ph->magic, HEADER_MAGIC, 16) ||
-        !memcmp(ph->magic, HEADER_MAGIC2, 16)) &&
-        (le32_to_cpu(ph->version) == HEADER_VERSION))
-        return 100;
+    *file = bs->file->bs;
+    return (offset << BDRV_SECTOR_BITS) |
+        BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
+}
+
+static coroutine_fn int parallels_co_writev(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    BDRVParallelsState *s = bs->opaque;
+    uint64_t bytes_done = 0;
+    QEMUIOVector hd_qiov;
+    int ret = 0;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    while (nb_sectors > 0) {
+        int64_t position;
+        int n, nbytes;
+
+        qemu_co_mutex_lock(&s->lock);
+        position = allocate_clusters(bs, sector_num, nb_sectors, &n);
+        qemu_co_mutex_unlock(&s->lock);
+        if (position < 0) {
+            ret = (int)position;
+            break;
+        }
+
+        nbytes = n << BDRV_SECTOR_BITS;
+
+        qemu_iovec_reset(&hd_qiov);
+        qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
+
+        ret = bdrv_co_writev(bs->file, position, n, &hd_qiov);
+        if (ret < 0) {
+            break;
+        }
+
+        nb_sectors -= n;
+        sector_num += n;
+        bytes_done += nbytes;
+    }
+
+    qemu_iovec_destroy(&hd_qiov);
+    return ret;
+}
+
+static coroutine_fn int parallels_co_readv(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov)
+{
+    BDRVParallelsState *s = bs->opaque;
+    uint64_t bytes_done = 0;
+    QEMUIOVector hd_qiov;
+    int ret = 0;
+
+    qemu_iovec_init(&hd_qiov, qiov->niov);
+
+    while (nb_sectors > 0) {
+        int64_t position;
+        int n, nbytes;
+
+        qemu_co_mutex_lock(&s->lock);
+        position = block_status(s, sector_num, nb_sectors, &n);
+        qemu_co_mutex_unlock(&s->lock);
+
+        nbytes = n << BDRV_SECTOR_BITS;
+
+        if (position < 0) {
+            qemu_iovec_memset(qiov, bytes_done, 0, nbytes);
+        } else {
+            qemu_iovec_reset(&hd_qiov);
+            qemu_iovec_concat(&hd_qiov, qiov, bytes_done, nbytes);
+
+            ret = bdrv_co_readv(bs->file, position, n, &hd_qiov);
+            if (ret < 0) {
+                break;
+            }
+        }
+
+        nb_sectors -= n;
+        sector_num += n;
+        bytes_done += nbytes;
+    }
+
+    qemu_iovec_destroy(&hd_qiov);
+    return ret;
+}
+
+
+static int parallels_check(BlockDriverState *bs, BdrvCheckResult *res,
+                           BdrvCheckMode fix)
+{
+    BDRVParallelsState *s = bs->opaque;
+    int64_t size, prev_off, high_off;
+    int ret;
+    uint32_t i;
+    bool flush_bat = false;
+    int cluster_size = s->tracks << BDRV_SECTOR_BITS;
+
+    size = bdrv_getlength(bs->file->bs);
+    if (size < 0) {
+        res->check_errors++;
+        return size;
+    }
+
+    if (s->header_unclean) {
+        fprintf(stderr, "%s image was not closed correctly\n",
+                fix & BDRV_FIX_ERRORS ? "Repairing" : "ERROR");
+        res->corruptions++;
+        if (fix & BDRV_FIX_ERRORS) {
+            /* parallels_close will do the job right */
+            res->corruptions_fixed++;
+            s->header_unclean = false;
+        }
+    }
+
+    res->bfi.total_clusters = s->bat_size;
+    res->bfi.compressed_clusters = 0; /* compression is not supported */
+
+    high_off = 0;
+    prev_off = 0;
+    for (i = 0; i < s->bat_size; i++) {
+        int64_t off = bat2sect(s, i) << BDRV_SECTOR_BITS;
+        if (off == 0) {
+            prev_off = 0;
+            continue;
+        }
+
+        /* cluster outside the image */
+        if (off > size) {
+            fprintf(stderr, "%s cluster %u is outside image\n",
+                    fix & BDRV_FIX_ERRORS ? "Repairing" : "ERROR", i);
+            res->corruptions++;
+            if (fix & BDRV_FIX_ERRORS) {
+                prev_off = 0;
+                s->bat_bitmap[i] = 0;
+                res->corruptions_fixed++;
+                flush_bat = true;
+                continue;
+            }
+        }
+
+        res->bfi.allocated_clusters++;
+        if (off > high_off) {
+            high_off = off;
+        }
+
+        if (prev_off != 0 && (prev_off + cluster_size) != off) {
+            res->bfi.fragmented_clusters++;
+        }
+        prev_off = off;
+    }
+
+    if (flush_bat) {
+        ret = bdrv_pwrite_sync(bs->file, 0, s->header, s->header_size);
+        if (ret < 0) {
+            res->check_errors++;
+            return ret;
+        }
+    }
+
+    res->image_end_offset = high_off + cluster_size;
+    if (size > res->image_end_offset) {
+        int64_t count;
+        count = DIV_ROUND_UP(size - res->image_end_offset, cluster_size);
+        fprintf(stderr, "%s space leaked at the end of the image %" PRId64 "\n",
+                fix & BDRV_FIX_LEAKS ? "Repairing" : "ERROR",
+                size - res->image_end_offset);
+        res->leaks += count;
+        if (fix & BDRV_FIX_LEAKS) {
+            ret = bdrv_truncate(bs->file->bs, res->image_end_offset);
+            if (ret < 0) {
+                res->check_errors++;
+                return ret;
+            }
+            res->leaks_fixed += count;
+        }
+    }
 
     return 0;
 }
 
+
+static int parallels_create(const char *filename, QemuOpts *opts, Error **errp)
+{
+    int64_t total_size, cl_size;
+    uint8_t tmp[BDRV_SECTOR_SIZE];
+    Error *local_err = NULL;
+    BlockBackend *file;
+    uint32_t bat_entries, bat_sectors;
+    ParallelsHeader header;
+    int ret;
+
+    total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
+                          BDRV_SECTOR_SIZE);
+    cl_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_CLUSTER_SIZE,
+                          DEFAULT_CLUSTER_SIZE), BDRV_SECTOR_SIZE);
+    if (total_size >= MAX_PARALLELS_IMAGE_FACTOR * cl_size) {
+        error_propagate(errp, local_err);
+        return -E2BIG;
+    }
+
+    ret = bdrv_create_file(filename, opts, &local_err);
+    if (ret < 0) {
+        error_propagate(errp, local_err);
+        return ret;
+    }
+
+    file = blk_new_open(filename, NULL, NULL,
+                        BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (file == NULL) {
+        error_propagate(errp, local_err);
+        return -EIO;
+    }
+
+    blk_set_allow_write_beyond_eof(file, true);
+
+    ret = blk_truncate(file, 0);
+    if (ret < 0) {
+        goto exit;
+    }
+
+    bat_entries = DIV_ROUND_UP(total_size, cl_size);
+    bat_sectors = DIV_ROUND_UP(bat_entry_off(bat_entries), cl_size);
+    bat_sectors = (bat_sectors *  cl_size) >> BDRV_SECTOR_BITS;
+
+    memset(&header, 0, sizeof(header));
+    memcpy(header.magic, HEADER_MAGIC2, sizeof(header.magic));
+    header.version = cpu_to_le32(HEADER_VERSION);
+    /* don't care much about geometry, it is not used on image level */
+    header.heads = cpu_to_le32(16);
+    header.cylinders = cpu_to_le32(total_size / BDRV_SECTOR_SIZE / 16 / 32);
+    header.tracks = cpu_to_le32(cl_size >> BDRV_SECTOR_BITS);
+    header.bat_entries = cpu_to_le32(bat_entries);
+    header.nb_sectors = cpu_to_le64(DIV_ROUND_UP(total_size, BDRV_SECTOR_SIZE));
+    header.data_off = cpu_to_le32(bat_sectors);
+
+    /* write all the data */
+    memset(tmp, 0, sizeof(tmp));
+    memcpy(tmp, &header, sizeof(header));
+
+    ret = blk_pwrite(file, 0, tmp, BDRV_SECTOR_SIZE, 0);
+    if (ret < 0) {
+        goto exit;
+    }
+    ret = blk_pwrite_zeroes(file, BDRV_SECTOR_SIZE,
+                            (bat_sectors - 1) << BDRV_SECTOR_BITS, 0);
+    if (ret < 0) {
+        goto exit;
+    }
+    ret = 0;
+
+done:
+    blk_unref(file);
+    return ret;
+
+exit:
+    error_setg_errno(errp, -ret, "Failed to create Parallels image");
+    goto done;
+}
+
+
+static int parallels_probe(const uint8_t *buf, int buf_size,
+                           const char *filename)
+{
+    const ParallelsHeader *ph = (const void *)buf;
+
+    if (buf_size < sizeof(ParallelsHeader)) {
+        return 0;
+    }
+
+    if ((!memcmp(ph->magic, HEADER_MAGIC, 16) ||
+           !memcmp(ph->magic, HEADER_MAGIC2, 16)) &&
+           (le32_to_cpu(ph->version) == HEADER_VERSION)) {
+        return 100;
+    }
+
+    return 0;
+}
+
+static int parallels_update_header(BlockDriverState *bs)
+{
+    BDRVParallelsState *s = bs->opaque;
+    unsigned size = MAX(bdrv_opt_mem_align(bs->file->bs),
+                        sizeof(ParallelsHeader));
+
+    if (size > s->header_size) {
+        size = s->header_size;
+    }
+    return bdrv_pwrite_sync(bs->file, 0, s->header, size);
+}
+
 static int parallels_open(BlockDriverState *bs, QDict *options, int flags,
                           Error **errp)
 {
     BDRVParallelsState *s = bs->opaque;
-    int i;
-    struct parallels_header ph;
-    int ret;
-
-    bs->read_only = 1; // no write support yet
+    ParallelsHeader ph;
+    int ret, size, i;
+    QemuOpts *opts = NULL;
+    Error *local_err = NULL;
+    char *buf;
 
     ret = bdrv_pread(bs->file, 0, &ph, sizeof(ph));
     if (ret < 0) {
@@ -115,25 +612,90 @@
         goto fail;
     }
 
-    s->catalog_size = le32_to_cpu(ph.catalog_entries);
-    if (s->catalog_size > INT_MAX / 4) {
+    s->bat_size = le32_to_cpu(ph.bat_entries);
+    if (s->bat_size > INT_MAX / sizeof(uint32_t)) {
         error_setg(errp, "Catalog too large");
         ret = -EFBIG;
         goto fail;
     }
-    s->catalog_bitmap = g_try_new(uint32_t, s->catalog_size);
-    if (s->catalog_size && s->catalog_bitmap == NULL) {
+
+    size = bat_entry_off(s->bat_size);
+    s->header_size = ROUND_UP(size, bdrv_opt_mem_align(bs->file->bs));
+    s->header = qemu_try_blockalign(bs->file->bs, s->header_size);
+    if (s->header == NULL) {
         ret = -ENOMEM;
         goto fail;
     }
+    s->data_end = le32_to_cpu(ph.data_off);
+    if (s->data_end == 0) {
+        s->data_end = ROUND_UP(bat_entry_off(s->bat_size), BDRV_SECTOR_SIZE);
+    }
+    if (s->data_end < s->header_size) {
+        /* there is not enough unused space to fit to block align between BAT
+           and actual data. We can't avoid read-modify-write... */
+        s->header_size = size;
+    }
 
-    ret = bdrv_pread(bs->file, 64, s->catalog_bitmap, s->catalog_size * 4);
+    ret = bdrv_pread(bs->file, 0, s->header, s->header_size);
     if (ret < 0) {
         goto fail;
     }
+    s->bat_bitmap = (uint32_t *)(s->header + 1);
 
-    for (i = 0; i < s->catalog_size; i++)
-        le32_to_cpus(&s->catalog_bitmap[i]);
+    for (i = 0; i < s->bat_size; i++) {
+        int64_t off = bat2sect(s, i);
+        if (off >= s->data_end) {
+            s->data_end = off + s->tracks;
+        }
+    }
+
+    if (le32_to_cpu(ph.inuse) == HEADER_INUSE_MAGIC) {
+        /* Image was not closed correctly. The check is mandatory */
+        s->header_unclean = true;
+        if ((flags & BDRV_O_RDWR) && !(flags & BDRV_O_CHECK)) {
+            error_setg(errp, "parallels: Image was not closed correctly; "
+                       "cannot be opened read/write");
+            ret = -EACCES;
+            goto fail;
+        }
+    }
+
+    opts = qemu_opts_create(&parallels_runtime_opts, NULL, 0, &local_err);
+    if (local_err != NULL) {
+        goto fail_options;
+    }
+
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err != NULL) {
+        goto fail_options;
+    }
+
+    s->prealloc_size =
+        qemu_opt_get_size_del(opts, PARALLELS_OPT_PREALLOC_SIZE, 0);
+    s->prealloc_size = MAX(s->tracks, s->prealloc_size >> BDRV_SECTOR_BITS);
+    buf = qemu_opt_get_del(opts, PARALLELS_OPT_PREALLOC_MODE);
+    s->prealloc_mode = qapi_enum_parse(prealloc_mode_lookup, buf,
+            PRL_PREALLOC_MODE__MAX, PRL_PREALLOC_MODE_FALLOCATE, &local_err);
+    g_free(buf);
+    if (local_err != NULL) {
+        goto fail_options;
+    }
+    if (!bdrv_has_zero_init(bs->file->bs) ||
+            bdrv_truncate(bs->file->bs, bdrv_getlength(bs->file->bs)) != 0) {
+        s->prealloc_mode = PRL_PREALLOC_MODE_FALLOCATE;
+    }
+
+    if (flags & BDRV_O_RDWR) {
+        s->header->inuse = cpu_to_le32(HEADER_INUSE_MAGIC);
+        ret = parallels_update_header(bs);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    s->bat_dirty_block = 4 * getpagesize();
+    s->bat_dirty_bmap =
+        bitmap_new(DIV_ROUND_UP(s->header_size, s->bat_dirty_block));
 
     qemu_co_mutex_init(&s->lock);
     return 0;
@@ -142,67 +704,67 @@
     error_setg(errp, "Image not in Parallels format");
     ret = -EINVAL;
 fail:
-    g_free(s->catalog_bitmap);
+    qemu_vfree(s->header);
     return ret;
+
+fail_options:
+    error_propagate(errp, local_err);
+    ret = -EINVAL;
+    goto fail;
 }
 
-static int64_t seek_to_sector(BlockDriverState *bs, int64_t sector_num)
-{
-    BDRVParallelsState *s = bs->opaque;
-    uint32_t index, offset;
-
-    index = sector_num / s->tracks;
-    offset = sector_num % s->tracks;
-
-    /* not allocated */
-    if ((index >= s->catalog_size) || (s->catalog_bitmap[index] == 0))
-        return -1;
-    return
-        ((uint64_t)s->catalog_bitmap[index] * s->off_multiplier + offset) * 512;
-}
-
-static int parallels_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
-{
-    while (nb_sectors > 0) {
-        int64_t position = seek_to_sector(bs, sector_num);
-        if (position >= 0) {
-            if (bdrv_pread(bs->file, position, buf, 512) != 512)
-                return -1;
-        } else {
-            memset(buf, 0, 512);
-        }
-        nb_sectors--;
-        sector_num++;
-        buf += 512;
-    }
-    return 0;
-}
-
-static coroutine_fn int parallels_co_read(BlockDriverState *bs, int64_t sector_num,
-                                          uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVParallelsState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = parallels_read(bs, sector_num, buf, nb_sectors);
-    qemu_co_mutex_unlock(&s->lock);
-    return ret;
-}
 
 static void parallels_close(BlockDriverState *bs)
 {
     BDRVParallelsState *s = bs->opaque;
-    g_free(s->catalog_bitmap);
+
+    if (bs->open_flags & BDRV_O_RDWR) {
+        s->header->inuse = 0;
+        parallels_update_header(bs);
+    }
+
+    if (bs->open_flags & BDRV_O_RDWR) {
+        bdrv_truncate(bs->file->bs, s->data_end << BDRV_SECTOR_BITS);
+    }
+
+    g_free(s->bat_dirty_bmap);
+    qemu_vfree(s->header);
 }
 
+static QemuOptsList parallels_create_opts = {
+    .name = "parallels-create-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(parallels_create_opts.head),
+    .desc = {
+        {
+            .name = BLOCK_OPT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Virtual disk size",
+        },
+        {
+            .name = BLOCK_OPT_CLUSTER_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Parallels image cluster size",
+            .def_value_str = stringify(DEFAULT_CLUSTER_SIZE),
+        },
+        { /* end of list */ }
+    }
+};
+
 static BlockDriver bdrv_parallels = {
     .format_name	= "parallels",
     .instance_size	= sizeof(BDRVParallelsState),
     .bdrv_probe		= parallels_probe,
     .bdrv_open		= parallels_open,
-    .bdrv_read          = parallels_co_read,
     .bdrv_close		= parallels_close,
+    .bdrv_co_get_block_status = parallels_co_get_block_status,
+    .bdrv_has_zero_init       = bdrv_has_zero_init_1,
+    .bdrv_co_flush_to_os      = parallels_co_flush_to_os,
+    .bdrv_co_readv  = parallels_co_readv,
+    .bdrv_co_writev = parallels_co_writev,
+
+    .bdrv_create    = parallels_create,
+    .bdrv_check     = parallels_check,
+    .create_opts    = &parallels_create_opts,
 };
 
 static void bdrv_parallels_init(void)
diff --git a/block/qapi.c b/block/qapi.c
index a87a34a..6f947e3 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -22,16 +22,23 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/qapi.h"
 #include "block/block_int.h"
+#include "block/throttle-groups.h"
+#include "block/write-threshold.h"
 #include "qmp-commands.h"
 #include "qapi-visit.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/qmp/types.h"
 #include "sysemu/block-backend.h"
+#include "qemu/cutils.h"
 
-BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs)
+BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
+                                        BlockDriverState *bs, Error **errp)
 {
+    ImageInfo **p_image_info;
+    BlockDriverState *bs0;
     BlockDeviceInfo *info = g_malloc0(sizeof(*info));
 
     info->file                   = g_strdup(bs->filename);
@@ -40,6 +47,13 @@
     info->encrypted              = bs->encrypted;
     info->encryption_key_missing = bdrv_key_required(bs);
 
+    info->cache = g_new(BlockdevCacheInfo, 1);
+    *info->cache = (BlockdevCacheInfo) {
+        .writeback      = blk ? blk_enable_write_cache(blk) : true,
+        .direct         = !!(bs->open_flags & BDRV_O_NOCACHE),
+        .no_flush       = !!(bs->open_flags & BDRV_O_NO_FLUSH),
+    };
+
     if (bs->node_name[0]) {
         info->has_node_name = true;
         info->node_name = g_strdup(bs->node_name);
@@ -53,9 +67,11 @@
     info->backing_file_depth = bdrv_get_backing_file_depth(bs);
     info->detect_zeroes = bs->detect_zeroes;
 
-    if (bs->io_limits_enabled) {
+    if (blk && blk_get_public(blk)->throttle_state) {
         ThrottleConfig cfg;
-        throttle_get_config(&bs->throttle_state, &cfg);
+
+        throttle_group_get_config(blk, &cfg);
+
         info->bps     = cfg.buckets[THROTTLE_BPS_TOTAL].avg;
         info->bps_rd  = cfg.buckets[THROTTLE_BPS_READ].avg;
         info->bps_wr  = cfg.buckets[THROTTLE_BPS_WRITE].avg;
@@ -78,8 +94,52 @@
         info->has_iops_wr_max = cfg.buckets[THROTTLE_OPS_WRITE].max;
         info->iops_wr_max     = cfg.buckets[THROTTLE_OPS_WRITE].max;
 
+        info->has_bps_max_length     = info->has_bps_max;
+        info->bps_max_length         =
+            cfg.buckets[THROTTLE_BPS_TOTAL].burst_length;
+        info->has_bps_rd_max_length  = info->has_bps_rd_max;
+        info->bps_rd_max_length      =
+            cfg.buckets[THROTTLE_BPS_READ].burst_length;
+        info->has_bps_wr_max_length  = info->has_bps_wr_max;
+        info->bps_wr_max_length      =
+            cfg.buckets[THROTTLE_BPS_WRITE].burst_length;
+
+        info->has_iops_max_length    = info->has_iops_max;
+        info->iops_max_length        =
+            cfg.buckets[THROTTLE_OPS_TOTAL].burst_length;
+        info->has_iops_rd_max_length = info->has_iops_rd_max;
+        info->iops_rd_max_length     =
+            cfg.buckets[THROTTLE_OPS_READ].burst_length;
+        info->has_iops_wr_max_length = info->has_iops_wr_max;
+        info->iops_wr_max_length     =
+            cfg.buckets[THROTTLE_OPS_WRITE].burst_length;
+
         info->has_iops_size = cfg.op_size;
         info->iops_size = cfg.op_size;
+
+        info->has_group = true;
+        info->group = g_strdup(throttle_group_get_name(blk));
+    }
+
+    info->write_threshold = bdrv_write_threshold_get(bs);
+
+    bs0 = bs;
+    p_image_info = &info->image;
+    while (1) {
+        Error *local_err = NULL;
+        bdrv_query_image_info(bs0, p_image_info, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            qapi_free_BlockDeviceInfo(info);
+            return NULL;
+        }
+        if (bs0->drv && bs0->backing) {
+            bs0 = bs0->backing->bs;
+            (*p_image_info)->has_backing_image = true;
+            p_image_info = &((*p_image_info)->backing_image);
+        } else {
+            break;
+        }
     }
 
     return info;
@@ -168,17 +228,18 @@
 {
     int64_t size;
     const char *backing_filename;
-    char backing_filename2[1024];
     BlockDriverInfo bdi;
     int ret;
     Error *err = NULL;
     ImageInfo *info;
 
+    aio_context_acquire(bdrv_get_aio_context(bs));
+
     size = bdrv_getlength(bs);
     if (size < 0) {
         error_setg_errno(errp, -size, "Can't get size of device '%s'",
                          bdrv_get_device_name(bs));
-        return;
+        goto out;
     }
 
     info = g_new0(ImageInfo, 1);
@@ -204,14 +265,23 @@
 
     backing_filename = bs->backing_file;
     if (backing_filename[0] != '\0') {
+        char *backing_filename2 = g_malloc0(PATH_MAX);
         info->backing_filename = g_strdup(backing_filename);
         info->has_backing_filename = true;
-        bdrv_get_full_backing_filename(bs, backing_filename2,
-                                       sizeof(backing_filename2));
+        bdrv_get_full_backing_filename(bs, backing_filename2, PATH_MAX, &err);
+        if (err) {
+            /* Can't reconstruct the full backing filename, so we must omit
+             * this field and apply a Best Effort to this query. */
+            g_free(backing_filename2);
+            backing_filename2 = NULL;
+            error_free(err);
+            err = NULL;
+        }
 
-        if (strcmp(backing_filename, backing_filename2) != 0) {
-            info->full_backing_filename =
-                        g_strdup(backing_filename2);
+        /* Always report the full_backing_filename if present, even if it's the
+         * same as backing_filename. That they are same is useful info. */
+        if (backing_filename2) {
+            info->full_backing_filename = g_strdup(backing_filename2);
             info->has_full_backing_filename = true;
         }
 
@@ -219,6 +289,7 @@
             info->backing_filename_format = g_strdup(bs->backing_format);
             info->has_backing_filename_format = true;
         }
+        g_free(backing_filename2);
     }
 
     ret = bdrv_query_snapshot_info_list(bs, &info->snapshots, &err);
@@ -236,10 +307,13 @@
     default:
         error_propagate(errp, err);
         qapi_free_ImageInfo(info);
-        return;
+        goto out;
     }
 
     *p_info = info;
+
+out:
+    aio_context_release(bdrv_get_aio_context(bs));
 }
 
 /* @p_info will be set only on success. */
@@ -248,48 +322,31 @@
 {
     BlockInfo *info = g_malloc0(sizeof(*info));
     BlockDriverState *bs = blk_bs(blk);
-    BlockDriverState *bs0;
-    ImageInfo **p_image_info;
-    Error *local_err = NULL;
     info->device = g_strdup(blk_name(blk));
     info->type = g_strdup("unknown");
     info->locked = blk_dev_is_medium_locked(blk);
     info->removable = blk_dev_has_removable_media(blk);
 
-    if (blk_dev_has_removable_media(blk)) {
+    if (blk_dev_has_tray(blk)) {
         info->has_tray_open = true;
         info->tray_open = blk_dev_is_tray_open(blk);
     }
 
-    if (bdrv_iostatus_is_enabled(bs)) {
+    if (blk_iostatus_is_enabled(blk)) {
         info->has_io_status = true;
-        info->io_status = bs->iostatus;
+        info->io_status = blk_iostatus(blk);
     }
 
-    if (!QLIST_EMPTY(&bs->dirty_bitmaps)) {
+    if (bs && !QLIST_EMPTY(&bs->dirty_bitmaps)) {
         info->has_dirty_bitmaps = true;
         info->dirty_bitmaps = bdrv_query_dirty_bitmaps(bs);
     }
 
-    if (bs->drv) {
+    if (bs && bs->drv) {
         info->has_inserted = true;
-        info->inserted = bdrv_block_device_info(bs);
-
-        bs0 = bs;
-        p_image_info = &info->inserted->image;
-        while (1) {
-            bdrv_query_image_info(bs0, p_image_info, &local_err);
-            if (local_err) {
-                error_propagate(errp, local_err);
-                goto err;
-            }
-            if (bs0->drv && bs0->backing_hd) {
-                bs0 = bs0->backing_hd;
-                (*p_image_info)->has_backing_image = true;
-                p_image_info = &((*p_image_info)->backing_image);
-            } else {
-                break;
-            }
+        info->inserted = bdrv_block_device_info(blk, bs, errp);
+        if (info->inserted == NULL) {
+            goto err;
         }
     }
 
@@ -300,37 +357,115 @@
     qapi_free_BlockInfo(info);
 }
 
-static BlockStats *bdrv_query_stats(const BlockDriverState *bs)
+static BlockStats *bdrv_query_stats(BlockBackend *blk,
+                                    const BlockDriverState *bs,
+                                    bool query_backing);
+
+static void bdrv_query_blk_stats(BlockDeviceStats *ds, BlockBackend *blk)
+{
+    BlockAcctStats *stats = blk_get_stats(blk);
+    BlockAcctTimedStats *ts = NULL;
+
+    ds->rd_bytes = stats->nr_bytes[BLOCK_ACCT_READ];
+    ds->wr_bytes = stats->nr_bytes[BLOCK_ACCT_WRITE];
+    ds->rd_operations = stats->nr_ops[BLOCK_ACCT_READ];
+    ds->wr_operations = stats->nr_ops[BLOCK_ACCT_WRITE];
+
+    ds->failed_rd_operations = stats->failed_ops[BLOCK_ACCT_READ];
+    ds->failed_wr_operations = stats->failed_ops[BLOCK_ACCT_WRITE];
+    ds->failed_flush_operations = stats->failed_ops[BLOCK_ACCT_FLUSH];
+
+    ds->invalid_rd_operations = stats->invalid_ops[BLOCK_ACCT_READ];
+    ds->invalid_wr_operations = stats->invalid_ops[BLOCK_ACCT_WRITE];
+    ds->invalid_flush_operations =
+        stats->invalid_ops[BLOCK_ACCT_FLUSH];
+
+    ds->rd_merged = stats->merged[BLOCK_ACCT_READ];
+    ds->wr_merged = stats->merged[BLOCK_ACCT_WRITE];
+    ds->flush_operations = stats->nr_ops[BLOCK_ACCT_FLUSH];
+    ds->wr_total_time_ns = stats->total_time_ns[BLOCK_ACCT_WRITE];
+    ds->rd_total_time_ns = stats->total_time_ns[BLOCK_ACCT_READ];
+    ds->flush_total_time_ns = stats->total_time_ns[BLOCK_ACCT_FLUSH];
+
+    ds->has_idle_time_ns = stats->last_access_time_ns > 0;
+    if (ds->has_idle_time_ns) {
+        ds->idle_time_ns = block_acct_idle_time_ns(stats);
+    }
+
+    ds->account_invalid = stats->account_invalid;
+    ds->account_failed = stats->account_failed;
+
+    while ((ts = block_acct_interval_next(stats, ts))) {
+        BlockDeviceTimedStatsList *timed_stats =
+            g_malloc0(sizeof(*timed_stats));
+        BlockDeviceTimedStats *dev_stats = g_malloc0(sizeof(*dev_stats));
+        timed_stats->next = ds->timed_stats;
+        timed_stats->value = dev_stats;
+        ds->timed_stats = timed_stats;
+
+        TimedAverage *rd = &ts->latency[BLOCK_ACCT_READ];
+        TimedAverage *wr = &ts->latency[BLOCK_ACCT_WRITE];
+        TimedAverage *fl = &ts->latency[BLOCK_ACCT_FLUSH];
+
+        dev_stats->interval_length = ts->interval_length;
+
+        dev_stats->min_rd_latency_ns = timed_average_min(rd);
+        dev_stats->max_rd_latency_ns = timed_average_max(rd);
+        dev_stats->avg_rd_latency_ns = timed_average_avg(rd);
+
+        dev_stats->min_wr_latency_ns = timed_average_min(wr);
+        dev_stats->max_wr_latency_ns = timed_average_max(wr);
+        dev_stats->avg_wr_latency_ns = timed_average_avg(wr);
+
+        dev_stats->min_flush_latency_ns = timed_average_min(fl);
+        dev_stats->max_flush_latency_ns = timed_average_max(fl);
+        dev_stats->avg_flush_latency_ns = timed_average_avg(fl);
+
+        dev_stats->avg_rd_queue_depth =
+            block_acct_queue_depth(ts, BLOCK_ACCT_READ);
+        dev_stats->avg_wr_queue_depth =
+            block_acct_queue_depth(ts, BLOCK_ACCT_WRITE);
+    }
+}
+
+static void bdrv_query_bds_stats(BlockStats *s, const BlockDriverState *bs,
+                                 bool query_backing)
+{
+    if (bdrv_get_node_name(bs)[0]) {
+        s->has_node_name = true;
+        s->node_name = g_strdup(bdrv_get_node_name(bs));
+    }
+
+    s->stats->wr_highest_offset = bs->wr_highest_offset;
+
+    if (bs->file) {
+        s->has_parent = true;
+        s->parent = bdrv_query_stats(NULL, bs->file->bs, query_backing);
+    }
+
+    if (query_backing && bs->backing) {
+        s->has_backing = true;
+        s->backing = bdrv_query_stats(NULL, bs->backing->bs, query_backing);
+    }
+
+}
+
+static BlockStats *bdrv_query_stats(BlockBackend *blk,
+                                    const BlockDriverState *bs,
+                                    bool query_backing)
 {
     BlockStats *s;
 
     s = g_malloc0(sizeof(*s));
-
-    if (bdrv_get_device_name(bs)[0]) {
-        s->has_device = true;
-        s->device = g_strdup(bdrv_get_device_name(bs));
-    }
-
     s->stats = g_malloc0(sizeof(*s->stats));
-    s->stats->rd_bytes = bs->stats.nr_bytes[BLOCK_ACCT_READ];
-    s->stats->wr_bytes = bs->stats.nr_bytes[BLOCK_ACCT_WRITE];
-    s->stats->rd_operations = bs->stats.nr_ops[BLOCK_ACCT_READ];
-    s->stats->wr_operations = bs->stats.nr_ops[BLOCK_ACCT_WRITE];
-    s->stats->wr_highest_offset =
-        bs->stats.wr_highest_sector * BDRV_SECTOR_SIZE;
-    s->stats->flush_operations = bs->stats.nr_ops[BLOCK_ACCT_FLUSH];
-    s->stats->wr_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_WRITE];
-    s->stats->rd_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_READ];
-    s->stats->flush_total_time_ns = bs->stats.total_time_ns[BLOCK_ACCT_FLUSH];
 
-    if (bs->file) {
-        s->has_parent = true;
-        s->parent = bdrv_query_stats(bs->file);
+    if (blk) {
+        s->has_device = true;
+        s->device = g_strdup(blk_name(blk));
+        bdrv_query_blk_stats(s->stats, blk);
     }
-
-    if (bs->backing_hd) {
-        s->has_backing = true;
-        s->backing = bdrv_query_stats(bs->backing_hd);
+    if (bs) {
+        bdrv_query_bds_stats(s, bs, query_backing);
     }
 
     return s;
@@ -347,7 +482,9 @@
         bdrv_query_info(blk, &info->value, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
-            goto err;
+            g_free(info);
+            qapi_free_BlockInfoList(head);
+            return NULL;
         }
 
         *p_next = info;
@@ -355,23 +492,40 @@
     }
 
     return head;
-
- err:
-    qapi_free_BlockInfoList(head);
-    return NULL;
 }
 
-BlockStatsList *qmp_query_blockstats(Error **errp)
+static bool next_query_bds(BlockBackend **blk, BlockDriverState **bs,
+                           bool query_nodes)
+{
+    if (query_nodes) {
+        *bs = bdrv_next_node(*bs);
+        return !!*bs;
+    }
+
+    *blk = blk_next(*blk);
+    *bs = *blk ? blk_bs(*blk) : NULL;
+
+    return !!*blk;
+}
+
+BlockStatsList *qmp_query_blockstats(bool has_query_nodes,
+                                     bool query_nodes,
+                                     Error **errp)
 {
     BlockStatsList *head = NULL, **p_next = &head;
+    BlockBackend *blk = NULL;
     BlockDriverState *bs = NULL;
 
-     while ((bs = bdrv_next(bs))) {
+    /* Just to be safe if query_nodes is not always initialized */
+    query_nodes = has_query_nodes && query_nodes;
+
+    while (next_query_bds(&blk, &bs, query_nodes)) {
         BlockStatsList *info = g_malloc0(sizeof(*info));
-        AioContext *ctx = bdrv_get_aio_context(bs);
+        AioContext *ctx = blk ? blk_get_aio_context(blk)
+                              : bdrv_get_aio_context(bs);
 
         aio_context_acquire(ctx);
-        info->value = bdrv_query_stats(bs);
+        info->value = bdrv_query_stats(blk, bs, !query_nodes);
         aio_context_release(ctx);
 
         *p_next = info;
@@ -385,7 +539,7 @@
 
 static char *get_human_readable_size(char *buf, int buf_size, int64_t size)
 {
-    static const char suffixes[NB_SUFFIXES] = "KMGT";
+    static const char suffixes[NB_SUFFIXES] = {'K', 'M', 'G', 'T'};
     int64_t base;
     int i;
 
@@ -481,18 +635,9 @@
         }
         case QTYPE_QBOOL: {
             QBool *value = qobject_to_qbool(obj);
-            func_fprintf(f, "%s", qbool_get_int(value) ? "true" : "false");
+            func_fprintf(f, "%s", qbool_get_bool(value) ? "true" : "false");
             break;
         }
-        case QTYPE_QERROR: {
-            QString *value = qerror_human((QError *)obj);
-            func_fprintf(f, "%s", qstring_get_str(value));
-            QDECREF(value);
-            break;
-        }
-        case QTYPE_NONE:
-            break;
-        case QTYPE_MAX:
         default:
             abort();
     }
@@ -505,11 +650,10 @@
     int i = 0;
 
     for (entry = qlist_first(list); entry; entry = qlist_next(entry), i++) {
-        qtype_code type = qobject_type(entry->value);
+        QType type = qobject_type(entry->value);
         bool composite = (type == QTYPE_QDICT || type == QTYPE_QLIST);
-        const char *format = composite ? "%*s[%i]:\n" : "%*s[%i]: ";
-
-        func_fprintf(f, format, indentation * 4, "", i);
+        func_fprintf(f, "%*s[%i]:%c", indentation * 4, "", i,
+                     composite ? '\n' : ' ');
         dump_qobject(func_fprintf, f, indentation + 1, entry->value);
         if (!composite) {
             func_fprintf(f, "\n");
@@ -523,10 +667,9 @@
     const QDictEntry *entry;
 
     for (entry = qdict_first(dict); entry; entry = qdict_next(dict, entry)) {
-        qtype_code type = qobject_type(entry->value);
+        QType type = qobject_type(entry->value);
         bool composite = (type == QTYPE_QDICT || type == QTYPE_QLIST);
-        const char *format = composite ? "%*s%s:\n" : "%*s%s: ";
-        char key[strlen(entry->key) + 1];
+        char *key = g_malloc(strlen(entry->key) + 1);
         int i;
 
         /* replace dashes with spaces in key (variable) names */
@@ -534,28 +677,28 @@
             key[i] = entry->key[i] == '-' ? ' ' : entry->key[i];
         }
         key[i] = 0;
-
-        func_fprintf(f, format, indentation * 4, "", key);
+        func_fprintf(f, "%*s%s:%c", indentation * 4, "", key,
+                     composite ? '\n' : ' ');
         dump_qobject(func_fprintf, f, indentation + 1, entry->value);
         if (!composite) {
             func_fprintf(f, "\n");
         }
+        g_free(key);
     }
 }
 
 void bdrv_image_info_specific_dump(fprintf_function func_fprintf, void *f,
                                    ImageInfoSpecific *info_spec)
 {
-    QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj, *data;
+    Visitor *v = qmp_output_visitor_new(&obj);
 
-    visit_type_ImageInfoSpecific(qmp_output_get_visitor(ov), &info_spec, NULL,
-                                 &error_abort);
-    obj = qmp_output_get_qobject(ov);
+    visit_type_ImageInfoSpecific(v, NULL, &info_spec, &error_abort);
+    visit_complete(v, &obj);
     assert(qobject_type(obj) == QTYPE_QDICT);
     data = qdict_get(qobject_to_qdict(obj), "data");
     dump_qobject(func_fprintf, f, 1, data);
-    qmp_output_visitor_cleanup(ov);
+    visit_free(v);
 }
 
 void bdrv_image_info_dump(fprintf_function func_fprintf, void *f,
@@ -593,7 +736,10 @@
 
     if (info->has_backing_filename) {
         func_fprintf(f, "backing file: %s", info->backing_filename);
-        if (info->has_full_backing_filename) {
+        if (!info->has_full_backing_filename) {
+            func_fprintf(f, " (cannot determine actual path)");
+        } else if (strcmp(info->backing_filename,
+                          info->full_backing_filename) != 0) {
             func_fprintf(f, " (actual path: %s)", info->full_backing_filename);
         }
         func_fprintf(f, "\n");
diff --git a/block/qcow.c b/block/qcow.c
index ece2269..6f9b2e2 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -21,11 +21,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 #include <zlib.h>
-#include "qemu/aes.h"
+#include "qapi/qmp/qerror.h"
+#include "crypto/cipher.h"
 #include "migration/migration.h"
 
 /**************************************************************/
@@ -71,10 +77,8 @@
     uint8_t *cluster_cache;
     uint8_t *cluster_data;
     uint64_t cluster_cache_offset;
-    uint32_t crypt_method; /* current crypt method, 0 if no key yet */
+    QCryptoCipher *cipher; /* NULL if no key yet */
     uint32_t crypt_method_header;
-    AES_KEY aes_encrypt_key;
-    AES_KEY aes_decrypt_key;
     CoMutex lock;
     Error *migration_blocker;
 } BDRVQcowState;
@@ -120,11 +124,7 @@
         goto fail;
     }
     if (header.version != QCOW_VERSION) {
-        char version[64];
-        snprintf(version, sizeof(version), "QCOW version %" PRIu32,
-                 header.version);
-        error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                  bdrv_get_device_name(bs), "qcow", version);
+        error_setg(errp, "Unsupported qcow version %" PRIu32, header.version);
         ret = -ENOTSUP;
         goto fail;
     }
@@ -153,9 +153,28 @@
         ret = -EINVAL;
         goto fail;
     }
+    if (!qcrypto_cipher_supports(QCRYPTO_CIPHER_ALG_AES_128)) {
+        error_setg(errp, "AES cipher not available");
+        ret = -EINVAL;
+        goto fail;
+    }
     s->crypt_method_header = header.crypt_method;
     if (s->crypt_method_header) {
-        bs->encrypted = 1;
+        if (bdrv_uses_whitelist() &&
+            s->crypt_method_header == QCOW_CRYPT_AES) {
+            error_setg(errp,
+                       "Use of AES-CBC encrypted qcow images is no longer "
+                       "supported in system emulators");
+            error_append_hint(errp,
+                              "You can use 'qemu-img convert' to convert your "
+                              "image to an alternative supported format, such "
+                              "as unencrypted qcow, or raw with the LUKS "
+                              "format instead.\n");
+            ret = -ENOSYS;
+            goto fail;
+        }
+
+        bs->encrypted = true;
     }
     s->cluster_bits = header.cluster_bits;
     s->cluster_size = 1 << s->cluster_bits;
@@ -201,7 +220,7 @@
 
     /* alloc L2 cache (max. 64k * 16 * 8 = 8 MB) */
     s->l2_cache =
-        qemu_try_blockalign(bs->file,
+        qemu_try_blockalign(bs->file->bs,
                             s->l2_size * L2_CACHE_SIZE * sizeof(uint64_t));
     if (s->l2_cache == NULL) {
         error_setg(errp, "Could not allocate L2 table cache");
@@ -215,7 +234,7 @@
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
         len = header.backing_file_size;
-        if (len > 1023) {
+        if (len > 1023 || len >= sizeof(bs->backing_file)) {
             error_setg(errp, "Backing file name too long");
             ret = -EINVAL;
             goto fail;
@@ -229,9 +248,9 @@
     }
 
     /* Disable migration when qcow images are used */
-    error_set(&s->migration_blocker,
-              QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "qcow", bdrv_get_device_name(bs), "live migration");
+    error_setg(&s->migration_blocker, "The qcow format used by node '%s' "
+               "does not support live migration",
+               bdrv_get_device_or_node_name(bs));
     migrate_add_blocker(s->migration_blocker);
 
     qemu_co_mutex_init(&s->lock);
@@ -259,6 +278,7 @@
     BDRVQcowState *s = bs->opaque;
     uint8_t keybuf[16];
     int len, i;
+    Error *err;
 
     memset(keybuf, 0, 16);
     len = strlen(key);
@@ -269,38 +289,68 @@
     for(i = 0;i < len;i++) {
         keybuf[i] = key[i];
     }
-    s->crypt_method = s->crypt_method_header;
+    assert(bs->encrypted);
 
-    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
+    qcrypto_cipher_free(s->cipher);
+    s->cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_128,
+        QCRYPTO_CIPHER_MODE_CBC,
+        keybuf, G_N_ELEMENTS(keybuf),
+        &err);
+
+    if (!s->cipher) {
+        /* XXX would be nice if errors in this method could
+         * be properly propagate to the caller. Would need
+         * the bdrv_set_key() API signature to be fixed. */
+        error_free(err);
         return -1;
-    if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0)
-        return -1;
+    }
     return 0;
 }
 
 /* The crypt function is compatible with the linux cryptoloop
    algorithm for < 4 GB images. NOTE: out_buf == in_buf is
    supported */
-static void encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
-                            uint8_t *out_buf, const uint8_t *in_buf,
-                            int nb_sectors, int enc,
-                            const AES_KEY *key)
+static int encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
+                           uint8_t *out_buf, const uint8_t *in_buf,
+                           int nb_sectors, bool enc, Error **errp)
 {
     union {
         uint64_t ll[2];
         uint8_t b[16];
     } ivec;
     int i;
+    int ret;
 
     for(i = 0; i < nb_sectors; i++) {
         ivec.ll[0] = cpu_to_le64(sector_num);
         ivec.ll[1] = 0;
-        AES_cbc_encrypt(in_buf, out_buf, 512, key,
-                        ivec.b, enc);
+        if (qcrypto_cipher_setiv(s->cipher,
+                                 ivec.b, G_N_ELEMENTS(ivec.b),
+                                 errp) < 0) {
+            return -1;
+        }
+        if (enc) {
+            ret = qcrypto_cipher_encrypt(s->cipher,
+                                         in_buf,
+                                         out_buf,
+                                         512,
+                                         errp);
+        } else {
+            ret = qcrypto_cipher_decrypt(s->cipher,
+                                         in_buf,
+                                         out_buf,
+                                         512,
+                                         errp);
+        }
+        if (ret < 0) {
+            return -1;
+        }
         sector_num++;
         in_buf += 512;
         out_buf += 512;
     }
+    return 0;
 }
 
 /* 'allocate' is:
@@ -334,7 +384,7 @@
         if (!allocate)
             return 0;
         /* allocate a new l2 entry */
-        l2_offset = bdrv_getlength(bs->file);
+        l2_offset = bdrv_getlength(bs->file->bs);
         /* round to cluster size */
         l2_offset = (l2_offset + s->cluster_size - 1) & ~(s->cluster_size - 1);
         /* update the L1 entry */
@@ -374,7 +424,8 @@
                 s->l2_size * sizeof(uint64_t)) < 0)
             return 0;
     } else {
-        if (bdrv_pread(bs->file, l2_offset, l2_table, s->l2_size * sizeof(uint64_t)) !=
+        if (bdrv_pread(bs->file, l2_offset, l2_table,
+                       s->l2_size * sizeof(uint64_t)) !=
             s->l2_size * sizeof(uint64_t))
             return 0;
     }
@@ -395,34 +446,42 @@
                overwritten */
             if (decompress_cluster(bs, cluster_offset) < 0)
                 return 0;
-            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = bdrv_getlength(bs->file->bs);
             cluster_offset = (cluster_offset + s->cluster_size - 1) &
                 ~(s->cluster_size - 1);
             /* write the cluster content */
-            if (bdrv_pwrite(bs->file, cluster_offset, s->cluster_cache, s->cluster_size) !=
+            if (bdrv_pwrite(bs->file, cluster_offset, s->cluster_cache,
+                            s->cluster_size) !=
                 s->cluster_size)
                 return -1;
         } else {
-            cluster_offset = bdrv_getlength(bs->file);
+            cluster_offset = bdrv_getlength(bs->file->bs);
             if (allocate == 1) {
                 /* round to cluster size */
                 cluster_offset = (cluster_offset + s->cluster_size - 1) &
                     ~(s->cluster_size - 1);
-                bdrv_truncate(bs->file, cluster_offset + s->cluster_size);
+                bdrv_truncate(bs->file->bs, cluster_offset + s->cluster_size);
                 /* if encrypted, we must initialize the cluster
                    content which won't be written */
-                if (s->crypt_method &&
+                if (bs->encrypted &&
                     (n_end - n_start) < s->cluster_sectors) {
                     uint64_t start_sect;
+                    assert(s->cipher);
                     start_sect = (offset & ~(s->cluster_size - 1)) >> 9;
                     memset(s->cluster_data + 512, 0x00, 512);
                     for(i = 0; i < s->cluster_sectors; i++) {
                         if (i < n_start || i >= n_end) {
-                            encrypt_sectors(s, start_sect + i,
-                                            s->cluster_data,
-                                            s->cluster_data + 512, 1, 1,
-                                            &s->aes_encrypt_key);
-                            if (bdrv_pwrite(bs->file, cluster_offset + i * 512,
+                            Error *err = NULL;
+                            if (encrypt_sectors(s, start_sect + i,
+                                                s->cluster_data,
+                                                s->cluster_data + 512, 1,
+                                                true, &err) < 0) {
+                                error_free(err);
+                                errno = EIO;
+                                return -1;
+                            }
+                            if (bdrv_pwrite(bs->file,
+                                            cluster_offset + i * 512,
                                             s->cluster_data, 512) != 512)
                                 return -1;
                         }
@@ -444,7 +503,7 @@
 }
 
 static int64_t coroutine_fn qcow_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     BDRVQcowState *s = bs->opaque;
     int index_in_cluster, n;
@@ -461,10 +520,11 @@
     if (!cluster_offset) {
         return 0;
     }
-    if ((cluster_offset & QCOW_OFLAG_COMPRESSED) || s->crypt_method) {
+    if ((cluster_offset & QCOW_OFLAG_COMPRESSED) || s->cipher) {
         return BDRV_BLOCK_DATA;
     }
     cluster_offset |= (index_in_cluster << BDRV_SECTOR_BITS);
+    *file = bs->file->bs;
     return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | cluster_offset;
 }
 
@@ -528,6 +588,7 @@
     QEMUIOVector hd_qiov;
     uint8_t *buf;
     void *orig_buf;
+    Error *err = NULL;
 
     if (qiov->niov > 1) {
         buf = orig_buf = qemu_try_blockalign(bs, qiov->size);
@@ -552,14 +613,13 @@
         }
 
         if (!cluster_offset) {
-            if (bs->backing_hd) {
+            if (bs->backing) {
                 /* read from the base image */
                 hd_iov.iov_base = (void *)buf;
                 hd_iov.iov_len = n * 512;
                 qemu_iovec_init_external(&hd_qiov, &hd_iov, 1);
                 qemu_co_mutex_unlock(&s->lock);
-                ret = bdrv_co_readv(bs->backing_hd, sector_num,
-                                    n, &hd_qiov);
+                ret = bdrv_co_readv(bs->backing, sector_num, n, &hd_qiov);
                 qemu_co_mutex_lock(&s->lock);
                 if (ret < 0) {
                     goto fail;
@@ -590,10 +650,12 @@
             if (ret < 0) {
                 break;
             }
-            if (s->crypt_method) {
-                encrypt_sectors(s, sector_num, buf, buf,
-                                n, 0,
-                                &s->aes_decrypt_key);
+            if (bs->encrypted) {
+                assert(s->cipher);
+                if (encrypt_sectors(s, sector_num, buf, buf,
+                                    n, false, &err) < 0) {
+                    goto fail;
+                }
             }
         }
         ret = 0;
@@ -614,6 +676,7 @@
     return ret;
 
 fail:
+    error_free(err);
     ret = -EIO;
     goto done;
 }
@@ -661,12 +724,18 @@
             ret = -EIO;
             break;
         }
-        if (s->crypt_method) {
+        if (bs->encrypted) {
+            Error *err = NULL;
+            assert(s->cipher);
             if (!cluster_data) {
                 cluster_data = g_malloc0(s->cluster_size);
             }
-            encrypt_sectors(s, sector_num, cluster_data, buf,
-                            n, 1, &s->aes_encrypt_key);
+            if (encrypt_sectors(s, sector_num, cluster_data, buf,
+                                n, true, &err) < 0) {
+                error_free(err);
+                ret = -EIO;
+                break;
+            }
             src_buf = cluster_data;
         } else {
             src_buf = buf;
@@ -703,6 +772,8 @@
 {
     BDRVQcowState *s = bs->opaque;
 
+    qcrypto_cipher_free(s->cipher);
+    s->cipher = NULL;
     g_free(s->l1_table);
     qemu_vfree(s->l2_cache);
     g_free(s->cluster_cache);
@@ -722,7 +793,7 @@
     int flags = 0;
     Error *local_err = NULL;
     int ret;
-    BlockDriverState *qcow_bs;
+    BlockBackend *qcow_blk;
 
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
@@ -738,15 +809,17 @@
         goto cleanup;
     }
 
-    qcow_bs = NULL;
-    ret = bdrv_open(&qcow_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
-    if (ret < 0) {
+    qcow_blk = blk_new_open(filename, NULL, NULL,
+                            BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (qcow_blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto cleanup;
     }
 
-    ret = bdrv_truncate(qcow_bs, 0);
+    blk_set_allow_write_beyond_eof(qcow_blk, true);
+
+    ret = blk_truncate(qcow_blk, 0);
     if (ret < 0) {
         goto exit;
     }
@@ -786,24 +859,24 @@
     }
 
     /* write all the data */
-    ret = bdrv_pwrite(qcow_bs, 0, &header, sizeof(header));
+    ret = blk_pwrite(qcow_blk, 0, &header, sizeof(header), 0);
     if (ret != sizeof(header)) {
         goto exit;
     }
 
     if (backing_file) {
-        ret = bdrv_pwrite(qcow_bs, sizeof(header),
-            backing_file, backing_filename_len);
+        ret = blk_pwrite(qcow_blk, sizeof(header),
+                         backing_file, backing_filename_len, 0);
         if (ret != backing_filename_len) {
             goto exit;
         }
     }
 
     tmp = g_malloc0(BDRV_SECTOR_SIZE);
-    for (i = 0; i < ((sizeof(uint64_t)*l1_size + BDRV_SECTOR_SIZE - 1)/
-        BDRV_SECTOR_SIZE); i++) {
-        ret = bdrv_pwrite(qcow_bs, header_size +
-            BDRV_SECTOR_SIZE*i, tmp, BDRV_SECTOR_SIZE);
+    for (i = 0; i < DIV_ROUND_UP(sizeof(uint64_t) * l1_size, BDRV_SECTOR_SIZE);
+         i++) {
+        ret = blk_pwrite(qcow_blk, header_size + BDRV_SECTOR_SIZE * i,
+                         tmp, BDRV_SECTOR_SIZE, 0);
         if (ret != BDRV_SECTOR_SIZE) {
             g_free(tmp);
             goto exit;
@@ -813,7 +886,7 @@
     g_free(tmp);
     ret = 0;
 exit:
-    bdrv_unref(qcow_bs);
+    blk_unref(qcow_blk);
 cleanup:
     g_free(backing_file);
     return ret;
@@ -829,7 +902,7 @@
     if (bdrv_pwrite_sync(bs->file, s->l1_table_offset, s->l1_table,
             l1_length) < 0)
         return -1;
-    ret = bdrv_truncate(bs->file, s->l1_table_offset + l1_length);
+    ret = bdrv_truncate(bs->file->bs, s->l1_table_offset + l1_length);
     if (ret < 0)
         return ret;
 
@@ -840,6 +913,49 @@
     return 0;
 }
 
+typedef struct QcowWriteCo {
+    BlockDriverState *bs;
+    int64_t sector_num;
+    const uint8_t *buf;
+    int nb_sectors;
+    int ret;
+} QcowWriteCo;
+
+static void qcow_write_co_entry(void *opaque)
+{
+    QcowWriteCo *co = opaque;
+    QEMUIOVector qiov;
+
+    struct iovec iov = (struct iovec) {
+        .iov_base   = (uint8_t*) co->buf,
+        .iov_len    = co->nb_sectors * BDRV_SECTOR_SIZE,
+    };
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    co->ret = qcow_co_writev(co->bs, co->sector_num, co->nb_sectors, &qiov);
+}
+
+/* Wrapper for non-coroutine contexts */
+static int qcow_write(BlockDriverState *bs, int64_t sector_num,
+                      const uint8_t *buf, int nb_sectors)
+{
+    Coroutine *co;
+    AioContext *aio_context = bdrv_get_aio_context(bs);
+    QcowWriteCo data = {
+        .bs         = bs,
+        .sector_num = sector_num,
+        .buf        = buf,
+        .nb_sectors = nb_sectors,
+        .ret        = -EINPROGRESS,
+    };
+    co = qemu_coroutine_create(qcow_write_co_entry, &data);
+    qemu_coroutine_enter(co);
+    while (data.ret == -EINPROGRESS) {
+        aio_poll(aio_context, true);
+    }
+    return data.ret;
+}
+
 /* XXX: put compressed sectors first, then all the cluster aligned
    tables to avoid losing bytes in alignment */
 static int qcow_write_compressed(BlockDriverState *bs, int64_t sector_num,
@@ -867,7 +983,7 @@
         return ret;
     }
 
-    out_buf = g_malloc(s->cluster_size + (s->cluster_size / 1000) + 128);
+    out_buf = g_malloc(s->cluster_size);
 
     /* best compression, small window, no zlib header */
     memset(&strm, 0, sizeof(strm));
@@ -896,7 +1012,7 @@
 
     if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
         /* could not compress: write normal cluster */
-        ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
+        ret = qcow_write(bs, sector_num, buf, s->cluster_sectors);
         if (ret < 0) {
             goto fail;
         }
diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c
index 904f6b1..6eaefed 100644
--- a/block/qcow2-cache.c
+++ b/block/qcow2-cache.c
@@ -22,68 +22,127 @@
  * THE SOFTWARE.
  */
 
+/* Needed for CONFIG_MADVISE */
+#include "qemu/osdep.h"
 #include "block/block_int.h"
 #include "qemu-common.h"
 #include "qcow2.h"
 #include "trace.h"
 
 typedef struct Qcow2CachedTable {
-    void*   table;
-    int64_t offset;
-    bool    dirty;
-    int     cache_hits;
-    int     ref;
+    int64_t  offset;
+    uint64_t lru_counter;
+    int      ref;
+    bool     dirty;
 } Qcow2CachedTable;
 
 struct Qcow2Cache {
-    Qcow2CachedTable*       entries;
-    struct Qcow2Cache*      depends;
+    Qcow2CachedTable       *entries;
+    struct Qcow2Cache      *depends;
     int                     size;
     bool                    depends_on_flush;
+    void                   *table_array;
+    uint64_t                lru_counter;
+    uint64_t                cache_clean_lru_counter;
 };
 
+static inline void *qcow2_cache_get_table_addr(BlockDriverState *bs,
+                    Qcow2Cache *c, int table)
+{
+    BDRVQcow2State *s = bs->opaque;
+    return (uint8_t *) c->table_array + (size_t) table * s->cluster_size;
+}
+
+static inline int qcow2_cache_get_table_idx(BlockDriverState *bs,
+                  Qcow2Cache *c, void *table)
+{
+    BDRVQcow2State *s = bs->opaque;
+    ptrdiff_t table_offset = (uint8_t *) table - (uint8_t *) c->table_array;
+    int idx = table_offset / s->cluster_size;
+    assert(idx >= 0 && idx < c->size && table_offset % s->cluster_size == 0);
+    return idx;
+}
+
+static void qcow2_cache_table_release(BlockDriverState *bs, Qcow2Cache *c,
+                                      int i, int num_tables)
+{
+#if QEMU_MADV_DONTNEED != QEMU_MADV_INVALID
+    BDRVQcow2State *s = bs->opaque;
+    void *t = qcow2_cache_get_table_addr(bs, c, i);
+    int align = getpagesize();
+    size_t mem_size = (size_t) s->cluster_size * num_tables;
+    size_t offset = QEMU_ALIGN_UP((uintptr_t) t, align) - (uintptr_t) t;
+    size_t length = QEMU_ALIGN_DOWN(mem_size - offset, align);
+    if (length > 0) {
+        qemu_madvise((uint8_t *) t + offset, length, QEMU_MADV_DONTNEED);
+    }
+#endif
+}
+
+static inline bool can_clean_entry(Qcow2Cache *c, int i)
+{
+    Qcow2CachedTable *t = &c->entries[i];
+    return t->ref == 0 && !t->dirty && t->offset != 0 &&
+        t->lru_counter <= c->cache_clean_lru_counter;
+}
+
+void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c)
+{
+    int i = 0;
+    while (i < c->size) {
+        int to_clean = 0;
+
+        /* Skip the entries that we don't need to clean */
+        while (i < c->size && !can_clean_entry(c, i)) {
+            i++;
+        }
+
+        /* And count how many we can clean in a row */
+        while (i < c->size && can_clean_entry(c, i)) {
+            c->entries[i].offset = 0;
+            c->entries[i].lru_counter = 0;
+            i++;
+            to_clean++;
+        }
+
+        if (to_clean > 0) {
+            qcow2_cache_table_release(bs, c, i - to_clean, to_clean);
+        }
+    }
+
+    c->cache_clean_lru_counter = c->lru_counter;
+}
+
 Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2Cache *c;
-    int i;
 
     c = g_new0(Qcow2Cache, 1);
     c->size = num_tables;
     c->entries = g_try_new0(Qcow2CachedTable, num_tables);
-    if (!c->entries) {
-        goto fail;
-    }
+    c->table_array = qemu_try_blockalign(bs->file->bs,
+                                         (size_t) num_tables * s->cluster_size);
 
-    for (i = 0; i < c->size; i++) {
-        c->entries[i].table = qemu_try_blockalign(bs->file, s->cluster_size);
-        if (c->entries[i].table == NULL) {
-            goto fail;
-        }
+    if (!c->entries || !c->table_array) {
+        qemu_vfree(c->table_array);
+        g_free(c->entries);
+        g_free(c);
+        c = NULL;
     }
 
     return c;
-
-fail:
-    if (c->entries) {
-        for (i = 0; i < c->size; i++) {
-            qemu_vfree(c->entries[i].table);
-        }
-    }
-    g_free(c->entries);
-    g_free(c);
-    return NULL;
 }
 
-int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c)
+int qcow2_cache_destroy(BlockDriverState *bs, Qcow2Cache *c)
 {
     int i;
 
     for (i = 0; i < c->size; i++) {
         assert(c->entries[i].ref == 0);
-        qemu_vfree(c->entries[i].table);
     }
 
+    qemu_vfree(c->table_array);
     g_free(c->entries);
     g_free(c);
 
@@ -107,7 +166,7 @@
 
 static int qcow2_cache_entry_flush(BlockDriverState *bs, Qcow2Cache *c, int i)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret = 0;
 
     if (!c->entries[i].dirty || !c->entries[i].offset) {
@@ -120,7 +179,7 @@
     if (c->depends) {
         ret = qcow2_cache_flush_dependency(bs, c);
     } else if (c->depends_on_flush) {
-        ret = bdrv_flush(bs->file);
+        ret = bdrv_flush(bs->file->bs);
         if (ret >= 0) {
             c->depends_on_flush = false;
         }
@@ -151,8 +210,8 @@
         BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE);
     }
 
-    ret = bdrv_pwrite(bs->file, c->entries[i].offset, c->entries[i].table,
-        s->cluster_size);
+    ret = bdrv_pwrite(bs->file, c->entries[i].offset,
+                      qcow2_cache_get_table_addr(bs, c, i), s->cluster_size);
     if (ret < 0) {
         return ret;
     }
@@ -162,9 +221,9 @@
     return 0;
 }
 
-int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
+int qcow2_cache_write(BlockDriverState *bs, Qcow2Cache *c)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int result = 0;
     int ret;
     int i;
@@ -178,8 +237,15 @@
         }
     }
 
+    return result;
+}
+
+int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c)
+{
+    int result = qcow2_cache_write(bs, c);
+
     if (result == 0) {
-        ret = bdrv_flush(bs->file);
+        int ret = bdrv_flush(bs->file->bs);
         if (ret < 0) {
             result = ret;
         }
@@ -228,66 +294,55 @@
     for (i = 0; i < c->size; i++) {
         assert(c->entries[i].ref == 0);
         c->entries[i].offset = 0;
-        c->entries[i].cache_hits = 0;
+        c->entries[i].lru_counter = 0;
     }
 
+    qcow2_cache_table_release(bs, c, 0, c->size);
+
+    c->lru_counter = 0;
+
     return 0;
 }
 
-static int qcow2_cache_find_entry_to_replace(Qcow2Cache *c)
-{
-    int i;
-    int min_count = INT_MAX;
-    int min_index = -1;
-
-
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].ref) {
-            continue;
-        }
-
-        if (c->entries[i].cache_hits < min_count) {
-            min_index = i;
-            min_count = c->entries[i].cache_hits;
-        }
-
-        /* Give newer hits priority */
-        /* TODO Check how to optimize the replacement strategy */
-        c->entries[i].cache_hits /= 2;
-    }
-
-    if (min_index == -1) {
-        /* This can't happen in current synchronous code, but leave the check
-         * here as a reminder for whoever starts using AIO with the cache */
-        abort();
-    }
-    return min_index;
-}
-
 static int qcow2_cache_do_get(BlockDriverState *bs, Qcow2Cache *c,
     uint64_t offset, void **table, bool read_from_disk)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
     int ret;
+    int lookup_index;
+    uint64_t min_lru_counter = UINT64_MAX;
+    int min_lru_index = -1;
 
     trace_qcow2_cache_get(qemu_coroutine_self(), c == s->l2_table_cache,
                           offset, read_from_disk);
 
     /* Check if the table is already cached */
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].offset == offset) {
+    i = lookup_index = (offset / s->cluster_size * 4) % c->size;
+    do {
+        const Qcow2CachedTable *t = &c->entries[i];
+        if (t->offset == offset) {
             goto found;
         }
+        if (t->ref == 0 && t->lru_counter < min_lru_counter) {
+            min_lru_counter = t->lru_counter;
+            min_lru_index = i;
+        }
+        if (++i == c->size) {
+            i = 0;
+        }
+    } while (i != lookup_index);
+
+    if (min_lru_index == -1) {
+        /* This can't happen in current synchronous code, but leave the check
+         * here as a reminder for whoever starts using AIO with the cache */
+        abort();
     }
 
-    /* If not, write a table back and replace it */
-    i = qcow2_cache_find_entry_to_replace(c);
+    /* Cache miss: write a table back and replace it */
+    i = min_lru_index;
     trace_qcow2_cache_get_replace_entry(qemu_coroutine_self(),
                                         c == s->l2_table_cache, i);
-    if (i < 0) {
-        return i;
-    }
 
     ret = qcow2_cache_entry_flush(bs, c, i);
     if (ret < 0) {
@@ -302,22 +357,20 @@
             BLKDBG_EVENT(bs->file, BLKDBG_L2_LOAD);
         }
 
-        ret = bdrv_pread(bs->file, offset, c->entries[i].table, s->cluster_size);
+        ret = bdrv_pread(bs->file, offset,
+                         qcow2_cache_get_table_addr(bs, c, i),
+                         s->cluster_size);
         if (ret < 0) {
             return ret;
         }
     }
 
-    /* Give the table some hits for the start so that it won't be replaced
-     * immediately. The number 32 is completely arbitrary. */
-    c->entries[i].cache_hits = 32;
     c->entries[i].offset = offset;
 
     /* And return the right table */
 found:
-    c->entries[i].cache_hits++;
     c->entries[i].ref++;
-    *table = c->entries[i].table;
+    *table = qcow2_cache_get_table_addr(bs, c, i);
 
     trace_qcow2_cache_get_done(qemu_coroutine_self(),
                                c == s->l2_table_cache, i);
@@ -337,36 +390,24 @@
     return qcow2_cache_do_get(bs, c, offset, table, false);
 }
 
-int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table)
+void qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table)
 {
-    int i;
+    int i = qcow2_cache_get_table_idx(bs, c, *table);
 
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].table == *table) {
-            goto found;
-        }
-    }
-    return -ENOENT;
-
-found:
     c->entries[i].ref--;
     *table = NULL;
 
+    if (c->entries[i].ref == 0) {
+        c->entries[i].lru_counter = ++c->lru_counter;
+    }
+
     assert(c->entries[i].ref >= 0);
-    return 0;
 }
 
-void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table)
+void qcow2_cache_entry_mark_dirty(BlockDriverState *bs, Qcow2Cache *c,
+     void *table)
 {
-    int i;
-
-    for (i = 0; i < c->size; i++) {
-        if (c->entries[i].table == table) {
-            goto found;
-        }
-    }
-    abort();
-
-found:
+    int i = qcow2_cache_get_table_idx(bs, c, table);
+    assert(c->entries[i].offset != 0);
     c->entries[i].dirty = true;
 }
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index df0b2c9..f941835 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -22,17 +22,20 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <zlib.h>
 
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "block/qcow2.h"
+#include "qemu/bswap.h"
 #include "trace.h"
 
 int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
                         bool exact_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int new_l1_size2, ret, i;
     uint64_t *new_l1_table;
     int64_t old_l1_table_offset, old_l1_size;
@@ -62,7 +65,8 @@
         }
     }
 
-    if (new_l1_size > INT_MAX / sizeof(uint64_t)) {
+    QEMU_BUILD_BUG_ON(QCOW_MAX_L1_SIZE > INT_MAX);
+    if (new_l1_size > QCOW_MAX_L1_SIZE / sizeof(uint64_t)) {
         return -EFBIG;
     }
 
@@ -72,7 +76,7 @@
 #endif
 
     new_l1_size2 = sizeof(uint64_t) * new_l1_size;
-    new_l1_table = qemu_try_blockalign(bs->file,
+    new_l1_table = qemu_try_blockalign(bs->file->bs,
                                        align_offset(new_l1_size2, 512));
     if (new_l1_table == NULL) {
         return -ENOMEM;
@@ -105,7 +109,8 @@
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_WRITE_TABLE);
     for(i = 0; i < s->l1_size; i++)
         new_l1_table[i] = cpu_to_be64(new_l1_table[i]);
-    ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset, new_l1_table, new_l1_size2);
+    ret = bdrv_pwrite_sync(bs->file, new_l1_table_offset,
+                           new_l1_table, new_l1_size2);
     if (ret < 0)
         goto fail;
     for(i = 0; i < s->l1_size; i++)
@@ -113,9 +118,10 @@
 
     /* set new table */
     BLKDBG_EVENT(bs->file, BLKDBG_L1_GROW_ACTIVATE_TABLE);
-    cpu_to_be32w((uint32_t*)data, new_l1_size);
+    stl_be_p(data, new_l1_size);
     stq_be_p(data + 4, new_l1_table_offset);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size), data,sizeof(data));
+    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_size),
+                           data, sizeof(data));
     if (ret < 0) {
         goto fail;
     }
@@ -148,12 +154,10 @@
 static int l2_load(BlockDriverState *bs, uint64_t l2_offset,
     uint64_t **l2_table)
 {
-    BDRVQcowState *s = bs->opaque;
-    int ret;
+    BDRVQcow2State *s = bs->opaque;
 
-    ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset, (void**) l2_table);
-
-    return ret;
+    return qcow2_cache_get(bs, s->l2_table_cache, l2_offset,
+                           (void **)l2_table);
 }
 
 /*
@@ -163,7 +167,7 @@
 #define L1_ENTRIES_PER_SECTOR (512 / 8)
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t buf[L1_ENTRIES_PER_SECTOR] = { 0 };
     int l1_start_index;
     int i, ret;
@@ -182,8 +186,9 @@
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_L1_UPDATE);
-    ret = bdrv_pwrite_sync(bs->file, s->l1_table_offset + 8 * l1_start_index,
-        buf, sizeof(buf));
+    ret = bdrv_pwrite_sync(bs->file,
+                           s->l1_table_offset + 8 * l1_start_index,
+                           buf, sizeof(buf));
     if (ret < 0) {
         return ret;
     }
@@ -203,7 +208,7 @@
 
 static int l2_allocate(BlockDriverState *bs, int l1_index, uint64_t **table)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t old_l2_offset;
     uint64_t *l2_table = NULL;
     int64_t l2_offset;
@@ -253,17 +258,14 @@
 
         memcpy(l2_table, old_table, s->cluster_size);
 
-        ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &old_table);
-        if (ret < 0) {
-            goto fail;
-        }
+        qcow2_cache_put(bs, s->l2_table_cache, (void **) &old_table);
     }
 
     /* write the l2 table to the file */
     BLKDBG_EVENT(bs->file, BLKDBG_L2_ALLOC_WRITE);
 
     trace_qcow2_l2_allocate_write_l2(bs, l1_index);
-    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
     ret = qcow2_cache_flush(bs, s->l2_table_cache);
     if (ret < 0) {
         goto fail;
@@ -301,7 +303,7 @@
  * as contiguous. (This allows it, for example, to stop at the first compressed
  * cluster which may require a different handling)
  */
-static int count_contiguous_clusters(uint64_t nb_clusters, int cluster_size,
+static int count_contiguous_clusters(int nb_clusters, int cluster_size,
         uint64_t *l2_table, uint64_t stop_flags)
 {
     int i;
@@ -312,7 +314,7 @@
     if (!offset)
         return 0;
 
-    assert(qcow2_get_cluster_type(first_entry) != QCOW2_CLUSTER_COMPRESSED);
+    assert(qcow2_get_cluster_type(first_entry) == QCOW2_CLUSTER_NORMAL);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t l2_entry = be64_to_cpu(l2_table[i]) & mask;
@@ -324,14 +326,16 @@
 	return i;
 }
 
-static int count_contiguous_free_clusters(uint64_t nb_clusters, uint64_t *l2_table)
+static int count_contiguous_clusters_by_type(int nb_clusters,
+                                             uint64_t *l2_table,
+                                             int wanted_type)
 {
     int i;
 
     for (i = 0; i < nb_clusters; i++) {
         int type = qcow2_get_cluster_type(be64_to_cpu(l2_table[i]));
 
-        if (type != QCOW2_CLUSTER_UNALLOCATED) {
+        if (type != wanted_type) {
             break;
         }
     }
@@ -342,44 +346,61 @@
 /* The crypt function is compatible with the linux cryptoloop
    algorithm for < 4 GB images. NOTE: out_buf == in_buf is
    supported */
-void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
-                           uint8_t *out_buf, const uint8_t *in_buf,
-                           int nb_sectors, int enc,
-                           const AES_KEY *key)
+int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,
+                          uint8_t *out_buf, const uint8_t *in_buf,
+                          int nb_sectors, bool enc,
+                          Error **errp)
 {
     union {
         uint64_t ll[2];
         uint8_t b[16];
     } ivec;
     int i;
+    int ret;
 
     for(i = 0; i < nb_sectors; i++) {
         ivec.ll[0] = cpu_to_le64(sector_num);
         ivec.ll[1] = 0;
-        AES_cbc_encrypt(in_buf, out_buf, 512, key,
-                        ivec.b, enc);
+        if (qcrypto_cipher_setiv(s->cipher,
+                                 ivec.b, G_N_ELEMENTS(ivec.b),
+                                 errp) < 0) {
+            return -1;
+        }
+        if (enc) {
+            ret = qcrypto_cipher_encrypt(s->cipher,
+                                         in_buf,
+                                         out_buf,
+                                         512,
+                                         errp);
+        } else {
+            ret = qcrypto_cipher_decrypt(s->cipher,
+                                         in_buf,
+                                         out_buf,
+                                         512,
+                                         errp);
+        }
+        if (ret < 0) {
+            return -1;
+        }
         sector_num++;
         in_buf += 512;
         out_buf += 512;
     }
+    return 0;
 }
 
-static int coroutine_fn copy_sectors(BlockDriverState *bs,
-                                     uint64_t start_sect,
-                                     uint64_t cluster_offset,
-                                     int n_start, int n_end)
+static int coroutine_fn do_perform_cow(BlockDriverState *bs,
+                                       uint64_t src_cluster_offset,
+                                       uint64_t cluster_offset,
+                                       int offset_in_cluster,
+                                       int bytes)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QEMUIOVector qiov;
     struct iovec iov;
-    int n, ret;
+    int ret;
 
-    n = n_end - n_start;
-    if (n <= 0) {
-        return 0;
-    }
-
-    iov.iov_len = n * BDRV_SECTOR_SIZE;
+    iov.iov_len = bytes;
     iov.iov_base = qemu_try_blockalign(bs, iov.iov_len);
     if (iov.iov_base == NULL) {
         return -ENOMEM;
@@ -398,25 +419,36 @@
      * interface.  This avoids double I/O throttling and request tracking,
      * which can lead to deadlock when block layer copy-on-read is enabled.
      */
-    ret = bs->drv->bdrv_co_readv(bs, start_sect + n_start, n, &qiov);
+    ret = bs->drv->bdrv_co_preadv(bs, src_cluster_offset + offset_in_cluster,
+                                  bytes, &qiov, 0);
     if (ret < 0) {
         goto out;
     }
 
-    if (s->crypt_method) {
-        qcow2_encrypt_sectors(s, start_sect + n_start,
-                        iov.iov_base, iov.iov_base, n, 1,
-                        &s->aes_encrypt_key);
+    if (bs->encrypted) {
+        Error *err = NULL;
+        int64_t sector = (cluster_offset + offset_in_cluster)
+                         >> BDRV_SECTOR_BITS;
+        assert(s->cipher);
+        assert((offset_in_cluster & ~BDRV_SECTOR_MASK) == 0);
+        assert((bytes & ~BDRV_SECTOR_MASK) == 0);
+        if (qcow2_encrypt_sectors(s, sector, iov.iov_base, iov.iov_base,
+                                  bytes >> BDRV_SECTOR_BITS, true, &err) < 0) {
+            ret = -EIO;
+            error_free(err);
+            goto out;
+        }
     }
 
     ret = qcow2_pre_write_overlap_check(bs, 0,
-            cluster_offset + n_start * BDRV_SECTOR_SIZE, n * BDRV_SECTOR_SIZE);
+            cluster_offset + offset_in_cluster, bytes);
     if (ret < 0) {
         goto out;
     }
 
     BLKDBG_EVENT(bs->file, BLKDBG_COW_WRITE);
-    ret = bdrv_co_writev(bs->file, (cluster_offset >> 9) + n_start, n, &qiov);
+    ret = bdrv_co_pwritev(bs->file, cluster_offset + offset_in_cluster,
+                          bytes, &qiov, 0);
     if (ret < 0) {
         goto out;
     }
@@ -431,50 +463,47 @@
 /*
  * get_cluster_offset
  *
- * For a given offset of the disk image, find the cluster offset in
- * qcow2 file. The offset is stored in *cluster_offset.
+ * For a given offset of the virtual disk, find the cluster type and offset in
+ * the qcow2 file. The offset is stored in *cluster_offset.
  *
- * on entry, *num is the number of contiguous sectors we'd like to
- * access following offset.
+ * On entry, *bytes is the maximum number of contiguous bytes starting at
+ * offset that we are interested in.
  *
- * on exit, *num is the number of contiguous sectors we can read.
+ * On exit, *bytes is the number of bytes starting at offset that have the same
+ * cluster type and (if applicable) are stored contiguously in the image file.
+ * Compressed clusters are always returned one by one.
  *
  * Returns the cluster type (QCOW2_CLUSTER_*) on success, -errno in error
  * cases.
  */
 int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int *num, uint64_t *cluster_offset)
+                             unsigned int *bytes, uint64_t *cluster_offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int l2_index;
     uint64_t l1_index, l2_offset, *l2_table;
     int l1_bits, c;
-    unsigned int index_in_cluster, nb_clusters;
-    uint64_t nb_available, nb_needed;
+    unsigned int offset_in_cluster;
+    uint64_t bytes_available, bytes_needed, nb_clusters;
     int ret;
 
-    index_in_cluster = (offset >> 9) & (s->cluster_sectors - 1);
-    nb_needed = *num + index_in_cluster;
+    offset_in_cluster = offset_into_cluster(s, offset);
+    bytes_needed = (uint64_t) *bytes + offset_in_cluster;
 
     l1_bits = s->l2_bits + s->cluster_bits;
 
-    /* compute how many bytes there are between the offset and
-     * the end of the l1 entry
-     */
+    /* compute how many bytes there are between the start of the cluster
+     * containing offset and the end of the l1 entry */
+    bytes_available = (1ULL << l1_bits) - (offset & ((1ULL << l1_bits) - 1))
+                    + offset_in_cluster;
 
-    nb_available = (1ULL << l1_bits) - (offset & ((1ULL << l1_bits) - 1));
-
-    /* compute the number of available sectors */
-
-    nb_available = (nb_available >> 9) + index_in_cluster;
-
-    if (nb_needed > nb_available) {
-        nb_needed = nb_available;
+    if (bytes_needed > bytes_available) {
+        bytes_needed = bytes_available;
     }
 
     *cluster_offset = 0;
 
-    /* seek the the l2 offset in the l1 table */
+    /* seek to the l2 offset in the l1 table */
 
     l1_index = offset >> l1_bits;
     if (l1_index >= s->l1_size) {
@@ -506,7 +535,12 @@
 
     l2_index = (offset >> s->cluster_bits) & (s->l2_size - 1);
     *cluster_offset = be64_to_cpu(l2_table[l2_index]);
-    nb_clusters = size_to_clusters(s, nb_needed << 9);
+
+    nb_clusters = size_to_clusters(s, bytes_needed);
+    /* bytes_needed <= *bytes + offset_in_cluster, both of which are unsigned
+     * integers; the minimum cluster size is 512, so this assertion is always
+     * true */
+    assert(nb_clusters <= INT_MAX);
 
     ret = qcow2_get_cluster_type(*cluster_offset);
     switch (ret) {
@@ -523,13 +557,14 @@
             ret = -EIO;
             goto fail;
         }
-        c = count_contiguous_clusters(nb_clusters, s->cluster_size,
-                &l2_table[l2_index], QCOW_OFLAG_ZERO);
+        c = count_contiguous_clusters_by_type(nb_clusters, &l2_table[l2_index],
+                                              QCOW2_CLUSTER_ZERO);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_UNALLOCATED:
         /* how many empty clusters ? */
-        c = count_contiguous_free_clusters(nb_clusters, &l2_table[l2_index]);
+        c = count_contiguous_clusters_by_type(nb_clusters, &l2_table[l2_index],
+                                              QCOW2_CLUSTER_UNALLOCATED);
         *cluster_offset = 0;
         break;
     case QCOW2_CLUSTER_NORMAL:
@@ -552,13 +587,18 @@
 
     qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
 
-    nb_available = (c * s->cluster_sectors);
+    bytes_available = (int64_t)c * s->cluster_size;
 
 out:
-    if (nb_available > nb_needed)
-        nb_available = nb_needed;
+    if (bytes_available > bytes_needed) {
+        bytes_available = bytes_needed;
+    }
 
-    *num = nb_available - index_in_cluster;
+    /* bytes_available <= bytes_needed <= *bytes + offset_in_cluster;
+     * subtracting offset_in_cluster will therefore definitely yield something
+     * not exceeding UINT_MAX */
+    assert(bytes_available - offset_in_cluster <= UINT_MAX);
+    *bytes = bytes_available - offset_in_cluster;
 
     return ret;
 
@@ -582,13 +622,13 @@
                              uint64_t **new_l2_table,
                              int *new_l2_index)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int l2_index;
     uint64_t l1_index, l2_offset;
     uint64_t *l2_table = NULL;
     int ret;
 
-    /* seek the the l2 offset in the l1 table */
+    /* seek to the l2 offset in the l1 table */
 
     l1_index = offset >> (s->l2_bits + s->cluster_bits);
     if (l1_index >= s->l1_size) {
@@ -656,7 +696,7 @@
                                                uint64_t offset,
                                                int compressed_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index, ret;
     uint64_t *l2_table;
     int64_t cluster_offset;
@@ -692,29 +732,24 @@
     /* compressed clusters never have the copied flag */
 
     BLKDBG_EVENT(bs->file, BLKDBG_L2_UPDATE_COMPRESSED);
-    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
     l2_table[l2_index] = cpu_to_be64(cluster_offset);
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return 0;
-    }
+    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
     return cluster_offset;
 }
 
 static int perform_cow(BlockDriverState *bs, QCowL2Meta *m, Qcow2COWRegion *r)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
-    if (r->nb_sectors == 0) {
+    if (r->nb_bytes == 0) {
         return 0;
     }
 
     qemu_co_mutex_unlock(&s->lock);
-    ret = copy_sectors(bs, m->offset / BDRV_SECTOR_SIZE, m->alloc_offset,
-                       r->offset / BDRV_SECTOR_SIZE,
-                       r->offset / BDRV_SECTOR_SIZE + r->nb_sectors);
+    ret = do_perform_cow(bs, m->offset, m->alloc_offset, r->offset, r->nb_bytes);
     qemu_co_mutex_lock(&s->lock);
 
     if (ret < 0) {
@@ -733,7 +768,7 @@
 
 int qcow2_alloc_cluster_link_l2(BlockDriverState *bs, QCowL2Meta *m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i, j = 0, l2_index, ret;
     uint64_t *old_cluster, *l2_table;
     uint64_t cluster_offset = m->alloc_offset;
@@ -771,32 +806,29 @@
     if (ret < 0) {
         goto err;
     }
-    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
 
     assert(l2_index + m->nb_clusters <= s->l2_size);
     for (i = 0; i < m->nb_clusters; i++) {
         /* if two concurrent writes happen to the same unallocated cluster
-	 * each write allocates separate cluster and writes data concurrently.
-	 * The first one to complete updates l2 table with pointer to its
-	 * cluster the second one has to do RMW (which is done above by
-	 * copy_sectors()), update l2 table with its cluster pointer and free
-	 * old cluster. This is what this loop does */
-        if(l2_table[l2_index + i] != 0)
+         * each write allocates separate cluster and writes data concurrently.
+         * The first one to complete updates l2 table with pointer to its
+         * cluster the second one has to do RMW (which is done above by
+         * perform_cow()), update l2 table with its cluster pointer and free
+         * old cluster. This is what this loop does */
+        if (l2_table[l2_index + i] != 0) {
             old_cluster[j++] = l2_table[l2_index + i];
+        }
 
         l2_table[l2_index + i] = cpu_to_be64((cluster_offset +
                     (i << s->cluster_bits)) | QCOW_OFLAG_COPIED);
      }
 
 
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        goto err;
-    }
+    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
     /*
      * If this was a COW, we need to decrease the refcount of the old cluster.
-     * Also flush bs->file to get the right order for L2 and refcount update.
      *
      * Don't discard clusters that reach a refcount of 0 (e.g. compressed
      * clusters), the next write will reuse them anyway.
@@ -819,7 +851,7 @@
  * write, but require COW to be performed (this includes yet unallocated space,
  * which must copy from the backing file)
  */
-static int count_cow_clusters(BDRVQcowState *s, int nb_clusters,
+static int count_cow_clusters(BDRVQcow2State *s, int nb_clusters,
     uint64_t *l2_table, int l2_index)
 {
     int i;
@@ -865,7 +897,7 @@
 static int handle_dependencies(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *cur_bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowL2Meta *old_alloc;
     uint64_t bytes = *cur_bytes;
 
@@ -938,13 +970,13 @@
 static int handle_copied(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index;
     uint64_t cluster_offset;
     uint64_t *l2_table;
-    unsigned int nb_clusters;
+    uint64_t nb_clusters;
     unsigned int keep_clusters;
-    int ret, pret;
+    int ret;
 
     trace_qcow2_handle_copied(qemu_coroutine_self(), guest_offset, *host_offset,
                               *bytes);
@@ -961,6 +993,7 @@
 
     l2_index = offset_to_l2_index(s, guest_offset);
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     /* Find L2 entry for the first involved cluster */
     ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
@@ -1011,10 +1044,7 @@
 
     /* Cleanup */
 out:
-    pret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (pret < 0) {
-        return pret;
-    }
+    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
     /* Only return a host offset if we actually made progress. Otherwise we
      * would make requirements for handle_alloc() that it can't fulfill */
@@ -1046,9 +1076,9 @@
  * restarted, but the whole request should not be failed.
  */
 static int do_alloc_cluster_offset(BlockDriverState *bs, uint64_t guest_offset,
-    uint64_t *host_offset, unsigned int *nb_clusters)
+                                   uint64_t *host_offset, uint64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     trace_qcow2_do_alloc_clusters_offset(qemu_coroutine_self(), guest_offset,
                                          *host_offset, *nb_clusters);
@@ -1064,7 +1094,7 @@
         *host_offset = cluster_offset;
         return 0;
     } else {
-        int ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
+        int64_t ret = qcow2_alloc_clusters_at(bs, *host_offset, *nb_clusters);
         if (ret < 0) {
             return ret;
         }
@@ -1096,11 +1126,11 @@
 static int handle_alloc(BlockDriverState *bs, uint64_t guest_offset,
     uint64_t *host_offset, uint64_t *bytes, QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int l2_index;
     uint64_t *l2_table;
     uint64_t entry;
-    unsigned int nb_clusters;
+    uint64_t nb_clusters;
     int ret;
 
     uint64_t alloc_cluster_offset;
@@ -1118,6 +1148,7 @@
 
     l2_index = offset_to_l2_index(s, guest_offset);
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     /* Find L2 entry for the first involved cluster */
     ret = get_cluster_table(bs, guest_offset, &l2_table, &l2_index);
@@ -1139,10 +1170,7 @@
      * wrong with our code. */
     assert(nb_clusters > 0);
 
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
+    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
     /* Allocate, if necessary at a given offset in the image file */
     alloc_cluster_offset = start_of_cluster(s, *host_offset);
@@ -1172,25 +1200,20 @@
     /*
      * Save info needed for meta data update.
      *
-     * requested_sectors: Number of sectors from the start of the first
+     * requested_bytes: Number of bytes from the start of the first
      * newly allocated cluster to the end of the (possibly shortened
      * before) write request.
      *
-     * avail_sectors: Number of sectors from the start of the first
+     * avail_bytes: Number of bytes from the start of the first
      * newly allocated to the end of the last newly allocated cluster.
      *
-     * nb_sectors: The number of sectors from the start of the first
+     * nb_bytes: The number of bytes from the start of the first
      * newly allocated cluster to the end of the area that the write
      * request actually writes to (excluding COW at the end)
      */
-    int requested_sectors =
-        (*bytes + offset_into_cluster(s, guest_offset))
-        >> BDRV_SECTOR_BITS;
-    int avail_sectors = nb_clusters
-                        << (s->cluster_bits - BDRV_SECTOR_BITS);
-    int alloc_n_start = offset_into_cluster(s, guest_offset)
-                        >> BDRV_SECTOR_BITS;
-    int nb_sectors = MIN(requested_sectors, avail_sectors);
+    uint64_t requested_bytes = *bytes + offset_into_cluster(s, guest_offset);
+    int avail_bytes = MIN(INT_MAX, nb_clusters << s->cluster_bits);
+    int nb_bytes = MIN(requested_bytes, avail_bytes);
     QCowL2Meta *old_m = *m;
 
     *m = g_malloc0(sizeof(**m));
@@ -1201,23 +1224,21 @@
         .alloc_offset   = alloc_cluster_offset,
         .offset         = start_of_cluster(s, guest_offset),
         .nb_clusters    = nb_clusters,
-        .nb_available   = nb_sectors,
 
         .cow_start = {
             .offset     = 0,
-            .nb_sectors = alloc_n_start,
+            .nb_bytes   = offset_into_cluster(s, guest_offset),
         },
         .cow_end = {
-            .offset     = nb_sectors * BDRV_SECTOR_SIZE,
-            .nb_sectors = avail_sectors - nb_sectors,
+            .offset     = nb_bytes,
+            .nb_bytes   = avail_bytes - nb_bytes,
         },
     };
     qemu_co_queue_init(&(*m)->dependent_requests);
     QLIST_INSERT_HEAD(&s->cluster_allocs, *m, next_in_flight);
 
     *host_offset = alloc_cluster_offset + offset_into_cluster(s, guest_offset);
-    *bytes = MIN(*bytes, (nb_sectors * BDRV_SECTOR_SIZE)
-                         - offset_into_cluster(s, guest_offset));
+    *bytes = MIN(*bytes, nb_bytes - offset_into_cluster(s, guest_offset));
     assert(*bytes != 0);
 
     return 1;
@@ -1249,21 +1270,20 @@
  * Return 0 on success and -errno in error cases
  */
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int *num, uint64_t *host_offset, QCowL2Meta **m)
+                               unsigned int *bytes, uint64_t *host_offset,
+                               QCowL2Meta **m)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start, remaining;
     uint64_t cluster_offset;
     uint64_t cur_bytes;
     int ret;
 
-    trace_qcow2_alloc_clusters_offset(qemu_coroutine_self(), offset, *num);
-
-    assert((offset & ~BDRV_SECTOR_MASK) == 0);
+    trace_qcow2_alloc_clusters_offset(qemu_coroutine_self(), offset, *bytes);
 
 again:
     start = offset;
-    remaining = *num << BDRV_SECTOR_BITS;
+    remaining = *bytes;
     cluster_offset = 0;
     *host_offset = 0;
     cur_bytes = 0;
@@ -1349,8 +1369,8 @@
         }
     }
 
-    *num -= remaining >> BDRV_SECTOR_BITS;
-    assert(*num > 0);
+    *bytes -= remaining;
+    assert(*bytes > 0);
     assert(*host_offset != 0);
 
     return 0;
@@ -1385,7 +1405,7 @@
 
 int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret, csize, nb_csectors, sector_offset;
     uint64_t coffset;
 
@@ -1395,7 +1415,8 @@
         sector_offset = coffset & 511;
         csize = nb_csectors * 512 - sector_offset;
         BLKDBG_EVENT(bs->file, BLKDBG_READ_COMPRESSED);
-        ret = bdrv_read(bs->file, coffset >> 9, s->cluster_data, nb_csectors);
+        ret = bdrv_read(bs->file, coffset >> 9, s->cluster_data,
+                        nb_csectors);
         if (ret < 0) {
             return ret;
         }
@@ -1414,9 +1435,10 @@
  * clusters.
  */
 static int discard_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters, enum qcow2_discard_type type, bool full_discard)
+                             uint64_t nb_clusters, enum qcow2_discard_type type,
+                             bool full_discard)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table;
     int l2_index;
     int ret;
@@ -1429,6 +1451,7 @@
 
     /* Limit nb_clusters to one L2 table */
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t old_l2_entry;
@@ -1450,7 +1473,7 @@
          */
         switch (qcow2_get_cluster_type(old_l2_entry)) {
             case QCOW2_CLUSTER_UNALLOCATED:
-                if (full_discard || !bs->backing_hd) {
+                if (full_discard || !bs->backing) {
                     continue;
                 }
                 break;
@@ -1470,7 +1493,7 @@
         }
 
         /* First remove L2 entries */
-        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+        qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
         if (!full_discard && s->qcow_version >= 3) {
             l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
         } else {
@@ -1481,10 +1504,7 @@
         qcow2_free_any_clusters(bs, old_l2_entry, 1, type);
     }
 
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
+    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
     return nb_clusters;
 }
@@ -1492,9 +1512,9 @@
 int qcow2_discard_clusters(BlockDriverState *bs, uint64_t offset,
     int nb_sectors, enum qcow2_discard_type type, bool full_discard)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t end_offset;
-    unsigned int nb_clusters;
+    uint64_t nb_clusters;
     int ret;
 
     end_offset = offset + (nb_sectors << BDRV_SECTOR_BITS);
@@ -1536,9 +1556,9 @@
  * clusters.
  */
 static int zero_single_l2(BlockDriverState *bs, uint64_t offset,
-    unsigned int nb_clusters)
+                          uint64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table;
     int l2_index;
     int ret;
@@ -1551,6 +1571,7 @@
 
     /* Limit nb_clusters to one L2 table */
     nb_clusters = MIN(nb_clusters, s->l2_size - l2_index);
+    assert(nb_clusters <= INT_MAX);
 
     for (i = 0; i < nb_clusters; i++) {
         uint64_t old_offset;
@@ -1558,7 +1579,7 @@
         old_offset = be64_to_cpu(l2_table[l2_index + i]);
 
         /* Update L2 entries */
-        qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+        qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
         if (old_offset & QCOW_OFLAG_COMPRESSED) {
             l2_table[l2_index + i] = cpu_to_be64(QCOW_OFLAG_ZERO);
             qcow2_free_any_clusters(bs, old_offset, 1, QCOW2_DISCARD_REQUEST);
@@ -1567,18 +1588,15 @@
         }
     }
 
-    ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-    if (ret < 0) {
-        return ret;
-    }
+    qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
     return nb_clusters;
 }
 
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors)
 {
-    BDRVQcowState *s = bs->opaque;
-    unsigned int nb_clusters;
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t nb_clusters;
     int ret;
 
     /* The zero flag is only supported by version 3 and newer */
@@ -1620,9 +1638,10 @@
 static int expand_zero_clusters_in_l1(BlockDriverState *bs, uint64_t *l1_table,
                                       int l1_size, int64_t *visited_l1_entries,
                                       int64_t l1_entries,
-                                      BlockDriverAmendStatusCB *status_cb)
+                                      BlockDriverAmendStatusCB *status_cb,
+                                      void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bool is_active_l1 = (l1_table == s->l1_table);
     uint64_t *l2_table = NULL;
     int ret;
@@ -1631,7 +1650,7 @@
     if (!is_active_l1) {
         /* inactive L2 tables require a buffer to be stored in when loading
          * them from disk */
-        l2_table = qemu_try_blockalign(bs->file, s->cluster_size);
+        l2_table = qemu_try_blockalign(bs->file->bs, s->cluster_size);
         if (l2_table == NULL) {
             return -ENOMEM;
         }
@@ -1640,17 +1659,25 @@
     for (i = 0; i < l1_size; i++) {
         uint64_t l2_offset = l1_table[i] & L1E_OFFSET_MASK;
         bool l2_dirty = false;
-        int l2_refcount;
+        uint64_t l2_refcount;
 
         if (!l2_offset) {
             /* unallocated */
             (*visited_l1_entries)++;
             if (status_cb) {
-                status_cb(bs, *visited_l1_entries, l1_entries);
+                status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
             }
             continue;
         }
 
+        if (offset_into_cluster(s, l2_offset)) {
+            qcow2_signal_corruption(bs, true, -1, -1, "L2 table offset %#"
+                                    PRIx64 " unaligned (L1 index: %#x)",
+                                    l2_offset, i);
+            ret = -EIO;
+            goto fail;
+        }
+
         if (is_active_l1) {
             /* get active L2 tables from cache */
             ret = qcow2_cache_get(bs, s->l2_table_cache, l2_offset,
@@ -1658,15 +1685,15 @@
         } else {
             /* load inactive L2 tables from disk */
             ret = bdrv_read(bs->file, l2_offset / BDRV_SECTOR_SIZE,
-                    (void *)l2_table, s->cluster_sectors);
+                            (void *)l2_table, s->cluster_sectors);
         }
         if (ret < 0) {
             goto fail;
         }
 
-        l2_refcount = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits);
-        if (l2_refcount < 0) {
-            ret = l2_refcount;
+        ret = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits,
+                                 &l2_refcount);
+        if (ret < 0) {
             goto fail;
         }
 
@@ -1681,7 +1708,7 @@
             }
 
             if (!preallocated) {
-                if (!bs->backing_hd) {
+                if (!bs->backing) {
                     /* not backed; therefore we can simply deallocate the
                      * cluster */
                     l2_table[j] = 0;
@@ -1699,7 +1726,8 @@
                     /* For shared L2 tables, set the refcount accordingly (it is
                      * already 1 and needs to be l2_refcount) */
                     ret = qcow2_update_cluster_refcount(bs,
-                            offset >> s->cluster_bits, l2_refcount - 1,
+                            offset >> s->cluster_bits,
+                            refcount_diff(1, l2_refcount), false,
                             QCOW2_DISCARD_OTHER);
                     if (ret < 0) {
                         qcow2_free_clusters(bs, offset, s->cluster_size,
@@ -1709,6 +1737,19 @@
                 }
             }
 
+            if (offset_into_cluster(s, offset)) {
+                qcow2_signal_corruption(bs, true, -1, -1, "Data cluster offset "
+                                        "%#" PRIx64 " unaligned (L2 offset: %#"
+                                        PRIx64 ", L2 index: %#x)", offset,
+                                        l2_offset, j);
+                if (!preallocated) {
+                    qcow2_free_clusters(bs, offset, s->cluster_size,
+                                        QCOW2_DISCARD_ALWAYS);
+                }
+                ret = -EIO;
+                goto fail;
+            }
+
             ret = qcow2_pre_write_overlap_check(bs, 0, offset, s->cluster_size);
             if (ret < 0) {
                 if (!preallocated) {
@@ -1718,8 +1759,7 @@
                 goto fail;
             }
 
-            ret = bdrv_write_zeroes(bs->file, offset / BDRV_SECTOR_SIZE,
-                                    s->cluster_sectors, 0);
+            ret = bdrv_pwrite_zeroes(bs->file, offset, s->cluster_size, 0);
             if (ret < 0) {
                 if (!preallocated) {
                     qcow2_free_clusters(bs, offset, s->cluster_size,
@@ -1738,14 +1778,10 @@
 
         if (is_active_l1) {
             if (l2_dirty) {
-                qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+                qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache, l2_table);
                 qcow2_cache_depends_on_flush(s->l2_table_cache);
             }
-            ret = qcow2_cache_put(bs, s->l2_table_cache, (void **)&l2_table);
-            if (ret < 0) {
-                l2_table = NULL;
-                goto fail;
-            }
+            qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
         } else {
             if (l2_dirty) {
                 ret = qcow2_pre_write_overlap_check(bs,
@@ -1756,7 +1792,7 @@
                 }
 
                 ret = bdrv_write(bs->file, l2_offset / BDRV_SECTOR_SIZE,
-                        (void *)l2_table, s->cluster_sectors);
+                                 (void *)l2_table, s->cluster_sectors);
                 if (ret < 0) {
                     goto fail;
                 }
@@ -1765,7 +1801,7 @@
 
         (*visited_l1_entries)++;
         if (status_cb) {
-            status_cb(bs, *visited_l1_entries, l1_entries);
+            status_cb(bs, *visited_l1_entries, l1_entries, cb_opaque);
         }
     }
 
@@ -1776,12 +1812,7 @@
         if (!is_active_l1) {
             qemu_vfree(l2_table);
         } else {
-            if (ret < 0) {
-                qcow2_cache_put(bs, s->l2_table_cache, (void **)&l2_table);
-            } else {
-                ret = qcow2_cache_put(bs, s->l2_table_cache,
-                        (void **)&l2_table);
-            }
+            qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
         }
     }
     return ret;
@@ -1794,9 +1825,10 @@
  * qcow2 version which doesn't yet support metadata zero clusters.
  */
 int qcow2_expand_zero_clusters(BlockDriverState *bs,
-                               BlockDriverAmendStatusCB *status_cb)
+                               BlockDriverAmendStatusCB *status_cb,
+                               void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table = NULL;
     int64_t l1_entries = 0, visited_l1_entries = 0;
     int ret;
@@ -1811,7 +1843,7 @@
 
     ret = expand_zero_clusters_in_l1(bs, s->l1_table, s->l1_size,
                                      &visited_l1_entries, l1_entries,
-                                     status_cb);
+                                     status_cb, cb_opaque);
     if (ret < 0) {
         goto fail;
     }
@@ -1829,13 +1861,14 @@
     }
 
     for (i = 0; i < s->nb_snapshots; i++) {
-        int l1_sectors = (s->snapshots[i].l1_size * sizeof(uint64_t) +
-                BDRV_SECTOR_SIZE - 1) / BDRV_SECTOR_SIZE;
+        int l1_sectors = DIV_ROUND_UP(s->snapshots[i].l1_size *
+                                      sizeof(uint64_t), BDRV_SECTOR_SIZE);
 
         l1_table = g_realloc(l1_table, l1_sectors * BDRV_SECTOR_SIZE);
 
-        ret = bdrv_read(bs->file, s->snapshots[i].l1_table_offset /
-                BDRV_SECTOR_SIZE, (void *)l1_table, l1_sectors);
+        ret = bdrv_read(bs->file,
+                        s->snapshots[i].l1_table_offset / BDRV_SECTOR_SIZE,
+                        (void *)l1_table, l1_sectors);
         if (ret < 0) {
             goto fail;
         }
@@ -1846,7 +1879,7 @@
 
         ret = expand_zero_clusters_in_l1(bs, l1_table, s->snapshots[i].l1_size,
                                          &visited_l1_entries, l1_entries,
-                                         status_cb);
+                                         status_cb, cb_opaque);
         if (ret < 0) {
             goto fail;
         }
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 9afdb40..cbfb3fe 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -22,15 +22,62 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
 #include "block/qcow2.h"
 #include "qemu/range.h"
+#include "qemu/bswap.h"
 
 static int64_t alloc_clusters_noref(BlockDriverState *bs, uint64_t size);
 static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
-                            int64_t offset, int64_t length,
-                            int addend, enum qcow2_discard_type type);
+                            int64_t offset, int64_t length, uint64_t addend,
+                            bool decrease, enum qcow2_discard_type type);
+
+static uint64_t get_refcount_ro0(const void *refcount_array, uint64_t index);
+static uint64_t get_refcount_ro1(const void *refcount_array, uint64_t index);
+static uint64_t get_refcount_ro2(const void *refcount_array, uint64_t index);
+static uint64_t get_refcount_ro3(const void *refcount_array, uint64_t index);
+static uint64_t get_refcount_ro4(const void *refcount_array, uint64_t index);
+static uint64_t get_refcount_ro5(const void *refcount_array, uint64_t index);
+static uint64_t get_refcount_ro6(const void *refcount_array, uint64_t index);
+
+static void set_refcount_ro0(void *refcount_array, uint64_t index,
+                             uint64_t value);
+static void set_refcount_ro1(void *refcount_array, uint64_t index,
+                             uint64_t value);
+static void set_refcount_ro2(void *refcount_array, uint64_t index,
+                             uint64_t value);
+static void set_refcount_ro3(void *refcount_array, uint64_t index,
+                             uint64_t value);
+static void set_refcount_ro4(void *refcount_array, uint64_t index,
+                             uint64_t value);
+static void set_refcount_ro5(void *refcount_array, uint64_t index,
+                             uint64_t value);
+static void set_refcount_ro6(void *refcount_array, uint64_t index,
+                             uint64_t value);
+
+
+static Qcow2GetRefcountFunc *const get_refcount_funcs[] = {
+    &get_refcount_ro0,
+    &get_refcount_ro1,
+    &get_refcount_ro2,
+    &get_refcount_ro3,
+    &get_refcount_ro4,
+    &get_refcount_ro5,
+    &get_refcount_ro6
+};
+
+static Qcow2SetRefcountFunc *const set_refcount_funcs[] = {
+    &set_refcount_ro0,
+    &set_refcount_ro1,
+    &set_refcount_ro2,
+    &set_refcount_ro3,
+    &set_refcount_ro4,
+    &set_refcount_ro5,
+    &set_refcount_ro6
+};
 
 
 /*********************************************************/
@@ -38,10 +85,15 @@
 
 int qcow2_refcount_init(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int refcount_table_size2, i;
     int ret;
 
+    assert(s->refcount_order >= 0 && s->refcount_order <= 6);
+
+    s->get_refcount = get_refcount_funcs[s->refcount_order];
+    s->set_refcount = set_refcount_funcs[s->refcount_order];
+
     assert(s->refcount_table_size <= INT_MAX / sizeof(uint64_t));
     refcount_table_size2 = s->refcount_table_size * sizeof(uint64_t);
     s->refcount_table = g_try_malloc(refcount_table_size2);
@@ -67,46 +119,135 @@
 
 void qcow2_refcount_close(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     g_free(s->refcount_table);
 }
 
 
+static uint64_t get_refcount_ro0(const void *refcount_array, uint64_t index)
+{
+    return (((const uint8_t *)refcount_array)[index / 8] >> (index % 8)) & 0x1;
+}
+
+static void set_refcount_ro0(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    assert(!(value >> 1));
+    ((uint8_t *)refcount_array)[index / 8] &= ~(0x1 << (index % 8));
+    ((uint8_t *)refcount_array)[index / 8] |= value << (index % 8);
+}
+
+static uint64_t get_refcount_ro1(const void *refcount_array, uint64_t index)
+{
+    return (((const uint8_t *)refcount_array)[index / 4] >> (2 * (index % 4)))
+           & 0x3;
+}
+
+static void set_refcount_ro1(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    assert(!(value >> 2));
+    ((uint8_t *)refcount_array)[index / 4] &= ~(0x3 << (2 * (index % 4)));
+    ((uint8_t *)refcount_array)[index / 4] |= value << (2 * (index % 4));
+}
+
+static uint64_t get_refcount_ro2(const void *refcount_array, uint64_t index)
+{
+    return (((const uint8_t *)refcount_array)[index / 2] >> (4 * (index % 2)))
+           & 0xf;
+}
+
+static void set_refcount_ro2(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    assert(!(value >> 4));
+    ((uint8_t *)refcount_array)[index / 2] &= ~(0xf << (4 * (index % 2)));
+    ((uint8_t *)refcount_array)[index / 2] |= value << (4 * (index % 2));
+}
+
+static uint64_t get_refcount_ro3(const void *refcount_array, uint64_t index)
+{
+    return ((const uint8_t *)refcount_array)[index];
+}
+
+static void set_refcount_ro3(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    assert(!(value >> 8));
+    ((uint8_t *)refcount_array)[index] = value;
+}
+
+static uint64_t get_refcount_ro4(const void *refcount_array, uint64_t index)
+{
+    return be16_to_cpu(((const uint16_t *)refcount_array)[index]);
+}
+
+static void set_refcount_ro4(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    assert(!(value >> 16));
+    ((uint16_t *)refcount_array)[index] = cpu_to_be16(value);
+}
+
+static uint64_t get_refcount_ro5(const void *refcount_array, uint64_t index)
+{
+    return be32_to_cpu(((const uint32_t *)refcount_array)[index]);
+}
+
+static void set_refcount_ro5(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    assert(!(value >> 32));
+    ((uint32_t *)refcount_array)[index] = cpu_to_be32(value);
+}
+
+static uint64_t get_refcount_ro6(const void *refcount_array, uint64_t index)
+{
+    return be64_to_cpu(((const uint64_t *)refcount_array)[index]);
+}
+
+static void set_refcount_ro6(void *refcount_array, uint64_t index,
+                             uint64_t value)
+{
+    ((uint64_t *)refcount_array)[index] = cpu_to_be64(value);
+}
+
+
 static int load_refcount_block(BlockDriverState *bs,
                                int64_t refcount_block_offset,
                                void **refcount_block)
 {
-    BDRVQcowState *s = bs->opaque;
-    int ret;
+    BDRVQcow2State *s = bs->opaque;
 
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_LOAD);
-    ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
-        refcount_block);
-
-    return ret;
+    return qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
+                           refcount_block);
 }
 
 /*
- * Returns the refcount of the cluster given by its index. Any non-negative
- * return value is the refcount of the cluster, negative values are -errno
- * and indicate an error.
+ * Retrieves the refcount of the cluster given by its index and stores it in
+ * *refcount. Returns 0 on success and -errno on failure.
  */
-int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index)
+int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index,
+                       uint64_t *refcount)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t refcount_table_index, block_index;
     int64_t refcount_block_offset;
     int ret;
-    uint16_t *refcount_block;
-    uint16_t refcount;
+    void *refcount_block;
 
     refcount_table_index = cluster_index >> s->refcount_block_bits;
-    if (refcount_table_index >= s->refcount_table_size)
+    if (refcount_table_index >= s->refcount_table_size) {
+        *refcount = 0;
         return 0;
+    }
     refcount_block_offset =
         s->refcount_table[refcount_table_index] & REFT_OFFSET_MASK;
-    if (!refcount_block_offset)
+    if (!refcount_block_offset) {
+        *refcount = 0;
         return 0;
+    }
 
     if (offset_into_cluster(s, refcount_block_offset)) {
         qcow2_signal_corruption(bs, true, -1, -1, "Refblock offset %#" PRIx64
@@ -116,28 +257,24 @@
     }
 
     ret = qcow2_cache_get(bs, s->refcount_block_cache, refcount_block_offset,
-        (void**) &refcount_block);
+                          &refcount_block);
     if (ret < 0) {
         return ret;
     }
 
     block_index = cluster_index & (s->refcount_block_size - 1);
-    refcount = be16_to_cpu(refcount_block[block_index]);
+    *refcount = s->get_refcount(refcount_block, block_index);
 
-    ret = qcow2_cache_put(bs, s->refcount_block_cache,
-        (void**) &refcount_block);
-    if (ret < 0) {
-        return ret;
-    }
+    qcow2_cache_put(bs, s->refcount_block_cache, &refcount_block);
 
-    return refcount;
+    return 0;
 }
 
 /*
  * Rounds the refcount table size up to avoid growing the table for each single
  * refcount block that is allocated.
  */
-static unsigned int next_refcount_table_size(BDRVQcowState *s,
+static unsigned int next_refcount_table_size(BDRVQcow2State *s,
     unsigned int min_size)
 {
     unsigned int min_clusters = (min_size >> (s->cluster_bits - 3)) + 1;
@@ -153,7 +290,7 @@
 
 
 /* Checks if two offsets are described by the same refcount block */
-static int in_same_refcount_block(BDRVQcowState *s, uint64_t offset_a,
+static int in_same_refcount_block(BDRVQcow2State *s, uint64_t offset_a,
     uint64_t offset_b)
 {
     uint64_t block_a = offset_a >> (s->cluster_bits + s->refcount_block_bits);
@@ -169,9 +306,9 @@
  * Returns 0 on success or -errno in error case
  */
 static int alloc_refcount_block(BlockDriverState *bs,
-    int64_t cluster_index, uint16_t **refcount_block)
+                                int64_t cluster_index, void **refcount_block)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     unsigned int refcount_table_index;
     int ret;
 
@@ -196,7 +333,7 @@
             }
 
              return load_refcount_block(bs, refcount_block_offset,
-                 (void**) refcount_block);
+                                        refcount_block);
         }
     }
 
@@ -246,7 +383,7 @@
     if (in_same_refcount_block(s, new_block, cluster_index << s->cluster_bits)) {
         /* Zero the new refcount block before updating it */
         ret = qcow2_cache_get_empty(bs, s->refcount_block_cache, new_block,
-            (void**) refcount_block);
+                                    refcount_block);
         if (ret < 0) {
             goto fail_block;
         }
@@ -256,11 +393,11 @@
         /* The block describes itself, need to update the cache */
         int block_index = (new_block >> s->cluster_bits) &
             (s->refcount_block_size - 1);
-        (*refcount_block)[block_index] = cpu_to_be16(1);
+        s->set_refcount(*refcount_block, block_index, 1);
     } else {
         /* Described somewhere else. This can recurse at most twice before we
          * arrive at a block that describes itself. */
-        ret = update_refcount(bs, new_block, s->cluster_size, 1,
+        ret = update_refcount(bs, new_block, s->cluster_size, 1, false,
                               QCOW2_DISCARD_NEVER);
         if (ret < 0) {
             goto fail_block;
@@ -274,7 +411,7 @@
         /* Initialize the new refcount block only after updating its refcount,
          * update_refcount uses the refcount cache itself */
         ret = qcow2_cache_get_empty(bs, s->refcount_block_cache, new_block,
-            (void**) refcount_block);
+                                    refcount_block);
         if (ret < 0) {
             goto fail_block;
         }
@@ -284,7 +421,7 @@
 
     /* Now the new refcount block needs to be written to disk */
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_WRITE);
-    qcow2_cache_entry_mark_dirty(s->refcount_block_cache, *refcount_block);
+    qcow2_cache_entry_mark_dirty(bs, s->refcount_block_cache, *refcount_block);
     ret = qcow2_cache_flush(bs, s->refcount_block_cache);
     if (ret < 0) {
         goto fail_block;
@@ -308,10 +445,7 @@
         return -EAGAIN;
     }
 
-    ret = qcow2_cache_put(bs, s->refcount_block_cache, (void**) refcount_block);
-    if (ret < 0) {
-        goto fail_block;
-    }
+    qcow2_cache_put(bs, s->refcount_block_cache, refcount_block);
 
     /*
      * If we come here, we need to grow the refcount table. Again, a new
@@ -326,8 +460,20 @@
      */
     BLKDBG_EVENT(bs->file, BLKDBG_REFTABLE_GROW);
 
-    /* Calculate the number of refcount blocks needed so far */
-    uint64_t blocks_used = DIV_ROUND_UP(cluster_index, s->refcount_block_size);
+    /* Calculate the number of refcount blocks needed so far; this will be the
+     * basis for calculating the index of the first cluster used for the
+     * self-describing refcount structures which we are about to create.
+     *
+     * Because we reached this point, there cannot be any refcount entries for
+     * cluster_index or higher indices yet. However, because new_block has been
+     * allocated to describe that cluster (and it will assume this role later
+     * on), we cannot use that index; also, new_block may actually have a higher
+     * cluster index than cluster_index, so it needs to be taken into account
+     * here (and 1 needs to be added to its value because that cluster is used).
+     */
+    uint64_t blocks_used = DIV_ROUND_UP(MAX(cluster_index + 1,
+                                            (new_block >> s->cluster_bits) + 1),
+                                        s->refcount_block_size);
 
     if (blocks_used > QCOW_MAX_REFTABLE_SIZE / sizeof(uint64_t)) {
         return -EFBIG;
@@ -341,14 +487,12 @@
         uint64_t table_clusters =
             size_to_clusters(s, table_size * sizeof(uint64_t));
         blocks_clusters = 1 +
-            ((table_clusters + s->refcount_block_size - 1)
-            / s->refcount_block_size);
+            DIV_ROUND_UP(table_clusters, s->refcount_block_size);
         uint64_t meta_clusters = table_clusters + blocks_clusters;
 
         last_table_size = table_size;
         table_size = next_refcount_table_size(s, blocks_used +
-            ((meta_clusters + s->refcount_block_size - 1)
-            / s->refcount_block_size));
+            DIV_ROUND_UP(meta_clusters, s->refcount_block_size));
 
     } while (last_table_size != table_size);
 
@@ -362,7 +506,7 @@
         s->cluster_size;
     uint64_t table_offset = meta_offset + blocks_clusters * s->cluster_size;
     uint64_t *new_table = g_try_new0(uint64_t, table_size);
-    uint16_t *new_blocks = g_try_malloc0(blocks_clusters * s->cluster_size);
+    void *new_blocks = g_try_malloc0(blocks_clusters * s->cluster_size);
 
     assert(table_size > 0 && blocks_clusters > 0);
     if (new_table == NULL || new_blocks == NULL) {
@@ -384,7 +528,7 @@
     uint64_t table_clusters = size_to_clusters(s, table_size * sizeof(uint64_t));
     int block = 0;
     for (i = 0; i < table_clusters + blocks_clusters; i++) {
-        new_blocks[block++] = cpu_to_be16(1);
+        s->set_refcount(new_blocks, block++, 1);
     }
 
     /* Write refcount blocks to disk */
@@ -414,12 +558,16 @@
     }
 
     /* Hook up the new refcount table in the qcow2 header */
-    uint8_t data[12];
-    cpu_to_be64w((uint64_t*)data, table_offset);
-    cpu_to_be32w((uint32_t*)(data + 8), table_clusters);
+    struct QEMU_PACKED {
+        uint64_t d64;
+        uint32_t d32;
+    } data;
+    data.d64 = cpu_to_be64(table_offset);
+    data.d32 = cpu_to_be32(table_clusters);
     BLKDBG_EVENT(bs->file, BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE);
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, refcount_table_offset),
-        data, sizeof(data));
+    ret = bdrv_pwrite_sync(bs->file,
+                           offsetof(QCowHeader, refcount_table_offset),
+                           &data, sizeof(data));
     if (ret < 0) {
         goto fail_table;
     }
@@ -437,7 +585,7 @@
     qcow2_free_clusters(bs, old_table_offset, old_table_size * sizeof(uint64_t),
                         QCOW2_DISCARD_OTHER);
 
-    ret = load_refcount_block(bs, new_block, (void**) refcount_block);
+    ret = load_refcount_block(bs, new_block, refcount_block);
     if (ret < 0) {
         return ret;
     }
@@ -452,14 +600,14 @@
     g_free(new_table);
 fail_block:
     if (*refcount_block != NULL) {
-        qcow2_cache_put(bs, s->refcount_block_cache, (void**) refcount_block);
+        qcow2_cache_put(bs, s->refcount_block_cache, refcount_block);
     }
     return ret;
 }
 
 void qcow2_process_discards(BlockDriverState *bs, int ret)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2DiscardRegion *d, *next;
 
     QTAILQ_FOREACH_SAFE(d, &s->discards, next, next) {
@@ -467,9 +615,7 @@
 
         /* Discard is optional, ignore the return value */
         if (ret >= 0) {
-            bdrv_discard(bs->file,
-                         d->offset >> BDRV_SECTOR_BITS,
-                         d->bytes >> BDRV_SECTOR_BITS);
+            bdrv_pdiscard(bs->file->bs, d->offset, d->bytes);
         }
 
         g_free(d);
@@ -479,7 +625,7 @@
 static void update_refcount_discard(BlockDriverState *bs,
                                     uint64_t offset, uint64_t length)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2DiscardRegion *d, *p, *next;
 
     QTAILQ_FOREACH(d, &s->discards, next) {
@@ -527,18 +673,25 @@
 }
 
 /* XXX: cache several refcount block clusters ? */
+/* @addend is the absolute value of the addend; if @decrease is set, @addend
+ * will be subtracted from the current refcount, otherwise it will be added */
 static int QEMU_WARN_UNUSED_RESULT update_refcount(BlockDriverState *bs,
-    int64_t offset, int64_t length, int addend, enum qcow2_discard_type type)
+                                                   int64_t offset,
+                                                   int64_t length,
+                                                   uint64_t addend,
+                                                   bool decrease,
+                                                   enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t start, last, cluster_offset;
-    uint16_t *refcount_block = NULL;
+    void *refcount_block = NULL;
     int64_t old_table_index = -1;
     int ret;
 
 #ifdef DEBUG_ALLOC2
-    fprintf(stderr, "update_refcount: offset=%" PRId64 " size=%" PRId64 " addend=%d\n",
-           offset, length, addend);
+    fprintf(stderr, "update_refcount: offset=%" PRId64 " size=%" PRId64
+            " addend=%s%" PRIu64 "\n", offset, length, decrease ? "-" : "",
+            addend);
 #endif
     if (length < 0) {
         return -EINVAL;
@@ -546,7 +699,7 @@
         return 0;
     }
 
-    if (addend < 0) {
+    if (decrease) {
         qcow2_cache_set_dependency(bs, s->refcount_block_cache,
             s->l2_table_cache);
     }
@@ -556,20 +709,16 @@
     for(cluster_offset = start; cluster_offset <= last;
         cluster_offset += s->cluster_size)
     {
-        int block_index, refcount;
+        int block_index;
+        uint64_t refcount;
         int64_t cluster_index = cluster_offset >> s->cluster_bits;
         int64_t table_index = cluster_index >> s->refcount_block_bits;
 
         /* Load the refcount block and allocate it if needed */
         if (table_index != old_table_index) {
             if (refcount_block) {
-                ret = qcow2_cache_put(bs, s->refcount_block_cache,
-                    (void**) &refcount_block);
-                if (ret < 0) {
-                    goto fail;
-                }
+                qcow2_cache_put(bs, s->refcount_block_cache, &refcount_block);
             }
-
             ret = alloc_refcount_block(bs, cluster_index, &refcount_block);
             if (ret < 0) {
                 goto fail;
@@ -577,21 +726,29 @@
         }
         old_table_index = table_index;
 
-        qcow2_cache_entry_mark_dirty(s->refcount_block_cache, refcount_block);
+        qcow2_cache_entry_mark_dirty(bs, s->refcount_block_cache,
+                                     refcount_block);
 
         /* we can update the count and save it */
         block_index = cluster_index & (s->refcount_block_size - 1);
 
-        refcount = be16_to_cpu(refcount_block[block_index]);
-        refcount += addend;
-        if (refcount < 0 || refcount > 0xffff) {
+        refcount = s->get_refcount(refcount_block, block_index);
+        if (decrease ? (refcount - addend > refcount)
+                     : (refcount + addend < refcount ||
+                        refcount + addend > s->refcount_max))
+        {
             ret = -EINVAL;
             goto fail;
         }
+        if (decrease) {
+            refcount -= addend;
+        } else {
+            refcount += addend;
+        }
         if (refcount == 0 && cluster_index < s->free_cluster_index) {
             s->free_cluster_index = cluster_index;
         }
-        refcount_block[block_index] = cpu_to_be16(refcount);
+        s->set_refcount(refcount_block, block_index, refcount);
 
         if (refcount == 0 && s->discard_passthrough[type]) {
             update_refcount_discard(bs, cluster_offset, s->cluster_size);
@@ -606,12 +763,7 @@
 
     /* Write last changed block to disk */
     if (refcount_block) {
-        int wret;
-        wret = qcow2_cache_put(bs, s->refcount_block_cache,
-            (void**) &refcount_block);
-        if (wret < 0) {
-            return ret < 0 ? ret : wret;
-        }
+        qcow2_cache_put(bs, s->refcount_block_cache, &refcount_block);
     }
 
     /*
@@ -620,8 +772,8 @@
      */
     if (ret < 0) {
         int dummy;
-        dummy = update_refcount(bs, offset, cluster_offset - offset, -addend,
-                                QCOW2_DISCARD_NEVER);
+        dummy = update_refcount(bs, offset, cluster_offset - offset, addend,
+                                !decrease, QCOW2_DISCARD_NEVER);
         (void)dummy;
     }
 
@@ -631,24 +783,26 @@
 /*
  * Increases or decreases the refcount of a given cluster.
  *
- * If the return value is non-negative, it is the new refcount of the cluster.
- * If it is negative, it is -errno and indicates an error.
+ * @addend is the absolute value of the addend; if @decrease is set, @addend
+ * will be subtracted from the current refcount, otherwise it will be added.
+ *
+ * On success 0 is returned; on failure -errno is returned.
  */
 int qcow2_update_cluster_refcount(BlockDriverState *bs,
                                   int64_t cluster_index,
-                                  int addend,
+                                  uint64_t addend, bool decrease,
                                   enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     ret = update_refcount(bs, cluster_index << s->cluster_bits, 1, addend,
-                          type);
+                          decrease, type);
     if (ret < 0) {
         return ret;
     }
 
-    return qcow2_get_refcount(bs, cluster_index);
+    return 0;
 }
 
 
@@ -661,18 +815,23 @@
 /* return < 0 if error */
 static int64_t alloc_clusters_noref(BlockDriverState *bs, uint64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
-    uint64_t i, nb_clusters;
-    int refcount;
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t i, nb_clusters, refcount;
+    int ret;
+
+    /* We can't allocate clusters if they may still be queued for discard. */
+    if (s->cache_discards) {
+        qcow2_process_discards(bs, 0);
+    }
 
     nb_clusters = size_to_clusters(s, size);
 retry:
     for(i = 0; i < nb_clusters; i++) {
         uint64_t next_cluster_index = s->free_cluster_index++;
-        refcount = qcow2_get_refcount(bs, next_cluster_index);
+        ret = qcow2_get_refcount(bs, next_cluster_index, &refcount);
 
-        if (refcount < 0) {
-            return refcount;
+        if (ret < 0) {
+            return ret;
         } else if (refcount != 0) {
             goto retry;
         }
@@ -706,7 +865,7 @@
             return offset;
         }
 
-        ret = update_refcount(bs, offset, size, 1, QCOW2_DISCARD_NEVER);
+        ret = update_refcount(bs, offset, size, 1, false, QCOW2_DISCARD_NEVER);
     } while (ret == -EAGAIN);
 
     if (ret < 0) {
@@ -716,13 +875,13 @@
     return offset;
 }
 
-int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
-    int nb_clusters)
+int64_t qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
+                                int64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
-    uint64_t cluster_index;
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t cluster_index, refcount;
     uint64_t i;
-    int refcount, ret;
+    int ret;
 
     assert(nb_clusters >= 0);
     if (nb_clusters == 0) {
@@ -733,17 +892,16 @@
         /* Check how many clusters there are free */
         cluster_index = offset >> s->cluster_bits;
         for(i = 0; i < nb_clusters; i++) {
-            refcount = qcow2_get_refcount(bs, cluster_index++);
-
-            if (refcount < 0) {
-                return refcount;
+            ret = qcow2_get_refcount(bs, cluster_index++, &refcount);
+            if (ret < 0) {
+                return ret;
             } else if (refcount != 0) {
                 break;
             }
         }
 
         /* And then allocate them */
-        ret = update_refcount(bs, offset, i << s->cluster_bits, 1,
+        ret = update_refcount(bs, offset, i << s->cluster_bits, 1, false,
                               QCOW2_DISCARD_NEVER);
     } while (ret == -EAGAIN);
 
@@ -758,55 +916,64 @@
    contiguous sectors. size must be <= cluster_size */
 int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size)
 {
-    BDRVQcowState *s = bs->opaque;
-    int64_t offset, cluster_offset;
-    int free_in_cluster;
+    BDRVQcow2State *s = bs->opaque;
+    int64_t offset;
+    size_t free_in_cluster;
+    int ret;
 
     BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_ALLOC_BYTES);
     assert(size > 0 && size <= s->cluster_size);
-    if (s->free_byte_offset == 0) {
-        offset = qcow2_alloc_clusters(bs, s->cluster_size);
-        if (offset < 0) {
-            return offset;
+    assert(!s->free_byte_offset || offset_into_cluster(s, s->free_byte_offset));
+
+    offset = s->free_byte_offset;
+
+    if (offset) {
+        uint64_t refcount;
+        ret = qcow2_get_refcount(bs, offset >> s->cluster_bits, &refcount);
+        if (ret < 0) {
+            return ret;
         }
-        s->free_byte_offset = offset;
-    }
- redo:
-    free_in_cluster = s->cluster_size -
-        offset_into_cluster(s, s->free_byte_offset);
-    if (size <= free_in_cluster) {
-        /* enough space in current cluster */
-        offset = s->free_byte_offset;
-        s->free_byte_offset += size;
-        free_in_cluster -= size;
-        if (free_in_cluster == 0)
-            s->free_byte_offset = 0;
-        if (offset_into_cluster(s, offset) != 0)
-            qcow2_update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
-                                          QCOW2_DISCARD_NEVER);
-    } else {
-        offset = qcow2_alloc_clusters(bs, s->cluster_size);
-        if (offset < 0) {
-            return offset;
-        }
-        cluster_offset = start_of_cluster(s, s->free_byte_offset);
-        if ((cluster_offset + s->cluster_size) == offset) {
-            /* we are lucky: contiguous data */
-            offset = s->free_byte_offset;
-            qcow2_update_cluster_refcount(bs, offset >> s->cluster_bits, 1,
-                                          QCOW2_DISCARD_NEVER);
-            s->free_byte_offset += size;
-        } else {
-            s->free_byte_offset = offset;
-            goto redo;
+
+        if (refcount == s->refcount_max) {
+            offset = 0;
         }
     }
 
-    /* The cluster refcount was incremented, either by qcow2_alloc_clusters()
-     * or explicitly by qcow2_update_cluster_refcount().  Refcount blocks must
-     * be flushed before the caller's L2 table updates.
-     */
+    free_in_cluster = s->cluster_size - offset_into_cluster(s, offset);
+    do {
+        if (!offset || free_in_cluster < size) {
+            int64_t new_cluster = alloc_clusters_noref(bs, s->cluster_size);
+            if (new_cluster < 0) {
+                return new_cluster;
+            }
+
+            if (!offset || ROUND_UP(offset, s->cluster_size) != new_cluster) {
+                offset = new_cluster;
+                free_in_cluster = s->cluster_size;
+            } else {
+                free_in_cluster += s->cluster_size;
+            }
+        }
+
+        assert(offset);
+        ret = update_refcount(bs, offset, size, 1, false, QCOW2_DISCARD_NEVER);
+        if (ret < 0) {
+            offset = 0;
+        }
+    } while (ret == -EAGAIN);
+    if (ret < 0) {
+        return ret;
+    }
+
+    /* The cluster refcount was incremented; refcount blocks must be flushed
+     * before the caller's L2 table updates. */
     qcow2_cache_set_dependency(bs, s->l2_table_cache, s->refcount_block_cache);
+
+    s->free_byte_offset = offset + size;
+    if (!offset_into_cluster(s, s->free_byte_offset)) {
+        s->free_byte_offset = 0;
+    }
+
     return offset;
 }
 
@@ -817,7 +984,7 @@
     int ret;
 
     BLKDBG_EVENT(bs->file, BLKDBG_CLUSTER_FREE);
-    ret = update_refcount(bs, offset, size, -1, type);
+    ret = update_refcount(bs, offset, size, 1, true, type);
     if (ret < 0) {
         fprintf(stderr, "qcow2_free_clusters failed: %s\n", strerror(-ret));
         /* TODO Remember the clusters to free them later and avoid leaking */
@@ -831,7 +998,7 @@
 void qcow2_free_any_clusters(BlockDriverState *bs, uint64_t l2_entry,
                              int nb_clusters, enum qcow2_discard_type type)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     switch (qcow2_get_cluster_type(l2_entry)) {
     case QCOW2_CLUSTER_COMPRESSED:
@@ -875,13 +1042,15 @@
 int qcow2_update_snapshot_refcount(BlockDriverState *bs,
     int64_t l1_table_offset, int l1_size, int addend)
 {
-    BDRVQcowState *s = bs->opaque;
-    uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2;
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t *l1_table, *l2_table, l2_offset, offset, l1_size2, refcount;
     bool l1_allocated = false;
     int64_t old_offset, old_l2_offset;
-    int i, j, l1_modified = 0, nb_csectors, refcount;
+    int i, j, l1_modified = 0, nb_csectors;
     int ret;
 
+    assert(addend >= -1 && addend <= 1);
+
     l2_table = NULL;
     l1_table = NULL;
     l1_size2 = l1_size * sizeof(uint64_t);
@@ -946,7 +1115,7 @@
                         if (addend != 0) {
                             ret = update_refcount(bs,
                                 (offset & s->cluster_offset_mask) & ~511,
-                                nb_csectors * 512, addend,
+                                nb_csectors * 512, abs(addend), addend < 0,
                                 QCOW2_DISCARD_SNAPSHOT);
                             if (ret < 0) {
                                 goto fail;
@@ -976,15 +1145,16 @@
                             break;
                         }
                         if (addend != 0) {
-                            refcount = qcow2_update_cluster_refcount(bs,
-                                    cluster_index, addend,
+                            ret = qcow2_update_cluster_refcount(bs,
+                                    cluster_index, abs(addend), addend < 0,
                                     QCOW2_DISCARD_SNAPSHOT);
-                        } else {
-                            refcount = qcow2_get_refcount(bs, cluster_index);
+                            if (ret < 0) {
+                                goto fail;
+                            }
                         }
 
-                        if (refcount < 0) {
-                            ret = refcount;
+                        ret = qcow2_get_refcount(bs, cluster_index, &refcount);
+                        if (ret < 0) {
                             goto fail;
                         }
                         break;
@@ -1006,24 +1176,25 @@
                             s->refcount_block_cache);
                     }
                     l2_table[j] = cpu_to_be64(offset);
-                    qcow2_cache_entry_mark_dirty(s->l2_table_cache, l2_table);
+                    qcow2_cache_entry_mark_dirty(bs, s->l2_table_cache,
+                                                 l2_table);
                 }
             }
 
-            ret = qcow2_cache_put(bs, s->l2_table_cache, (void**) &l2_table);
-            if (ret < 0) {
-                goto fail;
-            }
-
+            qcow2_cache_put(bs, s->l2_table_cache, (void **) &l2_table);
 
             if (addend != 0) {
-                refcount = qcow2_update_cluster_refcount(bs, l2_offset >>
-                        s->cluster_bits, addend, QCOW2_DISCARD_SNAPSHOT);
-            } else {
-                refcount = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits);
+                ret = qcow2_update_cluster_refcount(bs, l2_offset >>
+                                                        s->cluster_bits,
+                                                    abs(addend), addend < 0,
+                                                    QCOW2_DISCARD_SNAPSHOT);
+                if (ret < 0) {
+                    goto fail;
+                }
             }
-            if (refcount < 0) {
-                ret = refcount;
+            ret = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits,
+                                     &refcount);
+            if (ret < 0) {
                 goto fail;
             } else if (refcount == 1) {
                 l2_offset |= QCOW_OFLAG_COPIED;
@@ -1050,7 +1221,8 @@
             cpu_to_be64s(&l1_table[i]);
         }
 
-        ret = bdrv_pwrite_sync(bs->file, l1_table_offset, l1_table, l1_size2);
+        ret = bdrv_pwrite_sync(bs->file, l1_table_offset,
+                               l1_table, l1_size2);
 
         for (i = 0; i < l1_size; i++) {
             be64_to_cpus(&l1_table[i]);
@@ -1068,22 +1240,84 @@
 /* refcount checking functions */
 
 
+static uint64_t refcount_array_byte_size(BDRVQcow2State *s, uint64_t entries)
+{
+    /* This assertion holds because there is no way we can address more than
+     * 2^(64 - 9) clusters at once (with cluster size 512 = 2^9, and because
+     * offsets have to be representable in bytes); due to every cluster
+     * corresponding to one refcount entry, we are well below that limit */
+    assert(entries < (UINT64_C(1) << (64 - 9)));
+
+    /* Thanks to the assertion this will not overflow, because
+     * s->refcount_order < 7.
+     * (note: x << s->refcount_order == x * s->refcount_bits) */
+    return DIV_ROUND_UP(entries << s->refcount_order, 8);
+}
+
+/**
+ * Reallocates *array so that it can hold new_size entries. *size must contain
+ * the current number of entries in *array. If the reallocation fails, *array
+ * and *size will not be modified and -errno will be returned. If the
+ * reallocation is successful, *array will be set to the new buffer, *size
+ * will be set to new_size and 0 will be returned. The size of the reallocated
+ * refcount array buffer will be aligned to a cluster boundary, and the newly
+ * allocated area will be zeroed.
+ */
+static int realloc_refcount_array(BDRVQcow2State *s, void **array,
+                                  int64_t *size, int64_t new_size)
+{
+    int64_t old_byte_size, new_byte_size;
+    void *new_ptr;
+
+    /* Round to clusters so the array can be directly written to disk */
+    old_byte_size = size_to_clusters(s, refcount_array_byte_size(s, *size))
+                    * s->cluster_size;
+    new_byte_size = size_to_clusters(s, refcount_array_byte_size(s, new_size))
+                    * s->cluster_size;
+
+    if (new_byte_size == old_byte_size) {
+        *size = new_size;
+        return 0;
+    }
+
+    assert(new_byte_size > 0);
+
+    if (new_byte_size > SIZE_MAX) {
+        return -ENOMEM;
+    }
+
+    new_ptr = g_try_realloc(*array, new_byte_size);
+    if (!new_ptr) {
+        return -ENOMEM;
+    }
+
+    if (new_byte_size > old_byte_size) {
+        memset((char *)new_ptr + old_byte_size, 0,
+               new_byte_size - old_byte_size);
+    }
+
+    *array = new_ptr;
+    *size  = new_size;
+
+    return 0;
+}
 
 /*
  * Increases the refcount for a range of clusters in a given refcount table.
  * This is used to construct a temporary refcount table out of L1 and L2 tables
- * which can be compared the the refcount table saved in the image.
+ * which can be compared to the refcount table saved in the image.
  *
  * Modifies the number of errors in res.
  */
 static int inc_refcounts(BlockDriverState *bs,
                          BdrvCheckResult *res,
-                         uint16_t **refcount_table,
+                         void **refcount_table,
                          int64_t *refcount_table_size,
                          int64_t offset, int64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
-    uint64_t start, last, cluster_offset, k;
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t start, last, cluster_offset, k, refcount;
+    int ret;
 
     if (size <= 0) {
         return 0;
@@ -1095,30 +1329,25 @@
         cluster_offset += s->cluster_size) {
         k = cluster_offset >> s->cluster_bits;
         if (k >= *refcount_table_size) {
-            int64_t old_refcount_table_size = *refcount_table_size;
-            uint16_t *new_refcount_table;
-
-            *refcount_table_size = k + 1;
-            new_refcount_table = g_try_realloc(*refcount_table,
-                                               *refcount_table_size *
-                                               sizeof(**refcount_table));
-            if (!new_refcount_table) {
-                *refcount_table_size = old_refcount_table_size;
+            ret = realloc_refcount_array(s, refcount_table,
+                                         refcount_table_size, k + 1);
+            if (ret < 0) {
                 res->check_errors++;
-                return -ENOMEM;
+                return ret;
             }
-            *refcount_table = new_refcount_table;
-
-            memset(*refcount_table + old_refcount_table_size, 0,
-                   (*refcount_table_size - old_refcount_table_size) *
-                   sizeof(**refcount_table));
         }
 
-        if (++(*refcount_table)[k] == 0) {
+        refcount = s->get_refcount(*refcount_table, k);
+        if (refcount == s->refcount_max) {
             fprintf(stderr, "ERROR: overflow cluster offset=0x%" PRIx64
                     "\n", cluster_offset);
+            fprintf(stderr, "Use qemu-img amend to increase the refcount entry "
+                    "width or qemu-img convert to create a clean copy if the "
+                    "image cannot be opened for writing\n");
             res->corruptions++;
+            continue;
         }
+        s->set_refcount(*refcount_table, k, refcount + 1);
     }
 
     return 0;
@@ -1138,10 +1367,11 @@
  * error occurred.
  */
 static int check_refcounts_l2(BlockDriverState *bs, BdrvCheckResult *res,
-    uint16_t **refcount_table, int64_t *refcount_table_size, int64_t l2_offset,
-    int flags)
+                              void **refcount_table,
+                              int64_t *refcount_table_size, int64_t l2_offset,
+                              int flags)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table, l2_entry;
     uint64_t next_contiguous_offset = 0;
     int i, l2_size, nb_csectors, ret;
@@ -1256,12 +1486,12 @@
  */
 static int check_refcounts_l1(BlockDriverState *bs,
                               BdrvCheckResult *res,
-                              uint16_t **refcount_table,
+                              void **refcount_table,
                               int64_t *refcount_table_size,
                               int64_t l1_table_offset, int l1_size,
                               int flags)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l1_table = NULL, l2_offset, l1_size2;
     int i, ret;
 
@@ -1338,10 +1568,10 @@
 static int check_oflag_copied(BlockDriverState *bs, BdrvCheckResult *res,
                               BdrvCheckMode fix)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t *l2_table = qemu_blockalign(bs, s->cluster_size);
     int ret;
-    int refcount;
+    uint64_t refcount;
     int i, j;
 
     for (i = 0; i < s->l1_size; i++) {
@@ -1353,14 +1583,15 @@
             continue;
         }
 
-        refcount = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits);
-        if (refcount < 0) {
+        ret = qcow2_get_refcount(bs, l2_offset >> s->cluster_bits,
+                                 &refcount);
+        if (ret < 0) {
             /* don't print message nor increment check_errors */
             continue;
         }
         if ((refcount == 1) != ((l1_entry & QCOW_OFLAG_COPIED) != 0)) {
             fprintf(stderr, "%s OFLAG_COPIED L2 cluster: l1_index=%d "
-                    "l1_entry=%" PRIx64 " refcount=%d\n",
+                    "l1_entry=%" PRIx64 " refcount=%" PRIu64 "\n",
                     fix & BDRV_FIX_ERRORS ? "Repairing" :
                                             "ERROR",
                     i, l1_entry, refcount);
@@ -1395,15 +1626,16 @@
 
             if ((cluster_type == QCOW2_CLUSTER_NORMAL) ||
                 ((cluster_type == QCOW2_CLUSTER_ZERO) && (data_offset != 0))) {
-                refcount = qcow2_get_refcount(bs,
-                                              data_offset >> s->cluster_bits);
-                if (refcount < 0) {
+                ret = qcow2_get_refcount(bs,
+                                         data_offset >> s->cluster_bits,
+                                         &refcount);
+                if (ret < 0) {
                     /* don't print message nor increment check_errors */
                     continue;
                 }
                 if ((refcount == 1) != ((l2_entry & QCOW_OFLAG_COPIED) != 0)) {
                     fprintf(stderr, "%s OFLAG_COPIED data cluster: "
-                            "l2_entry=%" PRIx64 " refcount=%d\n",
+                            "l2_entry=%" PRIx64 " refcount=%" PRIu64 "\n",
                             fix & BDRV_FIX_ERRORS ? "Repairing" :
                                                     "ERROR",
                             l2_entry, refcount);
@@ -1430,7 +1662,8 @@
                 goto fail;
             }
 
-            ret = bdrv_pwrite(bs->file, l2_offset, l2_table, s->cluster_size);
+            ret = bdrv_pwrite(bs->file, l2_offset, l2_table,
+                              s->cluster_size);
             if (ret < 0) {
                 fprintf(stderr, "ERROR: Could not write L2 table: %s\n",
                         strerror(-ret));
@@ -1453,9 +1686,9 @@
  */
 static int check_refblocks(BlockDriverState *bs, BdrvCheckResult *res,
                            BdrvCheckMode fix, bool *rebuild,
-                           uint16_t **refcount_table, int64_t *nb_clusters)
+                           void **refcount_table, int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i, size;
     int ret;
 
@@ -1478,40 +1711,32 @@
                     fix & BDRV_FIX_ERRORS ? "Repairing" : "ERROR", i);
 
             if (fix & BDRV_FIX_ERRORS) {
-                int64_t old_nb_clusters = *nb_clusters;
-                uint16_t *new_refcount_table;
+                int64_t new_nb_clusters;
 
                 if (offset > INT64_MAX - s->cluster_size) {
                     ret = -EINVAL;
                     goto resize_fail;
                 }
 
-                ret = bdrv_truncate(bs->file, offset + s->cluster_size);
+                ret = bdrv_truncate(bs->file->bs, offset + s->cluster_size);
                 if (ret < 0) {
                     goto resize_fail;
                 }
-                size = bdrv_getlength(bs->file);
+                size = bdrv_getlength(bs->file->bs);
                 if (size < 0) {
                     ret = size;
                     goto resize_fail;
                 }
 
-                *nb_clusters = size_to_clusters(s, size);
-                assert(*nb_clusters >= old_nb_clusters);
+                new_nb_clusters = size_to_clusters(s, size);
+                assert(new_nb_clusters >= *nb_clusters);
 
-                new_refcount_table = g_try_realloc(*refcount_table,
-                                                   *nb_clusters *
-                                                   sizeof(**refcount_table));
-                if (!new_refcount_table) {
-                    *nb_clusters = old_nb_clusters;
+                ret = realloc_refcount_array(s, refcount_table,
+                                             nb_clusters, new_nb_clusters);
+                if (ret < 0) {
                     res->check_errors++;
-                    return -ENOMEM;
+                    return ret;
                 }
-                *refcount_table = new_refcount_table;
-
-                memset(*refcount_table + old_nb_clusters, 0,
-                       (*nb_clusters - old_nb_clusters) *
-                       sizeof(**refcount_table));
 
                 if (cluster >= *nb_clusters) {
                     ret = -EINVAL;
@@ -1546,9 +1771,10 @@
             if (ret < 0) {
                 return ret;
             }
-            if ((*refcount_table)[cluster] != 1) {
+            if (s->get_refcount(*refcount_table, cluster) != 1) {
                 fprintf(stderr, "ERROR refcount block %" PRId64
-                        " refcount=%d\n", i, (*refcount_table)[cluster]);
+                        " refcount=%" PRIu64 "\n", i,
+                        s->get_refcount(*refcount_table, cluster));
                 res->corruptions++;
                 *rebuild = true;
             }
@@ -1563,18 +1789,20 @@
  */
 static int calculate_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                                BdrvCheckMode fix, bool *rebuild,
-                               uint16_t **refcount_table, int64_t *nb_clusters)
+                               void **refcount_table, int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i;
     QCowSnapshot *sn;
     int ret;
 
     if (!*refcount_table) {
-        *refcount_table = g_try_new0(uint16_t, *nb_clusters);
-        if (*nb_clusters && *refcount_table == NULL) {
+        int64_t old_size = 0;
+        ret = realloc_refcount_array(s, refcount_table,
+                                     &old_size, *nb_clusters);
+        if (ret < 0) {
             res->check_errors++;
-            return -ENOMEM;
+            return ret;
         }
     }
 
@@ -1625,22 +1853,23 @@
 static void compare_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                               BdrvCheckMode fix, bool *rebuild,
                               int64_t *highest_cluster,
-                              uint16_t *refcount_table, int64_t nb_clusters)
+                              void *refcount_table, int64_t nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t i;
-    int refcount1, refcount2, ret;
+    uint64_t refcount1, refcount2;
+    int ret;
 
     for (i = 0, *highest_cluster = 0; i < nb_clusters; i++) {
-        refcount1 = qcow2_get_refcount(bs, i);
-        if (refcount1 < 0) {
+        ret = qcow2_get_refcount(bs, i, &refcount1);
+        if (ret < 0) {
             fprintf(stderr, "Can't get refcount for cluster %" PRId64 ": %s\n",
-                i, strerror(-refcount1));
+                    i, strerror(-ret));
             res->check_errors++;
             continue;
         }
 
-        refcount2 = refcount_table[i];
+        refcount2 = s->get_refcount(refcount_table, i);
 
         if (refcount1 > 0 || refcount2 > 0) {
             *highest_cluster = i;
@@ -1657,7 +1886,8 @@
                 num_fixed = &res->corruptions_fixed;
             }
 
-            fprintf(stderr, "%s cluster %" PRId64 " refcount=%d reference=%d\n",
+            fprintf(stderr, "%s cluster %" PRId64 " refcount=%" PRIu64
+                    " reference=%" PRIu64 "\n",
                    num_fixed != NULL     ? "Repairing" :
                    refcount1 < refcount2 ? "ERROR" :
                                            "Leaked",
@@ -1665,7 +1895,8 @@
 
             if (num_fixed) {
                 ret = update_refcount(bs, i << s->cluster_bits, 1,
-                                      refcount2 - refcount1,
+                                      refcount_diff(refcount1, refcount2),
+                                      refcount1 > refcount2,
                                       QCOW2_DISCARD_ALWAYS);
                 if (ret >= 0) {
                     (*num_fixed)++;
@@ -1697,14 +1928,15 @@
  */
 static int64_t alloc_clusters_imrt(BlockDriverState *bs,
                                    int cluster_count,
-                                   uint16_t **refcount_table,
+                                   void **refcount_table,
                                    int64_t *imrt_nb_clusters,
                                    int64_t *first_free_cluster)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t cluster = *first_free_cluster, i;
     bool first_gap = true;
     int contiguous_free_clusters;
+    int ret;
 
     /* Starting at *first_free_cluster, find a range of at least cluster_count
      * continuously free clusters */
@@ -1713,7 +1945,7 @@
          contiguous_free_clusters < cluster_count;
          cluster++)
     {
-        if (!(*refcount_table)[cluster]) {
+        if (!s->get_refcount(*refcount_table, cluster)) {
             contiguous_free_clusters++;
             if (first_gap) {
                 /* If this is the first free cluster found, update
@@ -1734,34 +1966,24 @@
     /* If no such range could be found, grow the in-memory refcount table
      * accordingly to append free clusters at the end of the image */
     if (contiguous_free_clusters < cluster_count) {
-        int64_t old_imrt_nb_clusters = *imrt_nb_clusters;
-        uint16_t *new_refcount_table;
-
         /* contiguous_free_clusters clusters are already empty at the image end;
          * we need cluster_count clusters; therefore, we have to allocate
          * cluster_count - contiguous_free_clusters new clusters at the end of
          * the image (which is the current value of cluster; note that cluster
          * may exceed old_imrt_nb_clusters if *first_free_cluster pointed beyond
          * the image end) */
-        *imrt_nb_clusters = cluster + cluster_count - contiguous_free_clusters;
-        new_refcount_table = g_try_realloc(*refcount_table,
-                                           *imrt_nb_clusters *
-                                           sizeof(**refcount_table));
-        if (!new_refcount_table) {
-            *imrt_nb_clusters = old_imrt_nb_clusters;
-            return -ENOMEM;
+        ret = realloc_refcount_array(s, refcount_table, imrt_nb_clusters,
+                                     cluster + cluster_count
+                                     - contiguous_free_clusters);
+        if (ret < 0) {
+            return ret;
         }
-        *refcount_table = new_refcount_table;
-
-        memset(*refcount_table + old_imrt_nb_clusters, 0,
-               (*imrt_nb_clusters - old_imrt_nb_clusters) *
-               sizeof(**refcount_table));
     }
 
     /* Go back to the first free cluster */
     cluster -= contiguous_free_clusters;
     for (i = 0; i < cluster_count; i++) {
-        (*refcount_table)[cluster + i] = 1;
+        s->set_refcount(*refcount_table, cluster + i, 1);
     }
 
     return cluster << s->cluster_bits;
@@ -1777,16 +1999,16 @@
  */
 static int rebuild_refcount_structure(BlockDriverState *bs,
                                       BdrvCheckResult *res,
-                                      uint16_t **refcount_table,
+                                      void **refcount_table,
                                       int64_t *nb_clusters)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t first_free_cluster = 0, reftable_offset = -1, cluster = 0;
     int64_t refblock_offset, refblock_start, refblock_index;
     uint32_t reftable_size = 0;
     uint64_t *on_disk_reftable = NULL;
-    uint16_t *on_disk_refblock;
-    int i, ret = 0;
+    void *on_disk_refblock;
+    int ret = 0;
     struct {
         uint64_t reftable_offset;
         uint32_t reftable_clusters;
@@ -1796,7 +2018,7 @@
 
 write_refblocks:
     for (; cluster < *nb_clusters; cluster++) {
-        if (!(*refcount_table)[cluster]) {
+        if (!s->get_refcount(*refcount_table, cluster)) {
             continue;
         }
 
@@ -1869,17 +2091,13 @@
             goto fail;
         }
 
-        on_disk_refblock = qemu_blockalign0(bs->file, s->cluster_size);
-        for (i = 0; i < s->refcount_block_size &&
-                    refblock_start + i < *nb_clusters; i++)
-        {
-            on_disk_refblock[i] =
-                cpu_to_be16((*refcount_table)[refblock_start + i]);
-        }
+        /* The size of *refcount_table is always cluster-aligned, therefore the
+         * write operation will not overflow */
+        on_disk_refblock = (void *)((char *) *refcount_table +
+                                    refblock_index * s->cluster_size);
 
         ret = bdrv_write(bs->file, refblock_offset / BDRV_SECTOR_SIZE,
-                         (void *)on_disk_refblock, s->cluster_sectors);
-        qemu_vfree(on_disk_refblock);
+                         on_disk_refblock, s->cluster_sectors);
         if (ret < 0) {
             fprintf(stderr, "ERROR writing refblock: %s\n", strerror(-ret));
             goto fail;
@@ -1935,12 +2153,11 @@
     }
 
     /* Enter new reftable into the image header */
-    cpu_to_be64w(&reftable_offset_and_clusters.reftable_offset,
-                 reftable_offset);
-    cpu_to_be32w(&reftable_offset_and_clusters.reftable_clusters,
-                 size_to_clusters(s, reftable_size * sizeof(uint64_t)));
-    ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader,
-                                              refcount_table_offset),
+    reftable_offset_and_clusters.reftable_offset = cpu_to_be64(reftable_offset);
+    reftable_offset_and_clusters.reftable_clusters =
+        cpu_to_be32(size_to_clusters(s, reftable_size * sizeof(uint64_t)));
+    ret = bdrv_pwrite_sync(bs->file,
+                           offsetof(QCowHeader, refcount_table_offset),
                            &reftable_offset_and_clusters,
                            sizeof(reftable_offset_and_clusters));
     if (ret < 0) {
@@ -1971,14 +2188,14 @@
 int qcow2_check_refcounts(BlockDriverState *bs, BdrvCheckResult *res,
                           BdrvCheckMode fix)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     BdrvCheckResult pre_compare_res;
     int64_t size, highest_cluster, nb_clusters;
-    uint16_t *refcount_table = NULL;
+    void *refcount_table = NULL;
     bool rebuild = false;
     int ret;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     if (size < 0) {
         res->check_errors++;
         return size;
@@ -2023,7 +2240,7 @@
         /* Because the old reftable has been exchanged for a new one the
          * references have to be recalculated */
         rebuild = false;
-        memset(refcount_table, 0, nb_clusters * sizeof(uint16_t));
+        memset(refcount_table, 0, refcount_array_byte_size(s, nb_clusters));
         ret = calculate_refcounts(bs, res, 0, &rebuild, &refcount_table,
                                   &nb_clusters);
         if (ret < 0) {
@@ -2108,7 +2325,7 @@
 int qcow2_check_metadata_overlap(BlockDriverState *bs, int ign, int64_t offset,
                                  int64_t size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int chk = s->overlap_check & ~ign;
     int i, j;
 
@@ -2237,7 +2454,7 @@
     if (ret < 0) {
         return ret;
     } else if (ret > 0) {
-        int metadata_ol_bitnr = ffs(ret) - 1;
+        int metadata_ol_bitnr = ctz32(ret);
         assert(metadata_ol_bitnr < QCOW2_OL_MAX_BITNR);
 
         qcow2_signal_corruption(bs, true, offset, size, "Preventing invalid "
@@ -2248,3 +2465,450 @@
 
     return 0;
 }
+
+/* A pointer to a function of this type is given to walk_over_reftable(). That
+ * function will create refblocks and pass them to a RefblockFinishOp once they
+ * are completed (@refblock). @refblock_empty is set if the refblock is
+ * completely empty.
+ *
+ * Along with the refblock, a corresponding reftable entry is passed, in the
+ * reftable @reftable (which may be reallocated) at @reftable_index.
+ *
+ * @allocated should be set to true if a new cluster has been allocated.
+ */
+typedef int (RefblockFinishOp)(BlockDriverState *bs, uint64_t **reftable,
+                               uint64_t reftable_index, uint64_t *reftable_size,
+                               void *refblock, bool refblock_empty,
+                               bool *allocated, Error **errp);
+
+/**
+ * This "operation" for walk_over_reftable() allocates the refblock on disk (if
+ * it is not empty) and inserts its offset into the new reftable. The size of
+ * this new reftable is increased as required.
+ */
+static int alloc_refblock(BlockDriverState *bs, uint64_t **reftable,
+                          uint64_t reftable_index, uint64_t *reftable_size,
+                          void *refblock, bool refblock_empty, bool *allocated,
+                          Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int64_t offset;
+
+    if (!refblock_empty && reftable_index >= *reftable_size) {
+        uint64_t *new_reftable;
+        uint64_t new_reftable_size;
+
+        new_reftable_size = ROUND_UP(reftable_index + 1,
+                                     s->cluster_size / sizeof(uint64_t));
+        if (new_reftable_size > QCOW_MAX_REFTABLE_SIZE / sizeof(uint64_t)) {
+            error_setg(errp,
+                       "This operation would make the refcount table grow "
+                       "beyond the maximum size supported by QEMU, aborting");
+            return -ENOTSUP;
+        }
+
+        new_reftable = g_try_realloc(*reftable, new_reftable_size *
+                                                sizeof(uint64_t));
+        if (!new_reftable) {
+            error_setg(errp, "Failed to increase reftable buffer size");
+            return -ENOMEM;
+        }
+
+        memset(new_reftable + *reftable_size, 0,
+               (new_reftable_size - *reftable_size) * sizeof(uint64_t));
+
+        *reftable      = new_reftable;
+        *reftable_size = new_reftable_size;
+    }
+
+    if (!refblock_empty && !(*reftable)[reftable_index]) {
+        offset = qcow2_alloc_clusters(bs, s->cluster_size);
+        if (offset < 0) {
+            error_setg_errno(errp, -offset, "Failed to allocate refblock");
+            return offset;
+        }
+        (*reftable)[reftable_index] = offset;
+        *allocated = true;
+    }
+
+    return 0;
+}
+
+/**
+ * This "operation" for walk_over_reftable() writes the refblock to disk at the
+ * offset specified by the new reftable's entry. It does not modify the new
+ * reftable or change any refcounts.
+ */
+static int flush_refblock(BlockDriverState *bs, uint64_t **reftable,
+                          uint64_t reftable_index, uint64_t *reftable_size,
+                          void *refblock, bool refblock_empty, bool *allocated,
+                          Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int64_t offset;
+    int ret;
+
+    if (reftable_index < *reftable_size && (*reftable)[reftable_index]) {
+        offset = (*reftable)[reftable_index];
+
+        ret = qcow2_pre_write_overlap_check(bs, 0, offset, s->cluster_size);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Overlap check failed");
+            return ret;
+        }
+
+        ret = bdrv_pwrite(bs->file, offset, refblock, s->cluster_size);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Failed to write refblock");
+            return ret;
+        }
+    } else {
+        assert(refblock_empty);
+    }
+
+    return 0;
+}
+
+/**
+ * This function walks over the existing reftable and every referenced refblock;
+ * if @new_set_refcount is non-NULL, it is called for every refcount entry to
+ * create an equal new entry in the passed @new_refblock. Once that
+ * @new_refblock is completely filled, @operation will be called.
+ *
+ * @status_cb and @cb_opaque are used for the amend operation's status callback.
+ * @index is the index of the walk_over_reftable() calls and @total is the total
+ * number of walk_over_reftable() calls per amend operation. Both are used for
+ * calculating the parameters for the status callback.
+ *
+ * @allocated is set to true if a new cluster has been allocated.
+ */
+static int walk_over_reftable(BlockDriverState *bs, uint64_t **new_reftable,
+                              uint64_t *new_reftable_index,
+                              uint64_t *new_reftable_size,
+                              void *new_refblock, int new_refblock_size,
+                              int new_refcount_bits,
+                              RefblockFinishOp *operation, bool *allocated,
+                              Qcow2SetRefcountFunc *new_set_refcount,
+                              BlockDriverAmendStatusCB *status_cb,
+                              void *cb_opaque, int index, int total,
+                              Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    uint64_t reftable_index;
+    bool new_refblock_empty = true;
+    int refblock_index;
+    int new_refblock_index = 0;
+    int ret;
+
+    for (reftable_index = 0; reftable_index < s->refcount_table_size;
+         reftable_index++)
+    {
+        uint64_t refblock_offset = s->refcount_table[reftable_index]
+                                 & REFT_OFFSET_MASK;
+
+        status_cb(bs, (uint64_t)index * s->refcount_table_size + reftable_index,
+                  (uint64_t)total * s->refcount_table_size, cb_opaque);
+
+        if (refblock_offset) {
+            void *refblock;
+
+            if (offset_into_cluster(s, refblock_offset)) {
+                qcow2_signal_corruption(bs, true, -1, -1, "Refblock offset %#"
+                                        PRIx64 " unaligned (reftable index: %#"
+                                        PRIx64 ")", refblock_offset,
+                                        reftable_index);
+                error_setg(errp,
+                           "Image is corrupt (unaligned refblock offset)");
+                return -EIO;
+            }
+
+            ret = qcow2_cache_get(bs, s->refcount_block_cache, refblock_offset,
+                                  &refblock);
+            if (ret < 0) {
+                error_setg_errno(errp, -ret, "Failed to retrieve refblock");
+                return ret;
+            }
+
+            for (refblock_index = 0; refblock_index < s->refcount_block_size;
+                 refblock_index++)
+            {
+                uint64_t refcount;
+
+                if (new_refblock_index >= new_refblock_size) {
+                    /* new_refblock is now complete */
+                    ret = operation(bs, new_reftable, *new_reftable_index,
+                                    new_reftable_size, new_refblock,
+                                    new_refblock_empty, allocated, errp);
+                    if (ret < 0) {
+                        qcow2_cache_put(bs, s->refcount_block_cache, &refblock);
+                        return ret;
+                    }
+
+                    (*new_reftable_index)++;
+                    new_refblock_index = 0;
+                    new_refblock_empty = true;
+                }
+
+                refcount = s->get_refcount(refblock, refblock_index);
+                if (new_refcount_bits < 64 && refcount >> new_refcount_bits) {
+                    uint64_t offset;
+
+                    qcow2_cache_put(bs, s->refcount_block_cache, &refblock);
+
+                    offset = ((reftable_index << s->refcount_block_bits)
+                              + refblock_index) << s->cluster_bits;
+
+                    error_setg(errp, "Cannot decrease refcount entry width to "
+                               "%i bits: Cluster at offset %#" PRIx64 " has a "
+                               "refcount of %" PRIu64, new_refcount_bits,
+                               offset, refcount);
+                    return -EINVAL;
+                }
+
+                if (new_set_refcount) {
+                    new_set_refcount(new_refblock, new_refblock_index++,
+                                     refcount);
+                } else {
+                    new_refblock_index++;
+                }
+                new_refblock_empty = new_refblock_empty && refcount == 0;
+            }
+
+            qcow2_cache_put(bs, s->refcount_block_cache, &refblock);
+        } else {
+            /* No refblock means every refcount is 0 */
+            for (refblock_index = 0; refblock_index < s->refcount_block_size;
+                 refblock_index++)
+            {
+                if (new_refblock_index >= new_refblock_size) {
+                    /* new_refblock is now complete */
+                    ret = operation(bs, new_reftable, *new_reftable_index,
+                                    new_reftable_size, new_refblock,
+                                    new_refblock_empty, allocated, errp);
+                    if (ret < 0) {
+                        return ret;
+                    }
+
+                    (*new_reftable_index)++;
+                    new_refblock_index = 0;
+                    new_refblock_empty = true;
+                }
+
+                if (new_set_refcount) {
+                    new_set_refcount(new_refblock, new_refblock_index++, 0);
+                } else {
+                    new_refblock_index++;
+                }
+            }
+        }
+    }
+
+    if (new_refblock_index > 0) {
+        /* Complete the potentially existing partially filled final refblock */
+        if (new_set_refcount) {
+            for (; new_refblock_index < new_refblock_size;
+                 new_refblock_index++)
+            {
+                new_set_refcount(new_refblock, new_refblock_index, 0);
+            }
+        }
+
+        ret = operation(bs, new_reftable, *new_reftable_index,
+                        new_reftable_size, new_refblock, new_refblock_empty,
+                        allocated, errp);
+        if (ret < 0) {
+            return ret;
+        }
+
+        (*new_reftable_index)++;
+    }
+
+    status_cb(bs, (uint64_t)(index + 1) * s->refcount_table_size,
+              (uint64_t)total * s->refcount_table_size, cb_opaque);
+
+    return 0;
+}
+
+int qcow2_change_refcount_order(BlockDriverState *bs, int refcount_order,
+                                BlockDriverAmendStatusCB *status_cb,
+                                void *cb_opaque, Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    Qcow2GetRefcountFunc *new_get_refcount;
+    Qcow2SetRefcountFunc *new_set_refcount;
+    void *new_refblock = qemu_blockalign(bs->file->bs, s->cluster_size);
+    uint64_t *new_reftable = NULL, new_reftable_size = 0;
+    uint64_t *old_reftable, old_reftable_size, old_reftable_offset;
+    uint64_t new_reftable_index = 0;
+    uint64_t i;
+    int64_t new_reftable_offset = 0, allocated_reftable_size = 0;
+    int new_refblock_size, new_refcount_bits = 1 << refcount_order;
+    int old_refcount_order;
+    int walk_index = 0;
+    int ret;
+    bool new_allocation;
+
+    assert(s->qcow_version >= 3);
+    assert(refcount_order >= 0 && refcount_order <= 6);
+
+    /* see qcow2_open() */
+    new_refblock_size = 1 << (s->cluster_bits - (refcount_order - 3));
+
+    new_get_refcount = get_refcount_funcs[refcount_order];
+    new_set_refcount = set_refcount_funcs[refcount_order];
+
+
+    do {
+        int total_walks;
+
+        new_allocation = false;
+
+        /* At least we have to do this walk and the one which writes the
+         * refblocks; also, at least we have to do this loop here at least
+         * twice (normally), first to do the allocations, and second to
+         * determine that everything is correctly allocated, this then makes
+         * three walks in total */
+        total_walks = MAX(walk_index + 2, 3);
+
+        /* First, allocate the structures so they are present in the refcount
+         * structures */
+        ret = walk_over_reftable(bs, &new_reftable, &new_reftable_index,
+                                 &new_reftable_size, NULL, new_refblock_size,
+                                 new_refcount_bits, &alloc_refblock,
+                                 &new_allocation, NULL, status_cb, cb_opaque,
+                                 walk_index++, total_walks, errp);
+        if (ret < 0) {
+            goto done;
+        }
+
+        new_reftable_index = 0;
+
+        if (new_allocation) {
+            if (new_reftable_offset) {
+                qcow2_free_clusters(bs, new_reftable_offset,
+                                    allocated_reftable_size * sizeof(uint64_t),
+                                    QCOW2_DISCARD_NEVER);
+            }
+
+            new_reftable_offset = qcow2_alloc_clusters(bs, new_reftable_size *
+                                                           sizeof(uint64_t));
+            if (new_reftable_offset < 0) {
+                error_setg_errno(errp, -new_reftable_offset,
+                                 "Failed to allocate the new reftable");
+                ret = new_reftable_offset;
+                goto done;
+            }
+            allocated_reftable_size = new_reftable_size;
+        }
+    } while (new_allocation);
+
+    /* Second, write the new refblocks */
+    ret = walk_over_reftable(bs, &new_reftable, &new_reftable_index,
+                             &new_reftable_size, new_refblock,
+                             new_refblock_size, new_refcount_bits,
+                             &flush_refblock, &new_allocation, new_set_refcount,
+                             status_cb, cb_opaque, walk_index, walk_index + 1,
+                             errp);
+    if (ret < 0) {
+        goto done;
+    }
+    assert(!new_allocation);
+
+
+    /* Write the new reftable */
+    ret = qcow2_pre_write_overlap_check(bs, 0, new_reftable_offset,
+                                        new_reftable_size * sizeof(uint64_t));
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Overlap check failed");
+        goto done;
+    }
+
+    for (i = 0; i < new_reftable_size; i++) {
+        cpu_to_be64s(&new_reftable[i]);
+    }
+
+    ret = bdrv_pwrite(bs->file, new_reftable_offset, new_reftable,
+                      new_reftable_size * sizeof(uint64_t));
+
+    for (i = 0; i < new_reftable_size; i++) {
+        be64_to_cpus(&new_reftable[i]);
+    }
+
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Failed to write the new reftable");
+        goto done;
+    }
+
+
+    /* Empty the refcount cache */
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Failed to flush the refblock cache");
+        goto done;
+    }
+
+    /* Update the image header to point to the new reftable; this only updates
+     * the fields which are relevant to qcow2_update_header(); other fields
+     * such as s->refcount_table or s->refcount_bits stay stale for now
+     * (because we have to restore everything if qcow2_update_header() fails) */
+    old_refcount_order  = s->refcount_order;
+    old_reftable_size   = s->refcount_table_size;
+    old_reftable_offset = s->refcount_table_offset;
+
+    s->refcount_order        = refcount_order;
+    s->refcount_table_size   = new_reftable_size;
+    s->refcount_table_offset = new_reftable_offset;
+
+    ret = qcow2_update_header(bs);
+    if (ret < 0) {
+        s->refcount_order        = old_refcount_order;
+        s->refcount_table_size   = old_reftable_size;
+        s->refcount_table_offset = old_reftable_offset;
+        error_setg_errno(errp, -ret, "Failed to update the qcow2 header");
+        goto done;
+    }
+
+    /* Now update the rest of the in-memory information */
+    old_reftable = s->refcount_table;
+    s->refcount_table = new_reftable;
+
+    s->refcount_bits = 1 << refcount_order;
+    s->refcount_max = UINT64_C(1) << (s->refcount_bits - 1);
+    s->refcount_max += s->refcount_max - 1;
+
+    s->refcount_block_bits = s->cluster_bits - (refcount_order - 3);
+    s->refcount_block_size = 1 << s->refcount_block_bits;
+
+    s->get_refcount = new_get_refcount;
+    s->set_refcount = new_set_refcount;
+
+    /* For cleaning up all old refblocks and the old reftable below the "done"
+     * label */
+    new_reftable        = old_reftable;
+    new_reftable_size   = old_reftable_size;
+    new_reftable_offset = old_reftable_offset;
+
+done:
+    if (new_reftable) {
+        /* On success, new_reftable actually points to the old reftable (and
+         * new_reftable_size is the old reftable's size); but that is just
+         * fine */
+        for (i = 0; i < new_reftable_size; i++) {
+            uint64_t offset = new_reftable[i] & REFT_OFFSET_MASK;
+            if (offset) {
+                qcow2_free_clusters(bs, offset, s->cluster_size,
+                                    QCOW2_DISCARD_OTHER);
+            }
+        }
+        g_free(new_reftable);
+
+        if (new_reftable_offset > 0) {
+            qcow2_free_clusters(bs, new_reftable_offset,
+                                new_reftable_size * sizeof(uint64_t),
+                                QCOW2_DISCARD_OTHER);
+        }
+    }
+
+    qemu_vfree(new_refblock);
+    return ret;
+}
diff --git a/block/qcow2-snapshot.c b/block/qcow2-snapshot.c
index 5b3903c..0324243 100644
--- a/block/qcow2-snapshot.c
+++ b/block/qcow2-snapshot.c
@@ -22,13 +22,17 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 #include "block/qcow2.h"
+#include "qemu/bswap.h"
+#include "qemu/error-report.h"
+#include "qemu/cutils.h"
 
 void qcow2_free_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
 
     for(i = 0; i < s->nb_snapshots; i++) {
@@ -42,7 +46,7 @@
 
 int qcow2_read_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshotHeader h;
     QCowSnapshotExtraData extra;
     QCowSnapshot *sn;
@@ -135,7 +139,7 @@
 /* add at the end of the file a new list of snapshots */
 static int qcow2_write_snapshots(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     QCowSnapshotHeader h;
     QCowSnapshotExtraData extra;
@@ -277,7 +281,7 @@
 static void find_new_snapshot_id(BlockDriverState *bs,
                                  char *id_str, int id_str_size)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     int i;
     unsigned long id, id_max = 0;
@@ -295,7 +299,7 @@
                                         const char *id,
                                         const char *name)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int i;
 
     if (id && name) {
@@ -337,7 +341,7 @@
 /* if no id is provided, a new one is constructed */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *new_snapshot_list = NULL;
     QCowSnapshot *old_snapshot_list = NULL;
     QCowSnapshot sn1, *sn = &sn1;
@@ -351,10 +355,8 @@
 
     memset(sn, 0, sizeof(*sn));
 
-    /* Generate an ID if it wasn't passed */
-    if (sn_info->id_str[0] == '\0') {
-        find_new_snapshot_id(bs, sn_info->id_str, sizeof(sn_info->id_str));
-    }
+    /* Generate an ID */
+    find_new_snapshot_id(bs, sn_info->id_str, sizeof(sn_info->id_str));
 
     /* Check that the ID is unique */
     if (find_snapshot_by_id_and_name(bs, sn_info->id_str, NULL) >= 0) {
@@ -462,7 +464,7 @@
 /* copy the snapshot 'snapshot_name' into the current disk image */
 int qcow2_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     int i, snapshot_index;
     int cur_l1_bytes, sn_l1_bytes;
@@ -510,7 +512,8 @@
         goto fail;
     }
 
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, sn_l1_table, sn_l1_bytes);
+    ret = bdrv_pread(bs->file, sn->l1_table_offset,
+                     sn_l1_table, sn_l1_bytes);
     if (ret < 0) {
         goto fail;
     }
@@ -588,7 +591,7 @@
                           const char *name,
                           Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot sn;
     int snapshot_index, ret;
 
@@ -651,7 +654,7 @@
 
 int qcow2_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QEMUSnapshotInfo *sn_tab, *sn_info;
     QCowSnapshot *sn;
     int i;
@@ -684,7 +687,7 @@
                             Error **errp)
 {
     int i, snapshot_index;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowSnapshot *sn;
     uint64_t *new_l1_table;
     int new_l1_bytes;
@@ -702,18 +705,19 @@
     sn = &s->snapshots[snapshot_index];
 
     /* Allocate and read in the snapshot's L1 table */
-    if (sn->l1_size > QCOW_MAX_L1_SIZE) {
+    if (sn->l1_size > QCOW_MAX_L1_SIZE / sizeof(uint64_t)) {
         error_setg(errp, "Snapshot L1 table too large");
         return -EFBIG;
     }
     new_l1_bytes = sn->l1_size * sizeof(uint64_t);
-    new_l1_table = qemu_try_blockalign(bs->file,
+    new_l1_table = qemu_try_blockalign(bs->file->bs,
                                        align_offset(new_l1_bytes, 512));
     if (new_l1_table == NULL) {
         return -ENOMEM;
     }
 
-    ret = bdrv_pread(bs->file, sn->l1_table_offset, new_l1_table, new_l1_bytes);
+    ret = bdrv_pread(bs->file, sn->l1_table_offset,
+                     new_l1_table, new_l1_bytes);
     if (ret < 0) {
         error_setg(errp, "Failed to read l1 table for snapshot");
         qemu_vfree(new_l1_table);
diff --git a/block/qcow2.c b/block/qcow2.c
index d120494..91ef4df 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -21,11 +21,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include <zlib.h>
-#include "qemu/aes.h"
 #include "block/qcow2.h"
 #include "qemu/error-report.h"
 #include "qapi/qmp/qerror.h"
@@ -35,6 +35,8 @@
 #include "qapi-event.h"
 #include "trace.h"
 #include "qemu/option_int.h"
+#include "qemu/cutils.h"
+#include "qemu/bswap.h"
 
 /*
   Differences with QCOW:
@@ -86,7 +88,7 @@
                                  uint64_t end_offset, void **p_feature_table,
                                  Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowExtension ext;
     uint64_t offset;
     int ret;
@@ -117,7 +119,7 @@
 #ifdef DEBUG_EXT
         printf("ext.magic = 0x%x\n", ext.magic);
 #endif
-        if (ext.len > end_offset - offset) {
+        if (offset > end_offset || ext.len > end_offset - offset) {
             error_setg(errp, "Header extension too large");
             return -EINVAL;
         }
@@ -140,6 +142,7 @@
                 return 3;
             }
             bs->backing_format[ext.len] = '\0';
+            s->image_backing_format = g_strdup(bs->backing_format);
 #ifdef DEBUG_EXT
             printf("Qcow2: Got format extension %s\n", bs->backing_format);
 #endif
@@ -187,7 +190,7 @@
 
 static void cleanup_unknown_header_ext(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     Qcow2UnknownHeaderExtension *uext, *next;
 
     QLIST_FOREACH_SAFE(uext, &s->unknown_header_ext, next, next) {
@@ -196,22 +199,8 @@
     }
 }
 
-static void GCC_FMT_ATTR(3, 4) report_unsupported(BlockDriverState *bs,
-    Error **errp, const char *fmt, ...)
-{
-    char msg[64];
-    va_list ap;
-
-    va_start(ap, fmt);
-    vsnprintf(msg, sizeof(msg), fmt, ap);
-    va_end(ap);
-
-    error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-              bdrv_get_device_name(bs), "qcow2", msg);
-}
-
-static void report_unsupported_feature(BlockDriverState *bs,
-    Error **errp, Qcow2Feature *table, uint64_t mask)
+static void report_unsupported_feature(Error **errp, Qcow2Feature *table,
+                                       uint64_t mask)
 {
     char *features = g_strdup("");
     char *old;
@@ -236,7 +225,7 @@
         g_free(old);
     }
 
-    report_unsupported(bs, errp, "%s", features);
+    error_setg(errp, "Unsupported qcow2 feature(s): %s", features);
     g_free(features);
 }
 
@@ -249,7 +238,7 @@
  */
 int qcow2_mark_dirty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t val;
     int ret;
 
@@ -265,7 +254,7 @@
     if (ret < 0) {
         return ret;
     }
-    ret = bdrv_flush(bs->file);
+    ret = bdrv_flush(bs->file->bs);
     if (ret < 0) {
         return ret;
     }
@@ -282,7 +271,7 @@
  */
 static int qcow2_mark_clean(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     if (s->incompatible_features & QCOW2_INCOMPAT_DIRTY) {
         int ret;
@@ -304,7 +293,7 @@
  */
 int qcow2_mark_corrupt(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     s->incompatible_features |= QCOW2_INCOMPAT_CORRUPT;
     return qcow2_update_header(bs);
@@ -316,7 +305,7 @@
  */
 int qcow2_mark_consistent(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     if (s->incompatible_features & QCOW2_INCOMPAT_CORRUPT) {
         int ret = bdrv_flush(bs);
@@ -351,7 +340,7 @@
 static int validate_table_offset(BlockDriverState *bs, uint64_t offset,
                                  uint64_t entries, size_t entry_len)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t size;
 
     /* Use signed INT64_MAX as the maximum even for uint64_t header fields,
@@ -467,6 +456,11 @@
             .type = QEMU_OPT_SIZE,
             .help = "Maximum refcount block cache size",
         },
+        {
+            .name = QCOW2_OPT_CACHE_CLEAN_INTERVAL,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Clean unused cache entries after this time (in seconds)",
+        },
         { /* end of list */ }
     },
 };
@@ -482,9 +476,54 @@
     [QCOW2_OL_INACTIVE_L2_BITNR]    = QCOW2_OPT_OVERLAP_INACTIVE_L2,
 };
 
-static void read_cache_sizes(QemuOpts *opts, uint64_t *l2_cache_size,
+static void cache_clean_timer_cb(void *opaque)
+{
+    BlockDriverState *bs = opaque;
+    BDRVQcow2State *s = bs->opaque;
+    qcow2_cache_clean_unused(bs, s->l2_table_cache);
+    qcow2_cache_clean_unused(bs, s->refcount_block_cache);
+    timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+              (int64_t) s->cache_clean_interval * 1000);
+}
+
+static void cache_clean_timer_init(BlockDriverState *bs, AioContext *context)
+{
+    BDRVQcow2State *s = bs->opaque;
+    if (s->cache_clean_interval > 0) {
+        s->cache_clean_timer = aio_timer_new(context, QEMU_CLOCK_VIRTUAL,
+                                             SCALE_MS, cache_clean_timer_cb,
+                                             bs);
+        timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+                  (int64_t) s->cache_clean_interval * 1000);
+    }
+}
+
+static void cache_clean_timer_del(BlockDriverState *bs)
+{
+    BDRVQcow2State *s = bs->opaque;
+    if (s->cache_clean_timer) {
+        timer_del(s->cache_clean_timer);
+        timer_free(s->cache_clean_timer);
+        s->cache_clean_timer = NULL;
+    }
+}
+
+static void qcow2_detach_aio_context(BlockDriverState *bs)
+{
+    cache_clean_timer_del(bs);
+}
+
+static void qcow2_attach_aio_context(BlockDriverState *bs,
+                                     AioContext *new_context)
+{
+    cache_clean_timer_init(bs, new_context);
+}
+
+static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts,
+                             uint64_t *l2_cache_size,
                              uint64_t *refcount_cache_size, Error **errp)
 {
+    BDRVQcow2State *s = bs->opaque;
     uint64_t combined_cache_size;
     bool l2_cache_size_set, refcount_cache_size_set, combined_cache_size_set;
 
@@ -524,7 +563,9 @@
         }
     } else {
         if (!l2_cache_size_set && !refcount_cache_size_set) {
-            *l2_cache_size = DEFAULT_L2_CACHE_BYTE_SIZE;
+            *l2_cache_size = MAX(DEFAULT_L2_CACHE_BYTE_SIZE,
+                                 (uint64_t)DEFAULT_L2_CACHE_CLUSTERS
+                                 * s->cluster_size);
             *refcount_cache_size = *l2_cache_size
                                  / DEFAULT_L2_REFCOUNT_SIZE_RATIO;
         } else if (!l2_cache_size_set) {
@@ -537,20 +578,244 @@
     }
 }
 
-static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
-                      Error **errp)
+typedef struct Qcow2ReopenState {
+    Qcow2Cache *l2_table_cache;
+    Qcow2Cache *refcount_block_cache;
+    bool use_lazy_refcounts;
+    int overlap_check;
+    bool discard_passthrough[QCOW2_DISCARD_MAX];
+    uint64_t cache_clean_interval;
+} Qcow2ReopenState;
+
+static int qcow2_update_options_prepare(BlockDriverState *bs,
+                                        Qcow2ReopenState *r,
+                                        QDict *options, int flags,
+                                        Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
-    unsigned int len, i;
-    int ret = 0;
-    QCowHeader header;
+    BDRVQcow2State *s = bs->opaque;
     QemuOpts *opts = NULL;
-    Error *local_err = NULL;
-    uint64_t ext_end;
-    uint64_t l1_vm_state_index;
     const char *opt_overlap_check, *opt_overlap_check_template;
     int overlap_check_template = 0;
     uint64_t l2_cache_size, refcount_cache_size;
+    int i;
+    Error *local_err = NULL;
+    int ret;
+
+    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* get L2 table/refcount block cache size from command line options */
+    read_cache_sizes(bs, opts, &l2_cache_size, &refcount_cache_size,
+                     &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    l2_cache_size /= s->cluster_size;
+    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
+        l2_cache_size = MIN_L2_CACHE_SIZE;
+    }
+    if (l2_cache_size > INT_MAX) {
+        error_setg(errp, "L2 cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    refcount_cache_size /= s->cluster_size;
+    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
+        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
+    }
+    if (refcount_cache_size > INT_MAX) {
+        error_setg(errp, "Refcount cache size too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* alloc new L2 table/refcount block cache, flush old one */
+    if (s->l2_table_cache) {
+        ret = qcow2_cache_flush(bs, s->l2_table_cache);
+        if (ret) {
+            error_setg_errno(errp, -ret, "Failed to flush the L2 table cache");
+            goto fail;
+        }
+    }
+
+    if (s->refcount_block_cache) {
+        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+        if (ret) {
+            error_setg_errno(errp, -ret,
+                             "Failed to flush the refcount block cache");
+            goto fail;
+        }
+    }
+
+    r->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
+    r->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
+    if (r->l2_table_cache == NULL || r->refcount_block_cache == NULL) {
+        error_setg(errp, "Could not allocate metadata caches");
+        ret = -ENOMEM;
+        goto fail;
+    }
+
+    /* New interval for cache cleanup timer */
+    r->cache_clean_interval =
+        qemu_opt_get_number(opts, QCOW2_OPT_CACHE_CLEAN_INTERVAL,
+                            s->cache_clean_interval);
+    if (r->cache_clean_interval > UINT_MAX) {
+        error_setg(errp, "Cache clean interval too big");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* lazy-refcounts; flush if going from enabled to disabled */
+    r->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
+        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
+    if (r->use_lazy_refcounts && s->qcow_version < 3) {
+        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
+                   "qemu 1.1 compatibility level");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    if (s->use_lazy_refcounts && !r->use_lazy_refcounts) {
+        ret = qcow2_mark_clean(bs);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Failed to disable lazy refcounts");
+            goto fail;
+        }
+    }
+
+    /* Overlap check options */
+    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
+    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
+    if (opt_overlap_check_template && opt_overlap_check &&
+        strcmp(opt_overlap_check_template, opt_overlap_check))
+    {
+        error_setg(errp, "Conflicting values for qcow2 options '"
+                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
+                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (!opt_overlap_check) {
+        opt_overlap_check = opt_overlap_check_template ?: "cached";
+    }
+
+    if (!strcmp(opt_overlap_check, "none")) {
+        overlap_check_template = 0;
+    } else if (!strcmp(opt_overlap_check, "constant")) {
+        overlap_check_template = QCOW2_OL_CONSTANT;
+    } else if (!strcmp(opt_overlap_check, "cached")) {
+        overlap_check_template = QCOW2_OL_CACHED;
+    } else if (!strcmp(opt_overlap_check, "all")) {
+        overlap_check_template = QCOW2_OL_ALL;
+    } else {
+        error_setg(errp, "Unsupported value '%s' for qcow2 option "
+                   "'overlap-check'. Allowed are any of the following: "
+                   "none, constant, cached, all", opt_overlap_check);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    r->overlap_check = 0;
+    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
+        /* overlap-check defines a template bitmask, but every flag may be
+         * overwritten through the associated boolean option */
+        r->overlap_check |=
+            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
+                              overlap_check_template & (1 << i)) << i;
+    }
+
+    r->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
+    r->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
+    r->discard_passthrough[QCOW2_DISCARD_REQUEST] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
+                          flags & BDRV_O_UNMAP);
+    r->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
+    r->discard_passthrough[QCOW2_DISCARD_OTHER] =
+        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
+
+    ret = 0;
+fail:
+    qemu_opts_del(opts);
+    opts = NULL;
+    return ret;
+}
+
+static void qcow2_update_options_commit(BlockDriverState *bs,
+                                        Qcow2ReopenState *r)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int i;
+
+    if (s->l2_table_cache) {
+        qcow2_cache_destroy(bs, s->l2_table_cache);
+    }
+    if (s->refcount_block_cache) {
+        qcow2_cache_destroy(bs, s->refcount_block_cache);
+    }
+    s->l2_table_cache = r->l2_table_cache;
+    s->refcount_block_cache = r->refcount_block_cache;
+
+    s->overlap_check = r->overlap_check;
+    s->use_lazy_refcounts = r->use_lazy_refcounts;
+
+    for (i = 0; i < QCOW2_DISCARD_MAX; i++) {
+        s->discard_passthrough[i] = r->discard_passthrough[i];
+    }
+
+    if (s->cache_clean_interval != r->cache_clean_interval) {
+        cache_clean_timer_del(bs);
+        s->cache_clean_interval = r->cache_clean_interval;
+        cache_clean_timer_init(bs, bdrv_get_aio_context(bs));
+    }
+}
+
+static void qcow2_update_options_abort(BlockDriverState *bs,
+                                       Qcow2ReopenState *r)
+{
+    if (r->l2_table_cache) {
+        qcow2_cache_destroy(bs, r->l2_table_cache);
+    }
+    if (r->refcount_block_cache) {
+        qcow2_cache_destroy(bs, r->refcount_block_cache);
+    }
+}
+
+static int qcow2_update_options(BlockDriverState *bs, QDict *options,
+                                int flags, Error **errp)
+{
+    Qcow2ReopenState r = {};
+    int ret;
+
+    ret = qcow2_update_options_prepare(bs, &r, options, flags, errp);
+    if (ret >= 0) {
+        qcow2_update_options_commit(bs, &r);
+    } else {
+        qcow2_update_options_abort(bs, &r);
+    }
+
+    return ret;
+}
+
+static int qcow2_open(BlockDriverState *bs, QDict *options, int flags,
+                      Error **errp)
+{
+    BDRVQcow2State *s = bs->opaque;
+    unsigned int len, i;
+    int ret = 0;
+    QCowHeader header;
+    Error *local_err = NULL;
+    uint64_t ext_end;
+    uint64_t l1_vm_state_index;
 
     ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
     if (ret < 0) {
@@ -577,7 +842,7 @@
         goto fail;
     }
     if (header.version < 2 || header.version > 3) {
-        report_unsupported(bs, errp, "QCOW version %" PRIu32, header.version);
+        error_setg(errp, "Unsupported qcow2 version %" PRIu32, header.version);
         ret = -ENOTSUP;
         goto fail;
     }
@@ -657,7 +922,7 @@
         void *feature_table = NULL;
         qcow2_read_extensions(bs, header.header_length, ext_end,
                               &feature_table, NULL);
-        report_unsupported_feature(bs, errp, feature_table,
+        report_unsupported_feature(errp, feature_table,
                                    s->incompatible_features &
                                    ~QCOW2_INCOMPAT_MASK);
         ret = -ENOTSUP;
@@ -677,13 +942,16 @@
     }
 
     /* Check support for various header values */
-    if (header.refcount_order != 4) {
-        report_unsupported(bs, errp, "%d bit reference counts",
-                           1 << header.refcount_order);
-        ret = -ENOTSUP;
+    if (header.refcount_order > 6) {
+        error_setg(errp, "Reference count entry width too large; may not "
+                   "exceed 64 bits");
+        ret = -EINVAL;
         goto fail;
     }
     s->refcount_order = header.refcount_order;
+    s->refcount_bits = 1 << s->refcount_order;
+    s->refcount_max = UINT64_C(1) << (s->refcount_bits - 1);
+    s->refcount_max += s->refcount_max - 1;
 
     if (header.crypt_method > QCOW_CRYPT_AES) {
         error_setg(errp, "Unsupported encryption method: %" PRIu32,
@@ -691,9 +959,28 @@
         ret = -EINVAL;
         goto fail;
     }
+    if (!qcrypto_cipher_supports(QCRYPTO_CIPHER_ALG_AES_128)) {
+        error_setg(errp, "AES cipher not available");
+        ret = -EINVAL;
+        goto fail;
+    }
     s->crypt_method_header = header.crypt_method;
     if (s->crypt_method_header) {
-        bs->encrypted = 1;
+        if (bdrv_uses_whitelist() &&
+            s->crypt_method_header == QCOW_CRYPT_AES) {
+            error_setg(errp,
+                       "Use of AES-CBC encrypted qcow2 images is no longer "
+                       "supported in system emulators");
+            error_append_hint(errp,
+                              "You can use 'qemu-img convert' to convert your "
+                              "image to an alternative supported format, such "
+                              "as unencrypted qcow2, or raw with the LUKS "
+                              "format instead.\n");
+            ret = -ENOSYS;
+            goto fail;
+        }
+
+        bs->encrypted = true;
     }
 
     s->l2_bits = s->cluster_bits - 3; /* L2 is always one cluster */
@@ -739,7 +1026,7 @@
     }
 
     /* read the level 1 table */
-    if (header.l1_size > QCOW_MAX_L1_SIZE) {
+    if (header.l1_size > QCOW_MAX_L1_SIZE / sizeof(uint64_t)) {
         error_setg(errp, "Active L1 table too large");
         ret = -EFBIG;
         goto fail;
@@ -772,7 +1059,7 @@
 
 
     if (s->l1_size > 0) {
-        s->l1_table = qemu_try_blockalign(bs->file,
+        s->l1_table = qemu_try_blockalign(bs->file->bs,
             align_offset(s->l1_size * sizeof(uint64_t), 512));
         if (s->l1_table == NULL) {
             error_setg(errp, "Could not allocate L1 table");
@@ -790,54 +1077,15 @@
         }
     }
 
-    /* get L2 table/refcount block cache size from command line options */
-    opts = qemu_opts_create(&qcow2_runtime_opts, NULL, 0, &error_abort);
-    qemu_opts_absorb_qdict(opts, options, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    read_cache_sizes(opts, &l2_cache_size, &refcount_cache_size, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    l2_cache_size /= s->cluster_size;
-    if (l2_cache_size < MIN_L2_CACHE_SIZE) {
-        l2_cache_size = MIN_L2_CACHE_SIZE;
-    }
-    if (l2_cache_size > INT_MAX) {
-        error_setg(errp, "L2 cache size too big");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    refcount_cache_size /= s->cluster_size;
-    if (refcount_cache_size < MIN_REFCOUNT_CACHE_SIZE) {
-        refcount_cache_size = MIN_REFCOUNT_CACHE_SIZE;
-    }
-    if (refcount_cache_size > INT_MAX) {
-        error_setg(errp, "Refcount cache size too big");
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    /* alloc L2 table/refcount block cache */
-    s->l2_table_cache = qcow2_cache_create(bs, l2_cache_size);
-    s->refcount_block_cache = qcow2_cache_create(bs, refcount_cache_size);
-    if (s->l2_table_cache == NULL || s->refcount_block_cache == NULL) {
-        error_setg(errp, "Could not allocate metadata caches");
-        ret = -ENOMEM;
+    /* Parse driver-specific options */
+    ret = qcow2_update_options(bs, options, flags, errp);
+    if (ret < 0) {
         goto fail;
     }
 
     s->cluster_cache = g_malloc(s->cluster_size);
     /* one more sector for decompressed data alignment */
-    s->cluster_data = qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
+    s->cluster_data = qemu_try_blockalign(bs->file->bs, QCOW_MAX_CRYPT_CLUSTERS
                                                     * s->cluster_size + 512);
     if (s->cluster_data == NULL) {
         error_setg(errp, "Could not allocate temporary cluster buffer");
@@ -868,7 +1116,8 @@
     /* read the backing file name */
     if (header.backing_file_offset != 0) {
         len = header.backing_file_size;
-        if (len > MIN(1023, s->cluster_size - header.backing_file_offset)) {
+        if (len > MIN(1023, s->cluster_size - header.backing_file_offset) ||
+            len >= sizeof(bs->backing_file)) {
             error_setg(errp, "Backing file name too long");
             ret = -EINVAL;
             goto fail;
@@ -880,6 +1129,7 @@
             goto fail;
         }
         bs->backing_file[len] = '\0';
+        s->image_backing_file = g_strdup(bs->backing_file);
     }
 
     /* Internal snapshots */
@@ -893,7 +1143,7 @@
     }
 
     /* Clear unknown autoclear feature bits */
-    if (!bs->read_only && !(flags & BDRV_O_INCOMING) && s->autoclear_features) {
+    if (!bs->read_only && !(flags & BDRV_O_INACTIVE) && s->autoclear_features) {
         s->autoclear_features = 0;
         ret = qcow2_update_header(bs);
         if (ret < 0) {
@@ -906,7 +1156,7 @@
     qemu_co_mutex_init(&s->lock);
 
     /* Repair image if dirty */
-    if (!(flags & (BDRV_O_CHECK | BDRV_O_INCOMING)) && !bs->read_only &&
+    if (!(flags & (BDRV_O_CHECK | BDRV_O_INACTIVE)) && !bs->read_only &&
         (s->incompatible_features & QCOW2_INCOMPAT_DIRTY)) {
         BdrvCheckResult result = {0};
 
@@ -917,70 +1167,6 @@
         }
     }
 
-    /* Enable lazy_refcounts according to image and command line options */
-    s->use_lazy_refcounts = qemu_opt_get_bool(opts, QCOW2_OPT_LAZY_REFCOUNTS,
-        (s->compatible_features & QCOW2_COMPAT_LAZY_REFCOUNTS));
-
-    s->discard_passthrough[QCOW2_DISCARD_NEVER] = false;
-    s->discard_passthrough[QCOW2_DISCARD_ALWAYS] = true;
-    s->discard_passthrough[QCOW2_DISCARD_REQUEST] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_REQUEST,
-                          flags & BDRV_O_UNMAP);
-    s->discard_passthrough[QCOW2_DISCARD_SNAPSHOT] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_SNAPSHOT, true);
-    s->discard_passthrough[QCOW2_DISCARD_OTHER] =
-        qemu_opt_get_bool(opts, QCOW2_OPT_DISCARD_OTHER, false);
-
-    opt_overlap_check = qemu_opt_get(opts, QCOW2_OPT_OVERLAP);
-    opt_overlap_check_template = qemu_opt_get(opts, QCOW2_OPT_OVERLAP_TEMPLATE);
-    if (opt_overlap_check_template && opt_overlap_check &&
-        strcmp(opt_overlap_check_template, opt_overlap_check))
-    {
-        error_setg(errp, "Conflicting values for qcow2 options '"
-                   QCOW2_OPT_OVERLAP "' ('%s') and '" QCOW2_OPT_OVERLAP_TEMPLATE
-                   "' ('%s')", opt_overlap_check, opt_overlap_check_template);
-        ret = -EINVAL;
-        goto fail;
-    }
-    if (!opt_overlap_check) {
-        opt_overlap_check = opt_overlap_check_template ?: "cached";
-    }
-
-    if (!strcmp(opt_overlap_check, "none")) {
-        overlap_check_template = 0;
-    } else if (!strcmp(opt_overlap_check, "constant")) {
-        overlap_check_template = QCOW2_OL_CONSTANT;
-    } else if (!strcmp(opt_overlap_check, "cached")) {
-        overlap_check_template = QCOW2_OL_CACHED;
-    } else if (!strcmp(opt_overlap_check, "all")) {
-        overlap_check_template = QCOW2_OL_ALL;
-    } else {
-        error_setg(errp, "Unsupported value '%s' for qcow2 option "
-                   "'overlap-check'. Allowed are either of the following: "
-                   "none, constant, cached, all", opt_overlap_check);
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    s->overlap_check = 0;
-    for (i = 0; i < QCOW2_OL_MAX_BITNR; i++) {
-        /* overlap-check defines a template bitmask, but every flag may be
-         * overwritten through the associated boolean option */
-        s->overlap_check |=
-            qemu_opt_get_bool(opts, overlap_bool_option_names[i],
-                              overlap_check_template & (1 << i)) << i;
-    }
-
-    qemu_opts_del(opts);
-    opts = NULL;
-
-    if (s->use_lazy_refcounts && s->qcow_version < 3) {
-        error_setg(errp, "Lazy refcounts require a qcow2 image with at least "
-                   "qemu 1.1 compatibility level");
-        ret = -EINVAL;
-        goto fail;
-    }
-
 #ifdef DEBUG_ALLOC
     {
         BdrvCheckResult result = {0};
@@ -990,7 +1176,6 @@
     return ret;
 
  fail:
-    qemu_opts_del(opts);
     g_free(s->unknown_header_fields);
     cleanup_unknown_header_ext(bs);
     qcow2_free_snapshots(bs);
@@ -998,6 +1183,7 @@
     qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
+    cache_clean_timer_del(bs);
     if (s->l2_table_cache) {
         qcow2_cache_destroy(bs, s->l2_table_cache);
     }
@@ -1011,16 +1197,21 @@
 
 static void qcow2_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
-    bs->bl.write_zeroes_alignment = s->cluster_sectors;
+    if (bs->encrypted) {
+        /* Encryption works on a sector granularity */
+        bs->bl.request_alignment = BDRV_SECTOR_SIZE;
+    }
+    bs->bl.pwrite_zeroes_alignment = s->cluster_size;
 }
 
 static int qcow2_set_key(BlockDriverState *bs, const char *key)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint8_t keybuf[16];
     int len, i;
+    Error *err = NULL;
 
     memset(keybuf, 0, 16);
     len = strlen(key);
@@ -1031,75 +1222,144 @@
     for(i = 0;i < len;i++) {
         keybuf[i] = key[i];
     }
-    s->crypt_method = s->crypt_method_header;
+    assert(bs->encrypted);
 
-    if (AES_set_encrypt_key(keybuf, 128, &s->aes_encrypt_key) != 0)
+    qcrypto_cipher_free(s->cipher);
+    s->cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_128,
+        QCRYPTO_CIPHER_MODE_CBC,
+        keybuf, G_N_ELEMENTS(keybuf),
+        &err);
+
+    if (!s->cipher) {
+        /* XXX would be nice if errors in this method could
+         * be properly propagate to the caller. Would need
+         * the bdrv_set_key() API signature to be fixed. */
+        error_free(err);
         return -1;
-    if (AES_set_decrypt_key(keybuf, 128, &s->aes_decrypt_key) != 0)
-        return -1;
-#if 0
-    /* test */
-    {
-        uint8_t in[16];
-        uint8_t out[16];
-        uint8_t tmp[16];
-        for(i=0;i<16;i++)
-            in[i] = i;
-        AES_encrypt(in, tmp, &s->aes_encrypt_key);
-        AES_decrypt(tmp, out, &s->aes_decrypt_key);
-        for(i = 0; i < 16; i++)
-            printf(" %02x", tmp[i]);
-        printf("\n");
-        for(i = 0; i < 16; i++)
-            printf(" %02x", out[i]);
-        printf("\n");
     }
-#endif
     return 0;
 }
 
-/* We have no actual commit/abort logic for qcow2, but we need to write out any
- * unwritten data if we reopen read-only. */
 static int qcow2_reopen_prepare(BDRVReopenState *state,
                                 BlockReopenQueue *queue, Error **errp)
 {
+    Qcow2ReopenState *r;
     int ret;
 
+    r = g_new0(Qcow2ReopenState, 1);
+    state->opaque = r;
+
+    ret = qcow2_update_options_prepare(state->bs, r, state->options,
+                                       state->flags, errp);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* We need to write out any unwritten data if we reopen read-only. */
     if ((state->flags & BDRV_O_RDWR) == 0) {
         ret = bdrv_flush(state->bs);
         if (ret < 0) {
-            return ret;
+            goto fail;
         }
 
         ret = qcow2_mark_clean(state->bs);
         if (ret < 0) {
-            return ret;
+            goto fail;
         }
     }
 
     return 0;
+
+fail:
+    qcow2_update_options_abort(state->bs, r);
+    g_free(r);
+    return ret;
+}
+
+static void qcow2_reopen_commit(BDRVReopenState *state)
+{
+    qcow2_update_options_commit(state->bs, state->opaque);
+    g_free(state->opaque);
+}
+
+static void qcow2_reopen_abort(BDRVReopenState *state)
+{
+    qcow2_update_options_abort(state->bs, state->opaque);
+    g_free(state->opaque);
+}
+
+static void qcow2_join_options(QDict *options, QDict *old_options)
+{
+    bool has_new_overlap_template =
+        qdict_haskey(options, QCOW2_OPT_OVERLAP) ||
+        qdict_haskey(options, QCOW2_OPT_OVERLAP_TEMPLATE);
+    bool has_new_total_cache_size =
+        qdict_haskey(options, QCOW2_OPT_CACHE_SIZE);
+    bool has_all_cache_options;
+
+    /* New overlap template overrides all old overlap options */
+    if (has_new_overlap_template) {
+        qdict_del(old_options, QCOW2_OPT_OVERLAP);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_TEMPLATE);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_MAIN_HEADER);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_ACTIVE_L1);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_ACTIVE_L2);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_REFCOUNT_TABLE);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_REFCOUNT_BLOCK);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_SNAPSHOT_TABLE);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_INACTIVE_L1);
+        qdict_del(old_options, QCOW2_OPT_OVERLAP_INACTIVE_L2);
+    }
+
+    /* New total cache size overrides all old options */
+    if (qdict_haskey(options, QCOW2_OPT_CACHE_SIZE)) {
+        qdict_del(old_options, QCOW2_OPT_L2_CACHE_SIZE);
+        qdict_del(old_options, QCOW2_OPT_REFCOUNT_CACHE_SIZE);
+    }
+
+    qdict_join(options, old_options, false);
+
+    /*
+     * If after merging all cache size options are set, an old total size is
+     * overwritten. Do keep all options, however, if all three are new. The
+     * resulting error message is what we want to happen.
+     */
+    has_all_cache_options =
+        qdict_haskey(options, QCOW2_OPT_CACHE_SIZE) ||
+        qdict_haskey(options, QCOW2_OPT_L2_CACHE_SIZE) ||
+        qdict_haskey(options, QCOW2_OPT_REFCOUNT_CACHE_SIZE);
+
+    if (has_all_cache_options && !has_new_total_cache_size) {
+        qdict_del(options, QCOW2_OPT_CACHE_SIZE);
+    }
 }
 
 static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t cluster_offset;
     int index_in_cluster, ret;
+    unsigned int bytes;
     int64_t status = 0;
 
-    *pnum = nb_sectors;
+    bytes = MIN(INT_MAX, nb_sectors * BDRV_SECTOR_SIZE);
     qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_get_cluster_offset(bs, sector_num << 9, pnum, &cluster_offset);
+    ret = qcow2_get_cluster_offset(bs, sector_num << 9, &bytes,
+                                   &cluster_offset);
     qemu_co_mutex_unlock(&s->lock);
     if (ret < 0) {
         return ret;
     }
 
+    *pnum = bytes >> BDRV_SECTOR_BITS;
+
     if (cluster_offset != 0 && ret != QCOW2_CLUSTER_COMPRESSED &&
-        !s->crypt_method) {
+        !s->cipher) {
         index_in_cluster = sector_num & (s->cluster_sectors - 1);
         cluster_offset |= (index_in_cluster << BDRV_SECTOR_BITS);
+        *file = bs->file->bs;
         status |= BDRV_BLOCK_OFFSET_VALID | cluster_offset;
     }
     if (ret == QCOW2_CLUSTER_ZERO) {
@@ -1112,28 +1372,34 @@
 
 /* handle reading after the end of the backing file */
 int qcow2_backing_read1(BlockDriverState *bs, QEMUIOVector *qiov,
-                  int64_t sector_num, int nb_sectors)
+                        int64_t offset, int bytes)
 {
+    uint64_t bs_size = bs->total_sectors * BDRV_SECTOR_SIZE;
     int n1;
-    if ((sector_num + nb_sectors) <= bs->total_sectors)
-        return nb_sectors;
-    if (sector_num >= bs->total_sectors)
-        n1 = 0;
-    else
-        n1 = bs->total_sectors - sector_num;
 
-    qemu_iovec_memset(qiov, 512 * n1, 0, 512 * (nb_sectors - n1));
+    if ((offset + bytes) <= bs_size) {
+        return bytes;
+    }
+
+    if (offset >= bs_size) {
+        n1 = 0;
+    } else {
+        n1 = bs_size - offset;
+    }
+
+    qemu_iovec_memset(qiov, n1, 0, bytes - n1);
 
     return n1;
 }
 
-static coroutine_fn int qcow2_co_readv(BlockDriverState *bs, int64_t sector_num,
-                          int remaining_sectors, QEMUIOVector *qiov)
+static coroutine_fn int qcow2_co_preadv(BlockDriverState *bs, uint64_t offset,
+                                        uint64_t bytes, QEMUIOVector *qiov,
+                                        int flags)
 {
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster, n1;
+    BDRVQcow2State *s = bs->opaque;
+    int offset_in_cluster, n1;
     int ret;
-    int cur_nr_sectors; /* number of sectors in current iteration */
+    unsigned int cur_bytes; /* number of bytes in current iteration */
     uint64_t cluster_offset = 0;
     uint64_t bytes_done = 0;
     QEMUIOVector hd_qiov;
@@ -1143,45 +1409,42 @@
 
     qemu_co_mutex_lock(&s->lock);
 
-    while (remaining_sectors != 0) {
+    while (bytes != 0) {
 
         /* prepare next request */
-        cur_nr_sectors = remaining_sectors;
-        if (s->crypt_method) {
-            cur_nr_sectors = MIN(cur_nr_sectors,
-                QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors);
+        cur_bytes = MIN(bytes, INT_MAX);
+        if (s->cipher) {
+            cur_bytes = MIN(cur_bytes,
+                            QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
         }
 
-        ret = qcow2_get_cluster_offset(bs, sector_num << 9,
-            &cur_nr_sectors, &cluster_offset);
+        ret = qcow2_get_cluster_offset(bs, offset, &cur_bytes, &cluster_offset);
         if (ret < 0) {
             goto fail;
         }
 
-        index_in_cluster = sector_num & (s->cluster_sectors - 1);
+        offset_in_cluster = offset_into_cluster(s, offset);
 
         qemu_iovec_reset(&hd_qiov);
-        qemu_iovec_concat(&hd_qiov, qiov, bytes_done,
-            cur_nr_sectors * 512);
+        qemu_iovec_concat(&hd_qiov, qiov, bytes_done, cur_bytes);
 
         switch (ret) {
         case QCOW2_CLUSTER_UNALLOCATED:
 
-            if (bs->backing_hd) {
+            if (bs->backing) {
                 /* read from the base image */
-                n1 = qcow2_backing_read1(bs->backing_hd, &hd_qiov,
-                    sector_num, cur_nr_sectors);
+                n1 = qcow2_backing_read1(bs->backing->bs, &hd_qiov,
+                                         offset, cur_bytes);
                 if (n1 > 0) {
                     QEMUIOVector local_qiov;
 
                     qemu_iovec_init(&local_qiov, hd_qiov.niov);
-                    qemu_iovec_concat(&local_qiov, &hd_qiov, 0,
-                                      n1 * BDRV_SECTOR_SIZE);
+                    qemu_iovec_concat(&local_qiov, &hd_qiov, 0, n1);
 
                     BLKDBG_EVENT(bs->file, BLKDBG_READ_BACKING_AIO);
                     qemu_co_mutex_unlock(&s->lock);
-                    ret = bdrv_co_readv(bs->backing_hd, sector_num,
-                                        n1, &local_qiov);
+                    ret = bdrv_co_preadv(bs->backing, offset, n1,
+                                         &local_qiov, 0);
                     qemu_co_mutex_lock(&s->lock);
 
                     qemu_iovec_destroy(&local_qiov);
@@ -1192,12 +1455,12 @@
                 }
             } else {
                 /* Note: in this case, no need to wait */
-                qemu_iovec_memset(&hd_qiov, 0, 0, 512 * cur_nr_sectors);
+                qemu_iovec_memset(&hd_qiov, 0, 0, cur_bytes);
             }
             break;
 
         case QCOW2_CLUSTER_ZERO:
-            qemu_iovec_memset(&hd_qiov, 0, 0, 512 * cur_nr_sectors);
+            qemu_iovec_memset(&hd_qiov, 0, 0, cur_bytes);
             break;
 
         case QCOW2_CLUSTER_COMPRESSED:
@@ -1208,8 +1471,8 @@
             }
 
             qemu_iovec_from_buf(&hd_qiov, 0,
-                s->cluster_cache + index_in_cluster * 512,
-                512 * cur_nr_sectors);
+                                s->cluster_cache + offset_in_cluster,
+                                cur_bytes);
             break;
 
         case QCOW2_CLUSTER_NORMAL:
@@ -1218,42 +1481,52 @@
                 goto fail;
             }
 
-            if (s->crypt_method) {
+            if (bs->encrypted) {
+                assert(s->cipher);
+
                 /*
                  * For encrypted images, read everything into a temporary
                  * contiguous buffer on which the AES functions can work.
                  */
                 if (!cluster_data) {
                     cluster_data =
-                        qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS
-                                                      * s->cluster_size);
+                        qemu_try_blockalign(bs->file->bs,
+                                            QCOW_MAX_CRYPT_CLUSTERS
+                                            * s->cluster_size);
                     if (cluster_data == NULL) {
                         ret = -ENOMEM;
                         goto fail;
                     }
                 }
 
-                assert(cur_nr_sectors <=
-                    QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors);
+                assert(cur_bytes <= QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
                 qemu_iovec_reset(&hd_qiov);
-                qemu_iovec_add(&hd_qiov, cluster_data,
-                    512 * cur_nr_sectors);
+                qemu_iovec_add(&hd_qiov, cluster_data, cur_bytes);
             }
 
             BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
             qemu_co_mutex_unlock(&s->lock);
-            ret = bdrv_co_readv(bs->file,
-                                (cluster_offset >> 9) + index_in_cluster,
-                                cur_nr_sectors, &hd_qiov);
+            ret = bdrv_co_preadv(bs->file,
+                                 cluster_offset + offset_in_cluster,
+                                 cur_bytes, &hd_qiov, 0);
             qemu_co_mutex_lock(&s->lock);
             if (ret < 0) {
                 goto fail;
             }
-            if (s->crypt_method) {
-                qcow2_encrypt_sectors(s, sector_num,  cluster_data,
-                    cluster_data, cur_nr_sectors, 0, &s->aes_decrypt_key);
-                qemu_iovec_from_buf(qiov, bytes_done,
-                    cluster_data, 512 * cur_nr_sectors);
+            if (bs->encrypted) {
+                assert(s->cipher);
+                assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+                assert((cur_bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+                Error *err = NULL;
+                if (qcow2_encrypt_sectors(s, offset >> BDRV_SECTOR_BITS,
+                                          cluster_data, cluster_data,
+                                          cur_bytes >> BDRV_SECTOR_BITS,
+                                          false, &err) < 0) {
+                    error_free(err);
+                    ret = -EIO;
+                    goto fail;
+                }
+                qemu_iovec_from_buf(qiov, bytes_done, cluster_data, cur_bytes);
             }
             break;
 
@@ -1263,9 +1536,9 @@
             goto fail;
         }
 
-        remaining_sectors -= cur_nr_sectors;
-        sector_num += cur_nr_sectors;
-        bytes_done += cur_nr_sectors * 512;
+        bytes -= cur_bytes;
+        offset += cur_bytes;
+        bytes_done += cur_bytes;
     }
     ret = 0;
 
@@ -1278,23 +1551,21 @@
     return ret;
 }
 
-static coroutine_fn int qcow2_co_writev(BlockDriverState *bs,
-                           int64_t sector_num,
-                           int remaining_sectors,
-                           QEMUIOVector *qiov)
+static coroutine_fn int qcow2_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                                         uint64_t bytes, QEMUIOVector *qiov,
+                                         int flags)
 {
-    BDRVQcowState *s = bs->opaque;
-    int index_in_cluster;
+    BDRVQcow2State *s = bs->opaque;
+    int offset_in_cluster;
     int ret;
-    int cur_nr_sectors; /* number of sectors in current iteration */
+    unsigned int cur_bytes; /* number of sectors in current iteration */
     uint64_t cluster_offset;
     QEMUIOVector hd_qiov;
     uint64_t bytes_done = 0;
     uint8_t *cluster_data = NULL;
     QCowL2Meta *l2meta = NULL;
 
-    trace_qcow2_writev_start_req(qemu_coroutine_self(), sector_num,
-                                 remaining_sectors);
+    trace_qcow2_writev_start_req(qemu_coroutine_self(), offset, bytes);
 
     qemu_iovec_init(&hd_qiov, qiov->niov);
 
@@ -1302,22 +1573,21 @@
 
     qemu_co_mutex_lock(&s->lock);
 
-    while (remaining_sectors != 0) {
+    while (bytes != 0) {
 
         l2meta = NULL;
 
         trace_qcow2_writev_start_part(qemu_coroutine_self());
-        index_in_cluster = sector_num & (s->cluster_sectors - 1);
-        cur_nr_sectors = remaining_sectors;
-        if (s->crypt_method &&
-            cur_nr_sectors >
-            QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors - index_in_cluster) {
-            cur_nr_sectors =
-                QCOW_MAX_CRYPT_CLUSTERS * s->cluster_sectors - index_in_cluster;
+        offset_in_cluster = offset_into_cluster(s, offset);
+        cur_bytes = MIN(bytes, INT_MAX);
+        if (bs->encrypted) {
+            cur_bytes = MIN(cur_bytes,
+                            QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size
+                            - offset_in_cluster);
         }
 
-        ret = qcow2_alloc_cluster_offset(bs, sector_num << 9,
-            &cur_nr_sectors, &cluster_offset, &l2meta);
+        ret = qcow2_alloc_cluster_offset(bs, offset, &cur_bytes,
+                                         &cluster_offset, &l2meta);
         if (ret < 0) {
             goto fail;
         }
@@ -1325,12 +1595,13 @@
         assert((cluster_offset & 511) == 0);
 
         qemu_iovec_reset(&hd_qiov);
-        qemu_iovec_concat(&hd_qiov, qiov, bytes_done,
-            cur_nr_sectors * 512);
+        qemu_iovec_concat(&hd_qiov, qiov, bytes_done, cur_bytes);
 
-        if (s->crypt_method) {
+        if (bs->encrypted) {
+            Error *err = NULL;
+            assert(s->cipher);
             if (!cluster_data) {
-                cluster_data = qemu_try_blockalign(bs->file,
+                cluster_data = qemu_try_blockalign(bs->file->bs,
                                                    QCOW_MAX_CRYPT_CLUSTERS
                                                    * s->cluster_size);
                 if (cluster_data == NULL) {
@@ -1343,17 +1614,21 @@
                    QCOW_MAX_CRYPT_CLUSTERS * s->cluster_size);
             qemu_iovec_to_buf(&hd_qiov, 0, cluster_data, hd_qiov.size);
 
-            qcow2_encrypt_sectors(s, sector_num, cluster_data,
-                cluster_data, cur_nr_sectors, 1, &s->aes_encrypt_key);
+            if (qcow2_encrypt_sectors(s, offset >> BDRV_SECTOR_BITS,
+                                      cluster_data, cluster_data,
+                                      cur_bytes >>BDRV_SECTOR_BITS,
+                                      true, &err) < 0) {
+                error_free(err);
+                ret = -EIO;
+                goto fail;
+            }
 
             qemu_iovec_reset(&hd_qiov);
-            qemu_iovec_add(&hd_qiov, cluster_data,
-                cur_nr_sectors * 512);
+            qemu_iovec_add(&hd_qiov, cluster_data, cur_bytes);
         }
 
         ret = qcow2_pre_write_overlap_check(bs, 0,
-                cluster_offset + index_in_cluster * BDRV_SECTOR_SIZE,
-                cur_nr_sectors * BDRV_SECTOR_SIZE);
+                cluster_offset + offset_in_cluster, cur_bytes);
         if (ret < 0) {
             goto fail;
         }
@@ -1361,10 +1636,10 @@
         qemu_co_mutex_unlock(&s->lock);
         BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
         trace_qcow2_writev_data(qemu_coroutine_self(),
-                                (cluster_offset >> 9) + index_in_cluster);
-        ret = bdrv_co_writev(bs->file,
-                             (cluster_offset >> 9) + index_in_cluster,
-                             cur_nr_sectors, &hd_qiov);
+                                cluster_offset + offset_in_cluster);
+        ret = bdrv_co_pwritev(bs->file,
+                              cluster_offset + offset_in_cluster,
+                              cur_bytes, &hd_qiov, 0);
         qemu_co_mutex_lock(&s->lock);
         if (ret < 0) {
             goto fail;
@@ -1390,10 +1665,10 @@
             l2meta = next;
         }
 
-        remaining_sectors -= cur_nr_sectors;
-        sector_num += cur_nr_sectors;
-        bytes_done += cur_nr_sectors * 512;
-        trace_qcow2_writev_done_part(qemu_coroutine_self(), cur_nr_sectors);
+        bytes -= cur_bytes;
+        offset += cur_bytes;
+        bytes_done += cur_bytes;
+        trace_qcow2_writev_done_part(qemu_coroutine_self(), cur_bytes);
     }
     ret = 0;
 
@@ -1420,26 +1695,56 @@
     return ret;
 }
 
+static int qcow2_inactivate(BlockDriverState *bs)
+{
+    BDRVQcow2State *s = bs->opaque;
+    int ret, result = 0;
+
+    ret = qcow2_cache_flush(bs, s->l2_table_cache);
+    if (ret) {
+        result = ret;
+        error_report("Failed to flush the L2 table cache: %s",
+                     strerror(-ret));
+    }
+
+    ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+    if (ret) {
+        result = ret;
+        error_report("Failed to flush the refcount block cache: %s",
+                     strerror(-ret));
+    }
+
+    if (result == 0) {
+        qcow2_mark_clean(bs);
+    }
+
+    return result;
+}
+
 static void qcow2_close(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     qemu_vfree(s->l1_table);
     /* else pre-write overlap checks in cache_destroy may crash */
     s->l1_table = NULL;
 
-    if (!(bs->open_flags & BDRV_O_INCOMING)) {
-        qcow2_cache_flush(bs, s->l2_table_cache);
-        qcow2_cache_flush(bs, s->refcount_block_cache);
-
-        qcow2_mark_clean(bs);
+    if (!(s->flags & BDRV_O_INACTIVE)) {
+        qcow2_inactivate(bs);
     }
 
+    cache_clean_timer_del(bs);
     qcow2_cache_destroy(bs, s->l2_table_cache);
     qcow2_cache_destroy(bs, s->refcount_block_cache);
 
+    qcrypto_cipher_free(s->cipher);
+    s->cipher = NULL;
+
     g_free(s->unknown_header_fields);
     cleanup_unknown_header_ext(bs);
 
+    g_free(s->image_backing_file);
+    g_free(s->image_backing_format);
+
     g_free(s->cluster_cache);
     qemu_vfree(s->cluster_data);
     qcow2_refcount_close(bs);
@@ -1448,11 +1753,9 @@
 
 static void qcow2_invalidate_cache(BlockDriverState *bs, Error **errp)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int flags = s->flags;
-    AES_KEY aes_encrypt_key;
-    AES_KEY aes_decrypt_key;
-    uint32_t crypt_method = 0;
+    QCryptoCipher *cipher = NULL;
     QDict *options;
     Error *local_err = NULL;
     int ret;
@@ -1462,40 +1765,29 @@
      * that means we don't have to worry about reopening them here.
      */
 
-    if (s->crypt_method) {
-        crypt_method = s->crypt_method;
-        memcpy(&aes_encrypt_key, &s->aes_encrypt_key, sizeof(aes_encrypt_key));
-        memcpy(&aes_decrypt_key, &s->aes_decrypt_key, sizeof(aes_decrypt_key));
-    }
+    cipher = s->cipher;
+    s->cipher = NULL;
 
     qcow2_close(bs);
 
-    bdrv_invalidate_cache(bs->file, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    memset(s, 0, sizeof(BDRVQcowState));
+    memset(s, 0, sizeof(BDRVQcow2State));
     options = qdict_clone_shallow(bs->options);
 
+    flags &= ~BDRV_O_INACTIVE;
     ret = qcow2_open(bs, options, flags, &local_err);
     QDECREF(options);
     if (local_err) {
-        error_setg(errp, "Could not reopen qcow2 layer: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+        error_propagate(errp, local_err);
+        error_prepend(errp, "Could not reopen qcow2 layer: ");
+        bs->drv = NULL;
         return;
     } else if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not reopen qcow2 layer");
+        bs->drv = NULL;
         return;
     }
 
-    if (crypt_method) {
-        s->crypt_method = crypt_method;
-        memcpy(&s->aes_encrypt_key, &aes_encrypt_key, sizeof(aes_encrypt_key));
-        memcpy(&s->aes_decrypt_key, &aes_decrypt_key, sizeof(aes_decrypt_key));
-    }
+    s->cipher = cipher;
 }
 
 static size_t header_ext_add(char *buf, uint32_t magic, const void *s,
@@ -1527,7 +1819,7 @@
  */
 int qcow2_update_header(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     QCowHeader *header;
     char *buf;
     size_t buflen = s->cluster_size;
@@ -1605,9 +1897,10 @@
     }
 
     /* Backing file format header extension */
-    if (*bs->backing_format) {
+    if (s->image_backing_format) {
         ret = header_ext_add(buf, QCOW2_EXT_MAGIC_BACKING_FORMAT,
-                             bs->backing_format, strlen(bs->backing_format),
+                             s->image_backing_format,
+                             strlen(s->image_backing_format),
                              buflen);
         if (ret < 0) {
             goto fail;
@@ -1618,31 +1911,33 @@
     }
 
     /* Feature table */
-    Qcow2Feature features[] = {
-        {
-            .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
-            .bit  = QCOW2_INCOMPAT_DIRTY_BITNR,
-            .name = "dirty bit",
-        },
-        {
-            .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
-            .bit  = QCOW2_INCOMPAT_CORRUPT_BITNR,
-            .name = "corrupt bit",
-        },
-        {
-            .type = QCOW2_FEAT_TYPE_COMPATIBLE,
-            .bit  = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
-            .name = "lazy refcounts",
-        },
-    };
+    if (s->qcow_version >= 3) {
+        Qcow2Feature features[] = {
+            {
+                .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
+                .bit  = QCOW2_INCOMPAT_DIRTY_BITNR,
+                .name = "dirty bit",
+            },
+            {
+                .type = QCOW2_FEAT_TYPE_INCOMPATIBLE,
+                .bit  = QCOW2_INCOMPAT_CORRUPT_BITNR,
+                .name = "corrupt bit",
+            },
+            {
+                .type = QCOW2_FEAT_TYPE_COMPATIBLE,
+                .bit  = QCOW2_COMPAT_LAZY_REFCOUNTS_BITNR,
+                .name = "lazy refcounts",
+            },
+        };
 
-    ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE,
-                         features, sizeof(features), buflen);
-    if (ret < 0) {
-        goto fail;
+        ret = header_ext_add(buf, QCOW2_EXT_MAGIC_FEATURE_TABLE,
+                             features, sizeof(features), buflen);
+        if (ret < 0) {
+            goto fail;
+        }
+        buf += ret;
+        buflen -= ret;
     }
-    buf += ret;
-    buflen -= ret;
 
     /* Keep unknown header extensions */
     QLIST_FOREACH(uext, &s->unknown_header_ext, next) {
@@ -1665,8 +1960,8 @@
     buflen -= ret;
 
     /* Backing file name */
-    if (*bs->backing_file) {
-        size_t backing_file_len = strlen(bs->backing_file);
+    if (s->image_backing_file) {
+        size_t backing_file_len = strlen(s->image_backing_file);
 
         if (buflen < backing_file_len) {
             ret = -ENOSPC;
@@ -1674,7 +1969,7 @@
         }
 
         /* Using strncpy is ok here, since buf is not NUL-terminated. */
-        strncpy(buf, bs->backing_file, buflen);
+        strncpy(buf, s->image_backing_file, buflen);
 
         header->backing_file_offset = cpu_to_be64(buf - ((char*) header));
         header->backing_file_size   = cpu_to_be32(backing_file_len);
@@ -1695,27 +1990,39 @@
 static int qcow2_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt)
 {
+    BDRVQcow2State *s = bs->opaque;
+
+    if (backing_file && strlen(backing_file) > 1023) {
+        return -EINVAL;
+    }
+
     pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_file ?: "");
     pstrcpy(bs->backing_format, sizeof(bs->backing_format), backing_fmt ?: "");
 
+    g_free(s->image_backing_file);
+    g_free(s->image_backing_format);
+
+    s->image_backing_file = backing_file ? g_strdup(bs->backing_file) : NULL;
+    s->image_backing_format = backing_fmt ? g_strdup(bs->backing_format) : NULL;
+
     return qcow2_update_header(bs);
 }
 
 static int preallocate(BlockDriverState *bs)
 {
-    uint64_t nb_sectors;
+    uint64_t bytes;
     uint64_t offset;
     uint64_t host_offset = 0;
-    int num;
+    unsigned int cur_bytes;
     int ret;
     QCowL2Meta *meta;
 
-    nb_sectors = bdrv_nb_sectors(bs);
+    bytes = bdrv_getlength(bs);
     offset = 0;
 
-    while (nb_sectors) {
-        num = MIN(nb_sectors, INT_MAX >> BDRV_SECTOR_BITS);
-        ret = qcow2_alloc_cluster_offset(bs, offset, &num,
+    while (bytes) {
+        cur_bytes = MIN(bytes, INT_MAX);
+        ret = qcow2_alloc_cluster_offset(bs, offset, &cur_bytes,
                                          &host_offset, &meta);
         if (ret < 0) {
             return ret;
@@ -1741,8 +2048,8 @@
 
         /* TODO Preallocate data if requested */
 
-        nb_sectors -= num;
-        offset += num << BDRV_SECTOR_BITS;
+        bytes -= cur_bytes;
+        offset += cur_bytes;
     }
 
     /*
@@ -1751,10 +2058,9 @@
      * EOF). Extend the image to the last allocated sector.
      */
     if (host_offset != 0) {
-        uint8_t buf[BDRV_SECTOR_SIZE];
-        memset(buf, 0, BDRV_SECTOR_SIZE);
-        ret = bdrv_write(bs->file, (host_offset >> BDRV_SECTOR_BITS) + num - 1,
-                         buf, 1);
+        uint8_t data = 0;
+        ret = bdrv_pwrite(bs->file, (host_offset + cur_bytes) - 1,
+                          &data, 1);
         if (ret < 0) {
             return ret;
         }
@@ -1766,12 +2072,14 @@
 static int qcow2_create2(const char *filename, int64_t total_size,
                          const char *backing_file, const char *backing_format,
                          int flags, size_t cluster_size, PreallocMode prealloc,
-                         QemuOpts *opts, int version,
+                         QemuOpts *opts, int version, int refcount_order,
                          Error **errp)
 {
-    /* Calculate cluster_bits */
     int cluster_bits;
-    cluster_bits = ffs(cluster_size) - 1;
+    QDict *options;
+
+    /* Calculate cluster_bits */
+    cluster_bits = ctz32(cluster_size);
     if (cluster_bits < MIN_CLUSTER_BITS || cluster_bits > MAX_CLUSTER_BITS ||
         (1 << cluster_bits) != cluster_size)
     {
@@ -1792,16 +2100,28 @@
      * 2 GB for 64k clusters, and we don't want to have a 2 GB initial file
      * size for any qcow2 image.
      */
-    BlockDriverState* bs;
+    BlockBackend *blk;
     QCowHeader *header;
     uint64_t* refcount_table;
     Error *local_err = NULL;
     int ret;
 
     if (prealloc == PREALLOC_MODE_FULL || prealloc == PREALLOC_MODE_FALLOC) {
+        /* Note: The following calculation does not need to be exact; if it is a
+         * bit off, either some bytes will be "leaked" (which is fine) or we
+         * will need to increase the file size by some bytes (which is fine,
+         * too, as long as the bulk is allocated here). Therefore, using
+         * floating point arithmetic is fine. */
         int64_t meta_size = 0;
         uint64_t nreftablee, nrefblocke, nl1e, nl2e;
         int64_t aligned_total_size = align_offset(total_size, cluster_size);
+        int refblock_bits, refblock_size;
+        /* refcount entry size in bytes */
+        double rces = (1 << refcount_order) / 8.;
+
+        /* see qcow2_open() */
+        refblock_bits = cluster_bits - (refcount_order - 3);
+        refblock_size = 1 << refblock_bits;
 
         /* header: 1 cluster */
         meta_size += cluster_size;
@@ -1826,26 +2146,27 @@
          *   c = cluster size
          *   y1 = number of refcount blocks entries
          *   y2 = meta size including everything
+         *   rces = refcount entry size in bytes
          * then,
          *   y1 = (y2 + a)/c
-         *   y2 = y1 * sizeof(u16) + y1 * sizeof(u16) * sizeof(u64) / c + m
+         *   y2 = y1 * rces + y1 * rces * sizeof(u64) / c + m
          * we can get y1:
-         *   y1 = (a + m) / (c - sizeof(u16) - sizeof(u16) * sizeof(u64) / c)
+         *   y1 = (a + m) / (c - rces - rces * sizeof(u64) / c)
          */
-        nrefblocke = (aligned_total_size + meta_size + cluster_size) /
-            (cluster_size - sizeof(uint16_t) -
-             1.0 * sizeof(uint16_t) * sizeof(uint64_t) / cluster_size);
-        nrefblocke = align_offset(nrefblocke, cluster_size / sizeof(uint16_t));
-        meta_size += nrefblocke * sizeof(uint16_t);
+        nrefblocke = (aligned_total_size + meta_size + cluster_size)
+                   / (cluster_size - rces - rces * sizeof(uint64_t)
+                                                 / cluster_size);
+        meta_size += DIV_ROUND_UP(nrefblocke, refblock_size) * cluster_size;
 
         /* total size of refcount tables */
-        nreftablee = nrefblocke * sizeof(uint16_t) / cluster_size;
+        nreftablee = nrefblocke / refblock_size;
         nreftablee = align_offset(nreftablee, cluster_size / sizeof(uint64_t));
         meta_size += nreftablee * sizeof(uint64_t);
 
         qemu_opt_set_number(opts, BLOCK_OPT_SIZE,
-                            aligned_total_size + meta_size);
-        qemu_opt_set(opts, BLOCK_OPT_PREALLOC, PreallocMode_lookup[prealloc]);
+                            aligned_total_size + meta_size, &error_abort);
+        qemu_opt_set(opts, BLOCK_OPT_PREALLOC, PreallocMode_lookup[prealloc],
+                     &error_abort);
     }
 
     ret = bdrv_create_file(filename, opts, &local_err);
@@ -1854,14 +2175,15 @@
         return ret;
     }
 
-    bs = NULL;
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
-        return ret;
+        return -EIO;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* Write the header */
     QEMU_BUILD_BUG_ON((1 << MIN_CLUSTER_BITS) < sizeof(*header));
     header = g_malloc0(cluster_size);
@@ -1874,7 +2196,7 @@
         .l1_size                    = cpu_to_be32(0),
         .refcount_table_offset      = cpu_to_be64(cluster_size),
         .refcount_table_clusters    = cpu_to_be32(1),
-        .refcount_order             = cpu_to_be32(4),
+        .refcount_order             = cpu_to_be32(refcount_order),
         .header_length              = cpu_to_be32(sizeof(*header)),
     };
 
@@ -1889,7 +2211,7 @@
             cpu_to_be64(QCOW2_COMPAT_LAZY_REFCOUNTS);
     }
 
-    ret = bdrv_pwrite(bs, 0, header, cluster_size);
+    ret = blk_pwrite(blk, 0, header, cluster_size, 0);
     g_free(header);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not write qcow2 header");
@@ -1899,7 +2221,7 @@
     /* Write a refcount table with one refcount block */
     refcount_table = g_malloc0(2 * cluster_size);
     refcount_table[0] = cpu_to_be64(2 * cluster_size);
-    ret = bdrv_pwrite(bs, cluster_size, refcount_table, 2 * cluster_size);
+    ret = blk_pwrite(blk, cluster_size, refcount_table, 2 * cluster_size, 0);
     g_free(refcount_table);
 
     if (ret < 0) {
@@ -1907,24 +2229,25 @@
         goto out;
     }
 
-    bdrv_unref(bs);
-    bs = NULL;
+    blk_unref(blk);
+    blk = NULL;
 
     /*
      * And now open the image and make it consistent first (i.e. increase the
      * refcount of the cluster that is occupied by the header and the refcount
      * table)
      */
-    BlockDriver* drv = bdrv_find_format("qcow2");
-    assert(drv != NULL);
-    ret = bdrv_open(&bs, filename, NULL, NULL,
-        BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, drv, &local_err);
-    if (ret < 0) {
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow2"));
+    blk = blk_new_open(filename, NULL, options,
+                       BDRV_O_RDWR | BDRV_O_NO_FLUSH, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto out;
     }
 
-    ret = qcow2_alloc_clusters(bs, 3 * cluster_size);
+    ret = qcow2_alloc_clusters(blk_bs(blk), 3 * cluster_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not allocate clusters for qcow2 "
                          "header and refcount table");
@@ -1935,8 +2258,15 @@
         abort();
     }
 
+    /* Create a full header (including things like feature table) */
+    ret = qcow2_update_header(blk_bs(blk));
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not update qcow2 header");
+        goto out;
+    }
+
     /* Okay, now that we have a valid image, let's give it the right size */
-    ret = bdrv_truncate(bs, total_size);
+    ret = blk_truncate(blk, total_size);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not resize image");
         goto out;
@@ -1944,7 +2274,7 @@
 
     /* Want a backing file? There you go.*/
     if (backing_file) {
-        ret = bdrv_change_backing_file(bs, backing_file, backing_format);
+        ret = bdrv_change_backing_file(blk_bs(blk), backing_file, backing_format);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not assign backing file '%s' "
                              "with format '%s'", backing_file, backing_format);
@@ -1954,9 +2284,9 @@
 
     /* And if we're supposed to preallocate metadata, do that now */
     if (prealloc != PREALLOC_MODE_OFF) {
-        BDRVQcowState *s = bs->opaque;
+        BDRVQcow2State *s = blk_bs(blk)->opaque;
         qemu_co_mutex_lock(&s->lock);
-        ret = preallocate(bs);
+        ret = preallocate(blk_bs(blk));
         qemu_co_mutex_unlock(&s->lock);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not preallocate metadata");
@@ -1964,22 +2294,24 @@
         }
     }
 
-    bdrv_unref(bs);
-    bs = NULL;
+    blk_unref(blk);
+    blk = NULL;
 
     /* Reopen the image without BDRV_O_NO_FLUSH to flush it before returning */
-    ret = bdrv_open(&bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_BACKING,
-                    drv, &local_err);
-    if (local_err) {
+    options = qdict_new();
+    qdict_put(options, "driver", qstring_from_str("qcow2"));
+    blk = blk_new_open(filename, NULL, options,
+                       BDRV_O_RDWR | BDRV_O_NO_BACKING, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto out;
     }
 
     ret = 0;
 out:
-    if (bs) {
-        bdrv_unref(bs);
+    if (blk) {
+        blk_unref(blk);
     }
     return ret;
 }
@@ -1994,6 +2326,8 @@
     size_t cluster_size = DEFAULT_CLUSTER_SIZE;
     PreallocMode prealloc;
     int version = 3;
+    uint64_t refcount_bits = 16;
+    int refcount_order;
     Error *local_err = NULL;
     int ret;
 
@@ -2009,7 +2343,7 @@
                                          DEFAULT_CLUSTER_SIZE);
     buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     prealloc = qapi_enum_parse(PreallocMode_lookup, buf,
-                               PREALLOC_MODE_MAX, PREALLOC_MODE_OFF,
+                               PREALLOC_MODE__MAX, PREALLOC_MODE_OFF,
                                &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -2048,12 +2382,30 @@
         goto finish;
     }
 
-    ret = qcow2_create2(filename, size, backing_file, backing_fmt, flags,
-                        cluster_size, prealloc, opts, version, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    refcount_bits = qemu_opt_get_number_del(opts, BLOCK_OPT_REFCOUNT_BITS,
+                                            refcount_bits);
+    if (refcount_bits > 64 || !is_power_of_2(refcount_bits)) {
+        error_setg(errp, "Refcount width must be a power of two and may not "
+                   "exceed 64 bits");
+        ret = -EINVAL;
+        goto finish;
     }
 
+    if (version < 3 && refcount_bits != 16) {
+        error_setg(errp, "Different refcount widths than 16 bits require "
+                   "compatibility level 1.1 or above (use compat=1.1 or "
+                   "greater)");
+        ret = -EINVAL;
+        goto finish;
+    }
+
+    refcount_order = ctz32(refcount_bits);
+
+    ret = qcow2_create2(filename, size, backing_file, backing_fmt, flags,
+                        cluster_size, prealloc, opts, version, refcount_order,
+                        &local_err);
+    error_propagate(errp, local_err);
+
 finish:
     g_free(backing_file);
     g_free(backing_fmt);
@@ -2061,42 +2413,88 @@
     return ret;
 }
 
-static coroutine_fn int qcow2_co_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
+
+static bool is_zero_sectors(BlockDriverState *bs, int64_t start,
+                            uint32_t count)
+{
+    int nr;
+    BlockDriverState *file;
+    int64_t res;
+
+    if (!count) {
+        return true;
+    }
+    res = bdrv_get_block_status_above(bs, NULL, start, count,
+                                      &nr, &file);
+    return res >= 0 && (res & BDRV_BLOCK_ZERO) && nr == count;
+}
+
+static coroutine_fn int qcow2_co_pwrite_zeroes(BlockDriverState *bs,
+    int64_t offset, int count, BdrvRequestFlags flags)
 {
     int ret;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
-    /* Emulate misaligned zero writes */
-    if (sector_num % s->cluster_sectors || nb_sectors % s->cluster_sectors) {
-        return -ENOTSUP;
+    uint32_t head = offset % s->cluster_size;
+    uint32_t tail = (offset + count) % s->cluster_size;
+
+    trace_qcow2_pwrite_zeroes_start_req(qemu_coroutine_self(), offset, count);
+
+    if (head || tail) {
+        int64_t cl_start = (offset - head) >> BDRV_SECTOR_BITS;
+        uint64_t off;
+        unsigned int nr;
+
+        assert(head + count <= s->cluster_size);
+
+        /* check whether remainder of cluster already reads as zero */
+        if (!(is_zero_sectors(bs, cl_start,
+                              DIV_ROUND_UP(head, BDRV_SECTOR_SIZE)) &&
+              is_zero_sectors(bs, (offset + count) >> BDRV_SECTOR_BITS,
+                              DIV_ROUND_UP(-tail & (s->cluster_size - 1),
+                                           BDRV_SECTOR_SIZE)))) {
+            return -ENOTSUP;
+        }
+
+        qemu_co_mutex_lock(&s->lock);
+        /* We can have new write after previous check */
+        offset = cl_start << BDRV_SECTOR_BITS;
+        count = s->cluster_size;
+        nr = s->cluster_size;
+        ret = qcow2_get_cluster_offset(bs, offset, &nr, &off);
+        if (ret != QCOW2_CLUSTER_UNALLOCATED && ret != QCOW2_CLUSTER_ZERO) {
+            qemu_co_mutex_unlock(&s->lock);
+            return -ENOTSUP;
+        }
+    } else {
+        qemu_co_mutex_lock(&s->lock);
     }
 
+    trace_qcow2_pwrite_zeroes(qemu_coroutine_self(), offset, count);
+
     /* Whatever is left can use real zero clusters */
-    qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_zero_clusters(bs, sector_num << BDRV_SECTOR_BITS,
-        nb_sectors);
+    ret = qcow2_zero_clusters(bs, offset, count >> BDRV_SECTOR_BITS);
     qemu_co_mutex_unlock(&s->lock);
 
     return ret;
 }
 
-static coroutine_fn int qcow2_co_discard(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors)
+static coroutine_fn int qcow2_co_pdiscard(BlockDriverState *bs,
+                                          int64_t offset, int count)
 {
     int ret;
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
 
     qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_discard_clusters(bs, sector_num << BDRV_SECTOR_BITS,
-        nb_sectors, QCOW2_DISCARD_REQUEST, false);
+    ret = qcow2_discard_clusters(bs, offset, count >> BDRV_SECTOR_BITS,
+                                 QCOW2_DISCARD_REQUEST, false);
     qemu_co_mutex_unlock(&s->lock);
     return ret;
 }
 
 static int qcow2_truncate(BlockDriverState *bs, int64_t offset)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t new_l1_size;
     int ret;
 
@@ -2135,12 +2533,57 @@
     return 0;
 }
 
+typedef struct Qcow2WriteCo {
+    BlockDriverState *bs;
+    int64_t sector_num;
+    const uint8_t *buf;
+    int nb_sectors;
+    int ret;
+} Qcow2WriteCo;
+
+static void qcow2_write_co_entry(void *opaque)
+{
+    Qcow2WriteCo *co = opaque;
+    QEMUIOVector qiov;
+    uint64_t offset = co->sector_num * BDRV_SECTOR_SIZE;
+    uint64_t bytes = co->nb_sectors * BDRV_SECTOR_SIZE;
+
+    struct iovec iov = (struct iovec) {
+        .iov_base   = (uint8_t*) co->buf,
+        .iov_len    = bytes,
+    };
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    co->ret = qcow2_co_pwritev(co->bs, offset, bytes, &qiov, 0);
+}
+
+/* Wrapper for non-coroutine contexts */
+static int qcow2_write(BlockDriverState *bs, int64_t sector_num,
+                       const uint8_t *buf, int nb_sectors)
+{
+    Coroutine *co;
+    AioContext *aio_context = bdrv_get_aio_context(bs);
+    Qcow2WriteCo data = {
+        .bs         = bs,
+        .sector_num = sector_num,
+        .buf        = buf,
+        .nb_sectors = nb_sectors,
+        .ret        = -EINPROGRESS,
+    };
+    co = qemu_coroutine_create(qcow2_write_co_entry, &data);
+    qemu_coroutine_enter(co);
+    while (data.ret == -EINPROGRESS) {
+        aio_poll(aio_context, true);
+    }
+    return data.ret;
+}
+
 /* XXX: put compressed sectors first, then all the cluster aligned
    tables to avoid losing bytes in alignment */
 static int qcow2_write_compressed(BlockDriverState *bs, int64_t sector_num,
                                   const uint8_t *buf, int nb_sectors)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     z_stream strm;
     int ret, out_len;
     uint8_t *out_buf;
@@ -2149,9 +2592,8 @@
     if (nb_sectors == 0) {
         /* align end of file to a sector boundary to ease reading with
            sector based I/Os */
-        cluster_offset = bdrv_getlength(bs->file);
-        bdrv_truncate(bs->file, cluster_offset);
-        return 0;
+        cluster_offset = bdrv_getlength(bs->file->bs);
+        return bdrv_truncate(bs->file->bs, cluster_offset);
     }
 
     if (nb_sectors != s->cluster_sectors) {
@@ -2170,7 +2612,7 @@
         return ret;
     }
 
-    out_buf = g_malloc(s->cluster_size + (s->cluster_size / 1000) + 128);
+    out_buf = g_malloc(s->cluster_size);
 
     /* best compression, small window, no zlib header */
     memset(&strm, 0, sizeof(strm));
@@ -2199,7 +2641,7 @@
 
     if (ret != Z_STREAM_END || out_len >= s->cluster_size) {
         /* could not compress: write normal cluster */
-        ret = bdrv_write(bs, sector_num, buf, s->cluster_sectors);
+        ret = qcow2_write(bs, sector_num, buf, s->cluster_sectors);
         if (ret < 0) {
             goto fail;
         }
@@ -2232,7 +2674,7 @@
 
 static int make_completely_empty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret, l1_clusters;
     int64_t offset;
     uint64_t *new_reftable = NULL;
@@ -2267,8 +2709,8 @@
     /* After this call, neither the in-memory nor the on-disk refcount
      * information accurately describe the actual references */
 
-    ret = bdrv_write_zeroes(bs->file, s->l1_table_offset / BDRV_SECTOR_SIZE,
-                            l1_clusters * s->cluster_sectors, 0);
+    ret = bdrv_pwrite_zeroes(bs->file, s->l1_table_offset,
+                             l1_clusters * s->cluster_size, 0);
     if (ret < 0) {
         goto fail_broken_refcounts;
     }
@@ -2281,9 +2723,8 @@
      * overwrite parts of the existing refcount and L1 table, which is not
      * an issue because the dirty flag is set, complete data loss is in fact
      * desired and partial data loss is consequently fine as well */
-    ret = bdrv_write_zeroes(bs->file, s->cluster_size / BDRV_SECTOR_SIZE,
-                            (2 + l1_clusters) * s->cluster_size /
-                            BDRV_SECTOR_SIZE, 0);
+    ret = bdrv_pwrite_zeroes(bs->file, s->cluster_size,
+                             (2 + l1_clusters) * s->cluster_size, 0);
     /* This call (even if it failed overall) may have overwritten on-disk
      * refcount structures; in that case, the in-memory refcount information
      * will probably differ from the on-disk information which makes the BDS
@@ -2298,9 +2739,9 @@
     /* "Create" an empty reftable (one cluster) directly after the image
      * header and an empty L1 table three clusters after the image header;
      * the cluster between those two will be used as the first refblock */
-    cpu_to_be64w(&l1_ofs_rt_ofs_cls.l1_offset, 3 * s->cluster_size);
-    cpu_to_be64w(&l1_ofs_rt_ofs_cls.reftable_offset, s->cluster_size);
-    cpu_to_be32w(&l1_ofs_rt_ofs_cls.reftable_clusters, 1);
+    l1_ofs_rt_ofs_cls.l1_offset = cpu_to_be64(3 * s->cluster_size);
+    l1_ofs_rt_ofs_cls.reftable_offset = cpu_to_be64(s->cluster_size);
+    l1_ofs_rt_ofs_cls.reftable_clusters = cpu_to_be32(1);
     ret = bdrv_pwrite_sync(bs->file, offsetof(QCowHeader, l1_table_offset),
                            &l1_ofs_rt_ofs_cls, sizeof(l1_ofs_rt_ofs_cls));
     if (ret < 0) {
@@ -2357,7 +2798,7 @@
         goto fail;
     }
 
-    ret = bdrv_truncate(bs->file, (3 + l1_clusters) * s->cluster_size);
+    ret = bdrv_truncate(bs->file->bs, (3 + l1_clusters) * s->cluster_size);
     if (ret < 0) {
         goto fail;
     }
@@ -2380,7 +2821,7 @@
 
 static int qcow2_make_empty(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     uint64_t start_sector;
     int sector_step = INT_MAX / BDRV_SECTOR_SIZE;
     int l1_clusters, ret = 0;
@@ -2421,18 +2862,18 @@
 
 static coroutine_fn int qcow2_co_flush_to_os(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int ret;
 
     qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_cache_flush(bs, s->l2_table_cache);
+    ret = qcow2_cache_write(bs, s->l2_table_cache);
     if (ret < 0) {
         qemu_co_mutex_unlock(&s->lock);
         return ret;
     }
 
     if (qcow2_need_accurate_refcounts(s)) {
-        ret = qcow2_cache_flush(bs, s->refcount_block_cache);
+        ret = qcow2_cache_write(bs, s->refcount_block_cache);
         if (ret < 0) {
             qemu_co_mutex_unlock(&s->lock);
             return ret;
@@ -2445,7 +2886,7 @@
 
 static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     bdi->unallocated_blocks_are_zero = true;
     bdi->can_write_zeroes_with_unmap = (s->qcow_version >= 3);
     bdi->cluster_size = s->cluster_size;
@@ -2455,21 +2896,20 @@
 
 static ImageInfoSpecific *qcow2_get_specific_info(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     ImageInfoSpecific *spec_info = g_new(ImageInfoSpecific, 1);
 
     *spec_info = (ImageInfoSpecific){
-        .kind  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
-        {
-            .qcow2 = g_new(ImageInfoSpecificQCow2, 1),
-        },
+        .type  = IMAGE_INFO_SPECIFIC_KIND_QCOW2,
+        .u.qcow2.data = g_new(ImageInfoSpecificQCow2, 1),
     };
     if (s->qcow_version == 2) {
-        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
-            .compat = g_strdup("0.10"),
+        *spec_info->u.qcow2.data = (ImageInfoSpecificQCow2){
+            .compat             = g_strdup("0.10"),
+            .refcount_bits      = s->refcount_bits,
         };
     } else if (s->qcow_version == 3) {
-        *spec_info->qcow2 = (ImageInfoSpecificQCow2){
+        *spec_info->u.qcow2.data = (ImageInfoSpecificQCow2){
             .compat             = g_strdup("1.1"),
             .lazy_refcounts     = s->compatible_features &
                                   QCOW2_COMPAT_LAZY_REFCOUNTS,
@@ -2477,7 +2917,12 @@
             .corrupt            = s->incompatible_features &
                                   QCOW2_INCOMPAT_CORRUPT,
             .has_corrupt        = true,
+            .refcount_bits      = s->refcount_bits,
         };
+    } else {
+        /* if this assertion fails, this probably means a new version was
+         * added without having it covered here */
+        assert(false);
     }
 
     return spec_info;
@@ -2486,11 +2931,11 @@
 #if 0
 static void dump_refcounts(BlockDriverState *bs)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int64_t nb_clusters, k, k1, size;
     int refcount;
 
-    size = bdrv_getlength(bs->file);
+    size = bdrv_getlength(bs->file->bs);
     nb_clusters = size_to_clusters(s, size);
     for(k = 0; k < nb_clusters;) {
         k1 = k;
@@ -2507,43 +2952,21 @@
 static int qcow2_save_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
                               int64_t pos)
 {
-    BDRVQcowState *s = bs->opaque;
-    int64_t total_sectors = bs->total_sectors;
-    int growable = bs->growable;
-    bool zero_beyond_eof = bs->zero_beyond_eof;
-    int ret;
+    BDRVQcow2State *s = bs->opaque;
 
     BLKDBG_EVENT(bs->file, BLKDBG_VMSTATE_SAVE);
-    bs->growable = 1;
-    bs->zero_beyond_eof = false;
-    ret = bdrv_pwritev(bs, qcow2_vm_state_offset(s) + pos, qiov);
-    bs->growable = growable;
-    bs->zero_beyond_eof = zero_beyond_eof;
-
-    /* bdrv_co_do_writev will have increased the total_sectors value to include
-     * the VM state - the VM state is however not an actual part of the block
-     * device, therefore, we need to restore the old value. */
-    bs->total_sectors = total_sectors;
-
-    return ret;
+    return bs->drv->bdrv_co_pwritev(bs, qcow2_vm_state_offset(s) + pos,
+                                    qiov->size, qiov, 0);
 }
 
-static int qcow2_load_vmstate(BlockDriverState *bs, uint8_t *buf,
-                              int64_t pos, int size)
+static int qcow2_load_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
+                              int64_t pos)
 {
-    BDRVQcowState *s = bs->opaque;
-    int growable = bs->growable;
-    bool zero_beyond_eof = bs->zero_beyond_eof;
-    int ret;
+    BDRVQcow2State *s = bs->opaque;
 
     BLKDBG_EVENT(bs->file, BLKDBG_VMSTATE_LOAD);
-    bs->growable = 1;
-    bs->zero_beyond_eof = false;
-    ret = bdrv_pread(bs, qcow2_vm_state_offset(s) + pos, buf, size);
-    bs->growable = growable;
-    bs->zero_beyond_eof = zero_beyond_eof;
-
-    return ret;
+    return bs->drv->bdrv_co_preadv(bs, qcow2_vm_state_offset(s) + pos,
+                                   qiov->size, qiov, 0);
 }
 
 /*
@@ -2551,9 +2974,9 @@
  * have to be removed.
  */
 static int qcow2_downgrade(BlockDriverState *bs, int target_version,
-                           BlockDriverAmendStatusCB *status_cb)
+                           BlockDriverAmendStatusCB *status_cb, void *cb_opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
     int current_version = s->qcow_version;
     int ret;
 
@@ -2566,13 +2989,7 @@
     }
 
     if (s->refcount_order != 4) {
-        /* we would have to convert the image to a refcount_order == 4 image
-         * here; however, since qemu (at the time of writing this) does not
-         * support anything different than 4 anyway, there is no point in doing
-         * so right now; however, we should error out (if qemu supports this in
-         * the future and this code has not been adapted) */
-        error_report("qcow2_downgrade: Image refcount orders other than 4 are "
-                     "currently not supported.");
+        error_report("compat=0.10 requires refcount_bits=16");
         return -ENOTSUP;
     }
 
@@ -2600,7 +3017,7 @@
     /* clearing autoclear features is trivial */
     s->autoclear_features = 0;
 
-    ret = qcow2_expand_zero_clusters(bs, status_cb);
+    ret = qcow2_expand_zero_clusters(bs, status_cb, cb_opaque);
     if (ret < 0) {
         return ret;
     }
@@ -2614,10 +3031,81 @@
     return 0;
 }
 
-static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
-                               BlockDriverAmendStatusCB *status_cb)
+typedef enum Qcow2AmendOperation {
+    /* This is the value Qcow2AmendHelperCBInfo::last_operation will be
+     * statically initialized to so that the helper CB can discern the first
+     * invocation from an operation change */
+    QCOW2_NO_OPERATION = 0,
+
+    QCOW2_CHANGING_REFCOUNT_ORDER,
+    QCOW2_DOWNGRADING,
+} Qcow2AmendOperation;
+
+typedef struct Qcow2AmendHelperCBInfo {
+    /* The code coordinating the amend operations should only modify
+     * these four fields; the rest will be managed by the CB */
+    BlockDriverAmendStatusCB *original_status_cb;
+    void *original_cb_opaque;
+
+    Qcow2AmendOperation current_operation;
+
+    /* Total number of operations to perform (only set once) */
+    int total_operations;
+
+    /* The following fields are managed by the CB */
+
+    /* Number of operations completed */
+    int operations_completed;
+
+    /* Cumulative offset of all completed operations */
+    int64_t offset_completed;
+
+    Qcow2AmendOperation last_operation;
+    int64_t last_work_size;
+} Qcow2AmendHelperCBInfo;
+
+static void qcow2_amend_helper_cb(BlockDriverState *bs,
+                                  int64_t operation_offset,
+                                  int64_t operation_work_size, void *opaque)
 {
-    BDRVQcowState *s = bs->opaque;
+    Qcow2AmendHelperCBInfo *info = opaque;
+    int64_t current_work_size;
+    int64_t projected_work_size;
+
+    if (info->current_operation != info->last_operation) {
+        if (info->last_operation != QCOW2_NO_OPERATION) {
+            info->offset_completed += info->last_work_size;
+            info->operations_completed++;
+        }
+
+        info->last_operation = info->current_operation;
+    }
+
+    assert(info->total_operations > 0);
+    assert(info->operations_completed < info->total_operations);
+
+    info->last_work_size = operation_work_size;
+
+    current_work_size = info->offset_completed + operation_work_size;
+
+    /* current_work_size is the total work size for (operations_completed + 1)
+     * operations (which includes this one), so multiply it by the number of
+     * operations not covered and divide it by the number of operations
+     * covered to get a projection for the operations not covered */
+    projected_work_size = current_work_size * (info->total_operations -
+                                               info->operations_completed - 1)
+                                            / (info->operations_completed + 1);
+
+    info->original_status_cb(bs, info->offset_completed + operation_offset,
+                             current_work_size + projected_work_size,
+                             info->original_cb_opaque);
+}
+
+static int qcow2_amend_options(BlockDriverState *bs, QemuOpts *opts,
+                               BlockDriverAmendStatusCB *status_cb,
+                               void *cb_opaque)
+{
+    BDRVQcow2State *s = bs->opaque;
     int old_version = s->qcow_version, new_version = old_version;
     uint64_t new_size = 0;
     const char *backing_file = NULL, *backing_format = NULL;
@@ -2625,8 +3113,10 @@
     const char *compat = NULL;
     uint64_t cluster_size = s->cluster_size;
     bool encrypt;
+    int refcount_bits = s->refcount_bits;
     int ret;
     QemuOptDesc *desc = opts->list->desc;
+    Qcow2AmendHelperCBInfo helper_cb_info;
 
     while (desc && desc->name) {
         if (!qemu_opt_find(opts, desc->name)) {
@@ -2635,8 +3125,8 @@
             continue;
         }
 
-        if (!strcmp(desc->name, "compat")) {
-            compat = qemu_opt_get(opts, "compat");
+        if (!strcmp(desc->name, BLOCK_OPT_COMPAT_LEVEL)) {
+            compat = qemu_opt_get(opts, BLOCK_OPT_COMPAT_LEVEL);
             if (!compat) {
                 /* preserve default */
             } else if (!strcmp(compat, "0.10")) {
@@ -2644,65 +3134,98 @@
             } else if (!strcmp(compat, "1.1")) {
                 new_version = 3;
             } else {
-                fprintf(stderr, "Unknown compatibility level %s.\n", compat);
+                error_report("Unknown compatibility level %s", compat);
                 return -EINVAL;
             }
-        } else if (!strcmp(desc->name, "preallocation")) {
-            fprintf(stderr, "Cannot change preallocation mode.\n");
+        } else if (!strcmp(desc->name, BLOCK_OPT_PREALLOC)) {
+            error_report("Cannot change preallocation mode");
             return -ENOTSUP;
-        } else if (!strcmp(desc->name, "size")) {
-            new_size = qemu_opt_get_size(opts, "size", 0);
-        } else if (!strcmp(desc->name, "backing_file")) {
-            backing_file = qemu_opt_get(opts, "backing_file");
-        } else if (!strcmp(desc->name, "backing_fmt")) {
-            backing_format = qemu_opt_get(opts, "backing_fmt");
-        } else if (!strcmp(desc->name, "encryption")) {
-            encrypt = qemu_opt_get_bool(opts, "encryption", s->crypt_method);
-            if (encrypt != !!s->crypt_method) {
-                fprintf(stderr, "Changing the encryption flag is not "
-                        "supported.\n");
+        } else if (!strcmp(desc->name, BLOCK_OPT_SIZE)) {
+            new_size = qemu_opt_get_size(opts, BLOCK_OPT_SIZE, 0);
+        } else if (!strcmp(desc->name, BLOCK_OPT_BACKING_FILE)) {
+            backing_file = qemu_opt_get(opts, BLOCK_OPT_BACKING_FILE);
+        } else if (!strcmp(desc->name, BLOCK_OPT_BACKING_FMT)) {
+            backing_format = qemu_opt_get(opts, BLOCK_OPT_BACKING_FMT);
+        } else if (!strcmp(desc->name, BLOCK_OPT_ENCRYPT)) {
+            encrypt = qemu_opt_get_bool(opts, BLOCK_OPT_ENCRYPT,
+                                        !!s->cipher);
+
+            if (encrypt != !!s->cipher) {
+                error_report("Changing the encryption flag is not supported");
                 return -ENOTSUP;
             }
-        } else if (!strcmp(desc->name, "cluster_size")) {
-            cluster_size = qemu_opt_get_size(opts, "cluster_size",
+        } else if (!strcmp(desc->name, BLOCK_OPT_CLUSTER_SIZE)) {
+            cluster_size = qemu_opt_get_size(opts, BLOCK_OPT_CLUSTER_SIZE,
                                              cluster_size);
             if (cluster_size != s->cluster_size) {
-                fprintf(stderr, "Changing the cluster size is not "
-                        "supported.\n");
+                error_report("Changing the cluster size is not supported");
                 return -ENOTSUP;
             }
-        } else if (!strcmp(desc->name, "lazy_refcounts")) {
-            lazy_refcounts = qemu_opt_get_bool(opts, "lazy_refcounts",
+        } else if (!strcmp(desc->name, BLOCK_OPT_LAZY_REFCOUNTS)) {
+            lazy_refcounts = qemu_opt_get_bool(opts, BLOCK_OPT_LAZY_REFCOUNTS,
                                                lazy_refcounts);
+        } else if (!strcmp(desc->name, BLOCK_OPT_REFCOUNT_BITS)) {
+            refcount_bits = qemu_opt_get_number(opts, BLOCK_OPT_REFCOUNT_BITS,
+                                                refcount_bits);
+
+            if (refcount_bits <= 0 || refcount_bits > 64 ||
+                !is_power_of_2(refcount_bits))
+            {
+                error_report("Refcount width must be a power of two and may "
+                             "not exceed 64 bits");
+                return -EINVAL;
+            }
         } else {
-            /* if this assertion fails, this probably means a new option was
+            /* if this point is reached, this probably means a new option was
              * added without having it covered here */
-            assert(false);
+            abort();
         }
 
         desc++;
     }
 
-    if (new_version != old_version) {
-        if (new_version > old_version) {
-            /* Upgrade */
-            s->qcow_version = new_version;
-            ret = qcow2_update_header(bs);
-            if (ret < 0) {
-                s->qcow_version = old_version;
-                return ret;
-            }
-        } else {
-            ret = qcow2_downgrade(bs, new_version, status_cb);
-            if (ret < 0) {
-                return ret;
-            }
+    helper_cb_info = (Qcow2AmendHelperCBInfo){
+        .original_status_cb = status_cb,
+        .original_cb_opaque = cb_opaque,
+        .total_operations = (new_version < old_version)
+                          + (s->refcount_bits != refcount_bits)
+    };
+
+    /* Upgrade first (some features may require compat=1.1) */
+    if (new_version > old_version) {
+        s->qcow_version = new_version;
+        ret = qcow2_update_header(bs);
+        if (ret < 0) {
+            s->qcow_version = old_version;
+            return ret;
+        }
+    }
+
+    if (s->refcount_bits != refcount_bits) {
+        int refcount_order = ctz32(refcount_bits);
+        Error *local_error = NULL;
+
+        if (new_version < 3 && refcount_bits != 16) {
+            error_report("Different refcount widths than 16 bits require "
+                         "compatibility level 1.1 or above (use compat=1.1 or "
+                         "greater)");
+            return -EINVAL;
+        }
+
+        helper_cb_info.current_operation = QCOW2_CHANGING_REFCOUNT_ORDER;
+        ret = qcow2_change_refcount_order(bs, refcount_order,
+                                          &qcow2_amend_helper_cb,
+                                          &helper_cb_info, &local_error);
+        if (ret < 0) {
+            error_report_err(local_error);
+            return ret;
         }
     }
 
     if (backing_file || backing_format) {
-        ret = qcow2_change_backing_file(bs, backing_file ?: bs->backing_file,
-                                        backing_format ?: bs->backing_format);
+        ret = qcow2_change_backing_file(bs,
+                    backing_file ?: s->image_backing_file,
+                    backing_format ?: s->image_backing_format);
         if (ret < 0) {
             return ret;
         }
@@ -2710,9 +3233,9 @@
 
     if (s->use_lazy_refcounts != lazy_refcounts) {
         if (lazy_refcounts) {
-            if (s->qcow_version < 3) {
-                fprintf(stderr, "Lazy refcounts only supported with compatibility "
-                        "level 1.1 and above (use compat=1.1 or greater)\n");
+            if (new_version < 3) {
+                error_report("Lazy refcounts only supported with compatibility "
+                             "level 1.1 and above (use compat=1.1 or greater)");
                 return -EINVAL;
             }
             s->compatible_features |= QCOW2_COMPAT_LAZY_REFCOUNTS;
@@ -2746,6 +3269,16 @@
         }
     }
 
+    /* Downgrade last (so unsupported features can be removed before) */
+    if (new_version < old_version) {
+        helper_cb_info.current_operation = QCOW2_DOWNGRADING;
+        ret = qcow2_downgrade(bs, new_version, &qcow2_amend_helper_cb,
+                              &helper_cb_info);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
     return 0;
 }
 
@@ -2758,7 +3291,8 @@
 void qcow2_signal_corruption(BlockDriverState *bs, bool fatal, int64_t offset,
                              int64_t size, const char *message_format, ...)
 {
-    BDRVQcowState *s = bs->opaque;
+    BDRVQcow2State *s = bs->opaque;
+    const char *node_name;
     char *message;
     va_list ap;
 
@@ -2782,8 +3316,11 @@
                 "corruption events will be suppressed\n", message);
     }
 
-    qapi_event_send_block_image_corrupted(bdrv_get_device_name(bs), message,
-                                          offset >= 0, offset, size >= 0, size,
+    node_name = bdrv_get_node_name(bs);
+    qapi_event_send_block_image_corrupted(bdrv_get_device_name(bs),
+                                          *node_name != '\0', node_name,
+                                          message, offset >= 0, offset,
+                                          size >= 0, size,
                                           fatal, &error_abort);
     g_free(message);
 
@@ -2843,28 +3380,37 @@
             .help = "Postpone refcount updates",
             .def_value_str = "off"
         },
+        {
+            .name = BLOCK_OPT_REFCOUNT_BITS,
+            .type = QEMU_OPT_NUMBER,
+            .help = "Width of a reference count entry in bits",
+            .def_value_str = "16"
+        },
         { /* end of list */ }
     }
 };
 
-static BlockDriver bdrv_qcow2 = {
+BlockDriver bdrv_qcow2 = {
     .format_name        = "qcow2",
-    .instance_size      = sizeof(BDRVQcowState),
+    .instance_size      = sizeof(BDRVQcow2State),
     .bdrv_probe         = qcow2_probe,
     .bdrv_open          = qcow2_open,
     .bdrv_close         = qcow2_close,
     .bdrv_reopen_prepare  = qcow2_reopen_prepare,
+    .bdrv_reopen_commit   = qcow2_reopen_commit,
+    .bdrv_reopen_abort    = qcow2_reopen_abort,
+    .bdrv_join_options    = qcow2_join_options,
     .bdrv_create        = qcow2_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
     .bdrv_co_get_block_status = qcow2_co_get_block_status,
     .bdrv_set_key       = qcow2_set_key,
 
-    .bdrv_co_readv          = qcow2_co_readv,
-    .bdrv_co_writev         = qcow2_co_writev,
+    .bdrv_co_preadv         = qcow2_co_preadv,
+    .bdrv_co_pwritev        = qcow2_co_pwritev,
     .bdrv_co_flush_to_os    = qcow2_co_flush_to_os,
 
-    .bdrv_co_write_zeroes   = qcow2_co_write_zeroes,
-    .bdrv_co_discard        = qcow2_co_discard,
+    .bdrv_co_pwrite_zeroes  = qcow2_co_pwrite_zeroes,
+    .bdrv_co_pdiscard       = qcow2_co_pdiscard,
     .bdrv_truncate          = qcow2_truncate,
     .bdrv_write_compressed  = qcow2_write_compressed,
     .bdrv_make_empty        = qcow2_make_empty,
@@ -2885,10 +3431,14 @@
 
     .bdrv_refresh_limits        = qcow2_refresh_limits,
     .bdrv_invalidate_cache      = qcow2_invalidate_cache,
+    .bdrv_inactivate            = qcow2_inactivate,
 
     .create_opts         = &qcow2_create_opts,
     .bdrv_check          = qcow2_check,
     .bdrv_amend_options  = qcow2_amend_options,
+
+    .bdrv_detach_aio_context  = qcow2_detach_aio_context,
+    .bdrv_attach_aio_context  = qcow2_attach_aio_context,
 };
 
 static void bdrv_qcow2_init(void)
diff --git a/block/qcow2.h b/block/qcow2.h
index 6e39a1b..b36a7bf 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -25,8 +25,8 @@
 #ifndef BLOCK_QCOW2_H
 #define BLOCK_QCOW2_H
 
-#include "qemu/aes.h"
-#include "block/coroutine.h"
+#include "crypto/cipher.h"
+#include "qemu/coroutine.h"
 
 //#define DEBUG_ALLOC
 //#define DEBUG_ALLOC2
@@ -62,11 +62,14 @@
 #define MIN_CLUSTER_BITS 9
 #define MAX_CLUSTER_BITS 21
 
-#define MIN_L2_CACHE_SIZE 1 /* cluster */
+/* Must be at least 2 to cover COW */
+#define MIN_L2_CACHE_SIZE 2 /* clusters */
 
 /* Must be at least 4 to cover all cases of refcount table growth */
 #define MIN_REFCOUNT_CACHE_SIZE 4 /* clusters */
 
+/* Whichever is more */
+#define DEFAULT_L2_CACHE_CLUSTERS 8 /* clusters */
 #define DEFAULT_L2_CACHE_BYTE_SIZE 1048576 /* bytes */
 
 /* The refblock cache needs only a fourth of the L2 cache size to cover as many
@@ -93,6 +96,7 @@
 #define QCOW2_OPT_CACHE_SIZE "cache-size"
 #define QCOW2_OPT_L2_CACHE_SIZE "l2-cache-size"
 #define QCOW2_OPT_REFCOUNT_CACHE_SIZE "refcount-cache-size"
+#define QCOW2_OPT_CACHE_CLEAN_INTERVAL "cache-clean-interval"
 
 typedef struct QCowHeader {
     uint32_t magic;
@@ -213,7 +217,12 @@
     QTAILQ_ENTRY(Qcow2DiscardRegion) next;
 } Qcow2DiscardRegion;
 
-typedef struct BDRVQcowState {
+typedef uint64_t Qcow2GetRefcountFunc(const void *refcount_array,
+                                      uint64_t index);
+typedef void Qcow2SetRefcountFunc(void *refcount_array,
+                                  uint64_t index, uint64_t value);
+
+typedef struct BDRVQcow2State {
     int cluster_bits;
     int cluster_size;
     int cluster_sectors;
@@ -231,6 +240,8 @@
 
     Qcow2Cache* l2_table_cache;
     Qcow2Cache* refcount_block_cache;
+    QEMUTimer *cache_clean_timer;
+    unsigned cache_clean_interval;
 
     uint8_t *cluster_cache;
     uint8_t *cluster_data;
@@ -245,10 +256,8 @@
 
     CoMutex lock;
 
-    uint32_t crypt_method; /* current crypt method, 0 if no key yet */
+    QCryptoCipher *cipher; /* current cipher, NULL if no key yet */
     uint32_t crypt_method_header;
-    AES_KEY aes_encrypt_key;
-    AES_KEY aes_decrypt_key;
     uint64_t snapshots_offset;
     int snapshots_size;
     unsigned int nb_snapshots;
@@ -258,6 +267,11 @@
     int qcow_version;
     bool use_lazy_refcounts;
     int refcount_order;
+    int refcount_bits;
+    uint64_t refcount_max;
+
+    Qcow2GetRefcountFunc *get_refcount;
+    Qcow2SetRefcountFunc *set_refcount;
 
     bool discard_passthrough[QCOW2_DISCARD_MAX];
 
@@ -273,20 +287,13 @@
     QLIST_HEAD(, Qcow2UnknownHeaderExtension) unknown_header_ext;
     QTAILQ_HEAD (, Qcow2DiscardRegion) discards;
     bool cache_discards;
-} BDRVQcowState;
 
-/* XXX: use std qcow open function ? */
-typedef struct QCowCreateState {
-    int cluster_size;
-    int cluster_bits;
-    uint16_t *refcount_block;
-    uint64_t *refcount_table;
-    int64_t l1_table_offset;
-    int64_t refcount_table_offset;
-    int64_t refcount_block_offset;
-} QCowCreateState;
-
-struct QCowAIOCB;
+    /* Backing file path and format as stored in the image (this is not the
+     * effective path/format, which may be the result of a runtime option
+     * override) */
+    char *image_backing_file;
+    char *image_backing_format;
+} BDRVQcow2State;
 
 typedef struct Qcow2COWRegion {
     /**
@@ -295,8 +302,8 @@
      */
     uint64_t    offset;
 
-    /** Number of sectors to copy */
-    int         nb_sectors;
+    /** Number of bytes to copy */
+    int         nb_bytes;
 } Qcow2COWRegion;
 
 /**
@@ -311,12 +318,6 @@
     /** Host offset of the first newly allocated cluster */
     uint64_t alloc_offset;
 
-    /**
-     * Number of sectors from the start of the first allocated cluster to
-     * the end of the (possibly shortened) request
-     */
-    int nb_available;
-
     /** Number of newly allocated clusters */
     int nb_clusters;
 
@@ -396,28 +397,28 @@
 
 #define REFT_OFFSET_MASK 0xfffffffffffffe00ULL
 
-static inline int64_t start_of_cluster(BDRVQcowState *s, int64_t offset)
+static inline int64_t start_of_cluster(BDRVQcow2State *s, int64_t offset)
 {
     return offset & ~(s->cluster_size - 1);
 }
 
-static inline int64_t offset_into_cluster(BDRVQcowState *s, int64_t offset)
+static inline int64_t offset_into_cluster(BDRVQcow2State *s, int64_t offset)
 {
     return offset & (s->cluster_size - 1);
 }
 
-static inline int size_to_clusters(BDRVQcowState *s, int64_t size)
+static inline uint64_t size_to_clusters(BDRVQcow2State *s, uint64_t size)
 {
     return (size + (s->cluster_size - 1)) >> s->cluster_bits;
 }
 
-static inline int64_t size_to_l1(BDRVQcowState *s, int64_t size)
+static inline int64_t size_to_l1(BDRVQcow2State *s, int64_t size)
 {
     int shift = s->cluster_bits + s->l2_bits;
     return (size + (1ULL << shift) - 1) >> shift;
 }
 
-static inline int offset_to_l2_index(BDRVQcowState *s, int64_t offset)
+static inline int offset_to_l2_index(BDRVQcow2State *s, int64_t offset)
 {
     return (offset >> s->cluster_bits) & (s->l2_size - 1);
 }
@@ -428,12 +429,12 @@
     return offset;
 }
 
-static inline int64_t qcow2_vm_state_offset(BDRVQcowState *s)
+static inline int64_t qcow2_vm_state_offset(BDRVQcow2State *s)
 {
     return (int64_t)s->l1_vm_state_index << (s->cluster_bits + s->l2_bits);
 }
 
-static inline uint64_t qcow2_max_refcount_clusters(BDRVQcowState *s)
+static inline uint64_t qcow2_max_refcount_clusters(BDRVQcow2State *s)
 {
     return QCOW_MAX_REFTABLE_SIZE >> s->cluster_bits;
 }
@@ -452,7 +453,7 @@
 }
 
 /* Check whether refcounts are eager or lazy */
-static inline bool qcow2_need_accurate_refcounts(BDRVQcowState *s)
+static inline bool qcow2_need_accurate_refcounts(BDRVQcow2State *s)
 {
     return !(s->incompatible_features & QCOW2_INCOMPAT_DIRTY);
 }
@@ -464,8 +465,12 @@
 
 static inline uint64_t l2meta_cow_end(QCowL2Meta *m)
 {
-    return m->offset + m->cow_end.offset
-        + (m->cow_end.nb_sectors << BDRV_SECTOR_BITS);
+    return m->offset + m->cow_end.offset + m->cow_end.nb_bytes;
+}
+
+static inline uint64_t refcount_diff(uint64_t r1, uint64_t r2)
+{
+    return r1 > r2 ? r1 - r2 : r2 - r1;
 }
 
 // FIXME Need qcow2_ prefix to global functions
@@ -487,14 +492,16 @@
 int qcow2_refcount_init(BlockDriverState *bs);
 void qcow2_refcount_close(BlockDriverState *bs);
 
-int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index);
+int qcow2_get_refcount(BlockDriverState *bs, int64_t cluster_index,
+                       uint64_t *refcount);
 
 int qcow2_update_cluster_refcount(BlockDriverState *bs, int64_t cluster_index,
-                                  int addend, enum qcow2_discard_type type);
+                                  uint64_t addend, bool decrease,
+                                  enum qcow2_discard_type type);
 
 int64_t qcow2_alloc_clusters(BlockDriverState *bs, uint64_t size);
-int qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
-    int nb_clusters);
+int64_t qcow2_alloc_clusters_at(BlockDriverState *bs, uint64_t offset,
+                                int64_t nb_clusters);
 int64_t qcow2_alloc_bytes(BlockDriverState *bs, int size);
 void qcow2_free_clusters(BlockDriverState *bs,
                           int64_t offset, int64_t size,
@@ -515,21 +522,25 @@
 int qcow2_pre_write_overlap_check(BlockDriverState *bs, int ign, int64_t offset,
                                   int64_t size);
 
+int qcow2_change_refcount_order(BlockDriverState *bs, int refcount_order,
+                                BlockDriverAmendStatusCB *status_cb,
+                                void *cb_opaque, Error **errp);
+
 /* qcow2-cluster.c functions */
 int qcow2_grow_l1_table(BlockDriverState *bs, uint64_t min_size,
                         bool exact_size);
 int qcow2_write_l1_entry(BlockDriverState *bs, int l1_index);
 void qcow2_l2_cache_reset(BlockDriverState *bs);
 int qcow2_decompress_cluster(BlockDriverState *bs, uint64_t cluster_offset);
-void qcow2_encrypt_sectors(BDRVQcowState *s, int64_t sector_num,
-                     uint8_t *out_buf, const uint8_t *in_buf,
-                     int nb_sectors, int enc,
-                     const AES_KEY *key);
+int qcow2_encrypt_sectors(BDRVQcow2State *s, int64_t sector_num,
+                          uint8_t *out_buf, const uint8_t *in_buf,
+                          int nb_sectors, bool enc, Error **errp);
 
 int qcow2_get_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int *num, uint64_t *cluster_offset);
+                             unsigned int *bytes, uint64_t *cluster_offset);
 int qcow2_alloc_cluster_offset(BlockDriverState *bs, uint64_t offset,
-    int *num, uint64_t *host_offset, QCowL2Meta **m);
+                               unsigned int *bytes, uint64_t *host_offset,
+                               QCowL2Meta **m);
 uint64_t qcow2_alloc_compressed_cluster_offset(BlockDriverState *bs,
                                          uint64_t offset,
                                          int compressed_size);
@@ -540,7 +551,8 @@
 int qcow2_zero_clusters(BlockDriverState *bs, uint64_t offset, int nb_sectors);
 
 int qcow2_expand_zero_clusters(BlockDriverState *bs,
-                               BlockDriverAmendStatusCB *status_cb);
+                               BlockDriverAmendStatusCB *status_cb,
+                               void *cb_opaque);
 
 /* qcow2-snapshot.c functions */
 int qcow2_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info);
@@ -562,18 +574,21 @@
 Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables);
 int qcow2_cache_destroy(BlockDriverState* bs, Qcow2Cache *c);
 
-void qcow2_cache_entry_mark_dirty(Qcow2Cache *c, void *table);
+void qcow2_cache_entry_mark_dirty(BlockDriverState *bs, Qcow2Cache *c,
+     void *table);
 int qcow2_cache_flush(BlockDriverState *bs, Qcow2Cache *c);
+int qcow2_cache_write(BlockDriverState *bs, Qcow2Cache *c);
 int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c,
     Qcow2Cache *dependency);
 void qcow2_cache_depends_on_flush(Qcow2Cache *c);
 
+void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c);
 int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c);
 
 int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
     void **table);
 int qcow2_cache_get_empty(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset,
     void **table);
-int qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table);
+void qcow2_cache_put(BlockDriverState *bs, Qcow2Cache *c, void **table);
 
 #endif
diff --git a/block/qed-check.c b/block/qed-check.c
index 36ecd29..dcd4f03 100644
--- a/block/qed-check.c
+++ b/block/qed-check.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qed.h"
 
 typedef struct {
@@ -233,8 +234,7 @@
     }
 
     check.result->bfi.total_clusters =
-        (s->header.image_size + s->header.cluster_size - 1) /
-            s->header.cluster_size;
+        DIV_ROUND_UP(s->header.image_size, s->header.cluster_size);
     ret = qed_check_l1_table(&check, s->l1_table);
     if (ret == 0) {
         /* Only check for leaks if entire image was scanned successfully */
diff --git a/block/qed-cluster.c b/block/qed-cluster.c
index f64b2af..c24e756 100644
--- a/block/qed-cluster.c
+++ b/block/qed-cluster.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qed.h"
 
 /**
diff --git a/block/qed-gencb.c b/block/qed-gencb.c
index b817a8b..faf8ecc 100644
--- a/block/qed-gencb.c
+++ b/block/qed-gencb.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qed.h"
 
 void *gencb_alloc(size_t len, BlockCompletionFunc *cb, void *opaque)
diff --git a/block/qed-l2-cache.c b/block/qed-l2-cache.c
index e9b2aae..5cba794 100644
--- a/block/qed-l2-cache.c
+++ b/block/qed-l2-cache.c
@@ -50,6 +50,7 @@
  * table will be deleted in favor of the existing cache entry.
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "qed.h"
 
diff --git a/block/qed-table.c b/block/qed-table.c
index 513aa87..1a731df 100644
--- a/block/qed-table.c
+++ b/block/qed-table.c
@@ -12,9 +12,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "qemu/sockets.h" /* for EINPROGRESS on Windows */
 #include "qed.h"
+#include "qemu/bswap.h"
 
 typedef struct {
     GenericCB gencb;
diff --git a/block/qed.c b/block/qed.c
index 80f18d8..426f3cb 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -12,11 +12,15 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
+#include "qemu/bswap.h"
 #include "trace.h"
 #include "qed.h"
 #include "qapi/qmp/qerror.h"
 #include "migration/migration.h"
+#include "sysemu/block-backend.h"
 
 static const AIOCBInfo qed_aiocb_info = {
     .aiocb_size         = sizeof(QEDAIOCB),
@@ -139,8 +143,7 @@
      * them, and write back.
      */
 
-    int nsectors = (sizeof(QEDHeader) + BDRV_SECTOR_SIZE - 1) /
-                   BDRV_SECTOR_SIZE;
+    int nsectors = DIV_ROUND_UP(sizeof(QEDHeader), BDRV_SECTOR_SIZE);
     size_t len = nsectors * BDRV_SECTOR_SIZE;
     QEDWriteHeaderCB *write_header_cb = gencb_alloc(sizeof(*write_header_cb),
                                                     cb, opaque);
@@ -215,7 +218,7 @@
  *
  * The string is NUL-terminated.
  */
-static int qed_read_string(BlockDriverState *file, uint64_t offset, size_t n,
+static int qed_read_string(BdrvChild *file, uint64_t offset, size_t n,
                            char *buf, size_t buflen)
 {
     int ret;
@@ -344,7 +347,7 @@
      * migration.
      */
     timer_mod(s->need_check_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() * QED_NEED_CHECK_TIMEOUT);
+                   NANOSECONDS_PER_SECOND * QED_NEED_CHECK_TIMEOUT);
 }
 
 /* It's okay to call this multiple times or when no timer is started */
@@ -354,12 +357,6 @@
     timer_del(s->need_check_timer);
 }
 
-static void bdrv_qed_rebind(BlockDriverState *bs)
-{
-    BDRVQEDState *s = bs->opaque;
-    s->bs = bs;
-}
-
 static void bdrv_qed_detach_aio_context(BlockDriverState *bs)
 {
     BDRVQEDState *s = bs->opaque;
@@ -404,11 +401,8 @@
     }
     if (s->header.features & ~QED_FEATURE_MASK) {
         /* image uses unsupported feature bits */
-        char buf[64];
-        snprintf(buf, sizeof(buf), "%" PRIx64,
-            s->header.features & ~QED_FEATURE_MASK);
-        error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-            bdrv_get_device_name(bs), "QED", buf);
+        error_setg(errp, "Unsupported QED features: %" PRIx64,
+                   s->header.features & ~QED_FEATURE_MASK);
         return -ENOTSUP;
     }
     if (!qed_is_cluster_size_valid(s->header.cluster_size)) {
@@ -416,7 +410,7 @@
     }
 
     /* Round down file size to the last cluster */
-    file_size = bdrv_getlength(bs->file);
+    file_size = bdrv_getlength(bs->file->bs);
     if (file_size < 0) {
         return file_size;
     }
@@ -436,9 +430,14 @@
 
     s->table_nelems = (s->header.cluster_size * s->header.table_size) /
                       sizeof(uint64_t);
-    s->l2_shift = ffs(s->header.cluster_size) - 1;
+    s->l2_shift = ctz32(s->header.cluster_size);
     s->l2_mask = s->table_nelems - 1;
-    s->l1_shift = s->l2_shift + ffs(s->table_nelems) - 1;
+    s->l1_shift = s->l2_shift + ctz32(s->table_nelems);
+
+    /* Header size calculation must not overflow uint32_t */
+    if (s->header.header_size > UINT32_MAX / s->header.cluster_size) {
+        return -EINVAL;
+    }
 
     if ((s->header.features & QED_F_BACKING_FILE)) {
         if ((uint64_t)s->header.backing_filename_offset +
@@ -466,7 +465,7 @@
      * feature is no longer valid.
      */
     if ((s->header.autoclear_features & ~QED_AUTOCLEAR_FEATURE_MASK) != 0 &&
-        !bdrv_is_read_only(bs->file) && !(flags & BDRV_O_INCOMING)) {
+        !bdrv_is_read_only(bs->file->bs) && !(flags & BDRV_O_INACTIVE)) {
         s->header.autoclear_features &= QED_AUTOCLEAR_FEATURE_MASK;
 
         ret = qed_write_header_sync(s);
@@ -475,7 +474,7 @@
         }
 
         /* From here on only known autoclear feature bits are valid */
-        bdrv_flush(bs->file);
+        bdrv_flush(bs->file->bs);
     }
 
     s->l1_table = qed_alloc_table(s);
@@ -493,8 +492,8 @@
          * potentially inconsistent images to be opened read-only.  This can
          * aid data recovery from an otherwise inconsistent image.
          */
-        if (!bdrv_is_read_only(bs->file) &&
-            !(flags & BDRV_O_INCOMING)) {
+        if (!bdrv_is_read_only(bs->file->bs) &&
+            !(flags & BDRV_O_INACTIVE)) {
             BdrvCheckResult result = {0};
 
             ret = qed_check(s, &result, true);
@@ -518,7 +517,7 @@
 {
     BDRVQEDState *s = bs->opaque;
 
-    bs->bl.write_zeroes_alignment = s->header.cluster_size >> BDRV_SECTOR_BITS;
+    bs->bl.pwrite_zeroes_alignment = s->header.cluster_size;
 }
 
 /* We have nothing to do for QED reopen, stubs just return
@@ -536,7 +535,7 @@
     bdrv_qed_detach_aio_context(bs);
 
     /* Ensure writes reach stable storage */
-    bdrv_flush(bs->file);
+    bdrv_flush(bs->file->bs);
 
     /* Clean shutdown, no check required on next open */
     if (s->header.features & QED_F_NEED_CHECK) {
@@ -568,7 +567,7 @@
     size_t l1_size = header.cluster_size * header.table_size;
     Error *local_err = NULL;
     int ret = 0;
-    BlockDriverState *bs;
+    BlockBackend *blk;
 
     ret = bdrv_create_file(filename, opts, &local_err);
     if (ret < 0) {
@@ -576,17 +575,17 @@
         return ret;
     }
 
-    bs = NULL;
-    ret = bdrv_open(&bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_PROTOCOL, NULL,
-                    &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
-        return ret;
+        return -EIO;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* File must start empty and grow, check truncate is supported */
-    ret = bdrv_truncate(bs, 0);
+    ret = blk_truncate(blk, 0);
     if (ret < 0) {
         goto out;
     }
@@ -602,18 +601,18 @@
     }
 
     qed_header_cpu_to_le(&header, &le_header);
-    ret = bdrv_pwrite(bs, 0, &le_header, sizeof(le_header));
+    ret = blk_pwrite(blk, 0, &le_header, sizeof(le_header), 0);
     if (ret < 0) {
         goto out;
     }
-    ret = bdrv_pwrite(bs, sizeof(le_header), backing_file,
-                      header.backing_filename_size);
+    ret = blk_pwrite(blk, sizeof(le_header), backing_file,
+                     header.backing_filename_size, 0);
     if (ret < 0) {
         goto out;
     }
 
     l1_table = g_malloc0(l1_size);
-    ret = bdrv_pwrite(bs, header.l1_table_offset, l1_table, l1_size);
+    ret = blk_pwrite(blk, header.l1_table_offset, l1_table, l1_size, 0);
     if (ret < 0) {
         goto out;
     }
@@ -621,7 +620,7 @@
     ret = 0; /* success */
 out:
     g_free(l1_table);
-    bdrv_unref(bs);
+    blk_unref(blk);
     return ret;
 }
 
@@ -681,6 +680,7 @@
     uint64_t pos;
     int64_t status;
     int *pnum;
+    BlockDriverState **file;
 } QEDIsAllocatedCB;
 
 static void qed_is_allocated_cb(void *opaque, int ret, uint64_t offset, size_t len)
@@ -692,6 +692,7 @@
     case QED_CLUSTER_FOUND:
         offset |= qed_offset_into_cluster(s, cb->pos);
         cb->status = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | offset;
+        *cb->file = cb->bs->file->bs;
         break;
     case QED_CLUSTER_ZERO:
         cb->status = BDRV_BLOCK_ZERO;
@@ -707,13 +708,14 @@
     }
 
     if (cb->co) {
-        qemu_coroutine_enter(cb->co, NULL);
+        qemu_coroutine_enter(cb->co);
     }
 }
 
 static int64_t coroutine_fn bdrv_qed_co_get_block_status(BlockDriverState *bs,
                                                  int64_t sector_num,
-                                                 int nb_sectors, int *pnum)
+                                                 int nb_sectors, int *pnum,
+                                                 BlockDriverState **file)
 {
     BDRVQEDState *s = bs->opaque;
     size_t len = (size_t)nb_sectors * BDRV_SECTOR_SIZE;
@@ -722,6 +724,7 @@
         .pos = (uint64_t)sector_num * BDRV_SECTOR_SIZE,
         .status = BDRV_BLOCK_OFFSET_MASK,
         .pnum = pnum,
+        .file = file,
     };
     QEDRequest request = { .l2_table = NULL };
 
@@ -767,8 +770,8 @@
     /* If there is a backing file, get its length.  Treat the absence of a
      * backing file like a zero length backing file.
      */
-    if (s->bs->backing_hd) {
-        int64_t l = bdrv_getlength(s->bs->backing_hd);
+    if (s->bs->backing) {
+        int64_t l = bdrv_getlength(s->bs->backing->bs);
         if (l < 0) {
             cb(opaque, l);
             return;
@@ -797,7 +800,7 @@
     qemu_iovec_concat(*backing_qiov, qiov, 0, size);
 
     BLKDBG_EVENT(s->bs->file, BLKDBG_READ_BACKING_AIO);
-    bdrv_aio_readv(s->bs->backing_hd, pos / BDRV_SECTOR_SIZE,
+    bdrv_aio_readv(s->bs->backing, pos / BDRV_SECTOR_SIZE,
                    *backing_qiov, size / BDRV_SECTOR_SIZE, cb, opaque);
 }
 
@@ -1050,7 +1053,7 @@
     QEDAIOCB *acb = opaque;
     BDRVQEDState *s = acb_to_s(acb);
 
-    if (!bdrv_aio_flush(s->bs->file, qed_aio_write_l2_update_cb, opaque)) {
+    if (!bdrv_aio_flush(s->bs->file->bs, qed_aio_write_l2_update_cb, opaque)) {
         qed_aio_complete(acb, -EIO);
     }
 }
@@ -1076,7 +1079,7 @@
     if (acb->find_cluster_ret == QED_CLUSTER_FOUND) {
         next_fn = qed_aio_next_io;
     } else {
-        if (s->bs->backing_hd) {
+        if (s->bs->backing) {
             next_fn = qed_aio_write_flush_before_l2_update;
         } else {
             next_fn = qed_aio_write_l2_update_cb;
@@ -1134,7 +1137,7 @@
 static bool qed_should_set_need_check(BDRVQEDState *s)
 {
     /* The flush before L2 update path ensures consistency */
-    if (s->bs->backing_hd) {
+    if (s->bs->backing) {
         return false;
     }
 
@@ -1415,21 +1418,21 @@
     bool done;
 } QEDWriteZeroesCB;
 
-static void coroutine_fn qed_co_write_zeroes_cb(void *opaque, int ret)
+static void coroutine_fn qed_co_pwrite_zeroes_cb(void *opaque, int ret)
 {
     QEDWriteZeroesCB *cb = opaque;
 
     cb->done = true;
     cb->ret = ret;
     if (cb->co) {
-        qemu_coroutine_enter(cb->co, NULL);
+        qemu_coroutine_enter(cb->co);
     }
 }
 
-static int coroutine_fn bdrv_qed_co_write_zeroes(BlockDriverState *bs,
-                                                 int64_t sector_num,
-                                                 int nb_sectors,
-                                                 BdrvRequestFlags flags)
+static int coroutine_fn bdrv_qed_co_pwrite_zeroes(BlockDriverState *bs,
+                                                  int64_t offset,
+                                                  int count,
+                                                  BdrvRequestFlags flags)
 {
     BlockAIOCB *blockacb;
     BDRVQEDState *s = bs->opaque;
@@ -1437,25 +1440,22 @@
     QEMUIOVector qiov;
     struct iovec iov;
 
-    /* Refuse if there are untouched backing file sectors */
-    if (bs->backing_hd) {
-        if (qed_offset_into_cluster(s, sector_num * BDRV_SECTOR_SIZE) != 0) {
-            return -ENOTSUP;
-        }
-        if (qed_offset_into_cluster(s, nb_sectors * BDRV_SECTOR_SIZE) != 0) {
-            return -ENOTSUP;
-        }
+    /* Fall back if the request is not aligned */
+    if (qed_offset_into_cluster(s, offset) ||
+        qed_offset_into_cluster(s, count)) {
+        return -ENOTSUP;
     }
 
     /* Zero writes start without an I/O buffer.  If a buffer becomes necessary
      * then it will be allocated during request processing.
      */
-    iov.iov_base = NULL,
-    iov.iov_len  = nb_sectors * BDRV_SECTOR_SIZE,
+    iov.iov_base = NULL;
+    iov.iov_len = count;
 
     qemu_iovec_init_external(&qiov, &iov, 1);
-    blockacb = qed_aio_setup(bs, sector_num, &qiov, nb_sectors,
-                             qed_co_write_zeroes_cb, &cb,
+    blockacb = qed_aio_setup(bs, offset >> BDRV_SECTOR_BITS, &qiov,
+                             count >> BDRV_SECTOR_BITS,
+                             qed_co_pwrite_zeroes_cb, &cb,
                              QED_AIOCB_WRITE | QED_AIOCB_ZERO);
     if (!blockacb) {
         return -EIO;
@@ -1591,18 +1591,11 @@
 
     bdrv_qed_close(bs);
 
-    bdrv_invalidate_cache(bs->file, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
     memset(s, 0, sizeof(BDRVQEDState));
     ret = bdrv_qed_open(bs, NULL, bs->open_flags, &local_err);
     if (local_err) {
-        error_setg(errp, "Could not reopen qed layer: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+        error_propagate(errp, local_err);
+        error_prepend(errp, "Could not reopen qed layer: ");
         return;
     } else if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not reopen qed layer");
@@ -1659,7 +1652,6 @@
     .supports_backing         = true,
 
     .bdrv_probe               = bdrv_qed_probe,
-    .bdrv_rebind              = bdrv_qed_rebind,
     .bdrv_open                = bdrv_qed_open,
     .bdrv_close               = bdrv_qed_close,
     .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
@@ -1668,7 +1660,7 @@
     .bdrv_co_get_block_status = bdrv_qed_co_get_block_status,
     .bdrv_aio_readv           = bdrv_qed_aio_readv,
     .bdrv_aio_writev          = bdrv_qed_aio_writev,
-    .bdrv_co_write_zeroes     = bdrv_qed_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes    = bdrv_qed_co_pwrite_zeroes,
     .bdrv_truncate            = bdrv_qed_truncate,
     .bdrv_getlength           = bdrv_qed_getlength,
     .bdrv_get_info            = bdrv_qed_get_info,
diff --git a/block/qed.h b/block/qed.h
index d3934a0..22b3198 100644
--- a/block/qed.h
+++ b/block/qed.h
@@ -16,6 +16,7 @@
 #define BLOCK_QED_H
 
 #include "block/block_int.h"
+#include "qemu/cutils.h"
 
 /* The layout of a QED file is as follows:
  *
@@ -133,7 +134,6 @@
     int bh_ret;                     /* final return status for completion bh */
     QSIMPLEQ_ENTRY(QEDAIOCB) next;  /* next request */
     int flags;                      /* QED_AIOCB_* bits ORed together */
-    bool *finished;                 /* signal for cancel completion */
     uint64_t end_pos;               /* request end on block device, in bytes */
 
     /* User scatter-gather list */
diff --git a/block/quorum.c b/block/quorum.c
index 437b122..9cf876f 100644
--- a/block/quorum.c
+++ b/block/quorum.c
@@ -13,16 +13,18 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <gnutls/gnutls.h>
-#include <gnutls/crypto.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "block/block_int.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/qlist.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi-event.h"
+#include "crypto/hash.h"
 
 #define HASH_LENGTH 32
 
@@ -33,7 +35,7 @@
 
 /* This union holds a vote hash value */
 typedef union QuorumVoteValue {
-    char h[HASH_LENGTH];       /* SHA-256 hash */
+    uint8_t h[HASH_LENGTH];    /* SHA-256 hash */
     int64_t l;                 /* simpler 64 bits hash */
 } QuorumVoteValue;
 
@@ -64,8 +66,11 @@
 
 /* the following structure holds the state of one quorum instance */
 typedef struct BDRVQuorumState {
-    BlockDriverState **bs; /* children BlockDriverStates */
+    BdrvChild **children;  /* children BlockDriverStates */
     int num_children;      /* children count */
+    unsigned next_child_index;  /* the index of the next child that should
+                                 * be added
+                                 */
     int threshold;         /* if less than threshold children reads gave the
                             * same result a quorum error occurs.
                             */
@@ -214,22 +219,21 @@
     return acb;
 }
 
-static void quorum_report_bad(QuorumAIOCB *acb, char *node_name, int ret)
+static void quorum_report_bad(QuorumOpType type, uint64_t sector_num,
+                              int nb_sectors, char *node_name, int ret)
 {
     const char *msg = NULL;
     if (ret < 0) {
         msg = strerror(-ret);
     }
-    qapi_event_send_quorum_report_bad(!!msg, msg, node_name,
-                                      acb->sector_num, acb->nb_sectors, &error_abort);
+
+    qapi_event_send_quorum_report_bad(type, !!msg, msg, node_name,
+                                      sector_num, nb_sectors, &error_abort);
 }
 
 static void quorum_report_failure(QuorumAIOCB *acb)
 {
-    const char *reference = bdrv_get_device_name(acb->common.bs)[0] ?
-                            bdrv_get_device_name(acb->common.bs) :
-                            acb->common.bs->node_name;
-
+    const char *reference = bdrv_get_device_or_node_name(acb->common.bs);
     qapi_event_send_quorum_failure(reference, acb->sector_num,
                                    acb->nb_sectors, &error_abort);
 }
@@ -286,9 +290,19 @@
     BDRVQuorumState *s = acb->common.bs->opaque;
     bool rewrite = false;
 
+    if (ret == 0) {
+        acb->success_count++;
+    } else {
+        QuorumOpType type;
+        type = acb->is_read ? QUORUM_OP_TYPE_READ : QUORUM_OP_TYPE_WRITE;
+        quorum_report_bad(type, acb->sector_num, acb->nb_sectors,
+                          sacb->aiocb->bs->node_name, ret);
+    }
+
     if (acb->is_read && s->read_pattern == QUORUM_READ_PATTERN_FIFO) {
         /* We try to read next child in FIFO order if we fail to read */
-        if (ret < 0 && ++acb->child_iter < s->num_children) {
+        if (ret < 0 && (acb->child_iter + 1) < s->num_children) {
+            acb->child_iter++;
             read_fifo_child(acb);
             return;
         }
@@ -303,11 +317,6 @@
 
     sacb->ret = ret;
     acb->count++;
-    if (ret == 0) {
-        acb->success_count++;
-    } else {
-        quorum_report_bad(acb, sacb->aiocb->bs->node_name, ret);
-    }
     assert(acb->count <= s->num_children);
     assert(acb->success_count <= s->num_children);
     if (acb->count < s->num_children) {
@@ -339,7 +348,9 @@
             continue;
         }
         QLIST_FOREACH(item, &version->items, next) {
-            quorum_report_bad(acb, s->bs[item->index]->node_name, 0);
+            quorum_report_bad(QUORUM_OP_TYPE_READ, acb->sector_num,
+                              acb->nb_sectors,
+                              s->children[item->index]->bs->node_name, 0);
         }
     }
 }
@@ -372,8 +383,9 @@
             continue;
         }
         QLIST_FOREACH(item, &version->items, next) {
-            bdrv_aio_writev(s->bs[item->index], acb->sector_num, acb->qiov,
-                            acb->nb_sectors, quorum_rewrite_aio_cb, acb);
+            bdrv_aio_writev(s->children[item->index], acb->sector_num,
+                            acb->qiov, acb->nb_sectors, quorum_rewrite_aio_cb,
+                            acb);
         }
     }
 
@@ -430,25 +442,21 @@
 
 static int quorum_compute_hash(QuorumAIOCB *acb, int i, QuorumVoteValue *hash)
 {
-    int j, ret;
-    gnutls_hash_hd_t dig;
     QEMUIOVector *qiov = &acb->qcrs[i].qiov;
+    size_t len = sizeof(hash->h);
+    uint8_t *data = hash->h;
 
-    ret = gnutls_hash_init(&dig, GNUTLS_DIG_SHA256);
-
-    if (ret < 0) {
-        return ret;
+    /* XXX - would be nice if we could pass in the Error **
+     * and propagate that back, but this quorum code is
+     * restricted to just errno values currently */
+    if (qcrypto_hash_bytesv(QCRYPTO_HASH_ALG_SHA256,
+                            qiov->iov, qiov->niov,
+                            &data, &len,
+                            NULL) < 0) {
+        return -EINVAL;
     }
 
-    for (j = 0; j < qiov->niov; j++) {
-        ret = gnutls_hash(dig, qiov->iov[j].iov_base, qiov->iov[j].iov_len);
-        if (ret < 0) {
-            break;
-        }
-    }
-
-    gnutls_hash_deinit(dig, (void *) hash);
-    return ret;
+    return 0;
 }
 
 static QuorumVoteVersion *quorum_get_vote_winner(QuorumVotes *votes)
@@ -646,14 +654,15 @@
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].buf = qemu_blockalign(s->bs[i], acb->qiov->size);
+        acb->qcrs[i].buf = qemu_blockalign(s->children[i]->bs, acb->qiov->size);
         qemu_iovec_init(&acb->qcrs[i].qiov, acb->qiov->niov);
         qemu_iovec_clone(&acb->qcrs[i].qiov, acb->qiov, acb->qcrs[i].buf);
     }
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_aio_readv(s->bs[i], acb->sector_num, &acb->qcrs[i].qiov,
-                       acb->nb_sectors, quorum_aio_cb, &acb->qcrs[i]);
+        acb->qcrs[i].aiocb = bdrv_aio_readv(s->children[i], acb->sector_num,
+                                            &acb->qcrs[i].qiov, acb->nb_sectors,
+                                            quorum_aio_cb, &acb->qcrs[i]);
     }
 
     return &acb->common;
@@ -663,14 +672,15 @@
 {
     BDRVQuorumState *s = acb->common.bs->opaque;
 
-    acb->qcrs[acb->child_iter].buf = qemu_blockalign(s->bs[acb->child_iter],
-                                                     acb->qiov->size);
+    acb->qcrs[acb->child_iter].buf =
+        qemu_blockalign(s->children[acb->child_iter]->bs, acb->qiov->size);
     qemu_iovec_init(&acb->qcrs[acb->child_iter].qiov, acb->qiov->niov);
     qemu_iovec_clone(&acb->qcrs[acb->child_iter].qiov, acb->qiov,
                      acb->qcrs[acb->child_iter].buf);
-    bdrv_aio_readv(s->bs[acb->child_iter], acb->sector_num,
-                   &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
-                   quorum_aio_cb, &acb->qcrs[acb->child_iter]);
+    acb->qcrs[acb->child_iter].aiocb =
+        bdrv_aio_readv(s->children[acb->child_iter], acb->sector_num,
+                       &acb->qcrs[acb->child_iter].qiov, acb->nb_sectors,
+                       quorum_aio_cb, &acb->qcrs[acb->child_iter]);
 
     return &acb->common;
 }
@@ -709,8 +719,8 @@
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        acb->qcrs[i].aiocb = bdrv_aio_writev(s->bs[i], sector_num, qiov,
-                                             nb_sectors, &quorum_aio_cb,
+        acb->qcrs[i].aiocb = bdrv_aio_writev(s->children[i], sector_num,
+                                             qiov, nb_sectors, &quorum_aio_cb,
                                              &acb->qcrs[i]);
     }
 
@@ -724,12 +734,12 @@
     int i;
 
     /* check that all file have the same length */
-    result = bdrv_getlength(s->bs[0]);
+    result = bdrv_getlength(s->children[0]->bs);
     if (result < 0) {
         return result;
     }
     for (i = 1; i < s->num_children; i++) {
-        int64_t value = bdrv_getlength(s->bs[i]);
+        int64_t value = bdrv_getlength(s->children[i]->bs);
         if (value < 0) {
             return value;
         }
@@ -741,21 +751,6 @@
     return result;
 }
 
-static void quorum_invalidate_cache(BlockDriverState *bs, Error **errp)
-{
-    BDRVQuorumState *s = bs->opaque;
-    Error *local_err = NULL;
-    int i;
-
-    for (i = 0; i < s->num_children; i++) {
-        bdrv_invalidate_cache(s->bs[i], &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return;
-        }
-    }
-}
-
 static coroutine_fn int quorum_co_flush(BlockDriverState *bs)
 {
     BDRVQuorumState *s = bs->opaque;
@@ -764,19 +759,30 @@
     QuorumVoteValue result_value;
     int i;
     int result = 0;
+    int success_count = 0;
 
     QLIST_INIT(&error_votes.vote_list);
     error_votes.compare = quorum_64bits_compare;
 
     for (i = 0; i < s->num_children; i++) {
-        result = bdrv_co_flush(s->bs[i]);
-        result_value.l = result;
-        quorum_count_vote(&error_votes, &result_value, i);
+        result = bdrv_co_flush(s->children[i]->bs);
+        if (result) {
+            quorum_report_bad(QUORUM_OP_TYPE_FLUSH, 0,
+                              bdrv_nb_sectors(s->children[i]->bs),
+                              s->children[i]->bs->node_name, result);
+            result_value.l = result;
+            quorum_count_vote(&error_votes, &result_value, i);
+        } else {
+            success_count++;
+        }
     }
 
-    winner = quorum_get_vote_winner(&error_votes);
-    result = winner->value.l;
-
+    if (success_count >= s->threshold) {
+        result = 0;
+    } else {
+        winner = quorum_get_vote_winner(&error_votes);
+        result = winner->value.l;
+    }
     quorum_free_vote_list(&error_votes);
 
     return result;
@@ -789,7 +795,7 @@
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bool perm = bdrv_recurse_is_first_non_filter(s->bs[i],
+        bool perm = bdrv_recurse_is_first_non_filter(s->children[i]->bs,
                                                      candidate);
         if (perm) {
             return true;
@@ -803,8 +809,8 @@
 {
 
     if (threshold < 1) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  "vote-threshold", "value >= 1");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "vote-threshold", "value >= 1");
         return -ERANGE;
     }
 
@@ -853,7 +859,7 @@
         return QUORUM_READ_PATTERN_QUORUM;
     }
 
-    for (i = 0; i < QUORUM_READ_PATTERN_MAX; i++) {
+    for (i = 0; i < QUORUM_READ_PATTERN__MAX; i++) {
         if (!strcmp(opt, QuorumReadPattern_lookup[i])) {
             return i;
         }
@@ -869,28 +875,21 @@
     Error *local_err = NULL;
     QemuOpts *opts = NULL;
     bool *opened;
-    QDict *sub = NULL;
-    QList *list = NULL;
-    const QListEntry *lentry;
     int i;
     int ret = 0;
 
     qdict_flatten(options);
-    qdict_extract_subqdict(options, &sub, "children.");
-    qdict_array_split(sub, &list);
 
-    if (qdict_size(sub)) {
-        error_setg(&local_err, "Invalid option children.%s",
-                   qdict_first(sub)->key);
+    /* count how many different children are present */
+    s->num_children = qdict_array_entries(options, "children.");
+    if (s->num_children < 0) {
+        error_setg(&local_err, "Option children is not a valid array");
         ret = -EINVAL;
         goto exit;
     }
-
-    /* count how many different children are present */
-    s->num_children = qlist_size(list);
-    if (s->num_children < 2) {
+    if (s->num_children < 1) {
         error_setg(&local_err,
-                   "Number of provided children must be greater than 1");
+                   "Number of provided children must be 1 or more");
         ret = -EINVAL;
         goto exit;
     }
@@ -903,6 +902,12 @@
     }
 
     s->threshold = qemu_opt_get_number(opts, QUORUM_OPT_VOTE_THRESHOLD, 0);
+    /* and validate it against s->num_children */
+    ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
+    if (ret < 0) {
+        goto exit;
+    }
+
     ret = parse_read_pattern(qemu_opt_get(opts, QUORUM_OPT_READ_PATTERN));
     if (ret < 0) {
         error_setg(&local_err, "Please set read-pattern as fifo or quorum");
@@ -911,12 +916,6 @@
     s->read_pattern = ret;
 
     if (s->read_pattern == QUORUM_READ_PATTERN_QUORUM) {
-        /* and validate it against s->num_children */
-        ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err);
-        if (ret < 0) {
-            goto exit;
-        }
-
         /* is the driver in blkverify mode */
         if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false) &&
             s->num_children == 2 && s->threshold == 2) {
@@ -936,43 +935,25 @@
         }
     }
 
-    /* allocate the children BlockDriverState array */
-    s->bs = g_new0(BlockDriverState *, s->num_children);
+    /* allocate the children array */
+    s->children = g_new0(BdrvChild *, s->num_children);
     opened = g_new0(bool, s->num_children);
 
-    for (i = 0, lentry = qlist_first(list); lentry;
-         lentry = qlist_next(lentry), i++) {
-        QDict *d;
-        QString *string;
+    for (i = 0; i < s->num_children; i++) {
+        char indexstr[32];
+        ret = snprintf(indexstr, 32, "children.%d", i);
+        assert(ret < 32);
 
-        switch (qobject_type(lentry->value))
-        {
-            /* List of options */
-            case QTYPE_QDICT:
-                d = qobject_to_qdict(lentry->value);
-                QINCREF(d);
-                ret = bdrv_open(&s->bs[i], NULL, NULL, d, flags, NULL,
-                                &local_err);
-                break;
-
-            /* QMP reference */
-            case QTYPE_QSTRING:
-                string = qobject_to_qstring(lentry->value);
-                ret = bdrv_open(&s->bs[i], NULL, qstring_get_str(string), NULL,
-                                flags, NULL, &local_err);
-                break;
-
-            default:
-                error_setg(&local_err, "Specification of child block device %i "
-                           "is invalid", i);
-                ret = -EINVAL;
-        }
-
-        if (ret < 0) {
+        s->children[i] = bdrv_open_child(NULL, options, indexstr, bs,
+                                         &child_format, false, &local_err);
+        if (local_err) {
+            ret = -EINVAL;
             goto close_exit;
         }
+
         opened[i] = true;
     }
+    s->next_child_index = s->num_children;
 
     g_free(opened);
     goto exit;
@@ -983,18 +964,14 @@
         if (!opened[i]) {
             continue;
         }
-        bdrv_unref(s->bs[i]);
+        bdrv_unref_child(bs, s->children[i]);
     }
-    g_free(s->bs);
+    g_free(s->children);
     g_free(opened);
 exit:
     qemu_opts_del(opts);
     /* propagate error */
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    QDECREF(list);
-    QDECREF(sub);
+    error_propagate(errp, local_err);
     return ret;
 }
 
@@ -1004,34 +981,79 @@
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_unref(s->bs[i]);
+        bdrv_unref_child(bs, s->children[i]);
     }
 
-    g_free(s->bs);
+    g_free(s->children);
 }
 
-static void quorum_detach_aio_context(BlockDriverState *bs)
+static void quorum_add_child(BlockDriverState *bs, BlockDriverState *child_bs,
+                             Error **errp)
+{
+    BDRVQuorumState *s = bs->opaque;
+    BdrvChild *child;
+    char indexstr[32];
+    int ret;
+
+    assert(s->num_children <= INT_MAX / sizeof(BdrvChild *));
+    if (s->num_children == INT_MAX / sizeof(BdrvChild *) ||
+        s->next_child_index == UINT_MAX) {
+        error_setg(errp, "Too many children");
+        return;
+    }
+
+    ret = snprintf(indexstr, 32, "children.%u", s->next_child_index);
+    if (ret < 0 || ret >= 32) {
+        error_setg(errp, "cannot generate child name");
+        return;
+    }
+    s->next_child_index++;
+
+    bdrv_drained_begin(bs);
+
+    /* We can safely add the child now */
+    bdrv_ref(child_bs);
+    child = bdrv_attach_child(bs, child_bs, indexstr, &child_format);
+    s->children = g_renew(BdrvChild *, s->children, s->num_children + 1);
+    s->children[s->num_children++] = child;
+
+    bdrv_drained_end(bs);
+}
+
+static void quorum_del_child(BlockDriverState *bs, BdrvChild *child,
+                             Error **errp)
 {
     BDRVQuorumState *s = bs->opaque;
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_detach_aio_context(s->bs[i]);
+        if (s->children[i] == child) {
+            break;
+        }
     }
+
+    /* we have checked it in bdrv_del_child() */
+    assert(i < s->num_children);
+
+    if (s->num_children <= s->threshold) {
+        error_setg(errp,
+            "The number of children cannot be lower than the vote threshold %d",
+            s->threshold);
+        return;
+    }
+
+    bdrv_drained_begin(bs);
+
+    /* We can safely remove this child now */
+    memmove(&s->children[i], &s->children[i + 1],
+            (s->num_children - i - 1) * sizeof(BdrvChild *));
+    s->children = g_renew(BdrvChild *, s->children, --s->num_children);
+    bdrv_unref_child(bs, child);
+
+    bdrv_drained_end(bs);
 }
 
-static void quorum_attach_aio_context(BlockDriverState *bs,
-                                      AioContext *new_context)
-{
-    BDRVQuorumState *s = bs->opaque;
-    int i;
-
-    for (i = 0; i < s->num_children; i++) {
-        bdrv_attach_aio_context(s->bs[i], new_context);
-    }
-}
-
-static void quorum_refresh_filename(BlockDriverState *bs)
+static void quorum_refresh_filename(BlockDriverState *bs, QDict *options)
 {
     BDRVQuorumState *s = bs->opaque;
     QDict *opts;
@@ -1039,16 +1061,17 @@
     int i;
 
     for (i = 0; i < s->num_children; i++) {
-        bdrv_refresh_filename(s->bs[i]);
-        if (!s->bs[i]->full_open_options) {
+        bdrv_refresh_filename(s->children[i]->bs);
+        if (!s->children[i]->bs->full_open_options) {
             return;
         }
     }
 
     children = qlist_new();
     for (i = 0; i < s->num_children; i++) {
-        QINCREF(s->bs[i]->full_open_options);
-        qlist_append_obj(children, QOBJECT(s->bs[i]->full_open_options));
+        QINCREF(s->children[i]->bs->full_open_options);
+        qlist_append_obj(children,
+                         QOBJECT(s->children[i]->bs->full_open_options));
     }
 
     opts = qdict_new();
@@ -1056,9 +1079,9 @@
     qdict_put_obj(opts, QUORUM_OPT_VOTE_THRESHOLD,
                   QOBJECT(qint_from_int(s->threshold)));
     qdict_put_obj(opts, QUORUM_OPT_BLKVERIFY,
-                  QOBJECT(qbool_from_int(s->is_blkverify)));
+                  QOBJECT(qbool_from_bool(s->is_blkverify)));
     qdict_put_obj(opts, QUORUM_OPT_REWRITE,
-                  QOBJECT(qbool_from_int(s->rewrite_corrupted)));
+                  QOBJECT(qbool_from_bool(s->rewrite_corrupted)));
     qdict_put_obj(opts, "children", QOBJECT(children));
 
     bs->full_open_options = opts;
@@ -1080,10 +1103,9 @@
 
     .bdrv_aio_readv                     = quorum_aio_readv,
     .bdrv_aio_writev                    = quorum_aio_writev,
-    .bdrv_invalidate_cache              = quorum_invalidate_cache,
 
-    .bdrv_detach_aio_context            = quorum_detach_aio_context,
-    .bdrv_attach_aio_context            = quorum_attach_aio_context,
+    .bdrv_add_child                     = quorum_add_child,
+    .bdrv_del_child                     = quorum_del_child,
 
     .is_filter                          = true,
     .bdrv_recurse_is_first_non_filter   = quorum_recurse_is_first_non_filter,
@@ -1091,6 +1113,10 @@
 
 static void bdrv_quorum_init(void)
 {
+    if (!qcrypto_hash_supports(QCRYPTO_HASH_ALG_SHA256)) {
+        /* SHA256 hash support is required for quorum device */
+        return;
+    }
     bdrv_register(&bdrv_quorum);
 }
 
diff --git a/block/raw-aio.h b/block/raw-aio.h
deleted file mode 100644
index 80681ce..0000000
--- a/block/raw-aio.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Declarations for AIO in the raw protocol
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-#ifndef QEMU_RAW_AIO_H
-#define QEMU_RAW_AIO_H
-
-/* AIO request types */
-#define QEMU_AIO_READ         0x0001
-#define QEMU_AIO_WRITE        0x0002
-#define QEMU_AIO_IOCTL        0x0004
-#define QEMU_AIO_FLUSH        0x0008
-#define QEMU_AIO_DISCARD      0x0010
-#define QEMU_AIO_WRITE_ZEROES 0x0020
-#define QEMU_AIO_TYPE_MASK \
-        (QEMU_AIO_READ|QEMU_AIO_WRITE|QEMU_AIO_IOCTL|QEMU_AIO_FLUSH| \
-         QEMU_AIO_DISCARD|QEMU_AIO_WRITE_ZEROES)
-
-/* AIO flags */
-#define QEMU_AIO_MISALIGNED   0x1000
-#define QEMU_AIO_BLKDEV       0x2000
-
-
-/* linux-aio.c - Linux native implementation */
-#ifdef CONFIG_LINUX_AIO
-void *laio_init(void);
-void laio_cleanup(void *s);
-BlockAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque, int type);
-void laio_detach_aio_context(void *s, AioContext *old_context);
-void laio_attach_aio_context(void *s, AioContext *new_context);
-void laio_io_plug(BlockDriverState *bs, void *aio_ctx);
-int laio_io_unplug(BlockDriverState *bs, void *aio_ctx, bool unplug);
-#endif
-
-#ifdef _WIN32
-typedef struct QEMUWin32AIOState QEMUWin32AIOState;
-QEMUWin32AIOState *win32_aio_init(void);
-void win32_aio_cleanup(QEMUWin32AIOState *aio);
-int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile);
-BlockAIOCB *win32_aio_submit(BlockDriverState *bs,
-        QEMUWin32AIOState *aio, HANDLE hfile,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque, int type);
-void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
-                                  AioContext *old_context);
-void win32_aio_attach_aio_context(QEMUWin32AIOState *aio,
-                                  AioContext *new_context);
-#endif
-
-#endif /* QEMU_RAW_AIO_H */
diff --git a/block/raw-posix.c b/block/raw-posix.c
index b1af77e..cabc4f3 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -21,7 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "qemu/log.h"
 #include "block/block_int.h"
@@ -29,8 +32,9 @@
 #include "trace.h"
 #include "block/thread-pool.h"
 #include "qemu/iov.h"
-#include "raw-aio.h"
+#include "block/raw-aio.h"
 #include "qapi/util.h"
+#include "qapi/qmp/qstring.h"
 
 #if defined(__APPLE__) && (__MACH__)
 #include <paths.h>
@@ -41,6 +45,7 @@
 #include <IOKit/storage/IOMedia.h>
 #include <IOKit/storage/IOCDMedia.h>
 //#include <IOKit/storage/IOCDTypes.h>
+#include <IOKit/storage/IODVDMedia.h>
 #include <CoreFoundation/CoreFoundation.h>
 #endif
 
@@ -49,18 +54,21 @@
 #include <sys/dkio.h>
 #endif
 #ifdef __linux__
-#include <sys/types.h>
-#include <sys/stat.h>
 #include <sys/ioctl.h>
 #include <sys/param.h>
 #include <linux/cdrom.h>
 #include <linux/fd.h>
 #include <linux/fs.h>
+#include <linux/hdreg.h>
+#include <scsi/sg.h>
+#ifdef __s390__
+#include <asm/dasd.h>
+#endif
 #ifndef FS_NOCOW_FL
 #define FS_NOCOW_FL                     0x00800000 /* Do not cow file */
 #endif
 #endif
-#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
+#if defined(CONFIG_FALLOCATE_PUNCH_HOLE) || defined(CONFIG_FALLOCATE_ZERO_RANGE)
 #include <linux/falloc.h>
 #endif
 #if defined (__FreeBSD__) || defined(__FreeBSD_kernel__)
@@ -90,15 +98,19 @@
 #include <xfs/xfs.h>
 #endif
 
-//#define DEBUG_FLOPPY
-
 //#define DEBUG_BLOCK
-#if defined(DEBUG_BLOCK)
-#define DEBUG_BLOCK_PRINT(formatCstr, ...) do { if (qemu_log_enabled()) \
-    { qemu_log(formatCstr, ## __VA_ARGS__); qemu_log_flush(); } } while (0)
+
+#ifdef DEBUG_BLOCK
+# define DEBUG_BLOCK_PRINT 1
 #else
-#define DEBUG_BLOCK_PRINT(formatCstr, ...)
+# define DEBUG_BLOCK_PRINT 0
 #endif
+#define DPRINTF(fmt, ...) \
+do { \
+    if (DEBUG_BLOCK_PRINT) { \
+        printf(fmt, ## __VA_ARGS__); \
+    } \
+} while (0)
 
 /* OS X does not have O_DSYNC */
 #ifndef O_DSYNC
@@ -116,11 +128,6 @@
 
 #define FTYPE_FILE   0
 #define FTYPE_CD     1
-#define FTYPE_FD     2
-
-/* if the FD is not accessed during that time (in ns), we try to
-   reopen it to see if the disk has been changed */
-#define FD_OPEN_TIMEOUT (1000000000)
 
 #define MAX_BLOCKSIZE	4096
 
@@ -130,32 +137,19 @@
     int open_flags;
     size_t buf_align;
 
-#if defined(__linux__)
-    /* linux floppy specific */
-    int64_t fd_open_time;
-    int64_t fd_error_time;
-    int fd_got_error;
-    int fd_media_changed;
-#endif
-#ifdef CONFIG_LINUX_AIO
-    int use_aio;
-    void *aio_ctx;
-#endif
 #ifdef CONFIG_XFS
     bool is_xfs:1;
 #endif
     bool has_discard:1;
     bool has_write_zeroes:1;
     bool discard_zeroes:1;
+    bool has_fallocate;
     bool needs_alignment;
 } BDRVRawState;
 
 typedef struct BDRVRawReopenState {
     int fd;
     int open_flags;
-#ifdef CONFIG_LINUX_AIO
-    int use_aio;
-#endif
 } BDRVRawReopenState;
 
 static int fd_open(BlockDriverState *bs);
@@ -188,7 +182,7 @@
 
     fname = *filename;
     dp = strrchr(fname, '/');
-    if (lstat(fname, &sb) < 0) {
+    if (qemu_lstat(fname, &sb) < 0) {
         fprintf(stderr, "%s: stat failed: %s\n",
             fname, strerror(errno));
         return -errno;
@@ -217,44 +211,106 @@
 }
 #endif
 
-static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
+/*
+ * Get logical block size via ioctl. On success store it in @sector_size_p.
+ */
+static int probe_logical_blocksize(int fd, unsigned int *sector_size_p)
 {
-    BDRVRawState *s = bs->opaque;
-    char *buf;
     unsigned int sector_size;
+    bool success = false;
 
-    /* For /dev/sg devices the alignment is not really used.
-       With buffered I/O, we don't have any restrictions. */
-    if (bs->sg || !s->needs_alignment) {
-        bs->request_alignment = 1;
-        s->buf_align = 1;
-        return;
-    }
+    errno = ENOTSUP;
 
     /* Try a few ioctls to get the right size */
-    bs->request_alignment = 0;
-    s->buf_align = 0;
-
 #ifdef BLKSSZGET
     if (ioctl(fd, BLKSSZGET, &sector_size) >= 0) {
-        bs->request_alignment = sector_size;
+        *sector_size_p = sector_size;
+        success = true;
     }
 #endif
 #ifdef DKIOCGETBLOCKSIZE
     if (ioctl(fd, DKIOCGETBLOCKSIZE, &sector_size) >= 0) {
-        bs->request_alignment = sector_size;
+        *sector_size_p = sector_size;
+        success = true;
     }
 #endif
 #ifdef DIOCGSECTORSIZE
     if (ioctl(fd, DIOCGSECTORSIZE, &sector_size) >= 0) {
-        bs->request_alignment = sector_size;
+        *sector_size_p = sector_size;
+        success = true;
     }
 #endif
+
+    return success ? 0 : -errno;
+}
+
+/**
+ * Get physical block size of @fd.
+ * On success, store it in @blk_size and return 0.
+ * On failure, return -errno.
+ */
+static int probe_physical_blocksize(int fd, unsigned int *blk_size)
+{
+#ifdef BLKPBSZGET
+    if (ioctl(fd, BLKPBSZGET, blk_size) < 0) {
+        return -errno;
+    }
+    return 0;
+#else
+    return -ENOTSUP;
+#endif
+}
+
+/* Check if read is allowed with given memory buffer and length.
+ *
+ * This function is used to check O_DIRECT memory buffer and request alignment.
+ */
+static bool raw_is_io_aligned(int fd, void *buf, size_t len)
+{
+    ssize_t ret = pread(fd, buf, len, 0);
+
+    if (ret >= 0) {
+        return true;
+    }
+
+#ifdef __linux__
+    /* The Linux kernel returns EINVAL for misaligned O_DIRECT reads.  Ignore
+     * other errors (e.g. real I/O error), which could happen on a failed
+     * drive, since we only care about probing alignment.
+     */
+    if (errno != EINVAL) {
+        return true;
+    }
+#endif
+
+    return false;
+}
+
+static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
+{
+    BDRVRawState *s = bs->opaque;
+    char *buf;
+    size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
+
+    /* For SCSI generic devices the alignment is not really used.
+       With buffered I/O, we don't have any restrictions. */
+    if (bdrv_is_sg(bs) || !s->needs_alignment) {
+        bs->bl.request_alignment = 1;
+        s->buf_align = 1;
+        return;
+    }
+
+    bs->bl.request_alignment = 0;
+    s->buf_align = 0;
+    /* Let's try to use the logical blocksize for the alignment. */
+    if (probe_logical_blocksize(fd, &bs->bl.request_alignment) < 0) {
+        bs->bl.request_alignment = 0;
+    }
 #ifdef CONFIG_XFS
     if (s->is_xfs) {
         struct dioattr da;
         if (xfsctl(NULL, fd, XFS_IOC_DIOINFO, &da) >= 0) {
-            bs->request_alignment = da.d_miniosz;
+            bs->bl.request_alignment = da.d_miniosz;
             /* The kernel returns wrong information for d_mem */
             /* s->buf_align = da.d_mem; */
         }
@@ -264,9 +320,9 @@
     /* If we could not get the sizes so far, we can only guess them */
     if (!s->buf_align) {
         size_t align;
-        buf = qemu_memalign(MAX_BLOCKSIZE, 2 * MAX_BLOCKSIZE);
-        for (align = 512; align <= MAX_BLOCKSIZE; align <<= 1) {
-            if (pread(fd, buf + align, MAX_BLOCKSIZE, 0) >= 0) {
+        buf = qemu_memalign(max_align, 2 * max_align);
+        for (align = 512; align <= max_align; align <<= 1) {
+            if (raw_is_io_aligned(fd, buf + align, max_align)) {
                 s->buf_align = align;
                 break;
             }
@@ -274,21 +330,21 @@
         qemu_vfree(buf);
     }
 
-    if (!bs->request_alignment) {
+    if (!bs->bl.request_alignment) {
         size_t align;
-        buf = qemu_memalign(s->buf_align, MAX_BLOCKSIZE);
-        for (align = 512; align <= MAX_BLOCKSIZE; align <<= 1) {
-            if (pread(fd, buf, align, 0) >= 0) {
-                bs->request_alignment = align;
+        buf = qemu_memalign(s->buf_align, max_align);
+        for (align = 512; align <= max_align; align <<= 1) {
+            if (raw_is_io_aligned(fd, buf, align)) {
+                bs->bl.request_alignment = align;
                 break;
             }
         }
         qemu_vfree(buf);
     }
 
-    if (!s->buf_align || !bs->request_alignment) {
-        error_setg(errp, "Could not find working O_DIRECT alignment. "
-                         "Try cache.direct=off.");
+    if (!s->buf_align || !bs->bl.request_alignment) {
+        error_setg(errp, "Could not find working O_DIRECT alignment");
+        error_append_hint(errp, "Try cache.direct=off\n");
     }
 }
 
@@ -311,58 +367,15 @@
     }
 }
 
-static void raw_detach_aio_context(BlockDriverState *bs)
-{
 #ifdef CONFIG_LINUX_AIO
-    BDRVRawState *s = bs->opaque;
-
-    if (s->use_aio) {
-        laio_detach_aio_context(s->aio_ctx, bdrv_get_aio_context(bs));
-    }
-#endif
-}
-
-static void raw_attach_aio_context(BlockDriverState *bs,
-                                   AioContext *new_context)
+static bool raw_use_aio(int bdrv_flags)
 {
-#ifdef CONFIG_LINUX_AIO
-    BDRVRawState *s = bs->opaque;
-
-    if (s->use_aio) {
-        laio_attach_aio_context(s->aio_ctx, new_context);
-    }
-#endif
-}
-
-#ifdef CONFIG_LINUX_AIO
-static int raw_set_aio(void **aio_ctx, int *use_aio, int bdrv_flags)
-{
-    int ret = -1;
-    assert(aio_ctx != NULL);
-    assert(use_aio != NULL);
     /*
      * Currently Linux do AIO only for files opened with O_DIRECT
      * specified so check NOCACHE flag too
      */
-    if ((bdrv_flags & (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) ==
-                      (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) {
-
-        /* if non-NULL, laio_init() has already been run */
-        if (*aio_ctx == NULL) {
-            *aio_ctx = laio_init();
-            if (!*aio_ctx) {
-                goto error;
-            }
-        }
-        *use_aio = 1;
-    } else {
-        *use_aio = 0;
-    }
-
-    ret = 0;
-
-error:
-    return ret;
+    return (bdrv_flags & (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) ==
+                         (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO);
 }
 #endif
 
@@ -431,26 +444,36 @@
     s->fd = fd;
 
 #ifdef CONFIG_LINUX_AIO
-    if (raw_set_aio(&s->aio_ctx, &s->use_aio, bdrv_flags)) {
-        qemu_close(fd);
-        ret = -errno;
-        error_setg_errno(errp, -ret, "Could not set AIO state");
+    if (!raw_use_aio(bdrv_flags) && (bdrv_flags & BDRV_O_NATIVE_AIO)) {
+        error_setg(errp, "aio=native was specified, but it requires "
+                         "cache.direct=on, which was not specified.");
+        ret = -EINVAL;
         goto fail;
     }
-#endif
+#else
+    if (bdrv_flags & BDRV_O_NATIVE_AIO) {
+        error_setg(errp, "aio=native was specified, but is not supported "
+                         "in this build.");
+        ret = -EINVAL;
+        goto fail;
+    }
+#endif /* !defined(CONFIG_LINUX_AIO) */
 
     s->has_discard = true;
     s->has_write_zeroes = true;
+    bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;
     if ((bs->open_flags & BDRV_O_NOCACHE) != 0) {
         s->needs_alignment = true;
     }
 
     if (fstat(s->fd, &st) < 0) {
+        ret = -errno;
         error_setg_errno(errp, errno, "Could not stat file");
         goto fail;
     }
     if (S_ISREG(st.st_mode)) {
         s->discard_zeroes = true;
+        s->has_fallocate = true;
     }
     if (S_ISBLK(st.st_mode)) {
 #ifdef BLKDISCARDZEROES
@@ -488,8 +511,6 @@
     }
 #endif
 
-    raw_attach_aio_context(bs, bdrv_get_aio_context(bs));
-
     ret = 0;
 fail:
     if (filename && (bdrv_flags & BDRV_O_TEMPORARY)) {
@@ -503,15 +524,9 @@
                     Error **errp)
 {
     BDRVRawState *s = bs->opaque;
-    Error *local_err = NULL;
-    int ret;
 
     s->type = FTYPE_FILE;
-    ret = raw_open_common(bs, options, flags, 0, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    return ret;
+    return raw_open_common(bs, options, flags, 0, errp);
 }
 
 static int raw_reopen_prepare(BDRVReopenState *state,
@@ -530,19 +545,7 @@
     state->opaque = g_new0(BDRVRawReopenState, 1);
     raw_s = state->opaque;
 
-#ifdef CONFIG_LINUX_AIO
-    raw_s->use_aio = s->use_aio;
-
-    /* we can use s->aio_ctx instead of a copy, because the use_aio flag is
-     * valid in the 'false' condition even if aio_ctx is set, and raw_set_aio()
-     * won't override aio_ctx if aio_ctx is non-NULL */
-    if (raw_set_aio(&s->aio_ctx, &raw_s->use_aio, state->flags)) {
-        error_setg(errp, "Could not set AIO state");
-        return -1;
-    }
-#endif
-
-    if (s->type == FTYPE_FD || s->type == FTYPE_CD) {
+    if (s->type == FTYPE_CD) {
         raw_s->open_flags |= O_NONBLOCK;
     }
 
@@ -566,15 +569,7 @@
 
     if ((raw_s->open_flags & ~fcntl_flags) == (s->open_flags & ~fcntl_flags)) {
         /* dup the original fd */
-        /* TODO: use qemu fcntl wrapper */
-#ifdef F_DUPFD_CLOEXEC
-        raw_s->fd = fcntl(s->fd, F_DUPFD_CLOEXEC, 0);
-#else
-        raw_s->fd = dup(s->fd);
-        if (raw_s->fd != -1) {
-            qemu_set_cloexec(raw_s->fd);
-        }
-#endif
+        raw_s->fd = qemu_dup(s->fd);
         if (raw_s->fd >= 0) {
             ret = fcntl_setfl(raw_s->fd, raw_s->open_flags);
             if (ret) {
@@ -586,11 +581,17 @@
 
     /* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */
     if (raw_s->fd == -1) {
-        assert(!(raw_s->open_flags & O_CREAT));
-        raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags);
-        if (raw_s->fd == -1) {
-            error_setg_errno(errp, errno, "Could not reopen file");
-            ret = -1;
+        const char *normalized_filename = state->bs->filename;
+        ret = raw_normalize_devicepath(&normalized_filename);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret, "Could not normalize device path");
+        } else {
+            assert(!(raw_s->open_flags & O_CREAT));
+            raw_s->fd = qemu_open(normalized_filename, raw_s->open_flags);
+            if (raw_s->fd == -1) {
+                error_setg_errno(errp, errno, "Could not reopen file");
+                ret = -1;
+            }
         }
     }
 
@@ -618,9 +619,6 @@
 
     qemu_close(s->fd);
     s->fd = raw_s->fd;
-#ifdef CONFIG_LINUX_AIO
-    s->use_aio = raw_s->use_aio;
-#endif
 
     g_free(state->opaque);
     state->opaque = NULL;
@@ -644,14 +642,112 @@
     state->opaque = NULL;
 }
 
+static int hdev_get_max_transfer_length(int fd)
+{
+#ifdef BLKSECTGET
+    int max_sectors = 0;
+    if (ioctl(fd, BLKSECTGET, &max_sectors) == 0) {
+        return max_sectors;
+    } else {
+        return -errno;
+    }
+#else
+    return -ENOSYS;
+#endif
+}
+
 static void raw_refresh_limits(BlockDriverState *bs, Error **errp)
 {
     BDRVRawState *s = bs->opaque;
+    struct stat st;
+
+    if (!fstat(s->fd, &st)) {
+        if (S_ISBLK(st.st_mode)) {
+            int ret = hdev_get_max_transfer_length(s->fd);
+            if (ret > 0 && ret <= BDRV_REQUEST_MAX_SECTORS) {
+                bs->bl.max_transfer = pow2floor(ret << BDRV_SECTOR_BITS);
+            }
+        }
+    }
 
     raw_probe_alignment(bs, s->fd, errp);
-    bs->bl.opt_mem_alignment = s->buf_align;
+    bs->bl.min_mem_alignment = s->buf_align;
+    bs->bl.opt_mem_alignment = MAX(s->buf_align, getpagesize());
 }
 
+static int check_for_dasd(int fd)
+{
+#ifdef BIODASDINFO2
+    struct dasd_information2_t info = {0};
+
+    return ioctl(fd, BIODASDINFO2, &info);
+#else
+    return -1;
+#endif
+}
+
+/**
+ * Try to get @bs's logical and physical block size.
+ * On success, store them in @bsz and return zero.
+ * On failure, return negative errno.
+ */
+static int hdev_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
+{
+    BDRVRawState *s = bs->opaque;
+    int ret;
+
+    /* If DASD, get blocksizes */
+    if (check_for_dasd(s->fd) < 0) {
+        return -ENOTSUP;
+    }
+    ret = probe_logical_blocksize(s->fd, &bsz->log);
+    if (ret < 0) {
+        return ret;
+    }
+    return probe_physical_blocksize(s->fd, &bsz->phys);
+}
+
+/**
+ * Try to get @bs's geometry: cyls, heads, sectors.
+ * On success, store them in @geo and return 0.
+ * On failure return -errno.
+ * (Allows block driver to assign default geometry values that guest sees)
+ */
+#ifdef __linux__
+static int hdev_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
+{
+    BDRVRawState *s = bs->opaque;
+    struct hd_geometry ioctl_geo = {0};
+
+    /* If DASD, get its geometry */
+    if (check_for_dasd(s->fd) < 0) {
+        return -ENOTSUP;
+    }
+    if (ioctl(s->fd, HDIO_GETGEO, &ioctl_geo) < 0) {
+        return -errno;
+    }
+    /* HDIO_GETGEO may return success even though geo contains zeros
+       (e.g. certain multipath setups) */
+    if (!ioctl_geo.heads || !ioctl_geo.sectors || !ioctl_geo.cylinders) {
+        return -ENOTSUP;
+    }
+    /* Do not return a geometry for partition */
+    if (ioctl_geo.start != 0) {
+        return -ENOTSUP;
+    }
+    geo->heads = ioctl_geo.heads;
+    geo->sectors = ioctl_geo.sectors;
+    geo->cylinders = ioctl_geo.cylinders;
+
+    return 0;
+}
+#else /* __linux__ */
+static int hdev_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
+{
+    return -ENOTSUP;
+}
+#endif
+
 static ssize_t handle_aiocb_ioctl(RawPosixAIOData *aiocb)
 {
     int ret;
@@ -860,6 +956,7 @@
 static int xfs_write_zeroes(BDRVRawState *s, int64_t offset, uint64_t bytes)
 {
     struct xfs_flock64 fl;
+    int err;
 
     memset(&fl, 0, sizeof(fl));
     fl.l_whence = SEEK_SET;
@@ -867,8 +964,9 @@
     fl.l_len = bytes;
 
     if (xfsctl(NULL, s->fd, XFS_IOC_ZERO_RANGE, &fl) < 0) {
-        DEBUG_BLOCK_PRINT("cannot write zero range (%s)\n", strerror(errno));
-        return -errno;
+        err = errno;
+        DPRINTF("cannot write zero range (%s)\n", strerror(errno));
+        return -err;
     }
 
     return 0;
@@ -877,6 +975,7 @@
 static int xfs_discard(BDRVRawState *s, int64_t offset, uint64_t bytes)
 {
     struct xfs_flock64 fl;
+    int err;
 
     memset(&fl, 0, sizeof(fl));
     fl.l_whence = SEEK_SET;
@@ -884,50 +983,121 @@
     fl.l_len = bytes;
 
     if (xfsctl(NULL, s->fd, XFS_IOC_UNRESVSP64, &fl) < 0) {
-        DEBUG_BLOCK_PRINT("cannot punch hole (%s)\n", strerror(errno));
-        return -errno;
+        err = errno;
+        DPRINTF("cannot punch hole (%s)\n", strerror(errno));
+        return -err;
     }
 
     return 0;
 }
 #endif
 
-static ssize_t handle_aiocb_write_zeroes(RawPosixAIOData *aiocb)
+static int translate_err(int err)
 {
-    int ret = -EOPNOTSUPP;
+    if (err == -ENODEV || err == -ENOSYS || err == -EOPNOTSUPP ||
+        err == -ENOTTY) {
+        err = -ENOTSUP;
+    }
+    return err;
+}
+
+#ifdef CONFIG_FALLOCATE
+static int do_fallocate(int fd, int mode, off_t offset, off_t len)
+{
+    do {
+        if (fallocate(fd, mode, offset, len) == 0) {
+            return 0;
+        }
+    } while (errno == EINTR);
+    return translate_err(-errno);
+}
+#endif
+
+static ssize_t handle_aiocb_write_zeroes_block(RawPosixAIOData *aiocb)
+{
+    int ret = -ENOTSUP;
     BDRVRawState *s = aiocb->bs->opaque;
 
-    if (s->has_write_zeroes == 0) {
+    if (!s->has_write_zeroes) {
         return -ENOTSUP;
     }
 
-    if (aiocb->aio_type & QEMU_AIO_BLKDEV) {
 #ifdef BLKZEROOUT
-        do {
-            uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };
-            if (ioctl(aiocb->aio_fildes, BLKZEROOUT, range) == 0) {
-                return 0;
-            }
-        } while (errno == EINTR);
-
-        ret = -errno;
-#endif
-    } else {
-#ifdef CONFIG_XFS
-        if (s->is_xfs) {
-            return xfs_write_zeroes(s, aiocb->aio_offset, aiocb->aio_nbytes);
+    do {
+        uint64_t range[2] = { aiocb->aio_offset, aiocb->aio_nbytes };
+        if (ioctl(aiocb->aio_fildes, BLKZEROOUT, range) == 0) {
+            return 0;
         }
-#endif
-    }
+    } while (errno == EINTR);
 
-    if (ret == -ENODEV || ret == -ENOSYS || ret == -EOPNOTSUPP ||
-        ret == -ENOTTY) {
+    ret = translate_err(-errno);
+#endif
+
+    if (ret == -ENOTSUP) {
         s->has_write_zeroes = false;
-        ret = -ENOTSUP;
     }
     return ret;
 }
 
+static ssize_t handle_aiocb_write_zeroes(RawPosixAIOData *aiocb)
+{
+#if defined(CONFIG_FALLOCATE) || defined(CONFIG_XFS)
+    BDRVRawState *s = aiocb->bs->opaque;
+#endif
+
+    if (aiocb->aio_type & QEMU_AIO_BLKDEV) {
+        return handle_aiocb_write_zeroes_block(aiocb);
+    }
+
+#ifdef CONFIG_XFS
+    if (s->is_xfs) {
+        return xfs_write_zeroes(s, aiocb->aio_offset, aiocb->aio_nbytes);
+    }
+#endif
+
+#ifdef CONFIG_FALLOCATE_ZERO_RANGE
+    if (s->has_write_zeroes) {
+        int ret = do_fallocate(s->fd, FALLOC_FL_ZERO_RANGE,
+                               aiocb->aio_offset, aiocb->aio_nbytes);
+        if (ret == 0 || ret != -ENOTSUP) {
+            return ret;
+        }
+        s->has_write_zeroes = false;
+    }
+#endif
+
+#ifdef CONFIG_FALLOCATE_PUNCH_HOLE
+    if (s->has_discard && s->has_fallocate) {
+        int ret = do_fallocate(s->fd,
+                               FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                               aiocb->aio_offset, aiocb->aio_nbytes);
+        if (ret == 0) {
+            ret = do_fallocate(s->fd, 0, aiocb->aio_offset, aiocb->aio_nbytes);
+            if (ret == 0 || ret != -ENOTSUP) {
+                return ret;
+            }
+            s->has_fallocate = false;
+        } else if (ret != -ENOTSUP) {
+            return ret;
+        } else {
+            s->has_discard = false;
+        }
+    }
+#endif
+
+#ifdef CONFIG_FALLOCATE
+    if (s->has_fallocate && aiocb->aio_offset >= bdrv_getlength(aiocb->bs)) {
+        int ret = do_fallocate(s->fd, 0, aiocb->aio_offset, aiocb->aio_nbytes);
+        if (ret == 0 || ret != -ENOTSUP) {
+            return ret;
+        }
+        s->has_fallocate = false;
+    }
+#endif
+
+    return -ENOTSUP;
+}
+
 static ssize_t handle_aiocb_discard(RawPosixAIOData *aiocb)
 {
     int ret = -EOPNOTSUPP;
@@ -956,21 +1126,14 @@
 #endif
 
 #ifdef CONFIG_FALLOCATE_PUNCH_HOLE
-        do {
-            if (fallocate(s->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
-                          aiocb->aio_offset, aiocb->aio_nbytes) == 0) {
-                return 0;
-            }
-        } while (errno == EINTR);
-
-        ret = -errno;
+        ret = do_fallocate(s->fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
+                           aiocb->aio_offset, aiocb->aio_nbytes);
 #endif
     }
 
-    if (ret == -ENODEV || ret == -ENOSYS || ret == -EOPNOTSUPP ||
-        ret == -ENOTTY) {
+    ret = translate_err(ret);
+    if (ret == -ENOTSUP) {
         s->has_discard = false;
-        ret = -ENOTSUP;
     }
     return ret;
 }
@@ -983,7 +1146,7 @@
     switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
     case QEMU_AIO_READ:
         ret = handle_aiocb_rw(aiocb);
-        if (ret >= 0 && ret < aiocb->aio_nbytes && aiocb->bs->growable) {
+        if (ret >= 0 && ret < aiocb->aio_nbytes) {
             iov_memset(aiocb->aio_iov, aiocb->aio_niov, ret,
                       0, aiocb->aio_nbytes - ret);
 
@@ -1021,48 +1184,48 @@
         break;
     }
 
-    g_slice_free(RawPosixAIOData, aiocb);
+    g_free(aiocb);
     return ret;
 }
 
 static int paio_submit_co(BlockDriverState *bs, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        int type)
+                          int64_t offset, QEMUIOVector *qiov,
+                          int count, int type)
 {
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+    RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
     acb->aio_type = type;
     acb->aio_fildes = fd;
 
-    acb->aio_nbytes = nb_sectors * BDRV_SECTOR_SIZE;
-    acb->aio_offset = sector_num * BDRV_SECTOR_SIZE;
+    acb->aio_nbytes = count;
+    acb->aio_offset = offset;
 
     if (qiov) {
         acb->aio_iov = qiov->iov;
         acb->aio_niov = qiov->niov;
-        assert(qiov->size == acb->aio_nbytes);
+        assert(qiov->size == count);
     }
 
-    trace_paio_submit_co(sector_num, nb_sectors, type);
+    trace_paio_submit_co(offset, count, type);
     pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
     return thread_pool_submit_co(pool, aio_worker, acb);
 }
 
 static BlockAIOCB *paio_submit(BlockDriverState *bs, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        int64_t offset, QEMUIOVector *qiov, int count,
         BlockCompletionFunc *cb, void *opaque, int type)
 {
-    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+    RawPosixAIOData *acb = g_new(RawPosixAIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
     acb->aio_type = type;
     acb->aio_fildes = fd;
 
-    acb->aio_nbytes = nb_sectors * BDRV_SECTOR_SIZE;
-    acb->aio_offset = sector_num * BDRV_SECTOR_SIZE;
+    acb->aio_nbytes = count;
+    acb->aio_offset = offset;
 
     if (qiov) {
         acb->aio_iov = qiov->iov;
@@ -1070,19 +1233,18 @@
         assert(qiov->size == acb->aio_nbytes);
     }
 
-    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
+    trace_paio_submit(acb, opaque, offset, count, type);
     pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
     return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
 }
 
-static BlockAIOCB *raw_aio_submit(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque, int type)
+static int coroutine_fn raw_co_prw(BlockDriverState *bs, uint64_t offset,
+                                   uint64_t bytes, QEMUIOVector *qiov, int type)
 {
     BDRVRawState *s = bs->opaque;
 
     if (fd_open(bs) < 0)
-        return NULL;
+        return -EIO;
 
     /*
      * Check if the underlying device requires requests to be aligned,
@@ -1094,23 +1256,38 @@
         if (!bdrv_qiov_is_aligned(bs, qiov)) {
             type |= QEMU_AIO_MISALIGNED;
 #ifdef CONFIG_LINUX_AIO
-        } else if (s->use_aio) {
-            return laio_submit(bs, s->aio_ctx, s->fd, sector_num, qiov,
-                               nb_sectors, cb, opaque, type);
+        } else if (bs->open_flags & BDRV_O_NATIVE_AIO) {
+            LinuxAioState *aio = aio_get_linux_aio(bdrv_get_aio_context(bs));
+            assert(qiov->size == bytes);
+            return laio_co_submit(bs, aio, s->fd, offset, qiov, type);
 #endif
         }
     }
 
-    return paio_submit(bs, s->fd, sector_num, qiov, nb_sectors,
-                       cb, opaque, type);
+    return paio_submit_co(bs, s->fd, offset, qiov, bytes, type);
+}
+
+static int coroutine_fn raw_co_preadv(BlockDriverState *bs, uint64_t offset,
+                                      uint64_t bytes, QEMUIOVector *qiov,
+                                      int flags)
+{
+    return raw_co_prw(bs, offset, bytes, qiov, QEMU_AIO_READ);
+}
+
+static int coroutine_fn raw_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                                       uint64_t bytes, QEMUIOVector *qiov,
+                                       int flags)
+{
+    assert(flags == 0);
+    return raw_co_prw(bs, offset, bytes, qiov, QEMU_AIO_WRITE);
 }
 
 static void raw_aio_plug(BlockDriverState *bs)
 {
 #ifdef CONFIG_LINUX_AIO
-    BDRVRawState *s = bs->opaque;
-    if (s->use_aio) {
-        laio_io_plug(bs, s->aio_ctx);
+    if (bs->open_flags & BDRV_O_NATIVE_AIO) {
+        LinuxAioState *aio = aio_get_linux_aio(bdrv_get_aio_context(bs));
+        laio_io_plug(bs, aio);
     }
 #endif
 }
@@ -1118,39 +1295,13 @@
 static void raw_aio_unplug(BlockDriverState *bs)
 {
 #ifdef CONFIG_LINUX_AIO
-    BDRVRawState *s = bs->opaque;
-    if (s->use_aio) {
-        laio_io_unplug(bs, s->aio_ctx, true);
+    if (bs->open_flags & BDRV_O_NATIVE_AIO) {
+        LinuxAioState *aio = aio_get_linux_aio(bdrv_get_aio_context(bs));
+        laio_io_unplug(bs, aio);
     }
 #endif
 }
 
-static void raw_aio_flush_io_queue(BlockDriverState *bs)
-{
-#ifdef CONFIG_LINUX_AIO
-    BDRVRawState *s = bs->opaque;
-    if (s->use_aio) {
-        laio_io_unplug(bs, s->aio_ctx, false);
-    }
-#endif
-}
-
-static BlockAIOCB *raw_aio_readv(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    return raw_aio_submit(bs, sector_num, qiov, nb_sectors,
-                          cb, opaque, QEMU_AIO_READ);
-}
-
-static BlockAIOCB *raw_aio_writev(BlockDriverState *bs,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque)
-{
-    return raw_aio_submit(bs, sector_num, qiov, nb_sectors,
-                          cb, opaque, QEMU_AIO_WRITE);
-}
-
 static BlockAIOCB *raw_aio_flush(BlockDriverState *bs,
         BlockCompletionFunc *cb, void *opaque)
 {
@@ -1166,13 +1317,6 @@
 {
     BDRVRawState *s = bs->opaque;
 
-    raw_detach_aio_context(bs);
-
-#ifdef CONFIG_LINUX_AIO
-    if (s->use_aio) {
-        laio_cleanup(s->aio_ctx);
-    }
-#endif
     if (s->fd >= 0) {
         qemu_close(s->fd);
         s->fd = -1;
@@ -1311,7 +1455,20 @@
         if (size == 0)
 #endif
 #if defined(__APPLE__) && defined(__MACH__)
-        size = LLONG_MAX;
+        {
+            uint64_t sectors = 0;
+            uint32_t sector_size = 0;
+
+            if (ioctl(fd, DKIOCGETBLOCKCOUNT, &sectors) == 0
+               && ioctl(fd, DKIOCGETBLOCKSIZE, &sector_size) == 0) {
+                size = sectors * sector_size;
+            } else {
+                size = lseek(fd, 0LL, SEEK_END);
+                if (size < 0) {
+                    return -errno;
+                }
+            }
+        }
 #else
         size = lseek(fd, 0LL, SEEK_END);
         if (size < 0) {
@@ -1388,7 +1545,7 @@
     nocow = qemu_opt_get_bool(opts, BLOCK_OPT_NOCOW, false);
     buf = qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC);
     prealloc = qapi_enum_parse(PreallocMode_lookup, buf,
-                               PREALLOC_MODE_MAX, PREALLOC_MODE_OFF,
+                               PREALLOC_MODE__MAX, PREALLOC_MODE_OFF,
                                &local_err);
     g_free(buf);
     if (local_err) {
@@ -1397,7 +1554,7 @@
         goto out;
     }
 
-    fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY,
+    fd = qemu_open(filename, O_RDWR | O_CREAT | O_TRUNC | O_BINARY,
                    0644);
     if (fd < 0) {
         result = -errno;
@@ -1582,7 +1739,8 @@
  */
 static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
                                                     int64_t sector_num,
-                                                    int nb_sectors, int *pnum)
+                                                    int nb_sectors, int *pnum,
+                                                    BlockDriverState **file)
 {
     off_t start, data = 0, hole = 0;
     int64_t total_size;
@@ -1614,8 +1772,9 @@
         *pnum = nb_sectors;
         ret = BDRV_BLOCK_DATA;
     } else if (data == start) {
-        /* On a data extent, compute sectors to the end of the extent.  */
-        *pnum = MIN(nb_sectors, (hole - start) / BDRV_SECTOR_SIZE);
+        /* On a data extent, compute sectors to the end of the extent,
+         * possibly including a partial sector at EOF. */
+        *pnum = MIN(nb_sectors, DIV_ROUND_UP(hole - start, BDRV_SECTOR_SIZE));
         ret = BDRV_BLOCK_DATA;
     } else {
         /* On a hole, compute sectors to the beginning of the next extent.  */
@@ -1623,30 +1782,31 @@
         *pnum = MIN(nb_sectors, (data - start) / BDRV_SECTOR_SIZE);
         ret = BDRV_BLOCK_ZERO;
     }
+    *file = bs;
     return ret | BDRV_BLOCK_OFFSET_VALID | start;
 }
 
-static coroutine_fn BlockAIOCB *raw_aio_discard(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors,
+static coroutine_fn BlockAIOCB *raw_aio_pdiscard(BlockDriverState *bs,
+    int64_t offset, int count,
     BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
 
-    return paio_submit(bs, s->fd, sector_num, NULL, nb_sectors,
+    return paio_submit(bs, s->fd, offset, NULL, count,
                        cb, opaque, QEMU_AIO_DISCARD);
 }
 
-static int coroutine_fn raw_co_write_zeroes(
-    BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, BdrvRequestFlags flags)
+static int coroutine_fn raw_co_pwrite_zeroes(
+    BlockDriverState *bs, int64_t offset,
+    int count, BdrvRequestFlags flags)
 {
     BDRVRawState *s = bs->opaque;
 
     if (!(flags & BDRV_REQ_MAY_UNMAP)) {
-        return paio_submit_co(bs, s->fd, sector_num, NULL, nb_sectors,
+        return paio_submit_co(bs, s->fd, offset, NULL, count,
                               QEMU_AIO_WRITE_ZEROES);
     } else if (s->discard_zeroes) {
-        return paio_submit_co(bs, s->fd, sector_num, NULL, nb_sectors,
+        return paio_submit_co(bs, s->fd, offset, NULL, count,
                               QEMU_AIO_DISCARD);
     }
     return -ENOTSUP;
@@ -1684,7 +1844,7 @@
     }
 };
 
-static BlockDriver bdrv_file = {
+BlockDriver bdrv_file = {
     .format_name = "file",
     .protocol_name = "file",
     .instance_size = sizeof(BDRVRawState),
@@ -1699,16 +1859,15 @@
     .bdrv_create = raw_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
     .bdrv_co_get_block_status = raw_co_get_block_status,
-    .bdrv_co_write_zeroes = raw_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes = raw_co_pwrite_zeroes,
 
-    .bdrv_aio_readv = raw_aio_readv,
-    .bdrv_aio_writev = raw_aio_writev,
+    .bdrv_co_preadv         = raw_co_preadv,
+    .bdrv_co_pwritev        = raw_co_pwritev,
     .bdrv_aio_flush = raw_aio_flush,
-    .bdrv_aio_discard = raw_aio_discard,
+    .bdrv_aio_pdiscard = raw_aio_pdiscard,
     .bdrv_refresh_limits = raw_refresh_limits,
     .bdrv_io_plug = raw_aio_plug,
     .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate = raw_truncate,
     .bdrv_getlength = raw_getlength,
@@ -1716,9 +1875,6 @@
     .bdrv_get_allocated_file_size
                         = raw_get_allocated_file_size,
 
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
-
     .create_opts = &raw_create_opts,
 };
 
@@ -1726,36 +1882,51 @@
 /* host device */
 
 #if defined(__APPLE__) && defined(__MACH__)
-static kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator );
-static kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize );
-
-kern_return_t FindEjectableCDMedia( io_iterator_t *mediaIterator )
+static kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
+                                CFIndex maxPathSize, int flags);
+static char *FindEjectableOpticalMedia(io_iterator_t *mediaIterator)
 {
-    kern_return_t       kernResult;
+    kern_return_t kernResult = KERN_FAILURE;
     mach_port_t     masterPort;
     CFMutableDictionaryRef  classesToMatch;
+    const char *matching_array[] = {kIODVDMediaClass, kIOCDMediaClass};
+    char *mediaType = NULL;
 
     kernResult = IOMasterPort( MACH_PORT_NULL, &masterPort );
     if ( KERN_SUCCESS != kernResult ) {
         printf( "IOMasterPort returned %d\n", kernResult );
     }
 
-    classesToMatch = IOServiceMatching( kIOCDMediaClass );
-    if ( classesToMatch == NULL ) {
-        printf( "IOServiceMatching returned a NULL dictionary.\n" );
-    } else {
-    CFDictionarySetValue( classesToMatch, CFSTR( kIOMediaEjectableKey ), kCFBooleanTrue );
-    }
-    kernResult = IOServiceGetMatchingServices( masterPort, classesToMatch, mediaIterator );
-    if ( KERN_SUCCESS != kernResult )
-    {
-        printf( "IOServiceGetMatchingServices returned %d\n", kernResult );
-    }
+    int index;
+    for (index = 0; index < ARRAY_SIZE(matching_array); index++) {
+        classesToMatch = IOServiceMatching(matching_array[index]);
+        if (classesToMatch == NULL) {
+            error_report("IOServiceMatching returned NULL for %s",
+                         matching_array[index]);
+            continue;
+        }
+        CFDictionarySetValue(classesToMatch, CFSTR(kIOMediaEjectableKey),
+                             kCFBooleanTrue);
+        kernResult = IOServiceGetMatchingServices(masterPort, classesToMatch,
+                                                  mediaIterator);
+        if (kernResult != KERN_SUCCESS) {
+            error_report("Note: IOServiceGetMatchingServices returned %d",
+                         kernResult);
+            continue;
+        }
 
-    return kernResult;
+        /* If a match was found, leave the loop */
+        if (*mediaIterator != 0) {
+            DPRINTF("Matching using %s\n", matching_array[index]);
+            mediaType = g_strdup(matching_array[index]);
+            break;
+        }
+    }
+    return mediaType;
 }
 
-kern_return_t GetBSDPath( io_iterator_t mediaIterator, char *bsdPath, CFIndex maxPathSize )
+kern_return_t GetBSDPath(io_iterator_t mediaIterator, char *bsdPath,
+                         CFIndex maxPathSize, int flags)
 {
     io_object_t     nextMedia;
     kern_return_t   kernResult = KERN_FAILURE;
@@ -1768,7 +1939,9 @@
         if ( bsdPathAsCFString ) {
             size_t devPathLength;
             strcpy( bsdPath, _PATH_DEV );
-            strcat( bsdPath, "r" );
+            if (flags & BDRV_O_NOCACHE) {
+                strcat(bsdPath, "r");
+            }
             devPathLength = strlen( bsdPath );
             if ( CFStringGetCString( bsdPathAsCFString, bsdPath + devPathLength, maxPathSize - devPathLength, kCFStringEncodingASCII ) ) {
                 kernResult = KERN_SUCCESS;
@@ -1781,7 +1954,46 @@
     return kernResult;
 }
 
-#endif
+/* Sets up a real cdrom for use in QEMU */
+static bool setup_cdrom(char *bsd_path, Error **errp)
+{
+    int index, num_of_test_partitions = 2, fd;
+    char test_partition[MAXPATHLEN];
+    bool partition_found = false;
+
+    /* look for a working partition */
+    for (index = 0; index < num_of_test_partitions; index++) {
+        snprintf(test_partition, sizeof(test_partition), "%ss%d", bsd_path,
+                 index);
+        fd = qemu_open(test_partition, O_RDONLY | O_BINARY | O_LARGEFILE);
+        if (fd >= 0) {
+            partition_found = true;
+            qemu_close(fd);
+            break;
+        }
+    }
+
+    /* if a working partition on the device was not found */
+    if (partition_found == false) {
+        error_setg(errp, "Failed to find a working partition on disc");
+    } else {
+        DPRINTF("Using %s as optical disc\n", test_partition);
+        pstrcpy(bsd_path, MAXPATHLEN, test_partition);
+    }
+    return partition_found;
+}
+
+/* Prints directions on mounting and unmounting a device */
+static void print_unmounting_directions(const char *file_name)
+{
+    error_report("If device %s is mounted on the desktop, unmount"
+                 " it first before using it in QEMU", file_name);
+    error_report("Command to unmount device: diskutil unmountDisk %s",
+                 file_name);
+    error_report("Command to mount device: diskutil mountDisk %s", file_name);
+}
+
+#endif /* defined(__APPLE__) && defined(__MACH__) */
 
 static int hdev_probe_device(const char *filename)
 {
@@ -1791,7 +2003,7 @@
     if (strstart(filename, "/dev/cdrom", NULL))
         return 50;
 
-    if (stat(filename, &st) >= 0 &&
+    if (qemu_stat(filename, &st) >= 0 &&
             (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode))) {
         return 100;
     }
@@ -1841,62 +2053,109 @@
     qdict_put_obj(options, "filename", QOBJECT(qstring_from_str(filename)));
 }
 
+static bool hdev_is_sg(BlockDriverState *bs)
+{
+
+#if defined(__linux__)
+
+    struct stat st;
+    struct sg_scsi_id scsiid;
+    int sg_version;
+
+    if (qemu_stat(bs->filename, &st) >= 0 && S_ISCHR(st.st_mode) &&
+        !bdrv_ioctl(bs, SG_GET_VERSION_NUM, &sg_version) &&
+        !bdrv_ioctl(bs, SG_GET_SCSI_ID, &scsiid)) {
+        DPRINTF("SG device found: type=%d, version=%d\n",
+            scsiid.scsi_type, sg_version);
+        return true;
+    }
+
+#endif
+
+    return false;
+}
+
 static int hdev_open(BlockDriverState *bs, QDict *options, int flags,
                      Error **errp)
 {
     BDRVRawState *s = bs->opaque;
     Error *local_err = NULL;
     int ret;
-    const char *filename = qdict_get_str(options, "filename");
 
 #if defined(__APPLE__) && defined(__MACH__)
-    if (strstart(filename, "/dev/cdrom", NULL)) {
-        kern_return_t kernResult;
-        io_iterator_t mediaIterator;
-        char bsdPath[ MAXPATHLEN ];
-        int fd;
+    const char *filename = qdict_get_str(options, "filename");
+    char bsd_path[MAXPATHLEN] = "";
+    bool error_occurred = false;
 
-        kernResult = FindEjectableCDMedia( &mediaIterator );
-        kernResult = GetBSDPath( mediaIterator, bsdPath, sizeof( bsdPath ) );
+    /* If using a real cdrom */
+    if (strcmp(filename, "/dev/cdrom") == 0) {
+        char *mediaType = NULL;
+        kern_return_t ret_val;
+        io_iterator_t mediaIterator = 0;
 
-        if ( bsdPath[ 0 ] != '\0' ) {
-            strcat(bsdPath,"s0");
-            /* some CDs don't have a partition 0 */
-            fd = qemu_open(bsdPath, O_RDONLY | O_BINARY | O_LARGEFILE);
-            if (fd < 0) {
-                bsdPath[strlen(bsdPath)-1] = '1';
-            } else {
-                qemu_close(fd);
-            }
-            filename = bsdPath;
-            qdict_put(options, "filename", qstring_from_str(filename));
+        mediaType = FindEjectableOpticalMedia(&mediaIterator);
+        if (mediaType == NULL) {
+            error_setg(errp, "Please make sure your CD/DVD is in the optical"
+                       " drive");
+            error_occurred = true;
+            goto hdev_open_Mac_error;
         }
 
-        if ( mediaIterator )
-            IOObjectRelease( mediaIterator );
+        ret_val = GetBSDPath(mediaIterator, bsd_path, sizeof(bsd_path), flags);
+        if (ret_val != KERN_SUCCESS) {
+            error_setg(errp, "Could not get BSD path for optical drive");
+            error_occurred = true;
+            goto hdev_open_Mac_error;
+        }
+
+        /* If a real optical drive was not found */
+        if (bsd_path[0] == '\0') {
+            error_setg(errp, "Failed to obtain bsd path for optical drive");
+            error_occurred = true;
+            goto hdev_open_Mac_error;
+        }
+
+        /* If using a cdrom disc and finding a partition on the disc failed */
+        if (strncmp(mediaType, kIOCDMediaClass, 9) == 0 &&
+            setup_cdrom(bsd_path, errp) == false) {
+            print_unmounting_directions(bsd_path);
+            error_occurred = true;
+            goto hdev_open_Mac_error;
+        }
+
+        qdict_put(options, "filename", qstring_from_str(bsd_path));
+
+hdev_open_Mac_error:
+        g_free(mediaType);
+        if (mediaIterator) {
+            IOObjectRelease(mediaIterator);
+        }
+        if (error_occurred) {
+            return -ENOENT;
+        }
     }
-#endif
+#endif /* defined(__APPLE__) && defined(__MACH__) */
 
     s->type = FTYPE_FILE;
-#if defined(__linux__)
-    {
-        char resolved_path[ MAXPATHLEN ], *temp;
-
-        temp = realpath(filename, resolved_path);
-        if (temp && strstart(temp, "/dev/sg", NULL)) {
-            bs->sg = 1;
-        }
-    }
-#endif
 
     ret = raw_open_common(bs, options, flags, 0, &local_err);
     if (ret < 0) {
-        if (local_err) {
-            error_propagate(errp, local_err);
+        error_propagate(errp, local_err);
+#if defined(__APPLE__) && defined(__MACH__)
+        if (*bsd_path) {
+            filename = bsd_path;
         }
+        /* if a physical device experienced an error while being opened */
+        if (strncmp(filename, "/dev/", 5) == 0) {
+            print_unmounting_directions(filename);
+        }
+#endif /* defined(__APPLE__) && defined(__MACH__) */
         return ret;
     }
 
+    /* Since this does ioctl the device must be already opened */
+    bs->sg = hdev_is_sg(bs);
+
     if (flags & BDRV_O_RDWR) {
         ret = check_hdev_writable(s);
         if (ret < 0) {
@@ -1910,61 +2169,6 @@
 }
 
 #if defined(__linux__)
-/* Note: we do not have a reliable method to detect if the floppy is
-   present. The current method is to try to open the floppy at every
-   I/O and to keep it opened during a few hundreds of ms. */
-static int fd_open(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    int last_media_present;
-
-    if (s->type != FTYPE_FD)
-        return 0;
-    last_media_present = (s->fd >= 0);
-    if (s->fd >= 0 &&
-        (get_clock() - s->fd_open_time) >= FD_OPEN_TIMEOUT) {
-        qemu_close(s->fd);
-        s->fd = -1;
-#ifdef DEBUG_FLOPPY
-        printf("Floppy closed\n");
-#endif
-    }
-    if (s->fd < 0) {
-        if (s->fd_got_error &&
-            (get_clock() - s->fd_error_time) < FD_OPEN_TIMEOUT) {
-#ifdef DEBUG_FLOPPY
-            printf("No floppy (open delayed)\n");
-#endif
-            return -EIO;
-        }
-        s->fd = qemu_open(bs->filename, s->open_flags & ~O_NONBLOCK);
-        if (s->fd < 0) {
-            s->fd_error_time = get_clock();
-            s->fd_got_error = 1;
-            if (last_media_present)
-                s->fd_media_changed = 1;
-#ifdef DEBUG_FLOPPY
-            printf("No floppy\n");
-#endif
-            return -EIO;
-        }
-#ifdef DEBUG_FLOPPY
-        printf("Floppy opened\n");
-#endif
-    }
-    if (!last_media_present)
-        s->fd_media_changed = 1;
-    s->fd_open_time = get_clock();
-    s->fd_got_error = 0;
-    return 0;
-}
-
-static int hdev_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    BDRVRawState *s = bs->opaque;
-
-    return ioctl(s->fd, req, buf);
-}
 
 static BlockAIOCB *hdev_aio_ioctl(BlockDriverState *bs,
         unsigned long int req, void *buf,
@@ -1977,7 +2181,7 @@
     if (fd_open(bs) < 0)
         return NULL;
 
-    acb = g_slice_new(RawPosixAIOData);
+    acb = g_new(RawPosixAIOData, 1);
     acb->bs = bs;
     acb->aio_type = QEMU_AIO_IOCTL;
     acb->aio_fildes = s->fd;
@@ -1987,8 +2191,8 @@
     pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
     return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
 }
+#endif /* linux */
 
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static int fd_open(BlockDriverState *bs)
 {
     BDRVRawState *s = bs->opaque;
@@ -1998,17 +2202,9 @@
         return 0;
     return -EIO;
 }
-#else /* !linux && !FreeBSD */
 
-static int fd_open(BlockDriverState *bs)
-{
-    return 0;
-}
-
-#endif /* !linux && !FreeBSD */
-
-static coroutine_fn BlockAIOCB *hdev_aio_discard(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors,
+static coroutine_fn BlockAIOCB *hdev_aio_pdiscard(BlockDriverState *bs,
+    int64_t offset, int count,
     BlockCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
@@ -2016,12 +2212,12 @@
     if (fd_open(bs) < 0) {
         return NULL;
     }
-    return paio_submit(bs, s->fd, sector_num, NULL, nb_sectors,
+    return paio_submit(bs, s->fd, offset, NULL, count,
                        cb, opaque, QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV);
 }
 
-static coroutine_fn int hdev_co_write_zeroes(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, BdrvRequestFlags flags)
+static coroutine_fn int hdev_co_pwrite_zeroes(BlockDriverState *bs,
+    int64_t offset, int count, BdrvRequestFlags flags)
 {
     BDRVRawState *s = bs->opaque;
     int rc;
@@ -2031,10 +2227,10 @@
         return rc;
     }
     if (!(flags & BDRV_REQ_MAY_UNMAP)) {
-        return paio_submit_co(bs, s->fd, sector_num, NULL, nb_sectors,
+        return paio_submit_co(bs, s->fd, offset, NULL, count,
                               QEMU_AIO_WRITE_ZEROES|QEMU_AIO_BLKDEV);
     } else if (s->discard_zeroes) {
-        return paio_submit_co(bs, s->fd, sector_num, NULL, nb_sectors,
+        return paio_submit_co(bs, s->fd, offset, NULL, count,
                               QEMU_AIO_DISCARD|QEMU_AIO_BLKDEV);
     }
     return -ENOTSUP;
@@ -2049,17 +2245,22 @@
     int64_t total_size = 0;
     bool has_prefix;
 
-    /* This function is used by all three protocol block drivers and therefore
-     * any of these three prefixes may be given.
+    /* This function is used by both protocol block drivers and therefore either
+     * of these prefixes may be given.
      * The return value has to be stored somewhere, otherwise this is an error
      * due to -Werror=unused-value. */
     has_prefix =
         strstart(filename, "host_device:", &filename) ||
-        strstart(filename, "host_cdrom:" , &filename) ||
-        strstart(filename, "host_floppy:", &filename);
+        strstart(filename, "host_cdrom:" , &filename);
 
     (void)has_prefix;
 
+    ret = raw_normalize_devicepath(&filename);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Could not normalize device path");
+        return ret;
+    }
+
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
                           BDRV_SECTOR_SIZE);
@@ -2101,181 +2302,30 @@
     .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create         = hdev_create,
     .create_opts         = &raw_create_opts,
-    .bdrv_co_write_zeroes = hdev_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes = hdev_co_pwrite_zeroes,
 
-    .bdrv_aio_readv	= raw_aio_readv,
-    .bdrv_aio_writev	= raw_aio_writev,
+    .bdrv_co_preadv         = raw_co_preadv,
+    .bdrv_co_pwritev        = raw_co_pwritev,
     .bdrv_aio_flush	= raw_aio_flush,
-    .bdrv_aio_discard   = hdev_aio_discard,
+    .bdrv_aio_pdiscard   = hdev_aio_pdiscard,
     .bdrv_refresh_limits = raw_refresh_limits,
     .bdrv_io_plug = raw_aio_plug,
     .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength	= raw_getlength,
     .bdrv_get_info = raw_get_info,
     .bdrv_get_allocated_file_size
                         = raw_get_allocated_file_size,
-
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
+    .bdrv_probe_blocksizes = hdev_probe_blocksizes,
+    .bdrv_probe_geometry = hdev_probe_geometry,
 
     /* generic scsi device */
 #ifdef __linux__
-    .bdrv_ioctl         = hdev_ioctl,
     .bdrv_aio_ioctl     = hdev_aio_ioctl,
 #endif
 };
 
-#ifdef __linux__
-static void floppy_parse_filename(const char *filename, QDict *options,
-                                  Error **errp)
-{
-    /* The prefix is optional, just as for "file". */
-    strstart(filename, "host_floppy:", &filename);
-
-    qdict_put_obj(options, "filename", QOBJECT(qstring_from_str(filename)));
-}
-
-static int floppy_open(BlockDriverState *bs, QDict *options, int flags,
-                       Error **errp)
-{
-    BDRVRawState *s = bs->opaque;
-    Error *local_err = NULL;
-    int ret;
-
-    s->type = FTYPE_FD;
-
-    /* open will not fail even if no floppy is inserted, so add O_NONBLOCK */
-    ret = raw_open_common(bs, options, flags, O_NONBLOCK, &local_err);
-    if (ret) {
-        if (local_err) {
-            error_propagate(errp, local_err);
-        }
-        return ret;
-    }
-
-    /* close fd so that we can reopen it as needed */
-    qemu_close(s->fd);
-    s->fd = -1;
-    s->fd_media_changed = 1;
-
-    return 0;
-}
-
-static int floppy_probe_device(const char *filename)
-{
-    int fd, ret;
-    int prio = 0;
-    struct floppy_struct fdparam;
-    struct stat st;
-
-    if (strstart(filename, "/dev/fd", NULL) &&
-        !strstart(filename, "/dev/fdset/", NULL)) {
-        prio = 50;
-    }
-
-    fd = qemu_open(filename, O_RDONLY | O_NONBLOCK);
-    if (fd < 0) {
-        goto out;
-    }
-    ret = fstat(fd, &st);
-    if (ret == -1 || !S_ISBLK(st.st_mode)) {
-        goto outc;
-    }
-
-    /* Attempt to detect via a floppy specific ioctl */
-    ret = ioctl(fd, FDGETPRM, &fdparam);
-    if (ret >= 0)
-        prio = 100;
-
-outc:
-    qemu_close(fd);
-out:
-    return prio;
-}
-
-
-static int floppy_is_inserted(BlockDriverState *bs)
-{
-    return fd_open(bs) >= 0;
-}
-
-static int floppy_media_changed(BlockDriverState *bs)
-{
-    BDRVRawState *s = bs->opaque;
-    int ret;
-
-    /*
-     * XXX: we do not have a true media changed indication.
-     * It does not work if the floppy is changed without trying to read it.
-     */
-    fd_open(bs);
-    ret = s->fd_media_changed;
-    s->fd_media_changed = 0;
-#ifdef DEBUG_FLOPPY
-    printf("Floppy changed=%d\n", ret);
-#endif
-    return ret;
-}
-
-static void floppy_eject(BlockDriverState *bs, bool eject_flag)
-{
-    BDRVRawState *s = bs->opaque;
-    int fd;
-
-    if (s->fd >= 0) {
-        qemu_close(s->fd);
-        s->fd = -1;
-    }
-    fd = qemu_open(bs->filename, s->open_flags | O_NONBLOCK);
-    if (fd >= 0) {
-        if (ioctl(fd, FDEJECT, 0) < 0)
-            perror("FDEJECT");
-        qemu_close(fd);
-    }
-}
-
-static BlockDriver bdrv_host_floppy = {
-    .format_name        = "host_floppy",
-    .protocol_name      = "host_floppy",
-    .instance_size      = sizeof(BDRVRawState),
-    .bdrv_needs_filename = true,
-    .bdrv_probe_device	= floppy_probe_device,
-    .bdrv_parse_filename = floppy_parse_filename,
-    .bdrv_file_open     = floppy_open,
-    .bdrv_close         = raw_close,
-    .bdrv_reopen_prepare = raw_reopen_prepare,
-    .bdrv_reopen_commit  = raw_reopen_commit,
-    .bdrv_reopen_abort   = raw_reopen_abort,
-    .bdrv_create         = hdev_create,
-    .create_opts         = &raw_create_opts,
-
-    .bdrv_aio_readv     = raw_aio_readv,
-    .bdrv_aio_writev    = raw_aio_writev,
-    .bdrv_aio_flush	= raw_aio_flush,
-    .bdrv_refresh_limits = raw_refresh_limits,
-    .bdrv_io_plug = raw_aio_plug,
-    .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
-
-    .bdrv_truncate      = raw_truncate,
-    .bdrv_getlength      = raw_getlength,
-    .has_variable_length = true,
-    .bdrv_get_allocated_file_size
-                        = raw_get_allocated_file_size,
-
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
-
-    /* removable device support */
-    .bdrv_is_inserted   = floppy_is_inserted,
-    .bdrv_media_changed = floppy_media_changed,
-    .bdrv_eject         = floppy_eject,
-};
-#endif
-
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static void cdrom_parse_filename(const char *filename, QDict *options,
                                  Error **errp)
@@ -2292,17 +2342,11 @@
                       Error **errp)
 {
     BDRVRawState *s = bs->opaque;
-    Error *local_err = NULL;
-    int ret;
 
     s->type = FTYPE_CD;
 
     /* open will not fail even if no CD is inserted, so add O_NONBLOCK */
-    ret = raw_open_common(bs, options, flags, O_NONBLOCK, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    return ret;
+    return raw_open_common(bs, options, flags, O_NONBLOCK, errp);
 }
 
 static int cdrom_probe_device(const char *filename)
@@ -2331,15 +2375,13 @@
     return prio;
 }
 
-static int cdrom_is_inserted(BlockDriverState *bs)
+static bool cdrom_is_inserted(BlockDriverState *bs)
 {
     BDRVRawState *s = bs->opaque;
     int ret;
 
     ret = ioctl(s->fd, CDROM_DRIVE_STATUS, CDSL_CURRENT);
-    if (ret == CDS_DISC_OK)
-        return 1;
-    return 0;
+    return ret == CDS_DISC_OK;
 }
 
 static void cdrom_eject(BlockDriverState *bs, bool eject_flag)
@@ -2383,13 +2425,13 @@
     .bdrv_create         = hdev_create,
     .create_opts         = &raw_create_opts,
 
-    .bdrv_aio_readv     = raw_aio_readv,
-    .bdrv_aio_writev    = raw_aio_writev,
+
+    .bdrv_co_preadv         = raw_co_preadv,
+    .bdrv_co_pwritev        = raw_co_pwritev,
     .bdrv_aio_flush	= raw_aio_flush,
     .bdrv_refresh_limits = raw_refresh_limits,
     .bdrv_io_plug = raw_aio_plug,
     .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength      = raw_getlength,
@@ -2397,16 +2439,12 @@
     .bdrv_get_allocated_file_size
                         = raw_get_allocated_file_size,
 
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
-
     /* removable device support */
     .bdrv_is_inserted   = cdrom_is_inserted,
     .bdrv_eject         = cdrom_eject,
     .bdrv_lock_medium   = cdrom_lock_medium,
 
     /* generic scsi device */
-    .bdrv_ioctl         = hdev_ioctl,
     .bdrv_aio_ioctl     = hdev_aio_ioctl,
 };
 #endif /* __linux__ */
@@ -2423,9 +2461,7 @@
 
     ret = raw_open_common(bs, options, flags, 0, &local_err);
     if (ret) {
-        if (local_err) {
-            error_propagate(errp, local_err);
-        }
+        error_propagate(errp, local_err);
         return ret;
     }
 
@@ -2465,7 +2501,7 @@
     return 0;
 }
 
-static int cdrom_is_inserted(BlockDriverState *bs)
+static bool cdrom_is_inserted(BlockDriverState *bs)
 {
     return raw_getlength(bs) > 0;
 }
@@ -2520,13 +2556,12 @@
     .bdrv_create        = hdev_create,
     .create_opts        = &raw_create_opts,
 
-    .bdrv_aio_readv     = raw_aio_readv,
-    .bdrv_aio_writev    = raw_aio_writev,
+    .bdrv_co_preadv         = raw_co_preadv,
+    .bdrv_co_pwritev        = raw_co_pwritev,
     .bdrv_aio_flush	= raw_aio_flush,
     .bdrv_refresh_limits = raw_refresh_limits,
     .bdrv_io_plug = raw_aio_plug,
     .bdrv_io_unplug = raw_aio_unplug,
-    .bdrv_flush_io_queue = raw_aio_flush_io_queue,
 
     .bdrv_truncate      = raw_truncate,
     .bdrv_getlength      = raw_getlength,
@@ -2534,9 +2569,6 @@
     .bdrv_get_allocated_file_size
                         = raw_get_allocated_file_size,
 
-    .bdrv_detach_aio_context = raw_detach_aio_context,
-    .bdrv_attach_aio_context = raw_attach_aio_context,
-
     /* removable device support */
     .bdrv_is_inserted   = cdrom_is_inserted,
     .bdrv_eject         = cdrom_eject,
@@ -2553,7 +2585,6 @@
     bdrv_register(&bdrv_file);
     bdrv_register(&bdrv_host_device);
 #ifdef __linux__
-    bdrv_register(&bdrv_host_floppy);
     bdrv_register(&bdrv_host_cdrom);
 #endif
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
diff --git a/block/raw-win32.c b/block/raw-win32.c
index bf264d4..b445a2c 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -21,19 +21,19 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/timer.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
-#include "raw-aio.h"
+#include "block/raw-aio.h"
 #include "trace.h"
 #include "block/thread-pool.h"
 #include "qemu/iov.h"
+#include "qapi/qmp/qstring.h"
 #include <windows.h>
 #include <winioctl.h>
-#ifdef CONFIG_ANDROID
-#include "android/utils/win32_unicode.h"
-#endif
 
 #define FTYPE_FILE 0
 #define FTYPE_CD     1
@@ -104,7 +104,7 @@
     switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
     case QEMU_AIO_READ:
         count = handle_aiocb_rw(aiocb);
-        if (count < aiocb->aio_nbytes && aiocb->bs->growable) {
+        if (count < aiocb->aio_nbytes) {
             /* A short read means that we have reached EOF. Pad the buffer
              * with zeros for bytes after EOF. */
             iov_memset(aiocb->aio_iov, aiocb->aio_niov, count,
@@ -121,9 +121,9 @@
     case QEMU_AIO_WRITE:
         count = handle_aiocb_rw(aiocb);
         if (count == aiocb->aio_nbytes) {
-            count = 0;
+            ret = 0;
         } else {
-            count = -EINVAL;
+            ret = -EINVAL;
         }
         break;
     case QEMU_AIO_FLUSH:
@@ -137,15 +137,15 @@
         break;
     }
 
-    g_slice_free(RawWin32AIOData, aiocb);
+    g_free(aiocb);
     return ret;
 }
 
 static BlockAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        int64_t offset, QEMUIOVector *qiov, int count,
         BlockCompletionFunc *cb, void *opaque, int type)
 {
-    RawWin32AIOData *acb = g_slice_new(RawWin32AIOData);
+    RawWin32AIOData *acb = g_new(RawWin32AIOData, 1);
     ThreadPool *pool;
 
     acb->bs = bs;
@@ -155,11 +155,12 @@
     if (qiov) {
         acb->aio_iov = qiov->iov;
         acb->aio_niov = qiov->niov;
+        assert(qiov->size == count);
     }
-    acb->aio_nbytes = nb_sectors * 512;
-    acb->aio_offset = sector_num * 512;
+    acb->aio_nbytes = count;
+    acb->aio_offset = offset;
 
-    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
+    trace_paio_submit(acb, opaque, offset, count, type);
     pool = aio_get_thread_pool(bdrv_get_aio_context(bs));
     return thread_pool_submit_aio(pool, aio_worker, acb, cb, opaque);
 }
@@ -222,7 +223,7 @@
     }
 }
 
-static void raw_probe_alignment(BlockDriverState *bs)
+static void raw_probe_alignment(BlockDriverState *bs, Error **errp)
 {
     BDRVRawState *s = bs->opaque;
     DWORD sectorsPerCluster, freeClusters, totalClusters, count;
@@ -230,14 +231,14 @@
     BOOL status;
 
     if (s->type == FTYPE_CD) {
-        bs->request_alignment = 2048;
+        bs->bl.request_alignment = 2048;
         return;
     }
     if (s->type == FTYPE_HARDDISK) {
         status = DeviceIoControl(s->hfile, IOCTL_DISK_GET_DRIVE_GEOMETRY_EX,
                                  NULL, 0, &dg, sizeof(dg), &count, NULL);
         if (status != 0) {
-            bs->request_alignment = dg.Geometry.BytesPerSector;
+            bs->bl.request_alignment = dg.Geometry.BytesPerSector;
             return;
         }
         /* try GetDiskFreeSpace too */
@@ -247,7 +248,7 @@
         GetDiskFreeSpace(s->drive_path, &sectorsPerCluster,
                          &dg.Geometry.BytesPerSector,
                          &freeClusters, &totalClusters);
-        bs->request_alignment = dg.Geometry.BytesPerSector;
+        bs->bl.request_alignment = dg.Geometry.BytesPerSector;
     }
 }
 
@@ -327,26 +328,13 @@
     } else {
         /* Relative path.  */
         char buf[MAX_PATH];
-        GetCurrentDirectory(MAX_PATH, buf);
+        win32GetCurrentDirectory(MAX_PATH, buf);
         snprintf(s->drive_path, sizeof(s->drive_path), "%c:\\", buf[0]);
     }
 
-#ifdef CONFIG_ANDROID
-    wchar_t* wideName = win32_utf8_to_utf16_str(filename);
-    if (wideName == NULL) {
-        ret = -EINVAL;
-        goto fail;
-    }
-
-    s->hfile = CreateFileW(wideName, access_flags,
-                           FILE_SHARE_READ, NULL,
-                           OPEN_EXISTING, overlapped, NULL);
-    free(wideName);
-#else
-    s->hfile = CreateFile(filename, access_flags,
+    s->hfile = win32CreateFile(filename, access_flags,
                           FILE_SHARE_READ, NULL,
                           OPEN_EXISTING, overlapped, NULL);
-#endif
     if (s->hfile == INVALID_HANDLE_VALUE) {
         int err = GetLastError();
 
@@ -378,7 +366,6 @@
         win32_aio_attach_aio_context(s->aio, bdrv_get_aio_context(bs));
     }
 
-    raw_probe_alignment(bs);
     ret = 0;
 fail:
     qemu_opts_del(opts);
@@ -392,9 +379,10 @@
     BDRVRawState *s = bs->opaque;
     if (s->aio) {
         return win32_aio_submit(bs, s->aio, s->hfile, sector_num, qiov,
-                                nb_sectors, cb, opaque, QEMU_AIO_READ); 
+                                nb_sectors, cb, opaque, QEMU_AIO_READ);
     } else {
-        return paio_submit(bs, s->hfile, sector_num, qiov, nb_sectors,
+        return paio_submit(bs, s->hfile, sector_num << BDRV_SECTOR_BITS, qiov,
+                           nb_sectors << BDRV_SECTOR_BITS,
                            cb, opaque, QEMU_AIO_READ);
     }
 }
@@ -406,9 +394,10 @@
     BDRVRawState *s = bs->opaque;
     if (s->aio) {
         return win32_aio_submit(bs, s->aio, s->hfile, sector_num, qiov,
-                                nb_sectors, cb, opaque, QEMU_AIO_WRITE); 
+                                nb_sectors, cb, opaque, QEMU_AIO_WRITE);
     } else {
-        return paio_submit(bs, s->hfile, sector_num, qiov, nb_sectors,
+        return paio_submit(bs, s->hfile, sector_num << BDRV_SECTOR_BITS, qiov,
+                           nb_sectors << BDRV_SECTOR_BITS,
                            cb, opaque, QEMU_AIO_WRITE);
     }
 }
@@ -556,13 +545,14 @@
     }
 };
 
-static BlockDriver bdrv_file = {
+BlockDriver bdrv_file = {
     .format_name	= "file",
     .protocol_name	= "file",
     .instance_size	= sizeof(BDRVRawState),
     .bdrv_needs_filename = true,
     .bdrv_parse_filename = raw_parse_filename,
     .bdrv_file_open     = raw_open,
+    .bdrv_refresh_limits = raw_probe_alignment,
     .bdrv_close         = raw_close,
     .bdrv_create        = raw_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -691,7 +681,7 @@
 
     create_flags = OPEN_EXISTING;
 
-    s->hfile = CreateFile(filename, access_flags,
+    s->hfile = win32CreateFile(filename, access_flags,
                           FILE_SHARE_READ, NULL,
                           create_flags, overlapped, NULL);
     if (s->hfile == INVALID_HANDLE_VALUE) {
diff --git a/block/raw_bsd.c b/block/raw_bsd.c
index 401b967..588d408 100644
--- a/block/raw_bsd.c
+++ b/block/raw_bsd.c
@@ -1,6 +1,6 @@
 /* BlockDriver implementation for "raw"
  *
- * Copyright (C) 2010, 2013, Red Hat, Inc.
+ * Copyright (C) 2010-2016 Red Hat, Inc.
  * Copyright (C) 2010, Blue Swirl <blauwirbel@gmail.com>
  * Copyright (C) 2009, Anthony Liguori <aliguori@us.ibm.com>
  *
@@ -26,7 +26,9 @@
  * IN THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/block_int.h"
+#include "qapi/error.h"
 #include "qemu/option.h"
 
 static QemuOptsList raw_create_opts = {
@@ -48,85 +50,130 @@
     return 0;
 }
 
-static int coroutine_fn raw_co_readv(BlockDriverState *bs, int64_t sector_num,
-                                     int nb_sectors, QEMUIOVector *qiov)
+static int coroutine_fn raw_co_preadv(BlockDriverState *bs, uint64_t offset,
+                                      uint64_t bytes, QEMUIOVector *qiov,
+                                      int flags)
 {
     BLKDBG_EVENT(bs->file, BLKDBG_READ_AIO);
-    return bdrv_co_readv(bs->file, sector_num, nb_sectors, qiov);
+    return bdrv_co_preadv(bs->file, offset, bytes, qiov, flags);
 }
 
-static int coroutine_fn raw_co_writev(BlockDriverState *bs, int64_t sector_num,
-                                      int nb_sectors, QEMUIOVector *qiov)
+static int coroutine_fn raw_co_pwritev(BlockDriverState *bs, uint64_t offset,
+                                       uint64_t bytes, QEMUIOVector *qiov,
+                                       int flags)
 {
+    void *buf = NULL;
+    BlockDriver *drv;
+    QEMUIOVector local_qiov;
+    int ret;
+
+    if (bs->probed && offset < BLOCK_PROBE_BUF_SIZE && bytes) {
+        /* Handling partial writes would be a pain - so we just
+         * require that guests have 512-byte request alignment if
+         * probing occurred */
+        QEMU_BUILD_BUG_ON(BLOCK_PROBE_BUF_SIZE != 512);
+        QEMU_BUILD_BUG_ON(BDRV_SECTOR_SIZE != 512);
+        assert(offset == 0 && bytes >= BLOCK_PROBE_BUF_SIZE);
+
+        buf = qemu_try_blockalign(bs->file->bs, 512);
+        if (!buf) {
+            ret = -ENOMEM;
+            goto fail;
+        }
+
+        ret = qemu_iovec_to_buf(qiov, 0, buf, 512);
+        if (ret != 512) {
+            ret = -EINVAL;
+            goto fail;
+        }
+
+        drv = bdrv_probe_all(buf, 512, NULL);
+        if (drv != bs->drv) {
+            ret = -EPERM;
+            goto fail;
+        }
+
+        /* Use the checked buffer, a malicious guest might be overwriting its
+         * original buffer in the background. */
+        qemu_iovec_init(&local_qiov, qiov->niov + 1);
+        qemu_iovec_add(&local_qiov, buf, 512);
+        qemu_iovec_concat(&local_qiov, qiov, 512, qiov->size - 512);
+        qiov = &local_qiov;
+    }
+
     BLKDBG_EVENT(bs->file, BLKDBG_WRITE_AIO);
-    return bdrv_co_writev(bs->file, sector_num, nb_sectors, qiov);
+    ret = bdrv_co_pwritev(bs->file, offset, bytes, qiov, flags);
+
+fail:
+    if (qiov == &local_qiov) {
+        qemu_iovec_destroy(&local_qiov);
+    }
+    qemu_vfree(buf);
+    return ret;
 }
 
 static int64_t coroutine_fn raw_co_get_block_status(BlockDriverState *bs,
                                             int64_t sector_num,
-                                            int nb_sectors, int *pnum)
+                                            int nb_sectors, int *pnum,
+                                            BlockDriverState **file)
 {
     *pnum = nb_sectors;
+    *file = bs->file->bs;
     return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | BDRV_BLOCK_DATA |
            (sector_num << BDRV_SECTOR_BITS);
 }
 
-static int coroutine_fn raw_co_write_zeroes(BlockDriverState *bs,
-                                            int64_t sector_num, int nb_sectors,
-                                            BdrvRequestFlags flags)
+static int coroutine_fn raw_co_pwrite_zeroes(BlockDriverState *bs,
+                                             int64_t offset, int count,
+                                             BdrvRequestFlags flags)
 {
-    return bdrv_co_write_zeroes(bs->file, sector_num, nb_sectors, flags);
+    return bdrv_co_pwrite_zeroes(bs->file, offset, count, flags);
 }
 
-static int coroutine_fn raw_co_discard(BlockDriverState *bs,
-                                       int64_t sector_num, int nb_sectors)
+static int coroutine_fn raw_co_pdiscard(BlockDriverState *bs,
+                                        int64_t offset, int count)
 {
-    return bdrv_co_discard(bs->file, sector_num, nb_sectors);
+    return bdrv_co_pdiscard(bs->file->bs, offset, count);
 }
 
 static int64_t raw_getlength(BlockDriverState *bs)
 {
-    return bdrv_getlength(bs->file);
+    return bdrv_getlength(bs->file->bs);
 }
 
 static int raw_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
-    return bdrv_get_info(bs->file, bdi);
+    return bdrv_get_info(bs->file->bs, bdi);
 }
 
 static void raw_refresh_limits(BlockDriverState *bs, Error **errp)
 {
-    bs->bl = bs->file->bl;
+    if (bs->probed) {
+        /* To make it easier to protect the first sector, any probed
+         * image is restricted to read-modify-write on sub-sector
+         * operations. */
+        bs->bl.request_alignment = BDRV_SECTOR_SIZE;
+    }
 }
 
 static int raw_truncate(BlockDriverState *bs, int64_t offset)
 {
-    return bdrv_truncate(bs->file, offset);
-}
-
-static int raw_is_inserted(BlockDriverState *bs)
-{
-    return bdrv_is_inserted(bs->file);
+    return bdrv_truncate(bs->file->bs, offset);
 }
 
 static int raw_media_changed(BlockDriverState *bs)
 {
-    return bdrv_media_changed(bs->file);
+    return bdrv_media_changed(bs->file->bs);
 }
 
 static void raw_eject(BlockDriverState *bs, bool eject_flag)
 {
-    bdrv_eject(bs->file, eject_flag);
+    bdrv_eject(bs->file->bs, eject_flag);
 }
 
 static void raw_lock_medium(BlockDriverState *bs, bool locked)
 {
-    bdrv_lock_medium(bs->file, locked);
-}
-
-static int raw_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
-{
-    return bdrv_ioctl(bs->file, req, buf);
+    bdrv_lock_medium(bs->file->bs, locked);
 }
 
 static BlockAIOCB *raw_aio_ioctl(BlockDriverState *bs,
@@ -134,30 +181,39 @@
                                  BlockCompletionFunc *cb,
                                  void *opaque)
 {
-    return bdrv_aio_ioctl(bs->file, req, buf, cb, opaque);
+    return bdrv_aio_ioctl(bs->file->bs, req, buf, cb, opaque);
 }
 
 static int raw_has_zero_init(BlockDriverState *bs)
 {
-    return bdrv_has_zero_init(bs->file);
+    return bdrv_has_zero_init(bs->file->bs);
 }
 
 static int raw_create(const char *filename, QemuOpts *opts, Error **errp)
 {
-    Error *local_err = NULL;
-    int ret;
-
-    ret = bdrv_create_file(filename, opts, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    return ret;
+    return bdrv_create_file(filename, opts, errp);
 }
 
 static int raw_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
-    bs->sg = bs->file->sg;
+    bs->sg = bs->file->bs->sg;
+    bs->supported_write_flags = BDRV_REQ_FUA &
+        bs->file->bs->supported_write_flags;
+    bs->supported_zero_flags = (BDRV_REQ_FUA | BDRV_REQ_MAY_UNMAP) &
+        bs->file->bs->supported_zero_flags;
+
+    if (bs->probed && !bdrv_is_read_only(bs)) {
+        fprintf(stderr,
+                "WARNING: Image format was not specified for '%s' and probing "
+                "guessed raw.\n"
+                "         Automatically detecting the format is dangerous for "
+                "raw images, write operations on block 0 will be restricted.\n"
+                "         Specify the 'raw' format explicitly to remove the "
+                "restrictions.\n",
+                bs->file->bs->filename);
+    }
+
     return 0;
 }
 
@@ -173,28 +229,38 @@
     return 1;
 }
 
-static BlockDriver bdrv_raw = {
+static int raw_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz)
+{
+    return bdrv_probe_blocksizes(bs->file->bs, bsz);
+}
+
+static int raw_probe_geometry(BlockDriverState *bs, HDGeometry *geo)
+{
+    return bdrv_probe_geometry(bs->file->bs, geo);
+}
+
+BlockDriver bdrv_raw = {
     .format_name          = "raw",
     .bdrv_probe           = &raw_probe,
     .bdrv_reopen_prepare  = &raw_reopen_prepare,
     .bdrv_open            = &raw_open,
     .bdrv_close           = &raw_close,
     .bdrv_create          = &raw_create,
-    .bdrv_co_readv        = &raw_co_readv,
-    .bdrv_co_writev       = &raw_co_writev,
-    .bdrv_co_write_zeroes = &raw_co_write_zeroes,
-    .bdrv_co_discard      = &raw_co_discard,
+    .bdrv_co_preadv       = &raw_co_preadv,
+    .bdrv_co_pwritev      = &raw_co_pwritev,
+    .bdrv_co_pwrite_zeroes = &raw_co_pwrite_zeroes,
+    .bdrv_co_pdiscard     = &raw_co_pdiscard,
     .bdrv_co_get_block_status = &raw_co_get_block_status,
     .bdrv_truncate        = &raw_truncate,
     .bdrv_getlength       = &raw_getlength,
     .has_variable_length  = true,
     .bdrv_get_info        = &raw_get_info,
     .bdrv_refresh_limits  = &raw_refresh_limits,
-    .bdrv_is_inserted     = &raw_is_inserted,
+    .bdrv_probe_blocksizes = &raw_probe_blocksizes,
+    .bdrv_probe_geometry  = &raw_probe_geometry,
     .bdrv_media_changed   = &raw_media_changed,
     .bdrv_eject           = &raw_eject,
     .bdrv_lock_medium     = &raw_lock_medium,
-    .bdrv_ioctl           = &raw_ioctl,
     .bdrv_aio_ioctl       = &raw_aio_ioctl,
     .create_opts          = &raw_create_opts,
     .bdrv_has_zero_init   = &raw_has_zero_init
diff --git a/block/rbd.c b/block/rbd.c
index 5b5a64a..0106fea 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -11,11 +11,13 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
-#include "qemu-common.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "block/block_int.h"
+#include "crypto/secret.h"
+#include "qemu/cutils.h"
 
 #include <rbd/librbd.h>
 
@@ -74,25 +76,18 @@
     QEMUIOVector *qiov;
     char *bounce;
     RBDAIOCmd cmd;
-    int64_t sector_num;
     int error;
     struct BDRVRBDState *s;
-    int status;
 } RBDAIOCB;
 
 typedef struct RADOSCB {
-    int rcbid;
     RBDAIOCB *acb;
     struct BDRVRBDState *s;
-    int done;
     int64_t size;
     char *buf;
     int64_t ret;
 } RADOSCB;
 
-#define RBD_FD_READ 0
-#define RBD_FD_WRITE 1
-
 typedef struct BDRVRBDState {
     rados_t cluster;
     rados_ioctx_t io_ctx;
@@ -235,7 +230,30 @@
     return NULL;
 }
 
-static int qemu_rbd_set_conf(rados_t cluster, const char *conf, Error **errp)
+
+static int qemu_rbd_set_auth(rados_t cluster, const char *secretid,
+                             Error **errp)
+{
+    if (secretid == 0) {
+        return 0;
+    }
+
+    gchar *secret = qcrypto_secret_lookup_as_base64(secretid,
+                                                    errp);
+    if (!secret) {
+        return -1;
+    }
+
+    rados_conf_set(cluster, "key", secret);
+    g_free(secret);
+
+    return 0;
+}
+
+
+static int qemu_rbd_set_conf(rados_t cluster, const char *conf,
+                             bool only_read_conf_file,
+                             Error **errp)
 {
     char *p, *buf;
     char name[RBD_MAX_CONF_NAME_SIZE];
@@ -267,17 +285,22 @@
         qemu_rbd_unescape(value);
 
         if (strcmp(name, "conf") == 0) {
-            ret = rados_conf_read_file(cluster, value);
-            if (ret < 0) {
-                error_setg(errp, "error reading conf file %s", value);
-                break;
+            /* read the conf file alone, so it doesn't override more
+               specific settings for a particular device */
+            if (only_read_conf_file) {
+                ret = rados_conf_read_file(cluster, value);
+                if (ret < 0) {
+                    error_setg_errno(errp, -ret, "error reading conf file %s",
+                                     value);
+                    break;
+                }
             }
         } else if (strcmp(name, "id") == 0) {
             /* ignore, this is parsed by qemu_rbd_parse_clientname() */
-        } else {
+        } else if (!only_read_conf_file) {
             ret = rados_conf_set(cluster, name, value);
             if (ret < 0) {
-                error_setg(errp, "invalid conf option %s", name);
+                error_setg_errno(errp, -ret, "invalid conf option %s", name);
                 ret = -EINVAL;
                 break;
             }
@@ -300,10 +323,13 @@
     char conf[RBD_MAX_CONF_SIZE];
     char clientname_buf[RBD_MAX_CONF_SIZE];
     char *clientname;
+    const char *secretid;
     rados_t cluster;
     rados_ioctx_t io_ctx;
     int ret;
 
+    secretid = qemu_opt_get(opts, "password-secret");
+
     if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                            snap_buf, sizeof(snap_buf),
                            name, sizeof(name),
@@ -325,42 +351,59 @@
             error_setg(errp, "obj size too small");
             return -EINVAL;
         }
-        obj_order = ffs(objsize) - 1;
+        obj_order = ctz32(objsize);
     }
 
     clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
-    if (rados_create(&cluster, clientname) < 0) {
-        error_setg(errp, "error initializing");
-        return -EIO;
+    ret = rados_create(&cluster, clientname);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "error initializing");
+        return ret;
     }
 
     if (strstr(conf, "conf=") == NULL) {
         /* try default location, but ignore failure */
         rados_conf_read_file(cluster, NULL);
-    }
-
-    if (conf[0] != '\0' &&
-        qemu_rbd_set_conf(cluster, conf, &local_err) < 0) {
+    } else if (conf[0] != '\0' &&
+               qemu_rbd_set_conf(cluster, conf, true, &local_err) < 0) {
         rados_shutdown(cluster);
         error_propagate(errp, local_err);
         return -EIO;
     }
 
-    if (rados_connect(cluster) < 0) {
-        error_setg(errp, "error connecting");
+    if (conf[0] != '\0' &&
+        qemu_rbd_set_conf(cluster, conf, false, &local_err) < 0) {
+        rados_shutdown(cluster);
+        error_propagate(errp, local_err);
+        return -EIO;
+    }
+
+    if (qemu_rbd_set_auth(cluster, secretid, errp) < 0) {
         rados_shutdown(cluster);
         return -EIO;
     }
 
-    if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
-        error_setg(errp, "error opening pool %s", pool);
+    ret = rados_connect(cluster);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "error connecting");
         rados_shutdown(cluster);
-        return -EIO;
+        return ret;
+    }
+
+    ret = rados_ioctx_create(cluster, pool, &io_ctx);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "error opening pool %s", pool);
+        rados_shutdown(cluster);
+        return ret;
     }
 
     ret = rbd_create(io_ctx, name, bytes, &obj_order);
     rados_ioctx_destroy(io_ctx);
     rados_shutdown(cluster);
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "error rbd create");
+        return ret;
+    }
 
     return ret;
 }
@@ -405,7 +448,6 @@
     }
     qemu_vfree(acb->bounce);
     acb->common.cb(acb->common.opaque, (acb->ret > 0 ? 0 : acb->ret));
-    acb->status = 0;
 
     qemu_aio_unref(acb);
 }
@@ -420,6 +462,11 @@
             .type = QEMU_OPT_STRING,
             .help = "Specification of the rbd image",
         },
+        {
+            .name = "password-secret",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret providing the password",
+        },
         { /* end of list */ }
     },
 };
@@ -433,6 +480,7 @@
     char conf[RBD_MAX_CONF_SIZE];
     char clientname_buf[RBD_MAX_CONF_SIZE];
     char *clientname;
+    const char *secretid;
     QemuOpts *opts;
     Error *local_err = NULL;
     const char *filename;
@@ -447,6 +495,7 @@
     }
 
     filename = qemu_opt_get(opts, "filename");
+    secretid = qemu_opt_get(opts, "password-secret");
 
     if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                            snap_buf, sizeof(snap_buf),
@@ -459,7 +508,7 @@
     clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
     r = rados_create(&s->cluster, clientname);
     if (r < 0) {
-        error_setg(&local_err, "error initializing");
+        error_setg_errno(errp, -r, "error initializing");
         goto failed_opts;
     }
 
@@ -468,6 +517,28 @@
         s->snap = g_strdup(snap_buf);
     }
 
+    if (strstr(conf, "conf=") == NULL) {
+        /* try default location, but ignore failure */
+        rados_conf_read_file(s->cluster, NULL);
+    } else if (conf[0] != '\0') {
+        r = qemu_rbd_set_conf(s->cluster, conf, true, errp);
+        if (r < 0) {
+            goto failed_shutdown;
+        }
+    }
+
+    if (conf[0] != '\0') {
+        r = qemu_rbd_set_conf(s->cluster, conf, false, errp);
+        if (r < 0) {
+            goto failed_shutdown;
+        }
+    }
+
+    if (qemu_rbd_set_auth(s->cluster, secretid, errp) < 0) {
+        r = -EIO;
+        goto failed_shutdown;
+    }
+
     /*
      * Fallback to more conservative semantics if setting cache
      * options fails. Ignore errors from setting rbd_cache because the
@@ -481,33 +552,21 @@
         rados_conf_set(s->cluster, "rbd_cache", "true");
     }
 
-    if (strstr(conf, "conf=") == NULL) {
-        /* try default location, but ignore failure */
-        rados_conf_read_file(s->cluster, NULL);
-    }
-
-    if (conf[0] != '\0') {
-        r = qemu_rbd_set_conf(s->cluster, conf, errp);
-        if (r < 0) {
-            goto failed_shutdown;
-        }
-    }
-
     r = rados_connect(s->cluster);
     if (r < 0) {
-        error_setg(&local_err, "error connecting");
+        error_setg_errno(errp, -r, "error connecting");
         goto failed_shutdown;
     }
 
     r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
     if (r < 0) {
-        error_setg(&local_err, "error opening pool %s", pool);
+        error_setg_errno(errp, -r, "error opening pool %s", pool);
         goto failed_shutdown;
     }
 
     r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
     if (r < 0) {
-        error_setg(&local_err, "error reading header from %s", s->name);
+        error_setg_errno(errp, -r, "error reading header from %s", s->name);
         goto failed_open;
     }
 
@@ -590,9 +649,9 @@
 }
 
 static BlockAIOCB *rbd_start_aio(BlockDriverState *bs,
-                                 int64_t sector_num,
+                                 int64_t off,
                                  QEMUIOVector *qiov,
-                                 int nb_sectors,
+                                 int64_t size,
                                  BlockCompletionFunc *cb,
                                  void *opaque,
                                  RBDAIOCmd cmd)
@@ -600,7 +659,6 @@
     RBDAIOCB *acb;
     RADOSCB *rcb = NULL;
     rbd_completion_t c;
-    int64_t off, size;
     char *buf;
     int r;
 
@@ -609,6 +667,7 @@
     acb = qemu_aio_get(&rbd_aiocb_info, bs, cb, opaque);
     acb->cmd = cmd;
     acb->qiov = qiov;
+    assert(!qiov || qiov->size == size);
     if (cmd == RBD_AIO_DISCARD || cmd == RBD_AIO_FLUSH) {
         acb->bounce = NULL;
     } else {
@@ -621,7 +680,6 @@
     acb->error = 0;
     acb->s = s;
     acb->bh = NULL;
-    acb->status = -EINPROGRESS;
 
     if (cmd == RBD_AIO_WRITE) {
         qemu_iovec_to_buf(acb->qiov, 0, acb->bounce, qiov->size);
@@ -629,11 +687,7 @@
 
     buf = acb->bounce;
 
-    off = sector_num * BDRV_SECTOR_SIZE;
-    size = nb_sectors * BDRV_SECTOR_SIZE;
-
     rcb = g_new(RADOSCB, 1);
-    rcb->done = 0;
     rcb->acb = acb;
     rcb->buf = buf;
     rcb->s = acb->s;
@@ -682,7 +736,8 @@
                                       BlockCompletionFunc *cb,
                                       void *opaque)
 {
-    return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque,
+    return rbd_start_aio(bs, sector_num << BDRV_SECTOR_BITS, qiov,
+                         nb_sectors << BDRV_SECTOR_BITS, cb, opaque,
                          RBD_AIO_READ);
 }
 
@@ -693,7 +748,8 @@
                                        BlockCompletionFunc *cb,
                                        void *opaque)
 {
-    return rbd_start_aio(bs, sector_num, qiov, nb_sectors, cb, opaque,
+    return rbd_start_aio(bs, sector_num << BDRV_SECTOR_BITS, qiov,
+                         nb_sectors << BDRV_SECTOR_BITS, cb, opaque,
                          RBD_AIO_WRITE);
 }
 
@@ -826,10 +882,8 @@
                                   const char *snapshot_name)
 {
     BDRVRBDState *s = bs->opaque;
-    int r;
 
-    r = rbd_snap_rollback(s->image, snapshot_name);
-    return r;
+    return rbd_snap_rollback(s->image, snapshot_name);
 }
 
 static int qemu_rbd_snap_list(BlockDriverState *bs,
@@ -876,13 +930,13 @@
 }
 
 #ifdef LIBRBD_SUPPORTS_DISCARD
-static BlockAIOCB* qemu_rbd_aio_discard(BlockDriverState *bs,
-                                        int64_t sector_num,
-                                        int nb_sectors,
-                                        BlockCompletionFunc *cb,
-                                        void *opaque)
+static BlockAIOCB *qemu_rbd_aio_pdiscard(BlockDriverState *bs,
+                                         int64_t offset,
+                                         int count,
+                                         BlockCompletionFunc *cb,
+                                         void *opaque)
 {
-    return rbd_start_aio(bs, sector_num, NULL, nb_sectors, cb, opaque,
+    return rbd_start_aio(bs, offset, NULL, count, cb, opaque,
                          RBD_AIO_DISCARD);
 }
 #endif
@@ -913,6 +967,11 @@
             .type = QEMU_OPT_SIZE,
             .help = "RBD object size"
         },
+        {
+            .name = "password-secret",
+            .type = QEMU_OPT_STRING,
+            .help = "ID of secret providing the password",
+        },
         { /* end of list */ }
     }
 };
@@ -941,7 +1000,7 @@
 #endif
 
 #ifdef LIBRBD_SUPPORTS_DISCARD
-    .bdrv_aio_discard       = qemu_rbd_aio_discard,
+    .bdrv_aio_pdiscard      = qemu_rbd_aio_pdiscard,
 #endif
 
     .bdrv_snapshot_create   = qemu_rbd_snap_create,
diff --git a/block/sheepdog.c b/block/sheepdog.c
index be3176f..66e1cb2 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -12,12 +12,15 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/uri.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/bitops.h"
+#include "qemu/cutils.h"
 
 #define SD_PROTO_VER 0x01
 
@@ -28,7 +31,6 @@
 #define SD_OP_READ_OBJ       0x02
 #define SD_OP_WRITE_OBJ      0x03
 /* 0x04 is used internally by Sheepdog */
-#define SD_OP_DISCARD_OBJ    0x05
 
 #define SD_OP_NEW_VDI        0x11
 #define SD_OP_LOCK_VDI       0x12
@@ -37,6 +39,7 @@
 #define SD_OP_READ_VDIS      0x15
 #define SD_OP_FLUSH_VDI      0x16
 #define SD_OP_DEL_VDI        0x17
+#define SD_OP_GET_CLUSTER_DEFAULT   0x18
 
 #define SD_FLAG_CMD_WRITE    0x01
 #define SD_FLAG_CMD_COW      0x02
@@ -91,6 +94,7 @@
 #define SD_NR_VDIS   (1U << 24)
 #define SD_DATA_OBJ_SIZE (UINT64_C(1) << 22)
 #define SD_MAX_VDI_SIZE (SD_DATA_OBJ_SIZE * MAX_DATA_OBJS)
+#define SD_DEFAULT_BLOCK_SIZE_SHIFT 22
 /*
  * For erasure coding, we use at most SD_EC_MAX_STRIP for data strips and
  * (SD_EC_MAX_STRIP - 1) for parity strips
@@ -167,7 +171,8 @@
     uint32_t base_vdi_id;
     uint8_t copies;
     uint8_t copy_policy;
-    uint8_t reserved[2];
+    uint8_t store_policy;
+    uint8_t block_size_shift;
     uint32_t snapid;
     uint32_t type;
     uint32_t pad[2];
@@ -186,6 +191,21 @@
     uint32_t pad[5];
 } SheepdogVdiRsp;
 
+typedef struct SheepdogClusterRsp {
+    uint8_t proto_ver;
+    uint8_t opcode;
+    uint16_t flags;
+    uint32_t epoch;
+    uint32_t id;
+    uint32_t data_length;
+    uint32_t result;
+    uint8_t nr_copies;
+    uint8_t copy_policy;
+    uint8_t block_size_shift;
+    uint8_t __pad1;
+    uint32_t __pad2[6];
+} SheepdogClusterRsp;
+
 typedef struct SheepdogInode {
     char name[SD_MAX_VDI_LEN];
     char tag[SD_MAX_VDI_TAG_LEN];
@@ -266,15 +286,24 @@
     return !!inode->snap_ctime;
 }
 
+static inline size_t count_data_objs(const struct SheepdogInode *inode)
+{
+    return DIV_ROUND_UP(inode->vdi_size,
+                        (1UL << inode->block_size_shift));
+}
+
 #undef DPRINTF
 #ifdef DEBUG_SDOG
-#define DPRINTF(fmt, args...)                                       \
-    do {                                                            \
-        fprintf(stdout, "%s %d: " fmt, __func__, __LINE__, ##args); \
-    } while (0)
+#define DEBUG_SDOG_PRINT 1
 #else
-#define DPRINTF(fmt, args...)
+#define DEBUG_SDOG_PRINT 0
 #endif
+#define DPRINTF(fmt, args...)                                           \
+    do {                                                                \
+        if (DEBUG_SDOG_PRINT) {                                         \
+            fprintf(stderr, "%s %d: " fmt, __func__, __LINE__, ##args); \
+        }                                                               \
+    } while (0)
 
 typedef struct SheepdogAIOCB SheepdogAIOCB;
 
@@ -300,6 +329,10 @@
     AIOCB_DISCARD_OBJ,
 };
 
+#define AIOCBOverlapping(x, y)                                 \
+    (!(x->max_affect_data_idx < y->min_affect_data_idx          \
+       || y->max_affect_data_idx < x->min_affect_data_idx))
+
 struct SheepdogAIOCB {
     BlockAIOCB common;
 
@@ -316,6 +349,20 @@
 
     bool cancelable;
     int nr_pending;
+
+    uint32_t min_affect_data_idx;
+    uint32_t max_affect_data_idx;
+
+    /*
+     * The difference between affect_data_idx and dirty_data_idx:
+     * affect_data_idx represents range of index of all request types.
+     * dirty_data_idx represents range of index updated by COW requests.
+     * dirty_data_idx is used for updating an inode object.
+     */
+    uint32_t min_dirty_data_idx;
+    uint32_t max_dirty_data_idx;
+
+    QLIST_ENTRY(SheepdogAIOCB) aiocb_siblings;
 };
 
 typedef struct BDRVSheepdogState {
@@ -324,9 +371,6 @@
 
     SheepdogInode inode;
 
-    uint32_t min_dirty_data_idx;
-    uint32_t max_dirty_data_idx;
-
     char name[SD_MAX_VDI_LEN];
     bool is_snapshot;
     uint32_t cache_flags;
@@ -344,10 +388,17 @@
 
     /* Every aio request must be linked to either of these queues. */
     QLIST_HEAD(inflight_aio_head, AIOReq) inflight_aio_head;
-    QLIST_HEAD(pending_aio_head, AIOReq) pending_aio_head;
     QLIST_HEAD(failed_aio_head, AIOReq) failed_aio_head;
+
+    CoQueue overlapping_queue;
+    QLIST_HEAD(inflight_aiocb_head, SheepdogAIOCB) inflight_aiocb_head;
 } BDRVSheepdogState;
 
+typedef struct BDRVSheepdogReopenState {
+    int fd;
+    int cache_flags;
+} BDRVSheepdogReopenState;
+
 static const char * sd_strerror(int err)
 {
     int i;
@@ -444,7 +495,7 @@
 
 static void coroutine_fn sd_finish_aiocb(SheepdogAIOCB *acb)
 {
-    qemu_coroutine_enter(acb->coroutine, NULL);
+    qemu_coroutine_enter(acb->coroutine);
     qemu_aio_unref(acb);
 }
 
@@ -480,13 +531,7 @@
     AIOReq *aioreq, *next;
 
     if (sd_acb_cancelable(acb)) {
-        /* Remove outstanding requests from pending and failed queues.  */
-        QLIST_FOREACH_SAFE(aioreq, &s->pending_aio_head, aio_siblings,
-                           next) {
-            if (aioreq->aiocb == acb) {
-                free_aio_req(s, aioreq);
-            }
-        }
+        /* Remove outstanding requests from failed queue.  */
         QLIST_FOREACH_SAFE(aioreq, &s->failed_aio_head, aio_siblings,
                            next) {
             if (aioreq->aiocb == acb) {
@@ -511,6 +556,10 @@
                                    int64_t sector_num, int nb_sectors)
 {
     SheepdogAIOCB *acb;
+    uint32_t object_size;
+    BDRVSheepdogState *s = bs->opaque;
+
+    object_size = (UINT32_C(1) << s->inode.block_size_shift);
 
     acb = qemu_aio_get(&sd_aiocb_info, bs, NULL, NULL);
 
@@ -524,9 +573,18 @@
     acb->coroutine = qemu_coroutine_self();
     acb->ret = 0;
     acb->nr_pending = 0;
+
+    acb->min_affect_data_idx = acb->sector_num * BDRV_SECTOR_SIZE / object_size;
+    acb->max_affect_data_idx = (acb->sector_num * BDRV_SECTOR_SIZE +
+                              acb->nb_sectors * BDRV_SECTOR_SIZE) / object_size;
+
+    acb->min_dirty_data_idx = UINT32_MAX;
+    acb->max_dirty_data_idx = 0;
+
     return acb;
 }
 
+/* Return -EIO in case of error, file descriptor on success */
 static int connect_to_sdog(BDRVSheepdogState *s, Error **errp)
 {
     int fd;
@@ -546,11 +604,14 @@
 
     if (fd >= 0) {
         qemu_set_nonblock(fd);
+    } else {
+        fd = -EIO;
     }
 
     return fd;
 }
 
+/* Return 0 on success and -errno in case of error */
 static coroutine_fn int send_co_req(int sockfd, SheepdogReq *hdr, void *data,
                                     unsigned int *wlen)
 {
@@ -559,12 +620,13 @@
     ret = qemu_co_send(sockfd, hdr, sizeof(*hdr));
     if (ret != sizeof(*hdr)) {
         error_report("failed to send a req, %s", strerror(errno));
-        return ret;
+        return -errno;
     }
 
     ret = qemu_co_send(sockfd, data, *wlen);
     if (ret != *wlen) {
         error_report("failed to send a req, %s", strerror(errno));
+        return -errno;
     }
 
     return ret;
@@ -574,7 +636,7 @@
 {
     Coroutine *co = opaque;
 
-    qemu_coroutine_enter(co, NULL);
+    qemu_coroutine_enter(co);
 }
 
 typedef struct SheepdogReqCo {
@@ -600,14 +662,16 @@
     unsigned int *rlen = srco->rlen;
 
     co = qemu_coroutine_self();
-    aio_set_fd_handler(srco->aio_context, sockfd, NULL, restart_co_req, co);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       NULL, restart_co_req, co);
 
     ret = send_co_req(sockfd, hdr, data, wlen);
     if (ret < 0) {
         goto out;
     }
 
-    aio_set_fd_handler(srco->aio_context, sockfd, restart_co_req, NULL, co);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       restart_co_req, NULL, co);
 
     ret = qemu_co_recv(sockfd, hdr, sizeof(*hdr));
     if (ret != sizeof(*hdr)) {
@@ -632,12 +696,18 @@
 out:
     /* there is at most one request for this sockfd, so it is safe to
      * set each handler to NULL. */
-    aio_set_fd_handler(srco->aio_context, sockfd, NULL, NULL, NULL);
+    aio_set_fd_handler(srco->aio_context, sockfd, false,
+                       NULL, NULL, NULL);
 
     srco->ret = ret;
     srco->finished = true;
 }
 
+/*
+ * Send the request to the sheep in a synchronous manner.
+ *
+ * Return 0 on success, -errno in case of error.
+ */
 static int do_req(int sockfd, AioContext *aio_context, SheepdogReq *hdr,
                   void *data, unsigned int *wlen, unsigned int *rlen)
 {
@@ -656,8 +726,8 @@
     if (qemu_in_coroutine()) {
         do_co_req(&srco);
     } else {
-        co = qemu_coroutine_create(do_co_req);
-        qemu_coroutine_enter(co, &srco);
+        co = qemu_coroutine_create(do_co_req, &srco);
+        qemu_coroutine_enter(co);
         while (!srco.finished) {
             aio_poll(aio_context, true);
         }
@@ -674,44 +744,13 @@
 static int get_sheep_fd(BDRVSheepdogState *s, Error **errp);
 static void co_write_request(void *opaque);
 
-static AIOReq *find_pending_req(BDRVSheepdogState *s, uint64_t oid)
-{
-    AIOReq *aio_req;
-
-    QLIST_FOREACH(aio_req, &s->pending_aio_head, aio_siblings) {
-        if (aio_req->oid == oid) {
-            return aio_req;
-        }
-    }
-
-    return NULL;
-}
-
-/*
- * This function searchs pending requests to the object `oid', and
- * sends them.
- */
-static void coroutine_fn send_pending_req(BDRVSheepdogState *s, uint64_t oid)
-{
-    AIOReq *aio_req;
-    SheepdogAIOCB *acb;
-
-    while ((aio_req = find_pending_req(s, oid)) != NULL) {
-        acb = aio_req->aiocb;
-        /* move aio_req from pending list to inflight one */
-        QLIST_REMOVE(aio_req, aio_siblings);
-        QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
-        add_aio_request(s, aio_req, acb->qiov->iov, acb->qiov->niov,
-                        acb->aiocb_type);
-    }
-}
-
 static coroutine_fn void reconnect_to_sdog(void *opaque)
 {
     BDRVSheepdogState *s = opaque;
     AIOReq *aio_req, *next;
 
-    aio_set_fd_handler(s->aio_context, s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(s->aio_context, s->fd, false, NULL,
+                       NULL, NULL);
     close(s->fd);
     s->fd = -1;
 
@@ -726,8 +765,7 @@
         s->fd = get_sheep_fd(s, &local_err);
         if (s->fd < 0) {
             DPRINTF("Wait for connection to be established\n");
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
+            error_report_err(local_err);
             co_aio_sleep_ns(bdrv_get_aio_context(s->bs), QEMU_CLOCK_REALTIME,
                             1000000000ULL);
         }
@@ -809,15 +847,9 @@
              */
             if (rsp.result == SD_RES_SUCCESS) {
                 s->inode.data_vdi_id[idx] = s->inode.vdi_id;
-                s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
-                s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
+                acb->max_dirty_data_idx = MAX(idx, acb->max_dirty_data_idx);
+                acb->min_dirty_data_idx = MIN(idx, acb->min_dirty_data_idx);
             }
-            /*
-             * Some requests may be blocked because simultaneous
-             * create requests are not allowed, so we search the
-             * pending requests here.
-             */
-            send_pending_req(s, aio_req->oid);
         }
         break;
     case AIOCB_READ_UDATA:
@@ -843,10 +875,6 @@
             rsp.result = SD_RES_SUCCESS;
             s->discard_supported = false;
             break;
-        case SD_RES_SUCCESS:
-            idx = data_oid_to_idx(aio_req->oid);
-            s->inode.data_vdi_id[idx] = 0;
-            break;
         default:
             break;
         }
@@ -897,17 +925,17 @@
     BDRVSheepdogState *s = opaque;
 
     if (!s->co_recv) {
-        s->co_recv = qemu_coroutine_create(aio_read_response);
+        s->co_recv = qemu_coroutine_create(aio_read_response, opaque);
     }
 
-    qemu_coroutine_enter(s->co_recv, opaque);
+    qemu_coroutine_enter(s->co_recv);
 }
 
 static void co_write_request(void *opaque)
 {
     BDRVSheepdogState *s = opaque;
 
-    qemu_coroutine_enter(s->co_send, NULL);
+    qemu_coroutine_enter(s->co_send);
 }
 
 /*
@@ -925,7 +953,8 @@
         return fd;
     }
 
-    aio_set_fd_handler(s->aio_context, fd, co_read_response, NULL, s);
+    aio_set_fd_handler(s->aio_context, fd, false,
+                       co_read_response, NULL, s);
     return fd;
 }
 
@@ -1161,7 +1190,13 @@
         hdr.flags = SD_FLAG_CMD_WRITE | flags;
         break;
     case AIOCB_DISCARD_OBJ:
-        hdr.opcode = SD_OP_DISCARD_OBJ;
+        hdr.opcode = SD_OP_WRITE_OBJ;
+        hdr.flags = SD_FLAG_CMD_WRITE | flags;
+        s->inode.data_vdi_id[data_oid_to_idx(oid)] = 0;
+        offset = offsetof(SheepdogInode,
+                          data_vdi_id[data_oid_to_idx(oid)]);
+        oid = vid_to_vdi_oid(s->inode.vdi_id);
+        wlen = datalen = sizeof(uint32_t);
         break;
     }
 
@@ -1180,7 +1215,7 @@
 
     qemu_co_mutex_lock(&s->lock);
     s->co_send = qemu_coroutine_self();
-    aio_set_fd_handler(s->aio_context, s->fd,
+    aio_set_fd_handler(s->aio_context, s->fd, false,
                        co_read_response, co_write_request, s);
     socket_set_cork(s->fd, 1);
 
@@ -1199,7 +1234,8 @@
     }
 out:
     socket_set_cork(s->fd, 0);
-    aio_set_fd_handler(s->aio_context, s->fd, co_read_response, NULL, s);
+    aio_set_fd_handler(s->aio_context, s->fd, false,
+                       co_read_response, NULL, s);
     s->co_send = NULL;
     qemu_co_mutex_unlock(&s->lock);
 }
@@ -1283,8 +1319,7 @@
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         return -EIO;
     }
 
@@ -1292,8 +1327,7 @@
 
     ret = find_vdi_name(s, s->name, snapid, tag, &vid, false, &local_err);
     if (ret) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         goto out;
     }
 
@@ -1315,30 +1349,6 @@
     return ret;
 }
 
-/* Return true if the specified request is linked to the pending list. */
-static bool check_simultaneous_create(BDRVSheepdogState *s, AIOReq *aio_req)
-{
-    AIOReq *areq;
-    QLIST_FOREACH(areq, &s->inflight_aio_head, aio_siblings) {
-        if (areq != aio_req && areq->oid == aio_req->oid) {
-            /*
-             * Sheepdog cannot handle simultaneous create requests to the same
-             * object, so we cannot send the request until the previous request
-             * finishes.
-             */
-            DPRINTF("simultaneous create to %" PRIx64 "\n", aio_req->oid);
-            aio_req->flags = 0;
-            aio_req->base_oid = 0;
-            aio_req->create = false;
-            QLIST_REMOVE(aio_req, aio_siblings);
-            QLIST_INSERT_HEAD(&s->pending_aio_head, aio_req, aio_siblings);
-            return true;
-        }
-    }
-
-    return false;
-}
-
 static void coroutine_fn resend_aioreq(BDRVSheepdogState *s, AIOReq *aio_req)
 {
     SheepdogAIOCB *acb = aio_req->aiocb;
@@ -1353,10 +1363,6 @@
             goto out;
         }
 
-        if (check_simultaneous_create(s, aio_req)) {
-            return;
-        }
-
         if (s->inode.data_vdi_id[idx]) {
             aio_req->base_oid = vid_to_data_oid(s->inode.data_vdi_id[idx], idx);
             aio_req->flags |= SD_FLAG_CMD_COW;
@@ -1379,7 +1385,8 @@
 {
     BDRVSheepdogState *s = bs->opaque;
 
-    aio_set_fd_handler(s->aio_context, s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(s->aio_context, s->fd, false, NULL,
+                       NULL, NULL);
 }
 
 static void sd_attach_aio_context(BlockDriverState *bs,
@@ -1388,7 +1395,8 @@
     BDRVSheepdogState *s = bs->opaque;
 
     s->aio_context = new_context;
-    aio_set_fd_handler(new_context, s->fd, co_read_response, NULL, s);
+    aio_set_fd_handler(new_context, s->fd, false,
+                       co_read_response, NULL, s);
 }
 
 /* TODO Convert to fine grained options */
@@ -1432,8 +1440,8 @@
     filename = qemu_opt_get(opts, "filename");
 
     QLIST_INIT(&s->inflight_aio_head);
-    QLIST_INIT(&s->pending_aio_head);
     QLIST_INIT(&s->failed_aio_head);
+    QLIST_INIT(&s->inflight_aiocb_head);
     s->fd = -1;
 
     memset(vdi, 0, sizeof(vdi));
@@ -1492,17 +1500,17 @@
     }
 
     memcpy(&s->inode, buf, sizeof(s->inode));
-    s->min_dirty_data_idx = UINT32_MAX;
-    s->max_dirty_data_idx = 0;
 
     bs->total_sectors = s->inode.vdi_size / BDRV_SECTOR_SIZE;
     pstrcpy(s->name, sizeof(s->name), vdi);
     qemu_co_mutex_init(&s->lock);
+    qemu_co_queue_init(&s->overlapping_queue);
     qemu_opts_del(opts);
     g_free(buf);
     return 0;
 out:
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd,
+                       false, NULL, NULL, NULL);
     if (s->fd >= 0) {
         closesocket(s->fd);
     }
@@ -1511,6 +1519,70 @@
     return ret;
 }
 
+static int sd_reopen_prepare(BDRVReopenState *state, BlockReopenQueue *queue,
+                             Error **errp)
+{
+    BDRVSheepdogState *s = state->bs->opaque;
+    BDRVSheepdogReopenState *re_s;
+    int ret = 0;
+
+    re_s = state->opaque = g_new0(BDRVSheepdogReopenState, 1);
+
+    re_s->cache_flags = SD_FLAG_CMD_CACHE;
+    if (state->flags & BDRV_O_NOCACHE) {
+        re_s->cache_flags = SD_FLAG_CMD_DIRECT;
+    }
+
+    re_s->fd = get_sheep_fd(s, errp);
+    if (re_s->fd < 0) {
+        ret = re_s->fd;
+        return ret;
+    }
+
+    return ret;
+}
+
+static void sd_reopen_commit(BDRVReopenState *state)
+{
+    BDRVSheepdogReopenState *re_s = state->opaque;
+    BDRVSheepdogState *s = state->bs->opaque;
+
+    if (s->fd) {
+        aio_set_fd_handler(s->aio_context, s->fd, false,
+                           NULL, NULL, NULL);
+        closesocket(s->fd);
+    }
+
+    s->fd = re_s->fd;
+    s->cache_flags = re_s->cache_flags;
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+
+    return;
+}
+
+static void sd_reopen_abort(BDRVReopenState *state)
+{
+    BDRVSheepdogReopenState *re_s = state->opaque;
+    BDRVSheepdogState *s = state->bs->opaque;
+
+    if (re_s == NULL) {
+        return;
+    }
+
+    if (re_s->fd) {
+        aio_set_fd_handler(s->aio_context, re_s->fd, false,
+                           NULL, NULL, NULL);
+        closesocket(re_s->fd);
+    }
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+
+    return;
+}
+
 static int do_sd_create(BDRVSheepdogState *s, uint32_t *vdi_id, int snapshot,
                         Error **errp)
 {
@@ -1544,6 +1616,7 @@
     hdr.vdi_size = s->inode.vdi_size;
     hdr.copy_policy = s->inode.copy_policy;
     hdr.copies = s->inode.nr_copies;
+    hdr.block_size_shift = s->inode.block_size_shift;
 
     ret = do_req(fd, s->aio_context, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
 
@@ -1568,47 +1641,60 @@
 
 static int sd_prealloc(const char *filename, Error **errp)
 {
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
+    BDRVSheepdogState *base = NULL;
+    unsigned long buf_size;
     uint32_t idx, max_idx;
+    uint32_t object_size;
     int64_t vdi_size;
-    void *buf = g_malloc0(SD_DATA_OBJ_SIZE);
+    void *buf = NULL;
     int ret;
 
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, errp);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, errp);
+    if (blk == NULL) {
+        ret = -EIO;
         goto out_with_err_set;
     }
 
-    vdi_size = bdrv_getlength(bs);
+    blk_set_allow_write_beyond_eof(blk, true);
+
+    vdi_size = blk_getlength(blk);
     if (vdi_size < 0) {
         ret = vdi_size;
         goto out;
     }
-    max_idx = DIV_ROUND_UP(vdi_size, SD_DATA_OBJ_SIZE);
+
+    base = blk_bs(blk)->opaque;
+    object_size = (UINT32_C(1) << base->inode.block_size_shift);
+    buf_size = MIN(object_size, SD_DATA_OBJ_SIZE);
+    buf = g_malloc0(buf_size);
+
+    max_idx = DIV_ROUND_UP(vdi_size, buf_size);
 
     for (idx = 0; idx < max_idx; idx++) {
         /*
          * The created image can be a cloned image, so we need to read
          * a data from the source image.
          */
-        ret = bdrv_pread(bs, idx * SD_DATA_OBJ_SIZE, buf, SD_DATA_OBJ_SIZE);
+        ret = blk_pread(blk, idx * buf_size, buf, buf_size);
         if (ret < 0) {
             goto out;
         }
-        ret = bdrv_pwrite(bs, idx * SD_DATA_OBJ_SIZE, buf, SD_DATA_OBJ_SIZE);
+        ret = blk_pwrite(blk, idx * buf_size, buf, buf_size, 0);
         if (ret < 0) {
             goto out;
         }
     }
 
+    ret = 0;
 out:
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Can't pre-allocate");
     }
 out_with_err_set:
-    if (bs) {
-        bdrv_unref(bs);
+    if (blk) {
+        blk_unref(blk);
     }
     g_free(buf);
 
@@ -1669,6 +1755,27 @@
     return 0;
 }
 
+static int parse_block_size_shift(BDRVSheepdogState *s, QemuOpts *opt)
+{
+    struct SheepdogInode *inode = &s->inode;
+    uint64_t object_size;
+    int obj_order;
+
+    object_size = qemu_opt_get_size_del(opt, BLOCK_OPT_OBJECT_SIZE, 0);
+    if (object_size) {
+        if ((object_size - 1) & object_size) {    /* not a power of 2? */
+            return -EINVAL;
+        }
+        obj_order = ctz32(object_size);
+        if (obj_order < 20 || obj_order > 31) {
+            return -EINVAL;
+        }
+        inode->block_size_shift = (uint8_t)obj_order;
+    }
+
+    return 0;
+}
+
 static int sd_create(const char *filename, QemuOpts *opts,
                      Error **errp)
 {
@@ -1679,6 +1786,7 @@
     BDRVSheepdogState *s;
     char tag[SD_MAX_VDI_TAG_LEN];
     uint32_t snapid;
+    uint64_t max_vdi_size;
     bool prealloc = false;
 
     s = g_new0(BDRVSheepdogState, 1);
@@ -1717,46 +1825,91 @@
             goto out;
         }
     }
-
-    if (s->inode.vdi_size > SD_MAX_VDI_SIZE) {
-        error_setg(errp, "too big image size");
-        ret = -EINVAL;
+    ret = parse_block_size_shift(s, opts);
+    if (ret < 0) {
+        error_setg(errp, "Invalid object_size."
+                         " obect_size needs to be power of 2"
+                         " and be limited from 2^20 to 2^31");
         goto out;
     }
 
     if (backing_file) {
-        BlockDriverState *bs;
+        BlockBackend *blk;
         BDRVSheepdogState *base;
         BlockDriver *drv;
 
         /* Currently, only Sheepdog backing image is supported. */
-        drv = bdrv_find_protocol(backing_file, true);
+        drv = bdrv_find_protocol(backing_file, true, NULL);
         if (!drv || strcmp(drv->protocol_name, "sheepdog") != 0) {
             error_setg(errp, "backing_file must be a sheepdog image");
             ret = -EINVAL;
             goto out;
         }
 
-        bs = NULL;
-        ret = bdrv_open(&bs, backing_file, NULL, NULL, BDRV_O_PROTOCOL, NULL,
-                        errp);
-        if (ret < 0) {
+        blk = blk_new_open(backing_file, NULL, NULL,
+                           BDRV_O_PROTOCOL, errp);
+        if (blk == NULL) {
+            ret = -EIO;
             goto out;
         }
 
-        base = bs->opaque;
+        base = blk_bs(blk)->opaque;
 
         if (!is_snapshot(&base->inode)) {
             error_setg(errp, "cannot clone from a non snapshot vdi");
-            bdrv_unref(bs);
+            blk_unref(blk);
             ret = -EINVAL;
             goto out;
         }
         s->inode.vdi_id = base->inode.vdi_id;
-        bdrv_unref(bs);
+        blk_unref(blk);
     }
 
     s->aio_context = qemu_get_aio_context();
+
+    /* if block_size_shift is not specified, get cluster default value */
+    if (s->inode.block_size_shift == 0) {
+        SheepdogVdiReq hdr;
+        SheepdogClusterRsp *rsp = (SheepdogClusterRsp *)&hdr;
+        Error *local_err = NULL;
+        int fd;
+        unsigned int wlen = 0, rlen = 0;
+
+        fd = connect_to_sdog(s, &local_err);
+        if (fd < 0) {
+            error_report_err(local_err);
+            ret = -EIO;
+            goto out;
+        }
+
+        memset(&hdr, 0, sizeof(hdr));
+        hdr.opcode = SD_OP_GET_CLUSTER_DEFAULT;
+        hdr.proto_ver = SD_PROTO_VER;
+
+        ret = do_req(fd, s->aio_context, (SheepdogReq *)&hdr,
+                     NULL, &wlen, &rlen);
+        closesocket(fd);
+        if (ret) {
+            error_setg_errno(errp, -ret, "failed to get cluster default");
+            goto out;
+        }
+        if (rsp->result == SD_RES_SUCCESS) {
+            s->inode.block_size_shift = rsp->block_size_shift;
+        } else {
+            s->inode.block_size_shift = SD_DEFAULT_BLOCK_SIZE_SHIFT;
+        }
+    }
+
+    max_vdi_size = (UINT64_C(1) << s->inode.block_size_shift) * MAX_DATA_OBJS;
+
+    if (s->inode.vdi_size > max_vdi_size) {
+        error_setg(errp, "An image is too large."
+                         " The maximum image size is %"PRIu64 "GB",
+                         max_vdi_size / 1024 / 1024 / 1024);
+        ret = -EINVAL;
+        goto out;
+    }
+
     ret = do_sd_create(s, &vid, 0, errp);
     if (ret) {
         goto out;
@@ -1785,8 +1938,7 @@
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         return;
     }
 
@@ -1809,7 +1961,8 @@
         error_report("%s, %s", sd_strerror(rsp->result), s->name);
     }
 
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->fd,
+                       false, NULL, NULL, NULL);
     closesocket(s->fd);
     g_free(s->host_spec);
 }
@@ -1827,19 +1980,20 @@
     BDRVSheepdogState *s = bs->opaque;
     int ret, fd;
     unsigned int datalen;
+    uint64_t max_vdi_size;
 
+    max_vdi_size = (UINT64_C(1) << s->inode.block_size_shift) * MAX_DATA_OBJS;
     if (offset < s->inode.vdi_size) {
         error_report("shrinking is not supported");
         return -EINVAL;
-    } else if (offset > SD_MAX_VDI_SIZE) {
+    } else if (offset > max_vdi_size) {
         error_report("too big image size");
         return -EINVAL;
     }
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         return fd;
     }
 
@@ -1870,16 +2024,16 @@
     AIOReq *aio_req;
     uint32_t offset, data_len, mn, mx;
 
-    mn = s->min_dirty_data_idx;
-    mx = s->max_dirty_data_idx;
+    mn = acb->min_dirty_data_idx;
+    mx = acb->max_dirty_data_idx;
     if (mn <= mx) {
         /* we need to update the vdi object. */
         offset = sizeof(s->inode) - sizeof(s->inode.data_vdi_id) +
             mn * sizeof(s->inode.data_vdi_id[0]);
         data_len = (mx - mn + 1) * sizeof(s->inode.data_vdi_id[0]);
 
-        s->min_dirty_data_idx = UINT32_MAX;
-        s->max_dirty_data_idx = 0;
+        acb->min_dirty_data_idx = UINT32_MAX;
+        acb->max_dirty_data_idx = 0;
 
         iov.iov_base = &s->inode;
         iov.iov_len = sizeof(s->inode);
@@ -1912,8 +2066,7 @@
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         return false;
     }
 
@@ -1960,8 +2113,7 @@
     deleted = sd_delete(s);
     ret = do_sd_create(s, &vid, !deleted, &local_err);
     if (ret) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         goto out;
     }
 
@@ -1969,8 +2121,7 @@
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         ret = fd;
         goto out;
     }
@@ -2013,9 +2164,10 @@
     SheepdogAIOCB *acb = p;
     int ret = 0;
     unsigned long len, done = 0, total = acb->nb_sectors * BDRV_SECTOR_SIZE;
-    unsigned long idx = acb->sector_num * BDRV_SECTOR_SIZE / SD_DATA_OBJ_SIZE;
+    unsigned long idx;
+    uint32_t object_size;
     uint64_t oid;
-    uint64_t offset = (acb->sector_num * BDRV_SECTOR_SIZE) % SD_DATA_OBJ_SIZE;
+    uint64_t offset;
     BDRVSheepdogState *s = acb->common.bs->opaque;
     SheepdogInode *inode = &s->inode;
     AIOReq *aio_req;
@@ -2032,6 +2184,10 @@
         }
     }
 
+    object_size = (UINT32_C(1) << inode->block_size_shift);
+    idx = acb->sector_num * BDRV_SECTOR_SIZE / object_size;
+    offset = (acb->sector_num * BDRV_SECTOR_SIZE) % object_size;
+
     /*
      * Make sure we don't free the aiocb before we are done with all requests.
      * This additional reference is dropped at the end of this function.
@@ -2045,7 +2201,7 @@
 
         oid = vid_to_data_oid(inode->data_vdi_id[idx], idx);
 
-        len = MIN(total - done, SD_DATA_OBJ_SIZE - offset);
+        len = MIN(total - done, object_size - offset);
 
         switch (acb->aiocb_type) {
         case AIOCB_READ_UDATA:
@@ -2069,7 +2225,7 @@
              * We discard the object only when the whole object is
              * 1) allocated 2) trimmed. Otherwise, simply skip it.
              */
-            if (len != SD_DATA_OBJ_SIZE || inode->data_vdi_id[idx] == 0) {
+            if (len != object_size || inode->data_vdi_id[idx] == 0) {
                 goto done;
             }
             break;
@@ -2086,15 +2242,11 @@
         }
 
         aio_req = alloc_aio_req(s, acb, oid, len, offset, flags, create,
-                                old_oid, done);
+                                old_oid,
+                                acb->aiocb_type == AIOCB_DISCARD_OBJ ?
+                                0 : done);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
 
-        if (create) {
-            if (check_simultaneous_create(s, aio_req)) {
-                goto done;
-            }
-        }
-
         add_aio_request(s, aio_req, acb->qiov->iov, acb->qiov->niov,
                         acb->aiocb_type);
     done:
@@ -2109,6 +2261,20 @@
     return 1;
 }
 
+static bool check_overlapping_aiocb(BDRVSheepdogState *s, SheepdogAIOCB *aiocb)
+{
+    SheepdogAIOCB *cb;
+
+    QLIST_FOREACH(cb, &s->inflight_aiocb_head, aiocb_siblings) {
+        if (AIOCBOverlapping(aiocb, cb)) {
+            return true;
+        }
+    }
+
+    QLIST_INSERT_HEAD(&s->inflight_aiocb_head, aiocb, aiocb_siblings);
+    return false;
+}
+
 static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
                         int nb_sectors, QEMUIOVector *qiov)
 {
@@ -2117,7 +2283,7 @@
     int64_t offset = (sector_num + nb_sectors) * BDRV_SECTOR_SIZE;
     BDRVSheepdogState *s = bs->opaque;
 
-    if (bs->growable && offset > s->inode.vdi_size) {
+    if (offset > s->inode.vdi_size) {
         ret = sd_truncate(bs, offset);
         if (ret < 0) {
             return ret;
@@ -2128,14 +2294,25 @@
     acb->aio_done_func = sd_write_done;
     acb->aiocb_type = AIOCB_WRITE_UDATA;
 
+retry:
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
+        goto retry;
+    }
+
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
+        QLIST_REMOVE(acb, aiocb_siblings);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
 
     qemu_coroutine_yield();
 
+    QLIST_REMOVE(acb, aiocb_siblings);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
+
     return acb->ret;
 }
 
@@ -2144,19 +2321,30 @@
 {
     SheepdogAIOCB *acb;
     int ret;
+    BDRVSheepdogState *s = bs->opaque;
 
     acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors);
     acb->aiocb_type = AIOCB_READ_UDATA;
     acb->aio_done_func = sd_finish_aiocb;
 
+retry:
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
+        goto retry;
+    }
+
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
+        QLIST_REMOVE(acb, aiocb_siblings);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
 
     qemu_coroutine_yield();
 
+    QLIST_REMOVE(acb, aiocb_siblings);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
     return acb->ret;
 }
 
@@ -2218,8 +2406,7 @@
     /* refresh inode. */
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         ret = fd;
         goto cleanup;
     }
@@ -2234,10 +2421,8 @@
 
     ret = do_sd_create(s, &new_vid, 1, &local_err);
     if (ret < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
-        error_report("failed to create inode for snapshot. %s",
-                     strerror(errno));
+        error_reportf_err(local_err,
+                          "failed to create inode for snapshot: ");
         goto cleanup;
     }
 
@@ -2308,13 +2493,131 @@
     return ret;
 }
 
+#define NR_BATCHED_DISCARD 128
+
+static bool remove_objects(BDRVSheepdogState *s)
+{
+    int fd, i = 0, nr_objs = 0;
+    Error *local_err = NULL;
+    int ret = 0;
+    bool result = true;
+    SheepdogInode *inode = &s->inode;
+
+    fd = connect_to_sdog(s, &local_err);
+    if (fd < 0) {
+        error_report_err(local_err);
+        return false;
+    }
+
+    nr_objs = count_data_objs(inode);
+    while (i < nr_objs) {
+        int start_idx, nr_filled_idx;
+
+        while (i < nr_objs && !inode->data_vdi_id[i]) {
+            i++;
+        }
+        start_idx = i;
+
+        nr_filled_idx = 0;
+        while (i < nr_objs && nr_filled_idx < NR_BATCHED_DISCARD) {
+            if (inode->data_vdi_id[i]) {
+                inode->data_vdi_id[i] = 0;
+                nr_filled_idx++;
+            }
+
+            i++;
+        }
+
+        ret = write_object(fd, s->aio_context,
+                           (char *)&inode->data_vdi_id[start_idx],
+                           vid_to_vdi_oid(s->inode.vdi_id), inode->nr_copies,
+                           (i - start_idx) * sizeof(uint32_t),
+                           offsetof(struct SheepdogInode,
+                                    data_vdi_id[start_idx]),
+                           false, s->cache_flags);
+        if (ret < 0) {
+            error_report("failed to discard snapshot inode.");
+            result = false;
+            goto out;
+        }
+    }
+
+out:
+    closesocket(fd);
+    return result;
+}
+
 static int sd_snapshot_delete(BlockDriverState *bs,
                               const char *snapshot_id,
                               const char *name,
                               Error **errp)
 {
-    /* FIXME: Delete specified snapshot id.  */
-    return 0;
+    unsigned long snap_id = 0;
+    char snap_tag[SD_MAX_VDI_TAG_LEN];
+    Error *local_err = NULL;
+    int fd, ret;
+    char buf[SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN];
+    BDRVSheepdogState *s = bs->opaque;
+    unsigned int wlen = SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN, rlen = 0;
+    uint32_t vid;
+    SheepdogVdiReq hdr = {
+        .opcode = SD_OP_DEL_VDI,
+        .data_length = wlen,
+        .flags = SD_FLAG_CMD_WRITE,
+    };
+    SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
+
+    if (!remove_objects(s)) {
+        return -1;
+    }
+
+    memset(buf, 0, sizeof(buf));
+    memset(snap_tag, 0, sizeof(snap_tag));
+    pstrcpy(buf, SD_MAX_VDI_LEN, s->name);
+    ret = qemu_strtoul(snapshot_id, NULL, 10, &snap_id);
+    if (ret || snap_id > UINT32_MAX) {
+        error_setg(errp, "Invalid snapshot ID: %s",
+                         snapshot_id ? snapshot_id : "<null>");
+        return -EINVAL;
+    }
+
+    if (snap_id) {
+        hdr.snapid = (uint32_t) snap_id;
+    } else {
+        pstrcpy(snap_tag, sizeof(snap_tag), snapshot_id);
+        pstrcpy(buf + SD_MAX_VDI_LEN, SD_MAX_VDI_TAG_LEN, snap_tag);
+    }
+
+    ret = find_vdi_name(s, s->name, snap_id, snap_tag, &vid, true,
+                        &local_err);
+    if (ret) {
+        return ret;
+    }
+
+    fd = connect_to_sdog(s, &local_err);
+    if (fd < 0) {
+        error_report_err(local_err);
+        return -1;
+    }
+
+    ret = do_req(fd, s->aio_context, (SheepdogReq *)&hdr,
+                 buf, &wlen, &rlen);
+    closesocket(fd);
+    if (ret) {
+        return ret;
+    }
+
+    switch (rsp->result) {
+    case SD_RES_NO_VDI:
+        error_report("%s was already deleted", s->name);
+    case SD_RES_SUCCESS:
+        break;
+    default:
+        error_report("%s, %s", sd_strerror(rsp->result), s->name);
+        return -1;
+    }
+
+    return ret;
 }
 
 static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
@@ -2336,8 +2639,7 @@
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         ret = fd;
         goto out;
     }
@@ -2366,8 +2668,7 @@
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         ret = fd;
         goto out;
     }
@@ -2426,19 +2727,19 @@
     uint64_t offset;
     uint32_t vdi_index;
     uint32_t vdi_id = load ? s->inode.parent_vdi_id : s->inode.vdi_id;
+    uint32_t object_size = (UINT32_C(1) << s->inode.block_size_shift);
 
     fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
-        error_report("%s", error_get_pretty(local_err));;
-        error_free(local_err);
+        error_report_err(local_err);
         return fd;
     }
 
     while (remaining) {
-        vdi_index = pos / SD_DATA_OBJ_SIZE;
-        offset = pos % SD_DATA_OBJ_SIZE;
+        vdi_index = pos / object_size;
+        offset = pos % object_size;
 
-        data_len = MIN(remaining, SD_DATA_OBJ_SIZE - offset);
+        data_len = MIN(remaining, object_size - offset);
 
         vmstate_oid = vid_to_vmstate_oid(vdi_id, vdi_index);
 
@@ -2483,52 +2784,82 @@
     return ret;
 }
 
-static int sd_load_vmstate(BlockDriverState *bs, uint8_t *data,
-                           int64_t pos, int size)
+static int sd_load_vmstate(BlockDriverState *bs, QEMUIOVector *qiov,
+                           int64_t pos)
 {
     BDRVSheepdogState *s = bs->opaque;
+    void *buf;
+    int ret;
 
-    return do_load_save_vmstate(s, data, pos, size, 1);
+    buf = qemu_blockalign(bs, qiov->size);
+    ret = do_load_save_vmstate(s, buf, pos, qiov->size, 1);
+    qemu_iovec_from_buf(qiov, 0, buf, qiov->size);
+    qemu_vfree(buf);
+
+    return ret;
 }
 
 
-static coroutine_fn int sd_co_discard(BlockDriverState *bs, int64_t sector_num,
-                                      int nb_sectors)
+static coroutine_fn int sd_co_pdiscard(BlockDriverState *bs, int64_t offset,
+                                      int count)
 {
     SheepdogAIOCB *acb;
-    QEMUIOVector dummy;
     BDRVSheepdogState *s = bs->opaque;
     int ret;
+    QEMUIOVector discard_iov;
+    struct iovec iov;
+    uint32_t zero = 0;
 
     if (!s->discard_supported) {
-            return 0;
+        return 0;
     }
 
-    acb = sd_aio_setup(bs, &dummy, sector_num, nb_sectors);
+    memset(&discard_iov, 0, sizeof(discard_iov));
+    memset(&iov, 0, sizeof(iov));
+    iov.iov_base = &zero;
+    iov.iov_len = sizeof(zero);
+    discard_iov.iov = &iov;
+    discard_iov.niov = 1;
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((count & (BDRV_SECTOR_SIZE - 1)) == 0);
+    acb = sd_aio_setup(bs, &discard_iov, offset >> BDRV_SECTOR_BITS,
+                       count >> BDRV_SECTOR_BITS);
     acb->aiocb_type = AIOCB_DISCARD_OBJ;
     acb->aio_done_func = sd_finish_aiocb;
 
+retry:
+    if (check_overlapping_aiocb(s, acb)) {
+        qemu_co_queue_wait(&s->overlapping_queue);
+        goto retry;
+    }
+
     ret = sd_co_rw_vector(acb);
     if (ret <= 0) {
+        QLIST_REMOVE(acb, aiocb_siblings);
+        qemu_co_queue_restart_all(&s->overlapping_queue);
         qemu_aio_unref(acb);
         return ret;
     }
 
     qemu_coroutine_yield();
 
+    QLIST_REMOVE(acb, aiocb_siblings);
+    qemu_co_queue_restart_all(&s->overlapping_queue);
+
     return acb->ret;
 }
 
 static coroutine_fn int64_t
 sd_co_get_block_status(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
-                       int *pnum)
+                       int *pnum, BlockDriverState **file)
 {
     BDRVSheepdogState *s = bs->opaque;
     SheepdogInode *inode = &s->inode;
+    uint32_t object_size = (UINT32_C(1) << inode->block_size_shift);
     uint64_t offset = sector_num * BDRV_SECTOR_SIZE;
-    unsigned long start = offset / SD_DATA_OBJ_SIZE,
+    unsigned long start = offset / object_size,
                   end = DIV_ROUND_UP((sector_num + nb_sectors) *
-                                     BDRV_SECTOR_SIZE, SD_DATA_OBJ_SIZE);
+                                     BDRV_SECTOR_SIZE, object_size);
     unsigned long idx;
     int64_t ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | offset;
 
@@ -2547,10 +2878,13 @@
         }
     }
 
-    *pnum = (idx - start) * SD_DATA_OBJ_SIZE / BDRV_SECTOR_SIZE;
+    *pnum = (idx - start) * object_size / BDRV_SECTOR_SIZE;
     if (*pnum > nb_sectors) {
         *pnum = nb_sectors;
     }
+    if (ret > 0 && ret & BDRV_BLOCK_OFFSET_VALID) {
+        *file = bs;
+    }
     return ret;
 }
 
@@ -2558,14 +2892,15 @@
 {
     BDRVSheepdogState *s = bs->opaque;
     SheepdogInode *inode = &s->inode;
-    unsigned long i, last = DIV_ROUND_UP(inode->vdi_size, SD_DATA_OBJ_SIZE);
+    uint32_t object_size = (UINT32_C(1) << inode->block_size_shift);
+    unsigned long i, last = DIV_ROUND_UP(inode->vdi_size, object_size);
     uint64_t size = 0;
 
     for (i = 0; i < last; i++) {
         if (inode->data_vdi_id[i] == 0) {
             continue;
         }
-        size += SD_DATA_OBJ_SIZE;
+        size += object_size;
     }
     return size;
 }
@@ -2594,6 +2929,11 @@
             .type = QEMU_OPT_STRING,
             .help = "Redundancy of the image"
         },
+        {
+            .name = BLOCK_OPT_OBJECT_SIZE,
+            .type = QEMU_OPT_SIZE,
+            .help = "Object size of the image"
+        },
         { /* end of list */ }
     }
 };
@@ -2604,6 +2944,9 @@
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2614,7 +2957,7 @@
     .bdrv_co_readv  = sd_co_readv,
     .bdrv_co_writev = sd_co_writev,
     .bdrv_co_flush_to_disk  = sd_co_flush_to_disk,
-    .bdrv_co_discard = sd_co_discard,
+    .bdrv_co_pdiscard = sd_co_pdiscard,
     .bdrv_co_get_block_status = sd_co_get_block_status,
 
     .bdrv_snapshot_create   = sd_snapshot_create,
@@ -2637,6 +2980,9 @@
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2647,7 +2993,7 @@
     .bdrv_co_readv  = sd_co_readv,
     .bdrv_co_writev = sd_co_writev,
     .bdrv_co_flush_to_disk  = sd_co_flush_to_disk,
-    .bdrv_co_discard = sd_co_discard,
+    .bdrv_co_pdiscard = sd_co_pdiscard,
     .bdrv_co_get_block_status = sd_co_get_block_status,
 
     .bdrv_snapshot_create   = sd_snapshot_create,
@@ -2670,6 +3016,9 @@
     .instance_size  = sizeof(BDRVSheepdogState),
     .bdrv_needs_filename = true,
     .bdrv_file_open = sd_open,
+    .bdrv_reopen_prepare    = sd_reopen_prepare,
+    .bdrv_reopen_commit     = sd_reopen_commit,
+    .bdrv_reopen_abort      = sd_reopen_abort,
     .bdrv_close     = sd_close,
     .bdrv_create    = sd_create,
     .bdrv_has_zero_init = bdrv_has_zero_init_1,
@@ -2680,7 +3029,7 @@
     .bdrv_co_readv  = sd_co_readv,
     .bdrv_co_writev = sd_co_writev,
     .bdrv_co_flush_to_disk  = sd_co_flush_to_disk,
-    .bdrv_co_discard = sd_co_discard,
+    .bdrv_co_pdiscard = sd_co_pdiscard,
     .bdrv_co_get_block_status = sd_co_get_block_status,
 
     .bdrv_snapshot_create   = sd_snapshot_create,
diff --git a/block/snapshot.c b/block/snapshot.c
index 698e1a1..bf5c2ca 100644
--- a/block/snapshot.c
+++ b/block/snapshot.c
@@ -22,8 +22,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "block/snapshot.h"
 #include "block/block_int.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 
 QemuOptsList internal_snapshot_opts = {
     .name = "snapshot",
@@ -148,7 +151,7 @@
 
     if (!drv->bdrv_snapshot_create) {
         if (bs->file != NULL) {
-            return bdrv_can_snapshot(bs->file);
+            return bdrv_can_snapshot(bs->file->bs);
         }
         return 0;
     }
@@ -167,7 +170,7 @@
         return drv->bdrv_snapshot_create(bs, sn_info);
     }
     if (bs->file) {
-        return bdrv_snapshot_create(bs->file, sn_info);
+        return bdrv_snapshot_create(bs->file->bs, sn_info);
     }
     return -ENOTSUP;
 }
@@ -187,10 +190,10 @@
 
     if (bs->file) {
         drv->bdrv_close(bs);
-        ret = bdrv_snapshot_goto(bs->file, snapshot_id);
+        ret = bdrv_snapshot_goto(bs->file->bs, snapshot_id);
         open_ret = drv->bdrv_open(bs, NULL, bs->open_flags, NULL);
         if (open_ret < 0) {
-            bdrv_unref(bs->file);
+            bdrv_unref(bs->file->bs);
             bs->drv = NULL;
             return open_ret;
         }
@@ -228,8 +231,10 @@
                          Error **errp)
 {
     BlockDriver *drv = bs->drv;
+    int ret;
+
     if (!drv) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, bdrv_get_device_name(bs));
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, bdrv_get_device_name(bs));
         return -ENOMEDIUM;
     }
     if (!snapshot_id && !name) {
@@ -238,23 +243,26 @@
     }
 
     /* drain all pending i/o before deleting snapshot */
-    bdrv_drain_all();
+    bdrv_drained_begin(bs);
 
     if (drv->bdrv_snapshot_delete) {
-        return drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
+        ret = drv->bdrv_snapshot_delete(bs, snapshot_id, name, errp);
+    } else if (bs->file) {
+        ret = bdrv_snapshot_delete(bs->file->bs, snapshot_id, name, errp);
+    } else {
+        error_setg(errp, "Block format '%s' used by device '%s' "
+                   "does not support internal snapshot deletion",
+                   drv->format_name, bdrv_get_device_name(bs));
+        ret = -ENOTSUP;
     }
-    if (bs->file) {
-        return bdrv_snapshot_delete(bs->file, snapshot_id, name, errp);
-    }
-    error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              drv->format_name, bdrv_get_device_name(bs),
-              "internal snapshot deletion");
-    return -ENOTSUP;
+
+    bdrv_drained_end(bs);
+    return ret;
 }
 
-void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
-                                        const char *id_or_name,
-                                        Error **errp)
+int bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name,
+                                       Error **errp)
 {
     int ret;
     Error *local_err = NULL;
@@ -269,6 +277,7 @@
     if (ret < 0) {
         error_propagate(errp, local_err);
     }
+    return ret;
 }
 
 int bdrv_snapshot_list(BlockDriverState *bs,
@@ -282,7 +291,7 @@
         return drv->bdrv_snapshot_list(bs, psn_info);
     }
     if (bs->file) {
-        return bdrv_snapshot_list(bs->file, psn_info);
+        return bdrv_snapshot_list(bs->file->bs, psn_info);
     }
     return -ENOTSUP;
 }
@@ -315,7 +324,7 @@
     BlockDriver *drv = bs->drv;
 
     if (!drv) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, bdrv_get_device_name(bs));
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, bdrv_get_device_name(bs));
         return -ENOMEDIUM;
     }
     if (!snapshot_id && !name) {
@@ -329,9 +338,9 @@
     if (drv->bdrv_snapshot_load_tmp) {
         return drv->bdrv_snapshot_load_tmp(bs, snapshot_id, name, errp);
     }
-    error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              drv->format_name, bdrv_get_device_name(bs),
-              "temporarily load internal snapshot");
+    error_setg(errp, "Block format '%s' used by device '%s' "
+               "does not support temporarily loading internal snapshots",
+               drv->format_name, bdrv_get_device_name(bs));
     return -ENOTSUP;
 }
 
@@ -349,9 +358,164 @@
         ret = bdrv_snapshot_load_tmp(bs, NULL, id_or_name, &local_err);
     }
 
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 
     return ret;
 }
+
+
+/* Group operations. All block drivers are involved.
+ * These functions will properly handle dataplane (take aio_context_acquire
+ * when appropriate for appropriate block drivers) */
+
+bool bdrv_all_can_snapshot(BlockDriverState **first_bad_bs)
+{
+    bool ok = true;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_is_inserted(bs) && !bdrv_is_read_only(bs)) {
+            ok = bdrv_can_snapshot(bs);
+        }
+        aio_context_release(ctx);
+        if (!ok) {
+            goto fail;
+        }
+    }
+
+fail:
+    *first_bad_bs = bs;
+    return ok;
+}
+
+int bdrv_all_delete_snapshot(const char *name, BlockDriverState **first_bad_bs,
+                             Error **err)
+{
+    int ret = 0;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+    QEMUSnapshotInfo sn1, *snapshot = &sn1;
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs) &&
+                bdrv_snapshot_find(bs, snapshot, name) >= 0) {
+            ret = bdrv_snapshot_delete_by_id_or_name(bs, name, err);
+        }
+        aio_context_release(ctx);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+fail:
+    *first_bad_bs = bs;
+    return ret;
+}
+
+
+int bdrv_all_goto_snapshot(const char *name, BlockDriverState **first_bad_bs)
+{
+    int err = 0;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            err = bdrv_snapshot_goto(bs, name);
+        }
+        aio_context_release(ctx);
+        if (err < 0) {
+            goto fail;
+        }
+    }
+
+fail:
+    *first_bad_bs = bs;
+    return err;
+}
+
+int bdrv_all_find_snapshot(const char *name, BlockDriverState **first_bad_bs)
+{
+    QEMUSnapshotInfo sn;
+    int err = 0;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            err = bdrv_snapshot_find(bs, &sn, name);
+        }
+        aio_context_release(ctx);
+        if (err < 0) {
+            goto fail;
+        }
+    }
+
+fail:
+    *first_bad_bs = bs;
+    return err;
+}
+
+int bdrv_all_create_snapshot(QEMUSnapshotInfo *sn,
+                             BlockDriverState *vm_state_bs,
+                             uint64_t vm_state_size,
+                             BlockDriverState **first_bad_bs)
+{
+    int err = 0;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        if (bs == vm_state_bs) {
+            sn->vm_state_size = vm_state_size;
+            err = bdrv_snapshot_create(bs, sn);
+        } else if (bdrv_can_snapshot(bs)) {
+            sn->vm_state_size = 0;
+            err = bdrv_snapshot_create(bs, sn);
+        }
+        aio_context_release(ctx);
+        if (err < 0) {
+            goto fail;
+        }
+    }
+
+fail:
+    *first_bad_bs = bs;
+    return err;
+}
+
+BlockDriverState *bdrv_all_find_vmstate_bs(void)
+{
+    BlockDriverState *bs;
+    BdrvNextIterator it;
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+        bool found;
+
+        aio_context_acquire(ctx);
+        found = bdrv_can_snapshot(bs);
+        aio_context_release(ctx);
+
+        if (found) {
+            break;
+        }
+    }
+    return bs;
+}
diff --git a/block/ssh.c b/block/ssh.c
index f466cbf..5ce12b6 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -22,17 +22,18 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include <libssh2.h>
 #include <libssh2_sftp.h>
 
 #include "block/block_int.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "qemu/uri.h"
 #include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
 
 /* DEBUG_SSH=1 enables the DPRINTF (debugging printf) statements in
  * this block driver code.
@@ -191,7 +192,7 @@
 static int parse_uri(const char *filename, QDict *options, Error **errp)
 {
     URI *uri = NULL;
-    QueryParams *qp = NULL;
+    QueryParams *qp;
     int i;
 
     uri = uri_parse(filename);
@@ -247,9 +248,6 @@
     return 0;
 
  err:
-    if (qp) {
-      query_params_free(qp);
-    }
     if (uri) {
       uri_free(uri);
     }
@@ -510,36 +508,73 @@
     return ret;
 }
 
+static QemuOptsList ssh_runtime_opts = {
+    .name = "ssh",
+    .head = QTAILQ_HEAD_INITIALIZER(ssh_runtime_opts.head),
+    .desc = {
+        {
+            .name = "host",
+            .type = QEMU_OPT_STRING,
+            .help = "Host to connect to",
+        },
+        {
+            .name = "port",
+            .type = QEMU_OPT_NUMBER,
+            .help = "Port to connect to",
+        },
+        {
+            .name = "path",
+            .type = QEMU_OPT_STRING,
+            .help = "Path of the image on the host",
+        },
+        {
+            .name = "user",
+            .type = QEMU_OPT_STRING,
+            .help = "User as which to connect",
+        },
+        {
+            .name = "host_key_check",
+            .type = QEMU_OPT_STRING,
+            .help = "Defines how and what to check the host key against",
+        },
+    },
+};
+
 static int connect_to_ssh(BDRVSSHState *s, QDict *options,
                           int ssh_flags, int creat_mode, Error **errp)
 {
     int r, ret;
+    QemuOpts *opts = NULL;
+    Error *local_err = NULL;
     const char *host, *user, *path, *host_key_check;
     int port;
 
-    if (!qdict_haskey(options, "host")) {
+    opts = qemu_opts_create(&ssh_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        ret = -EINVAL;
+        error_propagate(errp, local_err);
+        goto err;
+    }
+
+    host = qemu_opt_get(opts, "host");
+    if (!host) {
         ret = -EINVAL;
         error_setg(errp, "No hostname was specified");
         goto err;
     }
-    host = qdict_get_str(options, "host");
 
-    if (qdict_haskey(options, "port")) {
-        port = qdict_get_int(options, "port");
-    } else {
-        port = 22;
-    }
+    port = qemu_opt_get_number(opts, "port", 22);
 
-    if (!qdict_haskey(options, "path")) {
+    path = qemu_opt_get(opts, "path");
+    if (!path) {
         ret = -EINVAL;
         error_setg(errp, "No path was specified");
         goto err;
     }
-    path = qdict_get_str(options, "path");
 
-    if (qdict_haskey(options, "user")) {
-        user = qdict_get_str(options, "user");
-    } else {
+    user = qemu_opt_get(opts, "user");
+    if (!user) {
         user = g_get_user_name();
         if (!user) {
             error_setg_errno(errp, errno, "Can't get user name");
@@ -548,9 +583,8 @@
         }
     }
 
-    if (qdict_haskey(options, "host_key_check")) {
-        host_key_check = qdict_get_str(options, "host_key_check");
-    } else {
+    host_key_check = qemu_opt_get(opts, "host_key_check");
+    if (!host_key_check) {
         host_key_check = "yes";
     }
 
@@ -561,7 +595,7 @@
     /* Open the socket and connect. */
     s->sock = inet_connect(s->hostport, errp);
     if (s->sock < 0) {
-        ret = -errno;
+        ret = -EIO;
         goto err;
     }
 
@@ -614,21 +648,14 @@
         goto err;
     }
 
+    qemu_opts_del(opts);
+
     r = libssh2_sftp_fstat(s->sftp_handle, &s->attrs);
     if (r < 0) {
         sftp_error_setg(errp, s, "failed to read file attributes");
         return -EINVAL;
     }
 
-    /* Delete the options we've used; any not deleted will cause the
-     * block layer to give an error about unused options.
-     */
-    qdict_del(options, "host");
-    qdict_del(options, "port");
-    qdict_del(options, "user");
-    qdict_del(options, "path");
-    qdict_del(options, "host_key_check");
-
     return 0;
 
  err:
@@ -648,6 +675,8 @@
     }
     s->session = NULL;
 
+    qemu_opts_del(opts);
+
     return ret;
 }
 
@@ -779,7 +808,7 @@
 
     DPRINTF("co=%p", co);
 
-    qemu_coroutine_enter(co, NULL);
+    qemu_coroutine_enter(co);
 }
 
 static coroutine_fn void set_fd_handler(BDRVSSHState *s, BlockDriverState *bs)
@@ -801,14 +830,15 @@
             rd_handler, wr_handler);
 
     aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
-                       rd_handler, wr_handler, co);
+                       false, rd_handler, wr_handler, co);
 }
 
 static coroutine_fn void clear_fd_handler(BDRVSSHState *s,
                                           BlockDriverState *bs)
 {
     DPRINTF("s->sock=%d", s->sock);
-    aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock, NULL, NULL, NULL);
+    aio_set_fd_handler(bdrv_get_aio_context(bs), s->sock,
+                       false, NULL, NULL, NULL);
 }
 
 /* A non-blocking call returned EAGAIN, so yield, ensuring the
diff --git a/block/stream.c b/block/stream.c
index a628901..3187481 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -11,10 +11,14 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu/ratelimit.h"
+#include "sysemu/block-backend.h"
 
 enum {
     /*
@@ -35,7 +39,7 @@
     char *backing_file_str;
 } StreamBlockJob;
 
-static int coroutine_fn stream_populate(BlockDriverState *bs,
+static int coroutine_fn stream_populate(BlockBackend *blk,
                                         int64_t sector_num, int nb_sectors,
                                         void *buf)
 {
@@ -48,35 +52,8 @@
     qemu_iovec_init_external(&qiov, &iov, 1);
 
     /* Copy-on-read the unallocated clusters */
-    return bdrv_co_copy_on_readv(bs, sector_num, nb_sectors, &qiov);
-}
-
-static void close_unused_images(BlockDriverState *top, BlockDriverState *base,
-                                const char *base_id)
-{
-    BlockDriverState *intermediate;
-    intermediate = top->backing_hd;
-
-    /* Must assign before bdrv_delete() to prevent traversing dangling pointer
-     * while we delete backing image instances.
-     */
-    bdrv_set_backing_hd(top, base);
-
-    while (intermediate) {
-        BlockDriverState *unused;
-
-        /* reached base */
-        if (intermediate == base) {
-            break;
-        }
-
-        unused = intermediate;
-        intermediate = intermediate->backing_hd;
-        bdrv_set_backing_hd(unused, NULL);
-        bdrv_unref(unused);
-    }
-
-    bdrv_refresh_limits(top, NULL);
+    return blk_co_preadv(blk, sector_num * BDRV_SECTOR_SIZE, qiov.size, &qiov,
+                         BDRV_REQ_COPY_ON_READ);
 }
 
 typedef struct {
@@ -88,6 +65,7 @@
 {
     StreamBlockJob *s = container_of(job, StreamBlockJob, common);
     StreamCompleteData *data = opaque;
+    BlockDriverState *bs = blk_bs(job->blk);
     BlockDriverState *base = s->base;
 
     if (!block_job_is_cancelled(&s->common) && data->reached_end &&
@@ -99,8 +77,8 @@
                 base_fmt = base->drv->format_name;
             }
         }
-        data->ret = bdrv_change_backing_file(job->bs, base_id, base_fmt);
-        close_unused_images(job->bs, base, base_id);
+        data->ret = bdrv_change_backing_file(bs, base_id, base_fmt);
+        bdrv_set_backing_hd(bs, base);
     }
 
     g_free(s->backing_file_str);
@@ -112,23 +90,25 @@
 {
     StreamBlockJob *s = opaque;
     StreamCompleteData *data;
-    BlockDriverState *bs = s->common.bs;
+    BlockBackend *blk = s->common.blk;
+    BlockDriverState *bs = blk_bs(blk);
     BlockDriverState *base = s->base;
-    int64_t sector_num, end;
+    int64_t sector_num = 0;
+    int64_t end = -1;
+    uint64_t delay_ns = 0;
     int error = 0;
     int ret = 0;
     int n = 0;
     void *buf;
 
-    if (!bs->backing_hd) {
-        block_job_completed(&s->common, 0);
-        return;
+    if (!bs->backing) {
+        goto out;
     }
 
     s->common.len = bdrv_getlength(bs);
     if (s->common.len < 0) {
-        block_job_completed(&s->common, s->common.len);
-        return;
+        ret = s->common.len;
+        goto out;
     }
 
     end = s->common.len >> BDRV_SECTOR_BITS;
@@ -144,10 +124,8 @@
     }
 
     for (sector_num = 0; sector_num < end; sector_num += n) {
-        uint64_t delay_ns = 0;
         bool copy;
 
-wait:
         /* Note that even when no rate limit is applied we need to yield
          * with no pending I/O here so that bdrv_drain_all() returns.
          */
@@ -165,7 +143,7 @@
         } else if (ret >= 0) {
             /* Copy if allocated in the intermediate images.  Limit to the
              * known-unallocated area [sector_num, sector_num+n).  */
-            ret = bdrv_is_allocated_above(bs->backing_hd, base,
+            ret = bdrv_is_allocated_above(backing_bs(bs), base,
                                           sector_num, n, &n);
 
             /* Finish early if end of backing file has been reached */
@@ -177,18 +155,11 @@
         }
         trace_stream_one_iteration(s, sector_num, n, ret);
         if (copy) {
-            if (s->common.speed) {
-                delay_ns = ratelimit_calculate_delay(&s->limit, n);
-                if (delay_ns > 0) {
-                    goto wait;
-                }
-            }
-            ret = stream_populate(bs, sector_num, n, buf);
+            ret = stream_populate(blk, sector_num, n, buf);
         }
         if (ret < 0) {
             BlockErrorAction action =
-                block_job_error_action(&s->common, s->common.bs, s->on_error,
-                                       true, -ret);
+                block_job_error_action(&s->common, s->on_error, true, -ret);
             if (action == BLOCK_ERROR_ACTION_STOP) {
                 n = 0;
                 continue;
@@ -204,6 +175,9 @@
 
         /* Publish progress */
         s->common.offset += n * BDRV_SECTOR_SIZE;
+        if (copy && s->common.speed) {
+            delay_ns = ratelimit_calculate_delay(&s->limit, n);
+        }
     }
 
     if (!base) {
@@ -215,6 +189,7 @@
 
     qemu_vfree(buf);
 
+out:
     /* Modify backing chain and close BDSes in main loop */
     data = g_malloc(sizeof(*data));
     data->ret = ret;
@@ -227,7 +202,7 @@
     StreamBlockJob *s = container_of(job, StreamBlockJob, common);
 
     if (speed < 0) {
-        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        error_setg(errp, QERR_INVALID_PARAMETER, "speed");
         return;
     }
     ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
@@ -239,22 +214,15 @@
     .set_speed     = stream_set_speed,
 };
 
-void stream_start(BlockDriverState *bs, BlockDriverState *base,
-                  const char *backing_file_str, int64_t speed,
-                  BlockdevOnError on_error,
-                  BlockCompletionFunc *cb,
-                  void *opaque, Error **errp)
+void stream_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *base, const char *backing_file_str,
+                  int64_t speed, BlockdevOnError on_error,
+                  BlockCompletionFunc *cb, void *opaque, Error **errp)
 {
     StreamBlockJob *s;
 
-    if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
-         on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
-        !bdrv_iostatus_is_enabled(bs)) {
-        error_set(errp, QERR_INVALID_PARAMETER, "on-error");
-        return;
-    }
-
-    s = block_job_create(&stream_job_driver, bs, speed, cb, opaque, errp);
+    s = block_job_create(job_id, &stream_job_driver, bs, speed,
+                         cb, opaque, errp);
     if (!s) {
         return;
     }
@@ -263,7 +231,7 @@
     s->backing_file_str = g_strdup(backing_file_str);
 
     s->on_error = on_error;
-    s->common.co = qemu_coroutine_create(stream_run);
+    s->common.co = qemu_coroutine_create(stream_run, s);
     trace_stream_start(bs, base, s, s->common.co, opaque);
-    qemu_coroutine_enter(s->common.co, s);
+    qemu_coroutine_enter(s->common.co);
 }
diff --git a/block/throttle-groups.c b/block/throttle-groups.c
new file mode 100644
index 0000000..59545e2
--- /dev/null
+++ b/block/throttle-groups.c
@@ -0,0 +1,509 @@
+/*
+ * QEMU block throttling group infrastructure
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/block-backend.h"
+#include "block/throttle-groups.h"
+#include "qemu/queue.h"
+#include "qemu/thread.h"
+#include "sysemu/qtest.h"
+
+/* The ThrottleGroup structure (with its ThrottleState) is shared
+ * among different BlockBackends and it's independent from
+ * AioContext, so in order to use it from different threads it needs
+ * its own locking.
+ *
+ * This locking is however handled internally in this file, so it's
+ * transparent to outside users.
+ *
+ * The whole ThrottleGroup structure is private and invisible to
+ * outside users, that only use it through its ThrottleState.
+ *
+ * In addition to the ThrottleGroup structure, BlockBackendPublic has
+ * fields that need to be accessed by other members of the group and
+ * therefore also need to be protected by this lock. Once a
+ * BlockBackend is registered in a group those fields can be accessed
+ * by other threads any time.
+ *
+ * Again, all this is handled internally and is mostly transparent to
+ * the outside. The 'throttle_timers' field however has an additional
+ * constraint because it may be temporarily invalid (see for example
+ * bdrv_set_aio_context()). Therefore in this file a thread will
+ * access some other BlockBackend's timers only after verifying that
+ * that BlockBackend has throttled requests in the queue.
+ */
+typedef struct ThrottleGroup {
+    char *name; /* This is constant during the lifetime of the group */
+
+    QemuMutex lock; /* This lock protects the following four fields */
+    ThrottleState ts;
+    QLIST_HEAD(, BlockBackendPublic) head;
+    BlockBackend *tokens[2];
+    bool any_timer_armed[2];
+
+    /* These two are protected by the global throttle_groups_lock */
+    unsigned refcount;
+    QTAILQ_ENTRY(ThrottleGroup) list;
+} ThrottleGroup;
+
+static QemuMutex throttle_groups_lock;
+static QTAILQ_HEAD(, ThrottleGroup) throttle_groups =
+    QTAILQ_HEAD_INITIALIZER(throttle_groups);
+
+/* Increments the reference count of a ThrottleGroup given its name.
+ *
+ * If no ThrottleGroup is found with the given name a new one is
+ * created.
+ *
+ * @name: the name of the ThrottleGroup
+ * @ret:  the ThrottleState member of the ThrottleGroup
+ */
+ThrottleState *throttle_group_incref(const char *name)
+{
+    ThrottleGroup *tg = NULL;
+    ThrottleGroup *iter;
+
+    qemu_mutex_lock(&throttle_groups_lock);
+
+    /* Look for an existing group with that name */
+    QTAILQ_FOREACH(iter, &throttle_groups, list) {
+        if (!strcmp(name, iter->name)) {
+            tg = iter;
+            break;
+        }
+    }
+
+    /* Create a new one if not found */
+    if (!tg) {
+        tg = g_new0(ThrottleGroup, 1);
+        tg->name = g_strdup(name);
+        qemu_mutex_init(&tg->lock);
+        throttle_init(&tg->ts);
+        QLIST_INIT(&tg->head);
+
+        QTAILQ_INSERT_TAIL(&throttle_groups, tg, list);
+    }
+
+    tg->refcount++;
+
+    qemu_mutex_unlock(&throttle_groups_lock);
+
+    return &tg->ts;
+}
+
+/* Decrease the reference count of a ThrottleGroup.
+ *
+ * When the reference count reaches zero the ThrottleGroup is
+ * destroyed.
+ *
+ * @ts:  The ThrottleGroup to unref, given by its ThrottleState member
+ */
+void throttle_group_unref(ThrottleState *ts)
+{
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+
+    qemu_mutex_lock(&throttle_groups_lock);
+    if (--tg->refcount == 0) {
+        QTAILQ_REMOVE(&throttle_groups, tg, list);
+        qemu_mutex_destroy(&tg->lock);
+        g_free(tg->name);
+        g_free(tg);
+    }
+    qemu_mutex_unlock(&throttle_groups_lock);
+}
+
+/* Get the name from a BlockBackend's ThrottleGroup. The name (and the pointer)
+ * is guaranteed to remain constant during the lifetime of the group.
+ *
+ * @blk:  a BlockBackend that is member of a throttling group
+ * @ret:  the name of the group.
+ */
+const char *throttle_group_get_name(BlockBackend *blk)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
+    return tg->name;
+}
+
+/* Return the next BlockBackend in the round-robin sequence, simulating a
+ * circular list.
+ *
+ * This assumes that tg->lock is held.
+ *
+ * @blk: the current BlockBackend
+ * @ret: the next BlockBackend in the sequence
+ */
+static BlockBackend *throttle_group_next_blk(BlockBackend *blk)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleState *ts = blkp->throttle_state;
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+    BlockBackendPublic *next = QLIST_NEXT(blkp, round_robin);
+
+    if (!next) {
+        next = QLIST_FIRST(&tg->head);
+    }
+
+    return blk_by_public(next);
+}
+
+/* Return the next BlockBackend in the round-robin sequence with pending I/O
+ * requests.
+ *
+ * This assumes that tg->lock is held.
+ *
+ * @blk:       the current BlockBackend
+ * @is_write:  the type of operation (read/write)
+ * @ret:       the next BlockBackend with pending requests, or blk if there is
+ *             none.
+ */
+static BlockBackend *next_throttle_token(BlockBackend *blk, bool is_write)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
+    BlockBackend *token, *start;
+
+    start = token = tg->tokens[is_write];
+
+    /* get next bs round in round robin style */
+    token = throttle_group_next_blk(token);
+    while (token != start && !blkp->pending_reqs[is_write]) {
+        token = throttle_group_next_blk(token);
+    }
+
+    /* If no IO are queued for scheduling on the next round robin token
+     * then decide the token is the current bs because chances are
+     * the current bs get the current request queued.
+     */
+    if (token == start && !blkp->pending_reqs[is_write]) {
+        token = blk;
+    }
+
+    return token;
+}
+
+/* Check if the next I/O request for a BlockBackend needs to be throttled or
+ * not. If there's no timer set in this group, set one and update the token
+ * accordingly.
+ *
+ * This assumes that tg->lock is held.
+ *
+ * @blk:        the current BlockBackend
+ * @is_write:   the type of operation (read/write)
+ * @ret:        whether the I/O request needs to be throttled or not
+ */
+static bool throttle_group_schedule_timer(BlockBackend *blk, bool is_write)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleState *ts = blkp->throttle_state;
+    ThrottleTimers *tt = &blkp->throttle_timers;
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+    bool must_wait;
+
+    if (blkp->io_limits_disabled) {
+        return false;
+    }
+
+    /* Check if any of the timers in this group is already armed */
+    if (tg->any_timer_armed[is_write]) {
+        return true;
+    }
+
+    must_wait = throttle_schedule_timer(ts, tt, is_write);
+
+    /* If a timer just got armed, set blk as the current token */
+    if (must_wait) {
+        tg->tokens[is_write] = blk;
+        tg->any_timer_armed[is_write] = true;
+    }
+
+    return must_wait;
+}
+
+/* Look for the next pending I/O request and schedule it.
+ *
+ * This assumes that tg->lock is held.
+ *
+ * @blk:       the current BlockBackend
+ * @is_write:  the type of operation (read/write)
+ */
+static void schedule_next_request(BlockBackend *blk, bool is_write)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
+    bool must_wait;
+    BlockBackend *token;
+
+    /* Check if there's any pending request to schedule next */
+    token = next_throttle_token(blk, is_write);
+    if (!blkp->pending_reqs[is_write]) {
+        return;
+    }
+
+    /* Set a timer for the request if it needs to be throttled */
+    must_wait = throttle_group_schedule_timer(token, is_write);
+
+    /* If it doesn't have to wait, queue it for immediate execution */
+    if (!must_wait) {
+        /* Give preference to requests from the current blk */
+        if (qemu_in_coroutine() &&
+            qemu_co_queue_next(&blkp->throttled_reqs[is_write])) {
+            token = blk;
+        } else {
+            ThrottleTimers *tt = &blkp->throttle_timers;
+            int64_t now = qemu_clock_get_ns(tt->clock_type);
+            timer_mod(tt->timers[is_write], now + 1);
+            tg->any_timer_armed[is_write] = true;
+        }
+        tg->tokens[is_write] = token;
+    }
+}
+
+/* Check if an I/O request needs to be throttled, wait and set a timer
+ * if necessary, and schedule the next request using a round robin
+ * algorithm.
+ *
+ * @blk:       the current BlockBackend
+ * @bytes:     the number of bytes for this I/O
+ * @is_write:  the type of operation (read/write)
+ */
+void coroutine_fn throttle_group_co_io_limits_intercept(BlockBackend *blk,
+                                                        unsigned int bytes,
+                                                        bool is_write)
+{
+    bool must_wait;
+    BlockBackend *token;
+
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
+    qemu_mutex_lock(&tg->lock);
+
+    /* First we check if this I/O has to be throttled. */
+    token = next_throttle_token(blk, is_write);
+    must_wait = throttle_group_schedule_timer(token, is_write);
+
+    /* Wait if there's a timer set or queued requests of this type */
+    if (must_wait || blkp->pending_reqs[is_write]) {
+        blkp->pending_reqs[is_write]++;
+        qemu_mutex_unlock(&tg->lock);
+        qemu_co_queue_wait(&blkp->throttled_reqs[is_write]);
+        qemu_mutex_lock(&tg->lock);
+        blkp->pending_reqs[is_write]--;
+    }
+
+    /* The I/O will be executed, so do the accounting */
+    throttle_account(blkp->throttle_state, is_write, bytes);
+
+    /* Schedule the next request */
+    schedule_next_request(blk, is_write);
+
+    qemu_mutex_unlock(&tg->lock);
+}
+
+void throttle_group_restart_blk(BlockBackend *blk)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    int i;
+
+    for (i = 0; i < 2; i++) {
+        while (qemu_co_enter_next(&blkp->throttled_reqs[i])) {
+            ;
+        }
+    }
+}
+
+/* Update the throttle configuration for a particular group. Similar
+ * to throttle_config(), but guarantees atomicity within the
+ * throttling group.
+ *
+ * @blk: a BlockBackend that is a member of the group
+ * @cfg: the configuration to set
+ */
+void throttle_group_config(BlockBackend *blk, ThrottleConfig *cfg)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleTimers *tt = &blkp->throttle_timers;
+    ThrottleState *ts = blkp->throttle_state;
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+    qemu_mutex_lock(&tg->lock);
+    /* throttle_config() cancels the timers */
+    if (timer_pending(tt->timers[0])) {
+        tg->any_timer_armed[0] = false;
+    }
+    if (timer_pending(tt->timers[1])) {
+        tg->any_timer_armed[1] = false;
+    }
+    throttle_config(ts, tt, cfg);
+    qemu_mutex_unlock(&tg->lock);
+
+    qemu_co_enter_next(&blkp->throttled_reqs[0]);
+    qemu_co_enter_next(&blkp->throttled_reqs[1]);
+}
+
+/* Get the throttle configuration from a particular group. Similar to
+ * throttle_get_config(), but guarantees atomicity within the
+ * throttling group.
+ *
+ * @blk: a BlockBackend that is a member of the group
+ * @cfg: the configuration will be written here
+ */
+void throttle_group_get_config(BlockBackend *blk, ThrottleConfig *cfg)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleState *ts = blkp->throttle_state;
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+    qemu_mutex_lock(&tg->lock);
+    throttle_get_config(ts, cfg);
+    qemu_mutex_unlock(&tg->lock);
+}
+
+/* ThrottleTimers callback. This wakes up a request that was waiting
+ * because it had been throttled.
+ *
+ * @blk:       the BlockBackend whose request had been throttled
+ * @is_write:  the type of operation (read/write)
+ */
+static void timer_cb(BlockBackend *blk, bool is_write)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleState *ts = blkp->throttle_state;
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+    bool empty_queue;
+
+    /* The timer has just been fired, so we can update the flag */
+    qemu_mutex_lock(&tg->lock);
+    tg->any_timer_armed[is_write] = false;
+    qemu_mutex_unlock(&tg->lock);
+
+    /* Run the request that was waiting for this timer */
+    empty_queue = !qemu_co_enter_next(&blkp->throttled_reqs[is_write]);
+
+    /* If the request queue was empty then we have to take care of
+     * scheduling the next one */
+    if (empty_queue) {
+        qemu_mutex_lock(&tg->lock);
+        schedule_next_request(blk, is_write);
+        qemu_mutex_unlock(&tg->lock);
+    }
+}
+
+static void read_timer_cb(void *opaque)
+{
+    timer_cb(opaque, false);
+}
+
+static void write_timer_cb(void *opaque)
+{
+    timer_cb(opaque, true);
+}
+
+/* Register a BlockBackend in the throttling group, also initializing its
+ * timers and updating its throttle_state pointer to point to it. If a
+ * throttling group with that name does not exist yet, it will be created.
+ *
+ * @blk:       the BlockBackend to insert
+ * @groupname: the name of the group
+ */
+void throttle_group_register_blk(BlockBackend *blk, const char *groupname)
+{
+    int i;
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleState *ts = throttle_group_incref(groupname);
+    ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
+    int clock_type = QEMU_CLOCK_REALTIME;
+
+    if (qtest_enabled()) {
+        /* For testing block IO throttling only */
+        clock_type = QEMU_CLOCK_VIRTUAL;
+    }
+
+    blkp->throttle_state = ts;
+
+    qemu_mutex_lock(&tg->lock);
+    /* If the ThrottleGroup is new set this BlockBackend as the token */
+    for (i = 0; i < 2; i++) {
+        if (!tg->tokens[i]) {
+            tg->tokens[i] = blk;
+        }
+    }
+
+    QLIST_INSERT_HEAD(&tg->head, blkp, round_robin);
+
+    throttle_timers_init(&blkp->throttle_timers,
+                         blk_get_aio_context(blk),
+                         clock_type,
+                         read_timer_cb,
+                         write_timer_cb,
+                         blk);
+
+    qemu_mutex_unlock(&tg->lock);
+}
+
+/* Unregister a BlockBackend from its group, removing it from the list,
+ * destroying the timers and setting the throttle_state pointer to NULL.
+ *
+ * The BlockBackend must not have pending throttled requests, so the caller has
+ * to drain them first.
+ *
+ * The group will be destroyed if it's empty after this operation.
+ *
+ * @blk: the BlockBackend to remove
+ */
+void throttle_group_unregister_blk(BlockBackend *blk)
+{
+    BlockBackendPublic *blkp = blk_get_public(blk);
+    ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
+    int i;
+
+    assert(blkp->pending_reqs[0] == 0 && blkp->pending_reqs[1] == 0);
+    assert(qemu_co_queue_empty(&blkp->throttled_reqs[0]));
+    assert(qemu_co_queue_empty(&blkp->throttled_reqs[1]));
+
+    qemu_mutex_lock(&tg->lock);
+    for (i = 0; i < 2; i++) {
+        if (tg->tokens[i] == blk) {
+            BlockBackend *token = throttle_group_next_blk(blk);
+            /* Take care of the case where this is the last blk in the group */
+            if (token == blk) {
+                token = NULL;
+            }
+            tg->tokens[i] = token;
+        }
+    }
+
+    /* remove the current blk from the list */
+    QLIST_REMOVE(blkp, round_robin);
+    throttle_timers_destroy(&blkp->throttle_timers);
+    qemu_mutex_unlock(&tg->lock);
+
+    throttle_group_unref(&tg->ts);
+    blkp->throttle_state = NULL;
+}
+
+static void throttle_groups_init(void)
+{
+    qemu_mutex_init(&throttle_groups_lock);
+}
+
+block_init(throttle_groups_init);
diff --git a/block/trace-events b/block/trace-events
new file mode 100644
index 0000000..05fa13c
--- /dev/null
+++ b/block/trace-events
@@ -0,0 +1,116 @@
+# See docs/tracing.txt for syntax documentation.
+
+# block.c
+bdrv_open_common(void *bs, const char *filename, int flags, const char *format_name) "bs %p filename \"%s\" flags %#x format_name \"%s\""
+bdrv_lock_medium(void *bs, bool locked) "bs %p locked %d"
+
+# block/block-backend.c
+blk_co_preadv(void *blk, void *bs, int64_t offset, unsigned int bytes, int flags) "blk %p bs %p offset %"PRId64" bytes %u flags %x"
+blk_co_pwritev(void *blk, void *bs, int64_t offset, unsigned int bytes, int flags) "blk %p bs %p offset %"PRId64" bytes %u flags %x"
+
+# block/io.c
+bdrv_aio_pdiscard(void *bs, int64_t offset, int count, void *opaque) "bs %p offset %"PRId64" count %d opaque %p"
+bdrv_aio_flush(void *bs, void *opaque) "bs %p opaque %p"
+bdrv_aio_readv(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
+bdrv_aio_writev(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
+bdrv_co_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
+bdrv_co_writev(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
+bdrv_co_pwrite_zeroes(void *bs, int64_t offset, int count, int flags) "bs %p offset %"PRId64" count %d flags %#x"
+bdrv_co_do_copy_on_readv(void *bs, int64_t offset, unsigned int bytes, int64_t cluster_offset, unsigned int cluster_bytes) "bs %p offset %"PRId64" bytes %u cluster_offset %"PRId64" cluster_bytes %u"
+
+# block/stream.c
+stream_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
+stream_start(void *bs, void *base, void *s, void *co, void *opaque) "bs %p base %p s %p co %p opaque %p"
+
+# block/commit.c
+commit_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
+commit_start(void *bs, void *base, void *top, void *s, void *co, void *opaque) "bs %p base %p top %p s %p co %p opaque %p"
+
+# block/mirror.c
+mirror_start(void *bs, void *s, void *co, void *opaque) "bs %p s %p co %p opaque %p"
+mirror_restart_iter(void *s, int64_t cnt) "s %p dirty count %"PRId64
+mirror_before_flush(void *s) "s %p"
+mirror_before_drain(void *s, int64_t cnt) "s %p dirty count %"PRId64
+mirror_before_sleep(void *s, int64_t cnt, int synced, uint64_t delay_ns) "s %p dirty count %"PRId64" synced %d delay %"PRIu64"ns"
+mirror_one_iteration(void *s, int64_t sector_num, int nb_sectors) "s %p sector_num %"PRId64" nb_sectors %d"
+mirror_iteration_done(void *s, int64_t sector_num, int nb_sectors, int ret) "s %p sector_num %"PRId64" nb_sectors %d ret %d"
+mirror_yield(void *s, int64_t cnt, int buf_free_count, int in_flight) "s %p dirty count %"PRId64" free buffers %d in_flight %d"
+mirror_yield_in_flight(void *s, int64_t sector_num, int in_flight) "s %p sector_num %"PRId64" in_flight %d"
+mirror_yield_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
+mirror_break_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
+
+# block/backup.c
+backup_do_cow_enter(void *job, int64_t start, int64_t sector_num, int nb_sectors) "job %p start %"PRId64" sector_num %"PRId64" nb_sectors %d"
+backup_do_cow_return(void *job, int64_t sector_num, int nb_sectors, int ret) "job %p sector_num %"PRId64" nb_sectors %d ret %d"
+backup_do_cow_skip(void *job, int64_t start) "job %p start %"PRId64
+backup_do_cow_process(void *job, int64_t start) "job %p start %"PRId64
+backup_do_cow_read_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d"
+backup_do_cow_write_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d"
+
+# blockdev.c
+qmp_block_job_cancel(void *job) "job %p"
+qmp_block_job_pause(void *job) "job %p"
+qmp_block_job_resume(void *job) "job %p"
+qmp_block_job_complete(void *job) "job %p"
+block_job_cb(void *bs, void *job, int ret) "bs %p job %p ret %d"
+qmp_block_stream(void *bs, void *job) "bs %p job %p"
+
+# block/raw-win32.c
+# block/raw-posix.c
+paio_submit_co(int64_t offset, int count, int type) "offset %"PRId64" count %d type %d"
+paio_submit(void *acb, void *opaque, int64_t offset, int count, int type) "acb %p opaque %p offset %"PRId64" count %d type %d"
+
+# block/qcow2.c
+qcow2_writev_start_req(void *co, int64_t offset, int bytes) "co %p offset %" PRIx64 " bytes %d"
+qcow2_writev_done_req(void *co, int ret) "co %p ret %d"
+qcow2_writev_start_part(void *co) "co %p"
+qcow2_writev_done_part(void *co, int cur_bytes) "co %p cur_bytes %d"
+qcow2_writev_data(void *co, uint64_t offset) "co %p offset %" PRIx64
+qcow2_pwrite_zeroes_start_req(void *co, int64_t offset, int count) "co %p offset %" PRIx64 " count %d"
+qcow2_pwrite_zeroes(void *co, int64_t offset, int count) "co %p offset %" PRIx64 " count %d"
+
+# block/qcow2-cluster.c
+qcow2_alloc_clusters_offset(void *co, uint64_t offset, int bytes) "co %p offset %" PRIx64 " bytes %d"
+qcow2_handle_copied(void *co, uint64_t guest_offset, uint64_t host_offset, uint64_t bytes) "co %p guest_offset %" PRIx64 " host_offset %" PRIx64 " bytes %" PRIx64
+qcow2_handle_alloc(void *co, uint64_t guest_offset, uint64_t host_offset, uint64_t bytes) "co %p guest_offset %" PRIx64 " host_offset %" PRIx64 " bytes %" PRIx64
+qcow2_do_alloc_clusters_offset(void *co, uint64_t guest_offset, uint64_t host_offset, int nb_clusters) "co %p guest_offset %" PRIx64 " host_offset %" PRIx64 " nb_clusters %d"
+qcow2_cluster_alloc_phys(void *co) "co %p"
+qcow2_cluster_link_l2(void *co, int nb_clusters) "co %p nb_clusters %d"
+
+qcow2_l2_allocate(void *bs, int l1_index) "bs %p l1_index %d"
+qcow2_l2_allocate_get_empty(void *bs, int l1_index) "bs %p l1_index %d"
+qcow2_l2_allocate_write_l2(void *bs, int l1_index) "bs %p l1_index %d"
+qcow2_l2_allocate_write_l1(void *bs, int l1_index) "bs %p l1_index %d"
+qcow2_l2_allocate_done(void *bs, int l1_index, int ret) "bs %p l1_index %d ret %d"
+
+# block/qcow2-cache.c
+qcow2_cache_get(void *co, int c, uint64_t offset, bool read_from_disk) "co %p is_l2_cache %d offset %" PRIx64 " read_from_disk %d"
+qcow2_cache_get_replace_entry(void *co, int c, int i) "co %p is_l2_cache %d index %d"
+qcow2_cache_get_read(void *co, int c, int i) "co %p is_l2_cache %d index %d"
+qcow2_cache_get_done(void *co, int c, int i) "co %p is_l2_cache %d index %d"
+qcow2_cache_flush(void *co, int c) "co %p is_l2_cache %d"
+qcow2_cache_entry_flush(void *co, int c, int i) "co %p is_l2_cache %d index %d"
+
+# block/qed-l2-cache.c
+qed_alloc_l2_cache_entry(void *l2_cache, void *entry) "l2_cache %p entry %p"
+qed_unref_l2_cache_entry(void *entry, int ref) "entry %p ref %d"
+qed_find_l2_cache_entry(void *l2_cache, void *entry, uint64_t offset, int ref) "l2_cache %p entry %p offset %"PRIu64" ref %d"
+
+# block/qed-table.c
+qed_read_table(void *s, uint64_t offset, void *table) "s %p offset %"PRIu64" table %p"
+qed_read_table_cb(void *s, void *table, int ret) "s %p table %p ret %d"
+qed_write_table(void *s, uint64_t offset, void *table, unsigned int index, unsigned int n) "s %p offset %"PRIu64" table %p index %u n %u"
+qed_write_table_cb(void *s, void *table, int flush, int ret) "s %p table %p flush %d ret %d"
+
+# block/qed.c
+qed_need_check_timer_cb(void *s) "s %p"
+qed_start_need_check_timer(void *s) "s %p"
+qed_cancel_need_check_timer(void *s) "s %p"
+qed_aio_complete(void *s, void *acb, int ret) "s %p acb %p ret %d"
+qed_aio_setup(void *s, void *acb, int64_t sector_num, int nb_sectors, void *opaque, int flags) "s %p acb %p sector_num %"PRId64" nb_sectors %d opaque %p flags %#x"
+qed_aio_next_io(void *s, void *acb, int ret, uint64_t cur_pos) "s %p acb %p ret %d cur_pos %"PRIu64
+qed_aio_read_data(void *s, void *acb, int ret, uint64_t offset, size_t len) "s %p acb %p ret %d offset %"PRIu64" len %zu"
+qed_aio_write_data(void *s, void *acb, int ret, uint64_t offset, size_t len) "s %p acb %p ret %d offset %"PRIu64" len %zu"
+qed_aio_write_prefill(void *s, void *acb, uint64_t start, size_t len, uint64_t offset) "s %p acb %p start %"PRIu64" len %zu offset %"PRIu64
+qed_aio_write_postfill(void *s, void *acb, uint64_t start, size_t len, uint64_t offset) "s %p acb %p start %"PRIu64" len %zu offset %"PRIu64
+qed_aio_write_main(void *s, void *acb, int ret, uint64_t offset, size_t len) "s %p acb %p ret %d offset %"PRIu64" len %zu"
diff --git a/block/vdi.c b/block/vdi.c
index 39070b7..8a1cf97 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -49,10 +49,15 @@
  * so this seems to be reasonable.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 #include "migration/migration.h"
+#include "qemu/coroutine.h"
+#include "qemu/cutils.h"
 
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
@@ -196,6 +201,8 @@
     /* VDI header (converted to host endianness). */
     VdiHeader header;
 
+    CoMutex write_lock;
+
     Error *migration_blocker;
 } BDRVVdiState;
 
@@ -487,23 +494,26 @@
 
     bmap_size = header.blocks_in_image * sizeof(uint32_t);
     bmap_size = DIV_ROUND_UP(bmap_size, SECTOR_SIZE);
-    s->bmap = qemu_try_blockalign(bs->file, bmap_size * SECTOR_SIZE);
+    s->bmap = qemu_try_blockalign(bs->file->bs, bmap_size * SECTOR_SIZE);
     if (s->bmap == NULL) {
         ret = -ENOMEM;
         goto fail;
     }
 
-    ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap, bmap_size);
+    ret = bdrv_read(bs->file, s->bmap_sector, (uint8_t *)s->bmap,
+                    bmap_size);
     if (ret < 0) {
         goto fail_free_bmap;
     }
 
     /* Disable migration when vdi images are used */
-    error_set(&s->migration_blocker,
-              QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "vdi", bdrv_get_device_name(bs), "live migration");
+    error_setg(&s->migration_blocker, "The vdi format used by node '%s' "
+               "does not support live migration",
+               bdrv_get_device_or_node_name(bs));
     migrate_add_blocker(s->migration_blocker);
 
+    qemu_co_mutex_init(&s->write_lock);
+
     return 0;
 
  fail_free_bmap:
@@ -520,7 +530,7 @@
 }
 
 static int64_t coroutine_fn vdi_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     /* TODO: Check for too large sector_num (in bdrv_is_allocated or here). */
     BDRVVdiState *s = (BDRVVdiState *)bs->opaque;
@@ -544,116 +554,155 @@
     offset = s->header.offset_data +
                               (uint64_t)bmap_entry * s->block_size +
                               sector_in_block * SECTOR_SIZE;
+    *file = bs->file->bs;
     return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | offset;
 }
 
-static int vdi_co_read(BlockDriverState *bs,
-        int64_t sector_num, uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vdi_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+              QEMUIOVector *qiov, int flags)
 {
     BDRVVdiState *s = bs->opaque;
+    QEMUIOVector local_qiov;
     uint32_t bmap_entry;
     uint32_t block_index;
-    uint32_t sector_in_block;
-    uint32_t n_sectors;
+    uint32_t offset_in_block;
+    uint32_t n_bytes;
+    uint64_t bytes_done = 0;
     int ret = 0;
 
     logout("\n");
 
-    while (ret >= 0 && nb_sectors > 0) {
-        block_index = sector_num / s->block_sectors;
-        sector_in_block = sector_num % s->block_sectors;
-        n_sectors = s->block_sectors - sector_in_block;
-        if (n_sectors > nb_sectors) {
-            n_sectors = nb_sectors;
-        }
+    qemu_iovec_init(&local_qiov, qiov->niov);
 
-        logout("will read %u sectors starting at sector %" PRIu64 "\n",
-               n_sectors, sector_num);
+    while (ret >= 0 && bytes > 0) {
+        block_index = offset / s->block_size;
+        offset_in_block = offset % s->block_size;
+        n_bytes = MIN(bytes, s->block_size - offset_in_block);
+
+        logout("will read %u bytes starting at offset %" PRIu64 "\n",
+               n_bytes, offset);
 
         /* prepare next AIO request */
         bmap_entry = le32_to_cpu(s->bmap[block_index]);
         if (!VDI_IS_ALLOCATED(bmap_entry)) {
             /* Block not allocated, return zeros, no need to wait. */
-            memset(buf, 0, n_sectors * SECTOR_SIZE);
+            qemu_iovec_memset(qiov, bytes_done, 0, n_bytes);
             ret = 0;
         } else {
-            uint64_t offset = s->header.offset_data / SECTOR_SIZE +
-                              (uint64_t)bmap_entry * s->block_sectors +
-                              sector_in_block;
-            ret = bdrv_read(bs->file, offset, buf, n_sectors);
-        }
-        logout("%u sectors read\n", n_sectors);
+            uint64_t data_offset = s->header.offset_data +
+                                   (uint64_t)bmap_entry * s->block_size +
+                                   offset_in_block;
 
-        nb_sectors -= n_sectors;
-        sector_num += n_sectors;
-        buf += n_sectors * SECTOR_SIZE;
+            qemu_iovec_reset(&local_qiov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes_done, n_bytes);
+
+            ret = bdrv_co_preadv(bs->file, data_offset, n_bytes,
+                                 &local_qiov, 0);
+        }
+        logout("%u bytes read\n", n_bytes);
+
+        bytes -= n_bytes;
+        offset += n_bytes;
+        bytes_done += n_bytes;
     }
 
+    qemu_iovec_destroy(&local_qiov);
+
     return ret;
 }
 
-static int vdi_co_write(BlockDriverState *bs,
-        int64_t sector_num, const uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vdi_co_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+               QEMUIOVector *qiov, int flags)
 {
     BDRVVdiState *s = bs->opaque;
+    QEMUIOVector local_qiov;
     uint32_t bmap_entry;
     uint32_t block_index;
-    uint32_t sector_in_block;
-    uint32_t n_sectors;
+    uint32_t offset_in_block;
+    uint32_t n_bytes;
     uint32_t bmap_first = VDI_UNALLOCATED;
     uint32_t bmap_last = VDI_UNALLOCATED;
     uint8_t *block = NULL;
+    uint64_t bytes_done = 0;
     int ret = 0;
 
     logout("\n");
 
-    while (ret >= 0 && nb_sectors > 0) {
-        block_index = sector_num / s->block_sectors;
-        sector_in_block = sector_num % s->block_sectors;
-        n_sectors = s->block_sectors - sector_in_block;
-        if (n_sectors > nb_sectors) {
-            n_sectors = nb_sectors;
-        }
+    qemu_iovec_init(&local_qiov, qiov->niov);
 
-        logout("will write %u sectors starting at sector %" PRIu64 "\n",
-               n_sectors, sector_num);
+    while (ret >= 0 && bytes > 0) {
+        block_index = offset / s->block_size;
+        offset_in_block = offset % s->block_size;
+        n_bytes = MIN(bytes, s->block_size - offset_in_block);
+
+        logout("will write %u bytes starting at offset %" PRIu64 "\n",
+               n_bytes, offset);
 
         /* prepare next AIO request */
         bmap_entry = le32_to_cpu(s->bmap[block_index]);
         if (!VDI_IS_ALLOCATED(bmap_entry)) {
             /* Allocate new block and write to it. */
-            uint64_t offset;
+            uint64_t data_offset;
             bmap_entry = s->header.blocks_allocated;
             s->bmap[block_index] = cpu_to_le32(bmap_entry);
             s->header.blocks_allocated++;
-            offset = s->header.offset_data / SECTOR_SIZE +
-                     (uint64_t)bmap_entry * s->block_sectors;
+            data_offset = s->header.offset_data +
+                          (uint64_t)bmap_entry * s->block_size;
             if (block == NULL) {
                 block = g_malloc(s->block_size);
                 bmap_first = block_index;
             }
             bmap_last = block_index;
             /* Copy data to be written to new block and zero unused parts. */
-            memset(block, 0, sector_in_block * SECTOR_SIZE);
-            memcpy(block + sector_in_block * SECTOR_SIZE,
-                   buf, n_sectors * SECTOR_SIZE);
-            memset(block + (sector_in_block + n_sectors) * SECTOR_SIZE, 0,
-                   (s->block_sectors - n_sectors - sector_in_block) * SECTOR_SIZE);
-            ret = bdrv_write(bs->file, offset, block, s->block_sectors);
+            memset(block, 0, offset_in_block);
+            qemu_iovec_to_buf(qiov, bytes_done, block + offset_in_block,
+                              n_bytes);
+            memset(block + offset_in_block + n_bytes, 0,
+                   s->block_size - n_bytes - offset_in_block);
+
+            /* Note that this coroutine does not yield anywhere from reading the
+             * bmap entry until here, so in regards to all the coroutines trying
+             * to write to this cluster, the one doing the allocation will
+             * always be the first to try to acquire the lock.
+             * Therefore, it is also the first that will actually be able to
+             * acquire the lock and thus the padded cluster is written before
+             * the other coroutines can write to the affected area. */
+            qemu_co_mutex_lock(&s->write_lock);
+            ret = bdrv_pwrite(bs->file, data_offset, block, s->block_size);
+            qemu_co_mutex_unlock(&s->write_lock);
         } else {
-            uint64_t offset = s->header.offset_data / SECTOR_SIZE +
-                              (uint64_t)bmap_entry * s->block_sectors +
-                              sector_in_block;
-            ret = bdrv_write(bs->file, offset, buf, n_sectors);
+            uint64_t data_offset = s->header.offset_data +
+                                   (uint64_t)bmap_entry * s->block_size +
+                                   offset_in_block;
+            qemu_co_mutex_lock(&s->write_lock);
+            /* This lock is only used to make sure the following write operation
+             * is executed after the write issued by the coroutine allocating
+             * this cluster, therefore we do not need to keep it locked.
+             * As stated above, the allocating coroutine will always try to lock
+             * the mutex before all the other concurrent accesses to that
+             * cluster, therefore at this point we can be absolutely certain
+             * that that write operation has returned (there may be other writes
+             * in flight, but they do not concern this very operation). */
+            qemu_co_mutex_unlock(&s->write_lock);
+
+            qemu_iovec_reset(&local_qiov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes_done, n_bytes);
+
+            ret = bdrv_co_pwritev(bs->file, data_offset, n_bytes,
+                                  &local_qiov, 0);
         }
 
-        nb_sectors -= n_sectors;
-        sector_num += n_sectors;
-        buf += n_sectors * SECTOR_SIZE;
+        bytes -= n_bytes;
+        offset += n_bytes;
+        bytes_done += n_bytes;
 
-        logout("%u sectors written\n", n_sectors);
+        logout("%u bytes written\n", n_bytes);
     }
 
+    qemu_iovec_destroy(&local_qiov);
+
     logout("finished data write\n");
     if (ret < 0) {
         return ret;
@@ -664,6 +713,7 @@
         VdiHeader *header = (VdiHeader *) block;
         uint8_t *base;
         uint64_t offset;
+        uint32_t n_sectors;
 
         logout("now writing modified header\n");
         assert(VDI_IS_ALLOCATED(bmap_first));
@@ -705,7 +755,7 @@
     size_t bmap_size;
     int64_t offset = 0;
     Error *local_err = NULL;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
     uint32_t *bmap = NULL;
 
     logout("\n");
@@ -738,13 +788,17 @@
         error_propagate(errp, local_err);
         goto exit;
     }
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* We need enough blocks to store the given disk size,
        so always round up. */
     blocks = DIV_ROUND_UP(bytes, block_size);
@@ -774,7 +828,7 @@
     vdi_header_print(&header);
 #endif
     vdi_header_to_le(&header);
-    ret = bdrv_pwrite_sync(bs, offset, &header, sizeof(header));
+    ret = blk_pwrite(blk, offset, &header, sizeof(header), 0);
     if (ret < 0) {
         error_setg(errp, "Error writing header to %s", filename);
         goto exit;
@@ -795,7 +849,7 @@
                 bmap[i] = VDI_UNALLOCATED;
             }
         }
-        ret = bdrv_pwrite_sync(bs, offset, bmap, bmap_size);
+        ret = blk_pwrite(blk, offset, bmap, bmap_size, 0);
         if (ret < 0) {
             error_setg(errp, "Error writing bmap to %s", filename);
             goto exit;
@@ -804,7 +858,7 @@
     }
 
     if (image_type == VDI_TYPE_STATIC) {
-        ret = bdrv_truncate(bs, offset + blocks * block_size);
+        ret = blk_truncate(blk, offset + blocks * block_size);
         if (ret < 0) {
             error_setg(errp, "Failed to statically allocate %s", filename);
             goto exit;
@@ -812,7 +866,7 @@
     }
 
 exit:
-    bdrv_unref(bs);
+    blk_unref(blk);
     g_free(bmap);
     return ret;
 }
@@ -852,11 +906,6 @@
             .def_value_str = "off"
         },
 #endif
-        {
-            .name = BLOCK_OPT_NOCOW,
-            .type = QEMU_OPT_BOOL,
-            .help = "Turn off copy-on-write (valid only on btrfs)"
-        },
         /* TODO: An additional option to set UUID values might be useful. */
         { /* end of list */ }
     }
@@ -874,9 +923,9 @@
     .bdrv_co_get_block_status = vdi_co_get_block_status,
     .bdrv_make_empty = vdi_make_empty,
 
-    .bdrv_read = vdi_co_read,
+    .bdrv_co_preadv     = vdi_co_preadv,
 #if defined(CONFIG_VDI_WRITE)
-    .bdrv_write = vdi_co_write,
+    .bdrv_co_pwritev    = vdi_co_pwritev,
 #endif
 
     .bdrv_get_info = vdi_get_info,
diff --git a/block/vhdx-endian.c b/block/vhdx-endian.c
index 0640d3f..c306b90 100644
--- a/block/vhdx-endian.c
+++ b/block/vhdx-endian.c
@@ -15,8 +15,10 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "qemu/bswap.h"
 #include "block/vhdx.h"
 
 #include <uuid/uuid.h>
diff --git a/block/vhdx-log.c b/block/vhdx-log.c
index 6547bec..02eb104 100644
--- a/block/vhdx-log.c
+++ b/block/vhdx-log.c
@@ -17,9 +17,13 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "qemu/error-report.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 #include "block/vhdx.h"
 
 
@@ -190,7 +194,8 @@
             /* full */
             break;
         }
-        ret = bdrv_pwrite(bs->file, offset, buffer_tmp, VHDX_LOG_SECTOR_SIZE);
+        ret = bdrv_pwrite(bs->file, offset, buffer_tmp,
+                          VHDX_LOG_SECTOR_SIZE);
         if (ret < 0) {
             goto exit;
         }
@@ -352,7 +357,7 @@
     }
 
     desc_sectors = vhdx_compute_desc_sectors(hdr.descriptor_count);
-    desc_entries = qemu_try_blockalign(bs->file,
+    desc_entries = qemu_try_blockalign(bs->file->bs,
                                        desc_sectors * VHDX_LOG_SECTOR_SIZE);
     if (desc_entries == NULL) {
         ret = -ENOMEM;
@@ -508,7 +513,7 @@
         /* if the log shows a FlushedFileOffset larger than our current file
          * size, then that means the file has been truncated / corrupted, and
          * we must refused to open it / use it */
-        if (hdr_tmp.flushed_file_offset > bdrv_getlength(bs->file)) {
+        if (hdr_tmp.flushed_file_offset > bdrv_getlength(bs->file->bs)) {
             ret = -EINVAL;
             goto exit;
         }
@@ -538,12 +543,12 @@
                 goto exit;
             }
         }
-        if (bdrv_getlength(bs->file) < desc_entries->hdr.last_file_offset) {
+        if (bdrv_getlength(bs->file->bs) < desc_entries->hdr.last_file_offset) {
             new_file_size = desc_entries->hdr.last_file_offset;
             if (new_file_size % (1024*1024)) {
                 /* round up to nearest 1MB boundary */
                 new_file_size = ((new_file_size >> 20) + 1) << 20;
-                bdrv_truncate(bs->file, new_file_size);
+                bdrv_truncate(bs->file->bs, new_file_size);
             }
         }
         qemu_vfree(desc_entries);
@@ -782,12 +787,13 @@
     if (logs.valid) {
         if (bs->read_only) {
             ret = -EPERM;
-            error_setg_errno(errp, EPERM,
-                             "VHDX image file '%s' opened read-only, but "
-                             "contains a log that needs to be replayed.  To "
-                             "replay the log, execute:\n qemu-img check -r "
-                             "all '%s'",
-                             bs->filename, bs->filename);
+            error_setg(errp,
+                       "VHDX image file '%s' opened read-only, but "
+                       "contains a log that needs to be replayed",
+                       bs->filename);
+            error_append_hint(errp,  "To replay the log, run:\n"
+                              "qemu-img check -r all '%s'\n",
+                              bs->filename);
             goto exit;
         }
         /* now flush the log */
@@ -907,8 +913,8 @@
                 .sequence_number     = s->log.sequence,
                 .descriptor_count    = sectors,
                 .reserved            = 0,
-                .flushed_file_offset = bdrv_getlength(bs->file),
-                .last_file_offset    = bdrv_getlength(bs->file),
+                .flushed_file_offset = bdrv_getlength(bs->file->bs),
+                .last_file_offset    = bdrv_getlength(bs->file->bs),
               };
 
     new_hdr.log_guid = header->log_guid;
diff --git a/block/vhdx.c b/block/vhdx.c
index 12bfe75..75ef2b1 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -15,15 +15,18 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "qemu/crc32c.h"
+#include "qemu/bswap.h"
 #include "block/vhdx.h"
 #include "migration/migration.h"
 
 #include <uuid/uuid.h>
-#include <glib.h>
 
 /* Options for VHDX creation */
 
@@ -263,10 +266,10 @@
 
 static void vhdx_set_shift_bits(BDRVVHDXState *s)
 {
-    s->logical_sector_size_bits = 31 - clz32(s->logical_sector_size);
-    s->sectors_per_block_bits =   31 - clz32(s->sectors_per_block);
-    s->chunk_ratio_bits =         63 - clz64(s->chunk_ratio);
-    s->block_size_bits =          31 - clz32(s->block_size);
+    s->logical_sector_size_bits = ctz32(s->logical_sector_size);
+    s->sectors_per_block_bits =   ctz32(s->sectors_per_block);
+    s->chunk_ratio_bits =         ctz64(s->chunk_ratio);
+    s->block_size_bits =          ctz32(s->block_size);
 }
 
 /*
@@ -295,9 +298,10 @@
  * and then update the header checksum.  Header is converted to proper
  * endianness before being written to the specified file offset
  */
-static int vhdx_write_header(BlockDriverState *bs_file, VHDXHeader *hdr,
+static int vhdx_write_header(BdrvChild *file, VHDXHeader *hdr,
                              uint64_t offset, bool read)
 {
+    BlockDriverState *bs_file = file->bs;
     uint8_t *buffer = NULL;
     int ret;
     VHDXHeader *header_le;
@@ -312,7 +316,7 @@
     buffer = qemu_blockalign(bs_file, VHDX_HEADER_SIZE);
     if (read) {
         /* if true, we can't assume the extra reserved bytes are 0 */
-        ret = bdrv_pread(bs_file, offset, buffer, VHDX_HEADER_SIZE);
+        ret = bdrv_pread(file, offset, buffer, VHDX_HEADER_SIZE);
         if (ret < 0) {
             goto exit;
         }
@@ -326,7 +330,7 @@
     vhdx_header_le_export(hdr, header_le);
     vhdx_update_checksum(buffer, VHDX_HEADER_SIZE,
                          offsetof(VHDXHeader, checksum));
-    ret = bdrv_pwrite_sync(bs_file, offset, header_le, sizeof(VHDXHeader));
+    ret = bdrv_pwrite_sync(file, offset, header_le, sizeof(VHDXHeader));
 
 exit:
     qemu_vfree(buffer);
@@ -427,7 +431,8 @@
     /* We have to read the whole VHDX_HEADER_SIZE instead of
      * sizeof(VHDXHeader), because the checksum is over the whole
      * region */
-    ret = bdrv_pread(bs->file, VHDX_HEADER1_OFFSET, buffer, VHDX_HEADER_SIZE);
+    ret = bdrv_pread(bs->file, VHDX_HEADER1_OFFSET, buffer,
+                     VHDX_HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
@@ -443,7 +448,8 @@
         }
     }
 
-    ret = bdrv_pread(bs->file, VHDX_HEADER2_OFFSET, buffer, VHDX_HEADER_SIZE);
+    ret = bdrv_pread(bs->file, VHDX_HEADER2_OFFSET, buffer,
+                     VHDX_HEADER_SIZE);
     if (ret < 0) {
         goto fail;
     }
@@ -854,14 +860,8 @@
 {
     uint32_t data_blocks_cnt, bitmap_blocks_cnt;
 
-    data_blocks_cnt = s->virtual_disk_size >> s->block_size_bits;
-    if (s->virtual_disk_size - (data_blocks_cnt << s->block_size_bits)) {
-        data_blocks_cnt++;
-    }
-    bitmap_blocks_cnt = data_blocks_cnt >> s->chunk_ratio_bits;
-    if (data_blocks_cnt - (bitmap_blocks_cnt << s->chunk_ratio_bits)) {
-        bitmap_blocks_cnt++;
-    }
+    data_blocks_cnt = DIV_ROUND_UP(s->virtual_disk_size, s->block_size);
+    bitmap_blocks_cnt = DIV_ROUND_UP(data_blocks_cnt, s->chunk_ratio);
 
     if (s->parent_entries) {
         s->bat_entries = bitmap_blocks_cnt * (s->chunk_ratio + 1);
@@ -959,7 +959,7 @@
     }
 
     /* s->bat is freed in vhdx_close() */
-    s->bat = qemu_try_blockalign(bs->file, s->bat_rt.length);
+    s->bat = qemu_try_blockalign(bs->file->bs, s->bat_rt.length);
     if (s->bat == NULL) {
         ret = -ENOMEM;
         goto fail;
@@ -1002,9 +1002,9 @@
     /* TODO: differencing files */
 
     /* Disable migration when VHDX images are used */
-    error_set(&s->migration_blocker,
-            QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-            "vhdx", bdrv_get_device_name(bs), "live migration");
+    error_setg(&s->migration_blocker, "The vhdx format used by node '%s' "
+               "does not support live migration",
+               bdrv_get_device_or_node_name(bs));
     migrate_add_blocker(s->migration_blocker);
 
     return 0;
@@ -1109,8 +1109,9 @@
             /* check the payload block state */
             switch (s->bat[sinfo.bat_idx] & VHDX_BAT_STATE_BIT_MASK) {
             case PAYLOAD_BLOCK_NOT_PRESENT: /* fall through */
-            case PAYLOAD_BLOCK_UNDEFINED:   /* fall through */
-            case PAYLOAD_BLOCK_UNMAPPED:    /* fall through */
+            case PAYLOAD_BLOCK_UNDEFINED:
+            case PAYLOAD_BLOCK_UNMAPPED:
+            case PAYLOAD_BLOCK_UNMAPPED_v095:
             case PAYLOAD_BLOCK_ZERO:
                 /* return zero */
                 qemu_iovec_memset(&hd_qiov, 0, 0, sinfo.bytes_avail);
@@ -1155,12 +1156,12 @@
 static int vhdx_allocate_block(BlockDriverState *bs, BDRVVHDXState *s,
                                     uint64_t *new_offset)
 {
-    *new_offset = bdrv_getlength(bs->file);
+    *new_offset = bdrv_getlength(bs->file->bs);
 
     /* per the spec, the address for a block is in units of 1MB */
     *new_offset = ROUND_UP(*new_offset, 1024 * 1024);
 
-    return bdrv_truncate(bs->file, *new_offset + s->block_size);
+    return bdrv_truncate(bs->file->bs, *new_offset + s->block_size);
 }
 
 /*
@@ -1173,7 +1174,18 @@
 {
     /* The BAT entry is a uint64, with 44 bits for the file offset in units of
      * 1MB, and 3 bits for the block state. */
-    s->bat[sinfo->bat_idx]  = sinfo->file_offset;
+    if ((state == PAYLOAD_BLOCK_ZERO)        ||
+        (state == PAYLOAD_BLOCK_UNDEFINED)   ||
+        (state == PAYLOAD_BLOCK_NOT_PRESENT) ||
+        (state == PAYLOAD_BLOCK_UNMAPPED)) {
+        s->bat[sinfo->bat_idx]  = 0;  /* For PAYLOAD_BLOCK_ZERO, the
+                                         FileOffsetMB field is denoted as
+                                         'reserved' in the v1.0 spec.  If it is
+                                         non-zero, MS Hyper-V will fail to read
+                                         the disk image */
+    } else {
+        s->bat[sinfo->bat_idx]  = sinfo->file_offset;
+    }
 
     s->bat[sinfo->bat_idx] |= state & VHDX_BAT_STATE_BIT_MASK;
 
@@ -1248,7 +1260,7 @@
                 /* Queue another write of zero buffers if the underlying file
                  * does not zero-fill on file extension */
 
-                if (bdrv_has_zero_init(bs->file) == 0) {
+                if (bdrv_has_zero_init(bs->file->bs) == 0) {
                     use_zero_buffers = true;
 
                     /* zero fill the front, if any */
@@ -1257,7 +1269,7 @@
                         iov1.iov_base = qemu_blockalign(bs, iov1.iov_len);
                         memset(iov1.iov_base, 0, iov1.iov_len);
                         qemu_iovec_concat_iov(&hd_qiov, &iov1, 1, 0,
-                                              sinfo.block_offset);
+                                              iov1.iov_len);
                         sectors_to_write += iov1.iov_len >> BDRV_SECTOR_BITS;
                     }
 
@@ -1273,15 +1285,15 @@
                         iov2.iov_base = qemu_blockalign(bs, iov2.iov_len);
                         memset(iov2.iov_base, 0, iov2.iov_len);
                         qemu_iovec_concat_iov(&hd_qiov, &iov2, 1, 0,
-                                              sinfo.block_offset);
+                                              iov2.iov_len);
                         sectors_to_write += iov2.iov_len >> BDRV_SECTOR_BITS;
                     }
                 }
-
                 /* fall through */
             case PAYLOAD_BLOCK_NOT_PRESENT: /* fall through */
-            case PAYLOAD_BLOCK_UNMAPPED:    /* fall through */
-            case PAYLOAD_BLOCK_UNDEFINED:   /* fall through */
+            case PAYLOAD_BLOCK_UNMAPPED:
+            case PAYLOAD_BLOCK_UNMAPPED_v095:
+            case PAYLOAD_BLOCK_UNDEFINED:
                 bat_prior_offset = sinfo.file_offset;
                 ret = vhdx_allocate_block(bs, s, &sinfo.file_offset);
                 if (ret < 0) {
@@ -1376,9 +1388,11 @@
  * There are 2 headers, and the highest sequence number will represent
  * the active header
  */
-static int vhdx_create_new_headers(BlockDriverState *bs, uint64_t image_size,
+static int vhdx_create_new_headers(BlockBackend *blk, uint64_t image_size,
                                    uint32_t log_size)
 {
+    BlockDriverState *bs = blk_bs(blk);
+    BdrvChild *child;
     int ret = 0;
     VHDXHeader *hdr = NULL;
 
@@ -1393,12 +1407,18 @@
     vhdx_guid_generate(&hdr->file_write_guid);
     vhdx_guid_generate(&hdr->data_write_guid);
 
-    ret = vhdx_write_header(bs, hdr, VHDX_HEADER1_OFFSET, false);
+    /* XXX Ugly way to get blk->root, but that's a feature, not a bug. This
+     * hack makes it obvious that vhdx_write_header() bypasses the BlockBackend
+     * here, which it really shouldn't be doing. */
+    child = QLIST_FIRST(&bs->parents);
+    assert(!QLIST_NEXT(child, next_parent));
+
+    ret = vhdx_write_header(child, hdr, VHDX_HEADER1_OFFSET, false);
     if (ret < 0) {
         goto exit;
     }
     hdr->sequence_number++;
-    ret = vhdx_write_header(bs, hdr, VHDX_HEADER2_OFFSET, false);
+    ret = vhdx_write_header(child, hdr, VHDX_HEADER2_OFFSET, false);
     if (ret < 0) {
         goto exit;
     }
@@ -1431,7 +1451,7 @@
  * The first 64KB of the Metadata section is reserved for the metadata
  * header and entries; beyond that, the metadata items themselves reside.
  */
-static int vhdx_create_new_metadata(BlockDriverState *bs,
+static int vhdx_create_new_metadata(BlockBackend *blk,
                                     uint64_t image_size,
                                     uint32_t block_size,
                                     uint32_t sector_size,
@@ -1442,7 +1462,7 @@
     uint32_t offset = 0;
     void *buffer = NULL;
     void *entry_buffer;
-    VHDXMetadataTableHeader *md_table;;
+    VHDXMetadataTableHeader *md_table;
     VHDXMetadataTableEntry  *md_table_entry;
 
     /* Metadata entries */
@@ -1527,13 +1547,13 @@
                                    VHDX_META_FLAGS_IS_VIRTUAL_DISK;
     vhdx_metadata_entry_le_export(&md_table_entry[4]);
 
-    ret = bdrv_pwrite(bs, metadata_offset, buffer, VHDX_HEADER_BLOCK_SIZE);
+    ret = blk_pwrite(blk, metadata_offset, buffer, VHDX_HEADER_BLOCK_SIZE, 0);
     if (ret < 0) {
         goto exit;
     }
 
-    ret = bdrv_pwrite(bs, metadata_offset + (64 * KiB), entry_buffer,
-                      VHDX_METADATA_ENTRY_BUFFER_SIZE);
+    ret = blk_pwrite(blk, metadata_offset + (64 * KiB), entry_buffer,
+                     VHDX_METADATA_ENTRY_BUFFER_SIZE, 0);
     if (ret < 0) {
         goto exit;
     }
@@ -1553,7 +1573,7 @@
  *  Fixed images: default state of the BAT is fully populated, with
  *                file offsets and state PAYLOAD_BLOCK_FULLY_PRESENT.
  */
-static int vhdx_create_bat(BlockDriverState *bs, BDRVVHDXState *s,
+static int vhdx_create_bat(BlockBackend *blk, BDRVVHDXState *s,
                            uint64_t image_size, VHDXImageType type,
                            bool use_zero_blocks, uint64_t file_offset,
                            uint32_t length)
@@ -1577,12 +1597,12 @@
     if (type == VHDX_TYPE_DYNAMIC) {
         /* All zeroes, so we can just extend the file - the end of the BAT
          * is the furthest thing we have written yet */
-        ret = bdrv_truncate(bs, data_file_offset);
+        ret = blk_truncate(blk, data_file_offset);
         if (ret < 0) {
             goto exit;
         }
     } else if (type == VHDX_TYPE_FIXED) {
-        ret = bdrv_truncate(bs, data_file_offset + image_size);
+        ret = blk_truncate(blk, data_file_offset + image_size);
         if (ret < 0) {
             goto exit;
         }
@@ -1593,7 +1613,7 @@
 
     if (type == VHDX_TYPE_FIXED ||
                 use_zero_blocks ||
-                bdrv_has_zero_init(bs) == 0) {
+                bdrv_has_zero_init(blk_bs(blk)) == 0) {
         /* for a fixed file, the default BAT entry is not zero */
         s->bat = g_try_malloc0(length);
         if (length && s->bat == NULL) {
@@ -1609,12 +1629,12 @@
             sinfo.file_offset = data_file_offset +
                                 (sector_num << s->logical_sector_size_bits);
             sinfo.file_offset = ROUND_UP(sinfo.file_offset, MiB);
-            vhdx_update_bat_table_entry(bs, s, &sinfo, &unused, &unused,
+            vhdx_update_bat_table_entry(blk_bs(blk), s, &sinfo, &unused, &unused,
                                         block_state);
             cpu_to_le64s(&s->bat[sinfo.bat_idx]);
             sector_num += s->sectors_per_block;
         }
-        ret = bdrv_pwrite(bs, file_offset, s->bat, length);
+        ret = blk_pwrite(blk, file_offset, s->bat, length, 0);
         if (ret < 0) {
             goto exit;
         }
@@ -1634,7 +1654,7 @@
  * to create the BAT itself, we will also cause the BAT to be
  * created.
  */
-static int vhdx_create_new_region_table(BlockDriverState *bs,
+static int vhdx_create_new_region_table(BlockBackend *blk,
                                         uint64_t image_size,
                                         uint32_t block_size,
                                         uint32_t sector_size,
@@ -1709,21 +1729,21 @@
 
     /* The region table gives us the data we need to create the BAT,
      * so do that now */
-    ret = vhdx_create_bat(bs, s, image_size, type, use_zero_blocks,
+    ret = vhdx_create_bat(blk, s, image_size, type, use_zero_blocks,
                           bat_file_offset, bat_length);
     if (ret < 0) {
         goto exit;
     }
 
     /* Now write out the region headers to disk */
-    ret = bdrv_pwrite(bs, VHDX_REGION_TABLE_OFFSET, buffer,
-                      VHDX_HEADER_BLOCK_SIZE);
+    ret = blk_pwrite(blk, VHDX_REGION_TABLE_OFFSET, buffer,
+                     VHDX_HEADER_BLOCK_SIZE, 0);
     if (ret < 0) {
         goto exit;
     }
 
-    ret = bdrv_pwrite(bs, VHDX_REGION_TABLE2_OFFSET, buffer,
-                      VHDX_HEADER_BLOCK_SIZE);
+    ret = blk_pwrite(blk, VHDX_REGION_TABLE2_OFFSET, buffer,
+                     VHDX_HEADER_BLOCK_SIZE, 0);
     if (ret < 0) {
         goto exit;
     }
@@ -1763,7 +1783,7 @@
 
     gunichar2 *creator = NULL;
     glong creator_items;
-    BlockDriverState *bs;
+    BlockBackend *blk;
     char *type = NULL;
     VHDXImageType image_type;
     Error *local_err = NULL;
@@ -1773,7 +1793,7 @@
     log_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_LOG_SIZE, 0);
     block_size = qemu_opt_get_size_del(opts, VHDX_BLOCK_OPT_BLOCK_SIZE, 0);
     type = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
-    use_zero_blocks = qemu_opt_get_bool_del(opts, VHDX_BLOCK_OPT_ZERO, false);
+    use_zero_blocks = qemu_opt_get_bool_del(opts, VHDX_BLOCK_OPT_ZERO, true);
 
     if (image_size > VHDX_MAX_IMAGE_SIZE) {
         error_setg_errno(errp, EINVAL, "Image size too large; max of 64TB");
@@ -1828,14 +1848,16 @@
         goto exit;
     }
 
-    bs = NULL;
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /* Create (A) */
 
     /* The creator field is optional, but may be useful for
@@ -1843,13 +1865,14 @@
     creator = g_utf8_to_utf16("QEMU v" QEMU_VERSION, -1, NULL,
                               &creator_items, NULL);
     signature = cpu_to_le64(VHDX_FILE_SIGNATURE);
-    ret = bdrv_pwrite(bs, VHDX_FILE_ID_OFFSET, &signature, sizeof(signature));
+    ret = blk_pwrite(blk, VHDX_FILE_ID_OFFSET, &signature, sizeof(signature),
+                     0);
     if (ret < 0) {
         goto delete_and_exit;
     }
     if (creator) {
-        ret = bdrv_pwrite(bs, VHDX_FILE_ID_OFFSET + sizeof(signature),
-                          creator, creator_items * sizeof(gunichar2));
+        ret = blk_pwrite(blk, VHDX_FILE_ID_OFFSET + sizeof(signature),
+                         creator, creator_items * sizeof(gunichar2), 0);
         if (ret < 0) {
             goto delete_and_exit;
         }
@@ -1857,13 +1880,13 @@
 
 
     /* Creates (B),(C) */
-    ret = vhdx_create_new_headers(bs, image_size, log_size);
+    ret = vhdx_create_new_headers(blk, image_size, log_size);
     if (ret < 0) {
         goto delete_and_exit;
     }
 
     /* Creates (D),(E),(G) explicitly. (F) created as by-product */
-    ret = vhdx_create_new_region_table(bs, image_size, block_size, 512,
+    ret = vhdx_create_new_region_table(blk, image_size, block_size, 512,
                                        log_size, use_zero_blocks, image_type,
                                        &metadata_offset);
     if (ret < 0) {
@@ -1871,7 +1894,7 @@
     }
 
     /* Creates (H) */
-    ret = vhdx_create_new_metadata(bs, image_size, block_size, 512,
+    ret = vhdx_create_new_metadata(blk, image_size, block_size, 512,
                                    metadata_offset, image_type);
     if (ret < 0) {
         goto delete_and_exit;
@@ -1879,7 +1902,7 @@
 
 
 delete_and_exit:
-    bdrv_unref(bs);
+    blk_unref(blk);
 exit:
     g_free(type);
     g_free(creator);
@@ -1935,7 +1958,9 @@
        {
            .name = VHDX_BLOCK_OPT_ZERO,
            .type = QEMU_OPT_BOOL,
-           .help = "Force use of payload blocks of type 'ZERO'.  Non-standard."
+           .help = "Force use of payload blocks of type 'ZERO'. "\
+                   "Non-standard, but default.  Do not set to 'off' when "\
+                   "using 'qemu-img convert' with subformat=dynamic."
        },
        { NULL }
     }
@@ -1953,6 +1978,7 @@
     .bdrv_create            = vhdx_create,
     .bdrv_get_info          = vhdx_get_info,
     .bdrv_check             = vhdx_check,
+    .bdrv_has_zero_init     = bdrv_has_zero_init_1,
 
     .create_opts            = &vhdx_create_opts,
 };
diff --git a/block/vhdx.h b/block/vhdx.h
index b4a12a0..7003ab7 100644
--- a/block/vhdx.h
+++ b/block/vhdx.h
@@ -226,7 +226,8 @@
 #define PAYLOAD_BLOCK_NOT_PRESENT       0
 #define PAYLOAD_BLOCK_UNDEFINED         1
 #define PAYLOAD_BLOCK_ZERO              2
-#define PAYLOAD_BLOCK_UNMAPPED          5
+#define PAYLOAD_BLOCK_UNMAPPED          3
+#define PAYLOAD_BLOCK_UNMAPPED_v095     5
 #define PAYLOAD_BLOCK_FULLY_PRESENT     6
 #define PAYLOAD_BLOCK_PARTIALLY_PRESENT 7
 
diff --git a/block/vmdk.c b/block/vmdk.c
index 2cbfd3e..46d474e 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -23,10 +23,16 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 #include "migration/migration.h"
+#include "qemu/cutils.h"
 #include <zlib.h>
 
 #define VMDK3_MAGIC (('C' << 24) | ('O' << 16) | ('W' << 8) | 'D')
@@ -84,7 +90,7 @@
 #define L2_CACHE_SIZE 16
 
 typedef struct VmdkExtent {
-    BlockDriverState *file;
+    BdrvChild *file;
     bool flat;
     bool compressed;
     bool has_marker;
@@ -219,7 +225,7 @@
         g_free(e->l1_backup_table);
         g_free(e->type);
         if (e->file != bs->file) {
-            bdrv_unref(e->file);
+            bdrv_unref_child(bs, e->file);
         }
     }
     g_free(s->extents);
@@ -238,15 +244,17 @@
 
 static uint32_t vmdk_read_cid(BlockDriverState *bs, int parent)
 {
-    char desc[DESC_SIZE];
+    char *desc;
     uint32_t cid = 0xffffffff;
     const char *p_name, *cid_str;
     size_t cid_str_size;
     BDRVVmdkState *s = bs->opaque;
     int ret;
 
+    desc = g_malloc0(DESC_SIZE);
     ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
+        g_free(desc);
         return 0;
     }
 
@@ -265,50 +273,55 @@
         sscanf(p_name, "%" SCNx32, &cid);
     }
 
+    g_free(desc);
     return cid;
 }
 
 static int vmdk_write_cid(BlockDriverState *bs, uint32_t cid)
 {
-    char desc[DESC_SIZE], tmp_desc[DESC_SIZE];
+    char *desc, *tmp_desc;
     char *p_name, *tmp_str;
     BDRVVmdkState *s = bs->opaque;
-    int ret;
+    int ret = 0;
 
+    desc = g_malloc0(DESC_SIZE);
+    tmp_desc = g_malloc0(DESC_SIZE);
     ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
 
     desc[DESC_SIZE - 1] = '\0';
     tmp_str = strstr(desc, "parentCID");
     if (tmp_str == NULL) {
-        return -EINVAL;
+        ret = -EINVAL;
+        goto out;
     }
 
-    pstrcpy(tmp_desc, sizeof(tmp_desc), tmp_str);
+    pstrcpy(tmp_desc, DESC_SIZE, tmp_str);
     p_name = strstr(desc, "CID");
     if (p_name != NULL) {
         p_name += sizeof("CID");
-        snprintf(p_name, sizeof(desc) - (p_name - desc), "%" PRIx32 "\n", cid);
-        pstrcat(desc, sizeof(desc), tmp_desc);
+        snprintf(p_name, DESC_SIZE - (p_name - desc), "%" PRIx32 "\n", cid);
+        pstrcat(desc, DESC_SIZE, tmp_desc);
     }
 
     ret = bdrv_pwrite_sync(bs->file, s->desc_offset, desc, DESC_SIZE);
-    if (ret < 0) {
-        return ret;
-    }
 
-    return 0;
+out:
+    g_free(desc);
+    g_free(tmp_desc);
+    return ret;
 }
 
 static int vmdk_is_cid_valid(BlockDriverState *bs)
 {
     BDRVVmdkState *s = bs->opaque;
-    BlockDriverState *p_bs = bs->backing_hd;
     uint32_t cur_pcid;
 
-    if (!s->cid_checked && p_bs) {
+    if (!s->cid_checked && bs->backing) {
+        BlockDriverState *p_bs = bs->backing->bs;
+
         cur_pcid = vmdk_read_cid(p_bs, 0);
         if (s->parent_cid != cur_pcid) {
             /* CID not valid */
@@ -320,51 +333,28 @@
     return 1;
 }
 
-/* Queue extents, if any, for reopen() */
+/* We have nothing to do for VMDK reopen, stubs just return success */
 static int vmdk_reopen_prepare(BDRVReopenState *state,
                                BlockReopenQueue *queue, Error **errp)
 {
-    BDRVVmdkState *s;
-    int ret = -1;
-    int i;
-    VmdkExtent *e;
-
     assert(state != NULL);
     assert(state->bs != NULL);
-
-    if (queue == NULL) {
-        error_setg(errp, "No reopen queue for VMDK extents");
-        goto exit;
-    }
-
-    s = state->bs->opaque;
-
-    assert(s != NULL);
-
-    for (i = 0; i < s->num_extents; i++) {
-        e = &s->extents[i];
-        if (e->file != state->bs->file) {
-            bdrv_reopen_queue(queue, e->file, state->flags);
-        }
-    }
-    ret = 0;
-
-exit:
-    return ret;
+    return 0;
 }
 
 static int vmdk_parent_open(BlockDriverState *bs)
 {
     char *p_name;
-    char desc[DESC_SIZE + 1];
+    char *desc;
     BDRVVmdkState *s = bs->opaque;
     int ret;
 
-    desc[DESC_SIZE] = '\0';
+    desc = g_malloc0(DESC_SIZE + 1);
     ret = bdrv_pread(bs->file, s->desc_offset, desc, DESC_SIZE);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
+    ret = 0;
 
     p_name = strstr(desc, "parentFileNameHint");
     if (p_name != NULL) {
@@ -373,22 +363,26 @@
         p_name += sizeof("parentFileNameHint") + 1;
         end_name = strchr(p_name, '\"');
         if (end_name == NULL) {
-            return -EINVAL;
+            ret = -EINVAL;
+            goto out;
         }
         if ((end_name - p_name) > sizeof(bs->backing_file) - 1) {
-            return -EINVAL;
+            ret = -EINVAL;
+            goto out;
         }
 
         pstrcpy(bs->backing_file, end_name - p_name + 1, p_name);
     }
 
-    return 0;
+out:
+    g_free(desc);
+    return ret;
 }
 
 /* Create and append extent to the extent array. Return the added VmdkExtent
  * address. return NULL if allocation failed. */
 static int vmdk_add_extent(BlockDriverState *bs,
-                           BlockDriverState *file, bool flat, int64_t sectors,
+                           BdrvChild *file, bool flat, int64_t sectors,
                            int64_t l1_offset, int64_t l1_backup_offset,
                            uint32_t l1_size,
                            int l2_size, uint64_t cluster_sectors,
@@ -413,7 +407,7 @@
         return -EFBIG;
     }
 
-    nb_sectors = bdrv_nb_sectors(file);
+    nb_sectors = bdrv_nb_sectors(file->bs);
     if (nb_sectors < 0) {
         return nb_sectors;
     }
@@ -450,7 +444,8 @@
                             Error **errp)
 {
     int ret;
-    int l1_size, i;
+    size_t l1_size;
+    int i;
 
     /* read the L1 table */
     l1_size = extent->l1_size * sizeof(uint32_t);
@@ -466,7 +461,7 @@
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read l1 table from extent '%s'",
-                         extent->file->filename);
+                         extent->file->bs->filename);
         goto fail_l1;
     }
     for (i = 0; i < extent->l1_size; i++) {
@@ -486,7 +481,7 @@
         if (ret < 0) {
             error_setg_errno(errp, -ret,
                              "Could not read l1 backup table from extent '%s'",
-                             extent->file->filename);
+                             extent->file->bs->filename);
             goto fail_l1b;
         }
         for (i = 0; i < extent->l1_size; i++) {
@@ -505,7 +500,7 @@
 }
 
 static int vmdk_open_vmfs_sparse(BlockDriverState *bs,
-                                 BlockDriverState *file,
+                                 BdrvChild *file,
                                  int flags, Error **errp)
 {
     int ret;
@@ -517,12 +512,12 @@
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read header from file '%s'",
-                         file->filename);
+                         file->bs->filename);
         return ret;
     }
     ret = vmdk_add_extent(bs, file, false,
                           le32_to_cpu(header.disk_sectors),
-                          le32_to_cpu(header.l1dir_offset) << 9,
+                          (int64_t)le32_to_cpu(header.l1dir_offset) << 9,
                           0,
                           le32_to_cpu(header.l1dir_size),
                           4096,
@@ -541,23 +536,30 @@
 }
 
 static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
-                               Error **errp);
+                               QDict *options, Error **errp);
 
-static char *vmdk_read_desc(BlockDriverState *file, uint64_t desc_offset,
-                            Error **errp)
+static char *vmdk_read_desc(BdrvChild *file, uint64_t desc_offset, Error **errp)
 {
     int64_t size;
     char *buf;
     int ret;
 
-    size = bdrv_getlength(file);
+    size = bdrv_getlength(file->bs);
     if (size < 0) {
         error_setg_errno(errp, -size, "Could not access file");
         return NULL;
     }
 
-    size = MIN(size, 1 << 20);  /* avoid unbounded allocation */
-    buf = g_malloc0(size + 1);
+    if (size < 4) {
+        /* Both descriptor file and sparse image must be much larger than 4
+         * bytes, also callers of vmdk_read_desc want to compare the first 4
+         * bytes with VMDK4_MAGIC, let's error out if less is read. */
+        error_setg(errp, "File is too small, not a valid image");
+        return NULL;
+    }
+
+    size = MIN(size, (1 << 20) - 1);  /* avoid unbounded allocation */
+    buf = g_malloc(size + 1);
 
     ret = bdrv_pread(file, desc_offset, buf, size);
     if (ret < 0) {
@@ -565,13 +567,14 @@
         g_free(buf);
         return NULL;
     }
+    buf[ret] = 0;
 
     return buf;
 }
 
 static int vmdk_open_vmdk4(BlockDriverState *bs,
-                           BlockDriverState *file,
-                           int flags, Error **errp)
+                           BdrvChild *file,
+                           int flags, QDict *options, Error **errp)
 {
     int ret;
     uint32_t magic;
@@ -580,12 +583,13 @@
     VmdkExtent *extent;
     BDRVVmdkState *s = bs->opaque;
     int64_t l1_backup_offset = 0;
+    bool compressed;
 
     ret = bdrv_pread(file, sizeof(magic), &header, sizeof(header));
     if (ret < 0) {
         error_setg_errno(errp, -ret,
                          "Could not read header from file '%s'",
-                         file->filename);
+                         file->bs->filename);
         return -EINVAL;
     }
     if (header.capacity == 0) {
@@ -595,7 +599,7 @@
             if (!buf) {
                 return -EINVAL;
             }
-            ret = vmdk_open_desc_file(bs, flags, buf, errp);
+            ret = vmdk_open_desc_file(bs, flags, buf, options, errp);
             g_free(buf);
             return ret;
         }
@@ -632,9 +636,10 @@
         } QEMU_PACKED footer;
 
         ret = bdrv_pread(file,
-            bs->file->total_sectors * 512 - 1536,
+            bs->file->bs->total_sectors * 512 - 1536,
             &footer, sizeof(footer));
         if (ret < 0) {
+            error_setg_errno(errp, -ret, "Failed to read footer");
             return ret;
         }
 
@@ -646,20 +651,21 @@
             le32_to_cpu(footer.eos_marker.size) != 0  ||
             le32_to_cpu(footer.eos_marker.type) != MARKER_END_OF_STREAM)
         {
+            error_setg(errp, "Invalid footer");
             return -EINVAL;
         }
 
         header = footer.header;
     }
 
+    compressed =
+        le16_to_cpu(header.compressAlgorithm) == VMDK4_COMPRESSION_DEFLATE;
     if (le32_to_cpu(header.version) > 3) {
-        char buf[64];
-        snprintf(buf, sizeof(buf), "VMDK version %" PRId32,
-                 le32_to_cpu(header.version));
-        error_set(errp, QERR_UNKNOWN_BLOCK_FORMAT_FEATURE,
-                  bdrv_get_device_name(bs), "vmdk", buf);
+        error_setg(errp, "Unsupported VMDK version %" PRIu32,
+                   le32_to_cpu(header.version));
         return -ENOTSUP;
-    } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR)) {
+    } else if (le32_to_cpu(header.version) == 3 && (flags & BDRV_O_RDWR) &&
+               !compressed) {
         /* VMware KB 2064959 explains that version 3 added support for
          * persistent changed block tracking (CBT), and backup software can
          * read it as version=1 if it doesn't care about the changed area
@@ -676,6 +682,7 @@
     l1_entry_sectors = le32_to_cpu(header.num_gtes_per_gt)
                         * le64_to_cpu(header.granularity);
     if (l1_entry_sectors == 0) {
+        error_setg(errp, "L1 entry size is invalid");
         return -EINVAL;
     }
     l1_size = (le64_to_cpu(header.capacity) + l1_entry_sectors - 1)
@@ -683,7 +690,7 @@
     if (le32_to_cpu(header.flags) & VMDK4_FLAG_RGD) {
         l1_backup_offset = le64_to_cpu(header.rgd_offset) << 9;
     }
-    if (bdrv_nb_sectors(file) < le64_to_cpu(header.grain_offset)) {
+    if (bdrv_nb_sectors(file->bs) < le64_to_cpu(header.grain_offset)) {
         error_setg(errp, "File truncated, expecting at least %" PRId64 " bytes",
                    (int64_t)(le64_to_cpu(header.grain_offset)
                              * BDRV_SECTOR_SIZE));
@@ -747,9 +754,8 @@
 }
 
 /* Open an extent file and append to bs array */
-static int vmdk_open_sparse(BlockDriverState *bs,
-                            BlockDriverState *file, int flags,
-                            char *buf, Error **errp)
+static int vmdk_open_sparse(BlockDriverState *bs, BdrvChild *file, int flags,
+                            char *buf, QDict *options, Error **errp)
 {
     uint32_t magic;
 
@@ -759,7 +765,7 @@
             return vmdk_open_vmfs_sparse(bs, file, flags, errp);
             break;
         case VMDK4_MAGIC:
-            return vmdk_open_vmdk4(bs, file, flags, errp);
+            return vmdk_open_vmdk4(bs, file, flags, options, errp);
             break;
         default:
             error_setg(errp, "Image not in VMDK format");
@@ -768,63 +774,90 @@
     }
 }
 
+static const char *next_line(const char *s)
+{
+    while (*s) {
+        if (*s == '\n') {
+            return s + 1;
+        }
+        s++;
+    }
+    return s;
+}
+
 static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
-                              const char *desc_file_path, Error **errp)
+                              const char *desc_file_path, QDict *options,
+                              Error **errp)
 {
     int ret;
+    int matches;
     char access[11];
     char type[11];
     char fname[512];
-    const char *p = desc;
+    const char *p, *np;
     int64_t sectors = 0;
     int64_t flat_offset;
-    char extent_path[PATH_MAX];
-    BlockDriverState *extent_file;
+    char *extent_path;
+    BdrvChild *extent_file;
     BDRVVmdkState *s = bs->opaque;
     VmdkExtent *extent;
+    char extent_opt_prefix[32];
+    Error *local_err = NULL;
 
-    while (*p) {
-        /* parse extent line:
+    for (p = desc; *p; p = next_line(p)) {
+        /* parse extent line in one of below formats:
+         *
          * RW [size in sectors] FLAT "file-name.vmdk" OFFSET
-         * or
          * RW [size in sectors] SPARSE "file-name.vmdk"
+         * RW [size in sectors] VMFS "file-name.vmdk"
+         * RW [size in sectors] VMFSSPARSE "file-name.vmdk"
          */
         flat_offset = -1;
-        ret = sscanf(p, "%10s %" SCNd64 " %10s \"%511[^\n\r\"]\" %" SCNd64,
-                access, &sectors, type, fname, &flat_offset);
-        if (ret < 4 || strcmp(access, "RW")) {
-            goto next_line;
+        matches = sscanf(p, "%10s %" SCNd64 " %10s \"%511[^\n\r\"]\" %" SCNd64,
+                         access, &sectors, type, fname, &flat_offset);
+        if (matches < 4 || strcmp(access, "RW")) {
+            continue;
         } else if (!strcmp(type, "FLAT")) {
-            if (ret != 5 || flat_offset < 0) {
-                error_setg(errp, "Invalid extent lines: \n%s", p);
-                return -EINVAL;
+            if (matches != 5 || flat_offset < 0) {
+                goto invalid;
             }
         } else if (!strcmp(type, "VMFS")) {
-            if (ret == 4) {
+            if (matches == 4) {
                 flat_offset = 0;
             } else {
-                error_setg(errp, "Invalid extent lines:\n%s", p);
-                return -EINVAL;
+                goto invalid;
             }
-        } else if (ret != 4) {
-            error_setg(errp, "Invalid extent lines:\n%s", p);
-            return -EINVAL;
+        } else if (matches != 4) {
+            goto invalid;
         }
 
         if (sectors <= 0 ||
             (strcmp(type, "FLAT") && strcmp(type, "SPARSE") &&
              strcmp(type, "VMFS") && strcmp(type, "VMFSSPARSE")) ||
             (strcmp(access, "RW"))) {
-            goto next_line;
+            continue;
         }
 
-        path_combine(extent_path, sizeof(extent_path),
-                desc_file_path, fname);
-        extent_file = NULL;
-        ret = bdrv_open(&extent_file, extent_path, NULL, NULL,
-                        bs->open_flags | BDRV_O_PROTOCOL, NULL, errp);
-        if (ret) {
-            return ret;
+        if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
+            !desc_file_path[0])
+        {
+            error_setg(errp, "Cannot use relative extent paths with VMDK "
+                       "descriptor file '%s'", bs->file->bs->filename);
+            return -EINVAL;
+        }
+
+        extent_path = g_malloc0(PATH_MAX);
+        path_combine(extent_path, PATH_MAX, desc_file_path, fname);
+
+        ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
+        assert(ret < 32);
+
+        extent_file = bdrv_open_child(extent_path, options, extent_opt_prefix,
+                                      bs, &child_file, false, &local_err);
+        g_free(extent_path);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return -EINVAL;
         }
 
         /* save to extents array */
@@ -834,7 +867,7 @@
             ret = vmdk_add_extent(bs, extent_file, true, sectors,
                             0, 0, 0, 0, 0, &extent, errp);
             if (ret < 0) {
-                bdrv_unref(extent_file);
+                bdrv_unref_child(bs, extent_file);
                 return ret;
             }
             extent->flat_start_offset = flat_offset << 9;
@@ -844,35 +877,36 @@
             if (!buf) {
                 ret = -EINVAL;
             } else {
-                ret = vmdk_open_sparse(bs, extent_file, bs->open_flags, buf, errp);
+                ret = vmdk_open_sparse(bs, extent_file, bs->open_flags, buf,
+                                       options, errp);
             }
             g_free(buf);
             if (ret) {
-                bdrv_unref(extent_file);
+                bdrv_unref_child(bs, extent_file);
                 return ret;
             }
             extent = &s->extents[s->num_extents - 1];
         } else {
             error_setg(errp, "Unsupported extent type '%s'", type);
-            bdrv_unref(extent_file);
+            bdrv_unref_child(bs, extent_file);
             return -ENOTSUP;
         }
         extent->type = g_strdup(type);
-next_line:
-        /* move to next line */
-        while (*p) {
-            if (*p == '\n') {
-                p++;
-                break;
-            }
-            p++;
-        }
     }
     return 0;
+
+invalid:
+    np = next_line(p);
+    assert(np != p);
+    if (np[-1] == '\n') {
+        np--;
+    }
+    error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);
+    return -EINVAL;
 }
 
 static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
-                               Error **errp)
+                               QDict *options, Error **errp)
 {
     int ret;
     char ct[128];
@@ -894,7 +928,8 @@
     }
     s->create_type = g_strdup(ct);
     s->desc_offset = 0;
-    ret = vmdk_parse_extents(buf, bs, bs->file->filename, errp);
+    ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
+                             errp);
 exit:
     return ret;
 }
@@ -902,7 +937,7 @@
 static int vmdk_open(BlockDriverState *bs, QDict *options, int flags,
                      Error **errp)
 {
-    char *buf = NULL;
+    char *buf;
     int ret;
     BDRVVmdkState *s = bs->opaque;
     uint32_t magic;
@@ -916,11 +951,12 @@
     switch (magic) {
         case VMDK3_MAGIC:
         case VMDK4_MAGIC:
-            ret = vmdk_open_sparse(bs, bs->file, flags, buf, errp);
+            ret = vmdk_open_sparse(bs, bs->file, flags, buf, options,
+                                   errp);
             s->desc_offset = 0x200;
             break;
         default:
-            ret = vmdk_open_desc_file(bs, flags, buf, errp);
+            ret = vmdk_open_desc_file(bs, flags, buf, options, errp);
             break;
     }
     if (ret) {
@@ -937,9 +973,9 @@
     qemu_co_mutex_init(&s->lock);
 
     /* Disable migration when VMDK images are used */
-    error_set(&s->migration_blocker,
-              QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "vmdk", bdrv_get_device_name(bs), "live migration");
+    error_setg(&s->migration_blocker, "The vmdk format used by node '%s' "
+               "does not support live migration",
+               bdrv_get_device_or_node_name(bs));
     migrate_add_blocker(s->migration_blocker);
     g_free(buf);
     return 0;
@@ -960,9 +996,9 @@
 
     for (i = 0; i < s->num_extents; i++) {
         if (!s->extents[i].flat) {
-            bs->bl.write_zeroes_alignment =
-                MAX(bs->bl.write_zeroes_alignment,
-                    s->extents[i].cluster_sectors);
+            bs->bl.pwrite_zeroes_alignment =
+                MAX(bs->bl.pwrite_zeroes_alignment,
+                    s->extents[i].cluster_sectors << BDRV_SECTOR_BITS);
         }
     }
 }
@@ -979,71 +1015,71 @@
  */
 static int get_whole_cluster(BlockDriverState *bs,
                              VmdkExtent *extent,
-                             uint64_t cluster_sector_num,
-                             uint64_t sector_num,
-                             uint64_t skip_start_sector,
-                             uint64_t skip_end_sector)
+                             uint64_t cluster_offset,
+                             uint64_t offset,
+                             uint64_t skip_start_bytes,
+                             uint64_t skip_end_bytes)
 {
     int ret = VMDK_OK;
     int64_t cluster_bytes;
     uint8_t *whole_grain;
 
     /* For COW, align request sector_num to cluster start */
-    sector_num = QEMU_ALIGN_DOWN(sector_num, extent->cluster_sectors);
     cluster_bytes = extent->cluster_sectors << BDRV_SECTOR_BITS;
+    offset = QEMU_ALIGN_DOWN(offset, cluster_bytes);
     whole_grain = qemu_blockalign(bs, cluster_bytes);
 
-    if (!bs->backing_hd) {
-        memset(whole_grain, 0,  skip_start_sector << BDRV_SECTOR_BITS);
-        memset(whole_grain + (skip_end_sector << BDRV_SECTOR_BITS), 0,
-               cluster_bytes - (skip_end_sector << BDRV_SECTOR_BITS));
+    if (!bs->backing) {
+        memset(whole_grain, 0, skip_start_bytes);
+        memset(whole_grain + skip_end_bytes, 0, cluster_bytes - skip_end_bytes);
     }
 
-    assert(skip_end_sector <= extent->cluster_sectors);
+    assert(skip_end_bytes <= cluster_bytes);
     /* we will be here if it's first write on non-exist grain(cluster).
      * try to read from parent image, if exist */
-    if (bs->backing_hd && !vmdk_is_cid_valid(bs)) {
+    if (bs->backing && !vmdk_is_cid_valid(bs)) {
         ret = VMDK_ERROR;
         goto exit;
     }
 
     /* Read backing data before skip range */
-    if (skip_start_sector > 0) {
-        if (bs->backing_hd) {
-            ret = bdrv_read(bs->backing_hd, sector_num,
-                            whole_grain, skip_start_sector);
+    if (skip_start_bytes > 0) {
+        if (bs->backing) {
+            ret = bdrv_pread(bs->backing, offset, whole_grain,
+                             skip_start_bytes);
             if (ret < 0) {
                 ret = VMDK_ERROR;
                 goto exit;
             }
         }
-        ret = bdrv_write(extent->file, cluster_sector_num, whole_grain,
-                         skip_start_sector);
+        ret = bdrv_pwrite(extent->file, cluster_offset, whole_grain,
+                          skip_start_bytes);
         if (ret < 0) {
             ret = VMDK_ERROR;
             goto exit;
         }
     }
     /* Read backing data after skip range */
-    if (skip_end_sector < extent->cluster_sectors) {
-        if (bs->backing_hd) {
-            ret = bdrv_read(bs->backing_hd, sector_num + skip_end_sector,
-                            whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
-                            extent->cluster_sectors - skip_end_sector);
+    if (skip_end_bytes < cluster_bytes) {
+        if (bs->backing) {
+            ret = bdrv_pread(bs->backing, offset + skip_end_bytes,
+                             whole_grain + skip_end_bytes,
+                             cluster_bytes - skip_end_bytes);
             if (ret < 0) {
                 ret = VMDK_ERROR;
                 goto exit;
             }
         }
-        ret = bdrv_write(extent->file, cluster_sector_num + skip_end_sector,
-                         whole_grain + (skip_end_sector << BDRV_SECTOR_BITS),
-                         extent->cluster_sectors - skip_end_sector);
+        ret = bdrv_pwrite(extent->file, cluster_offset + skip_end_bytes,
+                          whole_grain + skip_end_bytes,
+                          cluster_bytes - skip_end_bytes);
         if (ret < 0) {
             ret = VMDK_ERROR;
             goto exit;
         }
     }
 
+    ret = VMDK_OK;
 exit:
     qemu_vfree(whole_grain);
     return ret;
@@ -1054,8 +1090,7 @@
 {
     offset = cpu_to_le32(offset);
     /* update L2 table */
-    if (bdrv_pwrite_sync(
-                extent->file,
+    if (bdrv_pwrite_sync(extent->file,
                 ((int64_t)m_data->l2_offset * 512)
                     + (m_data->l2_index * sizeof(offset)),
                 &offset, sizeof(offset)) < 0) {
@@ -1064,8 +1099,7 @@
     /* update backup L2 table */
     if (extent->l1_backup_table_offset != 0) {
         m_data->l2_offset = extent->l1_backup_table[m_data->l1_index];
-        if (bdrv_pwrite_sync(
-                    extent->file,
+        if (bdrv_pwrite_sync(extent->file,
                     ((int64_t)m_data->l2_offset * 512)
                         + (m_data->l2_index * sizeof(offset)),
                     &offset, sizeof(offset)) < 0) {
@@ -1105,8 +1139,8 @@
                               uint64_t offset,
                               bool allocate,
                               uint64_t *cluster_offset,
-                              uint64_t skip_start_sector,
-                              uint64_t skip_end_sector)
+                              uint64_t skip_start_bytes,
+                              uint64_t skip_end_bytes)
 {
     unsigned int l1_index, l2_offset, l2_index;
     int min_index, i, j;
@@ -1154,8 +1188,7 @@
         }
     }
     l2_table = extent->l2_cache + (min_index * extent->l2_size);
-    if (bdrv_pread(
-                extent->file,
+    if (bdrv_pread(extent->file,
                 (int64_t)l2_offset * 512,
                 l2_table,
                 extent->l2_size * sizeof(uint32_t)
@@ -1169,13 +1202,6 @@
     l2_index = ((offset >> 9) / extent->cluster_sectors) % extent->l2_size;
     cluster_sector = le32_to_cpu(l2_table[l2_index]);
 
-    if (m_data) {
-        m_data->valid = 1;
-        m_data->l1_index = l1_index;
-        m_data->l2_index = l2_index;
-        m_data->l2_offset = l2_offset;
-        m_data->l2_cache_entry = &l2_table[l2_index];
-    }
     if (extent->has_zero_grain && cluster_sector == VMDK_GTE_ZEROED) {
         zeroed = true;
     }
@@ -1193,13 +1219,18 @@
          * This problem may occur because of insufficient space on host disk
          * or inappropriate VM shutdown.
          */
-        ret = get_whole_cluster(bs, extent,
-                                cluster_sector,
-                                offset >> BDRV_SECTOR_BITS,
-                                skip_start_sector, skip_end_sector);
+        ret = get_whole_cluster(bs, extent, cluster_sector * BDRV_SECTOR_SIZE,
+                                offset, skip_start_bytes, skip_end_bytes);
         if (ret) {
             return ret;
         }
+        if (m_data) {
+            m_data->valid = 1;
+            m_data->l1_index = l1_index;
+            m_data->l2_index = l2_index;
+            m_data->l2_offset = l2_offset;
+            m_data->l2_cache_entry = &l2_table[l2_index];
+        }
     }
     *cluster_offset = cluster_sector << BDRV_SECTOR_BITS;
     return VMDK_OK;
@@ -1222,8 +1253,28 @@
     return NULL;
 }
 
+static inline uint64_t vmdk_find_offset_in_cluster(VmdkExtent *extent,
+                                                   int64_t offset)
+{
+    uint64_t extent_begin_offset, extent_relative_offset;
+    uint64_t cluster_size = extent->cluster_sectors * BDRV_SECTOR_SIZE;
+
+    extent_begin_offset =
+        (extent->end_sector - extent->sectors) * BDRV_SECTOR_SIZE;
+    extent_relative_offset = offset - extent_begin_offset;
+    return extent_relative_offset % cluster_size;
+}
+
+static inline uint64_t vmdk_find_index_in_cluster(VmdkExtent *extent,
+                                                  int64_t sector_num)
+{
+    uint64_t offset;
+    offset = vmdk_find_offset_in_cluster(extent, sector_num * BDRV_SECTOR_SIZE);
+    return offset / BDRV_SECTOR_SIZE;
+}
+
 static int64_t coroutine_fn vmdk_co_get_block_status(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum)
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
     BDRVVmdkState *s = bs->opaque;
     int64_t index_in_cluster, n, ret;
@@ -1240,6 +1291,7 @@
                              0, 0);
     qemu_co_mutex_unlock(&s->lock);
 
+    index_in_cluster = vmdk_find_index_in_cluster(extent, sector_num);
     switch (ret) {
     case VMDK_ERROR:
         ret = -EIO;
@@ -1252,14 +1304,15 @@
         break;
     case VMDK_OK:
         ret = BDRV_BLOCK_DATA;
-        if (extent->file == bs->file && !extent->compressed) {
-            ret |= BDRV_BLOCK_OFFSET_VALID | offset;
+        if (!extent->compressed) {
+            ret |= BDRV_BLOCK_OFFSET_VALID;
+            ret |= (offset + (index_in_cluster << BDRV_SECTOR_BITS))
+                    & BDRV_BLOCK_OFFSET_MASK;
         }
-
+        *file = extent->file->bs;
         break;
     }
 
-    index_in_cluster = sector_num % extent->cluster_sectors;
     n = extent->cluster_sectors - index_in_cluster;
     if (n > nb_sectors) {
         n = nb_sectors;
@@ -1269,49 +1322,80 @@
 }
 
 static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
-                            int64_t offset_in_cluster, const uint8_t *buf,
-                            int nb_sectors, int64_t sector_num)
+                            int64_t offset_in_cluster, QEMUIOVector *qiov,
+                            uint64_t qiov_offset, uint64_t n_bytes,
+                            uint64_t offset)
 {
     int ret;
     VmdkGrainMarker *data = NULL;
     uLongf buf_len;
-    const uint8_t *write_buf = buf;
-    int write_len = nb_sectors * 512;
+    QEMUIOVector local_qiov;
+    struct iovec iov;
+    int64_t write_offset;
+    int64_t write_end_sector;
 
     if (extent->compressed) {
+        void *compressed_data;
+
         if (!extent->has_marker) {
             ret = -EINVAL;
             goto out;
         }
         buf_len = (extent->cluster_sectors << 9) * 2;
         data = g_malloc(buf_len + sizeof(VmdkGrainMarker));
-        if (compress(data->data, &buf_len, buf, nb_sectors << 9) != Z_OK ||
-                buf_len == 0) {
+
+        compressed_data = g_malloc(n_bytes);
+        qemu_iovec_to_buf(qiov, qiov_offset, compressed_data, n_bytes);
+        ret = compress(data->data, &buf_len, compressed_data, n_bytes);
+        g_free(compressed_data);
+
+        if (ret != Z_OK || buf_len == 0) {
             ret = -EINVAL;
             goto out;
         }
-        data->lba = sector_num;
+
+        data->lba = offset >> BDRV_SECTOR_BITS;
         data->size = buf_len;
-        write_buf = (uint8_t *)data;
-        write_len = buf_len + sizeof(VmdkGrainMarker);
+
+        n_bytes = buf_len + sizeof(VmdkGrainMarker);
+        iov = (struct iovec) {
+            .iov_base   = data,
+            .iov_len    = n_bytes,
+        };
+        qemu_iovec_init_external(&local_qiov, &iov, 1);
+    } else {
+        qemu_iovec_init(&local_qiov, qiov->niov);
+        qemu_iovec_concat(&local_qiov, qiov, qiov_offset, n_bytes);
     }
-    ret = bdrv_pwrite(extent->file,
-                        cluster_offset + offset_in_cluster,
-                        write_buf,
-                        write_len);
-    if (ret != write_len) {
-        ret = ret < 0 ? ret : -EIO;
+
+    write_offset = cluster_offset + offset_in_cluster,
+    ret = bdrv_co_pwritev(extent->file, write_offset, n_bytes,
+                          &local_qiov, 0);
+
+    write_end_sector = DIV_ROUND_UP(write_offset + n_bytes, BDRV_SECTOR_SIZE);
+
+    if (extent->compressed) {
+        extent->next_cluster_sector = write_end_sector;
+    } else {
+        extent->next_cluster_sector = MAX(extent->next_cluster_sector,
+                                          write_end_sector);
+    }
+
+    if (ret < 0) {
         goto out;
     }
     ret = 0;
  out:
     g_free(data);
+    if (!extent->compressed) {
+        qemu_iovec_destroy(&local_qiov);
+    }
     return ret;
 }
 
 static int vmdk_read_extent(VmdkExtent *extent, int64_t cluster_offset,
-                            int64_t offset_in_cluster, uint8_t *buf,
-                            int nb_sectors)
+                            int64_t offset_in_cluster, QEMUIOVector *qiov,
+                            int bytes)
 {
     int ret;
     int cluster_bytes, buf_bytes;
@@ -1323,14 +1407,13 @@
 
 
     if (!extent->compressed) {
-        ret = bdrv_pread(extent->file,
-                          cluster_offset + offset_in_cluster,
-                          buf, nb_sectors * 512);
-        if (ret == nb_sectors * 512) {
-            return 0;
-        } else {
-            return -EIO;
+        ret = bdrv_co_preadv(extent->file,
+                             cluster_offset + offset_in_cluster, bytes,
+                             qiov, 0);
+        if (ret < 0) {
+            return ret;
         }
+        return 0;
     }
     cluster_bytes = extent->cluster_sectors * 512;
     /* Read two clusters in case GrainMarker + compressed data > one cluster */
@@ -1362,11 +1445,11 @@
 
     }
     if (offset_in_cluster < 0 ||
-            offset_in_cluster + nb_sectors * 512 > buf_len) {
+            offset_in_cluster + bytes > buf_len) {
         ret = -EINVAL;
         goto out;
     }
-    memcpy(buf, uncomp_buf + offset_in_cluster, nb_sectors * 512);
+    qemu_iovec_from_buf(qiov, 0, uncomp_buf + offset_in_cluster, bytes);
     ret = 0;
 
  out:
@@ -1375,67 +1458,73 @@
     return ret;
 }
 
-static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vmdk_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+               QEMUIOVector *qiov, int flags)
 {
     BDRVVmdkState *s = bs->opaque;
     int ret;
-    uint64_t n, index_in_cluster;
-    uint64_t extent_begin_sector, extent_relative_sector_num;
+    uint64_t n_bytes, offset_in_cluster;
     VmdkExtent *extent = NULL;
+    QEMUIOVector local_qiov;
     uint64_t cluster_offset;
+    uint64_t bytes_done = 0;
 
-    while (nb_sectors > 0) {
-        extent = find_extent(s, sector_num, extent);
+    qemu_iovec_init(&local_qiov, qiov->niov);
+    qemu_co_mutex_lock(&s->lock);
+
+    while (bytes > 0) {
+        extent = find_extent(s, offset >> BDRV_SECTOR_BITS, extent);
         if (!extent) {
-            return -EIO;
+            ret = -EIO;
+            goto fail;
         }
         ret = get_cluster_offset(bs, extent, NULL,
-                                 sector_num << 9, false, &cluster_offset,
-                                 0, 0);
-        extent_begin_sector = extent->end_sector - extent->sectors;
-        extent_relative_sector_num = sector_num - extent_begin_sector;
-        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
-        n = extent->cluster_sectors - index_in_cluster;
-        if (n > nb_sectors) {
-            n = nb_sectors;
-        }
+                                 offset, false, &cluster_offset, 0, 0);
+        offset_in_cluster = vmdk_find_offset_in_cluster(extent, offset);
+
+        n_bytes = MIN(bytes, extent->cluster_sectors * BDRV_SECTOR_SIZE
+                             - offset_in_cluster);
+
         if (ret != VMDK_OK) {
             /* if not allocated, try to read from parent image, if exist */
-            if (bs->backing_hd && ret != VMDK_ZEROED) {
+            if (bs->backing && ret != VMDK_ZEROED) {
                 if (!vmdk_is_cid_valid(bs)) {
-                    return -EINVAL;
+                    ret = -EINVAL;
+                    goto fail;
                 }
-                ret = bdrv_read(bs->backing_hd, sector_num, buf, n);
+
+                qemu_iovec_reset(&local_qiov);
+                qemu_iovec_concat(&local_qiov, qiov, bytes_done, n_bytes);
+
+                ret = bdrv_co_preadv(bs->backing, offset, n_bytes,
+                                     &local_qiov, 0);
                 if (ret < 0) {
-                    return ret;
+                    goto fail;
                 }
             } else {
-                memset(buf, 0, 512 * n);
+                qemu_iovec_memset(qiov, bytes_done, 0, n_bytes);
             }
         } else {
-            ret = vmdk_read_extent(extent,
-                            cluster_offset, index_in_cluster * 512,
-                            buf, n);
+            qemu_iovec_reset(&local_qiov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes_done, n_bytes);
+
+            ret = vmdk_read_extent(extent, cluster_offset, offset_in_cluster,
+                                   &local_qiov, n_bytes);
             if (ret) {
-                return ret;
+                goto fail;
             }
         }
-        nb_sectors -= n;
-        sector_num += n;
-        buf += n * 512;
+        bytes -= n_bytes;
+        offset += n_bytes;
+        bytes_done += n_bytes;
     }
-    return 0;
-}
 
-static coroutine_fn int vmdk_co_read(BlockDriverState *bs, int64_t sector_num,
-                                     uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVVmdkState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = vmdk_read(bs, sector_num, buf, nb_sectors);
+    ret = 0;
+fail:
     qemu_co_mutex_unlock(&s->lock);
+    qemu_iovec_destroy(&local_qiov);
+
     return ret;
 }
 
@@ -1449,41 +1538,38 @@
  *
  * Returns: error code with 0 for success.
  */
-static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
-                      const uint8_t *buf, int nb_sectors,
-                      bool zeroed, bool zero_dry_run)
+static int vmdk_pwritev(BlockDriverState *bs, uint64_t offset,
+                       uint64_t bytes, QEMUIOVector *qiov,
+                       bool zeroed, bool zero_dry_run)
 {
     BDRVVmdkState *s = bs->opaque;
     VmdkExtent *extent = NULL;
     int ret;
-    int64_t index_in_cluster, n;
-    uint64_t extent_begin_sector, extent_relative_sector_num;
+    int64_t offset_in_cluster, n_bytes;
     uint64_t cluster_offset;
+    uint64_t bytes_done = 0;
     VmdkMetaData m_data;
 
-    if (sector_num > bs->total_sectors) {
-        error_report("Wrong offset: sector_num=0x%" PRIx64
-                " total_sectors=0x%" PRIx64 "\n",
-                sector_num, bs->total_sectors);
+    if (DIV_ROUND_UP(offset, BDRV_SECTOR_SIZE) > bs->total_sectors) {
+        error_report("Wrong offset: offset=0x%" PRIx64
+                     " total_sectors=0x%" PRIx64,
+                     offset, bs->total_sectors);
         return -EIO;
     }
 
-    while (nb_sectors > 0) {
-        extent = find_extent(s, sector_num, extent);
+    while (bytes > 0) {
+        extent = find_extent(s, offset >> BDRV_SECTOR_BITS, extent);
         if (!extent) {
             return -EIO;
         }
-        extent_begin_sector = extent->end_sector - extent->sectors;
-        extent_relative_sector_num = sector_num - extent_begin_sector;
-        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
-        n = extent->cluster_sectors - index_in_cluster;
-        if (n > nb_sectors) {
-            n = nb_sectors;
-        }
-        ret = get_cluster_offset(bs, extent, &m_data, sector_num << 9,
+        offset_in_cluster = vmdk_find_offset_in_cluster(extent, offset);
+        n_bytes = MIN(bytes, extent->cluster_sectors * BDRV_SECTOR_SIZE
+                             - offset_in_cluster);
+
+        ret = get_cluster_offset(bs, extent, &m_data, offset,
                                  !(extent->compressed || zeroed),
-                                 &cluster_offset,
-                                 index_in_cluster, index_in_cluster + n);
+                                 &cluster_offset, offset_in_cluster,
+                                 offset_in_cluster + n_bytes);
         if (extent->compressed) {
             if (ret == VMDK_OK) {
                 /* Refuse write to allocated cluster for streamOptimized */
@@ -1492,7 +1578,7 @@
                 return -EIO;
             } else {
                 /* allocate */
-                ret = get_cluster_offset(bs, extent, &m_data, sector_num << 9,
+                ret = get_cluster_offset(bs, extent, &m_data, offset,
                                          true, &cluster_offset, 0, 0);
             }
         }
@@ -1502,9 +1588,9 @@
         if (zeroed) {
             /* Do zeroed write, buf is ignored */
             if (extent->has_zero_grain &&
-                    index_in_cluster == 0 &&
-                    n >= extent->cluster_sectors) {
-                n = extent->cluster_sectors;
+                    offset_in_cluster == 0 &&
+                    n_bytes >= extent->cluster_sectors * BDRV_SECTOR_SIZE) {
+                n_bytes = extent->cluster_sectors * BDRV_SECTOR_SIZE;
                 if (!zero_dry_run) {
                     /* update L2 tables */
                     if (vmdk_L2update(extent, &m_data, VMDK_GTE_ZEROED)
@@ -1516,9 +1602,8 @@
                 return -ENOTSUP;
             }
         } else {
-            ret = vmdk_write_extent(extent,
-                            cluster_offset, index_in_cluster * 512,
-                            buf, n, sector_num);
+            ret = vmdk_write_extent(extent, cluster_offset, offset_in_cluster,
+                                    qiov, bytes_done, n_bytes, offset);
             if (ret) {
                 return ret;
             }
@@ -1531,14 +1616,14 @@
                 }
             }
         }
-        nb_sectors -= n;
-        sector_num += n;
-        buf += n * 512;
+        bytes -= n_bytes;
+        offset += n_bytes;
+        bytes_done += n_bytes;
 
         /* update CID on the first write every time the virtual disk is
          * opened */
         if (!s->cid_updated) {
-            ret = vmdk_write_cid(bs, time(NULL));
+            ret = vmdk_write_cid(bs, g_random_int());
             if (ret < 0) {
                 return ret;
             }
@@ -1548,43 +1633,84 @@
     return 0;
 }
 
-static coroutine_fn int vmdk_co_write(BlockDriverState *bs, int64_t sector_num,
-                                      const uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vmdk_co_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+                QEMUIOVector *qiov, int flags)
 {
     int ret;
     BDRVVmdkState *s = bs->opaque;
     qemu_co_mutex_lock(&s->lock);
-    ret = vmdk_write(bs, sector_num, buf, nb_sectors, false, false);
+    ret = vmdk_pwritev(bs, offset, bytes, qiov, false, false);
     qemu_co_mutex_unlock(&s->lock);
     return ret;
 }
 
+typedef struct VmdkWriteCompressedCo {
+    BlockDriverState *bs;
+    int64_t sector_num;
+    const uint8_t *buf;
+    int nb_sectors;
+    int ret;
+} VmdkWriteCompressedCo;
+
+static void vmdk_co_write_compressed(void *opaque)
+{
+    VmdkWriteCompressedCo *co = opaque;
+    QEMUIOVector local_qiov;
+    uint64_t offset = co->sector_num * BDRV_SECTOR_SIZE;
+    uint64_t bytes = co->nb_sectors * BDRV_SECTOR_SIZE;
+
+    struct iovec iov = (struct iovec) {
+        .iov_base   = (uint8_t*) co->buf,
+        .iov_len    = bytes,
+    };
+    qemu_iovec_init_external(&local_qiov, &iov, 1);
+
+    co->ret = vmdk_pwritev(co->bs, offset, bytes, &local_qiov, false, false);
+}
+
 static int vmdk_write_compressed(BlockDriverState *bs,
                                  int64_t sector_num,
                                  const uint8_t *buf,
                                  int nb_sectors)
 {
     BDRVVmdkState *s = bs->opaque;
+
     if (s->num_extents == 1 && s->extents[0].compressed) {
-        return vmdk_write(bs, sector_num, buf, nb_sectors, false, false);
+        Coroutine *co;
+        AioContext *aio_context = bdrv_get_aio_context(bs);
+        VmdkWriteCompressedCo data = {
+            .bs         = bs,
+            .sector_num = sector_num,
+            .buf        = buf,
+            .nb_sectors = nb_sectors,
+            .ret        = -EINPROGRESS,
+        };
+        co = qemu_coroutine_create(vmdk_co_write_compressed, &data);
+        qemu_coroutine_enter(co);
+        while (data.ret == -EINPROGRESS) {
+            aio_poll(aio_context, true);
+        }
+        return data.ret;
     } else {
         return -ENOTSUP;
     }
 }
 
-static int coroutine_fn vmdk_co_write_zeroes(BlockDriverState *bs,
-                                             int64_t sector_num,
-                                             int nb_sectors,
-                                             BdrvRequestFlags flags)
+static int coroutine_fn vmdk_co_pwrite_zeroes(BlockDriverState *bs,
+                                              int64_t offset,
+                                              int bytes,
+                                              BdrvRequestFlags flags)
 {
     int ret;
     BDRVVmdkState *s = bs->opaque;
+
     qemu_co_mutex_lock(&s->lock);
     /* write zeroes could fail if sectors not aligned to cluster, test it with
      * dry_run == true before really updating image */
-    ret = vmdk_write(bs, sector_num, NULL, nb_sectors, true, true);
+    ret = vmdk_pwritev(bs, offset, bytes, NULL, true, true);
     if (!ret) {
-        ret = vmdk_write(bs, sector_num, NULL, nb_sectors, true, false);
+        ret = vmdk_pwritev(bs, offset, bytes, NULL, true, false);
     }
     qemu_co_mutex_unlock(&s->lock);
     return ret;
@@ -1595,7 +1721,7 @@
                               QemuOpts *opts, Error **errp)
 {
     int ret, i;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
     VMDK4Header header;
     Error *local_err = NULL;
     uint32_t tmp, magic, grains, gd_sectors, gt_size, gt_count;
@@ -1608,16 +1734,18 @@
         goto exit;
     }
 
-    assert(bs == NULL);
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     if (flat) {
-        ret = bdrv_truncate(bs, filesize);
+        ret = blk_truncate(blk, filesize);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not truncate file");
         }
@@ -1625,7 +1753,13 @@
     }
     magic = cpu_to_be32(VMDK4_MAGIC);
     memset(&header, 0, sizeof(header));
-    header.version = zeroed_grain ? 2 : 1;
+    if (compress) {
+        header.version = 3;
+    } else if (zeroed_grain) {
+        header.version = 2;
+    } else {
+        header.version = 1;
+    }
     header.flags = VMDK4_FLAG_RGD | VMDK4_FLAG_NL_DETECT
                    | (compress ? VMDK4_FLAG_COMPRESS | VMDK4_FLAG_MARKER : 0)
                    | (zeroed_grain ? VMDK4_FLAG_ZERO_GRAIN : 0);
@@ -1666,18 +1800,18 @@
     header.check_bytes[3] = 0xa;
 
     /* write all the data */
-    ret = bdrv_pwrite(bs, 0, &magic, sizeof(magic));
+    ret = blk_pwrite(blk, 0, &magic, sizeof(magic), 0);
     if (ret < 0) {
-        error_set(errp, QERR_IO_ERROR);
+        error_setg(errp, QERR_IO_ERROR);
         goto exit;
     }
-    ret = bdrv_pwrite(bs, sizeof(magic), &header, sizeof(header));
+    ret = blk_pwrite(blk, sizeof(magic), &header, sizeof(header), 0);
     if (ret < 0) {
-        error_set(errp, QERR_IO_ERROR);
+        error_setg(errp, QERR_IO_ERROR);
         goto exit;
     }
 
-    ret = bdrv_truncate(bs, le64_to_cpu(header.grain_offset) << 9);
+    ret = blk_truncate(blk, le64_to_cpu(header.grain_offset) << 9);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not truncate file");
         goto exit;
@@ -1690,10 +1824,10 @@
          i < gt_count; i++, tmp += gt_size) {
         gd_buf[i] = cpu_to_le32(tmp);
     }
-    ret = bdrv_pwrite(bs, le64_to_cpu(header.rgd_offset) * BDRV_SECTOR_SIZE,
-                      gd_buf, gd_buf_size);
+    ret = blk_pwrite(blk, le64_to_cpu(header.rgd_offset) * BDRV_SECTOR_SIZE,
+                     gd_buf, gd_buf_size, 0);
     if (ret < 0) {
-        error_set(errp, QERR_IO_ERROR);
+        error_setg(errp, QERR_IO_ERROR);
         goto exit;
     }
 
@@ -1702,17 +1836,17 @@
          i < gt_count; i++, tmp += gt_size) {
         gd_buf[i] = cpu_to_le32(tmp);
     }
-    ret = bdrv_pwrite(bs, le64_to_cpu(header.gd_offset) * BDRV_SECTOR_SIZE,
-                      gd_buf, gd_buf_size);
+    ret = blk_pwrite(blk, le64_to_cpu(header.gd_offset) * BDRV_SECTOR_SIZE,
+                     gd_buf, gd_buf_size, 0);
     if (ret < 0) {
-        error_set(errp, QERR_IO_ERROR);
+        error_setg(errp, QERR_IO_ERROR);
         goto exit;
     }
 
     ret = 0;
 exit:
-    if (bs) {
-        bdrv_unref(bs);
+    if (blk) {
+        blk_unref(blk);
     }
     g_free(gd_buf);
     return ret;
@@ -1761,21 +1895,26 @@
 static int vmdk_create(const char *filename, QemuOpts *opts, Error **errp)
 {
     int idx = 0;
-    BlockDriverState *new_bs = NULL;
+    BlockBackend *new_blk = NULL;
     Error *local_err = NULL;
     char *desc = NULL;
     int64_t total_size = 0, filesize;
     char *adapter_type = NULL;
     char *backing_file = NULL;
+    char *hw_version = NULL;
     char *fmt = NULL;
-    int flags = 0;
     int ret = 0;
     bool flat, split, compress;
     GString *ext_desc_lines;
-    char path[PATH_MAX], prefix[PATH_MAX], postfix[PATH_MAX];
+    char *path = g_malloc0(PATH_MAX);
+    char *prefix = g_malloc0(PATH_MAX);
+    char *postfix = g_malloc0(PATH_MAX);
+    char *desc_line = g_malloc0(BUF_SIZE);
+    char *ext_filename = g_malloc0(PATH_MAX);
+    char *desc_filename = g_malloc0(PATH_MAX);
     const int64_t split_size = 0x80000000;  /* VMDK has constant split size */
     const char *desc_extent_line;
-    char parent_desc_line[BUF_SIZE] = "";
+    char *parent_desc_line = g_malloc0(BUF_SIZE);
     uint32_t parent_cid = 0xffffffff;
     uint32_t number_heads = 16;
     bool zeroed_grain = false;
@@ -1794,7 +1933,7 @@
         "# The Disk Data Base\n"
         "#DDB\n"
         "\n"
-        "ddb.virtualHWVersion = \"%d\"\n"
+        "ddb.virtualHWVersion = \"%s\"\n"
         "ddb.geometry.cylinders = \"%" PRId64 "\"\n"
         "ddb.geometry.heads = \"%" PRIu32 "\"\n"
         "ddb.geometry.sectors = \"63\"\n"
@@ -1811,8 +1950,20 @@
                           BDRV_SECTOR_SIZE);
     adapter_type = qemu_opt_get_del(opts, BLOCK_OPT_ADAPTER_TYPE);
     backing_file = qemu_opt_get_del(opts, BLOCK_OPT_BACKING_FILE);
+    hw_version = qemu_opt_get_del(opts, BLOCK_OPT_HWVERSION);
     if (qemu_opt_get_bool_del(opts, BLOCK_OPT_COMPAT6, false)) {
-        flags |= BLOCK_FLAG_COMPAT6;
+        if (strcmp(hw_version, "undefined")) {
+            error_setg(errp,
+                       "compat6 cannot be enabled with hwversion set");
+            ret = -EINVAL;
+            goto exit;
+        }
+        g_free(hw_version);
+        hw_version = g_strdup("6");
+    }
+    if (strcmp(hw_version, "undefined") == 0) {
+        g_free(hw_version);
+        hw_version = g_strdup("4");
     }
     fmt = qemu_opt_get_del(opts, BLOCK_OPT_SUBFMT);
     if (qemu_opt_get_bool_del(opts, BLOCK_OPT_ZEROED_GRAIN, false)) {
@@ -1867,46 +2018,53 @@
         goto exit;
     }
     if (backing_file) {
-        BlockDriverState *bs = NULL;
-        ret = bdrv_open(&bs, backing_file, NULL, NULL, BDRV_O_NO_BACKING, NULL,
-                        errp);
-        if (ret != 0) {
+        BlockBackend *blk;
+        char *full_backing = g_new0(char, PATH_MAX);
+        bdrv_get_full_backing_filename_from_filename(filename, backing_file,
+                                                     full_backing, PATH_MAX,
+                                                     &local_err);
+        if (local_err) {
+            g_free(full_backing);
+            error_propagate(errp, local_err);
+            ret = -ENOENT;
             goto exit;
         }
-        if (strcmp(bs->drv->format_name, "vmdk")) {
-            bdrv_unref(bs);
+
+        blk = blk_new_open(full_backing, NULL, NULL,
+                           BDRV_O_NO_BACKING, errp);
+        g_free(full_backing);
+        if (blk == NULL) {
+            ret = -EIO;
+            goto exit;
+        }
+        if (strcmp(blk_bs(blk)->drv->format_name, "vmdk")) {
+            blk_unref(blk);
             ret = -EINVAL;
             goto exit;
         }
-        parent_cid = vmdk_read_cid(bs, 0);
-        bdrv_unref(bs);
-        snprintf(parent_desc_line, sizeof(parent_desc_line),
+        parent_cid = vmdk_read_cid(blk_bs(blk), 0);
+        blk_unref(blk);
+        snprintf(parent_desc_line, BUF_SIZE,
                 "parentFileNameHint=\"%s\"", backing_file);
     }
 
     /* Create extents */
     filesize = total_size;
     while (filesize > 0) {
-        char desc_line[BUF_SIZE];
-        char ext_filename[PATH_MAX];
-        char desc_filename[PATH_MAX];
         int64_t size = filesize;
 
         if (split && size > split_size) {
             size = split_size;
         }
         if (split) {
-            snprintf(desc_filename, sizeof(desc_filename), "%s-%c%03d%s",
+            snprintf(desc_filename, PATH_MAX, "%s-%c%03d%s",
                     prefix, flat ? 'f' : 's', ++idx, postfix);
         } else if (flat) {
-            snprintf(desc_filename, sizeof(desc_filename), "%s-flat%s",
-                    prefix, postfix);
+            snprintf(desc_filename, PATH_MAX, "%s-flat%s", prefix, postfix);
         } else {
-            snprintf(desc_filename, sizeof(desc_filename), "%s%s",
-                    prefix, postfix);
+            snprintf(desc_filename, PATH_MAX, "%s%s", prefix, postfix);
         }
-        snprintf(ext_filename, sizeof(ext_filename), "%s%s",
-                path, desc_filename);
+        snprintf(ext_filename, PATH_MAX, "%s%s", path, desc_filename);
 
         if (vmdk_create_extent(ext_filename, size,
                                flat, compress, zeroed_grain, opts, errp)) {
@@ -1916,18 +2074,18 @@
         filesize -= size;
 
         /* Format description line */
-        snprintf(desc_line, sizeof(desc_line),
+        snprintf(desc_line, BUF_SIZE,
                     desc_extent_line, size / BDRV_SECTOR_SIZE, desc_filename);
         g_string_append(ext_desc_lines, desc_line);
     }
     /* generate descriptor file */
     desc = g_strdup_printf(desc_template,
-                           (uint32_t)time(NULL),
+                           g_random_int(),
                            parent_cid,
                            fmt,
                            parent_desc_line,
                            ext_desc_lines->str,
-                           (flags & BLOCK_FLAG_COMPAT6 ? 6 : 4),
+                           hw_version,
                            total_size /
                                (int64_t)(63 * number_heads * BDRV_SECTOR_SIZE),
                            number_heads,
@@ -1943,14 +2101,18 @@
             goto exit;
         }
     }
-    assert(new_bs == NULL);
-    ret = bdrv_open(&new_bs, filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
-    if (ret < 0) {
+
+    new_blk = blk_new_open(filename, NULL, NULL,
+                           BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (new_blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto exit;
     }
-    ret = bdrv_pwrite(new_bs, desc_offset, desc, desc_len);
+
+    blk_set_allow_write_beyond_eof(new_blk, true);
+
+    ret = blk_pwrite(new_blk, desc_offset, desc, desc_len, 0);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "Could not write description");
         goto exit;
@@ -1958,19 +2120,27 @@
     /* bdrv_pwrite write padding zeros to align to sector, we don't need that
      * for description file */
     if (desc_offset == 0) {
-        ret = bdrv_truncate(new_bs, desc_len);
+        ret = blk_truncate(new_blk, desc_len);
         if (ret < 0) {
             error_setg_errno(errp, -ret, "Could not truncate file");
         }
     }
 exit:
-    if (new_bs) {
-        bdrv_unref(new_bs);
+    if (new_blk) {
+        blk_unref(new_blk);
     }
     g_free(adapter_type);
     g_free(backing_file);
+    g_free(hw_version);
     g_free(fmt);
     g_free(desc);
+    g_free(path);
+    g_free(prefix);
+    g_free(postfix);
+    g_free(desc_line);
+    g_free(ext_filename);
+    g_free(desc_filename);
+    g_free(parent_desc_line);
     g_string_free(ext_desc_lines, true);
     return ret;
 }
@@ -1993,7 +2163,7 @@
     int ret = 0;
 
     for (i = 0; i < s->num_extents; i++) {
-        err = bdrv_co_flush(s->extents[i].file);
+        err = bdrv_co_flush(s->extents[i].file->bs);
         if (err < 0) {
             ret = err;
         }
@@ -2008,7 +2178,7 @@
     int64_t r;
     BDRVVmdkState *s = bs->opaque;
 
-    ret = bdrv_get_allocated_file_size(bs->file);
+    ret = bdrv_get_allocated_file_size(bs->file->bs);
     if (ret < 0) {
         return ret;
     }
@@ -2016,7 +2186,7 @@
         if (s->extents[i].file == bs->file) {
             continue;
         }
-        r = bdrv_get_allocated_file_size(s->extents[i].file);
+        r = bdrv_get_allocated_file_size(s->extents[i].file->bs);
         if (r < 0) {
             return r;
         }
@@ -2034,7 +2204,7 @@
      * return 0. */
     for (i = 0; i < s->num_extents; i++) {
         if (s->extents[i].flat) {
-            if (!bdrv_has_zero_init(s->extents[i].file)) {
+            if (!bdrv_has_zero_init(s->extents[i].file->bs)) {
                 return 0;
             }
         }
@@ -2047,7 +2217,7 @@
     ImageInfo *info = g_new0(ImageInfo, 1);
 
     *info = (ImageInfo){
-        .filename         = g_strdup(extent->file->filename),
+        .filename         = g_strdup(extent->file->bs->filename),
         .format           = g_strdup(extent->type),
         .virtual_size     = extent->sectors * BDRV_SECTOR_SIZE,
         .compressed       = extent->compressed,
@@ -2093,7 +2263,9 @@
                     PRId64 "\n", sector_num);
             break;
         }
-        if (ret == VMDK_OK && cluster_offset >= bdrv_getlength(extent->file)) {
+        if (ret == VMDK_OK &&
+            cluster_offset >= bdrv_getlength(extent->file->bs))
+        {
             fprintf(stderr,
                     "ERROR: cluster offset for sector %"
                     PRId64 " points after EOF\n", sector_num);
@@ -2114,19 +2286,19 @@
     ImageInfoList **next;
 
     *spec_info = (ImageInfoSpecific){
-        .kind = IMAGE_INFO_SPECIFIC_KIND_VMDK,
-        {
-            .vmdk = g_new0(ImageInfoSpecificVmdk, 1),
+        .type = IMAGE_INFO_SPECIFIC_KIND_VMDK,
+        .u = {
+            .vmdk.data = g_new0(ImageInfoSpecificVmdk, 1),
         },
     };
 
-    *spec_info->vmdk = (ImageInfoSpecificVmdk) {
+    *spec_info->u.vmdk.data = (ImageInfoSpecificVmdk) {
         .create_type = g_strdup(s->create_type),
         .cid = s->cid,
         .parent_cid = s->parent_cid,
     };
 
-    next = &spec_info->vmdk->extents;
+    next = &spec_info->u.vmdk.data->extents;
     for (i = 0; i < s->num_extents; i++) {
         *next = g_new0(ImageInfoList, 1);
         (*next)->value = vmdk_get_extent_info(&s->extents[i]);
@@ -2163,27 +2335,6 @@
     return 0;
 }
 
-static void vmdk_detach_aio_context(BlockDriverState *bs)
-{
-    BDRVVmdkState *s = bs->opaque;
-    int i;
-
-    for (i = 0; i < s->num_extents; i++) {
-        bdrv_detach_aio_context(s->extents[i].file);
-    }
-}
-
-static void vmdk_attach_aio_context(BlockDriverState *bs,
-                                    AioContext *new_context)
-{
-    BDRVVmdkState *s = bs->opaque;
-    int i;
-
-    for (i = 0; i < s->num_extents; i++) {
-        bdrv_attach_aio_context(s->extents[i].file, new_context);
-    }
-}
-
 static QemuOptsList vmdk_create_opts = {
     .name = "vmdk-create-opts",
     .head = QTAILQ_HEAD_INITIALIZER(vmdk_create_opts.head),
@@ -2211,6 +2362,12 @@
             .def_value_str = "off"
         },
         {
+            .name = BLOCK_OPT_HWVERSION,
+            .type = QEMU_OPT_STRING,
+            .help = "VMDK hardware version",
+            .def_value_str = "undefined"
+        },
+        {
             .name = BLOCK_OPT_SUBFMT,
             .type = QEMU_OPT_STRING,
             .help =
@@ -2234,10 +2391,10 @@
     .bdrv_open                    = vmdk_open,
     .bdrv_check                   = vmdk_check,
     .bdrv_reopen_prepare          = vmdk_reopen_prepare,
-    .bdrv_read                    = vmdk_co_read,
-    .bdrv_write                   = vmdk_co_write,
+    .bdrv_co_preadv               = vmdk_co_preadv,
+    .bdrv_co_pwritev              = vmdk_co_pwritev,
     .bdrv_write_compressed        = vmdk_write_compressed,
-    .bdrv_co_write_zeroes         = vmdk_co_write_zeroes,
+    .bdrv_co_pwrite_zeroes        = vmdk_co_pwrite_zeroes,
     .bdrv_close                   = vmdk_close,
     .bdrv_create                  = vmdk_create,
     .bdrv_co_flush_to_disk        = vmdk_co_flush,
@@ -2247,8 +2404,6 @@
     .bdrv_get_specific_info       = vmdk_get_specific_info,
     .bdrv_refresh_limits          = vmdk_refresh_limits,
     .bdrv_get_info                = vmdk_get_info,
-    .bdrv_detach_aio_context      = vmdk_detach_aio_context,
-    .bdrv_attach_aio_context      = vmdk_attach_aio_context,
 
     .supports_backing             = true,
     .create_opts                  = &vmdk_create_opts,
diff --git a/block/vpc.c b/block/vpc.c
index 38c4f02..43707ed 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -22,10 +22,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
 #include "qemu/module.h"
 #include "migration/migration.h"
+#include "qemu/bswap.h"
 #if defined(CONFIG_UUID)
 #include <uuid/uuid.h>
 #endif
@@ -42,30 +46,37 @@
     VHD_DIFFERENCING    = 4,
 };
 
-// Seconds since Jan 1, 2000 0:00:00 (UTC)
+/* Seconds since Jan 1, 2000 0:00:00 (UTC) */
 #define VHD_TIMESTAMP_BASE 946684800
 
-#define VHD_MAX_SECTORS       (65535LL * 255 * 255)
+#define VHD_CHS_MAX_C   65535LL
+#define VHD_CHS_MAX_H   16
+#define VHD_CHS_MAX_S   255
 
-// always big-endian
+#define VHD_MAX_SECTORS       0xff000000    /* 2040 GiB max image size */
+#define VHD_MAX_GEOMETRY      (VHD_CHS_MAX_C * VHD_CHS_MAX_H * VHD_CHS_MAX_S)
+
+#define VPC_OPT_FORCE_SIZE "force_size"
+
+/* always big-endian */
 typedef struct vhd_footer {
-    char        creator[8]; // "conectix"
+    char        creator[8]; /* "conectix" */
     uint32_t    features;
     uint32_t    version;
 
-    // Offset of next header structure, 0xFFFFFFFF if none
+    /* Offset of next header structure, 0xFFFFFFFF if none */
     uint64_t    data_offset;
 
-    // Seconds since Jan 1, 2000 0:00:00 (UTC)
+    /* Seconds since Jan 1, 2000 0:00:00 (UTC) */
     uint32_t    timestamp;
 
-    char        creator_app[4]; // "vpc "
+    char        creator_app[4]; /*  e.g., "vpc " */
     uint16_t    major;
     uint16_t    minor;
-    char        creator_os[4]; // "Wi2k"
+    char        creator_os[4]; /* "Wi2k" */
 
     uint64_t    orig_size;
-    uint64_t    size;
+    uint64_t    current_size;
 
     uint16_t    cyls;
     uint8_t     heads;
@@ -73,29 +84,29 @@
 
     uint32_t    type;
 
-    // Checksum of the Hard Disk Footer ("one's complement of the sum of all
-    // the bytes in the footer without the checksum field")
+    /* Checksum of the Hard Disk Footer ("one's complement of the sum of all
+       the bytes in the footer without the checksum field") */
     uint32_t    checksum;
 
-    // UUID used to identify a parent hard disk (backing file)
+    /* UUID used to identify a parent hard disk (backing file) */
     uint8_t     uuid[16];
 
     uint8_t     in_saved_state;
 } QEMU_PACKED VHDFooter;
 
 typedef struct vhd_dyndisk_header {
-    char        magic[8]; // "cxsparse"
+    char        magic[8]; /* "cxsparse" */
 
-    // Offset of next header structure, 0xFFFFFFFF if none
+    /* Offset of next header structure, 0xFFFFFFFF if none */
     uint64_t    data_offset;
 
-    // Offset of the Block Allocation Table (BAT)
+    /* Offset of the Block Allocation Table (BAT) */
     uint64_t    table_offset;
 
     uint32_t    version;
-    uint32_t    max_table_entries; // 32bit/entry
+    uint32_t    max_table_entries; /* 32bit/entry */
 
-    // 2 MB by default, must be a power of two
+    /* 2 MB by default, must be a power of two */
     uint32_t    block_size;
 
     uint32_t    checksum;
@@ -103,7 +114,7 @@
     uint32_t    parent_timestamp;
     uint32_t    reserved;
 
-    // Backing file name (in UTF-16)
+    /* Backing file name (in UTF-16) */
     uint8_t     parent_name[512];
 
     struct {
@@ -126,6 +137,8 @@
 
     uint32_t block_size;
     uint32_t bitmap_size;
+    bool force_use_chs;
+    bool force_use_sz;
 
 #ifdef CACHE
     uint8_t *pageentry_u8;
@@ -138,6 +151,22 @@
     Error *migration_blocker;
 } BDRVVPCState;
 
+#define VPC_OPT_SIZE_CALC "force_size_calc"
+static QemuOptsList vpc_runtime_opts = {
+    .name = "vpc-runtime-opts",
+    .head = QTAILQ_HEAD_INITIALIZER(vpc_runtime_opts.head),
+    .desc = {
+        {
+            .name = VPC_OPT_SIZE_CALC,
+            .type = QEMU_OPT_STRING,
+            .help = "Force disk size calculation to use either CHS geometry, "
+                    "or use the disk current_size specified in the VHD footer. "
+                    "{chs, current_size}"
+        },
+        { /* end of list */ }
+    }
+};
+
 static uint32_t vpc_checksum(uint8_t* buf, size_t size)
 {
     uint32_t res = 0;
@@ -157,6 +186,25 @@
     return 0;
 }
 
+static void vpc_parse_options(BlockDriverState *bs, QemuOpts *opts,
+                              Error **errp)
+{
+    BDRVVPCState *s = bs->opaque;
+    const char *size_calc;
+
+    size_calc = qemu_opt_get(opts, VPC_OPT_SIZE_CALC);
+
+    if (!size_calc) {
+       /* no override, use autodetect only */
+    } else if (!strcmp(size_calc, "current_size")) {
+        s->force_use_sz = true;
+    } else if (!strcmp(size_calc, "chs")) {
+        s->force_use_chs = true;
+    } else {
+        error_setg(errp, "Invalid size calculation mode: '%s'", size_calc);
+    }
+}
+
 static int vpc_open(BlockDriverState *bs, QDict *options, int flags,
                     Error **errp)
 {
@@ -164,25 +212,47 @@
     int i;
     VHDFooter *footer;
     VHDDynDiskHeader *dyndisk_header;
+    QemuOpts *opts = NULL;
+    Error *local_err = NULL;
+    bool use_chs;
     uint8_t buf[HEADER_SIZE];
     uint32_t checksum;
     uint64_t computed_size;
+    uint64_t pagetable_size;
     int disk_type = VHD_DYNAMIC;
     int ret;
 
+    opts = qemu_opts_create(&vpc_runtime_opts, NULL, 0, &error_abort);
+    qemu_opts_absorb_qdict(opts, options, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    vpc_parse_options(bs, opts, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        ret = -EINVAL;
+        goto fail;
+    }
+
     ret = bdrv_pread(bs->file, 0, s->footer_buf, HEADER_SIZE);
     if (ret < 0) {
+        error_setg(errp, "Unable to read VHD header");
         goto fail;
     }
 
     footer = (VHDFooter *) s->footer_buf;
     if (strncmp(footer->creator, "conectix", 8)) {
-        int64_t offset = bdrv_getlength(bs->file);
+        int64_t offset = bdrv_getlength(bs->file->bs);
         if (offset < 0) {
             ret = offset;
+            error_setg(errp, "Invalid file size");
             goto fail;
         } else if (offset < HEADER_SIZE) {
             ret = -EINVAL;
+            error_setg(errp, "File too small for a VHD header");
             goto fail;
         }
 
@@ -209,23 +279,50 @@
     /* Write 'checksum' back to footer, or else will leave it with zero. */
     footer->checksum = cpu_to_be32(checksum);
 
-    // The visible size of a image in Virtual PC depends on the geometry
-    // rather than on the size stored in the footer (the size in the footer
-    // is too large usually)
+    /* The visible size of a image in Virtual PC depends on the geometry
+       rather than on the size stored in the footer (the size in the footer
+       is too large usually) */
     bs->total_sectors = (int64_t)
         be16_to_cpu(footer->cyls) * footer->heads * footer->secs_per_cyl;
 
-    /* images created with disk2vhd report a far higher virtual size
-     * than expected with the cyls * heads * sectors_per_cyl formula.
-     * use the footer->size instead if the image was created with
-     * disk2vhd.
+    /* Microsoft Virtual PC and Microsoft Hyper-V produce and read
+     * VHD image sizes differently.  VPC will rely on CHS geometry,
+     * while Hyper-V and disk2vhd use the size specified in the footer.
+     *
+     * We use a couple of approaches to try and determine the correct method:
+     * look at the Creator App field, and look for images that have CHS
+     * geometry that is the maximum value.
+     *
+     * If the CHS geometry is the maximum CHS geometry, then we assume that
+     * the size is the footer->current_size to avoid truncation.  Otherwise,
+     * we follow the table based on footer->creator_app:
+     *
+     *  Known creator apps:
+     *      'vpc '  :  CHS              Virtual PC (uses disk geometry)
+     *      'qemu'  :  CHS              QEMU (uses disk geometry)
+     *      'qem2'  :  current_size     QEMU (uses current_size)
+     *      'win '  :  current_size     Hyper-V
+     *      'd2v '  :  current_size     Disk2vhd
+     *      'tap\0' :  current_size     XenServer
+     *      'CTXS'  :  current_size     XenConverter
+     *
+     *  The user can override the table values via drive options, however
+     *  even with an override we will still use current_size for images
+     *  that have CHS geometry of the maximum size.
      */
-    if (!strncmp(footer->creator_app, "d2v", 4)) {
-        bs->total_sectors = be64_to_cpu(footer->size) / BDRV_SECTOR_SIZE;
+    use_chs = (!!strncmp(footer->creator_app, "win ", 4) &&
+               !!strncmp(footer->creator_app, "qem2", 4) &&
+               !!strncmp(footer->creator_app, "d2v ", 4) &&
+               !!strncmp(footer->creator_app, "CTXS", 4) &&
+               !!memcmp(footer->creator_app, "tap", 4)) || s->force_use_chs;
+
+    if (!use_chs || bs->total_sectors == VHD_MAX_GEOMETRY || s->force_use_sz) {
+        bs->total_sectors = be64_to_cpu(footer->current_size) /
+                                        BDRV_SECTOR_SIZE;
     }
 
-    /* Allow a maximum disk size of approximately 2 TB */
-    if (bs->total_sectors >= VHD_MAX_SECTORS) {
+    /* Allow a maximum disk size of 2040 GiB */
+    if (bs->total_sectors > VHD_MAX_SECTORS) {
         ret = -EFBIG;
         goto fail;
     }
@@ -234,12 +331,14 @@
         ret = bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf,
                          HEADER_SIZE);
         if (ret < 0) {
+            error_setg(errp, "Error reading dynamic VHD header");
             goto fail;
         }
 
         dyndisk_header = (VHDDynDiskHeader *) buf;
 
         if (strncmp(dyndisk_header->magic, "cxsparse", 8)) {
+            error_setg(errp, "Invalid header magic");
             ret = -EINVAL;
             goto fail;
         }
@@ -255,22 +354,31 @@
         s->max_table_entries = be32_to_cpu(dyndisk_header->max_table_entries);
 
         if ((bs->total_sectors * 512) / s->block_size > 0xffffffffU) {
-            ret = -EINVAL;
-            goto fail;
-        }
-        if (s->max_table_entries > (VHD_MAX_SECTORS * 512) / s->block_size) {
+            error_setg(errp, "Too many blocks");
             ret = -EINVAL;
             goto fail;
         }
 
         computed_size = (uint64_t) s->max_table_entries * s->block_size;
         if (computed_size < bs->total_sectors * 512) {
+            error_setg(errp, "Page table too small");
             ret = -EINVAL;
             goto fail;
         }
 
-        s->pagetable = qemu_try_blockalign(bs->file, s->max_table_entries * 4);
+        if (s->max_table_entries > SIZE_MAX / 4 ||
+            s->max_table_entries > (int) INT_MAX / 4) {
+            error_setg(errp, "Max Table Entries too large (%" PRId32 ")",
+                        s->max_table_entries);
+            ret = -EINVAL;
+            goto fail;
+        }
+
+        pagetable_size = (uint64_t) s->max_table_entries * 4;
+
+        s->pagetable = qemu_try_blockalign(bs->file->bs, pagetable_size);
         if (s->pagetable == NULL) {
+            error_setg(errp, "Unable to allocate memory for page table");
             ret = -ENOMEM;
             goto fail;
         }
@@ -278,13 +386,14 @@
         s->bat_offset = be64_to_cpu(dyndisk_header->table_offset);
 
         ret = bdrv_pread(bs->file, s->bat_offset, s->pagetable,
-                         s->max_table_entries * 4);
+                         pagetable_size);
         if (ret < 0) {
+            error_setg(errp, "Error reading pagetable");
             goto fail;
         }
 
         s->free_data_block_offset =
-            (s->bat_offset + (s->max_table_entries * 4) + 511) & ~511;
+            ROUND_UP(s->bat_offset + pagetable_size, 512);
 
         for (i = 0; i < s->max_table_entries; i++) {
             be32_to_cpus(&s->pagetable[i]);
@@ -298,7 +407,7 @@
             }
         }
 
-        if (s->free_data_block_offset > bdrv_getlength(bs->file)) {
+        if (s->free_data_block_offset > bdrv_getlength(bs->file->bs)) {
             error_setg(errp, "block-vpc: free_data_block_offset points after "
                              "the end of file. The image has been truncated.");
             ret = -EINVAL;
@@ -318,9 +427,9 @@
     qemu_co_mutex_init(&s->lock);
 
     /* Disable migration when VHD images are used */
-    error_set(&s->migration_blocker,
-              QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-              "vpc", bdrv_get_device_name(bs), "live migration");
+    error_setg(&s->migration_blocker, "The vpc format used by node '%s' "
+               "does not support live migration",
+               bdrv_get_device_or_node_name(bs));
     migrate_add_blocker(s->migration_blocker);
 
     return 0;
@@ -346,28 +455,27 @@
  * The parameter write must be 1 if the offset will be used for a write
  * operation (the block bitmaps is updated then), 0 otherwise.
  */
-static inline int64_t get_sector_offset(BlockDriverState *bs,
-    int64_t sector_num, int write)
+static inline int64_t get_image_offset(BlockDriverState *bs, uint64_t offset,
+                                       bool write)
 {
     BDRVVPCState *s = bs->opaque;
-    uint64_t offset = sector_num * 512;
     uint64_t bitmap_offset, block_offset;
-    uint32_t pagetable_index, pageentry_index;
+    uint32_t pagetable_index, offset_in_block;
 
     pagetable_index = offset / s->block_size;
-    pageentry_index = (offset % s->block_size) / 512;
+    offset_in_block = offset % s->block_size;
 
     if (pagetable_index >= s->max_table_entries || s->pagetable[pagetable_index] == 0xffffffff)
-        return -1; // not allocated
+        return -1; /* not allocated */
 
     bitmap_offset = 512 * (uint64_t) s->pagetable[pagetable_index];
-    block_offset = bitmap_offset + s->bitmap_size + (512 * pageentry_index);
+    block_offset = bitmap_offset + s->bitmap_size + offset_in_block;
 
-    // We must ensure that we don't write to any sectors which are marked as
-    // unused in the bitmap. We get away with setting all bits in the block
-    // bitmap each time we write to a new block. This might cause Virtual PC to
-    // miss sparse read optimization, but it's not a problem in terms of
-    // correctness.
+    /* We must ensure that we don't write to any sectors which are marked as
+       unused in the bitmap. We get away with setting all bits in the block
+       bitmap each time we write to a new block. This might cause Virtual PC to
+       miss sparse read optimization, but it's not a problem in terms of
+       correctness. */
     if (write && (s->last_bitmap_offset != bitmap_offset)) {
         uint8_t bitmap[s->bitmap_size];
 
@@ -376,41 +484,15 @@
         bdrv_pwrite_sync(bs->file, bitmap_offset, bitmap, s->bitmap_size);
     }
 
-//    printf("sector: %" PRIx64 ", index: %x, offset: %x, bioff: %" PRIx64 ", bloff: %" PRIx64 "\n",
-//	sector_num, pagetable_index, pageentry_index,
-//	bitmap_offset, block_offset);
-
-// disabled by reason
-#if 0
-#ifdef CACHE
-    if (bitmap_offset != s->last_bitmap)
-    {
-	lseek(s->fd, bitmap_offset, SEEK_SET);
-
-	s->last_bitmap = bitmap_offset;
-
-	// Scary! Bitmap is stored as big endian 32bit entries,
-	// while we used to look it up byte by byte
-	read(s->fd, s->pageentry_u8, 512);
-	for (i = 0; i < 128; i++)
-	    be32_to_cpus(&s->pageentry_u32[i]);
-    }
-
-    if ((s->pageentry_u8[pageentry_index / 8] >> (pageentry_index % 8)) & 1)
-	return -1;
-#else
-    lseek(s->fd, bitmap_offset + (pageentry_index / 8), SEEK_SET);
-
-    read(s->fd, &bitmap_entry, 1);
-
-    if ((bitmap_entry >> (pageentry_index % 8)) & 1)
-	return -1; // not allocated
-#endif
-#endif
-
     return block_offset;
 }
 
+static inline int64_t get_sector_offset(BlockDriverState *bs,
+                                        int64_t sector_num, bool write)
+{
+    return get_image_offset(bs, sector_num * BDRV_SECTOR_SIZE, write);
+}
+
 /*
  * Writes the footer to the end of the image file. This is needed when the
  * file grows as it overwrites the old footer
@@ -437,7 +519,7 @@
  *
  * Returns the sectors' offset in the image file on success and < 0 on error
  */
-static int64_t alloc_block(BlockDriverState* bs, int64_t sector_num)
+static int64_t alloc_block(BlockDriverState* bs, int64_t offset)
 {
     BDRVVPCState *s = bs->opaque;
     int64_t bat_offset;
@@ -445,18 +527,17 @@
     int ret;
     uint8_t bitmap[s->bitmap_size];
 
-    // Check if sector_num is valid
-    if ((sector_num < 0) || (sector_num > bs->total_sectors))
-        return -1;
+    /* Check if sector_num is valid */
+    if ((offset < 0) || (offset > bs->total_sectors * BDRV_SECTOR_SIZE)) {
+        return -EINVAL;
+    }
 
-    // Write entry into in-memory BAT
-    index = (sector_num * 512) / s->block_size;
-    if (s->pagetable[index] != 0xFFFFFFFF)
-        return -1;
-
+    /* Write entry into in-memory BAT */
+    index = offset / s->block_size;
+    assert(s->pagetable[index] == 0xFFFFFFFF);
     s->pagetable[index] = s->free_data_block_offset / 512;
 
-    // Initialize the block's bitmap
+    /* Initialize the block's bitmap */
     memset(bitmap, 0xff, s->bitmap_size);
     ret = bdrv_pwrite_sync(bs->file, s->free_data_block_offset, bitmap,
         s->bitmap_size);
@@ -464,24 +545,24 @@
         return ret;
     }
 
-    // Write new footer (the old one will be overwritten)
+    /* Write new footer (the old one will be overwritten) */
     s->free_data_block_offset += s->block_size + s->bitmap_size;
     ret = rewrite_footer(bs);
     if (ret < 0)
         goto fail;
 
-    // Write BAT entry to disk
+    /* Write BAT entry to disk */
     bat_offset = s->bat_offset + (4 * index);
     bat_value = cpu_to_be32(s->pagetable[index]);
     ret = bdrv_pwrite_sync(bs->file, bat_offset, &bat_value, 4);
     if (ret < 0)
         goto fail;
 
-    return get_sector_offset(bs, sector_num, 0);
+    return get_image_offset(bs, offset, false);
 
 fail:
     s->free_data_block_offset -= (s->block_size + s->bitmap_size);
-    return -1;
+    return ret;
 }
 
 static int vpc_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
@@ -497,104 +578,151 @@
     return 0;
 }
 
-static int vpc_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vpc_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+              QEMUIOVector *qiov, int flags)
 {
     BDRVVPCState *s = bs->opaque;
     int ret;
-    int64_t offset;
-    int64_t sectors, sectors_per_block;
+    int64_t image_offset;
+    int64_t n_bytes;
+    int64_t bytes_done = 0;
     VHDFooter *footer = (VHDFooter *) s->footer_buf;
+    QEMUIOVector local_qiov;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_read(bs->file, sector_num, buf, nb_sectors);
+        return bdrv_co_preadv(bs->file, offset, bytes, qiov, 0);
     }
-    while (nb_sectors > 0) {
-        offset = get_sector_offset(bs, sector_num, 0);
 
-        sectors_per_block = s->block_size >> BDRV_SECTOR_BITS;
-        sectors = sectors_per_block - (sector_num % sectors_per_block);
-        if (sectors > nb_sectors) {
-            sectors = nb_sectors;
-        }
+    qemu_co_mutex_lock(&s->lock);
+    qemu_iovec_init(&local_qiov, qiov->niov);
 
-        if (offset == -1) {
-            memset(buf, 0, sectors * BDRV_SECTOR_SIZE);
+    while (bytes > 0) {
+        image_offset = get_image_offset(bs, offset, false);
+        n_bytes = MIN(bytes, s->block_size - (offset % s->block_size));
+
+        if (image_offset == -1) {
+            qemu_iovec_memset(qiov, bytes_done, 0, n_bytes);
         } else {
-            ret = bdrv_pread(bs->file, offset, buf,
-                sectors * BDRV_SECTOR_SIZE);
-            if (ret != sectors * BDRV_SECTOR_SIZE) {
-                return -1;
+            qemu_iovec_reset(&local_qiov);
+            qemu_iovec_concat(&local_qiov, qiov, bytes_done, n_bytes);
+
+            ret = bdrv_co_preadv(bs->file, image_offset, n_bytes,
+                                 &local_qiov, 0);
+            if (ret < 0) {
+                goto fail;
             }
         }
 
-        nb_sectors -= sectors;
-        sector_num += sectors;
-        buf += sectors * BDRV_SECTOR_SIZE;
+        bytes -= n_bytes;
+        offset += n_bytes;
+        bytes_done += n_bytes;
     }
-    return 0;
-}
 
-static coroutine_fn int vpc_co_read(BlockDriverState *bs, int64_t sector_num,
-                                    uint8_t *buf, int nb_sectors)
-{
-    int ret;
-    BDRVVPCState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = vpc_read(bs, sector_num, buf, nb_sectors);
+    ret = 0;
+fail:
+    qemu_iovec_destroy(&local_qiov);
     qemu_co_mutex_unlock(&s->lock);
+
     return ret;
 }
 
-static int vpc_write(BlockDriverState *bs, int64_t sector_num,
-    const uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vpc_co_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+               QEMUIOVector *qiov, int flags)
 {
     BDRVVPCState *s = bs->opaque;
-    int64_t offset;
-    int64_t sectors, sectors_per_block;
+    int64_t image_offset;
+    int64_t n_bytes;
+    int64_t bytes_done = 0;
     int ret;
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
+    QEMUIOVector local_qiov;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_write(bs->file, sector_num, buf, nb_sectors);
-    }
-    while (nb_sectors > 0) {
-        offset = get_sector_offset(bs, sector_num, 1);
-
-        sectors_per_block = s->block_size >> BDRV_SECTOR_BITS;
-        sectors = sectors_per_block - (sector_num % sectors_per_block);
-        if (sectors > nb_sectors) {
-            sectors = nb_sectors;
-        }
-
-        if (offset == -1) {
-            offset = alloc_block(bs, sector_num);
-            if (offset < 0)
-                return -1;
-        }
-
-        ret = bdrv_pwrite(bs->file, offset, buf, sectors * BDRV_SECTOR_SIZE);
-        if (ret != sectors * BDRV_SECTOR_SIZE) {
-            return -1;
-        }
-
-        nb_sectors -= sectors;
-        sector_num += sectors;
-        buf += sectors * BDRV_SECTOR_SIZE;
+        return bdrv_co_pwritev(bs->file, offset, bytes, qiov, 0);
     }
 
-    return 0;
+    qemu_co_mutex_lock(&s->lock);
+    qemu_iovec_init(&local_qiov, qiov->niov);
+
+    while (bytes > 0) {
+        image_offset = get_image_offset(bs, offset, true);
+        n_bytes = MIN(bytes, s->block_size - (offset % s->block_size));
+
+        if (image_offset == -1) {
+            image_offset = alloc_block(bs, offset);
+            if (image_offset < 0) {
+                ret = image_offset;
+                goto fail;
+            }
+        }
+
+        qemu_iovec_reset(&local_qiov);
+        qemu_iovec_concat(&local_qiov, qiov, bytes_done, n_bytes);
+
+        ret = bdrv_co_pwritev(bs->file, image_offset, n_bytes,
+                              &local_qiov, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+
+        bytes -= n_bytes;
+        offset += n_bytes;
+        bytes_done += n_bytes;
+    }
+
+    ret = 0;
+fail:
+    qemu_iovec_destroy(&local_qiov);
+    qemu_co_mutex_unlock(&s->lock);
+
+    return ret;
 }
 
-static coroutine_fn int vpc_co_write(BlockDriverState *bs, int64_t sector_num,
-                                     const uint8_t *buf, int nb_sectors)
+static int64_t coroutine_fn vpc_co_get_block_status(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)
 {
-    int ret;
     BDRVVPCState *s = bs->opaque;
-    qemu_co_mutex_lock(&s->lock);
-    ret = vpc_write(bs, sector_num, buf, nb_sectors);
-    qemu_co_mutex_unlock(&s->lock);
-    return ret;
+    VHDFooter *footer = (VHDFooter*) s->footer_buf;
+    int64_t start, offset;
+    bool allocated;
+    int n;
+
+    if (be32_to_cpu(footer->type) == VHD_FIXED) {
+        *pnum = nb_sectors;
+        *file = bs->file->bs;
+        return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID | BDRV_BLOCK_DATA |
+               (sector_num << BDRV_SECTOR_BITS);
+    }
+
+    offset = get_sector_offset(bs, sector_num, 0);
+    start = offset;
+    allocated = (offset != -1);
+    *pnum = 0;
+
+    do {
+        /* All sectors in a block are contiguous (without using the bitmap) */
+        n = ROUND_UP(sector_num + 1, s->block_size / BDRV_SECTOR_SIZE)
+          - sector_num;
+        n = MIN(n, nb_sectors);
+
+        *pnum += n;
+        sector_num += n;
+        nb_sectors -= n;
+        /* *pnum can't be greater than one block for allocated
+         * sectors since there is always a bitmap in between. */
+        if (allocated) {
+            *file = bs->file->bs;
+            return BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID | start;
+        }
+        if (nb_sectors == 0) {
+            break;
+        }
+        offset = get_sector_offset(bs, sector_num, 0);
+    } while (offset == -1);
+
+    return 0;
 }
 
 /*
@@ -605,7 +733,7 @@
  * Note that the geometry doesn't always exactly match total_sectors but
  * may round it down.
  *
- * Returns 0 on success, -EFBIG if the size is larger than ~2 TB. Override
+ * Returns 0 on success, -EFBIG if the size is larger than 2040 GiB. Override
  * the hardware EIDE and ATA-2 limit of 16 heads (max disk size of 127 GB)
  * and instead allow up to 255 heads.
  */
@@ -614,26 +742,20 @@
 {
     uint32_t cyls_times_heads;
 
-    /* Allow a maximum disk size of approximately 2 TB */
-    if (total_sectors > 65535LL * 255 * 255) {
-        return -EFBIG;
-    }
+    total_sectors = MIN(total_sectors, VHD_MAX_GEOMETRY);
 
-    if (total_sectors > 65535 * 16 * 63) {
+    if (total_sectors >= 65535LL * 16 * 63) {
         *secs_per_cyl = 255;
-        if (total_sectors > 65535 * 16 * 255) {
-            *heads = 255;
-        } else {
-            *heads = 16;
-        }
+        *heads = 16;
         cyls_times_heads = total_sectors / *secs_per_cyl;
     } else {
         *secs_per_cyl = 17;
         cyls_times_heads = total_sectors / *secs_per_cyl;
         *heads = (cyls_times_heads + 1023) / 1024;
 
-        if (*heads < 4)
+        if (*heads < 4) {
             *heads = 4;
+        }
 
         if (cyls_times_heads >= (*heads * 1024) || *heads > 16) {
             *secs_per_cyl = 31;
@@ -653,7 +775,7 @@
     return 0;
 }
 
-static int create_dynamic_disk(BlockDriverState *bs, uint8_t *buf,
+static int create_dynamic_disk(BlockBackend *blk, uint8_t *buf,
                                int64_t total_sectors)
 {
     VHDDynDiskHeader *dyndisk_header =
@@ -663,34 +785,34 @@
     int ret;
     int64_t offset = 0;
 
-    // Write the footer (twice: at the beginning and at the end)
+    /* Write the footer (twice: at the beginning and at the end) */
     block_size = 0x200000;
     num_bat_entries = (total_sectors + block_size / 512) / (block_size / 512);
 
-    ret = bdrv_pwrite_sync(bs, offset, buf, HEADER_SIZE);
-    if (ret) {
-        goto fail;
-    }
-
-    offset = 1536 + ((num_bat_entries * 4 + 511) & ~511);
-    ret = bdrv_pwrite_sync(bs, offset, buf, HEADER_SIZE);
+    ret = blk_pwrite(blk, offset, buf, HEADER_SIZE, 0);
     if (ret < 0) {
         goto fail;
     }
 
-    // Write the initial BAT
+    offset = 1536 + ((num_bat_entries * 4 + 511) & ~511);
+    ret = blk_pwrite(blk, offset, buf, HEADER_SIZE, 0);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /* Write the initial BAT */
     offset = 3 * 512;
 
     memset(buf, 0xFF, 512);
     for (i = 0; i < (num_bat_entries * 4 + 511) / 512; i++) {
-        ret = bdrv_pwrite_sync(bs, offset, buf, 512);
+        ret = blk_pwrite(blk, offset, buf, 512, 0);
         if (ret < 0) {
             goto fail;
         }
         offset += 512;
     }
 
-    // Prepare the Dynamic Disk Header
+    /* Prepare the Dynamic Disk Header */
     memset(buf, 0, 1024);
 
     memcpy(dyndisk_header->magic, "cxsparse", 8);
@@ -707,10 +829,10 @@
 
     dyndisk_header->checksum = cpu_to_be32(vpc_checksum(buf, 1024));
 
-    // Write the header
+    /* Write the header */
     offset = 512;
 
-    ret = bdrv_pwrite_sync(bs, offset, buf, 1024);
+    ret = blk_pwrite(blk, offset, buf, 1024, 0);
     if (ret < 0) {
         goto fail;
     }
@@ -719,7 +841,7 @@
     return ret;
 }
 
-static int create_fixed_disk(BlockDriverState *bs, uint8_t *buf,
+static int create_fixed_disk(BlockBackend *blk, uint8_t *buf,
                              int64_t total_size)
 {
     int ret;
@@ -727,12 +849,12 @@
     /* Add footer to total size */
     total_size += HEADER_SIZE;
 
-    ret = bdrv_truncate(bs, total_size);
+    ret = blk_truncate(blk, total_size);
     if (ret < 0) {
         return ret;
     }
 
-    ret = bdrv_pwrite_sync(bs, total_size - HEADER_SIZE, buf, HEADER_SIZE);
+    ret = blk_pwrite(blk, total_size - HEADER_SIZE, buf, HEADER_SIZE, 0);
     if (ret < 0) {
         return ret;
     }
@@ -753,8 +875,9 @@
     int64_t total_size;
     int disk_type;
     int ret = -EIO;
+    bool force_size;
     Error *local_err = NULL;
-    BlockDriverState *bs = NULL;
+    BlockBackend *blk = NULL;
 
     /* Read out options */
     total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0),
@@ -766,6 +889,7 @@
         } else if (!strcmp(disk_type_param, "fixed")) {
             disk_type = VHD_FIXED;
         } else {
+            error_setg(errp, "Invalid disk type, %s", disk_type_param);
             ret = -EINVAL;
             goto out;
         }
@@ -773,41 +897,67 @@
         disk_type = VHD_DYNAMIC;
     }
 
+    force_size = qemu_opt_get_bool_del(opts, VPC_OPT_FORCE_SIZE, false);
+
     ret = bdrv_create_file(filename, opts, &local_err);
     if (ret < 0) {
         error_propagate(errp, local_err);
         goto out;
     }
-    ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
-    if (ret < 0) {
+
+    blk = blk_new_open(filename, NULL, NULL,
+                       BDRV_O_RDWR | BDRV_O_PROTOCOL, &local_err);
+    if (blk == NULL) {
         error_propagate(errp, local_err);
+        ret = -EIO;
         goto out;
     }
 
+    blk_set_allow_write_beyond_eof(blk, true);
+
     /*
      * Calculate matching total_size and geometry. Increase the number of
      * sectors requested until we get enough (or fail). This ensures that
      * qemu-img convert doesn't truncate images, but rather rounds up.
+     *
+     * If the image size can't be represented by a spec conformant CHS geometry,
+     * we set the geometry to 65535 x 16 x 255 (CxHxS) sectors and use
+     * the image size from the VHD footer to calculate total_sectors.
      */
-    total_sectors = total_size / BDRV_SECTOR_SIZE;
-    for (i = 0; total_sectors > (int64_t)cyls * heads * secs_per_cyl; i++) {
-        if (calculate_geometry(total_sectors + i, &cyls, &heads,
-                               &secs_per_cyl))
-        {
-            ret = -EFBIG;
-            goto out;
+    if (force_size) {
+        /* This will force the use of total_size for sector count, below */
+        cyls         = VHD_CHS_MAX_C;
+        heads        = VHD_CHS_MAX_H;
+        secs_per_cyl = VHD_CHS_MAX_S;
+    } else {
+        total_sectors = MIN(VHD_MAX_GEOMETRY, total_size / BDRV_SECTOR_SIZE);
+        for (i = 0; total_sectors > (int64_t)cyls * heads * secs_per_cyl; i++) {
+            calculate_geometry(total_sectors + i, &cyls, &heads, &secs_per_cyl);
         }
     }
 
-    total_sectors = (int64_t) cyls * heads * secs_per_cyl;
+    if ((int64_t)cyls * heads * secs_per_cyl == VHD_MAX_GEOMETRY) {
+        total_sectors = total_size / BDRV_SECTOR_SIZE;
+        /* Allow a maximum disk size of 2040 GiB */
+        if (total_sectors > VHD_MAX_SECTORS) {
+            error_setg(errp, "Disk size is too large, max size is 2040 GiB");
+            ret = -EFBIG;
+            goto out;
+        }
+    } else {
+        total_sectors = (int64_t)cyls * heads * secs_per_cyl;
+        total_size = total_sectors * BDRV_SECTOR_SIZE;
+    }
 
     /* Prepare the Hard Disk Footer */
     memset(buf, 0, 1024);
 
     memcpy(footer->creator, "conectix", 8);
-    /* TODO Check if "qemu" creator_app is ok for VPC */
-    memcpy(footer->creator_app, "qemu", 4);
+    if (force_size) {
+        memcpy(footer->creator_app, "qem2", 4);
+    } else {
+        memcpy(footer->creator_app, "qemu", 4);
+    }
     memcpy(footer->creator_os, "Wi2k", 4);
 
     footer->features = cpu_to_be32(0x02);
@@ -822,13 +972,8 @@
     /* Version of Virtual PC 2007 */
     footer->major = cpu_to_be16(0x0005);
     footer->minor = cpu_to_be16(0x0003);
-    if (disk_type == VHD_DYNAMIC) {
-        footer->orig_size = cpu_to_be64(total_sectors * 512);
-        footer->size = cpu_to_be64(total_sectors * 512);
-    } else {
-        footer->orig_size = cpu_to_be64(total_size);
-        footer->size = cpu_to_be64(total_size);
-    }
+    footer->orig_size = cpu_to_be64(total_size);
+    footer->current_size = cpu_to_be64(total_size);
     footer->cyls = cpu_to_be16(cyls);
     footer->heads = heads;
     footer->secs_per_cyl = secs_per_cyl;
@@ -842,13 +987,16 @@
     footer->checksum = cpu_to_be32(vpc_checksum(buf, HEADER_SIZE));
 
     if (disk_type == VHD_DYNAMIC) {
-        ret = create_dynamic_disk(bs, buf, total_sectors);
+        ret = create_dynamic_disk(blk, buf, total_sectors);
     } else {
-        ret = create_fixed_disk(bs, buf, total_size);
+        ret = create_fixed_disk(blk, buf, total_size);
+    }
+    if (ret < 0) {
+        error_setg(errp, "Unable to create or write VHD header");
     }
 
 out:
-    bdrv_unref(bs);
+    blk_unref(blk);
     g_free(disk_type_param);
     return ret;
 }
@@ -859,7 +1007,7 @@
     VHDFooter *footer =  (VHDFooter *) s->footer_buf;
 
     if (be32_to_cpu(footer->type) == VHD_FIXED) {
-        return bdrv_has_zero_init(bs->file);
+        return bdrv_has_zero_init(bs->file->bs);
     } else {
         return 1;
     }
@@ -894,9 +1042,11 @@
                 "{dynamic (default) | fixed} "
         },
         {
-            .name = BLOCK_OPT_NOCOW,
+            .name = VPC_OPT_FORCE_SIZE,
             .type = QEMU_OPT_BOOL,
-            .help = "Turn off copy-on-write (valid only on btrfs)"
+            .help = "Force disk size calculation to use the actual size "
+                    "specified, rather than using the nearest CHS-based "
+                    "calculation"
         },
         { /* end of list */ }
     }
@@ -912,8 +1062,9 @@
     .bdrv_reopen_prepare    = vpc_reopen_prepare,
     .bdrv_create            = vpc_create,
 
-    .bdrv_read              = vpc_co_read,
-    .bdrv_write             = vpc_co_write,
+    .bdrv_co_preadv             = vpc_co_preadv,
+    .bdrv_co_pwritev            = vpc_co_pwritev,
+    .bdrv_co_get_block_status   = vpc_co_get_block_status,
 
     .bdrv_get_info          = vpc_get_info,
 
diff --git a/block/vvfat.c b/block/vvfat.c
index e684991..a9c0716 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -22,17 +22,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <sys/stat.h>
+#include "qemu/osdep.h"
 #include <dirent.h>
-#include "qemu-common.h"
+#include "qapi/error.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
+#include "qemu/bswap.h"
 #include "migration/migration.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qbool.h"
-#ifdef CONFIG_ANDROID
-#include "android/utils/file_io.h"
-#endif  // CONFIG_ANDROID
+#include "qapi/qmp/qstring.h"
+#include "qemu/cutils.h"
 
 #ifndef S_IWGRP
 #define S_IWGRP 0
@@ -114,15 +114,12 @@
 
 static inline void* array_get_next(array_t* array) {
     unsigned int next = array->next;
-    void* result;
 
     if (array_ensure_allocated(array, next) < 0)
 	return NULL;
 
     array->next = next + 1;
-    result = array_get(array, next);
-
-    return result;
+    return array_get(array, next);
 }
 
 static inline void* array_insert(array_t* array,unsigned int index,unsigned int count) {
@@ -325,6 +322,7 @@
 
     int fat_type; /* 16 or 32 */
     array_t fat,directory,mapping;
+    char volume_label[11];
 
     unsigned int cluster_size;
     unsigned int sectors_per_cluster;
@@ -343,9 +341,8 @@
     unsigned int current_cluster;
 
     /* write support */
-    BlockDriverState* write_target;
     char* qcow_filename;
-    BlockDriverState* qcow;
+    BdrvChild* qcow;
     void* fat2;
     char* used_clusters;
     array_t commits;
@@ -734,11 +731,7 @@
 	buffer = g_malloc(length);
 	snprintf(buffer,length,"%s/%s",dirname,entry->d_name);
 
-#ifdef CONFIG_ANDROID
-	if(android_stat(buffer,&st)<0) {
-#else  // !CONFIG_ANDROID
-	if(stat(buffer,&st)<0) {
-#endif  // CONFIG_ANDROID
+	if(qemu_stat(buffer,&st)<0) {
             g_free(buffer);
             continue;
 	}
@@ -866,7 +859,7 @@
     {
 	direntry_t* entry=array_get_next(&(s->directory));
 	entry->attributes=0x28; /* archive | volume label */
-        memcpy(entry->name, "QEMU VVFAT ", sizeof(entry->name));
+        memcpy(entry->name, s->volume_label, sizeof(entry->name));
     }
 
     /* Now build FAT, and write back information into directory */
@@ -975,7 +968,8 @@
     bootsector->u.fat16.signature=0x29;
     bootsector->u.fat16.id=cpu_to_le32(0xfabe1afd);
 
-    memcpy(bootsector->u.fat16.volume_label,"QEMU VVFAT ",11);
+    memcpy(bootsector->u.fat16.volume_label, s->volume_label,
+           sizeof(bootsector->u.fat16.volume_label));
     memcpy(bootsector->fat_type,(s->fat_type==12?"FAT12   ":s->fat_type==16?"FAT16   ":"FAT32   "),8);
     bootsector->magic[0]=0x55; bootsector->magic[1]=0xaa;
 
@@ -986,15 +980,9 @@
 static BDRVVVFATState *vvv = NULL;
 #endif
 
-static int enable_write_target(BDRVVVFATState *s, Error **errp);
+static int enable_write_target(BlockDriverState *bs, Error **errp);
 static int is_consistent(BDRVVVFATState *s);
 
-static void vvfat_rebind(BlockDriverState *bs)
-{
-    BDRVVVFATState *s = bs->opaque;
-    s->bs = bs;
-}
-
 static QemuOptsList runtime_opts = {
     .name = "vvfat",
     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
@@ -1015,6 +1003,11 @@
             .help = "Create a floppy rather than a hard disk image",
         },
         {
+            .name = "label",
+            .type = QEMU_OPT_STRING,
+            .help = "Use a volume label other than QEMU VVFAT",
+        },
+        {
             .name = "rw",
             .type = QEMU_OPT_BOOL,
             .help = "Make the image writable",
@@ -1066,8 +1059,8 @@
     /* Fill in the options QDict */
     qdict_put(options, "dir", qstring_from_str(filename));
     qdict_put(options, "fat-type", qint_from_int(fat_type));
-    qdict_put(options, "floppy", qbool_from_int(floppy));
-    qdict_put(options, "rw", qbool_from_int(rw));
+    qdict_put(options, "floppy", qbool_from_bool(floppy));
+    qdict_put(options, "rw", qbool_from_bool(rw));
 }
 
 static int vvfat_open(BlockDriverState *bs, QDict *options, int flags,
@@ -1076,7 +1069,7 @@
     BDRVVVFATState *s = bs->opaque;
     int cyls, heads, secs;
     bool floppy;
-    const char *dirname;
+    const char *dirname, *label;
     QemuOpts *opts;
     Error *local_err = NULL;
     int ret;
@@ -1103,6 +1096,20 @@
     s->fat_type = qemu_opt_get_number(opts, "fat-type", 0);
     floppy = qemu_opt_get_bool(opts, "floppy", false);
 
+    memset(s->volume_label, ' ', sizeof(s->volume_label));
+    label = qemu_opt_get(opts, "label");
+    if (label) {
+        size_t label_length = strlen(label);
+        if (label_length > 11) {
+            error_setg(errp, "vvfat label cannot be longer than 11 bytes");
+            ret = -EINVAL;
+            goto fail;
+        }
+        memcpy(s->volume_label, label, label_length);
+    } else {
+        memcpy(s->volume_label, "QEMU VVFAT", 10);
+    }
+
     if (floppy) {
         /* 1.44MB or 2.88MB floppy.  2.88MB can be FAT12 (default) or FAT16. */
         if (!s->fat_type) {
@@ -1150,8 +1157,8 @@
     s->current_cluster=0xffffffff;
 
     /* read only is the default for safety */
-    bs->read_only = 1;
-    s->qcow = s->write_target = NULL;
+    bs->read_only = true;
+    s->qcow = NULL;
     s->qcow_filename = NULL;
     s->fat2 = NULL;
     s->downcase_short_names = 1;
@@ -1162,11 +1169,11 @@
     s->sector_count = cyls * heads * secs - (s->first_sectors_number - 1);
 
     if (qemu_opt_get_bool(opts, "rw", false)) {
-        ret = enable_write_target(s, errp);
+        ret = enable_write_target(bs, errp);
         if (ret < 0) {
             goto fail;
         }
-        bs->read_only = 0;
+        bs->read_only = false;
     }
 
     bs->total_sectors = cyls * heads * secs;
@@ -1187,9 +1194,10 @@
 
     /* Disable migration when vvfat is used rw */
     if (s->qcow) {
-        error_set(&s->migration_blocker,
-                  QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-                  "vvfat (rw)", bdrv_get_device_name(bs), "live migration");
+        error_setg(&s->migration_blocker,
+                   "The vvfat (rw) format used by node '%s' "
+                   "does not support live migration",
+                   bdrv_get_device_or_node_name(bs));
         migrate_add_blocker(s->migration_blocker);
     }
 
@@ -1199,6 +1207,11 @@
     return ret;
 }
 
+static void vvfat_refresh_limits(BlockDriverState *bs, Error **errp)
+{
+    bs->bl.request_alignment = BDRV_SECTOR_SIZE; /* No sub-sector I/O */
+}
+
 static inline void vvfat_close_current_file(BDRVVVFATState *s)
 {
     if(s->current_mapping) {
@@ -1377,9 +1390,10 @@
 	   return -1;
 	if (s->qcow) {
 	    int n;
-            if (bdrv_is_allocated(s->qcow, sector_num, nb_sectors-i, &n)) {
-DLOG(fprintf(stderr, "sectors %d+%d allocated\n", (int)sector_num, n));
-                if (bdrv_read(s->qcow, sector_num, buf + i*0x200, n)) {
+            if (bdrv_is_allocated(s->qcow->bs, sector_num, nb_sectors-i, &n)) {
+                DLOG(fprintf(stderr, "sectors %d+%d allocated\n",
+                             (int)sector_num, n));
+                if (bdrv_read(s->qcow, sector_num, buf + i * 0x200, n)) {
                     return -1;
                 }
                 i += n - 1;
@@ -1410,14 +1424,31 @@
     return 0;
 }
 
-static coroutine_fn int vvfat_co_read(BlockDriverState *bs, int64_t sector_num,
-                                      uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vvfat_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+                QEMUIOVector *qiov, int flags)
 {
     int ret;
     BDRVVVFATState *s = bs->opaque;
+    uint64_t sector_num = offset >> BDRV_SECTOR_BITS;
+    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
+    void *buf;
+
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+
+    buf = g_try_malloc(bytes);
+    if (bytes && buf == NULL) {
+        return -ENOMEM;
+    }
+
     qemu_co_mutex_lock(&s->lock);
     ret = vvfat_read(bs, sector_num, buf, nb_sectors);
     qemu_co_mutex_unlock(&s->lock);
+
+    qemu_iovec_from_buf(qiov, 0, buf, bytes);
+    g_free(buf);
+
     return ret;
 }
 
@@ -1638,12 +1669,15 @@
     int was_modified = 0;
     int i, dummy;
 
-    if (s->qcow == NULL)
-	return 0;
+    if (s->qcow == NULL) {
+        return 0;
+    }
 
-    for (i = 0; !was_modified && i < s->sectors_per_cluster; i++)
-	was_modified = bdrv_is_allocated(s->qcow,
-		cluster2sector(s, cluster_num) + i, 1, &dummy);
+    for (i = 0; !was_modified && i < s->sectors_per_cluster; i++) {
+        was_modified = bdrv_is_allocated(s->qcow->bs,
+                                         cluster2sector(s, cluster_num) + i,
+                                         1, &dummy);
+    }
 
     return was_modified;
 }
@@ -1792,11 +1826,16 @@
 
 		vvfat_close_current_file(s);
                 for (i = 0; i < s->sectors_per_cluster; i++) {
-                    if (!bdrv_is_allocated(s->qcow, offset + i, 1, &dummy)) {
-                        if (vvfat_read(s->bs, offset, s->cluster_buffer, 1)) {
+                    int res;
+
+                    res = bdrv_is_allocated(s->qcow->bs, offset + i, 1, &dummy);
+                    if (!res) {
+                        res = vvfat_read(s->bs, offset, s->cluster_buffer, 1);
+                        if (res) {
                             return -1;
                         }
-                        if (bdrv_write(s->qcow, offset, s->cluster_buffer, 1)) {
+                        res = bdrv_write(s->qcow, offset, s->cluster_buffer, 1);
+                        if (res) {
                             return -2;
                         }
                     }
@@ -1930,8 +1969,7 @@
 		/* check file size with FAT */
 		cluster_count = get_cluster_count_for_direntry(s, direntries + i, path2);
 		if (cluster_count !=
-			(le32_to_cpu(direntries[i].size) + s->cluster_size
-			 - 1) / s->cluster_size) {
+            DIV_ROUND_UP(le32_to_cpu(direntries[i].size), s->cluster_size)) {
 		    DLOG(fprintf(stderr, "Cluster count mismatch\n"));
 		    goto fail;
 		}
@@ -2274,12 +2312,17 @@
 		factor * (old_cluster_count - new_cluster_count));
 
     for (c = first_cluster; !fat_eof(s, c); c = modified_fat_get(s, c)) {
+        direntry_t *first_direntry;
 	void* direntry = array_get(&(s->directory), current_dir_index);
 	int ret = vvfat_read(s->bs, cluster2sector(s, c), direntry,
 		s->sectors_per_cluster);
 	if (ret)
 	    return ret;
-	assert(!strncmp(s->directory.pointer, "QEMU", 4));
+
+        /* The first directory entry on the filesystem is the volume name */
+        first_direntry = (direntry_t*) s->directory.pointer;
+        assert(!memcmp(first_direntry->name, s->volume_label, 11));
+
 	current_dir_index += factor;
     }
 
@@ -2748,8 +2791,8 @@
 	return ret;
     }
 
-    if (s->qcow->drv->bdrv_make_empty) {
-        s->qcow->drv->bdrv_make_empty(s->qcow);
+    if (s->qcow->bs->drv->bdrv_make_empty) {
+        s->qcow->bs->drv->bdrv_make_empty(s->qcow->bs);
     }
 
     memset(s->used_clusters, 0, sector2cluster(s, s->sector_count));
@@ -2864,19 +2907,36 @@
     return 0;
 }
 
-static coroutine_fn int vvfat_co_write(BlockDriverState *bs, int64_t sector_num,
-                                       const uint8_t *buf, int nb_sectors)
+static int coroutine_fn
+vvfat_co_pwritev(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+                 QEMUIOVector *qiov, int flags)
 {
     int ret;
     BDRVVVFATState *s = bs->opaque;
+    uint64_t sector_num = offset >> BDRV_SECTOR_BITS;
+    int nb_sectors = bytes >> BDRV_SECTOR_BITS;
+    void *buf;
+
+    assert((offset & (BDRV_SECTOR_SIZE - 1)) == 0);
+    assert((bytes & (BDRV_SECTOR_SIZE - 1)) == 0);
+
+    buf = g_try_malloc(bytes);
+    if (bytes && buf == NULL) {
+        return -ENOMEM;
+    }
+    qemu_iovec_to_buf(qiov, 0, buf, bytes);
+
     qemu_co_mutex_lock(&s->lock);
     ret = vvfat_write(bs, sector_num, buf, nb_sectors);
     qemu_co_mutex_unlock(&s->lock);
+
+    g_free(buf);
+
     return ret;
 }
 
 static int64_t coroutine_fn vvfat_co_get_block_status(BlockDriverState *bs,
-	int64_t sector_num, int nb_sectors, int* n)
+	int64_t sector_num, int nb_sectors, int *n, BlockDriverState **file)
 {
     BDRVVVFATState* s = bs->opaque;
     *n = s->sector_count - sector_num;
@@ -2888,45 +2948,68 @@
     return BDRV_BLOCK_DATA;
 }
 
-static int write_target_commit(BlockDriverState *bs, int64_t sector_num,
-	const uint8_t* buffer, int nb_sectors) {
+static int coroutine_fn
+write_target_commit(BlockDriverState *bs, uint64_t offset, uint64_t bytes,
+                    QEMUIOVector *qiov, int flags)
+{
     BDRVVVFATState* s = *((BDRVVVFATState**) bs->opaque);
     return try_commit(s);
 }
 
 static void write_target_close(BlockDriverState *bs) {
     BDRVVVFATState* s = *((BDRVVVFATState**) bs->opaque);
-    bdrv_unref(s->qcow);
+    bdrv_unref_child(s->bs, s->qcow);
     g_free(s->qcow_filename);
 }
 
 static BlockDriver vvfat_write_target = {
     .format_name        = "vvfat_write_target",
-    .bdrv_write         = write_target_commit,
+    .bdrv_co_pwritev    = write_target_commit,
     .bdrv_close         = write_target_close,
 };
 
-static int enable_write_target(BDRVVVFATState *s, Error **errp)
+static void vvfat_qcow_options(int *child_flags, QDict *child_options,
+                               int parent_flags, QDict *parent_options)
 {
+    *child_flags = BDRV_O_RDWR | BDRV_O_NO_FLUSH;
+}
+
+static const BdrvChildRole child_vvfat_qcow = {
+    .inherit_options    = vvfat_qcow_options,
+};
+
+static int enable_write_target(BlockDriverState *bs, Error **errp)
+{
+    BDRVVVFATState *s = bs->opaque;
     BlockDriver *bdrv_qcow = NULL;
+    BlockDriverState *backing;
     QemuOpts *opts = NULL;
     int ret;
     int size = sector2cluster(s, s->sector_count);
+    QDict *options;
+
     s->used_clusters = calloc(size, 1);
 
     array_init(&(s->commits), sizeof(commit_t));
 
-    s->qcow_filename = g_malloc(1024);
-    ret = get_tmp_filename(s->qcow_filename, 1024);
+    s->qcow_filename = g_malloc(PATH_MAX);
+    ret = get_tmp_filename(s->qcow_filename, PATH_MAX);
     if (ret < 0) {
         error_setg_errno(errp, -ret, "can't create temporary file");
         goto err;
     }
 
     bdrv_qcow = bdrv_find_format("qcow");
+    if (!bdrv_qcow) {
+        error_setg(errp, "Failed to locate qcow driver");
+        ret = -ENOENT;
+        goto err;
+    }
+
     opts = qemu_opts_create(bdrv_qcow->create_opts, NULL, 0, &error_abort);
-    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, s->sector_count * 512);
-    qemu_opt_set(opts, BLOCK_OPT_BACKING_FILE, "fat:");
+    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, s->sector_count * 512,
+                        &error_abort);
+    qemu_opt_set(opts, BLOCK_OPT_BACKING_FILE, "fat:", &error_abort);
 
     ret = bdrv_create(bdrv_qcow, s->qcow_filename, opts, errp);
     qemu_opts_del(opts);
@@ -2934,11 +3017,13 @@
         goto err;
     }
 
-    s->qcow = NULL;
-    ret = bdrv_open(&s->qcow, s->qcow_filename, NULL, NULL,
-                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH,
-                    bdrv_qcow, errp);
-    if (ret < 0) {
+    options = qdict_new();
+    qdict_put(options, "write-target.driver", qstring_from_str("qcow"));
+    s->qcow = bdrv_open_child(s->qcow_filename, options, "write-target", bs,
+                              &child_vvfat_qcow, false, errp);
+    QDECREF(options);
+    if (!s->qcow) {
+        ret = -EINVAL;
         goto err;
     }
 
@@ -2946,10 +3031,13 @@
     unlink(s->qcow_filename);
 #endif
 
-    bdrv_set_backing_hd(s->bs, bdrv_new());
-    s->bs->backing_hd->drv = &vvfat_write_target;
-    s->bs->backing_hd->opaque = g_new(void *, 1);
-    *(void**)s->bs->backing_hd->opaque = s;
+    backing = bdrv_new();
+    bdrv_set_backing_hd(s->bs, backing);
+    bdrv_unref(backing);
+
+    s->bs->backing->bs->drv = &vvfat_write_target;
+    s->bs->backing->bs->opaque = g_new(void *, 1);
+    *(void**)s->bs->backing->bs->opaque = s;
 
     return 0;
 
@@ -2982,11 +3070,11 @@
 
     .bdrv_parse_filename    = vvfat_parse_filename,
     .bdrv_file_open         = vvfat_open,
+    .bdrv_refresh_limits    = vvfat_refresh_limits,
     .bdrv_close             = vvfat_close,
-    .bdrv_rebind            = vvfat_rebind,
 
-    .bdrv_read              = vvfat_co_read,
-    .bdrv_write             = vvfat_co_write,
+    .bdrv_co_preadv         = vvfat_co_preadv,
+    .bdrv_co_pwritev        = vvfat_co_pwritev,
     .bdrv_co_get_block_status = vvfat_co_get_block_status,
 };
 
diff --git a/block/win32-aio.c b/block/win32-aio.c
index 64e8682..95e3ab1 100644
--- a/block/win32-aio.c
+++ b/block/win32-aio.c
@@ -21,12 +21,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "block/block_int.h"
 #include "qemu/module.h"
 #include "block/aio.h"
-#include "raw-aio.h"
+#include "block/raw-aio.h"
 #include "qemu/event_notifier.h"
 #include "qemu/iov.h"
 #include <windows.h>
@@ -174,7 +175,7 @@
 void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
                                   AioContext *old_context)
 {
-    aio_set_event_notifier(old_context, &aio->e, NULL);
+    aio_set_event_notifier(old_context, &aio->e, false, NULL);
     aio->is_aio_context_attached = false;
 }
 
@@ -182,7 +183,8 @@
                                   AioContext *new_context)
 {
     aio->is_aio_context_attached = true;
-    aio_set_event_notifier(new_context, &aio->e, win32_aio_completion_cb);
+    aio_set_event_notifier(new_context, &aio->e, false,
+                           win32_aio_completion_cb);
 }
 
 QEMUWin32AIOState *win32_aio_init(void)
diff --git a/block/write-threshold.c b/block/write-threshold.c
new file mode 100644
index 0000000..cc2ca71
--- /dev/null
+++ b/block/write-threshold.c
@@ -0,0 +1,126 @@
+/*
+ * QEMU System Emulator block write threshold notification
+ *
+ * Copyright Red Hat, Inc. 2014
+ *
+ * Authors:
+ *  Francesco Romani <fromani@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "block/block_int.h"
+#include "qemu/coroutine.h"
+#include "block/write-threshold.h"
+#include "qemu/notify.h"
+#include "qapi-event.h"
+#include "qmp-commands.h"
+
+
+uint64_t bdrv_write_threshold_get(const BlockDriverState *bs)
+{
+    return bs->write_threshold_offset;
+}
+
+bool bdrv_write_threshold_is_set(const BlockDriverState *bs)
+{
+    return bs->write_threshold_offset > 0;
+}
+
+static void write_threshold_disable(BlockDriverState *bs)
+{
+    if (bdrv_write_threshold_is_set(bs)) {
+        notifier_with_return_remove(&bs->write_threshold_notifier);
+        bs->write_threshold_offset = 0;
+    }
+}
+
+uint64_t bdrv_write_threshold_exceeded(const BlockDriverState *bs,
+                                       const BdrvTrackedRequest *req)
+{
+    if (bdrv_write_threshold_is_set(bs)) {
+        if (req->offset > bs->write_threshold_offset) {
+            return (req->offset - bs->write_threshold_offset) + req->bytes;
+        }
+        if ((req->offset + req->bytes) > bs->write_threshold_offset) {
+            return (req->offset + req->bytes) - bs->write_threshold_offset;
+        }
+    }
+    return 0;
+}
+
+static int coroutine_fn before_write_notify(NotifierWithReturn *notifier,
+                                            void *opaque)
+{
+    BdrvTrackedRequest *req = opaque;
+    BlockDriverState *bs = req->bs;
+    uint64_t amount = 0;
+
+    amount = bdrv_write_threshold_exceeded(bs, req);
+    if (amount > 0) {
+        qapi_event_send_block_write_threshold(
+            bs->node_name,
+            amount,
+            bs->write_threshold_offset,
+            &error_abort);
+
+        /* autodisable to avoid flooding the monitor */
+        write_threshold_disable(bs);
+    }
+
+    return 0; /* should always let other notifiers run */
+}
+
+static void write_threshold_register_notifier(BlockDriverState *bs)
+{
+    bs->write_threshold_notifier.notify = before_write_notify;
+    notifier_with_return_list_add(&bs->before_write_notifiers,
+                                  &bs->write_threshold_notifier);
+}
+
+static void write_threshold_update(BlockDriverState *bs,
+                                   int64_t threshold_bytes)
+{
+    bs->write_threshold_offset = threshold_bytes;
+}
+
+void bdrv_write_threshold_set(BlockDriverState *bs, uint64_t threshold_bytes)
+{
+    if (bdrv_write_threshold_is_set(bs)) {
+        if (threshold_bytes > 0) {
+            write_threshold_update(bs, threshold_bytes);
+        } else {
+            write_threshold_disable(bs);
+        }
+    } else {
+        if (threshold_bytes > 0) {
+            /* avoid multiple registration */
+            write_threshold_register_notifier(bs);
+            write_threshold_update(bs, threshold_bytes);
+        }
+        /* discard bogus disable request */
+    }
+}
+
+void qmp_block_set_write_threshold(const char *node_name,
+                                   uint64_t threshold_bytes,
+                                   Error **errp)
+{
+    BlockDriverState *bs;
+    AioContext *aio_context;
+
+    bs = bdrv_find_node(node_name);
+    if (!bs) {
+        error_setg(errp, "Device '%s' not found", node_name);
+        return;
+    }
+
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_write_threshold_set(bs, threshold_bytes);
+
+    aio_context_release(aio_context);
+}
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
index 4eed45d..12cae0e 100644
--- a/blockdev-nbd.c
+++ b/blockdev-nbd.c
@@ -9,75 +9,146 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "hw/block/block.h"
-#include "monitor/monitor.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
 #include "trace.h"
 #include "block/nbd.h"
-#include "qemu/sockets.h"
+#include "io/channel-socket.h"
 
-static int server_fd = -1;
+typedef struct NBDServerData {
+    QIOChannelSocket *listen_ioc;
+    int watch;
+    QCryptoTLSCreds *tlscreds;
+} NBDServerData;
 
-static void nbd_accept(void *opaque)
+static NBDServerData *nbd_server;
+
+
+static gboolean nbd_accept(QIOChannel *ioc, GIOCondition condition,
+                           gpointer opaque)
 {
-    struct sockaddr_in addr;
-    socklen_t addr_len = sizeof(addr);
+    QIOChannelSocket *cioc;
 
-    int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    if (fd >= 0 && !nbd_client_new(NULL, fd, nbd_client_put)) {
-        shutdown(fd, 2);
-        close(fd);
+    if (!nbd_server) {
+        return FALSE;
     }
+
+    cioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc),
+                                     NULL);
+    if (!cioc) {
+        return TRUE;
+    }
+
+    nbd_client_new(NULL, cioc,
+                   nbd_server->tlscreds, NULL,
+                   nbd_client_put);
+    object_unref(OBJECT(cioc));
+    return TRUE;
 }
 
-void qmp_nbd_server_start(SocketAddress *addr, Error **errp)
+
+static void nbd_server_free(NBDServerData *server)
 {
-    if (server_fd != -1) {
+    if (!server) {
+        return;
+    }
+
+    if (server->watch != -1) {
+        g_source_remove(server->watch);
+    }
+    object_unref(OBJECT(server->listen_ioc));
+    if (server->tlscreds) {
+        object_unref(OBJECT(server->tlscreds));
+    }
+
+    g_free(server);
+}
+
+static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp)
+{
+    Object *obj;
+    QCryptoTLSCreds *creds;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), id);
+    if (!obj) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   id);
+        return NULL;
+    }
+    creds = (QCryptoTLSCreds *)
+        object_dynamic_cast(obj, TYPE_QCRYPTO_TLS_CREDS);
+    if (!creds) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   id);
+        return NULL;
+    }
+
+    if (creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        error_setg(errp,
+                   "Expecting TLS credentials with a server endpoint");
+        return NULL;
+    }
+    object_ref(obj);
+    return creds;
+}
+
+
+void qmp_nbd_server_start(SocketAddress *addr,
+                          bool has_tls_creds, const char *tls_creds,
+                          Error **errp)
+{
+    if (nbd_server) {
         error_setg(errp, "NBD server already running");
         return;
     }
 
-    server_fd = socket_listen_addr(addr, errp);
-    if (server_fd != -1) {
-        qemu_set_fd_handler2(server_fd, NULL, nbd_accept, NULL, NULL);
+    nbd_server = g_new0(NBDServerData, 1);
+    nbd_server->watch = -1;
+    nbd_server->listen_ioc = qio_channel_socket_new();
+    if (qio_channel_socket_listen_sync(
+            nbd_server->listen_ioc, addr, errp) < 0) {
+        goto error;
     }
-}
 
-/* Hook into the BlockDriverState notifiers to close the export when
- * the file is closed.
- */
-typedef struct NBDCloseNotifier {
-    Notifier n;
-    NBDExport *exp;
-    QTAILQ_ENTRY(NBDCloseNotifier) next;
-} NBDCloseNotifier;
+    if (has_tls_creds) {
+        nbd_server->tlscreds = nbd_get_tls_creds(tls_creds, errp);
+        if (!nbd_server->tlscreds) {
+            goto error;
+        }
 
-static QTAILQ_HEAD(, NBDCloseNotifier) close_notifiers =
-    QTAILQ_HEAD_INITIALIZER(close_notifiers);
+        if (addr->type != SOCKET_ADDRESS_KIND_INET) {
+            error_setg(errp, "TLS is only supported with IPv4/IPv6");
+            goto error;
+        }
+    }
 
-static void nbd_close_notifier(Notifier *n, void *data)
-{
-    NBDCloseNotifier *cn = DO_UPCAST(NBDCloseNotifier, n, n);
+    nbd_server->watch = qio_channel_add_watch(
+        QIO_CHANNEL(nbd_server->listen_ioc),
+        G_IO_IN,
+        nbd_accept,
+        NULL,
+        NULL);
 
-    notifier_remove(&cn->n);
-    QTAILQ_REMOVE(&close_notifiers, cn, next);
+    return;
 
-    nbd_export_close(cn->exp);
-    nbd_export_put(cn->exp);
-    g_free(cn);
+ error:
+    nbd_server_free(nbd_server);
+    nbd_server = NULL;
 }
 
 void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
                         Error **errp)
 {
-    BlockDriverState *bs;
+    BlockBackend *blk;
     NBDExport *exp;
-    NBDCloseNotifier *n;
 
-    if (server_fd == -1) {
+    if (!nbd_server) {
         error_setg(errp, "NBD server not running");
         return;
     }
@@ -87,44 +158,42 @@
         return;
     }
 
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
-    if (!bdrv_is_inserted(bs)) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+    if (!blk_is_inserted(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         return;
     }
 
     if (!has_writable) {
         writable = false;
     }
-    if (bdrv_is_read_only(bs)) {
+    if (blk_is_read_only(blk)) {
         writable = false;
     }
 
-    exp = nbd_export_new(bs, 0, -1, writable ? 0 : NBD_FLAG_READ_ONLY, NULL);
+    exp = nbd_export_new(blk, 0, -1, writable ? 0 : NBD_FLAG_READ_ONLY, NULL,
+                         errp);
+    if (!exp) {
+        return;
+    }
 
     nbd_export_set_name(exp, device);
 
-    n = g_new0(NBDCloseNotifier, 1);
-    n->n.notify = nbd_close_notifier;
-    n->exp = exp;
-    bdrv_add_close_notifier(bs, &n->n);
-    QTAILQ_INSERT_TAIL(&close_notifiers, n, next);
+    /* The list of named exports has a strong reference to this export now and
+     * our only way of accessing it is through nbd_export_find(), so we can drop
+     * the strong reference that is @exp. */
+    nbd_export_put(exp);
 }
 
 void qmp_nbd_server_stop(Error **errp)
 {
-    while (!QTAILQ_EMPTY(&close_notifiers)) {
-        NBDCloseNotifier *cn = QTAILQ_FIRST(&close_notifiers);
-        nbd_close_notifier(&cn->n, nbd_export_get_blockdev(cn->exp));
-    }
+    nbd_export_close_all();
 
-    if (server_fd != -1) {
-        qemu_set_fd_handler2(server_fd, NULL, NULL, NULL, NULL);
-        close(server_fd);
-        server_fd = -1;
-    }
+    nbd_server_free(nbd_server);
+    nbd_server = NULL;
 }
diff --git a/blockdev.c b/blockdev.c
index 57910b8..2161400 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -30,15 +30,19 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "block/blockjob.h"
+#include "block/throttle-groups.h"
 #include "monitor/monitor.h"
+#include "qemu/error-report.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qapi/qmp/types.h"
 #include "qapi-visit.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/util.h"
 #include "sysemu/sysemu.h"
@@ -46,6 +50,13 @@
 #include "qmp-commands.h"
 #include "trace.h"
 #include "sysemu/arch_init.h"
+#include "qemu/cutils.h"
+#include "qemu/help_option.h"
+
+static QTAILQ_HEAD(, BlockDriverState) monitor_bdrv_states =
+    QTAILQ_HEAD_INITIALIZER(monitor_bdrv_states);
+
+static int do_open_tray(const char *device, bool force, Error **errp);
 
 static const char *const if_name[IF_COUNT] = {
     [IF_NONE] = "none",
@@ -64,7 +75,7 @@
      * Do not change these numbers!  They govern how drive option
      * index maps to unit and bus.  That mapping is ABI.
      *
-     * All controllers used to imlement if=T drives need to support
+     * All controllers used to implement if=T drives need to support
      * if_max_devs[T] units, for any T with if_max_devs[T] != 0.
      * Otherwise, some index values map to "impossible" bus, unit
      * values.
@@ -121,15 +132,17 @@
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
-    aio_context_acquire(aio_context);
+    if (bs) {
+        aio_context = bdrv_get_aio_context(bs);
+        aio_context_acquire(aio_context);
 
-    if (bs->job) {
-        block_job_cancel(bs->job);
+        if (bs->job) {
+            block_job_cancel(bs->job);
+        }
+
+        aio_context_release(aio_context);
     }
 
-    aio_context_release(aio_context);
-
     dinfo->auto_del = 1;
 }
 
@@ -138,6 +151,7 @@
     DriveInfo *dinfo = blk_legacy_dinfo(blk);
 
     if (dinfo && dinfo->auto_del) {
+        monitor_remove_blk(blk);
         blk_unref(blk);
     }
 }
@@ -173,28 +187,26 @@
 
 QemuOpts *drive_def(const char *optstr)
 {
-    return qemu_opts_parse(qemu_find_opts("drive"), optstr, 0);
+    return qemu_opts_parse_noisily(qemu_find_opts("drive"), optstr, false);
 }
 
 QemuOpts *drive_add(BlockInterfaceType type, int index, const char *file,
                     const char *optstr)
 {
     QemuOpts *opts;
-    char buf[32];
 
     opts = drive_def(optstr);
     if (!opts) {
         return NULL;
     }
     if (type != IF_DEFAULT) {
-        qemu_opt_set(opts, "if", if_name[type]);
+        qemu_opt_set(opts, "if", if_name[type], &error_abort);
     }
     if (index >= 0) {
-        snprintf(buf, sizeof(buf), "%d", index);
-        qemu_opt_set(opts, "index", buf);
+        qemu_opt_set_number(opts, "index", index, &error_abort);
     }
     if (file)
-        qemu_opt_set(opts, "file", file);
+        qemu_opt_set(opts, "file", file, &error_abort);
     return opts;
 }
 
@@ -228,8 +240,8 @@
             dinfo->type != IF_NONE) {
             fprintf(stderr, "Warning: Orphaned drive without device: "
                     "id=%s,file=%s,if=%s,bus=%d,unit=%d\n",
-                    blk_name(blk), blk_bs(blk)->filename, if_name[dinfo->type],
-                    dinfo->bus, dinfo->unit);
+                    blk_name(blk), blk_bs(blk) ? blk_bs(blk)->filename : "",
+                    if_name[dinfo->type], dinfo->bus, dinfo->unit);
             rs = true;
         }
     }
@@ -280,32 +292,6 @@
     BlockDriverState *bs;
 } BDRVPutRefBH;
 
-static void bdrv_put_ref_bh(void *opaque)
-{
-    BDRVPutRefBH *s = opaque;
-
-    bdrv_unref(s->bs);
-    qemu_bh_delete(s->bh);
-    g_free(s);
-}
-
-/*
- * Release a BDS reference in a BH
- *
- * It is not safe to use bdrv_unref() from a callback function when the callers
- * still need the BlockDriverState.  In such cases we schedule a BH to release
- * the reference.
- */
-static void bdrv_put_ref_bh_schedule(BlockDriverState *bs)
-{
-    BDRVPutRefBH *s;
-
-    s = g_new(BDRVPutRefBH, 1);
-    s->bh = qemu_bh_new(bdrv_put_ref_bh, s);
-    s->bs = bs;
-    qemu_bh_schedule(s->bh);
-}
-
 static int parse_block_error_action(const char *buf, bool is_read, Error **errp)
 {
     if (!strcmp(buf, "ignore")) {
@@ -323,44 +309,181 @@
     }
 }
 
-static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
+static bool parse_stats_intervals(BlockAcctStats *stats, QList *intervals,
+                                  Error **errp)
 {
-    if (throttle_conflicting(cfg)) {
-        error_setg(errp, "bps/iops/max total values and read/write values"
-                         " cannot be used at the same time");
-        return false;
-    }
+    const QListEntry *entry;
+    for (entry = qlist_first(intervals); entry; entry = qlist_next(entry)) {
+        switch (qobject_type(entry->value)) {
 
-    if (!throttle_is_valid(cfg)) {
-        error_setg(errp, "bps/iops/maxs values must be 0 or greater");
-        return false;
-    }
+        case QTYPE_QSTRING: {
+            unsigned long long length;
+            const char *str = qstring_get_str(qobject_to_qstring(entry->value));
+            if (parse_uint_full(str, &length, 10) == 0 &&
+                length > 0 && length <= UINT_MAX) {
+                block_acct_add_interval(stats, (unsigned) length);
+            } else {
+                error_setg(errp, "Invalid interval length: %s", str);
+                return false;
+            }
+            break;
+        }
 
+        case QTYPE_QINT: {
+            int64_t length = qint_get_int(qobject_to_qint(entry->value));
+            if (length > 0 && length <= UINT_MAX) {
+                block_acct_add_interval(stats, (unsigned) length);
+            } else {
+                error_setg(errp, "Invalid interval length: %" PRId64, length);
+                return false;
+            }
+            break;
+        }
+
+        default:
+            error_setg(errp, "The specification of stats-intervals is invalid");
+            return false;
+        }
+    }
     return true;
 }
 
 typedef enum { MEDIA_DISK, MEDIA_CDROM } DriveMediaType;
 
+/* All parameters but @opts are optional and may be set to NULL. */
+static void extract_common_blockdev_options(QemuOpts *opts, int *bdrv_flags,
+    const char **throttling_group, ThrottleConfig *throttle_cfg,
+    BlockdevDetectZeroesOptions *detect_zeroes, Error **errp)
+{
+    const char *discard;
+    Error *local_error = NULL;
+    const char *aio;
+
+    if (bdrv_flags) {
+        if (!qemu_opt_get_bool(opts, "read-only", false)) {
+            *bdrv_flags |= BDRV_O_RDWR;
+        }
+        if (qemu_opt_get_bool(opts, "copy-on-read", false)) {
+            *bdrv_flags |= BDRV_O_COPY_ON_READ;
+        }
+
+        if ((discard = qemu_opt_get(opts, "discard")) != NULL) {
+            if (bdrv_parse_discard_flags(discard, bdrv_flags) != 0) {
+                error_setg(errp, "Invalid discard option");
+                return;
+            }
+        }
+
+        if ((aio = qemu_opt_get(opts, "aio")) != NULL) {
+            if (!strcmp(aio, "native")) {
+                *bdrv_flags |= BDRV_O_NATIVE_AIO;
+            } else if (!strcmp(aio, "threads")) {
+                /* this is the default */
+            } else {
+               error_setg(errp, "invalid aio option");
+               return;
+            }
+        }
+    }
+
+    /* disk I/O throttling */
+    if (throttling_group) {
+        *throttling_group = qemu_opt_get(opts, "throttling.group");
+    }
+
+    if (throttle_cfg) {
+        throttle_config_init(throttle_cfg);
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].avg =
+            qemu_opt_get_number(opts, "throttling.bps-total", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].avg  =
+            qemu_opt_get_number(opts, "throttling.bps-read", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].avg =
+            qemu_opt_get_number(opts, "throttling.bps-write", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].avg =
+            qemu_opt_get_number(opts, "throttling.iops-total", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].avg =
+            qemu_opt_get_number(opts, "throttling.iops-read", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].avg =
+            qemu_opt_get_number(opts, "throttling.iops-write", 0);
+
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].max =
+            qemu_opt_get_number(opts, "throttling.bps-total-max", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].max  =
+            qemu_opt_get_number(opts, "throttling.bps-read-max", 0);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].max =
+            qemu_opt_get_number(opts, "throttling.bps-write-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].max =
+            qemu_opt_get_number(opts, "throttling.iops-total-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].max =
+            qemu_opt_get_number(opts, "throttling.iops-read-max", 0);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].max =
+            qemu_opt_get_number(opts, "throttling.iops-write-max", 0);
+
+        throttle_cfg->buckets[THROTTLE_BPS_TOTAL].burst_length =
+            qemu_opt_get_number(opts, "throttling.bps-total-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_BPS_READ].burst_length  =
+            qemu_opt_get_number(opts, "throttling.bps-read-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_BPS_WRITE].burst_length =
+            qemu_opt_get_number(opts, "throttling.bps-write-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_OPS_TOTAL].burst_length =
+            qemu_opt_get_number(opts, "throttling.iops-total-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_OPS_READ].burst_length =
+            qemu_opt_get_number(opts, "throttling.iops-read-max-length", 1);
+        throttle_cfg->buckets[THROTTLE_OPS_WRITE].burst_length =
+            qemu_opt_get_number(opts, "throttling.iops-write-max-length", 1);
+
+        throttle_cfg->op_size =
+            qemu_opt_get_number(opts, "throttling.iops-size", 0);
+
+        if (!throttle_is_valid(throttle_cfg, errp)) {
+            return;
+        }
+    }
+
+    if (detect_zeroes) {
+        *detect_zeroes =
+            qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
+                            qemu_opt_get(opts, "detect-zeroes"),
+                            BLOCKDEV_DETECT_ZEROES_OPTIONS__MAX,
+                            BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                            &local_error);
+        if (local_error) {
+            error_propagate(errp, local_error);
+            return;
+        }
+
+        if (bdrv_flags &&
+            *detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
+            !(*bdrv_flags & BDRV_O_UNMAP))
+        {
+            error_setg(errp, "setting detect-zeroes to unmap is not allowed "
+                             "without setting discard operation to unmap");
+            return;
+        }
+    }
+}
+
 /* Takes the ownership of bs_opts */
 static BlockBackend *blockdev_init(const char *file, QDict *bs_opts,
                                    Error **errp)
 {
     const char *buf;
-    int ro = 0;
     int bdrv_flags = 0;
     int on_read_error, on_write_error;
+    bool account_invalid, account_failed;
+    bool writethrough;
     BlockBackend *blk;
     BlockDriverState *bs;
     ThrottleConfig cfg;
     int snapshot = 0;
-    bool copy_on_read;
-    int ret;
     Error *error = NULL;
     QemuOpts *opts;
+    QDict *interval_dict = NULL;
+    QList *interval_list = NULL;
     const char *id;
-    bool has_driver_specific_opts;
-    BlockdevDetectZeroesOptions detect_zeroes;
-    BlockDriver *drv = NULL;
+    BlockdevDetectZeroesOptions detect_zeroes =
+        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
+    const char *throttling_group = NULL;
 
     /* Check common options by copying from bs_opts to opts, all other options
      * stay in bs_opts for processing by bdrv_open(). */
@@ -381,42 +504,31 @@
         qdict_del(bs_opts, "id");
     }
 
-    has_driver_specific_opts = !!qdict_size(bs_opts);
-
     /* extract parameters */
     snapshot = qemu_opt_get_bool(opts, "snapshot", 0);
-    ro = qemu_opt_get_bool(opts, "read-only", 0);
-    copy_on_read = qemu_opt_get_bool(opts, "copy-on-read", false);
 
-    if ((buf = qemu_opt_get(opts, "discard")) != NULL) {
-        if (bdrv_parse_discard_flags(buf, &bdrv_flags) != 0) {
-            error_setg(errp, "invalid discard option");
-            goto early_err;
-        }
+    account_invalid = qemu_opt_get_bool(opts, "stats-account-invalid", true);
+    account_failed = qemu_opt_get_bool(opts, "stats-account-failed", true);
+
+    writethrough = !qemu_opt_get_bool(opts, BDRV_OPT_CACHE_WB, true);
+
+    id = qemu_opts_id(opts);
+
+    qdict_extract_subqdict(bs_opts, &interval_dict, "stats-intervals.");
+    qdict_array_split(interval_dict, &interval_list);
+
+    if (qdict_size(interval_dict) != 0) {
+        error_setg(errp, "Invalid option stats-intervals.%s",
+                   qdict_first(interval_dict)->key);
+        goto early_err;
     }
 
-    if (qemu_opt_get_bool(opts, "cache.writeback", true)) {
-        bdrv_flags |= BDRV_O_CACHE_WB;
+    extract_common_blockdev_options(opts, &bdrv_flags, &throttling_group, &cfg,
+                                    &detect_zeroes, &error);
+    if (error) {
+        error_propagate(errp, error);
+        goto early_err;
     }
-    if (qemu_opt_get_bool(opts, "cache.direct", false)) {
-        bdrv_flags |= BDRV_O_NOCACHE;
-    }
-    if (qemu_opt_get_bool(opts, "cache.no-flush", false)) {
-        bdrv_flags |= BDRV_O_NO_FLUSH;
-    }
-
-#ifdef CONFIG_LINUX_AIO
-    if ((buf = qemu_opt_get(opts, "aio")) != NULL) {
-        if (!strcmp(buf, "native")) {
-            bdrv_flags |= BDRV_O_NATIVE_AIO;
-        } else if (!strcmp(buf, "threads")) {
-            /* this is the default */
-        } else {
-           error_setg(errp, "invalid aio option");
-           goto early_err;
-        }
-    }
-#endif
 
     if ((buf = qemu_opt_get(opts, "format")) != NULL) {
         if (is_help_option(buf)) {
@@ -426,46 +538,11 @@
             goto early_err;
         }
 
-        drv = bdrv_find_format(buf);
-        if (!drv) {
-            error_setg(errp, "'%s' invalid format", buf);
+        if (qdict_haskey(bs_opts, "driver")) {
+            error_setg(errp, "Cannot specify both 'driver' and 'format'");
             goto early_err;
         }
-    }
-
-    /* disk I/O throttling */
-    memset(&cfg, 0, sizeof(cfg));
-    cfg.buckets[THROTTLE_BPS_TOTAL].avg =
-        qemu_opt_get_number(opts, "throttling.bps-total", 0);
-    cfg.buckets[THROTTLE_BPS_READ].avg  =
-        qemu_opt_get_number(opts, "throttling.bps-read", 0);
-    cfg.buckets[THROTTLE_BPS_WRITE].avg =
-        qemu_opt_get_number(opts, "throttling.bps-write", 0);
-    cfg.buckets[THROTTLE_OPS_TOTAL].avg =
-        qemu_opt_get_number(opts, "throttling.iops-total", 0);
-    cfg.buckets[THROTTLE_OPS_READ].avg =
-        qemu_opt_get_number(opts, "throttling.iops-read", 0);
-    cfg.buckets[THROTTLE_OPS_WRITE].avg =
-        qemu_opt_get_number(opts, "throttling.iops-write", 0);
-
-    cfg.buckets[THROTTLE_BPS_TOTAL].max =
-        qemu_opt_get_number(opts, "throttling.bps-total-max", 0);
-    cfg.buckets[THROTTLE_BPS_READ].max  =
-        qemu_opt_get_number(opts, "throttling.bps-read-max", 0);
-    cfg.buckets[THROTTLE_BPS_WRITE].max =
-        qemu_opt_get_number(opts, "throttling.bps-write-max", 0);
-    cfg.buckets[THROTTLE_OPS_TOTAL].max =
-        qemu_opt_get_number(opts, "throttling.iops-total-max", 0);
-    cfg.buckets[THROTTLE_OPS_READ].max =
-        qemu_opt_get_number(opts, "throttling.iops-read-max", 0);
-    cfg.buckets[THROTTLE_OPS_WRITE].max =
-        qemu_opt_get_number(opts, "throttling.iops-write-max", 0);
-
-    cfg.op_size = qemu_opt_get_number(opts, "throttling.iops-size", 0);
-
-    if (!check_throttle_config(&cfg, &error)) {
-        error_propagate(errp, error);
-        goto early_err;
+        qdict_put(bs_opts, "driver", qstring_from_str(buf));
     }
 
     on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
@@ -486,96 +563,167 @@
         }
     }
 
-    detect_zeroes =
-        qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
-                        qemu_opt_get(opts, "detect-zeroes"),
-                        BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
-                        BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
-                        &error);
-    if (error) {
-        error_propagate(errp, error);
-        goto early_err;
-    }
-
-    if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
-        !(bdrv_flags & BDRV_O_UNMAP)) {
-        error_setg(errp, "setting detect-zeroes to unmap is not allowed "
-                         "without setting discard operation to unmap");
-        goto early_err;
+    if (snapshot) {
+        bdrv_flags |= BDRV_O_SNAPSHOT;
     }
 
     /* init */
-    blk = blk_new_with_bs(qemu_opts_id(opts), errp);
-    if (!blk) {
-        goto early_err;
-    }
-    bs = blk_bs(blk);
-    bs->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
-    bs->read_only = ro;
-    bs->detect_zeroes = detect_zeroes;
+    if ((!file || !*file) && !qdict_size(bs_opts)) {
+        BlockBackendRootState *blk_rs;
 
-    bdrv_set_on_error(bs, on_read_error, on_write_error);
+        blk = blk_new();
+        blk_rs = blk_get_root_state(blk);
+        blk_rs->open_flags    = bdrv_flags;
+        blk_rs->read_only     = !(bdrv_flags & BDRV_O_RDWR);
+        blk_rs->detect_zeroes = detect_zeroes;
+
+        QDECREF(bs_opts);
+    } else {
+        if (file && !*file) {
+            file = NULL;
+        }
+
+        /* bdrv_open() defaults to the values in bdrv_flags (for compatibility
+         * with other callers) rather than what we want as the real defaults.
+         * Apply the defaults here instead. */
+        qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_DIRECT, "off");
+        qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_NO_FLUSH, "off");
+        assert((bdrv_flags & BDRV_O_CACHE_MASK) == 0);
+
+        if (runstate_check(RUN_STATE_INMIGRATE)) {
+            bdrv_flags |= BDRV_O_INACTIVE;
+        }
+
+        blk = blk_new_open(file, NULL, bs_opts, bdrv_flags, errp);
+        if (!blk) {
+            goto err_no_bs_opts;
+        }
+        bs = blk_bs(blk);
+
+        bs->detect_zeroes = detect_zeroes;
+
+        if (bdrv_key_required(bs)) {
+            autostart = 0;
+        }
+
+        block_acct_init(blk_get_stats(blk), account_invalid, account_failed);
+
+        if (!parse_stats_intervals(blk_get_stats(blk), interval_list, errp)) {
+            blk_unref(blk);
+            blk = NULL;
+            goto err_no_bs_opts;
+        }
+    }
 
     /* disk I/O throttling */
     if (throttle_enabled(&cfg)) {
-        bdrv_io_limits_enable(bs);
-        bdrv_set_io_limits(bs, &cfg);
-    }
-
-    if (!file || !*file) {
-        if (has_driver_specific_opts) {
-            file = NULL;
-        } else {
-            QDECREF(bs_opts);
-            qemu_opts_del(opts);
-            return blk;
+        if (!throttling_group) {
+            throttling_group = id;
         }
-    }
-    if (snapshot) {
-        /* always use cache=unsafe with snapshot */
-        bdrv_flags &= ~BDRV_O_CACHE_MASK;
-        bdrv_flags |= (BDRV_O_SNAPSHOT|BDRV_O_CACHE_WB|BDRV_O_NO_FLUSH);
+        blk_io_limits_enable(blk, throttling_group);
+        blk_set_io_limits(blk, &cfg);
     }
 
-    if (copy_on_read) {
-        bdrv_flags |= BDRV_O_COPY_ON_READ;
+    blk_set_enable_write_cache(blk, !writethrough);
+    blk_set_on_error(blk, on_read_error, on_write_error);
+
+    if (!monitor_add_blk(blk, id, errp)) {
+        blk_unref(blk);
+        blk = NULL;
+        goto err_no_bs_opts;
     }
 
-    if (runstate_check(RUN_STATE_INMIGRATE)) {
-        bdrv_flags |= BDRV_O_INCOMING;
-    }
-
-    bdrv_flags |= ro ? 0 : BDRV_O_RDWR;
-
-    QINCREF(bs_opts);
-    ret = bdrv_open(&bs, file, NULL, bs_opts, bdrv_flags, drv, &error);
-    assert(bs == blk_bs(blk));
-
-    if (ret < 0) {
-        error_setg(errp, "could not open disk image %s: %s",
-                   file ?: blk_name(blk), error_get_pretty(error));
-        error_free(error);
-        goto err;
-    }
-
-    if (bdrv_key_required(bs)) {
-        autostart = 0;
-    }
-
-    QDECREF(bs_opts);
+err_no_bs_opts:
     qemu_opts_del(opts);
-
+    QDECREF(interval_dict);
+    QDECREF(interval_list);
     return blk;
 
-err:
-    blk_unref(blk);
 early_err:
     qemu_opts_del(opts);
+    QDECREF(interval_dict);
+    QDECREF(interval_list);
 err_no_opts:
     QDECREF(bs_opts);
     return NULL;
 }
 
+static QemuOptsList qemu_root_bds_opts;
+
+/* Takes the ownership of bs_opts */
+static BlockDriverState *bds_tree_init(QDict *bs_opts, Error **errp)
+{
+    BlockDriverState *bs;
+    QemuOpts *opts;
+    Error *local_error = NULL;
+    BlockdevDetectZeroesOptions detect_zeroes;
+    int bdrv_flags = 0;
+
+    opts = qemu_opts_create(&qemu_root_bds_opts, NULL, 1, errp);
+    if (!opts) {
+        goto fail;
+    }
+
+    qemu_opts_absorb_qdict(opts, bs_opts, &local_error);
+    if (local_error) {
+        error_propagate(errp, local_error);
+        goto fail;
+    }
+
+    extract_common_blockdev_options(opts, &bdrv_flags, NULL, NULL,
+                                    &detect_zeroes, &local_error);
+    if (local_error) {
+        error_propagate(errp, local_error);
+        goto fail;
+    }
+
+    /* bdrv_open() defaults to the values in bdrv_flags (for compatibility
+     * with other callers) rather than what we want as the real defaults.
+     * Apply the defaults here instead. */
+    qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_DIRECT, "off");
+    qdict_set_default_str(bs_opts, BDRV_OPT_CACHE_NO_FLUSH, "off");
+
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        bdrv_flags |= BDRV_O_INACTIVE;
+    }
+
+    bs = bdrv_open(NULL, NULL, bs_opts, bdrv_flags, errp);
+    if (!bs) {
+        goto fail_no_bs_opts;
+    }
+
+    bs->detect_zeroes = detect_zeroes;
+
+fail_no_bs_opts:
+    qemu_opts_del(opts);
+    return bs;
+
+fail:
+    qemu_opts_del(opts);
+    QDECREF(bs_opts);
+    return NULL;
+}
+
+void blockdev_close_all_bdrv_states(void)
+{
+    BlockDriverState *bs, *next_bs;
+
+    QTAILQ_FOREACH_SAFE(bs, &monitor_bdrv_states, monitor_list, next_bs) {
+        AioContext *ctx = bdrv_get_aio_context(bs);
+
+        aio_context_acquire(ctx);
+        bdrv_unref(bs);
+        aio_context_release(ctx);
+    }
+}
+
+/* Iterates over the list of monitor-owned BlockDriverStates */
+BlockDriverState *bdrv_next_monitor_owned(BlockDriverState *bs)
+{
+    return bs ? QTAILQ_NEXT(bs, monitor_list)
+              : QTAILQ_FIRST(&monitor_bdrv_states);
+}
+
 static void qemu_opt_rename(QemuOpts *opts, const char *from, const char *to,
                             Error **errp)
 {
@@ -592,7 +740,7 @@
 
     /* rename all items in opts */
     while ((value = qemu_opt_get(opts, from))) {
-        qemu_opt_set(opts, to, value);
+        qemu_opt_set(opts, to, value, &error_abort);
         qemu_opt_unset(opts, from);
     }
 }
@@ -721,6 +869,8 @@
 
         { "iops_size",      "throttling.iops-size" },
 
+        { "group",          "throttling.group" },
+
         { "readonly",       "read-only" },
     };
 
@@ -728,8 +878,7 @@
         qemu_opt_rename(all_opts, opt_renames[i].from, opt_renames[i].to,
                         &local_err);
         if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
+            error_report_err(local_err);
             return NULL;
         }
     }
@@ -737,24 +886,25 @@
     value = qemu_opt_get(all_opts, "cache");
     if (value) {
         int flags = 0;
+        bool writethrough;
 
-        if (bdrv_parse_cache_flags(value, &flags) != 0) {
+        if (bdrv_parse_cache_mode(value, &flags, &writethrough) != 0) {
             error_report("invalid cache option");
             return NULL;
         }
 
         /* Specific options take precedence */
-        if (!qemu_opt_get(all_opts, "cache.writeback")) {
-            qemu_opt_set_bool(all_opts, "cache.writeback",
-                              !!(flags & BDRV_O_CACHE_WB));
+        if (!qemu_opt_get(all_opts, BDRV_OPT_CACHE_WB)) {
+            qemu_opt_set_bool(all_opts, BDRV_OPT_CACHE_WB,
+                              !writethrough, &error_abort);
         }
-        if (!qemu_opt_get(all_opts, "cache.direct")) {
-            qemu_opt_set_bool(all_opts, "cache.direct",
-                              !!(flags & BDRV_O_NOCACHE));
+        if (!qemu_opt_get(all_opts, BDRV_OPT_CACHE_DIRECT)) {
+            qemu_opt_set_bool(all_opts, BDRV_OPT_CACHE_DIRECT,
+                              !!(flags & BDRV_O_NOCACHE), &error_abort);
         }
-        if (!qemu_opt_get(all_opts, "cache.no-flush")) {
-            qemu_opt_set_bool(all_opts, "cache.no-flush",
-                              !!(flags & BDRV_O_NO_FLUSH));
+        if (!qemu_opt_get(all_opts, BDRV_OPT_CACHE_NO_FLUSH)) {
+            qemu_opt_set_bool(all_opts, BDRV_OPT_CACHE_NO_FLUSH,
+                              !!(flags & BDRV_O_NO_FLUSH), &error_abort);
         }
         qemu_opt_unset(all_opts, "cache");
     }
@@ -767,8 +917,7 @@
                                    &error_abort);
     qemu_opts_absorb_qdict(legacy_opts, bs_opts, &local_err);
     if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         goto fail;
     }
 
@@ -945,13 +1094,14 @@
         devopts = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
                                    &error_abort);
         if (arch_type == QEMU_ARCH_S390X) {
-            qemu_opt_set(devopts, "driver", "virtio-blk-s390");
+            qemu_opt_set(devopts, "driver", "virtio-blk-ccw", &error_abort);
         } else {
-            qemu_opt_set(devopts, "driver", "virtio-blk-pci");
+            qemu_opt_set(devopts, "driver", "virtio-blk-pci", &error_abort);
         }
-        qemu_opt_set(devopts, "drive", qdict_get_str(bs_opts, "id"));
+        qemu_opt_set(devopts, "drive", qdict_get_str(bs_opts, "id"),
+                     &error_abort);
         if (devaddr) {
-            qemu_opt_set(devopts, "addr", devaddr);
+            qemu_opt_set(devopts, "addr", devaddr, &error_abort);
         }
     }
 
@@ -983,8 +1133,7 @@
     bs_opts = NULL;
     if (!blk) {
         if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
+            error_report_err(local_err);
         }
         goto fail;
     } else {
@@ -1025,21 +1174,35 @@
     return dinfo;
 }
 
-void do_commit(Monitor *mon, const QDict *qdict)
+void hmp_commit(Monitor *mon, const QDict *qdict)
 {
     const char *device = qdict_get_str(qdict, "device");
-    BlockDriverState *bs;
+    BlockBackend *blk;
     int ret;
 
     if (!strcmp(device, "all")) {
-        ret = bdrv_commit_all();
+        ret = blk_commit_all();
     } else {
-        bs = bdrv_find(device);
-        if (!bs) {
+        BlockDriverState *bs;
+        AioContext *aio_context;
+
+        blk = blk_by_name(device);
+        if (!blk) {
             monitor_printf(mon, "Device '%s' not found\n", device);
             return;
         }
+        if (!blk_is_available(blk)) {
+            monitor_printf(mon, "Device '%s' has no medium\n", device);
+            return;
+        }
+
+        bs = blk_bs(blk);
+        aio_context = bdrv_get_aio_context(bs);
+        aio_context_acquire(aio_context);
+
         ret = bdrv_commit(bs);
+
+        aio_context_release(aio_context);
     }
     if (ret < 0) {
         monitor_printf(mon, "'commit' error for '%s': %s\n", device,
@@ -1047,16 +1210,13 @@
     }
 }
 
-static void blockdev_do_action(int kind, void *data, Error **errp)
+static void blockdev_do_action(TransactionAction *action, Error **errp)
 {
-    TransactionAction action;
     TransactionActionList list;
 
-    action.kind = kind;
-    action.data = data;
-    list.value = &action;
+    list.value = action;
     list.next = NULL;
-    qmp_transaction(&list, errp);
+    qmp_transaction(&list, false, NULL, errp);
 }
 
 void qmp_blockdev_snapshot_sync(bool has_device, const char *device,
@@ -1067,7 +1227,7 @@
                                 bool has_format, const char *format,
                                 bool has_mode, NewImageMode mode, Error **errp)
 {
-    BlockdevSnapshot snapshot = {
+    BlockdevSnapshotSync snapshot = {
         .has_device = has_device,
         .device = (char *) device,
         .has_node_name = has_node_name,
@@ -1080,8 +1240,25 @@
         .has_mode = has_mode,
         .mode = mode,
     };
-    blockdev_do_action(TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC,
-                       &snapshot, errp);
+    TransactionAction action = {
+        .type = TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC,
+        .u.blockdev_snapshot_sync.data = &snapshot,
+    };
+    blockdev_do_action(&action, errp);
+}
+
+void qmp_blockdev_snapshot(const char *node, const char *overlay,
+                           Error **errp)
+{
+    BlockdevSnapshot snapshot_data = {
+        .node = (char *) node,
+        .overlay = (char *) overlay
+    };
+    TransactionAction action = {
+        .type = TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT,
+        .u.blockdev_snapshot.data = &snapshot_data,
+    };
+    blockdev_do_action(&action, errp);
 }
 
 void qmp_blockdev_snapshot_internal_sync(const char *device,
@@ -1092,9 +1269,11 @@
         .device = (char *) device,
         .name = (char *) name
     };
-
-    blockdev_do_action(TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC,
-                       &snapshot, errp);
+    TransactionAction action = {
+        .type = TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC,
+        .u.blockdev_snapshot_internal_sync.data = &snapshot,
+    };
+    blockdev_do_action(&action, errp);
 }
 
 SnapshotInfo *qmp_blockdev_snapshot_delete_internal_sync(const char *device,
@@ -1104,17 +1283,24 @@
                                                          const char *name,
                                                          Error **errp)
 {
-    BlockDriverState *bs = bdrv_find(device);
+    BlockDriverState *bs;
+    BlockBackend *blk;
+    AioContext *aio_context;
     QEMUSnapshotInfo sn;
     Error *local_err = NULL;
     SnapshotInfo *info = NULL;
     int ret;
 
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return NULL;
     }
 
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
+
     if (!has_id) {
         id = NULL;
     }
@@ -1125,28 +1311,40 @@
 
     if (!id && !name) {
         error_setg(errp, "Name or id must be provided");
-        return NULL;
+        goto out_aio_context;
+    }
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out_aio_context;
+    }
+    bs = blk_bs(blk);
+
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE, errp)) {
+        goto out_aio_context;
     }
 
     ret = bdrv_snapshot_find_by_id_and_name(bs, id, name, &sn, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return NULL;
+        goto out_aio_context;
     }
     if (!ret) {
         error_setg(errp,
                    "Snapshot with id '%s' and name '%s' does not exist on "
                    "device '%s'",
                    STR_OR_NULL(id), STR_OR_NULL(name), device);
-        return NULL;
+        goto out_aio_context;
     }
 
     bdrv_snapshot_delete(bs, id, name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return NULL;
+        goto out_aio_context;
     }
 
+    aio_context_release(aio_context);
+
     info = g_new0(SnapshotInfo, 1);
     info->id = g_strdup(sn.id_str);
     info->name = g_strdup(sn.name);
@@ -1157,51 +1355,151 @@
     info->vm_clock_sec = sn.vm_clock_nsec / 1000000000;
 
     return info;
+
+out_aio_context:
+    aio_context_release(aio_context);
+    return NULL;
 }
 
-/* New and old BlockDriverState structs for group snapshots */
-
-typedef struct BlkTransactionState BlkTransactionState;
-
-/* Only prepare() may fail. In a single transaction, only one of commit() or
-   abort() will be called, clean() will always be called if it present. */
-typedef struct BdrvActionOps {
-    /* Size of state struct, in bytes. */
-    size_t instance_size;
-    /* Prepare the work, must NOT be NULL. */
-    void (*prepare)(BlkTransactionState *common, Error **errp);
-    /* Commit the changes, can be NULL. */
-    void (*commit)(BlkTransactionState *common);
-    /* Abort the changes on fail, can be NULL. */
-    void (*abort)(BlkTransactionState *common);
-    /* Clean up resource in the end, can be NULL. */
-    void (*clean)(BlkTransactionState *common);
-} BdrvActionOps;
-
-/*
- * This structure must be arranged as first member in child type, assuming
- * that compiler will also arrange it to the same address with parent instance.
- * Later it will be used in free().
+/**
+ * block_dirty_bitmap_lookup:
+ * Return a dirty bitmap (if present), after validating
+ * the node reference and bitmap names.
+ *
+ * @node: The name of the BDS node to search for bitmaps
+ * @name: The name of the bitmap to search for
+ * @pbs: Output pointer for BDS lookup, if desired. Can be NULL.
+ * @paio: Output pointer for aio_context acquisition, if desired. Can be NULL.
+ * @errp: Output pointer for error information. Can be NULL.
+ *
+ * @return: A bitmap object on success, or NULL on failure.
  */
-struct BlkTransactionState {
+static BdrvDirtyBitmap *block_dirty_bitmap_lookup(const char *node,
+                                                  const char *name,
+                                                  BlockDriverState **pbs,
+                                                  AioContext **paio,
+                                                  Error **errp)
+{
+    BlockDriverState *bs;
+    BdrvDirtyBitmap *bitmap;
+    AioContext *aio_context;
+
+    if (!node) {
+        error_setg(errp, "Node cannot be NULL");
+        return NULL;
+    }
+    if (!name) {
+        error_setg(errp, "Bitmap name cannot be NULL");
+        return NULL;
+    }
+    bs = bdrv_lookup_bs(node, node, NULL);
+    if (!bs) {
+        error_setg(errp, "Node '%s' not found", node);
+        return NULL;
+    }
+
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    bitmap = bdrv_find_dirty_bitmap(bs, name);
+    if (!bitmap) {
+        error_setg(errp, "Dirty bitmap '%s' not found", name);
+        goto fail;
+    }
+
+    if (pbs) {
+        *pbs = bs;
+    }
+    if (paio) {
+        *paio = aio_context;
+    } else {
+        aio_context_release(aio_context);
+    }
+
+    return bitmap;
+
+ fail:
+    aio_context_release(aio_context);
+    return NULL;
+}
+
+/* New and old BlockDriverState structs for atomic group operations */
+
+typedef struct BlkActionState BlkActionState;
+
+/**
+ * BlkActionOps:
+ * Table of operations that define an Action.
+ *
+ * @instance_size: Size of state struct, in bytes.
+ * @prepare: Prepare the work, must NOT be NULL.
+ * @commit: Commit the changes, can be NULL.
+ * @abort: Abort the changes on fail, can be NULL.
+ * @clean: Clean up resources after all transaction actions have called
+ *         commit() or abort(). Can be NULL.
+ *
+ * Only prepare() may fail. In a single transaction, only one of commit() or
+ * abort() will be called. clean() will always be called if it is present.
+ */
+typedef struct BlkActionOps {
+    size_t instance_size;
+    void (*prepare)(BlkActionState *common, Error **errp);
+    void (*commit)(BlkActionState *common);
+    void (*abort)(BlkActionState *common);
+    void (*clean)(BlkActionState *common);
+} BlkActionOps;
+
+/**
+ * BlkActionState:
+ * Describes one Action's state within a Transaction.
+ *
+ * @action: QAPI-defined enum identifying which Action to perform.
+ * @ops: Table of ActionOps this Action can perform.
+ * @block_job_txn: Transaction which this action belongs to.
+ * @entry: List membership for all Actions in this Transaction.
+ *
+ * This structure must be arranged as first member in a subclassed type,
+ * assuming that the compiler will also arrange it to the same offsets as the
+ * base class.
+ */
+struct BlkActionState {
     TransactionAction *action;
-    const BdrvActionOps *ops;
-    QSIMPLEQ_ENTRY(BlkTransactionState) entry;
+    const BlkActionOps *ops;
+    BlockJobTxn *block_job_txn;
+    TransactionProperties *txn_props;
+    QSIMPLEQ_ENTRY(BlkActionState) entry;
 };
 
 /* internal snapshot private data */
 typedef struct InternalSnapshotState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
+    AioContext *aio_context;
     QEMUSnapshotInfo sn;
+    bool created;
 } InternalSnapshotState;
 
-static void internal_snapshot_prepare(BlkTransactionState *common,
+
+static int action_check_completion_mode(BlkActionState *s, Error **errp)
+{
+    if (s->txn_props->completion_mode != ACTION_COMPLETION_MODE_INDIVIDUAL) {
+        error_setg(errp,
+                   "Action '%s' does not support Transaction property "
+                   "completion-mode = %s",
+                   TransactionActionKind_lookup[s->action->type],
+                   ActionCompletionMode_lookup[s->txn_props->completion_mode]);
+        return -1;
+    }
+    return 0;
+}
+
+static void internal_snapshot_prepare(BlkActionState *common,
                                       Error **errp)
 {
     Error *local_err = NULL;
     const char *device;
     const char *name;
+    BlockBackend *blk;
     BlockDriverState *bs;
     QEMUSnapshotInfo old_sn, *sn;
     bool ret;
@@ -1210,9 +1508,9 @@
     InternalSnapshotState *state;
     int ret1;
 
-    g_assert(common->action->kind ==
+    g_assert(common->action->type ==
              TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_INTERNAL_SYNC);
-    internal = common->action->blockdev_snapshot_internal_sync;
+    internal = common->action->u.blockdev_snapshot_internal_sync.data;
     state = DO_UPCAST(InternalSnapshotState, common, common);
 
     /* 1. parse input */
@@ -1220,25 +1518,43 @@
     name = internal->name;
 
     /* 2. check for validation */
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    if (action_check_completion_mode(common, errp) < 0) {
         return;
     }
 
-    if (!bdrv_is_inserted(bs)) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
+    }
+
+    /* AioContext is released in .clean() */
+    state->aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(state->aio_context);
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+        return;
+    }
+    bs = blk_bs(blk);
+
+    state->bs = bs;
+    bdrv_drained_begin(bs);
+
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_INTERNAL_SNAPSHOT, errp)) {
         return;
     }
 
     if (bdrv_is_read_only(bs)) {
-        error_set(errp, QERR_DEVICE_IS_READ_ONLY, device);
+        error_setg(errp, "Device '%s' is read only", device);
         return;
     }
 
     if (!bdrv_can_snapshot(bs)) {
-        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
-                  bs->drv->format_name, device, "internal snapshot");
+        error_setg(errp, "Block format '%s' used by device '%s' "
+                   "does not support internal snapshots",
+                   bs->drv->format_name, device);
         return;
     }
 
@@ -1277,10 +1593,10 @@
     }
 
     /* 4. succeed, mark a snapshot is created */
-    state->bs = bs;
+    state->created = true;
 }
 
-static void internal_snapshot_abort(BlkTransactionState *common)
+static void internal_snapshot_abort(BlkActionState *common)
 {
     InternalSnapshotState *state =
                              DO_UPCAST(InternalSnapshotState, common, common);
@@ -1288,94 +1604,99 @@
     QEMUSnapshotInfo *sn = &state->sn;
     Error *local_error = NULL;
 
-    if (!bs) {
+    if (!state->created) {
         return;
     }
 
     if (bdrv_snapshot_delete(bs, sn->id_str, sn->name, &local_error) < 0) {
-        error_report("Failed to delete snapshot with id '%s' and name '%s' on "
-                     "device '%s' in abort: %s",
-                     sn->id_str,
-                     sn->name,
-                     bdrv_get_device_name(bs),
-                     error_get_pretty(local_error));
-        error_free(local_error);
+        error_reportf_err(local_error,
+                          "Failed to delete snapshot with id '%s' and "
+                          "name '%s' on device '%s' in abort: ",
+                          sn->id_str, sn->name,
+                          bdrv_get_device_name(bs));
+    }
+}
+
+static void internal_snapshot_clean(BlkActionState *common)
+{
+    InternalSnapshotState *state = DO_UPCAST(InternalSnapshotState,
+                                             common, common);
+
+    if (state->aio_context) {
+        if (state->bs) {
+            bdrv_drained_end(state->bs);
+        }
+        aio_context_release(state->aio_context);
     }
 }
 
 /* external snapshot private data */
 typedef struct ExternalSnapshotState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *old_bs;
     BlockDriverState *new_bs;
+    AioContext *aio_context;
 } ExternalSnapshotState;
 
-static void external_snapshot_prepare(BlkTransactionState *common,
+static void external_snapshot_prepare(BlkActionState *common,
                                       Error **errp)
 {
-    BlockDriver *drv;
-    int flags, ret;
+    int flags = 0;
     QDict *options = NULL;
     Error *local_err = NULL;
-    bool has_device = false;
+    /* Device and node name of the image to generate the snapshot from */
     const char *device;
-    bool has_node_name = false;
     const char *node_name;
-    bool has_snapshot_node_name = false;
-    const char *snapshot_node_name;
+    /* Reference to the new image (for 'blockdev-snapshot') */
+    const char *snapshot_ref;
+    /* File name of the new image (for 'blockdev-snapshot-sync') */
     const char *new_image_file;
-    const char *format = "qcow2";
-    enum NewImageMode mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
     TransactionAction *action = common->action;
 
-    /* get parameters */
-    g_assert(action->kind == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC);
-
-    has_device = action->blockdev_snapshot_sync->has_device;
-    device = action->blockdev_snapshot_sync->device;
-    has_node_name = action->blockdev_snapshot_sync->has_node_name;
-    node_name = action->blockdev_snapshot_sync->node_name;
-    has_snapshot_node_name =
-        action->blockdev_snapshot_sync->has_snapshot_node_name;
-    snapshot_node_name = action->blockdev_snapshot_sync->snapshot_node_name;
-
-    new_image_file = action->blockdev_snapshot_sync->snapshot_file;
-    if (action->blockdev_snapshot_sync->has_format) {
-        format = action->blockdev_snapshot_sync->format;
-    }
-    if (action->blockdev_snapshot_sync->has_mode) {
-        mode = action->blockdev_snapshot_sync->mode;
+    /* 'blockdev-snapshot' and 'blockdev-snapshot-sync' have similar
+     * purpose but a different set of parameters */
+    switch (action->type) {
+    case TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT:
+        {
+            BlockdevSnapshot *s = action->u.blockdev_snapshot.data;
+            device = s->node;
+            node_name = s->node;
+            new_image_file = NULL;
+            snapshot_ref = s->overlay;
+        }
+        break;
+    case TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC:
+        {
+            BlockdevSnapshotSync *s = action->u.blockdev_snapshot_sync.data;
+            device = s->has_device ? s->device : NULL;
+            node_name = s->has_node_name ? s->node_name : NULL;
+            new_image_file = s->snapshot_file;
+            snapshot_ref = NULL;
+        }
+        break;
+    default:
+        g_assert_not_reached();
     }
 
     /* start processing */
-    drv = bdrv_find_format(format);
-    if (!drv) {
-        error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
+    if (action_check_completion_mode(common, errp) < 0) {
         return;
     }
 
-    state->old_bs = bdrv_lookup_bs(has_device ? device : NULL,
-                                   has_node_name ? node_name : NULL,
-                                   &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
+    state->old_bs = bdrv_lookup_bs(device, node_name, errp);
+    if (!state->old_bs) {
         return;
     }
 
-    if (has_node_name && !has_snapshot_node_name) {
-        error_setg(errp, "New snapshot node name missing");
-        return;
-    }
-
-    if (has_snapshot_node_name && bdrv_find_node(snapshot_node_name)) {
-        error_setg(errp, "New snapshot node name already existing");
-        return;
-    }
+    /* Acquire AioContext now so any threads operating on old_bs stop */
+    state->aio_context = bdrv_get_aio_context(state->old_bs);
+    aio_context_acquire(state->aio_context);
+    bdrv_drained_begin(state->old_bs);
 
     if (!bdrv_is_inserted(state->old_bs)) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         return;
     }
 
@@ -1386,62 +1707,112 @@
 
     if (!bdrv_is_read_only(state->old_bs)) {
         if (bdrv_flush(state->old_bs)) {
-            error_set(errp, QERR_IO_ERROR);
+            error_setg(errp, QERR_IO_ERROR);
             return;
         }
     }
 
     if (!bdrv_is_first_non_filter(state->old_bs)) {
-        error_set(errp, QERR_FEATURE_DISABLED, "snapshot");
+        error_setg(errp, QERR_FEATURE_DISABLED, "snapshot");
         return;
     }
 
-    flags = state->old_bs->open_flags;
+    if (action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC) {
+        BlockdevSnapshotSync *s = action->u.blockdev_snapshot_sync.data;
+        const char *format = s->has_format ? s->format : "qcow2";
+        enum NewImageMode mode;
+        const char *snapshot_node_name =
+            s->has_snapshot_node_name ? s->snapshot_node_name : NULL;
 
-    /* create new image w/backing file */
-    if (mode != NEW_IMAGE_MODE_EXISTING) {
-        bdrv_img_create(new_image_file, format,
-                        state->old_bs->filename,
-                        state->old_bs->drv->format_name,
-                        NULL, -1, flags, &local_err, false);
-        if (local_err) {
-            error_propagate(errp, local_err);
+        if (node_name && !snapshot_node_name) {
+            error_setg(errp, "New snapshot node name missing");
             return;
         }
-    }
 
-    if (has_snapshot_node_name) {
+        if (snapshot_node_name &&
+            bdrv_lookup_bs(snapshot_node_name, snapshot_node_name, NULL)) {
+            error_setg(errp, "New snapshot node name already in use");
+            return;
+        }
+
+        flags = state->old_bs->open_flags;
+        flags &= ~(BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING | BDRV_O_COPY_ON_READ);
+
+        /* create new image w/backing file */
+        mode = s->has_mode ? s->mode : NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+        if (mode != NEW_IMAGE_MODE_EXISTING) {
+            int64_t size = bdrv_getlength(state->old_bs);
+            if (size < 0) {
+                error_setg_errno(errp, -size, "bdrv_getlength failed");
+                return;
+            }
+            bdrv_img_create(new_image_file, format,
+                            state->old_bs->filename,
+                            state->old_bs->drv->format_name,
+                            NULL, size, flags, &local_err, false);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+        }
+
         options = qdict_new();
-        qdict_put(options, "node-name",
-                  qstring_from_str(snapshot_node_name));
+        if (s->has_snapshot_node_name) {
+            qdict_put(options, "node-name",
+                      qstring_from_str(snapshot_node_name));
+        }
+        qdict_put(options, "driver", qstring_from_str(format));
+
+        flags |= BDRV_O_NO_BACKING;
     }
 
-    /* TODO Inherit bs->options or only take explicit options with an
-     * extended QMP command? */
-    assert(state->new_bs == NULL);
-    ret = bdrv_open(&state->new_bs, new_image_file, NULL, options,
-                    flags | BDRV_O_NO_BACKING, drv, &local_err);
+    state->new_bs = bdrv_open(new_image_file, snapshot_ref, options, flags,
+                              errp);
     /* We will manually add the backing_hd field to the bs later */
-    if (ret != 0) {
-        error_propagate(errp, local_err);
+    if (!state->new_bs) {
+        return;
+    }
+
+    if (bdrv_has_blk(state->new_bs)) {
+        error_setg(errp, "The snapshot is already in use by %s",
+                   bdrv_get_parent_name(state->new_bs));
+        return;
+    }
+
+    if (bdrv_op_is_blocked(state->new_bs, BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
+                           errp)) {
+        return;
+    }
+
+    if (state->new_bs->backing != NULL) {
+        error_setg(errp, "The snapshot already has a backing image");
+        return;
+    }
+
+    if (!state->new_bs->drv->supports_backing) {
+        error_setg(errp, "The snapshot does not support backing images");
     }
 }
 
-static void external_snapshot_commit(BlkTransactionState *common)
+static void external_snapshot_commit(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
 
+    bdrv_set_aio_context(state->new_bs, state->aio_context);
+
     /* This removes our old bs and adds the new bs */
     bdrv_append(state->new_bs, state->old_bs);
     /* We don't need (or want) to use the transactional
      * bdrv_reopen_multiple() across all the entries at once, because we
      * don't want to abort all of them if one of them fails the reopen */
-    bdrv_reopen(state->new_bs, state->new_bs->open_flags & ~BDRV_O_RDWR,
-                NULL);
+    if (!state->old_bs->copy_on_read) {
+        bdrv_reopen(state->old_bs, state->old_bs->open_flags & ~BDRV_O_RDWR,
+                    NULL);
+    }
 }
 
-static void external_snapshot_abort(BlkTransactionState *common)
+static void external_snapshot_abort(BlkActionState *common)
 {
     ExternalSnapshotState *state =
                              DO_UPCAST(ExternalSnapshotState, common, common);
@@ -1450,41 +1821,82 @@
     }
 }
 
+static void external_snapshot_clean(BlkActionState *common)
+{
+    ExternalSnapshotState *state =
+                             DO_UPCAST(ExternalSnapshotState, common, common);
+    if (state->aio_context) {
+        bdrv_drained_end(state->old_bs);
+        aio_context_release(state->aio_context);
+    }
+}
+
 typedef struct DriveBackupState {
-    BlkTransactionState common;
+    BlkActionState common;
     BlockDriverState *bs;
+    AioContext *aio_context;
     BlockJob *job;
 } DriveBackupState;
 
-static void drive_backup_prepare(BlkTransactionState *common, Error **errp)
+static void do_drive_backup(const char *job_id, const char *device,
+                            const char *target, bool has_format,
+                            const char *format, enum MirrorSyncMode sync,
+                            bool has_mode, enum NewImageMode mode,
+                            bool has_speed, int64_t speed,
+                            bool has_bitmap, const char *bitmap,
+                            bool has_on_source_error,
+                            BlockdevOnError on_source_error,
+                            bool has_on_target_error,
+                            BlockdevOnError on_target_error,
+                            BlockJobTxn *txn, Error **errp);
+
+static void drive_backup_prepare(BlkActionState *common, Error **errp)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+    BlockBackend *blk;
     DriveBackup *backup;
     Error *local_err = NULL;
 
-    assert(common->action->kind == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
-    backup = common->action->drive_backup;
+    assert(common->action->type == TRANSACTION_ACTION_KIND_DRIVE_BACKUP);
+    backup = common->action->u.drive_backup.data;
 
-    qmp_drive_backup(backup->device, backup->target,
-                     backup->has_format, backup->format,
-                     backup->sync,
-                     backup->has_mode, backup->mode,
-                     backup->has_speed, backup->speed,
-                     backup->has_on_source_error, backup->on_source_error,
-                     backup->has_on_target_error, backup->on_target_error,
-                     &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        state->bs = NULL;
-        state->job = NULL;
+    blk = blk_by_name(backup->device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", backup->device);
         return;
     }
 
-    state->bs = bdrv_find(backup->device);
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, backup->device);
+        return;
+    }
+
+    /* AioContext is released in .clean() */
+    state->aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(state->aio_context);
+    bdrv_drained_begin(blk_bs(blk));
+    state->bs = blk_bs(blk);
+
+    do_drive_backup(backup->has_job_id ? backup->job_id : NULL,
+                    backup->device, backup->target,
+                    backup->has_format, backup->format,
+                    backup->sync,
+                    backup->has_mode, backup->mode,
+                    backup->has_speed, backup->speed,
+                    backup->has_bitmap, backup->bitmap,
+                    backup->has_on_source_error, backup->on_source_error,
+                    backup->has_on_target_error, backup->on_target_error,
+                    common->block_job_txn, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     state->job = state->bs->job;
 }
 
-static void drive_backup_abort(BlkTransactionState *common)
+static void drive_backup_abort(BlkActionState *common)
 {
     DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
     BlockDriverState *bs = state->bs;
@@ -1495,30 +1907,252 @@
     }
 }
 
-static void abort_prepare(BlkTransactionState *common, Error **errp)
+static void drive_backup_clean(BlkActionState *common)
+{
+    DriveBackupState *state = DO_UPCAST(DriveBackupState, common, common);
+
+    if (state->aio_context) {
+        bdrv_drained_end(state->bs);
+        aio_context_release(state->aio_context);
+    }
+}
+
+typedef struct BlockdevBackupState {
+    BlkActionState common;
+    BlockDriverState *bs;
+    BlockJob *job;
+    AioContext *aio_context;
+} BlockdevBackupState;
+
+static void do_blockdev_backup(const char *job_id, const char *device,
+                               const char *target, enum MirrorSyncMode sync,
+                               bool has_speed, int64_t speed,
+                               bool has_on_source_error,
+                               BlockdevOnError on_source_error,
+                               bool has_on_target_error,
+                               BlockdevOnError on_target_error,
+                               BlockJobTxn *txn, Error **errp);
+
+static void blockdev_backup_prepare(BlkActionState *common, Error **errp)
+{
+    BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
+    BlockdevBackup *backup;
+    BlockBackend *blk;
+    BlockDriverState *target;
+    Error *local_err = NULL;
+
+    assert(common->action->type == TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP);
+    backup = common->action->u.blockdev_backup.data;
+
+    blk = blk_by_name(backup->device);
+    if (!blk) {
+        error_setg(errp, "Device '%s' not found", backup->device);
+        return;
+    }
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, backup->device);
+        return;
+    }
+
+    target = bdrv_lookup_bs(backup->target, backup->target, errp);
+    if (!target) {
+        return;
+    }
+
+    /* AioContext is released in .clean() */
+    state->aio_context = blk_get_aio_context(blk);
+    if (state->aio_context != bdrv_get_aio_context(target)) {
+        state->aio_context = NULL;
+        error_setg(errp, "Backup between two IO threads is not implemented");
+        return;
+    }
+    aio_context_acquire(state->aio_context);
+    state->bs = blk_bs(blk);
+    bdrv_drained_begin(state->bs);
+
+    do_blockdev_backup(backup->has_job_id ? backup->job_id : NULL,
+                       backup->device, backup->target, backup->sync,
+                       backup->has_speed, backup->speed,
+                       backup->has_on_source_error, backup->on_source_error,
+                       backup->has_on_target_error, backup->on_target_error,
+                       common->block_job_txn, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    state->job = state->bs->job;
+}
+
+static void blockdev_backup_abort(BlkActionState *common)
+{
+    BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
+    BlockDriverState *bs = state->bs;
+
+    /* Only cancel if it's the job we started */
+    if (bs && bs->job && bs->job == state->job) {
+        block_job_cancel_sync(bs->job);
+    }
+}
+
+static void blockdev_backup_clean(BlkActionState *common)
+{
+    BlockdevBackupState *state = DO_UPCAST(BlockdevBackupState, common, common);
+
+    if (state->aio_context) {
+        bdrv_drained_end(state->bs);
+        aio_context_release(state->aio_context);
+    }
+}
+
+typedef struct BlockDirtyBitmapState {
+    BlkActionState common;
+    BdrvDirtyBitmap *bitmap;
+    BlockDriverState *bs;
+    AioContext *aio_context;
+    HBitmap *backup;
+    bool prepared;
+} BlockDirtyBitmapState;
+
+static void block_dirty_bitmap_add_prepare(BlkActionState *common,
+                                           Error **errp)
+{
+    Error *local_err = NULL;
+    BlockDirtyBitmapAdd *action;
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
+    action = common->action->u.block_dirty_bitmap_add.data;
+    /* AIO context taken and released within qmp_block_dirty_bitmap_add */
+    qmp_block_dirty_bitmap_add(action->node, action->name,
+                               action->has_granularity, action->granularity,
+                               &local_err);
+
+    if (!local_err) {
+        state->prepared = true;
+    } else {
+        error_propagate(errp, local_err);
+    }
+}
+
+static void block_dirty_bitmap_add_abort(BlkActionState *common)
+{
+    BlockDirtyBitmapAdd *action;
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    action = common->action->u.block_dirty_bitmap_add.data;
+    /* Should not be able to fail: IF the bitmap was added via .prepare(),
+     * then the node reference and bitmap name must have been valid.
+     */
+    if (state->prepared) {
+        qmp_block_dirty_bitmap_remove(action->node, action->name, &error_abort);
+    }
+}
+
+static void block_dirty_bitmap_clear_prepare(BlkActionState *common,
+                                             Error **errp)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+    BlockDirtyBitmap *action;
+
+    if (action_check_completion_mode(common, errp) < 0) {
+        return;
+    }
+
+    action = common->action->u.block_dirty_bitmap_clear.data;
+    state->bitmap = block_dirty_bitmap_lookup(action->node,
+                                              action->name,
+                                              &state->bs,
+                                              &state->aio_context,
+                                              errp);
+    if (!state->bitmap) {
+        return;
+    }
+
+    if (bdrv_dirty_bitmap_frozen(state->bitmap)) {
+        error_setg(errp, "Cannot modify a frozen bitmap");
+        return;
+    } else if (!bdrv_dirty_bitmap_enabled(state->bitmap)) {
+        error_setg(errp, "Cannot clear a disabled bitmap");
+        return;
+    }
+
+    bdrv_clear_dirty_bitmap(state->bitmap, &state->backup);
+    /* AioContext is released in .clean() */
+}
+
+static void block_dirty_bitmap_clear_abort(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    bdrv_undo_clear_dirty_bitmap(state->bitmap, state->backup);
+}
+
+static void block_dirty_bitmap_clear_commit(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    hbitmap_free(state->backup);
+}
+
+static void block_dirty_bitmap_clear_clean(BlkActionState *common)
+{
+    BlockDirtyBitmapState *state = DO_UPCAST(BlockDirtyBitmapState,
+                                             common, common);
+
+    if (state->aio_context) {
+        aio_context_release(state->aio_context);
+    }
+}
+
+static void abort_prepare(BlkActionState *common, Error **errp)
 {
     error_setg(errp, "Transaction aborted using Abort action");
 }
 
-static void abort_commit(BlkTransactionState *common)
+static void abort_commit(BlkActionState *common)
 {
     g_assert_not_reached(); /* this action never succeeds */
 }
 
-static const BdrvActionOps actions[] = {
+static const BlkActionOps actions[] = {
+    [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT] = {
+        .instance_size = sizeof(ExternalSnapshotState),
+        .prepare  = external_snapshot_prepare,
+        .commit   = external_snapshot_commit,
+        .abort = external_snapshot_abort,
+        .clean = external_snapshot_clean,
+    },
     [TRANSACTION_ACTION_KIND_BLOCKDEV_SNAPSHOT_SYNC] = {
         .instance_size = sizeof(ExternalSnapshotState),
         .prepare  = external_snapshot_prepare,
         .commit   = external_snapshot_commit,
         .abort = external_snapshot_abort,
+        .clean = external_snapshot_clean,
     },
     [TRANSACTION_ACTION_KIND_DRIVE_BACKUP] = {
         .instance_size = sizeof(DriveBackupState),
         .prepare = drive_backup_prepare,
         .abort = drive_backup_abort,
+        .clean = drive_backup_clean,
+    },
+    [TRANSACTION_ACTION_KIND_BLOCKDEV_BACKUP] = {
+        .instance_size = sizeof(BlockdevBackupState),
+        .prepare = blockdev_backup_prepare,
+        .abort = blockdev_backup_abort,
+        .clean = blockdev_backup_clean,
     },
     [TRANSACTION_ACTION_KIND_ABORT] = {
-        .instance_size = sizeof(BlkTransactionState),
+        .instance_size = sizeof(BlkActionState),
         .prepare = abort_prepare,
         .commit = abort_commit,
     },
@@ -1526,42 +2160,87 @@
         .instance_size = sizeof(InternalSnapshotState),
         .prepare  = internal_snapshot_prepare,
         .abort = internal_snapshot_abort,
+        .clean = internal_snapshot_clean,
     },
+    [TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_ADD] = {
+        .instance_size = sizeof(BlockDirtyBitmapState),
+        .prepare = block_dirty_bitmap_add_prepare,
+        .abort = block_dirty_bitmap_add_abort,
+    },
+    [TRANSACTION_ACTION_KIND_BLOCK_DIRTY_BITMAP_CLEAR] = {
+        .instance_size = sizeof(BlockDirtyBitmapState),
+        .prepare = block_dirty_bitmap_clear_prepare,
+        .commit = block_dirty_bitmap_clear_commit,
+        .abort = block_dirty_bitmap_clear_abort,
+        .clean = block_dirty_bitmap_clear_clean,
+    }
 };
 
-/*
- * 'Atomic' group snapshots.  The snapshots are taken as a set, and if any fail
- *  then we do not pivot any of the devices in the group, and abandon the
- *  snapshots
+/**
+ * Allocate a TransactionProperties structure if necessary, and fill
+ * that structure with desired defaults if they are unset.
  */
-void qmp_transaction(TransactionActionList *dev_list, Error **errp)
+static TransactionProperties *get_transaction_properties(
+    TransactionProperties *props)
+{
+    if (!props) {
+        props = g_new0(TransactionProperties, 1);
+    }
+
+    if (!props->has_completion_mode) {
+        props->has_completion_mode = true;
+        props->completion_mode = ACTION_COMPLETION_MODE_INDIVIDUAL;
+    }
+
+    return props;
+}
+
+/*
+ * 'Atomic' group operations.  The operations are performed as a set, and if
+ * any fail then we roll back all operations in the group.
+ */
+void qmp_transaction(TransactionActionList *dev_list,
+                     bool has_props,
+                     struct TransactionProperties *props,
+                     Error **errp)
 {
     TransactionActionList *dev_entry = dev_list;
-    BlkTransactionState *state, *next;
+    BlockJobTxn *block_job_txn = NULL;
+    BlkActionState *state, *next;
     Error *local_err = NULL;
 
-    QSIMPLEQ_HEAD(snap_bdrv_states, BlkTransactionState) snap_bdrv_states;
+    QSIMPLEQ_HEAD(snap_bdrv_states, BlkActionState) snap_bdrv_states;
     QSIMPLEQ_INIT(&snap_bdrv_states);
 
-    /* drain all i/o before any snapshots */
+    /* Does this transaction get canceled as a group on failure?
+     * If not, we don't really need to make a BlockJobTxn.
+     */
+    props = get_transaction_properties(props);
+    if (props->completion_mode != ACTION_COMPLETION_MODE_INDIVIDUAL) {
+        block_job_txn = block_job_txn_new();
+    }
+
+    /* drain all i/o before any operations */
     bdrv_drain_all();
 
-    /* We don't do anything in this loop that commits us to the snapshot */
+    /* We don't do anything in this loop that commits us to the operations */
     while (NULL != dev_entry) {
         TransactionAction *dev_info = NULL;
-        const BdrvActionOps *ops;
+        const BlkActionOps *ops;
 
         dev_info = dev_entry->value;
         dev_entry = dev_entry->next;
 
-        assert(dev_info->kind < ARRAY_SIZE(actions));
+        assert(dev_info->type < ARRAY_SIZE(actions));
 
-        ops = &actions[dev_info->kind];
+        ops = &actions[dev_info->type];
         assert(ops->instance_size > 0);
 
         state = g_malloc0(ops->instance_size);
         state->ops = ops;
         state->action = dev_info;
+        state->block_job_txn = block_job_txn;
+        state->txn_props = props;
         QSIMPLEQ_INSERT_TAIL(&snap_bdrv_states, state, entry);
 
         state->ops->prepare(state, &local_err);
@@ -1581,10 +2260,7 @@
     goto exit;
 
 delete_and_fail:
-    /*
-    * failure, and it is all-or-none; abandon each new bs, and keep using
-    * the original bs for all images
-    */
+    /* failure, and it is all-or-none; roll back all operations */
     QSIMPLEQ_FOREACH(state, &snap_bdrv_states, entry) {
         if (state->ops->abort) {
             state->ops->abort(state);
@@ -1597,45 +2273,29 @@
         }
         g_free(state);
     }
-}
-
-
-static void eject_device(BlockBackend *blk, int force, Error **errp)
-{
-    BlockDriverState *bs = blk_bs(blk);
-
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
-        return;
+    if (!has_props) {
+        qapi_free_TransactionProperties(props);
     }
-    if (!blk_dev_has_removable_media(blk)) {
-        error_setg(errp, "Device '%s' is not removable",
-                   bdrv_get_device_name(bs));
-        return;
-    }
-
-    if (blk_dev_is_medium_locked(blk) && !blk_dev_is_tray_open(blk)) {
-        blk_dev_eject_request(blk, force);
-        if (!force) {
-            error_setg(errp, "Device '%s' is locked",
-                       bdrv_get_device_name(bs));
-            return;
-        }
-    }
-
-    bdrv_close(bs);
+    block_job_txn_unref(block_job_txn);
 }
 
 void qmp_eject(const char *device, bool has_force, bool force, Error **errp)
 {
-    BlockBackend *blk;
+    Error *local_err = NULL;
+    int rc;
 
-    blk = blk_by_name(device);
-    if (!blk) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
-        return;
+    if (!has_force) {
+        force = false;
     }
 
-    eject_device(blk, force, errp);
+    rc = do_open_tray(device, force, &local_err);
+    if (rc && rc != -ENOSYS) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    error_free(local_err);
+
+    qmp_x_blockdev_remove_medium(device, errp);
 }
 
 void qmp_block_passwd(bool has_device, const char *device,
@@ -1644,7 +2304,7 @@
 {
     Error *local_err = NULL;
     BlockDriverState *bs;
-    int err;
+    AioContext *aio_context;
 
     bs = bdrv_lookup_bs(has_device ? device : NULL,
                         has_node_name ? node_name : NULL,
@@ -1654,163 +2314,521 @@
         return;
     }
 
-    err = bdrv_set_key(bs, password);
-    if (err == -EINVAL) {
-        error_set(errp, QERR_DEVICE_NOT_ENCRYPTED, bdrv_get_device_name(bs));
-        return;
-    } else if (err < 0) {
-        error_set(errp, QERR_INVALID_PASSWORD);
-        return;
-    }
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_add_key(bs, password, errp);
+
+    aio_context_release(aio_context);
 }
 
-static void qmp_bdrv_open_encrypted(BlockDriverState *bs, const char *filename,
-                                    int bdrv_flags, BlockDriver *drv,
-                                    const char *password, Error **errp)
-{
-    Error *local_err = NULL;
-    int ret;
-
-    ret = bdrv_open(&bs, filename, NULL, NULL, bdrv_flags, drv, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    if (bdrv_key_required(bs)) {
-        if (password) {
-            if (bdrv_set_key(bs, password) < 0) {
-                error_set(errp, QERR_INVALID_PASSWORD);
-            }
-        } else {
-            error_set(errp, QERR_DEVICE_ENCRYPTED, bdrv_get_device_name(bs),
-                      bdrv_get_encrypted_filename(bs));
-        }
-    } else if (password) {
-        error_set(errp, QERR_DEVICE_NOT_ENCRYPTED, bdrv_get_device_name(bs));
-    }
-}
-
-void qmp_change_blockdev(const char *device, const char *filename,
-                         const char *format, Error **errp)
+/*
+ * Attempt to open the tray of @device.
+ * If @force, ignore its tray lock.
+ * Else, if the tray is locked, don't open it, but ask the guest to open it.
+ * On error, store an error through @errp and return -errno.
+ * If @device does not exist, return -ENODEV.
+ * If it has no removable media, return -ENOTSUP.
+ * If it has no tray, return -ENOSYS.
+ * If the guest was asked to open the tray, return -EINPROGRESS.
+ * Else, return 0.
+ */
+static int do_open_tray(const char *device, bool force, Error **errp)
 {
     BlockBackend *blk;
-    BlockDriverState *bs;
-    BlockDriver *drv = NULL;
-    int bdrv_flags;
-    Error *err = NULL;
+    bool locked;
 
     blk = blk_by_name(device);
     if (!blk) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
-        return;
-    }
-    bs = blk_bs(blk);
-
-    if (format) {
-        drv = bdrv_find_whitelisted_format(format, bs->read_only);
-        if (!drv) {
-            error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            return;
-        }
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return -ENODEV;
     }
 
-    eject_device(blk, 0, &err);
-    if (err) {
-        error_propagate(errp, err);
-        return;
+    if (!blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return -ENOTSUP;
     }
 
-    bdrv_flags = bdrv_is_read_only(bs) ? 0 : BDRV_O_RDWR;
-    bdrv_flags |= bdrv_is_snapshot(bs) ? BDRV_O_SNAPSHOT : 0;
+    if (!blk_dev_has_tray(blk)) {
+        error_setg(errp, "Device '%s' does not have a tray", device);
+        return -ENOSYS;
+    }
 
-    qmp_bdrv_open_encrypted(bs, filename, bdrv_flags, drv, NULL, errp);
+    if (blk_dev_is_tray_open(blk)) {
+        return 0;
+    }
+
+    locked = blk_dev_is_medium_locked(blk);
+    if (locked) {
+        blk_dev_eject_request(blk, force);
+    }
+
+    if (!locked || force) {
+        blk_dev_change_media_cb(blk, false);
+    }
+
+    if (locked && !force) {
+        error_setg(errp, "Device '%s' is locked and force was not specified, "
+                   "wait for tray to open and try again", device);
+        return -EINPROGRESS;
+    }
+
+    return 0;
 }
 
-/* throttling disk I/O limits */
-void qmp_block_set_io_throttle(const char *device, int64_t bps, int64_t bps_rd,
-                               int64_t bps_wr,
-                               int64_t iops,
-                               int64_t iops_rd,
-                               int64_t iops_wr,
-                               bool has_bps_max,
-                               int64_t bps_max,
-                               bool has_bps_rd_max,
-                               int64_t bps_rd_max,
-                               bool has_bps_wr_max,
-                               int64_t bps_wr_max,
-                               bool has_iops_max,
-                               int64_t iops_max,
-                               bool has_iops_rd_max,
-                               int64_t iops_rd_max,
-                               bool has_iops_wr_max,
-                               int64_t iops_wr_max,
-                               bool has_iops_size,
-                               int64_t iops_size, Error **errp)
+void qmp_blockdev_open_tray(const char *device, bool has_force, bool force,
+                            Error **errp)
 {
-    ThrottleConfig cfg;
-    BlockDriverState *bs;
-    AioContext *aio_context;
+    Error *local_err = NULL;
+    int rc;
 
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    if (!has_force) {
+        force = false;
+    }
+    rc = do_open_tray(device, force, &local_err);
+    if (rc && rc != -ENOSYS && rc != -EINPROGRESS) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    error_free(local_err);
+}
+
+void qmp_blockdev_close_tray(const char *device, Error **errp)
+{
+    BlockBackend *blk;
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
-    memset(&cfg, 0, sizeof(cfg));
-    cfg.buckets[THROTTLE_BPS_TOTAL].avg = bps;
-    cfg.buckets[THROTTLE_BPS_READ].avg  = bps_rd;
-    cfg.buckets[THROTTLE_BPS_WRITE].avg = bps_wr;
-
-    cfg.buckets[THROTTLE_OPS_TOTAL].avg = iops;
-    cfg.buckets[THROTTLE_OPS_READ].avg  = iops_rd;
-    cfg.buckets[THROTTLE_OPS_WRITE].avg = iops_wr;
-
-    if (has_bps_max) {
-        cfg.buckets[THROTTLE_BPS_TOTAL].max = bps_max;
-    }
-    if (has_bps_rd_max) {
-        cfg.buckets[THROTTLE_BPS_READ].max = bps_rd_max;
-    }
-    if (has_bps_wr_max) {
-        cfg.buckets[THROTTLE_BPS_WRITE].max = bps_wr_max;
-    }
-    if (has_iops_max) {
-        cfg.buckets[THROTTLE_OPS_TOTAL].max = iops_max;
-    }
-    if (has_iops_rd_max) {
-        cfg.buckets[THROTTLE_OPS_READ].max = iops_rd_max;
-    }
-    if (has_iops_wr_max) {
-        cfg.buckets[THROTTLE_OPS_WRITE].max = iops_wr_max;
+    if (!blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
     }
 
-    if (has_iops_size) {
-        cfg.op_size = iops_size;
+    if (!blk_dev_has_tray(blk)) {
+        /* Ignore this command on tray-less devices */
+        return;
     }
 
-    if (!check_throttle_config(&cfg, errp)) {
+    if (!blk_dev_is_tray_open(blk)) {
+        return;
+    }
+
+    blk_dev_change_media_cb(blk, true);
+}
+
+void qmp_x_blockdev_remove_medium(const char *device, Error **errp)
+{
+    BlockBackend *blk;
+    BlockDriverState *bs;
+    AioContext *aio_context;
+    bool has_device;
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
+    }
+
+    /* For BBs without a device, we can exchange the BDS tree at will */
+    has_device = blk_get_attached_dev(blk);
+
+    if (has_device && !blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (has_device && blk_dev_has_tray(blk) && !blk_dev_is_tray_open(blk)) {
+        error_setg(errp, "Tray of device '%s' is not open", device);
+        return;
+    }
+
+    bs = blk_bs(blk);
+    if (!bs) {
         return;
     }
 
     aio_context = bdrv_get_aio_context(bs);
     aio_context_acquire(aio_context);
 
-    if (!bs->io_limits_enabled && throttle_enabled(&cfg)) {
-        bdrv_io_limits_enable(bs);
-    } else if (bs->io_limits_enabled && !throttle_enabled(&cfg)) {
-        bdrv_io_limits_disable(bs);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
+        goto out;
     }
 
-    if (bs->io_limits_enabled) {
-        bdrv_set_io_limits(bs, &cfg);
+    blk_remove_bs(blk);
+
+    if (!blk_dev_has_tray(blk)) {
+        /* For tray-less devices, blockdev-open-tray is a no-op (or may not be
+         * called at all); therefore, the medium needs to be ejected here.
+         * Do it after blk_remove_bs() so blk_is_inserted(blk) returns the @load
+         * value passed here (i.e. false). */
+        blk_dev_change_media_cb(blk, false);
     }
 
+out:
     aio_context_release(aio_context);
 }
 
-int do_drive_del(Monitor *mon, const QDict *qdict, QObject **ret_data)
+static void qmp_blockdev_insert_anon_medium(const char *device,
+                                            BlockDriverState *bs, Error **errp)
+{
+    BlockBackend *blk;
+    bool has_device;
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        return;
+    }
+
+    /* For BBs without a device, we can exchange the BDS tree at will */
+    has_device = blk_get_attached_dev(blk);
+
+    if (has_device && !blk_dev_has_removable_media(blk)) {
+        error_setg(errp, "Device '%s' is not removable", device);
+        return;
+    }
+
+    if (has_device && blk_dev_has_tray(blk) && !blk_dev_is_tray_open(blk)) {
+        error_setg(errp, "Tray of device '%s' is not open", device);
+        return;
+    }
+
+    if (blk_bs(blk)) {
+        error_setg(errp, "There already is a medium in device '%s'", device);
+        return;
+    }
+
+    blk_insert_bs(blk, bs);
+
+    if (!blk_dev_has_tray(blk)) {
+        /* For tray-less devices, blockdev-close-tray is a no-op (or may not be
+         * called at all); therefore, the medium needs to be pushed into the
+         * slot here.
+         * Do it after blk_insert_bs() so blk_is_inserted(blk) returns the @load
+         * value passed here (i.e. true). */
+        blk_dev_change_media_cb(blk, true);
+    }
+}
+
+void qmp_x_blockdev_insert_medium(const char *device, const char *node_name,
+                                  Error **errp)
+{
+    BlockDriverState *bs;
+
+    bs = bdrv_find_node(node_name);
+    if (!bs) {
+        error_setg(errp, "Node '%s' not found", node_name);
+        return;
+    }
+
+    if (bdrv_has_blk(bs)) {
+        error_setg(errp, "Node '%s' is already in use by '%s'", node_name,
+                   bdrv_get_parent_name(bs));
+        return;
+    }
+
+    qmp_blockdev_insert_anon_medium(device, bs, errp);
+}
+
+void qmp_blockdev_change_medium(const char *device, const char *filename,
+                                bool has_format, const char *format,
+                                bool has_read_only,
+                                BlockdevChangeReadOnlyMode read_only,
+                                Error **errp)
+{
+    BlockBackend *blk;
+    BlockDriverState *medium_bs = NULL;
+    int bdrv_flags;
+    int rc;
+    QDict *options = NULL;
+    Error *err = NULL;
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
+        goto fail;
+    }
+
+    if (blk_bs(blk)) {
+        blk_update_root_state(blk);
+    }
+
+    bdrv_flags = blk_get_open_flags_from_root_state(blk);
+    bdrv_flags &= ~(BDRV_O_TEMPORARY | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING |
+        BDRV_O_PROTOCOL);
+
+    if (!has_read_only) {
+        read_only = BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN;
+    }
+
+    switch (read_only) {
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN:
+        break;
+
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_READ_ONLY:
+        bdrv_flags &= ~BDRV_O_RDWR;
+        break;
+
+    case BLOCKDEV_CHANGE_READ_ONLY_MODE_READ_WRITE:
+        bdrv_flags |= BDRV_O_RDWR;
+        break;
+
+    default:
+        abort();
+    }
+
+    if (has_format) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
+
+    medium_bs = bdrv_open(filename, NULL, options, bdrv_flags, errp);
+    if (!medium_bs) {
+        goto fail;
+    }
+
+    bdrv_add_key(medium_bs, NULL, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
+
+    rc = do_open_tray(device, false, &err);
+    if (rc && rc != -ENOSYS) {
+        error_propagate(errp, err);
+        goto fail;
+    }
+    error_free(err);
+    err = NULL;
+
+    qmp_x_blockdev_remove_medium(device, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
+
+    qmp_blockdev_insert_anon_medium(device, medium_bs, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto fail;
+    }
+
+    blk_apply_root_state(blk, medium_bs);
+
+    qmp_blockdev_close_tray(device, errp);
+
+fail:
+    /* If the medium has been inserted, the device has its own reference, so
+     * ours must be relinquished; and if it has not been inserted successfully,
+     * the reference must be relinquished anyway */
+    bdrv_unref(medium_bs);
+}
+
+/* throttling disk I/O limits */
+void qmp_block_set_io_throttle(BlockIOThrottle *arg, Error **errp)
+{
+    ThrottleConfig cfg;
+    BlockDriverState *bs;
+    BlockBackend *blk;
+    AioContext *aio_context;
+
+    blk = blk_by_name(arg->device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", arg->device);
+        return;
+    }
+
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
+
+    bs = blk_bs(blk);
+    if (!bs) {
+        error_setg(errp, "Device '%s' has no medium", arg->device);
+        goto out;
+    }
+
+    throttle_config_init(&cfg);
+    cfg.buckets[THROTTLE_BPS_TOTAL].avg = arg->bps;
+    cfg.buckets[THROTTLE_BPS_READ].avg  = arg->bps_rd;
+    cfg.buckets[THROTTLE_BPS_WRITE].avg = arg->bps_wr;
+
+    cfg.buckets[THROTTLE_OPS_TOTAL].avg = arg->iops;
+    cfg.buckets[THROTTLE_OPS_READ].avg  = arg->iops_rd;
+    cfg.buckets[THROTTLE_OPS_WRITE].avg = arg->iops_wr;
+
+    if (arg->has_bps_max) {
+        cfg.buckets[THROTTLE_BPS_TOTAL].max = arg->bps_max;
+    }
+    if (arg->has_bps_rd_max) {
+        cfg.buckets[THROTTLE_BPS_READ].max = arg->bps_rd_max;
+    }
+    if (arg->has_bps_wr_max) {
+        cfg.buckets[THROTTLE_BPS_WRITE].max = arg->bps_wr_max;
+    }
+    if (arg->has_iops_max) {
+        cfg.buckets[THROTTLE_OPS_TOTAL].max = arg->iops_max;
+    }
+    if (arg->has_iops_rd_max) {
+        cfg.buckets[THROTTLE_OPS_READ].max = arg->iops_rd_max;
+    }
+    if (arg->has_iops_wr_max) {
+        cfg.buckets[THROTTLE_OPS_WRITE].max = arg->iops_wr_max;
+    }
+
+    if (arg->has_bps_max_length) {
+        cfg.buckets[THROTTLE_BPS_TOTAL].burst_length = arg->bps_max_length;
+    }
+    if (arg->has_bps_rd_max_length) {
+        cfg.buckets[THROTTLE_BPS_READ].burst_length = arg->bps_rd_max_length;
+    }
+    if (arg->has_bps_wr_max_length) {
+        cfg.buckets[THROTTLE_BPS_WRITE].burst_length = arg->bps_wr_max_length;
+    }
+    if (arg->has_iops_max_length) {
+        cfg.buckets[THROTTLE_OPS_TOTAL].burst_length = arg->iops_max_length;
+    }
+    if (arg->has_iops_rd_max_length) {
+        cfg.buckets[THROTTLE_OPS_READ].burst_length = arg->iops_rd_max_length;
+    }
+    if (arg->has_iops_wr_max_length) {
+        cfg.buckets[THROTTLE_OPS_WRITE].burst_length = arg->iops_wr_max_length;
+    }
+
+    if (arg->has_iops_size) {
+        cfg.op_size = arg->iops_size;
+    }
+
+    if (!throttle_is_valid(&cfg, errp)) {
+        goto out;
+    }
+
+    if (throttle_enabled(&cfg)) {
+        /* Enable I/O limits if they're not enabled yet, otherwise
+         * just update the throttling group. */
+        if (!blk_get_public(blk)->throttle_state) {
+            blk_io_limits_enable(blk,
+                                 arg->has_group ? arg->group : arg->device);
+        } else if (arg->has_group) {
+            blk_io_limits_update_group(blk, arg->group);
+        }
+        /* Set the new throttling configuration */
+        blk_set_io_limits(blk, &cfg);
+    } else if (blk_get_public(blk)->throttle_state) {
+        /* If all throttling settings are set to 0, disable I/O limits */
+        blk_io_limits_disable(blk);
+    }
+
+out:
+    aio_context_release(aio_context);
+}
+
+void qmp_block_dirty_bitmap_add(const char *node, const char *name,
+                                bool has_granularity, uint32_t granularity,
+                                Error **errp)
+{
+    AioContext *aio_context;
+    BlockDriverState *bs;
+
+    if (!name || name[0] == '\0') {
+        error_setg(errp, "Bitmap name cannot be empty");
+        return;
+    }
+
+    bs = bdrv_lookup_bs(node, node, errp);
+    if (!bs) {
+        return;
+    }
+
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    if (has_granularity) {
+        if (granularity < 512 || !is_power_of_2(granularity)) {
+            error_setg(errp, "Granularity must be power of 2 "
+                             "and at least 512");
+            goto out;
+        }
+    } else {
+        /* Default to cluster size, if available: */
+        granularity = bdrv_get_default_bitmap_granularity(bs);
+    }
+
+    bdrv_create_dirty_bitmap(bs, granularity, name, errp);
+
+ out:
+    aio_context_release(aio_context);
+}
+
+void qmp_block_dirty_bitmap_remove(const char *node, const char *name,
+                                   Error **errp)
+{
+    AioContext *aio_context;
+    BlockDriverState *bs;
+    BdrvDirtyBitmap *bitmap;
+
+    bitmap = block_dirty_bitmap_lookup(node, name, &bs, &aio_context, errp);
+    if (!bitmap || !bs) {
+        return;
+    }
+
+    if (bdrv_dirty_bitmap_frozen(bitmap)) {
+        error_setg(errp,
+                   "Bitmap '%s' is currently frozen and cannot be removed",
+                   name);
+        goto out;
+    }
+    bdrv_dirty_bitmap_make_anon(bitmap);
+    bdrv_release_dirty_bitmap(bs, bitmap);
+
+ out:
+    aio_context_release(aio_context);
+}
+
+/**
+ * Completely clear a bitmap, for the purposes of synchronizing a bitmap
+ * immediately after a full backup operation.
+ */
+void qmp_block_dirty_bitmap_clear(const char *node, const char *name,
+                                  Error **errp)
+{
+    AioContext *aio_context;
+    BdrvDirtyBitmap *bitmap;
+    BlockDriverState *bs;
+
+    bitmap = block_dirty_bitmap_lookup(node, name, &bs, &aio_context, errp);
+    if (!bitmap || !bs) {
+        return;
+    }
+
+    if (bdrv_dirty_bitmap_frozen(bitmap)) {
+        error_setg(errp,
+                   "Bitmap '%s' is currently frozen and cannot be modified",
+                   name);
+        goto out;
+    } else if (!bdrv_dirty_bitmap_enabled(bitmap)) {
+        error_setg(errp,
+                   "Bitmap '%s' is currently disabled and cannot be cleared",
+                   name);
+        goto out;
+    }
+
+    bdrv_clear_dirty_bitmap(bitmap, NULL);
+
+ out:
+    aio_context_release(aio_context);
+}
+
+void hmp_drive_del(Monitor *mon, const QDict *qdict)
 {
     const char *id = qdict_get_str(qdict, "id");
     BlockBackend *blk;
@@ -1818,50 +2836,56 @@
     AioContext *aio_context;
     Error *local_err = NULL;
 
+    bs = bdrv_find_node(id);
+    if (bs) {
+        qmp_x_blockdev_del(false, NULL, true, id, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
+        }
+        return;
+    }
+
     blk = blk_by_name(id);
     if (!blk) {
         error_report("Device '%s' not found", id);
-        return -1;
+        return;
     }
-    bs = blk_bs(blk);
 
     if (!blk_legacy_dinfo(blk)) {
         error_report("Deleting device added with blockdev-add"
                      " is not supported");
-        return -1;
+        return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
-        aio_context_release(aio_context);
-        return -1;
+    bs = blk_bs(blk);
+    if (bs) {
+        if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
+            error_report_err(local_err);
+            aio_context_release(aio_context);
+            return;
+        }
+
+        blk_remove_bs(blk);
     }
 
-    /* quiesce block driver; prevent further io */
-    bdrv_drain_all();
-    bdrv_flush(bs);
-    bdrv_close(bs);
+    /* Make the BlockBackend and the attached BlockDriverState anonymous */
+    monitor_remove_blk(blk);
 
-    /* if we have a device attached to this BlockDriverState
-     * then we need to make the drive anonymous until the device
-     * can be removed.  If this is a drive with no device backing
-     * then we can just get rid of the block driver state right here.
+    /* If this BlockBackend has a device attached to it, its refcount will be
+     * decremented when the device is removed; otherwise we have to do so here.
      */
     if (blk_get_attached_dev(blk)) {
-        blk_hide_on_behalf_of_do_drive_del(blk);
         /* Further I/O must not pause the guest */
-        bdrv_set_on_error(bs, BLOCKDEV_ON_ERROR_REPORT,
-                          BLOCKDEV_ON_ERROR_REPORT);
+        blk_set_on_error(blk, BLOCKDEV_ON_ERROR_REPORT,
+                         BLOCKDEV_ON_ERROR_REPORT);
     } else {
         blk_unref(blk);
     }
 
     aio_context_release(aio_context);
-    return 0;
 }
 
 void qmp_block_resize(bool has_device, const char *device,
@@ -1885,17 +2909,17 @@
     aio_context_acquire(aio_context);
 
     if (!bdrv_is_first_non_filter(bs)) {
-        error_set(errp, QERR_FEATURE_DISABLED, "resize");
+        error_setg(errp, QERR_FEATURE_DISABLED, "resize");
         goto out;
     }
 
     if (size < 0) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "size", "a >0 size");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "size", "a >0 size");
         goto out;
     }
 
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_RESIZE, NULL)) {
-        error_set(errp, QERR_DEVICE_IN_USE, device);
+        error_setg(errp, QERR_DEVICE_IN_USE, device);
         goto out;
     }
 
@@ -1907,16 +2931,16 @@
     case 0:
         break;
     case -ENOMEDIUM:
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         break;
     case -ENOTSUP:
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         break;
     case -EACCES:
-        error_set(errp, QERR_DEVICE_IS_READ_ONLY, device);
+        error_setg(errp, "Device '%s' is read only", device);
         break;
     case -EBUSY:
-        error_set(errp, QERR_DEVICE_IN_USE, device);
+        error_setg(errp, QERR_DEVICE_IN_USE, device);
         break;
     default:
         error_setg_errno(errp, -ret, "Could not resize");
@@ -1950,17 +2974,16 @@
     } else {
         block_job_event_completed(bs->job, msg);
     }
-
-    bdrv_put_ref_bh_schedule(bs);
 }
 
-void qmp_block_stream(const char *device,
+void qmp_block_stream(bool has_job_id, const char *job_id, const char *device,
                       bool has_base, const char *base,
                       bool has_backing_file, const char *backing_file,
                       bool has_speed, int64_t speed,
                       bool has_on_error, BlockdevOnError on_error,
                       Error **errp)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     BlockDriverState *base_bs = NULL;
     AioContext *aio_context;
@@ -1971,15 +2994,22 @@
         on_error = BLOCKDEV_ON_ERROR_REPORT;
     }
 
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_STREAM, errp)) {
         goto out;
     }
@@ -1987,7 +3017,7 @@
     if (has_base) {
         base_bs = bdrv_find_backing_image(bs, base);
         if (base_bs == NULL) {
-            error_set(errp, QERR_BASE_NOT_FOUND, base);
+            error_setg(errp, QERR_BASE_NOT_FOUND, base);
             goto out;
         }
         assert(bdrv_get_aio_context(base_bs) == aio_context);
@@ -2005,8 +3035,8 @@
     /* backing_file string overrides base bs filename */
     base_name = has_backing_file ? backing_file : base_name;
 
-    stream_start(bs, base_bs, base_name, has_speed ? speed : 0,
-                 on_error, block_job_cb, bs, &local_err);
+    stream_start(has_job_id ? job_id : NULL, bs, base_bs, base_name,
+                 has_speed ? speed : 0, on_error, block_job_cb, bs, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto out;
@@ -2018,13 +3048,14 @@
     aio_context_release(aio_context);
 }
 
-void qmp_block_commit(const char *device,
+void qmp_block_commit(bool has_job_id, const char *job_id, const char *device,
                       bool has_base, const char *base,
                       bool has_top, const char *top,
                       bool has_backing_file, const char *backing_file,
                       bool has_speed, int64_t speed,
                       Error **errp)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     BlockDriverState *base_bs, *top_bs;
     AioContext *aio_context;
@@ -2043,19 +3074,23 @@
      *  live commit feature versions; for this to work, we must make sure to
      *  perform the device lookup before any generic errors that may occur in a
      *  scenario in which all optional arguments are omitted. */
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    /* drain all i/o before commits */
-    bdrv_drain_all();
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT, errp)) {
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT_SOURCE, errp)) {
         goto out;
     }
 
@@ -2082,12 +3117,16 @@
     }
 
     if (base_bs == NULL) {
-        error_set(errp, QERR_BASE_NOT_FOUND, base ? base : "NULL");
+        error_setg(errp, QERR_BASE_NOT_FOUND, base ? base : "NULL");
         goto out;
     }
 
     assert(bdrv_get_aio_context(base_bs) == aio_context);
 
+    if (bdrv_op_is_blocked(base_bs, BLOCK_OP_TYPE_COMMIT_TARGET, errp)) {
+        goto out;
+    }
+
     /* Do not allow attempts to commit an image into itself */
     if (top_bs == base_bs) {
         error_setg(errp, "cannot commit an image into itself");
@@ -2100,10 +3139,11 @@
                              " but 'top' is the active layer");
             goto out;
         }
-        commit_active_start(bs, base_bs, speed, on_error, block_job_cb,
-                            bs, &local_err);
+        commit_active_start(has_job_id ? job_id : NULL, bs, base_bs, speed,
+                            on_error, block_job_cb, bs, &local_err);
     } else {
-        commit_start(bs, base_bs, top_bs, speed, on_error, block_job_cb, bs,
+        commit_start(has_job_id ? job_id : NULL, bs, base_bs, top_bs, speed,
+                     on_error, block_job_cb, bs,
                      has_backing_file ? backing_file : NULL, &local_err);
     }
     if (local_err != NULL) {
@@ -2115,24 +3155,28 @@
     aio_context_release(aio_context);
 }
 
-void qmp_drive_backup(const char *device, const char *target,
-                      bool has_format, const char *format,
-                      enum MirrorSyncMode sync,
-                      bool has_mode, enum NewImageMode mode,
-                      bool has_speed, int64_t speed,
-                      bool has_on_source_error, BlockdevOnError on_source_error,
-                      bool has_on_target_error, BlockdevOnError on_target_error,
-                      Error **errp)
+static void do_drive_backup(const char *job_id, const char *device,
+                            const char *target, bool has_format,
+                            const char *format, enum MirrorSyncMode sync,
+                            bool has_mode, enum NewImageMode mode,
+                            bool has_speed, int64_t speed,
+                            bool has_bitmap, const char *bitmap,
+                            bool has_on_source_error,
+                            BlockdevOnError on_source_error,
+                            bool has_on_target_error,
+                            BlockdevOnError on_target_error,
+                            BlockJobTxn *txn, Error **errp)
 {
+    BlockBackend *blk;
     BlockDriverState *bs;
     BlockDriverState *target_bs;
     BlockDriverState *source = NULL;
+    BdrvDirtyBitmap *bmap = NULL;
     AioContext *aio_context;
-    BlockDriver *drv = NULL;
+    QDict *options = NULL;
     Error *local_err = NULL;
     int flags;
     int64_t size;
-    int ret;
 
     if (!has_speed) {
         speed = 0;
@@ -2147,31 +3191,29 @@
         mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     }
 
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (!bdrv_is_inserted(bs)) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+    /* Although backup_run has this check too, we need to use bs->drv below, so
+     * do an early check redundantly. */
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
         goto out;
     }
+    bs = blk_bs(blk);
 
     if (!has_format) {
         format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
     }
-    if (format) {
-        drv = bdrv_find_format(format);
-        if (!drv) {
-            error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
-    }
 
+    /* Early check to avoid creating target */
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
         goto out;
     }
@@ -2181,7 +3223,7 @@
     /* See if we have a backing HD we can use to create our new image
      * on top of. */
     if (sync == MIRROR_SYNC_MODE_TOP) {
-        source = bs->backing_hd;
+        source = backing_bs(bs);
         if (!source) {
             sync = MIRROR_SYNC_MODE_FULL;
         }
@@ -2197,7 +3239,7 @@
     }
 
     if (mode != NEW_IMAGE_MODE_EXISTING) {
-        assert(format && drv);
+        assert(format);
         if (source) {
             bdrv_img_create(target, format, source->filename,
                             source->drv->format_name, NULL,
@@ -2213,19 +3255,32 @@
         goto out;
     }
 
-    target_bs = NULL;
-    ret = bdrv_open(&target_bs, target, NULL, NULL, flags, drv, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
+    if (format) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(format));
+    }
+
+    target_bs = bdrv_open(target, NULL, options, flags, errp);
+    if (!target_bs) {
         goto out;
     }
 
     bdrv_set_aio_context(target_bs, aio_context);
 
-    backup_start(bs, target_bs, speed, sync, on_source_error, on_target_error,
-                 block_job_cb, bs, &local_err);
+    if (has_bitmap) {
+        bmap = bdrv_find_dirty_bitmap(bs, bitmap);
+        if (!bmap) {
+            error_setg(errp, "Bitmap '%s' could not be found", bitmap);
+            bdrv_unref(target_bs);
+            goto out;
+        }
+    }
+
+    backup_start(job_id, bs, target_bs, speed, sync, bmap,
+                 on_source_error, on_target_error,
+                 block_job_cb, bs, txn, &local_err);
+    bdrv_unref(target_bs);
     if (local_err != NULL) {
-        bdrv_unref(target_bs);
         error_propagate(errp, local_err);
         goto out;
     }
@@ -2234,35 +3289,45 @@
     aio_context_release(aio_context);
 }
 
-BlockDeviceInfoList *qmp_query_named_block_nodes(Error **errp)
-{
-    return bdrv_named_nodes_list();
-}
-
-#define DEFAULT_MIRROR_BUF_SIZE   (10 << 20)
-
-void qmp_drive_mirror(const char *device, const char *target,
+void qmp_drive_backup(bool has_job_id, const char *job_id,
+                      const char *device, const char *target,
                       bool has_format, const char *format,
-                      bool has_node_name, const char *node_name,
-                      bool has_replaces, const char *replaces,
                       enum MirrorSyncMode sync,
                       bool has_mode, enum NewImageMode mode,
                       bool has_speed, int64_t speed,
-                      bool has_granularity, uint32_t granularity,
-                      bool has_buf_size, int64_t buf_size,
+                      bool has_bitmap, const char *bitmap,
                       bool has_on_source_error, BlockdevOnError on_source_error,
                       bool has_on_target_error, BlockdevOnError on_target_error,
                       Error **errp)
 {
+    return do_drive_backup(has_job_id ? job_id : NULL, device, target,
+                           has_format, format, sync,
+                           has_mode, mode, has_speed, speed,
+                           has_bitmap, bitmap,
+                           has_on_source_error, on_source_error,
+                           has_on_target_error, on_target_error,
+                           NULL, errp);
+}
+
+BlockDeviceInfoList *qmp_query_named_block_nodes(Error **errp)
+{
+    return bdrv_named_nodes_list(errp);
+}
+
+void do_blockdev_backup(const char *job_id, const char *device,
+                        const char *target, enum MirrorSyncMode sync,
+                         bool has_speed, int64_t speed,
+                         bool has_on_source_error,
+                         BlockdevOnError on_source_error,
+                         bool has_on_target_error,
+                         BlockdevOnError on_target_error,
+                         BlockJobTxn *txn, Error **errp)
+{
+    BlockBackend *blk;
     BlockDriverState *bs;
-    BlockDriverState *source, *target_bs;
-    AioContext *aio_context;
-    BlockDriver *drv = NULL;
+    BlockDriverState *target_bs;
     Error *local_err = NULL;
-    QDict *options = NULL;
-    int flags;
-    int64_t size;
-    int ret;
+    AioContext *aio_context;
 
     if (!has_speed) {
         speed = 0;
@@ -2273,61 +3338,177 @@
     if (!has_on_target_error) {
         on_target_error = BLOCKDEV_ON_ERROR_REPORT;
     }
-    if (!has_mode) {
-        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_setg(errp, "Device '%s' not found", device);
+        return;
+    }
+
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
+    target_bs = bdrv_lookup_bs(target, target, errp);
+    if (!target_bs) {
+        goto out;
+    }
+
+    if (bdrv_get_aio_context(target_bs) != aio_context) {
+        if (!bdrv_has_blk(target_bs)) {
+            /* The target BDS is not attached, we can safely move it to another
+             * AioContext. */
+            bdrv_set_aio_context(target_bs, aio_context);
+        } else {
+            error_setg(errp, "Target is attached to a different thread from "
+                             "source.");
+            goto out;
+        }
+    }
+    backup_start(job_id, bs, target_bs, speed, sync, NULL, on_source_error,
+                 on_target_error, block_job_cb, bs, txn, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+    }
+out:
+    aio_context_release(aio_context);
+}
+
+void qmp_blockdev_backup(bool has_job_id, const char *job_id,
+                         const char *device, const char *target,
+                         enum MirrorSyncMode sync,
+                         bool has_speed, int64_t speed,
+                         bool has_on_source_error,
+                         BlockdevOnError on_source_error,
+                         bool has_on_target_error,
+                         BlockdevOnError on_target_error,
+                         Error **errp)
+{
+    do_blockdev_backup(has_job_id ? job_id : NULL, device, target,
+                       sync, has_speed, speed,
+                       has_on_source_error, on_source_error,
+                       has_on_target_error, on_target_error,
+                       NULL, errp);
+}
+
+/* Parameter check and block job starting for drive mirroring.
+ * Caller should hold @device and @target's aio context (must be the same).
+ **/
+static void blockdev_mirror_common(const char *job_id, BlockDriverState *bs,
+                                   BlockDriverState *target,
+                                   bool has_replaces, const char *replaces,
+                                   enum MirrorSyncMode sync,
+                                   BlockMirrorBackingMode backing_mode,
+                                   bool has_speed, int64_t speed,
+                                   bool has_granularity, uint32_t granularity,
+                                   bool has_buf_size, int64_t buf_size,
+                                   bool has_on_source_error,
+                                   BlockdevOnError on_source_error,
+                                   bool has_on_target_error,
+                                   BlockdevOnError on_target_error,
+                                   bool has_unmap, bool unmap,
+                                   Error **errp)
+{
+
+    if (!has_speed) {
+        speed = 0;
+    }
+    if (!has_on_source_error) {
+        on_source_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+    if (!has_on_target_error) {
+        on_target_error = BLOCKDEV_ON_ERROR_REPORT;
     }
     if (!has_granularity) {
         granularity = 0;
     }
     if (!has_buf_size) {
-        buf_size = DEFAULT_MIRROR_BUF_SIZE;
+        buf_size = 0;
+    }
+    if (!has_unmap) {
+        unmap = true;
     }
 
     if (granularity != 0 && (granularity < 512 || granularity > 1048576 * 64)) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "granularity",
-                  "a value in range [512B, 64MB]");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "granularity",
+                   "a value in range [512B, 64MB]");
         return;
     }
     if (granularity & (granularity - 1)) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "granularity", "power of 2");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "granularity",
+                   "power of 2");
         return;
     }
 
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR_SOURCE, errp)) {
+        return;
+    }
+    if (bdrv_op_is_blocked(target, BLOCK_OP_TYPE_MIRROR_TARGET, errp)) {
         return;
     }
 
-    aio_context = bdrv_get_aio_context(bs);
+    if (!bs->backing && sync == MIRROR_SYNC_MODE_TOP) {
+        sync = MIRROR_SYNC_MODE_FULL;
+    }
+
+    /* pass the node name to replace to mirror start since it's loose coupling
+     * and will allow to check whether the node still exist at mirror completion
+     */
+    mirror_start(job_id, bs, target,
+                 has_replaces ? replaces : NULL,
+                 speed, granularity, buf_size, sync, backing_mode,
+                 on_source_error, on_target_error, unmap,
+                 block_job_cb, bs, errp);
+}
+
+void qmp_drive_mirror(DriveMirror *arg, Error **errp)
+{
+    BlockDriverState *bs;
+    BlockBackend *blk;
+    BlockDriverState *source, *target_bs;
+    AioContext *aio_context;
+    BlockMirrorBackingMode backing_mode;
+    Error *local_err = NULL;
+    QDict *options = NULL;
+    int flags;
+    int64_t size;
+    const char *format = arg->format;
+
+    blk = blk_by_name(arg->device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", arg->device);
+        return;
+    }
+
+    aio_context = blk_get_aio_context(blk);
     aio_context_acquire(aio_context);
 
-    if (!bdrv_is_inserted(bs)) {
-        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+    if (!blk_is_available(blk)) {
+        error_setg(errp, QERR_DEVICE_HAS_NO_MEDIUM, arg->device);
         goto out;
     }
-
-    if (!has_format) {
-        format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
-    }
-    if (format) {
-        drv = bdrv_find_format(format);
-        if (!drv) {
-            error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
-            goto out;
-        }
+    bs = blk_bs(blk);
+    if (!arg->has_mode) {
+        arg->mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     }
 
-    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR, errp)) {
-        goto out;
+    if (!arg->has_format) {
+        format = (arg->mode == NEW_IMAGE_MODE_EXISTING
+                  ? NULL : bs->drv->format_name);
     }
 
     flags = bs->open_flags | BDRV_O_RDWR;
-    source = bs->backing_hd;
-    if (!source && sync == MIRROR_SYNC_MODE_TOP) {
-        sync = MIRROR_SYNC_MODE_FULL;
+    source = backing_bs(bs);
+    if (!source && arg->sync == MIRROR_SYNC_MODE_TOP) {
+        arg->sync = MIRROR_SYNC_MODE_FULL;
     }
-    if (sync == MIRROR_SYNC_MODE_NONE) {
+    if (arg->sync == MIRROR_SYNC_MODE_NONE) {
         source = bs;
     }
 
@@ -2337,18 +3518,18 @@
         goto out;
     }
 
-    if (has_replaces) {
+    if (arg->has_replaces) {
         BlockDriverState *to_replace_bs;
         AioContext *replace_aio_context;
         int64_t replace_size;
 
-        if (!has_node_name) {
+        if (!arg->has_node_name) {
             error_setg(errp, "a node-name must be provided when replacing a"
                              " named node of the graph");
             goto out;
         }
 
-        to_replace_bs = check_to_replace_node(replaces, &local_err);
+        to_replace_bs = check_to_replace_node(bs, arg->replaces, &local_err);
 
         if (!to_replace_bs) {
             error_propagate(errp, local_err);
@@ -2367,20 +3548,26 @@
         }
     }
 
-    if ((sync == MIRROR_SYNC_MODE_FULL || !source)
-        && mode != NEW_IMAGE_MODE_EXISTING)
+    if (arg->mode == NEW_IMAGE_MODE_ABSOLUTE_PATHS) {
+        backing_mode = MIRROR_SOURCE_BACKING_CHAIN;
+    } else {
+        backing_mode = MIRROR_OPEN_BACKING_CHAIN;
+    }
+
+    if ((arg->sync == MIRROR_SYNC_MODE_FULL || !source)
+        && arg->mode != NEW_IMAGE_MODE_EXISTING)
     {
         /* create new image w/o backing file */
-        assert(format && drv);
-        bdrv_img_create(target, format,
+        assert(format);
+        bdrv_img_create(arg->target, format,
                         NULL, NULL, NULL, size, flags, &local_err, false);
     } else {
-        switch (mode) {
+        switch (arg->mode) {
         case NEW_IMAGE_MODE_EXISTING:
             break;
         case NEW_IMAGE_MODE_ABSOLUTE_PATHS:
             /* create new image with backing file */
-            bdrv_img_create(target, format,
+            bdrv_img_create(arg->target, format,
                             source->filename,
                             source->drv->format_name,
                             NULL, size, flags, &local_err, false);
@@ -2395,74 +3582,126 @@
         goto out;
     }
 
-    if (has_node_name) {
-        options = qdict_new();
-        qdict_put(options, "node-name", qstring_from_str(node_name));
+    options = qdict_new();
+    if (arg->has_node_name) {
+        qdict_put(options, "node-name", qstring_from_str(arg->node_name));
+    }
+    if (format) {
+        qdict_put(options, "driver", qstring_from_str(format));
     }
 
     /* Mirroring takes care of copy-on-write using the source's backing
      * file.
      */
-    target_bs = NULL;
-    ret = bdrv_open(&target_bs, target, NULL, options,
-                    flags | BDRV_O_NO_BACKING, drv, &local_err);
-    if (ret < 0) {
-        error_propagate(errp, local_err);
+    target_bs = bdrv_open(arg->target, NULL, options,
+                          flags | BDRV_O_NO_BACKING, errp);
+    if (!target_bs) {
         goto out;
     }
 
     bdrv_set_aio_context(target_bs, aio_context);
 
-    /* pass the node name to replace to mirror start since it's loose coupling
-     * and will allow to check whether the node still exist at mirror completion
-     */
-    mirror_start(bs, target_bs,
-                 has_replaces ? replaces : NULL,
-                 speed, granularity, buf_size, sync,
-                 on_source_error, on_target_error,
-                 block_job_cb, bs, &local_err);
-    if (local_err != NULL) {
-        bdrv_unref(target_bs);
-        error_propagate(errp, local_err);
-        goto out;
-    }
-
+    blockdev_mirror_common(arg->has_job_id ? arg->job_id : NULL, bs, target_bs,
+                           arg->has_replaces, arg->replaces, arg->sync,
+                           backing_mode, arg->has_speed, arg->speed,
+                           arg->has_granularity, arg->granularity,
+                           arg->has_buf_size, arg->buf_size,
+                           arg->has_on_source_error, arg->on_source_error,
+                           arg->has_on_target_error, arg->on_target_error,
+                           arg->has_unmap, arg->unmap,
+                           &local_err);
+    bdrv_unref(target_bs);
+    error_propagate(errp, local_err);
 out:
     aio_context_release(aio_context);
 }
 
-/* Get the block job for a given device name and acquire its AioContext */
-static BlockJob *find_block_job(const char *device, AioContext **aio_context)
+void qmp_blockdev_mirror(bool has_job_id, const char *job_id,
+                         const char *device, const char *target,
+                         bool has_replaces, const char *replaces,
+                         MirrorSyncMode sync,
+                         bool has_speed, int64_t speed,
+                         bool has_granularity, uint32_t granularity,
+                         bool has_buf_size, int64_t buf_size,
+                         bool has_on_source_error,
+                         BlockdevOnError on_source_error,
+                         bool has_on_target_error,
+                         BlockdevOnError on_target_error,
+                         Error **errp)
 {
     BlockDriverState *bs;
+    BlockBackend *blk;
+    BlockDriverState *target_bs;
+    AioContext *aio_context;
+    BlockMirrorBackingMode backing_mode = MIRROR_LEAVE_BACKING_CHAIN;
+    Error *local_err = NULL;
 
-    bs = bdrv_find(device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_setg(errp, "Device '%s' not found", device);
+        return;
+    }
+    bs = blk_bs(blk);
+
     if (!bs) {
-        goto notfound;
+        error_setg(errp, "Device '%s' has no media", device);
+        return;
     }
 
-    *aio_context = bdrv_get_aio_context(bs);
+    target_bs = bdrv_lookup_bs(target, target, errp);
+    if (!target_bs) {
+        return;
+    }
+
+    aio_context = bdrv_get_aio_context(bs);
+    aio_context_acquire(aio_context);
+
+    bdrv_set_aio_context(target_bs, aio_context);
+
+    blockdev_mirror_common(has_job_id ? job_id : NULL, bs, target_bs,
+                           has_replaces, replaces, sync, backing_mode,
+                           has_speed, speed,
+                           has_granularity, granularity,
+                           has_buf_size, buf_size,
+                           has_on_source_error, on_source_error,
+                           has_on_target_error, on_target_error,
+                           true, true,
+                           &local_err);
+    error_propagate(errp, local_err);
+
+    aio_context_release(aio_context);
+}
+
+/* Get a block job using its ID and acquire its AioContext */
+static BlockJob *find_block_job(const char *id, AioContext **aio_context,
+                                Error **errp)
+{
+    BlockJob *job;
+
+    assert(id != NULL);
+
+    *aio_context = NULL;
+
+    job = block_job_get(id);
+
+    if (!job) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_ACTIVE,
+                  "Block job '%s' not found", id);
+        return NULL;
+    }
+
+    *aio_context = blk_get_aio_context(job->blk);
     aio_context_acquire(*aio_context);
 
-    if (!bs->job) {
-        aio_context_release(*aio_context);
-        goto notfound;
-    }
-
-    return bs->job;
-
-notfound:
-    *aio_context = NULL;
-    return NULL;
+    return job;
 }
 
 void qmp_block_job_set_speed(const char *device, int64_t speed, Error **errp)
 {
     AioContext *aio_context;
-    BlockJob *job = find_block_job(device, &aio_context);
+    BlockJob *job = find_block_job(device, &aio_context, errp);
 
     if (!job) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
         return;
     }
 
@@ -2474,10 +3713,9 @@
                           bool has_force, bool force, Error **errp)
 {
     AioContext *aio_context;
-    BlockJob *job = find_block_job(device, &aio_context);
+    BlockJob *job = find_block_job(device, &aio_context, errp);
 
     if (!job) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
         return;
     }
 
@@ -2485,7 +3723,7 @@
         force = false;
     }
 
-    if (job->paused && !force) {
+    if (job->user_paused && !force) {
         error_setg(errp, "The block job for device '%s' is currently paused",
                    device);
         goto out;
@@ -2500,13 +3738,13 @@
 void qmp_block_job_pause(const char *device, Error **errp)
 {
     AioContext *aio_context;
-    BlockJob *job = find_block_job(device, &aio_context);
+    BlockJob *job = find_block_job(device, &aio_context, errp);
 
-    if (!job) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+    if (!job || job->user_paused) {
         return;
     }
 
+    job->user_paused = true;
     trace_qmp_block_job_pause(job);
     block_job_pause(job);
     aio_context_release(aio_context);
@@ -2515,14 +3753,15 @@
 void qmp_block_job_resume(const char *device, Error **errp)
 {
     AioContext *aio_context;
-    BlockJob *job = find_block_job(device, &aio_context);
+    BlockJob *job = find_block_job(device, &aio_context, errp);
 
-    if (!job) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+    if (!job || !job->user_paused) {
         return;
     }
 
+    job->user_paused = false;
     trace_qmp_block_job_resume(job);
+    block_job_iostatus_reset(job);
     block_job_resume(job);
     aio_context_release(aio_context);
 }
@@ -2530,10 +3769,9 @@
 void qmp_block_job_complete(const char *device, Error **errp)
 {
     AioContext *aio_context;
-    BlockJob *job = find_block_job(device, &aio_context);
+    BlockJob *job = find_block_job(device, &aio_context, errp);
 
     if (!job) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
         return;
     }
 
@@ -2547,48 +3785,59 @@
                              const char *backing_file,
                              Error **errp)
 {
+    BlockBackend *blk;
     BlockDriverState *bs = NULL;
+    AioContext *aio_context;
     BlockDriverState *image_bs = NULL;
     Error *local_err = NULL;
     bool ro;
     int open_flags;
     int ret;
 
-    /* find the top layer BDS of the chain */
-    bs = bdrv_find(device);
-    if (!bs) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+    blk = blk_by_name(device);
+    if (!blk) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
         return;
     }
 
+    aio_context = blk_get_aio_context(blk);
+    aio_context_acquire(aio_context);
+
+    if (!blk_is_available(blk)) {
+        error_setg(errp, "Device '%s' has no medium", device);
+        goto out;
+    }
+    bs = blk_bs(blk);
+
     image_bs = bdrv_lookup_bs(NULL, image_node_name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        goto out;
     }
 
     if (!image_bs) {
         error_setg(errp, "image file not found");
-        return;
+        goto out;
     }
 
     if (bdrv_find_base(image_bs) == image_bs) {
         error_setg(errp, "not allowing backing file change on an image "
                          "without a backing file");
-        return;
+        goto out;
     }
 
     /* even though we are not necessarily operating on bs, we need it to
      * determine if block ops are currently prohibited on the chain */
     if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_CHANGE, errp)) {
-        return;
+        goto out;
     }
 
     /* final sanity check */
     if (!bdrv_chain_contains(bs, image_bs)) {
         error_setg(errp, "'%s' and image file are not in the same chain",
                    device);
-        return;
+        goto out;
     }
 
     /* if not r/w, reopen to make r/w */
@@ -2599,7 +3848,7 @@
         bdrv_reopen(image_bs, open_flags | BDRV_O_RDWR, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
-            return;
+            goto out;
         }
     }
 
@@ -2615,26 +3864,53 @@
 
     if (ro) {
         bdrv_reopen(image_bs, open_flags, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err); /* will preserve prior errp */
-        }
+        error_propagate(errp, local_err);
     }
+
+out:
+    aio_context_release(aio_context);
+}
+
+void hmp_drive_add_node(Monitor *mon, const char *optstr)
+{
+    QemuOpts *opts;
+    QDict *qdict;
+    Error *local_err = NULL;
+
+    opts = qemu_opts_parse_noisily(&qemu_drive_opts, optstr, false);
+    if (!opts) {
+        return;
+    }
+
+    qdict = qemu_opts_to_qdict(opts, NULL);
+
+    if (!qdict_get_try_str(qdict, "node-name")) {
+        QDECREF(qdict);
+        error_report("'node-name' needs to be specified");
+        goto out;
+    }
+
+    BlockDriverState *bs = bds_tree_init(qdict, &local_err);
+    if (!bs) {
+        error_report_err(local_err);
+        goto out;
+    }
+
+    QTAILQ_INSERT_TAIL(&monitor_bdrv_states, bs, monitor_list);
+
+out:
+    qemu_opts_del(opts);
 }
 
 void qmp_blockdev_add(BlockdevOptions *options, Error **errp)
 {
-    QmpOutputVisitor *ov = qmp_output_visitor_new();
-    BlockBackend *blk;
+    BlockDriverState *bs;
+    BlockBackend *blk = NULL;
     QObject *obj;
+    Visitor *v = qmp_output_visitor_new(&obj);
     QDict *qdict;
     Error *local_err = NULL;
 
-    /* Require an ID in the top level */
-    if (!options->has_id) {
-        error_setg(errp, "Block device needs an ID");
-        goto fail;
-    }
-
     /* TODO Sort it out in raw-posix and drive_new(): Reject aio=native with
      * cache.direct=false instead of silently switching to aio=threads, except
      * when called from drive_new().
@@ -2650,52 +3926,206 @@
         }
     }
 
-    visit_type_BlockdevOptions(qmp_output_get_visitor(ov),
-                               &options, NULL, &local_err);
+    visit_type_BlockdevOptions(v, NULL, &options, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto fail;
     }
 
-    obj = qmp_output_get_qobject(ov);
+    visit_complete(v, &obj);
     qdict = qobject_to_qdict(obj);
 
     qdict_flatten(qdict);
 
-    blk = blockdev_init(NULL, qdict, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        goto fail;
+    if (options->has_id) {
+        blk = blockdev_init(NULL, qdict, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+
+        bs = blk_bs(blk);
+    } else {
+        if (!qdict_get_try_str(qdict, "node-name")) {
+            error_setg(errp, "'id' and/or 'node-name' need to be specified for "
+                       "the root node");
+            goto fail;
+        }
+
+        bs = bds_tree_init(qdict, errp);
+        if (!bs) {
+            goto fail;
+        }
+
+        QTAILQ_INSERT_TAIL(&monitor_bdrv_states, bs, monitor_list);
     }
 
-    if (bdrv_key_required(blk_bs(blk))) {
-        blk_unref(blk);
+    if (bs && bdrv_key_required(bs)) {
+        if (blk) {
+            monitor_remove_blk(blk);
+            blk_unref(blk);
+        } else {
+            QTAILQ_REMOVE(&monitor_bdrv_states, bs, monitor_list);
+            bdrv_unref(bs);
+        }
         error_setg(errp, "blockdev-add doesn't support encrypted devices");
         goto fail;
     }
 
 fail:
-    qmp_output_visitor_cleanup(ov);
+    visit_free(v);
+}
+
+void qmp_x_blockdev_del(bool has_id, const char *id,
+                        bool has_node_name, const char *node_name, Error **errp)
+{
+    AioContext *aio_context;
+    BlockBackend *blk;
+    BlockDriverState *bs;
+
+    if (has_id && has_node_name) {
+        error_setg(errp, "Only one of id and node-name must be specified");
+        return;
+    } else if (!has_id && !has_node_name) {
+        error_setg(errp, "No block device specified");
+        return;
+    }
+
+    if (has_id) {
+        /* blk_by_name() never returns a BB that is not owned by the monitor */
+        blk = blk_by_name(id);
+        if (!blk) {
+            error_setg(errp, "Cannot find block backend %s", id);
+            return;
+        }
+        if (blk_legacy_dinfo(blk)) {
+            error_setg(errp, "Deleting block backend added with drive-add"
+                       " is not supported");
+            return;
+        }
+        if (blk_get_refcnt(blk) > 1) {
+            error_setg(errp, "Block backend %s is in use", id);
+            return;
+        }
+        bs = blk_bs(blk);
+        aio_context = blk_get_aio_context(blk);
+    } else {
+        blk = NULL;
+        bs = bdrv_find_node(node_name);
+        if (!bs) {
+            error_setg(errp, "Cannot find node %s", node_name);
+            return;
+        }
+        if (bdrv_has_blk(bs)) {
+            error_setg(errp, "Node %s is in use by %s",
+                       node_name, bdrv_get_parent_name(bs));
+            return;
+        }
+        aio_context = bdrv_get_aio_context(bs);
+    }
+
+    aio_context_acquire(aio_context);
+
+    if (bs) {
+        if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, errp)) {
+            goto out;
+        }
+
+        if (!blk && !bs->monitor_list.tqe_prev) {
+            error_setg(errp, "Node %s is not owned by the monitor",
+                       bs->node_name);
+            goto out;
+        }
+
+        if (bs->refcnt > 1) {
+            error_setg(errp, "Block device %s is in use",
+                       bdrv_get_device_or_node_name(bs));
+            goto out;
+        }
+    }
+
+    if (blk) {
+        monitor_remove_blk(blk);
+        blk_unref(blk);
+    } else {
+        QTAILQ_REMOVE(&monitor_bdrv_states, bs, monitor_list);
+        bdrv_unref(bs);
+    }
+
+out:
+    aio_context_release(aio_context);
+}
+
+static BdrvChild *bdrv_find_child(BlockDriverState *parent_bs,
+                                  const char *child_name)
+{
+    BdrvChild *child;
+
+    QLIST_FOREACH(child, &parent_bs->children, next) {
+        if (strcmp(child->name, child_name) == 0) {
+            return child;
+        }
+    }
+
+    return NULL;
+}
+
+void qmp_x_blockdev_change(const char *parent, bool has_child,
+                           const char *child, bool has_node,
+                           const char *node, Error **errp)
+{
+    BlockDriverState *parent_bs, *new_bs = NULL;
+    BdrvChild *p_child;
+
+    parent_bs = bdrv_lookup_bs(parent, parent, errp);
+    if (!parent_bs) {
+        return;
+    }
+
+    if (has_child == has_node) {
+        if (has_child) {
+            error_setg(errp, "The parameters child and node are in conflict");
+        } else {
+            error_setg(errp, "Either child or node must be specified");
+        }
+        return;
+    }
+
+    if (has_child) {
+        p_child = bdrv_find_child(parent_bs, child);
+        if (!p_child) {
+            error_setg(errp, "Node '%s' does not have child '%s'",
+                       parent, child);
+            return;
+        }
+        bdrv_del_child(parent_bs, p_child, errp);
+    }
+
+    if (has_node) {
+        new_bs = bdrv_find_node(node);
+        if (!new_bs) {
+            error_setg(errp, "Node '%s' not found", node);
+            return;
+        }
+        bdrv_add_child(parent_bs, new_bs, errp);
+    }
 }
 
 BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 {
     BlockJobInfoList *head = NULL, **p_next = &head;
-    BlockDriverState *bs;
+    BlockJob *job;
 
-    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-        AioContext *aio_context = bdrv_get_aio_context(bs);
+    for (job = block_job_next(NULL); job; job = block_job_next(job)) {
+        BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
+        AioContext *aio_context = blk_get_aio_context(job->blk);
 
         aio_context_acquire(aio_context);
-
-        if (bs->job) {
-            BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
-            elem->value = block_job_query(bs->job);
-            *p_next = elem;
-            p_next = &elem->next;
-        }
-
+        elem->value = block_job_query(job);
         aio_context_release(aio_context);
+
+        *p_next = elem;
+        p_next = &elem->next;
     }
 
     return head;
@@ -2714,22 +4144,14 @@
             .type = QEMU_OPT_STRING,
             .help = "discard operation (ignore/off, unmap/on)",
         },{
-            .name = "cache.writeback",
-            .type = QEMU_OPT_BOOL,
-            .help = "enables writeback mode for any caches",
-        },{
-            .name = "cache.direct",
-            .type = QEMU_OPT_BOOL,
-            .help = "enables use of O_DIRECT (bypass the host page cache)",
-        },{
-            .name = "cache.no-flush",
-            .type = QEMU_OPT_BOOL,
-            .help = "ignore any flush requests for the device",
-        },{
             .name = "aio",
             .type = QEMU_OPT_STRING,
             .help = "host AIO implementation (threads, native)",
         },{
+            .name = BDRV_OPT_CACHE_WB,
+            .type = QEMU_OPT_BOOL,
+            .help = "Enable writeback mode",
+        },{
             .name = "format",
             .type = QEMU_OPT_STRING,
             .help = "disk format (raw, qcow2, ...)",
@@ -2794,10 +4216,77 @@
             .type = QEMU_OPT_NUMBER,
             .help = "total bytes write burst",
         },{
+            .name = "throttling.iops-total-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the iops-total-max burst period, in seconds",
+        },{
+            .name = "throttling.iops-read-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the iops-read-max burst period, in seconds",
+        },{
+            .name = "throttling.iops-write-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the iops-write-max burst period, in seconds",
+        },{
+            .name = "throttling.bps-total-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the bps-total-max burst period, in seconds",
+        },{
+            .name = "throttling.bps-read-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the bps-read-max burst period, in seconds",
+        },{
+            .name = "throttling.bps-write-max-length",
+            .type = QEMU_OPT_NUMBER,
+            .help = "length of the bps-write-max burst period, in seconds",
+        },{
             .name = "throttling.iops-size",
             .type = QEMU_OPT_NUMBER,
             .help = "when limiting by iops max size of an I/O in bytes",
         },{
+            .name = "throttling.group",
+            .type = QEMU_OPT_STRING,
+            .help = "name of the block throttling group",
+        },{
+            .name = "copy-on-read",
+            .type = QEMU_OPT_BOOL,
+            .help = "copy read data from backing file into image file",
+        },{
+            .name = "detect-zeroes",
+            .type = QEMU_OPT_STRING,
+            .help = "try to optimize zero writes (off, on, unmap)",
+        },{
+            .name = "stats-account-invalid",
+            .type = QEMU_OPT_BOOL,
+            .help = "whether to account for invalid I/O operations "
+                    "in the statistics",
+        },{
+            .name = "stats-account-failed",
+            .type = QEMU_OPT_BOOL,
+            .help = "whether to account for failed I/O operations "
+                    "in the statistics",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_root_bds_opts = {
+    .name = "root-bds",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_root_bds_opts.head),
+    .desc = {
+        {
+            .name = "discard",
+            .type = QEMU_OPT_STRING,
+            .help = "discard operation (ignore/off, unmap/on)",
+        },{
+            .name = "aio",
+            .type = QEMU_OPT_STRING,
+            .help = "host AIO implementation (threads, native)",
+        },{
+            .name = "read-only",
+            .type = QEMU_OPT_BOOL,
+            .help = "open drive file as read-only",
+        },{
             .name = "copy-on-read",
             .type = QEMU_OPT_BOOL,
             .help = "copy read data from backing file into image file",
diff --git a/blockjob.c b/blockjob.c
index ba2255d..a5ba3be 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -23,29 +23,130 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "trace.h"
 #include "block/block.h"
 #include "block/blockjob.h"
 #include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qjson.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
+#include "qemu/id.h"
 #include "qmp-commands.h"
 #include "qemu/timer.h"
 #include "qapi-event.h"
 
-void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
-                       int64_t speed, BlockCompletionFunc *cb,
-                       void *opaque, Error **errp)
+/* Transactional group of block jobs */
+struct BlockJobTxn {
+
+    /* Is this txn being cancelled? */
+    bool aborting;
+
+    /* List of jobs */
+    QLIST_HEAD(, BlockJob) jobs;
+
+    /* Reference count */
+    int refcnt;
+};
+
+static QLIST_HEAD(, BlockJob) block_jobs = QLIST_HEAD_INITIALIZER(block_jobs);
+
+BlockJob *block_job_next(BlockJob *job)
+{
+    if (!job) {
+        return QLIST_FIRST(&block_jobs);
+    }
+    return QLIST_NEXT(job, job_list);
+}
+
+BlockJob *block_job_get(const char *id)
 {
     BlockJob *job;
 
+    QLIST_FOREACH(job, &block_jobs, job_list) {
+        if (!strcmp(id, job->id)) {
+            return job;
+        }
+    }
+
+    return NULL;
+}
+
+/* Normally the job runs in its BlockBackend's AioContext.  The exception is
+ * block_job_defer_to_main_loop() where it runs in the QEMU main loop.  Code
+ * that supports both cases uses this helper function.
+ */
+static AioContext *block_job_get_aio_context(BlockJob *job)
+{
+    return job->deferred_to_main_loop ?
+           qemu_get_aio_context() :
+           blk_get_aio_context(job->blk);
+}
+
+static void block_job_attached_aio_context(AioContext *new_context,
+                                           void *opaque)
+{
+    BlockJob *job = opaque;
+
+    if (job->driver->attached_aio_context) {
+        job->driver->attached_aio_context(job, new_context);
+    }
+
+    block_job_resume(job);
+}
+
+static void block_job_detach_aio_context(void *opaque)
+{
+    BlockJob *job = opaque;
+
+    /* In case the job terminates during aio_poll()... */
+    block_job_ref(job);
+
+    block_job_pause(job);
+
+    if (!job->paused) {
+        /* If job is !job->busy this kicks it into the next pause point. */
+        block_job_enter(job);
+    }
+    while (!job->paused && !job->completed) {
+        aio_poll(block_job_get_aio_context(job), true);
+    }
+
+    block_job_unref(job);
+}
+
+void *block_job_create(const char *job_id, const BlockJobDriver *driver,
+                       BlockDriverState *bs, int64_t speed,
+                       BlockCompletionFunc *cb, void *opaque, Error **errp)
+{
+    BlockBackend *blk;
+    BlockJob *job;
+
+    assert(cb);
     if (bs->job) {
-        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
+        error_setg(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
         return NULL;
     }
-    bdrv_ref(bs);
+
+    if (job_id == NULL) {
+        job_id = bdrv_get_device_name(bs);
+    }
+
+    if (!id_wellformed(job_id)) {
+        error_setg(errp, "Invalid job ID '%s'", job_id);
+        return NULL;
+    }
+
+    if (block_job_get(job_id)) {
+        error_setg(errp, "Job ID '%s' already in use", job_id);
+        return NULL;
+    }
+
+    blk = blk_new();
+    blk_insert_bs(blk, bs);
+
     job = g_malloc0(driver->instance_size);
     error_setg(&job->blocker, "block device is in use by block job: %s",
                BlockJobType_lookup[driver->job_type]);
@@ -53,22 +154,26 @@
     bdrv_op_unblock(bs, BLOCK_OP_TYPE_DATAPLANE, job->blocker);
 
     job->driver        = driver;
-    job->bs            = bs;
+    job->id            = g_strdup(job_id);
+    job->blk           = blk;
     job->cb            = cb;
     job->opaque        = opaque;
     job->busy          = true;
+    job->refcnt        = 1;
     bs->job = job;
 
+    QLIST_INSERT_HEAD(&block_jobs, job, job_list);
+
+    blk_add_aio_context_notifier(blk, block_job_attached_aio_context,
+                                 block_job_detach_aio_context, job);
+
     /* Only set speed when necessary to avoid NotSupported error */
     if (speed != 0) {
         Error *local_err = NULL;
 
         block_job_set_speed(job, speed, &local_err);
         if (local_err) {
-            bs->job = NULL;
-            bdrv_op_unblock_all(bs, job->blocker);
-            error_free(job->blocker);
-            g_free(job);
+            block_job_unref(job);
             error_propagate(errp, local_err);
             return NULL;
         }
@@ -76,16 +181,121 @@
     return job;
 }
 
+void block_job_ref(BlockJob *job)
+{
+    ++job->refcnt;
+}
+
+void block_job_unref(BlockJob *job)
+{
+    if (--job->refcnt == 0) {
+        BlockDriverState *bs = blk_bs(job->blk);
+        bs->job = NULL;
+        bdrv_op_unblock_all(bs, job->blocker);
+        blk_remove_aio_context_notifier(job->blk,
+                                        block_job_attached_aio_context,
+                                        block_job_detach_aio_context, job);
+        blk_unref(job->blk);
+        error_free(job->blocker);
+        g_free(job->id);
+        QLIST_REMOVE(job, job_list);
+        g_free(job);
+    }
+}
+
+static void block_job_completed_single(BlockJob *job)
+{
+    if (!job->ret) {
+        if (job->driver->commit) {
+            job->driver->commit(job);
+        }
+    } else {
+        if (job->driver->abort) {
+            job->driver->abort(job);
+        }
+    }
+    job->cb(job->opaque, job->ret);
+    if (job->txn) {
+        block_job_txn_unref(job->txn);
+    }
+    block_job_unref(job);
+}
+
+static void block_job_completed_txn_abort(BlockJob *job)
+{
+    AioContext *ctx;
+    BlockJobTxn *txn = job->txn;
+    BlockJob *other_job, *next;
+
+    if (txn->aborting) {
+        /*
+         * We are cancelled by another job, which will handle everything.
+         */
+        return;
+    }
+    txn->aborting = true;
+    /* We are the first failed job. Cancel other jobs. */
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        ctx = blk_get_aio_context(other_job->blk);
+        aio_context_acquire(ctx);
+    }
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        if (other_job == job || other_job->completed) {
+            /* Other jobs are "effectively" cancelled by us, set the status for
+             * them; this job, however, may or may not be cancelled, depending
+             * on the caller, so leave it. */
+            if (other_job != job) {
+                other_job->cancelled = true;
+            }
+            continue;
+        }
+        block_job_cancel_sync(other_job);
+        assert(other_job->completed);
+    }
+    QLIST_FOREACH_SAFE(other_job, &txn->jobs, txn_list, next) {
+        ctx = blk_get_aio_context(other_job->blk);
+        block_job_completed_single(other_job);
+        aio_context_release(ctx);
+    }
+}
+
+static void block_job_completed_txn_success(BlockJob *job)
+{
+    AioContext *ctx;
+    BlockJobTxn *txn = job->txn;
+    BlockJob *other_job, *next;
+    /*
+     * Successful completion, see if there are other running jobs in this
+     * txn.
+     */
+    QLIST_FOREACH(other_job, &txn->jobs, txn_list) {
+        if (!other_job->completed) {
+            return;
+        }
+    }
+    /* We are the last completed job, commit the transaction. */
+    QLIST_FOREACH_SAFE(other_job, &txn->jobs, txn_list, next) {
+        ctx = blk_get_aio_context(other_job->blk);
+        aio_context_acquire(ctx);
+        assert(other_job->ret == 0);
+        block_job_completed_single(other_job);
+        aio_context_release(ctx);
+    }
+}
+
 void block_job_completed(BlockJob *job, int ret)
 {
-    BlockDriverState *bs = job->bs;
-
-    assert(bs->job == job);
-    job->cb(job->opaque, ret);
-    bs->job = NULL;
-    bdrv_op_unblock_all(bs, job->blocker);
-    error_free(job->blocker);
-    g_free(job);
+    assert(blk_bs(job->blk)->job == job);
+    assert(!job->completed);
+    job->completed = true;
+    job->ret = ret;
+    if (!job->txn) {
+        block_job_completed_single(job);
+    } else if (ret < 0 || block_job_is_cancelled(job)) {
+        block_job_completed_txn_abort(job);
+    } else {
+        block_job_completed_txn_success(job);
+    }
 }
 
 void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -93,7 +303,7 @@
     Error *local_err = NULL;
 
     if (!job->driver->set_speed) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         return;
     }
     job->driver->set_speed(job, speed, &local_err);
@@ -107,9 +317,9 @@
 
 void block_job_complete(BlockJob *job, Error **errp)
 {
-    if (job->paused || job->cancelled || !job->driver->complete) {
-        error_set(errp, QERR_BLOCK_JOB_NOT_READY,
-                  bdrv_get_device_name(job->bs));
+    if (job->pause_count || job->cancelled || !job->driver->complete) {
+        error_setg(errp, "The active block job '%s' cannot be completed",
+                   job->id);
         return;
     }
 
@@ -118,27 +328,62 @@
 
 void block_job_pause(BlockJob *job)
 {
-    job->paused = true;
+    job->pause_count++;
 }
 
-bool block_job_is_paused(BlockJob *job)
+static bool block_job_should_pause(BlockJob *job)
 {
-    return job->paused;
+    return job->pause_count > 0;
+}
+
+void coroutine_fn block_job_pause_point(BlockJob *job)
+{
+    if (!block_job_should_pause(job)) {
+        return;
+    }
+    if (block_job_is_cancelled(job)) {
+        return;
+    }
+
+    if (job->driver->pause) {
+        job->driver->pause(job);
+    }
+
+    if (block_job_should_pause(job) && !block_job_is_cancelled(job)) {
+        job->paused = true;
+        job->busy = false;
+        qemu_coroutine_yield(); /* wait for block_job_resume() */
+        job->busy = true;
+        job->paused = false;
+    }
+
+    if (job->driver->resume) {
+        job->driver->resume(job);
+    }
 }
 
 void block_job_resume(BlockJob *job)
 {
-    job->paused = false;
-    block_job_iostatus_reset(job);
+    assert(job->pause_count > 0);
+    job->pause_count--;
+    if (job->pause_count) {
+        return;
+    }
+    block_job_enter(job);
+}
+
+void block_job_enter(BlockJob *job)
+{
     if (job->co && !job->busy) {
-        qemu_coroutine_enter(job->co, NULL);
+        qemu_coroutine_enter(job->co);
     }
 }
 
 void block_job_cancel(BlockJob *job)
 {
     job->cancelled = true;
-    block_job_resume(job);
+    block_job_iostatus_reset(job);
+    block_job_enter(job);
 }
 
 bool block_job_is_cancelled(BlockJob *job)
@@ -154,51 +399,28 @@
     }
 }
 
-struct BlockFinishData {
-    BlockJob *job;
-    BlockCompletionFunc *cb;
-    void *opaque;
-    bool cancelled;
-    int ret;
-};
-
-static void block_job_finish_cb(void *opaque, int ret)
-{
-    struct BlockFinishData *data = opaque;
-
-    data->cancelled = block_job_is_cancelled(data->job);
-    data->ret = ret;
-    data->cb(data->opaque, ret);
-}
-
 static int block_job_finish_sync(BlockJob *job,
                                  void (*finish)(BlockJob *, Error **errp),
                                  Error **errp)
 {
-    struct BlockFinishData data;
-    BlockDriverState *bs = job->bs;
     Error *local_err = NULL;
+    int ret;
 
-    assert(bs->job == job);
+    assert(blk_bs(job->blk)->job == job);
 
-    /* Set up our own callback to store the result and chain to
-     * the original callback.
-     */
-    data.job = job;
-    data.cb = job->cb;
-    data.opaque = job->opaque;
-    data.ret = -EINPROGRESS;
-    job->cb = block_job_finish_cb;
-    job->opaque = &data;
+    block_job_ref(job);
     finish(job, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
+        block_job_unref(job);
         return -EBUSY;
     }
-    while (data.ret == -EINPROGRESS) {
-        aio_poll(bdrv_get_aio_context(bs), true);
+    while (!job->completed) {
+        aio_poll(block_job_get_aio_context(job), true);
     }
-    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
+    ret = (job->cancelled && job->ret == 0) ? -ECANCELED : job->ret;
+    block_job_unref(job);
+    return ret;
 }
 
 /* A wrapper around block_job_cancel() taking an Error ** parameter so it may be
@@ -214,6 +436,19 @@
     return block_job_finish_sync(job, &block_job_cancel_err, NULL);
 }
 
+void block_job_cancel_sync_all(void)
+{
+    BlockJob *job;
+    AioContext *aio_context;
+
+    while ((job = QLIST_FIRST(&block_jobs))) {
+        aio_context = blk_get_aio_context(job->blk);
+        aio_context_acquire(aio_context);
+        block_job_cancel_sync(job);
+        aio_context_release(aio_context);
+    }
+}
+
 int block_job_complete_sync(BlockJob *job, Error **errp)
 {
     return block_job_finish_sync(job, &block_job_complete, errp);
@@ -229,12 +464,12 @@
     }
 
     job->busy = false;
-    if (block_job_is_paused(job)) {
-        qemu_coroutine_yield();
-    } else {
-        co_aio_sleep_ns(bdrv_get_aio_context(job->bs), type, ns);
+    if (!block_job_should_pause(job)) {
+        co_aio_sleep_ns(blk_get_aio_context(job->blk), type, ns);
     }
     job->busy = true;
+
+    block_job_pause_point(job);
 }
 
 void block_job_yield(BlockJob *job)
@@ -247,18 +482,22 @@
     }
 
     job->busy = false;
-    qemu_coroutine_yield();
+    if (!block_job_should_pause(job)) {
+        qemu_coroutine_yield();
+    }
     job->busy = true;
+
+    block_job_pause_point(job);
 }
 
 BlockJobInfo *block_job_query(BlockJob *job)
 {
     BlockJobInfo *info = g_new0(BlockJobInfo, 1);
     info->type      = g_strdup(BlockJobType_lookup[job->driver->job_type]);
-    info->device    = g_strdup(bdrv_get_device_name(job->bs));
+    info->device    = g_strdup(job->id);
     info->len       = job->len;
     info->busy      = job->busy;
-    info->paused    = job->paused;
+    info->paused    = job->pause_count > 0;
     info->offset    = job->offset;
     info->speed     = job->speed;
     info->io_status = job->iostatus;
@@ -277,7 +516,7 @@
 void block_job_event_cancelled(BlockJob *job)
 {
     qapi_event_send_block_job_cancelled(job->driver->job_type,
-                                        bdrv_get_device_name(job->bs),
+                                        job->id,
                                         job->len,
                                         job->offset,
                                         job->speed,
@@ -287,7 +526,7 @@
 void block_job_event_completed(BlockJob *job, const char *msg)
 {
     qapi_event_send_block_job_completed(job->driver->job_type,
-                                        bdrv_get_device_name(job->bs),
+                                        job->id,
                                         job->len,
                                         job->offset,
                                         job->speed,
@@ -301,20 +540,20 @@
     job->ready = true;
 
     qapi_event_send_block_job_ready(job->driver->job_type,
-                                    bdrv_get_device_name(job->bs),
+                                    job->id,
                                     job->len,
                                     job->offset,
                                     job->speed, &error_abort);
 }
 
-BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
-                                        BlockdevOnError on_err,
+BlockErrorAction block_job_error_action(BlockJob *job, BlockdevOnError on_err,
                                         int is_read, int error)
 {
     BlockErrorAction action;
 
     switch (on_err) {
     case BLOCKDEV_ON_ERROR_ENOSPC:
+    case BLOCKDEV_ON_ERROR_AUTO:
         action = (error == ENOSPC) ?
                  BLOCK_ERROR_ACTION_STOP : BLOCK_ERROR_ACTION_REPORT;
         break;
@@ -330,16 +569,15 @@
     default:
         abort();
     }
-    qapi_event_send_block_job_error(bdrv_get_device_name(job->bs),
+    qapi_event_send_block_job_error(job->id,
                                     is_read ? IO_OPERATION_TYPE_READ :
                                     IO_OPERATION_TYPE_WRITE,
                                     action, &error_abort);
     if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* make the pause user visible, which will be resumed from QMP. */
+        job->user_paused = true;
         block_job_pause(job);
         block_job_iostatus_set_err(job, error);
-        if (bs != job->bs) {
-            bdrv_iostatus_set_err(bs, error);
-        }
     }
     return action;
 }
@@ -363,9 +601,10 @@
     aio_context_acquire(data->aio_context);
 
     /* Fetch BDS AioContext again, in case it has changed */
-    aio_context = bdrv_get_aio_context(data->job->bs);
+    aio_context = blk_get_aio_context(data->job->blk);
     aio_context_acquire(aio_context);
 
+    data->job->deferred_to_main_loop = false;
     data->fn(data->job, data->opaque);
 
     aio_context_release(aio_context);
@@ -382,9 +621,43 @@
     BlockJobDeferToMainLoopData *data = g_malloc(sizeof(*data));
     data->job = job;
     data->bh = qemu_bh_new(block_job_defer_to_main_loop_bh, data);
-    data->aio_context = bdrv_get_aio_context(job->bs);
+    data->aio_context = blk_get_aio_context(job->blk);
     data->fn = fn;
     data->opaque = opaque;
+    job->deferred_to_main_loop = true;
 
     qemu_bh_schedule(data->bh);
 }
+
+BlockJobTxn *block_job_txn_new(void)
+{
+    BlockJobTxn *txn = g_new0(BlockJobTxn, 1);
+    QLIST_INIT(&txn->jobs);
+    txn->refcnt = 1;
+    return txn;
+}
+
+static void block_job_txn_ref(BlockJobTxn *txn)
+{
+    txn->refcnt++;
+}
+
+void block_job_txn_unref(BlockJobTxn *txn)
+{
+    if (txn && --txn->refcnt == 0) {
+        g_free(txn);
+    }
+}
+
+void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job)
+{
+    if (!txn) {
+        return;
+    }
+
+    assert(!job->txn);
+    job->txn = txn;
+
+    QLIST_INSERT_HEAD(&txn->jobs, job, txn_list);
+    block_job_txn_ref(txn);
+}
diff --git a/bootdevice.c b/bootdevice.c
index b29970c..33e3029 100644
--- a/bootdevice.c
+++ b/bootdevice.c
@@ -1,7 +1,7 @@
 /*
  * QEMU Boot Device Implement
  *
- * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO., LTD.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,9 +22,13 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "qapi/visitor.h"
 #include "qemu/error-report.h"
+#include "hw/hw.h"
+#include "hw/qdev-core.h"
 
 typedef struct FWBootEntry FWBootEntry;
 
@@ -37,6 +41,80 @@
 
 static QTAILQ_HEAD(, FWBootEntry) fw_boot_order =
     QTAILQ_HEAD_INITIALIZER(fw_boot_order);
+static QEMUBootSetHandler *boot_set_handler;
+static void *boot_set_opaque;
+
+void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque)
+{
+    boot_set_handler = func;
+    boot_set_opaque = opaque;
+}
+
+void qemu_boot_set(const char *boot_order, Error **errp)
+{
+    Error *local_err = NULL;
+
+    if (!boot_set_handler) {
+        error_setg(errp, "no function defined to set boot device list for"
+                         " this architecture");
+        return;
+    }
+
+    validate_bootdevices(boot_order, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    boot_set_handler(boot_set_opaque, boot_order, errp);
+}
+
+void validate_bootdevices(const char *devices, Error **errp)
+{
+    /* We just do some generic consistency checks */
+    const char *p;
+    int bitmap = 0;
+
+    for (p = devices; *p != '\0'; p++) {
+        /* Allowed boot devices are:
+         * a-b: floppy disk drives
+         * c-f: IDE disk drives
+         * g-m: machine implementation dependent drives
+         * n-p: network devices
+         * It's up to each machine implementation to check if the given boot
+         * devices match the actual hardware implementation and firmware
+         * features.
+         */
+        if (*p < 'a' || *p > 'p') {
+            error_setg(errp, "Invalid boot device '%c'", *p);
+            return;
+        }
+        if (bitmap & (1 << (*p - 'a'))) {
+            error_setg(errp, "Boot device '%c' was given twice", *p);
+            return;
+        }
+        bitmap |= 1 << (*p - 'a');
+    }
+}
+
+void restore_boot_order(void *opaque)
+{
+    char *normal_boot_order = opaque;
+    static int first = 1;
+
+    /* Restore boot order and remove ourselves after the first boot */
+    if (first) {
+        first = 0;
+        return;
+    }
+
+    if (boot_set_handler) {
+        qemu_boot_set(normal_boot_order, &error_abort);
+    }
+
+    qemu_unregister_reset(restore_boot_order, normal_boot_order);
+    g_free(normal_boot_order);
+}
 
 void check_boot_index(int32_t bootindex, Error **errp)
 {
@@ -137,7 +215,9 @@
     char *list = NULL;
 
     QTAILQ_FOREACH(i, &fw_boot_order, link) {
-        char *devpath = NULL, *bootpath;
+        char *devpath = NULL,  *suffix = NULL;
+        char *bootpath;
+        char *d;
         size_t len;
 
         if (i->dev) {
@@ -145,21 +225,27 @@
             assert(devpath);
         }
 
-        if (i->suffix && !ignore_suffixes && devpath) {
-            size_t bootpathlen = strlen(devpath) + strlen(i->suffix) + 1;
-
-            bootpath = g_malloc(bootpathlen);
-            snprintf(bootpath, bootpathlen, "%s%s", devpath, i->suffix);
-            g_free(devpath);
-        } else if (devpath) {
-            bootpath = devpath;
-        } else if (!ignore_suffixes) {
-            assert(i->suffix);
-            bootpath = g_strdup(i->suffix);
-        } else {
-            bootpath = g_strdup("");
+        if (!ignore_suffixes) {
+            if (i->dev) {
+                d = qdev_get_own_fw_dev_path_from_handler(i->dev->parent_bus,
+                                                          i->dev);
+                if (d) {
+                    assert(!i->suffix);
+                    suffix = d;
+                } else {
+                    suffix = g_strdup(i->suffix);
+                }
+            } else {
+                suffix = g_strdup(i->suffix);
+            }
         }
 
+        bootpath = g_strdup_printf("%s%s",
+                                   devpath ? devpath : "",
+                                   suffix ? suffix : "");
+        g_free(devpath);
+        g_free(suffix);
+
         if (total) {
             list[total-1] = '\n';
         }
@@ -187,21 +273,21 @@
     DeviceState *dev;
 } BootIndexProperty;
 
-static void device_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void device_get_bootindex(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     BootIndexProperty *prop = opaque;
-    visit_type_int32(v, prop->bootindex, name, errp);
+    visit_type_int32(v, name, prop->bootindex, errp);
 }
 
-static void device_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void device_set_bootindex(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     BootIndexProperty *prop = opaque;
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
@@ -216,9 +302,7 @@
     add_boot_device_path(*prop->bootindex, prop->dev, prop->suffix);
 
 out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
 static void property_release_bootindex(Object *obj, const char *name,
diff --git a/bsd-user/bsdload.c b/bsd-user/bsdload.c
index 2abc713..94eec36 100644
--- a/bsd-user/bsdload.c
+++ b/bsd-user/bsdload.c
@@ -1,12 +1,6 @@
 /* Code for loading BSD executables.  Mostly linux kernel code.  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
diff --git a/bsd-user/elfload.c b/bsd-user/elfload.c
index 93fd9e4..41a1309 100644
--- a/bsd-user/elfload.c
+++ b/bsd-user/elfload.c
@@ -1,16 +1,10 @@
 /* This is the Linux kernel elf-loading code, ported into user space */
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include <stdlib.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "disas/disas.h"
+#include "qemu/path.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
@@ -351,8 +345,10 @@
 
     _regs->gpr[1] = infop->start_stack;
 #if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
-    entry = ldq_raw(infop->entry) + infop->load_addr;
-    toc = ldq_raw(infop->entry + 8) + infop->load_addr;
+    get_user_u64(entry, infop->entry);
+    entry += infop->load_addr;
+    get_user_u64(toc, infop->entry + 8);
+    toc += infop->load_addr;
     _regs->gpr[2] = toc;
     infop->entry = entry;
 #endif
@@ -365,8 +361,9 @@
     get_user_ual(_regs->gpr[3], pos);
     pos += sizeof(abi_ulong);
     _regs->gpr[4] = pos;
-    for (tmp = 1; tmp != 0; pos += sizeof(abi_ulong))
-        tmp = ldl(pos);
+    for (tmp = 1; tmp != 0; pos += sizeof(abi_ulong)) {
+        get_user_ual(tmp, pos);
+    }
     _regs->gpr[5] = pos;
 }
 
@@ -737,8 +734,7 @@
            size must be known */
         if (qemu_real_host_page_size < qemu_host_page_size) {
             abi_ulong end_addr, end_addr1;
-            end_addr1 = (elf_bss + qemu_real_host_page_size - 1) &
-                ~(qemu_real_host_page_size - 1);
+            end_addr1 = REAL_HOST_PAGE_ALIGN(elf_bss);
             end_addr = HOST_PAGE_ALIGN(elf_bss);
             if (end_addr1 < end_addr) {
                 mmap((void *)g2h(end_addr1), end_addr - end_addr1,
@@ -1352,9 +1348,7 @@
             }
         }
         if (!bprm->p) {
-            if (elf_interpreter) {
-                free(elf_interpreter);
-            }
+            free(elf_interpreter);
             free (elf_phdata);
             close(bprm->fd);
             return -E2BIG;
@@ -1368,7 +1362,6 @@
     info->mmap = 0;
     elf_entry = (abi_ulong) elf_ex.e_entry;
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * In case where user has not explicitly set the guest_base, we
      * probe here that should we set it automatically.
@@ -1389,7 +1382,6 @@
             }
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     /* Do this so that we can load the interpreter, if need be.  We will
        change some of these later */
diff --git a/bsd-user/i386/syscall.h b/bsd-user/i386/syscall.h
deleted file mode 100644
index 9b34c61..0000000
--- a/bsd-user/i386/syscall.h
+++ /dev/null
@@ -1,161 +0,0 @@
-/* default linux values for the selectors */
-#define __USER_CS	(0x23)
-#define __USER_DS	(0x2B)
-
-struct target_pt_regs {
-	long ebx;
-	long ecx;
-	long edx;
-	long esi;
-	long edi;
-	long ebp;
-	long eax;
-	int  xds;
-	int  xes;
-	long orig_eax;
-	long eip;
-	int  xcs;
-	long eflags;
-	long esp;
-	int  xss;
-};
-
-/* ioctls */
-
-#define TARGET_LDT_ENTRIES      8192
-#define TARGET_LDT_ENTRY_SIZE	8
-
-#define TARGET_GDT_ENTRIES             9
-#define TARGET_GDT_ENTRY_TLS_ENTRIES   3
-#define TARGET_GDT_ENTRY_TLS_MIN       6
-#define TARGET_GDT_ENTRY_TLS_MAX       (TARGET_GDT_ENTRY_TLS_MIN + TARGET_GDT_ENTRY_TLS_ENTRIES - 1)
-
-struct target_modify_ldt_ldt_s {
-    unsigned int  entry_number;
-    abi_ulong base_addr;
-    unsigned int limit;
-    unsigned int flags;
-};
-
-/* vm86 defines */
-
-#define TARGET_BIOSSEG		0x0f000
-
-#define TARGET_CPU_086		0
-#define TARGET_CPU_186		1
-#define TARGET_CPU_286		2
-#define TARGET_CPU_386		3
-#define TARGET_CPU_486		4
-#define TARGET_CPU_586		5
-
-#define TARGET_VM86_SIGNAL	0	/* return due to signal */
-#define TARGET_VM86_UNKNOWN	1	/* unhandled GP fault - IO-instruction or similar */
-#define TARGET_VM86_INTx	2	/* int3/int x instruction (ARG = x) */
-#define TARGET_VM86_STI	3	/* sti/popf/iret instruction enabled virtual interrupts */
-
-/*
- * Additional return values when invoking new vm86()
- */
-#define TARGET_VM86_PICRETURN	4	/* return due to pending PIC request */
-#define TARGET_VM86_TRAP	6	/* return due to DOS-debugger request */
-
-/*
- * function codes when invoking new vm86()
- */
-#define TARGET_VM86_PLUS_INSTALL_CHECK	0
-#define TARGET_VM86_ENTER		1
-#define TARGET_VM86_ENTER_NO_BYPASS	2
-#define	TARGET_VM86_REQUEST_IRQ	3
-#define TARGET_VM86_FREE_IRQ		4
-#define TARGET_VM86_GET_IRQ_BITS	5
-#define TARGET_VM86_GET_AND_RESET_IRQ	6
-
-/*
- * This is the stack-layout seen by the user space program when we have
- * done a translation of "SAVE_ALL" from vm86 mode. The real kernel layout
- * is 'kernel_vm86_regs' (see below).
- */
-
-struct target_vm86_regs {
-/*
- * normal regs, with special meaning for the segment descriptors..
- */
-	abi_long ebx;
-	abi_long ecx;
-	abi_long edx;
-	abi_long esi;
-	abi_long edi;
-	abi_long ebp;
-	abi_long eax;
-	abi_long __null_ds;
-	abi_long __null_es;
-	abi_long __null_fs;
-	abi_long __null_gs;
-	abi_long orig_eax;
-	abi_long eip;
-	unsigned short cs, __csh;
-	abi_long eflags;
-	abi_long esp;
-	unsigned short ss, __ssh;
-/*
- * these are specific to v86 mode:
- */
-	unsigned short es, __esh;
-	unsigned short ds, __dsh;
-	unsigned short fs, __fsh;
-	unsigned short gs, __gsh;
-};
-
-struct target_revectored_struct {
-	abi_ulong __map[8];			/* 256 bits */
-};
-
-struct target_vm86_struct {
-	struct target_vm86_regs regs;
-	abi_ulong flags;
-	abi_ulong screen_bitmap;
-	abi_ulong cpu_type;
-	struct target_revectored_struct int_revectored;
-	struct target_revectored_struct int21_revectored;
-};
-
-/*
- * flags masks
- */
-#define TARGET_VM86_SCREEN_BITMAP	0x0001
-
-struct target_vm86plus_info_struct {
-        abi_ulong flags;
-#define TARGET_force_return_for_pic (1 << 0)
-#define TARGET_vm86dbg_active       (1 << 1)  /* for debugger */
-#define TARGET_vm86dbg_TFpendig     (1 << 2)  /* for debugger */
-#define TARGET_is_vm86pus           (1 << 31) /* for vm86 internal use */
-	unsigned char vm86dbg_intxxtab[32];   /* for debugger */
-};
-
-struct target_vm86plus_struct {
-	struct target_vm86_regs regs;
-	abi_ulong flags;
-	abi_ulong screen_bitmap;
-	abi_ulong cpu_type;
-	struct target_revectored_struct int_revectored;
-	struct target_revectored_struct int21_revectored;
-	struct target_vm86plus_info_struct vm86plus;
-};
-
-/* FreeBSD sysarch(2) */
-#define TARGET_FREEBSD_I386_GET_LDT	0
-#define TARGET_FREEBSD_I386_SET_LDT	1
-				/* I386_IOPL */
-#define TARGET_FREEBSD_I386_GET_IOPERM	3
-#define TARGET_FREEBSD_I386_SET_IOPERM	4
-				/* xxxxx */
-#define TARGET_FREEBSD_I386_VM86	6
-#define TARGET_FREEBSD_I386_GET_FSBASE	7
-#define TARGET_FREEBSD_I386_SET_FSBASE	8
-#define TARGET_FREEBSD_I386_GET_GSBASE	9
-#define TARGET_FREEBSD_I386_SET_GSBASE	10
-
-
-#define UNAME_MACHINE "i386"
-
diff --git a/bsd-user/i386/target_syscall.h b/bsd-user/i386/target_syscall.h
new file mode 100644
index 0000000..8f20138
--- /dev/null
+++ b/bsd-user/i386/target_syscall.h
@@ -0,0 +1,165 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
+/* default linux values for the selectors */
+#define __USER_CS	(0x23)
+#define __USER_DS	(0x2B)
+
+struct target_pt_regs {
+	long ebx;
+	long ecx;
+	long edx;
+	long esi;
+	long edi;
+	long ebp;
+	long eax;
+	int  xds;
+	int  xes;
+	long orig_eax;
+	long eip;
+	int  xcs;
+	long eflags;
+	long esp;
+	int  xss;
+};
+
+/* ioctls */
+
+#define TARGET_LDT_ENTRIES      8192
+#define TARGET_LDT_ENTRY_SIZE	8
+
+#define TARGET_GDT_ENTRIES             9
+#define TARGET_GDT_ENTRY_TLS_ENTRIES   3
+#define TARGET_GDT_ENTRY_TLS_MIN       6
+#define TARGET_GDT_ENTRY_TLS_MAX       (TARGET_GDT_ENTRY_TLS_MIN + TARGET_GDT_ENTRY_TLS_ENTRIES - 1)
+
+struct target_modify_ldt_ldt_s {
+    unsigned int  entry_number;
+    abi_ulong base_addr;
+    unsigned int limit;
+    unsigned int flags;
+};
+
+/* vm86 defines */
+
+#define TARGET_BIOSSEG		0x0f000
+
+#define TARGET_CPU_086		0
+#define TARGET_CPU_186		1
+#define TARGET_CPU_286		2
+#define TARGET_CPU_386		3
+#define TARGET_CPU_486		4
+#define TARGET_CPU_586		5
+
+#define TARGET_VM86_SIGNAL	0	/* return due to signal */
+#define TARGET_VM86_UNKNOWN	1	/* unhandled GP fault - IO-instruction or similar */
+#define TARGET_VM86_INTx	2	/* int3/int x instruction (ARG = x) */
+#define TARGET_VM86_STI	3	/* sti/popf/iret instruction enabled virtual interrupts */
+
+/*
+ * Additional return values when invoking new vm86()
+ */
+#define TARGET_VM86_PICRETURN	4	/* return due to pending PIC request */
+#define TARGET_VM86_TRAP	6	/* return due to DOS-debugger request */
+
+/*
+ * function codes when invoking new vm86()
+ */
+#define TARGET_VM86_PLUS_INSTALL_CHECK	0
+#define TARGET_VM86_ENTER		1
+#define TARGET_VM86_ENTER_NO_BYPASS	2
+#define	TARGET_VM86_REQUEST_IRQ	3
+#define TARGET_VM86_FREE_IRQ		4
+#define TARGET_VM86_GET_IRQ_BITS	5
+#define TARGET_VM86_GET_AND_RESET_IRQ	6
+
+/*
+ * This is the stack-layout seen by the user space program when we have
+ * done a translation of "SAVE_ALL" from vm86 mode. The real kernel layout
+ * is 'kernel_vm86_regs' (see below).
+ */
+
+struct target_vm86_regs {
+/*
+ * normal regs, with special meaning for the segment descriptors..
+ */
+	abi_long ebx;
+	abi_long ecx;
+	abi_long edx;
+	abi_long esi;
+	abi_long edi;
+	abi_long ebp;
+	abi_long eax;
+	abi_long __null_ds;
+	abi_long __null_es;
+	abi_long __null_fs;
+	abi_long __null_gs;
+	abi_long orig_eax;
+	abi_long eip;
+	unsigned short cs, __csh;
+	abi_long eflags;
+	abi_long esp;
+	unsigned short ss, __ssh;
+/*
+ * these are specific to v86 mode:
+ */
+	unsigned short es, __esh;
+	unsigned short ds, __dsh;
+	unsigned short fs, __fsh;
+	unsigned short gs, __gsh;
+};
+
+struct target_revectored_struct {
+	abi_ulong __map[8];			/* 256 bits */
+};
+
+struct target_vm86_struct {
+	struct target_vm86_regs regs;
+	abi_ulong flags;
+	abi_ulong screen_bitmap;
+	abi_ulong cpu_type;
+	struct target_revectored_struct int_revectored;
+	struct target_revectored_struct int21_revectored;
+};
+
+/*
+ * flags masks
+ */
+#define TARGET_VM86_SCREEN_BITMAP	0x0001
+
+struct target_vm86plus_info_struct {
+        abi_ulong flags;
+#define TARGET_force_return_for_pic (1 << 0)
+#define TARGET_vm86dbg_active       (1 << 1)  /* for debugger */
+#define TARGET_vm86dbg_TFpendig     (1 << 2)  /* for debugger */
+#define TARGET_is_vm86pus           (1 << 31) /* for vm86 internal use */
+	unsigned char vm86dbg_intxxtab[32];   /* for debugger */
+};
+
+struct target_vm86plus_struct {
+	struct target_vm86_regs regs;
+	abi_ulong flags;
+	abi_ulong screen_bitmap;
+	abi_ulong cpu_type;
+	struct target_revectored_struct int_revectored;
+	struct target_revectored_struct int21_revectored;
+	struct target_vm86plus_info_struct vm86plus;
+};
+
+/* FreeBSD sysarch(2) */
+#define TARGET_FREEBSD_I386_GET_LDT	0
+#define TARGET_FREEBSD_I386_SET_LDT	1
+				/* I386_IOPL */
+#define TARGET_FREEBSD_I386_GET_IOPERM	3
+#define TARGET_FREEBSD_I386_SET_IOPERM	4
+				/* xxxxx */
+#define TARGET_FREEBSD_I386_VM86	6
+#define TARGET_FREEBSD_I386_GET_FSBASE	7
+#define TARGET_FREEBSD_I386_SET_FSBASE	8
+#define TARGET_FREEBSD_I386_GET_GSBASE	9
+#define TARGET_FREEBSD_I386_SET_GSBASE	10
+
+
+#define UNAME_MACHINE "i386"
+
+#endif /* TARGET_SYSCALL_H */
diff --git a/bsd-user/main.c b/bsd-user/main.c
index 0e8c26c..0fb08e4 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -16,31 +16,30 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
+#include "qemu-version.h"
 #include <machine/trap.h>
-#include <sys/types.h>
-#include <sys/mman.h>
 
+#include "qapi/error.h"
 #include "qemu.h"
-#include "qemu-common.h"
+#include "qemu/config-file.h"
+#include "qemu/path.h"
+#include "qemu/help_option.h"
 /* For tb_lock */
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
 #include "qemu/timer.h"
 #include "qemu/envlist.h"
+#include "exec/log.h"
+#include "trace/control.h"
+#include "glib-compat.h"
 
 int singlestep;
-#if defined(CONFIG_USE_GUEST_BASE)
 unsigned long mmap_min_addr;
 unsigned long guest_base;
 int have_guest_base;
 unsigned long reserved_va;
-#endif
 
 static const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX;
 const char *qemu_uname_release;
@@ -92,7 +91,7 @@
 void fork_end(int child)
 {
     if (child) {
-        gdbserver_fork((CPUArchState *)thread_cpu->env_ptr);
+        gdbserver_fork(thread_cpu);
     }
 }
 
@@ -108,13 +107,9 @@
 /***********************************************************/
 /* CPUX86 core interface */
 
-void cpu_smm_update(CPUX86State *env)
-{
-}
-
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 static void write_dt(void *ptr, unsigned long addr, unsigned long limit,
@@ -170,12 +165,14 @@
 
 void cpu_loop(CPUX86State *env)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     int trapnr;
     abi_ulong pc;
     //target_siginfo_t info;
 
     for(;;) {
-        trapnr = cpu_x86_exec(env);
+        trapnr = cpu_exec(cs);
         switch(trapnr) {
         case 0x80:
             /* syscall from int $0x80 */
@@ -516,7 +513,7 @@
     //target_siginfo_t info;
 
     while (1) {
-        trapnr = cpu_sparc_exec (env);
+        trapnr = cpu_exec(cs);
 
         switch (trapnr) {
 #ifndef TARGET_SPARC64
@@ -671,7 +668,8 @@
 
 static void usage(void)
 {
-    printf("qemu-" TARGET_NAME " version " QEMU_VERSION ", Copyright (c) 2003-2008 Fabrice Bellard\n"
+    printf("qemu-" TARGET_NAME " version " QEMU_VERSION QEMU_PKGVERSION
+           ", " QEMU_COPYRIGHT "\n"
            "usage: qemu-" TARGET_NAME " [options] program [arguments...]\n"
            "BSD CPU emulator (compiled for %s emulation)\n"
            "\n"
@@ -684,9 +682,7 @@
            "-drop-ld-preload  drop LD_PRELOAD for target process\n"
            "-E var=value      sets/modifies targets environment variable(s)\n"
            "-U var            unsets targets environment variable(s)\n"
-#if defined(CONFIG_USE_GUEST_BASE)
            "-B address        set guest_base address to address\n"
-#endif
            "-bsd type         select emulated BSD type FreeBSD/NetBSD/OpenBSD (default)\n"
            "\n"
            "Debug options:\n"
@@ -696,6 +692,8 @@
            "-p pagesize       set the host page size to 'pagesize'\n"
            "-singlestep       always run in singlestep mode\n"
            "-strace           log system calls\n"
+           "-trace            [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
+           "                  specify tracing options\n"
            "\n"
            "Environment variables:\n"
            "QEMU_STRACE       Print system calls and arguments similar to the\n"
@@ -744,6 +742,7 @@
     int gdbstub_port = 0;
     char **target_environ, **wrk;
     envlist_t *envlist = NULL;
+    char *trace_file = NULL;
     bsd_type = target_openbsd;
 
     if (argc <= 1)
@@ -762,12 +761,11 @@
     }
 
     cpu_model = NULL;
-#if defined(cpudef_setup)
-    cpudef_setup(); /* parse cpu definitions in target config file (TBD) */
-#endif
+
+    qemu_add_opts(&qemu_trace_opts);
 
     optind = 1;
-    for(;;) {
+    for (;;) {
         if (optind >= argc)
             break;
         r = argv[optind];
@@ -832,11 +830,9 @@
 #endif
                 exit(1);
             }
-#if defined(CONFIG_USE_GUEST_BASE)
         } else if (!strcmp(r, "B")) {
            guest_base = strtol(argv[optind++], NULL, 0);
            have_guest_base = 1;
-#endif
         } else if (!strcmp(r, "drop-ld-preload")) {
             (void) envlist_unsetenv(envlist, "LD_PRELOAD");
         } else if (!strcmp(r, "bsd")) {
@@ -854,14 +850,17 @@
             singlestep = 1;
         } else if (!strcmp(r, "strace")) {
             do_strace = 1;
-        } else
-        {
+        } else if (!strcmp(r, "trace")) {
+            g_free(trace_file);
+            trace_file = trace_opt_parse(optarg);
+        } else {
             usage();
         }
     }
 
     /* init debug */
-    qemu_set_log_filename(log_file);
+    qemu_log_needs_buffers();
+    qemu_set_log_filename(log_file, &error_fatal);
     if (log_mask) {
         int mask;
 
@@ -878,6 +877,11 @@
     }
     filename = argv[optind];
 
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+
     /* Zero out regs */
     memset(regs, 0, sizeof(struct target_pt_regs));
 
@@ -905,15 +909,14 @@
 #endif
     }
     tcg_exec_init(0);
-    cpu_exec_init_all();
     /* NOTE: we need to init the CPU at this stage to get
        qemu_host_page_size */
-    env = cpu_init(cpu_model);
-    if (!env) {
+    cpu = cpu_init(cpu_model);
+    if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-    cpu = ENV_GET_CPU(env);
+    env = cpu->env_ptr;
 #if defined(TARGET_SPARC) || defined(TARGET_PPC)
     cpu_reset(cpu);
 #endif
@@ -926,7 +929,6 @@
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * Now that page sizes are configured in cpu_init() we can do
      * proper page alignment for guest_base.
@@ -948,12 +950,11 @@
             unsigned long tmp;
             if (fscanf(fp, "%lu", &tmp) == 1) {
                 mmap_min_addr = tmp;
-                qemu_log("host mmap_min_addr=0x%lx\n", mmap_min_addr);
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n", mmap_min_addr);
             }
             fclose(fp);
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     if (loader_exec(filename, argv+optind, target_environ, regs, info) != 0) {
         printf("Error loading %s\n", filename);
@@ -966,10 +967,8 @@
 
     free(target_environ);
 
-    if (qemu_log_enabled()) {
-#if defined(CONFIG_USE_GUEST_BASE)
+    if (qemu_loglevel_mask(CPU_LOG_PAGE)) {
         qemu_log("guest_base  0x%lx\n", guest_base);
-#endif
         log_page_dump();
 
         qemu_log("start_brk   0x" TARGET_ABI_FMT_lx "\n", info->start_brk);
@@ -989,12 +988,10 @@
     syscall_init();
     signal_init();
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /* Now that we've loaded the binary, GUEST_BASE is fixed.  Delay
        generating the prologue until now so that the prologue can take
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(&tcg_ctx);
-#endif
 
     /* build Task State */
     memset(ts, 0, sizeof(TaskState));
@@ -1136,6 +1133,7 @@
         gdbserver_start (gdbstub_port);
         gdb_handlesig(cpu, 0);
     }
+    trace_init_vcpu_events();
     cpu_loop(env);
     /* never exits */
     return 0;
diff --git a/bsd-user/mmap.c b/bsd-user/mmap.c
index 092bf7f..610f91b 100644
--- a/bsd-user/mmap.c
+++ b/bsd-user/mmap.c
@@ -16,13 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/mman.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "qemu-common.h"
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index 5362297..2b2b918 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -17,15 +17,13 @@
 #ifndef QEMU_H
 #define QEMU_H
 
-#include <signal.h>
-#include <string.h>
 
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #undef DEBUG_REMAP
 #ifdef DEBUG_REMAP
-#include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
 #include "exec/user/abitypes.h"
@@ -38,7 +36,7 @@
 extern enum BSDType bsd_type;
 
 #include "syscall_defs.h"
-#include "syscall.h"
+#include "target_syscall.h"
 #include "target_signal.h"
 #include "exec/gdbstub.h"
 
@@ -101,9 +99,7 @@
 
 void init_task_state(TaskState *ts);
 extern const char *qemu_uname_release;
-#if defined(CONFIG_USE_GUEST_BASE)
 extern unsigned long mmap_min_addr;
-#endif
 
 /* ??? See if we can avoid exposing so much of the loader internals.  */
 /*
@@ -213,10 +209,6 @@
                        abi_ulong new_addr);
 int target_msync(abi_ulong start, abi_ulong len, int flags);
 extern unsigned long last_brk;
-void mmap_lock(void);
-void mmap_unlock(void);
-void cpu_list_lock(void);
-void cpu_list_unlock(void);
 #if defined(CONFIG_USE_NPTL)
 void mmap_fork_start(void);
 void mmap_fork_end(int child);
@@ -364,7 +356,7 @@
 #ifdef DEBUG_REMAP
     {
         void *addr;
-        addr = malloc(len);
+        addr = g_malloc(len);
         if (copy)
             memcpy(addr, g2h(guest_addr), len);
         else
@@ -390,7 +382,7 @@
         return;
     if (len > 0)
         memcpy(g2h(guest_addr), host_ptr, len);
-    free(host_ptr);
+    g_free(host_ptr);
 #endif
 }
 
diff --git a/bsd-user/signal.c b/bsd-user/signal.c
index 445f69e..f6f7aa2 100644
--- a/bsd-user/signal.c
+++ b/bsd-user/signal.c
@@ -16,19 +16,11 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <signal.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "target_signal.h"
 
-//#define DEBUG_SIGNAL
-
 void signal_init(void)
 {
 }
diff --git a/bsd-user/sparc/syscall.h b/bsd-user/sparc/syscall.h
deleted file mode 100644
index 5a9bb7e..0000000
--- a/bsd-user/sparc/syscall.h
+++ /dev/null
@@ -1,9 +0,0 @@
-struct target_pt_regs {
-	abi_ulong psr;
-	abi_ulong pc;
-	abi_ulong npc;
-	abi_ulong y;
-	abi_ulong u_regs[16];
-};
-
-#define UNAME_MACHINE "sun4"
diff --git a/bsd-user/sparc/target_syscall.h b/bsd-user/sparc/target_syscall.h
new file mode 100644
index 0000000..dfdf9f8
--- /dev/null
+++ b/bsd-user/sparc/target_syscall.h
@@ -0,0 +1,14 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
+struct target_pt_regs {
+	abi_ulong psr;
+	abi_ulong pc;
+	abi_ulong npc;
+	abi_ulong y;
+	abi_ulong u_regs[16];
+};
+
+#define UNAME_MACHINE "sun4"
+
+#endif /* TARGET_SYSCALL_H */
diff --git a/bsd-user/sparc64/syscall.h b/bsd-user/sparc64/syscall.h
deleted file mode 100644
index 81a816d..0000000
--- a/bsd-user/sparc64/syscall.h
+++ /dev/null
@@ -1,10 +0,0 @@
-struct target_pt_regs {
-	abi_ulong u_regs[16];
-	abi_ulong tstate;
-	abi_ulong pc;
-	abi_ulong npc;
-	abi_ulong y;
-	abi_ulong fprs;
-};
-
-#define UNAME_MACHINE "sun4u"
diff --git a/bsd-user/sparc64/target_syscall.h b/bsd-user/sparc64/target_syscall.h
new file mode 100644
index 0000000..3a9f4c2
--- /dev/null
+++ b/bsd-user/sparc64/target_syscall.h
@@ -0,0 +1,15 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
+struct target_pt_regs {
+	abi_ulong u_regs[16];
+	abi_ulong tstate;
+	abi_ulong pc;
+	abi_ulong npc;
+	abi_ulong y;
+	abi_ulong fprs;
+};
+
+#define UNAME_MACHINE "sun4u"
+
+#endif /* TARGET_SYSCALL_H */
diff --git a/bsd-user/strace.c b/bsd-user/strace.c
index e33dd4d..fa66fe1 100644
--- a/bsd-user/strace.c
+++ b/bsd-user/strace.c
@@ -16,14 +16,10 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/select.h>
-#include <sys/types.h>
-#include <unistd.h>
 #include <sys/syscall.h>
 #include <sys/ioccom.h>
-#include <ctype.h>
 
 #include "qemu.h"
 
diff --git a/bsd-user/syscall.c b/bsd-user/syscall.c
index a4d1583..66492aa 100644
--- a/bsd-user/syscall.c
+++ b/bsd-user/syscall.c
@@ -16,18 +16,9 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <limits.h>
-#include <sys/types.h>
-#include <sys/mman.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/path.h"
 #include <sys/syscall.h>
 #include <sys/param.h>
 #include <sys/sysctl.h>
@@ -324,12 +315,14 @@
                             abi_long arg5, abi_long arg6, abi_long arg7,
                             abi_long arg8)
 {
+    CPUState *cpu = ENV_GET_CPU(cpu_env);
     abi_long ret;
     void *p;
 
 #ifdef DEBUG
     gemu_log("freebsd syscall %d\n", num);
 #endif
+    trace_guest_user_syscall(cpu, num, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
     if(do_strace)
         print_freebsd_syscall(num, arg1, arg2, arg3, arg4, arg5, arg6);
 
@@ -409,6 +402,7 @@
 #endif
     if (do_strace)
         print_freebsd_syscall_ret(num, ret);
+    trace_guest_user_syscall_ret(cpu, num, ret);
     return ret;
  efault:
     ret = -TARGET_EFAULT;
@@ -419,12 +413,14 @@
                            abi_long arg2, abi_long arg3, abi_long arg4,
                            abi_long arg5, abi_long arg6)
 {
+    CPUState *cpu = ENV_GET_CPU(cpu_env);
     abi_long ret;
     void *p;
 
 #ifdef DEBUG
     gemu_log("netbsd syscall %d\n", num);
 #endif
+    trace_guest_user_syscall(cpu, num, arg1, arg2, arg3, arg4, arg5, arg6, 0, 0);
     if(do_strace)
         print_netbsd_syscall(num, arg1, arg2, arg3, arg4, arg5, arg6);
 
@@ -481,6 +477,7 @@
 #endif
     if (do_strace)
         print_netbsd_syscall_ret(num, ret);
+    trace_guest_user_syscall_ret(cpu, num, ret);
     return ret;
  efault:
     ret = -TARGET_EFAULT;
@@ -491,12 +488,14 @@
                             abi_long arg2, abi_long arg3, abi_long arg4,
                             abi_long arg5, abi_long arg6)
 {
+    CPUState *cpu = ENV_GET_CPU(cpu_env);
     abi_long ret;
     void *p;
 
 #ifdef DEBUG
     gemu_log("openbsd syscall %d\n", num);
 #endif
+    trace_guest_user_syscall(cpu, num, arg1, arg2, arg3, arg4, arg5, arg6, 0, 0);
     if(do_strace)
         print_openbsd_syscall(num, arg1, arg2, arg3, arg4, arg5, arg6);
 
@@ -553,6 +552,7 @@
 #endif
     if (do_strace)
         print_openbsd_syscall_ret(num, ret);
+    trace_guest_user_syscall_ret(cpu, num, ret);
     return ret;
  efault:
     ret = -TARGET_EFAULT;
diff --git a/bsd-user/uaccess.c b/bsd-user/uaccess.c
index 677f19c..91e2067 100644
--- a/bsd-user/uaccess.c
+++ b/bsd-user/uaccess.c
@@ -1,6 +1,6 @@
 /* User memory access */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 
 #include "qemu.h"
 
@@ -51,7 +51,7 @@
         ptr = lock_user(VERIFY_READ, guest_addr, max_len, 1);
         if (!ptr)
             return -TARGET_EFAULT;
-        len = qemu_strnlen((char *)ptr, max_len);
+        len = qemu_strnlen((const char *)ptr, max_len);
         unlock_user(ptr, guest_addr, 0);
         guest_addr += len;
         /* we don't allow wrapping or integer overflow */
diff --git a/bsd-user/x86_64/syscall.h b/bsd-user/x86_64/syscall.h
deleted file mode 100644
index 630514a..0000000
--- a/bsd-user/x86_64/syscall.h
+++ /dev/null
@@ -1,116 +0,0 @@
-#define __USER_CS	(0x33)
-#define __USER_DS	(0x2B)
-
-struct target_pt_regs {
-	abi_ulong r15;
-	abi_ulong r14;
-	abi_ulong r13;
-	abi_ulong r12;
-	abi_ulong rbp;
-	abi_ulong rbx;
-/* arguments: non interrupts/non tracing syscalls only save up to here */
- 	abi_ulong r11;
-	abi_ulong r10;
-	abi_ulong r9;
-	abi_ulong r8;
-	abi_ulong rax;
-	abi_ulong rcx;
-	abi_ulong rdx;
-	abi_ulong rsi;
-	abi_ulong rdi;
-	abi_ulong orig_rax;
-/* end of arguments */
-/* cpu exception frame or undefined */
-	abi_ulong rip;
-	abi_ulong cs;
-	abi_ulong eflags;
-	abi_ulong rsp;
-	abi_ulong ss;
-/* top of stack page */
-};
-
-/* Maximum number of LDT entries supported. */
-#define TARGET_LDT_ENTRIES	8192
-/* The size of each LDT entry. */
-#define TARGET_LDT_ENTRY_SIZE	8
-
-#define TARGET_GDT_ENTRIES 16
-#define TARGET_GDT_ENTRY_TLS_ENTRIES 3
-#define TARGET_GDT_ENTRY_TLS_MIN 12
-#define TARGET_GDT_ENTRY_TLS_MAX 14
-
-#if 0 // Redefine this
-struct target_modify_ldt_ldt_s {
-	unsigned int  entry_number;
-        abi_ulong     base_addr;
-	unsigned int  limit;
-	unsigned int  seg_32bit:1;
-	unsigned int  contents:2;
-	unsigned int  read_exec_only:1;
-	unsigned int  limit_in_pages:1;
-	unsigned int  seg_not_present:1;
-	unsigned int  useable:1;
-	unsigned int  lm:1;
-};
-#else
-struct target_modify_ldt_ldt_s {
-	unsigned int  entry_number;
-        abi_ulong     base_addr;
-	unsigned int  limit;
-        unsigned int flags;
-};
-#endif
-
-struct target_ipc64_perm
-{
-	int		key;
-	uint32_t	uid;
-	uint32_t	gid;
-	uint32_t	cuid;
-	uint32_t	cgid;
-	unsigned short		mode;
-	unsigned short		__pad1;
-	unsigned short		seq;
-	unsigned short		__pad2;
-	abi_ulong		__unused1;
-	abi_ulong		__unused2;
-};
-
-struct target_msqid64_ds {
-	struct target_ipc64_perm msg_perm;
-	unsigned int msg_stime;	/* last msgsnd time */
-	unsigned int msg_rtime;	/* last msgrcv time */
-	unsigned int msg_ctime;	/* last change time */
-	abi_ulong  msg_cbytes;	/* current number of bytes on queue */
-	abi_ulong  msg_qnum;	/* number of messages in queue */
-	abi_ulong  msg_qbytes;	/* max number of bytes on queue */
-	unsigned int msg_lspid;	/* pid of last msgsnd */
-	unsigned int msg_lrpid;	/* last receive pid */
-	abi_ulong  __unused4;
-	abi_ulong  __unused5;
-};
-
-/* FreeBSD sysarch(2) */
-#define TARGET_FREEBSD_I386_GET_LDT	0
-#define TARGET_FREEBSD_I386_SET_LDT	1
-				/* I386_IOPL */
-#define TARGET_FREEBSD_I386_GET_IOPERM	3
-#define TARGET_FREEBSD_I386_SET_IOPERM	4
-				/* xxxxx */
-#define TARGET_FREEBSD_I386_GET_FSBASE	7
-#define TARGET_FREEBSD_I386_SET_FSBASE	8
-#define TARGET_FREEBSD_I386_GET_GSBASE	9
-#define TARGET_FREEBSD_I386_SET_GSBASE	10
-
-#define TARGET_FREEBSD_AMD64_GET_FSBASE	128
-#define TARGET_FREEBSD_AMD64_SET_FSBASE	129
-#define TARGET_FREEBSD_AMD64_GET_GSBASE	130
-#define TARGET_FREEBSD_AMD64_SET_GSBASE	131
-
-
-#define UNAME_MACHINE "x86_64"
-
-#define TARGET_ARCH_SET_GS 0x1001
-#define TARGET_ARCH_SET_FS 0x1002
-#define TARGET_ARCH_GET_FS 0x1003
-#define TARGET_ARCH_GET_GS 0x1004
diff --git a/bsd-user/x86_64/target_syscall.h b/bsd-user/x86_64/target_syscall.h
new file mode 100644
index 0000000..211ce29
--- /dev/null
+++ b/bsd-user/x86_64/target_syscall.h
@@ -0,0 +1,121 @@
+#ifndef TARGET_SYSCALL_H
+#define TARGET_SYSCALL_H
+
+#define __USER_CS	(0x33)
+#define __USER_DS	(0x2B)
+
+struct target_pt_regs {
+	abi_ulong r15;
+	abi_ulong r14;
+	abi_ulong r13;
+	abi_ulong r12;
+	abi_ulong rbp;
+	abi_ulong rbx;
+/* arguments: non interrupts/non tracing syscalls only save up to here */
+ 	abi_ulong r11;
+	abi_ulong r10;
+	abi_ulong r9;
+	abi_ulong r8;
+	abi_ulong rax;
+	abi_ulong rcx;
+	abi_ulong rdx;
+	abi_ulong rsi;
+	abi_ulong rdi;
+	abi_ulong orig_rax;
+/* end of arguments */
+/* cpu exception frame or undefined */
+	abi_ulong rip;
+	abi_ulong cs;
+	abi_ulong eflags;
+	abi_ulong rsp;
+	abi_ulong ss;
+/* top of stack page */
+};
+
+/* Maximum number of LDT entries supported. */
+#define TARGET_LDT_ENTRIES	8192
+/* The size of each LDT entry. */
+#define TARGET_LDT_ENTRY_SIZE	8
+
+#define TARGET_GDT_ENTRIES 16
+#define TARGET_GDT_ENTRY_TLS_ENTRIES 3
+#define TARGET_GDT_ENTRY_TLS_MIN 12
+#define TARGET_GDT_ENTRY_TLS_MAX 14
+
+#if 0 // Redefine this
+struct target_modify_ldt_ldt_s {
+	unsigned int  entry_number;
+        abi_ulong     base_addr;
+	unsigned int  limit;
+	unsigned int  seg_32bit:1;
+	unsigned int  contents:2;
+	unsigned int  read_exec_only:1;
+	unsigned int  limit_in_pages:1;
+	unsigned int  seg_not_present:1;
+	unsigned int  useable:1;
+	unsigned int  lm:1;
+};
+#else
+struct target_modify_ldt_ldt_s {
+	unsigned int  entry_number;
+        abi_ulong     base_addr;
+	unsigned int  limit;
+        unsigned int flags;
+};
+#endif
+
+struct target_ipc64_perm
+{
+	int		key;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	cuid;
+	uint32_t	cgid;
+	unsigned short		mode;
+	unsigned short		__pad1;
+	unsigned short		seq;
+	unsigned short		__pad2;
+	abi_ulong		__unused1;
+	abi_ulong		__unused2;
+};
+
+struct target_msqid64_ds {
+	struct target_ipc64_perm msg_perm;
+	unsigned int msg_stime;	/* last msgsnd time */
+	unsigned int msg_rtime;	/* last msgrcv time */
+	unsigned int msg_ctime;	/* last change time */
+	abi_ulong  msg_cbytes;	/* current number of bytes on queue */
+	abi_ulong  msg_qnum;	/* number of messages in queue */
+	abi_ulong  msg_qbytes;	/* max number of bytes on queue */
+	unsigned int msg_lspid;	/* pid of last msgsnd */
+	unsigned int msg_lrpid;	/* last receive pid */
+	abi_ulong  __unused4;
+	abi_ulong  __unused5;
+};
+
+/* FreeBSD sysarch(2) */
+#define TARGET_FREEBSD_I386_GET_LDT	0
+#define TARGET_FREEBSD_I386_SET_LDT	1
+				/* I386_IOPL */
+#define TARGET_FREEBSD_I386_GET_IOPERM	3
+#define TARGET_FREEBSD_I386_SET_IOPERM	4
+				/* xxxxx */
+#define TARGET_FREEBSD_I386_GET_FSBASE	7
+#define TARGET_FREEBSD_I386_SET_FSBASE	8
+#define TARGET_FREEBSD_I386_GET_GSBASE	9
+#define TARGET_FREEBSD_I386_SET_GSBASE	10
+
+#define TARGET_FREEBSD_AMD64_GET_FSBASE	128
+#define TARGET_FREEBSD_AMD64_SET_FSBASE	129
+#define TARGET_FREEBSD_AMD64_GET_GSBASE	130
+#define TARGET_FREEBSD_AMD64_SET_GSBASE	131
+
+
+#define UNAME_MACHINE "x86_64"
+
+#define TARGET_ARCH_SET_GS 0x1001
+#define TARGET_ARCH_SET_FS 0x1002
+#define TARGET_ARCH_GET_FS 0x1003
+#define TARGET_ARCH_GET_GS 0x1004
+
+#endif /* TARGET_SYSCALL_H */
diff --git a/bt-host.c b/bt-host.c
index 49205bf..2f8f631 100644
--- a/bt-host.c
+++ b/bt-host.c
@@ -17,12 +17,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/bt.h"
 #include "qemu/main-loop.h"
 
 #ifndef _WIN32
-# include <errno.h>
 # include <sys/ioctl.h>
 # include <sys/uio.h>
 # ifdef CONFIG_BLUEZ
diff --git a/bt-vhci.c b/bt-vhci.c
index e267c8a..9d277c3 100644
--- a/bt-vhci.c
+++ b/bt-vhci.c
@@ -17,6 +17,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
diff --git a/configure b/configure
index 137dd59..890c14c 100755
--- a/configure
+++ b/configure
@@ -8,6 +8,9 @@
 CLICOLOR_FORCE= GREP_OPTIONS=
 unset CLICOLOR_FORCE GREP_OPTIONS
 
+# Don't allow CCACHE, if present, to use cached results of compile tests!
+export CCACHE_RECACHE=yes
+
 # Temporary directory used for files created while
 # configure runs. Since it is in the build directory
 # we can safely blow away any previous version of it
@@ -28,6 +31,7 @@
 TMPL="${TMPDIR1}/${TMPB}.lo"
 TMPA="${TMPDIR1}/lib${TMPB}.la"
 TMPE="${TMPDIR1}/${TMPB}.exe"
+TMPMO="${TMPDIR1}/${TMPB}.mo"
 
 rm -f config.log
 
@@ -103,7 +107,8 @@
 }
 
 compile_object() {
-  do_cc $QEMU_CFLAGS -c -o $TMPO $TMPC
+  local_cflags="$1"
+  do_cc $QEMU_CFLAGS $local_cflags -c -o $TMPO $TMPC
 }
 
 compile_prog() {
@@ -112,38 +117,6 @@
   do_cc $QEMU_CFLAGS $local_cflags -o $TMPE $TMPC $LDFLAGS $local_ldflags
 }
 
-do_libtool() {
-    local mode=$1
-    shift
-    # Run the compiler, capturing its output to the log.
-    echo $libtool $mode --tag=CC $cc "$@" >> config.log
-    $libtool $mode --tag=CC $cc "$@" >> config.log 2>&1 || return $?
-    # Test passed. If this is an --enable-werror build, rerun
-    # the test with -Werror and bail out if it fails. This
-    # makes warning-generating-errors in configure test code
-    # obvious to developers.
-    if test "$werror" != "yes"; then
-        return 0
-    fi
-    # Don't bother rerunning the compile if we were already using -Werror
-    case "$*" in
-        *-Werror*)
-           return 0
-        ;;
-    esac
-    echo $libtool $mode --tag=CC $cc -Werror "$@" >> config.log
-    $libtool $mode --tag=CC $cc -Werror "$@" >> config.log 2>&1 && return $?
-    error_exit "configure test passed without -Werror but failed with -Werror." \
-        "This is probably a bug in the configure script. The failing command" \
-        "will be at the bottom of config.log." \
-        "You can run configure with --disable-werror to bypass this check."
-}
-
-libtool_prog() {
-    do_libtool --mode=compile $QEMU_CFLAGS -c -fPIE -DPIE -o $TMPO $TMPC || return $?
-    do_libtool --mode=link $LDFLAGS -o $TMPA $TMPL -rpath /usr/local/lib
-}
-
 # symbolically link $1 to $2.  Portable version of "ln -sf".
 symlink() {
   rm -rf "$2"
@@ -192,7 +165,7 @@
 }
 
 # default parameters
-source_path=`dirname "$0"`
+source_path=$(dirname "$0")
 cpu=""
 iasl="iasl"
 interp_prefix="/usr/gnemul/qemu-%M"
@@ -235,19 +208,18 @@
 netmap="no"
 pixman=""
 sdl=""
-sdlabi="1.2"
+sdlabi=""
 virtfs=""
 vnc="yes"
 sparse="no"
 uuid=""
 vde=""
-vnc_tls=""
 vnc_sasl=""
 vnc_jpeg=""
 vnc_png=""
-vnc_ws=""
 xen=""
 xen_ctrl_version=""
+xen_pv_domain_build="no"
 xen_pci_passthrough=""
 linux_aio=""
 cap_ng=""
@@ -263,6 +235,7 @@
 gprof="no"
 debug_tcg="no"
 debug="no"
+fortify_source=""
 strip_opt="yes"
 tcg_interpreter="no"
 bigendian="no"
@@ -285,8 +258,6 @@
 local_statedir="\${prefix}/var"
 confsuffix="/qemu"
 slirp="yes"
-fmod_lib=""
-fmod_inc=""
 oss_lib=""
 bsd="no"
 linux="no"
@@ -296,26 +267,29 @@
 softmmu="yes"
 linux_user="no"
 bsd_user="no"
-guest_base="yes"
 aix="no"
 blobs="yes"
 pkgversion=""
 pie=""
-zero_malloc=""
 qom_cast_debug="yes"
-trace_backends="nop"
+trace_backends="log"
 trace_file="trace"
 spice=""
 rbd=""
-smartcard_nss=""
+smartcard=""
 libusb=""
 usb_redir=""
-glx=""
+opengl=""
+opengl_dmabuf="no"
+avx2_opt="no"
 zlib="yes"
 lzo=""
 snappy=""
+bzip2=""
 guest_agent=""
 guest_agent_with_vss="no"
+guest_agent_ntddscsi="no"
+guest_agent_msi=""
 vss_win32_sdk=""
 win_sdk="no"
 want_tools="yes"
@@ -325,22 +299,32 @@
 coroutine_pool=""
 seccomp=""
 glusterfs=""
+glusterfs_xlator_opt="no"
 glusterfs_discard="no"
 glusterfs_zerofill="no"
-archipelago=""
+archipelago="no"
 gtk=""
 gtkabi=""
+gtk_gl="no"
+tls_priority="NORMAL"
+gnutls=""
+gnutls_rnd=""
+nettle=""
+nettle_kdf="no"
+gcrypt=""
+gcrypt_kdf="no"
 vte=""
+virglrenderer=""
 tpm="yes"
 libssh2=""
 vhdx=""
-quorum=""
 numa=""
-android="no"
+tcmalloc="no"
+jemalloc="no"
 
 # parse CC options first
 for opt do
-  optarg=`expr "x$opt" : 'x[^=]*=\(.*\)'`
+  optarg=$(expr "x$opt" : 'x[^=]*=\(.*\)')
   case "$opt" in
   --cross-prefix=*) cross_prefix="$optarg"
   ;;
@@ -352,10 +336,10 @@
   ;;
   --cpu=*) cpu="$optarg"
   ;;
-  --extra-cflags=*) QEMU_CFLAGS="$optarg $QEMU_CFLAGS"
+  --extra-cflags=*) QEMU_CFLAGS="$QEMU_CFLAGS $optarg"
                     EXTRA_CFLAGS="$optarg"
   ;;
-  --extra-ldflags=*) LDFLAGS="$optarg $LDFLAGS"
+  --extra-ldflags=*) LDFLAGS="$LDFLAGS $optarg"
                      EXTRA_LDFLAGS="$optarg"
   ;;
   --enable-debug-info) debug_info="yes"
@@ -386,10 +370,10 @@
 
 ar="${AR-${cross_prefix}ar}"
 as="${AS-${cross_prefix}as}"
+ccas="${CCAS-$cc}"
 cpp="${CPP-$cc -E}"
 objcopy="${OBJCOPY-${cross_prefix}objcopy}"
 ld="${LD-${cross_prefix}ld}"
-libtool="${LIBTOOL-${cross_prefix}libtool}"
 nm="${NM-${cross_prefix}nm}"
 strip="${STRIP-${cross_prefix}strip}"
 windres="${WINDRES-${cross_prefix}windres}"
@@ -416,7 +400,7 @@
 fi
 
 # make source path absolute
-source_path=`cd "$source_path"; pwd`
+source_path=$(cd "$source_path"; pwd)
 
 # running configure in the source tree?
 # we know that's the case if configure is there.
@@ -436,6 +420,20 @@
   compile_object
 }
 
+check_include() {
+cat > $TMPC <<EOF
+#include <$1>
+int main(void) { return 0; }
+EOF
+  compile_object
+}
+
+write_c_skeleton() {
+    cat > $TMPC <<EOF
+int main(void) { return 0; }
+EOF
+}
+
 if check_define __linux__ ; then
   targetos="Linux"
 elif check_define _WIN32 ; then
@@ -447,7 +445,7 @@
 elif check_define __HAIKU__ ; then
   targetos='Haiku'
 else
-  targetos=`uname -s`
+  targetos=$(uname -s)
 fi
 
 # Some host OSes need non-standard checks for which CPU to use.
@@ -465,7 +463,7 @@
   fi
   ;;
 SunOS)
-  # `uname -m` returns i86pc even on an x86_64 box, so default based on isainfo
+  # $(uname -m) returns i86pc even on an x86_64 box, so default based on isainfo
   if test -z "$cpu" && test "$(isainfo -k)" = "amd64"; then
     cpu="x86_64"
   fi
@@ -511,7 +509,7 @@
 elif check_define __hppa__ ; then
   cpu="hppa"
 else
-  cpu=`uname -m`
+  cpu=$(uname -m)
 fi
 
 ARCH=
@@ -557,25 +555,28 @@
 CYGWIN*)
   mingw32="yes"
   QEMU_CFLAGS="-mno-cygwin $QEMU_CFLAGS"
-  audio_possible_drivers="winwave sdl"
-  audio_drv_list="winwave"
+  audio_possible_drivers="sdl"
+  audio_drv_list="sdl"
 ;;
 MINGW32*)
   mingw32="yes"
-  hax="yes"
-  audio_possible_drivers="winwave dsound sdl fmod winaudio"
-  audio_drv_list="winwave"
+  audio_possible_drivers="dsound sdl winaudio"
+  if check_include dsound.h; then
+    audio_drv_list="dsound"
+  else
+    audio_drv_list=""
+  fi
 ;;
 GNU/kFreeBSD)
   bsd="yes"
   audio_drv_list="oss"
-  audio_possible_drivers="oss sdl esd pa"
+  audio_possible_drivers="oss sdl pa"
 ;;
 FreeBSD)
   bsd="yes"
   make="${MAKE-gmake}"
   audio_drv_list="oss"
-  audio_possible_drivers="oss sdl esd pa"
+  audio_possible_drivers="oss sdl pa"
   # needed for kinfo_getvmmap(3) in libutil.h
   LIBS="-lutil $LIBS"
   netmap=""  # enable netmap autodetect
@@ -585,14 +586,14 @@
   bsd="yes"
   make="${MAKE-gmake}"
   audio_drv_list="oss"
-  audio_possible_drivers="oss sdl esd pa"
+  audio_possible_drivers="oss sdl pa"
   HOST_VARIANT_DIR="dragonfly"
 ;;
 NetBSD)
   bsd="yes"
   make="${MAKE-gmake}"
   audio_drv_list="oss"
-  audio_possible_drivers="oss sdl esd"
+  audio_possible_drivers="oss sdl"
   oss_lib="-lossaudio"
   HOST_VARIANT_DIR="netbsd"
 ;;
@@ -600,7 +601,7 @@
   bsd="yes"
   make="${MAKE-gmake}"
   audio_drv_list="sdl"
-  audio_possible_drivers="sdl esd"
+  audio_possible_drivers="sdl"
   HOST_VARIANT_DIR="openbsd"
 ;;
 Darwin)
@@ -614,7 +615,7 @@
   fi
   cocoa="yes"
   audio_drv_list="coreaudio"
-  audio_possible_drivers="coreaudio sdl fmod"
+  audio_possible_drivers="coreaudio sdl"
   LDFLAGS="-framework CoreFoundation -framework IOKit $LDFLAGS"
   libs_softmmu="-F/System/Library/Frameworks -framework Cocoa -framework IOKit $libs_softmmu"
   # Disable attempts to use ObjectiveC features in os/object.h since they
@@ -629,7 +630,7 @@
   ld="gld"
   smbd="${SMBD-/usr/sfw/sbin/smbd}"
   needs_libsunmath="no"
-  solarisrev=`uname -r | cut -f2 -d.`
+  solarisrev=$(uname -r | cut -f2 -d.)
   if [ "$cpu" = "i386" -o "$cpu" = "x86_64" ] ; then
     if test "$solarisrev" -le 9 ; then
       if test -f /opt/SUNWspro/prod/lib/libsunmath.so.1; then
@@ -669,15 +670,12 @@
 ;;
 *)
   audio_drv_list="oss"
-  audio_possible_drivers="oss alsa sdl esd pa"
+  audio_possible_drivers="oss alsa sdl pa"
   linux="yes"
   linux_user="yes"
   kvm="yes"
   vhost_net="yes"
   vhost_scsi="yes"
-  if [ "$cpu" = "i386" -o "$cpu" = "x86_64" -o "$cpu" = "x32" ] ; then
-    audio_possible_drivers="$audio_possible_drivers fmod"
-  fi
   QEMU_INCLUDES="-I\$(SRC_PATH)/linux-headers -I$(pwd)/linux-headers $QEMU_INCLUDES"
 ;;
 esac
@@ -706,10 +704,10 @@
   QEMU_CFLAGS="-DWIN32_LEAN_AND_MEAN -DWINVER=0x501 $QEMU_CFLAGS"
   # enable C99/POSIX format strings (needs mingw32-runtime 3.15 or later)
   QEMU_CFLAGS="-D__USE_MINGW_ANSI_STDIO=1 $QEMU_CFLAGS"
+  # MinGW needs -mthreads for TLS and macro _MT.
+  QEMU_CFLAGS="-mthreads $QEMU_CFLAGS"
   LIBS="-lwinmm -lws2_32 -liphlpapi $LIBS"
-cat > $TMPC << EOF
-int main(void) { return 0; }
-EOF
+  write_c_skeleton;
   if compile_prog "" "-liberty" ; then
     LIBS="-liberty $LIBS"
   fi
@@ -721,13 +719,13 @@
   sysconfdir="\${prefix}"
   local_statedir=
   confsuffix=""
-  libs_qga="-lws2_32 -lwinmm -lpowrprof $libs_qga"
+  libs_qga="-lws2_32 -lwinmm -lpowrprof -liphlpapi -lnetapi32 $libs_qga"
 fi
 
 werror=""
 
 for opt do
-  optarg=`expr "x$opt" : 'x[^=]*=\(.*\)'`
+  optarg=$(expr "x$opt" : 'x[^=]*=\(.*\)')
   case "$opt" in
   --help|-h) show_help=yes
   ;;
@@ -772,6 +770,9 @@
   --enable-modules)
       modules="yes"
   ;;
+  --disable-modules)
+      modules="no"
+  ;;
   --cpu=*)
   ;;
   --target-list=*) target_list="$optarg"
@@ -844,17 +845,13 @@
   ;;
   --enable-vnc) vnc="yes"
   ;;
-  --fmod-lib=*) fmod_lib="$optarg"
-  ;;
-  --fmod-inc=*) fmod_inc="$optarg"
-  ;;
   --oss-lib=*) oss_lib="$optarg"
   ;;
   --audio-drv-list=*) audio_drv_list="$optarg"
   ;;
-  --block-drv-rw-whitelist=*|--block-drv-whitelist=*) block_drv_rw_whitelist=`echo "$optarg" | sed -e 's/,/ /g'`
+  --block-drv-rw-whitelist=*|--block-drv-whitelist=*) block_drv_rw_whitelist=$(echo "$optarg" | sed -e 's/,/ /g')
   ;;
-  --block-drv-ro-whitelist=*) block_drv_ro_whitelist=`echo "$optarg" | sed -e 's/,/ /g'`
+  --block-drv-ro-whitelist=*) block_drv_ro_whitelist=$(echo "$optarg" | sed -e 's/,/ /g')
   ;;
   --enable-debug-tcg) debug_tcg="yes"
   ;;
@@ -865,6 +862,7 @@
       debug_tcg="yes"
       debug="yes"
       strip_opt="no"
+      fortify_source="no"
   ;;
   --enable-sparse) sparse="yes"
   ;;
@@ -872,10 +870,6 @@
   ;;
   --disable-strip) strip_opt="no"
   ;;
-  --disable-vnc-tls) vnc_tls="no"
-  ;;
-  --enable-vnc-tls) vnc_tls="yes"
-  ;;
   --disable-vnc-sasl) vnc_sasl="no"
   ;;
   --enable-vnc-sasl) vnc_sasl="yes"
@@ -888,10 +882,6 @@
   ;;
   --enable-vnc-png) vnc_png="yes"
   ;;
-  --disable-vnc-ws) vnc_ws="no"
-  ;;
-  --enable-vnc-ws) vnc_ws="yes"
-  ;;
   --disable-slirp) slirp="no"
   ;;
   --disable-uuid) uuid="no"
@@ -914,6 +904,10 @@
   ;;
   --enable-xen-pci-passthrough) xen_pci_passthrough="yes"
   ;;
+  --disable-xen-pv-domain-build) xen_pv_domain_build="no"
+  ;;
+  --enable-xen-pv-domain-build) xen_pv_domain_build="yes"
+  ;;
   --disable-brlapi) brlapi="no"
   ;;
   --enable-brlapi) brlapi="yes"
@@ -956,8 +950,7 @@
   ;;
   --enable-cocoa)
       cocoa="yes" ;
-      sdl="no" ;
-      audio_drv_list="coreaudio `echo $audio_drv_list | sed s,coreaudio,,g`"
+      audio_drv_list="coreaudio $(echo $audio_drv_list | sed s,coreaudio,,g)"
   ;;
   --disable-system) softmmu="no"
   ;;
@@ -976,10 +969,6 @@
   ;;
   --enable-bsd-user) bsd_user="yes"
   ;;
-  --enable-guest-base) guest_base="yes"
-  ;;
-  --disable-guest-base) guest_base="no"
-  ;;
   --enable-pie) pie="yes"
   ;;
   --disable-pie) pie="no"
@@ -1034,9 +1023,9 @@
   ;;
   --enable-vhost-scsi) vhost_scsi="yes"
   ;;
-  --disable-glx) glx="no"
+  --disable-opengl) opengl="no"
   ;;
-  --enable-glx) glx="yes"
+  --enable-opengl) opengl="yes"
   ;;
   --disable-rbd) rbd="no"
   ;;
@@ -1046,9 +1035,9 @@
   ;;
   --enable-xfsctl) xfs="yes"
   ;;
-  --disable-smartcard-nss) smartcard_nss="no"
+  --disable-smartcard) smartcard="no"
   ;;
-  --enable-smartcard-nss) smartcard_nss="yes"
+  --enable-smartcard) smartcard="yes"
   ;;
   --disable-libusb) libusb="no"
   ;;
@@ -1068,10 +1057,18 @@
   ;;
   --enable-snappy) snappy="yes"
   ;;
+  --disable-bzip2) bzip2="no"
+  ;;
+  --enable-bzip2) bzip2="yes"
+  ;;
   --enable-guest-agent) guest_agent="yes"
   ;;
   --disable-guest-agent) guest_agent="no"
   ;;
+  --enable-guest-agent-msi) guest_agent_msi="yes"
+  ;;
+  --disable-guest-agent-msi) guest_agent_msi="no"
+  ;;
   --with-vss-sdk) vss_win32_sdk=""
   ;;
   --with-vss-sdk=*) vss_win32_sdk="$optarg"
@@ -1107,6 +1104,20 @@
   ;;
   --enable-gtk) gtk="yes"
   ;;
+  --tls-priority=*) tls_priority="$optarg"
+  ;;
+  --disable-gnutls) gnutls="no"
+  ;;
+  --enable-gnutls) gnutls="yes"
+  ;;
+  --disable-nettle) nettle="no"
+  ;;
+  --enable-nettle) nettle="yes"
+  ;;
+  --disable-gcrypt) gcrypt="no"
+  ;;
+  --enable-gcrypt) gcrypt="yes"
+  ;;
   --enable-rdma) rdma="yes"
   ;;
   --disable-rdma) rdma="no"
@@ -1117,6 +1128,10 @@
   ;;
   --enable-vte) vte="yes"
   ;;
+  --disable-virglrenderer) virglrenderer="no"
+  ;;
+  --enable-virglrenderer) virglrenderer="yes"
+  ;;
   --disable-tpm) tpm="no"
   ;;
   --enable-tpm) tpm="yes"
@@ -1129,15 +1144,17 @@
   ;;
   --disable-vhdx) vhdx="no"
   ;;
-  --disable-quorum) quorum="no"
-  ;;
-  --enable-quorum) quorum="yes"
-  ;;
   --disable-numa) numa="no"
   ;;
   --enable-numa) numa="yes"
   ;;
-  --enable-android) android="yes"
+  --disable-tcmalloc) tcmalloc="no"
+  ;;
+  --enable-tcmalloc) tcmalloc="yes"
+  ;;
+  --disable-jemalloc) jemalloc="no"
+  ;;
+  --enable-jemalloc) jemalloc="yes"
   ;;
   *)
       echo "ERROR: unknown option $opt"
@@ -1153,18 +1170,14 @@
 
 # Note that if the Python conditional here evaluates True we will exit
 # with status 1 which is a shell 'false' value.
-if ! $python -c 'import sys; sys.exit(sys.version_info < (2,4) or sys.version_info >= (3,))'; then
-  error_exit "Cannot use '$python', Python 2.4 or later is required." \
+if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then
+  error_exit "Cannot use '$python', Python 2.6 or later is required." \
       "Note that Python 3 or later is not yet supported." \
       "Use --python=/path/to/python to specify a supported Python."
 fi
 
-# The -B switch was added in Python 2.6.
-# If it is supplied, compiled files are not written.
-# Use it for Python versions which support it.
-if $python -B -c 'import sys; sys.exit(0)' 2>/dev/null; then
-  python="$python -B"
-fi
+# Suppress writing compiled files
+python="$python -B"
 
 case "$cpu" in
     ppc)
@@ -1212,6 +1225,13 @@
 QEMU_CFLAGS="$CPU_CFLAGS $QEMU_CFLAGS"
 EXTRA_CFLAGS="$CPU_CFLAGS $EXTRA_CFLAGS"
 
+# For user-mode emulation the host arch has to be one we explicitly
+# support, even if we're using TCI.
+if [ "$ARCH" = "unknown" ]; then
+  bsd_user="no"
+  linux_user="no"
+fi
+
 default_target_list=""
 
 mak_wilds=""
@@ -1269,29 +1289,10 @@
   --sysconfdir=PATH        install config in PATH$confsuffix
   --localstatedir=PATH     install local state in PATH (set at runtime on win32)
   --with-confsuffix=SUFFIX suffix for QEMU data inside datadir/libdir/sysconfdir [$confsuffix]
-  --enable-modules         enable modules support
-  --enable-debug-tcg       enable TCG debugging
-  --disable-debug-tcg      disable TCG debugging (default)
-  --enable-debug-info      enable debugging information (default)
-  --disable-debug-info     disable debugging information
   --enable-debug           enable common debug build options
-  --enable-sparse          enable sparse checker
-  --disable-sparse         disable sparse checker (default)
   --disable-strip          disable stripping binaries
   --disable-werror         disable compilation abort on warning
   --disable-stack-protector disable compiler-provided stack protection
-  --disable-sdl            disable SDL
-  --enable-sdl             enable SDL
-  --with-sdlabi            select preferred SDL ABI 1.2 or 2.0
-  --disable-gtk            disable gtk UI
-  --enable-gtk             enable gtk UI
-  --with-gtkabi            select preferred GTK ABI 2.0 or 3.0
-  --disable-virtfs         disable VirtFS
-  --enable-virtfs          enable VirtFS
-  --disable-vnc            disable VNC
-  --enable-vnc             enable VNC
-  --disable-cocoa          disable Cocoa (Mac OS X only)
-  --enable-cocoa           enable Cocoa (default on Mac OS X)
   --audio-drv-list=LIST    set audio drivers list:
                            Available drivers: $audio_possible_drivers
   --block-drv-whitelist=L  Same as --block-drv-rw-whitelist=L
@@ -1301,118 +1302,91 @@
   --block-drv-ro-whitelist=L
                            set block driver read-only whitelist
                            (affects only QEMU, not qemu-img)
-  --disable-xen            disable xen backend driver support
-  --enable-xen             enable xen backend driver support
-  --disable-xen-pci-passthrough
-  --enable-xen-pci-passthrough
-  --disable-brlapi         disable BrlAPI
-  --enable-brlapi          enable BrlAPI
-  --disable-vnc-tls        disable TLS encryption for VNC server
-  --enable-vnc-tls         enable TLS encryption for VNC server
-  --disable-vnc-sasl       disable SASL encryption for VNC server
-  --enable-vnc-sasl        enable SASL encryption for VNC server
-  --disable-vnc-jpeg       disable JPEG lossy compression for VNC server
-  --enable-vnc-jpeg        enable JPEG lossy compression for VNC server
-  --disable-vnc-png        disable PNG compression for VNC server (default)
-  --enable-vnc-png         enable PNG compression for VNC server
-  --disable-vnc-ws         disable Websockets support for VNC server
-  --enable-vnc-ws          enable Websockets support for VNC server
-  --disable-curses         disable curses output
-  --enable-curses          enable curses output
-  --disable-curl           disable curl connectivity
-  --enable-curl            enable curl connectivity
-  --disable-fdt            disable fdt device tree
-  --enable-fdt             enable fdt device tree
-  --disable-bluez          disable bluez stack connectivity
-  --enable-bluez           enable bluez stack connectivity
-  --disable-slirp          disable SLIRP userspace network connectivity
-  --disable-kvm            disable KVM acceleration support
-  --enable-kvm             enable KVM acceleration support
-  --disable-hax            disable HAX acceleration support
-  --enable-hax             enable HAX acceleration support
-  --disable-rdma           disable RDMA-based migration support
-  --enable-rdma            enable RDMA-based migration support
-  --enable-tcg-interpreter enable TCG with bytecode interpreter (TCI)
-  --enable-system          enable all system emulation targets
-  --disable-system         disable all system emulation targets
-  --enable-user            enable supported user emulation targets
-  --disable-user           disable all user emulation targets
-  --enable-linux-user      enable all linux usermode emulation targets
-  --disable-linux-user     disable all linux usermode emulation targets
-  --enable-bsd-user        enable all BSD usermode emulation targets
-  --disable-bsd-user       disable all BSD usermode emulation targets
-  --enable-guest-base      enable GUEST_BASE support for usermode
-                           emulation targets
-  --disable-guest-base     disable GUEST_BASE support
-  --enable-pie             build Position Independent Executables
-  --disable-pie            do not build Position Independent Executables
-  --fmod-lib               path to FMOD library
-  --fmod-inc               path to FMOD includes
-  --oss-lib                path to OSS library
-  --cpu=CPU                Build for host CPU [$cpu]
-  --disable-uuid           disable uuid support
-  --enable-uuid            enable uuid support
-  --disable-vde            disable support for vde network
-  --enable-vde             enable support for vde network
-  --disable-netmap         disable support for netmap network
-  --enable-netmap          enable support for netmap network
-  --disable-linux-aio      disable Linux AIO support
-  --enable-linux-aio       enable Linux AIO support
-  --disable-cap-ng         disable libcap-ng support
-  --enable-cap-ng          enable libcap-ng support
-  --disable-attr           disable attr and xattr support
-  --enable-attr            enable attr and xattr support
-  --disable-blobs          disable installing provided firmware blobs
-  --enable-docs            enable documentation build
-  --disable-docs           disable documentation build
-  --disable-vhost-net      disable vhost-net acceleration support
-  --enable-vhost-net       enable vhost-net acceleration support
   --enable-trace-backends=B Set trace backend
                            Available backends: $($python $source_path/scripts/tracetool.py --list-backends)
   --with-trace-file=NAME   Full PATH,NAME of file to store traces
                            Default:trace-<pid>
-  --disable-spice          disable spice
-  --enable-spice           enable spice
-  --enable-rbd             enable building the rados block device (rbd)
-  --disable-libiscsi       disable iscsi support
-  --enable-libiscsi        enable iscsi support
-  --disable-libnfs         disable nfs support
-  --enable-libnfs          enable nfs support
-  --disable-smartcard-nss  disable smartcard nss support
-  --enable-smartcard-nss   enable smartcard nss support
-  --disable-libusb         disable libusb (for usb passthrough)
-  --enable-libusb          enable libusb (for usb passthrough)
-  --disable-usb-redir      disable usb network redirection support
-  --enable-usb-redir       enable usb network redirection support
-  --enable-lzo             enable the support of lzo compression library
-  --enable-snappy          enable the support of snappy compression library
-  --disable-guest-agent    disable building of the QEMU Guest Agent
-  --enable-guest-agent     enable building of the QEMU Guest Agent
-  --with-vss-sdk=SDK-path  enable Windows VSS support in QEMU Guest Agent
-  --with-win-sdk=SDK-path  path to Windows Platform SDK (to build VSS .tlb)
-  --disable-seccomp        disable seccomp support
-  --enable-seccomp         enable seccomp support
+  --disable-slirp          disable SLIRP userspace network connectivity
+  --enable-tcg-interpreter enable TCG with bytecode interpreter (TCI)
+  --oss-lib                path to OSS library
+  --cpu=CPU                Build for host CPU [$cpu]
   --with-coroutine=BACKEND coroutine backend. Supported options:
                            gthread, ucontext, sigaltstack, windows
-  --disable-coroutine-pool disable coroutine freelist (worse performance)
-  --enable-coroutine-pool  enable coroutine freelist (better performance)
-  --enable-glusterfs       enable GlusterFS backend
-  --disable-glusterfs      disable GlusterFS backend
-  --enable-archipelago     enable Archipelago backend
-  --disable-archipelago    disable Archipelago backend
   --enable-gcov            enable test coverage analysis with gcov
   --gcov=GCOV              use specified gcov [$gcov_tool]
-  --disable-tpm            disable TPM support
-  --enable-tpm             enable TPM support
-  --disable-libssh2        disable ssh block device support
-  --enable-libssh2         enable ssh block device support
-  --disable-vhdx           disable support for the Microsoft VHDX image format
-  --enable-vhdx            enable support for the Microsoft VHDX image format
-  --disable-quorum         disable quorum block filter support
-  --enable-quorum          enable quorum block filter support
-  --disable-numa           disable libnuma support
-  --enable-numa            enable libnuma support
-  --enable-android         enable Android support
+  --disable-blobs          disable installing provided firmware blobs
+  --with-vss-sdk=SDK-path  enable Windows VSS support in QEMU Guest Agent
+  --with-win-sdk=SDK-path  path to Windows Platform SDK (to build VSS .tlb)
+  --tls-priority           default TLS protocol/cipher priority string
+
+Optional features, enabled with --enable-FEATURE and
+disabled with --disable-FEATURE, default is enabled if available:
+
+  system          all system emulation targets
+  user            supported user emulation targets
+  linux-user      all linux usermode emulation targets
+  bsd-user        all BSD usermode emulation targets
+  docs            build documentation
+  guest-agent     build the QEMU Guest Agent
+  guest-agent-msi build guest agent Windows MSI installation package
+  pie             Position Independent Executables
+  modules         modules support
+  debug-tcg       TCG debugging (default is disabled)
+  debug-info      debugging information
+  sparse          sparse checker
+
+  gnutls          GNUTLS cryptography support
+  nettle          nettle cryptography support
+  gcrypt          libgcrypt cryptography support
+  sdl             SDL UI
+  --with-sdlabi     select preferred SDL ABI 1.2 or 2.0
+  gtk             gtk UI
+  --with-gtkabi     select preferred GTK ABI 2.0 or 3.0
+  vte             vte support for the gtk UI
+  curses          curses UI
+  vnc             VNC UI support
+  vnc-sasl        SASL encryption for VNC server
+  vnc-jpeg        JPEG lossy compression for VNC server
+  vnc-png         PNG compression for VNC server
+  cocoa           Cocoa UI (Mac OS X only)
+  virtfs          VirtFS
+  xen             xen backend driver support
+  xen-pci-passthrough
+  brlapi          BrlAPI (Braile)
+  curl            curl connectivity
+  fdt             fdt device tree
+  bluez           bluez stack connectivity
+  kvm             KVM acceleration support
+  hax             HAX acceleration support
+  rdma            RDMA-based migration support
+  uuid            uuid support
+  vde             support for vde network
+  netmap          support for netmap network
+  linux-aio       Linux AIO support
+  cap-ng          libcap-ng support
+  attr            attr and xattr support
+  vhost-net       vhost-net acceleration support
+  spice           spice
+  rbd             rados block device (rbd)
+  libiscsi        iscsi support
+  libnfs          nfs support
+  smartcard       smartcard support (libcacard)
+  libusb          libusb (for usb passthrough)
+  usb-redir       usb network redirection support
+  lzo             support of lzo compression library
+  snappy          support of snappy compression library
+  bzip2           support of bzip2 compression library
+                  (for reading bzip2-compressed dmg images)
+  seccomp         seccomp support
+  coroutine-pool  coroutine freelist (better performance)
+  glusterfs       GlusterFS backend
+  archipelago     Archipelago backend
+  tpm             TPM support
+  libssh2         ssh block device support
+  vhdx            support for the Microsoft VHDX image format
+  numa            libnuma support
+  tcmalloc        tcmalloc support
+  jemalloc        jemalloc support
 
 NOTE: The object files are built at the place where configure is launched
 EOF
@@ -1424,7 +1398,6 @@
 if test "$ARCH" = "unknown"; then
     if test "$tcg_interpreter" = "yes" ; then
         echo "Unsupported CPU = $cpu, will use TCG with TCI (experimental)"
-        ARCH=tci
     else
         error_exit "Unsupported CPU = $cpu, try --enable-tcg-interpreter"
     fi
@@ -1432,11 +1405,9 @@
 
 # Consult white-list to determine whether to enable werror
 # by default.  Only enable by default for git builds
-z_version=`cut -f3 -d. $source_path/VERSION`
-
 if test -z "$werror" ; then
     if test -d "$source_path/.git" -a \
-        "$linux" = "yes" ; then
+        \( "$linux" = "yes" -o "$mingw32" = "yes" \) ; then
         werror="yes"
     else
         werror="no"
@@ -1444,15 +1415,15 @@
 fi
 
 # check that the C compiler works.
-cat > $TMPC <<EOF
-int main(void) { return 0; }
-EOF
-
+write_c_skeleton;
 if compile_object ; then
   : C compiler works ok
 else
     error_exit "\"$cc\" either does not exist or does not work"
 fi
+if ! compile_prog ; then
+    error_exit "\"$cc\" cannot build an executable (is your linker broken?)"
+fi
 
 # Check that the C++ compiler exists and works with the C compiler
 if has $cxx; then
@@ -1488,27 +1459,41 @@
 gcc_flags="-Wold-style-declaration -Wold-style-definition -Wtype-limits"
 gcc_flags="-Wformat-security -Wformat-y2k -Winit-self -Wignored-qualifiers $gcc_flags"
 gcc_flags="-Wmissing-include-dirs -Wempty-body -Wnested-externs $gcc_flags"
-gcc_flags="-Wendif-labels $gcc_flags"
+gcc_flags="-Wendif-labels -Wno-shift-negative-value $gcc_flags"
 gcc_flags="-Wno-initializer-overrides $gcc_flags"
 gcc_flags="-Wno-string-plus-int $gcc_flags"
 # Note that we do not add -Werror to gcc_flags here, because that would
 # enable it for all configure tests. If a configure test failed due
 # to -Werror this would just silently disable some features,
 # so it's too error prone.
-cat > $TMPC << EOF
-int main(void) { return 0; }
-EOF
-for flag in $gcc_flags; do
+
+cc_has_warning_flag() {
+    write_c_skeleton;
+
     # Use the positive sense of the flag when testing for -Wno-wombat
     # support (gcc will happily accept the -Wno- form of unknown
     # warning options).
-    optflag="$(echo $flag | sed -e 's/^-Wno-/-W/')"
-    if compile_prog "-Werror $optflag" "" ; then
-	QEMU_CFLAGS="$QEMU_CFLAGS $flag"
+    optflag="$(echo $1 | sed -e 's/^-Wno-/-W/')"
+    compile_prog "-Werror $optflag" ""
+}
+
+for flag in $gcc_flags; do
+    if cc_has_warning_flag $flag ; then
+        QEMU_CFLAGS="$QEMU_CFLAGS $flag"
     fi
 done
 
 if test "$stack_protector" != "no"; then
+  cat > $TMPC << EOF
+int main(int argc, char *argv[])
+{
+    char arr[64], *p = arr, *c = argv[0];
+    while (*c) {
+        *p++ = *c++;
+    }
+    return 0;
+}
+EOF
   gcc_flags="-fstack-protector-strong -fstack-protector-all"
   sp_on=0
   for flag in $gcc_flags; do
@@ -1517,7 +1502,6 @@
     if do_cc $QEMU_CFLAGS -Werror $flag -c -o $TMPO $TMPC &&
        compile_prog "-Werror $flag" ""; then
       QEMU_CFLAGS="$QEMU_CFLAGS $flag"
-      LIBTOOLFLAGS="$LIBTOOLFLAGS -Wc,$flag"
       sp_on=1
       break
     fi
@@ -1555,6 +1539,17 @@
   fi
 fi
 
+# Unconditional check for compiler __thread support
+  cat > $TMPC << EOF
+static __thread int tls_var;
+int main(void) { return tls_var; }
+EOF
+
+if ! compile_prog "-Werror" "" ; then
+    error_exit "Your compiler does not support the __thread specifier for " \
+	"Thread-Local Storage (TLS). Please upgrade to a version that does."
+fi
+
 if test "$pie" = ""; then
   case "$cpu-$targetos" in
     i386-Linux|x86_64-Linux|x32-Linux|i386-OpenBSD|x86_64-OpenBSD)
@@ -1595,38 +1590,12 @@
     fi
   fi
 
-  if compile_prog "-fno-pie" "-nopie"; then
+  if compile_prog "-Werror -fno-pie" "-nopie"; then
     CFLAGS_NOPIE="-fno-pie"
     LDFLAGS_NOPIE="-nopie"
   fi
 fi
 
-# check for broken gcc and libtool in RHEL5
-if test -n "$libtool" -a "$pie" != "no" ; then
-  cat > $TMPC <<EOF
-
-void *f(unsigned char *buf, int len);
-void *g(unsigned char *buf, int len);
-
-void *
-f(unsigned char *buf, int len)
-{
-    return (void*)0L;
-}
-
-void *
-g(unsigned char *buf, int len)
-{
-    return f(buf, len);
-}
-
-EOF
-  if ! libtool_prog; then
-    echo "Disabling libtool due to broken toolchain support"
-    libtool=
-  fi
-fi
-
 ##########################################
 # __sync_fetch_and_and requires at least -march=i486. Many toolchains
 # use i686 as default anyway, but for those that don't, an explicit
@@ -1663,7 +1632,7 @@
         "install fileutils from www.blastwave.org using pkg-get -i fileutils" \
         "to get ginstall which is used by default (which lives in /opt/csw/bin)"
   fi
-  if test "`path_of $install`" = "/usr/sbin/install" ; then
+  if test "$(path_of $install)" = "/usr/sbin/install" ; then
     error_exit "Solaris /usr/sbin/install is not an appropriate install program." \
         "try ginstall from the GNU fileutils available from www.blastwave.org" \
         "using pkg-get -i fileutils, or use --install=/usr/ucb/install"
@@ -1682,7 +1651,7 @@
 if test -z "${target_list+xxx}" ; then
     target_list="$default_target_list"
 else
-    target_list=`echo "$target_list" | sed -e 's/,/ /g'`
+    target_list=$(echo "$target_list" | sed -e 's/,/ /g')
 fi
 
 # Check that we recognised the target name; this allows a more
@@ -1738,6 +1707,21 @@
 fi
 
 ##########################################
+# cocoa implies not SDL or GTK
+# (the cocoa UI code currently assumes it is always the active UI
+# and doesn't interact well with other UI frontend code)
+if test "$cocoa" = "yes"; then
+    if test "$sdl" = "yes"; then
+        error_exit "Cocoa and SDL UIs cannot both be enabled at once"
+    fi
+    if test "$gtk" = "yes"; then
+        error_exit "Cocoa and GTK UIs cannot both be enabled at once"
+    fi
+    gtk=no
+    sdl=no
+fi
+
+##########################################
 # L2TPV3 probe
 
 cat > $TMPC <<EOF
@@ -1752,6 +1736,37 @@
 fi
 
 ##########################################
+# MinGW / Mingw-w64 localtime_r/gmtime_r check
+
+if test "$mingw32" = "yes"; then
+    # Some versions of MinGW / Mingw-w64 lack localtime_r
+    # and gmtime_r entirely.
+    #
+    # Some versions of Mingw-w64 define a macro for
+    # localtime_r/gmtime_r.
+    #
+    # Some versions of Mingw-w64 will define functions
+    # for localtime_r/gmtime_r, but only if you have
+    # _POSIX_THREAD_SAFE_FUNCTIONS defined. For fun
+    # though, unistd.h and pthread.h both define
+    # that for you.
+    #
+    # So this #undef localtime_r and #include <unistd.h>
+    # are not in fact redundant.
+cat > $TMPC << EOF
+#include <unistd.h>
+#include <time.h>
+#undef localtime_r
+int main(void) { localtime_r(NULL, NULL); return 0; }
+EOF
+    if compile_prog "" "" ; then
+        localtime_r="yes"
+    else
+        localtime_r="no"
+    fi
+fi
+
+##########################################
 # pkg-config probe
 
 if ! has "$pkg_config_exe"; then
@@ -1778,6 +1793,33 @@
 fi
 
 ##########################################
+# avx2 optimization requirement check
+
+
+if test "$static" = "no" ; then
+  cat > $TMPC << EOF
+#pragma GCC push_options
+#pragma GCC target("avx2")
+#include <cpuid.h>
+#include <immintrin.h>
+
+static int bar(void *a) {
+    return _mm256_movemask_epi8(_mm256_cmpeq_epi8(*(__m256i *)a, (__m256i){0}));
+}
+static void *bar_ifunc(void) {return (void*) bar;}
+int foo(void *a) __attribute__((ifunc("bar_ifunc")));
+int main(int argc, char *argv[]) { return foo(argv[0]);}
+EOF
+  if compile_object "" ; then
+      if has readelf; then
+          if readelf --syms $TMPO 2>/dev/null |grep -q "IFUNC.*foo"; then
+              avx2_opt="yes"
+          fi
+      fi
+  fi
+fi
+
+#########################################
 # zlib check
 
 if test "$zlib" != "no" ; then
@@ -1833,19 +1875,61 @@
 fi
 
 ##########################################
+# bzip2 check
+
+if test "$bzip2" != "no" ; then
+    cat > $TMPC << EOF
+#include <bzlib.h>
+int main(void) { BZ2_bzlibVersion(); return 0; }
+EOF
+    if compile_prog "" "-lbz2" ; then
+        bzip2="yes"
+    else
+        if test "$bzip2" = "yes"; then
+            feature_not_found "libbzip2" "Install libbzip2 devel"
+        fi
+        bzip2="no"
+    fi
+fi
+
+##########################################
 # libseccomp check
 
 if test "$seccomp" != "no" ; then
-    if test "$cpu" = "i386" || test "$cpu" = "x86_64" &&
-        $pkg_config --atleast-version=2.1.1 libseccomp; then
-        libs_softmmu="$libs_softmmu `$pkg_config --libs libseccomp`"
-        QEMU_CFLAGS="$QEMU_CFLAGS `$pkg_config --cflags libseccomp`"
-	seccomp="yes"
+    case "$cpu" in
+    i386|x86_64)
+        libseccomp_minver="2.1.0"
+        ;;
+    mips)
+        libseccomp_minver="2.2.0"
+        ;;
+    arm|aarch64)
+        libseccomp_minver="2.2.3"
+        ;;
+    ppc|ppc64)
+        libseccomp_minver="2.3.0"
+        ;;
+    *)
+        libseccomp_minver=""
+        ;;
+    esac
+
+    if test "$libseccomp_minver" != "" &&
+       $pkg_config --atleast-version=$libseccomp_minver libseccomp ; then
+        libs_softmmu="$libs_softmmu $($pkg_config --libs libseccomp)"
+        QEMU_CFLAGS="$QEMU_CFLAGS $($pkg_config --cflags libseccomp)"
+        seccomp="yes"
     else
-	if test "$seccomp" = "yes"; then
-            feature_not_found "libseccomp" "Install libseccomp devel >= 2.1.0"
-	fi
-	seccomp="no"
+        if test "$seccomp" = "yes" ; then
+            if test "$libseccomp_minver" != "" ; then
+                feature_not_found "libseccomp" \
+                    "Install libseccomp devel >= $libseccomp_minver"
+            else
+                feature_not_found "libseccomp" \
+                    "libseccomp is not supported for host cpu $cpu"
+            fi
+        fi
+        seccomp="no"
     fi
 fi
 ##########################################
@@ -1853,6 +1937,7 @@
 
 if test "$xen" != "no" ; then
   xen_libs="-lxenstore -lxenctrl -lxenguest"
+  xen_stable_libs="-lxenforeignmemory -lxengnttab -lxenevtchn"
 
   # First we test whether Xen headers and libraries are available.
   # If no, we are done and there is no Xen support.
@@ -1875,6 +1960,128 @@
   # Xen unstable
   elif
       cat > $TMPC <<EOF &&
+/*
+ * If we have stable libs the we don't want the libxc compat
+ * layers, regardless of what CFLAGS we may have been given.
+ */
+#undef XC_WANT_COMPAT_EVTCHN_API
+#undef XC_WANT_COMPAT_GNTTAB_API
+#undef XC_WANT_COMPAT_MAP_FOREIGN_API
+#include <xenctrl.h>
+#include <xenstore.h>
+#include <xenevtchn.h>
+#include <xengnttab.h>
+#include <xenforeignmemory.h>
+#include <stdint.h>
+#include <xen/hvm/hvm_info_table.h>
+#if !defined(HVM_MAX_VCPUS)
+# error HVM_MAX_VCPUS not defined
+#endif
+int main(void) {
+  xc_interface *xc = NULL;
+  xenforeignmemory_handle *xfmem;
+  xenevtchn_handle *xe;
+  xengnttab_handle *xg;
+  xen_domain_handle_t handle;
+
+  xs_daemon_open();
+
+  xc = xc_interface_open(0, 0, 0);
+  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
+  xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
+  xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
+  xc_hvm_create_ioreq_server(xc, 0, HVM_IOREQSRV_BUFIOREQ_ATOMIC, NULL);
+  xc_domain_create(xc, 0, handle, 0, NULL, NULL);
+
+  xfmem = xenforeignmemory_open(0, 0);
+  xenforeignmemory_map(xfmem, 0, 0, 0, 0, 0);
+
+  xe = xenevtchn_open(0, 0);
+  xenevtchn_fd(xe);
+
+  xg = xengnttab_open(0, 0);
+  xengnttab_map_grant_ref(xg, 0, 0, 0);
+
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs $xen_stable_libs"
+    then
+    xen_ctrl_version=471
+    xen=yes
+  elif
+      cat > $TMPC <<EOF &&
+#include <xenctrl.h>
+#include <stdint.h>
+int main(void) {
+  xc_interface *xc = NULL;
+  xen_domain_handle_t handle;
+  xc_domain_create(xc, 0, handle, 0, NULL, NULL);
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs"
+    then
+    xen_ctrl_version=470
+    xen=yes
+
+  # Xen 4.6
+  elif
+      cat > $TMPC <<EOF &&
+#include <xenctrl.h>
+#include <xenstore.h>
+#include <stdint.h>
+#include <xen/hvm/hvm_info_table.h>
+#if !defined(HVM_MAX_VCPUS)
+# error HVM_MAX_VCPUS not defined
+#endif
+int main(void) {
+  xc_interface *xc;
+  xs_daemon_open();
+  xc = xc_interface_open(0, 0, 0);
+  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
+  xc_gnttab_open(NULL, 0);
+  xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
+  xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
+  xc_hvm_create_ioreq_server(xc, 0, HVM_IOREQSRV_BUFIOREQ_ATOMIC, NULL);
+  xc_reserved_device_memory_map(xc, 0, 0, 0, 0, NULL, 0);
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs"
+    then
+    xen_ctrl_version=460
+    xen=yes
+
+  # Xen 4.5
+  elif
+      cat > $TMPC <<EOF &&
+#include <xenctrl.h>
+#include <xenstore.h>
+#include <stdint.h>
+#include <xen/hvm/hvm_info_table.h>
+#if !defined(HVM_MAX_VCPUS)
+# error HVM_MAX_VCPUS not defined
+#endif
+int main(void) {
+  xc_interface *xc;
+  xs_daemon_open();
+  xc = xc_interface_open(0, 0, 0);
+  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
+  xc_gnttab_open(NULL, 0);
+  xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
+  xc_hvm_inject_msi(xc, 0, 0xf0000000, 0x00000000);
+  xc_hvm_create_ioreq_server(xc, 0, 0, NULL);
+  return 0;
+}
+EOF
+      compile_prog "" "$xen_libs"
+    then
+    xen_ctrl_version=450
+    xen=yes
+
+  elif
+      cat > $TMPC <<EOF &&
 #include <xenctrl.h>
 #include <xenstore.h>
 #include <stdint.h>
@@ -1898,119 +2105,27 @@
     xen_ctrl_version=420
     xen=yes
 
-  elif
-      cat > $TMPC <<EOF &&
-#include <xenctrl.h>
-#include <xs.h>
-#include <stdint.h>
-#include <xen/hvm/hvm_info_table.h>
-#if !defined(HVM_MAX_VCPUS)
-# error HVM_MAX_VCPUS not defined
-#endif
-int main(void) {
-  xs_daemon_open();
-  xc_interface_open(0, 0, 0);
-  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  xc_gnttab_open(NULL, 0);
-  xc_domain_add_to_physmap(0, 0, XENMAPSPACE_gmfn, 0, 0);
-  return 0;
-}
-EOF
-      compile_prog "" "$xen_libs"
-    then
-    xen_ctrl_version=410
-    xen=yes
-
-  # Xen 4.0.0
-  elif
-      cat > $TMPC <<EOF &&
-#include <xenctrl.h>
-#include <xs.h>
-#include <stdint.h>
-#include <xen/hvm/hvm_info_table.h>
-#if !defined(HVM_MAX_VCPUS)
-# error HVM_MAX_VCPUS not defined
-#endif
-int main(void) {
-  struct xen_add_to_physmap xatp = {
-    .domid = 0, .space = XENMAPSPACE_gmfn, .idx = 0, .gpfn = 0,
-  };
-  xs_daemon_open();
-  xc_interface_open();
-  xc_gnttab_open();
-  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  xc_memory_op(0, XENMEM_add_to_physmap, &xatp);
-  return 0;
-}
-EOF
-      compile_prog "" "$xen_libs"
-    then
-    xen_ctrl_version=400
-    xen=yes
-
-  # Xen 3.4.0
-  elif
-      cat > $TMPC <<EOF &&
-#include <xenctrl.h>
-#include <xs.h>
-int main(void) {
-  struct xen_add_to_physmap xatp = {
-    .domid = 0, .space = XENMAPSPACE_gmfn, .idx = 0, .gpfn = 0,
-  };
-  xs_daemon_open();
-  xc_interface_open();
-  xc_gnttab_open();
-  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  xc_memory_op(0, XENMEM_add_to_physmap, &xatp);
-  return 0;
-}
-EOF
-      compile_prog "" "$xen_libs"
-    then
-    xen_ctrl_version=340
-    xen=yes
-
-  # Xen 3.3.0
-  elif
-      cat > $TMPC <<EOF &&
-#include <xenctrl.h>
-#include <xs.h>
-int main(void) {
-  xs_daemon_open();
-  xc_interface_open();
-  xc_gnttab_open();
-  xc_hvm_set_mem_type(0, 0, HVMMEM_ram_ro, 0, 0);
-  return 0;
-}
-EOF
-      compile_prog "" "$xen_libs"
-    then
-    xen_ctrl_version=330
-    xen=yes
-
-  # Xen version unsupported
   else
     if test "$xen" = "yes" ; then
-      feature_not_found "xen (unsupported version)" "Install supported xen (e.g. 4.0, 3.4, 3.3)"
+      feature_not_found "xen (unsupported version)" \
+                        "Install a supported xen (xen 4.2 or newer)"
     fi
     xen=no
   fi
 
   if test "$xen" = yes; then
+    if test $xen_ctrl_version -ge 471  ; then
+      libs_softmmu="$xen_stable_libs $libs_softmmu"
+    fi
     libs_softmmu="$xen_libs $libs_softmmu"
   fi
 fi
 
 if test "$xen_pci_passthrough" != "no"; then
-  if test "$xen" = "yes" && test "$linux" = "yes" &&
-    test "$xen_ctrl_version" -ge 340; then
+  if test "$xen" = "yes" && test "$linux" = "yes"; then
     xen_pci_passthrough=yes
   else
     if test "$xen_pci_passthrough" = "yes"; then
-      if test "$xen_ctrl_version" -lt 340; then
-        error_exit "User requested feature Xen PCI Passthrough" \
-            "This feature does not work with Xen 3.3"
-      fi
       error_exit "User requested feature Xen PCI Passthrough" \
           " but this feature requires /sys from Linux"
     fi
@@ -2018,19 +2133,10 @@
   fi
 fi
 
-##########################################
-# libtool probe
-
-if ! has $libtool; then
-    libtool=
-fi
-
-# MacOSX ships with a libtool which isn't the GNU one; weed this
-# out by checking whether libtool supports the --version switch
-if test -n "$libtool"; then
-  if ! "$libtool" --version >/dev/null 2>&1; then
-    libtool=
-  fi
+if test "$xen_pv_domain_build" = "yes" &&
+   test "$xen" != "yes"; then
+    error_exit "User requested Xen PV domain builder support" \
+	       "which requires Xen support."
 fi
 
 ##########################################
@@ -2047,6 +2153,15 @@
 fi
 
 ##########################################
+# X11 probe
+x11_cflags=
+x11_libs=-lX11
+if $pkg_config --exists "x11"; then
+    x11_cflags=$($pkg_config --cflags x11)
+    x11_libs=$($pkg_config --libs x11)
+fi
+
+##########################################
 # GTK probe
 
 if test "$gtkabi" = ""; then
@@ -2070,10 +2185,12 @@
       gtkversion="2.18.0"
     fi
     if $pkg_config --exists "$gtkpackage >= $gtkversion"; then
-        gtk_cflags=`$pkg_config --cflags $gtkpackage`
-        gtk_libs=`$pkg_config --libs $gtkpackage`
+        gtk_cflags=$($pkg_config --cflags $gtkpackage)
+        gtk_libs=$($pkg_config --libs $gtkpackage)
+        gtk_version=$($pkg_config --modversion $gtkpackage)
         if $pkg_config --exists "$gtkx11package >= $gtkversion"; then
-            gtk_libs="$gtk_libs -lX11"
+            gtk_cflags="$gtk_cflags $x11_cflags"
+            gtk_libs="$gtk_libs $x11_libs"
         fi
         libs_softmmu="$gtk_libs $libs_softmmu"
         gtk="yes"
@@ -2084,25 +2201,239 @@
     fi
 fi
 
+
+##########################################
+# GNUTLS probe
+
+gnutls_works() {
+    # Unfortunately some distros have bad pkg-config information for gnutls
+    # such that it claims to exist but you get a compiler error if you try
+    # to use the options returned by --libs. Specifically, Ubuntu for --static
+    # builds doesn't work:
+    # https://bugs.launchpad.net/ubuntu/+source/gnutls26/+bug/1478035
+    #
+    # So sanity check the cflags/libs before assuming gnutls can be used.
+    if ! $pkg_config --exists "gnutls"; then
+        return 1
+    fi
+
+    write_c_skeleton
+    compile_prog "$($pkg_config --cflags gnutls)" "$($pkg_config --libs gnutls)"
+}
+
+gnutls_gcrypt=no
+gnutls_nettle=no
+if test "$gnutls" != "no"; then
+    if gnutls_works; then
+        gnutls_cflags=$($pkg_config --cflags gnutls)
+        gnutls_libs=$($pkg_config --libs gnutls)
+        libs_softmmu="$gnutls_libs $libs_softmmu"
+        libs_tools="$gnutls_libs $libs_tools"
+	QEMU_CFLAGS="$QEMU_CFLAGS $gnutls_cflags"
+        gnutls="yes"
+
+	# gnutls_rnd requires >= 2.11.0
+	if $pkg_config --exists "gnutls >= 2.11.0"; then
+	    gnutls_rnd="yes"
+	else
+	    gnutls_rnd="no"
+	fi
+
+	if $pkg_config --exists 'gnutls >= 3.0'; then
+	    gnutls_gcrypt=no
+	    gnutls_nettle=yes
+	elif $pkg_config --exists 'gnutls >= 2.12'; then
+	    case $($pkg_config --libs --static gnutls) in
+		*gcrypt*)
+		    gnutls_gcrypt=yes
+		    gnutls_nettle=no
+		    ;;
+		*nettle*)
+		    gnutls_gcrypt=no
+		    gnutls_nettle=yes
+		    ;;
+		*)
+		    gnutls_gcrypt=yes
+		    gnutls_nettle=no
+		    ;;
+	    esac
+	else
+	    gnutls_gcrypt=yes
+	    gnutls_nettle=no
+	fi
+    elif test "$gnutls" = "yes"; then
+	feature_not_found "gnutls" "Install gnutls devel"
+    else
+        gnutls="no"
+        gnutls_rnd="no"
+    fi
+else
+    gnutls_rnd="no"
+fi
+
+
+# If user didn't give a --disable/enable-gcrypt flag,
+# then mark as disabled if user requested nettle
+# explicitly, or if gnutls links to nettle
+if test -z "$gcrypt"
+then
+    if test "$nettle" = "yes" || test "$gnutls_nettle" = "yes"
+    then
+        gcrypt="no"
+    fi
+fi
+
+# If user didn't give a --disable/enable-nettle flag,
+# then mark as disabled if user requested gcrypt
+# explicitly, or if gnutls links to gcrypt
+if test -z "$nettle"
+then
+    if test "$gcrypt" = "yes" || test "$gnutls_gcrypt" = "yes"
+    then
+        nettle="no"
+    fi
+fi
+
+has_libgcrypt_config() {
+    if ! has "libgcrypt-config"
+    then
+	return 1
+    fi
+
+    if test -n "$cross_prefix"
+    then
+	host=$(libgcrypt-config --host)
+	if test "$host-" != $cross_prefix
+	then
+	    return 1
+	fi
+    fi
+
+    return 0
+}
+
+if test "$gcrypt" != "no"; then
+    if has_libgcrypt_config; then
+        gcrypt_cflags=$(libgcrypt-config --cflags)
+        gcrypt_libs=$(libgcrypt-config --libs)
+        # Debian has remove -lgpg-error from libgcrypt-config
+        # as it "spreads unnecessary dependencies" which in
+        # turn breaks static builds...
+        if test "$static" = "yes"
+        then
+            gcrypt_libs="$gcrypt_libs -lgpg-error"
+        fi
+        libs_softmmu="$gcrypt_libs $libs_softmmu"
+        libs_tools="$gcrypt_libs $libs_tools"
+        QEMU_CFLAGS="$QEMU_CFLAGS $gcrypt_cflags"
+        gcrypt="yes"
+        if test -z "$nettle"; then
+           nettle="no"
+        fi
+
+        cat > $TMPC << EOF
+#include <gcrypt.h>
+int main(void) {
+  gcry_kdf_derive(NULL, 0, GCRY_KDF_PBKDF2,
+                  GCRY_MD_SHA256,
+                  NULL, 0, 0, 0, NULL);
+ return 0;
+}
+EOF
+        if compile_prog "$gcrypt_cflags" "$gcrypt_libs" ; then
+            gcrypt_kdf=yes
+        fi
+    else
+        if test "$gcrypt" = "yes"; then
+            feature_not_found "gcrypt" "Install gcrypt devel"
+        else
+            gcrypt="no"
+        fi
+    fi
+fi
+
+
+if test "$nettle" != "no"; then
+    if $pkg_config --exists "nettle"; then
+        nettle_cflags=$($pkg_config --cflags nettle)
+        nettle_libs=$($pkg_config --libs nettle)
+        nettle_version=$($pkg_config --modversion nettle)
+        libs_softmmu="$nettle_libs $libs_softmmu"
+        libs_tools="$nettle_libs $libs_tools"
+        QEMU_CFLAGS="$QEMU_CFLAGS $nettle_cflags"
+        nettle="yes"
+
+        cat > $TMPC << EOF
+#include <stddef.h>
+#include <nettle/pbkdf2.h>
+int main(void) {
+     pbkdf2_hmac_sha256(8, NULL, 1000, 8, NULL, 8, NULL);
+     return 0;
+}
+EOF
+        if compile_prog "$nettle_cflags" "$nettle_libs" ; then
+            nettle_kdf=yes
+        fi
+    else
+        if test "$nettle" = "yes"; then
+            feature_not_found "nettle" "Install nettle devel"
+        else
+            nettle="no"
+        fi
+    fi
+fi
+
+if test "$gcrypt" = "yes" && test "$nettle" = "yes"
+then
+    error_exit "Only one of gcrypt & nettle can be enabled"
+fi
+
+##########################################
+# libtasn1 - only for the TLS creds/session test suite
+
+tasn1=yes
+tasn1_cflags=""
+tasn1_libs=""
+if $pkg_config --exists "libtasn1"; then
+    tasn1_cflags=$($pkg_config --cflags libtasn1)
+    tasn1_libs=$($pkg_config --libs libtasn1)
+else
+    tasn1=no
+fi
+
+
+##########################################
+# getifaddrs (for tests/test-io-channel-socket )
+
+have_ifaddrs_h=yes
+if ! check_include "ifaddrs.h" ; then
+  have_ifaddrs_h=no
+fi
+
 ##########################################
 # VTE probe
 
 if test "$vte" != "no"; then
     if test "$gtkabi" = "3.0"; then
-      vtepackage="vte-2.90"
-      vteversion="0.32.0"
+      vteminversion="0.32.0"
+      if $pkg_config --exists "vte-2.91"; then
+        vtepackage="vte-2.91"
+      else
+        vtepackage="vte-2.90"
+      fi
     else
       vtepackage="vte"
-      vteversion="0.24.0"
+      vteminversion="0.24.0"
     fi
-    if $pkg_config --exists "$vtepackage >= $vteversion"; then
-        vte_cflags=`$pkg_config --cflags $vtepackage`
-        vte_libs=`$pkg_config --libs $vtepackage`
+    if $pkg_config --exists "$vtepackage >= $vteminversion"; then
+        vte_cflags=$($pkg_config --cflags $vtepackage)
+        vte_libs=$($pkg_config --libs $vtepackage)
+        vteversion=$($pkg_config --modversion $vtepackage)
         libs_softmmu="$vte_libs $libs_softmmu"
         vte="yes"
     elif test "$vte" = "yes"; then
         if test "$gtkabi" = "3.0"; then
-            feature_not_found "vte" "Install libvte-2.90 devel"
+            feature_not_found "vte" "Install libvte-2.90/2.91 devel"
         else
             feature_not_found "vte" "Install libvte devel"
         fi
@@ -2117,25 +2448,37 @@
 # Look for sdl configuration program (pkg-config or sdl-config).  Try
 # sdl-config even without cross prefix, and favour pkg-config over sdl-config.
 
+if test "$sdlabi" = ""; then
+    if $pkg_config --exists "sdl"; then
+        sdlabi=1.2
+    elif $pkg_config --exists "sdl2"; then
+        sdlabi=2.0
+    else
+        sdlabi=1.2
+    fi
+fi
+
 if test $sdlabi = "2.0"; then
     sdl_config=$sdl2_config
     sdlname=sdl2
     sdlconfigname=sdl2_config
-else
+elif test $sdlabi = "1.2"; then
     sdlname=sdl
     sdlconfigname=sdl_config
+else
+    error_exit "Unknown sdlabi $sdlabi, must be 1.2 or 2.0"
 fi
 
-if test "`basename $sdl_config`" != $sdlconfigname && ! has ${sdl_config}; then
+if test "$(basename $sdl_config)" != $sdlconfigname && ! has ${sdl_config}; then
   sdl_config=$sdlconfigname
 fi
 
 if $pkg_config $sdlname --exists; then
   sdlconfig="$pkg_config $sdlname"
-  _sdlversion=`$sdlconfig --modversion 2>/dev/null | sed 's/[^0-9]//g'`
+  sdlversion=$($sdlconfig --modversion 2>/dev/null)
 elif has ${sdl_config}; then
   sdlconfig="$sdl_config"
-  _sdlversion=`$sdlconfig --version | sed 's/[^0-9]//g'`
+  sdlversion=$($sdlconfig --version)
 else
   if test "$sdl" = "yes" ; then
     feature_not_found "sdl" "Install SDL devel"
@@ -2153,26 +2496,24 @@
 #undef main /* We don't want SDL to override our main() */
 int main( void ) { return SDL_Init (SDL_INIT_VIDEO); }
 EOF
-  sdl_cflags=`$sdlconfig --cflags 2> /dev/null`
+  sdl_cflags=$($sdlconfig --cflags 2>/dev/null)
   if test "$static" = "yes" ; then
-    sdl_libs=`$sdlconfig --static-libs 2>/dev/null`
+    sdl_libs=$($sdlconfig --static-libs 2>/dev/null)
   else
-    sdl_libs=`$sdlconfig --libs 2> /dev/null`
+    sdl_libs=$($sdlconfig --libs 2>/dev/null)
   fi
   if compile_prog "$sdl_cflags" "$sdl_libs" ; then
-    if test "$_sdlversion" -lt 121 ; then
+    if test $(echo $sdlversion | sed 's/[^0-9]//g') -lt 121 ; then
       sdl_too_old=yes
     else
-      if test "$cocoa" = "no" ; then
-        sdl=yes
-      fi
+      sdl=yes
     fi
 
     # static link with sdl ? (note: sdl.pc's --static --libs is broken)
     if test "$sdl" = "yes" -a "$static" = "yes" ; then
       if test $? = 0 && echo $sdl_libs | grep -- -laa > /dev/null; then
-         sdl_libs="$sdl_libs `aalib-config --static-libs 2>/dev/null`"
-         sdl_cflags="$sdl_cflags `aalib-config --cflags 2>/dev/null`"
+         sdl_libs="$sdl_libs $(aalib-config --static-libs 2>/dev/null)"
+         sdl_cflags="$sdl_cflags $(aalib-config --cflags 2>/dev/null)"
       fi
       if compile_prog "$sdl_cflags" "$sdl_libs" ; then
 	:
@@ -2198,8 +2539,9 @@
 #endif
 int main(void) { return 0; }
 EOF
-  if compile_prog "$sdl_cflags" "$sdl_libs" ; then
-    sdl_libs="$sdl_libs -lX11"
+  if compile_prog "$sdl_cflags $x11_cflags" "$sdl_libs $x11_libs" ; then
+    sdl_cflags="$sdl_cflags $x11_cflags"
+    sdl_libs="$sdl_libs $x11_libs"
   fi
   libs_softmmu="$sdl_libs $libs_softmmu"
 fi
@@ -2228,62 +2570,6 @@
   fi
 fi
 
-##########################################
-# VNC TLS/WS detection
-if test "$vnc" = "yes" -a \( "$vnc_tls" != "no" -o "$vnc_ws" != "no" \) ; then
-  cat > $TMPC <<EOF
-#include <gnutls/gnutls.h>
-int main(void) { gnutls_session_t s; gnutls_init(&s, GNUTLS_SERVER); return 0; }
-EOF
-  vnc_tls_cflags=`$pkg_config --cflags gnutls 2> /dev/null`
-  vnc_tls_libs=`$pkg_config --libs gnutls 2> /dev/null`
-  if compile_prog "$vnc_tls_cflags" "$vnc_tls_libs" ; then
-    if test "$vnc_tls" != "no" ; then
-      vnc_tls=yes
-    fi
-    if test "$vnc_ws" != "no" ; then
-      vnc_ws=yes
-    fi
-    libs_softmmu="$vnc_tls_libs $libs_softmmu"
-    QEMU_CFLAGS="$QEMU_CFLAGS $vnc_tls_cflags"
-  else
-    if test "$vnc_tls" = "yes" ; then
-      feature_not_found "vnc-tls" "Install gnutls devel"
-    fi
-    if test "$vnc_ws" = "yes" ; then
-      feature_not_found "vnc-ws" "Install gnutls devel"
-    fi
-    vnc_tls=no
-    vnc_ws=no
-  fi
-fi
-
-##########################################
-# Quorum probe (check for gnutls)
-if test "$quorum" != "no" ; then
-cat > $TMPC <<EOF
-#include <gnutls/gnutls.h>
-#include <gnutls/crypto.h>
-int main(void) {char data[4096], digest[32];
-gnutls_hash_fast(GNUTLS_DIG_SHA256, data, 4096, digest);
-return 0;
-}
-EOF
-quorum_tls_cflags=`$pkg_config --cflags gnutls 2> /dev/null`
-quorum_tls_libs=`$pkg_config --libs gnutls 2> /dev/null`
-if compile_prog "$quorum_tls_cflags" "$quorum_tls_libs" ; then
-  qcow_tls=yes
-  libs_softmmu="$quorum_tls_libs $libs_softmmu"
-  libs_tools="$quorum_tls_libs $libs_softmmu"
-  QEMU_CFLAGS="$QEMU_CFLAGS $quorum_tls_cflags"
-  quorum="yes"
-else
-  if test "$quorum" = "yes"; then
-    feature_not_found "gnutls" "gnutls > 2.10.0 required to compile Quorum"
-  fi
-  quorum="no"
-fi
-fi
 
 ##########################################
 # VNC SASL detection
@@ -2344,8 +2630,8 @@
 }
 EOF
   if $pkg_config libpng --exists; then
-    vnc_png_cflags=`$pkg_config libpng --cflags`
-    vnc_png_libs=`$pkg_config libpng --libs`
+    vnc_png_cflags=$($pkg_config libpng --cflags)
+    vnc_png_libs=$($pkg_config libpng --libs)
   else
     vnc_png_cflags=""
     vnc_png_libs="-lpng"
@@ -2539,7 +2825,7 @@
     fi
 }
 
-audio_drv_list=`echo "$audio_drv_list" | sed -e 's/,/ /g'`
+audio_drv_list=$(echo "$audio_drv_list" | sed -e 's/,/ /g')
 for drv in $audio_drv_list; do
     case $drv in
     alsa)
@@ -2548,21 +2834,6 @@
     libs_softmmu="-lasound $libs_softmmu"
     ;;
 
-    fmod)
-    if test -z $fmod_lib || test -z $fmod_inc; then
-        error_exit "You must specify path to FMOD library and headers" \
-            "Example: --fmod-inc=/path/include/fmod --fmod-lib=/path/lib/libfmod-3.74.so"
-    fi
-    audio_drv_probe $drv fmod.h $fmod_lib "return FSOUND_GetVersion();" "-I $fmod_inc"
-    libs_softmmu="$fmod_lib $libs_softmmu"
-    ;;
-
-    esd)
-    audio_drv_probe $drv esd.h -lesd 'return esd_play_stream(0, 0, "", 0);'
-    libs_softmmu="-lesd $libs_softmmu"
-    audio_pt_int="yes"
-    ;;
-
     pa)
     audio_drv_probe $drv pulse/mainloop.h "-lpulse" \
         "pa_mainloop *m = 0; pa_mainloop_free (m); return 0;"
@@ -2579,6 +2850,10 @@
       audio_win_int="yes"
     ;;
 
+    winaudio)
+        libs_softmmu="-lwinmm $libs_softmmu"
+    ;;
+
     oss)
       libs_softmmu="$oss_lib $libs_softmmu"
     ;;
@@ -2587,15 +2862,6 @@
     # XXX: Probes for CoreAudio, DirectSound, SDL(?)
     ;;
 
-    winwave)
-      libs_softmmu="-lwinmm $libs_softmmu"
-      audio_win_int="yes"
-    ;;
-
-    winaudio)
-      libs_softmmu="-lwinmm $libs_softmmu"
-    ;;
-
     *)
     echo "$audio_possible_drivers" | grep -q "\<$drv\>" || {
         error_exit "Unknown driver '$drv' selected" \
@@ -2630,7 +2896,7 @@
 # curses probe
 if test "$curses" != "no" ; then
   if test "$mingw32" = "yes" ; then
-    curses_list="-lpdcurses"
+    curses_list="$($pkg_config --libs ncurses 2>/dev/null):-lpdcurses"
   else
     curses_list="$($pkg_config --libs ncurses 2>/dev/null):-lncurses:-lcurses"
   fi
@@ -2675,8 +2941,8 @@
 #include <curl/curl.h>
 int main(void) { curl_easy_init(); curl_multi_setopt(0, 0, 0); return 0; }
 EOF
-  curl_cflags=`$curlconfig --cflags 2>/dev/null`
-  curl_libs=`$curlconfig --libs 2>/dev/null`
+  curl_cflags=$($curlconfig --cflags 2>/dev/null)
+  curl_libs=$($curlconfig --libs 2>/dev/null)
   if compile_prog "$curl_cflags" "$curl_libs" ; then
     curl=yes
   else
@@ -2694,8 +2960,8 @@
 #include <bluetooth/bluetooth.h>
 int main(void) { return bt_error(0); }
 EOF
-  bluez_cflags=`$pkg_config --cflags bluez 2> /dev/null`
-  bluez_libs=`$pkg_config --libs bluez 2> /dev/null`
+  bluez_cflags=$($pkg_config --cflags bluez 2>/dev/null)
+  bluez_libs=$($pkg_config --libs bluez 2>/dev/null)
   if compile_prog "$bluez_cflags" "$bluez_libs" ; then
     bluez=yes
     libs_softmmu="$bluez_libs $libs_softmmu"
@@ -2710,12 +2976,7 @@
 ##########################################
 # glib support probe
 
-if test "$mingw32" = yes; then
-    # g_poll is required in order to integrate with the glib main loop.
-    glib_req_ver=2.20
-else
-    glib_req_ver=2.12
-fi
+glib_req_ver=2.22
 glib_modules=gthread-2.0
 if test "$modules" = yes; then
     glib_modules="$glib_modules gmodule-2.0"
@@ -2723,8 +2984,8 @@
 
 for i in $glib_modules; do
     if $pkg_config --atleast-version=$glib_req_ver $i; then
-        glib_cflags=`$pkg_config --cflags $i`
-        glib_libs=`$pkg_config --libs $i`
+        glib_cflags=$($pkg_config --cflags $i)
+        glib_libs=$($pkg_config --libs $i)
         CFLAGS="$glib_cflags $CFLAGS"
         LIBS="$glib_libs $LIBS"
         libs_qga="$glib_libs $libs_qga"
@@ -2733,18 +2994,54 @@
     fi
 done
 
+# Sanity check that the current size_t matches the
+# size that glib thinks it should be. This catches
+# problems on multi-arch where people try to build
+# 32-bit QEMU while pointing at 64-bit glib headers
+cat > $TMPC <<EOF
+#include <glib.h>
+#include <unistd.h>
+
+#define QEMU_BUILD_BUG_ON(x) \
+  typedef char qemu_build_bug_on[(x)?-1:1] __attribute__((unused));
+
+int main(void) {
+   QEMU_BUILD_BUG_ON(sizeof(size_t) != GLIB_SIZEOF_SIZE_T);
+   return 0;
+}
+EOF
+
+if ! compile_prog "$CFLAGS" "$LIBS" ; then
+    error_exit "sizeof(size_t) doesn't match GLIB_SIZEOF_SIZE_T."\
+               "You probably need to set PKG_CONFIG_LIBDIR"\
+	       "to point to the right pkg-config files for your"\
+	       "build target"
+fi
+
 # g_test_trap_subprocess added in 2.38. Used by some tests.
 glib_subprocess=yes
 if ! $pkg_config --atleast-version=2.38 glib-2.0; then
     glib_subprocess=no
 fi
 
+# Silence clang 3.5.0 warnings about glib attribute __alloc_size__ usage
+cat > $TMPC << EOF
+#include <glib.h>
+int main(void) { return 0; }
+EOF
+if ! compile_prog "$glib_cflags -Werror" "$glib_libs" ; then
+    if cc_has_warning_flag "-Wno-unknown-attributes"; then
+        glib_cflags="-Wno-unknown-attributes $glib_cflags"
+        CFLAGS="-Wno-unknown-attributes $CFLAGS"
+    fi
+fi
+
 ##########################################
 # SHA command probe for modules
 if test "$modules" = yes; then
     shacmd_probe="sha1sum sha1 shasum"
     for c in $shacmd_probe; do
-        if which $c >/dev/null 2>&1; then
+        if has $c; then
             shacmd="$c"
             break
         fi
@@ -2777,8 +3074,8 @@
   pixman_libs=
 elif test "$pixman" = "system"; then
   # pixman version has been checked above
-  pixman_cflags=`$pkg_config --cflags pixman-1`
-  pixman_libs=`$pkg_config --libs pixman-1`
+  pixman_cflags=$($pkg_config --cflags pixman-1)
+  pixman_libs=$($pkg_config --libs pixman-1)
 else
   if test ! -d ${source_path}/pixman/pixman; then
     error_exit "pixman >= 0.21.8 not present. Your options:" \
@@ -2838,6 +3135,7 @@
       if test "$found" = "no"; then
         LIBS="$pthread_lib $LIBS"
       fi
+      PTHREAD_LIB="$pthread_lib"
       break
     fi
   done
@@ -2894,8 +3192,8 @@
 min_libssh2_version=1.2.8
 if test "$libssh2" != "no" ; then
   if $pkg_config --atleast-version=$min_libssh2_version libssh2; then
-    libssh2_cflags=`$pkg_config libssh2 --cflags`
-    libssh2_libs=`$pkg_config libssh2 --libs`
+    libssh2_cflags=$($pkg_config libssh2 --cflags)
+    libssh2_libs=$($pkg_config libssh2 --libs)
     libssh2=yes
   else
     if test "$libssh2" = "yes" ; then
@@ -3039,9 +3337,11 @@
 if test "$fdt" != "no" ; then
   fdt_libs="-lfdt"
   # explicitly check for libfdt_env.h as it is missing in some stable installs
+  # and test for required functions to make sure we are on a version >= 1.4.0
   cat > $TMPC << EOF
+#include <libfdt.h>
 #include <libfdt_env.h>
-int main(void) { return 0; }
+int main(void) { fdt_get_property_by_offset(0, 0, 0); return 0; }
 EOF
   if compile_prog "" "$fdt_libs" ; then
     # system DTC is good - use it
@@ -3059,7 +3359,7 @@
     fdt_libs="-L\$(BUILD_DIR)/dtc/libfdt $fdt_libs"
   elif test "$fdt" = "yes" ; then
     # have neither and want - prompt for system/submodule install
-    error_exit "DTC (libfdt) not present. Your options:" \
+    error_exit "DTC (libfdt) version >= 1.4.0 not present. Your options:" \
         "  (1) Preferred: Install the DTC (libfdt) devel package" \
         "  (2) Fetch the DTC submodule, using:" \
         "      git submodule update --init dtc"
@@ -3073,26 +3373,39 @@
 libs_softmmu="$libs_softmmu $fdt_libs"
 
 ##########################################
-# GLX probe, used by milkymist-tmu2
-if test "$glx" != "no" ; then
-  glx_libs="-lGL -lX11"
-  cat > $TMPC << EOF
-#include <X11/Xlib.h>
-#include <GL/gl.h>
-#include <GL/glx.h>
-int main(void) { glBegin(0); glXQueryVersion(0,0,0); return 0; }
-EOF
-  if compile_prog "" "-lGL -lX11" ; then
-    glx=yes
-  else
-    if test "$glx" = "yes" ; then
-      feature_not_found "glx" "Install GL devel (e.g. MESA)"
+# opengl probe (for sdl2, gtk, milkymist-tmu2)
+
+if test "$opengl" != "no" ; then
+  opengl_pkgs="epoxy libdrm gbm"
+  if $pkg_config $opengl_pkgs x11; then
+    opengl_cflags="$($pkg_config --cflags $opengl_pkgs) $x11_cflags"
+    opengl_libs="$($pkg_config --libs $opengl_pkgs) $x11_libs"
+    opengl=yes
+    if test "$gtk" = "yes" && $pkg_config --exists "$gtkpackage >= 3.16"; then
+        gtk_gl="yes"
     fi
-    glx_libs=
-    glx=no
+  else
+    if test "$opengl" = "yes" ; then
+      feature_not_found "opengl" "Please install opengl (mesa) devel pkgs: $opengl_pkgs"
+    fi
+    opengl_cflags=""
+    opengl_libs=""
+    opengl=no
   fi
 fi
 
+if test "$opengl" = "yes"; then
+  cat > $TMPC << EOF
+#include <epoxy/egl.h>
+#ifndef EGL_MESA_image_dma_buf_export
+# error mesa/epoxy lacks support for dmabufs (mesa 10.6+)
+#endif
+int main(void) { return 0; }
+EOF
+  if compile_prog "" "" ; then
+    opengl_dmabuf=yes
+  fi
+fi
 
 ##########################################
 # archipelago probe
@@ -3111,6 +3424,12 @@
         archipelago="yes"
         libs_tools="$archipelago_libs $libs_tools"
         libs_softmmu="$archipelago_libs $libs_softmmu"
+
+	echo "WARNING: Please check the licenses of QEMU and libxseg carefully."
+	echo "GPLv3 versions of libxseg may not be compatible with QEMU's"
+	echo "license and therefore prevent redistribution."
+	echo
+	echo "To disable Archipelago, use --disable-archipelago"
     else
       if test "$archipelago" = "yes" ; then
         feature_not_found "Archipelago backend support" "Install libxseg devel"
@@ -3125,8 +3444,11 @@
 if test "$glusterfs" != "no" ; then
   if $pkg_config --atleast-version=3 glusterfs-api; then
     glusterfs="yes"
-    glusterfs_cflags=`$pkg_config --cflags glusterfs-api`
-    glusterfs_libs=`$pkg_config --libs glusterfs-api`
+    glusterfs_cflags=$($pkg_config --cflags glusterfs-api)
+    glusterfs_libs=$($pkg_config --libs glusterfs-api)
+    if $pkg_config --atleast-version=4 glusterfs-api; then
+      glusterfs_xlator_opt="yes"
+    fi
     if $pkg_config --atleast-version=5 glusterfs-api; then
       glusterfs_discard="yes"
     fi
@@ -3267,6 +3589,43 @@
   fi
 fi
 
+if test "$tcmalloc" = "yes" && test "$jemalloc" = "yes" ; then
+    echo "ERROR: tcmalloc && jemalloc can't be used at the same time"
+    exit 1
+fi
+
+##########################################
+# tcmalloc probe
+
+if test "$tcmalloc" = "yes" ; then
+  cat > $TMPC << EOF
+#include <stdlib.h>
+int main(void) { malloc(1); return 0; }
+EOF
+
+  if compile_prog "" "-ltcmalloc" ; then
+    LIBS="-ltcmalloc $LIBS"
+  else
+    feature_not_found "tcmalloc" "install gperftools devel"
+  fi
+fi
+
+##########################################
+# jemalloc probe
+
+if test "$jemalloc" = "yes" ; then
+  cat > $TMPC << EOF
+#include <stdlib.h>
+int main(void) { malloc(1); return 0; }
+EOF
+
+  if compile_prog "" "-ljemalloc" ; then
+    LIBS="-ljemalloc $LIBS"
+  else
+    feature_not_found "jemalloc" "install jemalloc devel"
+  fi
+fi
+
 ##########################################
 # signalfd probe
 signalfd="no"
@@ -3295,6 +3654,22 @@
   eventfd=yes
 fi
 
+# check if memfd is supported
+memfd=no
+cat > $TMPC << EOF
+#include <sys/memfd.h>
+
+int main(void)
+{
+    return memfd_create("foo", MFD_ALLOW_SEALING);
+}
+EOF
+if compile_prog "" "" ; then
+  memfd=yes
+fi
+
+
+
 # check for fallocate
 fallocate=no
 cat > $TMPC << EOF
@@ -3326,6 +3701,22 @@
   fallocate_punch_hole=yes
 fi
 
+# check that fallocate supports range zeroing inside the file
+fallocate_zero_range=no
+cat > $TMPC << EOF
+#include <fcntl.h>
+#include <linux/falloc.h>
+
+int main(void)
+{
+    fallocate(0, FALLOC_FL_ZERO_RANGE, 0, 0);
+    return 0;
+}
+EOF
+if compile_prog "" "" ; then
+  fallocate_zero_range=yes
+fi
+
 # check for posix_fallocate
 posix_fallocate=no
 cat > $TMPC << EOF
@@ -3434,8 +3825,8 @@
   epoll=yes
 fi
 
-# epoll_create1 and epoll_pwait are later additions
-# so we must check separately for their presence
+# epoll_create1 is a later addition
+# so we must check separately for its presence
 epoll_create1=no
 cat > $TMPC << EOF
 #include <sys/epoll.h>
@@ -3457,20 +3848,6 @@
   epoll_create1=yes
 fi
 
-epoll_pwait=no
-cat > $TMPC << EOF
-#include <sys/epoll.h>
-
-int main(void)
-{
-    epoll_pwait(0, 0, 0, 0, 0);
-    return 0;
-}
-EOF
-if compile_prog "" "" ; then
-  epoll_pwait=yes
-fi
-
 # check for sendfile support
 sendfile=no
 cat > $TMPC << EOF
@@ -3634,34 +4011,20 @@
   fi
 fi
 
-# check for libcacard for smartcard support
+# check for smartcard support
 smartcard_cflags=""
-# TODO - what's the minimal nss version we support?
-if test "$smartcard_nss" != "no"; then
-  cat > $TMPC << EOF
-#include <pk11pub.h>
-int main(void) { PK11_FreeSlot(0); return 0; }
-EOF
-    # FIXME: do not include $glib_* in here
-    nss_libs="$($pkg_config --libs nss 2>/dev/null) $glib_libs"
-    nss_cflags="$($pkg_config --cflags nss 2>/dev/null) $glib_cflags"
-    test_cflags="$nss_cflags"
-    # The header files in nss < 3.13.3 have a bug which causes them to
-    # emit a warning. If we're going to compile QEMU with -Werror, then
-    # test that the headers don't have this bug. Otherwise we would pass
-    # the configure test but fail to compile QEMU later.
-    if test "$werror" = "yes"; then
-        test_cflags="-Werror $test_cflags"
-    fi
-    if test -n "$libtool" &&
-       $pkg_config --atleast-version=3.12.8 nss && \
-      compile_prog "$test_cflags" "$nss_libs"; then
-        smartcard_nss="yes"
+if test "$smartcard" != "no"; then
+    if $pkg_config libcacard; then
+        libcacard_cflags=$($pkg_config --cflags libcacard)
+        libcacard_libs=$($pkg_config --libs libcacard)
+        QEMU_CFLAGS="$QEMU_CFLAGS $libcacard_cflags"
+        libs_softmmu="$libs_softmmu $libcacard_libs"
+        smartcard="yes"
     else
-        if test "$smartcard_nss" = "yes"; then
-            feature_not_found "nss" "Install nss devel >= 3.12.8"
+        if test "$smartcard" = "yes"; then
+            feature_not_found "smartcard" "Install libcacard devel"
         fi
-        smartcard_nss="no"
+        smartcard="no"
     fi
 fi
 
@@ -3702,13 +4065,13 @@
 
 if test "$mingw32" = "yes" -a "$guest_agent" != "no" -a "$vss_win32_sdk" != "no" ; then
   case "$vss_win32_sdk" in
-    "")   vss_win32_include="-I$source_path" ;;
+    "")   vss_win32_include="-isystem $source_path" ;;
     *\ *) # The SDK is installed in "Program Files" by default, but we cannot
           # handle path with spaces. So we symlink the headers into ".sdk/vss".
-          vss_win32_include="-I$source_path/.sdk/vss"
+          vss_win32_include="-isystem $source_path/.sdk/vss"
 	  symlink "$vss_win32_sdk/inc" "$source_path/.sdk/vss/inc"
 	  ;;
-    *)    vss_win32_include="-I$vss_win32_sdk"
+    *)    vss_win32_include="-isystem $vss_win32_sdk"
   esac
   cat > $TMPC << EOF
 #define __MIDL_user_allocate_free_DEFINED__
@@ -3719,6 +4082,7 @@
     guest_agent_with_vss="yes"
     QEMU_CFLAGS="$QEMU_CFLAGS $vss_win32_include"
     libs_qga="-lole32 -loleaut32 -lshlwapi -luuid -lstdc++ -Wl,--enable-stdcall-fixup $libs_qga"
+    qga_vss_provider="qga/vss-win32/qga-vss.dll qga/vss-win32/qga-vss.tlb"
   else
     if test "$vss_win32_sdk" != "" ; then
       echo "ERROR: Please download and install Microsoft VSS SDK:"
@@ -3753,6 +4117,45 @@
 fi
 
 ##########################################
+# check if mingw environment provides a recent ntddscsi.h
+if test "$mingw32" = "yes" -a "$guest_agent" != "no"; then
+  cat > $TMPC << EOF
+#include <windows.h>
+#include <ntddscsi.h>
+int main(void) {
+#if !defined(IOCTL_SCSI_GET_ADDRESS)
+#error Missing required ioctl definitions
+#endif
+  SCSI_ADDRESS addr = { .Lun = 0, .TargetId = 0, .PathId = 0 };
+  return addr.Lun;
+}
+EOF
+  if compile_prog "" "" ; then
+    guest_agent_ntddscsi=yes
+    libs_qga="-lsetupapi $libs_qga"
+  fi
+fi
+
+##########################################
+# virgl renderer probe
+
+if test "$virglrenderer" != "no" ; then
+  cat > $TMPC << EOF
+#include <virglrenderer.h>
+int main(void) { virgl_renderer_poll(); return 0; }
+EOF
+  virgl_cflags=$($pkg_config --cflags virglrenderer 2>/dev/null)
+  virgl_libs=$($pkg_config --libs virglrenderer 2>/dev/null)
+  if $pkg_config virglrenderer >/dev/null 2>&1 && \
+     compile_prog "$virgl_cflags" "$virgl_libs" ; then
+    virglrenderer="yes"
+  else
+    if test "$virglrenderer" = "yes" ; then
+      feature_not_found "virglrenderer"
+    fi
+    virglrenderer="no"
+  fi
+fi
 
 ##########################################
 # check if we have fdatasync
@@ -3800,24 +4203,6 @@
 fi
 
 ##########################################
-# check if we have usable SIGEV_THREAD_ID
-
-sigev_thread_id=no
-cat > $TMPC << EOF
-#include <signal.h>
-int main(void) {
-  struct sigevent ev;
-  ev.sigev_notify = SIGEV_THREAD_ID;
-  ev._sigev_un._tid = 0;
-  asm volatile("" : : "g"(&ev));
-  return 0;
-}
-EOF
-if compile_prog "" "" ; then
-    sigev_thread_id=yes
-fi
-
-##########################################
 # check if trace backend exists
 
 $python "$source_path/scripts/tracetool.py" "--backends=$trace_backends" --check-backends  > /dev/null 2> /dev/null
@@ -3835,12 +4220,12 @@
 EOF
   if compile_prog "" "" ; then
     if $pkg_config lttng-ust --exists; then
-      lttng_ust_libs=`$pkg_config --libs lttng-ust`
+      lttng_ust_libs=$($pkg_config --libs lttng-ust)
     else
       lttng_ust_libs="-llttng-ust"
     fi
     if $pkg_config liburcu-bp --exists; then
-      urcu_bp_libs=`$pkg_config --libs liburcu-bp`
+      urcu_bp_libs=$($pkg_config --libs liburcu-bp)
     else
       urcu_bp_libs="-lurcu-bp"
     fi
@@ -4079,6 +4464,95 @@
     getauxval=yes
 fi
 
+########################################
+# check if ccache is interfering with
+# semantic analysis of macros
+
+unset CCACHE_CPP2
+ccache_cpp2=no
+cat > $TMPC << EOF
+static const int Z = 1;
+#define fn() ({ Z; })
+#define TAUT(X) ((X) == Z)
+#define PAREN(X, Y) (X == Y)
+#define ID(X) (X)
+int main(int argc, char *argv[])
+{
+    int x = 0, y = 0;
+    x = ID(x);
+    x = fn();
+    fn();
+    if (PAREN(x, y)) return 0;
+    if (TAUT(Z)) return 0;
+    return 0;
+}
+EOF
+
+if ! compile_object "-Werror"; then
+    ccache_cpp2=yes
+fi
+
+#################################################
+# clang does not support glibc + FORTIFY_SOURCE.
+
+if test "$fortify_source" != "no"; then
+  if echo | $cc -dM -E - | grep __clang__ > /dev/null 2>&1 ; then
+    fortify_source="no";
+  elif test -n "$cxx" &&
+       echo | $cxx -dM -E - | grep __clang__ >/dev/null 2>&1 ; then
+    fortify_source="no";
+  else
+    fortify_source="yes"
+  fi
+fi
+
+##########################################
+# check if struct fsxattr is available via linux/fs.h
+
+have_fsxattr=no
+cat > $TMPC << EOF
+#include <linux/fs.h>
+struct fsxattr foo;
+int main(void) {
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    have_fsxattr=yes
+fi
+
+##########################################
+# check if rtnetlink.h exists and is useful
+have_rtnetlink=no
+cat > $TMPC << EOF
+#include <linux/rtnetlink.h>
+int main(void) {
+  return IFLA_PROTO_DOWN;
+}
+EOF
+if compile_prog "" "" ; then
+    have_rtnetlink=yes
+fi
+
+#################################################
+# Sparc implicitly links with --relax, which is
+# incompatible with -r, so --no-relax should be
+# given. It does no harm to give it on other
+# platforms too.
+
+# Note: the prototype is needed since QEMU_CFLAGS
+#       contains -Wmissing-prototypes
+cat > $TMPC << EOF
+extern int foo(void);
+int foo(void) { return 0; }
+EOF
+if ! compile_object ""; then
+  error_exit "Failed to compile object file for LD_REL_FLAGS test"
+fi
+if do_cc -nostdlib -Wl,-r -Wl,--no-relax -o $TMPMO $TMPO; then
+  LD_REL_FLAGS="-Wl,--no-relax"
+fi
+
 ##########################################
 # End of CC checks
 # After here, no more $cc or $ld runs
@@ -4086,11 +4560,10 @@
 if test "$gcov" = "yes" ; then
   CFLAGS="-fprofile-arcs -ftest-coverage -g $CFLAGS"
   LDFLAGS="-fprofile-arcs -ftest-coverage $LDFLAGS"
-elif test "$debug" = "no" ; then
-    if test "${EXTRA_CFLAGS#*-O}" = "$EXTRA_CFLAGS"; then
-        CFLAGS="-O2 $CFLAGS"
-    fi
-    CFLAGS="-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $CFLAGS"
+elif test "$fortify_source" = "yes" ; then
+  CFLAGS="-O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 $CFLAGS"
+elif test "$debug" = "no"; then
+  CFLAGS="-O2 $CFLAGS"
 fi
 
 ##########################################
@@ -4108,16 +4581,6 @@
   fi
 fi
 
-# Disable zero malloc errors for official releases unless explicitly told to
-# enable/disable
-if test -z "$zero_malloc" ; then
-    if test "$z_version" = "50" ; then
-	zero_malloc="no"
-    else
-	zero_malloc="yes"
-    fi
-fi
-
 # Now we've finished running tests it's OK to add -Werror to the compiler flags
 if test "$werror" = "yes"; then
     QEMU_CFLAGS="-Werror $QEMU_CFLAGS"
@@ -4155,6 +4618,7 @@
   tools="qemu-img\$(EXESUF) qemu-io\$(EXESUF) $tools"
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
     tools="qemu-nbd\$(EXESUF) $tools"
+    tools="ivshmem-client\$(EXESUF) ivshmem-server\$(EXESUF) $tools"
   fi
 fi
 if test "$softmmu" = yes ; then
@@ -4164,18 +4628,18 @@
       tools="$tools fsdev/virtfs-proxy-helper\$(EXESUF)"
     else
       if test "$virtfs" = yes; then
-        error_exit "VirtFS is supported only on Linux and requires libcap-devel and libattr-devel"
+        error_exit "VirtFS is supported only on Linux and requires libcap devel and libattr devel"
       fi
       virtfs=no
     fi
   fi
 fi
+
+# Probe for guest agent support/options
+
 if [ "$guest_agent" != "no" ]; then
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" -o "$mingw32" = "yes" ] ; then
-      tools="qemu-ga\$(EXESUF) $tools"
-      if [ "$mingw32" = "yes" -a "$guest_agent_with_vss" = "yes" ]; then
-        tools="qga/vss-win32/qga-vss.dll qga/vss-win32/qga-vss.tlb $tools"
-      fi
+      tools="qemu-ga $tools"
       guest_agent=yes
   elif [ "$guest_agent" != yes ]; then
       guest_agent=no
@@ -4184,12 +4648,78 @@
   fi
 fi
 
+# Guest agent Window MSI  package
+
+if test "$guest_agent" != yes; then
+  if test "$guest_agent_msi" = yes; then
+    error_exit "MSI guest agent package requires guest agent enabled"
+  fi
+  guest_agent_msi=no
+elif test "$mingw32" != "yes"; then
+  if test "$guest_agent_msi" = "yes"; then
+    error_exit "MSI guest agent package is available only for MinGW Windows cross-compilation"
+  fi
+  guest_agent_msi=no
+elif ! has wixl; then
+  if test "$guest_agent_msi" = "yes"; then
+    error_exit "MSI guest agent package requires wixl tool installed ( usually from msitools package )"
+  fi
+  guest_agent_msi=no
+else
+  # we support qemu-ga, mingw32, and wixl: default to MSI enabled if it wasn't
+  # disabled explicitly
+  if test "$guest_agent_msi" != "no"; then
+    guest_agent_msi=yes
+  fi
+fi
+
+if test "$guest_agent_msi" = "yes"; then
+  if test "$guest_agent_with_vss" = "yes"; then
+    QEMU_GA_MSI_WITH_VSS="-D InstallVss"
+  fi
+
+  if test "$QEMU_GA_MANUFACTURER" = ""; then
+    QEMU_GA_MANUFACTURER=QEMU
+  fi
+
+  if test "$QEMU_GA_DISTRO" = ""; then
+    QEMU_GA_DISTRO=Linux
+  fi
+
+  if test "$QEMU_GA_VERSION" = ""; then
+      QEMU_GA_VERSION=$(cat $source_path/VERSION)
+  fi
+
+  QEMU_GA_MSI_MINGW_DLL_PATH="-D Mingw_dlls=$($pkg_config --variable=prefix glib-2.0)/bin"
+
+  case "$cpu" in
+  x86_64)
+    QEMU_GA_MSI_ARCH="-a x64 -D Arch=64"
+    ;;
+  i386)
+    QEMU_GA_MSI_ARCH="-D Arch=32"
+    ;;
+  *)
+    error_exit "CPU $cpu not supported for building installation package"
+    ;;
+  esac
+fi
+
 # Mac OS X ships with a broken assembler
 roms=
 if test \( "$cpu" = "i386" -o "$cpu" = "x86_64" \) -a \
         "$targetos" != "Darwin" -a "$targetos" != "SunOS" -a \
         "$softmmu" = yes ; then
-  roms="optionrom"
+    # Different host OS linkers have different ideas about the name of the ELF
+    # emulation. Linux and OpenBSD use 'elf_i386'; FreeBSD uses the _fbsd
+    # variant; and Windows uses i386pe.
+    for emu in elf_i386 elf_i386_fbsd i386pe; do
+        if "$ld" -verbose 2>&1 | grep -q "^[[:space:]]*$emu[[:space:]]*$"; then
+            ld_i386_emulation="$emu"
+            roms="optionrom"
+            break
+        fi
+    done
 fi
 if test "$cpu" = "ppc64" -a "$targetos" != "Darwin" ; then
   roms="$roms spapr-rtas"
@@ -4200,7 +4730,7 @@
 fi
 
 # Probe for the need for relocating the user-only binary.
-if test "$pie" = "no" ; then
+if ( [ "$linux_user" = yes ] || [ "$bsd_user" = yes ] ) && [ "$pie" = no ]; then
   textseg_addr=
   case "$cpu" in
     arm | i386 | ppc* | s390* | sparc* | x86_64 | x32)
@@ -4222,6 +4752,16 @@
       # In case ld does not support -Ttext-segment, edit the default linker
       # script via sed to set the .text start addr.  This is needed on FreeBSD
       # at least.
+      if ! $ld --verbose >/dev/null 2>&1; then
+        error_exit \
+            "We need to link the QEMU user mode binaries at a" \
+            "specific text address. Unfortunately your linker" \
+            "doesn't support either the -Ttext-segment option or" \
+            "printing the default linker script with --verbose." \
+            "If you don't want the user mode binaries, pass the" \
+            "--disable-user option to configure."
+      fi
+
       $ld --verbose | sed \
         -e '1,/==================================================/d' \
         -e '/==================================================/,$d' \
@@ -4232,21 +4772,27 @@
   fi
 fi
 
+echo_version() {
+    if test "$1" = "yes" ; then
+        echo "($2)"
+    fi
+}
+
 # prepend pixman and ftd flags after all config tests are done
 QEMU_CFLAGS="$pixman_cflags $fdt_cflags $QEMU_CFLAGS"
 libs_softmmu="$pixman_libs $libs_softmmu"
 
 echo "Install prefix    $prefix"
-echo "BIOS directory    `eval echo $qemu_datadir`"
-echo "binary directory  `eval echo $bindir`"
-echo "library directory `eval echo $libdir`"
-echo "module directory  `eval echo $qemu_moddir`"
-echo "libexec directory `eval echo $libexecdir`"
-echo "include directory `eval echo $includedir`"
-echo "config directory  `eval echo $sysconfdir`"
+echo "BIOS directory    $(eval echo $qemu_datadir)"
+echo "binary directory  $(eval echo $bindir)"
+echo "library directory $(eval echo $libdir)"
+echo "module directory  $(eval echo $qemu_moddir)"
+echo "libexec directory $(eval echo $libexecdir)"
+echo "include directory $(eval echo $includedir)"
+echo "config directory  $(eval echo $sysconfdir)"
 if test "$mingw32" = "no" ; then
-echo "local state directory   `eval echo $local_statedir`"
-echo "Manual directory  `eval echo $mandir`"
+echo "local state directory   $(eval echo $local_statedir)"
+echo "Manual directory  $(eval echo $mandir)"
 echo "ELF interp prefix $interp_prefix"
 else
 echo "local state directory   queried at runtime"
@@ -4281,10 +4827,20 @@
     echo "Cocoa support     $cocoa"
 fi
 echo "pixman            $pixman"
-echo "SDL support       $sdl"
-echo "GTK support       $gtk"
-echo "VTE support       $vte"
+echo "SDL support       $sdl $(echo_version $sdl $sdlversion)"
+echo "GTK support       $gtk $(echo_version $gtk $gtk_version)"
+echo "GTK GL support    $gtk_gl"
+echo "VTE support       $vte $(echo_version $vte $vteversion)"
+echo "TLS priority      $tls_priority"
+echo "GNUTLS support    $gnutls"
+echo "GNUTLS rnd        $gnutls_rnd"
+echo "libgcrypt         $gcrypt"
+echo "libgcrypt kdf     $gcrypt_kdf"
+echo "nettle            $nettle $(echo_version $nettle $nettle_version)"
+echo "nettle kdf        $nettle_kdf"
+echo "libtasn1          $tasn1"
 echo "curses support    $curses"
+echo "virgl support     $virglrenderer"
 echo "curl support      $curl"
 echo "mingw32 support   $mingw32"
 echo "Audio drivers     $audio_drv_list"
@@ -4293,20 +4849,21 @@
 echo "VirtFS support    $virtfs"
 echo "VNC support       $vnc"
 if test "$vnc" = "yes" ; then
-    echo "VNC TLS support   $vnc_tls"
     echo "VNC SASL support  $vnc_sasl"
     echo "VNC JPEG support  $vnc_jpeg"
     echo "VNC PNG support   $vnc_png"
-    echo "VNC WS support    $vnc_ws"
 fi
 if test -n "$sparc_cpu"; then
     echo "Target Sparc Arch $sparc_cpu"
 fi
 echo "xen support       $xen"
+if test "$xen" = "yes" ; then
+  echo "xen ctrl version  $xen_ctrl_version"
+  echo "pv dom build      $xen_pv_domain_build"
+fi
 echo "brlapi support    $brlapi"
 echo "bluez  support    $bluez"
 echo "Documentation     $docs"
-echo "GUEST_BASE        $guest_base"
 echo "PIE               $pie"
 echo "vde support       $vde"
 echo "netmap support    $netmap"
@@ -4322,30 +4879,28 @@
 echo "fdatasync         $fdatasync"
 echo "madvise           $madvise"
 echo "posix_madvise     $posix_madvise"
-echo "sigev_thread_id   $sigev_thread_id"
 echo "uuid support      $uuid"
 echo "libcap-ng support $cap_ng"
 echo "vhost-net support $vhost_net"
 echo "vhost-scsi support $vhost_scsi"
 echo "Trace backends    $trace_backends"
-if test "$trace_backend" = "simple"; then
+if have_backend "simple"; then
 echo "Trace output file $trace_file-<pid>"
 fi
-if test "$spice" = "yes"; then
-echo "spice support     $spice ($spice_protocol_version/$spice_server_version)"
-else
-echo "spice support     $spice"
-fi
+echo "spice support     $spice $(echo_version $spice $spice_protocol_version/$spice_server_version)"
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
-echo "nss used          $smartcard_nss"
+echo "smartcard support $smartcard"
 echo "libusb            $libusb"
 echo "usb net redir     $usb_redir"
-echo "GLX support       $glx"
+echo "OpenGL support    $opengl"
+echo "OpenGL dmabufs    $opengl_dmabuf"
 echo "libiscsi support  $libiscsi"
 echo "libnfs support    $libnfs"
 echo "build guest agent $guest_agent"
 echo "QGA VSS support   $guest_agent_with_vss"
+echo "QGA w32 disk info $guest_agent_ntddscsi"
+echo "QGA MSI support   $guest_agent_msi"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine"
 echo "coroutine pool    $coroutine_pool"
@@ -4358,11 +4913,13 @@
 echo "TPM passthrough   $tpm_passthrough"
 echo "QOM debugging     $qom_cast_debug"
 echo "vhdx              $vhdx"
-echo "Quorum            $quorum"
 echo "lzo support       $lzo"
 echo "snappy support    $snappy"
+echo "bzip2 support     $bzip2"
 echo "NUMA host support $numa"
-echo "Android support   $android"
+echo "tcmalloc support  $tcmalloc"
+echo "jemalloc support  $jemalloc"
+echo "avx2 optimization $avx2_opt"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -4409,7 +4966,7 @@
 fi
 if test "$mingw32" = "yes" ; then
   echo "CONFIG_WIN32=y" >> $config_host_mak
-  rc_version=`cat $source_path/VERSION`
+  rc_version=$(cat $source_path/VERSION)
   version_major=${rc_version%%.*}
   rc_version=${rc_version#*.}
   version_minor=${rc_version%%.*}
@@ -4420,8 +4977,21 @@
   echo "CONFIG_PRODUCTVERSION=$version_major,$version_minor,$version_subminor,$version_micro" >> $config_host_mak
   if test "$guest_agent_with_vss" = "yes" ; then
     echo "CONFIG_QGA_VSS=y" >> $config_host_mak
+    echo "QGA_VSS_PROVIDER=$qga_vss_provider" >> $config_host_mak
     echo "WIN_SDK=\"$win_sdk\"" >> $config_host_mak
   fi
+  if test "$guest_agent_ntddscsi" = "yes" ; then
+    echo "CONFIG_QGA_NTDDDISK=y" >> $config_host_mak
+  fi
+  if test "$guest_agent_msi" = "yes"; then
+    echo "QEMU_GA_MSI_ENABLED=yes" >> $config_host_mak  
+    echo "QEMU_GA_MSI_MINGW_DLL_PATH=${QEMU_GA_MSI_MINGW_DLL_PATH}" >> $config_host_mak
+    echo "QEMU_GA_MSI_WITH_VSS=${QEMU_GA_MSI_WITH_VSS}" >> $config_host_mak
+    echo "QEMU_GA_MSI_ARCH=${QEMU_GA_MSI_ARCH}" >> $config_host_mak
+    echo "QEMU_GA_MANUFACTURER=${QEMU_GA_MANUFACTURER}" >> $config_host_mak
+    echo "QEMU_GA_DISTRO=${QEMU_GA_DISTRO}" >> $config_host_mak
+    echo "QEMU_GA_VERSION=${QEMU_GA_VERSION}" >> $config_host_mak
+  fi
 else
   echo "CONFIG_POSIX=y" >> $config_host_mak
 fi
@@ -4472,11 +5042,8 @@
 fi
 echo "CONFIG_AUDIO_DRIVERS=$audio_drv_list" >> $config_host_mak
 for drv in $audio_drv_list; do
-    def=CONFIG_`echo $drv | LC_ALL=C tr '[a-z]' '[A-Z]'`
+    def=CONFIG_$(echo $drv | LC_ALL=C tr '[a-z]' '[A-Z]')
     echo "$def=y" >> $config_host_mak
-    if test "$drv" = "fmod"; then
-        echo "FMOD_CFLAGS=-I$fmod_inc" >> $config_host_mak
-    fi
 done
 if test "$audio_pt_int" = "yes" ; then
   echo "CONFIG_AUDIO_PT_INT=y" >> $config_host_mak
@@ -4489,9 +5056,6 @@
 if test "$vnc" = "yes" ; then
   echo "CONFIG_VNC=y" >> $config_host_mak
 fi
-if test "$vnc_tls" = "yes" ; then
-  echo "CONFIG_VNC_TLS=y" >> $config_host_mak
-fi
 if test "$vnc_sasl" = "yes" ; then
   echo "CONFIG_VNC_SASL=y" >> $config_host_mak
 fi
@@ -4501,10 +5065,6 @@
 if test "$vnc_png" = "yes" ; then
   echo "CONFIG_VNC_PNG=y" >> $config_host_mak
 fi
-if test "$vnc_ws" = "yes" ; then
-  echo "CONFIG_VNC_WS=y" >> $config_host_mak
-  echo "VNC_WS_CFLAGS=$vnc_ws_cflags" >> $config_host_mak
-fi
 if test "$fnmatch" = "yes" ; then
   echo "CONFIG_FNMATCH=y" >> $config_host_mak
 fi
@@ -4514,7 +5074,7 @@
 if test "$xfs" = "yes" ; then
   echo "CONFIG_XFS=y" >> $config_host_mak
 fi
-qemu_version=`head $source_path/VERSION`
+qemu_version=$(head $source_path/VERSION)
 echo "VERSION=$qemu_version" >>$config_host_mak
 echo "PKGVERSION=$pkgversion" >>$config_host_mak
 echo "SRC_PATH=$source_path" >> $config_host_mak
@@ -4525,7 +5085,7 @@
 if test "$modules" = "yes"; then
   # $shacmd can generate a hash started with digit, which the compiler doesn't
   # like as an symbol. So prefix it with an underscore
-  echo "CONFIG_STAMP=_`(echo $qemu_version; echo $pkgversion; cat $0) | $shacmd - | cut -f1 -d\ `" >> $config_host_mak
+  echo "CONFIG_STAMP=_$( (echo $qemu_version; echo $pkgversion; cat $0) | $shacmd - | cut -f1 -d\ )" >> $config_host_mak
   echo "CONFIG_MODULES=y" >> $config_host_mak
 fi
 if test "$sdl" = "yes" ; then
@@ -4554,12 +5114,18 @@
 if test "$eventfd" = "yes" ; then
   echo "CONFIG_EVENTFD=y" >> $config_host_mak
 fi
+if test "$memfd" = "yes" ; then
+  echo "CONFIG_MEMFD=y" >> $config_host_mak
+fi
 if test "$fallocate" = "yes" ; then
   echo "CONFIG_FALLOCATE=y" >> $config_host_mak
 fi
 if test "$fallocate_punch_hole" = "yes" ; then
   echo "CONFIG_FALLOCATE_PUNCH_HOLE=y" >> $config_host_mak
 fi
+if test "$fallocate_zero_range" = "yes" ; then
+  echo "CONFIG_FALLOCATE_ZERO_RANGE=y" >> $config_host_mak
+fi
 if test "$posix_fallocate" = "yes" ; then
   echo "CONFIG_POSIX_FALLOCATE=y" >> $config_host_mak
 fi
@@ -4584,9 +5150,6 @@
 if test "$epoll_create1" = "yes" ; then
   echo "CONFIG_EPOLL_CREATE1=y" >> $config_host_mak
 fi
-if test "$epoll_pwait" = "yes" ; then
-  echo "CONFIG_EPOLL_PWAIT=y" >> $config_host_mak
-fi
 if test "$sendfile" = "yes" ; then
   echo "CONFIG_SENDFILE=y" >> $config_host_mak
 fi
@@ -4620,7 +5183,7 @@
   echo "CONFIG_BLUEZ=y" >> $config_host_mak
   echo "BLUEZ_CFLAGS=$bluez_cflags" >> $config_host_mak
 fi
-if test "glib_subprocess" = "yes" ; then
+if test "$glib_subprocess" = "yes" ; then
   echo "CONFIG_HAS_GLIB_SUBPROCESS_TESTS=y" >> $config_host_mak
 fi
 echo "GLIB_CFLAGS=$glib_cflags" >> $config_host_mak
@@ -4628,14 +5191,60 @@
   echo "CONFIG_GTK=y" >> $config_host_mak
   echo "CONFIG_GTKABI=$gtkabi" >> $config_host_mak
   echo "GTK_CFLAGS=$gtk_cflags" >> $config_host_mak
+  echo "GTK_LIBS=$gtk_libs" >> $config_host_mak
+  if test "$gtk_gl" = "yes" ; then
+    echo "CONFIG_GTK_GL=y" >> $config_host_mak
+  fi
+fi
+echo "CONFIG_TLS_PRIORITY=\"$tls_priority\"" >> $config_host_mak
+if test "$gnutls" = "yes" ; then
+  echo "CONFIG_GNUTLS=y" >> $config_host_mak
+fi
+if test "$gnutls_rnd" = "yes" ; then
+  echo "CONFIG_GNUTLS_RND=y" >> $config_host_mak
+fi
+if test "$gcrypt" = "yes" ; then
+  echo "CONFIG_GCRYPT=y" >> $config_host_mak
+  if test "$gcrypt_kdf" = "yes" ; then
+    echo "CONFIG_GCRYPT_KDF=y" >> $config_host_mak
+  fi
+fi
+if test "$nettle" = "yes" ; then
+  echo "CONFIG_NETTLE=y" >> $config_host_mak
+  echo "CONFIG_NETTLE_VERSION_MAJOR=${nettle_version%%.*}" >> $config_host_mak
+  if test "$nettle_kdf" = "yes" ; then
+    echo "CONFIG_NETTLE_KDF=y" >> $config_host_mak
+  fi
+fi
+if test "$tasn1" = "yes" ; then
+  echo "CONFIG_TASN1=y" >> $config_host_mak
+fi
+if test "$have_ifaddrs_h" = "yes" ; then
+    echo "HAVE_IFADDRS_H=y" >> $config_host_mak
+fi
+
+# Work around a system header bug with some kernel/XFS header
+# versions where they both try to define 'struct fsxattr':
+# xfs headers will not try to redefine structs from linux headers
+# if this macro is set.
+if test "$have_fsxattr" = "yes" ; then
+    echo "HAVE_FSXATTR=y" >> $config_host_mak
 fi
 if test "$vte" = "yes" ; then
   echo "CONFIG_VTE=y" >> $config_host_mak
   echo "VTE_CFLAGS=$vte_cflags" >> $config_host_mak
 fi
+if test "$virglrenderer" = "yes" ; then
+  echo "CONFIG_VIRGL=y" >> $config_host_mak
+  echo "VIRGL_CFLAGS=$virgl_cflags" >> $config_host_mak
+  echo "VIRGL_LIBS=$virgl_libs" >> $config_host_mak
+fi
 if test "$xen" = "yes" ; then
   echo "CONFIG_XEN_BACKEND=y" >> $config_host_mak
   echo "CONFIG_XEN_CTRL_INTERFACE_VERSION=$xen_ctrl_version" >> $config_host_mak
+  if test "$xen_pv_domain_build" = "yes" ; then
+    echo "CONFIG_XEN_PV_DOMAIN_BUILD=y" >> $config_host_mak
+  fi
 fi
 if test "$linux_aio" = "yes" ; then
   echo "CONFIG_LINUX_AIO=y" >> $config_host_mak
@@ -4682,18 +5291,13 @@
 if test "$posix_madvise" = "yes" ; then
   echo "CONFIG_POSIX_MADVISE=y" >> $config_host_mak
 fi
-if test "$sigev_thread_id" = "yes" ; then
-  echo "CONFIG_SIGEV_THREAD_ID=y" >> $config_host_mak
-fi
 
 if test "$spice" = "yes" ; then
   echo "CONFIG_SPICE=y" >> $config_host_mak
 fi
 
-if test "$smartcard_nss" = "yes" ; then
-  echo "CONFIG_SMARTCARD_NSS=y" >> $config_host_mak
-  echo "NSS_LIBS=$nss_libs" >> $config_host_mak
-  echo "NSS_CFLAGS=$nss_cflags" >> $config_host_mak
+if test "$smartcard" = "yes" ; then
+  echo "CONFIG_SMARTCARD=y" >> $config_host_mak
 fi
 
 if test "$libusb" = "yes" ; then
@@ -4704,9 +5308,17 @@
   echo "CONFIG_USB_REDIR=y" >> $config_host_mak
 fi
 
-if test "$glx" = "yes" ; then
-  echo "CONFIG_GLX=y" >> $config_host_mak
-  echo "GLX_LIBS=$glx_libs" >> $config_host_mak
+if test "$opengl" = "yes" ; then
+  echo "CONFIG_OPENGL=y" >> $config_host_mak
+  echo "OPENGL_CFLAGS=$opengl_cflags" >> $config_host_mak
+  echo "OPENGL_LIBS=$opengl_libs" >> $config_host_mak
+  if test "$opengl_dmabuf" = "yes" ; then
+    echo "CONFIG_OPENGL_DMABUF=y" >> $config_host_mak
+  fi
+fi
+
+if test "$avx2_opt" = "yes" ; then
+  echo "CONFIG_AVX2_OPT=y" >> $config_host_mak
 fi
 
 if test "$lzo" = "yes" ; then
@@ -4717,6 +5329,11 @@
   echo "CONFIG_SNAPPY=y" >> $config_host_mak
 fi
 
+if test "$bzip2" = "yes" ; then
+  echo "CONFIG_BZIP2=y" >> $config_host_mak
+  echo "BZIP2_LIBS=-lbz2" >> $config_host_mak
+fi
+
 if test "$libiscsi" = "yes" ; then
   echo "CONFIG_LIBISCSI=m" >> $config_host_mak
   echo "LIBISCSI_CFLAGS=$libiscsi_cflags" >> $config_host_mak
@@ -4736,8 +5353,8 @@
   echo "CONFIG_BSD=y" >> $config_host_mak
 fi
 
-if test "$zero_malloc" = "yes" ; then
-  echo "CONFIG_ZERO_MALLOC=y" >> $config_host_mak
+if test "$localtime_r" = "yes" ; then
+  echo "CONFIG_LOCALTIME_R=y" >> $config_host_mak
 fi
 if test "$qom_cast_debug" = "yes" ; then
   echo "CONFIG_QOM_CAST_DEBUG=y" >> $config_host_mak
@@ -4793,6 +5410,10 @@
   echo "GLUSTERFS_LIBS=$glusterfs_libs" >> $config_host_mak
 fi
 
+if test "$glusterfs_xlator_opt" = "yes" ; then
+  echo "CONFIG_GLUSTERFS_XLATOR_OPT=y" >> $config_host_mak
+fi
+
 if test "$glusterfs_discard" = "yes" ; then
   echo "CONFIG_GLUSTERFS_DISCARD=y" >> $config_host_mak
 fi
@@ -4812,10 +5433,6 @@
   echo "LIBSSH2_LIBS=$libssh2_libs" >> $config_host_mak
 fi
 
-if test "$quorum" = "yes" ; then
-  echo "CONFIG_QUORUM=y" >> $config_host_mak
-fi
-
 if test "$vhdx" = "yes" ; then
   echo "CONFIG_VHDX=y" >> $config_host_mak
 fi
@@ -4844,8 +5461,8 @@
   # Set the appropriate trace file.
   trace_file="\"$trace_file-\" FMT_pid"
 fi
-if have_backend "stderr"; then
-  echo "CONFIG_TRACE_STDERR=y" >> $config_host_mak
+if have_backend "log"; then
+  echo "CONFIG_TRACE_LOG=y" >> $config_host_mak
 fi
 if have_backend "ust"; then
   echo "CONFIG_TRACE_UST=y" >> $config_host_mak
@@ -4869,6 +5486,10 @@
   echo "CONFIG_RDMA=y" >> $config_host_mak
 fi
 
+if test "$have_rtnetlink" = "yes" ; then
+  echo "CONFIG_RTNETLINK=y" >> $config_host_mak
+fi
+
 # Hold two types of flag:
 #   CONFIG_THREAD_SETNAME_BYTHREAD  - we've got a way of setting the name on
 #                                     a thread we have a handle to
@@ -4900,13 +5521,8 @@
 echo "INSTALL=$install" >> $config_host_mak
 echo "INSTALL_DIR=$install -d -m 0755" >> $config_host_mak
 echo "INSTALL_DATA=$install -c -m 0644" >> $config_host_mak
-if test -n "$libtool"; then
-  echo "INSTALL_PROG=\$(LIBTOOL) --mode=install $install -c -m 0755" >> $config_host_mak
-  echo "INSTALL_LIB=\$(LIBTOOL) --mode=install $install -c -m 0644" >> $config_host_mak
-else
-  echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
-  echo "INSTALL_LIB=$install -c -m 0644" >> $config_host_mak
-fi
+echo "INSTALL_PROG=$install -c -m 0755" >> $config_host_mak
+echo "INSTALL_LIB=$install -c -m 0644" >> $config_host_mak
 echo "PYTHON=$python" >> $config_host_mak
 echo "CC=$cc" >> $config_host_mak
 if $iasl -h > /dev/null 2>&1; then
@@ -4919,18 +5535,19 @@
 echo "AR=$ar" >> $config_host_mak
 echo "ARFLAGS=$ARFLAGS" >> $config_host_mak
 echo "AS=$as" >> $config_host_mak
+echo "CCAS=$ccas" >> $config_host_mak
 echo "CPP=$cpp" >> $config_host_mak
 echo "OBJCOPY=$objcopy" >> $config_host_mak
 echo "LD=$ld" >> $config_host_mak
 echo "NM=$nm" >> $config_host_mak
 echo "WINDRES=$windres" >> $config_host_mak
-echo "LIBTOOL=$libtool" >> $config_host_mak
 echo "CFLAGS=$CFLAGS" >> $config_host_mak
 echo "CFLAGS_NOPIE=$CFLAGS_NOPIE" >> $config_host_mak
 echo "QEMU_CFLAGS=$QEMU_CFLAGS" >> $config_host_mak
 echo "QEMU_INCLUDES=$QEMU_INCLUDES" >> $config_host_mak
 if test "$sparse" = "yes" ; then
   echo "CC           := REAL_CC=\"\$(CC)\" cgcc"       >> $config_host_mak
+  echo "CPP          := REAL_CC=\"\$(CPP)\" cgcc"      >> $config_host_mak
   echo "CXX          := REAL_CC=\"\$(CXX)\" cgcc"      >> $config_host_mak
   echo "HOST_CC      := REAL_CC=\"\$(HOST_CC)\" cgcc"  >> $config_host_mak
   echo "QEMU_CFLAGS  += -Wbitwise -Wno-transparent-union -Wno-old-initializer -Wno-non-pointer-null" >> $config_host_mak
@@ -4942,13 +5559,17 @@
 fi
 echo "LDFLAGS=$LDFLAGS" >> $config_host_mak
 echo "LDFLAGS_NOPIE=$LDFLAGS_NOPIE" >> $config_host_mak
-echo "LIBTOOLFLAGS=$LIBTOOLFLAGS" >> $config_host_mak
+echo "LD_REL_FLAGS=$LD_REL_FLAGS" >> $config_host_mak
+echo "LD_I386_EMULATION=$ld_i386_emulation" >> $config_host_mak
 echo "LIBS+=$LIBS" >> $config_host_mak
 echo "LIBS_TOOLS+=$libs_tools" >> $config_host_mak
+echo "PTHREAD_LIB=$PTHREAD_LIB" >> $config_host_mak
 echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "DSOSUF=$DSOSUF" >> $config_host_mak
 echo "LDFLAGS_SHARED=$LDFLAGS_SHARED" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
+echo "TASN1_LIBS=$tasn1_libs" >> $config_host_mak
+echo "TASN1_CFLAGS=$tasn1_cflags" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
 echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
 if test "$gcov" = "yes" ; then
@@ -4989,7 +5610,7 @@
 for target in $target_list; do
 target_dir="$target"
 config_target_mak=$target_dir/config-target.mak
-target_name=`echo $target | cut -d '-' -f 1`
+target_name=$(echo $target | cut -d '-' -f 1)
 target_bigendian="no"
 
 case "$target_name" in
@@ -5029,7 +5650,7 @@
 echo "# Automatically generated by configure - do not modify" > $config_target_mak
 
 bflt="no"
-interp_prefix1=`echo "$interp_prefix" | sed "s/%M/$target_name/g"`
+interp_prefix1=$(echo "$interp_prefix" | sed "s/%M/$target_name/g")
 gdb_xml_files=""
 
 TARGET_ARCH="$target_name"
@@ -5081,8 +5702,6 @@
     TARGET_BASE_ARCH=mips
     echo "TARGET_ABI_MIPSN64=y" >> $config_target_mak
   ;;
-  tricore)
-  ;;
   moxie)
   ;;
   or32)
@@ -5100,20 +5719,20 @@
   ppc64)
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
-    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
+    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml power-vsx.xml"
   ;;
   ppc64le)
     TARGET_ARCH=ppc64
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
-    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
+    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml power-vsx.xml"
   ;;
   ppc64abi32)
     TARGET_ARCH=ppc64
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
     echo "TARGET_ABI32=y" >> $config_target_mak
-    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
+    gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml power-vsx.xml"
   ;;
   sh4|sh4eb)
     TARGET_ARCH=sh4
@@ -5131,7 +5750,11 @@
     echo "TARGET_ABI32=y" >> $config_target_mak
   ;;
   s390x)
-    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml"
+    gdb_xml_files="s390x-core64.xml s390-acr.xml s390-fpr.xml s390-vx.xml s390-cr.xml s390-virt.xml"
+  ;;
+  tilegx)
+  ;;
+  tricore)
   ;;
   unicore32)
   ;;
@@ -5153,7 +5776,7 @@
     echo "$@"| LC_ALL=C tr '[a-z]' '[A-Z]'
 }
 
-target_arch_name="`upper $TARGET_ARCH`"
+target_arch_name="$(upper $TARGET_ARCH)"
 echo "TARGET_$target_arch_name=y" >> $config_target_mak
 echo "TARGET_NAME=$target_name" >> $config_target_mak
 echo "TARGET_BASE_ARCH=$TARGET_BASE_ARCH" >> $config_target_mak
@@ -5186,10 +5809,13 @@
       \( "$target_name" = "ppcemb" -a "$cpu" = "ppc64" \) -o \
       \( "$target_name" = "mipsel" -a "$cpu" = "mips" \) -o \
       \( "$target_name" = "x86_64" -a "$cpu" = "i386"   \) -o \
-      \( "$target_name" = "i386"   -a "$cpu" = "x86_64" \) \) ; then
+      \( "$target_name" = "i386"   -a "$cpu" = "x86_64" \) -o \
+      \( "$target_name" = "x86_64" -a "$cpu" = "x32"   \) -o \
+      \( "$target_name" = "i386"   -a "$cpu" = "x32" \) \) ; then
       echo "CONFIG_KVM=y" >> $config_target_mak
       if test "$vhost_net" = "yes" ; then
         echo "CONFIG_VHOST_NET=y" >> $config_target_mak
+        echo "CONFIG_VHOST_NET_TEST_$target_name=y" >> $config_host_mak
       fi
     fi
 esac
@@ -5226,9 +5852,6 @@
 if test "$target_user_only" = "yes" -a "$bflt" = "yes"; then
   echo "TARGET_HAS_BFLT=y" >> $config_target_mak
 fi
-if test "$target_user_only" = "yes" -a "$guest_base" = "yes"; then
-  echo "CONFIG_USE_GUEST_BASE=y" >> $config_target_mak
-fi
 if test "$target_bsd_user" = "yes" ; then
   echo "CONFIG_BSD_USER=y" >> $config_target_mak
 fi
@@ -5238,91 +5861,76 @@
 cflags=""
 ldflags=""
 
+disas_config() {
+  echo "CONFIG_${1}_DIS=y" >> $config_target_mak
+  echo "CONFIG_${1}_DIS=y" >> config-all-disas.mak
+}
+
 for i in $ARCH $TARGET_BASE_ARCH ; do
   case "$i" in
   alpha)
-    echo "CONFIG_ALPHA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ALPHA_DIS=y"  >> config-all-disas.mak
+    disas_config "ALPHA"
   ;;
   aarch64)
     if test -n "${cxx}"; then
-      echo "CONFIG_ARM_A64_DIS=y"  >> $config_target_mak
-      echo "CONFIG_ARM_A64_DIS=y"  >> config-all-disas.mak
+      disas_config "ARM_A64"
     fi
   ;;
   arm)
-    echo "CONFIG_ARM_DIS=y"  >> $config_target_mak
-    echo "CONFIG_ARM_DIS=y"  >> config-all-disas.mak
+    disas_config "ARM"
     if test -n "${cxx}"; then
-      echo "CONFIG_ARM_A64_DIS=y"  >> $config_target_mak
-      echo "CONFIG_ARM_A64_DIS=y"  >> config-all-disas.mak
+      disas_config "ARM_A64"
     fi
   ;;
   cris)
-    echo "CONFIG_CRIS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_CRIS_DIS=y"  >> config-all-disas.mak
+    disas_config "CRIS"
   ;;
   hppa)
-    echo "CONFIG_HPPA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_HPPA_DIS=y"  >> config-all-disas.mak
+    disas_config "HPPA"
   ;;
   i386|x86_64|x32)
-    echo "CONFIG_I386_DIS=y"  >> $config_target_mak
-    echo "CONFIG_I386_DIS=y"  >> config-all-disas.mak
+    disas_config "I386"
   ;;
   ia64*)
-    echo "CONFIG_IA64_DIS=y"  >> $config_target_mak
-    echo "CONFIG_IA64_DIS=y"  >> config-all-disas.mak
+    disas_config "IA64"
   ;;
   lm32)
-    echo "CONFIG_LM32_DIS=y"  >> $config_target_mak
-    echo "CONFIG_LM32_DIS=y"  >> config-all-disas.mak
+    disas_config "LM32"
   ;;
   m68k)
-    echo "CONFIG_M68K_DIS=y"  >> $config_target_mak
-    echo "CONFIG_M68K_DIS=y"  >> config-all-disas.mak
+    disas_config "M68K"
   ;;
   microblaze*)
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MICROBLAZE_DIS=y"  >> config-all-disas.mak
+    disas_config "MICROBLAZE"
   ;;
   mips*)
-    echo "CONFIG_MIPS_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MIPS_DIS=y"  >> config-all-disas.mak
+    disas_config "MIPS"
   ;;
   moxie*)
-    echo "CONFIG_MOXIE_DIS=y"  >> $config_target_mak
-    echo "CONFIG_MOXIE_DIS=y"  >> config-all-disas.mak
+    disas_config "MOXIE"
   ;;
   or32)
-    echo "CONFIG_OPENRISC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_OPENRISC_DIS=y"  >> config-all-disas.mak
+    disas_config "OPENRISC"
   ;;
   ppc*)
-    echo "CONFIG_PPC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_PPC_DIS=y"  >> config-all-disas.mak
+    disas_config "PPC"
   ;;
   s390*)
-    echo "CONFIG_S390_DIS=y"  >> $config_target_mak
-    echo "CONFIG_S390_DIS=y"  >> config-all-disas.mak
+    disas_config "S390"
   ;;
   sh4)
-    echo "CONFIG_SH4_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SH4_DIS=y"  >> config-all-disas.mak
+    disas_config "SH4"
   ;;
   sparc*)
-    echo "CONFIG_SPARC_DIS=y"  >> $config_target_mak
-    echo "CONFIG_SPARC_DIS=y"  >> config-all-disas.mak
+    disas_config "SPARC"
   ;;
   xtensa*)
-    echo "CONFIG_XTENSA_DIS=y"  >> $config_target_mak
-    echo "CONFIG_XTENSA_DIS=y"  >> config-all-disas.mak
+    disas_config "XTENSA"
   ;;
   esac
 done
 if test "$tcg_interpreter" = "yes" ; then
-  echo "CONFIG_TCI_DIS=y"  >> $config_target_mak
-  echo "CONFIG_TCI_DIS=y"  >> config-all-disas.mak
+  disas_config "TCI"
 fi
 
 case "$ARCH" in
@@ -5357,10 +5965,6 @@
   echo "config-host.h: subdir-pixman" >> $config_host_mak
 fi
 
-if test "$rdma" = "yes" ; then
-echo "CONFIG_RDMA=y" >> $config_host_mak
-fi
-
 if [ "$dtc_internal" = "yes" ]; then
   echo "config-host.h: subdir-dtc" >> $config_host_mak
 fi
@@ -5369,8 +5973,8 @@
   echo "CONFIG_NUMA=y" >> $config_host_mak
 fi
 
-if test "$android" = "yes"; then
-  echo "CONFIG_ANDROID=y" >> $config_host_mak
+if test "$ccache_cpp2" = "yes"; then
+  echo "export CCACHE_CPP2=y" >> $config_host_mak
 fi
 
 # build tree in object directory in case the source is not in the current directory
@@ -5397,11 +6001,11 @@
     $source_path/pc-bios/u-boot.* \
     $source_path/pc-bios/palcode-*
 do
-    FILES="$FILES pc-bios/`basename $bios_file`"
+    FILES="$FILES pc-bios/$(basename $bios_file)"
 done
-for test_file in `find $source_path/tests/acpi-test-data -type f`
+for test_file in $(find $source_path/tests/acpi-test-data -type f)
 do
-    FILES="$FILES tests/acpi-test-data`echo $test_file | sed -e 's/.*acpi-test-data//'`"
+    FILES="$FILES tests/acpi-test-data$(echo $test_file | sed -e 's/.*acpi-test-data//')"
 done
 mkdir -p $DIRS
 for f in $FILES ; do
@@ -5416,6 +6020,7 @@
     echo "# Automatically generated by configure - do not modify" > $config_mak
     echo "SRC_PATH=$source_path/roms/$rom" >> $config_mak
     echo "AS=$as" >> $config_mak
+    echo "CCAS=$ccas" >> $config_mak
     echo "CC=$cc" >> $config_mak
     echo "BCC=bcc" >> $config_mak
     echo "CPP=$cpp" >> $config_mak
@@ -5424,6 +6029,11 @@
     echo "LD=$ld" >> $config_mak
 done
 
+# set up tests data directory
+if [ ! -e tests/data ]; then
+    symlink "$source_path/tests/data" tests/data
+fi
+
 # set up qemu-iotests in this build directory
 iotests_common_env="tests/qemu-iotests/common.env"
 iotests_check="tests/qemu-iotests/check"
@@ -5446,7 +6056,7 @@
 EOD
 printf "exec" >>config.status
 printf " '%s'" "$0" "$@" >>config.status
-echo >>config.status
+echo ' "$@"' >>config.status
 chmod +x config.status
 
 rm -r "$TMPDIR1"
diff --git a/contrib/ivshmem-client/Makefile.objs b/contrib/ivshmem-client/Makefile.objs
new file mode 100644
index 0000000..bfab2d2
--- /dev/null
+++ b/contrib/ivshmem-client/Makefile.objs
@@ -0,0 +1 @@
+ivshmem-client-obj-y = ivshmem-client.o main.o
diff --git a/contrib/ivshmem-client/ivshmem-client.c b/contrib/ivshmem-client/ivshmem-client.c
new file mode 100644
index 0000000..44ae364
--- /dev/null
+++ b/contrib/ivshmem-client/ivshmem-client.c
@@ -0,0 +1,446 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "qemu-common.h"
+#include "qemu/queue.h"
+
+#include "ivshmem-client.h"
+
+/* log a message on stdout if verbose=1 */
+#define IVSHMEM_CLIENT_DEBUG(client, fmt, ...) do { \
+        if ((client)->verbose) {         \
+            printf(fmt, ## __VA_ARGS__); \
+        }                                \
+    } while (0)
+
+/* read message from the unix socket */
+static int
+ivshmem_client_read_one_msg(IvshmemClient *client, int64_t *index, int *fd)
+{
+    int ret;
+    struct msghdr msg;
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int))];
+    } msg_control;
+    struct cmsghdr *cmsg;
+
+    iov[0].iov_base = index;
+    iov[0].iov_len = sizeof(*index);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+    msg.msg_control = &msg_control;
+    msg.msg_controllen = sizeof(msg_control);
+
+    ret = recvmsg(client->sock_fd, &msg, 0);
+    if (ret < sizeof(*index)) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read message: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+    if (ret == 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "lost connection to server\n");
+        return -1;
+    }
+
+    *index = GINT64_FROM_LE(*index);
+    *fd = -1;
+
+    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+
+        if (cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
+            cmsg->cmsg_level != SOL_SOCKET ||
+            cmsg->cmsg_type != SCM_RIGHTS) {
+            continue;
+        }
+
+        memcpy(fd, CMSG_DATA(cmsg), sizeof(*fd));
+    }
+
+    return 0;
+}
+
+/* free a peer when the server advertises a disconnection or when the
+ * client is freed */
+static void
+ivshmem_client_free_peer(IvshmemClient *client, IvshmemClientPeer *peer)
+{
+    unsigned vector;
+
+    QTAILQ_REMOVE(&client->peer_list, peer, next);
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        close(peer->vectors[vector]);
+    }
+
+    g_free(peer);
+}
+
+/* handle message coming from server (new peer, new vectors) */
+static int
+ivshmem_client_handle_server_msg(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    int64_t peer_id;
+    int ret, fd;
+
+    ret = ivshmem_client_read_one_msg(client, &peer_id, &fd);
+    if (ret < 0) {
+        return -1;
+    }
+
+    /* can return a peer or the local client */
+    peer = ivshmem_client_search_peer(client, peer_id);
+
+    /* delete peer */
+    if (fd == -1) {
+
+        if (peer == NULL || peer == &client->local) {
+            IVSHMEM_CLIENT_DEBUG(client, "receive delete for invalid "
+                                 "peer %" PRId64 "\n", peer_id);
+            return -1;
+        }
+
+        IVSHMEM_CLIENT_DEBUG(client, "delete peer id = %" PRId64 "\n", peer_id);
+        ivshmem_client_free_peer(client, peer);
+        return 0;
+    }
+
+    /* new peer */
+    if (peer == NULL) {
+        peer = g_malloc0(sizeof(*peer));
+        peer->id = peer_id;
+        peer->vectors_count = 0;
+        QTAILQ_INSERT_TAIL(&client->peer_list, peer, next);
+        IVSHMEM_CLIENT_DEBUG(client, "new peer id = %" PRId64 "\n", peer_id);
+    }
+
+    /* new vector */
+    IVSHMEM_CLIENT_DEBUG(client, "  new vector %d (fd=%d) for peer id %"
+                         PRId64 "\n", peer->vectors_count, fd, peer->id);
+    if (peer->vectors_count >= G_N_ELEMENTS(peer->vectors)) {
+        IVSHMEM_CLIENT_DEBUG(client, "Too many vectors received, failing");
+        return -1;
+    }
+
+    peer->vectors[peer->vectors_count] = fd;
+    peer->vectors_count++;
+
+    return 0;
+}
+
+/* init a new ivshmem client */
+int
+ivshmem_client_init(IvshmemClient *client, const char *unix_sock_path,
+                    IvshmemClientNotifCb notif_cb, void *notif_arg,
+                    bool verbose)
+{
+    int ret;
+    unsigned i;
+
+    memset(client, 0, sizeof(*client));
+
+    ret = snprintf(client->unix_sock_path, sizeof(client->unix_sock_path),
+                   "%s", unix_sock_path);
+
+    if (ret < 0 || ret >= sizeof(client->unix_sock_path)) {
+        IVSHMEM_CLIENT_DEBUG(client, "could not copy unix socket path\n");
+        return -1;
+    }
+
+    for (i = 0; i < IVSHMEM_CLIENT_MAX_VECTORS; i++) {
+        client->local.vectors[i] = -1;
+    }
+
+    QTAILQ_INIT(&client->peer_list);
+    client->local.id = -1;
+
+    client->notif_cb = notif_cb;
+    client->notif_arg = notif_arg;
+    client->verbose = verbose;
+    client->shm_fd = -1;
+    client->sock_fd = -1;
+
+    return 0;
+}
+
+/* create and connect to the unix socket */
+int
+ivshmem_client_connect(IvshmemClient *client)
+{
+    struct sockaddr_un sun;
+    int fd, ret;
+    int64_t tmp;
+
+    IVSHMEM_CLIENT_DEBUG(client, "connect to client %s\n",
+                         client->unix_sock_path);
+
+    client->sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (client->sock_fd < 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot create socket: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    sun.sun_family = AF_UNIX;
+    ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
+                   client->unix_sock_path);
+    if (ret < 0 || ret >= sizeof(sun.sun_path)) {
+        IVSHMEM_CLIENT_DEBUG(client, "could not copy unix socket path\n");
+        goto err_close;
+    }
+
+    if (connect(client->sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot connect to %s: %s\n", sun.sun_path,
+                             strerror(errno));
+        goto err_close;
+    }
+
+    /* first, we expect a protocol version */
+    if (ivshmem_client_read_one_msg(client, &tmp, &fd) < 0 ||
+        (tmp != IVSHMEM_PROTOCOL_VERSION) || fd != -1) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server\n");
+        goto err_close;
+    }
+
+    /* then, we expect our index + a fd == -1 */
+    if (ivshmem_client_read_one_msg(client, &client->local.id, &fd) < 0 ||
+        client->local.id < 0 || fd != -1) {
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server (2)\n");
+        goto err_close;
+    }
+    IVSHMEM_CLIENT_DEBUG(client, "our_id=%" PRId64 "\n", client->local.id);
+
+    /* now, we expect shared mem fd + a -1 index, note that shm fd
+     * is not used */
+    if (ivshmem_client_read_one_msg(client, &tmp, &fd) < 0 ||
+        tmp != -1 || fd < 0) {
+        if (fd >= 0) {
+            close(fd);
+        }
+        IVSHMEM_CLIENT_DEBUG(client, "cannot read from server (3)\n");
+        goto err_close;
+    }
+    client->shm_fd = fd;
+    IVSHMEM_CLIENT_DEBUG(client, "shm_fd=%d\n", fd);
+
+    return 0;
+
+err_close:
+    close(client->sock_fd);
+    client->sock_fd = -1;
+    return -1;
+}
+
+/* close connection to the server, and free all peer structures */
+void
+ivshmem_client_close(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    unsigned i;
+
+    IVSHMEM_CLIENT_DEBUG(client, "close client\n");
+
+    while ((peer = QTAILQ_FIRST(&client->peer_list)) != NULL) {
+        ivshmem_client_free_peer(client, peer);
+    }
+
+    close(client->shm_fd);
+    client->shm_fd = -1;
+    close(client->sock_fd);
+    client->sock_fd = -1;
+    client->local.id = -1;
+    for (i = 0; i < IVSHMEM_CLIENT_MAX_VECTORS; i++) {
+        close(client->local.vectors[i]);
+        client->local.vectors[i] = -1;
+    }
+    client->local.vectors_count = 0;
+}
+
+/* get the fd_set according to the unix socket and peer list */
+void
+ivshmem_client_get_fds(const IvshmemClient *client, fd_set *fds, int *maxfd)
+{
+    int fd;
+    unsigned vector;
+
+    FD_SET(client->sock_fd, fds);
+    if (client->sock_fd >= *maxfd) {
+        *maxfd = client->sock_fd + 1;
+    }
+
+    for (vector = 0; vector < client->local.vectors_count; vector++) {
+        fd = client->local.vectors[vector];
+        FD_SET(fd, fds);
+        if (fd >= *maxfd) {
+            *maxfd = fd + 1;
+        }
+    }
+}
+
+/* handle events from eventfd: just print a message on notification */
+static int
+ivshmem_client_handle_event(IvshmemClient *client, const fd_set *cur, int maxfd)
+{
+    IvshmemClientPeer *peer;
+    uint64_t kick;
+    unsigned i;
+    int ret;
+
+    peer = &client->local;
+
+    for (i = 0; i < peer->vectors_count; i++) {
+        if (peer->vectors[i] >= maxfd || !FD_ISSET(peer->vectors[i], cur)) {
+            continue;
+        }
+
+        ret = read(peer->vectors[i], &kick, sizeof(kick));
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret != sizeof(kick)) {
+            IVSHMEM_CLIENT_DEBUG(client, "invalid read size = %d\n", ret);
+            errno = EINVAL;
+            return -1;
+        }
+        IVSHMEM_CLIENT_DEBUG(client, "received event on fd %d vector %d: %"
+                             PRIu64 "\n", peer->vectors[i], i, kick);
+        if (client->notif_cb != NULL) {
+            client->notif_cb(client, peer, i, client->notif_arg);
+        }
+    }
+
+    return 0;
+}
+
+/* read and handle new messages on the given fd_set */
+int
+ivshmem_client_handle_fds(IvshmemClient *client, fd_set *fds, int maxfd)
+{
+    if (client->sock_fd < maxfd && FD_ISSET(client->sock_fd, fds) &&
+        ivshmem_client_handle_server_msg(client) < 0 && errno != EINTR) {
+        IVSHMEM_CLIENT_DEBUG(client, "ivshmem_client_handle_server_msg() "
+                             "failed\n");
+        return -1;
+    } else if (ivshmem_client_handle_event(client, fds, maxfd) < 0 &&
+               errno != EINTR) {
+        IVSHMEM_CLIENT_DEBUG(client, "ivshmem_client_handle_event() failed\n");
+        return -1;
+    }
+
+    return 0;
+}
+
+/* send a notification on a vector of a peer */
+int
+ivshmem_client_notify(const IvshmemClient *client,
+                      const IvshmemClientPeer *peer, unsigned vector)
+{
+    uint64_t kick;
+    int fd;
+
+    if (vector >= peer->vectors_count) {
+        IVSHMEM_CLIENT_DEBUG(client, "invalid vector %u on peer %" PRId64 "\n",
+                             vector, peer->id);
+        return -1;
+    }
+    fd = peer->vectors[vector];
+    IVSHMEM_CLIENT_DEBUG(client, "notify peer %" PRId64
+                         " on vector %d, fd %d\n", peer->id, vector, fd);
+
+    kick = 1;
+    if (write(fd, &kick, sizeof(kick)) != sizeof(kick)) {
+        fprintf(stderr, "could not write to %d: %s\n", peer->vectors[vector],
+                strerror(errno));
+        return -1;
+    }
+    return 0;
+}
+
+/* send a notification to all vectors of a peer */
+int
+ivshmem_client_notify_all_vects(const IvshmemClient *client,
+                                const IvshmemClientPeer *peer)
+{
+    unsigned vector;
+    int ret = 0;
+
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        if (ivshmem_client_notify(client, peer, vector) < 0) {
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+/* send a notification to all peers */
+int
+ivshmem_client_notify_broadcast(const IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    int ret = 0;
+
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        if (ivshmem_client_notify_all_vects(client, peer) < 0) {
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+/* lookup peer from its id */
+IvshmemClientPeer *
+ivshmem_client_search_peer(IvshmemClient *client, int64_t peer_id)
+{
+    IvshmemClientPeer *peer;
+
+    if (peer_id == client->local.id) {
+        return &client->local;
+    }
+
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        if (peer->id == peer_id) {
+            return peer;
+        }
+    }
+    return NULL;
+}
+
+/* dump our info, the list of peers their vectors on stdout */
+void
+ivshmem_client_dump(const IvshmemClient *client)
+{
+    const IvshmemClientPeer *peer;
+    unsigned vector;
+
+    /* dump local infos */
+    peer = &client->local;
+    printf("our_id = %" PRId64 "\n", peer->id);
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        printf("  vector %d is enabled (fd=%d)\n", vector,
+               peer->vectors[vector]);
+    }
+
+    /* dump peers */
+    QTAILQ_FOREACH(peer, &client->peer_list, next) {
+        printf("peer_id = %" PRId64 "\n", peer->id);
+
+        for (vector = 0; vector < peer->vectors_count; vector++) {
+            printf("  vector %d is enabled (fd=%d)\n", vector,
+                   peer->vectors[vector]);
+        }
+    }
+}
diff --git a/contrib/ivshmem-client/ivshmem-client.h b/contrib/ivshmem-client/ivshmem-client.h
new file mode 100644
index 0000000..5ee9422
--- /dev/null
+++ b/contrib/ivshmem-client/ivshmem-client.h
@@ -0,0 +1,212 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef IVSHMEM_CLIENT_H
+#define IVSHMEM_CLIENT_H
+
+/**
+ * This file provides helper to implement an ivshmem client. It is used
+ * on the host to ask QEMU to send an interrupt to an ivshmem PCI device in a
+ * guest. QEMU also implements an ivshmem client similar to this one, they both
+ * connect to an ivshmem server.
+ *
+ * A standalone ivshmem client based on this file is provided for debug/test
+ * purposes.
+ */
+
+#include <sys/select.h>
+
+#include "qemu/queue.h"
+#include "hw/misc/ivshmem.h"
+
+/**
+ * Maximum number of notification vectors supported by the client
+ */
+#define IVSHMEM_CLIENT_MAX_VECTORS 64
+
+/**
+ * Structure storing a peer
+ *
+ * Each time a client connects to an ivshmem server, it is advertised to
+ * all connected clients through the unix socket. When our ivshmem
+ * client receives a notification, it creates a IvshmemClientPeer
+ * structure to store the infos of this peer.
+ *
+ * This structure is also used to store the information of our own
+ * client in (IvshmemClient)->local.
+ */
+typedef struct IvshmemClientPeer {
+    QTAILQ_ENTRY(IvshmemClientPeer) next;    /**< next in list*/
+    int64_t id;                              /**< the id of the peer */
+    int vectors[IVSHMEM_CLIENT_MAX_VECTORS]; /**< one fd per vector */
+    unsigned vectors_count;                  /**< number of vectors */
+} IvshmemClientPeer;
+QTAILQ_HEAD(IvshmemClientPeerList, IvshmemClientPeer);
+
+typedef struct IvshmemClientPeerList IvshmemClientPeerList;
+typedef struct IvshmemClient IvshmemClient;
+
+/**
+ * Typedef of callback function used when our IvshmemClient receives a
+ * notification from a peer.
+ */
+typedef void (*IvshmemClientNotifCb)(
+    const IvshmemClient *client,
+    const IvshmemClientPeer *peer,
+    unsigned vect, void *arg);
+
+/**
+ * Structure describing an ivshmem client
+ *
+ * This structure stores all information related to our client: the name
+ * of the server unix socket, the list of peers advertised by the
+ * server, our own client information, and a pointer the notification
+ * callback function used when we receive a notification from a peer.
+ */
+struct IvshmemClient {
+    char unix_sock_path[PATH_MAX];      /**< path to unix sock */
+    int sock_fd;                        /**< unix sock filedesc */
+    int shm_fd;                         /**< shm file descriptor */
+
+    IvshmemClientPeerList peer_list;    /**< list of peers */
+    IvshmemClientPeer local;            /**< our own infos */
+
+    IvshmemClientNotifCb notif_cb;      /**< notification callback */
+    void *notif_arg;                    /**< notification argument */
+
+    bool verbose;                       /**< true to enable debug */
+};
+
+/**
+ * Initialize an ivshmem client
+ *
+ * @client:         A pointer to an uninitialized IvshmemClient structure
+ * @unix_sock_path: The pointer to the unix socket file name
+ * @notif_cb:       If not NULL, the pointer to the function to be called when
+ *                  our IvshmemClient receives a notification from a peer
+ * @notif_arg:      Opaque pointer given as-is to the notification callback
+ *                  function
+ * @verbose:        True to enable debug
+ *
+ * Returns:         0 on success, or a negative value on error
+ */
+int ivshmem_client_init(IvshmemClient *client, const char *unix_sock_path,
+                        IvshmemClientNotifCb notif_cb, void *notif_arg,
+                        bool verbose);
+
+/**
+ * Connect to the server
+ *
+ * Connect to the server unix socket, and read the first initial
+ * messages sent by the server, giving the ID of the client and the file
+ * descriptor of the shared memory.
+ *
+ * @client: The ivshmem client
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_connect(IvshmemClient *client);
+
+/**
+ * Close connection to the server and free all peer structures
+ *
+ * @client: The ivshmem client
+ */
+void ivshmem_client_close(IvshmemClient *client);
+
+/**
+ * Fill a fd_set with file descriptors to be monitored
+ *
+ * This function will fill a fd_set with all file descriptors
+ * that must be polled (unix server socket and peers eventfd). The
+ * function will not initialize the fd_set, it is up to the caller
+ * to do this.
+ *
+ * @client: The ivshmem client
+ * @fds:    The fd_set to be updated
+ * @maxfd:  Must be set to the max file descriptor + 1 in fd_set. This value is
+ *          updated if this function adds a greater fd in fd_set.
+ */
+void ivshmem_client_get_fds(const IvshmemClient *client, fd_set *fds,
+                            int *maxfd);
+
+/**
+ * Read and handle new messages
+ *
+ * Given a fd_set filled by select(), handle incoming messages from
+ * server or peers.
+ *
+ * @client: The ivshmem client
+ * @fds:    The fd_set containing the file descriptors to be checked. Note
+ *          that file descriptors that are not related to our client are
+ *          ignored.
+ * @maxfd:  The maximum fd in fd_set, plus one.
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_handle_fds(IvshmemClient *client, fd_set *fds, int maxfd);
+
+/**
+ * Send a notification to a vector of a peer
+ *
+ * @client: The ivshmem client
+ * @peer:   The peer to be notified
+ * @vector: The number of the vector
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_client_notify(const IvshmemClient *client,
+                          const IvshmemClientPeer *peer, unsigned vector);
+
+/**
+ * Send a notification to all vectors of a peer
+ *
+ * @client: The ivshmem client
+ * @peer:   The peer to be notified
+ *
+ * Returns: 0 on success, or a negative value on error (at least one
+ *          notification failed)
+ */
+int ivshmem_client_notify_all_vects(const IvshmemClient *client,
+                                    const IvshmemClientPeer *peer);
+
+/**
+ * Broadcat a notification to all vectors of all peers
+ *
+ * @client: The ivshmem client
+ *
+ * Returns: 0 on success, or a negative value on error (at least one
+ *          notification failed)
+ */
+int ivshmem_client_notify_broadcast(const IvshmemClient *client);
+
+/**
+ * Search a peer from its identifier
+ *
+ * Return the peer structure from its peer_id. If the given peer_id is
+ * the local id, the function returns the local peer structure.
+ *
+ * @client:  The ivshmem client
+ * @peer_id: The identifier of the peer structure
+ *
+ * Returns:  The peer structure, or NULL if not found
+ */
+IvshmemClientPeer *
+ivshmem_client_search_peer(IvshmemClient *client, int64_t peer_id);
+
+/**
+ * Dump information of this ivshmem client on stdout
+ *
+ * Dump the id and the vectors of the given ivshmem client and the list
+ * of its peers and their vectors on stdout.
+ *
+ * @client: The ivshmem client
+ */
+void ivshmem_client_dump(const IvshmemClient *client);
+
+#endif /* IVSHMEM_CLIENT_H */
diff --git a/contrib/ivshmem-client/main.c b/contrib/ivshmem-client/main.c
new file mode 100644
index 0000000..33ae1da
--- /dev/null
+++ b/contrib/ivshmem-client/main.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "ivshmem-client.h"
+
+#define IVSHMEM_CLIENT_DEFAULT_VERBOSE        0
+#define IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH "/tmp/ivshmem_socket"
+
+typedef struct IvshmemClientArgs {
+    bool verbose;
+    const char *unix_sock_path;
+} IvshmemClientArgs;
+
+/* show ivshmem_client_usage and exit with given error code */
+static void
+ivshmem_client_usage(const char *name, int code)
+{
+    fprintf(stderr, "%s [opts]\n", name);
+    fprintf(stderr, "  -h: show this help\n");
+    fprintf(stderr, "  -v: verbose mode\n");
+    fprintf(stderr, "  -S <unix_sock_path>: path to the unix socket\n"
+                    "     to connect to.\n"
+                    "     default=%s\n", IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH);
+    exit(code);
+}
+
+/* parse the program arguments, exit on error */
+static void
+ivshmem_client_parse_args(IvshmemClientArgs *args, int argc, char *argv[])
+{
+    int c;
+
+    while ((c = getopt(argc, argv,
+                       "h"  /* help */
+                       "v"  /* verbose */
+                       "S:" /* unix_sock_path */
+                      )) != -1) {
+
+        switch (c) {
+        case 'h': /* help */
+            ivshmem_client_usage(argv[0], 0);
+            break;
+
+        case 'v': /* verbose */
+            args->verbose = 1;
+            break;
+
+        case 'S': /* unix_sock_path */
+            args->unix_sock_path = optarg;
+            break;
+
+        default:
+            ivshmem_client_usage(argv[0], 1);
+            break;
+        }
+    }
+}
+
+/* show command line help */
+static void
+ivshmem_client_cmdline_help(void)
+{
+    printf("dump: dump peers (including us)\n"
+           "int <peer> <vector>: notify one vector on a peer\n"
+           "int <peer> all: notify all vectors of a peer\n"
+           "int all: notify all vectors of all peers (excepting us)\n");
+}
+
+/* read stdin and handle commands */
+static int
+ivshmem_client_handle_stdin_command(IvshmemClient *client)
+{
+    IvshmemClientPeer *peer;
+    char buf[128];
+    char *s, *token;
+    int ret;
+    int peer_id, vector;
+
+    memset(buf, 0, sizeof(buf));
+    ret = read(0, buf, sizeof(buf) - 1);
+    if (ret < 0) {
+        return -1;
+    }
+
+    s = buf;
+    while ((token = strsep(&s, "\n\r;")) != NULL) {
+        if (!strcmp(token, "")) {
+            continue;
+        }
+        if (!strcmp(token, "?")) {
+            ivshmem_client_cmdline_help();
+        }
+        if (!strcmp(token, "help")) {
+            ivshmem_client_cmdline_help();
+        } else if (!strcmp(token, "dump")) {
+            ivshmem_client_dump(client);
+        } else if (!strcmp(token, "int all")) {
+            ivshmem_client_notify_broadcast(client);
+        } else if (sscanf(token, "int %d %d", &peer_id, &vector) == 2) {
+            peer = ivshmem_client_search_peer(client, peer_id);
+            if (peer == NULL) {
+                printf("cannot find peer_id = %d\n", peer_id);
+                continue;
+            }
+            ivshmem_client_notify(client, peer, vector);
+        } else if (sscanf(token, "int %d all", &peer_id) == 1) {
+            peer = ivshmem_client_search_peer(client, peer_id);
+            if (peer == NULL) {
+                printf("cannot find peer_id = %d\n", peer_id);
+                continue;
+            }
+            ivshmem_client_notify_all_vects(client, peer);
+        } else {
+            printf("invalid command, type help\n");
+        }
+    }
+
+    printf("cmd> ");
+    fflush(stdout);
+    return 0;
+}
+
+/* listen on stdin (command line), on unix socket (notifications of new
+ * and dead peers), and on eventfd (IRQ request) */
+static int
+ivshmem_client_poll_events(IvshmemClient *client)
+{
+    fd_set fds;
+    int ret, maxfd;
+
+    while (1) {
+
+        FD_ZERO(&fds);
+        FD_SET(0, &fds); /* add stdin in fd_set */
+        maxfd = 1;
+
+        ivshmem_client_get_fds(client, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            fprintf(stderr, "select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (FD_ISSET(0, &fds) &&
+            ivshmem_client_handle_stdin_command(client) < 0 && errno != EINTR) {
+            fprintf(stderr, "ivshmem_client_handle_stdin_command() failed\n");
+            break;
+        }
+
+        if (ivshmem_client_handle_fds(client, &fds, maxfd) < 0) {
+            fprintf(stderr, "ivshmem_client_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return ret;
+}
+
+/* callback when we receive a notification (just display it) */
+static void
+ivshmem_client_notification_cb(const IvshmemClient *client,
+                               const IvshmemClientPeer *peer,
+                               unsigned vect, void *arg)
+{
+    (void)client;
+    (void)arg;
+    printf("receive notification from peer_id=%" PRId64 " vector=%u\n",
+           peer->id, vect);
+}
+
+int
+main(int argc, char *argv[])
+{
+    struct sigaction sa;
+    IvshmemClient client;
+    IvshmemClientArgs args = {
+        .verbose = IVSHMEM_CLIENT_DEFAULT_VERBOSE,
+        .unix_sock_path = IVSHMEM_CLIENT_DEFAULT_UNIX_SOCK_PATH,
+    };
+
+    /* parse arguments, will exit on error */
+    ivshmem_client_parse_args(&args, argc, argv);
+
+    /* Ignore SIGPIPE, see this link for more info:
+     * http://www.mail-archive.com/libevent-users@monkey.org/msg01606.html */
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = 0;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPIPE, &sa, 0) == -1) {
+        perror("failed to ignore SIGPIPE; sigaction");
+        return 1;
+    }
+
+    ivshmem_client_cmdline_help();
+    printf("cmd> ");
+    fflush(stdout);
+
+    if (ivshmem_client_init(&client, args.unix_sock_path,
+                            ivshmem_client_notification_cb, NULL,
+                            args.verbose) < 0) {
+        fprintf(stderr, "cannot init client\n");
+        return 1;
+    }
+
+    while (1) {
+        if (ivshmem_client_connect(&client) < 0) {
+            fprintf(stderr, "cannot connect to server, retry in 1 second\n");
+            sleep(1);
+            continue;
+        }
+
+        fprintf(stdout, "listen on server socket %d\n", client.sock_fd);
+
+        if (ivshmem_client_poll_events(&client) == 0) {
+            continue;
+        }
+
+        /* disconnected from server, reset all peers */
+        fprintf(stdout, "disconnected from server\n");
+
+        ivshmem_client_close(&client);
+    }
+
+    return 0;
+}
diff --git a/contrib/ivshmem-server/Makefile.objs b/contrib/ivshmem-server/Makefile.objs
new file mode 100644
index 0000000..c060dd3
--- /dev/null
+++ b/contrib/ivshmem-server/Makefile.objs
@@ -0,0 +1 @@
+ivshmem-server-obj-y = ivshmem-server.o main.o
diff --git a/contrib/ivshmem-server/ivshmem-server.c b/contrib/ivshmem-server/ivshmem-server.c
new file mode 100644
index 0000000..e2f295b
--- /dev/null
+++ b/contrib/ivshmem-server/ivshmem-server.c
@@ -0,0 +1,457 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/host-utils.h"
+#include "qemu/sockets.h"
+
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "ivshmem-server.h"
+
+/* log a message on stdout if verbose=1 */
+#define IVSHMEM_SERVER_DEBUG(server, fmt, ...) do { \
+        if ((server)->verbose) {         \
+            printf(fmt, ## __VA_ARGS__); \
+        }                                \
+    } while (0)
+
+/** maximum size of a huge page, used by ivshmem_server_ftruncate() */
+#define IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE (1024 * 1024 * 1024)
+
+/** default listen backlog (number of sockets not accepted) */
+#define IVSHMEM_SERVER_LISTEN_BACKLOG 10
+
+/* send message to a client unix socket */
+static int
+ivshmem_server_send_one_msg(int sock_fd, int64_t peer_id, int fd)
+{
+    int ret;
+    struct msghdr msg;
+    struct iovec iov[1];
+    union {
+        struct cmsghdr cmsg;
+        char control[CMSG_SPACE(sizeof(int))];
+    } msg_control;
+    struct cmsghdr *cmsg;
+
+    peer_id = GINT64_TO_LE(peer_id);
+    iov[0].iov_base = &peer_id;
+    iov[0].iov_len = sizeof(peer_id);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = iov;
+    msg.msg_iovlen = 1;
+
+    /* if fd is specified, add it in a cmsg */
+    if (fd >= 0) {
+        memset(&msg_control, 0, sizeof(msg_control));
+        msg.msg_control = &msg_control;
+        msg.msg_controllen = sizeof(msg_control);
+        cmsg = CMSG_FIRSTHDR(&msg);
+        cmsg->cmsg_level = SOL_SOCKET;
+        cmsg->cmsg_type = SCM_RIGHTS;
+        cmsg->cmsg_len = CMSG_LEN(sizeof(int));
+        memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd));
+    }
+
+    ret = sendmsg(sock_fd, &msg, 0);
+    if (ret <= 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/* free a peer when the server advertises a disconnection or when the
+ * server is freed */
+static void
+ivshmem_server_free_peer(IvshmemServer *server, IvshmemServerPeer *peer)
+{
+    unsigned vector;
+    IvshmemServerPeer *other_peer;
+
+    IVSHMEM_SERVER_DEBUG(server, "free peer %" PRId64 "\n", peer->id);
+    close(peer->sock_fd);
+    QTAILQ_REMOVE(&server->peer_list, peer, next);
+
+    /* advertise the deletion to other peers */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id, -1);
+    }
+
+    for (vector = 0; vector < peer->vectors_count; vector++) {
+        event_notifier_cleanup(&peer->vectors[vector]);
+    }
+
+    g_free(peer);
+}
+
+/* send the peer id and the shm_fd just after a new client connection */
+static int
+ivshmem_server_send_initial_info(IvshmemServer *server, IvshmemServerPeer *peer)
+{
+    int ret;
+
+    /* send our protocol version first */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, IVSHMEM_PROTOCOL_VERSION,
+                                      -1);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send version: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    /* send the peer id to the client */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, peer->id, -1);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send peer id: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    /* send the shm_fd */
+    ret = ivshmem_server_send_one_msg(peer->sock_fd, -1, server->shm_fd);
+    if (ret < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send shm fd: %s\n",
+                             strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/* handle message on listening unix socket (new client connection) */
+static int
+ivshmem_server_handle_new_conn(IvshmemServer *server)
+{
+    IvshmemServerPeer *peer, *other_peer;
+    struct sockaddr_un unaddr;
+    socklen_t unaddr_len;
+    int newfd;
+    unsigned i;
+
+    /* accept the incoming connection */
+    unaddr_len = sizeof(unaddr);
+    newfd = qemu_accept(server->sock_fd,
+                        (struct sockaddr *)&unaddr, &unaddr_len);
+
+    if (newfd < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot accept() %s\n", strerror(errno));
+        return -1;
+    }
+
+    qemu_set_nonblock(newfd);
+    IVSHMEM_SERVER_DEBUG(server, "accept()=%d\n", newfd);
+
+    /* allocate new structure for this peer */
+    peer = g_malloc0(sizeof(*peer));
+    peer->sock_fd = newfd;
+
+    /* get an unused peer id */
+    /* XXX: this could use id allocation such as Linux IDA, or simply
+     * a free-list */
+    for (i = 0; i < G_MAXUINT16; i++) {
+        if (ivshmem_server_search_peer(server, server->cur_id) == NULL) {
+            break;
+        }
+        server->cur_id++;
+    }
+    if (i == G_MAXUINT16) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot allocate new client id\n");
+        close(newfd);
+        g_free(peer);
+        return -1;
+    }
+    peer->id = server->cur_id++;
+
+    /* create eventfd, one per vector */
+    peer->vectors_count = server->n_vectors;
+    for (i = 0; i < peer->vectors_count; i++) {
+        if (event_notifier_init(&peer->vectors[i], FALSE) < 0) {
+            IVSHMEM_SERVER_DEBUG(server, "cannot create eventfd\n");
+            goto fail;
+        }
+    }
+
+    /* send peer id and shm fd */
+    if (ivshmem_server_send_initial_info(server, peer) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot send initial info\n");
+        goto fail;
+    }
+
+    /* advertise the new peer to others */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        for (i = 0; i < peer->vectors_count; i++) {
+            ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id,
+                                        peer->vectors[i].wfd);
+        }
+    }
+
+    /* advertise the other peers to the new one */
+    QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
+        for (i = 0; i < peer->vectors_count; i++) {
+            ivshmem_server_send_one_msg(peer->sock_fd, other_peer->id,
+                                        other_peer->vectors[i].wfd);
+        }
+    }
+
+    /* advertise the new peer to itself */
+    for (i = 0; i < peer->vectors_count; i++) {
+        ivshmem_server_send_one_msg(peer->sock_fd, peer->id,
+                                    event_notifier_get_fd(&peer->vectors[i]));
+    }
+
+    QTAILQ_INSERT_TAIL(&server->peer_list, peer, next);
+    IVSHMEM_SERVER_DEBUG(server, "new peer id = %" PRId64 "\n",
+                         peer->id);
+    return 0;
+
+fail:
+    while (i--) {
+        event_notifier_cleanup(&peer->vectors[i]);
+    }
+    close(newfd);
+    g_free(peer);
+    return -1;
+}
+
+/* Try to ftruncate a file to next power of 2 of shmsize.
+ * If it fails; all power of 2 above shmsize are tested until
+ * we reach the maximum huge page size. This is useful
+ * if the shm file is in a hugetlbfs that cannot be truncated to the
+ * shm_size value. */
+static int
+ivshmem_server_ftruncate(int fd, unsigned shmsize)
+{
+    int ret;
+    struct stat mapstat;
+
+    /* align shmsize to next power of 2 */
+    shmsize = pow2ceil(shmsize);
+
+    if (fstat(fd, &mapstat) != -1 && mapstat.st_size == shmsize) {
+        return 0;
+    }
+
+    while (shmsize <= IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE) {
+        ret = ftruncate(fd, shmsize);
+        if (ret == 0) {
+            return ret;
+        }
+        shmsize *= 2;
+    }
+
+    return -1;
+}
+
+/* Init a new ivshmem server */
+int
+ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
+                    const char *shm_path, bool use_shm_open,
+                    size_t shm_size, unsigned n_vectors,
+                    bool verbose)
+{
+    int ret;
+
+    memset(server, 0, sizeof(*server));
+    server->verbose = verbose;
+
+    ret = snprintf(server->unix_sock_path, sizeof(server->unix_sock_path),
+                   "%s", unix_sock_path);
+    if (ret < 0 || ret >= sizeof(server->unix_sock_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
+        return -1;
+    }
+    ret = snprintf(server->shm_path, sizeof(server->shm_path),
+                   "%s", shm_path);
+    if (ret < 0 || ret >= sizeof(server->shm_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy shm path\n");
+        return -1;
+    }
+
+    server->use_shm_open = use_shm_open;
+    server->shm_size = shm_size;
+    server->n_vectors = n_vectors;
+
+    QTAILQ_INIT(&server->peer_list);
+
+    return 0;
+}
+
+/* open shm, create and bind to the unix socket */
+int
+ivshmem_server_start(IvshmemServer *server)
+{
+    struct sockaddr_un sun;
+    int shm_fd, sock_fd, ret;
+
+    /* open shm file */
+    if (server->use_shm_open) {
+        IVSHMEM_SERVER_DEBUG(server, "Using POSIX shared memory: %s\n",
+                             server->shm_path);
+        shm_fd = shm_open(server->shm_path, O_CREAT | O_RDWR, S_IRWXU);
+    } else {
+        gchar *filename = g_strdup_printf("%s/ivshmem.XXXXXX", server->shm_path);
+        IVSHMEM_SERVER_DEBUG(server, "Using file-backed shared memory: %s\n",
+                             server->shm_path);
+        shm_fd = mkstemp(filename);
+        unlink(filename);
+        g_free(filename);
+    }
+
+    if (shm_fd < 0) {
+        fprintf(stderr, "cannot open shm file %s: %s\n", server->shm_path,
+                strerror(errno));
+        return -1;
+    }
+    if (ivshmem_server_ftruncate(shm_fd, server->shm_size) < 0) {
+        fprintf(stderr, "ftruncate(%s) failed: %s\n", server->shm_path,
+                strerror(errno));
+        goto err_close_shm;
+    }
+
+    IVSHMEM_SERVER_DEBUG(server, "create & bind socket %s\n",
+                         server->unix_sock_path);
+
+    /* create the unix listening socket */
+    sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (sock_fd < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot create socket: %s\n",
+                             strerror(errno));
+        goto err_close_shm;
+    }
+
+    sun.sun_family = AF_UNIX;
+    ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
+                   server->unix_sock_path);
+    if (ret < 0 || ret >= sizeof(sun.sun_path)) {
+        IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
+        goto err_close_sock;
+    }
+    if (bind(sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "cannot connect to %s: %s\n", sun.sun_path,
+                             strerror(errno));
+        goto err_close_sock;
+    }
+
+    if (listen(sock_fd, IVSHMEM_SERVER_LISTEN_BACKLOG) < 0) {
+        IVSHMEM_SERVER_DEBUG(server, "listen() failed: %s\n", strerror(errno));
+        goto err_close_sock;
+    }
+
+    server->sock_fd = sock_fd;
+    server->shm_fd = shm_fd;
+
+    return 0;
+
+err_close_sock:
+    close(sock_fd);
+err_close_shm:
+    close(shm_fd);
+    return -1;
+}
+
+/* close connections to clients, the unix socket and the shm fd */
+void
+ivshmem_server_close(IvshmemServer *server)
+{
+    IvshmemServerPeer *peer, *npeer;
+
+    IVSHMEM_SERVER_DEBUG(server, "close server\n");
+
+    QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, npeer) {
+        ivshmem_server_free_peer(server, peer);
+    }
+
+    unlink(server->unix_sock_path);
+    close(server->sock_fd);
+    close(server->shm_fd);
+    server->sock_fd = -1;
+    server->shm_fd = -1;
+}
+
+/* get the fd_set according to the unix socket and the peer list */
+void
+ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd)
+{
+    IvshmemServerPeer *peer;
+
+    if (server->sock_fd == -1) {
+        return;
+    }
+
+    FD_SET(server->sock_fd, fds);
+    if (server->sock_fd >= *maxfd) {
+        *maxfd = server->sock_fd + 1;
+    }
+
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        FD_SET(peer->sock_fd, fds);
+        if (peer->sock_fd >= *maxfd) {
+            *maxfd = peer->sock_fd + 1;
+        }
+    }
+}
+
+/* process incoming messages on the sockets in fd_set */
+int
+ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd)
+{
+    IvshmemServerPeer *peer, *peer_next;
+
+    if (server->sock_fd < maxfd && FD_ISSET(server->sock_fd, fds) &&
+        ivshmem_server_handle_new_conn(server) < 0 && errno != EINTR) {
+        IVSHMEM_SERVER_DEBUG(server, "ivshmem_server_handle_new_conn() "
+                             "failed\n");
+        return -1;
+    }
+
+    QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, peer_next) {
+        /* any message from a peer socket result in a close() */
+        IVSHMEM_SERVER_DEBUG(server, "peer->sock_fd=%d\n", peer->sock_fd);
+        if (peer->sock_fd < maxfd && FD_ISSET(peer->sock_fd, fds)) {
+            ivshmem_server_free_peer(server, peer);
+        }
+    }
+
+    return 0;
+}
+
+/* lookup peer from its id */
+IvshmemServerPeer *
+ivshmem_server_search_peer(IvshmemServer *server, int64_t peer_id)
+{
+    IvshmemServerPeer *peer;
+
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        if (peer->id == peer_id) {
+            return peer;
+        }
+    }
+    return NULL;
+}
+
+/* dump our info, the list of peers their vectors on stdout */
+void
+ivshmem_server_dump(const IvshmemServer *server)
+{
+    const IvshmemServerPeer *peer;
+    unsigned vector;
+
+    /* dump peers */
+    QTAILQ_FOREACH(peer, &server->peer_list, next) {
+        printf("peer_id = %" PRId64 "\n", peer->id);
+
+        for (vector = 0; vector < peer->vectors_count; vector++) {
+            printf("  vector %d is enabled (fd=%d)\n", vector,
+                   event_notifier_get_fd(&peer->vectors[vector]));
+        }
+    }
+}
diff --git a/contrib/ivshmem-server/ivshmem-server.h b/contrib/ivshmem-server/ivshmem-server.h
new file mode 100644
index 0000000..4af08e1
--- /dev/null
+++ b/contrib/ivshmem-server/ivshmem-server.h
@@ -0,0 +1,166 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef IVSHMEM_SERVER_H
+#define IVSHMEM_SERVER_H
+
+/**
+ * The ivshmem server is a daemon that creates a unix socket in listen
+ * mode. The ivshmem clients (qemu or ivshmem-client) connect to this
+ * unix socket. For each client, the server will create some eventfd
+ * (see EVENTFD(2)), one per vector. These fd are transmitted to all
+ * clients using the SCM_RIGHTS cmsg message. Therefore, each client is
+ * able to send a notification to another client without being
+ * "profixied" by the server.
+ *
+ * We use this mechanism to send interruptions between guests.
+ * qemu is able to transform an event on a eventfd into a PCI MSI-x
+ * interruption in the guest.
+ *
+ * The ivshmem server is also able to share the file descriptor
+ * associated to the ivshmem shared memory.
+ */
+
+#include <sys/select.h>
+
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "hw/misc/ivshmem.h"
+
+/**
+ * Maximum number of notification vectors supported by the server
+ */
+#define IVSHMEM_SERVER_MAX_VECTORS 64
+
+/**
+ * Structure storing a peer
+ *
+ * Each time a client connects to an ivshmem server, a new
+ * IvshmemServerPeer structure is created. This peer and all its
+ * vectors are advertised to all connected clients through the connected
+ * unix sockets.
+ */
+typedef struct IvshmemServerPeer {
+    QTAILQ_ENTRY(IvshmemServerPeer) next;    /**< next in list*/
+    int sock_fd;                             /**< connected unix sock */
+    int64_t id;                              /**< the id of the peer */
+    EventNotifier vectors[IVSHMEM_SERVER_MAX_VECTORS]; /**< one per vector */
+    unsigned vectors_count;                  /**< number of vectors */
+} IvshmemServerPeer;
+QTAILQ_HEAD(IvshmemServerPeerList, IvshmemServerPeer);
+
+typedef struct IvshmemServerPeerList IvshmemServerPeerList;
+
+/**
+ * Structure describing an ivshmem server
+ *
+ * This structure stores all information related to our server: the name
+ * of the server unix socket and the list of connected peers.
+ */
+typedef struct IvshmemServer {
+    char unix_sock_path[PATH_MAX];   /**< path to unix socket */
+    int sock_fd;                     /**< unix sock file descriptor */
+    char shm_path[PATH_MAX];         /**< path to shm */
+    bool use_shm_open;
+    size_t shm_size;                 /**< size of shm */
+    int shm_fd;                      /**< shm file descriptor */
+    unsigned n_vectors;              /**< number of vectors */
+    uint16_t cur_id;                 /**< id to be given to next client */
+    bool verbose;                    /**< true in verbose mode */
+    IvshmemServerPeerList peer_list; /**< list of peers */
+} IvshmemServer;
+
+/**
+ * Initialize an ivshmem server
+ *
+ * @server:         A pointer to an uninitialized IvshmemServer structure
+ * @unix_sock_path: The pointer to the unix socket file name
+ * @shm_path:       Path to the shared memory. The path corresponds to a POSIX
+ *                  shm name or a hugetlbfs mount point.
+ * @shm_size:       Size of shared memory
+ * @n_vectors:      Number of interrupt vectors per client
+ * @verbose:        True to enable verbose mode
+ *
+ * Returns:         0 on success, or a negative value on error
+ */
+int
+ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
+                    const char *shm_path, bool use_shm_open,
+                    size_t shm_size, unsigned n_vectors,
+                    bool verbose);
+
+/**
+ * Open the shm, then create and bind to the unix socket
+ *
+ * @server: The pointer to the initialized IvshmemServer structure
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_server_start(IvshmemServer *server);
+
+/**
+ * Close the server
+ *
+ * Close connections to all clients, close the unix socket and the
+ * shared memory file descriptor. The structure remains initialized, so
+ * it is possible to call ivshmem_server_start() again after a call to
+ * ivshmem_server_close().
+ *
+ * @server: The ivshmem server
+ */
+void ivshmem_server_close(IvshmemServer *server);
+
+/**
+ * Fill a fd_set with file descriptors to be monitored
+ *
+ * This function will fill a fd_set with all file descriptors that must
+ * be polled (unix server socket and peers unix socket). The function
+ * will not initialize the fd_set, it is up to the caller to do it.
+ *
+ * @server: The ivshmem server
+ * @fds:    The fd_set to be updated
+ * @maxfd:  Must be set to the max file descriptor + 1 in fd_set. This value is
+ *          updated if this function adds a greater fd in fd_set.
+ */
+void
+ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd);
+
+/**
+ * Read and handle new messages
+ *
+ * Given a fd_set (for instance filled by a call to select()), handle
+ * incoming messages from peers.
+ *
+ * @server: The ivshmem server
+ * @fds:    The fd_set containing the file descriptors to be checked. Note that
+ *          file descriptors that are not related to our server are ignored.
+ * @maxfd:  The maximum fd in fd_set, plus one.
+ *
+ * Returns: 0 on success, or a negative value on error
+ */
+int ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd);
+
+/**
+ * Search a peer from its identifier
+ *
+ * @server:  The ivshmem server
+ * @peer_id: The identifier of the peer structure
+ *
+ * Returns:  The peer structure, or NULL if not found
+ */
+IvshmemServerPeer *
+ivshmem_server_search_peer(IvshmemServer *server, int64_t peer_id);
+
+/**
+ * Dump information of this ivshmem server and its peers on stdout
+ *
+ * @server: The ivshmem server
+ */
+void ivshmem_server_dump(const IvshmemServer *server);
+
+#endif /* IVSHMEM_SERVER_H */
diff --git a/contrib/ivshmem-server/main.c b/contrib/ivshmem-server/main.c
new file mode 100644
index 0000000..45776d8
--- /dev/null
+++ b/contrib/ivshmem-server/main.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright 6WIND S.A., 2014
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+
+#include "ivshmem-server.h"
+
+#define IVSHMEM_SERVER_DEFAULT_VERBOSE        0
+#define IVSHMEM_SERVER_DEFAULT_FOREGROUND     0
+#define IVSHMEM_SERVER_DEFAULT_PID_FILE       "/var/run/ivshmem-server.pid"
+#define IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH "/tmp/ivshmem_socket"
+#define IVSHMEM_SERVER_DEFAULT_SHM_PATH       "ivshmem"
+#define IVSHMEM_SERVER_DEFAULT_SHM_SIZE       (4*1024*1024)
+#define IVSHMEM_SERVER_DEFAULT_N_VECTORS      1
+
+/* used to quit on signal SIGTERM */
+static int ivshmem_server_quit;
+
+/* arguments given by the user */
+typedef struct IvshmemServerArgs {
+    bool verbose;
+    bool foreground;
+    const char *pid_file;
+    const char *unix_socket_path;
+    const char *shm_path;
+    bool use_shm_open;
+    uint64_t shm_size;
+    unsigned n_vectors;
+} IvshmemServerArgs;
+
+static void
+ivshmem_server_usage(const char *progname)
+{
+    printf("Usage: %s [OPTION]...\n"
+           "  -h: show this help\n"
+           "  -v: verbose mode\n"
+           "  -F: foreground mode (default is to daemonize)\n"
+           "  -p <pid-file>: path to the PID file (used in daemon mode only)\n"
+           "     default " IVSHMEM_SERVER_DEFAULT_PID_FILE "\n"
+           "  -S <unix-socket-path>: path to the unix socket to listen to\n"
+           "     default " IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH "\n"
+           "  -M <shm-name>: POSIX shared memory object to use\n"
+           "     default " IVSHMEM_SERVER_DEFAULT_SHM_PATH "\n"
+           "  -m <dir-name>: where to create shared memory\n"
+           "  -l <size>: size of shared memory in bytes\n"
+           "     suffixes K, M and G can be used, e.g. 1K means 1024\n"
+           "     default %u\n"
+           "  -n <nvectors>: number of vectors\n"
+           "     default %u\n",
+           progname, IVSHMEM_SERVER_DEFAULT_SHM_SIZE,
+           IVSHMEM_SERVER_DEFAULT_N_VECTORS);
+}
+
+static void
+ivshmem_server_help(const char *progname)
+{
+    fprintf(stderr, "Try '%s -h' for more information.\n", progname);
+}
+
+/* parse the program arguments, exit on error */
+static void
+ivshmem_server_parse_args(IvshmemServerArgs *args, int argc, char *argv[])
+{
+    int c;
+    unsigned long long v;
+    Error *err = NULL;
+
+    while ((c = getopt(argc, argv, "hvFp:S:m:M:l:n:")) != -1) {
+
+        switch (c) {
+        case 'h': /* help */
+            ivshmem_server_usage(argv[0]);
+            exit(0);
+            break;
+
+        case 'v': /* verbose */
+            args->verbose = 1;
+            break;
+
+        case 'F': /* foreground */
+            args->foreground = 1;
+            break;
+
+        case 'p': /* pid file */
+            args->pid_file = optarg;
+            break;
+
+        case 'S': /* unix socket path */
+            args->unix_socket_path = optarg;
+            break;
+
+        case 'M': /* shm name */
+        case 'm': /* dir name */
+            args->shm_path = optarg;
+            args->use_shm_open = c == 'M';
+            break;
+
+        case 'l': /* shm size */
+            parse_option_size("shm_size", optarg, &args->shm_size, &err);
+            if (err) {
+                error_report_err(err);
+                ivshmem_server_help(argv[0]);
+                exit(1);
+            }
+            break;
+
+        case 'n': /* number of vectors */
+            if (parse_uint_full(optarg, &v, 0) < 0) {
+                fprintf(stderr, "cannot parse n_vectors\n");
+                ivshmem_server_help(argv[0]);
+                exit(1);
+            }
+            args->n_vectors = v;
+            break;
+
+        default:
+            ivshmem_server_usage(argv[0]);
+            exit(1);
+            break;
+        }
+    }
+
+    if (args->n_vectors > IVSHMEM_SERVER_MAX_VECTORS) {
+        fprintf(stderr, "too many requested vectors (max is %d)\n",
+                IVSHMEM_SERVER_MAX_VECTORS);
+        ivshmem_server_help(argv[0]);
+        exit(1);
+    }
+
+    if (args->verbose == 1 && args->foreground == 0) {
+        fprintf(stderr, "cannot use verbose in daemon mode\n");
+        ivshmem_server_help(argv[0]);
+        exit(1);
+    }
+}
+
+/* wait for events on listening server unix socket and connected client
+ * sockets */
+static int
+ivshmem_server_poll_events(IvshmemServer *server)
+{
+    fd_set fds;
+    int ret = 0, maxfd;
+
+    while (!ivshmem_server_quit) {
+
+        FD_ZERO(&fds);
+        maxfd = 0;
+        ivshmem_server_get_fds(server, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            fprintf(stderr, "select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (ivshmem_server_handle_fds(server, &fds, maxfd) < 0) {
+            fprintf(stderr, "ivshmem_server_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return ret;
+}
+
+static void
+ivshmem_server_quit_cb(int signum)
+{
+    ivshmem_server_quit = 1;
+}
+
+int
+main(int argc, char *argv[])
+{
+    IvshmemServer server;
+    struct sigaction sa, sa_quit;
+    IvshmemServerArgs args = {
+        .verbose = IVSHMEM_SERVER_DEFAULT_VERBOSE,
+        .foreground = IVSHMEM_SERVER_DEFAULT_FOREGROUND,
+        .pid_file = IVSHMEM_SERVER_DEFAULT_PID_FILE,
+        .unix_socket_path = IVSHMEM_SERVER_DEFAULT_UNIX_SOCK_PATH,
+        .shm_path = IVSHMEM_SERVER_DEFAULT_SHM_PATH,
+        .use_shm_open = true,
+        .shm_size = IVSHMEM_SERVER_DEFAULT_SHM_SIZE,
+        .n_vectors = IVSHMEM_SERVER_DEFAULT_N_VECTORS,
+    };
+    int ret = 1;
+
+    /*
+     * Do not remove this notice without adding proper error handling!
+     * Start with handling ivshmem_server_send_one_msg() failure.
+     */
+    printf("*** Example code, do not use in production ***\n");
+
+    /* parse arguments, will exit on error */
+    ivshmem_server_parse_args(&args, argc, argv);
+
+    /* Ignore SIGPIPE, see this link for more info:
+     * http://www.mail-archive.com/libevent-users@monkey.org/msg01606.html */
+    sa.sa_handler = SIG_IGN;
+    sa.sa_flags = 0;
+    if (sigemptyset(&sa.sa_mask) == -1 ||
+        sigaction(SIGPIPE, &sa, 0) == -1) {
+        perror("failed to ignore SIGPIPE; sigaction");
+        goto err;
+    }
+
+    sa_quit.sa_handler = ivshmem_server_quit_cb;
+    sa_quit.sa_flags = 0;
+    if (sigemptyset(&sa_quit.sa_mask) == -1 ||
+        sigaction(SIGTERM, &sa_quit, 0) == -1) {
+        perror("failed to add SIGTERM handler; sigaction");
+        goto err;
+    }
+
+    /* init the ivshms structure */
+    if (ivshmem_server_init(&server, args.unix_socket_path,
+                            args.shm_path, args.use_shm_open,
+                            args.shm_size, args.n_vectors, args.verbose) < 0) {
+        fprintf(stderr, "cannot init server\n");
+        goto err;
+    }
+
+    /* start the ivshmem server (open shm & unix socket) */
+    if (ivshmem_server_start(&server) < 0) {
+        fprintf(stderr, "cannot bind\n");
+        goto err;
+    }
+
+    /* daemonize if asked to */
+    if (!args.foreground) {
+        FILE *fp;
+
+        if (qemu_daemon(1, 1) < 0) {
+            fprintf(stderr, "cannot daemonize: %s\n", strerror(errno));
+            goto err_close;
+        }
+
+        /* write pid file */
+        fp = fopen(args.pid_file, "w");
+        if (fp == NULL) {
+            fprintf(stderr, "cannot write pid file: %s\n", strerror(errno));
+            goto err_close;
+        }
+
+        fprintf(fp, "%d\n", (int) getpid());
+        fclose(fp);
+    }
+
+    ivshmem_server_poll_events(&server);
+    fprintf(stdout, "server disconnected\n");
+    ret = 0;
+
+err_close:
+    ivshmem_server_close(&server);
+err:
+    return ret;
+}
diff --git a/coroutine-gthread.c b/coroutine-gthread.c
deleted file mode 100644
index 6bd6d6b..0000000
--- a/coroutine-gthread.c
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * GThread coroutine initialization code
- *
- * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2011  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.0 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <glib.h>
-#include "qemu-common.h"
-#include "block/coroutine_int.h"
-
-typedef struct {
-    Coroutine base;
-    GThread *thread;
-    bool runnable;
-    bool free_on_thread_exit;
-    CoroutineAction action;
-} CoroutineGThread;
-
-static CompatGMutex coroutine_lock;
-static CompatGCond coroutine_cond;
-
-/* GLib 2.31 and beyond deprecated various parts of the thread API,
- * but the new interfaces are not available in older GLib versions
- * so we have to cope with both.
- */
-#if GLIB_CHECK_VERSION(2, 31, 0)
-/* Awkwardly, the GPrivate API doesn't provide a way to update the
- * GDestroyNotify handler for the coroutine key dynamically. So instead
- * we track whether or not the CoroutineGThread should be freed on
- * thread exit / coroutine key update using the free_on_thread_exit
- * field.
- */
-static void coroutine_destroy_notify(gpointer data)
-{
-    CoroutineGThread *co = data;
-    if (co && co->free_on_thread_exit) {
-        g_free(co);
-    }
-}
-
-static GPrivate coroutine_key = G_PRIVATE_INIT(coroutine_destroy_notify);
-
-static inline CoroutineGThread *get_coroutine_key(void)
-{
-    return g_private_get(&coroutine_key);
-}
-
-static inline void set_coroutine_key(CoroutineGThread *co,
-                                     bool free_on_thread_exit)
-{
-    /* Unlike g_static_private_set() this does not call the GDestroyNotify
-     * if the previous value of the key was NULL. Fortunately we only need
-     * the GDestroyNotify in the non-NULL key case.
-     */
-    co->free_on_thread_exit = free_on_thread_exit;
-    g_private_replace(&coroutine_key, co);
-}
-
-static inline GThread *create_thread(GThreadFunc func, gpointer data)
-{
-    return g_thread_new("coroutine", func, data);
-}
-
-#else
-
-/* Handle older GLib versions */
-
-static GStaticPrivate coroutine_key = G_STATIC_PRIVATE_INIT;
-
-static inline CoroutineGThread *get_coroutine_key(void)
-{
-    return g_static_private_get(&coroutine_key);
-}
-
-static inline void set_coroutine_key(CoroutineGThread *co,
-                                     bool free_on_thread_exit)
-{
-    g_static_private_set(&coroutine_key, co,
-                         free_on_thread_exit ? (GDestroyNotify)g_free : NULL);
-}
-
-static inline GThread *create_thread(GThreadFunc func, gpointer data)
-{
-    return g_thread_create_full(func, data, 0, TRUE, TRUE,
-                                G_THREAD_PRIORITY_NORMAL, NULL);
-}
-
-#endif
-
-
-static void __attribute__((constructor)) coroutine_init(void)
-{
-#if !GLIB_CHECK_VERSION(2, 31, 0)
-    if (!g_thread_supported()) {
-        g_thread_init(NULL);
-    }
-#endif
-}
-
-static void coroutine_wait_runnable_locked(CoroutineGThread *co)
-{
-    while (!co->runnable) {
-        g_cond_wait(&coroutine_cond, &coroutine_lock);
-    }
-}
-
-static void coroutine_wait_runnable(CoroutineGThread *co)
-{
-    g_mutex_lock(&coroutine_lock);
-    coroutine_wait_runnable_locked(co);
-    g_mutex_unlock(&coroutine_lock);
-}
-
-static gpointer coroutine_thread(gpointer opaque)
-{
-    CoroutineGThread *co = opaque;
-
-    set_coroutine_key(co, false);
-    coroutine_wait_runnable(co);
-    co->base.entry(co->base.entry_arg);
-    qemu_coroutine_switch(&co->base, co->base.caller, COROUTINE_TERMINATE);
-    return NULL;
-}
-
-Coroutine *qemu_coroutine_new(void)
-{
-    CoroutineGThread *co;
-
-    co = g_malloc0(sizeof(*co));
-    co->thread = create_thread(coroutine_thread, co);
-    if (!co->thread) {
-        g_free(co);
-        return NULL;
-    }
-    return &co->base;
-}
-
-void qemu_coroutine_delete(Coroutine *co_)
-{
-    CoroutineGThread *co = DO_UPCAST(CoroutineGThread, base, co_);
-
-    g_thread_join(co->thread);
-    g_free(co);
-}
-
-CoroutineAction qemu_coroutine_switch(Coroutine *from_,
-                                      Coroutine *to_,
-                                      CoroutineAction action)
-{
-    CoroutineGThread *from = DO_UPCAST(CoroutineGThread, base, from_);
-    CoroutineGThread *to = DO_UPCAST(CoroutineGThread, base, to_);
-
-    g_mutex_lock(&coroutine_lock);
-    from->runnable = false;
-    from->action = action;
-    to->runnable = true;
-    to->action = action;
-    g_cond_broadcast(&coroutine_cond);
-
-    if (action != COROUTINE_TERMINATE) {
-        coroutine_wait_runnable_locked(from);
-    }
-    g_mutex_unlock(&coroutine_lock);
-    return from->action;
-}
-
-Coroutine *qemu_coroutine_self(void)
-{
-    CoroutineGThread *co = get_coroutine_key();
-    if (!co) {
-        co = g_malloc0(sizeof(*co));
-        co->runnable = true;
-        set_coroutine_key(co, true);
-    }
-
-    return &co->base;
-}
-
-bool qemu_in_coroutine(void)
-{
-    CoroutineGThread *co = get_coroutine_key();
-
-    return co && co->base.caller;
-}
diff --git a/coroutine-sigaltstack.c b/coroutine-sigaltstack.c
deleted file mode 100644
index 63519ff..0000000
--- a/coroutine-sigaltstack.c
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * sigaltstack coroutine initialization code
- *
- * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
- * Copyright (C) 2012  Alex Barcelo <abarcelo@ac.upc.edu>
-** This file is partly based on pth_mctx.c, from the GNU Portable Threads
-**  Copyright (c) 1999-2006 Ralf S. Engelschall <rse@engelschall.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
-#ifdef _FORTIFY_SOURCE
-#undef _FORTIFY_SOURCE
-#endif
-#include <stdlib.h>
-#include <setjmp.h>
-#include <stdint.h>
-#include <pthread.h>
-#include <signal.h>
-#include "qemu-common.h"
-#include "block/coroutine_int.h"
-
-typedef struct {
-    Coroutine base;
-    void *stack;
-    sigjmp_buf env;
-} CoroutineUContext;
-
-/**
- * Per-thread coroutine bookkeeping
- */
-typedef struct {
-    /** Currently executing coroutine */
-    Coroutine *current;
-
-    /** The default coroutine */
-    CoroutineUContext leader;
-
-    /** Information for the signal handler (trampoline) */
-    sigjmp_buf tr_reenter;
-    volatile sig_atomic_t tr_called;
-    void *tr_handler;
-} CoroutineThreadState;
-
-static pthread_key_t thread_state_key;
-
-static CoroutineThreadState *coroutine_get_thread_state(void)
-{
-    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
-
-    if (!s) {
-        s = g_malloc0(sizeof(*s));
-        s->current = &s->leader.base;
-        pthread_setspecific(thread_state_key, s);
-    }
-    return s;
-}
-
-static void qemu_coroutine_thread_cleanup(void *opaque)
-{
-    CoroutineThreadState *s = opaque;
-
-    g_free(s);
-}
-
-static void __attribute__((constructor)) coroutine_init(void)
-{
-    int ret;
-
-    ret = pthread_key_create(&thread_state_key, qemu_coroutine_thread_cleanup);
-    if (ret != 0) {
-        fprintf(stderr, "unable to create leader key: %s\n", strerror(errno));
-        abort();
-    }
-}
-
-/* "boot" function
- * This is what starts the coroutine, is called from the trampoline
- * (from the signal handler when it is not signal handling, read ahead
- * for more information).
- */
-static void coroutine_bootstrap(CoroutineUContext *self, Coroutine *co)
-{
-    /* Initialize longjmp environment and switch back the caller */
-    if (!sigsetjmp(self->env, 0)) {
-        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
-    }
-
-    while (true) {
-        co->entry(co->entry_arg);
-        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
-    }
-}
-
-/*
- * This is used as the signal handler. This is called with the brand new stack
- * (thanks to sigaltstack). We have to return, given that this is a signal
- * handler and the sigmask and some other things are changed.
- */
-static void coroutine_trampoline(int signal)
-{
-    CoroutineUContext *self;
-    Coroutine *co;
-    CoroutineThreadState *coTS;
-
-    /* Get the thread specific information */
-    coTS = coroutine_get_thread_state();
-    self = coTS->tr_handler;
-    coTS->tr_called = 1;
-    co = &self->base;
-
-    /*
-     * Here we have to do a bit of a ping pong between the caller, given that
-     * this is a signal handler and we have to do a return "soon". Then the
-     * caller can reestablish everything and do a siglongjmp here again.
-     */
-    if (!sigsetjmp(coTS->tr_reenter, 0)) {
-        return;
-    }
-
-    /*
-     * Ok, the caller has siglongjmp'ed back to us, so now prepare
-     * us for the real machine state switching. We have to jump
-     * into another function here to get a new stack context for
-     * the auto variables (which have to be auto-variables
-     * because the start of the thread happens later). Else with
-     * PIC (i.e. Position Independent Code which is used when PTH
-     * is built as a shared library) most platforms would
-     * horrible core dump as experience showed.
-     */
-    coroutine_bootstrap(self, co);
-}
-
-Coroutine *qemu_coroutine_new(void)
-{
-    const size_t stack_size = 1 << 20;
-    CoroutineUContext *co;
-    CoroutineThreadState *coTS;
-    struct sigaction sa;
-    struct sigaction osa;
-    stack_t ss;
-    stack_t oss;
-    sigset_t sigs;
-    sigset_t osigs;
-    sigjmp_buf old_env;
-
-    /* The way to manipulate stack is with the sigaltstack function. We
-     * prepare a stack, with it delivering a signal to ourselves and then
-     * put sigsetjmp/siglongjmp where needed.
-     * This has been done keeping coroutine-ucontext as a model and with the
-     * pth ideas (GNU Portable Threads). See coroutine-ucontext for the basics
-     * of the coroutines and see pth_mctx.c (from the pth project) for the
-     * sigaltstack way of manipulating stacks.
-     */
-
-    co = g_malloc0(sizeof(*co));
-    co->stack = g_malloc(stack_size);
-    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
-
-    coTS = coroutine_get_thread_state();
-    coTS->tr_handler = co;
-
-    /*
-     * Preserve the SIGUSR2 signal state, block SIGUSR2,
-     * and establish our signal handler. The signal will
-     * later transfer control onto the signal stack.
-     */
-    sigemptyset(&sigs);
-    sigaddset(&sigs, SIGUSR2);
-    pthread_sigmask(SIG_BLOCK, &sigs, &osigs);
-    sa.sa_handler = coroutine_trampoline;
-    sigfillset(&sa.sa_mask);
-    sa.sa_flags = SA_ONSTACK;
-    if (sigaction(SIGUSR2, &sa, &osa) != 0) {
-        abort();
-    }
-
-    /*
-     * Set the new stack.
-     */
-    ss.ss_sp = co->stack;
-    ss.ss_size = stack_size;
-    ss.ss_flags = 0;
-    if (sigaltstack(&ss, &oss) < 0) {
-        abort();
-    }
-
-    /*
-     * Now transfer control onto the signal stack and set it up.
-     * It will return immediately via "return" after the sigsetjmp()
-     * was performed. Be careful here with race conditions.  The
-     * signal can be delivered the first time sigsuspend() is
-     * called.
-     */
-    coTS->tr_called = 0;
-    pthread_kill(pthread_self(), SIGUSR2);
-    sigfillset(&sigs);
-    sigdelset(&sigs, SIGUSR2);
-    while (!coTS->tr_called) {
-        sigsuspend(&sigs);
-    }
-
-    /*
-     * Inform the system that we are back off the signal stack by
-     * removing the alternative signal stack. Be careful here: It
-     * first has to be disabled, before it can be removed.
-     */
-    sigaltstack(NULL, &ss);
-    ss.ss_flags = SS_DISABLE;
-    if (sigaltstack(&ss, NULL) < 0) {
-        abort();
-    }
-    sigaltstack(NULL, &ss);
-    if (!(oss.ss_flags & SS_DISABLE)) {
-        sigaltstack(&oss, NULL);
-    }
-
-    /*
-     * Restore the old SIGUSR2 signal handler and mask
-     */
-    sigaction(SIGUSR2, &osa, NULL);
-    pthread_sigmask(SIG_SETMASK, &osigs, NULL);
-
-    /*
-     * Now enter the trampoline again, but this time not as a signal
-     * handler. Instead we jump into it directly. The functionally
-     * redundant ping-pong pointer arithmetic is necessary to avoid
-     * type-conversion warnings related to the `volatile' qualifier and
-     * the fact that `jmp_buf' usually is an array type.
-     */
-    if (!sigsetjmp(old_env, 0)) {
-        siglongjmp(coTS->tr_reenter, 1);
-    }
-
-    /*
-     * Ok, we returned again, so now we're finished
-     */
-
-    return &co->base;
-}
-
-void qemu_coroutine_delete(Coroutine *co_)
-{
-    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
-
-    g_free(co->stack);
-    g_free(co);
-}
-
-CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
-                                      CoroutineAction action)
-{
-    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
-    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
-    CoroutineThreadState *s = coroutine_get_thread_state();
-    int ret;
-
-    s->current = to_;
-
-    ret = sigsetjmp(from->env, 0);
-    if (ret == 0) {
-        siglongjmp(to->env, action);
-    }
-    return ret;
-}
-
-Coroutine *qemu_coroutine_self(void)
-{
-    CoroutineThreadState *s = coroutine_get_thread_state();
-
-    return s->current;
-}
-
-bool qemu_in_coroutine(void)
-{
-    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
-
-    return s && s->current->caller;
-}
-
diff --git a/coroutine-ucontext.c b/coroutine-ucontext.c
deleted file mode 100644
index 4bf2cde..0000000
--- a/coroutine-ucontext.c
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * ucontext coroutine initialization code
- *
- * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.0 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
-#ifdef _FORTIFY_SOURCE
-#undef _FORTIFY_SOURCE
-#endif
-#include <stdlib.h>
-#include <setjmp.h>
-#include <stdint.h>
-#include <pthread.h>
-#include <ucontext.h>
-#include "qemu-common.h"
-#include "block/coroutine_int.h"
-
-#ifdef CONFIG_VALGRIND_H
-#include <valgrind/valgrind.h>
-#endif
-
-typedef struct {
-    Coroutine base;
-    void *stack;
-    sigjmp_buf env;
-
-#ifdef CONFIG_VALGRIND_H
-    unsigned int valgrind_stack_id;
-#endif
-
-} CoroutineUContext;
-
-/**
- * Per-thread coroutine bookkeeping
- */
-typedef struct {
-    /** Currently executing coroutine */
-    Coroutine *current;
-
-    /** The default coroutine */
-    CoroutineUContext leader;
-} CoroutineThreadState;
-
-static pthread_key_t thread_state_key;
-
-/*
- * va_args to makecontext() must be type 'int', so passing
- * the pointer we need may require several int args. This
- * union is a quick hack to let us do that
- */
-union cc_arg {
-    void *p;
-    int i[2];
-};
-
-static CoroutineThreadState *coroutine_get_thread_state(void)
-{
-    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
-
-    if (!s) {
-        s = g_malloc0(sizeof(*s));
-        s->current = &s->leader.base;
-        pthread_setspecific(thread_state_key, s);
-    }
-    return s;
-}
-
-static void qemu_coroutine_thread_cleanup(void *opaque)
-{
-    CoroutineThreadState *s = opaque;
-
-    g_free(s);
-}
-
-static void __attribute__((constructor)) coroutine_init(void)
-{
-    int ret;
-
-    ret = pthread_key_create(&thread_state_key, qemu_coroutine_thread_cleanup);
-    if (ret != 0) {
-        fprintf(stderr, "unable to create leader key: %s\n", strerror(errno));
-        abort();
-    }
-}
-
-static void coroutine_trampoline(int i0, int i1)
-{
-    union cc_arg arg;
-    CoroutineUContext *self;
-    Coroutine *co;
-
-    arg.i[0] = i0;
-    arg.i[1] = i1;
-    self = arg.p;
-    co = &self->base;
-
-    /* Initialize longjmp environment and switch back the caller */
-    if (!sigsetjmp(self->env, 0)) {
-        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
-    }
-
-    while (true) {
-        co->entry(co->entry_arg);
-        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
-    }
-}
-
-Coroutine *qemu_coroutine_new(void)
-{
-    const size_t stack_size = 1 << 20;
-    CoroutineUContext *co;
-    ucontext_t old_uc, uc;
-    sigjmp_buf old_env;
-    union cc_arg arg = {0};
-
-    /* The ucontext functions preserve signal masks which incurs a
-     * system call overhead.  sigsetjmp(buf, 0)/siglongjmp() does not
-     * preserve signal masks but only works on the current stack.
-     * Since we need a way to create and switch to a new stack, use
-     * the ucontext functions for that but sigsetjmp()/siglongjmp() for
-     * everything else.
-     */
-
-    if (getcontext(&uc) == -1) {
-        abort();
-    }
-
-    co = g_malloc0(sizeof(*co));
-    co->stack = g_malloc(stack_size);
-    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
-
-    uc.uc_link = &old_uc;
-    uc.uc_stack.ss_sp = co->stack;
-    uc.uc_stack.ss_size = stack_size;
-    uc.uc_stack.ss_flags = 0;
-
-#ifdef CONFIG_VALGRIND_H
-    co->valgrind_stack_id =
-        VALGRIND_STACK_REGISTER(co->stack, co->stack + stack_size);
-#endif
-
-    arg.p = co;
-
-    makecontext(&uc, (void (*)(void))coroutine_trampoline,
-                2, arg.i[0], arg.i[1]);
-
-    /* swapcontext() in, siglongjmp() back out */
-    if (!sigsetjmp(old_env, 0)) {
-        swapcontext(&old_uc, &uc);
-    }
-    return &co->base;
-}
-
-#ifdef CONFIG_VALGRIND_H
-#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
-/* Work around an unused variable in the valgrind.h macro... */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
-#endif
-static inline void valgrind_stack_deregister(CoroutineUContext *co)
-{
-    VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
-}
-#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
-#pragma GCC diagnostic pop
-#endif
-#endif
-
-void qemu_coroutine_delete(Coroutine *co_)
-{
-    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
-
-#ifdef CONFIG_VALGRIND_H
-    valgrind_stack_deregister(co);
-#endif
-
-    g_free(co->stack);
-    g_free(co);
-}
-
-CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
-                                      CoroutineAction action)
-{
-    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
-    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
-    CoroutineThreadState *s = coroutine_get_thread_state();
-    int ret;
-
-    s->current = to_;
-
-    ret = sigsetjmp(from->env, 0);
-    if (ret == 0) {
-        siglongjmp(to->env, action);
-    }
-    return ret;
-}
-
-Coroutine *qemu_coroutine_self(void)
-{
-    CoroutineThreadState *s = coroutine_get_thread_state();
-
-    return s->current;
-}
-
-bool qemu_in_coroutine(void)
-{
-    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
-
-    return s && s->current->caller;
-}
diff --git a/coroutine-win32.c b/coroutine-win32.c
deleted file mode 100644
index 17ace37..0000000
--- a/coroutine-win32.c
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Win32 coroutine initialization code
- *
- * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-common.h"
-#include "block/coroutine_int.h"
-
-typedef struct
-{
-    Coroutine base;
-
-    LPVOID fiber;
-    CoroutineAction action;
-} CoroutineWin32;
-
-static __thread CoroutineWin32 leader;
-static __thread Coroutine *current;
-
-/* This function is marked noinline to prevent GCC from inlining it
- * into coroutine_trampoline(). If we allow it to do that then it
- * hoists the code to get the address of the TLS variable "current"
- * out of the while() loop. This is an invalid transformation because
- * the SwitchToFiber() call may be called when running thread A but
- * return in thread B, and so we might be in a different thread
- * context each time round the loop.
- */
-CoroutineAction __attribute__((noinline))
-qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
-                      CoroutineAction action)
-{
-    CoroutineWin32 *from = DO_UPCAST(CoroutineWin32, base, from_);
-    CoroutineWin32 *to = DO_UPCAST(CoroutineWin32, base, to_);
-
-    current = to_;
-
-    to->action = action;
-    SwitchToFiber(to->fiber);
-    return from->action;
-}
-
-static void CALLBACK coroutine_trampoline(void *co_)
-{
-    Coroutine *co = co_;
-
-    while (true) {
-        co->entry(co->entry_arg);
-        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
-    }
-}
-
-Coroutine *qemu_coroutine_new(void)
-{
-    const size_t stack_size = 1 << 20;
-    CoroutineWin32 *co;
-
-    co = g_malloc0(sizeof(*co));
-    co->fiber = CreateFiber(stack_size, coroutine_trampoline, &co->base);
-    return &co->base;
-}
-
-void qemu_coroutine_delete(Coroutine *co_)
-{
-    CoroutineWin32 *co = DO_UPCAST(CoroutineWin32, base, co_);
-
-    DeleteFiber(co->fiber);
-    g_free(co);
-}
-
-Coroutine *qemu_coroutine_self(void)
-{
-    if (!current) {
-        current = &leader.base;
-        leader.fiber = ConvertThreadToFiber(NULL);
-    }
-    return current;
-}
-
-bool qemu_in_coroutine(void)
-{
-    return current && current->caller;
-}
diff --git a/cpu-exec-common.c b/cpu-exec-common.c
new file mode 100644
index 0000000..0cb4ae6
--- /dev/null
+++ b/cpu-exec-common.c
@@ -0,0 +1,79 @@
+/*
+ *  emulator main execution loop
+ *
+ *  Copyright (c) 2003-2005 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "sysemu/cpus.h"
+#include "exec/exec-all.h"
+#include "exec/memory-internal.h"
+
+bool exit_request;
+CPUState *tcg_current_cpu;
+
+/* exit the current TB, but without causing any exception to be raised */
+void cpu_loop_exit_noexc(CPUState *cpu)
+{
+    /* XXX: restore cpu registers saved in host registers */
+
+    cpu->exception_index = -1;
+    siglongjmp(cpu->jmp_env, 1);
+}
+
+#if defined(CONFIG_SOFTMMU)
+void cpu_reloading_memory_map(void)
+{
+    if (qemu_in_vcpu_thread()) {
+        /* The guest can in theory prolong the RCU critical section as long
+         * as it feels like. The major problem with this is that because it
+         * can do multiple reconfigurations of the memory map within the
+         * critical section, we could potentially accumulate an unbounded
+         * collection of memory data structures awaiting reclamation.
+         *
+         * Because the only thing we're currently protecting with RCU is the
+         * memory data structures, it's sufficient to break the critical section
+         * in this callback, which we know will get called every time the
+         * memory map is rearranged.
+         *
+         * (If we add anything else in the system that uses RCU to protect
+         * its data structures, we will need to implement some other mechanism
+         * to force TCG CPUs to exit the critical section, at which point this
+         * part of this callback might become unnecessary.)
+         *
+         * This pair matches cpu_exec's rcu_read_lock()/rcu_read_unlock(), which
+         * only protects cpu->as->dispatch. Since we know our caller is about
+         * to reload it, it's safe to split the critical section.
+         */
+        rcu_read_unlock();
+        rcu_read_lock();
+    }
+}
+#endif
+
+void cpu_loop_exit(CPUState *cpu)
+{
+    siglongjmp(cpu->jmp_env, 1);
+}
+
+void cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc)
+{
+    if (pc) {
+        cpu_restore_state(cpu, pc);
+    }
+    siglongjmp(cpu->jmp_env, 1);
+}
diff --git a/cpu-exec.c b/cpu-exec.c
index 3f3529e..02e4c97 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -16,15 +16,26 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
 #include "qemu/atomic.h"
+#ifdef CONFIG_HAX
+#include "sysemu/hax.h"
+#endif /* CONFIG_HAX */
 #include "sysemu/qtest.h"
 #include "qemu/timer.h"
-#include "sysemu/hax.h"
+#include "exec/address-spaces.h"
+#include "qemu/rcu.h"
+#include "exec/tb-hash.h"
+#include "exec/log.h"
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+#include "hw/i386/apic.h"
+#endif
+#include "sysemu/replay.h"
 
 /* -icount align implementation. */
 
@@ -62,8 +73,7 @@
         sleep_delay.tv_sec = sc->diff_clk / 1000000000LL;
         sleep_delay.tv_nsec = sc->diff_clk % 1000000000LL;
         if (nanosleep(&sleep_delay, &rem_delay) < 0) {
-            sc->diff_clk -= (sleep_delay.tv_sec - rem_delay.tv_sec) * 1000000000LL;
-            sc->diff_clk -= sleep_delay.tv_nsec - rem_delay.tv_nsec;
+            sc->diff_clk = rem_delay.tv_sec * 1000000000LL + rem_delay.tv_nsec;
         } else {
             sc->diff_clk = 0;
         }
@@ -102,10 +112,8 @@
     if (!icount_align_option) {
         return;
     }
-    sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-    sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                   sc->realtime_clock +
-                   cpu_get_clock_offset();
+    sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
+    sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - sc->realtime_clock;
     sc->last_cpu_icount = cpu->icount_extra + cpu->icount_decr.u16.low;
     if (sc->diff_clk < max_delay) {
         max_delay = sc->diff_clk;
@@ -128,30 +136,18 @@
 }
 #endif /* CONFIG USER ONLY */
 
-void cpu_loop_exit(CPUState *cpu)
-{
-    cpu->current_tb = NULL;
-    siglongjmp(cpu->jmp_env, 1);
-}
-
-/* exit the current TB from a signal handler. The host registers are
-   restored in a state compatible with the CPU emulator
- */
-#if defined(CONFIG_SOFTMMU)
-void cpu_resume_from_signal(CPUState *cpu, void *puc)
-{
-    /* XXX: restore cpu registers saved in host registers */
-
-    cpu->exception_index = -1;
-    siglongjmp(cpu->jmp_env, 1);
-}
-#endif
-
 /* Execute a TB, and fix up the CPU state afterwards if necessary */
-static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, uint8_t *tb_ptr)
+static inline tcg_target_ulong cpu_tb_exec(CPUState *cpu, TranslationBlock *itb)
 {
     CPUArchState *env = cpu->env_ptr;
-    uintptr_t next_tb;
+    uintptr_t ret;
+    TranslationBlock *last_tb;
+    int tb_exit;
+    uint8_t *tb_ptr = itb->tc_ptr;
+
+    qemu_log_mask_and_addr(CPU_LOG_EXEC, itb->pc,
+                           "Trace %p [" TARGET_FMT_lx "] %s\n",
+                           itb->tc_ptr, itb->pc, lookup_symbol(itb->pc));
 
 #if defined(DEBUG_DISAS)
     if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
@@ -169,136 +165,230 @@
     }
 #endif /* DEBUG_DISAS */
 
-    next_tb = tcg_qemu_tb_exec(env, tb_ptr);
-    trace_exec_tb_exit((void *) (next_tb & ~TB_EXIT_MASK),
-                       next_tb & TB_EXIT_MASK);
+    cpu->can_do_io = !use_icount;
+    ret = tcg_qemu_tb_exec(env, tb_ptr);
+    cpu->can_do_io = 1;
+    last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK);
+    tb_exit = ret & TB_EXIT_MASK;
+    trace_exec_tb_exit(last_tb, tb_exit);
 
-    if ((next_tb & TB_EXIT_MASK) > TB_EXIT_IDX1) {
+    if (tb_exit > TB_EXIT_IDX1) {
         /* We didn't start executing this TB (eg because the instruction
          * counter hit zero); we must restore the guest PC to the address
          * of the start of the TB.
          */
         CPUClass *cc = CPU_GET_CLASS(cpu);
-        TranslationBlock *tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
+        qemu_log_mask_and_addr(CPU_LOG_EXEC, last_tb->pc,
+                               "Stopped execution of TB chain before %p ["
+                               TARGET_FMT_lx "] %s\n",
+                               last_tb->tc_ptr, last_tb->pc,
+                               lookup_symbol(last_tb->pc));
         if (cc->synchronize_from_tb) {
-            cc->synchronize_from_tb(cpu, tb);
+            cc->synchronize_from_tb(cpu, last_tb);
         } else {
             assert(cc->set_pc);
-            cc->set_pc(cpu, tb->pc);
+            cc->set_pc(cpu, last_tb->pc);
         }
     }
-    if ((next_tb & TB_EXIT_MASK) == TB_EXIT_REQUESTED) {
+    if (tb_exit == TB_EXIT_REQUESTED) {
         /* We were asked to stop executing TBs (probably a pending
          * interrupt. We've now stopped, so clear the flag.
          */
         cpu->tcg_exit_req = 0;
     }
-    return next_tb;
+    return ret;
 }
 
+#ifndef CONFIG_USER_ONLY
 /* Execute the code without caching the generated code. An interpreter
    could be used if available. */
-static void cpu_exec_nocache(CPUArchState *env, int max_cycles,
-                             TranslationBlock *orig_tb)
+static void cpu_exec_nocache(CPUState *cpu, int max_cycles,
+                             TranslationBlock *orig_tb, bool ignore_icount)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
     TranslationBlock *tb;
+    bool old_tb_flushed;
 
     /* Should never happen.
        We only end up here when an existing TB is too long.  */
     if (max_cycles > CF_COUNT_MASK)
         max_cycles = CF_COUNT_MASK;
 
+    old_tb_flushed = cpu->tb_flushed;
+    cpu->tb_flushed = false;
     tb = tb_gen_code(cpu, orig_tb->pc, orig_tb->cs_base, orig_tb->flags,
-                     max_cycles);
-    cpu->current_tb = tb;
+                     max_cycles | CF_NOCACHE
+                         | (ignore_icount ? CF_IGNORE_ICOUNT : 0));
+    tb->orig_tb = cpu->tb_flushed ? NULL : orig_tb;
+    cpu->tb_flushed |= old_tb_flushed;
     /* execute the generated code */
     trace_exec_tb_nocache(tb, tb->pc);
-    cpu_tb_exec(cpu, tb->tc_ptr);
-    cpu->current_tb = NULL;
+    cpu_tb_exec(cpu, tb);
     tb_phys_invalidate(tb, -1);
     tb_free(tb);
 }
+#endif
 
-static TranslationBlock *tb_find_slow(CPUArchState *env,
-                                      target_ulong pc,
-                                      target_ulong cs_base,
-                                      uint64_t flags)
+struct tb_desc {
+    target_ulong pc;
+    target_ulong cs_base;
+    CPUArchState *env;
+    tb_page_addr_t phys_page1;
+    uint32_t flags;
+};
+
+static bool tb_cmp(const void *p, const void *d)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
-    TranslationBlock *tb, **ptb1;
-    unsigned int h;
-    tb_page_addr_t phys_pc, phys_page1;
-    target_ulong virt_page2;
+    const TranslationBlock *tb = p;
+    const struct tb_desc *desc = d;
 
-    tcg_ctx.tb_ctx.tb_invalidated_flag = 0;
+    if (tb->pc == desc->pc &&
+        tb->page_addr[0] == desc->phys_page1 &&
+        tb->cs_base == desc->cs_base &&
+        tb->flags == desc->flags) {
+        /* check next page if needed */
+        if (tb->page_addr[1] == -1) {
+            return true;
+        } else {
+            tb_page_addr_t phys_page2;
+            target_ulong virt_page2;
 
-    /* find translated block using physical mappings */
-    phys_pc = get_page_addr_code(env, pc);
-    phys_page1 = phys_pc & TARGET_PAGE_MASK;
-    h = tb_phys_hash_func(phys_pc);
-    ptb1 = &tcg_ctx.tb_ctx.tb_phys_hash[h];
-    for(;;) {
-        tb = *ptb1;
-        if (!tb)
-            goto not_found;
-        if (tb->pc == pc &&
-            tb->page_addr[0] == phys_page1 &&
-            tb->cs_base == cs_base &&
-            tb->flags == flags) {
-            /* check next page if needed */
-            if (tb->page_addr[1] != -1) {
-                tb_page_addr_t phys_page2;
-
-                virt_page2 = (pc & TARGET_PAGE_MASK) +
-                    TARGET_PAGE_SIZE;
-                phys_page2 = get_page_addr_code(env, virt_page2);
-                if (tb->page_addr[1] == phys_page2)
-                    goto found;
-            } else {
-                goto found;
+            virt_page2 = (desc->pc & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+            phys_page2 = get_page_addr_code(desc->env, virt_page2);
+            if (tb->page_addr[1] == phys_page2) {
+                return true;
             }
         }
-        ptb1 = &tb->phys_hash_next;
     }
- not_found:
-   /* if no translated code available, then translate it now */
+    return false;
+}
+
+static TranslationBlock *tb_find_physical(CPUState *cpu,
+                                          target_ulong pc,
+                                          target_ulong cs_base,
+                                          uint32_t flags)
+{
+    tb_page_addr_t phys_pc;
+    struct tb_desc desc;
+    uint32_t h;
+
+    desc.env = (CPUArchState *)cpu->env_ptr;
+    desc.cs_base = cs_base;
+    desc.flags = flags;
+    desc.pc = pc;
+    phys_pc = get_page_addr_code(desc.env, pc);
+    desc.phys_page1 = phys_pc & TARGET_PAGE_MASK;
+    h = tb_hash_func(phys_pc, pc, flags);
+    return qht_lookup(&tcg_ctx.tb_ctx.htable, tb_cmp, &desc, h);
+}
+
+static TranslationBlock *tb_find_slow(CPUState *cpu,
+                                      target_ulong pc,
+                                      target_ulong cs_base,
+                                      uint32_t flags)
+{
+    TranslationBlock *tb;
+
+    tb = tb_find_physical(cpu, pc, cs_base, flags);
+    if (tb) {
+        goto found;
+    }
+
+#ifdef CONFIG_USER_ONLY
+    /* mmap_lock is needed by tb_gen_code, and mmap_lock must be
+     * taken outside tb_lock.  Since we're momentarily dropping
+     * tb_lock, there's a chance that our desired tb has been
+     * translated.
+     */
+    tb_unlock();
+    mmap_lock();
+    tb_lock();
+    tb = tb_find_physical(cpu, pc, cs_base, flags);
+    if (tb) {
+        mmap_unlock();
+        goto found;
+    }
+#endif
+
+    /* if no translated code available, then translate it now */
     tb = tb_gen_code(cpu, pc, cs_base, flags, 0);
 
- found:
-    /* Move the last found TB to the head of the list */
-    if (likely(*ptb1)) {
-        *ptb1 = tb->phys_hash_next;
-        tb->phys_hash_next = tcg_ctx.tb_ctx.tb_phys_hash[h];
-        tcg_ctx.tb_ctx.tb_phys_hash[h] = tb;
-    }
+#ifdef CONFIG_USER_ONLY
+    mmap_unlock();
+#endif
+
+found:
     /* we add the TB in the virtual pc hash table */
     cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)] = tb;
     return tb;
 }
 
-static inline TranslationBlock *tb_find_fast(CPUArchState *env)
+static inline TranslationBlock *tb_find_fast(CPUState *cpu,
+                                             TranslationBlock **last_tb,
+                                             int tb_exit)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
+    CPUArchState *env = (CPUArchState *)cpu->env_ptr;
     TranslationBlock *tb;
     target_ulong cs_base, pc;
-    int flags;
+    uint32_t flags;
 
     /* we record a subset of the CPU state. It will
        always be the same before a given translated block
        is executed. */
     cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
+    tb_lock();
     tb = cpu->tb_jmp_cache[tb_jmp_cache_hash_func(pc)];
     if (unlikely(!tb || tb->pc != pc || tb->cs_base != cs_base ||
                  tb->flags != flags)) {
-        tb = tb_find_slow(env, pc, cs_base, flags);
+        tb = tb_find_slow(cpu, pc, cs_base, flags);
     }
+    if (cpu->tb_flushed) {
+        /* Ensure that no TB jump will be modified as the
+         * translation buffer has been flushed.
+         */
+        *last_tb = NULL;
+        cpu->tb_flushed = false;
+    }
+#ifndef CONFIG_USER_ONLY
+    /* We don't take care of direct jumps when address mapping changes in
+     * system emulation. So it's not safe to make a direct jump to a TB
+     * spanning two pages because the mapping for the second page can change.
+     */
+    if (tb->page_addr[1] != -1) {
+        *last_tb = NULL;
+    }
+#endif
+    /* See if we can patch the calling TB. */
+    if (*last_tb && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
+        tb_add_jump(*last_tb, tb_exit, tb);
+    }
+    tb_unlock();
     return tb;
 }
 
-static void cpu_handle_debug_exception(CPUArchState *env)
+static inline bool cpu_handle_halt(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
+    if (cpu->halted) {
+#if defined(TARGET_I386) && !defined(CONFIG_USER_ONLY)
+        if ((cpu->interrupt_request & CPU_INTERRUPT_POLL)
+            && replay_interrupt()) {
+            X86CPU *x86_cpu = X86_CPU(cpu);
+            apic_poll_irq(x86_cpu->apic_state);
+            cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
+        }
+#endif
+        if (!cpu_has_work(cpu)) {
+            current_cpu = NULL;
+            return true;
+        }
+
+        cpu->halted = 0;
+    }
+
+    return false;
+}
+
+static inline void cpu_handle_debug_exception(CPUState *cpu)
+{
     CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUWatchpoint *wp;
 
@@ -311,67 +401,223 @@
     cc->debug_excp_handler(cpu);
 }
 
-/* main execution loop */
-
-volatile sig_atomic_t exit_request;
-
-/*
- * QEMU emulate can happens because of MMIO or emulation mode, i.e. non-PG mode,
- * when it's because of MMIO, the MMIO, the interrupt should not be emulated,
- * because MMIO is emulated for only one instruction now and then back to
- * HAX kernel
- */
-static int need_handle_intr_request(CPUState *cpu)
+static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
 {
-#ifdef CONFIG_HAX
-    if (!hax_enabled() || hax_vcpu_emulation_mode(cpu))
-        return cpu->interrupt_request;
-    return 0;
+    if (cpu->exception_index >= 0) {
+        if (cpu->exception_index >= EXCP_INTERRUPT) {
+            /* exit request from the cpu execution loop */
+            *ret = cpu->exception_index;
+            if (*ret == EXCP_DEBUG) {
+                cpu_handle_debug_exception(cpu);
+            }
+            cpu->exception_index = -1;
+            return true;
+        } else {
+#if defined(CONFIG_USER_ONLY)
+            /* if user mode only, we simulate a fake exception
+               which will be handled outside the cpu execution
+               loop */
+#if defined(TARGET_I386)
+            CPUClass *cc = CPU_GET_CLASS(cpu);
+            cc->do_interrupt(cpu);
+#endif
+            *ret = cpu->exception_index;
+            cpu->exception_index = -1;
+            return true;
 #else
-    return cpu->interrupt_request;
+            if (replay_exception()) {
+                CPUClass *cc = CPU_GET_CLASS(cpu);
+                cc->do_interrupt(cpu);
+                cpu->exception_index = -1;
+            } else if (!replay_has_interrupt()) {
+                /* give a chance to iothread in replay mode */
+                *ret = EXCP_INTERRUPT;
+                return true;
+            }
 #endif
-}
-
-int cpu_exec(CPUArchState *env)
-{
-    CPUState *cpu = ENV_GET_CPU(env);
-    CPUClass *cc = CPU_GET_CLASS(cpu);
-#ifdef TARGET_I386
-    X86CPU *x86_cpu = X86_CPU(cpu);
-#endif
-    int ret, interrupt_request;
-    TranslationBlock *tb;
-    uint8_t *tc_ptr;
-    uintptr_t next_tb;
-    SyncClocks sc;
-
-    /* This must be volatile so it is not trashed by longjmp() */
-    volatile bool have_tb_lock = false;
-
-    if (cpu->halted) {
-        if (!cpu_has_work(cpu)) {
-            return EXCP_HALTED;
         }
-
-        cpu->halted = 0;
+#ifndef CONFIG_USER_ONLY
+    } else if (replay_has_exception()
+               && cpu->icount_decr.u16.low + cpu->icount_extra == 0) {
+        /* try to cause an exception pending in the log */
+        TranslationBlock *last_tb = NULL; /* Avoid chaining TBs */
+        cpu_exec_nocache(cpu, 1, tb_find_fast(cpu, &last_tb, 0), true);
+        *ret = -1;
+        return true;
+#endif
     }
 
+    return false;
+}
+
+static inline int cpu_get_interrupt_request(CPUState *cpu)
+{
+#ifdef CONFIG_HAX
+    /* When HAX is enabled, there are two cases where TCG emulation might happen:
+     * MMIO instructions, or non-paged mode. When this is due to an MMIO, the interrupt
+     * should not be emulated because only one instruction will be translated and run
+     * through TCG before returning to the HAX kernel.
+     */
+    if (hax_enabled() && !hax_vcpu_emulation_mode(cpu)) {
+        /* Mask interrupt during MMIO emulation. */
+        return 0;
+    }
+#endif
+    return cpu->interrupt_request;
+}
+
+static inline void cpu_handle_interrupt(CPUState *cpu,
+                                        TranslationBlock **last_tb)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    int interrupt_request = cpu_get_interrupt_request(cpu);
+
+    if (unlikely(interrupt_request)) {
+        if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
+            /* Mask out external interrupts for this step. */
+            interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
+        }
+        if (interrupt_request & CPU_INTERRUPT_DEBUG) {
+            cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
+            cpu->exception_index = EXCP_DEBUG;
+            cpu_loop_exit(cpu);
+        }
+        if (replay_mode == REPLAY_MODE_PLAY && !replay_has_interrupt()) {
+            /* Do nothing */
+        } else if (interrupt_request & CPU_INTERRUPT_HALT) {
+            replay_interrupt();
+            cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
+            cpu->halted = 1;
+            cpu->exception_index = EXCP_HLT;
+            cpu_loop_exit(cpu);
+        }
+#if defined(TARGET_I386)
+        else if (interrupt_request & CPU_INTERRUPT_INIT) {
+            X86CPU *x86_cpu = X86_CPU(cpu);
+            CPUArchState *env = &x86_cpu->env;
+            replay_interrupt();
+            cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
+            do_cpu_init(x86_cpu);
+            cpu->exception_index = EXCP_HALTED;
+            cpu_loop_exit(cpu);
+        }
+#else
+        else if (interrupt_request & CPU_INTERRUPT_RESET) {
+            replay_interrupt();
+            cpu_reset(cpu);
+            cpu_loop_exit(cpu);
+        }
+#endif
+        /* The target hook has 3 exit conditions:
+           False when the interrupt isn't processed,
+           True when it is, and we should restart on a new TB,
+           and via longjmp via cpu_loop_exit.  */
+        else {
+            replay_interrupt();
+            if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
+                *last_tb = NULL;
+            }
+            /* The target hook may have updated the 'cpu->interrupt_request';
+             * reload the 'interrupt_request' value */
+            interrupt_request = cpu->interrupt_request;
+        }
+        if (interrupt_request & CPU_INTERRUPT_EXITTB) {
+            cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
+            /* ensure that no TB jump will be modified as
+               the program flow was changed */
+            *last_tb = NULL;
+        }
+    }
+    if (unlikely(cpu->exit_request || replay_has_interrupt())) {
+        cpu->exit_request = 0;
+        cpu->exception_index = EXCP_INTERRUPT;
+        cpu_loop_exit(cpu);
+    }
+}
+
+static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
+                                    TranslationBlock **last_tb, int *tb_exit,
+                                    SyncClocks *sc)
+{
+    uintptr_t ret;
+
+    if (unlikely(cpu->exit_request)) {
+        return;
+    }
+
+    trace_exec_tb(tb, tb->pc);
+    ret = cpu_tb_exec(cpu, tb);
+    *last_tb = (TranslationBlock *)(ret & ~TB_EXIT_MASK);
+    *tb_exit = ret & TB_EXIT_MASK;
+    switch (*tb_exit) {
+    case TB_EXIT_REQUESTED:
+        /* Something asked us to stop executing
+         * chained TBs; just continue round the main
+         * loop. Whatever requested the exit will also
+         * have set something else (eg exit_request or
+         * interrupt_request) which we will handle
+         * next time around the loop.  But we need to
+         * ensure the tcg_exit_req read in generated code
+         * comes before the next read of cpu->exit_request
+         * or cpu->interrupt_request.
+         */
+        smp_rmb();
+        *last_tb = NULL;
+        break;
+    case TB_EXIT_ICOUNT_EXPIRED:
+    {
+        /* Instruction counter expired.  */
+#ifdef CONFIG_USER_ONLY
+        abort();
+#else
+        int insns_left = cpu->icount_decr.u32;
+        if (cpu->icount_extra && insns_left >= 0) {
+            /* Refill decrementer and continue execution.  */
+            cpu->icount_extra += insns_left;
+            insns_left = MIN(0xffff, cpu->icount_extra);
+            cpu->icount_extra -= insns_left;
+            cpu->icount_decr.u16.low = insns_left;
+        } else {
+            if (insns_left > 0) {
+                /* Execute remaining instructions.  */
+                cpu_exec_nocache(cpu, insns_left, *last_tb, false);
+                align_clocks(sc, cpu);
+            }
+            cpu->exception_index = EXCP_INTERRUPT;
+            *last_tb = NULL;
+            cpu_loop_exit(cpu);
+        }
+        break;
+#endif
+    }
+    default:
+        break;
+    }
+}
+
+/* main execution loop */
+
+int cpu_exec(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    int ret;
+    SyncClocks sc = {};
+
+    /* replay_interrupt may need current_cpu */
     current_cpu = cpu;
 
-    /* As long as current_cpu is null, up to the assignment just above,
-     * requests by other threads to exit the execution loop are expected to
-     * be issued using the exit_request global. We must make sure that our
-     * evaluation of the global value is performed past the current_cpu
-     * value transition point, which requires a memory barrier as well as
-     * an instruction scheduling constraint on modern architectures.  */
-    smp_mb();
+    if (cpu_handle_halt(cpu)) {
+        return EXCP_HALTED;
+    }
 
-    if (unlikely(exit_request)) {
+    atomic_mb_set(&tcg_current_cpu, cpu);
+    rcu_read_lock();
+
+    if (unlikely(atomic_mb_read(&exit_request))) {
         cpu->exit_request = 1;
     }
 
     cc->cpu_exec_enter(cpu);
-    cpu->exception_index = -1;
 
     /* Calculate difference between guest clock and host clock.
      * This delay includes the delay of the last cycle, so
@@ -380,204 +626,75 @@
      */
     init_delay_params(&sc, cpu);
 
-    /* prepare setjmp context for exception handling */
     for(;;) {
+        /* prepare setjmp context for exception handling */
         if (sigsetjmp(cpu->jmp_env, 0) == 0) {
+            TranslationBlock *tb, *last_tb = NULL;
+            int tb_exit = 0;
+
             /* if an exception is pending, we execute it here */
-            if (cpu->exception_index >= 0) {
-                if (cpu->exception_index >= EXCP_INTERRUPT) {
-                    /* exit request from the cpu execution loop */
-                    ret = cpu->exception_index;
-                    if (ret == EXCP_DEBUG) {
-                        cpu_handle_debug_exception(env);
-                    }
-                    break;
-                } else {
-#if defined(CONFIG_USER_ONLY)
-                    /* if user mode only, we simulate a fake exception
-                       which will be handled outside the cpu execution
-                       loop */
-#if defined(TARGET_I386)
-                    cc->do_interrupt(cpu);
-#endif
-                    ret = cpu->exception_index;
-                    break;
-#else
-                    cc->do_interrupt(cpu);
-                    cpu->exception_index = -1;
-#endif
-                }
+            if (cpu_handle_exception(cpu, &ret)) {
+                break;
             }
 
 #ifdef CONFIG_HAX
-            if (hax_enabled() && !hax_vcpu_exec(cpu))
-                longjmp(cpu->jmp_env, 1);
-#endif
+            /* When HAX is enabled but VMX "unrestricted guest" mode is not
+             * supported, call hax_vcpu_exec() to run the current instructions.
+             * The function returns 1 when execution should stop immediately
+             * (e.g. if the vCPU is halted, or received an interrupt). However,
+             * it will return 0 to indicate that the next instructions need to
+             * be handled through TCG. This happens when the virtual CPU runs
+             * in "real mode", or to handle MMIO operations only. */
+            if (hax_enabled() && !hax_vcpu_exec(cpu)) {
+                break;
+            }
+#endif /* CONFIG_HAX */
 
-            next_tb = 0; /* force lookup of first TB */
+            cpu->tb_flushed = false; /* reset before first TB lookup */
             for(;;) {
-                interrupt_request = need_handle_intr_request(cpu);
-                if (unlikely(interrupt_request)) {
-                    if (unlikely(cpu->singlestep_enabled & SSTEP_NOIRQ)) {
-                        /* Mask out external interrupts for this step. */
-                        interrupt_request &= ~CPU_INTERRUPT_SSTEP_MASK;
-                    }
-                    if (interrupt_request & CPU_INTERRUPT_DEBUG) {
-                        cpu->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
-                        cpu->exception_index = EXCP_DEBUG;
-                        cpu_loop_exit(cpu);
-                    }
-                    if (interrupt_request & CPU_INTERRUPT_HALT) {
-                        cpu->interrupt_request &= ~CPU_INTERRUPT_HALT;
-                        cpu->halted = 1;
-                        cpu->exception_index = EXCP_HLT;
-                        cpu_loop_exit(cpu);
-                    }
-#if defined(TARGET_I386)
-                    if (interrupt_request & CPU_INTERRUPT_INIT) {
-                        cpu_svm_check_intercept_param(env, SVM_EXIT_INIT, 0);
-                        do_cpu_init(x86_cpu);
-                        cpu->exception_index = EXCP_HALTED;
-                        cpu_loop_exit(cpu);
-                    }
-#else
-                    if (interrupt_request & CPU_INTERRUPT_RESET) {
-                        cpu_reset(cpu);
-                    }
-#endif
-                    /* The target hook has 3 exit conditions:
-                       False when the interrupt isn't processed,
-                       True when it is, and we should restart on a new TB,
-                       and via longjmp via cpu_loop_exit.  */
-                    if (cc->cpu_exec_interrupt(cpu, interrupt_request)) {
-                        next_tb = 0;
-                    }
-                    /* Don't use the cached interrupt_request value,
-                       do_interrupt may have updated the EXITTB flag. */
-                    if (cpu->interrupt_request & CPU_INTERRUPT_EXITTB) {
-                        cpu->interrupt_request &= ~CPU_INTERRUPT_EXITTB;
-                        /* ensure that no TB jump will be modified as
-                           the program flow was changed */
-                        next_tb = 0;
-                    }
-                }
-                if (unlikely(cpu->exit_request)) {
-                    cpu->exit_request = 0;
-                    cpu->exception_index = EXCP_INTERRUPT;
-                    cpu_loop_exit(cpu);
-                }
-                spin_lock(&tcg_ctx.tb_ctx.tb_lock);
-                have_tb_lock = true;
-                tb = tb_find_fast(env);
-                /* Note: we do it here to avoid a gcc bug on Mac OS X when
-                   doing it in tb_find_slow */
-                if (tcg_ctx.tb_ctx.tb_invalidated_flag) {
-                    /* as some TB could have been invalidated because
-                       of memory exceptions while generating the code, we
-                       must recompute the hash index here */
-                    next_tb = 0;
-                    tcg_ctx.tb_ctx.tb_invalidated_flag = 0;
-                }
-                if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
-                    qemu_log("Trace %p [" TARGET_FMT_lx "] %s\n",
-                             tb->tc_ptr, tb->pc, lookup_symbol(tb->pc));
-                }
-                /* see if we can patch the calling TB. When the TB
-                   spans two pages, we cannot safely do a direct
-                   jump. */
-                if (next_tb != 0 && tb->page_addr[1] == -1) {
-                    tb_add_jump((TranslationBlock *)(next_tb & ~TB_EXIT_MASK),
-                                next_tb & TB_EXIT_MASK, tb);
-                }
-                have_tb_lock = false;
-                spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
-
-                /* cpu_interrupt might be called while translating the
-                   TB, but before it is linked into a potentially
-                   infinite loop and becomes env->current_tb. Avoid
-                   starting execution if there is a pending interrupt. */
-                cpu->current_tb = tb;
-                barrier();
-                if (likely(!cpu->exit_request)) {
-                    trace_exec_tb(tb, tb->pc);
-                    tc_ptr = tb->tc_ptr;
-                    /* execute the generated code */
-                    next_tb = cpu_tb_exec(cpu, tc_ptr);
-                    switch (next_tb & TB_EXIT_MASK) {
-                    case TB_EXIT_REQUESTED:
-                        /* Something asked us to stop executing
-                         * chained TBs; just continue round the main
-                         * loop. Whatever requested the exit will also
-                         * have set something else (eg exit_request or
-                         * interrupt_request) which we will handle
-                         * next time around the loop.
-                         */
-                        tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
-                        next_tb = 0;
-                        break;
-                    case TB_EXIT_ICOUNT_EXPIRED:
-                    {
-                        /* Instruction counter expired.  */
-                        int insns_left;
-                        tb = (TranslationBlock *)(next_tb & ~TB_EXIT_MASK);
-                        insns_left = cpu->icount_decr.u32;
-                        if (cpu->icount_extra && insns_left >= 0) {
-                            /* Refill decrementer and continue execution.  */
-                            cpu->icount_extra += insns_left;
-                            if (cpu->icount_extra > 0xffff) {
-                                insns_left = 0xffff;
-                            } else {
-                                insns_left = cpu->icount_extra;
-                            }
-                            cpu->icount_extra -= insns_left;
-                            cpu->icount_decr.u16.low = insns_left;
-                        } else {
-                            if (insns_left > 0) {
-                                /* Execute remaining instructions.  */
-                                cpu_exec_nocache(env, insns_left, tb);
-                                align_clocks(&sc, cpu);
-                            }
-                            cpu->exception_index = EXCP_INTERRUPT;
-                            next_tb = 0;
-                            cpu_loop_exit(cpu);
-                        }
-                        break;
-                    }
-                    default:
-                        break;
-                    }
-                }
-                cpu->current_tb = NULL;
+                cpu_handle_interrupt(cpu, &last_tb);
+                tb = tb_find_fast(cpu, &last_tb, tb_exit);
+                cpu_loop_exec_tb(cpu, tb, &last_tb, &tb_exit, &sc);
 #ifdef CONFIG_HAX
-                if (hax_enabled() && hax_stop_emulation(cpu))
+                if (hax_enabled() && hax_stop_emulation(cpu)) {
+                    /* This will end TCG emulation of instructions if the vCPU
+                     * just switched to paged-mode (which can be handled by
+                     * hax_vcpu_exec() in the next call to this function), or
+                     * if the single-instruction MMIO operation has completed.
+                     * (see target-i386/translate.c).
+                     */ 
                     cpu_loop_exit(cpu);
-#endif
-
+                }
+#endif /* CONFIG_HAX */
                 /* Try to align the host and virtual clocks
                    if the guest is in advance */
                 align_clocks(&sc, cpu);
-                /* reset soft MMU for next block (it can currently
-                   only be set by a memory fault) */
             } /* for(;;) */
         } else {
-            /* Reload env after longjmp - the compiler may have smashed all
-             * local variables as longjmp is marked 'noreturn'. */
+#if defined(__clang__) || !QEMU_GNUC_PREREQ(4, 6)
+            /* Some compilers wrongly smash all local variables after
+             * siglongjmp. There were bug reports for gcc 4.5.0 and clang.
+             * Reload essential local variables here for those compilers.
+             * Newer versions of gcc would complain about this code (-Wclobbered). */
             cpu = current_cpu;
-            env = cpu->env_ptr;
             cc = CPU_GET_CLASS(cpu);
-#ifdef TARGET_I386
-            x86_cpu = X86_CPU(cpu);
-#endif
-            if (have_tb_lock) {
-                spin_unlock(&tcg_ctx.tb_ctx.tb_lock);
-                have_tb_lock = false;
-            }
+#else /* buggy compiler */
+            /* Assert that the compiler does not smash local variables. */
+            g_assert(cpu == current_cpu);
+            g_assert(cc == CPU_GET_CLASS(cpu));
+#endif /* buggy compiler */
+            cpu->can_do_io = 1;
+            tb_lock_reset();
         }
     } /* for(;;) */
 
     cc->cpu_exec_exit(cpu);
+    rcu_read_unlock();
 
     /* fail safe : never use current_cpu outside cpu_exec() */
     current_cpu = NULL;
+
+    /* Does not need atomic_mb_set because a spurious wakeup is okay.  */
+    atomic_set(&tcg_current_cpu, NULL);
     return ret;
 }
diff --git a/cpus.c b/cpus.c
index a1df372..a2be482 100644
--- a/cpus.c
+++ b/cpus.c
@@ -23,16 +23,20 @@
  */
 
 /* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
-
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "monitor/monitor.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 #include "exec/gdbstub.h"
 #include "sysemu/dma.h"
 #include "sysemu/kvm.h"
 #include "sysemu/hax.h"
 #include "qmp-commands.h"
+#include "exec/exec-all.h"
 
 #include "qemu/thread.h"
 #include "sysemu/cpus.h"
@@ -42,6 +46,7 @@
 #include "qemu/seqlock.h"
 #include "qapi-event.h"
 #include "hw/nmi.h"
+#include "sysemu/replay.h"
 
 #ifndef _WIN32
 #include "qemu/compatfd.h"
@@ -69,6 +74,14 @@
 int64_t max_delay;
 int64_t max_advance;
 
+/* vcpu throttling controls */
+static QEMUTimer *throttle_timer;
+static unsigned int throttle_percentage;
+
+#define CPU_THROTTLE_PCT_MIN 1
+#define CPU_THROTTLE_PCT_MAX 99
+#define CPU_THROTTLE_TIMESLICE_NS 10000000
+
 bool cpu_is_stopped(CPUState *cpu)
 {
     return cpu->stopped || !runstate_is_running();
@@ -106,6 +119,7 @@
 
 /* Protected by TimersState seqlock */
 
+static bool icount_sleep = true;
 static int64_t vm_clock_warp_start = -1;
 /* Conversion factor from emulated instructions to virtual clock ticks.  */
 static int icount_time_shift;
@@ -137,19 +151,26 @@
 
 static TimersState timers_state;
 
-/* Return the virtual CPU time, based on the instruction counter.  */
-static int64_t cpu_get_icount_locked(void)
+int64_t cpu_get_icount_raw(void)
 {
     int64_t icount;
     CPUState *cpu = current_cpu;
 
     icount = timers_state.qemu_icount;
     if (cpu) {
-        if (!cpu_can_do_io(cpu)) {
-            fprintf(stderr, "Bad clock read\n");
+        if (!cpu->can_do_io) {
+            fprintf(stderr, "Bad icount read\n");
+            exit(1);
         }
         icount -= (cpu->icount_decr.u16.low + cpu->icount_extra);
     }
+    return icount;
+}
+
+/* Return the virtual CPU time, based on the instruction counter.  */
+static int64_t cpu_get_icount_locked(void)
+{
+    int64_t icount = cpu_get_icount_raw();
     return timers_state.qemu_icount_bias + cpu_icount_to_ns(icount);
 }
 
@@ -183,7 +204,7 @@
 
     ticks = timers_state.cpu_ticks_offset;
     if (timers_state.cpu_ticks_enabled) {
-        ticks += cpu_get_real_ticks();
+        ticks += cpu_get_host_ticks();
     }
 
     if (timers_state.cpu_ticks_prev > ticks) {
@@ -223,36 +244,19 @@
     return ti;
 }
 
-/* return the offset between the host clock and virtual CPU clock */
-int64_t cpu_get_clock_offset(void)
-{
-    int64_t ti;
-    unsigned start;
-
-    do {
-        start = seqlock_read_begin(&timers_state.vm_clock_seqlock);
-        ti = timers_state.cpu_clock_offset;
-        if (!timers_state.cpu_ticks_enabled) {
-            ti -= get_clock();
-        }
-    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, start));
-
-    return -ti;
-}
-
 /* enable cpu_get_ticks()
  * Caller must hold BQL which server as mutex for vm_clock_seqlock.
  */
 void cpu_enable_ticks(void)
 {
     /* Here, the really thing protected by seqlock is cpu_clock_offset. */
-    seqlock_write_lock(&timers_state.vm_clock_seqlock);
+    seqlock_write_begin(&timers_state.vm_clock_seqlock);
     if (!timers_state.cpu_ticks_enabled) {
-        timers_state.cpu_ticks_offset -= cpu_get_real_ticks();
+        timers_state.cpu_ticks_offset -= cpu_get_host_ticks();
         timers_state.cpu_clock_offset -= get_clock();
         timers_state.cpu_ticks_enabled = 1;
     }
-    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+    seqlock_write_end(&timers_state.vm_clock_seqlock);
 }
 
 /* disable cpu_get_ticks() : the clock is stopped. You must not call
@@ -262,20 +266,20 @@
 void cpu_disable_ticks(void)
 {
     /* Here, the really thing protected by seqlock is cpu_clock_offset. */
-    seqlock_write_lock(&timers_state.vm_clock_seqlock);
+    seqlock_write_begin(&timers_state.vm_clock_seqlock);
     if (timers_state.cpu_ticks_enabled) {
-        timers_state.cpu_ticks_offset += cpu_get_real_ticks();
+        timers_state.cpu_ticks_offset += cpu_get_host_ticks();
         timers_state.cpu_clock_offset = cpu_get_clock_locked();
         timers_state.cpu_ticks_enabled = 0;
     }
-    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+    seqlock_write_end(&timers_state.vm_clock_seqlock);
 }
 
 /* Correlation between real and virtual time is always going to be
    fairly approximate, so ignore small variation.
    When the guest is idle real and virtual time will be aligned in
    the IO wait loop.  */
-#define ICOUNT_WOBBLE (get_ticks_per_sec() / 10)
+#define ICOUNT_WOBBLE (NANOSECONDS_PER_SECOND / 10)
 
 static void icount_adjust(void)
 {
@@ -291,7 +295,7 @@
         return;
     }
 
-    seqlock_write_lock(&timers_state.vm_clock_seqlock);
+    seqlock_write_begin(&timers_state.vm_clock_seqlock);
     cur_time = cpu_get_clock_locked();
     cur_icount = cpu_get_icount_locked();
 
@@ -312,13 +316,13 @@
     last_delta = delta;
     timers_state.qemu_icount_bias = cur_icount
                               - (timers_state.qemu_icount << icount_time_shift);
-    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+    seqlock_write_end(&timers_state.vm_clock_seqlock);
 }
 
 static void icount_adjust_rt(void *opaque)
 {
     timer_mod(icount_rt_timer,
-                   qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + 1000);
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL_RT) + 1000);
     icount_adjust();
 }
 
@@ -326,7 +330,7 @@
 {
     timer_mod(icount_vm_timer,
                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / 10);
+                   NANOSECONDS_PER_SECOND / 10);
     icount_adjust();
 }
 
@@ -335,18 +339,27 @@
     return (count + (1 << icount_time_shift) - 1) >> icount_time_shift;
 }
 
-static void icount_warp_rt(void *opaque)
+static void icount_warp_rt(void)
 {
+    unsigned seq;
+    int64_t warp_start;
+
     /* The icount_warp_timer is rescheduled soon after vm_clock_warp_start
      * changes from -1 to another value, so the race here is okay.
      */
-    if (atomic_read(&vm_clock_warp_start) == -1) {
+    do {
+        seq = seqlock_read_begin(&timers_state.vm_clock_seqlock);
+        warp_start = vm_clock_warp_start;
+    } while (seqlock_read_retry(&timers_state.vm_clock_seqlock, seq));
+
+    if (warp_start == -1) {
         return;
     }
 
-    seqlock_write_lock(&timers_state.vm_clock_seqlock);
+    seqlock_write_begin(&timers_state.vm_clock_seqlock);
     if (runstate_is_running()) {
-        int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+        int64_t clock = REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT,
+                                     cpu_get_clock_locked());
         int64_t warp_delta;
 
         warp_delta = clock - vm_clock_warp_start;
@@ -355,74 +368,88 @@
              * In adaptive mode, do not let QEMU_CLOCK_VIRTUAL run too
              * far ahead of real time.
              */
-            int64_t cur_time = cpu_get_clock_locked();
             int64_t cur_icount = cpu_get_icount_locked();
-            int64_t delta = cur_time - cur_icount;
+            int64_t delta = clock - cur_icount;
             warp_delta = MIN(warp_delta, delta);
         }
         timers_state.qemu_icount_bias += warp_delta;
     }
     vm_clock_warp_start = -1;
-    seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+    seqlock_write_end(&timers_state.vm_clock_seqlock);
 
     if (qemu_clock_expired(QEMU_CLOCK_VIRTUAL)) {
         qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
     }
 }
 
+static void icount_timer_cb(void *opaque)
+{
+    /* No need for a checkpoint because the timer already synchronizes
+     * with CHECKPOINT_CLOCK_VIRTUAL_RT.
+     */
+    icount_warp_rt();
+}
+
 void qtest_clock_warp(int64_t dest)
 {
     int64_t clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    AioContext *aio_context;
     assert(qtest_enabled());
+    aio_context = qemu_get_aio_context();
     while (clock < dest) {
         int64_t deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
         int64_t warp = qemu_soonest_timeout(dest - clock, deadline);
-        seqlock_write_lock(&timers_state.vm_clock_seqlock);
+
+        seqlock_write_begin(&timers_state.vm_clock_seqlock);
         timers_state.qemu_icount_bias += warp;
-        seqlock_write_unlock(&timers_state.vm_clock_seqlock);
+        seqlock_write_end(&timers_state.vm_clock_seqlock);
 
         qemu_clock_run_timers(QEMU_CLOCK_VIRTUAL);
+        timerlist_run_timers(aio_context->tlg.tl[QEMU_CLOCK_VIRTUAL]);
         clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     }
     qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
 }
 
-void qemu_clock_warp(QEMUClockType type)
+void qemu_start_warp_timer(void)
 {
     int64_t clock;
     int64_t deadline;
 
-    /*
-     * There are too many global variables to make the "warp" behavior
-     * applicable to other clocks.  But a clock argument removes the
-     * need for if statements all over the place.
-     */
-    if (type != QEMU_CLOCK_VIRTUAL || !use_icount) {
+    if (!use_icount) {
         return;
     }
 
-    /*
-     * If the CPUs have been sleeping, advance QEMU_CLOCK_VIRTUAL timer now.
-     * This ensures that the deadline for the timer is computed correctly below.
-     * This also makes sure that the insn counter is synchronized before the
-     * CPU starts running, in case the CPU is woken by an event other than
-     * the earliest QEMU_CLOCK_VIRTUAL timer.
+    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
+     * do not fire, so computing the deadline does not make sense.
      */
-    icount_warp_rt(NULL);
-    timer_del(icount_warp_timer);
+    if (!runstate_is_running()) {
+        return;
+    }
+
+    /* warp clock deterministically in record/replay mode */
+    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_START)) {
+        return;
+    }
+
     if (!all_cpu_threads_idle()) {
         return;
     }
 
     if (qtest_enabled()) {
         /* When testing, qtest commands advance icount.  */
-	return;
+        return;
     }
 
     /* We want to use the earliest deadline from ALL vm_clocks */
-    clock = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
     deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
     if (deadline < 0) {
+        static bool notified;
+        if (!icount_sleep && !notified) {
+            error_report("WARNING: icount sleep disabled and no active timers");
+            notified = true;
+        }
         return;
     }
 
@@ -433,28 +460,62 @@
          * interrupt to wake it up, but the interrupt never comes because
          * the vCPU isn't running any insns and thus doesn't advance the
          * QEMU_CLOCK_VIRTUAL.
-         *
-         * An extreme solution for this problem would be to never let VCPUs
-         * sleep in icount mode if there is a pending QEMU_CLOCK_VIRTUAL
-         * timer; rather time could just advance to the next QEMU_CLOCK_VIRTUAL
-         * event.  Instead, we do stop VCPUs and only advance QEMU_CLOCK_VIRTUAL
-         * after some e"real" time, (related to the time left until the next
-         * event) has passed. The QEMU_CLOCK_REALTIME timer will do this.
-         * This avoids that the warps are visible externally; for example,
-         * you will not be sending network packets continuously instead of
-         * every 100ms.
          */
-        seqlock_write_lock(&timers_state.vm_clock_seqlock);
-        if (vm_clock_warp_start == -1 || vm_clock_warp_start > clock) {
-            vm_clock_warp_start = clock;
+        if (!icount_sleep) {
+            /*
+             * We never let VCPUs sleep in no sleep icount mode.
+             * If there is a pending QEMU_CLOCK_VIRTUAL timer we just advance
+             * to the next QEMU_CLOCK_VIRTUAL event and notify it.
+             * It is useful when we want a deterministic execution time,
+             * isolated from host latencies.
+             */
+            seqlock_write_begin(&timers_state.vm_clock_seqlock);
+            timers_state.qemu_icount_bias += deadline;
+            seqlock_write_end(&timers_state.vm_clock_seqlock);
+            qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
+        } else {
+            /*
+             * We do stop VCPUs and only advance QEMU_CLOCK_VIRTUAL after some
+             * "real" time, (related to the time left until the next event) has
+             * passed. The QEMU_CLOCK_VIRTUAL_RT clock will do this.
+             * This avoids that the warps are visible externally; for example,
+             * you will not be sending network packets continuously instead of
+             * every 100ms.
+             */
+            seqlock_write_begin(&timers_state.vm_clock_seqlock);
+            if (vm_clock_warp_start == -1 || vm_clock_warp_start > clock) {
+                vm_clock_warp_start = clock;
+            }
+            seqlock_write_end(&timers_state.vm_clock_seqlock);
+            timer_mod_anticipate(icount_warp_timer, clock + deadline);
         }
-        seqlock_write_unlock(&timers_state.vm_clock_seqlock);
-        timer_mod_anticipate(icount_warp_timer, clock + deadline);
     } else if (deadline == 0) {
         qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
     }
 }
 
+static void qemu_account_warp_timer(void)
+{
+    if (!use_icount || !icount_sleep) {
+        return;
+    }
+
+    /* Nothing to do if the VM is stopped: QEMU_CLOCK_VIRTUAL timers
+     * do not fire, so computing the deadline does not make sense.
+     */
+    if (!runstate_is_running()) {
+        return;
+    }
+
+    /* warp clock deterministically in record/replay mode */
+    if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_ACCOUNT)) {
+        return;
+    }
+
+    timer_del(icount_warp_timer);
+    icount_warp_rt();
+}
+
 static bool icount_state_needed(void *opaque)
 {
     return use_icount;
@@ -467,6 +528,7 @@
     .name = "timer/icount",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = icount_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT64(qemu_icount_bias, TimersState),
         VMSTATE_INT64(qemu_icount, TimersState),
@@ -484,20 +546,86 @@
         VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &icount_vmstate_timers,
-            .needed = icount_state_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &icount_vmstate_timers,
+        NULL
     }
 };
 
+static void cpu_throttle_thread(void *opaque)
+{
+    CPUState *cpu = opaque;
+    double pct;
+    double throttle_ratio;
+    long sleeptime_ns;
+
+    if (!cpu_throttle_get_percentage()) {
+        return;
+    }
+
+    pct = (double)cpu_throttle_get_percentage()/100;
+    throttle_ratio = pct / (1 - pct);
+    sleeptime_ns = (long)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS);
+
+    qemu_mutex_unlock_iothread();
+    atomic_set(&cpu->throttle_thread_scheduled, 0);
+    g_usleep(sleeptime_ns / 1000); /* Convert ns to us for usleep call */
+    qemu_mutex_lock_iothread();
+}
+
+static void cpu_throttle_timer_tick(void *opaque)
+{
+    CPUState *cpu;
+    double pct;
+
+    /* Stop the timer if needed */
+    if (!cpu_throttle_get_percentage()) {
+        return;
+    }
+    CPU_FOREACH(cpu) {
+        if (!atomic_xchg(&cpu->throttle_thread_scheduled, 1)) {
+            async_run_on_cpu(cpu, cpu_throttle_thread, cpu);
+        }
+    }
+
+    pct = (double)cpu_throttle_get_percentage()/100;
+    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
+                                   CPU_THROTTLE_TIMESLICE_NS / (1-pct));
+}
+
+void cpu_throttle_set(int new_throttle_pct)
+{
+    /* Ensure throttle percentage is within valid range */
+    new_throttle_pct = MIN(new_throttle_pct, CPU_THROTTLE_PCT_MAX);
+    new_throttle_pct = MAX(new_throttle_pct, CPU_THROTTLE_PCT_MIN);
+
+    atomic_set(&throttle_percentage, new_throttle_pct);
+
+    timer_mod(throttle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT) +
+                                       CPU_THROTTLE_TIMESLICE_NS);
+}
+
+void cpu_throttle_stop(void)
+{
+    atomic_set(&throttle_percentage, 0);
+}
+
+bool cpu_throttle_active(void)
+{
+    return (cpu_throttle_get_percentage() != 0);
+}
+
+int cpu_throttle_get_percentage(void)
+{
+    return atomic_read(&throttle_percentage);
+}
+
 void cpu_ticks_init(void)
 {
-    seqlock_init(&timers_state.vm_clock_seqlock, NULL);
+    seqlock_init(&timers_state.vm_clock_seqlock);
     vmstate_register(NULL, 0, &vmstate_timers, &timers_state);
+    throttle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
+                                           cpu_throttle_timer_tick, NULL);
 }
 
 void configure_icount(QemuOpts *opts, Error **errp)
@@ -512,9 +640,18 @@
         }
         return;
     }
+
+    icount_sleep = qemu_opt_get_bool(opts, "sleep", true);
+    if (icount_sleep) {
+        icount_warp_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL_RT,
+                                         icount_timer_cb, NULL);
+    }
+
     icount_align_option = qemu_opt_get_bool(opts, "align", false);
-    icount_warp_timer = timer_new_ns(QEMU_CLOCK_REALTIME,
-                                          icount_warp_rt, NULL);
+
+    if (icount_align_option && !icount_sleep) {
+        error_setg(errp, "align=on and sleep=off are incompatible");
+    }
     if (strcmp(option, "auto") != 0) {
         errno = 0;
         icount_time_shift = strtol(option, &rem_str, 0);
@@ -525,6 +662,8 @@
         return;
     } else if (icount_align_option) {
         error_setg(errp, "shift=auto and align=on are incompatible");
+    } else if (!icount_sleep) {
+        error_setg(errp, "shift=auto and sleep=off are incompatible");
     }
 
     use_icount = 2;
@@ -538,15 +677,15 @@
        the virtual time trigger catches emulated time passing too fast.
        Realtime triggers occur even when idle, so use them less frequently
        than VM triggers.  */
-    icount_rt_timer = timer_new_ms(QEMU_CLOCK_REALTIME,
-                                        icount_adjust_rt, NULL);
+    icount_rt_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL_RT,
+                                   icount_adjust_rt, NULL);
     timer_mod(icount_rt_timer,
-                   qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + 1000);
+                   qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL_RT) + 1000);
     icount_vm_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
                                         icount_adjust_vm, NULL);
     timer_mod(icount_vm_timer,
                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / 10);
+                   NANOSECONDS_PER_SECOND / 10);
 }
 
 /***********************************************************/
@@ -607,15 +746,6 @@
     }
 }
 
-void cpu_clean_all_dirty(void)
-{
-    CPUState *cpu;
-
-    CPU_FOREACH(cpu) {
-        cpu_clean_state(cpu);
-    }
-}
-
 static int do_vm_stop(RunState state)
 {
     int ret = 0;
@@ -629,7 +759,7 @@
     }
 
     bdrv_drain_all();
-    ret = bdrv_flush_all();
+    ret = blk_flush_all();
 
     return ret;
 }
@@ -652,14 +782,6 @@
     cpu->stopped = true;
 }
 
-static void cpu_signal(int sig)
-{
-    if (current_cpu) {
-        cpu_exit(current_cpu);
-    }
-    exit_request = 1;
-}
-
 #ifdef CONFIG_LINUX
 static void sigbus_reraise(void)
 {
@@ -672,7 +794,7 @@
         raise(SIGBUS);
         sigemptyset(&set);
         sigaddset(&set, SIGBUS);
-        sigprocmask(SIG_UNBLOCK, &set, NULL);
+        pthread_sigmask(SIG_UNBLOCK, &set, NULL);
     }
     perror("Failed to re-raise SIGBUS!\n");
     abort();
@@ -772,51 +894,18 @@
     }
 }
 
-static void qemu_tcg_init_cpu_signals(void)
-{
-    sigset_t set;
-    struct sigaction sigact;
-
-    memset(&sigact, 0, sizeof(sigact));
-    sigact.sa_handler = cpu_signal;
-    sigaction(SIG_IPI, &sigact, NULL);
-
-    sigemptyset(&set);
-    sigaddset(&set, SIG_IPI);
-    pthread_sigmask(SIG_UNBLOCK, &set, NULL);
-}
-
 #else /* _WIN32 */
 static void qemu_kvm_init_cpu_signals(CPUState *cpu)
 {
     abort();
 }
-
-static void qemu_tcg_init_cpu_signals(void)
-{
-}
 #endif /* _WIN32 */
 
 static QemuMutex qemu_global_mutex;
 static QemuCond qemu_io_proceeded_cond;
-static bool iothread_requesting_mutex;
+static unsigned iothread_requesting_mutex;
 
-#ifdef CONFIG_ANDROID
-static __thread bool qemu_global_mutex_held;
-
-static void qemu_global_cond_wait(QemuCond* cond) {
-    qemu_global_mutex_held = false;
-    qemu_cond_wait(cond, &qemu_global_mutex);
-    qemu_global_mutex_held = true;
-}
-#else
-static void qemu_global_cond_wait(QemuCond* cond) {
-    qemu_cond_wait(cond, &qemu_global_mutex);
-}
-#endif
-
-static QemuThread *tcg_cpu_thread;
-static QemuCond *tcg_halt_cond;
+static QemuThread io_thread;
 
 /* cpu creation */
 static QemuCond qemu_cpu_cond;
@@ -832,6 +921,8 @@
     qemu_cond_init(&qemu_work_cond);
     qemu_cond_init(&qemu_io_proceeded_cond);
     qemu_mutex_init(&qemu_global_mutex);
+
+    qemu_thread_get_self(&io_thread);
 }
 
 void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
@@ -846,6 +937,8 @@
     wi.func = func;
     wi.data = data;
     wi.free = false;
+
+    qemu_mutex_lock(&cpu->work_mutex);
     if (cpu->queued_work_first == NULL) {
         cpu->queued_work_first = &wi;
     } else {
@@ -854,12 +947,13 @@
     cpu->queued_work_last = &wi;
     wi.next = NULL;
     wi.done = false;
+    qemu_mutex_unlock(&cpu->work_mutex);
 
     qemu_cpu_kick(cpu);
-    while (!wi.done) {
+    while (!atomic_mb_read(&wi.done)) {
         CPUState *self_cpu = current_cpu;
 
-        qemu_global_cond_wait(&qemu_work_cond);
+        qemu_cond_wait(&qemu_work_cond, &qemu_global_mutex);
         current_cpu = self_cpu;
     }
 }
@@ -877,6 +971,8 @@
     wi->func = func;
     wi->data = data;
     wi->free = true;
+
+    qemu_mutex_lock(&cpu->work_mutex);
     if (cpu->queued_work_first == NULL) {
         cpu->queued_work_first = wi;
     } else {
@@ -885,10 +981,23 @@
     cpu->queued_work_last = wi;
     wi->next = NULL;
     wi->done = false;
+    qemu_mutex_unlock(&cpu->work_mutex);
 
     qemu_cpu_kick(cpu);
 }
 
+static void qemu_kvm_destroy_vcpu(CPUState *cpu)
+{
+    if (kvm_destroy_vcpu(cpu) < 0) {
+        error_report("kvm_destroy_vcpu failed");
+        exit(EXIT_FAILURE);
+    }
+}
+
+static void qemu_tcg_destroy_vcpu(CPUState *cpu)
+{
+}
+
 static void flush_queued_work(CPUState *cpu)
 {
     struct qemu_work_item *wi;
@@ -897,15 +1006,23 @@
         return;
     }
 
-    while ((wi = cpu->queued_work_first)) {
+    qemu_mutex_lock(&cpu->work_mutex);
+    while (cpu->queued_work_first != NULL) {
+        wi = cpu->queued_work_first;
         cpu->queued_work_first = wi->next;
+        if (!cpu->queued_work_first) {
+            cpu->queued_work_last = NULL;
+        }
+        qemu_mutex_unlock(&cpu->work_mutex);
         wi->func(wi->data);
-        wi->done = true;
+        qemu_mutex_lock(&cpu->work_mutex);
         if (wi->free) {
             g_free(wi);
+        } else {
+            atomic_mb_set(&wi->done, true);
         }
     }
-    cpu->queued_work_last = NULL;
+    qemu_mutex_unlock(&cpu->work_mutex);
     qemu_cond_broadcast(&qemu_work_cond);
 }
 
@@ -914,25 +1031,20 @@
     if (cpu->stop) {
         cpu->stop = false;
         cpu->stopped = true;
-        qemu_cond_signal(&qemu_pause_cond);
+        qemu_cond_broadcast(&qemu_pause_cond);
     }
     flush_queued_work(cpu);
     cpu->thread_kicked = false;
 }
 
-static void qemu_tcg_wait_io_event(void)
+static void qemu_tcg_wait_io_event(CPUState *cpu)
 {
-    CPUState *cpu;
-
     while (all_cpu_threads_idle()) {
-       /* Start accounting real time to the virtual clock if the CPUs
-          are idle.  */
-        qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
-        qemu_global_cond_wait(tcg_halt_cond);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
     while (iothread_requesting_mutex) {
-        qemu_global_cond_wait(&qemu_io_proceeded_cond);
+        qemu_cond_wait(&qemu_io_proceeded_cond, &qemu_global_mutex);
     }
 
     CPU_FOREACH(cpu) {
@@ -944,17 +1056,16 @@
 static void qemu_hax_wait_io_event(CPUState *cpu)
 {
     while (cpu_thread_is_idle(cpu)) {
-        qemu_global_cond_wait(cpu->halt_cond);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
-
     qemu_wait_io_event_common(cpu);
 }
-#endif
+#endif /* CONFIG_HAX */
 
 static void qemu_kvm_wait_io_event(CPUState *cpu)
 {
     while (cpu_thread_is_idle(cpu)) {
-        qemu_global_cond_wait(cpu->halt_cond);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
     qemu_kvm_eat_signals(cpu);
@@ -966,12 +1077,12 @@
     CPUState *cpu = arg;
     int r;
 
-    qemu_mutex_lock(&qemu_global_mutex);
-#ifdef CONFIG_ANDROID
-    qemu_global_mutex_held = true;
-#endif
+    rcu_register_thread();
+
+    qemu_mutex_lock_iothread();
     qemu_thread_get_self(cpu->thread);
     cpu->thread_id = qemu_get_thread_id();
+    cpu->can_do_io = 1;
     current_cpu = cpu;
 
     r = kvm_init_vcpu(cpu);
@@ -986,7 +1097,7 @@
     cpu->created = true;
     qemu_cond_signal(&qemu_cpu_cond);
 
-    while (1) {
+    do {
         if (cpu_can_run(cpu)) {
             r = kvm_cpu_exec(cpu);
             if (r == EXCP_DEBUG) {
@@ -994,8 +1105,12 @@
             }
         }
         qemu_kvm_wait_io_event(cpu);
-    }
+    } while (!cpu->unplug || cpu_can_run(cpu));
 
+    qemu_kvm_destroy_vcpu(cpu);
+    cpu->created = false;
+    qemu_cond_signal(&qemu_cpu_cond);
+    qemu_mutex_unlock_iothread();
     return NULL;
 }
 
@@ -1010,9 +1125,12 @@
     sigset_t waitset;
     int r;
 
+    rcu_register_thread();
+
     qemu_mutex_lock_iothread();
     qemu_thread_get_self(cpu->thread);
     cpu->thread_id = qemu_get_thread_id();
+    cpu->can_do_io = 1;
 
     sigemptyset(&waitset);
     sigaddset(&waitset, SIG_IPI);
@@ -1047,23 +1165,23 @@
 static void *qemu_tcg_cpu_thread_fn(void *arg)
 {
     CPUState *cpu = arg;
+    CPUState *remove_cpu = NULL;
 
-    qemu_tcg_init_cpu_signals();
+    rcu_register_thread();
+
+    qemu_mutex_lock_iothread();
     qemu_thread_get_self(cpu->thread);
 
-    qemu_mutex_lock(&qemu_global_mutex);
-#ifdef CONFIG_ANDROID
-    qemu_global_mutex_held = true;
-#endif
     CPU_FOREACH(cpu) {
         cpu->thread_id = qemu_get_thread_id();
         cpu->created = true;
+        cpu->can_do_io = 1;
     }
     qemu_cond_signal(&qemu_cpu_cond);
 
     /* wait for initial kick-off after machine start */
-    while (QTAILQ_FIRST(&cpus)->stopped) {
-        qemu_global_cond_wait(tcg_halt_cond);
+    while (first_cpu->stopped) {
+        qemu_cond_wait(first_cpu->halt_cond, &qemu_global_mutex);
 
         /* process any pending work */
         CPU_FOREACH(cpu) {
@@ -1071,6 +1189,9 @@
         }
     }
 
+    /* process any pending work */
+    atomic_mb_set(&exit_request, 1);
+
     while (1) {
         tcg_exec_all();
 
@@ -1081,31 +1202,52 @@
                 qemu_clock_notify(QEMU_CLOCK_VIRTUAL);
             }
         }
-        qemu_tcg_wait_io_event();
+        qemu_tcg_wait_io_event(QTAILQ_FIRST(&cpus));
+        CPU_FOREACH(cpu) {
+            if (cpu->unplug && !cpu_can_run(cpu)) {
+                remove_cpu = cpu;
+                break;
+            }
+        }
+        if (remove_cpu) {
+            qemu_tcg_destroy_vcpu(remove_cpu);
+            cpu->created = false;
+            qemu_cond_signal(&qemu_cpu_cond);
+            remove_cpu = NULL;
+        }
     }
 
     return NULL;
 }
 
 #ifdef CONFIG_HAX
+/* The HAX-specific vCPU thread function. This one should only run when the host
+ * CPU supports the VMX "unrestricted guest" feature. */
 static void *qemu_hax_cpu_thread_fn(void *arg)
 {
     CPUState *cpu = arg;
     int r;
-    qemu_thread_get_self(cpu->thread);
+
+    assert(hax_enabled() && hax_ug_platform());
+
+    rcu_register_thread();
+
     qemu_mutex_lock(&qemu_global_mutex);
-#ifdef CONFIG_ANDROID
-    qemu_global_mutex_held = true;
-#endif
+    qemu_thread_get_self(cpu->thread);
+
     cpu->thread_id = qemu_get_thread_id();
-    cpu->created = true;
-    cpu->halted = 0;
+    cpu->can_do_io = 1;
+//     cpu->created = true;
+//     cpu->halted = 0;
     current_cpu = cpu;
 
     hax_init_vcpu(cpu);
+
+    /* signal CPU creation */
+    cpu->created = true;
     qemu_cond_signal(&qemu_cpu_cond);
 
-    while (1) {
+    do {
         if (cpu_can_run(cpu)) {
             r = hax_smp_cpu_exec(cpu);
             if (r == EXCP_DEBUG) {
@@ -1113,33 +1255,47 @@
             }
         }
         qemu_hax_wait_io_event(cpu);
-    }
+    } while (!cpu->unplug || cpu_can_run(cpu));
+
+    hax_vcpu_destroy(cpu);
+    cpu->created = false;
+    qemu_cond_signal(&qemu_cpu_cond);
+    qemu_mutex_unlock_iothread();
     return NULL;
 }
-#endif
+#endif /* CONFIG_HAX */
 
 static void qemu_cpu_kick_thread(CPUState *cpu)
 {
 #ifndef _WIN32
     int err;
 
+    if (cpu->thread_kicked) {
+        return;
+    }
+    cpu->thread_kicked = true;
     err = pthread_kill(cpu->thread->thread, SIG_IPI);
     if (err) {
         fprintf(stderr, "qemu:%s: %s", __func__, strerror(err));
         exit(1);
     }
-
-#ifdef CONFIG_DARWIN
-    /* The cpu thread cannot catch it reliably when shutdown the guest on Mac.
-     * We can double check it and resend it
-     */
-    if (!exit_request)
-        cpu_signal(0);
-
-    if (hax_enabled() && hax_ug_platform())
-        cpu->exit_request = 1;
-#endif
+#ifdef __APPLE__
+    // On OS X, the signal isn't caught reliably during shutdown.
+    if (!atomic_mb_read(&exit_request)) {
+        cpu_exit(cpu);
+        atomic_mb_set(&exit_request, 1);
+    }
+#endif /* __APPLE__ */
+#ifdef CONFIG_HAX
+    if (hax_enabled() && hax_ug_platform()) {
+        cpu_exit(cpu);
+    }
+#endif /* CONFIG_HAX */
 #else /* _WIN32 */
+    if (cpu->thread_kicked) {
+        return;
+    }
+    cpu->thread_kicked = true;
     if (!qemu_cpu_is_self(cpu)) {
         CONTEXT tcgContext;
 
@@ -1150,18 +1306,14 @@
         }
 
         /* On multi-core systems, we are not sure that the thread is actually
-         * suspended until we can get the context.
-         */
+         * suspended until we can get the context. */
         tcgContext.ContextFlags = CONTEXT_CONTROL;
         while (GetThreadContext(cpu->hThread, &tcgContext) != 0) {
             continue;
         }
 
-        cpu_signal(0);
-#ifdef CONFIG_HAX
-        if (hax_enabled() && hax_ug_platform())
-            cpu->exit_request = 1;
-#endif
+        cpu_exit(cpu);
+        atomic_mb_set(&exit_request, 1);
 
         if (ResumeThread(cpu->hThread) == (DWORD)-1) {
             fprintf(stderr, "qemu:%s: GetLastError:%lu\n", __func__,
@@ -1169,34 +1321,50 @@
             exit(1);
         }
     }
-#endif
+#endif /* _WIN32 */
+}
+
+static void qemu_cpu_kick_no_halt(void)
+{
+    CPUState *cpu;
+    /* Ensure whatever caused the exit has reached the CPU threads before
+     * writing exit_request.
+     */
+    atomic_mb_set(&exit_request, 1);
+    cpu = atomic_mb_read(&tcg_current_cpu);
+    if (cpu) {
+        cpu_exit(cpu);
+    }
 }
 
 void qemu_cpu_kick(CPUState *cpu)
 {
     qemu_cond_broadcast(cpu->halt_cond);
+    /* There are three cases to consider here:
+     *
+     * - TCG is being used without HAX, then qemu_cpu_kick_no_halt() can be
+     *   called directly.
+     * 
+     * - TCG is being used with HAX, then kicking the thread with a signal (on Posix)
+     *   or with a thread suspend/resume (on Win32) is still needed.
+     * 
+     * - TCG is not being used, kick the thread with a signal or suspend/resume.
+     */
 #ifdef CONFIG_HAX
-    if (((hax_enabled() && hax_ug_platform()) || !tcg_enabled()) && !cpu->thread_kicked) {
+    if (tcg_enabled() && !(hax_enabled() && hax_ug_platform())) {
 #else
-    if (!tcg_enabled() && !cpu->thread_kicked) {
+    if (tcg_enabled()) {
 #endif
+        qemu_cpu_kick_no_halt();
+    } else {
         qemu_cpu_kick_thread(cpu);
-        cpu->thread_kicked = true;
     }
 }
 
 void qemu_cpu_kick_self(void)
 {
-#ifndef _WIN32
     assert(current_cpu);
-
-    if (!current_cpu->thread_kicked) {
-        qemu_cpu_kick_thread(current_cpu);
-        current_cpu->thread_kicked = true;
-    }
-#else
-    abort();
-#endif
+    qemu_cpu_kick_thread(current_cpu);
 }
 
 bool qemu_cpu_is_self(CPUState *cpu)
@@ -1204,48 +1372,86 @@
     return qemu_thread_is_self(cpu->thread);
 }
 
-static bool qemu_in_vcpu_thread(void)
+bool qemu_in_vcpu_thread(void)
 {
     return current_cpu && qemu_cpu_is_self(current_cpu);
 }
 
+static __thread bool iothread_locked = false;
+
+bool qemu_mutex_iothread_locked(void)
+{
+    return iothread_locked;
+}
+
 void qemu_mutex_lock_iothread(void)
 {
+    /* Technical note on what's going on here, because it's really subtle :-)
+     * 
+     * The single TCG vCPU thread always holds the global mutex when executing
+     * instructions, and only releases it very briefly in qemu_tcg_wait_io_event(),
+     * which gets called periodically to process interrupts.
+     *
+     * Under heavy guest CPU load, it will be hard for other threads to acquire
+     * the lock due to this. To counter that, several things are implemented here:
+     * 
+     * - First, |iothread_requesting_mutex| is used as a global atomic counter that
+     *   will be > 0 whenever other threads are trying to acquire the lock. It is
+     *   actually read by qemu_tcg_wait_io_event() to force the vCPU thread to
+     *   release the lock until its value reaches 0 again. The |qemu_io_proceeded_cond|
+     *   condition variable is used to do that.
+     * 
+     * - Second, if TCG is enabled, a trylock() is first tried to acquire the lock.
+     *   If this fail, the TCG vCPU thread is kicked(), which forces generated code
+     *   to exit to qemu_tcg_wait_io_event() as soon as possible.
+     * 
+     * NOTE: It looks like the use of |iothread_requesting_mutex| isn't needed at all
+     *       when KVM or HAX execution modes are being used, because the corresponding
+     *       vCPU threads actually _release_ the lock just before entering guest mode
+     *       (and re-acquire it just after exiting from it).
+     */
+    atomic_inc(&iothread_requesting_mutex);
+
+    /* A simple lock is sufficient in the following cases:
+     * 
+     * - TCG is not enabled (KVM execution mode).
+     *   [This is the !tcg_enabled() check]
+     * 
+     * - TCG is enabled, but this called from the TCG vCPU thread directly.
+     *   [This is the qemu_in_vcpu_thread() check]
+     *
+     * - TCG is enabled, but so is HAX in "unrestricted guest" mode, which allows it
+     *   to execute all guest code directly (i.e. there is no TCG vCPU thread).
+     *   [This is the (hax_enabled() && hax_ug_platform()) check].
+     * 
+     * - TCG is enabled, but its thread has not started yet (e.g. when this
+     *   function is called during virtual device realization).
+     *   [This is (!first_cpu || !first_cpu->created)].
+     */
+    if (!tcg_enabled() || qemu_in_vcpu_thread() ||
 #ifdef CONFIG_HAX
-    if ((hax_enabled() && hax_ug_platform()) || !tcg_enabled()) {
-#else
-    if (!tcg_enabled()) {
+        (hax_enabled() && hax_ug_platform()) ||
 #endif
+        !first_cpu || !first_cpu->created) {
         qemu_mutex_lock(&qemu_global_mutex);
+        atomic_dec(&iothread_requesting_mutex);
     } else {
-        iothread_requesting_mutex = true;
         if (qemu_mutex_trylock(&qemu_global_mutex)) {
-            qemu_cpu_kick_thread(first_cpu);
+            qemu_cpu_kick_no_halt();
             qemu_mutex_lock(&qemu_global_mutex);
         }
-        iothread_requesting_mutex = false;
+        atomic_dec(&iothread_requesting_mutex);
         qemu_cond_broadcast(&qemu_io_proceeded_cond);
     }
-#ifdef CONFIG_ANDROID
-    qemu_global_mutex_held = true;
-#endif
+    iothread_locked = true;
 }
 
 void qemu_mutex_unlock_iothread(void)
 {
-#ifdef CONFIG_ANDROID
-    qemu_global_mutex_held = false;
-#endif
+    iothread_locked = false;
     qemu_mutex_unlock(&qemu_global_mutex);
 }
 
-#ifdef CONFIG_ANDROID
-bool qemu_mutex_check_iothread(void)
-{
-    return qemu_global_mutex_held;
-}
-#endif
-
 static int all_vcpus_paused(void)
 {
     CPUState *cpu;
@@ -1281,7 +1487,7 @@
     }
 
     while (!all_vcpus_paused()) {
-        qemu_global_cond_wait(&qemu_pause_cond);
+        qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
         CPU_FOREACH(cpu) {
             qemu_cpu_kick(cpu);
         }
@@ -1305,19 +1511,40 @@
     }
 }
 
+void cpu_remove(CPUState *cpu)
+{
+    cpu->stop = true;
+    cpu->unplug = true;
+    qemu_cpu_kick(cpu);
+}
+
+void cpu_remove_sync(CPUState *cpu)
+{
+    cpu_remove(cpu);
+    while (cpu->created) {
+        qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
+    }
+}
+
 /* For temporary buffers for forming a name */
 #define VCPU_THREAD_NAME_SIZE 16
 
 static void qemu_tcg_init_vcpu(CPUState *cpu)
 {
     char thread_name[VCPU_THREAD_NAME_SIZE];
+    static QemuCond *tcg_halt_cond;
+    static QemuThread *tcg_cpu_thread;
 
 #ifdef CONFIG_HAX
-    if (hax_enabled())
+    if (hax_enabled()) {
+        /* This code path should only be taken when HAX is enabled but the
+         * CPU doesn't support "unrestricted guest" mode. */
+        assert(!hax_ug_platform());
+        /* Initialize HAX-related state for the TCG thread. This is required for
+         * cpu_exec() to work correctly when HAX is enabled. */
         hax_init_vcpu(cpu);
-#endif
-
-    tcg_cpu_address_space_init(cpu, cpu->as);
+    }
+#endif /* CONFIG_HAX */
 
     /* share a single thread for all cpus with TCG */
     if (!tcg_cpu_thread) {
@@ -1333,7 +1560,7 @@
         cpu->hThread = qemu_thread_get_handle(cpu->thread);
 #endif
         while (!cpu->created) {
-            qemu_global_cond_wait(&qemu_cpu_cond);
+            qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
         }
         tcg_cpu_thread = cpu->thread;
     } else {
@@ -1347,6 +1574,13 @@
 {
     char thread_name[VCPU_THREAD_NAME_SIZE];
 
+    /* This function shall only be called when HAX is enabled, and the host CPU
+     * supports "unrestricted guest" mode. This allows emulation of "real mode"
+     * and completely avoids the use of TCG. It's only the only way to get
+     * multi-core accelerated emulation with HAX. */
+    assert(hax_enabled());
+    assert(hax_ug_platform());
+
     cpu->thread = g_malloc0(sizeof(QemuThread));
     cpu->halt_cond = g_malloc0(sizeof(QemuCond));
     qemu_cond_init(cpu->halt_cond);
@@ -1359,10 +1593,10 @@
     cpu->hThread = qemu_thread_get_handle(cpu->thread);
 #endif
     while (!cpu->created) {
-        qemu_global_cond_wait(&qemu_cpu_cond);
+        qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
-#endif
+#endif /* CONFIG_HAX */
 
 static void qemu_kvm_start_vcpu(CPUState *cpu)
 {
@@ -1376,7 +1610,7 @@
     qemu_thread_create(cpu->thread, thread_name, qemu_kvm_cpu_thread_fn,
                        cpu, QEMU_THREAD_JOINABLE);
     while (!cpu->created) {
-        qemu_global_cond_wait(&qemu_cpu_cond);
+        qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
 
@@ -1392,7 +1626,7 @@
     qemu_thread_create(cpu->thread, thread_name, qemu_dummy_cpu_thread_fn, cpu,
                        QEMU_THREAD_JOINABLE);
     while (!cpu->created) {
-        qemu_global_cond_wait(&qemu_cpu_cond);
+        qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
 
@@ -1401,12 +1635,23 @@
     cpu->nr_cores = smp_cores;
     cpu->nr_threads = smp_threads;
     cpu->stopped = true;
+
+    if (!cpu->as) {
+        /* If the target cpu hasn't set up any address spaces itself,
+         * give it the default one.
+         */
+        AddressSpace *as = address_space_init_shareable(cpu->memory,
+                                                        "cpu-memory");
+        cpu->num_ases = 1;
+        cpu_address_space_init(cpu, as, 0);
+    }
+
     if (kvm_enabled()) {
         qemu_kvm_start_vcpu(cpu);
 #ifdef CONFIG_HAX
     } else if (hax_enabled() && hax_ug_platform()) {
         qemu_hax_start_vcpu(cpu);
-#endif
+#endif 
     } else if (tcg_enabled()) {
         qemu_tcg_init_vcpu(cpu);
     } else {
@@ -1420,7 +1665,7 @@
         current_cpu->stop = false;
         current_cpu->stopped = true;
         cpu_exit(current_cpu);
-        qemu_cond_signal(&qemu_pause_cond);
+        qemu_cond_broadcast(&qemu_pause_cond);
     }
 }
 
@@ -1448,31 +1693,19 @@
         return vm_stop(state);
     } else {
         runstate_set(state);
+
+        bdrv_drain_all();
         /* Make sure to return an error if the flush in a previous vm_stop()
          * failed. */
-        return bdrv_flush_all();
+        return blk_flush_all();
     }
 }
 
-static int tcg_cpu_exec(CPUArchState *env)
+static int64_t tcg_get_icount_limit(void)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
-    int ret;
-#ifdef CONFIG_PROFILER
-    int64_t ti;
-#endif
+    int64_t deadline;
 
-#ifdef CONFIG_PROFILER
-    ti = profile_getclock();
-#endif
-    if (use_icount) {
-        int64_t count;
-        int64_t deadline;
-        int decr;
-        timers_state.qemu_icount -= (cpu->icount_decr.u16.low
-                                    + cpu->icount_extra);
-        cpu->icount_decr.u16.low = 0;
-        cpu->icount_extra = 0;
+    if (replay_mode != REPLAY_MODE_PLAY) {
         deadline = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL);
 
         /* Maintain prior (possibly buggy) behaviour where if no deadline
@@ -1484,16 +1717,39 @@
             deadline = INT32_MAX;
         }
 
-        count = qemu_icount_round(deadline);
+        return qemu_icount_round(deadline);
+    } else {
+        return replay_get_instructions();
+    }
+}
+
+static int tcg_cpu_exec(CPUState *cpu)
+{
+    int ret;
+#ifdef CONFIG_PROFILER
+    int64_t ti;
+#endif
+
+#ifdef CONFIG_PROFILER
+    ti = profile_getclock();
+#endif
+    if (use_icount) {
+        int64_t count;
+        int decr;
+        timers_state.qemu_icount -= (cpu->icount_decr.u16.low
+                                    + cpu->icount_extra);
+        cpu->icount_decr.u16.low = 0;
+        cpu->icount_extra = 0;
+        count = tcg_get_icount_limit();
         timers_state.qemu_icount += count;
         decr = (count > 0xffff) ? 0xffff : count;
         count -= decr;
         cpu->icount_decr.u16.low = decr;
         cpu->icount_extra = count;
     }
-    ret = cpu_exec(env);
+    ret = cpu_exec(cpu);
 #ifdef CONFIG_PROFILER
-    qemu_time += profile_getclock() - ti;
+    tcg_time += profile_getclock() - ti;
 #endif
     if (use_icount) {
         /* Fold pending instructions back into the
@@ -1502,6 +1758,7 @@
                         + cpu->icount_extra);
         cpu->icount_decr.u32 = 0;
         cpu->icount_extra = 0;
+        replay_account_executed_instructions();
     }
     return ret;
 }
@@ -1511,29 +1768,33 @@
     int r;
 
     /* Account partial waits to QEMU_CLOCK_VIRTUAL.  */
-    qemu_clock_warp(QEMU_CLOCK_VIRTUAL);
+    qemu_account_warp_timer();
 
     if (next_cpu == NULL) {
         next_cpu = first_cpu;
     }
     for (; next_cpu != NULL && !exit_request; next_cpu = CPU_NEXT(next_cpu)) {
         CPUState *cpu = next_cpu;
-        CPUArchState *env = cpu->env_ptr;
 
         qemu_clock_enable(QEMU_CLOCK_VIRTUAL,
                           (cpu->singlestep_enabled & SSTEP_NOTIMER) == 0);
 
         if (cpu_can_run(cpu)) {
-            r = tcg_cpu_exec(env);
+            r = tcg_cpu_exec(cpu);
             if (r == EXCP_DEBUG) {
                 cpu_handle_guest_debug(cpu);
                 break;
             }
         } else if (cpu->stop || cpu->stopped) {
+            if (cpu->unplug) {
+                next_cpu = CPU_NEXT(cpu);
+            }
             break;
         }
     }
-    exit_request = 0;
+
+    /* Pairs with smp_wmb in qemu_cpu_kick.  */
+    atomic_mb_set(&exit_request, 0);
 }
 
 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
@@ -1575,24 +1836,26 @@
         info->value->CPU = cpu->cpu_index;
         info->value->current = (cpu == first_cpu);
         info->value->halted = cpu->halted;
+        info->value->qom_path = object_get_canonical_path(OBJECT(cpu));
         info->value->thread_id = cpu->thread_id;
 #if defined(TARGET_I386)
-        info->value->has_pc = true;
-        info->value->pc = env->eip + env->segs[R_CS].base;
+        info->value->arch = CPU_INFO_ARCH_X86;
+        info->value->u.x86.pc = env->eip + env->segs[R_CS].base;
 #elif defined(TARGET_PPC)
-        info->value->has_nip = true;
-        info->value->nip = env->nip;
+        info->value->arch = CPU_INFO_ARCH_PPC;
+        info->value->u.ppc.nip = env->nip;
 #elif defined(TARGET_SPARC)
-        info->value->has_pc = true;
-        info->value->pc = env->pc;
-        info->value->has_npc = true;
-        info->value->npc = env->npc;
+        info->value->arch = CPU_INFO_ARCH_SPARC;
+        info->value->u.q_sparc.pc = env->pc;
+        info->value->u.q_sparc.npc = env->npc;
 #elif defined(TARGET_MIPS)
-        info->value->has_PC = true;
-        info->value->PC = env->active_tc.PC;
+        info->value->arch = CPU_INFO_ARCH_MIPS;
+        info->value->u.q_mips.PC = env->active_tc.PC;
 #elif defined(TARGET_TRICORE)
-        info->value->has_PC = true;
-        info->value->PC = env->PC;
+        info->value->arch = CPU_INFO_ARCH_TRICORE;
+        info->value->u.tricore.PC = env->PC;
+#else
+        info->value->arch = CPU_INFO_ARCH_OTHER;
 #endif
 
         /* XXX: waiting for the qapi to support GSList */
@@ -1614,6 +1877,7 @@
     uint32_t l;
     CPUState *cpu;
     uint8_t buf[1024];
+    int64_t orig_addr = addr, orig_size = size;
 
     if (!has_cpu) {
         cpu_index = 0;
@@ -1621,8 +1885,8 @@
 
     cpu = qemu_get_cpu(cpu_index);
     if (cpu == NULL) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cpu-index",
-                  "a CPU number");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cpu-index",
+                   "a CPU number");
         return;
     }
 
@@ -1637,11 +1901,12 @@
         if (l > size)
             l = size;
         if (cpu_memory_rw_debug(cpu, addr, buf, l, 0) != 0) {
-            error_setg(errp, "Invalid addr 0x%016" PRIx64 "specified", addr);
+            error_setg(errp, "Invalid addr 0x%016" PRIx64 "/size %" PRId64
+                             " specified", orig_addr, orig_size);
             goto exit;
         }
         if (fwrite(buf, 1, l, f) != l) {
-            error_set(errp, QERR_IO_ERROR);
+            error_setg(errp, QERR_IO_ERROR);
             goto exit;
         }
         addr += l;
@@ -1671,7 +1936,7 @@
             l = size;
         cpu_physical_memory_read(addr, buf, l);
         if (fwrite(buf, 1, l, f) != l) {
-            error_set(errp, QERR_IO_ERROR);
+            error_setg(errp, QERR_IO_ERROR);
             goto exit;
         }
         addr += l;
@@ -1684,21 +1949,7 @@
 
 void qmp_inject_nmi(Error **errp)
 {
-#if defined(TARGET_I386)
-    CPUState *cs;
-
-    CPU_FOREACH(cs) {
-        X86CPU *cpu = X86_CPU(cs);
-
-        if (!cpu->apic_state) {
-            cpu_interrupt(cs, CPU_INTERRUPT_NMI);
-        } else {
-            apic_deliver_nmi(cpu->apic_state);
-        }
-    }
-#else
     nmi_monitor_handle(monitor_get_cpu_index(), errp);
-#endif
 }
 
 void dump_drift_info(FILE *f, fprintf_function cpu_fprintf)
diff --git a/cputlb.c b/cputlb.c
index a55518a..d068ee5 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/memory.h"
@@ -28,10 +28,35 @@
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
+#include "exec/exec-all.h"
 #include "tcg/tcg.h"
+#include "qemu/error-report.h"
+#include "exec/log.h"
 
-//#define DEBUG_TLB
-//#define DEBUG_TLB_CHECK
+/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
+/* #define DEBUG_TLB */
+/* #define DEBUG_TLB_LOG */
+
+#ifdef DEBUG_TLB
+# define DEBUG_TLB_GATE 1
+# ifdef DEBUG_TLB_LOG
+#  define DEBUG_TLB_LOG_GATE 1
+# else
+#  define DEBUG_TLB_LOG_GATE 0
+# endif
+#else
+# define DEBUG_TLB_GATE 0
+# define DEBUG_TLB_LOG_GATE 0
+#endif
+
+#define tlb_debug(fmt, ...) do { \
+    if (DEBUG_TLB_LOG_GATE) { \
+        qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
+                      ## __VA_ARGS__); \
+    } else if (DEBUG_TLB_GATE) { \
+        fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
+    } \
+} while (0)
 
 /* statistics */
 int tlb_flush_count;
@@ -52,12 +77,7 @@
 {
     CPUArchState *env = cpu->env_ptr;
 
-#if defined(DEBUG_TLB)
-    printf("tlb_flush:\n");
-#endif
-    /* must reset current TB so that interrupts cannot modify the
-       links while we are modifying them */
-    cpu->current_tb = NULL;
+    tlb_debug("(%d)\n", flush_global);
 
     memset(env->tlb_table, -1, sizeof(env->tlb_table));
     memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
@@ -69,6 +89,36 @@
     tlb_flush_count++;
 }
 
+static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
+{
+    CPUArchState *env = cpu->env_ptr;
+
+    tlb_debug("start\n");
+
+    for (;;) {
+        int mmu_idx = va_arg(argp, int);
+
+        if (mmu_idx < 0) {
+            break;
+        }
+
+        tlb_debug("%d\n", mmu_idx);
+
+        memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
+        memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
+    }
+
+    memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
+}
+
+void tlb_flush_by_mmuidx(CPUState *cpu, ...)
+{
+    va_list argp;
+    va_start(argp, cpu);
+    v_tlb_flush_by_mmuidx(cpu, argp);
+    va_end(argp);
+}
+
 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 {
     if (addr == (tlb_entry->addr_read &
@@ -87,22 +137,17 @@
     int i;
     int mmu_idx;
 
-#if defined(DEBUG_TLB)
-    printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
-#endif
+    tlb_debug("page :" TARGET_FMT_lx "\n", addr);
+
     /* Check if we need to flush due to large pages.  */
     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
-#if defined(DEBUG_TLB)
-        printf("tlb_flush_page: forced full flush ("
-               TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
-               env->tlb_flush_addr, env->tlb_flush_mask);
-#endif
+        tlb_debug("forcing full flush ("
+                  TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+                  env->tlb_flush_addr, env->tlb_flush_mask);
+
         tlb_flush(cpu, 1);
         return;
     }
-    /* must reset current TB so that interrupts cannot modify the
-       links while we are modifying them */
-    cpu->current_tb = NULL;
 
     addr &= TARGET_PAGE_MASK;
     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
@@ -121,18 +166,62 @@
     tb_flush_jmp_cache(cpu, addr);
 }
 
+void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
+{
+    CPUArchState *env = cpu->env_ptr;
+    int i, k;
+    va_list argp;
+
+    va_start(argp, addr);
+
+    tlb_debug("addr "TARGET_FMT_lx"\n", addr);
+
+    /* Check if we need to flush due to large pages.  */
+    if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
+        tlb_debug("forced full flush ("
+                  TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
+                  env->tlb_flush_addr, env->tlb_flush_mask);
+
+        v_tlb_flush_by_mmuidx(cpu, argp);
+        va_end(argp);
+        return;
+    }
+
+    addr &= TARGET_PAGE_MASK;
+    i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+
+    for (;;) {
+        int mmu_idx = va_arg(argp, int);
+
+        if (mmu_idx < 0) {
+            break;
+        }
+
+        tlb_debug("idx %d\n", mmu_idx);
+
+        tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
+
+        /* check whether there are vltb entries that need to be flushed */
+        for (k = 0; k < CPU_VTLB_SIZE; k++) {
+            tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
+        }
+    }
+    va_end(argp);
+
+    tb_flush_jmp_cache(cpu, addr);
+}
+
 /* update the TLBs so that writes to code in the virtual page 'addr'
    can be detected */
 void tlb_protect_code(ram_addr_t ram_addr)
 {
-    cpu_physical_memory_reset_dirty(ram_addr, TARGET_PAGE_SIZE,
-                                    DIRTY_MEMORY_CODE);
+    cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
+                                             DIRTY_MEMORY_CODE);
 }
 
 /* update the TLB so that writes in physical page 'phys_addr' are no longer
    tested for self modifying code */
-void tlb_unprotect_code_phys(CPUState *cpu, ram_addr_t ram_addr,
-                             target_ulong vaddr)
+void tlb_unprotect_code(ram_addr_t ram_addr)
 {
     cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
 }
@@ -159,34 +248,32 @@
 {
     ram_addr_t ram_addr;
 
-    if (qemu_ram_addr_from_host(ptr, &ram_addr) == NULL) {
+    ram_addr = qemu_ram_addr_from_host(ptr);
+    if (ram_addr == RAM_ADDR_INVALID) {
         fprintf(stderr, "Bad ram pointer %p\n", ptr);
         abort();
     }
     return ram_addr;
 }
 
-void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 {
-    CPUState *cpu;
     CPUArchState *env;
 
-    CPU_FOREACH(cpu) {
-        int mmu_idx;
+    int mmu_idx;
 
-        env = cpu->env_ptr;
-        for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
-            unsigned int i;
+    env = cpu->env_ptr;
+    for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+        unsigned int i;
 
-            for (i = 0; i < CPU_TLB_SIZE; i++) {
-                tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
-                                      start1, length);
-            }
+        for (i = 0; i < CPU_TLB_SIZE; i++) {
+            tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
+                                  start1, length);
+        }
 
-            for (i = 0; i < CPU_VTLB_SIZE; i++) {
-                tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
-                                      start1, length);
-            }
+        for (i = 0; i < CPU_VTLB_SIZE; i++) {
+            tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
+                                  start1, length);
         }
     }
 }
@@ -200,8 +287,9 @@
 
 /* update the TLB corresponding to virtual page vaddr
    so that it is no longer dirty */
-void tlb_set_dirty(CPUArchState *env, target_ulong vaddr)
+void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 {
+    CPUArchState *env = cpu->env_ptr;
     int i;
     int mmu_idx;
 
@@ -243,11 +331,15 @@
 }
 
 /* Add a new TLB entry. At most one entry for a given virtual address
-   is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
-   supplied size is only used by tlb_flush_page.  */
-void tlb_set_page(CPUState *cpu, target_ulong vaddr,
-                  hwaddr paddr, int prot,
-                  int mmu_idx, target_ulong size)
+ * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
+ * supplied size is only used by tlb_flush_page.
+ *
+ * Called from TCG-generated code, which is under an RCU read-side
+ * critical section.
+ */
+void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
+                             hwaddr paddr, MemTxAttrs attrs, int prot,
+                             int mmu_idx, target_ulong size)
 {
     CPUArchState *env = cpu->env_ptr;
     MemoryRegionSection *section;
@@ -258,6 +350,7 @@
     CPUTLBEntry *te;
     hwaddr iotlb, xlat, sz;
     unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
+    int asidx = cpu_asidx_from_attrs(cpu, attrs);
 
     assert(size >= TARGET_PAGE_SIZE);
     if (size != TARGET_PAGE_SIZE) {
@@ -265,15 +358,12 @@
     }
 
     sz = size;
-    section = address_space_translate_for_iotlb(cpu->as, paddr,
-                                                &xlat, &sz);
+    section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
     assert(sz >= TARGET_PAGE_SIZE);
 
-#if defined(DEBUG_TLB)
-    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
-           " prot=%x idx=%d\n",
-           vaddr, paddr, prot, mmu_idx);
-#endif
+    tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
+              " prot=%x idx=%d\n",
+              vaddr, paddr, prot, mmu_idx);
 
     address = vaddr;
     if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
@@ -297,7 +387,8 @@
     env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
 
     /* refill the tlb */
-    env->iotlb[mmu_idx][index] = iotlb - vaddr;
+    env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
+    env->iotlb[mmu_idx][index].attrs = attrs;
     te->addend = addend - vaddr;
     if (prot & PAGE_READ) {
         te->addr_read = address;
@@ -316,8 +407,8 @@
             /* Write access calls the I/O callback.  */
             te->addr_write = address | TLB_MMIO;
         } else if (memory_region_is_ram(section->mr)
-                   && cpu_physical_memory_is_clean(section->mr->ram_addr
-                                                   + xlat)) {
+                   && cpu_physical_memory_is_clean(
+                        memory_region_get_ram_addr(section->mr) + xlat)) {
             te->addr_write = address | TLB_NOTDIRTY;
         } else {
             te->addr_write = address;
@@ -327,6 +418,50 @@
     }
 }
 
+/* Add a new TLB entry, but without specifying the memory
+ * transaction attributes to be used.
+ */
+void tlb_set_page(CPUState *cpu, target_ulong vaddr,
+                  hwaddr paddr, int prot,
+                  int mmu_idx, target_ulong size)
+{
+    tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
+                            prot, mmu_idx, size);
+}
+
+static void report_bad_exec(CPUState *cpu, target_ulong addr)
+{
+    /* Accidentally executing outside RAM or ROM is quite common for
+     * several user-error situations, so report it in a way that
+     * makes it clear that this isn't a QEMU bug and provide suggestions
+     * about what a user could do to fix things.
+     */
+    error_report("Trying to execute code outside RAM or ROM at 0x"
+                 TARGET_FMT_lx, addr);
+    error_printf("This usually means one of the following happened:\n\n"
+                 "(1) You told QEMU to execute a kernel for the wrong machine "
+                 "type, and it crashed on startup (eg trying to run a "
+                 "raspberry pi kernel on a versatilepb QEMU machine)\n"
+                 "(2) You didn't give QEMU a kernel or BIOS filename at all, "
+                 "and QEMU executed a ROM full of no-op instructions until "
+                 "it fell off the end\n"
+                 "(3) Your guest kernel has a bug and crashed by jumping "
+                 "off into nowhere\n\n"
+                 "This is almost always one of the first two, so check your "
+                 "command line and that you are using the right type of kernel "
+                 "for this machine.\n"
+                 "If you think option (3) is likely then you can try debugging "
+                 "your guest with the -d debug options; in particular "
+                 "-d guest_errors will cause the log to include a dump of the "
+                 "guest register state at this point.\n\n"
+                 "Execution cannot continue; stopping here.\n\n");
+
+    /* Report also to the logs, with more detail including register dump */
+    qemu_log_mask(LOG_GUEST_ERROR, "qemu: fatal: Trying to execute code "
+                  "outside RAM or ROM at 0x" TARGET_FMT_lx "\n", addr);
+    log_cpu_state_mask(LOG_GUEST_ERROR, cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
+}
+
 /* NOTE: this function can trigger an exception */
 /* NOTE2: the returned address is not exactly the physical address: it
  * is actually a ram_addr_t (in system mode; the user mode emulation
@@ -338,29 +473,60 @@
     void *p;
     MemoryRegion *mr;
     CPUState *cpu = ENV_GET_CPU(env1);
+    CPUIOTLBEntry *iotlbentry;
 
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    mmu_idx = cpu_mmu_index(env1);
+    mmu_idx = cpu_mmu_index(env1, true);
     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
                  (addr & TARGET_PAGE_MASK))) {
         cpu_ldub_code(env1, addr);
     }
-    pd = env1->iotlb[mmu_idx][page_index] & ~TARGET_PAGE_MASK;
-    mr = iotlb_to_region(cpu->as, pd);
+    iotlbentry = &env1->iotlb[mmu_idx][page_index];
+    pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
+    mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
     if (memory_region_is_unassigned(mr)) {
         CPUClass *cc = CPU_GET_CLASS(cpu);
 
         if (cc->do_unassigned_access) {
             cc->do_unassigned_access(cpu, addr, false, true, 0, 4);
         } else {
-            cpu_abort(cpu, "Trying to execute code outside RAM or ROM at 0x"
-                      TARGET_FMT_lx "\n", addr);
+            report_bad_exec(cpu, addr);
+            exit(1);
         }
     }
     p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
     return qemu_ram_addr_from_host_nofail(p);
 }
 
+/* Return true if ADDR is present in the victim tlb, and has been copied
+   back to the main tlb.  */
+static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
+                           size_t elt_ofs, target_ulong page)
+{
+    size_t vidx;
+    for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
+        CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
+        target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
+
+        if (cmp == page) {
+            /* Found entry in victim tlb, swap tlb and iotlb.  */
+            CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
+            CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
+            CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
+
+            tmptlb = *tlb; *tlb = *vtlb; *vtlb = tmptlb;
+            tmpio = *io; *io = *vio; *vio = tmpio;
+            return true;
+        }
+    }
+    return false;
+}
+
+/* Macro to call the above, with local variables from the use context.  */
+#define VICTIM_TLB_HIT(TY, ADDR) \
+  victim_tlb_hit(env, mmu_idx, index, offsetof(CPUTLBEntry, TY), \
+                 (ADDR) & TARGET_PAGE_MASK)
+
 #define MMUSUFFIX _mmu
 
 #define SHIFT 0
diff --git a/crypto/Makefile.objs b/crypto/Makefile.objs
new file mode 100644
index 0000000..a36d2d9
--- /dev/null
+++ b/crypto/Makefile.objs
@@ -0,0 +1,33 @@
+crypto-obj-y = init.o
+crypto-obj-y += hash.o
+crypto-obj-$(CONFIG_NETTLE) += hash-nettle.o
+crypto-obj-$(if $(CONFIG_NETTLE),n,$(CONFIG_GCRYPT)) += hash-gcrypt.o
+crypto-obj-$(if $(CONFIG_NETTLE),n,$(if $(CONFIG_GCRYPT),n,y)) += hash-glib.o
+crypto-obj-y += aes.o
+crypto-obj-y += desrfb.o
+crypto-obj-y += cipher.o
+crypto-obj-y += tlscreds.o
+crypto-obj-y += tlscredsanon.o
+crypto-obj-y += tlscredsx509.o
+crypto-obj-y += tlssession.o
+crypto-obj-y += secret.o
+crypto-obj-$(CONFIG_GCRYPT) += random-gcrypt.o
+crypto-obj-$(if $(CONFIG_GCRYPT),n,$(CONFIG_GNUTLS_RND)) += random-gnutls.o
+crypto-obj-$(if $(CONFIG_GCRYPT),n,$(if $(CONFIG_GNUTLS_RND),n,y)) += random-platform.o
+crypto-obj-y += pbkdf.o
+crypto-obj-$(CONFIG_NETTLE_KDF) += pbkdf-nettle.o
+crypto-obj-$(if $(CONFIG_NETTLE_KDF),n,$(CONFIG_GCRYPT_KDF)) += pbkdf-gcrypt.o
+crypto-obj-y += ivgen.o
+crypto-obj-y += ivgen-essiv.o
+crypto-obj-y += ivgen-plain.o
+crypto-obj-y += ivgen-plain64.o
+crypto-obj-y += afsplit.o
+crypto-obj-y += xts.o
+crypto-obj-y += block.o
+crypto-obj-y += block-qcow.o
+crypto-obj-y += block-luks.o
+
+# Let the userspace emulators avoid linking gnutls/etc
+crypto-aes-obj-y = aes.o
+
+stub-obj-y += pbkdf-stub.o
diff --git a/crypto/aes.c b/crypto/aes.c
new file mode 100644
index 0000000..3456eac
--- /dev/null
+++ b/crypto/aes.c
@@ -0,0 +1,1653 @@
+/**
+ *
+ * aes.c - integrated in QEMU by Fabrice Bellard from the OpenSSL project.
+ */
+/*
+ * rijndael-alg-fst.c
+ *
+ * @version 3.0 (December 2000)
+ *
+ * Optimised ANSI C code for the Rijndael cipher (now AES)
+ *
+ * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
+ * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
+ * @author Paulo Barreto <paulo.barreto@terra.com.br>
+ *
+ * This code is hereby placed in the public domain.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
+ * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "crypto/aes.h"
+
+typedef uint32_t u32;
+typedef uint8_t u8;
+
+/* This controls loop-unrolling in aes_core.c */
+#undef FULL_UNROLL
+# define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
+# define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
+
+const uint8_t AES_sbox[256] = {
+    0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5,
+    0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
+    0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0,
+    0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
+    0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC,
+    0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
+    0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A,
+    0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
+    0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0,
+    0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
+    0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B,
+    0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
+    0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85,
+    0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
+    0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5,
+    0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
+    0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17,
+    0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
+    0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88,
+    0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
+    0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C,
+    0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
+    0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9,
+    0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
+    0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6,
+    0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
+    0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E,
+    0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
+    0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94,
+    0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
+    0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68,
+    0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16,
+};
+
+const uint8_t AES_isbox[256] = {
+    0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38,
+    0xBF, 0x40, 0xA3, 0x9E, 0x81, 0xF3, 0xD7, 0xFB,
+    0x7C, 0xE3, 0x39, 0x82, 0x9B, 0x2F, 0xFF, 0x87,
+    0x34, 0x8E, 0x43, 0x44, 0xC4, 0xDE, 0xE9, 0xCB,
+    0x54, 0x7B, 0x94, 0x32, 0xA6, 0xC2, 0x23, 0x3D,
+    0xEE, 0x4C, 0x95, 0x0B, 0x42, 0xFA, 0xC3, 0x4E,
+    0x08, 0x2E, 0xA1, 0x66, 0x28, 0xD9, 0x24, 0xB2,
+    0x76, 0x5B, 0xA2, 0x49, 0x6D, 0x8B, 0xD1, 0x25,
+    0x72, 0xF8, 0xF6, 0x64, 0x86, 0x68, 0x98, 0x16,
+    0xD4, 0xA4, 0x5C, 0xCC, 0x5D, 0x65, 0xB6, 0x92,
+    0x6C, 0x70, 0x48, 0x50, 0xFD, 0xED, 0xB9, 0xDA,
+    0x5E, 0x15, 0x46, 0x57, 0xA7, 0x8D, 0x9D, 0x84,
+    0x90, 0xD8, 0xAB, 0x00, 0x8C, 0xBC, 0xD3, 0x0A,
+    0xF7, 0xE4, 0x58, 0x05, 0xB8, 0xB3, 0x45, 0x06,
+    0xD0, 0x2C, 0x1E, 0x8F, 0xCA, 0x3F, 0x0F, 0x02,
+    0xC1, 0xAF, 0xBD, 0x03, 0x01, 0x13, 0x8A, 0x6B,
+    0x3A, 0x91, 0x11, 0x41, 0x4F, 0x67, 0xDC, 0xEA,
+    0x97, 0xF2, 0xCF, 0xCE, 0xF0, 0xB4, 0xE6, 0x73,
+    0x96, 0xAC, 0x74, 0x22, 0xE7, 0xAD, 0x35, 0x85,
+    0xE2, 0xF9, 0x37, 0xE8, 0x1C, 0x75, 0xDF, 0x6E,
+    0x47, 0xF1, 0x1A, 0x71, 0x1D, 0x29, 0xC5, 0x89,
+    0x6F, 0xB7, 0x62, 0x0E, 0xAA, 0x18, 0xBE, 0x1B,
+    0xFC, 0x56, 0x3E, 0x4B, 0xC6, 0xD2, 0x79, 0x20,
+    0x9A, 0xDB, 0xC0, 0xFE, 0x78, 0xCD, 0x5A, 0xF4,
+    0x1F, 0xDD, 0xA8, 0x33, 0x88, 0x07, 0xC7, 0x31,
+    0xB1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xEC, 0x5F,
+    0x60, 0x51, 0x7F, 0xA9, 0x19, 0xB5, 0x4A, 0x0D,
+    0x2D, 0xE5, 0x7A, 0x9F, 0x93, 0xC9, 0x9C, 0xEF,
+    0xA0, 0xE0, 0x3B, 0x4D, 0xAE, 0x2A, 0xF5, 0xB0,
+    0xC8, 0xEB, 0xBB, 0x3C, 0x83, 0x53, 0x99, 0x61,
+    0x17, 0x2B, 0x04, 0x7E, 0xBA, 0x77, 0xD6, 0x26,
+    0xE1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0C, 0x7D,
+};
+
+const uint8_t AES_shifts[16] = {
+    0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, 1, 6, 11
+};
+
+const uint8_t AES_ishifts[16] = {
+    0, 13, 10, 7, 4, 1, 14, 11, 8, 5, 2, 15, 12, 9, 6, 3
+};
+
+/* AES_imc[x][0] = [x].[0e, 09, 0d, 0b]; */
+/* AES_imc[x][1] = [x].[0b, 0e, 09, 0d]; */
+/* AES_imc[x][2] = [x].[0d, 0b, 0e, 09]; */
+/* AES_imc[x][3] = [x].[09, 0d, 0b, 0e]; */
+const uint32_t AES_imc[256][4] = {
+    { 0x00000000, 0x00000000, 0x00000000, 0x00000000, }, /* x=00 */
+    { 0x0E090D0B, 0x0B0E090D, 0x0D0B0E09, 0x090D0B0E, }, /* x=01 */
+    { 0x1C121A16, 0x161C121A, 0x1A161C12, 0x121A161C, }, /* x=02 */
+    { 0x121B171D, 0x1D121B17, 0x171D121B, 0x1B171D12, }, /* x=03 */
+    { 0x3824342C, 0x2C382434, 0x342C3824, 0x24342C38, }, /* x=04 */
+    { 0x362D3927, 0x27362D39, 0x3927362D, 0x2D392736, }, /* x=05 */
+    { 0x24362E3A, 0x3A24362E, 0x2E3A2436, 0x362E3A24, }, /* x=06 */
+    { 0x2A3F2331, 0x312A3F23, 0x23312A3F, 0x3F23312A, }, /* x=07 */
+    { 0x70486858, 0x58704868, 0x68587048, 0x48685870, }, /* x=08 */
+    { 0x7E416553, 0x537E4165, 0x65537E41, 0x4165537E, }, /* x=09 */
+    { 0x6C5A724E, 0x4E6C5A72, 0x724E6C5A, 0x5A724E6C, }, /* x=0A */
+    { 0x62537F45, 0x4562537F, 0x7F456253, 0x537F4562, }, /* x=0B */
+    { 0x486C5C74, 0x74486C5C, 0x5C74486C, 0x6C5C7448, }, /* x=0C */
+    { 0x4665517F, 0x7F466551, 0x517F4665, 0x65517F46, }, /* x=0D */
+    { 0x547E4662, 0x62547E46, 0x4662547E, 0x7E466254, }, /* x=0E */
+    { 0x5A774B69, 0x695A774B, 0x4B695A77, 0x774B695A, }, /* x=0F */
+    { 0xE090D0B0, 0xB0E090D0, 0xD0B0E090, 0x90D0B0E0, }, /* x=10 */
+    { 0xEE99DDBB, 0xBBEE99DD, 0xDDBBEE99, 0x99DDBBEE, }, /* x=11 */
+    { 0xFC82CAA6, 0xA6FC82CA, 0xCAA6FC82, 0x82CAA6FC, }, /* x=12 */
+    { 0xF28BC7AD, 0xADF28BC7, 0xC7ADF28B, 0x8BC7ADF2, }, /* x=13 */
+    { 0xD8B4E49C, 0x9CD8B4E4, 0xE49CD8B4, 0xB4E49CD8, }, /* x=14 */
+    { 0xD6BDE997, 0x97D6BDE9, 0xE997D6BD, 0xBDE997D6, }, /* x=15 */
+    { 0xC4A6FE8A, 0x8AC4A6FE, 0xFE8AC4A6, 0xA6FE8AC4, }, /* x=16 */
+    { 0xCAAFF381, 0x81CAAFF3, 0xF381CAAF, 0xAFF381CA, }, /* x=17 */
+    { 0x90D8B8E8, 0xE890D8B8, 0xB8E890D8, 0xD8B8E890, }, /* x=18 */
+    { 0x9ED1B5E3, 0xE39ED1B5, 0xB5E39ED1, 0xD1B5E39E, }, /* x=19 */
+    { 0x8CCAA2FE, 0xFE8CCAA2, 0xA2FE8CCA, 0xCAA2FE8C, }, /* x=1A */
+    { 0x82C3AFF5, 0xF582C3AF, 0xAFF582C3, 0xC3AFF582, }, /* x=1B */
+    { 0xA8FC8CC4, 0xC4A8FC8C, 0x8CC4A8FC, 0xFC8CC4A8, }, /* x=1C */
+    { 0xA6F581CF, 0xCFA6F581, 0x81CFA6F5, 0xF581CFA6, }, /* x=1D */
+    { 0xB4EE96D2, 0xD2B4EE96, 0x96D2B4EE, 0xEE96D2B4, }, /* x=1E */
+    { 0xBAE79BD9, 0xD9BAE79B, 0x9BD9BAE7, 0xE79BD9BA, }, /* x=1F */
+    { 0xDB3BBB7B, 0x7BDB3BBB, 0xBB7BDB3B, 0x3BBB7BDB, }, /* x=20 */
+    { 0xD532B670, 0x70D532B6, 0xB670D532, 0x32B670D5, }, /* x=21 */
+    { 0xC729A16D, 0x6DC729A1, 0xA16DC729, 0x29A16DC7, }, /* x=22 */
+    { 0xC920AC66, 0x66C920AC, 0xAC66C920, 0x20AC66C9, }, /* x=23 */
+    { 0xE31F8F57, 0x57E31F8F, 0x8F57E31F, 0x1F8F57E3, }, /* x=24 */
+    { 0xED16825C, 0x5CED1682, 0x825CED16, 0x16825CED, }, /* x=25 */
+    { 0xFF0D9541, 0x41FF0D95, 0x9541FF0D, 0x0D9541FF, }, /* x=26 */
+    { 0xF104984A, 0x4AF10498, 0x984AF104, 0x04984AF1, }, /* x=27 */
+    { 0xAB73D323, 0x23AB73D3, 0xD323AB73, 0x73D323AB, }, /* x=28 */
+    { 0xA57ADE28, 0x28A57ADE, 0xDE28A57A, 0x7ADE28A5, }, /* x=29 */
+    { 0xB761C935, 0x35B761C9, 0xC935B761, 0x61C935B7, }, /* x=2A */
+    { 0xB968C43E, 0x3EB968C4, 0xC43EB968, 0x68C43EB9, }, /* x=2B */
+    { 0x9357E70F, 0x0F9357E7, 0xE70F9357, 0x57E70F93, }, /* x=2C */
+    { 0x9D5EEA04, 0x049D5EEA, 0xEA049D5E, 0x5EEA049D, }, /* x=2D */
+    { 0x8F45FD19, 0x198F45FD, 0xFD198F45, 0x45FD198F, }, /* x=2E */
+    { 0x814CF012, 0x12814CF0, 0xF012814C, 0x4CF01281, }, /* x=2F */
+    { 0x3BAB6BCB, 0xCB3BAB6B, 0x6BCB3BAB, 0xAB6BCB3B, }, /* x=30 */
+    { 0x35A266C0, 0xC035A266, 0x66C035A2, 0xA266C035, }, /* x=31 */
+    { 0x27B971DD, 0xDD27B971, 0x71DD27B9, 0xB971DD27, }, /* x=32 */
+    { 0x29B07CD6, 0xD629B07C, 0x7CD629B0, 0xB07CD629, }, /* x=33 */
+    { 0x038F5FE7, 0xE7038F5F, 0x5FE7038F, 0x8F5FE703, }, /* x=34 */
+    { 0x0D8652EC, 0xEC0D8652, 0x52EC0D86, 0x8652EC0D, }, /* x=35 */
+    { 0x1F9D45F1, 0xF11F9D45, 0x45F11F9D, 0x9D45F11F, }, /* x=36 */
+    { 0x119448FA, 0xFA119448, 0x48FA1194, 0x9448FA11, }, /* x=37 */
+    { 0x4BE30393, 0x934BE303, 0x03934BE3, 0xE303934B, }, /* x=38 */
+    { 0x45EA0E98, 0x9845EA0E, 0x0E9845EA, 0xEA0E9845, }, /* x=39 */
+    { 0x57F11985, 0x8557F119, 0x198557F1, 0xF1198557, }, /* x=3A */
+    { 0x59F8148E, 0x8E59F814, 0x148E59F8, 0xF8148E59, }, /* x=3B */
+    { 0x73C737BF, 0xBF73C737, 0x37BF73C7, 0xC737BF73, }, /* x=3C */
+    { 0x7DCE3AB4, 0xB47DCE3A, 0x3AB47DCE, 0xCE3AB47D, }, /* x=3D */
+    { 0x6FD52DA9, 0xA96FD52D, 0x2DA96FD5, 0xD52DA96F, }, /* x=3E */
+    { 0x61DC20A2, 0xA261DC20, 0x20A261DC, 0xDC20A261, }, /* x=3F */
+    { 0xAD766DF6, 0xF6AD766D, 0x6DF6AD76, 0x766DF6AD, }, /* x=40 */
+    { 0xA37F60FD, 0xFDA37F60, 0x60FDA37F, 0x7F60FDA3, }, /* x=41 */
+    { 0xB16477E0, 0xE0B16477, 0x77E0B164, 0x6477E0B1, }, /* x=42 */
+    { 0xBF6D7AEB, 0xEBBF6D7A, 0x7AEBBF6D, 0x6D7AEBBF, }, /* x=43 */
+    { 0x955259DA, 0xDA955259, 0x59DA9552, 0x5259DA95, }, /* x=44 */
+    { 0x9B5B54D1, 0xD19B5B54, 0x54D19B5B, 0x5B54D19B, }, /* x=45 */
+    { 0x894043CC, 0xCC894043, 0x43CC8940, 0x4043CC89, }, /* x=46 */
+    { 0x87494EC7, 0xC787494E, 0x4EC78749, 0x494EC787, }, /* x=47 */
+    { 0xDD3E05AE, 0xAEDD3E05, 0x05AEDD3E, 0x3E05AEDD, }, /* x=48 */
+    { 0xD33708A5, 0xA5D33708, 0x08A5D337, 0x3708A5D3, }, /* x=49 */
+    { 0xC12C1FB8, 0xB8C12C1F, 0x1FB8C12C, 0x2C1FB8C1, }, /* x=4A */
+    { 0xCF2512B3, 0xB3CF2512, 0x12B3CF25, 0x2512B3CF, }, /* x=4B */
+    { 0xE51A3182, 0x82E51A31, 0x3182E51A, 0x1A3182E5, }, /* x=4C */
+    { 0xEB133C89, 0x89EB133C, 0x3C89EB13, 0x133C89EB, }, /* x=4D */
+    { 0xF9082B94, 0x94F9082B, 0x2B94F908, 0x082B94F9, }, /* x=4E */
+    { 0xF701269F, 0x9FF70126, 0x269FF701, 0x01269FF7, }, /* x=4F */
+    { 0x4DE6BD46, 0x464DE6BD, 0xBD464DE6, 0xE6BD464D, }, /* x=50 */
+    { 0x43EFB04D, 0x4D43EFB0, 0xB04D43EF, 0xEFB04D43, }, /* x=51 */
+    { 0x51F4A750, 0x5051F4A7, 0xA75051F4, 0xF4A75051, }, /* x=52 */
+    { 0x5FFDAA5B, 0x5B5FFDAA, 0xAA5B5FFD, 0xFDAA5B5F, }, /* x=53 */
+    { 0x75C2896A, 0x6A75C289, 0x896A75C2, 0xC2896A75, }, /* x=54 */
+    { 0x7BCB8461, 0x617BCB84, 0x84617BCB, 0xCB84617B, }, /* x=55 */
+    { 0x69D0937C, 0x7C69D093, 0x937C69D0, 0xD0937C69, }, /* x=56 */
+    { 0x67D99E77, 0x7767D99E, 0x9E7767D9, 0xD99E7767, }, /* x=57 */
+    { 0x3DAED51E, 0x1E3DAED5, 0xD51E3DAE, 0xAED51E3D, }, /* x=58 */
+    { 0x33A7D815, 0x1533A7D8, 0xD81533A7, 0xA7D81533, }, /* x=59 */
+    { 0x21BCCF08, 0x0821BCCF, 0xCF0821BC, 0xBCCF0821, }, /* x=5A */
+    { 0x2FB5C203, 0x032FB5C2, 0xC2032FB5, 0xB5C2032F, }, /* x=5B */
+    { 0x058AE132, 0x32058AE1, 0xE132058A, 0x8AE13205, }, /* x=5C */
+    { 0x0B83EC39, 0x390B83EC, 0xEC390B83, 0x83EC390B, }, /* x=5D */
+    { 0x1998FB24, 0x241998FB, 0xFB241998, 0x98FB2419, }, /* x=5E */
+    { 0x1791F62F, 0x2F1791F6, 0xF62F1791, 0x91F62F17, }, /* x=5F */
+    { 0x764DD68D, 0x8D764DD6, 0xD68D764D, 0x4DD68D76, }, /* x=60 */
+    { 0x7844DB86, 0x867844DB, 0xDB867844, 0x44DB8678, }, /* x=61 */
+    { 0x6A5FCC9B, 0x9B6A5FCC, 0xCC9B6A5F, 0x5FCC9B6A, }, /* x=62 */
+    { 0x6456C190, 0x906456C1, 0xC1906456, 0x56C19064, }, /* x=63 */
+    { 0x4E69E2A1, 0xA14E69E2, 0xE2A14E69, 0x69E2A14E, }, /* x=64 */
+    { 0x4060EFAA, 0xAA4060EF, 0xEFAA4060, 0x60EFAA40, }, /* x=65 */
+    { 0x527BF8B7, 0xB7527BF8, 0xF8B7527B, 0x7BF8B752, }, /* x=66 */
+    { 0x5C72F5BC, 0xBC5C72F5, 0xF5BC5C72, 0x72F5BC5C, }, /* x=67 */
+    { 0x0605BED5, 0xD50605BE, 0xBED50605, 0x05BED506, }, /* x=68 */
+    { 0x080CB3DE, 0xDE080CB3, 0xB3DE080C, 0x0CB3DE08, }, /* x=69 */
+    { 0x1A17A4C3, 0xC31A17A4, 0xA4C31A17, 0x17A4C31A, }, /* x=6A */
+    { 0x141EA9C8, 0xC8141EA9, 0xA9C8141E, 0x1EA9C814, }, /* x=6B */
+    { 0x3E218AF9, 0xF93E218A, 0x8AF93E21, 0x218AF93E, }, /* x=6C */
+    { 0x302887F2, 0xF2302887, 0x87F23028, 0x2887F230, }, /* x=6D */
+    { 0x223390EF, 0xEF223390, 0x90EF2233, 0x3390EF22, }, /* x=6E */
+    { 0x2C3A9DE4, 0xE42C3A9D, 0x9DE42C3A, 0x3A9DE42C, }, /* x=6F */
+    { 0x96DD063D, 0x3D96DD06, 0x063D96DD, 0xDD063D96, }, /* x=70 */
+    { 0x98D40B36, 0x3698D40B, 0x0B3698D4, 0xD40B3698, }, /* x=71 */
+    { 0x8ACF1C2B, 0x2B8ACF1C, 0x1C2B8ACF, 0xCF1C2B8A, }, /* x=72 */
+    { 0x84C61120, 0x2084C611, 0x112084C6, 0xC6112084, }, /* x=73 */
+    { 0xAEF93211, 0x11AEF932, 0x3211AEF9, 0xF93211AE, }, /* x=74 */
+    { 0xA0F03F1A, 0x1AA0F03F, 0x3F1AA0F0, 0xF03F1AA0, }, /* x=75 */
+    { 0xB2EB2807, 0x07B2EB28, 0x2807B2EB, 0xEB2807B2, }, /* x=76 */
+    { 0xBCE2250C, 0x0CBCE225, 0x250CBCE2, 0xE2250CBC, }, /* x=77 */
+    { 0xE6956E65, 0x65E6956E, 0x6E65E695, 0x956E65E6, }, /* x=78 */
+    { 0xE89C636E, 0x6EE89C63, 0x636EE89C, 0x9C636EE8, }, /* x=79 */
+    { 0xFA877473, 0x73FA8774, 0x7473FA87, 0x877473FA, }, /* x=7A */
+    { 0xF48E7978, 0x78F48E79, 0x7978F48E, 0x8E7978F4, }, /* x=7B */
+    { 0xDEB15A49, 0x49DEB15A, 0x5A49DEB1, 0xB15A49DE, }, /* x=7C */
+    { 0xD0B85742, 0x42D0B857, 0x5742D0B8, 0xB85742D0, }, /* x=7D */
+    { 0xC2A3405F, 0x5FC2A340, 0x405FC2A3, 0xA3405FC2, }, /* x=7E */
+    { 0xCCAA4D54, 0x54CCAA4D, 0x4D54CCAA, 0xAA4D54CC, }, /* x=7F */
+    { 0x41ECDAF7, 0xF741ECDA, 0xDAF741EC, 0xECDAF741, }, /* x=80 */
+    { 0x4FE5D7FC, 0xFC4FE5D7, 0xD7FC4FE5, 0xE5D7FC4F, }, /* x=81 */
+    { 0x5DFEC0E1, 0xE15DFEC0, 0xC0E15DFE, 0xFEC0E15D, }, /* x=82 */
+    { 0x53F7CDEA, 0xEA53F7CD, 0xCDEA53F7, 0xF7CDEA53, }, /* x=83 */
+    { 0x79C8EEDB, 0xDB79C8EE, 0xEEDB79C8, 0xC8EEDB79, }, /* x=84 */
+    { 0x77C1E3D0, 0xD077C1E3, 0xE3D077C1, 0xC1E3D077, }, /* x=85 */
+    { 0x65DAF4CD, 0xCD65DAF4, 0xF4CD65DA, 0xDAF4CD65, }, /* x=86 */
+    { 0x6BD3F9C6, 0xC66BD3F9, 0xF9C66BD3, 0xD3F9C66B, }, /* x=87 */
+    { 0x31A4B2AF, 0xAF31A4B2, 0xB2AF31A4, 0xA4B2AF31, }, /* x=88 */
+    { 0x3FADBFA4, 0xA43FADBF, 0xBFA43FAD, 0xADBFA43F, }, /* x=89 */
+    { 0x2DB6A8B9, 0xB92DB6A8, 0xA8B92DB6, 0xB6A8B92D, }, /* x=8A */
+    { 0x23BFA5B2, 0xB223BFA5, 0xA5B223BF, 0xBFA5B223, }, /* x=8B */
+    { 0x09808683, 0x83098086, 0x86830980, 0x80868309, }, /* x=8C */
+    { 0x07898B88, 0x8807898B, 0x8B880789, 0x898B8807, }, /* x=8D */
+    { 0x15929C95, 0x9515929C, 0x9C951592, 0x929C9515, }, /* x=8E */
+    { 0x1B9B919E, 0x9E1B9B91, 0x919E1B9B, 0x9B919E1B, }, /* x=8F */
+    { 0xA17C0A47, 0x47A17C0A, 0x0A47A17C, 0x7C0A47A1, }, /* x=90 */
+    { 0xAF75074C, 0x4CAF7507, 0x074CAF75, 0x75074CAF, }, /* x=91 */
+    { 0xBD6E1051, 0x51BD6E10, 0x1051BD6E, 0x6E1051BD, }, /* x=92 */
+    { 0xB3671D5A, 0x5AB3671D, 0x1D5AB367, 0x671D5AB3, }, /* x=93 */
+    { 0x99583E6B, 0x6B99583E, 0x3E6B9958, 0x583E6B99, }, /* x=94 */
+    { 0x97513360, 0x60975133, 0x33609751, 0x51336097, }, /* x=95 */
+    { 0x854A247D, 0x7D854A24, 0x247D854A, 0x4A247D85, }, /* x=96 */
+    { 0x8B432976, 0x768B4329, 0x29768B43, 0x4329768B, }, /* x=97 */
+    { 0xD134621F, 0x1FD13462, 0x621FD134, 0x34621FD1, }, /* x=98 */
+    { 0xDF3D6F14, 0x14DF3D6F, 0x6F14DF3D, 0x3D6F14DF, }, /* x=99 */
+    { 0xCD267809, 0x09CD2678, 0x7809CD26, 0x267809CD, }, /* x=9A */
+    { 0xC32F7502, 0x02C32F75, 0x7502C32F, 0x2F7502C3, }, /* x=9B */
+    { 0xE9105633, 0x33E91056, 0x5633E910, 0x105633E9, }, /* x=9C */
+    { 0xE7195B38, 0x38E7195B, 0x5B38E719, 0x195B38E7, }, /* x=9D */
+    { 0xF5024C25, 0x25F5024C, 0x4C25F502, 0x024C25F5, }, /* x=9E */
+    { 0xFB0B412E, 0x2EFB0B41, 0x412EFB0B, 0x0B412EFB, }, /* x=9F */
+    { 0x9AD7618C, 0x8C9AD761, 0x618C9AD7, 0xD7618C9A, }, /* x=A0 */
+    { 0x94DE6C87, 0x8794DE6C, 0x6C8794DE, 0xDE6C8794, }, /* x=A1 */
+    { 0x86C57B9A, 0x9A86C57B, 0x7B9A86C5, 0xC57B9A86, }, /* x=A2 */
+    { 0x88CC7691, 0x9188CC76, 0x769188CC, 0xCC769188, }, /* x=A3 */
+    { 0xA2F355A0, 0xA0A2F355, 0x55A0A2F3, 0xF355A0A2, }, /* x=A4 */
+    { 0xACFA58AB, 0xABACFA58, 0x58ABACFA, 0xFA58ABAC, }, /* x=A5 */
+    { 0xBEE14FB6, 0xB6BEE14F, 0x4FB6BEE1, 0xE14FB6BE, }, /* x=A6 */
+    { 0xB0E842BD, 0xBDB0E842, 0x42BDB0E8, 0xE842BDB0, }, /* x=A7 */
+    { 0xEA9F09D4, 0xD4EA9F09, 0x09D4EA9F, 0x9F09D4EA, }, /* x=A8 */
+    { 0xE49604DF, 0xDFE49604, 0x04DFE496, 0x9604DFE4, }, /* x=A9 */
+    { 0xF68D13C2, 0xC2F68D13, 0x13C2F68D, 0x8D13C2F6, }, /* x=AA */
+    { 0xF8841EC9, 0xC9F8841E, 0x1EC9F884, 0x841EC9F8, }, /* x=AB */
+    { 0xD2BB3DF8, 0xF8D2BB3D, 0x3DF8D2BB, 0xBB3DF8D2, }, /* x=AC */
+    { 0xDCB230F3, 0xF3DCB230, 0x30F3DCB2, 0xB230F3DC, }, /* x=AD */
+    { 0xCEA927EE, 0xEECEA927, 0x27EECEA9, 0xA927EECE, }, /* x=AE */
+    { 0xC0A02AE5, 0xE5C0A02A, 0x2AE5C0A0, 0xA02AE5C0, }, /* x=AF */
+    { 0x7A47B13C, 0x3C7A47B1, 0xB13C7A47, 0x47B13C7A, }, /* x=B0 */
+    { 0x744EBC37, 0x37744EBC, 0xBC37744E, 0x4EBC3774, }, /* x=B1 */
+    { 0x6655AB2A, 0x2A6655AB, 0xAB2A6655, 0x55AB2A66, }, /* x=B2 */
+    { 0x685CA621, 0x21685CA6, 0xA621685C, 0x5CA62168, }, /* x=B3 */
+    { 0x42638510, 0x10426385, 0x85104263, 0x63851042, }, /* x=B4 */
+    { 0x4C6A881B, 0x1B4C6A88, 0x881B4C6A, 0x6A881B4C, }, /* x=B5 */
+    { 0x5E719F06, 0x065E719F, 0x9F065E71, 0x719F065E, }, /* x=B6 */
+    { 0x5078920D, 0x0D507892, 0x920D5078, 0x78920D50, }, /* x=B7 */
+    { 0x0A0FD964, 0x640A0FD9, 0xD9640A0F, 0x0FD9640A, }, /* x=B8 */
+    { 0x0406D46F, 0x6F0406D4, 0xD46F0406, 0x06D46F04, }, /* x=B9 */
+    { 0x161DC372, 0x72161DC3, 0xC372161D, 0x1DC37216, }, /* x=BA */
+    { 0x1814CE79, 0x791814CE, 0xCE791814, 0x14CE7918, }, /* x=BB */
+    { 0x322BED48, 0x48322BED, 0xED48322B, 0x2BED4832, }, /* x=BC */
+    { 0x3C22E043, 0x433C22E0, 0xE0433C22, 0x22E0433C, }, /* x=BD */
+    { 0x2E39F75E, 0x5E2E39F7, 0xF75E2E39, 0x39F75E2E, }, /* x=BE */
+    { 0x2030FA55, 0x552030FA, 0xFA552030, 0x30FA5520, }, /* x=BF */
+    { 0xEC9AB701, 0x01EC9AB7, 0xB701EC9A, 0x9AB701EC, }, /* x=C0 */
+    { 0xE293BA0A, 0x0AE293BA, 0xBA0AE293, 0x93BA0AE2, }, /* x=C1 */
+    { 0xF088AD17, 0x17F088AD, 0xAD17F088, 0x88AD17F0, }, /* x=C2 */
+    { 0xFE81A01C, 0x1CFE81A0, 0xA01CFE81, 0x81A01CFE, }, /* x=C3 */
+    { 0xD4BE832D, 0x2DD4BE83, 0x832DD4BE, 0xBE832DD4, }, /* x=C4 */
+    { 0xDAB78E26, 0x26DAB78E, 0x8E26DAB7, 0xB78E26DA, }, /* x=C5 */
+    { 0xC8AC993B, 0x3BC8AC99, 0x993BC8AC, 0xAC993BC8, }, /* x=C6 */
+    { 0xC6A59430, 0x30C6A594, 0x9430C6A5, 0xA59430C6, }, /* x=C7 */
+    { 0x9CD2DF59, 0x599CD2DF, 0xDF599CD2, 0xD2DF599C, }, /* x=C8 */
+    { 0x92DBD252, 0x5292DBD2, 0xD25292DB, 0xDBD25292, }, /* x=C9 */
+    { 0x80C0C54F, 0x4F80C0C5, 0xC54F80C0, 0xC0C54F80, }, /* x=CA */
+    { 0x8EC9C844, 0x448EC9C8, 0xC8448EC9, 0xC9C8448E, }, /* x=CB */
+    { 0xA4F6EB75, 0x75A4F6EB, 0xEB75A4F6, 0xF6EB75A4, }, /* x=CC */
+    { 0xAAFFE67E, 0x7EAAFFE6, 0xE67EAAFF, 0xFFE67EAA, }, /* x=CD */
+    { 0xB8E4F163, 0x63B8E4F1, 0xF163B8E4, 0xE4F163B8, }, /* x=CE */
+    { 0xB6EDFC68, 0x68B6EDFC, 0xFC68B6ED, 0xEDFC68B6, }, /* x=CF */
+    { 0x0C0A67B1, 0xB10C0A67, 0x67B10C0A, 0x0A67B10C, }, /* x=D0 */
+    { 0x02036ABA, 0xBA02036A, 0x6ABA0203, 0x036ABA02, }, /* x=D1 */
+    { 0x10187DA7, 0xA710187D, 0x7DA71018, 0x187DA710, }, /* x=D2 */
+    { 0x1E1170AC, 0xAC1E1170, 0x70AC1E11, 0x1170AC1E, }, /* x=D3 */
+    { 0x342E539D, 0x9D342E53, 0x539D342E, 0x2E539D34, }, /* x=D4 */
+    { 0x3A275E96, 0x963A275E, 0x5E963A27, 0x275E963A, }, /* x=D5 */
+    { 0x283C498B, 0x8B283C49, 0x498B283C, 0x3C498B28, }, /* x=D6 */
+    { 0x26354480, 0x80263544, 0x44802635, 0x35448026, }, /* x=D7 */
+    { 0x7C420FE9, 0xE97C420F, 0x0FE97C42, 0x420FE97C, }, /* x=D8 */
+    { 0x724B02E2, 0xE2724B02, 0x02E2724B, 0x4B02E272, }, /* x=D9 */
+    { 0x605015FF, 0xFF605015, 0x15FF6050, 0x5015FF60, }, /* x=DA */
+    { 0x6E5918F4, 0xF46E5918, 0x18F46E59, 0x5918F46E, }, /* x=DB */
+    { 0x44663BC5, 0xC544663B, 0x3BC54466, 0x663BC544, }, /* x=DC */
+    { 0x4A6F36CE, 0xCE4A6F36, 0x36CE4A6F, 0x6F36CE4A, }, /* x=DD */
+    { 0x587421D3, 0xD3587421, 0x21D35874, 0x7421D358, }, /* x=DE */
+    { 0x567D2CD8, 0xD8567D2C, 0x2CD8567D, 0x7D2CD856, }, /* x=DF */
+    { 0x37A10C7A, 0x7A37A10C, 0x0C7A37A1, 0xA10C7A37, }, /* x=E0 */
+    { 0x39A80171, 0x7139A801, 0x017139A8, 0xA8017139, }, /* x=E1 */
+    { 0x2BB3166C, 0x6C2BB316, 0x166C2BB3, 0xB3166C2B, }, /* x=E2 */
+    { 0x25BA1B67, 0x6725BA1B, 0x1B6725BA, 0xBA1B6725, }, /* x=E3 */
+    { 0x0F853856, 0x560F8538, 0x38560F85, 0x8538560F, }, /* x=E4 */
+    { 0x018C355D, 0x5D018C35, 0x355D018C, 0x8C355D01, }, /* x=E5 */
+    { 0x13972240, 0x40139722, 0x22401397, 0x97224013, }, /* x=E6 */
+    { 0x1D9E2F4B, 0x4B1D9E2F, 0x2F4B1D9E, 0x9E2F4B1D, }, /* x=E7 */
+    { 0x47E96422, 0x2247E964, 0x642247E9, 0xE9642247, }, /* x=E8 */
+    { 0x49E06929, 0x2949E069, 0x692949E0, 0xE0692949, }, /* x=E9 */
+    { 0x5BFB7E34, 0x345BFB7E, 0x7E345BFB, 0xFB7E345B, }, /* x=EA */
+    { 0x55F2733F, 0x3F55F273, 0x733F55F2, 0xF2733F55, }, /* x=EB */
+    { 0x7FCD500E, 0x0E7FCD50, 0x500E7FCD, 0xCD500E7F, }, /* x=EC */
+    { 0x71C45D05, 0x0571C45D, 0x5D0571C4, 0xC45D0571, }, /* x=ED */
+    { 0x63DF4A18, 0x1863DF4A, 0x4A1863DF, 0xDF4A1863, }, /* x=EE */
+    { 0x6DD64713, 0x136DD647, 0x47136DD6, 0xD647136D, }, /* x=EF */
+    { 0xD731DCCA, 0xCAD731DC, 0xDCCAD731, 0x31DCCAD7, }, /* x=F0 */
+    { 0xD938D1C1, 0xC1D938D1, 0xD1C1D938, 0x38D1C1D9, }, /* x=F1 */
+    { 0xCB23C6DC, 0xDCCB23C6, 0xC6DCCB23, 0x23C6DCCB, }, /* x=F2 */
+    { 0xC52ACBD7, 0xD7C52ACB, 0xCBD7C52A, 0x2ACBD7C5, }, /* x=F3 */
+    { 0xEF15E8E6, 0xE6EF15E8, 0xE8E6EF15, 0x15E8E6EF, }, /* x=F4 */
+    { 0xE11CE5ED, 0xEDE11CE5, 0xE5EDE11C, 0x1CE5EDE1, }, /* x=F5 */
+    { 0xF307F2F0, 0xF0F307F2, 0xF2F0F307, 0x07F2F0F3, }, /* x=F6 */
+    { 0xFD0EFFFB, 0xFBFD0EFF, 0xFFFBFD0E, 0x0EFFFBFD, }, /* x=F7 */
+    { 0xA779B492, 0x92A779B4, 0xB492A779, 0x79B492A7, }, /* x=F8 */
+    { 0xA970B999, 0x99A970B9, 0xB999A970, 0x70B999A9, }, /* x=F9 */
+    { 0xBB6BAE84, 0x84BB6BAE, 0xAE84BB6B, 0x6BAE84BB, }, /* x=FA */
+    { 0xB562A38F, 0x8FB562A3, 0xA38FB562, 0x62A38FB5, }, /* x=FB */
+    { 0x9F5D80BE, 0xBE9F5D80, 0x80BE9F5D, 0x5D80BE9F, }, /* x=FC */
+    { 0x91548DB5, 0xB591548D, 0x8DB59154, 0x548DB591, }, /* x=FD */
+    { 0x834F9AA8, 0xA8834F9A, 0x9AA8834F, 0x4F9AA883, }, /* x=FE */
+    { 0x8D4697A3, 0xA38D4697, 0x97A38D46, 0x4697A38D, }, /* x=FF */
+};
+
+
+
+/*
+AES_Te0[x] = S [x].[02, 01, 01, 03];
+AES_Te1[x] = S [x].[03, 02, 01, 01];
+AES_Te2[x] = S [x].[01, 03, 02, 01];
+AES_Te3[x] = S [x].[01, 01, 03, 02];
+AES_Te4[x] = S [x].[01, 01, 01, 01];
+
+AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
+AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
+AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
+AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
+AES_Td4[x] = Si[x].[01, 01, 01, 01];
+*/
+
+const uint32_t AES_Te0[256] = {
+    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
+    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
+    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
+    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
+    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
+    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
+    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
+    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
+    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
+    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
+    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
+    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
+    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
+    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
+    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
+    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
+    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
+    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
+    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
+    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
+    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
+    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
+    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
+    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
+    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
+    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
+    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
+    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
+    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
+    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
+    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
+    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
+    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
+    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
+    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
+    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
+    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
+    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
+    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
+    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
+    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
+    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
+    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
+    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
+    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
+    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
+    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
+    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
+    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
+    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
+    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
+    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
+    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
+    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
+    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
+    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
+    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
+    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
+    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
+    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
+    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
+    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
+    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
+    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
+};
+const uint32_t AES_Te1[256] = {
+    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
+    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
+    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
+    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
+    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
+    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
+    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
+    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
+    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
+    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
+    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
+    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
+    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
+    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
+    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
+    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
+    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
+    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
+    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
+    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
+    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
+    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
+    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
+    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
+    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
+    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
+    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
+    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
+    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
+    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
+    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
+    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
+    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
+    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
+    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
+    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
+    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
+    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
+    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
+    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
+    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
+    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
+    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
+    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
+    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
+    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
+    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
+    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
+    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
+    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
+    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
+    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
+    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
+    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
+    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
+    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
+    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
+    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
+    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
+    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
+    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
+    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
+    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
+    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
+};
+const uint32_t AES_Te2[256] = {
+    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
+    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
+    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
+    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
+    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
+    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
+    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
+    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
+    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
+    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
+    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
+    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
+    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
+    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
+    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
+    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
+    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
+    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
+    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
+    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
+    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
+    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
+    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
+    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
+    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
+    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
+    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
+    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
+    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
+    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
+    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
+    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
+    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
+    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
+    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
+    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
+    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
+    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
+    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
+    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
+    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
+    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
+    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
+    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
+    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
+    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
+    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
+    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
+    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
+    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
+    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
+    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
+    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
+    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
+    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
+    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
+    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
+    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
+    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
+    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
+    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
+    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
+    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
+    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
+};
+const uint32_t AES_Te3[256] = {
+
+    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
+    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
+    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
+    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
+    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
+    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
+    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
+    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
+    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
+    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
+    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
+    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
+    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
+    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
+    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
+    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
+    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
+    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
+    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
+    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
+    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
+    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
+    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
+    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
+    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
+    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
+    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
+    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
+    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
+    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
+    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
+    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
+    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
+    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
+    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
+    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
+    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
+    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
+    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
+    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
+    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
+    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
+    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
+    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
+    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
+    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
+    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
+    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
+    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
+    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
+    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
+    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
+    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
+    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
+    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
+    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
+    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
+    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
+    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
+    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
+    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
+    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
+    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
+    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
+};
+const uint32_t AES_Te4[256] = {
+    0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
+    0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
+    0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
+    0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
+    0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
+    0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
+    0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
+    0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
+    0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
+    0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
+    0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
+    0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
+    0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
+    0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
+    0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
+    0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
+    0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
+    0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
+    0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
+    0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
+    0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
+    0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
+    0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
+    0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
+    0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
+    0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
+    0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
+    0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
+    0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
+    0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
+    0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
+    0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
+    0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
+    0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
+    0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
+    0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
+    0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
+    0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
+    0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
+    0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
+    0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
+    0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
+    0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
+    0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
+    0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
+    0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
+    0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
+    0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
+    0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
+    0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
+    0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
+    0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
+    0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
+    0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
+    0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
+    0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
+    0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
+    0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
+    0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
+    0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
+    0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
+    0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
+    0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
+    0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
+};
+const uint32_t AES_Td0[256] = {
+    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
+    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
+    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
+    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
+    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
+    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
+    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
+    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
+    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
+    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
+    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
+    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
+    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
+    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
+    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
+    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
+    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
+    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
+    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
+    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
+    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
+    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
+    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
+    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
+    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
+    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
+    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
+    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
+    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
+    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
+    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
+    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
+    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
+    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
+    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
+    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
+    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
+    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
+    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
+    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
+    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
+    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
+    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
+    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
+    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
+    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
+    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
+    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
+    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
+    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
+    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
+    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
+    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
+    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
+    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
+    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
+    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
+    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
+    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
+    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
+    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
+    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
+    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
+    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
+};
+const uint32_t AES_Td1[256] = {
+    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
+    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
+    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
+    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
+    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
+    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
+    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
+    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
+    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
+    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
+    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
+    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
+    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
+    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
+    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
+    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
+    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
+    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
+    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
+    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
+    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
+    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
+    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
+    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
+    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
+    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
+    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
+    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
+    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
+    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
+    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
+    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
+    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
+    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
+    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
+    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
+    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
+    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
+    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
+    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
+    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
+    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
+    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
+    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
+    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
+    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
+    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
+    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
+    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
+    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
+    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
+    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
+    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
+    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
+    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
+    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
+    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
+    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
+    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
+    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
+    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
+    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
+    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
+    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
+};
+const uint32_t AES_Td2[256] = {
+    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
+    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
+    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
+    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
+    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
+    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
+    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
+    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
+    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
+    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
+    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
+    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
+    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
+    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
+    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
+    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
+    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
+    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
+    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
+    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
+
+    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
+    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
+    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
+    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
+    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
+    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
+    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
+    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
+    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
+    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
+    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
+    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
+    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
+    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
+    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
+    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
+    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
+    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
+    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
+    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
+    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
+    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
+    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
+    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
+    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
+    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
+    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
+    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
+    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
+    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
+    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
+    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
+    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
+    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
+    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
+    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
+    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
+    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
+    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
+    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
+    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
+    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
+    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
+    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
+};
+const uint32_t AES_Td3[256] = {
+    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
+    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
+    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
+    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
+    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
+    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
+    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
+    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
+    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
+    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
+    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
+    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
+    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
+    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
+    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
+    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
+    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
+    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
+    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
+    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
+    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
+    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
+    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
+    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
+    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
+    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
+    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
+    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
+    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
+    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
+    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
+    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
+    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
+    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
+    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
+    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
+    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
+    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
+    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
+    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
+    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
+    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
+    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
+    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
+    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
+    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
+    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
+    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
+    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
+    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
+    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
+    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
+    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
+    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
+    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
+    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
+    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
+    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
+    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
+    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
+    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
+    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
+    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
+    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
+};
+const uint32_t AES_Td4[256] = {
+    0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U,
+    0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U,
+    0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU,
+    0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU,
+    0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U,
+    0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U,
+    0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U,
+    0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU,
+    0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U,
+    0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU,
+    0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU,
+    0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU,
+    0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U,
+    0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U,
+    0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U,
+    0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U,
+    0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U,
+    0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
+    0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU,
+    0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U,
+    0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U,
+    0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU,
+    0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U,
+    0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U,
+    0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U,
+    0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU,
+    0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U,
+    0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U,
+    0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU,
+    0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U,
+    0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U,
+    0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU,
+    0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U,
+    0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU,
+    0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU,
+    0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U,
+    0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U,
+    0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U,
+    0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U,
+    0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU,
+    0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U,
+    0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U,
+    0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU,
+    0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU,
+    0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU,
+    0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U,
+    0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU,
+    0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U,
+    0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U,
+    0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U,
+    0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U,
+    0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU,
+    0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U,
+    0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU,
+    0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU,
+    0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU,
+    0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU,
+    0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U,
+    0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU,
+    0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
+    0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU,
+    0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U,
+    0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U,
+    0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU,
+};
+static const u32 rcon[] = {
+	0x01000000, 0x02000000, 0x04000000, 0x08000000,
+	0x10000000, 0x20000000, 0x40000000, 0x80000000,
+	0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
+};
+
+/**
+ * Expand the cipher key into the encryption key schedule.
+ */
+int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
+			AES_KEY *key) {
+
+	u32 *rk;
+   	int i = 0;
+	u32 temp;
+
+	if (!userKey || !key)
+		return -1;
+	if (bits != 128 && bits != 192 && bits != 256)
+		return -2;
+
+	rk = key->rd_key;
+
+	if (bits==128)
+		key->rounds = 10;
+	else if (bits==192)
+		key->rounds = 12;
+	else
+		key->rounds = 14;
+
+	rk[0] = GETU32(userKey     );
+	rk[1] = GETU32(userKey +  4);
+	rk[2] = GETU32(userKey +  8);
+	rk[3] = GETU32(userKey + 12);
+	if (bits == 128) {
+		while (1) {
+			temp  = rk[3];
+			rk[4] = rk[0] ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
+				rcon[i];
+			rk[5] = rk[1] ^ rk[4];
+			rk[6] = rk[2] ^ rk[5];
+			rk[7] = rk[3] ^ rk[6];
+			if (++i == 10) {
+				return 0;
+			}
+			rk += 4;
+		}
+	}
+	rk[4] = GETU32(userKey + 16);
+	rk[5] = GETU32(userKey + 20);
+	if (bits == 192) {
+		while (1) {
+			temp = rk[ 5];
+			rk[ 6] = rk[ 0] ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
+				rcon[i];
+			rk[ 7] = rk[ 1] ^ rk[ 6];
+			rk[ 8] = rk[ 2] ^ rk[ 7];
+			rk[ 9] = rk[ 3] ^ rk[ 8];
+			if (++i == 8) {
+				return 0;
+			}
+			rk[10] = rk[ 4] ^ rk[ 9];
+			rk[11] = rk[ 5] ^ rk[10];
+			rk += 6;
+		}
+	}
+	rk[6] = GETU32(userKey + 24);
+	rk[7] = GETU32(userKey + 28);
+	if (bits == 256) {
+		while (1) {
+			temp = rk[ 7];
+			rk[ 8] = rk[ 0] ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
+				rcon[i];
+			rk[ 9] = rk[ 1] ^ rk[ 8];
+			rk[10] = rk[ 2] ^ rk[ 9];
+			rk[11] = rk[ 3] ^ rk[10];
+			if (++i == 7) {
+				return 0;
+			}
+			temp = rk[11];
+			rk[12] = rk[ 4] ^
+                                (AES_Te4[(temp >> 24)       ] & 0xff000000) ^
+                                (AES_Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^
+                                (AES_Te4[(temp >>  8) & 0xff] & 0x0000ff00) ^
+                                (AES_Te4[(temp      ) & 0xff] & 0x000000ff);
+			rk[13] = rk[ 5] ^ rk[12];
+			rk[14] = rk[ 6] ^ rk[13];
+			rk[15] = rk[ 7] ^ rk[14];
+
+			rk += 8;
+        	}
+	}
+        abort();
+}
+
+/**
+ * Expand the cipher key into the decryption key schedule.
+ */
+int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
+			 AES_KEY *key) {
+
+        u32 *rk;
+	int i, j, status;
+	u32 temp;
+
+	/* first, start with an encryption schedule */
+	status = AES_set_encrypt_key(userKey, bits, key);
+	if (status < 0)
+		return status;
+
+	rk = key->rd_key;
+
+	/* invert the order of the round keys: */
+	for (i = 0, j = 4*(key->rounds); i < j; i += 4, j -= 4) {
+		temp = rk[i    ]; rk[i    ] = rk[j    ]; rk[j    ] = temp;
+		temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
+		temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
+		temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
+	}
+	/* apply the inverse MixColumn transform to all round keys but the first and the last: */
+	for (i = 1; i < (key->rounds); i++) {
+		rk += 4;
+		rk[0] =
+                        AES_Td0[AES_Te4[(rk[0] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[0] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[0] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[0]      ) & 0xff] & 0xff];
+		rk[1] =
+                        AES_Td0[AES_Te4[(rk[1] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[1] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[1] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[1]      ) & 0xff] & 0xff];
+		rk[2] =
+                        AES_Td0[AES_Te4[(rk[2] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[2] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[2] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[2]      ) & 0xff] & 0xff];
+		rk[3] =
+                        AES_Td0[AES_Te4[(rk[3] >> 24)       ] & 0xff] ^
+                        AES_Td1[AES_Te4[(rk[3] >> 16) & 0xff] & 0xff] ^
+                        AES_Td2[AES_Te4[(rk[3] >>  8) & 0xff] & 0xff] ^
+                        AES_Td3[AES_Te4[(rk[3]      ) & 0xff] & 0xff];
+	}
+	return 0;
+}
+
+#ifndef AES_ASM
+/*
+ * Encrypt a single block
+ * in and out can overlap
+ */
+void AES_encrypt(const unsigned char *in, unsigned char *out,
+		 const AES_KEY *key) {
+
+	const u32 *rk;
+	u32 s0, s1, s2, s3, t0, t1, t2, t3;
+#ifndef FULL_UNROLL
+	int r;
+#endif /* ?FULL_UNROLL */
+
+	assert(in && out && key);
+	rk = key->rd_key;
+
+	/*
+	 * map byte array block to cipher state
+	 * and add initial round key:
+	 */
+	s0 = GETU32(in     ) ^ rk[0];
+	s1 = GETU32(in +  4) ^ rk[1];
+	s2 = GETU32(in +  8) ^ rk[2];
+	s3 = GETU32(in + 12) ^ rk[3];
+#ifdef FULL_UNROLL
+	/* round 1: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[ 4];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[ 5];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[ 6];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[ 7];
+   	/* round 2: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[ 8];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[ 9];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[10];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[11];
+	/* round 3: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[12];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[13];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[14];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[15];
+   	/* round 4: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[16];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[17];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[18];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[19];
+	/* round 5: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[20];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[21];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[22];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[23];
+   	/* round 6: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[24];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[25];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[26];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[27];
+	/* round 7: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[28];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[29];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[30];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[31];
+   	/* round 8: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[32];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[33];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[34];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[35];
+	/* round 9: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[36];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[37];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[38];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[39];
+    if (key->rounds > 10) {
+        /* round 10: */
+        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[40];
+        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[41];
+        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[42];
+        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[43];
+        /* round 11: */
+        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[44];
+        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[45];
+        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[46];
+        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[47];
+        if (key->rounds > 12) {
+            /* round 12: */
+            s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[48];
+            s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[49];
+            s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[50];
+            s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[51];
+            /* round 13: */
+            t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[52];
+            t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[53];
+            t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[54];
+            t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[55];
+        }
+    }
+    rk += key->rounds << 2;
+#else  /* !FULL_UNROLL */
+    /*
+     * Nr - 1 full rounds:
+     */
+    r = key->rounds >> 1;
+    for (;;) {
+        t0 =
+            AES_Te0[(s0 >> 24)       ] ^
+            AES_Te1[(s1 >> 16) & 0xff] ^
+            AES_Te2[(s2 >>  8) & 0xff] ^
+            AES_Te3[(s3      ) & 0xff] ^
+            rk[4];
+        t1 =
+            AES_Te0[(s1 >> 24)       ] ^
+            AES_Te1[(s2 >> 16) & 0xff] ^
+            AES_Te2[(s3 >>  8) & 0xff] ^
+            AES_Te3[(s0      ) & 0xff] ^
+            rk[5];
+        t2 =
+            AES_Te0[(s2 >> 24)       ] ^
+            AES_Te1[(s3 >> 16) & 0xff] ^
+            AES_Te2[(s0 >>  8) & 0xff] ^
+            AES_Te3[(s1      ) & 0xff] ^
+            rk[6];
+        t3 =
+            AES_Te0[(s3 >> 24)       ] ^
+            AES_Te1[(s0 >> 16) & 0xff] ^
+            AES_Te2[(s1 >>  8) & 0xff] ^
+            AES_Te3[(s2      ) & 0xff] ^
+            rk[7];
+
+        rk += 8;
+        if (--r == 0) {
+            break;
+        }
+
+        s0 =
+            AES_Te0[(t0 >> 24)       ] ^
+            AES_Te1[(t1 >> 16) & 0xff] ^
+            AES_Te2[(t2 >>  8) & 0xff] ^
+            AES_Te3[(t3      ) & 0xff] ^
+            rk[0];
+        s1 =
+            AES_Te0[(t1 >> 24)       ] ^
+            AES_Te1[(t2 >> 16) & 0xff] ^
+            AES_Te2[(t3 >>  8) & 0xff] ^
+            AES_Te3[(t0      ) & 0xff] ^
+            rk[1];
+        s2 =
+            AES_Te0[(t2 >> 24)       ] ^
+            AES_Te1[(t3 >> 16) & 0xff] ^
+            AES_Te2[(t0 >>  8) & 0xff] ^
+            AES_Te3[(t1      ) & 0xff] ^
+            rk[2];
+        s3 =
+            AES_Te0[(t3 >> 24)       ] ^
+            AES_Te1[(t0 >> 16) & 0xff] ^
+            AES_Te2[(t1 >>  8) & 0xff] ^
+            AES_Te3[(t2      ) & 0xff] ^
+            rk[3];
+    }
+#endif /* ?FULL_UNROLL */
+    /*
+	 * apply last round and
+	 * map cipher state to byte array block:
+	 */
+	s0 =
+                (AES_Te4[(t0 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t3      ) & 0xff] & 0x000000ff) ^
+		rk[0];
+	PUTU32(out     , s0);
+	s1 =
+                (AES_Te4[(t1 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t0      ) & 0xff] & 0x000000ff) ^
+		rk[1];
+	PUTU32(out +  4, s1);
+	s2 =
+                (AES_Te4[(t2 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t1      ) & 0xff] & 0x000000ff) ^
+		rk[2];
+	PUTU32(out +  8, s2);
+	s3 =
+                (AES_Te4[(t3 >> 24)       ] & 0xff000000) ^
+                (AES_Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Te4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Te4[(t2      ) & 0xff] & 0x000000ff) ^
+		rk[3];
+	PUTU32(out + 12, s3);
+}
+
+/*
+ * Decrypt a single block
+ * in and out can overlap
+ */
+void AES_decrypt(const unsigned char *in, unsigned char *out,
+		 const AES_KEY *key) {
+
+	const u32 *rk;
+	u32 s0, s1, s2, s3, t0, t1, t2, t3;
+#ifndef FULL_UNROLL
+	int r;
+#endif /* ?FULL_UNROLL */
+
+	assert(in && out && key);
+	rk = key->rd_key;
+
+	/*
+	 * map byte array block to cipher state
+	 * and add initial round key:
+	 */
+    s0 = GETU32(in     ) ^ rk[0];
+    s1 = GETU32(in +  4) ^ rk[1];
+    s2 = GETU32(in +  8) ^ rk[2];
+    s3 = GETU32(in + 12) ^ rk[3];
+#ifdef FULL_UNROLL
+    /* round 1: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[ 4];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[ 5];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[ 6];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[ 7];
+    /* round 2: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[ 8];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[ 9];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[10];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[11];
+    /* round 3: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[12];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[13];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[14];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[15];
+    /* round 4: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[16];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[17];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[18];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[19];
+    /* round 5: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[20];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[21];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[22];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[23];
+    /* round 6: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[24];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[25];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[26];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[27];
+    /* round 7: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[28];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[29];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[30];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[31];
+    /* round 8: */
+    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[32];
+    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[33];
+    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[34];
+    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[35];
+    /* round 9: */
+    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[36];
+    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[37];
+    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[38];
+    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[39];
+    if (key->rounds > 10) {
+        /* round 10: */
+        s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[40];
+        s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[41];
+        s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[42];
+        s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[43];
+        /* round 11: */
+        t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[44];
+        t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[45];
+        t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[46];
+        t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[47];
+        if (key->rounds > 12) {
+            /* round 12: */
+            s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[48];
+            s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[49];
+            s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[50];
+            s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[51];
+            /* round 13: */
+            t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[52];
+            t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[53];
+            t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[54];
+            t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[55];
+        }
+    }
+	rk += key->rounds << 2;
+#else  /* !FULL_UNROLL */
+    /*
+     * Nr - 1 full rounds:
+     */
+    r = key->rounds >> 1;
+    for (;;) {
+        t0 =
+            AES_Td0[(s0 >> 24)       ] ^
+            AES_Td1[(s3 >> 16) & 0xff] ^
+            AES_Td2[(s2 >>  8) & 0xff] ^
+            AES_Td3[(s1      ) & 0xff] ^
+            rk[4];
+        t1 =
+            AES_Td0[(s1 >> 24)       ] ^
+            AES_Td1[(s0 >> 16) & 0xff] ^
+            AES_Td2[(s3 >>  8) & 0xff] ^
+            AES_Td3[(s2      ) & 0xff] ^
+            rk[5];
+        t2 =
+            AES_Td0[(s2 >> 24)       ] ^
+            AES_Td1[(s1 >> 16) & 0xff] ^
+            AES_Td2[(s0 >>  8) & 0xff] ^
+            AES_Td3[(s3      ) & 0xff] ^
+            rk[6];
+        t3 =
+            AES_Td0[(s3 >> 24)       ] ^
+            AES_Td1[(s2 >> 16) & 0xff] ^
+            AES_Td2[(s1 >>  8) & 0xff] ^
+            AES_Td3[(s0      ) & 0xff] ^
+            rk[7];
+
+        rk += 8;
+        if (--r == 0) {
+            break;
+        }
+
+        s0 =
+            AES_Td0[(t0 >> 24)       ] ^
+            AES_Td1[(t3 >> 16) & 0xff] ^
+            AES_Td2[(t2 >>  8) & 0xff] ^
+            AES_Td3[(t1      ) & 0xff] ^
+            rk[0];
+        s1 =
+            AES_Td0[(t1 >> 24)       ] ^
+            AES_Td1[(t0 >> 16) & 0xff] ^
+            AES_Td2[(t3 >>  8) & 0xff] ^
+            AES_Td3[(t2      ) & 0xff] ^
+            rk[1];
+        s2 =
+            AES_Td0[(t2 >> 24)       ] ^
+            AES_Td1[(t1 >> 16) & 0xff] ^
+            AES_Td2[(t0 >>  8) & 0xff] ^
+            AES_Td3[(t3      ) & 0xff] ^
+            rk[2];
+        s3 =
+            AES_Td0[(t3 >> 24)       ] ^
+            AES_Td1[(t2 >> 16) & 0xff] ^
+            AES_Td2[(t1 >>  8) & 0xff] ^
+            AES_Td3[(t0      ) & 0xff] ^
+            rk[3];
+    }
+#endif /* ?FULL_UNROLL */
+    /*
+	 * apply last round and
+	 * map cipher state to byte array block:
+	 */
+   	s0 =
+                (AES_Td4[(t0 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t1      ) & 0xff] & 0x000000ff) ^
+   		rk[0];
+	PUTU32(out     , s0);
+   	s1 =
+                (AES_Td4[(t1 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t2      ) & 0xff] & 0x000000ff) ^
+   		rk[1];
+	PUTU32(out +  4, s1);
+   	s2 =
+                (AES_Td4[(t2 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t3      ) & 0xff] & 0x000000ff) ^
+   		rk[2];
+	PUTU32(out +  8, s2);
+   	s3 =
+                (AES_Td4[(t3 >> 24)       ] & 0xff000000) ^
+                (AES_Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
+                (AES_Td4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
+                (AES_Td4[(t0      ) & 0xff] & 0x000000ff) ^
+   		rk[3];
+	PUTU32(out + 12, s3);
+}
+
+#endif /* AES_ASM */
+
+void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
+		     const unsigned long length, const AES_KEY *key,
+		     unsigned char *ivec, const int enc)
+{
+
+	unsigned long n;
+	unsigned long len = length;
+	unsigned char tmp[AES_BLOCK_SIZE];
+
+	assert(in && out && key && ivec);
+
+	if (enc) {
+		while (len >= AES_BLOCK_SIZE) {
+			for(n=0; n < AES_BLOCK_SIZE; ++n)
+				tmp[n] = in[n] ^ ivec[n];
+			AES_encrypt(tmp, out, key);
+			memcpy(ivec, out, AES_BLOCK_SIZE);
+			len -= AES_BLOCK_SIZE;
+			in += AES_BLOCK_SIZE;
+			out += AES_BLOCK_SIZE;
+		}
+		if (len) {
+			for(n=0; n < len; ++n)
+				tmp[n] = in[n] ^ ivec[n];
+			for(n=len; n < AES_BLOCK_SIZE; ++n)
+				tmp[n] = ivec[n];
+			AES_encrypt(tmp, tmp, key);
+			memcpy(out, tmp, AES_BLOCK_SIZE);
+			memcpy(ivec, tmp, AES_BLOCK_SIZE);
+		}
+	} else {
+		while (len >= AES_BLOCK_SIZE) {
+			memcpy(tmp, in, AES_BLOCK_SIZE);
+			AES_decrypt(in, out, key);
+			for(n=0; n < AES_BLOCK_SIZE; ++n)
+				out[n] ^= ivec[n];
+			memcpy(ivec, tmp, AES_BLOCK_SIZE);
+			len -= AES_BLOCK_SIZE;
+			in += AES_BLOCK_SIZE;
+			out += AES_BLOCK_SIZE;
+		}
+		if (len) {
+			memcpy(tmp, in, AES_BLOCK_SIZE);
+			AES_decrypt(tmp, tmp, key);
+			for(n=0; n < len; ++n)
+				out[n] = tmp[n] ^ ivec[n];
+			memcpy(ivec, tmp, AES_BLOCK_SIZE);
+		}
+	}
+}
diff --git a/crypto/afsplit.c b/crypto/afsplit.c
new file mode 100644
index 0000000..825e2cf
--- /dev/null
+++ b/crypto/afsplit.c
@@ -0,0 +1,159 @@
+/*
+ * QEMU Crypto anti forensic information splitter
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * Derived from cryptsetup package lib/luks1/af.c
+ *
+ * Copyright (C) 2004, Clemens Fruhwirth <clemens@endorphin.org>
+ * Copyright (C) 2009-2012, Red Hat, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/bswap.h"
+#include "crypto/afsplit.h"
+#include "crypto/random.h"
+
+
+static void qcrypto_afsplit_xor(size_t blocklen,
+                                const uint8_t *in1,
+                                const uint8_t *in2,
+                                uint8_t *out)
+{
+    size_t i;
+    for (i = 0; i < blocklen; i++) {
+        out[i] = in1[i] ^ in2[i];
+    }
+}
+
+
+static int qcrypto_afsplit_hash(QCryptoHashAlgorithm hash,
+                                size_t blocklen,
+                                uint8_t *block,
+                                Error **errp)
+{
+    size_t digestlen = qcrypto_hash_digest_len(hash);
+
+    size_t hashcount = blocklen / digestlen;
+    size_t finallen = blocklen % digestlen;
+    uint32_t i;
+
+    if (finallen) {
+        hashcount++;
+    } else {
+        finallen = digestlen;
+    }
+
+    for (i = 0; i < hashcount; i++) {
+        uint8_t *out = NULL;
+        size_t outlen = 0;
+        uint32_t iv = cpu_to_be32(i);
+        struct iovec in[] = {
+            { .iov_base = &iv,
+              .iov_len = sizeof(iv) },
+            { .iov_base = block + (i * digestlen),
+              .iov_len = (i == (hashcount - 1)) ? finallen : digestlen },
+        };
+
+        if (qcrypto_hash_bytesv(hash,
+                                in,
+                                G_N_ELEMENTS(in),
+                                &out, &outlen,
+                                errp) < 0) {
+            return -1;
+        }
+
+        assert(outlen == digestlen);
+        memcpy(block + (i * digestlen), out,
+               (i == (hashcount - 1)) ? finallen : digestlen);
+        g_free(out);
+    }
+
+    return 0;
+}
+
+
+int qcrypto_afsplit_encode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp)
+{
+    uint8_t *block = g_new0(uint8_t, blocklen);
+    size_t i;
+    int ret = -1;
+
+    for (i = 0; i < (stripes - 1); i++) {
+        if (qcrypto_random_bytes(out + (i * blocklen), blocklen, errp) < 0) {
+            goto cleanup;
+        }
+
+        qcrypto_afsplit_xor(blocklen,
+                            out + (i * blocklen),
+                            block,
+                            block);
+        if (qcrypto_afsplit_hash(hash, blocklen, block,
+                                 errp) < 0) {
+            goto cleanup;
+        }
+    }
+    qcrypto_afsplit_xor(blocklen,
+                        in,
+                        block,
+                        out + (i * blocklen));
+    ret = 0;
+
+ cleanup:
+    g_free(block);
+    return ret;
+}
+
+
+int qcrypto_afsplit_decode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp)
+{
+    uint8_t *block = g_new0(uint8_t, blocklen);
+    size_t i;
+    int ret = -1;
+
+    for (i = 0; i < (stripes - 1); i++) {
+        qcrypto_afsplit_xor(blocklen,
+                            in + (i * blocklen),
+                            block,
+                            block);
+        if (qcrypto_afsplit_hash(hash, blocklen, block,
+                                 errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    qcrypto_afsplit_xor(blocklen,
+                        in + (i * blocklen),
+                        block,
+                        out);
+
+    ret = 0;
+
+ cleanup:
+    g_free(block);
+    return ret;
+}
diff --git a/crypto/block-luks.c b/crypto/block-luks.c
new file mode 100644
index 0000000..aba4455
--- /dev/null
+++ b/crypto/block-luks.c
@@ -0,0 +1,1416 @@
+/*
+ * QEMU Crypto block device encryption LUKS format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+
+#include "crypto/block-luks.h"
+
+#include "crypto/hash.h"
+#include "crypto/afsplit.h"
+#include "crypto/pbkdf.h"
+#include "crypto/secret.h"
+#include "crypto/random.h"
+
+#ifdef CONFIG_UUID
+#include <uuid/uuid.h>
+#endif
+
+#include "qemu/coroutine.h"
+
+/*
+ * Reference for the LUKS format implemented here is
+ *
+ *   docs/on-disk-format.pdf
+ *
+ * in 'cryptsetup' package source code
+ *
+ * This file implements the 1.2.1 specification, dated
+ * Oct 16, 2011.
+ */
+
+typedef struct QCryptoBlockLUKS QCryptoBlockLUKS;
+typedef struct QCryptoBlockLUKSHeader QCryptoBlockLUKSHeader;
+typedef struct QCryptoBlockLUKSKeySlot QCryptoBlockLUKSKeySlot;
+
+
+/* The following constants are all defined by the LUKS spec */
+#define QCRYPTO_BLOCK_LUKS_VERSION 1
+
+#define QCRYPTO_BLOCK_LUKS_MAGIC_LEN 6
+#define QCRYPTO_BLOCK_LUKS_CIPHER_NAME_LEN 32
+#define QCRYPTO_BLOCK_LUKS_CIPHER_MODE_LEN 32
+#define QCRYPTO_BLOCK_LUKS_HASH_SPEC_LEN 32
+#define QCRYPTO_BLOCK_LUKS_DIGEST_LEN 20
+#define QCRYPTO_BLOCK_LUKS_SALT_LEN 32
+#define QCRYPTO_BLOCK_LUKS_UUID_LEN 40
+#define QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS 8
+#define QCRYPTO_BLOCK_LUKS_STRIPES 4000
+#define QCRYPTO_BLOCK_LUKS_MIN_SLOT_KEY_ITERS 1000
+#define QCRYPTO_BLOCK_LUKS_MIN_MASTER_KEY_ITERS 1000
+#define QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET 4096
+
+#define QCRYPTO_BLOCK_LUKS_KEY_SLOT_DISABLED 0x0000DEAD
+#define QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED 0x00AC71F3
+
+#define QCRYPTO_BLOCK_LUKS_SECTOR_SIZE 512LL
+
+static const char qcrypto_block_luks_magic[QCRYPTO_BLOCK_LUKS_MAGIC_LEN] = {
+    'L', 'U', 'K', 'S', 0xBA, 0xBE
+};
+
+typedef struct QCryptoBlockLUKSNameMap QCryptoBlockLUKSNameMap;
+struct QCryptoBlockLUKSNameMap {
+    const char *name;
+    int id;
+};
+
+typedef struct QCryptoBlockLUKSCipherSizeMap QCryptoBlockLUKSCipherSizeMap;
+struct QCryptoBlockLUKSCipherSizeMap {
+    uint32_t key_bytes;
+    int id;
+};
+typedef struct QCryptoBlockLUKSCipherNameMap QCryptoBlockLUKSCipherNameMap;
+struct QCryptoBlockLUKSCipherNameMap {
+    const char *name;
+    const QCryptoBlockLUKSCipherSizeMap *sizes;
+};
+
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_aes[] = {
+    { 16, QCRYPTO_CIPHER_ALG_AES_128 },
+    { 24, QCRYPTO_CIPHER_ALG_AES_192 },
+    { 32, QCRYPTO_CIPHER_ALG_AES_256 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_cast5[] = {
+    { 16, QCRYPTO_CIPHER_ALG_CAST5_128 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_serpent[] = {
+    { 16, QCRYPTO_CIPHER_ALG_SERPENT_128 },
+    { 24, QCRYPTO_CIPHER_ALG_SERPENT_192 },
+    { 32, QCRYPTO_CIPHER_ALG_SERPENT_256 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherSizeMap
+qcrypto_block_luks_cipher_size_map_twofish[] = {
+    { 16, QCRYPTO_CIPHER_ALG_TWOFISH_128 },
+    { 24, QCRYPTO_CIPHER_ALG_TWOFISH_192 },
+    { 32, QCRYPTO_CIPHER_ALG_TWOFISH_256 },
+    { 0, 0 },
+};
+
+static const QCryptoBlockLUKSCipherNameMap
+qcrypto_block_luks_cipher_name_map[] = {
+    { "aes", qcrypto_block_luks_cipher_size_map_aes },
+    { "cast5", qcrypto_block_luks_cipher_size_map_cast5 },
+    { "serpent", qcrypto_block_luks_cipher_size_map_serpent },
+    { "twofish", qcrypto_block_luks_cipher_size_map_twofish },
+};
+
+
+/*
+ * This struct is written to disk in big-endian format,
+ * but operated upon in native-endian format.
+ */
+struct QCryptoBlockLUKSKeySlot {
+    /* state of keyslot, enabled/disable */
+    uint32_t active;
+    /* iterations for PBKDF2 */
+    uint32_t iterations;
+    /* salt for PBKDF2 */
+    uint8_t salt[QCRYPTO_BLOCK_LUKS_SALT_LEN];
+    /* start sector of key material */
+    uint32_t key_offset;
+    /* number of anti-forensic stripes */
+    uint32_t stripes;
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct QCryptoBlockLUKSKeySlot) != 48);
+
+
+/*
+ * This struct is written to disk in big-endian format,
+ * but operated upon in native-endian format.
+ */
+struct QCryptoBlockLUKSHeader {
+    /* 'L', 'U', 'K', 'S', '0xBA', '0xBE' */
+    char magic[QCRYPTO_BLOCK_LUKS_MAGIC_LEN];
+
+    /* LUKS version, currently 1 */
+    uint16_t version;
+
+    /* cipher name specification (aes, etc) */
+    char cipher_name[QCRYPTO_BLOCK_LUKS_CIPHER_NAME_LEN];
+
+    /* cipher mode specification (cbc-plain, xts-essiv:sha256, etc) */
+    char cipher_mode[QCRYPTO_BLOCK_LUKS_CIPHER_MODE_LEN];
+
+    /* hash specification (sha256, etc) */
+    char hash_spec[QCRYPTO_BLOCK_LUKS_HASH_SPEC_LEN];
+
+    /* start offset of the volume data (in 512 byte sectors) */
+    uint32_t payload_offset;
+
+    /* Number of key bytes */
+    uint32_t key_bytes;
+
+    /* master key checksum after PBKDF2 */
+    uint8_t master_key_digest[QCRYPTO_BLOCK_LUKS_DIGEST_LEN];
+
+    /* salt for master key PBKDF2 */
+    uint8_t master_key_salt[QCRYPTO_BLOCK_LUKS_SALT_LEN];
+
+    /* iterations for master key PBKDF2 */
+    uint32_t master_key_iterations;
+
+    /* UUID of the partition in standard ASCII representation */
+    uint8_t uuid[QCRYPTO_BLOCK_LUKS_UUID_LEN];
+
+    /* key slots */
+    QCryptoBlockLUKSKeySlot key_slots[QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct QCryptoBlockLUKSHeader) != 592);
+
+
+struct QCryptoBlockLUKS {
+    QCryptoBlockLUKSHeader header;
+
+    /* Cache parsed versions of what's in header fields,
+     * as we can't rely on QCryptoBlock.cipher being
+     * non-NULL */
+    QCryptoCipherAlgorithm cipher_alg;
+    QCryptoCipherMode cipher_mode;
+    QCryptoIVGenAlgorithm ivgen_alg;
+    QCryptoHashAlgorithm ivgen_hash_alg;
+    QCryptoHashAlgorithm hash_alg;
+};
+
+
+static int qcrypto_block_luks_cipher_name_lookup(const char *name,
+                                                 QCryptoCipherMode mode,
+                                                 uint32_t key_bytes,
+                                                 Error **errp)
+{
+    const QCryptoBlockLUKSCipherNameMap *map =
+        qcrypto_block_luks_cipher_name_map;
+    size_t maplen = G_N_ELEMENTS(qcrypto_block_luks_cipher_name_map);
+    size_t i, j;
+
+    if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+        key_bytes /= 2;
+    }
+
+    for (i = 0; i < maplen; i++) {
+        if (!g_str_equal(map[i].name, name)) {
+            continue;
+        }
+        for (j = 0; j < map[i].sizes[j].key_bytes; j++) {
+            if (map[i].sizes[j].key_bytes == key_bytes) {
+                return map[i].sizes[j].id;
+            }
+        }
+    }
+
+    error_setg(errp, "Algorithm %s with key size %d bytes not supported",
+               name, key_bytes);
+    return 0;
+}
+
+static const char *
+qcrypto_block_luks_cipher_alg_lookup(QCryptoCipherAlgorithm alg,
+                                     Error **errp)
+{
+    const QCryptoBlockLUKSCipherNameMap *map =
+        qcrypto_block_luks_cipher_name_map;
+    size_t maplen = G_N_ELEMENTS(qcrypto_block_luks_cipher_name_map);
+    size_t i, j;
+    for (i = 0; i < maplen; i++) {
+        for (j = 0; j < map[i].sizes[j].key_bytes; j++) {
+            if (map[i].sizes[j].id == alg) {
+                return map[i].name;
+            }
+        }
+    }
+
+    error_setg(errp, "Algorithm '%s' not supported",
+               QCryptoCipherAlgorithm_lookup[alg]);
+    return NULL;
+}
+
+/* XXX replace with qapi_enum_parse() in future, when we can
+ * make that function emit a more friendly error message */
+static int qcrypto_block_luks_name_lookup(const char *name,
+                                          const char *const *map,
+                                          size_t maplen,
+                                          const char *type,
+                                          Error **errp)
+{
+    size_t i;
+    for (i = 0; i < maplen; i++) {
+        if (g_str_equal(map[i], name)) {
+            return i;
+        }
+    }
+
+    error_setg(errp, "%s %s not supported", type, name);
+    return 0;
+}
+
+#define qcrypto_block_luks_cipher_mode_lookup(name, errp)               \
+    qcrypto_block_luks_name_lookup(name,                                \
+                                   QCryptoCipherMode_lookup,            \
+                                   QCRYPTO_CIPHER_MODE__MAX,            \
+                                   "Cipher mode",                       \
+                                   errp)
+
+#define qcrypto_block_luks_hash_name_lookup(name, errp)                 \
+    qcrypto_block_luks_name_lookup(name,                                \
+                                   QCryptoHashAlgorithm_lookup,         \
+                                   QCRYPTO_HASH_ALG__MAX,               \
+                                   "Hash algorithm",                    \
+                                   errp)
+
+#define qcrypto_block_luks_ivgen_name_lookup(name, errp)                \
+    qcrypto_block_luks_name_lookup(name,                                \
+                                   QCryptoIVGenAlgorithm_lookup,        \
+                                   QCRYPTO_IVGEN_ALG__MAX,              \
+                                   "IV generator",                      \
+                                   errp)
+
+
+static bool
+qcrypto_block_luks_has_format(const uint8_t *buf,
+                              size_t buf_size)
+{
+    const QCryptoBlockLUKSHeader *luks_header = (const void *)buf;
+
+    if (buf_size >= offsetof(QCryptoBlockLUKSHeader, cipher_name) &&
+        memcmp(luks_header->magic, qcrypto_block_luks_magic,
+               QCRYPTO_BLOCK_LUKS_MAGIC_LEN) == 0 &&
+        be16_to_cpu(luks_header->version) == QCRYPTO_BLOCK_LUKS_VERSION) {
+        return true;
+    } else {
+        return false;
+    }
+}
+
+
+/**
+ * Deal with a quirk of dm-crypt usage of ESSIV.
+ *
+ * When calculating ESSIV IVs, the cipher length used by ESSIV
+ * may be different from the cipher length used for the block
+ * encryption, becauses dm-crypt uses the hash digest length
+ * as the key size. ie, if you have AES 128 as the block cipher
+ * and SHA 256 as ESSIV hash, then ESSIV will use AES 256 as
+ * the cipher since that gets a key length matching the digest
+ * size, not AES 128 with truncated digest as might be imagined
+ */
+static QCryptoCipherAlgorithm
+qcrypto_block_luks_essiv_cipher(QCryptoCipherAlgorithm cipher,
+                                QCryptoHashAlgorithm hash,
+                                Error **errp)
+{
+    size_t digestlen = qcrypto_hash_digest_len(hash);
+    size_t keylen = qcrypto_cipher_get_key_len(cipher);
+    if (digestlen == keylen) {
+        return cipher;
+    }
+
+    switch (cipher) {
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+        if (digestlen == qcrypto_cipher_get_key_len(
+                QCRYPTO_CIPHER_ALG_AES_128)) {
+            return QCRYPTO_CIPHER_ALG_AES_128;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_AES_192)) {
+            return QCRYPTO_CIPHER_ALG_AES_192;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_AES_256)) {
+            return QCRYPTO_CIPHER_ALG_AES_256;
+        } else {
+            error_setg(errp, "No AES cipher with key size %zu available",
+                       digestlen);
+            return 0;
+        }
+        break;
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        if (digestlen == qcrypto_cipher_get_key_len(
+                QCRYPTO_CIPHER_ALG_SERPENT_128)) {
+            return QCRYPTO_CIPHER_ALG_SERPENT_128;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_SERPENT_192)) {
+            return QCRYPTO_CIPHER_ALG_SERPENT_192;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_SERPENT_256)) {
+            return QCRYPTO_CIPHER_ALG_SERPENT_256;
+        } else {
+            error_setg(errp, "No Serpent cipher with key size %zu available",
+                       digestlen);
+            return 0;
+        }
+        break;
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_192:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        if (digestlen == qcrypto_cipher_get_key_len(
+                QCRYPTO_CIPHER_ALG_TWOFISH_128)) {
+            return QCRYPTO_CIPHER_ALG_TWOFISH_128;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_TWOFISH_192)) {
+            return QCRYPTO_CIPHER_ALG_TWOFISH_192;
+        } else if (digestlen == qcrypto_cipher_get_key_len(
+                       QCRYPTO_CIPHER_ALG_TWOFISH_256)) {
+            return QCRYPTO_CIPHER_ALG_TWOFISH_256;
+        } else {
+            error_setg(errp, "No Twofish cipher with key size %zu available",
+                       digestlen);
+            return 0;
+        }
+        break;
+    default:
+        error_setg(errp, "Cipher %s not supported with essiv",
+                   QCryptoCipherAlgorithm_lookup[cipher]);
+        return 0;
+    }
+}
+
+/*
+ * Given a key slot, and user password, this will attempt to unlock
+ * the master encryption key from the key slot.
+ *
+ * Returns:
+ *    0 if the key slot is disabled, or key could not be decrypted
+ *      with the provided password
+ *    1 if the key slot is enabled, and key decrypted successfully
+ *      with the provided password
+ *   -1 if a fatal error occurred loading the key
+ */
+static int
+qcrypto_block_luks_load_key(QCryptoBlock *block,
+                            QCryptoBlockLUKSKeySlot *slot,
+                            const char *password,
+                            QCryptoCipherAlgorithm cipheralg,
+                            QCryptoCipherMode ciphermode,
+                            QCryptoHashAlgorithm hash,
+                            QCryptoIVGenAlgorithm ivalg,
+                            QCryptoCipherAlgorithm ivcipheralg,
+                            QCryptoHashAlgorithm ivhash,
+                            uint8_t *masterkey,
+                            size_t masterkeylen,
+                            QCryptoBlockReadFunc readfunc,
+                            void *opaque,
+                            Error **errp)
+{
+    QCryptoBlockLUKS *luks = block->opaque;
+    uint8_t *splitkey;
+    size_t splitkeylen;
+    uint8_t *possiblekey;
+    int ret = -1;
+    ssize_t rv;
+    QCryptoCipher *cipher = NULL;
+    uint8_t keydigest[QCRYPTO_BLOCK_LUKS_DIGEST_LEN];
+    QCryptoIVGen *ivgen = NULL;
+    size_t niv;
+
+    if (slot->active != QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED) {
+        return 0;
+    }
+
+    splitkeylen = masterkeylen * slot->stripes;
+    splitkey = g_new0(uint8_t, splitkeylen);
+    possiblekey = g_new0(uint8_t, masterkeylen);
+
+    /*
+     * The user password is used to generate a (possible)
+     * decryption key. This may or may not successfully
+     * decrypt the master key - we just blindly assume
+     * the key is correct and validate the results of
+     * decryption later.
+     */
+    if (qcrypto_pbkdf2(hash,
+                       (const uint8_t *)password, strlen(password),
+                       slot->salt, QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       slot->iterations,
+                       possiblekey, masterkeylen,
+                       errp) < 0) {
+        goto cleanup;
+    }
+
+    /*
+     * We need to read the master key material from the
+     * LUKS key material header. What we're reading is
+     * not the raw master key, but rather the data after
+     * it has been passed through AFSplit and the result
+     * then encrypted.
+     */
+    rv = readfunc(block,
+                  slot->key_offset * QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                  splitkey, splitkeylen,
+                  errp,
+                  opaque);
+    if (rv < 0) {
+        goto cleanup;
+    }
+
+
+    /* Setup the cipher/ivgen that we'll use to try to decrypt
+     * the split master key material */
+    cipher = qcrypto_cipher_new(cipheralg, ciphermode,
+                                possiblekey, masterkeylen,
+                                errp);
+    if (!cipher) {
+        goto cleanup;
+    }
+
+    niv = qcrypto_cipher_get_iv_len(cipheralg,
+                                    ciphermode);
+    ivgen = qcrypto_ivgen_new(ivalg,
+                              ivcipheralg,
+                              ivhash,
+                              possiblekey, masterkeylen,
+                              errp);
+    if (!ivgen) {
+        goto cleanup;
+    }
+
+
+    /*
+     * The master key needs to be decrypted in the same
+     * way that the block device payload will be decrypted
+     * later. In particular we'll be using the IV generator
+     * to reset the encryption cipher every time the master
+     * key crosses a sector boundary.
+     */
+    if (qcrypto_block_decrypt_helper(cipher,
+                                     niv,
+                                     ivgen,
+                                     QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                     0,
+                                     splitkey,
+                                     splitkeylen,
+                                     errp) < 0) {
+        goto cleanup;
+    }
+
+    /*
+     * Now we've decrypted the split master key, join
+     * it back together to get the actual master key.
+     */
+    if (qcrypto_afsplit_decode(hash,
+                               masterkeylen,
+                               slot->stripes,
+                               splitkey,
+                               masterkey,
+                               errp) < 0) {
+        goto cleanup;
+    }
+
+
+    /*
+     * We still don't know that the masterkey we got is valid,
+     * because we just blindly assumed the user's password
+     * was correct. This is where we now verify it. We are
+     * creating a hash of the master key using PBKDF and
+     * then comparing that to the hash stored in the key slot
+     * header
+     */
+    if (qcrypto_pbkdf2(hash,
+                       masterkey, masterkeylen,
+                       luks->header.master_key_salt,
+                       QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       luks->header.master_key_iterations,
+                       keydigest, G_N_ELEMENTS(keydigest),
+                       errp) < 0) {
+        goto cleanup;
+    }
+
+    if (memcmp(keydigest, luks->header.master_key_digest,
+               QCRYPTO_BLOCK_LUKS_DIGEST_LEN) == 0) {
+        /* Success, we got the right master key */
+        ret = 1;
+        goto cleanup;
+    }
+
+    /* Fail, user's password was not valid for this key slot,
+     * tell caller to try another slot */
+    ret = 0;
+
+ cleanup:
+    qcrypto_ivgen_free(ivgen);
+    qcrypto_cipher_free(cipher);
+    g_free(splitkey);
+    g_free(possiblekey);
+    return ret;
+}
+
+
+/*
+ * Given a user password, this will iterate over all key
+ * slots and try to unlock each active key slot using the
+ * password until it successfully obtains a master key.
+ *
+ * Returns 0 if a key was loaded, -1 if no keys could be loaded
+ */
+static int
+qcrypto_block_luks_find_key(QCryptoBlock *block,
+                            const char *password,
+                            QCryptoCipherAlgorithm cipheralg,
+                            QCryptoCipherMode ciphermode,
+                            QCryptoHashAlgorithm hash,
+                            QCryptoIVGenAlgorithm ivalg,
+                            QCryptoCipherAlgorithm ivcipheralg,
+                            QCryptoHashAlgorithm ivhash,
+                            uint8_t **masterkey,
+                            size_t *masterkeylen,
+                            QCryptoBlockReadFunc readfunc,
+                            void *opaque,
+                            Error **errp)
+{
+    QCryptoBlockLUKS *luks = block->opaque;
+    size_t i;
+    int rv;
+
+    *masterkey = g_new0(uint8_t, luks->header.key_bytes);
+    *masterkeylen = luks->header.key_bytes;
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        rv = qcrypto_block_luks_load_key(block,
+                                         &luks->header.key_slots[i],
+                                         password,
+                                         cipheralg,
+                                         ciphermode,
+                                         hash,
+                                         ivalg,
+                                         ivcipheralg,
+                                         ivhash,
+                                         *masterkey,
+                                         *masterkeylen,
+                                         readfunc,
+                                         opaque,
+                                         errp);
+        if (rv < 0) {
+            goto error;
+        }
+        if (rv == 1) {
+            return 0;
+        }
+    }
+
+    error_setg(errp, "Invalid password, cannot unlock any keyslot");
+
+ error:
+    g_free(*masterkey);
+    *masterkey = NULL;
+    *masterkeylen = 0;
+    return -1;
+}
+
+
+static int
+qcrypto_block_luks_open(QCryptoBlock *block,
+                        QCryptoBlockOpenOptions *options,
+                        QCryptoBlockReadFunc readfunc,
+                        void *opaque,
+                        unsigned int flags,
+                        Error **errp)
+{
+    QCryptoBlockLUKS *luks;
+    Error *local_err = NULL;
+    int ret = 0;
+    size_t i;
+    ssize_t rv;
+    uint8_t *masterkey = NULL;
+    size_t masterkeylen;
+    char *ivgen_name, *ivhash_name;
+    QCryptoCipherMode ciphermode;
+    QCryptoCipherAlgorithm cipheralg;
+    QCryptoIVGenAlgorithm ivalg;
+    QCryptoCipherAlgorithm ivcipheralg;
+    QCryptoHashAlgorithm hash;
+    QCryptoHashAlgorithm ivhash;
+    char *password = NULL;
+
+    if (!(flags & QCRYPTO_BLOCK_OPEN_NO_IO)) {
+        if (!options->u.luks.key_secret) {
+            error_setg(errp, "Parameter 'key-secret' is required for cipher");
+            return -1;
+        }
+        password = qcrypto_secret_lookup_as_utf8(
+            options->u.luks.key_secret, errp);
+        if (!password) {
+            return -1;
+        }
+    }
+
+    luks = g_new0(QCryptoBlockLUKS, 1);
+    block->opaque = luks;
+
+    /* Read the entire LUKS header, minus the key material from
+     * the underlying device */
+    rv = readfunc(block, 0,
+                  (uint8_t *)&luks->header,
+                  sizeof(luks->header),
+                  errp,
+                  opaque);
+    if (rv < 0) {
+        ret = rv;
+        goto fail;
+    }
+
+    /* The header is always stored in big-endian format, so
+     * convert everything to native */
+    be16_to_cpus(&luks->header.version);
+    be32_to_cpus(&luks->header.payload_offset);
+    be32_to_cpus(&luks->header.key_bytes);
+    be32_to_cpus(&luks->header.master_key_iterations);
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        be32_to_cpus(&luks->header.key_slots[i].active);
+        be32_to_cpus(&luks->header.key_slots[i].iterations);
+        be32_to_cpus(&luks->header.key_slots[i].key_offset);
+        be32_to_cpus(&luks->header.key_slots[i].stripes);
+    }
+
+    if (memcmp(luks->header.magic, qcrypto_block_luks_magic,
+               QCRYPTO_BLOCK_LUKS_MAGIC_LEN) != 0) {
+        error_setg(errp, "Volume is not in LUKS format");
+        ret = -EINVAL;
+        goto fail;
+    }
+    if (luks->header.version != QCRYPTO_BLOCK_LUKS_VERSION) {
+        error_setg(errp, "LUKS version %" PRIu32 " is not supported",
+                   luks->header.version);
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    /*
+     * The cipher_mode header contains a string that we have
+     * to further parse, of the format
+     *
+     *    <cipher-mode>-<iv-generator>[:<iv-hash>]
+     *
+     * eg  cbc-essiv:sha256, cbc-plain64
+     */
+    ivgen_name = strchr(luks->header.cipher_mode, '-');
+    if (!ivgen_name) {
+        ret = -EINVAL;
+        error_setg(errp, "Unexpected cipher mode string format %s",
+                   luks->header.cipher_mode);
+        goto fail;
+    }
+    *ivgen_name = '\0';
+    ivgen_name++;
+
+    ivhash_name = strchr(ivgen_name, ':');
+    if (!ivhash_name) {
+        ivhash = 0;
+    } else {
+        *ivhash_name = '\0';
+        ivhash_name++;
+
+        ivhash = qcrypto_block_luks_hash_name_lookup(ivhash_name,
+                                                     &local_err);
+        if (local_err) {
+            ret = -ENOTSUP;
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+    }
+
+    ciphermode = qcrypto_block_luks_cipher_mode_lookup(luks->header.cipher_mode,
+                                                       &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    cipheralg = qcrypto_block_luks_cipher_name_lookup(luks->header.cipher_name,
+                                                      ciphermode,
+                                                      luks->header.key_bytes,
+                                                      &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    hash = qcrypto_block_luks_hash_name_lookup(luks->header.hash_spec,
+                                               &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    ivalg = qcrypto_block_luks_ivgen_name_lookup(ivgen_name,
+                                                 &local_err);
+    if (local_err) {
+        ret = -ENOTSUP;
+        error_propagate(errp, local_err);
+        goto fail;
+    }
+
+    if (ivalg == QCRYPTO_IVGEN_ALG_ESSIV) {
+        if (!ivhash_name) {
+            ret = -EINVAL;
+            error_setg(errp, "Missing IV generator hash specification");
+            goto fail;
+        }
+        ivcipheralg = qcrypto_block_luks_essiv_cipher(cipheralg,
+                                                      ivhash,
+                                                      &local_err);
+        if (local_err) {
+            ret = -ENOTSUP;
+            error_propagate(errp, local_err);
+            goto fail;
+        }
+    } else {
+        /* Note we parsed the ivhash_name earlier in the cipher_mode
+         * spec string even with plain/plain64 ivgens, but we
+         * will ignore it, since it is irrelevant for these ivgens.
+         * This is for compat with dm-crypt which will silently
+         * ignore hash names with these ivgens rather than report
+         * an error about the invalid usage
+         */
+        ivcipheralg = cipheralg;
+    }
+
+    if (!(flags & QCRYPTO_BLOCK_OPEN_NO_IO)) {
+        /* Try to find which key slot our password is valid for
+         * and unlock the master key from that slot.
+         */
+        if (qcrypto_block_luks_find_key(block,
+                                        password,
+                                        cipheralg, ciphermode,
+                                        hash,
+                                        ivalg,
+                                        ivcipheralg,
+                                        ivhash,
+                                        &masterkey, &masterkeylen,
+                                        readfunc, opaque,
+                                        errp) < 0) {
+            ret = -EACCES;
+            goto fail;
+        }
+
+        /* We have a valid master key now, so can setup the
+         * block device payload decryption objects
+         */
+        block->kdfhash = hash;
+        block->niv = qcrypto_cipher_get_iv_len(cipheralg,
+                                               ciphermode);
+        block->ivgen = qcrypto_ivgen_new(ivalg,
+                                         ivcipheralg,
+                                         ivhash,
+                                         masterkey, masterkeylen,
+                                         errp);
+        if (!block->ivgen) {
+            ret = -ENOTSUP;
+            goto fail;
+        }
+
+        block->cipher = qcrypto_cipher_new(cipheralg,
+                                           ciphermode,
+                                           masterkey, masterkeylen,
+                                           errp);
+        if (!block->cipher) {
+            ret = -ENOTSUP;
+            goto fail;
+        }
+    }
+
+    block->payload_offset = luks->header.payload_offset *
+        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE;
+
+    luks->cipher_alg = cipheralg;
+    luks->cipher_mode = ciphermode;
+    luks->ivgen_alg = ivalg;
+    luks->ivgen_hash_alg = ivhash;
+    luks->hash_alg = hash;
+
+    g_free(masterkey);
+    g_free(password);
+
+    return 0;
+
+ fail:
+    g_free(masterkey);
+    qcrypto_cipher_free(block->cipher);
+    qcrypto_ivgen_free(block->ivgen);
+    g_free(luks);
+    g_free(password);
+    return ret;
+}
+
+
+static int
+qcrypto_block_luks_uuid_gen(uint8_t *uuidstr, Error **errp)
+{
+#ifdef CONFIG_UUID
+    uuid_t uuid;
+    uuid_generate(uuid);
+    uuid_unparse(uuid, (char *)uuidstr);
+    return 0;
+#else
+    error_setg(errp, "Unable to generate uuids on this platform");
+    return -1;
+#endif
+}
+
+static int
+qcrypto_block_luks_create(QCryptoBlock *block,
+                          QCryptoBlockCreateOptions *options,
+                          QCryptoBlockInitFunc initfunc,
+                          QCryptoBlockWriteFunc writefunc,
+                          void *opaque,
+                          Error **errp)
+{
+    QCryptoBlockLUKS *luks;
+    QCryptoBlockCreateOptionsLUKS luks_opts;
+    Error *local_err = NULL;
+    uint8_t *masterkey = NULL;
+    uint8_t *slotkey = NULL;
+    uint8_t *splitkey = NULL;
+    size_t splitkeylen = 0;
+    size_t i;
+    QCryptoCipher *cipher = NULL;
+    QCryptoIVGen *ivgen = NULL;
+    char *password;
+    const char *cipher_alg;
+    const char *cipher_mode;
+    const char *ivgen_alg;
+    const char *ivgen_hash_alg = NULL;
+    const char *hash_alg;
+    char *cipher_mode_spec = NULL;
+    QCryptoCipherAlgorithm ivcipheralg = 0;
+
+    memcpy(&luks_opts, &options->u.luks, sizeof(luks_opts));
+    if (!luks_opts.has_cipher_alg) {
+        luks_opts.cipher_alg = QCRYPTO_CIPHER_ALG_AES_256;
+    }
+    if (!luks_opts.has_cipher_mode) {
+        luks_opts.cipher_mode = QCRYPTO_CIPHER_MODE_XTS;
+    }
+    if (!luks_opts.has_ivgen_alg) {
+        luks_opts.ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64;
+    }
+    if (!luks_opts.has_hash_alg) {
+        luks_opts.hash_alg = QCRYPTO_HASH_ALG_SHA256;
+    }
+    if (luks_opts.ivgen_alg == QCRYPTO_IVGEN_ALG_ESSIV) {
+        if (!luks_opts.has_ivgen_hash_alg) {
+            luks_opts.ivgen_hash_alg = QCRYPTO_HASH_ALG_SHA256;
+            luks_opts.has_ivgen_hash_alg = true;
+        }
+    }
+    /* Note we're allowing ivgen_hash_alg to be set even for
+     * non-essiv iv generators that don't need a hash. It will
+     * be silently ignored, for compatibility with dm-crypt */
+
+    if (!options->u.luks.key_secret) {
+        error_setg(errp, "Parameter 'key-secret' is required for cipher");
+        return -1;
+    }
+    password = qcrypto_secret_lookup_as_utf8(luks_opts.key_secret, errp);
+    if (!password) {
+        return -1;
+    }
+
+    luks = g_new0(QCryptoBlockLUKS, 1);
+    block->opaque = luks;
+
+    memcpy(luks->header.magic, qcrypto_block_luks_magic,
+           QCRYPTO_BLOCK_LUKS_MAGIC_LEN);
+
+    /* We populate the header in native endianness initially and
+     * then convert everything to big endian just before writing
+     * it out to disk
+     */
+    luks->header.version = QCRYPTO_BLOCK_LUKS_VERSION;
+    if (qcrypto_block_luks_uuid_gen(luks->header.uuid,
+                                    errp) < 0) {
+        goto error;
+    }
+
+    cipher_alg = qcrypto_block_luks_cipher_alg_lookup(luks_opts.cipher_alg,
+                                                      errp);
+    if (!cipher_alg) {
+        goto error;
+    }
+
+    cipher_mode = QCryptoCipherMode_lookup[luks_opts.cipher_mode];
+    ivgen_alg = QCryptoIVGenAlgorithm_lookup[luks_opts.ivgen_alg];
+    if (luks_opts.has_ivgen_hash_alg) {
+        ivgen_hash_alg = QCryptoHashAlgorithm_lookup[luks_opts.ivgen_hash_alg];
+        cipher_mode_spec = g_strdup_printf("%s-%s:%s", cipher_mode, ivgen_alg,
+                                           ivgen_hash_alg);
+    } else {
+        cipher_mode_spec = g_strdup_printf("%s-%s", cipher_mode, ivgen_alg);
+    }
+    hash_alg = QCryptoHashAlgorithm_lookup[luks_opts.hash_alg];
+
+
+    if (strlen(cipher_alg) >= QCRYPTO_BLOCK_LUKS_CIPHER_NAME_LEN) {
+        error_setg(errp, "Cipher name '%s' is too long for LUKS header",
+                   cipher_alg);
+        goto error;
+    }
+    if (strlen(cipher_mode_spec) >= QCRYPTO_BLOCK_LUKS_CIPHER_MODE_LEN) {
+        error_setg(errp, "Cipher mode '%s' is too long for LUKS header",
+                   cipher_mode_spec);
+        goto error;
+    }
+    if (strlen(hash_alg) >= QCRYPTO_BLOCK_LUKS_HASH_SPEC_LEN) {
+        error_setg(errp, "Hash name '%s' is too long for LUKS header",
+                   hash_alg);
+        goto error;
+    }
+
+    if (luks_opts.ivgen_alg == QCRYPTO_IVGEN_ALG_ESSIV) {
+        ivcipheralg = qcrypto_block_luks_essiv_cipher(luks_opts.cipher_alg,
+                                                      luks_opts.ivgen_hash_alg,
+                                                      &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            goto error;
+        }
+    } else {
+        ivcipheralg = luks_opts.cipher_alg;
+    }
+
+    strcpy(luks->header.cipher_name, cipher_alg);
+    strcpy(luks->header.cipher_mode, cipher_mode_spec);
+    strcpy(luks->header.hash_spec, hash_alg);
+
+    luks->header.key_bytes = qcrypto_cipher_get_key_len(luks_opts.cipher_alg);
+    if (luks_opts.cipher_mode == QCRYPTO_CIPHER_MODE_XTS) {
+        luks->header.key_bytes *= 2;
+    }
+
+    /* Generate the salt used for hashing the master key
+     * with PBKDF later
+     */
+    if (qcrypto_random_bytes(luks->header.master_key_salt,
+                             QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                             errp) < 0) {
+        goto error;
+    }
+
+    /* Generate random master key */
+    masterkey = g_new0(uint8_t, luks->header.key_bytes);
+    if (qcrypto_random_bytes(masterkey,
+                             luks->header.key_bytes, errp) < 0) {
+        goto error;
+    }
+
+
+    /* Setup the block device payload encryption objects */
+    block->cipher = qcrypto_cipher_new(luks_opts.cipher_alg,
+                                       luks_opts.cipher_mode,
+                                       masterkey, luks->header.key_bytes,
+                                       errp);
+    if (!block->cipher) {
+        goto error;
+    }
+
+    block->kdfhash = luks_opts.hash_alg;
+    block->niv = qcrypto_cipher_get_iv_len(luks_opts.cipher_alg,
+                                           luks_opts.cipher_mode);
+    block->ivgen = qcrypto_ivgen_new(luks_opts.ivgen_alg,
+                                     ivcipheralg,
+                                     luks_opts.ivgen_hash_alg,
+                                     masterkey, luks->header.key_bytes,
+                                     errp);
+
+    if (!block->ivgen) {
+        goto error;
+    }
+
+
+    /* Determine how many iterations we need to hash the master
+     * key, in order to have 1 second of compute time used
+     */
+    luks->header.master_key_iterations =
+        qcrypto_pbkdf2_count_iters(luks_opts.hash_alg,
+                                   masterkey, luks->header.key_bytes,
+                                   luks->header.master_key_salt,
+                                   QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                                   &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+
+    /* Why /= 8 ?  That matches cryptsetup, but there's no
+     * explanation why they chose /= 8... Probably so that
+     * if all 8 keyslots are active we only spend 1 second
+     * in total time to check all keys */
+    luks->header.master_key_iterations /= 8;
+    luks->header.master_key_iterations = MAX(
+        luks->header.master_key_iterations,
+        QCRYPTO_BLOCK_LUKS_MIN_MASTER_KEY_ITERS);
+
+
+    /* Hash the master key, saving the result in the LUKS
+     * header. This hash is used when opening the encrypted
+     * device to verify that the user password unlocked a
+     * valid master key
+     */
+    if (qcrypto_pbkdf2(luks_opts.hash_alg,
+                       masterkey, luks->header.key_bytes,
+                       luks->header.master_key_salt,
+                       QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       luks->header.master_key_iterations,
+                       luks->header.master_key_digest,
+                       QCRYPTO_BLOCK_LUKS_DIGEST_LEN,
+                       errp) < 0) {
+        goto error;
+    }
+
+
+    /* Although LUKS has multiple key slots, we're just going
+     * to use the first key slot */
+    splitkeylen = luks->header.key_bytes * QCRYPTO_BLOCK_LUKS_STRIPES;
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        luks->header.key_slots[i].active = i == 0 ?
+            QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED :
+            QCRYPTO_BLOCK_LUKS_KEY_SLOT_DISABLED;
+        luks->header.key_slots[i].stripes = QCRYPTO_BLOCK_LUKS_STRIPES;
+
+        /* This calculation doesn't match that shown in the spec,
+         * but instead follows the cryptsetup implementation.
+         */
+        luks->header.key_slots[i].key_offset =
+            (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+             QCRYPTO_BLOCK_LUKS_SECTOR_SIZE) +
+            (ROUND_UP(DIV_ROUND_UP(splitkeylen, QCRYPTO_BLOCK_LUKS_SECTOR_SIZE),
+                      (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+                       QCRYPTO_BLOCK_LUKS_SECTOR_SIZE)) * i);
+    }
+
+    if (qcrypto_random_bytes(luks->header.key_slots[0].salt,
+                             QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                             errp) < 0) {
+        goto error;
+    }
+
+    /* Again we determine how many iterations are required to
+     * hash the user password while consuming 1 second of compute
+     * time */
+    luks->header.key_slots[0].iterations =
+        qcrypto_pbkdf2_count_iters(luks_opts.hash_alg,
+                                   (uint8_t *)password, strlen(password),
+                                   luks->header.key_slots[0].salt,
+                                   QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                                   &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+    /* Why /= 2 ?  That matches cryptsetup, but there's no
+     * explanation why they chose /= 2... */
+    luks->header.key_slots[0].iterations /= 2;
+    luks->header.key_slots[0].iterations = MAX(
+        luks->header.key_slots[0].iterations,
+        QCRYPTO_BLOCK_LUKS_MIN_SLOT_KEY_ITERS);
+
+
+    /* Generate a key that we'll use to encrypt the master
+     * key, from the user's password
+     */
+    slotkey = g_new0(uint8_t, luks->header.key_bytes);
+    if (qcrypto_pbkdf2(luks_opts.hash_alg,
+                       (uint8_t *)password, strlen(password),
+                       luks->header.key_slots[0].salt,
+                       QCRYPTO_BLOCK_LUKS_SALT_LEN,
+                       luks->header.key_slots[0].iterations,
+                       slotkey, luks->header.key_bytes,
+                       errp) < 0) {
+        goto error;
+    }
+
+
+    /* Setup the encryption objects needed to encrypt the
+     * master key material
+     */
+    cipher = qcrypto_cipher_new(luks_opts.cipher_alg,
+                                luks_opts.cipher_mode,
+                                slotkey, luks->header.key_bytes,
+                                errp);
+    if (!cipher) {
+        goto error;
+    }
+
+    ivgen = qcrypto_ivgen_new(luks_opts.ivgen_alg,
+                              ivcipheralg,
+                              luks_opts.ivgen_hash_alg,
+                              slotkey, luks->header.key_bytes,
+                              errp);
+    if (!ivgen) {
+        goto error;
+    }
+
+    /* Before storing the master key, we need to vastly
+     * increase its size, as protection against forensic
+     * disk data recovery */
+    splitkey = g_new0(uint8_t, splitkeylen);
+
+    if (qcrypto_afsplit_encode(luks_opts.hash_alg,
+                               luks->header.key_bytes,
+                               luks->header.key_slots[0].stripes,
+                               masterkey,
+                               splitkey,
+                               errp) < 0) {
+        goto error;
+    }
+
+    /* Now we encrypt the split master key with the key generated
+     * from the user's password, before storing it */
+    if (qcrypto_block_encrypt_helper(cipher, block->niv, ivgen,
+                                     QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                     0,
+                                     splitkey,
+                                     splitkeylen,
+                                     errp) < 0) {
+        goto error;
+    }
+
+
+    /* The total size of the LUKS headers is the partition header + key
+     * slot headers, rounded up to the nearest sector, combined with
+     * the size of each master key material region, also rounded up
+     * to the nearest sector */
+    luks->header.payload_offset =
+        (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+         QCRYPTO_BLOCK_LUKS_SECTOR_SIZE) +
+        (ROUND_UP(DIV_ROUND_UP(splitkeylen, QCRYPTO_BLOCK_LUKS_SECTOR_SIZE),
+                  (QCRYPTO_BLOCK_LUKS_KEY_SLOT_OFFSET /
+                   QCRYPTO_BLOCK_LUKS_SECTOR_SIZE)) *
+         QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS);
+
+    block->payload_offset = luks->header.payload_offset *
+        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE;
+
+    /* Reserve header space to match payload offset */
+    initfunc(block, block->payload_offset, &local_err, opaque);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+
+    /* Everything on disk uses Big Endian, so flip header fields
+     * before writing them */
+    cpu_to_be16s(&luks->header.version);
+    cpu_to_be32s(&luks->header.payload_offset);
+    cpu_to_be32s(&luks->header.key_bytes);
+    cpu_to_be32s(&luks->header.master_key_iterations);
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        cpu_to_be32s(&luks->header.key_slots[i].active);
+        cpu_to_be32s(&luks->header.key_slots[i].iterations);
+        cpu_to_be32s(&luks->header.key_slots[i].key_offset);
+        cpu_to_be32s(&luks->header.key_slots[i].stripes);
+    }
+
+
+    /* Write out the partition header and key slot headers */
+    writefunc(block, 0,
+              (const uint8_t *)&luks->header,
+              sizeof(luks->header),
+              &local_err,
+              opaque);
+
+    /* Delay checking local_err until we've byte-swapped */
+
+    /* Byte swap the header back to native, in case we need
+     * to read it again later */
+    be16_to_cpus(&luks->header.version);
+    be32_to_cpus(&luks->header.payload_offset);
+    be32_to_cpus(&luks->header.key_bytes);
+    be32_to_cpus(&luks->header.master_key_iterations);
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        be32_to_cpus(&luks->header.key_slots[i].active);
+        be32_to_cpus(&luks->header.key_slots[i].iterations);
+        be32_to_cpus(&luks->header.key_slots[i].key_offset);
+        be32_to_cpus(&luks->header.key_slots[i].stripes);
+    }
+
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto error;
+    }
+
+    /* Write out the master key material, starting at the
+     * sector immediately following the partition header. */
+    if (writefunc(block,
+                  luks->header.key_slots[0].key_offset *
+                  QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                  splitkey, splitkeylen,
+                  errp,
+                  opaque) != splitkeylen) {
+        goto error;
+    }
+
+    luks->cipher_alg = luks_opts.cipher_alg;
+    luks->cipher_mode = luks_opts.cipher_mode;
+    luks->ivgen_alg = luks_opts.ivgen_alg;
+    luks->ivgen_hash_alg = luks_opts.ivgen_hash_alg;
+    luks->hash_alg = luks_opts.hash_alg;
+
+    memset(masterkey, 0, luks->header.key_bytes);
+    g_free(masterkey);
+    memset(slotkey, 0, luks->header.key_bytes);
+    g_free(slotkey);
+    g_free(splitkey);
+    g_free(password);
+    g_free(cipher_mode_spec);
+
+    qcrypto_ivgen_free(ivgen);
+    qcrypto_cipher_free(cipher);
+
+    return 0;
+
+ error:
+    if (masterkey) {
+        memset(masterkey, 0, luks->header.key_bytes);
+    }
+    g_free(masterkey);
+    if (slotkey) {
+        memset(slotkey, 0, luks->header.key_bytes);
+    }
+    g_free(slotkey);
+    g_free(splitkey);
+    g_free(password);
+    g_free(cipher_mode_spec);
+
+    qcrypto_ivgen_free(ivgen);
+    qcrypto_cipher_free(cipher);
+
+    g_free(luks);
+    return -1;
+}
+
+
+static int qcrypto_block_luks_get_info(QCryptoBlock *block,
+                                       QCryptoBlockInfo *info,
+                                       Error **errp)
+{
+    QCryptoBlockLUKS *luks = block->opaque;
+    QCryptoBlockInfoLUKSSlot *slot;
+    QCryptoBlockInfoLUKSSlotList *slots = NULL, **prev = &info->u.luks.slots;
+    size_t i;
+
+    info->u.luks.cipher_alg = luks->cipher_alg;
+    info->u.luks.cipher_mode = luks->cipher_mode;
+    info->u.luks.ivgen_alg = luks->ivgen_alg;
+    if (info->u.luks.ivgen_alg == QCRYPTO_IVGEN_ALG_ESSIV) {
+        info->u.luks.has_ivgen_hash_alg = true;
+        info->u.luks.ivgen_hash_alg = luks->ivgen_hash_alg;
+    }
+    info->u.luks.hash_alg = luks->hash_alg;
+    info->u.luks.payload_offset = block->payload_offset;
+    info->u.luks.master_key_iters = luks->header.master_key_iterations;
+    info->u.luks.uuid = g_strndup((const char *)luks->header.uuid,
+                                  sizeof(luks->header.uuid));
+
+    for (i = 0; i < QCRYPTO_BLOCK_LUKS_NUM_KEY_SLOTS; i++) {
+        slots = g_new0(QCryptoBlockInfoLUKSSlotList, 1);
+        *prev = slots;
+
+        slots->value = slot = g_new0(QCryptoBlockInfoLUKSSlot, 1);
+        slot->active = luks->header.key_slots[i].active ==
+            QCRYPTO_BLOCK_LUKS_KEY_SLOT_ENABLED;
+        slot->key_offset = luks->header.key_slots[i].key_offset
+             * QCRYPTO_BLOCK_LUKS_SECTOR_SIZE;
+        if (slot->active) {
+            slot->has_iters = true;
+            slot->iters = luks->header.key_slots[i].iterations;
+            slot->has_stripes = true;
+            slot->stripes = luks->header.key_slots[i].stripes;
+        }
+
+        prev = &slots->next;
+    }
+
+    return 0;
+}
+
+
+static void qcrypto_block_luks_cleanup(QCryptoBlock *block)
+{
+    g_free(block->opaque);
+}
+
+
+static int
+qcrypto_block_luks_decrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_decrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+static int
+qcrypto_block_luks_encrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_encrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_LUKS_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+const QCryptoBlockDriver qcrypto_block_driver_luks = {
+    .open = qcrypto_block_luks_open,
+    .create = qcrypto_block_luks_create,
+    .get_info = qcrypto_block_luks_get_info,
+    .cleanup = qcrypto_block_luks_cleanup,
+    .decrypt = qcrypto_block_luks_decrypt,
+    .encrypt = qcrypto_block_luks_encrypt,
+    .has_format = qcrypto_block_luks_has_format,
+};
diff --git a/crypto/block-luks.h b/crypto/block-luks.h
new file mode 100644
index 0000000..b2d8a35
--- /dev/null
+++ b/crypto/block-luks.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block device encryption LUKS format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_LUKS_H
+#define QCRYPTO_BLOCK_LUKS_H
+
+#include "crypto/blockpriv.h"
+
+extern const QCryptoBlockDriver qcrypto_block_driver_luks;
+
+#endif /* QCRYPTO_BLOCK_LUKS_H */
diff --git a/crypto/block-qcow.c b/crypto/block-qcow.c
new file mode 100644
index 0000000..be88c6f
--- /dev/null
+++ b/crypto/block-qcow.c
@@ -0,0 +1,174 @@
+/*
+ * QEMU Crypto block device encryption QCow/QCow2 AES-CBC format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * Note that the block encryption implemented in this file is broken
+ * by design. This exists only to allow data to be liberated from
+ * existing qcow[2] images and should not be used in any new areas.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+
+#include "crypto/block-qcow.h"
+#include "crypto/secret.h"
+
+#define QCRYPTO_BLOCK_QCOW_SECTOR_SIZE 512
+
+
+static bool
+qcrypto_block_qcow_has_format(const uint8_t *buf G_GNUC_UNUSED,
+                              size_t buf_size G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+static int
+qcrypto_block_qcow_init(QCryptoBlock *block,
+                        const char *keysecret,
+                        Error **errp)
+{
+    char *password;
+    int ret;
+    uint8_t keybuf[16];
+    int len;
+
+    memset(keybuf, 0, 16);
+
+    password = qcrypto_secret_lookup_as_utf8(keysecret, errp);
+    if (!password) {
+        return -1;
+    }
+
+    len = strlen(password);
+    memcpy(keybuf, password, MIN(len, sizeof(keybuf)));
+    g_free(password);
+
+    block->niv = qcrypto_cipher_get_iv_len(QCRYPTO_CIPHER_ALG_AES_128,
+                                           QCRYPTO_CIPHER_MODE_CBC);
+    block->ivgen = qcrypto_ivgen_new(QCRYPTO_IVGEN_ALG_PLAIN64,
+                                     0, 0, NULL, 0, errp);
+    if (!block->ivgen) {
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    block->cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+                                       QCRYPTO_CIPHER_MODE_CBC,
+                                       keybuf, G_N_ELEMENTS(keybuf),
+                                       errp);
+    if (!block->cipher) {
+        ret = -ENOTSUP;
+        goto fail;
+    }
+
+    block->payload_offset = 0;
+
+    return 0;
+
+ fail:
+    qcrypto_cipher_free(block->cipher);
+    qcrypto_ivgen_free(block->ivgen);
+    return ret;
+}
+
+
+static int
+qcrypto_block_qcow_open(QCryptoBlock *block,
+                        QCryptoBlockOpenOptions *options,
+                        QCryptoBlockReadFunc readfunc G_GNUC_UNUSED,
+                        void *opaque G_GNUC_UNUSED,
+                        unsigned int flags,
+                        Error **errp)
+{
+    if (flags & QCRYPTO_BLOCK_OPEN_NO_IO) {
+        return 0;
+    } else {
+        if (!options->u.qcow.key_secret) {
+            error_setg(errp,
+                       "Parameter 'key-secret' is required for cipher");
+            return -1;
+        }
+        return qcrypto_block_qcow_init(block,
+                                       options->u.qcow.key_secret, errp);
+    }
+}
+
+
+static int
+qcrypto_block_qcow_create(QCryptoBlock *block,
+                          QCryptoBlockCreateOptions *options,
+                          QCryptoBlockInitFunc initfunc G_GNUC_UNUSED,
+                          QCryptoBlockWriteFunc writefunc G_GNUC_UNUSED,
+                          void *opaque G_GNUC_UNUSED,
+                          Error **errp)
+{
+    if (!options->u.qcow.key_secret) {
+        error_setg(errp, "Parameter 'key-secret' is required for cipher");
+        return -1;
+    }
+    /* QCow2 has no special header, since everything is hardwired */
+    return qcrypto_block_qcow_init(block, options->u.qcow.key_secret, errp);
+}
+
+
+static void
+qcrypto_block_qcow_cleanup(QCryptoBlock *block)
+{
+}
+
+
+static int
+qcrypto_block_qcow_decrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_decrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_QCOW_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+static int
+qcrypto_block_qcow_encrypt(QCryptoBlock *block,
+                           uint64_t startsector,
+                           uint8_t *buf,
+                           size_t len,
+                           Error **errp)
+{
+    return qcrypto_block_encrypt_helper(block->cipher,
+                                        block->niv, block->ivgen,
+                                        QCRYPTO_BLOCK_QCOW_SECTOR_SIZE,
+                                        startsector, buf, len, errp);
+}
+
+
+const QCryptoBlockDriver qcrypto_block_driver_qcow = {
+    .open = qcrypto_block_qcow_open,
+    .create = qcrypto_block_qcow_create,
+    .cleanup = qcrypto_block_qcow_cleanup,
+    .decrypt = qcrypto_block_qcow_decrypt,
+    .encrypt = qcrypto_block_qcow_encrypt,
+    .has_format = qcrypto_block_qcow_has_format,
+};
diff --git a/crypto/block-qcow.h b/crypto/block-qcow.h
new file mode 100644
index 0000000..3e2c0a8
--- /dev/null
+++ b/crypto/block-qcow.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block device encryption QCow/QCow2 AES-CBC format
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_QCOW_H
+#define QCRYPTO_BLOCK_QCOW_H
+
+#include "crypto/blockpriv.h"
+
+extern const QCryptoBlockDriver qcrypto_block_driver_qcow;
+
+#endif /* QCRYPTO_BLOCK_QCOW_H */
diff --git a/crypto/block.c b/crypto/block.c
new file mode 100644
index 0000000..be823ee
--- /dev/null
+++ b/crypto/block.c
@@ -0,0 +1,278 @@
+/*
+ * QEMU Crypto block device encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/blockpriv.h"
+#include "crypto/block-qcow.h"
+#include "crypto/block-luks.h"
+
+static const QCryptoBlockDriver *qcrypto_block_drivers[] = {
+    [Q_CRYPTO_BLOCK_FORMAT_QCOW] = &qcrypto_block_driver_qcow,
+    [Q_CRYPTO_BLOCK_FORMAT_LUKS] = &qcrypto_block_driver_luks,
+};
+
+
+bool qcrypto_block_has_format(QCryptoBlockFormat format,
+                              const uint8_t *buf,
+                              size_t len)
+{
+    const QCryptoBlockDriver *driver;
+
+    if (format >= G_N_ELEMENTS(qcrypto_block_drivers) ||
+        !qcrypto_block_drivers[format]) {
+        return false;
+    }
+
+    driver = qcrypto_block_drivers[format];
+
+    return driver->has_format(buf, len);
+}
+
+
+QCryptoBlock *qcrypto_block_open(QCryptoBlockOpenOptions *options,
+                                 QCryptoBlockReadFunc readfunc,
+                                 void *opaque,
+                                 unsigned int flags,
+                                 Error **errp)
+{
+    QCryptoBlock *block = g_new0(QCryptoBlock, 1);
+
+    block->format = options->format;
+
+    if (options->format >= G_N_ELEMENTS(qcrypto_block_drivers) ||
+        !qcrypto_block_drivers[options->format]) {
+        error_setg(errp, "Unsupported block driver %d", options->format);
+        g_free(block);
+        return NULL;
+    }
+
+    block->driver = qcrypto_block_drivers[options->format];
+
+    if (block->driver->open(block, options,
+                            readfunc, opaque, flags, errp) < 0) {
+        g_free(block);
+        return NULL;
+    }
+
+    return block;
+}
+
+
+QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
+                                   QCryptoBlockInitFunc initfunc,
+                                   QCryptoBlockWriteFunc writefunc,
+                                   void *opaque,
+                                   Error **errp)
+{
+    QCryptoBlock *block = g_new0(QCryptoBlock, 1);
+
+    block->format = options->format;
+
+    if (options->format >= G_N_ELEMENTS(qcrypto_block_drivers) ||
+        !qcrypto_block_drivers[options->format]) {
+        error_setg(errp, "Unsupported block driver %d", options->format);
+        g_free(block);
+        return NULL;
+    }
+
+    block->driver = qcrypto_block_drivers[options->format];
+
+    if (block->driver->create(block, options, initfunc,
+                              writefunc, opaque, errp) < 0) {
+        g_free(block);
+        return NULL;
+    }
+
+    return block;
+}
+
+
+QCryptoBlockInfo *qcrypto_block_get_info(QCryptoBlock *block,
+                                         Error **errp)
+{
+    QCryptoBlockInfo *info = g_new0(QCryptoBlockInfo, 1);
+
+    info->format = block->format;
+
+    if (block->driver->get_info &&
+        block->driver->get_info(block, info, errp) < 0) {
+        g_free(info);
+        return NULL;
+    }
+
+    return info;
+}
+
+
+int qcrypto_block_decrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp)
+{
+    return block->driver->decrypt(block, startsector, buf, len, errp);
+}
+
+
+int qcrypto_block_encrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp)
+{
+    return block->driver->encrypt(block, startsector, buf, len, errp);
+}
+
+
+QCryptoCipher *qcrypto_block_get_cipher(QCryptoBlock *block)
+{
+    return block->cipher;
+}
+
+
+QCryptoIVGen *qcrypto_block_get_ivgen(QCryptoBlock *block)
+{
+    return block->ivgen;
+}
+
+
+QCryptoHashAlgorithm qcrypto_block_get_kdf_hash(QCryptoBlock *block)
+{
+    return block->kdfhash;
+}
+
+
+uint64_t qcrypto_block_get_payload_offset(QCryptoBlock *block)
+{
+    return block->payload_offset;
+}
+
+
+void qcrypto_block_free(QCryptoBlock *block)
+{
+    if (!block) {
+        return;
+    }
+
+    block->driver->cleanup(block);
+
+    qcrypto_cipher_free(block->cipher);
+    qcrypto_ivgen_free(block->ivgen);
+    g_free(block);
+}
+
+
+int qcrypto_block_decrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp)
+{
+    uint8_t *iv;
+    int ret = -1;
+
+    iv = niv ? g_new0(uint8_t, niv) : NULL;
+
+    while (len > 0) {
+        size_t nbytes;
+        if (niv) {
+            if (qcrypto_ivgen_calculate(ivgen,
+                                        startsector,
+                                        iv, niv,
+                                        errp) < 0) {
+                goto cleanup;
+            }
+
+            if (qcrypto_cipher_setiv(cipher,
+                                     iv, niv,
+                                     errp) < 0) {
+                goto cleanup;
+            }
+        }
+
+        nbytes = len > sectorsize ? sectorsize : len;
+        if (qcrypto_cipher_decrypt(cipher, buf, buf,
+                                   nbytes, errp) < 0) {
+            goto cleanup;
+        }
+
+        startsector++;
+        buf += nbytes;
+        len -= nbytes;
+    }
+
+    ret = 0;
+ cleanup:
+    g_free(iv);
+    return ret;
+}
+
+
+int qcrypto_block_encrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp)
+{
+    uint8_t *iv;
+    int ret = -1;
+
+    iv = niv ? g_new0(uint8_t, niv) : NULL;
+
+    while (len > 0) {
+        size_t nbytes;
+        if (niv) {
+            if (qcrypto_ivgen_calculate(ivgen,
+                                        startsector,
+                                        iv, niv,
+                                        errp) < 0) {
+                goto cleanup;
+            }
+
+            if (qcrypto_cipher_setiv(cipher,
+                                     iv, niv,
+                                     errp) < 0) {
+                goto cleanup;
+            }
+        }
+
+        nbytes = len > sectorsize ? sectorsize : len;
+        if (qcrypto_cipher_encrypt(cipher, buf, buf,
+                                   nbytes, errp) < 0) {
+            goto cleanup;
+        }
+
+        startsector++;
+        buf += nbytes;
+        len -= nbytes;
+    }
+
+    ret = 0;
+ cleanup:
+    g_free(iv);
+    return ret;
+}
diff --git a/crypto/blockpriv.h b/crypto/blockpriv.h
new file mode 100644
index 0000000..68f0f06
--- /dev/null
+++ b/crypto/blockpriv.h
@@ -0,0 +1,96 @@
+/*
+ * QEMU Crypto block device encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCKPRIV_H
+#define QCRYPTO_BLOCKPRIV_H
+
+#include "crypto/block.h"
+
+typedef struct QCryptoBlockDriver QCryptoBlockDriver;
+
+struct QCryptoBlock {
+    QCryptoBlockFormat format;
+
+    const QCryptoBlockDriver *driver;
+    void *opaque;
+
+    QCryptoCipher *cipher;
+    QCryptoIVGen *ivgen;
+    QCryptoHashAlgorithm kdfhash;
+    size_t niv;
+    uint64_t payload_offset; /* In bytes */
+};
+
+struct QCryptoBlockDriver {
+    int (*open)(QCryptoBlock *block,
+                QCryptoBlockOpenOptions *options,
+                QCryptoBlockReadFunc readfunc,
+                void *opaque,
+                unsigned int flags,
+                Error **errp);
+
+    int (*create)(QCryptoBlock *block,
+                  QCryptoBlockCreateOptions *options,
+                  QCryptoBlockInitFunc initfunc,
+                  QCryptoBlockWriteFunc writefunc,
+                  void *opaque,
+                  Error **errp);
+
+    int (*get_info)(QCryptoBlock *block,
+                    QCryptoBlockInfo *info,
+                    Error **errp);
+
+    void (*cleanup)(QCryptoBlock *block);
+
+    int (*encrypt)(QCryptoBlock *block,
+                   uint64_t startsector,
+                   uint8_t *buf,
+                   size_t len,
+                   Error **errp);
+    int (*decrypt)(QCryptoBlock *block,
+                   uint64_t startsector,
+                   uint8_t *buf,
+                   size_t len,
+                   Error **errp);
+
+    bool (*has_format)(const uint8_t *buf,
+                       size_t buflen);
+};
+
+
+int qcrypto_block_decrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp);
+
+int qcrypto_block_encrypt_helper(QCryptoCipher *cipher,
+                                 size_t niv,
+                                 QCryptoIVGen *ivgen,
+                                 int sectorsize,
+                                 uint64_t startsector,
+                                 uint8_t *buf,
+                                 size_t len,
+                                 Error **errp);
+
+#endif /* QCRYPTO_BLOCKPRIV_H */
diff --git a/crypto/cipher-builtin.c b/crypto/cipher-builtin.c
new file mode 100644
index 0000000..88963f6
--- /dev/null
+++ b/crypto/cipher-builtin.c
@@ -0,0 +1,513 @@
+/*
+ * QEMU Crypto cipher built-in algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/aes.h"
+#include "crypto/desrfb.h"
+#include "crypto/xts.h"
+
+typedef struct QCryptoCipherBuiltinAESContext QCryptoCipherBuiltinAESContext;
+struct QCryptoCipherBuiltinAESContext {
+    AES_KEY enc;
+    AES_KEY dec;
+};
+typedef struct QCryptoCipherBuiltinAES QCryptoCipherBuiltinAES;
+struct QCryptoCipherBuiltinAES {
+    QCryptoCipherBuiltinAESContext key;
+    QCryptoCipherBuiltinAESContext key_tweak;
+    uint8_t iv[AES_BLOCK_SIZE];
+};
+typedef struct QCryptoCipherBuiltinDESRFB QCryptoCipherBuiltinDESRFB;
+struct QCryptoCipherBuiltinDESRFB {
+    uint8_t *key;
+    size_t nkey;
+};
+
+typedef struct QCryptoCipherBuiltin QCryptoCipherBuiltin;
+struct QCryptoCipherBuiltin {
+    union {
+        QCryptoCipherBuiltinAES aes;
+        QCryptoCipherBuiltinDESRFB desrfb;
+    } state;
+    size_t blocksize;
+    void (*free)(QCryptoCipher *cipher);
+    int (*setiv)(QCryptoCipher *cipher,
+                 const uint8_t *iv, size_t niv,
+                 Error **errp);
+    int (*encrypt)(QCryptoCipher *cipher,
+                   const void *in,
+                   void *out,
+                   size_t len,
+                   Error **errp);
+    int (*decrypt)(QCryptoCipher *cipher,
+                   const void *in,
+                   void *out,
+                   size_t len,
+                   Error **errp);
+};
+
+
+static void qcrypto_cipher_free_aes(QCryptoCipher *cipher)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    g_free(ctxt);
+    cipher->opaque = NULL;
+}
+
+
+static void qcrypto_cipher_aes_ecb_encrypt(AES_KEY *key,
+                                           const void *in,
+                                           void *out,
+                                           size_t len)
+{
+    const uint8_t *inptr = in;
+    uint8_t *outptr = out;
+    while (len) {
+        if (len > AES_BLOCK_SIZE) {
+            AES_encrypt(inptr, outptr, key);
+            inptr += AES_BLOCK_SIZE;
+            outptr += AES_BLOCK_SIZE;
+            len -= AES_BLOCK_SIZE;
+        } else {
+            uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+            memcpy(tmp1, inptr, len);
+            /* Fill with 0 to avoid valgrind uninitialized reads */
+            memset(tmp1 + len, 0, sizeof(tmp1) - len);
+            AES_encrypt(tmp1, tmp2, key);
+            memcpy(outptr, tmp2, len);
+            len = 0;
+        }
+    }
+}
+
+
+static void qcrypto_cipher_aes_ecb_decrypt(AES_KEY *key,
+                                           const void *in,
+                                           void *out,
+                                           size_t len)
+{
+    const uint8_t *inptr = in;
+    uint8_t *outptr = out;
+    while (len) {
+        if (len > AES_BLOCK_SIZE) {
+            AES_decrypt(inptr, outptr, key);
+            inptr += AES_BLOCK_SIZE;
+            outptr += AES_BLOCK_SIZE;
+            len -= AES_BLOCK_SIZE;
+        } else {
+            uint8_t tmp1[AES_BLOCK_SIZE], tmp2[AES_BLOCK_SIZE];
+            memcpy(tmp1, inptr, len);
+            /* Fill with 0 to avoid valgrind uninitialized reads */
+            memset(tmp1 + len, 0, sizeof(tmp1) - len);
+            AES_decrypt(tmp1, tmp2, key);
+            memcpy(outptr, tmp2, len);
+            len = 0;
+        }
+    }
+}
+
+
+static void qcrypto_cipher_aes_xts_encrypt(const void *ctx,
+                                           size_t length,
+                                           uint8_t *dst,
+                                           const uint8_t *src)
+{
+    const QCryptoCipherBuiltinAESContext *aesctx = ctx;
+
+    qcrypto_cipher_aes_ecb_encrypt((AES_KEY *)&aesctx->enc,
+                                   src, dst, length);
+}
+
+
+static void qcrypto_cipher_aes_xts_decrypt(const void *ctx,
+                                           size_t length,
+                                           uint8_t *dst,
+                                           const uint8_t *src)
+{
+    const QCryptoCipherBuiltinAESContext *aesctx = ctx;
+
+    qcrypto_cipher_aes_ecb_decrypt((AES_KEY *)&aesctx->dec,
+                                   src, dst, length);
+}
+
+
+static int qcrypto_cipher_encrypt_aes(QCryptoCipher *cipher,
+                                      const void *in,
+                                      void *out,
+                                      size_t len,
+                                      Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        qcrypto_cipher_aes_ecb_encrypt(&ctxt->state.aes.key.enc,
+                                       in, out, len);
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
+        AES_cbc_encrypt(in, out, len,
+                        &ctxt->state.aes.key.enc,
+                        ctxt->state.aes.iv, 1);
+        break;
+    case QCRYPTO_CIPHER_MODE_XTS:
+        xts_encrypt(&ctxt->state.aes.key,
+                    &ctxt->state.aes.key_tweak,
+                    qcrypto_cipher_aes_xts_encrypt,
+                    qcrypto_cipher_aes_xts_decrypt,
+                    ctxt->state.aes.iv,
+                    len, out, in);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    return 0;
+}
+
+
+static int qcrypto_cipher_decrypt_aes(QCryptoCipher *cipher,
+                                      const void *in,
+                                      void *out,
+                                      size_t len,
+                                      Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        qcrypto_cipher_aes_ecb_decrypt(&ctxt->state.aes.key.dec,
+                                       in, out, len);
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
+        AES_cbc_encrypt(in, out, len,
+                        &ctxt->state.aes.key.dec,
+                        ctxt->state.aes.iv, 0);
+        break;
+    case QCRYPTO_CIPHER_MODE_XTS:
+        xts_decrypt(&ctxt->state.aes.key,
+                    &ctxt->state.aes.key_tweak,
+                    qcrypto_cipher_aes_xts_encrypt,
+                    qcrypto_cipher_aes_xts_decrypt,
+                    ctxt->state.aes.iv,
+                    len, out, in);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    return 0;
+}
+
+static int qcrypto_cipher_setiv_aes(QCryptoCipher *cipher,
+                                     const uint8_t *iv, size_t niv,
+                                     Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+    if (niv != AES_BLOCK_SIZE) {
+        error_setg(errp, "IV must be %d bytes not %zu",
+                   AES_BLOCK_SIZE, niv);
+        return -1;
+    }
+
+    memcpy(ctxt->state.aes.iv, iv, AES_BLOCK_SIZE);
+
+    return 0;
+}
+
+
+
+
+static int qcrypto_cipher_init_aes(QCryptoCipher *cipher,
+                                   const uint8_t *key, size_t nkey,
+                                   Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt;
+
+    if (cipher->mode != QCRYPTO_CIPHER_MODE_CBC &&
+        cipher->mode != QCRYPTO_CIPHER_MODE_ECB &&
+        cipher->mode != QCRYPTO_CIPHER_MODE_XTS) {
+        error_setg(errp, "Unsupported cipher mode %d", cipher->mode);
+        return -1;
+    }
+
+    ctxt = g_new0(QCryptoCipherBuiltin, 1);
+
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        if (AES_set_encrypt_key(key, nkey * 4, &ctxt->state.aes.key.enc) != 0) {
+            error_setg(errp, "Failed to set encryption key");
+            goto error;
+        }
+
+        if (AES_set_decrypt_key(key, nkey * 4, &ctxt->state.aes.key.dec) != 0) {
+            error_setg(errp, "Failed to set decryption key");
+            goto error;
+        }
+
+        if (AES_set_encrypt_key(key + (nkey / 2), nkey * 4,
+                                &ctxt->state.aes.key_tweak.enc) != 0) {
+            error_setg(errp, "Failed to set encryption key");
+            goto error;
+        }
+
+        if (AES_set_decrypt_key(key + (nkey / 2), nkey * 4,
+                                &ctxt->state.aes.key_tweak.dec) != 0) {
+            error_setg(errp, "Failed to set decryption key");
+            goto error;
+        }
+    } else {
+        if (AES_set_encrypt_key(key, nkey * 8, &ctxt->state.aes.key.enc) != 0) {
+            error_setg(errp, "Failed to set encryption key");
+            goto error;
+        }
+
+        if (AES_set_decrypt_key(key, nkey * 8, &ctxt->state.aes.key.dec) != 0) {
+            error_setg(errp, "Failed to set decryption key");
+            goto error;
+        }
+    }
+
+    ctxt->blocksize = AES_BLOCK_SIZE;
+    ctxt->free = qcrypto_cipher_free_aes;
+    ctxt->setiv = qcrypto_cipher_setiv_aes;
+    ctxt->encrypt = qcrypto_cipher_encrypt_aes;
+    ctxt->decrypt = qcrypto_cipher_decrypt_aes;
+
+    cipher->opaque = ctxt;
+
+    return 0;
+
+ error:
+    g_free(ctxt);
+    return -1;
+}
+
+
+static void qcrypto_cipher_free_des_rfb(QCryptoCipher *cipher)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    g_free(ctxt->state.desrfb.key);
+    g_free(ctxt);
+    cipher->opaque = NULL;
+}
+
+
+static int qcrypto_cipher_encrypt_des_rfb(QCryptoCipher *cipher,
+                                          const void *in,
+                                          void *out,
+                                          size_t len,
+                                          Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+    size_t i;
+
+    if (len % 8) {
+        error_setg(errp, "Buffer size must be multiple of 8 not %zu",
+                   len);
+        return -1;
+    }
+
+    deskey(ctxt->state.desrfb.key, EN0);
+
+    for (i = 0; i < len; i += 8) {
+        des((void *)in + i, out + i);
+    }
+
+    return 0;
+}
+
+
+static int qcrypto_cipher_decrypt_des_rfb(QCryptoCipher *cipher,
+                                          const void *in,
+                                          void *out,
+                                          size_t len,
+                                          Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+    size_t i;
+
+    if (len % 8) {
+        error_setg(errp, "Buffer size must be multiple of 8 not %zu",
+                   len);
+        return -1;
+    }
+
+    deskey(ctxt->state.desrfb.key, DE1);
+
+    for (i = 0; i < len; i += 8) {
+        des((void *)in + i, out + i);
+    }
+
+    return 0;
+}
+
+
+static int qcrypto_cipher_setiv_des_rfb(QCryptoCipher *cipher,
+                                        const uint8_t *iv, size_t niv,
+                                        Error **errp)
+{
+    error_setg(errp, "Setting IV is not supported");
+    return -1;
+}
+
+
+static int qcrypto_cipher_init_des_rfb(QCryptoCipher *cipher,
+                                       const uint8_t *key, size_t nkey,
+                                       Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt;
+
+    if (cipher->mode != QCRYPTO_CIPHER_MODE_ECB) {
+        error_setg(errp, "Unsupported cipher mode %d", cipher->mode);
+        return -1;
+    }
+
+    ctxt = g_new0(QCryptoCipherBuiltin, 1);
+
+    ctxt->state.desrfb.key = g_new0(uint8_t, nkey);
+    memcpy(ctxt->state.desrfb.key, key, nkey);
+    ctxt->state.desrfb.nkey = nkey;
+
+    ctxt->blocksize = 8;
+    ctxt->free = qcrypto_cipher_free_des_rfb;
+    ctxt->setiv = qcrypto_cipher_setiv_des_rfb;
+    ctxt->encrypt = qcrypto_cipher_encrypt_des_rfb;
+    ctxt->decrypt = qcrypto_cipher_decrypt_des_rfb;
+
+    cipher->opaque = ctxt;
+
+    return 0;
+}
+
+
+bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
+{
+    switch (alg) {
+    case QCRYPTO_CIPHER_ALG_DES_RFB:
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+
+QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
+                                  QCryptoCipherMode mode,
+                                  const uint8_t *key, size_t nkey,
+                                  Error **errp)
+{
+    QCryptoCipher *cipher;
+
+    cipher = g_new0(QCryptoCipher, 1);
+    cipher->alg = alg;
+    cipher->mode = mode;
+
+    if (!qcrypto_cipher_validate_key_length(alg, mode, nkey, errp)) {
+        goto error;
+    }
+
+    switch (cipher->alg) {
+    case QCRYPTO_CIPHER_ALG_DES_RFB:
+        if (qcrypto_cipher_init_des_rfb(cipher, key, nkey, errp) < 0) {
+            goto error;
+        }
+        break;
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+        if (qcrypto_cipher_init_aes(cipher, key, nkey, errp) < 0) {
+            goto error;
+        }
+        break;
+    default:
+        error_setg(errp,
+                   "Unsupported cipher algorithm %d", cipher->alg);
+        goto error;
+    }
+
+    return cipher;
+
+ error:
+    g_free(cipher);
+    return NULL;
+}
+
+void qcrypto_cipher_free(QCryptoCipher *cipher)
+{
+    QCryptoCipherBuiltin *ctxt;
+
+    if (!cipher) {
+        return;
+    }
+
+    ctxt = cipher->opaque;
+    ctxt->free(cipher);
+    g_free(cipher);
+}
+
+
+int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    if (len % ctxt->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctxt->blocksize);
+        return -1;
+    }
+
+    return ctxt->encrypt(cipher, in, out, len, errp);
+}
+
+
+int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    if (len % ctxt->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctxt->blocksize);
+        return -1;
+    }
+
+    return ctxt->decrypt(cipher, in, out, len, errp);
+}
+
+
+int qcrypto_cipher_setiv(QCryptoCipher *cipher,
+                         const uint8_t *iv, size_t niv,
+                         Error **errp)
+{
+    QCryptoCipherBuiltin *ctxt = cipher->opaque;
+
+    return ctxt->setiv(cipher, iv, niv, errp);
+}
diff --git a/crypto/cipher-gcrypt.c b/crypto/cipher-gcrypt.c
new file mode 100644
index 0000000..ede2f70
--- /dev/null
+++ b/crypto/cipher-gcrypt.c
@@ -0,0 +1,344 @@
+/*
+ * QEMU Crypto cipher libgcrypt algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/xts.h"
+
+#include <gcrypt.h>
+
+
+bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
+{
+    switch (alg) {
+    case QCRYPTO_CIPHER_ALG_DES_RFB:
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+typedef struct QCryptoCipherGcrypt QCryptoCipherGcrypt;
+struct QCryptoCipherGcrypt {
+    gcry_cipher_hd_t handle;
+    gcry_cipher_hd_t tweakhandle;
+    size_t blocksize;
+    uint8_t *iv;
+};
+
+QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
+                                  QCryptoCipherMode mode,
+                                  const uint8_t *key, size_t nkey,
+                                  Error **errp)
+{
+    QCryptoCipher *cipher;
+    QCryptoCipherGcrypt *ctx;
+    gcry_error_t err;
+    int gcryalg, gcrymode;
+
+    switch (mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+    case QCRYPTO_CIPHER_MODE_XTS:
+        gcrymode = GCRY_CIPHER_MODE_ECB;
+        break;
+    case QCRYPTO_CIPHER_MODE_CBC:
+        gcrymode = GCRY_CIPHER_MODE_CBC;
+        break;
+    default:
+        error_setg(errp, "Unsupported cipher mode %d", mode);
+        return NULL;
+    }
+
+    if (!qcrypto_cipher_validate_key_length(alg, mode, nkey, errp)) {
+        return NULL;
+    }
+
+    switch (alg) {
+    case QCRYPTO_CIPHER_ALG_DES_RFB:
+        gcryalg = GCRY_CIPHER_DES;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_AES_128:
+        gcryalg = GCRY_CIPHER_AES128;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_AES_192:
+        gcryalg = GCRY_CIPHER_AES192;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_AES_256:
+        gcryalg = GCRY_CIPHER_AES256;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+        gcryalg = GCRY_CIPHER_CAST5;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+        gcryalg = GCRY_CIPHER_SERPENT128;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+        gcryalg = GCRY_CIPHER_SERPENT192;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        gcryalg = GCRY_CIPHER_SERPENT256;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+        gcryalg = GCRY_CIPHER_TWOFISH128;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        gcryalg = GCRY_CIPHER_TWOFISH;
+        break;
+
+    default:
+        error_setg(errp, "Unsupported cipher algorithm %d", alg);
+        return NULL;
+    }
+
+    cipher = g_new0(QCryptoCipher, 1);
+    cipher->alg = alg;
+    cipher->mode = mode;
+
+    ctx = g_new0(QCryptoCipherGcrypt, 1);
+
+    err = gcry_cipher_open(&ctx->handle, gcryalg, gcrymode, 0);
+    if (err != 0) {
+        error_setg(errp, "Cannot initialize cipher: %s",
+                   gcry_strerror(err));
+        goto error;
+    }
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        err = gcry_cipher_open(&ctx->tweakhandle, gcryalg, gcrymode, 0);
+        if (err != 0) {
+            error_setg(errp, "Cannot initialize cipher: %s",
+                       gcry_strerror(err));
+            goto error;
+        }
+    }
+
+    if (cipher->alg == QCRYPTO_CIPHER_ALG_DES_RFB) {
+        /* We're using standard DES cipher from gcrypt, so we need
+         * to munge the key so that the results are the same as the
+         * bizarre RFB variant of DES :-)
+         */
+        uint8_t *rfbkey = qcrypto_cipher_munge_des_rfb_key(key, nkey);
+        err = gcry_cipher_setkey(ctx->handle, rfbkey, nkey);
+        g_free(rfbkey);
+        ctx->blocksize = 8;
+    } else {
+        if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+            nkey /= 2;
+            err = gcry_cipher_setkey(ctx->handle, key, nkey);
+            if (err != 0) {
+                error_setg(errp, "Cannot set key: %s",
+                           gcry_strerror(err));
+                goto error;
+            }
+            err = gcry_cipher_setkey(ctx->tweakhandle, key + nkey, nkey);
+        } else {
+            err = gcry_cipher_setkey(ctx->handle, key, nkey);
+        }
+        if (err != 0) {
+            error_setg(errp, "Cannot set key: %s",
+                       gcry_strerror(err));
+            goto error;
+        }
+        switch (cipher->alg) {
+        case QCRYPTO_CIPHER_ALG_AES_128:
+        case QCRYPTO_CIPHER_ALG_AES_192:
+        case QCRYPTO_CIPHER_ALG_AES_256:
+        case QCRYPTO_CIPHER_ALG_SERPENT_128:
+        case QCRYPTO_CIPHER_ALG_SERPENT_192:
+        case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+        case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+            ctx->blocksize = 16;
+            break;
+        case QCRYPTO_CIPHER_ALG_CAST5_128:
+            ctx->blocksize = 8;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    }
+
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        ctx->iv = g_new0(uint8_t, ctx->blocksize);
+    }
+
+    cipher->opaque = ctx;
+    return cipher;
+
+ error:
+    gcry_cipher_close(ctx->handle);
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        gcry_cipher_close(ctx->tweakhandle);
+    }
+    g_free(ctx);
+    g_free(cipher);
+    return NULL;
+}
+
+
+void qcrypto_cipher_free(QCryptoCipher *cipher)
+{
+    QCryptoCipherGcrypt *ctx;
+    if (!cipher) {
+        return;
+    }
+    ctx = cipher->opaque;
+    gcry_cipher_close(ctx->handle);
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        gcry_cipher_close(ctx->tweakhandle);
+    }
+    g_free(ctx->iv);
+    g_free(ctx);
+    g_free(cipher);
+}
+
+
+static void qcrypto_gcrypt_xts_encrypt(const void *ctx,
+                                       size_t length,
+                                       uint8_t *dst,
+                                       const uint8_t *src)
+{
+    gcry_error_t err;
+    err = gcry_cipher_encrypt((gcry_cipher_hd_t)ctx, dst, length, src, length);
+    g_assert(err == 0);
+}
+
+static void qcrypto_gcrypt_xts_decrypt(const void *ctx,
+                                       size_t length,
+                                       uint8_t *dst,
+                                       const uint8_t *src)
+{
+    gcry_error_t err;
+    err = gcry_cipher_decrypt((gcry_cipher_hd_t)ctx, dst, length, src, length);
+    g_assert(err == 0);
+}
+
+int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp)
+{
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
+    gcry_error_t err;
+
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        xts_encrypt(ctx->handle, ctx->tweakhandle,
+                    qcrypto_gcrypt_xts_encrypt,
+                    qcrypto_gcrypt_xts_decrypt,
+                    ctx->iv, len, out, in);
+    } else {
+        err = gcry_cipher_encrypt(ctx->handle,
+                                  out, len,
+                                  in, len);
+        if (err != 0) {
+            error_setg(errp, "Cannot encrypt data: %s",
+                       gcry_strerror(err));
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp)
+{
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
+    gcry_error_t err;
+
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
+    if (cipher->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        xts_decrypt(ctx->handle, ctx->tweakhandle,
+                    qcrypto_gcrypt_xts_encrypt,
+                    qcrypto_gcrypt_xts_decrypt,
+                    ctx->iv, len, out, in);
+    } else {
+        err = gcry_cipher_decrypt(ctx->handle,
+                                  out, len,
+                                  in, len);
+        if (err != 0) {
+            error_setg(errp, "Cannot decrypt data: %s",
+                       gcry_strerror(err));
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+int qcrypto_cipher_setiv(QCryptoCipher *cipher,
+                         const uint8_t *iv, size_t niv,
+                         Error **errp)
+{
+    QCryptoCipherGcrypt *ctx = cipher->opaque;
+    gcry_error_t err;
+
+    if (niv != ctx->blocksize) {
+        error_setg(errp, "Expected IV size %zu not %zu",
+                   ctx->blocksize, niv);
+        return -1;
+    }
+
+    if (ctx->iv) {
+        memcpy(ctx->iv, iv, niv);
+    } else {
+        gcry_cipher_reset(ctx->handle);
+        err = gcry_cipher_setiv(ctx->handle, iv, niv);
+        if (err != 0) {
+            error_setg(errp, "Cannot set IV: %s",
+                   gcry_strerror(err));
+            return -1;
+        }
+    }
+
+    return 0;
+}
diff --git a/crypto/cipher-nettle.c b/crypto/cipher-nettle.c
new file mode 100644
index 0000000..70909fb
--- /dev/null
+++ b/crypto/cipher-nettle.c
@@ -0,0 +1,489 @@
+/*
+ * QEMU Crypto cipher nettle algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/xts.h"
+
+#include <nettle/nettle-types.h>
+#include <nettle/aes.h>
+#include <nettle/des.h>
+#include <nettle/cbc.h>
+#include <nettle/cast128.h>
+#include <nettle/serpent.h>
+#include <nettle/twofish.h>
+
+typedef void (*QCryptoCipherNettleFuncWrapper)(const void *ctx,
+                                               size_t length,
+                                               uint8_t *dst,
+                                               const uint8_t *src);
+
+#if CONFIG_NETTLE_VERSION_MAJOR < 3
+typedef nettle_crypt_func * QCryptoCipherNettleFuncNative;
+typedef void *       cipher_ctx_t;
+typedef unsigned     cipher_length_t;
+
+#define cast5_set_key cast128_set_key
+#else
+typedef nettle_cipher_func * QCryptoCipherNettleFuncNative;
+typedef const void * cipher_ctx_t;
+typedef size_t       cipher_length_t;
+#endif
+
+typedef struct QCryptoNettleAES {
+    struct aes_ctx enc;
+    struct aes_ctx dec;
+} QCryptoNettleAES;
+
+static void aes_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_encrypt(&aesctx->enc, length, dst, src);
+}
+
+static void aes_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_decrypt(&aesctx->dec, length, dst, src);
+}
+
+static void des_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    des_encrypt(ctx, length, dst, src);
+}
+
+static void des_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                               uint8_t *dst, const uint8_t *src)
+{
+    des_decrypt(ctx, length, dst, src);
+}
+
+static void cast128_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    cast128_encrypt(ctx, length, dst, src);
+}
+
+static void cast128_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    cast128_decrypt(ctx, length, dst, src);
+}
+
+static void serpent_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    serpent_encrypt(ctx, length, dst, src);
+}
+
+static void serpent_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    serpent_decrypt(ctx, length, dst, src);
+}
+
+static void twofish_encrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    twofish_encrypt(ctx, length, dst, src);
+}
+
+static void twofish_decrypt_native(cipher_ctx_t ctx, cipher_length_t length,
+                                   uint8_t *dst, const uint8_t *src)
+{
+    twofish_decrypt(ctx, length, dst, src);
+}
+
+static void aes_encrypt_wrapper(const void *ctx, size_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_encrypt(&aesctx->enc, length, dst, src);
+}
+
+static void aes_decrypt_wrapper(const void *ctx, size_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    const QCryptoNettleAES *aesctx = ctx;
+    aes_decrypt(&aesctx->dec, length, dst, src);
+}
+
+static void des_encrypt_wrapper(const void *ctx, size_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    des_encrypt(ctx, length, dst, src);
+}
+
+static void des_decrypt_wrapper(const void *ctx, size_t length,
+                                uint8_t *dst, const uint8_t *src)
+{
+    des_decrypt(ctx, length, dst, src);
+}
+
+static void cast128_encrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    cast128_encrypt(ctx, length, dst, src);
+}
+
+static void cast128_decrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    cast128_decrypt(ctx, length, dst, src);
+}
+
+static void serpent_encrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    serpent_encrypt(ctx, length, dst, src);
+}
+
+static void serpent_decrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    serpent_decrypt(ctx, length, dst, src);
+}
+
+static void twofish_encrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    twofish_encrypt(ctx, length, dst, src);
+}
+
+static void twofish_decrypt_wrapper(const void *ctx, size_t length,
+                                    uint8_t *dst, const uint8_t *src)
+{
+    twofish_decrypt(ctx, length, dst, src);
+}
+
+typedef struct QCryptoCipherNettle QCryptoCipherNettle;
+struct QCryptoCipherNettle {
+    /* Primary cipher context for all modes */
+    void *ctx;
+    /* Second cipher context for XTS mode only */
+    void *ctx_tweak;
+    /* Cipher callbacks for both contexts */
+    QCryptoCipherNettleFuncNative alg_encrypt_native;
+    QCryptoCipherNettleFuncNative alg_decrypt_native;
+    QCryptoCipherNettleFuncWrapper alg_encrypt_wrapper;
+    QCryptoCipherNettleFuncWrapper alg_decrypt_wrapper;
+
+    uint8_t *iv;
+    size_t blocksize;
+};
+
+bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg)
+{
+    switch (alg) {
+    case QCRYPTO_CIPHER_ALG_DES_RFB:
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_192:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+
+QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
+                                  QCryptoCipherMode mode,
+                                  const uint8_t *key, size_t nkey,
+                                  Error **errp)
+{
+    QCryptoCipher *cipher;
+    QCryptoCipherNettle *ctx;
+    uint8_t *rfbkey;
+
+    switch (mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+    case QCRYPTO_CIPHER_MODE_CBC:
+    case QCRYPTO_CIPHER_MODE_XTS:
+        break;
+    default:
+        error_setg(errp, "Unsupported cipher mode %d", mode);
+        return NULL;
+    }
+
+    if (!qcrypto_cipher_validate_key_length(alg, mode, nkey, errp)) {
+        return NULL;
+    }
+
+    cipher = g_new0(QCryptoCipher, 1);
+    cipher->alg = alg;
+    cipher->mode = mode;
+
+    ctx = g_new0(QCryptoCipherNettle, 1);
+
+    switch (alg) {
+    case QCRYPTO_CIPHER_ALG_DES_RFB:
+        ctx->ctx = g_new0(struct des_ctx, 1);
+        rfbkey = qcrypto_cipher_munge_des_rfb_key(key, nkey);
+        des_set_key(ctx->ctx, rfbkey);
+        g_free(rfbkey);
+
+        ctx->alg_encrypt_native = des_encrypt_native;
+        ctx->alg_decrypt_native = des_decrypt_native;
+        ctx->alg_encrypt_wrapper = des_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = des_decrypt_wrapper;
+
+        ctx->blocksize = DES_BLOCK_SIZE;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_AES_128:
+    case QCRYPTO_CIPHER_ALG_AES_192:
+    case QCRYPTO_CIPHER_ALG_AES_256:
+        ctx->ctx = g_new0(QCryptoNettleAES, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(QCryptoNettleAES, 1);
+
+            nkey /= 2;
+            aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx)->enc,
+                                nkey, key);
+            aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx)->dec,
+                                nkey, key);
+
+            aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx_tweak)->enc,
+                                nkey, key + nkey);
+            aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx_tweak)->dec,
+                                nkey, key + nkey);
+        } else {
+            aes_set_encrypt_key(&((QCryptoNettleAES *)ctx->ctx)->enc,
+                                nkey, key);
+            aes_set_decrypt_key(&((QCryptoNettleAES *)ctx->ctx)->dec,
+                                nkey, key);
+        }
+
+        ctx->alg_encrypt_native = aes_encrypt_native;
+        ctx->alg_decrypt_native = aes_decrypt_native;
+        ctx->alg_encrypt_wrapper = aes_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = aes_decrypt_wrapper;
+
+        ctx->blocksize = AES_BLOCK_SIZE;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_CAST5_128:
+        ctx->ctx = g_new0(struct cast128_ctx, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(struct cast128_ctx, 1);
+
+            nkey /= 2;
+            cast5_set_key(ctx->ctx, nkey, key);
+            cast5_set_key(ctx->ctx_tweak, nkey, key + nkey);
+        } else {
+            cast5_set_key(ctx->ctx, nkey, key);
+        }
+
+        ctx->alg_encrypt_native = cast128_encrypt_native;
+        ctx->alg_decrypt_native = cast128_decrypt_native;
+        ctx->alg_encrypt_wrapper = cast128_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = cast128_decrypt_wrapper;
+
+        ctx->blocksize = CAST128_BLOCK_SIZE;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_SERPENT_128:
+    case QCRYPTO_CIPHER_ALG_SERPENT_192:
+    case QCRYPTO_CIPHER_ALG_SERPENT_256:
+        ctx->ctx = g_new0(struct serpent_ctx, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(struct serpent_ctx, 1);
+
+            nkey /= 2;
+            serpent_set_key(ctx->ctx, nkey, key);
+            serpent_set_key(ctx->ctx_tweak, nkey, key + nkey);
+        } else {
+            serpent_set_key(ctx->ctx, nkey, key);
+        }
+
+        ctx->alg_encrypt_native = serpent_encrypt_native;
+        ctx->alg_decrypt_native = serpent_decrypt_native;
+        ctx->alg_encrypt_wrapper = serpent_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = serpent_decrypt_wrapper;
+
+        ctx->blocksize = SERPENT_BLOCK_SIZE;
+        break;
+
+    case QCRYPTO_CIPHER_ALG_TWOFISH_128:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_192:
+    case QCRYPTO_CIPHER_ALG_TWOFISH_256:
+        ctx->ctx = g_new0(struct twofish_ctx, 1);
+
+        if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+            ctx->ctx_tweak = g_new0(struct twofish_ctx, 1);
+
+            nkey /= 2;
+            twofish_set_key(ctx->ctx, nkey, key);
+            twofish_set_key(ctx->ctx_tweak, nkey, key + nkey);
+        } else {
+            twofish_set_key(ctx->ctx, nkey, key);
+        }
+
+        ctx->alg_encrypt_native = twofish_encrypt_native;
+        ctx->alg_decrypt_native = twofish_decrypt_native;
+        ctx->alg_encrypt_wrapper = twofish_encrypt_wrapper;
+        ctx->alg_decrypt_wrapper = twofish_decrypt_wrapper;
+
+        ctx->blocksize = TWOFISH_BLOCK_SIZE;
+        break;
+
+    default:
+        error_setg(errp, "Unsupported cipher algorithm %d", alg);
+        goto error;
+    }
+
+    ctx->iv = g_new0(uint8_t, ctx->blocksize);
+    cipher->opaque = ctx;
+
+    return cipher;
+
+ error:
+    g_free(cipher);
+    g_free(ctx);
+    return NULL;
+}
+
+
+void qcrypto_cipher_free(QCryptoCipher *cipher)
+{
+    QCryptoCipherNettle *ctx;
+
+    if (!cipher) {
+        return;
+    }
+
+    ctx = cipher->opaque;
+    g_free(ctx->iv);
+    g_free(ctx->ctx);
+    g_free(ctx->ctx_tweak);
+    g_free(ctx);
+    g_free(cipher);
+}
+
+
+int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp)
+{
+    QCryptoCipherNettle *ctx = cipher->opaque;
+
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        ctx->alg_encrypt_wrapper(ctx->ctx, len, out, in);
+        break;
+
+    case QCRYPTO_CIPHER_MODE_CBC:
+        cbc_encrypt(ctx->ctx, ctx->alg_encrypt_native,
+                    ctx->blocksize, ctx->iv,
+                    len, out, in);
+        break;
+
+    case QCRYPTO_CIPHER_MODE_XTS:
+        xts_encrypt(ctx->ctx, ctx->ctx_tweak,
+                    ctx->alg_encrypt_wrapper, ctx->alg_encrypt_wrapper,
+                    ctx->iv, len, out, in);
+        break;
+
+    default:
+        error_setg(errp, "Unsupported cipher algorithm %d",
+                   cipher->alg);
+        return -1;
+    }
+    return 0;
+}
+
+
+int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp)
+{
+    QCryptoCipherNettle *ctx = cipher->opaque;
+
+    if (len % ctx->blocksize) {
+        error_setg(errp, "Length %zu must be a multiple of block size %zu",
+                   len, ctx->blocksize);
+        return -1;
+    }
+
+    switch (cipher->mode) {
+    case QCRYPTO_CIPHER_MODE_ECB:
+        ctx->alg_decrypt_wrapper(ctx->ctx, len, out, in);
+        break;
+
+    case QCRYPTO_CIPHER_MODE_CBC:
+        cbc_decrypt(ctx->ctx, ctx->alg_decrypt_native,
+                    ctx->blocksize, ctx->iv,
+                    len, out, in);
+        break;
+
+    case QCRYPTO_CIPHER_MODE_XTS:
+        if (ctx->blocksize != XTS_BLOCK_SIZE) {
+            error_setg(errp, "Block size must be %d not %zu",
+                       XTS_BLOCK_SIZE, ctx->blocksize);
+            return -1;
+        }
+        xts_decrypt(ctx->ctx, ctx->ctx_tweak,
+                    ctx->alg_encrypt_wrapper, ctx->alg_decrypt_wrapper,
+                    ctx->iv, len, out, in);
+        break;
+
+    default:
+        error_setg(errp, "Unsupported cipher algorithm %d",
+                   cipher->alg);
+        return -1;
+    }
+    return 0;
+}
+
+int qcrypto_cipher_setiv(QCryptoCipher *cipher,
+                         const uint8_t *iv, size_t niv,
+                         Error **errp)
+{
+    QCryptoCipherNettle *ctx = cipher->opaque;
+    if (niv != ctx->blocksize) {
+        error_setg(errp, "Expected IV size %zu not %zu",
+                   ctx->blocksize, niv);
+        return -1;
+    }
+    memcpy(ctx->iv, iv, niv);
+    return 0;
+}
diff --git a/crypto/cipher.c b/crypto/cipher.c
new file mode 100644
index 0000000..cafb454
--- /dev/null
+++ b/crypto/cipher.c
@@ -0,0 +1,157 @@
+/*
+ * QEMU Crypto cipher algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/cipher.h"
+
+
+static size_t alg_key_len[QCRYPTO_CIPHER_ALG__MAX] = {
+    [QCRYPTO_CIPHER_ALG_AES_128] = 16,
+    [QCRYPTO_CIPHER_ALG_AES_192] = 24,
+    [QCRYPTO_CIPHER_ALG_AES_256] = 32,
+    [QCRYPTO_CIPHER_ALG_DES_RFB] = 8,
+    [QCRYPTO_CIPHER_ALG_CAST5_128] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_128] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_192] = 24,
+    [QCRYPTO_CIPHER_ALG_SERPENT_256] = 32,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_128] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_192] = 24,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_256] = 32,
+};
+
+static size_t alg_block_len[QCRYPTO_CIPHER_ALG__MAX] = {
+    [QCRYPTO_CIPHER_ALG_AES_128] = 16,
+    [QCRYPTO_CIPHER_ALG_AES_192] = 16,
+    [QCRYPTO_CIPHER_ALG_AES_256] = 16,
+    [QCRYPTO_CIPHER_ALG_DES_RFB] = 8,
+    [QCRYPTO_CIPHER_ALG_CAST5_128] = 8,
+    [QCRYPTO_CIPHER_ALG_SERPENT_128] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_192] = 16,
+    [QCRYPTO_CIPHER_ALG_SERPENT_256] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_128] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_192] = 16,
+    [QCRYPTO_CIPHER_ALG_TWOFISH_256] = 16,
+};
+
+static bool mode_need_iv[QCRYPTO_CIPHER_MODE__MAX] = {
+    [QCRYPTO_CIPHER_MODE_ECB] = false,
+    [QCRYPTO_CIPHER_MODE_CBC] = true,
+    [QCRYPTO_CIPHER_MODE_XTS] = true,
+};
+
+
+size_t qcrypto_cipher_get_block_len(QCryptoCipherAlgorithm alg)
+{
+    if (alg >= G_N_ELEMENTS(alg_key_len)) {
+        return 0;
+    }
+    return alg_block_len[alg];
+}
+
+
+size_t qcrypto_cipher_get_key_len(QCryptoCipherAlgorithm alg)
+{
+    if (alg >= G_N_ELEMENTS(alg_key_len)) {
+        return 0;
+    }
+    return alg_key_len[alg];
+}
+
+
+size_t qcrypto_cipher_get_iv_len(QCryptoCipherAlgorithm alg,
+                                 QCryptoCipherMode mode)
+{
+    if (alg >= G_N_ELEMENTS(alg_block_len)) {
+        return 0;
+    }
+    if (mode >= G_N_ELEMENTS(mode_need_iv)) {
+        return 0;
+    }
+
+    if (mode_need_iv[mode]) {
+        return alg_block_len[alg];
+    }
+    return 0;
+}
+
+
+static bool
+qcrypto_cipher_validate_key_length(QCryptoCipherAlgorithm alg,
+                                   QCryptoCipherMode mode,
+                                   size_t nkey,
+                                   Error **errp)
+{
+    if ((unsigned)alg >= QCRYPTO_CIPHER_ALG__MAX) {
+        error_setg(errp, "Cipher algorithm %d out of range",
+                   alg);
+        return false;
+    }
+
+    if (mode == QCRYPTO_CIPHER_MODE_XTS) {
+        if (alg == QCRYPTO_CIPHER_ALG_DES_RFB) {
+            error_setg(errp, "XTS mode not compatible with DES-RFB");
+            return false;
+        }
+        if (nkey % 2) {
+            error_setg(errp, "XTS cipher key length should be a multiple of 2");
+            return false;
+        }
+
+        if (alg_key_len[alg] != (nkey / 2)) {
+            error_setg(errp, "Cipher key length %zu should be %zu",
+                       nkey, alg_key_len[alg] * 2);
+            return false;
+        }
+    } else {
+        if (alg_key_len[alg] != nkey) {
+            error_setg(errp, "Cipher key length %zu should be %zu",
+                       nkey, alg_key_len[alg]);
+            return false;
+        }
+    }
+    return true;
+}
+
+#if defined(CONFIG_GCRYPT) || defined(CONFIG_NETTLE)
+static uint8_t *
+qcrypto_cipher_munge_des_rfb_key(const uint8_t *key,
+                                 size_t nkey)
+{
+    uint8_t *ret = g_new0(uint8_t, nkey);
+    size_t i;
+    for (i = 0; i < nkey; i++) {
+        uint8_t r = key[i];
+        r = (r & 0xf0) >> 4 | (r & 0x0f) << 4;
+        r = (r & 0xcc) >> 2 | (r & 0x33) << 2;
+        r = (r & 0xaa) >> 1 | (r & 0x55) << 1;
+        ret[i] = r;
+    }
+    return ret;
+}
+#endif /* CONFIG_GCRYPT || CONFIG_NETTLE */
+
+#ifdef CONFIG_GCRYPT
+#include "crypto/cipher-gcrypt.c"
+#elif defined CONFIG_NETTLE
+#include "crypto/cipher-nettle.c"
+#else
+#include "crypto/cipher-builtin.c"
+#endif
diff --git a/crypto/desrfb.c b/crypto/desrfb.c
new file mode 100644
index 0000000..ec47dea
--- /dev/null
+++ b/crypto/desrfb.c
@@ -0,0 +1,416 @@
+/*
+ * This is D3DES (V5.09) by Richard Outerbridge with the double and
+ * triple-length support removed for use in VNC.  Also the bytebit[] array
+ * has been reversed so that the most significant bit in each byte of the
+ * key is ignored, not the least significant.
+ *
+ * These changes are:
+ *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+/* D3DES (V5.09) -
+ *
+ * A portable, public domain, version of the Data Encryption Standard.
+ *
+ * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
+ * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
+ * code;  Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
+ * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
+ * for humouring me on.
+ *
+ * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
+ * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/desrfb.h"
+
+static void scrunch(unsigned char *, unsigned long *);
+static void unscrun(unsigned long *, unsigned char *);
+static void desfunc(unsigned long *, unsigned long *);
+static void cookey(unsigned long *);
+
+static unsigned long KnL[32] = { 0L };
+
+static const unsigned short bytebit[8]	= {
+	01, 02, 04, 010, 020, 040, 0100, 0200 };
+
+static const unsigned long bigbyte[24] = {
+	0x800000L,	0x400000L,	0x200000L,	0x100000L,
+	0x80000L,	0x40000L,	0x20000L,	0x10000L,
+	0x8000L,	0x4000L,	0x2000L,	0x1000L,
+	0x800L, 	0x400L, 	0x200L, 	0x100L,
+	0x80L,		0x40L,		0x20L,		0x10L,
+	0x8L,		0x4L,		0x2L,		0x1L	};
+
+/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
+
+static const unsigned char pc1[56] = {
+	56, 48, 40, 32, 24, 16,  8,	 0, 57, 49, 41, 33, 25, 17,
+	 9,  1, 58, 50, 42, 34, 26,	18, 10,  2, 59, 51, 43, 35,
+	62, 54, 46, 38, 30, 22, 14,	 6, 61, 53, 45, 37, 29, 21,
+	13,  5, 60, 52, 44, 36, 28,	20, 12,  4, 27, 19, 11,  3 };
+
+static const unsigned char totrot[16] = {
+	1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
+
+static const unsigned char pc2[48] = {
+	13, 16, 10, 23,  0,  4,  2, 27, 14,  5, 20,  9,
+	22, 18, 11,  3, 25,  7, 15,  6, 26, 19, 12,  1,
+	40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
+	43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
+
+/* Thanks to James Gillogly & Phil Karn! */
+void deskey(unsigned char *key, int edf)
+{
+	register int i, j, l, m, n;
+	unsigned char pc1m[56], pcr[56];
+	unsigned long kn[32];
+
+	for ( j = 0; j < 56; j++ ) {
+		l = pc1[j];
+		m = l & 07;
+		pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
+		}
+	for( i = 0; i < 16; i++ ) {
+		if( edf == DE1 ) m = (15 - i) << 1;
+		else m = i << 1;
+		n = m + 1;
+		kn[m] = kn[n] = 0L;
+		for( j = 0; j < 28; j++ ) {
+			l = j + totrot[i];
+			if( l < 28 ) pcr[j] = pc1m[l];
+			else pcr[j] = pc1m[l - 28];
+			}
+		for( j = 28; j < 56; j++ ) {
+		    l = j + totrot[i];
+		    if( l < 56 ) pcr[j] = pc1m[l];
+		    else pcr[j] = pc1m[l - 28];
+		    }
+		for( j = 0; j < 24; j++ ) {
+			if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
+			if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
+			}
+		}
+	cookey(kn);
+	return;
+	}
+
+static void cookey(register unsigned long *raw1)
+{
+	register unsigned long *cook, *raw0;
+	unsigned long dough[32];
+	register int i;
+
+	cook = dough;
+	for( i = 0; i < 16; i++, raw1++ ) {
+		raw0 = raw1++;
+		*cook	 = (*raw0 & 0x00fc0000L) << 6;
+		*cook	|= (*raw0 & 0x00000fc0L) << 10;
+		*cook	|= (*raw1 & 0x00fc0000L) >> 10;
+		*cook++ |= (*raw1 & 0x00000fc0L) >> 6;
+		*cook	 = (*raw0 & 0x0003f000L) << 12;
+		*cook	|= (*raw0 & 0x0000003fL) << 16;
+		*cook	|= (*raw1 & 0x0003f000L) >> 4;
+		*cook++ |= (*raw1 & 0x0000003fL);
+		}
+	usekey(dough);
+	return;
+	}
+
+void usekey(register unsigned long *from)
+{
+	register unsigned long *to, *endp;
+
+	to = KnL, endp = &KnL[32];
+	while( to < endp ) *to++ = *from++;
+	return;
+	}
+
+void des(unsigned char *inblock, unsigned char *outblock)
+{
+	unsigned long work[2];
+
+	scrunch(inblock, work);
+	desfunc(work, KnL);
+	unscrun(work, outblock);
+	return;
+	}
+
+static void scrunch(register unsigned char *outof, register unsigned long *into)
+{
+	*into	 = (*outof++ & 0xffL) << 24;
+	*into	|= (*outof++ & 0xffL) << 16;
+	*into	|= (*outof++ & 0xffL) << 8;
+	*into++ |= (*outof++ & 0xffL);
+	*into	 = (*outof++ & 0xffL) << 24;
+	*into	|= (*outof++ & 0xffL) << 16;
+	*into	|= (*outof++ & 0xffL) << 8;
+	*into	|= (*outof   & 0xffL);
+	return;
+	}
+
+static void unscrun(register unsigned long *outof, register unsigned char *into)
+{
+	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
+	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
+	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
+	*into++ = (unsigned char)(*outof++	 & 0xffL);
+	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
+	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
+	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
+	*into	=  (unsigned char)(*outof	 & 0xffL);
+	return;
+	}
+
+static const unsigned long SP1[64] = {
+	0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
+	0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
+	0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
+	0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
+	0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
+	0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
+	0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
+	0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
+	0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
+	0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
+	0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
+	0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
+	0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
+	0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
+	0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
+	0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
+
+static const unsigned long SP2[64] = {
+	0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
+	0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
+	0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
+	0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
+	0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
+	0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
+	0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
+	0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
+	0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
+	0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
+	0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
+	0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
+	0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
+	0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
+	0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
+	0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
+
+static const unsigned long SP3[64] = {
+	0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
+	0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
+	0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
+	0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
+	0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
+	0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
+	0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
+	0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
+	0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
+	0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
+	0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
+	0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
+	0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
+	0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
+	0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
+	0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
+
+static const unsigned long SP4[64] = {
+	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
+	0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
+	0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
+	0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
+	0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
+	0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
+	0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
+	0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
+	0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
+	0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
+	0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
+	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
+	0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
+	0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
+	0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
+	0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
+
+static const unsigned long SP5[64] = {
+	0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
+	0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
+	0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
+	0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
+	0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
+	0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
+	0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
+	0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
+	0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
+	0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
+	0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
+	0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
+	0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
+	0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
+	0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
+	0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
+
+static const unsigned long SP6[64] = {
+	0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
+	0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
+	0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
+	0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
+	0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
+	0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
+	0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
+	0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
+	0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
+	0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
+	0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
+	0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
+	0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
+	0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
+	0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
+	0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
+
+static const unsigned long SP7[64] = {
+	0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
+	0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
+	0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
+	0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
+	0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
+	0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
+	0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
+	0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
+	0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
+	0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
+	0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
+	0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
+	0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
+	0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
+	0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
+	0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
+
+static const unsigned long SP8[64] = {
+	0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
+	0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
+	0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
+	0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
+	0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
+	0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
+	0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
+	0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
+	0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
+	0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
+	0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
+	0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
+	0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
+	0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
+	0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
+	0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
+
+static void desfunc(register unsigned long *block, register unsigned long *keys)
+{
+	register unsigned long fval, work, right, leftt;
+	register int round;
+
+	leftt = block[0];
+	right = block[1];
+	work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
+	right ^= work;
+	leftt ^= (work << 4);
+	work = ((leftt >> 16) ^ right) & 0x0000ffffL;
+	right ^= work;
+	leftt ^= (work << 16);
+	work = ((right >> 2) ^ leftt) & 0x33333333L;
+	leftt ^= work;
+	right ^= (work << 2);
+	work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
+	leftt ^= work;
+	right ^= (work << 8);
+	right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
+	work = (leftt ^ right) & 0xaaaaaaaaL;
+	leftt ^= work;
+	right ^= work;
+	leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
+
+	for( round = 0; round < 8; round++ ) {
+		work  = (right << 28) | (right >> 4);
+		work ^= *keys++;
+		fval  = SP7[ work		 & 0x3fL];
+		fval |= SP5[(work >>  8) & 0x3fL];
+		fval |= SP3[(work >> 16) & 0x3fL];
+		fval |= SP1[(work >> 24) & 0x3fL];
+		work  = right ^ *keys++;
+		fval |= SP8[ work		 & 0x3fL];
+		fval |= SP6[(work >>  8) & 0x3fL];
+		fval |= SP4[(work >> 16) & 0x3fL];
+		fval |= SP2[(work >> 24) & 0x3fL];
+		leftt ^= fval;
+		work  = (leftt << 28) | (leftt >> 4);
+		work ^= *keys++;
+		fval  = SP7[ work		 & 0x3fL];
+		fval |= SP5[(work >>  8) & 0x3fL];
+		fval |= SP3[(work >> 16) & 0x3fL];
+		fval |= SP1[(work >> 24) & 0x3fL];
+		work  = leftt ^ *keys++;
+		fval |= SP8[ work		 & 0x3fL];
+		fval |= SP6[(work >>  8) & 0x3fL];
+		fval |= SP4[(work >> 16) & 0x3fL];
+		fval |= SP2[(work >> 24) & 0x3fL];
+		right ^= fval;
+		}
+
+	right = (right << 31) | (right >> 1);
+	work = (leftt ^ right) & 0xaaaaaaaaL;
+	leftt ^= work;
+	right ^= work;
+	leftt = (leftt << 31) | (leftt >> 1);
+	work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
+	right ^= work;
+	leftt ^= (work << 8);
+	work = ((leftt >> 2) ^ right) & 0x33333333L;
+	right ^= work;
+	leftt ^= (work << 2);
+	work = ((right >> 16) ^ leftt) & 0x0000ffffL;
+	leftt ^= work;
+	right ^= (work << 16);
+	work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
+	leftt ^= work;
+	right ^= (work << 4);
+	*block++ = right;
+	*block = leftt;
+	return;
+	}
+
+/* Validation sets:
+ *
+ * Single-length key, single-length plaintext -
+ * Key	  : 0123 4567 89ab cdef
+ * Plain  : 0123 4567 89ab cde7
+ * Cipher : c957 4425 6a5e d31d
+ *
+ * Double-length key, single-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
+ * Plain  : 0123 4567 89ab cde7
+ * Cipher : 7f1d 0a77 826b 8aff
+ *
+ * Double-length key, double-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
+ * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
+ * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
+ *
+ * Triple-length key, single-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
+ * Plain  : 0123 4567 89ab cde7
+ * Cipher : de0b 7c06 ae5e 0ed5
+ *
+ * Triple-length key, double-length plaintext -
+ * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
+ * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
+ * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
+ *
+ * d3des V5.0a rwo 9208.07 18:44 Graven Imagery
+ **********************************************************************/
diff --git a/crypto/hash-gcrypt.c b/crypto/hash-gcrypt.c
new file mode 100644
index 0000000..7690690
--- /dev/null
+++ b/crypto/hash-gcrypt.c
@@ -0,0 +1,109 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <gcrypt.h>
+#include "qapi/error.h"
+#include "crypto/hash.h"
+
+
+static int qcrypto_hash_alg_map[QCRYPTO_HASH_ALG__MAX] = {
+    [QCRYPTO_HASH_ALG_MD5] = GCRY_MD_MD5,
+    [QCRYPTO_HASH_ALG_SHA1] = GCRY_MD_SHA1,
+    [QCRYPTO_HASH_ALG_SHA224] = GCRY_MD_SHA224,
+    [QCRYPTO_HASH_ALG_SHA256] = GCRY_MD_SHA256,
+    [QCRYPTO_HASH_ALG_SHA384] = GCRY_MD_SHA384,
+    [QCRYPTO_HASH_ALG_SHA512] = GCRY_MD_SHA512,
+    [QCRYPTO_HASH_ALG_RIPEMD160] = GCRY_MD_RMD160,
+};
+
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
+{
+    if (alg < G_N_ELEMENTS(qcrypto_hash_alg_map) &&
+        qcrypto_hash_alg_map[alg] != GCRY_MD_NONE) {
+        return true;
+    }
+    return false;
+}
+
+
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov,
+                        size_t niov,
+                        uint8_t **result,
+                        size_t *resultlen,
+                        Error **errp)
+{
+    int i, ret;
+    gcry_md_hd_t md;
+    unsigned char *digest;
+
+    if (!qcrypto_hash_supports(alg)) {
+        error_setg(errp,
+                   "Unknown hash algorithm %d",
+                   alg);
+        return -1;
+    }
+
+    ret = gcry_md_open(&md, qcrypto_hash_alg_map[alg], 0);
+
+    if (ret < 0) {
+        error_setg(errp,
+                   "Unable to initialize hash algorithm: %s",
+                   gcry_strerror(ret));
+        return -1;
+    }
+
+    for (i = 0; i < niov; i++) {
+        gcry_md_write(md, iov[i].iov_base, iov[i].iov_len);
+    }
+
+    ret = gcry_md_get_algo_dlen(qcrypto_hash_alg_map[alg]);
+    if (ret <= 0) {
+        error_setg(errp,
+                   "Unable to get hash length: %s",
+                   gcry_strerror(ret));
+        goto error;
+    }
+    if (*resultlen == 0) {
+        *resultlen = ret;
+        *result = g_new0(uint8_t, *resultlen);
+    } else if (*resultlen != ret) {
+        error_setg(errp,
+                   "Result buffer size %zu is smaller than hash %d",
+                   *resultlen, ret);
+        goto error;
+    }
+
+    digest = gcry_md_read(md, 0);
+    if (!digest) {
+        error_setg(errp,
+                   "No digest produced");
+        goto error;
+    }
+    memcpy(*result, digest, *resultlen);
+
+    gcry_md_close(md);
+    return 0;
+
+ error:
+    gcry_md_close(md);
+    return -1;
+}
diff --git a/crypto/hash-glib.c b/crypto/hash-glib.c
new file mode 100644
index 0000000..ec99ac9
--- /dev/null
+++ b/crypto/hash-glib.c
@@ -0,0 +1,97 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/hash.h"
+
+
+static int qcrypto_hash_alg_map[QCRYPTO_HASH_ALG__MAX] = {
+    [QCRYPTO_HASH_ALG_MD5] = G_CHECKSUM_MD5,
+    [QCRYPTO_HASH_ALG_SHA1] = G_CHECKSUM_SHA1,
+    [QCRYPTO_HASH_ALG_SHA224] = -1,
+    [QCRYPTO_HASH_ALG_SHA256] = G_CHECKSUM_SHA256,
+    [QCRYPTO_HASH_ALG_SHA384] = -1,
+#if GLIB_CHECK_VERSION(2, 36, 0)
+    [QCRYPTO_HASH_ALG_SHA512] = G_CHECKSUM_SHA512,
+#else
+    [QCRYPTO_HASH_ALG_SHA512] = -1,
+#endif
+    [QCRYPTO_HASH_ALG_RIPEMD160] = -1,
+};
+
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
+{
+    if (alg < G_N_ELEMENTS(qcrypto_hash_alg_map) &&
+        qcrypto_hash_alg_map[alg] != -1) {
+        return true;
+    }
+    return false;
+}
+
+
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov,
+                        size_t niov,
+                        uint8_t **result,
+                        size_t *resultlen,
+                        Error **errp)
+{
+    int i, ret;
+    GChecksum *cs;
+
+    if (!qcrypto_hash_supports(alg)) {
+        error_setg(errp,
+                   "Unknown hash algorithm %d",
+                   alg);
+        return -1;
+    }
+
+    cs = g_checksum_new(qcrypto_hash_alg_map[alg]);
+
+    for (i = 0; i < niov; i++) {
+        g_checksum_update(cs, iov[i].iov_base, iov[i].iov_len);
+    }
+
+    ret = g_checksum_type_get_length(qcrypto_hash_alg_map[alg]);
+    if (ret < 0) {
+        error_setg(errp, "%s",
+                   "Unable to get hash length");
+        goto error;
+    }
+    if (*resultlen == 0) {
+        *resultlen = ret;
+        *result = g_new0(uint8_t, *resultlen);
+    } else if (*resultlen != ret) {
+        error_setg(errp,
+                   "Result buffer size %zu is smaller than hash %d",
+                   *resultlen, ret);
+        goto error;
+    }
+
+    g_checksum_get_digest(cs, *result, resultlen);
+
+    g_checksum_free(cs);
+    return 0;
+
+ error:
+    g_checksum_free(cs);
+    return -1;
+}
diff --git a/crypto/hash-nettle.c b/crypto/hash-nettle.c
new file mode 100644
index 0000000..6a206dc
--- /dev/null
+++ b/crypto/hash-nettle.c
@@ -0,0 +1,154 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/hash.h"
+#include <nettle/md5.h>
+#include <nettle/sha.h>
+#include <nettle/ripemd160.h>
+
+typedef void (*qcrypto_nettle_init)(void *ctx);
+typedef void (*qcrypto_nettle_write)(void *ctx,
+                                     unsigned int len,
+                                     const uint8_t *buf);
+typedef void (*qcrypto_nettle_result)(void *ctx,
+                                      unsigned int len,
+                                      uint8_t *buf);
+
+union qcrypto_hash_ctx {
+    struct md5_ctx md5;
+    struct sha1_ctx sha1;
+    struct sha224_ctx sha224;
+    struct sha256_ctx sha256;
+    struct sha384_ctx sha384;
+    struct sha512_ctx sha512;
+    struct ripemd160_ctx ripemd160;
+};
+
+struct qcrypto_hash_alg {
+    qcrypto_nettle_init init;
+    qcrypto_nettle_write write;
+    qcrypto_nettle_result result;
+    size_t len;
+} qcrypto_hash_alg_map[] = {
+    [QCRYPTO_HASH_ALG_MD5] = {
+        .init = (qcrypto_nettle_init)md5_init,
+        .write = (qcrypto_nettle_write)md5_update,
+        .result = (qcrypto_nettle_result)md5_digest,
+        .len = MD5_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA1] = {
+        .init = (qcrypto_nettle_init)sha1_init,
+        .write = (qcrypto_nettle_write)sha1_update,
+        .result = (qcrypto_nettle_result)sha1_digest,
+        .len = SHA1_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA224] = {
+        .init = (qcrypto_nettle_init)sha224_init,
+        .write = (qcrypto_nettle_write)sha224_update,
+        .result = (qcrypto_nettle_result)sha224_digest,
+        .len = SHA224_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA256] = {
+        .init = (qcrypto_nettle_init)sha256_init,
+        .write = (qcrypto_nettle_write)sha256_update,
+        .result = (qcrypto_nettle_result)sha256_digest,
+        .len = SHA256_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA384] = {
+        .init = (qcrypto_nettle_init)sha384_init,
+        .write = (qcrypto_nettle_write)sha384_update,
+        .result = (qcrypto_nettle_result)sha384_digest,
+        .len = SHA384_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_SHA512] = {
+        .init = (qcrypto_nettle_init)sha512_init,
+        .write = (qcrypto_nettle_write)sha512_update,
+        .result = (qcrypto_nettle_result)sha512_digest,
+        .len = SHA512_DIGEST_SIZE,
+    },
+    [QCRYPTO_HASH_ALG_RIPEMD160] = {
+        .init = (qcrypto_nettle_init)ripemd160_init,
+        .write = (qcrypto_nettle_write)ripemd160_update,
+        .result = (qcrypto_nettle_result)ripemd160_digest,
+        .len = RIPEMD160_DIGEST_SIZE,
+    },
+};
+
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg)
+{
+    if (alg < G_N_ELEMENTS(qcrypto_hash_alg_map) &&
+        qcrypto_hash_alg_map[alg].init != NULL) {
+        return true;
+    }
+    return false;
+}
+
+
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov,
+                        size_t niov,
+                        uint8_t **result,
+                        size_t *resultlen,
+                        Error **errp)
+{
+    int i;
+    union qcrypto_hash_ctx ctx;
+
+    if (!qcrypto_hash_supports(alg)) {
+        error_setg(errp,
+                   "Unknown hash algorithm %d",
+                   alg);
+        return -1;
+    }
+
+    qcrypto_hash_alg_map[alg].init(&ctx);
+
+    for (i = 0; i < niov; i++) {
+        /* Some versions of nettle have functions
+         * declared with 'int' instead of 'size_t'
+         * so to be safe avoid writing more than
+         * UINT_MAX bytes at a time
+         */
+        size_t len = iov[i].iov_len;
+        uint8_t *base = iov[i].iov_base;
+        while (len) {
+            size_t shortlen = MIN(len, UINT_MAX);
+            qcrypto_hash_alg_map[alg].write(&ctx, len, base);
+            len -= shortlen;
+            base += len;
+        }
+    }
+
+    if (*resultlen == 0) {
+        *resultlen = qcrypto_hash_alg_map[alg].len;
+        *result = g_new0(uint8_t, *resultlen);
+    } else if (*resultlen != qcrypto_hash_alg_map[alg].len) {
+        error_setg(errp,
+                   "Result buffer size %zu is smaller than hash %zu",
+                   *resultlen, qcrypto_hash_alg_map[alg].len);
+        return -1;
+    }
+
+    qcrypto_hash_alg_map[alg].result(&ctx, *resultlen, *result);
+
+    return 0;
+}
diff --git a/crypto/hash.c b/crypto/hash.c
new file mode 100644
index 0000000..0f1ceac
--- /dev/null
+++ b/crypto/hash.c
@@ -0,0 +1,118 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/hash.h"
+
+static size_t qcrypto_hash_alg_size[QCRYPTO_HASH_ALG__MAX] = {
+    [QCRYPTO_HASH_ALG_MD5] = 16,
+    [QCRYPTO_HASH_ALG_SHA1] = 20,
+    [QCRYPTO_HASH_ALG_SHA224] = 28,
+    [QCRYPTO_HASH_ALG_SHA256] = 32,
+    [QCRYPTO_HASH_ALG_SHA384] = 48,
+    [QCRYPTO_HASH_ALG_SHA512] = 64,
+    [QCRYPTO_HASH_ALG_RIPEMD160] = 20,
+};
+
+size_t qcrypto_hash_digest_len(QCryptoHashAlgorithm alg)
+{
+    assert(alg < G_N_ELEMENTS(qcrypto_hash_alg_size));
+    return qcrypto_hash_alg_size[alg];
+}
+
+
+int qcrypto_hash_bytes(QCryptoHashAlgorithm alg,
+                       const char *buf,
+                       size_t len,
+                       uint8_t **result,
+                       size_t *resultlen,
+                       Error **errp)
+{
+    struct iovec iov = { .iov_base = (char *)buf,
+                         .iov_len = len };
+    return qcrypto_hash_bytesv(alg, &iov, 1, result, resultlen, errp);
+}
+
+static const char hex[] = "0123456789abcdef";
+
+int qcrypto_hash_digestv(QCryptoHashAlgorithm alg,
+                         const struct iovec *iov,
+                         size_t niov,
+                         char **digest,
+                         Error **errp)
+{
+    uint8_t *result = NULL;
+    size_t resultlen = 0;
+    size_t i;
+
+    if (qcrypto_hash_bytesv(alg, iov, niov, &result, &resultlen, errp) < 0) {
+        return -1;
+    }
+
+    *digest = g_new0(char, (resultlen * 2) + 1);
+    for (i = 0 ; i < resultlen ; i++) {
+        (*digest)[(i * 2)] = hex[(result[i] >> 4) & 0xf];
+        (*digest)[(i * 2) + 1] = hex[result[i] & 0xf];
+    }
+    (*digest)[resultlen * 2] = '\0';
+    g_free(result);
+    return 0;
+}
+
+int qcrypto_hash_digest(QCryptoHashAlgorithm alg,
+                        const char *buf,
+                        size_t len,
+                        char **digest,
+                        Error **errp)
+{
+    struct iovec iov = { .iov_base = (char *)buf, .iov_len = len };
+
+    return qcrypto_hash_digestv(alg, &iov, 1, digest, errp);
+}
+
+int qcrypto_hash_base64v(QCryptoHashAlgorithm alg,
+                         const struct iovec *iov,
+                         size_t niov,
+                         char **base64,
+                         Error **errp)
+{
+    uint8_t *result = NULL;
+    size_t resultlen = 0;
+
+    if (qcrypto_hash_bytesv(alg, iov, niov, &result, &resultlen, errp) < 0) {
+        return -1;
+    }
+
+    *base64 = g_base64_encode(result, resultlen);
+    g_free(result);
+    return 0;
+}
+
+int qcrypto_hash_base64(QCryptoHashAlgorithm alg,
+                        const char *buf,
+                        size_t len,
+                        char **base64,
+                        Error **errp)
+{
+    struct iovec iov = { .iov_base = (char *)buf, .iov_len = len };
+
+    return qcrypto_hash_base64v(alg, &iov, 1, base64, errp);
+}
diff --git a/crypto/init.c b/crypto/init.c
new file mode 100644
index 0000000..1e564d9
--- /dev/null
+++ b/crypto/init.c
@@ -0,0 +1,150 @@
+/*
+ * QEMU Crypto initialization
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/init.h"
+#include "qapi/error.h"
+#include "qemu/thread.h"
+
+#ifdef CONFIG_GNUTLS
+#include <gnutls/gnutls.h>
+#include <gnutls/crypto.h>
+#endif
+
+#ifdef CONFIG_GCRYPT
+#include <gcrypt.h>
+#endif
+
+/* #define DEBUG_GNUTLS */
+
+/*
+ * If GNUTLS is built against GCrypt then
+ *
+ *  - When GNUTLS >= 2.12, we must not initialize gcrypt threading
+ *    because GNUTLS will do that itself
+ *  - When GNUTLS < 2.12 we must always initialize gcrypt threading
+ *  - When GNUTLS is disabled we must always initialize gcrypt threading
+ *
+ * But....
+ *
+ *    When gcrypt >= 1.6.0 we must not initialize gcrypt threading
+ *    because gcrypt will do that itself.
+ *
+ * So we need to init gcrypt threading if
+ *
+ *   - gcrypt < 1.6.0
+ * AND
+ *      - gnutls < 2.12
+ *   OR
+ *      - gnutls is disabled
+ *
+ */
+
+#if (defined(CONFIG_GCRYPT) &&                  \
+     (!defined(CONFIG_GNUTLS) ||                \
+      !defined(GNUTLS_VERSION_NUMBER) ||       \
+      (GNUTLS_VERSION_NUMBER < 0x020c00)) &&    \
+     (!defined(GCRYPT_VERSION_NUMBER) ||        \
+      (GCRYPT_VERSION_NUMBER < 0x010600)))
+#define QCRYPTO_INIT_GCRYPT_THREADS
+#else
+#undef QCRYPTO_INIT_GCRYPT_THREADS
+#endif
+
+#ifdef DEBUG_GNUTLS
+static void qcrypto_gnutls_log(int level, const char *str)
+{
+    fprintf(stderr, "%d: %s", level, str);
+}
+#endif
+
+#ifdef QCRYPTO_INIT_GCRYPT_THREADS
+static int qcrypto_gcrypt_mutex_init(void **priv)
+{                                                                             \
+    QemuMutex *lock = NULL;
+    lock = g_new0(QemuMutex, 1);
+    qemu_mutex_init(lock);
+    *priv = lock;
+    return 0;
+}
+
+static int qcrypto_gcrypt_mutex_destroy(void **priv)
+{
+    QemuMutex *lock = *priv;
+    qemu_mutex_destroy(lock);
+    g_free(lock);
+    return 0;
+}
+
+static int qcrypto_gcrypt_mutex_lock(void **priv)
+{
+    QemuMutex *lock = *priv;
+    qemu_mutex_lock(lock);
+    return 0;
+}
+
+static int qcrypto_gcrypt_mutex_unlock(void **priv)
+{
+    QemuMutex *lock = *priv;
+    qemu_mutex_unlock(lock);
+    return 0;
+}
+
+static struct gcry_thread_cbs qcrypto_gcrypt_thread_impl = {
+    (GCRY_THREAD_OPTION_PTHREAD | (GCRY_THREAD_OPTION_VERSION << 8)),
+    NULL,
+    qcrypto_gcrypt_mutex_init,
+    qcrypto_gcrypt_mutex_destroy,
+    qcrypto_gcrypt_mutex_lock,
+    qcrypto_gcrypt_mutex_unlock,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
+};
+#endif /* QCRYPTO_INIT_GCRYPT */
+
+int qcrypto_init(Error **errp)
+{
+#ifdef CONFIG_GNUTLS
+    int ret;
+    ret = gnutls_global_init();
+    if (ret < 0) {
+        error_setg(errp,
+                   "Unable to initialize GNUTLS library: %s",
+                   gnutls_strerror(ret));
+        return -1;
+    }
+#ifdef DEBUG_GNUTLS
+    gnutls_global_set_log_level(10);
+    gnutls_global_set_log_function(qcrypto_gnutls_log);
+#endif
+#endif
+
+#ifdef CONFIG_GCRYPT
+    if (!gcry_check_version(GCRYPT_VERSION)) {
+        error_setg(errp, "Unable to initialize gcrypt");
+        return -1;
+    }
+#ifdef QCRYPTO_INIT_GCRYPT_THREADS
+    gcry_control(GCRYCTL_SET_THREAD_CBS, &qcrypto_gcrypt_thread_impl);
+#endif /* QCRYPTO_INIT_GCRYPT_THREADS */
+    gcry_control(GCRYCTL_INITIALIZATION_FINISHED, 0);
+#endif
+
+    return 0;
+}
diff --git a/crypto/ivgen-essiv.c b/crypto/ivgen-essiv.c
new file mode 100644
index 0000000..634de63
--- /dev/null
+++ b/crypto/ivgen-essiv.c
@@ -0,0 +1,120 @@
+/*
+ * QEMU Crypto block IV generator - essiv
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "crypto/ivgen-essiv.h"
+
+typedef struct QCryptoIVGenESSIV QCryptoIVGenESSIV;
+struct QCryptoIVGenESSIV {
+    QCryptoCipher *cipher;
+};
+
+static int qcrypto_ivgen_essiv_init(QCryptoIVGen *ivgen,
+                                    const uint8_t *key, size_t nkey,
+                                    Error **errp)
+{
+    uint8_t *salt;
+    size_t nhash;
+    size_t nsalt;
+    QCryptoIVGenESSIV *essiv = g_new0(QCryptoIVGenESSIV, 1);
+
+    /* Not necessarily the same as nkey */
+    nsalt = qcrypto_cipher_get_key_len(ivgen->cipher);
+
+    nhash = qcrypto_hash_digest_len(ivgen->hash);
+    /* Salt must be larger of hash size or key size */
+    salt = g_new0(uint8_t, MAX(nhash, nsalt));
+
+    if (qcrypto_hash_bytes(ivgen->hash, (const gchar *)key, nkey,
+                           &salt, &nhash,
+                           errp) < 0) {
+        g_free(essiv);
+        return -1;
+    }
+
+    /* Now potentially truncate salt to match cipher key len */
+    essiv->cipher = qcrypto_cipher_new(ivgen->cipher,
+                                       QCRYPTO_CIPHER_MODE_ECB,
+                                       salt, MIN(nhash, nsalt),
+                                       errp);
+    if (!essiv->cipher) {
+        g_free(essiv);
+        g_free(salt);
+        return -1;
+    }
+
+    g_free(salt);
+    ivgen->private = essiv;
+
+    return 0;
+}
+
+static int qcrypto_ivgen_essiv_calculate(QCryptoIVGen *ivgen,
+                                         uint64_t sector,
+                                         uint8_t *iv, size_t niv,
+                                         Error **errp)
+{
+    QCryptoIVGenESSIV *essiv = ivgen->private;
+    size_t ndata = qcrypto_cipher_get_block_len(ivgen->cipher);
+    uint8_t *data = g_new(uint8_t, ndata);
+
+    sector = cpu_to_le64(sector);
+    memcpy(data, (uint8_t *)&sector, ndata);
+    if (sizeof(sector) < ndata) {
+        memset(data + sizeof(sector), 0, ndata - sizeof(sector));
+    }
+
+    if (qcrypto_cipher_encrypt(essiv->cipher,
+                               data,
+                               data,
+                               ndata,
+                               errp) < 0) {
+        g_free(data);
+        return -1;
+    }
+
+    if (ndata > niv) {
+        ndata = niv;
+    }
+    memcpy(iv, data, ndata);
+    if (ndata < niv) {
+        memset(iv + ndata, 0, niv - ndata);
+    }
+    g_free(data);
+    return 0;
+}
+
+static void qcrypto_ivgen_essiv_cleanup(QCryptoIVGen *ivgen)
+{
+    QCryptoIVGenESSIV *essiv = ivgen->private;
+
+    qcrypto_cipher_free(essiv->cipher);
+    g_free(essiv);
+}
+
+
+struct QCryptoIVGenDriver qcrypto_ivgen_essiv = {
+    .init = qcrypto_ivgen_essiv_init,
+    .calculate = qcrypto_ivgen_essiv_calculate,
+    .cleanup = qcrypto_ivgen_essiv_cleanup,
+};
+
diff --git a/crypto/ivgen-essiv.h b/crypto/ivgen-essiv.h
new file mode 100644
index 0000000..4a00af8
--- /dev/null
+++ b/crypto/ivgen-essiv.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block IV generator - essiv
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/ivgenpriv.h"
+
+#ifndef QCRYPTO_IVGEN_ESSIV_H__
+#define QCRYPTO_IVGEN_ESSIV_H__
+
+extern struct QCryptoIVGenDriver qcrypto_ivgen_essiv;
+
+#endif /* QCRYPTO_IVGEN_ESSIV_H__ */
diff --git a/crypto/ivgen-plain.c b/crypto/ivgen-plain.c
new file mode 100644
index 0000000..9b9b4ad
--- /dev/null
+++ b/crypto/ivgen-plain.c
@@ -0,0 +1,61 @@
+/*
+ * QEMU Crypto block IV generator - plain
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "crypto/ivgen-plain.h"
+
+static int qcrypto_ivgen_plain_init(QCryptoIVGen *ivgen,
+                                    const uint8_t *key, size_t nkey,
+                                    Error **errp)
+{
+    return 0;
+}
+
+static int qcrypto_ivgen_plain_calculate(QCryptoIVGen *ivgen,
+                                         uint64_t sector,
+                                         uint8_t *iv, size_t niv,
+                                         Error **errp)
+{
+    size_t ivprefix;
+    uint32_t shortsector = cpu_to_le32((sector & 0xffffffff));
+    ivprefix = sizeof(shortsector);
+    if (ivprefix > niv) {
+        ivprefix = niv;
+    }
+    memcpy(iv, &shortsector, ivprefix);
+    if (ivprefix < niv) {
+        memset(iv + ivprefix, 0, niv - ivprefix);
+    }
+    return 0;
+}
+
+static void qcrypto_ivgen_plain_cleanup(QCryptoIVGen *ivgen)
+{
+}
+
+
+struct QCryptoIVGenDriver qcrypto_ivgen_plain = {
+    .init = qcrypto_ivgen_plain_init,
+    .calculate = qcrypto_ivgen_plain_calculate,
+    .cleanup = qcrypto_ivgen_plain_cleanup,
+};
+
diff --git a/crypto/ivgen-plain.h b/crypto/ivgen-plain.h
new file mode 100644
index 0000000..0fe8835
--- /dev/null
+++ b/crypto/ivgen-plain.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block IV generator - plain
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/ivgenpriv.h"
+
+#ifndef QCRYPTO_IVGEN_PLAIN_H__
+#define QCRYPTO_IVGEN_PLAIN_H__
+
+extern struct QCryptoIVGenDriver qcrypto_ivgen_plain;
+
+#endif /* QCRYPTO_IVGEN_PLAIN_H__ */
diff --git a/crypto/ivgen-plain64.c b/crypto/ivgen-plain64.c
new file mode 100644
index 0000000..6c6b1b4
--- /dev/null
+++ b/crypto/ivgen-plain64.c
@@ -0,0 +1,61 @@
+/*
+ * QEMU Crypto block IV generator - plain
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "crypto/ivgen-plain.h"
+
+static int qcrypto_ivgen_plain_init(QCryptoIVGen *ivgen,
+                                    const uint8_t *key, size_t nkey,
+                                    Error **errp)
+{
+    return 0;
+}
+
+static int qcrypto_ivgen_plain_calculate(QCryptoIVGen *ivgen,
+                                         uint64_t sector,
+                                         uint8_t *iv, size_t niv,
+                                         Error **errp)
+{
+    size_t ivprefix;
+    ivprefix = sizeof(sector);
+    sector = cpu_to_le64(sector);
+    if (ivprefix > niv) {
+        ivprefix = niv;
+    }
+    memcpy(iv, &sector, ivprefix);
+    if (ivprefix < niv) {
+        memset(iv + ivprefix, 0, niv - ivprefix);
+    }
+    return 0;
+}
+
+static void qcrypto_ivgen_plain_cleanup(QCryptoIVGen *ivgen)
+{
+}
+
+
+struct QCryptoIVGenDriver qcrypto_ivgen_plain64 = {
+    .init = qcrypto_ivgen_plain_init,
+    .calculate = qcrypto_ivgen_plain_calculate,
+    .cleanup = qcrypto_ivgen_plain_cleanup,
+};
+
diff --git a/crypto/ivgen-plain64.h b/crypto/ivgen-plain64.h
new file mode 100644
index 0000000..c410445
--- /dev/null
+++ b/crypto/ivgen-plain64.h
@@ -0,0 +1,28 @@
+/*
+ * QEMU Crypto block IV generator - plain64
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "crypto/ivgenpriv.h"
+
+#ifndef QCRYPTO_IVGEN_PLAIN64_H__
+#define QCRYPTO_IVGEN_PLAIN64_H__
+
+extern struct QCryptoIVGenDriver qcrypto_ivgen_plain64;
+
+#endif /* QCRYPTO_IVGEN_PLAIN64_H__ */
diff --git a/crypto/ivgen.c b/crypto/ivgen.c
new file mode 100644
index 0000000..f664351
--- /dev/null
+++ b/crypto/ivgen.c
@@ -0,0 +1,101 @@
+/*
+ * QEMU Crypto block IV generator
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+
+#include "crypto/ivgenpriv.h"
+#include "crypto/ivgen-plain.h"
+#include "crypto/ivgen-plain64.h"
+#include "crypto/ivgen-essiv.h"
+
+
+QCryptoIVGen *qcrypto_ivgen_new(QCryptoIVGenAlgorithm alg,
+                                QCryptoCipherAlgorithm cipheralg,
+                                QCryptoHashAlgorithm hash,
+                                const uint8_t *key, size_t nkey,
+                                Error **errp)
+{
+    QCryptoIVGen *ivgen = g_new0(QCryptoIVGen, 1);
+
+    ivgen->algorithm = alg;
+    ivgen->cipher = cipheralg;
+    ivgen->hash = hash;
+
+    switch (alg) {
+    case QCRYPTO_IVGEN_ALG_PLAIN:
+        ivgen->driver = &qcrypto_ivgen_plain;
+        break;
+    case QCRYPTO_IVGEN_ALG_PLAIN64:
+        ivgen->driver = &qcrypto_ivgen_plain64;
+        break;
+    case QCRYPTO_IVGEN_ALG_ESSIV:
+        ivgen->driver = &qcrypto_ivgen_essiv;
+        break;
+    default:
+        error_setg(errp, "Unknown block IV generator algorithm %d", alg);
+        g_free(ivgen);
+        return NULL;
+    }
+
+    if (ivgen->driver->init(ivgen, key, nkey, errp) < 0) {
+        g_free(ivgen);
+        return NULL;
+    }
+
+    return ivgen;
+}
+
+
+int qcrypto_ivgen_calculate(QCryptoIVGen *ivgen,
+                            uint64_t sector,
+                            uint8_t *iv, size_t niv,
+                            Error **errp)
+{
+    return ivgen->driver->calculate(ivgen, sector, iv, niv, errp);
+}
+
+
+QCryptoIVGenAlgorithm qcrypto_ivgen_get_algorithm(QCryptoIVGen *ivgen)
+{
+    return ivgen->algorithm;
+}
+
+
+QCryptoCipherAlgorithm qcrypto_ivgen_get_cipher(QCryptoIVGen *ivgen)
+{
+    return ivgen->cipher;
+}
+
+
+QCryptoHashAlgorithm qcrypto_ivgen_get_hash(QCryptoIVGen *ivgen)
+{
+    return ivgen->hash;
+}
+
+
+void qcrypto_ivgen_free(QCryptoIVGen *ivgen)
+{
+    if (!ivgen) {
+        return;
+    }
+    ivgen->driver->cleanup(ivgen);
+    g_free(ivgen);
+}
diff --git a/crypto/ivgenpriv.h b/crypto/ivgenpriv.h
new file mode 100644
index 0000000..28e5c67
--- /dev/null
+++ b/crypto/ivgenpriv.h
@@ -0,0 +1,49 @@
+/*
+ * QEMU Crypto block IV generator
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_IVGENPRIV_H
+#define QCRYPTO_IVGENPRIV_H
+
+#include "crypto/ivgen.h"
+
+typedef struct QCryptoIVGenDriver QCryptoIVGenDriver;
+
+struct QCryptoIVGenDriver {
+    int (*init)(QCryptoIVGen *ivgen,
+                const uint8_t *key, size_t nkey,
+                Error **errp);
+    int (*calculate)(QCryptoIVGen *ivgen,
+                     uint64_t sector,
+                     uint8_t *iv, size_t niv,
+                     Error **errp);
+    void (*cleanup)(QCryptoIVGen *ivgen);
+};
+
+struct QCryptoIVGen {
+    QCryptoIVGenDriver *driver;
+    void *private;
+
+    QCryptoIVGenAlgorithm algorithm;
+    QCryptoCipherAlgorithm cipher;
+    QCryptoHashAlgorithm hash;
+};
+
+
+#endif /* QCRYPTO_IVGENPRIV_H */
diff --git a/crypto/pbkdf-gcrypt.c b/crypto/pbkdf-gcrypt.c
new file mode 100644
index 0000000..34af3a9
--- /dev/null
+++ b/crypto/pbkdf-gcrypt.c
@@ -0,0 +1,69 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <gcrypt.h>
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash)
+{
+    switch (hash) {
+    case QCRYPTO_HASH_ALG_MD5:
+    case QCRYPTO_HASH_ALG_SHA1:
+    case QCRYPTO_HASH_ALG_SHA256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash,
+                   const uint8_t *key, size_t nkey,
+                   const uint8_t *salt, size_t nsalt,
+                   unsigned int iterations,
+                   uint8_t *out, size_t nout,
+                   Error **errp)
+{
+    static const int hash_map[QCRYPTO_HASH_ALG__MAX] = {
+        [QCRYPTO_HASH_ALG_MD5] = GCRY_MD_MD5,
+        [QCRYPTO_HASH_ALG_SHA1] = GCRY_MD_SHA1,
+        [QCRYPTO_HASH_ALG_SHA256] = GCRY_MD_SHA256,
+    };
+    int ret;
+
+    if (hash >= G_N_ELEMENTS(hash_map) ||
+        hash_map[hash] == GCRY_MD_NONE) {
+        error_setg(errp, "Unexpected hash algorithm %d", hash);
+        return -1;
+    }
+
+    ret = gcry_kdf_derive(key, nkey, GCRY_KDF_PBKDF2,
+                          hash_map[hash],
+                          salt, nsalt, iterations,
+                          nout, out);
+    if (ret != 0) {
+        error_setg(errp, "Cannot derive password: %s",
+                   gcry_strerror(ret));
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/crypto/pbkdf-nettle.c b/crypto/pbkdf-nettle.c
new file mode 100644
index 0000000..d681a60
--- /dev/null
+++ b/crypto/pbkdf-nettle.c
@@ -0,0 +1,66 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <nettle/pbkdf2.h>
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+
+
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash)
+{
+    switch (hash) {
+    case QCRYPTO_HASH_ALG_SHA1:
+    case QCRYPTO_HASH_ALG_SHA256:
+        return true;
+    default:
+        return false;
+    }
+}
+
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash,
+                   const uint8_t *key, size_t nkey,
+                   const uint8_t *salt, size_t nsalt,
+                   unsigned int iterations,
+                   uint8_t *out, size_t nout,
+                   Error **errp)
+{
+    switch (hash) {
+    case QCRYPTO_HASH_ALG_SHA1:
+        pbkdf2_hmac_sha1(nkey, key,
+                         iterations,
+                         nsalt, salt,
+                         nout, out);
+        break;
+
+    case QCRYPTO_HASH_ALG_SHA256:
+        pbkdf2_hmac_sha256(nkey, key,
+                           iterations,
+                           nsalt, salt,
+                           nout, out);
+        break;
+
+    default:
+        error_setg_errno(errp, ENOSYS,
+                         "PBKDF does not support hash algorithm %d", hash);
+        return -1;
+    }
+    return 0;
+}
diff --git a/crypto/pbkdf-stub.c b/crypto/pbkdf-stub.c
new file mode 100644
index 0000000..266a505
--- /dev/null
+++ b/crypto/pbkdf-stub.c
@@ -0,0 +1,43 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash G_GNUC_UNUSED)
+{
+    return false;
+}
+
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash G_GNUC_UNUSED,
+                   const uint8_t *key G_GNUC_UNUSED,
+                   size_t nkey G_GNUC_UNUSED,
+                   const uint8_t *salt G_GNUC_UNUSED,
+                   size_t nsalt G_GNUC_UNUSED,
+                   unsigned int iterations G_GNUC_UNUSED,
+                   uint8_t *out G_GNUC_UNUSED,
+                   size_t nout G_GNUC_UNUSED,
+                   Error **errp)
+{
+    error_setg_errno(errp, ENOSYS,
+                     "No crypto library supporting PBKDF in this build");
+    return -1;
+}
diff --git a/crypto/pbkdf.c b/crypto/pbkdf.c
new file mode 100644
index 0000000..695cc35
--- /dev/null
+++ b/crypto/pbkdf.c
@@ -0,0 +1,110 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/pbkdf.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+
+static int qcrypto_pbkdf2_get_thread_cpu(unsigned long long *val_ms,
+                                         Error **errp)
+{
+#ifdef _WIN32
+    FILETIME creation_time, exit_time, kernel_time, user_time;
+    ULARGE_INTEGER thread_time;
+
+    if (!GetThreadTimes(GetCurrentThread(), &creation_time, &exit_time,
+                        &kernel_time, &user_time)) {
+        error_setg(errp, "Unable to get thread CPU usage");
+        return -1;
+    }
+
+    thread_time.LowPart = user_time.dwLowDateTime;
+    thread_time.HighPart = user_time.dwHighDateTime;
+
+    /* QuadPart is units of 100ns and we want ms as unit */
+    *val_ms = thread_time.QuadPart / 10000ll;
+    return 0;
+#elif defined(RUSAGE_THREAD)
+    struct rusage ru;
+    if (getrusage(RUSAGE_THREAD, &ru) < 0) {
+        error_setg_errno(errp, errno, "Unable to get thread CPU usage");
+        return -1;
+    }
+
+    *val_ms = ((ru.ru_utime.tv_sec * 1000ll) +
+               (ru.ru_utime.tv_usec / 1000));
+    return 0;
+#else
+    *val_ms = 0;
+    error_setg(errp, "Unable to calculate thread CPU usage on this platform");
+    return -1;
+#endif
+}
+
+int qcrypto_pbkdf2_count_iters(QCryptoHashAlgorithm hash,
+                               const uint8_t *key, size_t nkey,
+                               const uint8_t *salt, size_t nsalt,
+                               Error **errp)
+{
+    uint8_t out[32];
+    long long int iterations = (1 << 15);
+    unsigned long long delta_ms, start_ms, end_ms;
+
+    while (1) {
+        if (qcrypto_pbkdf2_get_thread_cpu(&start_ms, errp) < 0) {
+            return -1;
+        }
+        if (qcrypto_pbkdf2(hash,
+                           key, nkey,
+                           salt, nsalt,
+                           iterations,
+                           out, sizeof(out),
+                           errp) < 0) {
+            return -1;
+        }
+        if (qcrypto_pbkdf2_get_thread_cpu(&end_ms, errp) < 0) {
+            return -1;
+        }
+
+        delta_ms = end_ms - start_ms;
+
+        if (delta_ms > 500) {
+            break;
+        } else if (delta_ms < 100) {
+            iterations = iterations * 10;
+        } else {
+            iterations = (iterations * 1000 / delta_ms);
+        }
+    }
+
+    iterations = iterations * 1000 / delta_ms;
+
+    if (iterations > INT32_MAX) {
+        error_setg(errp, "Iterations %lld too large for a 32-bit int",
+                   iterations);
+        return -1;
+    }
+
+    return iterations;
+}
diff --git a/crypto/random-gcrypt.c b/crypto/random-gcrypt.c
new file mode 100644
index 0000000..0de9a09
--- /dev/null
+++ b/crypto/random-gcrypt.c
@@ -0,0 +1,33 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/random.h"
+
+#include <gcrypt.h>
+
+int qcrypto_random_bytes(uint8_t *buf,
+                         size_t buflen,
+                         Error **errp G_GNUC_UNUSED)
+{
+    gcry_randomize(buf, buflen, GCRY_STRONG_RANDOM);
+    return 0;
+}
diff --git a/crypto/random-gnutls.c b/crypto/random-gnutls.c
new file mode 100644
index 0000000..04b45a8
--- /dev/null
+++ b/crypto/random-gnutls.c
@@ -0,0 +1,43 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/random.h"
+
+#include <gnutls/gnutls.h>
+#include <gnutls/crypto.h>
+
+int qcrypto_random_bytes(uint8_t *buf,
+                         size_t buflen,
+                         Error **errp)
+{
+    int ret;
+
+    ret = gnutls_rnd(GNUTLS_RND_RANDOM, buf, buflen);
+
+    if (ret < 0) {
+        error_setg(errp, "Cannot get random bytes: %s",
+                   gnutls_strerror(ret));
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/crypto/random-platform.c b/crypto/random-platform.c
new file mode 100644
index 0000000..82b755a
--- /dev/null
+++ b/crypto/random-platform.c
@@ -0,0 +1,64 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/random.h"
+
+int qcrypto_random_bytes(uint8_t *buf G_GNUC_UNUSED,
+                         size_t buflen G_GNUC_UNUSED,
+                         Error **errp)
+{
+    int fd;
+    int ret = -1;
+    int got;
+
+    /* TBD perhaps also add support for BSD getentropy / Linux
+     * getrandom syscalls directly */
+    fd = open("/dev/urandom", O_RDONLY);
+    if (fd == -1 && errno == ENOENT) {
+        fd = open("/dev/random", O_RDONLY);
+    }
+
+    if (fd < 0) {
+        error_setg(errp, "No /dev/urandom or /dev/random found");
+        return -1;
+    }
+
+    while (buflen > 0) {
+        got = read(fd, buf, buflen);
+        if (got < 0) {
+            error_setg_errno(errp, errno,
+                             "Unable to read random bytes");
+            goto cleanup;
+        } else if (!got) {
+            error_setg(errp,
+                       "Unexpected EOF reading random bytes");
+            goto cleanup;
+        }
+        buflen -= got;
+        buf += got;
+    }
+
+    ret = 0;
+ cleanup:
+    close(fd);
+    return ret;
+}
diff --git a/crypto/secret.c b/crypto/secret.c
new file mode 100644
index 0000000..285ab7a
--- /dev/null
+++ b/crypto/secret.c
@@ -0,0 +1,509 @@
+/*
+ * QEMU crypto secret support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/secret.h"
+#include "crypto/cipher.h"
+#include "qapi/error.h"
+#include "qom/object_interfaces.h"
+#include "qemu/base64.h"
+#include "trace.h"
+
+
+static void
+qcrypto_secret_load_data(QCryptoSecret *secret,
+                         uint8_t **output,
+                         size_t *outputlen,
+                         Error **errp)
+{
+    char *data = NULL;
+    size_t length = 0;
+    GError *gerr = NULL;
+
+    *output = NULL;
+    *outputlen = 0;
+
+    if (secret->file) {
+        if (secret->data) {
+            error_setg(errp,
+                       "'file' and 'data' are mutually exclusive");
+            return;
+        }
+        if (!g_file_get_contents(secret->file, &data, &length, &gerr)) {
+            error_setg(errp,
+                       "Unable to read %s: %s",
+                       secret->file, gerr->message);
+            g_error_free(gerr);
+            return;
+        }
+        *output = (uint8_t *)data;
+        *outputlen = length;
+    } else if (secret->data) {
+        *outputlen = strlen(secret->data);
+        *output = (uint8_t *)g_strdup(secret->data);
+    } else {
+        error_setg(errp, "Either 'file' or 'data' must be provided");
+    }
+}
+
+
+static void qcrypto_secret_decrypt(QCryptoSecret *secret,
+                                   const uint8_t *input,
+                                   size_t inputlen,
+                                   uint8_t **output,
+                                   size_t *outputlen,
+                                   Error **errp)
+{
+    uint8_t *key = NULL, *ciphertext = NULL, *iv = NULL;
+    size_t keylen, ciphertextlen, ivlen;
+    QCryptoCipher *aes = NULL;
+    uint8_t *plaintext = NULL;
+
+    *output = NULL;
+    *outputlen = 0;
+
+    if (qcrypto_secret_lookup(secret->keyid,
+                              &key, &keylen,
+                              errp) < 0) {
+        goto cleanup;
+    }
+
+    if (keylen != 32) {
+        error_setg(errp, "Key should be 32 bytes in length");
+        goto cleanup;
+    }
+
+    if (!secret->iv) {
+        error_setg(errp, "IV is required to decrypt secret");
+        goto cleanup;
+    }
+
+    iv = qbase64_decode(secret->iv, -1, &ivlen, errp);
+    if (!iv) {
+        goto cleanup;
+    }
+    if (ivlen != 16) {
+        error_setg(errp, "IV should be 16 bytes in length not %zu",
+                   ivlen);
+        goto cleanup;
+    }
+
+    aes = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_256,
+                             QCRYPTO_CIPHER_MODE_CBC,
+                             key, keylen,
+                             errp);
+    if (!aes) {
+        goto cleanup;
+    }
+
+    if (qcrypto_cipher_setiv(aes, iv, ivlen, errp) < 0) {
+        goto cleanup;
+    }
+
+    if (secret->format == QCRYPTO_SECRET_FORMAT_BASE64) {
+        ciphertext = qbase64_decode((const gchar*)input,
+                                    inputlen,
+                                    &ciphertextlen,
+                                    errp);
+        if (!ciphertext) {
+            goto cleanup;
+        }
+        plaintext = g_new0(uint8_t, ciphertextlen + 1);
+    } else {
+        ciphertextlen = inputlen;
+        plaintext = g_new0(uint8_t, inputlen + 1);
+    }
+    if (qcrypto_cipher_decrypt(aes,
+                               ciphertext ? ciphertext : input,
+                               plaintext,
+                               ciphertextlen,
+                               errp) < 0) {
+        plaintext = NULL;
+        goto cleanup;
+    }
+
+    if (plaintext[ciphertextlen - 1] > 16 ||
+        plaintext[ciphertextlen - 1] > ciphertextlen) {
+        error_setg(errp, "Incorrect number of padding bytes (%d) "
+                   "found on decrypted data",
+                   (int)plaintext[ciphertextlen - 1]);
+        g_free(plaintext);
+        plaintext = NULL;
+        goto cleanup;
+    }
+
+    /* Even though plaintext may contain arbitrary NUL
+     * ensure it is explicitly NUL terminated.
+     */
+    ciphertextlen -= plaintext[ciphertextlen - 1];
+    plaintext[ciphertextlen] = '\0';
+
+    *output = plaintext;
+    *outputlen = ciphertextlen;
+
+ cleanup:
+    g_free(ciphertext);
+    g_free(iv);
+    g_free(key);
+    qcrypto_cipher_free(aes);
+}
+
+
+static void qcrypto_secret_decode(const uint8_t *input,
+                                  size_t inputlen,
+                                  uint8_t **output,
+                                  size_t *outputlen,
+                                  Error **errp)
+{
+    *output = qbase64_decode((const gchar*)input,
+                             inputlen,
+                             outputlen,
+                             errp);
+}
+
+
+static void
+qcrypto_secret_prop_set_loaded(Object *obj,
+                               bool value,
+                               Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    if (value) {
+        Error *local_err = NULL;
+        uint8_t *input = NULL;
+        size_t inputlen = 0;
+        uint8_t *output = NULL;
+        size_t outputlen = 0;
+
+        qcrypto_secret_load_data(secret, &input, &inputlen, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        if (secret->keyid) {
+            qcrypto_secret_decrypt(secret, input, inputlen,
+                                   &output, &outputlen, &local_err);
+            g_free(input);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                return;
+            }
+            input = output;
+            inputlen = outputlen;
+        } else {
+            if (secret->format != QCRYPTO_SECRET_FORMAT_RAW) {
+                qcrypto_secret_decode(input, inputlen,
+                                      &output, &outputlen, &local_err);
+                g_free(input);
+                if (local_err) {
+                    error_propagate(errp, local_err);
+                    return;
+                }
+                input = output;
+                inputlen = outputlen;
+            }
+        }
+
+        secret->rawdata = input;
+        secret->rawlen = inputlen;
+    } else {
+        g_free(secret->rawdata);
+        secret->rawlen = 0;
+    }
+}
+
+
+static bool
+qcrypto_secret_prop_get_loaded(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return secret->data != NULL;
+}
+
+
+static void
+qcrypto_secret_prop_set_format(Object *obj,
+                               int value,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoSecret *creds = QCRYPTO_SECRET(obj);
+
+    creds->format = value;
+}
+
+
+static int
+qcrypto_secret_prop_get_format(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoSecret *creds = QCRYPTO_SECRET(obj);
+
+    return creds->format;
+}
+
+
+static void
+qcrypto_secret_prop_set_data(Object *obj,
+                             const char *value,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->data);
+    secret->data = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_data(Object *obj,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->data);
+}
+
+
+static void
+qcrypto_secret_prop_set_file(Object *obj,
+                             const char *value,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->file);
+    secret->file = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_file(Object *obj,
+                             Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->file);
+}
+
+
+static void
+qcrypto_secret_prop_set_iv(Object *obj,
+                           const char *value,
+                           Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->iv);
+    secret->iv = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_iv(Object *obj,
+                           Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->iv);
+}
+
+
+static void
+qcrypto_secret_prop_set_keyid(Object *obj,
+                              const char *value,
+                              Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->keyid);
+    secret->keyid = g_strdup(value);
+}
+
+
+static char *
+qcrypto_secret_prop_get_keyid(Object *obj,
+                              Error **errp)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+    return g_strdup(secret->keyid);
+}
+
+
+static void
+qcrypto_secret_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_secret_finalize(Object *obj)
+{
+    QCryptoSecret *secret = QCRYPTO_SECRET(obj);
+
+    g_free(secret->iv);
+    g_free(secret->file);
+    g_free(secret->keyid);
+    g_free(secret->rawdata);
+    g_free(secret->data);
+}
+
+static void
+qcrypto_secret_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_secret_complete;
+
+    object_class_property_add_bool(oc, "loaded",
+                                   qcrypto_secret_prop_get_loaded,
+                                   qcrypto_secret_prop_set_loaded,
+                                   NULL);
+    object_class_property_add_enum(oc, "format",
+                                   "QCryptoSecretFormat",
+                                   QCryptoSecretFormat_lookup,
+                                   qcrypto_secret_prop_get_format,
+                                   qcrypto_secret_prop_set_format,
+                                   NULL);
+    object_class_property_add_str(oc, "data",
+                                  qcrypto_secret_prop_get_data,
+                                  qcrypto_secret_prop_set_data,
+                                  NULL);
+    object_class_property_add_str(oc, "file",
+                                  qcrypto_secret_prop_get_file,
+                                  qcrypto_secret_prop_set_file,
+                                  NULL);
+    object_class_property_add_str(oc, "keyid",
+                                  qcrypto_secret_prop_get_keyid,
+                                  qcrypto_secret_prop_set_keyid,
+                                  NULL);
+    object_class_property_add_str(oc, "iv",
+                                  qcrypto_secret_prop_get_iv,
+                                  qcrypto_secret_prop_set_iv,
+                                  NULL);
+}
+
+
+int qcrypto_secret_lookup(const char *secretid,
+                          uint8_t **data,
+                          size_t *datalen,
+                          Error **errp)
+{
+    Object *obj;
+    QCryptoSecret *secret;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), secretid);
+    if (!obj) {
+        error_setg(errp, "No secret with id '%s'", secretid);
+        return -1;
+    }
+
+    secret = (QCryptoSecret *)
+        object_dynamic_cast(obj,
+                            TYPE_QCRYPTO_SECRET);
+    if (!secret) {
+        error_setg(errp, "Object with id '%s' is not a secret",
+                   secretid);
+        return -1;
+    }
+
+    if (!secret->rawdata) {
+        error_setg(errp, "Secret with id '%s' has no data",
+                   secretid);
+        return -1;
+    }
+
+    *data = g_new0(uint8_t, secret->rawlen + 1);
+    memcpy(*data, secret->rawdata, secret->rawlen);
+    (*data)[secret->rawlen] = '\0';
+    *datalen = secret->rawlen;
+
+    return 0;
+}
+
+
+char *qcrypto_secret_lookup_as_utf8(const char *secretid,
+                                    Error **errp)
+{
+    uint8_t *data;
+    size_t datalen;
+
+    if (qcrypto_secret_lookup(secretid,
+                              &data,
+                              &datalen,
+                              errp) < 0) {
+        return NULL;
+    }
+
+    if (!g_utf8_validate((const gchar*)data, datalen, NULL)) {
+        error_setg(errp,
+                   "Data from secret %s is not valid UTF-8",
+                   secretid);
+        g_free(data);
+        return NULL;
+    }
+
+    return (char *)data;
+}
+
+
+char *qcrypto_secret_lookup_as_base64(const char *secretid,
+                                      Error **errp)
+{
+    uint8_t *data;
+    size_t datalen;
+    char *ret;
+
+    if (qcrypto_secret_lookup(secretid,
+                              &data,
+                              &datalen,
+                              errp) < 0) {
+        return NULL;
+    }
+
+    ret = g_base64_encode(data, datalen);
+    g_free(data);
+    return ret;
+}
+
+
+static const TypeInfo qcrypto_secret_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QCRYPTO_SECRET,
+    .instance_size = sizeof(QCryptoSecret),
+    .instance_finalize = qcrypto_secret_finalize,
+    .class_size = sizeof(QCryptoSecretClass),
+    .class_init = qcrypto_secret_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_secret_register_types(void)
+{
+    type_register_static(&qcrypto_secret_info);
+}
+
+
+type_init(qcrypto_secret_register_types);
diff --git a/crypto/tlscreds.c b/crypto/tlscreds.c
new file mode 100644
index 0000000..dd7232b
--- /dev/null
+++ b/crypto/tlscreds.c
@@ -0,0 +1,285 @@
+/*
+ * QEMU crypto TLS credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/tlscredspriv.h"
+#include "trace.h"
+
+#define DH_BITS 2048
+
+#ifdef CONFIG_GNUTLS
+int
+qcrypto_tls_creds_get_dh_params_file(QCryptoTLSCreds *creds,
+                                     const char *filename,
+                                     gnutls_dh_params_t *dh_params,
+                                     Error **errp)
+{
+    int ret;
+
+    trace_qcrypto_tls_creds_load_dh(creds, filename ? filename : "<generated>");
+
+    if (filename == NULL) {
+        ret = gnutls_dh_params_init(dh_params);
+        if (ret < 0) {
+            error_setg(errp, "Unable to initialize DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+        ret = gnutls_dh_params_generate2(*dh_params, DH_BITS);
+        if (ret < 0) {
+            gnutls_dh_params_deinit(*dh_params);
+            *dh_params = NULL;
+            error_setg(errp, "Unable to generate DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+    } else {
+        GError *gerr = NULL;
+        gchar *contents;
+        gsize len;
+        gnutls_datum_t data;
+        if (!g_file_get_contents(filename,
+                                 &contents,
+                                 &len,
+                                 &gerr)) {
+
+            error_setg(errp, "%s", gerr->message);
+            g_error_free(gerr);
+            return -1;
+        }
+        data.data = (unsigned char *)contents;
+        data.size = len;
+        ret = gnutls_dh_params_init(dh_params);
+        if (ret < 0) {
+            g_free(contents);
+            error_setg(errp, "Unable to initialize DH parameters: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+        ret = gnutls_dh_params_import_pkcs3(*dh_params,
+                                            &data,
+                                            GNUTLS_X509_FMT_PEM);
+        g_free(contents);
+        if (ret < 0) {
+            gnutls_dh_params_deinit(*dh_params);
+            *dh_params = NULL;
+            error_setg(errp, "Unable to load DH parameters from %s: %s",
+                       filename, gnutls_strerror(ret));
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+
+int
+qcrypto_tls_creds_get_path(QCryptoTLSCreds *creds,
+                           const char *filename,
+                           bool required,
+                           char **cred,
+                           Error **errp)
+{
+    struct stat sb;
+    int ret = -1;
+
+    if (!creds->dir) {
+        if (required) {
+            error_setg(errp, "Missing 'dir' property value");
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    *cred = g_strdup_printf("%s/%s", creds->dir, filename);
+
+    if (qemu_stat(*cred, &sb) < 0) {
+        if (errno == ENOENT && !required) {
+            ret = 0;
+        } else {
+            error_setg_errno(errp, errno,
+                             "Unable to access credentials %s",
+                             *cred);
+        }
+        g_free(*cred);
+        *cred = NULL;
+        goto cleanup;
+    }
+
+    ret = 0;
+ cleanup:
+    trace_qcrypto_tls_creds_get_path(creds, filename,
+                                     *cred ? *cred : "<none>");
+    return ret;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_prop_set_verify(Object *obj,
+                                  bool value,
+                                  Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->verifyPeer = value;
+}
+
+
+static bool
+qcrypto_tls_creds_prop_get_verify(Object *obj,
+                                  Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return creds->verifyPeer;
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_dir(Object *obj,
+                               const char *value,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->dir = g_strdup(value);
+}
+
+
+static char *
+qcrypto_tls_creds_prop_get_dir(Object *obj,
+                               Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return g_strdup(creds->dir);
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_priority(Object *obj,
+                                    const char *value,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->priority = g_strdup(value);
+}
+
+
+static char *
+qcrypto_tls_creds_prop_get_priority(Object *obj,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return g_strdup(creds->priority);
+}
+
+
+static void
+qcrypto_tls_creds_prop_set_endpoint(Object *obj,
+                                    int value,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->endpoint = value;
+}
+
+
+static int
+qcrypto_tls_creds_prop_get_endpoint(Object *obj,
+                                    Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    return creds->endpoint;
+}
+
+
+static void
+qcrypto_tls_creds_class_init(ObjectClass *oc, void *data)
+{
+    object_class_property_add_bool(oc, "verify-peer",
+                                   qcrypto_tls_creds_prop_get_verify,
+                                   qcrypto_tls_creds_prop_set_verify,
+                                   NULL);
+    object_class_property_add_str(oc, "dir",
+                                  qcrypto_tls_creds_prop_get_dir,
+                                  qcrypto_tls_creds_prop_set_dir,
+                                  NULL);
+    object_class_property_add_enum(oc, "endpoint",
+                                   "QCryptoTLSCredsEndpoint",
+                                   QCryptoTLSCredsEndpoint_lookup,
+                                   qcrypto_tls_creds_prop_get_endpoint,
+                                   qcrypto_tls_creds_prop_set_endpoint,
+                                   NULL);
+    object_class_property_add_str(oc, "priority",
+                                  qcrypto_tls_creds_prop_get_priority,
+                                  qcrypto_tls_creds_prop_set_priority,
+                                  NULL);
+}
+
+
+static void
+qcrypto_tls_creds_init(Object *obj)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    creds->verifyPeer = true;
+}
+
+
+static void
+qcrypto_tls_creds_finalize(Object *obj)
+{
+    QCryptoTLSCreds *creds = QCRYPTO_TLS_CREDS(obj);
+
+    g_free(creds->dir);
+    g_free(creds->priority);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QCRYPTO_TLS_CREDS,
+    .instance_size = sizeof(QCryptoTLSCreds),
+    .instance_init = qcrypto_tls_creds_init,
+    .instance_finalize = qcrypto_tls_creds_finalize,
+    .class_init = qcrypto_tls_creds_class_init,
+    .class_size = sizeof(QCryptoTLSCredsClass),
+    .abstract = true,
+};
+
+
+static void
+qcrypto_tls_creds_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_info);
+}
+
+
+type_init(qcrypto_tls_creds_register_types);
diff --git a/crypto/tlscredsanon.c b/crypto/tlscredsanon.c
new file mode 100644
index 0000000..1464220
--- /dev/null
+++ b/crypto/tlscredsanon.c
@@ -0,0 +1,219 @@
+/*
+ * QEMU crypto TLS anonymous credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredspriv.h"
+#include "qapi/error.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static int
+qcrypto_tls_creds_anon_load(QCryptoTLSCredsAnon *creds,
+                            Error **errp)
+{
+    char *dhparams = NULL;
+    int ret;
+    int rv = -1;
+
+    trace_qcrypto_tls_creds_anon_load(creds,
+            creds->parent_obj.dir ? creds->parent_obj.dir : "<nodir>");
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_DH_PARAMS,
+                                       false, &dhparams, errp) < 0) {
+            goto cleanup;
+        }
+
+        ret = gnutls_anon_allocate_server_credentials(&creds->data.server);
+        if (ret < 0) {
+            error_setg(errp, "Cannot allocate credentials: %s",
+                       gnutls_strerror(ret));
+            goto cleanup;
+        }
+
+        if (qcrypto_tls_creds_get_dh_params_file(&creds->parent_obj, dhparams,
+                                                 &creds->parent_obj.dh_params,
+                                                 errp) < 0) {
+            goto cleanup;
+        }
+
+        gnutls_anon_set_server_dh_params(creds->data.server,
+                                         creds->parent_obj.dh_params);
+    } else {
+        ret = gnutls_anon_allocate_client_credentials(&creds->data.client);
+        if (ret < 0) {
+            error_setg(errp, "Cannot allocate credentials: %s",
+                       gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    rv = 0;
+ cleanup:
+    g_free(dhparams);
+    return rv;
+}
+
+
+static void
+qcrypto_tls_creds_anon_unload(QCryptoTLSCredsAnon *creds)
+{
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+        if (creds->data.client) {
+            gnutls_anon_free_client_credentials(creds->data.client);
+            creds->data.client = NULL;
+        }
+    } else {
+        if (creds->data.server) {
+            gnutls_anon_free_server_credentials(creds->data.server);
+            creds->data.server = NULL;
+        }
+    }
+    if (creds->parent_obj.dh_params) {
+        gnutls_dh_params_deinit(creds->parent_obj.dh_params);
+        creds->parent_obj.dh_params = NULL;
+    }
+}
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_load(QCryptoTLSCredsAnon *creds G_GNUC_UNUSED,
+                            Error **errp)
+{
+    error_setg(errp, "TLS credentials support requires GNUTLS");
+}
+
+
+static void
+qcrypto_tls_creds_anon_unload(QCryptoTLSCredsAnon *creds G_GNUC_UNUSED)
+{
+    /* nada */
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_prop_set_loaded(Object *obj,
+                                       bool value,
+                                       Error **errp)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    if (value) {
+        qcrypto_tls_creds_anon_load(creds, errp);
+    } else {
+        qcrypto_tls_creds_anon_unload(creds);
+    }
+}
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static bool
+qcrypto_tls_creds_anon_prop_get_loaded(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        return creds->data.server != NULL;
+    } else {
+        return creds->data.client != NULL;
+    }
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static bool
+qcrypto_tls_creds_anon_prop_get_loaded(Object *obj G_GNUC_UNUSED,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_anon_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_tls_creds_anon_finalize(Object *obj)
+{
+    QCryptoTLSCredsAnon *creds = QCRYPTO_TLS_CREDS_ANON(obj);
+
+    qcrypto_tls_creds_anon_unload(creds);
+}
+
+
+static void
+qcrypto_tls_creds_anon_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_tls_creds_anon_complete;
+
+    object_class_property_add_bool(oc, "loaded",
+                                   qcrypto_tls_creds_anon_prop_get_loaded,
+                                   qcrypto_tls_creds_anon_prop_set_loaded,
+                                   NULL);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_anon_info = {
+    .parent = TYPE_QCRYPTO_TLS_CREDS,
+    .name = TYPE_QCRYPTO_TLS_CREDS_ANON,
+    .instance_size = sizeof(QCryptoTLSCredsAnon),
+    .instance_finalize = qcrypto_tls_creds_anon_finalize,
+    .class_size = sizeof(QCryptoTLSCredsAnonClass),
+    .class_init = qcrypto_tls_creds_anon_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_tls_creds_anon_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_anon_info);
+}
+
+
+type_init(qcrypto_tls_creds_anon_register_types);
diff --git a/crypto/tlscredspriv.h b/crypto/tlscredspriv.h
new file mode 100644
index 0000000..13e9b6c
--- /dev/null
+++ b/crypto/tlscredspriv.h
@@ -0,0 +1,41 @@
+/*
+ * QEMU crypto TLS credential support private helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCREDSPRIV_H
+#define QCRYPTO_TLSCREDSPRIV_H
+
+#include "crypto/tlscreds.h"
+
+#ifdef CONFIG_GNUTLS
+
+int qcrypto_tls_creds_get_path(QCryptoTLSCreds *creds,
+                               const char *filename,
+                               bool required,
+                               char **cred,
+                               Error **errp);
+
+int qcrypto_tls_creds_get_dh_params_file(QCryptoTLSCreds *creds,
+                                         const char *filename,
+                                         gnutls_dh_params_t *dh_params,
+                                         Error **errp);
+
+#endif
+
+#endif /* QCRYPTO_TLSCREDSPRIV_H */
diff --git a/crypto/tlscredsx509.c b/crypto/tlscredsx509.c
new file mode 100644
index 0000000..520d34d
--- /dev/null
+++ b/crypto/tlscredsx509.c
@@ -0,0 +1,870 @@
+/*
+ * QEMU crypto TLS x509 credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/tlscredsx509.h"
+#include "crypto/tlscredspriv.h"
+#include "crypto/secret.h"
+#include "qapi/error.h"
+#include "qom/object_interfaces.h"
+#include "trace.h"
+
+
+#ifdef CONFIG_GNUTLS
+
+#include <gnutls/x509.h>
+
+
+static int
+qcrypto_tls_creds_check_cert_times(gnutls_x509_crt_t cert,
+                                   const char *certFile,
+                                   bool isServer,
+                                   bool isCA,
+                                   Error **errp)
+{
+    time_t now = time(NULL);
+
+    if (now == ((time_t)-1)) {
+        error_setg_errno(errp, errno, "cannot get current time");
+        return -1;
+    }
+
+    if (gnutls_x509_crt_get_expiration_time(cert) < now) {
+        error_setg(errp,
+                   (isCA ?
+                    "The CA certificate %s has expired" :
+                    (isServer ?
+                     "The server certificate %s has expired" :
+                     "The client certificate %s has expired")),
+                   certFile);
+        return -1;
+    }
+
+    if (gnutls_x509_crt_get_activation_time(cert) > now) {
+        error_setg(errp,
+                   (isCA ?
+                    "The CA certificate %s is not yet active" :
+                    (isServer ?
+                     "The server certificate %s is not yet active" :
+                     "The client certificate %s is not yet active")),
+                   certFile);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+#if LIBGNUTLS_VERSION_NUMBER >= 2
+/*
+ * The gnutls_x509_crt_get_basic_constraints function isn't
+ * available in GNUTLS 1.0.x branches. This isn't critical
+ * though, since gnutls_certificate_verify_peers2 will do
+ * pretty much the same check at runtime, so we can just
+ * disable this code
+ */
+static int
+qcrypto_tls_creds_check_cert_basic_constraints(QCryptoTLSCredsX509 *creds,
+                                               gnutls_x509_crt_t cert,
+                                               const char *certFile,
+                                               bool isServer,
+                                               bool isCA,
+                                               Error **errp)
+{
+    int status;
+
+    status = gnutls_x509_crt_get_basic_constraints(cert, NULL, NULL, NULL);
+    trace_qcrypto_tls_creds_x509_check_basic_constraints(
+        creds, certFile, status);
+
+    if (status > 0) { /* It is a CA cert */
+        if (!isCA) {
+            error_setg(errp, isServer ?
+                       "The certificate %s basic constraints show a CA, "
+                       "but we need one for a server" :
+                       "The certificate %s basic constraints show a CA, "
+                       "but we need one for a client",
+                       certFile);
+            return -1;
+        }
+    } else if (status == 0) { /* It is not a CA cert */
+        if (isCA) {
+            error_setg(errp,
+                       "The certificate %s basic constraints do not "
+                       "show a CA",
+                       certFile);
+            return -1;
+        }
+    } else if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+        /* Missing basicConstraints */
+        if (isCA) {
+            error_setg(errp,
+                       "The certificate %s is missing basic constraints "
+                       "for a CA",
+                       certFile);
+            return -1;
+        }
+    } else { /* General error */
+        error_setg(errp,
+                   "Unable to query certificate %s basic constraints: %s",
+                   certFile, gnutls_strerror(status));
+        return -1;
+    }
+
+    return 0;
+}
+#endif
+
+
+static int
+qcrypto_tls_creds_check_cert_key_usage(QCryptoTLSCredsX509 *creds,
+                                       gnutls_x509_crt_t cert,
+                                       const char *certFile,
+                                       bool isCA,
+                                       Error **errp)
+{
+    int status;
+    unsigned int usage = 0;
+    unsigned int critical = 0;
+
+    status = gnutls_x509_crt_get_key_usage(cert, &usage, &critical);
+    trace_qcrypto_tls_creds_x509_check_key_usage(
+        creds, certFile, status, usage, critical);
+
+    if (status < 0) {
+        if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+            usage = isCA ? GNUTLS_KEY_KEY_CERT_SIGN :
+                GNUTLS_KEY_DIGITAL_SIGNATURE|GNUTLS_KEY_KEY_ENCIPHERMENT;
+        } else {
+            error_setg(errp,
+                       "Unable to query certificate %s key usage: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+    }
+
+    if (isCA) {
+        if (!(usage & GNUTLS_KEY_KEY_CERT_SIGN)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit "
+                           "certificate signing", certFile);
+                return -1;
+            }
+        }
+    } else {
+        if (!(usage & GNUTLS_KEY_DIGITAL_SIGNATURE)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit digital "
+                           "signature", certFile);
+                return -1;
+            }
+        }
+        if (!(usage & GNUTLS_KEY_KEY_ENCIPHERMENT)) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s usage does not permit key "
+                           "encipherment", certFile);
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert_key_purpose(QCryptoTLSCredsX509 *creds,
+                                         gnutls_x509_crt_t cert,
+                                         const char *certFile,
+                                         bool isServer,
+                                         Error **errp)
+{
+    int status;
+    size_t i;
+    unsigned int purposeCritical;
+    unsigned int critical;
+    char *buffer = NULL;
+    size_t size;
+    bool allowClient = false, allowServer = false;
+
+    critical = 0;
+    for (i = 0; ; i++) {
+        size = 0;
+        status = gnutls_x509_crt_get_key_purpose_oid(cert, i, buffer,
+                                                     &size, NULL);
+
+        if (status == GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+
+            /* If there is no data at all, then we must allow
+               client/server to pass */
+            if (i == 0) {
+                allowServer = allowClient = true;
+            }
+            break;
+        }
+        if (status != GNUTLS_E_SHORT_MEMORY_BUFFER) {
+            error_setg(errp,
+                       "Unable to query certificate %s key purpose: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+
+        buffer = g_new0(char, size);
+
+        status = gnutls_x509_crt_get_key_purpose_oid(cert, i, buffer,
+                                                     &size, &purposeCritical);
+
+        if (status < 0) {
+            trace_qcrypto_tls_creds_x509_check_key_purpose(
+                creds, certFile, status, "<none>", purposeCritical);
+            g_free(buffer);
+            error_setg(errp,
+                       "Unable to query certificate %s key purpose: %s",
+                       certFile, gnutls_strerror(status));
+            return -1;
+        }
+        trace_qcrypto_tls_creds_x509_check_key_purpose(
+            creds, certFile, status, buffer, purposeCritical);
+        if (purposeCritical) {
+            critical = true;
+        }
+
+        if (g_str_equal(buffer, GNUTLS_KP_TLS_WWW_SERVER)) {
+            allowServer = true;
+        } else if (g_str_equal(buffer, GNUTLS_KP_TLS_WWW_CLIENT)) {
+            allowClient = true;
+        } else if (g_str_equal(buffer, GNUTLS_KP_ANY)) {
+            allowServer = allowClient = true;
+        }
+
+        g_free(buffer);
+        buffer = NULL;
+    }
+
+    if (isServer) {
+        if (!allowServer) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s purpose does not allow "
+                           "use with a TLS server", certFile);
+                return -1;
+            }
+        }
+    } else {
+        if (!allowClient) {
+            if (critical) {
+                error_setg(errp,
+                           "Certificate %s purpose does not allow use "
+                           "with a TLS client", certFile);
+                return -1;
+            }
+        }
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert(QCryptoTLSCredsX509 *creds,
+                             gnutls_x509_crt_t cert,
+                             const char *certFile,
+                             bool isServer,
+                             bool isCA,
+                             Error **errp)
+{
+    if (qcrypto_tls_creds_check_cert_times(cert, certFile,
+                                           isServer, isCA,
+                                           errp) < 0) {
+        return -1;
+    }
+
+#if LIBGNUTLS_VERSION_NUMBER >= 2
+    if (qcrypto_tls_creds_check_cert_basic_constraints(creds,
+                                                       cert, certFile,
+                                                       isServer, isCA,
+                                                       errp) < 0) {
+        return -1;
+    }
+#endif
+
+    if (qcrypto_tls_creds_check_cert_key_usage(creds,
+                                               cert, certFile,
+                                               isCA, errp) < 0) {
+        return -1;
+    }
+
+    if (!isCA &&
+        qcrypto_tls_creds_check_cert_key_purpose(creds,
+                                                 cert, certFile,
+                                                 isServer, errp) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int
+qcrypto_tls_creds_check_cert_pair(gnutls_x509_crt_t cert,
+                                  const char *certFile,
+                                  gnutls_x509_crt_t *cacerts,
+                                  size_t ncacerts,
+                                  const char *cacertFile,
+                                  bool isServer,
+                                  Error **errp)
+{
+    unsigned int status;
+
+    if (gnutls_x509_crt_list_verify(&cert, 1,
+                                    cacerts, ncacerts,
+                                    NULL, 0,
+                                    0, &status) < 0) {
+        error_setg(errp, isServer ?
+                   "Unable to verify server certificate %s against "
+                   "CA certificate %s" :
+                   "Unable to verify client certificate %s against "
+                   "CA certificate %s",
+                   certFile, cacertFile);
+        return -1;
+    }
+
+    if (status != 0) {
+        const char *reason = "Invalid certificate";
+
+        if (status & GNUTLS_CERT_INVALID) {
+            reason = "The certificate is not trusted";
+        }
+
+        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
+            reason = "The certificate hasn't got a known issuer";
+        }
+
+        if (status & GNUTLS_CERT_REVOKED) {
+            reason = "The certificate has been revoked";
+        }
+
+#ifndef GNUTLS_1_0_COMPAT
+        if (status & GNUTLS_CERT_INSECURE_ALGORITHM) {
+            reason = "The certificate uses an insecure algorithm";
+        }
+#endif
+
+        error_setg(errp,
+                   "Our own certificate %s failed validation against %s: %s",
+                   certFile, cacertFile, reason);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static gnutls_x509_crt_t
+qcrypto_tls_creds_load_cert(QCryptoTLSCredsX509 *creds,
+                            const char *certFile,
+                            bool isServer,
+                            Error **errp)
+{
+    gnutls_datum_t data;
+    gnutls_x509_crt_t cert = NULL;
+    char *buf = NULL;
+    gsize buflen;
+    GError *gerr;
+    int ret = -1;
+    int err;
+
+    trace_qcrypto_tls_creds_x509_load_cert(creds, isServer, certFile);
+
+    err = gnutls_x509_crt_init(&cert);
+    if (err < 0) {
+        error_setg(errp, "Unable to initialize certificate: %s",
+                   gnutls_strerror(err));
+        goto cleanup;
+    }
+
+    if (!g_file_get_contents(certFile, &buf, &buflen, &gerr)) {
+        error_setg(errp, "Cannot load CA cert list %s: %s",
+                   certFile, gerr->message);
+        g_error_free(gerr);
+        goto cleanup;
+    }
+
+    data.data = (unsigned char *)buf;
+    data.size = strlen(buf);
+
+    err = gnutls_x509_crt_import(cert, &data, GNUTLS_X509_FMT_PEM);
+    if (err < 0) {
+        error_setg(errp, isServer ?
+                   "Unable to import server certificate %s: %s" :
+                   "Unable to import client certificate %s: %s",
+                   certFile,
+                   gnutls_strerror(err));
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    if (ret != 0) {
+        gnutls_x509_crt_deinit(cert);
+        cert = NULL;
+    }
+    g_free(buf);
+    return cert;
+}
+
+
+static int
+qcrypto_tls_creds_load_ca_cert_list(QCryptoTLSCredsX509 *creds,
+                                    const char *certFile,
+                                    gnutls_x509_crt_t *certs,
+                                    unsigned int certMax,
+                                    size_t *ncerts,
+                                    Error **errp)
+{
+    gnutls_datum_t data;
+    char *buf = NULL;
+    gsize buflen;
+    int ret = -1;
+    GError *gerr = NULL;
+
+    *ncerts = 0;
+    trace_qcrypto_tls_creds_x509_load_cert_list(creds, certFile);
+
+    if (!g_file_get_contents(certFile, &buf, &buflen, &gerr)) {
+        error_setg(errp, "Cannot load CA cert list %s: %s",
+                   certFile, gerr->message);
+        g_error_free(gerr);
+        goto cleanup;
+    }
+
+    data.data = (unsigned char *)buf;
+    data.size = strlen(buf);
+
+    if (gnutls_x509_crt_list_import(certs, &certMax, &data,
+                                    GNUTLS_X509_FMT_PEM, 0) < 0) {
+        error_setg(errp,
+                   "Unable to import CA certificate list %s",
+                   certFile);
+        goto cleanup;
+    }
+    *ncerts = certMax;
+
+    ret = 0;
+
+ cleanup:
+    g_free(buf);
+    return ret;
+}
+
+
+#define MAX_CERTS 16
+static int
+qcrypto_tls_creds_x509_sanity_check(QCryptoTLSCredsX509 *creds,
+                                    bool isServer,
+                                    const char *cacertFile,
+                                    const char *certFile,
+                                    Error **errp)
+{
+    gnutls_x509_crt_t cert = NULL;
+    gnutls_x509_crt_t cacerts[MAX_CERTS];
+    size_t ncacerts = 0;
+    size_t i;
+    int ret = -1;
+
+    memset(cacerts, 0, sizeof(cacerts));
+    if (certFile &&
+        access(certFile, R_OK) == 0) {
+        cert = qcrypto_tls_creds_load_cert(creds,
+                                           certFile, isServer,
+                                           errp);
+        if (!cert) {
+            goto cleanup;
+        }
+    }
+    if (access(cacertFile, R_OK) == 0) {
+        if (qcrypto_tls_creds_load_ca_cert_list(creds,
+                                                cacertFile, cacerts,
+                                                MAX_CERTS, &ncacerts,
+                                                errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (cert &&
+        qcrypto_tls_creds_check_cert(creds,
+                                     cert, certFile, isServer,
+                                     false, errp) < 0) {
+        goto cleanup;
+    }
+
+    for (i = 0; i < ncacerts; i++) {
+        if (qcrypto_tls_creds_check_cert(creds,
+                                         cacerts[i], cacertFile,
+                                         isServer, true, errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (cert && ncacerts &&
+        qcrypto_tls_creds_check_cert_pair(cert, certFile, cacerts,
+                                          ncacerts, cacertFile,
+                                          isServer, errp) < 0) {
+        goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    if (cert) {
+        gnutls_x509_crt_deinit(cert);
+    }
+    for (i = 0; i < ncacerts; i++) {
+        gnutls_x509_crt_deinit(cacerts[i]);
+    }
+    return ret;
+}
+
+
+static int
+qcrypto_tls_creds_x509_load(QCryptoTLSCredsX509 *creds,
+                            Error **errp)
+{
+    char *cacert = NULL, *cacrl = NULL, *cert = NULL,
+        *key = NULL, *dhparams = NULL;
+    int ret;
+    int rv = -1;
+
+    trace_qcrypto_tls_creds_x509_load(creds,
+            creds->parent_obj.dir ? creds->parent_obj.dir : "<nodir>");
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CERT,
+                                       true, &cacert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CRL,
+                                       false, &cacrl, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_SERVER_CERT,
+                                       true, &cert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_SERVER_KEY,
+                                       true, &key, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_DH_PARAMS,
+                                       false, &dhparams, errp) < 0) {
+            goto cleanup;
+        }
+    } else {
+        if (qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CA_CERT,
+                                       true, &cacert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CLIENT_CERT,
+                                       false, &cert, errp) < 0 ||
+            qcrypto_tls_creds_get_path(&creds->parent_obj,
+                                       QCRYPTO_TLS_CREDS_X509_CLIENT_KEY,
+                                       false, &key, errp) < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (creds->sanityCheck &&
+        qcrypto_tls_creds_x509_sanity_check(creds,
+            creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+            cacert, cert, errp) < 0) {
+        goto cleanup;
+    }
+
+    ret = gnutls_certificate_allocate_credentials(&creds->data);
+    if (ret < 0) {
+        error_setg(errp, "Cannot allocate credentials: '%s'",
+                   gnutls_strerror(ret));
+        goto cleanup;
+    }
+
+    ret = gnutls_certificate_set_x509_trust_file(creds->data,
+                                                 cacert,
+                                                 GNUTLS_X509_FMT_PEM);
+    if (ret < 0) {
+        error_setg(errp, "Cannot load CA certificate '%s': %s",
+                   cacert, gnutls_strerror(ret));
+        goto cleanup;
+    }
+
+    if (cert != NULL && key != NULL) {
+#if GNUTLS_VERSION_NUMBER >= 0x030111
+        char *password = NULL;
+        if (creds->passwordid) {
+            password = qcrypto_secret_lookup_as_utf8(creds->passwordid,
+                                                     errp);
+            if (!password) {
+                goto cleanup;
+            }
+        }
+        ret = gnutls_certificate_set_x509_key_file2(creds->data,
+                                                    cert, key,
+                                                    GNUTLS_X509_FMT_PEM,
+                                                    password,
+                                                    0);
+        g_free(password);
+#else /* GNUTLS_VERSION_NUMBER < 0x030111 */
+        if (creds->passwordid) {
+            error_setg(errp, "PKCS8 decryption requires GNUTLS >= 3.1.11");
+            goto cleanup;
+        }
+        ret = gnutls_certificate_set_x509_key_file(creds->data,
+                                                   cert, key,
+                                                   GNUTLS_X509_FMT_PEM);
+#endif /* GNUTLS_VERSION_NUMBER < 0x030111 */
+        if (ret < 0) {
+            error_setg(errp, "Cannot load certificate '%s' & key '%s': %s",
+                       cert, key, gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    if (cacrl != NULL) {
+        ret = gnutls_certificate_set_x509_crl_file(creds->data,
+                                                   cacrl,
+                                                   GNUTLS_X509_FMT_PEM);
+        if (ret < 0) {
+            error_setg(errp, "Cannot load CRL '%s': %s",
+                       cacrl, gnutls_strerror(ret));
+            goto cleanup;
+        }
+    }
+
+    if (creds->parent_obj.endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        if (qcrypto_tls_creds_get_dh_params_file(&creds->parent_obj, dhparams,
+                                                 &creds->parent_obj.dh_params,
+                                                 errp) < 0) {
+            goto cleanup;
+        }
+        gnutls_certificate_set_dh_params(creds->data,
+                                         creds->parent_obj.dh_params);
+    }
+
+    rv = 0;
+ cleanup:
+    g_free(cacert);
+    g_free(cacrl);
+    g_free(cert);
+    g_free(key);
+    g_free(dhparams);
+    return rv;
+}
+
+
+static void
+qcrypto_tls_creds_x509_unload(QCryptoTLSCredsX509 *creds)
+{
+    if (creds->data) {
+        gnutls_certificate_free_credentials(creds->data);
+        creds->data = NULL;
+    }
+    if (creds->parent_obj.dh_params) {
+        gnutls_dh_params_deinit(creds->parent_obj.dh_params);
+        creds->parent_obj.dh_params = NULL;
+    }
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_load(QCryptoTLSCredsX509 *creds G_GNUC_UNUSED,
+                            Error **errp)
+{
+    error_setg(errp, "TLS credentials support requires GNUTLS");
+}
+
+
+static void
+qcrypto_tls_creds_x509_unload(QCryptoTLSCredsX509 *creds G_GNUC_UNUSED)
+{
+    /* nada */
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_loaded(Object *obj,
+                                       bool value,
+                                       Error **errp)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    if (value) {
+        qcrypto_tls_creds_x509_load(creds, errp);
+    } else {
+        qcrypto_tls_creds_x509_unload(creds);
+    }
+}
+
+
+#ifdef CONFIG_GNUTLS
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_loaded(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return creds->data != NULL;
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_loaded(Object *obj G_GNUC_UNUSED,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    return false;
+}
+
+
+#endif /* ! CONFIG_GNUTLS */
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_sanity(Object *obj,
+                                       bool value,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->sanityCheck = value;
+}
+
+
+static void
+qcrypto_tls_creds_x509_prop_set_passwordid(Object *obj,
+                                           const char *value,
+                                           Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->passwordid = g_strdup(value);
+}
+
+
+static char *
+qcrypto_tls_creds_x509_prop_get_passwordid(Object *obj,
+                                           Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return g_strdup(creds->passwordid);
+}
+
+
+static bool
+qcrypto_tls_creds_x509_prop_get_sanity(Object *obj,
+                                       Error **errp G_GNUC_UNUSED)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    return creds->sanityCheck;
+}
+
+
+static void
+qcrypto_tls_creds_x509_complete(UserCreatable *uc, Error **errp)
+{
+    object_property_set_bool(OBJECT(uc), true, "loaded", errp);
+}
+
+
+static void
+qcrypto_tls_creds_x509_init(Object *obj)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    creds->sanityCheck = true;
+}
+
+
+static void
+qcrypto_tls_creds_x509_finalize(Object *obj)
+{
+    QCryptoTLSCredsX509 *creds = QCRYPTO_TLS_CREDS_X509(obj);
+
+    g_free(creds->passwordid);
+    qcrypto_tls_creds_x509_unload(creds);
+}
+
+
+static void
+qcrypto_tls_creds_x509_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = qcrypto_tls_creds_x509_complete;
+
+    object_class_property_add_bool(oc, "loaded",
+                                   qcrypto_tls_creds_x509_prop_get_loaded,
+                                   qcrypto_tls_creds_x509_prop_set_loaded,
+                                   NULL);
+    object_class_property_add_bool(oc, "sanity-check",
+                                   qcrypto_tls_creds_x509_prop_get_sanity,
+                                   qcrypto_tls_creds_x509_prop_set_sanity,
+                                   NULL);
+    object_class_property_add_str(oc, "passwordid",
+                                  qcrypto_tls_creds_x509_prop_get_passwordid,
+                                  qcrypto_tls_creds_x509_prop_set_passwordid,
+                                  NULL);
+}
+
+
+static const TypeInfo qcrypto_tls_creds_x509_info = {
+    .parent = TYPE_QCRYPTO_TLS_CREDS,
+    .name = TYPE_QCRYPTO_TLS_CREDS_X509,
+    .instance_size = sizeof(QCryptoTLSCredsX509),
+    .instance_init = qcrypto_tls_creds_x509_init,
+    .instance_finalize = qcrypto_tls_creds_x509_finalize,
+    .class_size = sizeof(QCryptoTLSCredsX509Class),
+    .class_init = qcrypto_tls_creds_x509_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+
+static void
+qcrypto_tls_creds_x509_register_types(void)
+{
+    type_register_static(&qcrypto_tls_creds_x509_info);
+}
+
+
+type_init(qcrypto_tls_creds_x509_register_types);
diff --git a/crypto/tlssession.c b/crypto/tlssession.c
new file mode 100644
index 0000000..2de42c6
--- /dev/null
+++ b/crypto/tlssession.c
@@ -0,0 +1,588 @@
+/*
+ * QEMU crypto TLS session support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/tlssession.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredsx509.h"
+#include "qapi/error.h"
+#include "qemu/acl.h"
+#include "trace.h"
+
+#ifdef CONFIG_GNUTLS
+
+
+#include <gnutls/x509.h>
+
+
+struct QCryptoTLSSession {
+    QCryptoTLSCreds *creds;
+    gnutls_session_t handle;
+    char *hostname;
+    char *aclname;
+    bool handshakeComplete;
+    QCryptoTLSSessionWriteFunc writeFunc;
+    QCryptoTLSSessionReadFunc readFunc;
+    void *opaque;
+    char *peername;
+};
+
+
+void
+qcrypto_tls_session_free(QCryptoTLSSession *session)
+{
+    if (!session) {
+        return;
+    }
+
+    gnutls_deinit(session->handle);
+    g_free(session->hostname);
+    g_free(session->peername);
+    g_free(session->aclname);
+    object_unref(OBJECT(session->creds));
+    g_free(session);
+}
+
+
+static ssize_t
+qcrypto_tls_session_push(void *opaque, const void *buf, size_t len)
+{
+    QCryptoTLSSession *session = opaque;
+
+    if (!session->writeFunc) {
+        errno = EIO;
+        return -1;
+    };
+
+    return session->writeFunc(buf, len, session->opaque);
+}
+
+
+static ssize_t
+qcrypto_tls_session_pull(void *opaque, void *buf, size_t len)
+{
+    QCryptoTLSSession *session = opaque;
+
+    if (!session->readFunc) {
+        errno = EIO;
+        return -1;
+    };
+
+    return session->readFunc(buf, len, session->opaque);
+}
+
+
+QCryptoTLSSession *
+qcrypto_tls_session_new(QCryptoTLSCreds *creds,
+                        const char *hostname,
+                        const char *aclname,
+                        QCryptoTLSCredsEndpoint endpoint,
+                        Error **errp)
+{
+    QCryptoTLSSession *session;
+    int ret;
+
+    session = g_new0(QCryptoTLSSession, 1);
+    trace_qcrypto_tls_session_new(
+        session, creds, hostname ? hostname : "<none>",
+        aclname ? aclname : "<none>", endpoint);
+
+    if (hostname) {
+        session->hostname = g_strdup(hostname);
+    }
+    if (aclname) {
+        session->aclname = g_strdup(aclname);
+    }
+    session->creds = creds;
+    object_ref(OBJECT(creds));
+
+    if (creds->endpoint != endpoint) {
+        error_setg(errp, "Credentials endpoint doesn't match session");
+        goto error;
+    }
+
+    if (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        ret = gnutls_init(&session->handle, GNUTLS_SERVER);
+    } else {
+        ret = gnutls_init(&session->handle, GNUTLS_CLIENT);
+    }
+    if (ret < 0) {
+        error_setg(errp, "Cannot initialize TLS session: %s",
+                   gnutls_strerror(ret));
+        goto error;
+    }
+
+    if (object_dynamic_cast(OBJECT(creds),
+                            TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+        QCryptoTLSCredsAnon *acreds = QCRYPTO_TLS_CREDS_ANON(creds);
+        char *prio;
+
+        if (creds->priority != NULL) {
+            prio = g_strdup_printf("%s:+ANON-DH", creds->priority);
+        } else {
+            prio = g_strdup(CONFIG_TLS_PRIORITY ":+ANON-DH");
+        }
+
+        ret = gnutls_priority_set_direct(session->handle, prio, NULL);
+        if (ret < 0) {
+            error_setg(errp, "Unable to set TLS session priority %s: %s",
+                       prio, gnutls_strerror(ret));
+            g_free(prio);
+            goto error;
+        }
+        g_free(prio);
+        if (creds->endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            ret = gnutls_credentials_set(session->handle,
+                                         GNUTLS_CRD_ANON,
+                                         acreds->data.server);
+        } else {
+            ret = gnutls_credentials_set(session->handle,
+                                         GNUTLS_CRD_ANON,
+                                         acreds->data.client);
+        }
+        if (ret < 0) {
+            error_setg(errp, "Cannot set session credentials: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+    } else if (object_dynamic_cast(OBJECT(creds),
+                                   TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        QCryptoTLSCredsX509 *tcreds = QCRYPTO_TLS_CREDS_X509(creds);
+        const char *prio = creds->priority;
+        if (!prio) {
+            prio = CONFIG_TLS_PRIORITY;
+        }
+
+        ret = gnutls_priority_set_direct(session->handle, prio, NULL);
+        if (ret < 0) {
+            error_setg(errp, "Cannot set default TLS session priority %s: %s",
+                       prio, gnutls_strerror(ret));
+            goto error;
+        }
+        ret = gnutls_credentials_set(session->handle,
+                                     GNUTLS_CRD_CERTIFICATE,
+                                     tcreds->data);
+        if (ret < 0) {
+            error_setg(errp, "Cannot set session credentials: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+
+        if (creds->endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            /* This requests, but does not enforce a client cert.
+             * The cert checking code later does enforcement */
+            gnutls_certificate_server_set_request(session->handle,
+                                                  GNUTLS_CERT_REQUEST);
+        }
+    } else {
+        error_setg(errp, "Unsupported TLS credentials type %s",
+                   object_get_typename(OBJECT(creds)));
+        goto error;
+    }
+
+    gnutls_transport_set_ptr(session->handle, session);
+    gnutls_transport_set_push_function(session->handle,
+                                       qcrypto_tls_session_push);
+    gnutls_transport_set_pull_function(session->handle,
+                                       qcrypto_tls_session_pull);
+
+    return session;
+
+ error:
+    qcrypto_tls_session_free(session);
+    return NULL;
+}
+
+static int
+qcrypto_tls_session_check_certificate(QCryptoTLSSession *session,
+                                      Error **errp)
+{
+    int ret;
+    unsigned int status;
+    const gnutls_datum_t *certs;
+    unsigned int nCerts, i;
+    time_t now;
+    gnutls_x509_crt_t cert = NULL;
+
+    now = time(NULL);
+    if (now == ((time_t)-1)) {
+        error_setg_errno(errp, errno, "Cannot get current time");
+        return -1;
+    }
+
+    ret = gnutls_certificate_verify_peers2(session->handle, &status);
+    if (ret < 0) {
+        error_setg(errp, "Verify failed: %s", gnutls_strerror(ret));
+        return -1;
+    }
+
+    if (status != 0) {
+        const char *reason = "Invalid certificate";
+
+        if (status & GNUTLS_CERT_INVALID) {
+            reason = "The certificate is not trusted";
+        }
+
+        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND) {
+            reason = "The certificate hasn't got a known issuer";
+        }
+
+        if (status & GNUTLS_CERT_REVOKED) {
+            reason = "The certificate has been revoked";
+        }
+
+        if (status & GNUTLS_CERT_INSECURE_ALGORITHM) {
+            reason = "The certificate uses an insecure algorithm";
+        }
+
+        error_setg(errp, "%s", reason);
+        return -1;
+    }
+
+    certs = gnutls_certificate_get_peers(session->handle, &nCerts);
+    if (!certs) {
+        error_setg(errp, "No certificate peers");
+        return -1;
+    }
+
+    for (i = 0; i < nCerts; i++) {
+        ret = gnutls_x509_crt_init(&cert);
+        if (ret < 0) {
+            error_setg(errp, "Cannot initialize certificate: %s",
+                       gnutls_strerror(ret));
+            return -1;
+        }
+
+        ret = gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER);
+        if (ret < 0) {
+            error_setg(errp, "Cannot import certificate: %s",
+                       gnutls_strerror(ret));
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_expiration_time(cert) < now) {
+            error_setg(errp, "The certificate has expired");
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_activation_time(cert) > now) {
+            error_setg(errp, "The certificate is not yet activated");
+            goto error;
+        }
+
+        if (gnutls_x509_crt_get_activation_time(cert) > now) {
+            error_setg(errp, "The certificate is not yet activated");
+            goto error;
+        }
+
+        if (i == 0) {
+            size_t dnameSize = 1024;
+            session->peername = g_malloc(dnameSize);
+        requery:
+            ret = gnutls_x509_crt_get_dn(cert, session->peername, &dnameSize);
+            if (ret < 0) {
+                if (ret == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+                    session->peername = g_realloc(session->peername,
+                                                  dnameSize);
+                    goto requery;
+                }
+                error_setg(errp, "Cannot get client distinguished name: %s",
+                           gnutls_strerror(ret));
+                goto error;
+            }
+            if (session->aclname) {
+                qemu_acl *acl = qemu_acl_find(session->aclname);
+                int allow;
+                if (!acl) {
+                    error_setg(errp, "Cannot find ACL %s",
+                               session->aclname);
+                    goto error;
+                }
+
+                allow = qemu_acl_party_is_allowed(acl, session->peername);
+
+                if (!allow) {
+                    error_setg(errp, "TLS x509 ACL check for %s is denied",
+                               session->peername);
+                    goto error;
+                }
+            }
+            if (session->hostname) {
+                if (!gnutls_x509_crt_check_hostname(cert, session->hostname)) {
+                    error_setg(errp,
+                               "Certificate does not match the hostname %s",
+                               session->hostname);
+                    goto error;
+                }
+            }
+        }
+
+        gnutls_x509_crt_deinit(cert);
+    }
+
+    return 0;
+
+ error:
+    gnutls_x509_crt_deinit(cert);
+    return -1;
+}
+
+
+int
+qcrypto_tls_session_check_credentials(QCryptoTLSSession *session,
+                                      Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(session->creds),
+                            TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+        return 0;
+    } else if (object_dynamic_cast(OBJECT(session->creds),
+                            TYPE_QCRYPTO_TLS_CREDS_X509)) {
+        if (session->creds->verifyPeer) {
+            return qcrypto_tls_session_check_certificate(session,
+                                                         errp);
+        } else {
+            return 0;
+        }
+    } else {
+        error_setg(errp, "Unexpected credential type %s",
+                   object_get_typename(OBJECT(session->creds)));
+        return -1;
+    }
+}
+
+
+void
+qcrypto_tls_session_set_callbacks(QCryptoTLSSession *session,
+                                  QCryptoTLSSessionWriteFunc writeFunc,
+                                  QCryptoTLSSessionReadFunc readFunc,
+                                  void *opaque)
+{
+    session->writeFunc = writeFunc;
+    session->readFunc = readFunc;
+    session->opaque = opaque;
+}
+
+
+ssize_t
+qcrypto_tls_session_write(QCryptoTLSSession *session,
+                          const char *buf,
+                          size_t len)
+{
+    ssize_t ret = gnutls_record_send(session->handle, buf, len);
+
+    if (ret < 0) {
+        switch (ret) {
+        case GNUTLS_E_AGAIN:
+            errno = EAGAIN;
+            break;
+        case GNUTLS_E_INTERRUPTED:
+            errno = EINTR;
+            break;
+        default:
+            errno = EIO;
+            break;
+        }
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+ssize_t
+qcrypto_tls_session_read(QCryptoTLSSession *session,
+                         char *buf,
+                         size_t len)
+{
+    ssize_t ret = gnutls_record_recv(session->handle, buf, len);
+
+    if (ret < 0) {
+        switch (ret) {
+        case GNUTLS_E_AGAIN:
+            errno = EAGAIN;
+            break;
+        case GNUTLS_E_INTERRUPTED:
+            errno = EINTR;
+            break;
+        default:
+            errno = EIO;
+            break;
+        }
+        ret = -1;
+    }
+
+    return ret;
+}
+
+
+int
+qcrypto_tls_session_handshake(QCryptoTLSSession *session,
+                              Error **errp)
+{
+    int ret = gnutls_handshake(session->handle);
+    if (ret == 0) {
+        session->handshakeComplete = true;
+    } else {
+        if (ret == GNUTLS_E_INTERRUPTED ||
+            ret == GNUTLS_E_AGAIN) {
+            ret = 1;
+        } else {
+            error_setg(errp, "TLS handshake failed: %s",
+                       gnutls_strerror(ret));
+            ret = -1;
+        }
+    }
+
+    return ret;
+}
+
+
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *session)
+{
+    if (session->handshakeComplete) {
+        return QCRYPTO_TLS_HANDSHAKE_COMPLETE;
+    } else if (gnutls_record_get_direction(session->handle) == 0) {
+        return QCRYPTO_TLS_HANDSHAKE_RECVING;
+    } else {
+        return QCRYPTO_TLS_HANDSHAKE_SENDING;
+    }
+}
+
+
+int
+qcrypto_tls_session_get_key_size(QCryptoTLSSession *session,
+                                 Error **errp)
+{
+    gnutls_cipher_algorithm_t cipher;
+    int ssf;
+
+    cipher = gnutls_cipher_get(session->handle);
+    ssf = gnutls_cipher_get_key_size(cipher);
+    if (!ssf) {
+        error_setg(errp, "Cannot get TLS cipher key size");
+        return -1;
+    }
+    return ssf;
+}
+
+
+char *
+qcrypto_tls_session_get_peer_name(QCryptoTLSSession *session)
+{
+    if (session->peername) {
+        return g_strdup(session->peername);
+    }
+    return NULL;
+}
+
+
+#else /* ! CONFIG_GNUTLS */
+
+
+QCryptoTLSSession *
+qcrypto_tls_session_new(QCryptoTLSCreds *creds G_GNUC_UNUSED,
+                        const char *hostname G_GNUC_UNUSED,
+                        const char *aclname G_GNUC_UNUSED,
+                        QCryptoTLSCredsEndpoint endpoint G_GNUC_UNUSED,
+                        Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return NULL;
+}
+
+
+void
+qcrypto_tls_session_free(QCryptoTLSSession *sess G_GNUC_UNUSED)
+{
+}
+
+
+int
+qcrypto_tls_session_check_credentials(QCryptoTLSSession *sess G_GNUC_UNUSED,
+                                      Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+void
+qcrypto_tls_session_set_callbacks(
+    QCryptoTLSSession *sess G_GNUC_UNUSED,
+    QCryptoTLSSessionWriteFunc writeFunc G_GNUC_UNUSED,
+    QCryptoTLSSessionReadFunc readFunc G_GNUC_UNUSED,
+    void *opaque G_GNUC_UNUSED)
+{
+}
+
+
+ssize_t
+qcrypto_tls_session_write(QCryptoTLSSession *sess,
+                          const char *buf,
+                          size_t len)
+{
+    errno = -EIO;
+    return -1;
+}
+
+
+ssize_t
+qcrypto_tls_session_read(QCryptoTLSSession *sess,
+                         char *buf,
+                         size_t len)
+{
+    errno = -EIO;
+    return -1;
+}
+
+
+int
+qcrypto_tls_session_handshake(QCryptoTLSSession *sess,
+                              Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *sess)
+{
+    return QCRYPTO_TLS_HANDSHAKE_COMPLETE;
+}
+
+
+int
+qcrypto_tls_session_get_key_size(QCryptoTLSSession *sess,
+                                 Error **errp)
+{
+    error_setg(errp, "TLS requires GNUTLS support");
+    return -1;
+}
+
+
+char *
+qcrypto_tls_session_get_peer_name(QCryptoTLSSession *sess)
+{
+    return NULL;
+}
+
+#endif
diff --git a/crypto/trace-events b/crypto/trace-events
new file mode 100644
index 0000000..8181843
--- /dev/null
+++ b/crypto/trace-events
@@ -0,0 +1,19 @@
+# See docs/tracing.txt for syntax documentation.
+
+# crypto/tlscreds.c
+qcrypto_tls_creds_load_dh(void *creds, const char *filename) "TLS creds load DH creds=%p filename=%s"
+qcrypto_tls_creds_get_path(void *creds, const char *filename, const char *path) "TLS creds path creds=%p filename=%s path=%s"
+
+# crypto/tlscredsanon.c
+qcrypto_tls_creds_anon_load(void *creds, const char *dir) "TLS creds anon load creds=%p dir=%s"
+
+# crypto/tlscredsx509.c
+qcrypto_tls_creds_x509_load(void *creds, const char *dir) "TLS creds x509 load creds=%p dir=%s"
+qcrypto_tls_creds_x509_check_basic_constraints(void *creds, const char *file, int status) "TLS creds x509 check basic constraints creds=%p file=%s status=%d"
+qcrypto_tls_creds_x509_check_key_usage(void *creds, const char *file, int status, int usage, int critical) "TLS creds x509 check key usage creds=%p file=%s status=%d usage=%d critical=%d"
+qcrypto_tls_creds_x509_check_key_purpose(void *creds, const char *file, int status, const char *usage, int critical) "TLS creds x509 check key usage creds=%p file=%s status=%d usage=%s critical=%d"
+qcrypto_tls_creds_x509_load_cert(void *creds, int isServer, const char *file) "TLS creds x509 load cert creds=%p isServer=%d file=%s"
+qcrypto_tls_creds_x509_load_cert_list(void *creds, const char *file) "TLS creds x509 load cert list creds=%p file=%s"
+
+# crypto/tlssession.c
+qcrypto_tls_session_new(void *session, void *creds, const char *hostname, const char *aclname, int endpoint) "TLS session new session=%p creds=%p hostname=%s aclname=%s endpoint=%d"
diff --git a/crypto/xts.c b/crypto/xts.c
new file mode 100644
index 0000000..9521234
--- /dev/null
+++ b/crypto/xts.c
@@ -0,0 +1,230 @@
+/*
+ * QEMU Crypto XTS cipher mode
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This code is originally derived from public domain / WTFPL code in
+ * LibTomCrypt crytographic library http://libtom.org. The XTS code
+ * was donated by Elliptic Semiconductor Inc (www.ellipticsemi.com)
+ * to the LibTom Projects
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/xts.h"
+
+static void xts_mult_x(uint8_t *I)
+{
+    int x;
+    uint8_t t, tt;
+
+    for (x = t = 0; x < 16; x++) {
+        tt = I[x] >> 7;
+        I[x] = ((I[x] << 1) | t) & 0xFF;
+        t = tt;
+    }
+    if (tt) {
+        I[0] ^= 0x87;
+    }
+}
+
+
+/**
+ * xts_tweak_uncrypt:
+ * @param ctxt: the cipher context
+ * @param func: the cipher function
+ * @src: buffer providing the cipher text of XTS_BLOCK_SIZE bytes
+ * @dst: buffer to output the plain text of XTS_BLOCK_SIZE bytes
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ *
+ * Decrypt data with a tweak
+ */
+static void xts_tweak_decrypt(const void *ctx,
+                              xts_cipher_func *func,
+                              const uint8_t *src,
+                              uint8_t *dst,
+                              uint8_t *iv)
+{
+    unsigned long x;
+
+    /* tweak encrypt block i */
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = src[x] ^ iv[x];
+    }
+
+    func(ctx, XTS_BLOCK_SIZE, dst, dst);
+
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = dst[x] ^ iv[x];
+    }
+
+    /* LFSR the tweak */
+    xts_mult_x(iv);
+}
+
+
+void xts_decrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src)
+{
+    uint8_t PP[XTS_BLOCK_SIZE], CC[XTS_BLOCK_SIZE], T[XTS_BLOCK_SIZE];
+    unsigned long i, m, mo, lim;
+
+    /* get number of blocks */
+    m = length >> 4;
+    mo = length & 15;
+
+    /* must have at least one full block */
+    g_assert(m != 0);
+
+    if (mo == 0) {
+        lim = m;
+    } else {
+        lim = m - 1;
+    }
+
+    /* encrypt the iv */
+    encfunc(tweakctx, XTS_BLOCK_SIZE, T, iv);
+
+    for (i = 0; i < lim; i++) {
+        xts_tweak_decrypt(datactx, decfunc, src, dst, T);
+
+        src += XTS_BLOCK_SIZE;
+        dst += XTS_BLOCK_SIZE;
+    }
+
+    /* if length is not a multiple of XTS_BLOCK_SIZE then */
+    if (mo > 0) {
+        memcpy(CC, T, XTS_BLOCK_SIZE);
+        xts_mult_x(CC);
+
+        /* PP = tweak decrypt block m-1 */
+        xts_tweak_decrypt(datactx, decfunc, src, PP, CC);
+
+        /* Pm = first length % XTS_BLOCK_SIZE bytes of PP */
+        for (i = 0; i < mo; i++) {
+            CC[i] = src[XTS_BLOCK_SIZE + i];
+            dst[XTS_BLOCK_SIZE + i] = PP[i];
+        }
+        for (; i < XTS_BLOCK_SIZE; i++) {
+            CC[i] = PP[i];
+        }
+
+        /* Pm-1 = Tweak uncrypt CC */
+        xts_tweak_decrypt(datactx, decfunc, CC, dst, T);
+    }
+
+    /* Decrypt the iv back */
+    decfunc(tweakctx, XTS_BLOCK_SIZE, iv, T);
+}
+
+
+/**
+ * xts_tweak_crypt:
+ * @param ctxt: the cipher context
+ * @param func: the cipher function
+ * @src: buffer providing the plain text of XTS_BLOCK_SIZE bytes
+ * @dst: buffer to output the cipher text of XTS_BLOCK_SIZE bytes
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ *
+ * Encrypt data with a tweak
+ */
+static void xts_tweak_encrypt(const void *ctx,
+                              xts_cipher_func *func,
+                              const uint8_t *src,
+                              uint8_t *dst,
+                              uint8_t *iv)
+{
+    unsigned long x;
+
+    /* tweak encrypt block i */
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = src[x] ^ iv[x];
+    }
+
+    func(ctx, XTS_BLOCK_SIZE, dst, dst);
+
+    for (x = 0; x < XTS_BLOCK_SIZE; x++) {
+        dst[x] = dst[x] ^ iv[x];
+    }
+
+    /* LFSR the tweak */
+    xts_mult_x(iv);
+}
+
+
+void xts_encrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src)
+{
+    uint8_t PP[XTS_BLOCK_SIZE], CC[XTS_BLOCK_SIZE], T[XTS_BLOCK_SIZE];
+    unsigned long i, m, mo, lim;
+
+    /* get number of blocks */
+    m = length >> 4;
+    mo = length & 15;
+
+    /* must have at least one full block */
+    g_assert(m != 0);
+
+    if (mo == 0) {
+        lim = m;
+    } else {
+        lim = m - 1;
+    }
+
+    /* encrypt the iv */
+    encfunc(tweakctx, XTS_BLOCK_SIZE, T, iv);
+
+    for (i = 0; i < lim; i++) {
+        xts_tweak_encrypt(datactx, encfunc, src, dst, T);
+
+        dst += XTS_BLOCK_SIZE;
+        src += XTS_BLOCK_SIZE;
+    }
+
+    /* if length is not a multiple of XTS_BLOCK_SIZE then */
+    if (mo > 0) {
+        /* CC = tweak encrypt block m-1 */
+        xts_tweak_encrypt(datactx, encfunc, src, CC, T);
+
+        /* Cm = first length % XTS_BLOCK_SIZE bytes of CC */
+        for (i = 0; i < mo; i++) {
+            PP[i] = src[XTS_BLOCK_SIZE + i];
+            dst[XTS_BLOCK_SIZE + i] = CC[i];
+        }
+
+        for (; i < XTS_BLOCK_SIZE; i++) {
+            PP[i] = CC[i];
+        }
+
+        /* Cm-1 = Tweak encrypt PP */
+        xts_tweak_encrypt(datactx, encfunc, PP, dst, T);
+    }
+
+    /* Decrypt the iv back */
+    decfunc(tweakctx, XTS_BLOCK_SIZE, iv, T);
+}
diff --git a/default-configs/aarch64-linux-user.mak b/default-configs/aarch64-linux-user.mak
index 3df7de5..0a5b08a 100644
--- a/default-configs/aarch64-linux-user.mak
+++ b/default-configs/aarch64-linux-user.mak
@@ -1,3 +1 @@
 # Default configuration for aarch64-linux-user
-
-CONFIG_GDBSTUB_XML=y
diff --git a/default-configs/aarch64-softmmu.mak b/default-configs/aarch64-softmmu.mak
index 6d3b5c7..2449483 100644
--- a/default-configs/aarch64-softmmu.mak
+++ b/default-configs/aarch64-softmmu.mak
@@ -3,4 +3,7 @@
 # We support all the 32 bit boards so need all their config
 include arm-softmmu.mak
 
-# Currently no 64-bit specific config requirements
+CONFIG_AUX=y
+CONFIG_DDC=y
+CONFIG_DPCD=y
+CONFIG_XLNX_ZYNQMP=y
diff --git a/default-configs/alpha-softmmu.mak b/default-configs/alpha-softmmu.mak
index bc07600..7f6161e 100644
--- a/default-configs/alpha-softmmu.mak
+++ b/default-configs/alpha-softmmu.mak
@@ -5,8 +5,6 @@
 CONFIG_SERIAL=y
 CONFIG_I8254=y
 CONFIG_PCKBD=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
 CONFIG_VGA_CIRRUS=y
 CONFIG_IDE_CORE=y
 CONFIG_IDE_QDEV=y
diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak
index aefd297..7a19863 100644
--- a/default-configs/arm-softmmu.mak
+++ b/default-configs/arm-softmmu.mak
@@ -28,11 +28,15 @@
 CONFIG_LAN9118=y
 CONFIG_SMC91C111=y
 CONFIG_ALLWINNER_EMAC=y
+CONFIG_IMX_FEC=y
 CONFIG_DS1338=y
 CONFIG_PFLASH_CFI01=y
 CONFIG_PFLASH_CFI02=y
 CONFIG_MICRODRIVE=y
+CONFIG_USB=y
 CONFIG_USB_MUSB=y
+CONFIG_USB_EHCI_SYSBUS=y
+CONFIG_PLATFORM_BUS=y
 
 CONFIG_ARM11MPCORE=y
 CONFIG_A9MPCORE=y
@@ -62,6 +66,7 @@
 CONFIG_BITBANG_I2C=y
 CONFIG_FRAMEBUFFER=y
 CONFIG_XILINX_SPIPS=y
+CONFIG_ZYNQ_DEVCFG=y
 
 CONFIG_ARM11SCU=y
 CONFIG_A9SCU=y
@@ -75,14 +80,20 @@
 CONFIG_IMX=y
 CONFIG_MAINSTONE=y
 CONFIG_NSERIES=y
+CONFIG_RASPI=y
 CONFIG_REALVIEW=y
 CONFIG_ZAURUS=y
 CONFIG_ZYNQ=y
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
+CONFIG_STM32F2XX_TIMER=y
+CONFIG_STM32F2XX_USART=y
+CONFIG_STM32F2XX_SYSCFG=y
+CONFIG_STM32F205_SOC=y
 
 CONFIG_VERSATILE_PCI=y
 CONFIG_VERSATILE_I2C=y
 
+CONFIG_PCI_GENERIC=y
+
 CONFIG_SDHCI=y
 CONFIG_INTEGRATOR_DEBUG=y
 
@@ -90,4 +101,16 @@
 CONFIG_ALLWINNER_A10_PIC=y
 CONFIG_ALLWINNER_A10=y
 
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
+CONFIG_FSL_IMX6=y
+CONFIG_FSL_IMX31=y
+CONFIG_FSL_IMX25=y
+
+CONFIG_IMX_I2C=y
+
+CONFIG_XIO3130=y
+CONFIG_IOH3420=y
+CONFIG_I82801B11=y
+CONFIG_ACPI=y
+CONFIG_SMBIOS=y
+CONFIG_ASPEED_SOC=y
+CONFIG_GPIO_KEY=y
diff --git a/default-configs/i386-softmmu.mak b/default-configs/i386-softmmu.mak
index 5f77aac..b177e52 100644
--- a/default-configs/i386-softmmu.mak
+++ b/default-configs/i386-softmmu.mak
@@ -3,13 +3,17 @@
 include pci.mak
 include sound.mak
 include usb.mak
-CONFIG_VGA=y
 CONFIG_QXL=$(CONFIG_SPICE)
-CONFIG_VGA_PCI=y
 CONFIG_VGA_ISA=y
 CONFIG_VGA_CIRRUS=y
 CONFIG_VMWARE_VGA=y
+CONFIG_VIRTIO_VGA=y
 CONFIG_VMMOUSE=y
+CONFIG_IPMI=y
+CONFIG_IPMI_LOCAL=y
+CONFIG_IPMI_EXTERN=y
+CONFIG_ISA_IPMI_KCS=y
+CONFIG_ISA_IPMI_BT=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_I8254=y
@@ -17,6 +21,10 @@
 CONFIG_PCKBD=y
 CONFIG_FDC=y
 CONFIG_ACPI=y
+CONFIG_ACPI_X86=y
+CONFIG_ACPI_X86_ICH=y
+CONFIG_ACPI_MEMORY_HOTPLUG=y
+CONFIG_ACPI_CPU_HOTPLUG=y
 CONFIG_APM=y
 CONFIG_I8257=y
 CONFIG_IDE_ISA=y
@@ -28,7 +36,6 @@
 CONFIG_I8259=y
 CONFIG_PFLASH_CFI01=y
 CONFIG_TPM_TIS=$(CONFIG_TPM)
-CONFIG_PCI_HOTPLUG_OLD=y
 CONFIG_MC146818RTC=y
 CONFIG_PAM=y
 CONFIG_PCI_PIIX=y
@@ -42,11 +49,12 @@
 CONFIG_PCI_Q35=y
 CONFIG_APIC=y
 CONFIG_IOAPIC=y
-CONFIG_ICC_BUS=y
 CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
-
-# Android emulator-specific options
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
-CONFIG_FRAMEBUFFER=$(CONFIG_ANDROID)
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
+CONFIG_NVDIMM=y
+CONFIG_ACPI_NVDIMM=y
+CONFIG_XIO3130=y
+CONFIG_IOH3420=y
+CONFIG_I82801B11=y
+CONFIG_SMBIOS=y
+CONFIG_HYPERV_TESTDEV=$(CONFIG_KVM)
diff --git a/default-configs/lm32-softmmu.mak b/default-configs/lm32-softmmu.mak
index 7df58c8..4889348 100644
--- a/default-configs/lm32-softmmu.mak
+++ b/default-configs/lm32-softmmu.mak
@@ -2,7 +2,7 @@
 
 CONFIG_LM32=y
 CONFIG_MILKYMIST=y
-CONFIG_MILKYMIST_TMU2=$(CONFIG_GLX)
+CONFIG_MILKYMIST_TMU2=$(CONFIG_OPENGL)
 CONFIG_FRAMEBUFFER=y
 CONFIG_PTIMER=y
 CONFIG_PFLASH_CFI01=y
diff --git a/default-configs/microblazeel-softmmu.mak b/default-configs/microblazeel-softmmu.mak
index acf22c5..2fcf442 100644
--- a/default-configs/microblazeel-softmmu.mak
+++ b/default-configs/microblazeel-softmmu.mak
@@ -1,11 +1,3 @@
 # Default configuration for microblazeel-softmmu
 
-CONFIG_PTIMER=y
-CONFIG_PFLASH_CFI01=y
-CONFIG_SERIAL=y
-CONFIG_XILINX=y
-CONFIG_XILINX_AXI=y
-CONFIG_XILINX_SPI=y
-CONFIG_XILINX_ETHLITE=y
-CONFIG_SSI=y
-CONFIG_SSI_M25P80=y
+include microblaze-softmmu.mak
diff --git a/default-configs/mips-softmmu-common.mak b/default-configs/mips-softmmu-common.mak
new file mode 100644
index 0000000..0394514
--- /dev/null
+++ b/default-configs/mips-softmmu-common.mak
@@ -0,0 +1,34 @@
+# Common mips*-softmmu CONFIG defines
+
+include pci.mak
+include sound.mak
+include usb.mak
+CONFIG_ESP=y
+CONFIG_VGA_ISA=y
+CONFIG_VGA_ISA_MM=y
+CONFIG_VGA_CIRRUS=y
+CONFIG_VMWARE_VGA=y
+CONFIG_SERIAL=y
+CONFIG_PARALLEL=y
+CONFIG_I8254=y
+CONFIG_PCSPK=y
+CONFIG_PCKBD=y
+CONFIG_FDC=y
+CONFIG_ACPI=y
+CONFIG_ACPI_X86=y
+CONFIG_ACPI_MEMORY_HOTPLUG=y
+CONFIG_ACPI_CPU_HOTPLUG=y
+CONFIG_APM=y
+CONFIG_I8257=y
+CONFIG_PIIX4=y
+CONFIG_IDE_ISA=y
+CONFIG_IDE_PIIX=y
+CONFIG_NE2000_ISA=y
+CONFIG_MIPSNET=y
+CONFIG_PFLASH_CFI01=y
+CONFIG_I8259=y
+CONFIG_MC146818RTC=y
+CONFIG_ISA_TESTDEV=y
+CONFIG_EMPTY_SLOT=y
+CONFIG_MIPS_CPS=y
+CONFIG_MIPS_ITU=y
diff --git a/default-configs/mips-softmmu.mak b/default-configs/mips-softmmu.mak
index f39b8c6..9fede6e 100644
--- a/default-configs/mips-softmmu.mak
+++ b/default-configs/mips-softmmu.mak
@@ -1,39 +1,3 @@
 # Default configuration for mips-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
-CONFIG_NE2000_ISA=y
-CONFIG_RC4030=y
-CONFIG_DP8393X=y
-CONFIG_DS1225Y=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
-CONFIG_G364FB=y
-CONFIG_I8259=y
-CONFIG_JAZZ_LED=y
-CONFIG_MC146818RTC=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
-CONFIG_FRAMEBUFFER=$(CONFIG_ANDROID)
+include mips-softmmu-common.mak
diff --git a/default-configs/mips64-softmmu.mak b/default-configs/mips64-softmmu.mak
index 1dedab4..bad7496 100644
--- a/default-configs/mips64-softmmu.mak
+++ b/default-configs/mips64-softmmu.mak
@@ -1,39 +1,9 @@
 # Default configuration for mips64-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
-CONFIG_NE2000_ISA=y
+include mips-softmmu-common.mak
 CONFIG_RC4030=y
 CONFIG_DP8393X=y
 CONFIG_DS1225Y=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
+CONFIG_JAZZ=y
 CONFIG_G364FB=y
-CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
-CONFIG_MC146818RTC=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
-CONFIG_FRAMEBUFFER=$(CONFIG_ANDROID)
diff --git a/default-configs/mips64el-softmmu.mak b/default-configs/mips64el-softmmu.mak
index 2fe17e8..485e218 100644
--- a/default-configs/mips64el-softmmu.mak
+++ b/default-configs/mips64el-softmmu.mak
@@ -1,42 +1,12 @@
 # Default configuration for mips64el-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
+include mips-softmmu-common.mak
 CONFIG_IDE_VIA=y
-CONFIG_NE2000_ISA=y
 CONFIG_RC4030=y
 CONFIG_DP8393X=y
 CONFIG_DS1225Y=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
 CONFIG_FULONG=y
+CONFIG_JAZZ=y
 CONFIG_G364FB=y
-CONFIG_I8259=y
 CONFIG_JAZZ_LED=y
-CONFIG_MC146818RTC=y
 CONFIG_VT82C686=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
-CONFIG_FRAMEBUFFER=$(CONFIG_ANDROID)
diff --git a/default-configs/mipsel-softmmu.mak b/default-configs/mipsel-softmmu.mak
index fd2d1c7..a7f6059 100644
--- a/default-configs/mipsel-softmmu.mak
+++ b/default-configs/mipsel-softmmu.mak
@@ -1,39 +1,3 @@
 # Default configuration for mipsel-softmmu
 
-include pci.mak
-include sound.mak
-include usb.mak
-CONFIG_ESP=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
-CONFIG_VGA_ISA=y
-CONFIG_VGA_ISA_MM=y
-CONFIG_VGA_CIRRUS=y
-CONFIG_VMWARE_VGA=y
-CONFIG_SERIAL=y
-CONFIG_PARALLEL=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_PCKBD=y
-CONFIG_FDC=y
-CONFIG_ACPI=y
-CONFIG_APM=y
-CONFIG_I8257=y
-CONFIG_PIIX4=y
-CONFIG_IDE_ISA=y
-CONFIG_IDE_PIIX=y
-CONFIG_NE2000_ISA=y
-CONFIG_RC4030=y
-CONFIG_DP8393X=y
-CONFIG_DS1225Y=y
-CONFIG_MIPSNET=y
-CONFIG_PFLASH_CFI01=y
-CONFIG_G364FB=y
-CONFIG_I8259=y
-CONFIG_JAZZ_LED=y
-CONFIG_MC146818RTC=y
-CONFIG_ISA_TESTDEV=y
-CONFIG_EMPTY_SLOT=y
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
-CONFIG_FRAMEBUFFER=$(CONFIG_ANDROID)
+include mips-softmmu-common.mak
diff --git a/default-configs/pci.mak b/default-configs/pci.mak
index 91b1e92..2ef761c 100644
--- a/default-configs/pci.mak
+++ b/default-configs/pci.mak
@@ -15,8 +15,10 @@
 CONFIG_LSI_SCSI_PCI=y
 CONFIG_VMW_PVSCSI_SCSI_PCI=y
 CONFIG_MEGASAS_SCSI_PCI=y
+CONFIG_MPTSAS_SCSI_PCI=y
 CONFIG_RTL8139_PCI=y
 CONFIG_E1000_PCI=y
+CONFIG_E1000E_PCI=y
 CONFIG_VMXNET3_PCI=y
 CONFIG_IDE_CORE=y
 CONFIG_IDE_QDEV=y
@@ -30,3 +32,11 @@
 CONFIG_WDT_IB6300ESB=y
 CONFIG_PCI_TESTDEV=y
 CONFIG_NVME_PCI=y
+CONFIG_SD=y
+CONFIG_SDHCI=y
+CONFIG_EDU=y
+CONFIG_VGA=y
+CONFIG_VGA_PCI=y
+CONFIG_IVSHMEM=$(CONFIG_EVENTFD)
+CONFIG_ROCKER=y
+CONFIG_GOLDFISH=y
diff --git a/default-configs/ppc-softmmu.mak b/default-configs/ppc-softmmu.mak
index d725b23..4befde3 100644
--- a/default-configs/ppc-softmmu.mak
+++ b/default-configs/ppc-softmmu.mak
@@ -6,8 +6,6 @@
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_I8254=y
@@ -40,11 +38,11 @@
 CONFIG_I8259=y
 CONFIG_XILINX=y
 CONFIG_XILINX_ETHLITE=y
-CONFIG_OPENPIC=y
 CONFIG_PREP=y
 CONFIG_MAC=y
 CONFIG_E500=y
 CONFIG_OPENPIC_KVM=$(and $(CONFIG_E500),$(CONFIG_KVM))
+CONFIG_PLATFORM_BUS=y
 CONFIG_ETSEC=y
 CONFIG_LIBDECNUMBER=y
 # For PReP
diff --git a/default-configs/ppc64-softmmu.mak b/default-configs/ppc64-softmmu.mak
index bd30d69..c4be59f 100644
--- a/default-configs/ppc64-softmmu.mak
+++ b/default-configs/ppc64-softmmu.mak
@@ -3,11 +3,10 @@
 include pci.mak
 include sound.mak
 include usb.mak
+CONFIG_VIRTIO_VGA=y
 CONFIG_ISA_MMIO=y
 CONFIG_ESCC=y
 CONFIG_M48T59=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_I8254=y
@@ -40,23 +39,19 @@
 CONFIG_I8259=y
 CONFIG_XILINX=y
 CONFIG_XILINX_ETHLITE=y
-CONFIG_OPENPIC=y
 CONFIG_PSERIES=y
 CONFIG_PREP=y
 CONFIG_MAC=y
 CONFIG_E500=y
 CONFIG_OPENPIC_KVM=$(and $(CONFIG_E500),$(CONFIG_KVM))
+CONFIG_PLATFORM_BUS=y
 CONFIG_ETSEC=y
 CONFIG_LIBDECNUMBER=y
 # For pSeries
 CONFIG_XICS=$(CONFIG_PSERIES)
+CONFIG_XICS_SPAPR=$(CONFIG_PSERIES)
 CONFIG_XICS_KVM=$(and $(CONFIG_PSERIES),$(CONFIG_KVM))
 # For PReP
-CONFIG_I82378=y
-CONFIG_I8259=y
-CONFIG_I8254=y
-CONFIG_PCSPK=y
-CONFIG_I82374=y
-CONFIG_I8257=y
 CONFIG_MC146818RTC=y
 CONFIG_ISA_TESTDEV=y
+CONFIG_MEM_HOTPLUG=y
diff --git a/default-configs/ppcemb-softmmu.mak b/default-configs/ppcemb-softmmu.mak
index e032761..54acc4d 100644
--- a/default-configs/ppcemb-softmmu.mak
+++ b/default-configs/ppcemb-softmmu.mak
@@ -4,8 +4,6 @@
 include sound.mak
 include usb.mak
 CONFIG_M48T59=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
 CONFIG_I8257=y
 CONFIG_OPENPIC=y
@@ -15,5 +13,4 @@
 CONFIG_I8259=y
 CONFIG_XILINX=y
 CONFIG_XILINX_ETHLITE=y
-CONFIG_OPENPIC=y
 CONFIG_LIBDECNUMBER=y
diff --git a/default-configs/s390x-softmmu.mak b/default-configs/s390x-softmmu.mak
index 126d88d..36e15de 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -1,4 +1,7 @@
+CONFIG_PCI=y
+CONFIG_VIRTIO_PCI=y
 CONFIG_VIRTIO=y
 CONFIG_SCLPCONSOLE=y
 CONFIG_S390_FLIC=y
 CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
+CONFIG_WDT_DIAG288=y
diff --git a/default-configs/sparc64-softmmu.mak b/default-configs/sparc64-softmmu.mak
index 299c97b..123bb99 100644
--- a/default-configs/sparc64-softmmu.mak
+++ b/default-configs/sparc64-softmmu.mak
@@ -5,8 +5,6 @@
 CONFIG_ISA_MMIO=y
 CONFIG_M48T59=y
 CONFIG_PTIMER=y
-CONFIG_VGA=y
-CONFIG_VGA_PCI=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_PCKBD=y
diff --git a/default-configs/tilegx-linux-user.mak b/default-configs/tilegx-linux-user.mak
new file mode 100644
index 0000000..3e47493
--- /dev/null
+++ b/default-configs/tilegx-linux-user.mak
@@ -0,0 +1 @@
+# Default configuration for tilegx-linux-user
diff --git a/default-configs/usb.mak b/default-configs/usb.mak
index 73d8489..f4b8568 100644
--- a/default-configs/usb.mak
+++ b/default-configs/usb.mak
@@ -1,3 +1,4 @@
+CONFIG_USB=y
 CONFIG_USB_TABLET_WACOM=y
 CONFIG_USB_STORAGE_BOT=y
 CONFIG_USB_STORAGE_UAS=y
diff --git a/default-configs/x86_64-softmmu.mak b/default-configs/x86_64-softmmu.mak
index 846aab6..6e3b312 100644
--- a/default-configs/x86_64-softmmu.mak
+++ b/default-configs/x86_64-softmmu.mak
@@ -3,13 +3,17 @@
 include pci.mak
 include sound.mak
 include usb.mak
-CONFIG_VGA=y
 CONFIG_QXL=$(CONFIG_SPICE)
-CONFIG_VGA_PCI=y
 CONFIG_VGA_ISA=y
 CONFIG_VGA_CIRRUS=y
 CONFIG_VMWARE_VGA=y
+CONFIG_VIRTIO_VGA=y
 CONFIG_VMMOUSE=y
+CONFIG_IPMI=y
+CONFIG_IPMI_LOCAL=y
+CONFIG_IPMI_EXTERN=y
+CONFIG_ISA_IPMI_KCS=y
+CONFIG_ISA_IPMI_BT=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_I8254=y
@@ -17,6 +21,10 @@
 CONFIG_PCKBD=y
 CONFIG_FDC=y
 CONFIG_ACPI=y
+CONFIG_ACPI_X86=y
+CONFIG_ACPI_X86_ICH=y
+CONFIG_ACPI_MEMORY_HOTPLUG=y
+CONFIG_ACPI_CPU_HOTPLUG=y
 CONFIG_APM=y
 CONFIG_I8257=y
 CONFIG_IDE_ISA=y
@@ -28,7 +36,6 @@
 CONFIG_I8259=y
 CONFIG_PFLASH_CFI01=y
 CONFIG_TPM_TIS=$(CONFIG_TPM)
-CONFIG_PCI_HOTPLUG_OLD=y
 CONFIG_MC146818RTC=y
 CONFIG_PAM=y
 CONFIG_PCI_PIIX=y
@@ -42,11 +49,12 @@
 CONFIG_PCI_Q35=y
 CONFIG_APIC=y
 CONFIG_IOAPIC=y
-CONFIG_ICC_BUS=y
 CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
-
-# Android emulator-specific options
-CONFIG_GOLDFISH=$(CONFIG_ANDROID)
-CONFIG_FRAMEBUFFER=$(CONFIG_ANDROID)
-CONFIG_ANDROID_PIPE=$(CONFIG_ANDROID)
+CONFIG_NVDIMM=y
+CONFIG_ACPI_NVDIMM=y
+CONFIG_XIO3130=y
+CONFIG_IOH3420=y
+CONFIG_I82801B11=y
+CONFIG_SMBIOS=y
+CONFIG_HYPERV_TESTDEV=$(CONFIG_KVM)
diff --git a/device-hotplug.c b/device-hotplug.c
index 9e38cc4..126f73c 100644
--- a/device-hotplug.c
+++ b/device-hotplug.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "sysemu/block-backend.h"
@@ -29,8 +30,9 @@
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
+#include "block/block_int.h"
 
-DriveInfo *add_init_drive(const char *optstr)
+static DriveInfo *add_init_drive(const char *optstr)
 {
     DriveInfo *dinfo;
     QemuOpts *opts;
@@ -50,10 +52,16 @@
     return dinfo;
 }
 
-void drive_hot_add(Monitor *mon, const QDict *qdict)
+void hmp_drive_add(Monitor *mon, const QDict *qdict)
 {
     DriveInfo *dinfo = NULL;
     const char *opts = qdict_get_str(qdict, "opts");
+    bool node = qdict_get_try_bool(qdict, "node", false);
+
+    if (node) {
+        hmp_drive_add_node(mon, opts);
+        return;
+    }
 
     dinfo = add_init_drive(opts);
     if (!dinfo) {
@@ -69,14 +77,15 @@
         monitor_printf(mon, "OK\n");
         break;
     default:
-        if (pci_drive_hot_add(mon, qdict, dinfo)) {
-            goto err;
-        }
+        monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
+        goto err;
     }
     return;
 
 err:
     if (dinfo) {
-        blk_unref(blk_by_legacy_dinfo(dinfo));
+        BlockBackend *blk = blk_by_legacy_dinfo(dinfo);
+        monitor_remove_blk(blk);
+        blk_unref(blk);
     }
 }
diff --git a/device_tree.c b/device_tree.c
index df9eed9..aafc410 100644
--- a/device_tree.c
+++ b/device_tree.c
@@ -11,20 +11,20 @@
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
+#ifdef CONFIG_LINUX
+#include <dirent.h>
+#endif
+
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
+#include "qemu/bswap.h"
 #include "sysemu/device_tree.h"
 #include "sysemu/sysemu.h"
 #include "hw/loader.h"
-#include "qemu/option.h"
+#include "hw/boards.h"
 #include "qemu/config-file.h"
 
 #include <libfdt.h>
@@ -118,6 +118,102 @@
     return NULL;
 }
 
+#ifdef CONFIG_LINUX
+
+#define SYSFS_DT_BASEDIR "/proc/device-tree"
+
+/**
+ * read_fstree: this function is inspired from dtc read_fstree
+ * @fdt: preallocated fdt blob buffer, to be populated
+ * @dirname: directory to scan under SYSFS_DT_BASEDIR
+ * the search is recursive and the tree is searched down to the
+ * leaves (property files).
+ *
+ * the function asserts in case of error
+ */
+static void read_fstree(void *fdt, const char *dirname)
+{
+    DIR *d;
+    struct dirent *de;
+    struct stat st;
+    const char *root_dir = SYSFS_DT_BASEDIR;
+    const char *parent_node;
+
+    if (strstr(dirname, root_dir) != dirname) {
+        error_setg(&error_fatal, "%s: %s must be searched within %s",
+                   __func__, dirname, root_dir);
+    }
+    parent_node = &dirname[strlen(SYSFS_DT_BASEDIR)];
+
+    d = opendir(dirname);
+    if (!d) {
+        error_setg(&error_fatal, "%s cannot open %s", __func__, dirname);
+    }
+
+    while ((de = readdir(d)) != NULL) {
+        char *tmpnam;
+
+        if (!g_strcmp0(de->d_name, ".")
+            || !g_strcmp0(de->d_name, "..")) {
+            continue;
+        }
+
+        tmpnam = g_strdup_printf("%s/%s", dirname, de->d_name);
+
+        if (qemu_lstat(tmpnam, &st) < 0) {
+            error_setg(&error_fatal, "%s cannot lstat %s", __func__, tmpnam);
+        }
+
+        if (S_ISREG(st.st_mode)) {
+            gchar *val;
+            gsize len;
+
+            if (!g_file_get_contents(tmpnam, &val, &len, NULL)) {
+                error_setg(&error_fatal, "%s not able to extract info from %s",
+                           __func__, tmpnam);
+            }
+
+            if (strlen(parent_node) > 0) {
+                qemu_fdt_setprop(fdt, parent_node,
+                                 de->d_name, val, len);
+            } else {
+                qemu_fdt_setprop(fdt, "/", de->d_name, val, len);
+            }
+            g_free(val);
+        } else if (S_ISDIR(st.st_mode)) {
+            char *node_name;
+
+            node_name = g_strdup_printf("%s/%s",
+                                        parent_node, de->d_name);
+            qemu_fdt_add_subnode(fdt, node_name);
+            g_free(node_name);
+            read_fstree(fdt, tmpnam);
+        }
+
+        g_free(tmpnam);
+    }
+
+    closedir(d);
+}
+
+/* load_device_tree_from_sysfs: extract the dt blob from host sysfs */
+void *load_device_tree_from_sysfs(void)
+{
+    void *host_fdt;
+    int host_fdt_size;
+
+    host_fdt = create_device_tree(&host_fdt_size);
+    read_fstree(host_fdt, SYSFS_DT_BASEDIR);
+    if (fdt_check_header(host_fdt)) {
+        error_setg(&error_fatal,
+                   "%s host device tree extracted into memory is invalid",
+                   __func__);
+    }
+    return host_fdt;
+}
+
+#endif /* CONFIG_LINUX */
+
 static int findnode_nofail(void *fdt, const char *node_path)
 {
     int offset;
@@ -132,6 +228,60 @@
     return offset;
 }
 
+char **qemu_fdt_node_path(void *fdt, const char *name, char *compat,
+                          Error **errp)
+{
+    int offset, len, ret;
+    const char *iter_name;
+    unsigned int path_len = 16, n = 0;
+    GSList *path_list = NULL, *iter;
+    char **path_array;
+
+    offset = fdt_node_offset_by_compatible(fdt, -1, compat);
+
+    while (offset >= 0) {
+        iter_name = fdt_get_name(fdt, offset, &len);
+        if (!iter_name) {
+            offset = len;
+            break;
+        }
+        if (!strcmp(iter_name, name)) {
+            char *path;
+
+            path = g_malloc(path_len);
+            while ((ret = fdt_get_path(fdt, offset, path, path_len))
+                  == -FDT_ERR_NOSPACE) {
+                path_len += 16;
+                path = g_realloc(path, path_len);
+            }
+            path_list = g_slist_prepend(path_list, path);
+            n++;
+        }
+        offset = fdt_node_offset_by_compatible(fdt, offset, compat);
+    }
+
+    if (offset < 0 && offset != -FDT_ERR_NOTFOUND) {
+        error_setg(errp, "%s: abort parsing dt for %s/%s: %s",
+                   __func__, name, compat, fdt_strerror(offset));
+        for (iter = path_list; iter; iter = iter->next) {
+            g_free(iter->data);
+        }
+        g_slist_free(path_list);
+        return NULL;
+    }
+
+    path_array = g_new(char *, n + 1);
+    path_array[n--] = NULL;
+
+    for (iter = path_list; iter; iter = iter->next) {
+        path_array[n--] = iter->data;
+    }
+
+    g_slist_free(path_list);
+
+    return path_array;
+}
+
 int qemu_fdt_setprop(void *fdt, const char *node_path,
                      const char *property, const void *val, int size)
 {
@@ -185,31 +335,39 @@
 }
 
 const void *qemu_fdt_getprop(void *fdt, const char *node_path,
-                             const char *property, int *lenp)
+                             const char *property, int *lenp, Error **errp)
 {
     int len;
     const void *r;
+
     if (!lenp) {
         lenp = &len;
     }
     r = fdt_getprop(fdt, findnode_nofail(fdt, node_path), property, lenp);
     if (!r) {
-        error_report("%s: Couldn't get %s/%s: %s", __func__,
-                     node_path, property, fdt_strerror(*lenp));
-        exit(1);
+        error_setg(errp, "%s: Couldn't get %s/%s: %s", __func__,
+                  node_path, property, fdt_strerror(*lenp));
     }
     return r;
 }
 
 uint32_t qemu_fdt_getprop_cell(void *fdt, const char *node_path,
-                               const char *property)
+                               const char *property, int *lenp, Error **errp)
 {
     int len;
-    const uint32_t *p = qemu_fdt_getprop(fdt, node_path, property, &len);
-    if (len != 4) {
-        error_report("%s: %s/%s not 4 bytes long (not a cell?)",
-                     __func__, node_path, property);
-        exit(1);
+    const uint32_t *p;
+
+    if (!lenp) {
+        lenp = &len;
+    }
+    p = qemu_fdt_getprop(fdt, node_path, property, lenp, errp);
+    if (!p) {
+        return 0;
+    } else if (*lenp != 4) {
+        error_setg(errp, "%s: %s/%s not 4 bytes long (not a cell?)",
+                   __func__, node_path, property);
+        *lenp = -EINVAL;
+        return 0;
     }
     return be32_to_cpu(*p);
 }
@@ -242,11 +400,10 @@
 
     /*
      * We need to find out if the user gave us special instruction at
-     * which phandle id to start allocting phandles.
+     * which phandle id to start allocating phandles.
      */
     if (!phandle) {
-        phandle = qemu_opt_get_number(qemu_get_machine_opts(),
-                                      "phandle_start", 0);
+        phandle = machine_phandle_start(current_machine);
     }
 
     if (!phandle) {
@@ -324,6 +481,7 @@
     uint64_t value;
     int cellnum, vnum, ncells;
     uint32_t hival;
+    int ret;
 
     propcells = g_new0(uint32_t, numvalues * 2);
 
@@ -331,18 +489,23 @@
     for (vnum = 0; vnum < numvalues; vnum++) {
         ncells = values[vnum * 2];
         if (ncells != 1 && ncells != 2) {
-            return -1;
+            ret = -1;
+            goto out;
         }
         value = values[vnum * 2 + 1];
         hival = cpu_to_be32(value >> 32);
         if (ncells > 1) {
             propcells[cellnum++] = hival;
         } else if (hival != 0) {
-            return -1;
+            ret = -1;
+            goto out;
         }
         propcells[cellnum++] = cpu_to_be32(value);
     }
 
-    return qemu_fdt_setprop(fdt, node_path, property, propcells,
-                            cellnum * sizeof(uint32_t));
+    ret = qemu_fdt_setprop(fdt, node_path, property, propcells,
+                           cellnum * sizeof(uint32_t));
+out:
+    g_free(propcells);
+    return ret;
 }
diff --git a/disas.c b/disas.c
index 44a019a..05a7a12 100644
--- a/disas.c
+++ b/disas.c
@@ -1,15 +1,15 @@
 /* General "disassemble this chunk" code.  Used for debugging. */
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "disas/bfd.h"
 #include "elf.h"
-#include <errno.h>
 
 #include "cpu.h"
 #include "disas/disas.h"
 
 typedef struct CPUDebug {
     struct disassemble_info info;
-    CPUArchState *env;
+    CPUState *cpu;
 } CPUDebug;
 
 /* Filled in by elfload.c.  Simplistic, but will do for now. */
@@ -39,7 +39,7 @@
 {
     CPUDebug *s = container_of(info, CPUDebug, info);
 
-    cpu_memory_rw_debug(ENV_GET_CPU(s->env), memaddr, myaddr, length, 0);
+    cpu_memory_rw_debug(s->cpu, memaddr, myaddr, length, 0);
     return 0;
 }
 
@@ -71,14 +71,6 @@
     (*info->fprintf_func) (info->stream, "0x%" PRIx64, addr);
 }
 
-/* Print address in hex, truncated to the width of a target virtual address. */
-static void
-generic_print_target_address(bfd_vma addr, struct disassemble_info *info)
-{
-    uint64_t mask = ~0ULL >> (64 - TARGET_VIRT_ADDR_SPACE_BITS);
-    generic_print_address(addr & mask, info);
-}
-
 /* Print address in hex, truncated to the width of a host virtual address. */
 static void
 generic_print_host_address(bfd_vma addr, struct disassemble_info *info)
@@ -150,14 +142,6 @@
   return (bfd_vma) v;
 }
 
-#ifdef TARGET_ARM
-static int
-print_insn_thumb1(bfd_vma pc, disassemble_info *info)
-{
-  return print_insn_arm(pc | 1, info);
-}
-#endif
-
 static int print_insn_objdump(bfd_vma pc, disassemble_info *info,
                               const char *prefix)
 {
@@ -190,32 +174,36 @@
 /* Disassemble this for me please... (debugging). 'flags' has the following
    values:
     i386 - 1 means 16 bit code, 2 means 64 bit code
-    arm  - bit 0 = thumb, bit 1 = reverse endian, bit 2 = A64
     ppc  - bits 0:15 specify (optionally) the machine instruction set;
            bit 16 indicates little endian.
     other targets - unused
  */
-void target_disas(FILE *out, CPUArchState *env, target_ulong code,
+void target_disas(FILE *out, CPUState *cpu, target_ulong code,
                   target_ulong size, int flags)
 {
+    CPUClass *cc = CPU_GET_CLASS(cpu);
     target_ulong pc;
     int count;
     CPUDebug s;
-    int (*print_insn)(bfd_vma pc, disassemble_info *info) = NULL;
 
     INIT_DISASSEMBLE_INFO(s.info, out, fprintf);
 
-    s.env = env;
+    s.cpu = cpu;
     s.info.read_memory_func = target_read_memory;
     s.info.buffer_vma = code;
     s.info.buffer_length = size;
-    s.info.print_address_func = generic_print_target_address;
+    s.info.print_address_func = generic_print_address;
 
 #ifdef TARGET_WORDS_BIGENDIAN
     s.info.endian = BFD_ENDIAN_BIG;
 #else
     s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
+
+    if (cc->disas_set_info) {
+        cc->disas_set_info(cpu, &s.info);
+    }
+
 #if defined(TARGET_I386)
     if (flags == 2) {
         s.info.mach = bfd_mach_x86_64;
@@ -224,33 +212,7 @@
     } else {
         s.info.mach = bfd_mach_i386_i386;
     }
-    print_insn = print_insn_i386;
-#elif defined(TARGET_ARM)
-    if (flags & 4) {
-        /* We might not be compiled with the A64 disassembler
-         * because it needs a C++ compiler; in that case we will
-         * fall through to the default print_insn_od case.
-         */
-#if defined(CONFIG_ARM_A64_DIS)
-        print_insn = print_insn_arm_a64;
-#endif
-    } else if (flags & 1) {
-        print_insn = print_insn_thumb1;
-    } else {
-        print_insn = print_insn_arm;
-    }
-    if (flags & 2) {
-#ifdef TARGET_WORDS_BIGENDIAN
-        s.info.endian = BFD_ENDIAN_LITTLE;
-#else
-        s.info.endian = BFD_ENDIAN_BIG;
-#endif
-    }
-#elif defined(TARGET_SPARC)
-    print_insn = print_insn_sparc;
-#ifdef TARGET_SPARC64
-    s.info.mach = bfd_mach_sparc_v9b;
-#endif
+    s.info.print_insn = print_insn_i386;
 #elif defined(TARGET_PPC)
     if ((flags >> 16) & 1) {
         s.info.endian = BFD_ENDIAN_LITTLE;
@@ -266,49 +228,15 @@
 #endif
     }
     s.info.disassembler_options = (char *)"any";
-    print_insn = print_insn_ppc;
-#elif defined(TARGET_M68K)
-    print_insn = print_insn_m68k;
-#elif defined(TARGET_MIPS)
-#ifdef TARGET_WORDS_BIGENDIAN
-    print_insn = print_insn_big_mips;
-#else
-    print_insn = print_insn_little_mips;
+    s.info.print_insn = print_insn_ppc;
 #endif
-#elif defined(TARGET_SH4)
-    s.info.mach = bfd_mach_sh4;
-    print_insn = print_insn_sh;
-#elif defined(TARGET_ALPHA)
-    s.info.mach = bfd_mach_alpha_ev6;
-    print_insn = print_insn_alpha;
-#elif defined(TARGET_CRIS)
-    if (flags != 32) {
-        s.info.mach = bfd_mach_cris_v0_v10;
-        print_insn = print_insn_crisv10;
-    } else {
-        s.info.mach = bfd_mach_cris_v32;
-        print_insn = print_insn_crisv32;
-    }
-#elif defined(TARGET_S390X)
-    s.info.mach = bfd_mach_s390_64;
-    print_insn = print_insn_s390;
-#elif defined(TARGET_MICROBLAZE)
-    s.info.mach = bfd_arch_microblaze;
-    print_insn = print_insn_microblaze;
-#elif defined(TARGET_MOXIE)
-    s.info.mach = bfd_arch_moxie;
-    print_insn = print_insn_moxie;
-#elif defined(TARGET_LM32)
-    s.info.mach = bfd_mach_lm32;
-    print_insn = print_insn_lm32;
-#endif
-    if (print_insn == NULL) {
-        print_insn = print_insn_od_target;
+    if (s.info.print_insn == NULL) {
+        s.info.print_insn = print_insn_od_target;
     }
 
     for (pc = code; size > 0; pc += count, size -= count) {
 	fprintf(out, "0x" TARGET_FMT_lx ":  ", pc);
-	count = print_insn(pc, &s.info);
+	count = s.info.print_insn(pc, &s.info);
 #if 0
         {
             int i;
@@ -430,36 +358,26 @@
     if (monitor_disas_is_physical) {
         cpu_physical_memory_read(memaddr, myaddr, length);
     } else {
-        cpu_memory_rw_debug(ENV_GET_CPU(s->env), memaddr, myaddr, length, 0);
+        cpu_memory_rw_debug(s->cpu, memaddr, myaddr, length, 0);
     }
     return 0;
 }
 
-static int GCC_FMT_ATTR(2, 3)
-monitor_fprintf(FILE *stream, const char *fmt, ...)
-{
-    va_list ap;
-    va_start(ap, fmt);
-    monitor_vprintf((Monitor *)stream, fmt, ap);
-    va_end(ap);
-    return 0;
-}
-
 /* Disassembler for the monitor.
    See target_disas for a description of flags. */
-void monitor_disas(Monitor *mon, CPUArchState *env,
+void monitor_disas(Monitor *mon, CPUState *cpu,
                    target_ulong pc, int nb_insn, int is_physical, int flags)
 {
+    CPUClass *cc = CPU_GET_CLASS(cpu);
     int count, i;
     CPUDebug s;
-    int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
     INIT_DISASSEMBLE_INFO(s.info, (FILE *)mon, monitor_fprintf);
 
-    s.env = env;
+    s.cpu = cpu;
     monitor_disas_is_physical = is_physical;
     s.info.read_memory_func = monitor_read_memory;
-    s.info.print_address_func = generic_print_target_address;
+    s.info.print_address_func = generic_print_address;
 
     s.info.buffer_vma = pc;
 
@@ -468,6 +386,11 @@
 #else
     s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
+
+    if (cc->disas_set_info) {
+        cc->disas_set_info(cpu, &s.info);
+    }
+
 #if defined(TARGET_I386)
     if (flags == 2) {
         s.info.mach = bfd_mach_x86_64;
@@ -476,16 +399,7 @@
     } else {
         s.info.mach = bfd_mach_i386_i386;
     }
-    print_insn = print_insn_i386;
-#elif defined(TARGET_ARM)
-    print_insn = print_insn_arm;
-#elif defined(TARGET_ALPHA)
-    print_insn = print_insn_alpha;
-#elif defined(TARGET_SPARC)
-    print_insn = print_insn_sparc;
-#ifdef TARGET_SPARC64
-    s.info.mach = bfd_mach_sparc_v9b;
-#endif
+    s.info.print_insn = print_insn_i386;
 #elif defined(TARGET_PPC)
     if (flags & 0xFFFF) {
         /* If we have a precise definition of the instruction set, use it. */
@@ -500,36 +414,17 @@
     if ((flags >> 16) & 1) {
         s.info.endian = BFD_ENDIAN_LITTLE;
     }
-    print_insn = print_insn_ppc;
-#elif defined(TARGET_M68K)
-    print_insn = print_insn_m68k;
-#elif defined(TARGET_MIPS)
-#ifdef TARGET_WORDS_BIGENDIAN
-    print_insn = print_insn_big_mips;
-#else
-    print_insn = print_insn_little_mips;
+    s.info.print_insn = print_insn_ppc;
 #endif
-#elif defined(TARGET_SH4)
-    s.info.mach = bfd_mach_sh4;
-    print_insn = print_insn_sh;
-#elif defined(TARGET_S390X)
-    s.info.mach = bfd_mach_s390_64;
-    print_insn = print_insn_s390;
-#elif defined(TARGET_MOXIE)
-    s.info.mach = bfd_arch_moxie;
-    print_insn = print_insn_moxie;
-#elif defined(TARGET_LM32)
-    s.info.mach = bfd_mach_lm32;
-    print_insn = print_insn_lm32;
-#else
-    monitor_printf(mon, "0x" TARGET_FMT_lx
-                   ": Asm output not supported on this arch\n", pc);
-    return;
-#endif
+    if (!s.info.print_insn) {
+        monitor_printf(mon, "0x" TARGET_FMT_lx
+                       ": Asm output not supported on this arch\n", pc);
+        return;
+    }
 
     for(i = 0; i < nb_insn; i++) {
 	monitor_printf(mon, "0x" TARGET_FMT_lx ":  ", pc);
-        count = print_insn(pc, &s.info);
+        count = s.info.print_insn(pc, &s.info);
 	monitor_printf(mon, "\n");
 	if (count < 0)
 	    break;
diff --git a/disas/Makefile.objs b/disas/Makefile.objs
index 8dae4da..abeba84 100644
--- a/disas/Makefile.objs
+++ b/disas/Makefile.objs
@@ -4,7 +4,10 @@
 common-obj-$(CONFIG_ARM_A64_DIS) += arm-a64.o
 common-obj-$(CONFIG_ARM_A64_DIS) += libvixl/
 libvixldir = $(SRC_PATH)/disas/libvixl
-arm-a64.o-cflags := -I$(libvixldir)
+# The -Wno-sign-compare is needed only for gcc 4.6, which complains about
+# some signed-unsigned equality comparisons in libvixl which later gcc
+# versions do not.
+arm-a64.o-cflags := -I$(libvixldir) -Wno-sign-compare
 common-obj-$(CONFIG_CRIS_DIS) += cris.o
 common-obj-$(CONFIG_HPPA_DIS) += hppa.o
 common-obj-$(CONFIG_I386_DIS) += i386.o
diff --git a/disas/alpha.c b/disas/alpha.c
index a950b9c..b7b0ae0 100644
--- a/disas/alpha.c
+++ b/disas/alpha.c
@@ -19,7 +19,7 @@
 along with this file; see the file COPYING.  If not, see
 <http://www.gnu.org/licenses/>. */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* MAX is redefined below, so remove any previous definition. */
@@ -521,7 +521,7 @@
 insert_bdisp(unsigned insn, int value, const char **errmsg)
 {
   if (errmsg != (const char **)NULL && (value & 3))
-    *errmsg = _("branch operand unaligned");
+    *errmsg = "branch operand unaligned";
   return insn | ((value / 4) & 0x1FFFFF);
 }
 
@@ -539,7 +539,7 @@
 insert_jhint(unsigned insn, int value, const char **errmsg)
 {
   if (errmsg != (const char **)NULL && (value & 3))
-    *errmsg = _("jump hint unaligned");
+    *errmsg = "jump hint unaligned";
   return insn | ((value / 4) & 0x3FFF);
 }
 
@@ -556,7 +556,7 @@
 insert_ev6hwjhint(unsigned insn, int value, const char **errmsg)
 {
   if (errmsg != (const char **)NULL && (value & 3))
-    *errmsg = _("jump hint unaligned");
+    *errmsg = "jump hint unaligned";
   return insn | ((value / 4) & 0x1FFF);
 }
 
diff --git a/disas/arm-a64.cc b/disas/arm-a64.cc
index ca29f6f..9280950 100644
--- a/disas/arm-a64.cc
+++ b/disas/arm-a64.cc
@@ -17,12 +17,13 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "a64/disasm-a64.h"
-
 extern "C" {
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 }
 
+#include "vixl/a64/disasm-a64.h"
+
 using namespace vixl;
 
 static Decoder *vixl_decoder = NULL;
@@ -35,16 +36,25 @@
  */
 class QEMUDisassembler : public Disassembler {
 public:
-    explicit QEMUDisassembler(FILE *stream) : stream_(stream) { }
+    QEMUDisassembler() : printf_(NULL), stream_(NULL) { }
     ~QEMUDisassembler() { }
 
+    void SetStream(FILE *stream) {
+        stream_ = stream;
+    }
+
+    void SetPrintf(fprintf_function printf_fn) {
+        printf_ = printf_fn;
+    }
+
 protected:
     virtual void ProcessOutput(const Instruction *instr) {
-        fprintf(stream_, "%08" PRIx32 "      %s",
+        printf_(stream_, "%08" PRIx32 "      %s",
                 instr->InstructionBits(), GetOutput());
     }
 
 private:
+    fprintf_function printf_;
     FILE *stream_;
 };
 
@@ -53,9 +63,9 @@
     return vixl_decoder != NULL;
 }
 
-static void vixl_init(FILE *f) {
+static void vixl_init() {
     vixl_decoder = new Decoder();
-    vixl_disasm = new QEMUDisassembler(f);
+    vixl_disasm = new QEMUDisassembler();
     vixl_decoder->AppendVisitor(vixl_disasm);
 }
 
@@ -67,7 +77,8 @@
 int print_insn_arm_a64(uint64_t addr, disassemble_info *info)
 {
     uint8_t bytes[INSN_SIZE];
-    uint32_t instr;
+    uint32_t instrval;
+    const Instruction *instr;
     int status;
 
     status = info->read_memory_func(addr, bytes, INSN_SIZE, info);
@@ -77,11 +88,16 @@
     }
 
     if (!vixl_is_initialized()) {
-        vixl_init(info->stream);
+        vixl_init();
     }
 
-    instr = bytes[0] | bytes[1] << 8 | bytes[2] << 16 | bytes[3] << 24;
-    vixl_decoder->Decode(reinterpret_cast<Instruction*>(&instr));
+    ((QEMUDisassembler *)vixl_disasm)->SetPrintf(info->fprintf_func);
+    ((QEMUDisassembler *)vixl_disasm)->SetStream(info->stream);
+
+    instrval = bytes[0] | bytes[1] << 8 | bytes[2] << 16 | bytes[3] << 24;
+    instr = reinterpret_cast<const Instruction *>(&instrval);
+    vixl_disasm->MapCodeAddress(addr, instr);
+    vixl_decoder->Decode(instr);
 
     return INSN_SIZE;
 }
diff --git a/disas/arm.c b/disas/arm.c
index 76e97a8..426270f 100644
--- a/disas/arm.c
+++ b/disas/arm.c
@@ -22,8 +22,8 @@
 /* Start of qemu specific additions.  Mostly this is stub definitions
    for things we don't care about.  */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
-#define ATTRIBUTE_UNUSED __attribute__((unused))
 #define ISSPACE(x) ((x) == ' ' || (x) == '\t' || (x) == '\n')
 
 #define ARM_EXT_V1	 0
@@ -1549,10 +1549,6 @@
   MAP_DATA
 };
 
-enum map_type last_type;
-int last_mapping_sym = -1;
-bfd_vma last_mapping_addr = 0;
-
 /* Decode a bitfield of the form matching regexp (N(-N)?,)*N(-N)?.
    Returns pointer to following character of the format string and
    fills in *VALUEP and *WIDTHP with the extracted value and number of
@@ -1783,7 +1779,7 @@
 
 			/* Is ``imm'' a negative number?  */
 			if (imm & 0x40)
-			  imm |= (-1 << 7);
+			  imm |= (~0u << 7);
 
 			func (stream, "%d", imm);
 		      }
@@ -1820,7 +1816,7 @@
 			  func (stream, "e");
 			  break;
 			default:
-			  func (stream, _("<illegal precision>"));
+			  func (stream, "<illegal precision>");
 			  break;
 			}
 		      break;
@@ -3878,135 +3874,11 @@
   int           is_data = false;
   unsigned int	size = 4;
   void	 	(*printer) (bfd_vma, struct disassemble_info *, long);
-#if 0
-  bfd_boolean   found = false;
-
-  if (info->disassembler_options)
-    {
-      parse_disassembler_options (info->disassembler_options);
-
-      /* To avoid repeated parsing of these options, we remove them here.  */
-      info->disassembler_options = NULL;
-    }
-
-  /* First check the full symtab for a mapping symbol, even if there
-     are no usable non-mapping symbols for this address.  */
-  if (info->symtab != NULL
-      && bfd_asymbol_flavour (*info->symtab) == bfd_target_elf_flavour)
-    {
-      bfd_vma addr;
-      int n;
-      int last_sym = -1;
-      enum map_type type = MAP_ARM;
-
-      if (pc <= last_mapping_addr)
-	last_mapping_sym = -1;
-      is_thumb = (last_type == MAP_THUMB);
-      found = false;
-      /* Start scanning at the start of the function, or wherever
-	 we finished last time.  */
-      n = info->symtab_pos + 1;
-      if (n < last_mapping_sym)
-	n = last_mapping_sym;
-
-      /* Scan up to the location being disassembled.  */
-      for (; n < info->symtab_size; n++)
-	{
-	  addr = bfd_asymbol_value (info->symtab[n]);
-	  if (addr > pc)
-	    break;
-	  if ((info->section == NULL
-	       || info->section == info->symtab[n]->section)
-	      && get_sym_code_type (info, n, &type))
-	    {
-	      last_sym = n;
-	      found = true;
-	    }
-	}
-
-      if (!found)
-	{
-	  n = info->symtab_pos;
-	  if (n < last_mapping_sym - 1)
-	    n = last_mapping_sym - 1;
-
-	  /* No mapping symbol found at this address.  Look backwards
-	     for a preceding one.  */
-	  for (; n >= 0; n--)
-	    {
-	      if (get_sym_code_type (info, n, &type))
-		{
-		  last_sym = n;
-		  found = true;
-		  break;
-		}
-	    }
-	}
-
-      last_mapping_sym = last_sym;
-      last_type = type;
-      is_thumb = (last_type == MAP_THUMB);
-      is_data = (last_type == MAP_DATA);
-
-      /* Look a little bit ahead to see if we should print out
-	 two or four bytes of data.  If there's a symbol,
-	 mapping or otherwise, after two bytes then don't
-	 print more.  */
-      if (is_data)
-	{
-	  size = 4 - (pc & 3);
-	  for (n = last_sym + 1; n < info->symtab_size; n++)
-	    {
-	      addr = bfd_asymbol_value (info->symtab[n]);
-	      if (addr > pc)
-		{
-		  if (addr - pc < size)
-		    size = addr - pc;
-		  break;
-		}
-	    }
-	  /* If the next symbol is after three bytes, we need to
-	     print only part of the data, so that we can use either
-	     .byte or .short.  */
-	  if (size == 3)
-	    size = (pc & 1) ? 1 : 2;
-	}
-    }
-
-  if (info->symbols != NULL)
-    {
-      if (bfd_asymbol_flavour (*info->symbols) == bfd_target_coff_flavour)
-	{
-	  coff_symbol_type * cs;
-
-	  cs = coffsymbol (*info->symbols);
-	  is_thumb = (   cs->native->u.syment.n_sclass == C_THUMBEXT
-		      || cs->native->u.syment.n_sclass == C_THUMBSTAT
-		      || cs->native->u.syment.n_sclass == C_THUMBLABEL
-		      || cs->native->u.syment.n_sclass == C_THUMBEXTFUNC
-		      || cs->native->u.syment.n_sclass == C_THUMBSTATFUNC);
-	}
-      else if (bfd_asymbol_flavour (*info->symbols) == bfd_target_elf_flavour
-	       && !found)
-	{
-	  /* If no mapping symbol has been found then fall back to the type
-	     of the function symbol.  */
-	  elf_symbol_type *  es;
-	  unsigned int       type;
-
-	  es = *(elf_symbol_type **)(info->symbols);
-	  type = ELF_ST_TYPE (es->internal_elf_sym.st_info);
-
-	  is_thumb = (type == STT_ARM_TFUNC) || (type == STT_ARM_16BIT);
-	}
-    }
-#else
   int little;
 
   little = (info->endian == BFD_ENDIAN_LITTLE);
   is_thumb |= (pc & 1);
   pc &= ~(bfd_vma)1;
-#endif
 
   if (force_thumb)
     is_thumb = true;
diff --git a/disas/cris.c b/disas/cris.c
index 9dfb4e3..7f35bc0 100644
--- a/disas/cris.c
+++ b/disas/cris.c
@@ -18,6 +18,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "disas/bfd.h"
 //#include "sysdep.h"
@@ -1210,21 +1211,10 @@
   "le",
   "a",
   /* This is a placeholder.  In v0, this would be "ext".  In v32, this
-     is "sb".  See cris_conds15.  */
+     is "sb". */
   "wf"
 };
 
-/* Different names and semantics for condition 1111 (0xf).  */
-const struct cris_cond15 cris_cond15s[] =
-{
-  /* FIXME: In what version did condition "ext" disappear?  */
-  {"ext", cris_ver_v0_3},
-  {"wf", cris_ver_v10},
-  {"sb", cris_ver_v32p},
-  {NULL, 0}
-};
-
-
 /*
  * Local variables:
  * eval: (c-set-style "gnu")
@@ -2503,7 +2493,7 @@
 	  = spec_reg_info ((insn >> 12) & 15, disdata->distype);
 
 	if (sregp->name == NULL)
-	  /* Should have been caught as a non-match eariler.  */
+	  /* Should have been caught as a non-match earlier.  */
 	  *tp++ = '?';
 	else
 	  {
@@ -2586,9 +2576,9 @@
      If we can't get any data, or we do not get enough data, we print
      the error message.  */
 
-  nbytes = info->buffer_length;
-  if (nbytes > MAX_BYTES_PER_CRIS_INSN)
-	  nbytes = MAX_BYTES_PER_CRIS_INSN;
+  nbytes = info->buffer_length ? info->buffer_length
+                               : MAX_BYTES_PER_CRIS_INSN;
+  nbytes = MIN(nbytes, MAX_BYTES_PER_CRIS_INSN);
   status = (*info->read_memory_func) (memaddr, buffer, nbytes, info);  
 
   /* If we did not get all we asked for, then clear the rest.
diff --git a/disas/hppa.c b/disas/hppa.c
index c7c8be6..43facdc 100644
--- a/disas/hppa.c
+++ b/disas/hppa.c
@@ -18,6 +18,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* HP PA-RISC SOM object file format:  definitions internal to BFD.
diff --git a/disas/i386.c b/disas/i386.c
index 00ceca9..57145d0 100644
--- a/disas/i386.c
+++ b/disas/i386.c
@@ -31,8 +31,10 @@
    and the small letter tells about the operand size.  Refer to
    the Intel manual for details.  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
+#include "qemu/cutils.h"
+
 /* include/opcode/i386.h r1.78 */
 
 /* opcode/i386.h -- Intel 80386 opcode macros
@@ -153,8 +155,6 @@
 /* opcodes/i386-dis.c r1.126 */
 #include "qemu-common.h"
 
-#include <setjmp.h>
-
 static int fetch_data2(struct disassemble_info *, bfd_byte *);
 static int fetch_data(struct disassemble_info *, bfd_byte *);
 static void ckprefix (void);
@@ -357,7 +357,7 @@
 #define Rd { OP_R, d_mode }
 #define Rm { OP_R, m_mode }
 #define Ib { OP_I, b_mode }
-#define sIb { OP_sI, b_mode }	/* sign extened byte */
+#define sIb { OP_sI, b_mode }	/* sign extended byte */
 #define Iv { OP_I, v_mode }
 #define Iq { OP_I, q_mode }
 #define Iv64 { OP_I64, v_mode }
@@ -3406,7 +3406,7 @@
   }
 };
 
-#define INTERNAL_DISASSEMBLER_ERROR _("<internal disassembler error>")
+#define INTERNAL_DISASSEMBLER_ERROR "<internal disassembler error>"
 
 static void
 ckprefix (void)
diff --git a/disas/ia64.c b/disas/ia64.c
index a8fe26c..140754c 100644
--- a/disas/ia64.c
+++ b/disas/ia64.c
@@ -18,8 +18,7 @@
    along with this file; see the file COPYING.  If not, see
    <http://www.gnu.org/licenses/>. */
 
-#include <assert.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "disas/bfd.h"
 
@@ -28,7 +27,6 @@
    Free Software Foundation, Inc.
    Contributed by David Mosberger-Tang <davidm@hpl.hp.com> */
 
-#include <sys/types.h>
 
 typedef uint64_t ia64_insn;
 
diff --git a/disas/libvixl/Makefile.objs b/disas/libvixl/Makefile.objs
index 17e6565..bbe7695 100644
--- a/disas/libvixl/Makefile.objs
+++ b/disas/libvixl/Makefile.objs
@@ -1,8 +1,11 @@
-libvixl_OBJS = utils.o \
-               a64/instructions-a64.o \
-               a64/decoder-a64.o \
-               a64/disasm-a64.o
+libvixl_OBJS = vixl/utils.o \
+               vixl/compiler-intrinsics.o \
+               vixl/a64/instructions-a64.o \
+               vixl/a64/decoder-a64.o \
+               vixl/a64/disasm-a64.o
 
-$(addprefix $(obj)/,$(libvixl_OBJS)): QEMU_CFLAGS := -I$(SRC_PATH)/disas/libvixl $(QEMU_CFLAGS)
+# The -Wno-sign-compare is needed only for gcc 4.6, which complains about
+# some signed-unsigned equality comparisons which later gcc versions do not.
+$(addprefix $(obj)/,$(libvixl_OBJS)): QEMU_CFLAGS := -I$(SRC_PATH)/disas/libvixl $(QEMU_CFLAGS) -Wno-sign-compare
 
 common-obj-$(CONFIG_ARM_A64_DIS) += $(libvixl_OBJS)
diff --git a/disas/libvixl/README b/disas/libvixl/README
index cba31b4..932a41a 100644
--- a/disas/libvixl/README
+++ b/disas/libvixl/README
@@ -2,11 +2,10 @@
 The code in this directory is a subset of libvixl:
  https://github.com/armvixl/vixl
 (specifically, it is the set of files needed for disassembly only,
-taken from libvixl 1.6).
+taken from libvixl 1.12).
 Bugfixes should preferably be sent upstream initially.
 
 The disassembler does not currently support the entire A64 instruction
 set. Notably:
- * No Advanced SIMD support.
  * Limited support for system instructions.
  * A few miscellaneous integer and floating point instructions are missing.
diff --git a/disas/libvixl/a64/assembler-a64.h b/disas/libvixl/a64/assembler-a64.h
deleted file mode 100644
index 16a704b..0000000
--- a/disas/libvixl/a64/assembler-a64.h
+++ /dev/null
@@ -1,2289 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_ASSEMBLER_A64_H_
-#define VIXL_A64_ASSEMBLER_A64_H_
-
-#include <list>
-#include <stack>
-
-#include "globals.h"
-#include "utils.h"
-#include "code-buffer.h"
-#include "a64/instructions-a64.h"
-
-namespace vixl {
-
-typedef uint64_t RegList;
-static const int kRegListSizeInBits = sizeof(RegList) * 8;
-
-
-// Registers.
-
-// Some CPURegister methods can return Register and FPRegister types, so we
-// need to declare them in advance.
-class Register;
-class FPRegister;
-
-
-class CPURegister {
- public:
-  enum RegisterType {
-    // The kInvalid value is used to detect uninitialized static instances,
-    // which are always zero-initialized before any constructors are called.
-    kInvalid = 0,
-    kRegister,
-    kFPRegister,
-    kNoRegister
-  };
-
-  CPURegister() : code_(0), size_(0), type_(kNoRegister) {
-    VIXL_ASSERT(!IsValid());
-    VIXL_ASSERT(IsNone());
-  }
-
-  CPURegister(unsigned code, unsigned size, RegisterType type)
-      : code_(code), size_(size), type_(type) {
-    VIXL_ASSERT(IsValidOrNone());
-  }
-
-  unsigned code() const {
-    VIXL_ASSERT(IsValid());
-    return code_;
-  }
-
-  RegisterType type() const {
-    VIXL_ASSERT(IsValidOrNone());
-    return type_;
-  }
-
-  RegList Bit() const {
-    VIXL_ASSERT(code_ < (sizeof(RegList) * 8));
-    return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
-  }
-
-  unsigned size() const {
-    VIXL_ASSERT(IsValid());
-    return size_;
-  }
-
-  int SizeInBytes() const {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(size() % 8 == 0);
-    return size_ / 8;
-  }
-
-  int SizeInBits() const {
-    VIXL_ASSERT(IsValid());
-    return size_;
-  }
-
-  bool Is32Bits() const {
-    VIXL_ASSERT(IsValid());
-    return size_ == 32;
-  }
-
-  bool Is64Bits() const {
-    VIXL_ASSERT(IsValid());
-    return size_ == 64;
-  }
-
-  bool IsValid() const {
-    if (IsValidRegister() || IsValidFPRegister()) {
-      VIXL_ASSERT(!IsNone());
-      return true;
-    } else {
-      VIXL_ASSERT(IsNone());
-      return false;
-    }
-  }
-
-  bool IsValidRegister() const {
-    return IsRegister() &&
-           ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
-           ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
-  }
-
-  bool IsValidFPRegister() const {
-    return IsFPRegister() &&
-           ((size_ == kSRegSize) || (size_ == kDRegSize)) &&
-           (code_ < kNumberOfFPRegisters);
-  }
-
-  bool IsNone() const {
-    // kNoRegister types should always have size 0 and code 0.
-    VIXL_ASSERT((type_ != kNoRegister) || (code_ == 0));
-    VIXL_ASSERT((type_ != kNoRegister) || (size_ == 0));
-
-    return type_ == kNoRegister;
-  }
-
-  bool Aliases(const CPURegister& other) const {
-    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
-    return (code_ == other.code_) && (type_ == other.type_);
-  }
-
-  bool Is(const CPURegister& other) const {
-    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
-    return Aliases(other) && (size_ == other.size_);
-  }
-
-  inline bool IsZero() const {
-    VIXL_ASSERT(IsValid());
-    return IsRegister() && (code_ == kZeroRegCode);
-  }
-
-  inline bool IsSP() const {
-    VIXL_ASSERT(IsValid());
-    return IsRegister() && (code_ == kSPRegInternalCode);
-  }
-
-  inline bool IsRegister() const {
-    return type_ == kRegister;
-  }
-
-  inline bool IsFPRegister() const {
-    return type_ == kFPRegister;
-  }
-
-  bool IsW() const { return IsValidRegister() && Is32Bits(); }
-  bool IsX() const { return IsValidRegister() && Is64Bits(); }
-  bool IsS() const { return IsValidFPRegister() && Is32Bits(); }
-  bool IsD() const { return IsValidFPRegister() && Is64Bits(); }
-
-  const Register& W() const;
-  const Register& X() const;
-  const FPRegister& S() const;
-  const FPRegister& D() const;
-
-  inline bool IsSameSizeAndType(const CPURegister& other) const {
-    return (size_ == other.size_) && (type_ == other.type_);
-  }
-
- protected:
-  unsigned code_;
-  unsigned size_;
-  RegisterType type_;
-
- private:
-  bool IsValidOrNone() const {
-    return IsValid() || IsNone();
-  }
-};
-
-
-class Register : public CPURegister {
- public:
-  Register() : CPURegister() {}
-  inline explicit Register(const CPURegister& other)
-      : CPURegister(other.code(), other.size(), other.type()) {
-    VIXL_ASSERT(IsValidRegister());
-  }
-  Register(unsigned code, unsigned size)
-      : CPURegister(code, size, kRegister) {}
-
-  bool IsValid() const {
-    VIXL_ASSERT(IsRegister() || IsNone());
-    return IsValidRegister();
-  }
-
-  static const Register& WRegFromCode(unsigned code);
-  static const Register& XRegFromCode(unsigned code);
-
-  // V8 compatibility.
-  static const int kNumRegisters = kNumberOfRegisters;
-  static const int kNumAllocatableRegisters = kNumberOfRegisters - 1;
-
- private:
-  static const Register wregisters[];
-  static const Register xregisters[];
-};
-
-
-class FPRegister : public CPURegister {
- public:
-  inline FPRegister() : CPURegister() {}
-  inline explicit FPRegister(const CPURegister& other)
-      : CPURegister(other.code(), other.size(), other.type()) {
-    VIXL_ASSERT(IsValidFPRegister());
-  }
-  inline FPRegister(unsigned code, unsigned size)
-      : CPURegister(code, size, kFPRegister) {}
-
-  bool IsValid() const {
-    VIXL_ASSERT(IsFPRegister() || IsNone());
-    return IsValidFPRegister();
-  }
-
-  static const FPRegister& SRegFromCode(unsigned code);
-  static const FPRegister& DRegFromCode(unsigned code);
-
-  // V8 compatibility.
-  static const int kNumRegisters = kNumberOfFPRegisters;
-  static const int kNumAllocatableRegisters = kNumberOfFPRegisters - 1;
-
- private:
-  static const FPRegister sregisters[];
-  static const FPRegister dregisters[];
-};
-
-
-// No*Reg is used to indicate an unused argument, or an error case. Note that
-// these all compare equal (using the Is() method). The Register and FPRegister
-// variants are provided for convenience.
-const Register NoReg;
-const FPRegister NoFPReg;
-const CPURegister NoCPUReg;
-
-
-#define DEFINE_REGISTERS(N)  \
-const Register w##N(N, kWRegSize);  \
-const Register x##N(N, kXRegSize);
-REGISTER_CODE_LIST(DEFINE_REGISTERS)
-#undef DEFINE_REGISTERS
-const Register wsp(kSPRegInternalCode, kWRegSize);
-const Register sp(kSPRegInternalCode, kXRegSize);
-
-
-#define DEFINE_FPREGISTERS(N)  \
-const FPRegister s##N(N, kSRegSize);  \
-const FPRegister d##N(N, kDRegSize);
-REGISTER_CODE_LIST(DEFINE_FPREGISTERS)
-#undef DEFINE_FPREGISTERS
-
-
-// Registers aliases.
-const Register ip0 = x16;
-const Register ip1 = x17;
-const Register lr = x30;
-const Register xzr = x31;
-const Register wzr = w31;
-
-
-// AreAliased returns true if any of the named registers overlap. Arguments
-// set to NoReg are ignored. The system stack pointer may be specified.
-bool AreAliased(const CPURegister& reg1,
-                const CPURegister& reg2,
-                const CPURegister& reg3 = NoReg,
-                const CPURegister& reg4 = NoReg,
-                const CPURegister& reg5 = NoReg,
-                const CPURegister& reg6 = NoReg,
-                const CPURegister& reg7 = NoReg,
-                const CPURegister& reg8 = NoReg);
-
-
-// AreSameSizeAndType returns true if all of the specified registers have the
-// same size, and are of the same type. The system stack pointer may be
-// specified. Arguments set to NoReg are ignored, as are any subsequent
-// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
-bool AreSameSizeAndType(const CPURegister& reg1,
-                        const CPURegister& reg2,
-                        const CPURegister& reg3 = NoCPUReg,
-                        const CPURegister& reg4 = NoCPUReg,
-                        const CPURegister& reg5 = NoCPUReg,
-                        const CPURegister& reg6 = NoCPUReg,
-                        const CPURegister& reg7 = NoCPUReg,
-                        const CPURegister& reg8 = NoCPUReg);
-
-
-// Lists of registers.
-class CPURegList {
- public:
-  inline explicit CPURegList(CPURegister reg1,
-                             CPURegister reg2 = NoCPUReg,
-                             CPURegister reg3 = NoCPUReg,
-                             CPURegister reg4 = NoCPUReg)
-      : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
-        size_(reg1.size()), type_(reg1.type()) {
-    VIXL_ASSERT(AreSameSizeAndType(reg1, reg2, reg3, reg4));
-    VIXL_ASSERT(IsValid());
-  }
-
-  inline CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
-      : list_(list), size_(size), type_(type) {
-    VIXL_ASSERT(IsValid());
-  }
-
-  inline CPURegList(CPURegister::RegisterType type, unsigned size,
-                    unsigned first_reg, unsigned last_reg)
-      : size_(size), type_(type) {
-    VIXL_ASSERT(((type == CPURegister::kRegister) &&
-                 (last_reg < kNumberOfRegisters)) ||
-                ((type == CPURegister::kFPRegister) &&
-                 (last_reg < kNumberOfFPRegisters)));
-    VIXL_ASSERT(last_reg >= first_reg);
-    list_ = (UINT64_C(1) << (last_reg + 1)) - 1;
-    list_ &= ~((UINT64_C(1) << first_reg) - 1);
-    VIXL_ASSERT(IsValid());
-  }
-
-  inline CPURegister::RegisterType type() const {
-    VIXL_ASSERT(IsValid());
-    return type_;
-  }
-
-  // Combine another CPURegList into this one. Registers that already exist in
-  // this list are left unchanged. The type and size of the registers in the
-  // 'other' list must match those in this list.
-  void Combine(const CPURegList& other) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
-    list_ |= other.list();
-  }
-
-  // Remove every register in the other CPURegList from this one. Registers that
-  // do not exist in this list are ignored. The type and size of the registers
-  // in the 'other' list must match those in this list.
-  void Remove(const CPURegList& other) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
-    list_ &= ~other.list();
-  }
-
-  // Variants of Combine and Remove which take a single register.
-  inline void Combine(const CPURegister& other) {
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.size() == size_);
-    Combine(other.code());
-  }
-
-  inline void Remove(const CPURegister& other) {
-    VIXL_ASSERT(other.type() == type_);
-    VIXL_ASSERT(other.size() == size_);
-    Remove(other.code());
-  }
-
-  // Variants of Combine and Remove which take a single register by its code;
-  // the type and size of the register is inferred from this list.
-  inline void Combine(int code) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
-    list_ |= (UINT64_C(1) << code);
-  }
-
-  inline void Remove(int code) {
-    VIXL_ASSERT(IsValid());
-    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
-    list_ &= ~(UINT64_C(1) << code);
-  }
-
-  inline RegList list() const {
-    VIXL_ASSERT(IsValid());
-    return list_;
-  }
-
-  inline void set_list(RegList new_list) {
-    VIXL_ASSERT(IsValid());
-    list_ = new_list;
-  }
-
-  // Remove all callee-saved registers from the list. This can be useful when
-  // preparing registers for an AAPCS64 function call, for example.
-  void RemoveCalleeSaved();
-
-  CPURegister PopLowestIndex();
-  CPURegister PopHighestIndex();
-
-  // AAPCS64 callee-saved registers.
-  static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
-  static CPURegList GetCalleeSavedFP(unsigned size = kDRegSize);
-
-  // AAPCS64 caller-saved registers. Note that this includes lr.
-  static CPURegList GetCallerSaved(unsigned size = kXRegSize);
-  static CPURegList GetCallerSavedFP(unsigned size = kDRegSize);
-
-  inline bool IsEmpty() const {
-    VIXL_ASSERT(IsValid());
-    return list_ == 0;
-  }
-
-  inline bool IncludesAliasOf(const CPURegister& other) const {
-    VIXL_ASSERT(IsValid());
-    return (type_ == other.type()) && ((other.Bit() & list_) != 0);
-  }
-
-  inline bool IncludesAliasOf(int code) const {
-    VIXL_ASSERT(IsValid());
-    return ((code & list_) != 0);
-  }
-
-  inline int Count() const {
-    VIXL_ASSERT(IsValid());
-    return CountSetBits(list_, kRegListSizeInBits);
-  }
-
-  inline unsigned RegisterSizeInBits() const {
-    VIXL_ASSERT(IsValid());
-    return size_;
-  }
-
-  inline unsigned RegisterSizeInBytes() const {
-    int size_in_bits = RegisterSizeInBits();
-    VIXL_ASSERT((size_in_bits % 8) == 0);
-    return size_in_bits / 8;
-  }
-
-  inline unsigned TotalSizeInBytes() const {
-    VIXL_ASSERT(IsValid());
-    return RegisterSizeInBytes() * Count();
-  }
-
- private:
-  RegList list_;
-  unsigned size_;
-  CPURegister::RegisterType type_;
-
-  bool IsValid() const;
-};
-
-
-// AAPCS64 callee-saved registers.
-extern const CPURegList kCalleeSaved;
-extern const CPURegList kCalleeSavedFP;
-
-
-// AAPCS64 caller-saved registers. Note that this includes lr.
-extern const CPURegList kCallerSaved;
-extern const CPURegList kCallerSavedFP;
-
-
-// Operand.
-class Operand {
- public:
-  // #<immediate>
-  // where <immediate> is int64_t.
-  // This is allowed to be an implicit constructor because Operand is
-  // a wrapper class that doesn't normally perform any type conversion.
-  Operand(int64_t immediate);           // NOLINT(runtime/explicit)
-
-  // rm, {<shift> #<shift_amount>}
-  // where <shift> is one of {LSL, LSR, ASR, ROR}.
-  //       <shift_amount> is uint6_t.
-  // This is allowed to be an implicit constructor because Operand is
-  // a wrapper class that doesn't normally perform any type conversion.
-  Operand(Register reg,
-          Shift shift = LSL,
-          unsigned shift_amount = 0);   // NOLINT(runtime/explicit)
-
-  // rm, {<extend> {#<shift_amount>}}
-  // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
-  //       <shift_amount> is uint2_t.
-  explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
-
-  bool IsImmediate() const;
-  bool IsShiftedRegister() const;
-  bool IsExtendedRegister() const;
-  bool IsZero() const;
-
-  // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
-  // which helps in the encoding of instructions that use the stack pointer.
-  Operand ToExtendedRegister() const;
-
-  int64_t immediate() const {
-    VIXL_ASSERT(IsImmediate());
-    return immediate_;
-  }
-
-  Register reg() const {
-    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
-    return reg_;
-  }
-
-  Shift shift() const {
-    VIXL_ASSERT(IsShiftedRegister());
-    return shift_;
-  }
-
-  Extend extend() const {
-    VIXL_ASSERT(IsExtendedRegister());
-    return extend_;
-  }
-
-  unsigned shift_amount() const {
-    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
-    return shift_amount_;
-  }
-
- private:
-  int64_t immediate_;
-  Register reg_;
-  Shift shift_;
-  Extend extend_;
-  unsigned shift_amount_;
-};
-
-
-// MemOperand represents the addressing mode of a load or store instruction.
-class MemOperand {
- public:
-  explicit MemOperand(Register base,
-                      int64_t offset = 0,
-                      AddrMode addrmode = Offset);
-  explicit MemOperand(Register base,
-                      Register regoffset,
-                      Shift shift = LSL,
-                      unsigned shift_amount = 0);
-  explicit MemOperand(Register base,
-                      Register regoffset,
-                      Extend extend,
-                      unsigned shift_amount = 0);
-  explicit MemOperand(Register base,
-                      const Operand& offset,
-                      AddrMode addrmode = Offset);
-
-  const Register& base() const { return base_; }
-  const Register& regoffset() const { return regoffset_; }
-  int64_t offset() const { return offset_; }
-  AddrMode addrmode() const { return addrmode_; }
-  Shift shift() const { return shift_; }
-  Extend extend() const { return extend_; }
-  unsigned shift_amount() const { return shift_amount_; }
-  bool IsImmediateOffset() const;
-  bool IsRegisterOffset() const;
-  bool IsPreIndex() const;
-  bool IsPostIndex() const;
-
- private:
-  Register base_;
-  Register regoffset_;
-  int64_t offset_;
-  AddrMode addrmode_;
-  Shift shift_;
-  Extend extend_;
-  unsigned shift_amount_;
-};
-
-
-class Label {
- public:
-  Label() : location_(kLocationUnbound) {}
-  ~Label() {
-    // If the label has been linked to, it needs to be bound to a target.
-    VIXL_ASSERT(!IsLinked() || IsBound());
-  }
-
-  inline bool IsBound() const { return location_ >= 0; }
-  inline bool IsLinked() const { return !links_.empty(); }
-
- private:
-  // The list of linked instructions is stored in a stack-like structure. We
-  // don't use std::stack directly because it's slow for the common case where
-  // only one or two instructions refer to a label, and labels themselves are
-  // short-lived. This class behaves like std::stack, but the first few links
-  // are preallocated (configured by kPreallocatedLinks).
-  //
-  // If more than N links are required, this falls back to std::stack.
-  class LinksStack {
-   public:
-    LinksStack() : size_(0), links_extended_(NULL) {}
-    ~LinksStack() {
-      delete links_extended_;
-    }
-
-    size_t size() const {
-      return size_;
-    }
-
-    bool empty() const {
-      return size_ == 0;
-    }
-
-    void push(ptrdiff_t value) {
-      if (size_ < kPreallocatedLinks) {
-        links_[size_] = value;
-      } else {
-        if (links_extended_ == NULL) {
-          links_extended_ = new std::stack<ptrdiff_t>();
-        }
-        VIXL_ASSERT(size_ == (links_extended_->size() + kPreallocatedLinks));
-        links_extended_->push(value);
-      }
-      size_++;
-    }
-
-    ptrdiff_t top() const {
-      return (size_ <= kPreallocatedLinks) ? links_[size_ - 1]
-                                           : links_extended_->top();
-    }
-
-    void pop() {
-      size_--;
-      if (size_ >= kPreallocatedLinks) {
-        links_extended_->pop();
-        VIXL_ASSERT(size_ == (links_extended_->size() + kPreallocatedLinks));
-      }
-    }
-
-   private:
-    static const size_t kPreallocatedLinks = 4;
-
-    size_t size_;
-    ptrdiff_t links_[kPreallocatedLinks];
-    std::stack<ptrdiff_t> * links_extended_;
-  };
-
-  inline ptrdiff_t location() const { return location_; }
-
-  inline void Bind(ptrdiff_t location) {
-    // Labels can only be bound once.
-    VIXL_ASSERT(!IsBound());
-    location_ = location;
-  }
-
-  inline void AddLink(ptrdiff_t instruction) {
-    // If a label is bound, the assembler already has the information it needs
-    // to write the instruction, so there is no need to add it to links_.
-    VIXL_ASSERT(!IsBound());
-    links_.push(instruction);
-  }
-
-  inline ptrdiff_t GetAndRemoveNextLink() {
-    VIXL_ASSERT(IsLinked());
-    ptrdiff_t link = links_.top();
-    links_.pop();
-    return link;
-  }
-
-  // The offsets of the instructions that have linked to this label.
-  LinksStack links_;
-  // The label location.
-  ptrdiff_t location_;
-
-  static const ptrdiff_t kLocationUnbound = -1;
-
-  // It is not safe to copy labels, so disable the copy constructor by declaring
-  // it private (without an implementation).
-  Label(const Label&);
-
-  // The Assembler class is responsible for binding and linking labels, since
-  // the stored offsets need to be consistent with the Assembler's buffer.
-  friend class Assembler;
-};
-
-
-// A literal is a 32-bit or 64-bit piece of data stored in the instruction
-// stream and loaded through a pc relative load. The same literal can be
-// referred to by multiple instructions but a literal can only reside at one
-// place in memory. A literal can be used by a load before or after being
-// placed in memory.
-//
-// Internally an offset of 0 is associated with a literal which has been
-// neither used nor placed. Then two possibilities arise:
-//  1) the label is placed, the offset (stored as offset + 1) is used to
-//     resolve any subsequent load using the label.
-//  2) the label is not placed and offset is the offset of the last load using
-//     the literal (stored as -offset -1). If multiple loads refer to this
-//     literal then the last load holds the offset of the preceding load and
-//     all loads form a chain. Once the offset is placed all the loads in the
-//     chain are resolved and future loads fall back to possibility 1.
-class RawLiteral {
- public:
-  RawLiteral() : size_(0), offset_(0), raw_value_(0) {}
-
-  size_t size() {
-    VIXL_STATIC_ASSERT(kDRegSizeInBytes == kXRegSizeInBytes);
-    VIXL_STATIC_ASSERT(kSRegSizeInBytes == kWRegSizeInBytes);
-    VIXL_ASSERT((size_ == kXRegSizeInBytes) || (size_ == kWRegSizeInBytes));
-    return size_;
-  }
-  uint64_t raw_value64() {
-    VIXL_ASSERT(size_ == kXRegSizeInBytes);
-    return raw_value_;
-  }
-  uint32_t raw_value32() {
-    VIXL_ASSERT(size_ == kWRegSizeInBytes);
-    VIXL_ASSERT(is_uint32(raw_value_) || is_int32(raw_value_));
-    return static_cast<uint32_t>(raw_value_);
-  }
-  bool IsUsed() { return offset_ < 0; }
-  bool IsPlaced() { return offset_ > 0; }
-
- protected:
-  ptrdiff_t offset() {
-    VIXL_ASSERT(IsPlaced());
-    return offset_ - 1;
-  }
-  void set_offset(ptrdiff_t offset) {
-    VIXL_ASSERT(offset >= 0);
-    VIXL_ASSERT(IsWordAligned(offset));
-    VIXL_ASSERT(!IsPlaced());
-    offset_ = offset + 1;
-  }
-  ptrdiff_t last_use() {
-    VIXL_ASSERT(IsUsed());
-    return -offset_ - 1;
-  }
-  void set_last_use(ptrdiff_t offset) {
-    VIXL_ASSERT(offset >= 0);
-    VIXL_ASSERT(IsWordAligned(offset));
-    VIXL_ASSERT(!IsPlaced());
-    offset_ = -offset - 1;
-  }
-
-  size_t size_;
-  ptrdiff_t offset_;
-  uint64_t raw_value_;
-
-  friend class Assembler;
-};
-
-
-template <typename T>
-class Literal : public RawLiteral {
- public:
-  explicit Literal(T value) {
-    size_ = sizeof(value);
-    memcpy(&raw_value_, &value, sizeof(value));
-  }
-};
-
-
-// Control whether or not position-independent code should be emitted.
-enum PositionIndependentCodeOption {
-  // All code generated will be position-independent; all branches and
-  // references to labels generated with the Label class will use PC-relative
-  // addressing.
-  PositionIndependentCode,
-
-  // Allow VIXL to generate code that refers to absolute addresses. With this
-  // option, it will not be possible to copy the code buffer and run it from a
-  // different address; code must be generated in its final location.
-  PositionDependentCode,
-
-  // Allow VIXL to assume that the bottom 12 bits of the address will be
-  // constant, but that the top 48 bits may change. This allows `adrp` to
-  // function in systems which copy code between pages, but otherwise maintain
-  // 4KB page alignment.
-  PageOffsetDependentCode
-};
-
-
-// Control how scaled- and unscaled-offset loads and stores are generated.
-enum LoadStoreScalingOption {
-  // Prefer scaled-immediate-offset instructions, but emit unscaled-offset,
-  // register-offset, pre-index or post-index instructions if necessary.
-  PreferScaledOffset,
-
-  // Prefer unscaled-immediate-offset instructions, but emit scaled-offset,
-  // register-offset, pre-index or post-index instructions if necessary.
-  PreferUnscaledOffset,
-
-  // Require scaled-immediate-offset instructions.
-  RequireScaledOffset,
-
-  // Require unscaled-immediate-offset instructions.
-  RequireUnscaledOffset
-};
-
-
-// Assembler.
-class Assembler {
- public:
-  Assembler(size_t capacity,
-            PositionIndependentCodeOption pic = PositionIndependentCode);
-  Assembler(byte* buffer, size_t capacity,
-            PositionIndependentCodeOption pic = PositionIndependentCode);
-
-  // The destructor asserts that one of the following is true:
-  //  * The Assembler object has not been used.
-  //  * Nothing has been emitted since the last Reset() call.
-  //  * Nothing has been emitted since the last FinalizeCode() call.
-  ~Assembler();
-
-  // System functions.
-
-  // Start generating code from the beginning of the buffer, discarding any code
-  // and data that has already been emitted into the buffer.
-  void Reset();
-
-  // Finalize a code buffer of generated instructions. This function must be
-  // called before executing or copying code from the buffer.
-  void FinalizeCode();
-
-  // Label.
-  // Bind a label to the current PC.
-  void bind(Label* label);
-
-  // Bind a label to a specified offset from the start of the buffer.
-  void BindToOffset(Label* label, ptrdiff_t offset);
-
-  // Place a literal at the current PC.
-  void place(RawLiteral* literal);
-
-  ptrdiff_t CursorOffset() const {
-    return buffer_->CursorOffset();
-  }
-
-  ptrdiff_t BufferEndOffset() const {
-    return static_cast<ptrdiff_t>(buffer_->capacity());
-  }
-
-  // Return the address of an offset in the buffer.
-  template <typename T>
-  inline T GetOffsetAddress(ptrdiff_t offset) {
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return buffer_->GetOffsetAddress<T>(offset);
-  }
-
-  // Return the address of a bound label.
-  template <typename T>
-  inline T GetLabelAddress(const Label * label) {
-    VIXL_ASSERT(label->IsBound());
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return GetOffsetAddress<T>(label->location());
-  }
-
-  // Return the address of the cursor.
-  template <typename T>
-  inline T GetCursorAddress() {
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return GetOffsetAddress<T>(CursorOffset());
-  }
-
-  // Return the address of the start of the buffer.
-  template <typename T>
-  inline T GetStartAddress() {
-    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
-    return GetOffsetAddress<T>(0);
-  }
-
-  // Instruction set functions.
-
-  // Branch / Jump instructions.
-  // Branch to register.
-  void br(const Register& xn);
-
-  // Branch with link to register.
-  void blr(const Register& xn);
-
-  // Branch to register with return hint.
-  void ret(const Register& xn = lr);
-
-  // Unconditional branch to label.
-  void b(Label* label);
-
-  // Conditional branch to label.
-  void b(Label* label, Condition cond);
-
-  // Unconditional branch to PC offset.
-  void b(int imm26);
-
-  // Conditional branch to PC offset.
-  void b(int imm19, Condition cond);
-
-  // Branch with link to label.
-  void bl(Label* label);
-
-  // Branch with link to PC offset.
-  void bl(int imm26);
-
-  // Compare and branch to label if zero.
-  void cbz(const Register& rt, Label* label);
-
-  // Compare and branch to PC offset if zero.
-  void cbz(const Register& rt, int imm19);
-
-  // Compare and branch to label if not zero.
-  void cbnz(const Register& rt, Label* label);
-
-  // Compare and branch to PC offset if not zero.
-  void cbnz(const Register& rt, int imm19);
-
-  // Test bit and branch to label if zero.
-  void tbz(const Register& rt, unsigned bit_pos, Label* label);
-
-  // Test bit and branch to PC offset if zero.
-  void tbz(const Register& rt, unsigned bit_pos, int imm14);
-
-  // Test bit and branch to label if not zero.
-  void tbnz(const Register& rt, unsigned bit_pos, Label* label);
-
-  // Test bit and branch to PC offset if not zero.
-  void tbnz(const Register& rt, unsigned bit_pos, int imm14);
-
-  // Address calculation instructions.
-  // Calculate a PC-relative address. Unlike for branches the offset in adr is
-  // unscaled (i.e. the result can be unaligned).
-
-  // Calculate the address of a label.
-  void adr(const Register& rd, Label* label);
-
-  // Calculate the address of a PC offset.
-  void adr(const Register& rd, int imm21);
-
-  // Calculate the page address of a label.
-  void adrp(const Register& rd, Label* label);
-
-  // Calculate the page address of a PC offset.
-  void adrp(const Register& rd, int imm21);
-
-  // Data Processing instructions.
-  // Add.
-  void add(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Add and update status flags.
-  void adds(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Compare negative.
-  void cmn(const Register& rn, const Operand& operand);
-
-  // Subtract.
-  void sub(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Subtract and update status flags.
-  void subs(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Compare.
-  void cmp(const Register& rn, const Operand& operand);
-
-  // Negate.
-  void neg(const Register& rd,
-           const Operand& operand);
-
-  // Negate and update status flags.
-  void negs(const Register& rd,
-            const Operand& operand);
-
-  // Add with carry bit.
-  void adc(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Add with carry bit and update status flags.
-  void adcs(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Subtract with carry bit.
-  void sbc(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Subtract with carry bit and update status flags.
-  void sbcs(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Negate with carry bit.
-  void ngc(const Register& rd,
-           const Operand& operand);
-
-  // Negate with carry bit and update status flags.
-  void ngcs(const Register& rd,
-            const Operand& operand);
-
-  // Logical instructions.
-  // Bitwise and (A & B).
-  void and_(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Bitwise and (A & B) and update status flags.
-  void ands(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Bit test and set flags.
-  void tst(const Register& rn, const Operand& operand);
-
-  // Bit clear (A & ~B).
-  void bic(const Register& rd,
-           const Register& rn,
-           const Operand& operand);
-
-  // Bit clear (A & ~B) and update status flags.
-  void bics(const Register& rd,
-            const Register& rn,
-            const Operand& operand);
-
-  // Bitwise or (A | B).
-  void orr(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Bitwise nor (A | ~B).
-  void orn(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Bitwise eor/xor (A ^ B).
-  void eor(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Bitwise enor/xnor (A ^ ~B).
-  void eon(const Register& rd, const Register& rn, const Operand& operand);
-
-  // Logical shift left by variable.
-  void lslv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Logical shift right by variable.
-  void lsrv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Arithmetic shift right by variable.
-  void asrv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Rotate right by variable.
-  void rorv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Bitfield instructions.
-  // Bitfield move.
-  void bfm(const Register& rd,
-           const Register& rn,
-           unsigned immr,
-           unsigned imms);
-
-  // Signed bitfield move.
-  void sbfm(const Register& rd,
-            const Register& rn,
-            unsigned immr,
-            unsigned imms);
-
-  // Unsigned bitfield move.
-  void ubfm(const Register& rd,
-            const Register& rn,
-            unsigned immr,
-            unsigned imms);
-
-  // Bfm aliases.
-  // Bitfield insert.
-  inline void bfi(const Register& rd,
-                  const Register& rn,
-                  unsigned lsb,
-                  unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
-  }
-
-  // Bitfield extract and insert low.
-  inline void bfxil(const Register& rd,
-                    const Register& rn,
-                    unsigned lsb,
-                    unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    bfm(rd, rn, lsb, lsb + width - 1);
-  }
-
-  // Sbfm aliases.
-  // Arithmetic shift right.
-  inline void asr(const Register& rd, const Register& rn, unsigned shift) {
-    VIXL_ASSERT(shift < rd.size());
-    sbfm(rd, rn, shift, rd.size() - 1);
-  }
-
-  // Signed bitfield insert with zero at right.
-  inline void sbfiz(const Register& rd,
-                    const Register& rn,
-                    unsigned lsb,
-                    unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
-  }
-
-  // Signed bitfield extract.
-  inline void sbfx(const Register& rd,
-                   const Register& rn,
-                   unsigned lsb,
-                   unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    sbfm(rd, rn, lsb, lsb + width - 1);
-  }
-
-  // Signed extend byte.
-  inline void sxtb(const Register& rd, const Register& rn) {
-    sbfm(rd, rn, 0, 7);
-  }
-
-  // Signed extend halfword.
-  inline void sxth(const Register& rd, const Register& rn) {
-    sbfm(rd, rn, 0, 15);
-  }
-
-  // Signed extend word.
-  inline void sxtw(const Register& rd, const Register& rn) {
-    sbfm(rd, rn, 0, 31);
-  }
-
-  // Ubfm aliases.
-  // Logical shift left.
-  inline void lsl(const Register& rd, const Register& rn, unsigned shift) {
-    unsigned reg_size = rd.size();
-    VIXL_ASSERT(shift < reg_size);
-    ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
-  }
-
-  // Logical shift right.
-  inline void lsr(const Register& rd, const Register& rn, unsigned shift) {
-    VIXL_ASSERT(shift < rd.size());
-    ubfm(rd, rn, shift, rd.size() - 1);
-  }
-
-  // Unsigned bitfield insert with zero at right.
-  inline void ubfiz(const Register& rd,
-                    const Register& rn,
-                    unsigned lsb,
-                    unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
-  }
-
-  // Unsigned bitfield extract.
-  inline void ubfx(const Register& rd,
-                   const Register& rn,
-                   unsigned lsb,
-                   unsigned width) {
-    VIXL_ASSERT(width >= 1);
-    VIXL_ASSERT(lsb + width <= rn.size());
-    ubfm(rd, rn, lsb, lsb + width - 1);
-  }
-
-  // Unsigned extend byte.
-  inline void uxtb(const Register& rd, const Register& rn) {
-    ubfm(rd, rn, 0, 7);
-  }
-
-  // Unsigned extend halfword.
-  inline void uxth(const Register& rd, const Register& rn) {
-    ubfm(rd, rn, 0, 15);
-  }
-
-  // Unsigned extend word.
-  inline void uxtw(const Register& rd, const Register& rn) {
-    ubfm(rd, rn, 0, 31);
-  }
-
-  // Extract.
-  void extr(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            unsigned lsb);
-
-  // Conditional select: rd = cond ? rn : rm.
-  void csel(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            Condition cond);
-
-  // Conditional select increment: rd = cond ? rn : rm + 1.
-  void csinc(const Register& rd,
-             const Register& rn,
-             const Register& rm,
-             Condition cond);
-
-  // Conditional select inversion: rd = cond ? rn : ~rm.
-  void csinv(const Register& rd,
-             const Register& rn,
-             const Register& rm,
-             Condition cond);
-
-  // Conditional select negation: rd = cond ? rn : -rm.
-  void csneg(const Register& rd,
-             const Register& rn,
-             const Register& rm,
-             Condition cond);
-
-  // Conditional set: rd = cond ? 1 : 0.
-  void cset(const Register& rd, Condition cond);
-
-  // Conditional set mask: rd = cond ? -1 : 0.
-  void csetm(const Register& rd, Condition cond);
-
-  // Conditional increment: rd = cond ? rn + 1 : rn.
-  void cinc(const Register& rd, const Register& rn, Condition cond);
-
-  // Conditional invert: rd = cond ? ~rn : rn.
-  void cinv(const Register& rd, const Register& rn, Condition cond);
-
-  // Conditional negate: rd = cond ? -rn : rn.
-  void cneg(const Register& rd, const Register& rn, Condition cond);
-
-  // Rotate right.
-  inline void ror(const Register& rd, const Register& rs, unsigned shift) {
-    extr(rd, rs, rs, shift);
-  }
-
-  // Conditional comparison.
-  // Conditional compare negative.
-  void ccmn(const Register& rn,
-            const Operand& operand,
-            StatusFlags nzcv,
-            Condition cond);
-
-  // Conditional compare.
-  void ccmp(const Register& rn,
-            const Operand& operand,
-            StatusFlags nzcv,
-            Condition cond);
-
-  // Multiply.
-  void mul(const Register& rd, const Register& rn, const Register& rm);
-
-  // Negated multiply.
-  void mneg(const Register& rd, const Register& rn, const Register& rm);
-
-  // Signed long multiply: 32 x 32 -> 64-bit.
-  void smull(const Register& rd, const Register& rn, const Register& rm);
-
-  // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
-  void smulh(const Register& xd, const Register& xn, const Register& xm);
-
-  // Multiply and accumulate.
-  void madd(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            const Register& ra);
-
-  // Multiply and subtract.
-  void msub(const Register& rd,
-            const Register& rn,
-            const Register& rm,
-            const Register& ra);
-
-  // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
-  void smaddl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
-  void umaddl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
-  void smsubl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
-  void umsubl(const Register& rd,
-              const Register& rn,
-              const Register& rm,
-              const Register& ra);
-
-  // Signed integer divide.
-  void sdiv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Unsigned integer divide.
-  void udiv(const Register& rd, const Register& rn, const Register& rm);
-
-  // Bit reverse.
-  void rbit(const Register& rd, const Register& rn);
-
-  // Reverse bytes in 16-bit half words.
-  void rev16(const Register& rd, const Register& rn);
-
-  // Reverse bytes in 32-bit words.
-  void rev32(const Register& rd, const Register& rn);
-
-  // Reverse bytes.
-  void rev(const Register& rd, const Register& rn);
-
-  // Count leading zeroes.
-  void clz(const Register& rd, const Register& rn);
-
-  // Count leading sign bits.
-  void cls(const Register& rd, const Register& rn);
-
-  // Memory instructions.
-  // Load integer or FP register.
-  void ldr(const CPURegister& rt, const MemOperand& src,
-           LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Store integer or FP register.
-  void str(const CPURegister& rt, const MemOperand& dst,
-           LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load word with sign extension.
-  void ldrsw(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load byte.
-  void ldrb(const Register& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Store byte.
-  void strb(const Register& rt, const MemOperand& dst,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load byte with sign extension.
-  void ldrsb(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load half-word.
-  void ldrh(const Register& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Store half-word.
-  void strh(const Register& rt, const MemOperand& dst,
-            LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load half-word with sign extension.
-  void ldrsh(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferScaledOffset);
-
-  // Load integer or FP register (with unscaled offset).
-  void ldur(const CPURegister& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Store integer or FP register (with unscaled offset).
-  void stur(const CPURegister& rt, const MemOperand& src,
-            LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load word with sign extension.
-  void ldursw(const Register& rt, const MemOperand& src,
-              LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load byte (with unscaled offset).
-  void ldurb(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Store byte (with unscaled offset).
-  void sturb(const Register& rt, const MemOperand& dst,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load byte with sign extension (and unscaled offset).
-  void ldursb(const Register& rt, const MemOperand& src,
-              LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load half-word (with unscaled offset).
-  void ldurh(const Register& rt, const MemOperand& src,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Store half-word (with unscaled offset).
-  void sturh(const Register& rt, const MemOperand& dst,
-             LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load half-word with sign extension (and unscaled offset).
-  void ldursh(const Register& rt, const MemOperand& src,
-              LoadStoreScalingOption option = PreferUnscaledOffset);
-
-  // Load integer or FP register pair.
-  void ldp(const CPURegister& rt, const CPURegister& rt2,
-           const MemOperand& src);
-
-  // Store integer or FP register pair.
-  void stp(const CPURegister& rt, const CPURegister& rt2,
-           const MemOperand& dst);
-
-  // Load word pair with sign extension.
-  void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
-
-  // Load integer or FP register pair, non-temporal.
-  void ldnp(const CPURegister& rt, const CPURegister& rt2,
-            const MemOperand& src);
-
-  // Store integer or FP register pair, non-temporal.
-  void stnp(const CPURegister& rt, const CPURegister& rt2,
-            const MemOperand& dst);
-
-  // Load integer or FP register from literal pool.
-  void ldr(const CPURegister& rt, RawLiteral* literal);
-
-  // Load word with sign extension from literal pool.
-  void ldrsw(const Register& rt, RawLiteral* literal);
-
-  // Load integer or FP register from pc + imm19 << 2.
-  void ldr(const CPURegister& rt, int imm19);
-
-  // Load word with sign extension from pc + imm19 << 2.
-  void ldrsw(const Register& rt, int imm19);
-
-  // Store exclusive byte.
-  void stxrb(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store exclusive half-word.
-  void stxrh(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store exclusive register.
-  void stxr(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Load exclusive byte.
-  void ldxrb(const Register& rt, const MemOperand& src);
-
-  // Load exclusive half-word.
-  void ldxrh(const Register& rt, const MemOperand& src);
-
-  // Load exclusive register.
-  void ldxr(const Register& rt, const MemOperand& src);
-
-  // Store exclusive register pair.
-  void stxp(const Register& rs,
-            const Register& rt,
-            const Register& rt2,
-            const MemOperand& dst);
-
-  // Load exclusive register pair.
-  void ldxp(const Register& rt, const Register& rt2, const MemOperand& src);
-
-  // Store-release exclusive byte.
-  void stlxrb(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store-release exclusive half-word.
-  void stlxrh(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Store-release exclusive register.
-  void stlxr(const Register& rs, const Register& rt, const MemOperand& dst);
-
-  // Load-acquire exclusive byte.
-  void ldaxrb(const Register& rt, const MemOperand& src);
-
-  // Load-acquire exclusive half-word.
-  void ldaxrh(const Register& rt, const MemOperand& src);
-
-  // Load-acquire exclusive register.
-  void ldaxr(const Register& rt, const MemOperand& src);
-
-  // Store-release exclusive register pair.
-  void stlxp(const Register& rs,
-             const Register& rt,
-             const Register& rt2,
-             const MemOperand& dst);
-
-  // Load-acquire exclusive register pair.
-  void ldaxp(const Register& rt, const Register& rt2, const MemOperand& src);
-
-  // Store-release byte.
-  void stlrb(const Register& rt, const MemOperand& dst);
-
-  // Store-release half-word.
-  void stlrh(const Register& rt, const MemOperand& dst);
-
-  // Store-release register.
-  void stlr(const Register& rt, const MemOperand& dst);
-
-  // Load-acquire byte.
-  void ldarb(const Register& rt, const MemOperand& src);
-
-  // Load-acquire half-word.
-  void ldarh(const Register& rt, const MemOperand& src);
-
-  // Load-acquire register.
-  void ldar(const Register& rt, const MemOperand& src);
-
-
-  // Move instructions. The default shift of -1 indicates that the move
-  // instruction will calculate an appropriate 16-bit immediate and left shift
-  // that is equal to the 64-bit immediate argument. If an explicit left shift
-  // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
-  //
-  // For movk, an explicit shift can be used to indicate which half word should
-  // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
-  // half word with zero, whereas movk(x0, 0, 48) will overwrite the
-  // most-significant.
-
-  // Move immediate and keep.
-  void movk(const Register& rd, uint64_t imm, int shift = -1) {
-    MoveWide(rd, imm, shift, MOVK);
-  }
-
-  // Move inverted immediate.
-  void movn(const Register& rd, uint64_t imm, int shift = -1) {
-    MoveWide(rd, imm, shift, MOVN);
-  }
-
-  // Move immediate.
-  void movz(const Register& rd, uint64_t imm, int shift = -1) {
-    MoveWide(rd, imm, shift, MOVZ);
-  }
-
-  // Misc instructions.
-  // Monitor debug-mode breakpoint.
-  void brk(int code);
-
-  // Halting debug-mode breakpoint.
-  void hlt(int code);
-
-  // Move register to register.
-  void mov(const Register& rd, const Register& rn);
-
-  // Move inverted operand to register.
-  void mvn(const Register& rd, const Operand& operand);
-
-  // System instructions.
-  // Move to register from system register.
-  void mrs(const Register& rt, SystemRegister sysreg);
-
-  // Move from register to system register.
-  void msr(SystemRegister sysreg, const Register& rt);
-
-  // System hint.
-  void hint(SystemHint code);
-
-  // Clear exclusive monitor.
-  void clrex(int imm4 = 0xf);
-
-  // Data memory barrier.
-  void dmb(BarrierDomain domain, BarrierType type);
-
-  // Data synchronization barrier.
-  void dsb(BarrierDomain domain, BarrierType type);
-
-  // Instruction synchronization barrier.
-  void isb();
-
-  // Alias for system instructions.
-  // No-op.
-  void nop() {
-    hint(NOP);
-  }
-
-  // FP instructions.
-  // Move double precision immediate to FP register.
-  void fmov(const FPRegister& fd, double imm);
-
-  // Move single precision immediate to FP register.
-  void fmov(const FPRegister& fd, float imm);
-
-  // Move FP register to register.
-  void fmov(const Register& rd, const FPRegister& fn);
-
-  // Move register to FP register.
-  void fmov(const FPRegister& fd, const Register& rn);
-
-  // Move FP register to FP register.
-  void fmov(const FPRegister& fd, const FPRegister& fn);
-
-  // FP add.
-  void fadd(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP subtract.
-  void fsub(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP multiply.
-  void fmul(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP fused multiply and add.
-  void fmadd(const FPRegister& fd,
-             const FPRegister& fn,
-             const FPRegister& fm,
-             const FPRegister& fa);
-
-  // FP fused multiply and subtract.
-  void fmsub(const FPRegister& fd,
-             const FPRegister& fn,
-             const FPRegister& fm,
-             const FPRegister& fa);
-
-  // FP fused multiply, add and negate.
-  void fnmadd(const FPRegister& fd,
-              const FPRegister& fn,
-              const FPRegister& fm,
-              const FPRegister& fa);
-
-  // FP fused multiply, subtract and negate.
-  void fnmsub(const FPRegister& fd,
-              const FPRegister& fn,
-              const FPRegister& fm,
-              const FPRegister& fa);
-
-  // FP divide.
-  void fdiv(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP maximum.
-  void fmax(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP minimum.
-  void fmin(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP maximum number.
-  void fmaxnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP minimum number.
-  void fminnm(const FPRegister& fd, const FPRegister& fn, const FPRegister& fm);
-
-  // FP absolute.
-  void fabs(const FPRegister& fd, const FPRegister& fn);
-
-  // FP negate.
-  void fneg(const FPRegister& fd, const FPRegister& fn);
-
-  // FP square root.
-  void fsqrt(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (nearest with ties to away).
-  void frinta(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (toward minus infinity).
-  void frintm(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (nearest with ties to even).
-  void frintn(const FPRegister& fd, const FPRegister& fn);
-
-  // FP round to integer (towards zero).
-  void frintz(const FPRegister& fd, const FPRegister& fn);
-
-  // FP compare registers.
-  void fcmp(const FPRegister& fn, const FPRegister& fm);
-
-  // FP compare immediate.
-  void fcmp(const FPRegister& fn, double value);
-
-  // FP conditional compare.
-  void fccmp(const FPRegister& fn,
-             const FPRegister& fm,
-             StatusFlags nzcv,
-             Condition cond);
-
-  // FP conditional select.
-  void fcsel(const FPRegister& fd,
-             const FPRegister& fn,
-             const FPRegister& fm,
-             Condition cond);
-
-  // Common FP Convert function.
-  void FPConvertToInt(const Register& rd,
-                      const FPRegister& fn,
-                      FPIntegerConvertOp op);
-
-  // FP convert between single and double precision.
-  void fcvt(const FPRegister& fd, const FPRegister& fn);
-
-  // Convert FP to signed integer (nearest with ties to away).
-  void fcvtas(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (nearest with ties to away).
-  void fcvtau(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to signed integer (round towards -infinity).
-  void fcvtms(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (round towards -infinity).
-  void fcvtmu(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to signed integer (nearest with ties to even).
-  void fcvtns(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (nearest with ties to even).
-  void fcvtnu(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to signed integer (round towards zero).
-  void fcvtzs(const Register& rd, const FPRegister& fn);
-
-  // Convert FP to unsigned integer (round towards zero).
-  void fcvtzu(const Register& rd, const FPRegister& fn);
-
-  // Convert signed integer or fixed point to FP.
-  void scvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
-
-  // Convert unsigned integer or fixed point to FP.
-  void ucvtf(const FPRegister& fd, const Register& rn, unsigned fbits = 0);
-
-  // Emit generic instructions.
-  // Emit raw instructions into the instruction stream.
-  inline void dci(Instr raw_inst) { Emit(raw_inst); }
-
-  // Emit 32 bits of data into the instruction stream.
-  inline void dc32(uint32_t data) {
-    VIXL_ASSERT(buffer_monitor_ > 0);
-    buffer_->Emit32(data);
-  }
-
-  // Emit 64 bits of data into the instruction stream.
-  inline void dc64(uint64_t data) {
-    VIXL_ASSERT(buffer_monitor_ > 0);
-    buffer_->Emit64(data);
-  }
-
-  // Copy a string into the instruction stream, including the terminating NULL
-  // character. The instruction pointer is then aligned correctly for
-  // subsequent instructions.
-  void EmitString(const char * string) {
-    VIXL_ASSERT(string != NULL);
-    VIXL_ASSERT(buffer_monitor_ > 0);
-
-    buffer_->EmitString(string);
-    buffer_->Align();
-  }
-
-  // Code generation helpers.
-
-  // Register encoding.
-  static Instr Rd(CPURegister rd) {
-    VIXL_ASSERT(rd.code() != kSPRegInternalCode);
-    return rd.code() << Rd_offset;
-  }
-
-  static Instr Rn(CPURegister rn) {
-    VIXL_ASSERT(rn.code() != kSPRegInternalCode);
-    return rn.code() << Rn_offset;
-  }
-
-  static Instr Rm(CPURegister rm) {
-    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
-    return rm.code() << Rm_offset;
-  }
-
-  static Instr Ra(CPURegister ra) {
-    VIXL_ASSERT(ra.code() != kSPRegInternalCode);
-    return ra.code() << Ra_offset;
-  }
-
-  static Instr Rt(CPURegister rt) {
-    VIXL_ASSERT(rt.code() != kSPRegInternalCode);
-    return rt.code() << Rt_offset;
-  }
-
-  static Instr Rt2(CPURegister rt2) {
-    VIXL_ASSERT(rt2.code() != kSPRegInternalCode);
-    return rt2.code() << Rt2_offset;
-  }
-
-  static Instr Rs(CPURegister rs) {
-    VIXL_ASSERT(rs.code() != kSPRegInternalCode);
-    return rs.code() << Rs_offset;
-  }
-
-  // These encoding functions allow the stack pointer to be encoded, and
-  // disallow the zero register.
-  static Instr RdSP(Register rd) {
-    VIXL_ASSERT(!rd.IsZero());
-    return (rd.code() & kRegCodeMask) << Rd_offset;
-  }
-
-  static Instr RnSP(Register rn) {
-    VIXL_ASSERT(!rn.IsZero());
-    return (rn.code() & kRegCodeMask) << Rn_offset;
-  }
-
-  // Flags encoding.
-  static Instr Flags(FlagsUpdate S) {
-    if (S == SetFlags) {
-      return 1 << FlagsUpdate_offset;
-    } else if (S == LeaveFlags) {
-      return 0 << FlagsUpdate_offset;
-    }
-    VIXL_UNREACHABLE();
-    return 0;
-  }
-
-  static Instr Cond(Condition cond) {
-    return cond << Condition_offset;
-  }
-
-  // PC-relative address encoding.
-  static Instr ImmPCRelAddress(int imm21) {
-    VIXL_ASSERT(is_int21(imm21));
-    Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
-    Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
-    Instr immlo = imm << ImmPCRelLo_offset;
-    return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
-  }
-
-  // Branch encoding.
-  static Instr ImmUncondBranch(int imm26) {
-    VIXL_ASSERT(is_int26(imm26));
-    return truncate_to_int26(imm26) << ImmUncondBranch_offset;
-  }
-
-  static Instr ImmCondBranch(int imm19) {
-    VIXL_ASSERT(is_int19(imm19));
-    return truncate_to_int19(imm19) << ImmCondBranch_offset;
-  }
-
-  static Instr ImmCmpBranch(int imm19) {
-    VIXL_ASSERT(is_int19(imm19));
-    return truncate_to_int19(imm19) << ImmCmpBranch_offset;
-  }
-
-  static Instr ImmTestBranch(int imm14) {
-    VIXL_ASSERT(is_int14(imm14));
-    return truncate_to_int14(imm14) << ImmTestBranch_offset;
-  }
-
-  static Instr ImmTestBranchBit(unsigned bit_pos) {
-    VIXL_ASSERT(is_uint6(bit_pos));
-    // Subtract five from the shift offset, as we need bit 5 from bit_pos.
-    unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
-    unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
-    b5 &= ImmTestBranchBit5_mask;
-    b40 &= ImmTestBranchBit40_mask;
-    return b5 | b40;
-  }
-
-  // Data Processing encoding.
-  static Instr SF(Register rd) {
-      return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
-  }
-
-  static Instr ImmAddSub(int64_t imm) {
-    VIXL_ASSERT(IsImmAddSub(imm));
-    if (is_uint12(imm)) {  // No shift required.
-      return imm << ImmAddSub_offset;
-    } else {
-      return ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
-    }
-  }
-
-  static inline Instr ImmS(unsigned imms, unsigned reg_size) {
-    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(imms)) ||
-           ((reg_size == kWRegSize) && is_uint5(imms)));
-    USE(reg_size);
-    return imms << ImmS_offset;
-  }
-
-  static inline Instr ImmR(unsigned immr, unsigned reg_size) {
-    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
-           ((reg_size == kWRegSize) && is_uint5(immr)));
-    USE(reg_size);
-    VIXL_ASSERT(is_uint6(immr));
-    return immr << ImmR_offset;
-  }
-
-  static inline Instr ImmSetBits(unsigned imms, unsigned reg_size) {
-    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-    VIXL_ASSERT(is_uint6(imms));
-    VIXL_ASSERT((reg_size == kXRegSize) || is_uint6(imms + 3));
-    USE(reg_size);
-    return imms << ImmSetBits_offset;
-  }
-
-  static inline Instr ImmRotate(unsigned immr, unsigned reg_size) {
-    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
-           ((reg_size == kWRegSize) && is_uint5(immr)));
-    USE(reg_size);
-    return immr << ImmRotate_offset;
-  }
-
-  static inline Instr ImmLLiteral(int imm19) {
-    VIXL_ASSERT(is_int19(imm19));
-    return truncate_to_int19(imm19) << ImmLLiteral_offset;
-  }
-
-  static inline Instr BitN(unsigned bitn, unsigned reg_size) {
-    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-    VIXL_ASSERT((reg_size == kXRegSize) || (bitn == 0));
-    USE(reg_size);
-    return bitn << BitN_offset;
-  }
-
-  static Instr ShiftDP(Shift shift) {
-    VIXL_ASSERT(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
-    return shift << ShiftDP_offset;
-  }
-
-  static Instr ImmDPShift(unsigned amount) {
-    VIXL_ASSERT(is_uint6(amount));
-    return amount << ImmDPShift_offset;
-  }
-
-  static Instr ExtendMode(Extend extend) {
-    return extend << ExtendMode_offset;
-  }
-
-  static Instr ImmExtendShift(unsigned left_shift) {
-    VIXL_ASSERT(left_shift <= 4);
-    return left_shift << ImmExtendShift_offset;
-  }
-
-  static Instr ImmCondCmp(unsigned imm) {
-    VIXL_ASSERT(is_uint5(imm));
-    return imm << ImmCondCmp_offset;
-  }
-
-  static Instr Nzcv(StatusFlags nzcv) {
-    return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
-  }
-
-  // MemOperand offset encoding.
-  static Instr ImmLSUnsigned(int imm12) {
-    VIXL_ASSERT(is_uint12(imm12));
-    return imm12 << ImmLSUnsigned_offset;
-  }
-
-  static Instr ImmLS(int imm9) {
-    VIXL_ASSERT(is_int9(imm9));
-    return truncate_to_int9(imm9) << ImmLS_offset;
-  }
-
-  static Instr ImmLSPair(int imm7, LSDataSize size) {
-    VIXL_ASSERT(((imm7 >> size) << size) == imm7);
-    int scaled_imm7 = imm7 >> size;
-    VIXL_ASSERT(is_int7(scaled_imm7));
-    return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
-  }
-
-  static Instr ImmShiftLS(unsigned shift_amount) {
-    VIXL_ASSERT(is_uint1(shift_amount));
-    return shift_amount << ImmShiftLS_offset;
-  }
-
-  static Instr ImmException(int imm16) {
-    VIXL_ASSERT(is_uint16(imm16));
-    return imm16 << ImmException_offset;
-  }
-
-  static Instr ImmSystemRegister(int imm15) {
-    VIXL_ASSERT(is_uint15(imm15));
-    return imm15 << ImmSystemRegister_offset;
-  }
-
-  static Instr ImmHint(int imm7) {
-    VIXL_ASSERT(is_uint7(imm7));
-    return imm7 << ImmHint_offset;
-  }
-
-  static Instr CRm(int imm4) {
-    VIXL_ASSERT(is_uint4(imm4));
-    return imm4 << CRm_offset;
-  }
-
-  static Instr ImmBarrierDomain(int imm2) {
-    VIXL_ASSERT(is_uint2(imm2));
-    return imm2 << ImmBarrierDomain_offset;
-  }
-
-  static Instr ImmBarrierType(int imm2) {
-    VIXL_ASSERT(is_uint2(imm2));
-    return imm2 << ImmBarrierType_offset;
-  }
-
-  static LSDataSize CalcLSDataSize(LoadStoreOp op) {
-    VIXL_ASSERT((SizeLS_offset + SizeLS_width) == (kInstructionSize * 8));
-    return static_cast<LSDataSize>(op >> SizeLS_offset);
-  }
-
-  // Move immediates encoding.
-  static Instr ImmMoveWide(uint64_t imm) {
-    VIXL_ASSERT(is_uint16(imm));
-    return imm << ImmMoveWide_offset;
-  }
-
-  static Instr ShiftMoveWide(int64_t shift) {
-    VIXL_ASSERT(is_uint2(shift));
-    return shift << ShiftMoveWide_offset;
-  }
-
-  // FP Immediates.
-  static Instr ImmFP32(float imm);
-  static Instr ImmFP64(double imm);
-
-  // FP register type.
-  static Instr FPType(FPRegister fd) {
-    return fd.Is64Bits() ? FP64 : FP32;
-  }
-
-  static Instr FPScale(unsigned scale) {
-    VIXL_ASSERT(is_uint6(scale));
-    return scale << FPScale_offset;
-  }
-
-  // Size of the code generated since label to the current position.
-  size_t SizeOfCodeGeneratedSince(Label* label) const {
-    VIXL_ASSERT(label->IsBound());
-    return buffer_->OffsetFrom(label->location());
-  }
-
-  size_t BufferCapacity() const { return buffer_->capacity(); }
-
-  size_t RemainingBufferSpace() const { return buffer_->RemainingBytes(); }
-
-  void EnsureSpaceFor(size_t amount) {
-    if (buffer_->RemainingBytes() < amount) {
-      size_t capacity = buffer_->capacity();
-      size_t size = buffer_->CursorOffset();
-      do {
-        // TODO(all): refine.
-        capacity *= 2;
-      } while ((capacity - size) <  amount);
-      buffer_->Grow(capacity);
-    }
-  }
-
-#ifdef DEBUG
-  void AcquireBuffer() {
-    VIXL_ASSERT(buffer_monitor_ >= 0);
-    buffer_monitor_++;
-  }
-
-  void ReleaseBuffer() {
-    buffer_monitor_--;
-    VIXL_ASSERT(buffer_monitor_ >= 0);
-  }
-#endif
-
-  inline PositionIndependentCodeOption pic() {
-    return pic_;
-  }
-
-  inline bool AllowPageOffsetDependentCode() {
-    return (pic() == PageOffsetDependentCode) ||
-           (pic() == PositionDependentCode);
-  }
-
-  static inline const Register& AppropriateZeroRegFor(const CPURegister& reg) {
-    return reg.Is64Bits() ? xzr : wzr;
-  }
-
-
- protected:
-  void LoadStore(const CPURegister& rt,
-                 const MemOperand& addr,
-                 LoadStoreOp op,
-                 LoadStoreScalingOption option = PreferScaledOffset);
-  static bool IsImmLSUnscaled(int64_t offset);
-  static bool IsImmLSScaled(int64_t offset, LSDataSize size);
-
-  void LoadStorePair(const CPURegister& rt,
-                     const CPURegister& rt2,
-                     const MemOperand& addr,
-                     LoadStorePairOp op);
-  static bool IsImmLSPair(int64_t offset, LSDataSize size);
-
-  // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
-  // reports a bogus uninitialised warning then.
-  void Logical(const Register& rd,
-               const Register& rn,
-               const Operand operand,
-               LogicalOp op);
-  void LogicalImmediate(const Register& rd,
-                        const Register& rn,
-                        unsigned n,
-                        unsigned imm_s,
-                        unsigned imm_r,
-                        LogicalOp op);
-  static bool IsImmLogical(uint64_t value,
-                           unsigned width,
-                           unsigned* n = NULL,
-                           unsigned* imm_s = NULL,
-                           unsigned* imm_r = NULL);
-
-  void ConditionalCompare(const Register& rn,
-                          const Operand& operand,
-                          StatusFlags nzcv,
-                          Condition cond,
-                          ConditionalCompareOp op);
-  static bool IsImmConditionalCompare(int64_t immediate);
-
-  void AddSubWithCarry(const Register& rd,
-                       const Register& rn,
-                       const Operand& operand,
-                       FlagsUpdate S,
-                       AddSubWithCarryOp op);
-
-  static bool IsImmFP32(float imm);
-  static bool IsImmFP64(double imm);
-
-  // Functions for emulating operands not directly supported by the instruction
-  // set.
-  void EmitShift(const Register& rd,
-                 const Register& rn,
-                 Shift shift,
-                 unsigned amount);
-  void EmitExtendShift(const Register& rd,
-                       const Register& rn,
-                       Extend extend,
-                       unsigned left_shift);
-
-  void AddSub(const Register& rd,
-              const Register& rn,
-              const Operand& operand,
-              FlagsUpdate S,
-              AddSubOp op);
-  static bool IsImmAddSub(int64_t immediate);
-
-  // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
-  // registers. Only simple loads are supported; sign- and zero-extension (such
-  // as in LDPSW_x or LDRB_w) are not supported.
-  static LoadStoreOp LoadOpFor(const CPURegister& rt);
-  static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
-                                       const CPURegister& rt2);
-  static LoadStoreOp StoreOpFor(const CPURegister& rt);
-  static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
-                                        const CPURegister& rt2);
-  static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
-    const CPURegister& rt, const CPURegister& rt2);
-  static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
-    const CPURegister& rt, const CPURegister& rt2);
-  static LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
-
-
- private:
-  // Instruction helpers.
-  void MoveWide(const Register& rd,
-                uint64_t imm,
-                int shift,
-                MoveWideImmediateOp mov_op);
-  void DataProcShiftedRegister(const Register& rd,
-                               const Register& rn,
-                               const Operand& operand,
-                               FlagsUpdate S,
-                               Instr op);
-  void DataProcExtendedRegister(const Register& rd,
-                                const Register& rn,
-                                const Operand& operand,
-                                FlagsUpdate S,
-                                Instr op);
-  void LoadStorePairNonTemporal(const CPURegister& rt,
-                                const CPURegister& rt2,
-                                const MemOperand& addr,
-                                LoadStorePairNonTemporalOp op);
-  void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
-  void ConditionalSelect(const Register& rd,
-                         const Register& rn,
-                         const Register& rm,
-                         Condition cond,
-                         ConditionalSelectOp op);
-  void DataProcessing1Source(const Register& rd,
-                             const Register& rn,
-                             DataProcessing1SourceOp op);
-  void DataProcessing3Source(const Register& rd,
-                             const Register& rn,
-                             const Register& rm,
-                             const Register& ra,
-                             DataProcessing3SourceOp op);
-  void FPDataProcessing1Source(const FPRegister& fd,
-                               const FPRegister& fn,
-                               FPDataProcessing1SourceOp op);
-  void FPDataProcessing2Source(const FPRegister& fd,
-                               const FPRegister& fn,
-                               const FPRegister& fm,
-                               FPDataProcessing2SourceOp op);
-  void FPDataProcessing3Source(const FPRegister& fd,
-                               const FPRegister& fn,
-                               const FPRegister& fm,
-                               const FPRegister& fa,
-                               FPDataProcessing3SourceOp op);
-
-  // Link the current (not-yet-emitted) instruction to the specified label, then
-  // return an offset to be encoded in the instruction. If the label is not yet
-  // bound, an offset of 0 is returned.
-  ptrdiff_t LinkAndGetByteOffsetTo(Label * label);
-  ptrdiff_t LinkAndGetInstructionOffsetTo(Label * label);
-  ptrdiff_t LinkAndGetPageOffsetTo(Label * label);
-
-  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
-  template <int element_shift>
-  ptrdiff_t LinkAndGetOffsetTo(Label* label);
-
-  // Literal load offset are in words (32-bit).
-  ptrdiff_t LinkAndGetWordOffsetTo(RawLiteral* literal);
-
-  // Emit the instruction in buffer_.
-  void Emit(Instr instruction) {
-    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
-    VIXL_ASSERT(buffer_monitor_ > 0);
-    buffer_->Emit32(instruction);
-  }
-
-  // Buffer where the code is emitted.
-  CodeBuffer* buffer_;
-  PositionIndependentCodeOption pic_;
-
-#ifdef DEBUG
-  int64_t buffer_monitor_;
-#endif
-};
-
-
-// All Assembler emits MUST acquire/release the underlying code buffer. The
-// helper scope below will do so and optionally ensure the buffer is big enough
-// to receive the emit. It is possible to request the scope not to perform any
-// checks (kNoCheck) if for example it is known in advance the buffer size is
-// adequate or there is some other size checking mechanism in place.
-class CodeBufferCheckScope {
- public:
-  // Tell whether or not the scope needs to ensure the associated CodeBuffer
-  // has enough space for the requested size.
-  enum CheckPolicy {
-    kNoCheck,
-    kCheck
-  };
-
-  // Tell whether or not the scope should assert the amount of code emitted
-  // within the scope is consistent with the requested amount.
-  enum AssertPolicy {
-    kNoAssert,    // No assert required.
-    kExactSize,   // The code emitted must be exactly size bytes.
-    kMaximumSize  // The code emitted must be at most size bytes.
-  };
-
-  CodeBufferCheckScope(Assembler* assm,
-                       size_t size,
-                       CheckPolicy check_policy = kCheck,
-                       AssertPolicy assert_policy = kMaximumSize)
-      : assm_(assm) {
-    if (check_policy == kCheck) assm->EnsureSpaceFor(size);
-#ifdef DEBUG
-    assm->bind(&start_);
-    size_ = size;
-    assert_policy_ = assert_policy;
-    assm->AcquireBuffer();
-#else
-    USE(assert_policy);
-#endif
-  }
-
-  // This is a shortcut for CodeBufferCheckScope(assm, 0, kNoCheck, kNoAssert).
-  explicit CodeBufferCheckScope(Assembler* assm) : assm_(assm) {
-#ifdef DEBUG
-    size_ = 0;
-    assert_policy_ = kNoAssert;
-    assm->AcquireBuffer();
-#endif
-  }
-
-  ~CodeBufferCheckScope() {
-#ifdef DEBUG
-    assm_->ReleaseBuffer();
-    switch (assert_policy_) {
-      case kNoAssert: break;
-      case kExactSize:
-        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) == size_);
-        break;
-      case kMaximumSize:
-        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) <= size_);
-        break;
-      default:
-        VIXL_UNREACHABLE();
-    }
-#endif
-  }
-
- protected:
-  Assembler* assm_;
-#ifdef DEBUG
-  Label start_;
-  size_t size_;
-  AssertPolicy assert_policy_;
-#endif
-};
-
-}  // namespace vixl
-
-#endif  // VIXL_A64_ASSEMBLER_A64_H_
diff --git a/disas/libvixl/a64/constants-a64.h b/disas/libvixl/a64/constants-a64.h
deleted file mode 100644
index 7a14f85..0000000
--- a/disas/libvixl/a64/constants-a64.h
+++ /dev/null
@@ -1,1186 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_CONSTANTS_A64_H_
-#define VIXL_A64_CONSTANTS_A64_H_
-
-namespace vixl {
-
-const unsigned kNumberOfRegisters = 32;
-const unsigned kNumberOfFPRegisters = 32;
-// Callee saved registers are x21-x30(lr).
-const int kNumberOfCalleeSavedRegisters = 10;
-const int kFirstCalleeSavedRegisterIndex = 21;
-// Callee saved FP registers are d8-d15.
-const int kNumberOfCalleeSavedFPRegisters = 8;
-const int kFirstCalleeSavedFPRegisterIndex = 8;
-
-#define REGISTER_CODE_LIST(R)                                                  \
-R(0)  R(1)  R(2)  R(3)  R(4)  R(5)  R(6)  R(7)                                 \
-R(8)  R(9)  R(10) R(11) R(12) R(13) R(14) R(15)                                \
-R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23)                                \
-R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
-
-#define INSTRUCTION_FIELDS_LIST(V_)                                            \
-/* Register fields */                                                          \
-V_(Rd, 4, 0, Bits)                        /* Destination register.        */   \
-V_(Rn, 9, 5, Bits)                        /* First source register.       */   \
-V_(Rm, 20, 16, Bits)                      /* Second source register.      */   \
-V_(Ra, 14, 10, Bits)                      /* Third source register.       */   \
-V_(Rt, 4, 0, Bits)                        /* Load/store register.         */   \
-V_(Rt2, 14, 10, Bits)                     /* Load/store second register.  */   \
-V_(Rs, 20, 16, Bits)                      /* Exclusive access status.     */   \
-V_(PrefetchMode, 4, 0, Bits)                                                   \
-                                                                               \
-/* Common bits */                                                              \
-V_(SixtyFourBits, 31, 31, Bits)                                                \
-V_(FlagsUpdate, 29, 29, Bits)                                                  \
-                                                                               \
-/* PC relative addressing */                                                   \
-V_(ImmPCRelHi, 23, 5, SignedBits)                                              \
-V_(ImmPCRelLo, 30, 29, Bits)                                                   \
-                                                                               \
-/* Add/subtract/logical shift register */                                      \
-V_(ShiftDP, 23, 22, Bits)                                                      \
-V_(ImmDPShift, 15, 10, Bits)                                                   \
-                                                                               \
-/* Add/subtract immediate */                                                   \
-V_(ImmAddSub, 21, 10, Bits)                                                    \
-V_(ShiftAddSub, 23, 22, Bits)                                                  \
-                                                                               \
-/* Add/substract extend */                                                     \
-V_(ImmExtendShift, 12, 10, Bits)                                               \
-V_(ExtendMode, 15, 13, Bits)                                                   \
-                                                                               \
-/* Move wide */                                                                \
-V_(ImmMoveWide, 20, 5, Bits)                                                   \
-V_(ShiftMoveWide, 22, 21, Bits)                                                \
-                                                                               \
-/* Logical immediate, bitfield and extract */                                  \
-V_(BitN, 22, 22, Bits)                                                         \
-V_(ImmRotate, 21, 16, Bits)                                                    \
-V_(ImmSetBits, 15, 10, Bits)                                                   \
-V_(ImmR, 21, 16, Bits)                                                         \
-V_(ImmS, 15, 10, Bits)                                                         \
-                                                                               \
-/* Test and branch immediate */                                                \
-V_(ImmTestBranch, 18, 5, SignedBits)                                           \
-V_(ImmTestBranchBit40, 23, 19, Bits)                                           \
-V_(ImmTestBranchBit5, 31, 31, Bits)                                            \
-                                                                               \
-/* Conditionals */                                                             \
-V_(Condition, 15, 12, Bits)                                                    \
-V_(ConditionBranch, 3, 0, Bits)                                                \
-V_(Nzcv, 3, 0, Bits)                                                           \
-V_(ImmCondCmp, 20, 16, Bits)                                                   \
-V_(ImmCondBranch, 23, 5, SignedBits)                                           \
-                                                                               \
-/* Floating point */                                                           \
-V_(FPType, 23, 22, Bits)                                                       \
-V_(ImmFP, 20, 13, Bits)                                                        \
-V_(FPScale, 15, 10, Bits)                                                      \
-                                                                               \
-/* Load Store */                                                               \
-V_(ImmLS, 20, 12, SignedBits)                                                  \
-V_(ImmLSUnsigned, 21, 10, Bits)                                                \
-V_(ImmLSPair, 21, 15, SignedBits)                                              \
-V_(SizeLS, 31, 30, Bits)                                                       \
-V_(ImmShiftLS, 12, 12, Bits)                                                   \
-                                                                               \
-/* Other immediates */                                                         \
-V_(ImmUncondBranch, 25, 0, SignedBits)                                         \
-V_(ImmCmpBranch, 23, 5, SignedBits)                                            \
-V_(ImmLLiteral, 23, 5, SignedBits)                                             \
-V_(ImmException, 20, 5, Bits)                                                  \
-V_(ImmHint, 11, 5, Bits)                                                       \
-V_(ImmBarrierDomain, 11, 10, Bits)                                             \
-V_(ImmBarrierType, 9, 8, Bits)                                                 \
-                                                                               \
-/* System (MRS, MSR) */                                                        \
-V_(ImmSystemRegister, 19, 5, Bits)                                             \
-V_(SysO0, 19, 19, Bits)                                                        \
-V_(SysOp1, 18, 16, Bits)                                                       \
-V_(SysOp2, 7, 5, Bits)                                                         \
-V_(CRn, 15, 12, Bits)                                                          \
-V_(CRm, 11, 8, Bits)                                                           \
-                                                                               \
-/* Load-/store-exclusive */                                                    \
-V_(LdStXLoad, 22, 22, Bits)                                                    \
-V_(LdStXNotExclusive, 23, 23, Bits)                                            \
-V_(LdStXAcquireRelease, 15, 15, Bits)                                          \
-V_(LdStXSizeLog2, 31, 30, Bits)                                                \
-V_(LdStXPair, 21, 21, Bits)                                                    \
-
-
-#define SYSTEM_REGISTER_FIELDS_LIST(V_, M_)                                    \
-/* NZCV */                                                                     \
-V_(Flags, 31, 28, Bits)                                                        \
-V_(N, 31, 31, Bits)                                                            \
-V_(Z, 30, 30, Bits)                                                            \
-V_(C, 29, 29, Bits)                                                            \
-V_(V, 28, 28, Bits)                                                            \
-M_(NZCV, Flags_mask)                                                           \
-                                                                               \
-/* FPCR */                                                                     \
-V_(AHP, 26, 26, Bits)                                                          \
-V_(DN, 25, 25, Bits)                                                           \
-V_(FZ, 24, 24, Bits)                                                           \
-V_(RMode, 23, 22, Bits)                                                        \
-M_(FPCR, AHP_mask | DN_mask | FZ_mask | RMode_mask)
-
-
-// Fields offsets.
-#define DECLARE_FIELDS_OFFSETS(Name, HighBit, LowBit, X)                       \
-const int Name##_offset = LowBit;                                              \
-const int Name##_width = HighBit - LowBit + 1;                                 \
-const uint32_t Name##_mask = ((1 << Name##_width) - 1) << LowBit;
-#define NOTHING(A, B)
-INSTRUCTION_FIELDS_LIST(DECLARE_FIELDS_OFFSETS)
-SYSTEM_REGISTER_FIELDS_LIST(DECLARE_FIELDS_OFFSETS, NOTHING)
-#undef NOTHING
-#undef DECLARE_FIELDS_BITS
-
-// ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST), formed
-// from ImmPCRelLo and ImmPCRelHi.
-const int ImmPCRel_mask = ImmPCRelLo_mask | ImmPCRelHi_mask;
-
-// Condition codes.
-enum Condition {
-  eq = 0,
-  ne = 1,
-  hs = 2,
-  lo = 3,
-  mi = 4,
-  pl = 5,
-  vs = 6,
-  vc = 7,
-  hi = 8,
-  ls = 9,
-  ge = 10,
-  lt = 11,
-  gt = 12,
-  le = 13,
-  al = 14,
-  nv = 15  // Behaves as always/al.
-};
-
-inline Condition InvertCondition(Condition cond) {
-  // Conditions al and nv behave identically, as "always true". They can't be
-  // inverted, because there is no "always false" condition.
-  VIXL_ASSERT((cond != al) && (cond != nv));
-  return static_cast<Condition>(cond ^ 1);
-}
-
-enum FlagsUpdate {
-  SetFlags   = 1,
-  LeaveFlags = 0
-};
-
-enum StatusFlags {
-  NoFlag    = 0,
-
-  // Derive the flag combinations from the system register bit descriptions.
-  NFlag     = N_mask,
-  ZFlag     = Z_mask,
-  CFlag     = C_mask,
-  VFlag     = V_mask,
-  NZFlag    = NFlag | ZFlag,
-  NCFlag    = NFlag | CFlag,
-  NVFlag    = NFlag | VFlag,
-  ZCFlag    = ZFlag | CFlag,
-  ZVFlag    = ZFlag | VFlag,
-  CVFlag    = CFlag | VFlag,
-  NZCFlag   = NFlag | ZFlag | CFlag,
-  NZVFlag   = NFlag | ZFlag | VFlag,
-  NCVFlag   = NFlag | CFlag | VFlag,
-  ZCVFlag   = ZFlag | CFlag | VFlag,
-  NZCVFlag  = NFlag | ZFlag | CFlag | VFlag,
-
-  // Floating-point comparison results.
-  FPEqualFlag       = ZCFlag,
-  FPLessThanFlag    = NFlag,
-  FPGreaterThanFlag = CFlag,
-  FPUnorderedFlag   = CVFlag
-};
-
-enum Shift {
-  NO_SHIFT = -1,
-  LSL = 0x0,
-  LSR = 0x1,
-  ASR = 0x2,
-  ROR = 0x3
-};
-
-enum Extend {
-  NO_EXTEND = -1,
-  UXTB      = 0,
-  UXTH      = 1,
-  UXTW      = 2,
-  UXTX      = 3,
-  SXTB      = 4,
-  SXTH      = 5,
-  SXTW      = 6,
-  SXTX      = 7
-};
-
-enum SystemHint {
-  NOP   = 0,
-  YIELD = 1,
-  WFE   = 2,
-  WFI   = 3,
-  SEV   = 4,
-  SEVL  = 5
-};
-
-enum BarrierDomain {
-  OuterShareable = 0,
-  NonShareable   = 1,
-  InnerShareable = 2,
-  FullSystem     = 3
-};
-
-enum BarrierType {
-  BarrierOther  = 0,
-  BarrierReads  = 1,
-  BarrierWrites = 2,
-  BarrierAll    = 3
-};
-
-// System/special register names.
-// This information is not encoded as one field but as the concatenation of
-// multiple fields (Op0<0>, Op1, Crn, Crm, Op2).
-enum SystemRegister {
-  NZCV = ((0x1 << SysO0_offset) |
-          (0x3 << SysOp1_offset) |
-          (0x4 << CRn_offset) |
-          (0x2 << CRm_offset) |
-          (0x0 << SysOp2_offset)) >> ImmSystemRegister_offset,
-  FPCR = ((0x1 << SysO0_offset) |
-          (0x3 << SysOp1_offset) |
-          (0x4 << CRn_offset) |
-          (0x4 << CRm_offset) |
-          (0x0 << SysOp2_offset)) >> ImmSystemRegister_offset
-};
-
-// Instruction enumerations.
-//
-// These are the masks that define a class of instructions, and the list of
-// instructions within each class. Each enumeration has a Fixed, FMask and
-// Mask value.
-//
-// Fixed: The fixed bits in this instruction class.
-// FMask: The mask used to extract the fixed bits in the class.
-// Mask:  The mask used to identify the instructions within a class.
-//
-// The enumerations can be used like this:
-//
-// VIXL_ASSERT(instr->Mask(PCRelAddressingFMask) == PCRelAddressingFixed);
-// switch(instr->Mask(PCRelAddressingMask)) {
-//   case ADR:  Format("adr 'Xd, 'AddrPCRelByte"); break;
-//   case ADRP: Format("adrp 'Xd, 'AddrPCRelPage"); break;
-//   default:   printf("Unknown instruction\n");
-// }
-
-
-// Generic fields.
-enum GenericInstrField {
-  SixtyFourBits        = 0x80000000,
-  ThirtyTwoBits        = 0x00000000,
-  FP32                 = 0x00000000,
-  FP64                 = 0x00400000
-};
-
-// PC relative addressing.
-enum PCRelAddressingOp {
-  PCRelAddressingFixed = 0x10000000,
-  PCRelAddressingFMask = 0x1F000000,
-  PCRelAddressingMask  = 0x9F000000,
-  ADR                  = PCRelAddressingFixed | 0x00000000,
-  ADRP                 = PCRelAddressingFixed | 0x80000000
-};
-
-// Add/sub (immediate, shifted and extended.)
-const int kSFOffset = 31;
-enum AddSubOp {
-  AddSubOpMask      = 0x60000000,
-  AddSubSetFlagsBit = 0x20000000,
-  ADD               = 0x00000000,
-  ADDS              = ADD | AddSubSetFlagsBit,
-  SUB               = 0x40000000,
-  SUBS              = SUB | AddSubSetFlagsBit
-};
-
-#define ADD_SUB_OP_LIST(V)  \
-  V(ADD),                   \
-  V(ADDS),                  \
-  V(SUB),                   \
-  V(SUBS)
-
-enum AddSubImmediateOp {
-  AddSubImmediateFixed = 0x11000000,
-  AddSubImmediateFMask = 0x1F000000,
-  AddSubImmediateMask  = 0xFF000000,
-  #define ADD_SUB_IMMEDIATE(A)           \
-  A##_w_imm = AddSubImmediateFixed | A,  \
-  A##_x_imm = AddSubImmediateFixed | A | SixtyFourBits
-  ADD_SUB_OP_LIST(ADD_SUB_IMMEDIATE)
-  #undef ADD_SUB_IMMEDIATE
-};
-
-enum AddSubShiftedOp {
-  AddSubShiftedFixed   = 0x0B000000,
-  AddSubShiftedFMask   = 0x1F200000,
-  AddSubShiftedMask    = 0xFF200000,
-  #define ADD_SUB_SHIFTED(A)             \
-  A##_w_shift = AddSubShiftedFixed | A,  \
-  A##_x_shift = AddSubShiftedFixed | A | SixtyFourBits
-  ADD_SUB_OP_LIST(ADD_SUB_SHIFTED)
-  #undef ADD_SUB_SHIFTED
-};
-
-enum AddSubExtendedOp {
-  AddSubExtendedFixed  = 0x0B200000,
-  AddSubExtendedFMask  = 0x1F200000,
-  AddSubExtendedMask   = 0xFFE00000,
-  #define ADD_SUB_EXTENDED(A)           \
-  A##_w_ext = AddSubExtendedFixed | A,  \
-  A##_x_ext = AddSubExtendedFixed | A | SixtyFourBits
-  ADD_SUB_OP_LIST(ADD_SUB_EXTENDED)
-  #undef ADD_SUB_EXTENDED
-};
-
-// Add/sub with carry.
-enum AddSubWithCarryOp {
-  AddSubWithCarryFixed = 0x1A000000,
-  AddSubWithCarryFMask = 0x1FE00000,
-  AddSubWithCarryMask  = 0xFFE0FC00,
-  ADC_w                = AddSubWithCarryFixed | ADD,
-  ADC_x                = AddSubWithCarryFixed | ADD | SixtyFourBits,
-  ADC                  = ADC_w,
-  ADCS_w               = AddSubWithCarryFixed | ADDS,
-  ADCS_x               = AddSubWithCarryFixed | ADDS | SixtyFourBits,
-  SBC_w                = AddSubWithCarryFixed | SUB,
-  SBC_x                = AddSubWithCarryFixed | SUB | SixtyFourBits,
-  SBC                  = SBC_w,
-  SBCS_w               = AddSubWithCarryFixed | SUBS,
-  SBCS_x               = AddSubWithCarryFixed | SUBS | SixtyFourBits
-};
-
-
-// Logical (immediate and shifted register).
-enum LogicalOp {
-  LogicalOpMask = 0x60200000,
-  NOT   = 0x00200000,
-  AND   = 0x00000000,
-  BIC   = AND | NOT,
-  ORR   = 0x20000000,
-  ORN   = ORR | NOT,
-  EOR   = 0x40000000,
-  EON   = EOR | NOT,
-  ANDS  = 0x60000000,
-  BICS  = ANDS | NOT
-};
-
-// Logical immediate.
-enum LogicalImmediateOp {
-  LogicalImmediateFixed = 0x12000000,
-  LogicalImmediateFMask = 0x1F800000,
-  LogicalImmediateMask  = 0xFF800000,
-  AND_w_imm   = LogicalImmediateFixed | AND,
-  AND_x_imm   = LogicalImmediateFixed | AND | SixtyFourBits,
-  ORR_w_imm   = LogicalImmediateFixed | ORR,
-  ORR_x_imm   = LogicalImmediateFixed | ORR | SixtyFourBits,
-  EOR_w_imm   = LogicalImmediateFixed | EOR,
-  EOR_x_imm   = LogicalImmediateFixed | EOR | SixtyFourBits,
-  ANDS_w_imm  = LogicalImmediateFixed | ANDS,
-  ANDS_x_imm  = LogicalImmediateFixed | ANDS | SixtyFourBits
-};
-
-// Logical shifted register.
-enum LogicalShiftedOp {
-  LogicalShiftedFixed = 0x0A000000,
-  LogicalShiftedFMask = 0x1F000000,
-  LogicalShiftedMask  = 0xFF200000,
-  AND_w               = LogicalShiftedFixed | AND,
-  AND_x               = LogicalShiftedFixed | AND | SixtyFourBits,
-  AND_shift           = AND_w,
-  BIC_w               = LogicalShiftedFixed | BIC,
-  BIC_x               = LogicalShiftedFixed | BIC | SixtyFourBits,
-  BIC_shift           = BIC_w,
-  ORR_w               = LogicalShiftedFixed | ORR,
-  ORR_x               = LogicalShiftedFixed | ORR | SixtyFourBits,
-  ORR_shift           = ORR_w,
-  ORN_w               = LogicalShiftedFixed | ORN,
-  ORN_x               = LogicalShiftedFixed | ORN | SixtyFourBits,
-  ORN_shift           = ORN_w,
-  EOR_w               = LogicalShiftedFixed | EOR,
-  EOR_x               = LogicalShiftedFixed | EOR | SixtyFourBits,
-  EOR_shift           = EOR_w,
-  EON_w               = LogicalShiftedFixed | EON,
-  EON_x               = LogicalShiftedFixed | EON | SixtyFourBits,
-  EON_shift           = EON_w,
-  ANDS_w              = LogicalShiftedFixed | ANDS,
-  ANDS_x              = LogicalShiftedFixed | ANDS | SixtyFourBits,
-  ANDS_shift          = ANDS_w,
-  BICS_w              = LogicalShiftedFixed | BICS,
-  BICS_x              = LogicalShiftedFixed | BICS | SixtyFourBits,
-  BICS_shift          = BICS_w
-};
-
-// Move wide immediate.
-enum MoveWideImmediateOp {
-  MoveWideImmediateFixed = 0x12800000,
-  MoveWideImmediateFMask = 0x1F800000,
-  MoveWideImmediateMask  = 0xFF800000,
-  MOVN                   = 0x00000000,
-  MOVZ                   = 0x40000000,
-  MOVK                   = 0x60000000,
-  MOVN_w                 = MoveWideImmediateFixed | MOVN,
-  MOVN_x                 = MoveWideImmediateFixed | MOVN | SixtyFourBits,
-  MOVZ_w                 = MoveWideImmediateFixed | MOVZ,
-  MOVZ_x                 = MoveWideImmediateFixed | MOVZ | SixtyFourBits,
-  MOVK_w                 = MoveWideImmediateFixed | MOVK,
-  MOVK_x                 = MoveWideImmediateFixed | MOVK | SixtyFourBits
-};
-
-// Bitfield.
-const int kBitfieldNOffset = 22;
-enum BitfieldOp {
-  BitfieldFixed = 0x13000000,
-  BitfieldFMask = 0x1F800000,
-  BitfieldMask  = 0xFF800000,
-  SBFM_w        = BitfieldFixed | 0x00000000,
-  SBFM_x        = BitfieldFixed | 0x80000000,
-  SBFM          = SBFM_w,
-  BFM_w         = BitfieldFixed | 0x20000000,
-  BFM_x         = BitfieldFixed | 0xA0000000,
-  BFM           = BFM_w,
-  UBFM_w        = BitfieldFixed | 0x40000000,
-  UBFM_x        = BitfieldFixed | 0xC0000000,
-  UBFM          = UBFM_w
-  // Bitfield N field.
-};
-
-// Extract.
-enum ExtractOp {
-  ExtractFixed = 0x13800000,
-  ExtractFMask = 0x1F800000,
-  ExtractMask  = 0xFFA00000,
-  EXTR_w       = ExtractFixed | 0x00000000,
-  EXTR_x       = ExtractFixed | 0x80000000,
-  EXTR         = EXTR_w
-};
-
-// Unconditional branch.
-enum UnconditionalBranchOp {
-  UnconditionalBranchFixed = 0x14000000,
-  UnconditionalBranchFMask = 0x7C000000,
-  UnconditionalBranchMask  = 0xFC000000,
-  B                        = UnconditionalBranchFixed | 0x00000000,
-  BL                       = UnconditionalBranchFixed | 0x80000000
-};
-
-// Unconditional branch to register.
-enum UnconditionalBranchToRegisterOp {
-  UnconditionalBranchToRegisterFixed = 0xD6000000,
-  UnconditionalBranchToRegisterFMask = 0xFE000000,
-  UnconditionalBranchToRegisterMask  = 0xFFFFFC1F,
-  BR      = UnconditionalBranchToRegisterFixed | 0x001F0000,
-  BLR     = UnconditionalBranchToRegisterFixed | 0x003F0000,
-  RET     = UnconditionalBranchToRegisterFixed | 0x005F0000
-};
-
-// Compare and branch.
-enum CompareBranchOp {
-  CompareBranchFixed = 0x34000000,
-  CompareBranchFMask = 0x7E000000,
-  CompareBranchMask  = 0xFF000000,
-  CBZ_w              = CompareBranchFixed | 0x00000000,
-  CBZ_x              = CompareBranchFixed | 0x80000000,
-  CBZ                = CBZ_w,
-  CBNZ_w             = CompareBranchFixed | 0x01000000,
-  CBNZ_x             = CompareBranchFixed | 0x81000000,
-  CBNZ               = CBNZ_w
-};
-
-// Test and branch.
-enum TestBranchOp {
-  TestBranchFixed = 0x36000000,
-  TestBranchFMask = 0x7E000000,
-  TestBranchMask  = 0x7F000000,
-  TBZ             = TestBranchFixed | 0x00000000,
-  TBNZ            = TestBranchFixed | 0x01000000
-};
-
-// Conditional branch.
-enum ConditionalBranchOp {
-  ConditionalBranchFixed = 0x54000000,
-  ConditionalBranchFMask = 0xFE000000,
-  ConditionalBranchMask  = 0xFF000010,
-  B_cond                 = ConditionalBranchFixed | 0x00000000
-};
-
-// System.
-// System instruction encoding is complicated because some instructions use op
-// and CR fields to encode parameters. To handle this cleanly, the system
-// instructions are split into more than one enum.
-
-enum SystemOp {
-  SystemFixed = 0xD5000000,
-  SystemFMask = 0xFFC00000
-};
-
-enum SystemSysRegOp {
-  SystemSysRegFixed = 0xD5100000,
-  SystemSysRegFMask = 0xFFD00000,
-  SystemSysRegMask  = 0xFFF00000,
-  MRS               = SystemSysRegFixed | 0x00200000,
-  MSR               = SystemSysRegFixed | 0x00000000
-};
-
-enum SystemHintOp {
-  SystemHintFixed = 0xD503201F,
-  SystemHintFMask = 0xFFFFF01F,
-  SystemHintMask  = 0xFFFFF01F,
-  HINT            = SystemHintFixed | 0x00000000
-};
-
-// Exception.
-enum ExceptionOp {
-  ExceptionFixed = 0xD4000000,
-  ExceptionFMask = 0xFF000000,
-  ExceptionMask  = 0xFFE0001F,
-  HLT            = ExceptionFixed | 0x00400000,
-  BRK            = ExceptionFixed | 0x00200000,
-  SVC            = ExceptionFixed | 0x00000001,
-  HVC            = ExceptionFixed | 0x00000002,
-  SMC            = ExceptionFixed | 0x00000003,
-  DCPS1          = ExceptionFixed | 0x00A00001,
-  DCPS2          = ExceptionFixed | 0x00A00002,
-  DCPS3          = ExceptionFixed | 0x00A00003
-};
-
-enum MemBarrierOp {
-  MemBarrierFixed = 0xD503309F,
-  MemBarrierFMask = 0xFFFFF09F,
-  MemBarrierMask  = 0xFFFFF0FF,
-  DSB             = MemBarrierFixed | 0x00000000,
-  DMB             = MemBarrierFixed | 0x00000020,
-  ISB             = MemBarrierFixed | 0x00000040
-};
-
-enum SystemExclusiveMonitorOp {
-  SystemExclusiveMonitorFixed = 0xD503305F,
-  SystemExclusiveMonitorFMask = 0xFFFFF0FF,
-  SystemExclusiveMonitorMask  = 0xFFFFF0FF,
-  CLREX                       = SystemExclusiveMonitorFixed
-};
-
-// Any load or store.
-enum LoadStoreAnyOp {
-  LoadStoreAnyFMask = 0x0a000000,
-  LoadStoreAnyFixed = 0x08000000
-};
-
-#define LOAD_STORE_PAIR_OP_LIST(V)  \
-  V(STP, w,   0x00000000),          \
-  V(LDP, w,   0x00400000),          \
-  V(LDPSW, x, 0x40400000),          \
-  V(STP, x,   0x80000000),          \
-  V(LDP, x,   0x80400000),          \
-  V(STP, s,   0x04000000),          \
-  V(LDP, s,   0x04400000),          \
-  V(STP, d,   0x44000000),          \
-  V(LDP, d,   0x44400000)
-
-// Load/store pair (post, pre and offset.)
-enum LoadStorePairOp {
-  LoadStorePairMask = 0xC4400000,
-  LoadStorePairLBit = 1 << 22,
-  #define LOAD_STORE_PAIR(A, B, C) \
-  A##_##B = C
-  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR)
-  #undef LOAD_STORE_PAIR
-};
-
-enum LoadStorePairPostIndexOp {
-  LoadStorePairPostIndexFixed = 0x28800000,
-  LoadStorePairPostIndexFMask = 0x3B800000,
-  LoadStorePairPostIndexMask  = 0xFFC00000,
-  #define LOAD_STORE_PAIR_POST_INDEX(A, B, C)  \
-  A##_##B##_post = LoadStorePairPostIndexFixed | A##_##B
-  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_POST_INDEX)
-  #undef LOAD_STORE_PAIR_POST_INDEX
-};
-
-enum LoadStorePairPreIndexOp {
-  LoadStorePairPreIndexFixed = 0x29800000,
-  LoadStorePairPreIndexFMask = 0x3B800000,
-  LoadStorePairPreIndexMask  = 0xFFC00000,
-  #define LOAD_STORE_PAIR_PRE_INDEX(A, B, C)  \
-  A##_##B##_pre = LoadStorePairPreIndexFixed | A##_##B
-  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_PRE_INDEX)
-  #undef LOAD_STORE_PAIR_PRE_INDEX
-};
-
-enum LoadStorePairOffsetOp {
-  LoadStorePairOffsetFixed = 0x29000000,
-  LoadStorePairOffsetFMask = 0x3B800000,
-  LoadStorePairOffsetMask  = 0xFFC00000,
-  #define LOAD_STORE_PAIR_OFFSET(A, B, C)  \
-  A##_##B##_off = LoadStorePairOffsetFixed | A##_##B
-  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_OFFSET)
-  #undef LOAD_STORE_PAIR_OFFSET
-};
-
-enum LoadStorePairNonTemporalOp {
-  LoadStorePairNonTemporalFixed = 0x28000000,
-  LoadStorePairNonTemporalFMask = 0x3B800000,
-  LoadStorePairNonTemporalMask  = 0xFFC00000,
-  STNP_w = LoadStorePairNonTemporalFixed | STP_w,
-  LDNP_w = LoadStorePairNonTemporalFixed | LDP_w,
-  STNP_x = LoadStorePairNonTemporalFixed | STP_x,
-  LDNP_x = LoadStorePairNonTemporalFixed | LDP_x,
-  STNP_s = LoadStorePairNonTemporalFixed | STP_s,
-  LDNP_s = LoadStorePairNonTemporalFixed | LDP_s,
-  STNP_d = LoadStorePairNonTemporalFixed | STP_d,
-  LDNP_d = LoadStorePairNonTemporalFixed | LDP_d
-};
-
-// Load literal.
-enum LoadLiteralOp {
-  LoadLiteralFixed = 0x18000000,
-  LoadLiteralFMask = 0x3B000000,
-  LoadLiteralMask  = 0xFF000000,
-  LDR_w_lit        = LoadLiteralFixed | 0x00000000,
-  LDR_x_lit        = LoadLiteralFixed | 0x40000000,
-  LDRSW_x_lit      = LoadLiteralFixed | 0x80000000,
-  PRFM_lit         = LoadLiteralFixed | 0xC0000000,
-  LDR_s_lit        = LoadLiteralFixed | 0x04000000,
-  LDR_d_lit        = LoadLiteralFixed | 0x44000000
-};
-
-#define LOAD_STORE_OP_LIST(V)     \
-  V(ST, RB, w,  0x00000000),  \
-  V(ST, RH, w,  0x40000000),  \
-  V(ST, R, w,   0x80000000),  \
-  V(ST, R, x,   0xC0000000),  \
-  V(LD, RB, w,  0x00400000),  \
-  V(LD, RH, w,  0x40400000),  \
-  V(LD, R, w,   0x80400000),  \
-  V(LD, R, x,   0xC0400000),  \
-  V(LD, RSB, x, 0x00800000),  \
-  V(LD, RSH, x, 0x40800000),  \
-  V(LD, RSW, x, 0x80800000),  \
-  V(LD, RSB, w, 0x00C00000),  \
-  V(LD, RSH, w, 0x40C00000),  \
-  V(ST, R, s,   0x84000000),  \
-  V(ST, R, d,   0xC4000000),  \
-  V(LD, R, s,   0x84400000),  \
-  V(LD, R, d,   0xC4400000)
-
-
-// Load/store unscaled offset.
-enum LoadStoreUnscaledOffsetOp {
-  LoadStoreUnscaledOffsetFixed = 0x38000000,
-  LoadStoreUnscaledOffsetFMask = 0x3B200C00,
-  LoadStoreUnscaledOffsetMask  = 0xFFE00C00,
-  #define LOAD_STORE_UNSCALED(A, B, C, D)  \
-  A##U##B##_##C = LoadStoreUnscaledOffsetFixed | D
-  LOAD_STORE_OP_LIST(LOAD_STORE_UNSCALED)
-  #undef LOAD_STORE_UNSCALED
-};
-
-// Load/store (post, pre, offset and unsigned.)
-enum LoadStoreOp {
-  LoadStoreOpMask = 0xC4C00000,
-  #define LOAD_STORE(A, B, C, D)  \
-  A##B##_##C = D
-  LOAD_STORE_OP_LIST(LOAD_STORE),
-  #undef LOAD_STORE
-  PRFM = 0xC0800000
-};
-
-// Load/store post index.
-enum LoadStorePostIndex {
-  LoadStorePostIndexFixed = 0x38000400,
-  LoadStorePostIndexFMask = 0x3B200C00,
-  LoadStorePostIndexMask  = 0xFFE00C00,
-  #define LOAD_STORE_POST_INDEX(A, B, C, D)  \
-  A##B##_##C##_post = LoadStorePostIndexFixed | D
-  LOAD_STORE_OP_LIST(LOAD_STORE_POST_INDEX)
-  #undef LOAD_STORE_POST_INDEX
-};
-
-// Load/store pre index.
-enum LoadStorePreIndex {
-  LoadStorePreIndexFixed = 0x38000C00,
-  LoadStorePreIndexFMask = 0x3B200C00,
-  LoadStorePreIndexMask  = 0xFFE00C00,
-  #define LOAD_STORE_PRE_INDEX(A, B, C, D)  \
-  A##B##_##C##_pre = LoadStorePreIndexFixed | D
-  LOAD_STORE_OP_LIST(LOAD_STORE_PRE_INDEX)
-  #undef LOAD_STORE_PRE_INDEX
-};
-
-// Load/store unsigned offset.
-enum LoadStoreUnsignedOffset {
-  LoadStoreUnsignedOffsetFixed = 0x39000000,
-  LoadStoreUnsignedOffsetFMask = 0x3B000000,
-  LoadStoreUnsignedOffsetMask  = 0xFFC00000,
-  PRFM_unsigned                = LoadStoreUnsignedOffsetFixed | PRFM,
-  #define LOAD_STORE_UNSIGNED_OFFSET(A, B, C, D) \
-  A##B##_##C##_unsigned = LoadStoreUnsignedOffsetFixed | D
-  LOAD_STORE_OP_LIST(LOAD_STORE_UNSIGNED_OFFSET)
-  #undef LOAD_STORE_UNSIGNED_OFFSET
-};
-
-// Load/store register offset.
-enum LoadStoreRegisterOffset {
-  LoadStoreRegisterOffsetFixed = 0x38200800,
-  LoadStoreRegisterOffsetFMask = 0x3B200C00,
-  LoadStoreRegisterOffsetMask  = 0xFFE00C00,
-  PRFM_reg                     = LoadStoreRegisterOffsetFixed | PRFM,
-  #define LOAD_STORE_REGISTER_OFFSET(A, B, C, D) \
-  A##B##_##C##_reg = LoadStoreRegisterOffsetFixed | D
-  LOAD_STORE_OP_LIST(LOAD_STORE_REGISTER_OFFSET)
-  #undef LOAD_STORE_REGISTER_OFFSET
-};
-
-enum LoadStoreExclusive {
-  LoadStoreExclusiveFixed = 0x08000000,
-  LoadStoreExclusiveFMask = 0x3F000000,
-  LoadStoreExclusiveMask  = 0xFFE08000,
-  STXRB_w  = LoadStoreExclusiveFixed | 0x00000000,
-  STXRH_w  = LoadStoreExclusiveFixed | 0x40000000,
-  STXR_w   = LoadStoreExclusiveFixed | 0x80000000,
-  STXR_x   = LoadStoreExclusiveFixed | 0xC0000000,
-  LDXRB_w  = LoadStoreExclusiveFixed | 0x00400000,
-  LDXRH_w  = LoadStoreExclusiveFixed | 0x40400000,
-  LDXR_w   = LoadStoreExclusiveFixed | 0x80400000,
-  LDXR_x   = LoadStoreExclusiveFixed | 0xC0400000,
-  STXP_w   = LoadStoreExclusiveFixed | 0x80200000,
-  STXP_x   = LoadStoreExclusiveFixed | 0xC0200000,
-  LDXP_w   = LoadStoreExclusiveFixed | 0x80600000,
-  LDXP_x   = LoadStoreExclusiveFixed | 0xC0600000,
-  STLXRB_w = LoadStoreExclusiveFixed | 0x00008000,
-  STLXRH_w = LoadStoreExclusiveFixed | 0x40008000,
-  STLXR_w  = LoadStoreExclusiveFixed | 0x80008000,
-  STLXR_x  = LoadStoreExclusiveFixed | 0xC0008000,
-  LDAXRB_w = LoadStoreExclusiveFixed | 0x00408000,
-  LDAXRH_w = LoadStoreExclusiveFixed | 0x40408000,
-  LDAXR_w  = LoadStoreExclusiveFixed | 0x80408000,
-  LDAXR_x  = LoadStoreExclusiveFixed | 0xC0408000,
-  STLXP_w  = LoadStoreExclusiveFixed | 0x80208000,
-  STLXP_x  = LoadStoreExclusiveFixed | 0xC0208000,
-  LDAXP_w  = LoadStoreExclusiveFixed | 0x80608000,
-  LDAXP_x  = LoadStoreExclusiveFixed | 0xC0608000,
-  STLRB_w  = LoadStoreExclusiveFixed | 0x00808000,
-  STLRH_w  = LoadStoreExclusiveFixed | 0x40808000,
-  STLR_w   = LoadStoreExclusiveFixed | 0x80808000,
-  STLR_x   = LoadStoreExclusiveFixed | 0xC0808000,
-  LDARB_w  = LoadStoreExclusiveFixed | 0x00C08000,
-  LDARH_w  = LoadStoreExclusiveFixed | 0x40C08000,
-  LDAR_w   = LoadStoreExclusiveFixed | 0x80C08000,
-  LDAR_x   = LoadStoreExclusiveFixed | 0xC0C08000
-};
-
-// Conditional compare.
-enum ConditionalCompareOp {
-  ConditionalCompareMask = 0x60000000,
-  CCMN                   = 0x20000000,
-  CCMP                   = 0x60000000
-};
-
-// Conditional compare register.
-enum ConditionalCompareRegisterOp {
-  ConditionalCompareRegisterFixed = 0x1A400000,
-  ConditionalCompareRegisterFMask = 0x1FE00800,
-  ConditionalCompareRegisterMask  = 0xFFE00C10,
-  CCMN_w = ConditionalCompareRegisterFixed | CCMN,
-  CCMN_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMN,
-  CCMP_w = ConditionalCompareRegisterFixed | CCMP,
-  CCMP_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMP
-};
-
-// Conditional compare immediate.
-enum ConditionalCompareImmediateOp {
-  ConditionalCompareImmediateFixed = 0x1A400800,
-  ConditionalCompareImmediateFMask = 0x1FE00800,
-  ConditionalCompareImmediateMask  = 0xFFE00C10,
-  CCMN_w_imm = ConditionalCompareImmediateFixed | CCMN,
-  CCMN_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMN,
-  CCMP_w_imm = ConditionalCompareImmediateFixed | CCMP,
-  CCMP_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMP
-};
-
-// Conditional select.
-enum ConditionalSelectOp {
-  ConditionalSelectFixed = 0x1A800000,
-  ConditionalSelectFMask = 0x1FE00000,
-  ConditionalSelectMask  = 0xFFE00C00,
-  CSEL_w                 = ConditionalSelectFixed | 0x00000000,
-  CSEL_x                 = ConditionalSelectFixed | 0x80000000,
-  CSEL                   = CSEL_w,
-  CSINC_w                = ConditionalSelectFixed | 0x00000400,
-  CSINC_x                = ConditionalSelectFixed | 0x80000400,
-  CSINC                  = CSINC_w,
-  CSINV_w                = ConditionalSelectFixed | 0x40000000,
-  CSINV_x                = ConditionalSelectFixed | 0xC0000000,
-  CSINV                  = CSINV_w,
-  CSNEG_w                = ConditionalSelectFixed | 0x40000400,
-  CSNEG_x                = ConditionalSelectFixed | 0xC0000400,
-  CSNEG                  = CSNEG_w
-};
-
-// Data processing 1 source.
-enum DataProcessing1SourceOp {
-  DataProcessing1SourceFixed = 0x5AC00000,
-  DataProcessing1SourceFMask = 0x5FE00000,
-  DataProcessing1SourceMask  = 0xFFFFFC00,
-  RBIT    = DataProcessing1SourceFixed | 0x00000000,
-  RBIT_w  = RBIT,
-  RBIT_x  = RBIT | SixtyFourBits,
-  REV16   = DataProcessing1SourceFixed | 0x00000400,
-  REV16_w = REV16,
-  REV16_x = REV16 | SixtyFourBits,
-  REV     = DataProcessing1SourceFixed | 0x00000800,
-  REV_w   = REV,
-  REV32_x = REV | SixtyFourBits,
-  REV_x   = DataProcessing1SourceFixed | SixtyFourBits | 0x00000C00,
-  CLZ     = DataProcessing1SourceFixed | 0x00001000,
-  CLZ_w   = CLZ,
-  CLZ_x   = CLZ | SixtyFourBits,
-  CLS     = DataProcessing1SourceFixed | 0x00001400,
-  CLS_w   = CLS,
-  CLS_x   = CLS | SixtyFourBits
-};
-
-// Data processing 2 source.
-enum DataProcessing2SourceOp {
-  DataProcessing2SourceFixed = 0x1AC00000,
-  DataProcessing2SourceFMask = 0x5FE00000,
-  DataProcessing2SourceMask  = 0xFFE0FC00,
-  UDIV_w  = DataProcessing2SourceFixed | 0x00000800,
-  UDIV_x  = DataProcessing2SourceFixed | 0x80000800,
-  UDIV    = UDIV_w,
-  SDIV_w  = DataProcessing2SourceFixed | 0x00000C00,
-  SDIV_x  = DataProcessing2SourceFixed | 0x80000C00,
-  SDIV    = SDIV_w,
-  LSLV_w  = DataProcessing2SourceFixed | 0x00002000,
-  LSLV_x  = DataProcessing2SourceFixed | 0x80002000,
-  LSLV    = LSLV_w,
-  LSRV_w  = DataProcessing2SourceFixed | 0x00002400,
-  LSRV_x  = DataProcessing2SourceFixed | 0x80002400,
-  LSRV    = LSRV_w,
-  ASRV_w  = DataProcessing2SourceFixed | 0x00002800,
-  ASRV_x  = DataProcessing2SourceFixed | 0x80002800,
-  ASRV    = ASRV_w,
-  RORV_w  = DataProcessing2SourceFixed | 0x00002C00,
-  RORV_x  = DataProcessing2SourceFixed | 0x80002C00,
-  RORV    = RORV_w,
-  CRC32B  = DataProcessing2SourceFixed | 0x00004000,
-  CRC32H  = DataProcessing2SourceFixed | 0x00004400,
-  CRC32W  = DataProcessing2SourceFixed | 0x00004800,
-  CRC32X  = DataProcessing2SourceFixed | SixtyFourBits | 0x00004C00,
-  CRC32CB = DataProcessing2SourceFixed | 0x00005000,
-  CRC32CH = DataProcessing2SourceFixed | 0x00005400,
-  CRC32CW = DataProcessing2SourceFixed | 0x00005800,
-  CRC32CX = DataProcessing2SourceFixed | SixtyFourBits | 0x00005C00
-};
-
-// Data processing 3 source.
-enum DataProcessing3SourceOp {
-  DataProcessing3SourceFixed = 0x1B000000,
-  DataProcessing3SourceFMask = 0x1F000000,
-  DataProcessing3SourceMask  = 0xFFE08000,
-  MADD_w                     = DataProcessing3SourceFixed | 0x00000000,
-  MADD_x                     = DataProcessing3SourceFixed | 0x80000000,
-  MADD                       = MADD_w,
-  MSUB_w                     = DataProcessing3SourceFixed | 0x00008000,
-  MSUB_x                     = DataProcessing3SourceFixed | 0x80008000,
-  MSUB                       = MSUB_w,
-  SMADDL_x                   = DataProcessing3SourceFixed | 0x80200000,
-  SMSUBL_x                   = DataProcessing3SourceFixed | 0x80208000,
-  SMULH_x                    = DataProcessing3SourceFixed | 0x80400000,
-  UMADDL_x                   = DataProcessing3SourceFixed | 0x80A00000,
-  UMSUBL_x                   = DataProcessing3SourceFixed | 0x80A08000,
-  UMULH_x                    = DataProcessing3SourceFixed | 0x80C00000
-};
-
-// Floating point compare.
-enum FPCompareOp {
-  FPCompareFixed = 0x1E202000,
-  FPCompareFMask = 0x5F203C00,
-  FPCompareMask  = 0xFFE0FC1F,
-  FCMP_s         = FPCompareFixed | 0x00000000,
-  FCMP_d         = FPCompareFixed | FP64 | 0x00000000,
-  FCMP           = FCMP_s,
-  FCMP_s_zero    = FPCompareFixed | 0x00000008,
-  FCMP_d_zero    = FPCompareFixed | FP64 | 0x00000008,
-  FCMP_zero      = FCMP_s_zero,
-  FCMPE_s        = FPCompareFixed | 0x00000010,
-  FCMPE_d        = FPCompareFixed | FP64 | 0x00000010,
-  FCMPE_s_zero   = FPCompareFixed | 0x00000018,
-  FCMPE_d_zero   = FPCompareFixed | FP64 | 0x00000018
-};
-
-// Floating point conditional compare.
-enum FPConditionalCompareOp {
-  FPConditionalCompareFixed = 0x1E200400,
-  FPConditionalCompareFMask = 0x5F200C00,
-  FPConditionalCompareMask  = 0xFFE00C10,
-  FCCMP_s                   = FPConditionalCompareFixed | 0x00000000,
-  FCCMP_d                   = FPConditionalCompareFixed | FP64 | 0x00000000,
-  FCCMP                     = FCCMP_s,
-  FCCMPE_s                  = FPConditionalCompareFixed | 0x00000010,
-  FCCMPE_d                  = FPConditionalCompareFixed | FP64 | 0x00000010,
-  FCCMPE                    = FCCMPE_s
-};
-
-// Floating point conditional select.
-enum FPConditionalSelectOp {
-  FPConditionalSelectFixed = 0x1E200C00,
-  FPConditionalSelectFMask = 0x5F200C00,
-  FPConditionalSelectMask  = 0xFFE00C00,
-  FCSEL_s                  = FPConditionalSelectFixed | 0x00000000,
-  FCSEL_d                  = FPConditionalSelectFixed | FP64 | 0x00000000,
-  FCSEL                    = FCSEL_s
-};
-
-// Floating point immediate.
-enum FPImmediateOp {
-  FPImmediateFixed = 0x1E201000,
-  FPImmediateFMask = 0x5F201C00,
-  FPImmediateMask  = 0xFFE01C00,
-  FMOV_s_imm       = FPImmediateFixed | 0x00000000,
-  FMOV_d_imm       = FPImmediateFixed | FP64 | 0x00000000
-};
-
-// Floating point data processing 1 source.
-enum FPDataProcessing1SourceOp {
-  FPDataProcessing1SourceFixed = 0x1E204000,
-  FPDataProcessing1SourceFMask = 0x5F207C00,
-  FPDataProcessing1SourceMask  = 0xFFFFFC00,
-  FMOV_s   = FPDataProcessing1SourceFixed | 0x00000000,
-  FMOV_d   = FPDataProcessing1SourceFixed | FP64 | 0x00000000,
-  FMOV     = FMOV_s,
-  FABS_s   = FPDataProcessing1SourceFixed | 0x00008000,
-  FABS_d   = FPDataProcessing1SourceFixed | FP64 | 0x00008000,
-  FABS     = FABS_s,
-  FNEG_s   = FPDataProcessing1SourceFixed | 0x00010000,
-  FNEG_d   = FPDataProcessing1SourceFixed | FP64 | 0x00010000,
-  FNEG     = FNEG_s,
-  FSQRT_s  = FPDataProcessing1SourceFixed | 0x00018000,
-  FSQRT_d  = FPDataProcessing1SourceFixed | FP64 | 0x00018000,
-  FSQRT    = FSQRT_s,
-  FCVT_ds  = FPDataProcessing1SourceFixed | 0x00028000,
-  FCVT_sd  = FPDataProcessing1SourceFixed | FP64 | 0x00020000,
-  FRINTN_s = FPDataProcessing1SourceFixed | 0x00040000,
-  FRINTN_d = FPDataProcessing1SourceFixed | FP64 | 0x00040000,
-  FRINTN   = FRINTN_s,
-  FRINTP_s = FPDataProcessing1SourceFixed | 0x00048000,
-  FRINTP_d = FPDataProcessing1SourceFixed | FP64 | 0x00048000,
-  FRINTP   = FRINTP_s,
-  FRINTM_s = FPDataProcessing1SourceFixed | 0x00050000,
-  FRINTM_d = FPDataProcessing1SourceFixed | FP64 | 0x00050000,
-  FRINTM   = FRINTM_s,
-  FRINTZ_s = FPDataProcessing1SourceFixed | 0x00058000,
-  FRINTZ_d = FPDataProcessing1SourceFixed | FP64 | 0x00058000,
-  FRINTZ   = FRINTZ_s,
-  FRINTA_s = FPDataProcessing1SourceFixed | 0x00060000,
-  FRINTA_d = FPDataProcessing1SourceFixed | FP64 | 0x00060000,
-  FRINTA   = FRINTA_s,
-  FRINTX_s = FPDataProcessing1SourceFixed | 0x00070000,
-  FRINTX_d = FPDataProcessing1SourceFixed | FP64 | 0x00070000,
-  FRINTX   = FRINTX_s,
-  FRINTI_s = FPDataProcessing1SourceFixed | 0x00078000,
-  FRINTI_d = FPDataProcessing1SourceFixed | FP64 | 0x00078000,
-  FRINTI   = FRINTI_s
-};
-
-// Floating point data processing 2 source.
-enum FPDataProcessing2SourceOp {
-  FPDataProcessing2SourceFixed = 0x1E200800,
-  FPDataProcessing2SourceFMask = 0x5F200C00,
-  FPDataProcessing2SourceMask  = 0xFFE0FC00,
-  FMUL     = FPDataProcessing2SourceFixed | 0x00000000,
-  FMUL_s   = FMUL,
-  FMUL_d   = FMUL | FP64,
-  FDIV     = FPDataProcessing2SourceFixed | 0x00001000,
-  FDIV_s   = FDIV,
-  FDIV_d   = FDIV | FP64,
-  FADD     = FPDataProcessing2SourceFixed | 0x00002000,
-  FADD_s   = FADD,
-  FADD_d   = FADD | FP64,
-  FSUB     = FPDataProcessing2SourceFixed | 0x00003000,
-  FSUB_s   = FSUB,
-  FSUB_d   = FSUB | FP64,
-  FMAX     = FPDataProcessing2SourceFixed | 0x00004000,
-  FMAX_s   = FMAX,
-  FMAX_d   = FMAX | FP64,
-  FMIN     = FPDataProcessing2SourceFixed | 0x00005000,
-  FMIN_s   = FMIN,
-  FMIN_d   = FMIN | FP64,
-  FMAXNM   = FPDataProcessing2SourceFixed | 0x00006000,
-  FMAXNM_s = FMAXNM,
-  FMAXNM_d = FMAXNM | FP64,
-  FMINNM   = FPDataProcessing2SourceFixed | 0x00007000,
-  FMINNM_s = FMINNM,
-  FMINNM_d = FMINNM | FP64,
-  FNMUL    = FPDataProcessing2SourceFixed | 0x00008000,
-  FNMUL_s  = FNMUL,
-  FNMUL_d  = FNMUL | FP64
-};
-
-// Floating point data processing 3 source.
-enum FPDataProcessing3SourceOp {
-  FPDataProcessing3SourceFixed = 0x1F000000,
-  FPDataProcessing3SourceFMask = 0x5F000000,
-  FPDataProcessing3SourceMask  = 0xFFE08000,
-  FMADD_s                      = FPDataProcessing3SourceFixed | 0x00000000,
-  FMSUB_s                      = FPDataProcessing3SourceFixed | 0x00008000,
-  FNMADD_s                     = FPDataProcessing3SourceFixed | 0x00200000,
-  FNMSUB_s                     = FPDataProcessing3SourceFixed | 0x00208000,
-  FMADD_d                      = FPDataProcessing3SourceFixed | 0x00400000,
-  FMSUB_d                      = FPDataProcessing3SourceFixed | 0x00408000,
-  FNMADD_d                     = FPDataProcessing3SourceFixed | 0x00600000,
-  FNMSUB_d                     = FPDataProcessing3SourceFixed | 0x00608000
-};
-
-// Conversion between floating point and integer.
-enum FPIntegerConvertOp {
-  FPIntegerConvertFixed = 0x1E200000,
-  FPIntegerConvertFMask = 0x5F20FC00,
-  FPIntegerConvertMask  = 0xFFFFFC00,
-  FCVTNS    = FPIntegerConvertFixed | 0x00000000,
-  FCVTNS_ws = FCVTNS,
-  FCVTNS_xs = FCVTNS | SixtyFourBits,
-  FCVTNS_wd = FCVTNS | FP64,
-  FCVTNS_xd = FCVTNS | SixtyFourBits | FP64,
-  FCVTNU    = FPIntegerConvertFixed | 0x00010000,
-  FCVTNU_ws = FCVTNU,
-  FCVTNU_xs = FCVTNU | SixtyFourBits,
-  FCVTNU_wd = FCVTNU | FP64,
-  FCVTNU_xd = FCVTNU | SixtyFourBits | FP64,
-  FCVTPS    = FPIntegerConvertFixed | 0x00080000,
-  FCVTPS_ws = FCVTPS,
-  FCVTPS_xs = FCVTPS | SixtyFourBits,
-  FCVTPS_wd = FCVTPS | FP64,
-  FCVTPS_xd = FCVTPS | SixtyFourBits | FP64,
-  FCVTPU    = FPIntegerConvertFixed | 0x00090000,
-  FCVTPU_ws = FCVTPU,
-  FCVTPU_xs = FCVTPU | SixtyFourBits,
-  FCVTPU_wd = FCVTPU | FP64,
-  FCVTPU_xd = FCVTPU | SixtyFourBits | FP64,
-  FCVTMS    = FPIntegerConvertFixed | 0x00100000,
-  FCVTMS_ws = FCVTMS,
-  FCVTMS_xs = FCVTMS | SixtyFourBits,
-  FCVTMS_wd = FCVTMS | FP64,
-  FCVTMS_xd = FCVTMS | SixtyFourBits | FP64,
-  FCVTMU    = FPIntegerConvertFixed | 0x00110000,
-  FCVTMU_ws = FCVTMU,
-  FCVTMU_xs = FCVTMU | SixtyFourBits,
-  FCVTMU_wd = FCVTMU | FP64,
-  FCVTMU_xd = FCVTMU | SixtyFourBits | FP64,
-  FCVTZS    = FPIntegerConvertFixed | 0x00180000,
-  FCVTZS_ws = FCVTZS,
-  FCVTZS_xs = FCVTZS | SixtyFourBits,
-  FCVTZS_wd = FCVTZS | FP64,
-  FCVTZS_xd = FCVTZS | SixtyFourBits | FP64,
-  FCVTZU    = FPIntegerConvertFixed | 0x00190000,
-  FCVTZU_ws = FCVTZU,
-  FCVTZU_xs = FCVTZU | SixtyFourBits,
-  FCVTZU_wd = FCVTZU | FP64,
-  FCVTZU_xd = FCVTZU | SixtyFourBits | FP64,
-  SCVTF     = FPIntegerConvertFixed | 0x00020000,
-  SCVTF_sw  = SCVTF,
-  SCVTF_sx  = SCVTF | SixtyFourBits,
-  SCVTF_dw  = SCVTF | FP64,
-  SCVTF_dx  = SCVTF | SixtyFourBits | FP64,
-  UCVTF     = FPIntegerConvertFixed | 0x00030000,
-  UCVTF_sw  = UCVTF,
-  UCVTF_sx  = UCVTF | SixtyFourBits,
-  UCVTF_dw  = UCVTF | FP64,
-  UCVTF_dx  = UCVTF | SixtyFourBits | FP64,
-  FCVTAS    = FPIntegerConvertFixed | 0x00040000,
-  FCVTAS_ws = FCVTAS,
-  FCVTAS_xs = FCVTAS | SixtyFourBits,
-  FCVTAS_wd = FCVTAS | FP64,
-  FCVTAS_xd = FCVTAS | SixtyFourBits | FP64,
-  FCVTAU    = FPIntegerConvertFixed | 0x00050000,
-  FCVTAU_ws = FCVTAU,
-  FCVTAU_xs = FCVTAU | SixtyFourBits,
-  FCVTAU_wd = FCVTAU | FP64,
-  FCVTAU_xd = FCVTAU | SixtyFourBits | FP64,
-  FMOV_ws   = FPIntegerConvertFixed | 0x00060000,
-  FMOV_sw   = FPIntegerConvertFixed | 0x00070000,
-  FMOV_xd   = FMOV_ws | SixtyFourBits | FP64,
-  FMOV_dx   = FMOV_sw | SixtyFourBits | FP64
-};
-
-// Conversion between fixed point and floating point.
-enum FPFixedPointConvertOp {
-  FPFixedPointConvertFixed = 0x1E000000,
-  FPFixedPointConvertFMask = 0x5F200000,
-  FPFixedPointConvertMask  = 0xFFFF0000,
-  FCVTZS_fixed    = FPFixedPointConvertFixed | 0x00180000,
-  FCVTZS_ws_fixed = FCVTZS_fixed,
-  FCVTZS_xs_fixed = FCVTZS_fixed | SixtyFourBits,
-  FCVTZS_wd_fixed = FCVTZS_fixed | FP64,
-  FCVTZS_xd_fixed = FCVTZS_fixed | SixtyFourBits | FP64,
-  FCVTZU_fixed    = FPFixedPointConvertFixed | 0x00190000,
-  FCVTZU_ws_fixed = FCVTZU_fixed,
-  FCVTZU_xs_fixed = FCVTZU_fixed | SixtyFourBits,
-  FCVTZU_wd_fixed = FCVTZU_fixed | FP64,
-  FCVTZU_xd_fixed = FCVTZU_fixed | SixtyFourBits | FP64,
-  SCVTF_fixed     = FPFixedPointConvertFixed | 0x00020000,
-  SCVTF_sw_fixed  = SCVTF_fixed,
-  SCVTF_sx_fixed  = SCVTF_fixed | SixtyFourBits,
-  SCVTF_dw_fixed  = SCVTF_fixed | FP64,
-  SCVTF_dx_fixed  = SCVTF_fixed | SixtyFourBits | FP64,
-  UCVTF_fixed     = FPFixedPointConvertFixed | 0x00030000,
-  UCVTF_sw_fixed  = UCVTF_fixed,
-  UCVTF_sx_fixed  = UCVTF_fixed | SixtyFourBits,
-  UCVTF_dw_fixed  = UCVTF_fixed | FP64,
-  UCVTF_dx_fixed  = UCVTF_fixed | SixtyFourBits | FP64
-};
-
-// Unimplemented and unallocated instructions. These are defined to make fixed
-// bit assertion easier.
-enum UnimplementedOp {
-  UnimplementedFixed = 0x00000000,
-  UnimplementedFMask = 0x00000000
-};
-
-enum UnallocatedOp {
-  UnallocatedFixed = 0x00000000,
-  UnallocatedFMask = 0x00000000
-};
-
-}  // namespace vixl
-
-#endif  // VIXL_A64_CONSTANTS_A64_H_
diff --git a/disas/libvixl/a64/cpu-a64.h b/disas/libvixl/a64/cpu-a64.h
deleted file mode 100644
index 59b7974..0000000
--- a/disas/libvixl/a64/cpu-a64.h
+++ /dev/null
@@ -1,83 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_CPU_A64_H
-#define VIXL_CPU_A64_H
-
-#include "globals.h"
-#include "instructions-a64.h"
-
-namespace vixl {
-
-class CPU {
- public:
-  // Initialise CPU support.
-  static void SetUp();
-
-  // Ensures the data at a given address and with a given size is the same for
-  // the I and D caches. I and D caches are not automatically coherent on ARM
-  // so this operation is required before any dynamically generated code can
-  // safely run.
-  static void EnsureIAndDCacheCoherency(void *address, size_t length);
-
-  // Handle tagged pointers.
-  template <typename T>
-  static T SetPointerTag(T pointer, uint64_t tag) {
-    VIXL_ASSERT(is_uintn(kAddressTagWidth, tag));
-
-    // Use C-style casts to get static_cast behaviour for integral types (T),
-    // and reinterpret_cast behaviour for other types.
-
-    uint64_t raw = (uint64_t)pointer;
-    VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
-
-    raw = (raw & ~kAddressTagMask) | (tag << kAddressTagOffset);
-    return (T)raw;
-  }
-
-  template <typename T>
-  static uint64_t GetPointerTag(T pointer) {
-    // Use C-style casts to get static_cast behaviour for integral types (T),
-    // and reinterpret_cast behaviour for other types.
-
-    uint64_t raw = (uint64_t)pointer;
-    VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
-
-    return (raw & kAddressTagMask) >> kAddressTagOffset;
-  }
-
- private:
-  // Return the content of the cache type register.
-  static uint32_t GetCacheType();
-
-  // I and D cache line size in bytes.
-  static unsigned icache_line_size_;
-  static unsigned dcache_line_size_;
-};
-
-}  // namespace vixl
-
-#endif  // VIXL_CPU_A64_H
diff --git a/disas/libvixl/a64/decoder-a64.cc b/disas/libvixl/a64/decoder-a64.cc
deleted file mode 100644
index 82591ca..0000000
--- a/disas/libvixl/a64/decoder-a64.cc
+++ /dev/null
@@ -1,707 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "globals.h"
-#include "utils.h"
-#include "a64/decoder-a64.h"
-
-namespace vixl {
-
-void Decoder::DecodeInstruction(const Instruction *instr) {
-  if (instr->Bits(28, 27) == 0) {
-    VisitUnallocated(instr);
-  } else {
-    switch (instr->Bits(27, 24)) {
-      // 0:   PC relative addressing.
-      case 0x0: DecodePCRelAddressing(instr); break;
-
-      // 1:   Add/sub immediate.
-      case 0x1: DecodeAddSubImmediate(instr); break;
-
-      // A:   Logical shifted register.
-      //      Add/sub with carry.
-      //      Conditional compare register.
-      //      Conditional compare immediate.
-      //      Conditional select.
-      //      Data processing 1 source.
-      //      Data processing 2 source.
-      // B:   Add/sub shifted register.
-      //      Add/sub extended register.
-      //      Data processing 3 source.
-      case 0xA:
-      case 0xB: DecodeDataProcessing(instr); break;
-
-      // 2:   Logical immediate.
-      //      Move wide immediate.
-      case 0x2: DecodeLogical(instr); break;
-
-      // 3:   Bitfield.
-      //      Extract.
-      case 0x3: DecodeBitfieldExtract(instr); break;
-
-      // 4:   Unconditional branch immediate.
-      //      Exception generation.
-      //      Compare and branch immediate.
-      // 5:   Compare and branch immediate.
-      //      Conditional branch.
-      //      System.
-      // 6,7: Unconditional branch.
-      //      Test and branch immediate.
-      case 0x4:
-      case 0x5:
-      case 0x6:
-      case 0x7: DecodeBranchSystemException(instr); break;
-
-      // 8,9: Load/store register pair post-index.
-      //      Load register literal.
-      //      Load/store register unscaled immediate.
-      //      Load/store register immediate post-index.
-      //      Load/store register immediate pre-index.
-      //      Load/store register offset.
-      //      Load/store exclusive.
-      // C,D: Load/store register pair offset.
-      //      Load/store register pair pre-index.
-      //      Load/store register unsigned immediate.
-      //      Advanced SIMD.
-      case 0x8:
-      case 0x9:
-      case 0xC:
-      case 0xD: DecodeLoadStore(instr); break;
-
-      // E:   FP fixed point conversion.
-      //      FP integer conversion.
-      //      FP data processing 1 source.
-      //      FP compare.
-      //      FP immediate.
-      //      FP data processing 2 source.
-      //      FP conditional compare.
-      //      FP conditional select.
-      //      Advanced SIMD.
-      // F:   FP data processing 3 source.
-      //      Advanced SIMD.
-      case 0xE:
-      case 0xF: DecodeFP(instr); break;
-    }
-  }
-}
-
-void Decoder::AppendVisitor(DecoderVisitor* new_visitor) {
-  visitors_.push_back(new_visitor);
-}
-
-
-void Decoder::PrependVisitor(DecoderVisitor* new_visitor) {
-  visitors_.push_front(new_visitor);
-}
-
-
-void Decoder::InsertVisitorBefore(DecoderVisitor* new_visitor,
-                                  DecoderVisitor* registered_visitor) {
-  std::list<DecoderVisitor*>::iterator it;
-  for (it = visitors_.begin(); it != visitors_.end(); it++) {
-    if (*it == registered_visitor) {
-      visitors_.insert(it, new_visitor);
-      return;
-    }
-  }
-  // We reached the end of the list. The last element must be
-  // registered_visitor.
-  VIXL_ASSERT(*it == registered_visitor);
-  visitors_.insert(it, new_visitor);
-}
-
-
-void Decoder::InsertVisitorAfter(DecoderVisitor* new_visitor,
-                                 DecoderVisitor* registered_visitor) {
-  std::list<DecoderVisitor*>::iterator it;
-  for (it = visitors_.begin(); it != visitors_.end(); it++) {
-    if (*it == registered_visitor) {
-      it++;
-      visitors_.insert(it, new_visitor);
-      return;
-    }
-  }
-  // We reached the end of the list. The last element must be
-  // registered_visitor.
-  VIXL_ASSERT(*it == registered_visitor);
-  visitors_.push_back(new_visitor);
-}
-
-
-void Decoder::RemoveVisitor(DecoderVisitor* visitor) {
-  visitors_.remove(visitor);
-}
-
-
-void Decoder::DecodePCRelAddressing(const Instruction* instr) {
-  VIXL_ASSERT(instr->Bits(27, 24) == 0x0);
-  // We know bit 28 is set, as <b28:b27> = 0 is filtered out at the top level
-  // decode.
-  VIXL_ASSERT(instr->Bit(28) == 0x1);
-  VisitPCRelAddressing(instr);
-}
-
-
-void Decoder::DecodeBranchSystemException(const Instruction* instr) {
-  VIXL_ASSERT((instr->Bits(27, 24) == 0x4) ||
-              (instr->Bits(27, 24) == 0x5) ||
-              (instr->Bits(27, 24) == 0x6) ||
-              (instr->Bits(27, 24) == 0x7) );
-
-  switch (instr->Bits(31, 29)) {
-    case 0:
-    case 4: {
-      VisitUnconditionalBranch(instr);
-      break;
-    }
-    case 1:
-    case 5: {
-      if (instr->Bit(25) == 0) {
-        VisitCompareBranch(instr);
-      } else {
-        VisitTestBranch(instr);
-      }
-      break;
-    }
-    case 2: {
-      if (instr->Bit(25) == 0) {
-        if ((instr->Bit(24) == 0x1) ||
-            (instr->Mask(0x01000010) == 0x00000010)) {
-          VisitUnallocated(instr);
-        } else {
-          VisitConditionalBranch(instr);
-        }
-      } else {
-        VisitUnallocated(instr);
-      }
-      break;
-    }
-    case 6: {
-      if (instr->Bit(25) == 0) {
-        if (instr->Bit(24) == 0) {
-          if ((instr->Bits(4, 2) != 0) ||
-              (instr->Mask(0x00E0001D) == 0x00200001) ||
-              (instr->Mask(0x00E0001D) == 0x00400001) ||
-              (instr->Mask(0x00E0001E) == 0x00200002) ||
-              (instr->Mask(0x00E0001E) == 0x00400002) ||
-              (instr->Mask(0x00E0001C) == 0x00600000) ||
-              (instr->Mask(0x00E0001C) == 0x00800000) ||
-              (instr->Mask(0x00E0001F) == 0x00A00000) ||
-              (instr->Mask(0x00C0001C) == 0x00C00000)) {
-            VisitUnallocated(instr);
-          } else {
-            VisitException(instr);
-          }
-        } else {
-          if (instr->Bits(23, 22) == 0) {
-            const Instr masked_003FF0E0 = instr->Mask(0x003FF0E0);
-            if ((instr->Bits(21, 19) == 0x4) ||
-                (masked_003FF0E0 == 0x00033000) ||
-                (masked_003FF0E0 == 0x003FF020) ||
-                (masked_003FF0E0 == 0x003FF060) ||
-                (masked_003FF0E0 == 0x003FF0E0) ||
-                (instr->Mask(0x00388000) == 0x00008000) ||
-                (instr->Mask(0x0038E000) == 0x00000000) ||
-                (instr->Mask(0x0039E000) == 0x00002000) ||
-                (instr->Mask(0x003AE000) == 0x00002000) ||
-                (instr->Mask(0x003CE000) == 0x00042000) ||
-                (instr->Mask(0x003FFFC0) == 0x000320C0) ||
-                (instr->Mask(0x003FF100) == 0x00032100) ||
-                (instr->Mask(0x003FF200) == 0x00032200) ||
-                (instr->Mask(0x003FF400) == 0x00032400) ||
-                (instr->Mask(0x003FF800) == 0x00032800) ||
-                (instr->Mask(0x0038F000) == 0x00005000) ||
-                (instr->Mask(0x0038E000) == 0x00006000)) {
-              VisitUnallocated(instr);
-            } else {
-              VisitSystem(instr);
-            }
-          } else {
-            VisitUnallocated(instr);
-          }
-        }
-      } else {
-        if ((instr->Bit(24) == 0x1) ||
-            (instr->Bits(20, 16) != 0x1F) ||
-            (instr->Bits(15, 10) != 0) ||
-            (instr->Bits(4, 0) != 0) ||
-            (instr->Bits(24, 21) == 0x3) ||
-            (instr->Bits(24, 22) == 0x3)) {
-          VisitUnallocated(instr);
-        } else {
-          VisitUnconditionalBranchToRegister(instr);
-        }
-      }
-      break;
-    }
-    case 3:
-    case 7: {
-      VisitUnallocated(instr);
-      break;
-    }
-  }
-}
-
-
-void Decoder::DecodeLoadStore(const Instruction* instr) {
-  VIXL_ASSERT((instr->Bits(27, 24) == 0x8) ||
-              (instr->Bits(27, 24) == 0x9) ||
-              (instr->Bits(27, 24) == 0xC) ||
-              (instr->Bits(27, 24) == 0xD) );
-
-  if (instr->Bit(24) == 0) {
-    if (instr->Bit(28) == 0) {
-      if (instr->Bit(29) == 0) {
-        if (instr->Bit(26) == 0) {
-          VisitLoadStoreExclusive(instr);
-        } else {
-          DecodeAdvSIMDLoadStore(instr);
-        }
-      } else {
-        if ((instr->Bits(31, 30) == 0x3) ||
-            (instr->Mask(0xC4400000) == 0x40000000)) {
-          VisitUnallocated(instr);
-        } else {
-          if (instr->Bit(23) == 0) {
-            if (instr->Mask(0xC4400000) == 0xC0400000) {
-              VisitUnallocated(instr);
-            } else {
-              VisitLoadStorePairNonTemporal(instr);
-            }
-          } else {
-            VisitLoadStorePairPostIndex(instr);
-          }
-        }
-      }
-    } else {
-      if (instr->Bit(29) == 0) {
-        if (instr->Mask(0xC4000000) == 0xC4000000) {
-          VisitUnallocated(instr);
-        } else {
-          VisitLoadLiteral(instr);
-        }
-      } else {
-        if ((instr->Mask(0x84C00000) == 0x80C00000) ||
-            (instr->Mask(0x44800000) == 0x44800000) ||
-            (instr->Mask(0x84800000) == 0x84800000)) {
-          VisitUnallocated(instr);
-        } else {
-          if (instr->Bit(21) == 0) {
-            switch (instr->Bits(11, 10)) {
-              case 0: {
-                VisitLoadStoreUnscaledOffset(instr);
-                break;
-              }
-              case 1: {
-                if (instr->Mask(0xC4C00000) == 0xC0800000) {
-                  VisitUnallocated(instr);
-                } else {
-                  VisitLoadStorePostIndex(instr);
-                }
-                break;
-              }
-              case 2: {
-                // TODO: VisitLoadStoreRegisterOffsetUnpriv.
-                VisitUnimplemented(instr);
-                break;
-              }
-              case 3: {
-                if (instr->Mask(0xC4C00000) == 0xC0800000) {
-                  VisitUnallocated(instr);
-                } else {
-                  VisitLoadStorePreIndex(instr);
-                }
-                break;
-              }
-            }
-          } else {
-            if (instr->Bits(11, 10) == 0x2) {
-              if (instr->Bit(14) == 0) {
-                VisitUnallocated(instr);
-              } else {
-                VisitLoadStoreRegisterOffset(instr);
-              }
-            } else {
-              VisitUnallocated(instr);
-            }
-          }
-        }
-      }
-    }
-  } else {
-    if (instr->Bit(28) == 0) {
-      if (instr->Bit(29) == 0) {
-        VisitUnallocated(instr);
-      } else {
-        if ((instr->Bits(31, 30) == 0x3) ||
-            (instr->Mask(0xC4400000) == 0x40000000)) {
-          VisitUnallocated(instr);
-        } else {
-          if (instr->Bit(23) == 0) {
-            VisitLoadStorePairOffset(instr);
-          } else {
-            VisitLoadStorePairPreIndex(instr);
-          }
-        }
-      }
-    } else {
-      if (instr->Bit(29) == 0) {
-        VisitUnallocated(instr);
-      } else {
-        if ((instr->Mask(0x84C00000) == 0x80C00000) ||
-            (instr->Mask(0x44800000) == 0x44800000) ||
-            (instr->Mask(0x84800000) == 0x84800000)) {
-          VisitUnallocated(instr);
-        } else {
-          VisitLoadStoreUnsignedOffset(instr);
-        }
-      }
-    }
-  }
-}
-
-
-void Decoder::DecodeLogical(const Instruction* instr) {
-  VIXL_ASSERT(instr->Bits(27, 24) == 0x2);
-
-  if (instr->Mask(0x80400000) == 0x00400000) {
-    VisitUnallocated(instr);
-  } else {
-    if (instr->Bit(23) == 0) {
-      VisitLogicalImmediate(instr);
-    } else {
-      if (instr->Bits(30, 29) == 0x1) {
-        VisitUnallocated(instr);
-      } else {
-        VisitMoveWideImmediate(instr);
-      }
-    }
-  }
-}
-
-
-void Decoder::DecodeBitfieldExtract(const Instruction* instr) {
-  VIXL_ASSERT(instr->Bits(27, 24) == 0x3);
-
-  if ((instr->Mask(0x80400000) == 0x80000000) ||
-      (instr->Mask(0x80400000) == 0x00400000) ||
-      (instr->Mask(0x80008000) == 0x00008000)) {
-    VisitUnallocated(instr);
-  } else if (instr->Bit(23) == 0) {
-    if ((instr->Mask(0x80200000) == 0x00200000) ||
-        (instr->Mask(0x60000000) == 0x60000000)) {
-      VisitUnallocated(instr);
-    } else {
-      VisitBitfield(instr);
-    }
-  } else {
-    if ((instr->Mask(0x60200000) == 0x00200000) ||
-        (instr->Mask(0x60000000) != 0x00000000)) {
-      VisitUnallocated(instr);
-    } else {
-      VisitExtract(instr);
-    }
-  }
-}
-
-
-void Decoder::DecodeAddSubImmediate(const Instruction* instr) {
-  VIXL_ASSERT(instr->Bits(27, 24) == 0x1);
-  if (instr->Bit(23) == 1) {
-    VisitUnallocated(instr);
-  } else {
-    VisitAddSubImmediate(instr);
-  }
-}
-
-
-void Decoder::DecodeDataProcessing(const Instruction* instr) {
-  VIXL_ASSERT((instr->Bits(27, 24) == 0xA) ||
-              (instr->Bits(27, 24) == 0xB));
-
-  if (instr->Bit(24) == 0) {
-    if (instr->Bit(28) == 0) {
-      if (instr->Mask(0x80008000) == 0x00008000) {
-        VisitUnallocated(instr);
-      } else {
-        VisitLogicalShifted(instr);
-      }
-    } else {
-      switch (instr->Bits(23, 21)) {
-        case 0: {
-          if (instr->Mask(0x0000FC00) != 0) {
-            VisitUnallocated(instr);
-          } else {
-            VisitAddSubWithCarry(instr);
-          }
-          break;
-        }
-        case 2: {
-          if ((instr->Bit(29) == 0) ||
-              (instr->Mask(0x00000410) != 0)) {
-            VisitUnallocated(instr);
-          } else {
-            if (instr->Bit(11) == 0) {
-              VisitConditionalCompareRegister(instr);
-            } else {
-              VisitConditionalCompareImmediate(instr);
-            }
-          }
-          break;
-        }
-        case 4: {
-          if (instr->Mask(0x20000800) != 0x00000000) {
-            VisitUnallocated(instr);
-          } else {
-            VisitConditionalSelect(instr);
-          }
-          break;
-        }
-        case 6: {
-          if (instr->Bit(29) == 0x1) {
-            VisitUnallocated(instr);
-          } else {
-            if (instr->Bit(30) == 0) {
-              if ((instr->Bit(15) == 0x1) ||
-                  (instr->Bits(15, 11) == 0) ||
-                  (instr->Bits(15, 12) == 0x1) ||
-                  (instr->Bits(15, 12) == 0x3) ||
-                  (instr->Bits(15, 13) == 0x3) ||
-                  (instr->Mask(0x8000EC00) == 0x00004C00) ||
-                  (instr->Mask(0x8000E800) == 0x80004000) ||
-                  (instr->Mask(0x8000E400) == 0x80004000)) {
-                VisitUnallocated(instr);
-              } else {
-                VisitDataProcessing2Source(instr);
-              }
-            } else {
-              if ((instr->Bit(13) == 1) ||
-                  (instr->Bits(20, 16) != 0) ||
-                  (instr->Bits(15, 14) != 0) ||
-                  (instr->Mask(0xA01FFC00) == 0x00000C00) ||
-                  (instr->Mask(0x201FF800) == 0x00001800)) {
-                VisitUnallocated(instr);
-              } else {
-                VisitDataProcessing1Source(instr);
-              }
-            }
-            break;
-          }
-        }
-        case 1:
-        case 3:
-        case 5:
-        case 7: VisitUnallocated(instr); break;
-      }
-    }
-  } else {
-    if (instr->Bit(28) == 0) {
-     if (instr->Bit(21) == 0) {
-        if ((instr->Bits(23, 22) == 0x3) ||
-            (instr->Mask(0x80008000) == 0x00008000)) {
-          VisitUnallocated(instr);
-        } else {
-          VisitAddSubShifted(instr);
-        }
-      } else {
-        if ((instr->Mask(0x00C00000) != 0x00000000) ||
-            (instr->Mask(0x00001400) == 0x00001400) ||
-            (instr->Mask(0x00001800) == 0x00001800)) {
-          VisitUnallocated(instr);
-        } else {
-          VisitAddSubExtended(instr);
-        }
-      }
-    } else {
-      if ((instr->Bit(30) == 0x1) ||
-          (instr->Bits(30, 29) == 0x1) ||
-          (instr->Mask(0xE0600000) == 0x00200000) ||
-          (instr->Mask(0xE0608000) == 0x00400000) ||
-          (instr->Mask(0x60608000) == 0x00408000) ||
-          (instr->Mask(0x60E00000) == 0x00E00000) ||
-          (instr->Mask(0x60E00000) == 0x00800000) ||
-          (instr->Mask(0x60E00000) == 0x00600000)) {
-        VisitUnallocated(instr);
-      } else {
-        VisitDataProcessing3Source(instr);
-      }
-    }
-  }
-}
-
-
-void Decoder::DecodeFP(const Instruction* instr) {
-  VIXL_ASSERT((instr->Bits(27, 24) == 0xE) ||
-              (instr->Bits(27, 24) == 0xF));
-
-  if (instr->Bit(28) == 0) {
-    DecodeAdvSIMDDataProcessing(instr);
-  } else {
-    if (instr->Bit(29) == 1) {
-      VisitUnallocated(instr);
-    } else {
-      if (instr->Bits(31, 30) == 0x3) {
-        VisitUnallocated(instr);
-      } else if (instr->Bits(31, 30) == 0x1) {
-        DecodeAdvSIMDDataProcessing(instr);
-      } else {
-        if (instr->Bit(24) == 0) {
-          if (instr->Bit(21) == 0) {
-            if ((instr->Bit(23) == 1) ||
-                (instr->Bit(18) == 1) ||
-                (instr->Mask(0x80008000) == 0x00000000) ||
-                (instr->Mask(0x000E0000) == 0x00000000) ||
-                (instr->Mask(0x000E0000) == 0x000A0000) ||
-                (instr->Mask(0x00160000) == 0x00000000) ||
-                (instr->Mask(0x00160000) == 0x00120000)) {
-              VisitUnallocated(instr);
-            } else {
-              VisitFPFixedPointConvert(instr);
-            }
-          } else {
-            if (instr->Bits(15, 10) == 32) {
-              VisitUnallocated(instr);
-            } else if (instr->Bits(15, 10) == 0) {
-              if ((instr->Bits(23, 22) == 0x3) ||
-                  (instr->Mask(0x000E0000) == 0x000A0000) ||
-                  (instr->Mask(0x000E0000) == 0x000C0000) ||
-                  (instr->Mask(0x00160000) == 0x00120000) ||
-                  (instr->Mask(0x00160000) == 0x00140000) ||
-                  (instr->Mask(0x20C40000) == 0x00800000) ||
-                  (instr->Mask(0x20C60000) == 0x00840000) ||
-                  (instr->Mask(0xA0C60000) == 0x80060000) ||
-                  (instr->Mask(0xA0C60000) == 0x00860000) ||
-                  (instr->Mask(0xA0C60000) == 0x00460000) ||
-                  (instr->Mask(0xA0CE0000) == 0x80860000) ||
-                  (instr->Mask(0xA0CE0000) == 0x804E0000) ||
-                  (instr->Mask(0xA0CE0000) == 0x000E0000) ||
-                  (instr->Mask(0xA0D60000) == 0x00160000) ||
-                  (instr->Mask(0xA0D60000) == 0x80560000) ||
-                  (instr->Mask(0xA0D60000) == 0x80960000)) {
-                VisitUnallocated(instr);
-              } else {
-                VisitFPIntegerConvert(instr);
-              }
-            } else if (instr->Bits(14, 10) == 16) {
-              const Instr masked_A0DF8000 = instr->Mask(0xA0DF8000);
-              if ((instr->Mask(0x80180000) != 0) ||
-                  (masked_A0DF8000 == 0x00020000) ||
-                  (masked_A0DF8000 == 0x00030000) ||
-                  (masked_A0DF8000 == 0x00068000) ||
-                  (masked_A0DF8000 == 0x00428000) ||
-                  (masked_A0DF8000 == 0x00430000) ||
-                  (masked_A0DF8000 == 0x00468000) ||
-                  (instr->Mask(0xA0D80000) == 0x00800000) ||
-                  (instr->Mask(0xA0DE0000) == 0x00C00000) ||
-                  (instr->Mask(0xA0DF0000) == 0x00C30000) ||
-                  (instr->Mask(0xA0DC0000) == 0x00C40000)) {
-                VisitUnallocated(instr);
-              } else {
-                VisitFPDataProcessing1Source(instr);
-              }
-            } else if (instr->Bits(13, 10) == 8) {
-              if ((instr->Bits(15, 14) != 0) ||
-                  (instr->Bits(2, 0) != 0) ||
-                  (instr->Mask(0x80800000) != 0x00000000)) {
-                VisitUnallocated(instr);
-              } else {
-                VisitFPCompare(instr);
-              }
-            } else if (instr->Bits(12, 10) == 4) {
-              if ((instr->Bits(9, 5) != 0) ||
-                  (instr->Mask(0x80800000) != 0x00000000)) {
-                VisitUnallocated(instr);
-              } else {
-                VisitFPImmediate(instr);
-              }
-            } else {
-              if (instr->Mask(0x80800000) != 0x00000000) {
-                VisitUnallocated(instr);
-              } else {
-                switch (instr->Bits(11, 10)) {
-                  case 1: {
-                    VisitFPConditionalCompare(instr);
-                    break;
-                  }
-                  case 2: {
-                    if ((instr->Bits(15, 14) == 0x3) ||
-                        (instr->Mask(0x00009000) == 0x00009000) ||
-                        (instr->Mask(0x0000A000) == 0x0000A000)) {
-                      VisitUnallocated(instr);
-                    } else {
-                      VisitFPDataProcessing2Source(instr);
-                    }
-                    break;
-                  }
-                  case 3: {
-                    VisitFPConditionalSelect(instr);
-                    break;
-                  }
-                  default: VIXL_UNREACHABLE();
-                }
-              }
-            }
-          }
-        } else {
-          // Bit 30 == 1 has been handled earlier.
-          VIXL_ASSERT(instr->Bit(30) == 0);
-          if (instr->Mask(0xA0800000) != 0) {
-            VisitUnallocated(instr);
-          } else {
-            VisitFPDataProcessing3Source(instr);
-          }
-        }
-      }
-    }
-  }
-}
-
-
-void Decoder::DecodeAdvSIMDLoadStore(const Instruction* instr) {
-  // TODO: Implement Advanced SIMD load/store instruction decode.
-  VIXL_ASSERT(instr->Bits(29, 25) == 0x6);
-  VisitUnimplemented(instr);
-}
-
-
-void Decoder::DecodeAdvSIMDDataProcessing(const Instruction* instr) {
-  // TODO: Implement Advanced SIMD data processing instruction decode.
-  VIXL_ASSERT(instr->Bits(27, 25) == 0x7);
-  VisitUnimplemented(instr);
-}
-
-
-#define DEFINE_VISITOR_CALLERS(A)                                              \
-  void Decoder::Visit##A(const Instruction *instr) {                           \
-    VIXL_ASSERT(instr->Mask(A##FMask) == A##Fixed);                            \
-    std::list<DecoderVisitor*>::iterator it;                                   \
-    for (it = visitors_.begin(); it != visitors_.end(); it++) {                \
-      (*it)->Visit##A(instr);                                                  \
-    }                                                                          \
-  }
-VISITOR_LIST(DEFINE_VISITOR_CALLERS)
-#undef DEFINE_VISITOR_CALLERS
-}  // namespace vixl
diff --git a/disas/libvixl/a64/decoder-a64.h b/disas/libvixl/a64/decoder-a64.h
deleted file mode 100644
index 172594c..0000000
--- a/disas/libvixl/a64/decoder-a64.h
+++ /dev/null
@@ -1,239 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_DECODER_A64_H_
-#define VIXL_A64_DECODER_A64_H_
-
-#include <list>
-
-#include "globals.h"
-#include "a64/instructions-a64.h"
-
-
-// List macro containing all visitors needed by the decoder class.
-
-#define VISITOR_LIST(V)             \
-  V(PCRelAddressing)                \
-  V(AddSubImmediate)                \
-  V(LogicalImmediate)               \
-  V(MoveWideImmediate)              \
-  V(Bitfield)                       \
-  V(Extract)                        \
-  V(UnconditionalBranch)            \
-  V(UnconditionalBranchToRegister)  \
-  V(CompareBranch)                  \
-  V(TestBranch)                     \
-  V(ConditionalBranch)              \
-  V(System)                         \
-  V(Exception)                      \
-  V(LoadStorePairPostIndex)         \
-  V(LoadStorePairOffset)            \
-  V(LoadStorePairPreIndex)          \
-  V(LoadStorePairNonTemporal)       \
-  V(LoadLiteral)                    \
-  V(LoadStoreUnscaledOffset)        \
-  V(LoadStorePostIndex)             \
-  V(LoadStorePreIndex)              \
-  V(LoadStoreRegisterOffset)        \
-  V(LoadStoreUnsignedOffset)        \
-  V(LoadStoreExclusive)             \
-  V(LogicalShifted)                 \
-  V(AddSubShifted)                  \
-  V(AddSubExtended)                 \
-  V(AddSubWithCarry)                \
-  V(ConditionalCompareRegister)     \
-  V(ConditionalCompareImmediate)    \
-  V(ConditionalSelect)              \
-  V(DataProcessing1Source)          \
-  V(DataProcessing2Source)          \
-  V(DataProcessing3Source)          \
-  V(FPCompare)                      \
-  V(FPConditionalCompare)           \
-  V(FPConditionalSelect)            \
-  V(FPImmediate)                    \
-  V(FPDataProcessing1Source)        \
-  V(FPDataProcessing2Source)        \
-  V(FPDataProcessing3Source)        \
-  V(FPIntegerConvert)               \
-  V(FPFixedPointConvert)            \
-  V(Unallocated)                    \
-  V(Unimplemented)
-
-namespace vixl {
-
-// The Visitor interface. Disassembler and simulator (and other tools)
-// must provide implementations for all of these functions.
-class DecoderVisitor {
- public:
-  enum VisitorConstness {
-    kConstVisitor,
-    kNonConstVisitor
-  };
-  explicit DecoderVisitor(VisitorConstness constness = kConstVisitor)
-      : constness_(constness) {}
-
-  virtual ~DecoderVisitor() {}
-
-  #define DECLARE(A) virtual void Visit##A(const Instruction* instr) = 0;
-  VISITOR_LIST(DECLARE)
-  #undef DECLARE
-
-  bool IsConstVisitor() const { return constness_ == kConstVisitor; }
-  Instruction* MutableInstruction(const Instruction* instr) {
-    VIXL_ASSERT(!IsConstVisitor());
-    return const_cast<Instruction*>(instr);
-  }
-
- private:
-  VisitorConstness constness_;
-};
-
-
-class Decoder {
- public:
-  Decoder() {}
-
-  // Top-level wrappers around the actual decoding function.
-  void Decode(const Instruction* instr) {
-    std::list<DecoderVisitor*>::iterator it;
-    for (it = visitors_.begin(); it != visitors_.end(); it++) {
-      VIXL_ASSERT((*it)->IsConstVisitor());
-    }
-    DecodeInstruction(instr);
-  }
-  void Decode(Instruction* instr) {
-    DecodeInstruction(const_cast<const Instruction*>(instr));
-  }
-
-  // Register a new visitor class with the decoder.
-  // Decode() will call the corresponding visitor method from all registered
-  // visitor classes when decoding reaches the leaf node of the instruction
-  // decode tree.
-  // Visitors are called in order.
-  // A visitor can be registered multiple times.
-  //
-  //   d.AppendVisitor(V1);
-  //   d.AppendVisitor(V2);
-  //   d.PrependVisitor(V2);
-  //   d.AppendVisitor(V3);
-  //
-  //   d.Decode(i);
-  //
-  // will call in order visitor methods in V2, V1, V2, V3.
-  void AppendVisitor(DecoderVisitor* visitor);
-  void PrependVisitor(DecoderVisitor* visitor);
-  // These helpers register `new_visitor` before or after the first instance of
-  // `registered_visiter` in the list.
-  // So if
-  //   V1, V2, V1, V2
-  // are registered in this order in the decoder, calls to
-  //   d.InsertVisitorAfter(V3, V1);
-  //   d.InsertVisitorBefore(V4, V2);
-  // will yield the order
-  //   V1, V3, V4, V2, V1, V2
-  //
-  // For more complex modifications of the order of registered visitors, one can
-  // directly access and modify the list of visitors via the `visitors()'
-  // accessor.
-  void InsertVisitorBefore(DecoderVisitor* new_visitor,
-                           DecoderVisitor* registered_visitor);
-  void InsertVisitorAfter(DecoderVisitor* new_visitor,
-                          DecoderVisitor* registered_visitor);
-
-  // Remove all instances of a previously registered visitor class from the list
-  // of visitors stored by the decoder.
-  void RemoveVisitor(DecoderVisitor* visitor);
-
-  #define DECLARE(A) void Visit##A(const Instruction* instr);
-  VISITOR_LIST(DECLARE)
-  #undef DECLARE
-
-
-  std::list<DecoderVisitor*>* visitors() { return &visitors_; }
-
- private:
-  // Decodes an instruction and calls the visitor functions registered with the
-  // Decoder class.
-  void DecodeInstruction(const Instruction* instr);
-
-  // Decode the PC relative addressing instruction, and call the corresponding
-  // visitors.
-  // On entry, instruction bits 27:24 = 0x0.
-  void DecodePCRelAddressing(const Instruction* instr);
-
-  // Decode the add/subtract immediate instruction, and call the correspoding
-  // visitors.
-  // On entry, instruction bits 27:24 = 0x1.
-  void DecodeAddSubImmediate(const Instruction* instr);
-
-  // Decode the branch, system command, and exception generation parts of
-  // the instruction tree, and call the corresponding visitors.
-  // On entry, instruction bits 27:24 = {0x4, 0x5, 0x6, 0x7}.
-  void DecodeBranchSystemException(const Instruction* instr);
-
-  // Decode the load and store parts of the instruction tree, and call
-  // the corresponding visitors.
-  // On entry, instruction bits 27:24 = {0x8, 0x9, 0xC, 0xD}.
-  void DecodeLoadStore(const Instruction* instr);
-
-  // Decode the logical immediate and move wide immediate parts of the
-  // instruction tree, and call the corresponding visitors.
-  // On entry, instruction bits 27:24 = 0x2.
-  void DecodeLogical(const Instruction* instr);
-
-  // Decode the bitfield and extraction parts of the instruction tree,
-  // and call the corresponding visitors.
-  // On entry, instruction bits 27:24 = 0x3.
-  void DecodeBitfieldExtract(const Instruction* instr);
-
-  // Decode the data processing parts of the instruction tree, and call the
-  // corresponding visitors.
-  // On entry, instruction bits 27:24 = {0x1, 0xA, 0xB}.
-  void DecodeDataProcessing(const Instruction* instr);
-
-  // Decode the floating point parts of the instruction tree, and call the
-  // corresponding visitors.
-  // On entry, instruction bits 27:24 = {0xE, 0xF}.
-  void DecodeFP(const Instruction* instr);
-
-  // Decode the Advanced SIMD (NEON) load/store part of the instruction tree,
-  // and call the corresponding visitors.
-  // On entry, instruction bits 29:25 = 0x6.
-  void DecodeAdvSIMDLoadStore(const Instruction* instr);
-
-  // Decode the Advanced SIMD (NEON) data processing part of the instruction
-  // tree, and call the corresponding visitors.
-  // On entry, instruction bits 27:25 = 0x7.
-  void DecodeAdvSIMDDataProcessing(const Instruction* instr);
-
- private:
-  // Visitors are registered in a list.
-  std::list<DecoderVisitor*> visitors_;
-};
-
-}  // namespace vixl
-
-#endif  // VIXL_A64_DECODER_A64_H_
diff --git a/disas/libvixl/a64/disasm-a64.cc b/disas/libvixl/a64/disasm-a64.cc
deleted file mode 100644
index 9df2f17..0000000
--- a/disas/libvixl/a64/disasm-a64.cc
+++ /dev/null
@@ -1,1882 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include <cstdlib>
-#include "a64/disasm-a64.h"
-
-#if defined(_WIN32)
-// Redefine PRI<f>64 values to avoid compiler warnings like
-// 'unknown conversion type character 'l' in format'. See:
-// http://stackoverflow.com/questions/10763854/printf-and-llx-in-gcc-under-windows-64x
-#undef PRIx64
-#undef PRIu64
-#undef PRId64
-#define PRIx64 "I64x"
-#define PRIu64 "I64u"
-#define PRId64 "I64d"
-#endif
-
-
-namespace vixl {
-
-Disassembler::Disassembler() {
-  buffer_size_ = 256;
-  buffer_ = reinterpret_cast<char*>(malloc(buffer_size_));
-  buffer_pos_ = 0;
-  own_buffer_ = true;
-}
-
-
-Disassembler::Disassembler(char* text_buffer, int buffer_size) {
-  buffer_size_ = buffer_size;
-  buffer_ = text_buffer;
-  buffer_pos_ = 0;
-  own_buffer_ = false;
-}
-
-
-Disassembler::~Disassembler() {
-  if (own_buffer_) {
-    free(buffer_);
-  }
-}
-
-
-char* Disassembler::GetOutput() {
-  return buffer_;
-}
-
-
-void Disassembler::VisitAddSubImmediate(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool stack_op = (rd_is_zr || RnIsZROrSP(instr)) &&
-                  (instr->ImmAddSub() == 0) ? true : false;
-  const char *mnemonic = "";
-  const char *form = "'Rds, 'Rns, 'IAddSub";
-  const char *form_cmp = "'Rns, 'IAddSub";
-  const char *form_mov = "'Rds, 'Rns";
-
-  switch (instr->Mask(AddSubImmediateMask)) {
-    case ADD_w_imm:
-    case ADD_x_imm: {
-      mnemonic = "add";
-      if (stack_op) {
-        mnemonic = "mov";
-        form = form_mov;
-      }
-      break;
-    }
-    case ADDS_w_imm:
-    case ADDS_x_imm: {
-      mnemonic = "adds";
-      if (rd_is_zr) {
-        mnemonic = "cmn";
-        form = form_cmp;
-      }
-      break;
-    }
-    case SUB_w_imm:
-    case SUB_x_imm: mnemonic = "sub"; break;
-    case SUBS_w_imm:
-    case SUBS_x_imm: {
-      mnemonic = "subs";
-      if (rd_is_zr) {
-        mnemonic = "cmp";
-        form = form_cmp;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitAddSubShifted(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm'HDP";
-  const char *form_cmp = "'Rn, 'Rm'HDP";
-  const char *form_neg = "'Rd, 'Rm'HDP";
-
-  switch (instr->Mask(AddSubShiftedMask)) {
-    case ADD_w_shift:
-    case ADD_x_shift: mnemonic = "add"; break;
-    case ADDS_w_shift:
-    case ADDS_x_shift: {
-      mnemonic = "adds";
-      if (rd_is_zr) {
-        mnemonic = "cmn";
-        form = form_cmp;
-      }
-      break;
-    }
-    case SUB_w_shift:
-    case SUB_x_shift: {
-      mnemonic = "sub";
-      if (rn_is_zr) {
-        mnemonic = "neg";
-        form = form_neg;
-      }
-      break;
-    }
-    case SUBS_w_shift:
-    case SUBS_x_shift: {
-      mnemonic = "subs";
-      if (rd_is_zr) {
-        mnemonic = "cmp";
-        form = form_cmp;
-      } else if (rn_is_zr) {
-        mnemonic = "negs";
-        form = form_neg;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitAddSubExtended(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  const char *mnemonic = "";
-  Extend mode = static_cast<Extend>(instr->ExtendMode());
-  const char *form = ((mode == UXTX) || (mode == SXTX)) ?
-                     "'Rds, 'Rns, 'Xm'Ext" : "'Rds, 'Rns, 'Wm'Ext";
-  const char *form_cmp = ((mode == UXTX) || (mode == SXTX)) ?
-                         "'Rns, 'Xm'Ext" : "'Rns, 'Wm'Ext";
-
-  switch (instr->Mask(AddSubExtendedMask)) {
-    case ADD_w_ext:
-    case ADD_x_ext: mnemonic = "add"; break;
-    case ADDS_w_ext:
-    case ADDS_x_ext: {
-      mnemonic = "adds";
-      if (rd_is_zr) {
-        mnemonic = "cmn";
-        form = form_cmp;
-      }
-      break;
-    }
-    case SUB_w_ext:
-    case SUB_x_ext: mnemonic = "sub"; break;
-    case SUBS_w_ext:
-    case SUBS_x_ext: {
-      mnemonic = "subs";
-      if (rd_is_zr) {
-        mnemonic = "cmp";
-        form = form_cmp;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitAddSubWithCarry(const Instruction* instr) {
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm";
-  const char *form_neg = "'Rd, 'Rm";
-
-  switch (instr->Mask(AddSubWithCarryMask)) {
-    case ADC_w:
-    case ADC_x: mnemonic = "adc"; break;
-    case ADCS_w:
-    case ADCS_x: mnemonic = "adcs"; break;
-    case SBC_w:
-    case SBC_x: {
-      mnemonic = "sbc";
-      if (rn_is_zr) {
-        mnemonic = "ngc";
-        form = form_neg;
-      }
-      break;
-    }
-    case SBCS_w:
-    case SBCS_x: {
-      mnemonic = "sbcs";
-      if (rn_is_zr) {
-        mnemonic = "ngcs";
-        form = form_neg;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLogicalImmediate(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rds, 'Rn, 'ITri";
-
-  if (instr->ImmLogical() == 0) {
-    // The immediate encoded in the instruction is not in the expected format.
-    Format(instr, "unallocated", "(LogicalImmediate)");
-    return;
-  }
-
-  switch (instr->Mask(LogicalImmediateMask)) {
-    case AND_w_imm:
-    case AND_x_imm: mnemonic = "and"; break;
-    case ORR_w_imm:
-    case ORR_x_imm: {
-      mnemonic = "orr";
-      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize
-                                                        : kWRegSize;
-      if (rn_is_zr && !IsMovzMovnImm(reg_size, instr->ImmLogical())) {
-        mnemonic = "mov";
-        form = "'Rds, 'ITri";
-      }
-      break;
-    }
-    case EOR_w_imm:
-    case EOR_x_imm: mnemonic = "eor"; break;
-    case ANDS_w_imm:
-    case ANDS_x_imm: {
-      mnemonic = "ands";
-      if (rd_is_zr) {
-        mnemonic = "tst";
-        form = "'Rn, 'ITri";
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-bool Disassembler::IsMovzMovnImm(unsigned reg_size, uint64_t value) {
-  VIXL_ASSERT((reg_size == kXRegSize) ||
-              ((reg_size == kWRegSize) && (value <= 0xffffffff)));
-
-  // Test for movz: 16 bits set at positions 0, 16, 32 or 48.
-  if (((value & UINT64_C(0xffffffffffff0000)) == 0) ||
-      ((value & UINT64_C(0xffffffff0000ffff)) == 0) ||
-      ((value & UINT64_C(0xffff0000ffffffff)) == 0) ||
-      ((value & UINT64_C(0x0000ffffffffffff)) == 0)) {
-    return true;
-  }
-
-  // Test for movn: NOT(16 bits set at positions 0, 16, 32 or 48).
-  if ((reg_size == kXRegSize) &&
-      (((~value & UINT64_C(0xffffffffffff0000)) == 0) ||
-       ((~value & UINT64_C(0xffffffff0000ffff)) == 0) ||
-       ((~value & UINT64_C(0xffff0000ffffffff)) == 0) ||
-       ((~value & UINT64_C(0x0000ffffffffffff)) == 0))) {
-    return true;
-  }
-  if ((reg_size == kWRegSize) &&
-      (((value & 0xffff0000) == 0xffff0000) ||
-       ((value & 0x0000ffff) == 0x0000ffff))) {
-    return true;
-  }
-  return false;
-}
-
-
-void Disassembler::VisitLogicalShifted(const Instruction* instr) {
-  bool rd_is_zr = RdIsZROrSP(instr);
-  bool rn_is_zr = RnIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm'HLo";
-
-  switch (instr->Mask(LogicalShiftedMask)) {
-    case AND_w:
-    case AND_x: mnemonic = "and"; break;
-    case BIC_w:
-    case BIC_x: mnemonic = "bic"; break;
-    case EOR_w:
-    case EOR_x: mnemonic = "eor"; break;
-    case EON_w:
-    case EON_x: mnemonic = "eon"; break;
-    case BICS_w:
-    case BICS_x: mnemonic = "bics"; break;
-    case ANDS_w:
-    case ANDS_x: {
-      mnemonic = "ands";
-      if (rd_is_zr) {
-        mnemonic = "tst";
-        form = "'Rn, 'Rm'HLo";
-      }
-      break;
-    }
-    case ORR_w:
-    case ORR_x: {
-      mnemonic = "orr";
-      if (rn_is_zr && (instr->ImmDPShift() == 0) && (instr->ShiftDP() == LSL)) {
-        mnemonic = "mov";
-        form = "'Rd, 'Rm";
-      }
-      break;
-    }
-    case ORN_w:
-    case ORN_x: {
-      mnemonic = "orn";
-      if (rn_is_zr) {
-        mnemonic = "mvn";
-        form = "'Rd, 'Rm'HLo";
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitConditionalCompareRegister(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rn, 'Rm, 'INzcv, 'Cond";
-
-  switch (instr->Mask(ConditionalCompareRegisterMask)) {
-    case CCMN_w:
-    case CCMN_x: mnemonic = "ccmn"; break;
-    case CCMP_w:
-    case CCMP_x: mnemonic = "ccmp"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitConditionalCompareImmediate(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rn, 'IP, 'INzcv, 'Cond";
-
-  switch (instr->Mask(ConditionalCompareImmediateMask)) {
-    case CCMN_w_imm:
-    case CCMN_x_imm: mnemonic = "ccmn"; break;
-    case CCMP_w_imm:
-    case CCMP_x_imm: mnemonic = "ccmp"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitConditionalSelect(const Instruction* instr) {
-  bool rnm_is_zr = (RnIsZROrSP(instr) && RmIsZROrSP(instr));
-  bool rn_is_rm = (instr->Rn() == instr->Rm());
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm, 'Cond";
-  const char *form_test = "'Rd, 'CInv";
-  const char *form_update = "'Rd, 'Rn, 'CInv";
-
-  Condition cond = static_cast<Condition>(instr->Condition());
-  bool invertible_cond = (cond != al) && (cond != nv);
-
-  switch (instr->Mask(ConditionalSelectMask)) {
-    case CSEL_w:
-    case CSEL_x: mnemonic = "csel"; break;
-    case CSINC_w:
-    case CSINC_x: {
-      mnemonic = "csinc";
-      if (rnm_is_zr && invertible_cond) {
-        mnemonic = "cset";
-        form = form_test;
-      } else if (rn_is_rm && invertible_cond) {
-        mnemonic = "cinc";
-        form = form_update;
-      }
-      break;
-    }
-    case CSINV_w:
-    case CSINV_x: {
-      mnemonic = "csinv";
-      if (rnm_is_zr && invertible_cond) {
-        mnemonic = "csetm";
-        form = form_test;
-      } else if (rn_is_rm && invertible_cond) {
-        mnemonic = "cinv";
-        form = form_update;
-      }
-      break;
-    }
-    case CSNEG_w:
-    case CSNEG_x: {
-      mnemonic = "csneg";
-      if (rn_is_rm && invertible_cond) {
-        mnemonic = "cneg";
-        form = form_update;
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitBitfield(const Instruction* instr) {
-  unsigned s = instr->ImmS();
-  unsigned r = instr->ImmR();
-  unsigned rd_size_minus_1 =
-    ((instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize) - 1;
-  const char *mnemonic = "";
-  const char *form = "";
-  const char *form_shift_right = "'Rd, 'Rn, 'IBr";
-  const char *form_extend = "'Rd, 'Wn";
-  const char *form_bfiz = "'Rd, 'Rn, 'IBZ-r, 'IBs+1";
-  const char *form_bfx = "'Rd, 'Rn, 'IBr, 'IBs-r+1";
-  const char *form_lsl = "'Rd, 'Rn, 'IBZ-r";
-
-  switch (instr->Mask(BitfieldMask)) {
-    case SBFM_w:
-    case SBFM_x: {
-      mnemonic = "sbfx";
-      form = form_bfx;
-      if (r == 0) {
-        form = form_extend;
-        if (s == 7) {
-          mnemonic = "sxtb";
-        } else if (s == 15) {
-          mnemonic = "sxth";
-        } else if ((s == 31) && (instr->SixtyFourBits() == 1)) {
-          mnemonic = "sxtw";
-        } else {
-          form = form_bfx;
-        }
-      } else if (s == rd_size_minus_1) {
-        mnemonic = "asr";
-        form = form_shift_right;
-      } else if (s < r) {
-        mnemonic = "sbfiz";
-        form = form_bfiz;
-      }
-      break;
-    }
-    case UBFM_w:
-    case UBFM_x: {
-      mnemonic = "ubfx";
-      form = form_bfx;
-      if (r == 0) {
-        form = form_extend;
-        if (s == 7) {
-          mnemonic = "uxtb";
-        } else if (s == 15) {
-          mnemonic = "uxth";
-        } else {
-          form = form_bfx;
-        }
-      }
-      if (s == rd_size_minus_1) {
-        mnemonic = "lsr";
-        form = form_shift_right;
-      } else if (r == s + 1) {
-        mnemonic = "lsl";
-        form = form_lsl;
-      } else if (s < r) {
-        mnemonic = "ubfiz";
-        form = form_bfiz;
-      }
-      break;
-    }
-    case BFM_w:
-    case BFM_x: {
-      mnemonic = "bfxil";
-      form = form_bfx;
-      if (s < r) {
-        mnemonic = "bfi";
-        form = form_bfiz;
-      }
-    }
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitExtract(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn, 'Rm, 'IExtract";
-
-  switch (instr->Mask(ExtractMask)) {
-    case EXTR_w:
-    case EXTR_x: {
-      if (instr->Rn() == instr->Rm()) {
-        mnemonic = "ror";
-        form = "'Rd, 'Rn, 'IExtract";
-      } else {
-        mnemonic = "extr";
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitPCRelAddressing(const Instruction* instr) {
-  switch (instr->Mask(PCRelAddressingMask)) {
-    case ADR: Format(instr, "adr", "'Xd, 'AddrPCRelByte"); break;
-    case ADRP: Format(instr, "adrp", "'Xd, 'AddrPCRelPage"); break;
-    default: Format(instr, "unimplemented", "(PCRelAddressing)");
-  }
-}
-
-
-void Disassembler::VisitConditionalBranch(const Instruction* instr) {
-  switch (instr->Mask(ConditionalBranchMask)) {
-    case B_cond: Format(instr, "b.'CBrn", "'BImmCond"); break;
-    default: VIXL_UNREACHABLE();
-  }
-}
-
-
-void Disassembler::VisitUnconditionalBranchToRegister(
-    const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Xn";
-
-  switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
-    case BR: mnemonic = "br"; break;
-    case BLR: mnemonic = "blr"; break;
-    case RET: {
-      mnemonic = "ret";
-      if (instr->Rn() == kLinkRegCode) {
-        form = NULL;
-      }
-      break;
-    }
-    default: form = "(UnconditionalBranchToRegister)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitUnconditionalBranch(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'BImmUncn";
-
-  switch (instr->Mask(UnconditionalBranchMask)) {
-    case B: mnemonic = "b"; break;
-    case BL: mnemonic = "bl"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitDataProcessing1Source(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Rn";
-
-  switch (instr->Mask(DataProcessing1SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_w:           \
-    case A##_x: mnemonic = B; break;
-    FORMAT(RBIT, "rbit");
-    FORMAT(REV16, "rev16");
-    FORMAT(REV, "rev");
-    FORMAT(CLZ, "clz");
-    FORMAT(CLS, "cls");
-    #undef FORMAT
-    case REV32_x: mnemonic = "rev32"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitDataProcessing2Source(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Rd, 'Rn, 'Rm";
-
-  switch (instr->Mask(DataProcessing2SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_w:           \
-    case A##_x: mnemonic = B; break;
-    FORMAT(UDIV, "udiv");
-    FORMAT(SDIV, "sdiv");
-    FORMAT(LSLV, "lsl");
-    FORMAT(LSRV, "lsr");
-    FORMAT(ASRV, "asr");
-    FORMAT(RORV, "ror");
-    #undef FORMAT
-    default: form = "(DataProcessing2Source)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitDataProcessing3Source(const Instruction* instr) {
-  bool ra_is_zr = RaIsZROrSP(instr);
-  const char *mnemonic = "";
-  const char *form = "'Xd, 'Wn, 'Wm, 'Xa";
-  const char *form_rrr = "'Rd, 'Rn, 'Rm";
-  const char *form_rrrr = "'Rd, 'Rn, 'Rm, 'Ra";
-  const char *form_xww = "'Xd, 'Wn, 'Wm";
-  const char *form_xxx = "'Xd, 'Xn, 'Xm";
-
-  switch (instr->Mask(DataProcessing3SourceMask)) {
-    case MADD_w:
-    case MADD_x: {
-      mnemonic = "madd";
-      form = form_rrrr;
-      if (ra_is_zr) {
-        mnemonic = "mul";
-        form = form_rrr;
-      }
-      break;
-    }
-    case MSUB_w:
-    case MSUB_x: {
-      mnemonic = "msub";
-      form = form_rrrr;
-      if (ra_is_zr) {
-        mnemonic = "mneg";
-        form = form_rrr;
-      }
-      break;
-    }
-    case SMADDL_x: {
-      mnemonic = "smaddl";
-      if (ra_is_zr) {
-        mnemonic = "smull";
-        form = form_xww;
-      }
-      break;
-    }
-    case SMSUBL_x: {
-      mnemonic = "smsubl";
-      if (ra_is_zr) {
-        mnemonic = "smnegl";
-        form = form_xww;
-      }
-      break;
-    }
-    case UMADDL_x: {
-      mnemonic = "umaddl";
-      if (ra_is_zr) {
-        mnemonic = "umull";
-        form = form_xww;
-      }
-      break;
-    }
-    case UMSUBL_x: {
-      mnemonic = "umsubl";
-      if (ra_is_zr) {
-        mnemonic = "umnegl";
-        form = form_xww;
-      }
-      break;
-    }
-    case SMULH_x: {
-      mnemonic = "smulh";
-      form = form_xxx;
-      break;
-    }
-    case UMULH_x: {
-      mnemonic = "umulh";
-      form = form_xxx;
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitCompareBranch(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rt, 'BImmCmpa";
-
-  switch (instr->Mask(CompareBranchMask)) {
-    case CBZ_w:
-    case CBZ_x: mnemonic = "cbz"; break;
-    case CBNZ_w:
-    case CBNZ_x: mnemonic = "cbnz"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitTestBranch(const Instruction* instr) {
-  const char *mnemonic = "";
-  // If the top bit of the immediate is clear, the tested register is
-  // disassembled as Wt, otherwise Xt. As the top bit of the immediate is
-  // encoded in bit 31 of the instruction, we can reuse the Rt form, which
-  // uses bit 31 (normally "sf") to choose the register size.
-  const char *form = "'Rt, 'IS, 'BImmTest";
-
-  switch (instr->Mask(TestBranchMask)) {
-    case TBZ: mnemonic = "tbz"; break;
-    case TBNZ: mnemonic = "tbnz"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitMoveWideImmediate(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'IMoveImm";
-
-  // Print the shift separately for movk, to make it clear which half word will
-  // be overwritten. Movn and movz print the computed immediate, which includes
-  // shift calculation.
-  switch (instr->Mask(MoveWideImmediateMask)) {
-    case MOVN_w:
-    case MOVN_x: mnemonic = "movn"; break;
-    case MOVZ_w:
-    case MOVZ_x: mnemonic = "movz"; break;
-    case MOVK_w:
-    case MOVK_x: mnemonic = "movk"; form = "'Rd, 'IMoveLSL"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-#define LOAD_STORE_LIST(V)    \
-  V(STRB_w, "strb", "'Wt")    \
-  V(STRH_w, "strh", "'Wt")    \
-  V(STR_w, "str", "'Wt")      \
-  V(STR_x, "str", "'Xt")      \
-  V(LDRB_w, "ldrb", "'Wt")    \
-  V(LDRH_w, "ldrh", "'Wt")    \
-  V(LDR_w, "ldr", "'Wt")      \
-  V(LDR_x, "ldr", "'Xt")      \
-  V(LDRSB_x, "ldrsb", "'Xt")  \
-  V(LDRSH_x, "ldrsh", "'Xt")  \
-  V(LDRSW_x, "ldrsw", "'Xt")  \
-  V(LDRSB_w, "ldrsb", "'Wt")  \
-  V(LDRSH_w, "ldrsh", "'Wt")  \
-  V(STR_s, "str", "'St")      \
-  V(STR_d, "str", "'Dt")      \
-  V(LDR_s, "ldr", "'St")      \
-  V(LDR_d, "ldr", "'Dt")
-
-void Disassembler::VisitLoadStorePreIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePreIndex)";
-
-  switch (instr->Mask(LoadStorePreIndexMask)) {
-    #define LS_PREINDEX(A, B, C) \
-    case A##_pre: mnemonic = B; form = C ", ['Xns'ILS]!"; break;
-    LOAD_STORE_LIST(LS_PREINDEX)
-    #undef LS_PREINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePostIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePostIndex)";
-
-  switch (instr->Mask(LoadStorePostIndexMask)) {
-    #define LS_POSTINDEX(A, B, C) \
-    case A##_post: mnemonic = B; form = C ", ['Xns]'ILS"; break;
-    LOAD_STORE_LIST(LS_POSTINDEX)
-    #undef LS_POSTINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStoreUnsignedOffset)";
-
-  switch (instr->Mask(LoadStoreUnsignedOffsetMask)) {
-    #define LS_UNSIGNEDOFFSET(A, B, C) \
-    case A##_unsigned: mnemonic = B; form = C ", ['Xns'ILU]"; break;
-    LOAD_STORE_LIST(LS_UNSIGNEDOFFSET)
-    #undef LS_UNSIGNEDOFFSET
-    case PRFM_unsigned: mnemonic = "prfm"; form = "'PrefOp, ['Xn'ILU]";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreRegisterOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStoreRegisterOffset)";
-
-  switch (instr->Mask(LoadStoreRegisterOffsetMask)) {
-    #define LS_REGISTEROFFSET(A, B, C) \
-    case A##_reg: mnemonic = B; form = C ", ['Xns, 'Offsetreg]"; break;
-    LOAD_STORE_LIST(LS_REGISTEROFFSET)
-    #undef LS_REGISTEROFFSET
-    case PRFM_reg: mnemonic = "prfm"; form = "'PrefOp, ['Xns, 'Offsetreg]";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Wt, ['Xns'ILS]";
-  const char *form_x = "'Xt, ['Xns'ILS]";
-  const char *form_s = "'St, ['Xns'ILS]";
-  const char *form_d = "'Dt, ['Xns'ILS]";
-
-  switch (instr->Mask(LoadStoreUnscaledOffsetMask)) {
-    case STURB_w:  mnemonic = "sturb"; break;
-    case STURH_w:  mnemonic = "sturh"; break;
-    case STUR_w:   mnemonic = "stur"; break;
-    case STUR_x:   mnemonic = "stur"; form = form_x; break;
-    case STUR_s:   mnemonic = "stur"; form = form_s; break;
-    case STUR_d:   mnemonic = "stur"; form = form_d; break;
-    case LDURB_w:  mnemonic = "ldurb"; break;
-    case LDURH_w:  mnemonic = "ldurh"; break;
-    case LDUR_w:   mnemonic = "ldur"; break;
-    case LDUR_x:   mnemonic = "ldur"; form = form_x; break;
-    case LDUR_s:   mnemonic = "ldur"; form = form_s; break;
-    case LDUR_d:   mnemonic = "ldur"; form = form_d; break;
-    case LDURSB_x: form = form_x;  // Fall through.
-    case LDURSB_w: mnemonic = "ldursb"; break;
-    case LDURSH_x: form = form_x;  // Fall through.
-    case LDURSH_w: mnemonic = "ldursh"; break;
-    case LDURSW_x: mnemonic = "ldursw"; form = form_x; break;
-    default: form = "(LoadStoreUnscaledOffset)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadLiteral(const Instruction* instr) {
-  const char *mnemonic = "ldr";
-  const char *form = "(LoadLiteral)";
-
-  switch (instr->Mask(LoadLiteralMask)) {
-    case LDR_w_lit: form = "'Wt, 'ILLiteral 'LValue"; break;
-    case LDR_x_lit: form = "'Xt, 'ILLiteral 'LValue"; break;
-    case LDR_s_lit: form = "'St, 'ILLiteral 'LValue"; break;
-    case LDR_d_lit: form = "'Dt, 'ILLiteral 'LValue"; break;
-    case LDRSW_x_lit: {
-      mnemonic = "ldrsw";
-      form = "'Xt, 'ILLiteral 'LValue";
-      break;
-    }
-    default: mnemonic = "unimplemented";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-#define LOAD_STORE_PAIR_LIST(V)         \
-  V(STP_w, "stp", "'Wt, 'Wt2", "4")     \
-  V(LDP_w, "ldp", "'Wt, 'Wt2", "4")     \
-  V(LDPSW_x, "ldpsw", "'Xt, 'Xt2", "4") \
-  V(STP_x, "stp", "'Xt, 'Xt2", "8")     \
-  V(LDP_x, "ldp", "'Xt, 'Xt2", "8")     \
-  V(STP_s, "stp", "'St, 'St2", "4")     \
-  V(LDP_s, "ldp", "'St, 'St2", "4")     \
-  V(STP_d, "stp", "'Dt, 'Dt2", "8")     \
-  V(LDP_d, "ldp", "'Dt, 'Dt2", "8")
-
-void Disassembler::VisitLoadStorePairPostIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePairPostIndex)";
-
-  switch (instr->Mask(LoadStorePairPostIndexMask)) {
-    #define LSP_POSTINDEX(A, B, C, D) \
-    case A##_post: mnemonic = B; form = C ", ['Xns]'ILP" D; break;
-    LOAD_STORE_PAIR_LIST(LSP_POSTINDEX)
-    #undef LSP_POSTINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePairPreIndex(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePairPreIndex)";
-
-  switch (instr->Mask(LoadStorePairPreIndexMask)) {
-    #define LSP_PREINDEX(A, B, C, D) \
-    case A##_pre: mnemonic = B; form = C ", ['Xns'ILP" D "]!"; break;
-    LOAD_STORE_PAIR_LIST(LSP_PREINDEX)
-    #undef LSP_PREINDEX
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePairOffset(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(LoadStorePairOffset)";
-
-  switch (instr->Mask(LoadStorePairOffsetMask)) {
-    #define LSP_OFFSET(A, B, C, D) \
-    case A##_off: mnemonic = B; form = C ", ['Xns'ILP" D "]"; break;
-    LOAD_STORE_PAIR_LIST(LSP_OFFSET)
-    #undef LSP_OFFSET
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStorePairNonTemporal(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form;
-
-  switch (instr->Mask(LoadStorePairNonTemporalMask)) {
-    case STNP_w: mnemonic = "stnp"; form = "'Wt, 'Wt2, ['Xns'ILP4]"; break;
-    case LDNP_w: mnemonic = "ldnp"; form = "'Wt, 'Wt2, ['Xns'ILP4]"; break;
-    case STNP_x: mnemonic = "stnp"; form = "'Xt, 'Xt2, ['Xns'ILP8]"; break;
-    case LDNP_x: mnemonic = "ldnp"; form = "'Xt, 'Xt2, ['Xns'ILP8]"; break;
-    case STNP_s: mnemonic = "stnp"; form = "'St, 'St2, ['Xns'ILP4]"; break;
-    case LDNP_s: mnemonic = "ldnp"; form = "'St, 'St2, ['Xns'ILP4]"; break;
-    case STNP_d: mnemonic = "stnp"; form = "'Dt, 'Dt2, ['Xns'ILP8]"; break;
-    case LDNP_d: mnemonic = "ldnp"; form = "'Dt, 'Dt2, ['Xns'ILP8]"; break;
-    default: form = "(LoadStorePairNonTemporal)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitLoadStoreExclusive(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form;
-
-  switch (instr->Mask(LoadStoreExclusiveMask)) {
-    case STXRB_w: mnemonic = "stxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STXRH_w: mnemonic = "stxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STXR_w: mnemonic = "stxr"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STXR_x: mnemonic = "stxr"; form = "'Ws, 'Xt, ['Xns]"; break;
-    case LDXRB_w: mnemonic = "ldxrb"; form = "'Wt, ['Xns]"; break;
-    case LDXRH_w: mnemonic = "ldxrh"; form = "'Wt, ['Xns]"; break;
-    case LDXR_w: mnemonic = "ldxr"; form = "'Wt, ['Xns]"; break;
-    case LDXR_x: mnemonic = "ldxr"; form = "'Xt, ['Xns]"; break;
-    case STXP_w: mnemonic = "stxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
-    case STXP_x: mnemonic = "stxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
-    case LDXP_w: mnemonic = "ldxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
-    case LDXP_x: mnemonic = "ldxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
-    case STLXRB_w: mnemonic = "stlxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STLXRH_w: mnemonic = "stlxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STLXR_w: mnemonic = "stlxr"; form = "'Ws, 'Wt, ['Xns]"; break;
-    case STLXR_x: mnemonic = "stlxr"; form = "'Ws, 'Xt, ['Xns]"; break;
-    case LDAXRB_w: mnemonic = "ldaxrb"; form = "'Wt, ['Xns]"; break;
-    case LDAXRH_w: mnemonic = "ldaxrh"; form = "'Wt, ['Xns]"; break;
-    case LDAXR_w: mnemonic = "ldaxr"; form = "'Wt, ['Xns]"; break;
-    case LDAXR_x: mnemonic = "ldaxr"; form = "'Xt, ['Xns]"; break;
-    case STLXP_w: mnemonic = "stlxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
-    case STLXP_x: mnemonic = "stlxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
-    case LDAXP_w: mnemonic = "ldaxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
-    case LDAXP_x: mnemonic = "ldaxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
-    case STLRB_w: mnemonic = "stlrb"; form = "'Wt, ['Xns]"; break;
-    case STLRH_w: mnemonic = "stlrh"; form = "'Wt, ['Xns]"; break;
-    case STLR_w: mnemonic = "stlr"; form = "'Wt, ['Xns]"; break;
-    case STLR_x: mnemonic = "stlr"; form = "'Xt, ['Xns]"; break;
-    case LDARB_w: mnemonic = "ldarb"; form = "'Wt, ['Xns]"; break;
-    case LDARH_w: mnemonic = "ldarh"; form = "'Wt, ['Xns]"; break;
-    case LDAR_w: mnemonic = "ldar"; form = "'Wt, ['Xns]"; break;
-    case LDAR_x: mnemonic = "ldar"; form = "'Xt, ['Xns]"; break;
-    default: form = "(LoadStoreExclusive)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPCompare(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Fn, 'Fm";
-  const char *form_zero = "'Fn, #0.0";
-
-  switch (instr->Mask(FPCompareMask)) {
-    case FCMP_s_zero:
-    case FCMP_d_zero: form = form_zero;  // Fall through.
-    case FCMP_s:
-    case FCMP_d: mnemonic = "fcmp"; break;
-    default: form = "(FPCompare)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPConditionalCompare(const Instruction* instr) {
-  const char *mnemonic = "unmplemented";
-  const char *form = "'Fn, 'Fm, 'INzcv, 'Cond";
-
-  switch (instr->Mask(FPConditionalCompareMask)) {
-    case FCCMP_s:
-    case FCCMP_d: mnemonic = "fccmp"; break;
-    case FCCMPE_s:
-    case FCCMPE_d: mnemonic = "fccmpe"; break;
-    default: form = "(FPConditionalCompare)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPConditionalSelect(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Fd, 'Fn, 'Fm, 'Cond";
-
-  switch (instr->Mask(FPConditionalSelectMask)) {
-    case FCSEL_s:
-    case FCSEL_d: mnemonic = "fcsel"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPDataProcessing1Source(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'Fd, 'Fn";
-
-  switch (instr->Mask(FPDataProcessing1SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_s:           \
-    case A##_d: mnemonic = B; break;
-    FORMAT(FMOV, "fmov");
-    FORMAT(FABS, "fabs");
-    FORMAT(FNEG, "fneg");
-    FORMAT(FSQRT, "fsqrt");
-    FORMAT(FRINTN, "frintn");
-    FORMAT(FRINTP, "frintp");
-    FORMAT(FRINTM, "frintm");
-    FORMAT(FRINTZ, "frintz");
-    FORMAT(FRINTA, "frinta");
-    FORMAT(FRINTX, "frintx");
-    FORMAT(FRINTI, "frinti");
-    #undef FORMAT
-    case FCVT_ds: mnemonic = "fcvt"; form = "'Dd, 'Sn"; break;
-    case FCVT_sd: mnemonic = "fcvt"; form = "'Sd, 'Dn"; break;
-    default: form = "(FPDataProcessing1Source)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPDataProcessing2Source(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Fd, 'Fn, 'Fm";
-
-  switch (instr->Mask(FPDataProcessing2SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_s:           \
-    case A##_d: mnemonic = B; break;
-    FORMAT(FMUL, "fmul");
-    FORMAT(FDIV, "fdiv");
-    FORMAT(FADD, "fadd");
-    FORMAT(FSUB, "fsub");
-    FORMAT(FMAX, "fmax");
-    FORMAT(FMIN, "fmin");
-    FORMAT(FMAXNM, "fmaxnm");
-    FORMAT(FMINNM, "fminnm");
-    FORMAT(FNMUL, "fnmul");
-    #undef FORMAT
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPDataProcessing3Source(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Fd, 'Fn, 'Fm, 'Fa";
-
-  switch (instr->Mask(FPDataProcessing3SourceMask)) {
-    #define FORMAT(A, B)  \
-    case A##_s:           \
-    case A##_d: mnemonic = B; break;
-    FORMAT(FMADD, "fmadd");
-    FORMAT(FMSUB, "fmsub");
-    FORMAT(FNMADD, "fnmadd");
-    FORMAT(FNMSUB, "fnmsub");
-    #undef FORMAT
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPImmediate(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "(FPImmediate)";
-
-  switch (instr->Mask(FPImmediateMask)) {
-    case FMOV_s_imm: mnemonic = "fmov"; form = "'Sd, 'IFPSingle"; break;
-    case FMOV_d_imm: mnemonic = "fmov"; form = "'Dd, 'IFPDouble"; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPIntegerConvert(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "(FPIntegerConvert)";
-  const char *form_rf = "'Rd, 'Fn";
-  const char *form_fr = "'Fd, 'Rn";
-
-  switch (instr->Mask(FPIntegerConvertMask)) {
-    case FMOV_ws:
-    case FMOV_xd: mnemonic = "fmov"; form = form_rf; break;
-    case FMOV_sw:
-    case FMOV_dx: mnemonic = "fmov"; form = form_fr; break;
-    case FCVTAS_ws:
-    case FCVTAS_xs:
-    case FCVTAS_wd:
-    case FCVTAS_xd: mnemonic = "fcvtas"; form = form_rf; break;
-    case FCVTAU_ws:
-    case FCVTAU_xs:
-    case FCVTAU_wd:
-    case FCVTAU_xd: mnemonic = "fcvtau"; form = form_rf; break;
-    case FCVTMS_ws:
-    case FCVTMS_xs:
-    case FCVTMS_wd:
-    case FCVTMS_xd: mnemonic = "fcvtms"; form = form_rf; break;
-    case FCVTMU_ws:
-    case FCVTMU_xs:
-    case FCVTMU_wd:
-    case FCVTMU_xd: mnemonic = "fcvtmu"; form = form_rf; break;
-    case FCVTNS_ws:
-    case FCVTNS_xs:
-    case FCVTNS_wd:
-    case FCVTNS_xd: mnemonic = "fcvtns"; form = form_rf; break;
-    case FCVTNU_ws:
-    case FCVTNU_xs:
-    case FCVTNU_wd:
-    case FCVTNU_xd: mnemonic = "fcvtnu"; form = form_rf; break;
-    case FCVTZU_xd:
-    case FCVTZU_ws:
-    case FCVTZU_wd:
-    case FCVTZU_xs: mnemonic = "fcvtzu"; form = form_rf; break;
-    case FCVTZS_xd:
-    case FCVTZS_wd:
-    case FCVTZS_xs:
-    case FCVTZS_ws: mnemonic = "fcvtzs"; form = form_rf; break;
-    case SCVTF_sw:
-    case SCVTF_sx:
-    case SCVTF_dw:
-    case SCVTF_dx: mnemonic = "scvtf"; form = form_fr; break;
-    case UCVTF_sw:
-    case UCVTF_sx:
-    case UCVTF_dw:
-    case UCVTF_dx: mnemonic = "ucvtf"; form = form_fr; break;
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitFPFixedPointConvert(const Instruction* instr) {
-  const char *mnemonic = "";
-  const char *form = "'Rd, 'Fn, 'IFPFBits";
-  const char *form_fr = "'Fd, 'Rn, 'IFPFBits";
-
-  switch (instr->Mask(FPFixedPointConvertMask)) {
-    case FCVTZS_ws_fixed:
-    case FCVTZS_xs_fixed:
-    case FCVTZS_wd_fixed:
-    case FCVTZS_xd_fixed: mnemonic = "fcvtzs"; break;
-    case FCVTZU_ws_fixed:
-    case FCVTZU_xs_fixed:
-    case FCVTZU_wd_fixed:
-    case FCVTZU_xd_fixed: mnemonic = "fcvtzu"; break;
-    case SCVTF_sw_fixed:
-    case SCVTF_sx_fixed:
-    case SCVTF_dw_fixed:
-    case SCVTF_dx_fixed: mnemonic = "scvtf"; form = form_fr; break;
-    case UCVTF_sw_fixed:
-    case UCVTF_sx_fixed:
-    case UCVTF_dw_fixed:
-    case UCVTF_dx_fixed: mnemonic = "ucvtf"; form = form_fr; break;
-    default: VIXL_UNREACHABLE();
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitSystem(const Instruction* instr) {
-  // Some system instructions hijack their Op and Cp fields to represent a
-  // range of immediates instead of indicating a different instruction. This
-  // makes the decoding tricky.
-  const char *mnemonic = "unimplemented";
-  const char *form = "(System)";
-
-  if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
-    switch (instr->Mask(SystemExclusiveMonitorMask)) {
-      case CLREX: {
-        mnemonic = "clrex";
-        form = (instr->CRm() == 0xf) ? NULL : "'IX";
-        break;
-      }
-    }
-  } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
-    switch (instr->Mask(SystemSysRegMask)) {
-      case MRS: {
-        mnemonic = "mrs";
-        switch (instr->ImmSystemRegister()) {
-          case NZCV: form = "'Xt, nzcv"; break;
-          case FPCR: form = "'Xt, fpcr"; break;
-          default: form = "'Xt, (unknown)"; break;
-        }
-        break;
-      }
-      case MSR: {
-        mnemonic = "msr";
-        switch (instr->ImmSystemRegister()) {
-          case NZCV: form = "nzcv, 'Xt"; break;
-          case FPCR: form = "fpcr, 'Xt"; break;
-          default: form = "(unknown), 'Xt"; break;
-        }
-        break;
-      }
-    }
-  } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
-    switch (instr->ImmHint()) {
-      case NOP: {
-        mnemonic = "nop";
-        form = NULL;
-        break;
-      }
-    }
-  } else if (instr->Mask(MemBarrierFMask) == MemBarrierFixed) {
-    switch (instr->Mask(MemBarrierMask)) {
-      case DMB: {
-        mnemonic = "dmb";
-        form = "'M";
-        break;
-      }
-      case DSB: {
-        mnemonic = "dsb";
-        form = "'M";
-        break;
-      }
-      case ISB: {
-        mnemonic = "isb";
-        form = NULL;
-        break;
-      }
-    }
-  }
-
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitException(const Instruction* instr) {
-  const char *mnemonic = "unimplemented";
-  const char *form = "'IDebug";
-
-  switch (instr->Mask(ExceptionMask)) {
-    case HLT: mnemonic = "hlt"; break;
-    case BRK: mnemonic = "brk"; break;
-    case SVC: mnemonic = "svc"; break;
-    case HVC: mnemonic = "hvc"; break;
-    case SMC: mnemonic = "smc"; break;
-    case DCPS1: mnemonic = "dcps1"; form = "{'IDebug}"; break;
-    case DCPS2: mnemonic = "dcps2"; form = "{'IDebug}"; break;
-    case DCPS3: mnemonic = "dcps3"; form = "{'IDebug}"; break;
-    default: form = "(Exception)";
-  }
-  Format(instr, mnemonic, form);
-}
-
-
-void Disassembler::VisitUnimplemented(const Instruction* instr) {
-  Format(instr, "unimplemented", "(Unimplemented)");
-}
-
-
-void Disassembler::VisitUnallocated(const Instruction* instr) {
-  Format(instr, "unallocated", "(Unallocated)");
-}
-
-
-void Disassembler::ProcessOutput(const Instruction* /*instr*/) {
-  // The base disasm does nothing more than disassembling into a buffer.
-}
-
-
-void Disassembler::AppendRegisterNameToOutput(const Instruction* instr,
-                                              const CPURegister& reg) {
-  USE(instr);
-  VIXL_ASSERT(reg.IsValid());
-  char reg_char;
-
-  if (reg.IsRegister()) {
-    reg_char = reg.Is64Bits() ? 'x' : 'w';
-  } else {
-    VIXL_ASSERT(reg.IsFPRegister());
-    reg_char = reg.Is64Bits() ? 'd' : 's';
-  }
-
-  if (reg.IsFPRegister() || !(reg.Aliases(sp) || reg.Aliases(xzr))) {
-    // A normal register: w0 - w30, x0 - x30, s0 - s31, d0 - d31.
-    AppendToOutput("%c%d", reg_char, reg.code());
-  } else if (reg.Aliases(sp)) {
-    // Disassemble w31/x31 as stack pointer wsp/sp.
-    AppendToOutput("%s", reg.Is64Bits() ? "sp" : "wsp");
-  } else {
-    // Disassemble w31/x31 as zero register wzr/xzr.
-    AppendToOutput("%czr", reg_char);
-  }
-}
-
-
-void Disassembler::AppendPCRelativeOffsetToOutput(const Instruction* instr,
-                                                  int64_t offset) {
-  USE(instr);
-  char sign = (offset < 0) ? '-' : '+';
-  AppendToOutput("#%c0x%" PRIx64, sign, std::abs(offset));
-}
-
-
-void Disassembler::AppendAddressToOutput(const Instruction* instr,
-                                         const void* addr) {
-  USE(instr);
-  AppendToOutput("(addr %p)", addr);
-}
-
-
-void Disassembler::AppendCodeAddressToOutput(const Instruction* instr,
-                                             const void* addr) {
-  AppendAddressToOutput(instr, addr);
-}
-
-
-void Disassembler::AppendDataAddressToOutput(const Instruction* instr,
-                                             const void* addr) {
-  AppendAddressToOutput(instr, addr);
-}
-
-
-void Disassembler::Format(const Instruction* instr, const char* mnemonic,
-                          const char* format) {
-  VIXL_ASSERT(mnemonic != NULL);
-  ResetOutput();
-  Substitute(instr, mnemonic);
-  if (format != NULL) {
-    buffer_[buffer_pos_++] = ' ';
-    Substitute(instr, format);
-  }
-  buffer_[buffer_pos_] = 0;
-  ProcessOutput(instr);
-}
-
-
-void Disassembler::Substitute(const Instruction* instr, const char* string) {
-  char chr = *string++;
-  while (chr != '\0') {
-    if (chr == '\'') {
-      string += SubstituteField(instr, string);
-    } else {
-      buffer_[buffer_pos_++] = chr;
-    }
-    chr = *string++;
-  }
-}
-
-
-int Disassembler::SubstituteField(const Instruction* instr,
-                                  const char* format) {
-  switch (format[0]) {
-    case 'R':  // Register. X or W, selected by sf bit.
-    case 'F':  // FP Register. S or D, selected by type field.
-    case 'W':
-    case 'X':
-    case 'S':
-    case 'D': return SubstituteRegisterField(instr, format);
-    case 'I': return SubstituteImmediateField(instr, format);
-    case 'L': return SubstituteLiteralField(instr, format);
-    case 'H': return SubstituteShiftField(instr, format);
-    case 'P': return SubstitutePrefetchField(instr, format);
-    case 'C': return SubstituteConditionField(instr, format);
-    case 'E': return SubstituteExtendField(instr, format);
-    case 'A': return SubstitutePCRelAddressField(instr, format);
-    case 'B': return SubstituteBranchTargetField(instr, format);
-    case 'O': return SubstituteLSRegOffsetField(instr, format);
-    case 'M': return SubstituteBarrierField(instr, format);
-    default: {
-      VIXL_UNREACHABLE();
-      return 1;
-    }
-  }
-}
-
-
-int Disassembler::SubstituteRegisterField(const Instruction* instr,
-                                          const char* format) {
-  unsigned reg_num = 0;
-  unsigned field_len = 2;
-  switch (format[1]) {
-    case 'd': reg_num = instr->Rd(); break;
-    case 'n': reg_num = instr->Rn(); break;
-    case 'm': reg_num = instr->Rm(); break;
-    case 'a': reg_num = instr->Ra(); break;
-    case 's': reg_num = instr->Rs(); break;
-    case 't': {
-      if (format[2] == '2') {
-        reg_num = instr->Rt2();
-        field_len = 3;
-      } else {
-        reg_num = instr->Rt();
-      }
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-
-  // Increase field length for registers tagged as stack.
-  if (format[2] == 's') {
-    field_len = 3;
-  }
-
-  CPURegister::RegisterType reg_type;
-  unsigned reg_size;
-
-  if (format[0] == 'R') {
-    // Register type is R: use sf bit to choose X and W.
-    reg_type = CPURegister::kRegister;
-    reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
-  } else if (format[0] == 'F') {
-    // Floating-point register: use type field to choose S or D.
-    reg_type = CPURegister::kFPRegister;
-    reg_size = ((instr->FPType() & 1) == 0) ? kSRegSize : kDRegSize;
-  } else {
-    // The register type is specified.
-    switch (format[0]) {
-      case 'W':
-        reg_type = CPURegister::kRegister; reg_size = kWRegSize; break;
-      case 'X':
-        reg_type = CPURegister::kRegister; reg_size = kXRegSize; break;
-      case 'S':
-        reg_type = CPURegister::kFPRegister; reg_size = kSRegSize; break;
-      case 'D':
-        reg_type = CPURegister::kFPRegister; reg_size = kDRegSize; break;
-      default:
-        VIXL_UNREACHABLE();
-        reg_type = CPURegister::kRegister;
-        reg_size = kXRegSize;
-    }
-  }
-
-  if ((reg_type == CPURegister::kRegister) &&
-      (reg_num == kZeroRegCode) && (format[2] == 's')) {
-    reg_num = kSPRegInternalCode;
-  }
-
-  AppendRegisterNameToOutput(instr, CPURegister(reg_num, reg_size, reg_type));
-
-  return field_len;
-}
-
-
-int Disassembler::SubstituteImmediateField(const Instruction* instr,
-                                           const char* format) {
-  VIXL_ASSERT(format[0] == 'I');
-
-  switch (format[1]) {
-    case 'M': {  // IMoveImm or IMoveLSL.
-      if (format[5] == 'I') {
-        uint64_t imm = instr->ImmMoveWide() << (16 * instr->ShiftMoveWide());
-        AppendToOutput("#0x%" PRIx64, imm);
-      } else {
-        VIXL_ASSERT(format[5] == 'L');
-        AppendToOutput("#0x%" PRIx64, instr->ImmMoveWide());
-        if (instr->ShiftMoveWide() > 0) {
-          AppendToOutput(", lsl #%" PRId64, 16 * instr->ShiftMoveWide());
-        }
-      }
-      return 8;
-    }
-    case 'L': {
-      switch (format[2]) {
-        case 'L': {  // ILLiteral - Immediate Load Literal.
-          AppendToOutput("pc%+" PRId64,
-                         instr->ImmLLiteral() << kLiteralEntrySizeLog2);
-          return 9;
-        }
-        case 'S': {  // ILS - Immediate Load/Store.
-          if (instr->ImmLS() != 0) {
-            AppendToOutput(", #%" PRId64, instr->ImmLS());
-          }
-          return 3;
-        }
-        case 'P': {  // ILPx - Immediate Load/Store Pair, x = access size.
-          if (instr->ImmLSPair() != 0) {
-            // format[3] is the scale value. Convert to a number.
-            int scale = format[3] - 0x30;
-            AppendToOutput(", #%" PRId64, instr->ImmLSPair() * scale);
-          }
-          return 4;
-        }
-        case 'U': {  // ILU - Immediate Load/Store Unsigned.
-          if (instr->ImmLSUnsigned() != 0) {
-            AppendToOutput(", #%" PRIu64,
-                           instr->ImmLSUnsigned() << instr->SizeLS());
-          }
-          return 3;
-        }
-      }
-    }
-    case 'C': {  // ICondB - Immediate Conditional Branch.
-      int64_t offset = instr->ImmCondBranch() << 2;
-      AppendPCRelativeOffsetToOutput(instr, offset);
-      return 6;
-    }
-    case 'A': {  // IAddSub.
-      VIXL_ASSERT(instr->ShiftAddSub() <= 1);
-      int64_t imm = instr->ImmAddSub() << (12 * instr->ShiftAddSub());
-      AppendToOutput("#0x%" PRIx64 " (%" PRId64 ")", imm, imm);
-      return 7;
-    }
-    case 'F': {  // IFPSingle, IFPDouble or IFPFBits.
-      if (format[3] == 'F') {  // IFPFbits.
-        AppendToOutput("#%" PRId64, 64 - instr->FPScale());
-        return 8;
-      } else {
-        AppendToOutput("#0x%" PRIx64 " (%.4f)", instr->ImmFP(),
-                       format[3] == 'S' ? instr->ImmFP32() : instr->ImmFP64());
-        return 9;
-      }
-    }
-    case 'T': {  // ITri - Immediate Triangular Encoded.
-      AppendToOutput("#0x%" PRIx64, instr->ImmLogical());
-      return 4;
-    }
-    case 'N': {  // INzcv.
-      int nzcv = (instr->Nzcv() << Flags_offset);
-      AppendToOutput("#%c%c%c%c", ((nzcv & NFlag) == 0) ? 'n' : 'N',
-                                  ((nzcv & ZFlag) == 0) ? 'z' : 'Z',
-                                  ((nzcv & CFlag) == 0) ? 'c' : 'C',
-                                  ((nzcv & VFlag) == 0) ? 'v' : 'V');
-      return 5;
-    }
-    case 'P': {  // IP - Conditional compare.
-      AppendToOutput("#%" PRId64, instr->ImmCondCmp());
-      return 2;
-    }
-    case 'B': {  // Bitfields.
-      return SubstituteBitfieldImmediateField(instr, format);
-    }
-    case 'E': {  // IExtract.
-      AppendToOutput("#%" PRId64, instr->ImmS());
-      return 8;
-    }
-    case 'S': {  // IS - Test and branch bit.
-      AppendToOutput("#%" PRId64, (instr->ImmTestBranchBit5() << 5) |
-                                  instr->ImmTestBranchBit40());
-      return 2;
-    }
-    case 'D': {  // IDebug - HLT and BRK instructions.
-      AppendToOutput("#0x%" PRIx64, instr->ImmException());
-      return 6;
-    }
-    case 'X': {  // IX - CLREX instruction.
-      AppendToOutput("#0x%" PRIx64, instr->CRm());
-      return 2;
-    }
-    default: {
-      VIXL_UNIMPLEMENTED();
-      return 0;
-    }
-  }
-}
-
-
-int Disassembler::SubstituteBitfieldImmediateField(const Instruction* instr,
-                                                   const char* format) {
-  VIXL_ASSERT((format[0] == 'I') && (format[1] == 'B'));
-  unsigned r = instr->ImmR();
-  unsigned s = instr->ImmS();
-
-  switch (format[2]) {
-    case 'r': {  // IBr.
-      AppendToOutput("#%d", r);
-      return 3;
-    }
-    case 's': {  // IBs+1 or IBs-r+1.
-      if (format[3] == '+') {
-        AppendToOutput("#%d", s + 1);
-        return 5;
-      } else {
-        VIXL_ASSERT(format[3] == '-');
-        AppendToOutput("#%d", s - r + 1);
-        return 7;
-      }
-    }
-    case 'Z': {  // IBZ-r.
-      VIXL_ASSERT((format[3] == '-') && (format[4] == 'r'));
-      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize;
-      AppendToOutput("#%d", reg_size - r);
-      return 5;
-    }
-    default: {
-      VIXL_UNREACHABLE();
-      return 0;
-    }
-  }
-}
-
-
-int Disassembler::SubstituteLiteralField(const Instruction* instr,
-                                         const char* format) {
-  VIXL_ASSERT(strncmp(format, "LValue", 6) == 0);
-  USE(format);
-
-  switch (instr->Mask(LoadLiteralMask)) {
-    case LDR_w_lit:
-    case LDR_x_lit:
-    case LDRSW_x_lit:
-    case LDR_s_lit:
-    case LDR_d_lit:
-      AppendDataAddressToOutput(instr, instr->LiteralAddress());
-      break;
-    default:
-      VIXL_UNREACHABLE();
-  }
-
-  return 6;
-}
-
-
-int Disassembler::SubstituteShiftField(const Instruction* instr,
-                                       const char* format) {
-  VIXL_ASSERT(format[0] == 'H');
-  VIXL_ASSERT(instr->ShiftDP() <= 0x3);
-
-  switch (format[1]) {
-    case 'D': {  // HDP.
-      VIXL_ASSERT(instr->ShiftDP() != ROR);
-    }  // Fall through.
-    case 'L': {  // HLo.
-      if (instr->ImmDPShift() != 0) {
-        const char* shift_type[] = {"lsl", "lsr", "asr", "ror"};
-        AppendToOutput(", %s #%" PRId64, shift_type[instr->ShiftDP()],
-                       instr->ImmDPShift());
-      }
-      return 3;
-    }
-    default:
-      VIXL_UNIMPLEMENTED();
-      return 0;
-  }
-}
-
-
-int Disassembler::SubstituteConditionField(const Instruction* instr,
-                                           const char* format) {
-  VIXL_ASSERT(format[0] == 'C');
-  const char* condition_code[] = { "eq", "ne", "hs", "lo",
-                                   "mi", "pl", "vs", "vc",
-                                   "hi", "ls", "ge", "lt",
-                                   "gt", "le", "al", "nv" };
-  int cond;
-  switch (format[1]) {
-    case 'B': cond = instr->ConditionBranch(); break;
-    case 'I': {
-      cond = InvertCondition(static_cast<Condition>(instr->Condition()));
-      break;
-    }
-    default: cond = instr->Condition();
-  }
-  AppendToOutput("%s", condition_code[cond]);
-  return 4;
-}
-
-
-int Disassembler::SubstitutePCRelAddressField(const Instruction* instr,
-                                              const char* format) {
-  VIXL_ASSERT((strcmp(format, "AddrPCRelByte") == 0) ||   // Used by `adr`.
-              (strcmp(format, "AddrPCRelPage") == 0));    // Used by `adrp`.
-
-  int64_t offset = instr->ImmPCRel();
-  const Instruction * base = instr;
-
-  if (format[9] == 'P') {
-    offset *= kPageSize;
-    base = AlignDown(base, kPageSize);
-  }
-
-  const void* target = reinterpret_cast<const void*>(base + offset);
-  AppendPCRelativeOffsetToOutput(instr, offset);
-  AppendToOutput(" ");
-  AppendAddressToOutput(instr, target);
-  return 13;
-}
-
-
-int Disassembler::SubstituteBranchTargetField(const Instruction* instr,
-                                              const char* format) {
-  VIXL_ASSERT(strncmp(format, "BImm", 4) == 0);
-
-  int64_t offset = 0;
-  switch (format[5]) {
-    // BImmUncn - unconditional branch immediate.
-    case 'n': offset = instr->ImmUncondBranch(); break;
-    // BImmCond - conditional branch immediate.
-    case 'o': offset = instr->ImmCondBranch(); break;
-    // BImmCmpa - compare and branch immediate.
-    case 'm': offset = instr->ImmCmpBranch(); break;
-    // BImmTest - test and branch immediate.
-    case 'e': offset = instr->ImmTestBranch(); break;
-    default: VIXL_UNIMPLEMENTED();
-  }
-  offset <<= kInstructionSizeLog2;
-  const void* target_address = reinterpret_cast<const void*>(instr + offset);
-  VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
-
-  AppendPCRelativeOffsetToOutput(instr, offset);
-  AppendToOutput(" ");
-  AppendCodeAddressToOutput(instr, target_address);
-
-  return 8;
-}
-
-
-int Disassembler::SubstituteExtendField(const Instruction* instr,
-                                        const char* format) {
-  VIXL_ASSERT(strncmp(format, "Ext", 3) == 0);
-  VIXL_ASSERT(instr->ExtendMode() <= 7);
-  USE(format);
-
-  const char* extend_mode[] = { "uxtb", "uxth", "uxtw", "uxtx",
-                                "sxtb", "sxth", "sxtw", "sxtx" };
-
-  // If rd or rn is SP, uxtw on 32-bit registers and uxtx on 64-bit
-  // registers becomes lsl.
-  if (((instr->Rd() == kZeroRegCode) || (instr->Rn() == kZeroRegCode)) &&
-      (((instr->ExtendMode() == UXTW) && (instr->SixtyFourBits() == 0)) ||
-       (instr->ExtendMode() == UXTX))) {
-    if (instr->ImmExtendShift() > 0) {
-      AppendToOutput(", lsl #%" PRId64, instr->ImmExtendShift());
-    }
-  } else {
-    AppendToOutput(", %s", extend_mode[instr->ExtendMode()]);
-    if (instr->ImmExtendShift() > 0) {
-      AppendToOutput(" #%" PRId64, instr->ImmExtendShift());
-    }
-  }
-  return 3;
-}
-
-
-int Disassembler::SubstituteLSRegOffsetField(const Instruction* instr,
-                                             const char* format) {
-  VIXL_ASSERT(strncmp(format, "Offsetreg", 9) == 0);
-  const char* extend_mode[] = { "undefined", "undefined", "uxtw", "lsl",
-                                "undefined", "undefined", "sxtw", "sxtx" };
-  USE(format);
-
-  unsigned shift = instr->ImmShiftLS();
-  Extend ext = static_cast<Extend>(instr->ExtendMode());
-  char reg_type = ((ext == UXTW) || (ext == SXTW)) ? 'w' : 'x';
-
-  unsigned rm = instr->Rm();
-  if (rm == kZeroRegCode) {
-    AppendToOutput("%czr", reg_type);
-  } else {
-    AppendToOutput("%c%d", reg_type, rm);
-  }
-
-  // Extend mode UXTX is an alias for shift mode LSL here.
-  if (!((ext == UXTX) && (shift == 0))) {
-    AppendToOutput(", %s", extend_mode[ext]);
-    if (shift != 0) {
-      AppendToOutput(" #%" PRId64, instr->SizeLS());
-    }
-  }
-  return 9;
-}
-
-
-int Disassembler::SubstitutePrefetchField(const Instruction* instr,
-                                          const char* format) {
-  VIXL_ASSERT(format[0] == 'P');
-  USE(format);
-
-  int prefetch_mode = instr->PrefetchMode();
-
-  const char* ls = (prefetch_mode & 0x10) ? "st" : "ld";
-  int level = (prefetch_mode >> 1) + 1;
-  const char* ks = (prefetch_mode & 1) ? "strm" : "keep";
-
-  AppendToOutput("p%sl%d%s", ls, level, ks);
-  return 6;
-}
-
-int Disassembler::SubstituteBarrierField(const Instruction* instr,
-                                         const char* format) {
-  VIXL_ASSERT(format[0] == 'M');
-  USE(format);
-
-  static const char* options[4][4] = {
-    { "sy (0b0000)", "oshld", "oshst", "osh" },
-    { "sy (0b0100)", "nshld", "nshst", "nsh" },
-    { "sy (0b1000)", "ishld", "ishst", "ish" },
-    { "sy (0b1100)", "ld", "st", "sy" }
-  };
-  int domain = instr->ImmBarrierDomain();
-  int type = instr->ImmBarrierType();
-
-  AppendToOutput("%s", options[domain][type]);
-  return 1;
-}
-
-void Disassembler::ResetOutput() {
-  buffer_pos_ = 0;
-  buffer_[buffer_pos_] = 0;
-}
-
-
-void Disassembler::AppendToOutput(const char* format, ...) {
-  va_list args;
-  va_start(args, format);
-  buffer_pos_ += vsnprintf(&buffer_[buffer_pos_], buffer_size_, format, args);
-  va_end(args);
-}
-
-
-void PrintDisassembler::ProcessOutput(const Instruction* instr) {
-#if defined(_WIN32)
-  // See http://comments.gmane.org/gmane.comp.web.curl.library/36161
-  // NOTE: snprintf() points to gnu_printf() that doesn't accept PRIx64
-  //       in this context. Yes, this is very confusing :-/
-  char hexInstr[32];
-  snprintf(hexInstr, sizeof(hexInstr), "%016llx",
-           reinterpret_cast<long long>(instr));
-  fprintf(stream_, "0x%s  %08" PRIx32 "\t\t%s\n",
-          hexInstr,
-          instr->InstructionBits(),
-          GetOutput());
-#else
-  fprintf(stream_, "0x%016" PRIx64 "  %08" PRIx32 "\t\t%s\n",
-          reinterpret_cast<uint64_t>(instr),
-          instr->InstructionBits(),
-          GetOutput());
-#endif
-}
-}  // namespace vixl
diff --git a/disas/libvixl/a64/disasm-a64.h b/disas/libvixl/a64/disasm-a64.h
deleted file mode 100644
index db04337..0000000
--- a/disas/libvixl/a64/disasm-a64.h
+++ /dev/null
@@ -1,146 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_DISASM_A64_H
-#define VIXL_A64_DISASM_A64_H
-
-#include "globals.h"
-#include "utils.h"
-#include "instructions-a64.h"
-#include "decoder-a64.h"
-#include "assembler-a64.h"
-
-namespace vixl {
-
-class Disassembler: public DecoderVisitor {
- public:
-  Disassembler();
-  Disassembler(char* text_buffer, int buffer_size);
-  virtual ~Disassembler();
-  char* GetOutput();
-
-  // Declare all Visitor functions.
-  #define DECLARE(A)  void Visit##A(const Instruction* instr);
-  VISITOR_LIST(DECLARE)
-  #undef DECLARE
-
- protected:
-  virtual void ProcessOutput(const Instruction* instr);
-
-  // Default output functions.  The functions below implement a default way of
-  // printing elements in the disassembly. A sub-class can override these to
-  // customize the disassembly output.
-
-  // Prints the name of a register.
-  virtual void AppendRegisterNameToOutput(const Instruction* instr,
-                                          const CPURegister& reg);
-
-  // Prints a PC-relative offset. This is used for example when disassembling
-  // branches to immediate offsets.
-  virtual void AppendPCRelativeOffsetToOutput(const Instruction* instr,
-                                              int64_t offset);
-
-  // Prints an address, in the general case. It can be code or data. This is
-  // used for example to print the target address of an ADR instruction.
-  virtual void AppendAddressToOutput(const Instruction* instr,
-                                     const void* addr);
-
-  // Prints the address of some code.
-  // This is used for example to print the target address of a branch to an
-  // immediate offset.
-  // A sub-class can for example override this method to lookup the address and
-  // print an appropriate name.
-  virtual void AppendCodeAddressToOutput(const Instruction* instr,
-                                         const void* addr);
-
-  // Prints the address of some data.
-  // This is used for example to print the source address of a load literal
-  // instruction.
-  virtual void AppendDataAddressToOutput(const Instruction* instr,
-                                         const void* addr);
-
- private:
-  void Format(
-      const Instruction* instr, const char* mnemonic, const char* format);
-  void Substitute(const Instruction* instr, const char* string);
-  int SubstituteField(const Instruction* instr, const char* format);
-  int SubstituteRegisterField(const Instruction* instr, const char* format);
-  int SubstituteImmediateField(const Instruction* instr, const char* format);
-  int SubstituteLiteralField(const Instruction* instr, const char* format);
-  int SubstituteBitfieldImmediateField(
-      const Instruction* instr, const char* format);
-  int SubstituteShiftField(const Instruction* instr, const char* format);
-  int SubstituteExtendField(const Instruction* instr, const char* format);
-  int SubstituteConditionField(const Instruction* instr, const char* format);
-  int SubstitutePCRelAddressField(const Instruction* instr, const char* format);
-  int SubstituteBranchTargetField(const Instruction* instr, const char* format);
-  int SubstituteLSRegOffsetField(const Instruction* instr, const char* format);
-  int SubstitutePrefetchField(const Instruction* instr, const char* format);
-  int SubstituteBarrierField(const Instruction* instr, const char* format);
-
-  inline bool RdIsZROrSP(const Instruction* instr) const {
-    return (instr->Rd() == kZeroRegCode);
-  }
-
-  inline bool RnIsZROrSP(const Instruction* instr) const {
-    return (instr->Rn() == kZeroRegCode);
-  }
-
-  inline bool RmIsZROrSP(const Instruction* instr) const {
-    return (instr->Rm() == kZeroRegCode);
-  }
-
-  inline bool RaIsZROrSP(const Instruction* instr) const {
-    return (instr->Ra() == kZeroRegCode);
-  }
-
-  bool IsMovzMovnImm(unsigned reg_size, uint64_t value);
-
- protected:
-  void ResetOutput();
-  void AppendToOutput(const char* string, ...) PRINTF_CHECK(2, 3);
-
-  char* buffer_;
-  uint32_t buffer_pos_;
-  uint32_t buffer_size_;
-  bool own_buffer_;
-};
-
-
-class PrintDisassembler: public Disassembler {
- public:
-  explicit PrintDisassembler(FILE* stream) : stream_(stream) { }
-  virtual ~PrintDisassembler() { }
-
- protected:
-  virtual void ProcessOutput(const Instruction* instr);
-
- private:
-  FILE *stream_;
-};
-}  // namespace vixl
-
-#endif  // VIXL_A64_DISASM_A64_H
diff --git a/disas/libvixl/a64/instructions-a64.cc b/disas/libvixl/a64/instructions-a64.cc
deleted file mode 100644
index 1f08c78..0000000
--- a/disas/libvixl/a64/instructions-a64.cc
+++ /dev/null
@@ -1,251 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "a64/instructions-a64.h"
-#include "a64/assembler-a64.h"
-
-namespace vixl {
-
-
-static uint64_t RotateRight(uint64_t value,
-                            unsigned int rotate,
-                            unsigned int width) {
-  VIXL_ASSERT(width <= 64);
-  rotate &= 63;
-  return ((value & ((UINT64_C(1) << rotate) - 1)) <<
-          (width - rotate)) | (value >> rotate);
-}
-
-
-static uint64_t RepeatBitsAcrossReg(unsigned reg_size,
-                                    uint64_t value,
-                                    unsigned width) {
-  VIXL_ASSERT((width == 2) || (width == 4) || (width == 8) || (width == 16) ||
-              (width == 32));
-  VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
-  uint64_t result = value & ((UINT64_C(1) << width) - 1);
-  for (unsigned i = width; i < reg_size; i *= 2) {
-    result |= (result << i);
-  }
-  return result;
-}
-
-
-// Logical immediates can't encode zero, so a return value of zero is used to
-// indicate a failure case. Specifically, where the constraints on imm_s are
-// not met.
-uint64_t Instruction::ImmLogical() const {
-  unsigned reg_size = SixtyFourBits() ? kXRegSize : kWRegSize;
-  int64_t n = BitN();
-  int64_t imm_s = ImmSetBits();
-  int64_t imm_r = ImmRotate();
-
-  // An integer is constructed from the n, imm_s and imm_r bits according to
-  // the following table:
-  //
-  //  N   imms    immr    size        S             R
-  //  1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
-  //  0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
-  //  0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
-  //  0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
-  //  0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
-  //  0  11110s  xxxxxr     2    UInt(s)       UInt(r)
-  // (s bits must not be all set)
-  //
-  // A pattern is constructed of size bits, where the least significant S+1
-  // bits are set. The pattern is rotated right by R, and repeated across a
-  // 32 or 64-bit value, depending on destination register width.
-  //
-
-  if (n == 1) {
-    if (imm_s == 0x3F) {
-      return 0;
-    }
-    uint64_t bits = (UINT64_C(1) << (imm_s + 1)) - 1;
-    return RotateRight(bits, imm_r, 64);
-  } else {
-    if ((imm_s >> 1) == 0x1F) {
-      return 0;
-    }
-    for (int width = 0x20; width >= 0x2; width >>= 1) {
-      if ((imm_s & width) == 0) {
-        int mask = width - 1;
-        if ((imm_s & mask) == mask) {
-          return 0;
-        }
-        uint64_t bits = (UINT64_C(1) << ((imm_s & mask) + 1)) - 1;
-        return RepeatBitsAcrossReg(reg_size,
-                                   RotateRight(bits, imm_r & mask, width),
-                                   width);
-      }
-    }
-  }
-  VIXL_UNREACHABLE();
-  return 0;
-}
-
-
-float Instruction::ImmFP32() const {
-  //  ImmFP: abcdefgh (8 bits)
-  // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
-  // where B is b ^ 1
-  uint32_t bits = ImmFP();
-  uint32_t bit7 = (bits >> 7) & 0x1;
-  uint32_t bit6 = (bits >> 6) & 0x1;
-  uint32_t bit5_to_0 = bits & 0x3f;
-  uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
-
-  return rawbits_to_float(result);
-}
-
-
-double Instruction::ImmFP64() const {
-  //  ImmFP: abcdefgh (8 bits)
-  // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
-  //         0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
-  // where B is b ^ 1
-  uint32_t bits = ImmFP();
-  uint64_t bit7 = (bits >> 7) & 0x1;
-  uint64_t bit6 = (bits >> 6) & 0x1;
-  uint64_t bit5_to_0 = bits & 0x3f;
-  uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
-
-  return rawbits_to_double(result);
-}
-
-
-LSDataSize CalcLSPairDataSize(LoadStorePairOp op) {
-  switch (op) {
-    case STP_x:
-    case LDP_x:
-    case STP_d:
-    case LDP_d: return LSDoubleWord;
-    default: return LSWord;
-  }
-}
-
-
-const Instruction* Instruction::ImmPCOffsetTarget() const {
-  const Instruction * base = this;
-  ptrdiff_t offset;
-  if (IsPCRelAddressing()) {
-    // ADR and ADRP.
-    offset = ImmPCRel();
-    if (Mask(PCRelAddressingMask) == ADRP) {
-      base = AlignDown(base, kPageSize);
-      offset *= kPageSize;
-    } else {
-      VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
-    }
-  } else {
-    // All PC-relative branches.
-    VIXL_ASSERT(BranchType() != UnknownBranchType);
-    // Relative branch offsets are instruction-size-aligned.
-    offset = ImmBranch() << kInstructionSizeLog2;
-  }
-  return base + offset;
-}
-
-
-inline int Instruction::ImmBranch() const {
-  switch (BranchType()) {
-    case CondBranchType: return ImmCondBranch();
-    case UncondBranchType: return ImmUncondBranch();
-    case CompareBranchType: return ImmCmpBranch();
-    case TestBranchType: return ImmTestBranch();
-    default: VIXL_UNREACHABLE();
-  }
-  return 0;
-}
-
-
-void Instruction::SetImmPCOffsetTarget(const Instruction* target) {
-  if (IsPCRelAddressing()) {
-    SetPCRelImmTarget(target);
-  } else {
-    SetBranchImmTarget(target);
-  }
-}
-
-
-void Instruction::SetPCRelImmTarget(const Instruction* target) {
-  int32_t imm21;
-  if ((Mask(PCRelAddressingMask) == ADR)) {
-    imm21 = target - this;
-  } else {
-    VIXL_ASSERT(Mask(PCRelAddressingMask) == ADRP);
-    uintptr_t this_page = reinterpret_cast<uintptr_t>(this) / kPageSize;
-    uintptr_t target_page = reinterpret_cast<uintptr_t>(target) / kPageSize;
-    imm21 = target_page - this_page;
-  }
-  Instr imm = Assembler::ImmPCRelAddress(imm21);
-
-  SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
-}
-
-
-void Instruction::SetBranchImmTarget(const Instruction* target) {
-  VIXL_ASSERT(((target - this) & 3) == 0);
-  Instr branch_imm = 0;
-  uint32_t imm_mask = 0;
-  int offset = (target - this) >> kInstructionSizeLog2;
-  switch (BranchType()) {
-    case CondBranchType: {
-      branch_imm = Assembler::ImmCondBranch(offset);
-      imm_mask = ImmCondBranch_mask;
-      break;
-    }
-    case UncondBranchType: {
-      branch_imm = Assembler::ImmUncondBranch(offset);
-      imm_mask = ImmUncondBranch_mask;
-      break;
-    }
-    case CompareBranchType: {
-      branch_imm = Assembler::ImmCmpBranch(offset);
-      imm_mask = ImmCmpBranch_mask;
-      break;
-    }
-    case TestBranchType: {
-      branch_imm = Assembler::ImmTestBranch(offset);
-      imm_mask = ImmTestBranch_mask;
-      break;
-    }
-    default: VIXL_UNREACHABLE();
-  }
-  SetInstructionBits(Mask(~imm_mask) | branch_imm);
-}
-
-
-void Instruction::SetImmLLiteral(const Instruction* source) {
-  VIXL_ASSERT(IsWordAligned(source));
-  ptrdiff_t offset = (source - this) >> kLiteralEntrySizeLog2;
-  Instr imm = Assembler::ImmLLiteral(offset);
-  Instr mask = ImmLLiteral_mask;
-
-  SetInstructionBits(Mask(~mask) | imm);
-}
-}  // namespace vixl
-
diff --git a/disas/libvixl/a64/instructions-a64.h b/disas/libvixl/a64/instructions-a64.h
deleted file mode 100644
index 29f9722..0000000
--- a/disas/libvixl/a64/instructions-a64.h
+++ /dev/null
@@ -1,354 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_A64_INSTRUCTIONS_A64_H_
-#define VIXL_A64_INSTRUCTIONS_A64_H_
-
-#include "globals.h"
-#include "utils.h"
-#include "a64/constants-a64.h"
-
-namespace vixl {
-// ISA constants. --------------------------------------------------------------
-
-typedef uint32_t Instr;
-const unsigned kInstructionSize = 4;
-const unsigned kInstructionSizeLog2 = 2;
-const unsigned kLiteralEntrySize = 4;
-const unsigned kLiteralEntrySizeLog2 = 2;
-const unsigned kMaxLoadLiteralRange = 1 * MBytes;
-
-// This is the nominal page size (as used by the adrp instruction); the actual
-// size of the memory pages allocated by the kernel is likely to differ.
-const unsigned kPageSize = 4 * KBytes;
-const unsigned kPageSizeLog2 = 12;
-
-const unsigned kWRegSize = 32;
-const unsigned kWRegSizeLog2 = 5;
-const unsigned kWRegSizeInBytes = kWRegSize / 8;
-const unsigned kWRegSizeInBytesLog2 = kWRegSizeLog2 - 3;
-const unsigned kXRegSize = 64;
-const unsigned kXRegSizeLog2 = 6;
-const unsigned kXRegSizeInBytes = kXRegSize / 8;
-const unsigned kXRegSizeInBytesLog2 = kXRegSizeLog2 - 3;
-const unsigned kSRegSize = 32;
-const unsigned kSRegSizeLog2 = 5;
-const unsigned kSRegSizeInBytes = kSRegSize / 8;
-const unsigned kSRegSizeInBytesLog2 = kSRegSizeLog2 - 3;
-const unsigned kDRegSize = 64;
-const unsigned kDRegSizeLog2 = 6;
-const unsigned kDRegSizeInBytes = kDRegSize / 8;
-const unsigned kDRegSizeInBytesLog2 = kDRegSizeLog2 - 3;
-const uint64_t kWRegMask = UINT64_C(0xffffffff);
-const uint64_t kXRegMask = UINT64_C(0xffffffffffffffff);
-const uint64_t kSRegMask = UINT64_C(0xffffffff);
-const uint64_t kDRegMask = UINT64_C(0xffffffffffffffff);
-const uint64_t kSSignMask = UINT64_C(0x80000000);
-const uint64_t kDSignMask = UINT64_C(0x8000000000000000);
-const uint64_t kWSignMask = UINT64_C(0x80000000);
-const uint64_t kXSignMask = UINT64_C(0x8000000000000000);
-const uint64_t kByteMask = UINT64_C(0xff);
-const uint64_t kHalfWordMask = UINT64_C(0xffff);
-const uint64_t kWordMask = UINT64_C(0xffffffff);
-const uint64_t kXMaxUInt = UINT64_C(0xffffffffffffffff);
-const uint64_t kWMaxUInt = UINT64_C(0xffffffff);
-const int64_t kXMaxInt = INT64_C(0x7fffffffffffffff);
-const int64_t kXMinInt = INT64_C(0x8000000000000000);
-const int32_t kWMaxInt = INT32_C(0x7fffffff);
-const int32_t kWMinInt = INT32_C(0x80000000);
-const unsigned kLinkRegCode = 30;
-const unsigned kZeroRegCode = 31;
-const unsigned kSPRegInternalCode = 63;
-const unsigned kRegCodeMask = 0x1f;
-
-const unsigned kAddressTagOffset = 56;
-const unsigned kAddressTagWidth = 8;
-const uint64_t kAddressTagMask =
-    ((UINT64_C(1) << kAddressTagWidth) - 1) << kAddressTagOffset;
-VIXL_STATIC_ASSERT(kAddressTagMask == UINT64_C(0xff00000000000000));
-
-// AArch64 floating-point specifics. These match IEEE-754.
-const unsigned kDoubleMantissaBits = 52;
-const unsigned kDoubleExponentBits = 11;
-const unsigned kFloatMantissaBits = 23;
-const unsigned kFloatExponentBits = 8;
-
-enum LSDataSize {
-  LSByte        = 0,
-  LSHalfword    = 1,
-  LSWord        = 2,
-  LSDoubleWord  = 3
-};
-
-LSDataSize CalcLSPairDataSize(LoadStorePairOp op);
-
-enum ImmBranchType {
-  UnknownBranchType = 0,
-  CondBranchType    = 1,
-  UncondBranchType  = 2,
-  CompareBranchType = 3,
-  TestBranchType    = 4
-};
-
-enum AddrMode {
-  Offset,
-  PreIndex,
-  PostIndex
-};
-
-enum FPRounding {
-  // The first four values are encodable directly by FPCR<RMode>.
-  FPTieEven = 0x0,
-  FPPositiveInfinity = 0x1,
-  FPNegativeInfinity = 0x2,
-  FPZero = 0x3,
-
-  // The final rounding mode is only available when explicitly specified by the
-  // instruction (such as with fcvta). It cannot be set in FPCR.
-  FPTieAway
-};
-
-enum Reg31Mode {
-  Reg31IsStackPointer,
-  Reg31IsZeroRegister
-};
-
-// Instructions. ---------------------------------------------------------------
-
-class Instruction {
- public:
-  inline Instr InstructionBits() const {
-    return *(reinterpret_cast<const Instr*>(this));
-  }
-
-  inline void SetInstructionBits(Instr new_instr) {
-    *(reinterpret_cast<Instr*>(this)) = new_instr;
-  }
-
-  inline int Bit(int pos) const {
-    return (InstructionBits() >> pos) & 1;
-  }
-
-  inline uint32_t Bits(int msb, int lsb) const {
-    return unsigned_bitextract_32(msb, lsb, InstructionBits());
-  }
-
-  inline int32_t SignedBits(int msb, int lsb) const {
-    int32_t bits = *(reinterpret_cast<const int32_t*>(this));
-    return signed_bitextract_32(msb, lsb, bits);
-  }
-
-  inline Instr Mask(uint32_t mask) const {
-    return InstructionBits() & mask;
-  }
-
-  #define DEFINE_GETTER(Name, HighBit, LowBit, Func)             \
-  inline int64_t Name() const { return Func(HighBit, LowBit); }
-  INSTRUCTION_FIELDS_LIST(DEFINE_GETTER)
-  #undef DEFINE_GETTER
-
-  // ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST),
-  // formed from ImmPCRelLo and ImmPCRelHi.
-  int ImmPCRel() const {
-    int const offset = ((ImmPCRelHi() << ImmPCRelLo_width) | ImmPCRelLo());
-    int const width = ImmPCRelLo_width + ImmPCRelHi_width;
-    return signed_bitextract_32(width-1, 0, offset);
-  }
-
-  uint64_t ImmLogical() const;
-  float ImmFP32() const;
-  double ImmFP64() const;
-
-  inline LSDataSize SizeLSPair() const {
-    return CalcLSPairDataSize(
-             static_cast<LoadStorePairOp>(Mask(LoadStorePairMask)));
-  }
-
-  // Helpers.
-  inline bool IsCondBranchImm() const {
-    return Mask(ConditionalBranchFMask) == ConditionalBranchFixed;
-  }
-
-  inline bool IsUncondBranchImm() const {
-    return Mask(UnconditionalBranchFMask) == UnconditionalBranchFixed;
-  }
-
-  inline bool IsCompareBranch() const {
-    return Mask(CompareBranchFMask) == CompareBranchFixed;
-  }
-
-  inline bool IsTestBranch() const {
-    return Mask(TestBranchFMask) == TestBranchFixed;
-  }
-
-  inline bool IsPCRelAddressing() const {
-    return Mask(PCRelAddressingFMask) == PCRelAddressingFixed;
-  }
-
-  inline bool IsLogicalImmediate() const {
-    return Mask(LogicalImmediateFMask) == LogicalImmediateFixed;
-  }
-
-  inline bool IsAddSubImmediate() const {
-    return Mask(AddSubImmediateFMask) == AddSubImmediateFixed;
-  }
-
-  inline bool IsAddSubExtended() const {
-    return Mask(AddSubExtendedFMask) == AddSubExtendedFixed;
-  }
-
-  inline bool IsLoadOrStore() const {
-    return Mask(LoadStoreAnyFMask) == LoadStoreAnyFixed;
-  }
-
-  inline bool IsMovn() const {
-    return (Mask(MoveWideImmediateMask) == MOVN_x) ||
-           (Mask(MoveWideImmediateMask) == MOVN_w);
-  }
-
-  // Indicate whether Rd can be the stack pointer or the zero register. This
-  // does not check that the instruction actually has an Rd field.
-  inline Reg31Mode RdMode() const {
-    // The following instructions use sp or wsp as Rd:
-    //  Add/sub (immediate) when not setting the flags.
-    //  Add/sub (extended) when not setting the flags.
-    //  Logical (immediate) when not setting the flags.
-    // Otherwise, r31 is the zero register.
-    if (IsAddSubImmediate() || IsAddSubExtended()) {
-      if (Mask(AddSubSetFlagsBit)) {
-        return Reg31IsZeroRegister;
-      } else {
-        return Reg31IsStackPointer;
-      }
-    }
-    if (IsLogicalImmediate()) {
-      // Of the logical (immediate) instructions, only ANDS (and its aliases)
-      // can set the flags. The others can all write into sp.
-      // Note that some logical operations are not available to
-      // immediate-operand instructions, so we have to combine two masks here.
-      if (Mask(LogicalImmediateMask & LogicalOpMask) == ANDS) {
-        return Reg31IsZeroRegister;
-      } else {
-        return Reg31IsStackPointer;
-      }
-    }
-    return Reg31IsZeroRegister;
-  }
-
-  // Indicate whether Rn can be the stack pointer or the zero register. This
-  // does not check that the instruction actually has an Rn field.
-  inline Reg31Mode RnMode() const {
-    // The following instructions use sp or wsp as Rn:
-    //  All loads and stores.
-    //  Add/sub (immediate).
-    //  Add/sub (extended).
-    // Otherwise, r31 is the zero register.
-    if (IsLoadOrStore() || IsAddSubImmediate() || IsAddSubExtended()) {
-      return Reg31IsStackPointer;
-    }
-    return Reg31IsZeroRegister;
-  }
-
-  inline ImmBranchType BranchType() const {
-    if (IsCondBranchImm()) {
-      return CondBranchType;
-    } else if (IsUncondBranchImm()) {
-      return UncondBranchType;
-    } else if (IsCompareBranch()) {
-      return CompareBranchType;
-    } else if (IsTestBranch()) {
-      return TestBranchType;
-    } else {
-      return UnknownBranchType;
-    }
-  }
-
-  // Find the target of this instruction. 'this' may be a branch or a
-  // PC-relative addressing instruction.
-  const Instruction* ImmPCOffsetTarget() const;
-
-  // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
-  // a PC-relative addressing instruction.
-  void SetImmPCOffsetTarget(const Instruction* target);
-  // Patch a literal load instruction to load from 'source'.
-  void SetImmLLiteral(const Instruction* source);
-
-  inline uint8_t* LiteralAddress() const {
-    int offset = ImmLLiteral() << kLiteralEntrySizeLog2;
-    const uint8_t* address = reinterpret_cast<const uint8_t*>(this) + offset;
-    // Note that the result is safely mutable only if the backing buffer is
-    // safely mutable.
-    return const_cast<uint8_t*>(address);
-  }
-
-  inline uint32_t Literal32() const {
-    uint32_t literal;
-    memcpy(&literal, LiteralAddress(), sizeof(literal));
-
-    return literal;
-  }
-
-  inline uint64_t Literal64() const {
-    uint64_t literal;
-    memcpy(&literal, LiteralAddress(), sizeof(literal));
-
-    return literal;
-  }
-
-  inline float LiteralFP32() const {
-    return rawbits_to_float(Literal32());
-  }
-
-  inline double LiteralFP64() const {
-    return rawbits_to_double(Literal64());
-  }
-
-  inline const Instruction* NextInstruction() const {
-    return this + kInstructionSize;
-  }
-
-  inline const Instruction* InstructionAtOffset(int64_t offset) const {
-    VIXL_ASSERT(IsWordAligned(this + offset));
-    return this + offset;
-  }
-
-  template<typename T> static inline Instruction* Cast(T src) {
-    return reinterpret_cast<Instruction*>(src);
-  }
-
-  template<typename T> static inline const Instruction* CastConst(T src) {
-    return reinterpret_cast<const Instruction*>(src);
-  }
-
- private:
-  inline int ImmBranch() const;
-
-  void SetPCRelImmTarget(const Instruction* target);
-  void SetBranchImmTarget(const Instruction* target);
-};
-}  // namespace vixl
-
-#endif  // VIXL_A64_INSTRUCTIONS_A64_H_
diff --git a/disas/libvixl/code-buffer.h b/disas/libvixl/code-buffer.h
deleted file mode 100644
index da6233d..0000000
--- a/disas/libvixl/code-buffer.h
+++ /dev/null
@@ -1,113 +0,0 @@
-// Copyright 2014, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_CODE_BUFFER_H
-#define VIXL_CODE_BUFFER_H
-
-#include <string.h>
-#include "globals.h"
-
-namespace vixl {
-
-class CodeBuffer {
- public:
-  explicit CodeBuffer(size_t capacity = 4 * KBytes);
-  CodeBuffer(void* buffer, size_t capacity);
-  ~CodeBuffer();
-
-  void Reset();
-
-  ptrdiff_t OffsetFrom(ptrdiff_t offset) const {
-    ptrdiff_t cursor_offset = cursor_ - buffer_;
-    VIXL_ASSERT((offset >= 0) && (offset <= cursor_offset));
-    return cursor_offset - offset;
-  }
-
-  ptrdiff_t CursorOffset() const {
-    return OffsetFrom(0);
-  }
-
-  template <typename T>
-  T GetOffsetAddress(ptrdiff_t offset) const {
-    VIXL_ASSERT((offset >= 0) && (offset <= (cursor_ - buffer_)));
-    return reinterpret_cast<T>(buffer_ + offset);
-  }
-
-  size_t RemainingBytes() const {
-    VIXL_ASSERT((cursor_ >= buffer_) && (cursor_ <= (buffer_ + capacity_)));
-    return (buffer_ + capacity_) - cursor_;
-  }
-
-  // A code buffer can emit:
-  //  * 32-bit data: instruction and constant.
-  //  * 64-bit data: constant.
-  //  * string: debug info.
-  void Emit32(uint32_t data) { Emit(data); }
-
-  void Emit64(uint64_t data) { Emit(data); }
-
-  void EmitString(const char* string);
-
-  // Align to kInstructionSize.
-  void Align();
-
-  size_t capacity() const { return capacity_; }
-
-  bool IsManaged() const { return managed_; }
-
-  void Grow(size_t new_capacity);
-
-  bool IsDirty() const { return dirty_; }
-
-  void SetClean() { dirty_ = false; }
-
- private:
-  template <typename T>
-  void Emit(T value) {
-    VIXL_ASSERT(RemainingBytes() >= sizeof(value));
-    dirty_ = true;
-    memcpy(cursor_, &value, sizeof(value));
-    cursor_ += sizeof(value);
-  }
-
-  // Backing store of the buffer.
-  byte* buffer_;
-  // If true the backing store is allocated and deallocated by the buffer. The
-  // backing store can then grow on demand. If false the backing store is
-  // provided by the user and cannot be resized internally.
-  bool managed_;
-  // Pointer to the next location to be written.
-  byte* cursor_;
-  // True if there has been any write since the buffer was created or cleaned.
-  bool dirty_;
-  // Capacity in bytes of the backing store.
-  size_t capacity_;
-};
-
-}  // namespace vixl
-
-#endif  // VIXL_CODE_BUFFER_H
-
diff --git a/disas/libvixl/globals.h b/disas/libvixl/globals.h
deleted file mode 100644
index e28dc66..0000000
--- a/disas/libvixl/globals.h
+++ /dev/null
@@ -1,85 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_GLOBALS_H
-#define VIXL_GLOBALS_H
-
-// Get standard C99 macros for integer types.
-#ifndef __STDC_CONSTANT_MACROS
-#define __STDC_CONSTANT_MACROS
-#endif
-
-#ifndef __STDC_LIMIT_MACROS
-#define __STDC_LIMIT_MACROS
-#endif
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <stdint.h>
-#include <inttypes.h>
-
-#include <assert.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include "platform.h"
-
-
-typedef uint8_t byte;
-
-const int KBytes = 1024;
-const int MBytes = 1024 * KBytes;
-
-#define VIXL_ABORT() printf("in %s, line %i", __FILE__, __LINE__); abort()
-#ifdef DEBUG
-  #define VIXL_ASSERT(condition) assert(condition)
-  #define VIXL_CHECK(condition) VIXL_ASSERT(condition)
-  #define VIXL_UNIMPLEMENTED() printf("UNIMPLEMENTED\t"); VIXL_ABORT()
-  #define VIXL_UNREACHABLE() printf("UNREACHABLE\t"); VIXL_ABORT()
-#else
-  #define VIXL_ASSERT(condition) ((void) 0)
-  #define VIXL_CHECK(condition) assert(condition)
-  #define VIXL_UNIMPLEMENTED() ((void) 0)
-  #define VIXL_UNREACHABLE() ((void) 0)
-#endif
-// This is not as powerful as template based assertions, but it is simple.
-// It assumes that the descriptions are unique. If this starts being a problem,
-// we can switch to a different implemention.
-#define VIXL_CONCAT(a, b) a##b
-#define VIXL_STATIC_ASSERT_LINE(line, condition) \
-  typedef char VIXL_CONCAT(STATIC_ASSERT_LINE_, line)[(condition) ? 1 : -1] \
-  __attribute__((unused))
-#define VIXL_STATIC_ASSERT(condition) VIXL_STATIC_ASSERT_LINE(__LINE__, condition) //NOLINT
-
-template <typename T> inline void USE(T) {}
-
-#define VIXL_ALIGNMENT_EXCEPTION() printf("ALIGNMENT EXCEPTION\t"); VIXL_ABORT()
-
-#endif  // VIXL_GLOBALS_H
diff --git a/disas/libvixl/platform.h b/disas/libvixl/platform.h
deleted file mode 100644
index de2b110..0000000
--- a/disas/libvixl/platform.h
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef PLATFORM_H
-#define PLATFORM_H
-
-// Define platform specific functionalities.
-#include <signal.h>
-
-namespace vixl {
-inline void HostBreakpoint() { raise(SIGINT); }
-}  // namespace vixl
-
-#endif
diff --git a/disas/libvixl/utils.cc b/disas/libvixl/utils.cc
deleted file mode 100644
index 21965d7..0000000
--- a/disas/libvixl/utils.cc
+++ /dev/null
@@ -1,138 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include "utils.h"
-#include <stdio.h>
-
-namespace vixl {
-
-uint32_t float_to_rawbits(float value) {
-  uint32_t bits = 0;
-  memcpy(&bits, &value, 4);
-  return bits;
-}
-
-
-uint64_t double_to_rawbits(double value) {
-  uint64_t bits = 0;
-  memcpy(&bits, &value, 8);
-  return bits;
-}
-
-
-float rawbits_to_float(uint32_t bits) {
-  float value = 0.0;
-  memcpy(&value, &bits, 4);
-  return value;
-}
-
-
-double rawbits_to_double(uint64_t bits) {
-  double value = 0.0;
-  memcpy(&value, &bits, 8);
-  return value;
-}
-
-
-int CountLeadingZeros(uint64_t value, int width) {
-  VIXL_ASSERT((width == 32) || (width == 64));
-  int count = 0;
-  uint64_t bit_test = UINT64_C(1) << (width - 1);
-  while ((count < width) && ((bit_test & value) == 0)) {
-    count++;
-    bit_test >>= 1;
-  }
-  return count;
-}
-
-
-int CountLeadingSignBits(int64_t value, int width) {
-  VIXL_ASSERT((width == 32) || (width == 64));
-  if (value >= 0) {
-    return CountLeadingZeros(value, width) - 1;
-  } else {
-    return CountLeadingZeros(~value, width) - 1;
-  }
-}
-
-
-int CountTrailingZeros(uint64_t value, int width) {
-  VIXL_ASSERT((width == 32) || (width == 64));
-  int count = 0;
-  while ((count < width) && (((value >> count) & 1) == 0)) {
-    count++;
-  }
-  return count;
-}
-
-
-int CountSetBits(uint64_t value, int width) {
-  // TODO: Other widths could be added here, as the implementation already
-  // supports them.
-  VIXL_ASSERT((width == 32) || (width == 64));
-
-  // Mask out unused bits to ensure that they are not counted.
-  value &= (UINT64_C(0xffffffffffffffff) >> (64-width));
-
-  // Add up the set bits.
-  // The algorithm works by adding pairs of bit fields together iteratively,
-  // where the size of each bit field doubles each time.
-  // An example for an 8-bit value:
-  // Bits:  h  g  f  e  d  c  b  a
-  //         \ |   \ |   \ |   \ |
-  // value = h+g   f+e   d+c   b+a
-  //            \    |      \    |
-  // value =   h+g+f+e     d+c+b+a
-  //                  \          |
-  // value =       h+g+f+e+d+c+b+a
-  const uint64_t kMasks[] = {
-    UINT64_C(0x5555555555555555),
-    UINT64_C(0x3333333333333333),
-    UINT64_C(0x0f0f0f0f0f0f0f0f),
-    UINT64_C(0x00ff00ff00ff00ff),
-    UINT64_C(0x0000ffff0000ffff),
-    UINT64_C(0x00000000ffffffff),
-  };
-
-  for (unsigned i = 0; i < (sizeof(kMasks) / sizeof(kMasks[0])); i++) {
-    int shift = 1 << i;
-    value = ((value >> shift) & kMasks[i]) + (value & kMasks[i]);
-  }
-
-  return value;
-}
-
-
-uint64_t LowestSetBit(uint64_t value) {
-  return value & -value;
-}
-
-
-bool IsPowerOf2(int64_t value) {
-  return (value != 0) && ((value & (value - 1)) == 0);
-}
-
-}  // namespace vixl
diff --git a/disas/libvixl/utils.h b/disas/libvixl/utils.h
deleted file mode 100644
index 1540c30..0000000
--- a/disas/libvixl/utils.h
+++ /dev/null
@@ -1,209 +0,0 @@
-// Copyright 2013, ARM Limited
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions are met:
-//
-//   * Redistributions of source code must retain the above copyright notice,
-//     this list of conditions and the following disclaimer.
-//   * Redistributions in binary form must reproduce the above copyright notice,
-//     this list of conditions and the following disclaimer in the documentation
-//     and/or other materials provided with the distribution.
-//   * Neither the name of ARM Limited nor the names of its contributors may be
-//     used to endorse or promote products derived from this software without
-//     specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
-// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifndef VIXL_UTILS_H
-#define VIXL_UTILS_H
-
-#include <math.h>
-#include <string.h>
-#include "globals.h"
-
-namespace vixl {
-
-// Macros for compile-time format checking.
-#if defined(__GNUC__)
-#define PRINTF_CHECK(format_index, varargs_index) \
-  __attribute__((format(printf, format_index, varargs_index)))
-#else
-#define PRINTF_CHECK(format_index, varargs_index)
-#endif
-
-// Check number width.
-inline bool is_intn(unsigned n, int64_t x) {
-  VIXL_ASSERT((0 < n) && (n < 64));
-  int64_t limit = INT64_C(1) << (n - 1);
-  return (-limit <= x) && (x < limit);
-}
-
-inline bool is_uintn(unsigned n, int64_t x) {
-  VIXL_ASSERT((0 < n) && (n < 64));
-  return !(x >> n);
-}
-
-inline unsigned truncate_to_intn(unsigned n, int64_t x) {
-  VIXL_ASSERT((0 < n) && (n < 64));
-  return (x & ((INT64_C(1) << n) - 1));
-}
-
-#define INT_1_TO_63_LIST(V)                                                    \
-V(1)  V(2)  V(3)  V(4)  V(5)  V(6)  V(7)  V(8)                                 \
-V(9)  V(10) V(11) V(12) V(13) V(14) V(15) V(16)                                \
-V(17) V(18) V(19) V(20) V(21) V(22) V(23) V(24)                                \
-V(25) V(26) V(27) V(28) V(29) V(30) V(31) V(32)                                \
-V(33) V(34) V(35) V(36) V(37) V(38) V(39) V(40)                                \
-V(41) V(42) V(43) V(44) V(45) V(46) V(47) V(48)                                \
-V(49) V(50) V(51) V(52) V(53) V(54) V(55) V(56)                                \
-V(57) V(58) V(59) V(60) V(61) V(62) V(63)
-
-#define DECLARE_IS_INT_N(N)                                                    \
-inline bool is_int##N(int64_t x) { return is_intn(N, x); }
-#define DECLARE_IS_UINT_N(N)                                                   \
-inline bool is_uint##N(int64_t x) { return is_uintn(N, x); }
-#define DECLARE_TRUNCATE_TO_INT_N(N)                                           \
-inline int truncate_to_int##N(int x) { return truncate_to_intn(N, x); }
-INT_1_TO_63_LIST(DECLARE_IS_INT_N)
-INT_1_TO_63_LIST(DECLARE_IS_UINT_N)
-INT_1_TO_63_LIST(DECLARE_TRUNCATE_TO_INT_N)
-#undef DECLARE_IS_INT_N
-#undef DECLARE_IS_UINT_N
-#undef DECLARE_TRUNCATE_TO_INT_N
-
-// Bit field extraction.
-inline uint32_t unsigned_bitextract_32(int msb, int lsb, uint32_t x) {
-  return (x >> lsb) & ((1 << (1 + msb - lsb)) - 1);
-}
-
-inline uint64_t unsigned_bitextract_64(int msb, int lsb, uint64_t x) {
-  return (x >> lsb) & ((static_cast<uint64_t>(1) << (1 + msb - lsb)) - 1);
-}
-
-inline int32_t signed_bitextract_32(int msb, int lsb, int32_t x) {
-  return (x << (31 - msb)) >> (lsb + 31 - msb);
-}
-
-inline int64_t signed_bitextract_64(int msb, int lsb, int64_t x) {
-  return (x << (63 - msb)) >> (lsb + 63 - msb);
-}
-
-// Floating point representation.
-uint32_t float_to_rawbits(float value);
-uint64_t double_to_rawbits(double value);
-float rawbits_to_float(uint32_t bits);
-double rawbits_to_double(uint64_t bits);
-
-
-// NaN tests.
-inline bool IsSignallingNaN(double num) {
-  const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
-  uint64_t raw = double_to_rawbits(num);
-  if (isnan(num) && ((raw & kFP64QuietNaNMask) == 0)) {
-    return true;
-  }
-  return false;
-}
-
-
-inline bool IsSignallingNaN(float num) {
-  const uint32_t kFP32QuietNaNMask = 0x00400000;
-  uint32_t raw = float_to_rawbits(num);
-  if (isnan(num) && ((raw & kFP32QuietNaNMask) == 0)) {
-    return true;
-  }
-  return false;
-}
-
-
-template <typename T>
-inline bool IsQuietNaN(T num) {
-  return isnan(num) && !IsSignallingNaN(num);
-}
-
-
-// Convert the NaN in 'num' to a quiet NaN.
-inline double ToQuietNaN(double num) {
-  const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
-  VIXL_ASSERT(isnan(num));
-  return rawbits_to_double(double_to_rawbits(num) | kFP64QuietNaNMask);
-}
-
-
-inline float ToQuietNaN(float num) {
-  const uint32_t kFP32QuietNaNMask = 0x00400000;
-  VIXL_ASSERT(isnan(num));
-  return rawbits_to_float(float_to_rawbits(num) | kFP32QuietNaNMask);
-}
-
-
-// Fused multiply-add.
-inline double FusedMultiplyAdd(double op1, double op2, double a) {
-  return fma(op1, op2, a);
-}
-
-
-inline float FusedMultiplyAdd(float op1, float op2, float a) {
-  return fmaf(op1, op2, a);
-}
-
-
-// Bit counting.
-int CountLeadingZeros(uint64_t value, int width);
-int CountLeadingSignBits(int64_t value, int width);
-int CountTrailingZeros(uint64_t value, int width);
-int CountSetBits(uint64_t value, int width);
-uint64_t LowestSetBit(uint64_t value);
-bool IsPowerOf2(int64_t value);
-
-// Pointer alignment
-// TODO: rename/refactor to make it specific to instructions.
-template<typename T>
-bool IsWordAligned(T pointer) {
-  VIXL_ASSERT(sizeof(pointer) == sizeof(intptr_t));   // NOLINT(runtime/sizeof)
-  return ((intptr_t)(pointer) & 3) == 0;
-}
-
-// Increment a pointer until it has the specified alignment.
-template<class T>
-T AlignUp(T pointer, size_t alignment) {
-  // Use C-style casts to get static_cast behaviour for integral types (T), and
-  // reinterpret_cast behaviour for other types.
-
-  uintptr_t pointer_raw = (uintptr_t)pointer;
-  VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(pointer_raw));
-
-  size_t align_step = (alignment - pointer_raw) % alignment;
-  VIXL_ASSERT((pointer_raw + align_step) % alignment == 0);
-
-  return (T)(pointer_raw + align_step);
-}
-
-// Decrement a pointer until it has the specified alignment.
-template<class T>
-T AlignDown(T pointer, size_t alignment) {
-  // Use C-style casts to get static_cast behaviour for integral types (T), and
-  // reinterpret_cast behaviour for other types.
-
-  uintptr_t pointer_raw = (uintptr_t)pointer;
-  VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(pointer_raw));
-
-  size_t align_step = pointer_raw % alignment;
-  VIXL_ASSERT((pointer_raw - align_step) % alignment == 0);
-
-  return (T)(pointer_raw - align_step);
-}
-
-}  // namespace vixl
-
-#endif  // VIXL_UTILS_H
diff --git a/disas/libvixl/vixl/a64/assembler-a64.h b/disas/libvixl/vixl/a64/assembler-a64.h
new file mode 100644
index 0000000..fda5ccc
--- /dev/null
+++ b/disas/libvixl/vixl/a64/assembler-a64.h
@@ -0,0 +1,4624 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_ASSEMBLER_A64_H_
+#define VIXL_A64_ASSEMBLER_A64_H_
+
+
+#include "vixl/globals.h"
+#include "vixl/invalset.h"
+#include "vixl/utils.h"
+#include "vixl/code-buffer.h"
+#include "vixl/a64/instructions-a64.h"
+
+namespace vixl {
+
+typedef uint64_t RegList;
+static const int kRegListSizeInBits = sizeof(RegList) * 8;
+
+
+// Registers.
+
+// Some CPURegister methods can return Register or VRegister types, so we need
+// to declare them in advance.
+class Register;
+class VRegister;
+
+class CPURegister {
+ public:
+  enum RegisterType {
+    // The kInvalid value is used to detect uninitialized static instances,
+    // which are always zero-initialized before any constructors are called.
+    kInvalid = 0,
+    kRegister,
+    kVRegister,
+    kFPRegister = kVRegister,
+    kNoRegister
+  };
+
+  CPURegister() : code_(0), size_(0), type_(kNoRegister) {
+    VIXL_ASSERT(!IsValid());
+    VIXL_ASSERT(IsNone());
+  }
+
+  CPURegister(unsigned code, unsigned size, RegisterType type)
+      : code_(code), size_(size), type_(type) {
+    VIXL_ASSERT(IsValidOrNone());
+  }
+
+  unsigned code() const {
+    VIXL_ASSERT(IsValid());
+    return code_;
+  }
+
+  RegisterType type() const {
+    VIXL_ASSERT(IsValidOrNone());
+    return type_;
+  }
+
+  RegList Bit() const {
+    VIXL_ASSERT(code_ < (sizeof(RegList) * 8));
+    return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
+  }
+
+  unsigned size() const {
+    VIXL_ASSERT(IsValid());
+    return size_;
+  }
+
+  int SizeInBytes() const {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(size() % 8 == 0);
+    return size_ / 8;
+  }
+
+  int SizeInBits() const {
+    VIXL_ASSERT(IsValid());
+    return size_;
+  }
+
+  bool Is8Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 8;
+  }
+
+  bool Is16Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 16;
+  }
+
+  bool Is32Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 32;
+  }
+
+  bool Is64Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 64;
+  }
+
+  bool Is128Bits() const {
+    VIXL_ASSERT(IsValid());
+    return size_ == 128;
+  }
+
+  bool IsValid() const {
+    if (IsValidRegister() || IsValidVRegister()) {
+      VIXL_ASSERT(!IsNone());
+      return true;
+    } else {
+      // This assert is hit when the register has not been properly initialized.
+      // One cause for this can be an initialisation order fiasco. See
+      // https://isocpp.org/wiki/faq/ctors#static-init-order for some details.
+      VIXL_ASSERT(IsNone());
+      return false;
+    }
+  }
+
+  bool IsValidRegister() const {
+    return IsRegister() &&
+           ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
+           ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
+  }
+
+  bool IsValidVRegister() const {
+    return IsVRegister() &&
+           ((size_ == kBRegSize) || (size_ == kHRegSize) ||
+            (size_ == kSRegSize) || (size_ == kDRegSize) ||
+            (size_ == kQRegSize)) &&
+           (code_ < kNumberOfVRegisters);
+  }
+
+  bool IsValidFPRegister() const {
+    return IsFPRegister() && (code_ < kNumberOfVRegisters);
+  }
+
+  bool IsNone() const {
+    // kNoRegister types should always have size 0 and code 0.
+    VIXL_ASSERT((type_ != kNoRegister) || (code_ == 0));
+    VIXL_ASSERT((type_ != kNoRegister) || (size_ == 0));
+
+    return type_ == kNoRegister;
+  }
+
+  bool Aliases(const CPURegister& other) const {
+    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
+    return (code_ == other.code_) && (type_ == other.type_);
+  }
+
+  bool Is(const CPURegister& other) const {
+    VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
+    return Aliases(other) && (size_ == other.size_);
+  }
+
+  bool IsZero() const {
+    VIXL_ASSERT(IsValid());
+    return IsRegister() && (code_ == kZeroRegCode);
+  }
+
+  bool IsSP() const {
+    VIXL_ASSERT(IsValid());
+    return IsRegister() && (code_ == kSPRegInternalCode);
+  }
+
+  bool IsRegister() const {
+    return type_ == kRegister;
+  }
+
+  bool IsVRegister() const {
+    return type_ == kVRegister;
+  }
+
+  bool IsFPRegister() const {
+    return IsS() || IsD();
+  }
+
+  bool IsW() const { return IsValidRegister() && Is32Bits(); }
+  bool IsX() const { return IsValidRegister() && Is64Bits(); }
+
+  // These assertions ensure that the size and type of the register are as
+  // described. They do not consider the number of lanes that make up a vector.
+  // So, for example, Is8B() implies IsD(), and Is1D() implies IsD, but IsD()
+  // does not imply Is1D() or Is8B().
+  // Check the number of lanes, ie. the format of the vector, using methods such
+  // as Is8B(), Is1D(), etc. in the VRegister class.
+  bool IsV() const { return IsVRegister(); }
+  bool IsB() const { return IsV() && Is8Bits(); }
+  bool IsH() const { return IsV() && Is16Bits(); }
+  bool IsS() const { return IsV() && Is32Bits(); }
+  bool IsD() const { return IsV() && Is64Bits(); }
+  bool IsQ() const { return IsV() && Is128Bits(); }
+
+  const Register& W() const;
+  const Register& X() const;
+  const VRegister& V() const;
+  const VRegister& B() const;
+  const VRegister& H() const;
+  const VRegister& S() const;
+  const VRegister& D() const;
+  const VRegister& Q() const;
+
+  bool IsSameSizeAndType(const CPURegister& other) const {
+    return (size_ == other.size_) && (type_ == other.type_);
+  }
+
+ protected:
+  unsigned code_;
+  unsigned size_;
+  RegisterType type_;
+
+ private:
+  bool IsValidOrNone() const {
+    return IsValid() || IsNone();
+  }
+};
+
+
+class Register : public CPURegister {
+ public:
+  Register() : CPURegister() {}
+  explicit Register(const CPURegister& other)
+      : CPURegister(other.code(), other.size(), other.type()) {
+    VIXL_ASSERT(IsValidRegister());
+  }
+  Register(unsigned code, unsigned size)
+      : CPURegister(code, size, kRegister) {}
+
+  bool IsValid() const {
+    VIXL_ASSERT(IsRegister() || IsNone());
+    return IsValidRegister();
+  }
+
+  static const Register& WRegFromCode(unsigned code);
+  static const Register& XRegFromCode(unsigned code);
+
+ private:
+  static const Register wregisters[];
+  static const Register xregisters[];
+};
+
+
+class VRegister : public CPURegister {
+ public:
+  VRegister() : CPURegister(), lanes_(1) {}
+  explicit VRegister(const CPURegister& other)
+      : CPURegister(other.code(), other.size(), other.type()), lanes_(1) {
+    VIXL_ASSERT(IsValidVRegister());
+    VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+  }
+  VRegister(unsigned code, unsigned size, unsigned lanes = 1)
+      : CPURegister(code, size, kVRegister), lanes_(lanes) {
+    VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+  }
+  VRegister(unsigned code, VectorFormat format)
+      : CPURegister(code, RegisterSizeInBitsFromFormat(format), kVRegister),
+        lanes_(IsVectorFormat(format) ? LaneCountFromFormat(format) : 1) {
+    VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+  }
+
+  bool IsValid() const {
+    VIXL_ASSERT(IsVRegister() || IsNone());
+    return IsValidVRegister();
+  }
+
+  static const VRegister& BRegFromCode(unsigned code);
+  static const VRegister& HRegFromCode(unsigned code);
+  static const VRegister& SRegFromCode(unsigned code);
+  static const VRegister& DRegFromCode(unsigned code);
+  static const VRegister& QRegFromCode(unsigned code);
+  static const VRegister& VRegFromCode(unsigned code);
+
+  VRegister V8B() const { return VRegister(code_, kDRegSize, 8); }
+  VRegister V16B() const { return VRegister(code_, kQRegSize, 16); }
+  VRegister V4H() const { return VRegister(code_, kDRegSize, 4); }
+  VRegister V8H() const { return VRegister(code_, kQRegSize, 8); }
+  VRegister V2S() const { return VRegister(code_, kDRegSize, 2); }
+  VRegister V4S() const { return VRegister(code_, kQRegSize, 4); }
+  VRegister V2D() const { return VRegister(code_, kQRegSize, 2); }
+  VRegister V1D() const { return VRegister(code_, kDRegSize, 1); }
+
+  bool Is8B() const { return (Is64Bits() && (lanes_ == 8)); }
+  bool Is16B() const { return (Is128Bits() && (lanes_ == 16)); }
+  bool Is4H() const { return (Is64Bits() && (lanes_ == 4)); }
+  bool Is8H() const { return (Is128Bits() && (lanes_ == 8)); }
+  bool Is2S() const { return (Is64Bits() && (lanes_ == 2)); }
+  bool Is4S() const { return (Is128Bits() && (lanes_ == 4)); }
+  bool Is1D() const { return (Is64Bits() && (lanes_ == 1)); }
+  bool Is2D() const { return (Is128Bits() && (lanes_ == 2)); }
+
+  // For consistency, we assert the number of lanes of these scalar registers,
+  // even though there are no vectors of equivalent total size with which they
+  // could alias.
+  bool Is1B() const {
+    VIXL_ASSERT(!(Is8Bits() && IsVector()));
+    return Is8Bits();
+  }
+  bool Is1H() const {
+    VIXL_ASSERT(!(Is16Bits() && IsVector()));
+    return Is16Bits();
+  }
+  bool Is1S() const {
+    VIXL_ASSERT(!(Is32Bits() && IsVector()));
+    return Is32Bits();
+  }
+
+  bool IsLaneSizeB() const { return LaneSizeInBits() == kBRegSize; }
+  bool IsLaneSizeH() const { return LaneSizeInBits() == kHRegSize; }
+  bool IsLaneSizeS() const { return LaneSizeInBits() == kSRegSize; }
+  bool IsLaneSizeD() const { return LaneSizeInBits() == kDRegSize; }
+
+  int lanes() const {
+    return lanes_;
+  }
+
+  bool IsScalar() const {
+    return lanes_ == 1;
+  }
+
+  bool IsVector() const {
+    return lanes_ > 1;
+  }
+
+  bool IsSameFormat(const VRegister& other) const {
+    return (size_ == other.size_) && (lanes_ == other.lanes_);
+  }
+
+  unsigned LaneSizeInBytes() const {
+    return SizeInBytes() / lanes_;
+  }
+
+  unsigned LaneSizeInBits() const {
+    return LaneSizeInBytes() * 8;
+  }
+
+ private:
+  static const VRegister bregisters[];
+  static const VRegister hregisters[];
+  static const VRegister sregisters[];
+  static const VRegister dregisters[];
+  static const VRegister qregisters[];
+  static const VRegister vregisters[];
+  int lanes_;
+};
+
+
+// Backward compatibility for FPRegisters.
+typedef VRegister FPRegister;
+
+// No*Reg is used to indicate an unused argument, or an error case. Note that
+// these all compare equal (using the Is() method). The Register and VRegister
+// variants are provided for convenience.
+const Register NoReg;
+const VRegister NoVReg;
+const FPRegister NoFPReg;  // For backward compatibility.
+const CPURegister NoCPUReg;
+
+
+#define DEFINE_REGISTERS(N)  \
+const Register w##N(N, kWRegSize);  \
+const Register x##N(N, kXRegSize);
+REGISTER_CODE_LIST(DEFINE_REGISTERS)
+#undef DEFINE_REGISTERS
+const Register wsp(kSPRegInternalCode, kWRegSize);
+const Register sp(kSPRegInternalCode, kXRegSize);
+
+
+#define DEFINE_VREGISTERS(N)  \
+const VRegister b##N(N, kBRegSize);  \
+const VRegister h##N(N, kHRegSize);  \
+const VRegister s##N(N, kSRegSize);  \
+const VRegister d##N(N, kDRegSize);  \
+const VRegister q##N(N, kQRegSize);  \
+const VRegister v##N(N, kQRegSize);
+REGISTER_CODE_LIST(DEFINE_VREGISTERS)
+#undef DEFINE_VREGISTERS
+
+
+// Registers aliases.
+const Register ip0 = x16;
+const Register ip1 = x17;
+const Register lr = x30;
+const Register xzr = x31;
+const Register wzr = w31;
+
+
+// AreAliased returns true if any of the named registers overlap. Arguments
+// set to NoReg are ignored. The system stack pointer may be specified.
+bool AreAliased(const CPURegister& reg1,
+                const CPURegister& reg2,
+                const CPURegister& reg3 = NoReg,
+                const CPURegister& reg4 = NoReg,
+                const CPURegister& reg5 = NoReg,
+                const CPURegister& reg6 = NoReg,
+                const CPURegister& reg7 = NoReg,
+                const CPURegister& reg8 = NoReg);
+
+
+// AreSameSizeAndType returns true if all of the specified registers have the
+// same size, and are of the same type. The system stack pointer may be
+// specified. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
+bool AreSameSizeAndType(const CPURegister& reg1,
+                        const CPURegister& reg2,
+                        const CPURegister& reg3 = NoCPUReg,
+                        const CPURegister& reg4 = NoCPUReg,
+                        const CPURegister& reg5 = NoCPUReg,
+                        const CPURegister& reg6 = NoCPUReg,
+                        const CPURegister& reg7 = NoCPUReg,
+                        const CPURegister& reg8 = NoCPUReg);
+
+
+// AreSameFormat returns true if all of the specified VRegisters have the same
+// vector format. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoVReg).
+bool AreSameFormat(const VRegister& reg1,
+                   const VRegister& reg2,
+                   const VRegister& reg3 = NoVReg,
+                   const VRegister& reg4 = NoVReg);
+
+
+// AreConsecutive returns true if all of the specified VRegisters are
+// consecutive in the register file. Arguments set to NoReg are ignored, as are
+// any subsequent arguments. At least one argument (reg1) must be valid
+// (not NoVReg).
+bool AreConsecutive(const VRegister& reg1,
+                    const VRegister& reg2,
+                    const VRegister& reg3 = NoVReg,
+                    const VRegister& reg4 = NoVReg);
+
+
+// Lists of registers.
+class CPURegList {
+ public:
+  explicit CPURegList(CPURegister reg1,
+                      CPURegister reg2 = NoCPUReg,
+                      CPURegister reg3 = NoCPUReg,
+                      CPURegister reg4 = NoCPUReg)
+      : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
+        size_(reg1.size()), type_(reg1.type()) {
+    VIXL_ASSERT(AreSameSizeAndType(reg1, reg2, reg3, reg4));
+    VIXL_ASSERT(IsValid());
+  }
+
+  CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
+      : list_(list), size_(size), type_(type) {
+    VIXL_ASSERT(IsValid());
+  }
+
+  CPURegList(CPURegister::RegisterType type, unsigned size,
+             unsigned first_reg, unsigned last_reg)
+      : size_(size), type_(type) {
+    VIXL_ASSERT(((type == CPURegister::kRegister) &&
+                 (last_reg < kNumberOfRegisters)) ||
+                ((type == CPURegister::kVRegister) &&
+                 (last_reg < kNumberOfVRegisters)));
+    VIXL_ASSERT(last_reg >= first_reg);
+    list_ = (UINT64_C(1) << (last_reg + 1)) - 1;
+    list_ &= ~((UINT64_C(1) << first_reg) - 1);
+    VIXL_ASSERT(IsValid());
+  }
+
+  CPURegister::RegisterType type() const {
+    VIXL_ASSERT(IsValid());
+    return type_;
+  }
+
+  // Combine another CPURegList into this one. Registers that already exist in
+  // this list are left unchanged. The type and size of the registers in the
+  // 'other' list must match those in this list.
+  void Combine(const CPURegList& other) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
+    list_ |= other.list();
+  }
+
+  // Remove every register in the other CPURegList from this one. Registers that
+  // do not exist in this list are ignored. The type and size of the registers
+  // in the 'other' list must match those in this list.
+  void Remove(const CPURegList& other) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.RegisterSizeInBits() == size_);
+    list_ &= ~other.list();
+  }
+
+  // Variants of Combine and Remove which take a single register.
+  void Combine(const CPURegister& other) {
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.size() == size_);
+    Combine(other.code());
+  }
+
+  void Remove(const CPURegister& other) {
+    VIXL_ASSERT(other.type() == type_);
+    VIXL_ASSERT(other.size() == size_);
+    Remove(other.code());
+  }
+
+  // Variants of Combine and Remove which take a single register by its code;
+  // the type and size of the register is inferred from this list.
+  void Combine(int code) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
+    list_ |= (UINT64_C(1) << code);
+  }
+
+  void Remove(int code) {
+    VIXL_ASSERT(IsValid());
+    VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
+    list_ &= ~(UINT64_C(1) << code);
+  }
+
+  static CPURegList Union(const CPURegList& list_1, const CPURegList& list_2) {
+    VIXL_ASSERT(list_1.type_ == list_2.type_);
+    VIXL_ASSERT(list_1.size_ == list_2.size_);
+    return CPURegList(list_1.type_, list_1.size_, list_1.list_ | list_2.list_);
+  }
+  static CPURegList Union(const CPURegList& list_1,
+                          const CPURegList& list_2,
+                          const CPURegList& list_3);
+  static CPURegList Union(const CPURegList& list_1,
+                          const CPURegList& list_2,
+                          const CPURegList& list_3,
+                          const CPURegList& list_4);
+
+  static CPURegList Intersection(const CPURegList& list_1,
+                                 const CPURegList& list_2) {
+    VIXL_ASSERT(list_1.type_ == list_2.type_);
+    VIXL_ASSERT(list_1.size_ == list_2.size_);
+    return CPURegList(list_1.type_, list_1.size_, list_1.list_ & list_2.list_);
+  }
+  static CPURegList Intersection(const CPURegList& list_1,
+                                 const CPURegList& list_2,
+                                 const CPURegList& list_3);
+  static CPURegList Intersection(const CPURegList& list_1,
+                                 const CPURegList& list_2,
+                                 const CPURegList& list_3,
+                                 const CPURegList& list_4);
+
+  bool Overlaps(const CPURegList& other) const {
+    return (type_ == other.type_) && ((list_ & other.list_) != 0);
+  }
+
+  RegList list() const {
+    VIXL_ASSERT(IsValid());
+    return list_;
+  }
+
+  void set_list(RegList new_list) {
+    VIXL_ASSERT(IsValid());
+    list_ = new_list;
+  }
+
+  // Remove all callee-saved registers from the list. This can be useful when
+  // preparing registers for an AAPCS64 function call, for example.
+  void RemoveCalleeSaved();
+
+  CPURegister PopLowestIndex();
+  CPURegister PopHighestIndex();
+
+  // AAPCS64 callee-saved registers.
+  static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
+  static CPURegList GetCalleeSavedV(unsigned size = kDRegSize);
+
+  // AAPCS64 caller-saved registers. Note that this includes lr.
+  // TODO(all): Determine how we handle d8-d15 being callee-saved, but the top
+  // 64-bits being caller-saved.
+  static CPURegList GetCallerSaved(unsigned size = kXRegSize);
+  static CPURegList GetCallerSavedV(unsigned size = kDRegSize);
+
+  bool IsEmpty() const {
+    VIXL_ASSERT(IsValid());
+    return list_ == 0;
+  }
+
+  bool IncludesAliasOf(const CPURegister& other) const {
+    VIXL_ASSERT(IsValid());
+    return (type_ == other.type()) && ((other.Bit() & list_) != 0);
+  }
+
+  bool IncludesAliasOf(int code) const {
+    VIXL_ASSERT(IsValid());
+    return ((code & list_) != 0);
+  }
+
+  int Count() const {
+    VIXL_ASSERT(IsValid());
+    return CountSetBits(list_);
+  }
+
+  unsigned RegisterSizeInBits() const {
+    VIXL_ASSERT(IsValid());
+    return size_;
+  }
+
+  unsigned RegisterSizeInBytes() const {
+    int size_in_bits = RegisterSizeInBits();
+    VIXL_ASSERT((size_in_bits % 8) == 0);
+    return size_in_bits / 8;
+  }
+
+  unsigned TotalSizeInBytes() const {
+    VIXL_ASSERT(IsValid());
+    return RegisterSizeInBytes() * Count();
+  }
+
+ private:
+  RegList list_;
+  unsigned size_;
+  CPURegister::RegisterType type_;
+
+  bool IsValid() const;
+};
+
+
+// AAPCS64 callee-saved registers.
+extern const CPURegList kCalleeSaved;
+extern const CPURegList kCalleeSavedV;
+
+
+// AAPCS64 caller-saved registers. Note that this includes lr.
+extern const CPURegList kCallerSaved;
+extern const CPURegList kCallerSavedV;
+
+
+// Operand.
+class Operand {
+ public:
+  // #<immediate>
+  // where <immediate> is int64_t.
+  // This is allowed to be an implicit constructor because Operand is
+  // a wrapper class that doesn't normally perform any type conversion.
+  Operand(int64_t immediate = 0);           // NOLINT(runtime/explicit)
+
+  // rm, {<shift> #<shift_amount>}
+  // where <shift> is one of {LSL, LSR, ASR, ROR}.
+  //       <shift_amount> is uint6_t.
+  // This is allowed to be an implicit constructor because Operand is
+  // a wrapper class that doesn't normally perform any type conversion.
+  Operand(Register reg,
+          Shift shift = LSL,
+          unsigned shift_amount = 0);   // NOLINT(runtime/explicit)
+
+  // rm, {<extend> {#<shift_amount>}}
+  // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
+  //       <shift_amount> is uint2_t.
+  explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
+
+  bool IsImmediate() const;
+  bool IsShiftedRegister() const;
+  bool IsExtendedRegister() const;
+  bool IsZero() const;
+
+  // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
+  // which helps in the encoding of instructions that use the stack pointer.
+  Operand ToExtendedRegister() const;
+
+  int64_t immediate() const {
+    VIXL_ASSERT(IsImmediate());
+    return immediate_;
+  }
+
+  Register reg() const {
+    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
+    return reg_;
+  }
+
+  Shift shift() const {
+    VIXL_ASSERT(IsShiftedRegister());
+    return shift_;
+  }
+
+  Extend extend() const {
+    VIXL_ASSERT(IsExtendedRegister());
+    return extend_;
+  }
+
+  unsigned shift_amount() const {
+    VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
+    return shift_amount_;
+  }
+
+ private:
+  int64_t immediate_;
+  Register reg_;
+  Shift shift_;
+  Extend extend_;
+  unsigned shift_amount_;
+};
+
+
+// MemOperand represents the addressing mode of a load or store instruction.
+class MemOperand {
+ public:
+  explicit MemOperand(Register base,
+                      int64_t offset = 0,
+                      AddrMode addrmode = Offset);
+  MemOperand(Register base,
+             Register regoffset,
+             Shift shift = LSL,
+             unsigned shift_amount = 0);
+  MemOperand(Register base,
+             Register regoffset,
+             Extend extend,
+             unsigned shift_amount = 0);
+  MemOperand(Register base,
+             const Operand& offset,
+             AddrMode addrmode = Offset);
+
+  const Register& base() const { return base_; }
+  const Register& regoffset() const { return regoffset_; }
+  int64_t offset() const { return offset_; }
+  AddrMode addrmode() const { return addrmode_; }
+  Shift shift() const { return shift_; }
+  Extend extend() const { return extend_; }
+  unsigned shift_amount() const { return shift_amount_; }
+  bool IsImmediateOffset() const;
+  bool IsRegisterOffset() const;
+  bool IsPreIndex() const;
+  bool IsPostIndex() const;
+
+  void AddOffset(int64_t offset);
+
+ private:
+  Register base_;
+  Register regoffset_;
+  int64_t offset_;
+  AddrMode addrmode_;
+  Shift shift_;
+  Extend extend_;
+  unsigned shift_amount_;
+};
+
+
+class LabelTestHelper;  // Forward declaration.
+
+
+class Label {
+ public:
+  Label() : location_(kLocationUnbound) {}
+  ~Label() {
+    // If the label has been linked to, it needs to be bound to a target.
+    VIXL_ASSERT(!IsLinked() || IsBound());
+  }
+
+  bool IsBound() const { return location_ >= 0; }
+  bool IsLinked() const { return !links_.empty(); }
+
+  ptrdiff_t location() const { return location_; }
+
+  static const int kNPreallocatedLinks = 4;
+  static const ptrdiff_t kInvalidLinkKey = PTRDIFF_MAX;
+  static const size_t kReclaimFrom = 512;
+  static const size_t kReclaimFactor = 2;
+
+  typedef InvalSet<ptrdiff_t,
+                   kNPreallocatedLinks,
+                   ptrdiff_t,
+                   kInvalidLinkKey,
+                   kReclaimFrom,
+                   kReclaimFactor> LinksSetBase;
+  typedef InvalSetIterator<LinksSetBase> LabelLinksIteratorBase;
+
+ private:
+  class LinksSet : public LinksSetBase {
+   public:
+    LinksSet() : LinksSetBase() {}
+  };
+
+  // Allows iterating over the links of a label. The behaviour is undefined if
+  // the list of links is modified in any way while iterating.
+  class LabelLinksIterator : public LabelLinksIteratorBase {
+   public:
+    explicit LabelLinksIterator(Label* label)
+        : LabelLinksIteratorBase(&label->links_) {}
+  };
+
+  void Bind(ptrdiff_t location) {
+    // Labels can only be bound once.
+    VIXL_ASSERT(!IsBound());
+    location_ = location;
+  }
+
+  void AddLink(ptrdiff_t instruction) {
+    // If a label is bound, the assembler already has the information it needs
+    // to write the instruction, so there is no need to add it to links_.
+    VIXL_ASSERT(!IsBound());
+    links_.insert(instruction);
+  }
+
+  void DeleteLink(ptrdiff_t instruction) {
+    links_.erase(instruction);
+  }
+
+  void ClearAllLinks() {
+    links_.clear();
+  }
+
+  // TODO: The comment below considers average case complexity for our
+  // usual use-cases. The elements of interest are:
+  // - Branches to a label are emitted in order: branch instructions to a label
+  // are generated at an offset in the code generation buffer greater than any
+  // other branch to that same label already generated. As an example, this can
+  // be broken when an instruction is patched to become a branch. Note that the
+  // code will still work, but the complexity considerations below may locally
+  // not apply any more.
+  // - Veneers are generated in order: for multiple branches of the same type
+  // branching to the same unbound label going out of range, veneers are
+  // generated in growing order of the branch instruction offset from the start
+  // of the buffer.
+  //
+  // When creating a veneer for a branch going out of range, the link for this
+  // branch needs to be removed from this `links_`. Since all branches are
+  // tracked in one underlying InvalSet, the complexity for this deletion is the
+  // same as for finding the element, ie. O(n), where n is the number of links
+  // in the set.
+  // This could be reduced to O(1) by using the same trick as used when tracking
+  // branch information for veneers: split the container to use one set per type
+  // of branch. With that setup, when a veneer is created and the link needs to
+  // be deleted, if the two points above hold, it must be the minimum element of
+  // the set for its type of branch, and that minimum element will be accessible
+  // in O(1).
+
+  // The offsets of the instructions that have linked to this label.
+  LinksSet links_;
+  // The label location.
+  ptrdiff_t location_;
+
+  static const ptrdiff_t kLocationUnbound = -1;
+
+  // It is not safe to copy labels, so disable the copy constructor and operator
+  // by declaring them private (without an implementation).
+  Label(const Label&);
+  void operator=(const Label&);
+
+  // The Assembler class is responsible for binding and linking labels, since
+  // the stored offsets need to be consistent with the Assembler's buffer.
+  friend class Assembler;
+  // The MacroAssembler and VeneerPool handle resolution of branches to distant
+  // targets.
+  friend class MacroAssembler;
+  friend class VeneerPool;
+};
+
+
+// Required InvalSet template specialisations.
+#define INVAL_SET_TEMPLATE_PARAMETERS \
+    ptrdiff_t,                        \
+    Label::kNPreallocatedLinks,       \
+    ptrdiff_t,                        \
+    Label::kInvalidLinkKey,           \
+    Label::kReclaimFrom,              \
+    Label::kReclaimFactor
+template<>
+inline ptrdiff_t InvalSet<INVAL_SET_TEMPLATE_PARAMETERS>::Key(
+    const ptrdiff_t& element) {
+  return element;
+}
+template<>
+inline void InvalSet<INVAL_SET_TEMPLATE_PARAMETERS>::SetKey(
+              ptrdiff_t* element, ptrdiff_t key) {
+  *element = key;
+}
+#undef INVAL_SET_TEMPLATE_PARAMETERS
+
+
+class Assembler;
+class LiteralPool;
+
+// A literal is a 32-bit or 64-bit piece of data stored in the instruction
+// stream and loaded through a pc relative load. The same literal can be
+// referred to by multiple instructions but a literal can only reside at one
+// place in memory. A literal can be used by a load before or after being
+// placed in memory.
+//
+// Internally an offset of 0 is associated with a literal which has been
+// neither used nor placed. Then two possibilities arise:
+//  1) the label is placed, the offset (stored as offset + 1) is used to
+//     resolve any subsequent load using the label.
+//  2) the label is not placed and offset is the offset of the last load using
+//     the literal (stored as -offset -1). If multiple loads refer to this
+//     literal then the last load holds the offset of the preceding load and
+//     all loads form a chain. Once the offset is placed all the loads in the
+//     chain are resolved and future loads fall back to possibility 1.
+class RawLiteral {
+ public:
+  enum DeletionPolicy {
+    kDeletedOnPlacementByPool,
+    kDeletedOnPoolDestruction,
+    kManuallyDeleted
+  };
+
+  RawLiteral(size_t size,
+             LiteralPool* literal_pool,
+             DeletionPolicy deletion_policy = kManuallyDeleted);
+
+  // The literal pool only sees and deletes `RawLiteral*` pointers, but they are
+  // actually pointing to `Literal<T>` objects.
+  virtual ~RawLiteral() {}
+
+  size_t size() {
+    VIXL_STATIC_ASSERT(kDRegSizeInBytes == kXRegSizeInBytes);
+    VIXL_STATIC_ASSERT(kSRegSizeInBytes == kWRegSizeInBytes);
+    VIXL_ASSERT((size_ == kXRegSizeInBytes) ||
+                (size_ == kWRegSizeInBytes) ||
+                (size_ == kQRegSizeInBytes));
+    return size_;
+  }
+  uint64_t raw_value128_low64() {
+    VIXL_ASSERT(size_ == kQRegSizeInBytes);
+    return low64_;
+  }
+  uint64_t raw_value128_high64() {
+    VIXL_ASSERT(size_ == kQRegSizeInBytes);
+    return high64_;
+  }
+  uint64_t raw_value64() {
+    VIXL_ASSERT(size_ == kXRegSizeInBytes);
+    VIXL_ASSERT(high64_ == 0);
+    return low64_;
+  }
+  uint32_t raw_value32() {
+    VIXL_ASSERT(size_ == kWRegSizeInBytes);
+    VIXL_ASSERT(high64_ == 0);
+    VIXL_ASSERT(is_uint32(low64_) || is_int32(low64_));
+    return static_cast<uint32_t>(low64_);
+  }
+  bool IsUsed() { return offset_ < 0; }
+  bool IsPlaced() { return offset_ > 0; }
+
+  LiteralPool* GetLiteralPool() const {
+    return literal_pool_;
+  }
+
+  ptrdiff_t offset() {
+    VIXL_ASSERT(IsPlaced());
+    return offset_ - 1;
+  }
+
+ protected:
+  void set_offset(ptrdiff_t offset) {
+    VIXL_ASSERT(offset >= 0);
+    VIXL_ASSERT(IsWordAligned(offset));
+    VIXL_ASSERT(!IsPlaced());
+    offset_ = offset + 1;
+  }
+  ptrdiff_t last_use() {
+    VIXL_ASSERT(IsUsed());
+    return -offset_ - 1;
+  }
+  void set_last_use(ptrdiff_t offset) {
+    VIXL_ASSERT(offset >= 0);
+    VIXL_ASSERT(IsWordAligned(offset));
+    VIXL_ASSERT(!IsPlaced());
+    offset_ = -offset - 1;
+  }
+
+  size_t size_;
+  ptrdiff_t offset_;
+  uint64_t low64_;
+  uint64_t high64_;
+
+ private:
+  LiteralPool* literal_pool_;
+  DeletionPolicy deletion_policy_;
+
+  friend class Assembler;
+  friend class LiteralPool;
+};
+
+
+template <typename T>
+class Literal : public RawLiteral {
+ public:
+  explicit Literal(T value,
+                   LiteralPool* literal_pool = NULL,
+                   RawLiteral::DeletionPolicy ownership = kManuallyDeleted)
+      : RawLiteral(sizeof(value), literal_pool, ownership) {
+    VIXL_STATIC_ASSERT(sizeof(value) <= kXRegSizeInBytes);
+    UpdateValue(value);
+  }
+
+  Literal(T high64, T low64,
+          LiteralPool* literal_pool = NULL,
+          RawLiteral::DeletionPolicy ownership = kManuallyDeleted)
+      : RawLiteral(kQRegSizeInBytes, literal_pool, ownership) {
+    VIXL_STATIC_ASSERT(sizeof(low64) == (kQRegSizeInBytes / 2));
+    UpdateValue(high64, low64);
+  }
+
+  virtual ~Literal() {}
+
+  // Update the value of this literal, if necessary by rewriting the value in
+  // the pool.
+  // If the literal has already been placed in a literal pool, the address of
+  // the start of the code buffer must be provided, as the literal only knows it
+  // offset from there.  This also allows patching the value after the code has
+  // been moved in memory.
+  void UpdateValue(T new_value, uint8_t* code_buffer = NULL) {
+    VIXL_ASSERT(sizeof(new_value) == size_);
+    memcpy(&low64_, &new_value, sizeof(new_value));
+    if (IsPlaced()) {
+      VIXL_ASSERT(code_buffer != NULL);
+      RewriteValueInCode(code_buffer);
+    }
+  }
+
+  void UpdateValue(T high64, T low64, uint8_t* code_buffer = NULL) {
+    VIXL_ASSERT(sizeof(low64) == size_ / 2);
+    memcpy(&low64_, &low64, sizeof(low64));
+    memcpy(&high64_, &high64, sizeof(high64));
+    if (IsPlaced()) {
+      VIXL_ASSERT(code_buffer != NULL);
+      RewriteValueInCode(code_buffer);
+    }
+  }
+
+  void UpdateValue(T new_value, const Assembler* assembler);
+  void UpdateValue(T high64, T low64, const Assembler* assembler);
+
+ private:
+  void RewriteValueInCode(uint8_t* code_buffer) {
+    VIXL_ASSERT(IsPlaced());
+    VIXL_STATIC_ASSERT(sizeof(T) <= kXRegSizeInBytes);
+    switch (size()) {
+      case kSRegSizeInBytes:
+        *reinterpret_cast<uint32_t*>(code_buffer + offset()) = raw_value32();
+        break;
+      case kDRegSizeInBytes:
+        *reinterpret_cast<uint64_t*>(code_buffer + offset()) = raw_value64();
+        break;
+      default:
+        VIXL_ASSERT(size() == kQRegSizeInBytes);
+        uint64_t* base_address =
+            reinterpret_cast<uint64_t*>(code_buffer + offset());
+        *base_address = raw_value128_low64();
+        *(base_address + 1) = raw_value128_high64();
+    }
+  }
+};
+
+
+// Control whether or not position-independent code should be emitted.
+enum PositionIndependentCodeOption {
+  // All code generated will be position-independent; all branches and
+  // references to labels generated with the Label class will use PC-relative
+  // addressing.
+  PositionIndependentCode,
+
+  // Allow VIXL to generate code that refers to absolute addresses. With this
+  // option, it will not be possible to copy the code buffer and run it from a
+  // different address; code must be generated in its final location.
+  PositionDependentCode,
+
+  // Allow VIXL to assume that the bottom 12 bits of the address will be
+  // constant, but that the top 48 bits may change. This allows `adrp` to
+  // function in systems which copy code between pages, but otherwise maintain
+  // 4KB page alignment.
+  PageOffsetDependentCode
+};
+
+
+// Control how scaled- and unscaled-offset loads and stores are generated.
+enum LoadStoreScalingOption {
+  // Prefer scaled-immediate-offset instructions, but emit unscaled-offset,
+  // register-offset, pre-index or post-index instructions if necessary.
+  PreferScaledOffset,
+
+  // Prefer unscaled-immediate-offset instructions, but emit scaled-offset,
+  // register-offset, pre-index or post-index instructions if necessary.
+  PreferUnscaledOffset,
+
+  // Require scaled-immediate-offset instructions.
+  RequireScaledOffset,
+
+  // Require unscaled-immediate-offset instructions.
+  RequireUnscaledOffset
+};
+
+
+// Assembler.
+class Assembler {
+ public:
+  Assembler(size_t capacity,
+            PositionIndependentCodeOption pic = PositionIndependentCode);
+  Assembler(byte* buffer, size_t capacity,
+            PositionIndependentCodeOption pic = PositionIndependentCode);
+
+  // The destructor asserts that one of the following is true:
+  //  * The Assembler object has not been used.
+  //  * Nothing has been emitted since the last Reset() call.
+  //  * Nothing has been emitted since the last FinalizeCode() call.
+  ~Assembler();
+
+  // System functions.
+
+  // Start generating code from the beginning of the buffer, discarding any code
+  // and data that has already been emitted into the buffer.
+  void Reset();
+
+  // Finalize a code buffer of generated instructions. This function must be
+  // called before executing or copying code from the buffer.
+  void FinalizeCode();
+
+  // Label.
+  // Bind a label to the current PC.
+  void bind(Label* label);
+
+  // Bind a label to a specified offset from the start of the buffer.
+  void BindToOffset(Label* label, ptrdiff_t offset);
+
+  // Place a literal at the current PC.
+  void place(RawLiteral* literal);
+
+  ptrdiff_t CursorOffset() const {
+    return buffer_->CursorOffset();
+  }
+
+  ptrdiff_t BufferEndOffset() const {
+    return static_cast<ptrdiff_t>(buffer_->capacity());
+  }
+
+  // Return the address of an offset in the buffer.
+  template <typename T>
+  T GetOffsetAddress(ptrdiff_t offset) const {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return buffer_->GetOffsetAddress<T>(offset);
+  }
+
+  // Return the address of a bound label.
+  template <typename T>
+  T GetLabelAddress(const Label * label) const {
+    VIXL_ASSERT(label->IsBound());
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(label->location());
+  }
+
+  // Return the address of the cursor.
+  template <typename T>
+  T GetCursorAddress() const {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(CursorOffset());
+  }
+
+  // Return the address of the start of the buffer.
+  template <typename T>
+  T GetStartAddress() const {
+    VIXL_STATIC_ASSERT(sizeof(T) >= sizeof(uintptr_t));
+    return GetOffsetAddress<T>(0);
+  }
+
+  Instruction* InstructionAt(ptrdiff_t instruction_offset) {
+    return GetOffsetAddress<Instruction*>(instruction_offset);
+  }
+
+  ptrdiff_t InstructionOffset(Instruction* instruction) {
+    VIXL_STATIC_ASSERT(sizeof(*instruction) == 1);
+    ptrdiff_t offset = instruction - GetStartAddress<Instruction*>();
+    VIXL_ASSERT((0 <= offset) &&
+                (offset < static_cast<ptrdiff_t>(BufferCapacity())));
+    return offset;
+  }
+
+  // Instruction set functions.
+
+  // Branch / Jump instructions.
+  // Branch to register.
+  void br(const Register& xn);
+
+  // Branch with link to register.
+  void blr(const Register& xn);
+
+  // Branch to register with return hint.
+  void ret(const Register& xn = lr);
+
+  // Unconditional branch to label.
+  void b(Label* label);
+
+  // Conditional branch to label.
+  void b(Label* label, Condition cond);
+
+  // Unconditional branch to PC offset.
+  void b(int imm26);
+
+  // Conditional branch to PC offset.
+  void b(int imm19, Condition cond);
+
+  // Branch with link to label.
+  void bl(Label* label);
+
+  // Branch with link to PC offset.
+  void bl(int imm26);
+
+  // Compare and branch to label if zero.
+  void cbz(const Register& rt, Label* label);
+
+  // Compare and branch to PC offset if zero.
+  void cbz(const Register& rt, int imm19);
+
+  // Compare and branch to label if not zero.
+  void cbnz(const Register& rt, Label* label);
+
+  // Compare and branch to PC offset if not zero.
+  void cbnz(const Register& rt, int imm19);
+
+  // Table lookup from one register.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Table lookup from two registers.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vm);
+
+  // Table lookup from three registers.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vm);
+
+  // Table lookup from four registers.
+  void tbl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vn4,
+           const VRegister& vm);
+
+  // Table lookup extension from one register.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Table lookup extension from two registers.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vm);
+
+  // Table lookup extension from three registers.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vm);
+
+  // Table lookup extension from four registers.
+  void tbx(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vn2,
+           const VRegister& vn3,
+           const VRegister& vn4,
+           const VRegister& vm);
+
+  // Test bit and branch to label if zero.
+  void tbz(const Register& rt, unsigned bit_pos, Label* label);
+
+  // Test bit and branch to PC offset if zero.
+  void tbz(const Register& rt, unsigned bit_pos, int imm14);
+
+  // Test bit and branch to label if not zero.
+  void tbnz(const Register& rt, unsigned bit_pos, Label* label);
+
+  // Test bit and branch to PC offset if not zero.
+  void tbnz(const Register& rt, unsigned bit_pos, int imm14);
+
+  // Address calculation instructions.
+  // Calculate a PC-relative address. Unlike for branches the offset in adr is
+  // unscaled (i.e. the result can be unaligned).
+
+  // Calculate the address of a label.
+  void adr(const Register& rd, Label* label);
+
+  // Calculate the address of a PC offset.
+  void adr(const Register& rd, int imm21);
+
+  // Calculate the page address of a label.
+  void adrp(const Register& rd, Label* label);
+
+  // Calculate the page address of a PC offset.
+  void adrp(const Register& rd, int imm21);
+
+  // Data Processing instructions.
+  // Add.
+  void add(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Add and update status flags.
+  void adds(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Compare negative.
+  void cmn(const Register& rn, const Operand& operand);
+
+  // Subtract.
+  void sub(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Subtract and update status flags.
+  void subs(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Compare.
+  void cmp(const Register& rn, const Operand& operand);
+
+  // Negate.
+  void neg(const Register& rd,
+           const Operand& operand);
+
+  // Negate and update status flags.
+  void negs(const Register& rd,
+            const Operand& operand);
+
+  // Add with carry bit.
+  void adc(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Add with carry bit and update status flags.
+  void adcs(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Subtract with carry bit.
+  void sbc(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Subtract with carry bit and update status flags.
+  void sbcs(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Negate with carry bit.
+  void ngc(const Register& rd,
+           const Operand& operand);
+
+  // Negate with carry bit and update status flags.
+  void ngcs(const Register& rd,
+            const Operand& operand);
+
+  // Logical instructions.
+  // Bitwise and (A & B).
+  void and_(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Bitwise and (A & B) and update status flags.
+  void ands(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Bit test and set flags.
+  void tst(const Register& rn, const Operand& operand);
+
+  // Bit clear (A & ~B).
+  void bic(const Register& rd,
+           const Register& rn,
+           const Operand& operand);
+
+  // Bit clear (A & ~B) and update status flags.
+  void bics(const Register& rd,
+            const Register& rn,
+            const Operand& operand);
+
+  // Bitwise or (A | B).
+  void orr(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Bitwise nor (A | ~B).
+  void orn(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Bitwise eor/xor (A ^ B).
+  void eor(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Bitwise enor/xnor (A ^ ~B).
+  void eon(const Register& rd, const Register& rn, const Operand& operand);
+
+  // Logical shift left by variable.
+  void lslv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Logical shift right by variable.
+  void lsrv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Arithmetic shift right by variable.
+  void asrv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Rotate right by variable.
+  void rorv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Bitfield instructions.
+  // Bitfield move.
+  void bfm(const Register& rd,
+           const Register& rn,
+           unsigned immr,
+           unsigned imms);
+
+  // Signed bitfield move.
+  void sbfm(const Register& rd,
+            const Register& rn,
+            unsigned immr,
+            unsigned imms);
+
+  // Unsigned bitfield move.
+  void ubfm(const Register& rd,
+            const Register& rn,
+            unsigned immr,
+            unsigned imms);
+
+  // Bfm aliases.
+  // Bitfield insert.
+  void bfi(const Register& rd,
+           const Register& rn,
+           unsigned lsb,
+           unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+  }
+
+  // Bitfield extract and insert low.
+  void bfxil(const Register& rd,
+             const Register& rn,
+             unsigned lsb,
+             unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    bfm(rd, rn, lsb, lsb + width - 1);
+  }
+
+  // Sbfm aliases.
+  // Arithmetic shift right.
+  void asr(const Register& rd, const Register& rn, unsigned shift) {
+    VIXL_ASSERT(shift < rd.size());
+    sbfm(rd, rn, shift, rd.size() - 1);
+  }
+
+  // Signed bitfield insert with zero at right.
+  void sbfiz(const Register& rd,
+             const Register& rn,
+             unsigned lsb,
+             unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+  }
+
+  // Signed bitfield extract.
+  void sbfx(const Register& rd,
+            const Register& rn,
+            unsigned lsb,
+            unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    sbfm(rd, rn, lsb, lsb + width - 1);
+  }
+
+  // Signed extend byte.
+  void sxtb(const Register& rd, const Register& rn) {
+    sbfm(rd, rn, 0, 7);
+  }
+
+  // Signed extend halfword.
+  void sxth(const Register& rd, const Register& rn) {
+    sbfm(rd, rn, 0, 15);
+  }
+
+  // Signed extend word.
+  void sxtw(const Register& rd, const Register& rn) {
+    sbfm(rd, rn, 0, 31);
+  }
+
+  // Ubfm aliases.
+  // Logical shift left.
+  void lsl(const Register& rd, const Register& rn, unsigned shift) {
+    unsigned reg_size = rd.size();
+    VIXL_ASSERT(shift < reg_size);
+    ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
+  }
+
+  // Logical shift right.
+  void lsr(const Register& rd, const Register& rn, unsigned shift) {
+    VIXL_ASSERT(shift < rd.size());
+    ubfm(rd, rn, shift, rd.size() - 1);
+  }
+
+  // Unsigned bitfield insert with zero at right.
+  void ubfiz(const Register& rd,
+             const Register& rn,
+             unsigned lsb,
+             unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+  }
+
+  // Unsigned bitfield extract.
+  void ubfx(const Register& rd,
+            const Register& rn,
+            unsigned lsb,
+            unsigned width) {
+    VIXL_ASSERT(width >= 1);
+    VIXL_ASSERT(lsb + width <= rn.size());
+    ubfm(rd, rn, lsb, lsb + width - 1);
+  }
+
+  // Unsigned extend byte.
+  void uxtb(const Register& rd, const Register& rn) {
+    ubfm(rd, rn, 0, 7);
+  }
+
+  // Unsigned extend halfword.
+  void uxth(const Register& rd, const Register& rn) {
+    ubfm(rd, rn, 0, 15);
+  }
+
+  // Unsigned extend word.
+  void uxtw(const Register& rd, const Register& rn) {
+    ubfm(rd, rn, 0, 31);
+  }
+
+  // Extract.
+  void extr(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            unsigned lsb);
+
+  // Conditional select: rd = cond ? rn : rm.
+  void csel(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            Condition cond);
+
+  // Conditional select increment: rd = cond ? rn : rm + 1.
+  void csinc(const Register& rd,
+             const Register& rn,
+             const Register& rm,
+             Condition cond);
+
+  // Conditional select inversion: rd = cond ? rn : ~rm.
+  void csinv(const Register& rd,
+             const Register& rn,
+             const Register& rm,
+             Condition cond);
+
+  // Conditional select negation: rd = cond ? rn : -rm.
+  void csneg(const Register& rd,
+             const Register& rn,
+             const Register& rm,
+             Condition cond);
+
+  // Conditional set: rd = cond ? 1 : 0.
+  void cset(const Register& rd, Condition cond);
+
+  // Conditional set mask: rd = cond ? -1 : 0.
+  void csetm(const Register& rd, Condition cond);
+
+  // Conditional increment: rd = cond ? rn + 1 : rn.
+  void cinc(const Register& rd, const Register& rn, Condition cond);
+
+  // Conditional invert: rd = cond ? ~rn : rn.
+  void cinv(const Register& rd, const Register& rn, Condition cond);
+
+  // Conditional negate: rd = cond ? -rn : rn.
+  void cneg(const Register& rd, const Register& rn, Condition cond);
+
+  // Rotate right.
+  void ror(const Register& rd, const Register& rs, unsigned shift) {
+    extr(rd, rs, rs, shift);
+  }
+
+  // Conditional comparison.
+  // Conditional compare negative.
+  void ccmn(const Register& rn,
+            const Operand& operand,
+            StatusFlags nzcv,
+            Condition cond);
+
+  // Conditional compare.
+  void ccmp(const Register& rn,
+            const Operand& operand,
+            StatusFlags nzcv,
+            Condition cond);
+
+  // CRC-32 checksum from byte.
+  void crc32b(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 checksum from half-word.
+  void crc32h(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 checksum from word.
+  void crc32w(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 checksum from double word.
+  void crc32x(const Register& rd,
+              const Register& rn,
+              const Register& rm);
+
+  // CRC-32 C checksum from byte.
+  void crc32cb(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // CRC-32 C checksum from half-word.
+  void crc32ch(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // CRC-32 C checksum from word.
+  void crc32cw(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // CRC-32C checksum from double word.
+  void crc32cx(const Register& rd,
+               const Register& rn,
+               const Register& rm);
+
+  // Multiply.
+  void mul(const Register& rd, const Register& rn, const Register& rm);
+
+  // Negated multiply.
+  void mneg(const Register& rd, const Register& rn, const Register& rm);
+
+  // Signed long multiply: 32 x 32 -> 64-bit.
+  void smull(const Register& rd, const Register& rn, const Register& rm);
+
+  // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
+  void smulh(const Register& xd, const Register& xn, const Register& xm);
+
+  // Multiply and accumulate.
+  void madd(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            const Register& ra);
+
+  // Multiply and subtract.
+  void msub(const Register& rd,
+            const Register& rn,
+            const Register& rm,
+            const Register& ra);
+
+  // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
+  void smaddl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
+  void umaddl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Unsigned long multiply: 32 x 32 -> 64-bit.
+  void umull(const Register& rd,
+             const Register& rn,
+             const Register& rm) {
+    umaddl(rd, rn, rm, xzr);
+  }
+
+  // Unsigned multiply high: 64 x 64 -> 64-bit <127:64>.
+  void umulh(const Register& xd,
+             const Register& xn,
+             const Register& xm);
+
+  // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
+  void smsubl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
+  void umsubl(const Register& rd,
+              const Register& rn,
+              const Register& rm,
+              const Register& ra);
+
+  // Signed integer divide.
+  void sdiv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Unsigned integer divide.
+  void udiv(const Register& rd, const Register& rn, const Register& rm);
+
+  // Bit reverse.
+  void rbit(const Register& rd, const Register& rn);
+
+  // Reverse bytes in 16-bit half words.
+  void rev16(const Register& rd, const Register& rn);
+
+  // Reverse bytes in 32-bit words.
+  void rev32(const Register& rd, const Register& rn);
+
+  // Reverse bytes.
+  void rev(const Register& rd, const Register& rn);
+
+  // Count leading zeroes.
+  void clz(const Register& rd, const Register& rn);
+
+  // Count leading sign bits.
+  void cls(const Register& rd, const Register& rn);
+
+  // Memory instructions.
+  // Load integer or FP register.
+  void ldr(const CPURegister& rt, const MemOperand& src,
+           LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Store integer or FP register.
+  void str(const CPURegister& rt, const MemOperand& dst,
+           LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load word with sign extension.
+  void ldrsw(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load byte.
+  void ldrb(const Register& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Store byte.
+  void strb(const Register& rt, const MemOperand& dst,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load byte with sign extension.
+  void ldrsb(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load half-word.
+  void ldrh(const Register& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Store half-word.
+  void strh(const Register& rt, const MemOperand& dst,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load half-word with sign extension.
+  void ldrsh(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Load integer or FP register (with unscaled offset).
+  void ldur(const CPURegister& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store integer or FP register (with unscaled offset).
+  void stur(const CPURegister& rt, const MemOperand& src,
+            LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load word with sign extension.
+  void ldursw(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load byte (with unscaled offset).
+  void ldurb(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store byte (with unscaled offset).
+  void sturb(const Register& rt, const MemOperand& dst,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load byte with sign extension (and unscaled offset).
+  void ldursb(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load half-word (with unscaled offset).
+  void ldurh(const Register& rt, const MemOperand& src,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Store half-word (with unscaled offset).
+  void sturh(const Register& rt, const MemOperand& dst,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load half-word with sign extension (and unscaled offset).
+  void ldursh(const Register& rt, const MemOperand& src,
+              LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Load integer or FP register pair.
+  void ldp(const CPURegister& rt, const CPURegister& rt2,
+           const MemOperand& src);
+
+  // Store integer or FP register pair.
+  void stp(const CPURegister& rt, const CPURegister& rt2,
+           const MemOperand& dst);
+
+  // Load word pair with sign extension.
+  void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Load integer or FP register pair, non-temporal.
+  void ldnp(const CPURegister& rt, const CPURegister& rt2,
+            const MemOperand& src);
+
+  // Store integer or FP register pair, non-temporal.
+  void stnp(const CPURegister& rt, const CPURegister& rt2,
+            const MemOperand& dst);
+
+  // Load integer or FP register from literal pool.
+  void ldr(const CPURegister& rt, RawLiteral* literal);
+
+  // Load word with sign extension from literal pool.
+  void ldrsw(const Register& rt, RawLiteral* literal);
+
+  // Load integer or FP register from pc + imm19 << 2.
+  void ldr(const CPURegister& rt, int imm19);
+
+  // Load word with sign extension from pc + imm19 << 2.
+  void ldrsw(const Register& rt, int imm19);
+
+  // Store exclusive byte.
+  void stxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store exclusive half-word.
+  void stxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store exclusive register.
+  void stxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Load exclusive byte.
+  void ldxrb(const Register& rt, const MemOperand& src);
+
+  // Load exclusive half-word.
+  void ldxrh(const Register& rt, const MemOperand& src);
+
+  // Load exclusive register.
+  void ldxr(const Register& rt, const MemOperand& src);
+
+  // Store exclusive register pair.
+  void stxp(const Register& rs,
+            const Register& rt,
+            const Register& rt2,
+            const MemOperand& dst);
+
+  // Load exclusive register pair.
+  void ldxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Store-release exclusive byte.
+  void stlxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store-release exclusive half-word.
+  void stlxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Store-release exclusive register.
+  void stlxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+  // Load-acquire exclusive byte.
+  void ldaxrb(const Register& rt, const MemOperand& src);
+
+  // Load-acquire exclusive half-word.
+  void ldaxrh(const Register& rt, const MemOperand& src);
+
+  // Load-acquire exclusive register.
+  void ldaxr(const Register& rt, const MemOperand& src);
+
+  // Store-release exclusive register pair.
+  void stlxp(const Register& rs,
+             const Register& rt,
+             const Register& rt2,
+             const MemOperand& dst);
+
+  // Load-acquire exclusive register pair.
+  void ldaxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+  // Store-release byte.
+  void stlrb(const Register& rt, const MemOperand& dst);
+
+  // Store-release half-word.
+  void stlrh(const Register& rt, const MemOperand& dst);
+
+  // Store-release register.
+  void stlr(const Register& rt, const MemOperand& dst);
+
+  // Load-acquire byte.
+  void ldarb(const Register& rt, const MemOperand& src);
+
+  // Load-acquire half-word.
+  void ldarh(const Register& rt, const MemOperand& src);
+
+  // Load-acquire register.
+  void ldar(const Register& rt, const MemOperand& src);
+
+  // Prefetch memory.
+  void prfm(PrefetchOperation op, const MemOperand& addr,
+            LoadStoreScalingOption option = PreferScaledOffset);
+
+  // Prefetch memory (with unscaled offset).
+  void prfum(PrefetchOperation op, const MemOperand& addr,
+             LoadStoreScalingOption option = PreferUnscaledOffset);
+
+  // Prefetch memory in the literal pool.
+  void prfm(PrefetchOperation op, RawLiteral* literal);
+
+  // Prefetch from pc + imm19 << 2.
+  void prfm(PrefetchOperation op, int imm19);
+
+  // Move instructions. The default shift of -1 indicates that the move
+  // instruction will calculate an appropriate 16-bit immediate and left shift
+  // that is equal to the 64-bit immediate argument. If an explicit left shift
+  // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
+  //
+  // For movk, an explicit shift can be used to indicate which half word should
+  // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
+  // half word with zero, whereas movk(x0, 0, 48) will overwrite the
+  // most-significant.
+
+  // Move immediate and keep.
+  void movk(const Register& rd, uint64_t imm, int shift = -1) {
+    MoveWide(rd, imm, shift, MOVK);
+  }
+
+  // Move inverted immediate.
+  void movn(const Register& rd, uint64_t imm, int shift = -1) {
+    MoveWide(rd, imm, shift, MOVN);
+  }
+
+  // Move immediate.
+  void movz(const Register& rd, uint64_t imm, int shift = -1) {
+    MoveWide(rd, imm, shift, MOVZ);
+  }
+
+  // Misc instructions.
+  // Monitor debug-mode breakpoint.
+  void brk(int code);
+
+  // Halting debug-mode breakpoint.
+  void hlt(int code);
+
+  // Generate exception targeting EL1.
+  void svc(int code);
+
+  // Move register to register.
+  void mov(const Register& rd, const Register& rn);
+
+  // Move inverted operand to register.
+  void mvn(const Register& rd, const Operand& operand);
+
+  // System instructions.
+  // Move to register from system register.
+  void mrs(const Register& rt, SystemRegister sysreg);
+
+  // Move from register to system register.
+  void msr(SystemRegister sysreg, const Register& rt);
+
+  // System instruction.
+  void sys(int op1, int crn, int crm, int op2, const Register& rt = xzr);
+
+  // System instruction with pre-encoded op (op1:crn:crm:op2).
+  void sys(int op, const Register& rt = xzr);
+
+  // System data cache operation.
+  void dc(DataCacheOp op, const Register& rt);
+
+  // System instruction cache operation.
+  void ic(InstructionCacheOp op, const Register& rt);
+
+  // System hint.
+  void hint(SystemHint code);
+
+  // Clear exclusive monitor.
+  void clrex(int imm4 = 0xf);
+
+  // Data memory barrier.
+  void dmb(BarrierDomain domain, BarrierType type);
+
+  // Data synchronization barrier.
+  void dsb(BarrierDomain domain, BarrierType type);
+
+  // Instruction synchronization barrier.
+  void isb();
+
+  // Alias for system instructions.
+  // No-op.
+  void nop() {
+    hint(NOP);
+  }
+
+  // FP and NEON instructions.
+  // Move double precision immediate to FP register.
+  void fmov(const VRegister& vd, double imm);
+
+  // Move single precision immediate to FP register.
+  void fmov(const VRegister& vd, float imm);
+
+  // Move FP register to register.
+  void fmov(const Register& rd, const VRegister& fn);
+
+  // Move register to FP register.
+  void fmov(const VRegister& vd, const Register& rn);
+
+  // Move FP register to FP register.
+  void fmov(const VRegister& vd, const VRegister& fn);
+
+  // Move 64-bit register to top half of 128-bit FP register.
+  void fmov(const VRegister& vd, int index, const Register& rn);
+
+  // Move top half of 128-bit FP register to 64-bit register.
+  void fmov(const Register& rd, const VRegister& vn, int index);
+
+  // FP add.
+  void fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+  // FP subtract.
+  void fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+  // FP multiply.
+  void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+  // FP fused multiply-add.
+  void fmadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             const VRegister& va);
+
+  // FP fused multiply-subtract.
+  void fmsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             const VRegister& va);
+
+  // FP fused multiply-add and negate.
+  void fnmadd(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              const VRegister& va);
+
+  // FP fused multiply-subtract and negate.
+  void fnmsub(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              const VRegister& va);
+
+  // FP multiply-negate scalar.
+  void fnmul(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP reciprocal exponent scalar.
+  void frecpx(const VRegister& vd,
+              const VRegister& vn);
+
+  // FP divide.
+  void fdiv(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP maximum.
+  void fmax(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP minimum.
+  void fmin(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP maximum number.
+  void fmaxnm(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP minimum number.
+  void fminnm(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+  // FP absolute.
+  void fabs(const VRegister& vd, const VRegister& vn);
+
+  // FP negate.
+  void fneg(const VRegister& vd, const VRegister& vn);
+
+  // FP square root.
+  void fsqrt(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, nearest with ties to away.
+  void frinta(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, implicit rounding.
+  void frinti(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, toward minus infinity.
+  void frintm(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, nearest with ties to even.
+  void frintn(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, toward plus infinity.
+  void frintp(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, exact, implicit rounding.
+  void frintx(const VRegister& vd, const VRegister& vn);
+
+  // FP round to integer, towards zero.
+  void frintz(const VRegister& vd, const VRegister& vn);
+
+  void FPCompareMacro(const VRegister& vn,
+                      double value,
+                      FPTrapFlags trap);
+
+  void FPCompareMacro(const VRegister& vn,
+                      const VRegister& vm,
+                      FPTrapFlags trap);
+
+  // FP compare registers.
+  void fcmp(const VRegister& vn, const VRegister& vm);
+
+  // FP compare immediate.
+  void fcmp(const VRegister& vn, double value);
+
+  void FPCCompareMacro(const VRegister& vn,
+                       const VRegister& vm,
+                       StatusFlags nzcv,
+                       Condition cond,
+                       FPTrapFlags trap);
+
+  // FP conditional compare.
+  void fccmp(const VRegister& vn,
+             const VRegister& vm,
+             StatusFlags nzcv,
+             Condition cond);
+
+  // FP signaling compare registers.
+  void fcmpe(const VRegister& vn, const VRegister& vm);
+
+  // FP signaling compare immediate.
+  void fcmpe(const VRegister& vn, double value);
+
+  // FP conditional signaling compare.
+  void fccmpe(const VRegister& vn,
+              const VRegister& vm,
+              StatusFlags nzcv,
+              Condition cond);
+
+  // FP conditional select.
+  void fcsel(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             Condition cond);
+
+  // Common FP Convert functions.
+  void NEONFPConvertToInt(const Register& rd,
+                          const VRegister& vn,
+                          Instr op);
+  void NEONFPConvertToInt(const VRegister& vd,
+                          const VRegister& vn,
+                          Instr op);
+
+  // FP convert between precisions.
+  void fcvt(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to higher precision.
+  void fcvtl(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to higher precision (second part).
+  void fcvtl2(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower precision.
+  void fcvtn(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower prevision (second part).
+  void fcvtn2(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower precision, rounding to odd.
+  void fcvtxn(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to lower precision, rounding to odd (second part).
+  void fcvtxn2(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to away.
+  void fcvtas(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to away.
+  void fcvtau(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to away.
+  void fcvtas(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to away.
+  void fcvtau(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to signed integer, round towards -infinity.
+  void fcvtms(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards -infinity.
+  void fcvtmu(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, round towards -infinity.
+  void fcvtms(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards -infinity.
+  void fcvtmu(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to even.
+  void fcvtns(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to even.
+  void fcvtnu(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, nearest with ties to even.
+  void fcvtns(const VRegister& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, nearest with ties to even.
+  void fcvtnu(const VRegister& rd, const VRegister& vn);
+
+  // FP convert to signed integer or fixed-point, round towards zero.
+  void fcvtzs(const Register& rd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to unsigned integer or fixed-point, round towards zero.
+  void fcvtzu(const Register& rd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to signed integer or fixed-point, round towards zero.
+  void fcvtzs(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to unsigned integer or fixed-point, round towards zero.
+  void fcvtzu(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+  // FP convert to signed integer, round towards +infinity.
+  void fcvtps(const Register& rd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards +infinity.
+  void fcvtpu(const Register& rd, const VRegister& vn);
+
+  // FP convert to signed integer, round towards +infinity.
+  void fcvtps(const VRegister& vd, const VRegister& vn);
+
+  // FP convert to unsigned integer, round towards +infinity.
+  void fcvtpu(const VRegister& vd, const VRegister& vn);
+
+  // Convert signed integer or fixed point to FP.
+  void scvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+  // Convert unsigned integer or fixed point to FP.
+  void ucvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+  // Convert signed integer or fixed-point to FP.
+  void scvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+  // Convert unsigned integer or fixed-point to FP.
+  void ucvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+  // Unsigned absolute difference.
+  void uabd(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed absolute difference.
+  void sabd(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned absolute difference and accumulate.
+  void uaba(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed absolute difference and accumulate.
+  void saba(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Add.
+  void add(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Subtract.
+  void sub(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Unsigned halving add.
+  void uhadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed halving add.
+  void shadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned rounding halving add.
+  void urhadd(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed rounding halving add.
+  void srhadd(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned halving sub.
+  void uhsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed halving sub.
+  void shsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned saturating add.
+  void uqadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed saturating add.
+  void sqadd(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned saturating subtract.
+  void uqsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed saturating subtract.
+  void sqsub(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Add pairwise.
+  void addp(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Add pair of elements scalar.
+  void addp(const VRegister& vd,
+            const VRegister& vn);
+
+  // Multiply-add to accumulator.
+  void mla(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Multiply-subtract to accumulator.
+  void mls(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Multiply.
+  void mul(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Multiply by scalar element.
+  void mul(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int vm_index);
+
+  // Multiply-add by scalar element.
+  void mla(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int vm_index);
+
+  // Multiply-subtract by scalar element.
+  void mls(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int vm_index);
+
+  // Signed long multiply-add by scalar element.
+  void smlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Signed long multiply-add by scalar element (second part).
+  void smlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Unsigned long multiply-add by scalar element.
+  void umlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Unsigned long multiply-add by scalar element (second part).
+  void umlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Signed long multiply-sub by scalar element.
+  void smlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Signed long multiply-sub by scalar element (second part).
+  void smlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Unsigned long multiply-sub by scalar element.
+  void umlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Unsigned long multiply-sub by scalar element (second part).
+  void umlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Signed long multiply by scalar element.
+  void smull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Signed long multiply by scalar element (second part).
+  void smull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Unsigned long multiply by scalar element.
+  void umull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // Unsigned long multiply by scalar element (second part).
+  void umull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm,
+              int vm_index);
+
+  // Signed saturating double long multiply by element.
+  void sqdmull(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating double long multiply by element (second part).
+  void sqdmull2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Signed saturating doubling long multiply-add by element.
+  void sqdmlal(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating doubling long multiply-add by element (second part).
+  void sqdmlal2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Signed saturating doubling long multiply-sub by element.
+  void sqdmlsl(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating doubling long multiply-sub by element (second part).
+  void sqdmlsl2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Compare equal.
+  void cmeq(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare signed greater than or equal.
+  void cmge(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare signed greater than.
+  void cmgt(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare unsigned higher.
+  void cmhi(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare unsigned higher or same.
+  void cmhs(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Compare bitwise test bits nonzero.
+  void cmtst(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Compare bitwise to zero.
+  void cmeq(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed greater than or equal to zero.
+  void cmge(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed greater than zero.
+  void cmgt(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed less than or equal to zero.
+  void cmle(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Compare signed less than zero.
+  void cmlt(const VRegister& vd,
+            const VRegister& vn,
+            int value);
+
+  // Signed shift left by register.
+  void sshl(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned shift left by register.
+  void ushl(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed saturating shift left by register.
+  void sqshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned saturating shift left by register.
+  void uqshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed rounding shift left by register.
+  void srshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned rounding shift left by register.
+  void urshl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed saturating rounding shift left by register.
+  void sqrshl(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned saturating rounding shift left by register.
+  void uqrshl(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Bitwise and.
+  void and_(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Bitwise or.
+  void orr(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise or immediate.
+  void orr(const VRegister& vd,
+           const int imm8,
+           const int left_shift = 0);
+
+  // Move register to register.
+  void mov(const VRegister& vd,
+           const VRegister& vn);
+
+  // Bitwise orn.
+  void orn(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise eor.
+  void eor(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bit clear immediate.
+  void bic(const VRegister& vd,
+           const int imm8,
+           const int left_shift = 0);
+
+  // Bit clear.
+  void bic(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise insert if false.
+  void bif(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise insert if true.
+  void bit(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Bitwise select.
+  void bsl(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm);
+
+  // Polynomial multiply.
+  void pmul(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Vector move immediate.
+  void movi(const VRegister& vd,
+            const uint64_t imm,
+            Shift shift = LSL,
+            const int shift_amount = 0);
+
+  // Bitwise not.
+  void mvn(const VRegister& vd,
+           const VRegister& vn);
+
+  // Vector move inverted immediate.
+  void mvni(const VRegister& vd,
+            const int imm8,
+            Shift shift = LSL,
+            const int shift_amount = 0);
+
+  // Signed saturating accumulate of unsigned value.
+  void suqadd(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned saturating accumulate of signed value.
+  void usqadd(const VRegister& vd,
+              const VRegister& vn);
+
+  // Absolute value.
+  void abs(const VRegister& vd,
+           const VRegister& vn);
+
+  // Signed saturating absolute value.
+  void sqabs(const VRegister& vd,
+             const VRegister& vn);
+
+  // Negate.
+  void neg(const VRegister& vd,
+           const VRegister& vn);
+
+  // Signed saturating negate.
+  void sqneg(const VRegister& vd,
+             const VRegister& vn);
+
+  // Bitwise not.
+  void not_(const VRegister& vd,
+            const VRegister& vn);
+
+  // Extract narrow.
+  void xtn(const VRegister& vd,
+           const VRegister& vn);
+
+  // Extract narrow (second part).
+  void xtn2(const VRegister& vd,
+            const VRegister& vn);
+
+  // Signed saturating extract narrow.
+  void sqxtn(const VRegister& vd,
+             const VRegister& vn);
+
+  // Signed saturating extract narrow (second part).
+  void sqxtn2(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned saturating extract narrow.
+  void uqxtn(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned saturating extract narrow (second part).
+  void uqxtn2(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed saturating extract unsigned narrow.
+  void sqxtun(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed saturating extract unsigned narrow (second part).
+  void sqxtun2(const VRegister& vd,
+               const VRegister& vn);
+
+  // Extract vector from pair of vectors.
+  void ext(const VRegister& vd,
+           const VRegister& vn,
+           const VRegister& vm,
+           int index);
+
+  // Duplicate vector element to vector or scalar.
+  void dup(const VRegister& vd,
+           const VRegister& vn,
+           int vn_index);
+
+  // Move vector element to scalar.
+  void mov(const VRegister& vd,
+           const VRegister& vn,
+           int vn_index);
+
+  // Duplicate general-purpose register to vector.
+  void dup(const VRegister& vd,
+           const Register& rn);
+
+  // Insert vector element from another vector element.
+  void ins(const VRegister& vd,
+           int vd_index,
+           const VRegister& vn,
+           int vn_index);
+
+  // Move vector element to another vector element.
+  void mov(const VRegister& vd,
+           int vd_index,
+           const VRegister& vn,
+           int vn_index);
+
+  // Insert vector element from general-purpose register.
+  void ins(const VRegister& vd,
+           int vd_index,
+           const Register& rn);
+
+  // Move general-purpose register to a vector element.
+  void mov(const VRegister& vd,
+           int vd_index,
+           const Register& rn);
+
+  // Unsigned move vector element to general-purpose register.
+  void umov(const Register& rd,
+            const VRegister& vn,
+            int vn_index);
+
+  // Move vector element to general-purpose register.
+  void mov(const Register& rd,
+           const VRegister& vn,
+           int vn_index);
+
+  // Signed move vector element to general-purpose register.
+  void smov(const Register& rd,
+            const VRegister& vn,
+            int vn_index);
+
+  // One-element structure load to one register.
+  void ld1(const VRegister& vt,
+           const MemOperand& src);
+
+  // One-element structure load to two registers.
+  void ld1(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // One-element structure load to three registers.
+  void ld1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // One-element structure load to four registers.
+  void ld1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // One-element single structure load to one lane.
+  void ld1(const VRegister& vt,
+           int lane,
+           const MemOperand& src);
+
+  // One-element single structure load to all lanes.
+  void ld1r(const VRegister& vt,
+            const MemOperand& src);
+
+  // Two-element structure load.
+  void ld2(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // Two-element single structure load to one lane.
+  void ld2(const VRegister& vt,
+           const VRegister& vt2,
+           int lane,
+           const MemOperand& src);
+
+  // Two-element single structure load to all lanes.
+  void ld2r(const VRegister& vt,
+            const VRegister& vt2,
+            const MemOperand& src);
+
+  // Three-element structure load.
+  void ld3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // Three-element single structure load to one lane.
+  void ld3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           int lane,
+           const MemOperand& src);
+
+  // Three-element single structure load to all lanes.
+  void ld3r(const VRegister& vt,
+            const VRegister& vt2,
+            const VRegister& vt3,
+            const MemOperand& src);
+
+  // Four-element structure load.
+  void ld4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // Four-element single structure load to one lane.
+  void ld4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           int lane,
+           const MemOperand& src);
+
+  // Four-element single structure load to all lanes.
+  void ld4r(const VRegister& vt,
+            const VRegister& vt2,
+            const VRegister& vt3,
+            const VRegister& vt4,
+            const MemOperand& src);
+
+  // Count leading sign bits.
+  void cls(const VRegister& vd,
+           const VRegister& vn);
+
+  // Count leading zero bits (vector).
+  void clz(const VRegister& vd,
+           const VRegister& vn);
+
+  // Population count per byte.
+  void cnt(const VRegister& vd,
+           const VRegister& vn);
+
+  // Reverse bit order.
+  void rbit(const VRegister& vd,
+            const VRegister& vn);
+
+  // Reverse elements in 16-bit halfwords.
+  void rev16(const VRegister& vd,
+             const VRegister& vn);
+
+  // Reverse elements in 32-bit words.
+  void rev32(const VRegister& vd,
+             const VRegister& vn);
+
+  // Reverse elements in 64-bit doublewords.
+  void rev64(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned reciprocal square root estimate.
+  void ursqrte(const VRegister& vd,
+               const VRegister& vn);
+
+  // Unsigned reciprocal estimate.
+  void urecpe(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed pairwise long add.
+  void saddlp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned pairwise long add.
+  void uaddlp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Signed pairwise long add and accumulate.
+  void sadalp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned pairwise long add and accumulate.
+  void uadalp(const VRegister& vd,
+              const VRegister& vn);
+
+  // Shift left by immediate.
+  void shl(const VRegister& vd,
+           const VRegister& vn,
+           int shift);
+
+  // Signed saturating shift left by immediate.
+  void sqshl(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed saturating shift left unsigned by immediate.
+  void sqshlu(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Unsigned saturating shift left by immediate.
+  void uqshl(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed shift left long by immediate.
+  void sshll(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed shift left long by immediate (second part).
+  void sshll2(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Signed extend long.
+  void sxtl(const VRegister& vd,
+            const VRegister& vn);
+
+  // Signed extend long (second part).
+  void sxtl2(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned shift left long by immediate.
+  void ushll(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned shift left long by immediate (second part).
+  void ushll2(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Shift left long by element size.
+  void shll(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Shift left long by element size (second part).
+  void shll2(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned extend long.
+  void uxtl(const VRegister& vd,
+            const VRegister& vn);
+
+  // Unsigned extend long (second part).
+  void uxtl2(const VRegister& vd,
+             const VRegister& vn);
+
+  // Shift left by immediate and insert.
+  void sli(const VRegister& vd,
+           const VRegister& vn,
+           int shift);
+
+  // Shift right by immediate and insert.
+  void sri(const VRegister& vd,
+           const VRegister& vn,
+           int shift);
+
+  // Signed maximum.
+  void smax(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed pairwise maximum.
+  void smaxp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Add across vector.
+  void addv(const VRegister& vd,
+            const VRegister& vn);
+
+  // Signed add long across vector.
+  void saddlv(const VRegister& vd,
+              const VRegister& vn);
+
+  // Unsigned add long across vector.
+  void uaddlv(const VRegister& vd,
+              const VRegister& vn);
+
+  // FP maximum number across vector.
+  void fmaxnmv(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP maximum across vector.
+  void fmaxv(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP minimum number across vector.
+  void fminnmv(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP minimum across vector.
+  void fminv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Signed maximum across vector.
+  void smaxv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Signed minimum.
+  void smin(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed minimum pairwise.
+  void sminp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed minimum across vector.
+  void sminv(const VRegister& vd,
+             const VRegister& vn);
+
+  // One-element structure store from one register.
+  void st1(const VRegister& vt,
+           const MemOperand& src);
+
+  // One-element structure store from two registers.
+  void st1(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // One-element structure store from three registers.
+  void st1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // One-element structure store from four registers.
+  void st1(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // One-element single structure store from one lane.
+  void st1(const VRegister& vt,
+           int lane,
+           const MemOperand& src);
+
+  // Two-element structure store from two registers.
+  void st2(const VRegister& vt,
+           const VRegister& vt2,
+           const MemOperand& src);
+
+  // Two-element single structure store from two lanes.
+  void st2(const VRegister& vt,
+           const VRegister& vt2,
+           int lane,
+           const MemOperand& src);
+
+  // Three-element structure store from three registers.
+  void st3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const MemOperand& src);
+
+  // Three-element single structure store from three lanes.
+  void st3(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           int lane,
+           const MemOperand& src);
+
+  // Four-element structure store from four registers.
+  void st4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           const MemOperand& src);
+
+  // Four-element single structure store from four lanes.
+  void st4(const VRegister& vt,
+           const VRegister& vt2,
+           const VRegister& vt3,
+           const VRegister& vt4,
+           int lane,
+           const MemOperand& src);
+
+  // Unsigned add long.
+  void uaddl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned add long (second part).
+  void uaddl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned add wide.
+  void uaddw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned add wide (second part).
+  void uaddw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed add long.
+  void saddl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed add long (second part).
+  void saddl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed add wide.
+  void saddw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed add wide (second part).
+  void saddw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned subtract long.
+  void usubl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned subtract long (second part).
+  void usubl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned subtract wide.
+  void usubw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned subtract wide (second part).
+  void usubw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed subtract long.
+  void ssubl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed subtract long (second part).
+  void ssubl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed integer subtract wide.
+  void ssubw(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed integer subtract wide (second part).
+  void ssubw2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned maximum.
+  void umax(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned pairwise maximum.
+  void umaxp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned maximum across vector.
+  void umaxv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Unsigned minimum.
+  void umin(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unsigned pairwise minimum.
+  void uminp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned minimum across vector.
+  void uminv(const VRegister& vd,
+             const VRegister& vn);
+
+  // Transpose vectors (primary).
+  void trn1(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Transpose vectors (secondary).
+  void trn2(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unzip vectors (primary).
+  void uzp1(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Unzip vectors (secondary).
+  void uzp2(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Zip vectors (primary).
+  void zip1(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Zip vectors (secondary).
+  void zip2(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // Signed shift right by immediate.
+  void sshr(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Unsigned shift right by immediate.
+  void ushr(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Signed rounding shift right by immediate.
+  void srshr(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned rounding shift right by immediate.
+  void urshr(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Signed shift right by immediate and accumulate.
+  void ssra(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Unsigned shift right by immediate and accumulate.
+  void usra(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Signed rounding shift right by immediate and accumulate.
+  void srsra(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Unsigned rounding shift right by immediate and accumulate.
+  void ursra(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Shift right narrow by immediate.
+  void shrn(const VRegister& vd,
+            const VRegister& vn,
+            int shift);
+
+  // Shift right narrow by immediate (second part).
+  void shrn2(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Rounding shift right narrow by immediate.
+  void rshrn(const VRegister& vd,
+             const VRegister& vn,
+             int shift);
+
+  // Rounding shift right narrow by immediate (second part).
+  void rshrn2(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Unsigned saturating shift right narrow by immediate.
+  void uqshrn(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Unsigned saturating shift right narrow by immediate (second part).
+  void uqshrn2(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Unsigned saturating rounding shift right narrow by immediate.
+  void uqrshrn(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Unsigned saturating rounding shift right narrow by immediate (second part).
+  void uqrshrn2(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed saturating shift right narrow by immediate.
+  void sqshrn(const VRegister& vd,
+              const VRegister& vn,
+              int shift);
+
+  // Signed saturating shift right narrow by immediate (second part).
+  void sqshrn2(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Signed saturating rounded shift right narrow by immediate.
+  void sqrshrn(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Signed saturating rounded shift right narrow by immediate (second part).
+  void sqrshrn2(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed saturating shift right unsigned narrow by immediate.
+  void sqshrun(const VRegister& vd,
+               const VRegister& vn,
+               int shift);
+
+  // Signed saturating shift right unsigned narrow by immediate (second part).
+  void sqshrun2(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed sat rounded shift right unsigned narrow by immediate.
+  void sqrshrun(const VRegister& vd,
+                const VRegister& vn,
+                int shift);
+
+  // Signed sat rounded shift right unsigned narrow by immediate (second part).
+  void sqrshrun2(const VRegister& vd,
+                 const VRegister& vn,
+                 int shift);
+
+  // FP reciprocal step.
+  void frecps(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // FP reciprocal estimate.
+  void frecpe(const VRegister& vd,
+              const VRegister& vn);
+
+  // FP reciprocal square root estimate.
+  void frsqrte(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP reciprocal square root step.
+  void frsqrts(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed absolute difference and accumulate long.
+  void sabal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed absolute difference and accumulate long (second part).
+  void sabal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned absolute difference and accumulate long.
+  void uabal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned absolute difference and accumulate long (second part).
+  void uabal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed absolute difference long.
+  void sabdl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed absolute difference long (second part).
+  void sabdl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned absolute difference long.
+  void uabdl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned absolute difference long (second part).
+  void uabdl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Polynomial multiply long.
+  void pmull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Polynomial multiply long (second part).
+  void pmull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed long multiply-add.
+  void smlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed long multiply-add (second part).
+  void smlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned long multiply-add.
+  void umlal(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned long multiply-add (second part).
+  void umlal2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed long multiply-sub.
+  void smlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed long multiply-sub (second part).
+  void smlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Unsigned long multiply-sub.
+  void umlsl(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned long multiply-sub (second part).
+  void umlsl2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed long multiply.
+  void smull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Signed long multiply (second part).
+  void smull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Signed saturating doubling long multiply-add.
+  void sqdmlal(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating doubling long multiply-add (second part).
+  void sqdmlal2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling long multiply-subtract.
+  void sqdmlsl(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating doubling long multiply-subtract (second part).
+  void sqdmlsl2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling long multiply.
+  void sqdmull(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating doubling long multiply (second part).
+  void sqdmull2(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling multiply returning high half.
+  void sqdmulh(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Signed saturating rounding doubling multiply returning high half.
+  void sqrdmulh(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm);
+
+  // Signed saturating doubling multiply element returning high half.
+  void sqdmulh(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm,
+               int vm_index);
+
+  // Signed saturating rounding doubling multiply element returning high half.
+  void sqrdmulh(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                int vm_index);
+
+  // Unsigned long multiply long.
+  void umull(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Unsigned long multiply (second part).
+  void umull2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Add narrow returning high half.
+  void addhn(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Add narrow returning high half (second part).
+  void addhn2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding add narrow returning high half.
+  void raddhn(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding add narrow returning high half (second part).
+  void raddhn2(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // Subtract narrow returning high half.
+  void subhn(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // Subtract narrow returning high half (second part).
+  void subhn2(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding subtract narrow returning high half.
+  void rsubhn(const VRegister& vd,
+              const VRegister& vn,
+              const VRegister& vm);
+
+  // Rounding subtract narrow returning high half (second part).
+  void rsubhn2(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // FP vector multiply accumulate.
+  void fmla(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // FP vector multiply subtract.
+  void fmls(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // FP vector multiply extended.
+  void fmulx(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP absolute greater than or equal.
+  void facge(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP absolute greater than.
+  void facgt(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP multiply by element.
+  void fmul(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm,
+            int vm_index);
+
+  // FP fused multiply-add to accumulator by element.
+  void fmla(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm,
+            int vm_index);
+
+  // FP fused multiply-sub from accumulator by element.
+  void fmls(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm,
+            int vm_index);
+
+  // FP multiply extended by element.
+  void fmulx(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm,
+             int vm_index);
+
+  // FP compare equal.
+  void fcmeq(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP greater than.
+  void fcmgt(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP greater than or equal.
+  void fcmge(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP compare equal to zero.
+  void fcmeq(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP greater than zero.
+  void fcmgt(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP greater than or equal to zero.
+  void fcmge(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP less than or equal to zero.
+  void fcmle(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP less than to zero.
+  void fcmlt(const VRegister& vd,
+             const VRegister& vn,
+             double imm);
+
+  // FP absolute difference.
+  void fabd(const VRegister& vd,
+            const VRegister& vn,
+            const VRegister& vm);
+
+  // FP pairwise add vector.
+  void faddp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP pairwise add scalar.
+  void faddp(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP pairwise maximum vector.
+  void fmaxp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP pairwise maximum scalar.
+  void fmaxp(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP pairwise minimum vector.
+  void fminp(const VRegister& vd,
+             const VRegister& vn,
+             const VRegister& vm);
+
+  // FP pairwise minimum scalar.
+  void fminp(const VRegister& vd,
+             const VRegister& vn);
+
+  // FP pairwise maximum number vector.
+  void fmaxnmp(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // FP pairwise maximum number scalar.
+  void fmaxnmp(const VRegister& vd,
+               const VRegister& vn);
+
+  // FP pairwise minimum number vector.
+  void fminnmp(const VRegister& vd,
+               const VRegister& vn,
+               const VRegister& vm);
+
+  // FP pairwise minimum number scalar.
+  void fminnmp(const VRegister& vd,
+               const VRegister& vn);
+
+  // Emit generic instructions.
+  // Emit raw instructions into the instruction stream.
+  void dci(Instr raw_inst) { Emit(raw_inst); }
+
+  // Emit 32 bits of data into the instruction stream.
+  void dc32(uint32_t data) {
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit32(data);
+  }
+
+  // Emit 64 bits of data into the instruction stream.
+  void dc64(uint64_t data) {
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit64(data);
+  }
+
+  // Copy a string into the instruction stream, including the terminating NULL
+  // character. The instruction pointer is then aligned correctly for
+  // subsequent instructions.
+  void EmitString(const char * string) {
+    VIXL_ASSERT(string != NULL);
+    VIXL_ASSERT(buffer_monitor_ > 0);
+
+    buffer_->EmitString(string);
+    buffer_->Align();
+  }
+
+  // Code generation helpers.
+
+  // Register encoding.
+  static Instr Rd(CPURegister rd) {
+    VIXL_ASSERT(rd.code() != kSPRegInternalCode);
+    return rd.code() << Rd_offset;
+  }
+
+  static Instr Rn(CPURegister rn) {
+    VIXL_ASSERT(rn.code() != kSPRegInternalCode);
+    return rn.code() << Rn_offset;
+  }
+
+  static Instr Rm(CPURegister rm) {
+    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
+    return rm.code() << Rm_offset;
+  }
+
+  static Instr RmNot31(CPURegister rm) {
+    VIXL_ASSERT(rm.code() != kSPRegInternalCode);
+    VIXL_ASSERT(!rm.IsZero());
+    return Rm(rm);
+  }
+
+  static Instr Ra(CPURegister ra) {
+    VIXL_ASSERT(ra.code() != kSPRegInternalCode);
+    return ra.code() << Ra_offset;
+  }
+
+  static Instr Rt(CPURegister rt) {
+    VIXL_ASSERT(rt.code() != kSPRegInternalCode);
+    return rt.code() << Rt_offset;
+  }
+
+  static Instr Rt2(CPURegister rt2) {
+    VIXL_ASSERT(rt2.code() != kSPRegInternalCode);
+    return rt2.code() << Rt2_offset;
+  }
+
+  static Instr Rs(CPURegister rs) {
+    VIXL_ASSERT(rs.code() != kSPRegInternalCode);
+    return rs.code() << Rs_offset;
+  }
+
+  // These encoding functions allow the stack pointer to be encoded, and
+  // disallow the zero register.
+  static Instr RdSP(Register rd) {
+    VIXL_ASSERT(!rd.IsZero());
+    return (rd.code() & kRegCodeMask) << Rd_offset;
+  }
+
+  static Instr RnSP(Register rn) {
+    VIXL_ASSERT(!rn.IsZero());
+    return (rn.code() & kRegCodeMask) << Rn_offset;
+  }
+
+  // Flags encoding.
+  static Instr Flags(FlagsUpdate S) {
+    if (S == SetFlags) {
+      return 1 << FlagsUpdate_offset;
+    } else if (S == LeaveFlags) {
+      return 0 << FlagsUpdate_offset;
+    }
+    VIXL_UNREACHABLE();
+    return 0;
+  }
+
+  static Instr Cond(Condition cond) {
+    return cond << Condition_offset;
+  }
+
+  // PC-relative address encoding.
+  static Instr ImmPCRelAddress(int imm21) {
+    VIXL_ASSERT(is_int21(imm21));
+    Instr imm = static_cast<Instr>(truncate_to_int21(imm21));
+    Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
+    Instr immlo = imm << ImmPCRelLo_offset;
+    return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
+  }
+
+  // Branch encoding.
+  static Instr ImmUncondBranch(int imm26) {
+    VIXL_ASSERT(is_int26(imm26));
+    return truncate_to_int26(imm26) << ImmUncondBranch_offset;
+  }
+
+  static Instr ImmCondBranch(int imm19) {
+    VIXL_ASSERT(is_int19(imm19));
+    return truncate_to_int19(imm19) << ImmCondBranch_offset;
+  }
+
+  static Instr ImmCmpBranch(int imm19) {
+    VIXL_ASSERT(is_int19(imm19));
+    return truncate_to_int19(imm19) << ImmCmpBranch_offset;
+  }
+
+  static Instr ImmTestBranch(int imm14) {
+    VIXL_ASSERT(is_int14(imm14));
+    return truncate_to_int14(imm14) << ImmTestBranch_offset;
+  }
+
+  static Instr ImmTestBranchBit(unsigned bit_pos) {
+    VIXL_ASSERT(is_uint6(bit_pos));
+    // Subtract five from the shift offset, as we need bit 5 from bit_pos.
+    unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
+    unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
+    b5 &= ImmTestBranchBit5_mask;
+    b40 &= ImmTestBranchBit40_mask;
+    return b5 | b40;
+  }
+
+  // Data Processing encoding.
+  static Instr SF(Register rd) {
+      return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
+  }
+
+  static Instr ImmAddSub(int imm) {
+    VIXL_ASSERT(IsImmAddSub(imm));
+    if (is_uint12(imm)) {  // No shift required.
+      imm <<= ImmAddSub_offset;
+    } else {
+      imm = ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
+    }
+    return imm;
+  }
+
+  static Instr ImmS(unsigned imms, unsigned reg_size) {
+    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(imms)) ||
+           ((reg_size == kWRegSize) && is_uint5(imms)));
+    USE(reg_size);
+    return imms << ImmS_offset;
+  }
+
+  static Instr ImmR(unsigned immr, unsigned reg_size) {
+    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
+           ((reg_size == kWRegSize) && is_uint5(immr)));
+    USE(reg_size);
+    VIXL_ASSERT(is_uint6(immr));
+    return immr << ImmR_offset;
+  }
+
+  static Instr ImmSetBits(unsigned imms, unsigned reg_size) {
+    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+    VIXL_ASSERT(is_uint6(imms));
+    VIXL_ASSERT((reg_size == kXRegSize) || is_uint6(imms + 3));
+    USE(reg_size);
+    return imms << ImmSetBits_offset;
+  }
+
+  static Instr ImmRotate(unsigned immr, unsigned reg_size) {
+    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+    VIXL_ASSERT(((reg_size == kXRegSize) && is_uint6(immr)) ||
+           ((reg_size == kWRegSize) && is_uint5(immr)));
+    USE(reg_size);
+    return immr << ImmRotate_offset;
+  }
+
+  static Instr ImmLLiteral(int imm19) {
+    VIXL_ASSERT(is_int19(imm19));
+    return truncate_to_int19(imm19) << ImmLLiteral_offset;
+  }
+
+  static Instr BitN(unsigned bitn, unsigned reg_size) {
+    VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+    VIXL_ASSERT((reg_size == kXRegSize) || (bitn == 0));
+    USE(reg_size);
+    return bitn << BitN_offset;
+  }
+
+  static Instr ShiftDP(Shift shift) {
+    VIXL_ASSERT(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
+    return shift << ShiftDP_offset;
+  }
+
+  static Instr ImmDPShift(unsigned amount) {
+    VIXL_ASSERT(is_uint6(amount));
+    return amount << ImmDPShift_offset;
+  }
+
+  static Instr ExtendMode(Extend extend) {
+    return extend << ExtendMode_offset;
+  }
+
+  static Instr ImmExtendShift(unsigned left_shift) {
+    VIXL_ASSERT(left_shift <= 4);
+    return left_shift << ImmExtendShift_offset;
+  }
+
+  static Instr ImmCondCmp(unsigned imm) {
+    VIXL_ASSERT(is_uint5(imm));
+    return imm << ImmCondCmp_offset;
+  }
+
+  static Instr Nzcv(StatusFlags nzcv) {
+    return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
+  }
+
+  // MemOperand offset encoding.
+  static Instr ImmLSUnsigned(int imm12) {
+    VIXL_ASSERT(is_uint12(imm12));
+    return imm12 << ImmLSUnsigned_offset;
+  }
+
+  static Instr ImmLS(int imm9) {
+    VIXL_ASSERT(is_int9(imm9));
+    return truncate_to_int9(imm9) << ImmLS_offset;
+  }
+
+  static Instr ImmLSPair(int imm7, unsigned access_size) {
+    VIXL_ASSERT(((imm7 >> access_size) << access_size) == imm7);
+    int scaled_imm7 = imm7 >> access_size;
+    VIXL_ASSERT(is_int7(scaled_imm7));
+    return truncate_to_int7(scaled_imm7) << ImmLSPair_offset;
+  }
+
+  static Instr ImmShiftLS(unsigned shift_amount) {
+    VIXL_ASSERT(is_uint1(shift_amount));
+    return shift_amount << ImmShiftLS_offset;
+  }
+
+  static Instr ImmPrefetchOperation(int imm5) {
+    VIXL_ASSERT(is_uint5(imm5));
+    return imm5 << ImmPrefetchOperation_offset;
+  }
+
+  static Instr ImmException(int imm16) {
+    VIXL_ASSERT(is_uint16(imm16));
+    return imm16 << ImmException_offset;
+  }
+
+  static Instr ImmSystemRegister(int imm15) {
+    VIXL_ASSERT(is_uint15(imm15));
+    return imm15 << ImmSystemRegister_offset;
+  }
+
+  static Instr ImmHint(int imm7) {
+    VIXL_ASSERT(is_uint7(imm7));
+    return imm7 << ImmHint_offset;
+  }
+
+  static Instr CRm(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << CRm_offset;
+  }
+
+  static Instr CRn(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << CRn_offset;
+  }
+
+  static Instr SysOp(int imm14) {
+    VIXL_ASSERT(is_uint14(imm14));
+    return imm14 << SysOp_offset;
+  }
+
+  static Instr ImmSysOp1(int imm3) {
+    VIXL_ASSERT(is_uint3(imm3));
+    return imm3 << SysOp1_offset;
+  }
+
+  static Instr ImmSysOp2(int imm3) {
+    VIXL_ASSERT(is_uint3(imm3));
+    return imm3 << SysOp2_offset;
+  }
+
+  static Instr ImmBarrierDomain(int imm2) {
+    VIXL_ASSERT(is_uint2(imm2));
+    return imm2 << ImmBarrierDomain_offset;
+  }
+
+  static Instr ImmBarrierType(int imm2) {
+    VIXL_ASSERT(is_uint2(imm2));
+    return imm2 << ImmBarrierType_offset;
+  }
+
+  // Move immediates encoding.
+  static Instr ImmMoveWide(uint64_t imm) {
+    VIXL_ASSERT(is_uint16(imm));
+    return static_cast<Instr>(imm << ImmMoveWide_offset);
+  }
+
+  static Instr ShiftMoveWide(int64_t shift) {
+    VIXL_ASSERT(is_uint2(shift));
+    return static_cast<Instr>(shift << ShiftMoveWide_offset);
+  }
+
+  // FP Immediates.
+  static Instr ImmFP32(float imm);
+  static Instr ImmFP64(double imm);
+
+  // FP register type.
+  static Instr FPType(FPRegister fd) {
+    return fd.Is64Bits() ? FP64 : FP32;
+  }
+
+  static Instr FPScale(unsigned scale) {
+    VIXL_ASSERT(is_uint6(scale));
+    return scale << FPScale_offset;
+  }
+
+  // Immediate field checking helpers.
+  static bool IsImmAddSub(int64_t immediate);
+  static bool IsImmConditionalCompare(int64_t immediate);
+  static bool IsImmFP32(float imm);
+  static bool IsImmFP64(double imm);
+  static bool IsImmLogical(uint64_t value,
+                           unsigned width,
+                           unsigned* n = NULL,
+                           unsigned* imm_s = NULL,
+                           unsigned* imm_r = NULL);
+  static bool IsImmLSPair(int64_t offset, unsigned access_size);
+  static bool IsImmLSScaled(int64_t offset, unsigned access_size);
+  static bool IsImmLSUnscaled(int64_t offset);
+  static bool IsImmMovn(uint64_t imm, unsigned reg_size);
+  static bool IsImmMovz(uint64_t imm, unsigned reg_size);
+
+  // Instruction bits for vector format in data processing operations.
+  static Instr VFormat(VRegister vd) {
+    if (vd.Is64Bits()) {
+      switch (vd.lanes()) {
+        case 2: return NEON_2S;
+        case 4: return NEON_4H;
+        case 8: return NEON_8B;
+        default: return 0xffffffff;
+      }
+    } else {
+      VIXL_ASSERT(vd.Is128Bits());
+      switch (vd.lanes()) {
+        case 2: return NEON_2D;
+        case 4: return NEON_4S;
+        case 8: return NEON_8H;
+        case 16: return NEON_16B;
+        default: return 0xffffffff;
+      }
+    }
+  }
+
+  // Instruction bits for vector format in floating point data processing
+  // operations.
+  static Instr FPFormat(VRegister vd) {
+    if (vd.lanes() == 1) {
+      // Floating point scalar formats.
+      VIXL_ASSERT(vd.Is32Bits() || vd.Is64Bits());
+      return vd.Is64Bits() ? FP64 : FP32;
+    }
+
+    // Two lane floating point vector formats.
+    if (vd.lanes() == 2) {
+      VIXL_ASSERT(vd.Is64Bits() || vd.Is128Bits());
+      return vd.Is128Bits() ? NEON_FP_2D : NEON_FP_2S;
+    }
+
+    // Four lane floating point vector format.
+    VIXL_ASSERT((vd.lanes() == 4) && vd.Is128Bits());
+    return NEON_FP_4S;
+  }
+
+  // Instruction bits for vector format in load and store operations.
+  static Instr LSVFormat(VRegister vd) {
+    if (vd.Is64Bits()) {
+      switch (vd.lanes()) {
+        case 1: return LS_NEON_1D;
+        case 2: return LS_NEON_2S;
+        case 4: return LS_NEON_4H;
+        case 8: return LS_NEON_8B;
+        default: return 0xffffffff;
+      }
+    } else {
+      VIXL_ASSERT(vd.Is128Bits());
+      switch (vd.lanes()) {
+        case 2: return LS_NEON_2D;
+        case 4: return LS_NEON_4S;
+        case 8: return LS_NEON_8H;
+        case 16: return LS_NEON_16B;
+        default: return 0xffffffff;
+      }
+    }
+  }
+
+  // Instruction bits for scalar format in data processing operations.
+  static Instr SFormat(VRegister vd) {
+    VIXL_ASSERT(vd.lanes() == 1);
+    switch (vd.SizeInBytes()) {
+      case 1: return NEON_B;
+      case 2: return NEON_H;
+      case 4: return NEON_S;
+      case 8: return NEON_D;
+      default: return 0xffffffff;
+    }
+  }
+
+  static Instr ImmNEONHLM(int index, int num_bits) {
+    int h, l, m;
+    if (num_bits == 3) {
+      VIXL_ASSERT(is_uint3(index));
+      h  = (index >> 2) & 1;
+      l  = (index >> 1) & 1;
+      m  = (index >> 0) & 1;
+    } else if (num_bits == 2) {
+      VIXL_ASSERT(is_uint2(index));
+      h  = (index >> 1) & 1;
+      l  = (index >> 0) & 1;
+      m  = 0;
+    } else {
+      VIXL_ASSERT(is_uint1(index) && (num_bits == 1));
+      h  = (index >> 0) & 1;
+      l  = 0;
+      m  = 0;
+    }
+    return (h << NEONH_offset) | (l << NEONL_offset) | (m << NEONM_offset);
+  }
+
+  static Instr ImmNEONExt(int imm4) {
+    VIXL_ASSERT(is_uint4(imm4));
+    return imm4 << ImmNEONExt_offset;
+  }
+
+  static Instr ImmNEON5(Instr format, int index) {
+    VIXL_ASSERT(is_uint4(index));
+    int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+    int imm5 = (index << (s + 1)) | (1 << s);
+    return imm5 << ImmNEON5_offset;
+  }
+
+  static Instr ImmNEON4(Instr format, int index) {
+    VIXL_ASSERT(is_uint4(index));
+    int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+    int imm4 = index << s;
+    return imm4 << ImmNEON4_offset;
+  }
+
+  static Instr ImmNEONabcdefgh(int imm8) {
+    VIXL_ASSERT(is_uint8(imm8));
+    Instr instr;
+    instr  = ((imm8 >> 5) & 7) << ImmNEONabc_offset;
+    instr |= (imm8 & 0x1f) << ImmNEONdefgh_offset;
+    return instr;
+  }
+
+  static Instr NEONCmode(int cmode) {
+    VIXL_ASSERT(is_uint4(cmode));
+    return cmode << NEONCmode_offset;
+  }
+
+  static Instr NEONModImmOp(int op) {
+    VIXL_ASSERT(is_uint1(op));
+    return op << NEONModImmOp_offset;
+  }
+
+  // Size of the code generated since label to the current position.
+  size_t SizeOfCodeGeneratedSince(Label* label) const {
+    VIXL_ASSERT(label->IsBound());
+    return buffer_->OffsetFrom(label->location());
+  }
+
+  size_t SizeOfCodeGenerated() const {
+    return buffer_->CursorOffset();
+  }
+
+  size_t BufferCapacity() const { return buffer_->capacity(); }
+
+  size_t RemainingBufferSpace() const { return buffer_->RemainingBytes(); }
+
+  void EnsureSpaceFor(size_t amount) {
+    if (buffer_->RemainingBytes() < amount) {
+      size_t capacity = buffer_->capacity();
+      size_t size = buffer_->CursorOffset();
+      do {
+        // TODO(all): refine.
+        capacity *= 2;
+      } while ((capacity - size) <  amount);
+      buffer_->Grow(capacity);
+    }
+  }
+
+#ifdef VIXL_DEBUG
+  void AcquireBuffer() {
+    VIXL_ASSERT(buffer_monitor_ >= 0);
+    buffer_monitor_++;
+  }
+
+  void ReleaseBuffer() {
+    buffer_monitor_--;
+    VIXL_ASSERT(buffer_monitor_ >= 0);
+  }
+#endif
+
+  PositionIndependentCodeOption pic() const {
+    return pic_;
+  }
+
+  bool AllowPageOffsetDependentCode() const {
+    return (pic() == PageOffsetDependentCode) ||
+           (pic() == PositionDependentCode);
+  }
+
+  static const Register& AppropriateZeroRegFor(const CPURegister& reg) {
+    return reg.Is64Bits() ? xzr : wzr;
+  }
+
+
+ protected:
+  void LoadStore(const CPURegister& rt,
+                 const MemOperand& addr,
+                 LoadStoreOp op,
+                 LoadStoreScalingOption option = PreferScaledOffset);
+
+  void LoadStorePair(const CPURegister& rt,
+                     const CPURegister& rt2,
+                     const MemOperand& addr,
+                     LoadStorePairOp op);
+  void LoadStoreStruct(const VRegister& vt,
+                       const MemOperand& addr,
+                       NEONLoadStoreMultiStructOp op);
+  void LoadStoreStruct1(const VRegister& vt,
+                        int reg_count,
+                        const MemOperand& addr);
+  void LoadStoreStructSingle(const VRegister& vt,
+                             uint32_t lane,
+                             const MemOperand& addr,
+                             NEONLoadStoreSingleStructOp op);
+  void LoadStoreStructSingleAllLanes(const VRegister& vt,
+                                     const MemOperand& addr,
+                                     NEONLoadStoreSingleStructOp op);
+  void LoadStoreStructVerify(const VRegister& vt,
+                             const MemOperand& addr,
+                             Instr op);
+
+  void Prefetch(PrefetchOperation op,
+                const MemOperand& addr,
+                LoadStoreScalingOption option = PreferScaledOffset);
+
+  // TODO(all): The third parameter should be passed by reference but gcc 4.8.2
+  // reports a bogus uninitialised warning then.
+  void Logical(const Register& rd,
+               const Register& rn,
+               const Operand operand,
+               LogicalOp op);
+  void LogicalImmediate(const Register& rd,
+                        const Register& rn,
+                        unsigned n,
+                        unsigned imm_s,
+                        unsigned imm_r,
+                        LogicalOp op);
+
+  void ConditionalCompare(const Register& rn,
+                          const Operand& operand,
+                          StatusFlags nzcv,
+                          Condition cond,
+                          ConditionalCompareOp op);
+
+  void AddSubWithCarry(const Register& rd,
+                       const Register& rn,
+                       const Operand& operand,
+                       FlagsUpdate S,
+                       AddSubWithCarryOp op);
+
+
+  // Functions for emulating operands not directly supported by the instruction
+  // set.
+  void EmitShift(const Register& rd,
+                 const Register& rn,
+                 Shift shift,
+                 unsigned amount);
+  void EmitExtendShift(const Register& rd,
+                       const Register& rn,
+                       Extend extend,
+                       unsigned left_shift);
+
+  void AddSub(const Register& rd,
+              const Register& rn,
+              const Operand& operand,
+              FlagsUpdate S,
+              AddSubOp op);
+
+  void NEONTable(const VRegister& vd,
+                 const VRegister& vn,
+                 const VRegister& vm,
+                 NEONTableOp op);
+
+  // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
+  // registers. Only simple loads are supported; sign- and zero-extension (such
+  // as in LDPSW_x or LDRB_w) are not supported.
+  static LoadStoreOp LoadOpFor(const CPURegister& rt);
+  static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
+                                       const CPURegister& rt2);
+  static LoadStoreOp StoreOpFor(const CPURegister& rt);
+  static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
+                                        const CPURegister& rt2);
+  static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
+    const CPURegister& rt, const CPURegister& rt2);
+  static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
+    const CPURegister& rt, const CPURegister& rt2);
+  static LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
+
+
+ private:
+  static uint32_t FP32ToImm8(float imm);
+  static uint32_t FP64ToImm8(double imm);
+
+  // Instruction helpers.
+  void MoveWide(const Register& rd,
+                uint64_t imm,
+                int shift,
+                MoveWideImmediateOp mov_op);
+  void DataProcShiftedRegister(const Register& rd,
+                               const Register& rn,
+                               const Operand& operand,
+                               FlagsUpdate S,
+                               Instr op);
+  void DataProcExtendedRegister(const Register& rd,
+                                const Register& rn,
+                                const Operand& operand,
+                                FlagsUpdate S,
+                                Instr op);
+  void LoadStorePairNonTemporal(const CPURegister& rt,
+                                const CPURegister& rt2,
+                                const MemOperand& addr,
+                                LoadStorePairNonTemporalOp op);
+  void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
+  void ConditionalSelect(const Register& rd,
+                         const Register& rn,
+                         const Register& rm,
+                         Condition cond,
+                         ConditionalSelectOp op);
+  void DataProcessing1Source(const Register& rd,
+                             const Register& rn,
+                             DataProcessing1SourceOp op);
+  void DataProcessing3Source(const Register& rd,
+                             const Register& rn,
+                             const Register& rm,
+                             const Register& ra,
+                             DataProcessing3SourceOp op);
+  void FPDataProcessing1Source(const VRegister& fd,
+                               const VRegister& fn,
+                               FPDataProcessing1SourceOp op);
+  void FPDataProcessing3Source(const VRegister& fd,
+                               const VRegister& fn,
+                               const VRegister& fm,
+                               const VRegister& fa,
+                               FPDataProcessing3SourceOp op);
+  void NEONAcrossLanesL(const VRegister& vd,
+                        const VRegister& vn,
+                        NEONAcrossLanesOp op);
+  void NEONAcrossLanes(const VRegister& vd,
+                       const VRegister& vn,
+                       NEONAcrossLanesOp op);
+  void NEONModifiedImmShiftLsl(const VRegister& vd,
+                               const int imm8,
+                               const int left_shift,
+                               NEONModifiedImmediateOp op);
+  void NEONModifiedImmShiftMsl(const VRegister& vd,
+                               const int imm8,
+                               const int shift_amount,
+                               NEONModifiedImmediateOp op);
+  void NEONFP2Same(const VRegister& vd,
+                   const VRegister& vn,
+                   Instr vop);
+  void NEON3Same(const VRegister& vd,
+                 const VRegister& vn,
+                 const VRegister& vm,
+                 NEON3SameOp vop);
+  void NEONFP3Same(const VRegister& vd,
+                   const VRegister& vn,
+                   const VRegister& vm,
+                   Instr op);
+  void NEON3DifferentL(const VRegister& vd,
+                       const VRegister& vn,
+                       const VRegister& vm,
+                       NEON3DifferentOp vop);
+  void NEON3DifferentW(const VRegister& vd,
+                       const VRegister& vn,
+                       const VRegister& vm,
+                       NEON3DifferentOp vop);
+  void NEON3DifferentHN(const VRegister& vd,
+                        const VRegister& vn,
+                        const VRegister& vm,
+                        NEON3DifferentOp vop);
+  void NEONFP2RegMisc(const VRegister& vd,
+                      const VRegister& vn,
+                      NEON2RegMiscOp vop,
+                      double value = 0.0);
+  void NEON2RegMisc(const VRegister& vd,
+                    const VRegister& vn,
+                    NEON2RegMiscOp vop,
+                    int value = 0);
+  void NEONFP2RegMisc(const VRegister& vd,
+                      const VRegister& vn,
+                      Instr op);
+  void NEONAddlp(const VRegister& vd,
+                 const VRegister& vn,
+                 NEON2RegMiscOp op);
+  void NEONPerm(const VRegister& vd,
+                const VRegister& vn,
+                const VRegister& vm,
+                NEONPermOp op);
+  void NEONFPByElement(const VRegister& vd,
+                       const VRegister& vn,
+                       const VRegister& vm,
+                       int vm_index,
+                       NEONByIndexedElementOp op);
+  void NEONByElement(const VRegister& vd,
+                     const VRegister& vn,
+                     const VRegister& vm,
+                     int vm_index,
+                     NEONByIndexedElementOp op);
+  void NEONByElementL(const VRegister& vd,
+                      const VRegister& vn,
+                      const VRegister& vm,
+                      int vm_index,
+                      NEONByIndexedElementOp op);
+  void NEONShiftImmediate(const VRegister& vd,
+                          const VRegister& vn,
+                          NEONShiftImmediateOp op,
+                          int immh_immb);
+  void NEONShiftLeftImmediate(const VRegister& vd,
+                              const VRegister& vn,
+                              int shift,
+                              NEONShiftImmediateOp op);
+  void NEONShiftRightImmediate(const VRegister& vd,
+                               const VRegister& vn,
+                               int shift,
+                               NEONShiftImmediateOp op);
+  void NEONShiftImmediateL(const VRegister& vd,
+                           const VRegister& vn,
+                           int shift,
+                           NEONShiftImmediateOp op);
+  void NEONShiftImmediateN(const VRegister& vd,
+                           const VRegister& vn,
+                           int shift,
+                           NEONShiftImmediateOp op);
+  void NEONXtn(const VRegister& vd,
+               const VRegister& vn,
+               NEON2RegMiscOp vop);
+
+  Instr LoadStoreStructAddrModeField(const MemOperand& addr);
+
+  // Encode the specified MemOperand for the specified access size and scaling
+  // preference.
+  Instr LoadStoreMemOperand(const MemOperand& addr,
+                            unsigned access_size,
+                            LoadStoreScalingOption option);
+
+  // Link the current (not-yet-emitted) instruction to the specified label, then
+  // return an offset to be encoded in the instruction. If the label is not yet
+  // bound, an offset of 0 is returned.
+  ptrdiff_t LinkAndGetByteOffsetTo(Label * label);
+  ptrdiff_t LinkAndGetInstructionOffsetTo(Label * label);
+  ptrdiff_t LinkAndGetPageOffsetTo(Label * label);
+
+  // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
+  template <int element_shift>
+  ptrdiff_t LinkAndGetOffsetTo(Label* label);
+
+  // Literal load offset are in words (32-bit).
+  ptrdiff_t LinkAndGetWordOffsetTo(RawLiteral* literal);
+
+  // Emit the instruction in buffer_.
+  void Emit(Instr instruction) {
+    VIXL_STATIC_ASSERT(sizeof(instruction) == kInstructionSize);
+    VIXL_ASSERT(buffer_monitor_ > 0);
+    buffer_->Emit32(instruction);
+  }
+
+  // Buffer where the code is emitted.
+  CodeBuffer* buffer_;
+  PositionIndependentCodeOption pic_;
+
+#ifdef VIXL_DEBUG
+  int64_t buffer_monitor_;
+#endif
+};
+
+
+// All Assembler emits MUST acquire/release the underlying code buffer. The
+// helper scope below will do so and optionally ensure the buffer is big enough
+// to receive the emit. It is possible to request the scope not to perform any
+// checks (kNoCheck) if for example it is known in advance the buffer size is
+// adequate or there is some other size checking mechanism in place.
+class CodeBufferCheckScope {
+ public:
+  // Tell whether or not the scope needs to ensure the associated CodeBuffer
+  // has enough space for the requested size.
+  enum CheckPolicy {
+    kNoCheck,
+    kCheck
+  };
+
+  // Tell whether or not the scope should assert the amount of code emitted
+  // within the scope is consistent with the requested amount.
+  enum AssertPolicy {
+    kNoAssert,    // No assert required.
+    kExactSize,   // The code emitted must be exactly size bytes.
+    kMaximumSize  // The code emitted must be at most size bytes.
+  };
+
+  CodeBufferCheckScope(Assembler* assm,
+                       size_t size,
+                       CheckPolicy check_policy = kCheck,
+                       AssertPolicy assert_policy = kMaximumSize)
+      : assm_(assm) {
+    if (check_policy == kCheck) assm->EnsureSpaceFor(size);
+#ifdef VIXL_DEBUG
+    assm->bind(&start_);
+    size_ = size;
+    assert_policy_ = assert_policy;
+    assm->AcquireBuffer();
+#else
+    USE(assert_policy);
+#endif
+  }
+
+  // This is a shortcut for CodeBufferCheckScope(assm, 0, kNoCheck, kNoAssert).
+  explicit CodeBufferCheckScope(Assembler* assm) : assm_(assm) {
+#ifdef VIXL_DEBUG
+    size_ = 0;
+    assert_policy_ = kNoAssert;
+    assm->AcquireBuffer();
+#endif
+  }
+
+  ~CodeBufferCheckScope() {
+#ifdef VIXL_DEBUG
+    assm_->ReleaseBuffer();
+    switch (assert_policy_) {
+      case kNoAssert: break;
+      case kExactSize:
+        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) == size_);
+        break;
+      case kMaximumSize:
+        VIXL_ASSERT(assm_->SizeOfCodeGeneratedSince(&start_) <= size_);
+        break;
+      default:
+        VIXL_UNREACHABLE();
+    }
+#endif
+  }
+
+ protected:
+  Assembler* assm_;
+#ifdef VIXL_DEBUG
+  Label start_;
+  size_t size_;
+  AssertPolicy assert_policy_;
+#endif
+};
+
+
+template <typename T>
+void Literal<T>::UpdateValue(T new_value, const Assembler* assembler) {
+  return UpdateValue(new_value, assembler->GetStartAddress<uint8_t*>());
+}
+
+
+template <typename T>
+void Literal<T>::UpdateValue(T high64, T low64, const Assembler* assembler) {
+  return UpdateValue(high64, low64, assembler->GetStartAddress<uint8_t*>());
+}
+
+
+}  // namespace vixl
+
+#endif  // VIXL_A64_ASSEMBLER_A64_H_
diff --git a/disas/libvixl/vixl/a64/constants-a64.h b/disas/libvixl/vixl/a64/constants-a64.h
new file mode 100644
index 0000000..2caa73a
--- /dev/null
+++ b/disas/libvixl/vixl/a64/constants-a64.h
@@ -0,0 +1,2116 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_CONSTANTS_A64_H_
+#define VIXL_A64_CONSTANTS_A64_H_
+
+namespace vixl {
+
+const unsigned kNumberOfRegisters = 32;
+const unsigned kNumberOfVRegisters = 32;
+const unsigned kNumberOfFPRegisters = kNumberOfVRegisters;
+// Callee saved registers are x21-x30(lr).
+const int kNumberOfCalleeSavedRegisters = 10;
+const int kFirstCalleeSavedRegisterIndex = 21;
+// Callee saved FP registers are d8-d15.
+const int kNumberOfCalleeSavedFPRegisters = 8;
+const int kFirstCalleeSavedFPRegisterIndex = 8;
+
+#define REGISTER_CODE_LIST(R)                                                  \
+R(0)  R(1)  R(2)  R(3)  R(4)  R(5)  R(6)  R(7)                                 \
+R(8)  R(9)  R(10) R(11) R(12) R(13) R(14) R(15)                                \
+R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23)                                \
+R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
+
+#define INSTRUCTION_FIELDS_LIST(V_)                                            \
+/* Register fields */                                                          \
+V_(Rd, 4, 0, Bits)                        /* Destination register.        */   \
+V_(Rn, 9, 5, Bits)                        /* First source register.       */   \
+V_(Rm, 20, 16, Bits)                      /* Second source register.      */   \
+V_(Ra, 14, 10, Bits)                      /* Third source register.       */   \
+V_(Rt, 4, 0, Bits)                        /* Load/store register.         */   \
+V_(Rt2, 14, 10, Bits)                     /* Load/store second register.  */   \
+V_(Rs, 20, 16, Bits)                      /* Exclusive access status.     */   \
+                                                                               \
+/* Common bits */                                                              \
+V_(SixtyFourBits, 31, 31, Bits)                                                \
+V_(FlagsUpdate, 29, 29, Bits)                                                  \
+                                                                               \
+/* PC relative addressing */                                                   \
+V_(ImmPCRelHi, 23, 5, SignedBits)                                              \
+V_(ImmPCRelLo, 30, 29, Bits)                                                   \
+                                                                               \
+/* Add/subtract/logical shift register */                                      \
+V_(ShiftDP, 23, 22, Bits)                                                      \
+V_(ImmDPShift, 15, 10, Bits)                                                   \
+                                                                               \
+/* Add/subtract immediate */                                                   \
+V_(ImmAddSub, 21, 10, Bits)                                                    \
+V_(ShiftAddSub, 23, 22, Bits)                                                  \
+                                                                               \
+/* Add/substract extend */                                                     \
+V_(ImmExtendShift, 12, 10, Bits)                                               \
+V_(ExtendMode, 15, 13, Bits)                                                   \
+                                                                               \
+/* Move wide */                                                                \
+V_(ImmMoveWide, 20, 5, Bits)                                                   \
+V_(ShiftMoveWide, 22, 21, Bits)                                                \
+                                                                               \
+/* Logical immediate, bitfield and extract */                                  \
+V_(BitN, 22, 22, Bits)                                                         \
+V_(ImmRotate, 21, 16, Bits)                                                    \
+V_(ImmSetBits, 15, 10, Bits)                                                   \
+V_(ImmR, 21, 16, Bits)                                                         \
+V_(ImmS, 15, 10, Bits)                                                         \
+                                                                               \
+/* Test and branch immediate */                                                \
+V_(ImmTestBranch, 18, 5, SignedBits)                                           \
+V_(ImmTestBranchBit40, 23, 19, Bits)                                           \
+V_(ImmTestBranchBit5, 31, 31, Bits)                                            \
+                                                                               \
+/* Conditionals */                                                             \
+V_(Condition, 15, 12, Bits)                                                    \
+V_(ConditionBranch, 3, 0, Bits)                                                \
+V_(Nzcv, 3, 0, Bits)                                                           \
+V_(ImmCondCmp, 20, 16, Bits)                                                   \
+V_(ImmCondBranch, 23, 5, SignedBits)                                           \
+                                                                               \
+/* Floating point */                                                           \
+V_(FPType, 23, 22, Bits)                                                       \
+V_(ImmFP, 20, 13, Bits)                                                        \
+V_(FPScale, 15, 10, Bits)                                                      \
+                                                                               \
+/* Load Store */                                                               \
+V_(ImmLS, 20, 12, SignedBits)                                                  \
+V_(ImmLSUnsigned, 21, 10, Bits)                                                \
+V_(ImmLSPair, 21, 15, SignedBits)                                              \
+V_(ImmShiftLS, 12, 12, Bits)                                                   \
+V_(LSOpc, 23, 22, Bits)                                                        \
+V_(LSVector, 26, 26, Bits)                                                     \
+V_(LSSize, 31, 30, Bits)                                                       \
+V_(ImmPrefetchOperation, 4, 0, Bits)                                           \
+V_(PrefetchHint, 4, 3, Bits)                                                   \
+V_(PrefetchTarget, 2, 1, Bits)                                                 \
+V_(PrefetchStream, 0, 0, Bits)                                                 \
+                                                                               \
+/* Other immediates */                                                         \
+V_(ImmUncondBranch, 25, 0, SignedBits)                                         \
+V_(ImmCmpBranch, 23, 5, SignedBits)                                            \
+V_(ImmLLiteral, 23, 5, SignedBits)                                             \
+V_(ImmException, 20, 5, Bits)                                                  \
+V_(ImmHint, 11, 5, Bits)                                                       \
+V_(ImmBarrierDomain, 11, 10, Bits)                                             \
+V_(ImmBarrierType, 9, 8, Bits)                                                 \
+                                                                               \
+/* System (MRS, MSR, SYS) */                                                   \
+V_(ImmSystemRegister, 19, 5, Bits)                                             \
+V_(SysO0, 19, 19, Bits)                                                        \
+V_(SysOp, 18, 5, Bits)                                                         \
+V_(SysOp1, 18, 16, Bits)                                                       \
+V_(SysOp2, 7, 5, Bits)                                                         \
+V_(CRn, 15, 12, Bits)                                                          \
+V_(CRm, 11, 8, Bits)                                                           \
+                                                                               \
+/* Load-/store-exclusive */                                                    \
+V_(LdStXLoad, 22, 22, Bits)                                                    \
+V_(LdStXNotExclusive, 23, 23, Bits)                                            \
+V_(LdStXAcquireRelease, 15, 15, Bits)                                          \
+V_(LdStXSizeLog2, 31, 30, Bits)                                                \
+V_(LdStXPair, 21, 21, Bits)                                                    \
+                                                                               \
+/* NEON generic fields */                                                      \
+V_(NEONQ, 30, 30, Bits)                                                        \
+V_(NEONSize, 23, 22, Bits)                                                     \
+V_(NEONLSSize, 11, 10, Bits)                                                   \
+V_(NEONS, 12, 12, Bits)                                                        \
+V_(NEONL, 21, 21, Bits)                                                        \
+V_(NEONM, 20, 20, Bits)                                                        \
+V_(NEONH, 11, 11, Bits)                                                        \
+V_(ImmNEONExt, 14, 11, Bits)                                                   \
+V_(ImmNEON5, 20, 16, Bits)                                                     \
+V_(ImmNEON4, 14, 11, Bits)                                                     \
+                                                                               \
+/* NEON Modified Immediate fields */                                           \
+V_(ImmNEONabc, 18, 16, Bits)                                                   \
+V_(ImmNEONdefgh, 9, 5, Bits)                                                   \
+V_(NEONModImmOp, 29, 29, Bits)                                                 \
+V_(NEONCmode, 15, 12, Bits)                                                    \
+                                                                               \
+/* NEON Shift Immediate fields */                                              \
+V_(ImmNEONImmhImmb, 22, 16, Bits)                                              \
+V_(ImmNEONImmh, 22, 19, Bits)                                                  \
+V_(ImmNEONImmb, 18, 16, Bits)
+
+#define SYSTEM_REGISTER_FIELDS_LIST(V_, M_)                                    \
+/* NZCV */                                                                     \
+V_(Flags, 31, 28, Bits)                                                        \
+V_(N, 31, 31, Bits)                                                            \
+V_(Z, 30, 30, Bits)                                                            \
+V_(C, 29, 29, Bits)                                                            \
+V_(V, 28, 28, Bits)                                                            \
+M_(NZCV, Flags_mask)                                                           \
+/* FPCR */                                                                     \
+V_(AHP, 26, 26, Bits)                                                          \
+V_(DN, 25, 25, Bits)                                                           \
+V_(FZ, 24, 24, Bits)                                                           \
+V_(RMode, 23, 22, Bits)                                                        \
+M_(FPCR, AHP_mask | DN_mask | FZ_mask | RMode_mask)
+
+// Fields offsets.
+#define DECLARE_FIELDS_OFFSETS(Name, HighBit, LowBit, X)                       \
+const int Name##_offset = LowBit;                                              \
+const int Name##_width = HighBit - LowBit + 1;                                 \
+const uint32_t Name##_mask = ((1 << Name##_width) - 1) << LowBit;
+#define NOTHING(A, B)
+INSTRUCTION_FIELDS_LIST(DECLARE_FIELDS_OFFSETS)
+SYSTEM_REGISTER_FIELDS_LIST(DECLARE_FIELDS_OFFSETS, NOTHING)
+#undef NOTHING
+#undef DECLARE_FIELDS_BITS
+
+// ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST), formed
+// from ImmPCRelLo and ImmPCRelHi.
+const int ImmPCRel_mask = ImmPCRelLo_mask | ImmPCRelHi_mask;
+
+// Condition codes.
+enum Condition {
+  eq = 0,   // Z set            Equal.
+  ne = 1,   // Z clear          Not equal.
+  cs = 2,   // C set            Carry set.
+  cc = 3,   // C clear          Carry clear.
+  mi = 4,   // N set            Negative.
+  pl = 5,   // N clear          Positive or zero.
+  vs = 6,   // V set            Overflow.
+  vc = 7,   // V clear          No overflow.
+  hi = 8,   // C set, Z clear   Unsigned higher.
+  ls = 9,   // C clear or Z set Unsigned lower or same.
+  ge = 10,  // N == V           Greater or equal.
+  lt = 11,  // N != V           Less than.
+  gt = 12,  // Z clear, N == V  Greater than.
+  le = 13,  // Z set or N != V  Less then or equal
+  al = 14,  //                  Always.
+  nv = 15,  // Behaves as always/al.
+
+  // Aliases.
+  hs = cs,  // C set            Unsigned higher or same.
+  lo = cc   // C clear          Unsigned lower.
+};
+
+inline Condition InvertCondition(Condition cond) {
+  // Conditions al and nv behave identically, as "always true". They can't be
+  // inverted, because there is no "always false" condition.
+  VIXL_ASSERT((cond != al) && (cond != nv));
+  return static_cast<Condition>(cond ^ 1);
+}
+
+enum FPTrapFlags {
+  EnableTrap   = 1,
+  DisableTrap = 0
+};
+
+enum FlagsUpdate {
+  SetFlags   = 1,
+  LeaveFlags = 0
+};
+
+enum StatusFlags {
+  NoFlag    = 0,
+
+  // Derive the flag combinations from the system register bit descriptions.
+  NFlag     = N_mask,
+  ZFlag     = Z_mask,
+  CFlag     = C_mask,
+  VFlag     = V_mask,
+  NZFlag    = NFlag | ZFlag,
+  NCFlag    = NFlag | CFlag,
+  NVFlag    = NFlag | VFlag,
+  ZCFlag    = ZFlag | CFlag,
+  ZVFlag    = ZFlag | VFlag,
+  CVFlag    = CFlag | VFlag,
+  NZCFlag   = NFlag | ZFlag | CFlag,
+  NZVFlag   = NFlag | ZFlag | VFlag,
+  NCVFlag   = NFlag | CFlag | VFlag,
+  ZCVFlag   = ZFlag | CFlag | VFlag,
+  NZCVFlag  = NFlag | ZFlag | CFlag | VFlag,
+
+  // Floating-point comparison results.
+  FPEqualFlag       = ZCFlag,
+  FPLessThanFlag    = NFlag,
+  FPGreaterThanFlag = CFlag,
+  FPUnorderedFlag   = CVFlag
+};
+
+enum Shift {
+  NO_SHIFT = -1,
+  LSL = 0x0,
+  LSR = 0x1,
+  ASR = 0x2,
+  ROR = 0x3,
+  MSL = 0x4
+};
+
+enum Extend {
+  NO_EXTEND = -1,
+  UXTB      = 0,
+  UXTH      = 1,
+  UXTW      = 2,
+  UXTX      = 3,
+  SXTB      = 4,
+  SXTH      = 5,
+  SXTW      = 6,
+  SXTX      = 7
+};
+
+enum SystemHint {
+  NOP   = 0,
+  YIELD = 1,
+  WFE   = 2,
+  WFI   = 3,
+  SEV   = 4,
+  SEVL  = 5
+};
+
+enum BarrierDomain {
+  OuterShareable = 0,
+  NonShareable   = 1,
+  InnerShareable = 2,
+  FullSystem     = 3
+};
+
+enum BarrierType {
+  BarrierOther  = 0,
+  BarrierReads  = 1,
+  BarrierWrites = 2,
+  BarrierAll    = 3
+};
+
+enum PrefetchOperation {
+  PLDL1KEEP = 0x00,
+  PLDL1STRM = 0x01,
+  PLDL2KEEP = 0x02,
+  PLDL2STRM = 0x03,
+  PLDL3KEEP = 0x04,
+  PLDL3STRM = 0x05,
+
+  PLIL1KEEP = 0x08,
+  PLIL1STRM = 0x09,
+  PLIL2KEEP = 0x0a,
+  PLIL2STRM = 0x0b,
+  PLIL3KEEP = 0x0c,
+  PLIL3STRM = 0x0d,
+
+  PSTL1KEEP = 0x10,
+  PSTL1STRM = 0x11,
+  PSTL2KEEP = 0x12,
+  PSTL2STRM = 0x13,
+  PSTL3KEEP = 0x14,
+  PSTL3STRM = 0x15
+};
+
+// System/special register names.
+// This information is not encoded as one field but as the concatenation of
+// multiple fields (Op0<0>, Op1, Crn, Crm, Op2).
+enum SystemRegister {
+  NZCV = ((0x1 << SysO0_offset) |
+          (0x3 << SysOp1_offset) |
+          (0x4 << CRn_offset) |
+          (0x2 << CRm_offset) |
+          (0x0 << SysOp2_offset)) >> ImmSystemRegister_offset,
+  FPCR = ((0x1 << SysO0_offset) |
+          (0x3 << SysOp1_offset) |
+          (0x4 << CRn_offset) |
+          (0x4 << CRm_offset) |
+          (0x0 << SysOp2_offset)) >> ImmSystemRegister_offset
+};
+
+enum InstructionCacheOp {
+  IVAU = ((0x3 << SysOp1_offset) |
+          (0x7 << CRn_offset) |
+          (0x5 << CRm_offset) |
+          (0x1 << SysOp2_offset)) >> SysOp_offset
+};
+
+enum DataCacheOp {
+  CVAC  = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0xa << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset,
+  CVAU  = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0xb << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset,
+  CIVAC = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0xe << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset,
+  ZVA   = ((0x3 << SysOp1_offset) |
+           (0x7 << CRn_offset) |
+           (0x4 << CRm_offset) |
+           (0x1 << SysOp2_offset)) >> SysOp_offset
+};
+
+// Instruction enumerations.
+//
+// These are the masks that define a class of instructions, and the list of
+// instructions within each class. Each enumeration has a Fixed, FMask and
+// Mask value.
+//
+// Fixed: The fixed bits in this instruction class.
+// FMask: The mask used to extract the fixed bits in the class.
+// Mask:  The mask used to identify the instructions within a class.
+//
+// The enumerations can be used like this:
+//
+// VIXL_ASSERT(instr->Mask(PCRelAddressingFMask) == PCRelAddressingFixed);
+// switch(instr->Mask(PCRelAddressingMask)) {
+//   case ADR:  Format("adr 'Xd, 'AddrPCRelByte"); break;
+//   case ADRP: Format("adrp 'Xd, 'AddrPCRelPage"); break;
+//   default:   printf("Unknown instruction\n");
+// }
+
+
+// Generic fields.
+enum GenericInstrField {
+  SixtyFourBits        = 0x80000000,
+  ThirtyTwoBits        = 0x00000000,
+  FP32                 = 0x00000000,
+  FP64                 = 0x00400000
+};
+
+enum NEONFormatField {
+  NEONFormatFieldMask   = 0x40C00000,
+  NEON_Q                = 0x40000000,
+  NEON_8B               = 0x00000000,
+  NEON_16B              = NEON_8B | NEON_Q,
+  NEON_4H               = 0x00400000,
+  NEON_8H               = NEON_4H | NEON_Q,
+  NEON_2S               = 0x00800000,
+  NEON_4S               = NEON_2S | NEON_Q,
+  NEON_1D               = 0x00C00000,
+  NEON_2D               = 0x00C00000 | NEON_Q
+};
+
+enum NEONFPFormatField {
+  NEONFPFormatFieldMask = 0x40400000,
+  NEON_FP_2S            = FP32,
+  NEON_FP_4S            = FP32 | NEON_Q,
+  NEON_FP_2D            = FP64 | NEON_Q
+};
+
+enum NEONLSFormatField {
+  NEONLSFormatFieldMask = 0x40000C00,
+  LS_NEON_8B            = 0x00000000,
+  LS_NEON_16B           = LS_NEON_8B | NEON_Q,
+  LS_NEON_4H            = 0x00000400,
+  LS_NEON_8H            = LS_NEON_4H | NEON_Q,
+  LS_NEON_2S            = 0x00000800,
+  LS_NEON_4S            = LS_NEON_2S | NEON_Q,
+  LS_NEON_1D            = 0x00000C00,
+  LS_NEON_2D            = LS_NEON_1D | NEON_Q
+};
+
+enum NEONScalarFormatField {
+  NEONScalarFormatFieldMask = 0x00C00000,
+  NEONScalar                = 0x10000000,
+  NEON_B                    = 0x00000000,
+  NEON_H                    = 0x00400000,
+  NEON_S                    = 0x00800000,
+  NEON_D                    = 0x00C00000
+};
+
+// PC relative addressing.
+enum PCRelAddressingOp {
+  PCRelAddressingFixed = 0x10000000,
+  PCRelAddressingFMask = 0x1F000000,
+  PCRelAddressingMask  = 0x9F000000,
+  ADR                  = PCRelAddressingFixed | 0x00000000,
+  ADRP                 = PCRelAddressingFixed | 0x80000000
+};
+
+// Add/sub (immediate, shifted and extended.)
+const int kSFOffset = 31;
+enum AddSubOp {
+  AddSubOpMask      = 0x60000000,
+  AddSubSetFlagsBit = 0x20000000,
+  ADD               = 0x00000000,
+  ADDS              = ADD | AddSubSetFlagsBit,
+  SUB               = 0x40000000,
+  SUBS              = SUB | AddSubSetFlagsBit
+};
+
+#define ADD_SUB_OP_LIST(V)  \
+  V(ADD),                   \
+  V(ADDS),                  \
+  V(SUB),                   \
+  V(SUBS)
+
+enum AddSubImmediateOp {
+  AddSubImmediateFixed = 0x11000000,
+  AddSubImmediateFMask = 0x1F000000,
+  AddSubImmediateMask  = 0xFF000000,
+  #define ADD_SUB_IMMEDIATE(A)           \
+  A##_w_imm = AddSubImmediateFixed | A,  \
+  A##_x_imm = AddSubImmediateFixed | A | SixtyFourBits
+  ADD_SUB_OP_LIST(ADD_SUB_IMMEDIATE)
+  #undef ADD_SUB_IMMEDIATE
+};
+
+enum AddSubShiftedOp {
+  AddSubShiftedFixed   = 0x0B000000,
+  AddSubShiftedFMask   = 0x1F200000,
+  AddSubShiftedMask    = 0xFF200000,
+  #define ADD_SUB_SHIFTED(A)             \
+  A##_w_shift = AddSubShiftedFixed | A,  \
+  A##_x_shift = AddSubShiftedFixed | A | SixtyFourBits
+  ADD_SUB_OP_LIST(ADD_SUB_SHIFTED)
+  #undef ADD_SUB_SHIFTED
+};
+
+enum AddSubExtendedOp {
+  AddSubExtendedFixed  = 0x0B200000,
+  AddSubExtendedFMask  = 0x1F200000,
+  AddSubExtendedMask   = 0xFFE00000,
+  #define ADD_SUB_EXTENDED(A)           \
+  A##_w_ext = AddSubExtendedFixed | A,  \
+  A##_x_ext = AddSubExtendedFixed | A | SixtyFourBits
+  ADD_SUB_OP_LIST(ADD_SUB_EXTENDED)
+  #undef ADD_SUB_EXTENDED
+};
+
+// Add/sub with carry.
+enum AddSubWithCarryOp {
+  AddSubWithCarryFixed = 0x1A000000,
+  AddSubWithCarryFMask = 0x1FE00000,
+  AddSubWithCarryMask  = 0xFFE0FC00,
+  ADC_w                = AddSubWithCarryFixed | ADD,
+  ADC_x                = AddSubWithCarryFixed | ADD | SixtyFourBits,
+  ADC                  = ADC_w,
+  ADCS_w               = AddSubWithCarryFixed | ADDS,
+  ADCS_x               = AddSubWithCarryFixed | ADDS | SixtyFourBits,
+  SBC_w                = AddSubWithCarryFixed | SUB,
+  SBC_x                = AddSubWithCarryFixed | SUB | SixtyFourBits,
+  SBC                  = SBC_w,
+  SBCS_w               = AddSubWithCarryFixed | SUBS,
+  SBCS_x               = AddSubWithCarryFixed | SUBS | SixtyFourBits
+};
+
+
+// Logical (immediate and shifted register).
+enum LogicalOp {
+  LogicalOpMask = 0x60200000,
+  NOT   = 0x00200000,
+  AND   = 0x00000000,
+  BIC   = AND | NOT,
+  ORR   = 0x20000000,
+  ORN   = ORR | NOT,
+  EOR   = 0x40000000,
+  EON   = EOR | NOT,
+  ANDS  = 0x60000000,
+  BICS  = ANDS | NOT
+};
+
+// Logical immediate.
+enum LogicalImmediateOp {
+  LogicalImmediateFixed = 0x12000000,
+  LogicalImmediateFMask = 0x1F800000,
+  LogicalImmediateMask  = 0xFF800000,
+  AND_w_imm   = LogicalImmediateFixed | AND,
+  AND_x_imm   = LogicalImmediateFixed | AND | SixtyFourBits,
+  ORR_w_imm   = LogicalImmediateFixed | ORR,
+  ORR_x_imm   = LogicalImmediateFixed | ORR | SixtyFourBits,
+  EOR_w_imm   = LogicalImmediateFixed | EOR,
+  EOR_x_imm   = LogicalImmediateFixed | EOR | SixtyFourBits,
+  ANDS_w_imm  = LogicalImmediateFixed | ANDS,
+  ANDS_x_imm  = LogicalImmediateFixed | ANDS | SixtyFourBits
+};
+
+// Logical shifted register.
+enum LogicalShiftedOp {
+  LogicalShiftedFixed = 0x0A000000,
+  LogicalShiftedFMask = 0x1F000000,
+  LogicalShiftedMask  = 0xFF200000,
+  AND_w               = LogicalShiftedFixed | AND,
+  AND_x               = LogicalShiftedFixed | AND | SixtyFourBits,
+  AND_shift           = AND_w,
+  BIC_w               = LogicalShiftedFixed | BIC,
+  BIC_x               = LogicalShiftedFixed | BIC | SixtyFourBits,
+  BIC_shift           = BIC_w,
+  ORR_w               = LogicalShiftedFixed | ORR,
+  ORR_x               = LogicalShiftedFixed | ORR | SixtyFourBits,
+  ORR_shift           = ORR_w,
+  ORN_w               = LogicalShiftedFixed | ORN,
+  ORN_x               = LogicalShiftedFixed | ORN | SixtyFourBits,
+  ORN_shift           = ORN_w,
+  EOR_w               = LogicalShiftedFixed | EOR,
+  EOR_x               = LogicalShiftedFixed | EOR | SixtyFourBits,
+  EOR_shift           = EOR_w,
+  EON_w               = LogicalShiftedFixed | EON,
+  EON_x               = LogicalShiftedFixed | EON | SixtyFourBits,
+  EON_shift           = EON_w,
+  ANDS_w              = LogicalShiftedFixed | ANDS,
+  ANDS_x              = LogicalShiftedFixed | ANDS | SixtyFourBits,
+  ANDS_shift          = ANDS_w,
+  BICS_w              = LogicalShiftedFixed | BICS,
+  BICS_x              = LogicalShiftedFixed | BICS | SixtyFourBits,
+  BICS_shift          = BICS_w
+};
+
+// Move wide immediate.
+enum MoveWideImmediateOp {
+  MoveWideImmediateFixed = 0x12800000,
+  MoveWideImmediateFMask = 0x1F800000,
+  MoveWideImmediateMask  = 0xFF800000,
+  MOVN                   = 0x00000000,
+  MOVZ                   = 0x40000000,
+  MOVK                   = 0x60000000,
+  MOVN_w                 = MoveWideImmediateFixed | MOVN,
+  MOVN_x                 = MoveWideImmediateFixed | MOVN | SixtyFourBits,
+  MOVZ_w                 = MoveWideImmediateFixed | MOVZ,
+  MOVZ_x                 = MoveWideImmediateFixed | MOVZ | SixtyFourBits,
+  MOVK_w                 = MoveWideImmediateFixed | MOVK,
+  MOVK_x                 = MoveWideImmediateFixed | MOVK | SixtyFourBits
+};
+
+// Bitfield.
+const int kBitfieldNOffset = 22;
+enum BitfieldOp {
+  BitfieldFixed = 0x13000000,
+  BitfieldFMask = 0x1F800000,
+  BitfieldMask  = 0xFF800000,
+  SBFM_w        = BitfieldFixed | 0x00000000,
+  SBFM_x        = BitfieldFixed | 0x80000000,
+  SBFM          = SBFM_w,
+  BFM_w         = BitfieldFixed | 0x20000000,
+  BFM_x         = BitfieldFixed | 0xA0000000,
+  BFM           = BFM_w,
+  UBFM_w        = BitfieldFixed | 0x40000000,
+  UBFM_x        = BitfieldFixed | 0xC0000000,
+  UBFM          = UBFM_w
+  // Bitfield N field.
+};
+
+// Extract.
+enum ExtractOp {
+  ExtractFixed = 0x13800000,
+  ExtractFMask = 0x1F800000,
+  ExtractMask  = 0xFFA00000,
+  EXTR_w       = ExtractFixed | 0x00000000,
+  EXTR_x       = ExtractFixed | 0x80000000,
+  EXTR         = EXTR_w
+};
+
+// Unconditional branch.
+enum UnconditionalBranchOp {
+  UnconditionalBranchFixed = 0x14000000,
+  UnconditionalBranchFMask = 0x7C000000,
+  UnconditionalBranchMask  = 0xFC000000,
+  B                        = UnconditionalBranchFixed | 0x00000000,
+  BL                       = UnconditionalBranchFixed | 0x80000000
+};
+
+// Unconditional branch to register.
+enum UnconditionalBranchToRegisterOp {
+  UnconditionalBranchToRegisterFixed = 0xD6000000,
+  UnconditionalBranchToRegisterFMask = 0xFE000000,
+  UnconditionalBranchToRegisterMask  = 0xFFFFFC1F,
+  BR      = UnconditionalBranchToRegisterFixed | 0x001F0000,
+  BLR     = UnconditionalBranchToRegisterFixed | 0x003F0000,
+  RET     = UnconditionalBranchToRegisterFixed | 0x005F0000
+};
+
+// Compare and branch.
+enum CompareBranchOp {
+  CompareBranchFixed = 0x34000000,
+  CompareBranchFMask = 0x7E000000,
+  CompareBranchMask  = 0xFF000000,
+  CBZ_w              = CompareBranchFixed | 0x00000000,
+  CBZ_x              = CompareBranchFixed | 0x80000000,
+  CBZ                = CBZ_w,
+  CBNZ_w             = CompareBranchFixed | 0x01000000,
+  CBNZ_x             = CompareBranchFixed | 0x81000000,
+  CBNZ               = CBNZ_w
+};
+
+// Test and branch.
+enum TestBranchOp {
+  TestBranchFixed = 0x36000000,
+  TestBranchFMask = 0x7E000000,
+  TestBranchMask  = 0x7F000000,
+  TBZ             = TestBranchFixed | 0x00000000,
+  TBNZ            = TestBranchFixed | 0x01000000
+};
+
+// Conditional branch.
+enum ConditionalBranchOp {
+  ConditionalBranchFixed = 0x54000000,
+  ConditionalBranchFMask = 0xFE000000,
+  ConditionalBranchMask  = 0xFF000010,
+  B_cond                 = ConditionalBranchFixed | 0x00000000
+};
+
+// System.
+// System instruction encoding is complicated because some instructions use op
+// and CR fields to encode parameters. To handle this cleanly, the system
+// instructions are split into more than one enum.
+
+enum SystemOp {
+  SystemFixed = 0xD5000000,
+  SystemFMask = 0xFFC00000
+};
+
+enum SystemSysRegOp {
+  SystemSysRegFixed = 0xD5100000,
+  SystemSysRegFMask = 0xFFD00000,
+  SystemSysRegMask  = 0xFFF00000,
+  MRS               = SystemSysRegFixed | 0x00200000,
+  MSR               = SystemSysRegFixed | 0x00000000
+};
+
+enum SystemHintOp {
+  SystemHintFixed = 0xD503201F,
+  SystemHintFMask = 0xFFFFF01F,
+  SystemHintMask  = 0xFFFFF01F,
+  HINT            = SystemHintFixed | 0x00000000
+};
+
+enum SystemSysOp {
+  SystemSysFixed  = 0xD5080000,
+  SystemSysFMask  = 0xFFF80000,
+  SystemSysMask   = 0xFFF80000,
+  SYS             = SystemSysFixed | 0x00000000
+};
+
+// Exception.
+enum ExceptionOp {
+  ExceptionFixed = 0xD4000000,
+  ExceptionFMask = 0xFF000000,
+  ExceptionMask  = 0xFFE0001F,
+  HLT            = ExceptionFixed | 0x00400000,
+  BRK            = ExceptionFixed | 0x00200000,
+  SVC            = ExceptionFixed | 0x00000001,
+  HVC            = ExceptionFixed | 0x00000002,
+  SMC            = ExceptionFixed | 0x00000003,
+  DCPS1          = ExceptionFixed | 0x00A00001,
+  DCPS2          = ExceptionFixed | 0x00A00002,
+  DCPS3          = ExceptionFixed | 0x00A00003
+};
+
+enum MemBarrierOp {
+  MemBarrierFixed = 0xD503309F,
+  MemBarrierFMask = 0xFFFFF09F,
+  MemBarrierMask  = 0xFFFFF0FF,
+  DSB             = MemBarrierFixed | 0x00000000,
+  DMB             = MemBarrierFixed | 0x00000020,
+  ISB             = MemBarrierFixed | 0x00000040
+};
+
+enum SystemExclusiveMonitorOp {
+  SystemExclusiveMonitorFixed = 0xD503305F,
+  SystemExclusiveMonitorFMask = 0xFFFFF0FF,
+  SystemExclusiveMonitorMask  = 0xFFFFF0FF,
+  CLREX                       = SystemExclusiveMonitorFixed
+};
+
+// Any load or store.
+enum LoadStoreAnyOp {
+  LoadStoreAnyFMask = 0x0a000000,
+  LoadStoreAnyFixed = 0x08000000
+};
+
+// Any load pair or store pair.
+enum LoadStorePairAnyOp {
+  LoadStorePairAnyFMask = 0x3a000000,
+  LoadStorePairAnyFixed = 0x28000000
+};
+
+#define LOAD_STORE_PAIR_OP_LIST(V)  \
+  V(STP, w,   0x00000000),          \
+  V(LDP, w,   0x00400000),          \
+  V(LDPSW, x, 0x40400000),          \
+  V(STP, x,   0x80000000),          \
+  V(LDP, x,   0x80400000),          \
+  V(STP, s,   0x04000000),          \
+  V(LDP, s,   0x04400000),          \
+  V(STP, d,   0x44000000),          \
+  V(LDP, d,   0x44400000),          \
+  V(STP, q,   0x84000000),          \
+  V(LDP, q,   0x84400000)
+
+// Load/store pair (post, pre and offset.)
+enum LoadStorePairOp {
+  LoadStorePairMask = 0xC4400000,
+  LoadStorePairLBit = 1 << 22,
+  #define LOAD_STORE_PAIR(A, B, C) \
+  A##_##B = C
+  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR)
+  #undef LOAD_STORE_PAIR
+};
+
+enum LoadStorePairPostIndexOp {
+  LoadStorePairPostIndexFixed = 0x28800000,
+  LoadStorePairPostIndexFMask = 0x3B800000,
+  LoadStorePairPostIndexMask  = 0xFFC00000,
+  #define LOAD_STORE_PAIR_POST_INDEX(A, B, C)  \
+  A##_##B##_post = LoadStorePairPostIndexFixed | A##_##B
+  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_POST_INDEX)
+  #undef LOAD_STORE_PAIR_POST_INDEX
+};
+
+enum LoadStorePairPreIndexOp {
+  LoadStorePairPreIndexFixed = 0x29800000,
+  LoadStorePairPreIndexFMask = 0x3B800000,
+  LoadStorePairPreIndexMask  = 0xFFC00000,
+  #define LOAD_STORE_PAIR_PRE_INDEX(A, B, C)  \
+  A##_##B##_pre = LoadStorePairPreIndexFixed | A##_##B
+  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_PRE_INDEX)
+  #undef LOAD_STORE_PAIR_PRE_INDEX
+};
+
+enum LoadStorePairOffsetOp {
+  LoadStorePairOffsetFixed = 0x29000000,
+  LoadStorePairOffsetFMask = 0x3B800000,
+  LoadStorePairOffsetMask  = 0xFFC00000,
+  #define LOAD_STORE_PAIR_OFFSET(A, B, C)  \
+  A##_##B##_off = LoadStorePairOffsetFixed | A##_##B
+  LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_OFFSET)
+  #undef LOAD_STORE_PAIR_OFFSET
+};
+
+enum LoadStorePairNonTemporalOp {
+  LoadStorePairNonTemporalFixed = 0x28000000,
+  LoadStorePairNonTemporalFMask = 0x3B800000,
+  LoadStorePairNonTemporalMask  = 0xFFC00000,
+  LoadStorePairNonTemporalLBit = 1 << 22,
+  STNP_w = LoadStorePairNonTemporalFixed | STP_w,
+  LDNP_w = LoadStorePairNonTemporalFixed | LDP_w,
+  STNP_x = LoadStorePairNonTemporalFixed | STP_x,
+  LDNP_x = LoadStorePairNonTemporalFixed | LDP_x,
+  STNP_s = LoadStorePairNonTemporalFixed | STP_s,
+  LDNP_s = LoadStorePairNonTemporalFixed | LDP_s,
+  STNP_d = LoadStorePairNonTemporalFixed | STP_d,
+  LDNP_d = LoadStorePairNonTemporalFixed | LDP_d,
+  STNP_q = LoadStorePairNonTemporalFixed | STP_q,
+  LDNP_q = LoadStorePairNonTemporalFixed | LDP_q
+};
+
+// Load literal.
+enum LoadLiteralOp {
+  LoadLiteralFixed = 0x18000000,
+  LoadLiteralFMask = 0x3B000000,
+  LoadLiteralMask  = 0xFF000000,
+  LDR_w_lit        = LoadLiteralFixed | 0x00000000,
+  LDR_x_lit        = LoadLiteralFixed | 0x40000000,
+  LDRSW_x_lit      = LoadLiteralFixed | 0x80000000,
+  PRFM_lit         = LoadLiteralFixed | 0xC0000000,
+  LDR_s_lit        = LoadLiteralFixed | 0x04000000,
+  LDR_d_lit        = LoadLiteralFixed | 0x44000000,
+  LDR_q_lit        = LoadLiteralFixed | 0x84000000
+};
+
+#define LOAD_STORE_OP_LIST(V)     \
+  V(ST, RB, w,  0x00000000),  \
+  V(ST, RH, w,  0x40000000),  \
+  V(ST, R, w,   0x80000000),  \
+  V(ST, R, x,   0xC0000000),  \
+  V(LD, RB, w,  0x00400000),  \
+  V(LD, RH, w,  0x40400000),  \
+  V(LD, R, w,   0x80400000),  \
+  V(LD, R, x,   0xC0400000),  \
+  V(LD, RSB, x, 0x00800000),  \
+  V(LD, RSH, x, 0x40800000),  \
+  V(LD, RSW, x, 0x80800000),  \
+  V(LD, RSB, w, 0x00C00000),  \
+  V(LD, RSH, w, 0x40C00000),  \
+  V(ST, R, b,   0x04000000),  \
+  V(ST, R, h,   0x44000000),  \
+  V(ST, R, s,   0x84000000),  \
+  V(ST, R, d,   0xC4000000),  \
+  V(ST, R, q,   0x04800000),  \
+  V(LD, R, b,   0x04400000),  \
+  V(LD, R, h,   0x44400000),  \
+  V(LD, R, s,   0x84400000),  \
+  V(LD, R, d,   0xC4400000),  \
+  V(LD, R, q,   0x04C00000)
+
+// Load/store (post, pre, offset and unsigned.)
+enum LoadStoreOp {
+  LoadStoreMask = 0xC4C00000,
+  LoadStoreVMask = 0x04000000,
+  #define LOAD_STORE(A, B, C, D)  \
+  A##B##_##C = D
+  LOAD_STORE_OP_LIST(LOAD_STORE),
+  #undef LOAD_STORE
+  PRFM = 0xC0800000
+};
+
+// Load/store unscaled offset.
+enum LoadStoreUnscaledOffsetOp {
+  LoadStoreUnscaledOffsetFixed = 0x38000000,
+  LoadStoreUnscaledOffsetFMask = 0x3B200C00,
+  LoadStoreUnscaledOffsetMask  = 0xFFE00C00,
+  PRFUM                        = LoadStoreUnscaledOffsetFixed | PRFM,
+  #define LOAD_STORE_UNSCALED(A, B, C, D)  \
+  A##U##B##_##C = LoadStoreUnscaledOffsetFixed | D
+  LOAD_STORE_OP_LIST(LOAD_STORE_UNSCALED)
+  #undef LOAD_STORE_UNSCALED
+};
+
+// Load/store post index.
+enum LoadStorePostIndex {
+  LoadStorePostIndexFixed = 0x38000400,
+  LoadStorePostIndexFMask = 0x3B200C00,
+  LoadStorePostIndexMask  = 0xFFE00C00,
+  #define LOAD_STORE_POST_INDEX(A, B, C, D)  \
+  A##B##_##C##_post = LoadStorePostIndexFixed | D
+  LOAD_STORE_OP_LIST(LOAD_STORE_POST_INDEX)
+  #undef LOAD_STORE_POST_INDEX
+};
+
+// Load/store pre index.
+enum LoadStorePreIndex {
+  LoadStorePreIndexFixed = 0x38000C00,
+  LoadStorePreIndexFMask = 0x3B200C00,
+  LoadStorePreIndexMask  = 0xFFE00C00,
+  #define LOAD_STORE_PRE_INDEX(A, B, C, D)  \
+  A##B##_##C##_pre = LoadStorePreIndexFixed | D
+  LOAD_STORE_OP_LIST(LOAD_STORE_PRE_INDEX)
+  #undef LOAD_STORE_PRE_INDEX
+};
+
+// Load/store unsigned offset.
+enum LoadStoreUnsignedOffset {
+  LoadStoreUnsignedOffsetFixed = 0x39000000,
+  LoadStoreUnsignedOffsetFMask = 0x3B000000,
+  LoadStoreUnsignedOffsetMask  = 0xFFC00000,
+  PRFM_unsigned                = LoadStoreUnsignedOffsetFixed | PRFM,
+  #define LOAD_STORE_UNSIGNED_OFFSET(A, B, C, D) \
+  A##B##_##C##_unsigned = LoadStoreUnsignedOffsetFixed | D
+  LOAD_STORE_OP_LIST(LOAD_STORE_UNSIGNED_OFFSET)
+  #undef LOAD_STORE_UNSIGNED_OFFSET
+};
+
+// Load/store register offset.
+enum LoadStoreRegisterOffset {
+  LoadStoreRegisterOffsetFixed = 0x38200800,
+  LoadStoreRegisterOffsetFMask = 0x3B200C00,
+  LoadStoreRegisterOffsetMask  = 0xFFE00C00,
+  PRFM_reg                     = LoadStoreRegisterOffsetFixed | PRFM,
+  #define LOAD_STORE_REGISTER_OFFSET(A, B, C, D) \
+  A##B##_##C##_reg = LoadStoreRegisterOffsetFixed | D
+  LOAD_STORE_OP_LIST(LOAD_STORE_REGISTER_OFFSET)
+  #undef LOAD_STORE_REGISTER_OFFSET
+};
+
+enum LoadStoreExclusive {
+  LoadStoreExclusiveFixed = 0x08000000,
+  LoadStoreExclusiveFMask = 0x3F000000,
+  LoadStoreExclusiveMask  = 0xFFE08000,
+  STXRB_w  = LoadStoreExclusiveFixed | 0x00000000,
+  STXRH_w  = LoadStoreExclusiveFixed | 0x40000000,
+  STXR_w   = LoadStoreExclusiveFixed | 0x80000000,
+  STXR_x   = LoadStoreExclusiveFixed | 0xC0000000,
+  LDXRB_w  = LoadStoreExclusiveFixed | 0x00400000,
+  LDXRH_w  = LoadStoreExclusiveFixed | 0x40400000,
+  LDXR_w   = LoadStoreExclusiveFixed | 0x80400000,
+  LDXR_x   = LoadStoreExclusiveFixed | 0xC0400000,
+  STXP_w   = LoadStoreExclusiveFixed | 0x80200000,
+  STXP_x   = LoadStoreExclusiveFixed | 0xC0200000,
+  LDXP_w   = LoadStoreExclusiveFixed | 0x80600000,
+  LDXP_x   = LoadStoreExclusiveFixed | 0xC0600000,
+  STLXRB_w = LoadStoreExclusiveFixed | 0x00008000,
+  STLXRH_w = LoadStoreExclusiveFixed | 0x40008000,
+  STLXR_w  = LoadStoreExclusiveFixed | 0x80008000,
+  STLXR_x  = LoadStoreExclusiveFixed | 0xC0008000,
+  LDAXRB_w = LoadStoreExclusiveFixed | 0x00408000,
+  LDAXRH_w = LoadStoreExclusiveFixed | 0x40408000,
+  LDAXR_w  = LoadStoreExclusiveFixed | 0x80408000,
+  LDAXR_x  = LoadStoreExclusiveFixed | 0xC0408000,
+  STLXP_w  = LoadStoreExclusiveFixed | 0x80208000,
+  STLXP_x  = LoadStoreExclusiveFixed | 0xC0208000,
+  LDAXP_w  = LoadStoreExclusiveFixed | 0x80608000,
+  LDAXP_x  = LoadStoreExclusiveFixed | 0xC0608000,
+  STLRB_w  = LoadStoreExclusiveFixed | 0x00808000,
+  STLRH_w  = LoadStoreExclusiveFixed | 0x40808000,
+  STLR_w   = LoadStoreExclusiveFixed | 0x80808000,
+  STLR_x   = LoadStoreExclusiveFixed | 0xC0808000,
+  LDARB_w  = LoadStoreExclusiveFixed | 0x00C08000,
+  LDARH_w  = LoadStoreExclusiveFixed | 0x40C08000,
+  LDAR_w   = LoadStoreExclusiveFixed | 0x80C08000,
+  LDAR_x   = LoadStoreExclusiveFixed | 0xC0C08000
+};
+
+// Conditional compare.
+enum ConditionalCompareOp {
+  ConditionalCompareMask = 0x60000000,
+  CCMN                   = 0x20000000,
+  CCMP                   = 0x60000000
+};
+
+// Conditional compare register.
+enum ConditionalCompareRegisterOp {
+  ConditionalCompareRegisterFixed = 0x1A400000,
+  ConditionalCompareRegisterFMask = 0x1FE00800,
+  ConditionalCompareRegisterMask  = 0xFFE00C10,
+  CCMN_w = ConditionalCompareRegisterFixed | CCMN,
+  CCMN_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMN,
+  CCMP_w = ConditionalCompareRegisterFixed | CCMP,
+  CCMP_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMP
+};
+
+// Conditional compare immediate.
+enum ConditionalCompareImmediateOp {
+  ConditionalCompareImmediateFixed = 0x1A400800,
+  ConditionalCompareImmediateFMask = 0x1FE00800,
+  ConditionalCompareImmediateMask  = 0xFFE00C10,
+  CCMN_w_imm = ConditionalCompareImmediateFixed | CCMN,
+  CCMN_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMN,
+  CCMP_w_imm = ConditionalCompareImmediateFixed | CCMP,
+  CCMP_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMP
+};
+
+// Conditional select.
+enum ConditionalSelectOp {
+  ConditionalSelectFixed = 0x1A800000,
+  ConditionalSelectFMask = 0x1FE00000,
+  ConditionalSelectMask  = 0xFFE00C00,
+  CSEL_w                 = ConditionalSelectFixed | 0x00000000,
+  CSEL_x                 = ConditionalSelectFixed | 0x80000000,
+  CSEL                   = CSEL_w,
+  CSINC_w                = ConditionalSelectFixed | 0x00000400,
+  CSINC_x                = ConditionalSelectFixed | 0x80000400,
+  CSINC                  = CSINC_w,
+  CSINV_w                = ConditionalSelectFixed | 0x40000000,
+  CSINV_x                = ConditionalSelectFixed | 0xC0000000,
+  CSINV                  = CSINV_w,
+  CSNEG_w                = ConditionalSelectFixed | 0x40000400,
+  CSNEG_x                = ConditionalSelectFixed | 0xC0000400,
+  CSNEG                  = CSNEG_w
+};
+
+// Data processing 1 source.
+enum DataProcessing1SourceOp {
+  DataProcessing1SourceFixed = 0x5AC00000,
+  DataProcessing1SourceFMask = 0x5FE00000,
+  DataProcessing1SourceMask  = 0xFFFFFC00,
+  RBIT    = DataProcessing1SourceFixed | 0x00000000,
+  RBIT_w  = RBIT,
+  RBIT_x  = RBIT | SixtyFourBits,
+  REV16   = DataProcessing1SourceFixed | 0x00000400,
+  REV16_w = REV16,
+  REV16_x = REV16 | SixtyFourBits,
+  REV     = DataProcessing1SourceFixed | 0x00000800,
+  REV_w   = REV,
+  REV32_x = REV | SixtyFourBits,
+  REV_x   = DataProcessing1SourceFixed | SixtyFourBits | 0x00000C00,
+  CLZ     = DataProcessing1SourceFixed | 0x00001000,
+  CLZ_w   = CLZ,
+  CLZ_x   = CLZ | SixtyFourBits,
+  CLS     = DataProcessing1SourceFixed | 0x00001400,
+  CLS_w   = CLS,
+  CLS_x   = CLS | SixtyFourBits
+};
+
+// Data processing 2 source.
+enum DataProcessing2SourceOp {
+  DataProcessing2SourceFixed = 0x1AC00000,
+  DataProcessing2SourceFMask = 0x5FE00000,
+  DataProcessing2SourceMask  = 0xFFE0FC00,
+  UDIV_w  = DataProcessing2SourceFixed | 0x00000800,
+  UDIV_x  = DataProcessing2SourceFixed | 0x80000800,
+  UDIV    = UDIV_w,
+  SDIV_w  = DataProcessing2SourceFixed | 0x00000C00,
+  SDIV_x  = DataProcessing2SourceFixed | 0x80000C00,
+  SDIV    = SDIV_w,
+  LSLV_w  = DataProcessing2SourceFixed | 0x00002000,
+  LSLV_x  = DataProcessing2SourceFixed | 0x80002000,
+  LSLV    = LSLV_w,
+  LSRV_w  = DataProcessing2SourceFixed | 0x00002400,
+  LSRV_x  = DataProcessing2SourceFixed | 0x80002400,
+  LSRV    = LSRV_w,
+  ASRV_w  = DataProcessing2SourceFixed | 0x00002800,
+  ASRV_x  = DataProcessing2SourceFixed | 0x80002800,
+  ASRV    = ASRV_w,
+  RORV_w  = DataProcessing2SourceFixed | 0x00002C00,
+  RORV_x  = DataProcessing2SourceFixed | 0x80002C00,
+  RORV    = RORV_w,
+  CRC32B  = DataProcessing2SourceFixed | 0x00004000,
+  CRC32H  = DataProcessing2SourceFixed | 0x00004400,
+  CRC32W  = DataProcessing2SourceFixed | 0x00004800,
+  CRC32X  = DataProcessing2SourceFixed | SixtyFourBits | 0x00004C00,
+  CRC32CB = DataProcessing2SourceFixed | 0x00005000,
+  CRC32CH = DataProcessing2SourceFixed | 0x00005400,
+  CRC32CW = DataProcessing2SourceFixed | 0x00005800,
+  CRC32CX = DataProcessing2SourceFixed | SixtyFourBits | 0x00005C00
+};
+
+// Data processing 3 source.
+enum DataProcessing3SourceOp {
+  DataProcessing3SourceFixed = 0x1B000000,
+  DataProcessing3SourceFMask = 0x1F000000,
+  DataProcessing3SourceMask  = 0xFFE08000,
+  MADD_w                     = DataProcessing3SourceFixed | 0x00000000,
+  MADD_x                     = DataProcessing3SourceFixed | 0x80000000,
+  MADD                       = MADD_w,
+  MSUB_w                     = DataProcessing3SourceFixed | 0x00008000,
+  MSUB_x                     = DataProcessing3SourceFixed | 0x80008000,
+  MSUB                       = MSUB_w,
+  SMADDL_x                   = DataProcessing3SourceFixed | 0x80200000,
+  SMSUBL_x                   = DataProcessing3SourceFixed | 0x80208000,
+  SMULH_x                    = DataProcessing3SourceFixed | 0x80400000,
+  UMADDL_x                   = DataProcessing3SourceFixed | 0x80A00000,
+  UMSUBL_x                   = DataProcessing3SourceFixed | 0x80A08000,
+  UMULH_x                    = DataProcessing3SourceFixed | 0x80C00000
+};
+
+// Floating point compare.
+enum FPCompareOp {
+  FPCompareFixed = 0x1E202000,
+  FPCompareFMask = 0x5F203C00,
+  FPCompareMask  = 0xFFE0FC1F,
+  FCMP_s         = FPCompareFixed | 0x00000000,
+  FCMP_d         = FPCompareFixed | FP64 | 0x00000000,
+  FCMP           = FCMP_s,
+  FCMP_s_zero    = FPCompareFixed | 0x00000008,
+  FCMP_d_zero    = FPCompareFixed | FP64 | 0x00000008,
+  FCMP_zero      = FCMP_s_zero,
+  FCMPE_s        = FPCompareFixed | 0x00000010,
+  FCMPE_d        = FPCompareFixed | FP64 | 0x00000010,
+  FCMPE          = FCMPE_s,
+  FCMPE_s_zero   = FPCompareFixed | 0x00000018,
+  FCMPE_d_zero   = FPCompareFixed | FP64 | 0x00000018,
+  FCMPE_zero     = FCMPE_s_zero
+};
+
+// Floating point conditional compare.
+enum FPConditionalCompareOp {
+  FPConditionalCompareFixed = 0x1E200400,
+  FPConditionalCompareFMask = 0x5F200C00,
+  FPConditionalCompareMask  = 0xFFE00C10,
+  FCCMP_s                   = FPConditionalCompareFixed | 0x00000000,
+  FCCMP_d                   = FPConditionalCompareFixed | FP64 | 0x00000000,
+  FCCMP                     = FCCMP_s,
+  FCCMPE_s                  = FPConditionalCompareFixed | 0x00000010,
+  FCCMPE_d                  = FPConditionalCompareFixed | FP64 | 0x00000010,
+  FCCMPE                    = FCCMPE_s
+};
+
+// Floating point conditional select.
+enum FPConditionalSelectOp {
+  FPConditionalSelectFixed = 0x1E200C00,
+  FPConditionalSelectFMask = 0x5F200C00,
+  FPConditionalSelectMask  = 0xFFE00C00,
+  FCSEL_s                  = FPConditionalSelectFixed | 0x00000000,
+  FCSEL_d                  = FPConditionalSelectFixed | FP64 | 0x00000000,
+  FCSEL                    = FCSEL_s
+};
+
+// Floating point immediate.
+enum FPImmediateOp {
+  FPImmediateFixed = 0x1E201000,
+  FPImmediateFMask = 0x5F201C00,
+  FPImmediateMask  = 0xFFE01C00,
+  FMOV_s_imm       = FPImmediateFixed | 0x00000000,
+  FMOV_d_imm       = FPImmediateFixed | FP64 | 0x00000000
+};
+
+// Floating point data processing 1 source.
+enum FPDataProcessing1SourceOp {
+  FPDataProcessing1SourceFixed = 0x1E204000,
+  FPDataProcessing1SourceFMask = 0x5F207C00,
+  FPDataProcessing1SourceMask  = 0xFFFFFC00,
+  FMOV_s   = FPDataProcessing1SourceFixed | 0x00000000,
+  FMOV_d   = FPDataProcessing1SourceFixed | FP64 | 0x00000000,
+  FMOV     = FMOV_s,
+  FABS_s   = FPDataProcessing1SourceFixed | 0x00008000,
+  FABS_d   = FPDataProcessing1SourceFixed | FP64 | 0x00008000,
+  FABS     = FABS_s,
+  FNEG_s   = FPDataProcessing1SourceFixed | 0x00010000,
+  FNEG_d   = FPDataProcessing1SourceFixed | FP64 | 0x00010000,
+  FNEG     = FNEG_s,
+  FSQRT_s  = FPDataProcessing1SourceFixed | 0x00018000,
+  FSQRT_d  = FPDataProcessing1SourceFixed | FP64 | 0x00018000,
+  FSQRT    = FSQRT_s,
+  FCVT_ds  = FPDataProcessing1SourceFixed | 0x00028000,
+  FCVT_sd  = FPDataProcessing1SourceFixed | FP64 | 0x00020000,
+  FCVT_hs  = FPDataProcessing1SourceFixed | 0x00038000,
+  FCVT_hd  = FPDataProcessing1SourceFixed | FP64 | 0x00038000,
+  FCVT_sh  = FPDataProcessing1SourceFixed | 0x00C20000,
+  FCVT_dh  = FPDataProcessing1SourceFixed | 0x00C28000,
+  FRINTN_s = FPDataProcessing1SourceFixed | 0x00040000,
+  FRINTN_d = FPDataProcessing1SourceFixed | FP64 | 0x00040000,
+  FRINTN   = FRINTN_s,
+  FRINTP_s = FPDataProcessing1SourceFixed | 0x00048000,
+  FRINTP_d = FPDataProcessing1SourceFixed | FP64 | 0x00048000,
+  FRINTP   = FRINTP_s,
+  FRINTM_s = FPDataProcessing1SourceFixed | 0x00050000,
+  FRINTM_d = FPDataProcessing1SourceFixed | FP64 | 0x00050000,
+  FRINTM   = FRINTM_s,
+  FRINTZ_s = FPDataProcessing1SourceFixed | 0x00058000,
+  FRINTZ_d = FPDataProcessing1SourceFixed | FP64 | 0x00058000,
+  FRINTZ   = FRINTZ_s,
+  FRINTA_s = FPDataProcessing1SourceFixed | 0x00060000,
+  FRINTA_d = FPDataProcessing1SourceFixed | FP64 | 0x00060000,
+  FRINTA   = FRINTA_s,
+  FRINTX_s = FPDataProcessing1SourceFixed | 0x00070000,
+  FRINTX_d = FPDataProcessing1SourceFixed | FP64 | 0x00070000,
+  FRINTX   = FRINTX_s,
+  FRINTI_s = FPDataProcessing1SourceFixed | 0x00078000,
+  FRINTI_d = FPDataProcessing1SourceFixed | FP64 | 0x00078000,
+  FRINTI   = FRINTI_s
+};
+
+// Floating point data processing 2 source.
+enum FPDataProcessing2SourceOp {
+  FPDataProcessing2SourceFixed = 0x1E200800,
+  FPDataProcessing2SourceFMask = 0x5F200C00,
+  FPDataProcessing2SourceMask  = 0xFFE0FC00,
+  FMUL     = FPDataProcessing2SourceFixed | 0x00000000,
+  FMUL_s   = FMUL,
+  FMUL_d   = FMUL | FP64,
+  FDIV     = FPDataProcessing2SourceFixed | 0x00001000,
+  FDIV_s   = FDIV,
+  FDIV_d   = FDIV | FP64,
+  FADD     = FPDataProcessing2SourceFixed | 0x00002000,
+  FADD_s   = FADD,
+  FADD_d   = FADD | FP64,
+  FSUB     = FPDataProcessing2SourceFixed | 0x00003000,
+  FSUB_s   = FSUB,
+  FSUB_d   = FSUB | FP64,
+  FMAX     = FPDataProcessing2SourceFixed | 0x00004000,
+  FMAX_s   = FMAX,
+  FMAX_d   = FMAX | FP64,
+  FMIN     = FPDataProcessing2SourceFixed | 0x00005000,
+  FMIN_s   = FMIN,
+  FMIN_d   = FMIN | FP64,
+  FMAXNM   = FPDataProcessing2SourceFixed | 0x00006000,
+  FMAXNM_s = FMAXNM,
+  FMAXNM_d = FMAXNM | FP64,
+  FMINNM   = FPDataProcessing2SourceFixed | 0x00007000,
+  FMINNM_s = FMINNM,
+  FMINNM_d = FMINNM | FP64,
+  FNMUL    = FPDataProcessing2SourceFixed | 0x00008000,
+  FNMUL_s  = FNMUL,
+  FNMUL_d  = FNMUL | FP64
+};
+
+// Floating point data processing 3 source.
+enum FPDataProcessing3SourceOp {
+  FPDataProcessing3SourceFixed = 0x1F000000,
+  FPDataProcessing3SourceFMask = 0x5F000000,
+  FPDataProcessing3SourceMask  = 0xFFE08000,
+  FMADD_s                      = FPDataProcessing3SourceFixed | 0x00000000,
+  FMSUB_s                      = FPDataProcessing3SourceFixed | 0x00008000,
+  FNMADD_s                     = FPDataProcessing3SourceFixed | 0x00200000,
+  FNMSUB_s                     = FPDataProcessing3SourceFixed | 0x00208000,
+  FMADD_d                      = FPDataProcessing3SourceFixed | 0x00400000,
+  FMSUB_d                      = FPDataProcessing3SourceFixed | 0x00408000,
+  FNMADD_d                     = FPDataProcessing3SourceFixed | 0x00600000,
+  FNMSUB_d                     = FPDataProcessing3SourceFixed | 0x00608000
+};
+
+// Conversion between floating point and integer.
+enum FPIntegerConvertOp {
+  FPIntegerConvertFixed = 0x1E200000,
+  FPIntegerConvertFMask = 0x5F20FC00,
+  FPIntegerConvertMask  = 0xFFFFFC00,
+  FCVTNS    = FPIntegerConvertFixed | 0x00000000,
+  FCVTNS_ws = FCVTNS,
+  FCVTNS_xs = FCVTNS | SixtyFourBits,
+  FCVTNS_wd = FCVTNS | FP64,
+  FCVTNS_xd = FCVTNS | SixtyFourBits | FP64,
+  FCVTNU    = FPIntegerConvertFixed | 0x00010000,
+  FCVTNU_ws = FCVTNU,
+  FCVTNU_xs = FCVTNU | SixtyFourBits,
+  FCVTNU_wd = FCVTNU | FP64,
+  FCVTNU_xd = FCVTNU | SixtyFourBits | FP64,
+  FCVTPS    = FPIntegerConvertFixed | 0x00080000,
+  FCVTPS_ws = FCVTPS,
+  FCVTPS_xs = FCVTPS | SixtyFourBits,
+  FCVTPS_wd = FCVTPS | FP64,
+  FCVTPS_xd = FCVTPS | SixtyFourBits | FP64,
+  FCVTPU    = FPIntegerConvertFixed | 0x00090000,
+  FCVTPU_ws = FCVTPU,
+  FCVTPU_xs = FCVTPU | SixtyFourBits,
+  FCVTPU_wd = FCVTPU | FP64,
+  FCVTPU_xd = FCVTPU | SixtyFourBits | FP64,
+  FCVTMS    = FPIntegerConvertFixed | 0x00100000,
+  FCVTMS_ws = FCVTMS,
+  FCVTMS_xs = FCVTMS | SixtyFourBits,
+  FCVTMS_wd = FCVTMS | FP64,
+  FCVTMS_xd = FCVTMS | SixtyFourBits | FP64,
+  FCVTMU    = FPIntegerConvertFixed | 0x00110000,
+  FCVTMU_ws = FCVTMU,
+  FCVTMU_xs = FCVTMU | SixtyFourBits,
+  FCVTMU_wd = FCVTMU | FP64,
+  FCVTMU_xd = FCVTMU | SixtyFourBits | FP64,
+  FCVTZS    = FPIntegerConvertFixed | 0x00180000,
+  FCVTZS_ws = FCVTZS,
+  FCVTZS_xs = FCVTZS | SixtyFourBits,
+  FCVTZS_wd = FCVTZS | FP64,
+  FCVTZS_xd = FCVTZS | SixtyFourBits | FP64,
+  FCVTZU    = FPIntegerConvertFixed | 0x00190000,
+  FCVTZU_ws = FCVTZU,
+  FCVTZU_xs = FCVTZU | SixtyFourBits,
+  FCVTZU_wd = FCVTZU | FP64,
+  FCVTZU_xd = FCVTZU | SixtyFourBits | FP64,
+  SCVTF     = FPIntegerConvertFixed | 0x00020000,
+  SCVTF_sw  = SCVTF,
+  SCVTF_sx  = SCVTF | SixtyFourBits,
+  SCVTF_dw  = SCVTF | FP64,
+  SCVTF_dx  = SCVTF | SixtyFourBits | FP64,
+  UCVTF     = FPIntegerConvertFixed | 0x00030000,
+  UCVTF_sw  = UCVTF,
+  UCVTF_sx  = UCVTF | SixtyFourBits,
+  UCVTF_dw  = UCVTF | FP64,
+  UCVTF_dx  = UCVTF | SixtyFourBits | FP64,
+  FCVTAS    = FPIntegerConvertFixed | 0x00040000,
+  FCVTAS_ws = FCVTAS,
+  FCVTAS_xs = FCVTAS | SixtyFourBits,
+  FCVTAS_wd = FCVTAS | FP64,
+  FCVTAS_xd = FCVTAS | SixtyFourBits | FP64,
+  FCVTAU    = FPIntegerConvertFixed | 0x00050000,
+  FCVTAU_ws = FCVTAU,
+  FCVTAU_xs = FCVTAU | SixtyFourBits,
+  FCVTAU_wd = FCVTAU | FP64,
+  FCVTAU_xd = FCVTAU | SixtyFourBits | FP64,
+  FMOV_ws   = FPIntegerConvertFixed | 0x00060000,
+  FMOV_sw   = FPIntegerConvertFixed | 0x00070000,
+  FMOV_xd   = FMOV_ws | SixtyFourBits | FP64,
+  FMOV_dx   = FMOV_sw | SixtyFourBits | FP64,
+  FMOV_d1_x = FPIntegerConvertFixed | SixtyFourBits | 0x008F0000,
+  FMOV_x_d1 = FPIntegerConvertFixed | SixtyFourBits | 0x008E0000
+};
+
+// Conversion between fixed point and floating point.
+enum FPFixedPointConvertOp {
+  FPFixedPointConvertFixed = 0x1E000000,
+  FPFixedPointConvertFMask = 0x5F200000,
+  FPFixedPointConvertMask  = 0xFFFF0000,
+  FCVTZS_fixed    = FPFixedPointConvertFixed | 0x00180000,
+  FCVTZS_ws_fixed = FCVTZS_fixed,
+  FCVTZS_xs_fixed = FCVTZS_fixed | SixtyFourBits,
+  FCVTZS_wd_fixed = FCVTZS_fixed | FP64,
+  FCVTZS_xd_fixed = FCVTZS_fixed | SixtyFourBits | FP64,
+  FCVTZU_fixed    = FPFixedPointConvertFixed | 0x00190000,
+  FCVTZU_ws_fixed = FCVTZU_fixed,
+  FCVTZU_xs_fixed = FCVTZU_fixed | SixtyFourBits,
+  FCVTZU_wd_fixed = FCVTZU_fixed | FP64,
+  FCVTZU_xd_fixed = FCVTZU_fixed | SixtyFourBits | FP64,
+  SCVTF_fixed     = FPFixedPointConvertFixed | 0x00020000,
+  SCVTF_sw_fixed  = SCVTF_fixed,
+  SCVTF_sx_fixed  = SCVTF_fixed | SixtyFourBits,
+  SCVTF_dw_fixed  = SCVTF_fixed | FP64,
+  SCVTF_dx_fixed  = SCVTF_fixed | SixtyFourBits | FP64,
+  UCVTF_fixed     = FPFixedPointConvertFixed | 0x00030000,
+  UCVTF_sw_fixed  = UCVTF_fixed,
+  UCVTF_sx_fixed  = UCVTF_fixed | SixtyFourBits,
+  UCVTF_dw_fixed  = UCVTF_fixed | FP64,
+  UCVTF_dx_fixed  = UCVTF_fixed | SixtyFourBits | FP64
+};
+
+// Crypto - two register SHA.
+enum Crypto2RegSHAOp {
+  Crypto2RegSHAFixed = 0x5E280800,
+  Crypto2RegSHAFMask = 0xFF3E0C00
+};
+
+// Crypto - three register SHA.
+enum Crypto3RegSHAOp {
+  Crypto3RegSHAFixed = 0x5E000000,
+  Crypto3RegSHAFMask = 0xFF208C00
+};
+
+// Crypto - AES.
+enum CryptoAESOp {
+  CryptoAESFixed = 0x4E280800,
+  CryptoAESFMask = 0xFF3E0C00
+};
+
+// NEON instructions with two register operands.
+enum NEON2RegMiscOp {
+  NEON2RegMiscFixed = 0x0E200800,
+  NEON2RegMiscFMask = 0x9F3E0C00,
+  NEON2RegMiscMask  = 0xBF3FFC00,
+  NEON2RegMiscUBit  = 0x20000000,
+  NEON_REV64     = NEON2RegMiscFixed | 0x00000000,
+  NEON_REV32     = NEON2RegMiscFixed | 0x20000000,
+  NEON_REV16     = NEON2RegMiscFixed | 0x00001000,
+  NEON_SADDLP    = NEON2RegMiscFixed | 0x00002000,
+  NEON_UADDLP    = NEON_SADDLP | NEON2RegMiscUBit,
+  NEON_SUQADD    = NEON2RegMiscFixed | 0x00003000,
+  NEON_USQADD    = NEON_SUQADD | NEON2RegMiscUBit,
+  NEON_CLS       = NEON2RegMiscFixed | 0x00004000,
+  NEON_CLZ       = NEON2RegMiscFixed | 0x20004000,
+  NEON_CNT       = NEON2RegMiscFixed | 0x00005000,
+  NEON_RBIT_NOT  = NEON2RegMiscFixed | 0x20005000,
+  NEON_SADALP    = NEON2RegMiscFixed | 0x00006000,
+  NEON_UADALP    = NEON_SADALP | NEON2RegMiscUBit,
+  NEON_SQABS     = NEON2RegMiscFixed | 0x00007000,
+  NEON_SQNEG     = NEON2RegMiscFixed | 0x20007000,
+  NEON_CMGT_zero = NEON2RegMiscFixed | 0x00008000,
+  NEON_CMGE_zero = NEON2RegMiscFixed | 0x20008000,
+  NEON_CMEQ_zero = NEON2RegMiscFixed | 0x00009000,
+  NEON_CMLE_zero = NEON2RegMiscFixed | 0x20009000,
+  NEON_CMLT_zero = NEON2RegMiscFixed | 0x0000A000,
+  NEON_ABS       = NEON2RegMiscFixed | 0x0000B000,
+  NEON_NEG       = NEON2RegMiscFixed | 0x2000B000,
+  NEON_XTN       = NEON2RegMiscFixed | 0x00012000,
+  NEON_SQXTUN    = NEON2RegMiscFixed | 0x20012000,
+  NEON_SHLL      = NEON2RegMiscFixed | 0x20013000,
+  NEON_SQXTN     = NEON2RegMiscFixed | 0x00014000,
+  NEON_UQXTN     = NEON_SQXTN | NEON2RegMiscUBit,
+
+  NEON2RegMiscOpcode = 0x0001F000,
+  NEON_RBIT_NOT_opcode = NEON_RBIT_NOT & NEON2RegMiscOpcode,
+  NEON_NEG_opcode = NEON_NEG & NEON2RegMiscOpcode,
+  NEON_XTN_opcode = NEON_XTN & NEON2RegMiscOpcode,
+  NEON_UQXTN_opcode = NEON_UQXTN & NEON2RegMiscOpcode,
+
+  // These instructions use only one bit of the size field. The other bit is
+  // used to distinguish between instructions.
+  NEON2RegMiscFPMask = NEON2RegMiscMask | 0x00800000,
+  NEON_FABS   = NEON2RegMiscFixed | 0x0080F000,
+  NEON_FNEG   = NEON2RegMiscFixed | 0x2080F000,
+  NEON_FCVTN  = NEON2RegMiscFixed | 0x00016000,
+  NEON_FCVTXN = NEON2RegMiscFixed | 0x20016000,
+  NEON_FCVTL  = NEON2RegMiscFixed | 0x00017000,
+  NEON_FRINTN = NEON2RegMiscFixed | 0x00018000,
+  NEON_FRINTA = NEON2RegMiscFixed | 0x20018000,
+  NEON_FRINTP = NEON2RegMiscFixed | 0x00818000,
+  NEON_FRINTM = NEON2RegMiscFixed | 0x00019000,
+  NEON_FRINTX = NEON2RegMiscFixed | 0x20019000,
+  NEON_FRINTZ = NEON2RegMiscFixed | 0x00819000,
+  NEON_FRINTI = NEON2RegMiscFixed | 0x20819000,
+  NEON_FCVTNS = NEON2RegMiscFixed | 0x0001A000,
+  NEON_FCVTNU = NEON_FCVTNS | NEON2RegMiscUBit,
+  NEON_FCVTPS = NEON2RegMiscFixed | 0x0081A000,
+  NEON_FCVTPU = NEON_FCVTPS | NEON2RegMiscUBit,
+  NEON_FCVTMS = NEON2RegMiscFixed | 0x0001B000,
+  NEON_FCVTMU = NEON_FCVTMS | NEON2RegMiscUBit,
+  NEON_FCVTZS = NEON2RegMiscFixed | 0x0081B000,
+  NEON_FCVTZU = NEON_FCVTZS | NEON2RegMiscUBit,
+  NEON_FCVTAS = NEON2RegMiscFixed | 0x0001C000,
+  NEON_FCVTAU = NEON_FCVTAS | NEON2RegMiscUBit,
+  NEON_FSQRT  = NEON2RegMiscFixed | 0x2081F000,
+  NEON_SCVTF  = NEON2RegMiscFixed | 0x0001D000,
+  NEON_UCVTF  = NEON_SCVTF | NEON2RegMiscUBit,
+  NEON_URSQRTE = NEON2RegMiscFixed | 0x2081C000,
+  NEON_URECPE  = NEON2RegMiscFixed | 0x0081C000,
+  NEON_FRSQRTE = NEON2RegMiscFixed | 0x2081D000,
+  NEON_FRECPE  = NEON2RegMiscFixed | 0x0081D000,
+  NEON_FCMGT_zero = NEON2RegMiscFixed | 0x0080C000,
+  NEON_FCMGE_zero = NEON2RegMiscFixed | 0x2080C000,
+  NEON_FCMEQ_zero = NEON2RegMiscFixed | 0x0080D000,
+  NEON_FCMLE_zero = NEON2RegMiscFixed | 0x2080D000,
+  NEON_FCMLT_zero = NEON2RegMiscFixed | 0x0080E000,
+
+  NEON_FCVTL_opcode = NEON_FCVTL & NEON2RegMiscOpcode,
+  NEON_FCVTN_opcode = NEON_FCVTN & NEON2RegMiscOpcode
+};
+
+// NEON instructions with three same-type operands.
+enum NEON3SameOp {
+  NEON3SameFixed = 0x0E200400,
+  NEON3SameFMask = 0x9F200400,
+  NEON3SameMask = 0xBF20FC00,
+  NEON3SameUBit = 0x20000000,
+  NEON_ADD    = NEON3SameFixed | 0x00008000,
+  NEON_ADDP   = NEON3SameFixed | 0x0000B800,
+  NEON_SHADD  = NEON3SameFixed | 0x00000000,
+  NEON_SHSUB  = NEON3SameFixed | 0x00002000,
+  NEON_SRHADD = NEON3SameFixed | 0x00001000,
+  NEON_CMEQ   = NEON3SameFixed | NEON3SameUBit | 0x00008800,
+  NEON_CMGE   = NEON3SameFixed | 0x00003800,
+  NEON_CMGT   = NEON3SameFixed | 0x00003000,
+  NEON_CMHI   = NEON3SameFixed | NEON3SameUBit | NEON_CMGT,
+  NEON_CMHS   = NEON3SameFixed | NEON3SameUBit | NEON_CMGE,
+  NEON_CMTST  = NEON3SameFixed | 0x00008800,
+  NEON_MLA    = NEON3SameFixed | 0x00009000,
+  NEON_MLS    = NEON3SameFixed | 0x20009000,
+  NEON_MUL    = NEON3SameFixed | 0x00009800,
+  NEON_PMUL   = NEON3SameFixed | 0x20009800,
+  NEON_SRSHL  = NEON3SameFixed | 0x00005000,
+  NEON_SQSHL  = NEON3SameFixed | 0x00004800,
+  NEON_SQRSHL = NEON3SameFixed | 0x00005800,
+  NEON_SSHL   = NEON3SameFixed | 0x00004000,
+  NEON_SMAX   = NEON3SameFixed | 0x00006000,
+  NEON_SMAXP  = NEON3SameFixed | 0x0000A000,
+  NEON_SMIN   = NEON3SameFixed | 0x00006800,
+  NEON_SMINP  = NEON3SameFixed | 0x0000A800,
+  NEON_SABD   = NEON3SameFixed | 0x00007000,
+  NEON_SABA   = NEON3SameFixed | 0x00007800,
+  NEON_UABD   = NEON3SameFixed | NEON3SameUBit | NEON_SABD,
+  NEON_UABA   = NEON3SameFixed | NEON3SameUBit | NEON_SABA,
+  NEON_SQADD  = NEON3SameFixed | 0x00000800,
+  NEON_SQSUB  = NEON3SameFixed | 0x00002800,
+  NEON_SUB    = NEON3SameFixed | NEON3SameUBit | 0x00008000,
+  NEON_UHADD  = NEON3SameFixed | NEON3SameUBit | NEON_SHADD,
+  NEON_UHSUB  = NEON3SameFixed | NEON3SameUBit | NEON_SHSUB,
+  NEON_URHADD = NEON3SameFixed | NEON3SameUBit | NEON_SRHADD,
+  NEON_UMAX   = NEON3SameFixed | NEON3SameUBit | NEON_SMAX,
+  NEON_UMAXP  = NEON3SameFixed | NEON3SameUBit | NEON_SMAXP,
+  NEON_UMIN   = NEON3SameFixed | NEON3SameUBit | NEON_SMIN,
+  NEON_UMINP  = NEON3SameFixed | NEON3SameUBit | NEON_SMINP,
+  NEON_URSHL  = NEON3SameFixed | NEON3SameUBit | NEON_SRSHL,
+  NEON_UQADD  = NEON3SameFixed | NEON3SameUBit | NEON_SQADD,
+  NEON_UQRSHL = NEON3SameFixed | NEON3SameUBit | NEON_SQRSHL,
+  NEON_UQSHL  = NEON3SameFixed | NEON3SameUBit | NEON_SQSHL,
+  NEON_UQSUB  = NEON3SameFixed | NEON3SameUBit | NEON_SQSUB,
+  NEON_USHL   = NEON3SameFixed | NEON3SameUBit | NEON_SSHL,
+  NEON_SQDMULH  = NEON3SameFixed | 0x0000B000,
+  NEON_SQRDMULH = NEON3SameFixed | 0x2000B000,
+
+  // NEON floating point instructions with three same-type operands.
+  NEON3SameFPFixed = NEON3SameFixed | 0x0000C000,
+  NEON3SameFPFMask = NEON3SameFMask | 0x0000C000,
+  NEON3SameFPMask = NEON3SameMask | 0x00800000,
+  NEON_FADD    = NEON3SameFixed | 0x0000D000,
+  NEON_FSUB    = NEON3SameFixed | 0x0080D000,
+  NEON_FMUL    = NEON3SameFixed | 0x2000D800,
+  NEON_FDIV    = NEON3SameFixed | 0x2000F800,
+  NEON_FMAX    = NEON3SameFixed | 0x0000F000,
+  NEON_FMAXNM  = NEON3SameFixed | 0x0000C000,
+  NEON_FMAXP   = NEON3SameFixed | 0x2000F000,
+  NEON_FMAXNMP = NEON3SameFixed | 0x2000C000,
+  NEON_FMIN    = NEON3SameFixed | 0x0080F000,
+  NEON_FMINNM  = NEON3SameFixed | 0x0080C000,
+  NEON_FMINP   = NEON3SameFixed | 0x2080F000,
+  NEON_FMINNMP = NEON3SameFixed | 0x2080C000,
+  NEON_FMLA    = NEON3SameFixed | 0x0000C800,
+  NEON_FMLS    = NEON3SameFixed | 0x0080C800,
+  NEON_FMULX   = NEON3SameFixed | 0x0000D800,
+  NEON_FRECPS  = NEON3SameFixed | 0x0000F800,
+  NEON_FRSQRTS = NEON3SameFixed | 0x0080F800,
+  NEON_FABD    = NEON3SameFixed | 0x2080D000,
+  NEON_FADDP   = NEON3SameFixed | 0x2000D000,
+  NEON_FCMEQ   = NEON3SameFixed | 0x0000E000,
+  NEON_FCMGE   = NEON3SameFixed | 0x2000E000,
+  NEON_FCMGT   = NEON3SameFixed | 0x2080E000,
+  NEON_FACGE   = NEON3SameFixed | 0x2000E800,
+  NEON_FACGT   = NEON3SameFixed | 0x2080E800,
+
+  // NEON logical instructions with three same-type operands.
+  NEON3SameLogicalFixed = NEON3SameFixed | 0x00001800,
+  NEON3SameLogicalFMask = NEON3SameFMask | 0x0000F800,
+  NEON3SameLogicalMask = 0xBFE0FC00,
+  NEON3SameLogicalFormatMask = NEON_Q,
+  NEON_AND = NEON3SameLogicalFixed | 0x00000000,
+  NEON_ORR = NEON3SameLogicalFixed | 0x00A00000,
+  NEON_ORN = NEON3SameLogicalFixed | 0x00C00000,
+  NEON_EOR = NEON3SameLogicalFixed | 0x20000000,
+  NEON_BIC = NEON3SameLogicalFixed | 0x00400000,
+  NEON_BIF = NEON3SameLogicalFixed | 0x20C00000,
+  NEON_BIT = NEON3SameLogicalFixed | 0x20800000,
+  NEON_BSL = NEON3SameLogicalFixed | 0x20400000
+};
+
+// NEON instructions with three different-type operands.
+enum NEON3DifferentOp {
+  NEON3DifferentFixed = 0x0E200000,
+  NEON3DifferentFMask = 0x9F200C00,
+  NEON3DifferentMask  = 0xFF20FC00,
+  NEON_ADDHN    = NEON3DifferentFixed | 0x00004000,
+  NEON_ADDHN2   = NEON_ADDHN | NEON_Q,
+  NEON_PMULL    = NEON3DifferentFixed | 0x0000E000,
+  NEON_PMULL2   = NEON_PMULL | NEON_Q,
+  NEON_RADDHN   = NEON3DifferentFixed | 0x20004000,
+  NEON_RADDHN2  = NEON_RADDHN | NEON_Q,
+  NEON_RSUBHN   = NEON3DifferentFixed | 0x20006000,
+  NEON_RSUBHN2  = NEON_RSUBHN | NEON_Q,
+  NEON_SABAL    = NEON3DifferentFixed | 0x00005000,
+  NEON_SABAL2   = NEON_SABAL | NEON_Q,
+  NEON_SABDL    = NEON3DifferentFixed | 0x00007000,
+  NEON_SABDL2   = NEON_SABDL | NEON_Q,
+  NEON_SADDL    = NEON3DifferentFixed | 0x00000000,
+  NEON_SADDL2   = NEON_SADDL | NEON_Q,
+  NEON_SADDW    = NEON3DifferentFixed | 0x00001000,
+  NEON_SADDW2   = NEON_SADDW | NEON_Q,
+  NEON_SMLAL    = NEON3DifferentFixed | 0x00008000,
+  NEON_SMLAL2   = NEON_SMLAL | NEON_Q,
+  NEON_SMLSL    = NEON3DifferentFixed | 0x0000A000,
+  NEON_SMLSL2   = NEON_SMLSL | NEON_Q,
+  NEON_SMULL    = NEON3DifferentFixed | 0x0000C000,
+  NEON_SMULL2   = NEON_SMULL | NEON_Q,
+  NEON_SSUBL    = NEON3DifferentFixed | 0x00002000,
+  NEON_SSUBL2   = NEON_SSUBL | NEON_Q,
+  NEON_SSUBW    = NEON3DifferentFixed | 0x00003000,
+  NEON_SSUBW2   = NEON_SSUBW | NEON_Q,
+  NEON_SQDMLAL  = NEON3DifferentFixed | 0x00009000,
+  NEON_SQDMLAL2 = NEON_SQDMLAL | NEON_Q,
+  NEON_SQDMLSL  = NEON3DifferentFixed | 0x0000B000,
+  NEON_SQDMLSL2 = NEON_SQDMLSL | NEON_Q,
+  NEON_SQDMULL  = NEON3DifferentFixed | 0x0000D000,
+  NEON_SQDMULL2 = NEON_SQDMULL | NEON_Q,
+  NEON_SUBHN    = NEON3DifferentFixed | 0x00006000,
+  NEON_SUBHN2   = NEON_SUBHN | NEON_Q,
+  NEON_UABAL    = NEON_SABAL | NEON3SameUBit,
+  NEON_UABAL2   = NEON_UABAL | NEON_Q,
+  NEON_UABDL    = NEON_SABDL | NEON3SameUBit,
+  NEON_UABDL2   = NEON_UABDL | NEON_Q,
+  NEON_UADDL    = NEON_SADDL | NEON3SameUBit,
+  NEON_UADDL2   = NEON_UADDL | NEON_Q,
+  NEON_UADDW    = NEON_SADDW | NEON3SameUBit,
+  NEON_UADDW2   = NEON_UADDW | NEON_Q,
+  NEON_UMLAL    = NEON_SMLAL | NEON3SameUBit,
+  NEON_UMLAL2   = NEON_UMLAL | NEON_Q,
+  NEON_UMLSL    = NEON_SMLSL | NEON3SameUBit,
+  NEON_UMLSL2   = NEON_UMLSL | NEON_Q,
+  NEON_UMULL    = NEON_SMULL | NEON3SameUBit,
+  NEON_UMULL2   = NEON_UMULL | NEON_Q,
+  NEON_USUBL    = NEON_SSUBL | NEON3SameUBit,
+  NEON_USUBL2   = NEON_USUBL | NEON_Q,
+  NEON_USUBW    = NEON_SSUBW | NEON3SameUBit,
+  NEON_USUBW2   = NEON_USUBW | NEON_Q
+};
+
+// NEON instructions operating across vectors.
+enum NEONAcrossLanesOp {
+  NEONAcrossLanesFixed = 0x0E300800,
+  NEONAcrossLanesFMask = 0x9F3E0C00,
+  NEONAcrossLanesMask  = 0xBF3FFC00,
+  NEON_ADDV   = NEONAcrossLanesFixed | 0x0001B000,
+  NEON_SADDLV = NEONAcrossLanesFixed | 0x00003000,
+  NEON_UADDLV = NEONAcrossLanesFixed | 0x20003000,
+  NEON_SMAXV  = NEONAcrossLanesFixed | 0x0000A000,
+  NEON_SMINV  = NEONAcrossLanesFixed | 0x0001A000,
+  NEON_UMAXV  = NEONAcrossLanesFixed | 0x2000A000,
+  NEON_UMINV  = NEONAcrossLanesFixed | 0x2001A000,
+
+  // NEON floating point across instructions.
+  NEONAcrossLanesFPFixed = NEONAcrossLanesFixed | 0x0000C000,
+  NEONAcrossLanesFPFMask = NEONAcrossLanesFMask | 0x0000C000,
+  NEONAcrossLanesFPMask  = NEONAcrossLanesMask  | 0x00800000,
+
+  NEON_FMAXV   = NEONAcrossLanesFPFixed | 0x2000F000,
+  NEON_FMINV   = NEONAcrossLanesFPFixed | 0x2080F000,
+  NEON_FMAXNMV = NEONAcrossLanesFPFixed | 0x2000C000,
+  NEON_FMINNMV = NEONAcrossLanesFPFixed | 0x2080C000
+};
+
+// NEON instructions with indexed element operand.
+enum NEONByIndexedElementOp {
+  NEONByIndexedElementFixed = 0x0F000000,
+  NEONByIndexedElementFMask = 0x9F000400,
+  NEONByIndexedElementMask  = 0xBF00F400,
+  NEON_MUL_byelement   = NEONByIndexedElementFixed | 0x00008000,
+  NEON_MLA_byelement   = NEONByIndexedElementFixed | 0x20000000,
+  NEON_MLS_byelement   = NEONByIndexedElementFixed | 0x20004000,
+  NEON_SMULL_byelement = NEONByIndexedElementFixed | 0x0000A000,
+  NEON_SMLAL_byelement = NEONByIndexedElementFixed | 0x00002000,
+  NEON_SMLSL_byelement = NEONByIndexedElementFixed | 0x00006000,
+  NEON_UMULL_byelement = NEONByIndexedElementFixed | 0x2000A000,
+  NEON_UMLAL_byelement = NEONByIndexedElementFixed | 0x20002000,
+  NEON_UMLSL_byelement = NEONByIndexedElementFixed | 0x20006000,
+  NEON_SQDMULL_byelement = NEONByIndexedElementFixed | 0x0000B000,
+  NEON_SQDMLAL_byelement = NEONByIndexedElementFixed | 0x00003000,
+  NEON_SQDMLSL_byelement = NEONByIndexedElementFixed | 0x00007000,
+  NEON_SQDMULH_byelement  = NEONByIndexedElementFixed | 0x0000C000,
+  NEON_SQRDMULH_byelement = NEONByIndexedElementFixed | 0x0000D000,
+
+  // Floating point instructions.
+  NEONByIndexedElementFPFixed = NEONByIndexedElementFixed | 0x00800000,
+  NEONByIndexedElementFPMask = NEONByIndexedElementMask | 0x00800000,
+  NEON_FMLA_byelement  = NEONByIndexedElementFPFixed | 0x00001000,
+  NEON_FMLS_byelement  = NEONByIndexedElementFPFixed | 0x00005000,
+  NEON_FMUL_byelement  = NEONByIndexedElementFPFixed | 0x00009000,
+  NEON_FMULX_byelement = NEONByIndexedElementFPFixed | 0x20009000
+};
+
+// NEON register copy.
+enum NEONCopyOp {
+  NEONCopyFixed = 0x0E000400,
+  NEONCopyFMask = 0x9FE08400,
+  NEONCopyMask  = 0x3FE08400,
+  NEONCopyInsElementMask = NEONCopyMask | 0x40000000,
+  NEONCopyInsGeneralMask = NEONCopyMask | 0x40007800,
+  NEONCopyDupElementMask = NEONCopyMask | 0x20007800,
+  NEONCopyDupGeneralMask = NEONCopyDupElementMask,
+  NEONCopyUmovMask       = NEONCopyMask | 0x20007800,
+  NEONCopySmovMask       = NEONCopyMask | 0x20007800,
+  NEON_INS_ELEMENT       = NEONCopyFixed | 0x60000000,
+  NEON_INS_GENERAL       = NEONCopyFixed | 0x40001800,
+  NEON_DUP_ELEMENT       = NEONCopyFixed | 0x00000000,
+  NEON_DUP_GENERAL       = NEONCopyFixed | 0x00000800,
+  NEON_SMOV              = NEONCopyFixed | 0x00002800,
+  NEON_UMOV              = NEONCopyFixed | 0x00003800
+};
+
+// NEON extract.
+enum NEONExtractOp {
+  NEONExtractFixed = 0x2E000000,
+  NEONExtractFMask = 0xBF208400,
+  NEONExtractMask = 0xBFE08400,
+  NEON_EXT = NEONExtractFixed | 0x00000000
+};
+
+enum NEONLoadStoreMultiOp {
+  NEONLoadStoreMultiL    = 0x00400000,
+  NEONLoadStoreMulti1_1v = 0x00007000,
+  NEONLoadStoreMulti1_2v = 0x0000A000,
+  NEONLoadStoreMulti1_3v = 0x00006000,
+  NEONLoadStoreMulti1_4v = 0x00002000,
+  NEONLoadStoreMulti2    = 0x00008000,
+  NEONLoadStoreMulti3    = 0x00004000,
+  NEONLoadStoreMulti4    = 0x00000000
+};
+
+// NEON load/store multiple structures.
+enum NEONLoadStoreMultiStructOp {
+  NEONLoadStoreMultiStructFixed = 0x0C000000,
+  NEONLoadStoreMultiStructFMask = 0xBFBF0000,
+  NEONLoadStoreMultiStructMask  = 0xBFFFF000,
+  NEONLoadStoreMultiStructStore = NEONLoadStoreMultiStructFixed,
+  NEONLoadStoreMultiStructLoad  = NEONLoadStoreMultiStructFixed |
+                                  NEONLoadStoreMultiL,
+  NEON_LD1_1v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_1v,
+  NEON_LD1_2v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_2v,
+  NEON_LD1_3v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_3v,
+  NEON_LD1_4v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_4v,
+  NEON_LD2    = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti2,
+  NEON_LD3    = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti3,
+  NEON_LD4    = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti4,
+  NEON_ST1_1v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_1v,
+  NEON_ST1_2v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_2v,
+  NEON_ST1_3v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_3v,
+  NEON_ST1_4v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_4v,
+  NEON_ST2    = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti2,
+  NEON_ST3    = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti3,
+  NEON_ST4    = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti4
+};
+
+// NEON load/store multiple structures with post-index addressing.
+enum NEONLoadStoreMultiStructPostIndexOp {
+  NEONLoadStoreMultiStructPostIndexFixed = 0x0C800000,
+  NEONLoadStoreMultiStructPostIndexFMask = 0xBFA00000,
+  NEONLoadStoreMultiStructPostIndexMask  = 0xBFE0F000,
+  NEONLoadStoreMultiStructPostIndex = 0x00800000,
+  NEON_LD1_1v_post = NEON_LD1_1v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD1_2v_post = NEON_LD1_2v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD1_3v_post = NEON_LD1_3v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD1_4v_post = NEON_LD1_4v | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD2_post = NEON_LD2 | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD3_post = NEON_LD3 | NEONLoadStoreMultiStructPostIndex,
+  NEON_LD4_post = NEON_LD4 | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_1v_post = NEON_ST1_1v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_2v_post = NEON_ST1_2v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_3v_post = NEON_ST1_3v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST1_4v_post = NEON_ST1_4v | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST2_post = NEON_ST2 | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST3_post = NEON_ST3 | NEONLoadStoreMultiStructPostIndex,
+  NEON_ST4_post = NEON_ST4 | NEONLoadStoreMultiStructPostIndex
+};
+
+enum NEONLoadStoreSingleOp {
+  NEONLoadStoreSingle1        = 0x00000000,
+  NEONLoadStoreSingle2        = 0x00200000,
+  NEONLoadStoreSingle3        = 0x00002000,
+  NEONLoadStoreSingle4        = 0x00202000,
+  NEONLoadStoreSingleL        = 0x00400000,
+  NEONLoadStoreSingle_b       = 0x00000000,
+  NEONLoadStoreSingle_h       = 0x00004000,
+  NEONLoadStoreSingle_s       = 0x00008000,
+  NEONLoadStoreSingle_d       = 0x00008400,
+  NEONLoadStoreSingleAllLanes = 0x0000C000,
+  NEONLoadStoreSingleLenMask  = 0x00202000
+};
+
+// NEON load/store single structure.
+enum NEONLoadStoreSingleStructOp {
+  NEONLoadStoreSingleStructFixed = 0x0D000000,
+  NEONLoadStoreSingleStructFMask = 0xBF9F0000,
+  NEONLoadStoreSingleStructMask  = 0xBFFFE000,
+  NEONLoadStoreSingleStructStore = NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructLoad  = NEONLoadStoreSingleStructFixed |
+                                   NEONLoadStoreSingleL,
+  NEONLoadStoreSingleStructLoad1 = NEONLoadStoreSingle1 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructLoad2 = NEONLoadStoreSingle2 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructLoad3 = NEONLoadStoreSingle3 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructLoad4 = NEONLoadStoreSingle4 |
+                                   NEONLoadStoreSingleStructLoad,
+  NEONLoadStoreSingleStructStore1 = NEONLoadStoreSingle1 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructStore2 = NEONLoadStoreSingle2 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructStore3 = NEONLoadStoreSingle3 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEONLoadStoreSingleStructStore4 = NEONLoadStoreSingle4 |
+                                    NEONLoadStoreSingleStructFixed,
+  NEON_LD1_b = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_b,
+  NEON_LD1_h = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_h,
+  NEON_LD1_s = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_s,
+  NEON_LD1_d = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_d,
+  NEON_LD1R  = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingleAllLanes,
+  NEON_ST1_b = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_b,
+  NEON_ST1_h = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_h,
+  NEON_ST1_s = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_s,
+  NEON_ST1_d = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_d,
+
+  NEON_LD2_b = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_b,
+  NEON_LD2_h = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_h,
+  NEON_LD2_s = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_s,
+  NEON_LD2_d = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_d,
+  NEON_LD2R  = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingleAllLanes,
+  NEON_ST2_b = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_b,
+  NEON_ST2_h = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_h,
+  NEON_ST2_s = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_s,
+  NEON_ST2_d = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_d,
+
+  NEON_LD3_b = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_b,
+  NEON_LD3_h = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_h,
+  NEON_LD3_s = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_s,
+  NEON_LD3_d = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_d,
+  NEON_LD3R  = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingleAllLanes,
+  NEON_ST3_b = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_b,
+  NEON_ST3_h = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_h,
+  NEON_ST3_s = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_s,
+  NEON_ST3_d = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_d,
+
+  NEON_LD4_b = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_b,
+  NEON_LD4_h = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_h,
+  NEON_LD4_s = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_s,
+  NEON_LD4_d = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_d,
+  NEON_LD4R  = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingleAllLanes,
+  NEON_ST4_b = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_b,
+  NEON_ST4_h = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_h,
+  NEON_ST4_s = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_s,
+  NEON_ST4_d = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_d
+};
+
+// NEON load/store single structure with post-index addressing.
+enum NEONLoadStoreSingleStructPostIndexOp {
+  NEONLoadStoreSingleStructPostIndexFixed = 0x0D800000,
+  NEONLoadStoreSingleStructPostIndexFMask = 0xBF800000,
+  NEONLoadStoreSingleStructPostIndexMask  = 0xBFE0E000,
+  NEONLoadStoreSingleStructPostIndex = 0x00800000,
+  NEON_LD1_b_post = NEON_LD1_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1_h_post = NEON_LD1_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1_s_post = NEON_LD1_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1_d_post = NEON_LD1_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD1R_post  = NEON_LD1R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_b_post = NEON_ST1_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_h_post = NEON_ST1_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_s_post = NEON_ST1_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST1_d_post = NEON_ST1_d | NEONLoadStoreSingleStructPostIndex,
+
+  NEON_LD2_b_post = NEON_LD2_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2_h_post = NEON_LD2_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2_s_post = NEON_LD2_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2_d_post = NEON_LD2_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD2R_post  = NEON_LD2R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_b_post = NEON_ST2_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_h_post = NEON_ST2_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_s_post = NEON_ST2_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST2_d_post = NEON_ST2_d | NEONLoadStoreSingleStructPostIndex,
+
+  NEON_LD3_b_post = NEON_LD3_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3_h_post = NEON_LD3_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3_s_post = NEON_LD3_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3_d_post = NEON_LD3_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD3R_post  = NEON_LD3R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_b_post = NEON_ST3_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_h_post = NEON_ST3_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_s_post = NEON_ST3_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST3_d_post = NEON_ST3_d | NEONLoadStoreSingleStructPostIndex,
+
+  NEON_LD4_b_post = NEON_LD4_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4_h_post = NEON_LD4_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4_s_post = NEON_LD4_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4_d_post = NEON_LD4_d | NEONLoadStoreSingleStructPostIndex,
+  NEON_LD4R_post  = NEON_LD4R | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_b_post = NEON_ST4_b | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_h_post = NEON_ST4_h | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_s_post = NEON_ST4_s | NEONLoadStoreSingleStructPostIndex,
+  NEON_ST4_d_post = NEON_ST4_d | NEONLoadStoreSingleStructPostIndex
+};
+
+// NEON modified immediate.
+enum NEONModifiedImmediateOp {
+  NEONModifiedImmediateFixed = 0x0F000400,
+  NEONModifiedImmediateFMask = 0x9FF80400,
+  NEONModifiedImmediateOpBit = 0x20000000,
+  NEONModifiedImmediate_MOVI = NEONModifiedImmediateFixed | 0x00000000,
+  NEONModifiedImmediate_MVNI = NEONModifiedImmediateFixed | 0x20000000,
+  NEONModifiedImmediate_ORR  = NEONModifiedImmediateFixed | 0x00001000,
+  NEONModifiedImmediate_BIC  = NEONModifiedImmediateFixed | 0x20001000
+};
+
+// NEON shift immediate.
+enum NEONShiftImmediateOp {
+  NEONShiftImmediateFixed = 0x0F000400,
+  NEONShiftImmediateFMask = 0x9F800400,
+  NEONShiftImmediateMask  = 0xBF80FC00,
+  NEONShiftImmediateUBit  = 0x20000000,
+  NEON_SHL      = NEONShiftImmediateFixed | 0x00005000,
+  NEON_SSHLL    = NEONShiftImmediateFixed | 0x0000A000,
+  NEON_USHLL    = NEONShiftImmediateFixed | 0x2000A000,
+  NEON_SLI      = NEONShiftImmediateFixed | 0x20005000,
+  NEON_SRI      = NEONShiftImmediateFixed | 0x20004000,
+  NEON_SHRN     = NEONShiftImmediateFixed | 0x00008000,
+  NEON_RSHRN    = NEONShiftImmediateFixed | 0x00008800,
+  NEON_UQSHRN   = NEONShiftImmediateFixed | 0x20009000,
+  NEON_UQRSHRN  = NEONShiftImmediateFixed | 0x20009800,
+  NEON_SQSHRN   = NEONShiftImmediateFixed | 0x00009000,
+  NEON_SQRSHRN  = NEONShiftImmediateFixed | 0x00009800,
+  NEON_SQSHRUN  = NEONShiftImmediateFixed | 0x20008000,
+  NEON_SQRSHRUN = NEONShiftImmediateFixed | 0x20008800,
+  NEON_SSHR     = NEONShiftImmediateFixed | 0x00000000,
+  NEON_SRSHR    = NEONShiftImmediateFixed | 0x00002000,
+  NEON_USHR     = NEONShiftImmediateFixed | 0x20000000,
+  NEON_URSHR    = NEONShiftImmediateFixed | 0x20002000,
+  NEON_SSRA     = NEONShiftImmediateFixed | 0x00001000,
+  NEON_SRSRA    = NEONShiftImmediateFixed | 0x00003000,
+  NEON_USRA     = NEONShiftImmediateFixed | 0x20001000,
+  NEON_URSRA    = NEONShiftImmediateFixed | 0x20003000,
+  NEON_SQSHLU   = NEONShiftImmediateFixed | 0x20006000,
+  NEON_SCVTF_imm = NEONShiftImmediateFixed | 0x0000E000,
+  NEON_UCVTF_imm = NEONShiftImmediateFixed | 0x2000E000,
+  NEON_FCVTZS_imm = NEONShiftImmediateFixed | 0x0000F800,
+  NEON_FCVTZU_imm = NEONShiftImmediateFixed | 0x2000F800,
+  NEON_SQSHL_imm = NEONShiftImmediateFixed | 0x00007000,
+  NEON_UQSHL_imm = NEONShiftImmediateFixed | 0x20007000
+};
+
+// NEON table.
+enum NEONTableOp {
+  NEONTableFixed = 0x0E000000,
+  NEONTableFMask = 0xBF208C00,
+  NEONTableExt   = 0x00001000,
+  NEONTableMask  = 0xBF20FC00,
+  NEON_TBL_1v    = NEONTableFixed | 0x00000000,
+  NEON_TBL_2v    = NEONTableFixed | 0x00002000,
+  NEON_TBL_3v    = NEONTableFixed | 0x00004000,
+  NEON_TBL_4v    = NEONTableFixed | 0x00006000,
+  NEON_TBX_1v    = NEON_TBL_1v | NEONTableExt,
+  NEON_TBX_2v    = NEON_TBL_2v | NEONTableExt,
+  NEON_TBX_3v    = NEON_TBL_3v | NEONTableExt,
+  NEON_TBX_4v    = NEON_TBL_4v | NEONTableExt
+};
+
+// NEON perm.
+enum NEONPermOp {
+  NEONPermFixed = 0x0E000800,
+  NEONPermFMask = 0xBF208C00,
+  NEONPermMask  = 0x3F20FC00,
+  NEON_UZP1 = NEONPermFixed | 0x00001000,
+  NEON_TRN1 = NEONPermFixed | 0x00002000,
+  NEON_ZIP1 = NEONPermFixed | 0x00003000,
+  NEON_UZP2 = NEONPermFixed | 0x00005000,
+  NEON_TRN2 = NEONPermFixed | 0x00006000,
+  NEON_ZIP2 = NEONPermFixed | 0x00007000
+};
+
+// NEON scalar instructions with two register operands.
+enum NEONScalar2RegMiscOp {
+  NEONScalar2RegMiscFixed = 0x5E200800,
+  NEONScalar2RegMiscFMask = 0xDF3E0C00,
+  NEONScalar2RegMiscMask = NEON_Q | NEONScalar | NEON2RegMiscMask,
+  NEON_CMGT_zero_scalar = NEON_Q | NEONScalar | NEON_CMGT_zero,
+  NEON_CMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_CMEQ_zero,
+  NEON_CMLT_zero_scalar = NEON_Q | NEONScalar | NEON_CMLT_zero,
+  NEON_CMGE_zero_scalar = NEON_Q | NEONScalar | NEON_CMGE_zero,
+  NEON_CMLE_zero_scalar = NEON_Q | NEONScalar | NEON_CMLE_zero,
+  NEON_ABS_scalar       = NEON_Q | NEONScalar | NEON_ABS,
+  NEON_SQABS_scalar     = NEON_Q | NEONScalar | NEON_SQABS,
+  NEON_NEG_scalar       = NEON_Q | NEONScalar | NEON_NEG,
+  NEON_SQNEG_scalar     = NEON_Q | NEONScalar | NEON_SQNEG,
+  NEON_SQXTN_scalar     = NEON_Q | NEONScalar | NEON_SQXTN,
+  NEON_UQXTN_scalar     = NEON_Q | NEONScalar | NEON_UQXTN,
+  NEON_SQXTUN_scalar    = NEON_Q | NEONScalar | NEON_SQXTUN,
+  NEON_SUQADD_scalar    = NEON_Q | NEONScalar | NEON_SUQADD,
+  NEON_USQADD_scalar    = NEON_Q | NEONScalar | NEON_USQADD,
+
+  NEONScalar2RegMiscOpcode = NEON2RegMiscOpcode,
+  NEON_NEG_scalar_opcode = NEON_NEG_scalar & NEONScalar2RegMiscOpcode,
+
+  NEONScalar2RegMiscFPMask  = NEONScalar2RegMiscMask | 0x00800000,
+  NEON_FRSQRTE_scalar    = NEON_Q | NEONScalar | NEON_FRSQRTE,
+  NEON_FRECPE_scalar     = NEON_Q | NEONScalar | NEON_FRECPE,
+  NEON_SCVTF_scalar      = NEON_Q | NEONScalar | NEON_SCVTF,
+  NEON_UCVTF_scalar      = NEON_Q | NEONScalar | NEON_UCVTF,
+  NEON_FCMGT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGT_zero,
+  NEON_FCMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_FCMEQ_zero,
+  NEON_FCMLT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLT_zero,
+  NEON_FCMGE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGE_zero,
+  NEON_FCMLE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLE_zero,
+  NEON_FRECPX_scalar     = NEONScalar2RegMiscFixed | 0x0081F000,
+  NEON_FCVTNS_scalar     = NEON_Q | NEONScalar | NEON_FCVTNS,
+  NEON_FCVTNU_scalar     = NEON_Q | NEONScalar | NEON_FCVTNU,
+  NEON_FCVTPS_scalar     = NEON_Q | NEONScalar | NEON_FCVTPS,
+  NEON_FCVTPU_scalar     = NEON_Q | NEONScalar | NEON_FCVTPU,
+  NEON_FCVTMS_scalar     = NEON_Q | NEONScalar | NEON_FCVTMS,
+  NEON_FCVTMU_scalar     = NEON_Q | NEONScalar | NEON_FCVTMU,
+  NEON_FCVTZS_scalar     = NEON_Q | NEONScalar | NEON_FCVTZS,
+  NEON_FCVTZU_scalar     = NEON_Q | NEONScalar | NEON_FCVTZU,
+  NEON_FCVTAS_scalar     = NEON_Q | NEONScalar | NEON_FCVTAS,
+  NEON_FCVTAU_scalar     = NEON_Q | NEONScalar | NEON_FCVTAU,
+  NEON_FCVTXN_scalar     = NEON_Q | NEONScalar | NEON_FCVTXN
+};
+
+// NEON scalar instructions with three same-type operands.
+enum NEONScalar3SameOp {
+  NEONScalar3SameFixed = 0x5E200400,
+  NEONScalar3SameFMask = 0xDF200400,
+  NEONScalar3SameMask  = 0xFF20FC00,
+  NEON_ADD_scalar    = NEON_Q | NEONScalar | NEON_ADD,
+  NEON_CMEQ_scalar   = NEON_Q | NEONScalar | NEON_CMEQ,
+  NEON_CMGE_scalar   = NEON_Q | NEONScalar | NEON_CMGE,
+  NEON_CMGT_scalar   = NEON_Q | NEONScalar | NEON_CMGT,
+  NEON_CMHI_scalar   = NEON_Q | NEONScalar | NEON_CMHI,
+  NEON_CMHS_scalar   = NEON_Q | NEONScalar | NEON_CMHS,
+  NEON_CMTST_scalar  = NEON_Q | NEONScalar | NEON_CMTST,
+  NEON_SUB_scalar    = NEON_Q | NEONScalar | NEON_SUB,
+  NEON_UQADD_scalar  = NEON_Q | NEONScalar | NEON_UQADD,
+  NEON_SQADD_scalar  = NEON_Q | NEONScalar | NEON_SQADD,
+  NEON_UQSUB_scalar  = NEON_Q | NEONScalar | NEON_UQSUB,
+  NEON_SQSUB_scalar  = NEON_Q | NEONScalar | NEON_SQSUB,
+  NEON_USHL_scalar   = NEON_Q | NEONScalar | NEON_USHL,
+  NEON_SSHL_scalar   = NEON_Q | NEONScalar | NEON_SSHL,
+  NEON_UQSHL_scalar  = NEON_Q | NEONScalar | NEON_UQSHL,
+  NEON_SQSHL_scalar  = NEON_Q | NEONScalar | NEON_SQSHL,
+  NEON_URSHL_scalar  = NEON_Q | NEONScalar | NEON_URSHL,
+  NEON_SRSHL_scalar  = NEON_Q | NEONScalar | NEON_SRSHL,
+  NEON_UQRSHL_scalar = NEON_Q | NEONScalar | NEON_UQRSHL,
+  NEON_SQRSHL_scalar = NEON_Q | NEONScalar | NEON_SQRSHL,
+  NEON_SQDMULH_scalar = NEON_Q | NEONScalar | NEON_SQDMULH,
+  NEON_SQRDMULH_scalar = NEON_Q | NEONScalar | NEON_SQRDMULH,
+
+  // NEON floating point scalar instructions with three same-type operands.
+  NEONScalar3SameFPFixed = NEONScalar3SameFixed | 0x0000C000,
+  NEONScalar3SameFPFMask = NEONScalar3SameFMask | 0x0000C000,
+  NEONScalar3SameFPMask  = NEONScalar3SameMask | 0x00800000,
+  NEON_FACGE_scalar   = NEON_Q | NEONScalar | NEON_FACGE,
+  NEON_FACGT_scalar   = NEON_Q | NEONScalar | NEON_FACGT,
+  NEON_FCMEQ_scalar   = NEON_Q | NEONScalar | NEON_FCMEQ,
+  NEON_FCMGE_scalar   = NEON_Q | NEONScalar | NEON_FCMGE,
+  NEON_FCMGT_scalar   = NEON_Q | NEONScalar | NEON_FCMGT,
+  NEON_FMULX_scalar   = NEON_Q | NEONScalar | NEON_FMULX,
+  NEON_FRECPS_scalar  = NEON_Q | NEONScalar | NEON_FRECPS,
+  NEON_FRSQRTS_scalar = NEON_Q | NEONScalar | NEON_FRSQRTS,
+  NEON_FABD_scalar    = NEON_Q | NEONScalar | NEON_FABD
+};
+
+// NEON scalar instructions with three different-type operands.
+enum NEONScalar3DiffOp {
+  NEONScalar3DiffFixed = 0x5E200000,
+  NEONScalar3DiffFMask = 0xDF200C00,
+  NEONScalar3DiffMask  = NEON_Q | NEONScalar | NEON3DifferentMask,
+  NEON_SQDMLAL_scalar  = NEON_Q | NEONScalar | NEON_SQDMLAL,
+  NEON_SQDMLSL_scalar  = NEON_Q | NEONScalar | NEON_SQDMLSL,
+  NEON_SQDMULL_scalar  = NEON_Q | NEONScalar | NEON_SQDMULL
+};
+
+// NEON scalar instructions with indexed element operand.
+enum NEONScalarByIndexedElementOp {
+  NEONScalarByIndexedElementFixed = 0x5F000000,
+  NEONScalarByIndexedElementFMask = 0xDF000400,
+  NEONScalarByIndexedElementMask  = 0xFF00F400,
+  NEON_SQDMLAL_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMLAL_byelement,
+  NEON_SQDMLSL_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMLSL_byelement,
+  NEON_SQDMULL_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMULL_byelement,
+  NEON_SQDMULH_byelement_scalar  = NEON_Q | NEONScalar | NEON_SQDMULH_byelement,
+  NEON_SQRDMULH_byelement_scalar
+    = NEON_Q | NEONScalar | NEON_SQRDMULH_byelement,
+
+  // Floating point instructions.
+  NEONScalarByIndexedElementFPFixed
+    = NEONScalarByIndexedElementFixed | 0x00800000,
+  NEONScalarByIndexedElementFPMask
+    = NEONScalarByIndexedElementMask | 0x00800000,
+  NEON_FMLA_byelement_scalar  = NEON_Q | NEONScalar | NEON_FMLA_byelement,
+  NEON_FMLS_byelement_scalar  = NEON_Q | NEONScalar | NEON_FMLS_byelement,
+  NEON_FMUL_byelement_scalar  = NEON_Q | NEONScalar | NEON_FMUL_byelement,
+  NEON_FMULX_byelement_scalar = NEON_Q | NEONScalar | NEON_FMULX_byelement
+};
+
+// NEON scalar register copy.
+enum NEONScalarCopyOp {
+  NEONScalarCopyFixed = 0x5E000400,
+  NEONScalarCopyFMask = 0xDFE08400,
+  NEONScalarCopyMask  = 0xFFE0FC00,
+  NEON_DUP_ELEMENT_scalar = NEON_Q | NEONScalar | NEON_DUP_ELEMENT
+};
+
+// NEON scalar pairwise instructions.
+enum NEONScalarPairwiseOp {
+  NEONScalarPairwiseFixed = 0x5E300800,
+  NEONScalarPairwiseFMask = 0xDF3E0C00,
+  NEONScalarPairwiseMask  = 0xFFB1F800,
+  NEON_ADDP_scalar    = NEONScalarPairwiseFixed | 0x0081B000,
+  NEON_FMAXNMP_scalar = NEONScalarPairwiseFixed | 0x2000C000,
+  NEON_FMINNMP_scalar = NEONScalarPairwiseFixed | 0x2080C000,
+  NEON_FADDP_scalar   = NEONScalarPairwiseFixed | 0x2000D000,
+  NEON_FMAXP_scalar   = NEONScalarPairwiseFixed | 0x2000F000,
+  NEON_FMINP_scalar   = NEONScalarPairwiseFixed | 0x2080F000
+};
+
+// NEON scalar shift immediate.
+enum NEONScalarShiftImmediateOp {
+  NEONScalarShiftImmediateFixed = 0x5F000400,
+  NEONScalarShiftImmediateFMask = 0xDF800400,
+  NEONScalarShiftImmediateMask  = 0xFF80FC00,
+  NEON_SHL_scalar  = NEON_Q | NEONScalar | NEON_SHL,
+  NEON_SLI_scalar  = NEON_Q | NEONScalar | NEON_SLI,
+  NEON_SRI_scalar  = NEON_Q | NEONScalar | NEON_SRI,
+  NEON_SSHR_scalar = NEON_Q | NEONScalar | NEON_SSHR,
+  NEON_USHR_scalar = NEON_Q | NEONScalar | NEON_USHR,
+  NEON_SRSHR_scalar = NEON_Q | NEONScalar | NEON_SRSHR,
+  NEON_URSHR_scalar = NEON_Q | NEONScalar | NEON_URSHR,
+  NEON_SSRA_scalar = NEON_Q | NEONScalar | NEON_SSRA,
+  NEON_USRA_scalar = NEON_Q | NEONScalar | NEON_USRA,
+  NEON_SRSRA_scalar = NEON_Q | NEONScalar | NEON_SRSRA,
+  NEON_URSRA_scalar = NEON_Q | NEONScalar | NEON_URSRA,
+  NEON_UQSHRN_scalar = NEON_Q | NEONScalar | NEON_UQSHRN,
+  NEON_UQRSHRN_scalar = NEON_Q | NEONScalar | NEON_UQRSHRN,
+  NEON_SQSHRN_scalar = NEON_Q | NEONScalar | NEON_SQSHRN,
+  NEON_SQRSHRN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRN,
+  NEON_SQSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQSHRUN,
+  NEON_SQRSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRUN,
+  NEON_SQSHLU_scalar = NEON_Q | NEONScalar | NEON_SQSHLU,
+  NEON_SQSHL_imm_scalar  = NEON_Q | NEONScalar | NEON_SQSHL_imm,
+  NEON_UQSHL_imm_scalar  = NEON_Q | NEONScalar | NEON_UQSHL_imm,
+  NEON_SCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_SCVTF_imm,
+  NEON_UCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_UCVTF_imm,
+  NEON_FCVTZS_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZS_imm,
+  NEON_FCVTZU_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZU_imm
+};
+
+// Unimplemented and unallocated instructions. These are defined to make fixed
+// bit assertion easier.
+enum UnimplementedOp {
+  UnimplementedFixed = 0x00000000,
+  UnimplementedFMask = 0x00000000
+};
+
+enum UnallocatedOp {
+  UnallocatedFixed = 0x00000000,
+  UnallocatedFMask = 0x00000000
+};
+
+}  // namespace vixl
+
+#endif  // VIXL_A64_CONSTANTS_A64_H_
diff --git a/disas/libvixl/vixl/a64/cpu-a64.h b/disas/libvixl/vixl/a64/cpu-a64.h
new file mode 100644
index 0000000..cdf09a6
--- /dev/null
+++ b/disas/libvixl/vixl/a64/cpu-a64.h
@@ -0,0 +1,83 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_CPU_A64_H
+#define VIXL_CPU_A64_H
+
+#include "vixl/globals.h"
+#include "vixl/a64/instructions-a64.h"
+
+namespace vixl {
+
+class CPU {
+ public:
+  // Initialise CPU support.
+  static void SetUp();
+
+  // Ensures the data at a given address and with a given size is the same for
+  // the I and D caches. I and D caches are not automatically coherent on ARM
+  // so this operation is required before any dynamically generated code can
+  // safely run.
+  static void EnsureIAndDCacheCoherency(void *address, size_t length);
+
+  // Handle tagged pointers.
+  template <typename T>
+  static T SetPointerTag(T pointer, uint64_t tag) {
+    VIXL_ASSERT(is_uintn(kAddressTagWidth, tag));
+
+    // Use C-style casts to get static_cast behaviour for integral types (T),
+    // and reinterpret_cast behaviour for other types.
+
+    uint64_t raw = (uint64_t)pointer;
+    VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
+
+    raw = (raw & ~kAddressTagMask) | (tag << kAddressTagOffset);
+    return (T)raw;
+  }
+
+  template <typename T>
+  static uint64_t GetPointerTag(T pointer) {
+    // Use C-style casts to get static_cast behaviour for integral types (T),
+    // and reinterpret_cast behaviour for other types.
+
+    uint64_t raw = (uint64_t)pointer;
+    VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
+
+    return (raw & kAddressTagMask) >> kAddressTagOffset;
+  }
+
+ private:
+  // Return the content of the cache type register.
+  static uint32_t GetCacheType();
+
+  // I and D cache line size in bytes.
+  static unsigned icache_line_size_;
+  static unsigned dcache_line_size_;
+};
+
+}  // namespace vixl
+
+#endif  // VIXL_CPU_A64_H
diff --git a/disas/libvixl/vixl/a64/decoder-a64.cc b/disas/libvixl/vixl/a64/decoder-a64.cc
new file mode 100644
index 0000000..5ba2d3c
--- /dev/null
+++ b/disas/libvixl/vixl/a64/decoder-a64.cc
@@ -0,0 +1,877 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "vixl/globals.h"
+#include "vixl/utils.h"
+#include "vixl/a64/decoder-a64.h"
+
+namespace vixl {
+
+void Decoder::DecodeInstruction(const Instruction *instr) {
+  if (instr->Bits(28, 27) == 0) {
+    VisitUnallocated(instr);
+  } else {
+    switch (instr->Bits(27, 24)) {
+      // 0:   PC relative addressing.
+      case 0x0: DecodePCRelAddressing(instr); break;
+
+      // 1:   Add/sub immediate.
+      case 0x1: DecodeAddSubImmediate(instr); break;
+
+      // A:   Logical shifted register.
+      //      Add/sub with carry.
+      //      Conditional compare register.
+      //      Conditional compare immediate.
+      //      Conditional select.
+      //      Data processing 1 source.
+      //      Data processing 2 source.
+      // B:   Add/sub shifted register.
+      //      Add/sub extended register.
+      //      Data processing 3 source.
+      case 0xA:
+      case 0xB: DecodeDataProcessing(instr); break;
+
+      // 2:   Logical immediate.
+      //      Move wide immediate.
+      case 0x2: DecodeLogical(instr); break;
+
+      // 3:   Bitfield.
+      //      Extract.
+      case 0x3: DecodeBitfieldExtract(instr); break;
+
+      // 4:   Unconditional branch immediate.
+      //      Exception generation.
+      //      Compare and branch immediate.
+      // 5:   Compare and branch immediate.
+      //      Conditional branch.
+      //      System.
+      // 6,7: Unconditional branch.
+      //      Test and branch immediate.
+      case 0x4:
+      case 0x5:
+      case 0x6:
+      case 0x7: DecodeBranchSystemException(instr); break;
+
+      // 8,9: Load/store register pair post-index.
+      //      Load register literal.
+      //      Load/store register unscaled immediate.
+      //      Load/store register immediate post-index.
+      //      Load/store register immediate pre-index.
+      //      Load/store register offset.
+      //      Load/store exclusive.
+      // C,D: Load/store register pair offset.
+      //      Load/store register pair pre-index.
+      //      Load/store register unsigned immediate.
+      //      Advanced SIMD.
+      case 0x8:
+      case 0x9:
+      case 0xC:
+      case 0xD: DecodeLoadStore(instr); break;
+
+      // E:   FP fixed point conversion.
+      //      FP integer conversion.
+      //      FP data processing 1 source.
+      //      FP compare.
+      //      FP immediate.
+      //      FP data processing 2 source.
+      //      FP conditional compare.
+      //      FP conditional select.
+      //      Advanced SIMD.
+      // F:   FP data processing 3 source.
+      //      Advanced SIMD.
+      case 0xE:
+      case 0xF: DecodeFP(instr); break;
+    }
+  }
+}
+
+void Decoder::AppendVisitor(DecoderVisitor* new_visitor) {
+  visitors_.push_back(new_visitor);
+}
+
+
+void Decoder::PrependVisitor(DecoderVisitor* new_visitor) {
+  visitors_.push_front(new_visitor);
+}
+
+
+void Decoder::InsertVisitorBefore(DecoderVisitor* new_visitor,
+                                  DecoderVisitor* registered_visitor) {
+  std::list<DecoderVisitor*>::iterator it;
+  for (it = visitors_.begin(); it != visitors_.end(); it++) {
+    if (*it == registered_visitor) {
+      visitors_.insert(it, new_visitor);
+      return;
+    }
+  }
+  // We reached the end of the list. The last element must be
+  // registered_visitor.
+  VIXL_ASSERT(*it == registered_visitor);
+  visitors_.insert(it, new_visitor);
+}
+
+
+void Decoder::InsertVisitorAfter(DecoderVisitor* new_visitor,
+                                 DecoderVisitor* registered_visitor) {
+  std::list<DecoderVisitor*>::iterator it;
+  for (it = visitors_.begin(); it != visitors_.end(); it++) {
+    if (*it == registered_visitor) {
+      it++;
+      visitors_.insert(it, new_visitor);
+      return;
+    }
+  }
+  // We reached the end of the list. The last element must be
+  // registered_visitor.
+  VIXL_ASSERT(*it == registered_visitor);
+  visitors_.push_back(new_visitor);
+}
+
+
+void Decoder::RemoveVisitor(DecoderVisitor* visitor) {
+  visitors_.remove(visitor);
+}
+
+
+void Decoder::DecodePCRelAddressing(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(27, 24) == 0x0);
+  // We know bit 28 is set, as <b28:b27> = 0 is filtered out at the top level
+  // decode.
+  VIXL_ASSERT(instr->Bit(28) == 0x1);
+  VisitPCRelAddressing(instr);
+}
+
+
+void Decoder::DecodeBranchSystemException(const Instruction* instr) {
+  VIXL_ASSERT((instr->Bits(27, 24) == 0x4) ||
+              (instr->Bits(27, 24) == 0x5) ||
+              (instr->Bits(27, 24) == 0x6) ||
+              (instr->Bits(27, 24) == 0x7) );
+
+  switch (instr->Bits(31, 29)) {
+    case 0:
+    case 4: {
+      VisitUnconditionalBranch(instr);
+      break;
+    }
+    case 1:
+    case 5: {
+      if (instr->Bit(25) == 0) {
+        VisitCompareBranch(instr);
+      } else {
+        VisitTestBranch(instr);
+      }
+      break;
+    }
+    case 2: {
+      if (instr->Bit(25) == 0) {
+        if ((instr->Bit(24) == 0x1) ||
+            (instr->Mask(0x01000010) == 0x00000010)) {
+          VisitUnallocated(instr);
+        } else {
+          VisitConditionalBranch(instr);
+        }
+      } else {
+        VisitUnallocated(instr);
+      }
+      break;
+    }
+    case 6: {
+      if (instr->Bit(25) == 0) {
+        if (instr->Bit(24) == 0) {
+          if ((instr->Bits(4, 2) != 0) ||
+              (instr->Mask(0x00E0001D) == 0x00200001) ||
+              (instr->Mask(0x00E0001D) == 0x00400001) ||
+              (instr->Mask(0x00E0001E) == 0x00200002) ||
+              (instr->Mask(0x00E0001E) == 0x00400002) ||
+              (instr->Mask(0x00E0001C) == 0x00600000) ||
+              (instr->Mask(0x00E0001C) == 0x00800000) ||
+              (instr->Mask(0x00E0001F) == 0x00A00000) ||
+              (instr->Mask(0x00C0001C) == 0x00C00000)) {
+            VisitUnallocated(instr);
+          } else {
+            VisitException(instr);
+          }
+        } else {
+          if (instr->Bits(23, 22) == 0) {
+            const Instr masked_003FF0E0 = instr->Mask(0x003FF0E0);
+            if ((instr->Bits(21, 19) == 0x4) ||
+                (masked_003FF0E0 == 0x00033000) ||
+                (masked_003FF0E0 == 0x003FF020) ||
+                (masked_003FF0E0 == 0x003FF060) ||
+                (masked_003FF0E0 == 0x003FF0E0) ||
+                (instr->Mask(0x00388000) == 0x00008000) ||
+                (instr->Mask(0x0038E000) == 0x00000000) ||
+                (instr->Mask(0x0039E000) == 0x00002000) ||
+                (instr->Mask(0x003AE000) == 0x00002000) ||
+                (instr->Mask(0x003CE000) == 0x00042000) ||
+                (instr->Mask(0x003FFFC0) == 0x000320C0) ||
+                (instr->Mask(0x003FF100) == 0x00032100) ||
+                (instr->Mask(0x003FF200) == 0x00032200) ||
+                (instr->Mask(0x003FF400) == 0x00032400) ||
+                (instr->Mask(0x003FF800) == 0x00032800) ||
+                (instr->Mask(0x0038F000) == 0x00005000) ||
+                (instr->Mask(0x0038E000) == 0x00006000)) {
+              VisitUnallocated(instr);
+            } else {
+              VisitSystem(instr);
+            }
+          } else {
+            VisitUnallocated(instr);
+          }
+        }
+      } else {
+        if ((instr->Bit(24) == 0x1) ||
+            (instr->Bits(20, 16) != 0x1F) ||
+            (instr->Bits(15, 10) != 0) ||
+            (instr->Bits(4, 0) != 0) ||
+            (instr->Bits(24, 21) == 0x3) ||
+            (instr->Bits(24, 22) == 0x3)) {
+          VisitUnallocated(instr);
+        } else {
+          VisitUnconditionalBranchToRegister(instr);
+        }
+      }
+      break;
+    }
+    case 3:
+    case 7: {
+      VisitUnallocated(instr);
+      break;
+    }
+  }
+}
+
+
+void Decoder::DecodeLoadStore(const Instruction* instr) {
+  VIXL_ASSERT((instr->Bits(27, 24) == 0x8) ||
+              (instr->Bits(27, 24) == 0x9) ||
+              (instr->Bits(27, 24) == 0xC) ||
+              (instr->Bits(27, 24) == 0xD) );
+  // TODO(all): rearrange the tree to integrate this branch.
+  if ((instr->Bit(28) == 0) && (instr->Bit(29) == 0) && (instr->Bit(26) == 1)) {
+    DecodeNEONLoadStore(instr);
+    return;
+  }
+
+  if (instr->Bit(24) == 0) {
+    if (instr->Bit(28) == 0) {
+      if (instr->Bit(29) == 0) {
+        if (instr->Bit(26) == 0) {
+          VisitLoadStoreExclusive(instr);
+        } else {
+          VIXL_UNREACHABLE();
+        }
+      } else {
+        if ((instr->Bits(31, 30) == 0x3) ||
+            (instr->Mask(0xC4400000) == 0x40000000)) {
+          VisitUnallocated(instr);
+        } else {
+          if (instr->Bit(23) == 0) {
+            if (instr->Mask(0xC4400000) == 0xC0400000) {
+              VisitUnallocated(instr);
+            } else {
+              VisitLoadStorePairNonTemporal(instr);
+            }
+          } else {
+            VisitLoadStorePairPostIndex(instr);
+          }
+        }
+      }
+    } else {
+      if (instr->Bit(29) == 0) {
+        if (instr->Mask(0xC4000000) == 0xC4000000) {
+          VisitUnallocated(instr);
+        } else {
+          VisitLoadLiteral(instr);
+        }
+      } else {
+        if ((instr->Mask(0x84C00000) == 0x80C00000) ||
+            (instr->Mask(0x44800000) == 0x44800000) ||
+            (instr->Mask(0x84800000) == 0x84800000)) {
+          VisitUnallocated(instr);
+        } else {
+          if (instr->Bit(21) == 0) {
+            switch (instr->Bits(11, 10)) {
+              case 0: {
+                VisitLoadStoreUnscaledOffset(instr);
+                break;
+              }
+              case 1: {
+                if (instr->Mask(0xC4C00000) == 0xC0800000) {
+                  VisitUnallocated(instr);
+                } else {
+                  VisitLoadStorePostIndex(instr);
+                }
+                break;
+              }
+              case 2: {
+                // TODO: VisitLoadStoreRegisterOffsetUnpriv.
+                VisitUnimplemented(instr);
+                break;
+              }
+              case 3: {
+                if (instr->Mask(0xC4C00000) == 0xC0800000) {
+                  VisitUnallocated(instr);
+                } else {
+                  VisitLoadStorePreIndex(instr);
+                }
+                break;
+              }
+            }
+          } else {
+            if (instr->Bits(11, 10) == 0x2) {
+              if (instr->Bit(14) == 0) {
+                VisitUnallocated(instr);
+              } else {
+                VisitLoadStoreRegisterOffset(instr);
+              }
+            } else {
+              VisitUnallocated(instr);
+            }
+          }
+        }
+      }
+    }
+  } else {
+    if (instr->Bit(28) == 0) {
+      if (instr->Bit(29) == 0) {
+        VisitUnallocated(instr);
+      } else {
+        if ((instr->Bits(31, 30) == 0x3) ||
+            (instr->Mask(0xC4400000) == 0x40000000)) {
+          VisitUnallocated(instr);
+        } else {
+          if (instr->Bit(23) == 0) {
+            VisitLoadStorePairOffset(instr);
+          } else {
+            VisitLoadStorePairPreIndex(instr);
+          }
+        }
+      }
+    } else {
+      if (instr->Bit(29) == 0) {
+        VisitUnallocated(instr);
+      } else {
+        if ((instr->Mask(0x84C00000) == 0x80C00000) ||
+            (instr->Mask(0x44800000) == 0x44800000) ||
+            (instr->Mask(0x84800000) == 0x84800000)) {
+          VisitUnallocated(instr);
+        } else {
+          VisitLoadStoreUnsignedOffset(instr);
+        }
+      }
+    }
+  }
+}
+
+
+void Decoder::DecodeLogical(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(27, 24) == 0x2);
+
+  if (instr->Mask(0x80400000) == 0x00400000) {
+    VisitUnallocated(instr);
+  } else {
+    if (instr->Bit(23) == 0) {
+      VisitLogicalImmediate(instr);
+    } else {
+      if (instr->Bits(30, 29) == 0x1) {
+        VisitUnallocated(instr);
+      } else {
+        VisitMoveWideImmediate(instr);
+      }
+    }
+  }
+}
+
+
+void Decoder::DecodeBitfieldExtract(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(27, 24) == 0x3);
+
+  if ((instr->Mask(0x80400000) == 0x80000000) ||
+      (instr->Mask(0x80400000) == 0x00400000) ||
+      (instr->Mask(0x80008000) == 0x00008000)) {
+    VisitUnallocated(instr);
+  } else if (instr->Bit(23) == 0) {
+    if ((instr->Mask(0x80200000) == 0x00200000) ||
+        (instr->Mask(0x60000000) == 0x60000000)) {
+      VisitUnallocated(instr);
+    } else {
+      VisitBitfield(instr);
+    }
+  } else {
+    if ((instr->Mask(0x60200000) == 0x00200000) ||
+        (instr->Mask(0x60000000) != 0x00000000)) {
+      VisitUnallocated(instr);
+    } else {
+      VisitExtract(instr);
+    }
+  }
+}
+
+
+void Decoder::DecodeAddSubImmediate(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(27, 24) == 0x1);
+  if (instr->Bit(23) == 1) {
+    VisitUnallocated(instr);
+  } else {
+    VisitAddSubImmediate(instr);
+  }
+}
+
+
+void Decoder::DecodeDataProcessing(const Instruction* instr) {
+  VIXL_ASSERT((instr->Bits(27, 24) == 0xA) ||
+              (instr->Bits(27, 24) == 0xB));
+
+  if (instr->Bit(24) == 0) {
+    if (instr->Bit(28) == 0) {
+      if (instr->Mask(0x80008000) == 0x00008000) {
+        VisitUnallocated(instr);
+      } else {
+        VisitLogicalShifted(instr);
+      }
+    } else {
+      switch (instr->Bits(23, 21)) {
+        case 0: {
+          if (instr->Mask(0x0000FC00) != 0) {
+            VisitUnallocated(instr);
+          } else {
+            VisitAddSubWithCarry(instr);
+          }
+          break;
+        }
+        case 2: {
+          if ((instr->Bit(29) == 0) ||
+              (instr->Mask(0x00000410) != 0)) {
+            VisitUnallocated(instr);
+          } else {
+            if (instr->Bit(11) == 0) {
+              VisitConditionalCompareRegister(instr);
+            } else {
+              VisitConditionalCompareImmediate(instr);
+            }
+          }
+          break;
+        }
+        case 4: {
+          if (instr->Mask(0x20000800) != 0x00000000) {
+            VisitUnallocated(instr);
+          } else {
+            VisitConditionalSelect(instr);
+          }
+          break;
+        }
+        case 6: {
+          if (instr->Bit(29) == 0x1) {
+            VisitUnallocated(instr);
+            VIXL_FALLTHROUGH();
+          } else {
+            if (instr->Bit(30) == 0) {
+              if ((instr->Bit(15) == 0x1) ||
+                  (instr->Bits(15, 11) == 0) ||
+                  (instr->Bits(15, 12) == 0x1) ||
+                  (instr->Bits(15, 12) == 0x3) ||
+                  (instr->Bits(15, 13) == 0x3) ||
+                  (instr->Mask(0x8000EC00) == 0x00004C00) ||
+                  (instr->Mask(0x8000E800) == 0x80004000) ||
+                  (instr->Mask(0x8000E400) == 0x80004000)) {
+                VisitUnallocated(instr);
+              } else {
+                VisitDataProcessing2Source(instr);
+              }
+            } else {
+              if ((instr->Bit(13) == 1) ||
+                  (instr->Bits(20, 16) != 0) ||
+                  (instr->Bits(15, 14) != 0) ||
+                  (instr->Mask(0xA01FFC00) == 0x00000C00) ||
+                  (instr->Mask(0x201FF800) == 0x00001800)) {
+                VisitUnallocated(instr);
+              } else {
+                VisitDataProcessing1Source(instr);
+              }
+            }
+            break;
+          }
+        }
+        case 1:
+        case 3:
+        case 5:
+        case 7: VisitUnallocated(instr); break;
+      }
+    }
+  } else {
+    if (instr->Bit(28) == 0) {
+     if (instr->Bit(21) == 0) {
+        if ((instr->Bits(23, 22) == 0x3) ||
+            (instr->Mask(0x80008000) == 0x00008000)) {
+          VisitUnallocated(instr);
+        } else {
+          VisitAddSubShifted(instr);
+        }
+      } else {
+        if ((instr->Mask(0x00C00000) != 0x00000000) ||
+            (instr->Mask(0x00001400) == 0x00001400) ||
+            (instr->Mask(0x00001800) == 0x00001800)) {
+          VisitUnallocated(instr);
+        } else {
+          VisitAddSubExtended(instr);
+        }
+      }
+    } else {
+      if ((instr->Bit(30) == 0x1) ||
+          (instr->Bits(30, 29) == 0x1) ||
+          (instr->Mask(0xE0600000) == 0x00200000) ||
+          (instr->Mask(0xE0608000) == 0x00400000) ||
+          (instr->Mask(0x60608000) == 0x00408000) ||
+          (instr->Mask(0x60E00000) == 0x00E00000) ||
+          (instr->Mask(0x60E00000) == 0x00800000) ||
+          (instr->Mask(0x60E00000) == 0x00600000)) {
+        VisitUnallocated(instr);
+      } else {
+        VisitDataProcessing3Source(instr);
+      }
+    }
+  }
+}
+
+
+void Decoder::DecodeFP(const Instruction* instr) {
+  VIXL_ASSERT((instr->Bits(27, 24) == 0xE) ||
+              (instr->Bits(27, 24) == 0xF));
+  if (instr->Bit(28) == 0) {
+    DecodeNEONVectorDataProcessing(instr);
+  } else {
+    if (instr->Bits(31, 30) == 0x3) {
+      VisitUnallocated(instr);
+    } else if (instr->Bits(31, 30) == 0x1) {
+      DecodeNEONScalarDataProcessing(instr);
+    } else {
+      if (instr->Bit(29) == 0) {
+        if (instr->Bit(24) == 0) {
+          if (instr->Bit(21) == 0) {
+            if ((instr->Bit(23) == 1) ||
+                (instr->Bit(18) == 1) ||
+                (instr->Mask(0x80008000) == 0x00000000) ||
+                (instr->Mask(0x000E0000) == 0x00000000) ||
+                (instr->Mask(0x000E0000) == 0x000A0000) ||
+                (instr->Mask(0x00160000) == 0x00000000) ||
+                (instr->Mask(0x00160000) == 0x00120000)) {
+              VisitUnallocated(instr);
+            } else {
+              VisitFPFixedPointConvert(instr);
+            }
+          } else {
+            if (instr->Bits(15, 10) == 32) {
+              VisitUnallocated(instr);
+            } else if (instr->Bits(15, 10) == 0) {
+              if ((instr->Bits(23, 22) == 0x3) ||
+                  (instr->Mask(0x000E0000) == 0x000A0000) ||
+                  (instr->Mask(0x000E0000) == 0x000C0000) ||
+                  (instr->Mask(0x00160000) == 0x00120000) ||
+                  (instr->Mask(0x00160000) == 0x00140000) ||
+                  (instr->Mask(0x20C40000) == 0x00800000) ||
+                  (instr->Mask(0x20C60000) == 0x00840000) ||
+                  (instr->Mask(0xA0C60000) == 0x80060000) ||
+                  (instr->Mask(0xA0C60000) == 0x00860000) ||
+                  (instr->Mask(0xA0C60000) == 0x00460000) ||
+                  (instr->Mask(0xA0CE0000) == 0x80860000) ||
+                  (instr->Mask(0xA0CE0000) == 0x804E0000) ||
+                  (instr->Mask(0xA0CE0000) == 0x000E0000) ||
+                  (instr->Mask(0xA0D60000) == 0x00160000) ||
+                  (instr->Mask(0xA0D60000) == 0x80560000) ||
+                  (instr->Mask(0xA0D60000) == 0x80960000)) {
+                VisitUnallocated(instr);
+              } else {
+                VisitFPIntegerConvert(instr);
+              }
+            } else if (instr->Bits(14, 10) == 16) {
+              const Instr masked_A0DF8000 = instr->Mask(0xA0DF8000);
+              if ((instr->Mask(0x80180000) != 0) ||
+                  (masked_A0DF8000 == 0x00020000) ||
+                  (masked_A0DF8000 == 0x00030000) ||
+                  (masked_A0DF8000 == 0x00068000) ||
+                  (masked_A0DF8000 == 0x00428000) ||
+                  (masked_A0DF8000 == 0x00430000) ||
+                  (masked_A0DF8000 == 0x00468000) ||
+                  (instr->Mask(0xA0D80000) == 0x00800000) ||
+                  (instr->Mask(0xA0DE0000) == 0x00C00000) ||
+                  (instr->Mask(0xA0DF0000) == 0x00C30000) ||
+                  (instr->Mask(0xA0DC0000) == 0x00C40000)) {
+                VisitUnallocated(instr);
+              } else {
+                VisitFPDataProcessing1Source(instr);
+              }
+            } else if (instr->Bits(13, 10) == 8) {
+              if ((instr->Bits(15, 14) != 0) ||
+                  (instr->Bits(2, 0) != 0) ||
+                  (instr->Mask(0x80800000) != 0x00000000)) {
+                VisitUnallocated(instr);
+              } else {
+                VisitFPCompare(instr);
+              }
+            } else if (instr->Bits(12, 10) == 4) {
+              if ((instr->Bits(9, 5) != 0) ||
+                  (instr->Mask(0x80800000) != 0x00000000)) {
+                VisitUnallocated(instr);
+              } else {
+                VisitFPImmediate(instr);
+              }
+            } else {
+              if (instr->Mask(0x80800000) != 0x00000000) {
+                VisitUnallocated(instr);
+              } else {
+                switch (instr->Bits(11, 10)) {
+                  case 1: {
+                    VisitFPConditionalCompare(instr);
+                    break;
+                  }
+                  case 2: {
+                    if ((instr->Bits(15, 14) == 0x3) ||
+                        (instr->Mask(0x00009000) == 0x00009000) ||
+                        (instr->Mask(0x0000A000) == 0x0000A000)) {
+                      VisitUnallocated(instr);
+                    } else {
+                      VisitFPDataProcessing2Source(instr);
+                    }
+                    break;
+                  }
+                  case 3: {
+                    VisitFPConditionalSelect(instr);
+                    break;
+                  }
+                  default: VIXL_UNREACHABLE();
+                }
+              }
+            }
+          }
+        } else {
+          // Bit 30 == 1 has been handled earlier.
+          VIXL_ASSERT(instr->Bit(30) == 0);
+          if (instr->Mask(0xA0800000) != 0) {
+            VisitUnallocated(instr);
+          } else {
+            VisitFPDataProcessing3Source(instr);
+          }
+        }
+      } else {
+        VisitUnallocated(instr);
+      }
+    }
+  }
+}
+
+
+void Decoder::DecodeNEONLoadStore(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(29, 25) == 0x6);
+  if (instr->Bit(31) == 0) {
+    if ((instr->Bit(24) == 0) && (instr->Bit(21) == 1)) {
+      VisitUnallocated(instr);
+      return;
+    }
+
+    if (instr->Bit(23) == 0) {
+      if (instr->Bits(20, 16) == 0) {
+        if (instr->Bit(24) == 0) {
+          VisitNEONLoadStoreMultiStruct(instr);
+        } else {
+          VisitNEONLoadStoreSingleStruct(instr);
+        }
+      } else {
+        VisitUnallocated(instr);
+      }
+    } else {
+      if (instr->Bit(24) == 0) {
+        VisitNEONLoadStoreMultiStructPostIndex(instr);
+      } else {
+        VisitNEONLoadStoreSingleStructPostIndex(instr);
+      }
+    }
+  } else {
+    VisitUnallocated(instr);
+  }
+}
+
+
+void Decoder::DecodeNEONVectorDataProcessing(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(28, 25) == 0x7);
+  if (instr->Bit(31) == 0) {
+    if (instr->Bit(24) == 0) {
+      if (instr->Bit(21) == 0) {
+        if (instr->Bit(15) == 0) {
+          if (instr->Bit(10) == 0) {
+            if (instr->Bit(29) == 0) {
+              if (instr->Bit(11) == 0) {
+                VisitNEONTable(instr);
+              } else {
+                VisitNEONPerm(instr);
+              }
+            } else {
+              VisitNEONExtract(instr);
+            }
+          } else {
+            if (instr->Bits(23, 22) == 0) {
+              VisitNEONCopy(instr);
+            } else {
+              VisitUnallocated(instr);
+            }
+          }
+        } else {
+          VisitUnallocated(instr);
+        }
+      } else {
+        if (instr->Bit(10) == 0) {
+          if (instr->Bit(11) == 0) {
+            VisitNEON3Different(instr);
+          } else {
+            if (instr->Bits(18, 17) == 0) {
+              if (instr->Bit(20) == 0) {
+                if (instr->Bit(19) == 0) {
+                  VisitNEON2RegMisc(instr);
+                } else {
+                  if (instr->Bits(30, 29) == 0x2) {
+                    VisitCryptoAES(instr);
+                  } else {
+                    VisitUnallocated(instr);
+                  }
+                }
+              } else {
+                if (instr->Bit(19) == 0) {
+                  VisitNEONAcrossLanes(instr);
+                } else {
+                  VisitUnallocated(instr);
+                }
+              }
+            } else {
+              VisitUnallocated(instr);
+            }
+          }
+        } else {
+          VisitNEON3Same(instr);
+        }
+      }
+    } else {
+      if (instr->Bit(10) == 0) {
+        VisitNEONByIndexedElement(instr);
+      } else {
+        if (instr->Bit(23) == 0) {
+          if (instr->Bits(22, 19) == 0) {
+            VisitNEONModifiedImmediate(instr);
+          } else {
+            VisitNEONShiftImmediate(instr);
+          }
+        } else {
+          VisitUnallocated(instr);
+        }
+      }
+    }
+  } else {
+    VisitUnallocated(instr);
+  }
+}
+
+
+void Decoder::DecodeNEONScalarDataProcessing(const Instruction* instr) {
+  VIXL_ASSERT(instr->Bits(28, 25) == 0xF);
+  if (instr->Bit(24) == 0) {
+    if (instr->Bit(21) == 0) {
+      if (instr->Bit(15) == 0) {
+        if (instr->Bit(10) == 0) {
+          if (instr->Bit(29) == 0) {
+            if (instr->Bit(11) == 0) {
+              VisitCrypto3RegSHA(instr);
+            } else {
+              VisitUnallocated(instr);
+            }
+          } else {
+            VisitUnallocated(instr);
+          }
+        } else {
+          if (instr->Bits(23, 22) == 0) {
+            VisitNEONScalarCopy(instr);
+          } else {
+            VisitUnallocated(instr);
+          }
+        }
+      } else {
+        VisitUnallocated(instr);
+      }
+    } else {
+      if (instr->Bit(10) == 0) {
+        if (instr->Bit(11) == 0) {
+          VisitNEONScalar3Diff(instr);
+        } else {
+          if (instr->Bits(18, 17) == 0) {
+            if (instr->Bit(20) == 0) {
+              if (instr->Bit(19) == 0) {
+                VisitNEONScalar2RegMisc(instr);
+              } else {
+                if (instr->Bit(29) == 0) {
+                  VisitCrypto2RegSHA(instr);
+                } else {
+                  VisitUnallocated(instr);
+                }
+              }
+            } else {
+              if (instr->Bit(19) == 0) {
+                VisitNEONScalarPairwise(instr);
+              } else {
+                VisitUnallocated(instr);
+              }
+            }
+          } else {
+            VisitUnallocated(instr);
+          }
+        }
+      } else {
+        VisitNEONScalar3Same(instr);
+      }
+    }
+  } else {
+    if (instr->Bit(10) == 0) {
+      VisitNEONScalarByIndexedElement(instr);
+    } else {
+      if (instr->Bit(23) == 0) {
+        VisitNEONScalarShiftImmediate(instr);
+      } else {
+        VisitUnallocated(instr);
+      }
+    }
+  }
+}
+
+
+#define DEFINE_VISITOR_CALLERS(A)                                              \
+  void Decoder::Visit##A(const Instruction *instr) {                           \
+    VIXL_ASSERT(instr->Mask(A##FMask) == A##Fixed);                            \
+    std::list<DecoderVisitor*>::iterator it;                                   \
+    for (it = visitors_.begin(); it != visitors_.end(); it++) {                \
+      (*it)->Visit##A(instr);                                                  \
+    }                                                                          \
+  }
+VISITOR_LIST(DEFINE_VISITOR_CALLERS)
+#undef DEFINE_VISITOR_CALLERS
+}  // namespace vixl
diff --git a/disas/libvixl/vixl/a64/decoder-a64.h b/disas/libvixl/vixl/a64/decoder-a64.h
new file mode 100644
index 0000000..b3f04f6
--- /dev/null
+++ b/disas/libvixl/vixl/a64/decoder-a64.h
@@ -0,0 +1,275 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_DECODER_A64_H_
+#define VIXL_A64_DECODER_A64_H_
+
+#include <list>
+
+#include "vixl/globals.h"
+#include "vixl/a64/instructions-a64.h"
+
+
+// List macro containing all visitors needed by the decoder class.
+
+#define VISITOR_LIST_THAT_RETURN(V) \
+  V(PCRelAddressing)                \
+  V(AddSubImmediate)                \
+  V(LogicalImmediate)               \
+  V(MoveWideImmediate)              \
+  V(Bitfield)                       \
+  V(Extract)                        \
+  V(UnconditionalBranch)            \
+  V(UnconditionalBranchToRegister)  \
+  V(CompareBranch)                  \
+  V(TestBranch)                     \
+  V(ConditionalBranch)              \
+  V(System)                         \
+  V(Exception)                      \
+  V(LoadStorePairPostIndex)         \
+  V(LoadStorePairOffset)            \
+  V(LoadStorePairPreIndex)          \
+  V(LoadStorePairNonTemporal)       \
+  V(LoadLiteral)                    \
+  V(LoadStoreUnscaledOffset)        \
+  V(LoadStorePostIndex)             \
+  V(LoadStorePreIndex)              \
+  V(LoadStoreRegisterOffset)        \
+  V(LoadStoreUnsignedOffset)        \
+  V(LoadStoreExclusive)             \
+  V(LogicalShifted)                 \
+  V(AddSubShifted)                  \
+  V(AddSubExtended)                 \
+  V(AddSubWithCarry)                \
+  V(ConditionalCompareRegister)     \
+  V(ConditionalCompareImmediate)    \
+  V(ConditionalSelect)              \
+  V(DataProcessing1Source)          \
+  V(DataProcessing2Source)          \
+  V(DataProcessing3Source)          \
+  V(FPCompare)                      \
+  V(FPConditionalCompare)           \
+  V(FPConditionalSelect)            \
+  V(FPImmediate)                    \
+  V(FPDataProcessing1Source)        \
+  V(FPDataProcessing2Source)        \
+  V(FPDataProcessing3Source)        \
+  V(FPIntegerConvert)               \
+  V(FPFixedPointConvert)            \
+  V(Crypto2RegSHA)                  \
+  V(Crypto3RegSHA)                  \
+  V(CryptoAES)                      \
+  V(NEON2RegMisc)                   \
+  V(NEON3Different)                 \
+  V(NEON3Same)                      \
+  V(NEONAcrossLanes)                \
+  V(NEONByIndexedElement)           \
+  V(NEONCopy)                       \
+  V(NEONExtract)                    \
+  V(NEONLoadStoreMultiStruct)       \
+  V(NEONLoadStoreMultiStructPostIndex)  \
+  V(NEONLoadStoreSingleStruct)      \
+  V(NEONLoadStoreSingleStructPostIndex) \
+  V(NEONModifiedImmediate)          \
+  V(NEONScalar2RegMisc)             \
+  V(NEONScalar3Diff)                \
+  V(NEONScalar3Same)                \
+  V(NEONScalarByIndexedElement)     \
+  V(NEONScalarCopy)                 \
+  V(NEONScalarPairwise)             \
+  V(NEONScalarShiftImmediate)       \
+  V(NEONShiftImmediate)             \
+  V(NEONTable)                      \
+  V(NEONPerm)                       \
+
+#define VISITOR_LIST_THAT_DONT_RETURN(V)  \
+  V(Unallocated)                          \
+  V(Unimplemented)                        \
+
+#define VISITOR_LIST(V)             \
+  VISITOR_LIST_THAT_RETURN(V)       \
+  VISITOR_LIST_THAT_DONT_RETURN(V)  \
+
+namespace vixl {
+
+// The Visitor interface. Disassembler and simulator (and other tools)
+// must provide implementations for all of these functions.
+class DecoderVisitor {
+ public:
+  enum VisitorConstness {
+    kConstVisitor,
+    kNonConstVisitor
+  };
+  explicit DecoderVisitor(VisitorConstness constness = kConstVisitor)
+      : constness_(constness) {}
+
+  virtual ~DecoderVisitor() {}
+
+  #define DECLARE(A) virtual void Visit##A(const Instruction* instr) = 0;
+  VISITOR_LIST(DECLARE)
+  #undef DECLARE
+
+  bool IsConstVisitor() const { return constness_ == kConstVisitor; }
+  Instruction* MutableInstruction(const Instruction* instr) {
+    VIXL_ASSERT(!IsConstVisitor());
+    return const_cast<Instruction*>(instr);
+  }
+
+ private:
+  const VisitorConstness constness_;
+};
+
+
+class Decoder {
+ public:
+  Decoder() {}
+
+  // Top-level wrappers around the actual decoding function.
+  void Decode(const Instruction* instr) {
+    std::list<DecoderVisitor*>::iterator it;
+    for (it = visitors_.begin(); it != visitors_.end(); it++) {
+      VIXL_ASSERT((*it)->IsConstVisitor());
+    }
+    DecodeInstruction(instr);
+  }
+  void Decode(Instruction* instr) {
+    DecodeInstruction(const_cast<const Instruction*>(instr));
+  }
+
+  // Register a new visitor class with the decoder.
+  // Decode() will call the corresponding visitor method from all registered
+  // visitor classes when decoding reaches the leaf node of the instruction
+  // decode tree.
+  // Visitors are called in order.
+  // A visitor can be registered multiple times.
+  //
+  //   d.AppendVisitor(V1);
+  //   d.AppendVisitor(V2);
+  //   d.PrependVisitor(V2);
+  //   d.AppendVisitor(V3);
+  //
+  //   d.Decode(i);
+  //
+  // will call in order visitor methods in V2, V1, V2, V3.
+  void AppendVisitor(DecoderVisitor* visitor);
+  void PrependVisitor(DecoderVisitor* visitor);
+  // These helpers register `new_visitor` before or after the first instance of
+  // `registered_visiter` in the list.
+  // So if
+  //   V1, V2, V1, V2
+  // are registered in this order in the decoder, calls to
+  //   d.InsertVisitorAfter(V3, V1);
+  //   d.InsertVisitorBefore(V4, V2);
+  // will yield the order
+  //   V1, V3, V4, V2, V1, V2
+  //
+  // For more complex modifications of the order of registered visitors, one can
+  // directly access and modify the list of visitors via the `visitors()'
+  // accessor.
+  void InsertVisitorBefore(DecoderVisitor* new_visitor,
+                           DecoderVisitor* registered_visitor);
+  void InsertVisitorAfter(DecoderVisitor* new_visitor,
+                          DecoderVisitor* registered_visitor);
+
+  // Remove all instances of a previously registered visitor class from the list
+  // of visitors stored by the decoder.
+  void RemoveVisitor(DecoderVisitor* visitor);
+
+  #define DECLARE(A) void Visit##A(const Instruction* instr);
+  VISITOR_LIST(DECLARE)
+  #undef DECLARE
+
+
+  std::list<DecoderVisitor*>* visitors() { return &visitors_; }
+
+ private:
+  // Decodes an instruction and calls the visitor functions registered with the
+  // Decoder class.
+  void DecodeInstruction(const Instruction* instr);
+
+  // Decode the PC relative addressing instruction, and call the corresponding
+  // visitors.
+  // On entry, instruction bits 27:24 = 0x0.
+  void DecodePCRelAddressing(const Instruction* instr);
+
+  // Decode the add/subtract immediate instruction, and call the correspoding
+  // visitors.
+  // On entry, instruction bits 27:24 = 0x1.
+  void DecodeAddSubImmediate(const Instruction* instr);
+
+  // Decode the branch, system command, and exception generation parts of
+  // the instruction tree, and call the corresponding visitors.
+  // On entry, instruction bits 27:24 = {0x4, 0x5, 0x6, 0x7}.
+  void DecodeBranchSystemException(const Instruction* instr);
+
+  // Decode the load and store parts of the instruction tree, and call
+  // the corresponding visitors.
+  // On entry, instruction bits 27:24 = {0x8, 0x9, 0xC, 0xD}.
+  void DecodeLoadStore(const Instruction* instr);
+
+  // Decode the logical immediate and move wide immediate parts of the
+  // instruction tree, and call the corresponding visitors.
+  // On entry, instruction bits 27:24 = 0x2.
+  void DecodeLogical(const Instruction* instr);
+
+  // Decode the bitfield and extraction parts of the instruction tree,
+  // and call the corresponding visitors.
+  // On entry, instruction bits 27:24 = 0x3.
+  void DecodeBitfieldExtract(const Instruction* instr);
+
+  // Decode the data processing parts of the instruction tree, and call the
+  // corresponding visitors.
+  // On entry, instruction bits 27:24 = {0x1, 0xA, 0xB}.
+  void DecodeDataProcessing(const Instruction* instr);
+
+  // Decode the floating point parts of the instruction tree, and call the
+  // corresponding visitors.
+  // On entry, instruction bits 27:24 = {0xE, 0xF}.
+  void DecodeFP(const Instruction* instr);
+
+  // Decode the Advanced SIMD (NEON) load/store part of the instruction tree,
+  // and call the corresponding visitors.
+  // On entry, instruction bits 29:25 = 0x6.
+  void DecodeNEONLoadStore(const Instruction* instr);
+
+  // Decode the Advanced SIMD (NEON) vector data processing part of the
+  // instruction tree, and call the corresponding visitors.
+  // On entry, instruction bits 28:25 = 0x7.
+  void DecodeNEONVectorDataProcessing(const Instruction* instr);
+
+  // Decode the Advanced SIMD (NEON) scalar data processing part of the
+  // instruction tree, and call the corresponding visitors.
+  // On entry, instruction bits 28:25 = 0xF.
+  void DecodeNEONScalarDataProcessing(const Instruction* instr);
+
+ private:
+  // Visitors are registered in a list.
+  std::list<DecoderVisitor*> visitors_;
+};
+
+}  // namespace vixl
+
+#endif  // VIXL_A64_DECODER_A64_H_
diff --git a/disas/libvixl/vixl/a64/disasm-a64.cc b/disas/libvixl/vixl/a64/disasm-a64.cc
new file mode 100644
index 0000000..7a58a5c
--- /dev/null
+++ b/disas/libvixl/vixl/a64/disasm-a64.cc
@@ -0,0 +1,3491 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstdlib>
+#include "vixl/a64/disasm-a64.h"
+
+namespace vixl {
+
+Disassembler::Disassembler() {
+  buffer_size_ = 256;
+  buffer_ = reinterpret_cast<char*>(malloc(buffer_size_));
+  buffer_pos_ = 0;
+  own_buffer_ = true;
+  code_address_offset_ = 0;
+}
+
+
+Disassembler::Disassembler(char* text_buffer, int buffer_size) {
+  buffer_size_ = buffer_size;
+  buffer_ = text_buffer;
+  buffer_pos_ = 0;
+  own_buffer_ = false;
+  code_address_offset_ = 0;
+}
+
+
+Disassembler::~Disassembler() {
+  if (own_buffer_) {
+    free(buffer_);
+  }
+}
+
+
+char* Disassembler::GetOutput() {
+  return buffer_;
+}
+
+
+void Disassembler::VisitAddSubImmediate(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool stack_op = (rd_is_zr || RnIsZROrSP(instr)) &&
+                  (instr->ImmAddSub() == 0) ? true : false;
+  const char *mnemonic = "";
+  const char *form = "'Rds, 'Rns, 'IAddSub";
+  const char *form_cmp = "'Rns, 'IAddSub";
+  const char *form_mov = "'Rds, 'Rns";
+
+  switch (instr->Mask(AddSubImmediateMask)) {
+    case ADD_w_imm:
+    case ADD_x_imm: {
+      mnemonic = "add";
+      if (stack_op) {
+        mnemonic = "mov";
+        form = form_mov;
+      }
+      break;
+    }
+    case ADDS_w_imm:
+    case ADDS_x_imm: {
+      mnemonic = "adds";
+      if (rd_is_zr) {
+        mnemonic = "cmn";
+        form = form_cmp;
+      }
+      break;
+    }
+    case SUB_w_imm:
+    case SUB_x_imm: mnemonic = "sub"; break;
+    case SUBS_w_imm:
+    case SUBS_x_imm: {
+      mnemonic = "subs";
+      if (rd_is_zr) {
+        mnemonic = "cmp";
+        form = form_cmp;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubShifted(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm'NDP";
+  const char *form_cmp = "'Rn, 'Rm'NDP";
+  const char *form_neg = "'Rd, 'Rm'NDP";
+
+  switch (instr->Mask(AddSubShiftedMask)) {
+    case ADD_w_shift:
+    case ADD_x_shift: mnemonic = "add"; break;
+    case ADDS_w_shift:
+    case ADDS_x_shift: {
+      mnemonic = "adds";
+      if (rd_is_zr) {
+        mnemonic = "cmn";
+        form = form_cmp;
+      }
+      break;
+    }
+    case SUB_w_shift:
+    case SUB_x_shift: {
+      mnemonic = "sub";
+      if (rn_is_zr) {
+        mnemonic = "neg";
+        form = form_neg;
+      }
+      break;
+    }
+    case SUBS_w_shift:
+    case SUBS_x_shift: {
+      mnemonic = "subs";
+      if (rd_is_zr) {
+        mnemonic = "cmp";
+        form = form_cmp;
+      } else if (rn_is_zr) {
+        mnemonic = "negs";
+        form = form_neg;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubExtended(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  const char *mnemonic = "";
+  Extend mode = static_cast<Extend>(instr->ExtendMode());
+  const char *form = ((mode == UXTX) || (mode == SXTX)) ?
+                     "'Rds, 'Rns, 'Xm'Ext" : "'Rds, 'Rns, 'Wm'Ext";
+  const char *form_cmp = ((mode == UXTX) || (mode == SXTX)) ?
+                         "'Rns, 'Xm'Ext" : "'Rns, 'Wm'Ext";
+
+  switch (instr->Mask(AddSubExtendedMask)) {
+    case ADD_w_ext:
+    case ADD_x_ext: mnemonic = "add"; break;
+    case ADDS_w_ext:
+    case ADDS_x_ext: {
+      mnemonic = "adds";
+      if (rd_is_zr) {
+        mnemonic = "cmn";
+        form = form_cmp;
+      }
+      break;
+    }
+    case SUB_w_ext:
+    case SUB_x_ext: mnemonic = "sub"; break;
+    case SUBS_w_ext:
+    case SUBS_x_ext: {
+      mnemonic = "subs";
+      if (rd_is_zr) {
+        mnemonic = "cmp";
+        form = form_cmp;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubWithCarry(const Instruction* instr) {
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm";
+  const char *form_neg = "'Rd, 'Rm";
+
+  switch (instr->Mask(AddSubWithCarryMask)) {
+    case ADC_w:
+    case ADC_x: mnemonic = "adc"; break;
+    case ADCS_w:
+    case ADCS_x: mnemonic = "adcs"; break;
+    case SBC_w:
+    case SBC_x: {
+      mnemonic = "sbc";
+      if (rn_is_zr) {
+        mnemonic = "ngc";
+        form = form_neg;
+      }
+      break;
+    }
+    case SBCS_w:
+    case SBCS_x: {
+      mnemonic = "sbcs";
+      if (rn_is_zr) {
+        mnemonic = "ngcs";
+        form = form_neg;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLogicalImmediate(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rds, 'Rn, 'ITri";
+
+  if (instr->ImmLogical() == 0) {
+    // The immediate encoded in the instruction is not in the expected format.
+    Format(instr, "unallocated", "(LogicalImmediate)");
+    return;
+  }
+
+  switch (instr->Mask(LogicalImmediateMask)) {
+    case AND_w_imm:
+    case AND_x_imm: mnemonic = "and"; break;
+    case ORR_w_imm:
+    case ORR_x_imm: {
+      mnemonic = "orr";
+      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize
+                                                        : kWRegSize;
+      if (rn_is_zr && !IsMovzMovnImm(reg_size, instr->ImmLogical())) {
+        mnemonic = "mov";
+        form = "'Rds, 'ITri";
+      }
+      break;
+    }
+    case EOR_w_imm:
+    case EOR_x_imm: mnemonic = "eor"; break;
+    case ANDS_w_imm:
+    case ANDS_x_imm: {
+      mnemonic = "ands";
+      if (rd_is_zr) {
+        mnemonic = "tst";
+        form = "'Rn, 'ITri";
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+bool Disassembler::IsMovzMovnImm(unsigned reg_size, uint64_t value) {
+  VIXL_ASSERT((reg_size == kXRegSize) ||
+              ((reg_size == kWRegSize) && (value <= 0xffffffff)));
+
+  // Test for movz: 16 bits set at positions 0, 16, 32 or 48.
+  if (((value & UINT64_C(0xffffffffffff0000)) == 0) ||
+      ((value & UINT64_C(0xffffffff0000ffff)) == 0) ||
+      ((value & UINT64_C(0xffff0000ffffffff)) == 0) ||
+      ((value & UINT64_C(0x0000ffffffffffff)) == 0)) {
+    return true;
+  }
+
+  // Test for movn: NOT(16 bits set at positions 0, 16, 32 or 48).
+  if ((reg_size == kXRegSize) &&
+      (((~value & UINT64_C(0xffffffffffff0000)) == 0) ||
+       ((~value & UINT64_C(0xffffffff0000ffff)) == 0) ||
+       ((~value & UINT64_C(0xffff0000ffffffff)) == 0) ||
+       ((~value & UINT64_C(0x0000ffffffffffff)) == 0))) {
+    return true;
+  }
+  if ((reg_size == kWRegSize) &&
+      (((value & 0xffff0000) == 0xffff0000) ||
+       ((value & 0x0000ffff) == 0x0000ffff))) {
+    return true;
+  }
+  return false;
+}
+
+
+void Disassembler::VisitLogicalShifted(const Instruction* instr) {
+  bool rd_is_zr = RdIsZROrSP(instr);
+  bool rn_is_zr = RnIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm'NLo";
+
+  switch (instr->Mask(LogicalShiftedMask)) {
+    case AND_w:
+    case AND_x: mnemonic = "and"; break;
+    case BIC_w:
+    case BIC_x: mnemonic = "bic"; break;
+    case EOR_w:
+    case EOR_x: mnemonic = "eor"; break;
+    case EON_w:
+    case EON_x: mnemonic = "eon"; break;
+    case BICS_w:
+    case BICS_x: mnemonic = "bics"; break;
+    case ANDS_w:
+    case ANDS_x: {
+      mnemonic = "ands";
+      if (rd_is_zr) {
+        mnemonic = "tst";
+        form = "'Rn, 'Rm'NLo";
+      }
+      break;
+    }
+    case ORR_w:
+    case ORR_x: {
+      mnemonic = "orr";
+      if (rn_is_zr && (instr->ImmDPShift() == 0) && (instr->ShiftDP() == LSL)) {
+        mnemonic = "mov";
+        form = "'Rd, 'Rm";
+      }
+      break;
+    }
+    case ORN_w:
+    case ORN_x: {
+      mnemonic = "orn";
+      if (rn_is_zr) {
+        mnemonic = "mvn";
+        form = "'Rd, 'Rm'NLo";
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalCompareRegister(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rn, 'Rm, 'INzcv, 'Cond";
+
+  switch (instr->Mask(ConditionalCompareRegisterMask)) {
+    case CCMN_w:
+    case CCMN_x: mnemonic = "ccmn"; break;
+    case CCMP_w:
+    case CCMP_x: mnemonic = "ccmp"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalCompareImmediate(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rn, 'IP, 'INzcv, 'Cond";
+
+  switch (instr->Mask(ConditionalCompareImmediateMask)) {
+    case CCMN_w_imm:
+    case CCMN_x_imm: mnemonic = "ccmn"; break;
+    case CCMP_w_imm:
+    case CCMP_x_imm: mnemonic = "ccmp"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalSelect(const Instruction* instr) {
+  bool rnm_is_zr = (RnIsZROrSP(instr) && RmIsZROrSP(instr));
+  bool rn_is_rm = (instr->Rn() == instr->Rm());
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm, 'Cond";
+  const char *form_test = "'Rd, 'CInv";
+  const char *form_update = "'Rd, 'Rn, 'CInv";
+
+  Condition cond = static_cast<Condition>(instr->Condition());
+  bool invertible_cond = (cond != al) && (cond != nv);
+
+  switch (instr->Mask(ConditionalSelectMask)) {
+    case CSEL_w:
+    case CSEL_x: mnemonic = "csel"; break;
+    case CSINC_w:
+    case CSINC_x: {
+      mnemonic = "csinc";
+      if (rnm_is_zr && invertible_cond) {
+        mnemonic = "cset";
+        form = form_test;
+      } else if (rn_is_rm && invertible_cond) {
+        mnemonic = "cinc";
+        form = form_update;
+      }
+      break;
+    }
+    case CSINV_w:
+    case CSINV_x: {
+      mnemonic = "csinv";
+      if (rnm_is_zr && invertible_cond) {
+        mnemonic = "csetm";
+        form = form_test;
+      } else if (rn_is_rm && invertible_cond) {
+        mnemonic = "cinv";
+        form = form_update;
+      }
+      break;
+    }
+    case CSNEG_w:
+    case CSNEG_x: {
+      mnemonic = "csneg";
+      if (rn_is_rm && invertible_cond) {
+        mnemonic = "cneg";
+        form = form_update;
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitBitfield(const Instruction* instr) {
+  unsigned s = instr->ImmS();
+  unsigned r = instr->ImmR();
+  unsigned rd_size_minus_1 =
+    ((instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize) - 1;
+  const char *mnemonic = "";
+  const char *form = "";
+  const char *form_shift_right = "'Rd, 'Rn, 'IBr";
+  const char *form_extend = "'Rd, 'Wn";
+  const char *form_bfiz = "'Rd, 'Rn, 'IBZ-r, 'IBs+1";
+  const char *form_bfx = "'Rd, 'Rn, 'IBr, 'IBs-r+1";
+  const char *form_lsl = "'Rd, 'Rn, 'IBZ-r";
+
+  switch (instr->Mask(BitfieldMask)) {
+    case SBFM_w:
+    case SBFM_x: {
+      mnemonic = "sbfx";
+      form = form_bfx;
+      if (r == 0) {
+        form = form_extend;
+        if (s == 7) {
+          mnemonic = "sxtb";
+        } else if (s == 15) {
+          mnemonic = "sxth";
+        } else if ((s == 31) && (instr->SixtyFourBits() == 1)) {
+          mnemonic = "sxtw";
+        } else {
+          form = form_bfx;
+        }
+      } else if (s == rd_size_minus_1) {
+        mnemonic = "asr";
+        form = form_shift_right;
+      } else if (s < r) {
+        mnemonic = "sbfiz";
+        form = form_bfiz;
+      }
+      break;
+    }
+    case UBFM_w:
+    case UBFM_x: {
+      mnemonic = "ubfx";
+      form = form_bfx;
+      if (r == 0) {
+        form = form_extend;
+        if (s == 7) {
+          mnemonic = "uxtb";
+        } else if (s == 15) {
+          mnemonic = "uxth";
+        } else {
+          form = form_bfx;
+        }
+      }
+      if (s == rd_size_minus_1) {
+        mnemonic = "lsr";
+        form = form_shift_right;
+      } else if (r == s + 1) {
+        mnemonic = "lsl";
+        form = form_lsl;
+      } else if (s < r) {
+        mnemonic = "ubfiz";
+        form = form_bfiz;
+      }
+      break;
+    }
+    case BFM_w:
+    case BFM_x: {
+      mnemonic = "bfxil";
+      form = form_bfx;
+      if (s < r) {
+        mnemonic = "bfi";
+        form = form_bfiz;
+      }
+    }
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitExtract(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn, 'Rm, 'IExtract";
+
+  switch (instr->Mask(ExtractMask)) {
+    case EXTR_w:
+    case EXTR_x: {
+      if (instr->Rn() == instr->Rm()) {
+        mnemonic = "ror";
+        form = "'Rd, 'Rn, 'IExtract";
+      } else {
+        mnemonic = "extr";
+      }
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitPCRelAddressing(const Instruction* instr) {
+  switch (instr->Mask(PCRelAddressingMask)) {
+    case ADR: Format(instr, "adr", "'Xd, 'AddrPCRelByte"); break;
+    case ADRP: Format(instr, "adrp", "'Xd, 'AddrPCRelPage"); break;
+    default: Format(instr, "unimplemented", "(PCRelAddressing)");
+  }
+}
+
+
+void Disassembler::VisitConditionalBranch(const Instruction* instr) {
+  switch (instr->Mask(ConditionalBranchMask)) {
+    case B_cond: Format(instr, "b.'CBrn", "'TImmCond"); break;
+    default: VIXL_UNREACHABLE();
+  }
+}
+
+
+void Disassembler::VisitUnconditionalBranchToRegister(
+    const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Xn";
+
+  switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
+    case BR: mnemonic = "br"; break;
+    case BLR: mnemonic = "blr"; break;
+    case RET: {
+      mnemonic = "ret";
+      if (instr->Rn() == kLinkRegCode) {
+        form = NULL;
+      }
+      break;
+    }
+    default: form = "(UnconditionalBranchToRegister)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitUnconditionalBranch(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'TImmUncn";
+
+  switch (instr->Mask(UnconditionalBranchMask)) {
+    case B: mnemonic = "b"; break;
+    case BL: mnemonic = "bl"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing1Source(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Rn";
+
+  switch (instr->Mask(DataProcessing1SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_w:           \
+    case A##_x: mnemonic = B; break;
+    FORMAT(RBIT, "rbit");
+    FORMAT(REV16, "rev16");
+    FORMAT(REV, "rev");
+    FORMAT(CLZ, "clz");
+    FORMAT(CLS, "cls");
+    #undef FORMAT
+    case REV32_x: mnemonic = "rev32"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing2Source(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Rd, 'Rn, 'Rm";
+  const char *form_wwx = "'Wd, 'Wn, 'Xm";
+
+  switch (instr->Mask(DataProcessing2SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_w:           \
+    case A##_x: mnemonic = B; break;
+    FORMAT(UDIV, "udiv");
+    FORMAT(SDIV, "sdiv");
+    FORMAT(LSLV, "lsl");
+    FORMAT(LSRV, "lsr");
+    FORMAT(ASRV, "asr");
+    FORMAT(RORV, "ror");
+    #undef FORMAT
+    case CRC32B: mnemonic = "crc32b"; break;
+    case CRC32H: mnemonic = "crc32h"; break;
+    case CRC32W: mnemonic = "crc32w"; break;
+    case CRC32X: mnemonic = "crc32x"; form = form_wwx; break;
+    case CRC32CB: mnemonic = "crc32cb"; break;
+    case CRC32CH: mnemonic = "crc32ch"; break;
+    case CRC32CW: mnemonic = "crc32cw"; break;
+    case CRC32CX: mnemonic = "crc32cx"; form = form_wwx; break;
+    default: form = "(DataProcessing2Source)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing3Source(const Instruction* instr) {
+  bool ra_is_zr = RaIsZROrSP(instr);
+  const char *mnemonic = "";
+  const char *form = "'Xd, 'Wn, 'Wm, 'Xa";
+  const char *form_rrr = "'Rd, 'Rn, 'Rm";
+  const char *form_rrrr = "'Rd, 'Rn, 'Rm, 'Ra";
+  const char *form_xww = "'Xd, 'Wn, 'Wm";
+  const char *form_xxx = "'Xd, 'Xn, 'Xm";
+
+  switch (instr->Mask(DataProcessing3SourceMask)) {
+    case MADD_w:
+    case MADD_x: {
+      mnemonic = "madd";
+      form = form_rrrr;
+      if (ra_is_zr) {
+        mnemonic = "mul";
+        form = form_rrr;
+      }
+      break;
+    }
+    case MSUB_w:
+    case MSUB_x: {
+      mnemonic = "msub";
+      form = form_rrrr;
+      if (ra_is_zr) {
+        mnemonic = "mneg";
+        form = form_rrr;
+      }
+      break;
+    }
+    case SMADDL_x: {
+      mnemonic = "smaddl";
+      if (ra_is_zr) {
+        mnemonic = "smull";
+        form = form_xww;
+      }
+      break;
+    }
+    case SMSUBL_x: {
+      mnemonic = "smsubl";
+      if (ra_is_zr) {
+        mnemonic = "smnegl";
+        form = form_xww;
+      }
+      break;
+    }
+    case UMADDL_x: {
+      mnemonic = "umaddl";
+      if (ra_is_zr) {
+        mnemonic = "umull";
+        form = form_xww;
+      }
+      break;
+    }
+    case UMSUBL_x: {
+      mnemonic = "umsubl";
+      if (ra_is_zr) {
+        mnemonic = "umnegl";
+        form = form_xww;
+      }
+      break;
+    }
+    case SMULH_x: {
+      mnemonic = "smulh";
+      form = form_xxx;
+      break;
+    }
+    case UMULH_x: {
+      mnemonic = "umulh";
+      form = form_xxx;
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitCompareBranch(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rt, 'TImmCmpa";
+
+  switch (instr->Mask(CompareBranchMask)) {
+    case CBZ_w:
+    case CBZ_x: mnemonic = "cbz"; break;
+    case CBNZ_w:
+    case CBNZ_x: mnemonic = "cbnz"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitTestBranch(const Instruction* instr) {
+  const char *mnemonic = "";
+  // If the top bit of the immediate is clear, the tested register is
+  // disassembled as Wt, otherwise Xt. As the top bit of the immediate is
+  // encoded in bit 31 of the instruction, we can reuse the Rt form, which
+  // uses bit 31 (normally "sf") to choose the register size.
+  const char *form = "'Rt, 'IS, 'TImmTest";
+
+  switch (instr->Mask(TestBranchMask)) {
+    case TBZ: mnemonic = "tbz"; break;
+    case TBNZ: mnemonic = "tbnz"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitMoveWideImmediate(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'IMoveImm";
+
+  // Print the shift separately for movk, to make it clear which half word will
+  // be overwritten. Movn and movz print the computed immediate, which includes
+  // shift calculation.
+  switch (instr->Mask(MoveWideImmediateMask)) {
+    case MOVN_w:
+    case MOVN_x:
+      if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0)) {
+        if ((instr->SixtyFourBits() == 0) && (instr->ImmMoveWide() == 0xffff)) {
+          mnemonic = "movn";
+        } else {
+          mnemonic = "mov";
+          form = "'Rd, 'IMoveNeg";
+        }
+      } else {
+        mnemonic = "movn";
+      }
+      break;
+    case MOVZ_w:
+    case MOVZ_x:
+      if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0))
+        mnemonic = "mov";
+      else
+        mnemonic = "movz";
+      break;
+    case MOVK_w:
+    case MOVK_x: mnemonic = "movk"; form = "'Rd, 'IMoveLSL"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+#define LOAD_STORE_LIST(V)    \
+  V(STRB_w, "strb", "'Wt")    \
+  V(STRH_w, "strh", "'Wt")    \
+  V(STR_w, "str", "'Wt")      \
+  V(STR_x, "str", "'Xt")      \
+  V(LDRB_w, "ldrb", "'Wt")    \
+  V(LDRH_w, "ldrh", "'Wt")    \
+  V(LDR_w, "ldr", "'Wt")      \
+  V(LDR_x, "ldr", "'Xt")      \
+  V(LDRSB_x, "ldrsb", "'Xt")  \
+  V(LDRSH_x, "ldrsh", "'Xt")  \
+  V(LDRSW_x, "ldrsw", "'Xt")  \
+  V(LDRSB_w, "ldrsb", "'Wt")  \
+  V(LDRSH_w, "ldrsh", "'Wt")  \
+  V(STR_b, "str", "'Bt")      \
+  V(STR_h, "str", "'Ht")      \
+  V(STR_s, "str", "'St")      \
+  V(STR_d, "str", "'Dt")      \
+  V(LDR_b, "ldr", "'Bt")      \
+  V(LDR_h, "ldr", "'Ht")      \
+  V(LDR_s, "ldr", "'St")      \
+  V(LDR_d, "ldr", "'Dt")      \
+  V(STR_q, "str", "'Qt")      \
+  V(LDR_q, "ldr", "'Qt")
+
+void Disassembler::VisitLoadStorePreIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePreIndex)";
+
+  switch (instr->Mask(LoadStorePreIndexMask)) {
+    #define LS_PREINDEX(A, B, C) \
+    case A##_pre: mnemonic = B; form = C ", ['Xns'ILS]!"; break;
+    LOAD_STORE_LIST(LS_PREINDEX)
+    #undef LS_PREINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePostIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePostIndex)";
+
+  switch (instr->Mask(LoadStorePostIndexMask)) {
+    #define LS_POSTINDEX(A, B, C) \
+    case A##_post: mnemonic = B; form = C ", ['Xns]'ILS"; break;
+    LOAD_STORE_LIST(LS_POSTINDEX)
+    #undef LS_POSTINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStoreUnsignedOffset)";
+
+  switch (instr->Mask(LoadStoreUnsignedOffsetMask)) {
+    #define LS_UNSIGNEDOFFSET(A, B, C) \
+    case A##_unsigned: mnemonic = B; form = C ", ['Xns'ILU]"; break;
+    LOAD_STORE_LIST(LS_UNSIGNEDOFFSET)
+    #undef LS_UNSIGNEDOFFSET
+    case PRFM_unsigned: mnemonic = "prfm"; form = "'PrefOp, ['Xns'ILU]";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreRegisterOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStoreRegisterOffset)";
+
+  switch (instr->Mask(LoadStoreRegisterOffsetMask)) {
+    #define LS_REGISTEROFFSET(A, B, C) \
+    case A##_reg: mnemonic = B; form = C ", ['Xns, 'Offsetreg]"; break;
+    LOAD_STORE_LIST(LS_REGISTEROFFSET)
+    #undef LS_REGISTEROFFSET
+    case PRFM_reg: mnemonic = "prfm"; form = "'PrefOp, ['Xns, 'Offsetreg]";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Wt, ['Xns'ILS]";
+  const char *form_x = "'Xt, ['Xns'ILS]";
+  const char *form_b = "'Bt, ['Xns'ILS]";
+  const char *form_h = "'Ht, ['Xns'ILS]";
+  const char *form_s = "'St, ['Xns'ILS]";
+  const char *form_d = "'Dt, ['Xns'ILS]";
+  const char *form_q = "'Qt, ['Xns'ILS]";
+  const char *form_prefetch = "'PrefOp, ['Xns'ILS]";
+
+  switch (instr->Mask(LoadStoreUnscaledOffsetMask)) {
+    case STURB_w:  mnemonic = "sturb"; break;
+    case STURH_w:  mnemonic = "sturh"; break;
+    case STUR_w:   mnemonic = "stur"; break;
+    case STUR_x:   mnemonic = "stur"; form = form_x; break;
+    case STUR_b:   mnemonic = "stur"; form = form_b; break;
+    case STUR_h:   mnemonic = "stur"; form = form_h; break;
+    case STUR_s:   mnemonic = "stur"; form = form_s; break;
+    case STUR_d:   mnemonic = "stur"; form = form_d; break;
+    case STUR_q:   mnemonic = "stur"; form = form_q; break;
+    case LDURB_w:  mnemonic = "ldurb"; break;
+    case LDURH_w:  mnemonic = "ldurh"; break;
+    case LDUR_w:   mnemonic = "ldur"; break;
+    case LDUR_x:   mnemonic = "ldur"; form = form_x; break;
+    case LDUR_b:   mnemonic = "ldur"; form = form_b; break;
+    case LDUR_h:   mnemonic = "ldur"; form = form_h; break;
+    case LDUR_s:   mnemonic = "ldur"; form = form_s; break;
+    case LDUR_d:   mnemonic = "ldur"; form = form_d; break;
+    case LDUR_q:   mnemonic = "ldur"; form = form_q; break;
+    case LDURSB_x: form = form_x; VIXL_FALLTHROUGH();
+    case LDURSB_w: mnemonic = "ldursb"; break;
+    case LDURSH_x: form = form_x; VIXL_FALLTHROUGH();
+    case LDURSH_w: mnemonic = "ldursh"; break;
+    case LDURSW_x: mnemonic = "ldursw"; form = form_x; break;
+    case PRFUM:    mnemonic = "prfum"; form = form_prefetch; break;
+    default: form = "(LoadStoreUnscaledOffset)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadLiteral(const Instruction* instr) {
+  const char *mnemonic = "ldr";
+  const char *form = "(LoadLiteral)";
+
+  switch (instr->Mask(LoadLiteralMask)) {
+    case LDR_w_lit: form = "'Wt, 'ILLiteral 'LValue"; break;
+    case LDR_x_lit: form = "'Xt, 'ILLiteral 'LValue"; break;
+    case LDR_s_lit: form = "'St, 'ILLiteral 'LValue"; break;
+    case LDR_d_lit: form = "'Dt, 'ILLiteral 'LValue"; break;
+    case LDR_q_lit: form = "'Qt, 'ILLiteral 'LValue"; break;
+    case LDRSW_x_lit: {
+      mnemonic = "ldrsw";
+      form = "'Xt, 'ILLiteral 'LValue";
+      break;
+    }
+    case PRFM_lit: {
+      mnemonic = "prfm";
+      form = "'PrefOp, 'ILLiteral 'LValue";
+      break;
+    }
+    default: mnemonic = "unimplemented";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+#define LOAD_STORE_PAIR_LIST(V)         \
+  V(STP_w, "stp", "'Wt, 'Wt2", "2")     \
+  V(LDP_w, "ldp", "'Wt, 'Wt2", "2")     \
+  V(LDPSW_x, "ldpsw", "'Xt, 'Xt2", "2") \
+  V(STP_x, "stp", "'Xt, 'Xt2", "3")     \
+  V(LDP_x, "ldp", "'Xt, 'Xt2", "3")     \
+  V(STP_s, "stp", "'St, 'St2", "2")     \
+  V(LDP_s, "ldp", "'St, 'St2", "2")     \
+  V(STP_d, "stp", "'Dt, 'Dt2", "3")     \
+  V(LDP_d, "ldp", "'Dt, 'Dt2", "3")     \
+  V(LDP_q, "ldp", "'Qt, 'Qt2", "4")     \
+  V(STP_q, "stp", "'Qt, 'Qt2", "4")
+
+void Disassembler::VisitLoadStorePairPostIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePairPostIndex)";
+
+  switch (instr->Mask(LoadStorePairPostIndexMask)) {
+    #define LSP_POSTINDEX(A, B, C, D) \
+    case A##_post: mnemonic = B; form = C ", ['Xns]'ILP" D; break;
+    LOAD_STORE_PAIR_LIST(LSP_POSTINDEX)
+    #undef LSP_POSTINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairPreIndex(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePairPreIndex)";
+
+  switch (instr->Mask(LoadStorePairPreIndexMask)) {
+    #define LSP_PREINDEX(A, B, C, D) \
+    case A##_pre: mnemonic = B; form = C ", ['Xns'ILP" D "]!"; break;
+    LOAD_STORE_PAIR_LIST(LSP_PREINDEX)
+    #undef LSP_PREINDEX
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairOffset(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(LoadStorePairOffset)";
+
+  switch (instr->Mask(LoadStorePairOffsetMask)) {
+    #define LSP_OFFSET(A, B, C, D) \
+    case A##_off: mnemonic = B; form = C ", ['Xns'ILP" D "]"; break;
+    LOAD_STORE_PAIR_LIST(LSP_OFFSET)
+    #undef LSP_OFFSET
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairNonTemporal(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form;
+
+  switch (instr->Mask(LoadStorePairNonTemporalMask)) {
+    case STNP_w: mnemonic = "stnp"; form = "'Wt, 'Wt2, ['Xns'ILP2]"; break;
+    case LDNP_w: mnemonic = "ldnp"; form = "'Wt, 'Wt2, ['Xns'ILP2]"; break;
+    case STNP_x: mnemonic = "stnp"; form = "'Xt, 'Xt2, ['Xns'ILP3]"; break;
+    case LDNP_x: mnemonic = "ldnp"; form = "'Xt, 'Xt2, ['Xns'ILP3]"; break;
+    case STNP_s: mnemonic = "stnp"; form = "'St, 'St2, ['Xns'ILP2]"; break;
+    case LDNP_s: mnemonic = "ldnp"; form = "'St, 'St2, ['Xns'ILP2]"; break;
+    case STNP_d: mnemonic = "stnp"; form = "'Dt, 'Dt2, ['Xns'ILP3]"; break;
+    case LDNP_d: mnemonic = "ldnp"; form = "'Dt, 'Dt2, ['Xns'ILP3]"; break;
+    case STNP_q: mnemonic = "stnp"; form = "'Qt, 'Qt2, ['Xns'ILP4]"; break;
+    case LDNP_q: mnemonic = "ldnp"; form = "'Qt, 'Qt2, ['Xns'ILP4]"; break;
+    default: form = "(LoadStorePairNonTemporal)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreExclusive(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form;
+
+  switch (instr->Mask(LoadStoreExclusiveMask)) {
+    case STXRB_w: mnemonic = "stxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXRH_w: mnemonic = "stxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXR_w: mnemonic = "stxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STXR_x: mnemonic = "stxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+    case LDXRB_w: mnemonic = "ldxrb"; form = "'Wt, ['Xns]"; break;
+    case LDXRH_w: mnemonic = "ldxrh"; form = "'Wt, ['Xns]"; break;
+    case LDXR_w: mnemonic = "ldxr"; form = "'Wt, ['Xns]"; break;
+    case LDXR_x: mnemonic = "ldxr"; form = "'Xt, ['Xns]"; break;
+    case STXP_w: mnemonic = "stxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+    case STXP_x: mnemonic = "stxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+    case LDXP_w: mnemonic = "ldxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+    case LDXP_x: mnemonic = "ldxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+    case STLXRB_w: mnemonic = "stlxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXRH_w: mnemonic = "stlxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXR_w: mnemonic = "stlxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+    case STLXR_x: mnemonic = "stlxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+    case LDAXRB_w: mnemonic = "ldaxrb"; form = "'Wt, ['Xns]"; break;
+    case LDAXRH_w: mnemonic = "ldaxrh"; form = "'Wt, ['Xns]"; break;
+    case LDAXR_w: mnemonic = "ldaxr"; form = "'Wt, ['Xns]"; break;
+    case LDAXR_x: mnemonic = "ldaxr"; form = "'Xt, ['Xns]"; break;
+    case STLXP_w: mnemonic = "stlxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+    case STLXP_x: mnemonic = "stlxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+    case LDAXP_w: mnemonic = "ldaxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+    case LDAXP_x: mnemonic = "ldaxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+    case STLRB_w: mnemonic = "stlrb"; form = "'Wt, ['Xns]"; break;
+    case STLRH_w: mnemonic = "stlrh"; form = "'Wt, ['Xns]"; break;
+    case STLR_w: mnemonic = "stlr"; form = "'Wt, ['Xns]"; break;
+    case STLR_x: mnemonic = "stlr"; form = "'Xt, ['Xns]"; break;
+    case LDARB_w: mnemonic = "ldarb"; form = "'Wt, ['Xns]"; break;
+    case LDARH_w: mnemonic = "ldarh"; form = "'Wt, ['Xns]"; break;
+    case LDAR_w: mnemonic = "ldar"; form = "'Wt, ['Xns]"; break;
+    case LDAR_x: mnemonic = "ldar"; form = "'Xt, ['Xns]"; break;
+    default: form = "(LoadStoreExclusive)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPCompare(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Fn, 'Fm";
+  const char *form_zero = "'Fn, #0.0";
+
+  switch (instr->Mask(FPCompareMask)) {
+    case FCMP_s_zero:
+    case FCMP_d_zero: form = form_zero; VIXL_FALLTHROUGH();
+    case FCMP_s:
+    case FCMP_d: mnemonic = "fcmp"; break;
+    case FCMPE_s_zero:
+    case FCMPE_d_zero: form = form_zero; VIXL_FALLTHROUGH();
+    case FCMPE_s:
+    case FCMPE_d: mnemonic = "fcmpe"; break;
+    default: form = "(FPCompare)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPConditionalCompare(const Instruction* instr) {
+  const char *mnemonic = "unmplemented";
+  const char *form = "'Fn, 'Fm, 'INzcv, 'Cond";
+
+  switch (instr->Mask(FPConditionalCompareMask)) {
+    case FCCMP_s:
+    case FCCMP_d: mnemonic = "fccmp"; break;
+    case FCCMPE_s:
+    case FCCMPE_d: mnemonic = "fccmpe"; break;
+    default: form = "(FPConditionalCompare)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPConditionalSelect(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Fd, 'Fn, 'Fm, 'Cond";
+
+  switch (instr->Mask(FPConditionalSelectMask)) {
+    case FCSEL_s:
+    case FCSEL_d: mnemonic = "fcsel"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing1Source(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Fd, 'Fn";
+
+  switch (instr->Mask(FPDataProcessing1SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_s:           \
+    case A##_d: mnemonic = B; break;
+    FORMAT(FMOV, "fmov");
+    FORMAT(FABS, "fabs");
+    FORMAT(FNEG, "fneg");
+    FORMAT(FSQRT, "fsqrt");
+    FORMAT(FRINTN, "frintn");
+    FORMAT(FRINTP, "frintp");
+    FORMAT(FRINTM, "frintm");
+    FORMAT(FRINTZ, "frintz");
+    FORMAT(FRINTA, "frinta");
+    FORMAT(FRINTX, "frintx");
+    FORMAT(FRINTI, "frinti");
+    #undef FORMAT
+    case FCVT_ds: mnemonic = "fcvt"; form = "'Dd, 'Sn"; break;
+    case FCVT_sd: mnemonic = "fcvt"; form = "'Sd, 'Dn"; break;
+    case FCVT_hs: mnemonic = "fcvt"; form = "'Hd, 'Sn"; break;
+    case FCVT_sh: mnemonic = "fcvt"; form = "'Sd, 'Hn"; break;
+    case FCVT_dh: mnemonic = "fcvt"; form = "'Dd, 'Hn"; break;
+    case FCVT_hd: mnemonic = "fcvt"; form = "'Hd, 'Dn"; break;
+    default: form = "(FPDataProcessing1Source)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing2Source(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Fd, 'Fn, 'Fm";
+
+  switch (instr->Mask(FPDataProcessing2SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_s:           \
+    case A##_d: mnemonic = B; break;
+    FORMAT(FMUL, "fmul");
+    FORMAT(FDIV, "fdiv");
+    FORMAT(FADD, "fadd");
+    FORMAT(FSUB, "fsub");
+    FORMAT(FMAX, "fmax");
+    FORMAT(FMIN, "fmin");
+    FORMAT(FMAXNM, "fmaxnm");
+    FORMAT(FMINNM, "fminnm");
+    FORMAT(FNMUL, "fnmul");
+    #undef FORMAT
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing3Source(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Fd, 'Fn, 'Fm, 'Fa";
+
+  switch (instr->Mask(FPDataProcessing3SourceMask)) {
+    #define FORMAT(A, B)  \
+    case A##_s:           \
+    case A##_d: mnemonic = B; break;
+    FORMAT(FMADD, "fmadd");
+    FORMAT(FMSUB, "fmsub");
+    FORMAT(FNMADD, "fnmadd");
+    FORMAT(FNMSUB, "fnmsub");
+    #undef FORMAT
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPImmediate(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "(FPImmediate)";
+
+  switch (instr->Mask(FPImmediateMask)) {
+    case FMOV_s_imm: mnemonic = "fmov"; form = "'Sd, 'IFPSingle"; break;
+    case FMOV_d_imm: mnemonic = "fmov"; form = "'Dd, 'IFPDouble"; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPIntegerConvert(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(FPIntegerConvert)";
+  const char *form_rf = "'Rd, 'Fn";
+  const char *form_fr = "'Fd, 'Rn";
+
+  switch (instr->Mask(FPIntegerConvertMask)) {
+    case FMOV_ws:
+    case FMOV_xd: mnemonic = "fmov"; form = form_rf; break;
+    case FMOV_sw:
+    case FMOV_dx: mnemonic = "fmov"; form = form_fr; break;
+    case FMOV_d1_x: mnemonic = "fmov"; form = "'Vd.D[1], 'Rn"; break;
+    case FMOV_x_d1: mnemonic = "fmov"; form = "'Rd, 'Vn.D[1]"; break;
+    case FCVTAS_ws:
+    case FCVTAS_xs:
+    case FCVTAS_wd:
+    case FCVTAS_xd: mnemonic = "fcvtas"; form = form_rf; break;
+    case FCVTAU_ws:
+    case FCVTAU_xs:
+    case FCVTAU_wd:
+    case FCVTAU_xd: mnemonic = "fcvtau"; form = form_rf; break;
+    case FCVTMS_ws:
+    case FCVTMS_xs:
+    case FCVTMS_wd:
+    case FCVTMS_xd: mnemonic = "fcvtms"; form = form_rf; break;
+    case FCVTMU_ws:
+    case FCVTMU_xs:
+    case FCVTMU_wd:
+    case FCVTMU_xd: mnemonic = "fcvtmu"; form = form_rf; break;
+    case FCVTNS_ws:
+    case FCVTNS_xs:
+    case FCVTNS_wd:
+    case FCVTNS_xd: mnemonic = "fcvtns"; form = form_rf; break;
+    case FCVTNU_ws:
+    case FCVTNU_xs:
+    case FCVTNU_wd:
+    case FCVTNU_xd: mnemonic = "fcvtnu"; form = form_rf; break;
+    case FCVTZU_xd:
+    case FCVTZU_ws:
+    case FCVTZU_wd:
+    case FCVTZU_xs: mnemonic = "fcvtzu"; form = form_rf; break;
+    case FCVTZS_xd:
+    case FCVTZS_wd:
+    case FCVTZS_xs:
+    case FCVTZS_ws: mnemonic = "fcvtzs"; form = form_rf; break;
+    case FCVTPU_xd:
+    case FCVTPU_ws:
+    case FCVTPU_wd:
+    case FCVTPU_xs: mnemonic = "fcvtpu"; form = form_rf; break;
+    case FCVTPS_xd:
+    case FCVTPS_wd:
+    case FCVTPS_xs:
+    case FCVTPS_ws: mnemonic = "fcvtps"; form = form_rf; break;
+    case SCVTF_sw:
+    case SCVTF_sx:
+    case SCVTF_dw:
+    case SCVTF_dx: mnemonic = "scvtf"; form = form_fr; break;
+    case UCVTF_sw:
+    case UCVTF_sx:
+    case UCVTF_dw:
+    case UCVTF_dx: mnemonic = "ucvtf"; form = form_fr; break;
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPFixedPointConvert(const Instruction* instr) {
+  const char *mnemonic = "";
+  const char *form = "'Rd, 'Fn, 'IFPFBits";
+  const char *form_fr = "'Fd, 'Rn, 'IFPFBits";
+
+  switch (instr->Mask(FPFixedPointConvertMask)) {
+    case FCVTZS_ws_fixed:
+    case FCVTZS_xs_fixed:
+    case FCVTZS_wd_fixed:
+    case FCVTZS_xd_fixed: mnemonic = "fcvtzs"; break;
+    case FCVTZU_ws_fixed:
+    case FCVTZU_xs_fixed:
+    case FCVTZU_wd_fixed:
+    case FCVTZU_xd_fixed: mnemonic = "fcvtzu"; break;
+    case SCVTF_sw_fixed:
+    case SCVTF_sx_fixed:
+    case SCVTF_dw_fixed:
+    case SCVTF_dx_fixed: mnemonic = "scvtf"; form = form_fr; break;
+    case UCVTF_sw_fixed:
+    case UCVTF_sx_fixed:
+    case UCVTF_dw_fixed:
+    case UCVTF_dx_fixed: mnemonic = "ucvtf"; form = form_fr; break;
+    default: VIXL_UNREACHABLE();
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitSystem(const Instruction* instr) {
+  // Some system instructions hijack their Op and Cp fields to represent a
+  // range of immediates instead of indicating a different instruction. This
+  // makes the decoding tricky.
+  const char *mnemonic = "unimplemented";
+  const char *form = "(System)";
+
+  if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
+    switch (instr->Mask(SystemExclusiveMonitorMask)) {
+      case CLREX: {
+        mnemonic = "clrex";
+        form = (instr->CRm() == 0xf) ? NULL : "'IX";
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
+    switch (instr->Mask(SystemSysRegMask)) {
+      case MRS: {
+        mnemonic = "mrs";
+        switch (instr->ImmSystemRegister()) {
+          case NZCV: form = "'Xt, nzcv"; break;
+          case FPCR: form = "'Xt, fpcr"; break;
+          default: form = "'Xt, (unknown)"; break;
+        }
+        break;
+      }
+      case MSR: {
+        mnemonic = "msr";
+        switch (instr->ImmSystemRegister()) {
+          case NZCV: form = "nzcv, 'Xt"; break;
+          case FPCR: form = "fpcr, 'Xt"; break;
+          default: form = "(unknown), 'Xt"; break;
+        }
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
+    switch (instr->ImmHint()) {
+      case NOP: {
+        mnemonic = "nop";
+        form = NULL;
+        break;
+      }
+    }
+  } else if (instr->Mask(MemBarrierFMask) == MemBarrierFixed) {
+    switch (instr->Mask(MemBarrierMask)) {
+      case DMB: {
+        mnemonic = "dmb";
+        form = "'M";
+        break;
+      }
+      case DSB: {
+        mnemonic = "dsb";
+        form = "'M";
+        break;
+      }
+      case ISB: {
+        mnemonic = "isb";
+        form = NULL;
+        break;
+      }
+    }
+  } else if (instr->Mask(SystemSysFMask) == SystemSysFixed) {
+    switch (instr->SysOp()) {
+      case IVAU:
+        mnemonic = "ic";
+        form = "ivau, 'Xt";
+        break;
+      case CVAC:
+        mnemonic = "dc";
+        form = "cvac, 'Xt";
+        break;
+      case CVAU:
+        mnemonic = "dc";
+        form = "cvau, 'Xt";
+        break;
+      case CIVAC:
+        mnemonic = "dc";
+        form = "civac, 'Xt";
+        break;
+      case ZVA:
+        mnemonic = "dc";
+        form = "zva, 'Xt";
+        break;
+      default:
+        mnemonic = "sys";
+        if (instr->Rt() == 31) {
+          form = "'G1, 'Kn, 'Km, 'G2";
+        } else {
+          form = "'G1, 'Kn, 'Km, 'G2, 'Xt";
+        }
+        break;
+      }
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitException(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'IDebug";
+
+  switch (instr->Mask(ExceptionMask)) {
+    case HLT: mnemonic = "hlt"; break;
+    case BRK: mnemonic = "brk"; break;
+    case SVC: mnemonic = "svc"; break;
+    case HVC: mnemonic = "hvc"; break;
+    case SMC: mnemonic = "smc"; break;
+    case DCPS1: mnemonic = "dcps1"; form = "{'IDebug}"; break;
+    case DCPS2: mnemonic = "dcps2"; form = "{'IDebug}"; break;
+    case DCPS3: mnemonic = "dcps3"; form = "{'IDebug}"; break;
+    default: form = "(Exception)";
+  }
+  Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitCrypto2RegSHA(const Instruction* instr) {
+  VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitCrypto3RegSHA(const Instruction* instr) {
+  VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitCryptoAES(const Instruction* instr) {
+  VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitNEON2RegMisc(const Instruction* instr) {
+  const char *mnemonic       = "unimplemented";
+  const char *form           = "'Vd.%s, 'Vn.%s";
+  const char *form_cmp_zero  = "'Vd.%s, 'Vn.%s, #0";
+  const char *form_fcmp_zero = "'Vd.%s, 'Vn.%s, #0.0";
+  NEONFormatDecoder nfd(instr);
+
+  static const NEONFormatMap map_lp_ta = {
+    {23, 22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+  };
+
+  static const NEONFormatMap map_cvt_ta = {
+    {22}, {NF_4S, NF_2D}
+  };
+
+  static const NEONFormatMap map_cvt_tb = {
+    {22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S}
+  };
+
+  if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_opcode) {
+    // These instructions all use a two bit size field, except NOT and RBIT,
+    // which use the field to encode the operation.
+    switch (instr->Mask(NEON2RegMiscMask)) {
+      case NEON_REV64:     mnemonic = "rev64"; break;
+      case NEON_REV32:     mnemonic = "rev32"; break;
+      case NEON_REV16:     mnemonic = "rev16"; break;
+      case NEON_SADDLP:
+        mnemonic = "saddlp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_UADDLP:
+        mnemonic = "uaddlp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_SUQADD:    mnemonic = "suqadd"; break;
+      case NEON_USQADD:    mnemonic = "usqadd"; break;
+      case NEON_CLS:       mnemonic = "cls"; break;
+      case NEON_CLZ:       mnemonic = "clz"; break;
+      case NEON_CNT:       mnemonic = "cnt"; break;
+      case NEON_SADALP:
+        mnemonic = "sadalp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_UADALP:
+        mnemonic = "uadalp";
+        nfd.SetFormatMap(0, &map_lp_ta);
+        break;
+      case NEON_SQABS:     mnemonic = "sqabs"; break;
+      case NEON_SQNEG:     mnemonic = "sqneg"; break;
+      case NEON_CMGT_zero: mnemonic = "cmgt"; form = form_cmp_zero; break;
+      case NEON_CMGE_zero: mnemonic = "cmge"; form = form_cmp_zero; break;
+      case NEON_CMEQ_zero: mnemonic = "cmeq"; form = form_cmp_zero; break;
+      case NEON_CMLE_zero: mnemonic = "cmle"; form = form_cmp_zero; break;
+      case NEON_CMLT_zero: mnemonic = "cmlt"; form = form_cmp_zero; break;
+      case NEON_ABS:       mnemonic = "abs"; break;
+      case NEON_NEG:       mnemonic = "neg"; break;
+      case NEON_RBIT_NOT:
+        switch (instr->FPType()) {
+          case 0: mnemonic = "mvn"; break;
+          case 1: mnemonic = "rbit"; break;
+          default: form = "(NEON2RegMisc)";
+        }
+        nfd.SetFormatMaps(nfd.LogicalFormatMap());
+        break;
+    }
+  } else {
+    // These instructions all use a one bit size field, except XTN, SQXTUN,
+    // SHLL, SQXTN and UQXTN, which use a two bit size field.
+    nfd.SetFormatMaps(nfd.FPFormatMap());
+    switch (instr->Mask(NEON2RegMiscFPMask)) {
+      case NEON_FABS:   mnemonic = "fabs"; break;
+      case NEON_FNEG:   mnemonic = "fneg"; break;
+      case NEON_FCVTN:
+        mnemonic = instr->Mask(NEON_Q) ? "fcvtn2" : "fcvtn";
+        nfd.SetFormatMap(0, &map_cvt_tb);
+        nfd.SetFormatMap(1, &map_cvt_ta);
+        break;
+      case NEON_FCVTXN:
+        mnemonic = instr->Mask(NEON_Q) ? "fcvtxn2" : "fcvtxn";
+        nfd.SetFormatMap(0, &map_cvt_tb);
+        nfd.SetFormatMap(1, &map_cvt_ta);
+        break;
+      case NEON_FCVTL:
+        mnemonic = instr->Mask(NEON_Q) ? "fcvtl2" : "fcvtl";
+        nfd.SetFormatMap(0, &map_cvt_ta);
+        nfd.SetFormatMap(1, &map_cvt_tb);
+        break;
+      case NEON_FRINTN: mnemonic = "frintn"; break;
+      case NEON_FRINTA: mnemonic = "frinta"; break;
+      case NEON_FRINTP: mnemonic = "frintp"; break;
+      case NEON_FRINTM: mnemonic = "frintm"; break;
+      case NEON_FRINTX: mnemonic = "frintx"; break;
+      case NEON_FRINTZ: mnemonic = "frintz"; break;
+      case NEON_FRINTI: mnemonic = "frinti"; break;
+      case NEON_FCVTNS: mnemonic = "fcvtns"; break;
+      case NEON_FCVTNU: mnemonic = "fcvtnu"; break;
+      case NEON_FCVTPS: mnemonic = "fcvtps"; break;
+      case NEON_FCVTPU: mnemonic = "fcvtpu"; break;
+      case NEON_FCVTMS: mnemonic = "fcvtms"; break;
+      case NEON_FCVTMU: mnemonic = "fcvtmu"; break;
+      case NEON_FCVTZS: mnemonic = "fcvtzs"; break;
+      case NEON_FCVTZU: mnemonic = "fcvtzu"; break;
+      case NEON_FCVTAS: mnemonic = "fcvtas"; break;
+      case NEON_FCVTAU: mnemonic = "fcvtau"; break;
+      case NEON_FSQRT:  mnemonic = "fsqrt"; break;
+      case NEON_SCVTF:  mnemonic = "scvtf"; break;
+      case NEON_UCVTF:  mnemonic = "ucvtf"; break;
+      case NEON_URSQRTE: mnemonic = "ursqrte"; break;
+      case NEON_URECPE:  mnemonic = "urecpe";  break;
+      case NEON_FRSQRTE: mnemonic = "frsqrte"; break;
+      case NEON_FRECPE:  mnemonic = "frecpe";  break;
+      case NEON_FCMGT_zero: mnemonic = "fcmgt"; form = form_fcmp_zero; break;
+      case NEON_FCMGE_zero: mnemonic = "fcmge"; form = form_fcmp_zero; break;
+      case NEON_FCMEQ_zero: mnemonic = "fcmeq"; form = form_fcmp_zero; break;
+      case NEON_FCMLE_zero: mnemonic = "fcmle"; form = form_fcmp_zero; break;
+      case NEON_FCMLT_zero: mnemonic = "fcmlt"; form = form_fcmp_zero; break;
+      default:
+        if ((NEON_XTN_opcode <= instr->Mask(NEON2RegMiscOpcode)) &&
+            (instr->Mask(NEON2RegMiscOpcode) <= NEON_UQXTN_opcode)) {
+          nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+          nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+
+          switch (instr->Mask(NEON2RegMiscMask)) {
+            case NEON_XTN:    mnemonic = "xtn"; break;
+            case NEON_SQXTN:  mnemonic = "sqxtn"; break;
+            case NEON_UQXTN:  mnemonic = "uqxtn"; break;
+            case NEON_SQXTUN: mnemonic = "sqxtun"; break;
+            case NEON_SHLL:
+              mnemonic = "shll";
+              nfd.SetFormatMap(0, nfd.LongIntegerFormatMap());
+              nfd.SetFormatMap(1, nfd.IntegerFormatMap());
+              switch (instr->NEONSize()) {
+                case 0: form = "'Vd.%s, 'Vn.%s, #8"; break;
+                case 1: form = "'Vd.%s, 'Vn.%s, #16"; break;
+                case 2: form = "'Vd.%s, 'Vn.%s, #32"; break;
+                default: form = "(NEON2RegMisc)";
+              }
+          }
+          Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+          return;
+        } else {
+          form = "(NEON2RegMisc)";
+        }
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEON3Same(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+  NEONFormatDecoder nfd(instr);
+
+  if (instr->Mask(NEON3SameLogicalFMask) == NEON3SameLogicalFixed) {
+    switch (instr->Mask(NEON3SameLogicalMask)) {
+      case NEON_AND: mnemonic = "and"; break;
+      case NEON_ORR:
+        mnemonic = "orr";
+        if (instr->Rm() == instr->Rn()) {
+          mnemonic = "mov";
+          form = "'Vd.%s, 'Vn.%s";
+        }
+        break;
+      case NEON_ORN: mnemonic = "orn"; break;
+      case NEON_EOR: mnemonic = "eor"; break;
+      case NEON_BIC: mnemonic = "bic"; break;
+      case NEON_BIF: mnemonic = "bif"; break;
+      case NEON_BIT: mnemonic = "bit"; break;
+      case NEON_BSL: mnemonic = "bsl"; break;
+      default: form = "(NEON3Same)";
+    }
+    nfd.SetFormatMaps(nfd.LogicalFormatMap());
+  } else {
+    static const char *mnemonics[] = {
+        "shadd", "uhadd", "shadd", "uhadd",
+        "sqadd", "uqadd", "sqadd", "uqadd",
+        "srhadd", "urhadd", "srhadd", "urhadd",
+        NULL, NULL, NULL, NULL,  // Handled by logical cases above.
+        "shsub", "uhsub", "shsub", "uhsub",
+        "sqsub", "uqsub", "sqsub", "uqsub",
+        "cmgt", "cmhi", "cmgt", "cmhi",
+        "cmge", "cmhs", "cmge", "cmhs",
+        "sshl", "ushl", "sshl", "ushl",
+        "sqshl", "uqshl", "sqshl", "uqshl",
+        "srshl", "urshl", "srshl", "urshl",
+        "sqrshl", "uqrshl", "sqrshl", "uqrshl",
+        "smax", "umax", "smax", "umax",
+        "smin", "umin", "smin", "umin",
+        "sabd", "uabd", "sabd", "uabd",
+        "saba", "uaba", "saba", "uaba",
+        "add", "sub", "add", "sub",
+        "cmtst", "cmeq", "cmtst", "cmeq",
+        "mla", "mls", "mla", "mls",
+        "mul", "pmul", "mul", "pmul",
+        "smaxp", "umaxp", "smaxp", "umaxp",
+        "sminp", "uminp", "sminp", "uminp",
+        "sqdmulh", "sqrdmulh", "sqdmulh", "sqrdmulh",
+        "addp", "unallocated", "addp", "unallocated",
+        "fmaxnm", "fmaxnmp", "fminnm", "fminnmp",
+        "fmla", "unallocated", "fmls", "unallocated",
+        "fadd", "faddp", "fsub", "fabd",
+        "fmulx", "fmul", "unallocated", "unallocated",
+        "fcmeq", "fcmge", "unallocated", "fcmgt",
+        "unallocated", "facge", "unallocated", "facgt",
+        "fmax", "fmaxp", "fmin", "fminp",
+        "frecps", "fdiv", "frsqrts", "unallocated"};
+
+    // Operation is determined by the opcode bits (15-11), the top bit of
+    // size (23) and the U bit (29).
+    unsigned index = (instr->Bits(15, 11) << 2) | (instr->Bit(23) << 1) |
+                     instr->Bit(29);
+    VIXL_ASSERT(index < (sizeof(mnemonics) / sizeof(mnemonics[0])));
+    mnemonic = mnemonics[index];
+    // Assert that index is not one of the previously handled logical
+    // instructions.
+    VIXL_ASSERT(mnemonic != NULL);
+
+    if (instr->Mask(NEON3SameFPFMask) == NEON3SameFPFixed) {
+      nfd.SetFormatMaps(nfd.FPFormatMap());
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEON3Different(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+
+  NEONFormatDecoder nfd(instr);
+  nfd.SetFormatMap(0, nfd.LongIntegerFormatMap());
+
+  // Ignore the Q bit. Appending a "2" suffix is handled later.
+  switch (instr->Mask(NEON3DifferentMask) & ~NEON_Q) {
+    case NEON_PMULL:    mnemonic = "pmull";   break;
+    case NEON_SABAL:    mnemonic = "sabal";   break;
+    case NEON_SABDL:    mnemonic = "sabdl";   break;
+    case NEON_SADDL:    mnemonic = "saddl";   break;
+    case NEON_SMLAL:    mnemonic = "smlal";   break;
+    case NEON_SMLSL:    mnemonic = "smlsl";   break;
+    case NEON_SMULL:    mnemonic = "smull";   break;
+    case NEON_SSUBL:    mnemonic = "ssubl";   break;
+    case NEON_SQDMLAL:  mnemonic = "sqdmlal"; break;
+    case NEON_SQDMLSL:  mnemonic = "sqdmlsl"; break;
+    case NEON_SQDMULL:  mnemonic = "sqdmull"; break;
+    case NEON_UABAL:    mnemonic = "uabal";   break;
+    case NEON_UABDL:    mnemonic = "uabdl";   break;
+    case NEON_UADDL:    mnemonic = "uaddl";   break;
+    case NEON_UMLAL:    mnemonic = "umlal";   break;
+    case NEON_UMLSL:    mnemonic = "umlsl";   break;
+    case NEON_UMULL:    mnemonic = "umull";   break;
+    case NEON_USUBL:    mnemonic = "usubl";   break;
+    case NEON_SADDW:
+      mnemonic = "saddw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_SSUBW:
+      mnemonic = "ssubw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_UADDW:
+      mnemonic = "uaddw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_USUBW:
+      mnemonic = "usubw";
+      nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+      break;
+    case NEON_ADDHN:
+      mnemonic = "addhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    case NEON_RADDHN:
+      mnemonic = "raddhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    case NEON_RSUBHN:
+      mnemonic = "rsubhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    case NEON_SUBHN:
+      mnemonic = "subhn";
+      nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+      nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+      break;
+    default: form = "(NEON3Different)";
+  }
+  Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONAcrossLanes(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, 'Vn.%s";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap(),
+                               NEONFormatDecoder::IntegerFormatMap());
+
+  if (instr->Mask(NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
+    nfd.SetFormatMap(0, nfd.FPScalarFormatMap());
+    nfd.SetFormatMap(1, nfd.FPFormatMap());
+    switch (instr->Mask(NEONAcrossLanesFPMask)) {
+      case NEON_FMAXV: mnemonic = "fmaxv"; break;
+      case NEON_FMINV: mnemonic = "fminv"; break;
+      case NEON_FMAXNMV: mnemonic = "fmaxnmv"; break;
+      case NEON_FMINNMV: mnemonic = "fminnmv"; break;
+      default: form = "(NEONAcrossLanes)"; break;
+    }
+  } else if (instr->Mask(NEONAcrossLanesFMask) == NEONAcrossLanesFixed) {
+    switch (instr->Mask(NEONAcrossLanesMask)) {
+      case NEON_ADDV:  mnemonic = "addv"; break;
+      case NEON_SMAXV: mnemonic = "smaxv"; break;
+      case NEON_SMINV: mnemonic = "sminv"; break;
+      case NEON_UMAXV: mnemonic = "umaxv"; break;
+      case NEON_UMINV: mnemonic = "uminv"; break;
+      case NEON_SADDLV:
+        mnemonic = "saddlv";
+        nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+        break;
+      case NEON_UADDLV:
+        mnemonic = "uaddlv";
+        nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+        break;
+      default: form = "(NEONAcrossLanes)"; break;
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form,
+      NEONFormatDecoder::kPlaceholder, NEONFormatDecoder::kFormat));
+}
+
+
+void Disassembler::VisitNEONByIndexedElement(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  bool l_instr = false;
+  bool fp_instr = false;
+
+  const char *form = "'Vd.%s, 'Vn.%s, 'Ve.%s['IVByElemIndex]";
+
+  static const NEONFormatMap map_ta = {
+    {23, 22}, {NF_UNDEF, NF_4S, NF_2D}
+  };
+  NEONFormatDecoder nfd(instr, &map_ta,
+                        NEONFormatDecoder::IntegerFormatMap(),
+                        NEONFormatDecoder::ScalarFormatMap());
+
+  switch (instr->Mask(NEONByIndexedElementMask)) {
+    case NEON_SMULL_byelement:    mnemonic = "smull"; l_instr = true; break;
+    case NEON_UMULL_byelement:    mnemonic = "umull"; l_instr = true; break;
+    case NEON_SMLAL_byelement:    mnemonic = "smlal"; l_instr = true; break;
+    case NEON_UMLAL_byelement:    mnemonic = "umlal"; l_instr = true; break;
+    case NEON_SMLSL_byelement:    mnemonic = "smlsl"; l_instr = true; break;
+    case NEON_UMLSL_byelement:    mnemonic = "umlsl"; l_instr = true; break;
+    case NEON_SQDMULL_byelement:  mnemonic = "sqdmull"; l_instr = true; break;
+    case NEON_SQDMLAL_byelement:  mnemonic = "sqdmlal"; l_instr = true; break;
+    case NEON_SQDMLSL_byelement:  mnemonic = "sqdmlsl"; l_instr = true; break;
+    case NEON_MUL_byelement:      mnemonic = "mul"; break;
+    case NEON_MLA_byelement:      mnemonic = "mla"; break;
+    case NEON_MLS_byelement:      mnemonic = "mls"; break;
+    case NEON_SQDMULH_byelement:  mnemonic = "sqdmulh";  break;
+    case NEON_SQRDMULH_byelement: mnemonic = "sqrdmulh"; break;
+    default:
+      switch (instr->Mask(NEONByIndexedElementFPMask)) {
+        case NEON_FMUL_byelement:  mnemonic = "fmul";  fp_instr = true; break;
+        case NEON_FMLA_byelement:  mnemonic = "fmla";  fp_instr = true; break;
+        case NEON_FMLS_byelement:  mnemonic = "fmls";  fp_instr = true; break;
+        case NEON_FMULX_byelement: mnemonic = "fmulx"; fp_instr = true; break;
+      }
+  }
+
+  if (l_instr) {
+    Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+  } else if (fp_instr) {
+    nfd.SetFormatMap(0, nfd.FPFormatMap());
+    Format(instr, mnemonic, nfd.Substitute(form));
+  } else {
+    nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+    Format(instr, mnemonic, nfd.Substitute(form));
+  }
+}
+
+
+void Disassembler::VisitNEONCopy(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONCopy)";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularFormatMap(),
+                               NEONFormatDecoder::TriangularScalarFormatMap());
+
+  if (instr->Mask(NEONCopyInsElementMask) == NEON_INS_ELEMENT) {
+    mnemonic = "mov";
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    form = "'Vd.%s['IVInsIndex1], 'Vn.%s['IVInsIndex2]";
+  } else if (instr->Mask(NEONCopyInsGeneralMask) == NEON_INS_GENERAL) {
+    mnemonic = "mov";
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    if (nfd.GetVectorFormat() == kFormatD) {
+      form = "'Vd.%s['IVInsIndex1], 'Xn";
+    } else {
+      form = "'Vd.%s['IVInsIndex1], 'Wn";
+    }
+  } else if (instr->Mask(NEONCopyUmovMask) == NEON_UMOV) {
+    if (instr->Mask(NEON_Q) || ((instr->ImmNEON5() & 7) == 4)) {
+      mnemonic = "mov";
+    } else {
+      mnemonic = "umov";
+    }
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    if (nfd.GetVectorFormat() == kFormatD) {
+      form = "'Xd, 'Vn.%s['IVInsIndex1]";
+    } else {
+      form = "'Wd, 'Vn.%s['IVInsIndex1]";
+    }
+  } else if (instr->Mask(NEONCopySmovMask) == NEON_SMOV) {
+    mnemonic = "smov";
+    nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+    form = "'Rdq, 'Vn.%s['IVInsIndex1]";
+  } else if (instr->Mask(NEONCopyDupElementMask) == NEON_DUP_ELEMENT) {
+    mnemonic = "dup";
+    form = "'Vd.%s, 'Vn.%s['IVInsIndex1]";
+  } else if (instr->Mask(NEONCopyDupGeneralMask) == NEON_DUP_GENERAL) {
+    mnemonic = "dup";
+    if (nfd.GetVectorFormat() == kFormat2D) {
+      form = "'Vd.%s, 'Xn";
+    } else {
+      form = "'Vd.%s, 'Wn";
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONExtract(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONExtract)";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LogicalFormatMap());
+  if (instr->Mask(NEONExtractMask) == NEON_EXT) {
+    mnemonic = "ext";
+    form = "'Vd.%s, 'Vn.%s, 'Vm.%s, 'IVExtract";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreMultiStruct(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreMultiStruct)";
+  const char *form_1v = "{'Vt.%1$s}, ['Xns]";
+  const char *form_2v = "{'Vt.%1$s, 'Vt2.%1$s}, ['Xns]";
+  const char *form_3v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s}, ['Xns]";
+  const char *form_4v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns]";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreMultiStructMask)) {
+    case NEON_LD1_1v: mnemonic = "ld1"; form = form_1v; break;
+    case NEON_LD1_2v: mnemonic = "ld1"; form = form_2v; break;
+    case NEON_LD1_3v: mnemonic = "ld1"; form = form_3v; break;
+    case NEON_LD1_4v: mnemonic = "ld1"; form = form_4v; break;
+    case NEON_LD2: mnemonic = "ld2"; form = form_2v; break;
+    case NEON_LD3: mnemonic = "ld3"; form = form_3v; break;
+    case NEON_LD4: mnemonic = "ld4"; form = form_4v; break;
+    case NEON_ST1_1v: mnemonic = "st1"; form = form_1v; break;
+    case NEON_ST1_2v: mnemonic = "st1"; form = form_2v; break;
+    case NEON_ST1_3v: mnemonic = "st1"; form = form_3v; break;
+    case NEON_ST1_4v: mnemonic = "st1"; form = form_4v; break;
+    case NEON_ST2: mnemonic = "st2"; form = form_2v; break;
+    case NEON_ST3: mnemonic = "st3"; form = form_3v; break;
+    case NEON_ST4: mnemonic = "st4"; form = form_4v; break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreMultiStructPostIndex(
+    const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreMultiStructPostIndex)";
+  const char *form_1v = "{'Vt.%1$s}, ['Xns], 'Xmr1";
+  const char *form_2v = "{'Vt.%1$s, 'Vt2.%1$s}, ['Xns], 'Xmr2";
+  const char *form_3v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s}, ['Xns], 'Xmr3";
+  const char *form_4v =
+      "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns], 'Xmr4";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreMultiStructPostIndexMask)) {
+    case NEON_LD1_1v_post: mnemonic = "ld1"; form = form_1v; break;
+    case NEON_LD1_2v_post: mnemonic = "ld1"; form = form_2v; break;
+    case NEON_LD1_3v_post: mnemonic = "ld1"; form = form_3v; break;
+    case NEON_LD1_4v_post: mnemonic = "ld1"; form = form_4v; break;
+    case NEON_LD2_post: mnemonic = "ld2"; form = form_2v; break;
+    case NEON_LD3_post: mnemonic = "ld3"; form = form_3v; break;
+    case NEON_LD4_post: mnemonic = "ld4"; form = form_4v; break;
+    case NEON_ST1_1v_post: mnemonic = "st1"; form = form_1v; break;
+    case NEON_ST1_2v_post: mnemonic = "st1"; form = form_2v; break;
+    case NEON_ST1_3v_post: mnemonic = "st1"; form = form_3v; break;
+    case NEON_ST1_4v_post: mnemonic = "st1"; form = form_4v; break;
+    case NEON_ST2_post: mnemonic = "st2"; form = form_2v; break;
+    case NEON_ST3_post: mnemonic = "st3"; form = form_3v; break;
+    case NEON_ST4_post: mnemonic = "st4"; form = form_4v; break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreSingleStruct(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreSingleStruct)";
+
+  const char *form_1b = "{'Vt.b}['IVLSLane0], ['Xns]";
+  const char *form_1h = "{'Vt.h}['IVLSLane1], ['Xns]";
+  const char *form_1s = "{'Vt.s}['IVLSLane2], ['Xns]";
+  const char *form_1d = "{'Vt.d}['IVLSLane3], ['Xns]";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreSingleStructMask)) {
+    case NEON_LD1_b: mnemonic = "ld1"; form = form_1b; break;
+    case NEON_LD1_h: mnemonic = "ld1"; form = form_1h; break;
+    case NEON_LD1_s:
+      mnemonic = "ld1";
+      VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_ST1_b: mnemonic = "st1"; form = form_1b; break;
+    case NEON_ST1_h: mnemonic = "st1"; form = form_1h; break;
+    case NEON_ST1_s:
+      mnemonic = "st1";
+      VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_LD1R:
+      mnemonic = "ld1r";
+      form = "{'Vt.%s}, ['Xns]";
+      break;
+    case NEON_LD2_b:
+    case NEON_ST2_b:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.b, 'Vt2.b}['IVLSLane0], ['Xns]";
+      break;
+    case NEON_LD2_h:
+    case NEON_ST2_h:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.h, 'Vt2.h}['IVLSLane1], ['Xns]";
+      break;
+    case NEON_LD2_s:
+    case NEON_ST2_s:
+      VIXL_STATIC_ASSERT((NEON_ST2_s | (1 << NEONLSSize_offset)) == NEON_ST2_d);
+      VIXL_STATIC_ASSERT((NEON_LD2_s | (1 << NEONLSSize_offset)) == NEON_LD2_d);
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s}['IVLSLane2], ['Xns]";
+      else
+        form = "{'Vt.d, 'Vt2.d}['IVLSLane3], ['Xns]";
+      break;
+    case NEON_LD2R:
+      mnemonic = "ld2r";
+      form = "{'Vt.%s, 'Vt2.%s}, ['Xns]";
+      break;
+    case NEON_LD3_b:
+    case NEON_ST3_b:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b}['IVLSLane0], ['Xns]";
+      break;
+    case NEON_LD3_h:
+    case NEON_ST3_h:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h}['IVLSLane1], ['Xns]";
+      break;
+    case NEON_LD3_s:
+    case NEON_ST3_s:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s}['IVLSLane2], ['Xns]";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d}['IVLSLane3], ['Xns]";
+      break;
+    case NEON_LD3R:
+      mnemonic = "ld3r";
+      form = "{'Vt.%s, 'Vt2.%s, 'Vt3.%s}, ['Xns]";
+      break;
+    case NEON_LD4_b:
+     case NEON_ST4_b:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b, 'Vt4.b}['IVLSLane0], ['Xns]";
+      break;
+    case NEON_LD4_h:
+    case NEON_ST4_h:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h, 'Vt4.h}['IVLSLane1], ['Xns]";
+      break;
+    case NEON_LD4_s:
+    case NEON_ST4_s:
+      VIXL_STATIC_ASSERT((NEON_LD4_s | (1 << NEONLSSize_offset)) == NEON_LD4_d);
+      VIXL_STATIC_ASSERT((NEON_ST4_s | (1 << NEONLSSize_offset)) == NEON_ST4_d);
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s, 'Vt4.s}['IVLSLane2], ['Xns]";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d, 'Vt4.d}['IVLSLane3], ['Xns]";
+      break;
+    case NEON_LD4R:
+      mnemonic = "ld4r";
+      form = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns]";
+      break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreSingleStructPostIndex(
+    const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONLoadStoreSingleStructPostIndex)";
+
+  const char *form_1b = "{'Vt.b}['IVLSLane0], ['Xns], 'Xmb1";
+  const char *form_1h = "{'Vt.h}['IVLSLane1], ['Xns], 'Xmb2";
+  const char *form_1s = "{'Vt.s}['IVLSLane2], ['Xns], 'Xmb4";
+  const char *form_1d = "{'Vt.d}['IVLSLane3], ['Xns], 'Xmb8";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+  switch (instr->Mask(NEONLoadStoreSingleStructPostIndexMask)) {
+    case NEON_LD1_b_post: mnemonic = "ld1"; form = form_1b; break;
+    case NEON_LD1_h_post: mnemonic = "ld1"; form = form_1h; break;
+    case NEON_LD1_s_post:
+      mnemonic = "ld1";
+      VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_ST1_b_post: mnemonic = "st1"; form = form_1b; break;
+    case NEON_ST1_h_post: mnemonic = "st1"; form = form_1h; break;
+    case NEON_ST1_s_post:
+      mnemonic = "st1";
+      VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+      form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+      break;
+    case NEON_LD1R_post:
+      mnemonic = "ld1r";
+      form = "{'Vt.%s}, ['Xns], 'Xmz1";
+      break;
+    case NEON_LD2_b_post:
+    case NEON_ST2_b_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.b, 'Vt2.b}['IVLSLane0], ['Xns], 'Xmb2";
+      break;
+    case NEON_ST2_h_post:
+    case NEON_LD2_h_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      form = "{'Vt.h, 'Vt2.h}['IVLSLane1], ['Xns], 'Xmb4";
+      break;
+    case NEON_LD2_s_post:
+    case NEON_ST2_s_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s}['IVLSLane2], ['Xns], 'Xmb8";
+      else
+        form = "{'Vt.d, 'Vt2.d}['IVLSLane3], ['Xns], 'Xmb16";
+      break;
+    case NEON_LD2R_post:
+      mnemonic = "ld2r";
+      form = "{'Vt.%s, 'Vt2.%s}, ['Xns], 'Xmz2";
+      break;
+    case NEON_LD3_b_post:
+    case NEON_ST3_b_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b}['IVLSLane0], ['Xns], 'Xmb3";
+      break;
+    case NEON_LD3_h_post:
+    case NEON_ST3_h_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h}['IVLSLane1], ['Xns], 'Xmb6";
+      break;
+    case NEON_LD3_s_post:
+    case NEON_ST3_s_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s}['IVLSLane2], ['Xns], 'Xmb12";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d}['IVLSLane3], ['Xns], 'Xmr3";
+      break;
+    case NEON_LD3R_post:
+      mnemonic = "ld3r";
+      form = "{'Vt.%s, 'Vt2.%s, 'Vt3.%s}, ['Xns], 'Xmz3";
+      break;
+    case NEON_LD4_b_post:
+    case NEON_ST4_b_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      form = "{'Vt.b, 'Vt2.b, 'Vt3.b, 'Vt4.b}['IVLSLane0], ['Xns], 'Xmb4";
+      break;
+    case NEON_LD4_h_post:
+    case NEON_ST4_h_post:
+      mnemonic = (instr->LdStXLoad()) == 1 ? "ld4" : "st4";
+      form = "{'Vt.h, 'Vt2.h, 'Vt3.h, 'Vt4.h}['IVLSLane1], ['Xns], 'Xmb8";
+      break;
+    case NEON_LD4_s_post:
+    case NEON_ST4_s_post:
+      mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+      if ((instr->NEONLSSize() & 1) == 0)
+        form = "{'Vt.s, 'Vt2.s, 'Vt3.s, 'Vt4.s}['IVLSLane2], ['Xns], 'Xmb16";
+      else
+        form = "{'Vt.d, 'Vt2.d, 'Vt3.d, 'Vt4.d}['IVLSLane3], ['Xns], 'Xmb32";
+      break;
+    case NEON_LD4R_post:
+      mnemonic = "ld4r";
+      form = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns], 'Xmz4";
+      break;
+    default: break;
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONModifiedImmediate(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vt.%s, 'IVMIImm8, lsl 'IVMIShiftAmt1";
+
+  int cmode   = instr->NEONCmode();
+  int cmode_3 = (cmode >> 3) & 1;
+  int cmode_2 = (cmode >> 2) & 1;
+  int cmode_1 = (cmode >> 1) & 1;
+  int cmode_0 = cmode & 1;
+  int q = instr->NEONQ();
+  int op = instr->NEONModImmOp();
+
+  static const NEONFormatMap map_b = { {30}, {NF_8B, NF_16B} };
+  static const NEONFormatMap map_h = { {30}, {NF_4H, NF_8H} };
+  static const NEONFormatMap map_s = { {30}, {NF_2S, NF_4S} };
+  NEONFormatDecoder nfd(instr, &map_b);
+
+  if (cmode_3 == 0) {
+    if (cmode_0 == 0) {
+      mnemonic = (op == 1) ? "mvni" : "movi";
+    } else {  // cmode<0> == '1'.
+      mnemonic = (op == 1) ? "bic" : "orr";
+    }
+    nfd.SetFormatMap(0, &map_s);
+  } else {  // cmode<3> == '1'.
+    if (cmode_2 == 0) {
+      if (cmode_0 == 0) {
+        mnemonic = (op == 1) ? "mvni" : "movi";
+      } else {  // cmode<0> == '1'.
+        mnemonic = (op == 1) ? "bic" : "orr";
+      }
+      nfd.SetFormatMap(0, &map_h);
+    } else {  // cmode<2> == '1'.
+      if (cmode_1 == 0) {
+        mnemonic = (op == 1) ? "mvni" : "movi";
+        form = "'Vt.%s, 'IVMIImm8, msl 'IVMIShiftAmt2";
+        nfd.SetFormatMap(0, &map_s);
+      } else {   // cmode<1> == '1'.
+        if (cmode_0 == 0) {
+          mnemonic = "movi";
+          if (op == 0) {
+            form = "'Vt.%s, 'IVMIImm8";
+          } else {
+            form = (q == 0) ? "'Dd, 'IVMIImm" : "'Vt.2d, 'IVMIImm";
+          }
+        } else {  // cmode<0> == '1'
+          mnemonic = "fmov";
+          if (op == 0) {
+            form  = "'Vt.%s, 'IVMIImmFPSingle";
+            nfd.SetFormatMap(0, &map_s);
+          } else {
+            if (q == 1) {
+              form = "'Vt.2d, 'IVMIImmFPDouble";
+            }
+          }
+        }
+      }
+    }
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONScalar2RegMisc(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form     = "%sd, %sn";
+  const char *form_0   = "%sd, %sn, #0";
+  const char *form_fp0 = "%sd, %sn, #0.0";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+
+  if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_scalar_opcode) {
+    // These instructions all use a two bit size field, except NOT and RBIT,
+    // which use the field to encode the operation.
+    switch (instr->Mask(NEONScalar2RegMiscMask)) {
+      case NEON_CMGT_zero_scalar: mnemonic = "cmgt"; form = form_0; break;
+      case NEON_CMGE_zero_scalar: mnemonic = "cmge"; form = form_0; break;
+      case NEON_CMLE_zero_scalar: mnemonic = "cmle"; form = form_0; break;
+      case NEON_CMLT_zero_scalar: mnemonic = "cmlt"; form = form_0; break;
+      case NEON_CMEQ_zero_scalar: mnemonic = "cmeq"; form = form_0; break;
+      case NEON_NEG_scalar:       mnemonic = "neg";   break;
+      case NEON_SQNEG_scalar:     mnemonic = "sqneg"; break;
+      case NEON_ABS_scalar:       mnemonic = "abs";   break;
+      case NEON_SQABS_scalar:     mnemonic = "sqabs"; break;
+      case NEON_SUQADD_scalar:    mnemonic = "suqadd"; break;
+      case NEON_USQADD_scalar:    mnemonic = "usqadd"; break;
+      default: form = "(NEONScalar2RegMisc)";
+    }
+  } else {
+    // These instructions all use a one bit size field, except SQXTUN, SQXTN
+    // and UQXTN, which use a two bit size field.
+    nfd.SetFormatMaps(nfd.FPScalarFormatMap());
+    switch (instr->Mask(NEONScalar2RegMiscFPMask)) {
+      case NEON_FRSQRTE_scalar:    mnemonic = "frsqrte"; break;
+      case NEON_FRECPE_scalar:     mnemonic = "frecpe";  break;
+      case NEON_SCVTF_scalar:      mnemonic = "scvtf"; break;
+      case NEON_UCVTF_scalar:      mnemonic = "ucvtf"; break;
+      case NEON_FCMGT_zero_scalar: mnemonic = "fcmgt"; form = form_fp0; break;
+      case NEON_FCMGE_zero_scalar: mnemonic = "fcmge"; form = form_fp0; break;
+      case NEON_FCMLE_zero_scalar: mnemonic = "fcmle"; form = form_fp0; break;
+      case NEON_FCMLT_zero_scalar: mnemonic = "fcmlt"; form = form_fp0; break;
+      case NEON_FCMEQ_zero_scalar: mnemonic = "fcmeq"; form = form_fp0; break;
+      case NEON_FRECPX_scalar:     mnemonic = "frecpx"; break;
+      case NEON_FCVTNS_scalar:     mnemonic = "fcvtns"; break;
+      case NEON_FCVTNU_scalar:     mnemonic = "fcvtnu"; break;
+      case NEON_FCVTPS_scalar:     mnemonic = "fcvtps"; break;
+      case NEON_FCVTPU_scalar:     mnemonic = "fcvtpu"; break;
+      case NEON_FCVTMS_scalar:     mnemonic = "fcvtms"; break;
+      case NEON_FCVTMU_scalar:     mnemonic = "fcvtmu"; break;
+      case NEON_FCVTZS_scalar:     mnemonic = "fcvtzs"; break;
+      case NEON_FCVTZU_scalar:     mnemonic = "fcvtzu"; break;
+      case NEON_FCVTAS_scalar:     mnemonic = "fcvtas"; break;
+      case NEON_FCVTAU_scalar:     mnemonic = "fcvtau"; break;
+      case NEON_FCVTXN_scalar:
+        nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+        mnemonic = "fcvtxn";
+        break;
+      default:
+        nfd.SetFormatMap(0, nfd.ScalarFormatMap());
+        nfd.SetFormatMap(1, nfd.LongScalarFormatMap());
+        switch (instr->Mask(NEONScalar2RegMiscMask)) {
+          case NEON_SQXTN_scalar:  mnemonic = "sqxtn"; break;
+          case NEON_UQXTN_scalar:  mnemonic = "uqxtn"; break;
+          case NEON_SQXTUN_scalar: mnemonic = "sqxtun"; break;
+          default: form = "(NEONScalar2RegMisc)";
+        }
+    }
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalar3Diff(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, %sn, %sm";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::LongScalarFormatMap(),
+                               NEONFormatDecoder::ScalarFormatMap());
+
+  switch (instr->Mask(NEONScalar3DiffMask)) {
+    case NEON_SQDMLAL_scalar  : mnemonic = "sqdmlal"; break;
+    case NEON_SQDMLSL_scalar  : mnemonic = "sqdmlsl"; break;
+    case NEON_SQDMULL_scalar  : mnemonic = "sqdmull"; break;
+    default: form = "(NEONScalar3Diff)";
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalar3Same(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, %sn, %sm";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+
+  if (instr->Mask(NEONScalar3SameFPFMask) == NEONScalar3SameFPFixed) {
+    nfd.SetFormatMaps(nfd.FPScalarFormatMap());
+    switch (instr->Mask(NEONScalar3SameFPMask)) {
+      case NEON_FACGE_scalar:   mnemonic = "facge"; break;
+      case NEON_FACGT_scalar:   mnemonic = "facgt"; break;
+      case NEON_FCMEQ_scalar:   mnemonic = "fcmeq"; break;
+      case NEON_FCMGE_scalar:   mnemonic = "fcmge"; break;
+      case NEON_FCMGT_scalar:   mnemonic = "fcmgt"; break;
+      case NEON_FMULX_scalar:   mnemonic = "fmulx"; break;
+      case NEON_FRECPS_scalar:  mnemonic = "frecps"; break;
+      case NEON_FRSQRTS_scalar: mnemonic = "frsqrts"; break;
+      case NEON_FABD_scalar:    mnemonic = "fabd"; break;
+      default: form = "(NEONScalar3Same)";
+    }
+  } else {
+    switch (instr->Mask(NEONScalar3SameMask)) {
+      case NEON_ADD_scalar:    mnemonic = "add";    break;
+      case NEON_SUB_scalar:    mnemonic = "sub";    break;
+      case NEON_CMEQ_scalar:   mnemonic = "cmeq";   break;
+      case NEON_CMGE_scalar:   mnemonic = "cmge";   break;
+      case NEON_CMGT_scalar:   mnemonic = "cmgt";   break;
+      case NEON_CMHI_scalar:   mnemonic = "cmhi";   break;
+      case NEON_CMHS_scalar:   mnemonic = "cmhs";   break;
+      case NEON_CMTST_scalar:  mnemonic = "cmtst";  break;
+      case NEON_UQADD_scalar:  mnemonic = "uqadd";  break;
+      case NEON_SQADD_scalar:  mnemonic = "sqadd";  break;
+      case NEON_UQSUB_scalar:  mnemonic = "uqsub";  break;
+      case NEON_SQSUB_scalar:  mnemonic = "sqsub";  break;
+      case NEON_USHL_scalar:   mnemonic = "ushl";   break;
+      case NEON_SSHL_scalar:   mnemonic = "sshl";   break;
+      case NEON_UQSHL_scalar:  mnemonic = "uqshl";  break;
+      case NEON_SQSHL_scalar:  mnemonic = "sqshl";  break;
+      case NEON_URSHL_scalar:  mnemonic = "urshl";  break;
+      case NEON_SRSHL_scalar:  mnemonic = "srshl";  break;
+      case NEON_UQRSHL_scalar: mnemonic = "uqrshl"; break;
+      case NEON_SQRSHL_scalar: mnemonic = "sqrshl"; break;
+      case NEON_SQDMULH_scalar:  mnemonic = "sqdmulh";  break;
+      case NEON_SQRDMULH_scalar: mnemonic = "sqrdmulh"; break;
+      default: form = "(NEONScalar3Same)";
+    }
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalarByIndexedElement(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, %sn, 'Ve.%s['IVByElemIndex]";
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+  bool long_instr = false;
+
+  switch (instr->Mask(NEONScalarByIndexedElementMask)) {
+    case NEON_SQDMULL_byelement_scalar:
+      mnemonic = "sqdmull";
+      long_instr = true;
+      break;
+    case NEON_SQDMLAL_byelement_scalar:
+      mnemonic = "sqdmlal";
+      long_instr = true;
+      break;
+    case NEON_SQDMLSL_byelement_scalar:
+      mnemonic = "sqdmlsl";
+      long_instr = true;
+      break;
+    case NEON_SQDMULH_byelement_scalar:
+      mnemonic = "sqdmulh";
+      break;
+    case NEON_SQRDMULH_byelement_scalar:
+      mnemonic = "sqrdmulh";
+      break;
+    default:
+      nfd.SetFormatMap(0, nfd.FPScalarFormatMap());
+      switch (instr->Mask(NEONScalarByIndexedElementFPMask)) {
+        case NEON_FMUL_byelement_scalar: mnemonic = "fmul"; break;
+        case NEON_FMLA_byelement_scalar: mnemonic = "fmla"; break;
+        case NEON_FMLS_byelement_scalar: mnemonic = "fmls"; break;
+        case NEON_FMULX_byelement_scalar: mnemonic = "fmulx"; break;
+        default: form = "(NEONScalarByIndexedElement)";
+      }
+  }
+
+  if (long_instr) {
+    nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(
+    form, nfd.kPlaceholder, nfd.kPlaceholder, nfd.kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarCopy(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONScalarCopy)";
+
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularScalarFormatMap());
+
+  if (instr->Mask(NEONScalarCopyMask) == NEON_DUP_ELEMENT_scalar) {
+    mnemonic = "mov";
+    form = "%sd, 'Vn.%s['IVInsIndex1]";
+  }
+
+  Format(instr, mnemonic, nfd.Substitute(form, nfd.kPlaceholder, nfd.kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarPairwise(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "%sd, 'Vn.%s";
+  NEONFormatMap map = { {22}, {NF_2S, NF_2D} };
+  NEONFormatDecoder nfd(instr, NEONFormatDecoder::FPScalarFormatMap(), &map);
+
+  switch (instr->Mask(NEONScalarPairwiseMask)) {
+    case NEON_ADDP_scalar:    mnemonic = "addp"; break;
+    case NEON_FADDP_scalar:   mnemonic = "faddp"; break;
+    case NEON_FMAXP_scalar:   mnemonic = "fmaxp"; break;
+    case NEON_FMAXNMP_scalar: mnemonic = "fmaxnmp"; break;
+    case NEON_FMINP_scalar:   mnemonic = "fminp"; break;
+    case NEON_FMINNMP_scalar: mnemonic = "fminnmp"; break;
+    default: form = "(NEONScalarPairwise)";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form,
+      NEONFormatDecoder::kPlaceholder, NEONFormatDecoder::kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarShiftImmediate(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form   = "%sd, %sn, 'Is1";
+  const char *form_2 = "%sd, %sn, 'Is2";
+
+  static const NEONFormatMap map_shift = {
+    {22, 21, 20, 19},
+    {NF_UNDEF, NF_B, NF_H, NF_H, NF_S, NF_S, NF_S, NF_S,
+     NF_D,     NF_D, NF_D, NF_D, NF_D, NF_D, NF_D, NF_D}
+  };
+  static const NEONFormatMap map_shift_narrow = {
+    {21, 20, 19},
+    {NF_UNDEF, NF_H, NF_S, NF_S, NF_D, NF_D, NF_D, NF_D}
+  };
+  NEONFormatDecoder nfd(instr, &map_shift);
+
+  if (instr->ImmNEONImmh()) {  // immh has to be non-zero.
+    switch (instr->Mask(NEONScalarShiftImmediateMask)) {
+      case NEON_FCVTZU_imm_scalar: mnemonic = "fcvtzu";    break;
+      case NEON_FCVTZS_imm_scalar: mnemonic = "fcvtzs";   break;
+      case NEON_SCVTF_imm_scalar: mnemonic = "scvtf";    break;
+      case NEON_UCVTF_imm_scalar: mnemonic = "ucvtf";   break;
+      case NEON_SRI_scalar:       mnemonic = "sri";    break;
+      case NEON_SSHR_scalar:      mnemonic = "sshr";   break;
+      case NEON_USHR_scalar:      mnemonic = "ushr";   break;
+      case NEON_SRSHR_scalar:     mnemonic = "srshr";  break;
+      case NEON_URSHR_scalar:     mnemonic = "urshr";  break;
+      case NEON_SSRA_scalar:      mnemonic = "ssra";   break;
+      case NEON_USRA_scalar:      mnemonic = "usra";   break;
+      case NEON_SRSRA_scalar:     mnemonic = "srsra";  break;
+      case NEON_URSRA_scalar:     mnemonic = "ursra";  break;
+      case NEON_SHL_scalar:       mnemonic = "shl";    form = form_2; break;
+      case NEON_SLI_scalar:       mnemonic = "sli";    form = form_2; break;
+      case NEON_SQSHLU_scalar:    mnemonic = "sqshlu"; form = form_2; break;
+      case NEON_SQSHL_imm_scalar: mnemonic = "sqshl";  form = form_2; break;
+      case NEON_UQSHL_imm_scalar: mnemonic = "uqshl";  form = form_2; break;
+      case NEON_UQSHRN_scalar:
+        mnemonic = "uqshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_UQRSHRN_scalar:
+        mnemonic = "uqrshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQSHRN_scalar:
+        mnemonic = "sqshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQRSHRN_scalar:
+        mnemonic = "sqrshrn";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQSHRUN_scalar:
+        mnemonic = "sqshrun";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      case NEON_SQRSHRUN_scalar:
+        mnemonic = "sqrshrun";
+        nfd.SetFormatMap(1, &map_shift_narrow);
+        break;
+      default:
+        form = "(NEONScalarShiftImmediate)";
+    }
+  } else {
+    form = "(NEONScalarShiftImmediate)";
+  }
+  Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONShiftImmediate(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form         = "'Vd.%s, 'Vn.%s, 'Is1";
+  const char *form_shift_2 = "'Vd.%s, 'Vn.%s, 'Is2";
+  const char *form_xtl     = "'Vd.%s, 'Vn.%s";
+
+  // 0001->8H, 001x->4S, 01xx->2D, all others undefined.
+  static const NEONFormatMap map_shift_ta = {
+    {22, 21, 20, 19},
+    {NF_UNDEF, NF_8H, NF_4S, NF_4S, NF_2D, NF_2D, NF_2D, NF_2D}
+  };
+
+  // 00010->8B, 00011->16B, 001x0->4H, 001x1->8H,
+  // 01xx0->2S, 01xx1->4S, 1xxx1->2D, all others undefined.
+  static const NEONFormatMap map_shift_tb = {
+    {22, 21, 20, 19, 30},
+    {NF_UNDEF, NF_UNDEF, NF_8B,    NF_16B, NF_4H,    NF_8H, NF_4H,    NF_8H,
+     NF_2S,    NF_4S,    NF_2S,    NF_4S,  NF_2S,    NF_4S, NF_2S,    NF_4S,
+     NF_UNDEF, NF_2D,    NF_UNDEF, NF_2D,  NF_UNDEF, NF_2D, NF_UNDEF, NF_2D,
+     NF_UNDEF, NF_2D,    NF_UNDEF, NF_2D,  NF_UNDEF, NF_2D, NF_UNDEF, NF_2D}
+  };
+
+  NEONFormatDecoder nfd(instr, &map_shift_tb);
+
+  if (instr->ImmNEONImmh()) {  // immh has to be non-zero.
+    switch (instr->Mask(NEONShiftImmediateMask)) {
+      case NEON_SQSHLU:     mnemonic = "sqshlu"; form = form_shift_2; break;
+      case NEON_SQSHL_imm:  mnemonic = "sqshl";  form = form_shift_2; break;
+      case NEON_UQSHL_imm:  mnemonic = "uqshl";  form = form_shift_2; break;
+      case NEON_SHL:        mnemonic = "shl";    form = form_shift_2; break;
+      case NEON_SLI:        mnemonic = "sli";    form = form_shift_2; break;
+      case NEON_SCVTF_imm:  mnemonic = "scvtf";  break;
+      case NEON_UCVTF_imm:  mnemonic = "ucvtf";  break;
+      case NEON_FCVTZU_imm: mnemonic = "fcvtzu"; break;
+      case NEON_FCVTZS_imm: mnemonic = "fcvtzs"; break;
+      case NEON_SRI:        mnemonic = "sri";    break;
+      case NEON_SSHR:       mnemonic = "sshr";   break;
+      case NEON_USHR:       mnemonic = "ushr";   break;
+      case NEON_SRSHR:      mnemonic = "srshr";  break;
+      case NEON_URSHR:      mnemonic = "urshr";  break;
+      case NEON_SSRA:       mnemonic = "ssra";   break;
+      case NEON_USRA:       mnemonic = "usra";   break;
+      case NEON_SRSRA:      mnemonic = "srsra";  break;
+      case NEON_URSRA:      mnemonic = "ursra";  break;
+      case NEON_SHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "shrn2" : "shrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_RSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "rshrn2" : "rshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_UQSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "uqshrn2" : "uqshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_UQRSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "uqrshrn2" : "uqrshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqshrn2" : "sqshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQRSHRN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqrshrn2" : "sqrshrn";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQSHRUN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqshrun2" : "sqshrun";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SQRSHRUN:
+        mnemonic = instr->Mask(NEON_Q) ? "sqrshrun2" : "sqrshrun";
+        nfd.SetFormatMap(1, &map_shift_ta);
+        break;
+      case NEON_SSHLL:
+        nfd.SetFormatMap(0, &map_shift_ta);
+        if (instr->ImmNEONImmb() == 0 &&
+            CountSetBits(instr->ImmNEONImmh(), 32) == 1) {  // sxtl variant.
+          form = form_xtl;
+          mnemonic = instr->Mask(NEON_Q) ? "sxtl2" : "sxtl";
+        } else {  // sshll variant.
+          form = form_shift_2;
+          mnemonic = instr->Mask(NEON_Q) ? "sshll2" : "sshll";
+        }
+        break;
+      case NEON_USHLL:
+        nfd.SetFormatMap(0, &map_shift_ta);
+        if (instr->ImmNEONImmb() == 0 &&
+            CountSetBits(instr->ImmNEONImmh(), 32) == 1) {  // uxtl variant.
+          form = form_xtl;
+          mnemonic = instr->Mask(NEON_Q) ? "uxtl2" : "uxtl";
+        } else {  // ushll variant.
+          form = form_shift_2;
+          mnemonic = instr->Mask(NEON_Q) ? "ushll2" : "ushll";
+        }
+        break;
+      default: form = "(NEONShiftImmediate)";
+    }
+  } else {
+    form = "(NEONShiftImmediate)";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONTable(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "(NEONTable)";
+  const char form_1v[] = "'Vd.%%s, {'Vn.16b}, 'Vm.%%s";
+  const char form_2v[] = "'Vd.%%s, {'Vn.16b, v%d.16b}, 'Vm.%%s";
+  const char form_3v[] = "'Vd.%%s, {'Vn.16b, v%d.16b, v%d.16b}, 'Vm.%%s";
+  const char form_4v[] =
+      "'Vd.%%s, {'Vn.16b, v%d.16b, v%d.16b, v%d.16b}, 'Vm.%%s";
+  static const NEONFormatMap map_b = { {30}, {NF_8B, NF_16B} };
+  NEONFormatDecoder nfd(instr, &map_b);
+
+  switch (instr->Mask(NEONTableMask)) {
+    case NEON_TBL_1v: mnemonic = "tbl"; form = form_1v; break;
+    case NEON_TBL_2v: mnemonic = "tbl"; form = form_2v; break;
+    case NEON_TBL_3v: mnemonic = "tbl"; form = form_3v; break;
+    case NEON_TBL_4v: mnemonic = "tbl"; form = form_4v; break;
+    case NEON_TBX_1v: mnemonic = "tbx"; form = form_1v; break;
+    case NEON_TBX_2v: mnemonic = "tbx"; form = form_2v; break;
+    case NEON_TBX_3v: mnemonic = "tbx"; form = form_3v; break;
+    case NEON_TBX_4v: mnemonic = "tbx"; form = form_4v; break;
+    default: break;
+  }
+
+  char re_form[sizeof(form_4v) + 6];
+  int reg_num = instr->Rn();
+  snprintf(re_form, sizeof(re_form), form,
+           (reg_num + 1) % kNumberOfVRegisters,
+           (reg_num + 2) % kNumberOfVRegisters,
+           (reg_num + 3) % kNumberOfVRegisters);
+
+  Format(instr, mnemonic, nfd.Substitute(re_form));
+}
+
+
+void Disassembler::VisitNEONPerm(const Instruction* instr) {
+  const char *mnemonic = "unimplemented";
+  const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+  NEONFormatDecoder nfd(instr);
+
+  switch (instr->Mask(NEONPermMask)) {
+    case NEON_TRN1: mnemonic = "trn1";   break;
+    case NEON_TRN2: mnemonic = "trn2";  break;
+    case NEON_UZP1: mnemonic = "uzp1"; break;
+    case NEON_UZP2: mnemonic = "uzp2";  break;
+    case NEON_ZIP1: mnemonic = "zip1"; break;
+    case NEON_ZIP2: mnemonic = "zip2"; break;
+    default: form = "(NEONPerm)";
+  }
+  Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitUnimplemented(const Instruction* instr) {
+  Format(instr, "unimplemented", "(Unimplemented)");
+}
+
+
+void Disassembler::VisitUnallocated(const Instruction* instr) {
+  Format(instr, "unallocated", "(Unallocated)");
+}
+
+
+void Disassembler::ProcessOutput(const Instruction* /*instr*/) {
+  // The base disasm does nothing more than disassembling into a buffer.
+}
+
+
+void Disassembler::AppendRegisterNameToOutput(const Instruction* instr,
+                                              const CPURegister& reg) {
+  USE(instr);
+  VIXL_ASSERT(reg.IsValid());
+  char reg_char;
+
+  if (reg.IsRegister()) {
+    reg_char = reg.Is64Bits() ? 'x' : 'w';
+  } else {
+    VIXL_ASSERT(reg.IsVRegister());
+    switch (reg.SizeInBits()) {
+      case kBRegSize: reg_char = 'b'; break;
+      case kHRegSize: reg_char = 'h'; break;
+      case kSRegSize: reg_char = 's'; break;
+      case kDRegSize: reg_char = 'd'; break;
+      default:
+        VIXL_ASSERT(reg.Is128Bits());
+        reg_char = 'q';
+    }
+  }
+
+  if (reg.IsVRegister() || !(reg.Aliases(sp) || reg.Aliases(xzr))) {
+    // A core or scalar/vector register: [wx]0 - 30, [bhsdq]0 - 31.
+    AppendToOutput("%c%d", reg_char, reg.code());
+  } else if (reg.Aliases(sp)) {
+    // Disassemble w31/x31 as stack pointer wsp/sp.
+    AppendToOutput("%s", reg.Is64Bits() ? "sp" : "wsp");
+  } else {
+    // Disassemble w31/x31 as zero register wzr/xzr.
+    AppendToOutput("%czr", reg_char);
+  }
+}
+
+
+void Disassembler::AppendPCRelativeOffsetToOutput(const Instruction* instr,
+                                                  int64_t offset) {
+  USE(instr);
+  uint64_t abs_offset = offset;
+  char sign = (offset < 0) ? '-' : '+';
+  if (offset < 0) {
+    abs_offset = -abs_offset;
+  }
+  AppendToOutput("#%c0x%" PRIx64, sign, abs_offset);
+}
+
+
+void Disassembler::AppendAddressToOutput(const Instruction* instr,
+                                         const void* addr) {
+  USE(instr);
+  AppendToOutput("(addr 0x%" PRIxPTR ")", reinterpret_cast<uintptr_t>(addr));
+}
+
+
+void Disassembler::AppendCodeAddressToOutput(const Instruction* instr,
+                                             const void* addr) {
+  AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendDataAddressToOutput(const Instruction* instr,
+                                             const void* addr) {
+  AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendCodeRelativeAddressToOutput(const Instruction* instr,
+                                                     const void* addr) {
+  USE(instr);
+  int64_t rel_addr = CodeRelativeAddress(addr);
+  if (rel_addr >= 0) {
+    AppendToOutput("(addr 0x%" PRIx64 ")", rel_addr);
+  } else {
+    AppendToOutput("(addr -0x%" PRIx64 ")", -rel_addr);
+  }
+}
+
+
+void Disassembler::AppendCodeRelativeCodeAddressToOutput(
+    const Instruction* instr, const void* addr) {
+  AppendCodeRelativeAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendCodeRelativeDataAddressToOutput(
+    const Instruction* instr, const void* addr) {
+  AppendCodeRelativeAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::MapCodeAddress(int64_t base_address,
+                                  const Instruction* instr_address) {
+  set_code_address_offset(
+      base_address - reinterpret_cast<intptr_t>(instr_address));
+}
+int64_t Disassembler::CodeRelativeAddress(const void* addr) {
+  return reinterpret_cast<intptr_t>(addr) + code_address_offset();
+}
+
+
+void Disassembler::Format(const Instruction* instr, const char* mnemonic,
+                          const char* format) {
+  VIXL_ASSERT(mnemonic != NULL);
+  ResetOutput();
+  Substitute(instr, mnemonic);
+  if (format != NULL) {
+    VIXL_ASSERT(buffer_pos_ < buffer_size_);
+    buffer_[buffer_pos_++] = ' ';
+    Substitute(instr, format);
+  }
+  VIXL_ASSERT(buffer_pos_ < buffer_size_);
+  buffer_[buffer_pos_] = 0;
+  ProcessOutput(instr);
+}
+
+
+void Disassembler::Substitute(const Instruction* instr, const char* string) {
+  char chr = *string++;
+  while (chr != '\0') {
+    if (chr == '\'') {
+      string += SubstituteField(instr, string);
+    } else {
+      VIXL_ASSERT(buffer_pos_ < buffer_size_);
+      buffer_[buffer_pos_++] = chr;
+    }
+    chr = *string++;
+  }
+}
+
+
+int Disassembler::SubstituteField(const Instruction* instr,
+                                  const char* format) {
+  switch (format[0]) {
+    // NB. The remaining substitution prefix characters are: GJKUZ.
+    case 'R':  // Register. X or W, selected by sf bit.
+    case 'F':  // FP register. S or D, selected by type field.
+    case 'V':  // Vector register, V, vector format.
+    case 'W':
+    case 'X':
+    case 'B':
+    case 'H':
+    case 'S':
+    case 'D':
+    case 'Q': return SubstituteRegisterField(instr, format);
+    case 'I': return SubstituteImmediateField(instr, format);
+    case 'L': return SubstituteLiteralField(instr, format);
+    case 'N': return SubstituteShiftField(instr, format);
+    case 'P': return SubstitutePrefetchField(instr, format);
+    case 'C': return SubstituteConditionField(instr, format);
+    case 'E': return SubstituteExtendField(instr, format);
+    case 'A': return SubstitutePCRelAddressField(instr, format);
+    case 'T': return SubstituteBranchTargetField(instr, format);
+    case 'O': return SubstituteLSRegOffsetField(instr, format);
+    case 'M': return SubstituteBarrierField(instr, format);
+    case 'K': return SubstituteCrField(instr, format);
+    case 'G': return SubstituteSysOpField(instr, format);
+    default: {
+      VIXL_UNREACHABLE();
+      return 1;
+    }
+  }
+}
+
+
+int Disassembler::SubstituteRegisterField(const Instruction* instr,
+                                          const char* format) {
+  char reg_prefix = format[0];
+  unsigned reg_num = 0;
+  unsigned field_len = 2;
+
+  switch (format[1]) {
+    case 'd':
+      reg_num = instr->Rd();
+      if (format[2] == 'q') {
+        reg_prefix = instr->NEONQ() ? 'X' : 'W';
+        field_len = 3;
+      }
+      break;
+    case 'n': reg_num = instr->Rn(); break;
+    case 'm':
+      reg_num = instr->Rm();
+      switch (format[2]) {
+          // Handle registers tagged with b (bytes), z (instruction), or
+          // r (registers), used for address updates in
+          // NEON load/store instructions.
+        case 'r':
+        case 'b':
+        case 'z': {
+          field_len = 3;
+          char* eimm;
+          int imm = static_cast<int>(strtol(&format[3], &eimm, 10));
+          field_len += eimm - &format[3];
+          if (reg_num == 31) {
+            switch (format[2]) {
+              case 'z':
+                imm *= (1 << instr->NEONLSSize());
+                break;
+              case 'r':
+                imm *= (instr->NEONQ() == 0) ? kDRegSizeInBytes
+                                             : kQRegSizeInBytes;
+                break;
+              case 'b':
+                break;
+            }
+            AppendToOutput("#%d", imm);
+            return field_len;
+          }
+          break;
+        }
+      }
+      break;
+    case 'e':
+      // This is register Rm, but using a 4-bit specifier. Used in NEON
+      // by-element instructions.
+      reg_num = (instr->Rm() & 0xf);
+      break;
+    case 'a': reg_num = instr->Ra(); break;
+    case 's': reg_num = instr->Rs(); break;
+    case 't':
+      reg_num = instr->Rt();
+      if (format[0] == 'V') {
+        if ((format[2] >= '2') && (format[2] <= '4')) {
+          // Handle consecutive vector register specifiers Vt2, Vt3 and Vt4.
+          reg_num = (reg_num + format[2] - '1') % 32;
+          field_len = 3;
+        }
+      } else {
+        if (format[2] == '2') {
+        // Handle register specifier Rt2.
+          reg_num = instr->Rt2();
+          field_len = 3;
+        }
+      }
+      break;
+    default: VIXL_UNREACHABLE();
+  }
+
+  // Increase field length for registers tagged as stack.
+  if (format[2] == 's') {
+    field_len = 3;
+  }
+
+  CPURegister::RegisterType reg_type = CPURegister::kRegister;
+  unsigned reg_size = kXRegSize;
+
+  if (reg_prefix == 'R') {
+    reg_prefix = instr->SixtyFourBits() ? 'X' : 'W';
+  } else if (reg_prefix == 'F') {
+    reg_prefix = ((instr->FPType() & 1) == 0) ? 'S' : 'D';
+  }
+
+  switch (reg_prefix) {
+    case 'W':
+      reg_type = CPURegister::kRegister; reg_size = kWRegSize; break;
+    case 'X':
+      reg_type = CPURegister::kRegister; reg_size = kXRegSize; break;
+    case 'B':
+      reg_type = CPURegister::kVRegister; reg_size = kBRegSize; break;
+    case 'H':
+      reg_type = CPURegister::kVRegister; reg_size = kHRegSize; break;
+    case 'S':
+      reg_type = CPURegister::kVRegister; reg_size = kSRegSize; break;
+    case 'D':
+      reg_type = CPURegister::kVRegister; reg_size = kDRegSize; break;
+    case 'Q':
+      reg_type = CPURegister::kVRegister; reg_size = kQRegSize; break;
+    case 'V':
+      AppendToOutput("v%d", reg_num);
+      return field_len;
+    default:
+      VIXL_UNREACHABLE();
+  }
+
+  if ((reg_type == CPURegister::kRegister) &&
+      (reg_num == kZeroRegCode) && (format[2] == 's')) {
+    reg_num = kSPRegInternalCode;
+  }
+
+  AppendRegisterNameToOutput(instr, CPURegister(reg_num, reg_size, reg_type));
+
+  return field_len;
+}
+
+
+int Disassembler::SubstituteImmediateField(const Instruction* instr,
+                                           const char* format) {
+  VIXL_ASSERT(format[0] == 'I');
+
+  switch (format[1]) {
+    case 'M': {  // IMoveImm, IMoveNeg or IMoveLSL.
+      if (format[5] == 'L') {
+        AppendToOutput("#0x%" PRIx32, instr->ImmMoveWide());
+        if (instr->ShiftMoveWide() > 0) {
+          AppendToOutput(", lsl #%" PRId32, 16 * instr->ShiftMoveWide());
+        }
+      } else {
+        VIXL_ASSERT((format[5] == 'I') || (format[5] == 'N'));
+        uint64_t imm = static_cast<uint64_t>(instr->ImmMoveWide()) <<
+            (16 * instr->ShiftMoveWide());
+        if (format[5] == 'N')
+          imm = ~imm;
+        if (!instr->SixtyFourBits())
+          imm &= UINT64_C(0xffffffff);
+        AppendToOutput("#0x%" PRIx64, imm);
+      }
+      return 8;
+    }
+    case 'L': {
+      switch (format[2]) {
+        case 'L': {  // ILLiteral - Immediate Load Literal.
+          AppendToOutput("pc%+" PRId32,
+                         instr->ImmLLiteral() << kLiteralEntrySizeLog2);
+          return 9;
+        }
+        case 'S': {  // ILS - Immediate Load/Store.
+          if (instr->ImmLS() != 0) {
+            AppendToOutput(", #%" PRId32, instr->ImmLS());
+          }
+          return 3;
+        }
+        case 'P': {  // ILPx - Immediate Load/Store Pair, x = access size.
+          if (instr->ImmLSPair() != 0) {
+            // format[3] is the scale value. Convert to a number.
+            int scale = 1 << (format[3] - '0');
+            AppendToOutput(", #%" PRId32, instr->ImmLSPair() * scale);
+          }
+          return 4;
+        }
+        case 'U': {  // ILU - Immediate Load/Store Unsigned.
+          if (instr->ImmLSUnsigned() != 0) {
+            int shift = instr->SizeLS();
+            AppendToOutput(", #%" PRId32, instr->ImmLSUnsigned() << shift);
+          }
+          return 3;
+        }
+      }
+    }
+    case 'C': {  // ICondB - Immediate Conditional Branch.
+      int64_t offset = instr->ImmCondBranch() << 2;
+      AppendPCRelativeOffsetToOutput(instr, offset);
+      return 6;
+    }
+    case 'A': {  // IAddSub.
+      VIXL_ASSERT(instr->ShiftAddSub() <= 1);
+      int64_t imm = instr->ImmAddSub() << (12 * instr->ShiftAddSub());
+      AppendToOutput("#0x%" PRIx64 " (%" PRId64 ")", imm, imm);
+      return 7;
+    }
+    case 'F': {  // IFPSingle, IFPDouble or IFPFBits.
+      if (format[3] == 'F') {  // IFPFbits.
+        AppendToOutput("#%" PRId32, 64 - instr->FPScale());
+        return 8;
+      } else {
+        AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmFP(),
+                       format[3] == 'S' ? instr->ImmFP32() : instr->ImmFP64());
+        return 9;
+      }
+    }
+    case 'T': {  // ITri - Immediate Triangular Encoded.
+      AppendToOutput("#0x%" PRIx64, instr->ImmLogical());
+      return 4;
+    }
+    case 'N': {  // INzcv.
+      int nzcv = (instr->Nzcv() << Flags_offset);
+      AppendToOutput("#%c%c%c%c", ((nzcv & NFlag) == 0) ? 'n' : 'N',
+                                  ((nzcv & ZFlag) == 0) ? 'z' : 'Z',
+                                  ((nzcv & CFlag) == 0) ? 'c' : 'C',
+                                  ((nzcv & VFlag) == 0) ? 'v' : 'V');
+      return 5;
+    }
+    case 'P': {  // IP - Conditional compare.
+      AppendToOutput("#%" PRId32, instr->ImmCondCmp());
+      return 2;
+    }
+    case 'B': {  // Bitfields.
+      return SubstituteBitfieldImmediateField(instr, format);
+    }
+    case 'E': {  // IExtract.
+      AppendToOutput("#%" PRId32, instr->ImmS());
+      return 8;
+    }
+    case 'S': {  // IS - Test and branch bit.
+      AppendToOutput("#%" PRId32, (instr->ImmTestBranchBit5() << 5) |
+                                  instr->ImmTestBranchBit40());
+      return 2;
+    }
+    case 's': {  // Is - Shift (immediate).
+      switch (format[2]) {
+        case '1': {  // Is1 - SSHR.
+          int shift = 16 << HighestSetBitPosition(instr->ImmNEONImmh());
+          shift -= instr->ImmNEONImmhImmb();
+          AppendToOutput("#%d", shift);
+          return 3;
+        }
+        case '2': {  // Is2 - SLI.
+          int shift = instr->ImmNEONImmhImmb();
+          shift -= 8 << HighestSetBitPosition(instr->ImmNEONImmh());
+          AppendToOutput("#%d", shift);
+          return 3;
+        }
+        default: {
+          VIXL_UNIMPLEMENTED();
+          return 0;
+        }
+      }
+    }
+    case 'D': {  // IDebug - HLT and BRK instructions.
+      AppendToOutput("#0x%" PRIx32, instr->ImmException());
+      return 6;
+    }
+    case 'V': {  // Immediate Vector.
+      switch (format[2]) {
+        case 'E': {  // IVExtract.
+          AppendToOutput("#%" PRId32, instr->ImmNEONExt());
+          return 9;
+        }
+        case 'B': {  // IVByElemIndex.
+          int vm_index = (instr->NEONH() << 1) | instr->NEONL();
+          if (instr->NEONSize() == 1) {
+            vm_index = (vm_index << 1) | instr->NEONM();
+          }
+          AppendToOutput("%d", vm_index);
+          return strlen("IVByElemIndex");
+        }
+        case 'I': {  // INS element.
+          if (strncmp(format, "IVInsIndex", strlen("IVInsIndex")) == 0) {
+            int rd_index, rn_index;
+            int imm5 = instr->ImmNEON5();
+            int imm4 = instr->ImmNEON4();
+            int tz = CountTrailingZeros(imm5, 32);
+            rd_index = imm5 >> (tz + 1);
+            rn_index = imm4 >> tz;
+            if (strncmp(format, "IVInsIndex1", strlen("IVInsIndex1")) == 0) {
+              AppendToOutput("%d", rd_index);
+              return strlen("IVInsIndex1");
+            } else if (strncmp(format, "IVInsIndex2",
+                       strlen("IVInsIndex2")) == 0) {
+              AppendToOutput("%d", rn_index);
+              return strlen("IVInsIndex2");
+            } else {
+              VIXL_UNIMPLEMENTED();
+              return 0;
+            }
+          }
+          VIXL_FALLTHROUGH();
+        }
+        case 'L': {  // IVLSLane[0123] - suffix indicates access size shift.
+          AppendToOutput("%d", instr->NEONLSIndex(format[8] - '0'));
+          return 9;
+        }
+        case 'M': {  // Modified Immediate cases.
+          if (strncmp(format,
+                      "IVMIImmFPSingle",
+                      strlen("IVMIImmFPSingle")) == 0) {
+            AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmNEONabcdefgh(),
+                           instr->ImmNEONFP32());
+            return strlen("IVMIImmFPSingle");
+          } else if (strncmp(format,
+                             "IVMIImmFPDouble",
+                             strlen("IVMIImmFPDouble")) == 0) {
+            AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmNEONabcdefgh(),
+                           instr->ImmNEONFP64());
+            return strlen("IVMIImmFPDouble");
+          } else if (strncmp(format, "IVMIImm8", strlen("IVMIImm8")) == 0) {
+            uint64_t imm8 = instr->ImmNEONabcdefgh();
+            AppendToOutput("#0x%" PRIx64, imm8);
+            return strlen("IVMIImm8");
+          } else if (strncmp(format, "IVMIImm", strlen("IVMIImm")) == 0) {
+            uint64_t imm8 = instr->ImmNEONabcdefgh();
+            uint64_t imm = 0;
+            for (int i = 0; i < 8; ++i) {
+              if (imm8 & (1 << i)) {
+                imm |= (UINT64_C(0xff) << (8 * i));
+              }
+            }
+            AppendToOutput("#0x%" PRIx64, imm);
+            return strlen("IVMIImm");
+          } else if (strncmp(format, "IVMIShiftAmt1",
+                             strlen("IVMIShiftAmt1")) == 0) {
+            int cmode = instr->NEONCmode();
+            int shift_amount = 8 * ((cmode >> 1) & 3);
+            AppendToOutput("#%d", shift_amount);
+            return strlen("IVMIShiftAmt1");
+          } else if (strncmp(format, "IVMIShiftAmt2",
+                             strlen("IVMIShiftAmt2")) == 0) {
+            int cmode = instr->NEONCmode();
+            int shift_amount = 8 << (cmode & 1);
+            AppendToOutput("#%d", shift_amount);
+            return strlen("IVMIShiftAmt2");
+          } else {
+            VIXL_UNIMPLEMENTED();
+            return 0;
+          }
+        }
+        default: {
+          VIXL_UNIMPLEMENTED();
+          return 0;
+        }
+      }
+    }
+    case 'X': {  // IX - CLREX instruction.
+      AppendToOutput("#0x%" PRIx32, instr->CRm());
+      return 2;
+    }
+    default: {
+      VIXL_UNIMPLEMENTED();
+      return 0;
+    }
+  }
+}
+
+
+int Disassembler::SubstituteBitfieldImmediateField(const Instruction* instr,
+                                                   const char* format) {
+  VIXL_ASSERT((format[0] == 'I') && (format[1] == 'B'));
+  unsigned r = instr->ImmR();
+  unsigned s = instr->ImmS();
+
+  switch (format[2]) {
+    case 'r': {  // IBr.
+      AppendToOutput("#%d", r);
+      return 3;
+    }
+    case 's': {  // IBs+1 or IBs-r+1.
+      if (format[3] == '+') {
+        AppendToOutput("#%d", s + 1);
+        return 5;
+      } else {
+        VIXL_ASSERT(format[3] == '-');
+        AppendToOutput("#%d", s - r + 1);
+        return 7;
+      }
+    }
+    case 'Z': {  // IBZ-r.
+      VIXL_ASSERT((format[3] == '-') && (format[4] == 'r'));
+      unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize;
+      AppendToOutput("#%d", reg_size - r);
+      return 5;
+    }
+    default: {
+      VIXL_UNREACHABLE();
+      return 0;
+    }
+  }
+}
+
+
+int Disassembler::SubstituteLiteralField(const Instruction* instr,
+                                         const char* format) {
+  VIXL_ASSERT(strncmp(format, "LValue", 6) == 0);
+  USE(format);
+
+  const void * address = instr->LiteralAddress<const void *>();
+  switch (instr->Mask(LoadLiteralMask)) {
+    case LDR_w_lit:
+    case LDR_x_lit:
+    case LDRSW_x_lit:
+    case LDR_s_lit:
+    case LDR_d_lit:
+    case LDR_q_lit:
+      AppendCodeRelativeDataAddressToOutput(instr, address);
+      break;
+    case PRFM_lit: {
+      // Use the prefetch hint to decide how to print the address.
+      switch (instr->PrefetchHint()) {
+        case 0x0:     // PLD: prefetch for load.
+        case 0x2:     // PST: prepare for store.
+          AppendCodeRelativeDataAddressToOutput(instr, address);
+          break;
+        case 0x1:     // PLI: preload instructions.
+          AppendCodeRelativeCodeAddressToOutput(instr, address);
+          break;
+        case 0x3:     // Unallocated hint.
+          AppendCodeRelativeAddressToOutput(instr, address);
+          break;
+      }
+      break;
+    }
+    default:
+      VIXL_UNREACHABLE();
+  }
+
+  return 6;
+}
+
+
+int Disassembler::SubstituteShiftField(const Instruction* instr,
+                                       const char* format) {
+  VIXL_ASSERT(format[0] == 'N');
+  VIXL_ASSERT(instr->ShiftDP() <= 0x3);
+
+  switch (format[1]) {
+    case 'D': {  // HDP.
+      VIXL_ASSERT(instr->ShiftDP() != ROR);
+      VIXL_FALLTHROUGH();
+    }
+    case 'L': {  // HLo.
+      if (instr->ImmDPShift() != 0) {
+        const char* shift_type[] = {"lsl", "lsr", "asr", "ror"};
+        AppendToOutput(", %s #%" PRId32, shift_type[instr->ShiftDP()],
+                       instr->ImmDPShift());
+      }
+      return 3;
+    }
+    default:
+      VIXL_UNIMPLEMENTED();
+      return 0;
+  }
+}
+
+
+int Disassembler::SubstituteConditionField(const Instruction* instr,
+                                           const char* format) {
+  VIXL_ASSERT(format[0] == 'C');
+  const char* condition_code[] = { "eq", "ne", "hs", "lo",
+                                   "mi", "pl", "vs", "vc",
+                                   "hi", "ls", "ge", "lt",
+                                   "gt", "le", "al", "nv" };
+  int cond;
+  switch (format[1]) {
+    case 'B': cond = instr->ConditionBranch(); break;
+    case 'I': {
+      cond = InvertCondition(static_cast<Condition>(instr->Condition()));
+      break;
+    }
+    default: cond = instr->Condition();
+  }
+  AppendToOutput("%s", condition_code[cond]);
+  return 4;
+}
+
+
+int Disassembler::SubstitutePCRelAddressField(const Instruction* instr,
+                                              const char* format) {
+  VIXL_ASSERT((strcmp(format, "AddrPCRelByte") == 0) ||   // Used by `adr`.
+              (strcmp(format, "AddrPCRelPage") == 0));    // Used by `adrp`.
+
+  int64_t offset = instr->ImmPCRel();
+
+  // Compute the target address based on the effective address (after applying
+  // code_address_offset). This is required for correct behaviour of adrp.
+  const Instruction* base = instr + code_address_offset();
+  if (format[9] == 'P') {
+    offset *= kPageSize;
+    base = AlignDown(base, kPageSize);
+  }
+  // Strip code_address_offset before printing, so we can use the
+  // semantically-correct AppendCodeRelativeAddressToOutput.
+  const void* target =
+      reinterpret_cast<const void*>(base + offset - code_address_offset());
+
+  AppendPCRelativeOffsetToOutput(instr, offset);
+  AppendToOutput(" ");
+  AppendCodeRelativeAddressToOutput(instr, target);
+  return 13;
+}
+
+
+int Disassembler::SubstituteBranchTargetField(const Instruction* instr,
+                                              const char* format) {
+  VIXL_ASSERT(strncmp(format, "TImm", 4) == 0);
+
+  int64_t offset = 0;
+  switch (format[5]) {
+    // BImmUncn - unconditional branch immediate.
+    case 'n': offset = instr->ImmUncondBranch(); break;
+    // BImmCond - conditional branch immediate.
+    case 'o': offset = instr->ImmCondBranch(); break;
+    // BImmCmpa - compare and branch immediate.
+    case 'm': offset = instr->ImmCmpBranch(); break;
+    // BImmTest - test and branch immediate.
+    case 'e': offset = instr->ImmTestBranch(); break;
+    default: VIXL_UNIMPLEMENTED();
+  }
+  offset <<= kInstructionSizeLog2;
+  const void* target_address = reinterpret_cast<const void*>(instr + offset);
+  VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
+
+  AppendPCRelativeOffsetToOutput(instr, offset);
+  AppendToOutput(" ");
+  AppendCodeRelativeCodeAddressToOutput(instr, target_address);
+
+  return 8;
+}
+
+
+int Disassembler::SubstituteExtendField(const Instruction* instr,
+                                        const char* format) {
+  VIXL_ASSERT(strncmp(format, "Ext", 3) == 0);
+  VIXL_ASSERT(instr->ExtendMode() <= 7);
+  USE(format);
+
+  const char* extend_mode[] = { "uxtb", "uxth", "uxtw", "uxtx",
+                                "sxtb", "sxth", "sxtw", "sxtx" };
+
+  // If rd or rn is SP, uxtw on 32-bit registers and uxtx on 64-bit
+  // registers becomes lsl.
+  if (((instr->Rd() == kZeroRegCode) || (instr->Rn() == kZeroRegCode)) &&
+      (((instr->ExtendMode() == UXTW) && (instr->SixtyFourBits() == 0)) ||
+       (instr->ExtendMode() == UXTX))) {
+    if (instr->ImmExtendShift() > 0) {
+      AppendToOutput(", lsl #%" PRId32, instr->ImmExtendShift());
+    }
+  } else {
+    AppendToOutput(", %s", extend_mode[instr->ExtendMode()]);
+    if (instr->ImmExtendShift() > 0) {
+      AppendToOutput(" #%" PRId32, instr->ImmExtendShift());
+    }
+  }
+  return 3;
+}
+
+
+int Disassembler::SubstituteLSRegOffsetField(const Instruction* instr,
+                                             const char* format) {
+  VIXL_ASSERT(strncmp(format, "Offsetreg", 9) == 0);
+  const char* extend_mode[] = { "undefined", "undefined", "uxtw", "lsl",
+                                "undefined", "undefined", "sxtw", "sxtx" };
+  USE(format);
+
+  unsigned shift = instr->ImmShiftLS();
+  Extend ext = static_cast<Extend>(instr->ExtendMode());
+  char reg_type = ((ext == UXTW) || (ext == SXTW)) ? 'w' : 'x';
+
+  unsigned rm = instr->Rm();
+  if (rm == kZeroRegCode) {
+    AppendToOutput("%czr", reg_type);
+  } else {
+    AppendToOutput("%c%d", reg_type, rm);
+  }
+
+  // Extend mode UXTX is an alias for shift mode LSL here.
+  if (!((ext == UXTX) && (shift == 0))) {
+    AppendToOutput(", %s", extend_mode[ext]);
+    if (shift != 0) {
+      AppendToOutput(" #%d", instr->SizeLS());
+    }
+  }
+  return 9;
+}
+
+
+int Disassembler::SubstitutePrefetchField(const Instruction* instr,
+                                          const char* format) {
+  VIXL_ASSERT(format[0] == 'P');
+  USE(format);
+
+  static const char* hints[] = {"ld", "li", "st"};
+  static const char* stream_options[] = {"keep", "strm"};
+
+  unsigned hint = instr->PrefetchHint();
+  unsigned target = instr->PrefetchTarget() + 1;
+  unsigned stream = instr->PrefetchStream();
+
+  if ((hint >= (sizeof(hints) / sizeof(hints[0]))) || (target > 3)) {
+    // Unallocated prefetch operations.
+    int prefetch_mode = instr->ImmPrefetchOperation();
+    AppendToOutput("#0b%c%c%c%c%c",
+                   (prefetch_mode & (1 << 4)) ? '1' : '0',
+                   (prefetch_mode & (1 << 3)) ? '1' : '0',
+                   (prefetch_mode & (1 << 2)) ? '1' : '0',
+                   (prefetch_mode & (1 << 1)) ? '1' : '0',
+                   (prefetch_mode & (1 << 0)) ? '1' : '0');
+  } else {
+    VIXL_ASSERT(stream < (sizeof(stream_options) / sizeof(stream_options[0])));
+    AppendToOutput("p%sl%d%s", hints[hint], target, stream_options[stream]);
+  }
+  return 6;
+}
+
+int Disassembler::SubstituteBarrierField(const Instruction* instr,
+                                         const char* format) {
+  VIXL_ASSERT(format[0] == 'M');
+  USE(format);
+
+  static const char* options[4][4] = {
+    { "sy (0b0000)", "oshld", "oshst", "osh" },
+    { "sy (0b0100)", "nshld", "nshst", "nsh" },
+    { "sy (0b1000)", "ishld", "ishst", "ish" },
+    { "sy (0b1100)", "ld", "st", "sy" }
+  };
+  int domain = instr->ImmBarrierDomain();
+  int type = instr->ImmBarrierType();
+
+  AppendToOutput("%s", options[domain][type]);
+  return 1;
+}
+
+int Disassembler::SubstituteSysOpField(const Instruction* instr,
+                                       const char* format) {
+  VIXL_ASSERT(format[0] == 'G');
+  int op = -1;
+  switch (format[1]) {
+    case '1': op = instr->SysOp1(); break;
+    case '2': op = instr->SysOp2(); break;
+    default:
+      VIXL_UNREACHABLE();
+  }
+  AppendToOutput("#%d", op);
+  return 2;
+}
+
+int Disassembler::SubstituteCrField(const Instruction* instr,
+                                    const char* format) {
+  VIXL_ASSERT(format[0] == 'K');
+  int cr = -1;
+  switch (format[1]) {
+    case 'n': cr = instr->CRn(); break;
+    case 'm': cr = instr->CRm(); break;
+    default:
+      VIXL_UNREACHABLE();
+  }
+  AppendToOutput("C%d", cr);
+  return 2;
+}
+
+void Disassembler::ResetOutput() {
+  buffer_pos_ = 0;
+  buffer_[buffer_pos_] = 0;
+}
+
+
+void Disassembler::AppendToOutput(const char* format, ...) {
+  va_list args;
+  va_start(args, format);
+  buffer_pos_ += vsnprintf(&buffer_[buffer_pos_], buffer_size_ - buffer_pos_,
+          format, args);
+  va_end(args);
+}
+
+
+void PrintDisassembler::ProcessOutput(const Instruction* instr) {
+  fprintf(stream_, "0x%016" PRIx64 "  %08" PRIx32 "\t\t%s\n",
+          reinterpret_cast<uint64_t>(instr),
+          instr->InstructionBits(),
+          GetOutput());
+}
+
+}  // namespace vixl
diff --git a/disas/libvixl/vixl/a64/disasm-a64.h b/disas/libvixl/vixl/a64/disasm-a64.h
new file mode 100644
index 0000000..930df6e
--- /dev/null
+++ b/disas/libvixl/vixl/a64/disasm-a64.h
@@ -0,0 +1,177 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_DISASM_A64_H
+#define VIXL_A64_DISASM_A64_H
+
+#include "vixl/globals.h"
+#include "vixl/utils.h"
+#include "vixl/a64/instructions-a64.h"
+#include "vixl/a64/decoder-a64.h"
+#include "vixl/a64/assembler-a64.h"
+
+namespace vixl {
+
+class Disassembler: public DecoderVisitor {
+ public:
+  Disassembler();
+  Disassembler(char* text_buffer, int buffer_size);
+  virtual ~Disassembler();
+  char* GetOutput();
+
+  // Declare all Visitor functions.
+  #define DECLARE(A) virtual void Visit##A(const Instruction* instr);
+  VISITOR_LIST(DECLARE)
+  #undef DECLARE
+
+ protected:
+  virtual void ProcessOutput(const Instruction* instr);
+
+  // Default output functions.  The functions below implement a default way of
+  // printing elements in the disassembly. A sub-class can override these to
+  // customize the disassembly output.
+
+  // Prints the name of a register.
+  // TODO: This currently doesn't allow renaming of V registers.
+  virtual void AppendRegisterNameToOutput(const Instruction* instr,
+                                          const CPURegister& reg);
+
+  // Prints a PC-relative offset. This is used for example when disassembling
+  // branches to immediate offsets.
+  virtual void AppendPCRelativeOffsetToOutput(const Instruction* instr,
+                                              int64_t offset);
+
+  // Prints an address, in the general case. It can be code or data. This is
+  // used for example to print the target address of an ADR instruction.
+  virtual void AppendCodeRelativeAddressToOutput(const Instruction* instr,
+                                                 const void* addr);
+
+  // Prints the address of some code.
+  // This is used for example to print the target address of a branch to an
+  // immediate offset.
+  // A sub-class can for example override this method to lookup the address and
+  // print an appropriate name.
+  virtual void AppendCodeRelativeCodeAddressToOutput(const Instruction* instr,
+                                                     const void* addr);
+
+  // Prints the address of some data.
+  // This is used for example to print the source address of a load literal
+  // instruction.
+  virtual void AppendCodeRelativeDataAddressToOutput(const Instruction* instr,
+                                                     const void* addr);
+
+  // Same as the above, but for addresses that are not relative to the code
+  // buffer. They are currently not used by VIXL.
+  virtual void AppendAddressToOutput(const Instruction* instr,
+                                     const void* addr);
+  virtual void AppendCodeAddressToOutput(const Instruction* instr,
+                                         const void* addr);
+  virtual void AppendDataAddressToOutput(const Instruction* instr,
+                                         const void* addr);
+
+ public:
+  // Get/Set the offset that should be added to code addresses when printing
+  // code-relative addresses in the AppendCodeRelative<Type>AddressToOutput()
+  // helpers.
+  // Below is an example of how a branch immediate instruction in memory at
+  // address 0xb010200 would disassemble with different offsets.
+  // Base address | Disassembly
+  //          0x0 | 0xb010200:  b #+0xcc  (addr 0xb0102cc)
+  //      0x10000 | 0xb000200:  b #+0xcc  (addr 0xb0002cc)
+  //    0xb010200 |       0x0:  b #+0xcc  (addr 0xcc)
+  void MapCodeAddress(int64_t base_address, const Instruction* instr_address);
+  int64_t CodeRelativeAddress(const void* instr);
+
+ private:
+  void Format(
+      const Instruction* instr, const char* mnemonic, const char* format);
+  void Substitute(const Instruction* instr, const char* string);
+  int SubstituteField(const Instruction* instr, const char* format);
+  int SubstituteRegisterField(const Instruction* instr, const char* format);
+  int SubstituteImmediateField(const Instruction* instr, const char* format);
+  int SubstituteLiteralField(const Instruction* instr, const char* format);
+  int SubstituteBitfieldImmediateField(
+      const Instruction* instr, const char* format);
+  int SubstituteShiftField(const Instruction* instr, const char* format);
+  int SubstituteExtendField(const Instruction* instr, const char* format);
+  int SubstituteConditionField(const Instruction* instr, const char* format);
+  int SubstitutePCRelAddressField(const Instruction* instr, const char* format);
+  int SubstituteBranchTargetField(const Instruction* instr, const char* format);
+  int SubstituteLSRegOffsetField(const Instruction* instr, const char* format);
+  int SubstitutePrefetchField(const Instruction* instr, const char* format);
+  int SubstituteBarrierField(const Instruction* instr, const char* format);
+  int SubstituteSysOpField(const Instruction* instr, const char* format);
+  int SubstituteCrField(const Instruction* instr, const char* format);
+  bool RdIsZROrSP(const Instruction* instr) const {
+    return (instr->Rd() == kZeroRegCode);
+  }
+
+  bool RnIsZROrSP(const Instruction* instr) const {
+    return (instr->Rn() == kZeroRegCode);
+  }
+
+  bool RmIsZROrSP(const Instruction* instr) const {
+    return (instr->Rm() == kZeroRegCode);
+  }
+
+  bool RaIsZROrSP(const Instruction* instr) const {
+    return (instr->Ra() == kZeroRegCode);
+  }
+
+  bool IsMovzMovnImm(unsigned reg_size, uint64_t value);
+
+  int64_t code_address_offset() const { return code_address_offset_; }
+
+ protected:
+  void ResetOutput();
+  void AppendToOutput(const char* string, ...) PRINTF_CHECK(2, 3);
+
+  void set_code_address_offset(int64_t code_address_offset) {
+    code_address_offset_ = code_address_offset;
+  }
+
+  char* buffer_;
+  uint32_t buffer_pos_;
+  uint32_t buffer_size_;
+  bool own_buffer_;
+
+  int64_t code_address_offset_;
+};
+
+
+class PrintDisassembler: public Disassembler {
+ public:
+  explicit PrintDisassembler(FILE* stream) : stream_(stream) { }
+
+ protected:
+  virtual void ProcessOutput(const Instruction* instr);
+
+ private:
+  FILE *stream_;
+};
+}  // namespace vixl
+
+#endif  // VIXL_A64_DISASM_A64_H
diff --git a/disas/libvixl/vixl/a64/instructions-a64.cc b/disas/libvixl/vixl/a64/instructions-a64.cc
new file mode 100644
index 0000000..33992f8
--- /dev/null
+++ b/disas/libvixl/vixl/a64/instructions-a64.cc
@@ -0,0 +1,622 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "vixl/a64/instructions-a64.h"
+#include "vixl/a64/assembler-a64.h"
+
+namespace vixl {
+
+
+// Floating-point infinity values.
+const float16 kFP16PositiveInfinity = 0x7c00;
+const float16 kFP16NegativeInfinity = 0xfc00;
+const float kFP32PositiveInfinity = rawbits_to_float(0x7f800000);
+const float kFP32NegativeInfinity = rawbits_to_float(0xff800000);
+const double kFP64PositiveInfinity =
+    rawbits_to_double(UINT64_C(0x7ff0000000000000));
+const double kFP64NegativeInfinity =
+    rawbits_to_double(UINT64_C(0xfff0000000000000));
+
+
+// The default NaN values (for FPCR.DN=1).
+const double kFP64DefaultNaN = rawbits_to_double(UINT64_C(0x7ff8000000000000));
+const float kFP32DefaultNaN = rawbits_to_float(0x7fc00000);
+const float16 kFP16DefaultNaN = 0x7e00;
+
+
+static uint64_t RotateRight(uint64_t value,
+                            unsigned int rotate,
+                            unsigned int width) {
+  VIXL_ASSERT(width <= 64);
+  rotate &= 63;
+  return ((value & ((UINT64_C(1) << rotate) - 1)) <<
+          (width - rotate)) | (value >> rotate);
+}
+
+
+static uint64_t RepeatBitsAcrossReg(unsigned reg_size,
+                                    uint64_t value,
+                                    unsigned width) {
+  VIXL_ASSERT((width == 2) || (width == 4) || (width == 8) || (width == 16) ||
+              (width == 32));
+  VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+  uint64_t result = value & ((UINT64_C(1) << width) - 1);
+  for (unsigned i = width; i < reg_size; i *= 2) {
+    result |= (result << i);
+  }
+  return result;
+}
+
+
+bool Instruction::IsLoad() const {
+  if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+    return false;
+  }
+
+  if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+    return Mask(LoadStorePairLBit) != 0;
+  } else {
+    LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+    switch (op) {
+      case LDRB_w:
+      case LDRH_w:
+      case LDR_w:
+      case LDR_x:
+      case LDRSB_w:
+      case LDRSB_x:
+      case LDRSH_w:
+      case LDRSH_x:
+      case LDRSW_x:
+      case LDR_b:
+      case LDR_h:
+      case LDR_s:
+      case LDR_d:
+      case LDR_q: return true;
+      default: return false;
+    }
+  }
+}
+
+
+bool Instruction::IsStore() const {
+  if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+    return false;
+  }
+
+  if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+    return Mask(LoadStorePairLBit) == 0;
+  } else {
+    LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+    switch (op) {
+      case STRB_w:
+      case STRH_w:
+      case STR_w:
+      case STR_x:
+      case STR_b:
+      case STR_h:
+      case STR_s:
+      case STR_d:
+      case STR_q: return true;
+      default: return false;
+    }
+  }
+}
+
+
+// Logical immediates can't encode zero, so a return value of zero is used to
+// indicate a failure case. Specifically, where the constraints on imm_s are
+// not met.
+uint64_t Instruction::ImmLogical() const {
+  unsigned reg_size = SixtyFourBits() ? kXRegSize : kWRegSize;
+  int32_t n = BitN();
+  int32_t imm_s = ImmSetBits();
+  int32_t imm_r = ImmRotate();
+
+  // An integer is constructed from the n, imm_s and imm_r bits according to
+  // the following table:
+  //
+  //  N   imms    immr    size        S             R
+  //  1  ssssss  rrrrrr    64    UInt(ssssss)  UInt(rrrrrr)
+  //  0  0sssss  xrrrrr    32    UInt(sssss)   UInt(rrrrr)
+  //  0  10ssss  xxrrrr    16    UInt(ssss)    UInt(rrrr)
+  //  0  110sss  xxxrrr     8    UInt(sss)     UInt(rrr)
+  //  0  1110ss  xxxxrr     4    UInt(ss)      UInt(rr)
+  //  0  11110s  xxxxxr     2    UInt(s)       UInt(r)
+  // (s bits must not be all set)
+  //
+  // A pattern is constructed of size bits, where the least significant S+1
+  // bits are set. The pattern is rotated right by R, and repeated across a
+  // 32 or 64-bit value, depending on destination register width.
+  //
+
+  if (n == 1) {
+    if (imm_s == 0x3f) {
+      return 0;
+    }
+    uint64_t bits = (UINT64_C(1) << (imm_s + 1)) - 1;
+    return RotateRight(bits, imm_r, 64);
+  } else {
+    if ((imm_s >> 1) == 0x1f) {
+      return 0;
+    }
+    for (int width = 0x20; width >= 0x2; width >>= 1) {
+      if ((imm_s & width) == 0) {
+        int mask = width - 1;
+        if ((imm_s & mask) == mask) {
+          return 0;
+        }
+        uint64_t bits = (UINT64_C(1) << ((imm_s & mask) + 1)) - 1;
+        return RepeatBitsAcrossReg(reg_size,
+                                   RotateRight(bits, imm_r & mask, width),
+                                   width);
+      }
+    }
+  }
+  VIXL_UNREACHABLE();
+  return 0;
+}
+
+
+uint32_t Instruction::ImmNEONabcdefgh() const {
+  return ImmNEONabc() << 5 | ImmNEONdefgh();
+}
+
+
+float Instruction::Imm8ToFP32(uint32_t imm8) {
+  //   Imm8: abcdefgh (8 bits)
+  // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
+  // where B is b ^ 1
+  uint32_t bits = imm8;
+  uint32_t bit7 = (bits >> 7) & 0x1;
+  uint32_t bit6 = (bits >> 6) & 0x1;
+  uint32_t bit5_to_0 = bits & 0x3f;
+  uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
+
+  return rawbits_to_float(result);
+}
+
+
+float Instruction::ImmFP32() const {
+  return Imm8ToFP32(ImmFP());
+}
+
+
+double Instruction::Imm8ToFP64(uint32_t imm8) {
+  //   Imm8: abcdefgh (8 bits)
+  // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+  //         0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
+  // where B is b ^ 1
+  uint32_t bits = imm8;
+  uint64_t bit7 = (bits >> 7) & 0x1;
+  uint64_t bit6 = (bits >> 6) & 0x1;
+  uint64_t bit5_to_0 = bits & 0x3f;
+  uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
+
+  return rawbits_to_double(result);
+}
+
+
+double Instruction::ImmFP64() const {
+  return Imm8ToFP64(ImmFP());
+}
+
+
+float Instruction::ImmNEONFP32() const {
+  return Imm8ToFP32(ImmNEONabcdefgh());
+}
+
+
+double Instruction::ImmNEONFP64() const {
+  return Imm8ToFP64(ImmNEONabcdefgh());
+}
+
+
+unsigned CalcLSDataSize(LoadStoreOp op) {
+  VIXL_ASSERT((LSSize_offset + LSSize_width) == (kInstructionSize * 8));
+  unsigned size = static_cast<Instr>(op) >> LSSize_offset;
+  if ((op & LSVector_mask) != 0) {
+    // Vector register memory operations encode the access size in the "size"
+    // and "opc" fields.
+    if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) {
+      size = kQRegSizeInBytesLog2;
+    }
+  }
+  return size;
+}
+
+
+unsigned CalcLSPairDataSize(LoadStorePairOp op) {
+  VIXL_STATIC_ASSERT(kXRegSizeInBytes == kDRegSizeInBytes);
+  VIXL_STATIC_ASSERT(kWRegSizeInBytes == kSRegSizeInBytes);
+  switch (op) {
+    case STP_q:
+    case LDP_q: return kQRegSizeInBytesLog2;
+    case STP_x:
+    case LDP_x:
+    case STP_d:
+    case LDP_d: return kXRegSizeInBytesLog2;
+    default: return kWRegSizeInBytesLog2;
+  }
+}
+
+
+int Instruction::ImmBranchRangeBitwidth(ImmBranchType branch_type) {
+  switch (branch_type) {
+    case UncondBranchType:
+      return ImmUncondBranch_width;
+    case CondBranchType:
+      return ImmCondBranch_width;
+    case CompareBranchType:
+      return ImmCmpBranch_width;
+    case TestBranchType:
+      return ImmTestBranch_width;
+    default:
+      VIXL_UNREACHABLE();
+      return 0;
+  }
+}
+
+
+int32_t Instruction::ImmBranchForwardRange(ImmBranchType branch_type) {
+  int32_t encoded_max = 1 << (ImmBranchRangeBitwidth(branch_type) - 1);
+  return encoded_max * kInstructionSize;
+}
+
+
+bool Instruction::IsValidImmPCOffset(ImmBranchType branch_type,
+                                     int64_t offset) {
+  return is_intn(ImmBranchRangeBitwidth(branch_type), offset);
+}
+
+
+const Instruction* Instruction::ImmPCOffsetTarget() const {
+  const Instruction * base = this;
+  ptrdiff_t offset;
+  if (IsPCRelAddressing()) {
+    // ADR and ADRP.
+    offset = ImmPCRel();
+    if (Mask(PCRelAddressingMask) == ADRP) {
+      base = AlignDown(base, kPageSize);
+      offset *= kPageSize;
+    } else {
+      VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
+    }
+  } else {
+    // All PC-relative branches.
+    VIXL_ASSERT(BranchType() != UnknownBranchType);
+    // Relative branch offsets are instruction-size-aligned.
+    offset = ImmBranch() << kInstructionSizeLog2;
+  }
+  return base + offset;
+}
+
+
+int Instruction::ImmBranch() const {
+  switch (BranchType()) {
+    case CondBranchType: return ImmCondBranch();
+    case UncondBranchType: return ImmUncondBranch();
+    case CompareBranchType: return ImmCmpBranch();
+    case TestBranchType: return ImmTestBranch();
+    default: VIXL_UNREACHABLE();
+  }
+  return 0;
+}
+
+
+void Instruction::SetImmPCOffsetTarget(const Instruction* target) {
+  if (IsPCRelAddressing()) {
+    SetPCRelImmTarget(target);
+  } else {
+    SetBranchImmTarget(target);
+  }
+}
+
+
+void Instruction::SetPCRelImmTarget(const Instruction* target) {
+  ptrdiff_t imm21;
+  if ((Mask(PCRelAddressingMask) == ADR)) {
+    imm21 = target - this;
+  } else {
+    VIXL_ASSERT(Mask(PCRelAddressingMask) == ADRP);
+    uintptr_t this_page = reinterpret_cast<uintptr_t>(this) / kPageSize;
+    uintptr_t target_page = reinterpret_cast<uintptr_t>(target) / kPageSize;
+    imm21 = target_page - this_page;
+  }
+  Instr imm = Assembler::ImmPCRelAddress(static_cast<int32_t>(imm21));
+
+  SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
+}
+
+
+void Instruction::SetBranchImmTarget(const Instruction* target) {
+  VIXL_ASSERT(((target - this) & 3) == 0);
+  Instr branch_imm = 0;
+  uint32_t imm_mask = 0;
+  int offset = static_cast<int>((target - this) >> kInstructionSizeLog2);
+  switch (BranchType()) {
+    case CondBranchType: {
+      branch_imm = Assembler::ImmCondBranch(offset);
+      imm_mask = ImmCondBranch_mask;
+      break;
+    }
+    case UncondBranchType: {
+      branch_imm = Assembler::ImmUncondBranch(offset);
+      imm_mask = ImmUncondBranch_mask;
+      break;
+    }
+    case CompareBranchType: {
+      branch_imm = Assembler::ImmCmpBranch(offset);
+      imm_mask = ImmCmpBranch_mask;
+      break;
+    }
+    case TestBranchType: {
+      branch_imm = Assembler::ImmTestBranch(offset);
+      imm_mask = ImmTestBranch_mask;
+      break;
+    }
+    default: VIXL_UNREACHABLE();
+  }
+  SetInstructionBits(Mask(~imm_mask) | branch_imm);
+}
+
+
+void Instruction::SetImmLLiteral(const Instruction* source) {
+  VIXL_ASSERT(IsWordAligned(source));
+  ptrdiff_t offset = (source - this) >> kLiteralEntrySizeLog2;
+  Instr imm = Assembler::ImmLLiteral(static_cast<int>(offset));
+  Instr mask = ImmLLiteral_mask;
+
+  SetInstructionBits(Mask(~mask) | imm);
+}
+
+
+VectorFormat VectorFormatHalfWidth(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat8H || vform == kFormat4S || vform == kFormat2D ||
+              vform == kFormatH || vform == kFormatS || vform == kFormatD);
+  switch (vform) {
+    case kFormat8H: return kFormat8B;
+    case kFormat4S: return kFormat4H;
+    case kFormat2D: return kFormat2S;
+    case kFormatH:  return kFormatB;
+    case kFormatS:  return kFormatH;
+    case kFormatD:  return kFormatS;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat VectorFormatDoubleWidth(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S ||
+              vform == kFormatB || vform == kFormatH || vform == kFormatS);
+  switch (vform) {
+    case kFormat8B: return kFormat8H;
+    case kFormat4H: return kFormat4S;
+    case kFormat2S: return kFormat2D;
+    case kFormatB:  return kFormatH;
+    case kFormatH:  return kFormatS;
+    case kFormatS:  return kFormatD;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat VectorFormatFillQ(const VectorFormat vform) {
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return kFormat16B;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H:  return kFormat8H;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S:  return kFormat4S;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D:  return kFormat2D;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+VectorFormat VectorFormatHalfWidthDoubleLanes(const VectorFormat vform) {
+  switch (vform) {
+    case kFormat4H: return kFormat8B;
+    case kFormat8H: return kFormat16B;
+    case kFormat2S: return kFormat4H;
+    case kFormat4S: return kFormat8H;
+    case kFormat1D: return kFormat2S;
+    case kFormat2D: return kFormat4S;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+VectorFormat VectorFormatDoubleLanes(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S);
+  switch (vform) {
+    case kFormat8B: return kFormat16B;
+    case kFormat4H: return kFormat8H;
+    case kFormat2S: return kFormat4S;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat VectorFormatHalfLanes(const VectorFormat vform) {
+  VIXL_ASSERT(vform == kFormat16B || vform == kFormat8H || vform == kFormat4S);
+  switch (vform) {
+    case kFormat16B: return kFormat8B;
+    case kFormat8H: return kFormat4H;
+    case kFormat4S: return kFormat2S;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+VectorFormat ScalarFormatFromLaneSize(int laneSize) {
+  switch (laneSize) {
+    case 8:  return kFormatB;
+    case 16: return kFormatH;
+    case 32: return kFormatS;
+    case 64: return kFormatD;
+    default: VIXL_UNREACHABLE(); return kFormatUndefined;
+  }
+}
+
+
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB: return kBRegSize;
+    case kFormatH: return kHRegSize;
+    case kFormatS: return kSRegSize;
+    case kFormatD: return kDRegSize;
+    case kFormat8B:
+    case kFormat4H:
+    case kFormat2S:
+    case kFormat1D: return kDRegSize;
+    default: return kQRegSize;
+  }
+}
+
+
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform) {
+  return RegisterSizeInBitsFromFormat(vform) / 8;
+}
+
+
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return 8;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H: return 16;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S: return 32;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D: return 64;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+int LaneSizeInBytesFromFormat(VectorFormat vform) {
+  return LaneSizeInBitsFromFormat(vform) / 8;
+}
+
+
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return 0;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H: return 1;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S: return 2;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D: return 3;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+int LaneCountFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormat16B: return 16;
+    case kFormat8B:
+    case kFormat8H: return 8;
+    case kFormat4H:
+    case kFormat4S: return 4;
+    case kFormat2S:
+    case kFormat2D: return 2;
+    case kFormat1D:
+    case kFormatB:
+    case kFormatH:
+    case kFormatS:
+    case kFormatD: return 1;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+int MaxLaneCountFromFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormat8B:
+    case kFormat16B: return 16;
+    case kFormatH:
+    case kFormat4H:
+    case kFormat8H: return 8;
+    case kFormatS:
+    case kFormat2S:
+    case kFormat4S: return 4;
+    case kFormatD:
+    case kFormat1D:
+    case kFormat2D: return 2;
+    default: VIXL_UNREACHABLE(); return 0;
+  }
+}
+
+
+// Does 'vform' indicate a vector format or a scalar format?
+bool IsVectorFormat(VectorFormat vform) {
+  VIXL_ASSERT(vform != kFormatUndefined);
+  switch (vform) {
+    case kFormatB:
+    case kFormatH:
+    case kFormatS:
+    case kFormatD: return false;
+    default: return true;
+  }
+}
+
+
+int64_t MaxIntFromFormat(VectorFormat vform) {
+  return INT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+
+int64_t MinIntFromFormat(VectorFormat vform) {
+  return INT64_MIN >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+
+uint64_t MaxUintFromFormat(VectorFormat vform) {
+  return UINT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+}  // namespace vixl
+
diff --git a/disas/libvixl/vixl/a64/instructions-a64.h b/disas/libvixl/vixl/a64/instructions-a64.h
new file mode 100644
index 0000000..7e0dbae
--- /dev/null
+++ b/disas/libvixl/vixl/a64/instructions-a64.h
@@ -0,0 +1,757 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_INSTRUCTIONS_A64_H_
+#define VIXL_A64_INSTRUCTIONS_A64_H_
+
+#include "vixl/globals.h"
+#include "vixl/utils.h"
+#include "vixl/a64/constants-a64.h"
+
+namespace vixl {
+// ISA constants. --------------------------------------------------------------
+
+typedef uint32_t Instr;
+const unsigned kInstructionSize = 4;
+const unsigned kInstructionSizeLog2 = 2;
+const unsigned kLiteralEntrySize = 4;
+const unsigned kLiteralEntrySizeLog2 = 2;
+const unsigned kMaxLoadLiteralRange = 1 * MBytes;
+
+// This is the nominal page size (as used by the adrp instruction); the actual
+// size of the memory pages allocated by the kernel is likely to differ.
+const unsigned kPageSize = 4 * KBytes;
+const unsigned kPageSizeLog2 = 12;
+
+const unsigned kBRegSize = 8;
+const unsigned kBRegSizeLog2 = 3;
+const unsigned kBRegSizeInBytes = kBRegSize / 8;
+const unsigned kBRegSizeInBytesLog2 = kBRegSizeLog2 - 3;
+const unsigned kHRegSize = 16;
+const unsigned kHRegSizeLog2 = 4;
+const unsigned kHRegSizeInBytes = kHRegSize / 8;
+const unsigned kHRegSizeInBytesLog2 = kHRegSizeLog2 - 3;
+const unsigned kWRegSize = 32;
+const unsigned kWRegSizeLog2 = 5;
+const unsigned kWRegSizeInBytes = kWRegSize / 8;
+const unsigned kWRegSizeInBytesLog2 = kWRegSizeLog2 - 3;
+const unsigned kXRegSize = 64;
+const unsigned kXRegSizeLog2 = 6;
+const unsigned kXRegSizeInBytes = kXRegSize / 8;
+const unsigned kXRegSizeInBytesLog2 = kXRegSizeLog2 - 3;
+const unsigned kSRegSize = 32;
+const unsigned kSRegSizeLog2 = 5;
+const unsigned kSRegSizeInBytes = kSRegSize / 8;
+const unsigned kSRegSizeInBytesLog2 = kSRegSizeLog2 - 3;
+const unsigned kDRegSize = 64;
+const unsigned kDRegSizeLog2 = 6;
+const unsigned kDRegSizeInBytes = kDRegSize / 8;
+const unsigned kDRegSizeInBytesLog2 = kDRegSizeLog2 - 3;
+const unsigned kQRegSize = 128;
+const unsigned kQRegSizeLog2 = 7;
+const unsigned kQRegSizeInBytes = kQRegSize / 8;
+const unsigned kQRegSizeInBytesLog2 = kQRegSizeLog2 - 3;
+const uint64_t kWRegMask = UINT64_C(0xffffffff);
+const uint64_t kXRegMask = UINT64_C(0xffffffffffffffff);
+const uint64_t kSRegMask = UINT64_C(0xffffffff);
+const uint64_t kDRegMask = UINT64_C(0xffffffffffffffff);
+const uint64_t kSSignMask = UINT64_C(0x80000000);
+const uint64_t kDSignMask = UINT64_C(0x8000000000000000);
+const uint64_t kWSignMask = UINT64_C(0x80000000);
+const uint64_t kXSignMask = UINT64_C(0x8000000000000000);
+const uint64_t kByteMask = UINT64_C(0xff);
+const uint64_t kHalfWordMask = UINT64_C(0xffff);
+const uint64_t kWordMask = UINT64_C(0xffffffff);
+const uint64_t kXMaxUInt = UINT64_C(0xffffffffffffffff);
+const uint64_t kWMaxUInt = UINT64_C(0xffffffff);
+const int64_t kXMaxInt = INT64_C(0x7fffffffffffffff);
+const int64_t kXMinInt = INT64_C(0x8000000000000000);
+const int32_t kWMaxInt = INT32_C(0x7fffffff);
+const int32_t kWMinInt = INT32_C(0x80000000);
+const unsigned kLinkRegCode = 30;
+const unsigned kZeroRegCode = 31;
+const unsigned kSPRegInternalCode = 63;
+const unsigned kRegCodeMask = 0x1f;
+
+const unsigned kAddressTagOffset = 56;
+const unsigned kAddressTagWidth = 8;
+const uint64_t kAddressTagMask =
+    ((UINT64_C(1) << kAddressTagWidth) - 1) << kAddressTagOffset;
+VIXL_STATIC_ASSERT(kAddressTagMask == UINT64_C(0xff00000000000000));
+
+// AArch64 floating-point specifics. These match IEEE-754.
+const unsigned kDoubleMantissaBits = 52;
+const unsigned kDoubleExponentBits = 11;
+const unsigned kFloatMantissaBits = 23;
+const unsigned kFloatExponentBits = 8;
+const unsigned kFloat16MantissaBits = 10;
+const unsigned kFloat16ExponentBits = 5;
+
+// Floating-point infinity values.
+extern const float16 kFP16PositiveInfinity;
+extern const float16 kFP16NegativeInfinity;
+extern const float kFP32PositiveInfinity;
+extern const float kFP32NegativeInfinity;
+extern const double kFP64PositiveInfinity;
+extern const double kFP64NegativeInfinity;
+
+// The default NaN values (for FPCR.DN=1).
+extern const float16 kFP16DefaultNaN;
+extern const float kFP32DefaultNaN;
+extern const double kFP64DefaultNaN;
+
+unsigned CalcLSDataSize(LoadStoreOp op);
+unsigned CalcLSPairDataSize(LoadStorePairOp op);
+
+enum ImmBranchType {
+  UnknownBranchType = 0,
+  CondBranchType    = 1,
+  UncondBranchType  = 2,
+  CompareBranchType = 3,
+  TestBranchType    = 4
+};
+
+enum AddrMode {
+  Offset,
+  PreIndex,
+  PostIndex
+};
+
+enum FPRounding {
+  // The first four values are encodable directly by FPCR<RMode>.
+  FPTieEven = 0x0,
+  FPPositiveInfinity = 0x1,
+  FPNegativeInfinity = 0x2,
+  FPZero = 0x3,
+
+  // The final rounding modes are only available when explicitly specified by
+  // the instruction (such as with fcvta). It cannot be set in FPCR.
+  FPTieAway,
+  FPRoundOdd
+};
+
+enum Reg31Mode {
+  Reg31IsStackPointer,
+  Reg31IsZeroRegister
+};
+
+// Instructions. ---------------------------------------------------------------
+
+class Instruction {
+ public:
+  Instr InstructionBits() const {
+    return *(reinterpret_cast<const Instr*>(this));
+  }
+
+  void SetInstructionBits(Instr new_instr) {
+    *(reinterpret_cast<Instr*>(this)) = new_instr;
+  }
+
+  int Bit(int pos) const {
+    return (InstructionBits() >> pos) & 1;
+  }
+
+  uint32_t Bits(int msb, int lsb) const {
+    return unsigned_bitextract_32(msb, lsb, InstructionBits());
+  }
+
+  int32_t SignedBits(int msb, int lsb) const {
+    int32_t bits = *(reinterpret_cast<const int32_t*>(this));
+    return signed_bitextract_32(msb, lsb, bits);
+  }
+
+  Instr Mask(uint32_t mask) const {
+    return InstructionBits() & mask;
+  }
+
+  #define DEFINE_GETTER(Name, HighBit, LowBit, Func)             \
+  int32_t Name() const { return Func(HighBit, LowBit); }
+  INSTRUCTION_FIELDS_LIST(DEFINE_GETTER)
+  #undef DEFINE_GETTER
+
+  // ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST),
+  // formed from ImmPCRelLo and ImmPCRelHi.
+  int ImmPCRel() const {
+    int offset =
+        static_cast<int>((ImmPCRelHi() << ImmPCRelLo_width) | ImmPCRelLo());
+    int width = ImmPCRelLo_width + ImmPCRelHi_width;
+    return signed_bitextract_32(width - 1, 0, offset);
+  }
+
+  uint64_t ImmLogical() const;
+  unsigned ImmNEONabcdefgh() const;
+  float ImmFP32() const;
+  double ImmFP64() const;
+  float ImmNEONFP32() const;
+  double ImmNEONFP64() const;
+
+  unsigned SizeLS() const {
+    return CalcLSDataSize(static_cast<LoadStoreOp>(Mask(LoadStoreMask)));
+  }
+
+  unsigned SizeLSPair() const {
+    return CalcLSPairDataSize(
+        static_cast<LoadStorePairOp>(Mask(LoadStorePairMask)));
+  }
+
+  int NEONLSIndex(int access_size_shift) const {
+    int64_t q = NEONQ();
+    int64_t s = NEONS();
+    int64_t size = NEONLSSize();
+    int64_t index = (q << 3) | (s << 2) | size;
+    return static_cast<int>(index >> access_size_shift);
+  }
+
+  // Helpers.
+  bool IsCondBranchImm() const {
+    return Mask(ConditionalBranchFMask) == ConditionalBranchFixed;
+  }
+
+  bool IsUncondBranchImm() const {
+    return Mask(UnconditionalBranchFMask) == UnconditionalBranchFixed;
+  }
+
+  bool IsCompareBranch() const {
+    return Mask(CompareBranchFMask) == CompareBranchFixed;
+  }
+
+  bool IsTestBranch() const {
+    return Mask(TestBranchFMask) == TestBranchFixed;
+  }
+
+  bool IsImmBranch() const {
+    return BranchType() != UnknownBranchType;
+  }
+
+  bool IsPCRelAddressing() const {
+    return Mask(PCRelAddressingFMask) == PCRelAddressingFixed;
+  }
+
+  bool IsLogicalImmediate() const {
+    return Mask(LogicalImmediateFMask) == LogicalImmediateFixed;
+  }
+
+  bool IsAddSubImmediate() const {
+    return Mask(AddSubImmediateFMask) == AddSubImmediateFixed;
+  }
+
+  bool IsAddSubExtended() const {
+    return Mask(AddSubExtendedFMask) == AddSubExtendedFixed;
+  }
+
+  bool IsLoadOrStore() const {
+    return Mask(LoadStoreAnyFMask) == LoadStoreAnyFixed;
+  }
+
+  bool IsLoad() const;
+  bool IsStore() const;
+
+  bool IsLoadLiteral() const {
+    // This includes PRFM_lit.
+    return Mask(LoadLiteralFMask) == LoadLiteralFixed;
+  }
+
+  bool IsMovn() const {
+    return (Mask(MoveWideImmediateMask) == MOVN_x) ||
+           (Mask(MoveWideImmediateMask) == MOVN_w);
+  }
+
+  static int ImmBranchRangeBitwidth(ImmBranchType branch_type);
+  static int32_t ImmBranchForwardRange(ImmBranchType branch_type);
+  static bool IsValidImmPCOffset(ImmBranchType branch_type, int64_t offset);
+
+  // Indicate whether Rd can be the stack pointer or the zero register. This
+  // does not check that the instruction actually has an Rd field.
+  Reg31Mode RdMode() const {
+    // The following instructions use sp or wsp as Rd:
+    //  Add/sub (immediate) when not setting the flags.
+    //  Add/sub (extended) when not setting the flags.
+    //  Logical (immediate) when not setting the flags.
+    // Otherwise, r31 is the zero register.
+    if (IsAddSubImmediate() || IsAddSubExtended()) {
+      if (Mask(AddSubSetFlagsBit)) {
+        return Reg31IsZeroRegister;
+      } else {
+        return Reg31IsStackPointer;
+      }
+    }
+    if (IsLogicalImmediate()) {
+      // Of the logical (immediate) instructions, only ANDS (and its aliases)
+      // can set the flags. The others can all write into sp.
+      // Note that some logical operations are not available to
+      // immediate-operand instructions, so we have to combine two masks here.
+      if (Mask(LogicalImmediateMask & LogicalOpMask) == ANDS) {
+        return Reg31IsZeroRegister;
+      } else {
+        return Reg31IsStackPointer;
+      }
+    }
+    return Reg31IsZeroRegister;
+  }
+
+  // Indicate whether Rn can be the stack pointer or the zero register. This
+  // does not check that the instruction actually has an Rn field.
+  Reg31Mode RnMode() const {
+    // The following instructions use sp or wsp as Rn:
+    //  All loads and stores.
+    //  Add/sub (immediate).
+    //  Add/sub (extended).
+    // Otherwise, r31 is the zero register.
+    if (IsLoadOrStore() || IsAddSubImmediate() || IsAddSubExtended()) {
+      return Reg31IsStackPointer;
+    }
+    return Reg31IsZeroRegister;
+  }
+
+  ImmBranchType BranchType() const {
+    if (IsCondBranchImm()) {
+      return CondBranchType;
+    } else if (IsUncondBranchImm()) {
+      return UncondBranchType;
+    } else if (IsCompareBranch()) {
+      return CompareBranchType;
+    } else if (IsTestBranch()) {
+      return TestBranchType;
+    } else {
+      return UnknownBranchType;
+    }
+  }
+
+  // Find the target of this instruction. 'this' may be a branch or a
+  // PC-relative addressing instruction.
+  const Instruction* ImmPCOffsetTarget() const;
+
+  // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
+  // a PC-relative addressing instruction.
+  void SetImmPCOffsetTarget(const Instruction* target);
+  // Patch a literal load instruction to load from 'source'.
+  void SetImmLLiteral(const Instruction* source);
+
+  // The range of a load literal instruction, expressed as 'instr +- range'.
+  // The range is actually the 'positive' range; the branch instruction can
+  // target [instr - range - kInstructionSize, instr + range].
+  static const int kLoadLiteralImmBitwidth = 19;
+  static const int kLoadLiteralRange =
+      (1 << kLoadLiteralImmBitwidth) / 2 - kInstructionSize;
+
+  // Calculate the address of a literal referred to by a load-literal
+  // instruction, and return it as the specified type.
+  //
+  // The literal itself is safely mutable only if the backing buffer is safely
+  // mutable.
+  template <typename T>
+  T LiteralAddress() const {
+    uint64_t base_raw = reinterpret_cast<uint64_t>(this);
+    int64_t offset = ImmLLiteral() << kLiteralEntrySizeLog2;
+    uint64_t address_raw = base_raw + offset;
+
+    // Cast the address using a C-style cast. A reinterpret_cast would be
+    // appropriate, but it can't cast one integral type to another.
+    T address = (T)(address_raw);
+
+    // Assert that the address can be represented by the specified type.
+    VIXL_ASSERT((uint64_t)(address) == address_raw);
+
+    return address;
+  }
+
+  uint32_t Literal32() const {
+    uint32_t literal;
+    memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
+    return literal;
+  }
+
+  uint64_t Literal64() const {
+    uint64_t literal;
+    memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
+    return literal;
+  }
+
+  float LiteralFP32() const {
+    return rawbits_to_float(Literal32());
+  }
+
+  double LiteralFP64() const {
+    return rawbits_to_double(Literal64());
+  }
+
+  const Instruction* NextInstruction() const {
+    return this + kInstructionSize;
+  }
+
+  const Instruction* InstructionAtOffset(int64_t offset) const {
+    VIXL_ASSERT(IsWordAligned(this + offset));
+    return this + offset;
+  }
+
+  template<typename T> static Instruction* Cast(T src) {
+    return reinterpret_cast<Instruction*>(src);
+  }
+
+  template<typename T> static const Instruction* CastConst(T src) {
+    return reinterpret_cast<const Instruction*>(src);
+  }
+
+ private:
+  int ImmBranch() const;
+
+  static float Imm8ToFP32(uint32_t imm8);
+  static double Imm8ToFP64(uint32_t imm8);
+
+  void SetPCRelImmTarget(const Instruction* target);
+  void SetBranchImmTarget(const Instruction* target);
+};
+
+
+// Functions for handling NEON vector format information.
+enum VectorFormat {
+  kFormatUndefined = 0xffffffff,
+  kFormat8B  = NEON_8B,
+  kFormat16B = NEON_16B,
+  kFormat4H  = NEON_4H,
+  kFormat8H  = NEON_8H,
+  kFormat2S  = NEON_2S,
+  kFormat4S  = NEON_4S,
+  kFormat1D  = NEON_1D,
+  kFormat2D  = NEON_2D,
+
+  // Scalar formats. We add the scalar bit to distinguish between scalar and
+  // vector enumerations; the bit is always set in the encoding of scalar ops
+  // and always clear for vector ops. Although kFormatD and kFormat1D appear
+  // to be the same, their meaning is subtly different. The first is a scalar
+  // operation, the second a vector operation that only affects one lane.
+  kFormatB = NEON_B | NEONScalar,
+  kFormatH = NEON_H | NEONScalar,
+  kFormatS = NEON_S | NEONScalar,
+  kFormatD = NEON_D | NEONScalar
+};
+
+VectorFormat VectorFormatHalfWidth(const VectorFormat vform);
+VectorFormat VectorFormatDoubleWidth(const VectorFormat vform);
+VectorFormat VectorFormatDoubleLanes(const VectorFormat vform);
+VectorFormat VectorFormatHalfLanes(const VectorFormat vform);
+VectorFormat ScalarFormatFromLaneSize(int lanesize);
+VectorFormat VectorFormatHalfWidthDoubleLanes(const VectorFormat vform);
+VectorFormat VectorFormatFillQ(const VectorFormat vform);
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform);
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform);
+// TODO: Make the return types of these functions consistent.
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform);
+int LaneSizeInBytesFromFormat(VectorFormat vform);
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform);
+int LaneCountFromFormat(VectorFormat vform);
+int MaxLaneCountFromFormat(VectorFormat vform);
+bool IsVectorFormat(VectorFormat vform);
+int64_t MaxIntFromFormat(VectorFormat vform);
+int64_t MinIntFromFormat(VectorFormat vform);
+uint64_t MaxUintFromFormat(VectorFormat vform);
+
+
+enum NEONFormat {
+  NF_UNDEF = 0,
+  NF_8B    = 1,
+  NF_16B   = 2,
+  NF_4H    = 3,
+  NF_8H    = 4,
+  NF_2S    = 5,
+  NF_4S    = 6,
+  NF_1D    = 7,
+  NF_2D    = 8,
+  NF_B     = 9,
+  NF_H     = 10,
+  NF_S     = 11,
+  NF_D     = 12
+};
+
+static const unsigned kNEONFormatMaxBits = 6;
+
+struct NEONFormatMap {
+  // The bit positions in the instruction to consider.
+  uint8_t bits[kNEONFormatMaxBits];
+
+  // Mapping from concatenated bits to format.
+  NEONFormat map[1 << kNEONFormatMaxBits];
+};
+
+class NEONFormatDecoder {
+ public:
+  enum SubstitutionMode {
+    kPlaceholder,
+    kFormat
+  };
+
+  // Construct a format decoder with increasingly specific format maps for each
+  // subsitution. If no format map is specified, the default is the integer
+  // format map.
+  explicit NEONFormatDecoder(const Instruction* instr) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(IntegerFormatMap());
+  }
+  NEONFormatDecoder(const Instruction* instr,
+                    const NEONFormatMap* format) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(format);
+  }
+  NEONFormatDecoder(const Instruction* instr,
+                    const NEONFormatMap* format0,
+                    const NEONFormatMap* format1) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(format0, format1);
+  }
+  NEONFormatDecoder(const Instruction* instr,
+                    const NEONFormatMap* format0,
+                    const NEONFormatMap* format1,
+                    const NEONFormatMap* format2) {
+    instrbits_ = instr->InstructionBits();
+    SetFormatMaps(format0, format1, format2);
+  }
+
+  // Set the format mapping for all or individual substitutions.
+  void SetFormatMaps(const NEONFormatMap* format0,
+                     const NEONFormatMap* format1 = NULL,
+                     const NEONFormatMap* format2 = NULL) {
+    VIXL_ASSERT(format0 != NULL);
+    formats_[0] = format0;
+    formats_[1] = (format1 == NULL) ? formats_[0] : format1;
+    formats_[2] = (format2 == NULL) ? formats_[1] : format2;
+  }
+  void SetFormatMap(unsigned index, const NEONFormatMap* format) {
+    VIXL_ASSERT(index <= (sizeof(formats_) / sizeof(formats_[0])));
+    VIXL_ASSERT(format != NULL);
+    formats_[index] = format;
+  }
+
+  // Substitute %s in the input string with the placeholder string for each
+  // register, ie. "'B", "'H", etc.
+  const char* SubstitutePlaceholders(const char* string) {
+    return Substitute(string, kPlaceholder, kPlaceholder, kPlaceholder);
+  }
+
+  // Substitute %s in the input string with a new string based on the
+  // substitution mode.
+  const char* Substitute(const char* string,
+                         SubstitutionMode mode0 = kFormat,
+                         SubstitutionMode mode1 = kFormat,
+                         SubstitutionMode mode2 = kFormat) {
+    snprintf(form_buffer_, sizeof(form_buffer_), string,
+             GetSubstitute(0, mode0),
+             GetSubstitute(1, mode1),
+             GetSubstitute(2, mode2));
+    return form_buffer_;
+  }
+
+  // Append a "2" to a mnemonic string based of the state of the Q bit.
+  const char* Mnemonic(const char* mnemonic) {
+    if ((instrbits_ & NEON_Q) != 0) {
+      snprintf(mne_buffer_, sizeof(mne_buffer_), "%s2", mnemonic);
+      return mne_buffer_;
+    }
+    return mnemonic;
+  }
+
+  VectorFormat GetVectorFormat(int format_index = 0) {
+    return GetVectorFormat(formats_[format_index]);
+  }
+
+  VectorFormat GetVectorFormat(const NEONFormatMap* format_map) {
+    static const VectorFormat vform[] = {
+      kFormatUndefined,
+      kFormat8B, kFormat16B, kFormat4H, kFormat8H,
+      kFormat2S, kFormat4S, kFormat1D, kFormat2D,
+      kFormatB, kFormatH, kFormatS, kFormatD
+    };
+    VIXL_ASSERT(GetNEONFormat(format_map) < (sizeof(vform) / sizeof(vform[0])));
+    return vform[GetNEONFormat(format_map)];
+  }
+
+  // Built in mappings for common cases.
+
+  // The integer format map uses three bits (Q, size<1:0>) to encode the
+  // "standard" set of NEON integer vector formats.
+  static const NEONFormatMap* IntegerFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22, 30},
+      {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_UNDEF, NF_2D}
+    };
+    return &map;
+  }
+
+  // The long integer format map uses two bits (size<1:0>) to encode the
+  // long set of NEON integer vector formats. These are used in narrow, wide
+  // and long operations.
+  static const NEONFormatMap* LongIntegerFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22}, {NF_8H, NF_4S, NF_2D}
+    };
+    return &map;
+  }
+
+  // The FP format map uses two bits (Q, size<0>) to encode the NEON FP vector
+  // formats: NF_2S, NF_4S, NF_2D.
+  static const NEONFormatMap* FPFormatMap() {
+    // The FP format map assumes two bits (Q, size<0>) are used to encode the
+    // NEON FP vector formats: NF_2S, NF_4S, NF_2D.
+    static const NEONFormatMap map = {
+      {22, 30}, {NF_2S, NF_4S, NF_UNDEF, NF_2D}
+    };
+    return &map;
+  }
+
+  // The load/store format map uses three bits (Q, 11, 10) to encode the
+  // set of NEON vector formats.
+  static const NEONFormatMap* LoadStoreFormatMap() {
+    static const NEONFormatMap map = {
+      {11, 10, 30},
+      {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+    };
+    return &map;
+  }
+
+  // The logical format map uses one bit (Q) to encode the NEON vector format:
+  // NF_8B, NF_16B.
+  static const NEONFormatMap* LogicalFormatMap() {
+    static const NEONFormatMap map = {
+      {30}, {NF_8B, NF_16B}
+    };
+    return &map;
+  }
+
+  // The triangular format map uses between two and five bits to encode the NEON
+  // vector format:
+  // xxx10->8B, xxx11->16B, xx100->4H, xx101->8H
+  // x1000->2S, x1001->4S,  10001->2D, all others undefined.
+  static const NEONFormatMap* TriangularFormatMap() {
+    static const NEONFormatMap map = {
+      {19, 18, 17, 16, 30},
+      {NF_UNDEF, NF_UNDEF, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_2S,
+       NF_4S, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_UNDEF, NF_2D,
+       NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_2S, NF_4S, NF_8B, NF_16B,
+       NF_4H, NF_8H, NF_8B, NF_16B}
+    };
+    return &map;
+  }
+
+  // The scalar format map uses two bits (size<1:0>) to encode the NEON scalar
+  // formats: NF_B, NF_H, NF_S, NF_D.
+  static const NEONFormatMap* ScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22}, {NF_B, NF_H, NF_S, NF_D}
+    };
+    return &map;
+  }
+
+  // The long scalar format map uses two bits (size<1:0>) to encode the longer
+  // NEON scalar formats: NF_H, NF_S, NF_D.
+  static const NEONFormatMap* LongScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {23, 22}, {NF_H, NF_S, NF_D}
+    };
+    return &map;
+  }
+
+  // The FP scalar format map assumes one bit (size<0>) is used to encode the
+  // NEON FP scalar formats: NF_S, NF_D.
+  static const NEONFormatMap* FPScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {22}, {NF_S, NF_D}
+    };
+    return &map;
+  }
+
+  // The triangular scalar format map uses between one and four bits to encode
+  // the NEON FP scalar formats:
+  // xxx1->B, xx10->H, x100->S, 1000->D, all others undefined.
+  static const NEONFormatMap* TriangularScalarFormatMap() {
+    static const NEONFormatMap map = {
+      {19, 18, 17, 16},
+      {NF_UNDEF, NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B,
+       NF_D,     NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B}
+    };
+    return &map;
+  }
+
+ private:
+  // Get a pointer to a string that represents the format or placeholder for
+  // the specified substitution index, based on the format map and instruction.
+  const char* GetSubstitute(int index, SubstitutionMode mode) {
+    if (mode == kFormat) {
+      return NEONFormatAsString(GetNEONFormat(formats_[index]));
+    }
+    VIXL_ASSERT(mode == kPlaceholder);
+    return NEONFormatAsPlaceholder(GetNEONFormat(formats_[index]));
+  }
+
+  // Get the NEONFormat enumerated value for bits obtained from the
+  // instruction based on the specified format mapping.
+  NEONFormat GetNEONFormat(const NEONFormatMap* format_map) {
+    return format_map->map[PickBits(format_map->bits)];
+  }
+
+  // Convert a NEONFormat into a string.
+  static const char* NEONFormatAsString(NEONFormat format) {
+    static const char* formats[] = {
+      "undefined",
+      "8b", "16b", "4h", "8h", "2s", "4s", "1d", "2d",
+      "b", "h", "s", "d"
+    };
+    VIXL_ASSERT(format < (sizeof(formats) / sizeof(formats[0])));
+    return formats[format];
+  }
+
+  // Convert a NEONFormat into a register placeholder string.
+  static const char* NEONFormatAsPlaceholder(NEONFormat format) {
+    VIXL_ASSERT((format == NF_B) || (format == NF_H) ||
+                (format == NF_S) || (format == NF_D) ||
+                (format == NF_UNDEF));
+    static const char* formats[] = {
+      "undefined",
+      "undefined", "undefined", "undefined", "undefined",
+      "undefined", "undefined", "undefined", "undefined",
+      "'B", "'H", "'S", "'D"
+    };
+    return formats[format];
+  }
+
+  // Select bits from instrbits_ defined by the bits array, concatenate them,
+  // and return the value.
+  uint8_t PickBits(const uint8_t bits[]) {
+    uint8_t result = 0;
+    for (unsigned b = 0; b < kNEONFormatMaxBits; b++) {
+      if (bits[b] == 0) break;
+      result <<= 1;
+      result |= ((instrbits_ & (1 << bits[b])) == 0) ? 0 : 1;
+    }
+    return result;
+  }
+
+  Instr instrbits_;
+  const NEONFormatMap* formats_[3];
+  char form_buffer_[64];
+  char mne_buffer_[16];
+};
+}  // namespace vixl
+
+#endif  // VIXL_A64_INSTRUCTIONS_A64_H_
diff --git a/disas/libvixl/vixl/code-buffer.h b/disas/libvixl/vixl/code-buffer.h
new file mode 100644
index 0000000..f93ebb6
--- /dev/null
+++ b/disas/libvixl/vixl/code-buffer.h
@@ -0,0 +1,113 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_CODE_BUFFER_H
+#define VIXL_CODE_BUFFER_H
+
+#include <string.h>
+#include "vixl/globals.h"
+
+namespace vixl {
+
+class CodeBuffer {
+ public:
+  explicit CodeBuffer(size_t capacity = 4 * KBytes);
+  CodeBuffer(void* buffer, size_t capacity);
+  ~CodeBuffer();
+
+  void Reset();
+
+  ptrdiff_t OffsetFrom(ptrdiff_t offset) const {
+    ptrdiff_t cursor_offset = cursor_ - buffer_;
+    VIXL_ASSERT((offset >= 0) && (offset <= cursor_offset));
+    return cursor_offset - offset;
+  }
+
+  ptrdiff_t CursorOffset() const {
+    return OffsetFrom(0);
+  }
+
+  template <typename T>
+  T GetOffsetAddress(ptrdiff_t offset) const {
+    VIXL_ASSERT((offset >= 0) && (offset <= (cursor_ - buffer_)));
+    return reinterpret_cast<T>(buffer_ + offset);
+  }
+
+  size_t RemainingBytes() const {
+    VIXL_ASSERT((cursor_ >= buffer_) && (cursor_ <= (buffer_ + capacity_)));
+    return (buffer_ + capacity_) - cursor_;
+  }
+
+  // A code buffer can emit:
+  //  * 32-bit data: instruction and constant.
+  //  * 64-bit data: constant.
+  //  * string: debug info.
+  void Emit32(uint32_t data) { Emit(data); }
+
+  void Emit64(uint64_t data) { Emit(data); }
+
+  void EmitString(const char* string);
+
+  // Align to kInstructionSize.
+  void Align();
+
+  size_t capacity() const { return capacity_; }
+
+  bool IsManaged() const { return managed_; }
+
+  void Grow(size_t new_capacity);
+
+  bool IsDirty() const { return dirty_; }
+
+  void SetClean() { dirty_ = false; }
+
+ private:
+  template <typename T>
+  void Emit(T value) {
+    VIXL_ASSERT(RemainingBytes() >= sizeof(value));
+    dirty_ = true;
+    memcpy(cursor_, &value, sizeof(value));
+    cursor_ += sizeof(value);
+  }
+
+  // Backing store of the buffer.
+  byte* buffer_;
+  // If true the backing store is allocated and deallocated by the buffer. The
+  // backing store can then grow on demand. If false the backing store is
+  // provided by the user and cannot be resized internally.
+  bool managed_;
+  // Pointer to the next location to be written.
+  byte* cursor_;
+  // True if there has been any write since the buffer was created or cleaned.
+  bool dirty_;
+  // Capacity in bytes of the backing store.
+  size_t capacity_;
+};
+
+}  // namespace vixl
+
+#endif  // VIXL_CODE_BUFFER_H
+
diff --git a/disas/libvixl/vixl/compiler-intrinsics.cc b/disas/libvixl/vixl/compiler-intrinsics.cc
new file mode 100644
index 0000000..fd551fa
--- /dev/null
+++ b/disas/libvixl/vixl/compiler-intrinsics.cc
@@ -0,0 +1,144 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "compiler-intrinsics.h"
+
+namespace vixl {
+
+
+int CountLeadingSignBitsFallBack(int64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
+  if (value >= 0) {
+    return CountLeadingZeros(value, width) - 1;
+  } else {
+    return CountLeadingZeros(~value, width) - 1;
+  }
+}
+
+
+int CountLeadingZerosFallBack(uint64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
+  if (value == 0) {
+    return width;
+  }
+  int count = 0;
+  value = value << (64 - width);
+  if ((value & UINT64_C(0xffffffff00000000)) == 0) {
+    count += 32;
+    value = value << 32;
+  }
+  if ((value & UINT64_C(0xffff000000000000)) == 0) {
+    count += 16;
+    value = value << 16;
+  }
+  if ((value & UINT64_C(0xff00000000000000)) == 0) {
+    count += 8;
+    value = value << 8;
+  }
+  if ((value & UINT64_C(0xf000000000000000)) == 0) {
+    count += 4;
+    value = value << 4;
+  }
+  if ((value & UINT64_C(0xc000000000000000)) == 0) {
+    count += 2;
+    value = value << 2;
+  }
+  if ((value & UINT64_C(0x8000000000000000)) == 0) {
+    count += 1;
+  }
+  count += (value == 0);
+  return count;
+}
+
+
+int CountSetBitsFallBack(uint64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
+
+  // Mask out unused bits to ensure that they are not counted.
+  value &= (UINT64_C(0xffffffffffffffff) >> (64 - width));
+
+  // Add up the set bits.
+  // The algorithm works by adding pairs of bit fields together iteratively,
+  // where the size of each bit field doubles each time.
+  // An example for an 8-bit value:
+  // Bits:  h  g  f  e  d  c  b  a
+  //         \ |   \ |   \ |   \ |
+  // value = h+g   f+e   d+c   b+a
+  //            \    |      \    |
+  // value =   h+g+f+e     d+c+b+a
+  //                  \          |
+  // value =       h+g+f+e+d+c+b+a
+  const uint64_t kMasks[] = {
+    UINT64_C(0x5555555555555555),
+    UINT64_C(0x3333333333333333),
+    UINT64_C(0x0f0f0f0f0f0f0f0f),
+    UINT64_C(0x00ff00ff00ff00ff),
+    UINT64_C(0x0000ffff0000ffff),
+    UINT64_C(0x00000000ffffffff),
+  };
+
+  for (unsigned i = 0; i < (sizeof(kMasks) / sizeof(kMasks[0])); i++) {
+    int shift = 1 << i;
+    value = ((value >> shift) & kMasks[i]) + (value & kMasks[i]);
+  }
+
+  return static_cast<int>(value);
+}
+
+
+int CountTrailingZerosFallBack(uint64_t value, int width) {
+  VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
+  int count = 0;
+  value = value << (64 - width);
+  if ((value & UINT64_C(0xffffffff)) == 0) {
+    count += 32;
+    value = value >> 32;
+  }
+  if ((value & 0xffff) == 0) {
+    count += 16;
+    value = value >> 16;
+  }
+  if ((value & 0xff) == 0) {
+    count += 8;
+    value = value >> 8;
+  }
+  if ((value & 0xf) == 0) {
+    count += 4;
+    value = value >> 4;
+  }
+  if ((value & 0x3) == 0) {
+    count += 2;
+    value = value >> 2;
+  }
+  if ((value & 0x1) == 0) {
+    count += 1;
+  }
+  count += (value == 0);
+  return count - (64 - width);
+}
+
+
+}  // namespace vixl
diff --git a/disas/libvixl/vixl/compiler-intrinsics.h b/disas/libvixl/vixl/compiler-intrinsics.h
new file mode 100644
index 0000000..9431bed
--- /dev/null
+++ b/disas/libvixl/vixl/compiler-intrinsics.h
@@ -0,0 +1,155 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+#ifndef VIXL_COMPILER_INTRINSICS_H
+#define VIXL_COMPILER_INTRINSICS_H
+
+#include "globals.h"
+
+namespace vixl {
+
+// Helper to check whether the version of GCC used is greater than the specified
+// requirement.
+#define MAJOR 1000000
+#define MINOR 1000
+#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel)                         \
+    ((__GNUC__ * MAJOR + __GNUC_MINOR__ * MINOR + __GNUC_PATCHLEVEL__) >=      \
+     ((major) * MAJOR + (minor) * MINOR + (patchlevel)))
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel)                         \
+    ((__GNUC__ * MAJOR + __GNUC_MINOR__ * MINOR) >=                            \
+     ((major) * MAJOR + (minor) * MINOR + (patchlevel)))
+#else
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel) 0
+#endif
+
+
+#if defined(__clang__) && !defined(VIXL_NO_COMPILER_BUILTINS)
+
+#define COMPILER_HAS_BUILTIN_CLRSB    (__has_builtin(__builtin_clrsb))
+#define COMPILER_HAS_BUILTIN_CLZ      (__has_builtin(__builtin_clz))
+#define COMPILER_HAS_BUILTIN_CTZ      (__has_builtin(__builtin_ctz))
+#define COMPILER_HAS_BUILTIN_FFS      (__has_builtin(__builtin_ffs))
+#define COMPILER_HAS_BUILTIN_POPCOUNT (__has_builtin(__builtin_popcount))
+
+#elif defined(__GNUC__) && !defined(VIXL_NO_COMPILER_BUILTINS)
+// The documentation for these builtins is available at:
+// https://gcc.gnu.org/onlinedocs/gcc-$MAJOR.$MINOR.$PATCHLEVEL/gcc//Other-Builtins.html
+
+# define COMPILER_HAS_BUILTIN_CLRSB    (GCC_VERSION_OR_NEWER(4, 7, 0))
+# define COMPILER_HAS_BUILTIN_CLZ      (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_CTZ      (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_FFS      (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_POPCOUNT (GCC_VERSION_OR_NEWER(3, 4, 0))
+
+#else
+// One can define VIXL_NO_COMPILER_BUILTINS to force using the manually
+// implemented C++ methods.
+
+#define COMPILER_HAS_BUILTIN_BSWAP    false
+#define COMPILER_HAS_BUILTIN_CLRSB    false
+#define COMPILER_HAS_BUILTIN_CLZ      false
+#define COMPILER_HAS_BUILTIN_CTZ      false
+#define COMPILER_HAS_BUILTIN_FFS      false
+#define COMPILER_HAS_BUILTIN_POPCOUNT false
+
+#endif
+
+
+template<typename V>
+inline bool IsPowerOf2(V value) {
+  return (value != 0) && ((value & (value - 1)) == 0);
+}
+
+
+// Declaration of fallback functions.
+int CountLeadingSignBitsFallBack(int64_t value, int width);
+int CountLeadingZerosFallBack(uint64_t value, int width);
+int CountSetBitsFallBack(uint64_t value, int width);
+int CountTrailingZerosFallBack(uint64_t value, int width);
+
+
+// Implementation of intrinsics functions.
+// TODO: The implementations could be improved for sizes different from 32bit
+// and 64bit: we could mask the values and call the appropriate builtin.
+
+template<typename V>
+inline int CountLeadingSignBits(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CLRSB
+  if (width == 32) {
+    return __builtin_clrsb(value);
+  } else if (width == 64) {
+    return __builtin_clrsbll(value);
+  }
+#endif
+  return CountLeadingSignBitsFallBack(value, width);
+}
+
+
+template<typename V>
+inline int CountLeadingZeros(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CLZ
+  if (width == 32) {
+    return (value == 0) ? 32 : __builtin_clz(static_cast<unsigned>(value));
+  } else if (width == 64) {
+    return (value == 0) ? 64 : __builtin_clzll(value);
+  }
+#endif
+  return CountLeadingZerosFallBack(value, width);
+}
+
+
+template<typename V>
+inline int CountSetBits(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_POPCOUNT
+  if (width == 32) {
+    return __builtin_popcount(static_cast<unsigned>(value));
+  } else if (width == 64) {
+    return __builtin_popcountll(value);
+  }
+#endif
+  return CountSetBitsFallBack(value, width);
+}
+
+
+template<typename V>
+inline int CountTrailingZeros(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CTZ
+  if (width == 32) {
+    return (value == 0) ? 32 : __builtin_ctz(static_cast<unsigned>(value));
+  } else if (width == 64) {
+    return (value == 0) ? 64 : __builtin_ctzll(value);
+  }
+#endif
+  return CountTrailingZerosFallBack(value, width);
+}
+
+}  // namespace vixl
+
+#endif  // VIXL_COMPILER_INTRINSICS_H
+
diff --git a/disas/libvixl/vixl/globals.h b/disas/libvixl/vixl/globals.h
new file mode 100644
index 0000000..61dc9f7
--- /dev/null
+++ b/disas/libvixl/vixl/globals.h
@@ -0,0 +1,151 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_GLOBALS_H
+#define VIXL_GLOBALS_H
+
+// Get standard C99 macros for integer types.
+#ifndef __STDC_CONSTANT_MACROS
+#define __STDC_CONSTANT_MACROS
+#endif
+
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS
+#endif
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include <stdint.h>
+#include <inttypes.h>
+
+#include <assert.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include "vixl/platform.h"
+
+
+typedef uint8_t byte;
+
+// Type for half-precision (16 bit) floating point numbers.
+typedef uint16_t float16;
+
+const int KBytes = 1024;
+const int MBytes = 1024 * KBytes;
+
+#define VIXL_ABORT() \
+    do { printf("in %s, line %i", __FILE__, __LINE__); abort(); } while (false)
+#ifdef VIXL_DEBUG
+  #define VIXL_ASSERT(condition) assert(condition)
+  #define VIXL_CHECK(condition) VIXL_ASSERT(condition)
+  #define VIXL_UNIMPLEMENTED() \
+    do { fprintf(stderr, "UNIMPLEMENTED\t"); VIXL_ABORT(); } while (false)
+  #define VIXL_UNREACHABLE() \
+    do { fprintf(stderr, "UNREACHABLE\t"); VIXL_ABORT(); } while (false)
+#else
+  #define VIXL_ASSERT(condition) ((void) 0)
+  #define VIXL_CHECK(condition) assert(condition)
+  #define VIXL_UNIMPLEMENTED() ((void) 0)
+  #define VIXL_UNREACHABLE() ((void) 0)
+#endif
+// This is not as powerful as template based assertions, but it is simple.
+// It assumes that the descriptions are unique. If this starts being a problem,
+// we can switch to a different implemention.
+#define VIXL_CONCAT(a, b) a##b
+#define VIXL_STATIC_ASSERT_LINE(line, condition) \
+  typedef char VIXL_CONCAT(STATIC_ASSERT_LINE_, line)[(condition) ? 1 : -1] \
+  __attribute__((unused))
+#define VIXL_STATIC_ASSERT(condition) \
+    VIXL_STATIC_ASSERT_LINE(__LINE__, condition)
+
+template <typename T1>
+inline void USE(T1) {}
+
+template <typename T1, typename T2>
+inline void USE(T1, T2) {}
+
+template <typename T1, typename T2, typename T3>
+inline void USE(T1, T2, T3) {}
+
+template <typename T1, typename T2, typename T3, typename T4>
+inline void USE(T1, T2, T3, T4) {}
+
+#define VIXL_ALIGNMENT_EXCEPTION() \
+    do { fprintf(stderr, "ALIGNMENT EXCEPTION\t"); VIXL_ABORT(); } while (0)
+
+// The clang::fallthrough attribute is used along with the Wimplicit-fallthrough
+// argument to annotate intentional fall-through between switch labels.
+// For more information please refer to:
+// http://clang.llvm.org/docs/AttributeReference.html#fallthrough-clang-fallthrough
+#ifndef __has_warning
+  #define __has_warning(x)  0
+#endif
+
+// Note: This option is only available for Clang. And will only be enabled for
+// C++11(201103L).
+#if __has_warning("-Wimplicit-fallthrough") && __cplusplus >= 201103L
+  #define VIXL_FALLTHROUGH() [[clang::fallthrough]] //NOLINT
+#else
+  #define VIXL_FALLTHROUGH() do {} while (0)
+#endif
+
+#if __cplusplus >= 201103L
+  #define VIXL_NO_RETURN  [[noreturn]] //NOLINT
+#else
+  #define VIXL_NO_RETURN  __attribute__((noreturn))
+#endif
+
+// Some functions might only be marked as "noreturn" for the DEBUG build. This
+// macro should be used for such cases (for more details see what
+// VIXL_UNREACHABLE expands to).
+#ifdef VIXL_DEBUG
+  #define VIXL_DEBUG_NO_RETURN  VIXL_NO_RETURN
+#else
+  #define VIXL_DEBUG_NO_RETURN
+#endif
+
+#ifdef VIXL_INCLUDE_SIMULATOR
+#ifndef VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE
+  #define VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE  1
+#endif
+#else
+#ifndef VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE
+  #define VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE  0
+#endif
+#if VIXL_GENERATE_SIMULATOR_INSTRUCTIONS_VALUE
+  #warning "Generating Simulator instructions without Simulator support."
+#endif
+#endif
+
+#ifdef USE_SIMULATOR
+  #error "Please see the release notes for USE_SIMULATOR."
+#endif
+
+#endif  // VIXL_GLOBALS_H
diff --git a/disas/libvixl/vixl/invalset.h b/disas/libvixl/vixl/invalset.h
new file mode 100644
index 0000000..ffdc023
--- /dev/null
+++ b/disas/libvixl/vixl/invalset.h
@@ -0,0 +1,775 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_INVALSET_H_
+#define VIXL_INVALSET_H_
+
+#include <string.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "vixl/globals.h"
+
+namespace vixl {
+
+// We define a custom data structure template and its iterator as `std`
+// containers do not fit the performance requirements for some of our use cases.
+//
+// The structure behaves like an iterable unordered set with special properties
+// and restrictions. "InvalSet" stands for "Invalidatable Set".
+//
+// Restrictions and requirements:
+// - Adding an element already present in the set is illegal. In debug mode,
+//   this is checked at insertion time.
+// - The templated class `ElementType` must provide comparison operators so that
+//   `std::sort()` can be used.
+// - A key must be available to represent invalid elements.
+// - Elements with an invalid key must compare higher or equal to any other
+//   element.
+//
+// Use cases and performance considerations:
+// Our use cases present two specificities that allow us to design this
+// structure to provide fast insertion *and* fast search and deletion
+// operations:
+// - Elements are (generally) inserted in order (sorted according to their key).
+// - A key is available to mark elements as invalid (deleted).
+// The backing `std::vector` allows for fast insertions. When
+// searching for an element we ensure the elements are sorted (this is generally
+// the case) and perform a binary search. When deleting an element we do not
+// free the associated memory immediately. Instead, an element to be deleted is
+// marked with the 'invalid' key. Other methods of the container take care of
+// ignoring entries marked as invalid.
+// To avoid the overhead of the `std::vector` container when only few entries
+// are used, a number of elements are preallocated.
+
+// 'ElementType' and 'KeyType' are respectively the types of the elements and
+// their key.  The structure only reclaims memory when safe to do so, if the
+// number of elements that can be reclaimed is greater than `RECLAIM_FROM` and
+// greater than `<total number of elements> / RECLAIM_FACTOR.
+#define TEMPLATE_INVALSET_P_DECL                                               \
+  class ElementType,                                                           \
+  unsigned N_PREALLOCATED_ELEMENTS,                                            \
+  class KeyType,                                                               \
+  KeyType INVALID_KEY,                                                         \
+  size_t RECLAIM_FROM,                                                         \
+  unsigned RECLAIM_FACTOR
+
+#define TEMPLATE_INVALSET_P_DEF                                                \
+ElementType, N_PREALLOCATED_ELEMENTS,                                          \
+KeyType, INVALID_KEY, RECLAIM_FROM, RECLAIM_FACTOR
+
+template<class S> class InvalSetIterator;  // Forward declaration.
+
+template<TEMPLATE_INVALSET_P_DECL> class InvalSet {
+ public:
+  InvalSet();
+  ~InvalSet();
+
+  static const size_t kNPreallocatedElements = N_PREALLOCATED_ELEMENTS;
+  static const KeyType kInvalidKey = INVALID_KEY;
+
+  // It is illegal to insert an element already present in the set.
+  void insert(const ElementType& element);
+
+  // Looks for the specified element in the set and - if found - deletes it.
+  void erase(const ElementType& element);
+
+  // This indicates the number of (valid) elements stored in this set.
+  size_t size() const;
+
+  // Returns true if no elements are stored in the set.
+  // Note that this does not mean the the backing storage is empty: it can still
+  // contain invalid elements.
+  bool empty() const;
+
+  void clear();
+
+  const ElementType min_element();
+
+  // This returns the key of the minimum element in the set.
+  KeyType min_element_key();
+
+  static bool IsValid(const ElementType& element);
+  static KeyType Key(const ElementType& element);
+  static void SetKey(ElementType* element, KeyType key);
+
+ protected:
+  // Returns a pointer to the element in vector_ if it was found, or NULL
+  // otherwise.
+  ElementType* Search(const ElementType& element);
+
+  // The argument *must* point to an element stored in *this* set.
+  // This function is not allowed to move elements in the backing vector
+  // storage.
+  void EraseInternal(ElementType* element);
+
+  // The elements in the range searched must be sorted.
+  ElementType* BinarySearch(const ElementType& element,
+                            ElementType* start,
+                            ElementType* end) const;
+
+  // Sort the elements.
+  enum SortType {
+    // The 'hard' version guarantees that invalid elements are moved to the end
+    // of the container.
+    kHardSort,
+    // The 'soft' version only guarantees that the elements will be sorted.
+    // Invalid elements may still be present anywhere in the set.
+    kSoftSort
+  };
+  void Sort(SortType sort_type);
+
+  // Delete the elements that have an invalid key. The complexity is linear
+  // with the size of the vector.
+  void Clean();
+
+  const ElementType Front() const;
+  const ElementType Back() const;
+
+  // Delete invalid trailing elements and return the last valid element in the
+  // set.
+  const ElementType CleanBack();
+
+  // Returns a pointer to the start or end of the backing storage.
+  const ElementType* StorageBegin() const;
+  const ElementType* StorageEnd() const;
+  ElementType* StorageBegin();
+  ElementType* StorageEnd();
+
+  // Returns the index of the element within the backing storage. The element
+  // must belong to the backing storage.
+  size_t ElementIndex(const ElementType* element) const;
+
+  // Returns the element at the specified index in the backing storage.
+  const ElementType* ElementAt(size_t index) const;
+  ElementType* ElementAt(size_t index);
+
+  static const ElementType* FirstValidElement(const ElementType* from,
+                                              const ElementType* end);
+
+  void CacheMinElement();
+  const ElementType CachedMinElement() const;
+
+  bool ShouldReclaimMemory() const;
+  void ReclaimMemory();
+
+  bool IsUsingVector() const { return vector_ != NULL; }
+  void set_sorted(bool sorted) { sorted_ = sorted; }
+
+  // We cache some data commonly required by users to improve performance.
+  // We cannot cache pointers to elements as we do not control the backing
+  // storage.
+  bool valid_cached_min_;
+  size_t cached_min_index_;  // Valid iff `valid_cached_min_` is true.
+  KeyType cached_min_key_;         // Valid iff `valid_cached_min_` is true.
+
+  // Indicates whether the elements are sorted.
+  bool sorted_;
+
+  // This represents the number of (valid) elements in this set.
+  size_t size_;
+
+  // The backing storage is either the array of preallocated elements or the
+  // vector. The structure starts by using the preallocated elements, and
+  // transitions (permanently) to using the vector once more than
+  // kNPreallocatedElements are used.
+  // Elements are only invalidated when using the vector. The preallocated
+  // storage always only contains valid elements.
+  ElementType preallocated_[kNPreallocatedElements];
+  std::vector<ElementType>* vector_;
+
+#ifdef VIXL_DEBUG
+  // Iterators acquire and release this monitor. While a set is acquired,
+  // certain operations are illegal to ensure that the iterator will
+  // correctly iterate over the elements in the set.
+  int monitor_;
+  int monitor() const { return monitor_; }
+  void Acquire() { monitor_++; }
+  void Release() {
+    monitor_--;
+    VIXL_ASSERT(monitor_ >= 0);
+  }
+#endif
+
+  friend class InvalSetIterator<InvalSet<TEMPLATE_INVALSET_P_DEF> >;
+  typedef ElementType _ElementType;
+  typedef KeyType _KeyType;
+};
+
+
+template<class S> class InvalSetIterator {
+ private:
+  // Redefine types to mirror the associated set types.
+  typedef typename S::_ElementType ElementType;
+  typedef typename S::_KeyType KeyType;
+
+ public:
+  explicit InvalSetIterator(S* inval_set);
+  ~InvalSetIterator();
+
+  ElementType* Current() const;
+  void Advance();
+  bool Done() const;
+
+  // Mark this iterator as 'done'.
+  void Finish();
+
+  // Delete the current element and advance the iterator to point to the next
+  // element.
+  void DeleteCurrentAndAdvance();
+
+  static bool IsValid(const ElementType& element);
+  static KeyType Key(const ElementType& element);
+
+ protected:
+  void MoveToValidElement();
+
+  // Indicates if the iterator is looking at the vector or at the preallocated
+  // elements.
+  const bool using_vector_;
+  // Used when looking at the preallocated elements, or in debug mode when using
+  // the vector to track how many times the iterator has advanced.
+  size_t index_;
+  typename std::vector<ElementType>::iterator iterator_;
+  S* inval_set_;
+};
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+InvalSet<TEMPLATE_INVALSET_P_DEF>::InvalSet()
+  : valid_cached_min_(false),
+    sorted_(true), size_(0), vector_(NULL) {
+#ifdef VIXL_DEBUG
+  monitor_ = 0;
+#endif
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+InvalSet<TEMPLATE_INVALSET_P_DEF>::~InvalSet() {
+  VIXL_ASSERT(monitor_ == 0);
+  delete vector_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::insert(const ElementType& element) {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(IsValid(element));
+  VIXL_ASSERT(Search(element) == NULL);
+  set_sorted(empty() || (sorted_ && (element > CleanBack())));
+  if (IsUsingVector()) {
+    vector_->push_back(element);
+  } else {
+    if (size_ < kNPreallocatedElements) {
+      preallocated_[size_] = element;
+    } else {
+      // Transition to using the vector.
+      vector_ = new std::vector<ElementType>(preallocated_,
+                                             preallocated_ + size_);
+      vector_->push_back(element);
+    }
+  }
+  size_++;
+
+  if (valid_cached_min_ && (element < min_element())) {
+    cached_min_index_ = IsUsingVector() ? vector_->size() - 1 : size_ - 1;
+    cached_min_key_ = Key(element);
+    valid_cached_min_ = true;
+  }
+
+  if (ShouldReclaimMemory()) {
+    ReclaimMemory();
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::erase(const ElementType& element) {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(IsValid(element));
+  ElementType* local_element = Search(element);
+  if (local_element != NULL) {
+    EraseInternal(local_element);
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::Search(
+    const ElementType& element) {
+  VIXL_ASSERT(monitor() == 0);
+  if (empty()) {
+    return NULL;
+  }
+  if (ShouldReclaimMemory()) {
+    ReclaimMemory();
+  }
+  if (!sorted_) {
+    Sort(kHardSort);
+  }
+  if (!valid_cached_min_) {
+    CacheMinElement();
+  }
+  return BinarySearch(element, ElementAt(cached_min_index_), StorageEnd());
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+size_t InvalSet<TEMPLATE_INVALSET_P_DEF>::size() const {
+  return size_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+bool InvalSet<TEMPLATE_INVALSET_P_DEF>::empty() const {
+  return size_ == 0;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::clear() {
+  VIXL_ASSERT(monitor() == 0);
+  size_ = 0;
+  if (IsUsingVector()) {
+    vector_->clear();
+  }
+  set_sorted(true);
+  valid_cached_min_ = false;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::min_element() {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(!empty());
+  CacheMinElement();
+  return *ElementAt(cached_min_index_);
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+KeyType InvalSet<TEMPLATE_INVALSET_P_DEF>::min_element_key() {
+  VIXL_ASSERT(monitor() == 0);
+  if (valid_cached_min_) {
+    return cached_min_key_;
+  } else {
+    return Key(min_element());
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+bool InvalSet<TEMPLATE_INVALSET_P_DEF>::IsValid(const ElementType& element) {
+  return Key(element) != kInvalidKey;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::EraseInternal(ElementType* element) {
+  // Note that this function must be safe even while an iterator has acquired
+  // this set.
+  VIXL_ASSERT(element != NULL);
+  size_t deleted_index = ElementIndex(element);
+  if (IsUsingVector()) {
+    VIXL_ASSERT((&(vector_->front()) <= element) &&
+                (element <= &(vector_->back())));
+    SetKey(element, kInvalidKey);
+  } else {
+    VIXL_ASSERT((preallocated_ <= element) &&
+                (element < (preallocated_ + kNPreallocatedElements)));
+    ElementType* end = preallocated_ + kNPreallocatedElements;
+    size_t copy_size = sizeof(*element) * (end - element - 1);
+    memmove(element, element + 1, copy_size);
+  }
+  size_--;
+
+  if (valid_cached_min_ &&
+      (deleted_index == cached_min_index_)) {
+    if (sorted_ && !empty()) {
+      const ElementType* min = FirstValidElement(element, StorageEnd());
+      cached_min_index_ = ElementIndex(min);
+      cached_min_key_ = Key(*min);
+      valid_cached_min_ = true;
+    } else {
+      valid_cached_min_ = false;
+    }
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::BinarySearch(
+    const ElementType& element, ElementType* start, ElementType* end) const {
+  if (start == end) {
+    return NULL;
+  }
+  VIXL_ASSERT(sorted_);
+  VIXL_ASSERT(start < end);
+  VIXL_ASSERT(!empty());
+
+  // Perform a binary search through the elements while ignoring invalid
+  // elements.
+  ElementType* elements = start;
+  size_t low = 0;
+  size_t high = (end - start) - 1;
+  while (low < high) {
+    // Find valid bounds.
+    while (!IsValid(elements[low]) && (low < high)) ++low;
+    while (!IsValid(elements[high]) && (low < high)) --high;
+    VIXL_ASSERT(low <= high);
+    // Avoid overflow when computing the middle index.
+    size_t middle = low / 2 + high / 2 + (low & high & 1);
+    if ((middle == low) || (middle == high)) {
+      break;
+    }
+    while (!IsValid(elements[middle]) && (middle < high - 1)) ++middle;
+    while (!IsValid(elements[middle]) && (low + 1 < middle)) --middle;
+    if (!IsValid(elements[middle])) {
+      break;
+    }
+    if (elements[middle] < element) {
+      low = middle;
+    } else {
+      high = middle;
+    }
+  }
+
+  if (elements[low] == element) return &elements[low];
+  if (elements[high] == element) return &elements[high];
+  return NULL;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::Sort(SortType sort_type) {
+  VIXL_ASSERT(monitor() == 0);
+  if (sort_type == kSoftSort) {
+    if (sorted_) {
+      return;
+    }
+  }
+  if (empty()) {
+    return;
+  }
+
+  Clean();
+  std::sort(StorageBegin(), StorageEnd());
+
+  set_sorted(true);
+  cached_min_index_ = 0;
+  cached_min_key_ = Key(Front());
+  valid_cached_min_ = true;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::Clean() {
+  VIXL_ASSERT(monitor() == 0);
+  if (empty() || !IsUsingVector()) {
+    return;
+  }
+  // Manually iterate through the vector storage to discard invalid elements.
+  ElementType* start = &(vector_->front());
+  ElementType* end = start + vector_->size();
+  ElementType* c = start;
+  ElementType* first_invalid;
+  ElementType* first_valid;
+  ElementType* next_invalid;
+
+  while (c < end && IsValid(*c)) { c++; }
+  first_invalid = c;
+
+  while (c < end) {
+    while (c < end && !IsValid(*c)) { c++; }
+    first_valid = c;
+    while (c < end && IsValid(*c)) { c++; }
+    next_invalid = c;
+
+    ptrdiff_t n_moved_elements = (next_invalid - first_valid);
+    memmove(first_invalid, first_valid,  n_moved_elements * sizeof(*c));
+    first_invalid = first_invalid + n_moved_elements;
+    c = next_invalid;
+  }
+
+  // Delete the trailing invalid elements.
+  vector_->erase(vector_->begin() + (first_invalid - start), vector_->end());
+  VIXL_ASSERT(vector_->size() == size_);
+
+  if (sorted_) {
+    valid_cached_min_ = true;
+    cached_min_index_ = 0;
+    cached_min_key_ = Key(*ElementAt(0));
+  } else {
+    valid_cached_min_ = false;
+  }
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::Front() const {
+  VIXL_ASSERT(!empty());
+  return IsUsingVector() ? vector_->front() : preallocated_[0];
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::Back() const {
+  VIXL_ASSERT(!empty());
+  return IsUsingVector() ? vector_->back() : preallocated_[size_ - 1];
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType InvalSet<TEMPLATE_INVALSET_P_DEF>::CleanBack() {
+  VIXL_ASSERT(monitor() == 0);
+  if (IsUsingVector()) {
+    // Delete the invalid trailing elements.
+    typename std::vector<ElementType>::reverse_iterator it = vector_->rbegin();
+    while (!IsValid(*it)) {
+      it++;
+    }
+    vector_->erase(it.base(), vector_->end());
+  }
+  return Back();
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageBegin() const {
+  return IsUsingVector() ? &(vector_->front()) : preallocated_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageEnd() const {
+  return IsUsingVector() ? &(vector_->back()) + 1 : preallocated_ + size_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageBegin() {
+  return IsUsingVector() ? &(vector_->front()) : preallocated_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::StorageEnd() {
+  return IsUsingVector() ? &(vector_->back()) + 1 : preallocated_ + size_;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+size_t InvalSet<TEMPLATE_INVALSET_P_DEF>::ElementIndex(
+    const ElementType* element) const {
+  VIXL_ASSERT((StorageBegin() <= element) && (element < StorageEnd()));
+  return element - StorageBegin();
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::ElementAt(
+    size_t index) const {
+  VIXL_ASSERT(
+      (IsUsingVector() && (index < vector_->size())) || (index < size_));
+  return StorageBegin() + index;
+}
+
+template<TEMPLATE_INVALSET_P_DECL>
+ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::ElementAt(size_t index) {
+  VIXL_ASSERT(
+      (IsUsingVector() && (index < vector_->size())) || (index < size_));
+  return StorageBegin() + index;
+}
+
+template<TEMPLATE_INVALSET_P_DECL>
+const ElementType* InvalSet<TEMPLATE_INVALSET_P_DEF>::FirstValidElement(
+    const ElementType* from, const ElementType* end) {
+  while ((from < end) && !IsValid(*from)) {
+    from++;
+  }
+  return from;
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::CacheMinElement() {
+  VIXL_ASSERT(monitor() == 0);
+  VIXL_ASSERT(!empty());
+
+  if (valid_cached_min_) {
+    return;
+  }
+
+  if (sorted_) {
+    const ElementType* min = FirstValidElement(StorageBegin(), StorageEnd());
+    cached_min_index_ = ElementIndex(min);
+    cached_min_key_ = Key(*min);
+    valid_cached_min_ = true;
+  } else {
+    Sort(kHardSort);
+  }
+  VIXL_ASSERT(valid_cached_min_);
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+bool InvalSet<TEMPLATE_INVALSET_P_DEF>::ShouldReclaimMemory() const {
+  if (!IsUsingVector()) {
+    return false;
+  }
+  size_t n_invalid_elements = vector_->size() - size_;
+  return (n_invalid_elements > RECLAIM_FROM) &&
+         (n_invalid_elements > vector_->size() / RECLAIM_FACTOR);
+}
+
+
+template<TEMPLATE_INVALSET_P_DECL>
+void InvalSet<TEMPLATE_INVALSET_P_DEF>::ReclaimMemory() {
+  VIXL_ASSERT(monitor() == 0);
+  Clean();
+}
+
+
+template<class S>
+InvalSetIterator<S>::InvalSetIterator(S* inval_set)
+    : using_vector_((inval_set != NULL) && inval_set->IsUsingVector()),
+      index_(0),
+      inval_set_(inval_set) {
+  if (inval_set != NULL) {
+    inval_set->Sort(S::kSoftSort);
+#ifdef VIXL_DEBUG
+    inval_set->Acquire();
+#endif
+    if (using_vector_) {
+      iterator_ = typename std::vector<ElementType>::iterator(
+          inval_set_->vector_->begin());
+    }
+    MoveToValidElement();
+  }
+}
+
+
+template<class S>
+InvalSetIterator<S>::~InvalSetIterator() {
+#ifdef VIXL_DEBUG
+  if (inval_set_ != NULL) {
+    inval_set_->Release();
+  }
+#endif
+}
+
+
+template<class S>
+typename S::_ElementType* InvalSetIterator<S>::Current() const {
+  VIXL_ASSERT(!Done());
+  if (using_vector_) {
+    return &(*iterator_);
+  } else {
+    return &(inval_set_->preallocated_[index_]);
+  }
+}
+
+
+template<class S>
+void InvalSetIterator<S>::Advance() {
+  VIXL_ASSERT(!Done());
+  if (using_vector_) {
+    iterator_++;
+#ifdef VIXL_DEBUG
+    index_++;
+#endif
+    MoveToValidElement();
+  } else {
+    index_++;
+  }
+}
+
+
+template<class S>
+bool InvalSetIterator<S>::Done() const {
+  if (using_vector_) {
+    bool done = (iterator_ == inval_set_->vector_->end());
+    VIXL_ASSERT(done == (index_ == inval_set_->size()));
+    return done;
+  } else {
+    return index_ == inval_set_->size();
+  }
+}
+
+
+template<class S>
+void InvalSetIterator<S>::Finish() {
+  VIXL_ASSERT(inval_set_->sorted_);
+  if (using_vector_) {
+    iterator_ = inval_set_->vector_->end();
+  }
+  index_ = inval_set_->size();
+}
+
+
+template<class S>
+void InvalSetIterator<S>::DeleteCurrentAndAdvance() {
+  if (using_vector_) {
+    inval_set_->EraseInternal(&(*iterator_));
+    MoveToValidElement();
+  } else {
+    inval_set_->EraseInternal(inval_set_->preallocated_ + index_);
+  }
+}
+
+
+template<class S>
+bool InvalSetIterator<S>::IsValid(const ElementType& element) {
+  return S::IsValid(element);
+}
+
+
+template<class S>
+typename S::_KeyType InvalSetIterator<S>::Key(const ElementType& element) {
+  return S::Key(element);
+}
+
+
+template<class S>
+void InvalSetIterator<S>::MoveToValidElement() {
+  if (using_vector_) {
+    while ((iterator_ != inval_set_->vector_->end()) && !IsValid(*iterator_)) {
+      iterator_++;
+    }
+  } else {
+    VIXL_ASSERT(inval_set_->empty() || IsValid(inval_set_->preallocated_[0]));
+    // Nothing to do.
+  }
+}
+
+#undef TEMPLATE_INVALSET_P_DECL
+#undef TEMPLATE_INVALSET_P_DEF
+
+}  // namespace vixl
+
+#endif  // VIXL_INVALSET_H_
diff --git a/disas/libvixl/vixl/platform.h b/disas/libvixl/vixl/platform.h
new file mode 100644
index 0000000..ab588f0
--- /dev/null
+++ b/disas/libvixl/vixl/platform.h
@@ -0,0 +1,37 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef PLATFORM_H
+#define PLATFORM_H
+
+// Define platform specific functionalities.
+#include <signal.h>
+
+namespace vixl {
+inline void HostBreakpoint() { raise(SIGINT); }
+}  // namespace vixl
+
+#endif
diff --git a/disas/libvixl/vixl/utils.cc b/disas/libvixl/vixl/utils.cc
new file mode 100644
index 0000000..3b8bd75
--- /dev/null
+++ b/disas/libvixl/vixl/utils.cc
@@ -0,0 +1,142 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "vixl/utils.h"
+#include <stdio.h>
+
+namespace vixl {
+
+uint32_t float_to_rawbits(float value) {
+  uint32_t bits = 0;
+  memcpy(&bits, &value, 4);
+  return bits;
+}
+
+
+uint64_t double_to_rawbits(double value) {
+  uint64_t bits = 0;
+  memcpy(&bits, &value, 8);
+  return bits;
+}
+
+
+float rawbits_to_float(uint32_t bits) {
+  float value = 0.0;
+  memcpy(&value, &bits, 4);
+  return value;
+}
+
+
+double rawbits_to_double(uint64_t bits) {
+  double value = 0.0;
+  memcpy(&value, &bits, 8);
+  return value;
+}
+
+
+uint32_t float_sign(float val) {
+  uint32_t rawbits = float_to_rawbits(val);
+  return unsigned_bitextract_32(31, 31, rawbits);
+}
+
+
+uint32_t float_exp(float val) {
+  uint32_t rawbits = float_to_rawbits(val);
+  return unsigned_bitextract_32(30, 23, rawbits);
+}
+
+
+uint32_t float_mantissa(float val) {
+  uint32_t rawbits = float_to_rawbits(val);
+  return unsigned_bitextract_32(22, 0, rawbits);
+}
+
+
+uint32_t double_sign(double val) {
+  uint64_t rawbits = double_to_rawbits(val);
+  return static_cast<uint32_t>(unsigned_bitextract_64(63, 63, rawbits));
+}
+
+
+uint32_t double_exp(double val) {
+  uint64_t rawbits = double_to_rawbits(val);
+  return static_cast<uint32_t>(unsigned_bitextract_64(62, 52, rawbits));
+}
+
+
+uint64_t double_mantissa(double val) {
+  uint64_t rawbits = double_to_rawbits(val);
+  return unsigned_bitextract_64(51, 0, rawbits);
+}
+
+
+float float_pack(uint32_t sign, uint32_t exp, uint32_t mantissa) {
+  uint32_t bits = (sign << 31) | (exp << 23) | mantissa;
+  return rawbits_to_float(bits);
+}
+
+
+double double_pack(uint64_t sign, uint64_t exp, uint64_t mantissa) {
+  uint64_t bits = (sign << 63) | (exp << 52) | mantissa;
+  return rawbits_to_double(bits);
+}
+
+
+int float16classify(float16 value) {
+  uint16_t exponent_max = (1 << 5) - 1;
+  uint16_t exponent_mask = exponent_max << 10;
+  uint16_t mantissa_mask = (1 << 10) - 1;
+
+  uint16_t exponent = (value & exponent_mask) >> 10;
+  uint16_t mantissa = value & mantissa_mask;
+  if (exponent == 0) {
+    if (mantissa == 0) {
+      return FP_ZERO;
+    }
+    return FP_SUBNORMAL;
+  } else if (exponent == exponent_max) {
+    if (mantissa == 0) {
+      return FP_INFINITE;
+    }
+    return FP_NAN;
+  }
+  return FP_NORMAL;
+}
+
+
+unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size) {
+  VIXL_ASSERT((reg_size % 8) == 0);
+  int count = 0;
+  for (unsigned i = 0; i < (reg_size / 16); i++) {
+    if ((imm & 0xffff) == 0) {
+      count++;
+    }
+    imm >>= 16;
+  }
+  return count;
+}
+
+}  // namespace vixl
diff --git a/disas/libvixl/vixl/utils.h b/disas/libvixl/vixl/utils.h
new file mode 100644
index 0000000..5ab134e
--- /dev/null
+++ b/disas/libvixl/vixl/utils.h
@@ -0,0 +1,286 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+//   * Redistributions of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//   * Redistributions in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//   * Neither the name of ARM Limited nor the names of its contributors may be
+//     used to endorse or promote products derived from this software without
+//     specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_UTILS_H
+#define VIXL_UTILS_H
+
+#include <string.h>
+#include <cmath>
+#include "vixl/globals.h"
+#include "vixl/compiler-intrinsics.h"
+
+namespace vixl {
+
+// Macros for compile-time format checking.
+#if GCC_VERSION_OR_NEWER(4, 4, 0)
+#define PRINTF_CHECK(format_index, varargs_index) \
+  __attribute__((format(gnu_printf, format_index, varargs_index)))
+#else
+#define PRINTF_CHECK(format_index, varargs_index)
+#endif
+
+// Check number width.
+inline bool is_intn(unsigned n, int64_t x) {
+  VIXL_ASSERT((0 < n) && (n < 64));
+  int64_t limit = INT64_C(1) << (n - 1);
+  return (-limit <= x) && (x < limit);
+}
+
+inline bool is_uintn(unsigned n, int64_t x) {
+  VIXL_ASSERT((0 < n) && (n < 64));
+  return !(x >> n);
+}
+
+inline uint32_t truncate_to_intn(unsigned n, int64_t x) {
+  VIXL_ASSERT((0 < n) && (n < 64));
+  return static_cast<uint32_t>(x & ((INT64_C(1) << n) - 1));
+}
+
+#define INT_1_TO_63_LIST(V)                                                    \
+V(1)  V(2)  V(3)  V(4)  V(5)  V(6)  V(7)  V(8)                                 \
+V(9)  V(10) V(11) V(12) V(13) V(14) V(15) V(16)                                \
+V(17) V(18) V(19) V(20) V(21) V(22) V(23) V(24)                                \
+V(25) V(26) V(27) V(28) V(29) V(30) V(31) V(32)                                \
+V(33) V(34) V(35) V(36) V(37) V(38) V(39) V(40)                                \
+V(41) V(42) V(43) V(44) V(45) V(46) V(47) V(48)                                \
+V(49) V(50) V(51) V(52) V(53) V(54) V(55) V(56)                                \
+V(57) V(58) V(59) V(60) V(61) V(62) V(63)
+
+#define DECLARE_IS_INT_N(N)                                                    \
+inline bool is_int##N(int64_t x) { return is_intn(N, x); }
+#define DECLARE_IS_UINT_N(N)                                                   \
+inline bool is_uint##N(int64_t x) { return is_uintn(N, x); }
+#define DECLARE_TRUNCATE_TO_INT_N(N)                                           \
+inline uint32_t truncate_to_int##N(int x) { return truncate_to_intn(N, x); }
+INT_1_TO_63_LIST(DECLARE_IS_INT_N)
+INT_1_TO_63_LIST(DECLARE_IS_UINT_N)
+INT_1_TO_63_LIST(DECLARE_TRUNCATE_TO_INT_N)
+#undef DECLARE_IS_INT_N
+#undef DECLARE_IS_UINT_N
+#undef DECLARE_TRUNCATE_TO_INT_N
+
+// Bit field extraction.
+inline uint32_t unsigned_bitextract_32(int msb, int lsb, uint32_t x) {
+  return (x >> lsb) & ((1 << (1 + msb - lsb)) - 1);
+}
+
+inline uint64_t unsigned_bitextract_64(int msb, int lsb, uint64_t x) {
+  return (x >> lsb) & ((static_cast<uint64_t>(1) << (1 + msb - lsb)) - 1);
+}
+
+inline int32_t signed_bitextract_32(int msb, int lsb, int32_t x) {
+  return (x << (31 - msb)) >> (lsb + 31 - msb);
+}
+
+inline int64_t signed_bitextract_64(int msb, int lsb, int64_t x) {
+  return (x << (63 - msb)) >> (lsb + 63 - msb);
+}
+
+// Floating point representation.
+uint32_t float_to_rawbits(float value);
+uint64_t double_to_rawbits(double value);
+float rawbits_to_float(uint32_t bits);
+double rawbits_to_double(uint64_t bits);
+
+uint32_t float_sign(float val);
+uint32_t float_exp(float val);
+uint32_t float_mantissa(float val);
+uint32_t double_sign(double val);
+uint32_t double_exp(double val);
+uint64_t double_mantissa(double val);
+
+float float_pack(uint32_t sign, uint32_t exp, uint32_t mantissa);
+double double_pack(uint64_t sign, uint64_t exp, uint64_t mantissa);
+
+// An fpclassify() function for 16-bit half-precision floats.
+int float16classify(float16 value);
+
+// NaN tests.
+inline bool IsSignallingNaN(double num) {
+  const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
+  uint64_t raw = double_to_rawbits(num);
+  if (std::isnan(num) && ((raw & kFP64QuietNaNMask) == 0)) {
+    return true;
+  }
+  return false;
+}
+
+
+inline bool IsSignallingNaN(float num) {
+  const uint32_t kFP32QuietNaNMask = 0x00400000;
+  uint32_t raw = float_to_rawbits(num);
+  if (std::isnan(num) && ((raw & kFP32QuietNaNMask) == 0)) {
+    return true;
+  }
+  return false;
+}
+
+
+inline bool IsSignallingNaN(float16 num) {
+  const uint16_t kFP16QuietNaNMask = 0x0200;
+  return (float16classify(num) == FP_NAN) &&
+         ((num & kFP16QuietNaNMask) == 0);
+}
+
+
+template <typename T>
+inline bool IsQuietNaN(T num) {
+  return std::isnan(num) && !IsSignallingNaN(num);
+}
+
+
+// Convert the NaN in 'num' to a quiet NaN.
+inline double ToQuietNaN(double num) {
+  const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
+  VIXL_ASSERT(std::isnan(num));
+  return rawbits_to_double(double_to_rawbits(num) | kFP64QuietNaNMask);
+}
+
+
+inline float ToQuietNaN(float num) {
+  const uint32_t kFP32QuietNaNMask = 0x00400000;
+  VIXL_ASSERT(std::isnan(num));
+  return rawbits_to_float(float_to_rawbits(num) | kFP32QuietNaNMask);
+}
+
+
+// Fused multiply-add.
+inline double FusedMultiplyAdd(double op1, double op2, double a) {
+  return fma(op1, op2, a);
+}
+
+
+inline float FusedMultiplyAdd(float op1, float op2, float a) {
+  return fmaf(op1, op2, a);
+}
+
+
+inline uint64_t LowestSetBit(uint64_t value) {
+  return value & -value;
+}
+
+
+template<typename T>
+inline int HighestSetBitPosition(T value) {
+  VIXL_ASSERT(value != 0);
+  return (sizeof(value) * 8 - 1) - CountLeadingZeros(value);
+}
+
+
+template<typename V>
+inline int WhichPowerOf2(V value) {
+  VIXL_ASSERT(IsPowerOf2(value));
+  return CountTrailingZeros(value);
+}
+
+
+unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size);
+
+
+template <typename T>
+T ReverseBits(T value) {
+  VIXL_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) ||
+              (sizeof(value) == 4) || (sizeof(value) == 8));
+  T result = 0;
+  for (unsigned i = 0; i < (sizeof(value) * 8); i++) {
+    result = (result << 1) | (value & 1);
+    value >>= 1;
+  }
+  return result;
+}
+
+
+template <typename T>
+T ReverseBytes(T value, int block_bytes_log2) {
+  VIXL_ASSERT((sizeof(value) == 4) || (sizeof(value) == 8));
+  VIXL_ASSERT((1U << block_bytes_log2) <= sizeof(value));
+  // Split the 64-bit value into an 8-bit array, where b[0] is the least
+  // significant byte, and b[7] is the most significant.
+  uint8_t bytes[8];
+  uint64_t mask = UINT64_C(0xff00000000000000);
+  for (int i = 7; i >= 0; i--) {
+    bytes[i] = (static_cast<uint64_t>(value) & mask) >> (i * 8);
+    mask >>= 8;
+  }
+
+  // Permutation tables for REV instructions.
+  //  permute_table[0] is used by REV16_x, REV16_w
+  //  permute_table[1] is used by REV32_x, REV_w
+  //  permute_table[2] is used by REV_x
+  VIXL_ASSERT((0 < block_bytes_log2) && (block_bytes_log2 < 4));
+  static const uint8_t permute_table[3][8] = { {6, 7, 4, 5, 2, 3, 0, 1},
+                                               {4, 5, 6, 7, 0, 1, 2, 3},
+                                               {0, 1, 2, 3, 4, 5, 6, 7} };
+  T result = 0;
+  for (int i = 0; i < 8; i++) {
+    result <<= 8;
+    result |= bytes[permute_table[block_bytes_log2 - 1][i]];
+  }
+  return result;
+}
+
+
+// Pointer alignment
+// TODO: rename/refactor to make it specific to instructions.
+template<typename T>
+bool IsWordAligned(T pointer) {
+  VIXL_ASSERT(sizeof(pointer) == sizeof(intptr_t));   // NOLINT(runtime/sizeof)
+  return ((intptr_t)(pointer) & 3) == 0;
+}
+
+// Increment a pointer (up to 64 bits) until it has the specified alignment.
+template<class T>
+T AlignUp(T pointer, size_t alignment) {
+  // Use C-style casts to get static_cast behaviour for integral types (T), and
+  // reinterpret_cast behaviour for other types.
+
+  uint64_t pointer_raw = (uint64_t)pointer;
+  VIXL_STATIC_ASSERT(sizeof(pointer) <= sizeof(pointer_raw));
+
+  size_t align_step = (alignment - pointer_raw) % alignment;
+  VIXL_ASSERT((pointer_raw + align_step) % alignment == 0);
+
+  return (T)(pointer_raw + align_step);
+}
+
+// Decrement a pointer (up to 64 bits) until it has the specified alignment.
+template<class T>
+T AlignDown(T pointer, size_t alignment) {
+  // Use C-style casts to get static_cast behaviour for integral types (T), and
+  // reinterpret_cast behaviour for other types.
+
+  uint64_t pointer_raw = (uint64_t)pointer;
+  VIXL_STATIC_ASSERT(sizeof(pointer) <= sizeof(pointer_raw));
+
+  size_t align_step = pointer_raw % alignment;
+  VIXL_ASSERT((pointer_raw - align_step) % alignment == 0);
+
+  return (T)(pointer_raw - align_step);
+}
+
+}  // namespace vixl
+
+#endif  // VIXL_UTILS_H
diff --git a/disas/lm32.c b/disas/lm32.c
index 1718c86..fcc2cde 100644
--- a/disas/lm32.c
+++ b/disas/lm32.c
@@ -18,7 +18,7 @@
  *
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 typedef enum {
diff --git a/disas/m68k.c b/disas/m68k.c
index cc0db96..8e7c3f7 100644
--- a/disas/m68k.c
+++ b/disas/m68k.c
@@ -1,9 +1,8 @@
 /* This file is composed of several different files from the upstream
    sourceware.org CVS.  Original file boundaries marked with **** */
 
-#include <string.h>
+#include "qemu/osdep.h"
 #include <math.h>
-#include <stdio.h>
 
 #include "disas/bfd.h"
 
@@ -616,8 +615,6 @@
 /* Maximum length of an instruction.  */
 #define MAXLEN 22
 
-#include <setjmp.h>
-
 struct private
 {
   /* Points to first byte not fetched.  */
@@ -1679,7 +1676,7 @@
 	  (*info->fprintf_func) (info->stream, "%%sfc");
 	else
 	  /* xgettext:c-format */
-	  (*info->fprintf_func) (info->stream, _("<function code %d>"), fc);
+	  (*info->fprintf_func) (info->stream, "<function code %d>", fc);
       }
       break;
 
@@ -1830,7 +1827,7 @@
 	{
 	  info->fprintf_func (info->stream,
 			      /* xgettext:c-format */
-			      _("<internal error in opcode table: %s %s>\n"),
+			      "<internal error in opcode table: %s %s>\n",
 			      best->name,  best->args);
 	  info->fprintf_func = save_printer;
 	  info->print_address_func = save_print_address;
diff --git a/disas/microblaze.c b/disas/microblaze.c
index ec91af3..91b30ac 100644
--- a/disas/microblaze.c
+++ b/disas/microblaze.c
@@ -33,13 +33,10 @@
  */
 
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #define STATIC_TABLE
 #define DEFINE_TABLE
 
-#define TRUE   1
-#define FALSE  0
-
 #ifndef MICROBLAZE_OPC
 #define MICROBLAZE_OPC
 /* Assembler instructions for Xilinx's microblaze processor
@@ -275,7 +272,7 @@
 
 #define MAX_OPCODES 280
 
-struct op_code_struct {
+static struct op_code_struct {
   const char *name;
   short inst_type; /* registers and immediate values involved */
   short inst_offset_type; /* immediate vals offset from PC? (= 1 for branches) */
@@ -567,10 +564,9 @@
 };
 
 /* prefix for register names */
-char register_prefix[] = "r";
-char special_register_prefix[] = "spr";
-char fsl_register_prefix[] = "rfsl";
-char pvr_register_prefix[] = "rpvr";
+static const char register_prefix[] = "r";
+static const char fsl_register_prefix[] = "rfsl";
+static const char pvr_register_prefix[] = "rpvr";
 
 
 /* #defines for valid immediate range */
@@ -583,7 +579,6 @@
 #endif /* MICROBLAZE_OPC */
 
 #include "disas/bfd.h"
-#include <strings.h>
 
 #define get_field_rd(instr) get_field(instr, RD_MASK, RD_LOW)
 #define get_field_r1(instr) get_field(instr, RA_MASK, RA_LOW)
@@ -605,25 +600,6 @@
 unsigned long read_insn_microblaze (bfd_vma memaddr, 
 		      struct disassemble_info *info,
 		      struct op_code_struct **opr);
-enum microblaze_instr get_insn_microblaze (long inst,
-  		     bfd_boolean *isunsignedimm,
-  		     enum microblaze_instr_type *insn_type,
-  		     short *delay_slots);
-short get_delay_slots_microblaze (long inst);
-enum microblaze_instr microblaze_decode_insn (long insn,
-		        int *rd, 
-			int *ra, 
-			int *rb, 
-			int *imm);
-unsigned long
-microblaze_get_target_address (long inst,
-			       bfd_boolean immfound,
-			       int immval,
-			       long pcval,
-			       long r1val,
-			       long r2val,
-			       bfd_boolean *targetvalid,
-			       bfd_boolean *unconditionalbranch);
 
 static char *
 get_field (long instr, long mask, unsigned short low)
@@ -738,7 +714,9 @@
    default :
      {
        if ( ((((instr & IMM_MASK) >> IMM_LOW) ^ op->immval_mask) & 0xE000) == REG_PVR_MASK) {
-	 sprintf(tmpstr, "%spvr%d", register_prefix, (unsigned short)(((instr & IMM_MASK) >> IMM_LOW) ^ op->immval_mask) ^ REG_PVR_MASK);
+	  sprintf(tmpstr, "%s%u", pvr_register_prefix,
+                 (unsigned short)(((instr & IMM_MASK) >> IMM_LOW) ^
+                                  op->immval_mask) ^ REG_PVR_MASK);
 	 return(strdup(tmpstr));
        } else {
 	 strcpy(spr, "pc");
@@ -967,134 +945,3 @@
   /* Say how many bytes we consumed? */
   return 4;
 }
-
-enum microblaze_instr
-get_insn_microblaze (long inst,
-  		     bfd_boolean *isunsignedimm,
-  		     enum microblaze_instr_type *insn_type,
-  		     short *delay_slots)
-{
-  struct op_code_struct * op;
-  *isunsignedimm = FALSE;
-
-  /* Just a linear search of the table.  */
-  for (op = opcodes; op->name != 0; op ++)
-    if (op->bit_sequence == (inst & op->opcode_mask))
-      break;
-
-  if (op->name == 0)
-    return invalid_inst;
-  else {
-    *isunsignedimm = (op->inst_type == INST_TYPE_RD_R1_UNSIGNED_IMM);
-    *insn_type = op->instr_type;
-    *delay_slots = op->delay_slots;
-    return op->instr;
-  }
-}
-
-short
-get_delay_slots_microblaze (long inst)
-{
-  bfd_boolean isunsignedimm;
-  enum microblaze_instr_type insn_type;
-  enum microblaze_instr op;
-  short delay_slots;
-
-  op = get_insn_microblaze( inst, &isunsignedimm, &insn_type, &delay_slots);
-  if (op == invalid_inst)
-    return 0;
-  else 
-    return delay_slots;
-}
-
-enum microblaze_instr
-microblaze_decode_insn (long insn,
-		        int *rd, 
-			int *ra, 
-			int *rb, 
-			int *imm)
-{
-  enum microblaze_instr op;
-  bfd_boolean t1;
-  enum microblaze_instr_type t2;
-  short t3;
-
-  op = get_insn_microblaze(insn, &t1, &t2, &t3);
-  *rd = (insn & RD_MASK) >> RD_LOW;
-  *ra = (insn & RA_MASK) >> RA_LOW;
-  *rb = (insn & RB_MASK) >> RB_LOW;
-  t3 = (insn & IMM_MASK) >> IMM_LOW;
-  *imm = (int) t3;
-  return (op);
-}
-
-unsigned long
-microblaze_get_target_address (long inst,
-			       bfd_boolean immfound,
-			       int immval,
-			       long pcval,
-			       long r1val,
-			       long r2val,
-			       bfd_boolean *targetvalid,
-			       bfd_boolean *unconditionalbranch)
-{
-  struct op_code_struct * op;
-  long targetaddr = 0;
-
-  *unconditionalbranch = FALSE;
-  /* Just a linear search of the table.  */
-  for (op = opcodes; op->name != 0; op ++)
-    if (op->bit_sequence == (inst & op->opcode_mask))
-      break;
-
-  if (op->name == 0) {
-    *targetvalid = FALSE;
-  } else if (op->instr_type == branch_inst) {
-    switch (op->inst_type) {
-    case INST_TYPE_R2:
-      *unconditionalbranch = TRUE;
-      /* fallthru */
-    case INST_TYPE_RD_R2:
-    case INST_TYPE_R1_R2:
-      targetaddr = r2val;
-      *targetvalid = TRUE;
-      if (op->inst_offset_type == INST_PC_OFFSET)
-	targetaddr += pcval;
-      break;
-    case INST_TYPE_IMM:
-      *unconditionalbranch = TRUE;
-      /* fallthru */
-    case INST_TYPE_RD_IMM:
-    case INST_TYPE_R1_IMM:
-      if (immfound) {
-	targetaddr = (immval << 16) & 0xffff0000;
-	targetaddr |= (get_int_field_imm(inst) & 0x0000ffff);
-      } else {
-	targetaddr = get_int_field_imm(inst);
-	if (targetaddr & 0x8000)
-	  targetaddr |= 0xFFFF0000;
-      }
-      if (op->inst_offset_type == INST_PC_OFFSET)
-	targetaddr += pcval;
-      *targetvalid = TRUE;
-      break;
-    default:
-      *targetvalid = FALSE;
-      break;
-    }
-  } else if (op->instr_type == return_inst) {
-      if (immfound) {
-	targetaddr = (immval << 16) & 0xffff0000;
-	targetaddr |= (get_int_field_imm(inst) & 0x0000ffff);
-      } else {
-	targetaddr = get_int_field_imm(inst);
-	if (targetaddr & 0x8000)
-	  targetaddr |= 0xFFFF0000;
-      }
-      targetaddr += r1val;
-      *targetvalid = TRUE;
-  } else {
-    *targetvalid = FALSE;
-  }
-  return targetaddr;
-}
diff --git a/disas/mips.c b/disas/mips.c
index 66a5275..97f661a 100644
--- a/disas/mips.c
+++ b/disas/mips.c
@@ -19,6 +19,7 @@
 You should have received a copy of the GNU General Public License
 along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* mips.h.  Mips opcode list for GDB, the GNU debugger.
@@ -1296,12 +1297,12 @@
 {"dmod",    "d,s,t",    0x000000de, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
 {"ddivu",   "d,s,t",    0x0000009f, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
 {"dmodu",   "d,s,t",    0x000000df, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I64R6},
-{"ll",      "t,o(b)",   0x7c000036, 0xfc00007f, LDD|RD_b|WR_t,        0, I32R6},
-{"sc",      "t,o(b)",   0x7c000026, 0xfc00007f, LDD|RD_b|WR_t,        0, I32R6},
-{"lld",     "t,o(b)",   0x7c000037, 0xfc00007f, LDD|RD_b|WR_t,        0, I64R6},
-{"scd",     "t,o(b)",   0x7c000027, 0xfc00007f, LDD|RD_b|WR_t,        0, I64R6},
-{"pref",    "h,o(b)",   0x7c000035, 0xfc00007f, RD_b,                 0, I32R6},
-{"cache",   "k,o(b)",   0x7c000025, 0xfc00007f, RD_b,                 0, I32R6},
+{"ll",      "t,+o(b)",  0x7c000036, 0xfc00007f, LDD|RD_b|WR_t,        0, I32R6},
+{"sc",      "t,+o(b)",  0x7c000026, 0xfc00007f, LDD|RD_b|WR_t,        0, I32R6},
+{"lld",     "t,+o(b)",  0x7c000037, 0xfc00007f, LDD|RD_b|WR_t,        0, I64R6},
+{"scd",     "t,+o(b)",  0x7c000027, 0xfc00007f, LDD|RD_b|WR_t,        0, I64R6},
+{"pref",    "h,+o(b)",  0x7c000035, 0xfc00007f, RD_b,                 0, I32R6},
+{"cache",   "k,+o(b)",  0x7c000025, 0xfc00007f, RD_b,                 0, I32R6},
 {"seleqz",  "d,v,t",    0x00000035, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
 {"selnez",  "d,v,t",    0x00000037, 0xfc0007ff, WR_d|RD_s|RD_t,       0, I32R6},
 {"maddf.s", "D,S,T",    0x46000018, 0xffe0003f, WR_D|RD_S|RD_T|FP_S,  0, I32R6},
@@ -1404,6 +1405,10 @@
 {"cmp.sor.d",  "D,S,T", 0x46a00019, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
 {"cmp.sune.d", "D,S,T", 0x46a0001a, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
 {"cmp.sne.d",  "D,S,T", 0x46a0001b, 0xffe0003f, RD_S|RD_T|WR_D|FP_D,  0, I32R6},
+{"dvp",        "",      0x41600024, 0xffffffff, TRAP,                 0, I32R6},
+{"dvp",        "t",     0x41600024, 0xffe0ffff, TRAP|WR_t,            0, I32R6},
+{"evp",        "",      0x41600004, 0xffffffff, TRAP,                 0, I32R6},
+{"evp",        "t",     0x41600004, 0xffe0ffff, TRAP|WR_t,            0, I32R6},
 
 /* MSA */
 {"sll.b",   "+d,+e,+f", 0x7800000d, 0xffe0003f, WR_VD|RD_VS|RD_VT,  0, MSA},
@@ -2238,6 +2243,8 @@
 {"ceil.l.s", "D,S",	0x4600000a, 0xffff003f, WR_D|RD_S|FP_S|FP_D,	0,		I3|I33	},
 {"ceil.w.d", "D,S",	0x4620000e, 0xffff003f, WR_D|RD_S|FP_S|FP_D,	0,		I2	},
 {"ceil.w.s", "D,S",	0x4600000e, 0xffff003f, WR_D|RD_S|FP_S,		0,		I2	},
+{"mfhc0",   "t,G,H",    0x40400000, 0xffe007f8, LCD|WR_t|RD_C0,       0, I33},
+{"mthc0",   "t,G,H",    0x40c00000, 0xffe007f8, COD|RD_t|WR_C0|WR_CC, 0, I33},
 {"cfc0",    "t,G",	0x40400000, 0xffe007ff,	LCD|WR_t|RD_C0,		0,		I1	},
 {"cfc1",    "t,G",	0x44400000, 0xffe007ff,	LCD|WR_t|RD_C1|FP_S,	0,		I1	},
 {"cfc1",    "t,S",	0x44400000, 0xffe007ff,	LCD|WR_t|RD_C1|FP_S,	0,		I1	},
@@ -2406,8 +2413,8 @@
 {"ei",      "t",	0x41606020, 0xffe0ffff,	WR_t|WR_C0,		0,		I33	},
 {"emt",     "",		0x41600be1, 0xffffffff, TRAP,			0,		MT32	},
 {"emt",     "t",	0x41600be1, 0xffe0ffff, TRAP|WR_t,		0,		MT32	},
-{"eretnc",  "",         0x42000058, 0xffffffff, 0,                    0, I33 },
 {"eret",    "",         0x42000018, 0xffffffff, 0,      		0,		I3|I32	},
+{"eretnc",  "",         0x42000058, 0xffffffff, 0,                    0, I33},
 {"evpe",    "",		0x41600021, 0xffffffff, TRAP,			0,		MT32	},
 {"evpe",    "t",	0x41600021, 0xffe0ffff, TRAP|WR_t,		0,		MT32	},
 {"ext",     "t,r,+A,+C", 0x7c000000, 0xfc00003f, WR_t|RD_s,    		0,		I33	},
@@ -2418,9 +2425,11 @@
 {"hibernate","",        0x42000023, 0xffffffff,	0, 			0,		V1	},
 {"ins",     "t,r,+A,+B", 0x7c000004, 0xfc00003f, WR_t|RD_s,    		0,		I33	},
 {"jr",      "s",	0x00000008, 0xfc1fffff,	UBD|RD_s,		0,		I1	},
+{"jr",      "s",	0x00000009, 0xfc1fffff,	UBD|RD_s,		0,		I32R6	}, /* jalr */
 /* jr.hb is officially MIPS{32,64}R2, but it works on R1 as jr with
    the same hazard barrier effect.  */
 {"jr.hb",   "s",	0x00000408, 0xfc1fffff,	UBD|RD_s,		0,		I32	},
+{"jr.hb",   "s",	0x00000409, 0xfc1fffff,	UBD|RD_s,		0,		I32R6	}, /* jalr.hb */
 {"j",       "s",	0x00000008, 0xfc1fffff,	UBD|RD_s,		0,		I1	}, /* jr */
 /* SVR4 PIC code requires special handling for j, so it must be a
    macro.  */
@@ -3512,6 +3521,7 @@
   const char * const name;
 };
 
+#if 0
 /* The mips16 registers.  */
 static const unsigned int mips16_to_32_reg_map[] =
 {
@@ -3519,7 +3529,7 @@
 };
 
 #define mips16_reg_names(rn)	mips_gpr_names[mips16_to_32_reg_map[rn]]
-
+#endif
 
 static const char * const mips_gpr_names_numeric[32] =
 {
@@ -3802,13 +3812,6 @@
   "$24",  "$25",  "$26",  "$27",  "$28",  "$29",  "$30",  "$31"
 };
 
-static const char * const mips_msa_control_names_numeric[32] = {
-  "$0",   "$1",   "$2",   "$3",   "$4",   "$5",   "$6",   "$7",
-  "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
-  "$16",  "$17",  "$18",  "$19",  "$20",  "$21",  "$22",  "$23",
-  "$24",  "$25",  "$26",  "$27",  "$28",  "$29",  "$30",  "$31"
-};
-
 static const char * const mips_msa_control_names_mips3264r2[32] = {
   "MSAIR", "MSACSR", "$2", "$3",  "$4",   "$5",   "$6",   "$7",
   "$8",   "$9",   "$10",  "$11",  "$12",  "$13",  "$14",  "$15",
@@ -4254,7 +4257,7 @@
 	    case '\0':
 	      /* xgettext:c-format */
 	      (*info->fprintf_func) (info->stream,
-				     _("# internal error, incomplete extension sequence (+)"));
+				     "# internal error, incomplete extension sequence (+)");
 	      return;
 
 	    case 'A':
@@ -4512,7 +4515,7 @@
 	    default:
 	      /* xgettext:c-format */
 	      (*info->fprintf_func) (info->stream,
-				     _("# internal error, undefined extension sequence (+%c)"),
+				     "# internal error, undefined extension sequence (+%c)",
 				     *d);
 	      return;
 	    }
@@ -4872,7 +4875,7 @@
 	default:
 	  /* xgettext:c-format */
 	  (*info->fprintf_func) (info->stream,
-				 _("# internal error, undefined modifier(%c)"),
+				 "# internal error, undefined modifier(%c)",
 				 *d);
 	  return;
 	}
@@ -5736,7 +5739,7 @@
       /* xgettext:c-format */
       (*info->fprintf_func)
 	(info->stream,
-	 _("# internal disassembler error, unrecognised modifier (%c)"),
+	 "# internal disassembler error, unrecognised modifier (%c)",
 	 type);
       abort ();
     }
@@ -5747,51 +5750,51 @@
 {
   unsigned int i;
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
 The following MIPS specific disassembler options are supported for use\n\
-with the -M switch (multiple options should be separated by commas):\n"));
+with the -M switch (multiple options should be separated by commas):\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   gpr-names=ABI            Print GPR names according to  specified ABI.\n\
-                           Default: based on binary being disassembled.\n"));
+                           Default: based on binary being disassembled.\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   fpr-names=ABI            Print FPR names according to specified ABI.\n\
-                           Default: numeric.\n"));
+                           Default: numeric.\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   cp0-names=ARCH           Print CP0 register names according to\n\
                            specified architecture.\n\
-                           Default: based on binary being disassembled.\n"));
+                           Default: based on binary being disassembled.\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   hwr-names=ARCH           Print HWR names according to specified\n\
 			   architecture.\n\
-                           Default: based on binary being disassembled.\n"));
+                           Default: based on binary being disassembled.\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   reg-names=ABI            Print GPR and FPR names according to\n\
-                           specified ABI.\n"));
+                           specified ABI.\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   reg-names=ARCH           Print CP0 register and HWR names according to\n\
-                           specified architecture.\n"));
+                           specified architecture.\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   For the options above, the following values are supported for \"ABI\":\n\
-   "));
+   ");
   for (i = 0; i < ARRAY_SIZE (mips_abi_choices); i++)
     fprintf (stream, " %s", mips_abi_choices[i].name);
-  fprintf (stream, _("\n"));
+  fprintf (stream, "\n");
 
-  fprintf (stream, _("\n\
+  fprintf (stream, "\n\
   For the options above, The following values are supported for \"ARCH\":\n\
-   "));
+   ");
   for (i = 0; i < ARRAY_SIZE (mips_arch_choices); i++)
     if (*mips_arch_choices[i].name != '\0')
       fprintf (stream, " %s", mips_arch_choices[i].name);
-  fprintf (stream, _("\n"));
+  fprintf (stream, "\n");
 
-  fprintf (stream, _("\n"));
+  fprintf (stream, "\n");
 }
 #endif
diff --git a/disas/moxie.c b/disas/moxie.c
index 4c5f180..70b49ed 100644
--- a/disas/moxie.c
+++ b/disas/moxie.c
@@ -14,7 +14,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>. */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #define STATIC_TABLE
 #define DEFINE_TABLE
 
diff --git a/disas/ppc.c b/disas/ppc.c
index 99c4cbc..052cebe 100644
--- a/disas/ppc.c
+++ b/disas/ppc.c
@@ -18,6 +18,7 @@
 You should have received a copy of the GNU General Public License
 along with this file; see the file COPYING.  If not,
 see <http://www.gnu.org/licenses/>.  */
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 #define BFD_DEFAULT_TARGET_SIZE 64
 
@@ -1119,7 +1120,7 @@
 	   const char **errmsg)
 {
   if (!valid_bo (value, dialect, 0))
-    *errmsg = _("invalid conditional option");
+    *errmsg = "invalid conditional option";
   return insn | ((value & 0x1f) << 21);
 }
 
@@ -1147,9 +1148,9 @@
 	    const char **errmsg)
 {
   if (!valid_bo (value, dialect, 0))
-    *errmsg = _("invalid conditional option");
+    *errmsg = "invalid conditional option";
   else if ((value & 1) != 0)
-    *errmsg = _("attempt to set y bit when using + or - modifier");
+    *errmsg = "attempt to set y bit when using + or - modifier";
 
   return insn | ((value & 0x1f) << 21);
 }
@@ -1181,7 +1182,7 @@
     {
       if (value == 0 || (value & -value) != value)
 	{
-	  *errmsg = _("invalid mask field");
+	  *errmsg = "invalid mask field";
 	  value = 0;
 	}
     }
@@ -1207,7 +1208,7 @@
   /* Any other value on mfcr is an error.  */
   else if ((insn & (0x3ff << 1)) == 19 << 1)
     {
-      *errmsg = _("ignoring invalid mfcr mask");
+      *errmsg = "ignoring invalid mfcr mask";
       value = 0;
     }
 
@@ -1257,7 +1258,7 @@
 
   if (uval == 0)
     {
-      *errmsg = _("illegal bitmask");
+      *errmsg = "illegal bitmask";
       return insn;
     }
 
@@ -1292,7 +1293,7 @@
     me = 32;
 
   if (count != 2 && (count != 0 || ! last))
-    *errmsg = _("illegal bitmask");
+    *errmsg = "illegal bitmask";
 
   return insn | (mb << 6) | ((me - 1) << 1);
 }
@@ -1412,7 +1413,7 @@
 	    const char **errmsg)
 {
   if ((unsigned long) value >= ((insn >> 21) & 0x1f))
-    *errmsg = _("index register in load range");
+    *errmsg = "index register in load range";
   return insn | ((value & 0x1f) << 16);
 }
 
@@ -1428,7 +1429,7 @@
   long rtvalue = (insn & RT_MASK) >> 21;
 
   if (value == rtvalue)
-    *errmsg = _("source and target register operands must be different");
+    *errmsg = "source and target register operands must be different";
   return insn | ((value & 0x1f) << 16);
 }
 
@@ -1443,7 +1444,7 @@
 	    const char **errmsg)
 {
   if (value == 0)
-    *errmsg = _("invalid register operand when updating");
+    *errmsg = "invalid register operand when updating";
   return insn | ((value & 0x1f) << 16);
 }
 
@@ -1525,7 +1526,7 @@
   if (value > 7
       || (value > 3
 	  && (dialect & (PPC_OPCODE_BOOKE | PPC_OPCODE_403)) == 0))
-    *errmsg = _("invalid sprg number");
+    *errmsg = "invalid sprg number";
 
   /* If this is mfsprg4..7 then use spr 260..263 which can be read in
      user mode.  Anything else must use spr 272..279.  */
diff --git a/disas/s390.c b/disas/s390.c
index 25499ba..1f167d2 100644
--- a/disas/s390.c
+++ b/disas/s390.c
@@ -20,6 +20,7 @@
    Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston, MA
    02110-1301, USA.  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "disas/bfd.h"
 
@@ -106,10 +107,6 @@
 static const struct s390_opcode s390_opcodes[];
 static const int                s390_num_opcodes;
 
-/* A opcode format table for the .insn pseudo mnemonic.  */
-static const struct s390_opcode s390_opformats[];
-static const int                s390_num_opformats;
-
 /* Values defined for the flags field of a struct powerpc_opcode.  */
 
 /* The operands table is an array of struct s390_operand.  */
@@ -617,7 +614,7 @@
       names of the instruction format that you can find in the principals
       of operation.
    2) the last part of the definition (y in INSTR_x_y) gives you an idea
-      which operands the binary represenation of the instruction has.
+      which operands the binary representation of the instruction has.
       The meanings of the letters in y are:
       a - access register
       c - control register
@@ -631,7 +628,7 @@
       m - mode field, 4 bit
       0 - operand skipped.
       The order of the letters reflects the layout of the format in
-      storage and not the order of the paramaters of the instructions.
+      storage and not the order of the parameters of the instructions.
       The use of the letters is not a 100% match with the PoP but it is
       quite close.
 
@@ -844,37 +841,6 @@
 #define MASK_SIY_DRI     { 0xff, 0x00, 0x00, 0x00, 0x00, 0xff }
 /* QEMU-END */
 
-/* The opcode formats table (blueprints for .insn pseudo mnemonic).  */
-
-static const struct s390_opcode s390_opformats[] =
-  {
-  { "e",	OP8(0x00LL),	MASK_E,		INSTR_E,	3, 0 },
-  { "ri",	OP8(0x00LL),	MASK_RI_RI,	INSTR_RI_RI,	3, 0 },
-  { "rie",	OP8(0x00LL),	MASK_RIE_RRP,	INSTR_RIE_RRP,	3, 0 },
-  { "ril",	OP8(0x00LL),	MASK_RIL_RP,	INSTR_RIL_RP,	3, 0 },
-  { "rilu",	OP8(0x00LL),	MASK_RIL_RU,	INSTR_RIL_RU,	3, 0 },
-  { "rr",	OP8(0x00LL),	MASK_RR_RR,	INSTR_RR_RR,	3, 0 },
-  { "rre",	OP8(0x00LL),	MASK_RRE_RR,	INSTR_RRE_RR,	3, 0 },
-  { "rrf",	OP8(0x00LL),	MASK_RRF_RURR,	INSTR_RRF_RURR,	3, 0 },
-  { "rs",	OP8(0x00LL),	MASK_RS_RRRD,	INSTR_RS_RRRD,	3, 0 },
-  { "rse",	OP8(0x00LL),	MASK_RSE_RRRD,	INSTR_RSE_RRRD,	3, 0 },
-  { "rsi",	OP8(0x00LL),	MASK_RSI_RRP,	INSTR_RSI_RRP,	3, 0 },
-  { "rsy",	OP8(0x00LL),	MASK_RSY_RRRD,	INSTR_RSY_RRRD,	3, 3 },
-  { "rx",	OP8(0x00LL),	MASK_RX_RRRD,	INSTR_RX_RRRD,	3, 0 },
-  { "rxe",	OP8(0x00LL),	MASK_RXE_RRRD,	INSTR_RXE_RRRD,	3, 0 },
-  { "rxf",	OP8(0x00LL),	MASK_RXF_RRRDR,	INSTR_RXF_RRRDR,3, 0 },
-  { "rxy",	OP8(0x00LL),	MASK_RXY_RRRD,	INSTR_RXY_RRRD,	3, 3 },
-  { "s",	OP8(0x00LL),	MASK_S_RD,	INSTR_S_RD,	3, 0 },
-  { "si",	OP8(0x00LL),	MASK_SI_URD,	INSTR_SI_URD,	3, 0 },
-  { "siy",	OP8(0x00LL),	MASK_SIY_URD,	INSTR_SIY_URD,	3, 3 },
-  { "ss",	OP8(0x00LL),	MASK_SS_RRRDRD,	INSTR_SS_RRRDRD,3, 0 },
-  { "sse",	OP8(0x00LL),	MASK_SSE_RDRD,	INSTR_SSE_RDRD,	3, 0 },
-  { "ssf",	OP8(0x00LL),	MASK_SSF_RRDRD,	INSTR_SSF_RRDRD,3, 0 },
-};
-
-static const int s390_num_opformats =
-  sizeof (s390_opformats) / sizeof (s390_opformats[0]);
-
 /* include "s390-opc.tab" generated from opcodes/s390-opc.txt rev 1.17 */
 /* The opcode table. This file was generated by s390-mkopc.
 
diff --git a/disas/sh4.c b/disas/sh4.c
index f6cadd5..8b0415d 100644
--- a/disas/sh4.c
+++ b/disas/sh4.c
@@ -15,7 +15,7 @@
    You should have received a copy of the GNU General Public License
    along with this program; if not, see <http://www.gnu.org/licenses/>.  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 #define DEFINE_TABLE
@@ -332,7 +332,7 @@
 
 #ifdef DEFINE_TABLE
 
-const sh_opcode_info sh_table[] =
+static const sh_opcode_info sh_table[] =
   {
 /* 0111nnnni8*1.... add #<imm>,<REG_N>  */{"add",{A_IMM,A_REG_N},{HEX_7,REG_N,IMM0_8}, arch_sh1_up},
 
diff --git a/disas/sparc.c b/disas/sparc.c
index 8e755d1..f120f4e 100644
--- a/disas/sparc.c
+++ b/disas/sparc.c
@@ -26,7 +26,7 @@
    along with GAS or GDB; see the file COPYING. If not,
    see <http://www.gnu.org/licenses/>.  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 
 /* The SPARC opcode table (and other related data) is defined in
@@ -80,19 +80,6 @@
   short supported;
 } sparc_opcode_arch;
 
-static const struct sparc_opcode_arch sparc_opcode_archs[];
-
-/* Return the bitmask of supported architectures for ARCH.  */
-#define SPARC_OPCODE_SUPPORTED(ARCH) (sparc_opcode_archs[ARCH].supported)
-
-/* Non-zero if ARCH1 conflicts with ARCH2.
-   IE: ARCH1 as a supported bit set that ARCH2 doesn't, and vice versa.  */
-#define SPARC_OPCODE_CONFLICT_P(ARCH1, ARCH2) \
- (((SPARC_OPCODE_SUPPORTED (ARCH1) & SPARC_OPCODE_SUPPORTED (ARCH2)) \
-   != SPARC_OPCODE_SUPPORTED (ARCH1)) \
-  && ((SPARC_OPCODE_SUPPORTED (ARCH1) & SPARC_OPCODE_SUPPORTED (ARCH2)) \
-     != SPARC_OPCODE_SUPPORTED (ARCH2)))
-
 /* Structure of an opcode table entry.  */
 
 typedef struct sparc_opcode
@@ -301,25 +288,6 @@
    otherwise.  */
 #define v9notv9a        (MASK_V9)
 
-/* Table of opcode architectures.
-   The order is defined in opcode/sparc.h.  */
-
-static const struct sparc_opcode_arch sparc_opcode_archs[] =
-{
-  { "v6", MASK_V6 },
-  { "v7", MASK_V6 | MASK_V7 },
-  { "v8", MASK_V6 | MASK_V7 | MASK_V8 },
-  { "sparclet", MASK_V6 | MASK_V7 | MASK_V8 | MASK_SPARCLET },
-  { "sparclite", MASK_V6 | MASK_V7 | MASK_V8 | MASK_SPARCLITE },
-  /* ??? Don't some v8 privileged insns conflict with v9?  */
-  { "v9", MASK_V6 | MASK_V7 | MASK_V8 | MASK_V9 },
-  /* v9 with ultrasparc additions */
-  { "v9a", MASK_V6 | MASK_V7 | MASK_V8 | MASK_V9 | MASK_V9A },
-  /* v9 with cheetah additions */
-  { "v9b", MASK_V6 | MASK_V7 | MASK_V8 | MASK_V9 | MASK_V9A | MASK_V9B },
-  { NULL, 0 }
-};
-
 /* Branch condition field.  */
 #define COND(x)         (((x) & 0xf) << 25)
 
@@ -2526,7 +2494,7 @@
       fprintf
         (stderr,
          /* xgettext:c-format */
-         _("Internal error:  bad sparc-opcode.h: \"%s\", %#.8lx, %#.8lx\n"),
+         "Internal error:  bad sparc-opcode.h: \"%s\", %#.8lx, %#.8lx\n",
          op0->name, match0, lose0);
       op0->lose &= ~op0->match;
       lose0 = op0->lose;
@@ -2537,7 +2505,7 @@
       fprintf
         (stderr,
          /* xgettext:c-format */
-         _("Internal error: bad sparc-opcode.h: \"%s\", %#.8lx, %#.8lx\n"),
+         "Internal error: bad sparc-opcode.h: \"%s\", %#.8lx, %#.8lx\n",
          op1->name, match1, lose1);
       op1->lose &= ~op1->match;
       lose1 = op1->lose;
@@ -2587,7 +2555,7 @@
       else
         fprintf (stderr,
                  /* xgettext:c-format */
-                 _("Internal error: bad sparc-opcode.h: \"%s\" == \"%s\"\n"),
+                 "Internal error: bad sparc-opcode.h: \"%s\" == \"%s\"\n",
                  op0->name, op1->name);
     }
 
@@ -2654,8 +2622,7 @@
 
   memset (hash_table, 0, HASH_SIZE * sizeof (hash_table[0]));
   memset (hash_count, 0, HASH_SIZE * sizeof (hash_count[0]));
-  if (hash_buf != NULL)
-    free (hash_buf);
+  free(hash_buf);
   hash_buf = malloc (sizeof (* hash_buf) * num_opcodes);
   for (i = num_opcodes - 1; i >= 0; --i)
     {
diff --git a/disas/tci.c b/disas/tci.c
index a606b63..1cdf5ee 100644
--- a/disas/tci.c
+++ b/disas/tci.c
@@ -17,6 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "disas/bfd.h"
 #include "tcg/tcg.h"
 
diff --git a/dma-helpers.c b/dma-helpers.c
index 6918572..9defc10 100644
--- a/dma-helpers.c
+++ b/dma-helpers.c
@@ -7,10 +7,10 @@
  * (GNU GPL), version 2 or later.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 #include "trace.h"
-#include "qemu/range.h"
 #include "qemu/thread.h"
 #include "qemu/main-loop.h"
 
@@ -28,7 +28,8 @@
     memset(fillbuf, c, FILLBUF_SIZE);
     while (len > 0) {
         l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
-        error |= address_space_rw(as, addr, fillbuf, l, true);
+        error |= address_space_rw(as, addr, MEMTXATTRS_UNSPECIFIED,
+                                  fillbuf, l, true);
         len -= l;
         addr += l;
     }
@@ -69,16 +70,17 @@
 
 typedef struct {
     BlockAIOCB common;
-    BlockBackend *blk;
+    AioContext *ctx;
     BlockAIOCB *acb;
     QEMUSGList *sg;
-    uint64_t sector_num;
+    uint64_t offset;
     DMADirection dir;
     int sg_cur_index;
     dma_addr_t sg_cur_byte;
     QEMUIOVector iov;
     QEMUBH *bh;
     DMAIOFunc *io_func;
+    void *io_func_opaque;
 } DMAAIOCB;
 
 static void dma_blk_cb(void *opaque, int ret);
@@ -92,14 +94,6 @@
     dma_blk_cb(dbs, 0);
 }
 
-static void continue_after_map_failure(void *opaque)
-{
-    DMAAIOCB *dbs = (DMAAIOCB *)opaque;
-
-    dbs->bh = qemu_bh_new(reschedule_dma, dbs);
-    qemu_bh_schedule(dbs->bh);
-}
-
 static void dma_blk_unmap(DMAAIOCB *dbs)
 {
     int i;
@@ -137,7 +131,7 @@
     trace_dma_blk_cb(dbs, ret);
 
     dbs->acb = NULL;
-    dbs->sector_num += dbs->iov.size / 512;
+    dbs->offset += dbs->iov.size;
 
     if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) {
         dma_complete(dbs, ret);
@@ -161,7 +155,8 @@
 
     if (dbs->iov.size == 0) {
         trace_dma_map_wait(dbs);
-        cpu_register_map_client(dbs, continue_after_map_failure);
+        dbs->bh = aio_bh_new(dbs->ctx, reschedule_dma, dbs);
+        cpu_register_map_client(dbs->bh);
         return;
     }
 
@@ -169,8 +164,8 @@
         qemu_iovec_discard_back(&dbs->iov, dbs->iov.size & ~BDRV_SECTOR_MASK);
     }
 
-    dbs->acb = dbs->io_func(dbs->blk, dbs->sector_num, &dbs->iov,
-                            dbs->iov.size / 512, dma_blk_cb, dbs);
+    dbs->acb = dbs->io_func(dbs->offset, &dbs->iov,
+                            dma_blk_cb, dbs, dbs->io_func_opaque);
     assert(dbs->acb);
 }
 
@@ -183,31 +178,45 @@
     if (dbs->acb) {
         blk_aio_cancel_async(dbs->acb);
     }
+    if (dbs->bh) {
+        cpu_unregister_map_client(dbs->bh);
+        qemu_bh_delete(dbs->bh);
+        dbs->bh = NULL;
+    }
 }
 
+static AioContext *dma_get_aio_context(BlockAIOCB *acb)
+{
+    DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);
+
+    return dbs->ctx;
+}
 
 static const AIOCBInfo dma_aiocb_info = {
     .aiocb_size         = sizeof(DMAAIOCB),
     .cancel_async       = dma_aio_cancel,
+    .get_aio_context    = dma_get_aio_context,
 };
 
-BlockAIOCB *dma_blk_io(
-    BlockBackend *blk, QEMUSGList *sg, uint64_t sector_num,
-    DMAIOFunc *io_func, BlockCompletionFunc *cb,
+BlockAIOCB *dma_blk_io(AioContext *ctx,
+    QEMUSGList *sg, uint64_t offset,
+    DMAIOFunc *io_func, void *io_func_opaque,
+    BlockCompletionFunc *cb,
     void *opaque, DMADirection dir)
 {
-    DMAAIOCB *dbs = blk_aio_get(&dma_aiocb_info, blk, cb, opaque);
+    DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, NULL, cb, opaque);
 
-    trace_dma_blk_io(dbs, blk, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
+    trace_dma_blk_io(dbs, io_func_opaque, offset, (dir == DMA_DIRECTION_TO_DEVICE));
 
     dbs->acb = NULL;
-    dbs->blk = blk;
     dbs->sg = sg;
-    dbs->sector_num = sector_num;
+    dbs->ctx = ctx;
+    dbs->offset = offset;
     dbs->sg_cur_index = 0;
     dbs->sg_cur_byte = 0;
     dbs->dir = dir;
     dbs->io_func = io_func;
+    dbs->io_func_opaque = io_func_opaque;
     dbs->bh = NULL;
     qemu_iovec_init(&dbs->iov, sg->nsg);
     dma_blk_cb(dbs, 0);
@@ -215,19 +224,39 @@
 }
 
 
+static
+BlockAIOCB *dma_blk_read_io_func(int64_t offset, QEMUIOVector *iov,
+                                 BlockCompletionFunc *cb, void *cb_opaque,
+                                 void *opaque)
+{
+    BlockBackend *blk = opaque;
+    return blk_aio_preadv(blk, offset, iov, 0, cb, cb_opaque);
+}
+
 BlockAIOCB *dma_blk_read(BlockBackend *blk,
-                         QEMUSGList *sg, uint64_t sector,
+                         QEMUSGList *sg, uint64_t offset,
                          void (*cb)(void *opaque, int ret), void *opaque)
 {
-    return dma_blk_io(blk, sg, sector, blk_aio_readv, cb, opaque,
+    return dma_blk_io(blk_get_aio_context(blk),
+                      sg, offset, dma_blk_read_io_func, blk, cb, opaque,
                       DMA_DIRECTION_FROM_DEVICE);
 }
 
+static
+BlockAIOCB *dma_blk_write_io_func(int64_t offset, QEMUIOVector *iov,
+                                  BlockCompletionFunc *cb, void *cb_opaque,
+                                  void *opaque)
+{
+    BlockBackend *blk = opaque;
+    return blk_aio_pwritev(blk, offset, iov, 0, cb, cb_opaque);
+}
+
 BlockAIOCB *dma_blk_write(BlockBackend *blk,
-                          QEMUSGList *sg, uint64_t sector,
+                          QEMUSGList *sg, uint64_t offset,
                           void (*cb)(void *opaque, int ret), void *opaque)
 {
-    return dma_blk_io(blk, sg, sector, blk_aio_writev, cb, opaque,
+    return dma_blk_io(blk_get_aio_context(blk),
+                      sg, offset, dma_blk_write_io_func, blk, cb, opaque,
                       DMA_DIRECTION_TO_DEVICE);
 }
 
diff --git a/docs/aio_notify.promela b/docs/aio_notify.promela
index ad3f6f0..fccc7ee 100644
--- a/docs/aio_notify.promela
+++ b/docs/aio_notify.promela
@@ -1,5 +1,5 @@
 /*
- * This model describes the interaction between aio_set_dispatching()
+ * This model describes the interaction between ctx->notify_me
  * and aio_notify().
  *
  * Author: Paolo Bonzini <pbonzini@redhat.com>
@@ -14,57 +14,53 @@
  *     spin -a docs/aio_notify.promela
  *     gcc -O2 pan.c
  *     ./a.out -a
+ *
+ * To verify it (with a bug planted in the model):
+ *     spin -a -DBUG docs/aio_notify.promela
+ *     gcc -O2 pan.c
+ *     ./a.out -a
  */
 
 #define MAX   4
 #define LAST  (1 << (MAX - 1))
 #define FINAL ((LAST << 1) - 1)
 
-bool dispatching;
+bool notify_me;
 bool event;
 
-int req, done;
+int req;
+int done;
 
 active proctype waiter()
 {
-     int fetch, blocking;
+    int fetch;
 
-     do
-        :: done != FINAL -> {
-            // Computing "blocking" is separate from execution of the
-            // "bottom half"
-            blocking = (req == 0);
+    do
+        :: true -> {
+            notify_me++;
 
-            // This is our "bottom half"
+            if
+#ifndef BUG
+                :: (req > 0) -> skip;
+#endif
+                :: else ->
+                    // Wait for a nudge from the other side
+                    do
+                        :: event == 1 -> { event = 0; break; }
+                    od;
+            fi;
+
+            notify_me--;
+
             atomic { fetch = req; req = 0; }
             done = done | fetch;
-
-            // Wait for a nudge from the other side
-            do
-                :: event == 1 -> { event = 0; break; }
-                :: !blocking  -> break;
-            od;
-
-            dispatching = 1;
-
-            // If you are simulating this model, you may want to add
-            // something like this here:
-            //
-            //      int foo; foo++; foo++; foo++;
-            //
-            // This only wastes some time and makes it more likely
-            // that the notifier process hits the "fast path".
-
-            dispatching = 0;
         }
-        :: else -> break;
     od
 }
 
 active proctype notifier()
 {
     int next = 1;
-    int sets = 0;
 
     do
         :: next <= LAST -> {
@@ -74,8 +70,8 @@
 
             // aio_notify
             if
-                :: dispatching == 0 -> sets++; event = 1;
-                :: else             -> skip;
+                :: notify_me == 1 -> event = 1;
+                :: else           -> printf("Skipped event_notifier_set\n"); skip;
             fi;
 
             // Test both synchronous and asynchronous delivery
@@ -86,19 +82,12 @@
                 :: 1 -> skip;
             fi;
         }
-        :: else -> break;
     od;
-    printf("Skipped %d event_notifier_set\n", MAX - sets);
 }
 
-#define p (done == FINAL)
-
-never  {
-    do
-        :: 1                      // after an arbitrarily long prefix
-        :: p -> break             // p becomes true
-    od;
-    do
-        :: !p -> accept: break    // it then must remains true forever after
-    od
+never { /* [] done < FINAL */
+accept_init:
+        do
+        :: done < FINAL -> skip;
+        od;
 }
diff --git a/docs/aio_notify_accept.promela b/docs/aio_notify_accept.promela
new file mode 100644
index 0000000..9cef2c9
--- /dev/null
+++ b/docs/aio_notify_accept.promela
@@ -0,0 +1,152 @@
+/*
+ * This model describes the interaction between ctx->notified
+ * and ctx->notifier.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This file is in the public domain.  If you really want a license,
+ * the WTFPL will do.
+ *
+ * To verify the buggy version:
+ *     spin -a -DBUG1 docs/aio_notify_bug.promela
+ *     gcc -O2 pan.c
+ *     ./a.out -a -f
+ * (or -DBUG2)
+ *
+ * To verify the fixed version:
+ *     spin -a docs/aio_notify_bug.promela
+ *     gcc -O2 pan.c
+ *     ./a.out -a -f
+ *
+ * Add -DCHECK_REQ to test an alternative invariant and the
+ * "notify_me" optimization.
+ */
+
+int notify_me;
+bool notified;
+bool event;
+bool req;
+bool notifier_done;
+
+#ifdef CHECK_REQ
+#define USE_NOTIFY_ME 1
+#else
+#define USE_NOTIFY_ME 0
+#endif
+
+#ifdef BUG
+#error Please define BUG1 or BUG2 instead.
+#endif
+
+active proctype notifier()
+{
+    do
+        :: true -> {
+            req = 1;
+            if
+               :: !USE_NOTIFY_ME || notify_me ->
+#if defined BUG1
+                   /* CHECK_REQ does not detect this bug! */
+                   notified = 1;
+                   event = 1;
+#elif defined BUG2
+                   if
+                      :: !notified -> event = 1;
+                      :: else -> skip;
+                   fi;
+                   notified = 1;
+#else
+                   event = 1;
+                   notified = 1;
+#endif
+               :: else -> skip;
+            fi
+        }
+        :: true -> break;
+    od;
+    notifier_done = 1;
+}
+
+#define AIO_POLL                                                    \
+    notify_me++;                                                    \
+    if                                                              \
+        :: !req -> {                                                \
+            if                                                      \
+                :: event -> skip;                                   \
+            fi;                                                     \
+        }                                                           \
+        :: else -> skip;                                            \
+    fi;                                                             \
+    notify_me--;                                                    \
+                                                                    \
+    atomic { old = notified; notified = 0; }                        \
+    if                                                              \
+       :: old -> event = 0;                                         \
+       :: else -> skip;                                             \
+    fi;                                                             \
+                                                                    \
+    req = 0;
+
+active proctype waiter()
+{
+    bool old;
+
+    do
+       :: true -> AIO_POLL;
+    od;
+}
+
+/* Same as waiter(), but disappears after a while.  */
+active proctype temporary_waiter()
+{
+    bool old;
+
+    do
+       :: true -> AIO_POLL;
+       :: true -> break;
+    od;
+}
+
+#ifdef CHECK_REQ
+never {
+    do
+        :: req -> goto accept_if_req_not_eventually_false;
+        :: true -> skip;
+    od;
+
+accept_if_req_not_eventually_false:
+    if
+        :: req -> goto accept_if_req_not_eventually_false;
+    fi;
+    assert(0);
+}
+
+#else
+/* There must be infinitely many transitions of event as long
+ * as the notifier does not exit.
+ *
+ * If event stayed always true, the waiters would be busy looping.
+ * If event stayed always false, the waiters would be sleeping
+ * forever.
+ */
+never {
+    do
+        :: !event    -> goto accept_if_event_not_eventually_true;
+        :: event     -> goto accept_if_event_not_eventually_false;
+        :: true      -> skip;
+    od;
+
+accept_if_event_not_eventually_true:
+    if
+        :: !event && notifier_done  -> do :: true -> skip; od;
+        :: !event && !notifier_done -> goto accept_if_event_not_eventually_true;
+    fi;
+    assert(0);
+
+accept_if_event_not_eventually_false:
+    if
+        :: event     -> goto accept_if_event_not_eventually_false;
+    fi;
+    assert(0);
+}
+#endif
diff --git a/docs/aio_notify_bug.promela b/docs/aio_notify_bug.promela
new file mode 100644
index 0000000..b3bfca1
--- /dev/null
+++ b/docs/aio_notify_bug.promela
@@ -0,0 +1,140 @@
+/*
+ * This model describes a bug in aio_notify.  If ctx->notifier is
+ * cleared too late, a wakeup could be lost.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This file is in the public domain.  If you really want a license,
+ * the WTFPL will do.
+ *
+ * To verify the buggy version:
+ *     spin -a -DBUG docs/aio_notify_bug.promela
+ *     gcc -O2 pan.c
+ *     ./a.out -a -f
+ *
+ * To verify the fixed version:
+ *     spin -a docs/aio_notify_bug.promela
+ *     gcc -O2 pan.c
+ *     ./a.out -a -f
+ *
+ * Add -DCHECK_REQ to test an alternative invariant and the
+ * "notify_me" optimization.
+ */
+
+int notify_me;
+bool event;
+bool req;
+bool notifier_done;
+
+#ifdef CHECK_REQ
+#define USE_NOTIFY_ME 1
+#else
+#define USE_NOTIFY_ME 0
+#endif
+
+active proctype notifier()
+{
+    do
+        :: true -> {
+            req = 1;
+            if
+               :: !USE_NOTIFY_ME || notify_me -> event = 1;
+               :: else -> skip;
+            fi
+        }
+        :: true -> break;
+    od;
+    notifier_done = 1;
+}
+
+#ifdef BUG
+#define AIO_POLL                                                    \
+    notify_me++;                                                    \
+    if                                                              \
+        :: !req -> {                                                \
+            if                                                      \
+                :: event -> skip;                                   \
+            fi;                                                     \
+        }                                                           \
+        :: else -> skip;                                            \
+    fi;                                                             \
+    notify_me--;                                                    \
+                                                                    \
+    req = 0;                                                        \
+    event = 0;
+#else
+#define AIO_POLL                                                    \
+    notify_me++;                                                    \
+    if                                                              \
+        :: !req -> {                                                \
+            if                                                      \
+                :: event -> skip;                                   \
+            fi;                                                     \
+        }                                                           \
+        :: else -> skip;                                            \
+    fi;                                                             \
+    notify_me--;                                                    \
+                                                                    \
+    event = 0;                                                      \
+    req = 0;
+#endif
+
+active proctype waiter()
+{
+    do
+       :: true -> AIO_POLL;
+    od;
+}
+
+/* Same as waiter(), but disappears after a while.  */
+active proctype temporary_waiter()
+{
+    do
+       :: true -> AIO_POLL;
+       :: true -> break;
+    od;
+}
+
+#ifdef CHECK_REQ
+never {
+    do
+        :: req -> goto accept_if_req_not_eventually_false;
+        :: true -> skip;
+    od;
+
+accept_if_req_not_eventually_false:
+    if
+        :: req -> goto accept_if_req_not_eventually_false;
+    fi;
+    assert(0);
+}
+
+#else
+/* There must be infinitely many transitions of event as long
+ * as the notifier does not exit.
+ *
+ * If event stayed always true, the waiters would be busy looping.
+ * If event stayed always false, the waiters would be sleeping
+ * forever.
+ */
+never {
+    do
+        :: !event    -> goto accept_if_event_not_eventually_true;
+        :: event     -> goto accept_if_event_not_eventually_false;
+        :: true      -> skip;
+    od;
+
+accept_if_event_not_eventually_true:
+    if
+        :: !event && notifier_done  -> do :: true -> skip; od;
+        :: !event && !notifier_done -> goto accept_if_event_not_eventually_true;
+    fi;
+    assert(0);
+
+accept_if_event_not_eventually_false:
+    if
+        :: event     -> goto accept_if_event_not_eventually_false;
+    fi;
+    assert(0);
+}
+#endif
diff --git a/docs/atomics.txt b/docs/atomics.txt
index 6f2997b..c95950b 100644
--- a/docs/atomics.txt
+++ b/docs/atomics.txt
@@ -62,7 +62,7 @@
     typeof(*ptr) atomic_fetch_sub(ptr, val)
     typeof(*ptr) atomic_fetch_and(ptr, val)
     typeof(*ptr) atomic_fetch_or(ptr, val)
-    typeof(*ptr) atomic_xchg(ptr, val
+    typeof(*ptr) atomic_xchg(ptr, val)
     typeof(*ptr) atomic_cmpxchg(ptr, old, new)
 
 all of which return the old value of *ptr.  These operations are
@@ -281,7 +281,7 @@
 the guest!
 
 For the purposes of pairing, smp_read_barrier_depends() and smp_rmb()
-both count as read barriers.  A read barriers shall pair with a write
+both count as read barriers.  A read barrier shall pair with a write
 barrier or a full barrier; a write barrier shall pair with a read
 barrier or a full barrier.  A full barrier can pair with anything.
 For example:
@@ -294,7 +294,7 @@
                              smp_rmb();
                              y = a;
 
-Note that the "writing" thread are accessing the variables in the
+Note that the "writing" thread is accessing the variables in the
 opposite order as the "reading" thread.  This is expected: stores
 before the write barrier will normally match the loads after the
 read barrier, and vice versa.  The same is true for more than 2
@@ -326,21 +326,41 @@
   use a boxed atomic_t type; atomic operations in QEMU are polymorphic
   and use normal C types.
 
-- atomic_read and atomic_set in Linux give no guarantee at all;
-  atomic_read and atomic_set in QEMU include a compiler barrier
-  (similar to the ACCESS_ONCE macro in Linux).
+- Originally, atomic_read and atomic_set in Linux gave no guarantee
+  at all. Linux 4.1 updated them to implement volatile
+  semantics via ACCESS_ONCE (or the more recent READ/WRITE_ONCE).
 
-- most atomic read-modify-write operations in Linux return void;
-  in QEMU, all of them return the old value of the variable.
+  QEMU's atomic_read/set implement, if the compiler supports it, C11
+  atomic relaxed semantics, and volatile semantics otherwise.
+  Both semantics prevent the compiler from doing certain transformations;
+  the difference is that atomic accesses are guaranteed to be atomic,
+  while volatile accesses aren't. Thus, in the volatile case we just cross
+  our fingers hoping that the compiler will generate atomic accesses,
+  since we assume the variables passed are machine-word sized and
+  properly aligned.
+  No barriers are implied by atomic_read/set in either Linux or QEMU.
+
+- atomic read-modify-write operations in Linux are of three kinds:
+
+         atomic_OP          returns void
+         atomic_OP_return   returns new value of the variable
+         atomic_fetch_OP    returns the old value of the variable
+         atomic_cmpxchg     returns the old value of the variable
+
+  In QEMU, the second kind does not exist.  Currently Linux has
+  atomic_fetch_or only.  QEMU provides and, or, inc, dec, add, sub.
 
 - different atomic read-modify-write operations in Linux imply
   a different set of memory barriers; in QEMU, all of them enforce
   sequential consistency, which means they imply full memory barriers
   before and after the operation.
 
-- Linux does not have an equivalent of atomic_mb_read() and
-  atomic_mb_set().  In particular, note that set_mb() is a little
-  weaker than atomic_mb_set().
+- Linux does not have an equivalent of atomic_mb_set().  In particular,
+  note that smp_store_mb() is a little weaker than atomic_mb_set().
+  atomic_mb_read() compiles to the same instructions as Linux's
+  smp_load_acquire(), but this should be treated as an implementation
+  detail.  If required, QEMU might later add atomic_load_acquire() and
+  atomic_store_release() macros.
 
 
 SOURCES
diff --git a/docs/bitmaps.md b/docs/bitmaps.md
new file mode 100644
index 0000000..a2e8d51
--- /dev/null
+++ b/docs/bitmaps.md
@@ -0,0 +1,505 @@
+<!--
+Copyright 2015 John Snow <jsnow@redhat.com> and Red Hat, Inc.
+All rights reserved.
+
+This file is licensed via The FreeBSD Documentation License, the full text of
+which is included at the end of this document.
+-->
+
+# Dirty Bitmaps and Incremental Backup
+
+* Dirty Bitmaps are objects that track which data needs to be backed up for the
+  next incremental backup.
+
+* Dirty bitmaps can be created at any time and attached to any node
+  (not just complete drives.)
+
+## Dirty Bitmap Names
+
+* A dirty bitmap's name is unique to the node, but bitmaps attached to different
+  nodes can share the same name.
+
+* Dirty bitmaps created for internal use by QEMU may be anonymous and have no
+  name, but any user-created bitmaps may not be. There can be any number of
+  anonymous bitmaps per node.
+
+* The name of a user-created bitmap must not be empty ("").
+
+## Bitmap Modes
+
+* A Bitmap can be "frozen," which means that it is currently in-use by a backup
+  operation and cannot be deleted, renamed, written to, reset,
+  etc.
+
+* The normal operating mode for a bitmap is "active."
+
+## Basic QMP Usage
+
+### Supported Commands ###
+
+* block-dirty-bitmap-add
+* block-dirty-bitmap-remove
+* block-dirty-bitmap-clear
+
+### Creation
+
+* To create a new bitmap, enabled, on the drive with id=drive0:
+
+```json
+{ "execute": "block-dirty-bitmap-add",
+  "arguments": {
+    "node": "drive0",
+    "name": "bitmap0"
+  }
+}
+```
+
+* This bitmap will have a default granularity that matches the cluster size of
+  its associated drive, if available, clamped to between [4KiB, 64KiB].
+  The current default for qcow2 is 64KiB.
+
+* To create a new bitmap that tracks changes in 32KiB segments:
+
+```json
+{ "execute": "block-dirty-bitmap-add",
+  "arguments": {
+    "node": "drive0",
+    "name": "bitmap0",
+    "granularity": 32768
+  }
+}
+```
+
+### Deletion
+
+* Bitmaps that are frozen cannot be deleted.
+
+* Deleting the bitmap does not impact any other bitmaps attached to the same
+  node, nor does it affect any backups already created from this node.
+
+* Because bitmaps are only unique to the node to which they are attached,
+  you must specify the node/drive name here, too.
+
+```json
+{ "execute": "block-dirty-bitmap-remove",
+  "arguments": {
+    "node": "drive0",
+    "name": "bitmap0"
+  }
+}
+```
+
+### Resetting
+
+* Resetting a bitmap will clear all information it holds.
+
+* An incremental backup created from an empty bitmap will copy no data,
+  as if nothing has changed.
+
+```json
+{ "execute": "block-dirty-bitmap-clear",
+  "arguments": {
+    "node": "drive0",
+    "name": "bitmap0"
+  }
+}
+```
+
+## Transactions
+
+### Justification
+
+Bitmaps can be safely modified when the VM is paused or halted by using
+the basic QMP commands. For instance, you might perform the following actions:
+
+1. Boot the VM in a paused state.
+2. Create a full drive backup of drive0.
+3. Create a new bitmap attached to drive0.
+4. Resume execution of the VM.
+5. Incremental backups are ready to be created.
+
+At this point, the bitmap and drive backup would be correctly in sync,
+and incremental backups made from this point forward would be correctly aligned
+to the full drive backup.
+
+This is not particularly useful if we decide we want to start incremental
+backups after the VM has been running for a while, for which we will need to
+perform actions such as the following:
+
+1. Boot the VM and begin execution.
+2. Using a single transaction, perform the following operations:
+    * Create bitmap0.
+    * Create a full drive backup of drive0.
+3. Incremental backups are now ready to be created.
+
+### Supported Bitmap Transactions
+
+* block-dirty-bitmap-add
+* block-dirty-bitmap-clear
+
+The usages are identical to their respective QMP commands, but see below
+for examples.
+
+### Example: New Incremental Backup
+
+As outlined in the justification, perhaps we want to create a new incremental
+backup chain attached to a drive.
+
+```json
+{ "execute": "transaction",
+  "arguments": {
+    "actions": [
+      {"type": "block-dirty-bitmap-add",
+       "data": {"node": "drive0", "name": "bitmap0"} },
+      {"type": "drive-backup",
+       "data": {"device": "drive0", "target": "/path/to/full_backup.img",
+                "sync": "full", "format": "qcow2"} }
+    ]
+  }
+}
+```
+
+### Example: New Incremental Backup Anchor Point
+
+Maybe we just want to create a new full backup with an existing bitmap and
+want to reset the bitmap to track the new chain.
+
+```json
+{ "execute": "transaction",
+  "arguments": {
+    "actions": [
+      {"type": "block-dirty-bitmap-clear",
+       "data": {"node": "drive0", "name": "bitmap0"} },
+      {"type": "drive-backup",
+       "data": {"device": "drive0", "target": "/path/to/new_full_backup.img",
+                "sync": "full", "format": "qcow2"} }
+    ]
+  }
+}
+```
+
+## Incremental Backups
+
+The star of the show.
+
+**Nota Bene!** Only incremental backups of entire drives are supported for now.
+So despite the fact that you can attach a bitmap to any arbitrary node, they are
+only currently useful when attached to the root node. This is because
+drive-backup only supports drives/devices instead of arbitrary nodes.
+
+### Example: First Incremental Backup
+
+1. Create a full backup and sync it to the dirty bitmap, as in the transactional
+examples above; or with the VM offline, manually create a full copy and then
+create a new bitmap before the VM begins execution.
+
+    * Let's assume the full backup is named 'full_backup.img'.
+    * Let's assume the bitmap you created is 'bitmap0' attached to 'drive0'.
+
+2. Create a destination image for the incremental backup that utilizes the
+full backup as a backing image.
+
+    * Let's assume it is named 'incremental.0.img'.
+
+    ```sh
+    # qemu-img create -f qcow2 incremental.0.img -b full_backup.img -F qcow2
+    ```
+
+3. Issue the incremental backup command:
+
+    ```json
+    { "execute": "drive-backup",
+      "arguments": {
+        "device": "drive0",
+        "bitmap": "bitmap0",
+        "target": "incremental.0.img",
+        "format": "qcow2",
+        "sync": "incremental",
+        "mode": "existing"
+      }
+    }
+    ```
+
+### Example: Second Incremental Backup
+
+1. Create a new destination image for the incremental backup that points to the
+   previous one, e.g.: 'incremental.1.img'
+
+    ```sh
+    # qemu-img create -f qcow2 incremental.1.img -b incremental.0.img -F qcow2
+    ```
+
+2. Issue a new incremental backup command. The only difference here is that we
+   have changed the target image below.
+
+    ```json
+    { "execute": "drive-backup",
+      "arguments": {
+        "device": "drive0",
+        "bitmap": "bitmap0",
+        "target": "incremental.1.img",
+        "format": "qcow2",
+        "sync": "incremental",
+        "mode": "existing"
+      }
+    }
+    ```
+
+## Errors
+
+* In the event of an error that occurs after a backup job is successfully
+  launched, either by a direct QMP command or a QMP transaction, the user
+  will receive a BLOCK_JOB_COMPLETE event with a failure message, accompanied
+  by a BLOCK_JOB_ERROR event.
+
+* In the case of an event being cancelled, the user will receive a
+  BLOCK_JOB_CANCELLED event instead of a pair of COMPLETE and ERROR events.
+
+* In either case, the incremental backup data contained within the bitmap is
+  safely rolled back, and the data within the bitmap is not lost. The image
+  file created for the failed attempt can be safely deleted.
+
+* Once the underlying problem is fixed (e.g. more storage space is freed up),
+  you can simply retry the incremental backup command with the same bitmap.
+
+### Example
+
+1. Create a target image:
+
+    ```sh
+    # qemu-img create -f qcow2 incremental.0.img -b full_backup.img -F qcow2
+    ```
+
+2. Attempt to create an incremental backup via QMP:
+
+    ```json
+    { "execute": "drive-backup",
+      "arguments": {
+        "device": "drive0",
+        "bitmap": "bitmap0",
+        "target": "incremental.0.img",
+        "format": "qcow2",
+        "sync": "incremental",
+        "mode": "existing"
+      }
+    }
+    ```
+
+3. Receive an event notifying us of failure:
+
+    ```json
+    { "timestamp": { "seconds": 1424709442, "microseconds": 844524 },
+      "data": { "speed": 0, "offset": 0, "len": 67108864,
+                "error": "No space left on device",
+                "device": "drive1", "type": "backup" },
+      "event": "BLOCK_JOB_COMPLETED" }
+    ```
+
+4. Delete the failed incremental, and re-create the image.
+
+    ```sh
+    # rm incremental.0.img
+    # qemu-img create -f qcow2 incremental.0.img -b full_backup.img -F qcow2
+    ```
+
+5. Retry the command after fixing the underlying problem,
+   such as freeing up space on the backup volume:
+
+    ```json
+    { "execute": "drive-backup",
+      "arguments": {
+        "device": "drive0",
+        "bitmap": "bitmap0",
+        "target": "incremental.0.img",
+        "format": "qcow2",
+        "sync": "incremental",
+        "mode": "existing"
+      }
+    }
+    ```
+
+6. Receive confirmation that the job completed successfully:
+
+    ```json
+    { "timestamp": { "seconds": 1424709668, "microseconds": 526525 },
+      "data": { "device": "drive1", "type": "backup",
+                "speed": 0, "len": 67108864, "offset": 67108864},
+      "event": "BLOCK_JOB_COMPLETED" }
+    ```
+
+### Partial Transactional Failures
+
+* Sometimes, a transaction will succeed in launching and return success,
+  but then later the backup jobs themselves may fail. It is possible that
+  a management application may have to deal with a partial backup failure
+  after a successful transaction.
+
+* If multiple backup jobs are specified in a single transaction, when one of
+  them fails, it will not interact with the other backup jobs in any way.
+
+* The job(s) that succeeded will clear the dirty bitmap associated with the
+  operation, but the job(s) that failed will not. It is not "safe" to delete
+  any incremental backups that were created successfully in this scenario,
+  even though others failed.
+
+#### Example
+
+* QMP example highlighting two backup jobs:
+
+    ```json
+    { "execute": "transaction",
+      "arguments": {
+        "actions": [
+          { "type": "drive-backup",
+            "data": { "device": "drive0", "bitmap": "bitmap0",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d0-incr-1.qcow2" } },
+          { "type": "drive-backup",
+            "data": { "device": "drive1", "bitmap": "bitmap1",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d1-incr-1.qcow2" } },
+        ]
+      }
+    }
+    ```
+
+* QMP example response, highlighting one success and one failure:
+    * Acknowledgement that the Transaction was accepted and jobs were launched:
+        ```json
+        { "return": {} }
+        ```
+
+    * Later, QEMU sends notice that the first job was completed:
+        ```json
+        { "timestamp": { "seconds": 1447192343, "microseconds": 615698 },
+          "data": { "device": "drive0", "type": "backup",
+                     "speed": 0, "len": 67108864, "offset": 67108864 },
+          "event": "BLOCK_JOB_COMPLETED"
+        }
+        ```
+
+    * Later yet, QEMU sends notice that the second job has failed:
+        ```json
+        { "timestamp": { "seconds": 1447192399, "microseconds": 683015 },
+          "data": { "device": "drive1", "action": "report",
+                    "operation": "read" },
+          "event": "BLOCK_JOB_ERROR" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447192399, "microseconds": 685853 },
+          "data": { "speed": 0, "offset": 0, "len": 67108864,
+                    "error": "Input/output error",
+                    "device": "drive1", "type": "backup" },
+          "event": "BLOCK_JOB_COMPLETED" }
+
+* In the above example, "d0-incr-1.qcow2" is valid and must be kept,
+  but "d1-incr-1.qcow2" is invalid and should be deleted. If a VM-wide
+  incremental backup of all drives at a point-in-time is to be made,
+  new backups for both drives will need to be made, taking into account
+  that a new incremental backup for drive0 needs to be based on top of
+  "d0-incr-1.qcow2."
+
+### Grouped Completion Mode
+
+* While jobs launched by transactions normally complete or fail on their own,
+  it is possible to instruct them to complete or fail together as a group.
+
+* QMP transactions take an optional properties structure that can affect
+  the semantics of the transaction.
+
+* The "completion-mode" transaction property can be either "individual"
+  which is the default, legacy behavior described above, or "grouped,"
+  a new behavior detailed below.
+
+* Delayed Completion: In grouped completion mode, no jobs will report
+  success until all jobs are ready to report success.
+
+* Grouped failure: If any job fails in grouped completion mode, all remaining
+  jobs will be cancelled. Any incremental backups will restore their dirty
+  bitmap objects as if no backup command was ever issued.
+
+    * Regardless of if QEMU reports a particular incremental backup job as
+      CANCELLED or as an ERROR, the in-memory bitmap will be restored.
+
+#### Example
+
+* Here's the same example scenario from above with the new property:
+
+    ```json
+    { "execute": "transaction",
+      "arguments": {
+        "actions": [
+          { "type": "drive-backup",
+            "data": { "device": "drive0", "bitmap": "bitmap0",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d0-incr-1.qcow2" } },
+          { "type": "drive-backup",
+            "data": { "device": "drive1", "bitmap": "bitmap1",
+                      "format": "qcow2", "mode": "existing",
+                      "sync": "incremental", "target": "d1-incr-1.qcow2" } },
+        ],
+        "properties": {
+          "completion-mode": "grouped"
+        }
+      }
+    }
+    ```
+
+* QMP example response, highlighting a failure for drive2:
+    * Acknowledgement that the Transaction was accepted and jobs were launched:
+        ```json
+        { "return": {} }
+        ```
+
+    * Later, QEMU sends notice that the second job has errored out,
+      but that the first job was also cancelled:
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 632377 },
+          "data": { "device": "drive1", "action": "report",
+                    "operation": "read" },
+          "event": "BLOCK_JOB_ERROR" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 640074 },
+          "data": { "speed": 0, "offset": 0, "len": 67108864,
+                    "error": "Input/output error",
+                    "device": "drive1", "type": "backup" },
+          "event": "BLOCK_JOB_COMPLETED" }
+        ```
+
+        ```json
+        { "timestamp": { "seconds": 1447193702, "microseconds": 640163 },
+          "data": { "device": "drive0", "type": "backup", "speed": 0,
+                    "len": 67108864, "offset": 16777216 },
+          "event": "BLOCK_JOB_CANCELLED" }
+        ```
+
+<!--
+The FreeBSD Documentation License
+
+Redistribution and use in source (Markdown) and 'compiled' forms (SGML, HTML,
+PDF, PostScript, RTF and so forth) with or without modification, are permitted
+provided that the following conditions are met:
+
+Redistributions of source code (Markdown) must retain the above copyright
+notice, this list of conditions and the following disclaimer of this file
+unmodified.
+
+Redistributions in compiled form (transformed to other DTDs, converted to PDF,
+PostScript, RTF and other formats) must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS  BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+THIS DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+-->
diff --git a/docs/blkdebug.txt b/docs/blkdebug.txt
index b67a36d..43d8e8f 100644
--- a/docs/blkdebug.txt
+++ b/docs/blkdebug.txt
@@ -1,6 +1,6 @@
 Block I/O error injection using blkdebug
 ----------------------------------------
-Copyright (C) 2014 Red Hat Inc
+Copyright (C) 2014-2015 Red Hat Inc
 
 This work is licensed under the terms of the GNU GPL, version 2 or later.  See
 the COPYING file in the top-level directory.
@@ -92,8 +92,9 @@
 
   flush_to_disk - flush the host block device's disk cache
 
-See block/blkdebug.c:event_names[] for the full list of events.  You may need
-to grep block driver source code to understand the meaning of specific events.
+See qapi/block-core.json:BlkdebugEvent for the full list of events.
+You may need to grep block driver source code to understand the
+meaning of specific events.
 
 State transitions
 -----------------
diff --git a/docs/build-system.txt b/docs/build-system.txt
new file mode 100644
index 0000000..2af1e66
--- /dev/null
+++ b/docs/build-system.txt
@@ -0,0 +1,512 @@
+    The QEMU build system architecture
+    ==================================
+
+This document aims to help developers understand the architecture of the
+QEMU build system. As with projects using GNU autotools, the QEMU build
+system has two stages, first the developer runs the "configure" script
+to determine the local build environment characteristics, then they run
+"make" to build the project. There is about where the similarities with
+GNU autotools end, so try to forget what you know about them.
+
+
+Stage 1: configure
+==================
+
+The QEMU configure script is written directly in shell, and should be
+compatible with any POSIX shell, hence it uses #!/bin/sh. An important
+implication of this is that it is important to avoid using bash-isms on
+development platforms where bash is the primary host.
+
+In contrast to autoconf scripts, QEMU's configure is expected to be
+silent while it is checking for features. It will only display output
+when an error occurs, or to show the final feature enablement summary
+on completion.
+
+Adding new checks to the configure script usually comprises the
+following tasks:
+
+ - Initialize one or more variables with the default feature state.
+
+   Ideally features should auto-detect whether they are present,
+   so try to avoid hardcoding the initial state to either enabled
+   or disabled, as that forces the user to pass a --enable-XXX
+   / --disable-XXX flag on every invocation of configure.
+
+ - Add support to the command line arg parser to handle any new
+   --enable-XXX / --disable-XXX flags required by the feature XXX.
+
+ - Add information to the help output message to report on the new
+   feature flag.
+
+ - Add code to perform the actual feature check. As noted above, try to
+   be fully dynamic in checking enablement/disablement.
+
+ - Add code to print out the feature status in the configure summary
+   upon completion.
+
+ - Add any new makefile variables to $config_host_mak on completion.
+
+
+Taking (a simplified version of) the probe for gnutls from configure,
+we have the following pieces:
+
+  # Initial variable state
+  gnutls=""
+
+  ..snip..
+
+  # Configure flag processing
+  --disable-gnutls) gnutls="no"
+  ;;
+  --enable-gnutls) gnutls="yes"
+  ;;
+
+  ..snip..
+
+  # Help output feature message
+  gnutls          GNUTLS cryptography support
+
+  ..snip..
+
+  # Test for gnutls
+  if test "$gnutls" != "no"; then
+     if ! $pkg_config --exists "gnutls"; then
+        gnutls_cflags=`$pkg_config --cflags gnutls`
+        gnutls_libs=`$pkg_config --libs gnutls`
+        libs_softmmu="$gnutls_libs $libs_softmmu"
+        libs_tools="$gnutls_libs $libs_tools"
+        QEMU_CFLAGS="$QEMU_CFLAGS $gnutls_cflags"
+        gnutls="yes"
+     elif test "$gnutls" = "yes"; then
+        feature_not_found "gnutls" "Install gnutls devel"
+     else
+        gnutls="no"
+     fi
+  fi
+
+  ..snip..
+
+  # Completion feature summary
+  echo "GNUTLS support    $gnutls"
+
+  ..snip..
+
+  # Define make variables
+  if test "$gnutls" = "yes" ; then
+     echo "CONFIG_GNUTLS=y" >> $config_host_mak
+  fi
+
+
+Helper functions
+----------------
+
+The configure script provides a variety of helper functions to assist
+developers in checking for system features:
+
+ - do_cc $ARGS...
+
+   Attempt to run the system C compiler passing it $ARGS...
+
+ - do_cxx $ARGS...
+
+   Attempt to run the system C++ compiler passing it $ARGS...
+
+ - compile_object $CFLAGS
+
+   Attempt to compile a test program with the system C compiler using
+   $CFLAGS. The test program must have been previously written to a file
+   called $TMPC.
+
+ - compile_prog $CFLAGS $LDFLAGS
+
+   Attempt to compile a test program with the system C compiler using
+   $CFLAGS and link it with the system linker using $LDFLAGS. The test
+   program must have been previously written to a file called $TMPC.
+
+ - has $COMMAND
+
+   Determine if $COMMAND exists in the current environment, either as a
+   shell builtin, or executable binary, returning 0 on success.
+
+ - path_of $COMMAND
+
+   Return the fully qualified path of $COMMAND, printing it to stdout,
+   and returning 0 on success.
+
+ - check_define $NAME
+
+   Determine if the macro $NAME is defined by the system C compiler
+
+ - check_include $NAME
+
+   Determine if the include $NAME file is available to the system C
+   compiler
+
+ - write_c_skeleton
+
+   Write a minimal C program main() function to the temporary file
+   indicated by $TMPC
+
+ - feature_not_found $NAME $REMEDY
+
+   Print a message to stderr that the feature $NAME was not available
+   on the system, suggesting the user try $REMEDY to address the
+   problem.
+
+ - error_exit $MESSAGE $MORE...
+
+   Print $MESSAGE to stderr, followed by $MORE... and then exit from the
+   configure script with non-zero status
+
+ - query_pkg_config $ARGS...
+
+   Run pkg-config passing it $ARGS. If QEMU is doing a static build,
+   then --static will be automatically added to $ARGS
+
+
+Stage 2: makefiles
+==================
+
+The use of GNU make is required with the QEMU build system.
+
+Although the source code is spread across multiple subdirectories, the
+build system should be considered largely non-recursive in nature, in
+contrast to common practices seen with automake. There is some recursive
+invocation of make, but this is related to the things being built,
+rather than the source directory structure.
+
+QEMU currently supports both VPATH and non-VPATH builds, so there are
+three general ways to invoke configure & perform a build.
+
+ - VPATH, build artifacts outside of QEMU source tree entirely
+
+     cd ../
+     mkdir build
+     cd build
+     ../qemu/configure
+     make
+
+ - VPATH, build artifacts in a subdir of QEMU source tree
+
+     mkdir build
+     cd build
+     ../configure
+     make
+
+ - non-VPATH, build artifacts everywhere
+
+     ./configure
+     make
+
+The QEMU maintainers generally recommend that a VPATH build is used by
+developers. Patches to QEMU are expected to ensure VPATH build still
+works.
+
+
+Module structure
+----------------
+
+There are a number of key outputs of the QEMU build system:
+
+ - Tools - qemu-img, qemu-nbd, qga (guest agent), etc
+ - System emulators - qemu-system-$ARCH
+ - Userspace emulators - qemu-$ARCH
+ - Unit tests
+
+The source code is highly modularized, split across many files to
+facilitate building of all of these components with as little duplicated
+compilation as possible. There can be considered to be two distinct
+groups of files, those which are independent of the QEMU emulation
+target and those which are dependent on the QEMU emulation target.
+
+In the target-independent set lives various general purpose helper code,
+such as error handling infrastructure, standard data structures,
+platform portability wrapper functions, etc. This code can be compiled
+once only and the .o files linked into all output binaries.
+
+In the target-dependent set lives CPU emulation, device emulation and
+much glue code. This sometimes also has to be compiled multiple times,
+once for each target being built.
+
+The utility code that is used by all binaries is built into a
+static archive called libqemuutil.a, which is then linked to all the
+binaries. In order to provide hooks that are only needed by some of the
+binaries, code in libqemuutil.a may depend on other functions that are
+not fully implemented by all QEMU binaries. To deal with this there is a
+second library called libqemustub.a which provides dummy stubs for all
+these functions. These will get lazy linked into the binary if the real
+implementation is not present. In this way, the libqemustub.a static
+library can be thought of as a portable implementation of the weak
+symbols concept. All binaries should link to both libqemuutil.a and
+libqemustub.a. e.g.
+
+ qemu-img$(EXESUF): qemu-img.o ..snip.. libqemuutil.a libqemustub.a
+
+
+Windows platform portability
+----------------------------
+
+On Windows, all binaries have the suffix '.exe', so all Makefile rules
+which create binaries must include the $(EXESUF) variable on the binary
+name. e.g.
+
+ qemu-img$(EXESUF): qemu-img.o ..snip..
+
+This expands to '.exe' on Windows, or '' on other platforms.
+
+A further complication for the system emulator binaries is that
+two separate binaries need to be generated.
+
+The main binary (e.g. qemu-system-x86_64.exe) is linked against the
+Windows console runtime subsystem. These are expected to be run from a
+command prompt window, and so will print stderr to the console that
+launched them.
+
+The second binary generated has a 'w' on the end of its name (e.g.
+qemu-system-x86_64w.exe) and is linked against the Windows graphical
+runtime subsystem. These are expected to be run directly from the
+desktop and will open up a dedicated console window for stderr output.
+
+The Makefile.target will generate the binary for the graphical subsystem
+first, and then use objcopy to relink it against the console subsystem
+to generate the second binary.
+
+
+Object variable naming
+----------------------
+
+The QEMU convention is to define variables to list different groups of
+object files. These are named with the convention $PREFIX-obj-y. For
+example the libqemuutil.a file will be linked with all objects listed
+in a variable 'util-obj-y'. So, for example, util/Makefile.obj will
+contain a set of definitions looking like
+
+  util-obj-y += bitmap.o bitops.o hbitmap.o
+  util-obj-y += fifo8.o
+  util-obj-y += acl.o
+  util-obj-y += error.o qemu-error.o
+
+When there is an object file which needs to be conditionally built based
+on some characteristic of the host system, the configure script will
+define a variable for the conditional. For example, on Windows it will
+define $(CONFIG_POSIX) with a value of 'n' and $(CONFIG_WIN32) with a
+value of 'y'. It is now possible to use the config variables when
+listing object files. For example,
+
+  util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o
+  util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o
+
+On Windows this expands to
+
+  util-obj-y += oslib-win32.o qemu-thread-win32.o
+  util-obj-n += oslib-posix.o qemu-thread-posix.o
+
+Since libqemutil.a links in $(util-obj-y), the POSIX specific files
+listed against $(util-obj-n) are ignored on the Windows platform builds.
+
+
+CFLAGS / LDFLAGS / LIBS handling
+--------------------------------
+
+There are many different binaries being built with differing purposes,
+and some of them might even be 3rd party libraries pulled in via git
+submodules. As such the use of the global CFLAGS variable is generally
+avoided in QEMU, since it would apply to too many build targets.
+
+Flags that are needed by any QEMU code (i.e. everything *except* GIT
+submodule projects) are put in $(QEMU_CFLAGS) variable. For linker
+flags the $(LIBS) variable is sometimes used, but a couple of more
+targeted variables are preferred. $(libs_softmmu) is used for
+libraries that must be linked to system emulator targets, $(LIBS_TOOLS)
+is used for tools like qemu-img, qemu-nbd, etc and $(LIBS_QGA) is used
+for the QEMU guest agent. There is currently no specific variable for
+the userspace emulator targets as the global $(LIBS), or more targeted
+variables shown below, are sufficient.
+
+In addition to these variables, it is possible to provide cflags and
+libs against individual source code files, by defining variables of the
+form $FILENAME-cflags and $FILENAME-libs. For example, the curl block
+driver needs to link to the libcurl library, so block/Makefile defines
+some variables:
+
+  curl.o-cflags      := $(CURL_CFLAGS)
+  curl.o-libs        := $(CURL_LIBS)
+
+The scope is a little different between the two variables. The libs get
+used when linking any target binary that includes the curl.o object
+file, while the cflags get used when compiling the curl.c file only.
+
+
+Statically defined files
+------------------------
+
+The following key files are statically defined in the source tree, with
+the rules needed to build QEMU. Their behaviour is influenced by a
+number of dynamically created files listed later.
+
+- Makefile
+
+The main entry point used when invoking make to build all the components
+of QEMU. The default 'all' target will naturally result in the build of
+every component. The various tools and helper binaries are built
+directly via a non-recursive set of rules.
+
+Each system/userspace emulation target needs to have a slightly
+different set of make rules / variables. Thus, make will be recursively
+invoked for each of the emulation targets.
+
+The recursive invocation will end up processing the toplevel
+Makefile.target file (more on that later).
+
+
+- */Makefile.objs
+
+Since the source code is spread across multiple directories, the rules
+for each file are similarly modularized. Thus each subdirectory
+containing .c files will usually also contain a Makefile.objs file.
+These files are not directly invoked by a recursive make, but instead
+they are imported by the top level Makefile and/or Makefile.target
+
+Each Makefile.objs usually just declares a set of variables listing the
+.o files that need building from the source files in the directory. They
+will also define any custom linker or compiler flags. For example in
+block/Makefile.objs
+
+  block-obj-$(CONFIG_LIBISCSI) += iscsi.o
+  block-obj-$(CONFIG_CURL) += curl.o
+
+  ..snip...
+
+  iscsi.o-cflags     := $(LIBISCSI_CFLAGS)
+  iscsi.o-libs       := $(LIBISCSI_LIBS)
+  curl.o-cflags      := $(CURL_CFLAGS)
+  curl.o-libs        := $(CURL_LIBS)
+
+If there are any rules defined in the Makefile.objs file, they should
+all use $(obj) as a prefix to the target, e.g.
+
+  $(obj)/generated-tcg-tracers.h: $(obj)/generated-tcg-tracers.h-timestamp
+
+
+- Makefile.target
+
+This file provides the entry point used to build each individual system
+or userspace emulator target. Each enabled target has its own
+subdirectory. For example if configure is run with the argument
+'--target-list=x86_64-softmmu', then a sub-directory 'x86_64-softmu'
+will be created, containing a 'Makefile' which symlinks back to
+Makefile.target
+
+So when the recursive '$(MAKE) -C x86_64-softmmu' is invoked, it ends up
+using Makefile.target for the build rules.
+
+
+- rules.mak
+
+This file provides the generic helper rules for invoking build tools, in
+particular the compiler and linker. This also contains the magic (hairy)
+'unnest-vars' function which is used to merge the variable definitions
+from all Makefile.objs in the source tree down into the main Makefile
+context.
+
+
+- default-configs/*.mak
+
+The files under default-configs/ control what emulated hardware is built
+into each QEMU system and userspace emulator targets. They merely
+contain a long list of config variable definitions. For example,
+default-configs/x86_64-softmmu.mak has:
+
+  include pci.mak
+  include sound.mak
+  include usb.mak
+  CONFIG_QXL=$(CONFIG_SPICE)
+  CONFIG_VGA_ISA=y
+  CONFIG_VGA_CIRRUS=y
+  CONFIG_VMWARE_VGA=y
+  CONFIG_VIRTIO_VGA=y
+  ...snip...
+
+These files rarely need changing unless new devices / hardware need to
+be enabled for a particular system/userspace emulation target
+
+
+- tests/Makefile
+
+Rules for building the unit tests. This file is included directly by the
+top level Makefile, so anything defined in this file will influence the
+entire build system. Care needs to be taken when writing rules for tests
+to ensure they only apply to the unit test execution / build.
+
+- tests/docker/Makefile.include
+
+Rules for Docker tests. Like tests/Makefile, this file is included
+directly by the top level Makefile, anything defined in this file will
+influence the entire build system.
+
+- po/Makefile
+
+Rules for building and installing the binary message catalogs from the
+text .po file sources. This almost never needs changing for any reason.
+
+
+Dynamically created files
+-------------------------
+
+The following files are generated dynamically by configure in order to
+control the behaviour of the statically defined makefiles. This avoids
+the need for QEMU makefiles to go through any pre-processing as seen
+with autotools, where Makefile.am generates Makefile.in which generates
+Makefile.
+
+
+- config-host.mak
+
+When configure has determined the characteristics of the build host it
+will write a long list of variables to config-host.mak file. This
+provides the various install directories, compiler / linker flags and a
+variety of CONFIG_* variables related to optionally enabled features.
+This is imported by the top level Makefile in order to tailor the build
+output.
+
+The variables defined here are those which are applicable to all QEMU
+build outputs. Variables which are potentially different for each
+emulator target are defined by the next file...
+
+It is also used as a dependency checking mechanism. If make sees that
+the modification timestamp on configure is newer than that on
+config-host.mak, then configure will be re-run.
+
+
+- config-host.h
+
+The config-host.h file is used by source code to determine what features
+are enabled. It is generated from the contents of config-host.mak using
+the scripts/create_config program. This extracts all the CONFIG_* variables,
+most of the HOST_* variables and a few other misc variables from
+config-host.mak, formatting them as C preprocessor macros.
+
+
+- $TARGET-NAME/config-target.mak
+
+TARGET-NAME is the name of a system or userspace emulator, for example,
+x86_64-softmmu denotes the system emulator for the x86_64 architecture.
+This file contains the variables which need to vary on a per-target
+basis. For example, it will indicate whether KVM or Xen are enabled for
+the target and any other potential custom libraries needed for linking
+the target.
+
+
+- $TARGET-NAME/config-devices.mak
+
+TARGET-NAME is again the name of a system or userspace emulator. The
+config-devices.mak file is automatically generated by make using the
+scripts/make_device_config.sh program, feeding it the
+default-configs/$TARGET-NAME file as input.
+
+
+- $TARGET-NAME/Makefile
+
+This is the entrypoint used when make recurses to build a single system
+or userspace emulator target. It is merely a symlink back to the
+Makefile.target in the top level.
diff --git a/docs/igd-assign.txt b/docs/igd-assign.txt
new file mode 100644
index 0000000..e17bb50
--- /dev/null
+++ b/docs/igd-assign.txt
@@ -0,0 +1,133 @@
+Intel Graphics Device (IGD) assignment with vfio-pci
+====================================================
+
+IGD has two different modes for assignment using vfio-pci:
+
+1) Universal Pass-Through (UPT) mode:
+
+   In this mode the IGD device is added as a *secondary* (ie. non-primary)
+   graphics device in combination with an emulated primary graphics device.
+   This mode *requires* guest driver support to remove the external
+   dependencies generally associated with IGD (see below).  Those guest
+   drivers only support this mode for Broadwell and newer IGD, according to
+   Intel.  Additionally, this mode by default, and as officially supported
+   by Intel, does not support direct video output.  The intention is to use
+   this mode either to provide hardware acceleration to the emulated graphics
+   or to use this mode in combination with guest-based remote access software,
+   for example VNC (see below for optional output support).  This mode
+   theoretically has no device specific handling dependencies on vfio-pci or
+   the VM firmware.
+
+2) "Legacy" mode:
+
+   In this mode the IGD device is intended to be the primary and exclusive
+   graphics device in the VM[1], as such QEMU does not facilitate any sort
+   of remote graphics to the VM in this mode.  A connected physical monitor
+   is the intended output device for IGD.  This mode includes several
+   requirements and restrictions:
+
+    * IGD must be given address 02.0 on the PCI root bus in the VM
+    * The host kernel must support vfio extensions for IGD (v4.6)
+    * vfio VGA support very likely needs to be enabled in the host kernel
+    * The VM firmware must support specific fw_cfg enablers for IGD
+    * The VM machine type must support a PCI host bridge at 00.0 (standard)
+    * The VM machine type must provide or allow to be created a special
+      ISA/LPC bridge device (vfio-pci-igd-lpc-bridge) on the root bus at
+      PCI address 1f.0.
+    * The IGD device must have a VGA ROM, either provided via the romfile
+      option or loaded automatically through vfio (standard).  rombar=0
+      will disable legacy mode support.
+    * Hotplug of the IGD device is not supported.
+    * The IGD device must be a SandyBridge or newer model device.
+
+For either mode, depending on the host kernel, the i915 driver in the host
+may generate faults and errors upon re-binding to an IGD device after it
+has been assigned to a VM.  It's therefore generally recommended to prevent
+such driver binding unless the host driver is known to work well for this.
+There are numerous ways to do this, i915 can be blacklisted on the host,
+the driver_override option can be used to ensure that only vfio-pci can bind
+to the device on the host[2], virsh nodedev-detach can be used to bind the
+device to vfio drivers and then managed='no' set in the VM xml to prevent
+re-binding to i915, etc.  Also note that IGD is also typically the primary
+graphics in the host and special options may be required beyond simply
+blacklisting i915 or using pci-stub/vfio-pci to take ownership of IGD as a
+PCI class device.  Lower level drivers exist that may still claim the device.
+It may therefore be necessary to use kernel boot options video=vesafb:off or
+video=efifb:off (depending on host BIOS/UEFI) or these can be combined to
+a catch-all, video=vesafb:off,efifb:off.  Error messages such as:
+
+    Failed to mmap 0000:00:02.0 BAR <>. Performance may be slow
+
+are a good indicator that such a problem exists.  The host files /proc/iomem
+and /proc/ioports are often useful for identifying drivers consuming ranges
+of the device to cause such conflicts.
+
+Additionally, IGD device are known to generate small numbers of DMAR faults
+when initially assigned.  It is believed that this is simply the IGD attempting
+to access the reserved GTT space after reset, which it no longer has access to
+when accessed from userspace.  So long as the DMAR faults are small in number
+and most importantly, not ongoing, these are not an indication of an error.
+
+Additionally++, analog VGA output (as opposed to digital outputs like HDMI,
+DVI, or DisplayPort) may be unsupported in some use cases.  In the author's
+experience, even DP to VGA adapters can be troublesome while adapters between
+digital formats work well.
+
+Usage
+=====
+The intention is for IGD assignment to be transparent for users and thus for
+management tools like libvirt.  To make use of legacy mode, simply remove all
+other graphics options and use "-nographic" and either "-vga none" or
+"-nodefaults", along with adding the device using vfio-pci:
+
+    -device vfio-pci,host=00:02.0,id=hostdev0,bus=pci.0,addr=0x2
+
+For UPT mode, retain the default emulated graphics and simply add the vfio-pci
+device making use of any other bus address other than 02.0.  libvirt will
+default to assigning the device a UPT compatible address while legacy mode
+users will need to manually edit the XML if using a tool like virt-manager
+where the VM device address is not expressly specified.
+
+An experimental vfio-pci option also exists to enable OpRegion, and thus
+external monitor support, for UPT mode.  This can be enabled by adding
+"x-igd-opregion=on" to the vfio-pci device options for the IGD device.  As
+with legacy mode, this requires the host to support features introduced in
+the v4.6 kernel.  If Intel chooses to embrace this support, the option may
+be made non-experimental in the future, opening it to libvirt support.
+
+Developer ABI
+=============
+Legacy mode IGD support imposes two fw_cfg requirements on the VM firmware:
+
+1) "etc/igd-opregion"
+
+   This fw_cfg file exposes the OpRegion for the IGD device.  A reserved
+   region should be created below 4GB (recommended 4KB alignment), sized
+   sufficient for the fw_cfg file size, and the content of this file copied
+   to it.  The dword based address of this reserved memory region must also
+   be written to the ASLS register at offset 0xFC on the IGD device.  It is
+   recommended that firmware should make use of this fw_cfg entry for any
+   PCI class VGA device with Intel vendor ID.  Multiple of such devices
+   within a VM is undefined.
+
+2) "etc/igd-bdsm-size"
+
+   This fw_cfg file contains an 8-byte, little endian integer indicating
+   the size of the reserved memory region required for IGD stolen memory.
+   Firmware must allocate a reserved memory below 4GB with required 1MB
+   alignment equal to this size.  Additionally the base address of this
+   reserved region must be written to the dword BDSM register in PCI config
+   space of the IGD device at offset 0x5C.  As this support is related to
+   running the IGD ROM, which has other dependencies on the device appearing
+   at guest address 00:02.0, it's expected that this fw_cfg file is only
+   relevant to a single PCI class VGA device with Intel vendor ID, appearing
+   at PCI bus address 00:02.0.
+
+Footnotes
+=========
+[1] Nothing precludes adding additional emulated or assigned graphics devices
+    as non-primary, other than the combination typically not working.  I only
+    intend to set user expectations, others are welcome to find working
+    combinations or fix whatever issues prevent this from working in the common
+    case.
+[2] # echo "vfio-pci" > /sys/bus/pci/devices/0000:00:02.0/driver_override
diff --git a/docs/libcacard.txt b/docs/libcacard.txt
deleted file mode 100644
index 8db421d..0000000
--- a/docs/libcacard.txt
+++ /dev/null
@@ -1,483 +0,0 @@
-This file documents the CAC (Common Access Card) library in the libcacard
-subdirectory.
-
-Virtual Smart Card Emulator
-
-This emulator is designed to provide emulation of actual smart cards to a
-virtual card reader running in a guest virtual machine. The emulated smart
-cards can be representations of real smart cards, where the necessary functions
-such as signing, card removal/insertion, etc. are mapped to real, physical
-cards which are shared with the client machine the emulator is running on, or
-the cards could be pure software constructs.
-
-The emulator is structured to allow multiple replaceable or additional pieces,
-so it can be easily modified for future requirements. The primary envisioned
-modifications are:
-
-1) The socket connection to the virtual card reader (presumably a CCID reader,
-but other ISO-7816 compatible readers could be used). The code that handles
-this is in vscclient.c.
-
-2) The virtual card low level emulation. This is currently supplied by using
-NSS. This emulation could be replaced by implementations based on other
-security libraries, including but not limitted to openssl+pkcs#11 library,
-raw pkcs#11, Microsoft CAPI, direct opensc calls, etc. The code that handles
-this is in vcard_emul_nss.c.
-
-3) Emulation for new types of cards. The current implementation emulates the
-original DoD CAC standard with separate pki containers. This emulator lives in
-cac.c. More than one card type emulator could be included. Other cards could
-be emulated as well, including PIV, newer versions of CAC, PKCS #15, etc.
-
---------------------
-Replacing the Socket Based Virtual Reader Interface.
-
-The current implementation contains a replaceable module vscclient.c. The
-current vscclient.c implements a sockets interface to the virtual ccid reader
-on the guest. CCID commands that are pertinent to emulation are passed
-across the socket, and their responses are passed back along that same socket.
-The protocol that vscclient uses is defined in vscard_common.h and connects
-to a qemu ccid usb device. Since this socket runs as a client, vscclient.c
-implements a program with a main entry. It also handles argument parsing for
-the emulator.
-
-An application that wants to use the virtual reader can replace vscclient.c
-with its own implementation that connects to its own CCID reader.  The calls
-that the CCID reader can call are:
-
-      VReaderList * vreader_get_reader_list();
-
-  This function returns a list of virtual readers.  These readers may map to
-  physical devices, or simulated devices depending on vcard the back end. Each
-  reader in the list should represent a reader to the virtual machine. Virtual
-  USB address mapping is left to the CCID reader front end. This call can be
-  made any time to get an updated list. The returned list is a copy of the
-  internal list that can be referenced by the caller without locking. This copy
-  must be freed by the caller with vreader_list_delete when it is no longer
-  needed.
-
-      VReaderListEntry *vreader_list_get_first(VReaderList *);
-
-  This function gets the first entry on the reader list. Along with
-  vreader_list_get_next(), vreader_list_get_first() can be used to walk the
-  reader list returned from vreader_get_reader_list(). VReaderListEntries are
-  part of the list themselves and do not need to be freed separately from the
-  list. If there are no entries on the list, it will return NULL.
-
-      VReaderListEntry *vreader_list_get_next(VReaderListEntry *);
-
-  This function gets the next entry in the list. If there are no more entries
-  it will return NULL.
-
-      VReader * vreader_list_get_reader(VReaderListEntry *)
-
-  This function returns the reader stored in the reader List entry. Caller gets
-  a new reference to a reader. The caller must free its reference when it is
-  finished with vreader_free().
-
-      void vreader_free(VReader *reader);
-
-   This function frees a reference to a reader. Readers are reference counted
-   and are automatically deleted when the last reference is freed.
-
-      void vreader_list_delete(VReaderList *list);
-
-   This function frees the list, all the elements on the list, and all the
-   reader references held by the list.
-
-      VReaderStatus vreader_power_on(VReader *reader, char *atr, int *len);
-
-  This function simulates a card power on. A virtual card does not care about
-  the actual voltage and other physical parameters, but it does care that the
-  card is actually on or off. Cycling the card causes the card to reset. If
-  the caller provides enough space, vreader_power_on will return the ATR of
-  the virtual card. The amount of space provided in atr should be indicated
-  in *len. The function modifies *len to be the actual length of of the
-  returned ATR.
-
-      VReaderStatus vreader_power_off(VReader *reader);
-
-  This function simulates a power off of a virtual card.
-
-      VReaderStatus vreader_xfer_bytes(VReader *reader, unsigne char *send_buf,
-                                       int send_buf_len,
-                                       unsigned char *receive_buf,
-                                       int receive_buf_len);
-
-  This function sends a raw apdu to a card and returns the card's response.
-  The CCID front end should return the response back. Most of the emulation
-  is driven from these APDUs.
-
-      VReaderStatus vreader_card_is_present(VReader *reader);
-
-  This function returns whether or not the reader has a card inserted. The
-  vreader_power_on, vreader_power_off, and vreader_xfer_bytes will return
-  VREADER_NO_CARD.
-
-       const char *vreader_get_name(VReader *reader);
-
-  This function returns the name of the reader. The name comes from the card
-  emulator level and is usually related to the name of the physical reader.
-
-       VReaderID vreader_get_id(VReader *reader);
-
-  This function returns the id of a reader. All readers start out with an id
-  of -1. The application can set the id with vreader_set_id.
-
-       VReaderStatus vreader_get_id(VReader *reader, VReaderID id);
-
-  This function sets the reader id. The application is responsible for making
-  sure that the id is unique for all readers it is actively using.
-
-       VReader *vreader_find_reader_by_id(VReaderID id);
-
-  This function returns the reader which matches the id. If two readers match,
-  only one is returned. The function returns NULL if the id is -1.
-
-       Event *vevent_wait_next_vevent();
-
-  This function blocks waiting for reader and card insertion events. There
-  will be one event for each card insertion, each card removal, each reader
-  insertion and each reader removal. At start up, events are created for all
-  the initial readers found, as well as all the cards that are inserted.
-
-       Event *vevent_get_next_vevent();
-
-  This function returns a pending event if it exists, otherwise it returns
-  NULL. It does not block.
-
-----------------
-Card Type Emulator: Adding a New Virtual Card Type
-
-The ISO 7816 card spec describes 2 types of cards:
- 1) File system cards, where the smartcard is managed by reading and writing
-data to files in a file system. There is currently only boiler plate
-implemented for file system cards.
- 2) VM cards, where the card has loadable applets which perform the card
-functions. The current implementation supports VM cards.
-
-In the case of VM cards, the difference between various types of cards is
-really what applets have been installed in that card. This structure is
-mirrored in card type emulators. The 7816 emulator already handles the basic
-ISO 7186 commands. Card type emulators simply need to add the virtual applets
-which emulate the real card applets. Card type emulators have exactly one
-public entry point:
-
-       VCARDStatus xxx_card_init(VCard *card, const char *flags,
-                               const unsigned char *cert[],
-                               int cert_len[],
-                               VCardKey *key[],
-                               int cert_count);
-
-  The parameters for this are:
-  card       - the virtual card structure which will represent this card.
-  flags      - option flags that may be specific to this card type.
-  cert       - array of binary certificates.
-  cert_len   - array of lengths of each of the certificates specified in cert.
-  key        - array of opaque key structures representing the private keys on
-               the card.
-  cert_count - number of entries in cert, cert_len, and key arrays.
-
-  Any cert, cert_len, or key with the same index are matching sets. That is
-  cert[0] is cert_len[0] long and has the corresponding private key of key[0].
-
-The card type emulator is expected to own the VCardKeys, but it should copy
-any raw cert data it wants to save. It can create new applets and add them to
-the card using the following functions:
-
-       VCardApplet *vcard_new_applet(VCardProcessAPDU apdu_func,
-                                     VCardResetApplet reset_func,
-                                     const unsigned char *aid,
-                                     int aid_len);
-
-  This function creates a new applet. Applet structures store the following
-  information:
-     1) the AID of the applet (set by aid and aid_len).
-     2) a function to handle APDUs for this applet. (set by apdu_func, more on
-        this below).
-     3) a function to reset the applet state when the applet is selected.
-        (set by reset_func, more on this below).
-     3) applet private data, a data pointer used by the card type emulator to
-        store any data or state it needs to complete requests. (set by a
-        separate call).
-     4) applet private data free, a function used to free the applet private
-        data when the applet itself is destroyed.
-  The created applet can be added to the card with vcard_add_applet below.
-
-        void vcard_set_applet_private(VCardApplet *applet,
-                                      VCardAppletPrivate *private,
-                                      VCardAppletPrivateFree private_free);
-  This function sets the private data and the corresponding free function.
-  VCardAppletPrivate is an opaque data structure to the rest of the emulator.
-  The card type emulator can define it any way it wants by defining
-  struct VCardAppletPrivateStruct {};. If there is already a private data
-  structure on the applet, the old one is freed before the new one is set up.
-  passing two NULL clear any existing private data.
-
-         VCardStatus vcard_add_applet(VCard *card, VCardApplet *applet);
-
-  Add an applet onto the list of applets attached to the card. Once an applet
-  has been added, it can be selected by its AID, and then commands will be
-  routed to it VCardProcessAPDU function. This function adopts the applet that
-  is passed into it. Note: 2 applets with the same AID should not be added to
-  the same card. It is permissible to add more than one applet. Multiple applets
-  may have the same VCardPRocessAPDU entry point.
-
-The certs and keys should be attached to private data associated with one or
-more appropriate applets for that card. Control will come to the card type
-emulators once one of its applets are selected through the VCardProcessAPDU
-function it specified when it created the applet.
-
-The signature of VCardResetApplet is:
-        VCardStatus (*VCardResetApplet) (VCard *card, int channel);
-  This function will reset the any internal applet state that needs to be
-  cleared after a select applet call. It should return VCARD_DONE;
-
-The signature of VCardProcessAPDU is:
-        VCardStatus (*VCardProcessAPDU)(VCard *card, VCardAPDU *apdu,
-                                         VCardResponse **response);
-  This function examines the APDU and determines whether it should process
-  the apdu directly, reject the apdu as invalid, or pass the apdu on to
-  the basic 7816 emulator for processing.
-      If the 7816 emulator should process the apdu, then the VCardProcessAPDU
-  should return VCARD_NEXT.
-      If there is an error, then VCardProcessAPDU should return an error
-  response using vcard_make_response and the appropriate 7816 error code
-  (see card_7816t.h) or vcard_make_response with a card type specific error
-  code. It should then return VCARD_DONE.
-      If the apdu can be processed correctly, VCardProcessAPDU should do so,
-  set the response value appropriately for that APDU, and return VCARD_DONE.
-  VCardProcessAPDU should always set the response if it returns VCARD_DONE.
-  It should always either return VCARD_DONE or VCARD_NEXT.
-
-Parsing the APDU --
-
-Prior to processing calling the card type emulator's VCardProcessAPDU function, the emulator has already decoded the APDU header and set several fields:
-
-   apdu->a_data - The raw apdu data bytes.
-   apdu->a_len  - The len of the raw apdu data.
-   apdu->a_body - The start of any post header parameter data.
-   apdu->a_Lc   - The parameter length value.
-   apdu->a_Le   - The expected length of any returned data.
-   apdu->a_cla  - The raw apdu class.
-   apdu->a_channel - The channel (decoded from the class).
-   apdu->a_secure_messaging_type - The decoded secure messaging type
-                                   (from class).
-   apdu->a_type - The decode class type.
-   apdu->a_gen_type - the generic class type (7816, PROPRIETARY, RFU, PTS).
-   apdu->a_ins  - The instruction byte.
-   apdu->a_p1   - Parameter 1.
-   apdu->a_p2   - Parameter 2.
-
-Creating a Response --
-
-The expected result of any APDU call is a response. The card type emulator must
-set *response with an appropriate VCardResponse value if it returns VCARD_DONE.
-Responses could be as simple as returning a 2 byte status word response, to as
-complex as returning a block of data along with a 2 byte response. Which is
-returned will depend on the semantics of the APDU. The following functions will
-create card responses.
-
-        VCardResponse *vcard_make_response(VCard7816Status status);
-
-    This is the most basic function to get a response. This function will
-    return a response the consists solely one 2 byte status code. If that status
-    code is defined in card_7816t.h, then this function is guaranteed to
-    return a response with that status. If a cart type specific status code
-    is passed and vcard_make_response fails to allocate the appropriate memory
-    for that response, then vcard_make_response will return a VCardResponse
-    of VCARD7816_STATUS_EXC_ERROR_MEMORY. In any case, this function is
-    guaranteed to return a valid VCardResponse.
-
-        VCardResponse *vcard_response_new(unsigned char *buf, int len,
-                                          VCard7816Status status);
-
-    This function is similar to vcard_make_response except it includes some
-    returned data with the response. It could also fail to allocate enough
-    memory, in which case it will return NULL.
-
-        VCardResponse *vcard_response_new_status_bytes(unsigned char sw1,
-                                                       unsigned char sw2);
-
-    Sometimes in 7816 the response bytes are treated as two separate bytes with
-    split meanings. This function allows you to create a response based on
-    two separate bytes. This function could fail, in which case it will return
-    NULL.
-
-       VCardResponse *vcard_response_new_bytes(unsigned char *buf, int len,
-                                               unsigned char sw1,
-                                               unsigned char sw2);
-
-    This function is the same as vcard_response_new except you may specify
-    the status as two separate bytes like vcard_response_new_status_bytes.
-
-
-Implementing functionality ---
-
-The following helper functions access information about the current card
-and applet.
-
-        VCARDAppletPrivate *vcard_get_current_applet_private(VCard *card,
-                                                             int channel);
-
-    This function returns any private data set by the card type emulator on
-    the currently selected applet. The card type emulator keeps track of the
-    current applet state in this data structure. Any certs and keys associated
-    with a particular applet is also stored here.
-
-        int vcard_emul_get_login_count(VCard *card);
-
-    This function returns the the number of remaining login attempts for this
-    card. If the card emulator does not know, or the card does not have a
-    way of giving this information, this function returns -1.
-
-
-         VCard7816Status vcard_emul_login(VCard *card, unsigned char *pin,
-                                          int pin_len);
-
-    This function logs into the card and returns the standard 7816 status
-    word depending on the success or failure of the call.
-
-         void vcard_emul_delete_key(VCardKey *key);
-
-     This function frees the VCardKey passed in to xxxx_card_init. The card
-     type emulator is responsible for freeing this key when it no longer needs
-     it.
-
-         VCard7816Status vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                                           unsigned char *buffer,
-                                           int buffer_size);
-
-     This function does a raw rsa op on the buffer with the given key.
-
-The sample card type emulator is found in cac.c. It implements the cac specific
-applets.  Only those applets needed by the coolkey pkcs#11 driver on the guest
-have been implemented. To support the full range CAC middleware, a complete CAC
-card according to the CAC specs should be implemented here.
-
-------------------------------
-Virtual Card Emulator
-
-This code accesses both real smart cards and simulated smart cards through
-services provided on the client. The current implementation uses NSS, which
-already knows how to talk to various PKCS #11 modules on the client, and is
-portable to most operating systems. A particular emulator can have only one
-virtual card implementation at a time.
-
-The virtual card emulator consists of a series of virtual card services. In
-addition to the services describe above (services starting with
-vcard_emul_xxxx), the virtual card emulator also provides the following
-functions:
-
-    VCardEmulError vcard_emul_init(cont VCardEmulOptions *options);
-
-  The options structure is built by another function in the virtual card
-  interface where a string of virtual card emulator specific strings are
-  mapped to the options. The actual structure is defined by the virtual card
-  emulator and is used to determine the configuration of soft cards, or to
-  determine which physical cards to present to the guest.
-
-  The vcard_emul_init function will build up sets of readers, create any
-  threads that are needed to watch for changes in the reader state. If readers
-  have cards present in them, they are also initialized.
-
-  Readers are created with the function.
-
-          VReader *vreader_new(VReaderEmul *reader_emul,
-                               VReaderEmulFree reader_emul_free);
-
-      The freeFunc is used to free the VReaderEmul * when the reader is
-      destroyed.  The VReaderEmul structure is an opaque structure to the
-      rest of the code, but defined by the virtual card emulator, which can
-      use it to store any reader specific state.
-
-  Once the reader has been created, it can be added to the front end with the
-  call:
-
-           VReaderStatus vreader_add_reader(VReader *reader);
-
-      This function will automatically generate the appropriate new reader
-      events and add the reader to the list.
-
-  To create a new card, the virtual card emulator will call a similar
-  function.
-
-           VCard *vcard_new(VCardEmul *card_emul,
-                            VCardEmulFree card_emul_free);
-
-      Like vreader_new, this function takes a virtual card emulator specific
-      structure which it uses to keep track of the card state.
-
-  Once the card is created, it is attached to a card type emulator with the
-  following function:
-
-            VCardStatus vcard_init(VCard *vcard, VCardEmulType type,
-                                   const char *flags,
-                                   unsigned char *const *certs,
-                                   int *cert_len,
-                                   VCardKey *key[],
-                                   int cert_count);
-
-      The vcard is the value returned from vcard_new. The type is the
-      card type emulator that this card should presented to the guest as.
-      The flags are card type emulator specific options. The certs,
-      cert_len, and keys are all arrays of length cert_count. These are the
-      the same of the parameters xxxx_card_init() accepts.
-
-   Finally the card is associated with its reader by the call:
-
-            VReaderStatus vreader_insert_card(VReader *vreader, VCard *vcard);
-
-      This function, like vreader_add_reader, will take care of any event
-      notification for the card insert.
-
-
-    VCardEmulError vcard_emul_force_card_remove(VReader *vreader);
-
-  Force a card that is present to appear to be removed to the guest, even if
-  that card is a physical card and is present.
-
-
-    VCardEmulError vcard_emul_force_card_insert(VReader *reader);
-
-  Force a card that has been removed by vcard_emul_force_card_remove to be
-  reinserted from the point of view of the guest. This will only work if the
-  card is physically present (which is always true fro a soft card).
-
-     void vcard_emul_get_atr(Vcard *card, unsigned char *atr, int *atr_len);
-
-  Return the virtual ATR for the card. By convention this should be the value
-  VCARD_ATR_PREFIX(size) followed by several ascii bytes related to this
-  particular emulator. For instance the NSS emulator returns
-  {VCARD_ATR_PREFIX(3), 'N', 'S', 'S' }. Do ot return more data then *atr_len;
-
-     void vcard_emul_reset(VCard *card, VCardPower power)
-
-   Set the state of 'card' to the current power level and reset its internal
-   state (logout, etc).
-
--------------------------------------------------------
-List of files and their function:
-README - This file
-card_7816.c - emulate basic 7816 functionality. Parse APDUs.
-card_7816.h - apdu and response services definitions.
-card_7816t.h - 7816 specific structures, types and definitions.
-event.c - event handling code.
-event.h - event handling services definitions.
-eventt.h - event handling structures and types
-vcard.c - handle common virtual card services like creation, destruction, and
-          applet management.
-vcard.h - common virtual card services function definitions.
-vcardt.h - comon virtual card types
-vreader.c - common virtual reader services.
-vreader.h - common virtual reader services definitions.
-vreadert.h - comon virtual reader types.
-vcard_emul_type.c - manage the card type emulators.
-vcard_emul_type.h - definitions for card type emulators.
-cac.c - card type emulator for CAC cards
-vcard_emul.h - virtual card emulator service definitions.
-vcard_emul_nss.c - virtual card emulator implementation for nss.
-vscclient.c - socket connection to guest qemu usb driver.
-vscard_common.h - common header with the guest qemu usb driver.
-mutex.h - header file for machine independent mutexes.
-link_test.c - static test to make sure all the symbols are properly defined.
diff --git a/docs/memory-hotplug.txt b/docs/memory-hotplug.txt
new file mode 100644
index 0000000..56bdd0a
--- /dev/null
+++ b/docs/memory-hotplug.txt
@@ -0,0 +1,93 @@
+QEMU memory hotplug
+===================
+
+This document explains how to use the memory hotplug feature in QEMU,
+which is present since v2.1.0.
+
+Guest support is required for memory hotplug to work.
+
+Basic RAM hotplug
+-----------------
+
+In order to be able to hotplug memory, QEMU has to be told how many
+hotpluggable memory slots to create and what is the maximum amount of
+memory the guest can grow. This is done at startup time by means of
+the -m command-line option, which has the following format:
+
+ -m [size=]megs[,slots=n,maxmem=size]
+
+Where,
+
+ - "megs" is the startup RAM. It is the RAM the guest will boot with
+ - "slots" is the number of hotpluggable memory slots
+ - "maxmem" is the maximum RAM size the guest can have
+
+For example, the following command-line:
+
+ qemu [...] 1G,slots=3,maxmem=4G
+
+Creates a guest with 1GB of memory and three hotpluggable memory slots.
+The hotpluggable memory slots are empty when the guest is booted, so all
+memory the guest will see after boot is 1GB. The maximum memory the
+guest can reach is 4GB. This means that three additional gigabytes can be
+hotplugged by using any combination of the available memory slots.
+
+Two monitor commands are used to hotplug memory:
+
+ - "object_add": creates a memory backend object
+ - "device_add": creates a front-end pc-dimm device and inserts it
+                 into the first empty slot
+
+For example, the following commands add another 1GB to the guest
+discussed earlier:
+
+  (qemu) object_add memory-backend-ram,id=mem1,size=1G
+  (qemu) device_add pc-dimm,id=dimm1,memdev=mem1
+
+Using the file backend
+----------------------
+
+Besides basic RAM hotplug, QEMU also supports using files as a memory
+backend. This is useful for using hugetlbfs in Linux, which provides
+access to bigger page sizes.
+
+For example, assuming that the host has 1GB hugepages available in
+the /mnt/hugepages-1GB directory, a 1GB hugepage could be hotplugged
+into the guest from the previous section with the following commands:
+
+  (qemu) object_add memory-backend-file,id=mem1,size=1G,mem-path=/mnt/hugepages-1GB
+  (qemu) device_add pc-dimm,id=dimm1,memdev=mem1
+
+It's also possible to start a guest with memory cold-plugged into the
+hotpluggable memory slots. This might seem counterintuitive at first,
+but this allows for a lot of flexibility when using the file backend.
+
+In the following command-line example, a 8GB guest is created where 6GB
+comes from regular RAM, 1GB is a 1GB hugepage page and 256MB is from
+2MB pages. Also, the guest has additional memory slots to hotplug more
+2GB if needed:
+
+ qemu [...] -m 6GB,slots=4,maxmem=10G \
+   -object memory-backend-file,id=mem1,size=1G,mem-path=/mnt/hugepages-1G \
+   -device pc-dimm,id=dimm1,memdev=mem1 \
+   -object memory-backend-file,id=mem2,size=256M,mem-path=/mnt/hugepages-2MB \
+   -device pc-dimm,id=dimm2,memdev=mem2
+
+
+RAM hot-unplug
+---------------
+
+In order to be able to hot unplug pc-dimm device, QEMU has to be told the ids
+of pc-dimm device and memory backend object. The ids were assigned when you hot
+plugged memory.
+
+Two monitor commands are used to hot unplug memory:
+
+ - "device_del": deletes a front-end pc-dimm device
+ - "object_del": deletes a memory backend object
+
+For example, assuming that the pc-dimm device with id "dimm1" exists, and its memory
+backend is "mem1", the following commands tries to remove it.
+
+  (qemu) device_del dimm1
+  (qemu) object_del mem1
diff --git a/docs/memory.txt b/docs/memory.txt
index b12f1f0..811b1bd 100644
--- a/docs/memory.txt
+++ b/docs/memory.txt
@@ -26,14 +26,33 @@
 Types of regions
 ----------------
 
-There are four types of memory regions (all represented by a single C type
+There are multiple types of memory regions (all represented by a single C type
 MemoryRegion):
 
 - RAM: a RAM region is simply a range of host memory that can be made available
   to the guest.
+  You typically initialize these with memory_region_init_ram().  Some special
+  purposes require the variants memory_region_init_resizeable_ram(),
+  memory_region_init_ram_from_file(), or memory_region_init_ram_ptr().
 
 - MMIO: a range of guest memory that is implemented by host callbacks;
   each read or write causes a callback to be called on the host.
+  You initialize these with memory_region_init_io(), passing it a
+  MemoryRegionOps structure describing the callbacks.
+
+- ROM: a ROM memory region works like RAM for reads (directly accessing
+  a region of host memory), and forbids writes. You initialize these with
+  memory_region_init_rom().
+
+- ROM device: a ROM device memory region works like RAM for reads
+  (directly accessing a region of host memory), but like MMIO for
+  writes (invoking a callback).  You initialize these with
+  memory_region_init_rom_device().
+
+- IOMMU region: an IOMMU region translates addresses of accesses made to it
+  and forwards them to some other target memory region.  As the name suggests,
+  these are only needed for modelling an IOMMU, not for simple devices.
+  You initialize these with memory_region_init_iommu().
 
 - container: a container simply includes other memory regions, each at
   a different offset.  Containers are useful for grouping several regions
@@ -45,12 +64,22 @@
   can overlay a subregion of RAM with MMIO or ROM, or a PCI controller
   that does not prevent card from claiming overlapping BARs.
 
+  You initialize a pure container with memory_region_init().
+
 - alias: a subsection of another region.  Aliases allow a region to be
   split apart into discontiguous regions.  Examples of uses are memory banks
   used when the guest address space is smaller than the amount of RAM
   addressed, or a memory controller that splits main memory to expose a "PCI
   hole".  Aliases may point to any type of region, including other aliases,
   but an alias may not point back to itself, directly or indirectly.
+  You initialize these with memory_region_init_alias().
+
+- reservation region: a reservation region is primarily for debugging.
+  It claims I/O space that is not supposed to be handled by QEMU itself.
+  The typical use is to track parts of the address space which will be
+  handled by the host kernel when KVM is enabled.
+  You initialize these with memory_region_init_reservation(), or by
+  passing a NULL callback parameter to memory_region_init_io().
 
 It is valid to add subregions to a region which is not a pure container
 (that is, to an MMIO, RAM or ROM region). This means that the region
@@ -73,17 +102,66 @@
 Region lifecycle
 ----------------
 
-A region is created by one of the constructor functions (memory_region_init*())
-and attached to an object.  It is then destroyed by object_unparent() or simply
-when the parent object dies.
+A region is created by one of the memory_region_init*() functions and
+attached to an object, which acts as its owner or parent.  QEMU ensures
+that the owner object remains alive as long as the region is visible to
+the guest, or as long as the region is in use by a virtual CPU or another
+device.  For example, the owner object will not die between an
+address_space_map operation and the corresponding address_space_unmap.
 
-In between, a region can be added to an address space
-by using memory_region_add_subregion() and removed using
-memory_region_del_subregion().  Destroying the region implicitly
-removes the region from the address space.
+After creation, a region can be added to an address space or a
+container with memory_region_add_subregion(), and removed using
+memory_region_del_subregion().
 
-Region attributes may be changed at any point; they take effect once
-the region becomes exposed to the guest.
+Various region attributes (read-only, dirty logging, coalesced mmio,
+ioeventfd) can be changed during the region lifecycle.  They take effect
+as soon as the region is made visible.  This can be immediately, later,
+or never.
+
+Destruction of a memory region happens automatically when the owner
+object dies.
+
+If however the memory region is part of a dynamically allocated data
+structure, you should call object_unparent() to destroy the memory region
+before the data structure is freed.  For an example see VFIOMSIXInfo
+and VFIOQuirk in hw/vfio/pci.c.
+
+You must not destroy a memory region as long as it may be in use by a
+device or CPU.  In order to do this, as a general rule do not create or
+destroy memory regions dynamically during a device's lifetime, and only
+call object_unparent() in the memory region owner's instance_finalize
+callback.  The dynamically allocated data structure that contains the
+memory region then should obviously be freed in the instance_finalize
+callback as well.
+
+If you break this rule, the following situation can happen:
+
+- the memory region's owner had a reference taken via memory_region_ref
+  (for example by address_space_map)
+
+- the region is unparented, and has no owner anymore
+
+- when address_space_unmap is called, the reference to the memory region's
+  owner is leaked.
+
+
+There is an exception to the above rule: it is okay to call
+object_unparent at any time for an alias or a container region.  It is
+therefore also okay to create or destroy alias and container regions
+dynamically during a device's lifetime.
+
+This exceptional usage is valid because aliases and containers only help
+QEMU building the guest's memory map; they are never accessed directly.
+memory_region_ref and memory_region_unref are never called on aliases
+or containers, and the above situation then cannot happen.  Exploiting
+this exception is rarely necessary, and therefore it is discouraged,
+but nevertheless it is used in a few places.
+
+For regions that "have no owner" (NULL is passed at creation time), the
+machine object is actually used as the owner.  Since instance_finalize is
+never called for the machine object, you must never call object_unparent
+on regions that have no owner, unless they are aliases or containers.
+
 
 Overlapping regions and priority
 --------------------------------
@@ -107,14 +185,14 @@
 holes too.)
 
 For example, suppose we have a container A of size 0x8000 with two subregions
-B and C. B is a container mapped at 0x2000, size 0x4000, priority 1; C is
-an MMIO region mapped at 0x0, size 0x6000, priority 2. B currently has two
+B and C. B is a container mapped at 0x2000, size 0x4000, priority 2; C is
+an MMIO region mapped at 0x0, size 0x6000, priority 1. B currently has two
 of its own subregions: D of size 0x1000 at offset 0 and E of size 0x1000 at
 offset 0x2000. As a diagram:
 
-        0      1000   2000   3000   4000   5000   6000   7000    8000
-        |------|------|------|------|------|------|------|-------|
-  A:    [                                                       ]
+        0      1000   2000   3000   4000   5000   6000   7000   8000
+        |------|------|------|------|------|------|------|------|
+  A:    [                                                      ]
   C:    [CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC]
   B:                  [                          ]
   D:                  [DDDDD]
@@ -174,7 +252,7 @@
  |
  +---- himem: alias@0x100000000-0x11fffffff ---> #ram (0xe0000000-0xffffffff)
  |
- +---- vga-window: alias@0xa0000-0xbfffff ---> #pci (0xa0000-0xbffff)
+ +---- vga-window: alias@0xa0000-0xbffff ---> #pci (0xa0000-0xbffff)
  |      (prio 1)
  |
  +---- pci-hole: alias@0xe0000000-0xffffffff ---> #pci (0xe0000000-0xffffffff)
@@ -215,13 +293,6 @@
 Note that if the guest maps a BAR outside the PCI hole, it would not be
 visible as the pci-hole alias clips it to a 0.5GB range.
 
-Attributes
-----------
-
-Various region attributes (read-only, dirty logging, coalesced mmio, ioeventfd)
-can be changed during the region lifecycle.  They take effect once the region
-is made visible (which can be immediately, later, or never).
-
 MMIO Operations
 ---------------
 
@@ -231,8 +302,9 @@
  - .valid.min_access_size, .valid.max_access_size define the access sizes
    (in bytes) which the device accepts; accesses outside this range will
    have device and bus specific behaviour (ignored, or machine check)
- - .valid.aligned specifies that the device only accepts naturally aligned
-   accesses.  Unaligned accesses invoke device and bus specific behaviour.
+ - .valid.unaligned specifies that the *device being modelled* supports
+    unaligned accesses; if false, unaligned accesses will invoke the
+    appropriate bus or CPU specific behaviour.
  - .impl.min_access_size, .impl.max_access_size define the access sizes
    (in bytes) supported by the *implementation*; other access sizes will be
    emulated using the ones available.  For example a 4-byte write will be
@@ -240,5 +312,5 @@
  - .impl.unaligned specifies that the *implementation* supports unaligned
    accesses; if false, unaligned accesses will be emulated by two aligned
    accesses.
- - .old_mmio can be used to ease porting from code using
+ - .old_mmio eases the porting of code that was formerly using
    cpu_register_io_memory(). It should not be used in new code.
diff --git a/docs/migration.txt b/docs/migration.txt
index 0492a45..6503c17 100644
--- a/docs/migration.txt
+++ b/docs/migration.txt
@@ -257,6 +257,7 @@
     .minimum_version_id = 1,
     .pre_save = ide_drive_pio_pre_save,
     .post_load = ide_drive_pio_post_load,
+    .needed = ide_drive_pio_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(req_nb_sectors, IDEState),
         VMSTATE_VARRAY_INT32(io_buffer, IDEState, io_buffer_total_len, 1,
@@ -279,13 +280,9 @@
         .... several fields ....
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_ide_drive_pio_state,
-            .needed = ide_drive_pio_state_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_ide_drive_pio_state,
+        NULL
     }
 };
 
@@ -294,3 +291,194 @@
 (that is what ide_drive_pio_state_needed() checks).  If DRQ_STAT is
 not enabled, the values on that fields are garbage and don't need to
 be sent.
+
+= Return path =
+
+In most migration scenarios there is only a single data path that runs
+from the source VM to the destination, typically along a single fd (although
+possibly with another fd or similar for some fast way of throwing pages across).
+
+However, some uses need two way communication; in particular the Postcopy
+destination needs to be able to request pages on demand from the source.
+
+For these scenarios there is a 'return path' from the destination to the source;
+qemu_file_get_return_path(QEMUFile* fwdpath) gives the QEMUFile* for the return
+path.
+
+  Source side
+     Forward path - written by migration thread
+     Return path  - opened by main thread, read by return-path thread
+
+  Destination side
+     Forward path - read by main thread
+     Return path  - opened by main thread, written by main thread AND postcopy
+                    thread (protected by rp_mutex)
+
+= Postcopy =
+'Postcopy' migration is a way to deal with migrations that refuse to converge
+(or take too long to converge) its plus side is that there is an upper bound on
+the amount of migration traffic and time it takes, the down side is that during
+the postcopy phase, a failure of *either* side or the network connection causes
+the guest to be lost.
+
+In postcopy the destination CPUs are started before all the memory has been
+transferred, and accesses to pages that are yet to be transferred cause
+a fault that's translated by QEMU into a request to the source QEMU.
+
+Postcopy can be combined with precopy (i.e. normal migration) so that if precopy
+doesn't finish in a given time the switch is made to postcopy.
+
+=== Enabling postcopy ===
+
+To enable postcopy, issue this command on the monitor prior to the
+start of migration:
+
+migrate_set_capability postcopy-ram on
+
+The normal commands are then used to start a migration, which is still
+started in precopy mode.  Issuing:
+
+migrate_start_postcopy
+
+will now cause the transition from precopy to postcopy.
+It can be issued immediately after migration is started or any
+time later on.  Issuing it after the end of a migration is harmless.
+
+Note: During the postcopy phase, the bandwidth limits set using
+migrate_set_speed is ignored (to avoid delaying requested pages that
+the destination is waiting for).
+
+=== Postcopy device transfer ===
+
+Loading of device data may cause the device emulation to access guest RAM
+that may trigger faults that have to be resolved by the source, as such
+the migration stream has to be able to respond with page data *during* the
+device load, and hence the device data has to be read from the stream completely
+before the device load begins to free the stream up.  This is achieved by
+'packaging' the device data into a blob that's read in one go.
+
+Source behaviour
+
+Until postcopy is entered the migration stream is identical to normal
+precopy, except for the addition of a 'postcopy advise' command at
+the beginning, to tell the destination that postcopy might happen.
+When postcopy starts the source sends the page discard data and then
+forms the 'package' containing:
+
+   Command: 'postcopy listen'
+   The device state
+      A series of sections, identical to the precopy streams device state stream
+      containing everything except postcopiable devices (i.e. RAM)
+   Command: 'postcopy run'
+
+The 'package' is sent as the data part of a Command: 'CMD_PACKAGED', and the
+contents are formatted in the same way as the main migration stream.
+
+During postcopy the source scans the list of dirty pages and sends them
+to the destination without being requested (in much the same way as precopy),
+however when a page request is received from the destination, the dirty page
+scanning restarts from the requested location.  This causes requested pages
+to be sent quickly, and also causes pages directly after the requested page
+to be sent quickly in the hope that those pages are likely to be used
+by the destination soon.
+
+Destination behaviour
+
+Initially the destination looks the same as precopy, with a single thread
+reading the migration stream; the 'postcopy advise' and 'discard' commands
+are processed to change the way RAM is managed, but don't affect the stream
+processing.
+
+------------------------------------------------------------------------------
+                        1      2   3     4 5                      6   7
+main -----DISCARD-CMD_PACKAGED ( LISTEN  DEVICE     DEVICE DEVICE RUN )
+thread                             |       |
+                                   |     (page request)
+                                   |        \___
+                                   v            \
+listen thread:                     --- page -- page -- page -- page -- page --
+
+                                   a   b        c
+------------------------------------------------------------------------------
+
+On receipt of CMD_PACKAGED (1)
+   All the data associated with the package - the ( ... ) section in the
+diagram - is read into memory, and the main thread recurses into
+qemu_loadvm_state_main to process the contents of the package (2)
+which contains commands (3,6) and devices (4...)
+
+On receipt of 'postcopy listen' - 3 -(i.e. the 1st command in the package)
+a new thread (a) is started that takes over servicing the migration stream,
+while the main thread carries on loading the package.   It loads normal
+background page data (b) but if during a device load a fault happens (5) the
+returned page (c) is loaded by the listen thread allowing the main threads
+device load to carry on.
+
+The last thing in the CMD_PACKAGED is a 'RUN' command (6) letting the destination
+CPUs start running.
+At the end of the CMD_PACKAGED (7) the main thread returns to normal running behaviour
+and is no longer used by migration, while the listen thread carries
+on servicing page data until the end of migration.
+
+=== Postcopy states ===
+
+Postcopy moves through a series of states (see postcopy_state) from
+ADVISE->DISCARD->LISTEN->RUNNING->END
+
+  Advise:  Set at the start of migration if postcopy is enabled, even
+           if it hasn't had the start command; here the destination
+           checks that its OS has the support needed for postcopy, and performs
+           setup to ensure the RAM mappings are suitable for later postcopy.
+           The destination will fail early in migration at this point if the
+           required OS support is not present.
+           (Triggered by reception of POSTCOPY_ADVISE command)
+
+  Discard: Entered on receipt of the first 'discard' command; prior to
+           the first Discard being performed, hugepages are switched off
+           (using madvise) to ensure that no new huge pages are created
+           during the postcopy phase, and to cause any huge pages that
+           have discards on them to be broken.
+
+  Listen:  The first command in the package, POSTCOPY_LISTEN, switches
+           the destination state to Listen, and starts a new thread
+           (the 'listen thread') which takes over the job of receiving
+           pages off the migration stream, while the main thread carries
+           on processing the blob.  With this thread able to process page
+           reception, the destination now 'sensitises' the RAM to detect
+           any access to missing pages (on Linux using the 'userfault'
+           system).
+
+  Running: POSTCOPY_RUN causes the destination to synchronise all
+           state and start the CPUs and IO devices running.  The main
+           thread now finishes processing the migration package and
+           now carries on as it would for normal precopy migration
+           (although it can't do the cleanup it would do as it
+           finishes a normal migration).
+
+  End:     The listen thread can now quit, and perform the cleanup of migration
+           state, the migration is now complete.
+
+=== Source side page maps ===
+
+The source side keeps two bitmaps during postcopy; 'the migration bitmap'
+and 'unsent map'.  The 'migration bitmap' is basically the same as in
+the precopy case, and holds a bit to indicate that page is 'dirty' -
+i.e. needs sending.  During the precopy phase this is updated as the CPU
+dirties pages, however during postcopy the CPUs are stopped and nothing
+should dirty anything any more.
+
+The 'unsent map' is used for the transition to postcopy. It is a bitmap that
+has a bit cleared whenever a page is sent to the destination, however during
+the transition to postcopy mode it is combined with the migration bitmap
+to form a set of pages that:
+   a) Have been sent but then redirtied (which must be discarded)
+   b) Have not yet been sent - which also must be discarded to cause any
+      transparent huge pages built during precopy to be broken.
+
+Note that the contents of the unsentmap are sacrificed during the calculation
+of the discard set and thus aren't valid once in postcopy.  The dirtymap
+is still valid and is used to ensure that no page is sent more than once.  Any
+request for a page that has already been sent is ignored.  Duplicate requests
+such as this can happen as a page is sent at about the same time the
+destination accesses it.
+
diff --git a/docs/multi-thread-compression.txt b/docs/multi-thread-compression.txt
new file mode 100644
index 0000000..d0caaf7
--- /dev/null
+++ b/docs/multi-thread-compression.txt
@@ -0,0 +1,149 @@
+Use multiple thread (de)compression in live migration
+=====================================================
+Copyright (C) 2015 Intel Corporation
+Author: Liang Li <liang.z.li@intel.com>
+
+This work is licensed under the terms of the GNU GPLv2 or later. See
+the COPYING file in the top-level directory.
+
+Contents:
+=========
+* Introduction
+* When to use
+* Performance
+* Usage
+* TODO
+
+Introduction
+============
+Instead of sending the guest memory directly, this solution will
+compress the RAM page before sending; after receiving, the data will
+be decompressed. Using compression in live migration can help
+to reduce the data transferred about 60%, this is very useful when the
+bandwidth is limited, and the total migration time can also be reduced
+about 70% in a typical case. In addition to this, the VM downtime can be
+reduced about 50%. The benefit depends on data's compressibility in VM.
+
+The process of compression will consume additional CPU cycles, and the
+extra CPU cycles will increase the migration time. On the other hand,
+the amount of data transferred will decrease; this factor can reduce
+the total migration time. If the process of the compression is quick
+enough, then the total migration time can be reduced, and multiple
+thread compression can be used to accelerate the compression process.
+
+The decompression speed of Zlib is at least 4 times as quick as
+compression, if the source and destination CPU have equal speed,
+keeping the compression thread count 4 times the decompression
+thread count can avoid resource waste.
+
+Compression level can be used to control the compression speed and the
+compression ratio. High compression ratio will take more time, level 0
+stands for no compression, level 1 stands for the best compression
+speed, and level 9 stands for the best compression ratio. Users can
+select a level number between 0 and 9.
+
+
+When to use the multiple thread compression in live migration
+=============================================================
+Compression of data will consume extra CPU cycles; so in a system with
+high overhead of CPU, avoid using this feature. When the network
+bandwidth is very limited and the CPU resource is adequate, use of
+multiple thread compression will be very helpful. If both the CPU and
+the network bandwidth are adequate, use of multiple thread compression
+can still help to reduce the migration time.
+
+Performance
+===========
+Test environment:
+
+CPU: Intel(R) Xeon(R) CPU E5-2680 0 @ 2.70GHz
+Socket Count: 2
+RAM: 128G
+NIC: Intel I350 (10/100/1000Mbps)
+Host OS: CentOS 7 64-bit
+Guest OS: RHEL 6.5 64-bit
+Parameter: qemu-system-x86_64 -enable-kvm -smp 4 -m 4096
+ /share/ia32e_rhel6u5.qcow -monitor stdio
+
+There is no additional application is running on the guest when doing
+the test.
+
+
+Speed limit: 1000Gb/s
+---------------------------------------------------------------
+                    | original  | compress thread: 8
+                    |   way     | decompress thread: 2
+                    |           | compression level: 1
+---------------------------------------------------------------
+total time(msec):   |   3333    |  1833
+---------------------------------------------------------------
+downtime(msec):     |    100    |   27
+---------------------------------------------------------------
+transferred ram(kB):|  363536   | 107819
+---------------------------------------------------------------
+throughput(mbps):   |  893.73   | 482.22
+---------------------------------------------------------------
+total ram(kB):      |  4211524  | 4211524
+---------------------------------------------------------------
+
+There is an application running on the guest which write random numbers
+to RAM block areas periodically.
+
+Speed limit: 1000Gb/s
+---------------------------------------------------------------
+                    | original  | compress thread: 8
+                    |   way     | decompress thread: 2
+                    |           | compression level: 1
+---------------------------------------------------------------
+total time(msec):   |   37369   | 15989
+---------------------------------------------------------------
+downtime(msec):     |    337    |  173
+---------------------------------------------------------------
+transferred ram(kB):|  4274143  | 1699824
+---------------------------------------------------------------
+throughput(mbps):   |  936.99   | 870.95
+---------------------------------------------------------------
+total ram(kB):      |  4211524  | 4211524
+---------------------------------------------------------------
+
+Usage
+=====
+1. Verify both the source and destination QEMU are able
+to support the multiple thread compression migration:
+    {qemu} info migrate_capabilities
+    {qemu} ... compress: off ...
+
+2. Activate compression on the source:
+    {qemu} migrate_set_capability compress on
+
+3. Set the compression thread count on source:
+    {qemu} migrate_set_parameter compress_threads 12
+
+4. Set the compression level on the source:
+    {qemu} migrate_set_parameter compress_level 1
+
+5. Set the decompression thread count on destination:
+    {qemu} migrate_set_parameter decompress_threads 3
+
+6. Start outgoing migration:
+    {qemu} migrate -d tcp:destination.host:4444
+    {qemu} info migrate
+    Capabilities: ... compress: on
+    ...
+
+The following are the default settings:
+    compress: off
+    compress_threads: 8
+    decompress_threads: 2
+    compress_level: 1 (which means best speed)
+
+So, only the first two steps are required to use the multiple
+thread compression in migration. You can do more if the default
+settings are not appropriate.
+
+TODO
+====
+Some faster (de)compression method such as LZ4 and Quicklz can help
+to reduce the CPU consumption when doing (de)compression. If using
+these faster (de)compression method, less (de)compression threads
+are needed when doing the migration.
diff --git a/docs/multiseat.txt b/docs/multiseat.txt
index 67151e0..807518c 100644
--- a/docs/multiseat.txt
+++ b/docs/multiseat.txt
@@ -2,12 +2,12 @@
 multiseat howto (with some multihead coverage)
 ==============================================
 
-host side
----------
+host devices
+------------
 
 First you must compile qemu with a user interface supporting
 multihead/multiseat and input event routing.  Right now this
-list includes sdl2 and gtk (both 2+3):
+list includes sdl2, gtk (both 2+3) and vnc:
 
   ./configure --enable-sdl --with-sdlabi=2.0
 
@@ -16,16 +16,16 @@
   ./configure --enable-gtk
 
 
-Next put together the qemu command line:
+Next put together the qemu command line (sdk/gtk):
 
 qemu	-enable-kvm -usb $memory $disk $whatever \
 	-display [ sdl | gtk ] \
 	-vga std \
 	-device usb-tablet
 
-That is it for the first head, which will use the standard vga, the
+That is it for the first seat, which will use the standard vga, the
 standard ps/2 keyboard (implicitly there) and the usb-tablet.  Now the
-additional switches for the second head:
+additional switches for the second seat:
 
 	-device pci-bridge,addr=12.0,chassis_nr=2,id=head.2 \
 	-device secondary-vga,bus=head.2,addr=02.0,id=video.2 \
@@ -41,12 +41,35 @@
 the window which belongs to the video.2 display adapter will be routed
 to these input devices.
 
+Starting with qemu 2.4 and linux kernel 4.1 you can also use virtio
+for the input devices, using this ...
+
+	-device pci-bridge,addr=12.0,chassis_nr=2,id=head.2 \
+	-device secondary-vga,bus=head.2,addr=02.0,id=video.2 \
+	-device virtio-keyboard-pci,bus=head.2,addr=03.0,display=video.2 \
+	-device virtio-tablet-pci,bus=head.2,addr=03.0,display=video.2
+
+... instead of xhci and usb hid devices.
+
+host ui
+-------
+
 The sdl2 ui will start up with two windows, one for each display
 device.  The gtk ui will start with a single window and each display
 in a separate tab.  You can either simply switch tabs to switch heads,
 or use the "View / Detach tab" menu item to move one of the displays
 to its own window so you can see both display devices side-by-side.
 
+For vnc some additional configuration on the command line is needed.
+We'll create two vnc server instances, and bind the second one to the
+second seat, simliar to input devices:
+
+	-display vnc=:1,id=primary \
+	-display vnc=:2,id=secondary,display=video.2
+
+Connecting to vnc display :1 gives you access to the first seat, and
+likewise connecting to vnc display :2 shows the second seat.
+
 Note on spice: Spice handles multihead just fine.  But it can't do
 multiseat.  For tablet events the event source is sent to the spice
 agent.  But qemu can't figure it, so it can't do input routing.
@@ -96,6 +119,26 @@
 Background info is here:
   http://www.freedesktop.org/wiki/Software/systemd/multiseat/
 
+
+guest side with pci-bridge-seat
+-------------------------------
+
+Qemu version 2.4 and newer has a new pci-bridge-seat device which
+can be used instead of pci-bridge.  Just swap the device name in the
+qemu command line above.  The only difference between the two devices
+is the pci id.  We can match the pci id instead of the device path
+with a nice generic rule now, which simplifies the guest
+configuration:
+
+    [root@fedora ~]# cat /etc/udev/rules.d/70-qemu-pci-bridge-seat.rules
+    SUBSYSTEM=="pci", ATTR{vendor}=="0x1b36", ATTR{device}=="0x000a", \
+            TAG+="seat", ENV{ID_AUTOSEAT}="1"
+
+Patch with this rule has been submitted to upstream udev/systemd, was
+accepted and should be included in the next systemd release (222).
+So, if your guest has this or a newer version, multiseat will work just
+fine without any manual guest configuration.
+
 Enjoy!
 
 --
diff --git a/docs/pci_expander_bridge.txt b/docs/pci_expander_bridge.txt
new file mode 100644
index 0000000..3675027
--- /dev/null
+++ b/docs/pci_expander_bridge.txt
@@ -0,0 +1,58 @@
+PCI EXPANDER BRIDGE (PXB)
+=========================
+
+Description
+===========
+PXB is a "light-weight" host bridge in the same PCI domain
+as the main host bridge whose purpose is to enable
+the main host bridge to support multiple PCI root buses.
+It is implemented only for i440fx and can be placed only
+on bus 0 (pci.0).
+
+As opposed to PCI-2-PCI bridge's secondary bus, PXB's bus
+is a primary bus and can be associated with a NUMA node
+(different from the main host bridge) allowing the guest OS
+to recognize the proximity of a pass-through device to
+other resources as RAM and CPUs.
+
+Usage
+=====
+A detailed command line would be:
+
+[qemu-bin + storage options]
+-m 2G
+-object memory-backend-ram,size=1024M,policy=bind,host-nodes=0,id=ram-node0 -numa node,nodeid=0,cpus=0,memdev=ram-node0
+-object memory-backend-ram,size=1024M,policy=bind,host-nodes=1,id=ram-node1 -numa node,nodeid=1,cpus=1,memdev=ram-node1
+-device pxb,id=bridge1,bus=pci.0,numa_node=1,bus_nr=4 -netdev user,id=nd -device e1000,bus=bridge1,addr=0x4,netdev=nd
+-device pxb,id=bridge2,bus=pci.0,numa_node=0,bus_nr=8 -device e1000,bus=bridge2,addr=0x3
+-device pxb,id=bridge3,bus=pci.0,bus_nr=40 -drive if=none,id=drive0,file=[img] -device virtio-blk-pci,drive=drive0,scsi=off,bus=bridge3,addr=1
+
+Here you have:
+ - 2 NUMA nodes for the guest, 0 and 1. (both mapped to the same NUMA node in host, but you can and should put it in different host NUMA nodes)
+ - a pxb host bridge attached to NUMA 1 with an e1000 behind it
+ - a pxb host bridge attached to NUMA 0 with an e1000 behind it
+ - a pxb host bridge not attached to any NUMA with a hard drive behind it.
+
+Limitations
+===========
+Please observe that we specified the bus "pci.0" for the second and third pxb.
+This is because when no bus is given, another pxb can be selected by QEMU as default bus,
+however, PXBs can be placed only under the root bus.
+
+Implementation
+==============
+The PXB is composed by:
+- HostBridge (TYPE_PXB_HOST)
+  The host bridge allows to register and query the PXB's PCI root bus in QEMU.
+- PXBDev(TYPE_PXB_DEVICE)
+  It is a regular PCI Device that resides on the piix host-bridge bus and its bus uses the same PCI domain.
+  However, the bus behind is exposed through ACPI as a primary PCI bus and starts a new PCI hierarchy.
+  The interrupts from devices behind the PXB are routed through this device the same as if it were a
+  PCI-2-PCI bridge. The _PRT follows the i440fx model.
+- PCIBridgeDev(TYPE_PCI_BRIDGE_DEV)
+  Created automatically as part of init sequence.
+  When adding a device to PXB it is attached to the bridge for two reasons:
+  - Using the bridge will enable hotplug support
+  - All the devices behind the bridge will use bridge's IO/MEM windows compacting
+    the PCI address space.
+
diff --git a/docs/qapi-code-gen.txt b/docs/qapi-code-gen.txt
index 8313ba6..de298dc 100644
--- a/docs/qapi-code-gen.txt
+++ b/docs/qapi-code-gen.txt
@@ -1,61 +1,203 @@
 = How to use the QAPI code generator =
 
+Copyright IBM Corp. 2011
+Copyright (C) 2012-2016 Red Hat, Inc.
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+== Introduction ==
+
 QAPI is a native C API within QEMU which provides management-level
-functionality to internal/external users. For external
-users/processes, this interface is made available by a JSON-based
-QEMU Monitor protocol that is provided by the QMP server.
+functionality to internal and external users. For external
+users/processes, this interface is made available by a JSON-based wire
+format for the QEMU Monitor Protocol (QMP) for controlling qemu, as
+well as the QEMU Guest Agent (QGA) for communicating with the guest.
+The remainder of this document uses "Client JSON Protocol" when
+referring to the wire contents of a QMP or QGA connection.
 
-To map QMP-defined interfaces to the native C QAPI implementations,
-a JSON-based schema is used to define types and function
-signatures, and a set of scripts is used to generate types/signatures,
-and marshaling/dispatch code. The QEMU Guest Agent also uses these
-scripts, paired with a separate schema, to generate
-marshaling/dispatch code for the guest agent server running in the
-guest.
-
-This document will describe how the schemas, scripts, and resulting
-code are used.
+To map Client JSON Protocol interfaces to the native C QAPI
+implementations, a JSON-based schema is used to define types and
+function signatures, and a set of scripts is used to generate types,
+signatures, and marshaling/dispatch code. This document will describe
+how the schemas, scripts, and resulting code are used.
 
 
 == QMP/Guest agent schema ==
 
-This file defines the types, commands, and events used by QMP.  It should
-fully describe the interface used by QMP.
+A QAPI schema file is designed to be loosely based on JSON
+(http://www.ietf.org/rfc/rfc7159.txt) with changes for quoting style
+and the use of comments; a QAPI schema file is then parsed by a python
+code generation program.  A valid QAPI schema consists of a series of
+top-level expressions, with no commas between them.  Where
+dictionaries (JSON objects) are used, they are parsed as python
+OrderedDicts so that ordering is preserved (for predictable layout of
+generated C structs and parameter lists).  Ordering doesn't matter
+between top-level expressions or the keys within an expression, but
+does matter within dictionary values for 'data' and 'returns' members
+of a single expression.  QAPI schema input is written using 'single
+quotes' instead of JSON's "double quotes" (in contrast, Client JSON
+Protocol uses no comments, and while input accepts 'single quotes' as
+an extension, output is strict JSON using only "double quotes").  As
+in JSON, trailing commas are not permitted in arrays or dictionaries.
+Input must be ASCII (although QMP supports full Unicode strings, the
+QAPI parser does not).  At present, there is no place where a QAPI
+schema requires the use of JSON numbers or null.
 
-This file is designed to be loosely based on JSON although it's technically
-executable Python.  While dictionaries are used, they are parsed as
-OrderedDicts so that ordering is preserved.
+Comments are allowed; anything between an unquoted # and the following
+newline is ignored.  Although there is not yet a documentation
+generator, a form of stylized comments has developed for consistently
+documenting details about an expression and when it was added to the
+schema.  The documentation is delimited between two lines of ##, then
+the first line names the expression, an optional overview is provided,
+then individual documentation about each member of 'data' is provided,
+and finally, a 'Since: x.y.z' tag lists the release that introduced
+the expression.  Optional members are tagged with the phrase
+'#optional', often with their default value; and extensions added
+after the expression was first released are also given a '(since
+x.y.z)' comment.  For example:
 
-There are two basic syntaxes used, type definitions and command definitions.
+    ##
+    # @BlockStats:
+    #
+    # Statistics of a virtual block device or a block backing device.
+    #
+    # @device: #optional If the stats are for a virtual block device, the name
+    #          corresponding to the virtual block device.
+    #
+    # @stats:  A @BlockDeviceStats for the device.
+    #
+    # @parent: #optional This describes the file block device if it has one.
+    #
+    # @backing: #optional This describes the backing block device if it has one.
+    #           (Since 2.0)
+    #
+    # Since: 0.14.0
+    ##
+    { 'struct': 'BlockStats',
+      'data': {'*device': 'str', 'stats': 'BlockDeviceStats',
+               '*parent': 'BlockStats',
+               '*backing': 'BlockStats'} }
 
-The first syntax defines a type and is represented by a dictionary.  There are
-three kinds of user-defined types that are supported: complex types,
-enumeration types and union types.
+The schema sets up a series of types, as well as commands and events
+that will use those types.  Forward references are allowed: the parser
+scans in two passes, where the first pass learns all type names, and
+the second validates the schema and generates the code.  This allows
+the definition of complex structs that can have mutually recursive
+types, and allows for indefinite nesting of Client JSON Protocol that
+satisfies the schema.  A type name should not be defined more than
+once.  It is permissible for the schema to contain additional types
+not used by any commands or events in the Client JSON Protocol, for
+the side effect of generated C code used internally.
 
-Generally speaking, types definitions should always use CamelCase for the type
-names. Command names should be all lower case with words separated by a hyphen.
+There are seven top-level expressions recognized by the parser:
+'include', 'command', 'struct', 'enum', 'union', 'alternate', and
+'event'.  There are several groups of types: simple types (a number of
+built-in types, such as 'int' and 'str'; as well as enumerations),
+complex types (structs and two flavors of unions), and alternate types
+(a choice between other types).  The 'command' and 'event' expressions
+can refer to existing types by name, or list an anonymous type as a
+dictionary. Listing a type name inside an array refers to a
+single-dimension array of that type; multi-dimension arrays are not
+directly supported (although an array of a complex struct that
+contains an array member is possible).
+
+Types, commands, and events share a common namespace.  Therefore,
+generally speaking, type definitions should always use CamelCase for
+user-defined type names, while built-in types are lowercase. Type
+definitions should not end in 'Kind', as this namespace is used for
+creating implicit C enums for visiting union types, or in 'List', as
+this namespace is used for creating array types.  Command names,
+and member names within a type, should be all lower case with words
+separated by a hyphen.  However, some existing older commands and
+complex types use underscore; when extending such expressions,
+consistency is preferred over blindly avoiding underscore.  Event
+names should be ALL_CAPS with words separated by underscore.  Member
+names cannot start with 'has-' or 'has_', as this is reserved for
+tracking optional members.
+
+Any name (command, event, type, member, or enum value) beginning with
+"x-" is marked experimental, and may be withdrawn or changed
+incompatibly in a future release.  All names must begin with a letter,
+and contain only ASCII letters, digits, dash, and underscore.  There
+are two exceptions: enum values may start with a digit, and any
+extensions added by downstream vendors should start with a prefix
+matching "__RFQDN_" (for the reverse-fully-qualified-domain-name of
+the vendor), even if the rest of the name uses dash (example:
+__com.redhat_drive-mirror).  Names beginning with 'q_' are reserved
+for the generator: QMP names that resemble C keywords or other
+problematic strings will be munged in C to use this prefix.  For
+example, a member named "default" in qapi becomes "q_default" in the
+generated C code.
+
+In the rest of this document, usage lines are given for each
+expression type, with literal strings written in lower case and
+placeholders written in capitals.  If a literal string includes a
+prefix of '*', that key/value pair can be omitted from the expression.
+For example, a usage statement that includes '*base':STRUCT-NAME
+means that an expression has an optional key 'base', which if present
+must have a value that forms a struct name.
+
+
+=== Built-in Types ===
+
+The following types are predefined, and map to C as follows:
+
+  Schema    C          JSON
+  str       char *     any JSON string, UTF-8
+  number    double     any JSON number
+  int       int64_t    a JSON number without fractional part
+                       that fits into the C integer type
+  int8      int8_t     likewise
+  int16     int16_t    likewise
+  int32     int32_t    likewise
+  int64     int64_t    likewise
+  uint8     uint8_t    likewise
+  uint16    uint16_t   likewise
+  uint32    uint32_t   likewise
+  uint64    uint64_t   likewise
+  size      uint64_t   like uint64_t, except StringInputVisitor
+                       accepts size suffixes
+  bool      bool       JSON true or false
+  any       QObject *  any JSON value
+  QType     QType      JSON string matching enum QType values
 
 
 === Includes ===
 
+Usage: { 'include': STRING }
+
 The QAPI schema definitions can be modularized using the 'include' directive:
 
- { 'include': 'path/to/file.json'}
+ { 'include': 'path/to/file.json' }
 
 The directive is evaluated recursively, and include paths are relative to the
-file using the directive. Multiple includes of the same file are safe.
+file using the directive. Multiple includes of the same file are
+idempotent.  No other keys should appear in the expression, and the include
+value should be a string.
+
+As a matter of style, it is a good idea to have all files be
+self-contained, but at the moment, nothing prevents an included file
+from making a forward reference to a type that is only introduced by
+an outer file.  The parser may be made stricter in the future to
+prevent incomplete include files.
 
 
-=== Complex types ===
+=== Struct types ===
 
-A complex type is a dictionary containing a single key whose value is a
-dictionary.  This corresponds to a struct in C or an Object in JSON.  An
-example of a complex type is:
+Usage: { 'struct': STRING, 'data': DICT, '*base': STRUCT-NAME }
 
- { 'type': 'MyType',
+A struct is a dictionary containing a single 'data' key whose value is
+a dictionary; the dictionary may be empty.  This corresponds to a
+struct in C or an Object in JSON. Each value of the 'data' dictionary
+must be the name of a type, or a one-element array containing a type
+name.  An example of a struct is:
+
+ { 'struct': 'MyType',
    'data': { 'member1': 'str', 'member2': 'int', '*member3': 'str' } }
 
-The use of '*' as a prefix to the name means the member is optional.
+The use of '*' as a prefix to the name means the member is optional in
+the corresponding JSON protocol usage.
 
 The default initialization value of an optional argument should not be changed
 between versions of QEMU unless the new default maintains backward
@@ -75,148 +217,297 @@
 
 On output structures (only mentioned in the 'returns' side of a command),
 changing from mandatory to optional is in general unsafe (older clients may be
-expecting the field, and could crash if it is missing), although it can be done
-if the only way that the optional argument will be omitted is when it is
-triggered by the presence of a new input flag to the command that older clients
-don't know to send.  Changing from optional to mandatory is safe.
+expecting the member, and could crash if it is missing), although it
+can be done if the only way that the optional argument will be omitted
+is when it is triggered by the presence of a new input flag to the
+command that older clients don't know to send.  Changing from optional
+to mandatory is safe.
 
 A structure that is used in both input and output of various commands
 must consider the backwards compatibility constraints of both directions
 of use.
 
-A complex type definition can specify another complex type as its base.
-In this case, the fields of the base type are included as top-level fields
-of the new complex type's dictionary in the QMP wire format. An example
-definition is:
+A struct definition can specify another struct as its base.
+In this case, the members of the base type are included as top-level members
+of the new struct's dictionary in the Client JSON Protocol wire
+format. An example definition is:
 
- { 'type': 'BlockdevOptionsGenericFormat', 'data': { 'file': 'str' } }
- { 'type': 'BlockdevOptionsGenericCOWFormat',
+ { 'struct': 'BlockdevOptionsGenericFormat', 'data': { 'file': 'str' } }
+ { 'struct': 'BlockdevOptionsGenericCOWFormat',
    'base': 'BlockdevOptionsGenericFormat',
    'data': { '*backing': 'str' } }
 
 An example BlockdevOptionsGenericCOWFormat object on the wire could use
-both fields like this:
+both members like this:
 
  { "file": "/some/place/my-image",
    "backing": "/some/place/my-backing-file" }
 
+
 === Enumeration types ===
 
-An enumeration type is a dictionary containing a single key whose value is a
-list of strings.  An example enumeration is:
+Usage: { 'enum': STRING, 'data': ARRAY-OF-STRING }
+       { 'enum': STRING, '*prefix': STRING, 'data': ARRAY-OF-STRING }
+
+An enumeration type is a dictionary containing a single 'data' key
+whose value is a list of strings.  An example enumeration is:
 
  { 'enum': 'MyEnum', 'data': [ 'value1', 'value2', 'value3' ] }
 
+Nothing prevents an empty enumeration, although it is probably not
+useful.  The list of strings should be lower case; if an enum name
+represents multiple words, use '-' between words.  The string 'max' is
+not allowed as an enum value, and values should not be repeated.
+
+The enum constants will be named by using a heuristic to turn the
+type name into a set of underscore separated words. For the example
+above, 'MyEnum' will turn into 'MY_ENUM' giving a constant name
+of 'MY_ENUM_VALUE1' for the first value. If the default heuristic
+does not result in a desirable name, the optional 'prefix' member
+can be used when defining the enum.
+
+The enumeration values are passed as strings over the Client JSON
+Protocol, but are encoded as C enum integral values in generated code.
+While the C code starts numbering at 0, it is better to use explicit
+comparisons to enum values than implicit comparisons to 0; the C code
+will also include a generated enum member ending in _MAX for tracking
+the size of the enum, useful when using common functions for
+converting between strings and enum values.  Since the wire format
+always passes by name, it is acceptable to reorder or add new
+enumeration members in any location without breaking clients of Client
+JSON Protocol; however, removing enum values would break
+compatibility.  For any struct that has a member that will only contain
+a finite set of string values, using an enum type for that member is
+better than open-coding the member to be type 'str'.
+
+
 === Union types ===
 
-Union types are used to let the user choose between several different data
-types.  A union type is defined using a dictionary as explained in the
-following paragraphs.
+Usage: { 'union': STRING, 'data': DICT }
+or:    { 'union': STRING, 'data': DICT, 'base': STRUCT-NAME-OR-DICT,
+         'discriminator': ENUM-MEMBER-OF-BASE }
 
+Union types are used to let the user choose between several different
+variants for an object.  There are two flavors: simple (no
+discriminator or base), and flat (both discriminator and base).  A union
+type is defined using a data dictionary as explained in the following
+paragraphs.  The data dictionary for either type of union must not
+be empty.
 
-A simple union type defines a mapping from discriminator values to data types
-like in this example:
+A simple union type defines a mapping from automatic discriminator
+values to data types like in this example:
 
- { 'type': 'FileOptions', 'data': { 'filename': 'str' } }
- { 'type': 'Qcow2Options',
-   'data': { 'backing-file': 'str', 'lazy-refcounts': 'bool' } }
+ { 'struct': 'BlockdevOptionsFile', 'data': { 'filename': 'str' } }
+ { 'struct': 'BlockdevOptionsQcow2',
+   'data': { 'backing': 'str', '*lazy-refcounts': 'bool' } }
 
+ { 'union': 'BlockdevOptionsSimple',
+   'data': { 'file': 'BlockdevOptionsFile',
+             'qcow2': 'BlockdevOptionsQcow2' } }
+
+In the Client JSON Protocol, a simple union is represented by a
+dictionary that contains the 'type' member as a discriminator, and a
+'data' member that is of the specified data type corresponding to the
+discriminator value, as in these examples:
+
+ { "type": "file", "data": { "filename": "/some/place/my-image" } }
+ { "type": "qcow2", "data": { "backing": "/some/place/my-image",
+                              "lazy-refcounts": true } }
+
+The generated C code uses a struct containing a union. Additionally,
+an implicit C enum 'NameKind' is created, corresponding to the union
+'Name', for accessing the various branches of the union.  No branch of
+the union can be named 'max', as this would collide with the implicit
+enum.  The value for each branch can be of any type.
+
+A flat union definition avoids nesting on the wire, and specifies a
+set of common members that occur in all variants of the union.  The
+'base' key must specify either a type name (the type must be a
+struct, not a union), or a dictionary representing an anonymous type.
+All branches of the union must be complex types, and the top-level
+members of the union dictionary on the wire will be combination of
+members from both the base type and the appropriate branch type (when
+merging two dictionaries, there must be no keys in common).  The
+'discriminator' member must be the name of a non-optional enum-typed
+member of the base struct.
+
+The following example enhances the above simple union example by
+adding an optional common member 'read-only', renaming the
+discriminator to something more applicable than the simple union's
+default of 'type', and reducing the number of {} required on the wire:
+
+ { 'enum': 'BlockdevDriver', 'data': [ 'file', 'qcow2' ] }
  { 'union': 'BlockdevOptions',
-   'data': { 'file': 'FileOptions',
-             'qcow2': 'Qcow2Options' } }
-
-In the QMP wire format, a simple union is represented by a dictionary that
-contains the 'type' field as a discriminator, and a 'data' field that is of the
-specified data type corresponding to the discriminator value:
-
- { "type": "qcow2", "data" : { "backing-file": "/some/place/my-image",
-                               "lazy-refcounts": true } }
-
-
-A union definition can specify a complex type as its base. In this case, the
-fields of the complex type are included as top-level fields of the union
-dictionary in the QMP wire format. An example definition is:
-
- { 'type': 'BlockdevCommonOptions', 'data': { 'readonly': 'bool' } }
- { 'union': 'BlockdevOptions',
-   'base': 'BlockdevCommonOptions',
-   'data': { 'raw': 'RawOptions',
-             'qcow2': 'Qcow2Options' } }
-
-And it looks like this on the wire:
-
- { "type": "qcow2",
-   "readonly": false,
-   "data" : { "backing-file": "/some/place/my-image",
-              "lazy-refcounts": true } }
-
-
-Flat union types avoid the nesting on the wire. They are used whenever a
-specific field of the base type is declared as the discriminator ('type' is
-then no longer generated). The discriminator must be of enumeration type.
-The above example can then be modified as follows:
-
- { 'enum': 'BlockdevDriver', 'data': [ 'raw', 'qcow2' ] }
- { 'type': 'BlockdevCommonOptions',
-   'data': { 'driver': 'BlockdevDriver', 'readonly': 'bool' } }
- { 'union': 'BlockdevOptions',
-   'base': 'BlockdevCommonOptions',
+   'base': { 'driver': 'BlockdevDriver', '*read-only': 'bool' },
    'discriminator': 'driver',
-   'data': { 'raw': 'RawOptions',
-             'qcow2': 'Qcow2Options' } }
+   'data': { 'file': 'BlockdevOptionsFile',
+             'qcow2': 'BlockdevOptionsQcow2' } }
 
-Resulting in this JSON object:
+Resulting in these JSON objects:
 
- { "driver": "qcow2",
-   "readonly": false,
-   "backing-file": "/some/place/my-image",
-   "lazy-refcounts": true }
+ { "driver": "file", "read-only": true,
+   "filename": "/some/place/my-image" }
+ { "driver": "qcow2", "read-only": false,
+   "backing": "/some/place/my-image", "lazy-refcounts": true }
+
+Notice that in a flat union, the discriminator name is controlled by
+the user, but because it must map to a base member with enum type, the
+code generator can ensure that branches exist for all values of the
+enum (although the order of the keys need not match the declaration of
+the enum).  In the resulting generated C data types, a flat union is
+represented as a struct with the base members included directly, and
+then a union of structures for each branch of the struct.
+
+A simple union can always be re-written as a flat union where the base
+class has a single member named 'type', and where each branch of the
+union has a struct with a single member named 'data'.  That is,
+
+ { 'union': 'Simple', 'data': { 'one': 'str', 'two': 'int' } }
+
+is identical on the wire to:
+
+ { 'enum': 'Enum', 'data': ['one', 'two'] }
+ { 'struct': 'Branch1', 'data': { 'data': 'str' } }
+ { 'struct': 'Branch2', 'data': { 'data': 'int' } }
+ { 'union': 'Flat': 'base': { 'type': 'Enum' }, 'discriminator': 'type',
+   'data': { 'one': 'Branch1', 'two': 'Branch2' } }
 
 
-A special type of unions are anonymous unions. They don't form a dictionary in
-the wire format but allow the direct use of different types in their place. As
-they aren't structured, they don't have any explicit discriminator but use
-the (QObject) data type of their value as an implicit discriminator. This means
-that they are restricted to using only one discriminator value per QObject
-type. For example, you cannot have two different complex types in an anonymous
-union, or two different integer types.
+=== Alternate types ===
 
-Anonymous unions are declared using an empty dictionary as their discriminator.
-The discriminator values never appear on the wire, they are only used in the
-generated C code. Anonymous unions cannot have a base type.
+Usage: { 'alternate': STRING, 'data': DICT }
 
- { 'union': 'BlockRef',
-   'discriminator': {},
+An alternate type is one that allows a choice between two or more JSON
+data types (string, integer, number, or object, but currently not
+array) on the wire.  The definition is similar to a simple union type,
+where each branch of the union names a QAPI type.  For example:
+
+ { 'alternate': 'BlockdevRef',
    'data': { 'definition': 'BlockdevOptions',
              'reference': 'str' } }
 
-This example allows using both of the following example objects:
+Unlike a union, the discriminator string is never passed on the wire
+for the Client JSON Protocol.  Instead, the value's JSON type serves
+as an implicit discriminator, which in turn means that an alternate
+can only express a choice between types represented differently in
+JSON.  If a branch is typed as the 'bool' built-in, the alternate
+accepts true and false; if it is typed as any of the various numeric
+built-ins, it accepts a JSON number; if it is typed as a 'str'
+built-in or named enum type, it accepts a JSON string; and if it is
+typed as a complex type (struct or union), it accepts a JSON object.
+Two different complex types, for instance, aren't permitted, because
+both are represented as a JSON object.
+
+The example alternate declaration above allows using both of the
+following example objects:
 
  { "file": "my_existing_block_device_id" }
  { "file": { "driver": "file",
-             "readonly": false,
+             "read-only": false,
              "filename": "/tmp/mydisk.qcow2" } }
 
 
 === Commands ===
 
-Commands are defined by using a list containing three members.  The first
-member is the command name, the second member is a dictionary containing
-arguments, and the third member is the return type.
+Usage: { 'command': STRING, '*data': COMPLEX-TYPE-NAME-OR-DICT,
+         '*returns': TYPE-NAME, '*boxed': true,
+         '*gen': false, '*success-response': false }
 
-An example command is:
+Commands are defined by using a dictionary containing several members,
+where three members are most common.  The 'command' member is a
+mandatory string, and determines the "execute" value passed in a
+Client JSON Protocol command exchange.
 
- { 'command': 'my-command',
-   'data': { 'arg1': 'str', '*arg2': 'str' },
-   'returns': 'str' }
+The 'data' argument maps to the "arguments" dictionary passed in as
+part of a Client JSON Protocol command.  The 'data' member is optional
+and defaults to {} (an empty dictionary).  If present, it must be the
+string name of a complex type, or a dictionary that declares an
+anonymous type with the same semantics as a 'struct' expression, with
+one exception noted below when 'gen' is used.
+
+The 'returns' member describes what will appear in the "return" member
+of a Client JSON Protocol reply on successful completion of a command.
+The member is optional from the command declaration; if absent, the
+"return" member will be an empty dictionary.  If 'returns' is present,
+it must be the string name of a complex or built-in type, a
+one-element array containing the name of a complex or built-in type,
+with one exception noted below when 'gen' is used.  Although it is
+permitted to have the 'returns' member name a built-in type or an
+array of built-in types, any command that does this cannot be extended
+to return additional information in the future; thus, new commands
+should strongly consider returning a dictionary-based type or an array
+of dictionaries, even if the dictionary only contains one member at the
+present.
+
+All commands in Client JSON Protocol use a dictionary to report
+failure, with no way to specify that in QAPI.  Where the error return
+is different than the usual GenericError class in order to help the
+client react differently to certain error conditions, it is worth
+documenting this in the comments before the command declaration.
+
+Some example commands:
+
+ { 'command': 'my-first-command',
+   'data': { 'arg1': 'str', '*arg2': 'str' } }
+ { 'struct': 'MyType', 'data': { '*value': 'str' } }
+ { 'command': 'my-second-command',
+   'returns': [ 'MyType' ] }
+
+which would validate this Client JSON Protocol transaction:
+
+ => { "execute": "my-first-command",
+      "arguments": { "arg1": "hello" } }
+ <= { "return": { } }
+ => { "execute": "my-second-command" }
+ <= { "return": [ { "value": "one" }, { } ] }
+
+The generator emits a prototype for the user's function implementing
+the command.  Normally, 'data' is a dictionary for an anonymous type,
+or names a struct type (possibly empty, but not a union), and its
+members are passed as separate arguments to this function.  If the
+command definition includes a key 'boxed' with the boolean value true,
+then 'data' is instead the name of any non-empty complex type
+(struct, union, or alternate), and a pointer to that QAPI type is
+passed as a single argument.
+
+The generator also emits a marshalling function that extracts
+arguments for the user's function out of an input QDict, calls the
+user's function, and if it succeeded, builds an output QObject from
+its return value.
+
+In rare cases, QAPI cannot express a type-safe representation of a
+corresponding Client JSON Protocol command.  You then have to suppress
+generation of a marshalling function by including a key 'gen' with
+boolean value false, and instead write your own function.  Please try
+to avoid adding new commands that rely on this, and instead use
+type-safe unions.  For an example of this usage:
+
+ { 'command': 'netdev_add',
+   'data': {'type': 'str', 'id': 'str'},
+   'gen': false }
+
+Normally, the QAPI schema is used to describe synchronous exchanges,
+where a response is expected.  But in some cases, the action of a
+command is expected to change state in a way that a successful
+response is not possible (although the command will still return a
+normal dictionary error on failure).  When a successful reply is not
+possible, the command expression should include the optional key
+'success-response' with boolean value false.  So far, only QGA makes
+use of this member.
+
 
 === Events ===
 
-Events are defined with the keyword 'event'.  When 'data' is also specified,
-additional info will be included in the event.  Finally there will be C API
-generated in qapi-event.h; when called by QEMU code, a message with timestamp
-will be emitted on the wire.  If timestamp is -1, it means failure to retrieve
-host time.
+Usage: { 'event': STRING, '*data': COMPLEX-TYPE-NAME-OR-DICT,
+         '*boxed': true }
+
+Events are defined with the keyword 'event'.  It is not allowed to
+name an event 'MAX', since the generator also produces a C enumeration
+of all event names with a generated _MAX value at the end.  When
+'data' is also specified, additional info will be included in the
+event, with similar semantics to a 'struct' expression.  Finally there
+will be C API generated in qapi-event.h; when called by QEMU code, a
+message with timestamp will be emitted on the wire.
 
 An example event is:
 
@@ -229,35 +520,264 @@
   "data": { "b": "test string" },
   "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
 
+The generator emits a function to send the event.  Normally, 'data' is
+a dictionary for an anonymous type, or names a struct type (possibly
+empty, but not a union), and its members are passed as separate
+arguments to this function.  If the event definition includes a key
+'boxed' with the boolean value true, then 'data' is instead the name of
+any non-empty complex type (struct, union, or alternate), and a
+pointer to that QAPI type is passed as a single argument.
+
+
+== Client JSON Protocol introspection ==
+
+Clients of a Client JSON Protocol commonly need to figure out what
+exactly the server (QEMU) supports.
+
+For this purpose, QMP provides introspection via command
+query-qmp-schema.  QGA currently doesn't support introspection.
+
+While Client JSON Protocol wire compatibility should be maintained
+between qemu versions, we cannot make the same guarantees for
+introspection stability.  For example, one version of qemu may provide
+a non-variant optional member of a struct, and a later version rework
+the member to instead be non-optional and associated with a variant.
+Likewise, one version of qemu may list a member with open-ended type
+'str', and a later version could convert it to a finite set of strings
+via an enum type; or a member may be converted from a specific type to
+an alternate that represents a choice between the original type and
+something else.
+
+query-qmp-schema returns a JSON array of SchemaInfo objects.  These
+objects together describe the wire ABI, as defined in the QAPI schema.
+There is no specified order to the SchemaInfo objects returned; a
+client must search for a particular name throughout the entire array
+to learn more about that name, but is at least guaranteed that there
+will be no collisions between type, command, and event names.
+
+However, the SchemaInfo can't reflect all the rules and restrictions
+that apply to QMP.  It's interface introspection (figuring out what's
+there), not interface specification.  The specification is in the QAPI
+schema.  To understand how QMP is to be used, you need to study the
+QAPI schema.
+
+Like any other command, query-qmp-schema is itself defined in the QAPI
+schema, along with the SchemaInfo type.  This text attempts to give an
+overview how things work.  For details you need to consult the QAPI
+schema.
+
+SchemaInfo objects have common members "name" and "meta-type", and
+additional variant members depending on the value of meta-type.
+
+Each SchemaInfo object describes a wire ABI entity of a certain
+meta-type: a command, event or one of several kinds of type.
+
+SchemaInfo for commands and events have the same name as in the QAPI
+schema.
+
+Command and event names are part of the wire ABI, but type names are
+not.  Therefore, the SchemaInfo for types have auto-generated
+meaningless names.  For readability, the examples in this section use
+meaningful type names instead.
+
+To examine a type, start with a command or event using it, then follow
+references by name.
+
+QAPI schema definitions not reachable that way are omitted.
+
+The SchemaInfo for a command has meta-type "command", and variant
+members "arg-type" and "ret-type".  On the wire, the "arguments"
+member of a client's "execute" command must conform to the object type
+named by "arg-type".  The "return" member that the server passes in a
+success response conforms to the type named by "ret-type".
+
+If the command takes no arguments, "arg-type" names an object type
+without members.  Likewise, if the command returns nothing, "ret-type"
+names an object type without members.
+
+Example: the SchemaInfo for command query-qmp-schema
+
+    { "name": "query-qmp-schema", "meta-type": "command",
+      "arg-type": "q_empty", "ret-type": "SchemaInfoList" }
+
+    Type "q_empty" is an automatic object type without members, and type
+    "SchemaInfoList" is the array of SchemaInfo type.
+
+The SchemaInfo for an event has meta-type "event", and variant member
+"arg-type".  On the wire, a "data" member that the server passes in an
+event conforms to the object type named by "arg-type".
+
+If the event carries no additional information, "arg-type" names an
+object type without members.  The event may not have a data member on
+the wire then.
+
+Each command or event defined with dictionary-valued 'data' in the
+QAPI schema implicitly defines an object type.
+
+Example: the SchemaInfo for EVENT_C from section Events
+
+    { "name": "EVENT_C", "meta-type": "event",
+      "arg-type": "q_obj-EVENT_C-arg" }
+
+    Type "q_obj-EVENT_C-arg" is an implicitly defined object type with
+    the two members from the event's definition.
+
+The SchemaInfo for struct and union types has meta-type "object".
+
+The SchemaInfo for a struct type has variant member "members".
+
+The SchemaInfo for a union type additionally has variant members "tag"
+and "variants".
+
+"members" is a JSON array describing the object's common members, if
+any.  Each element is a JSON object with members "name" (the member's
+name), "type" (the name of its type), and optionally "default".  The
+member is optional if "default" is present.  Currently, "default" can
+only have value null.  Other values are reserved for future
+extensions.  The "members" array is in no particular order; clients
+must search the entire object when learning whether a particular
+member is supported.
+
+Example: the SchemaInfo for MyType from section Struct types
+
+    { "name": "MyType", "meta-type": "object",
+      "members": [
+          { "name": "member1", "type": "str" },
+          { "name": "member2", "type": "int" },
+          { "name": "member3", "type": "str", "default": null } ] }
+
+"tag" is the name of the common member serving as type tag.
+"variants" is a JSON array describing the object's variant members.
+Each element is a JSON object with members "case" (the value of type
+tag this element applies to) and "type" (the name of an object type
+that provides the variant members for this type tag value).  The
+"variants" array is in no particular order, and is not guaranteed to
+list cases in the same order as the corresponding "tag" enum type.
+
+Example: the SchemaInfo for flat union BlockdevOptions from section
+Union types
+
+    { "name": "BlockdevOptions", "meta-type": "object",
+      "members": [
+          { "name": "driver", "type": "BlockdevDriver" },
+          { "name": "read-only", "type": "bool", "default": null } ],
+      "tag": "driver",
+      "variants": [
+          { "case": "file", "type": "BlockdevOptionsFile" },
+          { "case": "qcow2", "type": "BlockdevOptionsQcow2" } ] }
+
+Note that base types are "flattened": its members are included in the
+"members" array.
+
+A simple union implicitly defines an enumeration type for its implicit
+discriminator (called "type" on the wire, see section Union types).
+
+A simple union implicitly defines an object type for each of its
+variants.
+
+Example: the SchemaInfo for simple union BlockdevOptionsSimple from section
+Union types
+
+    { "name": "BlockdevOptionsSimple", "meta-type": "object",
+      "members": [
+          { "name": "type", "type": "BlockdevOptionsSimpleKind" } ],
+      "tag": "type",
+      "variants": [
+          { "case": "file", "type": "q_obj-BlockdevOptionsFile-wrapper" },
+          { "case": "qcow2", "type": "q_obj-BlockdevOptionsQcow2-wrapper" } ] }
+
+    Enumeration type "BlockdevOptionsSimpleKind" and the object types
+    "q_obj-BlockdevOptionsFile-wrapper", "q_obj-BlockdevOptionsQcow2-wrapper"
+    are implicitly defined.
+
+The SchemaInfo for an alternate type has meta-type "alternate", and
+variant member "members".  "members" is a JSON array.  Each element is
+a JSON object with member "type", which names a type.  Values of the
+alternate type conform to exactly one of its member types.  There is
+no guarantee on the order in which "members" will be listed.
+
+Example: the SchemaInfo for BlockdevRef from section Alternate types
+
+    { "name": "BlockdevRef", "meta-type": "alternate",
+      "members": [
+          { "type": "BlockdevOptions" },
+          { "type": "str" } ] }
+
+The SchemaInfo for an array type has meta-type "array", and variant
+member "element-type", which names the array's element type.  Array
+types are implicitly defined.  For convenience, the array's name may
+resemble the element type; however, clients should examine member
+"element-type" instead of making assumptions based on parsing member
+"name".
+
+Example: the SchemaInfo for ['str']
+
+    { "name": "[str]", "meta-type": "array",
+      "element-type": "str" }
+
+The SchemaInfo for an enumeration type has meta-type "enum" and
+variant member "values".  The values are listed in no particular
+order; clients must search the entire enum when learning whether a
+particular value is supported.
+
+Example: the SchemaInfo for MyEnum from section Enumeration types
+
+    { "name": "MyEnum", "meta-type": "enum",
+      "values": [ "value1", "value2", "value3" ] }
+
+The SchemaInfo for a built-in type has the same name as the type in
+the QAPI schema (see section Built-in Types), with one exception
+detailed below.  It has variant member "json-type" that shows how
+values of this type are encoded on the wire.
+
+Example: the SchemaInfo for str
+
+    { "name": "str", "meta-type": "builtin", "json-type": "string" }
+
+The QAPI schema supports a number of integer types that only differ in
+how they map to C.  They are identical as far as SchemaInfo is
+concerned.  Therefore, they get all mapped to a single type "int" in
+SchemaInfo.
+
+As explained above, type names are not part of the wire ABI.  Not even
+the names of built-in types.  Clients should examine member
+"json-type" instead of hard-coding names of built-in types.
+
 
 == Code generation ==
 
-Schemas are fed into 3 scripts to generate all the code/files that, paired
-with the core QAPI libraries, comprise everything required to take JSON
-commands read in by a QMP/guest agent server, unmarshal the arguments into
-the underlying C types, call into the corresponding C function, and map the
-response back to a QMP/guest agent response to be returned to the user.
+Schemas are fed into five scripts to generate all the code/files that,
+paired with the core QAPI libraries, comprise everything required to
+take JSON commands read in by a Client JSON Protocol server, unmarshal
+the arguments into the underlying C types, call into the corresponding
+C function, map the response back to a Client JSON Protocol response
+to be returned to the user, and introspect the commands.
 
-As an example, we'll use the following schema, which describes a single
-complex user-defined type (which will produce a C struct, along with a list
-node structure that can be used to chain together a list of such types in
-case we want to accept/return a list of this type with a command), and a
-command which takes that type as a parameter and returns the same type:
+As an example, we'll use the following schema, which describes a
+single complex user-defined type, along with command which takes a
+list of that type as a parameter, and returns a single element of that
+type.  The user is responsible for writing the implementation of
+qmp_my_command(); everything else is produced by the generator.
 
     $ cat example-schema.json
-    { 'type': 'UserDefOne',
-      'data': { 'integer': 'int', 'string': 'str' } }
+    { 'struct': 'UserDefOne',
+      'data': { 'integer': 'int', '*string': 'str' } }
 
     { 'command': 'my-command',
-      'data':    {'arg1': 'UserDefOne'},
+      'data': { 'arg1': ['UserDefOne'] },
       'returns': 'UserDefOne' }
 
     { 'event': 'MY_EVENT' }
 
+For a more thorough look at generated code, the testsuite includes
+tests/qapi-schema/qapi-schema-tests.json that covers more examples of
+what the generator will accept, and compiles the resulting C code as
+part of 'make check-unit'.
+
 === scripts/qapi-types.py ===
 
-Used to generate the C types defined by a schema. The following files are
-created:
+Used to generate the C types defined by a schema, along with
+supporting code. The following files are created:
 
 $(prefix)qapi-types.h - C types corresponding to types defined in
                         the schema you pass in
@@ -271,78 +791,70 @@
 Example:
 
     $ python scripts/qapi-types.py --output-dir="qapi-generated" \
-    --prefix="example-" --input-file=example-schema.json
-    $ cat qapi-generated/example-qapi-types.c
-[Uninteresting stuff omitted...]
-
-    void qapi_free_UserDefOneList(UserDefOneList *obj)
-    {
-        QapiDeallocVisitor *md;
-        Visitor *v;
-
-        if (!obj) {
-            return;
-        }
-
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOneList(v, &obj, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(md);
-    }
-
-    void qapi_free_UserDefOne(UserDefOne *obj)
-    {
-        QapiDeallocVisitor *md;
-        Visitor *v;
-
-        if (!obj) {
-            return;
-        }
-
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &obj, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(md);
-    }
-
+    --prefix="example-" example-schema.json
     $ cat qapi-generated/example-qapi-types.h
 [Uninteresting stuff omitted...]
 
     #ifndef EXAMPLE_QAPI_TYPES_H
     #define EXAMPLE_QAPI_TYPES_H
 
-[Builtin types omitted...]
+[Built-in types omitted...]
 
     typedef struct UserDefOne UserDefOne;
 
-    typedef struct UserDefOneList
-    {
-        union {
-            UserDefOne *value;
-            uint64_t padding;
-        };
-        struct UserDefOneList *next;
-    } UserDefOneList;
+    typedef struct UserDefOneList UserDefOneList;
 
-[Functions on builtin types omitted...]
-
-    struct UserDefOne
-    {
+    struct UserDefOne {
         int64_t integer;
+        bool has_string;
         char *string;
     };
 
-    void qapi_free_UserDefOneList(UserDefOneList *obj);
     void qapi_free_UserDefOne(UserDefOne *obj);
 
+    struct UserDefOneList {
+        UserDefOneList *next;
+        UserDefOne *value;
+    };
+
+    void qapi_free_UserDefOneList(UserDefOneList *obj);
+
     #endif
+    $ cat qapi-generated/example-qapi-types.c
+[Uninteresting stuff omitted...]
+
+    void qapi_free_UserDefOne(UserDefOne *obj)
+    {
+        Visitor *v;
+
+        if (!obj) {
+            return;
+        }
+
+        v = qapi_dealloc_visitor_new();
+        visit_type_UserDefOne(v, NULL, &obj, NULL);
+        visit_free(v);
+    }
+
+    void qapi_free_UserDefOneList(UserDefOneList *obj)
+    {
+        Visitor *v;
+
+        if (!obj) {
+            return;
+        }
+
+        v = qapi_dealloc_visitor_new();
+        visit_type_UserDefOneList(v, NULL, &obj, NULL);
+        visit_free(v);
+    }
 
 === scripts/qapi-visit.py ===
 
-Used to generate the visitor functions used to walk through and convert
-a QObject (as provided by QMP) to a native C data structure and
-vice-versa, as well as the visitor function used to dealloc a complex
-schema-defined C type.
+Used to generate the visitor functions used to walk through and
+convert between a native QAPI C data structure and some other format
+(such as QObject); the generated functions are named visit_type_FOO()
+and visit_type_FOO_members().
 
 The following files are generated:
 
@@ -358,82 +870,103 @@
 Example:
 
     $ python scripts/qapi-visit.py --output-dir="qapi-generated"
-    --prefix="example-" --input-file=example-schema.json
-    $ cat qapi-generated/example-qapi-visit.c
-[Uninteresting stuff omitted...]
-
-    static void visit_type_UserDefOne_fields(Visitor *m, UserDefOne **obj, Error **errp)
-    {
-        Error *err = NULL;
-        visit_type_int(m, &(*obj)->integer, "integer", &err);
-        if (err) {
-            goto out;
-        }
-        visit_type_str(m, &(*obj)->string, "string", &err);
-        if (err) {
-            goto out;
-        }
-
-    out:
-        error_propagate(errp, err);
-    }
-
-    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp)
-    {
-        Error *err = NULL;
-
-        visit_start_struct(m, (void **)obj, "UserDefOne", name, sizeof(UserDefOne), &err);
-        if (!err) {
-            if (*obj) {
-                visit_type_UserDefOne_fields(m, obj, errp);
-            }
-            visit_end_struct(m, &err);
-        }
-        error_propagate(errp, err);
-    }
-
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp)
-    {
-        Error *err = NULL;
-        GenericList *i, **prev;
-
-        visit_start_list(m, name, &err);
-        if (err) {
-            goto out;
-        }
-
-        for (prev = (GenericList **)obj;
-             !err && (i = visit_next_list(m, prev, &err)) != NULL;
-             prev = &i) {
-            UserDefOneList *native_i = (UserDefOneList *)i;
-            visit_type_UserDefOne(m, &native_i->value, NULL, &err);
-        }
-
-        error_propagate(errp, err);
-        err = NULL;
-        visit_end_list(m, &err);
-    out:
-        error_propagate(errp, err);
-    }
-    $ python scripts/qapi-commands.py --output-dir="qapi-generated" \
-    --prefix="example-" --input-file=example-schema.json
+    --prefix="example-" example-schema.json
     $ cat qapi-generated/example-qapi-visit.h
 [Uninteresting stuff omitted...]
 
     #ifndef EXAMPLE_QAPI_VISIT_H
     #define EXAMPLE_QAPI_VISIT_H
 
-[Visitors for builtin types omitted...]
+[Visitors for built-in types omitted...]
 
-    void visit_type_UserDefOne(Visitor *m, UserDefOne **obj, const char *name, Error **errp);
-    void visit_type_UserDefOneList(Visitor *m, UserDefOneList **obj, const char *name, Error **errp);
+    void visit_type_UserDefOne_members(Visitor *v, UserDefOne *obj, Error **errp);
+    void visit_type_UserDefOne(Visitor *v, const char *name, UserDefOne **obj, Error **errp);
+    void visit_type_UserDefOneList(Visitor *v, const char *name, UserDefOneList **obj, Error **errp);
 
     #endif
+    $ cat qapi-generated/example-qapi-visit.c
+[Uninteresting stuff omitted...]
+
+    void visit_type_UserDefOne_members(Visitor *v, UserDefOne *obj, Error **errp)
+    {
+        Error *err = NULL;
+
+        visit_type_int(v, "integer", &obj->integer, &err);
+        if (err) {
+            goto out;
+        }
+        if (visit_optional(v, "string", &obj->has_string)) {
+            visit_type_str(v, "string", &obj->string, &err);
+            if (err) {
+                goto out;
+            }
+        }
+
+    out:
+        error_propagate(errp, err);
+    }
+
+    void visit_type_UserDefOne(Visitor *v, const char *name, UserDefOne **obj, Error **errp)
+    {
+        Error *err = NULL;
+
+        visit_start_struct(v, name, (void **)obj, sizeof(UserDefOne), &err);
+        if (err) {
+            goto out;
+        }
+        if (!*obj) {
+            goto out_obj;
+        }
+        visit_type_UserDefOne_members(v, *obj, &err);
+        if (err) {
+            goto out_obj;
+        }
+        visit_check_struct(v, &err);
+    out_obj:
+        visit_end_struct(v, (void **)obj);
+        if (err && visit_is_input(v)) {
+            qapi_free_UserDefOne(*obj);
+            *obj = NULL;
+        }
+    out:
+        error_propagate(errp, err);
+    }
+
+    void visit_type_UserDefOneList(Visitor *v, const char *name, UserDefOneList **obj, Error **errp)
+    {
+        Error *err = NULL;
+        UserDefOneList *tail;
+        size_t size = sizeof(**obj);
+
+        visit_start_list(v, name, (GenericList **)obj, size, &err);
+        if (err) {
+            goto out;
+        }
+
+        for (tail = *obj; tail;
+             tail = (UserDefOneList *)visit_next_list(v, (GenericList *)tail, size)) {
+            visit_type_UserDefOne(v, NULL, &tail->value, &err);
+            if (err) {
+                break;
+            }
+        }
+
+        visit_end_list(v, (void **)obj);
+        if (err && visit_is_input(v)) {
+            qapi_free_UserDefOneList(*obj);
+            *obj = NULL;
+        }
+    out:
+        error_propagate(errp, err);
+    }
 
 === scripts/qapi-commands.py ===
 
-Used to generate the marshaling/dispatch functions for the commands defined
-in the schema. The following files are generated:
+Used to generate the marshaling/dispatch functions for the commands
+defined in the schema. The generated code implements
+qmp_marshal_COMMAND() (mentioned in qmp-commands.hx, and registered
+automatically), and declares qmp_COMMAND() that the user must
+implement.  The following files are generated:
 
 $(prefix)qmp-marshal.c: command marshal/dispatch functions for each
                         QMP command defined in the schema. Functions
@@ -450,71 +983,7 @@
 Example:
 
     $ python scripts/qapi-commands.py --output-dir="qapi-generated"
-    --prefix="example-" --input-file=example-schema.json
-    $ cat qapi-generated/example-qmp-marshal.c
-[Uninteresting stuff omitted...]
-
-    static void qmp_marshal_output_my_command(UserDefOne *ret_in, QObject **ret_out, Error **errp)
-    {
-        Error *local_err = NULL;
-        QmpOutputVisitor *mo = qmp_output_visitor_new();
-        QapiDeallocVisitor *md;
-        Visitor *v;
-
-        v = qmp_output_get_visitor(mo);
-        visit_type_UserDefOne(v, &ret_in, "unused", &local_err);
-        if (local_err) {
-            goto out;
-        }
-        *ret_out = qmp_output_get_qobject(mo);
-
-    out:
-        error_propagate(errp, local_err);
-        qmp_output_visitor_cleanup(mo);
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &ret_in, "unused", NULL);
-        qapi_dealloc_visitor_cleanup(md);
-    }
-
-    static void qmp_marshal_input_my_command(QDict *args, QObject **ret, Error **errp)
-    {
-        Error *local_err = NULL;
-        UserDefOne *retval = NULL;
-        QmpInputVisitor *mi = qmp_input_visitor_new_strict(QOBJECT(args));
-        QapiDeallocVisitor *md;
-        Visitor *v;
-        UserDefOne *arg1 = NULL;
-
-        v = qmp_input_get_visitor(mi);
-        visit_type_UserDefOne(v, &arg1, "arg1", &local_err);
-        if (local_err) {
-            goto out;
-        }
-
-        retval = qmp_my_command(arg1, &local_err);
-        if (local_err) {
-            goto out;
-        }
-
-        qmp_marshal_output_my_command(retval, ret, &local_err);
-
-    out:
-        error_propagate(errp, local_err);
-        qmp_input_visitor_cleanup(mi);
-        md = qapi_dealloc_visitor_new();
-        v = qapi_dealloc_get_visitor(md);
-        visit_type_UserDefOne(v, &arg1, "arg1", NULL);
-        qapi_dealloc_visitor_cleanup(md);
-        return;
-    }
-
-    static void qmp_init_marshal(void)
-    {
-        qmp_register_command("my-command", qmp_marshal_input_my_command, QCO_NO_OPTIONS);
-    }
-
-    qapi_init(qmp_init_marshal);
+    --prefix="example-" example-schema.json
     $ cat qapi-generated/example-qmp-commands.h
 [Uninteresting stuff omitted...]
 
@@ -525,14 +994,79 @@
     #include "qapi/qmp/qdict.h"
     #include "qapi/error.h"
 
-    UserDefOne *qmp_my_command(UserDefOne *arg1, Error **errp);
+    UserDefOne *qmp_my_command(UserDefOneList *arg1, Error **errp);
 
     #endif
+    $ cat qapi-generated/example-qmp-marshal.c
+[Uninteresting stuff omitted...]
+
+    static void qmp_marshal_output_UserDefOne(UserDefOne *ret_in, QObject **ret_out, Error **errp)
+    {
+        Error *err = NULL;
+        Visitor *v;
+
+        v = qmp_output_visitor_new(ret_out);
+        visit_type_UserDefOne(v, "unused", &ret_in, &err);
+        if (!err) {
+            visit_complete(v, ret_out);
+        }
+        error_propagate(errp, err);
+        visit_free(v);
+        v = qapi_dealloc_visitor_new();
+        visit_type_UserDefOne(v, "unused", &ret_in, NULL);
+        visit_free(v);
+    }
+
+    static void qmp_marshal_my_command(QDict *args, QObject **ret, Error **errp)
+    {
+        Error *err = NULL;
+        UserDefOne *retval;
+        Visitor *v;
+        UserDefOneList *arg1 = NULL;
+
+        v = qmp_input_visitor_new(QOBJECT(args), true);
+        visit_start_struct(v, NULL, NULL, 0, &err);
+        if (err) {
+            goto out;
+        }
+        visit_type_UserDefOneList(v, "arg1", &arg1, &err);
+        if (!err) {
+            visit_check_struct(v, &err);
+        }
+        visit_end_struct(v, NULL);
+        if (err) {
+            goto out;
+        }
+
+        retval = qmp_my_command(arg1, &err);
+        if (err) {
+            goto out;
+        }
+
+        qmp_marshal_output_UserDefOne(retval, ret, &err);
+
+    out:
+        error_propagate(errp, err);
+        visit_free(v);
+        v = qapi_dealloc_visitor_new();
+        visit_start_struct(v, NULL, NULL, 0, NULL);
+        visit_type_UserDefOneList(v, "arg1", &arg1, NULL);
+        visit_end_struct(v, NULL);
+        visit_free(v);
+    }
+
+    static void qmp_init_marshal(void)
+    {
+        qmp_register_command("my-command", qmp_marshal_my_command, QCO_NO_OPTIONS);
+    }
+
+    qapi_init(qmp_init_marshal);
 
 === scripts/qapi-event.py ===
 
-Used to generate the event-related C code defined by a schema. The
-following files are created:
+Used to generate the event-related C code defined by a schema, with
+implementations for qapi_event_send_FOO(). The following files are
+created:
 
 $(prefix)qapi-event.h - Function prototypes for each event type, plus an
                         enumeration of all event names
@@ -541,32 +1075,7 @@
 Example:
 
     $ python scripts/qapi-event.py --output-dir="qapi-generated"
-    --prefix="example-" --input-file=example-schema.json
-    $ cat qapi-generated/example-qapi-event.c
-[Uninteresting stuff omitted...]
-
-    void qapi_event_send_my_event(Error **errp)
-    {
-        QDict *qmp;
-        Error *local_err = NULL;
-        QMPEventFuncEmit emit;
-        emit = qmp_event_get_func_emit();
-        if (!emit) {
-            return;
-        }
-
-        qmp = qmp_event_build_dict("MY_EVENT");
-
-        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &local_err);
-
-        error_propagate(errp, local_err);
-        QDECREF(qmp);
-    }
-
-    const char *EXAMPLE_QAPIEvent_lookup[] = {
-        "MY_EVENT",
-        NULL,
-    };
+    --prefix="example-" example-schema.json
     $ cat qapi-generated/example-qapi-event.h
 [Uninteresting stuff omitted...]
 
@@ -580,11 +1089,71 @@
 
     void qapi_event_send_my_event(Error **errp);
 
-    extern const char *EXAMPLE_QAPIEvent_lookup[];
-    typedef enum EXAMPLE_QAPIEvent
-    {
+    typedef enum example_QAPIEvent {
         EXAMPLE_QAPI_EVENT_MY_EVENT = 0,
-        EXAMPLE_QAPI_EVENT_MAX = 1,
-    } EXAMPLE_QAPIEvent;
+        EXAMPLE_QAPI_EVENT__MAX = 1,
+    } example_QAPIEvent;
+
+    extern const char *const example_QAPIEvent_lookup[];
 
     #endif
+    $ cat qapi-generated/example-qapi-event.c
+[Uninteresting stuff omitted...]
+
+    void qapi_event_send_my_event(Error **errp)
+    {
+        QDict *qmp;
+        Error *err = NULL;
+        QMPEventFuncEmit emit;
+        emit = qmp_event_get_func_emit();
+        if (!emit) {
+            return;
+        }
+
+        qmp = qmp_event_build_dict("MY_EVENT");
+
+        emit(EXAMPLE_QAPI_EVENT_MY_EVENT, qmp, &err);
+
+        error_propagate(errp, err);
+        QDECREF(qmp);
+    }
+
+    const char *const example_QAPIEvent_lookup[] = {
+        [EXAMPLE_QAPI_EVENT_MY_EVENT] = "MY_EVENT",
+        [EXAMPLE_QAPI_EVENT__MAX] = NULL,
+    };
+
+=== scripts/qapi-introspect.py ===
+
+Used to generate the introspection C code for a schema. The following
+files are created:
+
+$(prefix)qmp-introspect.c - Defines a string holding a JSON
+                            description of the schema.
+$(prefix)qmp-introspect.h - Declares the above string.
+
+Example:
+
+    $ python scripts/qapi-introspect.py --output-dir="qapi-generated"
+    --prefix="example-" example-schema.json
+    $ cat qapi-generated/example-qmp-introspect.h
+[Uninteresting stuff omitted...]
+
+    #ifndef EXAMPLE_QMP_INTROSPECT_H
+    #define EXAMPLE_QMP_INTROSPECT_H
+
+    extern const char example_qmp_schema_json[];
+
+    #endif
+    $ cat qapi-generated/example-qmp-introspect.c
+[Uninteresting stuff omitted...]
+
+    const char example_qmp_schema_json[] = "["
+        "{\"arg-type\": \"0\", \"meta-type\": \"event\", \"name\": \"MY_EVENT\"}, "
+        "{\"arg-type\": \"1\", \"meta-type\": \"command\", \"name\": \"my-command\", \"ret-type\": \"2\"}, "
+        "{\"members\": [], \"meta-type\": \"object\", \"name\": \"0\"}, "
+        "{\"members\": [{\"name\": \"arg1\", \"type\": \"[2]\"}], \"meta-type\": \"object\", \"name\": \"1\"}, "
+        "{\"members\": [{\"name\": \"integer\", \"type\": \"int\"}, {\"default\": null, \"name\": \"string\", \"type\": \"str\"}], \"meta-type\": \"object\", \"name\": \"2\"}, "
+        "{\"element-type\": \"2\", \"meta-type\": \"array\", \"name\": \"[2]\"}, "
+        "{\"json-type\": \"int\", \"meta-type\": \"builtin\", \"name\": \"int\"}, "
+        "{\"json-type\": \"string\", \"meta-type\": \"builtin\", \"name\": \"str\"}]";
diff --git a/docs/qcow2-cache.txt b/docs/qcow2-cache.txt
new file mode 100644
index 0000000..5bb0607
--- /dev/null
+++ b/docs/qcow2-cache.txt
@@ -0,0 +1,164 @@
+qcow2 L2/refcount cache configuration
+=====================================
+Copyright (C) 2015 Igalia, S.L.
+Author: Alberto Garcia <berto@igalia.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+Introduction
+------------
+The QEMU qcow2 driver has two caches that can improve the I/O
+performance significantly. However, setting the right cache sizes is
+not a straightforward operation.
+
+This document attempts to give an overview of the L2 and refcount
+caches, and how to configure them.
+
+Please refer to the docs/specs/qcow2.txt file for an in-depth
+technical description of the qcow2 file format.
+
+
+Clusters
+--------
+A qcow2 file is organized in units of constant size called clusters.
+
+The cluster size is configurable, but it must be a power of two and
+its value 512 bytes or higher. QEMU currently defaults to 64 KB
+clusters, and it does not support sizes larger than 2MB.
+
+The 'qemu-img create' command supports specifying the size using the
+cluster_size option:
+
+   qemu-img create -f qcow2 -o cluster_size=128K hd.qcow2 4G
+
+
+The L2 tables
+-------------
+The qcow2 format uses a two-level structure to map the virtual disk as
+seen by the guest to the disk image in the host. These structures are
+called the L1 and L2 tables.
+
+There is one single L1 table per disk image. The table is small and is
+always kept in memory.
+
+There can be many L2 tables, depending on how much space has been
+allocated in the image. Each table is one cluster in size. In order to
+read or write data from the virtual disk, QEMU needs to read its
+corresponding L2 table to find out where that data is located. Since
+reading the table for each I/O operation can be expensive, QEMU keeps
+an L2 cache in memory to speed up disk access.
+
+The size of the L2 cache can be configured, and setting the right
+value can improve the I/O performance significantly.
+
+
+The refcount blocks
+-------------------
+The qcow2 format also mantains a reference count for each cluster.
+Reference counts are used for cluster allocation and internal
+snapshots. The data is stored in a two-level structure similar to the
+L1/L2 tables described above.
+
+The second level structures are called refcount blocks, are also one
+cluster in size and the number is also variable and dependent on the
+amount of allocated space.
+
+Each block contains a number of refcount entries. Their size (in bits)
+is a power of two and must not be higher than 64. It defaults to 16
+bits, but a different value can be set using the refcount_bits option:
+
+   qemu-img create -f qcow2 -o refcount_bits=8 hd.qcow2 4G
+
+QEMU keeps a refcount cache to speed up I/O much like the
+aforementioned L2 cache, and its size can also be configured.
+
+
+Choosing the right cache sizes
+------------------------------
+In order to choose the cache sizes we need to know how they relate to
+the amount of allocated space.
+
+The amount of virtual disk that can be mapped by the L2 and refcount
+caches (in bytes) is:
+
+   disk_size = l2_cache_size * cluster_size / 8
+   disk_size = refcount_cache_size * cluster_size * 8 / refcount_bits
+
+With the default values for cluster_size (64KB) and refcount_bits
+(16), that is
+
+   disk_size = l2_cache_size * 8192
+   disk_size = refcount_cache_size * 32768
+
+So in order to cover n GB of disk space with the default values we
+need:
+
+   l2_cache_size = disk_size_GB * 131072
+   refcount_cache_size = disk_size_GB * 32768
+
+QEMU has a default L2 cache of 1MB (1048576 bytes) and a refcount
+cache of 256KB (262144 bytes), so using the formulas we've just seen
+we have
+
+   1048576 / 131072 = 8 GB of virtual disk covered by that cache
+    262144 /  32768 = 8 GB
+
+
+How to configure the cache sizes
+--------------------------------
+Cache sizes can be configured using the -drive option in the
+command-line, or the 'blockdev-add' QMP command.
+
+There are three options available, and all of them take bytes:
+
+"l2-cache-size":         maximum size of the L2 table cache
+"refcount-cache-size":   maximum size of the refcount block cache
+"cache-size":            maximum size of both caches combined
+
+There are two things that need to be taken into account:
+
+ - Both caches must have a size that is a multiple of the cluster
+   size.
+
+ - If you only set one of the options above, QEMU will automatically
+   adjust the others so that the L2 cache is 4 times bigger than the
+   refcount cache.
+
+This means that these options are equivalent:
+
+   -drive file=hd.qcow2,l2-cache-size=2097152
+   -drive file=hd.qcow2,refcount-cache-size=524288
+   -drive file=hd.qcow2,cache-size=2621440
+
+The reason for this 1/4 ratio is to ensure that both caches cover the
+same amount of disk space. Note however that this is only valid with
+the default value of refcount_bits (16). If you are using a different
+value you might want to calculate both cache sizes yourself since QEMU
+will always use the same 1/4 ratio.
+
+It's also worth mentioning that there's no strict need for both caches
+to cover the same amount of disk space. The refcount cache is used
+much less often than the L2 cache, so it's perfectly reasonable to
+keep it small.
+
+
+Reducing the memory usage
+-------------------------
+It is possible to clean unused cache entries in order to reduce the
+memory usage during periods of low I/O activity.
+
+The parameter "cache-clean-interval" defines an interval (in seconds).
+All cache entries that haven't been accessed during that interval are
+removed from memory.
+
+This example removes all unused cache entries every 15 minutes:
+
+   -drive file=hd.qcow2,cache-clean-interval=900
+
+If unset, the default value for this parameter is 0 and it disables
+this feature.
+
+Note that this functionality currently relies on the MADV_DONTNEED
+argument for madvise() to actually free the memory, so it is not
+useful in systems that don't follow that behavior.
diff --git a/docs/qmp-events.txt b/docs/qmp-events.txt
new file mode 100644
index 0000000..7967ec4
--- /dev/null
+++ b/docs/qmp-events.txt
@@ -0,0 +1,721 @@
+                   QEMU Machine Protocol Events
+                   ============================
+
+ACPI_DEVICE_OST
+---------------
+
+Emitted when guest executes ACPI _OST method.
+
+ - data: ACPIOSTInfo type as described in qapi-schema.json
+
+{ "event": "ACPI_DEVICE_OST",
+     "data": { "device": "d1", "slot": "0", "slot-type": "DIMM", "source": 1, "status": 0 } }
+
+BALLOON_CHANGE
+--------------
+
+Emitted when the guest changes the actual BALLOON level. This
+value is equivalent to the 'actual' field return by the
+'query-balloon' command
+
+Data:
+
+- "actual": actual level of the guest memory balloon in bytes (json-number)
+
+Example:
+
+{ "event": "BALLOON_CHANGE",
+    "data": { "actual": 944766976 },
+    "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
+
+Note: this event is rate-limited.
+
+BLOCK_IMAGE_CORRUPTED
+---------------------
+
+Emitted when a disk image is being marked corrupt. The image can be
+identified by its device or node name. The 'device' field is always
+present for compatibility reasons, but it can be empty ("") if the
+image does not have a device name associated.
+
+Data:
+
+- "device":    Device name (json-string)
+- "node-name": Node name (json-string, optional)
+- "msg":       Informative message (e.g., reason for the corruption)
+               (json-string)
+- "offset":    If the corruption resulted from an image access, this
+               is the host's access offset into the image
+               (json-int, optional)
+- "size":      If the corruption resulted from an image access, this
+               is the access size (json-int, optional)
+
+Example:
+
+{ "event": "BLOCK_IMAGE_CORRUPTED",
+    "data": { "device": "ide0-hd0", "node-name": "node0",
+        "msg": "Prevented active L1 table overwrite", "offset": 196608,
+        "size": 65536 },
+    "timestamp": { "seconds": 1378126126, "microseconds": 966463 } }
+
+BLOCK_IO_ERROR
+--------------
+
+Emitted when a disk I/O error occurs.
+
+Data:
+
+- "device": device name (json-string)
+- "operation": I/O operation (json-string, "read" or "write")
+- "action": action that has been taken, it's one of the following (json-string):
+    "ignore": error has been ignored
+    "report": error has been reported to the device
+    "stop": the VM is going to stop because of the error
+
+Example:
+
+{ "event": "BLOCK_IO_ERROR",
+    "data": { "device": "ide0-hd1",
+              "operation": "write",
+              "action": "stop" },
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+Note: If action is "stop", a STOP event will eventually follow the
+BLOCK_IO_ERROR event.
+
+BLOCK_JOB_CANCELLED
+-------------------
+
+Emitted when a block job has been cancelled.
+
+Data:
+
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
+- "device":   Job identifier. Originally the device name but other
+              values are allowed since QEMU 2.7 (json-string)
+- "len":      Maximum progress value (json-int)
+- "offset":   Current progress value (json-int)
+              On success this is equal to len.
+              On failure this is less than len.
+- "speed":    Rate limit, bytes per second (json-int)
+
+Example:
+
+{ "event": "BLOCK_JOB_CANCELLED",
+     "data": { "type": "stream", "device": "virtio-disk0",
+               "len": 10737418240, "offset": 134217728,
+               "speed": 0 },
+     "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
+
+BLOCK_JOB_COMPLETED
+-------------------
+
+Emitted when a block job has completed.
+
+Data:
+
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
+- "device":   Job identifier. Originally the device name but other
+              values are allowed since QEMU 2.7 (json-string)
+- "len":      Maximum progress value (json-int)
+- "offset":   Current progress value (json-int)
+              On success this is equal to len.
+              On failure this is less than len.
+- "speed":    Rate limit, bytes per second (json-int)
+- "error":    Error message (json-string, optional)
+              Only present on failure.  This field contains a human-readable
+              error message.  There are no semantics other than that streaming
+              has failed and clients should not try to interpret the error
+              string.
+
+Example:
+
+{ "event": "BLOCK_JOB_COMPLETED",
+     "data": { "type": "stream", "device": "virtio-disk0",
+               "len": 10737418240, "offset": 10737418240,
+               "speed": 0 },
+     "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
+
+BLOCK_JOB_ERROR
+---------------
+
+Emitted when a block job encounters an error.
+
+Data:
+
+- "device": Job identifier. Originally the device name but other
+            values are allowed since QEMU 2.7 (json-string)
+- "operation": I/O operation (json-string, "read" or "write")
+- "action": action that has been taken, it's one of the following (json-string):
+    "ignore": error has been ignored, the job may fail later
+    "report": error will be reported and the job canceled
+    "stop": error caused job to be paused
+
+Example:
+
+{ "event": "BLOCK_JOB_ERROR",
+    "data": { "device": "ide0-hd1",
+              "operation": "write",
+              "action": "stop" },
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+BLOCK_JOB_READY
+---------------
+
+Emitted when a block job is ready to complete.
+
+Data:
+
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
+- "device":   Job identifier. Originally the device name but other
+              values are allowed since QEMU 2.7 (json-string)
+- "len":      Maximum progress value (json-int)
+- "offset":   Current progress value (json-int)
+              On success this is equal to len.
+              On failure this is less than len.
+- "speed":    Rate limit, bytes per second (json-int)
+
+Example:
+
+{ "event": "BLOCK_JOB_READY",
+    "data": { "device": "drive0", "type": "mirror", "speed": 0,
+              "len": 2097152, "offset": 2097152 }
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+Note: The "ready to complete" status is always reset by a BLOCK_JOB_ERROR
+event.
+
+DEVICE_DELETED
+--------------
+
+Emitted whenever the device removal completion is acknowledged
+by the guest.
+At this point, it's safe to reuse the specified device ID.
+Device removal can be initiated by the guest or by HMP/QMP commands.
+
+Data:
+
+- "device": device name (json-string, optional)
+- "path": device path (json-string)
+
+{ "event": "DEVICE_DELETED",
+  "data": { "device": "virtio-net-pci-0",
+            "path": "/machine/peripheral/virtio-net-pci-0" },
+  "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+DEVICE_TRAY_MOVED
+-----------------
+
+It's emitted whenever the tray of a removable device is moved by the guest
+or by HMP/QMP commands.
+
+Data:
+
+- "device": device name (json-string)
+- "tray-open": true if the tray has been opened or false if it has been closed
+               (json-bool)
+
+{ "event": "DEVICE_TRAY_MOVED",
+  "data": { "device": "ide1-cd0",
+            "tray-open": true
+  },
+  "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+DUMP_COMPLETED
+--------------
+
+Emitted when the guest has finished one memory dump.
+
+Data:
+
+- "result": DumpQueryResult type described in qapi-schema.json
+- "error": Error message when dump failed. This is only a
+  human-readable string provided when dump failed. It should not be
+  parsed in any way (json-string, optional)
+
+Example:
+
+{ "event": "DUMP_COMPLETED",
+  "data": {"result": {"total": 1090650112, "status": "completed",
+                      "completed": 1090650112} } }
+
+GUEST_PANICKED
+--------------
+
+Emitted when guest OS panic is detected.
+
+Data:
+
+- "action": Action that has been taken (json-string, currently always "pause").
+
+Example:
+
+{ "event": "GUEST_PANICKED",
+     "data": { "action": "pause" } }
+
+MEM_UNPLUG_ERROR
+--------------------
+Emitted when memory hot unplug error occurs.
+
+Data:
+
+- "device": device name (json-string)
+- "msg": Informative message (e.g., reason for the error) (json-string)
+
+Example:
+
+{ "event": "MEM_UNPLUG_ERROR"
+  "data": { "device": "dimm1",
+            "msg": "acpi: device unplug for unsupported device"
+  },
+  "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+NIC_RX_FILTER_CHANGED
+---------------------
+
+The event is emitted once until the query command is executed,
+the first event will always be emitted.
+
+Data:
+
+- "name": net client name (json-string)
+- "path": device path (json-string)
+
+{ "event": "NIC_RX_FILTER_CHANGED",
+  "data": { "name": "vnet0",
+            "path": "/machine/peripheral/vnet0/virtio-backend" },
+  "timestamp": { "seconds": 1368697518, "microseconds": 326866 } }
+}
+
+POWERDOWN
+---------
+
+Emitted when the Virtual Machine is powered down through the power
+control system, such as via ACPI.
+
+Data: None.
+
+Example:
+
+{ "event": "POWERDOWN",
+    "timestamp": { "seconds": 1267040730, "microseconds": 682951 } }
+
+QUORUM_FAILURE
+--------------
+
+Emitted by the Quorum block driver if it fails to establish a quorum.
+
+Data:
+
+- "reference":     device name if defined else node name.
+- "sector-num":    Number of the first sector of the failed read operation.
+- "sectors-count": Failed read operation sector count.
+
+Example:
+
+{ "event": "QUORUM_FAILURE",
+     "data": { "reference": "usr1", "sector-num": 345435, "sectors-count": 5 },
+     "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
+
+Note: this event is rate-limited.
+
+QUORUM_REPORT_BAD
+-----------------
+
+Emitted to report a corruption of a Quorum file.
+
+Data:
+
+- "type":          Quorum operation type
+- "error":         Error message (json-string, optional)
+                   Only present on failure.  This field contains a human-readable
+                   error message.  There are no semantics other than that the
+                   block layer reported an error and clients should not try to
+                   interpret the error string.
+- "node-name":     The graph node name of the block driver state.
+- "sector-num":    Number of the first sector of the failed read operation.
+- "sectors-count": Failed read operation sector count.
+
+Example:
+
+Read operation:
+{ "event": "QUORUM_REPORT_BAD",
+     "data": { "node-name": "node0", "sector-num": 345435, "sectors-count": 5,
+               "type": "read" },
+     "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
+
+Flush operation:
+{ "event": "QUORUM_REPORT_BAD",
+     "data": { "node-name": "node0", "sector-num": 0, "sectors-count": 2097120,
+               "type": "flush", "error": "Broken pipe" },
+     "timestamp": { "seconds": 1456406829, "microseconds": 291763 } }
+
+Note: this event is rate-limited.
+
+RESET
+-----
+
+Emitted when the Virtual Machine is reset.
+
+Data: None.
+
+Example:
+
+{ "event": "RESET",
+    "timestamp": { "seconds": 1267041653, "microseconds": 9518 } }
+
+RESUME
+------
+
+Emitted when the Virtual Machine resumes execution.
+
+Data: None.
+
+Example:
+
+{ "event": "RESUME",
+    "timestamp": { "seconds": 1271770767, "microseconds": 582542 } }
+
+RTC_CHANGE
+----------
+
+Emitted when the guest changes the RTC time.
+
+Data:
+
+- "offset": Offset between base RTC clock (as specified by -rtc base), and
+new RTC clock value (json-number)
+
+Example:
+
+{ "event": "RTC_CHANGE",
+    "data": { "offset": 78 },
+    "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
+
+Note: this event is rate-limited.
+
+SHUTDOWN
+--------
+
+Emitted when the Virtual Machine has shut down, indicating that qemu
+is about to exit.
+
+Data: None.
+
+Example:
+
+{ "event": "SHUTDOWN",
+    "timestamp": { "seconds": 1267040730, "microseconds": 682951 } }
+
+Note: If the command-line option "-no-shutdown" has been specified, a STOP
+event will eventually follow the SHUTDOWN event.
+
+SPICE_CONNECTED
+---------------
+
+Emitted when a SPICE client connects.
+
+Data:
+
+- "server": Server information (json-object)
+  - "host": IP address (json-string)
+  - "port": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+- "client": Client information (json-object)
+  - "host": IP address (json-string)
+  - "port": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+
+Example:
+
+{ "timestamp": {"seconds": 1290688046, "microseconds": 388707},
+  "event": "SPICE_CONNECTED",
+  "data": {
+    "server": { "port": "5920", "family": "ipv4", "host": "127.0.0.1"},
+    "client": {"port": "52873", "family": "ipv4", "host": "127.0.0.1"}
+}}
+
+SPICE_DISCONNECTED
+------------------
+
+Emitted when a SPICE client disconnects.
+
+Data:
+
+- "server": Server information (json-object)
+  - "host": IP address (json-string)
+  - "port": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+- "client": Client information (json-object)
+  - "host": IP address (json-string)
+  - "port": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+
+Example:
+
+{ "timestamp": {"seconds": 1290688046, "microseconds": 388707},
+  "event": "SPICE_DISCONNECTED",
+  "data": {
+    "server": { "port": "5920", "family": "ipv4", "host": "127.0.0.1"},
+    "client": {"port": "52873", "family": "ipv4", "host": "127.0.0.1"}
+}}
+
+SPICE_INITIALIZED
+-----------------
+
+Emitted after initial handshake and authentication takes place (if any)
+and the SPICE channel is up and running
+
+Data:
+
+- "server": Server information (json-object)
+  - "host": IP address (json-string)
+  - "port": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "auth": authentication method (json-string, optional)
+- "client": Client information (json-object)
+  - "host": IP address (json-string)
+  - "port": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "connection-id": spice connection id.  All channels with the same id
+                     belong to the same spice session (json-int)
+  - "channel-type": channel type.  "1" is the main control channel, filter for
+                    this one if you want track spice sessions only (json-int)
+  - "channel-id": channel id.  Usually "0", might be different needed when
+                  multiple channels of the same type exist, such as multiple
+                  display channels in a multihead setup (json-int)
+  - "tls": whevener the channel is encrypted (json-bool)
+
+Example:
+
+{ "timestamp": {"seconds": 1290688046, "microseconds": 417172},
+  "event": "SPICE_INITIALIZED",
+  "data": {"server": {"auth": "spice", "port": "5921",
+                      "family": "ipv4", "host": "127.0.0.1"},
+           "client": {"port": "49004", "family": "ipv4", "channel-type": 3,
+                      "connection-id": 1804289383, "host": "127.0.0.1",
+                      "channel-id": 0, "tls": true}
+}}
+
+SPICE_MIGRATE_COMPLETED
+-----------------------
+
+Emitted when SPICE migration has completed
+
+Data: None.
+
+Example:
+
+{ "timestamp": {"seconds": 1290688046, "microseconds": 417172},
+  "event": "SPICE_MIGRATE_COMPLETED" }
+
+MIGRATION
+---------
+
+Emitted when a migration event happens
+
+Data: None.
+
+ - "status": migration status
+     See MigrationStatus in ~/qapi-schema.json for possible values
+
+Example:
+
+{"timestamp": {"seconds": 1432121972, "microseconds": 744001},
+ "event": "MIGRATION", "data": {"status": "completed"}}
+
+MIGRATION_PASS
+--------------
+
+Emitted from the source side of a migration at the start of each pass
+(when it syncs the dirty bitmap)
+
+Data: None.
+
+  - "pass": An incrementing count (starting at 1 on the first pass)
+
+Example:
+{"timestamp": {"seconds": 1449669631, "microseconds": 239225},
+ "event": "MIGRATION_PASS", "data": {"pass": 2}}
+
+STOP
+----
+
+Emitted when the Virtual Machine is stopped.
+
+Data: None.
+
+Example:
+
+{ "event": "STOP",
+    "timestamp": { "seconds": 1267041730, "microseconds": 281295 } }
+
+SUSPEND
+-------
+
+Emitted when guest enters S3 state.
+
+Data: None.
+
+Example:
+
+{ "event": "SUSPEND",
+     "timestamp": { "seconds": 1344456160, "microseconds": 309119 } }
+
+SUSPEND_DISK
+------------
+
+Emitted when the guest makes a request to enter S4 state.
+
+Data: None.
+
+Example:
+
+{ "event": "SUSPEND_DISK",
+     "timestamp": { "seconds": 1344456160, "microseconds": 309119 } }
+
+Note: QEMU shuts down when entering S4 state.
+
+VNC_CONNECTED
+-------------
+
+Emitted when a VNC client establishes a connection.
+
+Data:
+
+- "server": Server information (json-object)
+  - "host": IP address (json-string)
+  - "service": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "auth": authentication method (json-string, optional)
+- "client": Client information (json-object)
+  - "host": IP address (json-string)
+  - "service": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+
+Example:
+
+{ "event": "VNC_CONNECTED",
+    "data": {
+        "server": { "auth": "sasl", "family": "ipv4",
+                    "service": "5901", "host": "0.0.0.0" },
+        "client": { "family": "ipv4", "service": "58425",
+                    "host": "127.0.0.1" } },
+    "timestamp": { "seconds": 1262976601, "microseconds": 975795 } }
+
+
+Note: This event is emitted before any authentication takes place, thus
+the authentication ID is not provided.
+
+VNC_DISCONNECTED
+----------------
+
+Emitted when the connection is closed.
+
+Data:
+
+- "server": Server information (json-object)
+  - "host": IP address (json-string)
+  - "service": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "auth": authentication method (json-string, optional)
+- "client": Client information (json-object)
+  - "host": IP address (json-string)
+  - "service": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "x509_dname": TLS dname (json-string, optional)
+  - "sasl_username": SASL username (json-string, optional)
+
+Example:
+
+{ "event": "VNC_DISCONNECTED",
+    "data": {
+        "server": { "auth": "sasl", "family": "ipv4",
+                    "service": "5901", "host": "0.0.0.0" },
+        "client": { "family": "ipv4", "service": "58425",
+                    "host": "127.0.0.1", "sasl_username": "luiz" } },
+    "timestamp": { "seconds": 1262976601, "microseconds": 975795 } }
+
+VNC_INITIALIZED
+---------------
+
+Emitted after authentication takes place (if any) and the VNC session is
+made active.
+
+Data:
+
+- "server": Server information (json-object)
+  - "host": IP address (json-string)
+  - "service": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "auth": authentication method (json-string, optional)
+- "client": Client information (json-object)
+  - "host": IP address (json-string)
+  - "service": port number (json-string)
+  - "family": address family (json-string, "ipv4" or "ipv6")
+  - "x509_dname": TLS dname (json-string, optional)
+  - "sasl_username": SASL username (json-string, optional)
+
+Example:
+
+{ "event": "VNC_INITIALIZED",
+    "data": {
+        "server": { "auth": "sasl", "family": "ipv4",
+                    "service": "5901", "host": "0.0.0.0"},
+        "client": { "family": "ipv4", "service": "46089",
+                    "host": "127.0.0.1", "sasl_username": "luiz" } },
+        "timestamp": { "seconds": 1263475302, "microseconds": 150772 } }
+
+VSERPORT_CHANGE
+---------------
+
+Emitted when the guest opens or closes a virtio-serial port.
+
+Data:
+
+- "id": device identifier of the virtio-serial port (json-string)
+- "open": true if the guest has opened the virtio-serial port (json-bool)
+
+Example:
+
+{ "event": "VSERPORT_CHANGE",
+    "data": { "id": "channel0", "open": true },
+    "timestamp": { "seconds": 1401385907, "microseconds": 422329 } }
+
+Note: this event is rate-limited separately for each "id".
+
+WAKEUP
+------
+
+Emitted when the guest has woken up from S3 and is running.
+
+Data: None.
+
+Example:
+
+{ "event": "WAKEUP",
+     "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
+
+WATCHDOG
+--------
+
+Emitted when the watchdog device's timer is expired.
+
+Data:
+
+- "action": Action that has been taken, it's one of the following (json-string):
+            "reset", "shutdown", "poweroff", "pause", "debug", or "none"
+
+Example:
+
+{ "event": "WATCHDOG",
+     "data": { "action": "reset" },
+     "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
+
+Note: If action is "reset", "shutdown", or "pause" the WATCHDOG event is
+followed respectively by the RESET, SHUTDOWN, or STOP events.
+
+Note: this event is rate-limited.
diff --git a/docs/qmp/README b/docs/qmp-intro.txt
similarity index 100%
rename from docs/qmp/README
rename to docs/qmp-intro.txt
diff --git a/docs/qmp-spec.txt b/docs/qmp-spec.txt
new file mode 100644
index 0000000..f8b5356
--- /dev/null
+++ b/docs/qmp-spec.txt
@@ -0,0 +1,341 @@
+                      QEMU Machine Protocol Specification
+
+0. About This Document
+======================
+
+Copyright (C) 2009-2016 Red Hat, Inc.
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+1. Introduction
+===============
+
+This document specifies the QEMU Machine Protocol (QMP), a JSON-based
+protocol which is available for applications to operate QEMU at the
+machine-level.  It is also in use by the QEMU Guest Agent (QGA), which
+is available for host applications to interact with the guest
+operating system.
+
+2. Protocol Specification
+=========================
+
+This section details the protocol format. For the purpose of this document
+"Client" is any application which is using QMP to communicate with QEMU and
+"Server" is QEMU itself.
+
+JSON data structures, when mentioned in this document, are always in the
+following format:
+
+    json-DATA-STRUCTURE-NAME
+
+Where DATA-STRUCTURE-NAME is any valid JSON data structure, as defined
+by the JSON standard:
+
+http://www.ietf.org/rfc/rfc7159.txt
+
+The protocol is always encoded in UTF-8 except for synchronization
+bytes (documented below); although thanks to json-string escape
+sequences, the server will reply using only the strict ASCII subset.
+
+For convenience, json-object members mentioned in this document will
+be in a certain order. However, in real protocol usage they can be in
+ANY order, thus no particular order should be assumed. On the other
+hand, use of json-array elements presumes that preserving order is
+important unless specifically documented otherwise.  Repeating a key
+within a json-object gives unpredictable results.
+
+Also for convenience, the server will accept an extension of
+'single-quoted' strings in place of the usual "double-quoted"
+json-string, and both input forms of strings understand an additional
+escape sequence of "\'" for a single quote. The server will only use
+double quoting on output.
+
+2.1 General Definitions
+-----------------------
+
+2.1.1 All interactions transmitted by the Server are json-objects, always
+      terminating with CRLF
+
+2.1.2 All json-objects members are mandatory when not specified otherwise
+
+2.2 Server Greeting
+-------------------
+
+Right when connected the Server will issue a greeting message, which signals
+that the connection has been successfully established and that the Server is
+ready for capabilities negotiation (for more information refer to section
+'4. Capabilities Negotiation').
+
+The greeting message format is:
+
+{ "QMP": { "version": json-object, "capabilities": json-array } }
+
+ Where,
+
+- The "version" member contains the Server's version information (the format
+  is the same of the query-version command)
+- The "capabilities" member specify the availability of features beyond the
+  baseline specification; the order of elements in this array has no
+  particular significance, so a client must search the entire array
+  when looking for a particular capability
+
+2.2.1 Capabilities
+------------------
+
+As of the date this document was last revised, no server or client
+capability strings have been defined.
+
+
+2.3 Issuing Commands
+--------------------
+
+The format for command execution is:
+
+{ "execute": json-string, "arguments": json-object, "id": json-value }
+
+ Where,
+
+- The "execute" member identifies the command to be executed by the Server
+- The "arguments" member is used to pass any arguments required for the
+  execution of the command, it is optional when no arguments are
+  required. Each command documents what contents will be considered
+  valid when handling the json-argument
+- The "id" member is a transaction identification associated with the
+  command execution, it is optional and will be part of the response if
+  provided. The "id" member can be any json-value, although most
+  clients merely use a json-number incremented for each successive
+  command
+
+2.4 Commands Responses
+----------------------
+
+There are two possible responses which the Server will issue as the result
+of a command execution: success or error.
+
+2.4.1 success
+-------------
+
+The format of a success response is:
+
+{ "return": json-value, "id": json-value }
+
+ Where,
+
+- The "return" member contains the data returned by the command, which
+  is defined on a per-command basis (usually a json-object or
+  json-array of json-objects, but sometimes a json-number, json-string,
+  or json-array of json-strings); it is an empty json-object if the
+  command does not return data
+- The "id" member contains the transaction identification associated
+  with the command execution if issued by the Client
+
+2.4.2 error
+-----------
+
+The format of an error response is:
+
+{ "error": { "class": json-string, "desc": json-string }, "id": json-value }
+
+ Where,
+
+- The "class" member contains the error class name (eg. "GenericError")
+- The "desc" member is a human-readable error message. Clients should
+  not attempt to parse this message.
+- The "id" member contains the transaction identification associated with
+  the command execution if issued by the Client
+
+NOTE: Some errors can occur before the Server is able to read the "id" member,
+in these cases the "id" member will not be part of the error response, even
+if provided by the client.
+
+2.5 Asynchronous events
+-----------------------
+
+As a result of state changes, the Server may send messages unilaterally
+to the Client at any time, when not in the middle of any other
+response. They are called "asynchronous events".
+
+The format of asynchronous events is:
+
+{ "event": json-string, "data": json-object,
+  "timestamp": { "seconds": json-number, "microseconds": json-number } }
+
+ Where,
+
+- The "event" member contains the event's name
+- The "data" member contains event specific data, which is defined in a
+  per-event basis, it is optional
+- The "timestamp" member contains the exact time of when the event
+  occurred in the Server. It is a fixed json-object with time in
+  seconds and microseconds relative to the Unix Epoch (1 Jan 1970); if
+  there is a failure to retrieve host time, both members of the
+  timestamp will be set to -1.
+
+For a listing of supported asynchronous events, please, refer to the
+qmp-events.txt file.
+
+Some events are rate-limited to at most one per second.  If additional
+"similar" events arrive within one second, all but the last one are
+dropped, and the last one is delayed.  "Similar" normally means same
+event type.  See qmp-events.txt for details.
+
+2.6 QGA Synchronization
+-----------------------
+
+When using QGA, an additional synchronization feature is built into
+the protocol.  If the Client sends a raw 0xFF sentinel byte (not valid
+JSON), then the Server will reset its state and discard all pending
+data prior to the sentinel.  Conversely, if the Client makes use of
+the 'guest-sync-delimited' command, the Server will send a raw 0xFF
+sentinel byte prior to its response, to aid the Client in discarding
+any data prior to the sentinel.
+
+
+3. QMP Examples
+===============
+
+This section provides some examples of real QMP usage, in all of them
+"C" stands for "Client" and "S" stands for "Server".
+
+3.1 Server greeting
+-------------------
+
+S: { "QMP": { "version": { "qemu": { "micro": 50, "minor": 6, "major": 1 },
+     "package": ""}, "capabilities": []}}
+
+3.2 Client QMP negotiation
+--------------------------
+C: { "execute": "qmp_capabilities" }
+S: { "return": {}}
+
+3.3 Simple 'stop' execution
+---------------------------
+
+C: { "execute": "stop" }
+S: { "return": {} }
+
+3.4 KVM information
+-------------------
+
+C: { "execute": "query-kvm", "id": "example" }
+S: { "return": { "enabled": true, "present": true }, "id": "example"}
+
+3.5 Parsing error
+------------------
+
+C: { "execute": }
+S: { "error": { "class": "GenericError", "desc": "Invalid JSON syntax" } }
+
+3.6 Powerdown event
+-------------------
+
+S: { "timestamp": { "seconds": 1258551470, "microseconds": 802384 },
+    "event": "POWERDOWN" }
+
+4. Capabilities Negotiation
+===========================
+
+When a Client successfully establishes a connection, the Server is in
+Capabilities Negotiation mode.
+
+In this mode only the qmp_capabilities command is allowed to run, all
+other commands will return the CommandNotFound error. Asynchronous
+messages are not delivered either.
+
+Clients should use the qmp_capabilities command to enable capabilities
+advertised in the Server's greeting (section '2.2 Server Greeting') they
+support.
+
+When the qmp_capabilities command is issued, and if it does not return an
+error, the Server enters in Command mode where capabilities changes take
+effect, all commands (except qmp_capabilities) are allowed and asynchronous
+messages are delivered.
+
+5 Compatibility Considerations
+==============================
+
+All protocol changes or new features which modify the protocol format in an
+incompatible way are disabled by default and will be advertised by the
+capabilities array (section '2.2 Server Greeting'). Thus, Clients can check
+that array and enable the capabilities they support.
+
+The QMP Server performs a type check on the arguments to a command.  It
+generates an error if a value does not have the expected type for its
+key, or if it does not understand a key that the Client included.  The
+strictness of the Server catches wrong assumptions of Clients about
+the Server's schema.  Clients can assume that, when such validation
+errors occur, they will be reported before the command generated any
+side effect.
+
+However, Clients must not assume any particular:
+
+- Length of json-arrays
+- Size of json-objects; in particular, future versions of QEMU may add
+  new keys and Clients should be able to ignore them.
+- Order of json-object members or json-array elements
+- Amount of errors generated by a command, that is, new errors can be added
+  to any existing command in newer versions of the Server
+
+Any command or member name beginning with "x-" is deemed experimental,
+and may be withdrawn or changed in an incompatible manner in a future
+release.
+
+Of course, the Server does guarantee to send valid JSON.  But apart from
+this, a Client should be "conservative in what they send, and liberal in
+what they accept".
+
+6. Downstream extension of QMP
+==============================
+
+We recommend that downstream consumers of QEMU do *not* modify QMP.
+Management tools should be able to support both upstream and downstream
+versions of QMP without special logic, and downstream extensions are
+inherently at odds with that.
+
+However, we recognize that it is sometimes impossible for downstreams to
+avoid modifying QMP.  Both upstream and downstream need to take care to
+preserve long-term compatibility and interoperability.
+
+To help with that, QMP reserves JSON object member names beginning with
+'__' (double underscore) for downstream use ("downstream names").  This
+means upstream will never use any downstream names for its commands,
+arguments, errors, asynchronous events, and so forth.
+
+Any new names downstream wishes to add must begin with '__'.  To
+ensure compatibility with other downstreams, it is strongly
+recommended that you prefix your downstream names with '__RFQDN_' where
+RFQDN is a valid, reverse fully qualified domain name which you
+control.  For example, a qemu-kvm specific monitor command would be:
+
+    (qemu) __org.linux-kvm_enable_irqchip
+
+Downstream must not change the server greeting (section 2.2) other than
+to offer additional capabilities.  But see below for why even that is
+discouraged.
+
+Section '5 Compatibility Considerations' applies to downstream as well
+as to upstream, obviously.  It follows that downstream must behave
+exactly like upstream for any input not containing members with
+downstream names ("downstream members"), except it may add members
+with downstream names to its output.
+
+Thus, a client should not be able to distinguish downstream from
+upstream as long as it doesn't send input with downstream members, and
+properly ignores any downstream members in the output it receives.
+
+Advice on downstream modifications:
+
+1. Introducing new commands is okay.  If you want to extend an existing
+   command, consider introducing a new one with the new behaviour
+   instead.
+
+2. Introducing new asynchronous messages is okay.  If you want to extend
+   an existing message, consider adding a new one instead.
+
+3. Introducing new errors for use in new commands is okay.  Adding new
+   errors to existing commands counts as extension, so 1. applies.
+
+4. New capabilities are strongly discouraged.  Capabilities are for
+   evolving the basic protocol, and multiple diverging basic protocol
+   dialects are most undesirable.
diff --git a/docs/qmp/qmp-events.txt b/docs/qmp/qmp-events.txt
deleted file mode 100644
index d759d19..0000000
--- a/docs/qmp/qmp-events.txt
+++ /dev/null
@@ -1,627 +0,0 @@
-                   QEMU Machine Protocol Events
-                   ============================
-
-ACPI_DEVICE_OST
----------------
-
-Emitted when guest executes ACPI _OST method.
-
- - data: ACPIOSTInfo type as described in qapi-schema.json
-
-{ "event": "ACPI_DEVICE_OST",
-     "data": { "device": "d1", "slot": "0", "slot-type": "DIMM", "source": 1, "status": 0 } }
-
-BALLOON_CHANGE
---------------
-
-Emitted when the guest changes the actual BALLOON level. This
-value is equivalent to the 'actual' field return by the
-'query-balloon' command
-
-Data:
-
-- "actual": actual level of the guest memory balloon in bytes (json-number)
-
-Example:
-
-{ "event": "BALLOON_CHANGE",
-    "data": { "actual": 944766976 },
-    "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
-
-BLOCK_IMAGE_CORRUPTED
----------------------
-
-Emitted when a disk image is being marked corrupt.
-
-Data:
-
-- "device": Device name (json-string)
-- "msg":    Informative message (e.g., reason for the corruption) (json-string)
-- "offset": If the corruption resulted from an image access, this is the access
-            offset into the image (json-int)
-- "size":   If the corruption resulted from an image access, this is the access
-            size (json-int)
-
-Example:
-
-{ "event": "BLOCK_IMAGE_CORRUPTED",
-    "data": { "device": "ide0-hd0",
-        "msg": "Prevented active L1 table overwrite", "offset": 196608,
-        "size": 65536 },
-    "timestamp": { "seconds": 1378126126, "microseconds": 966463 } }
-
-BLOCK_IO_ERROR
---------------
-
-Emitted when a disk I/O error occurs.
-
-Data:
-
-- "device": device name (json-string)
-- "operation": I/O operation (json-string, "read" or "write")
-- "action": action that has been taken, it's one of the following (json-string):
-    "ignore": error has been ignored
-    "report": error has been reported to the device
-    "stop": the VM is going to stop because of the error
-
-Example:
-
-{ "event": "BLOCK_IO_ERROR",
-    "data": { "device": "ide0-hd1",
-              "operation": "write",
-              "action": "stop" },
-    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
-
-Note: If action is "stop", a STOP event will eventually follow the
-BLOCK_IO_ERROR event.
-
-BLOCK_JOB_CANCELLED
--------------------
-
-Emitted when a block job has been cancelled.
-
-Data:
-
-- "type":     Job type (json-string; "stream" for image streaming
-                                     "commit" for block commit)
-- "device":   Device name (json-string)
-- "len":      Maximum progress value (json-int)
-- "offset":   Current progress value (json-int)
-              On success this is equal to len.
-              On failure this is less than len.
-- "speed":    Rate limit, bytes per second (json-int)
-
-Example:
-
-{ "event": "BLOCK_JOB_CANCELLED",
-     "data": { "type": "stream", "device": "virtio-disk0",
-               "len": 10737418240, "offset": 134217728,
-               "speed": 0 },
-     "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
-
-BLOCK_JOB_COMPLETED
--------------------
-
-Emitted when a block job has completed.
-
-Data:
-
-- "type":     Job type (json-string; "stream" for image streaming
-                                     "commit" for block commit)
-- "device":   Device name (json-string)
-- "len":      Maximum progress value (json-int)
-- "offset":   Current progress value (json-int)
-              On success this is equal to len.
-              On failure this is less than len.
-- "speed":    Rate limit, bytes per second (json-int)
-- "error":    Error message (json-string, optional)
-              Only present on failure.  This field contains a human-readable
-              error message.  There are no semantics other than that streaming
-              has failed and clients should not try to interpret the error
-              string.
-
-Example:
-
-{ "event": "BLOCK_JOB_COMPLETED",
-     "data": { "type": "stream", "device": "virtio-disk0",
-               "len": 10737418240, "offset": 10737418240,
-               "speed": 0 },
-     "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
-
-BLOCK_JOB_ERROR
----------------
-
-Emitted when a block job encounters an error.
-
-Data:
-
-- "device": device name (json-string)
-- "operation": I/O operation (json-string, "read" or "write")
-- "action": action that has been taken, it's one of the following (json-string):
-    "ignore": error has been ignored, the job may fail later
-    "report": error will be reported and the job canceled
-    "stop": error caused job to be paused
-
-Example:
-
-{ "event": "BLOCK_JOB_ERROR",
-    "data": { "device": "ide0-hd1",
-              "operation": "write",
-              "action": "stop" },
-    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
-
-BLOCK_JOB_READY
----------------
-
-Emitted when a block job is ready to complete.
-
-Data:
-
-- "type":     Job type (json-string; "stream" for image streaming
-                                     "commit" for block commit)
-- "device":   Device name (json-string)
-- "len":      Maximum progress value (json-int)
-- "offset":   Current progress value (json-int)
-              On success this is equal to len.
-              On failure this is less than len.
-- "speed":    Rate limit, bytes per second (json-int)
-
-Example:
-
-{ "event": "BLOCK_JOB_READY",
-    "data": { "device": "drive0", "type": "mirror", "speed": 0,
-              "len": 2097152, "offset": 2097152 }
-    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
-
-Note: The "ready to complete" status is always reset by a BLOCK_JOB_ERROR
-event.
-
-DEVICE_DELETED
---------------
-
-Emitted whenever the device removal completion is acknowledged
-by the guest.
-At this point, it's safe to reuse the specified device ID.
-Device removal can be initiated by the guest or by HMP/QMP commands.
-
-Data:
-
-- "device": device name (json-string, optional)
-- "path": device path (json-string)
-
-{ "event": "DEVICE_DELETED",
-  "data": { "device": "virtio-net-pci-0",
-            "path": "/machine/peripheral/virtio-net-pci-0" },
-  "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
-
-DEVICE_TRAY_MOVED
------------------
-
-It's emitted whenever the tray of a removable device is moved by the guest
-or by HMP/QMP commands.
-
-Data:
-
-- "device": device name (json-string)
-- "tray-open": true if the tray has been opened or false if it has been closed
-               (json-bool)
-
-{ "event": "DEVICE_TRAY_MOVED",
-  "data": { "device": "ide1-cd0",
-            "tray-open": true
-  },
-  "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
-
-GUEST_PANICKED
---------------
-
-Emitted when guest OS panic is detected.
-
-Data:
-
-- "action": Action that has been taken (json-string, currently always "pause").
-
-Example:
-
-{ "event": "GUEST_PANICKED",
-     "data": { "action": "pause" } }
-
-NIC_RX_FILTER_CHANGED
----------------------
-
-The event is emitted once until the query command is executed,
-the first event will always be emitted.
-
-Data:
-
-- "name": net client name (json-string)
-- "path": device path (json-string)
-
-{ "event": "NIC_RX_FILTER_CHANGED",
-  "data": { "name": "vnet0",
-            "path": "/machine/peripheral/vnet0/virtio-backend" },
-  "timestamp": { "seconds": 1368697518, "microseconds": 326866 } }
-}
-
-POWERDOWN
----------
-
-Emitted when the Virtual Machine is powered down through the power
-control system, such as via ACPI.
-
-Data: None.
-
-Example:
-
-{ "event": "POWERDOWN",
-    "timestamp": { "seconds": 1267040730, "microseconds": 682951 } }
-
-QUORUM_FAILURE
---------------
-
-Emitted by the Quorum block driver if it fails to establish a quorum.
-
-Data:
-
-- "reference":     device name if defined else node name.
-- "sector-num":    Number of the first sector of the failed read operation.
-- "sectors-count": Failed read operation sector count.
-
-Example:
-
-{ "event": "QUORUM_FAILURE",
-     "data": { "reference": "usr1", "sector-num": 345435, "sectors-count": 5 },
-     "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
-
-QUORUM_REPORT_BAD
------------------
-
-Emitted to report a corruption of a Quorum file.
-
-Data:
-
-- "error":         Error message (json-string, optional)
-                   Only present on failure.  This field contains a human-readable
-                   error message.  There are no semantics other than that the
-                   block layer reported an error and clients should not try to
-                   interpret the error string.
-- "node-name":     The graph node name of the block driver state.
-- "sector-num":    Number of the first sector of the failed read operation.
-- "sectors-count": Failed read operation sector count.
-
-Example:
-
-{ "event": "QUORUM_REPORT_BAD",
-     "data": { "node-name": "1.raw", "sector-num": 345435, "sectors-count": 5 },
-     "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
-
-RESET
------
-
-Emitted when the Virtual Machine is reset.
-
-Data: None.
-
-Example:
-
-{ "event": "RESET",
-    "timestamp": { "seconds": 1267041653, "microseconds": 9518 } }
-
-RESUME
-------
-
-Emitted when the Virtual Machine resumes execution.
-
-Data: None.
-
-Example:
-
-{ "event": "RESUME",
-    "timestamp": { "seconds": 1271770767, "microseconds": 582542 } }
-
-RTC_CHANGE
-----------
-
-Emitted when the guest changes the RTC time.
-
-Data:
-
-- "offset": Offset between base RTC clock (as specified by -rtc base), and
-new RTC clock value (json-number)
-
-Example:
-
-{ "event": "RTC_CHANGE",
-    "data": { "offset": 78 },
-    "timestamp": { "seconds": 1267020223, "microseconds": 435656 } }
-
-SHUTDOWN
---------
-
-Emitted when the Virtual Machine has shut down, indicating that qemu
-is about to exit.
-
-Data: None.
-
-Example:
-
-{ "event": "SHUTDOWN",
-    "timestamp": { "seconds": 1267040730, "microseconds": 682951 } }
-
-Note: If the command-line option "-no-shutdown" has been specified, a STOP
-event will eventually follow the SHUTDOWN event.
-
-SPICE_CONNECTED
----------------
-
-Emitted when a SPICE client connects.
-
-Data:
-
-- "server": Server information (json-object)
-  - "host": IP address (json-string)
-  - "port": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-- "client": Client information (json-object)
-  - "host": IP address (json-string)
-  - "port": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-
-Example:
-
-{ "timestamp": {"seconds": 1290688046, "microseconds": 388707},
-  "event": "SPICE_CONNECTED",
-  "data": {
-    "server": { "port": "5920", "family": "ipv4", "host": "127.0.0.1"},
-    "client": {"port": "52873", "family": "ipv4", "host": "127.0.0.1"}
-}}
-
-SPICE_DISCONNECTED
-------------------
-
-Emitted when a SPICE client disconnects.
-
-Data:
-
-- "server": Server information (json-object)
-  - "host": IP address (json-string)
-  - "port": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-- "client": Client information (json-object)
-  - "host": IP address (json-string)
-  - "port": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-
-Example:
-
-{ "timestamp": {"seconds": 1290688046, "microseconds": 388707},
-  "event": "SPICE_DISCONNECTED",
-  "data": {
-    "server": { "port": "5920", "family": "ipv4", "host": "127.0.0.1"},
-    "client": {"port": "52873", "family": "ipv4", "host": "127.0.0.1"}
-}}
-
-SPICE_INITIALIZED
------------------
-
-Emitted after initial handshake and authentication takes place (if any)
-and the SPICE channel is up and running
-
-Data:
-
-- "server": Server information (json-object)
-  - "host": IP address (json-string)
-  - "port": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "auth": authentication method (json-string, optional)
-- "client": Client information (json-object)
-  - "host": IP address (json-string)
-  - "port": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "connection-id": spice connection id.  All channels with the same id
-                     belong to the same spice session (json-int)
-  - "channel-type": channel type.  "1" is the main control channel, filter for
-                    this one if you want track spice sessions only (json-int)
-  - "channel-id": channel id.  Usually "0", might be different needed when
-                  multiple channels of the same type exist, such as multiple
-                  display channels in a multihead setup (json-int)
-  - "tls": whevener the channel is encrypted (json-bool)
-
-Example:
-
-{ "timestamp": {"seconds": 1290688046, "microseconds": 417172},
-  "event": "SPICE_INITIALIZED",
-  "data": {"server": {"auth": "spice", "port": "5921",
-                      "family": "ipv4", "host": "127.0.0.1"},
-           "client": {"port": "49004", "family": "ipv4", "channel-type": 3,
-                      "connection-id": 1804289383, "host": "127.0.0.1",
-                      "channel-id": 0, "tls": true}
-}}
-
-SPICE_MIGRATE_COMPLETED
------------------------
-
-Emitted when SPICE migration has completed
-
-Data: None.
-
-Example:
-
-{ "timestamp": {"seconds": 1290688046, "microseconds": 417172},
-  "event": "SPICE_MIGRATE_COMPLETED" }
-
-
-STOP
-----
-
-Emitted when the Virtual Machine is stopped.
-
-Data: None.
-
-Example:
-
-{ "event": "STOP",
-    "timestamp": { "seconds": 1267041730, "microseconds": 281295 } }
-
-SUSPEND
--------
-
-Emitted when guest enters S3 state.
-
-Data: None.
-
-Example:
-
-{ "event": "SUSPEND",
-     "timestamp": { "seconds": 1344456160, "microseconds": 309119 } }
-
-SUSPEND_DISK
-------------
-
-Emitted when the guest makes a request to enter S4 state.
-
-Data: None.
-
-Example:
-
-{ "event": "SUSPEND_DISK",
-     "timestamp": { "seconds": 1344456160, "microseconds": 309119 } }
-
-Note: QEMU shuts down when entering S4 state.
-
-VNC_CONNECTED
--------------
-
-Emitted when a VNC client establishes a connection.
-
-Data:
-
-- "server": Server information (json-object)
-  - "host": IP address (json-string)
-  - "service": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "auth": authentication method (json-string, optional)
-- "client": Client information (json-object)
-  - "host": IP address (json-string)
-  - "service": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-
-Example:
-
-{ "event": "VNC_CONNECTED",
-    "data": {
-        "server": { "auth": "sasl", "family": "ipv4",
-                    "service": "5901", "host": "0.0.0.0" },
-        "client": { "family": "ipv4", "service": "58425",
-                    "host": "127.0.0.1" } },
-    "timestamp": { "seconds": 1262976601, "microseconds": 975795 } }
-
-
-Note: This event is emitted before any authentication takes place, thus
-the authentication ID is not provided.
-
-VNC_DISCONNECTED
-----------------
-
-Emitted when the connection is closed.
-
-Data:
-
-- "server": Server information (json-object)
-  - "host": IP address (json-string)
-  - "service": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "auth": authentication method (json-string, optional)
-- "client": Client information (json-object)
-  - "host": IP address (json-string)
-  - "service": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "x509_dname": TLS dname (json-string, optional)
-  - "sasl_username": SASL username (json-string, optional)
-
-Example:
-
-{ "event": "VNC_DISCONNECTED",
-    "data": {
-        "server": { "auth": "sasl", "family": "ipv4",
-                    "service": "5901", "host": "0.0.0.0" },
-        "client": { "family": "ipv4", "service": "58425",
-                    "host": "127.0.0.1", "sasl_username": "luiz" } },
-    "timestamp": { "seconds": 1262976601, "microseconds": 975795 } }
-
-VNC_INITIALIZED
----------------
-
-Emitted after authentication takes place (if any) and the VNC session is
-made active.
-
-Data:
-
-- "server": Server information (json-object)
-  - "host": IP address (json-string)
-  - "service": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "auth": authentication method (json-string, optional)
-- "client": Client information (json-object)
-  - "host": IP address (json-string)
-  - "service": port number (json-string)
-  - "family": address family (json-string, "ipv4" or "ipv6")
-  - "x509_dname": TLS dname (json-string, optional)
-  - "sasl_username": SASL username (json-string, optional)
-
-Example:
-
-{ "event": "VNC_INITIALIZED",
-    "data": {
-        "server": { "auth": "sasl", "family": "ipv4",
-                    "service": "5901", "host": "0.0.0.0"},
-        "client": { "family": "ipv4", "service": "46089",
-                    "host": "127.0.0.1", "sasl_username": "luiz" } },
-        "timestamp": { "seconds": 1263475302, "microseconds": 150772 } }
-
-VSERPORT_CHANGE
----------------
-
-Emitted when the guest opens or closes a virtio-serial port.
-
-Data:
-
-- "id": device identifier of the virtio-serial port (json-string)
-- "open": true if the guest has opened the virtio-serial port (json-bool)
-
-Example:
-
-{ "event": "VSERPORT_CHANGE",
-    "data": { "id": "channel0", "open": true },
-    "timestamp": { "seconds": 1401385907, "microseconds": 422329 } }
-
-WAKEUP
-------
-
-Emitted when the guest has woken up from S3 and is running.
-
-Data: None.
-
-Example:
-
-{ "event": "WAKEUP",
-     "timestamp": { "seconds": 1344522075, "microseconds": 745528 } }
-
-WATCHDOG
---------
-
-Emitted when the watchdog device's timer is expired.
-
-Data:
-
-- "action": Action that has been taken, it's one of the following (json-string):
-            "reset", "shutdown", "poweroff", "pause", "debug", or "none"
-
-Example:
-
-{ "event": "WATCHDOG",
-     "data": { "action": "reset" },
-     "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
-
-Note: If action is "reset", "shutdown", or "pause" the WATCHDOG event is
-followed respectively by the RESET, SHUTDOWN, or STOP events.
diff --git a/docs/qmp/qmp-spec.txt b/docs/qmp/qmp-spec.txt
deleted file mode 100644
index 22568c6..0000000
--- a/docs/qmp/qmp-spec.txt
+++ /dev/null
@@ -1,273 +0,0 @@
-                      QEMU Machine Protocol Specification
-
-1. Introduction
-===============
-
-This document specifies the QEMU Machine Protocol (QMP), a JSON-based protocol
-which is available for applications to operate QEMU at the machine-level.
-
-2. Protocol Specification
-=========================
-
-This section details the protocol format. For the purpose of this document
-"Client" is any application which is using QMP to communicate with QEMU and
-"Server" is QEMU itself.
-
-JSON data structures, when mentioned in this document, are always in the
-following format:
-
-    json-DATA-STRUCTURE-NAME
-
-Where DATA-STRUCTURE-NAME is any valid JSON data structure, as defined by
-the JSON standard:
-
-http://www.ietf.org/rfc/rfc4627.txt
-
-For convenience, json-object members and json-array elements mentioned in
-this document will be in a certain order. However, in real protocol usage
-they can be in ANY order, thus no particular order should be assumed.
-
-2.1 General Definitions
------------------------
-
-2.1.1 All interactions transmitted by the Server are json-objects, always
-      terminating with CRLF
-
-2.1.2 All json-objects members are mandatory when not specified otherwise
-
-2.2 Server Greeting
--------------------
-
-Right when connected the Server will issue a greeting message, which signals
-that the connection has been successfully established and that the Server is
-ready for capabilities negotiation (for more information refer to section
-'4. Capabilities Negotiation').
-
-The greeting message format is:
-
-{ "QMP": { "version": json-object, "capabilities": json-array } }
-
- Where,
-
-- The "version" member contains the Server's version information (the format
-  is the same of the query-version command)
-- The "capabilities" member specify the availability of features beyond the
-  baseline specification
-
-2.3 Issuing Commands
---------------------
-
-The format for command execution is:
-
-{ "execute": json-string, "arguments": json-object, "id": json-value }
-
- Where,
-
-- The "execute" member identifies the command to be executed by the Server
-- The "arguments" member is used to pass any arguments required for the
-  execution of the command, it is optional when no arguments are required
-- The "id" member is a transaction identification associated with the
-  command execution, it is optional and will be part of the response if
-  provided
-
-2.4 Commands Responses
-----------------------
-
-There are two possible responses which the Server will issue as the result
-of a command execution: success or error.
-
-2.4.1 success
--------------
-
-The format of a success response is:
-
-{ "return": json-object, "id": json-value }
-
- Where,
-
-- The "return" member contains the command returned data, which is defined
-  in a per-command basis or an empty json-object if the command does not
-  return data
-- The "id" member contains the transaction identification associated
-  with the command execution if issued by the Client
-
-2.4.2 error
------------
-
-The format of an error response is:
-
-{ "error": { "class": json-string, "desc": json-string }, "id": json-value }
-
- Where,
-
-- The "class" member contains the error class name (eg. "GenericError")
-- The "desc" member is a human-readable error message. Clients should
-  not attempt to parse this message.
-- The "id" member contains the transaction identification associated with
-  the command execution if issued by the Client
-
-NOTE: Some errors can occur before the Server is able to read the "id" member,
-in these cases the "id" member will not be part of the error response, even
-if provided by the client.
-
-2.5 Asynchronous events
------------------------
-
-As a result of state changes, the Server may send messages unilaterally
-to the Client at any time. They are called "asynchronous events".
-
-The format of asynchronous events is:
-
-{ "event": json-string, "data": json-object,
-  "timestamp": { "seconds": json-number, "microseconds": json-number } }
-
- Where,
-
-- The "event" member contains the event's name
-- The "data" member contains event specific data, which is defined in a
-  per-event basis, it is optional
-- The "timestamp" member contains the exact time of when the event occurred
-  in the Server. It is a fixed json-object with time in seconds and
-  microseconds
-
-For a listing of supported asynchronous events, please, refer to the
-qmp-events.txt file.
-
-3. QMP Examples
-===============
-
-This section provides some examples of real QMP usage, in all of them
-"C" stands for "Client" and "S" stands for "Server".
-
-3.1 Server greeting
--------------------
-
-S: { "QMP": { "version": { "qemu": { "micro": 50, "minor": 6, "major": 1 },
-     "package": ""}, "capabilities": []}}
-
-3.2 Simple 'stop' execution
----------------------------
-
-C: { "execute": "stop" }
-S: { "return": {} }
-
-3.3 KVM information
--------------------
-
-C: { "execute": "query-kvm", "id": "example" }
-S: { "return": { "enabled": true, "present": true }, "id": "example"}
-
-3.4 Parsing error
-------------------
-
-C: { "execute": }
-S: { "error": { "class": "GenericError", "desc": "Invalid JSON syntax" } }
-
-3.5 Powerdown event
--------------------
-
-S: { "timestamp": { "seconds": 1258551470, "microseconds": 802384 },
-    "event": "POWERDOWN" }
-
-4. Capabilities Negotiation
-----------------------------
-
-When a Client successfully establishes a connection, the Server is in
-Capabilities Negotiation mode.
-
-In this mode only the qmp_capabilities command is allowed to run, all
-other commands will return the CommandNotFound error. Asynchronous
-messages are not delivered either.
-
-Clients should use the qmp_capabilities command to enable capabilities
-advertised in the Server's greeting (section '2.2 Server Greeting') they
-support.
-
-When the qmp_capabilities command is issued, and if it does not return an
-error, the Server enters in Command mode where capabilities changes take
-effect, all commands (except qmp_capabilities) are allowed and asynchronous
-messages are delivered.
-
-5 Compatibility Considerations
-------------------------------
-
-All protocol changes or new features which modify the protocol format in an
-incompatible way are disabled by default and will be advertised by the
-capabilities array (section '2.2 Server Greeting'). Thus, Clients can check
-that array and enable the capabilities they support.
-
-The QMP Server performs a type check on the arguments to a command.  It
-generates an error if a value does not have the expected type for its
-key, or if it does not understand a key that the Client included.  The
-strictness of the Server catches wrong assumptions of Clients about
-the Server's schema.  Clients can assume that, when such validation
-errors occur, they will be reported before the command generated any
-side effect.
-
-However, Clients must not assume any particular:
-
-- Length of json-arrays
-- Size of json-objects; in particular, future versions of QEMU may add
-  new keys and Clients should be able to ignore them.
-- Order of json-object members or json-array elements
-- Amount of errors generated by a command, that is, new errors can be added
-  to any existing command in newer versions of the Server
-
-Of course, the Server does guarantee to send valid JSON.  But apart from
-this, a Client should be "conservative in what they send, and liberal in
-what they accept".
-
-6. Downstream extension of QMP
-------------------------------
-
-We recommend that downstream consumers of QEMU do *not* modify QMP.
-Management tools should be able to support both upstream and downstream
-versions of QMP without special logic, and downstream extensions are
-inherently at odds with that.
-
-However, we recognize that it is sometimes impossible for downstreams to
-avoid modifying QMP.  Both upstream and downstream need to take care to
-preserve long-term compatibility and interoperability.
-
-To help with that, QMP reserves JSON object member names beginning with
-'__' (double underscore) for downstream use ("downstream names").  This
-means upstream will never use any downstream names for its commands,
-arguments, errors, asynchronous events, and so forth.
-
-Any new names downstream wishes to add must begin with '__'.  To
-ensure compatibility with other downstreams, it is strongly
-recommended that you prefix your downstream names with '__RFQDN_' where
-RFQDN is a valid, reverse fully qualified domain name which you
-control.  For example, a qemu-kvm specific monitor command would be:
-
-    (qemu) __org.linux-kvm_enable_irqchip
-
-Downstream must not change the server greeting (section 2.2) other than
-to offer additional capabilities.  But see below for why even that is
-discouraged.
-
-Section '5 Compatibility Considerations' applies to downstream as well
-as to upstream, obviously.  It follows that downstream must behave
-exactly like upstream for any input not containing members with
-downstream names ("downstream members"), except it may add members
-with downstream names to its output.
-
-Thus, a client should not be able to distinguish downstream from
-upstream as long as it doesn't send input with downstream members, and
-properly ignores any downstream members in the output it receives.
-
-Advice on downstream modifications:
-
-1. Introducing new commands is okay.  If you want to extend an existing
-   command, consider introducing a new one with the new behaviour
-   instead.
-
-2. Introducing new asynchronous messages is okay.  If you want to extend
-   an existing message, consider adding a new one instead.
-
-3. Introducing new errors for use in new commands is okay.  Adding new
-   errors to existing commands counts as extension, so 1. applies.
-
-4. New capabilities are strongly discouraged.  Capabilities are for
-   evolving the basic protocol, and multiple diverging basic protocol
-   dialects are most undesirable.
diff --git a/docs/rcu.txt b/docs/rcu.txt
new file mode 100644
index 0000000..2f70954
--- /dev/null
+++ b/docs/rcu.txt
@@ -0,0 +1,390 @@
+Using RCU (Read-Copy-Update) for synchronization
+================================================
+
+Read-copy update (RCU) is a synchronization mechanism that is used to
+protect read-mostly data structures.  RCU is very efficient and scalable
+on the read side (it is wait-free), and thus can make the read paths
+extremely fast.
+
+RCU supports concurrency between a single writer and multiple readers,
+thus it is not used alone.  Typically, the write-side will use a lock to
+serialize multiple updates, but other approaches are possible (e.g.,
+restricting updates to a single task).  In QEMU, when a lock is used,
+this will often be the "iothread mutex", also known as the "big QEMU
+lock" (BQL).  Also, restricting updates to a single task is done in
+QEMU using the "bottom half" API.
+
+RCU is fundamentally a "wait-to-finish" mechanism.  The read side marks
+sections of code with "critical sections", and the update side will wait
+for the execution of all *currently running* critical sections before
+proceeding, or before asynchronously executing a callback.
+
+The key point here is that only the currently running critical sections
+are waited for; critical sections that are started _after_ the beginning
+of the wait do not extend the wait, despite running concurrently with
+the updater.  This is the reason why RCU is more scalable than,
+for example, reader-writer locks.  It is so much more scalable that
+the system will have a single instance of the RCU mechanism; a single
+mechanism can be used for an arbitrary number of "things", without
+having to worry about things such as contention or deadlocks.
+
+How is this possible?  The basic idea is to split updates in two phases,
+"removal" and "reclamation".  During removal, we ensure that subsequent
+readers will not be able to get a reference to the old data.  After
+removal has completed, a critical section will not be able to access
+the old data.  Therefore, critical sections that begin after removal
+do not matter; as soon as all previous critical sections have finished,
+there cannot be any readers who hold references to the data structure,
+and these can now be safely reclaimed (e.g., freed or unref'ed).
+
+Here is a picutre:
+
+        thread 1                  thread 2                  thread 3
+    -------------------    ------------------------    -------------------
+    enter RCU crit.sec.
+           |                finish removal phase
+           |                begin wait
+           |                      |                    enter RCU crit.sec.
+    exit RCU crit.sec             |                           |
+                            complete wait                     |
+                            begin reclamation phase           |
+                                                       exit RCU crit.sec.
+
+
+Note how thread 3 is still executing its critical section when thread 2
+starts reclaiming data.  This is possible, because the old version of the
+data structure was not accessible at the time thread 3 began executing
+that critical section.
+
+
+RCU API
+=======
+
+The core RCU API is small:
+
+     void rcu_read_lock(void);
+
+        Used by a reader to inform the reclaimer that the reader is
+        entering an RCU read-side critical section.
+
+     void rcu_read_unlock(void);
+
+        Used by a reader to inform the reclaimer that the reader is
+        exiting an RCU read-side critical section.  Note that RCU
+        read-side critical sections may be nested and/or overlapping.
+
+     void synchronize_rcu(void);
+
+        Blocks until all pre-existing RCU read-side critical sections
+        on all threads have completed.  This marks the end of the removal
+        phase and the beginning of reclamation phase.
+
+        Note that it would be valid for another update to come while
+        synchronize_rcu is running.  Because of this, it is better that
+        the updater releases any locks it may hold before calling
+        synchronize_rcu.  If this is not possible (for example, because
+        the updater is protected by the BQL), you can use call_rcu.
+
+     void call_rcu1(struct rcu_head * head,
+                    void (*func)(struct rcu_head *head));
+
+        This function invokes func(head) after all pre-existing RCU
+        read-side critical sections on all threads have completed.  This
+        marks the end of the removal phase, with func taking care
+        asynchronously of the reclamation phase.
+
+        The foo struct needs to have an rcu_head structure added,
+        perhaps as follows:
+
+            struct foo {
+                struct rcu_head rcu;
+                int a;
+                char b;
+                long c;
+            };
+
+        so that the reclaimer function can fetch the struct foo address
+        and free it:
+
+            call_rcu1(&foo.rcu, foo_reclaim);
+
+            void foo_reclaim(struct rcu_head *rp)
+            {
+                struct foo *fp = container_of(rp, struct foo, rcu);
+                g_free(fp);
+            }
+
+        For the common case where the rcu_head member is the first of the
+        struct, you can use the following macro.
+
+     void call_rcu(T *p,
+                   void (*func)(T *p),
+                   field-name);
+     void g_free_rcu(T *p,
+                     field-name);
+
+        call_rcu1 is typically used through these macro, in the common case
+        where the "struct rcu_head" is the first field in the struct.  If
+        the callback function is g_free, in particular, g_free_rcu can be
+        used.  In the above case, one could have written simply:
+
+            g_free_rcu(&foo, rcu);
+
+     typeof(*p) atomic_rcu_read(p);
+
+        atomic_rcu_read() is similar to atomic_mb_read(), but it makes
+        some assumptions on the code that calls it.  This allows a more
+        optimized implementation.
+
+        atomic_rcu_read assumes that whenever a single RCU critical
+        section reads multiple shared data, these reads are either
+        data-dependent or need no ordering.  This is almost always the
+        case when using RCU, because read-side critical sections typically
+        navigate one or more pointers (the pointers that are changed on
+        every update) until reaching a data structure of interest,
+        and then read from there.
+
+        RCU read-side critical sections must use atomic_rcu_read() to
+        read data, unless concurrent writes are presented by another
+        synchronization mechanism.
+
+        Furthermore, RCU read-side critical sections should traverse the
+        data structure in a single direction, opposite to the direction
+        in which the updater initializes it.
+
+     void atomic_rcu_set(p, typeof(*p) v);
+
+        atomic_rcu_set() is also similar to atomic_mb_set(), and it also
+        makes assumptions on the code that calls it in order to allow a more
+        optimized implementation.
+
+        In particular, atomic_rcu_set() suffices for synchronization
+        with readers, if the updater never mutates a field within a
+        data item that is already accessible to readers.  This is the
+        case when initializing a new copy of the RCU-protected data
+        structure; just ensure that initialization of *p is carried out
+        before atomic_rcu_set() makes the data item visible to readers.
+        If this rule is observed, writes will happen in the opposite
+        order as reads in the RCU read-side critical sections (or if
+        there is just one update), and there will be no need for other
+        synchronization mechanism to coordinate the accesses.
+
+The following APIs must be used before RCU is used in a thread:
+
+     void rcu_register_thread(void);
+
+        Mark a thread as taking part in the RCU mechanism.  Such a thread
+        will have to report quiescent points regularly, either manually
+        or through the QemuCond/QemuSemaphore/QemuEvent APIs.
+
+     void rcu_unregister_thread(void);
+
+        Mark a thread as not taking part anymore in the RCU mechanism.
+        It is not a problem if such a thread reports quiescent points,
+        either manually or by using the QemuCond/QemuSemaphore/QemuEvent
+        APIs.
+
+Note that these APIs are relatively heavyweight, and should _not_ be
+nested.
+
+
+DIFFERENCES WITH LINUX
+======================
+
+- Waiting on a mutex is possible, though discouraged, within an RCU critical
+  section.  This is because spinlocks are rarely (if ever) used in userspace
+  programming; not allowing this would prevent upgrading an RCU read-side
+  critical section to become an updater.
+
+- atomic_rcu_read and atomic_rcu_set replace rcu_dereference and
+  rcu_assign_pointer.  They take a _pointer_ to the variable being accessed.
+
+- call_rcu is a macro that has an extra argument (the name of the first
+  field in the struct, which must be a struct rcu_head), and expects the
+  type of the callback's argument to be the type of the first argument.
+  call_rcu1 is the same as Linux's call_rcu.
+
+
+RCU PATTERNS
+============
+
+Many patterns using read-writer locks translate directly to RCU, with
+the advantages of higher scalability and deadlock immunity.
+
+In general, RCU can be used whenever it is possible to create a new
+"version" of a data structure every time the updater runs.  This may
+sound like a very strict restriction, however:
+
+- the updater does not mean "everything that writes to a data structure",
+  but rather "everything that involves a reclamation step".  See the
+  array example below
+
+- in some cases, creating a new version of a data structure may actually
+  be very cheap.  For example, modifying the "next" pointer of a singly
+  linked list is effectively creating a new version of the list.
+
+Here are some frequently-used RCU idioms that are worth noting.
+
+
+RCU list processing
+-------------------
+
+TBD (not yet used in QEMU)
+
+
+RCU reference counting
+----------------------
+
+Because grace periods are not allowed to complete while there is an RCU
+read-side critical section in progress, the RCU read-side primitives
+may be used as a restricted reference-counting mechanism.  For example,
+consider the following code fragment:
+
+    rcu_read_lock();
+    p = atomic_rcu_read(&foo);
+    /* do something with p. */
+    rcu_read_unlock();
+
+The RCU read-side critical section ensures that the value of "p" remains
+valid until after the rcu_read_unlock().  In some sense, it is acquiring
+a reference to p that is later released when the critical section ends.
+The write side looks simply like this (with appropriate locking):
+
+    qemu_mutex_lock(&foo_mutex);
+    old = foo;
+    atomic_rcu_set(&foo, new);
+    qemu_mutex_unlock(&foo_mutex);
+    synchronize_rcu();
+    free(old);
+
+If the processing cannot be done purely within the critical section, it
+is possible to combine this idiom with a "real" reference count:
+
+    rcu_read_lock();
+    p = atomic_rcu_read(&foo);
+    foo_ref(p);
+    rcu_read_unlock();
+    /* do something with p. */
+    foo_unref(p);
+
+The write side can be like this:
+
+    qemu_mutex_lock(&foo_mutex);
+    old = foo;
+    atomic_rcu_set(&foo, new);
+    qemu_mutex_unlock(&foo_mutex);
+    synchronize_rcu();
+    foo_unref(old);
+
+or with call_rcu:
+
+    qemu_mutex_lock(&foo_mutex);
+    old = foo;
+    atomic_rcu_set(&foo, new);
+    qemu_mutex_unlock(&foo_mutex);
+    call_rcu(foo_unref, old, rcu);
+
+In both cases, the write side only performs removal.  Reclamation
+happens when the last reference to a "foo" object is dropped.
+Using synchronize_rcu() is undesirably expensive, because the
+last reference may be dropped on the read side.  Hence you can
+use call_rcu() instead:
+
+     foo_unref(struct foo *p) {
+        if (atomic_fetch_dec(&p->refcount) == 1) {
+            call_rcu(foo_destroy, p, rcu);
+        }
+    }
+
+
+Note that the same idioms would be possible with reader/writer
+locks:
+
+    read_lock(&foo_rwlock);         write_mutex_lock(&foo_rwlock);
+    p = foo;                        p = foo;
+    /* do something with p. */      foo = new;
+    read_unlock(&foo_rwlock);       free(p);
+                                    write_mutex_unlock(&foo_rwlock);
+                                    free(p);
+
+    ------------------------------------------------------------------
+
+    read_lock(&foo_rwlock);         write_mutex_lock(&foo_rwlock);
+    p = foo;                        old = foo;
+    foo_ref(p);                     foo = new;
+    read_unlock(&foo_rwlock);       foo_unref(old);
+    /* do something with p. */      write_mutex_unlock(&foo_rwlock);
+    read_lock(&foo_rwlock);
+    foo_unref(p);
+    read_unlock(&foo_rwlock);
+
+foo_unref could use a mechanism such as bottom halves to move deallocation
+out of the write-side critical section.
+
+
+RCU resizable arrays
+--------------------
+
+Resizable arrays can be used with RCU.  The expensive RCU synchronization
+(or call_rcu) only needs to take place when the array is resized.
+The two items to take care of are:
+
+- ensuring that the old version of the array is available between removal
+  and reclamation;
+
+- avoiding mismatches in the read side between the array data and the
+  array size.
+
+The first problem is avoided simply by not using realloc.  Instead,
+each resize will allocate a new array and copy the old data into it.
+The second problem would arise if the size and the data pointers were
+two members of a larger struct:
+
+    struct mystuff {
+        ...
+        int data_size;
+        int data_alloc;
+        T   *data;
+        ...
+    };
+
+Instead, we store the size of the array with the array itself:
+
+    struct arr {
+        int size;
+        int alloc;
+        T   data[];
+    };
+    struct arr *global_array;
+
+    read side:
+        rcu_read_lock();
+        struct arr *array = atomic_rcu_read(&global_array);
+        x = i < array->size ? array->data[i] : -1;
+        rcu_read_unlock();
+        return x;
+
+    write side (running under a lock):
+        if (global_array->size == global_array->alloc) {
+            /* Creating a new version.  */
+            new_array = g_malloc(sizeof(struct arr) +
+                                 global_array->alloc * 2 * sizeof(T));
+            new_array->size = global_array->size;
+            new_array->alloc = global_array->alloc * 2;
+            memcpy(new_array->data, global_array->data,
+                   global_array->alloc * sizeof(T));
+
+            /* Removal phase.  */
+            old_array = global_array;
+            atomic_rcu_set(&new_array->data, new_array);
+            synchronize_rcu();
+
+            /* Reclamation phase.  */
+            free(old_array);
+        }
+
+
+SOURCES
+=======
+
+* Documentation/RCU/ from the Linux kernel
diff --git a/docs/replay.txt b/docs/replay.txt
new file mode 100644
index 0000000..779c6c0
--- /dev/null
+++ b/docs/replay.txt
@@ -0,0 +1,197 @@
+Copyright (c) 2010-2015 Institute for System Programming
+                        of the Russian Academy of Sciences.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.
+See the COPYING file in the top-level directory.
+
+Record/replay
+-------------
+
+Record/replay functions are used for the reverse execution and deterministic
+replay of qemu execution. This implementation of deterministic replay can
+be used for deterministic debugging of guest code through a gdb remote
+interface.
+
+Execution recording writes a non-deterministic events log, which can be later
+used for replaying the execution anywhere and for unlimited number of times.
+It also supports checkpointing for faster rewinding during reverse debugging.
+Execution replaying reads the log and replays all non-deterministic events
+including external input, hardware clocks, and interrupts.
+
+Deterministic replay has the following features:
+ * Deterministically replays whole system execution and all contents of
+   the memory, state of the hardware devices, clocks, and screen of the VM.
+ * Writes execution log into the file for later replaying for multiple times
+   on different machines.
+ * Supports i386, x86_64, and ARM hardware platforms.
+ * Performs deterministic replay of all operations with keyboard and mouse
+   input devices.
+
+Usage of the record/replay:
+ * First, record the execution, by adding the following arguments to the command line:
+   '-icount shift=7,rr=record,rrfile=replay.bin -net none'.
+   Block devices' images are not actually changed in the recording mode,
+   because all of the changes are written to the temporary overlay file.
+ * Then you can replay it by using another command
+   line option: '-icount shift=7,rr=replay,rrfile=replay.bin -net none'
+ * '-net none' option should also be specified if network replay patches
+   are not applied.
+
+Papers with description of deterministic replay implementation:
+http://www.computer.org/csdl/proceedings/csmr/2012/4666/00/4666a553-abs.html
+http://dl.acm.org/citation.cfm?id=2786805.2803179
+
+Modifications of qemu include:
+ * wrappers for clock and time functions to save their return values in the log
+ * saving different asynchronous events (e.g. system shutdown) into the log
+ * synchronization of the bottom halves execution
+ * synchronization of the threads from thread pool
+ * recording/replaying user input (mouse and keyboard)
+ * adding internal checkpoints for cpu and io synchronization
+
+Non-deterministic events
+------------------------
+
+Our record/replay system is based on saving and replaying non-deterministic
+events (e.g. keyboard input) and simulating deterministic ones (e.g. reading
+from HDD or memory of the VM). Saving only non-deterministic events makes
+log file smaller, simulation faster, and allows using reverse debugging even
+for realtime applications.
+
+The following non-deterministic data from peripheral devices is saved into
+the log: mouse and keyboard input, network packets, audio controller input,
+USB packets, serial port input, and hardware clocks (they are non-deterministic
+too, because their values are taken from the host machine). Inputs from
+simulated hardware, memory of VM, software interrupts, and execution of
+instructions are not saved into the log, because they are deterministic and
+can be replayed by simulating the behavior of virtual machine starting from
+initial state.
+
+We had to solve three tasks to implement deterministic replay: recording
+non-deterministic events, replaying non-deterministic events, and checking
+that there is no divergence between record and replay modes.
+
+We changed several parts of QEMU to make event log recording and replaying.
+Devices' models that have non-deterministic input from external devices were
+changed to write every external event into the execution log immediately.
+E.g. network packets are written into the log when they arrive into the virtual
+network adapter.
+
+All non-deterministic events are coming from these devices. But to
+replay them we need to know at which moments they occur. We specify
+these moments by counting the number of instructions executed between
+every pair of consecutive events.
+
+Instruction counting
+--------------------
+
+QEMU should work in icount mode to use record/replay feature. icount was
+designed to allow deterministic execution in absence of external inputs
+of the virtual machine. We also use icount to control the occurrence of the
+non-deterministic events. The number of instructions elapsed from the last event
+is written to the log while recording the execution. In replay mode we
+can predict when to inject that event using the instruction counter.
+
+Timers
+------
+
+Timers are used to execute callbacks from different subsystems of QEMU
+at the specified moments of time. There are several kinds of timers:
+ * Real time clock. Based on host time and used only for callbacks that
+   do not change the virtual machine state. For this reason real time
+   clock and timers does not affect deterministic replay at all.
+ * Virtual clock. These timers run only during the emulation. In icount
+   mode virtual clock value is calculated using executed instructions counter.
+   That is why it is completely deterministic and does not have to be recorded.
+ * Host clock. This clock is used by device models that simulate real time
+   sources (e.g. real time clock chip). Host clock is the one of the sources
+   of non-determinism. Host clock read operations should be logged to
+   make the execution deterministic.
+ * Virtual real time clock. This clock is similar to real time clock but
+   it is used only for increasing virtual clock while virtual machine is
+   sleeping. Due to its nature it is also non-deterministic as the host clock
+   and has to be logged too.
+
+Checkpoints
+-----------
+
+Replaying of the execution of virtual machine is bound by sources of
+non-determinism. These are inputs from clock and peripheral devices,
+and QEMU thread scheduling. Thread scheduling affect on processing events
+from timers, asynchronous input-output, and bottom halves.
+
+Invocations of timers are coupled with clock reads and changing the state
+of the virtual machine. Reads produce non-deterministic data taken from
+host clock. And VM state changes should preserve their order. Their relative
+order in replay mode must replicate the order of callbacks in record mode.
+To preserve this order we use checkpoints. When a specific clock is processed
+in record mode we save to the log special "checkpoint" event.
+Checkpoints here do not refer to virtual machine snapshots. They are just
+record/replay events used for synchronization.
+
+QEMU in replay mode will try to invoke timers processing in random moment
+of time. That's why we do not process a group of timers until the checkpoint
+event will be read from the log. Such an event allows synchronizing CPU
+execution and timer events.
+
+Two other checkpoints govern the "warping" of the virtual clock.
+While the virtual machine is idle, the virtual clock increments at
+1 ns per *real time* nanosecond.  This is done by setting up a timer
+(called the warp timer) on the virtual real time clock, so that the
+timer fires at the next deadline of the virtual clock; the virtual clock
+is then incremented (which is called "warping" the virtual clock) as
+soon as the timer fires or the CPUs need to go out of the idle state.
+Two functions are used for this purpose; because these actions change
+virtual machine state and must be deterministic, each of them creates a
+checkpoint.  qemu_start_warp_timer checks if the CPUs are idle and if so
+starts accounting real time to virtual clock.  qemu_account_warp_timer
+is called when the CPUs get an interrupt or when the warp timer fires,
+and it warps the virtual clock by the amount of real time that has passed
+since qemu_start_warp_timer.
+
+Bottom halves
+-------------
+
+Disk I/O events are completely deterministic in our model, because
+in both record and replay modes we start virtual machine from the same
+disk state. But callbacks that virtual disk controller uses for reading and
+writing the disk may occur at different moments of time in record and replay
+modes.
+
+Reading and writing requests are created by CPU thread of QEMU. Later these
+requests proceed to block layer which creates "bottom halves". Bottom
+halves consist of callback and its parameters. They are processed when
+main loop locks the global mutex. These locks are not synchronized with
+replaying process because main loop also processes the events that do not
+affect the virtual machine state (like user interaction with monitor).
+
+That is why we had to implement saving and replaying bottom halves callbacks
+synchronously to the CPU execution. When the callback is about to execute
+it is added to the queue in the replay module. This queue is written to the
+log when its callbacks are executed. In replay mode callbacks are not processed
+until the corresponding event is read from the events log file.
+
+Sometimes the block layer uses asynchronous callbacks for its internal purposes
+(like reading or writing VM snapshots or disk image cluster tables). In this
+case bottom halves are not marked as "replayable" and do not saved
+into the log.
+
+Block devices
+-------------
+
+Block devices record/replay module intercepts calls of
+bdrv coroutine functions at the top of block drivers stack.
+To record and replay block operations the drive must be configured
+as following:
+ -drive file=disk.qcow,if=none,id=img-direct
+ -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay
+ -device ide-hd,drive=img-blkreplay
+
+blkreplay driver should be inserted between disk image and virtual driver
+controller. Therefore all disk requests may be recorded and replayed.
+
+All block completion operations are added to the queue in the coroutines.
+Queue is flushed at checkpoints and information about processed requests
+is recorded to the log. In replay phase the queue is matched with
+events read from the log. Therefore block devices requests are processed
+deterministically.
diff --git a/docs/specs/acpi_cpu_hotplug.txt b/docs/specs/acpi_cpu_hotplug.txt
index 340b751..ee219c8 100644
--- a/docs/specs/acpi_cpu_hotplug.txt
+++ b/docs/specs/acpi_cpu_hotplug.txt
@@ -4,21 +4,91 @@
 QEMU supports CPU hotplug via ACPI. This document
 describes the interface between QEMU and the ACPI BIOS.
 
-ACPI GPE block (IO ports 0xafe0-0xafe3, byte access):
------------------------------------------
+ACPI BIOS GPE.2 handler is dedicated for notifying OS about CPU hot-add
+and hot-remove events.
 
-Generic ACPI GPE block. Bit 2 (GPE.2) used to notify CPU
-hot-add/remove event to ACPI BIOS, via SCI interrupt.
-
+============================================
+Legacy ACPI CPU hotplug interface registers:
+--------------------------------------------
 CPU present bitmap for:
   ICH9-LPC (IO port 0x0cd8-0xcf7, 1-byte access)
   PIIX-PM  (IO port 0xaf00-0xaf1f, 1-byte access)
+  One bit per CPU. Bit position reflects corresponding CPU APIC ID. Read-only.
+  The first DWORD in bitmap is used in write mode to switch from legacy
+  to new CPU hotplug interface, write 0 into it to do switch.
 ---------------------------------------------------------------
-One bit per CPU. Bit position reflects corresponding CPU APIC ID.
-Read-only.
+QEMU sets corresponding CPU bit on hot-add event and issues SCI
+with GPE.2 event set. CPU present map is read by ACPI BIOS GPE.2 handler
+to notify OS about CPU hot-add events. CPU hot-remove isn't supported.
 
-CPU hot-add/remove notification:
------------------------------------------------------
-QEMU sets/clears corresponding CPU bit on hot-add/remove event.
-CPU present map read by ACPI BIOS GPE.2 handler to notify OS of CPU
-hot-(un)plug events.
+=====================================
+ACPI CPU hotplug interface registers:
+-------------------------------------
+Register block base address:
+    ICH9-LPC IO port 0x0cd8
+    PIIX-PM  IO port 0xaf00
+Register block size:
+    ACPI_CPU_HOTPLUG_REG_LEN = 12
+
+read access:
+    offset:
+    [0x0-0x3] reserved
+    [0x4] CPU device status fields: (1 byte access)
+        bits:
+           0: Device is enabled and may be used by guest
+           1: Device insert event, used to distinguish device for which
+              no device check event to OSPM was issued.
+              It's valid only when bit 0 is set.
+           2: Device remove event, used to distinguish device for which
+              no device eject request to OSPM was issued.
+           3-7: reserved and should be ignored by OSPM
+    [0x5-0x7] reserved
+    [0x8] Command data: (DWORD access)
+          in case of error or unsupported command reads is 0xFFFFFFFF
+          current 'Command field' value:
+              0: returns PXM value corresponding to device
+
+write access:
+    offset:
+    [0x0-0x3] CPU selector: (DWORD access)
+              selects active CPU device. All following accesses to other
+              registers will read/store data from/to selected CPU.
+    [0x4] CPU device control fields: (1 byte access)
+        bits:
+            0: reserved, OSPM must clear it before writing to register.
+            1: if set to 1 clears device insert event, set by OSPM
+               after it has emitted device check event for the
+               selected CPU device
+            2: if set to 1 clears device remove event, set by OSPM
+               after it has emitted device eject request for the
+               selected CPU device
+            3: if set to 1 initiates device eject, set by OSPM when it
+               triggers CPU device removal and calls _EJ0 method
+            4-7: reserved, OSPM must clear them before writing to register
+    [0x5] Command field: (1 byte access)
+          value:
+            0: selects a CPU device with inserting/removing events and
+               following reads from 'Command data' register return
+               selected CPU (CPU selector value). If no CPU with events
+               found, the current CPU selector doesn't change and
+               corresponding insert/remove event flags are not set.
+            1: following writes to 'Command data' register set OST event
+               register in QEMU
+            2: following writes to 'Command data' register set OST status
+               register in QEMU
+            other values: reserved
+    [0x6-0x7] reserved
+    [0x8] Command data: (DWORD access)
+          current 'Command field' value:
+              0: OSPM reads value of CPU selector
+              1: stores value into OST event register
+              2: stores value into OST status register, triggers
+                 ACPI_DEVICE_OST QMP event from QEMU to external applications
+                 with current values of OST event and status registers.
+            other values: reserved
+
+Selecting CPU device beyond possible range has no effect on platform:
+   - write accesses to CPU hot-plug registers not documented above are
+     ignored
+   - read accesses to CPU hot-plug registers not documented above return
+     all bits set to 0.
diff --git a/docs/specs/acpi_mem_hotplug.txt b/docs/specs/acpi_mem_hotplug.txt
index 1290994..3df3620 100644
--- a/docs/specs/acpi_mem_hotplug.txt
+++ b/docs/specs/acpi_mem_hotplug.txt
@@ -2,7 +2,7 @@
 --------------------------------------
 
 ACPI BIOS GPE.3 handler is dedicated for notifying OS about memory hot-add
-events.
+and hot-remove events.
 
 Memory hot-plug interface (IO port 0xa00-0xa17, 1-4 byte access):
 ---------------------------------------------------------------
@@ -19,7 +19,9 @@
               1: Device insert event, used to distinguish device for which
                  no device check event to OSPM was issued.
                  It's valid only when bit 1 is set.
-              2-7: reserved and should be ignored by OSPM
+              2: Device remove event, used to distinguish device for which
+                 no device eject request to OSPM was issued.
+              3-7: reserved and should be ignored by OSPM
       [0x15-0x17] reserved
 
   write access:
@@ -31,14 +33,62 @@
       [0xc-0x13] reserved, writes into it are ignored
       [0x14] Memory device control fields
           bits:
-              0: reserved, OSPM must clear it before writing to register
+              0: reserved, OSPM must clear it before writing to register.
+                 Due to BUG in versions prior 2.4 that field isn't cleared
+                 when other fields are written. Keep it reserved and don't
+                 try to reuse it.
               1: if set to 1 clears device insert event, set by OSPM
                  after it has emitted device check event for the
                  selected memory device
-              2-7: reserved, OSPM must clear them before writing to register
+              2: if set to 1 clears device remove event, set by OSPM
+                 after it has emitted device eject request for the
+                 selected memory device
+              3: if set to 1 initiates device eject, set by OSPM when it
+                 triggers memory device removal and calls _EJ0 method
+              4-7: reserved, OSPM must clear them before writing to register
 
 Selecting memory device slot beyond present range has no effect on platform:
    - write accesses to memory hot-plug registers not documented above are
      ignored
    - read accesses to memory hot-plug registers not documented above return
      all bits set to 1.
+
+Memory hot remove process diagram:
+----------------------------------
+ +-------------+     +-----------------------+      +------------------+     
+ |  1. QEMU    |     | 2. QEMU               |      |3. QEMU           |     
+ |  device_del +---->+ device unplug request +----->+Send SCI to guest,|     
+ |             |     |         cb            |      |return control to |     
+ +-------------+     +-----------------------+      |management        |     
+                                                    +------------------+     
+                                                                             
+ +---------------------------------------------------------------------+     
+                                                                             
+ +---------------------+              +-------------------------+            
+ | OSPM:               | remove event | OSPM:                   |            
+ | send Eject Request, |              | Scan memory devices     |            
+ | clear remove event  +<-------------+ for event flags         |            
+ |                     |              |                         |            
+ +---------------------+              +-------------------------+            
+           |                                                                 
+           |                                                                 
+ +---------v--------+            +-----------------------+                   
+ | Guest OS:        |  success   | OSPM:                 |                   
+ | process Ejection +----------->+ Execute _EJ0 method,  |                   
+ | request          |            | set eject bit in flags|                   
+ +------------------+            +-----------------------+                   
+           |failure                         |                                
+           v                                v                                
+ +------------------------+      +-----------------------+                   
+ | OSPM:                  |      | QEMU:                 |                   
+ | set OST event & status |      | call device unplug cb |                   
+ | fields                 |      |                       |                   
+ +------------------------+      +-----------------------+                   
+          |                                  |                               
+          v                                  v                               
+ +------------------+              +-------------------+                     
+ |QEMU:             |              |QEMU:              |                     
+ |Send OST QMP event|              |Send device deleted|                     
+ |                  |              |QMP event          |                     
+ +------------------+              |                   |                     
+                                   +-------------------+
diff --git a/docs/specs/acpi_nvdimm.txt b/docs/specs/acpi_nvdimm.txt
new file mode 100644
index 0000000..0fdd251
--- /dev/null
+++ b/docs/specs/acpi_nvdimm.txt
@@ -0,0 +1,132 @@
+QEMU<->ACPI BIOS NVDIMM interface
+---------------------------------
+
+QEMU supports NVDIMM via ACPI. This document describes the basic concepts of
+NVDIMM ACPI and the interface between QEMU and the ACPI BIOS.
+
+NVDIMM ACPI Background
+----------------------
+NVDIMM is introduced in ACPI 6.0 which defines an NVDIMM root device under
+_SB scope with a _HID of “ACPI0012”. For each NVDIMM present or intended
+to be supported by platform, platform firmware also exposes an ACPI
+Namespace Device under the root device.
+
+The NVDIMM child devices under the NVDIMM root device are defined with _ADR
+corresponding to the NFIT device handle. The NVDIMM root device and the
+NVDIMM devices can have device specific methods (_DSM) to provide additional
+functions specific to a particular NVDIMM implementation.
+
+This is an example from ACPI 6.0, a platform contains one NVDIMM:
+
+Scope (\_SB){
+   Device (NVDR) // Root device
+   {
+      Name (_HID, “ACPI0012”)
+      Method (_STA) {...}
+      Method (_FIT) {...}
+      Method (_DSM, ...) {...}
+      Device (NVD)
+      {
+         Name(_ADR, h) //where h is NFIT Device Handle for this NVDIMM
+         Method (_DSM, ...) {...}
+      }
+   }
+}
+
+Method supported on both NVDIMM root device and NVDIMM device
+_DSM (Device Specific Method)
+   It is a control method that enables devices to provide device specific
+   control functions that are consumed by the device driver.
+   The NVDIMM DSM specification can be found at:
+        http://pmem.io/documents/NVDIMM_DSM_Interface_Example.pdf
+
+   Arguments:
+   Arg0 – A Buffer containing a UUID (16 Bytes)
+   Arg1 – An Integer containing the Revision ID (4 Bytes)
+   Arg2 – An Integer containing the Function Index (4 Bytes)
+   Arg3 – A package containing parameters for the function specified by the
+          UUID, Revision ID, and Function Index
+
+   Return Value:
+   If Function Index = 0, a Buffer containing a function index bitfield.
+   Otherwise, the return value and type depends on the UUID, revision ID
+   and function index which are described in the DSM specification.
+
+Methods on NVDIMM ROOT Device
+_FIT(Firmware Interface Table)
+   It evaluates to a buffer returning data in the format of a series of NFIT
+   Type Structure.
+
+   Arguments: None
+
+   Return Value:
+   A Buffer containing a list of NFIT Type structure entries.
+
+   The detailed definition of the structure can be found at ACPI 6.0: 5.2.25
+   NVDIMM Firmware Interface Table (NFIT).
+
+QEMU NVDIMM Implemention
+========================
+QEMU uses 4 bytes IO Port starting from 0x0a18 and a RAM-based memory page
+for NVDIMM ACPI.
+
+Memory:
+   QEMU uses BIOS Linker/loader feature to ask BIOS to allocate a memory
+   page and dynamically patch its into a int32 object named "MEMA" in ACPI.
+
+   This page is RAM-based and it is used to transfer data between _DSM
+   method and QEMU. If ACPI has control, this pages is owned by ACPI which
+   writes _DSM input data to it, otherwise, it is owned by QEMU which
+   emulates _DSM access and writes the output data to it.
+
+   ACPI writes _DSM Input Data (based on the offset in the page):
+   [0x0 - 0x3]: 4 bytes, NVDIMM Device Handle, 0 is reserved for NVDIMM
+                Root device.
+   [0x4 - 0x7]: 4 bytes, Revision ID, that is the Arg1 of _DSM method.
+   [0x8 - 0xB]: 4 bytes. Function Index, that is the Arg2 of _DSM method.
+   [0xC - 0xFFF]: 4084 bytes, the Arg3 of _DSM method.
+
+   QEMU Writes Output Data (based on the offset in the page):
+   [0x0 - 0x3]: 4 bytes, the length of result
+   [0x4 - 0xFFF]: 4092 bytes, the DSM result filled by QEMU
+
+IO Port 0x0a18 - 0xa1b:
+   ACPI writes the address of the memory page allocated by BIOS to this
+   port then QEMU gets the control and fills the result in the memory page.
+
+   write Access:
+   [0x0a18 - 0xa1b]: 4 bytes, the address of the memory page allocated
+                     by BIOS.
+
+_DSM process diagram:
+---------------------
+"MEMA" indicates the address of memory page allocated by BIOS.
+
+ +----------------------+      +-----------------------+
+ |    1. OSPM           |      |    2. OSPM            |
+ | save _DSM input data |      |  write "MEMA" to      | Exit to QEMU
+ | to the page          +----->|  IO port 0x0a18       +------------+
+ | indicated by "MEMA"  |      |                       |            |
+ +----------------------+      +-----------------------+            |
+                                                                    |
+                                                                    v
+ +-------------   ----+       +-----------+      +------------------+--------+
+ |      5 QEMU        |       | 4 QEMU    |      |        3. QEMU            |
+ | write _DSM result  |       |  emulate  |      | get _DSM input data from  |
+ | to the page        +<------+ _DSM      +<-----+ the page indicated by the |
+ |                    |       |           |      | value from the IO port    |
+ +--------+-----------+       +-----------+      +---------------------------+
+          |
+          | Enter Guest
+          |
+          v
+ +--------------------------+      +--------------+
+ |     6 OSPM               |      |   7 OSPM     |
+ | result size is returned  |      |  _DSM return |
+ | by reading  DSM          +----->+              |
+ | result from the page     |      |              |
+ +--------------------------+      +--------------+
+
+ _FIT implementation
+ -------------------
+ TODO (will fill it when nvdimm hotplug is introduced)
diff --git a/docs/specs/edu.txt b/docs/specs/edu.txt
new file mode 100644
index 0000000..7f81467
--- /dev/null
+++ b/docs/specs/edu.txt
@@ -0,0 +1,110 @@
+
+EDU device
+==========
+
+Copyright (c) 2014-2015 Jiri Slaby
+
+This document is licensed under the GPLv2 (or later).
+
+This is an educational device for writing (kernel) drivers. Its original
+intention was to support the Linux kernel lectures taught at the Masaryk
+University. Students are given this virtual device and are expected to write a
+driver with I/Os, IRQs, DMAs and such.
+
+The devices behaves very similar to the PCI bridge present in the COMBO6 cards
+developed under the Liberouter wings. Both PCI device ID and PCI space is
+inherited from that device.
+
+Command line switches:
+    -device edu[,dma_mask=mask]
+
+    dma_mask makes the virtual device work with DMA addresses with the given
+    mask. For educational purposes, the device supports only 28 bits (256 MiB)
+    by default. Students shall set dma_mask for the device in the OS driver
+    properly.
+
+PCI specs
+---------
+
+PCI ID: 1234:11e8
+
+PCI Region 0:
+   I/O memory, 1 MB in size. Users are supposed to communicate with the card
+   through this memory.
+
+MMIO area spec
+--------------
+
+Only size == 4 accesses are allowed for addresses < 0x80. size == 4 or
+size == 8 for the rest.
+
+0x00 (RO) : identification (0xRRrr00edu)
+	    RR -- major version
+	    rr -- minor version
+
+0x04 (RW) : card liveness check
+	    It is a simple value inversion (~ C operator).
+
+0x08 (RW) : factorial computation
+	    The stored value is taken and factorial of it is put back here.
+	    This happens only after factorial bit in the status register (0x20
+	    below) is cleared.
+
+0x20 (RW) : status register, bitwise OR
+	    0x01 -- computing factorial (RO)
+	    0x80 -- raise interrupt 0x01 after finishing factorial computation
+
+0x24 (RO) : interrupt status register
+	    It contains values which raised the interrupt (see interrupt raise
+	    register below).
+
+0x60 (WO) : interrupt raise register
+	    Raise an interrupt. The value will be put to the interrupt status
+	    register (using bitwise OR).
+
+0x64 (WO) : interrupt acknowledge register
+	    Clear an interrupt. The value will be cleared from the interrupt
+	    status register. This needs to be done from the ISR to stop
+	    generating interrupts.
+
+0x80 (RW) : DMA source address
+	    Where to perform the DMA from.
+
+0x88 (RW) : DMA destination address
+	    Where to perform the DMA to.
+
+0x90 (RW) : DMA transfer count
+	    The size of the area to perform the DMA on.
+
+0x98 (RW) : DMA command register, bitwise OR
+	    0x01 -- start transfer
+	    0x02 -- direction (0: from RAM to EDU, 1: from EDU to RAM)
+	    0x04 -- raise interrupt 0x100 after finishing the DMA
+
+IRQ controller
+--------------
+An IRQ is generated when written to the interrupt raise register. The value
+appears in interrupt status register when the interrupt is raised and has to
+be written to the interrupt acknowledge register to lower it.
+
+DMA controller
+--------------
+One has to specify, source, destination, size, and start the transfer. One
+4096 bytes long buffer at offset 0x40000 is available in the EDU device. I.e.
+one can perform DMA to/from this space when programmed properly.
+
+Example of transferring a 100 byte block to and from the buffer using a given
+PCI address 'addr':
+addr     -> DMA source address
+0x40000  -> DMA destination address
+100      -> DMA transfer count
+1        -> DMA command register
+while (DMA command register & 1)
+	;
+
+0x40000  -> DMA source address
+addr+100 -> DMA destination address
+100      -> DMA transfer count
+3        -> DMA command register
+while (DMA command register & 1)
+	;
diff --git a/docs/specs/fw_cfg.txt b/docs/specs/fw_cfg.txt
new file mode 100644
index 0000000..7a5f8c7
--- /dev/null
+++ b/docs/specs/fw_cfg.txt
@@ -0,0 +1,236 @@
+QEMU Firmware Configuration (fw_cfg) Device
+===========================================
+
+= Guest-side Hardware Interface =
+
+This hardware interface allows the guest to retrieve various data items
+(blobs) that can influence how the firmware configures itself, or may
+contain tables to be installed for the guest OS. Examples include device
+boot order, ACPI and SMBIOS tables, virtual machine UUID, SMP and NUMA
+information, kernel/initrd images for direct (Linux) kernel booting, etc.
+
+== Selector (Control) Register ==
+
+* Write only
+* Location: platform dependent (IOport or MMIO)
+* Width: 16-bit
+* Endianness: little-endian (if IOport), or big-endian (if MMIO)
+
+A write to this register sets the index of a firmware configuration
+item which can subsequently be accessed via the data register.
+
+Setting the selector register will cause the data offset to be set
+to zero. The data offset impacts which data is accessed via the data
+register, and is explained below.
+
+Bit14 of the selector register indicates whether the configuration
+setting is being written. A value of 0 means the item is only being
+read, and all write access to the data port will be ignored. A value
+of 1 means the item's data can be overwritten by writes to the data
+register. In other words, configuration write mode is enabled when
+the selector value is between 0x4000-0x7fff or 0xc000-0xffff.
+
+NOTE: As of QEMU v2.4, writes to the fw_cfg data register are no
+      longer supported, and will be ignored (treated as no-ops)!
+
+Bit15 of the selector register indicates whether the configuration
+setting is architecture specific. A value of 0 means the item is a
+generic configuration item. A value of 1 means the item is specific
+to a particular architecture. In other words, generic configuration
+items are accessed with a selector value between 0x0000-0x7fff, and
+architecture specific configuration items are accessed with a selector
+value between 0x8000-0xffff.
+
+== Data Register ==
+
+* Read/Write (writes ignored as of QEMU v2.4)
+* Location: platform dependent (IOport [*] or MMIO)
+* Width: 8-bit (if IOport), 8/16/32/64-bit (if MMIO)
+* Endianness: string-preserving
+
+[*] On platforms where the data register is exposed as an IOport, its
+port number will always be one greater than the port number of the
+selector register. In other words, the two ports overlap, and can not
+be mapped separately.
+
+The data register allows access to an array of bytes for each firmware
+configuration data item. The specific item is selected by writing to
+the selector register, as described above.
+
+Initially following a write to the selector register, the data offset
+will be set to zero. Each successful access to the data register will
+increment the data offset by the appropriate access width.
+
+Each firmware configuration item has a maximum length of data
+associated with the item. After the data offset has passed the
+end of this maximum data length, then any reads will return a data
+value of 0x00, and all writes will be ignored.
+
+An N-byte wide read of the data register will return the next available
+N bytes of the selected firmware configuration item, as a substring, in
+increasing address order, similar to memcpy().
+
+== Register Locations ==
+
+=== x86, x86_64 Register Locations ===
+
+Selector Register IOport: 0x510
+Data Register IOport:     0x511
+DMA Address IOport:       0x514
+
+=== ARM Register Locations ===
+
+Selector Register address: Base + 8 (2 bytes)
+Data Register address:     Base + 0 (8 bytes)
+DMA Address address:       Base + 16 (8 bytes)
+
+== ACPI Interface ==
+
+The fw_cfg device is defined with ACPI ID "QEMU0002". Since we expect
+ACPI tables to be passed into the guest through the fw_cfg device itself,
+the guest-side firmware can not use ACPI to find fw_cfg. However, once the
+firmware is finished setting up ACPI tables and hands control over to the
+guest kernel, the latter can use the fw_cfg ACPI node for a more accurate
+inventory of in-use IOport or MMIO regions.
+
+== Firmware Configuration Items ==
+
+=== Signature (Key 0x0000, FW_CFG_SIGNATURE) ===
+
+The presence of the fw_cfg selector and data registers can be verified
+by selecting the "signature" item using key 0x0000 (FW_CFG_SIGNATURE),
+and reading four bytes from the data register. If the fw_cfg device is
+present, the four bytes read will contain the characters "QEMU".
+
+If the DMA interface is available, then reading the DMA Address
+Register returns 0x51454d5520434647 ("QEMU CFG" in big-endian format).
+
+=== Revision / feature bitmap (Key 0x0001, FW_CFG_ID) ===
+
+A 32-bit little-endian unsigned int, this item is used to check for enabled
+features.
+ - Bit 0: traditional interface. Always set.
+ - Bit 1: DMA interface.
+
+=== File Directory (Key 0x0019, FW_CFG_FILE_DIR) ===
+
+Firmware configuration items stored at selector keys 0x0020 or higher
+(FW_CFG_FILE_FIRST or higher) have an associated entry in a directory
+structure, which makes it easier for guest-side firmware to identify
+and retrieve them. The format of this file directory (from fw_cfg.h in
+the QEMU source tree) is shown here, slightly annotated for clarity:
+
+struct FWCfgFiles {		/* the entire file directory fw_cfg item */
+    uint32_t count;		/* number of entries, in big-endian format */
+    struct FWCfgFile f[];	/* array of file entries, see below */
+};
+
+struct FWCfgFile {		/* an individual file entry, 64 bytes total */
+    uint32_t size;		/* size of referenced fw_cfg item, big-endian */
+    uint16_t select;		/* selector key of fw_cfg item, big-endian */
+    uint16_t reserved;
+    char name[56];		/* fw_cfg item name, NUL-terminated ascii */
+};
+
+=== All Other Data Items ===
+
+Please consult the QEMU source for the most up-to-date and authoritative
+list of selector keys and their respective items' purpose and format.
+
+=== Ranges ===
+
+Theoretically, there may be up to 0x4000 generic firmware configuration
+items, and up to 0x4000 architecturally specific ones.
+
+Selector Reg.    Range Usage
+---------------  -----------
+0x0000 - 0x3fff  Generic (0x0000 - 0x3fff, RO)
+0x4000 - 0x7fff  Generic (0x0000 - 0x3fff, RW, ignored in QEMU v2.4+)
+0x8000 - 0xbfff  Arch. Specific (0x0000 - 0x3fff, RO)
+0xc000 - 0xffff  Arch. Specific (0x0000 - 0x3fff, RW, ignored in v2.4+)
+
+In practice, the number of allowed firmware configuration items is given
+by the value of FW_CFG_MAX_ENTRY (see fw_cfg.h).
+
+= Guest-side DMA Interface =
+
+If bit 1 of the feature bitmap is set, the DMA interface is present. This does
+not replace the existing fw_cfg interface, it is an add-on. This interface
+can be used through the 64-bit wide address register.
+
+The address register is in big-endian format. The value for the register is 0
+at startup and after an operation. A write to the least significant half (at
+offset 4) triggers an operation. This means that operations with 32-bit
+addresses can be triggered with just one write, whereas operations with
+64-bit addresses can be triggered with one 64-bit write or two 32-bit writes,
+starting with the most significant half (at offset 0).
+
+In this register, the physical address of a FWCfgDmaAccess structure in RAM
+should be written. This is the format of the FWCfgDmaAccess structure:
+
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} FWCfgDmaAccess;
+
+The fields of the structure are in big endian mode, and the field at the lowest
+address is the "control" field.
+
+The "control" field has the following bits:
+ - Bit 0: Error
+ - Bit 1: Read
+ - Bit 2: Skip
+ - Bit 3: Select. The upper 16 bits are the selected index.
+
+When an operation is triggered, if the "control" field has bit 3 set, the
+upper 16 bits are interpreted as an index of a firmware configuration item.
+This has the same effect as writing the selector register.
+
+If the "control" field has bit 1 set, a read operation will be performed.
+"length" bytes for the current selector and offset will be copied into the
+physical RAM address specified by the "address" field.
+
+If the "control" field has bit 2 set (and not bit 1), a skip operation will be
+performed. The offset for the current selector will be advanced "length" bytes.
+
+To check the result, read the "control" field:
+   error bit set        ->  something went wrong.
+   all bits cleared     ->  transfer finished successfully.
+   otherwise            ->  transfer still in progress (doesn't happen
+                            today due to implementation not being async,
+                            but may in the future).
+
+= Externally Provided Items =
+
+As of v2.4, "file" fw_cfg items (i.e., items with selector keys above
+FW_CFG_FILE_FIRST, and with a corresponding entry in the fw_cfg file
+directory structure) may be inserted via the QEMU command line, using
+the following syntax:
+
+    -fw_cfg [name=]<item_name>,file=<path>
+
+Or
+
+    -fw_cfg [name=]<item_name>,string=<string>
+
+See QEMU man page for more documentation.
+
+Using item_name with plain ASCII characters only is recommended.
+
+Item names beginning with "opt/" are reserved for users.  QEMU will
+never create entries with such names unless explicitly ordered by the
+user.
+
+To avoid clashes among different users, it is strongly recommended
+that you use names beginning with opt/RFQDN/, where RFQDN is a reverse
+fully qualified domain name you control.  For instance, if SeaBIOS
+wanted to define additional names, the prefix "opt/org.seabios/" would
+be appropriate.
+
+For historical reasons, "opt/ovmf/" is reserved for OVMF firmware.
+
+Prefix "opt/org.qemu/" is reserved for QEMU itself.
+
+Use of names not beginning with "opt/" is potentially dangerous and
+entirely unsupported.  QEMU will warn if you try.
diff --git a/docs/specs/ivshmem-spec.txt b/docs/specs/ivshmem-spec.txt
new file mode 100644
index 0000000..a1f5499
--- /dev/null
+++ b/docs/specs/ivshmem-spec.txt
@@ -0,0 +1,254 @@
+= Device Specification for Inter-VM shared memory device =
+
+The Inter-VM shared memory device (ivshmem) is designed to share a
+memory region between multiple QEMU processes running different guests
+and the host.  In order for all guests to be able to pick up the
+shared memory area, it is modeled by QEMU as a PCI device exposing
+said memory to the guest as a PCI BAR.
+
+The device can use a shared memory object on the host directly, or it
+can obtain one from an ivshmem server.
+
+In the latter case, the device can additionally interrupt its peers, and
+get interrupted by its peers.
+
+
+== Configuring the ivshmem PCI device ==
+
+There are two basic configurations:
+
+- Just shared memory: -device ivshmem-plain,memdev=HMB,...
+
+  This uses host memory backend HMB.  It should have option "share"
+  set.
+
+- Shared memory plus interrupts: -device ivshmem,chardev=CHR,vectors=N,...
+
+  An ivshmem server must already be running on the host.  The device
+  connects to the server's UNIX domain socket via character device
+  CHR.
+
+  Each peer gets assigned a unique ID by the server.  IDs must be
+  between 0 and 65535.
+
+  Interrupts are message-signaled (MSI-X).  vectors=N configures the
+  number of vectors to use.
+
+For more details on ivshmem device properties, see The QEMU Emulator
+User Documentation (qemu-doc.*).
+
+
+== The ivshmem PCI device's guest interface ==
+
+The device has vendor ID 1af4, device ID 1110, revision 1.  Before
+QEMU 2.6.0, it had revision 0.
+
+=== PCI BARs ===
+
+The ivshmem PCI device has two or three BARs:
+
+- BAR0 holds device registers (256 Byte MMIO)
+- BAR1 holds MSI-X table and PBA (only ivshmem-doorbell)
+- BAR2 maps the shared memory object
+
+There are two ways to use this device:
+
+- If you only need the shared memory part, BAR2 suffices.  This way,
+  you have access to the shared memory in the guest and can use it as
+  you see fit.  Memnic, for example, uses ivshmem this way from guest
+  user space (see http://dpdk.org/browse/memnic).
+
+- If you additionally need the capability for peers to interrupt each
+  other, you need BAR0 and BAR1.  You will most likely want to write a
+  kernel driver to handle interrupts.  Requires the device to be
+  configured for interrupts, obviously.
+
+Before QEMU 2.6.0, BAR2 can initially be invalid if the device is
+configured for interrupts.  It becomes safely accessible only after
+the ivshmem server provided the shared memory.  These devices have PCI
+revision 0 rather than 1.  Guest software should wait for the
+IVPosition register (described below) to become non-negative before
+accessing BAR2.
+
+Revision 0 of the device is not capable to tell guest software whether
+it is configured for interrupts.
+
+=== PCI device registers ===
+
+BAR 0 contains the following registers:
+
+    Offset  Size  Access      On reset  Function
+        0     4   read/write        0   Interrupt Mask
+                                        bit 0: peer interrupt (rev 0)
+                                               reserved       (rev 1)
+                                        bit 1..31: reserved
+        4     4   read/write        0   Interrupt Status
+                                        bit 0: peer interrupt (rev 0)
+                                               reserved       (rev 1)
+                                        bit 1..31: reserved
+        8     4   read-only   0 or ID   IVPosition
+       12     4   write-only      N/A   Doorbell
+                                        bit 0..15: vector
+                                        bit 16..31: peer ID
+       16   240   none            N/A   reserved
+
+Software should only access the registers as specified in column
+"Access".  Reserved bits should be ignored on read, and preserved on
+write.
+
+In revision 0 of the device, Interrupt Status and Mask Register
+together control the legacy INTx interrupt when the device has no
+MSI-X capability: INTx is asserted when the bit-wise AND of Status and
+Mask is non-zero and the device has no MSI-X capability.  Interrupt
+Status Register bit 0 becomes 1 when an interrupt request from a peer
+is received.  Reading the register clears it.
+
+IVPosition Register: if the device is not configured for interrupts,
+this is zero.  Else, it is the device's ID (between 0 and 65535).
+
+Before QEMU 2.6.0, the register may read -1 for a short while after
+reset.  These devices have PCI revision 0 rather than 1.
+
+There is no good way for software to find out whether the device is
+configured for interrupts.  A positive IVPosition means interrupts,
+but zero could be either.
+
+Doorbell Register: writing this register requests to interrupt a peer.
+The written value's high 16 bits are the ID of the peer to interrupt,
+and its low 16 bits select an interrupt vector.
+
+If the device is not configured for interrupts, the write is ignored.
+
+If the interrupt hasn't completed setup, the write is ignored.  The
+device is not capable to tell guest software whether setup is
+complete.  Interrupts can regress to this state on migration.
+
+If the peer with the requested ID isn't connected, or it has fewer
+interrupt vectors connected, the write is ignored.  The device is not
+capable to tell guest software what peers are connected, or how many
+interrupt vectors are connected.
+
+The peer's interrupt for this vector then becomes pending.  There is
+no way for software to clear the pending bit, and a polling mode of
+operation is therefore impossible.
+
+If the peer is a revision 0 device without MSI-X capability, its
+Interrupt Status register is set to 1.  This asserts INTx unless
+masked by the Interrupt Mask register.  The device is not capable to
+communicate the interrupt vector to guest software then.
+
+With multiple MSI-X vectors, different vectors can be used to indicate
+different events have occurred.  The semantics of interrupt vectors
+are left to the application.
+
+
+== Interrupt infrastructure ==
+
+When configured for interrupts, the peers share eventfd objects in
+addition to shared memory.  The shared resources are managed by an
+ivshmem server.
+
+=== The ivshmem server ===
+
+The server listens on a UNIX domain socket.
+
+For each new client that connects to the server, the server
+- picks an ID,
+- creates eventfd file descriptors for the interrupt vectors,
+- sends the ID and the file descriptor for the shared memory to the
+  new client,
+- sends connect notifications for the new client to the other clients
+  (these contain file descriptors for sending interrupts),
+- sends connect notifications for the other clients to the new client,
+  and
+- sends interrupt setup messages to the new client (these contain file
+  descriptors for receiving interrupts).
+
+The first client to connect to the server receives ID zero.
+
+When a client disconnects from the server, the server sends disconnect
+notifications to the other clients.
+
+The next section describes the protocol in detail.
+
+If the server terminates without sending disconnect notifications for
+its connected clients, the clients can elect to continue.  They can
+communicate with each other normally, but won't receive disconnect
+notification on disconnect, and no new clients can connect.  There is
+no way for the clients to connect to a restarted server.  The device
+is not capable to tell guest software whether the server is still up.
+
+Example server code is in contrib/ivshmem-server/.  Not to be used in
+production.  It assumes all clients use the same number of interrupt
+vectors.
+
+A standalone client is in contrib/ivshmem-client/.  It can be useful
+for debugging.
+
+=== The ivshmem Client-Server Protocol ===
+
+An ivshmem device configured for interrupts connects to an ivshmem
+server.  This section details the protocol between the two.
+
+The connection is one-way: the server sends messages to the client.
+Each message consists of a single 8 byte little-endian signed number,
+and may be accompanied by a file descriptor via SCM_RIGHTS.  Both
+client and server close the connection on error.
+
+Note: QEMU currently doesn't close the connection right on error, but
+only when the character device is destroyed.
+
+On connect, the server sends the following messages in order:
+
+1. The protocol version number, currently zero.  The client should
+   close the connection on receipt of versions it can't handle.
+
+2. The client's ID.  This is unique among all clients of this server.
+   IDs must be between 0 and 65535, because the Doorbell register
+   provides only 16 bits for them.
+
+3. The number -1, accompanied by the file descriptor for the shared
+   memory.
+
+4. Connect notifications for existing other clients, if any.  This is
+   a peer ID (number between 0 and 65535 other than the client's ID),
+   repeated N times.  Each repetition is accompanied by one file
+   descriptor.  These are for interrupting the peer with that ID using
+   vector 0,..,N-1, in order.  If the client is configured for fewer
+   vectors, it closes the extra file descriptors.  If it is configured
+   for more, the extra vectors remain unconnected.
+
+5. Interrupt setup.  This is the client's own ID, repeated N times.
+   Each repetition is accompanied by one file descriptor.  These are
+   for receiving interrupts from peers using vector 0,..,N-1, in
+   order.  If the client is configured for fewer vectors, it closes
+   the extra file descriptors.  If it is configured for more, the
+   extra vectors remain unconnected.
+
+From then on, the server sends these kinds of messages:
+
+6. Connection / disconnection notification.  This is a peer ID.
+
+  - If the number comes with a file descriptor, it's a connection
+    notification, exactly like in step 4.
+
+  - Else, it's a disconnection notification for the peer with that ID.
+
+Known bugs:
+
+* The protocol changed incompatibly in QEMU 2.5.  Before, messages
+  were native endian long, and there was no version number.
+
+* The protocol is poorly designed.
+
+=== The ivshmem Client-Client Protocol ===
+
+An ivshmem device configured for interrupts receives eventfd file
+descriptors for interrupting peers and getting interrupted by peers
+from the server, as explained in the previous section.
+
+To interrupt a peer, the device writes the 8-byte integer 1 in native
+byte order to the respective file descriptor.
+
+To receive an interrupt, the device reads and discards as many 8-byte
+integers as it can.
diff --git a/docs/specs/ivshmem_device_spec.txt b/docs/specs/ivshmem_device_spec.txt
deleted file mode 100644
index 667a862..0000000
--- a/docs/specs/ivshmem_device_spec.txt
+++ /dev/null
@@ -1,96 +0,0 @@
-
-Device Specification for Inter-VM shared memory device
-------------------------------------------------------
-
-The Inter-VM shared memory device is designed to share a region of memory to
-userspace in multiple virtual guests.  The memory region does not belong to any
-guest, but is a POSIX memory object on the host.  Optionally, the device may
-support sending interrupts to other guests sharing the same memory region.
-
-
-The Inter-VM PCI device
------------------------
-
-*BARs*
-
-The device supports three BARs.  BAR0 is a 1 Kbyte MMIO region to support
-registers.  BAR1 is used for MSI-X when it is enabled in the device.  BAR2 is
-used to map the shared memory object from the host.  The size of BAR2 is
-specified when the guest is started and must be a power of 2 in size.
-
-*Registers*
-
-The device currently supports 4 registers of 32-bits each.  Registers
-are used for synchronization between guests sharing the same memory object when
-interrupts are supported (this requires using the shared memory server).
-
-The server assigns each VM an ID number and sends this ID number to the QEMU
-process when the guest starts.
-
-enum ivshmem_registers {
-    IntrMask = 0,
-    IntrStatus = 4,
-    IVPosition = 8,
-    Doorbell = 12
-};
-
-The first two registers are the interrupt mask and status registers.  Mask and
-status are only used with pin-based interrupts.  They are unused with MSI
-interrupts.
-
-Status Register: The status register is set to 1 when an interrupt occurs.
-
-Mask Register: The mask register is bitwise ANDed with the interrupt status
-and the result will raise an interrupt if it is non-zero.  However, since 1 is
-the only value the status will be set to, it is only the first bit of the mask
-that has any effect.  Therefore interrupts can be masked by setting the first
-bit to 0 and unmasked by setting the first bit to 1.
-
-IVPosition Register: The IVPosition register is read-only and reports the
-guest's ID number.  The guest IDs are non-negative integers.  When using the
-server, since the server is a separate process, the VM ID will only be set when
-the device is ready (shared memory is received from the server and accessible via
-the device).  If the device is not ready, the IVPosition will return -1.
-Applications should ensure that they have a valid VM ID before accessing the
-shared memory.
-
-Doorbell Register:  To interrupt another guest, a guest must write to the
-Doorbell register.  The doorbell register is 32-bits, logically divided into
-two 16-bit fields.  The high 16-bits are the guest ID to interrupt and the low
-16-bits are the interrupt vector to trigger.  The semantics of the value
-written to the doorbell depends on whether the device is using MSI or a regular
-pin-based interrupt.  In short, MSI uses vectors while regular interrupts set the
-status register.
-
-Regular Interrupts
-
-If regular interrupts are used (due to either a guest not supporting MSI or the
-user specifying not to use them on startup) then the value written to the lower
-16-bits of the Doorbell register results is arbitrary and will trigger an
-interrupt in the destination guest.
-
-Message Signalled Interrupts
-
-A ivshmem device may support multiple MSI vectors.  If so, the lower 16-bits
-written to the Doorbell register must be between 0 and the maximum number of
-vectors the guest supports.  The lower 16 bits written to the doorbell is the
-MSI vector that will be raised in the destination guest.  The number of MSI
-vectors is configurable but it is set when the VM is started.
-
-The important thing to remember with MSI is that it is only a signal, no status
-is set (since MSI interrupts are not shared).  All information other than the
-interrupt itself should be communicated via the shared memory region.  Devices
-supporting multiple MSI vectors can use different vectors to indicate different
-events have occurred.  The semantics of interrupt vectors are left to the
-user's discretion.
-
-
-Usage in the Guest
-------------------
-
-The shared memory device is intended to be used with the provided UIO driver.
-Very little configuration is needed.  The guest should map BAR0 to access the
-registers (an array of 32-bit ints allows simple writing) and map BAR2 to
-access the shared memory region itself.  The size of the shared memory region
-is specified when the guest (or shared memory server) is started.  A guest may
-map the whole shared memory region or only part of it.
diff --git a/docs/specs/parallels.txt b/docs/specs/parallels.txt
new file mode 100644
index 0000000..e9271eb
--- /dev/null
+++ b/docs/specs/parallels.txt
@@ -0,0 +1,228 @@
+= License =
+
+Copyright (c) 2015 Denis Lunev
+Copyright (c) 2015 Vladimir Sementsov-Ogievskiy
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.
+See the COPYING file in the top-level directory.
+
+= Parallels Expandable Image File Format =
+
+A Parallels expandable image file consists of three consecutive parts:
+    * header
+    * BAT
+    * data area
+
+All numbers in a Parallels expandable image are stored in little-endian byte
+order.
+
+
+== Definitions ==
+
+    Sector    A 512-byte data chunk.
+
+    Cluster   A data chunk of the size specified in the image header.
+              Currently, the default size is 1MiB (2048 sectors). In previous
+              versions, cluster sizes of 63 sectors, 256 and 252 kilobytes were
+              used.
+
+    BAT       Block Allocation Table, an entity that contains information for
+              guest-to-host I/O data address translation.
+
+
+== Header ==
+
+The header is placed at the start of an image and contains the following
+fields:
+
+Bytes:
+   0 - 15:    magic
+              Must contain "WithoutFreeSpace" or "WithouFreSpacExt".
+
+  16 - 19:    version
+              Must be 2.
+
+  20 - 23:    heads
+              Disk geometry parameter for guest.
+
+  24 - 27:    cylinders
+              Disk geometry parameter for guest.
+
+  28 - 31:    tracks
+              Cluster size, in sectors.
+
+  32 - 35:    nb_bat_entries
+              Disk size, in clusters (BAT size).
+
+  36 - 43:    nb_sectors
+              Disk size, in sectors.
+
+              For "WithoutFreeSpace" images:
+              Only the lowest 4 bytes are used. The highest 4 bytes must be
+              cleared in this case.
+
+              For "WithouFreSpacExt" images, there are no such
+              restrictions.
+
+  44 - 47:    in_use
+              Set to 0x746F6E59 when the image is opened by software in R/W
+              mode; set to 0x312e3276 when the image is closed.
+
+              A zero in this field means that the image was opened by an old
+              version of the software that doesn't support Format Extension
+              (see below).
+
+              Other values are not allowed.
+
+  48 - 51:    data_off
+              An offset, in sectors, from the start of the file to the start of
+              the data area.
+
+              For "WithoutFreeSpace" images:
+              - If data_off is zero, the offset is calculated as the end of BAT
+                table plus some padding to ensure sector size alignment.
+              - If data_off is non-zero, the offset should be aligned to sector
+                size. However it is recommended to align it to cluster size for
+                newly created images.
+
+              For "WithouFreSpacExt" images:
+              data_off must be non-zero and aligned to cluster size.
+
+  52 - 55:    flags
+              Miscellaneous flags.
+
+              Bit 0: Empty Image bit. If set, the image should be
+                     considered clear.
+
+              Bits 1-31: Unused.
+
+  56 - 63:    ext_off
+              Format Extension offset, an offset, in sectors, from the start of
+              the file to the start of the Format Extension Cluster.
+
+              ext_off must meet the same requirements as cluster offsets
+              defined by BAT entries (see below).
+
+
+== BAT ==
+
+BAT is placed immediately after the image header. In the file, BAT is a
+contiguous array of 32-bit unsigned little-endian integers with
+(bat_entries * 4) bytes size.
+
+Each BAT entry contains an offset from the start of the file to the
+corresponding cluster. The offset set in clusters for "WithouFreSpacExt" images
+and in sectors for "WithoutFreeSpace" images.
+
+If a BAT entry is zero, the corresponding cluster is not allocated and should
+be considered as filled with zeroes.
+
+Cluster offsets specified by BAT entries must meet the following requirements:
+    - the value must not be lower than data offset (provided by header.data_off
+      or calculated as specified above),
+    - the value must be lower than the desired file size,
+    - the value must be unique among all BAT entries,
+    - the result of (cluster offset - data offset) must be aligned to cluster
+      size.
+
+
+== Data Area ==
+
+The data area is an area from the data offset (provided by header.data_off or
+calculated as specified above) to the end of the file. It represents a
+contiguous array of clusters. Most of them are allocated by the BAT, some may
+be allocated by the ext_off field in the header while other may be allocated by
+extensions. All clusters allocated by ext_off and extensions should meet the
+same requirements as clusters specified by BAT entries.
+
+
+== Format Extension ==
+
+The Format Extension is an area 1 cluster in size that provides additional
+format features. This cluster is addressed by the ext_off field in the header.
+The format of the Format Extension area is the following:
+
+   0 -  7:    magic
+              Must be 0xAB234CEF23DCEA87
+
+   8 - 23:    m_CheckSum
+              The MD5 checksum of the entire Header Extension cluster except
+              the first 24 bytes.
+
+    The above are followed by feature sections or "extensions". The last
+    extension must be "End of features" (see below).
+
+Each feature section has the following format:
+
+   0 -  7:    magic
+              The identifier of the feature:
+              0x0000000000000000 - End of features
+              0x20385FAE252CB34A - Dirty bitmap
+
+   8 - 15:    flags
+              External flags for extension:
+
+              Bit 0: NECESSARY
+                     If the software cannot load the extension (due to an
+                     unknown magic number or error), the file should not be
+                     changed. If this flag is unset and there is an error on
+                     loading the extension, said extension should be dropped.
+
+              Bit 1: TRANSIT
+                     If there is an unknown extension with this flag set,
+                     said extension should be left as is.
+
+              If neither NECESSARY nor TRANSIT are set, the extension should be
+              dropped.
+
+  16 - 19:    data_size
+              The size of the following feature data, in bytes.
+
+  20 - 23:    unused32
+              Align header to 8 bytes boundary.
+
+  variable:   data (data_size bytes)
+
+    The above is followed by padding to the next 8 bytes boundary, then the
+    next extension starts.
+
+    The last extension must be "End of features" with all the fields set to 0.
+
+
+=== Dirty bitmaps feature ===
+
+This feature provides a way of storing dirty bitmaps in the image. The fields
+of its data area are:
+
+   0 -  7:    size
+              The bitmap size, should be equal to disk size in sectors.
+
+   8 - 23:    id
+              An identifier for backup consistency checking.
+
+  24 - 27:    granularity
+              Bitmap granularity, in sectors. I.e., the number of sectors
+              corresponding to one bit of the bitmap. Granularity must be
+              a power of 2.
+
+  28 - 31:    l1_size
+              The number of entries in the L1 table of the bitmap.
+
+  variable:   l1 (64 * l1_size bytes)
+              L1 offset table (in bytes)
+
+A dirty bitmap is stored using a one-level structure for the mapping to host
+clusters - an L1 table.
+
+Given an offset in bytes into the bitmap data, the offset in bytes into the
+image file can be obtained as follows:
+
+    offset = l1_table[offset / cluster_size] + (offset % cluster_size)
+
+If an L1 table entry is 0, the corresponding cluster of the bitmap is assumed
+to be zero.
+
+If an L1 table entry is 1, the corresponding cluster of the bitmap is assumed
+to have all bits set.
+
+If an L1 table entry is not 0 or 1, it allocates a cluster from the data area.
diff --git a/docs/specs/pci-ids.txt b/docs/specs/pci-ids.txt
index 3c65e1a..fd27c67 100644
--- a/docs/specs/pci-ids.txt
+++ b/docs/specs/pci-ids.txt
@@ -15,13 +15,23 @@
 Note that this allocation separate from the virtio device IDs, which are
 maintained as part of the virtio specification.
 
-1af4:1000  network device
-1af4:1001  block device
-1af4:1002  balloon device
-1af4:1003  console device
-1af4:1004  SCSI host bus adapter device
-1af4:1005  entropy generator device
-1af4:1009  9p filesystem device
+1af4:1000  network device (legacy)
+1af4:1001  block device (legacy)
+1af4:1002  balloon device (legacy)
+1af4:1003  console device (legacy)
+1af4:1004  SCSI host bus adapter device (legacy)
+1af4:1005  entropy generator device (legacy)
+1af4:1009  9p filesystem device (legacy)
+
+1af4:1041  network device (modern)
+1af4:1042  block device (modern)
+1af4:1043  console device (modern)
+1af4:1044  entropy generator device (modern)
+1af4:1045  balloon device (modern)
+1af4:1048  SCSI host bus adapter device (modern)
+1af4:1049  9p filesystem device (modern)
+1af4:1050  virtio gpu device (modern)
+1af4:1052  virtio input device (modern)
 
 1af4:10f0  Available for experimental usage without registration.  Must get
    to      official ID when the code leaves the test lab (i.e. when seeking
@@ -44,6 +54,10 @@
 1b36:0002  PCI serial port (16550A) adapter (docs/specs/pci-serial.txt)
 1b36:0003  PCI Dual-port 16550A adapter (docs/specs/pci-serial.txt)
 1b36:0004  PCI Quad-port 16550A adapter (docs/specs/pci-serial.txt)
+1b36:0005  PCI test device (docs/specs/pci-testdev.txt)
+1b36:0006  PCI Rocker Ethernet switch device
+1b36:0007  PCI SD Card Host Controller Interface (SDHCI)
+1b36:000a  PCI-PCI bridge (multiseat)
 
 All these devices are documented in docs/specs.
 
diff --git a/docs/specs/ppc-spapr-hcalls.txt b/docs/specs/ppc-spapr-hcalls.txt
index 667b3fa..5bd8eab 100644
--- a/docs/specs/ppc-spapr-hcalls.txt
+++ b/docs/specs/ppc-spapr-hcalls.txt
@@ -41,8 +41,8 @@
 with MMU disabled, ie guest effective == guest physical), it only
 has access to a subset of memory and no IOs.
 
-PAPR provides a set of hypervisor calls to perform cachable or
-non-cachable accesses to any guest physical addresses that the
+PAPR provides a set of hypervisor calls to perform cacheable or
+non-cacheable accesses to any guest physical addresses that the
 guest can use in order to access IO devices while in real mode.
 
 This is typically used by the firmware running in the guest.
diff --git a/docs/specs/ppc-spapr-hotplug.txt b/docs/specs/ppc-spapr-hotplug.txt
new file mode 100644
index 0000000..631b0ca
--- /dev/null
+++ b/docs/specs/ppc-spapr-hotplug.txt
@@ -0,0 +1,353 @@
+= sPAPR Dynamic Reconfiguration =
+
+sPAPR/"pseries" guests make use of a facility called dynamic-reconfiguration
+to handle hotplugging of dynamic "physical" resources like PCI cards, or
+"logical"/paravirtual resources like memory, CPUs, and "physical"
+host-bridges, which are generally managed by the host/hypervisor and provided
+to guests as virtualized resources. The specifics of dynamic-reconfiguration
+are documented extensively in PAPR+ v2.7, Section 13.1. This document
+provides a summary of that information as it applies to the implementation
+within QEMU.
+
+== Dynamic-reconfiguration Connectors ==
+
+To manage hotplug/unplug of these resources, a firmware abstraction known as
+a Dynamic Resource Connector (DRC) is used to assign a particular dynamic
+resource to the guest, and provide an interface for the guest to manage
+configuration/removal of the resource associated with it.
+
+== Device-tree description of DRCs ==
+
+A set of 4 Open Firmware device tree array properties are used to describe
+the name/index/power-domain/type of each DRC allocated to a guest at
+boot-time. There may be multiple sets of these arrays, rooted at different
+paths in the device tree depending on the type of resource the DRCs manage.
+
+In some cases, the DRCs themselves may be provided by a dynamic resource,
+such as the DRCs managing PCI slots on a hotplugged PHB. In this case the
+arrays would be fetched as part of the device tree retrieval interfaces
+for hotplugged resources described under "Guest->Host interface".
+
+The array properties are described below. Each entry/element in an array
+describes the DRC identified by the element in the corresponding position
+of ibm,drc-indexes:
+
+ibm,drc-names:
+  first 4-bytes: BE-encoded integer denoting the number of entries
+  each entry: a NULL-terminated <name> string encoded as a byte array
+
+  <name> values for logical/virtual resources are defined in PAPR+ v2.7,
+  Section 13.5.2.4, and basically consist of the type of the resource
+  followed by a space and a numerical value that's unique across resources
+  of that type.
+
+  <name> values for "physical" resources such as PCI or VIO devices are
+  defined as being "location codes", which are the "location labels" of
+  each encapsulating device, starting from the chassis down to the
+  individual slot for the device, concatenated by a hyphen. This provides
+  a mapping of resources to a physical location in a chassis for debugging
+  purposes. For QEMU, this mapping is less important, so we assign a
+  location code that conforms to naming specifications, but is simply a
+  location label for the slot by itself to simplify the implementation.
+  The naming convention for location labels is documented in detail in
+  PAPR+ v2.7, Section 12.3.1.5, and in our case amounts to using "C<n>"
+  for PCI/VIO device slots, where <n> is unique across all PCI/VIO
+  device slots.
+
+ibm,drc-indexes:
+  first 4-bytes: BE-encoded integer denoting the number of entries
+  each 4-byte entry: BE-encoded <index> integer that is unique across all DRCs
+    in the machine
+
+  <index> is arbitrary, but in the case of QEMU we try to maintain the
+  convention used to assign them to pSeries guests on pHyp:
+
+    bit[31:28]: integer encoding of <type>, where <type> is:
+                  1 for CPU resource
+                  2 for PHB resource
+                  3 for VIO resource
+                  4 for PCI resource
+                  8 for Memory resource
+    bit[27:0]: integer encoding of <id>, where <id> is unique across
+                 all resources of specified type
+
+ibm,drc-power-domains:
+  first 4-bytes: BE-encoded integer denoting the number of entries
+  each 4-byte entry: 32-bit, BE-encoded <index> integer that specifies the
+    power domain the resource will be assigned to. In the case of QEMU
+    we associated all resources with a "live insertion" domain, where the
+    power is assumed to be managed automatically. The integer value for
+    this domain is a special value of -1.
+
+
+ibm,drc-types:
+  first 4-bytes: BE-encoded integer denoting the number of entries
+  each entry: a NULL-terminated <type> string encoded as a byte array
+
+  <type> is assigned as follows:
+    "CPU" for a CPU
+    "PHB" for a physical host-bridge
+    "SLOT" for a VIO slot
+    "28" for a PCI slot
+    "MEM" for memory resource
+
+== Guest->Host interface to manage dynamic resources ==
+
+Each DRC is given a globally unique DRC Index, and resources associated with
+a particular DRC are configured/managed by the guest via a number of RTAS
+calls which reference individual DRCs based on the DRC index. This can be
+considered the guest->host interface.
+
+rtas-set-power-level:
+  arg[0]: integer identifying power domain
+  arg[1]: new power level for the domain, 0-100
+  output[0]: status, 0 on success
+  output[1]: power level after command
+
+  Set the power level for a specified power domain
+
+rtas-get-power-level:
+  arg[0]: integer identifying power domain
+  output[0]: status, 0 on success
+  output[1]: current power level
+
+  Get the power level for a specified power domain
+
+rtas-set-indicator:
+  arg[0]: integer identifying sensor/indicator type
+  arg[1]: index of sensor, for DR-related sensors this is generally the
+          DRC index
+  arg[2]: desired sensor value
+  output[0]: status, 0 on success
+
+  Set the state of an indicator or sensor. For the purpose of this document we
+  focus on the indicator/sensor types associated with a DRC. The types are:
+
+    9001: isolation-state, controls/indicates whether a device has been made
+          accessible to a guest
+
+          supported sensor values:
+            0: isolate, device is made unaccessible by guest OS
+            1: unisolate, device is made available to guest OS
+
+    9002: dr-indicator, controls "visual" indicator associated with device
+
+          supported sensor values:
+            0: inactive, resource may be safely removed
+            1: active, resource is in use and cannot be safely removed
+            2: identify, used to visually identify slot for interactive hotplug
+            3: action, in most cases, used in the same manner as identify
+
+    9003: allocation-state, generally only used for "logical" DR resources to
+          request the allocation/deallocation of a resource prior to acquiring
+          it via isolation-state->unisolate, or after releasing it via
+          isolation-state->isolate, respectively. for "physical" DR (like PCI
+          hotplug/unplug) the pre-allocation of the resource is implied and
+          this sensor is unused.
+
+          supported sensor values:
+            0: unusable, tell firmware/system the resource can be
+               unallocated/reclaimed and added back to the system resource pool
+            1: usable, request the resource be allocated/reserved for use by
+               guest OS
+            2: exchange, used to allocate a spare resource to use for fail-over
+               in certain situations. unused in QEMU
+            3: recover, used to reclaim a previously allocated resource that's
+               not currently allocated to the guest OS. unused in QEMU
+
+rtas-get-sensor-state:
+  arg[0]: integer identifying sensor/indicator type
+  arg[1]: index of sensor, for DR-related sensors this is generally the
+          DRC index
+  output[0]: status, 0 on success
+
+  Used to read an indicator or sensor value.
+
+  For DR-related operations, the only noteworthy sensor is dr-entity-sense,
+  which has a type value of 9003, as allocation-state does in the case of
+  rtas-set-indicator. The semantics/encodings of the sensor values are distinct
+  however:
+
+  supported sensor values for dr-entity-sense (9003) sensor:
+    0: empty,
+         for physical resources: DRC/slot is empty
+         for logical resources: unused
+    1: present,
+         for physical resources: DRC/slot is populated with a device/resource
+         for logical resources: resource has been allocated to the DRC
+    2: unusable,
+         for physical resources: unused
+         for logical resources: DRC has no resource allocated to it
+    3: exchange,
+         for physical resources: unused
+         for logical resources: resource available for exchange (see
+           allocation-state sensor semantics above)
+    4: recovery,
+         for physical resources: unused
+         for logical resources: resource available for recovery (see
+           allocation-state sensor semantics above)
+
+rtas-ibm-configure-connector:
+  arg[0]: guest physical address of 4096-byte work area buffer
+  arg[1]: 0, or address of additional 4096-byte work area buffer. only non-zero
+          if a prior RTAS response indicated a need for additional memory
+  output[0]: status:
+               0: completed transmittal of device-tree node
+               1: instruct guest to prepare for next DT sibling node
+               2: instruct guest to prepare for next DT child node
+               3: instruct guest to prepare for next DT property
+               4: instruct guest to ascend to parent DT node
+               5: instruct guest to provide additional work-area buffer
+                  via arg[1]
+            990x: instruct guest that operation took too long and to try
+                  again later
+
+  Used to fetch an OF device-tree description of the resource associated with
+  a particular DRC. The DRC index is encoded in the first 4-bytes of the first
+  work area buffer.
+
+  Work area layout, using 4-byte offsets:
+    wa[0]: DRC index of the DRC to fetch device-tree nodes from
+    wa[1]: 0 (hard-coded)
+    wa[2]: for next-sibling/next-child response:
+             wa offset of null-terminated string denoting the new node's name
+           for next-property response:
+             wa offset of null-terminated string denoting new property's name
+    wa[3]: for next-property response (unused otherwise):
+             byte-length of new property's value
+    wa[4]: for next-property response (unused otherwise):
+             new property's value, encoded as an OFDT-compatible byte array
+
+== hotplug/unplug events ==
+
+For most DR operations, the hypervisor will issue host->guest add/remove events
+using the EPOW/check-exception notification framework, where the host issues a
+check-exception interrupt, then provides an RTAS event log via an
+rtas-check-exception call issued by the guest in response. This framework is
+documented by PAPR+ v2.7, and already use in by QEMU for generating powerdown
+requests via EPOW events.
+
+For DR, this framework has been extended to include hotplug events, which were
+previously unneeded due to direct manipulation of DR-related guest userspace
+tools by host-level management such as an HMC. This level of management is not
+applicable to PowerKVM, hence the reason for extending the notification
+framework to support hotplug events.
+
+Note that these events are not yet formally part of the PAPR+ specification,
+but support for this format has already been implemented in DR-related
+guest tools such as powerpc-utils/librtas, as well as kernel patches that have
+been submitted to handle in-kernel processing of memory/cpu-related hotplug
+events[1], and is planned for formal inclusion is PAPR+ specification. The
+hotplug-specific payload is QEMU implemented as follows (with all values
+encoded in big-endian format):
+
+struct rtas_event_log_v6_hp {
+#define SECTION_ID_HOTPLUG              0x4850 /* HP */
+    struct section_header {
+        uint16_t section_id;            /* set to SECTION_ID_HOTPLUG */
+        uint16_t section_length;        /* sizeof(rtas_event_log_v6_hp),
+                                         * plus the length of the DRC name
+                                         * if a DRC name identifier is
+                                         * specified for hotplug_identifier
+                                         */
+        uint8_t section_version;        /* version 1 */
+        uint8_t section_subtype;        /* unused */
+        uint16_t creator_component_id;  /* unused */
+    } hdr;
+#define RTAS_LOG_V6_HP_TYPE_CPU         1
+#define RTAS_LOG_V6_HP_TYPE_MEMORY      2
+#define RTAS_LOG_V6_HP_TYPE_SLOT        3
+#define RTAS_LOG_V6_HP_TYPE_PHB         4
+#define RTAS_LOG_V6_HP_TYPE_PCI         5
+    uint8_t hotplug_type;               /* type of resource/device */
+#define RTAS_LOG_V6_HP_ACTION_ADD       1
+#define RTAS_LOG_V6_HP_ACTION_REMOVE    2
+    uint8_t hotplug_action;             /* action (add/remove) */
+#define RTAS_LOG_V6_HP_ID_DRC_NAME      1
+#define RTAS_LOG_V6_HP_ID_DRC_INDEX     2
+#define RTAS_LOG_V6_HP_ID_DRC_COUNT     3
+    uint8_t hotplug_identifier;         /* type of the resource identifier,
+                                         * which serves as the discriminator
+                                         * for the 'drc' union field below
+                                         */
+    uint8_t reserved;
+    union {
+        uint32_t index;                 /* DRC index of resource to take action
+                                         * on
+                                         */
+        uint32_t count;                 /* number of DR resources to take
+                                         * action on (guest chooses which)
+                                         */
+        char name[1];                   /* string representing the name of the
+                                         * DRC to take action on
+                                         */
+    } drc;
+} QEMU_PACKED;
+
+== ibm,lrdr-capacity ==
+
+ibm,lrdr-capacity is a property in the /rtas device tree node that identifies
+the dynamic reconfiguration capabilities of the guest. It consists of a triple
+consisting of <phys>, <size> and <maxcpus>.
+
+  <phys>, encoded in BE format represents the maximum address in bytes and
+  hence the maximum memory that can be allocated to the guest.
+
+  <size>, encoded in BE format represents the size increments in which
+  memory can be hot-plugged to the guest.
+
+  <maxcpus>, a BE-encoded integer, represents the maximum number of
+  processors that the guest can have.
+
+pseries guests use this property to note the maximum allowed CPUs for the
+guest.
+
+== ibm,dynamic-reconfiguration-memory ==
+
+ibm,dynamic-reconfiguration-memory is a device tree node that represents
+dynamically reconfigurable logical memory blocks (LMB). This node
+is generated only when the guest advertises the support for it via
+ibm,client-architecture-support call. Memory that is not dynamically
+reconfigurable is represented by /memory nodes. The properties of this
+node that are of interest to the sPAPR memory hotplug implementation
+in QEMU are described here.
+
+ibm,lmb-size
+
+This 64bit integer defines the size of each dynamically reconfigurable LMB.
+
+ibm,associativity-lookup-arrays
+
+This property defines a lookup array in which the NUMA associativity
+information for each LMB can be found. It is a property encoded array
+that begins with an integer M, the number of associativity lists followed
+by an integer N, the number of entries per associativity list and terminated
+by M associativity lists each of length N integers.
+
+This property provides the same information as given by ibm,associativity
+property in a /memory node. Each assigned LMB has an index value between
+0 and M-1 which is used as an index into this table to select which
+associativity list to use for the LMB. This index value for each LMB
+is defined in ibm,dynamic-memory property.
+
+ibm,dynamic-memory
+
+This property describes the dynamically reconfigurable memory. It is a
+property encoded array that has an integer N, the number of LMBs followed
+by N LMB list entires.
+
+Each LMB list entry consists of the following elements:
+
+- Logical address of the start of the LMB encoded as a 64bit integer. This
+  corresponds to reg property in /memory node.
+- DRC index of the LMB that corresponds to ibm,my-drc-index property
+  in a /memory node.
+- Four bytes reserved for expansion.
+- Associativity list index for the LMB that is used as an index into
+  ibm,associativity-lookup-arrays property described earlier. This
+  is used to retrieve the right associativity list to be used for this
+  LMB.
+- A 32bit flags word. The bit at bit position 0x00000008 defines whether
+  the LMB is assigned to the the partition as of boot time.
+
+[1] http://thread.gmane.org/gmane.linux.ports.ppc.embedded/75350/focus=106867
diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt
index 121dfc8..80cdfd0 100644
--- a/docs/specs/qcow2.txt
+++ b/docs/specs/qcow2.txt
@@ -103,7 +103,18 @@
                     write to an image with unknown auto-clear features if it
                     clears the respective bits from this field first.
 
-                    Bits 0-63:  Reserved (set to 0)
+                    Bit 0:      Bitmaps extension bit
+                                This bit indicates consistency for the bitmaps
+                                extension data.
+
+                                It is an error if this bit is set without the
+                                bitmaps extension present.
+
+                                If the bitmaps extension is present but this
+                                bit is unset, the bitmaps extension data must be
+                                considered inconsistent.
+
+                    Bits 1-63:  Reserved (set to 0)
 
          96 -  99:  refcount_order
                     Describes the width of a reference count block entry (width
@@ -123,6 +134,7 @@
                         0x00000000 - End of the header extension area
                         0xE2792ACA - Backing file format name
                         0x6803f857 - Feature name table
+                        0x23852875 - Bitmaps extension
                         other      - Unknown header extension, can be safely
                                      ignored
 
@@ -166,6 +178,36 @@
                     terminated if it has full length)
 
 
+== Bitmaps extension ==
+
+The bitmaps extension is an optional header extension. It provides the ability
+to store bitmaps related to a virtual disk. For now, there is only one bitmap
+type: the dirty tracking bitmap, which tracks virtual disk changes from some
+point in time.
+
+The data of the extension should be considered consistent only if the
+corresponding auto-clear feature bit is set, see autoclear_features above.
+
+The fields of the bitmaps extension are:
+
+    Byte  0 -  3:  nb_bitmaps
+                   The number of bitmaps contained in the image. Must be
+                   greater than or equal to 1.
+
+                   Note: Qemu currently only supports up to 65535 bitmaps per
+                   image.
+
+          4 -  7:  Reserved, must be zero.
+
+          8 - 15:  bitmap_directory_size
+                   Size of the bitmap directory in bytes. It is the cumulative
+                   size of all (nb_bitmaps) bitmap headers.
+
+         16 - 23:  bitmap_directory_offset
+                   Offset into the image file at which the bitmap directory
+                   starts. Must be aligned to a cluster boundary.
+
+
 == Host cluster management ==
 
 qcow2 manages the allocation of host clusters by maintaining a reference count
@@ -257,7 +299,7 @@
 
               63:   0 for a cluster that is unused or requires COW, 1 if its
                     refcount is exactly one. This information is only accurate
-                    in L2 tables that are reachable from the the active L1
+                    in L2 tables that are reachable from the active L1
                     table.
 
 Standard Cluster Descriptor:
@@ -360,3 +402,180 @@
 
         variable:   Padding to round up the snapshot table entry size to the
                     next multiple of 8.
+
+
+== Bitmaps ==
+
+As mentioned above, the bitmaps extension provides the ability to store bitmaps
+related to a virtual disk. This section describes how these bitmaps are stored.
+
+All stored bitmaps are related to the virtual disk stored in the same image, so
+each bitmap size is equal to the virtual disk size.
+
+Each bit of the bitmap is responsible for strictly defined range of the virtual
+disk. For bit number bit_nr the corresponding range (in bytes) will be:
+
+    [bit_nr * bitmap_granularity .. (bit_nr + 1) * bitmap_granularity - 1]
+
+Granularity is a property of the concrete bitmap, see below.
+
+
+=== Bitmap directory ===
+
+Each bitmap saved in the image is described in a bitmap directory entry. The
+bitmap directory is a contiguous area in the image file, whose starting offset
+and length are given by the header extension fields bitmap_directory_offset and
+bitmap_directory_size. The entries of the bitmap directory have variable
+length, depending on the lengths of the bitmap name and extra data. These
+entries are also called bitmap headers.
+
+Structure of a bitmap directory entry:
+
+    Byte 0 -  7:    bitmap_table_offset
+                    Offset into the image file at which the bitmap table
+                    (described below) for the bitmap starts. Must be aligned to
+                    a cluster boundary.
+
+         8 - 11:    bitmap_table_size
+                    Number of entries in the bitmap table of the bitmap.
+
+        12 - 15:    flags
+                    Bit
+                      0: in_use
+                         The bitmap was not saved correctly and may be
+                         inconsistent.
+
+                      1: auto
+                         The bitmap must reflect all changes of the virtual
+                         disk by any application that would write to this qcow2
+                         file (including writes, snapshot switching, etc.). The
+                         type of this bitmap must be 'dirty tracking bitmap'.
+
+                      2: extra_data_compatible
+                         This flags is meaningful when the extra data is
+                         unknown to the software (currently any extra data is
+                         unknown to Qemu).
+                         If it is set, the bitmap may be used as expected, extra
+                         data must be left as is.
+                         If it is not set, the bitmap must not be used, but
+                         both it and its extra data be left as is.
+
+                    Bits 3 - 31 are reserved and must be 0.
+
+             16:    type
+                    This field describes the sort of the bitmap.
+                    Values:
+                      1: Dirty tracking bitmap
+
+                    Values 0, 2 - 255 are reserved.
+
+             17:    granularity_bits
+                    Granularity bits. Valid values: 0 - 63.
+
+                    Note: Qemu currently doesn't support granularity_bits
+                    greater than 31.
+
+                    Granularity is calculated as
+                        granularity = 1 << granularity_bits
+
+                    A bitmap's granularity is how many bytes of the image
+                    accounts for one bit of the bitmap.
+
+        18 - 19:    name_size
+                    Size of the bitmap name. Must be non-zero.
+
+                    Note: Qemu currently doesn't support values greater than
+                    1023.
+
+        20 - 23:    extra_data_size
+                    Size of type-specific extra data.
+
+                    For now, as no extra data is defined, extra_data_size is
+                    reserved and should be zero. If it is non-zero the
+                    behavior is defined by extra_data_compatible flag.
+
+        variable:   extra_data
+                    Extra data for the bitmap, occupying extra_data_size bytes.
+                    Extra data must never contain references to clusters or in
+                    some other way allocate additional clusters.
+
+        variable:   name
+                    The name of the bitmap (not null terminated), occupying
+                    name_size bytes. Must be unique among all bitmap names
+                    within the bitmaps extension.
+
+        variable:   Padding to round up the bitmap directory entry size to the
+                    next multiple of 8. All bytes of the padding must be zero.
+
+
+=== Bitmap table ===
+
+Each bitmap is stored using a one-level structure (as opposed to two-level
+structures like for refcounts and guest clusters mapping) for the mapping of
+bitmap data to host clusters. This structure is called the bitmap table.
+
+Each bitmap table has a variable size (stored in the bitmap directory entry)
+and may use multiple clusters, however, it must be contiguous in the image
+file.
+
+Structure of a bitmap table entry:
+
+    Bit       0:    Reserved and must be zero if bits 9 - 55 are non-zero.
+                    If bits 9 - 55 are zero:
+                      0: Cluster should be read as all zeros.
+                      1: Cluster should be read as all ones.
+
+         1 -  8:    Reserved and must be zero.
+
+         9 - 55:    Bits 9 - 55 of the host cluster offset. Must be aligned to
+                    a cluster boundary. If the offset is 0, the cluster is
+                    unallocated; in that case, bit 0 determines how this
+                    cluster should be treated during reads.
+
+        56 - 63:    Reserved and must be zero.
+
+
+=== Bitmap data ===
+
+As noted above, bitmap data is stored in separate clusters, described by the
+bitmap table. Given an offset (in bytes) into the bitmap data, the offset into
+the image file can be obtained as follows:
+
+    image_offset(bitmap_data_offset) =
+        bitmap_table[bitmap_data_offset / cluster_size] +
+            (bitmap_data_offset % cluster_size)
+
+This offset is not defined if bits 9 - 55 of bitmap table entry are zero (see
+above).
+
+Given an offset byte_nr into the virtual disk and the bitmap's granularity, the
+bit offset into the image file to the corresponding bit of the bitmap can be
+calculated like this:
+
+    bit_offset(byte_nr) =
+        image_offset(byte_nr / granularity / 8) * 8 +
+            (byte_nr / granularity) % 8
+
+If the size of the bitmap data is not a multiple of the cluster size then the
+last cluster of the bitmap data contains some unused tail bits. These bits must
+be zero.
+
+
+=== Dirty tracking bitmaps ===
+
+Bitmaps with 'type' field equal to one are dirty tracking bitmaps.
+
+When the virtual disk is in use dirty tracking bitmap may be 'enabled' or
+'disabled'. While the bitmap is 'enabled', all writes to the virtual disk
+should be reflected in the bitmap. A set bit in the bitmap means that the
+corresponding range of the virtual disk (see above) was written to while the
+bitmap was 'enabled'. An unset bit means that this range was not written to.
+
+The software doesn't have to sync the bitmap in the image file with its
+representation in RAM after each write. Flag 'in_use' should be set while the
+bitmap is not synced.
+
+In the image file the 'enabled' state is reflected by the 'auto' flag. If this
+flag is set, the software must consider the bitmap as 'enabled' and start
+tracking virtual disk changes to this bitmap from the first write to the
+virtual disk. If this flag is not set then the bitmap is disabled.
diff --git a/docs/specs/rocker.txt b/docs/specs/rocker.txt
new file mode 100644
index 0000000..1857b31
--- /dev/null
+++ b/docs/specs/rocker.txt
@@ -0,0 +1,1014 @@
+Rocker Network Switch Register Programming Guide
+Copyright (c) Scott Feldman <sfeldma@gmail.com>
+Copyright (c) Neil Horman <nhorman@tuxdriver.com>
+Version 0.11, 12/29/2014
+
+LICENSE
+=======
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+SECTION 1: Introduction
+=======================
+
+Overview
+--------
+
+This document describes the hardware/software interface for the Rocker switch
+device.  The intended audience is authors of OS drivers and device emulation
+software.
+
+Notations and Conventions
+-------------------------
+
+o In register descriptions, [n:m] indicates a range from bit n to bit m,
+inclusive.
+o Use of leading 0x indicates a hexadecimal number.
+o Use of leading 0b indicates a binary number.
+o The use of RSVD or Reserved indicates that a bit or field is reserved for
+future use.
+o Field width is in bytes, unless otherwise noted.
+o Register are (R) read-only, (R/W) read/write, (W) write-only, or (COR) clear
+on read
+o TLV values in network-byte-order are designated with (N).
+
+
+SECTION 2: PCI Configuration Registers
+======================================
+
+PCI Configuration Space
+-----------------------
+
+Each switch instance registers as a PCI device with PCI configuration space:
+
+	offset	width	description		value
+	---------------------------------------------
+	0x0	2	Vendor ID		0x1b36
+	0x2	2	Device ID		0x0006
+	0x4	4	Command/Status
+	0x8	1	Revision ID		0x01
+	0x9	3	Class code		0x2800
+	0xC	1	Cache line size
+	0xD	1	Latency timer
+	0xE	1	Header type
+	0xF	1	Built-in self test
+	0x10	4	Base address low
+	0x14	4	Base address high
+	0x18-28		Reserved
+	0x2C	2	Subsystem vendor ID	*
+	0x2E	2	Subsystem ID		*
+	0x30-38		Reserved
+	0x3C	1	Interrupt line
+	0x3D	1	Interrupt pin		0x00
+	0x3E	1	Min grant		0x00
+	0x3D	1	Max latency		0x00
+	0x40	1	TRDY timeout
+	0x41	1	Retry count
+	0x42	2	Reserved
+
+
+* Assigned by sub-system implementation
+
+SECTION 3: Memory-Mapped Register Space
+=======================================
+
+There are two memory-mapped BARs.  BAR0 maps device register space and is
+0x2000 in size.  BAR1 maps MSI-X vector and PBA tables and is also 0x2000 in
+size, allowing for 256 MSI-X vectors.
+
+All registers are 4 or 8 bytes long.  It is assumed host software will access 4
+byte registers with one 4-byte access, and 8 byte registers with either two
+4-byte accesses or a single 8-byte access.  In the case of two 4-byte accesses,
+access must be lower and then upper 4-bytes, in that order.
+
+BAR0 device register space is organized as follows:
+
+	offset		description
+	------------------------------------------------------
+	0x0000-0x000f	Bogus registers to catch misbehaving
+			drivers.  Writes do nothing.  Reads
+			back as 0xDEADBABE.
+	0x0010-0x00ff	Test registers
+	0x0300-0x03ff	General purpose registers
+	0x1000-0x1fff	Descriptor control
+
+Holes in register space are reserved.  Writes to reserved registers do nothing.
+Reads to reserved registers read back as 0.
+
+No fancy stuff like write-combining is enabled on any of the registers.
+
+BAR1 MSI-X register space is organized as follows:
+
+	offset		description
+	------------------------------------------------------
+	0x0000-0x0fff	MSI-X vector table (256 vectors total)
+	0x1000-0x1fff	MSI-X PBA table
+
+
+SECTION 4: Interrupts, DMA, and Endianness
+==========================================
+
+PCI Interrupts
+--------------
+
+The device supports only MSI-X interrupts.  BAR1 memory-mapped region contains
+the MSI-X vector and PBA tables, with support for up to 256 MSI-X vectors.
+
+The vector assignment is:
+
+	vector		description
+	-----------------------------------------------------
+	0		Command descriptor ring completion
+	1		Event descriptor ring completion
+	2		Test operation completion
+	3		RSVD
+	4-255		Tx and Rx descriptor ring completion
+			  Tx vector is even
+			  Rx vector is odd
+
+A MSI-X vector table entry is 16 bytes:
+
+	field		offset	width	description
+	-------------------------------------------------------------
+	lower_addr	0x0	4	[31:2] message address[31:2]
+					[1:0] Rsvd (4 byte alignment
+						    required)
+	upper_addr	0x4	4	[31:19] Rsvd
+					[14:0] message address[46:32]
+	data		0x8	4	message data[31:0]
+	control		0xc	4	[31:1] Rsvd
+					[0] mask (0 = enable,
+						  1 = masked)
+
+Software should install the Interrupt Service Routine (ISR) before any ports
+are enabled or any commands are issued on the command ring.
+
+DMA Operations
+--------------
+
+DMA operations are used for packet DMA to/from the CPU, command and event
+processing.  Command processing includes statistical counters and table dumps,
+table insertion/deletion, and more.  Event processing provides an async
+notification method for device-originating events.  Each DMA operation has a
+set of control registers to manage a descriptor ring.  The descriptor rings are
+allocated from contiguous host DMA-able memory and registers specify the rings
+base address, size and current head and tail indices.  Software always writes
+the head, and hardware always writes the tail.
+
+The higher-order bit of DMA_DESC_COMP_ERR is used to mark hardware completion
+of a descriptor.  Software will clear this bit when posting a descriptor to the
+ring, and hardware will set this bit when the descriptor is complete.
+
+Descriptor ring sizes must be a power of 2 and range from 2 to 64K entries.
+Descriptor rings' base address must be 8-byte aligned.  Descriptors must be
+packed within ring.  Each descriptor in each ring must also be aligned on an 8
+byte boundary.  Each descriptor ring will have these registers:
+
+	DMA_DESC_xxx_BASE_ADDR, offset 0x1000 + (x * 32), 64-bit, (R/W)
+	DMA_DESC_xxx_SIZE, offset 0x1008 + (x * 32), 32-bit, (R/W)
+	DMA_DESC_xxx_HEAD, offset 0x100c + (x * 32), 32-bit, (R/W)
+	DMA_DESC_xxx_TAIL, offset 0x1010 + (x * 32), 32-bit, (R)
+	DMA_DESC_xxx_CTRL, offset 0x1014 + (x * 32), 32-bit, (W)
+	DMA_DESC_xxx_CREDITS, offset 0x1018 + (x * 32), 32-bit, (R/W)
+	DMA_DESC_xxx_RSVD1, offset 0x101c + (x * 32), 32-bit, (R/W)
+
+Where x is descriptor ring index:
+
+	index		ring
+	--------------------
+	0		CMD
+	1		EVENT
+	2		TX (port 0)
+	3		RX (port 0)
+	4		TX (port 1)
+	5		RX (port 1)
+	.
+	.
+	.
+	124		TX (port 61)
+	125		RX (port 61)
+	126		Resv
+	127		Resv
+
+Writing BASE_ADDR or SIZE will reset HEAD and TAIL to zero.  HEAD cannot be
+written past TAIL.  To do so would wrap the ring.  An empty ring is when HEAD
+== TAIL.  A full ring is when HEAD is one position behind TAIL.  Both HEAD and
+TAIL increment and modulo wrap at the ring size.
+
+CTRL register bits:
+
+	bit	name		description
+	------------------------------------------------------------------------
+	[0]	CTRL_RESET	Reset the descriptor ring
+	[1:31]	Reserved
+
+All descriptor types share some common fields:
+
+	field			width	description
+	-------------------------------------------------------------------
+	DMA_DESC_BUF_ADDR	8	Phys addr of desc payload, 8-byte
+					aligned
+	DMA_DESC_COOKIE		8	Desc cookie for completion matching,
+					upper-most bit is reserved
+	DMA_DESC_BUF_SIZE	2	Desc payload size in bytes
+	DMA_DESC_TLV_SIZE	2	Desc payload total size in bytes
+					used for TLVs.  Must be <=
+					DMA_DESC_BUF_SIZE.
+	DMA_DESC_COMP_ERR	2	Completion status of associated
+					desc payload.  High order bit is
+					clear on new descs, toggled by
+					hw for completed items.
+
+To support forward- and backward-compatibility, descriptor and completion
+payloads are specified in TLV format.  Fields are packed with Type=field name,
+Length=field length, and Value=field value.  Software will ignore unknown fields
+filled in by the switch.  Likewise, the switch will ignore unknown fields
+filled in by software.
+
+Descriptor payload buffer is 8-byte aligned and TLVs are 8-byte aligned.  The
+value within a TLV is also 8-byte aligned.  The (packed, 8 byte) TLV header is:
+
+	field	width	description
+	-----------------------------
+	type	4	TLV type
+	len	2	TLV value length
+	pad	2	Reserved
+
+The alignment requirements for descriptors and TLVs are to avoid unaligned
+access exceptions in software.  Note that the payload for each TLV is also
+8 byte aligned.
+
+Figure 1 shows an example descriptor buffer with two TLVs.
+
+                  <------- 8 bytes ------->
+
+  8-byte  +––––+  +–––––––––––+–––––+–––––+                     +–+
+  align           |   type    | len | pad |    TLV#1 hdr          |
+                  +–––––––––––+–––––+–––––+    (len=22)           |
+                  |                       |                       |
+                  |  value                |    TVL#1 value        |
+                  |                       |    (padded to 8-byte  |
+                  |                 +–––––+     alignment)        |
+                  |                 |/////|                       |
+   8-byte +––––+  +–––––––––––+–––––––––––+                       |
+   align          |   type    | len | pad |    TLV#2 hdr    DESC_BUF_SIZE
+                  +–––––+–––––+–––––+–––––+    (len=2)            |
+                  |value|/////////////////|    TLV#2 value        |
+                  +–––––+/////////////////|                       |
+                  |///////////////////////|                       |
+                  |///////////////////////|                       |
+                  |///////////////////////|                       |
+                  |////////unused/////////|                       |
+                  |////////space//////////|                       |
+                  |///////////////////////|                       |
+                  |///////////////////////|                       |
+                  |///////////////////////|                       |
+                  +–––––––––––––––––––––––+                     +–+
+
+				fig. 1
+
+TLVs can be nested within the NEST TLV type.
+
+Interrupt credits
+^^^^^^^^^^^^^^^^^
+
+MSI-X vectors used for descriptor ring completions use a credit mechanism for
+efficient device, PCIe bus, OS and driver operations.  Each descriptor ring has
+a credit count which represents the number of outstanding descriptors to be
+processed by the driver.  As the device marks descriptors complete, the credit
+count is incremented.  As the driver processes those outstanding descriptors,
+it returns credits back to the device.  This way, the device knows the driver's
+progress and can make decisions about when to fire the next interrupt or not.
+When the credit count is zero, and the first descriptors are posted for the
+driver, a single interrupt is fired.  Once the interrupt is fired, the
+interrupt is disabled (auto-masked*).  In response to the interrupt, the driver
+will process descriptors and PIO write a returned credit value for that
+descriptor ring.  If the driver returns all credits (the driver caught up with
+the device and there is no outstanding work), then the interrupt is unmasked,
+but not fired.  If only partial credits are returned, the interrupt remains
+masked but the device generates an interrupt, signaling the driver that more
+outstanding work is available.
+
+(* this masking is unrelated to the MSI-X interrupt mask register)
+
+Endianness
+----------
+
+Device registers are hard-coded to little-endian (LE).  The driver should
+convert to/from host endianness to LE for device register accesses.
+
+Descriptors are LE.  Descriptor buffer TLVs will have LE type and length
+fields, but the value field can either be LE or network-byte-order, depending
+on context.  TLV values containing network packet data will be in network-byte
+order.  A TLV value containing a field or mask used to compare against network
+packet data is network-byte order.  For example, flow match fields (and masks)
+are network-byte-order since they're matched directly, byte-by-byte, against
+network packet data.  All non-network-packet TLV multi-byte values will be LE.
+
+TLV values in network-byte-order are designated with (N).
+
+
+SECTION 5: Test Registers
+=========================
+
+Rocker has several test registers to support troubleshooting register access,
+interrupt generation, and DMA operations:
+
+	TEST_REG, offset 0x0010, 32-bit (R/W)
+	TEST_REG64, offset 0x0018, 64-bit (R/W)
+	TEST_IRQ, offset 0x0020, 32-bit (R/W)
+	TEST_DMA_ADDR, offset 0x0028, 64-bit (R/W)
+	TEST_DMA_SIZE, offset 0x0030, 32-bit (R/W)
+	TEST_DMA_CTRL, offset 0x0034, 32-bit (R/W)
+
+Reads to TEST_REG and TEST_REG64 will read a value equal to twice the last
+value written to the register.  The 32-bit and 64-bit versions are for testing
+32-bit and 64-bit host accesses.
+
+A vector can be written to TEST_IRQ and the device will generate an interrupt
+for that vector.
+
+To test basic DMA operations, allocate a DMA-able host buffer and put the
+buffer address into TEST_DMA_ADDR and size into TEST_DMA_SIZE.  Then, write to
+TEST_DMA_CTRL to manipulate the buffer contents.  TEST_DMA_CTRL operations are:
+
+	operation		value	description
+	-----------------------------------------------------------
+	TEST_DMA_CTRL_CLEAR	1	clear buffer
+	TEST_DMA_CTRL_FILL	2	fill buffer bytes with 0x96
+	TEST_DMA_CTRL_INVERT	4	invert bytes in buffer
+
+Various buffer address and sizes should be tested to verify no address boundary
+issue exists.  In particular, buffers that start on odd-8-byte boundary and/or
+span multiple PAGE sizes should be tested.
+
+
+SECTION 6: Ports
+================
+
+Physical and Logical Ports
+------------------------------------
+
+The switch supports up to 62 physical (front-panel) ports.  Register
+PORT_PHYS_COUNT returns the actual number of physical ports available:
+
+	PORT_PHYS_COUNT, offset 0x0304, 32-bit, (R)
+
+In addition to front-panel ports, the switch supports logical ports for
+tunnels.
+
+Front-panel ports and logical tunnel ports are mapped into a single 32-bit port
+space.  A special CPU port is assigned port 0.  The front-panel ports are
+mapped to ports 1-62.  A special loopback port is assigned port 63.  Logical
+tunnel ports are assigned ports 0x0001000-0x0001ffff.
+To summarize the port assignments:
+
+	port			mapping
+	-------------------------------------------------------
+	0			CPU port (for packets to/from host CPU)
+	1-62			front-panel physical ports
+	63			loopback port
+	64-0x0000ffff		RSVD
+	0x00010000-0x0001ffff	logical tunnel ports
+	0x00020000-0xffffffff	RSVD
+
+Physical Port Mode
+------------------
+
+Switch front-panel ports operate in a mode.  Currently, the only mode is
+OF-DPA.  OF-DPA[1] mode is based on OpenFlow Data Plane Abstraction (OF-DPA)
+Abstract Switch Specification, Version 1.0, from Broadcom Corporation.  To
+set/get the mode for front-panel ports, see port settings, below.
+
+Port Settings
+-------------
+
+Link status for all front-panel ports is available via PORT_PHYS_LINK_STATUS:
+
+	PORT_PHYS_LINK_STATUS, offset 0x0310, 64-bit, (R)
+
+	Value is port bitmap.  Bits 0 and 63 always read 0.  Bits 1-62
+	read 1 for link UP and 0 for link DOWN for respective front-panel ports.
+
+Other properties for front-panel ports are available via DMA CMD descriptors:
+
+	Get PORT_SETTINGS descriptor:
+
+		field		width	description
+		----------------------------------------------
+		PORT_SETTINGS	2	CMD_GET
+		PPORT		4	Physical port #
+
+	Get PORT_SETTINGS completion:
+
+		field		width	description
+		----------------------------------------------
+		PPORT		4	Physical port #
+		SPEED		4	Current port interface speed, in Mbps
+		DUPLEX		1	1 = Full, 0 = Half
+		AUTONEG		1	1 = enabled, 0 = disabled
+		MACADDR		6	Port MAC address
+		MODE		1	0 = OF-DPA
+		LEARNING	1	MAC address learning on port
+						1 = enabled
+						0 = disabled
+		PHYS_NAME	<var>	Physical port name (string)
+
+	Set PORT_SETTINGS descriptor:
+
+		field		width	description
+		----------------------------------------------
+		PORT_SETTINGS	2	CMD_SET
+		PPORT		4	Physical port #
+		SPEED		4	Port interface speed, in Mbps
+		DUPLEX		1	1 = Full, 0 = Half
+		AUTONEG		1	1 = enabled, 0 = disabled
+		MACADDR		6	Port MAC address
+		MODE		1	0 = OF-DPA
+
+Port Enable
+-----------
+
+Front-panel ports are initially disabled, which means port ingress and egress
+packets will be dropped.  To enable or disable a port, use PORT_PHYS_ENABLE:
+
+	PORT_PHYS_ENABLE: offset 0x0318, 64-bit, (R/W)
+
+	Value is bitmap of first 64 ports.  Bits 0 and 63 are ignored
+	and always read as 0.  Write 1 to enable port; write 0 to disable it.
+	Default is 0.
+
+
+SECTION 7: Switch Control
+=========================
+
+This section covers switch-wide register settings.
+
+Control
+-------
+
+This register is used for low level control of the switch.
+
+	CONTROL: offset 0x0300, 32-bit, (W)
+
+	bit	name		description
+	------------------------------------------------------------------------
+	[0]	CONTROL_RESET	If set, device will perform reset
+	[1:31]	Reserved
+
+Switch ID
+---------
+
+The switch has a SWITCH_ID to be used by software to uniquely identify the
+switch:
+
+	SWITCH_ID: offset 0x0320, 64-bit, (R)
+
+	Value is opaque to switch software and no special encoding is implied.
+
+
+SECTION 8: Events
+=================
+
+Non-I/O asynchronous events from the device are notified to the host using the
+event ring.  The TLV structure for events is:
+
+	field		width	description
+	---------------------------------------------------
+	TYPE		4	Event type, one of:
+					1: LINK_CHANGED
+					2: MAC_VLAN_SEEN
+	INFO		<nest>	Event info (details below)
+
+Link Changed Event
+------------------
+
+When link status changes on a physical port, this event is generated.
+
+	field		width	description
+	---------------------------------------------------
+	INFO		<nest>
+	  PPORT		4	Physical port
+	  LINKUP	1	Link status:
+					0: down
+					1: up
+
+MAC VLAN Seen Event
+-------------------
+
+When a packet ingresses on a port and the source MAC/VLAN isn't known to the
+device, the device will generate this event.  In response to the event, the
+driver should install to the device the MAC/VLAN on the port into the bridge
+table.  Once installed, the MAC/VLAN is known on the port and this event will
+no longer be generated.
+
+	field		width	description
+	---------------------------------------------------
+	INFO		<nest>
+	  PPORT		4	Physical port
+	  MAC		6	MAC address
+	  VLAN		2	VLAN ID
+
+
+SECTION 9: CPU Packet Processing
+================================
+
+Ingress packets directed to the host CPU for further processing are delivered
+in the DMA RX ring.  Likewise, host CPU originating packets destined to egress
+on switch ports are scheduled by software using the DMA TX ring.
+
+Tx Packet Processing
+--------------------
+
+Software schedules packets for egress on switch ports using the DMA TX ring.  A
+TX descriptor buffer describes the packet location and size in host DMA-able
+memory, the destination port, and any hardware-offload functions (such as L3
+payload checksum offload).  Software then bumps the descriptor head to signal
+hardware of new Tx work.  In response, hardware will DMA read Tx descriptors up
+to head, DMA read descriptor buffer and packet data, perform offloading
+functions, and finally frame packet on wire (network).  Once packet processing
+is complete, hardware will writeback status to descriptor(s) to signal to
+software that Tx is complete and software resources (e.g. skb) backing packet
+can be released.
+
+Figure 2 shows an example 3-fragment packet queued with one Tx descriptor.  A
+TLV is used for each packet fragment.
+
+	                                           pkt frag 1
+	                                           +–––––––+  +–+
+	                                       +–––+       |    |
+	                         desc buf      |   |       |    |
+	                        +––––––––+     |   |       |    |
+	        Tx ring     +–––+        +–––––+   |       |    |
+	      +–––––––––+   |   |  TLVs  |         +–––––––+    |
+	      |         +–––+   +––––––––+         pkt frag 2   |
+	      | desc 0  |       |        +–––––+   +–––––––+    |
+	      +–––––––––+       |  TLVs  |     +–––+       |    |
+	head+–+         |       +––––––––+         |       |    |
+	      | desc 1  |       |        +–––––+   +–––––––+    |pkt
+	      +–––––––––+       |  TLVs  |     |                |
+	      |         |       +––––––––+     |   pkt frag 3   |
+	      |         |                      |   +–––––––+    |
+	      +–––––––––+                      +–––+       |    |
+	      |         |                          |       |    |
+	      |         |                          |       |    |
+	      +–––––––––+                          |       |    |
+	      |         |                          |       |    |
+	      |         |                          |       |    |
+	      +–––––––––+                          |       |    |
+	      |         |                          +–––––––+  +–+
+	      |         |
+	      +–––––––––+
+
+				fig 2.
+
+The TLVs for Tx descriptor buffer are:
+
+	field			width	description
+	---------------------------------------------------------------------
+	PPORT			4	Destination physical port #
+	TX_OFFLOAD		1	Hardware offload modes:
+					  0: no offload
+					  1: insert IP csum (ipv4 only)
+					  2: insert TCP/UDP csum
+					  3: L3 csum calc and insert
+                        	             into csum offset (TX_L3_CSUM_OFF)
+                 	                    16-bit 1's complement csum value.
+                                	     IPv4 pseudo-header and IP
+                        	             already calculated by OS
+                  	                   and inserted.
+					  4: TSO (TCP Segmentation Offload)
+	TX_L3_CSUM_OFF		2	For L3 csum offload mode, the offset,
+					from the beginning of the packet,
+					of the csum field in the L3 header
+	TX_TSO_MSS		2	For TSO offload mode, the
+					Maximum Segment Size in bytes
+        TX_TSO_HDR_LEN		2	For TSO offload mode, the
+					length of ethernet, IP, and
+					TCP/UDP headers, including IP
+					and TCP options.
+	TX_FRAGS		<array>	Packet fragments
+	  TX_FRAG		<nest>	Packet fragment
+	    TX_FRAG_ADDR	8	DMA address of packet fragment
+	    TX_FRAG_LEN		2	Packet fragment length
+
+Possible status return codes in descriptor on completion are:
+
+	DESC_COMP_ERR	reason
+	--------------------------------------------------------------------
+	0		OK
+	-ROCKER_ENXIO	address or data read err on desc buf or packet
+			fragment
+	-ROCKER_EINVAL	bad pport or TSO or csum offloading error
+	-ROCKER_ENOMEM	no memory for internal staging tx fragment
+
+Rx Packet Processing
+--------------------
+
+For packets ingressing on switch ports that are not forwarded by the switch but
+rather directed to the host CPU for further processing are delivered in the DMA
+RX ring.  Rx descriptor buffers are allocated by software and placed on the
+ring.  Hardware will fill Rx descriptor buffers with packet data, write the
+completion, and signal to software that a new packet is ready.  Since Rx packet
+size is not known a-priori, the Rx descriptor buffer must be allocated for
+worst-case packet size.  A single Rx descriptor will contain the entire Rx
+packet data in one RX_FRAG.  Other Rx TLVs describe and hardware offloads
+performed on the packet, such as checksum validation.
+
+The TLVs for Rx descriptor buffer are:
+
+	field		width	description
+	---------------------------------------------------
+	PPORT		4	Source physical port #
+	RX_FLAGS	2	Packet parsing flags:
+				  (1 << 0): IPv4 packet
+				  (1 << 1): IPv6 packet
+				  (1 << 2): csum calculated
+				  (1 << 3): IPv4 csum good
+				  (1 << 4): IP fragment
+				  (1 << 5): TCP packet
+				  (1 << 6): UDP packet
+				  (1 << 7): TCP/UDP csum good
+				  (1 << 8): Offload forward
+	RX_CSUM		2	IP calculated checksum:
+				  IPv4: IP payload csum
+				  IPv6: header and payload csum
+				(Only valid is RX_FLAGS:csum calc is set)
+	RX_FRAG_ADDR	8	DMA address of packet fragment
+	RX_FRAG_MAX_LEN	2	Packet maximum fragment length
+	RX_FRAG_LEN	2	Actual packet fragment length after receive
+
+Offload forward RX_FLAG indicates the device has already forwarded the packet
+so the host CPU should not also forward the packet.
+
+Possible status return codes in descriptor on completion are:
+
+	DESC_COMP_ERR	reason
+	--------------------------------------------------------------------
+	0		OK
+	-ROCKER_ENXIO	address or data read err on desc buf
+	-ROCKER_ENOMEM	no memory for internal staging desc buf
+	-ROCKER_EMSGSIZE Rx descriptor buffer wasn't big enough to contain
+			packet data TLV and other TLVs.
+
+
+SECTION 10: OF-DPA Mode
+======================
+
+OF-DPA mode allows the switch to offload flow packet processing functions to
+hardware.  An OpenFlow controller would communicate with an OpenFlow agent
+installed on the switch.  The OpenFlow agent would (directly or indirectly)
+communicate with the Rocker switch driver, which in turn would program switch
+hardware with flow functionality, as defined in OF-DPA.  The block diagram is:
+
+		+–––––––––––––––----–––+
+		|        OF            |
+		|  Remote Controller   |
+		+––––––––+––----–––––––+
+		         |
+		         |
+		+––––––––+–––––––––+
+		|       OF         |
+		|   Local Agent    |
+		+––––––––––––––––––+
+		|                  |
+		|   Rocker Driver  |
+		+––––––––––––––––––+
+		    <this spec>
+		+––––––––––––––––––+
+		|                  |
+		|   Rocker Switch  |
+		+––––––––––––––––––+
+
+To participate in flow functions, ports must be configure for OF-DPA mode
+during switch initialization.
+
+OF-DPA Flow Table Interface
+---------------------------
+
+There are commands to add, modify, delete, and get stats of flow table entries.
+The commands are issued using the DMA CMD descriptor ring.  The following
+commands are defined:
+
+	CMD_ADD:		add an entry to flow table
+	CMD_MOD:		modify an entry in flow table
+	CMD_DEL:		delete an entry from flow table
+	CMD_GET_STATS:		get stats for flow entry
+
+TLVs for add and modify commands are:
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_CMD		2	CMD_[ADD|MOD]
+	OF_DPA_TBL		2	Flow table ID
+					  0: ingress port
+					  10: vlan
+					  20: termination mac
+					  30: unicast routing
+					  40: multicast routing
+					  50: bridging
+					  60: ACL policy
+	OF_DPA_PRIORITY		4	Flow priority
+	OF_DPA_HARDTIME		4	Hard timeout for flow
+	OF_DPA_IDLETIME		4	Idle timeout for flow
+	OF_DPA_COOKIE		8	Cookie
+
+Additional TLVs based on flow table ID:
+
+Table ID 0: ingress port
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_IN_PPORT		4	ingress physical port number
+	OF_DPA_GOTO_TBL		2	goto table ID; zero to drop
+
+Table ID 10: vlan
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_IN_PPORT		4	ingress physical port number
+	OF_DPA_VLAN_ID		2 (N)	vlan ID
+	OF_DPA_VLAN_ID_MASK	2 (N)	vlan ID mask
+	OF_DPA_GOTO_TBL		2	goto table ID; zero to drop
+	OF_DPA_NEW_VLAN_ID	2 (N)	new vlan ID
+
+Table ID 20: termination mac
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_IN_PPORT		4	ingress physical port number
+	OF_DPA_IN_PPORT_MASK	4	ingress physical port number mask
+	OF_DPA_ETHERTYPE	2 (N)	must be either 0x0800 or 0x86dd
+	OF_DPA_DST_MAC		6 (N)	destination MAC
+	OF_DPA_DST_MAC_MASK	6 (N)	destination MAC mask
+	OF_DPA_VLAN_ID		2 (N)	vlan ID
+	OF_DPA_VLAN_ID_MASK	2 (N)	vlan ID mask
+	OF_DPA_GOTO_TBL		2	only acceptable values are
+					unicast or multicast routing
+					table IDs
+	OF_DPA_OUT_PPORT	2	if specified, must be
+					controller, set zero otherwise
+
+Table ID 30: unicast routing
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_ETHERTYPE	2 (N)	must be either 0x0800 or 0x86dd
+	OF_DPA_DST_IP		4 (N)	destination IPv4 address.
+					Must be unicast address
+	OF_DPA_DST_IP_MASK	4 (N)	IP mask.  Must be prefix mask
+	OF_DPA_DST_IPV6		16 (N)	destination IPv6 address.
+					Must be unicast address
+	OF_DPA_DST_IPV6_MASK	16 (N)	IPv6 mask. Must be prefix mask
+	OF_DPA_GOTO_TBL		2	goto table ID; zero to drop
+	OF_DPA_GROUP_ID		4	data for GROUP action must
+					be an L3 Unicast group entry
+
+Table ID 40: multicast routing
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_ETHERTYPE	2 (N)	must be either 0x0800 or 0x86dd
+	OF_DPA_VLAN_ID		2 (N)	vlan ID
+	OF_DPA_SRC_IP		4 (N)	source IPv4. Optional,
+					can contain IPv4 address,
+					must be completely masked
+					if not used
+	OF_DPA_SRC_IP_MASK	4 (N)	IP Mask
+	OF_DPA_DST_IP		4 (N)	destination IPv4 address.
+					Must be multicast address
+	OF_DPA_SRC_IPV6		16 (N)	source IPv6 Address. Optional.
+					Can contain IPv6 address,
+					must be completely masked
+					if not used
+	OF_DPA_SRC_IPV6_MASK	16 (N)	IPv6 mask.
+	OF_DPA_DST_IPV6		16 (N)	destination IPv6 Address. Must
+					be multicast address
+					Must be multicast address
+	OF_DPA_GOTO_TBL		2	goto table ID; zero to drop
+	OF_DPA_GROUP_ID		4	data for GROUP action must
+					be an L3 multicast group entry
+
+Table ID 50: bridging
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_VLAN_ID		2 (N)	vlan ID
+	OF_DPA_TUNNEL_ID	4	tunnel ID
+	OF_DPA_DST_MAC		6 (N)	destination MAC
+	OF_DPA_DST_MAC_MASK	6 (N)	destination MAC mask
+	OF_DPA_GOTO_TBL		2	goto table ID; zero to drop
+	OF_DPA_GROUP_ID		4	data for GROUP action must
+					be a L2 Interface, L2
+					Multicast, L2 Flood,
+					or L2 Overlay group entry
+					as appropriate
+	OF_DPA_TUNNEL_LPORT	4	unicast Tenant Bridging
+					flows specify a tunnel
+					logical port ID
+	OF_DPA_OUT_PPORT	2	data for OUTPUT action,
+					restricted to CONTROLLER,
+					set to 0 otherwise
+
+Table ID 60: acl policy
+
+	field			width	description
+	----------------------------------------------------
+	OF_DPA_IN_PPORT		4	ingress physical port number
+	OF_DPA_IN_PPORT_MASK	4	ingress physical port number mask
+	OF_DPA_ETHERTYPE	2 (N)	ethertype
+	OF_DPA_VLAN_ID		2 (N)	vlan ID
+	OF_DPA_VLAN_ID_MASK	2 (N)	vlan ID mask
+	OF_DPA_VLAN_PCP		2 (N)	vlan Priority Code Point
+	OF_DPA_VLAN_PCP_MASK	2 (N)	vlan Priority Code Point mask
+	OF_DPA_SRC_MAC		6 (N)	source MAC
+	OF_DPA_SRC_MAC_MASK	6 (N)	source MAC mask
+	OF_DPA_DST_MAC		6 (N)	destination MAC
+	OF_DPA_DST_MAC_MASK	6 (N)	destination MAC mask
+	OF_DPA_TUNNEL_ID	4	tunnel ID
+	OF_DPA_SRC_IP		4 (N)	source IPv4. Optional,
+					can contain IPv4 address,
+					must be completely masked
+					if not used
+	OF_DPA_SRC_IP_MASK	4 (N)	IP Mask
+	OF_DPA_DST_IP		4 (N)	destination IPv4 address.
+					Must be multicast address
+	OF_DPA_DST_IP_MASK	4 (N)	IP Mask
+	OF_DPA_SRC_IPV6		16 (N)	source IPv6 Address. Optional.
+					Can contain IPv6 address,
+					must be completely masked
+					if not used
+	OF_DPA_SRC_IPV6_MASK	16 (N)	IPv6 mask
+	OF_DPA_DST_IPV6		16 (N)	destination IPv6 Address. Must
+					be multicast address.
+	OF_DPA_DST_IPV6_MASK	16 (N)	IPv6 mask
+	OF_DPA_SRC_ARP_IP	4 (N)	source IPv4 address in the ARP
+					payload.  Only used if ethertype
+					== 0x0806.
+	OF_DPA_SRC_ARP_IP_MASK	4 (N)	IP Mask
+	OF_DPA_IP_PROTO		1	IP protocol
+	OF_DPA_IP_PROTO_MASK	1	IP protocol mask
+	OF_DPA_IP_DSCP		1	DSCP
+	OF_DPA_IP_DSCP_MASK	1	DSCP mask
+	OF_DPA_IP_ECN		1	ECN
+	OF_DPA_IP_ECN_MASK		1	ECN mask
+	OF_DPA_L4_SRC_PORT	2 (N)	L4 source port, only for
+					TCP, UDP, or SCTP
+	OF_DPA_L4_SRC_PORT_MASK	2 (N)	L4 source port mask
+	OF_DPA_L4_DST_PORT	2 (N)	L4 source port, only for
+					TCP, UDP, or SCTP
+	OF_DPA_L4_DST_PORT_MASK	2 (N)	L4 source port mask
+	OF_DPA_ICMP_TYPE	1	ICMP type, only if IP
+					protocol is 1
+	OF_DPA_ICMP_TYPE_MASK	1	ICMP type mask
+	OF_DPA_ICMP_CODE	1	ICMP code
+	OF_DPA_ICMP_CODE_MASK	1	ICMP code mask
+	OF_DPA_IPV6_LABEL	4 (N)	IPv6 flow label
+	OF_DPA_IPV6_LABEL_MASK	4 (N)	IPv6 flow label mask
+	OF_DPA_GROUP_ID		4	data for GROUP action
+	OF_DPA_QUEUE_ID_ACTION	1	write the queue ID
+	OF_DPA_NEW_QUEUE_ID	1	queue ID
+	OF_DPA_VLAN_PCP_ACTION	1	write the VLAN priority
+	OF_DPA_NEW_VLAN_PCP	1	VLAN priority
+	OF_DPA_IP_DSCP_ACTION	1	write the DSCP
+	OF_DPA_NEW_IP_DSCP	1	new DSCP
+	OF_DPA_TUNNEL_LPORT	4	restrct to valid tunnel
+					logical port, set to 0
+					otherwise.
+	OF_DPA_OUT_PPORT	2	data for OUTPUT action,
+					restricted to CONTROLLER,
+					set to 0 otherwise
+	OF_DPA_CLEAR_ACTIONS	4	if 1 packets matching flow are
+					dropped (all other instructions
+					ignored)
+
+TLVs for flow delete and get stats command are:
+
+	field			width	description
+	---------------------------------------------------
+	OF_DPA_CMD		2	CMD_[DEL|GET_STATS]
+	OF_DPA_COOKIE		8	Cookie
+
+On completion of get stats command, the descriptor buffer is written back with
+the following TLVs:
+
+	field			width	description
+	---------------------------------------------------
+	OF_DPA_STAT_DURATION	4	Flow duration
+	OF_DPA_STAT_RX_PKTS	8	Received packets
+	OF_DPA_STAT_TX_PKTS	8	Transmit packets
+
+Possible status return codes in descriptor on completion are:
+
+	DESC_COMP_ERR	command			reason
+	--------------------------------------------------------------------
+	0		all			OK
+	-ROCKER_EFAULT	all			head or tail index outside
+						of ring
+	-ROCKER_ENXIO	all			address or data read err on
+						desc buf
+	-ROCKER_EMSGSIZE GET_STATS		cmd descriptor buffer wasn't
+						big enough to contain write-back
+						TLVs
+	-ROCKER_EINVAL	all			invalid parameters passed in
+	-ROCKER_EEXIST	ADD			entry already exists
+	-ROCKER_ENOSPC	ADD			no space left in flow table
+	-ROCKER_ENOENT	MOD|DEL|GET_STATS	cookie invalid
+
+Group Table Interface
+---------------------
+
+There are commands to add, modify, delete, and get stats of group table
+entries.  The commands are issued using the DMA CMD descriptor ring.  The
+following commands are defined:
+
+	CMD_ADD:		add an entry to group table
+	CMD_MOD:		modify an entry in group table
+	CMD_DEL:		delete an entry from group table
+	CMD_GET_STATS:		get stats for group entry
+
+TLVs for add and modify commands are:
+
+	field			width	description
+	-----------------------------------------------------------
+	FLOW_GROUP_CMD		2	CMD_[ADD|MOD]
+	FLOW_GROUP_ID		2	Flow group ID
+	FLOW_GROUP_TYPE		1	Group type:
+					  0: L2 interface
+					  1: L2 rewrite
+					  2: L3 unicast
+					  3: L2 multicast
+					  4: L2 flood
+					  5: L3 interface
+					  6: L3 multicast
+					  7: L3 ECMP
+					  8: L2 overlay
+	FLOW_VLAN_ID		2	Vlan ID (types 0, 3, 4, 6)
+	FLOW_L2_PORT		2	Port (types 0)
+	FLOW_INDEX		4	Index (all types but 0)
+	FLOW_OVERLAY_TYPE	1	Overlay sub-type (type 8):
+					  0: Flood unicast tunnel
+					  1: Flood multicast tunnel
+					  2: Multicast unicast tunnel
+					  3: Multicast multicast tunnel
+	FLOW_GROUP_ACTION		nest
+	  FLOW_GROUP_ID		2	next group ID in chain (all
+					types except 0)
+	  FLOW_OUT_PORT		4	egress port (types 0, 8)
+	  FLOW_POP_VLAN_TAG	1	strip outer VLAN tag (type 1
+					only)
+	  FLOW_VLAN_ID		2	(types 1, 5)
+	  FLOW_SRC_MAC		6	(types 1, 2, 5)
+	  FLOW_DST_MAC		6	(types 1, 2)
+
+TLVs for flow delete and get stats command are:
+
+	field			width	description
+	-----------------------------------------------------------
+	FLOW_GROUP_CMD		2	CMD_[DEL|GET_STATS]
+	FLOW_GROUP_ID		2	Flow group ID
+
+On completion of get stats command, the descriptor buffer is written back with
+the following TLVs:
+
+	field			width	description
+	---------------------------------------------------
+	FLOW_GROUP_ID		2	Flow group ID
+	FLOW_STAT_DURATION	4	Flow duration
+	FLOW_STAT_REF_COUNT	4	Flow reference count
+	FLOW_STAT_BUCKET_COUNT	4	Flow bucket count
+
+Possible status return codes in descriptor on completion are:
+
+	DESC_COMP_ERR	command			reason
+	--------------------------------------------------------------------
+	0		all			OK
+	-ROCKER_EFAULT	all			head or tail index outside
+						of ring
+	-ROCKER_ENXIO	all			address or data read err on
+						desc buf
+	-ROCKER_ENOSPC	GET_STATS		cmd descriptor buffer wasn't
+						big enough to contain write-back
+						TLVs
+	-ROCKER_EINVAL	ADD|MOD			invalid parameters passed in
+	-ROCKER_EEXIST	ADD			entry already exists
+	-ROCKER_ENOSPC	ADD			no space left in flow table
+	-ROCKER_ENOENT	MOD|DEL|GET_STATS	group ID invalid
+	-ROCKER_EBUSY	DEL			group reference count non-zero
+	-ROCKER_ENODEV	ADD			next group ID doesn't exist
+
+
+
+References
+==========
+
+[1] OpenFlow Data Plane Abstraction (OF-DPA) Abstract Switch Specification,
+Version 1.0, from Broadcom Corporation, February 21, 2014.
diff --git a/docs/specs/vhost-user.txt b/docs/specs/vhost-user.txt
index 650bb18..7890d71 100644
--- a/docs/specs/vhost-user.txt
+++ b/docs/specs/vhost-user.txt
@@ -37,6 +37,8 @@
  * Flags: 32-bit bit field:
    - Lower 2 bits are the version (currently 0x01)
    - Bit 2 is the reply flag - needs to be sent on each reply from the slave
+   - Bit 3 is the need_reply flag - see VHOST_USER_PROTOCOL_F_REPLY_ACK for
+     details.
  * Size - 32-bit size of the payload
 
 
@@ -87,6 +89,14 @@
    User address: a 64-bit user address
    mmap offset: 64-bit offset where region starts in the mapped memory
 
+* Log description
+   ---------------------------
+   | log size | log offset |
+   ---------------------------
+   log size: size of area used for logging
+   log offset: offset from start of supplied file descriptor
+       where logging starts (i.e. where guest address 0 would be logged)
+
 In QEMU the vhost-user message is implemented with the following struct:
 
 typedef struct VhostUserMsg {
@@ -98,6 +108,7 @@
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
+        VhostUserLog log;
     };
 } QEMU_PACKED VhostUserMsg;
 
@@ -113,12 +124,17 @@
 the ones that do:
 
  * VHOST_GET_FEATURES
+ * VHOST_GET_PROTOCOL_FEATURES
  * VHOST_GET_VRING_BASE
+ * VHOST_SET_LOG_BASE (if VHOST_USER_PROTOCOL_F_LOG_SHMFD)
+
+[ Also see the section on REPLY_ACK protocol extension. ]
 
 There are several messages that the master sends with file descriptors passed
 in the ancillary data:
 
  * VHOST_SET_MEM_TABLE
+ * VHOST_SET_LOG_BASE (if VHOST_USER_PROTOCOL_F_LOG_SHMFD)
  * VHOST_SET_LOG_FD
  * VHOST_SET_VRING_KICK
  * VHOST_SET_VRING_CALL
@@ -127,6 +143,123 @@
 If Master is unable to send the full message or receives a wrong reply it will
 close the connection. An optional reconnection mechanism can be implemented.
 
+Any protocol extensions are gated by protocol feature bits,
+which allows full backwards compatibility on both master
+and slave.
+As older slaves don't support negotiating protocol features,
+a feature bit was dedicated for this purpose:
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+Starting and stopping rings
+----------------------
+Client must only process each ring when it is started.
+
+Client must only pass data between the ring and the
+backend, when the ring is enabled.
+
+If ring is started but disabled, client must process the
+ring without talking to the backend.
+
+For example, for a networking device, in the disabled state
+client must not supply any new RX packets, but must process
+and discard any TX packets.
+
+If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated, the ring is initialized
+in an enabled state.
+
+If VHOST_USER_F_PROTOCOL_FEATURES has been negotiated, the ring is initialized
+in a disabled state. Client must not pass data to/from the backend until ring is enabled by
+VHOST_USER_SET_VRING_ENABLE with parameter 1, or after it has been disabled by
+VHOST_USER_SET_VRING_ENABLE with parameter 0.
+
+Each ring is initialized in a stopped state, client must not process it until
+ring is started, or after it has been stopped.
+
+Client must start ring upon receiving a kick (that is, detecting that file
+descriptor is readable) on the descriptor specified by
+VHOST_USER_SET_VRING_KICK, and stop ring upon receiving
+VHOST_USER_GET_VRING_BASE.
+
+While processing the rings (whether they are enabled or not), client must
+support changing some configuration aspects on the fly.
+
+Multiple queue support
+----------------------
+
+Multiple queue is treated as a protocol extension, hence the slave has to
+implement protocol features first. The multiple queues feature is supported
+only when the protocol feature VHOST_USER_PROTOCOL_F_MQ (bit 0) is set.
+
+The max number of queues the slave supports can be queried with message
+VHOST_USER_GET_PROTOCOL_FEATURES. Master should stop when the number of
+requested queues is bigger than that.
+
+As all queues share one connection, the master uses a unique index for each
+queue in the sent message to identify a specified queue. One queue pair
+is enabled initially. More queues are enabled dynamically, by sending
+message VHOST_USER_SET_VRING_ENABLE.
+
+Migration
+---------
+
+During live migration, the master may need to track the modifications
+the slave makes to the memory mapped regions. The client should mark
+the dirty pages in a log. Once it complies to this logging, it may
+declare the VHOST_F_LOG_ALL vhost feature.
+
+To start/stop logging of data/used ring writes, server may send messages
+VHOST_USER_SET_FEATURES with VHOST_F_LOG_ALL and VHOST_USER_SET_VRING_ADDR with
+VHOST_VRING_F_LOG in ring's flags set to 1/0, respectively.
+
+All the modifications to memory pointed by vring "descriptor" should
+be marked. Modifications to "used" vring should be marked if
+VHOST_VRING_F_LOG is part of ring's flags.
+
+Dirty pages are of size:
+#define VHOST_LOG_PAGE 0x1000
+
+The log memory fd is provided in the ancillary data of
+VHOST_USER_SET_LOG_BASE message when the slave has
+VHOST_USER_PROTOCOL_F_LOG_SHMFD protocol feature.
+
+The size of the log is supplied as part of VhostUserMsg
+which should be large enough to cover all known guest
+addresses. Log starts at the supplied offset in the
+supplied file descriptor.
+The log covers from address 0 to the maximum of guest
+regions. In pseudo-code, to mark page at "addr" as dirty:
+
+page = addr / VHOST_LOG_PAGE
+log[page / 8] |= 1 << page % 8
+
+Where addr is the guest physical address.
+
+Use atomic operations, as the log may be concurrently manipulated.
+
+Note that when logging modifications to the used ring (when VHOST_VRING_F_LOG
+is set for this ring), log_guest_addr should be used to calculate the log
+offset: the write to first byte of the used ring is logged at this offset from
+log start. Also note that this value might be outside the legal guest physical
+address range (i.e. does not have to be covered by the VhostUserMemory table),
+but the bit offset of the last byte of the ring must fall within
+the size supplied by VhostUserLog.
+
+VHOST_USER_SET_LOG_FD is an optional message with an eventfd in
+ancillary data, it may be used to inform the master that the log has
+been modified.
+
+Once the source has finished migration, rings will be stopped by
+the source. No further update must be done before rings are
+restarted.
+
+Protocol features
+-----------------
+
+#define VHOST_USER_PROTOCOL_F_MQ             0
+#define VHOST_USER_PROTOCOL_F_LOG_SHMFD      1
+#define VHOST_USER_PROTOCOL_F_RARP           2
+#define VHOST_USER_PROTOCOL_F_REPLY_ACK      3
+
 Message types
 -------------
 
@@ -138,6 +271,8 @@
       Slave payload: u64
 
       Get from the underlying vhost implementation the features bitmask.
+      Feature bit VHOST_USER_F_PROTOCOL_FEATURES signals slave support for
+      VHOST_USER_GET_PROTOCOL_FEATURES and VHOST_USER_SET_PROTOCOL_FEATURES.
 
  * VHOST_USER_SET_FEATURES
 
@@ -146,6 +281,33 @@
       Master payload: u64
 
       Enable features in the underlying vhost implementation using a bitmask.
+      Feature bit VHOST_USER_F_PROTOCOL_FEATURES signals slave support for
+      VHOST_USER_GET_PROTOCOL_FEATURES and VHOST_USER_SET_PROTOCOL_FEATURES.
+
+ * VHOST_USER_GET_PROTOCOL_FEATURES
+
+      Id: 15
+      Equivalent ioctl: VHOST_GET_FEATURES
+      Master payload: N/A
+      Slave payload: u64
+
+      Get the protocol feature bitmask from the underlying vhost implementation.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES.
+      Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must support
+      this message even before VHOST_USER_SET_FEATURES was called.
+
+ * VHOST_USER_SET_PROTOCOL_FEATURES
+
+      Id: 16
+      Ioctl: VHOST_SET_FEATURES
+      Master payload: u64
+
+      Enable protocol features in the underlying vhost implementation.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES.
+      Note: slave that reported VHOST_USER_F_PROTOCOL_FEATURES must support
+      this message even before VHOST_USER_SET_FEATURES was called.
 
  * VHOST_USER_SET_OWNER
 
@@ -160,11 +322,13 @@
  * VHOST_USER_RESET_OWNER
 
       Id: 4
-      Equivalent ioctl: VHOST_RESET_OWNER
       Master payload: N/A
 
-      Issued when a new connection is about to be closed. The Master will no
-      longer own this connection (and will usually close it).
+      This is no longer used. Used to be sent to request disabling
+      all rings, but some clients interpreted it to also discard
+      connection state (this interpretation would lead to bugs).
+      It is recommended that clients either ignore this message,
+      or use it to disable all rings.
 
  * VHOST_USER_SET_MEM_TABLE
 
@@ -182,8 +346,14 @@
       Id: 6
       Equivalent ioctl: VHOST_SET_LOG_BASE
       Master payload: u64
+      Slave payload: N/A
 
-      Sets the logging base address.
+      Sets logging shared memory space.
+      When slave has VHOST_USER_PROTOCOL_F_LOG_SHMFD protocol
+      feature, the log memory fd is provided in the ancillary data of
+      VHOST_USER_SET_LOG_BASE message, the size and offset of shared
+      memory area provided in the message.
+
 
  * VHOST_USER_SET_LOG_FD
 
@@ -199,7 +369,7 @@
       Equivalent ioctl: VHOST_SET_VRING_NUM
       Master payload: vring state description
 
-      Sets the number of vrings for this owner.
+      Set the size of the queue.
 
  * VHOST_USER_SET_VRING_ADDR
 
@@ -264,3 +434,59 @@
       Bits (0-7) of the payload contain the vring index. Bit 8 is the
       invalid FD flag. This flag is set when there is no file descriptor
       in the ancillary data.
+
+ * VHOST_USER_GET_QUEUE_NUM
+
+      Id: 17
+      Equivalent ioctl: N/A
+      Master payload: N/A
+      Slave payload: u64
+
+      Query how many queues the backend supports. This request should be
+      sent only when VHOST_USER_PROTOCOL_F_MQ is set in queried protocol
+      features by VHOST_USER_GET_PROTOCOL_FEATURES.
+
+ * VHOST_USER_SET_VRING_ENABLE
+
+      Id: 18
+      Equivalent ioctl: N/A
+      Master payload: vring state description
+
+      Signal slave to enable or disable corresponding vring.
+      This request should be sent only when VHOST_USER_F_PROTOCOL_FEATURES
+      has been negotiated.
+
+ * VHOST_USER_SEND_RARP
+
+      Id: 19
+      Equivalent ioctl: N/A
+      Master payload: u64
+
+      Ask vhost user backend to broadcast a fake RARP to notify the migration
+      is terminated for guest that does not support GUEST_ANNOUNCE.
+      Only legal if feature bit VHOST_USER_F_PROTOCOL_FEATURES is present in
+      VHOST_USER_GET_FEATURES and protocol feature bit VHOST_USER_PROTOCOL_F_RARP
+      is present in VHOST_USER_GET_PROTOCOL_FEATURES.
+      The first 6 bytes of the payload contain the mac address of the guest to
+      allow the vhost user backend to construct and broadcast the fake RARP.
+
+VHOST_USER_PROTOCOL_F_REPLY_ACK:
+-------------------------------
+The original vhost-user specification only demands replies for certain
+commands. This differs from the vhost protocol implementation where commands
+are sent over an ioctl() call and block until the client has completed.
+
+With this protocol extension negotiated, the sender (QEMU) can set the
+"need_reply" [Bit 3] flag to any command. This indicates that
+the client MUST respond with a Payload VhostUserMsg indicating success or
+failure. The payload should be set to zero on success or non-zero on failure,
+unless the message already has an explicit reply body.
+
+The response payload gives QEMU a deterministic indication of the result
+of the command. Today, QEMU is expected to terminate the main vhost-user
+loop upon receiving such errors. In future, qemu could be taught to be more
+resilient for selective requests.
+
+For the message types that already solicit a reply from the client, the
+presence of VHOST_USER_PROTOCOL_F_REPLY_ACK or need_reply bit being set brings
+no behavioural change. (See the 'Communication' section for details.)
diff --git a/docs/throttle.txt b/docs/throttle.txt
new file mode 100644
index 0000000..26d4d51
--- /dev/null
+++ b/docs/throttle.txt
@@ -0,0 +1,252 @@
+The QEMU throttling infrastructure
+==================================
+Copyright (C) 2016 Igalia, S.L.
+Author: Alberto Garcia <berto@igalia.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or
+later. See the COPYING file in the top-level directory.
+
+Introduction
+------------
+QEMU includes a throttling module that can be used to set limits to
+I/O operations. The code itself is generic and independent of the I/O
+units, but it is currently used to limit the number of bytes per second
+and operations per second (IOPS) when performing disk I/O.
+
+This document explains how to use the throttling code in QEMU, and how
+it works internally. The implementation is in throttle.c.
+
+
+Using throttling to limit disk I/O
+----------------------------------
+Two aspects of the disk I/O can be limited: the number of bytes per
+second and the number of operations per second (IOPS). For each one of
+them the user can set a global limit or separate limits for read and
+write operations. This gives us a total of six different parameters.
+
+I/O limits can be set using the throttling.* parameters of -drive, or
+using the QMP 'block_set_io_throttle' command. These are the names of
+the parameters for both cases:
+
+|-----------------------+-----------------------|
+| -drive                | block_set_io_throttle |
+|-----------------------+-----------------------|
+| throttling.iops-total | iops                  |
+| throttling.iops-read  | iops_rd               |
+| throttling.iops-write | iops_wr               |
+| throttling.bps-total  | bps                   |
+| throttling.bps-read   | bps_rd                |
+| throttling.bps-write  | bps_wr                |
+|-----------------------+-----------------------|
+
+It is possible to set limits for both IOPS and bps at the same time,
+and for each case we can decide whether to have separate read and
+write limits or not, but note that if iops-total is set then neither
+iops-read nor iops-write can be set. The same applies to bps-total and
+bps-read/write.
+
+The default value of these parameters is 0, and it means 'unlimited'.
+
+In its most basic usage, the user can add a drive to QEMU with a limit
+of 100 IOPS with the following -drive line:
+
+   -drive file=hd0.qcow2,throttling.iops-total=100
+
+We can do the same using QMP. In this case all these parameters are
+mandatory, so we must set to 0 the ones that we don't want to limit:
+
+   { "execute": "block_set_io_throttle",
+     "arguments": {
+        "device": "virtio0",
+        "iops": 100,
+        "iops_rd": 0,
+        "iops_wr": 0,
+        "bps": 0,
+        "bps_rd": 0,
+        "bps_wr": 0
+     }
+   }
+
+
+I/O bursts
+----------
+In addition to the basic limits we have just seen, QEMU allows the
+user to do bursts of I/O for a configurable amount of time. A burst is
+an amount of I/O that can exceed the basic limit. Bursts are useful to
+allow better performance when there are peaks of activity (the OS
+boots, a service needs to be restarted) while keeping the average
+limits lower the rest of the time.
+
+Two parameters control bursts: their length and the maximum amount of
+I/O they allow. These two can be configured separately for each one of
+the six basic parameters described in the previous section, but in
+this section we'll use 'iops-total' as an example.
+
+The I/O limit during bursts is set using 'iops-total-max', and the
+maximum length (in seconds) is set with 'iops-total-max-length'. So if
+we want to configure a drive with a basic limit of 100 IOPS and allow
+bursts of 2000 IOPS for 60 seconds, we would do it like this (the line
+is split for clarity):
+
+   -drive file=hd0.qcow2,
+          throttling.iops-total=100,
+          throttling.iops-total-max=2000,
+          throttling.iops-total-max-length=60
+
+Or, with QMP:
+
+   { "execute": "block_set_io_throttle",
+     "arguments": {
+        "device": "virtio0",
+        "iops": 100,
+        "iops_rd": 0,
+        "iops_wr": 0,
+        "bps": 0,
+        "bps_rd": 0,
+        "bps_wr": 0,
+        "iops_max": 2000,
+        "iops_max_length": 60,
+     }
+   }
+
+With this, the user can perform I/O on hd0.qcow2 at a rate of 2000
+IOPS for 1 minute before it's throttled down to 100 IOPS.
+
+The user will be able to do bursts again if there's a sufficiently
+long period of time with unused I/O (see below for details).
+
+The default value for 'iops-total-max' is 0 and it means that bursts
+are not allowed. 'iops-total-max-length' can only be set if
+'iops-total-max' is set as well, and its default value is 1 second.
+
+Here's the complete list of parameters for configuring bursts:
+
+|----------------------------------+-----------------------|
+| -drive                           | block_set_io_throttle |
+|----------------------------------+-----------------------|
+| throttling.iops-total-max        | iops_max              |
+| throttling.iops-total-max-length | iops_max_length       |
+| throttling.iops-read-max         | iops_rd_max           |
+| throttling.iops-read-max-length  | iops_rd_max_length    |
+| throttling.iops-write-max        | iops_wr_max           |
+| throttling.iops-write-max-length | iops_wr_max_length    |
+| throttling.bps-total-max         | bps_max               |
+| throttling.bps-total-max-length  | bps_max_length        |
+| throttling.bps-read-max          | bps_rd_max            |
+| throttling.bps-read-max-length   | bps_rd_max_length     |
+| throttling.bps-write-max         | bps_wr_max            |
+| throttling.bps-write-max-length  | bps_wr_max_length     |
+|----------------------------------+-----------------------|
+
+
+Controlling the size of I/O operations
+--------------------------------------
+When applying IOPS limits all I/O operations are treated equally
+regardless of their size. This means that the user can take advantage
+of this in order to circumvent the limits and submit one huge I/O
+request instead of several smaller ones.
+
+QEMU provides a setting called throttling.iops-size to prevent this
+from happening. This setting specifies the size (in bytes) of an I/O
+request for accounting purposes. Larger requests will be counted
+proportionally to this size.
+
+For example, if iops-size is set to 4096 then an 8KB request will be
+counted as two, and a 6KB request will be counted as one and a
+half. This only applies to requests larger than iops-size: smaller
+requests will be always counted as one, no matter their size.
+
+The default value of iops-size is 0 and it means that the size of the
+requests is never taken into account when applying IOPS limits.
+
+
+Applying I/O limits to groups of disks
+--------------------------------------
+In all the examples so far we have seen how to apply limits to the I/O
+performed on individual drives, but QEMU allows grouping drives so
+they all share the same limits.
+
+The way it works is that each drive with I/O limits is assigned to a
+group named using the throttling.group parameter. If this parameter is
+not specified, then the device name (i.e. 'virtio0', 'ide0-hd0') will
+be used as the group name.
+
+Limits set using the throttling.* parameters discussed earlier in this
+document apply to the combined I/O of all members of a group.
+
+Consider this example:
+
+   -drive file=hd1.qcow2,throttling.iops-total=6000,throttling.group=foo
+   -drive file=hd2.qcow2,throttling.iops-total=6000,throttling.group=foo
+   -drive file=hd3.qcow2,throttling.iops-total=3000,throttling.group=bar
+   -drive file=hd4.qcow2,throttling.iops-total=6000,throttling.group=foo
+   -drive file=hd5.qcow2,throttling.iops-total=3000,throttling.group=bar
+   -drive file=hd6.qcow2,throttling.iops-total=5000
+
+Here hd1, hd2 and hd4 are all members of a group named 'foo' with a
+combined IOPS limit of 6000, and hd3 and hd5 are members of 'bar'. hd6
+is left alone (technically it is part of a 1-member group).
+
+Limits are applied in a round-robin fashion so if there are concurrent
+I/O requests on several drives of the same group they will be
+distributed evenly.
+
+When I/O limits are applied to an existing drive using the QMP command
+'block_set_io_throttle', the following things need to be taken into
+account:
+
+   - I/O limits are shared within the same group, so new values will
+     affect all members and overwrite the previous settings. In other
+     words: if different limits are applied to members of the same
+     group, the last one wins.
+
+   - If 'group' is unset it is assumed to be the current group of that
+     drive. If the drive is not in a group yet, it will be added to a
+     group named after the device name.
+
+   - If 'group' is set then the drive will be moved to that group if
+     it was member of a different one. In this case the limits
+     specified in the parameters will be applied to the new group
+     only.
+
+   - I/O limits can be disabled by setting all of them to 0. In this
+     case the device will be removed from its group and the rest of
+     its members will not be affected. The 'group' parameter is
+     ignored.
+
+
+The Leaky Bucket algorithm
+--------------------------
+I/O limits in QEMU are implemented using the leaky bucket algorithm
+(specifically the "Leaky bucket as a meter" variant).
+
+This algorithm uses the analogy of a bucket that leaks water
+constantly. The water that gets into the bucket represents the I/O
+that has been performed, and no more I/O is allowed once the bucket is
+full.
+
+To see the way this corresponds to the throttling parameters in QEMU,
+consider the following values:
+
+  iops-total=100
+  iops-total-max=2000
+  iops-total-max-length=60
+
+  - Water leaks from the bucket at a rate of 100 IOPS.
+  - Water can be added to the bucket at a rate of 2000 IOPS.
+  - The size of the bucket is 2000 x 60 = 120000
+  - If 'iops-total-max' is unset then the bucket size is 100 x 60.
+
+The bucket is initially empty, therefore water can be added until it's
+full at a rate of 2000 IOPS (the burst rate). Once the bucket is full
+we can only add as much water as it leaks, therefore the I/O rate is
+reduced to 100 IOPS. If we add less water than it leaks then the
+bucket will start to empty, allowing for bursts again.
+
+Note that since water is leaking from the bucket even during bursts,
+it will take a bit more than 60 seconds at 2000 IOPS to fill it
+up. After those 60 seconds the bucket will have leaked 60 x 100 =
+6000, allowing for 3 more seconds of I/O at 2000 IOPS.
+
+Also, due to the way the algorithm works, longer burst can be done at
+a lower I/O rate, e.g. 1000 IOPS during 120 seconds.
diff --git a/docs/tracing.txt b/docs/tracing.txt
index 7117c5e..29f2f9a 100644
--- a/docs/tracing.txt
+++ b/docs/tracing.txt
@@ -23,20 +23,24 @@
 
 4. Pretty-print the binary trace file:
 
-    ./scripts/simpletrace.py trace-events trace-* # Override * with QEMU <pid>
+    ./scripts/simpletrace.py trace-events-all trace-* # Override * with QEMU <pid>
 
 == Trace events ==
 
-There is a set of static trace events declared in the "trace-events" source
-file.  Each trace event declaration names the event, its arguments, and the
-format string which can be used for pretty-printing:
+Each directory in the source tree can declare a set of static trace events
+in a "trace-events" file. Each trace event declaration names the event, its
+arguments, and the format string which can be used for pretty-printing:
 
     qemu_vmalloc(size_t size, void *ptr) "size %zu ptr %p"
     qemu_vfree(void *ptr) "ptr %p"
 
-The "trace-events" file is processed by the "tracetool" script during build to
-generate code for the trace events.  Trace events are invoked directly from
-source code like this:
+All "trace-events" files must be listed in the "trace-event-y" make variable
+in the top level Makefile.objs. During build the individual files are combined
+to create a "trace-events-all" file, which is processed by the "tracetool"
+script during build to generate code for the trace events. The
+"trace-events-all" file is also installed into "/usr/share/qemu".
+
+Trace events are invoked directly from source code like this:
 
     #include "trace.h"  /* needed for trace event prototype */
     
@@ -157,9 +161,9 @@
 Note that regardless of the selected trace backend, events with the "disable"
 property will be generated with the "nop" backend.
 
-=== Stderr ===
+=== Log ===
 
-The "stderr" backend sends trace events directly to standard error.  This
+The "log" backend sends trace events directly to standard error.  This
 effectively turns trace events into debug printfs.
 
 This is the simplest backend and can be used together with existing code that
@@ -172,9 +176,6 @@
 trace backends but it is portable.  This is the recommended trace backend
 unless you have specific needs for more advanced backends.
 
-The "simple" backend currently does not capture string arguments, it simply
-records the char* pointer value instead of the string that is pointed to.
-
 === Ftrace ===
 
 The "ftrace" backend writes trace data to ftrace marker. This effectively
@@ -199,12 +200,12 @@
 ==== Analyzing trace files ====
 
 The "simple" backend produces binary trace files that can be formatted with the
-simpletrace.py script.  The script takes the "trace-events" file and the binary
-trace:
+simpletrace.py script.  The script takes the "trace-events-all" file and the
+binary trace:
 
-    ./scripts/simpletrace.py trace-events trace-12345
+    ./scripts/simpletrace.py trace-events-all trace-12345
 
-You must ensure that the same "trace-events" file was used to build QEMU,
+You must ensure that the same "trace-events-all" file was used to build QEMU,
 otherwise trace event declarations may have changed and output will not be
 consistent.
 
@@ -258,15 +259,15 @@
 performed manually after a build in order to change the binary name in the .stp
 probes:
 
-    scripts/tracetool --dtrace --stap \
-                      --binary path/to/qemu-binary \
-                      --target-type system \
-                      --target-name x86_64 \
-                      <trace-events >qemu.stp
+    scripts/tracetool.py --backends=dtrace --format=stap \
+                         --binary path/to/qemu-binary \
+                         --target-type system \
+                         --target-name x86_64 \
+                         <trace-events-all >qemu.stp
 
 == Trace event properties ==
 
-Each event in the "trace-events" file can be prefixed with a space-separated
+Each event in the "trace-events-all" file can be prefixed with a space-separated
 list of zero or more of the following event properties.
 
 === "disable" ===
@@ -278,7 +279,7 @@
 In this case you should declare such event with the "disable" property. This
 will effectively disable the event at compile time (by using the "nop" backend),
 thus having no performance impact at all on regular builds (i.e., unless you
-edit the "trace-events" file).
+edit the "trace-events-all" file).
 
 In addition, there might be cases where relatively complex computations must be
 performed to generate values that are only used as arguments for a trace
@@ -347,3 +348,44 @@
 and will generate the TCG code to call:
 
     void trace_foo(uint8_t a1, uint32_t a2);
+
+=== "vcpu" ===
+
+Identifies events that trace vCPU-specific information. It implicitly adds a
+"CPUState*" argument, and extends the tracing print format to show the vCPU
+information. If used together with the "tcg" property, it adds a second
+"TCGv_env" argument that must point to the per-target global TCG register that
+points to the vCPU when guest code is executed (usually the "cpu_env" variable).
+
+The following example events:
+
+    foo(uint32_t a) "a=%x"
+    vcpu bar(uint32_t a) "a=%x"
+    tcg vcpu baz(uint32_t a) "a=%x", "a=%x"
+
+Can be used as:
+
+    #include "trace-tcg.h"
+    
+    CPUArchState *env;
+    TCGv_ptr cpu_env;
+    
+    void some_disassembly_func(...)
+    {
+        /* trace emitted at this point */
+        trace_foo(0xd1);
+        /* trace emitted at this point */
+        trace_bar(ENV_GET_CPU(env), 0xd2);
+        /* trace emitted at this point (env) and when guest code is executed (cpu_env) */
+        trace_baz_tcg(ENV_GET_CPU(env), cpu_env, 0xd3);
+    }
+
+If the translating vCPU has address 0xc1 and code is later executed by vCPU
+0xc2, this would be an example output:
+
+    // at guest code translation
+    foo a=0xd1
+    bar cpu=0xc1 a=0xd2
+    baz_trans cpu=0xc1 a=0xd3
+    // at guest code execution
+    baz_exec cpu=0xc2 a=0xd3
diff --git a/docs/usb-storage.txt b/docs/usb-storage.txt
index c5a3866..fbc1f2e 100644
--- a/docs/usb-storage.txt
+++ b/docs/usb-storage.txt
@@ -40,6 +40,18 @@
 The 0+1+5 numbering from the "usb-uas" example isn't going to work
 with "usb-bot".
 
+Starting with qemu version 2.7 usb-bot and usb-uas devices can be
+hotplugged.  In the hotplug case they are added with "attached =
+false" so the guest will not see the device until the "attached"
+property is explicitly set to true.  That allows to attach one or more
+scsi devices before making the device visible to the guest, i.e. the
+workflow looks like this:
+
+   (1) device-add usb-bot,id=foo
+   (2) device-add scsi-{hd,cd},bus=foo.0,lun=0
+   (2b) optionally add more devices (luns 1 ... 15).
+   (3) scripts/qmp/qom-set foo.attached = true
+
 enjoy,
   Gerd
 
diff --git a/docs/virtio-migration.txt b/docs/virtio-migration.txt
new file mode 100644
index 0000000..98a6b0f
--- /dev/null
+++ b/docs/virtio-migration.txt
@@ -0,0 +1,108 @@
+Virtio devices and migration
+============================
+
+Copyright 2015 IBM Corp.
+
+This work is licensed under the terms of the GNU GPL, version 2 or later.  See
+the COPYING file in the top-level directory.
+
+Saving and restoring the state of virtio devices is a bit of a twisty maze,
+for several reasons:
+- state is distributed between several parts:
+  - virtio core, for common fields like features, number of queues, ...
+  - virtio transport (pci, ccw, ...), for the different proxy devices and
+    transport specific state (msix vectors, indicators, ...)
+  - virtio device (net, blk, ...), for the different device types and their
+    state (mac address, request queue, ...)
+- most fields are saved via the stream interface; subsequently, subsections
+  have been added to make cross-version migration possible
+
+This file attempts to document the current procedure and point out some
+caveats.
+
+
+Save state procedure
+====================
+
+virtio core               virtio transport          virtio device
+-----------               ----------------          -------------
+
+                                                    save() function registered
+                                                    via VMState wrapper on
+                                                    device class
+virtio_save()                                       <----------
+             ------>      save_config()
+                          - save proxy device
+                          - save transport-specific
+                            device fields
+- save common device
+  fields
+- save common virtqueue
+  fields
+             ------>      save_queue()
+                          - save transport-specific
+                            virtqueue fields
+             ------>                               save_device()
+                                                   - save device-specific
+                                                     fields
+- save subsections
+  - device endianness,
+    if changed from
+    default endianness
+  - 64 bit features, if
+    any high feature bit
+    is set
+  - virtio-1 virtqueue
+    fields, if VERSION_1
+    is set
+
+
+Load state procedure
+====================
+
+virtio core               virtio transport          virtio device
+-----------               ----------------          -------------
+
+                                                    load() function registered
+                                                    via VMState wrapper on
+                                                    device class
+virtio_load()                                       <----------
+             ------>      load_config()
+                          - load proxy device
+                          - load transport-specific
+                            device fields
+- load common device
+  fields
+- load common virtqueue
+  fields
+             ------>      load_queue()
+                          - load transport-specific
+                            virtqueue fields
+- notify guest
+             ------>                               load_device()
+                                                   - load device-specific
+                                                     fields
+- load subsections
+  - device endianness
+  - 64 bit features
+  - virtio-1 virtqueue
+    fields
+- sanitize endianness
+- sanitize features
+- virtqueue index sanity
+  check
+                                                   - feature-dependent setup
+
+
+Implications of this setup
+==========================
+
+Devices need to be careful in their state processing during load: The
+load_device() procedure is invoked by the core before subsections have
+been loaded. Any code that depends on information transmitted in subsections
+therefore has to be invoked in the device's load() function _after_
+virtio_load() returned (like e.g. code depending on features).
+
+Any extension of the state being migrated should be done in subsections
+added to the core for compatibility reasons. If transport or device specific
+state is added, core needs to invoke a callback from the new subsection.
diff --git a/docs/win32-qemu-event.promela b/docs/win32-qemu-event.promela
new file mode 100644
index 0000000..c446a71
--- /dev/null
+++ b/docs/win32-qemu-event.promela
@@ -0,0 +1,98 @@
+/*
+ * This model describes the implementation of QemuEvent in
+ * util/qemu-thread-win32.c.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This file is in the public domain.  If you really want a license,
+ * the WTFPL will do.
+ *
+ * To verify it:
+ *     spin -a docs/event.promela
+ *     gcc -O2 pan.c -DSAFETY
+ *     ./a.out
+ */
+
+bool event;
+int value;
+
+/* Primitives for a Win32 event */
+#define RAW_RESET event = false
+#define RAW_SET   event = true
+#define RAW_WAIT  do :: event -> break; od
+
+#if 0
+/* Basic sanity checking: test the Win32 event primitives */
+#define RESET     RAW_RESET
+#define SET       RAW_SET
+#define WAIT      RAW_WAIT
+#else
+/* Full model: layer a userspace-only fast path on top of the RAW_*
+ * primitives.  SET/RESET/WAIT have exactly the same semantics as
+ * RAW_SET/RAW_RESET/RAW_WAIT, but try to avoid invoking them.
+ */
+#define EV_SET 0
+#define EV_FREE 1
+#define EV_BUSY -1
+
+int state = EV_FREE;
+
+int xchg_result;
+#define SET   if :: state != EV_SET ->                                      \
+                    atomic { /* xchg_result=xchg(state, EV_SET) */          \
+                        xchg_result = state;                                \
+                        state = EV_SET;                                     \
+                    }                                                       \
+                    if :: xchg_result == EV_BUSY -> RAW_SET;                \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                 :: else -> skip;                                           \
+              fi
+
+#define RESET if :: state == EV_SET -> atomic { state = state | EV_FREE; }  \
+                 :: else            -> skip;                                \
+              fi
+
+int tmp1, tmp2;
+#define WAIT  tmp1 = state;                                                 \
+              if :: tmp1 != EV_SET ->                                       \
+                    if :: tmp1 == EV_FREE ->                                \
+                          RAW_RESET;                                        \
+                          atomic { /* tmp2=cas(state, EV_FREE, EV_BUSY) */  \
+                              tmp2 = state;                                 \
+                              if :: tmp2 == EV_FREE -> state = EV_BUSY;     \
+                                 :: else            -> skip;                \
+                              fi;                                           \
+                          }                                                 \
+                          if :: tmp2 == EV_SET -> tmp1 = EV_SET;            \
+                             :: else           -> tmp1 = EV_BUSY;           \
+                          fi;                                               \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                    assert(tmp1 != EV_FREE);                                \
+                    if :: tmp1 == EV_BUSY -> RAW_WAIT;                      \
+                       :: else -> skip;                                     \
+                    fi;                                                     \
+                 :: else -> skip;                                           \
+              fi
+#endif
+
+active proctype waiter()
+{
+     if
+         :: !value ->
+             RESET;
+             if
+                 :: !value -> WAIT;
+                 :: else   -> skip;
+             fi;
+        :: else -> skip;
+    fi;
+    assert(value);
+}
+
+active proctype notifier()
+{
+    value = true;
+    SET;
+}
diff --git a/docs/writing-qmp-commands.txt b/docs/writing-qmp-commands.txt
index f3df206..59aa77a 100644
--- a/docs/writing-qmp-commands.txt
+++ b/docs/writing-qmp-commands.txt
@@ -122,12 +122,12 @@
 Now a little hack is needed. As we're still using the old QMP server we need
 to add the new command to its internal dispatch table. This step won't be
 required in the near future. Open the qmp-commands.hx file and add the
-following in the botton:
+following at the bottom:
 
     {
         .name       = "hello-world",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_hello_world,
+        .mhandler.cmd_new = qmp_marshal_hello_world,
     },
 
 You're done. Now build qemu, run it as suggested in the "Testing" section,
@@ -179,7 +179,7 @@
     {
         .name       = "hello-world",
         .args_type  = "message:s?",
-        .mhandler.cmd_new = qmp_marshal_input_hello_world,
+        .mhandler.cmd_new = qmp_marshal_hello_world,
     },
 
 Notice that the "args_type" member got our "message" argument. The character
@@ -210,7 +210,7 @@
 === Errors ===
 
 QMP commands should use the error interface exported by the error.h header
-file. Basically, errors are set by calling the error_set() function.
+file. Basically, most errors are set by calling the error_setg() function.
 
 Let's say we don't accept the string "message" to contain the word "love". If
 it does contain it, we want the "hello-world" command to return an error:
@@ -219,8 +219,7 @@
 {
     if (has_message) {
         if (strstr(message, "love")) {
-            error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                      "the word 'love' is not allowed");
+            error_setg(errp, "the word 'love' is not allowed");
             return;
         }
         printf("%s\n", message);
@@ -229,10 +228,8 @@
     }
 }
 
-The first argument to the error_set() function is the Error pointer to pointer,
-which is passed to all QMP functions. The second argument is a ErrorClass
-value, which should be ERROR_CLASS_GENERIC_ERROR most of the time (more
-details about error classes are given below). The third argument is a human
+The first argument to the error_setg() function is the Error pointer
+to pointer, which is passed to all QMP functions. The next argument is a human
 description of the error, this is a free-form printf-like string.
 
 Let's test the example above. Build qemu, run it as defined in the "Testing"
@@ -249,8 +246,9 @@
     }
 }
 
-As a general rule, all QMP errors should use ERROR_CLASS_GENERIC_ERROR. There
-are two exceptions to this rule:
+As a general rule, all QMP errors should use ERROR_CLASS_GENERIC_ERROR
+(done by default when using error_setg()). There are two exceptions to
+this rule:
 
  1. A non-generic ErrorClass value exists* for the failure you want to report
     (eg. DeviceNotFound)
@@ -259,8 +257,8 @@
     want to report, hence you have to add a new ErrorClass value so that they
     can check for it
 
-If the failure you want to report doesn't fall in one of the two cases above,
-just report ERROR_CLASS_GENERIC_ERROR.
+If the failure you want to report falls into one of the two cases above,
+use error_set() with a second argument of an ErrorClass value.
 
  * All existing ErrorClass values are defined in the qapi-schema.json file
 
@@ -461,7 +459,7 @@
     {
         .name       = "query-alarm-clock",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_alarm_clock,
+        .mhandler.cmd_new = qmp_marshal_query_alarm_clock,
     },
 
 Time to test the new command. Build qemu, run it as described in the "Testing"
@@ -598,7 +596,7 @@
 to an TimerAlarmMethod instance.
 
 Notice that the "current" variable is used as "true" only in the first
-interation of the loop. That's because the alarm timer method in use is the
+iteration of the loop. That's because the alarm timer method in use is the
 first element of the alarm_timers array. Also notice that QAPI lists are handled
 by hand and we return the head of the list.
 
@@ -607,7 +605,7 @@
     {
         .name       = "query-alarm-methods",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_alarm_methods,
+        .mhandler.cmd_new = qmp_marshal_query_alarm_methods,
     },
 
 Now Build qemu, run it as explained in the "Testing" section and try our new
diff --git a/docs/xbzrle.txt b/docs/xbzrle.txt
index cc3a26a..52c8511 100644
--- a/docs/xbzrle.txt
+++ b/docs/xbzrle.txt
@@ -71,6 +71,14 @@
 encoded length 24
 e9 07 0f 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 03 01 67 01 01 69
 
+Cache update strategy
+=====================
+Keeping the hot pages in the cache is effective for decreased cache
+misses. XBZRLE uses a counter as the age of each page. The counter will
+increase after each ram dirty bitmap sync. When a cache conflict is
+detected, XBZRLE will only evict pages in the cache that are older than
+a threshold.
+
 Usage
 ======================
 1. Verify the destination QEMU version is able to decode the new format.
diff --git a/dtc b/dtc
index bc895d6..65cc4d2 160000
--- a/dtc
+++ b/dtc
@@ -1 +1 @@
-Subproject commit bc895d6d09695d05ceb8b52486ffe861d6cfbdde
+Subproject commit 65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf
diff --git a/dump.c b/dump.c
index 9c7dad8..f7b80d8 100644
--- a/dump.c
+++ b/dump.c
@@ -11,7 +11,8 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 #include "elf.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
@@ -22,8 +23,9 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/memory_mapping.h"
 #include "sysemu/cpus.h"
-#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
 #include "qmp-commands.h"
+#include "qapi-event.h"
 
 #include <zlib.h>
 #ifdef CONFIG_LZO
@@ -81,12 +83,6 @@
     return 0;
 }
 
-static void dump_error(DumpState *s, const char *reason, Error **errp)
-{
-    dump_cleanup(s);
-    error_setg(errp, "%s", reason);
-}
-
 static int fd_write_vmcore(const void *buf, size_t size, void *opaque)
 {
     DumpState *s = opaque;
@@ -127,7 +123,7 @@
 
     ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write elf header", errp);
+        error_setg(errp, "dump: failed to write elf header");
     }
 }
 
@@ -158,7 +154,7 @@
 
     ret = fd_write_vmcore(&elf_header, sizeof(elf_header), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write elf header", errp);
+        error_setg(errp, "dump: failed to write elf header");
     }
 }
 
@@ -181,7 +177,7 @@
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -204,7 +200,7 @@
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -224,7 +220,7 @@
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf64_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -244,7 +240,7 @@
         id = cpu_index(cpu);
         ret = cpu_write_elf64_note(f, cpu, id, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write elf notes", errp);
+            error_setg(errp, "dump: failed to write elf notes");
             return;
         }
     }
@@ -252,7 +248,7 @@
     CPU_FOREACH(cpu) {
         ret = cpu_write_elf64_qemunote(f, cpu, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write CPU status", errp);
+            error_setg(errp, "dump: failed to write CPU status");
             return;
         }
     }
@@ -274,7 +270,7 @@
 
     ret = fd_write_vmcore(&phdr, sizeof(Elf32_Phdr), s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write program header table", errp);
+        error_setg(errp, "dump: failed to write program header table");
     }
 }
 
@@ -289,7 +285,7 @@
         id = cpu_index(cpu);
         ret = cpu_write_elf32_note(f, cpu, id, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write elf notes", errp);
+            error_setg(errp, "dump: failed to write elf notes");
             return;
         }
     }
@@ -297,7 +293,7 @@
     CPU_FOREACH(cpu) {
         ret = cpu_write_elf32_qemunote(f, cpu, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to write CPU status", errp);
+            error_setg(errp, "dump: failed to write CPU status");
             return;
         }
     }
@@ -325,7 +321,7 @@
 
     ret = fd_write_vmcore(&shdr, shdr_size, s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write section header table", errp);
+        error_setg(errp, "dump: failed to write section header table");
     }
 }
 
@@ -335,7 +331,9 @@
 
     ret = fd_write_vmcore(buf, length, s);
     if (ret < 0) {
-        dump_error(s, "dump: failed to save memory", errp);
+        error_setg(errp, "dump: failed to save memory");
+    } else {
+        s->written_size += length;
     }
 }
 
@@ -346,18 +344,18 @@
     int64_t i;
     Error *local_err = NULL;
 
-    for (i = 0; i < size / TARGET_PAGE_SIZE; i++) {
-        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
-                   TARGET_PAGE_SIZE, &local_err);
+    for (i = 0; i < size / s->dump_info.page_size; i++) {
+        write_data(s, block->host_addr + start + i * s->dump_info.page_size,
+                   s->dump_info.page_size, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
         }
     }
 
-    if ((size % TARGET_PAGE_SIZE) != 0) {
-        write_data(s, block->host_addr + start + i * TARGET_PAGE_SIZE,
-                   size % TARGET_PAGE_SIZE, &local_err);
+    if ((size % s->dump_info.page_size) != 0) {
+        write_data(s, block->host_addr + start + i * s->dump_info.page_size,
+                   size % s->dump_info.page_size, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
@@ -567,11 +565,6 @@
     }
 }
 
-static void dump_completed(DumpState *s)
-{
-    dump_cleanup(s);
-}
-
 static int get_next_block(DumpState *s, GuestPhysBlock *block)
 {
     while (1) {
@@ -623,8 +616,6 @@
         }
 
     } while (!get_next_block(s, block));
-
-    dump_completed(s);
 }
 
 static void create_vmcore(DumpState *s, Error **errp)
@@ -736,7 +727,7 @@
 
     strncpy(dh->signature, KDUMP_SIGNATURE, strlen(KDUMP_SIGNATURE));
     dh->header_version = cpu_to_dump32(s, 6);
-    block_size = TARGET_PAGE_SIZE;
+    block_size = s->dump_info.page_size;
     dh->block_size = cpu_to_dump32(s, block_size);
     sub_hdr_size = sizeof(struct KdumpSubHeader32) + s->note_size;
     sub_hdr_size = DIV_ROUND_UP(sub_hdr_size, block_size);
@@ -764,7 +755,7 @@
     dh->status = cpu_to_dump32(s, status);
 
     if (write_buffer(s->fd, 0, dh, size) < 0) {
-        dump_error(s, "dump: failed to write disk dump header", errp);
+        error_setg(errp, "dump: failed to write disk dump header");
         goto out;
     }
 
@@ -774,7 +765,7 @@
 
     /* 64bit max_mapnr_64 */
     kh->max_mapnr_64 = cpu_to_dump64(s, s->max_mapnr);
-    kh->phys_base = cpu_to_dump32(s, PHYS_BASE);
+    kh->phys_base = cpu_to_dump32(s, s->dump_info.phys_base);
     kh->dump_level = cpu_to_dump32(s, DUMP_LEVEL);
 
     offset_note = DISKDUMP_HEADER_BLOCKS * block_size + size;
@@ -783,7 +774,7 @@
 
     if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
                      block_size, kh, size) < 0) {
-        dump_error(s, "dump: failed to write kdump sub header", errp);
+        error_setg(errp, "dump: failed to write kdump sub header");
         goto out;
     }
 
@@ -799,7 +790,7 @@
     }
     if (write_buffer(s->fd, offset_note, s->note_buf,
                      s->note_size) < 0) {
-        dump_error(s, "dump: failed to write notes", errp);
+        error_setg(errp, "dump: failed to write notes");
         goto out;
     }
 
@@ -836,7 +827,7 @@
 
     strncpy(dh->signature, KDUMP_SIGNATURE, strlen(KDUMP_SIGNATURE));
     dh->header_version = cpu_to_dump32(s, 6);
-    block_size = TARGET_PAGE_SIZE;
+    block_size = s->dump_info.page_size;
     dh->block_size = cpu_to_dump32(s, block_size);
     sub_hdr_size = sizeof(struct KdumpSubHeader64) + s->note_size;
     sub_hdr_size = DIV_ROUND_UP(sub_hdr_size, block_size);
@@ -864,7 +855,7 @@
     dh->status = cpu_to_dump32(s, status);
 
     if (write_buffer(s->fd, 0, dh, size) < 0) {
-        dump_error(s, "dump: failed to write disk dump header", errp);
+        error_setg(errp, "dump: failed to write disk dump header");
         goto out;
     }
 
@@ -874,7 +865,7 @@
 
     /* 64bit max_mapnr_64 */
     kh->max_mapnr_64 = cpu_to_dump64(s, s->max_mapnr);
-    kh->phys_base = cpu_to_dump64(s, PHYS_BASE);
+    kh->phys_base = cpu_to_dump64(s, s->dump_info.phys_base);
     kh->dump_level = cpu_to_dump32(s, DUMP_LEVEL);
 
     offset_note = DISKDUMP_HEADER_BLOCKS * block_size + size;
@@ -883,7 +874,7 @@
 
     if (write_buffer(s->fd, DISKDUMP_HEADER_BLOCKS *
                      block_size, kh, size) < 0) {
-        dump_error(s, "dump: failed to write kdump sub header", errp);
+        error_setg(errp, "dump: failed to write kdump sub header");
         goto out;
     }
 
@@ -900,7 +891,7 @@
 
     if (write_buffer(s->fd, offset_note, s->note_buf,
                      s->note_size) < 0) {
-        dump_error(s, "dump: failed to write notes", errp);
+        error_setg(errp, "dump: failed to write notes");
         goto out;
     }
 
@@ -927,9 +918,12 @@
     } else {
         create_header64(s, &local_err);
     }
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
+}
+
+static size_t dump_bitmap_get_bufsize(DumpState *s)
+{
+    return s->dump_info.page_size;
 }
 
 /*
@@ -945,6 +939,8 @@
     off_t old_offset, new_offset;
     off_t offset_bitmap1, offset_bitmap2;
     uint32_t byte, bit;
+    size_t bitmap_bufsize = dump_bitmap_get_bufsize(s);
+    size_t bits_per_buf = bitmap_bufsize * CHAR_BIT;
 
     /* should not set the previous place */
     assert(last_pfn <= pfn);
@@ -955,14 +951,14 @@
      * making new_offset be bigger than old_offset can also sync remained data
      * into vmcore.
      */
-    old_offset = BUFSIZE_BITMAP * (last_pfn / PFN_BUFBITMAP);
-    new_offset = BUFSIZE_BITMAP * (pfn / PFN_BUFBITMAP);
+    old_offset = bitmap_bufsize * (last_pfn / bits_per_buf);
+    new_offset = bitmap_bufsize * (pfn / bits_per_buf);
 
     while (old_offset < new_offset) {
         /* calculate the offset and write dump_bitmap */
         offset_bitmap1 = s->offset_dump_bitmap + old_offset;
         if (write_buffer(s->fd, offset_bitmap1, buf,
-                         BUFSIZE_BITMAP) < 0) {
+                         bitmap_bufsize) < 0) {
             return -1;
         }
 
@@ -970,17 +966,17 @@
         offset_bitmap2 = s->offset_dump_bitmap + s->len_dump_bitmap +
                          old_offset;
         if (write_buffer(s->fd, offset_bitmap2, buf,
-                         BUFSIZE_BITMAP) < 0) {
+                         bitmap_bufsize) < 0) {
             return -1;
         }
 
-        memset(buf, 0, BUFSIZE_BITMAP);
-        old_offset += BUFSIZE_BITMAP;
+        memset(buf, 0, bitmap_bufsize);
+        old_offset += bitmap_bufsize;
     }
 
     /* get the exact place of the bit in the buf, and set it */
-    byte = (pfn % PFN_BUFBITMAP) / CHAR_BIT;
-    bit = (pfn % PFN_BUFBITMAP) % CHAR_BIT;
+    byte = (pfn % bits_per_buf) / CHAR_BIT;
+    bit = (pfn % bits_per_buf) % CHAR_BIT;
     if (value) {
         buf[byte] |= 1u << bit;
     } else {
@@ -990,6 +986,20 @@
     return 0;
 }
 
+static uint64_t dump_paddr_to_pfn(DumpState *s, uint64_t addr)
+{
+    int target_page_shift = ctz32(s->dump_info.page_size);
+
+    return (addr >> target_page_shift) - ARCH_PFN_OFFSET;
+}
+
+static uint64_t dump_pfn_to_paddr(DumpState *s, uint64_t pfn)
+{
+    int target_page_shift = ctz32(s->dump_info.page_size);
+
+    return (pfn + ARCH_PFN_OFFSET) << target_page_shift;
+}
+
 /*
  * exam every page and return the page frame number and the address of the page.
  * bufptr can be NULL. note: the blocks here is supposed to reflect guest-phys
@@ -1000,16 +1010,16 @@
                           uint8_t **bufptr, DumpState *s)
 {
     GuestPhysBlock *block = *blockptr;
-    hwaddr addr;
+    hwaddr addr, target_page_mask = ~((hwaddr)s->dump_info.page_size - 1);
     uint8_t *buf;
 
     /* block == NULL means the start of the iteration */
     if (!block) {
         block = QTAILQ_FIRST(&s->guest_phys_blocks.head);
         *blockptr = block;
-        assert((block->target_start & ~TARGET_PAGE_MASK) == 0);
-        assert((block->target_end & ~TARGET_PAGE_MASK) == 0);
-        *pfnptr = paddr_to_pfn(block->target_start);
+        assert((block->target_start & ~target_page_mask) == 0);
+        assert((block->target_end & ~target_page_mask) == 0);
+        *pfnptr = dump_paddr_to_pfn(s, block->target_start);
         if (bufptr) {
             *bufptr = block->host_addr;
         }
@@ -1017,10 +1027,10 @@
     }
 
     *pfnptr = *pfnptr + 1;
-    addr = pfn_to_paddr(*pfnptr);
+    addr = dump_pfn_to_paddr(s, *pfnptr);
 
     if ((addr >= block->target_start) &&
-        (addr + TARGET_PAGE_SIZE <= block->target_end)) {
+        (addr + s->dump_info.page_size <= block->target_end)) {
         buf = block->host_addr + (addr - block->target_start);
     } else {
         /* the next page is in the next block */
@@ -1029,9 +1039,9 @@
         if (!block) {
             return false;
         }
-        assert((block->target_start & ~TARGET_PAGE_MASK) == 0);
-        assert((block->target_end & ~TARGET_PAGE_MASK) == 0);
-        *pfnptr = paddr_to_pfn(block->target_start);
+        assert((block->target_start & ~target_page_mask) == 0);
+        assert((block->target_end & ~target_page_mask) == 0);
+        *pfnptr = dump_paddr_to_pfn(s, block->target_start);
         buf = block->host_addr;
     }
 
@@ -1049,9 +1059,11 @@
     void *dump_bitmap_buf;
     size_t num_dumpable;
     GuestPhysBlock *block_iter = NULL;
+    size_t bitmap_bufsize = dump_bitmap_get_bufsize(s);
+    size_t bits_per_buf = bitmap_bufsize * CHAR_BIT;
 
     /* dump_bitmap_buf is used to store dump_bitmap temporarily */
-    dump_bitmap_buf = g_malloc0(BUFSIZE_BITMAP);
+    dump_bitmap_buf = g_malloc0(bitmap_bufsize);
 
     num_dumpable = 0;
     last_pfn = 0;
@@ -1063,7 +1075,7 @@
     while (get_next_page(&block_iter, &pfn, NULL, s)) {
         ret = set_dump_bitmap(last_pfn, pfn, true, dump_bitmap_buf, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to set dump_bitmap", errp);
+            error_setg(errp, "dump: failed to set dump_bitmap");
             goto out;
         }
 
@@ -1073,14 +1085,14 @@
 
     /*
      * set_dump_bitmap will always leave the recently set bit un-sync. Here we
-     * set last_pfn + PFN_BUFBITMAP to 0 and those set but un-sync bit will be
-     * synchronized into vmcore.
+     * set the remaining bits from last_pfn to the end of the bitmap buffer to
+     * 0. With those set, the un-sync bit will be synchronized into the vmcore.
      */
     if (num_dumpable > 0) {
-        ret = set_dump_bitmap(last_pfn, last_pfn + PFN_BUFBITMAP, false,
+        ret = set_dump_bitmap(last_pfn, last_pfn + bits_per_buf, false,
                               dump_bitmap_buf, s);
         if (ret < 0) {
-            dump_error(s, "dump: failed to sync dump_bitmap", errp);
+            error_setg(errp, "dump: failed to sync dump_bitmap");
             goto out;
         }
     }
@@ -1097,8 +1109,8 @@
 {
     data_cache->fd = s->fd;
     data_cache->data_size = 0;
-    data_cache->buf_size = BUFSIZE_DATA_CACHE;
-    data_cache->buf = g_malloc0(BUFSIZE_DATA_CACHE);
+    data_cache->buf_size = 4 * dump_bitmap_get_bufsize(s);
+    data_cache->buf = g_malloc0(data_cache->buf_size);
     data_cache->offset = offset;
 }
 
@@ -1192,7 +1204,7 @@
     prepare_data_cache(&page_data, s, offset_data);
 
     /* prepare buffer to store compressed data */
-    len_buf_out = get_len_buf_out(TARGET_PAGE_SIZE, s->flag_compress);
+    len_buf_out = get_len_buf_out(s->dump_info.page_size, s->flag_compress);
     assert(len_buf_out != 0);
 
 #ifdef CONFIG_LZO
@@ -1205,19 +1217,19 @@
      * init zero page's page_desc and page_data, because every zero page
      * uses the same page_data
      */
-    pd_zero.size = cpu_to_dump32(s, TARGET_PAGE_SIZE);
+    pd_zero.size = cpu_to_dump32(s, s->dump_info.page_size);
     pd_zero.flags = cpu_to_dump32(s, 0);
     pd_zero.offset = cpu_to_dump64(s, offset_data);
     pd_zero.page_flags = cpu_to_dump64(s, 0);
-    buf = g_malloc0(TARGET_PAGE_SIZE);
-    ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
+    buf = g_malloc0(s->dump_info.page_size);
+    ret = write_cache(&page_data, buf, s->dump_info.page_size, false);
     g_free(buf);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write page data (zero page)", errp);
+        error_setg(errp, "dump: failed to write page data (zero page)");
         goto out;
     }
 
-    offset_data += TARGET_PAGE_SIZE;
+    offset_data += s->dump_info.page_size;
 
     /*
      * dump memory to vmcore page by page. zero page will all be resided in the
@@ -1225,11 +1237,11 @@
      */
     while (get_next_page(&block_iter, &pfn_iter, &buf, s)) {
         /* check zero page */
-        if (is_zero_page(buf, TARGET_PAGE_SIZE)) {
+        if (is_zero_page(buf, s->dump_info.page_size)) {
             ret = write_cache(&page_desc, &pd_zero, sizeof(PageDescriptor),
                               false);
             if (ret < 0) {
-                dump_error(s, "dump: failed to write page desc", errp);
+                error_setg(errp, "dump: failed to write page desc");
                 goto out;
             }
         } else {
@@ -1247,56 +1259,57 @@
              size_out = len_buf_out;
              if ((s->flag_compress & DUMP_DH_COMPRESSED_ZLIB) &&
                     (compress2(buf_out, (uLongf *)&size_out, buf,
-                               TARGET_PAGE_SIZE, Z_BEST_SPEED) == Z_OK) &&
-                    (size_out < TARGET_PAGE_SIZE)) {
+                               s->dump_info.page_size, Z_BEST_SPEED) == Z_OK) &&
+                    (size_out < s->dump_info.page_size)) {
                 pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_ZLIB);
                 pd.size  = cpu_to_dump32(s, size_out);
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
 #ifdef CONFIG_LZO
             } else if ((s->flag_compress & DUMP_DH_COMPRESSED_LZO) &&
-                    (lzo1x_1_compress(buf, TARGET_PAGE_SIZE, buf_out,
+                    (lzo1x_1_compress(buf, s->dump_info.page_size, buf_out,
                     (lzo_uint *)&size_out, wrkmem) == LZO_E_OK) &&
-                    (size_out < TARGET_PAGE_SIZE)) {
+                    (size_out < s->dump_info.page_size)) {
                 pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_LZO);
                 pd.size  = cpu_to_dump32(s, size_out);
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
 #endif
 #ifdef CONFIG_SNAPPY
             } else if ((s->flag_compress & DUMP_DH_COMPRESSED_SNAPPY) &&
-                    (snappy_compress((char *)buf, TARGET_PAGE_SIZE,
+                    (snappy_compress((char *)buf, s->dump_info.page_size,
                     (char *)buf_out, &size_out) == SNAPPY_OK) &&
-                    (size_out < TARGET_PAGE_SIZE)) {
+                    (size_out < s->dump_info.page_size)) {
                 pd.flags = cpu_to_dump32(s, DUMP_DH_COMPRESSED_SNAPPY);
                 pd.size  = cpu_to_dump32(s, size_out);
 
                 ret = write_cache(&page_data, buf_out, size_out, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
 #endif
             } else {
                 /*
                  * fall back to save in plaintext, size_out should be
-                 * assigned TARGET_PAGE_SIZE
+                 * assigned the target's page size
                  */
                 pd.flags = cpu_to_dump32(s, 0);
-                size_out = TARGET_PAGE_SIZE;
+                size_out = s->dump_info.page_size;
                 pd.size = cpu_to_dump32(s, size_out);
 
-                ret = write_cache(&page_data, buf, TARGET_PAGE_SIZE, false);
+                ret = write_cache(&page_data, buf,
+                                  s->dump_info.page_size, false);
                 if (ret < 0) {
-                    dump_error(s, "dump: failed to write page data", errp);
+                    error_setg(errp, "dump: failed to write page data");
                     goto out;
                 }
             }
@@ -1308,20 +1321,21 @@
 
             ret = write_cache(&page_desc, &pd, sizeof(PageDescriptor), false);
             if (ret < 0) {
-                dump_error(s, "dump: failed to write page desc", errp);
+                error_setg(errp, "dump: failed to write page desc");
                 goto out;
             }
         }
+        s->written_size += s->dump_info.page_size;
     }
 
     ret = write_cache(&page_desc, NULL, 0, true);
     if (ret < 0) {
-        dump_error(s, "dump: failed to sync cache for page_desc", errp);
+        error_setg(errp, "dump: failed to sync cache for page_desc");
         goto out;
     }
     ret = write_cache(&page_data, NULL, 0, true);
     if (ret < 0) {
-        dump_error(s, "dump: failed to sync cache for page_data", errp);
+        error_setg(errp, "dump: failed to sync cache for page_data");
         goto out;
     }
 
@@ -1365,7 +1379,7 @@
 
     ret = write_start_flat_header(s->fd);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write start flat header", errp);
+        error_setg(errp, "dump: failed to write start flat header");
         return;
     }
 
@@ -1389,11 +1403,9 @@
 
     ret = write_end_flat_header(s->fd);
     if (ret < 0) {
-        dump_error(s, "dump: failed to write end flat header", errp);
+        error_setg(errp, "dump: failed to write end flat header");
         return;
     }
-
-    dump_completed(s);
 }
 
 static ram_addr_t get_start_block(DumpState *s)
@@ -1429,7 +1441,45 @@
     GuestPhysBlock *last_block;
 
     last_block = QTAILQ_LAST(&s->guest_phys_blocks.head, GuestPhysBlockHead);
-    s->max_mapnr = paddr_to_pfn(last_block->target_end);
+    s->max_mapnr = dump_paddr_to_pfn(s, last_block->target_end);
+}
+
+static DumpState dump_state_global = { .status = DUMP_STATUS_NONE };
+
+static void dump_state_prepare(DumpState *s)
+{
+    /* zero the struct, setting status to active */
+    *s = (DumpState) { .status = DUMP_STATUS_ACTIVE };
+}
+
+bool dump_in_progress(void)
+{
+    DumpState *state = &dump_state_global;
+    return (atomic_read(&state->status) == DUMP_STATUS_ACTIVE);
+}
+
+/* calculate total size of memory to be dumped (taking filter into
+ * acoount.) */
+static int64_t dump_calculate_size(DumpState *s)
+{
+    GuestPhysBlock *block;
+    int64_t size = 0, total = 0, left = 0, right = 0;
+
+    QTAILQ_FOREACH(block, &s->guest_phys_blocks.head, next) {
+        if (s->has_filter) {
+            /* calculate the overlapped region. */
+            left = MAX(s->begin, block->target_start);
+            right = MIN(s->begin + s->length, block->target_end);
+            size = right - left;
+            size = size > 0 ? size : 0;
+        } else {
+            /* count the whole region in */
+            size = (block->target_end - block->target_start);
+        }
+        total += size;
+    }
+
+    return total;
 }
 
 static void dump_init(DumpState *s, int fd, bool has_format,
@@ -1441,6 +1491,10 @@
     Error *err = NULL;
     int ret;
 
+    s->has_format = has_format;
+    s->format = format;
+    s->written_size = 0;
+
     /* kdump-compressed is conflict with paging and filter */
     if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
         assert(!paging && !has_filter);
@@ -1471,10 +1525,14 @@
 
     guest_phys_blocks_init(&s->guest_phys_blocks);
     guest_phys_blocks_append(&s->guest_phys_blocks);
+    s->total_size = dump_calculate_size(s);
+#ifdef DEBUG_DUMP_GUEST_MEMORY
+    fprintf(stderr, "DUMP: total memory to dump: %lu\n", s->total_size);
+#endif
 
     s->start = get_start_block(s);
     if (s->start == -1) {
-        error_set(errp, QERR_INVALID_PARAMETER, "begin");
+        error_setg(errp, QERR_INVALID_PARAMETER, "begin");
         goto cleanup;
     }
 
@@ -1484,14 +1542,18 @@
      */
     ret = cpu_get_dump_info(&s->dump_info, &s->guest_phys_blocks);
     if (ret < 0) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         goto cleanup;
     }
 
+    if (!s->dump_info.page_size) {
+        s->dump_info.page_size = TARGET_PAGE_SIZE;
+    }
+
     s->note_size = cpu_get_note_size(s->dump_info.d_class,
                                      s->dump_info.d_machine, nr_cpus);
     if (s->note_size < 0) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         goto cleanup;
     }
 
@@ -1511,8 +1573,9 @@
     get_max_mapnr(s);
 
     uint64_t tmp;
-    tmp = DIV_ROUND_UP(DIV_ROUND_UP(s->max_mapnr, CHAR_BIT), TARGET_PAGE_SIZE);
-    s->len_dump_bitmap = tmp * TARGET_PAGE_SIZE;
+    tmp = DIV_ROUND_UP(DIV_ROUND_UP(s->max_mapnr, CHAR_BIT),
+                       s->dump_info.page_size);
+    s->len_dump_bitmap = tmp * s->dump_info.page_size;
 
     /* init for kdump-compressed format */
     if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
@@ -1594,8 +1657,60 @@
     dump_cleanup(s);
 }
 
-void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
-                           int64_t begin, bool has_length,
+/* this operation might be time consuming. */
+static void dump_process(DumpState *s, Error **errp)
+{
+    Error *local_err = NULL;
+    DumpQueryResult *result = NULL;
+
+    if (s->has_format && s->format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
+        create_kdump_vmcore(s, &local_err);
+    } else {
+        create_vmcore(s, &local_err);
+    }
+
+    /* make sure status is written after written_size updates */
+    smp_wmb();
+    atomic_set(&s->status,
+               (local_err ? DUMP_STATUS_FAILED : DUMP_STATUS_COMPLETED));
+
+    /* send DUMP_COMPLETED message (unconditionally) */
+    result = qmp_query_dump(NULL);
+    /* should never fail */
+    assert(result);
+    qapi_event_send_dump_completed(result, !!local_err, (local_err ? \
+                                   error_get_pretty(local_err) : NULL),
+                                   &error_abort);
+    qapi_free_DumpQueryResult(result);
+
+    error_propagate(errp, local_err);
+    dump_cleanup(s);
+}
+
+static void *dump_thread(void *data)
+{
+    Error *err = NULL;
+    DumpState *s = (DumpState *)data;
+    dump_process(s, &err);
+    error_free(err);
+    return NULL;
+}
+
+DumpQueryResult *qmp_query_dump(Error **errp)
+{
+    DumpQueryResult *result = g_new(DumpQueryResult, 1);
+    DumpState *state = &dump_state_global;
+    result->status = atomic_read(&state->status);
+    /* make sure we are reading status and written_size in order */
+    smp_rmb();
+    result->completed = state->written_size;
+    result->total = state->total_size;
+    return result;
+}
+
+void qmp_dump_guest_memory(bool paging, const char *file,
+                           bool has_detach, bool detach,
+                           bool has_begin, int64_t begin, bool has_length,
                            int64_t length, bool has_format,
                            DumpGuestMemoryFormat format, Error **errp)
 {
@@ -1603,6 +1718,19 @@
     int fd = -1;
     DumpState *s;
     Error *local_err = NULL;
+    bool detach_p = false;
+
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        error_setg(errp, "Dump not allowed during incoming migration.");
+        return;
+    }
+
+    /* if there is a dump in background, we should wait until the dump
+     * finished */
+    if (dump_in_progress()) {
+        error_setg(errp, "There is a dump in process, please wait.");
+        return;
+    }
 
     /*
      * kdump-compressed format need the whole memory dumped, so paging or
@@ -1615,13 +1743,16 @@
         return;
     }
     if (has_begin && !has_length) {
-        error_set(errp, QERR_MISSING_PARAMETER, "length");
+        error_setg(errp, QERR_MISSING_PARAMETER, "length");
         return;
     }
     if (!has_begin && has_length) {
-        error_set(errp, QERR_MISSING_PARAMETER, "begin");
+        error_setg(errp, QERR_MISSING_PARAMETER, "begin");
         return;
     }
+    if (has_detach) {
+        detach_p = detach;
+    }
 
     /* check whether lzo/snappy is supported */
 #ifndef CONFIG_LZO
@@ -1656,27 +1787,29 @@
     }
 
     if (fd == -1) {
-        error_set(errp, QERR_INVALID_PARAMETER, "protocol");
+        error_setg(errp, QERR_INVALID_PARAMETER, "protocol");
         return;
     }
 
-    s = g_malloc0(sizeof(DumpState));
+    s = &dump_state_global;
+    dump_state_prepare(s);
 
     dump_init(s, fd, has_format, format, paging, has_begin,
               begin, length, &local_err);
     if (local_err) {
-        g_free(s);
         error_propagate(errp, local_err);
+        atomic_set(&s->status, DUMP_STATUS_FAILED);
         return;
     }
 
-    if (has_format && format != DUMP_GUEST_MEMORY_FORMAT_ELF) {
-        create_kdump_vmcore(s, errp);
+    if (detach_p) {
+        /* detached dump */
+        qemu_thread_create(&s->dump_thread, "dump_thread", dump_thread,
+                           s, QEMU_THREAD_DETACHED);
     } else {
-        create_vmcore(s, errp);
+        /* sync dump */
+        dump_process(s, errp);
     }
-
-    g_free(s);
 }
 
 DumpGuestMemoryCapability *qmp_query_dump_guest_memory_capability(Error **errp)
diff --git a/exec.c b/exec.c
index 9c2856c..62b4d46 100644
--- a/exec.c
+++ b/exec.c
@@ -16,54 +16,63 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #ifndef _WIN32
-#include <sys/types.h>
-#include <sys/mman.h>
 #endif
 
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
-#include "hw/hw.h"
-#include "hw/qdev.h"
-#include "qemu/osdep.h"
-#include "sysemu/kvm.h"
-#include "sysemu/hax.h"
-#include "sysemu/sysemu.h"
+#include "hw/qdev-core.h"
+#if !defined(CONFIG_USER_ONLY)
+#include "hw/boards.h"
 #include "hw/xen/xen.h"
+#endif
+#include "sysemu/kvm.h"
+#ifdef CONFIG_HAX
+#include "sysemu/hax.h"
+#endif /* CONFIG_HAX */
+#include "sysemu/sysemu.h"
 #include "qemu/timer.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#if defined(CONFIG_USER_ONLY)
+#include "qemu.h"
+#else /* !CONFIG_USER_ONLY */
+#include "hw/hw.h"
 #include "exec/memory.h"
+#include "exec/ioport.h"
 #include "sysemu/dma.h"
 #include "exec/address-spaces.h"
-#if defined(CONFIG_USER_ONLY)
-#include <qemu.h>
-#else /* !CONFIG_USER_ONLY */
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
 #endif
 #include "exec/cpu-all.h"
-
-#include "exec/cputlb.h"
+#include "qemu/rcu_queue.h"
+#include "qemu/main-loop.h"
 #include "translate-all.h"
+#include "sysemu/replay.h"
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
+#include "exec/log.h"
+
+#include "migration/vmstate.h"
 
 #include "qemu/range.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/error-messages.h"
+#ifndef _WIN32
+#include "qemu/mmap-alloc.h"
 #endif
 
 //#define DEBUG_SUBPAGE
 
 #if !defined(CONFIG_USER_ONLY)
-static bool in_migration;
-
-RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
+/* ram_list is read under rcu_read_lock()/rcu_read_unlock().  Writes
+ * are protected by the ramlist lock.
+ */
+RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
 
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;
@@ -80,12 +89,17 @@
 /* RAM is mmap-ed with MAP_SHARED */
 #define RAM_SHARED     (1 << 1)
 
+/* Only a portion of RAM (used_length) is actually used, and migrated.
+ * This used_length size can change across reboots.
+ */
+#define RAM_RESIZEABLE (1 << 2)
+
 #endif
 
 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
 /* current CPU in the current thread. It is only valid inside
    cpu_exec() */
-DEFINE_TLS(CPUState *, current_cpu);
+__thread CPUState *current_cpu;
 /* 0 = Do not count executed instructions.
    1 = Precise instruction counting.
    2 = Adaptive rate instruction counting.  */
@@ -115,6 +129,8 @@
 typedef PhysPageEntry Node[P_L2_SIZE];
 
 typedef struct PhysPageMap {
+    struct rcu_head rcu;
+
     unsigned sections_nb;
     unsigned sections_nb_alloc;
     unsigned nodes_nb;
@@ -124,6 +140,9 @@
 } PhysPageMap;
 
 struct AddressSpaceDispatch {
+    struct rcu_head rcu;
+
+    MemoryRegionSection *mru_section;
     /* This is a multi-level map on the physical address space.
      * The bottom level has pointers to MemoryRegionSections.
      */
@@ -150,30 +169,52 @@
 static void tcg_commit(MemoryListener *listener);
 
 static MemoryRegion io_mem_watch;
+
+/**
+ * CPUAddressSpace: all the information a CPU needs about an AddressSpace
+ * @cpu: the CPU whose AddressSpace this is
+ * @as: the AddressSpace itself
+ * @memory_dispatch: its dispatch pointer (cached, RCU protected)
+ * @tcg_as_listener: listener for tracking changes to the AddressSpace
+ */
+struct CPUAddressSpace {
+    CPUState *cpu;
+    AddressSpace *as;
+    struct AddressSpaceDispatch *memory_dispatch;
+    MemoryListener tcg_as_listener;
+};
+
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
 
 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
 {
+    static unsigned alloc_hint = 16;
     if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
-        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
+        map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, alloc_hint);
         map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
         map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
+        alloc_hint = map->nodes_nb_alloc;
     }
 }
 
-static uint32_t phys_map_node_alloc(PhysPageMap *map)
+static uint32_t phys_map_node_alloc(PhysPageMap *map, bool leaf)
 {
     unsigned i;
     uint32_t ret;
+    PhysPageEntry e;
+    PhysPageEntry *p;
 
     ret = map->nodes_nb++;
+    p = map->nodes[ret];
     assert(ret != PHYS_MAP_NODE_NIL);
     assert(ret != map->nodes_nb_alloc);
+
+    e.skip = leaf ? 0 : 1;
+    e.ptr = leaf ? PHYS_SECTION_UNASSIGNED : PHYS_MAP_NODE_NIL;
     for (i = 0; i < P_L2_SIZE; ++i) {
-        map->nodes[ret][i].skip = 1;
-        map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
+        memcpy(&p[i], &e, sizeof(e));
     }
     return ret;
 }
@@ -183,21 +224,12 @@
                                 int level)
 {
     PhysPageEntry *p;
-    int i;
     hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
 
     if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
-        lp->ptr = phys_map_node_alloc(map);
-        p = map->nodes[lp->ptr];
-        if (level == 0) {
-            for (i = 0; i < P_L2_SIZE; i++) {
-                p[i].skip = 0;
-                p[i].ptr = PHYS_SECTION_UNASSIGNED;
-            }
-        }
-    } else {
-        p = map->nodes[lp->ptr];
+        lp->ptr = phys_map_node_alloc(map, level == 0);
     }
+    p = map->nodes[lp->ptr];
     lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
 
     while (*nb && lp < &p[P_L2_SIZE]) {
@@ -285,6 +317,17 @@
     }
 }
 
+static inline bool section_covers_addr(const MemoryRegionSection *section,
+                                       hwaddr addr)
+{
+    /* Memory topology clips a memory region to [0, 2^64); size.hi > 0 means
+     * the section must cover the entire address space.
+     */
+    return section->size.hi ||
+           range_covers_byte(section->offset_within_address_space,
+                             section->size.lo, addr);
+}
+
 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
                                            Node *nodes, MemoryRegionSection *sections)
 {
@@ -300,9 +343,7 @@
         lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
     }
 
-    if (sections[lp.ptr].size.hi ||
-        range_covers_byte(sections[lp.ptr].offset_within_address_space,
-                          sections[lp.ptr].size.lo, addr)) {
+    if (section_covers_addr(&sections[lp.ptr], addr)) {
         return &sections[lp.ptr];
     } else {
         return &sections[PHYS_SECTION_UNASSIGNED];
@@ -315,26 +356,40 @@
         && mr != &io_mem_watch;
 }
 
+/* Called from RCU critical section */
 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
                                                         hwaddr addr,
                                                         bool resolve_subpage)
 {
-    MemoryRegionSection *section;
+    MemoryRegionSection *section = atomic_read(&d->mru_section);
     subpage_t *subpage;
+    bool update;
 
-    section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
+    if (section && section != &d->map.sections[PHYS_SECTION_UNASSIGNED] &&
+        section_covers_addr(section, addr)) {
+        update = false;
+    } else {
+        section = phys_page_find(d->phys_map, addr, d->map.nodes,
+                                 d->map.sections);
+        update = true;
+    }
     if (resolve_subpage && section->mr->subpage) {
         subpage = container_of(section->mr, subpage_t, iomem);
         section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
     }
+    if (update) {
+        atomic_set(&d->mru_section, section);
+    }
     return section;
 }
 
+/* Called from RCU critical section */
 static MemoryRegionSection *
 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
                                  hwaddr *plen, bool resolve_subpage)
 {
     MemoryRegionSection *section;
+    MemoryRegion *mr;
     Int128 diff;
 
     section = address_space_lookup_region(d, addr, resolve_subpage);
@@ -344,23 +399,27 @@
     /* Compute offset within MemoryRegion */
     *xlat = addr + section->offset_within_region;
 
-    diff = int128_sub(section->mr->size, int128_make64(addr));
-    *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
+    mr = section->mr;
+
+    /* MMIO registers can be expected to perform full-width accesses based only
+     * on their address, without considering adjacent registers that could
+     * decode to completely different MemoryRegions.  When such registers
+     * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO
+     * regions overlap wildly.  For this reason we cannot clamp the accesses
+     * here.
+     *
+     * If the length is small (as is the case for address_space_ldl/stl),
+     * everything works fine.  If the incoming length is large, however,
+     * the caller really has to do the clamping through memory_access_size.
+     */
+    if (memory_region_is_ram(mr)) {
+        diff = int128_sub(section->size, int128_make64(addr));
+        *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
+    }
     return section;
 }
 
-static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
-{
-    if (memory_region_is_ram(mr)) {
-        return !(is_write && mr->readonly);
-    }
-    if (memory_region_is_romd(mr)) {
-        return !is_write;
-    }
-
-    return false;
-}
-
+/* Called from RCU critical section */
 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
                                       hwaddr *xlat, hwaddr *plen,
                                       bool is_write)
@@ -368,10 +427,10 @@
     IOMMUTLBEntry iotlb;
     MemoryRegionSection *section;
     MemoryRegion *mr;
-    hwaddr len = *plen;
 
     for (;;) {
-        section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
+        AddressSpaceDispatch *d = atomic_rcu_read(&as->dispatch);
+        section = address_space_translate_internal(d, addr, &addr, plen, true);
         mr = section->mr;
 
         if (!mr->iommu_ops) {
@@ -381,7 +440,7 @@
         iotlb = mr->iommu_ops->translate(mr, addr, is_write);
         addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
                 | (addr & iotlb.addr_mask));
-        len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
+        *plen = MIN(*plen, (addr | iotlb.addr_mask) - addr + 1);
         if (!(iotlb.perm & (1 << is_write))) {
             mr = &io_mem_unassigned;
             break;
@@ -392,35 +451,28 @@
 
     if (xen_enabled() && memory_access_is_direct(mr, is_write)) {
         hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
-        len = MIN(page, len);
+        *plen = MIN(page, *plen);
     }
 
-    *plen = len;
     *xlat = addr;
     return mr;
 }
 
+/* Called from RCU critical section */
 MemoryRegionSection *
-address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
-                                  hwaddr *plen)
+address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
+                                  hwaddr *xlat, hwaddr *plen)
 {
     MemoryRegionSection *section;
-    section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
+    AddressSpaceDispatch *d = cpu->cpu_ases[asidx].memory_dispatch;
+
+    section = address_space_translate_internal(d, addr, xlat, plen, false);
 
     assert(!section->mr->iommu_ops);
     return section;
 }
 #endif
 
-void cpu_exec_init_all(void)
-{
-#if !defined(CONFIG_USER_ONLY)
-    qemu_mutex_init(&ram_list.mutex);
-    memory_map_init();
-    io_mem_init();
-#endif
-}
-
 #if !defined(CONFIG_USER_ONLY)
 
 static int cpu_common_post_load(void *opaque, int version_id)
@@ -439,7 +491,7 @@
 {
     CPUState *cpu = opaque;
 
-    cpu->exception_index = 0;
+    cpu->exception_index = -1;
 
     return 0;
 }
@@ -448,19 +500,38 @@
 {
     CPUState *cpu = opaque;
 
-    return cpu->exception_index != 0;
+    return tcg_enabled() && cpu->exception_index != -1;
 }
 
 static const VMStateDescription vmstate_cpu_common_exception_index = {
     .name = "cpu_common/exception_index",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = cpu_common_exception_index_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(exception_index, CPUState),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static bool cpu_common_crash_occurred_needed(void *opaque)
+{
+    CPUState *cpu = opaque;
+
+    return cpu->crash_occurred;
+}
+
+static const VMStateDescription vmstate_cpu_common_crash_occurred = {
+    .name = "cpu_common/crash_occurred",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = cpu_common_crash_occurred_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(crash_occurred, CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_cpu_common = {
     .name = "cpu_common",
     .version_id = 1,
@@ -472,13 +543,10 @@
         VMSTATE_UINT32(interrupt_request, CPUState),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_cpu_common_exception_index,
-            .needed = cpu_common_exception_index_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_cpu_common_exception_index,
+        &vmstate_cpu_common_crash_occurred,
+        NULL
     }
 };
 
@@ -498,62 +566,120 @@
 }
 
 #if !defined(CONFIG_USER_ONLY)
-void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
+void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx)
 {
-    /* We only support one address space per cpu at the moment.  */
-    assert(cpu->as == as);
+    CPUAddressSpace *newas;
 
-    if (cpu->tcg_as_listener) {
-        memory_listener_unregister(cpu->tcg_as_listener);
-    } else {
-        cpu->tcg_as_listener = g_new0(MemoryListener, 1);
+    /* Target code should have set num_ases before calling us */
+    assert(asidx < cpu->num_ases);
+
+    if (asidx == 0) {
+        /* address space 0 gets the convenience alias */
+        cpu->as = as;
     }
-    cpu->tcg_as_listener->commit = tcg_commit;
-    memory_listener_register(cpu->tcg_as_listener, as);
+
+    /* KVM cannot currently support multiple address spaces. */
+    assert(asidx == 0 || !kvm_enabled());
+
+    if (!cpu->cpu_ases) {
+        cpu->cpu_ases = g_new0(CPUAddressSpace, cpu->num_ases);
+    }
+
+    newas = &cpu->cpu_ases[asidx];
+    newas->cpu = cpu;
+    newas->as = as;
+    if (tcg_enabled()) {
+        newas->tcg_as_listener.commit = tcg_commit;
+        memory_listener_register(&newas->tcg_as_listener, as);
+    }
+}
+
+AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
+{
+    /* Return the AddressSpace corresponding to the specified index */
+    return cpu->cpu_ases[asidx].as;
 }
 #endif
 
-void cpu_exec_init(CPUArchState *env)
+static int cpu_get_free_index(void)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
-    CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUState *some_cpu;
-    int cpu_index;
+    int cpu_index = 0;
 
-#if defined(CONFIG_USER_ONLY)
-    cpu_list_lock();
-#endif
-    cpu_index = 0;
     CPU_FOREACH(some_cpu) {
         cpu_index++;
     }
-    cpu->cpu_index = cpu_index;
-    cpu->numa_node = 0;
-    QTAILQ_INIT(&cpu->breakpoints);
-    QTAILQ_INIT(&cpu->watchpoints);
-#ifndef CONFIG_USER_ONLY
-    cpu->as = &address_space_memory;
-    cpu->thread_id = qemu_get_thread_id();
-#endif
-    QTAILQ_INSERT_TAIL(&cpus, cpu, node);
-#if defined(CONFIG_USER_ONLY)
-    cpu_list_unlock();
-#endif
-    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
-        vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
+    return cpu_index;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    cpu_list_lock();
+    if (cpu->node.tqe_prev == NULL) {
+        /* there is nothing to undo since cpu_exec_init() hasn't been called */
+        cpu_list_unlock();
+        return;
     }
-#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
-    register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
-                    cpu_save, cpu_load, env);
-    assert(cc->vmsd == NULL);
-    assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
-#endif
+
+    QTAILQ_REMOVE(&cpus, cpu, node);
+    cpu->node.tqe_prev = NULL;
+    cpu->cpu_index = UNASSIGNED_CPU_INDEX;
+    cpu_list_unlock();
+
     if (cc->vmsd != NULL) {
-        vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
+        vmstate_unregister(NULL, cc->vmsd, cpu);
+    }
+    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
+        vmstate_unregister(NULL, &vmstate_cpu_common, cpu);
     }
 }
 
-#if defined(TARGET_HAS_ICE)
+void cpu_exec_init(CPUState *cpu, Error **errp)
+{
+    CPUClass *cc ATTRIBUTE_UNUSED = CPU_GET_CLASS(cpu);
+    Error *local_err ATTRIBUTE_UNUSED = NULL;
+
+    cpu->as = NULL;
+    cpu->num_ases = 0;
+
+#ifndef CONFIG_USER_ONLY
+    cpu->thread_id = qemu_get_thread_id();
+
+    /* This is a softmmu CPU object, so create a property for it
+     * so users can wire up its memory. (This can't go in qom/cpu.c
+     * because that file is compiled only once for both user-mode
+     * and system builds.) The default if no link is set up is to use
+     * the system address space.
+     */
+    object_property_add_link(OBJECT(cpu), "memory", TYPE_MEMORY_REGION,
+                             (Object **)&cpu->memory,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+    cpu->memory = system_memory;
+    object_ref(OBJECT(cpu->memory));
+#endif
+
+    cpu_list_lock();
+    if (cpu->cpu_index == UNASSIGNED_CPU_INDEX) {
+        cpu->cpu_index = cpu_get_free_index();
+        assert(cpu->cpu_index != UNASSIGNED_CPU_INDEX);
+    }
+    QTAILQ_INSERT_TAIL(&cpus, cpu, node);
+    cpu_list_unlock();
+
+#ifndef CONFIG_USER_ONLY
+    if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
+        vmstate_register(NULL, cpu->cpu_index, &vmstate_cpu_common, cpu);
+    }
+    if (cc->vmsd != NULL) {
+        vmstate_register(NULL, cpu->cpu_index, cc->vmsd, cpu);
+    }
+#endif
+}
+
 #if defined(CONFIG_USER_ONLY)
 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 {
@@ -562,14 +688,15 @@
 #else
 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
 {
-    hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
+    MemTxAttrs attrs;
+    hwaddr phys = cpu_get_phys_page_attrs_debug(cpu, pc, &attrs);
+    int asidx = cpu_asidx_from_attrs(cpu, attrs);
     if (phys != -1) {
-        tb_invalidate_phys_addr(cpu->as,
+        tb_invalidate_phys_addr(cpu->cpu_ases[asidx].as,
                                 phys | (pc & ~TARGET_PAGE_MASK));
     }
 }
 #endif
-#endif /* TARGET_HAS_ICE */
 
 #if defined(CONFIG_USER_ONLY)
 void cpu_watchpoint_remove_all(CPUState *cpu, int mask)
@@ -689,7 +816,6 @@
 int cpu_breakpoint_insert(CPUState *cpu, vaddr pc, int flags,
                           CPUBreakpoint **breakpoint)
 {
-#if defined(TARGET_HAS_ICE)
     CPUBreakpoint *bp;
 
     bp = g_malloc(sizeof(*bp));
@@ -710,15 +836,11 @@
         *breakpoint = bp;
     }
     return 0;
-#else
-    return -ENOSYS;
-#endif
 }
 
 /* Remove a specific breakpoint.  */
 int cpu_breakpoint_remove(CPUState *cpu, vaddr pc, int flags)
 {
-#if defined(TARGET_HAS_ICE)
     CPUBreakpoint *bp;
 
     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
@@ -728,27 +850,21 @@
         }
     }
     return -ENOENT;
-#else
-    return -ENOSYS;
-#endif
 }
 
 /* Remove a specific breakpoint by reference.  */
 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint)
 {
-#if defined(TARGET_HAS_ICE)
     QTAILQ_REMOVE(&cpu->breakpoints, breakpoint, entry);
 
     breakpoint_invalidate(cpu, breakpoint->pc);
 
     g_free(breakpoint);
-#endif
 }
 
 /* Remove all matching breakpoints. */
 void cpu_breakpoint_remove_all(CPUState *cpu, int mask)
 {
-#if defined(TARGET_HAS_ICE)
     CPUBreakpoint *bp, *next;
 
     QTAILQ_FOREACH_SAFE(bp, &cpu->breakpoints, entry, next) {
@@ -756,14 +872,12 @@
             cpu_breakpoint_remove_by_ref(cpu, bp);
         }
     }
-#endif
 }
 
 /* enable or disable single step mode. EXCP_DEBUG is returned by the
    CPU loop after each instruction */
 void cpu_single_step(CPUState *cpu, int enabled)
 {
-#if defined(TARGET_HAS_ICE)
     if (cpu->singlestep_enabled != enabled) {
         cpu->singlestep_enabled = enabled;
         if (kvm_enabled()) {
@@ -771,11 +885,9 @@
         } else {
             /* must flush all the translated code to avoid inconsistencies */
             /* XXX: only flush what is necessary */
-            CPUArchState *env = cpu->env_ptr;
-            tb_flush(env);
+            tb_flush(cpu);
         }
     }
-#endif
 }
 
 void cpu_abort(CPUState *cpu, const char *fmt, ...)
@@ -789,7 +901,7 @@
     vfprintf(stderr, fmt, ap);
     fprintf(stderr, "\n");
     cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
-    if (qemu_log_enabled()) {
+    if (qemu_log_separate()) {
         qemu_log("qemu: fatal: ");
         qemu_log_vprintf(fmt, ap2);
         qemu_log("\n");
@@ -799,6 +911,7 @@
     }
     va_end(ap2);
     va_end(ap);
+    replay_finish();
 #if defined(CONFIG_USER_ONLY)
     {
         struct sigaction act;
@@ -811,17 +924,17 @@
 }
 
 #if !defined(CONFIG_USER_ONLY)
+/* Called from RCU critical section */
 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
 {
     RAMBlock *block;
 
-    /* The list is protected by the iothread lock here.  */
-    block = ram_list.mru_block;
-    if (block && addr - block->offset < block->length) {
-        goto found;
+    block = atomic_rcu_read(&ram_list.mru_block);
+    if (block && addr - block->offset < block->max_length) {
+        return block;
     }
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (addr - block->offset < block->length) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (addr - block->offset < block->max_length) {
             goto found;
         }
     }
@@ -830,12 +943,29 @@
     abort();
 
 found:
+    /* It is safe to write mru_block outside the iothread lock.  This
+     * is what happens:
+     *
+     *     mru_block = xxx
+     *     rcu_read_unlock()
+     *                                        xxx removed from list
+     *                  rcu_read_lock()
+     *                  read mru_block
+     *                                        mru_block = NULL;
+     *                                        call_rcu(reclaim_ramblock, xxx);
+     *                  rcu_read_unlock()
+     *
+     * atomic_rcu_set is not needed here.  The block was already published
+     * when it was placed into the list.  Here we're just making an extra
+     * copy of the pointer.
+     */
     ram_list.mru_block = block;
     return block;
 }
 
 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
 {
+    CPUState *cpu;
     ram_addr_t start1;
     RAMBlock *block;
     ram_addr_t end;
@@ -843,31 +973,56 @@
     end = TARGET_PAGE_ALIGN(start + length);
     start &= TARGET_PAGE_MASK;
 
+    rcu_read_lock();
     block = qemu_get_ram_block(start);
     assert(block == qemu_get_ram_block(end - 1));
-    (void)end;
-    start1 = (uintptr_t)block->host + (start - block->offset);
-    cpu_tlb_reset_dirty_all(start1, length);
+    start1 = (uintptr_t)ramblock_ptr(block, start - block->offset);
+    CPU_FOREACH(cpu) {
+        tlb_reset_dirty(cpu, start1, length);
+    }
+    rcu_read_unlock();
 }
 
 /* Note: start and end must be within the same ram block.  */
-void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
-                                     unsigned client)
+bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
+                                              ram_addr_t length,
+                                              unsigned client)
 {
-    if (length == 0)
-        return;
-    cpu_physical_memory_clear_dirty_range(start, length, client);
+    DirtyMemoryBlocks *blocks;
+    unsigned long end, page;
+    bool dirty = false;
 
-    if (tcg_enabled()) {
+    if (length == 0) {
+        return false;
+    }
+
+    end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
+    page = start >> TARGET_PAGE_BITS;
+
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    while (page < end) {
+        unsigned long idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+        unsigned long offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+        unsigned long num = MIN(end - page, DIRTY_MEMORY_BLOCK_SIZE - offset);
+
+        dirty |= bitmap_test_and_clear_atomic(blocks->blocks[idx],
+                                              offset, num);
+        page += num;
+    }
+
+    rcu_read_unlock();
+
+    if (dirty && tcg_enabled()) {
         tlb_reset_dirty_range_all(start, length);
     }
+
+    return dirty;
 }
 
-static void cpu_physical_memory_set_dirty_tracking(bool enable)
-{
-    in_migration = enable;
-}
-
+/* Called from RCU critical section */
 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
                                        MemoryRegionSection *section,
                                        target_ulong vaddr,
@@ -880,15 +1035,17 @@
 
     if (memory_region_is_ram(section->mr)) {
         /* Normal RAM.  */
-        iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
-            + xlat;
+        iotlb = memory_region_get_ram_addr(section->mr) + xlat;
         if (!section->readonly) {
             iotlb |= PHYS_SECTION_NOTDIRTY;
         } else {
             iotlb |= PHYS_SECTION_ROM;
         }
     } else {
-        iotlb = section - section->address_space->dispatch->map.sections;
+        AddressSpaceDispatch *d;
+
+        d = atomic_rcu_read(&section->address_space->dispatch);
+        iotlb = section - d->map.sections;
         iotlb += xlat;
     }
 
@@ -949,9 +1106,11 @@
 
 static void phys_section_destroy(MemoryRegion *mr)
 {
+    bool have_sub_page = mr->subpage;
+
     memory_region_unref(mr);
 
-    if (mr->subpage) {
+    if (have_sub_page) {
         subpage_t *subpage = container_of(mr, subpage_t, iomem);
         object_unref(OBJECT(&subpage->iomem));
         g_free(subpage);
@@ -1061,87 +1220,83 @@
 }
 
 #ifdef __linux__
-
-#include <sys/vfs.h>
-
-#define HUGETLBFS_MAGIC       0x958458f6
-
-static long gethugepagesize(const char *path, Error **errp)
-{
-    struct statfs fs;
-    int ret;
-
-    do {
-        ret = statfs(path, &fs);
-    } while (ret != 0 && errno == EINTR);
-
-    if (ret != 0) {
-        error_setg_errno(errp, errno, "failed to get page size of file %s",
-                         path);
-        return 0;
-    }
-
-    if (fs.f_type != HUGETLBFS_MAGIC)
-        fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
-
-    return fs.f_bsize;
-}
-
 static void *file_ram_alloc(RAMBlock *block,
                             ram_addr_t memory,
                             const char *path,
                             Error **errp)
 {
+    bool unlink_on_error = false;
     char *filename;
     char *sanitized_name;
     char *c;
-    void *area = NULL;
-    int fd;
-    uint64_t hpagesize;
-    Error *local_err = NULL;
-
-    hpagesize = gethugepagesize(path, &local_err);
-    if (local_err) {
-        error_propagate(errp, local_err);
-        goto error;
-    }
-    block->mr->align = hpagesize;
-
-    if (memory < hpagesize) {
-        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
-                   "or larger than huge page size 0x%" PRIx64,
-                   memory, hpagesize);
-        goto error;
-    }
+    void *area = MAP_FAILED;
+    int fd = -1;
+    int64_t page_size;
 
     if (kvm_enabled() && !kvm_has_sync_mmu()) {
         error_setg(errp,
                    "host lacks kvm mmu notifiers, -mem-path unsupported");
+        return NULL;
+    }
+
+    for (;;) {
+        fd = open(path, O_RDWR);
+        if (fd >= 0) {
+            /* @path names an existing file, use it */
+            break;
+        }
+        if (errno == ENOENT) {
+            /* @path names a file that doesn't exist, create it */
+            fd = open(path, O_RDWR | O_CREAT | O_EXCL, 0644);
+            if (fd >= 0) {
+                unlink_on_error = true;
+                break;
+            }
+        } else if (errno == EISDIR) {
+            /* @path names a directory, create a file there */
+            /* Make name safe to use with mkstemp by replacing '/' with '_'. */
+            sanitized_name = g_strdup(memory_region_name(block->mr));
+            for (c = sanitized_name; *c != '\0'; c++) {
+                if (*c == '/') {
+                    *c = '_';
+                }
+            }
+
+            filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
+                                       sanitized_name);
+            g_free(sanitized_name);
+
+            fd = mkstemp(filename);
+            if (fd >= 0) {
+                unlink(filename);
+                g_free(filename);
+                break;
+            }
+            g_free(filename);
+        }
+        if (errno != EEXIST && errno != EINTR) {
+            error_setg_errno(errp, errno,
+                             "can't open backing store %s for guest RAM",
+                             path);
+            goto error;
+        }
+        /*
+         * Try again on EINTR and EEXIST.  The latter happens when
+         * something else creates the file between our two open().
+         */
+    }
+
+    page_size = qemu_fd_getpagesize(fd);
+    block->mr->align = MAX(page_size, QEMU_VMALLOC_ALIGN);
+
+    if (memory < page_size) {
+        error_setg(errp, "memory size 0x" RAM_ADDR_FMT " must be equal to "
+                   "or larger than page size 0x%" PRIx64,
+                   memory, page_size);
         goto error;
     }
 
-    /* Make name safe to use with mkstemp by replacing '/' with '_'. */
-    sanitized_name = g_strdup(memory_region_name(block->mr));
-    for (c = sanitized_name; *c != '\0'; c++) {
-        if (*c == '/')
-            *c = '_';
-    }
-
-    filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
-                               sanitized_name);
-    g_free(sanitized_name);
-
-    fd = mkstemp(filename);
-    if (fd < 0) {
-        error_setg_errno(errp, errno,
-                         "unable to create backing store for hugepages");
-        g_free(filename);
-        goto error;
-    }
-    unlink(filename);
-    g_free(filename);
-
-    memory = (memory+hpagesize-1) & ~(hpagesize-1);
+    memory = ROUND_UP(memory, page_size);
 
     /*
      * ftruncate is not supported by hugetlbfs in older
@@ -1153,32 +1308,39 @@
         perror("ftruncate");
     }
 
-    area = mmap(0, memory, PROT_READ | PROT_WRITE,
-                (block->flags & RAM_SHARED ? MAP_SHARED : MAP_PRIVATE),
-                fd, 0);
+    area = qemu_ram_mmap(fd, memory, block->mr->align,
+                         block->flags & RAM_SHARED);
     if (area == MAP_FAILED) {
         error_setg_errno(errp, errno,
-                         "unable to map backing store for hugepages");
-        close(fd);
+                         "unable to map backing store for guest RAM");
         goto error;
     }
 
     if (mem_prealloc) {
-        os_mem_prealloc(fd, area, memory);
+        os_mem_prealloc(fd, area, memory, errp);
+        if (errp && *errp) {
+            goto error;
+        }
     }
 
     block->fd = fd;
     return area;
 
 error:
-    if (mem_prealloc) {
-        error_report("%s\n", error_get_pretty(*errp));
-        exit(1);
+    if (area != MAP_FAILED) {
+        qemu_ram_munmap(area, memory);
+    }
+    if (unlink_on_error) {
+        unlink(path);
+    }
+    if (fd != -1) {
+        close(fd);
     }
     return NULL;
 }
 #endif
 
+/* Called with the ramlist lock held.  */
 static ram_addr_t find_ram_offset(ram_addr_t size)
 {
     RAMBlock *block, *next_block;
@@ -1186,15 +1348,16 @@
 
     assert(size != 0); /* it would hand out same offset multiple times */
 
-    if (QTAILQ_EMPTY(&ram_list.blocks))
+    if (QLIST_EMPTY_RCU(&ram_list.blocks)) {
         return 0;
+    }
 
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
         ram_addr_t end, next = RAM_ADDR_MAX;
 
-        end = block->offset + block->length;
+        end = block->offset + block->max_length;
 
-        QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
+        QLIST_FOREACH_RCU(next_block, &ram_list.blocks, next) {
             if (next_block->offset >= end) {
                 next = MIN(next, next_block->offset);
             }
@@ -1219,9 +1382,11 @@
     RAMBlock *block;
     ram_addr_t last = 0;
 
-    QTAILQ_FOREACH(block, &ram_list.blocks, next)
-        last = MAX(last, block->offset + block->length);
-
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        last = MAX(last, block->offset + block->max_length);
+    }
+    rcu_read_unlock();
     return last;
 }
 
@@ -1230,8 +1395,7 @@
     int ret;
 
     /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
-    if (!qemu_opt_get_bool(qemu_get_machine_opts(),
-                           "dump-guest-core", true)) {
+    if (!machine_dump_guest_core(current_machine)) {
         ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
         if (ret) {
             perror("qemu_madvise");
@@ -1241,22 +1405,14 @@
     }
 }
 
-static RAMBlock *find_ram_block(ram_addr_t addr)
+const char *qemu_ram_get_idstr(RAMBlock *rb)
 {
-    RAMBlock *block;
-
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (block->offset == addr) {
-            return block;
-        }
-    }
-
-    return NULL;
+    return rb->idstr;
 }
 
-void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
+/* Called with iothread lock held.  */
+void qemu_ram_set_idstr(RAMBlock *new_block, const char *name, DeviceState *dev)
 {
-    RAMBlock *new_block = find_ram_block(addr);
     RAMBlock *block;
 
     assert(new_block);
@@ -1271,22 +1427,25 @@
     }
     pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
 
-    /* This assumes the iothread lock is taken here too.  */
-    qemu_mutex_lock_ramlist();
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (block != new_block &&
+            !strcmp(block->idstr, new_block->idstr)) {
             fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
                     new_block->idstr);
             abort();
         }
     }
-    qemu_mutex_unlock_ramlist();
+    rcu_read_unlock();
 }
 
-void qemu_ram_unset_idstr(ram_addr_t addr)
+/* Called with iothread lock held.  */
+void qemu_ram_unset_idstr(RAMBlock *block)
 {
-    RAMBlock *block = find_ram_block(addr);
-
+    /* FIXME: arch_init.c assumes that this is not called throughout
+     * migration.  Ignore the problem since hot-unplug during migration
+     * does not work anyway.
+     */
     if (block) {
         memset(block->idstr, 0, sizeof(block->idstr));
     }
@@ -1294,7 +1453,7 @@
 
 static int memory_try_enable_merging(void *addr, size_t len)
 {
-    if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
+    if (!machine_mem_merge(current_machine)) {
         /* disabled by the user */
         return 0;
     }
@@ -1302,23 +1461,122 @@
     return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
 }
 
-static ram_addr_t ram_block_add(RAMBlock *new_block, Error **errp)
+/* Only legal before guest might have detected the memory size: e.g. on
+ * incoming migration, or right after reset.
+ *
+ * As memory core doesn't know how is memory accessed, it is up to
+ * resize callback to update device state and/or add assertions to detect
+ * misuse, if necessary.
+ */
+int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp)
+{
+    assert(block);
+
+    newsize = HOST_PAGE_ALIGN(newsize);
+
+    if (block->used_length == newsize) {
+        return 0;
+    }
+
+    if (!(block->flags & RAM_RESIZEABLE)) {
+        error_setg_errno(errp, EINVAL,
+                         "Length mismatch: %s: 0x" RAM_ADDR_FMT
+                         " in != 0x" RAM_ADDR_FMT, block->idstr,
+                         newsize, block->used_length);
+        return -EINVAL;
+    }
+
+    if (block->max_length < newsize) {
+        error_setg_errno(errp, EINVAL,
+                         "Length too large: %s: 0x" RAM_ADDR_FMT
+                         " > 0x" RAM_ADDR_FMT, block->idstr,
+                         newsize, block->max_length);
+        return -EINVAL;
+    }
+
+    cpu_physical_memory_clear_dirty_range(block->offset, block->used_length);
+    block->used_length = newsize;
+    cpu_physical_memory_set_dirty_range(block->offset, block->used_length,
+                                        DIRTY_CLIENTS_ALL);
+    memory_region_set_size(block->mr, newsize);
+    if (block->resized) {
+        block->resized(block->idstr, newsize, block->host);
+    }
+    return 0;
+}
+
+/* Called with ram_list.mutex held */
+static void dirty_memory_extend(ram_addr_t old_ram_size,
+                                ram_addr_t new_ram_size)
+{
+    ram_addr_t old_num_blocks = DIV_ROUND_UP(old_ram_size,
+                                             DIRTY_MEMORY_BLOCK_SIZE);
+    ram_addr_t new_num_blocks = DIV_ROUND_UP(new_ram_size,
+                                             DIRTY_MEMORY_BLOCK_SIZE);
+    int i;
+
+    /* Only need to extend if block count increased */
+    if (new_num_blocks <= old_num_blocks) {
+        return;
+    }
+
+    for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+        DirtyMemoryBlocks *old_blocks;
+        DirtyMemoryBlocks *new_blocks;
+        int j;
+
+        old_blocks = atomic_rcu_read(&ram_list.dirty_memory[i]);
+        new_blocks = g_malloc(sizeof(*new_blocks) +
+                              sizeof(new_blocks->blocks[0]) * new_num_blocks);
+
+        if (old_num_blocks) {
+            memcpy(new_blocks->blocks, old_blocks->blocks,
+                   old_num_blocks * sizeof(old_blocks->blocks[0]));
+        }
+
+        for (j = old_num_blocks; j < new_num_blocks; j++) {
+            new_blocks->blocks[j] = bitmap_new(DIRTY_MEMORY_BLOCK_SIZE);
+        }
+
+        atomic_rcu_set(&ram_list.dirty_memory[i], new_blocks);
+
+        if (old_blocks) {
+            g_free_rcu(old_blocks, rcu);
+        }
+    }
+}
+
+static void ram_block_add(RAMBlock *new_block, Error **errp)
 {
     RAMBlock *block;
+    RAMBlock *last_block = NULL;
     ram_addr_t old_ram_size, new_ram_size;
+    Error *err = NULL;
 
     old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
 
-    /* This assumes the iothread lock is taken here too.  */
     qemu_mutex_lock_ramlist();
-    new_block->offset = find_ram_offset(new_block->length);
+    new_block->offset = find_ram_offset(new_block->max_length);
 
     if (!new_block->host) {
         if (xen_enabled()) {
-            xen_ram_alloc(new_block->offset, new_block->length, new_block->mr);
+            xen_ram_alloc(new_block->offset, new_block->max_length,
+                          new_block->mr, &err);
+            if (err) {
+                error_propagate(errp, err);
+                qemu_mutex_unlock_ramlist();
+                return;
+            }
         } else {
-            new_block->host = phys_mem_alloc(new_block->length,
+            new_block->host = phys_mem_alloc(new_block->max_length,
                                              &new_block->mr->align);
+            if (!new_block->host) {
+                error_setg_errno(errp, errno,
+                                 "cannot set up guest memory '%s'",
+                                 memory_region_name(new_block->mr));
+                qemu_mutex_unlock_ramlist();
+                return;
+            }
 #ifdef CONFIG_HAX
             /*
              * In Hax, the qemu allocate the virtual address, and HAX kernel
@@ -1328,90 +1586,75 @@
             if (hax_enabled()) {
                 int ret;
                 ret = hax_populate_ram((uint64_t)(uintptr_t)new_block->host,
-                                       new_block->length);
+                                       new_block->max_length);
                 if (ret < 0) {
-                    fprintf(stderr, "Hax failed to populate ram\n");
-#ifdef CONFIG_ANDROID
-                    if (phys_mem_alloc == qemu_anon_ram_alloc) {
-                        // We can only free the allocated memory if we know how
-                        // it was allocated. Else, we'll leak it :(
-                        qemu_anon_ram_free(new_block->host, new_block->length);
-                    }
-                    new_block->host = NULL;
-#else  // !CONFIG_ANDROID
-                    exit(-1);
-#endif  // !CONFIG_ANDROID
+                    error_setg_errno(errp, errno,
+                                     "Hax failed to populate RAM for: '%s'",
+                                     memory_region_name(new_block->mr));
+                    qemu_mutex_unlock_ramlist();
+                    return;
                 }
             }
 #endif
-            if (!new_block->host) {
-#ifdef CONFIG_ANDROID
-                error_setg_errno(errp, errno, kNotEnoughMemForGuestError,
-                                 new_block->length / (1024.0 * 1024.0 * 1024.0),
-                                 memory_region_name(new_block->mr));
-#else  // !CONFIG_ANDROID
-                error_setg_errno(errp, errno,
-                                 "cannot set up guest memory '%s'",
-                                 memory_region_name(new_block->mr));
-#endif  // !CONFIG_ANDROID
-                qemu_mutex_unlock_ramlist();
-                return -1;
-            }
-            memory_try_enable_merging(new_block->host, new_block->length);
+            memory_try_enable_merging(new_block->host, new_block->max_length);
         }
     }
 
-    /* Keep the list sorted from biggest to smallest block.  */
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (block->length < new_block->length) {
+    new_ram_size = MAX(old_ram_size,
+              (new_block->offset + new_block->max_length) >> TARGET_PAGE_BITS);
+    if (new_ram_size > old_ram_size) {
+        migration_bitmap_extend(old_ram_size, new_ram_size);
+        dirty_memory_extend(old_ram_size, new_ram_size);
+    }
+    /* Keep the list sorted from biggest to smallest block.  Unlike QTAILQ,
+     * QLIST (which has an RCU-friendly variant) does not have insertion at
+     * tail, so save the last element in last_block.
+     */
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        last_block = block;
+        if (block->max_length < new_block->max_length) {
             break;
         }
     }
     if (block) {
-        QTAILQ_INSERT_BEFORE(block, new_block, next);
-    } else {
-        QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
+        QLIST_INSERT_BEFORE_RCU(block, new_block, next);
+    } else if (last_block) {
+        QLIST_INSERT_AFTER_RCU(last_block, new_block, next);
+    } else { /* list is empty */
+        QLIST_INSERT_HEAD_RCU(&ram_list.blocks, new_block, next);
     }
     ram_list.mru_block = NULL;
 
+    /* Write list before version */
+    smp_wmb();
     ram_list.version++;
     qemu_mutex_unlock_ramlist();
 
-    new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
+    cpu_physical_memory_set_dirty_range(new_block->offset,
+                                        new_block->used_length,
+                                        DIRTY_CLIENTS_ALL);
 
-    if (new_ram_size > old_ram_size) {
-        int i;
-        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
-            ram_list.dirty_memory[i] =
-                bitmap_zero_extend(ram_list.dirty_memory[i],
-                                   old_ram_size, new_ram_size);
-       }
+    if (new_block->host) {
+        qemu_ram_setup_dump(new_block->host, new_block->max_length);
+        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_HUGEPAGE);
+        qemu_madvise(new_block->host, new_block->max_length, QEMU_MADV_DONTFORK);
+        if (kvm_enabled()) {
+            kvm_setup_guest_memory(new_block->host, new_block->max_length);
+        }
     }
-    cpu_physical_memory_set_dirty_range(new_block->offset, new_block->length);
-
-    qemu_ram_setup_dump(new_block->host, new_block->length);
-    qemu_madvise(new_block->host, new_block->length, QEMU_MADV_HUGEPAGE);
-    qemu_madvise(new_block->host, new_block->length, QEMU_MADV_DONTFORK);
-
-    if (kvm_enabled()) {
-        kvm_setup_guest_memory(new_block->host, new_block->length);
-    }
-
-    return new_block->offset;
 }
 
 #ifdef __linux__
-ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
-                                    bool share, const char *mem_path,
-                                    Error **errp)
+RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
+                                   bool share, const char *mem_path,
+                                   Error **errp)
 {
     RAMBlock *new_block;
-    ram_addr_t addr;
     Error *local_err = NULL;
 
     if (xen_enabled()) {
         error_setg(errp, "-mem-path not supported with Xen");
-        return -1;
+        return NULL;
     }
 
     if (phys_mem_alloc != qemu_anon_ram_alloc) {
@@ -1422,108 +1665,119 @@
          */
         error_setg(errp,
                    "-mem-path not supported with this accelerator");
-        return -1;
+        return NULL;
     }
 
-    size = TARGET_PAGE_ALIGN(size);
+    size = HOST_PAGE_ALIGN(size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
-    new_block->length = size;
+    new_block->used_length = size;
+    new_block->max_length = size;
     new_block->flags = share ? RAM_SHARED : 0;
     new_block->host = file_ram_alloc(new_block, size,
                                      mem_path, errp);
     if (!new_block->host) {
         g_free(new_block);
-        return -1;
+        return NULL;
     }
 
-    addr = ram_block_add(new_block, &local_err);
+    ram_block_add(new_block, &local_err);
     if (local_err) {
         g_free(new_block);
         error_propagate(errp, local_err);
-        return -1;
+        return NULL;
     }
-    return addr;
+    return new_block;
 }
 #endif
 
-ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
-                                   MemoryRegion *mr, Error **errp)
+static
+RAMBlock *qemu_ram_alloc_internal(ram_addr_t size, ram_addr_t max_size,
+                                  void (*resized)(const char*,
+                                                  uint64_t length,
+                                                  void *host),
+                                  void *host, bool resizeable,
+                                  MemoryRegion *mr, Error **errp)
 {
     RAMBlock *new_block;
-    ram_addr_t addr;
     Error *local_err = NULL;
 
-    size = TARGET_PAGE_ALIGN(size);
+    size = HOST_PAGE_ALIGN(size);
+    max_size = HOST_PAGE_ALIGN(max_size);
     new_block = g_malloc0(sizeof(*new_block));
     new_block->mr = mr;
-    new_block->length = size;
+    new_block->resized = resized;
+    new_block->used_length = size;
+    new_block->max_length = max_size;
+    assert(max_size >= size);
     new_block->fd = -1;
     new_block->host = host;
     if (host) {
         new_block->flags |= RAM_PREALLOC;
     }
-    addr = ram_block_add(new_block, &local_err);
+    if (resizeable) {
+        new_block->flags |= RAM_RESIZEABLE;
+    }
+    ram_block_add(new_block, &local_err);
     if (local_err) {
         g_free(new_block);
         error_propagate(errp, local_err);
-        return -1;
+        return NULL;
     }
-    return addr;
+    return new_block;
 }
 
-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
+RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
+                                   MemoryRegion *mr, Error **errp)
 {
-    return qemu_ram_alloc_from_ptr(size, NULL, mr, errp);
+    return qemu_ram_alloc_internal(size, size, NULL, host, false, mr, errp);
 }
 
-void qemu_ram_free_from_ptr(ram_addr_t addr)
+RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp)
 {
-    RAMBlock *block;
-
-    /* This assumes the iothread lock is taken here too.  */
-    qemu_mutex_lock_ramlist();
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (addr == block->offset) {
-            QTAILQ_REMOVE(&ram_list.blocks, block, next);
-            ram_list.mru_block = NULL;
-            ram_list.version++;
-            g_free(block);
-            break;
-        }
-    }
-    qemu_mutex_unlock_ramlist();
+    return qemu_ram_alloc_internal(size, size, NULL, NULL, false, mr, errp);
 }
 
-void qemu_ram_free(ram_addr_t addr)
+RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t maxsz,
+                                     void (*resized)(const char*,
+                                                     uint64_t length,
+                                                     void *host),
+                                     MemoryRegion *mr, Error **errp)
 {
-    RAMBlock *block;
+    return qemu_ram_alloc_internal(size, maxsz, resized, NULL, true, mr, errp);
+}
 
-    /* This assumes the iothread lock is taken here too.  */
-    qemu_mutex_lock_ramlist();
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        if (addr == block->offset) {
-            QTAILQ_REMOVE(&ram_list.blocks, block, next);
-            ram_list.mru_block = NULL;
-            ram_list.version++;
-            if (block->flags & RAM_PREALLOC) {
-                ;
-            } else if (xen_enabled()) {
-                xen_invalidate_map_cache_entry(block->host);
+static void reclaim_ramblock(RAMBlock *block)
+{
+    if (block->flags & RAM_PREALLOC) {
+        ;
+    } else if (xen_enabled()) {
+        xen_invalidate_map_cache_entry(block->host);
 #ifndef _WIN32
-            } else if (block->fd >= 0) {
-                munmap(block->host, block->length);
-                close(block->fd);
+    } else if (block->fd >= 0) {
+        qemu_ram_munmap(block->host, block->max_length);
+        close(block->fd);
 #endif
-            } else {
-                qemu_anon_ram_free(block->host, block->length);
-            }
-            g_free(block);
-            break;
-        }
+    } else {
+        qemu_anon_ram_free(block->host, block->max_length);
     }
-    qemu_mutex_unlock_ramlist();
+    g_free(block);
+}
 
+void qemu_ram_free(RAMBlock *block)
+{
+    if (!block) {
+        return;
+    }
+
+    qemu_mutex_lock_ramlist();
+    QLIST_REMOVE_RCU(block, next);
+    ram_list.mru_block = NULL;
+    /* Write list before version */
+    smp_wmb();
+    ram_list.version++;
+    call_rcu(block, reclaim_ramblock, rcu);
+    qemu_mutex_unlock_ramlist();
 }
 
 #ifndef _WIN32
@@ -1534,17 +1788,16 @@
     int flags;
     void *area, *vaddr;
 
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
         offset = addr - block->offset;
-        if (offset < block->length) {
-            vaddr = block->host + offset;
+        if (offset < block->max_length) {
+            vaddr = ramblock_ptr(block, offset);
             if (block->flags & RAM_PREALLOC) {
                 ;
             } else if (xen_enabled()) {
                 abort();
             } else {
                 flags = MAP_FIXED;
-                munmap(vaddr, length);
                 if (block->fd >= 0) {
                     flags |= (block->flags & RAM_SHARED ?
                               MAP_SHARED : MAP_PRIVATE);
@@ -1571,112 +1824,174 @@
                 memory_try_enable_merging(vaddr, length);
                 qemu_ram_setup_dump(vaddr, length);
             }
-            return;
         }
     }
 }
 #endif /* !_WIN32 */
 
-int qemu_get_ram_fd(ram_addr_t addr)
-{
-    RAMBlock *block = qemu_get_ram_block(addr);
-
-    return block->fd;
-}
-
-void *qemu_get_ram_block_host_ptr(ram_addr_t addr)
-{
-    RAMBlock *block = qemu_get_ram_block(addr);
-
-    return block->host;
-}
-
 /* Return a host pointer to ram allocated with qemu_ram_alloc.
-   With the exception of the softmmu code in this file, this should
-   only be used for local memory (e.g. video ram) that the device owns,
-   and knows it isn't going to access beyond the end of the block.
-
-   It should not be used for general purpose DMA.
-   Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
+ * This should not be used for general purpose DMA.  Use address_space_map
+ * or address_space_rw instead. For local memory (e.g. video ram) that the
+ * device owns, use memory_region_get_ram_ptr.
+ *
+ * Called within RCU critical section.
  */
-void *qemu_get_ram_ptr(ram_addr_t addr)
+void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr)
 {
-    RAMBlock *block = qemu_get_ram_block(addr);
+    RAMBlock *block = ram_block;
 
-    if (xen_enabled()) {
+    if (block == NULL) {
+        block = qemu_get_ram_block(addr);
+        addr -= block->offset;
+    }
+
+    if (xen_enabled() && block->host == NULL) {
         /* We need to check if the requested address is in the RAM
          * because we don't want to map the entire memory in QEMU.
          * In that case just map until the end of the page.
          */
         if (block->offset == 0) {
             return xen_map_cache(addr, 0, 0);
-        } else if (block->host == NULL) {
-            block->host =
-                xen_map_cache(block->offset, block->length, 1);
         }
+
+        block->host = xen_map_cache(block->offset, block->max_length, 1);
     }
-    return block->host + (addr - block->offset);
+    return ramblock_ptr(block, addr);
 }
 
-/* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
- * but takes a size argument */
-static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
+/* Return a host pointer to guest's ram. Similar to qemu_map_ram_ptr
+ * but takes a size argument.
+ *
+ * Called within RCU critical section.
+ */
+static void *qemu_ram_ptr_length(RAMBlock *ram_block, ram_addr_t addr,
+                                 hwaddr *size)
 {
+    RAMBlock *block = ram_block;
     if (*size == 0) {
         return NULL;
     }
-    if (xen_enabled()) {
-        return xen_map_cache(addr, *size, 1);
-    } else {
-        RAMBlock *block;
 
-        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-            if (addr - block->offset < block->length) {
-                if (addr - block->offset + *size > block->length)
-                    *size = block->length - addr + block->offset;
-                return block->host + (addr - block->offset);
-            }
+    if (block == NULL) {
+        block = qemu_get_ram_block(addr);
+        addr -= block->offset;
+    }
+    *size = MIN(*size, block->max_length - addr);
+
+    if (xen_enabled() && block->host == NULL) {
+        /* We need to check if the requested address is in the RAM
+         * because we don't want to map the entire memory in QEMU.
+         * In that case just map the requested area.
+         */
+        if (block->offset == 0) {
+            return xen_map_cache(addr, *size, 1);
         }
 
-        fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
-        abort();
+        block->host = xen_map_cache(block->offset, block->max_length, 1);
     }
+
+    return ramblock_ptr(block, addr);
 }
 
-/* Some of the softmmu routines need to translate from a host pointer
-   (typically a TLB entry) back to a ram offset.  */
-MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
+/*
+ * Translates a host ptr back to a RAMBlock, a ram_addr and an offset
+ * in that RAMBlock.
+ *
+ * ptr: Host pointer to look up
+ * round_offset: If true round the result offset down to a page boundary
+ * *ram_addr: set to result ram_addr
+ * *offset: set to result offset within the RAMBlock
+ *
+ * Returns: RAMBlock (or NULL if not found)
+ *
+ * By the time this function returns, the returned pointer is not protected
+ * by RCU anymore.  If the caller is not within an RCU critical section and
+ * does not hold the iothread lock, it must have other means of protecting the
+ * pointer, such as a reference to the region that includes the incoming
+ * ram_addr_t.
+ */
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *offset)
 {
     RAMBlock *block;
     uint8_t *host = ptr;
 
     if (xen_enabled()) {
-        *ram_addr = xen_ram_addr_from_mapcache(ptr);
-        return qemu_get_ram_block(*ram_addr)->mr;
+        ram_addr_t ram_addr;
+        rcu_read_lock();
+        ram_addr = xen_ram_addr_from_mapcache(ptr);
+        block = qemu_get_ram_block(ram_addr);
+        if (block) {
+            *offset = ram_addr - block->offset;
+        }
+        rcu_read_unlock();
+        return block;
     }
 
-    block = ram_list.mru_block;
-    if (block && block->host && host - block->host < block->length) {
+    rcu_read_lock();
+    block = atomic_rcu_read(&ram_list.mru_block);
+    if (block && block->host && host - block->host < block->max_length) {
         goto found;
     }
 
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
         /* This case append when the block is not mapped. */
         if (block->host == NULL) {
             continue;
         }
-        if (host - block->host < block->length) {
+        if (host - block->host < block->max_length) {
             goto found;
         }
     }
 
+    rcu_read_unlock();
     return NULL;
 
 found:
-    *ram_addr = block->offset + (host - block->host);
-    return block->mr;
+    *offset = (host - block->host);
+    if (round_offset) {
+        *offset &= TARGET_PAGE_MASK;
+    }
+    rcu_read_unlock();
+    return block;
 }
 
+/*
+ * Finds the named RAMBlock
+ *
+ * name: The name of RAMBlock to find
+ *
+ * Returns: RAMBlock (or NULL if not found)
+ */
+RAMBlock *qemu_ram_block_by_name(const char *name)
+{
+    RAMBlock *block;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        if (!strcmp(name, block->idstr)) {
+            return block;
+        }
+    }
+
+    return NULL;
+}
+
+/* Some of the softmmu routines need to translate from a host pointer
+   (typically a TLB entry) back to a ram offset.  */
+ram_addr_t qemu_ram_addr_from_host(void *ptr)
+{
+    RAMBlock *block;
+    ram_addr_t offset;
+
+    block = qemu_ram_block_from_host(ptr, false, &offset);
+    if (!block) {
+        return RAM_ADDR_INVALID;
+    }
+
+    return block->offset + offset;
+}
+
+/* Called within RCU critical section.  */
 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
                                uint64_t val, unsigned size)
 {
@@ -1685,23 +2000,26 @@
     }
     switch (size) {
     case 1:
-        stb_p(qemu_get_ram_ptr(ram_addr), val);
+        stb_p(qemu_map_ram_ptr(NULL, ram_addr), val);
         break;
     case 2:
-        stw_p(qemu_get_ram_ptr(ram_addr), val);
+        stw_p(qemu_map_ram_ptr(NULL, ram_addr), val);
         break;
     case 4:
-        stl_p(qemu_get_ram_ptr(ram_addr), val);
+        stl_p(qemu_map_ram_ptr(NULL, ram_addr), val);
         break;
     default:
         abort();
     }
-    cpu_physical_memory_set_dirty_range_nocode(ram_addr, size);
+    /* Set both VGA and migration bits for simplicity and to remove
+     * the notdirty callback faster.
+     */
+    cpu_physical_memory_set_dirty_range(ram_addr, size,
+                                        DIRTY_CLIENTS_NOCODE);
     /* we remove the notdirty callback only if the code has been
        flushed */
     if (!cpu_physical_memory_is_clean(ram_addr)) {
-        CPUArchState *env = current_cpu->env_ptr;
-        tlb_set_dirty(env, current_cpu->mem_io_vaddr);
+        tlb_set_dirty(current_cpu, current_cpu->mem_io_vaddr);
     }
 }
 
@@ -1718,14 +2036,15 @@
 };
 
 /* Generate a debug exception if a watchpoint has been hit.  */
-static void check_watchpoint(int offset, int len, int flags)
+static void check_watchpoint(int offset, int len, MemTxAttrs attrs, int flags)
 {
     CPUState *cpu = current_cpu;
+    CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUArchState *env = cpu->env_ptr;
     target_ulong pc, cs_base;
     target_ulong vaddr;
     CPUWatchpoint *wp;
-    int cpu_flags;
+    uint32_t cpu_flags;
 
     if (cpu->watchpoint_hit) {
         /* We re-entered the check after replacing the TB. Now raise
@@ -1744,7 +2063,13 @@
                 wp->flags |= BP_WATCHPOINT_HIT_WRITE;
             }
             wp->hitaddr = vaddr;
+            wp->hitattrs = attrs;
             if (!cpu->watchpoint_hit) {
+                if (wp->flags & BP_CPU &&
+                    !cc->debug_check_watchpoint(cpu, wp)) {
+                    wp->flags &= ~BP_WATCHPOINT_HIT;
+                    continue;
+                }
                 cpu->watchpoint_hit = wp;
                 tb_check_watchpoint(cpu);
                 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
@@ -1753,7 +2078,7 @@
                 } else {
                     cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
                     tb_gen_code(cpu, pc, cs_base, cpu_flags, 1);
-                    cpu_resume_from_signal(cpu, NULL);
+                    cpu_loop_exit_noexc(cpu);
                 }
             }
         } else {
@@ -1765,70 +2090,100 @@
 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
    so these check for a hit then pass through to the normal out-of-line
    phys routines.  */
-static uint64_t watch_mem_read(void *opaque, hwaddr addr,
-                               unsigned size)
+static MemTxResult watch_mem_read(void *opaque, hwaddr addr, uint64_t *pdata,
+                                  unsigned size, MemTxAttrs attrs)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_READ);
-    switch (size) {
-    case 1: return ldub_phys(&address_space_memory, addr);
-    case 2: return lduw_phys(&address_space_memory, addr);
-    case 4: return ldl_phys(&address_space_memory, addr);
-    default: abort();
-    }
-}
+    MemTxResult res;
+    uint64_t data;
+    int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
+    AddressSpace *as = current_cpu->cpu_ases[asidx].as;
 
-static void watch_mem_write(void *opaque, hwaddr addr,
-                            uint64_t val, unsigned size)
-{
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, BP_MEM_WRITE);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_READ);
     switch (size) {
     case 1:
-        stb_phys(&address_space_memory, addr, val);
+        data = address_space_ldub(as, addr, attrs, &res);
         break;
     case 2:
-        stw_phys(&address_space_memory, addr, val);
+        data = address_space_lduw(as, addr, attrs, &res);
         break;
     case 4:
-        stl_phys(&address_space_memory, addr, val);
+        data = address_space_ldl(as, addr, attrs, &res);
         break;
     default: abort();
     }
+    *pdata = data;
+    return res;
+}
+
+static MemTxResult watch_mem_write(void *opaque, hwaddr addr,
+                                   uint64_t val, unsigned size,
+                                   MemTxAttrs attrs)
+{
+    MemTxResult res;
+    int asidx = cpu_asidx_from_attrs(current_cpu, attrs);
+    AddressSpace *as = current_cpu->cpu_ases[asidx].as;
+
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, size, attrs, BP_MEM_WRITE);
+    switch (size) {
+    case 1:
+        address_space_stb(as, addr, val, attrs, &res);
+        break;
+    case 2:
+        address_space_stw(as, addr, val, attrs, &res);
+        break;
+    case 4:
+        address_space_stl(as, addr, val, attrs, &res);
+        break;
+    default: abort();
+    }
+    return res;
 }
 
 static const MemoryRegionOps watch_mem_ops = {
-    .read = watch_mem_read,
-    .write = watch_mem_write,
+    .read_with_attrs = watch_mem_read,
+    .write_with_attrs = watch_mem_write,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t subpage_read(void *opaque, hwaddr addr,
-                             unsigned len)
+static MemTxResult subpage_read(void *opaque, hwaddr addr, uint64_t *data,
+                                unsigned len, MemTxAttrs attrs)
 {
     subpage_t *subpage = opaque;
-    uint8_t buf[4];
+    uint8_t buf[8];
+    MemTxResult res;
 
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
            subpage, len, addr);
 #endif
-    address_space_read(subpage->as, addr + subpage->base, buf, len);
+    res = address_space_read(subpage->as, addr + subpage->base,
+                             attrs, buf, len);
+    if (res) {
+        return res;
+    }
     switch (len) {
     case 1:
-        return ldub_p(buf);
+        *data = ldub_p(buf);
+        return MEMTX_OK;
     case 2:
-        return lduw_p(buf);
+        *data = lduw_p(buf);
+        return MEMTX_OK;
     case 4:
-        return ldl_p(buf);
+        *data = ldl_p(buf);
+        return MEMTX_OK;
+    case 8:
+        *data = ldq_p(buf);
+        return MEMTX_OK;
     default:
         abort();
     }
 }
 
-static void subpage_write(void *opaque, hwaddr addr,
-                          uint64_t value, unsigned len)
+static MemTxResult subpage_write(void *opaque, hwaddr addr,
+                                 uint64_t value, unsigned len, MemTxAttrs attrs)
 {
     subpage_t *subpage = opaque;
-    uint8_t buf[4];
+    uint8_t buf[8];
 
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %u addr " TARGET_FMT_plx
@@ -1845,10 +2200,14 @@
     case 4:
         stl_p(buf, value);
         break;
+    case 8:
+        stq_p(buf, value);
+        break;
     default:
         abort();
     }
-    address_space_write(subpage->as, addr + subpage->base, buf, len);
+    return address_space_write(subpage->as, addr + subpage->base,
+                               attrs, buf, len);
 }
 
 static bool subpage_accepts(void *opaque, hwaddr addr,
@@ -1865,8 +2224,12 @@
 }
 
 static const MemoryRegionOps subpage_ops = {
-    .read = subpage_read,
-    .write = subpage_write,
+    .read_with_attrs = subpage_read,
+    .write_with_attrs = subpage_write,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 8,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 8,
     .valid.accepts = subpage_accepts,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
@@ -1926,9 +2289,14 @@
     return phys_section_add(map, &section);
 }
 
-MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index)
+MemoryRegion *iotlb_to_region(CPUState *cpu, hwaddr index, MemTxAttrs attrs)
 {
-    return as->dispatch->map.sections[index & ~TARGET_PAGE_MASK].mr;
+    int asidx = cpu_asidx_from_attrs(cpu, attrs);
+    CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
+    AddressSpaceDispatch *d = atomic_rcu_read(&cpuas->memory_dispatch);
+    MemoryRegionSection *sections = d->map.sections;
+
+    return sections[index & ~TARGET_PAGE_MASK].mr;
 }
 
 static void io_mem_init(void)
@@ -1966,6 +2334,12 @@
     as->next_dispatch = d;
 }
 
+static void address_space_dispatch_free(AddressSpaceDispatch *d)
+{
+    phys_sections_free(&d->map);
+    g_free(d);
+}
+
 static void mem_commit(MemoryListener *listener)
 {
     AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
@@ -1974,47 +2348,30 @@
 
     phys_page_compact_all(next, next->map.nodes_nb);
 
-    as->dispatch = next;
-
+    atomic_rcu_set(&as->dispatch, next);
     if (cur) {
-        phys_sections_free(&cur->map);
-        g_free(cur);
+        call_rcu(cur, address_space_dispatch_free, rcu);
     }
 }
 
 static void tcg_commit(MemoryListener *listener)
 {
-    CPUState *cpu;
+    CPUAddressSpace *cpuas;
+    AddressSpaceDispatch *d;
 
     /* since each CPU stores ram addresses in its TLB cache, we must
        reset the modified entries */
-    /* XXX: slow ! */
-    CPU_FOREACH(cpu) {
-        /* FIXME: Disentangle the cpu.h circular files deps so we can
-           directly get the right CPU from listener.  */
-        if (cpu->tcg_as_listener != listener) {
-            continue;
-        }
-        tlb_flush(cpu, 1);
-    }
+    cpuas = container_of(listener, CPUAddressSpace, tcg_as_listener);
+    cpu_reloading_memory_map();
+    /* The CPU and TLB are protected by the iothread lock.
+     * We reload the dispatch pointer now because cpu_reloading_memory_map()
+     * may have split the RCU critical section.
+     */
+    d = atomic_rcu_read(&cpuas->as->dispatch);
+    cpuas->memory_dispatch = d;
+    tlb_flush(cpuas->cpu, 1);
 }
 
-static void core_log_global_start(MemoryListener *listener)
-{
-    cpu_physical_memory_set_dirty_tracking(true);
-}
-
-static void core_log_global_stop(MemoryListener *listener)
-{
-    cpu_physical_memory_set_dirty_tracking(false);
-}
-
-static MemoryListener core_memory_listener = {
-    .log_global_start = core_log_global_start,
-    .log_global_stop = core_log_global_stop,
-    .priority = 1,
-};
-
 void address_space_init_dispatch(AddressSpace *as)
 {
     as->dispatch = NULL;
@@ -2028,13 +2385,19 @@
     memory_listener_register(&as->dispatch_listener, as);
 }
 
+void address_space_unregister(AddressSpace *as)
+{
+    memory_listener_unregister(&as->dispatch_listener);
+}
+
 void address_space_destroy_dispatch(AddressSpace *as)
 {
     AddressSpaceDispatch *d = as->dispatch;
 
-    memory_listener_unregister(&as->dispatch_listener);
-    g_free(d);
-    as->dispatch = NULL;
+    atomic_rcu_set(&as->dispatch, NULL);
+    if (d) {
+        call_rcu(d, address_space_dispatch_free, rcu);
+    }
 }
 
 static void memory_map_init(void)
@@ -2048,8 +2411,6 @@
     memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
                           65536);
     address_space_init(&address_space_io, system_io, "I/O");
-
-    memory_listener_register(&core_memory_listener, &address_space_memory);
 }
 
 MemoryRegion *get_system_memory(void)
@@ -2107,14 +2468,25 @@
 
 #else
 
-static void invalidate_and_set_dirty(hwaddr addr,
+static void invalidate_and_set_dirty(MemoryRegion *mr, hwaddr addr,
                                      hwaddr length)
 {
-    if (cpu_physical_memory_range_includes_clean(addr, length)) {
-        tb_invalidate_phys_range(addr, addr + length, 0);
-        cpu_physical_memory_set_dirty_range_nocode(addr, length);
+    uint8_t dirty_log_mask = memory_region_get_dirty_log_mask(mr);
+    addr += memory_region_get_ram_addr(mr);
+
+    /* No early return if dirty_log_mask is or becomes 0, because
+     * cpu_physical_memory_set_dirty_range will still call
+     * xen_modified_memory.
+     */
+    if (dirty_log_mask) {
+        dirty_log_mask =
+            cpu_physical_memory_range_includes_clean(addr, length, dirty_log_mask);
     }
-    xen_modified_memory(addr, length);
+    if (dirty_log_mask & (1 << DIRTY_MEMORY_CODE)) {
+        tb_invalidate_phys_range(addr, addr + length);
+        dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
+    }
+    cpu_physical_memory_set_dirty_range(addr, length, dirty_log_mask);
 }
 
 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
@@ -2139,121 +2511,232 @@
     if (l > access_size_max) {
         l = access_size_max;
     }
-    if (l & (l - 1)) {
-        l = 1 << (qemu_fls(l) - 1);
-    }
+    l = pow2floor(l);
 
     return l;
 }
 
-bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
-                      int len, bool is_write)
+static bool prepare_mmio_access(MemoryRegion *mr)
 {
-    hwaddr l;
+    bool unlocked = !qemu_mutex_iothread_locked();
+    bool release_lock = false;
+
+    if (unlocked && mr->global_locking) {
+        qemu_mutex_lock_iothread();
+        unlocked = false;
+        release_lock = true;
+    }
+    if (mr->flush_coalesced_mmio) {
+        if (unlocked) {
+            qemu_mutex_lock_iothread();
+        }
+        qemu_flush_coalesced_mmio_buffer();
+        if (unlocked) {
+            qemu_mutex_unlock_iothread();
+        }
+    }
+
+    return release_lock;
+}
+
+/* Called within RCU critical section.  */
+static MemTxResult address_space_write_continue(AddressSpace *as, hwaddr addr,
+                                                MemTxAttrs attrs,
+                                                const uint8_t *buf,
+                                                int len, hwaddr addr1,
+                                                hwaddr l, MemoryRegion *mr)
+{
     uint8_t *ptr;
     uint64_t val;
-    hwaddr addr1;
-    MemoryRegion *mr;
-    bool error = false;
+    MemTxResult result = MEMTX_OK;
+    bool release_lock = false;
 
-    while (len > 0) {
-        l = len;
-        mr = address_space_translate(as, addr, &addr1, &l, is_write);
-
-        if (is_write) {
-            if (!memory_access_is_direct(mr, is_write)) {
-                l = memory_access_size(mr, l, addr1);
-                /* XXX: could force current_cpu to NULL to avoid
-                   potential bugs */
-                switch (l) {
-                case 8:
-                    /* 64 bit write access */
-                    val = ldq_p(buf);
-                    error |= io_mem_write(mr, addr1, val, 8);
-                    break;
-                case 4:
-                    /* 32 bit write access */
-                    val = ldl_p(buf);
-                    error |= io_mem_write(mr, addr1, val, 4);
-                    break;
-                case 2:
-                    /* 16 bit write access */
-                    val = lduw_p(buf);
-                    error |= io_mem_write(mr, addr1, val, 2);
-                    break;
-                case 1:
-                    /* 8 bit write access */
-                    val = ldub_p(buf);
-                    error |= io_mem_write(mr, addr1, val, 1);
-                    break;
-                default:
-                    abort();
-                }
-            } else {
-                addr1 += memory_region_get_ram_addr(mr);
-                /* RAM case */
-                ptr = qemu_get_ram_ptr(addr1);
-                memcpy(ptr, buf, l);
-                invalidate_and_set_dirty(addr1, l);
+    for (;;) {
+        if (!memory_access_is_direct(mr, true)) {
+            release_lock |= prepare_mmio_access(mr);
+            l = memory_access_size(mr, l, addr1);
+            /* XXX: could force current_cpu to NULL to avoid
+               potential bugs */
+            switch (l) {
+            case 8:
+                /* 64 bit write access */
+                val = ldq_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 8,
+                                                       attrs);
+                break;
+            case 4:
+                /* 32 bit write access */
+                val = ldl_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 4,
+                                                       attrs);
+                break;
+            case 2:
+                /* 16 bit write access */
+                val = lduw_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 2,
+                                                       attrs);
+                break;
+            case 1:
+                /* 8 bit write access */
+                val = ldub_p(buf);
+                result |= memory_region_dispatch_write(mr, addr1, val, 1,
+                                                       attrs);
+                break;
+            default:
+                abort();
             }
         } else {
-            if (!memory_access_is_direct(mr, is_write)) {
-                /* I/O case */
-                l = memory_access_size(mr, l, addr1);
-                switch (l) {
-                case 8:
-                    /* 64 bit read access */
-                    error |= io_mem_read(mr, addr1, &val, 8);
-                    stq_p(buf, val);
-                    break;
-                case 4:
-                    /* 32 bit read access */
-                    error |= io_mem_read(mr, addr1, &val, 4);
-                    stl_p(buf, val);
-                    break;
-                case 2:
-                    /* 16 bit read access */
-                    error |= io_mem_read(mr, addr1, &val, 2);
-                    stw_p(buf, val);
-                    break;
-                case 1:
-                    /* 8 bit read access */
-                    error |= io_mem_read(mr, addr1, &val, 1);
-                    stb_p(buf, val);
-                    break;
-                default:
-                    abort();
-                }
-            } else {
-                /* RAM case */
-                ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
-                memcpy(buf, ptr, l);
-            }
+            /* RAM case */
+            ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
+            memcpy(ptr, buf, l);
+            invalidate_and_set_dirty(mr, addr1, l);
         }
+
+        if (release_lock) {
+            qemu_mutex_unlock_iothread();
+            release_lock = false;
+        }
+
         len -= l;
         buf += l;
         addr += l;
+
+        if (!len) {
+            break;
+        }
+
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
     }
 
-    return error;
+    return result;
 }
 
-bool address_space_write(AddressSpace *as, hwaddr addr,
-                         const uint8_t *buf, int len)
+MemTxResult address_space_write(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                                const uint8_t *buf, int len)
 {
-    return address_space_rw(as, addr, (uint8_t *)buf, len, true);
+    hwaddr l;
+    hwaddr addr1;
+    MemoryRegion *mr;
+    MemTxResult result = MEMTX_OK;
+
+    if (len > 0) {
+        rcu_read_lock();
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, true);
+        result = address_space_write_continue(as, addr, attrs, buf, len,
+                                              addr1, l, mr);
+        rcu_read_unlock();
+    }
+
+    return result;
 }
 
-bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
+/* Called within RCU critical section.  */
+MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
+                                        MemTxAttrs attrs, uint8_t *buf,
+                                        int len, hwaddr addr1, hwaddr l,
+                                        MemoryRegion *mr)
 {
-    return address_space_rw(as, addr, buf, len, false);
+    uint8_t *ptr;
+    uint64_t val;
+    MemTxResult result = MEMTX_OK;
+    bool release_lock = false;
+
+    for (;;) {
+        if (!memory_access_is_direct(mr, false)) {
+            /* I/O case */
+            release_lock |= prepare_mmio_access(mr);
+            l = memory_access_size(mr, l, addr1);
+            switch (l) {
+            case 8:
+                /* 64 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 8,
+                                                      attrs);
+                stq_p(buf, val);
+                break;
+            case 4:
+                /* 32 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 4,
+                                                      attrs);
+                stl_p(buf, val);
+                break;
+            case 2:
+                /* 16 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 2,
+                                                      attrs);
+                stw_p(buf, val);
+                break;
+            case 1:
+                /* 8 bit read access */
+                result |= memory_region_dispatch_read(mr, addr1, &val, 1,
+                                                      attrs);
+                stb_p(buf, val);
+                break;
+            default:
+                abort();
+            }
+        } else {
+            /* RAM case */
+            ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
+            memcpy(buf, ptr, l);
+        }
+
+        if (release_lock) {
+            qemu_mutex_unlock_iothread();
+            release_lock = false;
+        }
+
+        len -= l;
+        buf += l;
+        addr += l;
+
+        if (!len) {
+            break;
+        }
+
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, false);
+    }
+
+    return result;
 }
 
+MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
+                                    MemTxAttrs attrs, uint8_t *buf, int len)
+{
+    hwaddr l;
+    hwaddr addr1;
+    MemoryRegion *mr;
+    MemTxResult result = MEMTX_OK;
+
+    if (len > 0) {
+        rcu_read_lock();
+        l = len;
+        mr = address_space_translate(as, addr, &addr1, &l, false);
+        result = address_space_read_continue(as, addr, attrs, buf, len,
+                                             addr1, l, mr);
+        rcu_read_unlock();
+    }
+
+    return result;
+}
+
+MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                             uint8_t *buf, int len, bool is_write)
+{
+    if (is_write) {
+        return address_space_write(as, addr, attrs, (uint8_t *)buf, len);
+    } else {
+        return address_space_read(as, addr, attrs, (uint8_t *)buf, len);
+    }
+}
 
 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                             int len, int is_write)
 {
-    address_space_rw(&address_space_memory, addr, buf, len, is_write);
+    address_space_rw(&address_space_memory, addr, MEMTXATTRS_UNSPECIFIED,
+                     buf, len, is_write);
 }
 
 enum write_rom_type {
@@ -2269,21 +2752,21 @@
     hwaddr addr1;
     MemoryRegion *mr;
 
+    rcu_read_lock();
     while (len > 0) {
         l = len;
         mr = address_space_translate(as, addr, &addr1, &l, true);
 
         if (!(memory_region_is_ram(mr) ||
               memory_region_is_romd(mr))) {
-            /* do nothing */
+            l = memory_access_size(mr, l, addr1);
         } else {
-            addr1 += memory_region_get_ram_addr(mr);
             /* ROM/RAM case */
-            ptr = qemu_get_ram_ptr(addr1);
+            ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
             switch (type) {
             case WRITE_DATA:
                 memcpy(ptr, buf, l);
-                invalidate_and_set_dirty(addr1, l);
+                invalidate_and_set_dirty(mr, addr1, l);
                 break;
             case FLUSH_CACHE:
                 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
@@ -2294,6 +2777,7 @@
         buf += l;
         addr += l;
     }
+    rcu_read_unlock();
 }
 
 /* used for ROM loading : can write in RAM and ROM */
@@ -2324,53 +2808,85 @@
     void *buffer;
     hwaddr addr;
     hwaddr len;
+    bool in_use;
 } BounceBuffer;
 
 static BounceBuffer bounce;
 
 typedef struct MapClient {
-    void *opaque;
-    void (*callback)(void *opaque);
+    QEMUBH *bh;
     QLIST_ENTRY(MapClient) link;
 } MapClient;
 
+QemuMutex map_client_list_lock;
 static QLIST_HEAD(map_client_list, MapClient) map_client_list
     = QLIST_HEAD_INITIALIZER(map_client_list);
 
-void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
+static void cpu_unregister_map_client_do(MapClient *client)
 {
-    MapClient *client = g_malloc(sizeof(*client));
-
-    client->opaque = opaque;
-    client->callback = callback;
-    QLIST_INSERT_HEAD(&map_client_list, client, link);
-    return client;
-}
-
-static void cpu_unregister_map_client(void *_client)
-{
-    MapClient *client = (MapClient *)_client;
-
     QLIST_REMOVE(client, link);
     g_free(client);
 }
 
-static void cpu_notify_map_clients(void)
+static void cpu_notify_map_clients_locked(void)
 {
     MapClient *client;
 
     while (!QLIST_EMPTY(&map_client_list)) {
         client = QLIST_FIRST(&map_client_list);
-        client->callback(client->opaque);
-        cpu_unregister_map_client(client);
+        qemu_bh_schedule(client->bh);
+        cpu_unregister_map_client_do(client);
     }
 }
 
+void cpu_register_map_client(QEMUBH *bh)
+{
+    MapClient *client = g_malloc(sizeof(*client));
+
+    qemu_mutex_lock(&map_client_list_lock);
+    client->bh = bh;
+    QLIST_INSERT_HEAD(&map_client_list, client, link);
+    if (!atomic_read(&bounce.in_use)) {
+        cpu_notify_map_clients_locked();
+    }
+    qemu_mutex_unlock(&map_client_list_lock);
+}
+
+void cpu_exec_init_all(void)
+{
+    qemu_mutex_init(&ram_list.mutex);
+    io_mem_init();
+    memory_map_init();
+    qemu_mutex_init(&map_client_list_lock);
+}
+
+void cpu_unregister_map_client(QEMUBH *bh)
+{
+    MapClient *client;
+
+    qemu_mutex_lock(&map_client_list_lock);
+    QLIST_FOREACH(client, &map_client_list, link) {
+        if (client->bh == bh) {
+            cpu_unregister_map_client_do(client);
+            break;
+        }
+    }
+    qemu_mutex_unlock(&map_client_list_lock);
+}
+
+static void cpu_notify_map_clients(void)
+{
+    qemu_mutex_lock(&map_client_list_lock);
+    cpu_notify_map_clients_locked();
+    qemu_mutex_unlock(&map_client_list_lock);
+}
+
 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
 {
     MemoryRegion *mr;
     hwaddr l, xlat;
 
+    rcu_read_lock();
     while (len > 0) {
         l = len;
         mr = address_space_translate(as, addr, &xlat, &l, is_write);
@@ -2384,6 +2900,7 @@
         len -= l;
         addr += l;
     }
+    rcu_read_unlock();
     return true;
 }
 
@@ -2403,16 +2920,19 @@
     hwaddr done = 0;
     hwaddr l, xlat, base;
     MemoryRegion *mr, *this_mr;
-    ram_addr_t raddr;
+    void *ptr;
 
     if (len == 0) {
         return NULL;
     }
 
     l = len;
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &xlat, &l, is_write);
+
     if (!memory_access_is_direct(mr, is_write)) {
-        if (bounce.buffer) {
+        if (atomic_xchg(&bounce.in_use, true)) {
+            rcu_read_unlock();
             return NULL;
         }
         /* Avoid unbounded allocations */
@@ -2424,15 +2944,16 @@
         memory_region_ref(mr);
         bounce.mr = mr;
         if (!is_write) {
-            address_space_read(as, addr, bounce.buffer, l);
+            address_space_read(as, addr, MEMTXATTRS_UNSPECIFIED,
+                               bounce.buffer, l);
         }
 
+        rcu_read_unlock();
         *plen = l;
         return bounce.buffer;
     }
 
     base = xlat;
-    raddr = memory_region_get_ram_addr(mr);
 
     for (;;) {
         len -= l;
@@ -2451,7 +2972,10 @@
 
     memory_region_ref(mr);
     *plen = done;
-    return qemu_ram_ptr_length(raddr + base, plen);
+    ptr = qemu_ram_ptr_length(mr->ram_block, base, plen);
+    rcu_read_unlock();
+
+    return ptr;
 }
 
 /* Unmaps a memory region previously mapped by address_space_map().
@@ -2465,10 +2989,10 @@
         MemoryRegion *mr;
         ram_addr_t addr1;
 
-        mr = qemu_ram_addr_from_host(buffer, &addr1);
+        mr = memory_region_from_host(buffer, &addr1);
         assert(mr != NULL);
         if (is_write) {
-            invalidate_and_set_dirty(addr1, access_len);
+            invalidate_and_set_dirty(mr, addr1, access_len);
         }
         if (xen_enabled()) {
             xen_invalidate_map_cache_entry(buffer);
@@ -2477,11 +3001,13 @@
         return;
     }
     if (is_write) {
-        address_space_write(as, bounce.addr, bounce.buffer, access_len);
+        address_space_write(as, bounce.addr, MEMTXATTRS_UNSPECIFIED,
+                            bounce.buffer, access_len);
     }
     qemu_vfree(bounce.buffer);
     bounce.buffer = NULL;
     memory_region_unref(bounce.mr);
+    atomic_mb_set(&bounce.in_use, false);
     cpu_notify_map_clients();
 }
 
@@ -2499,19 +3025,26 @@
 }
 
 /* warning: addr must be aligned */
-static inline uint32_t ldl_phys_internal(AddressSpace *as, hwaddr addr,
-                                         enum device_endian endian)
+static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr,
+                                                  MemTxAttrs attrs,
+                                                  MemTxResult *result,
+                                                  enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegion *mr;
     hwaddr l = 4;
     hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr1, &l, false);
     if (l < 4 || !memory_access_is_direct(mr, false)) {
+        release_lock |= prepare_mmio_access(mr);
+
         /* I/O case */
-        io_mem_read(mr, addr1, &val, 4);
+        r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap32(val);
@@ -2523,9 +3056,7 @@
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
-                                & TARGET_PAGE_MASK)
-                               + addr1);
+        ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             val = ldl_le_p(ptr);
@@ -2537,40 +3068,76 @@
             val = ldl_p(ptr);
             break;
         }
+        r = MEMTX_OK;
     }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
     return val;
 }
 
+uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldl_internal(as, addr, attrs, result,
+                                      DEVICE_NATIVE_ENDIAN);
+}
+
+uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
+                              MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldl_internal(as, addr, attrs, result,
+                                      DEVICE_LITTLE_ENDIAN);
+}
+
+uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
+                              MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldl_internal(as, addr, attrs, result,
+                                      DEVICE_BIG_ENDIAN);
+}
+
 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
 {
-    return ldl_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
+    return address_space_ldl(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
 {
-    return ldl_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
+    return address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
 {
-    return ldl_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
+    return address_space_ldl_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* warning: addr must be aligned */
-static inline uint64_t ldq_phys_internal(AddressSpace *as, hwaddr addr,
-                                         enum device_endian endian)
+static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr,
+                                                  MemTxAttrs attrs,
+                                                  MemTxResult *result,
+                                                  enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegion *mr;
     hwaddr l = 8;
     hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr1, &l,
                                  false);
     if (l < 8 || !memory_access_is_direct(mr, false)) {
+        release_lock |= prepare_mmio_access(mr);
+
         /* I/O case */
-        io_mem_read(mr, addr1, &val, 8);
+        r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap64(val);
@@ -2582,9 +3149,7 @@
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
-                                & TARGET_PAGE_MASK)
-                               + addr1);
+        ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             val = ldq_le_p(ptr);
@@ -2596,48 +3161,96 @@
             val = ldq_p(ptr);
             break;
         }
+        r = MEMTX_OK;
     }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
     return val;
 }
 
+uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldq_internal(as, addr, attrs, result,
+                                      DEVICE_NATIVE_ENDIAN);
+}
+
+uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldq_internal(as, addr, attrs, result,
+                                      DEVICE_LITTLE_ENDIAN);
+}
+
+uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_ldq_internal(as, addr, attrs, result,
+                                      DEVICE_BIG_ENDIAN);
+}
+
 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
 {
-    return ldq_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
+    return address_space_ldq(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
 {
-    return ldq_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
+    return address_space_ldq_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
 {
-    return ldq_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
+    return address_space_ldq_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* XXX: optimize */
-uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
+uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result)
 {
     uint8_t val;
-    address_space_rw(as, addr, &val, 1, 0);
+    MemTxResult r;
+
+    r = address_space_rw(as, addr, attrs, &val, 1, 0);
+    if (result) {
+        *result = r;
+    }
     return val;
 }
 
+uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
+{
+    return address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+}
+
 /* warning: addr must be aligned */
-static inline uint32_t lduw_phys_internal(AddressSpace *as, hwaddr addr,
-                                          enum device_endian endian)
+static inline uint32_t address_space_lduw_internal(AddressSpace *as,
+                                                   hwaddr addr,
+                                                   MemTxAttrs attrs,
+                                                   MemTxResult *result,
+                                                   enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegion *mr;
     hwaddr l = 2;
     hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr1, &l,
                                  false);
     if (l < 2 || !memory_access_is_direct(mr, false)) {
+        release_lock |= prepare_mmio_access(mr);
+
         /* I/O case */
-        io_mem_read(mr, addr1, &val, 2);
+        r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs);
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap16(val);
@@ -2649,9 +3262,7 @@
 #endif
     } else {
         /* RAM case */
-        ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
-                                & TARGET_PAGE_MASK)
-                               + addr1);
+        ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             val = lduw_le_p(ptr);
@@ -2663,68 +3274,119 @@
             val = lduw_p(ptr);
             break;
         }
+        r = MEMTX_OK;
     }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
     return val;
 }
 
+uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_lduw_internal(as, addr, attrs, result,
+                                       DEVICE_NATIVE_ENDIAN);
+}
+
+uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_lduw_internal(as, addr, attrs, result,
+                                       DEVICE_LITTLE_ENDIAN);
+}
+
+uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
+                           MemTxAttrs attrs, MemTxResult *result)
+{
+    return address_space_lduw_internal(as, addr, attrs, result,
+                                       DEVICE_BIG_ENDIAN);
+}
+
 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
 {
-    return lduw_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
+    return address_space_lduw(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
 {
-    return lduw_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
+    return address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
 {
-    return lduw_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
+    return address_space_lduw_be(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* warning: addr must be aligned. The ram page is not masked as dirty
    and the code inside is not invalidated. It is useful if the dirty
    bits are used to track modified PTEs */
-void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
+void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
+                                MemTxAttrs attrs, MemTxResult *result)
 {
     uint8_t *ptr;
     MemoryRegion *mr;
     hwaddr l = 4;
     hwaddr addr1;
+    MemTxResult r;
+    uint8_t dirty_log_mask;
+    bool release_lock = false;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr1, &l,
                                  true);
     if (l < 4 || !memory_access_is_direct(mr, true)) {
-        io_mem_write(mr, addr1, val, 4);
+        release_lock |= prepare_mmio_access(mr);
+
+        r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
     } else {
-        addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
-        ptr = qemu_get_ram_ptr(addr1);
+        ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
         stl_p(ptr, val);
 
-        if (unlikely(in_migration)) {
-            if (cpu_physical_memory_is_clean(addr1)) {
-                /* invalidate code */
-                tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
-                /* set dirty bit */
-                cpu_physical_memory_set_dirty_range_nocode(addr1, 4);
-            }
-        }
+        dirty_log_mask = memory_region_get_dirty_log_mask(mr);
+        dirty_log_mask &= ~(1 << DIRTY_MEMORY_CODE);
+        cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
+                                            4, dirty_log_mask);
+        r = MEMTX_OK;
     }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
+}
+
+void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
+{
+    address_space_stl_notdirty(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* warning: addr must be aligned */
-static inline void stl_phys_internal(AddressSpace *as,
-                                     hwaddr addr, uint32_t val,
-                                     enum device_endian endian)
+static inline void address_space_stl_internal(AddressSpace *as,
+                                              hwaddr addr, uint32_t val,
+                                              MemTxAttrs attrs,
+                                              MemTxResult *result,
+                                              enum device_endian endian)
 {
     uint8_t *ptr;
     MemoryRegion *mr;
     hwaddr l = 4;
     hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr1, &l,
                                  true);
     if (l < 4 || !memory_access_is_direct(mr, true)) {
+        release_lock |= prepare_mmio_access(mr);
+
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap32(val);
@@ -2734,11 +3396,10 @@
             val = bswap32(val);
         }
 #endif
-        io_mem_write(mr, addr1, val, 4);
+        r = memory_region_dispatch_write(mr, addr1, val, 4, attrs);
     } else {
         /* RAM case */
-        addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
-        ptr = qemu_get_ram_ptr(addr1);
+        ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             stl_le_p(ptr, val);
@@ -2750,44 +3411,91 @@
             stl_p(ptr, val);
             break;
         }
-        invalidate_and_set_dirty(addr1, 4);
+        invalidate_and_set_dirty(mr, addr1, 4);
+        r = MEMTX_OK;
     }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
+}
+
+void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    address_space_stl_internal(as, addr, val, attrs, result,
+                               DEVICE_NATIVE_ENDIAN);
+}
+
+void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    address_space_stl_internal(as, addr, val, attrs, result,
+                               DEVICE_LITTLE_ENDIAN);
+}
+
+void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    address_space_stl_internal(as, addr, val, attrs, result,
+                               DEVICE_BIG_ENDIAN);
 }
 
 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
 {
-    stl_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
+    address_space_stl(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
 {
-    stl_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
+    address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
 {
-    stl_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
+    address_space_stl_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* XXX: optimize */
-void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
+void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
 {
     uint8_t v = val;
-    address_space_rw(as, addr, &v, 1, 1);
+    MemTxResult r;
+
+    r = address_space_rw(as, addr, attrs, &v, 1, 1);
+    if (result) {
+        *result = r;
+    }
+}
+
+void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
+{
+    address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* warning: addr must be aligned */
-static inline void stw_phys_internal(AddressSpace *as,
-                                     hwaddr addr, uint32_t val,
-                                     enum device_endian endian)
+static inline void address_space_stw_internal(AddressSpace *as,
+                                              hwaddr addr, uint32_t val,
+                                              MemTxAttrs attrs,
+                                              MemTxResult *result,
+                                              enum device_endian endian)
 {
     uint8_t *ptr;
     MemoryRegion *mr;
     hwaddr l = 2;
     hwaddr addr1;
+    MemTxResult r;
+    bool release_lock = false;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr1, &l, true);
     if (l < 2 || !memory_access_is_direct(mr, true)) {
+        release_lock |= prepare_mmio_access(mr);
+
 #if defined(TARGET_WORDS_BIGENDIAN)
         if (endian == DEVICE_LITTLE_ENDIAN) {
             val = bswap16(val);
@@ -2797,11 +3505,10 @@
             val = bswap16(val);
         }
 #endif
-        io_mem_write(mr, addr1, val, 2);
+        r = memory_region_dispatch_write(mr, addr1, val, 2, attrs);
     } else {
         /* RAM case */
-        addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
-        ptr = qemu_get_ram_ptr(addr1);
+        ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
         switch (endian) {
         case DEVICE_LITTLE_ENDIAN:
             stw_le_p(ptr, val);
@@ -2813,42 +3520,100 @@
             stw_p(ptr, val);
             break;
         }
-        invalidate_and_set_dirty(addr1, 2);
+        invalidate_and_set_dirty(mr, addr1, 2);
+        r = MEMTX_OK;
     }
+    if (result) {
+        *result = r;
+    }
+    if (release_lock) {
+        qemu_mutex_unlock_iothread();
+    }
+    rcu_read_unlock();
+}
+
+void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    address_space_stw_internal(as, addr, val, attrs, result,
+                               DEVICE_NATIVE_ENDIAN);
+}
+
+void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    address_space_stw_internal(as, addr, val, attrs, result,
+                               DEVICE_LITTLE_ENDIAN);
+}
+
+void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    address_space_stw_internal(as, addr, val, attrs, result,
+                               DEVICE_BIG_ENDIAN);
 }
 
 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
 {
-    stw_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
+    address_space_stw(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
 {
-    stw_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
+    address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
 {
-    stw_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
+    address_space_stw_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* XXX: optimize */
+void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    MemTxResult r;
+    val = tswap64(val);
+    r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
+    if (result) {
+        *result = r;
+    }
+}
+
+void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    MemTxResult r;
+    val = cpu_to_le64(val);
+    r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
+    if (result) {
+        *result = r;
+    }
+}
+void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
+                       MemTxAttrs attrs, MemTxResult *result)
+{
+    MemTxResult r;
+    val = cpu_to_be64(val);
+    r = address_space_rw(as, addr, attrs, (void *) &val, 8, 1);
+    if (result) {
+        *result = r;
+    }
+}
+
 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
 {
-    val = tswap64(val);
-    address_space_rw(as, addr, (void *) &val, 8, 1);
+    address_space_stq(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
 {
-    val = cpu_to_le64(val);
-    address_space_rw(as, addr, (void *) &val, 8, 1);
+    address_space_stq_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
 {
-    val = cpu_to_be64(val);
-    address_space_rw(as, addr, (void *) &val, 8, 1);
+    address_space_stq_be(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* virtual memory access for debug (includes writing to ROM) */
@@ -2860,8 +3625,12 @@
     target_ulong page;
 
     while (len > 0) {
+        int asidx;
+        MemTxAttrs attrs;
+
         page = addr & TARGET_PAGE_MASK;
-        phys_addr = cpu_get_phys_page_debug(cpu, page);
+        phys_addr = cpu_get_phys_page_attrs_debug(cpu, page, &attrs);
+        asidx = cpu_asidx_from_attrs(cpu, attrs);
         /* if no physical page mapped, return an error */
         if (phys_addr == -1)
             return -1;
@@ -2870,9 +3639,12 @@
             l = len;
         phys_addr += (addr & ~TARGET_PAGE_MASK);
         if (is_write) {
-            cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
+            cpu_physical_memory_write_rom(cpu->cpu_ases[asidx].as,
+                                          phys_addr, buf, l);
         } else {
-            address_space_rw(cpu->as, phys_addr, buf, l, 0);
+            address_space_rw(cpu->cpu_ases[asidx].as, phys_addr,
+                             MEMTXATTRS_UNSPECIFIED,
+                             buf, l, 0);
         }
         len -= l;
         buf += l;
@@ -2880,6 +3652,16 @@
     }
     return 0;
 }
+
+/*
+ * Allows code that needs to deal with migration bitmaps etc to still be built
+ * target independent.
+ */
+size_t qemu_target_page_bits(void)
+{
+    return TARGET_PAGE_BITS;
+}
+
 #endif
 
 /*
@@ -2901,20 +3683,31 @@
 {
     MemoryRegion*mr;
     hwaddr l = 1;
+    bool res;
 
+    rcu_read_lock();
     mr = address_space_translate(&address_space_memory,
                                  phys_addr, &phys_addr, &l, false);
 
-    return !(memory_region_is_ram(mr) ||
-             memory_region_is_romd(mr));
+    res = !(memory_region_is_ram(mr) || memory_region_is_romd(mr));
+    rcu_read_unlock();
+    return res;
 }
 
-void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
+int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
 {
     RAMBlock *block;
+    int ret = 0;
 
-    QTAILQ_FOREACH(block, &ram_list.blocks, next) {
-        func(block->host, block->offset, block->length, opaque);
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        ret = func(block->idstr, block->host, block->offset,
+                   block->used_length, opaque);
+        if (ret) {
+            break;
+        }
     }
+    rcu_read_unlock();
+    return ret;
 }
 #endif
diff --git a/fpu/softfloat-macros.h b/fpu/softfloat-macros.h
index 0dcda93..9cc6158 100644
--- a/fpu/softfloat-macros.h
+++ b/fpu/softfloat-macros.h
@@ -1,13 +1,24 @@
 /*
  * QEMU float support macros
  *
- * Derived from SoftFloat.
+ * The code in this source file is derived from release 2a of the SoftFloat
+ * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and
+ * some later contributions) are provided under that license, as detailed below.
+ * It has subsequently been modified by contributors to the QEMU Project,
+ * so some portions are provided under:
+ *  the SoftFloat-2a license
+ *  the BSD license
+ *  GPL-v2-or-later
+ *
+ * Any future contributions to this file after December 1st 2014 will be
+ * taken to be licensed under the Softfloat-2a license unless specifically
+ * indicated otherwise.
  */
 
-/*============================================================================
-
+/*
+===============================================================================
 This C source fragment is part of the SoftFloat IEC/IEEE Floating-point
-Arithmetic Package, Release 2b.
+Arithmetic Package, Release 2a.
 
 Written by John R. Hauser.  This work was made possible in part by the
 International Computer Science Institute, located at Suite 600, 1947 Center
@@ -16,24 +27,57 @@
 of this code was written as part of a project to build a fixed-point vector
 processor in collaboration with the University of California at Berkeley,
 overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
-is available through the Web page `http://www.cs.berkeley.edu/~jhauser/
+is available through the Web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
 arithmetic/SoftFloat.html'.
 
-THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort has
-been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT TIMES
-RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO PERSONS
-AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ALL LOSSES,
-COSTS, OR OTHER PROBLEMS THEY INCUR DUE TO THE SOFTWARE, AND WHO FURTHERMORE
-EFFECTIVELY INDEMNIFY JOHN HAUSER AND THE INTERNATIONAL COMPUTER SCIENCE
-INSTITUTE (possibly via similar legal notice) AGAINST ALL LOSSES, COSTS, OR
-OTHER PROBLEMS INCURRED BY THEIR CUSTOMERS AND CLIENTS DUE TO THE SOFTWARE.
+THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort
+has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
+TIMES RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO
+PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
+AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
 
 Derivative works are acceptable, even for commercial purposes, so long as
-(1) the source code for the derivative work includes prominent notice that
-the work is derivative, and (2) the source code includes prominent notice with
-these four paragraphs for those parts of this code that are retained.
+(1) they include prominent notice that the work is derivative, and (2) they
+include prominent notice akin to these four paragraphs for those parts of
+this code that are retained.
 
-=============================================================================*/
+===============================================================================
+*/
+
+/* BSD licensing:
+ * Copyright (c) 2006, Fabrice Bellard
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Portions of this work are licensed under the terms of the GNU GPL,
+ * version 2 or later. See the COPYING file in the top-level directory.
+ */
 
 /*----------------------------------------------------------------------------
 | This macro tests for minimum version of the GNU C compiler.
@@ -55,7 +99,7 @@
 | The result is stored in the location pointed to by `zPtr'.
 *----------------------------------------------------------------------------*/
 
-static inline void shift32RightJamming(uint32_t a, int_fast16_t count, uint32_t *zPtr)
+static inline void shift32RightJamming(uint32_t a, int count, uint32_t *zPtr)
 {
     uint32_t z;
 
@@ -81,7 +125,7 @@
 | The result is stored in the location pointed to by `zPtr'.
 *----------------------------------------------------------------------------*/
 
-static inline void shift64RightJamming(uint64_t a, int_fast16_t count, uint64_t *zPtr)
+static inline void shift64RightJamming(uint64_t a, int count, uint64_t *zPtr)
 {
     uint64_t z;
 
@@ -107,20 +151,20 @@
 | 63 bits of the extra result are all zero if and only if _all_but_the_last_
 | bits shifted off were all zero.  This extra result is stored in the location
 | pointed to by `z1Ptr'.  The value of `count' can be arbitrarily large.
-|     (This routine makes more sense if `a0' and `a1' are considered to form
-| a fixed-point value with binary point between `a0' and `a1'.  This fixed-
-| point value is shifted right by the number of bits given in `count', and
-| the integer part of the result is returned at the location pointed to by
+|     (This routine makes more sense if `a0' and `a1' are considered to form a
+| fixed-point value with binary point between `a0' and `a1'.  This fixed-point
+| value is shifted right by the number of bits given in `count', and the
+| integer part of the result is returned at the location pointed to by
 | `z0Ptr'.  The fractional part of the result may be slightly corrupted as
 | described above, and is returned at the location pointed to by `z1Ptr'.)
 *----------------------------------------------------------------------------*/
 
 static inline void
  shift64ExtraRightJamming(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
     uint64_t z0, z1;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z1 = a1;
@@ -154,10 +198,10 @@
 
 static inline void
  shift128Right(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
     uint64_t z0, z1;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z1 = a1;
@@ -189,10 +233,10 @@
 
 static inline void
  shift128RightJamming(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
     uint64_t z0, z1;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z1 = a1;
@@ -243,14 +287,14 @@
      uint64_t a0,
      uint64_t a1,
      uint64_t a2,
-     int_fast16_t count,
+     int count,
      uint64_t *z0Ptr,
      uint64_t *z1Ptr,
      uint64_t *z2Ptr
  )
 {
     uint64_t z0, z1, z2;
-    int8 negCount = ( - count ) & 63;
+    int8_t negCount = ( - count ) & 63;
 
     if ( count == 0 ) {
         z2 = a2;
@@ -298,7 +342,7 @@
 
 static inline void
  shortShift128Left(
-     uint64_t a0, uint64_t a1, int_fast16_t count, uint64_t *z0Ptr, uint64_t *z1Ptr)
+     uint64_t a0, uint64_t a1, int count, uint64_t *z0Ptr, uint64_t *z1Ptr)
 {
 
     *z1Ptr = a1<<count;
@@ -320,14 +364,14 @@
      uint64_t a0,
      uint64_t a1,
      uint64_t a2,
-     int_fast16_t count,
+     int count,
      uint64_t *z0Ptr,
      uint64_t *z1Ptr,
      uint64_t *z2Ptr
  )
 {
     uint64_t z0, z1, z2;
-    int8 negCount;
+    int8_t negCount;
 
     z2 = a2<<count;
     z1 = a1<<count;
@@ -384,7 +428,7 @@
  )
 {
     uint64_t z0, z1, z2;
-    int8 carry0, carry1;
+    int8_t carry0, carry1;
 
     z2 = a2 + b2;
     carry1 = ( z2 < a2 );
@@ -440,7 +484,7 @@
  )
 {
     uint64_t z0, z1, z2;
-    int8 borrow0, borrow1;
+    int8_t borrow0, borrow1;
 
     z2 = a2 - b2;
     borrow1 = ( a2 < b2 );
@@ -591,7 +635,7 @@
 | value.
 *----------------------------------------------------------------------------*/
 
-static uint32_t estimateSqrt32(int_fast16_t aExp, uint32_t a)
+static uint32_t estimateSqrt32(int aExp, uint32_t a)
 {
     static const uint16_t sqrtOddAdjustments[] = {
         0x0004, 0x0022, 0x005D, 0x00B1, 0x011D, 0x019F, 0x0236, 0x02E0,
@@ -601,7 +645,7 @@
         0x0A2D, 0x08AF, 0x075A, 0x0629, 0x051A, 0x0429, 0x0356, 0x029E,
         0x0200, 0x0179, 0x0109, 0x00AF, 0x0068, 0x0034, 0x0012, 0x0002
     };
-    int8 index;
+    int8_t index;
     uint32_t z;
 
     index = ( a>>27 ) & 15;
@@ -625,7 +669,7 @@
 | `a'.  If `a' is zero, 32 is returned.
 *----------------------------------------------------------------------------*/
 
-static int8 countLeadingZeros32( uint32_t a )
+static int8_t countLeadingZeros32( uint32_t a )
 {
 #if SOFTFLOAT_GNUC_PREREQ(3, 4)
     if (a) {
@@ -634,7 +678,7 @@
         return 32;
     }
 #else
-    static const int8 countLeadingZerosHigh[] = {
+    static const int8_t countLeadingZerosHigh[] = {
         8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,
         3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
@@ -652,7 +696,7 @@
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
     };
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = 0;
     if ( a < 0x10000 ) {
@@ -673,7 +717,7 @@
 | `a'.  If `a' is zero, 64 is returned.
 *----------------------------------------------------------------------------*/
 
-static int8 countLeadingZeros64( uint64_t a )
+static int8_t countLeadingZeros64( uint64_t a )
 {
 #if SOFTFLOAT_GNUC_PREREQ(3, 4)
     if (a) {
@@ -682,7 +726,7 @@
         return 64;
     }
 #else
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = 0;
     if ( a < ( (uint64_t) 1 )<<32 ) {
diff --git a/fpu/softfloat-specialize.h b/fpu/softfloat-specialize.h
index 518f694..f5aed72 100644
--- a/fpu/softfloat-specialize.h
+++ b/fpu/softfloat-specialize.h
@@ -1,13 +1,24 @@
 /*
  * QEMU float support
  *
- * Derived from SoftFloat.
+ * The code in this source file is derived from release 2a of the SoftFloat
+ * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and
+ * some later contributions) are provided under that license, as detailed below.
+ * It has subsequently been modified by contributors to the QEMU Project,
+ * so some portions are provided under:
+ *  the SoftFloat-2a license
+ *  the BSD license
+ *  GPL-v2-or-later
+ *
+ * Any future contributions to this file after December 1st 2014 will be
+ * taken to be licensed under the Softfloat-2a license unless specifically
+ * indicated otherwise.
  */
 
-/*============================================================================
-
+/*
+===============================================================================
 This C source fragment is part of the SoftFloat IEC/IEEE Floating-point
-Arithmetic Package, Release 2b.
+Arithmetic Package, Release 2a.
 
 Written by John R. Hauser.  This work was made possible in part by the
 International Computer Science Institute, located at Suite 600, 1947 Center
@@ -16,30 +27,57 @@
 of this code was written as part of a project to build a fixed-point vector
 processor in collaboration with the University of California at Berkeley,
 overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
-is available through the Web page `http://www.cs.berkeley.edu/~jhauser/
+is available through the Web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
 arithmetic/SoftFloat.html'.
 
-THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort has
-been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT TIMES
-RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO PERSONS
-AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ALL LOSSES,
-COSTS, OR OTHER PROBLEMS THEY INCUR DUE TO THE SOFTWARE, AND WHO FURTHERMORE
-EFFECTIVELY INDEMNIFY JOHN HAUSER AND THE INTERNATIONAL COMPUTER SCIENCE
-INSTITUTE (possibly via similar legal warning) AGAINST ALL LOSSES, COSTS, OR
-OTHER PROBLEMS INCURRED BY THEIR CUSTOMERS AND CLIENTS DUE TO THE SOFTWARE.
+THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort
+has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
+TIMES RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO
+PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
+AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
 
 Derivative works are acceptable, even for commercial purposes, so long as
-(1) the source code for the derivative work includes prominent notice that
-the work is derivative, and (2) the source code includes prominent notice with
-these four paragraphs for those parts of this code that are retained.
+(1) they include prominent notice that the work is derivative, and (2) they
+include prominent notice akin to these four paragraphs for those parts of
+this code that are retained.
 
-=============================================================================*/
+===============================================================================
+*/
 
-#if defined(TARGET_MIPS) || defined(TARGET_SH4) || defined(TARGET_UNICORE32)
-#define SNAN_BIT_IS_ONE		1
-#else
-#define SNAN_BIT_IS_ONE		0
-#endif
+/* BSD licensing:
+ * Copyright (c) 2006, Fabrice Bellard
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Portions of this work are licensed under the terms of the GNU GPL,
+ * version 2 or later. See the COPYING file in the top-level directory.
+ */
 
 #if defined(TARGET_XTENSA)
 /* Define for architectures which deviate from IEEE in not supporting
@@ -51,69 +89,106 @@
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
+float16 float16_default_nan(float_status *status)
+{
 #if defined(TARGET_ARM)
-const float16 float16_default_nan = const_float16(0x7E00);
-#elif SNAN_BIT_IS_ONE
-const float16 float16_default_nan = const_float16(0x7DFF);
+    return const_float16(0x7E00);
 #else
-const float16 float16_default_nan = const_float16(0xFE00);
+    if (status->snan_bit_is_one) {
+        return const_float16(0x7DFF);
+    } else {
+#if defined(TARGET_MIPS)
+        return const_float16(0x7E00);
+#else
+        return const_float16(0xFE00);
 #endif
+    }
+#endif
+}
 
 /*----------------------------------------------------------------------------
 | The pattern for a default generated single-precision NaN.
 *----------------------------------------------------------------------------*/
+float32 float32_default_nan(float_status *status)
+{
 #if defined(TARGET_SPARC)
-const float32 float32_default_nan = const_float32(0x7FFFFFFF);
+    return const_float32(0x7FFFFFFF);
 #elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA) || \
-      defined(TARGET_XTENSA)
-const float32 float32_default_nan = const_float32(0x7FC00000);
-#elif SNAN_BIT_IS_ONE
-const float32 float32_default_nan = const_float32(0x7FBFFFFF);
+      defined(TARGET_XTENSA) || defined(TARGET_S390X) || defined(TARGET_TRICORE)
+    return const_float32(0x7FC00000);
 #else
-const float32 float32_default_nan = const_float32(0xFFC00000);
+    if (status->snan_bit_is_one) {
+        return const_float32(0x7FBFFFFF);
+    } else {
+#if defined(TARGET_MIPS)
+        return const_float32(0x7FC00000);
+#else
+        return const_float32(0xFFC00000);
 #endif
+    }
+#endif
+}
 
 /*----------------------------------------------------------------------------
 | The pattern for a default generated double-precision NaN.
 *----------------------------------------------------------------------------*/
+float64 float64_default_nan(float_status *status)
+{
 #if defined(TARGET_SPARC)
-const float64 float64_default_nan = const_float64(LIT64( 0x7FFFFFFFFFFFFFFF ));
-#elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA)
-const float64 float64_default_nan = const_float64(LIT64( 0x7FF8000000000000 ));
-#elif SNAN_BIT_IS_ONE
-const float64 float64_default_nan = const_float64(LIT64( 0x7FF7FFFFFFFFFFFF ));
+    return const_float64(LIT64(0x7FFFFFFFFFFFFFFF));
+#elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA) || \
+      defined(TARGET_S390X)
+    return const_float64(LIT64(0x7FF8000000000000));
 #else
-const float64 float64_default_nan = const_float64(LIT64( 0xFFF8000000000000 ));
+    if (status->snan_bit_is_one) {
+        return const_float64(LIT64(0x7FF7FFFFFFFFFFFF));
+    } else {
+#if defined(TARGET_MIPS)
+        return const_float64(LIT64(0x7FF8000000000000));
+#else
+        return const_float64(LIT64(0xFFF8000000000000));
 #endif
+    }
+#endif
+}
 
 /*----------------------------------------------------------------------------
 | The pattern for a default generated extended double-precision NaN.
 *----------------------------------------------------------------------------*/
-#if SNAN_BIT_IS_ONE
-#define floatx80_default_nan_high 0x7FFF
-#define floatx80_default_nan_low  LIT64( 0xBFFFFFFFFFFFFFFF )
-#else
-#define floatx80_default_nan_high 0xFFFF
-#define floatx80_default_nan_low  LIT64( 0xC000000000000000 )
-#endif
+floatx80 floatx80_default_nan(float_status *status)
+{
+    floatx80 r;
 
-const floatx80 floatx80_default_nan
-    = make_floatx80_init(floatx80_default_nan_high, floatx80_default_nan_low);
+    if (status->snan_bit_is_one) {
+        r.low = LIT64(0xBFFFFFFFFFFFFFFF);
+        r.high = 0x7FFF;
+    } else {
+        r.low = LIT64(0xC000000000000000);
+        r.high = 0xFFFF;
+    }
+    return r;
+}
 
 /*----------------------------------------------------------------------------
-| The pattern for a default generated quadruple-precision NaN.  The `high' and
-| `low' values hold the most- and least-significant bits, respectively.
+| The pattern for a default generated quadruple-precision NaN.
 *----------------------------------------------------------------------------*/
-#if SNAN_BIT_IS_ONE
-#define float128_default_nan_high LIT64( 0x7FFF7FFFFFFFFFFF )
-#define float128_default_nan_low  LIT64( 0xFFFFFFFFFFFFFFFF )
-#else
-#define float128_default_nan_high LIT64( 0xFFFF800000000000 )
-#define float128_default_nan_low  LIT64( 0x0000000000000000 )
-#endif
+float128 float128_default_nan(float_status *status)
+{
+    float128 r;
 
-const float128 float128_default_nan
-    = make_float128_init(float128_default_nan_high, float128_default_nan_low);
+    if (status->snan_bit_is_one) {
+        r.low = LIT64(0xFFFFFFFFFFFFFFFF);
+        r.high = LIT64(0x7FFF7FFFFFFFFFFF);
+    } else {
+        r.low = LIT64(0x0000000000000000);
+#if defined(TARGET_S390X)
+        r.high = LIT64(0x7FFF800000000000);
+#else
+        r.high = LIT64(0xFFFF800000000000);
+#endif
+    }
+    return r;
+}
 
 /*----------------------------------------------------------------------------
 | Raises the exceptions specified by `flags'.  Floating-point traps can be
@@ -122,9 +197,9 @@
 | should be simply `float_exception_flags |= flags;'.
 *----------------------------------------------------------------------------*/
 
-void float_raise( int8 flags STATUS_PARAM )
+void float_raise(uint8_t flags, float_status *status)
 {
-    STATUS(float_exception_flags) |= flags;
+    status->float_exception_flags |= flags;
 }
 
 /*----------------------------------------------------------------------------
@@ -136,12 +211,12 @@
 } commonNaNT;
 
 #ifdef NO_SIGNALING_NANS
-int float16_is_quiet_nan(float16 a_)
+int float16_is_quiet_nan(float16 a_, float_status *status)
 {
     return float16_is_any_nan(a_);
 }
 
-int float16_is_signaling_nan(float16 a_)
+int float16_is_signaling_nan(float16 a_, float_status *status)
 {
     return 0;
 }
@@ -151,14 +226,14 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float16_is_quiet_nan(float16 a_)
+int float16_is_quiet_nan(float16 a_, float_status *status)
 {
     uint16_t a = float16_val(a_);
-#if SNAN_BIT_IS_ONE
-    return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
-#else
-    return ((a & ~0x8000) >= 0x7c80);
-#endif
+    if (status->snan_bit_is_one) {
+        return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
+    } else {
+        return ((a & ~0x8000) >= 0x7C80);
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -166,14 +241,14 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float16_is_signaling_nan(float16 a_)
+int float16_is_signaling_nan(float16 a_, float_status *status)
 {
     uint16_t a = float16_val(a_);
-#if SNAN_BIT_IS_ONE
-    return ((a & ~0x8000) >= 0x7c80);
-#else
-    return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
-#endif
+    if (status->snan_bit_is_one) {
+        return ((a & ~0x8000) >= 0x7C80);
+    } else {
+        return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
+    }
 }
 #endif
 
@@ -181,20 +256,16 @@
 | Returns a quiet NaN if the half-precision floating point value `a' is a
 | signaling NaN; otherwise returns `a'.
 *----------------------------------------------------------------------------*/
-float16 float16_maybe_silence_nan(float16 a_)
+float16 float16_maybe_silence_nan(float16 a_, float_status *status)
 {
-    if (float16_is_signaling_nan(a_)) {
-#if SNAN_BIT_IS_ONE
-#  if defined(TARGET_MIPS) || defined(TARGET_SH4) || defined(TARGET_UNICORE32)
-        return float16_default_nan;
-#  else
-#    error Rules for silencing a signaling NaN are target-specific
-#  endif
-#else
-        uint16_t a = float16_val(a_);
-        a |= (1 << 9);
-        return make_float16(a);
-#endif
+    if (float16_is_signaling_nan(a_, status)) {
+        if (status->snan_bit_is_one) {
+            return float16_default_nan(status);
+        } else {
+            uint16_t a = float16_val(a_);
+            a |= (1 << 9);
+            return make_float16(a);
+        }
     }
     return a_;
 }
@@ -205,14 +276,16 @@
 | exception is raised.
 *----------------------------------------------------------------------------*/
 
-static commonNaNT float16ToCommonNaN( float16 a STATUS_PARAM )
+static commonNaNT float16ToCommonNaN(float16 a, float_status *status)
 {
     commonNaNT z;
 
-    if ( float16_is_signaling_nan( a ) ) float_raise( float_flag_invalid STATUS_VAR );
+    if (float16_is_signaling_nan(a, status)) {
+        float_raise(float_flag_invalid, status);
+    }
     z.sign = float16_val(a) >> 15;
     z.low = 0;
-    z.high = ((uint64_t) float16_val(a))<<54;
+    z.high = ((uint64_t) float16_val(a)) << 54;
     return z;
 }
 
@@ -221,29 +294,29 @@
 | precision floating-point format.
 *----------------------------------------------------------------------------*/
 
-static float16 commonNaNToFloat16(commonNaNT a STATUS_PARAM)
+static float16 commonNaNToFloat16(commonNaNT a, float_status *status)
 {
-    uint16_t mantissa = a.high>>54;
+    uint16_t mantissa = a.high >> 54;
 
-    if (STATUS(default_nan_mode)) {
-        return float16_default_nan;
+    if (status->default_nan_mode) {
+        return float16_default_nan(status);
     }
 
     if (mantissa) {
         return make_float16(((((uint16_t) a.sign) << 15)
                              | (0x1F << 10) | mantissa));
     } else {
-        return float16_default_nan;
+        return float16_default_nan(status);
     }
 }
 
 #ifdef NO_SIGNALING_NANS
-int float32_is_quiet_nan(float32 a_)
+int float32_is_quiet_nan(float32 a_, float_status *status)
 {
     return float32_is_any_nan(a_);
 }
 
-int float32_is_signaling_nan(float32 a_)
+int float32_is_signaling_nan(float32 a_, float_status *status)
 {
     return 0;
 }
@@ -253,14 +326,14 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float32_is_quiet_nan( float32 a_ )
+int float32_is_quiet_nan(float32 a_, float_status *status)
 {
     uint32_t a = float32_val(a_);
-#if SNAN_BIT_IS_ONE
-    return ( ( ( a>>22 ) & 0x1FF ) == 0x1FE ) && ( a & 0x003FFFFF );
-#else
-    return ( 0xFF800000 <= (uint32_t) ( a<<1 ) );
-#endif
+    if (status->snan_bit_is_one) {
+        return (((a >> 22) & 0x1FF) == 0x1FE) && (a & 0x003FFFFF);
+    } else {
+        return ((uint32_t)(a << 1) >= 0xFF800000);
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -268,14 +341,14 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float32_is_signaling_nan( float32 a_ )
+int float32_is_signaling_nan(float32 a_, float_status *status)
 {
     uint32_t a = float32_val(a_);
-#if SNAN_BIT_IS_ONE
-    return ( 0xFF800000 <= (uint32_t) ( a<<1 ) );
-#else
-    return ( ( ( a>>22 ) & 0x1FF ) == 0x1FE ) && ( a & 0x003FFFFF );
-#endif
+    if (status->snan_bit_is_one) {
+        return ((uint32_t)(a << 1) >= 0xFF800000);
+    } else {
+        return (((a >> 22) & 0x1FF) == 0x1FE) && (a & 0x003FFFFF);
+    }
 }
 #endif
 
@@ -284,20 +357,16 @@
 | signaling NaN; otherwise returns `a'.
 *----------------------------------------------------------------------------*/
 
-float32 float32_maybe_silence_nan( float32 a_ )
+float32 float32_maybe_silence_nan(float32 a_, float_status *status)
 {
-    if (float32_is_signaling_nan(a_)) {
-#if SNAN_BIT_IS_ONE
-#  if defined(TARGET_MIPS) || defined(TARGET_SH4) || defined(TARGET_UNICORE32)
-        return float32_default_nan;
-#  else
-#    error Rules for silencing a signaling NaN are target-specific
-#  endif
-#else
-        uint32_t a = float32_val(a_);
-        a |= (1 << 22);
-        return make_float32(a);
-#endif
+    if (float32_is_signaling_nan(a_, status)) {
+        if (status->snan_bit_is_one) {
+            return float32_default_nan(status);
+        } else {
+            uint32_t a = float32_val(a_);
+            a |= (1 << 22);
+            return make_float32(a);
+        }
     }
     return a_;
 }
@@ -308,14 +377,16 @@
 | exception is raised.
 *----------------------------------------------------------------------------*/
 
-static commonNaNT float32ToCommonNaN( float32 a STATUS_PARAM )
+static commonNaNT float32ToCommonNaN(float32 a, float_status *status)
 {
     commonNaNT z;
 
-    if ( float32_is_signaling_nan( a ) ) float_raise( float_flag_invalid STATUS_VAR );
-    z.sign = float32_val(a)>>31;
+    if (float32_is_signaling_nan(a, status)) {
+        float_raise(float_flag_invalid, status);
+    }
+    z.sign = float32_val(a) >> 31;
     z.low = 0;
-    z.high = ( (uint64_t) float32_val(a) )<<41;
+    z.high = ((uint64_t)float32_val(a)) << 41;
     return z;
 }
 
@@ -324,19 +395,20 @@
 | precision floating-point format.
 *----------------------------------------------------------------------------*/
 
-static float32 commonNaNToFloat32( commonNaNT a STATUS_PARAM)
+static float32 commonNaNToFloat32(commonNaNT a, float_status *status)
 {
-    uint32_t mantissa = a.high>>41;
+    uint32_t mantissa = a.high >> 41;
 
-    if ( STATUS(default_nan_mode) ) {
-        return float32_default_nan;
+    if (status->default_nan_mode) {
+        return float32_default_nan(status);
     }
 
-    if ( mantissa )
+    if (mantissa) {
         return make_float32(
-            ( ( (uint32_t) a.sign )<<31 ) | 0x7F800000 | ( a.high>>41 ) );
-    else
-        return float32_default_nan;
+            (((uint32_t)a.sign) << 31) | 0x7F800000 | (a.high >> 41));
+    } else {
+        return float32_default_nan(status);
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -438,11 +510,10 @@
             return aIsLargerSignificand ? 0 : 1;
         }
         return bIsQNaN ? 1 : 0;
-    }
-    else if (aIsQNaN) {
-        if (bIsSNaN || !bIsQNaN)
+    } else if (aIsQNaN) {
+        if (bIsSNaN || !bIsQNaN) {
             return 0;
-        else {
+        } else {
             return aIsLargerSignificand ? 0 : 1;
         }
     } else {
@@ -459,13 +530,14 @@
 *----------------------------------------------------------------------------*/
 #if defined(TARGET_ARM)
 static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
-                         flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
+                         flag cIsQNaN, flag cIsSNaN, flag infzero,
+                         float_status *status)
 {
     /* For ARM, the (inf,zero,qnan) case sets InvalidOp and returns
      * the default NaN
      */
     if (infzero && cIsQNaN) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 3;
     }
 
@@ -488,41 +560,60 @@
 }
 #elif defined(TARGET_MIPS)
 static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
-                         flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
+                         flag cIsQNaN, flag cIsSNaN, flag infzero,
+                         float_status *status)
 {
     /* For MIPS, the (inf,zero,qnan) case sets InvalidOp and returns
      * the default NaN
      */
     if (infzero) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 3;
     }
 
-    /* Prefer sNaN over qNaN, in the a, b, c order. */
-    if (aIsSNaN) {
-        return 0;
-    } else if (bIsSNaN) {
-        return 1;
-    } else if (cIsSNaN) {
-        return 2;
-    } else if (aIsQNaN) {
-        return 0;
-    } else if (bIsQNaN) {
-        return 1;
+    if (status->snan_bit_is_one) {
+        /* Prefer sNaN over qNaN, in the a, b, c order. */
+        if (aIsSNaN) {
+            return 0;
+        } else if (bIsSNaN) {
+            return 1;
+        } else if (cIsSNaN) {
+            return 2;
+        } else if (aIsQNaN) {
+            return 0;
+        } else if (bIsQNaN) {
+            return 1;
+        } else {
+            return 2;
+        }
     } else {
-        return 2;
+        /* Prefer sNaN over qNaN, in the c, a, b order. */
+        if (cIsSNaN) {
+            return 2;
+        } else if (aIsSNaN) {
+            return 0;
+        } else if (bIsSNaN) {
+            return 1;
+        } else if (cIsQNaN) {
+            return 2;
+        } else if (aIsQNaN) {
+            return 0;
+        } else {
+            return 1;
+        }
     }
 }
 #elif defined(TARGET_PPC)
 static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
-                         flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
+                         flag cIsQNaN, flag cIsSNaN, flag infzero,
+                         float_status *status)
 {
     /* For PPC, the (inf,zero,qnan) case sets InvalidOp, but we prefer
      * to return an input NaN if we have one (ie c) rather than generating
      * a default NaN
      */
     if (infzero) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 2;
     }
 
@@ -542,7 +633,8 @@
  * This is unlikely to actually match any real implementation.
  */
 static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
-                         flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
+                         flag cIsQNaN, flag cIsSNaN, flag infzero,
+                         float_status *status)
 {
     if (aIsSNaN || aIsQNaN) {
         return 0;
@@ -560,27 +652,30 @@
 | signaling NaN, the invalid exception is raised.
 *----------------------------------------------------------------------------*/
 
-static float32 propagateFloat32NaN( float32 a, float32 b STATUS_PARAM)
+static float32 propagateFloat32NaN(float32 a, float32 b, float_status *status)
 {
     flag aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN;
     flag aIsLargerSignificand;
     uint32_t av, bv;
 
-    aIsQuietNaN = float32_is_quiet_nan( a );
-    aIsSignalingNaN = float32_is_signaling_nan( a );
-    bIsQuietNaN = float32_is_quiet_nan( b );
-    bIsSignalingNaN = float32_is_signaling_nan( b );
+    aIsQuietNaN = float32_is_quiet_nan(a, status);
+    aIsSignalingNaN = float32_is_signaling_nan(a, status);
+    bIsQuietNaN = float32_is_quiet_nan(b, status);
+    bIsSignalingNaN = float32_is_signaling_nan(b, status);
     av = float32_val(a);
     bv = float32_val(b);
 
-    if ( aIsSignalingNaN | bIsSignalingNaN ) float_raise( float_flag_invalid STATUS_VAR);
+    if (aIsSignalingNaN | bIsSignalingNaN) {
+        float_raise(float_flag_invalid, status);
+    }
 
-    if ( STATUS(default_nan_mode) )
-        return float32_default_nan;
+    if (status->default_nan_mode) {
+        return float32_default_nan(status);
+    }
 
-    if ((uint32_t)(av<<1) < (uint32_t)(bv<<1)) {
+    if ((uint32_t)(av << 1) < (uint32_t)(bv << 1)) {
         aIsLargerSignificand = 0;
-    } else if ((uint32_t)(bv<<1) < (uint32_t)(av<<1)) {
+    } else if ((uint32_t)(bv << 1) < (uint32_t)(av << 1)) {
         aIsLargerSignificand = 1;
     } else {
         aIsLargerSignificand = (av < bv) ? 1 : 0;
@@ -588,9 +683,9 @@
 
     if (pickNaN(aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN,
                 aIsLargerSignificand)) {
-        return float32_maybe_silence_nan(b);
+        return float32_maybe_silence_nan(b, status);
     } else {
-        return float32_maybe_silence_nan(a);
+        return float32_maybe_silence_nan(a, status);
     }
 }
 
@@ -604,54 +699,55 @@
 *----------------------------------------------------------------------------*/
 
 static float32 propagateFloat32MulAddNaN(float32 a, float32 b,
-                                         float32 c, flag infzero STATUS_PARAM)
+                                         float32 c, flag infzero,
+                                         float_status *status)
 {
     flag aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN,
         cIsQuietNaN, cIsSignalingNaN;
     int which;
 
-    aIsQuietNaN = float32_is_quiet_nan(a);
-    aIsSignalingNaN = float32_is_signaling_nan(a);
-    bIsQuietNaN = float32_is_quiet_nan(b);
-    bIsSignalingNaN = float32_is_signaling_nan(b);
-    cIsQuietNaN = float32_is_quiet_nan(c);
-    cIsSignalingNaN = float32_is_signaling_nan(c);
+    aIsQuietNaN = float32_is_quiet_nan(a, status);
+    aIsSignalingNaN = float32_is_signaling_nan(a, status);
+    bIsQuietNaN = float32_is_quiet_nan(b, status);
+    bIsSignalingNaN = float32_is_signaling_nan(b, status);
+    cIsQuietNaN = float32_is_quiet_nan(c, status);
+    cIsSignalingNaN = float32_is_signaling_nan(c, status);
 
     if (aIsSignalingNaN | bIsSignalingNaN | cIsSignalingNaN) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
     }
 
     which = pickNaNMulAdd(aIsQuietNaN, aIsSignalingNaN,
                           bIsQuietNaN, bIsSignalingNaN,
-                          cIsQuietNaN, cIsSignalingNaN, infzero STATUS_VAR);
+                          cIsQuietNaN, cIsSignalingNaN, infzero, status);
 
-    if (STATUS(default_nan_mode)) {
+    if (status->default_nan_mode) {
         /* Note that this check is after pickNaNMulAdd so that function
          * has an opportunity to set the Invalid flag.
          */
-        return float32_default_nan;
+        return float32_default_nan(status);
     }
 
     switch (which) {
     case 0:
-        return float32_maybe_silence_nan(a);
+        return float32_maybe_silence_nan(a, status);
     case 1:
-        return float32_maybe_silence_nan(b);
+        return float32_maybe_silence_nan(b, status);
     case 2:
-        return float32_maybe_silence_nan(c);
+        return float32_maybe_silence_nan(c, status);
     case 3:
     default:
-        return float32_default_nan;
+        return float32_default_nan(status);
     }
 }
 
 #ifdef NO_SIGNALING_NANS
-int float64_is_quiet_nan(float64 a_)
+int float64_is_quiet_nan(float64 a_, float_status *status)
 {
     return float64_is_any_nan(a_);
 }
 
-int float64_is_signaling_nan(float64 a_)
+int float64_is_signaling_nan(float64 a_, float_status *status)
 {
     return 0;
 }
@@ -661,16 +757,15 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float64_is_quiet_nan( float64 a_ )
+int float64_is_quiet_nan(float64 a_, float_status *status)
 {
     uint64_t a = float64_val(a_);
-#if SNAN_BIT_IS_ONE
-    return
-           ( ( ( a>>51 ) & 0xFFF ) == 0xFFE )
-        && ( a & LIT64( 0x0007FFFFFFFFFFFF ) );
-#else
-    return ( LIT64( 0xFFF0000000000000 ) <= (uint64_t) ( a<<1 ) );
-#endif
+    if (status->snan_bit_is_one) {
+        return (((a >> 51) & 0xFFF) == 0xFFE)
+            && (a & 0x0007FFFFFFFFFFFFULL);
+    } else {
+        return ((a << 1) >= 0xFFF0000000000000ULL);
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -678,16 +773,15 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float64_is_signaling_nan( float64 a_ )
+int float64_is_signaling_nan(float64 a_, float_status *status)
 {
     uint64_t a = float64_val(a_);
-#if SNAN_BIT_IS_ONE
-    return ( LIT64( 0xFFF0000000000000 ) <= (uint64_t) ( a<<1 ) );
-#else
-    return
-           ( ( ( a>>51 ) & 0xFFF ) == 0xFFE )
-        && ( a & LIT64( 0x0007FFFFFFFFFFFF ) );
-#endif
+    if (status->snan_bit_is_one) {
+        return ((a << 1) >= 0xFFF0000000000000ULL);
+    } else {
+        return (((a >> 51) & 0xFFF) == 0xFFE)
+            && (a & LIT64(0x0007FFFFFFFFFFFF));
+    }
 }
 #endif
 
@@ -696,20 +790,16 @@
 | signaling NaN; otherwise returns `a'.
 *----------------------------------------------------------------------------*/
 
-float64 float64_maybe_silence_nan( float64 a_ )
+float64 float64_maybe_silence_nan(float64 a_, float_status *status)
 {
-    if (float64_is_signaling_nan(a_)) {
-#if SNAN_BIT_IS_ONE
-#  if defined(TARGET_MIPS) || defined(TARGET_SH4) || defined(TARGET_UNICORE32)
-        return float64_default_nan;
-#  else
-#    error Rules for silencing a signaling NaN are target-specific
-#  endif
-#else
-        uint64_t a = float64_val(a_);
-        a |= LIT64( 0x0008000000000000 );
-        return make_float64(a);
-#endif
+    if (float64_is_signaling_nan(a_, status)) {
+        if (status->snan_bit_is_one) {
+            return float64_default_nan(status);
+        } else {
+            uint64_t a = float64_val(a_);
+            a |= LIT64(0x0008000000000000);
+            return make_float64(a);
+        }
     }
     return a_;
 }
@@ -720,14 +810,16 @@
 | exception is raised.
 *----------------------------------------------------------------------------*/
 
-static commonNaNT float64ToCommonNaN( float64 a STATUS_PARAM)
+static commonNaNT float64ToCommonNaN(float64 a, float_status *status)
 {
     commonNaNT z;
 
-    if ( float64_is_signaling_nan( a ) ) float_raise( float_flag_invalid STATUS_VAR);
-    z.sign = float64_val(a)>>63;
+    if (float64_is_signaling_nan(a, status)) {
+        float_raise(float_flag_invalid, status);
+    }
+    z.sign = float64_val(a) >> 63;
     z.low = 0;
-    z.high = float64_val(a)<<12;
+    z.high = float64_val(a) << 12;
     return z;
 }
 
@@ -736,21 +828,22 @@
 | precision floating-point format.
 *----------------------------------------------------------------------------*/
 
-static float64 commonNaNToFloat64( commonNaNT a STATUS_PARAM)
+static float64 commonNaNToFloat64(commonNaNT a, float_status *status)
 {
-    uint64_t mantissa = a.high>>12;
+    uint64_t mantissa = a.high >> 12;
 
-    if ( STATUS(default_nan_mode) ) {
-        return float64_default_nan;
+    if (status->default_nan_mode) {
+        return float64_default_nan(status);
     }
 
-    if ( mantissa )
+    if (mantissa) {
         return make_float64(
-              ( ( (uint64_t) a.sign )<<63 )
-            | LIT64( 0x7FF0000000000000 )
-            | ( a.high>>12 ));
-    else
-        return float64_default_nan;
+              (((uint64_t) a.sign) << 63)
+            | LIT64(0x7FF0000000000000)
+            | (a.high >> 12));
+    } else {
+        return float64_default_nan(status);
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -759,27 +852,30 @@
 | signaling NaN, the invalid exception is raised.
 *----------------------------------------------------------------------------*/
 
-static float64 propagateFloat64NaN( float64 a, float64 b STATUS_PARAM)
+static float64 propagateFloat64NaN(float64 a, float64 b, float_status *status)
 {
     flag aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN;
     flag aIsLargerSignificand;
     uint64_t av, bv;
 
-    aIsQuietNaN = float64_is_quiet_nan( a );
-    aIsSignalingNaN = float64_is_signaling_nan( a );
-    bIsQuietNaN = float64_is_quiet_nan( b );
-    bIsSignalingNaN = float64_is_signaling_nan( b );
+    aIsQuietNaN = float64_is_quiet_nan(a, status);
+    aIsSignalingNaN = float64_is_signaling_nan(a, status);
+    bIsQuietNaN = float64_is_quiet_nan(b, status);
+    bIsSignalingNaN = float64_is_signaling_nan(b, status);
     av = float64_val(a);
     bv = float64_val(b);
 
-    if ( aIsSignalingNaN | bIsSignalingNaN ) float_raise( float_flag_invalid STATUS_VAR);
+    if (aIsSignalingNaN | bIsSignalingNaN) {
+        float_raise(float_flag_invalid, status);
+    }
 
-    if ( STATUS(default_nan_mode) )
-        return float64_default_nan;
+    if (status->default_nan_mode) {
+        return float64_default_nan(status);
+    }
 
-    if ((uint64_t)(av<<1) < (uint64_t)(bv<<1)) {
+    if ((uint64_t)(av << 1) < (uint64_t)(bv << 1)) {
         aIsLargerSignificand = 0;
-    } else if ((uint64_t)(bv<<1) < (uint64_t)(av<<1)) {
+    } else if ((uint64_t)(bv << 1) < (uint64_t)(av << 1)) {
         aIsLargerSignificand = 1;
     } else {
         aIsLargerSignificand = (av < bv) ? 1 : 0;
@@ -787,9 +883,9 @@
 
     if (pickNaN(aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN,
                 aIsLargerSignificand)) {
-        return float64_maybe_silence_nan(b);
+        return float64_maybe_silence_nan(b, status);
     } else {
-        return float64_maybe_silence_nan(a);
+        return float64_maybe_silence_nan(a, status);
     }
 }
 
@@ -803,54 +899,55 @@
 *----------------------------------------------------------------------------*/
 
 static float64 propagateFloat64MulAddNaN(float64 a, float64 b,
-                                         float64 c, flag infzero STATUS_PARAM)
+                                         float64 c, flag infzero,
+                                         float_status *status)
 {
     flag aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN,
         cIsQuietNaN, cIsSignalingNaN;
     int which;
 
-    aIsQuietNaN = float64_is_quiet_nan(a);
-    aIsSignalingNaN = float64_is_signaling_nan(a);
-    bIsQuietNaN = float64_is_quiet_nan(b);
-    bIsSignalingNaN = float64_is_signaling_nan(b);
-    cIsQuietNaN = float64_is_quiet_nan(c);
-    cIsSignalingNaN = float64_is_signaling_nan(c);
+    aIsQuietNaN = float64_is_quiet_nan(a, status);
+    aIsSignalingNaN = float64_is_signaling_nan(a, status);
+    bIsQuietNaN = float64_is_quiet_nan(b, status);
+    bIsSignalingNaN = float64_is_signaling_nan(b, status);
+    cIsQuietNaN = float64_is_quiet_nan(c, status);
+    cIsSignalingNaN = float64_is_signaling_nan(c, status);
 
     if (aIsSignalingNaN | bIsSignalingNaN | cIsSignalingNaN) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
     }
 
     which = pickNaNMulAdd(aIsQuietNaN, aIsSignalingNaN,
                           bIsQuietNaN, bIsSignalingNaN,
-                          cIsQuietNaN, cIsSignalingNaN, infzero STATUS_VAR);
+                          cIsQuietNaN, cIsSignalingNaN, infzero, status);
 
-    if (STATUS(default_nan_mode)) {
+    if (status->default_nan_mode) {
         /* Note that this check is after pickNaNMulAdd so that function
          * has an opportunity to set the Invalid flag.
          */
-        return float64_default_nan;
+        return float64_default_nan(status);
     }
 
     switch (which) {
     case 0:
-        return float64_maybe_silence_nan(a);
+        return float64_maybe_silence_nan(a, status);
     case 1:
-        return float64_maybe_silence_nan(b);
+        return float64_maybe_silence_nan(b, status);
     case 2:
-        return float64_maybe_silence_nan(c);
+        return float64_maybe_silence_nan(c, status);
     case 3:
     default:
-        return float64_default_nan;
+        return float64_default_nan(status);
     }
 }
 
 #ifdef NO_SIGNALING_NANS
-int floatx80_is_quiet_nan(floatx80 a_)
+int floatx80_is_quiet_nan(floatx80 a_, float_status *status)
 {
     return floatx80_is_any_nan(a_);
 }
 
-int floatx80_is_signaling_nan(floatx80 a_)
+int floatx80_is_signaling_nan(floatx80 a_, float_status *status)
 {
     return 0;
 }
@@ -861,20 +958,19 @@
 | function for other types as floatx80 has an explicit bit.
 *----------------------------------------------------------------------------*/
 
-int floatx80_is_quiet_nan( floatx80 a )
+int floatx80_is_quiet_nan(floatx80 a, float_status *status)
 {
-#if SNAN_BIT_IS_ONE
-    uint64_t aLow;
+    if (status->snan_bit_is_one) {
+        uint64_t aLow;
 
-    aLow = a.low & ~ LIT64( 0x4000000000000000 );
-    return
-           ( ( a.high & 0x7FFF ) == 0x7FFF )
-        && (uint64_t) ( aLow<<1 )
-        && ( a.low == aLow );
-#else
-    return ( ( a.high & 0x7FFF ) == 0x7FFF )
-        && (LIT64( 0x8000000000000000 ) <= ((uint64_t) ( a.low<<1 )));
-#endif
+        aLow = a.low & ~0x4000000000000000ULL;
+        return ((a.high & 0x7FFF) == 0x7FFF)
+            && (aLow << 1)
+            && (a.low == aLow);
+    } else {
+        return ((a.high & 0x7FFF) == 0x7FFF)
+            && (LIT64(0x8000000000000000) <= ((uint64_t)(a.low << 1)));
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -883,20 +979,19 @@
 | function for other types as floatx80 has an explicit bit.
 *----------------------------------------------------------------------------*/
 
-int floatx80_is_signaling_nan( floatx80 a )
+int floatx80_is_signaling_nan(floatx80 a, float_status *status)
 {
-#if SNAN_BIT_IS_ONE
-    return ( ( a.high & 0x7FFF ) == 0x7FFF )
-        && (LIT64( 0x8000000000000000 ) <= ((uint64_t) ( a.low<<1 )));
-#else
-    uint64_t aLow;
+    if (status->snan_bit_is_one) {
+        return ((a.high & 0x7FFF) == 0x7FFF)
+            && ((a.low << 1) >= 0x8000000000000000ULL);
+    } else {
+        uint64_t aLow;
 
-    aLow = a.low & ~ LIT64( 0x4000000000000000 );
-    return
-           ( ( a.high & 0x7FFF ) == 0x7FFF )
-        && (uint64_t) ( aLow<<1 )
-        && ( a.low == aLow );
-#endif
+        aLow = a.low & ~LIT64(0x4000000000000000);
+        return ((a.high & 0x7FFF) == 0x7FFF)
+            && (uint64_t)(aLow << 1)
+            && (a.low == aLow);
+    }
 }
 #endif
 
@@ -905,20 +1000,15 @@
 | `a' is a signaling NaN; otherwise returns `a'.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_maybe_silence_nan( floatx80 a )
+floatx80 floatx80_maybe_silence_nan(floatx80 a, float_status *status)
 {
-    if (floatx80_is_signaling_nan(a)) {
-#if SNAN_BIT_IS_ONE
-#  if defined(TARGET_MIPS) || defined(TARGET_SH4) || defined(TARGET_UNICORE32)
-        a.low = floatx80_default_nan_low;
-        a.high = floatx80_default_nan_high;
-#  else
-#    error Rules for silencing a signaling NaN are target-specific
-#  endif
-#else
-        a.low |= LIT64( 0xC000000000000000 );
-        return a;
-#endif
+    if (floatx80_is_signaling_nan(a, status)) {
+        if (status->snan_bit_is_one) {
+            a = floatx80_default_nan(status);
+        } else {
+            a.low |= LIT64(0xC000000000000000);
+            return a;
+        }
     }
     return a;
 }
@@ -929,19 +1019,23 @@
 | invalid exception is raised.
 *----------------------------------------------------------------------------*/
 
-static commonNaNT floatx80ToCommonNaN( floatx80 a STATUS_PARAM)
+static commonNaNT floatx80ToCommonNaN(floatx80 a, float_status *status)
 {
+    floatx80 dflt;
     commonNaNT z;
 
-    if ( floatx80_is_signaling_nan( a ) ) float_raise( float_flag_invalid STATUS_VAR);
-    if ( a.low >> 63 ) {
+    if (floatx80_is_signaling_nan(a, status)) {
+        float_raise(float_flag_invalid, status);
+    }
+    if (a.low >> 63) {
         z.sign = a.high >> 15;
         z.low = 0;
         z.high = a.low << 1;
     } else {
-        z.sign = floatx80_default_nan_high >> 15;
+        dflt = floatx80_default_nan(status);
+        z.sign = dflt.high >> 15;
         z.low = 0;
-        z.high = floatx80_default_nan_low << 1;
+        z.high = dflt.low << 1;
     }
     return z;
 }
@@ -951,24 +1045,20 @@
 | double-precision floating-point format.
 *----------------------------------------------------------------------------*/
 
-static floatx80 commonNaNToFloatx80( commonNaNT a STATUS_PARAM)
+static floatx80 commonNaNToFloatx80(commonNaNT a, float_status *status)
 {
     floatx80 z;
 
-    if ( STATUS(default_nan_mode) ) {
-        z.low = floatx80_default_nan_low;
-        z.high = floatx80_default_nan_high;
-        return z;
+    if (status->default_nan_mode) {
+        return floatx80_default_nan(status);
     }
 
     if (a.high >> 1) {
-        z.low = LIT64( 0x8000000000000000 ) | a.high >> 1;
-        z.high = ( ( (uint16_t) a.sign )<<15 ) | 0x7FFF;
+        z.low = LIT64(0x8000000000000000) | a.high >> 1;
+        z.high = (((uint16_t)a.sign) << 15) | 0x7FFF;
     } else {
-        z.low = floatx80_default_nan_low;
-        z.high = floatx80_default_nan_high;
+        z = floatx80_default_nan(status);
     }
-
     return z;
 }
 
@@ -978,22 +1068,23 @@
 | `b' is a signaling NaN, the invalid exception is raised.
 *----------------------------------------------------------------------------*/
 
-static floatx80 propagateFloatx80NaN( floatx80 a, floatx80 b STATUS_PARAM)
+static floatx80 propagateFloatx80NaN(floatx80 a, floatx80 b,
+                                     float_status *status)
 {
     flag aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN;
     flag aIsLargerSignificand;
 
-    aIsQuietNaN = floatx80_is_quiet_nan( a );
-    aIsSignalingNaN = floatx80_is_signaling_nan( a );
-    bIsQuietNaN = floatx80_is_quiet_nan( b );
-    bIsSignalingNaN = floatx80_is_signaling_nan( b );
+    aIsQuietNaN = floatx80_is_quiet_nan(a, status);
+    aIsSignalingNaN = floatx80_is_signaling_nan(a, status);
+    bIsQuietNaN = floatx80_is_quiet_nan(b, status);
+    bIsSignalingNaN = floatx80_is_signaling_nan(b, status);
 
-    if ( aIsSignalingNaN | bIsSignalingNaN ) float_raise( float_flag_invalid STATUS_VAR);
+    if (aIsSignalingNaN | bIsSignalingNaN) {
+        float_raise(float_flag_invalid, status);
+    }
 
-    if ( STATUS(default_nan_mode) ) {
-        a.low = floatx80_default_nan_low;
-        a.high = floatx80_default_nan_high;
-        return a;
+    if (status->default_nan_mode) {
+        return floatx80_default_nan(status);
     }
 
     if (a.low < b.low) {
@@ -1006,19 +1097,19 @@
 
     if (pickNaN(aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN,
                 aIsLargerSignificand)) {
-        return floatx80_maybe_silence_nan(b);
+        return floatx80_maybe_silence_nan(b, status);
     } else {
-        return floatx80_maybe_silence_nan(a);
+        return floatx80_maybe_silence_nan(a, status);
     }
 }
 
 #ifdef NO_SIGNALING_NANS
-int float128_is_quiet_nan(float128 a_)
+int float128_is_quiet_nan(float128 a_, float_status *status)
 {
     return float128_is_any_nan(a_);
 }
 
-int float128_is_signaling_nan(float128 a_)
+int float128_is_signaling_nan(float128 a_, float_status *status)
 {
     return 0;
 }
@@ -1028,17 +1119,15 @@
 | NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float128_is_quiet_nan( float128 a )
+int float128_is_quiet_nan(float128 a, float_status *status)
 {
-#if SNAN_BIT_IS_ONE
-    return
-           ( ( ( a.high>>47 ) & 0xFFFF ) == 0xFFFE )
-        && ( a.low || ( a.high & LIT64( 0x00007FFFFFFFFFFF ) ) );
-#else
-    return
-           ( LIT64( 0xFFFE000000000000 ) <= (uint64_t) ( a.high<<1 ) )
-        && ( a.low || ( a.high & LIT64( 0x0000FFFFFFFFFFFF ) ) );
-#endif
+    if (status->snan_bit_is_one) {
+        return (((a.high >> 47) & 0xFFFF) == 0xFFFE)
+            && (a.low || (a.high & 0x00007FFFFFFFFFFFULL));
+    } else {
+        return ((a.high << 1) >= 0xFFFF000000000000ULL)
+            && (a.low || (a.high & 0x0000FFFFFFFFFFFFULL));
+    }
 }
 
 /*----------------------------------------------------------------------------
@@ -1046,17 +1135,15 @@
 | signaling NaN; otherwise returns 0.
 *----------------------------------------------------------------------------*/
 
-int float128_is_signaling_nan( float128 a )
+int float128_is_signaling_nan(float128 a, float_status *status)
 {
-#if SNAN_BIT_IS_ONE
-    return
-           ( LIT64( 0xFFFE000000000000 ) <= (uint64_t) ( a.high<<1 ) )
-        && ( a.low || ( a.high & LIT64( 0x0000FFFFFFFFFFFF ) ) );
-#else
-    return
-           ( ( ( a.high>>47 ) & 0xFFFF ) == 0xFFFE )
-        && ( a.low || ( a.high & LIT64( 0x00007FFFFFFFFFFF ) ) );
-#endif
+    if (status->snan_bit_is_one) {
+        return ((a.high << 1) >= 0xFFFF000000000000ULL)
+            && (a.low || (a.high & 0x0000FFFFFFFFFFFFULL));
+    } else {
+        return (((a.high >> 47) & 0xFFFF) == 0xFFFE)
+            && (a.low || (a.high & LIT64(0x00007FFFFFFFFFFF)));
+    }
 }
 #endif
 
@@ -1065,20 +1152,15 @@
 | a signaling NaN; otherwise returns `a'.
 *----------------------------------------------------------------------------*/
 
-float128 float128_maybe_silence_nan( float128 a )
+float128 float128_maybe_silence_nan(float128 a, float_status *status)
 {
-    if (float128_is_signaling_nan(a)) {
-#if SNAN_BIT_IS_ONE
-#  if defined(TARGET_MIPS) || defined(TARGET_SH4) || defined(TARGET_UNICORE32)
-        a.low = float128_default_nan_low;
-        a.high = float128_default_nan_high;
-#  else
-#    error Rules for silencing a signaling NaN are target-specific
-#  endif
-#else
-        a.high |= LIT64( 0x0000800000000000 );
-        return a;
-#endif
+    if (float128_is_signaling_nan(a, status)) {
+        if (status->snan_bit_is_one) {
+            a = float128_default_nan(status);
+        } else {
+            a.high |= LIT64(0x0000800000000000);
+            return a;
+        }
     }
     return a;
 }
@@ -1089,13 +1171,15 @@
 | exception is raised.
 *----------------------------------------------------------------------------*/
 
-static commonNaNT float128ToCommonNaN( float128 a STATUS_PARAM)
+static commonNaNT float128ToCommonNaN(float128 a, float_status *status)
 {
     commonNaNT z;
 
-    if ( float128_is_signaling_nan( a ) ) float_raise( float_flag_invalid STATUS_VAR);
-    z.sign = a.high>>63;
-    shortShift128Left( a.high, a.low, 16, &z.high, &z.low );
+    if (float128_is_signaling_nan(a, status)) {
+        float_raise(float_flag_invalid, status);
+    }
+    z.sign = a.high >> 63;
+    shortShift128Left(a.high, a.low, 16, &z.high, &z.low);
     return z;
 }
 
@@ -1104,18 +1188,16 @@
 | precision floating-point format.
 *----------------------------------------------------------------------------*/
 
-static float128 commonNaNToFloat128( commonNaNT a STATUS_PARAM)
+static float128 commonNaNToFloat128(commonNaNT a, float_status *status)
 {
     float128 z;
 
-    if ( STATUS(default_nan_mode) ) {
-        z.low = float128_default_nan_low;
-        z.high = float128_default_nan_high;
-        return z;
+    if (status->default_nan_mode) {
+        return float128_default_nan(status);
     }
 
-    shift128Right( a.high, a.low, 16, &z.high, &z.low );
-    z.high |= ( ( (uint64_t) a.sign )<<63 ) | LIT64( 0x7FFF000000000000 );
+    shift128Right(a.high, a.low, 16, &z.high, &z.low);
+    z.high |= (((uint64_t)a.sign) << 63) | LIT64(0x7FFF000000000000);
     return z;
 }
 
@@ -1125,27 +1207,28 @@
 | `b' is a signaling NaN, the invalid exception is raised.
 *----------------------------------------------------------------------------*/
 
-static float128 propagateFloat128NaN( float128 a, float128 b STATUS_PARAM)
+static float128 propagateFloat128NaN(float128 a, float128 b,
+                                     float_status *status)
 {
     flag aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN;
     flag aIsLargerSignificand;
 
-    aIsQuietNaN = float128_is_quiet_nan( a );
-    aIsSignalingNaN = float128_is_signaling_nan( a );
-    bIsQuietNaN = float128_is_quiet_nan( b );
-    bIsSignalingNaN = float128_is_signaling_nan( b );
+    aIsQuietNaN = float128_is_quiet_nan(a, status);
+    aIsSignalingNaN = float128_is_signaling_nan(a, status);
+    bIsQuietNaN = float128_is_quiet_nan(b, status);
+    bIsSignalingNaN = float128_is_signaling_nan(b, status);
 
-    if ( aIsSignalingNaN | bIsSignalingNaN ) float_raise( float_flag_invalid STATUS_VAR);
-
-    if ( STATUS(default_nan_mode) ) {
-        a.low = float128_default_nan_low;
-        a.high = float128_default_nan_high;
-        return a;
+    if (aIsSignalingNaN | bIsSignalingNaN) {
+        float_raise(float_flag_invalid, status);
     }
 
-    if (lt128(a.high<<1, a.low, b.high<<1, b.low)) {
+    if (status->default_nan_mode) {
+        return float128_default_nan(status);
+    }
+
+    if (lt128(a.high << 1, a.low, b.high << 1, b.low)) {
         aIsLargerSignificand = 0;
-    } else if (lt128(b.high<<1, b.low, a.high<<1, a.low)) {
+    } else if (lt128(b.high << 1, b.low, a.high << 1, a.low)) {
         aIsLargerSignificand = 1;
     } else {
         aIsLargerSignificand = (a.high < b.high) ? 1 : 0;
@@ -1153,9 +1236,8 @@
 
     if (pickNaN(aIsQuietNaN, aIsSignalingNaN, bIsQuietNaN, bIsSignalingNaN,
                 aIsLargerSignificand)) {
-        return float128_maybe_silence_nan(b);
+        return float128_maybe_silence_nan(b, status);
     } else {
-        return float128_maybe_silence_nan(a);
+        return float128_maybe_silence_nan(a, status);
     }
 }
-
diff --git a/fpu/softfloat.c b/fpu/softfloat.c
index 16b21eb..9b1eccf 100644
--- a/fpu/softfloat.c
+++ b/fpu/softfloat.c
@@ -1,13 +1,24 @@
 /*
  * QEMU float support
  *
- * Derived from SoftFloat.
+ * The code in this source file is derived from release 2a of the SoftFloat
+ * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and
+ * some later contributions) are provided under that license, as detailed below.
+ * It has subsequently been modified by contributors to the QEMU Project,
+ * so some portions are provided under:
+ *  the SoftFloat-2a license
+ *  the BSD license
+ *  GPL-v2-or-later
+ *
+ * Any future contributions to this file after December 1st 2014 will be
+ * taken to be licensed under the Softfloat-2a license unless specifically
+ * indicated otherwise.
  */
 
-/*============================================================================
-
-This C source file is part of the SoftFloat IEC/IEEE Floating-point Arithmetic
-Package, Release 2b.
+/*
+===============================================================================
+This C source file is part of the SoftFloat IEC/IEEE Floating-point
+Arithmetic Package, Release 2a.
 
 Written by John R. Hauser.  This work was made possible in part by the
 International Computer Science Institute, located at Suite 600, 1947 Center
@@ -16,34 +27,66 @@
 of this code was written as part of a project to build a fixed-point vector
 processor in collaboration with the University of California at Berkeley,
 overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
-is available through the Web page `http://www.cs.berkeley.edu/~jhauser/
+is available through the Web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
 arithmetic/SoftFloat.html'.
 
-THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort has
-been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT TIMES
-RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO PERSONS
-AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ALL LOSSES,
-COSTS, OR OTHER PROBLEMS THEY INCUR DUE TO THE SOFTWARE, AND WHO FURTHERMORE
-EFFECTIVELY INDEMNIFY JOHN HAUSER AND THE INTERNATIONAL COMPUTER SCIENCE
-INSTITUTE (possibly via similar legal warning) AGAINST ALL LOSSES, COSTS, OR
-OTHER PROBLEMS INCURRED BY THEIR CUSTOMERS AND CLIENTS DUE TO THE SOFTWARE.
+THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort
+has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
+TIMES RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO
+PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
+AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
 
 Derivative works are acceptable, even for commercial purposes, so long as
-(1) the source code for the derivative work includes prominent notice that
-the work is derivative, and (2) the source code includes prominent notice with
-these four paragraphs for those parts of this code that are retained.
+(1) they include prominent notice that the work is derivative, and (2) they
+include prominent notice akin to these four paragraphs for those parts of
+this code that are retained.
 
-=============================================================================*/
+===============================================================================
+*/
+
+/* BSD licensing:
+ * Copyright (c) 2006, Fabrice Bellard
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Portions of this work are licensed under the terms of the GNU GPL,
+ * version 2 or later. See the COPYING file in the top-level directory.
+ */
 
 /* softfloat (and in particular the code in softfloat-specialize.h) is
  * target-dependent and needs the TARGET_* macros.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 
 #include "fpu/softfloat.h"
 
 /* We only need stdlib for abort() */
-#include <stdlib.h>
 
 /*----------------------------------------------------------------------------
 | Primitive arithmetic functions, including multi-word arithmetic, and
@@ -75,7 +118,7 @@
 | Returns the exponent bits of the half-precision floating-point value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int_fast16_t extractFloat16Exp(float16 a)
+static inline int extractFloat16Exp(float16 a)
 {
     return (float16_val(a) >> 10) & 0x1f;
 }
@@ -100,14 +143,14 @@
 | positive or negative integer is returned.
 *----------------------------------------------------------------------------*/
 
-static int32 roundAndPackInt32( flag zSign, uint64_t absZ STATUS_PARAM)
+static int32_t roundAndPackInt32(flag zSign, uint64_t absZ, float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven;
-    int8 roundIncrement, roundBits;
+    int8_t roundIncrement, roundBits;
     int32_t z;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
     switch (roundingMode) {
     case float_round_nearest_even:
@@ -132,10 +175,12 @@
     z = absZ;
     if ( zSign ) z = - z;
     if ( ( absZ>>32 ) || ( z && ( ( z < 0 ) ^ zSign ) ) ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return zSign ? (int32_t) 0x80000000 : 0x7FFFFFFF;
     }
-    if ( roundBits ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (roundBits) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     return z;
 
 }
@@ -152,13 +197,14 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-static int64 roundAndPackInt64( flag zSign, uint64_t absZ0, uint64_t absZ1 STATUS_PARAM)
+static int64_t roundAndPackInt64(flag zSign, uint64_t absZ0, uint64_t absZ1,
+                               float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment;
     int64_t z;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
     switch (roundingMode) {
     case float_round_nearest_even:
@@ -186,12 +232,14 @@
     if ( zSign ) z = - z;
     if ( z && ( ( z < 0 ) ^ zSign ) ) {
  overflow:
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return
               zSign ? (int64_t) LIT64( 0x8000000000000000 )
             : LIT64( 0x7FFFFFFFFFFFFFFF );
     }
-    if ( absZ1 ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (absZ1) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     return z;
 
 }
@@ -206,13 +254,13 @@
 | exception is raised and the largest unsigned integer is returned.
 *----------------------------------------------------------------------------*/
 
-static int64 roundAndPackUint64(flag zSign, uint64_t absZ0,
-                                uint64_t absZ1 STATUS_PARAM)
+static int64_t roundAndPackUint64(flag zSign, uint64_t absZ0,
+                                uint64_t absZ1, float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = (roundingMode == float_round_nearest_even);
     switch (roundingMode) {
     case float_round_nearest_even:
@@ -234,19 +282,19 @@
     if (increment) {
         ++absZ0;
         if (absZ0 == 0) {
-            float_raise(float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             return LIT64(0xFFFFFFFFFFFFFFFF);
         }
         absZ0 &= ~(((uint64_t)(absZ1<<1) == 0) & roundNearestEven);
     }
 
     if (zSign && absZ0) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
 
     if (absZ1) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     return absZ0;
 }
@@ -266,7 +314,7 @@
 | Returns the exponent bits of the single-precision floating-point value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int_fast16_t extractFloat32Exp(float32 a)
+static inline int extractFloat32Exp(float32 a)
 {
 
     return ( float32_val(a)>>23 ) & 0xFF;
@@ -288,11 +336,11 @@
 | If `a' is denormal and we are in flush-to-zero mode then set the
 | input-denormal exception and return zero. Otherwise just return the value.
 *----------------------------------------------------------------------------*/
-float32 float32_squash_input_denormal(float32 a STATUS_PARAM)
+float32 float32_squash_input_denormal(float32 a, float_status *status)
 {
-    if (STATUS(flush_inputs_to_zero)) {
+    if (status->flush_inputs_to_zero) {
         if (extractFloat32Exp(a) == 0 && extractFloat32Frac(a) != 0) {
-            float_raise(float_flag_input_denormal STATUS_VAR);
+            float_raise(float_flag_input_denormal, status);
             return make_float32(float32_val(a) & 0x80000000);
         }
     }
@@ -307,9 +355,9 @@
 *----------------------------------------------------------------------------*/
 
 static void
- normalizeFloat32Subnormal(uint32_t aSig, int_fast16_t *zExpPtr, uint32_t *zSigPtr)
+ normalizeFloat32Subnormal(uint32_t aSig, int *zExpPtr, uint32_t *zSigPtr)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros32( aSig ) - 8;
     *zSigPtr = aSig<<shiftCount;
@@ -328,7 +376,7 @@
 | significand.
 *----------------------------------------------------------------------------*/
 
-static inline float32 packFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig)
+static inline float32 packFloat32(flag zSign, int zExp, uint32_t zSig)
 {
 
     return make_float32(
@@ -358,14 +406,15 @@
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float32 roundAndPackFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig STATUS_PARAM)
+static float32 roundAndPackFloat32(flag zSign, int zExp, uint32_t zSig,
+                                   float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven;
-    int8 roundIncrement, roundBits;
+    int8_t roundIncrement, roundBits;
     flag isTiny;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
     switch (roundingMode) {
     case float_round_nearest_even:
@@ -391,25 +440,30 @@
              || (    ( zExp == 0xFD )
                   && ( (int32_t) ( zSig + roundIncrement ) < 0 ) )
            ) {
-            float_raise( float_flag_overflow | float_flag_inexact STATUS_VAR);
+            float_raise(float_flag_overflow | float_flag_inexact, status);
             return packFloat32( zSign, 0xFF, - ( roundIncrement == 0 ));
         }
         if ( zExp < 0 ) {
-            if (STATUS(flush_to_zero)) {
-                float_raise(float_flag_output_denormal STATUS_VAR);
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
                 return packFloat32(zSign, 0, 0);
             }
             isTiny =
-                   ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
+                (status->float_detect_tininess
+                 == float_tininess_before_rounding)
                 || ( zExp < -1 )
                 || ( zSig + roundIncrement < 0x80000000 );
             shift32RightJamming( zSig, - zExp, &zSig );
             zExp = 0;
             roundBits = zSig & 0x7F;
-            if ( isTiny && roundBits ) float_raise( float_flag_underflow STATUS_VAR);
+            if (isTiny && roundBits) {
+                float_raise(float_flag_underflow, status);
+            }
         }
     }
-    if ( roundBits ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (roundBits) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     zSig = ( zSig + roundIncrement )>>7;
     zSig &= ~ ( ( ( roundBits ^ 0x40 ) == 0 ) & roundNearestEven );
     if ( zSig == 0 ) zExp = 0;
@@ -427,12 +481,14 @@
 *----------------------------------------------------------------------------*/
 
 static float32
- normalizeRoundAndPackFloat32(flag zSign, int_fast16_t zExp, uint32_t zSig STATUS_PARAM)
+ normalizeRoundAndPackFloat32(flag zSign, int zExp, uint32_t zSig,
+                              float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros32( zSig ) - 1;
-    return roundAndPackFloat32( zSign, zExp - shiftCount, zSig<<shiftCount STATUS_VAR);
+    return roundAndPackFloat32(zSign, zExp - shiftCount, zSig<<shiftCount,
+                               status);
 
 }
 
@@ -451,7 +507,7 @@
 | Returns the exponent bits of the double-precision floating-point value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int_fast16_t extractFloat64Exp(float64 a)
+static inline int extractFloat64Exp(float64 a)
 {
 
     return ( float64_val(a)>>52 ) & 0x7FF;
@@ -473,11 +529,11 @@
 | If `a' is denormal and we are in flush-to-zero mode then set the
 | input-denormal exception and return zero. Otherwise just return the value.
 *----------------------------------------------------------------------------*/
-float64 float64_squash_input_denormal(float64 a STATUS_PARAM)
+float64 float64_squash_input_denormal(float64 a, float_status *status)
 {
-    if (STATUS(flush_inputs_to_zero)) {
+    if (status->flush_inputs_to_zero) {
         if (extractFloat64Exp(a) == 0 && extractFloat64Frac(a) != 0) {
-            float_raise(float_flag_input_denormal STATUS_VAR);
+            float_raise(float_flag_input_denormal, status);
             return make_float64(float64_val(a) & (1ULL << 63));
         }
     }
@@ -492,9 +548,9 @@
 *----------------------------------------------------------------------------*/
 
 static void
- normalizeFloat64Subnormal(uint64_t aSig, int_fast16_t *zExpPtr, uint64_t *zSigPtr)
+ normalizeFloat64Subnormal(uint64_t aSig, int *zExpPtr, uint64_t *zSigPtr)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros64( aSig ) - 11;
     *zSigPtr = aSig<<shiftCount;
@@ -513,7 +569,7 @@
 | significand.
 *----------------------------------------------------------------------------*/
 
-static inline float64 packFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig)
+static inline float64 packFloat64(flag zSign, int zExp, uint64_t zSig)
 {
 
     return make_float64(
@@ -529,9 +585,9 @@
 | the inexact exception raised if the abstract input cannot be represented
 | exactly.  However, if the abstract value is too large, the overflow and
 | inexact exceptions are raised and an infinity or maximal finite value is
-| returned.  If the abstract value is too small, the input value is rounded
-| to a subnormal number, and the underflow and inexact exceptions are raised
-| if the abstract input cannot be represented exactly as a subnormal double-
+| returned.  If the abstract value is too small, the input value is rounded to
+| a subnormal number, and the underflow and inexact exceptions are raised if
+| the abstract input cannot be represented exactly as a subnormal double-
 | precision floating-point number.
 |     The input significand `zSig' has its binary point between bits 62
 | and 61, which is 10 bits to the left of the usual location.  This shifted
@@ -543,14 +599,15 @@
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float64 roundAndPackFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig STATUS_PARAM)
+static float64 roundAndPackFloat64(flag zSign, int zExp, uint64_t zSig,
+                                   float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven;
-    int_fast16_t roundIncrement, roundBits;
+    int roundIncrement, roundBits;
     flag isTiny;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
     switch (roundingMode) {
     case float_round_nearest_even:
@@ -575,25 +632,30 @@
              || (    ( zExp == 0x7FD )
                   && ( (int64_t) ( zSig + roundIncrement ) < 0 ) )
            ) {
-            float_raise( float_flag_overflow | float_flag_inexact STATUS_VAR);
+            float_raise(float_flag_overflow | float_flag_inexact, status);
             return packFloat64( zSign, 0x7FF, - ( roundIncrement == 0 ));
         }
         if ( zExp < 0 ) {
-            if (STATUS(flush_to_zero)) {
-                float_raise(float_flag_output_denormal STATUS_VAR);
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
                 return packFloat64(zSign, 0, 0);
             }
             isTiny =
-                   ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
+                   (status->float_detect_tininess
+                    == float_tininess_before_rounding)
                 || ( zExp < -1 )
                 || ( zSig + roundIncrement < LIT64( 0x8000000000000000 ) );
             shift64RightJamming( zSig, - zExp, &zSig );
             zExp = 0;
             roundBits = zSig & 0x3FF;
-            if ( isTiny && roundBits ) float_raise( float_flag_underflow STATUS_VAR);
+            if (isTiny && roundBits) {
+                float_raise(float_flag_underflow, status);
+            }
         }
     }
-    if ( roundBits ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (roundBits) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     zSig = ( zSig + roundIncrement )>>10;
     zSig &= ~ ( ( ( roundBits ^ 0x200 ) == 0 ) & roundNearestEven );
     if ( zSig == 0 ) zExp = 0;
@@ -611,12 +673,14 @@
 *----------------------------------------------------------------------------*/
 
 static float64
- normalizeRoundAndPackFloat64(flag zSign, int_fast16_t zExp, uint64_t zSig STATUS_PARAM)
+ normalizeRoundAndPackFloat64(flag zSign, int zExp, uint64_t zSig,
+                              float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros64( zSig ) - 1;
-    return roundAndPackFloat64( zSign, zExp - shiftCount, zSig<<shiftCount STATUS_VAR);
+    return roundAndPackFloat64(zSign, zExp - shiftCount, zSig<<shiftCount,
+                               status);
 
 }
 
@@ -637,7 +701,7 @@
 | value `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int32 extractFloatx80Exp( floatx80 a )
+static inline int32_t extractFloatx80Exp( floatx80 a )
 {
 
     return a.high & 0x7FFF;
@@ -664,9 +728,9 @@
 *----------------------------------------------------------------------------*/
 
 static void
- normalizeFloatx80Subnormal( uint64_t aSig, int32 *zExpPtr, uint64_t *zSigPtr )
+ normalizeFloatx80Subnormal( uint64_t aSig, int32_t *zExpPtr, uint64_t *zSigPtr )
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     shiftCount = countLeadingZeros64( aSig );
     *zSigPtr = aSig<<shiftCount;
@@ -679,7 +743,7 @@
 | extended double-precision floating-point value, returning the result.
 *----------------------------------------------------------------------------*/
 
-static inline floatx80 packFloatx80( flag zSign, int32 zExp, uint64_t zSig )
+static inline floatx80 packFloatx80( flag zSign, int32_t zExp, uint64_t zSig )
 {
     floatx80 z;
 
@@ -713,16 +777,15 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static floatx80
- roundAndPackFloatx80(
-     int8 roundingPrecision, flag zSign, int32 zExp, uint64_t zSig0, uint64_t zSig1
- STATUS_PARAM)
+static floatx80 roundAndPackFloatx80(int8_t roundingPrecision, flag zSign,
+                                     int32_t zExp, uint64_t zSig0, uint64_t zSig1,
+                                     float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment, isTiny;
-    int64 roundIncrement, roundMask, roundBits;
+    int64_t roundIncrement, roundMask, roundBits;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
     if ( roundingPrecision == 80 ) goto precision80;
     if ( roundingPrecision == 64 ) {
@@ -761,19 +824,24 @@
             goto overflow;
         }
         if ( zExp <= 0 ) {
-            if (STATUS(flush_to_zero)) {
-                float_raise(float_flag_output_denormal STATUS_VAR);
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
                 return packFloatx80(zSign, 0, 0);
             }
             isTiny =
-                   ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
+                   (status->float_detect_tininess
+                    == float_tininess_before_rounding)
                 || ( zExp < 0 )
                 || ( zSig0 <= zSig0 + roundIncrement );
             shift64RightJamming( zSig0, 1 - zExp, &zSig0 );
             zExp = 0;
             roundBits = zSig0 & roundMask;
-            if ( isTiny && roundBits ) float_raise( float_flag_underflow STATUS_VAR);
-            if ( roundBits ) STATUS(float_exception_flags) |= float_flag_inexact;
+            if (isTiny && roundBits) {
+                float_raise(float_flag_underflow, status);
+            }
+            if (roundBits) {
+                status->float_exception_flags |= float_flag_inexact;
+            }
             zSig0 += roundIncrement;
             if ( (int64_t) zSig0 < 0 ) zExp = 1;
             roundIncrement = roundMask + 1;
@@ -784,7 +852,9 @@
             return packFloatx80( zSign, zExp, zSig0 );
         }
     }
-    if ( roundBits ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (roundBits) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     zSig0 += roundIncrement;
     if ( zSig0 < roundIncrement ) {
         ++zExp;
@@ -824,7 +894,7 @@
            ) {
             roundMask = 0;
  overflow:
-            float_raise( float_flag_overflow | float_flag_inexact STATUS_VAR);
+            float_raise(float_flag_overflow | float_flag_inexact, status);
             if (    ( roundingMode == float_round_to_zero )
                  || ( zSign && ( roundingMode == float_round_up ) )
                  || ( ! zSign && ( roundingMode == float_round_down ) )
@@ -835,14 +905,19 @@
         }
         if ( zExp <= 0 ) {
             isTiny =
-                   ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
+                   (status->float_detect_tininess
+                    == float_tininess_before_rounding)
                 || ( zExp < 0 )
                 || ! increment
                 || ( zSig0 < LIT64( 0xFFFFFFFFFFFFFFFF ) );
             shift64ExtraRightJamming( zSig0, zSig1, 1 - zExp, &zSig0, &zSig1 );
             zExp = 0;
-            if ( isTiny && zSig1 ) float_raise( float_flag_underflow STATUS_VAR);
-            if ( zSig1 ) STATUS(float_exception_flags) |= float_flag_inexact;
+            if (isTiny && zSig1) {
+                float_raise(float_flag_underflow, status);
+            }
+            if (zSig1) {
+                status->float_exception_flags |= float_flag_inexact;
+            }
             switch (roundingMode) {
             case float_round_nearest_even:
             case float_round_ties_away:
@@ -869,7 +944,9 @@
             return packFloatx80( zSign, zExp, zSig0 );
         }
     }
-    if ( zSig1 ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (zSig1) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     if ( increment ) {
         ++zSig0;
         if ( zSig0 == 0 ) {
@@ -896,12 +973,12 @@
 | normalized.
 *----------------------------------------------------------------------------*/
 
-static floatx80
- normalizeRoundAndPackFloatx80(
-     int8 roundingPrecision, flag zSign, int32 zExp, uint64_t zSig0, uint64_t zSig1
- STATUS_PARAM)
+static floatx80 normalizeRoundAndPackFloatx80(int8_t roundingPrecision,
+                                              flag zSign, int32_t zExp,
+                                              uint64_t zSig0, uint64_t zSig1,
+                                              float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     if ( zSig0 == 0 ) {
         zSig0 = zSig1;
@@ -911,8 +988,8 @@
     shiftCount = countLeadingZeros64( zSig0 );
     shortShift128Left( zSig0, zSig1, shiftCount, &zSig0, &zSig1 );
     zExp -= shiftCount;
-    return
-        roundAndPackFloatx80( roundingPrecision, zSign, zExp, zSig0, zSig1 STATUS_VAR);
+    return roundAndPackFloatx80(roundingPrecision, zSign, zExp,
+                                zSig0, zSig1, status);
 
 }
 
@@ -945,7 +1022,7 @@
 | `a'.
 *----------------------------------------------------------------------------*/
 
-static inline int32 extractFloat128Exp( float128 a )
+static inline int32_t extractFloat128Exp( float128 a )
 {
 
     return ( a.high>>48 ) & 0x7FFF;
@@ -977,12 +1054,12 @@
  normalizeFloat128Subnormal(
      uint64_t aSig0,
      uint64_t aSig1,
-     int32 *zExpPtr,
+     int32_t *zExpPtr,
      uint64_t *zSig0Ptr,
      uint64_t *zSig1Ptr
  )
 {
-    int8 shiftCount;
+    int8_t shiftCount;
 
     if ( aSig0 == 0 ) {
         shiftCount = countLeadingZeros64( aSig1 ) - 15;
@@ -1018,7 +1095,7 @@
 *----------------------------------------------------------------------------*/
 
 static inline float128
- packFloat128( flag zSign, int32 zExp, uint64_t zSig0, uint64_t zSig1 )
+ packFloat128( flag zSign, int32_t zExp, uint64_t zSig0, uint64_t zSig1 )
 {
     float128 z;
 
@@ -1049,14 +1126,14 @@
 | overflow follows the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float128
- roundAndPackFloat128(
-     flag zSign, int32 zExp, uint64_t zSig0, uint64_t zSig1, uint64_t zSig2 STATUS_PARAM)
+static float128 roundAndPackFloat128(flag zSign, int32_t zExp,
+                                     uint64_t zSig0, uint64_t zSig1,
+                                     uint64_t zSig2, float_status *status)
 {
-    int8 roundingMode;
+    int8_t roundingMode;
     flag roundNearestEven, increment, isTiny;
 
-    roundingMode = STATUS(float_rounding_mode);
+    roundingMode = status->float_rounding_mode;
     roundNearestEven = ( roundingMode == float_round_nearest_even );
     switch (roundingMode) {
     case float_round_nearest_even:
@@ -1087,7 +1164,7 @@
                   && increment
                 )
            ) {
-            float_raise( float_flag_overflow | float_flag_inexact STATUS_VAR);
+            float_raise(float_flag_overflow | float_flag_inexact, status);
             if (    ( roundingMode == float_round_to_zero )
                  || ( zSign && ( roundingMode == float_round_up ) )
                  || ( ! zSign && ( roundingMode == float_round_down ) )
@@ -1103,12 +1180,13 @@
             return packFloat128( zSign, 0x7FFF, 0, 0 );
         }
         if ( zExp < 0 ) {
-            if (STATUS(flush_to_zero)) {
-                float_raise(float_flag_output_denormal STATUS_VAR);
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
                 return packFloat128(zSign, 0, 0, 0);
             }
             isTiny =
-                   ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
+                   (status->float_detect_tininess
+                    == float_tininess_before_rounding)
                 || ( zExp < -1 )
                 || ! increment
                 || lt128(
@@ -1120,7 +1198,9 @@
             shift128ExtraRightJamming(
                 zSig0, zSig1, zSig2, - zExp, &zSig0, &zSig1, &zSig2 );
             zExp = 0;
-            if ( isTiny && zSig2 ) float_raise( float_flag_underflow STATUS_VAR);
+            if (isTiny && zSig2) {
+                float_raise(float_flag_underflow, status);
+            }
             switch (roundingMode) {
             case float_round_nearest_even:
             case float_round_ties_away:
@@ -1140,7 +1220,9 @@
             }
         }
     }
-    if ( zSig2 ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (zSig2) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     if ( increment ) {
         add128( zSig0, zSig1, 0, 1, &zSig0, &zSig1 );
         zSig1 &= ~ ( ( zSig2 + zSig2 == 0 ) & roundNearestEven );
@@ -1162,11 +1244,11 @@
 | point exponent.
 *----------------------------------------------------------------------------*/
 
-static float128
- normalizeRoundAndPackFloat128(
-     flag zSign, int32 zExp, uint64_t zSig0, uint64_t zSig1 STATUS_PARAM)
+static float128 normalizeRoundAndPackFloat128(flag zSign, int32_t zExp,
+                                              uint64_t zSig0, uint64_t zSig1,
+                                              float_status *status)
 {
-    int8 shiftCount;
+    int8_t shiftCount;
     uint64_t zSig2;
 
     if ( zSig0 == 0 ) {
@@ -1184,7 +1266,7 @@
             zSig0, zSig1, 0, - shiftCount, &zSig0, &zSig1, &zSig2 );
     }
     zExp -= shiftCount;
-    return roundAndPackFloat128( zSign, zExp, zSig0, zSig1, zSig2 STATUS_VAR);
+    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
 
 }
 
@@ -1194,15 +1276,14 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 int32_to_float32(int32_t a STATUS_PARAM)
+float32 int32_to_float32(int32_t a, float_status *status)
 {
     flag zSign;
 
     if ( a == 0 ) return float32_zero;
     if ( a == (int32_t) 0x80000000 ) return packFloat32( 1, 0x9E, 0 );
     zSign = ( a < 0 );
-    return normalizeRoundAndPackFloat32( zSign, 0x9C, zSign ? - a : a STATUS_VAR );
-
+    return normalizeRoundAndPackFloat32(zSign, 0x9C, zSign ? -a : a, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -1211,11 +1292,11 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 int32_to_float64(int32_t a STATUS_PARAM)
+float64 int32_to_float64(int32_t a, float_status *status)
 {
     flag zSign;
-    uint32 absA;
-    int8 shiftCount;
+    uint32_t absA;
+    int8_t shiftCount;
     uint64_t zSig;
 
     if ( a == 0 ) return float64_zero;
@@ -1234,11 +1315,11 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 int32_to_floatx80(int32_t a STATUS_PARAM)
+floatx80 int32_to_floatx80(int32_t a, float_status *status)
 {
     flag zSign;
-    uint32 absA;
-    int8 shiftCount;
+    uint32_t absA;
+    int8_t shiftCount;
     uint64_t zSig;
 
     if ( a == 0 ) return packFloatx80( 0, 0, 0 );
@@ -1256,11 +1337,11 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 int32_to_float128(int32_t a STATUS_PARAM)
+float128 int32_to_float128(int32_t a, float_status *status)
 {
     flag zSign;
-    uint32 absA;
-    int8 shiftCount;
+    uint32_t absA;
+    int8_t shiftCount;
     uint64_t zSig0;
 
     if ( a == 0 ) return packFloat128( 0, 0, 0, 0 );
@@ -1278,11 +1359,11 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 int64_to_float32(int64_t a STATUS_PARAM)
+float32 int64_to_float32(int64_t a, float_status *status)
 {
     flag zSign;
-    uint64 absA;
-    int8 shiftCount;
+    uint64_t absA;
+    int8_t shiftCount;
 
     if ( a == 0 ) return float32_zero;
     zSign = ( a < 0 );
@@ -1299,39 +1380,18 @@
         else {
             absA <<= shiftCount;
         }
-        return roundAndPackFloat32( zSign, 0x9C - shiftCount, absA STATUS_VAR );
+        return roundAndPackFloat32(zSign, 0x9C - shiftCount, absA, status);
     }
 
 }
 
-float32 uint64_to_float32(uint64_t a STATUS_PARAM)
-{
-    int8 shiftCount;
-
-    if ( a == 0 ) return float32_zero;
-    shiftCount = countLeadingZeros64( a ) - 40;
-    if ( 0 <= shiftCount ) {
-        return packFloat32(0, 0x95 - shiftCount, a<<shiftCount);
-    }
-    else {
-        shiftCount += 7;
-        if ( shiftCount < 0 ) {
-            shift64RightJamming( a, - shiftCount, &a );
-        }
-        else {
-            a <<= shiftCount;
-        }
-        return roundAndPackFloat32(0, 0x9C - shiftCount, a STATUS_VAR);
-    }
-}
-
 /*----------------------------------------------------------------------------
 | Returns the result of converting the 64-bit two's complement integer `a'
 | to the double-precision floating-point format.  The conversion is performed
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 int64_to_float64(int64_t a STATUS_PARAM)
+float64 int64_to_float64(int64_t a, float_status *status)
 {
     flag zSign;
 
@@ -1340,22 +1400,7 @@
         return packFloat64( 1, 0x43E, 0 );
     }
     zSign = ( a < 0 );
-    return normalizeRoundAndPackFloat64( zSign, 0x43C, zSign ? - a : a STATUS_VAR );
-
-}
-
-float64 uint64_to_float64(uint64_t a STATUS_PARAM)
-{
-    int exp =  0x43C;
-
-    if (a == 0) {
-        return float64_zero;
-    }
-    if ((int64_t)a < 0) {
-        shift64RightJamming(a, 1, &a);
-        exp += 1;
-    }
-    return normalizeRoundAndPackFloat64(0, exp, a STATUS_VAR);
+    return normalizeRoundAndPackFloat64(zSign, 0x43C, zSign ? -a : a, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -1365,11 +1410,11 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 int64_to_floatx80(int64_t a STATUS_PARAM)
+floatx80 int64_to_floatx80(int64_t a, float_status *status)
 {
     flag zSign;
-    uint64 absA;
-    int8 shiftCount;
+    uint64_t absA;
+    int8_t shiftCount;
 
     if ( a == 0 ) return packFloatx80( 0, 0, 0 );
     zSign = ( a < 0 );
@@ -1385,12 +1430,12 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 int64_to_float128(int64_t a STATUS_PARAM)
+float128 int64_to_float128(int64_t a, float_status *status)
 {
     flag zSign;
-    uint64 absA;
-    int8 shiftCount;
-    int32 zExp;
+    uint64_t absA;
+    int8_t shiftCount;
+    int32_t zExp;
     uint64_t zSig0, zSig1;
 
     if ( a == 0 ) return packFloat128( 0, 0, 0, 0 );
@@ -1412,12 +1457,77 @@
 
 }
 
-float128 uint64_to_float128(uint64_t a STATUS_PARAM)
+/*----------------------------------------------------------------------------
+| Returns the result of converting the 64-bit unsigned integer `a'
+| to the single-precision floating-point format.  The conversion is performed
+| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+*----------------------------------------------------------------------------*/
+
+float32 uint64_to_float32(uint64_t a, float_status *status)
+{
+    int shiftcount;
+
+    if (a == 0) {
+        return float32_zero;
+    }
+
+    /* Determine (left) shift needed to put first set bit into bit posn 23
+     * (since packFloat32() expects the binary point between bits 23 and 22);
+     * this is the fast case for smallish numbers.
+     */
+    shiftcount = countLeadingZeros64(a) - 40;
+    if (shiftcount >= 0) {
+        return packFloat32(0, 0x95 - shiftcount, a << shiftcount);
+    }
+    /* Otherwise we need to do a round-and-pack. roundAndPackFloat32()
+     * expects the binary point between bits 30 and 29, hence the + 7.
+     */
+    shiftcount += 7;
+    if (shiftcount < 0) {
+        shift64RightJamming(a, -shiftcount, &a);
+    } else {
+        a <<= shiftcount;
+    }
+
+    return roundAndPackFloat32(0, 0x9c - shiftcount, a, status);
+}
+
+/*----------------------------------------------------------------------------
+| Returns the result of converting the 64-bit unsigned integer `a'
+| to the double-precision floating-point format.  The conversion is performed
+| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+*----------------------------------------------------------------------------*/
+
+float64 uint64_to_float64(uint64_t a, float_status *status)
+{
+    int exp = 0x43C;
+    int shiftcount;
+
+    if (a == 0) {
+        return float64_zero;
+    }
+
+    shiftcount = countLeadingZeros64(a) - 1;
+    if (shiftcount < 0) {
+        shift64RightJamming(a, -shiftcount, &a);
+    } else {
+        a <<= shiftcount;
+    }
+    return roundAndPackFloat64(0, exp - shiftcount, a, status);
+}
+
+/*----------------------------------------------------------------------------
+| Returns the result of converting the 64-bit unsigned integer `a'
+| to the quadruple-precision floating-point format.  The conversion is performed
+| according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
+*----------------------------------------------------------------------------*/
+
+float128 uint64_to_float128(uint64_t a, float_status *status)
 {
     if (a == 0) {
         return float128_zero;
     }
-    return normalizeRoundAndPackFloat128(0, 0x406E, a, 0 STATUS_VAR);
+    return normalizeRoundAndPackFloat128(0, 0x406E, a, 0, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -1430,14 +1540,15 @@
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 float32_to_int32( float32 a STATUS_PARAM )
+int32_t float32_to_int32(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
@@ -1447,7 +1558,7 @@
     aSig64 = aSig;
     aSig64 <<= 32;
     if ( 0 < shiftCount ) shift64RightJamming( aSig64, shiftCount, &aSig64 );
-    return roundAndPackInt32( aSign, aSig64 STATUS_VAR );
+    return roundAndPackInt32(aSign, aSig64, status);
 
 }
 
@@ -1461,13 +1572,14 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int32 float32_to_int32_round_to_zero( float32 a STATUS_PARAM )
+int32_t float32_to_int32_round_to_zero(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     int32_t z;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -1475,19 +1587,21 @@
     shiftCount = aExp - 0x9E;
     if ( 0 <= shiftCount ) {
         if ( float32_val(a) != 0xCF000000 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if ( ! aSign || ( ( aExp == 0xFF ) && aSig ) ) return 0x7FFFFFFF;
         }
         return (int32_t) 0x80000000;
     }
     else if ( aExp <= 0x7E ) {
-        if ( aExp | aSig ) STATUS(float_exception_flags) |= float_flag_inexact;
+        if (aExp | aSig) {
+            status->float_exception_flags |= float_flag_inexact;
+        }
         return 0;
     }
     aSig = ( aSig | 0x00800000 )<<8;
     z = aSig>>( - shiftCount );
     if ( (uint32_t) ( aSig<<( shiftCount & 31 ) ) ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     if ( aSign ) z = - z;
     return z;
@@ -1504,12 +1618,13 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int_fast16_t float32_to_int16_round_to_zero(float32 a STATUS_PARAM)
+int16_t float32_to_int16_round_to_zero(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
-    int32 z;
+    int32_t z;
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -1517,7 +1632,7 @@
     shiftCount = aExp - 0x8E;
     if ( 0 <= shiftCount ) {
         if ( float32_val(a) != 0xC7000000 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if ( ! aSign || ( ( aExp == 0xFF ) && aSig ) ) {
                 return 0x7FFF;
             }
@@ -1526,7 +1641,7 @@
     }
     else if ( aExp <= 0x7E ) {
         if ( aExp | aSig ) {
-            STATUS(float_exception_flags) |= float_flag_inexact;
+            status->float_exception_flags |= float_flag_inexact;
         }
         return 0;
     }
@@ -1534,7 +1649,7 @@
     aSig = ( aSig | 0x00800000 )<<8;
     z = aSig>>( - shiftCount );
     if ( (uint32_t) ( aSig<<( shiftCount & 31 ) ) ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     if ( aSign ) {
         z = - z;
@@ -1553,20 +1668,21 @@
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 float32_to_int64( float32 a STATUS_PARAM )
+int64_t float32_to_int64(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64, aSigExtra;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
     shiftCount = 0xBE - aExp;
     if ( shiftCount < 0 ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         if ( ! aSign || ( ( aExp == 0xFF ) && aSig ) ) {
             return LIT64( 0x7FFFFFFFFFFFFFFF );
         }
@@ -1576,7 +1692,7 @@
     aSig64 = aSig;
     aSig64 <<= 40;
     shift64ExtraRightJamming( aSig64, 0, shiftCount, &aSig64, &aSigExtra );
-    return roundAndPackInt64( aSign, aSig64, aSigExtra STATUS_VAR );
+    return roundAndPackInt64(aSign, aSig64, aSigExtra, status);
 
 }
 
@@ -1592,19 +1708,20 @@
 | raise the inexact exception flag.
 *----------------------------------------------------------------------------*/
 
-uint64 float32_to_uint64(float32 a STATUS_PARAM)
+uint64_t float32_to_uint64(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64, aSigExtra;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac(a);
     aExp = extractFloat32Exp(a);
     aSign = extractFloat32Sign(a);
     if ((aSign) && (aExp > 126)) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         if (float32_is_any_nan(a)) {
             return LIT64(0xFFFFFFFFFFFFFFFF);
         } else {
@@ -1616,14 +1733,14 @@
         aSig |= 0x00800000;
     }
     if (shiftCount < 0) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return LIT64(0xFFFFFFFFFFFFFFFF);
     }
 
     aSig64 = aSig;
     aSig64 <<= 40;
     shift64ExtraRightJamming(aSig64, 0, shiftCount, &aSig64, &aSigExtra);
-    return roundAndPackUint64(aSign, aSig64, aSigExtra STATUS_VAR);
+    return roundAndPackUint64(aSign, aSig64, aSigExtra, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -1637,12 +1754,12 @@
 | not round to zero will raise the inexact flag.
 *----------------------------------------------------------------------------*/
 
-uint64 float32_to_uint64_round_to_zero(float32 a STATUS_PARAM)
+uint64_t float32_to_uint64_round_to_zero(float32 a, float_status *status)
 {
-    signed char current_rounding_mode = STATUS(float_rounding_mode);
-    set_float_rounding_mode(float_round_to_zero STATUS_VAR);
-    int64_t v = float32_to_uint64(a STATUS_VAR);
-    set_float_rounding_mode(current_rounding_mode STATUS_VAR);
+    signed char current_rounding_mode = status->float_rounding_mode;
+    set_float_rounding_mode(float_round_to_zero, status);
+    int64_t v = float32_to_uint64(a, status);
+    set_float_rounding_mode(current_rounding_mode, status);
     return v;
 }
 
@@ -1656,14 +1773,15 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int64 float32_to_int64_round_to_zero( float32 a STATUS_PARAM )
+int64_t float32_to_int64_round_to_zero(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint32_t aSig;
     uint64_t aSig64;
-    int64 z;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    int64_t z;
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -1671,7 +1789,7 @@
     shiftCount = aExp - 0xBE;
     if ( 0 <= shiftCount ) {
         if ( float32_val(a) != 0xDF000000 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if ( ! aSign || ( ( aExp == 0xFF ) && aSig ) ) {
                 return LIT64( 0x7FFFFFFFFFFFFFFF );
             }
@@ -1679,14 +1797,16 @@
         return (int64_t) LIT64( 0x8000000000000000 );
     }
     else if ( aExp <= 0x7E ) {
-        if ( aExp | aSig ) STATUS(float_exception_flags) |= float_flag_inexact;
+        if (aExp | aSig) {
+            status->float_exception_flags |= float_flag_inexact;
+        }
         return 0;
     }
     aSig64 = aSig | 0x00800000;
     aSig64 <<= 40;
     z = aSig64>>( - shiftCount );
     if ( (uint64_t) ( aSig64<<( shiftCount & 63 ) ) ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     if ( aSign ) z = - z;
     return z;
@@ -1700,18 +1820,20 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float32_to_float64( float32 a STATUS_PARAM )
+float64 float32_to_float64(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
     if ( aExp == 0xFF ) {
-        if ( aSig ) return commonNaNToFloat64( float32ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        if (aSig) {
+            return commonNaNToFloat64(float32ToCommonNaN(a, status), status);
+        }
         return packFloat64( aSign, 0x7FF, 0 );
     }
     if ( aExp == 0 ) {
@@ -1730,18 +1852,20 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 float32_to_floatx80( float32 a STATUS_PARAM )
+floatx80 float32_to_floatx80(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
     if ( aExp == 0xFF ) {
-        if ( aSig ) return commonNaNToFloatx80( float32ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        if (aSig) {
+            return commonNaNToFloatx80(float32ToCommonNaN(a, status), status);
+        }
         return packFloatx80( aSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
     if ( aExp == 0 ) {
@@ -1760,18 +1884,20 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float32_to_float128( float32 a STATUS_PARAM )
+float128 float32_to_float128(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
     if ( aExp == 0xFF ) {
-        if ( aSig ) return commonNaNToFloat128( float32ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        if (aSig) {
+            return commonNaNToFloat128(float32ToCommonNaN(a, status), status);
+        }
         return packFloat128( aSign, 0x7FFF, 0, 0 );
     }
     if ( aExp == 0 ) {
@@ -1790,26 +1916,26 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_round_to_int( float32 a STATUS_PARAM)
+float32 float32_round_to_int(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t lastBitMask, roundBitsMask;
     uint32_t z;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aExp = extractFloat32Exp( a );
     if ( 0x96 <= aExp ) {
         if ( ( aExp == 0xFF ) && extractFloat32Frac( a ) ) {
-            return propagateFloat32NaN( a, a STATUS_VAR );
+            return propagateFloat32NaN(a, a, status);
         }
         return a;
     }
     if ( aExp <= 0x7E ) {
         if ( (uint32_t) ( float32_val(a)<<1 ) == 0 ) return a;
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
         aSign = extractFloat32Sign( a );
-        switch ( STATUS(float_rounding_mode) ) {
+        switch (status->float_rounding_mode) {
          case float_round_nearest_even:
             if ( ( aExp == 0x7E ) && extractFloat32Frac( a ) ) {
                 return packFloat32( aSign, 0x7F, 0 );
@@ -1831,7 +1957,7 @@
     lastBitMask <<= 0x96 - aExp;
     roundBitsMask = lastBitMask - 1;
     z = float32_val(a);
-    switch (STATUS(float_rounding_mode)) {
+    switch (status->float_rounding_mode) {
     case float_round_nearest_even:
         z += lastBitMask>>1;
         if ((z & roundBitsMask) == 0) {
@@ -1857,7 +1983,9 @@
         abort();
     }
     z &= ~ roundBitsMask;
-    if ( z != float32_val(a) ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (z != float32_val(a)) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     return make_float32(z);
 
 }
@@ -1870,11 +1998,12 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float32 addFloat32Sigs( float32 a, float32 b, flag zSign STATUS_PARAM)
+static float32 addFloat32Sigs(float32 a, float32 b, flag zSign,
+                              float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -1885,7 +2014,9 @@
     bSig <<= 6;
     if ( 0 < expDiff ) {
         if ( aExp == 0xFF ) {
-            if ( aSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+            if (aSig) {
+                return propagateFloat32NaN(a, b, status);
+            }
             return a;
         }
         if ( bExp == 0 ) {
@@ -1899,7 +2030,9 @@
     }
     else if ( expDiff < 0 ) {
         if ( bExp == 0xFF ) {
-            if ( bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+            if (bSig) {
+                return propagateFloat32NaN(a, b, status);
+            }
             return packFloat32( zSign, 0xFF, 0 );
         }
         if ( aExp == 0 ) {
@@ -1913,13 +2046,15 @@
     }
     else {
         if ( aExp == 0xFF ) {
-            if ( aSig | bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+            if (aSig | bSig) {
+                return propagateFloat32NaN(a, b, status);
+            }
             return a;
         }
         if ( aExp == 0 ) {
-            if (STATUS(flush_to_zero)) {
+            if (status->flush_to_zero) {
                 if (aSig | bSig) {
-                    float_raise(float_flag_output_denormal STATUS_VAR);
+                    float_raise(float_flag_output_denormal, status);
                 }
                 return packFloat32(zSign, 0, 0);
             }
@@ -1937,7 +2072,7 @@
         ++zExp;
     }
  roundAndPack:
-    return roundAndPackFloat32( zSign, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat32(zSign, zExp, zSig, status);
 
 }
 
@@ -1949,11 +2084,12 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float32 subFloat32Sigs( float32 a, float32 b, flag zSign STATUS_PARAM)
+static float32 subFloat32Sigs(float32 a, float32 b, flag zSign,
+                              float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -1965,9 +2101,11 @@
     if ( 0 < expDiff ) goto aExpBigger;
     if ( expDiff < 0 ) goto bExpBigger;
     if ( aExp == 0xFF ) {
-        if ( aSig | bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float32_default_nan;
+        if (aSig | bSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
+        float_raise(float_flag_invalid, status);
+        return float32_default_nan(status);
     }
     if ( aExp == 0 ) {
         aExp = 1;
@@ -1975,10 +2113,12 @@
     }
     if ( bSig < aSig ) goto aBigger;
     if ( aSig < bSig ) goto bBigger;
-    return packFloat32( STATUS(float_rounding_mode) == float_round_down, 0, 0 );
+    return packFloat32(status->float_rounding_mode == float_round_down, 0, 0);
  bExpBigger:
     if ( bExp == 0xFF ) {
-        if ( bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
         return packFloat32( zSign ^ 1, 0xFF, 0 );
     }
     if ( aExp == 0 ) {
@@ -1996,7 +2136,9 @@
     goto normalizeRoundAndPack;
  aExpBigger:
     if ( aExp == 0xFF ) {
-        if ( aSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+        if (aSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
@@ -2012,7 +2154,7 @@
     zExp = aExp;
  normalizeRoundAndPack:
     --zExp;
-    return normalizeRoundAndPackFloat32( zSign, zExp, zSig STATUS_VAR );
+    return normalizeRoundAndPackFloat32(zSign, zExp, zSig, status);
 
 }
 
@@ -2022,19 +2164,19 @@
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_add( float32 a, float32 b STATUS_PARAM )
+float32 float32_add(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     aSign = extractFloat32Sign( a );
     bSign = extractFloat32Sign( b );
     if ( aSign == bSign ) {
-        return addFloat32Sigs( a, b, aSign STATUS_VAR);
+        return addFloat32Sigs(a, b, aSign, status);
     }
     else {
-        return subFloat32Sigs( a, b, aSign STATUS_VAR );
+        return subFloat32Sigs(a, b, aSign, status);
     }
 
 }
@@ -2045,19 +2187,19 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_sub( float32 a, float32 b STATUS_PARAM )
+float32 float32_sub(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     aSign = extractFloat32Sign( a );
     bSign = extractFloat32Sign( b );
     if ( aSign == bSign ) {
-        return subFloat32Sigs( a, b, aSign STATUS_VAR );
+        return subFloat32Sigs(a, b, aSign, status);
     }
     else {
-        return addFloat32Sigs( a, b, aSign STATUS_VAR );
+        return addFloat32Sigs(a, b, aSign, status);
     }
 
 }
@@ -2068,16 +2210,16 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_mul( float32 a, float32 b STATUS_PARAM )
+float32 float32_mul(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig;
     uint64_t zSig64;
     uint32_t zSig;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -2088,19 +2230,21 @@
     zSign = aSign ^ bSign;
     if ( aExp == 0xFF ) {
         if ( aSig || ( ( bExp == 0xFF ) && bSig ) ) {
-            return propagateFloat32NaN( a, b STATUS_VAR );
+            return propagateFloat32NaN(a, b, status);
         }
         if ( ( bExp | bSig ) == 0 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float32_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float32_default_nan(status);
         }
         return packFloat32( zSign, 0xFF, 0 );
     }
     if ( bExp == 0xFF ) {
-        if ( bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
         if ( ( aExp | aSig ) == 0 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float32_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float32_default_nan(status);
         }
         return packFloat32( zSign, 0xFF, 0 );
     }
@@ -2121,7 +2265,7 @@
         zSig <<= 1;
         --zExp;
     }
-    return roundAndPackFloat32( zSign, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat32(zSign, zExp, zSig, status);
 
 }
 
@@ -2131,13 +2275,13 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_div( float32 a, float32 b STATUS_PARAM )
+float32 float32_div(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint32_t aSig, bSig, zSig;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -2147,25 +2291,31 @@
     bSign = extractFloat32Sign( b );
     zSign = aSign ^ bSign;
     if ( aExp == 0xFF ) {
-        if ( aSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+        if (aSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
         if ( bExp == 0xFF ) {
-            if ( bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float32_default_nan;
+            if (bSig) {
+                return propagateFloat32NaN(a, b, status);
+            }
+            float_raise(float_flag_invalid, status);
+            return float32_default_nan(status);
         }
         return packFloat32( zSign, 0xFF, 0 );
     }
     if ( bExp == 0xFF ) {
-        if ( bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
         return packFloat32( zSign, 0, 0 );
     }
     if ( bExp == 0 ) {
         if ( bSig == 0 ) {
             if ( ( aExp | aSig ) == 0 ) {
-                float_raise( float_flag_invalid STATUS_VAR);
-                return float32_default_nan;
+                float_raise(float_flag_invalid, status);
+                return float32_default_nan(status);
             }
-            float_raise( float_flag_divbyzero STATUS_VAR);
+            float_raise(float_flag_divbyzero, status);
             return packFloat32( zSign, 0xFF, 0 );
         }
         normalizeFloat32Subnormal( bSig, &bExp, &bSig );
@@ -2185,7 +2335,7 @@
     if ( ( zSig & 0x3F ) == 0 ) {
         zSig |= ( (uint64_t) bSig * zSig != ( (uint64_t) aSig )<<32 );
     }
-    return roundAndPackFloat32( zSign, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat32(zSign, zExp, zSig, status);
 
 }
 
@@ -2195,17 +2345,17 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_rem( float32 a, float32 b STATUS_PARAM )
+float32 float32_rem(float32 a, float32 b, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp, bExp, expDiff;
+    int aExp, bExp, expDiff;
     uint32_t aSig, bSig;
     uint32_t q;
     uint64_t aSig64, bSig64, q64;
     uint32_t alternateASig;
     int32_t sigMean;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -2214,19 +2364,21 @@
     bExp = extractFloat32Exp( b );
     if ( aExp == 0xFF ) {
         if ( aSig || ( ( bExp == 0xFF ) && bSig ) ) {
-            return propagateFloat32NaN( a, b STATUS_VAR );
+            return propagateFloat32NaN(a, b, status);
         }
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float32_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float32_default_nan(status);
     }
     if ( bExp == 0xFF ) {
-        if ( bSig ) return propagateFloat32NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat32NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
         if ( bSig == 0 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float32_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float32_default_nan(status);
         }
         normalizeFloat32Subnormal( bSig, &bExp, &bSig );
     }
@@ -2286,8 +2438,7 @@
     }
     zSign = ( (int32_t) aSig < 0 );
     if ( zSign ) aSig = - aSig;
-    return normalizeRoundAndPackFloat32( aSign ^ zSign, bExp, aSig STATUS_VAR );
-
+    return normalizeRoundAndPackFloat32(aSign ^ zSign, bExp, aSig, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -2301,10 +2452,11 @@
 | externally will flip the sign bit on NaNs.)
 *----------------------------------------------------------------------------*/
 
-float32 float32_muladd(float32 a, float32 b, float32 c, int flags STATUS_PARAM)
+float32 float32_muladd(float32 a, float32 b, float32 c, int flags,
+                       float_status *status)
 {
     flag aSign, bSign, cSign, zSign;
-    int_fast16_t aExp, bExp, cExp, pExp, zExp, expDiff;
+    int aExp, bExp, cExp, pExp, zExp, expDiff;
     uint32_t aSig, bSig, cSig;
     flag pInf, pZero, pSign;
     uint64_t pSig64, cSig64, zSig64;
@@ -2312,9 +2464,9 @@
     int shiftcount;
     flag signflip, infzero;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
-    c = float32_squash_input_denormal(c STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
+    c = float32_squash_input_denormal(c, status);
     aSig = extractFloat32Frac(a);
     aExp = extractFloat32Exp(a);
     aSign = extractFloat32Sign(a);
@@ -2336,12 +2488,12 @@
     if (((aExp == 0xff) && aSig) ||
         ((bExp == 0xff) && bSig) ||
         ((cExp == 0xff) && cSig)) {
-        return propagateFloat32MulAddNaN(a, b, c, infzero STATUS_VAR);
+        return propagateFloat32MulAddNaN(a, b, c, infzero, status);
     }
 
     if (infzero) {
-        float_raise(float_flag_invalid STATUS_VAR);
-        return float32_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float32_default_nan(status);
     }
 
     if (flags & float_muladd_negate_c) {
@@ -2361,8 +2513,8 @@
     if (cExp == 0xff) {
         if (pInf && (pSign ^ cSign)) {
             /* addition of opposite-signed infinities => InvalidOperation */
-            float_raise(float_flag_invalid STATUS_VAR);
-            return float32_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float32_default_nan(status);
         }
         /* Otherwise generate an infinity of the same sign */
         return packFloat32(cSign ^ signflip, 0xff, 0);
@@ -2378,7 +2530,7 @@
                 /* Adding two exact zeroes */
                 if (pSign == cSign) {
                     zSign = pSign;
-                } else if (STATUS(float_rounding_mode) == float_round_down) {
+                } else if (status->float_rounding_mode == float_round_down) {
                     zSign = 1;
                 } else {
                     zSign = 0;
@@ -2386,8 +2538,8 @@
                 return packFloat32(zSign ^ signflip, 0, 0);
             }
             /* Exact zero plus a denorm */
-            if (STATUS(flush_to_zero)) {
-                float_raise(float_flag_output_denormal STATUS_VAR);
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
                 return packFloat32(cSign ^ signflip, 0, 0);
             }
         }
@@ -2401,7 +2553,7 @@
              */
             cExp -= 2;
             cSig = (cSig | 0x00800000) << 7;
-            return roundAndPackFloat32(cSign ^ signflip, cExp, cSig STATUS_VAR);
+            return roundAndPackFloat32(cSign ^ signflip, cExp, cSig, status);
         }
         return packFloat32(cSign ^ signflip, cExp, cSig);
     }
@@ -2443,7 +2595,7 @@
                 pExp--;
             }
             return roundAndPackFloat32(zSign, pExp - 1,
-                                       pSig STATUS_VAR);
+                                       pSig, status);
         }
         normalizeFloat32Subnormal(cSig, &cExp, &cSig);
     }
@@ -2494,7 +2646,7 @@
             } else {
                 /* Exact zero */
                 zSign = signflip;
-                if (STATUS(float_rounding_mode) == float_round_down) {
+                if (status->float_rounding_mode == float_round_down) {
                     zSign ^= 1;
                 }
                 return packFloat32(zSign, 0, 0);
@@ -2511,7 +2663,7 @@
     }
 
     shift64RightJamming(zSig64, 32, &zSig64);
-    return roundAndPackFloat32(zSign, zExp, zSig64 STATUS_VAR);
+    return roundAndPackFloat32(zSign, zExp, zSig64, status);
 }
 
 
@@ -2521,27 +2673,29 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float32_sqrt( float32 a STATUS_PARAM )
+float32 float32_sqrt(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, zExp;
+    int aExp, zExp;
     uint32_t aSig, zSig;
     uint64_t rem, term;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
     if ( aExp == 0xFF ) {
-        if ( aSig ) return propagateFloat32NaN( a, float32_zero STATUS_VAR );
+        if (aSig) {
+            return propagateFloat32NaN(a, float32_zero, status);
+        }
         if ( ! aSign ) return a;
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float32_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float32_default_nan(status);
     }
     if ( aSign ) {
         if ( ( aExp | aSig ) == 0 ) return a;
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float32_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float32_default_nan(status);
     }
     if ( aExp == 0 ) {
         if ( aSig == 0 ) return float32_zero;
@@ -2566,7 +2720,7 @@
     }
     shift32RightJamming( zSig, 1, &zSig );
  roundAndPack:
-    return roundAndPackFloat32( 0, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat32(0, zExp, zSig, status);
 
 }
 
@@ -2607,44 +2761,46 @@
     const_float64( 0x3d6ae7f3e733b81fll ), /* 15 */
 };
 
-float32 float32_exp2( float32 a STATUS_PARAM )
+float32 float32_exp2(float32 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
     float64 r, x, xn;
     int i;
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
 
     if ( aExp == 0xFF) {
-        if ( aSig ) return propagateFloat32NaN( a, float32_zero STATUS_VAR );
+        if (aSig) {
+            return propagateFloat32NaN(a, float32_zero, status);
+        }
         return (aSign) ? float32_zero : a;
     }
     if (aExp == 0) {
         if (aSig == 0) return float32_one;
     }
 
-    float_raise( float_flag_inexact STATUS_VAR);
+    float_raise(float_flag_inexact, status);
 
     /* ******************************* */
     /* using float64 for approximation */
     /* ******************************* */
-    x = float32_to_float64(a STATUS_VAR);
-    x = float64_mul(x, float64_ln2 STATUS_VAR);
+    x = float32_to_float64(a, status);
+    x = float64_mul(x, float64_ln2, status);
 
     xn = x;
     r = float64_one;
     for (i = 0 ; i < 15 ; i++) {
         float64 f;
 
-        f = float64_mul(xn, float32_exp2_coefficients[i] STATUS_VAR);
-        r = float64_add(r, f STATUS_VAR);
+        f = float64_mul(xn, float32_exp2_coefficients[i], status);
+        r = float64_add(r, f, status);
 
-        xn = float64_mul(xn, x STATUS_VAR);
+        xn = float64_mul(xn, x, status);
     }
 
     return float64_to_float32(r, status);
@@ -2655,13 +2811,13 @@
 | The operation is performed according to the IEC/IEEE Standard for Binary
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
-float32 float32_log2( float32 a STATUS_PARAM )
+float32 float32_log2(float32 a, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig, zSig, i;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
@@ -2671,11 +2827,13 @@
         normalizeFloat32Subnormal( aSig, &aExp, &aSig );
     }
     if ( aSign ) {
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float32_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float32_default_nan(status);
     }
     if ( aExp == 0xFF ) {
-        if ( aSig ) return propagateFloat32NaN( a, float32_zero STATUS_VAR );
+        if (aSig) {
+            return propagateFloat32NaN(a, float32_zero, status);
+        }
         return a;
     }
 
@@ -2695,7 +2853,7 @@
     if ( zSign )
         zSig = -zSig;
 
-    return normalizeRoundAndPackFloat32( zSign, 0x85, zSig STATUS_VAR );
+    return normalizeRoundAndPackFloat32(zSign, 0x85, zSig, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -2705,16 +2863,16 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_eq( float32 a, float32 b STATUS_PARAM )
+int float32_eq(float32 a, float32 b, float_status *status)
 {
     uint32_t av, bv;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     av = float32_val(a);
@@ -2729,17 +2887,17 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_le( float32 a, float32 b STATUS_PARAM )
+int float32_le(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign;
     uint32_t av, bv;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloat32Sign( a );
@@ -2758,17 +2916,17 @@
 | to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_lt( float32 a, float32 b STATUS_PARAM )
+int float32_lt(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign;
     uint32_t av, bv;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloat32Sign( a );
@@ -2787,15 +2945,15 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_unordered( float32 a, float32 b STATUS_PARAM )
+int float32_unordered(float32 a, float32 b, float_status *status)
 {
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 1;
     }
     return 0;
@@ -2808,16 +2966,17 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_eq_quiet( float32 a, float32 b STATUS_PARAM )
+int float32_eq_quiet(float32 a, float32 b, float_status *status)
 {
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        if ( float32_is_signaling_nan( a ) || float32_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float32_is_signaling_nan(a, status)
+         || float32_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -2832,18 +2991,19 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_le_quiet( float32 a, float32 b STATUS_PARAM )
+int float32_le_quiet(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign;
     uint32_t av, bv;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        if ( float32_is_signaling_nan( a ) || float32_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float32_is_signaling_nan(a, status)
+         || float32_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -2863,18 +3023,19 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_lt_quiet( float32 a, float32 b STATUS_PARAM )
+int float32_lt_quiet(float32 a, float32 b, float_status *status)
 {
     flag aSign, bSign;
     uint32_t av, bv;
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        if ( float32_is_signaling_nan( a ) || float32_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float32_is_signaling_nan(a, status)
+         || float32_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -2894,16 +3055,17 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float32_unordered_quiet( float32 a, float32 b STATUS_PARAM )
+int float32_unordered_quiet(float32 a, float32 b, float_status *status)
 {
-    a = float32_squash_input_denormal(a STATUS_VAR);
-    b = float32_squash_input_denormal(b STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
+    b = float32_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat32Exp( a ) == 0xFF ) && extractFloat32Frac( a ) )
          || ( ( extractFloat32Exp( b ) == 0xFF ) && extractFloat32Frac( b ) )
        ) {
-        if ( float32_is_signaling_nan( a ) || float32_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float32_is_signaling_nan(a, status)
+         || float32_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 1;
     }
@@ -2920,12 +3082,13 @@
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 float64_to_int32( float64 a STATUS_PARAM )
+int32_t float64_to_int32(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -2934,7 +3097,7 @@
     if ( aExp ) aSig |= LIT64( 0x0010000000000000 );
     shiftCount = 0x42C - aExp;
     if ( 0 < shiftCount ) shift64RightJamming( aSig, shiftCount, &aSig );
-    return roundAndPackInt32( aSign, aSig STATUS_VAR );
+    return roundAndPackInt32(aSign, aSig, status);
 
 }
 
@@ -2948,13 +3111,14 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int32 float64_to_int32_round_to_zero( float64 a STATUS_PARAM )
+int32_t float64_to_int32_round_to_zero(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, savedASig;
     int32_t z;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -2964,7 +3128,9 @@
         goto invalid;
     }
     else if ( aExp < 0x3FF ) {
-        if ( aExp || aSig ) STATUS(float_exception_flags) |= float_flag_inexact;
+        if (aExp || aSig) {
+            status->float_exception_flags |= float_flag_inexact;
+        }
         return 0;
     }
     aSig |= LIT64( 0x0010000000000000 );
@@ -2975,11 +3141,11 @@
     if ( aSign ) z = - z;
     if ( ( z < 0 ) ^ aSign ) {
  invalid:
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return aSign ? (int32_t) 0x80000000 : 0x7FFFFFFF;
     }
     if ( ( aSig<<shiftCount ) != savedASig ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     return z;
 
@@ -2995,12 +3161,13 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int_fast16_t float64_to_int16_round_to_zero(float64 a STATUS_PARAM)
+int16_t float64_to_int16_round_to_zero(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, savedASig;
-    int32 z;
+    int32_t z;
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3013,7 +3180,7 @@
     }
     else if ( aExp < 0x3FF ) {
         if ( aExp || aSig ) {
-            STATUS(float_exception_flags) |= float_flag_inexact;
+            status->float_exception_flags |= float_flag_inexact;
         }
         return 0;
     }
@@ -3027,11 +3194,11 @@
     }
     if ( ( (int16_t)z < 0 ) ^ aSign ) {
  invalid:
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return aSign ? (int32_t) 0xffff8000 : 0x7FFF;
     }
     if ( ( aSig<<shiftCount ) != savedASig ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     return z;
 }
@@ -3046,12 +3213,13 @@
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 float64_to_int64( float64 a STATUS_PARAM )
+int64_t float64_to_int64(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, aSigExtra;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3060,7 +3228,7 @@
     shiftCount = 0x433 - aExp;
     if ( shiftCount <= 0 ) {
         if ( 0x43E < aExp ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if (    ! aSign
                  || (    ( aExp == 0x7FF )
                       && ( aSig != LIT64( 0x0010000000000000 ) ) )
@@ -3075,7 +3243,7 @@
     else {
         shift64ExtraRightJamming( aSig, 0, shiftCount, &aSig, &aSigExtra );
     }
-    return roundAndPackInt64( aSign, aSig, aSigExtra STATUS_VAR );
+    return roundAndPackInt64(aSign, aSig, aSigExtra, status);
 
 }
 
@@ -3089,13 +3257,14 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int64 float64_to_int64_round_to_zero( float64 a STATUS_PARAM )
+int64_t float64_to_int64_round_to_zero(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig;
-    int64 z;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    int64_t z;
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3105,7 +3274,7 @@
     if ( 0 <= shiftCount ) {
         if ( 0x43E <= aExp ) {
             if ( float64_val(a) != LIT64( 0xC3E0000000000000 ) ) {
-                float_raise( float_flag_invalid STATUS_VAR);
+                float_raise(float_flag_invalid, status);
                 if (    ! aSign
                      || (    ( aExp == 0x7FF )
                           && ( aSig != LIT64( 0x0010000000000000 ) ) )
@@ -3119,12 +3288,14 @@
     }
     else {
         if ( aExp < 0x3FE ) {
-            if ( aExp | aSig ) STATUS(float_exception_flags) |= float_flag_inexact;
+            if (aExp | aSig) {
+                status->float_exception_flags |= float_flag_inexact;
+            }
             return 0;
         }
         z = aSig>>( - shiftCount );
         if ( (uint64_t) ( aSig<<( shiftCount & 63 ) ) ) {
-            STATUS(float_exception_flags) |= float_flag_inexact;
+            status->float_exception_flags |= float_flag_inexact;
         }
     }
     if ( aSign ) z = - z;
@@ -3139,19 +3310,21 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float64_to_float32( float64 a STATUS_PARAM )
+float32 float64_to_float32(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig;
     uint32_t zSig;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
     aSign = extractFloat64Sign( a );
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return commonNaNToFloat32( float64ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        if (aSig) {
+            return commonNaNToFloat32(float64ToCommonNaN(a, status), status);
+        }
         return packFloat32( aSign, 0xFF, 0 );
     }
     shift64RightJamming( aSig, 22, &aSig );
@@ -3160,7 +3333,7 @@
         zSig |= 0x40000000;
         aExp -= 0x381;
     }
-    return roundAndPackFloat32( aSign, aExp, zSig STATUS_VAR );
+    return roundAndPackFloat32(aSign, aExp, zSig, status);
 
 }
 
@@ -3175,7 +3348,7 @@
 | than the desired result exponent whenever `zSig' is a complete, normalized
 | significand.
 *----------------------------------------------------------------------------*/
-static float16 packFloat16(flag zSign, int_fast16_t zExp, uint16_t zSig)
+static float16 packFloat16(flag zSign, int zExp, uint16_t zSig)
 {
     return make_float16(
         (((uint32_t)zSign) << 15) + (((uint32_t)zExp) << 10) + zSig);
@@ -3209,8 +3382,9 @@
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float32 roundAndPackFloat16(flag zSign, int_fast16_t zExp,
-                                   uint32_t zSig, flag ieee STATUS_PARAM)
+static float16 roundAndPackFloat16(flag zSign, int zExp,
+                                   uint32_t zSig, flag ieee,
+                                   float_status *status)
 {
     int maxexp = ieee ? 29 : 30;
     uint32_t mask;
@@ -3232,7 +3406,7 @@
         mask = 0x00001fff;
     }
 
-    switch (STATUS(float_rounding_mode)) {
+    switch (status->float_rounding_mode) {
     case float_round_nearest_even:
         increment = (mask + 1) >> 1;
         if ((zSig & mask) == increment) {
@@ -3257,10 +3431,10 @@
 
     if (zExp > maxexp || (zExp == maxexp && rounding_bumps_exp)) {
         if (ieee) {
-            float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+            float_raise(float_flag_overflow | float_flag_inexact, status);
             return packFloat16(zSign, 0x1f, 0);
         } else {
-            float_raise(float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             return packFloat16(zSign, 0x1f, 0x3ff);
         }
     }
@@ -3268,14 +3442,14 @@
     if (zExp < 0) {
         /* Note that flush-to-zero does not affect half-precision results */
         is_tiny =
-            (STATUS(float_detect_tininess) == float_tininess_before_rounding)
+            (status->float_detect_tininess == float_tininess_before_rounding)
             || (zExp < -1)
             || (!rounding_bumps_exp);
     }
     if (zSig & mask) {
-        float_raise(float_flag_inexact STATUS_VAR);
+        float_raise(float_flag_inexact, status);
         if (is_tiny) {
-            float_raise(float_flag_underflow STATUS_VAR);
+            float_raise(float_flag_underflow, status);
         }
     }
 
@@ -3295,7 +3469,7 @@
     return packFloat16(zSign, zExp, zSig >> 13);
 }
 
-static void normalizeFloat16Subnormal(uint32_t aSig, int_fast16_t *zExpPtr,
+static void normalizeFloat16Subnormal(uint32_t aSig, int *zExpPtr,
                                       uint32_t *zSigPtr)
 {
     int8_t shiftCount = countLeadingZeros32(aSig) - 21;
@@ -3306,10 +3480,10 @@
 /* Half precision floats come in two formats: standard IEEE and "ARM" format.
    The latter gains extra exponent range by omitting the NaN/Inf encodings.  */
 
-float32 float16_to_float32(float16 a, flag ieee STATUS_PARAM)
+float32 float16_to_float32(float16 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     aSign = extractFloat16Sign(a);
@@ -3318,7 +3492,7 @@
 
     if (aExp == 0x1f && ieee) {
         if (aSig) {
-            return commonNaNToFloat32(float16ToCommonNaN(a STATUS_VAR) STATUS_VAR);
+            return commonNaNToFloat32(float16ToCommonNaN(a, status), status);
         }
         return packFloat32(aSign, 0xff, 0);
     }
@@ -3333,13 +3507,13 @@
     return packFloat32( aSign, aExp + 0x70, aSig << 13);
 }
 
-float16 float32_to_float16(float32 a, flag ieee STATUS_PARAM)
+float16 float32_to_float16(float32 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
 
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
@@ -3348,15 +3522,15 @@
         if (aSig) {
             /* Input is a NaN */
             if (!ieee) {
-                float_raise(float_flag_invalid STATUS_VAR);
+                float_raise(float_flag_invalid, status);
                 return packFloat16(aSign, 0, 0);
             }
             return commonNaNToFloat16(
-                float32ToCommonNaN(a STATUS_VAR) STATUS_VAR);
+                float32ToCommonNaN(a, status), status);
         }
         /* Infinity */
         if (!ieee) {
-            float_raise(float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             return packFloat16(aSign, 0x1f, 0x3ff);
         }
         return packFloat16(aSign, 0x1f, 0);
@@ -3374,13 +3548,13 @@
     aSig |= 0x00800000;
     aExp -= 0x71;
 
-    return roundAndPackFloat16(aSign, aExp, aSig, ieee STATUS_VAR);
+    return roundAndPackFloat16(aSign, aExp, aSig, ieee, status);
 }
 
-float64 float16_to_float64(float16 a, flag ieee STATUS_PARAM)
+float64 float16_to_float64(float16 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint32_t aSig;
 
     aSign = extractFloat16Sign(a);
@@ -3390,7 +3564,7 @@
     if (aExp == 0x1f && ieee) {
         if (aSig) {
             return commonNaNToFloat64(
-                float16ToCommonNaN(a STATUS_VAR) STATUS_VAR);
+                float16ToCommonNaN(a, status), status);
         }
         return packFloat64(aSign, 0x7ff, 0);
     }
@@ -3405,14 +3579,14 @@
     return packFloat64(aSign, aExp + 0x3f0, ((uint64_t)aSig) << 42);
 }
 
-float16 float64_to_float16(float64 a, flag ieee STATUS_PARAM)
+float16 float64_to_float16(float64 a, flag ieee, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig;
     uint32_t zSig;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac(a);
     aExp = extractFloat64Exp(a);
@@ -3421,15 +3595,15 @@
         if (aSig) {
             /* Input is a NaN */
             if (!ieee) {
-                float_raise(float_flag_invalid STATUS_VAR);
+                float_raise(float_flag_invalid, status);
                 return packFloat16(aSign, 0, 0);
             }
             return commonNaNToFloat16(
-                float64ToCommonNaN(a STATUS_VAR) STATUS_VAR);
+                float64ToCommonNaN(a, status), status);
         }
         /* Infinity */
         if (!ieee) {
-            float_raise(float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             return packFloat16(aSign, 0x1f, 0x3ff);
         }
         return packFloat16(aSign, 0x1f, 0);
@@ -3449,7 +3623,7 @@
     zSig |= 0x00800000;
     aExp -= 0x3F1;
 
-    return roundAndPackFloat16(aSign, aExp, zSig, ieee STATUS_VAR);
+    return roundAndPackFloat16(aSign, aExp, zSig, ieee, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -3459,18 +3633,20 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 float64_to_floatx80( float64 a STATUS_PARAM )
+floatx80 float64_to_floatx80(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
     aSign = extractFloat64Sign( a );
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return commonNaNToFloatx80( float64ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        if (aSig) {
+            return commonNaNToFloatx80(float64ToCommonNaN(a, status), status);
+        }
         return packFloatx80( aSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
     if ( aExp == 0 ) {
@@ -3490,18 +3666,20 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float64_to_float128( float64 a STATUS_PARAM )
+float128 float64_to_float128(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig, zSig0, zSig1;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
     aSign = extractFloat64Sign( a );
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return commonNaNToFloat128( float64ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        if (aSig) {
+            return commonNaNToFloat128(float64ToCommonNaN(a, status), status);
+        }
         return packFloat128( aSign, 0x7FFF, 0, 0 );
     }
     if ( aExp == 0 ) {
@@ -3521,26 +3699,26 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_round_to_int( float64 a STATUS_PARAM )
+float64 float64_round_to_int(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t lastBitMask, roundBitsMask;
     uint64_t z;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aExp = extractFloat64Exp( a );
     if ( 0x433 <= aExp ) {
         if ( ( aExp == 0x7FF ) && extractFloat64Frac( a ) ) {
-            return propagateFloat64NaN( a, a STATUS_VAR );
+            return propagateFloat64NaN(a, a, status);
         }
         return a;
     }
     if ( aExp < 0x3FF ) {
         if ( (uint64_t) ( float64_val(a)<<1 ) == 0 ) return a;
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
         aSign = extractFloat64Sign( a );
-        switch ( STATUS(float_rounding_mode) ) {
+        switch (status->float_rounding_mode) {
          case float_round_nearest_even:
             if ( ( aExp == 0x3FE ) && extractFloat64Frac( a ) ) {
                 return packFloat64( aSign, 0x3FF, 0 );
@@ -3563,7 +3741,7 @@
     lastBitMask <<= 0x433 - aExp;
     roundBitsMask = lastBitMask - 1;
     z = float64_val(a);
-    switch (STATUS(float_rounding_mode)) {
+    switch (status->float_rounding_mode) {
     case float_round_nearest_even:
         z += lastBitMask >> 1;
         if ((z & roundBitsMask) == 0) {
@@ -3589,20 +3767,21 @@
         abort();
     }
     z &= ~ roundBitsMask;
-    if ( z != float64_val(a) )
-        STATUS(float_exception_flags) |= float_flag_inexact;
+    if (z != float64_val(a)) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     return make_float64(z);
 
 }
 
-float64 float64_trunc_to_int( float64 a STATUS_PARAM)
+float64 float64_trunc_to_int(float64 a, float_status *status)
 {
     int oldmode;
     float64 res;
-    oldmode = STATUS(float_rounding_mode);
-    STATUS(float_rounding_mode) = float_round_to_zero;
-    res = float64_round_to_int(a STATUS_VAR);
-    STATUS(float_rounding_mode) = oldmode;
+    oldmode = status->float_rounding_mode;
+    status->float_rounding_mode = float_round_to_zero;
+    res = float64_round_to_int(a, status);
+    status->float_rounding_mode = oldmode;
     return res;
 }
 
@@ -3614,11 +3793,12 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float64 addFloat64Sigs( float64 a, float64 b, flag zSign STATUS_PARAM )
+static float64 addFloat64Sigs(float64 a, float64 b, flag zSign,
+                              float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3629,7 +3809,9 @@
     bSig <<= 9;
     if ( 0 < expDiff ) {
         if ( aExp == 0x7FF ) {
-            if ( aSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+            if (aSig) {
+                return propagateFloat64NaN(a, b, status);
+            }
             return a;
         }
         if ( bExp == 0 ) {
@@ -3643,7 +3825,9 @@
     }
     else if ( expDiff < 0 ) {
         if ( bExp == 0x7FF ) {
-            if ( bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+            if (bSig) {
+                return propagateFloat64NaN(a, b, status);
+            }
             return packFloat64( zSign, 0x7FF, 0 );
         }
         if ( aExp == 0 ) {
@@ -3657,13 +3841,15 @@
     }
     else {
         if ( aExp == 0x7FF ) {
-            if ( aSig | bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+            if (aSig | bSig) {
+                return propagateFloat64NaN(a, b, status);
+            }
             return a;
         }
         if ( aExp == 0 ) {
-            if (STATUS(flush_to_zero)) {
+            if (status->flush_to_zero) {
                 if (aSig | bSig) {
-                    float_raise(float_flag_output_denormal STATUS_VAR);
+                    float_raise(float_flag_output_denormal, status);
                 }
                 return packFloat64(zSign, 0, 0);
             }
@@ -3681,7 +3867,7 @@
         ++zExp;
     }
  roundAndPack:
-    return roundAndPackFloat64( zSign, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat64(zSign, zExp, zSig, status);
 
 }
 
@@ -3693,11 +3879,12 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float64 subFloat64Sigs( float64 a, float64 b, flag zSign STATUS_PARAM )
+static float64 subFloat64Sigs(float64 a, float64 b, flag zSign,
+                              float_status *status)
 {
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig;
-    int_fast16_t expDiff;
+    int expDiff;
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3709,9 +3896,11 @@
     if ( 0 < expDiff ) goto aExpBigger;
     if ( expDiff < 0 ) goto bExpBigger;
     if ( aExp == 0x7FF ) {
-        if ( aSig | bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float64_default_nan;
+        if (aSig | bSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
+        float_raise(float_flag_invalid, status);
+        return float64_default_nan(status);
     }
     if ( aExp == 0 ) {
         aExp = 1;
@@ -3719,10 +3908,12 @@
     }
     if ( bSig < aSig ) goto aBigger;
     if ( aSig < bSig ) goto bBigger;
-    return packFloat64( STATUS(float_rounding_mode) == float_round_down, 0, 0 );
+    return packFloat64(status->float_rounding_mode == float_round_down, 0, 0);
  bExpBigger:
     if ( bExp == 0x7FF ) {
-        if ( bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
         return packFloat64( zSign ^ 1, 0x7FF, 0 );
     }
     if ( aExp == 0 ) {
@@ -3740,7 +3931,9 @@
     goto normalizeRoundAndPack;
  aExpBigger:
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+        if (aSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
@@ -3756,7 +3949,7 @@
     zExp = aExp;
  normalizeRoundAndPack:
     --zExp;
-    return normalizeRoundAndPackFloat64( zSign, zExp, zSig STATUS_VAR );
+    return normalizeRoundAndPackFloat64(zSign, zExp, zSig, status);
 
 }
 
@@ -3766,19 +3959,19 @@
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_add( float64 a, float64 b STATUS_PARAM )
+float64 float64_add(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     aSign = extractFloat64Sign( a );
     bSign = extractFloat64Sign( b );
     if ( aSign == bSign ) {
-        return addFloat64Sigs( a, b, aSign STATUS_VAR );
+        return addFloat64Sigs(a, b, aSign, status);
     }
     else {
-        return subFloat64Sigs( a, b, aSign STATUS_VAR );
+        return subFloat64Sigs(a, b, aSign, status);
     }
 
 }
@@ -3789,19 +3982,19 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_sub( float64 a, float64 b STATUS_PARAM )
+float64 float64_sub(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     aSign = extractFloat64Sign( a );
     bSign = extractFloat64Sign( b );
     if ( aSign == bSign ) {
-        return subFloat64Sigs( a, b, aSign STATUS_VAR );
+        return subFloat64Sigs(a, b, aSign, status);
     }
     else {
-        return addFloat64Sigs( a, b, aSign STATUS_VAR );
+        return addFloat64Sigs(a, b, aSign, status);
     }
 
 }
@@ -3812,14 +4005,14 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_mul( float64 a, float64 b STATUS_PARAM )
+float64 float64_mul(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3830,19 +4023,21 @@
     zSign = aSign ^ bSign;
     if ( aExp == 0x7FF ) {
         if ( aSig || ( ( bExp == 0x7FF ) && bSig ) ) {
-            return propagateFloat64NaN( a, b STATUS_VAR );
+            return propagateFloat64NaN(a, b, status);
         }
         if ( ( bExp | bSig ) == 0 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float64_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float64_default_nan(status);
         }
         return packFloat64( zSign, 0x7FF, 0 );
     }
     if ( bExp == 0x7FF ) {
-        if ( bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
         if ( ( aExp | aSig ) == 0 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float64_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float64_default_nan(status);
         }
         return packFloat64( zSign, 0x7FF, 0 );
     }
@@ -3863,7 +4058,7 @@
         zSig0 <<= 1;
         --zExp;
     }
-    return roundAndPackFloat64( zSign, zExp, zSig0 STATUS_VAR );
+    return roundAndPackFloat64(zSign, zExp, zSig0, status);
 
 }
 
@@ -3873,15 +4068,15 @@
 | the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_div( float64 a, float64 b STATUS_PARAM )
+float64 float64_div(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int_fast16_t aExp, bExp, zExp;
+    int aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig;
     uint64_t rem0, rem1;
     uint64_t term0, term1;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -3891,25 +4086,31 @@
     bSign = extractFloat64Sign( b );
     zSign = aSign ^ bSign;
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+        if (aSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
         if ( bExp == 0x7FF ) {
-            if ( bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float64_default_nan;
+            if (bSig) {
+                return propagateFloat64NaN(a, b, status);
+            }
+            float_raise(float_flag_invalid, status);
+            return float64_default_nan(status);
         }
         return packFloat64( zSign, 0x7FF, 0 );
     }
     if ( bExp == 0x7FF ) {
-        if ( bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
         return packFloat64( zSign, 0, 0 );
     }
     if ( bExp == 0 ) {
         if ( bSig == 0 ) {
             if ( ( aExp | aSig ) == 0 ) {
-                float_raise( float_flag_invalid STATUS_VAR);
-                return float64_default_nan;
+                float_raise(float_flag_invalid, status);
+                return float64_default_nan(status);
             }
-            float_raise( float_flag_divbyzero STATUS_VAR);
+            float_raise(float_flag_divbyzero, status);
             return packFloat64( zSign, 0x7FF, 0 );
         }
         normalizeFloat64Subnormal( bSig, &bExp, &bSig );
@@ -3935,7 +4136,7 @@
         }
         zSig |= ( rem1 != 0 );
     }
-    return roundAndPackFloat64( zSign, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat64(zSign, zExp, zSig, status);
 
 }
 
@@ -3945,16 +4146,16 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_rem( float64 a, float64 b STATUS_PARAM )
+float64 float64_rem(float64 a, float64 b, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp, bExp, expDiff;
+    int aExp, bExp, expDiff;
     uint64_t aSig, bSig;
     uint64_t q, alternateASig;
     int64_t sigMean;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
     aSign = extractFloat64Sign( a );
@@ -3962,19 +4163,21 @@
     bExp = extractFloat64Exp( b );
     if ( aExp == 0x7FF ) {
         if ( aSig || ( ( bExp == 0x7FF ) && bSig ) ) {
-            return propagateFloat64NaN( a, b STATUS_VAR );
+            return propagateFloat64NaN(a, b, status);
         }
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float64_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float64_default_nan(status);
     }
     if ( bExp == 0x7FF ) {
-        if ( bSig ) return propagateFloat64NaN( a, b STATUS_VAR );
+        if (bSig) {
+            return propagateFloat64NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
         if ( bSig == 0 ) {
-            float_raise( float_flag_invalid STATUS_VAR);
-            return float64_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float64_default_nan(status);
         }
         normalizeFloat64Subnormal( bSig, &bExp, &bSig );
     }
@@ -4021,7 +4224,7 @@
     }
     zSign = ( (int64_t) aSig < 0 );
     if ( zSign ) aSig = - aSig;
-    return normalizeRoundAndPackFloat64( aSign ^ zSign, bExp, aSig STATUS_VAR );
+    return normalizeRoundAndPackFloat64(aSign ^ zSign, bExp, aSig, status);
 
 }
 
@@ -4036,19 +4239,20 @@
 | externally will flip the sign bit on NaNs.)
 *----------------------------------------------------------------------------*/
 
-float64 float64_muladd(float64 a, float64 b, float64 c, int flags STATUS_PARAM)
+float64 float64_muladd(float64 a, float64 b, float64 c, int flags,
+                       float_status *status)
 {
     flag aSign, bSign, cSign, zSign;
-    int_fast16_t aExp, bExp, cExp, pExp, zExp, expDiff;
+    int aExp, bExp, cExp, pExp, zExp, expDiff;
     uint64_t aSig, bSig, cSig;
     flag pInf, pZero, pSign;
     uint64_t pSig0, pSig1, cSig0, cSig1, zSig0, zSig1;
     int shiftcount;
     flag signflip, infzero;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
-    c = float64_squash_input_denormal(c STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
+    c = float64_squash_input_denormal(c, status);
     aSig = extractFloat64Frac(a);
     aExp = extractFloat64Exp(a);
     aSign = extractFloat64Sign(a);
@@ -4070,12 +4274,12 @@
     if (((aExp == 0x7ff) && aSig) ||
         ((bExp == 0x7ff) && bSig) ||
         ((cExp == 0x7ff) && cSig)) {
-        return propagateFloat64MulAddNaN(a, b, c, infzero STATUS_VAR);
+        return propagateFloat64MulAddNaN(a, b, c, infzero, status);
     }
 
     if (infzero) {
-        float_raise(float_flag_invalid STATUS_VAR);
-        return float64_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float64_default_nan(status);
     }
 
     if (flags & float_muladd_negate_c) {
@@ -4095,8 +4299,8 @@
     if (cExp == 0x7ff) {
         if (pInf && (pSign ^ cSign)) {
             /* addition of opposite-signed infinities => InvalidOperation */
-            float_raise(float_flag_invalid STATUS_VAR);
-            return float64_default_nan;
+            float_raise(float_flag_invalid, status);
+            return float64_default_nan(status);
         }
         /* Otherwise generate an infinity of the same sign */
         return packFloat64(cSign ^ signflip, 0x7ff, 0);
@@ -4112,7 +4316,7 @@
                 /* Adding two exact zeroes */
                 if (pSign == cSign) {
                     zSign = pSign;
-                } else if (STATUS(float_rounding_mode) == float_round_down) {
+                } else if (status->float_rounding_mode == float_round_down) {
                     zSign = 1;
                 } else {
                     zSign = 0;
@@ -4120,8 +4324,8 @@
                 return packFloat64(zSign ^ signflip, 0, 0);
             }
             /* Exact zero plus a denorm */
-            if (STATUS(flush_to_zero)) {
-                float_raise(float_flag_output_denormal STATUS_VAR);
+            if (status->flush_to_zero) {
+                float_raise(float_flag_output_denormal, status);
                 return packFloat64(cSign ^ signflip, 0, 0);
             }
         }
@@ -4135,7 +4339,7 @@
              */
             cExp -= 2;
             cSig = (cSig | 0x0010000000000000ULL) << 10;
-            return roundAndPackFloat64(cSign ^ signflip, cExp, cSig STATUS_VAR);
+            return roundAndPackFloat64(cSign ^ signflip, cExp, cSig, status);
         }
         return packFloat64(cSign ^ signflip, cExp, cSig);
     }
@@ -4176,7 +4380,7 @@
                 pExp--;
             }
             return roundAndPackFloat64(zSign, pExp - 1,
-                                       pSig1 STATUS_VAR);
+                                       pSig1, status);
         }
         normalizeFloat64Subnormal(cSig, &cExp, &cSig);
     }
@@ -4214,7 +4418,7 @@
         if (flags & float_muladd_halve_result) {
             zExp--;
         }
-        return roundAndPackFloat64(zSign, zExp, zSig1 STATUS_VAR);
+        return roundAndPackFloat64(zSign, zExp, zSig1, status);
     } else {
         /* Subtraction */
         if (expDiff > 0) {
@@ -4236,7 +4440,7 @@
             } else {
                 /* Exact zero */
                 zSign = signflip;
-                if (STATUS(float_rounding_mode) == float_round_down) {
+                if (status->float_rounding_mode == float_round_down) {
                     zSign ^= 1;
                 }
                 return packFloat64(zSign, 0, 0);
@@ -4267,7 +4471,7 @@
         if (flags & float_muladd_halve_result) {
             zExp--;
         }
-        return roundAndPackFloat64(zSign, zExp, zSig0 STATUS_VAR);
+        return roundAndPackFloat64(zSign, zExp, zSig0, status);
     }
 }
 
@@ -4277,27 +4481,29 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float64_sqrt( float64 a STATUS_PARAM )
+float64 float64_sqrt(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, zExp;
+    int aExp, zExp;
     uint64_t aSig, zSig, doubleZSig;
     uint64_t rem0, rem1, term0, term1;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
     aSign = extractFloat64Sign( a );
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return propagateFloat64NaN( a, a STATUS_VAR );
+        if (aSig) {
+            return propagateFloat64NaN(a, a, status);
+        }
         if ( ! aSign ) return a;
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float64_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float64_default_nan(status);
     }
     if ( aSign ) {
         if ( ( aExp | aSig ) == 0 ) return a;
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float64_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float64_default_nan(status);
     }
     if ( aExp == 0 ) {
         if ( aSig == 0 ) return float64_zero;
@@ -4319,7 +4525,7 @@
         }
         zSig |= ( ( rem0 | rem1 ) != 0 );
     }
-    return roundAndPackFloat64( 0, zExp, zSig STATUS_VAR );
+    return roundAndPackFloat64(0, zExp, zSig, status);
 
 }
 
@@ -4328,12 +4534,12 @@
 | The operation is performed according to the IEC/IEEE Standard for Binary
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
-float64 float64_log2( float64 a STATUS_PARAM )
+float64 float64_log2(float64 a, float_status *status)
 {
     flag aSign, zSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig, aSig0, aSig1, zSig, i;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
@@ -4344,11 +4550,13 @@
         normalizeFloat64Subnormal( aSig, &aExp, &aSig );
     }
     if ( aSign ) {
-        float_raise( float_flag_invalid STATUS_VAR);
-        return float64_default_nan;
+        float_raise(float_flag_invalid, status);
+        return float64_default_nan(status);
     }
     if ( aExp == 0x7FF ) {
-        if ( aSig ) return propagateFloat64NaN( a, float64_zero STATUS_VAR );
+        if (aSig) {
+            return propagateFloat64NaN(a, float64_zero, status);
+        }
         return a;
     }
 
@@ -4367,7 +4575,7 @@
 
     if ( zSign )
         zSig = -zSig;
-    return normalizeRoundAndPackFloat64( zSign, 0x408, zSig STATUS_VAR );
+    return normalizeRoundAndPackFloat64(zSign, 0x408, zSig, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -4377,16 +4585,16 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_eq( float64 a, float64 b STATUS_PARAM )
+int float64_eq(float64 a, float64 b, float_status *status)
 {
     uint64_t av, bv;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     av = float64_val(a);
@@ -4402,17 +4610,17 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_le( float64 a, float64 b STATUS_PARAM )
+int float64_le(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign;
     uint64_t av, bv;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloat64Sign( a );
@@ -4431,17 +4639,17 @@
 | to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_lt( float64 a, float64 b STATUS_PARAM )
+int float64_lt(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign;
     uint64_t av, bv;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloat64Sign( a );
@@ -4460,15 +4668,15 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_unordered( float64 a, float64 b STATUS_PARAM )
+int float64_unordered(float64 a, float64 b, float_status *status)
 {
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 1;
     }
     return 0;
@@ -4481,17 +4689,18 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_eq_quiet( float64 a, float64 b STATUS_PARAM )
+int float64_eq_quiet(float64 a, float64 b, float_status *status)
 {
     uint64_t av, bv;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        if ( float64_is_signaling_nan( a ) || float64_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float64_is_signaling_nan(a, status)
+         || float64_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -4508,18 +4717,19 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_le_quiet( float64 a, float64 b STATUS_PARAM )
+int float64_le_quiet(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign;
     uint64_t av, bv;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        if ( float64_is_signaling_nan( a ) || float64_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float64_is_signaling_nan(a, status)
+         || float64_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -4539,18 +4749,19 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_lt_quiet( float64 a, float64 b STATUS_PARAM )
+int float64_lt_quiet(float64 a, float64 b, float_status *status)
 {
     flag aSign, bSign;
     uint64_t av, bv;
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        if ( float64_is_signaling_nan( a ) || float64_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float64_is_signaling_nan(a, status)
+         || float64_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -4570,16 +4781,17 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float64_unordered_quiet( float64 a, float64 b STATUS_PARAM )
+int float64_unordered_quiet(float64 a, float64 b, float_status *status)
 {
-    a = float64_squash_input_denormal(a STATUS_VAR);
-    b = float64_squash_input_denormal(b STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
+    b = float64_squash_input_denormal(b, status);
 
     if (    ( ( extractFloat64Exp( a ) == 0x7FF ) && extractFloat64Frac( a ) )
          || ( ( extractFloat64Exp( b ) == 0x7FF ) && extractFloat64Frac( b ) )
        ) {
-        if ( float64_is_signaling_nan( a ) || float64_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float64_is_signaling_nan(a, status)
+         || float64_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 1;
     }
@@ -4596,10 +4808,10 @@
 | overflows, the largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 floatx80_to_int32( floatx80 a STATUS_PARAM )
+int32_t floatx80_to_int32(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig;
 
     aSig = extractFloatx80Frac( a );
@@ -4609,7 +4821,7 @@
     shiftCount = 0x4037 - aExp;
     if ( shiftCount <= 0 ) shiftCount = 1;
     shift64RightJamming( aSig, shiftCount, &aSig );
-    return roundAndPackInt32( aSign, aSig STATUS_VAR );
+    return roundAndPackInt32(aSign, aSig, status);
 
 }
 
@@ -4623,10 +4835,10 @@
 | sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 floatx80_to_int32_round_to_zero( floatx80 a STATUS_PARAM )
+int32_t floatx80_to_int32_round_to_zero(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig, savedASig;
     int32_t z;
 
@@ -4638,7 +4850,9 @@
         goto invalid;
     }
     else if ( aExp < 0x3FFF ) {
-        if ( aExp || aSig ) STATUS(float_exception_flags) |= float_flag_inexact;
+        if (aExp || aSig) {
+            status->float_exception_flags |= float_flag_inexact;
+        }
         return 0;
     }
     shiftCount = 0x403E - aExp;
@@ -4648,11 +4862,11 @@
     if ( aSign ) z = - z;
     if ( ( z < 0 ) ^ aSign ) {
  invalid:
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return aSign ? (int32_t) 0x80000000 : 0x7FFFFFFF;
     }
     if ( ( aSig<<shiftCount ) != savedASig ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     return z;
 
@@ -4668,10 +4882,10 @@
 | overflows, the largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 floatx80_to_int64( floatx80 a STATUS_PARAM )
+int64_t floatx80_to_int64(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig, aSigExtra;
 
     aSig = extractFloatx80Frac( a );
@@ -4680,7 +4894,7 @@
     shiftCount = 0x403E - aExp;
     if ( shiftCount <= 0 ) {
         if ( shiftCount ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if (    ! aSign
                  || (    ( aExp == 0x7FFF )
                       && ( aSig != LIT64( 0x8000000000000000 ) ) )
@@ -4694,7 +4908,7 @@
     else {
         shift64ExtraRightJamming( aSig, 0, shiftCount, &aSig, &aSigExtra );
     }
-    return roundAndPackInt64( aSign, aSig, aSigExtra STATUS_VAR );
+    return roundAndPackInt64(aSign, aSig, aSigExtra, status);
 
 }
 
@@ -4708,12 +4922,12 @@
 | sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 floatx80_to_int64_round_to_zero( floatx80 a STATUS_PARAM )
+int64_t floatx80_to_int64_round_to_zero(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig;
-    int64 z;
+    int64_t z;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -4722,7 +4936,7 @@
     if ( 0 <= shiftCount ) {
         aSig &= LIT64( 0x7FFFFFFFFFFFFFFF );
         if ( ( a.high != 0xC03E ) || aSig ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if ( ! aSign || ( ( aExp == 0x7FFF ) && aSig ) ) {
                 return LIT64( 0x7FFFFFFFFFFFFFFF );
             }
@@ -4730,12 +4944,14 @@
         return (int64_t) LIT64( 0x8000000000000000 );
     }
     else if ( aExp < 0x3FFF ) {
-        if ( aExp | aSig ) STATUS(float_exception_flags) |= float_flag_inexact;
+        if (aExp | aSig) {
+            status->float_exception_flags |= float_flag_inexact;
+        }
         return 0;
     }
     z = aSig>>( - shiftCount );
     if ( (uint64_t) ( aSig<<( shiftCount & 63 ) ) ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     if ( aSign ) z = - z;
     return z;
@@ -4749,10 +4965,10 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 floatx80_to_float32( floatx80 a STATUS_PARAM )
+float32 floatx80_to_float32(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig;
 
     aSig = extractFloatx80Frac( a );
@@ -4760,13 +4976,13 @@
     aSign = extractFloatx80Sign( a );
     if ( aExp == 0x7FFF ) {
         if ( (uint64_t) ( aSig<<1 ) ) {
-            return commonNaNToFloat32( floatx80ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+            return commonNaNToFloat32(floatx80ToCommonNaN(a, status), status);
         }
         return packFloat32( aSign, 0xFF, 0 );
     }
     shift64RightJamming( aSig, 33, &aSig );
     if ( aExp || aSig ) aExp -= 0x3F81;
-    return roundAndPackFloat32( aSign, aExp, aSig STATUS_VAR );
+    return roundAndPackFloat32(aSign, aExp, aSig, status);
 
 }
 
@@ -4777,10 +4993,10 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 floatx80_to_float64( floatx80 a STATUS_PARAM )
+float64 floatx80_to_float64(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig, zSig;
 
     aSig = extractFloatx80Frac( a );
@@ -4788,13 +5004,13 @@
     aSign = extractFloatx80Sign( a );
     if ( aExp == 0x7FFF ) {
         if ( (uint64_t) ( aSig<<1 ) ) {
-            return commonNaNToFloat64( floatx80ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+            return commonNaNToFloat64(floatx80ToCommonNaN(a, status), status);
         }
         return packFloat64( aSign, 0x7FF, 0 );
     }
     shift64RightJamming( aSig, 1, &zSig );
     if ( aExp || aSig ) aExp -= 0x3C01;
-    return roundAndPackFloat64( aSign, aExp, zSig STATUS_VAR );
+    return roundAndPackFloat64(aSign, aExp, zSig, status);
 
 }
 
@@ -4805,17 +5021,17 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 floatx80_to_float128( floatx80 a STATUS_PARAM )
+float128 floatx80_to_float128(floatx80 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp;
+    int aExp;
     uint64_t aSig, zSig0, zSig1;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
     aSign = extractFloatx80Sign( a );
     if ( ( aExp == 0x7FFF ) && (uint64_t) ( aSig<<1 ) ) {
-        return commonNaNToFloat128( floatx80ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+        return commonNaNToFloat128(floatx80ToCommonNaN(a, status), status);
     }
     shift128Right( aSig<<1, 0, 16, &zSig0, &zSig1 );
     return packFloat128( aSign, aExp, zSig0, zSig1 );
@@ -4829,17 +5045,17 @@
 | Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_round_to_int( floatx80 a STATUS_PARAM )
+floatx80 floatx80_round_to_int(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t lastBitMask, roundBitsMask;
     floatx80 z;
 
     aExp = extractFloatx80Exp( a );
     if ( 0x403E <= aExp ) {
         if ( ( aExp == 0x7FFF ) && (uint64_t) ( extractFloatx80Frac( a )<<1 ) ) {
-            return propagateFloatx80NaN( a, a STATUS_VAR );
+            return propagateFloatx80NaN(a, a, status);
         }
         return a;
     }
@@ -4848,9 +5064,9 @@
              && ( (uint64_t) ( extractFloatx80Frac( a )<<1 ) == 0 ) ) {
             return a;
         }
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
         aSign = extractFloatx80Sign( a );
-        switch ( STATUS(float_rounding_mode) ) {
+        switch (status->float_rounding_mode) {
          case float_round_nearest_even:
             if ( ( aExp == 0x3FFE ) && (uint64_t) ( extractFloatx80Frac( a )<<1 )
                ) {
@@ -4879,7 +5095,7 @@
     lastBitMask <<= 0x403E - aExp;
     roundBitsMask = lastBitMask - 1;
     z = a;
-    switch (STATUS(float_rounding_mode)) {
+    switch (status->float_rounding_mode) {
     case float_round_nearest_even:
         z.low += lastBitMask>>1;
         if ((z.low & roundBitsMask) == 0) {
@@ -4909,7 +5125,9 @@
         ++z.high;
         z.low = LIT64( 0x8000000000000000 );
     }
-    if ( z.low != a.low ) STATUS(float_exception_flags) |= float_flag_inexact;
+    if (z.low != a.low) {
+        status->float_exception_flags |= float_flag_inexact;
+    }
     return z;
 
 }
@@ -4922,11 +5140,12 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static floatx80 addFloatx80Sigs( floatx80 a, floatx80 b, flag zSign STATUS_PARAM)
+static floatx80 addFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
+                                float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
-    int32 expDiff;
+    int32_t expDiff;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -4935,7 +5154,9 @@
     expDiff = aExp - bExp;
     if ( 0 < expDiff ) {
         if ( aExp == 0x7FFF ) {
-            if ( (uint64_t) ( aSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+            if ((uint64_t)(aSig << 1)) {
+                return propagateFloatx80NaN(a, b, status);
+            }
             return a;
         }
         if ( bExp == 0 ) --expDiff;
@@ -4944,7 +5165,9 @@
     }
     else if ( expDiff < 0 ) {
         if ( bExp == 0x7FFF ) {
-            if ( (uint64_t) ( bSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+            if ((uint64_t)(bSig << 1)) {
+                return propagateFloatx80NaN(a, b, status);
+            }
             return packFloatx80( zSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
         }
         if ( aExp == 0 ) ++expDiff;
@@ -4954,7 +5177,7 @@
     else {
         if ( aExp == 0x7FFF ) {
             if ( (uint64_t) ( ( aSig | bSig )<<1 ) ) {
-                return propagateFloatx80NaN( a, b STATUS_VAR );
+                return propagateFloatx80NaN(a, b, status);
             }
             return a;
         }
@@ -4974,10 +5197,8 @@
     zSig0 |= LIT64( 0x8000000000000000 );
     ++zExp;
  roundAndPack:
-    return
-        roundAndPackFloatx80(
-            STATUS(floatx80_rounding_precision), zSign, zExp, zSig0, zSig1 STATUS_VAR );
-
+    return roundAndPackFloatx80(status->floatx80_rounding_precision,
+                                zSign, zExp, zSig0, zSig1, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -4988,12 +5209,12 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static floatx80 subFloatx80Sigs( floatx80 a, floatx80 b, flag zSign STATUS_PARAM )
+static floatx80 subFloatx80Sigs(floatx80 a, floatx80 b, flag zSign,
+                                float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
-    int32 expDiff;
-    floatx80 z;
+    int32_t expDiff;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -5004,12 +5225,10 @@
     if ( expDiff < 0 ) goto bExpBigger;
     if ( aExp == 0x7FFF ) {
         if ( (uint64_t) ( ( aSig | bSig )<<1 ) ) {
-            return propagateFloatx80NaN( a, b STATUS_VAR );
+            return propagateFloatx80NaN(a, b, status);
         }
-        float_raise( float_flag_invalid STATUS_VAR);
-        z.low = floatx80_default_nan_low;
-        z.high = floatx80_default_nan_high;
-        return z;
+        float_raise(float_flag_invalid, status);
+        return floatx80_default_nan(status);
     }
     if ( aExp == 0 ) {
         aExp = 1;
@@ -5018,10 +5237,12 @@
     zSig1 = 0;
     if ( bSig < aSig ) goto aBigger;
     if ( aSig < bSig ) goto bBigger;
-    return packFloatx80( STATUS(float_rounding_mode) == float_round_down, 0, 0 );
+    return packFloatx80(status->float_rounding_mode == float_round_down, 0, 0);
  bExpBigger:
     if ( bExp == 0x7FFF ) {
-        if ( (uint64_t) ( bSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+        if ((uint64_t)(bSig << 1)) {
+            return propagateFloatx80NaN(a, b, status);
+        }
         return packFloatx80( zSign ^ 1, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
     if ( aExp == 0 ) ++expDiff;
@@ -5033,7 +5254,9 @@
     goto normalizeRoundAndPack;
  aExpBigger:
     if ( aExp == 0x7FFF ) {
-        if ( (uint64_t) ( aSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+        if ((uint64_t)(aSig << 1)) {
+            return propagateFloatx80NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) --expDiff;
@@ -5042,10 +5265,8 @@
     sub128( aSig, 0, bSig, zSig1, &zSig0, &zSig1 );
     zExp = aExp;
  normalizeRoundAndPack:
-    return
-        normalizeRoundAndPackFloatx80(
-            STATUS(floatx80_rounding_precision), zSign, zExp, zSig0, zSig1 STATUS_VAR );
-
+    return normalizeRoundAndPackFloatx80(status->floatx80_rounding_precision,
+                                         zSign, zExp, zSig0, zSig1, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -5054,17 +5275,17 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_add( floatx80 a, floatx80 b STATUS_PARAM )
+floatx80 floatx80_add(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign;
 
     aSign = extractFloatx80Sign( a );
     bSign = extractFloatx80Sign( b );
     if ( aSign == bSign ) {
-        return addFloatx80Sigs( a, b, aSign STATUS_VAR );
+        return addFloatx80Sigs(a, b, aSign, status);
     }
     else {
-        return subFloatx80Sigs( a, b, aSign STATUS_VAR );
+        return subFloatx80Sigs(a, b, aSign, status);
     }
 
 }
@@ -5075,17 +5296,17 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_sub( floatx80 a, floatx80 b STATUS_PARAM )
+floatx80 floatx80_sub(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign;
 
     aSign = extractFloatx80Sign( a );
     bSign = extractFloatx80Sign( b );
     if ( aSign == bSign ) {
-        return subFloatx80Sigs( a, b, aSign STATUS_VAR );
+        return subFloatx80Sigs(a, b, aSign, status);
     }
     else {
-        return addFloatx80Sigs( a, b, aSign STATUS_VAR );
+        return addFloatx80Sigs(a, b, aSign, status);
     }
 
 }
@@ -5096,12 +5317,11 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_mul( floatx80 a, floatx80 b STATUS_PARAM )
+floatx80 floatx80_mul(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
-    floatx80 z;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -5113,19 +5333,19 @@
     if ( aExp == 0x7FFF ) {
         if (    (uint64_t) ( aSig<<1 )
              || ( ( bExp == 0x7FFF ) && (uint64_t) ( bSig<<1 ) ) ) {
-            return propagateFloatx80NaN( a, b STATUS_VAR );
+            return propagateFloatx80NaN(a, b, status);
         }
         if ( ( bExp | bSig ) == 0 ) goto invalid;
         return packFloatx80( zSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
     if ( bExp == 0x7FFF ) {
-        if ( (uint64_t) ( bSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+        if ((uint64_t)(bSig << 1)) {
+            return propagateFloatx80NaN(a, b, status);
+        }
         if ( ( aExp | aSig ) == 0 ) {
  invalid:
-            float_raise( float_flag_invalid STATUS_VAR);
-            z.low = floatx80_default_nan_low;
-            z.high = floatx80_default_nan_high;
-            return z;
+            float_raise(float_flag_invalid, status);
+            return floatx80_default_nan(status);
         }
         return packFloatx80( zSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
@@ -5143,10 +5363,8 @@
         shortShift128Left( zSig0, zSig1, 1, &zSig0, &zSig1 );
         --zExp;
     }
-    return
-        roundAndPackFloatx80(
-            STATUS(floatx80_rounding_precision), zSign, zExp, zSig0, zSig1 STATUS_VAR );
-
+    return roundAndPackFloatx80(status->floatx80_rounding_precision,
+                                zSign, zExp, zSig0, zSig1, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -5155,13 +5373,12 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_div( floatx80 a, floatx80 b STATUS_PARAM )
+floatx80 floatx80_div(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig, bSig, zSig0, zSig1;
     uint64_t rem0, rem1, rem2, term0, term1, term2;
-    floatx80 z;
 
     aSig = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -5171,27 +5388,31 @@
     bSign = extractFloatx80Sign( b );
     zSign = aSign ^ bSign;
     if ( aExp == 0x7FFF ) {
-        if ( (uint64_t) ( aSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+        if ((uint64_t)(aSig << 1)) {
+            return propagateFloatx80NaN(a, b, status);
+        }
         if ( bExp == 0x7FFF ) {
-            if ( (uint64_t) ( bSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+            if ((uint64_t)(bSig << 1)) {
+                return propagateFloatx80NaN(a, b, status);
+            }
             goto invalid;
         }
         return packFloatx80( zSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
     if ( bExp == 0x7FFF ) {
-        if ( (uint64_t) ( bSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+        if ((uint64_t)(bSig << 1)) {
+            return propagateFloatx80NaN(a, b, status);
+        }
         return packFloatx80( zSign, 0, 0 );
     }
     if ( bExp == 0 ) {
         if ( bSig == 0 ) {
             if ( ( aExp | aSig ) == 0 ) {
  invalid:
-                float_raise( float_flag_invalid STATUS_VAR);
-                z.low = floatx80_default_nan_low;
-                z.high = floatx80_default_nan_high;
-                return z;
+                float_raise(float_flag_invalid, status);
+                return floatx80_default_nan(status);
             }
-            float_raise( float_flag_divbyzero STATUS_VAR);
+            float_raise(float_flag_divbyzero, status);
             return packFloatx80( zSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
         }
         normalizeFloatx80Subnormal( bSig, &bExp, &bSig );
@@ -5223,10 +5444,8 @@
         }
         zSig1 |= ( ( rem1 | rem2 ) != 0 );
     }
-    return
-        roundAndPackFloatx80(
-            STATUS(floatx80_rounding_precision), zSign, zExp, zSig0, zSig1 STATUS_VAR );
-
+    return roundAndPackFloatx80(status->floatx80_rounding_precision,
+                                zSign, zExp, zSig0, zSig1, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -5235,13 +5454,12 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_rem( floatx80 a, floatx80 b STATUS_PARAM )
+floatx80 floatx80_rem(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, zSign;
-    int32 aExp, bExp, expDiff;
+    int32_t aExp, bExp, expDiff;
     uint64_t aSig0, aSig1, bSig;
     uint64_t q, term0, term1, alternateASig0, alternateASig1;
-    floatx80 z;
 
     aSig0 = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
@@ -5251,21 +5469,21 @@
     if ( aExp == 0x7FFF ) {
         if (    (uint64_t) ( aSig0<<1 )
              || ( ( bExp == 0x7FFF ) && (uint64_t) ( bSig<<1 ) ) ) {
-            return propagateFloatx80NaN( a, b STATUS_VAR );
+            return propagateFloatx80NaN(a, b, status);
         }
         goto invalid;
     }
     if ( bExp == 0x7FFF ) {
-        if ( (uint64_t) ( bSig<<1 ) ) return propagateFloatx80NaN( a, b STATUS_VAR );
+        if ((uint64_t)(bSig << 1)) {
+            return propagateFloatx80NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
         if ( bSig == 0 ) {
  invalid:
-            float_raise( float_flag_invalid STATUS_VAR);
-            z.low = floatx80_default_nan_low;
-            z.high = floatx80_default_nan_high;
-            return z;
+            float_raise(float_flag_invalid, status);
+            return floatx80_default_nan(status);
         }
         normalizeFloatx80Subnormal( bSig, &bExp, &bSig );
     }
@@ -5321,7 +5539,7 @@
     }
     return
         normalizeRoundAndPackFloatx80(
-            80, zSign, bExp + expDiff, aSig0, aSig1 STATUS_VAR );
+            80, zSign, bExp + expDiff, aSig0, aSig1, status);
 
 }
 
@@ -5331,29 +5549,28 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 floatx80_sqrt( floatx80 a STATUS_PARAM )
+floatx80 floatx80_sqrt(floatx80 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, zExp;
+    int32_t aExp, zExp;
     uint64_t aSig0, aSig1, zSig0, zSig1, doubleZSig0;
     uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
-    floatx80 z;
 
     aSig0 = extractFloatx80Frac( a );
     aExp = extractFloatx80Exp( a );
     aSign = extractFloatx80Sign( a );
     if ( aExp == 0x7FFF ) {
-        if ( (uint64_t) ( aSig0<<1 ) ) return propagateFloatx80NaN( a, a STATUS_VAR );
+        if ((uint64_t)(aSig0 << 1)) {
+            return propagateFloatx80NaN(a, a, status);
+        }
         if ( ! aSign ) return a;
         goto invalid;
     }
     if ( aSign ) {
         if ( ( aExp | aSig0 ) == 0 ) return a;
  invalid:
-        float_raise( float_flag_invalid STATUS_VAR);
-        z.low = floatx80_default_nan_low;
-        z.high = floatx80_default_nan_high;
-        return z;
+        float_raise(float_flag_invalid, status);
+        return floatx80_default_nan(status);
     }
     if ( aExp == 0 ) {
         if ( aSig0 == 0 ) return packFloatx80( 0, 0, 0 );
@@ -5389,10 +5606,8 @@
     }
     shortShift128Left( 0, zSig1, 1, &zSig0, &zSig1 );
     zSig0 |= doubleZSig0;
-    return
-        roundAndPackFloatx80(
-            STATUS(floatx80_rounding_precision), 0, zExp, zSig0, zSig1 STATUS_VAR );
-
+    return roundAndPackFloatx80(status->floatx80_rounding_precision,
+                                0, zExp, zSig0, zSig1, status);
 }
 
 /*----------------------------------------------------------------------------
@@ -5402,7 +5617,7 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int floatx80_eq( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_eq(floatx80 a, floatx80 b, float_status *status)
 {
 
     if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
@@ -5410,7 +5625,7 @@
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     return
@@ -5430,7 +5645,7 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int floatx80_le( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_le(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -5439,7 +5654,7 @@
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloatx80Sign( a );
@@ -5463,7 +5678,7 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int floatx80_lt( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_lt(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -5472,7 +5687,7 @@
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloatx80Sign( a );
@@ -5495,14 +5710,14 @@
 | either operand is a NaN.   The comparison is performed according to the
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
-int floatx80_unordered( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_unordered(floatx80 a, floatx80 b, float_status *status)
 {
     if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( a )<<1 ) )
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 1;
     }
     return 0;
@@ -5515,7 +5730,7 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int floatx80_eq_quiet( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_eq_quiet(floatx80 a, floatx80 b, float_status *status)
 {
 
     if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
@@ -5523,9 +5738,9 @@
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        if (    floatx80_is_signaling_nan( a )
-             || floatx80_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (floatx80_is_signaling_nan(a, status)
+         || floatx80_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -5545,7 +5760,7 @@
 | to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int floatx80_le_quiet( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_le_quiet(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -5554,9 +5769,9 @@
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        if (    floatx80_is_signaling_nan( a )
-             || floatx80_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (floatx80_is_signaling_nan(a, status)
+         || floatx80_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -5581,7 +5796,7 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int floatx80_lt_quiet( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_lt_quiet(floatx80 a, floatx80 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -5590,9 +5805,9 @@
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        if (    floatx80_is_signaling_nan( a )
-             || floatx80_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (floatx80_is_signaling_nan(a, status)
+         || floatx80_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -5616,16 +5831,16 @@
 | The comparison is performed according to the IEC/IEEE Standard for Binary
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
-int floatx80_unordered_quiet( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_unordered_quiet(floatx80 a, floatx80 b, float_status *status)
 {
     if (    (    ( extractFloatx80Exp( a ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( a )<<1 ) )
          || (    ( extractFloatx80Exp( b ) == 0x7FFF )
               && (uint64_t) ( extractFloatx80Frac( b )<<1 ) )
        ) {
-        if (    floatx80_is_signaling_nan( a )
-             || floatx80_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (floatx80_is_signaling_nan(a, status)
+         || floatx80_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 1;
     }
@@ -5642,10 +5857,10 @@
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int32 float128_to_int32( float128 a STATUS_PARAM )
+int32_t float128_to_int32(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -5657,7 +5872,7 @@
     aSig0 |= ( aSig1 != 0 );
     shiftCount = 0x4028 - aExp;
     if ( 0 < shiftCount ) shift64RightJamming( aSig0, shiftCount, &aSig0 );
-    return roundAndPackInt32( aSign, aSig0 STATUS_VAR );
+    return roundAndPackInt32(aSign, aSig0, status);
 
 }
 
@@ -5671,10 +5886,10 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int32 float128_to_int32_round_to_zero( float128 a STATUS_PARAM )
+int32_t float128_to_int32_round_to_zero(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1, savedASig;
     int32_t z;
 
@@ -5688,7 +5903,9 @@
         goto invalid;
     }
     else if ( aExp < 0x3FFF ) {
-        if ( aExp || aSig0 ) STATUS(float_exception_flags) |= float_flag_inexact;
+        if (aExp || aSig0) {
+            status->float_exception_flags |= float_flag_inexact;
+        }
         return 0;
     }
     aSig0 |= LIT64( 0x0001000000000000 );
@@ -5699,11 +5916,11 @@
     if ( aSign ) z = - z;
     if ( ( z < 0 ) ^ aSign ) {
  invalid:
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return aSign ? (int32_t) 0x80000000 : 0x7FFFFFFF;
     }
     if ( ( aSig0<<shiftCount ) != savedASig ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     return z;
 
@@ -5719,10 +5936,10 @@
 | largest integer with the same sign as `a' is returned.
 *----------------------------------------------------------------------------*/
 
-int64 float128_to_int64( float128 a STATUS_PARAM )
+int64_t float128_to_int64(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -5733,7 +5950,7 @@
     shiftCount = 0x402F - aExp;
     if ( shiftCount <= 0 ) {
         if ( 0x403E < aExp ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             if (    ! aSign
                  || (    ( aExp == 0x7FFF )
                       && ( aSig1 || ( aSig0 != LIT64( 0x0001000000000000 ) ) )
@@ -5748,7 +5965,7 @@
     else {
         shift64ExtraRightJamming( aSig0, aSig1, shiftCount, &aSig0, &aSig1 );
     }
-    return roundAndPackInt64( aSign, aSig0, aSig1 STATUS_VAR );
+    return roundAndPackInt64(aSign, aSig0, aSig1, status);
 
 }
 
@@ -5762,12 +5979,12 @@
 | returned.
 *----------------------------------------------------------------------------*/
 
-int64 float128_to_int64_round_to_zero( float128 a STATUS_PARAM )
+int64_t float128_to_int64_round_to_zero(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, shiftCount;
+    int32_t aExp, shiftCount;
     uint64_t aSig0, aSig1;
-    int64 z;
+    int64_t z;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -5780,10 +5997,12 @@
             aSig0 &= LIT64( 0x0000FFFFFFFFFFFF );
             if (    ( a.high == LIT64( 0xC03E000000000000 ) )
                  && ( aSig1 < LIT64( 0x0002000000000000 ) ) ) {
-                if ( aSig1 ) STATUS(float_exception_flags) |= float_flag_inexact;
+                if (aSig1) {
+                    status->float_exception_flags |= float_flag_inexact;
+                }
             }
             else {
-                float_raise( float_flag_invalid STATUS_VAR);
+                float_raise(float_flag_invalid, status);
                 if ( ! aSign || ( ( aExp == 0x7FFF ) && ( aSig0 | aSig1 ) ) ) {
                     return LIT64( 0x7FFFFFFFFFFFFFFF );
                 }
@@ -5792,20 +6011,20 @@
         }
         z = ( aSig0<<shiftCount ) | ( aSig1>>( ( - shiftCount ) & 63 ) );
         if ( (uint64_t) ( aSig1<<shiftCount ) ) {
-            STATUS(float_exception_flags) |= float_flag_inexact;
+            status->float_exception_flags |= float_flag_inexact;
         }
     }
     else {
         if ( aExp < 0x3FFF ) {
             if ( aExp | aSig0 | aSig1 ) {
-                STATUS(float_exception_flags) |= float_flag_inexact;
+                status->float_exception_flags |= float_flag_inexact;
             }
             return 0;
         }
         z = aSig0>>( - shiftCount );
         if (    aSig1
              || ( shiftCount && (uint64_t) ( aSig0<<( shiftCount & 63 ) ) ) ) {
-            STATUS(float_exception_flags) |= float_flag_inexact;
+            status->float_exception_flags |= float_flag_inexact;
         }
     }
     if ( aSign ) z = - z;
@@ -5820,10 +6039,10 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float32 float128_to_float32( float128 a STATUS_PARAM )
+float32 float128_to_float32(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig0, aSig1;
     uint32_t zSig;
 
@@ -5833,7 +6052,7 @@
     aSign = extractFloat128Sign( a );
     if ( aExp == 0x7FFF ) {
         if ( aSig0 | aSig1 ) {
-            return commonNaNToFloat32( float128ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+            return commonNaNToFloat32(float128ToCommonNaN(a, status), status);
         }
         return packFloat32( aSign, 0xFF, 0 );
     }
@@ -5844,7 +6063,7 @@
         zSig |= 0x40000000;
         aExp -= 0x3F81;
     }
-    return roundAndPackFloat32( aSign, aExp, zSig STATUS_VAR );
+    return roundAndPackFloat32(aSign, aExp, zSig, status);
 
 }
 
@@ -5855,10 +6074,10 @@
 | Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float64 float128_to_float64( float128 a STATUS_PARAM )
+float64 float128_to_float64(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -5867,7 +6086,7 @@
     aSign = extractFloat128Sign( a );
     if ( aExp == 0x7FFF ) {
         if ( aSig0 | aSig1 ) {
-            return commonNaNToFloat64( float128ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+            return commonNaNToFloat64(float128ToCommonNaN(a, status), status);
         }
         return packFloat64( aSign, 0x7FF, 0 );
     }
@@ -5877,7 +6096,7 @@
         aSig0 |= LIT64( 0x4000000000000000 );
         aExp -= 0x3C01;
     }
-    return roundAndPackFloat64( aSign, aExp, aSig0 STATUS_VAR );
+    return roundAndPackFloat64(aSign, aExp, aSig0, status);
 
 }
 
@@ -5888,10 +6107,10 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-floatx80 float128_to_floatx80( float128 a STATUS_PARAM )
+floatx80 float128_to_floatx80(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t aSig0, aSig1;
 
     aSig1 = extractFloat128Frac1( a );
@@ -5900,7 +6119,7 @@
     aSign = extractFloat128Sign( a );
     if ( aExp == 0x7FFF ) {
         if ( aSig0 | aSig1 ) {
-            return commonNaNToFloatx80( float128ToCommonNaN( a STATUS_VAR ) STATUS_VAR );
+            return commonNaNToFloatx80(float128ToCommonNaN(a, status), status);
         }
         return packFloatx80( aSign, 0x7FFF, LIT64( 0x8000000000000000 ) );
     }
@@ -5912,7 +6131,7 @@
         aSig0 |= LIT64( 0x0001000000000000 );
     }
     shortShift128Left( aSig0, aSig1, 15, &aSig0, &aSig1 );
-    return roundAndPackFloatx80( 80, aSign, aExp, aSig0, aSig1 STATUS_VAR );
+    return roundAndPackFloatx80(80, aSign, aExp, aSig0, aSig1, status);
 
 }
 
@@ -5923,10 +6142,10 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_round_to_int( float128 a STATUS_PARAM )
+float128 float128_round_to_int(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp;
+    int32_t aExp;
     uint64_t lastBitMask, roundBitsMask;
     float128 z;
 
@@ -5936,7 +6155,7 @@
             if (    ( aExp == 0x7FFF )
                  && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) )
                ) {
-                return propagateFloat128NaN( a, a STATUS_VAR );
+                return propagateFloat128NaN(a, a, status);
             }
             return a;
         }
@@ -5944,7 +6163,7 @@
         lastBitMask = ( lastBitMask<<( 0x406E - aExp ) )<<1;
         roundBitsMask = lastBitMask - 1;
         z = a;
-        switch (STATUS(float_rounding_mode)) {
+        switch (status->float_rounding_mode) {
         case float_round_nearest_even:
             if ( lastBitMask ) {
                 add128( z.high, z.low, 0, lastBitMask>>1, &z.high, &z.low );
@@ -5986,9 +6205,9 @@
     else {
         if ( aExp < 0x3FFF ) {
             if ( ( ( (uint64_t) ( a.high<<1 ) ) | a.low ) == 0 ) return a;
-            STATUS(float_exception_flags) |= float_flag_inexact;
+            status->float_exception_flags |= float_flag_inexact;
             aSign = extractFloat128Sign( a );
-            switch ( STATUS(float_rounding_mode) ) {
+            switch (status->float_rounding_mode) {
              case float_round_nearest_even:
                 if (    ( aExp == 0x3FFE )
                      && (   extractFloat128Frac0( a )
@@ -6018,7 +6237,7 @@
         roundBitsMask = lastBitMask - 1;
         z.low = 0;
         z.high = a.high;
-        switch (STATUS(float_rounding_mode)) {
+        switch (status->float_rounding_mode) {
         case float_round_nearest_even:
             z.high += lastBitMask>>1;
             if ( ( ( z.high & roundBitsMask ) | a.low ) == 0 ) {
@@ -6048,7 +6267,7 @@
         z.high &= ~ roundBitsMask;
     }
     if ( ( z.low != a.low ) || ( z.high != a.high ) ) {
-        STATUS(float_exception_flags) |= float_flag_inexact;
+        status->float_exception_flags |= float_flag_inexact;
     }
     return z;
 
@@ -6062,11 +6281,12 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float128 addFloat128Sigs( float128 a, float128 b, flag zSign STATUS_PARAM)
+static float128 addFloat128Sigs(float128 a, float128 b, flag zSign,
+                                float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2;
-    int32 expDiff;
+    int32_t expDiff;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6077,7 +6297,9 @@
     expDiff = aExp - bExp;
     if ( 0 < expDiff ) {
         if ( aExp == 0x7FFF ) {
-            if ( aSig0 | aSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+            if (aSig0 | aSig1) {
+                return propagateFloat128NaN(a, b, status);
+            }
             return a;
         }
         if ( bExp == 0 ) {
@@ -6092,7 +6314,9 @@
     }
     else if ( expDiff < 0 ) {
         if ( bExp == 0x7FFF ) {
-            if ( bSig0 | bSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+            if (bSig0 | bSig1) {
+                return propagateFloat128NaN(a, b, status);
+            }
             return packFloat128( zSign, 0x7FFF, 0, 0 );
         }
         if ( aExp == 0 ) {
@@ -6108,15 +6332,15 @@
     else {
         if ( aExp == 0x7FFF ) {
             if ( aSig0 | aSig1 | bSig0 | bSig1 ) {
-                return propagateFloat128NaN( a, b STATUS_VAR );
+                return propagateFloat128NaN(a, b, status);
             }
             return a;
         }
         add128( aSig0, aSig1, bSig0, bSig1, &zSig0, &zSig1 );
         if ( aExp == 0 ) {
-            if (STATUS(flush_to_zero)) {
+            if (status->flush_to_zero) {
                 if (zSig0 | zSig1) {
-                    float_raise(float_flag_output_denormal STATUS_VAR);
+                    float_raise(float_flag_output_denormal, status);
                 }
                 return packFloat128(zSign, 0, 0, 0);
             }
@@ -6136,7 +6360,7 @@
     shift128ExtraRightJamming(
         zSig0, zSig1, zSig2, 1, &zSig0, &zSig1, &zSig2 );
  roundAndPack:
-    return roundAndPackFloat128( zSign, zExp, zSig0, zSig1, zSig2 STATUS_VAR );
+    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
 
 }
 
@@ -6148,12 +6372,12 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-static float128 subFloat128Sigs( float128 a, float128 b, flag zSign STATUS_PARAM)
+static float128 subFloat128Sigs(float128 a, float128 b, flag zSign,
+                                float_status *status)
 {
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1;
-    int32 expDiff;
-    float128 z;
+    int32_t expDiff;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6168,12 +6392,10 @@
     if ( expDiff < 0 ) goto bExpBigger;
     if ( aExp == 0x7FFF ) {
         if ( aSig0 | aSig1 | bSig0 | bSig1 ) {
-            return propagateFloat128NaN( a, b STATUS_VAR );
+            return propagateFloat128NaN(a, b, status);
         }
-        float_raise( float_flag_invalid STATUS_VAR);
-        z.low = float128_default_nan_low;
-        z.high = float128_default_nan_high;
-        return z;
+        float_raise(float_flag_invalid, status);
+        return float128_default_nan(status);
     }
     if ( aExp == 0 ) {
         aExp = 1;
@@ -6183,10 +6405,13 @@
     if ( aSig0 < bSig0 ) goto bBigger;
     if ( bSig1 < aSig1 ) goto aBigger;
     if ( aSig1 < bSig1 ) goto bBigger;
-    return packFloat128( STATUS(float_rounding_mode) == float_round_down, 0, 0, 0 );
+    return packFloat128(status->float_rounding_mode == float_round_down,
+                        0, 0, 0);
  bExpBigger:
     if ( bExp == 0x7FFF ) {
-        if ( bSig0 | bSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+        if (bSig0 | bSig1) {
+            return propagateFloat128NaN(a, b, status);
+        }
         return packFloat128( zSign ^ 1, 0x7FFF, 0, 0 );
     }
     if ( aExp == 0 ) {
@@ -6204,7 +6429,9 @@
     goto normalizeRoundAndPack;
  aExpBigger:
     if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+        if (aSig0 | aSig1) {
+            return propagateFloat128NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
@@ -6220,7 +6447,8 @@
     zExp = aExp;
  normalizeRoundAndPack:
     --zExp;
-    return normalizeRoundAndPackFloat128( zSign, zExp - 14, zSig0, zSig1 STATUS_VAR );
+    return normalizeRoundAndPackFloat128(zSign, zExp - 14, zSig0, zSig1,
+                                         status);
 
 }
 
@@ -6230,17 +6458,17 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_add( float128 a, float128 b STATUS_PARAM )
+float128 float128_add(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign;
 
     aSign = extractFloat128Sign( a );
     bSign = extractFloat128Sign( b );
     if ( aSign == bSign ) {
-        return addFloat128Sigs( a, b, aSign STATUS_VAR );
+        return addFloat128Sigs(a, b, aSign, status);
     }
     else {
-        return subFloat128Sigs( a, b, aSign STATUS_VAR );
+        return subFloat128Sigs(a, b, aSign, status);
     }
 
 }
@@ -6251,17 +6479,17 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_sub( float128 a, float128 b STATUS_PARAM )
+float128 float128_sub(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign;
 
     aSign = extractFloat128Sign( a );
     bSign = extractFloat128Sign( b );
     if ( aSign == bSign ) {
-        return subFloat128Sigs( a, b, aSign STATUS_VAR );
+        return subFloat128Sigs(a, b, aSign, status);
     }
     else {
-        return addFloat128Sigs( a, b, aSign STATUS_VAR );
+        return addFloat128Sigs(a, b, aSign, status);
     }
 
 }
@@ -6272,12 +6500,11 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_mul( float128 a, float128 b STATUS_PARAM )
+float128 float128_mul(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2, zSig3;
-    float128 z;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6291,19 +6518,19 @@
     if ( aExp == 0x7FFF ) {
         if (    ( aSig0 | aSig1 )
              || ( ( bExp == 0x7FFF ) && ( bSig0 | bSig1 ) ) ) {
-            return propagateFloat128NaN( a, b STATUS_VAR );
+            return propagateFloat128NaN(a, b, status);
         }
         if ( ( bExp | bSig0 | bSig1 ) == 0 ) goto invalid;
         return packFloat128( zSign, 0x7FFF, 0, 0 );
     }
     if ( bExp == 0x7FFF ) {
-        if ( bSig0 | bSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+        if (bSig0 | bSig1) {
+            return propagateFloat128NaN(a, b, status);
+        }
         if ( ( aExp | aSig0 | aSig1 ) == 0 ) {
  invalid:
-            float_raise( float_flag_invalid STATUS_VAR);
-            z.low = float128_default_nan_low;
-            z.high = float128_default_nan_high;
-            return z;
+            float_raise(float_flag_invalid, status);
+            return float128_default_nan(status);
         }
         return packFloat128( zSign, 0x7FFF, 0, 0 );
     }
@@ -6326,7 +6553,7 @@
             zSig0, zSig1, zSig2, 1, &zSig0, &zSig1, &zSig2 );
         ++zExp;
     }
-    return roundAndPackFloat128( zSign, zExp, zSig0, zSig1, zSig2 STATUS_VAR );
+    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
 
 }
 
@@ -6336,13 +6563,12 @@
 | the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_div( float128 a, float128 b STATUS_PARAM )
+float128 float128_div(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign, zSign;
-    int32 aExp, bExp, zExp;
+    int32_t aExp, bExp, zExp;
     uint64_t aSig0, aSig1, bSig0, bSig1, zSig0, zSig1, zSig2;
     uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
-    float128 z;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6354,27 +6580,31 @@
     bSign = extractFloat128Sign( b );
     zSign = aSign ^ bSign;
     if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+        if (aSig0 | aSig1) {
+            return propagateFloat128NaN(a, b, status);
+        }
         if ( bExp == 0x7FFF ) {
-            if ( bSig0 | bSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+            if (bSig0 | bSig1) {
+                return propagateFloat128NaN(a, b, status);
+            }
             goto invalid;
         }
         return packFloat128( zSign, 0x7FFF, 0, 0 );
     }
     if ( bExp == 0x7FFF ) {
-        if ( bSig0 | bSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+        if (bSig0 | bSig1) {
+            return propagateFloat128NaN(a, b, status);
+        }
         return packFloat128( zSign, 0, 0, 0 );
     }
     if ( bExp == 0 ) {
         if ( ( bSig0 | bSig1 ) == 0 ) {
             if ( ( aExp | aSig0 | aSig1 ) == 0 ) {
  invalid:
-                float_raise( float_flag_invalid STATUS_VAR);
-                z.low = float128_default_nan_low;
-                z.high = float128_default_nan_high;
-                return z;
+                float_raise(float_flag_invalid, status);
+                return float128_default_nan(status);
             }
-            float_raise( float_flag_divbyzero STATUS_VAR);
+            float_raise(float_flag_divbyzero, status);
             return packFloat128( zSign, 0x7FFF, 0, 0 );
         }
         normalizeFloat128Subnormal( bSig0, bSig1, &bExp, &bSig0, &bSig1 );
@@ -6410,7 +6640,7 @@
         zSig1 |= ( ( rem1 | rem2 | rem3 ) != 0 );
     }
     shift128ExtraRightJamming( zSig0, zSig1, 0, 15, &zSig0, &zSig1, &zSig2 );
-    return roundAndPackFloat128( zSign, zExp, zSig0, zSig1, zSig2 STATUS_VAR );
+    return roundAndPackFloat128(zSign, zExp, zSig0, zSig1, zSig2, status);
 
 }
 
@@ -6420,14 +6650,13 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_rem( float128 a, float128 b STATUS_PARAM )
+float128 float128_rem(float128 a, float128 b, float_status *status)
 {
     flag aSign, zSign;
-    int32 aExp, bExp, expDiff;
+    int32_t aExp, bExp, expDiff;
     uint64_t aSig0, aSig1, bSig0, bSig1, q, term0, term1, term2;
     uint64_t allZero, alternateASig0, alternateASig1, sigMean1;
     int64_t sigMean0;
-    float128 z;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
@@ -6439,21 +6668,21 @@
     if ( aExp == 0x7FFF ) {
         if (    ( aSig0 | aSig1 )
              || ( ( bExp == 0x7FFF ) && ( bSig0 | bSig1 ) ) ) {
-            return propagateFloat128NaN( a, b STATUS_VAR );
+            return propagateFloat128NaN(a, b, status);
         }
         goto invalid;
     }
     if ( bExp == 0x7FFF ) {
-        if ( bSig0 | bSig1 ) return propagateFloat128NaN( a, b STATUS_VAR );
+        if (bSig0 | bSig1) {
+            return propagateFloat128NaN(a, b, status);
+        }
         return a;
     }
     if ( bExp == 0 ) {
         if ( ( bSig0 | bSig1 ) == 0 ) {
  invalid:
-            float_raise( float_flag_invalid STATUS_VAR);
-            z.low = float128_default_nan_low;
-            z.high = float128_default_nan_high;
-            return z;
+            float_raise(float_flag_invalid, status);
+            return float128_default_nan(status);
         }
         normalizeFloat128Subnormal( bSig0, bSig1, &bExp, &bSig0, &bSig1 );
     }
@@ -6518,9 +6747,8 @@
     }
     zSign = ( (int64_t) aSig0 < 0 );
     if ( zSign ) sub128( 0, 0, aSig0, aSig1, &aSig0, &aSig1 );
-    return
-        normalizeRoundAndPackFloat128( aSign ^ zSign, bExp - 4, aSig0, aSig1 STATUS_VAR );
-
+    return normalizeRoundAndPackFloat128(aSign ^ zSign, bExp - 4, aSig0, aSig1,
+                                         status);
 }
 
 /*----------------------------------------------------------------------------
@@ -6529,30 +6757,29 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-float128 float128_sqrt( float128 a STATUS_PARAM )
+float128 float128_sqrt(float128 a, float_status *status)
 {
     flag aSign;
-    int32 aExp, zExp;
+    int32_t aExp, zExp;
     uint64_t aSig0, aSig1, zSig0, zSig1, zSig2, doubleZSig0;
     uint64_t rem0, rem1, rem2, rem3, term0, term1, term2, term3;
-    float128 z;
 
     aSig1 = extractFloat128Frac1( a );
     aSig0 = extractFloat128Frac0( a );
     aExp = extractFloat128Exp( a );
     aSign = extractFloat128Sign( a );
     if ( aExp == 0x7FFF ) {
-        if ( aSig0 | aSig1 ) return propagateFloat128NaN( a, a STATUS_VAR );
+        if (aSig0 | aSig1) {
+            return propagateFloat128NaN(a, a, status);
+        }
         if ( ! aSign ) return a;
         goto invalid;
     }
     if ( aSign ) {
         if ( ( aExp | aSig0 | aSig1 ) == 0 ) return a;
  invalid:
-        float_raise( float_flag_invalid STATUS_VAR);
-        z.low = float128_default_nan_low;
-        z.high = float128_default_nan_high;
-        return z;
+        float_raise(float_flag_invalid, status);
+        return float128_default_nan(status);
     }
     if ( aExp == 0 ) {
         if ( ( aSig0 | aSig1 ) == 0 ) return packFloat128( 0, 0, 0, 0 );
@@ -6588,7 +6815,7 @@
         zSig1 |= ( ( rem1 | rem2 | rem3 ) != 0 );
     }
     shift128ExtraRightJamming( zSig0, zSig1, 0, 14, &zSig0, &zSig1, &zSig2 );
-    return roundAndPackFloat128( 0, zExp, zSig0, zSig1, zSig2 STATUS_VAR );
+    return roundAndPackFloat128(0, zExp, zSig0, zSig1, zSig2, status);
 
 }
 
@@ -6599,7 +6826,7 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_eq( float128 a, float128 b STATUS_PARAM )
+int float128_eq(float128 a, float128 b, float_status *status)
 {
 
     if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
@@ -6607,7 +6834,7 @@
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     return
@@ -6626,7 +6853,7 @@
 | according to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_le( float128 a, float128 b STATUS_PARAM )
+int float128_le(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -6635,7 +6862,7 @@
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloat128Sign( a );
@@ -6659,7 +6886,7 @@
 | to the IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_lt( float128 a, float128 b STATUS_PARAM )
+int float128_lt(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -6668,7 +6895,7 @@
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
     aSign = extractFloat128Sign( a );
@@ -6692,14 +6919,14 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_unordered( float128 a, float128 b STATUS_PARAM )
+int float128_unordered(float128 a, float128 b, float_status *status)
 {
     if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
               && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        float_raise( float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 1;
     }
     return 0;
@@ -6712,7 +6939,7 @@
 | for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_eq_quiet( float128 a, float128 b STATUS_PARAM )
+int float128_eq_quiet(float128 a, float128 b, float_status *status)
 {
 
     if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
@@ -6720,9 +6947,9 @@
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        if (    float128_is_signaling_nan( a )
-             || float128_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float128_is_signaling_nan(a, status)
+         || float128_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -6742,7 +6969,7 @@
 | IEC/IEEE Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_le_quiet( float128 a, float128 b STATUS_PARAM )
+int float128_le_quiet(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -6751,9 +6978,9 @@
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        if (    float128_is_signaling_nan( a )
-             || float128_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float128_is_signaling_nan(a, status)
+         || float128_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -6778,7 +7005,7 @@
 | Standard for Binary Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_lt_quiet( float128 a, float128 b STATUS_PARAM )
+int float128_lt_quiet(float128 a, float128 b, float_status *status)
 {
     flag aSign, bSign;
 
@@ -6787,9 +7014,9 @@
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        if (    float128_is_signaling_nan( a )
-             || float128_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float128_is_signaling_nan(a, status)
+         || float128_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 0;
     }
@@ -6814,16 +7041,16 @@
 | Floating-Point Arithmetic.
 *----------------------------------------------------------------------------*/
 
-int float128_unordered_quiet( float128 a, float128 b STATUS_PARAM )
+int float128_unordered_quiet(float128 a, float128 b, float_status *status)
 {
     if (    (    ( extractFloat128Exp( a ) == 0x7FFF )
               && ( extractFloat128Frac0( a ) | extractFloat128Frac1( a ) ) )
          || (    ( extractFloat128Exp( b ) == 0x7FFF )
               && ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )
        ) {
-        if (    float128_is_signaling_nan( a )
-             || float128_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+        if (float128_is_signaling_nan(a, status)
+         || float128_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return 1;
     }
@@ -6831,23 +7058,23 @@
 }
 
 /* misc functions */
-float32 uint32_to_float32(uint32_t a STATUS_PARAM)
+float32 uint32_to_float32(uint32_t a, float_status *status)
 {
-    return int64_to_float32(a STATUS_VAR);
+    return int64_to_float32(a, status);
 }
 
-float64 uint32_to_float64(uint32_t a STATUS_PARAM)
+float64 uint32_to_float64(uint32_t a, float_status *status)
 {
-    return int64_to_float64(a STATUS_VAR);
+    return int64_to_float64(a, status);
 }
 
-uint32 float32_to_uint32( float32 a STATUS_PARAM )
+uint32_t float32_to_uint32(float32 a, float_status *status)
 {
     int64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float32_to_int64(a STATUS_VAR);
+    v = float32_to_int64(a, status);
     if (v < 0) {
         res = 0;
     } else if (v > 0xffffffff) {
@@ -6856,17 +7083,17 @@
         return v;
     }
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint32 float32_to_uint32_round_to_zero( float32 a STATUS_PARAM )
+uint32_t float32_to_uint32_round_to_zero(float32 a, float_status *status)
 {
     int64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float32_to_int64_round_to_zero(a STATUS_VAR);
+    v = float32_to_int64_round_to_zero(a, status);
     if (v < 0) {
         res = 0;
     } else if (v > 0xffffffff) {
@@ -6875,17 +7102,17 @@
         return v;
     }
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-int_fast16_t float32_to_int16(float32 a STATUS_PARAM)
+int16_t float32_to_int16(float32 a, float_status *status)
 {
     int32_t v;
-    int_fast16_t res;
+    int16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float32_to_int32(a STATUS_VAR);
+    v = float32_to_int32(a, status);
     if (v < -0x8000) {
         res = -0x8000;
     } else if (v > 0x7fff) {
@@ -6895,17 +7122,17 @@
     }
 
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint_fast16_t float32_to_uint16(float32 a STATUS_PARAM)
+uint16_t float32_to_uint16(float32 a, float_status *status)
 {
     int32_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float32_to_int32(a STATUS_VAR);
+    v = float32_to_int32(a, status);
     if (v < 0) {
         res = 0;
     } else if (v > 0xffff) {
@@ -6915,17 +7142,17 @@
     }
 
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint_fast16_t float32_to_uint16_round_to_zero(float32 a STATUS_PARAM)
+uint16_t float32_to_uint16_round_to_zero(float32 a, float_status *status)
 {
     int64_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float32_to_int64_round_to_zero(a STATUS_VAR);
+    v = float32_to_int64_round_to_zero(a, status);
     if (v < 0) {
         res = 0;
     } else if (v > 0xffff) {
@@ -6934,51 +7161,51 @@
         return v;
     }
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint32 float64_to_uint32( float64 a STATUS_PARAM )
+uint32_t float64_to_uint32(float64 a, float_status *status)
 {
     uint64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float64_to_uint64(a STATUS_VAR);
+    v = float64_to_uint64(a, status);
     if (v > 0xffffffff) {
         res = 0xffffffff;
     } else {
         return v;
     }
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint32 float64_to_uint32_round_to_zero( float64 a STATUS_PARAM )
+uint32_t float64_to_uint32_round_to_zero(float64 a, float_status *status)
 {
     uint64_t v;
-    uint32 res;
+    uint32_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float64_to_uint64_round_to_zero(a STATUS_VAR);
+    v = float64_to_uint64_round_to_zero(a, status);
     if (v > 0xffffffff) {
         res = 0xffffffff;
     } else {
         return v;
     }
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-int_fast16_t float64_to_int16(float64 a STATUS_PARAM)
+int16_t float64_to_int16(float64 a, float_status *status)
 {
     int64_t v;
-    int_fast16_t res;
+    int16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float64_to_int32(a STATUS_VAR);
+    v = float64_to_int32(a, status);
     if (v < -0x8000) {
         res = -0x8000;
     } else if (v > 0x7fff) {
@@ -6988,17 +7215,17 @@
     }
 
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint_fast16_t float64_to_uint16(float64 a STATUS_PARAM)
+uint16_t float64_to_uint16(float64 a, float_status *status)
 {
     int64_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float64_to_int32(a STATUS_VAR);
+    v = float64_to_int32(a, status);
     if (v < 0) {
         res = 0;
     } else if (v > 0xffff) {
@@ -7008,17 +7235,17 @@
     }
 
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
-uint_fast16_t float64_to_uint16_round_to_zero(float64 a STATUS_PARAM)
+uint16_t float64_to_uint16_round_to_zero(float64 a, float_status *status)
 {
     int64_t v;
-    uint_fast16_t res;
+    uint16_t res;
     int old_exc_flags = get_float_exception_flags(status);
 
-    v = float64_to_int64_round_to_zero(a STATUS_VAR);
+    v = float64_to_int64_round_to_zero(a, status);
     if (v < 0) {
         res = 0;
     } else if (v > 0xffff) {
@@ -7027,7 +7254,7 @@
         return v;
     }
     set_float_exception_flags(old_exc_flags, status);
-    float_raise(float_flag_invalid STATUS_VAR);
+    float_raise(float_flag_invalid, status);
     return res;
 }
 
@@ -7043,18 +7270,19 @@
 | will raise the inexact exception.
 *----------------------------------------------------------------------------*/
 
-uint64_t float64_to_uint64(float64 a STATUS_PARAM)
+uint64_t float64_to_uint64(float64 a, float_status *status)
 {
     flag aSign;
-    int_fast16_t aExp, shiftCount;
+    int aExp;
+    int shiftCount;
     uint64_t aSig, aSigExtra;
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
 
     aSig = extractFloat64Frac(a);
     aExp = extractFloat64Exp(a);
     aSign = extractFloat64Sign(a);
     if (aSign && (aExp > 1022)) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         if (float64_is_any_nan(a)) {
             return LIT64(0xFFFFFFFFFFFFFFFF);
         } else {
@@ -7067,7 +7295,7 @@
     shiftCount = 0x433 - aExp;
     if (shiftCount <= 0) {
         if (0x43E < aExp) {
-            float_raise(float_flag_invalid STATUS_VAR);
+            float_raise(float_flag_invalid, status);
             return LIT64(0xFFFFFFFFFFFFFFFF);
         }
         aSigExtra = 0;
@@ -7075,35 +7303,35 @@
     } else {
         shift64ExtraRightJamming(aSig, 0, shiftCount, &aSig, &aSigExtra);
     }
-    return roundAndPackUint64(aSign, aSig, aSigExtra STATUS_VAR);
+    return roundAndPackUint64(aSign, aSig, aSigExtra, status);
 }
 
-uint64_t float64_to_uint64_round_to_zero (float64 a STATUS_PARAM)
+uint64_t float64_to_uint64_round_to_zero(float64 a, float_status *status)
 {
-    signed char current_rounding_mode = STATUS(float_rounding_mode);
-    set_float_rounding_mode(float_round_to_zero STATUS_VAR);
-    int64_t v = float64_to_uint64(a STATUS_VAR);
-    set_float_rounding_mode(current_rounding_mode STATUS_VAR);
+    signed char current_rounding_mode = status->float_rounding_mode;
+    set_float_rounding_mode(float_round_to_zero, status);
+    int64_t v = float64_to_uint64(a, status);
+    set_float_rounding_mode(current_rounding_mode, status);
     return v;
 }
 
 #define COMPARE(s, nan_exp)                                                  \
-static inline int float ## s ## _compare_internal( float ## s a, float ## s b,      \
-                                      int is_quiet STATUS_PARAM )            \
+static inline int float ## s ## _compare_internal(float ## s a, float ## s b,\
+                                      int is_quiet, float_status *status)    \
 {                                                                            \
     flag aSign, bSign;                                                       \
     uint ## s ## _t av, bv;                                                  \
-    a = float ## s ## _squash_input_denormal(a STATUS_VAR);                  \
-    b = float ## s ## _squash_input_denormal(b STATUS_VAR);                  \
+    a = float ## s ## _squash_input_denormal(a, status);                     \
+    b = float ## s ## _squash_input_denormal(b, status);                     \
                                                                              \
     if (( ( extractFloat ## s ## Exp( a ) == nan_exp ) &&                    \
          extractFloat ## s ## Frac( a ) ) ||                                 \
         ( ( extractFloat ## s ## Exp( b ) == nan_exp ) &&                    \
           extractFloat ## s ## Frac( b ) )) {                                \
         if (!is_quiet ||                                                     \
-            float ## s ## _is_signaling_nan( a ) ||                          \
-            float ## s ## _is_signaling_nan( b ) ) {                         \
-            float_raise( float_flag_invalid STATUS_VAR);                     \
+            float ## s ## _is_signaling_nan(a, status) ||                  \
+            float ## s ## _is_signaling_nan(b, status)) {                 \
+            float_raise(float_flag_invalid, status);                         \
         }                                                                    \
         return float_relation_unordered;                                     \
     }                                                                        \
@@ -7127,21 +7355,22 @@
     }                                                                        \
 }                                                                            \
                                                                              \
-int float ## s ## _compare( float ## s a, float ## s b STATUS_PARAM )        \
+int float ## s ## _compare(float ## s a, float ## s b, float_status *status) \
 {                                                                            \
-    return float ## s ## _compare_internal(a, b, 0 STATUS_VAR);              \
+    return float ## s ## _compare_internal(a, b, 0, status);                 \
 }                                                                            \
                                                                              \
-int float ## s ## _compare_quiet( float ## s a, float ## s b STATUS_PARAM )  \
+int float ## s ## _compare_quiet(float ## s a, float ## s b,                 \
+                                 float_status *status)                       \
 {                                                                            \
-    return float ## s ## _compare_internal(a, b, 1 STATUS_VAR);              \
+    return float ## s ## _compare_internal(a, b, 1, status);                 \
 }
 
 COMPARE(32, 0xff)
 COMPARE(64, 0x7ff)
 
-static inline int floatx80_compare_internal( floatx80 a, floatx80 b,
-                                      int is_quiet STATUS_PARAM )
+static inline int floatx80_compare_internal(floatx80 a, floatx80 b,
+                                            int is_quiet, float_status *status)
 {
     flag aSign, bSign;
 
@@ -7150,9 +7379,9 @@
         ( ( extractFloatx80Exp( b ) == 0x7fff ) &&
           ( extractFloatx80Frac( b )<<1 ) )) {
         if (!is_quiet ||
-            floatx80_is_signaling_nan( a ) ||
-            floatx80_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            floatx80_is_signaling_nan(a, status) ||
+            floatx80_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return float_relation_unordered;
     }
@@ -7176,18 +7405,18 @@
     }
 }
 
-int floatx80_compare( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_compare(floatx80 a, floatx80 b, float_status *status)
 {
-    return floatx80_compare_internal(a, b, 0 STATUS_VAR);
+    return floatx80_compare_internal(a, b, 0, status);
 }
 
-int floatx80_compare_quiet( floatx80 a, floatx80 b STATUS_PARAM )
+int floatx80_compare_quiet(floatx80 a, floatx80 b, float_status *status)
 {
-    return floatx80_compare_internal(a, b, 1 STATUS_VAR);
+    return floatx80_compare_internal(a, b, 1, status);
 }
 
-static inline int float128_compare_internal( float128 a, float128 b,
-                                      int is_quiet STATUS_PARAM )
+static inline int float128_compare_internal(float128 a, float128 b,
+                                            int is_quiet, float_status *status)
 {
     flag aSign, bSign;
 
@@ -7196,9 +7425,9 @@
         ( ( extractFloat128Exp( b ) == 0x7fff ) &&
           ( extractFloat128Frac0( b ) | extractFloat128Frac1( b ) ) )) {
         if (!is_quiet ||
-            float128_is_signaling_nan( a ) ||
-            float128_is_signaling_nan( b ) ) {
-            float_raise( float_flag_invalid STATUS_VAR);
+            float128_is_signaling_nan(a, status) ||
+            float128_is_signaling_nan(b, status)) {
+            float_raise(float_flag_invalid, status);
         }
         return float_relation_unordered;
     }
@@ -7220,14 +7449,14 @@
     }
 }
 
-int float128_compare( float128 a, float128 b STATUS_PARAM )
+int float128_compare(float128 a, float128 b, float_status *status)
 {
-    return float128_compare_internal(a, b, 0 STATUS_VAR);
+    return float128_compare_internal(a, b, 0, status);
 }
 
-int float128_compare_quiet( float128 a, float128 b STATUS_PARAM )
+int float128_compare_quiet(float128 a, float128 b, float_status *status)
 {
-    return float128_compare_internal(a, b, 1 STATUS_VAR);
+    return float128_compare_internal(a, b, 1, status);
 }
 
 /* min() and max() functions. These can't be implemented as
@@ -7247,24 +7476,25 @@
 #define MINMAX(s)                                                       \
 static inline float ## s float ## s ## _minmax(float ## s a, float ## s b,     \
                                                int ismin, int isieee,   \
-                                               int ismag STATUS_PARAM)  \
+                                               int ismag,               \
+                                               float_status *status)    \
 {                                                                       \
     flag aSign, bSign;                                                  \
     uint ## s ## _t av, bv, aav, abv;                                   \
-    a = float ## s ## _squash_input_denormal(a STATUS_VAR);             \
-    b = float ## s ## _squash_input_denormal(b STATUS_VAR);             \
+    a = float ## s ## _squash_input_denormal(a, status);                \
+    b = float ## s ## _squash_input_denormal(b, status);                \
     if (float ## s ## _is_any_nan(a) ||                                 \
         float ## s ## _is_any_nan(b)) {                                 \
         if (isieee) {                                                   \
-            if (float ## s ## _is_quiet_nan(a) &&                       \
+            if (float ## s ## _is_quiet_nan(a, status) &&               \
                 !float ## s ##_is_any_nan(b)) {                         \
                 return b;                                               \
-            } else if (float ## s ## _is_quiet_nan(b) &&                \
-                       !float ## s ## _is_any_nan(a)) {                 \
+            } else if (float ## s ## _is_quiet_nan(b, status) &&        \
+                       !float ## s ## _is_any_nan(a)) {                \
                 return a;                                               \
             }                                                           \
         }                                                               \
-        return propagateFloat ## s ## NaN(a, b STATUS_VAR);             \
+        return propagateFloat ## s ## NaN(a, b, status);                \
     }                                                                   \
     aSign = extractFloat ## s ## Sign(a);                               \
     bSign = extractFloat ## s ## Sign(b);                               \
@@ -7296,34 +7526,40 @@
     }                                                                   \
 }                                                                       \
                                                                         \
-float ## s float ## s ## _min(float ## s a, float ## s b STATUS_PARAM)  \
+float ## s float ## s ## _min(float ## s a, float ## s b,               \
+                              float_status *status)                     \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 1, 0, 0 STATUS_VAR);             \
+    return float ## s ## _minmax(a, b, 1, 0, 0, status);                \
 }                                                                       \
                                                                         \
-float ## s float ## s ## _max(float ## s a, float ## s b STATUS_PARAM)  \
+float ## s float ## s ## _max(float ## s a, float ## s b,               \
+                              float_status *status)                     \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 0, 0, 0 STATUS_VAR);             \
+    return float ## s ## _minmax(a, b, 0, 0, 0, status);                \
 }                                                                       \
                                                                         \
-float ## s float ## s ## _minnum(float ## s a, float ## s b STATUS_PARAM) \
+float ## s float ## s ## _minnum(float ## s a, float ## s b,            \
+                                 float_status *status)                  \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 1, 1, 0 STATUS_VAR);             \
+    return float ## s ## _minmax(a, b, 1, 1, 0, status);                \
 }                                                                       \
                                                                         \
-float ## s float ## s ## _maxnum(float ## s a, float ## s b STATUS_PARAM) \
+float ## s float ## s ## _maxnum(float ## s a, float ## s b,            \
+                                 float_status *status)                  \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 0, 1, 0 STATUS_VAR);             \
+    return float ## s ## _minmax(a, b, 0, 1, 0, status);                \
 }                                                                       \
                                                                         \
-float ## s float ## s ## _minnummag(float ## s a, float ## s b STATUS_PARAM) \
+float ## s float ## s ## _minnummag(float ## s a, float ## s b,         \
+                                    float_status *status)               \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 1, 1, 1 STATUS_VAR);             \
+    return float ## s ## _minmax(a, b, 1, 1, 1, status);                \
 }                                                                       \
                                                                         \
-float ## s float ## s ## _maxnummag(float ## s a, float ## s b STATUS_PARAM) \
+float ## s float ## s ## _maxnummag(float ## s a, float ## s b,         \
+                                    float_status *status)               \
 {                                                                       \
-    return float ## s ## _minmax(a, b, 0, 1, 1 STATUS_VAR);             \
+    return float ## s ## _minmax(a, b, 0, 1, 1, status);                \
 }
 
 MINMAX(32)
@@ -7331,20 +7567,20 @@
 
 
 /* Multiply A by 2 raised to the power N.  */
-float32 float32_scalbn( float32 a, int n STATUS_PARAM )
+float32 float32_scalbn(float32 a, int n, float_status *status)
 {
     flag aSign;
     int16_t aExp;
     uint32_t aSig;
 
-    a = float32_squash_input_denormal(a STATUS_VAR);
+    a = float32_squash_input_denormal(a, status);
     aSig = extractFloat32Frac( a );
     aExp = extractFloat32Exp( a );
     aSign = extractFloat32Sign( a );
 
     if ( aExp == 0xFF ) {
         if ( aSig ) {
-            return propagateFloat32NaN( a, a STATUS_VAR );
+            return propagateFloat32NaN(a, a, status);
         }
         return a;
     }
@@ -7364,23 +7600,23 @@
 
     aExp += n - 1;
     aSig <<= 7;
-    return normalizeRoundAndPackFloat32( aSign, aExp, aSig STATUS_VAR );
+    return normalizeRoundAndPackFloat32(aSign, aExp, aSig, status);
 }
 
-float64 float64_scalbn( float64 a, int n STATUS_PARAM )
+float64 float64_scalbn(float64 a, int n, float_status *status)
 {
     flag aSign;
     int16_t aExp;
     uint64_t aSig;
 
-    a = float64_squash_input_denormal(a STATUS_VAR);
+    a = float64_squash_input_denormal(a, status);
     aSig = extractFloat64Frac( a );
     aExp = extractFloat64Exp( a );
     aSign = extractFloat64Sign( a );
 
     if ( aExp == 0x7FF ) {
         if ( aSig ) {
-            return propagateFloat64NaN( a, a STATUS_VAR );
+            return propagateFloat64NaN(a, a, status);
         }
         return a;
     }
@@ -7400,10 +7636,10 @@
 
     aExp += n - 1;
     aSig <<= 10;
-    return normalizeRoundAndPackFloat64( aSign, aExp, aSig STATUS_VAR );
+    return normalizeRoundAndPackFloat64(aSign, aExp, aSig, status);
 }
 
-floatx80 floatx80_scalbn( floatx80 a, int n STATUS_PARAM )
+floatx80 floatx80_scalbn(floatx80 a, int n, float_status *status)
 {
     flag aSign;
     int32_t aExp;
@@ -7415,7 +7651,7 @@
 
     if ( aExp == 0x7FFF ) {
         if ( aSig<<1 ) {
-            return propagateFloatx80NaN( a, a STATUS_VAR );
+            return propagateFloatx80NaN(a, a, status);
         }
         return a;
     }
@@ -7434,11 +7670,11 @@
     }
 
     aExp += n;
-    return normalizeRoundAndPackFloatx80( STATUS(floatx80_rounding_precision),
-                                          aSign, aExp, aSig, 0 STATUS_VAR );
+    return normalizeRoundAndPackFloatx80(status->floatx80_rounding_precision,
+                                         aSign, aExp, aSig, 0, status);
 }
 
-float128 float128_scalbn( float128 a, int n STATUS_PARAM )
+float128 float128_scalbn(float128 a, int n, float_status *status)
 {
     flag aSign;
     int32_t aExp;
@@ -7450,7 +7686,7 @@
     aSign = extractFloat128Sign( a );
     if ( aExp == 0x7FFF ) {
         if ( aSig0 | aSig1 ) {
-            return propagateFloat128NaN( a, a STATUS_VAR );
+            return propagateFloat128NaN(a, a, status);
         }
         return a;
     }
@@ -7470,6 +7706,6 @@
 
     aExp += n - 1;
     return normalizeRoundAndPackFloat128( aSign, aExp, aSig0, aSig1
-                                          STATUS_VAR );
+                                         , status);
 
 }
diff --git a/fsdev/9p-iov-marshal.c b/fsdev/9p-iov-marshal.c
new file mode 100644
index 0000000..663cad5
--- /dev/null
+++ b/fsdev/9p-iov-marshal.c
@@ -0,0 +1,306 @@
+/*
+ * 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib/gprintf.h>
+#include <utime.h>
+
+#include "9p-iov-marshal.h"
+#include "qemu/bswap.h"
+
+static ssize_t v9fs_packunpack(void *addr, struct iovec *sg, int sg_count,
+                               size_t offset, size_t size, int pack)
+{
+    int i = 0;
+    size_t copied = 0;
+    size_t req_size = size;
+
+
+    for (i = 0; size && i < sg_count; i++) {
+        size_t len;
+        if (offset >= sg[i].iov_len) {
+            /* skip this sg */
+            offset -= sg[i].iov_len;
+            continue;
+        } else {
+            len = MIN(sg[i].iov_len - offset, size);
+            if (pack) {
+                memcpy(sg[i].iov_base + offset, addr, len);
+            } else {
+                memcpy(addr, sg[i].iov_base + offset, len);
+            }
+            size -= len;
+            copied += len;
+            addr += len;
+            if (size) {
+                offset = 0;
+                continue;
+            }
+        }
+    }
+    if (copied < req_size) {
+        /*
+         * We copied less that requested size. error out
+         */
+        return -ENOBUFS;
+    }
+    return copied;
+}
+
+static ssize_t v9fs_unpack(void *dst, struct iovec *out_sg, int out_num,
+                           size_t offset, size_t size)
+{
+    return v9fs_packunpack(dst, out_sg, out_num, offset, size, 0);
+}
+
+ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
+                  const void *src, size_t size)
+{
+    return v9fs_packunpack((void *)src, in_sg, in_num, offset, size, 1);
+}
+
+ssize_t v9fs_iov_vunmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                            int bswap, const char *fmt, va_list ap)
+{
+    int i;
+    ssize_t copied = 0;
+    size_t old_offset = offset;
+
+    for (i = 0; fmt[i]; i++) {
+        switch (fmt[i]) {
+        case 'b': {
+            uint8_t *valp = va_arg(ap, uint8_t *);
+            copied = v9fs_unpack(valp, out_sg, out_num, offset, sizeof(*valp));
+            break;
+        }
+        case 'w': {
+            uint16_t val, *valp;
+            valp = va_arg(ap, uint16_t *);
+            copied = v9fs_unpack(&val, out_sg, out_num, offset, sizeof(val));
+            if (bswap) {
+                *valp = le16_to_cpu(val);
+            } else {
+                *valp = val;
+            }
+            break;
+        }
+        case 'd': {
+            uint32_t val, *valp;
+            valp = va_arg(ap, uint32_t *);
+            copied = v9fs_unpack(&val, out_sg, out_num, offset, sizeof(val));
+            if (bswap) {
+                *valp = le32_to_cpu(val);
+            } else {
+                *valp = val;
+            }
+            break;
+        }
+        case 'q': {
+            uint64_t val, *valp;
+            valp = va_arg(ap, uint64_t *);
+            copied = v9fs_unpack(&val, out_sg, out_num, offset, sizeof(val));
+            if (bswap) {
+                *valp = le64_to_cpu(val);
+            } else {
+                *valp = val;
+            }
+            break;
+        }
+        case 's': {
+            V9fsString *str = va_arg(ap, V9fsString *);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "w", &str->size);
+            if (copied > 0) {
+                offset += copied;
+                str->data = g_malloc(str->size + 1);
+                copied = v9fs_unpack(str->data, out_sg, out_num, offset,
+                                     str->size);
+                if (copied > 0) {
+                    str->data[str->size] = 0;
+                } else {
+                    v9fs_string_free(str);
+                }
+            }
+            break;
+        }
+        case 'Q': {
+            V9fsQID *qidp = va_arg(ap, V9fsQID *);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "bdq", &qidp->type, &qidp->version,
+                                        &qidp->path);
+            break;
+        }
+        case 'S': {
+            V9fsStat *statp = va_arg(ap, V9fsStat *);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "wwdQdddqsssssddd",
+                                        &statp->size, &statp->type,
+                                        &statp->dev, &statp->qid,
+                                        &statp->mode, &statp->atime,
+                                        &statp->mtime, &statp->length,
+                                        &statp->name, &statp->uid,
+                                        &statp->gid, &statp->muid,
+                                        &statp->extension,
+                                        &statp->n_uid, &statp->n_gid,
+                                        &statp->n_muid);
+            break;
+        }
+        case 'I': {
+            V9fsIattr *iattr = va_arg(ap, V9fsIattr *);
+            copied = v9fs_iov_unmarshal(out_sg, out_num, offset, bswap,
+                                        "ddddqqqqq",
+                                        &iattr->valid, &iattr->mode,
+                                        &iattr->uid, &iattr->gid,
+                                        &iattr->size, &iattr->atime_sec,
+                                        &iattr->atime_nsec,
+                                        &iattr->mtime_sec,
+                                        &iattr->mtime_nsec);
+            break;
+        }
+        default:
+            break;
+        }
+        if (copied < 0) {
+            return copied;
+        }
+        offset += copied;
+    }
+
+    return offset - old_offset;
+}
+
+ssize_t v9fs_iov_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                           int bswap, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = v9fs_iov_vunmarshal(out_sg, out_num, offset, bswap, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+ssize_t v9fs_iov_vmarshal(struct iovec *in_sg, int in_num, size_t offset,
+                          int bswap, const char *fmt, va_list ap)
+{
+    int i;
+    ssize_t copied = 0;
+    size_t old_offset = offset;
+
+    for (i = 0; fmt[i]; i++) {
+        switch (fmt[i]) {
+        case 'b': {
+            uint8_t val = va_arg(ap, int);
+            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
+            break;
+        }
+        case 'w': {
+            uint16_t val = va_arg(ap, int);
+            if (bswap) {
+                val = cpu_to_le16(val);
+            }
+            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
+            break;
+        }
+        case 'd': {
+            uint32_t val = va_arg(ap, uint32_t);
+            if (bswap) {
+                val = cpu_to_le32(val);
+            }
+            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
+            break;
+        }
+        case 'q': {
+            uint64_t val = va_arg(ap, uint64_t);
+            if (bswap) {
+                val = cpu_to_le64(val);
+            }
+            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
+            break;
+        }
+        case 's': {
+            V9fsString *str = va_arg(ap, V9fsString *);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap,
+                                      "w", str->size);
+            if (copied > 0) {
+                offset += copied;
+                copied = v9fs_pack(in_sg, in_num, offset, str->data, str->size);
+            }
+            break;
+        }
+        case 'Q': {
+            V9fsQID *qidp = va_arg(ap, V9fsQID *);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap, "bdq",
+                                      qidp->type, qidp->version,
+                                      qidp->path);
+            break;
+        }
+        case 'S': {
+            V9fsStat *statp = va_arg(ap, V9fsStat *);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap,
+                                      "wwdQdddqsssssddd",
+                                      statp->size, statp->type, statp->dev,
+                                      &statp->qid, statp->mode, statp->atime,
+                                      statp->mtime, statp->length,
+                                      &statp->name,
+                                      &statp->uid, &statp->gid, &statp->muid,
+                                      &statp->extension, statp->n_uid,
+                                      statp->n_gid, statp->n_muid);
+            break;
+        }
+        case 'A': {
+            V9fsStatDotl *statp = va_arg(ap, V9fsStatDotl *);
+            copied = v9fs_iov_marshal(in_sg, in_num, offset, bswap,
+                                      "qQdddqqqqqqqqqqqqqqq",
+                                      statp->st_result_mask,
+                                      &statp->qid, statp->st_mode,
+                                      statp->st_uid, statp->st_gid,
+                                      statp->st_nlink, statp->st_rdev,
+                                      statp->st_size, statp->st_blksize,
+                                      statp->st_blocks, statp->st_atime_sec,
+                                      statp->st_atime_nsec,
+                                      statp->st_mtime_sec,
+                                      statp->st_mtime_nsec,
+                                      statp->st_ctime_sec,
+                                      statp->st_ctime_nsec,
+                                      statp->st_btime_sec,
+                                      statp->st_btime_nsec, statp->st_gen,
+                                      statp->st_data_version);
+            break;
+        }
+        default:
+            break;
+        }
+        if (copied < 0) {
+            return copied;
+        }
+        offset += copied;
+    }
+
+    return offset - old_offset;
+}
+
+ssize_t v9fs_iov_marshal(struct iovec *in_sg, int in_num, size_t offset,
+                         int bswap, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = v9fs_iov_vmarshal(in_sg, in_num, offset, bswap, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
diff --git a/fsdev/9p-iov-marshal.h b/fsdev/9p-iov-marshal.h
new file mode 100644
index 0000000..1fb5016
--- /dev/null
+++ b/fsdev/9p-iov-marshal.h
@@ -0,0 +1,18 @@
+#ifndef QEMU_9P_IOV_MARSHAL_H
+#define QEMU_9P_IOV_MARSHAL_H
+
+#include "9p-marshal.h"
+
+
+ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
+                  const void *src, size_t size);
+ssize_t v9fs_iov_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                           int bswap, const char *fmt, ...);
+ssize_t v9fs_iov_marshal(struct iovec *in_sg, int in_num, size_t offset,
+                         int bswap, const char *fmt, ...);
+
+ssize_t v9fs_iov_vunmarshal(struct iovec *out_sg, int out_num, size_t offset,
+                            int bswap, const char *fmt, va_list ap);
+ssize_t v9fs_iov_vmarshal(struct iovec *in_sg, int in_num, size_t offset,
+                          int bswap, const char *fmt, va_list ap);
+#endif
diff --git a/fsdev/9p-marshal.c b/fsdev/9p-marshal.c
new file mode 100644
index 0000000..238dbf2
--- /dev/null
+++ b/fsdev/9p-marshal.c
@@ -0,0 +1,49 @@
+/*
+ * 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <glib/gprintf.h>
+#include <dirent.h>
+#include <utime.h>
+
+#include "9p-marshal.h"
+
+void v9fs_string_free(V9fsString *str)
+{
+    g_free(str->data);
+    str->data = NULL;
+    str->size = 0;
+}
+
+void v9fs_string_null(V9fsString *str)
+{
+    v9fs_string_free(str);
+}
+
+void GCC_FMT_ATTR(2, 3)
+v9fs_string_sprintf(V9fsString *str, const char *fmt, ...)
+{
+    va_list ap;
+
+    v9fs_string_free(str);
+
+    va_start(ap, fmt);
+    str->size = g_vasprintf(&str->data, fmt, ap);
+    va_end(ap);
+}
+
+void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs)
+{
+    v9fs_string_free(lhs);
+    v9fs_string_sprintf(lhs, "%s", rhs->data);
+}
diff --git a/fsdev/9p-marshal.h b/fsdev/9p-marshal.h
new file mode 100644
index 0000000..140db6d
--- /dev/null
+++ b/fsdev/9p-marshal.h
@@ -0,0 +1,84 @@
+#ifndef QEMU_9P_MARSHAL_H
+#define QEMU_9P_MARSHAL_H
+
+typedef struct V9fsString
+{
+    uint16_t size;
+    char *data;
+} V9fsString;
+
+typedef struct V9fsQID
+{
+    int8_t type;
+    int32_t version;
+    int64_t path;
+} V9fsQID;
+
+typedef struct V9fsStat
+{
+    int16_t size;
+    int16_t type;
+    int32_t dev;
+    V9fsQID qid;
+    int32_t mode;
+    int32_t atime;
+    int32_t mtime;
+    int64_t length;
+    V9fsString name;
+    V9fsString uid;
+    V9fsString gid;
+    V9fsString muid;
+    /* 9p2000.u */
+    V9fsString extension;
+    int32_t n_uid;
+    int32_t n_gid;
+    int32_t n_muid;
+} V9fsStat;
+
+typedef struct V9fsIattr
+{
+    int32_t valid;
+    int32_t mode;
+    int32_t uid;
+    int32_t gid;
+    int64_t size;
+    int64_t atime_sec;
+    int64_t atime_nsec;
+    int64_t mtime_sec;
+    int64_t mtime_nsec;
+} V9fsIattr;
+
+typedef struct V9fsStatDotl {
+    uint64_t st_result_mask;
+    V9fsQID qid;
+    uint32_t st_mode;
+    uint32_t st_uid;
+    uint32_t st_gid;
+    uint64_t st_nlink;
+    uint64_t st_rdev;
+    uint64_t st_size;
+    uint64_t st_blksize;
+    uint64_t st_blocks;
+    uint64_t st_atime_sec;
+    uint64_t st_atime_nsec;
+    uint64_t st_mtime_sec;
+    uint64_t st_mtime_nsec;
+    uint64_t st_ctime_sec;
+    uint64_t st_ctime_nsec;
+    uint64_t st_btime_sec;
+    uint64_t st_btime_nsec;
+    uint64_t st_gen;
+    uint64_t st_data_version;
+} V9fsStatDotl;
+
+static inline void v9fs_string_init(V9fsString *str)
+{
+    str->data = NULL;
+    str->size = 0;
+}
+extern void v9fs_string_free(V9fsString *str);
+extern void v9fs_string_null(V9fsString *str);
+extern void v9fs_string_sprintf(V9fsString *str, const char *fmt, ...);
+extern void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs);
+
+#endif
diff --git a/fsdev/Makefile.objs b/fsdev/Makefile.objs
index c27dad3..1b120a4 100644
--- a/fsdev/Makefile.objs
+++ b/fsdev/Makefile.objs
@@ -1,7 +1,7 @@
 ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
 # Lots of the fsdev/9pcode is pulled in by vl.c via qemu_fsdev_add.
 # only pull in the actual virtio-9p device if we also enabled virtio.
-common-obj-y = qemu-fsdev.o virtio-9p-marshal.o
+common-obj-y = qemu-fsdev.o 9p-marshal.o 9p-iov-marshal.o
 else
 common-obj-y = qemu-fsdev-dummy.o
 endif
diff --git a/fsdev/file-op-9p.h b/fsdev/file-op-9p.h
index 956fda0..6db9fea 100644
--- a/fsdev/file-op-9p.h
+++ b/fsdev/file-op-9p.h
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * Copyright IBM, Corp. 2010
  *
@@ -10,14 +10,12 @@
  * the COPYING file in the top-level directory.
  *
  */
-#ifndef _FILEOP_H
-#define _FILEOP_H
-#include <sys/types.h>
+
+#ifndef FILE_OP_9P_H
+#define FILE_OP_9P_H
+
 #include <dirent.h>
-#include <sys/time.h>
 #include <utime.h>
-#include <sys/stat.h>
-#include <sys/uio.h>
 #include <sys/vfs.h>
 
 #define SM_LOCAL_MODE_BITS    0600
@@ -121,8 +119,7 @@
                  int, FsCred *, V9fsFidOpenState *);
     void (*rewinddir)(FsContext *, V9fsFidOpenState *);
     off_t (*telldir)(FsContext *, V9fsFidOpenState *);
-    int (*readdir_r)(FsContext *, V9fsFidOpenState *,
-                     struct dirent *, struct dirent **);
+    struct dirent * (*readdir)(FsContext *, V9fsFidOpenState *);
     void (*seekdir)(FsContext *, V9fsFidOpenState *, off_t);
     ssize_t (*preadv)(FsContext *, V9fsFidOpenState *,
                       const struct iovec *, int, off_t);
diff --git a/fsdev/qemu-fsdev-dummy.c b/fsdev/qemu-fsdev-dummy.c
index 7dc2630..6dc0fbc 100644
--- a/fsdev/qemu-fsdev-dummy.c
+++ b/fsdev/qemu-fsdev-dummy.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * Copyright IBM, Corp. 2010
  *
@@ -10,8 +10,7 @@
  * the COPYING file in the top-level directory.
  *
  */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include "qemu-fsdev.h"
 #include "qemu/config-file.h"
 #include "qemu/module.h"
diff --git a/fsdev/qemu-fsdev-opts.c b/fsdev/qemu-fsdev-opts.c
index 6311c7a..1dd8c7a 100644
--- a/fsdev/qemu-fsdev-opts.c
+++ b/fsdev/qemu-fsdev-opts.c
@@ -1,10 +1,11 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/config-file.h"
 #include "qemu/option.h"
 #include "qemu/module.h"
@@ -82,4 +83,4 @@
     qemu_add_opts(&qemu_fsdev_opts);
     qemu_add_opts(&qemu_virtfs_opts);
 }
-machine_init(fsdev_register_config);
+opts_init(fsdev_register_config);
diff --git a/fsdev/qemu-fsdev.c b/fsdev/qemu-fsdev.c
index ccfec13..266e442 100644
--- a/fsdev/qemu-fsdev.c
+++ b/fsdev/qemu-fsdev.c
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * Copyright IBM, Corp. 2010
  *
@@ -10,13 +10,12 @@
  * the COPYING file in the top-level directory.
  *
  */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include "qemu-fsdev.h"
 #include "qemu/queue.h"
-#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 
 static QTAILQ_HEAD(FsDriverEntry_head, FsDriverListEntry) fsdriver_entries =
     QTAILQ_HEAD_INITIALIZER(fsdriver_entries);
@@ -40,7 +39,7 @@
     bool ro = qemu_opt_get_bool(opts, "readonly", 0);
 
     if (!fsdev_id) {
-        fprintf(stderr, "fsdev: No id specified\n");
+        error_report("fsdev: No id specified");
         return -1;
     }
 
@@ -52,11 +51,11 @@
         }
 
         if (i == ARRAY_SIZE(FsDrivers)) {
-            fprintf(stderr, "fsdev: fsdriver %s not found\n", fsdriver);
+            error_report("fsdev: fsdriver %s not found", fsdriver);
             return -1;
         }
     } else {
-        fprintf(stderr, "fsdev: No fsdriver specified\n");
+        error_report("fsdev: No fsdriver specified");
         return -1;
     }
 
diff --git a/fsdev/qemu-fsdev.h b/fsdev/qemu-fsdev.h
index 9fa45bf..29c9622 100644
--- a/fsdev/qemu-fsdev.h
+++ b/fsdev/qemu-fsdev.h
@@ -1,5 +1,5 @@
 /*
- * Virtio 9p
+ * 9p
  *
  * Copyright IBM, Corp. 2010
  *
diff --git a/fsdev/virtfs-proxy-helper.c b/fsdev/virtfs-proxy-helper.c
index cd291d3..c244d61 100644
--- a/fsdev/virtfs-proxy-helper.c
+++ b/fsdev/virtfs-proxy-helper.c
@@ -9,6 +9,7 @@
  * the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include <sys/resource.h>
 #include <getopt.h>
 #include <syslog.h>
@@ -23,9 +24,9 @@
 #include "qemu-common.h"
 #include "qemu/sockets.h"
 #include "qemu/xattr.h"
-#include "virtio-9p-marshal.h"
-#include "hw/9pfs/virtio-9p-proxy.h"
-#include "fsdev/virtio-9p-marshal.h"
+#include "9p-iov-marshal.h"
+#include "hw/9pfs/9p-proxy.h"
+#include "fsdev/9p-iov-marshal.h"
 
 #define PROGNAME "virtfs-proxy-helper"
 
@@ -49,6 +50,7 @@
     {"socket", required_argument, NULL, 's'},
     {"uid", required_argument, NULL, 'u'},
     {"gid", required_argument, NULL, 'g'},
+    {},
 };
 
 static bool is_daemon;
@@ -117,7 +119,7 @@
 
 static int init_capabilities(void)
 {
-    /* helper needs following capbabilities only */
+    /* helper needs following capabilities only */
     cap_value_t cap_list[] = {
         CAP_CHOWN,
         CAP_DAC_OVERRIDE,
@@ -262,6 +264,9 @@
      */
     msg_size = proxy_marshal(iovec, 0, "ddd", header.type,
                              header.size, status);
+    if (msg_size < 0) {
+        return msg_size;
+    }
     retval = socket_write(sockfd, iovec->iov_base, msg_size);
     if (retval < 0) {
         return retval;
@@ -544,7 +549,7 @@
 
     switch (type) {
     case T_LSTAT:
-        retval = lstat(path.data, &st_buf);
+        retval = qemu_lstat(path.data, &st_buf);
         if (retval < 0) {
             retval = -errno;
         } else {
@@ -735,6 +740,12 @@
         return -1;
     }
 
+    if (strlen(path) >= sizeof(proxy.sun_path)) {
+        do_log(LOG_CRIT, "UNIX domain socket path exceeds %zu characters\n",
+               sizeof(proxy.sun_path));
+        return -1;
+    }
+
     sock = socket(AF_UNIX, SOCK_STREAM, 0);
     if (sock < 0) {
         do_perror("socket");
@@ -749,24 +760,29 @@
     if (bind(sock, (struct sockaddr *)&proxy,
             sizeof(struct sockaddr_un)) < 0) {
         do_perror("bind");
-        return -1;
+        goto error;
     }
     if (chown(proxy.sun_path, uid, gid) < 0) {
         do_perror("chown");
-        return -1;
+        goto error;
     }
     if (listen(sock, 1) < 0) {
         do_perror("listen");
-        return -1;
+        goto error;
     }
 
     size = sizeof(qemu);
     client = accept(sock, (struct sockaddr *)&qemu, &size);
     if (client < 0) {
         do_perror("accept");
-        return -1;
+        goto error;
     }
+    close(sock);
     return client;
+
+error:
+    close(sock);
+    return -1;
 }
 
 static void usage(char *prog)
@@ -1086,7 +1102,7 @@
         exit(EXIT_FAILURE);
     }
 
-    if (lstat(rpath, &stbuf) < 0) {
+    if (qemu_lstat(rpath, &stbuf) < 0) {
         fprintf(stderr, "invalid path \"%s\" specified, %s\n",
                 rpath, strerror(errno));
         exit(EXIT_FAILURE);
@@ -1113,10 +1129,19 @@
         }
     }
 
+    if (chdir("/") < 0) {
+        do_perror("chdir");
+        goto error;
+    }
+    if (chroot(rpath) < 0) {
+        do_perror("chroot");
+        goto error;
+    }
+
     get_version = false;
 #ifdef FS_IOC_GETVERSION
     /* check whether underlying FS support IOC_GETVERSION */
-    retval = statfs(rpath, &st_fs);
+    retval = statfs("/", &st_fs);
     if (!retval) {
         switch (st_fs.f_type) {
         case EXT2_SUPER_MAGIC:
@@ -1129,16 +1154,7 @@
     }
 #endif
 
-    if (chdir("/") < 0) {
-        do_perror("chdir");
-        goto error;
-    }
-    if (chroot(rpath) < 0) {
-        do_perror("chroot");
-        goto error;
-    }
     umask(0);
-
     if (init_capabilities() < 0) {
         goto error;
     }
diff --git a/fsdev/virtfs-proxy-helper.texi b/fsdev/virtfs-proxy-helper.texi
index e60e3b9..f4cbb60 100644
--- a/fsdev/virtfs-proxy-helper.texi
+++ b/fsdev/virtfs-proxy-helper.texi
@@ -1,6 +1,6 @@
 @example
 @c man begin SYNOPSIS
-usage: virtfs-proxy-helper options
+@command{virtfs-proxy-helper} @var{options}
 @c man end
 @end example
 
@@ -15,7 +15,7 @@
 
 2) Running QEMU with root privilege could be a security issue.
 
-To overcome above issues, following approach is used: A new filesytem
+To overcome above issues, following approach is used: A new filesystem
 type 'proxy' is introduced. Proxy FS uses chroot + socket combination
 for securing the vulnerability known with following symbolic links.
 Intention of adding a new filesystem type is to allow qemu to run
@@ -28,8 +28,8 @@
 driver sends filesystem request to proxy helper and receives the
 response from it.
 
-Proxy helper is designed so that it can drop the root privilege with
-retaining capbilities needed for doing filesystem operations only.
+The proxy helper is designed so that it can drop root privileges except
+for the capabilities needed for doing filesystem operations.
 
 @end table
 @c man end
diff --git a/fsdev/virtio-9p-marshal.c b/fsdev/virtio-9p-marshal.c
deleted file mode 100644
index 20f308b..0000000
--- a/fsdev/virtio-9p-marshal.c
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Virtio 9p backend
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include <glib.h>
-#include <glib/gprintf.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/time.h>
-#include <utime.h>
-#include <sys/uio.h>
-#include <string.h>
-#include <stdint.h>
-#include <errno.h>
-
-#include "qemu/compiler.h"
-#include "virtio-9p-marshal.h"
-#include "qemu/bswap.h"
-
-void v9fs_string_free(V9fsString *str)
-{
-    g_free(str->data);
-    str->data = NULL;
-    str->size = 0;
-}
-
-void v9fs_string_null(V9fsString *str)
-{
-    v9fs_string_free(str);
-}
-
-void GCC_FMT_ATTR(2, 3)
-v9fs_string_sprintf(V9fsString *str, const char *fmt, ...)
-{
-    va_list ap;
-
-    v9fs_string_free(str);
-
-    va_start(ap, fmt);
-    str->size = g_vasprintf(&str->data, fmt, ap);
-    va_end(ap);
-}
-
-void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs)
-{
-    v9fs_string_free(lhs);
-    v9fs_string_sprintf(lhs, "%s", rhs->data);
-}
-
-
-static ssize_t v9fs_packunpack(void *addr, struct iovec *sg, int sg_count,
-                               size_t offset, size_t size, int pack)
-{
-    int i = 0;
-    size_t copied = 0;
-    size_t req_size = size;
-
-
-    for (i = 0; size && i < sg_count; i++) {
-        size_t len;
-        if (offset >= sg[i].iov_len) {
-            /* skip this sg */
-            offset -= sg[i].iov_len;
-            continue;
-        } else {
-            len = MIN(sg[i].iov_len - offset, size);
-            if (pack) {
-                memcpy(sg[i].iov_base + offset, addr, len);
-            } else {
-                memcpy(addr, sg[i].iov_base + offset, len);
-            }
-            size -= len;
-            copied += len;
-            addr += len;
-            if (size) {
-                offset = 0;
-                continue;
-            }
-        }
-    }
-    if (copied < req_size) {
-        /*
-         * We copied less that requested size. error out
-         */
-        return -ENOBUFS;
-    }
-    return copied;
-}
-
-static ssize_t v9fs_unpack(void *dst, struct iovec *out_sg, int out_num,
-                           size_t offset, size_t size)
-{
-    return v9fs_packunpack(dst, out_sg, out_num, offset, size, 0);
-}
-
-ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
-                  const void *src, size_t size)
-{
-    return v9fs_packunpack((void *)src, in_sg, in_num, offset, size, 1);
-}
-
-ssize_t v9fs_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
-                       int bswap, const char *fmt, ...)
-{
-    int i;
-    va_list ap;
-    ssize_t copied = 0;
-    size_t old_offset = offset;
-
-    va_start(ap, fmt);
-    for (i = 0; fmt[i]; i++) {
-        switch (fmt[i]) {
-        case 'b': {
-            uint8_t *valp = va_arg(ap, uint8_t *);
-            copied = v9fs_unpack(valp, out_sg, out_num, offset, sizeof(*valp));
-            break;
-        }
-        case 'w': {
-            uint16_t val, *valp;
-            valp = va_arg(ap, uint16_t *);
-            copied = v9fs_unpack(&val, out_sg, out_num, offset, sizeof(val));
-            if (bswap) {
-                *valp = le16_to_cpu(val);
-            } else {
-                *valp = val;
-            }
-            break;
-        }
-        case 'd': {
-            uint32_t val, *valp;
-            valp = va_arg(ap, uint32_t *);
-            copied = v9fs_unpack(&val, out_sg, out_num, offset, sizeof(val));
-            if (bswap) {
-                *valp = le32_to_cpu(val);
-            } else {
-                *valp = val;
-            }
-            break;
-        }
-        case 'q': {
-            uint64_t val, *valp;
-            valp = va_arg(ap, uint64_t *);
-            copied = v9fs_unpack(&val, out_sg, out_num, offset, sizeof(val));
-            if (bswap) {
-                *valp = le64_to_cpu(val);
-            } else {
-                *valp = val;
-            }
-            break;
-        }
-        case 's': {
-            V9fsString *str = va_arg(ap, V9fsString *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap,
-                                    "w", &str->size);
-            if (copied > 0) {
-                offset += copied;
-                str->data = g_malloc(str->size + 1);
-                copied = v9fs_unpack(str->data, out_sg, out_num, offset,
-                                     str->size);
-                if (copied > 0) {
-                    str->data[str->size] = 0;
-                } else {
-                    v9fs_string_free(str);
-                }
-            }
-            break;
-        }
-        case 'Q': {
-            V9fsQID *qidp = va_arg(ap, V9fsQID *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap, "bdq",
-                                    &qidp->type, &qidp->version, &qidp->path);
-            break;
-        }
-        case 'S': {
-            V9fsStat *statp = va_arg(ap, V9fsStat *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap,
-                                    "wwdQdddqsssssddd",
-                                    &statp->size, &statp->type, &statp->dev,
-                                    &statp->qid, &statp->mode, &statp->atime,
-                                    &statp->mtime, &statp->length,
-                                    &statp->name, &statp->uid, &statp->gid,
-                                    &statp->muid, &statp->extension,
-                                    &statp->n_uid, &statp->n_gid,
-                                    &statp->n_muid);
-            break;
-        }
-        case 'I': {
-            V9fsIattr *iattr = va_arg(ap, V9fsIattr *);
-            copied = v9fs_unmarshal(out_sg, out_num, offset, bswap,
-                                    "ddddqqqqq",
-                                    &iattr->valid, &iattr->mode,
-                                    &iattr->uid, &iattr->gid, &iattr->size,
-                                    &iattr->atime_sec, &iattr->atime_nsec,
-                                    &iattr->mtime_sec, &iattr->mtime_nsec);
-            break;
-        }
-        default:
-            break;
-        }
-        if (copied < 0) {
-            va_end(ap);
-            return copied;
-        }
-        offset += copied;
-    }
-    va_end(ap);
-
-    return offset - old_offset;
-}
-
-ssize_t v9fs_marshal(struct iovec *in_sg, int in_num, size_t offset,
-                     int bswap, const char *fmt, ...)
-{
-    int i;
-    va_list ap;
-    ssize_t copied = 0;
-    size_t old_offset = offset;
-
-    va_start(ap, fmt);
-    for (i = 0; fmt[i]; i++) {
-        switch (fmt[i]) {
-        case 'b': {
-            uint8_t val = va_arg(ap, int);
-            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
-            break;
-        }
-        case 'w': {
-            uint16_t val;
-            if (bswap) {
-                cpu_to_le16w(&val, va_arg(ap, int));
-            } else {
-                val =  va_arg(ap, int);
-            }
-            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
-            break;
-        }
-        case 'd': {
-            uint32_t val;
-            if (bswap) {
-                cpu_to_le32w(&val, va_arg(ap, uint32_t));
-            } else {
-                val =  va_arg(ap, uint32_t);
-            }
-            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
-            break;
-        }
-        case 'q': {
-            uint64_t val;
-            if (bswap) {
-                cpu_to_le64w(&val, va_arg(ap, uint64_t));
-            } else {
-                val =  va_arg(ap, uint64_t);
-            }
-            copied = v9fs_pack(in_sg, in_num, offset, &val, sizeof(val));
-            break;
-        }
-        case 's': {
-            V9fsString *str = va_arg(ap, V9fsString *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap,
-                                  "w", str->size);
-            if (copied > 0) {
-                offset += copied;
-                copied = v9fs_pack(in_sg, in_num, offset, str->data, str->size);
-            }
-            break;
-        }
-        case 'Q': {
-            V9fsQID *qidp = va_arg(ap, V9fsQID *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap, "bdq",
-                                  qidp->type, qidp->version, qidp->path);
-            break;
-        }
-        case 'S': {
-            V9fsStat *statp = va_arg(ap, V9fsStat *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap,
-                                  "wwdQdddqsssssddd",
-                                  statp->size, statp->type, statp->dev,
-                                  &statp->qid, statp->mode, statp->atime,
-                                  statp->mtime, statp->length, &statp->name,
-                                  &statp->uid, &statp->gid, &statp->muid,
-                                  &statp->extension, statp->n_uid,
-                                  statp->n_gid, statp->n_muid);
-            break;
-        }
-        case 'A': {
-            V9fsStatDotl *statp = va_arg(ap, V9fsStatDotl *);
-            copied = v9fs_marshal(in_sg, in_num, offset, bswap,
-                                   "qQdddqqqqqqqqqqqqqqq",
-                                   statp->st_result_mask,
-                                   &statp->qid, statp->st_mode,
-                                   statp->st_uid, statp->st_gid,
-                                   statp->st_nlink, statp->st_rdev,
-                                   statp->st_size, statp->st_blksize,
-                                   statp->st_blocks, statp->st_atime_sec,
-                                   statp->st_atime_nsec, statp->st_mtime_sec,
-                                   statp->st_mtime_nsec, statp->st_ctime_sec,
-                                   statp->st_ctime_nsec, statp->st_btime_sec,
-                                   statp->st_btime_nsec, statp->st_gen,
-                                   statp->st_data_version);
-            break;
-        }
-        default:
-            break;
-        }
-        if (copied < 0) {
-            va_end(ap);
-            return copied;
-        }
-        offset += copied;
-    }
-    va_end(ap);
-
-    return offset - old_offset;
-}
diff --git a/fsdev/virtio-9p-marshal.h b/fsdev/virtio-9p-marshal.h
deleted file mode 100644
index 5df65a8..0000000
--- a/fsdev/virtio-9p-marshal.h
+++ /dev/null
@@ -1,90 +0,0 @@
-#ifndef _QEMU_VIRTIO_9P_MARSHAL_H
-#define _QEMU_VIRTIO_9P_MARSHAL_H
-
-typedef struct V9fsString
-{
-    uint16_t size;
-    char *data;
-} V9fsString;
-
-typedef struct V9fsQID
-{
-    int8_t type;
-    int32_t version;
-    int64_t path;
-} V9fsQID;
-
-typedef struct V9fsStat
-{
-    int16_t size;
-    int16_t type;
-    int32_t dev;
-    V9fsQID qid;
-    int32_t mode;
-    int32_t atime;
-    int32_t mtime;
-    int64_t length;
-    V9fsString name;
-    V9fsString uid;
-    V9fsString gid;
-    V9fsString muid;
-    /* 9p2000.u */
-    V9fsString extension;
-   int32_t n_uid;
-    int32_t n_gid;
-    int32_t n_muid;
-} V9fsStat;
-
-typedef struct V9fsIattr
-{
-    int32_t valid;
-    int32_t mode;
-    int32_t uid;
-    int32_t gid;
-    int64_t size;
-    int64_t atime_sec;
-    int64_t atime_nsec;
-    int64_t mtime_sec;
-    int64_t mtime_nsec;
-} V9fsIattr;
-
-typedef struct V9fsStatDotl {
-    uint64_t st_result_mask;
-    V9fsQID qid;
-    uint32_t st_mode;
-    uint32_t st_uid;
-    uint32_t st_gid;
-    uint64_t st_nlink;
-    uint64_t st_rdev;
-    uint64_t st_size;
-    uint64_t st_blksize;
-    uint64_t st_blocks;
-    uint64_t st_atime_sec;
-    uint64_t st_atime_nsec;
-    uint64_t st_mtime_sec;
-    uint64_t st_mtime_nsec;
-    uint64_t st_ctime_sec;
-    uint64_t st_ctime_nsec;
-    uint64_t st_btime_sec;
-    uint64_t st_btime_nsec;
-    uint64_t st_gen;
-    uint64_t st_data_version;
-} V9fsStatDotl;
-
-static inline void v9fs_string_init(V9fsString *str)
-{
-    str->data = NULL;
-    str->size = 0;
-}
-extern void v9fs_string_free(V9fsString *str);
-extern void v9fs_string_null(V9fsString *str);
-extern void v9fs_string_sprintf(V9fsString *str, const char *fmt, ...);
-extern void v9fs_string_copy(V9fsString *lhs, V9fsString *rhs);
-
-ssize_t v9fs_pack(struct iovec *in_sg, int in_num, size_t offset,
-                  const void *src, size_t size);
-ssize_t v9fs_unmarshal(struct iovec *out_sg, int out_num, size_t offset,
-                       int bswap, const char *fmt, ...);
-ssize_t v9fs_marshal(struct iovec *in_sg, int in_num, size_t offset,
-                     int bswap, const char *fmt, ...);
-#endif
diff --git a/gdb-xml/power-vsx.xml b/gdb-xml/power-vsx.xml
new file mode 100644
index 0000000..fd290e9
--- /dev/null
+++ b/gdb-xml/power-vsx.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2008-2015 Free Software Foundation, Inc.
+
+     Copying and distribution of this file, with or without modification,
+     are permitted in any medium without royalty provided the copyright
+     notice and this notice are preserved.  -->
+
+<!-- POWER7 VSX registers that do not overlap existing FP and VMX
+     registers.  -->
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.power.vsx">
+  <reg name="vs0h" bitsize="64" type="uint64"/>
+  <reg name="vs1h" bitsize="64" type="uint64"/>
+  <reg name="vs2h" bitsize="64" type="uint64"/>
+  <reg name="vs3h" bitsize="64" type="uint64"/>
+  <reg name="vs4h" bitsize="64" type="uint64"/>
+  <reg name="vs5h" bitsize="64" type="uint64"/>
+  <reg name="vs6h" bitsize="64" type="uint64"/>
+  <reg name="vs7h" bitsize="64" type="uint64"/>
+  <reg name="vs8h" bitsize="64" type="uint64"/>
+  <reg name="vs9h" bitsize="64" type="uint64"/>
+  <reg name="vs10h" bitsize="64" type="uint64"/>
+  <reg name="vs11h" bitsize="64" type="uint64"/>
+  <reg name="vs12h" bitsize="64" type="uint64"/>
+  <reg name="vs13h" bitsize="64" type="uint64"/>
+  <reg name="vs14h" bitsize="64" type="uint64"/>
+  <reg name="vs15h" bitsize="64" type="uint64"/>
+  <reg name="vs16h" bitsize="64" type="uint64"/>
+  <reg name="vs17h" bitsize="64" type="uint64"/>
+  <reg name="vs18h" bitsize="64" type="uint64"/>
+  <reg name="vs19h" bitsize="64" type="uint64"/>
+  <reg name="vs20h" bitsize="64" type="uint64"/>
+  <reg name="vs21h" bitsize="64" type="uint64"/>
+  <reg name="vs22h" bitsize="64" type="uint64"/>
+  <reg name="vs23h" bitsize="64" type="uint64"/>
+  <reg name="vs24h" bitsize="64" type="uint64"/>
+  <reg name="vs25h" bitsize="64" type="uint64"/>
+  <reg name="vs26h" bitsize="64" type="uint64"/>
+  <reg name="vs27h" bitsize="64" type="uint64"/>
+  <reg name="vs28h" bitsize="64" type="uint64"/>
+  <reg name="vs29h" bitsize="64" type="uint64"/>
+  <reg name="vs30h" bitsize="64" type="uint64"/>
+  <reg name="vs31h" bitsize="64" type="uint64"/>
+</feature>
diff --git a/gdb-xml/s390-cr.xml b/gdb-xml/s390-cr.xml
new file mode 100644
index 0000000..5246bea
--- /dev/null
+++ b/gdb-xml/s390-cr.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!-- Copyright 2015 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.cr">
+  <reg name="cr0" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr1" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr2" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr3" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr4" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr5" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr6" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr7" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr8" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr9" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr10" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr11" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr12" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr13" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr14" bitsize="64" type="uint64" group="control"/>
+  <reg name="cr15" bitsize="64" type="uint64" group="control"/>
+</feature>
diff --git a/gdb-xml/s390-virt.xml b/gdb-xml/s390-virt.xml
new file mode 100644
index 0000000..e2e9a7a
--- /dev/null
+++ b/gdb-xml/s390-virt.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0"?>
+<!-- Copyright 2015 IBM Corp.
+
+     This work is licensed under the terms of the GNU GPL, version 2 or
+     (at your option) any later version. See the COPYING file in the
+     top-level directory. -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.virt">
+  <reg name="ckc" bitsize="64" type="uint64" group="system"/>
+  <reg name="cputm" bitsize="64" type="uint64" group="system"/>
+  <reg name="last_break" bitsize="64" type="code_ptr" group="system"/>
+  <reg name="prefix" bitsize="64" type="data_ptr" group="system"/>
+  <reg name="pp" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_token" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_select" bitsize="64" type="uint64" group="system"/>
+  <reg name="pfault_compare" bitsize="64" type="uint64" group="system"/>
+</feature>
diff --git a/gdb-xml/s390-vx.xml b/gdb-xml/s390-vx.xml
new file mode 100644
index 0000000..8239c11
--- /dev/null
+++ b/gdb-xml/s390-vx.xml
@@ -0,0 +1,59 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2010-2014 Free Software Foundation, Inc.
+
+     Copying and distribution of this file, with or without modification,
+     are permitted in any medium without royalty provided the copyright
+     notice and this notice are preserved.  -->
+
+<!DOCTYPE feature SYSTEM "gdb-target.dtd">
+<feature name="org.gnu.gdb.s390.vx">
+  <vector id="v4f" type="ieee_single" count="4"/>
+  <vector id="v2d" type="ieee_double" count="2"/>
+  <vector id="v16i8" type="int8" count="16"/>
+  <vector id="v8i16" type="int16" count="8"/>
+  <vector id="v4i32" type="int32" count="4"/>
+  <vector id="v2i64" type="int64" count="2"/>
+  <union id="vec128">
+    <field name="v4_float" type="v4f"/>
+    <field name="v2_double" type="v2d"/>
+    <field name="v16_int8" type="v16i8"/>
+    <field name="v8_int16" type="v8i16"/>
+    <field name="v4_int32" type="v4i32"/>
+    <field name="v2_int64" type="v2i64"/>
+    <field name="uint128" type="uint128"/>
+  </union>
+
+  <reg name="v0l" bitsize="64" type="uint64"/>
+  <reg name="v1l" bitsize="64" type="uint64"/>
+  <reg name="v2l" bitsize="64" type="uint64"/>
+  <reg name="v3l" bitsize="64" type="uint64"/>
+  <reg name="v4l" bitsize="64" type="uint64"/>
+  <reg name="v5l" bitsize="64" type="uint64"/>
+  <reg name="v6l" bitsize="64" type="uint64"/>
+  <reg name="v7l" bitsize="64" type="uint64"/>
+  <reg name="v8l" bitsize="64" type="uint64"/>
+  <reg name="v9l" bitsize="64" type="uint64"/>
+  <reg name="v10l" bitsize="64" type="uint64"/>
+  <reg name="v11l" bitsize="64" type="uint64"/>
+  <reg name="v12l" bitsize="64" type="uint64"/>
+  <reg name="v13l" bitsize="64" type="uint64"/>
+  <reg name="v14l" bitsize="64" type="uint64"/>
+  <reg name="v15l" bitsize="64" type="uint64"/>
+
+  <reg name="v16" bitsize="128" type="vec128"/>
+  <reg name="v17" bitsize="128" type="vec128"/>
+  <reg name="v18" bitsize="128" type="vec128"/>
+  <reg name="v19" bitsize="128" type="vec128"/>
+  <reg name="v20" bitsize="128" type="vec128"/>
+  <reg name="v21" bitsize="128" type="vec128"/>
+  <reg name="v22" bitsize="128" type="vec128"/>
+  <reg name="v23" bitsize="128" type="vec128"/>
+  <reg name="v24" bitsize="128" type="vec128"/>
+  <reg name="v25" bitsize="128" type="vec128"/>
+  <reg name="v26" bitsize="128" type="vec128"/>
+  <reg name="v27" bitsize="128" type="vec128"/>
+  <reg name="v28" bitsize="128" type="vec128"/>
+  <reg name="v29" bitsize="128" type="vec128"/>
+  <reg name="v30" bitsize="128" type="vec128"/>
+  <reg name="v31" bitsize="128" type="vec128"/>
+</feature>
diff --git a/gdbstub.c b/gdbstub.c
index 016c89d..5da66f1 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -16,17 +16,11 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+#include "cpu.h"
 #ifdef CONFIG_USER_ONLY
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-
 #include "qemu.h"
 #else
 #include "monitor/monitor.h"
@@ -40,6 +34,8 @@
 #include "cpu.h"
 #include "qemu/sockets.h"
 #include "sysemu/kvm.h"
+#include "exec/semihost.h"
+#include "exec/exec-all.h"
 
 #ifdef CONFIG_USER_ONLY
 #define GDB_ATTACHED "0"
@@ -337,7 +333,7 @@
         if (ret < 0) {
             if (errno == ECONNRESET)
                 s->fd = -1;
-            if (errno != EINTR && errno != EAGAIN)
+            if (errno != EINTR)
                 return -1;
         } else if (ret == 0) {
             close(s->fd);
@@ -357,10 +353,20 @@
     GDB_SYS_DISABLED,
 } gdb_syscall_mode;
 
-/* If gdb is connected when the first semihosting syscall occurs then use
-   remote gdb syscalls.  Otherwise use native file IO.  */
+/* Decide if either remote gdb syscalls or native file IO should be used. */
 int use_gdb_syscalls(void)
 {
+    SemihostingTarget target = semihosting_get_target();
+    if (target == SEMIHOSTING_TARGET_NATIVE) {
+        /* -semihosting-config target=native */
+        return false;
+    } else if (target == SEMIHOSTING_TARGET_GDB) {
+        /* -semihosting-config target=gdb */
+        return true;
+    }
+
+    /* -semihosting-config target=auto */
+    /* On the first call check if gdb is connected and remember. */
     if (gdb_syscall_mode == GDB_SYS_UNKNOWN) {
         gdb_syscall_mode = (gdbserver_state ? GDB_SYS_ENABLED
                                             : GDB_SYS_DISABLED);
@@ -388,7 +394,7 @@
     while (len > 0) {
         ret = send(s->fd, buf, len, 0);
         if (ret < 0) {
-            if (errno != EINTR && errno != EAGAIN)
+            if (errno != EINTR)
                 return;
         } else {
             buf += ret;
@@ -750,12 +756,9 @@
 static void gdb_set_cpu_pc(GDBState *s, target_ulong pc)
 {
     CPUState *cpu = s->c_cpu;
-    CPUClass *cc = CPU_GET_CLASS(cpu);
 
     cpu_synchronize_state(cpu);
-    if (cc->set_pc) {
-        cc->set_pc(cpu, pc);
-    }
+    cpu_set_pc(cpu, pc);
 }
 
 static CPUState *find_cpu(uint32_t thread_id)
@@ -884,11 +887,9 @@
             goto unknown_command;
         }
     case 'k':
-#ifdef CONFIG_USER_ONLY
         /* Kill the target */
         fprintf(stderr, "\nQEMU: Terminated via GDBstub\n");
         exit(0);
-#endif
     case 'D':
         /* Detach packet */
         gdb_breakpoint_remove_all();
@@ -1238,7 +1239,6 @@
 static void gdb_vm_state_change(void *opaque, int running, RunState state)
 {
     GDBState *s = gdbserver_state;
-    CPUArchState *env = s->c_cpu->env_ptr;
     CPUState *cpu = s->c_cpu;
     char buf[256];
     const char *type;
@@ -1273,7 +1273,7 @@
             cpu->watchpoint_hit = NULL;
             goto send_packet;
         }
-        tb_flush(env);
+        tb_flush(cpu);
         ret = GDB_SIGNAL_TRAP;
         break;
     case RUN_STATE_PAUSED:
@@ -1301,6 +1301,7 @@
         ret = GDB_SIGNAL_UNKNOWN;
         break;
     }
+    gdb_set_stop_cpu(cpu);
     snprintf(buf, sizeof(buf), "T%02xthread:%02x;", ret, cpu_index(cpu));
 
 send_packet:
@@ -1316,9 +1317,8 @@
     %x  - target_ulong argument printed in hex.
     %lx - 64-bit argument printed in hex.
     %s  - string pointer (target_ulong) and length (int) pair.  */
-void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
+void gdb_do_syscallv(gdb_syscall_complete_cb cb, const char *fmt, va_list va)
 {
-    va_list va;
     char *p;
     char *p_end;
     target_ulong addr;
@@ -1332,7 +1332,6 @@
 #ifndef CONFIG_USER_ONLY
     vm_stop(RUN_STATE_DEBUG);
 #endif
-    va_start(va, fmt);
     p = s->syscall_buf;
     p_end = &s->syscall_buf[sizeof(s->syscall_buf)];
     *(p++) = 'F';
@@ -1366,7 +1365,6 @@
         }
     }
     *p = 0;
-    va_end(va);
 #ifdef CONFIG_USER_ONLY
     put_packet(s, s->syscall_buf);
     gdb_handlesig(s->c_cpu, 0);
@@ -1377,10 +1375,19 @@
        is still in the running state, which can cause packets to be dropped
        and state transition 'T' packets to be sent while the syscall is still
        being processed.  */
-    cpu_exit(s->c_cpu);
+    qemu_cpu_kick(s->c_cpu);
 #endif
 }
 
+void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...)
+{
+    va_list va;
+
+    va_start(va, fmt);
+    gdb_do_syscallv(cb, fmt, va);
+    va_end(va);
+}
+
 static void gdb_read_byte(GDBState *s, int ch)
 {
     int i, csum;
@@ -1487,22 +1494,8 @@
 
 #ifdef CONFIG_USER_ONLY
 int
-gdb_queuesig (void)
-{
-    GDBState *s;
-
-    s = gdbserver_state;
-
-    if (gdbserver_fd < 0 || s->fd < 0)
-        return 0;
-    else
-        return 1;
-}
-
-int
 gdb_handlesig(CPUState *cpu, int sig)
 {
-    CPUArchState *env = cpu->env_ptr;
     GDBState *s;
     char buf[256];
     int n;
@@ -1514,7 +1507,7 @@
 
     /* disable single step if it was enabled */
     cpu_single_step(cpu, 0);
-    tb_flush(env);
+    tb_flush(cpu);
 
     if (sig != 0) {
         snprintf(buf, sizeof(buf), "S%02x", target_signal_to_gdb(sig));
@@ -1537,9 +1530,13 @@
             for (i = 0; i < n; i++) {
                 gdb_read_byte(s, buf[i]);
             }
-        } else if (n == 0 || errno != EAGAIN) {
+        } else {
             /* XXX: Connection closed.  Should probably wait for another
                connection before continuing.  */
+            if (n == 0) {
+                close(s->fd);
+            }
+            s->fd = -1;
             return sig;
         }
     }
@@ -1594,8 +1591,6 @@
     gdb_has_xml = false;
 
     gdbserver_state = s;
-
-    fcntl(fd, F_SETFL, O_NONBLOCK);
 }
 
 static int gdbserver_open(int port)
@@ -1623,7 +1618,7 @@
         close(fd);
         return -1;
     }
-    ret = listen(fd, 0);
+    ret = listen(fd, 1);
     if (ret < 0) {
         perror("listen");
         close(fd);
@@ -1643,9 +1638,8 @@
 }
 
 /* Disable gdb stub for child processes.  */
-void gdbserver_fork(CPUArchState *env)
+void gdbserver_fork(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
     GDBState *s = gdbserver_state;
 
     if (gdbserver_fd < 0 || s->fd < 0) {
@@ -1729,6 +1723,7 @@
     char gdbstub_device_name[128];
     CharDriverState *chr = NULL;
     CharDriverState *mon_chr;
+    ChardevCommon common = { 0 };
 
     if (!device)
         return -1;
@@ -1748,7 +1743,7 @@
             sigaction(SIGINT, &act, NULL);
         }
 #endif
-        chr = qemu_chr_new("gdb", device, NULL);
+        chr = qemu_chr_new_noreplay("gdb", device, NULL);
         if (!chr)
             return -1;
 
@@ -1765,7 +1760,7 @@
         qemu_add_vm_change_state_handler(gdb_vm_state_change, NULL);
 
         /* Initialize a monitor terminal for gdb */
-        mon_chr = qemu_chr_alloc();
+        mon_chr = qemu_chr_alloc(&common, &error_abort);
         mon_chr->chr_write = gdb_monitor_write;
         monitor_init(mon_chr, 0);
     } else {
diff --git a/hmp-commands-info.hx b/hmp-commands-info.hx
new file mode 100644
index 0000000..74446c6
--- /dev/null
+++ b/hmp-commands-info.hx
@@ -0,0 +1,825 @@
+HXCOMM Use DEFHEADING() to define headings in both help text and texi
+HXCOMM Text between STEXI and ETEXI are copied to texi version and
+HXCOMM discarded from C version
+HXCOMM DEF(command, args, callback, arg_string, help) is used to construct
+HXCOMM monitor info commands
+HXCOMM HXCOMM can be used for comments, discarded from both texi and C
+
+STEXI
+@table @option
+@item info @var{subcommand}
+@findex info
+Show various information about the system state.
+@table @option
+ETEXI
+
+    {
+        .name       = "version",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the version of QEMU",
+        .mhandler.cmd = hmp_info_version,
+    },
+
+STEXI
+@item info version
+@findex version
+Show the version of QEMU.
+ETEXI
+
+    {
+        .name       = "network",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the network state",
+        .mhandler.cmd = hmp_info_network,
+    },
+
+STEXI
+@item info network
+@findex network
+Show the network state.
+ETEXI
+
+    {
+        .name       = "chardev",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the character devices",
+        .mhandler.cmd = hmp_info_chardev,
+    },
+
+STEXI
+@item info chardev
+@findex chardev
+Show the character devices.
+ETEXI
+
+    {
+        .name       = "block",
+        .args_type  = "nodes:-n,verbose:-v,device:B?",
+        .params     = "[-n] [-v] [device]",
+        .help       = "show info of one block device or all block devices "
+                      "(-n: show named nodes; -v: show details)",
+        .mhandler.cmd = hmp_info_block,
+    },
+
+STEXI
+@item info block
+@findex block
+Show info of one block device or all block devices.
+ETEXI
+
+    {
+        .name       = "blockstats",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show block device statistics",
+        .mhandler.cmd = hmp_info_blockstats,
+    },
+
+STEXI
+@item info blockstats
+@findex blockstats
+Show block device statistics.
+ETEXI
+
+    {
+        .name       = "block-jobs",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show progress of ongoing block device operations",
+        .mhandler.cmd = hmp_info_block_jobs,
+    },
+
+STEXI
+@item info block-jobs
+@findex block-jobs
+Show progress of ongoing block device operations.
+ETEXI
+
+    {
+        .name       = "registers",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the cpu registers",
+        .mhandler.cmd = hmp_info_registers,
+    },
+
+STEXI
+@item info registers
+@findex registers
+Show the cpu registers.
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "lapic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show local apic state",
+        .mhandler.cmd = hmp_info_local_apic,
+    },
+#endif
+
+STEXI
+@item info lapic
+@findex lapic
+Show local APIC state
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "ioapic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show io apic state",
+        .mhandler.cmd = hmp_info_io_apic,
+    },
+#endif
+
+STEXI
+@item info ioapic
+@findex ioapic
+Show io APIC state
+ETEXI
+
+    {
+        .name       = "cpus",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show infos for each CPU",
+        .mhandler.cmd = hmp_info_cpus,
+    },
+
+STEXI
+@item info cpus
+@findex cpus
+Show infos for each CPU.
+ETEXI
+
+    {
+        .name       = "history",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the command line history",
+        .mhandler.cmd = hmp_info_history,
+    },
+
+STEXI
+@item info history
+@findex history
+Show the command line history.
+ETEXI
+
+#if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
+    defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
+    {
+        .name       = "irq",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the interrupts statistics (if available)",
+#ifdef TARGET_SPARC
+        .mhandler.cmd = sun4m_hmp_info_irq,
+#elif defined(TARGET_LM32)
+        .mhandler.cmd = lm32_hmp_info_irq,
+#else
+        .mhandler.cmd = hmp_info_irq,
+#endif
+    },
+
+STEXI
+@item info irq
+@findex irq
+Show the interrupts statistics (if available).
+ETEXI
+
+    {
+        .name       = "pic",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show i8259 (PIC) state",
+#ifdef TARGET_SPARC
+        .mhandler.cmd = sun4m_hmp_info_pic,
+#elif defined(TARGET_LM32)
+        .mhandler.cmd = lm32_hmp_info_pic,
+#else
+        .mhandler.cmd = hmp_info_pic,
+#endif
+    },
+#endif
+
+STEXI
+@item info pic
+@findex pic
+Show i8259 (PIC) state.
+ETEXI
+
+    {
+        .name       = "pci",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show PCI info",
+        .mhandler.cmd = hmp_info_pci,
+    },
+
+STEXI
+@item info pci
+@findex pci
+Show PCI information.
+ETEXI
+
+#if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
+    defined(TARGET_PPC) || defined(TARGET_XTENSA)
+    {
+        .name       = "tlb",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show virtual to physical memory mappings",
+        .mhandler.cmd = hmp_info_tlb,
+    },
+#endif
+
+STEXI
+@item info tlb
+@findex tlb
+Show virtual to physical memory mappings.
+ETEXI
+
+#if defined(TARGET_I386)
+    {
+        .name       = "mem",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the active virtual memory mappings",
+        .mhandler.cmd = hmp_info_mem,
+    },
+#endif
+
+STEXI
+@item info mem
+@findex mem
+Show the active virtual memory mappings.
+ETEXI
+
+    {
+        .name       = "mtree",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory tree",
+        .mhandler.cmd = hmp_info_mtree,
+    },
+
+STEXI
+@item info mtree
+@findex mtree
+Show memory tree.
+ETEXI
+
+    {
+        .name       = "jit",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show dynamic compiler info",
+        .mhandler.cmd = hmp_info_jit,
+    },
+
+STEXI
+@item info jit
+@findex jit
+Show dynamic compiler info.
+ETEXI
+
+    {
+        .name       = "opcount",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show dynamic compiler opcode counters",
+        .mhandler.cmd = hmp_info_opcount,
+    },
+
+STEXI
+@item info opcount
+@findex opcount
+Show dynamic compiler opcode counters
+ETEXI
+
+    {
+        .name       = "kvm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show KVM information",
+        .mhandler.cmd = hmp_info_kvm,
+    },
+
+STEXI
+@item info kvm
+@findex kvm
+Show KVM information.
+ETEXI
+
+    {
+        .name       = "numa",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show NUMA information",
+        .mhandler.cmd = hmp_info_numa,
+    },
+
+STEXI
+@item info numa
+@findex numa
+Show NUMA information.
+ETEXI
+
+    {
+        .name       = "usb",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show guest USB devices",
+        .mhandler.cmd = hmp_info_usb,
+    },
+
+STEXI
+@item info usb
+@findex usb
+Show guest USB devices.
+ETEXI
+
+    {
+        .name       = "usbhost",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show host USB devices",
+        .mhandler.cmd = hmp_info_usbhost,
+    },
+
+STEXI
+@item info usbhost
+@findex usbhost
+Show host USB devices.
+ETEXI
+
+    {
+        .name       = "profile",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show profiling information",
+        .mhandler.cmd = hmp_info_profile,
+    },
+
+STEXI
+@item info profile
+@findex profile
+Show profiling information.
+ETEXI
+
+    {
+        .name       = "capture",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show capture information",
+        .mhandler.cmd = hmp_info_capture,
+    },
+
+STEXI
+@item info capture
+@findex capture
+Show capture information.
+ETEXI
+
+    {
+        .name       = "snapshots",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the currently saved VM snapshots",
+        .mhandler.cmd = hmp_info_snapshots,
+    },
+
+STEXI
+@item info snapshots
+@findex snapshots
+Show the currently saved VM snapshots.
+ETEXI
+
+    {
+        .name       = "status",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM status (running|paused)",
+        .mhandler.cmd = hmp_info_status,
+    },
+
+STEXI
+@item info status
+@findex status
+Show the current VM status (running|paused).
+ETEXI
+
+    {
+        .name       = "mice",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show which guest mouse is receiving events",
+        .mhandler.cmd = hmp_info_mice,
+    },
+
+STEXI
+@item info mice
+@findex mice
+Show which guest mouse is receiving events.
+ETEXI
+
+    {
+        .name       = "vnc",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the vnc server status",
+        .mhandler.cmd = hmp_info_vnc,
+    },
+
+STEXI
+@item info vnc
+@findex vnc
+Show the vnc server status.
+ETEXI
+
+#if defined(CONFIG_SPICE)
+    {
+        .name       = "spice",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the spice server status",
+        .mhandler.cmd = hmp_info_spice,
+    },
+#endif
+
+STEXI
+@item info spice
+@findex spice
+Show the spice server status.
+ETEXI
+
+    {
+        .name       = "name",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM name",
+        .mhandler.cmd = hmp_info_name,
+    },
+
+STEXI
+@item info name
+@findex name
+Show the current VM name.
+ETEXI
+
+    {
+        .name       = "uuid",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the current VM UUID",
+        .mhandler.cmd = hmp_info_uuid,
+    },
+
+STEXI
+@item info uuid
+@findex uuid
+Show the current VM UUID.
+ETEXI
+
+    {
+        .name       = "cpustats",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show CPU statistics",
+        .mhandler.cmd = hmp_info_cpustats,
+    },
+
+STEXI
+@item info cpustats
+@findex cpustats
+Show CPU statistics.
+ETEXI
+
+#if defined(CONFIG_SLIRP)
+    {
+        .name       = "usernet",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show user network stack connection states",
+        .mhandler.cmd = hmp_info_usernet,
+    },
+#endif
+
+STEXI
+@item info usernet
+@findex usernet
+Show user network stack connection states.
+ETEXI
+
+    {
+        .name       = "migrate",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show migration status",
+        .mhandler.cmd = hmp_info_migrate,
+    },
+
+STEXI
+@item info migrate
+@findex migrate
+Show migration status.
+ETEXI
+
+    {
+        .name       = "migrate_capabilities",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration capabilities",
+        .mhandler.cmd = hmp_info_migrate_capabilities,
+    },
+
+STEXI
+@item info migrate_capabilities
+@findex migrate_capabilities
+Show current migration capabilities.
+ETEXI
+
+    {
+        .name       = "migrate_parameters",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration parameters",
+        .mhandler.cmd = hmp_info_migrate_parameters,
+    },
+
+STEXI
+@item info migrate_parameters
+@findex migrate_parameters
+Show current migration parameters.
+ETEXI
+
+    {
+        .name       = "migrate_cache_size",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show current migration xbzrle cache size",
+        .mhandler.cmd = hmp_info_migrate_cache_size,
+    },
+
+STEXI
+@item info migrate_cache_size
+@findex migrate_cache_size
+Show current migration xbzrle cache size.
+ETEXI
+
+    {
+        .name       = "balloon",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show balloon information",
+        .mhandler.cmd = hmp_info_balloon,
+    },
+
+STEXI
+@item info balloon
+@findex balloon
+Show balloon information.
+ETEXI
+
+    {
+        .name       = "qtree",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show device tree",
+        .mhandler.cmd = hmp_info_qtree,
+    },
+
+STEXI
+@item info qtree
+@findex qtree
+Show device tree.
+ETEXI
+
+    {
+        .name       = "qdm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show qdev device model list",
+        .mhandler.cmd = hmp_info_qdm,
+    },
+
+STEXI
+@item info qdm
+@findex qdm
+Show qdev device model list.
+ETEXI
+
+    {
+        .name       = "qom-tree",
+        .args_type  = "path:s?",
+        .params     = "[path]",
+        .help       = "show QOM composition tree",
+        .mhandler.cmd = hmp_info_qom_tree,
+    },
+
+STEXI
+@item info qom-tree
+@findex qom-tree
+Show QOM composition tree.
+ETEXI
+
+    {
+        .name       = "roms",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show roms",
+        .mhandler.cmd = hmp_info_roms,
+    },
+
+STEXI
+@item info roms
+@findex roms
+Show roms.
+ETEXI
+
+    {
+        .name       = "trace-events",
+        .args_type  = "name:s?,vcpu:i?",
+        .params     = "[name] [vcpu]",
+        .help       = "show available trace-events & their state "
+                      "(name: event name pattern; vcpu: vCPU to query, default is any)",
+        .mhandler.cmd = hmp_info_trace_events,
+        .command_completion = info_trace_events_completion,
+    },
+
+STEXI
+@item info trace-events
+@findex trace-events
+Show available trace-events & their state.
+ETEXI
+
+    {
+        .name       = "tpm",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show the TPM device",
+        .mhandler.cmd = hmp_info_tpm,
+    },
+
+STEXI
+@item info tpm
+@findex tpm
+Show the TPM device.
+ETEXI
+
+    {
+        .name       = "memdev",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory backends",
+        .mhandler.cmd = hmp_info_memdev,
+    },
+
+STEXI
+@item info memdev
+@findex memdev
+Show memory backends
+ETEXI
+
+    {
+        .name       = "memory-devices",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show memory devices",
+        .mhandler.cmd = hmp_info_memory_devices,
+    },
+
+STEXI
+@item info memory-devices
+@findex memory-devices
+Show memory devices.
+ETEXI
+
+    {
+        .name       = "iothreads",
+        .args_type  = "",
+        .params     = "",
+        .help       = "show iothreads",
+        .mhandler.cmd = hmp_info_iothreads,
+    },
+
+STEXI
+@item info iothreads
+@findex iothreads
+Show iothread's identifiers.
+ETEXI
+
+    {
+        .name       = "rocker",
+        .args_type  = "name:s",
+        .params     = "name",
+        .help       = "Show rocker switch",
+        .mhandler.cmd = hmp_rocker,
+    },
+
+STEXI
+@item info rocker @var{name}
+@findex rocker
+Show rocker switch.
+ETEXI
+
+    {
+        .name       = "rocker-ports",
+        .args_type  = "name:s",
+        .params     = "name",
+        .help       = "Show rocker ports",
+        .mhandler.cmd = hmp_rocker_ports,
+    },
+
+STEXI
+@item info rocker_ports @var{name}-ports
+@findex ocker-ports
+Show rocker ports.
+ETEXI
+
+    {
+        .name       = "rocker-of-dpa-flows",
+        .args_type  = "name:s,tbl_id:i?",
+        .params     = "name [tbl_id]",
+        .help       = "Show rocker OF-DPA flow tables",
+        .mhandler.cmd = hmp_rocker_of_dpa_flows,
+    },
+
+STEXI
+@item info rocker_of_dpa_flows @var{name} [@var{tbl_id}]
+@findex rocker-of-dpa-flows
+Show rocker OF-DPA flow tables.
+ETEXI
+
+    {
+        .name       = "rocker-of-dpa-groups",
+        .args_type  = "name:s,type:i?",
+        .params     = "name [type]",
+        .help       = "Show rocker OF-DPA groups",
+        .mhandler.cmd = hmp_rocker_of_dpa_groups,
+    },
+
+STEXI
+@item info rocker-of-dpa-groups @var{name} [@var{type}]
+@findex rocker-of-dpa-groups
+Show rocker OF-DPA groups.
+ETEXI
+
+#if defined(TARGET_S390X)
+    {
+        .name       = "skeys",
+        .args_type  = "addr:l",
+        .params     = "address",
+        .help       = "Display the value of a storage key",
+        .mhandler.cmd = hmp_info_skeys,
+    },
+#endif
+
+STEXI
+@item info skeys @var{address}
+@findex skeys
+Display the value of a storage key (s390 only)
+ETEXI
+
+    {
+        .name       = "dump",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Display the latest dump status",
+        .mhandler.cmd = hmp_info_dump,
+    },
+
+STEXI
+@item info dump
+@findex dump
+Display the latest dump status.
+ETEXI
+
+    {
+        .name       = "hotpluggable-cpus",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Show information about hotpluggable CPUs",
+        .mhandler.cmd = hmp_hotpluggable_cpus,
+    },
+
+STEXI
+@item info hotpluggable-cpus
+@findex hotpluggable-cpus
+Show information about hotpluggable CPUs
+ETEXI
+
+STEXI
+@end table
+ETEXI
+
+STEXI
+@end table
+ETEXI
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 1990e66..848efee 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -28,7 +28,7 @@
         .args_type  = "device:B",
         .params     = "device|all",
         .help       = "commit changes to the disk images (if -snapshot is used) or backing files",
-        .mhandler.cmd = do_commit,
+        .mhandler.cmd = hmp_commit,
     },
 
 STEXI
@@ -47,7 +47,6 @@
         .args_type  = "",
         .params     = "",
         .help       = "quit the emulator",
-        .user_print = monitor_user_noop,
         .mhandler.cmd = hmp_quit,
     },
 
@@ -179,8 +178,7 @@
         .args_type  = "id:B",
         .params     = "device",
         .help       = "remove host block device",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_drive_del,
+        .mhandler.cmd = hmp_drive_del,
     },
 
 STEXI
@@ -196,8 +194,8 @@
 
     {
         .name       = "change",
-        .args_type  = "device:B,target:F,arg:s?",
-        .params     = "device filename [format]",
+        .args_type  = "device:B,target:F,arg:s?,read-only-mode:s?",
+        .params     = "device filename [format [read-only-mode]]",
         .help       = "change a removable medium, optional format",
         .mhandler.cmd = hmp_change,
     },
@@ -205,11 +203,10 @@
 STEXI
 @item change @var{device} @var{setting}
 @findex change
-
 Change the configuration of a device.
 
 @table @option
-@item change @var{diskdevice} @var{filename} [@var{format}]
+@item change @var{diskdevice} @var{filename} [@var{format} [@var{read-only-mode}]]
 Change the medium for a removable disk device to point to @var{filename}. eg
 
 @example
@@ -218,6 +215,20 @@
 
 @var{format} is optional.
 
+@var{read-only-mode} may be used to change the read-only status of the device.
+It accepts the following values:
+
+@table @var
+@item retain
+Retains the current status; this is the default.
+
+@item read-only
+Makes the device read-only.
+
+@item read-write
+Makes the device writable.
+@end table
+
 @item change vnc @var{display},@var{options}
 Change the configuration of the VNC server. The valid syntax for @var{display}
 and @var{options} are described at @ref{sec_invocation}. eg
@@ -245,7 +256,7 @@
         .args_type  = "filename:F",
         .params     = "filename",
         .help       = "save screen into PPM image 'filename'",
-        .mhandler.cmd = hmp_screen_dump,
+        .mhandler.cmd = hmp_screendump,
     },
 
 STEXI
@@ -259,7 +270,7 @@
         .args_type  = "filename:F",
         .params     = "filename",
         .help       = "output logs to 'filename'",
-        .mhandler.cmd = do_logfile,
+        .mhandler.cmd = hmp_logfile,
     },
 
 STEXI
@@ -270,10 +281,12 @@
 
     {
         .name       = "trace-event",
-        .args_type  = "name:s,option:b",
-        .params     = "name on|off",
-        .help       = "changes status of a specific trace event",
-        .mhandler.cmd = do_trace_event_set_state,
+        .args_type  = "name:s,option:b,vcpu:i?",
+        .params     = "name on|off [vcpu]",
+        .help       = "changes status of a specific trace event "
+                      "(vcpu: vCPU to set, default is all)",
+        .mhandler.cmd = hmp_trace_event,
+        .command_completion = trace_event_completion,
     },
 
 STEXI
@@ -288,7 +301,7 @@
         .args_type  = "op:s?,arg:F?",
         .params     = "on|off|flush|set [arg]",
         .help       = "open, close, or flush trace file, or set a new file name",
-        .mhandler.cmd = do_trace_file,
+        .mhandler.cmd = hmp_trace_file,
     },
 
 STEXI
@@ -303,7 +316,7 @@
         .args_type  = "items:s",
         .params     = "item1[,...]",
         .help       = "activate logging of the specified items",
-        .mhandler.cmd = do_log,
+        .mhandler.cmd = hmp_log,
     },
 
 STEXI
@@ -317,7 +330,7 @@
         .args_type  = "name:s?",
         .params     = "[tag|id]",
         .help       = "save a VM snapshot. If no tag or id are provided, a new snapshot is created",
-        .mhandler.cmd = do_savevm,
+        .mhandler.cmd = hmp_savevm,
     },
 
 STEXI
@@ -334,7 +347,7 @@
         .args_type  = "name:s",
         .params     = "tag|id",
         .help       = "restore a VM snapshot from its tag or id",
-        .mhandler.cmd = do_loadvm,
+        .mhandler.cmd = hmp_loadvm,
         .command_completion = loadvm_completion,
     },
 
@@ -350,7 +363,7 @@
         .args_type  = "name:s",
         .params     = "tag|id",
         .help       = "delete a VM snapshot from its tag or id",
-        .mhandler.cmd = do_delvm,
+        .mhandler.cmd = hmp_delvm,
         .command_completion = delvm_completion,
     },
 
@@ -365,7 +378,7 @@
         .args_type  = "option:s?",
         .params     = "[on|off]",
         .help       = "run emulation in singlestep mode or switch to normal mode",
-        .mhandler.cmd = do_singlestep,
+        .mhandler.cmd = hmp_singlestep,
     },
 
 STEXI
@@ -422,7 +435,7 @@
         .args_type  = "device:s?",
         .params     = "[device]",
         .help       = "start gdbserver on given device (default 'tcp::1234'), stop with 'none'",
-        .mhandler.cmd = do_gdbserver,
+        .mhandler.cmd = hmp_gdbserver,
     },
 
 STEXI
@@ -436,7 +449,7 @@
         .args_type  = "fmt:/,addr:l",
         .params     = "/fmt addr",
         .help       = "virtual memory dump starting at 'addr'",
-        .mhandler.cmd = do_memory_dump,
+        .mhandler.cmd = hmp_memory_dump,
     },
 
 STEXI
@@ -450,7 +463,7 @@
         .args_type  = "fmt:/,addr:l",
         .params     = "/fmt addr",
         .help       = "physical memory dump starting at 'addr'",
-        .mhandler.cmd = do_physical_memory_dump,
+        .mhandler.cmd = hmp_physical_memory_dump,
     },
 
 STEXI
@@ -523,7 +536,6 @@
 STEXI
 @item p or print/@var{fmt} @var{expr}
 @findex print
-
 Print expression value. Only the @var{format} part of @var{fmt} is
 used.
 ETEXI
@@ -533,10 +545,12 @@
         .args_type  = "fmt:/,addr:i,index:i.",
         .params     = "/fmt addr",
         .help       = "I/O port read",
-        .mhandler.cmd = do_ioport_read,
+        .mhandler.cmd = hmp_ioport_read,
     },
 
 STEXI
+@item i/@var{fmt} @var{addr} [.@var{index}]
+@findex i
 Read I/O port.
 ETEXI
 
@@ -545,10 +559,12 @@
         .args_type  = "fmt:/,addr:i,val:i",
         .params     = "/fmt addr value",
         .help       = "I/O port write",
-        .mhandler.cmd = do_ioport_write,
+        .mhandler.cmd = hmp_ioport_write,
     },
 
 STEXI
+@item o/@var{fmt} @var{addr} @var{val}
+@findex o
 Write to I/O port.
 ETEXI
 
@@ -557,14 +573,13 @@
         .args_type  = "keys:s,hold-time:i?",
         .params     = "keys [hold_ms]",
         .help       = "send keys to the VM (e.g. 'sendkey ctrl-alt-f1', default hold time=100 ms)",
-        .mhandler.cmd = hmp_send_key,
+        .mhandler.cmd = hmp_sendkey,
         .command_completion = sendkey_completion,
     },
 
 STEXI
 @item sendkey @var{keys}
 @findex sendkey
-
 Send @var{keys} to the guest. @var{keys} could be the name of the
 key or the raw value in hexadecimal format. Use @code{-} to press
 several keys simultaneously. Example:
@@ -587,7 +602,6 @@
 STEXI
 @item system_reset
 @findex system_reset
-
 Reset the system.
 ETEXI
 
@@ -602,7 +616,6 @@
 STEXI
 @item system_powerdown
 @findex system_powerdown
-
 Power down the system (if supported).
 ETEXI
 
@@ -611,13 +624,12 @@
         .args_type  = "start:i,size:i",
         .params     = "addr size",
         .help       = "compute the checksum of a memory region",
-        .mhandler.cmd = do_sum,
+        .mhandler.cmd = hmp_sum,
     },
 
 STEXI
 @item sum @var{addr} @var{size}
 @findex sum
-
 Compute the checksum of a memory region.
 ETEXI
 
@@ -626,13 +638,12 @@
         .args_type  = "devname:s",
         .params     = "device",
         .help       = "add USB device (e.g. 'host:bus.addr' or 'host:vendor_id:product_id')",
-        .mhandler.cmd = do_usb_add,
+        .mhandler.cmd = hmp_usb_add,
     },
 
 STEXI
 @item usb_add @var{devname}
 @findex usb_add
-
 Add the USB device @var{devname}.  For details of available devices see
 @ref{usb_devices}
 ETEXI
@@ -642,13 +653,12 @@
         .args_type  = "devname:s",
         .params     = "device",
         .help       = "remove USB device 'bus.addr'",
-        .mhandler.cmd = do_usb_del,
+        .mhandler.cmd = hmp_usb_del,
     },
 
 STEXI
 @item usb_del @var{devname}
 @findex usb_del
-
 Remove the USB device @var{devname} from the QEMU virtual USB
 hub. @var{devname} has the syntax @code{bus.addr}. Use the monitor
 command @code{info usb} to see the devices you can remove.
@@ -659,15 +669,13 @@
         .args_type  = "device:O",
         .params     = "driver[,prop=value][,...]",
         .help       = "add device, like -device on the command line",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_device_add,
+        .mhandler.cmd = hmp_device_add,
         .command_completion = device_add_completion,
     },
 
 STEXI
 @item device_add @var{config}
 @findex device_add
-
 Add device.
 ETEXI
 
@@ -683,8 +691,8 @@
 STEXI
 @item device_del @var{id}
 @findex device_del
-
-Remove device @var{id}.
+Remove device @var{id}. @var{id} may be a short ID
+or a QOM object path.
 ETEXI
 
     {
@@ -706,7 +714,7 @@
         .args_type  = "dx_str:s,dy_str:s,dz_str:s?",
         .params     = "dx dy [dz]",
         .help       = "send mouse move events",
-        .mhandler.cmd = do_mouse_move,
+        .mhandler.cmd = hmp_mouse_move,
     },
 
 STEXI
@@ -721,7 +729,7 @@
         .args_type  = "button_state:i",
         .params     = "state",
         .help       = "change mouse button state (1=L, 2=M, 4=R)",
-        .mhandler.cmd = do_mouse_button,
+        .mhandler.cmd = hmp_mouse_button,
     },
 
 STEXI
@@ -735,7 +743,7 @@
         .args_type  = "index:i",
         .params     = "index",
         .help       = "set which mouse device receives events",
-        .mhandler.cmd = do_mouse_set,
+        .mhandler.cmd = hmp_mouse_set,
     },
 
 STEXI
@@ -753,7 +761,7 @@
         .args_type  = "path:F,freq:i?,bits:i?,nchannels:i?",
         .params     = "path [frequency [bits [channels]]]",
         .help       = "capture audio to a wave file (default frequency=44100 bits=16 channels=2)",
-        .mhandler.cmd = do_wav_capture,
+        .mhandler.cmd = hmp_wavcapture,
     },
 STEXI
 @item wavcapture @var{filename} [@var{frequency} [@var{bits} [@var{channels}]]]
@@ -774,7 +782,7 @@
         .args_type  = "n:i",
         .params     = "capture index",
         .help       = "stop capture",
-        .mhandler.cmd = do_stop_capture,
+        .mhandler.cmd = hmp_stopcapture,
     },
 STEXI
 @item stopcapture @var{index}
@@ -818,13 +826,12 @@
         .args_type  = "bootdevice:s",
         .params     = "bootdevice",
         .help       = "define new values for the boot device list",
-        .mhandler.cmd = do_boot_set,
+        .mhandler.cmd = hmp_boot_set,
     },
 
 STEXI
 @item boot_set @var{bootdevicelist}
 @findex boot_set
-
 Define new values for the boot device list. Those values will override
 the values specified on the command line through the @code{-boot} option.
 
@@ -837,7 +844,7 @@
         .args_type  = "",
         .params     = "",
         .help       = "inject an NMI",
-        .mhandler.cmd = hmp_inject_nmi,
+        .mhandler.cmd = hmp_nmi,
     },
 STEXI
 @item nmi @var{cpu}
@@ -922,6 +929,22 @@
 ETEXI
 
     {
+        .name       = "migrate_incoming",
+        .args_type  = "uri:s",
+        .params     = "uri",
+        .help       = "Continue an incoming migration from an -incoming defer",
+        .mhandler.cmd = hmp_migrate_incoming,
+    },
+
+STEXI
+@item migrate_incoming @var{uri}
+@findex migrate_incoming
+Continue an incoming migration using the @var{uri} (that has the same syntax
+as the -incoming option).
+
+ETEXI
+
+    {
         .name       = "migrate_set_cache_size",
         .args_type  = "value:o",
         .params     = "value",
@@ -985,29 +1008,60 @@
 ETEXI
 
     {
+        .name       = "migrate_set_parameter",
+        .args_type  = "parameter:s,value:s",
+        .params     = "parameter value",
+        .help       = "Set the parameter for migration",
+        .mhandler.cmd = hmp_migrate_set_parameter,
+        .command_completion = migrate_set_parameter_completion,
+    },
+
+STEXI
+@item migrate_set_parameter @var{parameter} @var{value}
+@findex migrate_set_parameter
+Set the parameter @var{parameter} for migration.
+ETEXI
+
+    {
+        .name       = "migrate_start_postcopy",
+        .args_type  = "",
+        .params     = "",
+        .help       = "Followup to a migration command to switch the migration"
+                      " to postcopy mode. The postcopy-ram capability must "
+                      "be set before the original migration command.",
+        .mhandler.cmd = hmp_migrate_start_postcopy,
+    },
+
+STEXI
+@item migrate_start_postcopy
+@findex migrate_start_postcopy
+Switch in-progress migration to postcopy mode. Ignored after the end of
+migration (or once already in postcopy).
+ETEXI
+
+    {
         .name       = "client_migrate_info",
         .args_type  = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
         .params     = "protocol hostname port tls-port cert-subject",
-        .help       = "send migration info to spice/vnc client",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_async = client_migrate_info,
-        .flags      = MONITOR_CMD_ASYNC,
+        .help       = "set migration information for remote display",
+        .mhandler.cmd = hmp_client_migrate_info,
     },
 
 STEXI
 @item client_migrate_info @var{protocol} @var{hostname} @var{port} @var{tls-port} @var{cert-subject}
 @findex client_migrate_info
-Set the spice/vnc connection info for the migration target.  The spice/vnc
-server will ask the spice/vnc client to automatically reconnect using the
-new parameters (if specified) once the vm migration finished successfully.
+Set migration information for remote display.  This makes the server
+ask the client to automatically reconnect using the new parameters
+once migration finished successfully.  Only implemented for SPICE.
 ETEXI
 
     {
         .name       = "dump-guest-memory",
-        .args_type  = "paging:-p,zlib:-z,lzo:-l,snappy:-s,filename:F,begin:i?,length:i?",
-        .params     = "[-p] [-z|-l|-s] filename [begin length]",
+        .args_type  = "paging:-p,detach:-d,zlib:-z,lzo:-l,snappy:-s,filename:F,begin:i?,length:i?",
+        .params     = "[-p] [-d] [-z|-l|-s] filename [begin length]",
         .help       = "dump guest memory into file 'filename'.\n\t\t\t"
                       "-p: do paging to get guest's memory mapping.\n\t\t\t"
+                      "-d: return immediately (do not wait for completion).\n\t\t\t"
                       "-z: dump in kdump-compressed format, with zlib compression.\n\t\t\t"
                       "-l: dump in kdump-compressed format, with lzo compression.\n\t\t\t"
                       "-s: dump in kdump-compressed format, with snappy compression.\n\t\t\t"
@@ -1034,6 +1088,22 @@
             together with begin.
 ETEXI
 
+#if defined(TARGET_S390X)
+    {
+        .name       = "dump-skeys",
+        .args_type  = "filename:F",
+        .params     = "",
+        .help       = "Save guest storage keys into file 'filename'.\n",
+        .mhandler.cmd = hmp_dump_skeys,
+    },
+#endif
+
+STEXI
+@item dump-skeys @var{filename}
+@findex dump-skeys
+Save guest storage keys to a file.
+ETEXI
+
     {
         .name       = "snapshot_blkdev",
         .args_type  = "reuse:-n,device:B,snapshot-file:s?,format:s?",
@@ -1132,15 +1202,15 @@
 
     {
         .name       = "drive_add",
-        .args_type  = "pci_addr:s,opts:s",
-        .params     = "[[<domain>:]<bus>:]<slot>\n"
+        .args_type  = "node:-n,pci_addr:s,opts:s",
+        .params     = "[-n] [[<domain>:]<bus>:]<slot>\n"
                       "[file=file][,if=type][,bus=n]\n"
                       "[,unit=m][,media=d][,index=i]\n"
                       "[,cyls=c,heads=h,secs=s[,trans=t]]\n"
                       "[,snapshot=on|off][,cache=on|off]\n"
                       "[,readonly=on|off][,copy-on-read=on|off]",
         .help       = "add drive to PCI storage controller",
-        .mhandler.cmd = drive_hot_add,
+        .mhandler.cmd = hmp_drive_add,
     },
 
 STEXI
@@ -1149,38 +1219,6 @@
 Add drive to PCI storage controller.
 ETEXI
 
-#if defined(CONFIG_PCI_HOTPLUG_OLD)
-    {
-        .name       = "pci_add",
-        .args_type  = "pci_addr:s,type:s,opts:s?",
-        .params     = "auto|[[<domain>:]<bus>:]<slot> nic|storage [[vlan=n][,macaddr=addr][,model=type]] [file=file][,if=type][,bus=nr]...",
-        .help       = "hot-add PCI device",
-        .mhandler.cmd = pci_device_hot_add,
-    },
-#endif
-
-STEXI
-@item pci_add
-@findex pci_add
-Hot-add PCI device.
-ETEXI
-
-#if defined(CONFIG_PCI_HOTPLUG_OLD)
-    {
-        .name       = "pci_del",
-        .args_type  = "pci_addr:s",
-        .params     = "[[<domain>:]<bus>:]<slot>",
-        .help       = "hot remove PCI device",
-        .mhandler.cmd = do_pci_device_hot_remove,
-    },
-#endif
-
-STEXI
-@item pci_del
-@findex pci_del
-Hot remove PCI device.
-ETEXI
-
     {
         .name       = "pcie_aer_inject_error",
         .args_type  = "advisory_non_fatal:-a,correctable:-c,"
@@ -1196,8 +1234,7 @@
                       "<error_status> = error string or 32bit\n\t\t\t"
                       "<tlb header> = 32bit x 4\n\t\t\t"
                       "<tlb header prefix> = 32bit x 4",
-        .user_print  = pcie_aer_inject_error_print,
-        .mhandler.cmd_new = do_pcie_aer_inject_error,
+        .mhandler.cmd = hmp_pcie_aer_inject_error,
     },
 
 STEXI
@@ -1211,7 +1248,7 @@
         .args_type  = "device:s,opts:s?",
         .params     = "tap|user|socket|vde|netmap|bridge|vhost-user|dump [options]",
         .help       = "add host VLAN client",
-        .mhandler.cmd = net_host_device_add,
+        .mhandler.cmd = hmp_host_net_add,
         .command_completion = host_net_add_completion,
     },
 
@@ -1226,7 +1263,7 @@
         .args_type  = "vlan_id:i,device:s",
         .params     = "vlan_id name",
         .help       = "remove host VLAN client",
-        .mhandler.cmd = net_host_device_remove,
+        .mhandler.cmd = hmp_host_net_remove,
         .command_completion = host_net_remove_completion,
     },
 
@@ -1302,7 +1339,7 @@
         .args_type  = "arg1:s,arg2:s?,arg3:s?",
         .params     = "[vlan_id name] [tcp|udp]:[hostaddr]:hostport-[guestaddr]:guestport",
         .help       = "redirect TCP or UDP connections from host to guest (requires -net user)",
-        .mhandler.cmd = net_slirp_hostfwd_add,
+        .mhandler.cmd = hmp_hostfwd_add,
     },
 #endif
 STEXI
@@ -1317,7 +1354,7 @@
         .args_type  = "arg1:s,arg2:s?,arg3:s?",
         .params     = "[vlan_id name] [tcp|udp]:[hostaddr]:hostport",
         .help       = "remove host-to-guest TCP or UDP redirection",
-        .mhandler.cmd = net_slirp_hostfwd_remove,
+        .mhandler.cmd = hmp_hostfwd_remove,
     },
 
 #endif
@@ -1361,7 +1398,7 @@
         .args_type  = "action:s",
         .params     = "[reset|shutdown|poweroff|pause|debug|none]",
         .help       = "change watchdog action",
-        .mhandler.cmd = do_watchdog_action,
+        .mhandler.cmd = hmp_watchdog_action,
         .command_completion = watchdog_action_completion,
     },
 
@@ -1376,7 +1413,7 @@
         .args_type  = "aclname:s",
         .params     = "aclname",
         .help       = "list rules in the access control list",
-        .mhandler.cmd = do_acl_show,
+        .mhandler.cmd = hmp_acl_show,
     },
 
 STEXI
@@ -1393,7 +1430,7 @@
         .args_type  = "aclname:s,policy:s",
         .params     = "aclname allow|deny",
         .help       = "set default access control list policy",
-        .mhandler.cmd = do_acl_policy,
+        .mhandler.cmd = hmp_acl_policy,
     },
 
 STEXI
@@ -1409,7 +1446,7 @@
         .args_type  = "aclname:s,match:s,policy:s,index:i?",
         .params     = "aclname match allow|deny [index]",
         .help       = "add a match rule to the access control list",
-        .mhandler.cmd = do_acl_add,
+        .mhandler.cmd = hmp_acl_add,
     },
 
 STEXI
@@ -1428,7 +1465,7 @@
         .args_type  = "aclname:s,match:s",
         .params     = "aclname match",
         .help       = "remove a match rule from the access control list",
-        .mhandler.cmd = do_acl_remove,
+        .mhandler.cmd = hmp_acl_remove,
     },
 
 STEXI
@@ -1442,7 +1479,7 @@
         .args_type  = "aclname:s",
         .params     = "aclname",
         .help       = "reset the access control list",
-        .mhandler.cmd = do_acl_reset,
+        .mhandler.cmd = hmp_acl_reset,
     },
 
 STEXI
@@ -1504,7 +1541,7 @@
         .args_type  = "broadcast:-b,cpu_index:i,bank:i,status:l,mcg_status:l,addr:l,misc:l",
         .params     = "[-b] cpu bank status mcgstatus addr misc",
         .help       = "inject a MCE on the given CPU [and broadcast to other CPUs with -b option]",
-        .mhandler.cmd = do_inject_mce,
+        .mhandler.cmd = hmp_mce,
     },
 
 #endif
@@ -1555,9 +1592,9 @@
     },
 
 STEXI
-@item block_set_io_throttle @var{device} @var{bps} @var{bps_rd} @var{bps_wr} @var{iops} @var{iops_rd} @var{iops_wr}
-@findex block_set_io_throttle
-Change I/O throttle limits for a block drive to @var{bps} @var{bps_rd} @var{bps_wr} @var{iops} @var{iops_rd} @var{iops_wr}
+@item block_passwd @var{device} @var{password}
+@findex block_passwd
+Set the encrypted device @var{device} password to @var{password}
 ETEXI
 
     {
@@ -1569,9 +1606,9 @@
     },
 
 STEXI
-@item block_passwd @var{device} @var{password}
-@findex block_passwd
-Set the encrypted device @var{device} password to @var{password}
+@item block_set_io_throttle @var{device} @var{bps} @var{bps_rd} @var{bps_wr} @var{iops} @var{iops_rd} @var{iops_wr}
+@findex block_set_io_throttle
+Change I/O throttle limits for a block drive to @var{bps} @var{bps_rd} @var{bps_wr} @var{iops} @var{iops_rd} @var{iops_wr}
 ETEXI
 
     {
@@ -1585,7 +1622,6 @@
 STEXI
 @item set_password [ vnc | spice ] password [ action-if-connected ]
 @findex set_password
-
 Change spice/vnc password.  Use zero to make the password stay valid
 forever.  @var{action-if-connected} specifies what should happen in
 case a connection is established: @var{fail} makes the password change
@@ -1605,7 +1641,6 @@
 STEXI
 @item expire_password [ vnc | spice ] expire-time
 @findex expire_password
-
 Specify when a password for spice/vnc becomes
 invalid. @var{expire-time} accepts:
 
@@ -1636,9 +1671,8 @@
     },
 
 STEXI
-@item chardev_add args
-@findex chardev_add
-
+@item chardev-add args
+@findex chardev-add
 chardev_add accepts the same parameters as the -chardev command line switch.
 
 ETEXI
@@ -1653,9 +1687,8 @@
     },
 
 STEXI
-@item chardev_remove id
-@findex chardev_remove
-
+@item chardev-remove id
+@findex chardev-remove
 Removes the chardev @var{id}.
 
 ETEXI
@@ -1671,7 +1704,6 @@
 STEXI
 @item qemu-io @var{device} @var{command}
 @findex qemu-io
-
 Executes a qemu-io command on the given block device.
 
 ETEXI
@@ -1686,106 +1718,45 @@
 
 STEXI
 @item cpu-add @var{id}
+@findex cpu-add
 Add CPU with id @var{id}
 ETEXI
 
     {
+        .name       = "qom-list",
+        .args_type  = "path:s?",
+        .params     = "path",
+        .help       = "list QOM properties",
+        .mhandler.cmd  = hmp_qom_list,
+    },
+
+STEXI
+@item qom-list [@var{path}]
+Print QOM properties of object at location @var{path}
+ETEXI
+
+    {
+        .name       = "qom-set",
+        .args_type  = "path:s,property:s,value:s",
+        .params     = "path property value",
+        .help       = "set QOM property",
+        .mhandler.cmd  = hmp_qom_set,
+    },
+
+STEXI
+@item qom-set @var{path} @var{property} @var{value}
+Set QOM property @var{property} of object at location @var{path} to value @var{value}
+ETEXI
+
+    {
         .name       = "info",
         .args_type  = "item:s?",
         .params     = "[subcommand]",
         .help       = "show various information about the system state",
-        .mhandler.cmd = do_info_help,
-        .sub_cmds.static_table = info_cmds,
+        .mhandler.cmd = hmp_info_help,
+        .sub_table = info_cmds,
     },
 
 STEXI
-@item info @var{subcommand}
-@findex info
-Show various information about the system state.
-
-@table @option
-@item info version
-show the version of QEMU
-@item info network
-show the various VLANs and the associated devices
-@item info chardev
-show the character devices
-@item info block
-show the block devices
-@item info blockstats
-show block device statistics
-@item info registers
-show the cpu registers
-@item info cpus
-show infos for each CPU
-@item info history
-show the command line history
-@item info irq
-show the interrupts statistics (if available)
-@item info pic
-show i8259 (PIC) state
-@item info pci
-show emulated PCI device info
-@item info tlb
-show virtual to physical memory mappings (i386, SH4, SPARC, PPC, and Xtensa only)
-@item info mem
-show the active virtual memory mappings (i386 only)
-@item info jit
-show dynamic compiler info
-@item info numa
-show NUMA information
-@item info kvm
-show KVM information
-@item info usb
-show USB devices plugged on the virtual USB hub
-@item info usbhost
-show all USB host devices
-@item info profile
-show profiling information
-@item info capture
-show information about active capturing
-@item info snapshots
-show list of VM snapshots
-@item info status
-show the current VM status (running|paused)
-@item info mice
-show which guest mouse is receiving events
-@item info vnc
-show the vnc server status
-@item info name
-show the current VM name
-@item info uuid
-show the current VM UUID
-@item info cpustats
-show CPU statistics
-@item info usernet
-show user network stack connection states
-@item info migrate
-show migration status
-@item info migrate_capabilities
-show current migration capabilities
-@item info migrate_cache_size
-show current migration XBZRLE cache size
-@item info balloon
-show balloon information
-@item info qtree
-show device tree
-@item info qdm
-show qdev device model list
-@item info roms
-show roms
-@item info tpm
-show the TPM device
-@item info memory-devices
-show the memory devices
-@end table
-ETEXI
-
-STEXI
-@item info trace-events
-show available trace events and their state
-ETEXI
-
-STEXI
 @end table
 ETEXI
diff --git a/hmp.c b/hmp.c
index 63d7686..cc2056e 100644
--- a/hmp.c
+++ b/hmp.c
@@ -13,27 +13,39 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hmp.h"
 #include "net/net.h"
+#include "net/eth.h"
 #include "sysemu/char.h"
+#include "sysemu/block-backend.h"
 #include "qemu/option.h"
 #include "qemu/timer.h"
 #include "qmp-commands.h"
 #include "qemu/sockets.h"
 #include "monitor/monitor.h"
+#include "monitor/qdev.h"
 #include "qapi/opts-visitor.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/string-output-visitor.h"
+#include "qapi/util.h"
 #include "qapi-visit.h"
+#include "qom/object_interfaces.h"
 #include "ui/console.h"
 #include "block/qapi.h"
 #include "qemu-io.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
+
+#ifdef CONFIG_SPICE
+#include <spice/enums.h>
+#endif
 
 static void hmp_handle_error(Monitor *mon, Error **errp)
 {
     assert(errp);
     if (*errp) {
-        monitor_printf(mon, "%s\n", error_get_pretty(*errp));
-        error_free(*errp);
+        error_report_err(*errp);
     }
 }
 
@@ -55,7 +67,7 @@
     info = qmp_query_version(NULL);
 
     monitor_printf(mon, "%" PRId64 ".%" PRId64 ".%" PRId64 "%s\n",
-                   info->qemu.major, info->qemu.minor, info->qemu.micro,
+                   info->qemu->major, info->qemu->minor, info->qemu->micro,
                    info->package);
 
     qapi_free_VersionInfo(info);
@@ -157,7 +169,15 @@
     }
 
     if (info->has_status) {
-        monitor_printf(mon, "Migration status: %s\n", info->status);
+        monitor_printf(mon, "Migration status: %s",
+                       MigrationStatus_lookup[info->status]);
+        if (info->status == MIGRATION_STATUS_FAILED &&
+            info->has_error_desc) {
+            monitor_printf(mon, " (%s)\n", info->error_desc);
+        } else {
+            monitor_printf(mon, "\n");
+        }
+
         monitor_printf(mon, "total time: %" PRIu64 " milliseconds\n",
                        info->total_time);
         if (info->has_expected_downtime) {
@@ -197,6 +217,10 @@
             monitor_printf(mon, "dirty pages rate: %" PRIu64 " pages\n",
                            info->ram->dirty_pages_rate);
         }
+        if (info->ram->postcopy_requests) {
+            monitor_printf(mon, "postcopy request count: %" PRIu64 "\n",
+                           info->ram->postcopy_requests);
+        }
     }
 
     if (info->has_disk) {
@@ -223,6 +247,11 @@
                        info->xbzrle_cache->overflow);
     }
 
+    if (info->has_cpu_throttle_percentage) {
+        monitor_printf(mon, "cpu throttle percentage: %" PRIu64 "\n",
+                       info->cpu_throttle_percentage);
+    }
+
     qapi_free_MigrationInfo(info);
     qapi_free_MigrationCapabilityStatusList(caps);
 }
@@ -246,6 +275,41 @@
     qapi_free_MigrationCapabilityStatusList(caps);
 }
 
+void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict)
+{
+    MigrationParameters *params;
+
+    params = qmp_query_migrate_parameters(NULL);
+
+    if (params) {
+        monitor_printf(mon, "parameters:");
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_COMPRESS_LEVEL],
+            params->compress_level);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_COMPRESS_THREADS],
+            params->compress_threads);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_DECOMPRESS_THREADS],
+            params->decompress_threads);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_CPU_THROTTLE_INITIAL],
+            params->cpu_throttle_initial);
+        monitor_printf(mon, " %s: %" PRId64,
+            MigrationParameter_lookup[MIGRATION_PARAMETER_CPU_THROTTLE_INCREMENT],
+            params->cpu_throttle_increment);
+        monitor_printf(mon, " %s: '%s'",
+            MigrationParameter_lookup[MIGRATION_PARAMETER_TLS_CREDS],
+            params->tls_creds ? : "");
+        monitor_printf(mon, " %s: '%s'",
+            MigrationParameter_lookup[MIGRATION_PARAMETER_TLS_HOSTNAME],
+            params->tls_hostname ? : "");
+        monitor_printf(mon, "\n");
+    }
+
+    qapi_free_MigrationParameters(params);
+}
+
 void hmp_info_migrate_cache_size(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "xbzrel cache size: %" PRId64 " kbytes\n",
@@ -267,17 +331,27 @@
 
         monitor_printf(mon, "%c CPU #%" PRId64 ":", active, cpu->value->CPU);
 
-        if (cpu->value->has_pc) {
-            monitor_printf(mon, " pc=0x%016" PRIx64, cpu->value->pc);
-        }
-        if (cpu->value->has_nip) {
-            monitor_printf(mon, " nip=0x%016" PRIx64, cpu->value->nip);
-        }
-        if (cpu->value->has_npc) {
-            monitor_printf(mon, " npc=0x%016" PRIx64, cpu->value->npc);
-        }
-        if (cpu->value->has_PC) {
-            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->PC);
+        switch (cpu->value->arch) {
+        case CPU_INFO_ARCH_X86:
+            monitor_printf(mon, " pc=0x%016" PRIx64, cpu->value->u.x86.pc);
+            break;
+        case CPU_INFO_ARCH_PPC:
+            monitor_printf(mon, " nip=0x%016" PRIx64, cpu->value->u.ppc.nip);
+            break;
+        case CPU_INFO_ARCH_SPARC:
+            monitor_printf(mon, " pc=0x%016" PRIx64,
+                           cpu->value->u.q_sparc.pc);
+            monitor_printf(mon, " npc=0x%016" PRIx64,
+                           cpu->value->u.q_sparc.npc);
+            break;
+        case CPU_INFO_ARCH_MIPS:
+            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->u.q_mips.PC);
+            break;
+        case CPU_INFO_ARCH_TRICORE:
+            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->u.tricore.PC);
+            break;
+        default:
+            break;
         }
 
         if (cpu->value->halted) {
@@ -290,14 +364,133 @@
     qapi_free_CpuInfoList(cpu_list);
 }
 
+static void print_block_info(Monitor *mon, BlockInfo *info,
+                             BlockDeviceInfo *inserted, bool verbose)
+{
+    ImageInfo *image_info;
+
+    assert(!info || !info->has_inserted || info->inserted == inserted);
+
+    if (info) {
+        monitor_printf(mon, "%s", info->device);
+        if (inserted && inserted->has_node_name) {
+            monitor_printf(mon, " (%s)", inserted->node_name);
+        }
+    } else {
+        assert(inserted);
+        monitor_printf(mon, "%s",
+                       inserted->has_node_name
+                       ? inserted->node_name
+                       : "<anonymous>");
+    }
+
+    if (inserted) {
+        monitor_printf(mon, ": %s (%s%s%s)\n",
+                       inserted->file,
+                       inserted->drv,
+                       inserted->ro ? ", read-only" : "",
+                       inserted->encrypted ? ", encrypted" : "");
+    } else {
+        monitor_printf(mon, ": [not inserted]\n");
+    }
+
+    if (info) {
+        if (info->has_io_status && info->io_status != BLOCK_DEVICE_IO_STATUS_OK) {
+            monitor_printf(mon, "    I/O status:       %s\n",
+                           BlockDeviceIoStatus_lookup[info->io_status]);
+        }
+
+        if (info->removable) {
+            monitor_printf(mon, "    Removable device: %slocked, tray %s\n",
+                           info->locked ? "" : "not ",
+                           info->tray_open ? "open" : "closed");
+        }
+    }
+
+
+    if (!inserted) {
+        return;
+    }
+
+    monitor_printf(mon, "    Cache mode:       %s%s%s\n",
+                   inserted->cache->writeback ? "writeback" : "writethrough",
+                   inserted->cache->direct ? ", direct" : "",
+                   inserted->cache->no_flush ? ", ignore flushes" : "");
+
+    if (inserted->has_backing_file) {
+        monitor_printf(mon,
+                       "    Backing file:     %s "
+                       "(chain depth: %" PRId64 ")\n",
+                       inserted->backing_file,
+                       inserted->backing_file_depth);
+    }
+
+    if (inserted->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF) {
+        monitor_printf(mon, "    Detect zeroes:    %s\n",
+                       BlockdevDetectZeroesOptions_lookup[inserted->detect_zeroes]);
+    }
+
+    if (inserted->bps  || inserted->bps_rd  || inserted->bps_wr  ||
+        inserted->iops || inserted->iops_rd || inserted->iops_wr)
+    {
+        monitor_printf(mon, "    I/O throttling:   bps=%" PRId64
+                        " bps_rd=%" PRId64  " bps_wr=%" PRId64
+                        " bps_max=%" PRId64
+                        " bps_rd_max=%" PRId64
+                        " bps_wr_max=%" PRId64
+                        " iops=%" PRId64 " iops_rd=%" PRId64
+                        " iops_wr=%" PRId64
+                        " iops_max=%" PRId64
+                        " iops_rd_max=%" PRId64
+                        " iops_wr_max=%" PRId64
+                        " iops_size=%" PRId64
+                        " group=%s\n",
+                        inserted->bps,
+                        inserted->bps_rd,
+                        inserted->bps_wr,
+                        inserted->bps_max,
+                        inserted->bps_rd_max,
+                        inserted->bps_wr_max,
+                        inserted->iops,
+                        inserted->iops_rd,
+                        inserted->iops_wr,
+                        inserted->iops_max,
+                        inserted->iops_rd_max,
+                        inserted->iops_wr_max,
+                        inserted->iops_size,
+                        inserted->group);
+    }
+
+    if (verbose) {
+        monitor_printf(mon, "\nImages:\n");
+        image_info = inserted->image;
+        while (1) {
+                bdrv_image_info_dump((fprintf_function)monitor_printf,
+                                     mon, image_info);
+            if (image_info->has_backing_image) {
+                image_info = image_info->backing_image;
+            } else {
+                break;
+            }
+        }
+    }
+}
+
 void hmp_info_block(Monitor *mon, const QDict *qdict)
 {
     BlockInfoList *block_list, *info;
-    ImageInfo *image_info;
+    BlockDeviceInfoList *blockdev_list, *blockdev;
     const char *device = qdict_get_try_str(qdict, "device");
-    bool verbose = qdict_get_try_bool(qdict, "verbose", 0);
+    bool verbose = qdict_get_try_bool(qdict, "verbose", false);
+    bool nodes = qdict_get_try_bool(qdict, "nodes", false);
+    bool printed = false;
 
-    block_list = qmp_query_block(NULL);
+    /* Print BlockBackend information */
+    if (!nodes) {
+        block_list = qmp_query_block(NULL);
+    } else {
+        block_list = NULL;
+    }
 
     for (info = block_list; info; info = info->next) {
         if (device && strcmp(device, info->value->device)) {
@@ -308,102 +501,40 @@
             monitor_printf(mon, "\n");
         }
 
-        monitor_printf(mon, "%s", info->value->device);
-        if (info->value->has_inserted) {
-            monitor_printf(mon, ": %s (%s%s%s)\n",
-                           info->value->inserted->file,
-                           info->value->inserted->drv,
-                           info->value->inserted->ro ? ", read-only" : "",
-                           info->value->inserted->encrypted ? ", encrypted" : "");
-        } else {
-            monitor_printf(mon, ": [not inserted]\n");
-        }
-
-        if (info->value->has_io_status && info->value->io_status != BLOCK_DEVICE_IO_STATUS_OK) {
-            monitor_printf(mon, "    I/O status:       %s\n",
-                           BlockDeviceIoStatus_lookup[info->value->io_status]);
-        }
-
-        if (info->value->removable) {
-            monitor_printf(mon, "    Removable device: %slocked, tray %s\n",
-                           info->value->locked ? "" : "not ",
-                           info->value->tray_open ? "open" : "closed");
-        }
-
-
-        if (!info->value->has_inserted) {
-            continue;
-        }
-
-        if (info->value->inserted->has_backing_file) {
-            monitor_printf(mon,
-                           "    Backing file:     %s "
-                           "(chain depth: %" PRId64 ")\n",
-                           info->value->inserted->backing_file,
-                           info->value->inserted->backing_file_depth);
-        }
-
-        if (info->value->inserted->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF) {
-            monitor_printf(mon, "    Detect zeroes:    %s\n",
-                           BlockdevDetectZeroesOptions_lookup[info->value->inserted->detect_zeroes]);
-        }
-
-        if (info->value->inserted->bps
-            || info->value->inserted->bps_rd
-            || info->value->inserted->bps_wr
-            || info->value->inserted->iops
-            || info->value->inserted->iops_rd
-            || info->value->inserted->iops_wr)
-        {
-            monitor_printf(mon, "    I/O throttling:   bps=%" PRId64
-                            " bps_rd=%" PRId64  " bps_wr=%" PRId64
-                            " bps_max=%" PRId64
-                            " bps_rd_max=%" PRId64
-                            " bps_wr_max=%" PRId64
-                            " iops=%" PRId64 " iops_rd=%" PRId64
-                            " iops_wr=%" PRId64
-                            " iops_max=%" PRId64
-                            " iops_rd_max=%" PRId64
-                            " iops_wr_max=%" PRId64
-                            " iops_size=%" PRId64 "\n",
-                            info->value->inserted->bps,
-                            info->value->inserted->bps_rd,
-                            info->value->inserted->bps_wr,
-                            info->value->inserted->bps_max,
-                            info->value->inserted->bps_rd_max,
-                            info->value->inserted->bps_wr_max,
-                            info->value->inserted->iops,
-                            info->value->inserted->iops_rd,
-                            info->value->inserted->iops_wr,
-                            info->value->inserted->iops_max,
-                            info->value->inserted->iops_rd_max,
-                            info->value->inserted->iops_wr_max,
-                            info->value->inserted->iops_size);
-        }
-
-        if (verbose) {
-            monitor_printf(mon, "\nImages:\n");
-            image_info = info->value->inserted->image;
-            while (1) {
-                    bdrv_image_info_dump((fprintf_function)monitor_printf,
-                                         mon, image_info);
-                if (image_info->has_backing_image) {
-                    image_info = image_info->backing_image;
-                } else {
-                    break;
-                }
-            }
-        }
+        print_block_info(mon, info->value, info->value->has_inserted
+                                           ? info->value->inserted : NULL,
+                         verbose);
+        printed = true;
     }
 
     qapi_free_BlockInfoList(block_list);
+
+    if ((!device && !nodes) || printed) {
+        return;
+    }
+
+    /* Print node information */
+    blockdev_list = qmp_query_named_block_nodes(NULL);
+    for (blockdev = blockdev_list; blockdev; blockdev = blockdev->next) {
+        assert(blockdev->value->has_node_name);
+        if (device && strcmp(device, blockdev->value->node_name)) {
+            continue;
+        }
+
+        if (blockdev != blockdev_list) {
+            monitor_printf(mon, "\n");
+        }
+
+        print_block_info(mon, NULL, blockdev->value, verbose);
+    }
+    qapi_free_BlockDeviceInfoList(blockdev_list);
 }
 
 void hmp_info_blockstats(Monitor *mon, const QDict *qdict)
 {
     BlockStatsList *stats_list, *stats;
 
-    stats_list = qmp_query_blockstats(NULL);
+    stats_list = qmp_query_blockstats(false, false, NULL);
 
     for (stats = stats_list; stats; stats = stats->next) {
         if (!stats->value->has_device) {
@@ -419,6 +550,9 @@
                        " wr_total_time_ns=%" PRId64
                        " rd_total_time_ns=%" PRId64
                        " flush_total_time_ns=%" PRId64
+                       " rd_merged=%" PRId64
+                       " wr_merged=%" PRId64
+                       " idle_time_ns=%" PRId64
                        "\n",
                        stats->value->stats->rd_bytes,
                        stats->value->stats->wr_bytes,
@@ -427,7 +561,10 @@
                        stats->value->stats->flush_operations,
                        stats->value->stats->wr_total_time_ns,
                        stats->value->stats->rd_total_time_ns,
-                       stats->value->stats->flush_total_time_ns);
+                       stats->value->stats->flush_total_time_ns,
+                       stats->value->stats->rd_merged,
+                       stats->value->stats->wr_merged,
+                       stats->value->stats->idle_time_ns);
     }
 
     qapi_free_BlockStatsList(stats_list);
@@ -441,8 +578,7 @@
 
     info = qmp_query_vnc(&err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -465,8 +601,8 @@
         for (client = info->clients; client; client = client->next) {
             monitor_printf(mon, "Client:\n");
             monitor_printf(mon, "     address: %s:%s\n",
-                           client->value->base->host,
-                           client->value->base->service);
+                           client->value->host,
+                           client->value->service);
             monitor_printf(mon, "  x509_dname: %s\n",
                            client->value->x509_dname ?
                            client->value->x509_dname : "none");
@@ -480,10 +616,30 @@
     qapi_free_VncInfo(info);
 }
 
+#ifdef CONFIG_SPICE
 void hmp_info_spice(Monitor *mon, const QDict *qdict)
 {
     SpiceChannelList *chan;
     SpiceInfo *info;
+    const char *channel_name;
+    const char * const channel_names[] = {
+        [SPICE_CHANNEL_MAIN] = "main",
+        [SPICE_CHANNEL_DISPLAY] = "display",
+        [SPICE_CHANNEL_INPUTS] = "inputs",
+        [SPICE_CHANNEL_CURSOR] = "cursor",
+        [SPICE_CHANNEL_PLAYBACK] = "playback",
+        [SPICE_CHANNEL_RECORD] = "record",
+        [SPICE_CHANNEL_TUNNEL] = "tunnel",
+        [SPICE_CHANNEL_SMARTCARD] = "smartcard",
+        [SPICE_CHANNEL_USBREDIR] = "usbredir",
+        [SPICE_CHANNEL_PORT] = "port",
+#if 0
+        /* minimum spice-protocol is 0.12.3, webdav was added in 0.12.7,
+         * no easy way to #ifdef (SPICE_CHANNEL_* is a enum).  Disable
+         * as quick fix for build failures with older versions. */
+        [SPICE_CHANNEL_WEBDAV] = "webdav",
+#endif
+    };
 
     info = qmp_query_spice(NULL);
 
@@ -514,18 +670,28 @@
         for (chan = info->channels; chan; chan = chan->next) {
             monitor_printf(mon, "Channel:\n");
             monitor_printf(mon, "     address: %s:%s%s\n",
-                           chan->value->base->host, chan->value->base->port,
+                           chan->value->host, chan->value->port,
                            chan->value->tls ? " [tls]" : "");
             monitor_printf(mon, "     session: %" PRId64 "\n",
                            chan->value->connection_id);
             monitor_printf(mon, "     channel: %" PRId64 ":%" PRId64 "\n",
                            chan->value->channel_type, chan->value->channel_id);
+
+            channel_name = "unknown";
+            if (chan->value->channel_type > 0 &&
+                chan->value->channel_type < ARRAY_SIZE(channel_names) &&
+                channel_names[chan->value->channel_type]) {
+                channel_name = channel_names[chan->value->channel_type];
+            }
+
+            monitor_printf(mon, "     channel name: %s\n", channel_name);
         }
     }
 
 out:
     qapi_free_SpiceInfo(info);
 }
+#endif
 
 void hmp_info_balloon(Monitor *mon, const QDict *qdict)
 {
@@ -534,8 +700,7 @@
 
     info = qmp_query_balloon(&err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -553,14 +718,14 @@
                    dev->slot, dev->function);
     monitor_printf(mon, "    ");
 
-    if (dev->class_info.has_desc) {
-        monitor_printf(mon, "%s", dev->class_info.desc);
+    if (dev->class_info->has_desc) {
+        monitor_printf(mon, "%s", dev->class_info->desc);
     } else {
-        monitor_printf(mon, "Class %04" PRId64, dev->class_info.q_class);
+        monitor_printf(mon, "Class %04" PRId64, dev->class_info->q_class);
     }
 
     monitor_printf(mon, ": PCI device %04" PRIx64 ":%04" PRIx64 "\n",
-                   dev->id.vendor, dev->id.device);
+                   dev->id->vendor, dev->id->device);
 
     if (dev->has_irq) {
         monitor_printf(mon, "      IRQ %" PRId64 ".\n", dev->irq);
@@ -568,25 +733,25 @@
 
     if (dev->has_pci_bridge) {
         monitor_printf(mon, "      BUS %" PRId64 ".\n",
-                       dev->pci_bridge->bus.number);
+                       dev->pci_bridge->bus->number);
         monitor_printf(mon, "      secondary bus %" PRId64 ".\n",
-                       dev->pci_bridge->bus.secondary);
+                       dev->pci_bridge->bus->secondary);
         monitor_printf(mon, "      subordinate bus %" PRId64 ".\n",
-                       dev->pci_bridge->bus.subordinate);
+                       dev->pci_bridge->bus->subordinate);
 
         monitor_printf(mon, "      IO range [0x%04"PRIx64", 0x%04"PRIx64"]\n",
-                       dev->pci_bridge->bus.io_range->base,
-                       dev->pci_bridge->bus.io_range->limit);
+                       dev->pci_bridge->bus->io_range->base,
+                       dev->pci_bridge->bus->io_range->limit);
 
         monitor_printf(mon,
                        "      memory range [0x%08"PRIx64", 0x%08"PRIx64"]\n",
-                       dev->pci_bridge->bus.memory_range->base,
-                       dev->pci_bridge->bus.memory_range->limit);
+                       dev->pci_bridge->bus->memory_range->base,
+                       dev->pci_bridge->bus->memory_range->limit);
 
         monitor_printf(mon, "      prefetchable memory range "
                        "[0x%08"PRIx64", 0x%08"PRIx64"]\n",
-                       dev->pci_bridge->bus.prefetchable_range->base,
-                       dev->pci_bridge->bus.prefetchable_range->limit);
+                       dev->pci_bridge->bus->prefetchable_range->base,
+                       dev->pci_bridge->bus->prefetchable_range->limit);
     }
 
     for (region = dev->regions; region; region = region->next) {
@@ -707,18 +872,18 @@
                        c, TpmModel_lookup[ti->model]);
 
         monitor_printf(mon, "  \\ %s: type=%s",
-                       ti->id, TpmTypeOptionsKind_lookup[ti->options->kind]);
+                       ti->id, TpmTypeOptionsKind_lookup[ti->options->type]);
 
-        switch (ti->options->kind) {
+        switch (ti->options->type) {
         case TPM_TYPE_OPTIONS_KIND_PASSTHROUGH:
-            tpo = ti->options->passthrough;
+            tpo = ti->options->u.passthrough.data;
             monitor_printf(mon, "%s%s%s%s",
                            tpo->has_path ? ",path=" : "",
                            tpo->has_path ? tpo->path : "",
                            tpo->has_cancel_path ? ",cancel-path=" : "",
                            tpo->has_cancel_path ? tpo->cancel_path : "");
             break;
-        case TPM_TYPE_OPTIONS_KIND_MAX:
+        case TPM_TYPE_OPTIONS_KIND__MAX:
             break;
         }
         monitor_printf(mon, "\n");
@@ -803,8 +968,7 @@
 
     data = qmp_ringbuf_read(chardev, size, false, 0, &err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
@@ -862,7 +1026,7 @@
     qmp_system_wakeup(NULL);
 }
 
-void hmp_inject_nmi(Monitor *mon, const QDict *qdict)
+void hmp_nmi(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
 
@@ -873,7 +1037,7 @@
 void hmp_set_link(Monitor *mon, const QDict *qdict)
 {
     const char *name = qdict_get_str(qdict, "name");
-    int up = qdict_get_bool(qdict, "up");
+    bool up = qdict_get_bool(qdict, "up");
     Error *err = NULL;
 
     qmp_set_link(name, up, &err);
@@ -897,8 +1061,7 @@
 
     qmp_balloon(value, &err);
     if (err) {
-        monitor_printf(mon, "balloon: %s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
     }
 }
 
@@ -914,31 +1077,28 @@
 
 void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
 {
-    const char *device = qdict_get_str(qdict, "device");
     const char *filename = qdict_get_str(qdict, "target");
     const char *format = qdict_get_try_str(qdict, "format");
-    int reuse = qdict_get_try_bool(qdict, "reuse", 0);
-    int full = qdict_get_try_bool(qdict, "full", 0);
-    enum NewImageMode mode;
+    bool reuse = qdict_get_try_bool(qdict, "reuse", false);
+    bool full = qdict_get_try_bool(qdict, "full", false);
     Error *err = NULL;
+    DriveMirror mirror = {
+        .device = (char *)qdict_get_str(qdict, "device"),
+        .target = (char *)filename,
+        .has_format = !!format,
+        .format = (char *)format,
+        .sync = full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
+        .has_mode = true,
+        .mode = reuse ? NEW_IMAGE_MODE_EXISTING : NEW_IMAGE_MODE_ABSOLUTE_PATHS,
+        .unmap = true,
+    };
 
     if (!filename) {
-        error_set(&err, QERR_MISSING_PARAMETER, "target");
+        error_setg(&err, QERR_MISSING_PARAMETER, "target");
         hmp_handle_error(mon, &err);
         return;
     }
-
-    if (reuse) {
-        mode = NEW_IMAGE_MODE_EXISTING;
-    } else {
-        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
-    }
-
-    qmp_drive_mirror(device, filename, !!format, format,
-                     false, NULL, false, NULL,
-                     full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
-                     true, mode, false, 0, false, 0, false, 0,
-                     false, 0, false, 0, &err);
+    qmp_drive_mirror(&mirror, &err);
     hmp_handle_error(mon, &err);
 }
 
@@ -947,13 +1107,13 @@
     const char *device = qdict_get_str(qdict, "device");
     const char *filename = qdict_get_str(qdict, "target");
     const char *format = qdict_get_try_str(qdict, "format");
-    int reuse = qdict_get_try_bool(qdict, "reuse", 0);
-    int full = qdict_get_try_bool(qdict, "full", 0);
+    bool reuse = qdict_get_try_bool(qdict, "reuse", false);
+    bool full = qdict_get_try_bool(qdict, "full", false);
     enum NewImageMode mode;
     Error *err = NULL;
 
     if (!filename) {
-        error_set(&err, QERR_MISSING_PARAMETER, "target");
+        error_setg(&err, QERR_MISSING_PARAMETER, "target");
         hmp_handle_error(mon, &err);
         return;
     }
@@ -964,9 +1124,10 @@
         mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
     }
 
-    qmp_drive_backup(device, filename, !!format, format,
+    qmp_drive_backup(false, NULL, device, filename, !!format, format,
                      full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
-                     true, mode, false, 0, false, 0, false, 0, &err);
+                     true, mode, false, 0, false, NULL,
+                     false, 0, false, 0, &err);
     hmp_handle_error(mon, &err);
 }
 
@@ -975,14 +1136,14 @@
     const char *device = qdict_get_str(qdict, "device");
     const char *filename = qdict_get_try_str(qdict, "snapshot-file");
     const char *format = qdict_get_try_str(qdict, "format");
-    int reuse = qdict_get_try_bool(qdict, "reuse", 0);
+    bool reuse = qdict_get_try_bool(qdict, "reuse", false);
     enum NewImageMode mode;
     Error *err = NULL;
 
     if (!filename) {
         /* In the future, if 'snapshot-file' is not specified, the snapshot
            will be taken internally. Today it's actually required. */
-        error_set(&err, QERR_MISSING_PARAMETER, "snapshot-file");
+        error_setg(&err, QERR_MISSING_PARAMETER, "snapshot-file");
         hmp_handle_error(mon, &err);
         return;
     }
@@ -1022,6 +1183,16 @@
     qmp_migrate_cancel(NULL);
 }
 
+void hmp_migrate_incoming(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    const char *uri = qdict_get_str(qdict, "uri");
+
+    qmp_migrate_incoming(uri, &err);
+
+    hmp_handle_error(mon, &err);
+}
+
 void hmp_migrate_set_downtime(Monitor *mon, const QDict *qdict)
 {
     double value = qdict_get_double(qdict, "value");
@@ -1035,8 +1206,7 @@
 
     qmp_migrate_set_cache_size(value, &err);
     if (err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 }
@@ -1055,7 +1225,7 @@
     MigrationCapabilityStatusList *caps = g_malloc0(sizeof(*caps));
     int i;
 
-    for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
+    for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
         if (strcmp(cap, MigrationCapability_lookup[i]) == 0) {
             caps->value = g_malloc0(sizeof(*caps->value));
             caps->value->capability = i;
@@ -1066,19 +1236,117 @@
         }
     }
 
-    if (i == MIGRATION_CAPABILITY_MAX) {
-        error_set(&err, QERR_INVALID_PARAMETER, cap);
+    if (i == MIGRATION_CAPABILITY__MAX) {
+        error_setg(&err, QERR_INVALID_PARAMETER, cap);
     }
 
     qapi_free_MigrationCapabilityStatusList(caps);
 
     if (err) {
-        monitor_printf(mon, "migrate_set_capability: %s\n",
-                       error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
     }
 }
 
+void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict)
+{
+    const char *param = qdict_get_str(qdict, "parameter");
+    const char *valuestr = qdict_get_str(qdict, "value");
+    long valueint = 0;
+    Error *err = NULL;
+    bool has_compress_level = false;
+    bool has_compress_threads = false;
+    bool has_decompress_threads = false;
+    bool has_cpu_throttle_initial = false;
+    bool has_cpu_throttle_increment = false;
+    bool has_tls_creds = false;
+    bool has_tls_hostname = false;
+    bool use_int_value = false;
+    int i;
+
+    for (i = 0; i < MIGRATION_PARAMETER__MAX; i++) {
+        if (strcmp(param, MigrationParameter_lookup[i]) == 0) {
+            switch (i) {
+            case MIGRATION_PARAMETER_COMPRESS_LEVEL:
+                has_compress_level = true;
+                use_int_value = true;
+                break;
+            case MIGRATION_PARAMETER_COMPRESS_THREADS:
+                has_compress_threads = true;
+                use_int_value = true;
+                break;
+            case MIGRATION_PARAMETER_DECOMPRESS_THREADS:
+                has_decompress_threads = true;
+                use_int_value = true;
+                break;
+            case MIGRATION_PARAMETER_CPU_THROTTLE_INITIAL:
+                has_cpu_throttle_initial = true;
+                use_int_value = true;
+                break;
+            case MIGRATION_PARAMETER_CPU_THROTTLE_INCREMENT:
+                has_cpu_throttle_increment = true;
+                break;
+            case MIGRATION_PARAMETER_TLS_CREDS:
+                has_tls_creds = true;
+                break;
+            case MIGRATION_PARAMETER_TLS_HOSTNAME:
+                has_tls_hostname = true;
+                break;
+            }
+
+            if (use_int_value) {
+                if (qemu_strtol(valuestr, NULL, 10, &valueint) < 0) {
+                    error_setg(&err, "Unable to parse '%s' as an int",
+                               valuestr);
+                    goto cleanup;
+                }
+            }
+
+            qmp_migrate_set_parameters(has_compress_level, valueint,
+                                       has_compress_threads, valueint,
+                                       has_decompress_threads, valueint,
+                                       has_cpu_throttle_initial, valueint,
+                                       has_cpu_throttle_increment, valueint,
+                                       has_tls_creds, valuestr,
+                                       has_tls_hostname, valuestr,
+                                       &err);
+            break;
+        }
+    }
+
+    if (i == MIGRATION_PARAMETER__MAX) {
+        error_setg(&err, QERR_INVALID_PARAMETER, param);
+    }
+
+ cleanup:
+    if (err) {
+        error_report_err(err);
+    }
+}
+
+void hmp_client_migrate_info(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    const char *protocol = qdict_get_str(qdict, "protocol");
+    const char *hostname = qdict_get_str(qdict, "hostname");
+    bool has_port        = qdict_haskey(qdict, "port");
+    int port             = qdict_get_try_int(qdict, "port", -1);
+    bool has_tls_port    = qdict_haskey(qdict, "tls-port");
+    int tls_port         = qdict_get_try_int(qdict, "tls-port", -1);
+    const char *cert_subject = qdict_get_try_str(qdict, "cert-subject");
+
+    qmp_client_migrate_info(protocol, hostname,
+                            has_port, port, has_tls_port, tls_port,
+                            !!cert_subject, cert_subject, &err);
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    qmp_migrate_start_postcopy(&err);
+    hmp_handle_error(mon, &err);
+}
+
 void hmp_set_password(Monitor *mon, const QDict *qdict)
 {
     const char *protocol  = qdict_get_str(qdict, "protocol");
@@ -1102,7 +1370,7 @@
 
 void hmp_eject(Monitor *mon, const QDict *qdict)
 {
-    int force = qdict_get_try_bool(qdict, "force", 0);
+    bool force = qdict_get_try_bool(qdict, "force", false);
     const char *device = qdict_get_str(qdict, "device");
     Error *err = NULL;
 
@@ -1122,52 +1390,63 @@
     const char *device = qdict_get_str(qdict, "device");
     const char *target = qdict_get_str(qdict, "target");
     const char *arg = qdict_get_try_str(qdict, "arg");
+    const char *read_only = qdict_get_try_str(qdict, "read-only-mode");
+    BlockdevChangeReadOnlyMode read_only_mode = 0;
     Error *err = NULL;
 
-    if (strcmp(device, "vnc") == 0 &&
-            (strcmp(target, "passwd") == 0 ||
-             strcmp(target, "password") == 0)) {
-        if (!arg) {
-            monitor_read_password(mon, hmp_change_read_arg, NULL);
+    if (strcmp(device, "vnc") == 0) {
+        if (read_only) {
+            monitor_printf(mon,
+                           "Parameter 'read-only-mode' is invalid for VNC\n");
+            return;
+        }
+        if (strcmp(target, "passwd") == 0 ||
+            strcmp(target, "password") == 0) {
+            if (!arg) {
+                monitor_read_password(mon, hmp_change_read_arg, NULL);
+                return;
+            }
+        }
+        qmp_change("vnc", target, !!arg, arg, &err);
+    } else {
+        if (read_only) {
+            read_only_mode =
+                qapi_enum_parse(BlockdevChangeReadOnlyMode_lookup,
+                                read_only, BLOCKDEV_CHANGE_READ_ONLY_MODE__MAX,
+                                BLOCKDEV_CHANGE_READ_ONLY_MODE_RETAIN, &err);
+            if (err) {
+                hmp_handle_error(mon, &err);
+                return;
+            }
+        }
+
+        qmp_blockdev_change_medium(device, target, !!arg, arg,
+                                   !!read_only, read_only_mode, &err);
+        if (err &&
+            error_get_class(err) == ERROR_CLASS_DEVICE_ENCRYPTED) {
+            error_free(err);
+            monitor_read_block_device_key(mon, device, NULL, NULL);
             return;
         }
     }
 
-    qmp_change(device, target, !!arg, arg, &err);
-    if (err &&
-        error_get_class(err) == ERROR_CLASS_DEVICE_ENCRYPTED) {
-        error_free(err);
-        monitor_read_block_device_key(mon, device, NULL, NULL);
-        return;
-    }
     hmp_handle_error(mon, &err);
 }
 
 void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
+    BlockIOThrottle throttle = {
+        .device = (char *) qdict_get_str(qdict, "device"),
+        .bps = qdict_get_int(qdict, "bps"),
+        .bps_rd = qdict_get_int(qdict, "bps_rd"),
+        .bps_wr = qdict_get_int(qdict, "bps_wr"),
+        .iops = qdict_get_int(qdict, "iops"),
+        .iops_rd = qdict_get_int(qdict, "iops_rd"),
+        .iops_wr = qdict_get_int(qdict, "iops_wr"),
+    };
 
-    qmp_block_set_io_throttle(qdict_get_str(qdict, "device"),
-                              qdict_get_int(qdict, "bps"),
-                              qdict_get_int(qdict, "bps_rd"),
-                              qdict_get_int(qdict, "bps_wr"),
-                              qdict_get_int(qdict, "iops"),
-                              qdict_get_int(qdict, "iops_rd"),
-                              qdict_get_int(qdict, "iops_wr"),
-                              false, /* no burst max via HMP */
-                              0,
-                              false,
-                              0,
-                              false,
-                              0,
-                              false,
-                              0,
-                              false,
-                              0,
-                              false,
-                              0,
-                              false, /* No default I/O size */
-                              0, &err);
+    qmp_block_set_io_throttle(&throttle, &err);
     hmp_handle_error(mon, &err);
 }
 
@@ -1178,7 +1457,7 @@
     const char *base = qdict_get_try_str(qdict, "base");
     int64_t speed = qdict_get_try_int(qdict, "speed", 0);
 
-    qmp_block_stream(device, base != NULL, base, false, NULL,
+    qmp_block_stream(false, NULL, device, base != NULL, base, false, NULL,
                      qdict_haskey(qdict, "speed"), speed,
                      true, BLOCKDEV_ON_ERROR_REPORT, &error);
 
@@ -1200,7 +1479,7 @@
 {
     Error *error = NULL;
     const char *device = qdict_get_str(qdict, "device");
-    bool force = qdict_get_try_bool(qdict, "force", 0);
+    bool force = qdict_get_try_bool(qdict, "force", false);
 
     qmp_block_job_cancel(device, true, force, &error);
 
@@ -1237,21 +1516,21 @@
     hmp_handle_error(mon, &error);
 }
 
-typedef struct MigrationStatus
+typedef struct HMPMigrationStatus
 {
     QEMUTimer *timer;
     Monitor *mon;
     bool is_block_migration;
-} MigrationStatus;
+} HMPMigrationStatus;
 
 static void hmp_migrate_status_cb(void *opaque)
 {
-    MigrationStatus *status = opaque;
+    HMPMigrationStatus *status = opaque;
     MigrationInfo *info;
 
     info = qmp_query_migrate(NULL);
-    if (!info->has_status || strcmp(info->status, "active") == 0 ||
-        strcmp(info->status, "setup") == 0) {
+    if (!info->has_status || info->status == MIGRATION_STATUS_ACTIVE ||
+        info->status == MIGRATION_STATUS_SETUP) {
         if (info->has_disk) {
             int progress;
 
@@ -1270,6 +1549,9 @@
         if (status->is_block_migration) {
             monitor_printf(status->mon, "\n");
         }
+        if (info->has_error_desc) {
+            error_report("%s", info->error_desc);
+        }
         monitor_resume(status->mon);
         timer_del(status->timer);
         g_free(status);
@@ -1280,21 +1562,20 @@
 
 void hmp_migrate(Monitor *mon, const QDict *qdict)
 {
-    int detach = qdict_get_try_bool(qdict, "detach", 0);
-    int blk = qdict_get_try_bool(qdict, "blk", 0);
-    int inc = qdict_get_try_bool(qdict, "inc", 0);
+    bool detach = qdict_get_try_bool(qdict, "detach", false);
+    bool blk = qdict_get_try_bool(qdict, "blk", false);
+    bool inc = qdict_get_try_bool(qdict, "inc", false);
     const char *uri = qdict_get_str(qdict, "uri");
     Error *err = NULL;
 
     qmp_migrate(uri, !!blk, blk, !!inc, inc, false, false, &err);
     if (err) {
-        monitor_printf(mon, "migrate: %s\n", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return;
     }
 
     if (!detach) {
-        MigrationStatus *status;
+        HMPMigrationStatus *status;
 
         if (monitor_suspend(mon) < 0) {
             monitor_printf(mon, "terminal does not allow synchronous "
@@ -1311,6 +1592,14 @@
     }
 }
 
+void hmp_device_add(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+
+    qmp_device_add((QDict *)qdict, NULL, &err);
+    hmp_handle_error(mon, &err);
+}
+
 void hmp_device_del(Monitor *mon, const QDict *qdict)
 {
     const char *id = qdict_get_str(qdict, "id");
@@ -1323,15 +1612,17 @@
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
-    int paging = qdict_get_try_bool(qdict, "paging", 0);
-    int zlib = qdict_get_try_bool(qdict, "zlib", 0);
-    int lzo = qdict_get_try_bool(qdict, "lzo", 0);
-    int snappy = qdict_get_try_bool(qdict, "snappy", 0);
+    bool paging = qdict_get_try_bool(qdict, "paging", false);
+    bool zlib = qdict_get_try_bool(qdict, "zlib", false);
+    bool lzo = qdict_get_try_bool(qdict, "lzo", false);
+    bool snappy = qdict_get_try_bool(qdict, "snappy", false);
     const char *file = qdict_get_str(qdict, "filename");
     bool has_begin = qdict_haskey(qdict, "begin");
     bool has_length = qdict_haskey(qdict, "length");
+    bool has_detach = qdict_haskey(qdict, "detach");
     int64_t begin = 0;
     int64_t length = 0;
+    bool detach = false;
     enum DumpGuestMemoryFormat dump_format = DUMP_GUEST_MEMORY_FORMAT_ELF;
     char *prot;
 
@@ -1359,11 +1650,14 @@
     if (has_length) {
         length = qdict_get_int(qdict, "length");
     }
+    if (has_detach) {
+        detach = qdict_get_bool(qdict, "detach");
+    }
 
     prot = g_strconcat("file:", file, NULL);
 
-    qmp_dump_guest_memory(paging, prot, has_begin, begin, has_length, length,
-                          true, dump_format, &err);
+    qmp_dump_guest_memory(paging, prot, true, detach, has_begin, begin,
+                          has_length, length, true, dump_format, &err);
     hmp_handle_error(mon, &err);
     g_free(prot);
 }
@@ -1399,58 +1693,27 @@
 void hmp_object_add(Monitor *mon, const QDict *qdict)
 {
     Error *err = NULL;
-    Error *err_end = NULL;
     QemuOpts *opts;
-    char *type = NULL;
-    char *id = NULL;
-    void *dummy = NULL;
-    OptsVisitor *ov;
-    QDict *pdict;
+    Visitor *v;
+    Object *obj = NULL;
 
     opts = qemu_opts_from_qdict(qemu_find_opts("object"), qdict, &err);
     if (err) {
-        goto out;
+        hmp_handle_error(mon, &err);
+        return;
     }
 
-    ov = opts_visitor_new(opts);
-    pdict = qdict_clone_shallow(qdict);
-
-    visit_start_struct(opts_get_visitor(ov), &dummy, NULL, NULL, 0, &err);
-    if (err) {
-        goto out_clean;
-    }
-
-    qdict_del(pdict, "qom-type");
-    visit_type_str(opts_get_visitor(ov), &type, "qom-type", &err);
-    if (err) {
-        goto out_end;
-    }
-
-    qdict_del(pdict, "id");
-    visit_type_str(opts_get_visitor(ov), &id, "id", &err);
-    if (err) {
-        goto out_end;
-    }
-
-    object_add(type, id, pdict, opts_get_visitor(ov), &err);
-
-out_end:
-    visit_end_struct(opts_get_visitor(ov), &err_end);
-    if (!err && err_end) {
-        qmp_object_del(id, NULL);
-    }
-    error_propagate(&err, err_end);
-out_clean:
-    opts_visitor_cleanup(ov);
-
-    QDECREF(pdict);
+    v = opts_visitor_new(opts);
+    obj = user_creatable_add(qdict, v, &err);
+    visit_free(v);
     qemu_opts_del(opts);
-    g_free(id);
-    g_free(type);
-    g_free(dummy);
 
-out:
-    hmp_handle_error(mon, &err);
+    if (err) {
+        hmp_handle_error(mon, &err);
+    }
+    if (obj) {
+        object_unref(obj);
+    }
 }
 
 void hmp_getfd(Monitor *mon, const QDict *qdict)
@@ -1471,28 +1734,25 @@
     hmp_handle_error(mon, &err);
 }
 
-void hmp_send_key(Monitor *mon, const QDict *qdict)
+void hmp_sendkey(Monitor *mon, const QDict *qdict)
 {
     const char *keys = qdict_get_str(qdict, "keys");
     KeyValueList *keylist, *head = NULL, *tmp = NULL;
     int has_hold_time = qdict_haskey(qdict, "hold-time");
     int hold_time = qdict_get_try_int(qdict, "hold-time", -1);
     Error *err = NULL;
-    char keyname_buf[16];
     char *separator;
     int keyname_len;
 
     while (1) {
         separator = strchr(keys, '-');
         keyname_len = separator ? separator - keys : strlen(keys);
-        pstrcpy(keyname_buf, sizeof(keyname_buf), keys);
 
         /* Be compatible with old interface, convert user inputted "<" */
-        if (!strncmp(keyname_buf, "<", 1) && keyname_len == 1) {
-            pstrcpy(keyname_buf, sizeof(keyname_buf), "less");
+        if (keys[0] == '<' && keyname_len == 1) {
+            keys = "less";
             keyname_len = 4;
         }
-        keyname_buf[keyname_len] = 0;
 
         keylist = g_malloc0(sizeof(*keylist));
         keylist->value = g_malloc0(sizeof(*keylist->value));
@@ -1505,21 +1765,22 @@
         }
         tmp = keylist;
 
-        if (strstart(keyname_buf, "0x", NULL)) {
+        if (strstart(keys, "0x", NULL)) {
             char *endp;
-            int value = strtoul(keyname_buf, &endp, 0);
-            if (*endp != '\0') {
+            int value = strtoul(keys, &endp, 0);
+            assert(endp <= keys + keyname_len);
+            if (endp != keys + keyname_len) {
                 goto err_out;
             }
-            keylist->value->kind = KEY_VALUE_KIND_NUMBER;
-            keylist->value->number = value;
+            keylist->value->type = KEY_VALUE_KIND_NUMBER;
+            keylist->value->u.number.data = value;
         } else {
-            int idx = index_from_key(keyname_buf);
-            if (idx == Q_KEY_CODE_MAX) {
+            int idx = index_from_key(keys, keyname_len);
+            if (idx == Q_KEY_CODE__MAX) {
                 goto err_out;
             }
-            keylist->value->kind = KEY_VALUE_KIND_QCODE;
-            keylist->value->qcode = idx;
+            keylist->value->type = KEY_VALUE_KIND_QCODE;
+            keylist->value->u.qcode.data = idx;
         }
 
         if (!separator) {
@@ -1536,11 +1797,11 @@
     return;
 
 err_out:
-    monitor_printf(mon, "invalid parameter: %s\n", keyname_buf);
+    monitor_printf(mon, "invalid parameter: %.*s\n", keyname_len, keys);
     goto out;
 }
 
-void hmp_screen_dump(Monitor *mon, const QDict *qdict)
+void hmp_screendump(Monitor *mon, const QDict *qdict)
 {
     const char *filename = qdict_get_str(qdict, "filename");
     Error *err = NULL;
@@ -1552,8 +1813,8 @@
 void hmp_nbd_server_start(Monitor *mon, const QDict *qdict)
 {
     const char *uri = qdict_get_str(qdict, "uri");
-    int writable = qdict_get_try_bool(qdict, "writable", 0);
-    int all = qdict_get_try_bool(qdict, "all", 0);
+    bool writable = qdict_get_try_bool(qdict, "writable", false);
+    bool all = qdict_get_try_bool(qdict, "all", false);
     Error *local_err = NULL;
     BlockInfoList *block_list, *info;
     SocketAddress *addr;
@@ -1569,7 +1830,7 @@
         goto exit;
     }
 
-    qmp_nbd_server_start(addr, &local_err);
+    qmp_nbd_server_start(addr, false, NULL, &local_err);
     qapi_free_SocketAddress(addr);
     if (local_err != NULL) {
         goto exit;
@@ -1606,7 +1867,7 @@
 void hmp_nbd_server_add(Monitor *mon, const QDict *qdict)
 {
     const char *device = qdict_get_str(qdict, "device");
-    int writable = qdict_get_try_bool(qdict, "writable", 0);
+    bool writable = qdict_get_try_bool(qdict, "writable", false);
     Error *local_err = NULL;
 
     qmp_nbd_server_add(device, true, writable, &local_err);
@@ -1640,7 +1901,7 @@
     Error *err = NULL;
     QemuOpts *opts;
 
-    opts = qemu_opts_parse(qemu_find_opts("chardev"), args, 1);
+    opts = qemu_opts_parse_noisily(qemu_find_opts("chardev"), args, true);
     if (opts == NULL) {
         error_setg(&err, "Parsing chardev args failed");
     } else {
@@ -1659,16 +1920,22 @@
 
 void hmp_qemu_io(Monitor *mon, const QDict *qdict)
 {
-    BlockDriverState *bs;
+    BlockBackend *blk;
     const char* device = qdict_get_str(qdict, "device");
     const char* command = qdict_get_str(qdict, "command");
     Error *err = NULL;
 
-    bs = bdrv_find(device);
-    if (bs) {
-        qemuio_command(bs, command);
+    blk = blk_by_name(device);
+    if (blk) {
+        AioContext *aio_context = blk_get_aio_context(blk);
+        aio_context_acquire(aio_context);
+
+        qemuio_command(blk, command);
+
+        aio_context_release(aio_context);
     } else {
-        error_set(&err, QERR_DEVICE_NOT_FOUND, device);
+        error_set(&err, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device);
     }
 
     hmp_handle_error(mon, &err);
@@ -1679,7 +1946,7 @@
     const char *id = qdict_get_str(qdict, "id");
     Error *err = NULL;
 
-    qmp_object_del(id, &err);
+    user_creatable_del(id, &err);
     hmp_handle_error(mon, &err);
 }
 
@@ -1688,15 +1955,14 @@
     Error *err = NULL;
     MemdevList *memdev_list = qmp_query_memdev(&err);
     MemdevList *m = memdev_list;
-    StringOutputVisitor *ov;
+    Visitor *v;
     char *str;
     int i = 0;
 
 
     while (m) {
-        ov = string_output_visitor_new(false);
-        visit_type_uint16List(string_output_get_visitor(ov),
-                              &m->value->host_nodes, NULL, NULL);
+        v = string_output_visitor_new(false, &str);
+        visit_type_uint16List(v, NULL, &m->value->host_nodes, NULL);
         monitor_printf(mon, "memory backend: %d\n", i);
         monitor_printf(mon, "  size:  %" PRId64 "\n", m->value->size);
         monitor_printf(mon, "  merge: %s\n",
@@ -1707,11 +1973,11 @@
                        m->value->prealloc ? "true" : "false");
         monitor_printf(mon, "  policy: %s\n",
                        HostMemPolicy_lookup[m->value->policy]);
-        str = string_output_get_string(ov);
+        visit_complete(v, &str);
         monitor_printf(mon, "  host nodes: %s\n", str);
 
         g_free(str);
-        string_output_visitor_cleanup(ov);
+        visit_free(v);
         m = m->next;
         i++;
     }
@@ -1733,12 +1999,12 @@
         value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                di = value->dimm;
+                di = value->u.dimm.data;
 
                 monitor_printf(mon, "Memory device [%s]: \"%s\"\n",
-                               MemoryDeviceInfoKind_lookup[value->kind],
+                               MemoryDeviceInfoKind_lookup[value->type],
                                di->id ? di->id : "");
                 monitor_printf(mon, "  addr: 0x%" PRIx64 "\n", di->addr);
                 monitor_printf(mon, "  slot: %" PRId64 "\n", di->slot);
@@ -1758,3 +2024,425 @@
 
     qapi_free_MemoryDeviceInfoList(info_list);
 }
+
+void hmp_info_iothreads(Monitor *mon, const QDict *qdict)
+{
+    IOThreadInfoList *info_list = qmp_query_iothreads(NULL);
+    IOThreadInfoList *info;
+
+    for (info = info_list; info; info = info->next) {
+        monitor_printf(mon, "%s: thread_id=%" PRId64 "\n",
+                       info->value->id, info->value->thread_id);
+    }
+
+    qapi_free_IOThreadInfoList(info_list);
+}
+
+void hmp_qom_list(Monitor *mon, const QDict *qdict)
+{
+    const char *path = qdict_get_try_str(qdict, "path");
+    ObjectPropertyInfoList *list;
+    Error *err = NULL;
+
+    if (path == NULL) {
+        monitor_printf(mon, "/\n");
+        return;
+    }
+
+    list = qmp_qom_list(path, &err);
+    if (err == NULL) {
+        ObjectPropertyInfoList *start = list;
+        while (list != NULL) {
+            ObjectPropertyInfo *value = list->value;
+
+            monitor_printf(mon, "%s (%s)\n",
+                           value->name, value->type);
+            list = list->next;
+        }
+        qapi_free_ObjectPropertyInfoList(start);
+    }
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_qom_set(Monitor *mon, const QDict *qdict)
+{
+    const char *path = qdict_get_str(qdict, "path");
+    const char *property = qdict_get_str(qdict, "property");
+    const char *value = qdict_get_str(qdict, "value");
+    Error *err = NULL;
+    bool ambiguous = false;
+    Object *obj;
+
+    obj = object_resolve_path(path, &ambiguous);
+    if (obj == NULL) {
+        error_set(&err, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", path);
+    } else {
+        if (ambiguous) {
+            monitor_printf(mon, "Warning: Path '%s' is ambiguous\n", path);
+        }
+        object_property_parse(obj, value, property, &err);
+    }
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_rocker(Monitor *mon, const QDict *qdict)
+{
+    const char *name = qdict_get_str(qdict, "name");
+    RockerSwitch *rocker;
+    Error *err = NULL;
+
+    rocker = qmp_query_rocker(name, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
+        return;
+    }
+
+    monitor_printf(mon, "name: %s\n", rocker->name);
+    monitor_printf(mon, "id: 0x%" PRIx64 "\n", rocker->id);
+    monitor_printf(mon, "ports: %d\n", rocker->ports);
+
+    qapi_free_RockerSwitch(rocker);
+}
+
+void hmp_rocker_ports(Monitor *mon, const QDict *qdict)
+{
+    RockerPortList *list, *port;
+    const char *name = qdict_get_str(qdict, "name");
+    Error *err = NULL;
+
+    list = qmp_query_rocker_ports(name, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
+        return;
+    }
+
+    monitor_printf(mon, "            ena/    speed/ auto\n");
+    monitor_printf(mon, "      port  link    duplex neg?\n");
+
+    for (port = list; port; port = port->next) {
+        monitor_printf(mon, "%10s  %-4s   %-3s  %2s  %-3s\n",
+                       port->value->name,
+                       port->value->enabled ? port->value->link_up ?
+                       "up" : "down" : "!ena",
+                       port->value->speed == 10000 ? "10G" : "??",
+                       port->value->duplex ? "FD" : "HD",
+                       port->value->autoneg ? "Yes" : "No");
+    }
+
+    qapi_free_RockerPortList(list);
+}
+
+void hmp_rocker_of_dpa_flows(Monitor *mon, const QDict *qdict)
+{
+    RockerOfDpaFlowList *list, *info;
+    const char *name = qdict_get_str(qdict, "name");
+    uint32_t tbl_id = qdict_get_try_int(qdict, "tbl_id", -1);
+    Error *err = NULL;
+
+    list = qmp_query_rocker_of_dpa_flows(name, tbl_id != -1, tbl_id, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
+        return;
+    }
+
+    monitor_printf(mon, "prio tbl hits key(mask) --> actions\n");
+
+    for (info = list; info; info = info->next) {
+        RockerOfDpaFlow *flow = info->value;
+        RockerOfDpaFlowKey *key = flow->key;
+        RockerOfDpaFlowMask *mask = flow->mask;
+        RockerOfDpaFlowAction *action = flow->action;
+
+        if (flow->hits) {
+            monitor_printf(mon, "%-4d %-3d %-4" PRIu64,
+                           key->priority, key->tbl_id, flow->hits);
+        } else {
+            monitor_printf(mon, "%-4d %-3d     ",
+                           key->priority, key->tbl_id);
+        }
+
+        if (key->has_in_pport) {
+            monitor_printf(mon, " pport %d", key->in_pport);
+            if (mask->has_in_pport) {
+                monitor_printf(mon, "(0x%x)", mask->in_pport);
+            }
+        }
+
+        if (key->has_vlan_id) {
+            monitor_printf(mon, " vlan %d",
+                           key->vlan_id & VLAN_VID_MASK);
+            if (mask->has_vlan_id) {
+                monitor_printf(mon, "(0x%x)", mask->vlan_id);
+            }
+        }
+
+        if (key->has_tunnel_id) {
+            monitor_printf(mon, " tunnel %d", key->tunnel_id);
+            if (mask->has_tunnel_id) {
+                monitor_printf(mon, "(0x%x)", mask->tunnel_id);
+            }
+        }
+
+        if (key->has_eth_type) {
+            switch (key->eth_type) {
+            case 0x0806:
+                monitor_printf(mon, " ARP");
+                break;
+            case 0x0800:
+                monitor_printf(mon, " IP");
+                break;
+            case 0x86dd:
+                monitor_printf(mon, " IPv6");
+                break;
+            case 0x8809:
+                monitor_printf(mon, " LACP");
+                break;
+            case 0x88cc:
+                monitor_printf(mon, " LLDP");
+                break;
+            default:
+                monitor_printf(mon, " eth type 0x%04x", key->eth_type);
+                break;
+            }
+        }
+
+        if (key->has_eth_src) {
+            if ((strcmp(key->eth_src, "01:00:00:00:00:00") == 0) &&
+                (mask->has_eth_src) &&
+                (strcmp(mask->eth_src, "01:00:00:00:00:00") == 0)) {
+                monitor_printf(mon, " src <any mcast/bcast>");
+            } else if ((strcmp(key->eth_src, "00:00:00:00:00:00") == 0) &&
+                (mask->has_eth_src) &&
+                (strcmp(mask->eth_src, "01:00:00:00:00:00") == 0)) {
+                monitor_printf(mon, " src <any ucast>");
+            } else {
+                monitor_printf(mon, " src %s", key->eth_src);
+                if (mask->has_eth_src) {
+                    monitor_printf(mon, "(%s)", mask->eth_src);
+                }
+            }
+        }
+
+        if (key->has_eth_dst) {
+            if ((strcmp(key->eth_dst, "01:00:00:00:00:00") == 0) &&
+                (mask->has_eth_dst) &&
+                (strcmp(mask->eth_dst, "01:00:00:00:00:00") == 0)) {
+                monitor_printf(mon, " dst <any mcast/bcast>");
+            } else if ((strcmp(key->eth_dst, "00:00:00:00:00:00") == 0) &&
+                (mask->has_eth_dst) &&
+                (strcmp(mask->eth_dst, "01:00:00:00:00:00") == 0)) {
+                monitor_printf(mon, " dst <any ucast>");
+            } else {
+                monitor_printf(mon, " dst %s", key->eth_dst);
+                if (mask->has_eth_dst) {
+                    monitor_printf(mon, "(%s)", mask->eth_dst);
+                }
+            }
+        }
+
+        if (key->has_ip_proto) {
+            monitor_printf(mon, " proto %d", key->ip_proto);
+            if (mask->has_ip_proto) {
+                monitor_printf(mon, "(0x%x)", mask->ip_proto);
+            }
+        }
+
+        if (key->has_ip_tos) {
+            monitor_printf(mon, " TOS %d", key->ip_tos);
+            if (mask->has_ip_tos) {
+                monitor_printf(mon, "(0x%x)", mask->ip_tos);
+            }
+        }
+
+        if (key->has_ip_dst) {
+            monitor_printf(mon, " dst %s", key->ip_dst);
+        }
+
+        if (action->has_goto_tbl || action->has_group_id ||
+            action->has_new_vlan_id) {
+            monitor_printf(mon, " -->");
+        }
+
+        if (action->has_new_vlan_id) {
+            monitor_printf(mon, " apply new vlan %d",
+                           ntohs(action->new_vlan_id));
+        }
+
+        if (action->has_group_id) {
+            monitor_printf(mon, " write group 0x%08x", action->group_id);
+        }
+
+        if (action->has_goto_tbl) {
+            monitor_printf(mon, " goto tbl %d", action->goto_tbl);
+        }
+
+        monitor_printf(mon, "\n");
+    }
+
+    qapi_free_RockerOfDpaFlowList(list);
+}
+
+void hmp_rocker_of_dpa_groups(Monitor *mon, const QDict *qdict)
+{
+    RockerOfDpaGroupList *list, *g;
+    const char *name = qdict_get_str(qdict, "name");
+    uint8_t type = qdict_get_try_int(qdict, "type", 9);
+    Error *err = NULL;
+    bool set = false;
+
+    list = qmp_query_rocker_of_dpa_groups(name, type != 9, type, &err);
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
+        return;
+    }
+
+    monitor_printf(mon, "id (decode) --> buckets\n");
+
+    for (g = list; g; g = g->next) {
+        RockerOfDpaGroup *group = g->value;
+
+        monitor_printf(mon, "0x%08x", group->id);
+
+        monitor_printf(mon, " (type %s", group->type == 0 ? "L2 interface" :
+                                         group->type == 1 ? "L2 rewrite" :
+                                         group->type == 2 ? "L3 unicast" :
+                                         group->type == 3 ? "L2 multicast" :
+                                         group->type == 4 ? "L2 flood" :
+                                         group->type == 5 ? "L3 interface" :
+                                         group->type == 6 ? "L3 multicast" :
+                                         group->type == 7 ? "L3 ECMP" :
+                                         group->type == 8 ? "L2 overlay" :
+                                         "unknown");
+
+        if (group->has_vlan_id) {
+            monitor_printf(mon, " vlan %d", group->vlan_id);
+        }
+
+        if (group->has_pport) {
+            monitor_printf(mon, " pport %d", group->pport);
+        }
+
+        if (group->has_index) {
+            monitor_printf(mon, " index %d", group->index);
+        }
+
+        monitor_printf(mon, ") -->");
+
+        if (group->has_set_vlan_id && group->set_vlan_id) {
+            set = true;
+            monitor_printf(mon, " set vlan %d",
+                           group->set_vlan_id & VLAN_VID_MASK);
+        }
+
+        if (group->has_set_eth_src) {
+            if (!set) {
+                set = true;
+                monitor_printf(mon, " set");
+            }
+            monitor_printf(mon, " src %s", group->set_eth_src);
+        }
+
+        if (group->has_set_eth_dst) {
+            if (!set) {
+                set = true;
+                monitor_printf(mon, " set");
+            }
+            monitor_printf(mon, " dst %s", group->set_eth_dst);
+        }
+
+        set = false;
+
+        if (group->has_ttl_check && group->ttl_check) {
+            monitor_printf(mon, " check TTL");
+        }
+
+        if (group->has_group_id && group->group_id) {
+            monitor_printf(mon, " group id 0x%08x", group->group_id);
+        }
+
+        if (group->has_pop_vlan && group->pop_vlan) {
+            monitor_printf(mon, " pop vlan");
+        }
+
+        if (group->has_out_pport) {
+            monitor_printf(mon, " out pport %d", group->out_pport);
+        }
+
+        if (group->has_group_ids) {
+            struct uint32List *id;
+
+            monitor_printf(mon, " groups [");
+            for (id = group->group_ids; id; id = id->next) {
+                monitor_printf(mon, "0x%08x", id->value);
+                if (id->next) {
+                    monitor_printf(mon, ",");
+                }
+            }
+            monitor_printf(mon, "]");
+        }
+
+        monitor_printf(mon, "\n");
+    }
+
+    qapi_free_RockerOfDpaGroupList(list);
+}
+
+void hmp_info_dump(Monitor *mon, const QDict *qdict)
+{
+    DumpQueryResult *result = qmp_query_dump(NULL);
+
+    assert(result && result->status < DUMP_STATUS__MAX);
+    monitor_printf(mon, "Status: %s\n", DumpStatus_lookup[result->status]);
+
+    if (result->status == DUMP_STATUS_ACTIVE) {
+        float percent = 0;
+        assert(result->total != 0);
+        percent = 100.0 * result->completed / result->total;
+        monitor_printf(mon, "Finished: %.2f %%\n", percent);
+    }
+
+    qapi_free_DumpQueryResult(result);
+}
+
+void hmp_hotpluggable_cpus(Monitor *mon, const QDict *qdict)
+{
+    Error *err = NULL;
+    HotpluggableCPUList *l = qmp_query_hotpluggable_cpus(&err);
+    HotpluggableCPUList *saved = l;
+    CpuInstanceProperties *c;
+
+    if (err != NULL) {
+        hmp_handle_error(mon, &err);
+        return;
+    }
+
+    monitor_printf(mon, "Hotpluggable CPUs:\n");
+    while (l) {
+        monitor_printf(mon, "  type: \"%s\"\n", l->value->type);
+        monitor_printf(mon, "  vcpus_count: \"%" PRIu64 "\"\n",
+                       l->value->vcpus_count);
+        if (l->value->has_qom_path) {
+            monitor_printf(mon, "  qom_path: \"%s\"\n", l->value->qom_path);
+        }
+
+        c = l->value->props;
+        monitor_printf(mon, "  CPUInstance Properties:\n");
+        if (c->has_node_id) {
+            monitor_printf(mon, "    node-id: \"%" PRIu64 "\"\n", c->node_id);
+        }
+        if (c->has_socket_id) {
+            monitor_printf(mon, "    socket-id: \"%" PRIu64 "\"\n", c->socket_id);
+        }
+        if (c->has_core_id) {
+            monitor_printf(mon, "    core-id: \"%" PRIu64 "\"\n", c->core_id);
+        }
+        if (c->has_thread_id) {
+            monitor_printf(mon, "    thread-id: \"%" PRIu64 "\"\n", c->thread_id);
+        }
+
+        l = l->next;
+    }
+
+    qapi_free_HotpluggableCPUList(saved);
+}
diff --git a/hmp.h b/hmp.h
index 4bb5dca..0876ec0 100644
--- a/hmp.h
+++ b/hmp.h
@@ -28,6 +28,7 @@
 void hmp_info_mice(Monitor *mon, const QDict *qdict);
 void hmp_info_migrate(Monitor *mon, const QDict *qdict);
 void hmp_info_migrate_capabilities(Monitor *mon, const QDict *qdict);
+void hmp_info_migrate_parameters(Monitor *mon, const QDict *qdict);
 void hmp_info_migrate_cache_size(Monitor *mon, const QDict *qdict);
 void hmp_info_cpus(Monitor *mon, const QDict *qdict);
 void hmp_info_block(Monitor *mon, const QDict *qdict);
@@ -38,6 +39,7 @@
 void hmp_info_pci(Monitor *mon, const QDict *qdict);
 void hmp_info_block_jobs(Monitor *mon, const QDict *qdict);
 void hmp_info_tpm(Monitor *mon, const QDict *qdict);
+void hmp_info_iothreads(Monitor *mon, const QDict *qdict);
 void hmp_quit(Monitor *mon, const QDict *qdict);
 void hmp_stop(Monitor *mon, const QDict *qdict);
 void hmp_system_reset(Monitor *mon, const QDict *qdict);
@@ -49,7 +51,7 @@
 void hmp_ringbuf_read(Monitor *mon, const QDict *qdict);
 void hmp_cont(Monitor *mon, const QDict *qdict);
 void hmp_system_wakeup(Monitor *mon, const QDict *qdict);
-void hmp_inject_nmi(Monitor *mon, const QDict *qdict);
+void hmp_nmi(Monitor *mon, const QDict *qdict);
 void hmp_set_link(Monitor *mon, const QDict *qdict);
 void hmp_block_passwd(Monitor *mon, const QDict *qdict);
 void hmp_balloon(Monitor *mon, const QDict *qdict);
@@ -60,10 +62,14 @@
 void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
 void hmp_drive_backup(Monitor *mon, const QDict *qdict);
 void hmp_migrate_cancel(Monitor *mon, const QDict *qdict);
+void hmp_migrate_incoming(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_downtime(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_speed(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_capability(Monitor *mon, const QDict *qdict);
+void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_cache_size(Monitor *mon, const QDict *qdict);
+void hmp_client_migrate_info(Monitor *mon, const QDict *qdict);
+void hmp_migrate_start_postcopy(Monitor *mon, const QDict *qdict);
 void hmp_set_password(Monitor *mon, const QDict *qdict);
 void hmp_expire_password(Monitor *mon, const QDict *qdict);
 void hmp_eject(Monitor *mon, const QDict *qdict);
@@ -76,14 +82,15 @@
 void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
 void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
 void hmp_migrate(Monitor *mon, const QDict *qdict);
+void hmp_device_add(Monitor *mon, const QDict *qdict);
 void hmp_device_del(Monitor *mon, const QDict *qdict);
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict);
 void hmp_netdev_add(Monitor *mon, const QDict *qdict);
 void hmp_netdev_del(Monitor *mon, const QDict *qdict);
 void hmp_getfd(Monitor *mon, const QDict *qdict);
 void hmp_closefd(Monitor *mon, const QDict *qdict);
-void hmp_send_key(Monitor *mon, const QDict *qdict);
-void hmp_screen_dump(Monitor *mon, const QDict *qdict);
+void hmp_sendkey(Monitor *mon, const QDict *qdict);
+void hmp_screendump(Monitor *mon, const QDict *qdict);
 void hmp_nbd_server_start(Monitor *mon, const QDict *qdict);
 void hmp_nbd_server_add(Monitor *mon, const QDict *qdict);
 void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict);
@@ -95,6 +102,8 @@
 void hmp_object_del(Monitor *mon, const QDict *qdict);
 void hmp_info_memdev(Monitor *mon, const QDict *qdict);
 void hmp_info_memory_devices(Monitor *mon, const QDict *qdict);
+void hmp_qom_list(Monitor *mon, const QDict *qdict);
+void hmp_qom_set(Monitor *mon, const QDict *qdict);
 void object_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void object_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void device_add_completion(ReadLineState *rs, int nb_args, const char *str);
@@ -106,15 +115,24 @@
 void netdev_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str);
 void ringbuf_write_completion(ReadLineState *rs, int nb_args, const char *str);
-void ringbuf_read_completion(ReadLineState *rs, int nb_args, const char *str);
+void info_trace_events_completion(ReadLineState *rs, int nb_args, const char *str);
+void trace_event_completion(ReadLineState *rs, int nb_args, const char *str);
 void watchdog_action_completion(ReadLineState *rs, int nb_args,
                                 const char *str);
 void migrate_set_capability_completion(ReadLineState *rs, int nb_args,
                                        const char *str);
+void migrate_set_parameter_completion(ReadLineState *rs, int nb_args,
+                                      const char *str);
 void host_net_add_completion(ReadLineState *rs, int nb_args, const char *str);
 void host_net_remove_completion(ReadLineState *rs, int nb_args,
                                 const char *str);
 void delvm_completion(ReadLineState *rs, int nb_args, const char *str);
 void loadvm_completion(ReadLineState *rs, int nb_args, const char *str);
+void hmp_rocker(Monitor *mon, const QDict *qdict);
+void hmp_rocker_ports(Monitor *mon, const QDict *qdict);
+void hmp_rocker_of_dpa_flows(Monitor *mon, const QDict *qdict);
+void hmp_rocker_of_dpa_groups(Monitor *mon, const QDict *qdict);
+void hmp_info_dump(Monitor *mon, const QDict *qdict);
+void hmp_hotpluggable_cpus(Monitor *mon, const QDict *qdict);
 
 #endif
diff --git a/hw/9pfs/9p-handle.c b/hw/9pfs/9p-handle.c
new file mode 100644
index 0000000..3d77594
--- /dev/null
+++ b/hw/9pfs/9p-handle.c
@@ -0,0 +1,707 @@
+/*
+ * 9p handle callback
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *    Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "9p-xattr.h"
+#include <arpa/inet.h>
+#include <pwd.h>
+#include <grp.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include "qemu/xattr.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
+#include <linux/fs.h>
+#ifdef CONFIG_LINUX_MAGIC_H
+#include <linux/magic.h>
+#endif
+#include <sys/ioctl.h>
+
+#ifndef XFS_SUPER_MAGIC
+#define XFS_SUPER_MAGIC  0x58465342
+#endif
+#ifndef EXT2_SUPER_MAGIC
+#define EXT2_SUPER_MAGIC 0xEF53
+#endif
+#ifndef REISERFS_SUPER_MAGIC
+#define REISERFS_SUPER_MAGIC 0x52654973
+#endif
+#ifndef BTRFS_SUPER_MAGIC
+#define BTRFS_SUPER_MAGIC 0x9123683E
+#endif
+
+struct handle_data {
+    int mountfd;
+    int handle_bytes;
+};
+
+static inline int name_to_handle(int dirfd, const char *name,
+                                 struct file_handle *fh, int *mnt_id, int flags)
+{
+    return name_to_handle_at(dirfd, name, fh, mnt_id, flags);
+}
+
+static inline int open_by_handle(int mountfd, const char *fh, int flags)
+{
+    return open_by_handle_at(mountfd, (struct file_handle *)fh, flags);
+}
+
+static int handle_update_file_cred(int dirfd, const char *name, FsCred *credp)
+{
+    int fd, ret;
+    fd = openat(dirfd, name, O_NONBLOCK | O_NOFOLLOW);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fchownat(fd, "", credp->fc_uid, credp->fc_gid, AT_EMPTY_PATH);
+    if (ret < 0) {
+        goto err_out;
+    }
+    ret = fchmod(fd, credp->fc_mode & 07777);
+err_out:
+    close(fd);
+    return ret;
+}
+
+
+static int handle_lstat(FsContext *fs_ctx, V9fsPath *fs_path,
+                        struct stat *stbuf)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_PATH);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fstatat(fd, "", stbuf, AT_EMPTY_PATH);
+    close(fd);
+    return ret;
+}
+
+static ssize_t handle_readlink(FsContext *fs_ctx, V9fsPath *fs_path,
+                               char *buf, size_t bufsz)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_PATH);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = readlinkat(fd, "", buf, bufsz);
+    close(fd);
+    return ret;
+}
+
+static int handle_close(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return close(fs->fd);
+}
+
+static int handle_closedir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return closedir(fs->dir.stream);
+}
+
+static int handle_open(FsContext *ctx, V9fsPath *fs_path,
+                       int flags, V9fsFidOpenState *fs)
+{
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fs->fd = open_by_handle(data->mountfd, fs_path->data, flags);
+    return fs->fd;
+}
+
+static int handle_opendir(FsContext *ctx,
+                          V9fsPath *fs_path, V9fsFidOpenState *fs)
+{
+    int ret;
+    ret = handle_open(ctx, fs_path, O_DIRECTORY, fs);
+    if (ret < 0) {
+        return -1;
+    }
+    fs->dir.stream = fdopendir(ret);
+    if (!fs->dir.stream) {
+        return -1;
+    }
+    return 0;
+}
+
+static void handle_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    rewinddir(fs->dir.stream);
+}
+
+static off_t handle_telldir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return telldir(fs->dir.stream);
+}
+
+static struct dirent *handle_readdir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return readdir(fs->dir.stream);
+}
+
+static void handle_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
+{
+    seekdir(fs->dir.stream, off);
+}
+
+static ssize_t handle_preadv(FsContext *ctx, V9fsFidOpenState *fs,
+                             const struct iovec *iov,
+                             int iovcnt, off_t offset)
+{
+#ifdef CONFIG_PREADV
+    return preadv(fs->fd, iov, iovcnt, offset);
+#else
+    int err = lseek(fs->fd, offset, SEEK_SET);
+    if (err == -1) {
+        return err;
+    } else {
+        return readv(fs->fd, iov, iovcnt);
+    }
+#endif
+}
+
+static ssize_t handle_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
+                              const struct iovec *iov,
+                              int iovcnt, off_t offset)
+{
+    ssize_t ret;
+#ifdef CONFIG_PREADV
+    ret = pwritev(fs->fd, iov, iovcnt, offset);
+#else
+    int err = lseek(fs->fd, offset, SEEK_SET);
+    if (err == -1) {
+        return err;
+    } else {
+        ret = writev(fs->fd, iov, iovcnt);
+    }
+#endif
+#ifdef CONFIG_SYNC_FILE_RANGE
+    if (ret > 0 && ctx->export_flags & V9FS_IMMEDIATE_WRITEOUT) {
+        /*
+         * Initiate a writeback. This is not a data integrity sync.
+         * We want to ensure that we don't leave dirty pages in the cache
+         * after write when writeout=immediate is sepcified.
+         */
+        sync_file_range(fs->fd, offset, ret,
+                        SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
+    }
+#endif
+    return ret;
+}
+
+static int handle_chmod(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fchmod(fd, credp->fc_mode);
+    close(fd);
+    return ret;
+}
+
+static int handle_mknod(FsContext *fs_ctx, V9fsPath *dir_path,
+                       const char *name, FsCred *credp)
+{
+    int dirfd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
+    if (dirfd < 0) {
+        return dirfd;
+    }
+    ret = mknodat(dirfd, name, credp->fc_mode, credp->fc_rdev);
+    if (!ret) {
+        ret = handle_update_file_cred(dirfd, name, credp);
+    }
+    close(dirfd);
+    return ret;
+}
+
+static int handle_mkdir(FsContext *fs_ctx, V9fsPath *dir_path,
+                       const char *name, FsCred *credp)
+{
+    int dirfd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
+    if (dirfd < 0) {
+        return dirfd;
+    }
+    ret = mkdirat(dirfd, name, credp->fc_mode);
+    if (!ret) {
+        ret = handle_update_file_cred(dirfd, name, credp);
+    }
+    close(dirfd);
+    return ret;
+}
+
+static int handle_fstat(FsContext *fs_ctx, int fid_type,
+                        V9fsFidOpenState *fs, struct stat *stbuf)
+{
+    int fd;
+
+    if (fid_type == P9_FID_DIR) {
+        fd = dirfd(fs->dir.stream);
+    } else {
+        fd = fs->fd;
+    }
+    return fstat(fd, stbuf);
+}
+
+static int handle_open2(FsContext *fs_ctx, V9fsPath *dir_path, const char *name,
+                        int flags, FsCred *credp, V9fsFidOpenState *fs)
+{
+    int ret;
+    int dirfd, fd;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
+    if (dirfd < 0) {
+        return dirfd;
+    }
+    fd = openat(dirfd, name, flags | O_NOFOLLOW, credp->fc_mode);
+    if (fd >= 0) {
+        ret = handle_update_file_cred(dirfd, name, credp);
+        if (ret < 0) {
+            close(fd);
+            fd = ret;
+        } else {
+            fs->fd = fd;
+        }
+    }
+    close(dirfd);
+    return fd;
+}
+
+
+static int handle_symlink(FsContext *fs_ctx, const char *oldpath,
+                          V9fsPath *dir_path, const char *name, FsCred *credp)
+{
+    int fd, dirfd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
+    if (dirfd < 0) {
+        return dirfd;
+    }
+    ret = symlinkat(oldpath, dirfd, name);
+    if (!ret) {
+        fd = openat(dirfd, name, O_PATH | O_NOFOLLOW);
+        if (fd < 0) {
+            ret = fd;
+            goto err_out;
+        }
+        ret = fchownat(fd, "", credp->fc_uid, credp->fc_gid, AT_EMPTY_PATH);
+        close(fd);
+    }
+err_out:
+    close(dirfd);
+    return ret;
+}
+
+static int handle_link(FsContext *ctx, V9fsPath *oldpath,
+                       V9fsPath *dirpath, const char *name)
+{
+    int oldfd, newdirfd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    oldfd = open_by_handle(data->mountfd, oldpath->data, O_PATH);
+    if (oldfd < 0) {
+        return oldfd;
+    }
+    newdirfd = open_by_handle(data->mountfd, dirpath->data, O_PATH);
+    if (newdirfd < 0) {
+        close(oldfd);
+        return newdirfd;
+    }
+    ret = linkat(oldfd, "", newdirfd, name, AT_EMPTY_PATH);
+    close(newdirfd);
+    close(oldfd);
+    return ret;
+}
+
+static int handle_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK | O_WRONLY);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = ftruncate(fd, size);
+    close(fd);
+    return ret;
+}
+
+static int handle_rename(FsContext *ctx, const char *oldpath,
+                         const char *newpath)
+{
+    errno = EOPNOTSUPP;
+    return -1;
+}
+
+static int handle_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)fs_ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_PATH);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fchownat(fd, "", credp->fc_uid, credp->fc_gid, AT_EMPTY_PATH);
+    close(fd);
+    return ret;
+}
+
+static int handle_utimensat(FsContext *ctx, V9fsPath *fs_path,
+                            const struct timespec *buf)
+{
+    int ret;
+#ifdef CONFIG_UTIMENSAT
+    int fd;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = futimens(fd, buf);
+    close(fd);
+#else
+    ret = -1;
+    errno = ENOSYS;
+#endif
+    return ret;
+}
+
+static int handle_remove(FsContext *ctx, const char *path)
+{
+    errno = EOPNOTSUPP;
+    return -1;
+}
+
+static int handle_fsync(FsContext *ctx, int fid_type,
+                        V9fsFidOpenState *fs, int datasync)
+{
+    int fd;
+
+    if (fid_type == P9_FID_DIR) {
+        fd = dirfd(fs->dir.stream);
+    } else {
+        fd = fs->fd;
+    }
+
+    if (datasync) {
+        return qemu_fdatasync(fd);
+    } else {
+        return fsync(fd);
+    }
+}
+
+static int handle_statfs(FsContext *ctx, V9fsPath *fs_path,
+                         struct statfs *stbuf)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fstatfs(fd, stbuf);
+    close(fd);
+    return ret;
+}
+
+static ssize_t handle_lgetxattr(FsContext *ctx, V9fsPath *fs_path,
+                                const char *name, void *value, size_t size)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fgetxattr(fd, name, value, size);
+    close(fd);
+    return ret;
+}
+
+static ssize_t handle_llistxattr(FsContext *ctx, V9fsPath *fs_path,
+                                 void *value, size_t size)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = flistxattr(fd, value, size);
+    close(fd);
+    return ret;
+}
+
+static int handle_lsetxattr(FsContext *ctx, V9fsPath *fs_path, const char *name,
+                            void *value, size_t size, int flags)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fsetxattr(fd, name, value, size, flags);
+    close(fd);
+    return ret;
+}
+
+static int handle_lremovexattr(FsContext *ctx, V9fsPath *fs_path,
+                               const char *name)
+{
+    int fd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
+    if (fd < 0) {
+        return fd;
+    }
+    ret = fremovexattr(fd, name);
+    close(fd);
+    return ret;
+}
+
+static int handle_name_to_path(FsContext *ctx, V9fsPath *dir_path,
+                              const char *name, V9fsPath *target)
+{
+    char *buffer;
+    struct file_handle *fh;
+    int dirfd, ret, mnt_id;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    /* "." and ".." are not allowed */
+    if (!strcmp(name, ".") || !strcmp(name, "..")) {
+        errno = EINVAL;
+        return -1;
+
+    }
+    if (dir_path) {
+        dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
+    } else {
+        /* relative to export root */
+        buffer = rpath(ctx, ".");
+        dirfd = open(buffer, O_DIRECTORY);
+        g_free(buffer);
+    }
+    if (dirfd < 0) {
+        return dirfd;
+    }
+    fh = g_malloc(sizeof(struct file_handle) + data->handle_bytes);
+    fh->handle_bytes = data->handle_bytes;
+    /* add a "./" at the beginning of the path */
+    buffer = g_strdup_printf("./%s", name);
+    /* flag = 0 imply don't follow symlink */
+    ret = name_to_handle(dirfd, buffer, fh, &mnt_id, 0);
+    if (!ret) {
+        target->data = (char *)fh;
+        target->size = sizeof(struct file_handle) + data->handle_bytes;
+    } else {
+        g_free(fh);
+    }
+    close(dirfd);
+    g_free(buffer);
+    return ret;
+}
+
+static int handle_renameat(FsContext *ctx, V9fsPath *olddir,
+                           const char *old_name, V9fsPath *newdir,
+                           const char *new_name)
+{
+    int olddirfd, newdirfd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+
+    olddirfd = open_by_handle(data->mountfd, olddir->data, O_PATH);
+    if (olddirfd < 0) {
+        return olddirfd;
+    }
+    newdirfd = open_by_handle(data->mountfd, newdir->data, O_PATH);
+    if (newdirfd < 0) {
+        close(olddirfd);
+        return newdirfd;
+    }
+    ret = renameat(olddirfd, old_name, newdirfd, new_name);
+    close(newdirfd);
+    close(olddirfd);
+    return ret;
+}
+
+static int handle_unlinkat(FsContext *ctx, V9fsPath *dir,
+                           const char *name, int flags)
+{
+    int dirfd, ret;
+    struct handle_data *data = (struct handle_data *)ctx->private;
+    int rflags;
+
+    dirfd = open_by_handle(data->mountfd, dir->data, O_PATH);
+    if (dirfd < 0) {
+        return dirfd;
+    }
+
+    rflags = 0;
+    if (flags & P9_DOTL_AT_REMOVEDIR) {
+        rflags |= AT_REMOVEDIR;
+    }
+
+    ret = unlinkat(dirfd, name, rflags);
+
+    close(dirfd);
+    return ret;
+}
+
+static int handle_ioc_getversion(FsContext *ctx, V9fsPath *path,
+                                 mode_t st_mode, uint64_t *st_gen)
+{
+#ifdef FS_IOC_GETVERSION
+    int err;
+    V9fsFidOpenState fid_open;
+
+    /*
+     * Do not try to open special files like device nodes, fifos etc
+     * We can get fd for regular files and directories only
+     */
+    if (!S_ISREG(st_mode) && !S_ISDIR(st_mode)) {
+        errno = ENOTTY;
+        return -1;
+    }
+    err = handle_open(ctx, path, O_RDONLY, &fid_open);
+    if (err < 0) {
+        return err;
+    }
+    err = ioctl(fid_open.fd, FS_IOC_GETVERSION, st_gen);
+    handle_close(ctx, &fid_open);
+    return err;
+#else
+    errno = ENOTTY;
+    return -1;
+#endif
+}
+
+static int handle_init(FsContext *ctx)
+{
+    int ret, mnt_id;
+    struct statfs stbuf;
+    struct file_handle fh;
+    struct handle_data *data = g_malloc(sizeof(struct handle_data));
+
+    data->mountfd = open(ctx->fs_root, O_DIRECTORY);
+    if (data->mountfd < 0) {
+        ret = data->mountfd;
+        goto err_out;
+    }
+    ret = statfs(ctx->fs_root, &stbuf);
+    if (!ret) {
+        switch (stbuf.f_type) {
+        case EXT2_SUPER_MAGIC:
+        case BTRFS_SUPER_MAGIC:
+        case REISERFS_SUPER_MAGIC:
+        case XFS_SUPER_MAGIC:
+            ctx->exops.get_st_gen = handle_ioc_getversion;
+            break;
+        }
+    }
+    memset(&fh, 0, sizeof(struct file_handle));
+    ret = name_to_handle(data->mountfd, ".", &fh, &mnt_id, 0);
+    if (ret && errno == EOVERFLOW) {
+        data->handle_bytes = fh.handle_bytes;
+        ctx->private = data;
+        ret = 0;
+        goto out;
+    }
+    /* we got 0 byte handle ? */
+    ret = -1;
+    close(data->mountfd);
+err_out:
+    g_free(data);
+out:
+    return ret;
+}
+
+static int handle_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
+{
+    const char *sec_model = qemu_opt_get(opts, "security_model");
+    const char *path = qemu_opt_get(opts, "path");
+
+    if (sec_model) {
+        error_report("Invalid argument security_model specified with handle fsdriver");
+        return -1;
+    }
+
+    if (!path) {
+        error_report("fsdev: No path specified");
+        return -1;
+    }
+    fse->path = g_strdup(path);
+    return 0;
+
+}
+
+FileOperations handle_ops = {
+    .parse_opts   = handle_parse_opts,
+    .init         = handle_init,
+    .lstat        = handle_lstat,
+    .readlink     = handle_readlink,
+    .close        = handle_close,
+    .closedir     = handle_closedir,
+    .open         = handle_open,
+    .opendir      = handle_opendir,
+    .rewinddir    = handle_rewinddir,
+    .telldir      = handle_telldir,
+    .readdir      = handle_readdir,
+    .seekdir      = handle_seekdir,
+    .preadv       = handle_preadv,
+    .pwritev      = handle_pwritev,
+    .chmod        = handle_chmod,
+    .mknod        = handle_mknod,
+    .mkdir        = handle_mkdir,
+    .fstat        = handle_fstat,
+    .open2        = handle_open2,
+    .symlink      = handle_symlink,
+    .link         = handle_link,
+    .truncate     = handle_truncate,
+    .rename       = handle_rename,
+    .chown        = handle_chown,
+    .utimensat    = handle_utimensat,
+    .remove       = handle_remove,
+    .fsync        = handle_fsync,
+    .statfs       = handle_statfs,
+    .lgetxattr    = handle_lgetxattr,
+    .llistxattr   = handle_llistxattr,
+    .lsetxattr    = handle_lsetxattr,
+    .lremovexattr = handle_lremovexattr,
+    .name_to_path = handle_name_to_path,
+    .renameat     = handle_renameat,
+    .unlinkat     = handle_unlinkat,
+};
diff --git a/hw/9pfs/9p-local.c b/hw/9pfs/9p-local.c
new file mode 100644
index 0000000..9fe224e
--- /dev/null
+++ b/hw/9pfs/9p-local.c
@@ -0,0 +1,1284 @@
+/*
+ * 9p Posix callback
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "9p-xattr.h"
+#include "fsdev/qemu-fsdev.h"   /* local_ops */
+#include <arpa/inet.h>
+#include <pwd.h>
+#include <grp.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include "qemu/xattr.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
+#include <libgen.h>
+#include <linux/fs.h>
+#ifdef CONFIG_LINUX_MAGIC_H
+#include <linux/magic.h>
+#endif
+#include <sys/ioctl.h>
+
+#ifndef XFS_SUPER_MAGIC
+#define XFS_SUPER_MAGIC  0x58465342
+#endif
+#ifndef EXT2_SUPER_MAGIC
+#define EXT2_SUPER_MAGIC 0xEF53
+#endif
+#ifndef REISERFS_SUPER_MAGIC
+#define REISERFS_SUPER_MAGIC 0x52654973
+#endif
+#ifndef BTRFS_SUPER_MAGIC
+#define BTRFS_SUPER_MAGIC 0x9123683E
+#endif
+
+#define VIRTFS_META_DIR ".virtfs_metadata"
+
+static char *local_mapped_attr_path(FsContext *ctx, const char *path)
+{
+    int dirlen;
+    const char *name = strrchr(path, '/');
+    if (name) {
+        dirlen = name - path;
+        ++name;
+    } else {
+        name = path;
+        dirlen = 0;
+    }
+    return g_strdup_printf("%s/%.*s/%s/%s", ctx->fs_root,
+                           dirlen, path, VIRTFS_META_DIR, name);
+}
+
+static FILE *local_fopen(const char *path, const char *mode)
+{
+    int fd, o_mode = 0;
+    FILE *fp;
+    int flags = O_NOFOLLOW;
+    /*
+     * only supports two modes
+     */
+    if (mode[0] == 'r') {
+        flags |= O_RDONLY;
+    } else if (mode[0] == 'w') {
+        flags |= O_WRONLY | O_TRUNC | O_CREAT;
+        o_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
+    } else {
+        return NULL;
+    }
+    fd = open(path, flags, o_mode);
+    if (fd == -1) {
+        return NULL;
+    }
+    fp = fdopen(fd, mode);
+    if (!fp) {
+        close(fd);
+    }
+    return fp;
+}
+
+#define ATTR_MAX 100
+static void local_mapped_file_attr(FsContext *ctx, const char *path,
+                                   struct stat *stbuf)
+{
+    FILE *fp;
+    char buf[ATTR_MAX];
+    char *attr_path;
+
+    attr_path = local_mapped_attr_path(ctx, path);
+    fp = local_fopen(attr_path, "r");
+    g_free(attr_path);
+    if (!fp) {
+        return;
+    }
+    memset(buf, 0, ATTR_MAX);
+    while (fgets(buf, ATTR_MAX, fp)) {
+        if (!strncmp(buf, "virtfs.uid", 10)) {
+            stbuf->st_uid = atoi(buf+11);
+        } else if (!strncmp(buf, "virtfs.gid", 10)) {
+            stbuf->st_gid = atoi(buf+11);
+        } else if (!strncmp(buf, "virtfs.mode", 11)) {
+            stbuf->st_mode = atoi(buf+12);
+        } else if (!strncmp(buf, "virtfs.rdev", 11)) {
+            stbuf->st_rdev = atoi(buf+12);
+        }
+        memset(buf, 0, ATTR_MAX);
+    }
+    fclose(fp);
+}
+
+static int local_lstat(FsContext *fs_ctx, V9fsPath *fs_path, struct stat *stbuf)
+{
+    int err;
+    char *buffer;
+    char *path = fs_path->data;
+
+    buffer = rpath(fs_ctx, path);
+    err =  qemu_lstat(buffer, stbuf);
+    if (err) {
+        goto err_out;
+    }
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        /* Actual credentials are part of extended attrs */
+        uid_t tmp_uid;
+        gid_t tmp_gid;
+        mode_t tmp_mode;
+        dev_t tmp_dev;
+        if (getxattr(buffer, "user.virtfs.uid", &tmp_uid, sizeof(uid_t)) > 0) {
+            stbuf->st_uid = le32_to_cpu(tmp_uid);
+        }
+        if (getxattr(buffer, "user.virtfs.gid", &tmp_gid, sizeof(gid_t)) > 0) {
+            stbuf->st_gid = le32_to_cpu(tmp_gid);
+        }
+        if (getxattr(buffer, "user.virtfs.mode",
+                    &tmp_mode, sizeof(mode_t)) > 0) {
+            stbuf->st_mode = le32_to_cpu(tmp_mode);
+        }
+        if (getxattr(buffer, "user.virtfs.rdev", &tmp_dev, sizeof(dev_t)) > 0) {
+            stbuf->st_rdev = le64_to_cpu(tmp_dev);
+        }
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        local_mapped_file_attr(fs_ctx, path, stbuf);
+    }
+
+err_out:
+    g_free(buffer);
+    return err;
+}
+
+static int local_create_mapped_attr_dir(FsContext *ctx, const char *path)
+{
+    int err;
+    char *attr_dir;
+    char *tmp_path = g_strdup(path);
+
+    attr_dir = g_strdup_printf("%s/%s/%s",
+             ctx->fs_root, dirname(tmp_path), VIRTFS_META_DIR);
+
+    err = mkdir(attr_dir, 0700);
+    if (err < 0 && errno == EEXIST) {
+        err = 0;
+    }
+    g_free(attr_dir);
+    g_free(tmp_path);
+    return err;
+}
+
+static int local_set_mapped_file_attr(FsContext *ctx,
+                                      const char *path, FsCred *credp)
+{
+    FILE *fp;
+    int ret = 0;
+    char buf[ATTR_MAX];
+    char *attr_path;
+    int uid = -1, gid = -1, mode = -1, rdev = -1;
+
+    attr_path = local_mapped_attr_path(ctx, path);
+    fp = local_fopen(attr_path, "r");
+    if (!fp) {
+        goto create_map_file;
+    }
+    memset(buf, 0, ATTR_MAX);
+    while (fgets(buf, ATTR_MAX, fp)) {
+        if (!strncmp(buf, "virtfs.uid", 10)) {
+            uid = atoi(buf+11);
+        } else if (!strncmp(buf, "virtfs.gid", 10)) {
+            gid = atoi(buf+11);
+        } else if (!strncmp(buf, "virtfs.mode", 11)) {
+            mode = atoi(buf+12);
+        } else if (!strncmp(buf, "virtfs.rdev", 11)) {
+            rdev = atoi(buf+12);
+        }
+        memset(buf, 0, ATTR_MAX);
+    }
+    fclose(fp);
+    goto update_map_file;
+
+create_map_file:
+    ret = local_create_mapped_attr_dir(ctx, path);
+    if (ret < 0) {
+        goto err_out;
+    }
+
+update_map_file:
+    fp = local_fopen(attr_path, "w");
+    if (!fp) {
+        ret = -1;
+        goto err_out;
+    }
+
+    if (credp->fc_uid != -1) {
+        uid = credp->fc_uid;
+    }
+    if (credp->fc_gid != -1) {
+        gid = credp->fc_gid;
+    }
+    if (credp->fc_mode != -1) {
+        mode = credp->fc_mode;
+    }
+    if (credp->fc_rdev != -1) {
+        rdev = credp->fc_rdev;
+    }
+
+
+    if (uid != -1) {
+        fprintf(fp, "virtfs.uid=%d\n", uid);
+    }
+    if (gid != -1) {
+        fprintf(fp, "virtfs.gid=%d\n", gid);
+    }
+    if (mode != -1) {
+        fprintf(fp, "virtfs.mode=%d\n", mode);
+    }
+    if (rdev != -1) {
+        fprintf(fp, "virtfs.rdev=%d\n", rdev);
+    }
+    fclose(fp);
+
+err_out:
+    g_free(attr_path);
+    return ret;
+}
+
+static int local_set_xattr(const char *path, FsCred *credp)
+{
+    int err;
+
+    if (credp->fc_uid != -1) {
+        uint32_t tmp_uid = cpu_to_le32(credp->fc_uid);
+        err = setxattr(path, "user.virtfs.uid", &tmp_uid, sizeof(uid_t), 0);
+        if (err) {
+            return err;
+        }
+    }
+    if (credp->fc_gid != -1) {
+        uint32_t tmp_gid = cpu_to_le32(credp->fc_gid);
+        err = setxattr(path, "user.virtfs.gid", &tmp_gid, sizeof(gid_t), 0);
+        if (err) {
+            return err;
+        }
+    }
+    if (credp->fc_mode != -1) {
+        uint32_t tmp_mode = cpu_to_le32(credp->fc_mode);
+        err = setxattr(path, "user.virtfs.mode", &tmp_mode, sizeof(mode_t), 0);
+        if (err) {
+            return err;
+        }
+    }
+    if (credp->fc_rdev != -1) {
+        uint64_t tmp_rdev = cpu_to_le64(credp->fc_rdev);
+        err = setxattr(path, "user.virtfs.rdev", &tmp_rdev, sizeof(dev_t), 0);
+        if (err) {
+            return err;
+        }
+    }
+    return 0;
+}
+
+static int local_post_create_passthrough(FsContext *fs_ctx, const char *path,
+                                         FsCred *credp)
+{
+    char *buffer;
+
+    buffer = rpath(fs_ctx, path);
+    if (lchown(buffer, credp->fc_uid, credp->fc_gid) < 0) {
+        /*
+         * If we fail to change ownership and if we are
+         * using security model none. Ignore the error
+         */
+        if ((fs_ctx->export_flags & V9FS_SEC_MASK) != V9FS_SM_NONE) {
+            goto err;
+        }
+    }
+
+    if (chmod(buffer, credp->fc_mode & 07777) < 0) {
+        goto err;
+    }
+
+    g_free(buffer);
+    return 0;
+err:
+    g_free(buffer);
+    return -1;
+}
+
+static ssize_t local_readlink(FsContext *fs_ctx, V9fsPath *fs_path,
+                              char *buf, size_t bufsz)
+{
+    ssize_t tsize = -1;
+    char *buffer;
+    char *path = fs_path->data;
+
+    if ((fs_ctx->export_flags & V9FS_SM_MAPPED) ||
+        (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE)) {
+        int fd;
+        buffer = rpath(fs_ctx, path);
+        fd = open(buffer, O_RDONLY | O_NOFOLLOW);
+        g_free(buffer);
+        if (fd == -1) {
+            return -1;
+        }
+        do {
+            tsize = read(fd, (void *)buf, bufsz);
+        } while (tsize == -1 && errno == EINTR);
+        close(fd);
+    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+               (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, path);
+        tsize = readlink(buffer, buf, bufsz);
+        g_free(buffer);
+    }
+    return tsize;
+}
+
+static int local_close(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return close(fs->fd);
+}
+
+static int local_closedir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return closedir(fs->dir.stream);
+}
+
+static int local_open(FsContext *ctx, V9fsPath *fs_path,
+                      int flags, V9fsFidOpenState *fs)
+{
+    char *buffer;
+    char *path = fs_path->data;
+
+    buffer = rpath(ctx, path);
+    fs->fd = open(buffer, flags | O_NOFOLLOW);
+    g_free(buffer);
+    return fs->fd;
+}
+
+static int local_opendir(FsContext *ctx,
+                         V9fsPath *fs_path, V9fsFidOpenState *fs)
+{
+    char *buffer;
+    char *path = fs_path->data;
+
+    buffer = rpath(ctx, path);
+    fs->dir.stream = opendir(buffer);
+    g_free(buffer);
+    if (!fs->dir.stream) {
+        return -1;
+    }
+    return 0;
+}
+
+static void local_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    rewinddir(fs->dir.stream);
+}
+
+static off_t local_telldir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return telldir(fs->dir.stream);
+}
+
+static struct dirent *local_readdir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    struct dirent *entry;
+
+again:
+    entry = readdir(fs->dir.stream);
+    if (!entry) {
+        return NULL;
+    }
+
+    if (ctx->export_flags & V9FS_SM_MAPPED) {
+        entry->d_type = DT_UNKNOWN;
+    } else if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        if (!strcmp(entry->d_name, VIRTFS_META_DIR)) {
+            /* skp the meta data directory */
+            goto again;
+        }
+        entry->d_type = DT_UNKNOWN;
+    }
+
+    return entry;
+}
+
+static void local_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
+{
+    seekdir(fs->dir.stream, off);
+}
+
+static ssize_t local_preadv(FsContext *ctx, V9fsFidOpenState *fs,
+                            const struct iovec *iov,
+                            int iovcnt, off_t offset)
+{
+#ifdef CONFIG_PREADV
+    return preadv(fs->fd, iov, iovcnt, offset);
+#else
+    int err = lseek(fs->fd, offset, SEEK_SET);
+    if (err == -1) {
+        return err;
+    } else {
+        return readv(fs->fd, iov, iovcnt);
+    }
+#endif
+}
+
+static ssize_t local_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
+                             const struct iovec *iov,
+                             int iovcnt, off_t offset)
+{
+    ssize_t ret
+;
+#ifdef CONFIG_PREADV
+    ret = pwritev(fs->fd, iov, iovcnt, offset);
+#else
+    int err = lseek(fs->fd, offset, SEEK_SET);
+    if (err == -1) {
+        return err;
+    } else {
+        ret = writev(fs->fd, iov, iovcnt);
+    }
+#endif
+#ifdef CONFIG_SYNC_FILE_RANGE
+    if (ret > 0 && ctx->export_flags & V9FS_IMMEDIATE_WRITEOUT) {
+        /*
+         * Initiate a writeback. This is not a data integrity sync.
+         * We want to ensure that we don't leave dirty pages in the cache
+         * after write when writeout=immediate is sepcified.
+         */
+        sync_file_range(fs->fd, offset, ret,
+                        SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
+    }
+#endif
+    return ret;
+}
+
+static int local_chmod(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
+{
+    char *buffer;
+    int ret = -1;
+    char *path = fs_path->data;
+
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        buffer = rpath(fs_ctx, path);
+        ret = local_set_xattr(buffer, credp);
+        g_free(buffer);
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        return local_set_mapped_file_attr(fs_ctx, path, credp);
+    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+               (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, path);
+        ret = chmod(buffer, credp->fc_mode);
+        g_free(buffer);
+    }
+    return ret;
+}
+
+static int local_mknod(FsContext *fs_ctx, V9fsPath *dir_path,
+                       const char *name, FsCred *credp)
+{
+    char *path;
+    int err = -1;
+    int serrno = 0;
+    V9fsString fullname;
+    char *buffer = NULL;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+    path = fullname.data;
+
+    /* Determine the security model */
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        buffer = rpath(fs_ctx, path);
+        err = mknod(buffer, SM_LOCAL_MODE_BITS|S_IFREG, 0);
+        if (err == -1) {
+            goto out;
+        }
+        err = local_set_xattr(buffer, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+
+        buffer = rpath(fs_ctx, path);
+        err = mknod(buffer, SM_LOCAL_MODE_BITS|S_IFREG, 0);
+        if (err == -1) {
+            goto out;
+        }
+        err = local_set_mapped_file_attr(fs_ctx, path, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+               (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, path);
+        err = mknod(buffer, credp->fc_mode, credp->fc_rdev);
+        if (err == -1) {
+            goto out;
+        }
+        err = local_post_create_passthrough(fs_ctx, path, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    }
+    goto out;
+
+err_end:
+    remove(buffer);
+    errno = serrno;
+out:
+    g_free(buffer);
+    v9fs_string_free(&fullname);
+    return err;
+}
+
+static int local_mkdir(FsContext *fs_ctx, V9fsPath *dir_path,
+                       const char *name, FsCred *credp)
+{
+    char *path;
+    int err = -1;
+    int serrno = 0;
+    V9fsString fullname;
+    char *buffer = NULL;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+    path = fullname.data;
+
+    /* Determine the security model */
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        buffer = rpath(fs_ctx, path);
+        err = mkdir(buffer, SM_LOCAL_DIR_MODE_BITS);
+        if (err == -1) {
+            goto out;
+        }
+        credp->fc_mode = credp->fc_mode|S_IFDIR;
+        err = local_set_xattr(buffer, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        buffer = rpath(fs_ctx, path);
+        err = mkdir(buffer, SM_LOCAL_DIR_MODE_BITS);
+        if (err == -1) {
+            goto out;
+        }
+        credp->fc_mode = credp->fc_mode|S_IFDIR;
+        err = local_set_mapped_file_attr(fs_ctx, path, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+               (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, path);
+        err = mkdir(buffer, credp->fc_mode);
+        if (err == -1) {
+            goto out;
+        }
+        err = local_post_create_passthrough(fs_ctx, path, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    }
+    goto out;
+
+err_end:
+    remove(buffer);
+    errno = serrno;
+out:
+    g_free(buffer);
+    v9fs_string_free(&fullname);
+    return err;
+}
+
+static int local_fstat(FsContext *fs_ctx, int fid_type,
+                       V9fsFidOpenState *fs, struct stat *stbuf)
+{
+    int err, fd;
+
+    if (fid_type == P9_FID_DIR) {
+        fd = dirfd(fs->dir.stream);
+    } else {
+        fd = fs->fd;
+    }
+
+    err = fstat(fd, stbuf);
+    if (err) {
+        return err;
+    }
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        /* Actual credentials are part of extended attrs */
+        uid_t tmp_uid;
+        gid_t tmp_gid;
+        mode_t tmp_mode;
+        dev_t tmp_dev;
+
+        if (fgetxattr(fd, "user.virtfs.uid", &tmp_uid, sizeof(uid_t)) > 0) {
+            stbuf->st_uid = le32_to_cpu(tmp_uid);
+        }
+        if (fgetxattr(fd, "user.virtfs.gid", &tmp_gid, sizeof(gid_t)) > 0) {
+            stbuf->st_gid = le32_to_cpu(tmp_gid);
+        }
+        if (fgetxattr(fd, "user.virtfs.mode", &tmp_mode, sizeof(mode_t)) > 0) {
+            stbuf->st_mode = le32_to_cpu(tmp_mode);
+        }
+        if (fgetxattr(fd, "user.virtfs.rdev", &tmp_dev, sizeof(dev_t)) > 0) {
+            stbuf->st_rdev = le64_to_cpu(tmp_dev);
+        }
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        errno = EOPNOTSUPP;
+        return -1;
+    }
+    return err;
+}
+
+static int local_open2(FsContext *fs_ctx, V9fsPath *dir_path, const char *name,
+                       int flags, FsCred *credp, V9fsFidOpenState *fs)
+{
+    char *path;
+    int fd = -1;
+    int err = -1;
+    int serrno = 0;
+    V9fsString fullname;
+    char *buffer = NULL;
+
+    /*
+     * Mark all the open to not follow symlinks
+     */
+    flags |= O_NOFOLLOW;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+    path = fullname.data;
+
+    /* Determine the security model */
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        buffer = rpath(fs_ctx, path);
+        fd = open(buffer, flags, SM_LOCAL_MODE_BITS);
+        if (fd == -1) {
+            err = fd;
+            goto out;
+        }
+        credp->fc_mode = credp->fc_mode|S_IFREG;
+        /* Set cleint credentials in xattr */
+        err = local_set_xattr(buffer, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        buffer = rpath(fs_ctx, path);
+        fd = open(buffer, flags, SM_LOCAL_MODE_BITS);
+        if (fd == -1) {
+            err = fd;
+            goto out;
+        }
+        credp->fc_mode = credp->fc_mode|S_IFREG;
+        /* Set client credentials in .virtfs_metadata directory files */
+        err = local_set_mapped_file_attr(fs_ctx, path, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+               (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, path);
+        fd = open(buffer, flags, credp->fc_mode);
+        if (fd == -1) {
+            err = fd;
+            goto out;
+        }
+        err = local_post_create_passthrough(fs_ctx, path, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    }
+    err = fd;
+    fs->fd = fd;
+    goto out;
+
+err_end:
+    close(fd);
+    remove(buffer);
+    errno = serrno;
+out:
+    g_free(buffer);
+    v9fs_string_free(&fullname);
+    return err;
+}
+
+
+static int local_symlink(FsContext *fs_ctx, const char *oldpath,
+                         V9fsPath *dir_path, const char *name, FsCred *credp)
+{
+    int err = -1;
+    int serrno = 0;
+    char *newpath;
+    V9fsString fullname;
+    char *buffer = NULL;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+    newpath = fullname.data;
+
+    /* Determine the security model */
+    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        int fd;
+        ssize_t oldpath_size, write_size;
+        buffer = rpath(fs_ctx, newpath);
+        fd = open(buffer, O_CREAT|O_EXCL|O_RDWR|O_NOFOLLOW, SM_LOCAL_MODE_BITS);
+        if (fd == -1) {
+            err = fd;
+            goto out;
+        }
+        /* Write the oldpath (target) to the file. */
+        oldpath_size = strlen(oldpath);
+        do {
+            write_size = write(fd, (void *)oldpath, oldpath_size);
+        } while (write_size == -1 && errno == EINTR);
+
+        if (write_size != oldpath_size) {
+            serrno = errno;
+            close(fd);
+            err = -1;
+            goto err_end;
+        }
+        close(fd);
+        /* Set cleint credentials in symlink's xattr */
+        credp->fc_mode = credp->fc_mode|S_IFLNK;
+        err = local_set_xattr(buffer, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        int fd;
+        ssize_t oldpath_size, write_size;
+        buffer = rpath(fs_ctx, newpath);
+        fd = open(buffer, O_CREAT|O_EXCL|O_RDWR|O_NOFOLLOW, SM_LOCAL_MODE_BITS);
+        if (fd == -1) {
+            err = fd;
+            goto out;
+        }
+        /* Write the oldpath (target) to the file. */
+        oldpath_size = strlen(oldpath);
+        do {
+            write_size = write(fd, (void *)oldpath, oldpath_size);
+        } while (write_size == -1 && errno == EINTR);
+
+        if (write_size != oldpath_size) {
+            serrno = errno;
+            close(fd);
+            err = -1;
+            goto err_end;
+        }
+        close(fd);
+        /* Set cleint credentials in symlink's xattr */
+        credp->fc_mode = credp->fc_mode|S_IFLNK;
+        err = local_set_mapped_file_attr(fs_ctx, newpath, credp);
+        if (err == -1) {
+            serrno = errno;
+            goto err_end;
+        }
+    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+               (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, newpath);
+        err = symlink(oldpath, buffer);
+        if (err) {
+            goto out;
+        }
+        err = lchown(buffer, credp->fc_uid, credp->fc_gid);
+        if (err == -1) {
+            /*
+             * If we fail to change ownership and if we are
+             * using security model none. Ignore the error
+             */
+            if ((fs_ctx->export_flags & V9FS_SEC_MASK) != V9FS_SM_NONE) {
+                serrno = errno;
+                goto err_end;
+            } else
+                err = 0;
+        }
+    }
+    goto out;
+
+err_end:
+    remove(buffer);
+    errno = serrno;
+out:
+    g_free(buffer);
+    v9fs_string_free(&fullname);
+    return err;
+}
+
+static int local_link(FsContext *ctx, V9fsPath *oldpath,
+                      V9fsPath *dirpath, const char *name)
+{
+    int ret;
+    V9fsString newpath;
+    char *buffer, *buffer1;
+
+    v9fs_string_init(&newpath);
+    v9fs_string_sprintf(&newpath, "%s/%s", dirpath->data, name);
+
+    buffer = rpath(ctx, oldpath->data);
+    buffer1 = rpath(ctx, newpath.data);
+    ret = link(buffer, buffer1);
+    g_free(buffer);
+    g_free(buffer1);
+
+    /* now link the virtfs_metadata files */
+    if (!ret && (ctx->export_flags & V9FS_SM_MAPPED_FILE)) {
+        /* Link the .virtfs_metadata files. Create the metada directory */
+        ret = local_create_mapped_attr_dir(ctx, newpath.data);
+        if (ret < 0) {
+            goto err_out;
+        }
+        buffer = local_mapped_attr_path(ctx, oldpath->data);
+        buffer1 = local_mapped_attr_path(ctx, newpath.data);
+        ret = link(buffer, buffer1);
+        g_free(buffer);
+        g_free(buffer1);
+        if (ret < 0 && errno != ENOENT) {
+            goto err_out;
+        }
+    }
+err_out:
+    v9fs_string_free(&newpath);
+    return ret;
+}
+
+static int local_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size)
+{
+    char *buffer;
+    int ret;
+    char *path = fs_path->data;
+
+    buffer = rpath(ctx, path);
+    ret = truncate(buffer, size);
+    g_free(buffer);
+    return ret;
+}
+
+static int local_rename(FsContext *ctx, const char *oldpath,
+                        const char *newpath)
+{
+    int err;
+    char *buffer, *buffer1;
+
+    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        err = local_create_mapped_attr_dir(ctx, newpath);
+        if (err < 0) {
+            return err;
+        }
+        /* rename the .virtfs_metadata files */
+        buffer = local_mapped_attr_path(ctx, oldpath);
+        buffer1 = local_mapped_attr_path(ctx, newpath);
+        err = rename(buffer, buffer1);
+        g_free(buffer);
+        g_free(buffer1);
+        if (err < 0 && errno != ENOENT) {
+            return err;
+        }
+    }
+
+    buffer = rpath(ctx, oldpath);
+    buffer1 = rpath(ctx, newpath);
+    err = rename(buffer, buffer1);
+    g_free(buffer);
+    g_free(buffer1);
+    return err;
+}
+
+static int local_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
+{
+    char *buffer;
+    int ret = -1;
+    char *path = fs_path->data;
+
+    if ((credp->fc_uid == -1 && credp->fc_gid == -1) ||
+        (fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
+        (fs_ctx->export_flags & V9FS_SM_NONE)) {
+        buffer = rpath(fs_ctx, path);
+        ret = lchown(buffer, credp->fc_uid, credp->fc_gid);
+        g_free(buffer);
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
+        buffer = rpath(fs_ctx, path);
+        ret = local_set_xattr(buffer, credp);
+        g_free(buffer);
+    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        return local_set_mapped_file_attr(fs_ctx, path, credp);
+    }
+    return ret;
+}
+
+static int local_utimensat(FsContext *s, V9fsPath *fs_path,
+                           const struct timespec *buf)
+{
+    char *buffer;
+    int ret;
+    char *path = fs_path->data;
+
+    buffer = rpath(s, path);
+    ret = qemu_utimens(buffer, buf);
+    g_free(buffer);
+    return ret;
+}
+
+static int local_remove(FsContext *ctx, const char *path)
+{
+    int err;
+    struct stat stbuf;
+    char *buffer;
+
+    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        buffer = rpath(ctx, path);
+        err =  qemu_lstat(buffer, &stbuf);
+        g_free(buffer);
+        if (err) {
+            goto err_out;
+        }
+        /*
+         * If directory remove .virtfs_metadata contained in the
+         * directory
+         */
+        if (S_ISDIR(stbuf.st_mode)) {
+            buffer = g_strdup_printf("%s/%s/%s", ctx->fs_root,
+                                     path, VIRTFS_META_DIR);
+            err = remove(buffer);
+            g_free(buffer);
+            if (err < 0 && errno != ENOENT) {
+                /*
+                 * We didn't had the .virtfs_metadata file. May be file created
+                 * in non-mapped mode ?. Ignore ENOENT.
+                 */
+                goto err_out;
+            }
+        }
+        /*
+         * Now remove the name from parent directory
+         * .virtfs_metadata directory
+         */
+        buffer = local_mapped_attr_path(ctx, path);
+        err = remove(buffer);
+        g_free(buffer);
+        if (err < 0 && errno != ENOENT) {
+            /*
+             * We didn't had the .virtfs_metadata file. May be file created
+             * in non-mapped mode ?. Ignore ENOENT.
+             */
+            goto err_out;
+        }
+    }
+
+    buffer = rpath(ctx, path);
+    err = remove(buffer);
+    g_free(buffer);
+err_out:
+    return err;
+}
+
+static int local_fsync(FsContext *ctx, int fid_type,
+                       V9fsFidOpenState *fs, int datasync)
+{
+    int fd;
+
+    if (fid_type == P9_FID_DIR) {
+        fd = dirfd(fs->dir.stream);
+    } else {
+        fd = fs->fd;
+    }
+
+    if (datasync) {
+        return qemu_fdatasync(fd);
+    } else {
+        return fsync(fd);
+    }
+}
+
+static int local_statfs(FsContext *s, V9fsPath *fs_path, struct statfs *stbuf)
+{
+    char *buffer;
+    int ret;
+    char *path = fs_path->data;
+
+    buffer = rpath(s, path);
+    ret = statfs(buffer, stbuf);
+    g_free(buffer);
+    return ret;
+}
+
+static ssize_t local_lgetxattr(FsContext *ctx, V9fsPath *fs_path,
+                               const char *name, void *value, size_t size)
+{
+    char *path = fs_path->data;
+
+    return v9fs_get_xattr(ctx, path, name, value, size);
+}
+
+static ssize_t local_llistxattr(FsContext *ctx, V9fsPath *fs_path,
+                                void *value, size_t size)
+{
+    char *path = fs_path->data;
+
+    return v9fs_list_xattr(ctx, path, value, size);
+}
+
+static int local_lsetxattr(FsContext *ctx, V9fsPath *fs_path, const char *name,
+                           void *value, size_t size, int flags)
+{
+    char *path = fs_path->data;
+
+    return v9fs_set_xattr(ctx, path, name, value, size, flags);
+}
+
+static int local_lremovexattr(FsContext *ctx, V9fsPath *fs_path,
+                              const char *name)
+{
+    char *path = fs_path->data;
+
+    return v9fs_remove_xattr(ctx, path, name);
+}
+
+static int local_name_to_path(FsContext *ctx, V9fsPath *dir_path,
+                              const char *name, V9fsPath *target)
+{
+    if (dir_path) {
+        v9fs_string_sprintf((V9fsString *)target, "%s/%s",
+                            dir_path->data, name);
+    } else {
+        v9fs_string_sprintf((V9fsString *)target, "%s", name);
+    }
+    /* Bump the size for including terminating NULL */
+    target->size++;
+    return 0;
+}
+
+static int local_renameat(FsContext *ctx, V9fsPath *olddir,
+                          const char *old_name, V9fsPath *newdir,
+                          const char *new_name)
+{
+    int ret;
+    V9fsString old_full_name, new_full_name;
+
+    v9fs_string_init(&old_full_name);
+    v9fs_string_init(&new_full_name);
+
+    v9fs_string_sprintf(&old_full_name, "%s/%s", olddir->data, old_name);
+    v9fs_string_sprintf(&new_full_name, "%s/%s", newdir->data, new_name);
+
+    ret = local_rename(ctx, old_full_name.data, new_full_name.data);
+    v9fs_string_free(&old_full_name);
+    v9fs_string_free(&new_full_name);
+    return ret;
+}
+
+static int local_unlinkat(FsContext *ctx, V9fsPath *dir,
+                          const char *name, int flags)
+{
+    int ret;
+    V9fsString fullname;
+    char *buffer;
+
+    v9fs_string_init(&fullname);
+
+    v9fs_string_sprintf(&fullname, "%s/%s", dir->data, name);
+    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        if (flags == AT_REMOVEDIR) {
+            /*
+             * If directory remove .virtfs_metadata contained in the
+             * directory
+             */
+            buffer = g_strdup_printf("%s/%s/%s", ctx->fs_root,
+                                     fullname.data, VIRTFS_META_DIR);
+            ret = remove(buffer);
+            g_free(buffer);
+            if (ret < 0 && errno != ENOENT) {
+                /*
+                 * We didn't had the .virtfs_metadata file. May be file created
+                 * in non-mapped mode ?. Ignore ENOENT.
+                 */
+                goto err_out;
+            }
+        }
+        /*
+         * Now remove the name from parent directory
+         * .virtfs_metadata directory.
+         */
+        buffer = local_mapped_attr_path(ctx, fullname.data);
+        ret = remove(buffer);
+        g_free(buffer);
+        if (ret < 0 && errno != ENOENT) {
+            /*
+             * We didn't had the .virtfs_metadata file. May be file created
+             * in non-mapped mode ?. Ignore ENOENT.
+             */
+            goto err_out;
+        }
+    }
+    /* Remove the name finally */
+    buffer = rpath(ctx, fullname.data);
+    ret = remove(buffer);
+    g_free(buffer);
+
+err_out:
+    v9fs_string_free(&fullname);
+    return ret;
+}
+
+static int local_ioc_getversion(FsContext *ctx, V9fsPath *path,
+                                mode_t st_mode, uint64_t *st_gen)
+{
+#ifdef FS_IOC_GETVERSION
+    int err;
+    V9fsFidOpenState fid_open;
+
+    /*
+     * Do not try to open special files like device nodes, fifos etc
+     * We can get fd for regular files and directories only
+     */
+    if (!S_ISREG(st_mode) && !S_ISDIR(st_mode)) {
+        errno = ENOTTY;
+        return -1;
+    }
+    err = local_open(ctx, path, O_RDONLY, &fid_open);
+    if (err < 0) {
+        return err;
+    }
+    err = ioctl(fid_open.fd, FS_IOC_GETVERSION, st_gen);
+    local_close(ctx, &fid_open);
+    return err;
+#else
+    errno = ENOTTY;
+    return -1;
+#endif
+}
+
+static int local_init(FsContext *ctx)
+{
+    int err = 0;
+    struct statfs stbuf;
+
+    if (ctx->export_flags & V9FS_SM_PASSTHROUGH) {
+        ctx->xops = passthrough_xattr_ops;
+    } else if (ctx->export_flags & V9FS_SM_MAPPED) {
+        ctx->xops = mapped_xattr_ops;
+    } else if (ctx->export_flags & V9FS_SM_NONE) {
+        ctx->xops = none_xattr_ops;
+    } else if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
+        /*
+         * xattr operation for mapped-file and passthrough
+         * remain same.
+         */
+        ctx->xops = passthrough_xattr_ops;
+    }
+    ctx->export_flags |= V9FS_PATHNAME_FSCONTEXT;
+#ifdef FS_IOC_GETVERSION
+    /*
+     * use ioc_getversion only if the iocl is definied
+     */
+    err = statfs(ctx->fs_root, &stbuf);
+    if (!err) {
+        switch (stbuf.f_type) {
+        case EXT2_SUPER_MAGIC:
+        case BTRFS_SUPER_MAGIC:
+        case REISERFS_SUPER_MAGIC:
+        case XFS_SUPER_MAGIC:
+            ctx->exops.get_st_gen = local_ioc_getversion;
+            break;
+        }
+    }
+#endif
+    return err;
+}
+
+static int local_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
+{
+    const char *sec_model = qemu_opt_get(opts, "security_model");
+    const char *path = qemu_opt_get(opts, "path");
+
+    if (!sec_model) {
+        error_report("Security model not specified, local fs needs security model");
+        error_printf("valid options are:"
+                     "\tsecurity_model=[passthrough|mapped-xattr|mapped-file|none]\n");
+        return -1;
+    }
+
+    if (!strcmp(sec_model, "passthrough")) {
+        fse->export_flags |= V9FS_SM_PASSTHROUGH;
+    } else if (!strcmp(sec_model, "mapped") ||
+               !strcmp(sec_model, "mapped-xattr")) {
+        fse->export_flags |= V9FS_SM_MAPPED;
+    } else if (!strcmp(sec_model, "none")) {
+        fse->export_flags |= V9FS_SM_NONE;
+    } else if (!strcmp(sec_model, "mapped-file")) {
+        fse->export_flags |= V9FS_SM_MAPPED_FILE;
+    } else {
+        error_report("Invalid security model %s specified", sec_model);
+        error_printf("valid options are:"
+                     "\t[passthrough|mapped-xattr|mapped-file|none]\n");
+        return -1;
+    }
+
+    if (!path) {
+        error_report("fsdev: No path specified");
+        return -1;
+    }
+    fse->path = g_strdup(path);
+
+    return 0;
+}
+
+FileOperations local_ops = {
+    .parse_opts = local_parse_opts,
+    .init  = local_init,
+    .lstat = local_lstat,
+    .readlink = local_readlink,
+    .close = local_close,
+    .closedir = local_closedir,
+    .open = local_open,
+    .opendir = local_opendir,
+    .rewinddir = local_rewinddir,
+    .telldir = local_telldir,
+    .readdir = local_readdir,
+    .seekdir = local_seekdir,
+    .preadv = local_preadv,
+    .pwritev = local_pwritev,
+    .chmod = local_chmod,
+    .mknod = local_mknod,
+    .mkdir = local_mkdir,
+    .fstat = local_fstat,
+    .open2 = local_open2,
+    .symlink = local_symlink,
+    .link = local_link,
+    .truncate = local_truncate,
+    .rename = local_rename,
+    .chown = local_chown,
+    .utimensat = local_utimensat,
+    .remove = local_remove,
+    .fsync = local_fsync,
+    .statfs = local_statfs,
+    .lgetxattr = local_lgetxattr,
+    .llistxattr = local_llistxattr,
+    .lsetxattr = local_lsetxattr,
+    .lremovexattr = local_lremovexattr,
+    .name_to_path = local_name_to_path,
+    .renameat  = local_renameat,
+    .unlinkat = local_unlinkat,
+};
diff --git a/hw/9pfs/9p-posix-acl.c b/hw/9pfs/9p-posix-acl.c
new file mode 100644
index 0000000..ec00318
--- /dev/null
+++ b/hw/9pfs/9p-posix-acl.c
@@ -0,0 +1,184 @@
+/*
+ * 9p system.posix* xattr callback
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/xattr.h"
+#include "9p.h"
+#include "fsdev/file-op-9p.h"
+#include "9p-xattr.h"
+
+#define MAP_ACL_ACCESS "user.virtfs.system.posix_acl_access"
+#define MAP_ACL_DEFAULT "user.virtfs.system.posix_acl_default"
+#define ACL_ACCESS "system.posix_acl_access"
+#define ACL_DEFAULT "system.posix_acl_default"
+
+static ssize_t mp_pacl_getxattr(FsContext *ctx, const char *path,
+                                const char *name, void *value, size_t size)
+{
+    char *buffer;
+    ssize_t ret;
+
+    buffer = rpath(ctx, path);
+    ret = lgetxattr(buffer, MAP_ACL_ACCESS, value, size);
+    g_free(buffer);
+    return ret;
+}
+
+static ssize_t mp_pacl_listxattr(FsContext *ctx, const char *path,
+                                 char *name, void *value, size_t osize)
+{
+    ssize_t len = sizeof(ACL_ACCESS);
+
+    if (!value) {
+        return len;
+    }
+
+    if (osize < len) {
+        errno = ERANGE;
+        return -1;
+    }
+
+    /* len includes the trailing NUL */
+    memcpy(value, ACL_ACCESS, len);
+    return 0;
+}
+
+static int mp_pacl_setxattr(FsContext *ctx, const char *path, const char *name,
+                            void *value, size_t size, int flags)
+{
+    char *buffer;
+    int ret;
+
+    buffer = rpath(ctx, path);
+    ret = lsetxattr(buffer, MAP_ACL_ACCESS, value, size, flags);
+    g_free(buffer);
+    return ret;
+}
+
+static int mp_pacl_removexattr(FsContext *ctx,
+                               const char *path, const char *name)
+{
+    int ret;
+    char *buffer;
+
+    buffer = rpath(ctx, path);
+    ret  = lremovexattr(buffer, MAP_ACL_ACCESS);
+    if (ret == -1 && errno == ENODATA) {
+        /*
+         * We don't get ENODATA error when trying to remove a
+         * posix acl that is not present. So don't throw the error
+         * even in case of mapped security model
+         */
+        errno = 0;
+        ret = 0;
+    }
+    g_free(buffer);
+    return ret;
+}
+
+static ssize_t mp_dacl_getxattr(FsContext *ctx, const char *path,
+                                const char *name, void *value, size_t size)
+{
+    char *buffer;
+    ssize_t ret;
+
+    buffer = rpath(ctx, path);
+    ret = lgetxattr(buffer, MAP_ACL_DEFAULT, value, size);
+    g_free(buffer);
+    return ret;
+}
+
+static ssize_t mp_dacl_listxattr(FsContext *ctx, const char *path,
+                                 char *name, void *value, size_t osize)
+{
+    ssize_t len = sizeof(ACL_DEFAULT);
+
+    if (!value) {
+        return len;
+    }
+
+    if (osize < len) {
+        errno = ERANGE;
+        return -1;
+    }
+
+    /* len includes the trailing NUL */
+    memcpy(value, ACL_DEFAULT, len);
+    return 0;
+}
+
+static int mp_dacl_setxattr(FsContext *ctx, const char *path, const char *name,
+                            void *value, size_t size, int flags)
+{
+    char *buffer;
+    int ret;
+
+    buffer = rpath(ctx, path);
+    ret = lsetxattr(buffer, MAP_ACL_DEFAULT, value, size, flags);
+    g_free(buffer);
+    return ret;
+}
+
+static int mp_dacl_removexattr(FsContext *ctx,
+                               const char *path, const char *name)
+{
+    int ret;
+    char *buffer;
+
+    buffer = rpath(ctx, path);
+    ret  = lremovexattr(buffer, MAP_ACL_DEFAULT);
+    if (ret == -1 && errno == ENODATA) {
+        /*
+         * We don't get ENODATA error when trying to remove a
+         * posix acl that is not present. So don't throw the error
+         * even in case of mapped security model
+         */
+        errno = 0;
+        ret = 0;
+    }
+    g_free(buffer);
+    return ret;
+}
+
+
+XattrOperations mapped_pacl_xattr = {
+    .name = "system.posix_acl_access",
+    .getxattr = mp_pacl_getxattr,
+    .setxattr = mp_pacl_setxattr,
+    .listxattr = mp_pacl_listxattr,
+    .removexattr = mp_pacl_removexattr,
+};
+
+XattrOperations mapped_dacl_xattr = {
+    .name = "system.posix_acl_default",
+    .getxattr = mp_dacl_getxattr,
+    .setxattr = mp_dacl_setxattr,
+    .listxattr = mp_dacl_listxattr,
+    .removexattr = mp_dacl_removexattr,
+};
+
+XattrOperations passthrough_acl_xattr = {
+    .name = "system.posix_acl_",
+    .getxattr = pt_getxattr,
+    .setxattr = pt_setxattr,
+    .listxattr = pt_listxattr,
+    .removexattr = pt_removexattr,
+};
+
+XattrOperations none_acl_xattr = {
+    .name = "system.posix_acl_",
+    .getxattr = notsup_getxattr,
+    .setxattr = notsup_setxattr,
+    .listxattr = notsup_listxattr,
+    .removexattr = notsup_removexattr,
+};
diff --git a/hw/9pfs/9p-proxy.c b/hw/9pfs/9p-proxy.c
new file mode 100644
index 0000000..f265501
--- /dev/null
+++ b/hw/9pfs/9p-proxy.c
@@ -0,0 +1,1218 @@
+/*
+ * 9p Proxy callback
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ * M. Mohan Kumar <mohan@in.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include <sys/socket.h>
+#include <sys/un.h>
+#include "9p.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
+#include "fsdev/qemu-fsdev.h"
+#include "9p-proxy.h"
+
+typedef struct V9fsProxy {
+    int sockfd;
+    QemuMutex mutex;
+    struct iovec in_iovec;
+    struct iovec out_iovec;
+} V9fsProxy;
+
+/*
+ * Return received file descriptor on success in *status.
+ * errno is also returned on *status (which will be < 0)
+ * return < 0 on transport error.
+ */
+static int v9fs_receivefd(int sockfd, int *status)
+{
+    struct iovec iov;
+    struct msghdr msg;
+    struct cmsghdr *cmsg;
+    int retval, data, fd;
+    union MsgControl msg_control;
+
+    iov.iov_base = &data;
+    iov.iov_len = sizeof(data);
+
+    memset(&msg, 0, sizeof(msg));
+    msg.msg_iov = &iov;
+    msg.msg_iovlen = 1;
+    msg.msg_control = &msg_control;
+    msg.msg_controllen = sizeof(msg_control);
+
+    do {
+        retval = recvmsg(sockfd, &msg, 0);
+    } while (retval < 0 && errno == EINTR);
+    if (retval <= 0) {
+        return retval;
+    }
+    /*
+     * data is set to V9FS_FD_VALID, if ancillary data is sent.  If this
+     * request doesn't need ancillary data (fd) or an error occurred,
+     * data is set to negative errno value.
+     */
+    if (data != V9FS_FD_VALID) {
+        *status = data;
+        return 0;
+    }
+    /*
+     * File descriptor (fd) is sent in the ancillary data. Check if we
+     * indeed received it. One of the reasons to fail to receive it is if
+     * we exceeded the maximum number of file descriptors!
+     */
+    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
+        if (cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
+            cmsg->cmsg_level != SOL_SOCKET ||
+            cmsg->cmsg_type != SCM_RIGHTS) {
+            continue;
+        }
+        fd = *((int *)CMSG_DATA(cmsg));
+        *status = fd;
+        return 0;
+    }
+    *status = -ENFILE;  /* Ancillary data sent but not received */
+    return 0;
+}
+
+static ssize_t socket_read(int sockfd, void *buff, size_t size)
+{
+    ssize_t retval, total = 0;
+
+    while (size) {
+        retval = read(sockfd, buff, size);
+        if (retval == 0) {
+            return -EIO;
+        }
+        if (retval < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+            return -errno;
+        }
+        size -= retval;
+        buff += retval;
+        total += retval;
+    }
+    return total;
+}
+
+/* Converts proxy_statfs to VFS statfs structure */
+static void prstatfs_to_statfs(struct statfs *stfs, ProxyStatFS *prstfs)
+{
+    memset(stfs, 0, sizeof(*stfs));
+    stfs->f_type = prstfs->f_type;
+    stfs->f_bsize = prstfs->f_bsize;
+    stfs->f_blocks = prstfs->f_blocks;
+    stfs->f_bfree = prstfs->f_bfree;
+    stfs->f_bavail = prstfs->f_bavail;
+    stfs->f_files = prstfs->f_files;
+    stfs->f_ffree = prstfs->f_ffree;
+    stfs->f_fsid.__val[0] = prstfs->f_fsid[0] & 0xFFFFFFFFU;
+    stfs->f_fsid.__val[1] = prstfs->f_fsid[1] >> 32 & 0xFFFFFFFFU;
+    stfs->f_namelen = prstfs->f_namelen;
+    stfs->f_frsize = prstfs->f_frsize;
+}
+
+/* Converts proxy_stat structure to VFS stat structure */
+static void prstat_to_stat(struct stat *stbuf, ProxyStat *prstat)
+{
+   memset(stbuf, 0, sizeof(*stbuf));
+   stbuf->st_dev = prstat->st_dev;
+   stbuf->st_ino = prstat->st_ino;
+   stbuf->st_nlink = prstat->st_nlink;
+   stbuf->st_mode = prstat->st_mode;
+   stbuf->st_uid = prstat->st_uid;
+   stbuf->st_gid = prstat->st_gid;
+   stbuf->st_rdev = prstat->st_rdev;
+   stbuf->st_size = prstat->st_size;
+   stbuf->st_blksize = prstat->st_blksize;
+   stbuf->st_blocks = prstat->st_blocks;
+   stbuf->st_atim.tv_sec = prstat->st_atim_sec;
+   stbuf->st_atim.tv_nsec = prstat->st_atim_nsec;
+   stbuf->st_mtime = prstat->st_mtim_sec;
+   stbuf->st_mtim.tv_nsec = prstat->st_mtim_nsec;
+   stbuf->st_ctime = prstat->st_ctim_sec;
+   stbuf->st_ctim.tv_nsec = prstat->st_ctim_nsec;
+}
+
+/*
+ * Response contains two parts
+ * {header, data}
+ * header.type == T_ERROR, data -> -errno
+ * header.type == T_SUCCESS, data -> response
+ * size of errno/response is given by header.size
+ * returns < 0, on transport error. response is
+ * valid only if status >= 0.
+ */
+static int v9fs_receive_response(V9fsProxy *proxy, int type,
+                                 int *status, void *response)
+{
+    int retval;
+    ProxyHeader header;
+    struct iovec *reply = &proxy->in_iovec;
+
+    *status = 0;
+    reply->iov_len = 0;
+    retval = socket_read(proxy->sockfd, reply->iov_base, PROXY_HDR_SZ);
+    if (retval < 0) {
+        return retval;
+    }
+    reply->iov_len = PROXY_HDR_SZ;
+    proxy_unmarshal(reply, 0, "dd", &header.type, &header.size);
+    /*
+     * if response size > PROXY_MAX_IO_SZ, read the response but ignore it and
+     * return -ENOBUFS
+     */
+    if (header.size > PROXY_MAX_IO_SZ) {
+        int count;
+        while (header.size > 0) {
+            count = MIN(PROXY_MAX_IO_SZ, header.size);
+            count = socket_read(proxy->sockfd, reply->iov_base, count);
+            if (count < 0) {
+                return count;
+            }
+            header.size -= count;
+        }
+        *status = -ENOBUFS;
+        return 0;
+    }
+
+    retval = socket_read(proxy->sockfd,
+                         reply->iov_base + PROXY_HDR_SZ, header.size);
+    if (retval < 0) {
+        return retval;
+    }
+    reply->iov_len += header.size;
+    /* there was an error during processing request */
+    if (header.type == T_ERROR) {
+        int ret;
+        ret = proxy_unmarshal(reply, PROXY_HDR_SZ, "d", status);
+        if (ret < 0) {
+            *status = ret;
+        }
+        return 0;
+    }
+
+    switch (type) {
+    case T_LSTAT: {
+        ProxyStat prstat;
+        retval = proxy_unmarshal(reply, PROXY_HDR_SZ,
+                                 "qqqdddqqqqqqqqqq", &prstat.st_dev,
+                                 &prstat.st_ino, &prstat.st_nlink,
+                                 &prstat.st_mode, &prstat.st_uid,
+                                 &prstat.st_gid, &prstat.st_rdev,
+                                 &prstat.st_size, &prstat.st_blksize,
+                                 &prstat.st_blocks,
+                                 &prstat.st_atim_sec, &prstat.st_atim_nsec,
+                                 &prstat.st_mtim_sec, &prstat.st_mtim_nsec,
+                                 &prstat.st_ctim_sec, &prstat.st_ctim_nsec);
+        prstat_to_stat(response, &prstat);
+        break;
+    }
+    case T_STATFS: {
+        ProxyStatFS prstfs;
+        retval = proxy_unmarshal(reply, PROXY_HDR_SZ,
+                                 "qqqqqqqqqqq", &prstfs.f_type,
+                                 &prstfs.f_bsize, &prstfs.f_blocks,
+                                 &prstfs.f_bfree, &prstfs.f_bavail,
+                                 &prstfs.f_files, &prstfs.f_ffree,
+                                 &prstfs.f_fsid[0], &prstfs.f_fsid[1],
+                                 &prstfs.f_namelen, &prstfs.f_frsize);
+        prstatfs_to_statfs(response, &prstfs);
+        break;
+    }
+    case T_READLINK: {
+        V9fsString target;
+        v9fs_string_init(&target);
+        retval = proxy_unmarshal(reply, PROXY_HDR_SZ, "s", &target);
+        strcpy(response, target.data);
+        v9fs_string_free(&target);
+        break;
+    }
+    case T_LGETXATTR:
+    case T_LLISTXATTR: {
+        V9fsString xattr;
+        v9fs_string_init(&xattr);
+        retval = proxy_unmarshal(reply, PROXY_HDR_SZ, "s", &xattr);
+        memcpy(response, xattr.data, xattr.size);
+        v9fs_string_free(&xattr);
+        break;
+    }
+    case T_GETVERSION:
+        proxy_unmarshal(reply, PROXY_HDR_SZ, "q", response);
+        break;
+    default:
+        return -1;
+    }
+    if (retval < 0) {
+        *status  = retval;
+    }
+    return 0;
+}
+
+/*
+ * return < 0 on transport error.
+ * *status is valid only if return >= 0
+ */
+static int v9fs_receive_status(V9fsProxy *proxy,
+                               struct iovec *reply, int *status)
+{
+    int retval;
+    ProxyHeader header;
+
+    *status = 0;
+    reply->iov_len = 0;
+    retval = socket_read(proxy->sockfd, reply->iov_base, PROXY_HDR_SZ);
+    if (retval < 0) {
+        return retval;
+    }
+    reply->iov_len = PROXY_HDR_SZ;
+    proxy_unmarshal(reply, 0, "dd", &header.type, &header.size);
+    if (header.size != sizeof(int)) {
+        *status = -ENOBUFS;
+        return 0;
+    }
+    retval = socket_read(proxy->sockfd,
+                         reply->iov_base + PROXY_HDR_SZ, header.size);
+    if (retval < 0) {
+        return retval;
+    }
+    reply->iov_len += header.size;
+    proxy_unmarshal(reply, PROXY_HDR_SZ, "d", status);
+    return 0;
+}
+
+/*
+ * Proxy->header and proxy->request written to socket by QEMU process.
+ * This request read by proxy helper process
+ * returns 0 on success and -errno on error
+ */
+static int v9fs_request(V9fsProxy *proxy, int type,
+                        void *response, const char *fmt, ...)
+{
+    dev_t rdev;
+    va_list ap;
+    int size = 0;
+    int retval = 0;
+    uint64_t offset;
+    ProxyHeader header = { 0, 0};
+    struct timespec spec[2];
+    int flags, mode, uid, gid;
+    V9fsString *name, *value;
+    V9fsString *path, *oldpath;
+    struct iovec *iovec = NULL, *reply = NULL;
+
+    qemu_mutex_lock(&proxy->mutex);
+
+    if (proxy->sockfd == -1) {
+        retval = -EIO;
+        goto err_out;
+    }
+    iovec = &proxy->out_iovec;
+    reply = &proxy->in_iovec;
+    va_start(ap, fmt);
+    switch (type) {
+    case T_OPEN:
+        path = va_arg(ap, V9fsString *);
+        flags = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sd", path, flags);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_OPEN;
+        }
+        break;
+    case T_CREATE:
+        path = va_arg(ap, V9fsString *);
+        flags = va_arg(ap, int);
+        mode = va_arg(ap, int);
+        uid = va_arg(ap, int);
+        gid = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sdddd", path,
+                                    flags, mode, uid, gid);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_CREATE;
+        }
+        break;
+    case T_MKNOD:
+        path = va_arg(ap, V9fsString *);
+        mode = va_arg(ap, int);
+        rdev = va_arg(ap, long int);
+        uid = va_arg(ap, int);
+        gid = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ddsdq",
+                                    uid, gid, path, mode, rdev);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_MKNOD;
+        }
+        break;
+    case T_MKDIR:
+        path = va_arg(ap, V9fsString *);
+        mode = va_arg(ap, int);
+        uid = va_arg(ap, int);
+        gid = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ddsd",
+                                    uid, gid, path, mode);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_MKDIR;
+        }
+        break;
+    case T_SYMLINK:
+        oldpath = va_arg(ap, V9fsString *);
+        path = va_arg(ap, V9fsString *);
+        uid = va_arg(ap, int);
+        gid = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ddss",
+                                    uid, gid, oldpath, path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_SYMLINK;
+        }
+        break;
+    case T_LINK:
+        oldpath = va_arg(ap, V9fsString *);
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ss",
+                                    oldpath, path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_LINK;
+        }
+        break;
+    case T_LSTAT:
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_LSTAT;
+        }
+        break;
+    case T_READLINK:
+        path = va_arg(ap, V9fsString *);
+        size = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sd", path, size);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_READLINK;
+        }
+        break;
+    case T_STATFS:
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_STATFS;
+        }
+        break;
+    case T_CHMOD:
+        path = va_arg(ap, V9fsString *);
+        mode = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sd", path, mode);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_CHMOD;
+        }
+        break;
+    case T_CHOWN:
+        path = va_arg(ap, V9fsString *);
+        uid = va_arg(ap, int);
+        gid = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sdd", path, uid, gid);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_CHOWN;
+        }
+        break;
+    case T_TRUNCATE:
+        path = va_arg(ap, V9fsString *);
+        offset = va_arg(ap, uint64_t);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sq", path, offset);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_TRUNCATE;
+        }
+        break;
+    case T_UTIME:
+        path = va_arg(ap, V9fsString *);
+        spec[0].tv_sec = va_arg(ap, long);
+        spec[0].tv_nsec = va_arg(ap, long);
+        spec[1].tv_sec = va_arg(ap, long);
+        spec[1].tv_nsec = va_arg(ap, long);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sqqqq", path,
+                                    spec[0].tv_sec, spec[1].tv_nsec,
+                                    spec[1].tv_sec, spec[1].tv_nsec);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_UTIME;
+        }
+        break;
+    case T_RENAME:
+        oldpath = va_arg(ap, V9fsString *);
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ss", oldpath, path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_RENAME;
+        }
+        break;
+    case T_REMOVE:
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_REMOVE;
+        }
+        break;
+    case T_LGETXATTR:
+        size = va_arg(ap, int);
+        path = va_arg(ap, V9fsString *);
+        name = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ,
+                                    "dss", size, path, name);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_LGETXATTR;
+        }
+        break;
+    case T_LLISTXATTR:
+        size = va_arg(ap, int);
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ds", size, path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_LLISTXATTR;
+        }
+        break;
+    case T_LSETXATTR:
+        path = va_arg(ap, V9fsString *);
+        name = va_arg(ap, V9fsString *);
+        value = va_arg(ap, V9fsString *);
+        size = va_arg(ap, int);
+        flags = va_arg(ap, int);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sssdd",
+                                    path, name, value, size, flags);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_LSETXATTR;
+        }
+        break;
+    case T_LREMOVEXATTR:
+        path = va_arg(ap, V9fsString *);
+        name = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ss", path, name);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_LREMOVEXATTR;
+        }
+        break;
+    case T_GETVERSION:
+        path = va_arg(ap, V9fsString *);
+        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
+        if (retval > 0) {
+            header.size = retval;
+            header.type = T_GETVERSION;
+        }
+        break;
+    default:
+        error_report("Invalid type %d", type);
+        retval = -EINVAL;
+        break;
+    }
+    va_end(ap);
+
+    if (retval < 0) {
+        goto err_out;
+    }
+
+    /* marshal the header details */
+    proxy_marshal(iovec, 0, "dd", header.type, header.size);
+    header.size += PROXY_HDR_SZ;
+
+    retval = qemu_write_full(proxy->sockfd, iovec->iov_base, header.size);
+    if (retval != header.size) {
+        goto close_error;
+    }
+
+    switch (type) {
+    case T_OPEN:
+    case T_CREATE:
+        /*
+         * A file descriptor is returned as response for
+         * T_OPEN,T_CREATE on success
+         */
+        if (v9fs_receivefd(proxy->sockfd, &retval) < 0) {
+            goto close_error;
+        }
+        break;
+    case T_MKNOD:
+    case T_MKDIR:
+    case T_SYMLINK:
+    case T_LINK:
+    case T_CHMOD:
+    case T_CHOWN:
+    case T_RENAME:
+    case T_TRUNCATE:
+    case T_UTIME:
+    case T_REMOVE:
+    case T_LSETXATTR:
+    case T_LREMOVEXATTR:
+        if (v9fs_receive_status(proxy, reply, &retval) < 0) {
+            goto close_error;
+        }
+        break;
+    case T_LSTAT:
+    case T_READLINK:
+    case T_STATFS:
+    case T_GETVERSION:
+        if (v9fs_receive_response(proxy, type, &retval, response) < 0) {
+            goto close_error;
+        }
+        break;
+    case T_LGETXATTR:
+    case T_LLISTXATTR:
+        if (!size) {
+            if (v9fs_receive_status(proxy, reply, &retval) < 0) {
+                goto close_error;
+            }
+        } else {
+            if (v9fs_receive_response(proxy, type, &retval, response) < 0) {
+                goto close_error;
+            }
+        }
+        break;
+    }
+
+err_out:
+    qemu_mutex_unlock(&proxy->mutex);
+    return retval;
+
+close_error:
+    close(proxy->sockfd);
+    proxy->sockfd = -1;
+    qemu_mutex_unlock(&proxy->mutex);
+    return -EIO;
+}
+
+static int proxy_lstat(FsContext *fs_ctx, V9fsPath *fs_path, struct stat *stbuf)
+{
+    int retval;
+    retval = v9fs_request(fs_ctx->private, T_LSTAT, stbuf, "s", fs_path);
+    if (retval < 0) {
+        errno = -retval;
+        return -1;
+    }
+    return retval;
+}
+
+static ssize_t proxy_readlink(FsContext *fs_ctx, V9fsPath *fs_path,
+                              char *buf, size_t bufsz)
+{
+    int retval;
+    retval = v9fs_request(fs_ctx->private, T_READLINK, buf, "sd",
+                          fs_path, bufsz);
+    if (retval < 0) {
+        errno = -retval;
+        return -1;
+    }
+    return strlen(buf);
+}
+
+static int proxy_close(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return close(fs->fd);
+}
+
+static int proxy_closedir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return closedir(fs->dir.stream);
+}
+
+static int proxy_open(FsContext *ctx, V9fsPath *fs_path,
+                      int flags, V9fsFidOpenState *fs)
+{
+    fs->fd = v9fs_request(ctx->private, T_OPEN, NULL, "sd", fs_path, flags);
+    if (fs->fd < 0) {
+        errno = -fs->fd;
+        fs->fd = -1;
+    }
+    return fs->fd;
+}
+
+static int proxy_opendir(FsContext *ctx,
+                         V9fsPath *fs_path, V9fsFidOpenState *fs)
+{
+    int serrno, fd;
+
+    fs->dir.stream = NULL;
+    fd = v9fs_request(ctx->private, T_OPEN, NULL, "sd", fs_path, O_DIRECTORY);
+    if (fd < 0) {
+        errno = -fd;
+        return -1;
+    }
+    fs->dir.stream = fdopendir(fd);
+    if (!fs->dir.stream) {
+        serrno = errno;
+        close(fd);
+        errno = serrno;
+        return -1;
+    }
+    return 0;
+}
+
+static void proxy_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    rewinddir(fs->dir.stream);
+}
+
+static off_t proxy_telldir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return telldir(fs->dir.stream);
+}
+
+static struct dirent *proxy_readdir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    return readdir(fs->dir.stream);
+}
+
+static void proxy_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
+{
+    seekdir(fs->dir.stream, off);
+}
+
+static ssize_t proxy_preadv(FsContext *ctx, V9fsFidOpenState *fs,
+                            const struct iovec *iov,
+                            int iovcnt, off_t offset)
+{
+    ssize_t ret;
+#ifdef CONFIG_PREADV
+    ret = preadv(fs->fd, iov, iovcnt, offset);
+#else
+    ret = lseek(fs->fd, offset, SEEK_SET);
+    if (ret >= 0) {
+        ret = readv(fs->fd, iov, iovcnt);
+    }
+#endif
+    return ret;
+}
+
+static ssize_t proxy_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
+                             const struct iovec *iov,
+                             int iovcnt, off_t offset)
+{
+    ssize_t ret;
+
+#ifdef CONFIG_PREADV
+    ret = pwritev(fs->fd, iov, iovcnt, offset);
+#else
+    ret = lseek(fs->fd, offset, SEEK_SET);
+    if (ret >= 0) {
+        ret = writev(fs->fd, iov, iovcnt);
+    }
+#endif
+#ifdef CONFIG_SYNC_FILE_RANGE
+    if (ret > 0 && ctx->export_flags & V9FS_IMMEDIATE_WRITEOUT) {
+        /*
+         * Initiate a writeback. This is not a data integrity sync.
+         * We want to ensure that we don't leave dirty pages in the cache
+         * after write when writeout=immediate is sepcified.
+         */
+        sync_file_range(fs->fd, offset, ret,
+                        SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
+    }
+#endif
+    return ret;
+}
+
+static int proxy_chmod(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
+{
+    int retval;
+    retval = v9fs_request(fs_ctx->private, T_CHMOD, NULL, "sd",
+                          fs_path, credp->fc_mode);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_mknod(FsContext *fs_ctx, V9fsPath *dir_path,
+                       const char *name, FsCred *credp)
+{
+    int retval;
+    V9fsString fullname;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+
+    retval = v9fs_request(fs_ctx->private, T_MKNOD, NULL, "sdqdd",
+                          &fullname, credp->fc_mode, credp->fc_rdev,
+                          credp->fc_uid, credp->fc_gid);
+    v9fs_string_free(&fullname);
+    if (retval < 0) {
+        errno = -retval;
+        retval = -1;
+    }
+    return retval;
+}
+
+static int proxy_mkdir(FsContext *fs_ctx, V9fsPath *dir_path,
+                       const char *name, FsCred *credp)
+{
+    int retval;
+    V9fsString fullname;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+
+    retval = v9fs_request(fs_ctx->private, T_MKDIR, NULL, "sddd", &fullname,
+                          credp->fc_mode, credp->fc_uid, credp->fc_gid);
+    v9fs_string_free(&fullname);
+    if (retval < 0) {
+        errno = -retval;
+        retval = -1;
+    }
+    v9fs_string_free(&fullname);
+    return retval;
+}
+
+static int proxy_fstat(FsContext *fs_ctx, int fid_type,
+                       V9fsFidOpenState *fs, struct stat *stbuf)
+{
+    int fd;
+
+    if (fid_type == P9_FID_DIR) {
+        fd = dirfd(fs->dir.stream);
+    } else {
+        fd = fs->fd;
+    }
+    return fstat(fd, stbuf);
+}
+
+static int proxy_open2(FsContext *fs_ctx, V9fsPath *dir_path, const char *name,
+                       int flags, FsCred *credp, V9fsFidOpenState *fs)
+{
+    V9fsString fullname;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+
+    fs->fd = v9fs_request(fs_ctx->private, T_CREATE, NULL, "sdddd",
+                          &fullname, flags, credp->fc_mode,
+                          credp->fc_uid, credp->fc_gid);
+    v9fs_string_free(&fullname);
+    if (fs->fd < 0) {
+        errno = -fs->fd;
+        fs->fd = -1;
+    }
+    return fs->fd;
+}
+
+static int proxy_symlink(FsContext *fs_ctx, const char *oldpath,
+                         V9fsPath *dir_path, const char *name, FsCred *credp)
+{
+    int retval;
+    V9fsString fullname, target;
+
+    v9fs_string_init(&fullname);
+    v9fs_string_init(&target);
+
+    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
+    v9fs_string_sprintf(&target, "%s", oldpath);
+
+    retval = v9fs_request(fs_ctx->private, T_SYMLINK, NULL, "ssdd",
+                          &target, &fullname, credp->fc_uid, credp->fc_gid);
+    v9fs_string_free(&fullname);
+    v9fs_string_free(&target);
+    if (retval < 0) {
+        errno = -retval;
+        retval = -1;
+    }
+    return retval;
+}
+
+static int proxy_link(FsContext *ctx, V9fsPath *oldpath,
+                      V9fsPath *dirpath, const char *name)
+{
+    int retval;
+    V9fsString newpath;
+
+    v9fs_string_init(&newpath);
+    v9fs_string_sprintf(&newpath, "%s/%s", dirpath->data, name);
+
+    retval = v9fs_request(ctx->private, T_LINK, NULL, "ss", oldpath, &newpath);
+    v9fs_string_free(&newpath);
+    if (retval < 0) {
+        errno = -retval;
+        retval = -1;
+    }
+    return retval;
+}
+
+static int proxy_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size)
+{
+    int retval;
+
+    retval = v9fs_request(ctx->private, T_TRUNCATE, NULL, "sq", fs_path, size);
+    if (retval < 0) {
+        errno = -retval;
+        return -1;
+    }
+    return 0;
+}
+
+static int proxy_rename(FsContext *ctx, const char *oldpath,
+                        const char *newpath)
+{
+    int retval;
+    V9fsString oldname, newname;
+
+    v9fs_string_init(&oldname);
+    v9fs_string_init(&newname);
+
+    v9fs_string_sprintf(&oldname, "%s", oldpath);
+    v9fs_string_sprintf(&newname, "%s", newpath);
+    retval = v9fs_request(ctx->private, T_RENAME, NULL, "ss",
+                          &oldname, &newname);
+    v9fs_string_free(&oldname);
+    v9fs_string_free(&newname);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
+{
+    int retval;
+    retval = v9fs_request(fs_ctx->private, T_CHOWN, NULL, "sdd",
+                          fs_path, credp->fc_uid, credp->fc_gid);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_utimensat(FsContext *s, V9fsPath *fs_path,
+                           const struct timespec *buf)
+{
+    int retval;
+    retval = v9fs_request(s->private, T_UTIME, NULL, "sqqqq",
+                          fs_path,
+                          buf[0].tv_sec, buf[0].tv_nsec,
+                          buf[1].tv_sec, buf[1].tv_nsec);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_remove(FsContext *ctx, const char *path)
+{
+    int retval;
+    V9fsString name;
+    v9fs_string_init(&name);
+    v9fs_string_sprintf(&name, "%s", path);
+    retval = v9fs_request(ctx->private, T_REMOVE, NULL, "s", &name);
+    v9fs_string_free(&name);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_fsync(FsContext *ctx, int fid_type,
+                       V9fsFidOpenState *fs, int datasync)
+{
+    int fd;
+
+    if (fid_type == P9_FID_DIR) {
+        fd = dirfd(fs->dir.stream);
+    } else {
+        fd = fs->fd;
+    }
+
+    if (datasync) {
+        return qemu_fdatasync(fd);
+    } else {
+        return fsync(fd);
+    }
+}
+
+static int proxy_statfs(FsContext *s, V9fsPath *fs_path, struct statfs *stbuf)
+{
+    int retval;
+    retval = v9fs_request(s->private, T_STATFS, stbuf, "s", fs_path);
+    if (retval < 0) {
+        errno = -retval;
+        return -1;
+    }
+    return retval;
+}
+
+static ssize_t proxy_lgetxattr(FsContext *ctx, V9fsPath *fs_path,
+                               const char *name, void *value, size_t size)
+{
+    int retval;
+    V9fsString xname;
+
+    v9fs_string_init(&xname);
+    v9fs_string_sprintf(&xname, "%s", name);
+    retval = v9fs_request(ctx->private, T_LGETXATTR, value, "dss", size,
+                          fs_path, &xname);
+    v9fs_string_free(&xname);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static ssize_t proxy_llistxattr(FsContext *ctx, V9fsPath *fs_path,
+                                void *value, size_t size)
+{
+    int retval;
+    retval = v9fs_request(ctx->private, T_LLISTXATTR, value, "ds", size,
+                        fs_path);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_lsetxattr(FsContext *ctx, V9fsPath *fs_path, const char *name,
+                           void *value, size_t size, int flags)
+{
+    int retval;
+    V9fsString xname, xvalue;
+
+    v9fs_string_init(&xname);
+    v9fs_string_sprintf(&xname, "%s", name);
+
+    v9fs_string_init(&xvalue);
+    xvalue.size = size;
+    xvalue.data = g_malloc(size);
+    memcpy(xvalue.data, value, size);
+
+    retval = v9fs_request(ctx->private, T_LSETXATTR, value, "sssdd",
+                          fs_path, &xname, &xvalue, size, flags);
+    v9fs_string_free(&xname);
+    v9fs_string_free(&xvalue);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_lremovexattr(FsContext *ctx, V9fsPath *fs_path,
+                              const char *name)
+{
+    int retval;
+    V9fsString xname;
+
+    v9fs_string_init(&xname);
+    v9fs_string_sprintf(&xname, "%s", name);
+    retval = v9fs_request(ctx->private, T_LREMOVEXATTR, NULL, "ss",
+                          fs_path, &xname);
+    v9fs_string_free(&xname);
+    if (retval < 0) {
+        errno = -retval;
+    }
+    return retval;
+}
+
+static int proxy_name_to_path(FsContext *ctx, V9fsPath *dir_path,
+                              const char *name, V9fsPath *target)
+{
+    if (dir_path) {
+        v9fs_string_sprintf((V9fsString *)target, "%s/%s",
+                            dir_path->data, name);
+    } else {
+        v9fs_string_sprintf((V9fsString *)target, "%s", name);
+    }
+    /* Bump the size for including terminating NULL */
+    target->size++;
+    return 0;
+}
+
+static int proxy_renameat(FsContext *ctx, V9fsPath *olddir,
+                          const char *old_name, V9fsPath *newdir,
+                          const char *new_name)
+{
+    int ret;
+    V9fsString old_full_name, new_full_name;
+
+    v9fs_string_init(&old_full_name);
+    v9fs_string_init(&new_full_name);
+
+    v9fs_string_sprintf(&old_full_name, "%s/%s", olddir->data, old_name);
+    v9fs_string_sprintf(&new_full_name, "%s/%s", newdir->data, new_name);
+
+    ret = proxy_rename(ctx, old_full_name.data, new_full_name.data);
+    v9fs_string_free(&old_full_name);
+    v9fs_string_free(&new_full_name);
+    return ret;
+}
+
+static int proxy_unlinkat(FsContext *ctx, V9fsPath *dir,
+                          const char *name, int flags)
+{
+    int ret;
+    V9fsString fullname;
+    v9fs_string_init(&fullname);
+
+    v9fs_string_sprintf(&fullname, "%s/%s", dir->data, name);
+    ret = proxy_remove(ctx, fullname.data);
+    v9fs_string_free(&fullname);
+
+    return ret;
+}
+
+static int proxy_ioc_getversion(FsContext *fs_ctx, V9fsPath *path,
+                                mode_t st_mode, uint64_t *st_gen)
+{
+    int err;
+
+    /* Do not try to open special files like device nodes, fifos etc
+     * we can get fd for regular files and directories only
+     */
+    if (!S_ISREG(st_mode) && !S_ISDIR(st_mode)) {
+        errno = ENOTTY;
+        return -1;
+    }
+    err = v9fs_request(fs_ctx->private, T_GETVERSION, st_gen, "s", path);
+    if (err < 0) {
+        errno = -err;
+        err = -1;
+    }
+    return err;
+}
+
+static int connect_namedsocket(const char *path)
+{
+    int sockfd, size;
+    struct sockaddr_un helper;
+
+    if (strlen(path) >= sizeof(helper.sun_path)) {
+        error_report("Socket name too long");
+        return -1;
+    }
+    sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (sockfd < 0) {
+        error_report("Failed to create socket: %s", strerror(errno));
+        return -1;
+    }
+    strcpy(helper.sun_path, path);
+    helper.sun_family = AF_UNIX;
+    size = strlen(helper.sun_path) + sizeof(helper.sun_family);
+    if (connect(sockfd, (struct sockaddr *)&helper, size) < 0) {
+        error_report("Failed to connect to %s: %s", path, strerror(errno));
+        close(sockfd);
+        return -1;
+    }
+
+    /* remove the socket for security reasons */
+    unlink(path);
+    return sockfd;
+}
+
+static int proxy_parse_opts(QemuOpts *opts, struct FsDriverEntry *fs)
+{
+    const char *socket = qemu_opt_get(opts, "socket");
+    const char *sock_fd = qemu_opt_get(opts, "sock_fd");
+
+    if (!socket && !sock_fd) {
+        error_report("Must specify either socket or sock_fd");
+        return -1;
+    }
+    if (socket && sock_fd) {
+        error_report("Both socket and sock_fd options specified");
+        return -1;
+    }
+    if (socket) {
+        fs->path = g_strdup(socket);
+        fs->export_flags = V9FS_PROXY_SOCK_NAME;
+    } else {
+        fs->path = g_strdup(sock_fd);
+        fs->export_flags = V9FS_PROXY_SOCK_FD;
+    }
+    return 0;
+}
+
+static int proxy_init(FsContext *ctx)
+{
+    V9fsProxy *proxy = g_malloc(sizeof(V9fsProxy));
+    int sock_id;
+
+    if (ctx->export_flags & V9FS_PROXY_SOCK_NAME) {
+        sock_id = connect_namedsocket(ctx->fs_root);
+    } else {
+        sock_id = atoi(ctx->fs_root);
+        if (sock_id < 0) {
+            error_report("Socket descriptor not initialized");
+        }
+    }
+    if (sock_id < 0) {
+        g_free(proxy);
+        return -1;
+    }
+    g_free(ctx->fs_root);
+    ctx->fs_root = NULL;
+
+    proxy->in_iovec.iov_base  = g_malloc(PROXY_MAX_IO_SZ + PROXY_HDR_SZ);
+    proxy->in_iovec.iov_len   = PROXY_MAX_IO_SZ + PROXY_HDR_SZ;
+    proxy->out_iovec.iov_base = g_malloc(PROXY_MAX_IO_SZ + PROXY_HDR_SZ);
+    proxy->out_iovec.iov_len  = PROXY_MAX_IO_SZ + PROXY_HDR_SZ;
+
+    ctx->private = proxy;
+    proxy->sockfd = sock_id;
+    qemu_mutex_init(&proxy->mutex);
+
+    ctx->export_flags |= V9FS_PATHNAME_FSCONTEXT;
+    ctx->exops.get_st_gen = proxy_ioc_getversion;
+    return 0;
+}
+
+FileOperations proxy_ops = {
+    .parse_opts   = proxy_parse_opts,
+    .init         = proxy_init,
+    .lstat        = proxy_lstat,
+    .readlink     = proxy_readlink,
+    .close        = proxy_close,
+    .closedir     = proxy_closedir,
+    .open         = proxy_open,
+    .opendir      = proxy_opendir,
+    .rewinddir    = proxy_rewinddir,
+    .telldir      = proxy_telldir,
+    .readdir      = proxy_readdir,
+    .seekdir      = proxy_seekdir,
+    .preadv       = proxy_preadv,
+    .pwritev      = proxy_pwritev,
+    .chmod        = proxy_chmod,
+    .mknod        = proxy_mknod,
+    .mkdir        = proxy_mkdir,
+    .fstat        = proxy_fstat,
+    .open2        = proxy_open2,
+    .symlink      = proxy_symlink,
+    .link         = proxy_link,
+    .truncate     = proxy_truncate,
+    .rename       = proxy_rename,
+    .chown        = proxy_chown,
+    .utimensat    = proxy_utimensat,
+    .remove       = proxy_remove,
+    .fsync        = proxy_fsync,
+    .statfs       = proxy_statfs,
+    .lgetxattr    = proxy_lgetxattr,
+    .llistxattr   = proxy_llistxattr,
+    .lsetxattr    = proxy_lsetxattr,
+    .lremovexattr = proxy_lremovexattr,
+    .name_to_path = proxy_name_to_path,
+    .renameat     = proxy_renameat,
+    .unlinkat     = proxy_unlinkat,
+};
diff --git a/hw/9pfs/9p-proxy.h b/hw/9pfs/9p-proxy.h
new file mode 100644
index 0000000..b84301d
--- /dev/null
+++ b/hw/9pfs/9p-proxy.h
@@ -0,0 +1,96 @@
+/*
+ * 9p Proxy callback
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ * M. Mohan Kumar <mohan@in.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_9P_PROXY_H
+#define QEMU_9P_PROXY_H
+
+#define PROXY_MAX_IO_SZ (64 * 1024)
+#define V9FS_FD_VALID INT_MAX
+
+/*
+ * proxy iovec only support one element and
+ * marsha/unmarshal doesn't do little endian conversion.
+ */
+#define proxy_unmarshal(in_sg, offset, fmt, args...) \
+    v9fs_iov_unmarshal(in_sg, 1, offset, 0, fmt, ##args)
+#define proxy_marshal(out_sg, offset, fmt, args...) \
+    v9fs_iov_marshal(out_sg, 1, offset, 0, fmt, ##args)
+
+union MsgControl {
+    struct cmsghdr cmsg;
+    char control[CMSG_SPACE(sizeof(int))];
+};
+
+typedef struct {
+    uint32_t type;
+    uint32_t size;
+} ProxyHeader;
+
+#define PROXY_HDR_SZ (sizeof(ProxyHeader))
+
+enum {
+    T_SUCCESS = 0,
+    T_ERROR,
+    T_OPEN,
+    T_CREATE,
+    T_MKNOD,
+    T_MKDIR,
+    T_SYMLINK,
+    T_LINK,
+    T_LSTAT,
+    T_READLINK,
+    T_STATFS,
+    T_CHMOD,
+    T_CHOWN,
+    T_TRUNCATE,
+    T_UTIME,
+    T_RENAME,
+    T_REMOVE,
+    T_LGETXATTR,
+    T_LLISTXATTR,
+    T_LSETXATTR,
+    T_LREMOVEXATTR,
+    T_GETVERSION,
+};
+
+typedef struct {
+    uint64_t st_dev;
+    uint64_t st_ino;
+    uint64_t st_nlink;
+    uint32_t st_mode;
+    uint32_t st_uid;
+    uint32_t st_gid;
+    uint64_t st_rdev;
+    uint64_t st_size;
+    uint64_t st_blksize;
+    uint64_t st_blocks;
+    uint64_t st_atim_sec;
+    uint64_t st_atim_nsec;
+    uint64_t st_mtim_sec;
+    uint64_t st_mtim_nsec;
+    uint64_t st_ctim_sec;
+    uint64_t st_ctim_nsec;
+} ProxyStat;
+
+typedef struct {
+    uint64_t f_type;
+    uint64_t f_bsize;
+    uint64_t f_blocks;
+    uint64_t f_bfree;
+    uint64_t f_bavail;
+    uint64_t f_files;
+    uint64_t f_ffree;
+    uint64_t f_fsid[2];
+    uint64_t f_namelen;
+    uint64_t f_frsize;
+} ProxyStatFS;
+#endif
diff --git a/hw/9pfs/9p-synth.c b/hw/9pfs/9p-synth.c
new file mode 100644
index 0000000..4b6d4e6
--- /dev/null
+++ b/hw/9pfs/9p-synth.c
@@ -0,0 +1,569 @@
+/*
+ * 9p synthetic file system support
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Malahal Naineni <malahal@us.ibm.com>
+ *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "fsdev/qemu-fsdev.h"
+#include "9p-synth.h"
+#include "qemu/rcu.h"
+#include "qemu/rcu_queue.h"
+#include "qemu/cutils.h"
+
+/* Root node for synth file system */
+static V9fsSynthNode synth_root = {
+    .name = "/",
+    .actual_attr = {
+        .mode = 0555 | S_IFDIR,
+        .nlink = 1,
+    },
+    .attr = &synth_root.actual_attr,
+};
+
+static QemuMutex  synth_mutex;
+static int synth_node_count;
+/* set to 1 when the synth fs is ready */
+static int synth_fs;
+
+static V9fsSynthNode *v9fs_add_dir_node(V9fsSynthNode *parent, int mode,
+                                        const char *name,
+                                        V9fsSynthNodeAttr *attr, int inode)
+{
+    V9fsSynthNode *node;
+
+    /* Add directory type and remove write bits */
+    mode = ((mode & 0777) | S_IFDIR) & ~(S_IWUSR | S_IWGRP | S_IWOTH);
+    node = g_malloc0(sizeof(V9fsSynthNode));
+    if (attr) {
+        /* We are adding .. or . entries */
+        node->attr = attr;
+        node->attr->nlink++;
+    } else {
+        node->attr = &node->actual_attr;
+        node->attr->inode = inode;
+        node->attr->nlink = 1;
+        /* We don't allow write to directories */
+        node->attr->mode   = mode;
+        node->attr->write = NULL;
+        node->attr->read  = NULL;
+    }
+    node->private = node;
+    pstrcpy(node->name, sizeof(node->name), name);
+    QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
+    return node;
+}
+
+int qemu_v9fs_synth_mkdir(V9fsSynthNode *parent, int mode,
+                          const char *name, V9fsSynthNode **result)
+{
+    int ret;
+    V9fsSynthNode *node, *tmp;
+
+    if (!synth_fs) {
+        return EAGAIN;
+    }
+    if (!name || (strlen(name) >= NAME_MAX)) {
+        return EINVAL;
+    }
+    if (!parent) {
+        parent = &synth_root;
+    }
+    qemu_mutex_lock(&synth_mutex);
+    QLIST_FOREACH(tmp, &parent->child, sibling) {
+        if (!strcmp(tmp->name, name)) {
+            ret = EEXIST;
+            goto err_out;
+        }
+    }
+    /* Add the name */
+    node = v9fs_add_dir_node(parent, mode, name, NULL, synth_node_count++);
+    v9fs_add_dir_node(node, parent->attr->mode, "..",
+                      parent->attr, parent->attr->inode);
+    v9fs_add_dir_node(node, node->attr->mode, ".",
+                      node->attr, node->attr->inode);
+    *result = node;
+    ret = 0;
+err_out:
+    qemu_mutex_unlock(&synth_mutex);
+    return ret;
+}
+
+int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
+                             const char *name, v9fs_synth_read read,
+                             v9fs_synth_write write, void *arg)
+{
+    int ret;
+    V9fsSynthNode *node, *tmp;
+
+    if (!synth_fs) {
+        return EAGAIN;
+    }
+    if (!name || (strlen(name) >= NAME_MAX)) {
+        return EINVAL;
+    }
+    if (!parent) {
+        parent = &synth_root;
+    }
+
+    qemu_mutex_lock(&synth_mutex);
+    QLIST_FOREACH(tmp, &parent->child, sibling) {
+        if (!strcmp(tmp->name, name)) {
+            ret = EEXIST;
+            goto err_out;
+        }
+    }
+    /* Add file type and remove write bits */
+    mode = ((mode & 0777) | S_IFREG);
+    node = g_malloc0(sizeof(V9fsSynthNode));
+    node->attr         = &node->actual_attr;
+    node->attr->inode  = synth_node_count++;
+    node->attr->nlink  = 1;
+    node->attr->read   = read;
+    node->attr->write  = write;
+    node->attr->mode   = mode;
+    node->private      = arg;
+    pstrcpy(node->name, sizeof(node->name), name);
+    QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
+    ret = 0;
+err_out:
+    qemu_mutex_unlock(&synth_mutex);
+    return ret;
+}
+
+static void synth_fill_statbuf(V9fsSynthNode *node, struct stat *stbuf)
+{
+    stbuf->st_dev = 0;
+    stbuf->st_ino = node->attr->inode;
+    stbuf->st_mode = node->attr->mode;
+    stbuf->st_nlink = node->attr->nlink;
+    stbuf->st_uid = 0;
+    stbuf->st_gid = 0;
+    stbuf->st_rdev = 0;
+    stbuf->st_size = 0;
+    stbuf->st_blksize = 0;
+    stbuf->st_blocks = 0;
+    stbuf->st_atime = 0;
+    stbuf->st_mtime = 0;
+    stbuf->st_ctime = 0;
+}
+
+static int synth_lstat(FsContext *fs_ctx,
+                            V9fsPath *fs_path, struct stat *stbuf)
+{
+    V9fsSynthNode *node = *(V9fsSynthNode **)fs_path->data;
+
+    synth_fill_statbuf(node, stbuf);
+    return 0;
+}
+
+static int synth_fstat(FsContext *fs_ctx, int fid_type,
+                            V9fsFidOpenState *fs, struct stat *stbuf)
+{
+    V9fsSynthOpenState *synth_open = fs->private;
+    synth_fill_statbuf(synth_open->node, stbuf);
+    return 0;
+}
+
+static int synth_opendir(FsContext *ctx,
+                             V9fsPath *fs_path, V9fsFidOpenState *fs)
+{
+    V9fsSynthOpenState *synth_open;
+    V9fsSynthNode *node = *(V9fsSynthNode **)fs_path->data;
+
+    synth_open = g_malloc(sizeof(*synth_open));
+    synth_open->node = node;
+    node->open_count++;
+    fs->private = synth_open;
+    return 0;
+}
+
+static int synth_closedir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    V9fsSynthOpenState *synth_open = fs->private;
+    V9fsSynthNode *node = synth_open->node;
+
+    node->open_count--;
+    g_free(synth_open);
+    fs->private = NULL;
+    return 0;
+}
+
+static off_t synth_telldir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    V9fsSynthOpenState *synth_open = fs->private;
+    return synth_open->offset;
+}
+
+static void synth_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
+{
+    V9fsSynthOpenState *synth_open = fs->private;
+    synth_open->offset = off;
+}
+
+static void synth_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    synth_seekdir(ctx, fs, 0);
+}
+
+static void synth_direntry(V9fsSynthNode *node,
+                                struct dirent *entry, off_t off)
+{
+    strcpy(entry->d_name, node->name);
+    entry->d_ino = node->attr->inode;
+    entry->d_off = off + 1;
+}
+
+static struct dirent *synth_get_dentry(V9fsSynthNode *dir,
+                                            struct dirent *entry, off_t off)
+{
+    int i = 0;
+    V9fsSynthNode *node;
+
+    rcu_read_lock();
+    QLIST_FOREACH(node, &dir->child, sibling) {
+        /* This is the off child of the directory */
+        if (i == off) {
+            break;
+        }
+        i++;
+    }
+    rcu_read_unlock();
+    if (!node) {
+        /* end of directory */
+        return NULL;
+    }
+    synth_direntry(node, entry, off);
+    return entry;
+}
+
+static struct dirent *synth_readdir(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    struct dirent *entry;
+    V9fsSynthOpenState *synth_open = fs->private;
+    V9fsSynthNode *node = synth_open->node;
+    entry = synth_get_dentry(node, &synth_open->dent, synth_open->offset);
+    if (entry) {
+        synth_open->offset++;
+    }
+    return entry;
+}
+
+static int synth_open(FsContext *ctx, V9fsPath *fs_path,
+                           int flags, V9fsFidOpenState *fs)
+{
+    V9fsSynthOpenState *synth_open;
+    V9fsSynthNode *node = *(V9fsSynthNode **)fs_path->data;
+
+    synth_open = g_malloc(sizeof(*synth_open));
+    synth_open->node = node;
+    node->open_count++;
+    fs->private = synth_open;
+    return 0;
+}
+
+static int synth_open2(FsContext *fs_ctx, V9fsPath *dir_path,
+                            const char *name, int flags,
+                            FsCred *credp, V9fsFidOpenState *fs)
+{
+    errno = ENOSYS;
+    return -1;
+}
+
+static int synth_close(FsContext *ctx, V9fsFidOpenState *fs)
+{
+    V9fsSynthOpenState *synth_open = fs->private;
+    V9fsSynthNode *node = synth_open->node;
+
+    node->open_count--;
+    g_free(synth_open);
+    fs->private = NULL;
+    return 0;
+}
+
+static ssize_t synth_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
+                                  const struct iovec *iov,
+                                  int iovcnt, off_t offset)
+{
+    int i, count = 0, wcount;
+    V9fsSynthOpenState *synth_open = fs->private;
+    V9fsSynthNode *node = synth_open->node;
+    if (!node->attr->write) {
+        errno = EPERM;
+        return -1;
+    }
+    for (i = 0; i < iovcnt; i++) {
+        wcount = node->attr->write(iov[i].iov_base, iov[i].iov_len,
+                                   offset, node->private);
+        offset += wcount;
+        count  += wcount;
+        /* If we wrote less than requested. we are done */
+        if (wcount < iov[i].iov_len) {
+            break;
+        }
+    }
+    return count;
+}
+
+static ssize_t synth_preadv(FsContext *ctx, V9fsFidOpenState *fs,
+                                 const struct iovec *iov,
+                                 int iovcnt, off_t offset)
+{
+    int i, count = 0, rcount;
+    V9fsSynthOpenState *synth_open = fs->private;
+    V9fsSynthNode *node = synth_open->node;
+    if (!node->attr->read) {
+        errno = EPERM;
+        return -1;
+    }
+    for (i = 0; i < iovcnt; i++) {
+        rcount = node->attr->read(iov[i].iov_base, iov[i].iov_len,
+                                  offset, node->private);
+        offset += rcount;
+        count  += rcount;
+        /* If we read less than requested. we are done */
+        if (rcount < iov[i].iov_len) {
+            break;
+        }
+    }
+    return count;
+}
+
+static int synth_truncate(FsContext *ctx, V9fsPath *path, off_t offset)
+{
+    errno = ENOSYS;
+    return -1;
+}
+
+static int synth_chmod(FsContext *fs_ctx, V9fsPath *path, FsCred *credp)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_mknod(FsContext *fs_ctx, V9fsPath *path,
+                       const char *buf, FsCred *credp)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_mkdir(FsContext *fs_ctx, V9fsPath *path,
+                       const char *buf, FsCred *credp)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static ssize_t synth_readlink(FsContext *fs_ctx, V9fsPath *path,
+                                   char *buf, size_t bufsz)
+{
+    errno = ENOSYS;
+    return -1;
+}
+
+static int synth_symlink(FsContext *fs_ctx, const char *oldpath,
+                              V9fsPath *newpath, const char *buf, FsCred *credp)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_link(FsContext *fs_ctx, V9fsPath *oldpath,
+                           V9fsPath *newpath, const char *buf)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_rename(FsContext *ctx, const char *oldpath,
+                             const char *newpath)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_chown(FsContext *fs_ctx, V9fsPath *path, FsCred *credp)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_utimensat(FsContext *fs_ctx, V9fsPath *path,
+                                const struct timespec *buf)
+{
+    errno = EPERM;
+    return 0;
+}
+
+static int synth_remove(FsContext *ctx, const char *path)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_fsync(FsContext *ctx, int fid_type,
+                            V9fsFidOpenState *fs, int datasync)
+{
+    errno = ENOSYS;
+    return 0;
+}
+
+static int synth_statfs(FsContext *s, V9fsPath *fs_path,
+                             struct statfs *stbuf)
+{
+    stbuf->f_type = 0xABCD;
+    stbuf->f_bsize = 512;
+    stbuf->f_blocks = 0;
+    stbuf->f_files = synth_node_count;
+    stbuf->f_namelen = NAME_MAX;
+    return 0;
+}
+
+static ssize_t synth_lgetxattr(FsContext *ctx, V9fsPath *path,
+                                    const char *name, void *value, size_t size)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static ssize_t synth_llistxattr(FsContext *ctx, V9fsPath *path,
+                                     void *value, size_t size)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int synth_lsetxattr(FsContext *ctx, V9fsPath *path,
+                                const char *name, void *value,
+                                size_t size, int flags)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int synth_lremovexattr(FsContext *ctx,
+                                   V9fsPath *path, const char *name)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static int synth_name_to_path(FsContext *ctx, V9fsPath *dir_path,
+                                   const char *name, V9fsPath *target)
+{
+    V9fsSynthNode *node;
+    V9fsSynthNode *dir_node;
+
+    /* "." and ".." are not allowed */
+    if (!strcmp(name, ".") || !strcmp(name, "..")) {
+        errno = EINVAL;
+        return -1;
+
+    }
+    if (!dir_path) {
+        dir_node = &synth_root;
+    } else {
+        dir_node = *(V9fsSynthNode **)dir_path->data;
+    }
+    if (!strcmp(name, "/")) {
+        node = dir_node;
+        goto out;
+    }
+    /* search for the name in the childern */
+    rcu_read_lock();
+    QLIST_FOREACH(node, &dir_node->child, sibling) {
+        if (!strcmp(node->name, name)) {
+            break;
+        }
+    }
+    rcu_read_unlock();
+
+    if (!node) {
+        errno = ENOENT;
+        return -1;
+    }
+out:
+    /* Copy the node pointer to fid */
+    target->data = g_malloc(sizeof(void *));
+    memcpy(target->data, &node, sizeof(void *));
+    target->size = sizeof(void *);
+    return 0;
+}
+
+static int synth_renameat(FsContext *ctx, V9fsPath *olddir,
+                               const char *old_name, V9fsPath *newdir,
+                               const char *new_name)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_unlinkat(FsContext *ctx, V9fsPath *dir,
+                               const char *name, int flags)
+{
+    errno = EPERM;
+    return -1;
+}
+
+static int synth_init(FsContext *ctx)
+{
+    QLIST_INIT(&synth_root.child);
+    qemu_mutex_init(&synth_mutex);
+
+    /* Add "." and ".." entries for root */
+    v9fs_add_dir_node(&synth_root, synth_root.attr->mode,
+                      "..", synth_root.attr, synth_root.attr->inode);
+    v9fs_add_dir_node(&synth_root, synth_root.attr->mode,
+                      ".", synth_root.attr, synth_root.attr->inode);
+
+    /* Mark the subsystem is ready for use */
+    synth_fs = 1;
+    return 0;
+}
+
+FileOperations synth_ops = {
+    .init         = synth_init,
+    .lstat        = synth_lstat,
+    .readlink     = synth_readlink,
+    .close        = synth_close,
+    .closedir     = synth_closedir,
+    .open         = synth_open,
+    .opendir      = synth_opendir,
+    .rewinddir    = synth_rewinddir,
+    .telldir      = synth_telldir,
+    .readdir      = synth_readdir,
+    .seekdir      = synth_seekdir,
+    .preadv       = synth_preadv,
+    .pwritev      = synth_pwritev,
+    .chmod        = synth_chmod,
+    .mknod        = synth_mknod,
+    .mkdir        = synth_mkdir,
+    .fstat        = synth_fstat,
+    .open2        = synth_open2,
+    .symlink      = synth_symlink,
+    .link         = synth_link,
+    .truncate     = synth_truncate,
+    .rename       = synth_rename,
+    .chown        = synth_chown,
+    .utimensat    = synth_utimensat,
+    .remove       = synth_remove,
+    .fsync        = synth_fsync,
+    .statfs       = synth_statfs,
+    .lgetxattr    = synth_lgetxattr,
+    .llistxattr   = synth_llistxattr,
+    .lsetxattr    = synth_lsetxattr,
+    .lremovexattr = synth_lremovexattr,
+    .name_to_path = synth_name_to_path,
+    .renameat     = synth_renameat,
+    .unlinkat     = synth_unlinkat,
+};
diff --git a/hw/9pfs/9p-synth.h b/hw/9pfs/9p-synth.h
new file mode 100644
index 0000000..6bcb44a
--- /dev/null
+++ b/hw/9pfs/9p-synth.h
@@ -0,0 +1,52 @@
+/*
+ * 9p
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_9P_SYNTH_H
+#define QEMU_9P_SYNTH_H
+
+typedef struct V9fsSynthNode V9fsSynthNode;
+typedef ssize_t (*v9fs_synth_read)(void *buf, int len, off_t offset,
+                                   void *arg);
+typedef ssize_t (*v9fs_synth_write)(void *buf, int len, off_t offset,
+                                    void *arg);
+typedef struct V9fsSynthNodeAttr {
+    int mode;
+    int inode;
+    int nlink;
+    v9fs_synth_read read;
+    v9fs_synth_write write;
+} V9fsSynthNodeAttr;
+
+struct V9fsSynthNode {
+    QLIST_HEAD(, V9fsSynthNode) child;
+    QLIST_ENTRY(V9fsSynthNode) sibling;
+    char name[NAME_MAX];
+    V9fsSynthNodeAttr *attr;
+    V9fsSynthNodeAttr actual_attr;
+    void *private;
+    int open_count;
+};
+
+typedef struct V9fsSynthOpenState {
+    off_t offset;
+    V9fsSynthNode *node;
+    struct dirent dent;
+} V9fsSynthOpenState;
+
+extern int qemu_v9fs_synth_mkdir(V9fsSynthNode *parent, int mode,
+                                 const char *name, V9fsSynthNode **result);
+extern int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
+                                    const char *name, v9fs_synth_read read,
+                                    v9fs_synth_write write, void *arg);
+
+#endif
diff --git a/hw/9pfs/9p-xattr-user.c b/hw/9pfs/9p-xattr-user.c
new file mode 100644
index 0000000..f87530c
--- /dev/null
+++ b/hw/9pfs/9p-xattr-user.c
@@ -0,0 +1,127 @@
+/*
+ * 9p user. xattr callback
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "fsdev/file-op-9p.h"
+#include "9p-xattr.h"
+
+
+static ssize_t mp_user_getxattr(FsContext *ctx, const char *path,
+                                const char *name, void *value, size_t size)
+{
+    char *buffer;
+    ssize_t ret;
+
+    if (strncmp(name, "user.virtfs.", 12) == 0) {
+        /*
+         * Don't allow fetch of user.virtfs namesapce
+         * in case of mapped security
+         */
+        errno = ENOATTR;
+        return -1;
+    }
+    buffer = rpath(ctx, path);
+    ret = lgetxattr(buffer, name, value, size);
+    g_free(buffer);
+    return ret;
+}
+
+static ssize_t mp_user_listxattr(FsContext *ctx, const char *path,
+                                 char *name, void *value, size_t size)
+{
+    int name_size = strlen(name) + 1;
+    if (strncmp(name, "user.virtfs.", 12) == 0) {
+
+        /*  check if it is a mapped posix acl */
+        if (strncmp(name, "user.virtfs.system.posix_acl_", 29) == 0) {
+            /* adjust the name and size */
+            name += 12;
+            name_size -= 12;
+        } else {
+            /*
+             * Don't allow fetch of user.virtfs namesapce
+             * in case of mapped security
+             */
+            return 0;
+        }
+    }
+    if (!value) {
+        return name_size;
+    }
+
+    if (size < name_size) {
+        errno = ERANGE;
+        return -1;
+    }
+
+    /* name_size includes the trailing NUL. */
+    memcpy(value, name, name_size);
+    return name_size;
+}
+
+static int mp_user_setxattr(FsContext *ctx, const char *path, const char *name,
+                            void *value, size_t size, int flags)
+{
+    char *buffer;
+    int ret;
+
+    if (strncmp(name, "user.virtfs.", 12) == 0) {
+        /*
+         * Don't allow fetch of user.virtfs namesapce
+         * in case of mapped security
+         */
+        errno = EACCES;
+        return -1;
+    }
+    buffer = rpath(ctx, path);
+    ret = lsetxattr(buffer, name, value, size, flags);
+    g_free(buffer);
+    return ret;
+}
+
+static int mp_user_removexattr(FsContext *ctx,
+                               const char *path, const char *name)
+{
+    char *buffer;
+    int ret;
+
+    if (strncmp(name, "user.virtfs.", 12) == 0) {
+        /*
+         * Don't allow fetch of user.virtfs namesapce
+         * in case of mapped security
+         */
+        errno = EACCES;
+        return -1;
+    }
+    buffer = rpath(ctx, path);
+    ret = lremovexattr(buffer, name);
+    g_free(buffer);
+    return ret;
+}
+
+XattrOperations mapped_user_xattr = {
+    .name = "user.",
+    .getxattr = mp_user_getxattr,
+    .setxattr = mp_user_setxattr,
+    .listxattr = mp_user_listxattr,
+    .removexattr = mp_user_removexattr,
+};
+
+XattrOperations passthrough_user_xattr = {
+    .name = "user.",
+    .getxattr = pt_getxattr,
+    .setxattr = pt_setxattr,
+    .listxattr = pt_listxattr,
+    .removexattr = pt_removexattr,
+};
diff --git a/hw/9pfs/9p-xattr.c b/hw/9pfs/9p-xattr.c
new file mode 100644
index 0000000..5d8595e
--- /dev/null
+++ b/hw/9pfs/9p-xattr.c
@@ -0,0 +1,164 @@
+/*
+ * 9p  xattr callback
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "9p.h"
+#include "fsdev/file-op-9p.h"
+#include "9p-xattr.h"
+
+
+static XattrOperations *get_xattr_operations(XattrOperations **h,
+                                             const char *name)
+{
+    XattrOperations *xops;
+    for (xops = *(h)++; xops != NULL; xops = *(h)++) {
+        if (!strncmp(name, xops->name, strlen(xops->name))) {
+            return xops;
+        }
+    }
+    return NULL;
+}
+
+ssize_t v9fs_get_xattr(FsContext *ctx, const char *path,
+                       const char *name, void *value, size_t size)
+{
+    XattrOperations *xops = get_xattr_operations(ctx->xops, name);
+    if (xops) {
+        return xops->getxattr(ctx, path, name, value, size);
+    }
+    errno = EOPNOTSUPP;
+    return -1;
+}
+
+ssize_t pt_listxattr(FsContext *ctx, const char *path,
+                     char *name, void *value, size_t size)
+{
+    int name_size = strlen(name) + 1;
+    if (!value) {
+        return name_size;
+    }
+
+    if (size < name_size) {
+        errno = ERANGE;
+        return -1;
+    }
+
+    /* no need for strncpy: name_size is strlen(name)+1 */
+    memcpy(value, name, name_size);
+    return name_size;
+}
+
+
+/*
+ * Get the list and pass to each layer to find out whether
+ * to send the data or not
+ */
+ssize_t v9fs_list_xattr(FsContext *ctx, const char *path,
+                        void *value, size_t vsize)
+{
+    ssize_t size = 0;
+    char *buffer;
+    void *ovalue = value;
+    XattrOperations *xops;
+    char *orig_value, *orig_value_start;
+    ssize_t xattr_len, parsed_len = 0, attr_len;
+
+    /* Get the actual len */
+    buffer = rpath(ctx, path);
+    xattr_len = llistxattr(buffer, value, 0);
+    if (xattr_len <= 0) {
+        g_free(buffer);
+        return xattr_len;
+    }
+
+    /* Now fetch the xattr and find the actual size */
+    orig_value = g_malloc(xattr_len);
+    xattr_len = llistxattr(buffer, orig_value, xattr_len);
+    g_free(buffer);
+
+    /* store the orig pointer */
+    orig_value_start = orig_value;
+    while (xattr_len > parsed_len) {
+        xops = get_xattr_operations(ctx->xops, orig_value);
+        if (!xops) {
+            goto next_entry;
+        }
+
+        if (!value) {
+            size += xops->listxattr(ctx, path, orig_value, value, vsize);
+        } else {
+            size = xops->listxattr(ctx, path, orig_value, value, vsize);
+            if (size < 0) {
+                goto err_out;
+            }
+            value += size;
+            vsize -= size;
+        }
+next_entry:
+        /* Got the next entry */
+        attr_len = strlen(orig_value) + 1;
+        parsed_len += attr_len;
+        orig_value += attr_len;
+    }
+    if (value) {
+        size = value - ovalue;
+    }
+
+err_out:
+    g_free(orig_value_start);
+    return size;
+}
+
+int v9fs_set_xattr(FsContext *ctx, const char *path, const char *name,
+                   void *value, size_t size, int flags)
+{
+    XattrOperations *xops = get_xattr_operations(ctx->xops, name);
+    if (xops) {
+        return xops->setxattr(ctx, path, name, value, size, flags);
+    }
+    errno = EOPNOTSUPP;
+    return -1;
+
+}
+
+int v9fs_remove_xattr(FsContext *ctx,
+                      const char *path, const char *name)
+{
+    XattrOperations *xops = get_xattr_operations(ctx->xops, name);
+    if (xops) {
+        return xops->removexattr(ctx, path, name);
+    }
+    errno = EOPNOTSUPP;
+    return -1;
+
+}
+
+XattrOperations *mapped_xattr_ops[] = {
+    &mapped_user_xattr,
+    &mapped_pacl_xattr,
+    &mapped_dacl_xattr,
+    NULL,
+};
+
+XattrOperations *passthrough_xattr_ops[] = {
+    &passthrough_user_xattr,
+    &passthrough_acl_xattr,
+    NULL,
+};
+
+/* for .user none model should be same as passthrough */
+XattrOperations *none_xattr_ops[] = {
+    &passthrough_user_xattr,
+    &none_acl_xattr,
+    NULL,
+};
diff --git a/hw/9pfs/9p-xattr.h b/hw/9pfs/9p-xattr.h
new file mode 100644
index 0000000..a853ea6
--- /dev/null
+++ b/hw/9pfs/9p-xattr.h
@@ -0,0 +1,121 @@
+/*
+ * 9p
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_9P_XATTR_H
+#define QEMU_9P_XATTR_H
+
+#include "qemu/xattr.h"
+
+typedef struct xattr_operations
+{
+    const char *name;
+    ssize_t (*getxattr)(FsContext *ctx, const char *path,
+                        const char *name, void *value, size_t size);
+    ssize_t (*listxattr)(FsContext *ctx, const char *path,
+                         char *name, void *value, size_t size);
+    int (*setxattr)(FsContext *ctx, const char *path, const char *name,
+                    void *value, size_t size, int flags);
+    int (*removexattr)(FsContext *ctx,
+                       const char *path, const char *name);
+} XattrOperations;
+
+
+extern XattrOperations mapped_user_xattr;
+extern XattrOperations passthrough_user_xattr;
+
+extern XattrOperations mapped_pacl_xattr;
+extern XattrOperations mapped_dacl_xattr;
+extern XattrOperations passthrough_acl_xattr;
+extern XattrOperations none_acl_xattr;
+
+extern XattrOperations *mapped_xattr_ops[];
+extern XattrOperations *passthrough_xattr_ops[];
+extern XattrOperations *none_xattr_ops[];
+
+ssize_t v9fs_get_xattr(FsContext *ctx, const char *path, const char *name,
+                       void *value, size_t size);
+ssize_t v9fs_list_xattr(FsContext *ctx, const char *path, void *value,
+                        size_t vsize);
+int v9fs_set_xattr(FsContext *ctx, const char *path, const char *name,
+                          void *value, size_t size, int flags);
+int v9fs_remove_xattr(FsContext *ctx, const char *path, const char *name);
+ssize_t pt_listxattr(FsContext *ctx, const char *path, char *name, void *value,
+                     size_t size);
+
+static inline ssize_t pt_getxattr(FsContext *ctx, const char *path,
+                                  const char *name, void *value, size_t size)
+{
+    char *buffer;
+    ssize_t ret;
+
+    buffer = rpath(ctx, path);
+    ret = lgetxattr(buffer, name, value, size);
+    g_free(buffer);
+    return ret;
+}
+
+static inline int pt_setxattr(FsContext *ctx, const char *path,
+                              const char *name, void *value,
+                              size_t size, int flags)
+{
+    char *buffer;
+    int ret;
+
+    buffer = rpath(ctx, path);
+    ret = lsetxattr(buffer, name, value, size, flags);
+    g_free(buffer);
+    return ret;
+}
+
+static inline int pt_removexattr(FsContext *ctx,
+                                 const char *path, const char *name)
+{
+    char *buffer;
+    int ret;
+
+    buffer = rpath(ctx, path);
+    ret = lremovexattr(path, name);
+    g_free(buffer);
+    return ret;
+}
+
+static inline ssize_t notsup_getxattr(FsContext *ctx, const char *path,
+                                      const char *name, void *value,
+                                      size_t size)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static inline int notsup_setxattr(FsContext *ctx, const char *path,
+                                  const char *name, void *value,
+                                  size_t size, int flags)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+static inline ssize_t notsup_listxattr(FsContext *ctx, const char *path,
+                                       char *name, void *value, size_t size)
+{
+    return 0;
+}
+
+static inline int notsup_removexattr(FsContext *ctx,
+                                     const char *path, const char *name)
+{
+    errno = ENOTSUP;
+    return -1;
+}
+
+#endif
diff --git a/hw/9pfs/9p.c b/hw/9pfs/9p.c
new file mode 100644
index 0000000..dfe293d
--- /dev/null
+++ b/hw/9pfs/9p.c
@@ -0,0 +1,3520 @@
+/*
+ * Virtio 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/virtio/virtio.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "virtio-9p.h"
+#include "fsdev/qemu-fsdev.h"
+#include "9p-xattr.h"
+#include "coth.h"
+#include "trace.h"
+#include "migration/migration.h"
+
+int open_fd_hw;
+int total_open_fd;
+static int open_fd_rc;
+
+enum {
+    Oread   = 0x00,
+    Owrite  = 0x01,
+    Ordwr   = 0x02,
+    Oexec   = 0x03,
+    Oexcl   = 0x04,
+    Otrunc  = 0x10,
+    Orexec  = 0x20,
+    Orclose = 0x40,
+    Oappend = 0x80,
+};
+
+ssize_t pdu_marshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = virtio_pdu_vmarshal(pdu, offset, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+ssize_t pdu_unmarshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...)
+{
+    ssize_t ret;
+    va_list ap;
+
+    va_start(ap, fmt);
+    ret = virtio_pdu_vunmarshal(pdu, offset, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+static void pdu_push_and_notify(V9fsPDU *pdu)
+{
+    virtio_9p_push_and_notify(pdu);
+}
+
+static int omode_to_uflags(int8_t mode)
+{
+    int ret = 0;
+
+    switch (mode & 3) {
+    case Oread:
+        ret = O_RDONLY;
+        break;
+    case Ordwr:
+        ret = O_RDWR;
+        break;
+    case Owrite:
+        ret = O_WRONLY;
+        break;
+    case Oexec:
+        ret = O_RDONLY;
+        break;
+    }
+
+    if (mode & Otrunc) {
+        ret |= O_TRUNC;
+    }
+
+    if (mode & Oappend) {
+        ret |= O_APPEND;
+    }
+
+    if (mode & Oexcl) {
+        ret |= O_EXCL;
+    }
+
+    return ret;
+}
+
+struct dotl_openflag_map {
+    int dotl_flag;
+    int open_flag;
+};
+
+static int dotl_to_open_flags(int flags)
+{
+    int i;
+    /*
+     * We have same bits for P9_DOTL_READONLY, P9_DOTL_WRONLY
+     * and P9_DOTL_NOACCESS
+     */
+    int oflags = flags & O_ACCMODE;
+
+    struct dotl_openflag_map dotl_oflag_map[] = {
+        { P9_DOTL_CREATE, O_CREAT },
+        { P9_DOTL_EXCL, O_EXCL },
+        { P9_DOTL_NOCTTY , O_NOCTTY },
+        { P9_DOTL_TRUNC, O_TRUNC },
+        { P9_DOTL_APPEND, O_APPEND },
+        { P9_DOTL_NONBLOCK, O_NONBLOCK } ,
+        { P9_DOTL_DSYNC, O_DSYNC },
+        { P9_DOTL_FASYNC, FASYNC },
+        { P9_DOTL_DIRECT, O_DIRECT },
+        { P9_DOTL_LARGEFILE, O_LARGEFILE },
+        { P9_DOTL_DIRECTORY, O_DIRECTORY },
+        { P9_DOTL_NOFOLLOW, O_NOFOLLOW },
+        { P9_DOTL_NOATIME, O_NOATIME },
+        { P9_DOTL_SYNC, O_SYNC },
+    };
+
+    for (i = 0; i < ARRAY_SIZE(dotl_oflag_map); i++) {
+        if (flags & dotl_oflag_map[i].dotl_flag) {
+            oflags |= dotl_oflag_map[i].open_flag;
+        }
+    }
+
+    return oflags;
+}
+
+void cred_init(FsCred *credp)
+{
+    credp->fc_uid = -1;
+    credp->fc_gid = -1;
+    credp->fc_mode = -1;
+    credp->fc_rdev = -1;
+}
+
+static int get_dotl_openflags(V9fsState *s, int oflags)
+{
+    int flags;
+    /*
+     * Filter the client open flags
+     */
+    flags = dotl_to_open_flags(oflags);
+    flags &= ~(O_NOCTTY | O_ASYNC | O_CREAT);
+    /*
+     * Ignore direct disk access hint until the server supports it.
+     */
+    flags &= ~O_DIRECT;
+    return flags;
+}
+
+void v9fs_path_init(V9fsPath *path)
+{
+    path->data = NULL;
+    path->size = 0;
+}
+
+void v9fs_path_free(V9fsPath *path)
+{
+    g_free(path->data);
+    path->data = NULL;
+    path->size = 0;
+}
+
+void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs)
+{
+    v9fs_path_free(lhs);
+    lhs->data = g_malloc(rhs->size);
+    memcpy(lhs->data, rhs->data, rhs->size);
+    lhs->size = rhs->size;
+}
+
+int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
+                      const char *name, V9fsPath *path)
+{
+    int err;
+    err = s->ops->name_to_path(&s->ctx, dirpath, name, path);
+    if (err < 0) {
+        err = -errno;
+    }
+    return err;
+}
+
+/*
+ * Return TRUE if s1 is an ancestor of s2.
+ *
+ * E.g. "a/b" is an ancestor of "a/b/c" but not of "a/bc/d".
+ * As a special case, We treat s1 as ancestor of s2 if they are same!
+ */
+static int v9fs_path_is_ancestor(V9fsPath *s1, V9fsPath *s2)
+{
+    if (!strncmp(s1->data, s2->data, s1->size - 1)) {
+        if (s2->data[s1->size - 1] == '\0' || s2->data[s1->size - 1] == '/') {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+static size_t v9fs_string_size(V9fsString *str)
+{
+    return str->size;
+}
+
+/*
+ * returns 0 if fid got re-opened, 1 if not, < 0 on error */
+static int v9fs_reopen_fid(V9fsPDU *pdu, V9fsFidState *f)
+{
+    int err = 1;
+    if (f->fid_type == P9_FID_FILE) {
+        if (f->fs.fd == -1) {
+            do {
+                err = v9fs_co_open(pdu, f, f->open_flags);
+            } while (err == -EINTR && !pdu->cancelled);
+        }
+    } else if (f->fid_type == P9_FID_DIR) {
+        if (f->fs.dir.stream == NULL) {
+            do {
+                err = v9fs_co_opendir(pdu, f);
+            } while (err == -EINTR && !pdu->cancelled);
+        }
+    }
+    return err;
+}
+
+static V9fsFidState *get_fid(V9fsPDU *pdu, int32_t fid)
+{
+    int err;
+    V9fsFidState *f;
+    V9fsState *s = pdu->s;
+
+    for (f = s->fid_list; f; f = f->next) {
+        BUG_ON(f->clunked);
+        if (f->fid == fid) {
+            /*
+             * Update the fid ref upfront so that
+             * we don't get reclaimed when we yield
+             * in open later.
+             */
+            f->ref++;
+            /*
+             * check whether we need to reopen the
+             * file. We might have closed the fd
+             * while trying to free up some file
+             * descriptors.
+             */
+            err = v9fs_reopen_fid(pdu, f);
+            if (err < 0) {
+                f->ref--;
+                return NULL;
+            }
+            /*
+             * Mark the fid as referenced so that the LRU
+             * reclaim won't close the file descriptor
+             */
+            f->flags |= FID_REFERENCED;
+            return f;
+        }
+    }
+    return NULL;
+}
+
+static V9fsFidState *alloc_fid(V9fsState *s, int32_t fid)
+{
+    V9fsFidState *f;
+
+    for (f = s->fid_list; f; f = f->next) {
+        /* If fid is already there return NULL */
+        BUG_ON(f->clunked);
+        if (f->fid == fid) {
+            return NULL;
+        }
+    }
+    f = g_malloc0(sizeof(V9fsFidState));
+    f->fid = fid;
+    f->fid_type = P9_FID_NONE;
+    f->ref = 1;
+    /*
+     * Mark the fid as referenced so that the LRU
+     * reclaim won't close the file descriptor
+     */
+    f->flags |= FID_REFERENCED;
+    f->next = s->fid_list;
+    s->fid_list = f;
+
+    v9fs_readdir_init(&f->fs.dir);
+    v9fs_readdir_init(&f->fs_reclaim.dir);
+
+    return f;
+}
+
+static int v9fs_xattr_fid_clunk(V9fsPDU *pdu, V9fsFidState *fidp)
+{
+    int retval = 0;
+
+    if (fidp->fs.xattr.copied_len == -1) {
+        /* getxattr/listxattr fid */
+        goto free_value;
+    }
+    /*
+     * if this is fid for setxattr. clunk should
+     * result in setxattr localcall
+     */
+    if (fidp->fs.xattr.len != fidp->fs.xattr.copied_len) {
+        /* clunk after partial write */
+        retval = -EINVAL;
+        goto free_out;
+    }
+    if (fidp->fs.xattr.len) {
+        retval = v9fs_co_lsetxattr(pdu, &fidp->path, &fidp->fs.xattr.name,
+                                   fidp->fs.xattr.value,
+                                   fidp->fs.xattr.len,
+                                   fidp->fs.xattr.flags);
+    } else {
+        retval = v9fs_co_lremovexattr(pdu, &fidp->path, &fidp->fs.xattr.name);
+    }
+free_out:
+    v9fs_string_free(&fidp->fs.xattr.name);
+free_value:
+    g_free(fidp->fs.xattr.value);
+    return retval;
+}
+
+static int free_fid(V9fsPDU *pdu, V9fsFidState *fidp)
+{
+    int retval = 0;
+
+    if (fidp->fid_type == P9_FID_FILE) {
+        /* If we reclaimed the fd no need to close */
+        if (fidp->fs.fd != -1) {
+            retval = v9fs_co_close(pdu, &fidp->fs);
+        }
+    } else if (fidp->fid_type == P9_FID_DIR) {
+        if (fidp->fs.dir.stream != NULL) {
+            retval = v9fs_co_closedir(pdu, &fidp->fs);
+        }
+    } else if (fidp->fid_type == P9_FID_XATTR) {
+        retval = v9fs_xattr_fid_clunk(pdu, fidp);
+    }
+    v9fs_path_free(&fidp->path);
+    g_free(fidp);
+    return retval;
+}
+
+static int put_fid(V9fsPDU *pdu, V9fsFidState *fidp)
+{
+    BUG_ON(!fidp->ref);
+    fidp->ref--;
+    /*
+     * Don't free the fid if it is in reclaim list
+     */
+    if (!fidp->ref && fidp->clunked) {
+        if (fidp->fid == pdu->s->root_fid) {
+            /*
+             * if the clunked fid is root fid then we
+             * have unmounted the fs on the client side.
+             * delete the migration blocker. Ideally, this
+             * should be hooked to transport close notification
+             */
+            if (pdu->s->migration_blocker) {
+                migrate_del_blocker(pdu->s->migration_blocker);
+                error_free(pdu->s->migration_blocker);
+                pdu->s->migration_blocker = NULL;
+            }
+        }
+        return free_fid(pdu, fidp);
+    }
+    return 0;
+}
+
+static V9fsFidState *clunk_fid(V9fsState *s, int32_t fid)
+{
+    V9fsFidState **fidpp, *fidp;
+
+    for (fidpp = &s->fid_list; *fidpp; fidpp = &(*fidpp)->next) {
+        if ((*fidpp)->fid == fid) {
+            break;
+        }
+    }
+    if (*fidpp == NULL) {
+        return NULL;
+    }
+    fidp = *fidpp;
+    *fidpp = fidp->next;
+    fidp->clunked = 1;
+    return fidp;
+}
+
+void v9fs_reclaim_fd(V9fsPDU *pdu)
+{
+    int reclaim_count = 0;
+    V9fsState *s = pdu->s;
+    V9fsFidState *f, *reclaim_list = NULL;
+
+    for (f = s->fid_list; f; f = f->next) {
+        /*
+         * Unlink fids cannot be reclaimed. Check
+         * for them and skip them. Also skip fids
+         * currently being operated on.
+         */
+        if (f->ref || f->flags & FID_NON_RECLAIMABLE) {
+            continue;
+        }
+        /*
+         * if it is a recently referenced fid
+         * we leave the fid untouched and clear the
+         * reference bit. We come back to it later
+         * in the next iteration. (a simple LRU without
+         * moving list elements around)
+         */
+        if (f->flags & FID_REFERENCED) {
+            f->flags &= ~FID_REFERENCED;
+            continue;
+        }
+        /*
+         * Add fids to reclaim list.
+         */
+        if (f->fid_type == P9_FID_FILE) {
+            if (f->fs.fd != -1) {
+                /*
+                 * Up the reference count so that
+                 * a clunk request won't free this fid
+                 */
+                f->ref++;
+                f->rclm_lst = reclaim_list;
+                reclaim_list = f;
+                f->fs_reclaim.fd = f->fs.fd;
+                f->fs.fd = -1;
+                reclaim_count++;
+            }
+        } else if (f->fid_type == P9_FID_DIR) {
+            if (f->fs.dir.stream != NULL) {
+                /*
+                 * Up the reference count so that
+                 * a clunk request won't free this fid
+                 */
+                f->ref++;
+                f->rclm_lst = reclaim_list;
+                reclaim_list = f;
+                f->fs_reclaim.dir.stream = f->fs.dir.stream;
+                f->fs.dir.stream = NULL;
+                reclaim_count++;
+            }
+        }
+        if (reclaim_count >= open_fd_rc) {
+            break;
+        }
+    }
+    /*
+     * Now close the fid in reclaim list. Free them if they
+     * are already clunked.
+     */
+    while (reclaim_list) {
+        f = reclaim_list;
+        reclaim_list = f->rclm_lst;
+        if (f->fid_type == P9_FID_FILE) {
+            v9fs_co_close(pdu, &f->fs_reclaim);
+        } else if (f->fid_type == P9_FID_DIR) {
+            v9fs_co_closedir(pdu, &f->fs_reclaim);
+        }
+        f->rclm_lst = NULL;
+        /*
+         * Now drop the fid reference, free it
+         * if clunked.
+         */
+        put_fid(pdu, f);
+    }
+}
+
+static int v9fs_mark_fids_unreclaim(V9fsPDU *pdu, V9fsPath *path)
+{
+    int err;
+    V9fsState *s = pdu->s;
+    V9fsFidState *fidp, head_fid;
+
+    head_fid.next = s->fid_list;
+    for (fidp = s->fid_list; fidp; fidp = fidp->next) {
+        if (fidp->path.size != path->size) {
+            continue;
+        }
+        if (!memcmp(fidp->path.data, path->data, path->size)) {
+            /* Mark the fid non reclaimable. */
+            fidp->flags |= FID_NON_RECLAIMABLE;
+
+            /* reopen the file/dir if already closed */
+            err = v9fs_reopen_fid(pdu, fidp);
+            if (err < 0) {
+                return -1;
+            }
+            /*
+             * Go back to head of fid list because
+             * the list could have got updated when
+             * switched to the worker thread
+             */
+            if (err == 0) {
+                fidp = &head_fid;
+            }
+        }
+    }
+    return 0;
+}
+
+static void virtfs_reset(V9fsPDU *pdu)
+{
+    V9fsState *s = pdu->s;
+    V9fsFidState *fidp = NULL;
+
+    /* Free all fids */
+    while (s->fid_list) {
+        fidp = s->fid_list;
+        s->fid_list = fidp->next;
+
+        if (fidp->ref) {
+            fidp->clunked = 1;
+        } else {
+            free_fid(pdu, fidp);
+        }
+    }
+    if (fidp) {
+        /* One or more unclunked fids found... */
+        error_report("9pfs:%s: One or more uncluncked fids "
+                     "found during reset", __func__);
+    }
+}
+
+#define P9_QID_TYPE_DIR         0x80
+#define P9_QID_TYPE_SYMLINK     0x02
+
+#define P9_STAT_MODE_DIR        0x80000000
+#define P9_STAT_MODE_APPEND     0x40000000
+#define P9_STAT_MODE_EXCL       0x20000000
+#define P9_STAT_MODE_MOUNT      0x10000000
+#define P9_STAT_MODE_AUTH       0x08000000
+#define P9_STAT_MODE_TMP        0x04000000
+#define P9_STAT_MODE_SYMLINK    0x02000000
+#define P9_STAT_MODE_LINK       0x01000000
+#define P9_STAT_MODE_DEVICE     0x00800000
+#define P9_STAT_MODE_NAMED_PIPE 0x00200000
+#define P9_STAT_MODE_SOCKET     0x00100000
+#define P9_STAT_MODE_SETUID     0x00080000
+#define P9_STAT_MODE_SETGID     0x00040000
+#define P9_STAT_MODE_SETVTX     0x00010000
+
+#define P9_STAT_MODE_TYPE_BITS (P9_STAT_MODE_DIR |          \
+                                P9_STAT_MODE_SYMLINK |      \
+                                P9_STAT_MODE_LINK |         \
+                                P9_STAT_MODE_DEVICE |       \
+                                P9_STAT_MODE_NAMED_PIPE |   \
+                                P9_STAT_MODE_SOCKET)
+
+/* This is the algorithm from ufs in spfs */
+static void stat_to_qid(const struct stat *stbuf, V9fsQID *qidp)
+{
+    size_t size;
+
+    memset(&qidp->path, 0, sizeof(qidp->path));
+    size = MIN(sizeof(stbuf->st_ino), sizeof(qidp->path));
+    memcpy(&qidp->path, &stbuf->st_ino, size);
+    qidp->version = stbuf->st_mtime ^ (stbuf->st_size << 8);
+    qidp->type = 0;
+    if (S_ISDIR(stbuf->st_mode)) {
+        qidp->type |= P9_QID_TYPE_DIR;
+    }
+    if (S_ISLNK(stbuf->st_mode)) {
+        qidp->type |= P9_QID_TYPE_SYMLINK;
+    }
+}
+
+static int fid_to_qid(V9fsPDU *pdu, V9fsFidState *fidp, V9fsQID *qidp)
+{
+    struct stat stbuf;
+    int err;
+
+    err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
+    if (err < 0) {
+        return err;
+    }
+    stat_to_qid(&stbuf, qidp);
+    return 0;
+}
+
+V9fsPDU *pdu_alloc(V9fsState *s)
+{
+    V9fsPDU *pdu = NULL;
+
+    if (!QLIST_EMPTY(&s->free_list)) {
+        pdu = QLIST_FIRST(&s->free_list);
+        QLIST_REMOVE(pdu, next);
+        QLIST_INSERT_HEAD(&s->active_list, pdu, next);
+    }
+    return pdu;
+}
+
+void pdu_free(V9fsPDU *pdu)
+{
+    if (pdu) {
+        V9fsState *s = pdu->s;
+        /*
+         * Cancelled pdu are added back to the freelist
+         * by flush request .
+         */
+        if (!pdu->cancelled) {
+            QLIST_REMOVE(pdu, next);
+            QLIST_INSERT_HEAD(&s->free_list, pdu, next);
+        }
+    }
+}
+
+/*
+ * We don't do error checking for pdu_marshal/unmarshal here
+ * because we always expect to have enough space to encode
+ * error details
+ */
+static void pdu_complete(V9fsPDU *pdu, ssize_t len)
+{
+    int8_t id = pdu->id + 1; /* Response */
+    V9fsState *s = pdu->s;
+
+    if (len < 0) {
+        int err = -len;
+        len = 7;
+
+        if (s->proto_version != V9FS_PROTO_2000L) {
+            V9fsString str;
+
+            str.data = strerror(err);
+            str.size = strlen(str.data);
+
+            len += pdu_marshal(pdu, len, "s", &str);
+            id = P9_RERROR;
+        }
+
+        len += pdu_marshal(pdu, len, "d", err);
+
+        if (s->proto_version == V9FS_PROTO_2000L) {
+            id = P9_RLERROR;
+        }
+        trace_v9fs_rerror(pdu->tag, pdu->id, err); /* Trace ERROR */
+    }
+
+    /* fill out the header */
+    pdu_marshal(pdu, 0, "dbw", (int32_t)len, id, pdu->tag);
+
+    /* keep these in sync */
+    pdu->size = len;
+    pdu->id = id;
+
+    pdu_push_and_notify(pdu);
+
+    /* Now wakeup anybody waiting in flush for this request */
+    qemu_co_queue_next(&pdu->complete);
+
+    pdu_free(pdu);
+}
+
+static mode_t v9mode_to_mode(uint32_t mode, V9fsString *extension)
+{
+    mode_t ret;
+
+    ret = mode & 0777;
+    if (mode & P9_STAT_MODE_DIR) {
+        ret |= S_IFDIR;
+    }
+
+    if (mode & P9_STAT_MODE_SYMLINK) {
+        ret |= S_IFLNK;
+    }
+    if (mode & P9_STAT_MODE_SOCKET) {
+        ret |= S_IFSOCK;
+    }
+    if (mode & P9_STAT_MODE_NAMED_PIPE) {
+        ret |= S_IFIFO;
+    }
+    if (mode & P9_STAT_MODE_DEVICE) {
+        if (extension->size && extension->data[0] == 'c') {
+            ret |= S_IFCHR;
+        } else {
+            ret |= S_IFBLK;
+        }
+    }
+
+    if (!(ret&~0777)) {
+        ret |= S_IFREG;
+    }
+
+    if (mode & P9_STAT_MODE_SETUID) {
+        ret |= S_ISUID;
+    }
+    if (mode & P9_STAT_MODE_SETGID) {
+        ret |= S_ISGID;
+    }
+    if (mode & P9_STAT_MODE_SETVTX) {
+        ret |= S_ISVTX;
+    }
+
+    return ret;
+}
+
+static int donttouch_stat(V9fsStat *stat)
+{
+    if (stat->type == -1 &&
+        stat->dev == -1 &&
+        stat->qid.type == -1 &&
+        stat->qid.version == -1 &&
+        stat->qid.path == -1 &&
+        stat->mode == -1 &&
+        stat->atime == -1 &&
+        stat->mtime == -1 &&
+        stat->length == -1 &&
+        !stat->name.size &&
+        !stat->uid.size &&
+        !stat->gid.size &&
+        !stat->muid.size &&
+        stat->n_uid == -1 &&
+        stat->n_gid == -1 &&
+        stat->n_muid == -1) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static void v9fs_stat_init(V9fsStat *stat)
+{
+    v9fs_string_init(&stat->name);
+    v9fs_string_init(&stat->uid);
+    v9fs_string_init(&stat->gid);
+    v9fs_string_init(&stat->muid);
+    v9fs_string_init(&stat->extension);
+}
+
+static void v9fs_stat_free(V9fsStat *stat)
+{
+    v9fs_string_free(&stat->name);
+    v9fs_string_free(&stat->uid);
+    v9fs_string_free(&stat->gid);
+    v9fs_string_free(&stat->muid);
+    v9fs_string_free(&stat->extension);
+}
+
+static uint32_t stat_to_v9mode(const struct stat *stbuf)
+{
+    uint32_t mode;
+
+    mode = stbuf->st_mode & 0777;
+    if (S_ISDIR(stbuf->st_mode)) {
+        mode |= P9_STAT_MODE_DIR;
+    }
+
+    if (S_ISLNK(stbuf->st_mode)) {
+        mode |= P9_STAT_MODE_SYMLINK;
+    }
+
+    if (S_ISSOCK(stbuf->st_mode)) {
+        mode |= P9_STAT_MODE_SOCKET;
+    }
+
+    if (S_ISFIFO(stbuf->st_mode)) {
+        mode |= P9_STAT_MODE_NAMED_PIPE;
+    }
+
+    if (S_ISBLK(stbuf->st_mode) || S_ISCHR(stbuf->st_mode)) {
+        mode |= P9_STAT_MODE_DEVICE;
+    }
+
+    if (stbuf->st_mode & S_ISUID) {
+        mode |= P9_STAT_MODE_SETUID;
+    }
+
+    if (stbuf->st_mode & S_ISGID) {
+        mode |= P9_STAT_MODE_SETGID;
+    }
+
+    if (stbuf->st_mode & S_ISVTX) {
+        mode |= P9_STAT_MODE_SETVTX;
+    }
+
+    return mode;
+}
+
+static int stat_to_v9stat(V9fsPDU *pdu, V9fsPath *name,
+                            const struct stat *stbuf,
+                            V9fsStat *v9stat)
+{
+    int err;
+    const char *str;
+
+    memset(v9stat, 0, sizeof(*v9stat));
+
+    stat_to_qid(stbuf, &v9stat->qid);
+    v9stat->mode = stat_to_v9mode(stbuf);
+    v9stat->atime = stbuf->st_atime;
+    v9stat->mtime = stbuf->st_mtime;
+    v9stat->length = stbuf->st_size;
+
+    v9fs_string_null(&v9stat->uid);
+    v9fs_string_null(&v9stat->gid);
+    v9fs_string_null(&v9stat->muid);
+
+    v9stat->n_uid = stbuf->st_uid;
+    v9stat->n_gid = stbuf->st_gid;
+    v9stat->n_muid = 0;
+
+    v9fs_string_null(&v9stat->extension);
+
+    if (v9stat->mode & P9_STAT_MODE_SYMLINK) {
+        err = v9fs_co_readlink(pdu, name, &v9stat->extension);
+        if (err < 0) {
+            return err;
+        }
+    } else if (v9stat->mode & P9_STAT_MODE_DEVICE) {
+        v9fs_string_sprintf(&v9stat->extension, "%c %u %u",
+                S_ISCHR(stbuf->st_mode) ? 'c' : 'b',
+                major(stbuf->st_rdev), minor(stbuf->st_rdev));
+    } else if (S_ISDIR(stbuf->st_mode) || S_ISREG(stbuf->st_mode)) {
+        v9fs_string_sprintf(&v9stat->extension, "%s %lu",
+                "HARDLINKCOUNT", (unsigned long)stbuf->st_nlink);
+    }
+
+    str = strrchr(name->data, '/');
+    if (str) {
+        str += 1;
+    } else {
+        str = name->data;
+    }
+
+    v9fs_string_sprintf(&v9stat->name, "%s", str);
+
+    v9stat->size = 61 +
+        v9fs_string_size(&v9stat->name) +
+        v9fs_string_size(&v9stat->uid) +
+        v9fs_string_size(&v9stat->gid) +
+        v9fs_string_size(&v9stat->muid) +
+        v9fs_string_size(&v9stat->extension);
+    return 0;
+}
+
+#define P9_STATS_MODE          0x00000001ULL
+#define P9_STATS_NLINK         0x00000002ULL
+#define P9_STATS_UID           0x00000004ULL
+#define P9_STATS_GID           0x00000008ULL
+#define P9_STATS_RDEV          0x00000010ULL
+#define P9_STATS_ATIME         0x00000020ULL
+#define P9_STATS_MTIME         0x00000040ULL
+#define P9_STATS_CTIME         0x00000080ULL
+#define P9_STATS_INO           0x00000100ULL
+#define P9_STATS_SIZE          0x00000200ULL
+#define P9_STATS_BLOCKS        0x00000400ULL
+
+#define P9_STATS_BTIME         0x00000800ULL
+#define P9_STATS_GEN           0x00001000ULL
+#define P9_STATS_DATA_VERSION  0x00002000ULL
+
+#define P9_STATS_BASIC         0x000007ffULL /* Mask for fields up to BLOCKS */
+#define P9_STATS_ALL           0x00003fffULL /* Mask for All fields above */
+
+
+static void stat_to_v9stat_dotl(V9fsState *s, const struct stat *stbuf,
+                                V9fsStatDotl *v9lstat)
+{
+    memset(v9lstat, 0, sizeof(*v9lstat));
+
+    v9lstat->st_mode = stbuf->st_mode;
+    v9lstat->st_nlink = stbuf->st_nlink;
+    v9lstat->st_uid = stbuf->st_uid;
+    v9lstat->st_gid = stbuf->st_gid;
+    v9lstat->st_rdev = stbuf->st_rdev;
+    v9lstat->st_size = stbuf->st_size;
+    v9lstat->st_blksize = stbuf->st_blksize;
+    v9lstat->st_blocks = stbuf->st_blocks;
+    v9lstat->st_atime_sec = stbuf->st_atime;
+    v9lstat->st_atime_nsec = stbuf->st_atim.tv_nsec;
+    v9lstat->st_mtime_sec = stbuf->st_mtime;
+    v9lstat->st_mtime_nsec = stbuf->st_mtim.tv_nsec;
+    v9lstat->st_ctime_sec = stbuf->st_ctime;
+    v9lstat->st_ctime_nsec = stbuf->st_ctim.tv_nsec;
+    /* Currently we only support BASIC fields in stat */
+    v9lstat->st_result_mask = P9_STATS_BASIC;
+
+    stat_to_qid(stbuf, &v9lstat->qid);
+}
+
+static void print_sg(struct iovec *sg, int cnt)
+{
+    int i;
+
+    printf("sg[%d]: {", cnt);
+    for (i = 0; i < cnt; i++) {
+        if (i) {
+            printf(", ");
+        }
+        printf("(%p, %zd)", sg[i].iov_base, sg[i].iov_len);
+    }
+    printf("}\n");
+}
+
+/* Will call this only for path name based fid */
+static void v9fs_fix_path(V9fsPath *dst, V9fsPath *src, int len)
+{
+    V9fsPath str;
+    v9fs_path_init(&str);
+    v9fs_path_copy(&str, dst);
+    v9fs_string_sprintf((V9fsString *)dst, "%s%s", src->data, str.data+len);
+    v9fs_path_free(&str);
+    /* +1 to include terminating NULL */
+    dst->size++;
+}
+
+static inline bool is_ro_export(FsContext *ctx)
+{
+    return ctx->export_flags & V9FS_RDONLY;
+}
+
+static void v9fs_version(void *opaque)
+{
+    ssize_t err;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+    V9fsString version;
+    size_t offset = 7;
+
+    v9fs_string_init(&version);
+    err = pdu_unmarshal(pdu, offset, "ds", &s->msize, &version);
+    if (err < 0) {
+        offset = err;
+        goto out;
+    }
+    trace_v9fs_version(pdu->tag, pdu->id, s->msize, version.data);
+
+    virtfs_reset(pdu);
+
+    if (!strcmp(version.data, "9P2000.u")) {
+        s->proto_version = V9FS_PROTO_2000U;
+    } else if (!strcmp(version.data, "9P2000.L")) {
+        s->proto_version = V9FS_PROTO_2000L;
+    } else {
+        v9fs_string_sprintf(&version, "unknown");
+    }
+
+    err = pdu_marshal(pdu, offset, "ds", s->msize, &version);
+    if (err < 0) {
+        offset = err;
+        goto out;
+    }
+    offset += err;
+    trace_v9fs_version_return(pdu->tag, pdu->id, s->msize, version.data);
+out:
+    pdu_complete(pdu, offset);
+    v9fs_string_free(&version);
+}
+
+static void v9fs_attach(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+    int32_t fid, afid, n_uname;
+    V9fsString uname, aname;
+    V9fsFidState *fidp;
+    size_t offset = 7;
+    V9fsQID qid;
+    ssize_t err;
+
+    v9fs_string_init(&uname);
+    v9fs_string_init(&aname);
+    err = pdu_unmarshal(pdu, offset, "ddssd", &fid,
+                        &afid, &uname, &aname, &n_uname);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_attach(pdu->tag, pdu->id, fid, afid, uname.data, aname.data);
+
+    fidp = alloc_fid(s, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    fidp->uid = n_uname;
+    err = v9fs_co_name_to_path(pdu, NULL, "/", &fidp->path);
+    if (err < 0) {
+        err = -EINVAL;
+        clunk_fid(s, fid);
+        goto out;
+    }
+    err = fid_to_qid(pdu, fidp, &qid);
+    if (err < 0) {
+        err = -EINVAL;
+        clunk_fid(s, fid);
+        goto out;
+    }
+    err = pdu_marshal(pdu, offset, "Q", &qid);
+    if (err < 0) {
+        clunk_fid(s, fid);
+        goto out;
+    }
+    err += offset;
+    memcpy(&s->root_qid, &qid, sizeof(qid));
+    trace_v9fs_attach_return(pdu->tag, pdu->id,
+                             qid.type, qid.version, qid.path);
+    /*
+     * disable migration if we haven't done already.
+     * attach could get called multiple times for the same export.
+     */
+    if (!s->migration_blocker) {
+        s->root_fid = fid;
+        error_setg(&s->migration_blocker,
+                   "Migration is disabled when VirtFS export path '%s' is mounted in the guest using mount_tag '%s'",
+                   s->ctx.fs_root ? s->ctx.fs_root : "NULL", s->tag);
+        migrate_add_blocker(s->migration_blocker);
+    }
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&uname);
+    v9fs_string_free(&aname);
+}
+
+static void v9fs_stat(void *opaque)
+{
+    int32_t fid;
+    V9fsStat v9stat;
+    ssize_t err = 0;
+    size_t offset = 7;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+
+    err = pdu_unmarshal(pdu, offset, "d", &fid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_stat(pdu->tag, pdu->id, fid);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    err = stat_to_v9stat(pdu, &fidp->path, &stbuf, &v9stat);
+    if (err < 0) {
+        goto out;
+    }
+    err = pdu_marshal(pdu, offset, "wS", 0, &v9stat);
+    if (err < 0) {
+        v9fs_stat_free(&v9stat);
+        goto out;
+    }
+    trace_v9fs_stat_return(pdu->tag, pdu->id, v9stat.mode,
+                           v9stat.atime, v9stat.mtime, v9stat.length);
+    err += offset;
+    v9fs_stat_free(&v9stat);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static void v9fs_getattr(void *opaque)
+{
+    int32_t fid;
+    size_t offset = 7;
+    ssize_t retval = 0;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    uint64_t request_mask;
+    V9fsStatDotl v9stat_dotl;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    retval = pdu_unmarshal(pdu, offset, "dq", &fid, &request_mask);
+    if (retval < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_getattr(pdu->tag, pdu->id, fid, request_mask);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        retval = -ENOENT;
+        goto out_nofid;
+    }
+    /*
+     * Currently we only support BASIC fields in stat, so there is no
+     * need to look at request_mask.
+     */
+    retval = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
+    if (retval < 0) {
+        goto out;
+    }
+    stat_to_v9stat_dotl(s, &stbuf, &v9stat_dotl);
+
+    /*  fill st_gen if requested and supported by underlying fs */
+    if (request_mask & P9_STATS_GEN) {
+        retval = v9fs_co_st_gen(pdu, &fidp->path, stbuf.st_mode, &v9stat_dotl);
+        switch (retval) {
+        case 0:
+            /* we have valid st_gen: update result mask */
+            v9stat_dotl.st_result_mask |= P9_STATS_GEN;
+            break;
+        case -EINTR:
+            /* request cancelled, e.g. by Tflush */
+            goto out;
+        default:
+            /* failed to get st_gen: not fatal, ignore */
+            break;
+        }
+    }
+    retval = pdu_marshal(pdu, offset, "A", &v9stat_dotl);
+    if (retval < 0) {
+        goto out;
+    }
+    retval += offset;
+    trace_v9fs_getattr_return(pdu->tag, pdu->id, v9stat_dotl.st_result_mask,
+                              v9stat_dotl.st_mode, v9stat_dotl.st_uid,
+                              v9stat_dotl.st_gid);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, retval);
+}
+
+/* Attribute flags */
+#define P9_ATTR_MODE       (1 << 0)
+#define P9_ATTR_UID        (1 << 1)
+#define P9_ATTR_GID        (1 << 2)
+#define P9_ATTR_SIZE       (1 << 3)
+#define P9_ATTR_ATIME      (1 << 4)
+#define P9_ATTR_MTIME      (1 << 5)
+#define P9_ATTR_CTIME      (1 << 6)
+#define P9_ATTR_ATIME_SET  (1 << 7)
+#define P9_ATTR_MTIME_SET  (1 << 8)
+
+#define P9_ATTR_MASK    127
+
+static void v9fs_setattr(void *opaque)
+{
+    int err = 0;
+    int32_t fid;
+    V9fsFidState *fidp;
+    size_t offset = 7;
+    V9fsIattr v9iattr;
+    V9fsPDU *pdu = opaque;
+
+    err = pdu_unmarshal(pdu, offset, "dI", &fid, &v9iattr);
+    if (err < 0) {
+        goto out_nofid;
+    }
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    if (v9iattr.valid & P9_ATTR_MODE) {
+        err = v9fs_co_chmod(pdu, &fidp->path, v9iattr.mode);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    if (v9iattr.valid & (P9_ATTR_ATIME | P9_ATTR_MTIME)) {
+        struct timespec times[2];
+        if (v9iattr.valid & P9_ATTR_ATIME) {
+            if (v9iattr.valid & P9_ATTR_ATIME_SET) {
+                times[0].tv_sec = v9iattr.atime_sec;
+                times[0].tv_nsec = v9iattr.atime_nsec;
+            } else {
+                times[0].tv_nsec = UTIME_NOW;
+            }
+        } else {
+            times[0].tv_nsec = UTIME_OMIT;
+        }
+        if (v9iattr.valid & P9_ATTR_MTIME) {
+            if (v9iattr.valid & P9_ATTR_MTIME_SET) {
+                times[1].tv_sec = v9iattr.mtime_sec;
+                times[1].tv_nsec = v9iattr.mtime_nsec;
+            } else {
+                times[1].tv_nsec = UTIME_NOW;
+            }
+        } else {
+            times[1].tv_nsec = UTIME_OMIT;
+        }
+        err = v9fs_co_utimensat(pdu, &fidp->path, times);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    /*
+     * If the only valid entry in iattr is ctime we can call
+     * chown(-1,-1) to update the ctime of the file
+     */
+    if ((v9iattr.valid & (P9_ATTR_UID | P9_ATTR_GID)) ||
+        ((v9iattr.valid & P9_ATTR_CTIME)
+         && !((v9iattr.valid & P9_ATTR_MASK) & ~P9_ATTR_CTIME))) {
+        if (!(v9iattr.valid & P9_ATTR_UID)) {
+            v9iattr.uid = -1;
+        }
+        if (!(v9iattr.valid & P9_ATTR_GID)) {
+            v9iattr.gid = -1;
+        }
+        err = v9fs_co_chown(pdu, &fidp->path, v9iattr.uid,
+                            v9iattr.gid);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    if (v9iattr.valid & (P9_ATTR_SIZE)) {
+        err = v9fs_co_truncate(pdu, &fidp->path, v9iattr.size);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    err = offset;
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static int v9fs_walk_marshal(V9fsPDU *pdu, uint16_t nwnames, V9fsQID *qids)
+{
+    int i;
+    ssize_t err;
+    size_t offset = 7;
+
+    err = pdu_marshal(pdu, offset, "w", nwnames);
+    if (err < 0) {
+        return err;
+    }
+    offset += err;
+    for (i = 0; i < nwnames; i++) {
+        err = pdu_marshal(pdu, offset, "Q", &qids[i]);
+        if (err < 0) {
+            return err;
+        }
+        offset += err;
+    }
+    return offset;
+}
+
+static bool name_is_illegal(const char *name)
+{
+    return !*name || strchr(name, '/') != NULL;
+}
+
+static bool not_same_qid(const V9fsQID *qid1, const V9fsQID *qid2)
+{
+    return
+        qid1->type != qid2->type ||
+        qid1->version != qid2->version ||
+        qid1->path != qid2->path;
+}
+
+static void v9fs_walk(void *opaque)
+{
+    int name_idx;
+    V9fsQID *qids = NULL;
+    int i, err = 0;
+    V9fsPath dpath, path;
+    uint16_t nwnames;
+    struct stat stbuf;
+    size_t offset = 7;
+    int32_t fid, newfid;
+    V9fsString *wnames = NULL;
+    V9fsFidState *fidp;
+    V9fsFidState *newfidp = NULL;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+    V9fsQID qid;
+
+    err = pdu_unmarshal(pdu, offset, "ddw", &fid, &newfid, &nwnames);
+    if (err < 0) {
+        pdu_complete(pdu, err);
+        return ;
+    }
+    offset += err;
+
+    trace_v9fs_walk(pdu->tag, pdu->id, fid, newfid, nwnames);
+
+    if (nwnames && nwnames <= P9_MAXWELEM) {
+        wnames = g_malloc0(sizeof(wnames[0]) * nwnames);
+        qids   = g_malloc0(sizeof(qids[0]) * nwnames);
+        for (i = 0; i < nwnames; i++) {
+            err = pdu_unmarshal(pdu, offset, "s", &wnames[i]);
+            if (err < 0) {
+                goto out_nofid;
+            }
+            if (name_is_illegal(wnames[i].data)) {
+                err = -ENOENT;
+                goto out_nofid;
+            }
+            offset += err;
+        }
+    } else if (nwnames > P9_MAXWELEM) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    err = fid_to_qid(pdu, fidp, &qid);
+    if (err < 0) {
+        goto out;
+    }
+
+    v9fs_path_init(&dpath);
+    v9fs_path_init(&path);
+    /*
+     * Both dpath and path initially poin to fidp.
+     * Needed to handle request with nwnames == 0
+     */
+    v9fs_path_copy(&dpath, &fidp->path);
+    v9fs_path_copy(&path, &fidp->path);
+    for (name_idx = 0; name_idx < nwnames; name_idx++) {
+        if (not_same_qid(&pdu->s->root_qid, &qid) ||
+            strcmp("..", wnames[name_idx].data)) {
+            err = v9fs_co_name_to_path(pdu, &dpath, wnames[name_idx].data,
+                                       &path);
+            if (err < 0) {
+                goto out;
+            }
+
+            err = v9fs_co_lstat(pdu, &path, &stbuf);
+            if (err < 0) {
+                goto out;
+            }
+            stat_to_qid(&stbuf, &qid);
+            v9fs_path_copy(&dpath, &path);
+        }
+        memcpy(&qids[name_idx], &qid, sizeof(qid));
+    }
+    if (fid == newfid) {
+        BUG_ON(fidp->fid_type != P9_FID_NONE);
+        v9fs_path_copy(&fidp->path, &path);
+    } else {
+        newfidp = alloc_fid(s, newfid);
+        if (newfidp == NULL) {
+            err = -EINVAL;
+            goto out;
+        }
+        newfidp->uid = fidp->uid;
+        v9fs_path_copy(&newfidp->path, &path);
+    }
+    err = v9fs_walk_marshal(pdu, nwnames, qids);
+    trace_v9fs_walk_return(pdu->tag, pdu->id, nwnames, qids);
+out:
+    put_fid(pdu, fidp);
+    if (newfidp) {
+        put_fid(pdu, newfidp);
+    }
+    v9fs_path_free(&dpath);
+    v9fs_path_free(&path);
+out_nofid:
+    pdu_complete(pdu, err);
+    if (nwnames && nwnames <= P9_MAXWELEM) {
+        for (name_idx = 0; name_idx < nwnames; name_idx++) {
+            v9fs_string_free(&wnames[name_idx]);
+        }
+        g_free(wnames);
+        g_free(qids);
+    }
+}
+
+static int32_t get_iounit(V9fsPDU *pdu, V9fsPath *path)
+{
+    struct statfs stbuf;
+    int32_t iounit = 0;
+    V9fsState *s = pdu->s;
+
+    /*
+     * iounit should be multiples of f_bsize (host filesystem block size
+     * and as well as less than (client msize - P9_IOHDRSZ))
+     */
+    if (!v9fs_co_statfs(pdu, path, &stbuf)) {
+        iounit = stbuf.f_bsize;
+        iounit *= (s->msize - P9_IOHDRSZ)/stbuf.f_bsize;
+    }
+    if (!iounit) {
+        iounit = s->msize - P9_IOHDRSZ;
+    }
+    return iounit;
+}
+
+static void v9fs_open(void *opaque)
+{
+    int flags;
+    int32_t fid;
+    int32_t mode;
+    V9fsQID qid;
+    int iounit = 0;
+    ssize_t err = 0;
+    size_t offset = 7;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    if (s->proto_version == V9FS_PROTO_2000L) {
+        err = pdu_unmarshal(pdu, offset, "dd", &fid, &mode);
+    } else {
+        uint8_t modebyte;
+        err = pdu_unmarshal(pdu, offset, "db", &fid, &modebyte);
+        mode = modebyte;
+    }
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_open(pdu->tag, pdu->id, fid, mode);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    BUG_ON(fidp->fid_type != P9_FID_NONE);
+
+    err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    stat_to_qid(&stbuf, &qid);
+    if (S_ISDIR(stbuf.st_mode)) {
+        err = v9fs_co_opendir(pdu, fidp);
+        if (err < 0) {
+            goto out;
+        }
+        fidp->fid_type = P9_FID_DIR;
+        err = pdu_marshal(pdu, offset, "Qd", &qid, 0);
+        if (err < 0) {
+            goto out;
+        }
+        err += offset;
+    } else {
+        if (s->proto_version == V9FS_PROTO_2000L) {
+            flags = get_dotl_openflags(s, mode);
+        } else {
+            flags = omode_to_uflags(mode);
+        }
+        if (is_ro_export(&s->ctx)) {
+            if (mode & O_WRONLY || mode & O_RDWR ||
+                mode & O_APPEND || mode & O_TRUNC) {
+                err = -EROFS;
+                goto out;
+            }
+        }
+        err = v9fs_co_open(pdu, fidp, flags);
+        if (err < 0) {
+            goto out;
+        }
+        fidp->fid_type = P9_FID_FILE;
+        fidp->open_flags = flags;
+        if (flags & O_EXCL) {
+            /*
+             * We let the host file system do O_EXCL check
+             * We should not reclaim such fd
+             */
+            fidp->flags |= FID_NON_RECLAIMABLE;
+        }
+        iounit = get_iounit(pdu, &fidp->path);
+        err = pdu_marshal(pdu, offset, "Qd", &qid, iounit);
+        if (err < 0) {
+            goto out;
+        }
+        err += offset;
+    }
+    trace_v9fs_open_return(pdu->tag, pdu->id,
+                           qid.type, qid.version, qid.path, iounit);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static void v9fs_lcreate(void *opaque)
+{
+    int32_t dfid, flags, mode;
+    gid_t gid;
+    ssize_t err = 0;
+    ssize_t offset = 7;
+    V9fsString name;
+    V9fsFidState *fidp;
+    struct stat stbuf;
+    V9fsQID qid;
+    int32_t iounit;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dsddd", &dfid,
+                        &name, &flags, &mode, &gid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_lcreate(pdu->tag, pdu->id, dfid, flags, mode, gid);
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EEXIST;
+        goto out_nofid;
+    }
+
+    fidp = get_fid(pdu, dfid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    flags = get_dotl_openflags(pdu->s, flags);
+    err = v9fs_co_open2(pdu, fidp, &name, gid,
+                        flags | O_CREAT, mode, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    fidp->fid_type = P9_FID_FILE;
+    fidp->open_flags = flags;
+    if (flags & O_EXCL) {
+        /*
+         * We let the host file system do O_EXCL check
+         * We should not reclaim such fd
+         */
+        fidp->flags |= FID_NON_RECLAIMABLE;
+    }
+    iounit =  get_iounit(pdu, &fidp->path);
+    stat_to_qid(&stbuf, &qid);
+    err = pdu_marshal(pdu, offset, "Qd", &qid, iounit);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_lcreate_return(pdu->tag, pdu->id,
+                              qid.type, qid.version, qid.path, iounit);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+static void v9fs_fsync(void *opaque)
+{
+    int err;
+    int32_t fid;
+    int datasync;
+    size_t offset = 7;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+
+    err = pdu_unmarshal(pdu, offset, "dd", &fid, &datasync);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_fsync(pdu->tag, pdu->id, fid, datasync);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    err = v9fs_co_fsync(pdu, fidp, datasync);
+    if (!err) {
+        err = offset;
+    }
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static void v9fs_clunk(void *opaque)
+{
+    int err;
+    int32_t fid;
+    size_t offset = 7;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    err = pdu_unmarshal(pdu, offset, "d", &fid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_clunk(pdu->tag, pdu->id, fid);
+
+    fidp = clunk_fid(s, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    /*
+     * Bump the ref so that put_fid will
+     * free the fid.
+     */
+    fidp->ref++;
+    err = put_fid(pdu, fidp);
+    if (!err) {
+        err = offset;
+    }
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static int v9fs_xattr_read(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
+                           uint64_t off, uint32_t max_count)
+{
+    ssize_t err;
+    size_t offset = 7;
+    int read_count;
+    int64_t xattr_len;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    xattr_len = fidp->fs.xattr.len;
+    read_count = xattr_len - off;
+    if (read_count > max_count) {
+        read_count = max_count;
+    } else if (read_count < 0) {
+        /*
+         * read beyond XATTR value
+         */
+        read_count = 0;
+    }
+    err = pdu_marshal(pdu, offset, "d", read_count);
+    if (err < 0) {
+        return err;
+    }
+    offset += err;
+
+    err = v9fs_pack(elem->in_sg, elem->in_num, offset,
+                    ((char *)fidp->fs.xattr.value) + off,
+                    read_count);
+    if (err < 0) {
+        return err;
+    }
+    offset += err;
+    return offset;
+}
+
+static int v9fs_do_readdir_with_stat(V9fsPDU *pdu,
+                                     V9fsFidState *fidp, uint32_t max_count)
+{
+    V9fsPath path;
+    V9fsStat v9stat;
+    int len, err = 0;
+    int32_t count = 0;
+    struct stat stbuf;
+    off_t saved_dir_pos;
+    struct dirent *dent;
+
+    /* save the directory position */
+    saved_dir_pos = v9fs_co_telldir(pdu, fidp);
+    if (saved_dir_pos < 0) {
+        return saved_dir_pos;
+    }
+
+    while (1) {
+        v9fs_path_init(&path);
+
+        v9fs_readdir_lock(&fidp->fs.dir);
+
+        err = v9fs_co_readdir(pdu, fidp, &dent);
+        if (err || !dent) {
+            break;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, dent->d_name, &path);
+        if (err < 0) {
+            break;
+        }
+        err = v9fs_co_lstat(pdu, &path, &stbuf);
+        if (err < 0) {
+            break;
+        }
+        err = stat_to_v9stat(pdu, &path, &stbuf, &v9stat);
+        if (err < 0) {
+            break;
+        }
+        /* 11 = 7 + 4 (7 = start offset, 4 = space for storing count) */
+        len = pdu_marshal(pdu, 11 + count, "S", &v9stat);
+
+        v9fs_readdir_unlock(&fidp->fs.dir);
+
+        if ((len != (v9stat.size + 2)) || ((count + len) > max_count)) {
+            /* Ran out of buffer. Set dir back to old position and return */
+            v9fs_co_seekdir(pdu, fidp, saved_dir_pos);
+            v9fs_stat_free(&v9stat);
+            v9fs_path_free(&path);
+            return count;
+        }
+        count += len;
+        v9fs_stat_free(&v9stat);
+        v9fs_path_free(&path);
+        saved_dir_pos = dent->d_off;
+    }
+
+    v9fs_readdir_unlock(&fidp->fs.dir);
+
+    v9fs_path_free(&path);
+    if (err < 0) {
+        return err;
+    }
+    return count;
+}
+
+/*
+ * Create a QEMUIOVector for a sub-region of PDU iovecs
+ *
+ * @qiov:       uninitialized QEMUIOVector
+ * @skip:       number of bytes to skip from beginning of PDU
+ * @size:       number of bytes to include
+ * @is_write:   true - write, false - read
+ *
+ * The resulting QEMUIOVector has heap-allocated iovecs and must be cleaned up
+ * with qemu_iovec_destroy().
+ */
+static void v9fs_init_qiov_from_pdu(QEMUIOVector *qiov, V9fsPDU *pdu,
+                                    size_t skip, size_t size,
+                                    bool is_write)
+{
+    QEMUIOVector elem;
+    struct iovec *iov;
+    unsigned int niov;
+
+    virtio_init_iov_from_pdu(pdu, &iov, &niov, is_write);
+
+    qemu_iovec_init_external(&elem, iov, niov);
+    qemu_iovec_init(qiov, niov);
+    qemu_iovec_concat(qiov, &elem, skip, size);
+}
+
+static void v9fs_read(void *opaque)
+{
+    int32_t fid;
+    uint64_t off;
+    ssize_t err = 0;
+    int32_t count = 0;
+    size_t offset = 7;
+    uint32_t max_count;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    err = pdu_unmarshal(pdu, offset, "dqd", &fid, &off, &max_count);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_read(pdu->tag, pdu->id, fid, off, max_count);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    if (fidp->fid_type == P9_FID_DIR) {
+
+        if (off == 0) {
+            v9fs_co_rewinddir(pdu, fidp);
+        }
+        count = v9fs_do_readdir_with_stat(pdu, fidp, max_count);
+        if (count < 0) {
+            err = count;
+            goto out;
+        }
+        err = pdu_marshal(pdu, offset, "d", count);
+        if (err < 0) {
+            goto out;
+        }
+        err += offset + count;
+    } else if (fidp->fid_type == P9_FID_FILE) {
+        QEMUIOVector qiov_full;
+        QEMUIOVector qiov;
+        int32_t len;
+
+        v9fs_init_qiov_from_pdu(&qiov_full, pdu, offset + 4, max_count, false);
+        qemu_iovec_init(&qiov, qiov_full.niov);
+        do {
+            qemu_iovec_reset(&qiov);
+            qemu_iovec_concat(&qiov, &qiov_full, count, qiov_full.size - count);
+            if (0) {
+                print_sg(qiov.iov, qiov.niov);
+            }
+            /* Loop in case of EINTR */
+            do {
+                len = v9fs_co_preadv(pdu, fidp, qiov.iov, qiov.niov, off);
+                if (len >= 0) {
+                    off   += len;
+                    count += len;
+                }
+            } while (len == -EINTR && !pdu->cancelled);
+            if (len < 0) {
+                /* IO error return the error */
+                err = len;
+                goto out;
+            }
+        } while (count < max_count && len > 0);
+        err = pdu_marshal(pdu, offset, "d", count);
+        if (err < 0) {
+            goto out;
+        }
+        err += offset + count;
+        qemu_iovec_destroy(&qiov);
+        qemu_iovec_destroy(&qiov_full);
+    } else if (fidp->fid_type == P9_FID_XATTR) {
+        err = v9fs_xattr_read(s, pdu, fidp, off, max_count);
+    } else {
+        err = -EINVAL;
+    }
+    trace_v9fs_read_return(pdu->tag, pdu->id, count, err);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static size_t v9fs_readdir_data_size(V9fsString *name)
+{
+    /*
+     * Size of each dirent on the wire: size of qid (13) + size of offset (8)
+     * size of type (1) + size of name.size (2) + strlen(name.data)
+     */
+    return 24 + v9fs_string_size(name);
+}
+
+static int v9fs_do_readdir(V9fsPDU *pdu,
+                           V9fsFidState *fidp, int32_t max_count)
+{
+    size_t size;
+    V9fsQID qid;
+    V9fsString name;
+    int len, err = 0;
+    int32_t count = 0;
+    off_t saved_dir_pos;
+    struct dirent *dent;
+
+    /* save the directory position */
+    saved_dir_pos = v9fs_co_telldir(pdu, fidp);
+    if (saved_dir_pos < 0) {
+        return saved_dir_pos;
+    }
+
+    while (1) {
+        v9fs_readdir_lock(&fidp->fs.dir);
+
+        err = v9fs_co_readdir(pdu, fidp, &dent);
+        if (err || !dent) {
+            break;
+        }
+        v9fs_string_init(&name);
+        v9fs_string_sprintf(&name, "%s", dent->d_name);
+        if ((count + v9fs_readdir_data_size(&name)) > max_count) {
+            v9fs_readdir_unlock(&fidp->fs.dir);
+
+            /* Ran out of buffer. Set dir back to old position and return */
+            v9fs_co_seekdir(pdu, fidp, saved_dir_pos);
+            v9fs_string_free(&name);
+            return count;
+        }
+        /*
+         * Fill up just the path field of qid because the client uses
+         * only that. To fill the entire qid structure we will have
+         * to stat each dirent found, which is expensive
+         */
+        size = MIN(sizeof(dent->d_ino), sizeof(qid.path));
+        memcpy(&qid.path, &dent->d_ino, size);
+        /* Fill the other fields with dummy values */
+        qid.type = 0;
+        qid.version = 0;
+
+        /* 11 = 7 + 4 (7 = start offset, 4 = space for storing count) */
+        len = pdu_marshal(pdu, 11 + count, "Qqbs",
+                          &qid, dent->d_off,
+                          dent->d_type, &name);
+
+        v9fs_readdir_unlock(&fidp->fs.dir);
+
+        if (len < 0) {
+            v9fs_co_seekdir(pdu, fidp, saved_dir_pos);
+            v9fs_string_free(&name);
+            return len;
+        }
+        count += len;
+        v9fs_string_free(&name);
+        saved_dir_pos = dent->d_off;
+    }
+
+    v9fs_readdir_unlock(&fidp->fs.dir);
+
+    if (err < 0) {
+        return err;
+    }
+    return count;
+}
+
+static void v9fs_readdir(void *opaque)
+{
+    int32_t fid;
+    V9fsFidState *fidp;
+    ssize_t retval = 0;
+    size_t offset = 7;
+    uint64_t initial_offset;
+    int32_t count;
+    uint32_t max_count;
+    V9fsPDU *pdu = opaque;
+
+    retval = pdu_unmarshal(pdu, offset, "dqd", &fid,
+                           &initial_offset, &max_count);
+    if (retval < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_readdir(pdu->tag, pdu->id, fid, initial_offset, max_count);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        retval = -EINVAL;
+        goto out_nofid;
+    }
+    if (!fidp->fs.dir.stream) {
+        retval = -EINVAL;
+        goto out;
+    }
+    if (initial_offset == 0) {
+        v9fs_co_rewinddir(pdu, fidp);
+    } else {
+        v9fs_co_seekdir(pdu, fidp, initial_offset);
+    }
+    count = v9fs_do_readdir(pdu, fidp, max_count);
+    if (count < 0) {
+        retval = count;
+        goto out;
+    }
+    retval = pdu_marshal(pdu, offset, "d", count);
+    if (retval < 0) {
+        goto out;
+    }
+    retval += count + offset;
+    trace_v9fs_readdir_return(pdu->tag, pdu->id, count, retval);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, retval);
+}
+
+static int v9fs_xattr_write(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
+                            uint64_t off, uint32_t count,
+                            struct iovec *sg, int cnt)
+{
+    int i, to_copy;
+    ssize_t err = 0;
+    int write_count;
+    int64_t xattr_len;
+    size_t offset = 7;
+
+
+    xattr_len = fidp->fs.xattr.len;
+    write_count = xattr_len - off;
+    if (write_count > count) {
+        write_count = count;
+    } else if (write_count < 0) {
+        /*
+         * write beyond XATTR value len specified in
+         * xattrcreate
+         */
+        err = -ENOSPC;
+        goto out;
+    }
+    err = pdu_marshal(pdu, offset, "d", write_count);
+    if (err < 0) {
+        return err;
+    }
+    err += offset;
+    fidp->fs.xattr.copied_len += write_count;
+    /*
+     * Now copy the content from sg list
+     */
+    for (i = 0; i < cnt; i++) {
+        if (write_count > sg[i].iov_len) {
+            to_copy = sg[i].iov_len;
+        } else {
+            to_copy = write_count;
+        }
+        memcpy((char *)fidp->fs.xattr.value + off, sg[i].iov_base, to_copy);
+        /* updating vs->off since we are not using below */
+        off += to_copy;
+        write_count -= to_copy;
+    }
+out:
+    return err;
+}
+
+static void v9fs_write(void *opaque)
+{
+    ssize_t err;
+    int32_t fid;
+    uint64_t off;
+    uint32_t count;
+    int32_t len = 0;
+    int32_t total = 0;
+    size_t offset = 7;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+    QEMUIOVector qiov_full;
+    QEMUIOVector qiov;
+
+    err = pdu_unmarshal(pdu, offset, "dqd", &fid, &off, &count);
+    if (err < 0) {
+        pdu_complete(pdu, err);
+        return;
+    }
+    offset += err;
+    v9fs_init_qiov_from_pdu(&qiov_full, pdu, offset, count, true);
+    trace_v9fs_write(pdu->tag, pdu->id, fid, off, count, qiov_full.niov);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    if (fidp->fid_type == P9_FID_FILE) {
+        if (fidp->fs.fd == -1) {
+            err = -EINVAL;
+            goto out;
+        }
+    } else if (fidp->fid_type == P9_FID_XATTR) {
+        /*
+         * setxattr operation
+         */
+        err = v9fs_xattr_write(s, pdu, fidp, off, count,
+                               qiov_full.iov, qiov_full.niov);
+        goto out;
+    } else {
+        err = -EINVAL;
+        goto out;
+    }
+    qemu_iovec_init(&qiov, qiov_full.niov);
+    do {
+        qemu_iovec_reset(&qiov);
+        qemu_iovec_concat(&qiov, &qiov_full, total, qiov_full.size - total);
+        if (0) {
+            print_sg(qiov.iov, qiov.niov);
+        }
+        /* Loop in case of EINTR */
+        do {
+            len = v9fs_co_pwritev(pdu, fidp, qiov.iov, qiov.niov, off);
+            if (len >= 0) {
+                off   += len;
+                total += len;
+            }
+        } while (len == -EINTR && !pdu->cancelled);
+        if (len < 0) {
+            /* IO error return the error */
+            err = len;
+            goto out_qiov;
+        }
+    } while (total < count && len > 0);
+
+    offset = 7;
+    err = pdu_marshal(pdu, offset, "d", total);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_write_return(pdu->tag, pdu->id, total, err);
+out_qiov:
+    qemu_iovec_destroy(&qiov);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    qemu_iovec_destroy(&qiov_full);
+    pdu_complete(pdu, err);
+}
+
+static void v9fs_create(void *opaque)
+{
+    int32_t fid;
+    int err = 0;
+    size_t offset = 7;
+    V9fsFidState *fidp;
+    V9fsQID qid;
+    int32_t perm;
+    int8_t mode;
+    V9fsPath path;
+    struct stat stbuf;
+    V9fsString name;
+    V9fsString extension;
+    int iounit;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_path_init(&path);
+    v9fs_string_init(&name);
+    v9fs_string_init(&extension);
+    err = pdu_unmarshal(pdu, offset, "dsdbs", &fid, &name,
+                        &perm, &mode, &extension);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_create(pdu->tag, pdu->id, fid, name.data, perm, mode);
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EEXIST;
+        goto out_nofid;
+    }
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    if (perm & P9_STAT_MODE_DIR) {
+        err = v9fs_co_mkdir(pdu, fidp, &name, perm & 0777,
+                            fidp->uid, -1, &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
+        if (err < 0) {
+            goto out;
+        }
+        v9fs_path_copy(&fidp->path, &path);
+        err = v9fs_co_opendir(pdu, fidp);
+        if (err < 0) {
+            goto out;
+        }
+        fidp->fid_type = P9_FID_DIR;
+    } else if (perm & P9_STAT_MODE_SYMLINK) {
+        err = v9fs_co_symlink(pdu, fidp, &name,
+                              extension.data, -1 , &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
+        if (err < 0) {
+            goto out;
+        }
+        v9fs_path_copy(&fidp->path, &path);
+    } else if (perm & P9_STAT_MODE_LINK) {
+        int32_t ofid = atoi(extension.data);
+        V9fsFidState *ofidp = get_fid(pdu, ofid);
+        if (ofidp == NULL) {
+            err = -EINVAL;
+            goto out;
+        }
+        err = v9fs_co_link(pdu, ofidp, fidp, &name);
+        put_fid(pdu, ofidp);
+        if (err < 0) {
+            goto out;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
+        if (err < 0) {
+            fidp->fid_type = P9_FID_NONE;
+            goto out;
+        }
+        v9fs_path_copy(&fidp->path, &path);
+        err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
+        if (err < 0) {
+            fidp->fid_type = P9_FID_NONE;
+            goto out;
+        }
+    } else if (perm & P9_STAT_MODE_DEVICE) {
+        char ctype;
+        uint32_t major, minor;
+        mode_t nmode = 0;
+
+        if (sscanf(extension.data, "%c %u %u", &ctype, &major, &minor) != 3) {
+            err = -errno;
+            goto out;
+        }
+
+        switch (ctype) {
+        case 'c':
+            nmode = S_IFCHR;
+            break;
+        case 'b':
+            nmode = S_IFBLK;
+            break;
+        default:
+            err = -EIO;
+            goto out;
+        }
+
+        nmode |= perm & 0777;
+        err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, -1,
+                            makedev(major, minor), nmode, &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
+        if (err < 0) {
+            goto out;
+        }
+        v9fs_path_copy(&fidp->path, &path);
+    } else if (perm & P9_STAT_MODE_NAMED_PIPE) {
+        err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, -1,
+                            0, S_IFIFO | (perm & 0777), &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
+        if (err < 0) {
+            goto out;
+        }
+        v9fs_path_copy(&fidp->path, &path);
+    } else if (perm & P9_STAT_MODE_SOCKET) {
+        err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, -1,
+                            0, S_IFSOCK | (perm & 0777), &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
+        if (err < 0) {
+            goto out;
+        }
+        v9fs_path_copy(&fidp->path, &path);
+    } else {
+        err = v9fs_co_open2(pdu, fidp, &name, -1,
+                            omode_to_uflags(mode)|O_CREAT, perm, &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        fidp->fid_type = P9_FID_FILE;
+        fidp->open_flags = omode_to_uflags(mode);
+        if (fidp->open_flags & O_EXCL) {
+            /*
+             * We let the host file system do O_EXCL check
+             * We should not reclaim such fd
+             */
+            fidp->flags |= FID_NON_RECLAIMABLE;
+        }
+    }
+    iounit = get_iounit(pdu, &fidp->path);
+    stat_to_qid(&stbuf, &qid);
+    err = pdu_marshal(pdu, offset, "Qd", &qid, iounit);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_create_return(pdu->tag, pdu->id,
+                             qid.type, qid.version, qid.path, iounit);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+   pdu_complete(pdu, err);
+   v9fs_string_free(&name);
+   v9fs_string_free(&extension);
+   v9fs_path_free(&path);
+}
+
+static void v9fs_symlink(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    V9fsString name;
+    V9fsString symname;
+    V9fsFidState *dfidp;
+    V9fsQID qid;
+    struct stat stbuf;
+    int32_t dfid;
+    int err = 0;
+    gid_t gid;
+    size_t offset = 7;
+
+    v9fs_string_init(&name);
+    v9fs_string_init(&symname);
+    err = pdu_unmarshal(pdu, offset, "dssd", &dfid, &name, &symname, &gid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_symlink(pdu->tag, pdu->id, dfid, name.data, symname.data, gid);
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EEXIST;
+        goto out_nofid;
+    }
+
+    dfidp = get_fid(pdu, dfid);
+    if (dfidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    err = v9fs_co_symlink(pdu, dfidp, &name, symname.data, gid, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    stat_to_qid(&stbuf, &qid);
+    err =  pdu_marshal(pdu, offset, "Q", &qid);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_symlink_return(pdu->tag, pdu->id,
+                              qid.type, qid.version, qid.path);
+out:
+    put_fid(pdu, dfidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+    v9fs_string_free(&symname);
+}
+
+static void v9fs_flush(void *opaque)
+{
+    ssize_t err;
+    int16_t tag;
+    size_t offset = 7;
+    V9fsPDU *cancel_pdu;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    err = pdu_unmarshal(pdu, offset, "w", &tag);
+    if (err < 0) {
+        pdu_complete(pdu, err);
+        return;
+    }
+    trace_v9fs_flush(pdu->tag, pdu->id, tag);
+
+    QLIST_FOREACH(cancel_pdu, &s->active_list, next) {
+        if (cancel_pdu->tag == tag) {
+            break;
+        }
+    }
+    if (cancel_pdu) {
+        cancel_pdu->cancelled = 1;
+        /*
+         * Wait for pdu to complete.
+         */
+        qemu_co_queue_wait(&cancel_pdu->complete);
+        cancel_pdu->cancelled = 0;
+        pdu_free(cancel_pdu);
+    }
+    pdu_complete(pdu, 7);
+}
+
+static void v9fs_link(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    int32_t dfid, oldfid;
+    V9fsFidState *dfidp, *oldfidp;
+    V9fsString name;
+    size_t offset = 7;
+    int err = 0;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dds", &dfid, &oldfid, &name);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_link(pdu->tag, pdu->id, dfid, oldfid, name.data);
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EEXIST;
+        goto out_nofid;
+    }
+
+    dfidp = get_fid(pdu, dfid);
+    if (dfidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    oldfidp = get_fid(pdu, oldfid);
+    if (oldfidp == NULL) {
+        err = -ENOENT;
+        goto out;
+    }
+    err = v9fs_co_link(pdu, oldfidp, dfidp, &name);
+    if (!err) {
+        err = offset;
+    }
+out:
+    put_fid(pdu, dfidp);
+out_nofid:
+    v9fs_string_free(&name);
+    pdu_complete(pdu, err);
+}
+
+/* Only works with path name based fid */
+static void v9fs_remove(void *opaque)
+{
+    int32_t fid;
+    int err = 0;
+    size_t offset = 7;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+
+    err = pdu_unmarshal(pdu, offset, "d", &fid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_remove(pdu->tag, pdu->id, fid);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    /* if fs driver is not path based, return EOPNOTSUPP */
+    if (!(pdu->s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT)) {
+        err = -EOPNOTSUPP;
+        goto out_err;
+    }
+    /*
+     * IF the file is unlinked, we cannot reopen
+     * the file later. So don't reclaim fd
+     */
+    err = v9fs_mark_fids_unreclaim(pdu, &fidp->path);
+    if (err < 0) {
+        goto out_err;
+    }
+    err = v9fs_co_remove(pdu, &fidp->path);
+    if (!err) {
+        err = offset;
+    }
+out_err:
+    /* For TREMOVE we need to clunk the fid even on failed remove */
+    clunk_fid(pdu->s, fidp->fid);
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static void v9fs_unlinkat(void *opaque)
+{
+    int err = 0;
+    V9fsString name;
+    int32_t dfid, flags;
+    size_t offset = 7;
+    V9fsPath path;
+    V9fsFidState *dfidp;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dsd", &dfid, &name, &flags);
+    if (err < 0) {
+        goto out_nofid;
+    }
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data)) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+
+    if (!strcmp("..", name.data)) {
+        err = -ENOTEMPTY;
+        goto out_nofid;
+    }
+
+    dfidp = get_fid(pdu, dfid);
+    if (dfidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    /*
+     * IF the file is unlinked, we cannot reopen
+     * the file later. So don't reclaim fd
+     */
+    v9fs_path_init(&path);
+    err = v9fs_co_name_to_path(pdu, &dfidp->path, name.data, &path);
+    if (err < 0) {
+        goto out_err;
+    }
+    err = v9fs_mark_fids_unreclaim(pdu, &path);
+    if (err < 0) {
+        goto out_err;
+    }
+    err = v9fs_co_unlinkat(pdu, &dfidp->path, &name, flags);
+    if (!err) {
+        err = offset;
+    }
+out_err:
+    put_fid(pdu, dfidp);
+    v9fs_path_free(&path);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+
+/* Only works with path name based fid */
+static int v9fs_complete_rename(V9fsPDU *pdu, V9fsFidState *fidp,
+                                int32_t newdirfid, V9fsString *name)
+{
+    char *end;
+    int err = 0;
+    V9fsPath new_path;
+    V9fsFidState *tfidp;
+    V9fsState *s = pdu->s;
+    V9fsFidState *dirfidp = NULL;
+    char *old_name, *new_name;
+
+    v9fs_path_init(&new_path);
+    if (newdirfid != -1) {
+        dirfidp = get_fid(pdu, newdirfid);
+        if (dirfidp == NULL) {
+            err = -ENOENT;
+            goto out_nofid;
+        }
+        BUG_ON(dirfidp->fid_type != P9_FID_NONE);
+        v9fs_co_name_to_path(pdu, &dirfidp->path, name->data, &new_path);
+    } else {
+        old_name = fidp->path.data;
+        end = strrchr(old_name, '/');
+        if (end) {
+            end++;
+        } else {
+            end = old_name;
+        }
+        new_name = g_malloc0(end - old_name + name->size + 1);
+        strncat(new_name, old_name, end - old_name);
+        strncat(new_name + (end - old_name), name->data, name->size);
+        v9fs_co_name_to_path(pdu, NULL, new_name, &new_path);
+        g_free(new_name);
+    }
+    err = v9fs_co_rename(pdu, &fidp->path, &new_path);
+    if (err < 0) {
+        goto out;
+    }
+    /*
+     * Fixup fid's pointing to the old name to
+     * start pointing to the new name
+     */
+    for (tfidp = s->fid_list; tfidp; tfidp = tfidp->next) {
+        if (v9fs_path_is_ancestor(&fidp->path, &tfidp->path)) {
+            /* replace the name */
+            v9fs_fix_path(&tfidp->path, &new_path, strlen(fidp->path.data));
+        }
+    }
+out:
+    if (dirfidp) {
+        put_fid(pdu, dirfidp);
+    }
+    v9fs_path_free(&new_path);
+out_nofid:
+    return err;
+}
+
+/* Only works with path name based fid */
+static void v9fs_rename(void *opaque)
+{
+    int32_t fid;
+    ssize_t err = 0;
+    size_t offset = 7;
+    V9fsString name;
+    int32_t newdirfid;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dds", &fid, &newdirfid, &name);
+    if (err < 0) {
+        goto out_nofid;
+    }
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EISDIR;
+        goto out_nofid;
+    }
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    BUG_ON(fidp->fid_type != P9_FID_NONE);
+    /* if fs driver is not path based, return EOPNOTSUPP */
+    if (!(pdu->s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT)) {
+        err = -EOPNOTSUPP;
+        goto out;
+    }
+    v9fs_path_write_lock(s);
+    err = v9fs_complete_rename(pdu, fidp, newdirfid, &name);
+    v9fs_path_unlock(s);
+    if (!err) {
+        err = offset;
+    }
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+static void v9fs_fix_fid_paths(V9fsPDU *pdu, V9fsPath *olddir,
+                               V9fsString *old_name, V9fsPath *newdir,
+                               V9fsString *new_name)
+{
+    V9fsFidState *tfidp;
+    V9fsPath oldpath, newpath;
+    V9fsState *s = pdu->s;
+
+
+    v9fs_path_init(&oldpath);
+    v9fs_path_init(&newpath);
+    v9fs_co_name_to_path(pdu, olddir, old_name->data, &oldpath);
+    v9fs_co_name_to_path(pdu, newdir, new_name->data, &newpath);
+
+    /*
+     * Fixup fid's pointing to the old name to
+     * start pointing to the new name
+     */
+    for (tfidp = s->fid_list; tfidp; tfidp = tfidp->next) {
+        if (v9fs_path_is_ancestor(&oldpath, &tfidp->path)) {
+            /* replace the name */
+            v9fs_fix_path(&tfidp->path, &newpath, strlen(oldpath.data));
+        }
+    }
+    v9fs_path_free(&oldpath);
+    v9fs_path_free(&newpath);
+}
+
+static int v9fs_complete_renameat(V9fsPDU *pdu, int32_t olddirfid,
+                                  V9fsString *old_name, int32_t newdirfid,
+                                  V9fsString *new_name)
+{
+    int err = 0;
+    V9fsState *s = pdu->s;
+    V9fsFidState *newdirfidp = NULL, *olddirfidp = NULL;
+
+    olddirfidp = get_fid(pdu, olddirfid);
+    if (olddirfidp == NULL) {
+        err = -ENOENT;
+        goto out;
+    }
+    if (newdirfid != -1) {
+        newdirfidp = get_fid(pdu, newdirfid);
+        if (newdirfidp == NULL) {
+            err = -ENOENT;
+            goto out;
+        }
+    } else {
+        newdirfidp = get_fid(pdu, olddirfid);
+    }
+
+    err = v9fs_co_renameat(pdu, &olddirfidp->path, old_name,
+                           &newdirfidp->path, new_name);
+    if (err < 0) {
+        goto out;
+    }
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        /* Only for path based fid  we need to do the below fixup */
+        v9fs_fix_fid_paths(pdu, &olddirfidp->path, old_name,
+                           &newdirfidp->path, new_name);
+    }
+out:
+    if (olddirfidp) {
+        put_fid(pdu, olddirfidp);
+    }
+    if (newdirfidp) {
+        put_fid(pdu, newdirfidp);
+    }
+    return err;
+}
+
+static void v9fs_renameat(void *opaque)
+{
+    ssize_t err = 0;
+    size_t offset = 7;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+    int32_t olddirfid, newdirfid;
+    V9fsString old_name, new_name;
+
+    v9fs_string_init(&old_name);
+    v9fs_string_init(&new_name);
+    err = pdu_unmarshal(pdu, offset, "dsds", &olddirfid,
+                        &old_name, &newdirfid, &new_name);
+    if (err < 0) {
+        goto out_err;
+    }
+
+    if (name_is_illegal(old_name.data) || name_is_illegal(new_name.data)) {
+        err = -ENOENT;
+        goto out_err;
+    }
+
+    if (!strcmp(".", old_name.data) || !strcmp("..", old_name.data) ||
+        !strcmp(".", new_name.data) || !strcmp("..", new_name.data)) {
+        err = -EISDIR;
+        goto out_err;
+    }
+
+    v9fs_path_write_lock(s);
+    err = v9fs_complete_renameat(pdu, olddirfid,
+                                 &old_name, newdirfid, &new_name);
+    v9fs_path_unlock(s);
+    if (!err) {
+        err = offset;
+    }
+
+out_err:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&old_name);
+    v9fs_string_free(&new_name);
+}
+
+static void v9fs_wstat(void *opaque)
+{
+    int32_t fid;
+    int err = 0;
+    int16_t unused;
+    V9fsStat v9stat;
+    size_t offset = 7;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_stat_init(&v9stat);
+    err = pdu_unmarshal(pdu, offset, "dwS", &fid, &unused, &v9stat);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_wstat(pdu->tag, pdu->id, fid,
+                     v9stat.mode, v9stat.atime, v9stat.mtime);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    /* do we need to sync the file? */
+    if (donttouch_stat(&v9stat)) {
+        err = v9fs_co_fsync(pdu, fidp, 0);
+        goto out;
+    }
+    if (v9stat.mode != -1) {
+        uint32_t v9_mode;
+        err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
+        if (err < 0) {
+            goto out;
+        }
+        v9_mode = stat_to_v9mode(&stbuf);
+        if ((v9stat.mode & P9_STAT_MODE_TYPE_BITS) !=
+            (v9_mode & P9_STAT_MODE_TYPE_BITS)) {
+            /* Attempting to change the type */
+            err = -EIO;
+            goto out;
+        }
+        err = v9fs_co_chmod(pdu, &fidp->path,
+                            v9mode_to_mode(v9stat.mode,
+                                           &v9stat.extension));
+        if (err < 0) {
+            goto out;
+        }
+    }
+    if (v9stat.mtime != -1 || v9stat.atime != -1) {
+        struct timespec times[2];
+        if (v9stat.atime != -1) {
+            times[0].tv_sec = v9stat.atime;
+            times[0].tv_nsec = 0;
+        } else {
+            times[0].tv_nsec = UTIME_OMIT;
+        }
+        if (v9stat.mtime != -1) {
+            times[1].tv_sec = v9stat.mtime;
+            times[1].tv_nsec = 0;
+        } else {
+            times[1].tv_nsec = UTIME_OMIT;
+        }
+        err = v9fs_co_utimensat(pdu, &fidp->path, times);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    if (v9stat.n_gid != -1 || v9stat.n_uid != -1) {
+        err = v9fs_co_chown(pdu, &fidp->path, v9stat.n_uid, v9stat.n_gid);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    if (v9stat.name.size != 0) {
+        err = v9fs_complete_rename(pdu, fidp, -1, &v9stat.name);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    if (v9stat.length != -1) {
+        err = v9fs_co_truncate(pdu, &fidp->path, v9stat.length);
+        if (err < 0) {
+            goto out;
+        }
+    }
+    err = offset;
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    v9fs_stat_free(&v9stat);
+    pdu_complete(pdu, err);
+}
+
+static int v9fs_fill_statfs(V9fsState *s, V9fsPDU *pdu, struct statfs *stbuf)
+{
+    uint32_t f_type;
+    uint32_t f_bsize;
+    uint64_t f_blocks;
+    uint64_t f_bfree;
+    uint64_t f_bavail;
+    uint64_t f_files;
+    uint64_t f_ffree;
+    uint64_t fsid_val;
+    uint32_t f_namelen;
+    size_t offset = 7;
+    int32_t bsize_factor;
+
+    /*
+     * compute bsize factor based on host file system block size
+     * and client msize
+     */
+    bsize_factor = (s->msize - P9_IOHDRSZ)/stbuf->f_bsize;
+    if (!bsize_factor) {
+        bsize_factor = 1;
+    }
+    f_type  = stbuf->f_type;
+    f_bsize = stbuf->f_bsize;
+    f_bsize *= bsize_factor;
+    /*
+     * f_bsize is adjusted(multiplied) by bsize factor, so we need to
+     * adjust(divide) the number of blocks, free blocks and available
+     * blocks by bsize factor
+     */
+    f_blocks = stbuf->f_blocks/bsize_factor;
+    f_bfree  = stbuf->f_bfree/bsize_factor;
+    f_bavail = stbuf->f_bavail/bsize_factor;
+    f_files  = stbuf->f_files;
+    f_ffree  = stbuf->f_ffree;
+    fsid_val = (unsigned int) stbuf->f_fsid.__val[0] |
+               (unsigned long long)stbuf->f_fsid.__val[1] << 32;
+    f_namelen = stbuf->f_namelen;
+
+    return pdu_marshal(pdu, offset, "ddqqqqqqd",
+                       f_type, f_bsize, f_blocks, f_bfree,
+                       f_bavail, f_files, f_ffree,
+                       fsid_val, f_namelen);
+}
+
+static void v9fs_statfs(void *opaque)
+{
+    int32_t fid;
+    ssize_t retval = 0;
+    size_t offset = 7;
+    V9fsFidState *fidp;
+    struct statfs stbuf;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    retval = pdu_unmarshal(pdu, offset, "d", &fid);
+    if (retval < 0) {
+        goto out_nofid;
+    }
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        retval = -ENOENT;
+        goto out_nofid;
+    }
+    retval = v9fs_co_statfs(pdu, &fidp->path, &stbuf);
+    if (retval < 0) {
+        goto out;
+    }
+    retval = v9fs_fill_statfs(s, pdu, &stbuf);
+    if (retval < 0) {
+        goto out;
+    }
+    retval += offset;
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, retval);
+}
+
+static void v9fs_mknod(void *opaque)
+{
+
+    int mode;
+    gid_t gid;
+    int32_t fid;
+    V9fsQID qid;
+    int err = 0;
+    int major, minor;
+    size_t offset = 7;
+    V9fsString name;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dsdddd", &fid, &name, &mode,
+                        &major, &minor, &gid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_mknod(pdu->tag, pdu->id, fid, mode, major, minor);
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EEXIST;
+        goto out_nofid;
+    }
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, gid,
+                        makedev(major, minor), mode, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    stat_to_qid(&stbuf, &qid);
+    err = pdu_marshal(pdu, offset, "Q", &qid);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_mknod_return(pdu->tag, pdu->id,
+                            qid.type, qid.version, qid.path);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+/*
+ * Implement posix byte range locking code
+ * Server side handling of locking code is very simple, because 9p server in
+ * QEMU can handle only one client. And most of the lock handling
+ * (like conflict, merging) etc is done by the VFS layer itself, so no need to
+ * do any thing in * qemu 9p server side lock code path.
+ * So when a TLOCK request comes, always return success
+ */
+static void v9fs_lock(void *opaque)
+{
+    int8_t status;
+    V9fsFlock flock;
+    size_t offset = 7;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    int32_t fid, err = 0;
+    V9fsPDU *pdu = opaque;
+
+    status = P9_LOCK_ERROR;
+    v9fs_string_init(&flock.client_id);
+    err = pdu_unmarshal(pdu, offset, "dbdqqds", &fid, &flock.type,
+                        &flock.flags, &flock.start, &flock.length,
+                        &flock.proc_id, &flock.client_id);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_lock(pdu->tag, pdu->id, fid,
+                    flock.type, flock.start, flock.length);
+
+
+    /* We support only block flag now (that too ignored currently) */
+    if (flock.flags & ~P9_LOCK_FLAGS_BLOCK) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    err = v9fs_co_fstat(pdu, fidp, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    status = P9_LOCK_SUCCESS;
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    err = pdu_marshal(pdu, offset, "b", status);
+    if (err > 0) {
+        err += offset;
+    }
+    trace_v9fs_lock_return(pdu->tag, pdu->id, status);
+    pdu_complete(pdu, err);
+    v9fs_string_free(&flock.client_id);
+}
+
+/*
+ * When a TGETLOCK request comes, always return success because all lock
+ * handling is done by client's VFS layer.
+ */
+static void v9fs_getlock(void *opaque)
+{
+    size_t offset = 7;
+    struct stat stbuf;
+    V9fsFidState *fidp;
+    V9fsGetlock glock;
+    int32_t fid, err = 0;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_string_init(&glock.client_id);
+    err = pdu_unmarshal(pdu, offset, "dbqqds", &fid, &glock.type,
+                        &glock.start, &glock.length, &glock.proc_id,
+                        &glock.client_id);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_getlock(pdu->tag, pdu->id, fid,
+                       glock.type, glock.start, glock.length);
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    err = v9fs_co_fstat(pdu, fidp, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    glock.type = P9_LOCK_TYPE_UNLCK;
+    err = pdu_marshal(pdu, offset, "bqqds", glock.type,
+                          glock.start, glock.length, glock.proc_id,
+                          &glock.client_id);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_getlock_return(pdu->tag, pdu->id, glock.type, glock.start,
+                              glock.length, glock.proc_id);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&glock.client_id);
+}
+
+static void v9fs_mkdir(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    size_t offset = 7;
+    int32_t fid;
+    struct stat stbuf;
+    V9fsQID qid;
+    V9fsString name;
+    V9fsFidState *fidp;
+    gid_t gid;
+    int mode;
+    int err = 0;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dsdd", &fid, &name, &mode, &gid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_mkdir(pdu->tag, pdu->id, fid, name.data, mode, gid);
+
+    if (name_is_illegal(name.data)) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    if (!strcmp(".", name.data) || !strcmp("..", name.data)) {
+        err = -EEXIST;
+        goto out_nofid;
+    }
+
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    err = v9fs_co_mkdir(pdu, fidp, &name, mode, fidp->uid, gid, &stbuf);
+    if (err < 0) {
+        goto out;
+    }
+    stat_to_qid(&stbuf, &qid);
+    err = pdu_marshal(pdu, offset, "Q", &qid);
+    if (err < 0) {
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_mkdir_return(pdu->tag, pdu->id,
+                            qid.type, qid.version, qid.path, err);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+static void v9fs_xattrwalk(void *opaque)
+{
+    int64_t size;
+    V9fsString name;
+    ssize_t err = 0;
+    size_t offset = 7;
+    int32_t fid, newfid;
+    V9fsFidState *file_fidp;
+    V9fsFidState *xattr_fidp = NULL;
+    V9fsPDU *pdu = opaque;
+    V9fsState *s = pdu->s;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dds", &fid, &newfid, &name);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_xattrwalk(pdu->tag, pdu->id, fid, newfid, name.data);
+
+    file_fidp = get_fid(pdu, fid);
+    if (file_fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+    xattr_fidp = alloc_fid(s, newfid);
+    if (xattr_fidp == NULL) {
+        err = -EINVAL;
+        goto out;
+    }
+    v9fs_path_copy(&xattr_fidp->path, &file_fidp->path);
+    if (name.data == NULL) {
+        /*
+         * listxattr request. Get the size first
+         */
+        size = v9fs_co_llistxattr(pdu, &xattr_fidp->path, NULL, 0);
+        if (size < 0) {
+            err = size;
+            clunk_fid(s, xattr_fidp->fid);
+            goto out;
+        }
+        /*
+         * Read the xattr value
+         */
+        xattr_fidp->fs.xattr.len = size;
+        xattr_fidp->fid_type = P9_FID_XATTR;
+        xattr_fidp->fs.xattr.copied_len = -1;
+        if (size) {
+            xattr_fidp->fs.xattr.value = g_malloc(size);
+            err = v9fs_co_llistxattr(pdu, &xattr_fidp->path,
+                                     xattr_fidp->fs.xattr.value,
+                                     xattr_fidp->fs.xattr.len);
+            if (err < 0) {
+                clunk_fid(s, xattr_fidp->fid);
+                goto out;
+            }
+        }
+        err = pdu_marshal(pdu, offset, "q", size);
+        if (err < 0) {
+            goto out;
+        }
+        err += offset;
+    } else {
+        /*
+         * specific xattr fid. We check for xattr
+         * presence also collect the xattr size
+         */
+        size = v9fs_co_lgetxattr(pdu, &xattr_fidp->path,
+                                 &name, NULL, 0);
+        if (size < 0) {
+            err = size;
+            clunk_fid(s, xattr_fidp->fid);
+            goto out;
+        }
+        /*
+         * Read the xattr value
+         */
+        xattr_fidp->fs.xattr.len = size;
+        xattr_fidp->fid_type = P9_FID_XATTR;
+        xattr_fidp->fs.xattr.copied_len = -1;
+        if (size) {
+            xattr_fidp->fs.xattr.value = g_malloc(size);
+            err = v9fs_co_lgetxattr(pdu, &xattr_fidp->path,
+                                    &name, xattr_fidp->fs.xattr.value,
+                                    xattr_fidp->fs.xattr.len);
+            if (err < 0) {
+                clunk_fid(s, xattr_fidp->fid);
+                goto out;
+            }
+        }
+        err = pdu_marshal(pdu, offset, "q", size);
+        if (err < 0) {
+            goto out;
+        }
+        err += offset;
+    }
+    trace_v9fs_xattrwalk_return(pdu->tag, pdu->id, size);
+out:
+    put_fid(pdu, file_fidp);
+    if (xattr_fidp) {
+        put_fid(pdu, xattr_fidp);
+    }
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+static void v9fs_xattrcreate(void *opaque)
+{
+    int flags;
+    int32_t fid;
+    int64_t size;
+    ssize_t err = 0;
+    V9fsString name;
+    size_t offset = 7;
+    V9fsFidState *file_fidp;
+    V9fsFidState *xattr_fidp;
+    V9fsPDU *pdu = opaque;
+
+    v9fs_string_init(&name);
+    err = pdu_unmarshal(pdu, offset, "dsqd", &fid, &name, &size, &flags);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_xattrcreate(pdu->tag, pdu->id, fid, name.data, size, flags);
+
+    file_fidp = get_fid(pdu, fid);
+    if (file_fidp == NULL) {
+        err = -EINVAL;
+        goto out_nofid;
+    }
+    /* Make the file fid point to xattr */
+    xattr_fidp = file_fidp;
+    xattr_fidp->fid_type = P9_FID_XATTR;
+    xattr_fidp->fs.xattr.copied_len = 0;
+    xattr_fidp->fs.xattr.len = size;
+    xattr_fidp->fs.xattr.flags = flags;
+    v9fs_string_init(&xattr_fidp->fs.xattr.name);
+    v9fs_string_copy(&xattr_fidp->fs.xattr.name, &name);
+    xattr_fidp->fs.xattr.value = g_malloc(size);
+    err = offset;
+    put_fid(pdu, file_fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+    v9fs_string_free(&name);
+}
+
+static void v9fs_readlink(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    size_t offset = 7;
+    V9fsString target;
+    int32_t fid;
+    int err = 0;
+    V9fsFidState *fidp;
+
+    err = pdu_unmarshal(pdu, offset, "d", &fid);
+    if (err < 0) {
+        goto out_nofid;
+    }
+    trace_v9fs_readlink(pdu->tag, pdu->id, fid);
+    fidp = get_fid(pdu, fid);
+    if (fidp == NULL) {
+        err = -ENOENT;
+        goto out_nofid;
+    }
+
+    v9fs_string_init(&target);
+    err = v9fs_co_readlink(pdu, &fidp->path, &target);
+    if (err < 0) {
+        goto out;
+    }
+    err = pdu_marshal(pdu, offset, "s", &target);
+    if (err < 0) {
+        v9fs_string_free(&target);
+        goto out;
+    }
+    err += offset;
+    trace_v9fs_readlink_return(pdu->tag, pdu->id, target.data);
+    v9fs_string_free(&target);
+out:
+    put_fid(pdu, fidp);
+out_nofid:
+    pdu_complete(pdu, err);
+}
+
+static CoroutineEntry *pdu_co_handlers[] = {
+    [P9_TREADDIR] = v9fs_readdir,
+    [P9_TSTATFS] = v9fs_statfs,
+    [P9_TGETATTR] = v9fs_getattr,
+    [P9_TSETATTR] = v9fs_setattr,
+    [P9_TXATTRWALK] = v9fs_xattrwalk,
+    [P9_TXATTRCREATE] = v9fs_xattrcreate,
+    [P9_TMKNOD] = v9fs_mknod,
+    [P9_TRENAME] = v9fs_rename,
+    [P9_TLOCK] = v9fs_lock,
+    [P9_TGETLOCK] = v9fs_getlock,
+    [P9_TRENAMEAT] = v9fs_renameat,
+    [P9_TREADLINK] = v9fs_readlink,
+    [P9_TUNLINKAT] = v9fs_unlinkat,
+    [P9_TMKDIR] = v9fs_mkdir,
+    [P9_TVERSION] = v9fs_version,
+    [P9_TLOPEN] = v9fs_open,
+    [P9_TATTACH] = v9fs_attach,
+    [P9_TSTAT] = v9fs_stat,
+    [P9_TWALK] = v9fs_walk,
+    [P9_TCLUNK] = v9fs_clunk,
+    [P9_TFSYNC] = v9fs_fsync,
+    [P9_TOPEN] = v9fs_open,
+    [P9_TREAD] = v9fs_read,
+#if 0
+    [P9_TAUTH] = v9fs_auth,
+#endif
+    [P9_TFLUSH] = v9fs_flush,
+    [P9_TLINK] = v9fs_link,
+    [P9_TSYMLINK] = v9fs_symlink,
+    [P9_TCREATE] = v9fs_create,
+    [P9_TLCREATE] = v9fs_lcreate,
+    [P9_TWRITE] = v9fs_write,
+    [P9_TWSTAT] = v9fs_wstat,
+    [P9_TREMOVE] = v9fs_remove,
+};
+
+static void v9fs_op_not_supp(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    pdu_complete(pdu, -EOPNOTSUPP);
+}
+
+static void v9fs_fs_ro(void *opaque)
+{
+    V9fsPDU *pdu = opaque;
+    pdu_complete(pdu, -EROFS);
+}
+
+static inline bool is_read_only_op(V9fsPDU *pdu)
+{
+    switch (pdu->id) {
+    case P9_TREADDIR:
+    case P9_TSTATFS:
+    case P9_TGETATTR:
+    case P9_TXATTRWALK:
+    case P9_TLOCK:
+    case P9_TGETLOCK:
+    case P9_TREADLINK:
+    case P9_TVERSION:
+    case P9_TLOPEN:
+    case P9_TATTACH:
+    case P9_TSTAT:
+    case P9_TWALK:
+    case P9_TCLUNK:
+    case P9_TFSYNC:
+    case P9_TOPEN:
+    case P9_TREAD:
+    case P9_TAUTH:
+    case P9_TFLUSH:
+        return 1;
+    default:
+        return 0;
+    }
+}
+
+void pdu_submit(V9fsPDU *pdu)
+{
+    Coroutine *co;
+    CoroutineEntry *handler;
+    V9fsState *s = pdu->s;
+
+    if (pdu->id >= ARRAY_SIZE(pdu_co_handlers) ||
+        (pdu_co_handlers[pdu->id] == NULL)) {
+        handler = v9fs_op_not_supp;
+    } else {
+        handler = pdu_co_handlers[pdu->id];
+    }
+
+    if (is_ro_export(&s->ctx) && !is_read_only_op(pdu)) {
+        handler = v9fs_fs_ro;
+    }
+    co = qemu_coroutine_create(handler, pdu);
+    qemu_coroutine_enter(co);
+}
+
+/* Returns 0 on success, 1 on failure. */
+int v9fs_device_realize_common(V9fsState *s, Error **errp)
+{
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    int i, len;
+    struct stat stat;
+    FsDriverEntry *fse;
+    V9fsPath path;
+    int rc = 1;
+
+    /* initialize pdu allocator */
+    QLIST_INIT(&s->free_list);
+    QLIST_INIT(&s->active_list);
+    for (i = 0; i < (MAX_REQ - 1); i++) {
+        QLIST_INSERT_HEAD(&s->free_list, &v->pdus[i], next);
+        v->pdus[i].s = s;
+        v->pdus[i].idx = i;
+    }
+
+    v9fs_path_init(&path);
+
+    fse = get_fsdev_fsentry(s->fsconf.fsdev_id);
+
+    if (!fse) {
+        /* We don't have a fsdev identified by fsdev_id */
+        error_setg(errp, "9pfs device couldn't find fsdev with the "
+                   "id = %s",
+                   s->fsconf.fsdev_id ? s->fsconf.fsdev_id : "NULL");
+        goto out;
+    }
+
+    if (!s->fsconf.tag) {
+        /* we haven't specified a mount_tag */
+        error_setg(errp, "fsdev with id %s needs mount_tag arguments",
+                   s->fsconf.fsdev_id);
+        goto out;
+    }
+
+    s->ctx.export_flags = fse->export_flags;
+    s->ctx.fs_root = g_strdup(fse->path);
+    s->ctx.exops.get_st_gen = NULL;
+    len = strlen(s->fsconf.tag);
+    if (len > MAX_TAG_LEN - 1) {
+        error_setg(errp, "mount tag '%s' (%d bytes) is longer than "
+                   "maximum (%d bytes)", s->fsconf.tag, len, MAX_TAG_LEN - 1);
+        goto out;
+    }
+
+    s->tag = g_strdup(s->fsconf.tag);
+    s->ctx.uid = -1;
+
+    s->ops = fse->ops;
+
+    s->fid_list = NULL;
+    qemu_co_rwlock_init(&s->rename_lock);
+
+    if (s->ops->init(&s->ctx) < 0) {
+        error_setg(errp, "9pfs Failed to initialize fs-driver with id:%s"
+                   " and export path:%s", s->fsconf.fsdev_id, s->ctx.fs_root);
+        goto out;
+    }
+
+    /*
+     * Check details of export path, We need to use fs driver
+     * call back to do that. Since we are in the init path, we don't
+     * use co-routines here.
+     */
+    if (s->ops->name_to_path(&s->ctx, NULL, "/", &path) < 0) {
+        error_setg(errp,
+                   "error in converting name to path %s", strerror(errno));
+        goto out;
+    }
+    if (s->ops->lstat(&s->ctx, &path, &stat)) {
+        error_setg(errp, "share path %s does not exist", fse->path);
+        goto out;
+    } else if (!S_ISDIR(stat.st_mode)) {
+        error_setg(errp, "share path %s is not a directory", fse->path);
+        goto out;
+    }
+    v9fs_path_free(&path);
+
+    rc = 0;
+out:
+    if (rc) {
+        g_free(s->ctx.fs_root);
+        g_free(s->tag);
+        v9fs_path_free(&path);
+    }
+    return rc;
+}
+
+void v9fs_device_unrealize_common(V9fsState *s, Error **errp)
+{
+    g_free(s->ctx.fs_root);
+    g_free(s->tag);
+}
+
+static void __attribute__((__constructor__)) v9fs_set_fd_limit(void)
+{
+    struct rlimit rlim;
+    if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
+        error_report("Failed to get the resource limit");
+        exit(1);
+    }
+    open_fd_hw = rlim.rlim_cur - MIN(400, rlim.rlim_cur/3);
+    open_fd_rc = rlim.rlim_cur/2;
+}
diff --git a/hw/9pfs/9p.h b/hw/9pfs/9p.h
new file mode 100644
index 0000000..a386033
--- /dev/null
+++ b/hw/9pfs/9p.h
@@ -0,0 +1,342 @@
+#ifndef QEMU_9P_H
+#define QEMU_9P_H
+
+#include <dirent.h>
+#include <utime.h>
+#include <sys/resource.h>
+#include "fsdev/file-op-9p.h"
+#include "fsdev/9p-iov-marshal.h"
+#include "qemu/thread.h"
+#include "qemu/coroutine.h"
+
+enum {
+    P9_TLERROR = 6,
+    P9_RLERROR,
+    P9_TSTATFS = 8,
+    P9_RSTATFS,
+    P9_TLOPEN = 12,
+    P9_RLOPEN,
+    P9_TLCREATE = 14,
+    P9_RLCREATE,
+    P9_TSYMLINK = 16,
+    P9_RSYMLINK,
+    P9_TMKNOD = 18,
+    P9_RMKNOD,
+    P9_TRENAME = 20,
+    P9_RRENAME,
+    P9_TREADLINK = 22,
+    P9_RREADLINK,
+    P9_TGETATTR = 24,
+    P9_RGETATTR,
+    P9_TSETATTR = 26,
+    P9_RSETATTR,
+    P9_TXATTRWALK = 30,
+    P9_RXATTRWALK,
+    P9_TXATTRCREATE = 32,
+    P9_RXATTRCREATE,
+    P9_TREADDIR = 40,
+    P9_RREADDIR,
+    P9_TFSYNC = 50,
+    P9_RFSYNC,
+    P9_TLOCK = 52,
+    P9_RLOCK,
+    P9_TGETLOCK = 54,
+    P9_RGETLOCK,
+    P9_TLINK = 70,
+    P9_RLINK,
+    P9_TMKDIR = 72,
+    P9_RMKDIR,
+    P9_TRENAMEAT = 74,
+    P9_RRENAMEAT,
+    P9_TUNLINKAT = 76,
+    P9_RUNLINKAT,
+    P9_TVERSION = 100,
+    P9_RVERSION,
+    P9_TAUTH = 102,
+    P9_RAUTH,
+    P9_TATTACH = 104,
+    P9_RATTACH,
+    P9_TERROR = 106,
+    P9_RERROR,
+    P9_TFLUSH = 108,
+    P9_RFLUSH,
+    P9_TWALK = 110,
+    P9_RWALK,
+    P9_TOPEN = 112,
+    P9_ROPEN,
+    P9_TCREATE = 114,
+    P9_RCREATE,
+    P9_TREAD = 116,
+    P9_RREAD,
+    P9_TWRITE = 118,
+    P9_RWRITE,
+    P9_TCLUNK = 120,
+    P9_RCLUNK,
+    P9_TREMOVE = 122,
+    P9_RREMOVE,
+    P9_TSTAT = 124,
+    P9_RSTAT,
+    P9_TWSTAT = 126,
+    P9_RWSTAT,
+};
+
+
+/* qid.types */
+enum {
+    P9_QTDIR = 0x80,
+    P9_QTAPPEND = 0x40,
+    P9_QTEXCL = 0x20,
+    P9_QTMOUNT = 0x10,
+    P9_QTAUTH = 0x08,
+    P9_QTTMP = 0x04,
+    P9_QTSYMLINK = 0x02,
+    P9_QTLINK = 0x01,
+    P9_QTFILE = 0x00,
+};
+
+enum p9_proto_version {
+    V9FS_PROTO_2000U = 0x01,
+    V9FS_PROTO_2000L = 0x02,
+};
+
+#define P9_NOTAG    (u16)(~0)
+#define P9_NOFID    (u32)(~0)
+#define P9_MAXWELEM 16
+
+#define FID_REFERENCED          0x1
+#define FID_NON_RECLAIMABLE     0x2
+static inline char *rpath(FsContext *ctx, const char *path)
+{
+    return g_strdup_printf("%s/%s", ctx->fs_root, path);
+}
+
+/*
+ * ample room for Twrite/Rread header
+ * size[4] Tread/Twrite tag[2] fid[4] offset[8] count[4]
+ */
+#define P9_IOHDRSZ 24
+
+typedef struct V9fsPDU V9fsPDU;
+struct V9fsState;
+
+struct V9fsPDU
+{
+    uint32_t size;
+    uint16_t tag;
+    uint8_t id;
+    uint8_t cancelled;
+    CoQueue complete;
+    struct V9fsState *s;
+    QLIST_ENTRY(V9fsPDU) next;
+    uint32_t idx;
+};
+
+
+/* FIXME
+ * 1) change user needs to set groups and stuff
+ */
+
+#define MAX_REQ         128
+#define MAX_TAG_LEN     32
+
+#define BUG_ON(cond) assert(!(cond))
+
+typedef struct V9fsFidState V9fsFidState;
+
+enum {
+    P9_FID_NONE = 0,
+    P9_FID_FILE,
+    P9_FID_DIR,
+    P9_FID_XATTR,
+};
+
+typedef struct V9fsConf
+{
+    /* tag name for the device */
+    char *tag;
+    char *fsdev_id;
+} V9fsConf;
+
+typedef struct V9fsXattr
+{
+    int64_t copied_len;
+    int64_t len;
+    void *value;
+    V9fsString name;
+    int flags;
+} V9fsXattr;
+
+typedef struct V9fsDir {
+    DIR *stream;
+    QemuMutex readdir_mutex;
+} V9fsDir;
+
+static inline void v9fs_readdir_lock(V9fsDir *dir)
+{
+    qemu_mutex_lock(&dir->readdir_mutex);
+}
+
+static inline void v9fs_readdir_unlock(V9fsDir *dir)
+{
+    qemu_mutex_unlock(&dir->readdir_mutex);
+}
+
+static inline void v9fs_readdir_init(V9fsDir *dir)
+{
+    qemu_mutex_init(&dir->readdir_mutex);
+}
+
+/*
+ * Filled by fs driver on open and other
+ * calls.
+ */
+union V9fsFidOpenState {
+    int fd;
+    V9fsDir dir;
+    V9fsXattr xattr;
+    /*
+     * private pointer for fs drivers, that
+     * have its own internal representation of
+     * open files.
+     */
+    void *private;
+};
+
+struct V9fsFidState
+{
+    int fid_type;
+    int32_t fid;
+    V9fsPath path;
+    V9fsFidOpenState fs;
+    V9fsFidOpenState fs_reclaim;
+    int flags;
+    int open_flags;
+    uid_t uid;
+    int ref;
+    int clunked;
+    V9fsFidState *next;
+    V9fsFidState *rclm_lst;
+};
+
+typedef struct V9fsState
+{
+    QLIST_HEAD(, V9fsPDU) free_list;
+    QLIST_HEAD(, V9fsPDU) active_list;
+    V9fsFidState *fid_list;
+    FileOperations *ops;
+    FsContext ctx;
+    char *tag;
+    enum p9_proto_version proto_version;
+    int32_t msize;
+    /*
+     * lock ensuring atomic path update
+     * on rename.
+     */
+    CoRwlock rename_lock;
+    int32_t root_fid;
+    Error *migration_blocker;
+    V9fsConf fsconf;
+    V9fsQID root_qid;
+} V9fsState;
+
+/* 9p2000.L open flags */
+#define P9_DOTL_RDONLY        00000000
+#define P9_DOTL_WRONLY        00000001
+#define P9_DOTL_RDWR          00000002
+#define P9_DOTL_NOACCESS      00000003
+#define P9_DOTL_CREATE        00000100
+#define P9_DOTL_EXCL          00000200
+#define P9_DOTL_NOCTTY        00000400
+#define P9_DOTL_TRUNC         00001000
+#define P9_DOTL_APPEND        00002000
+#define P9_DOTL_NONBLOCK      00004000
+#define P9_DOTL_DSYNC         00010000
+#define P9_DOTL_FASYNC        00020000
+#define P9_DOTL_DIRECT        00040000
+#define P9_DOTL_LARGEFILE     00100000
+#define P9_DOTL_DIRECTORY     00200000
+#define P9_DOTL_NOFOLLOW      00400000
+#define P9_DOTL_NOATIME       01000000
+#define P9_DOTL_CLOEXEC       02000000
+#define P9_DOTL_SYNC          04000000
+
+/* 9p2000.L at flags */
+#define P9_DOTL_AT_REMOVEDIR         0x200
+
+/* 9P2000.L lock type */
+#define P9_LOCK_TYPE_RDLCK 0
+#define P9_LOCK_TYPE_WRLCK 1
+#define P9_LOCK_TYPE_UNLCK 2
+
+#define P9_LOCK_SUCCESS 0
+#define P9_LOCK_BLOCKED 1
+#define P9_LOCK_ERROR 2
+#define P9_LOCK_GRACE 3
+
+#define P9_LOCK_FLAGS_BLOCK 1
+#define P9_LOCK_FLAGS_RECLAIM 2
+
+typedef struct V9fsFlock
+{
+    uint8_t type;
+    uint32_t flags;
+    uint64_t start; /* absolute offset */
+    uint64_t length;
+    uint32_t proc_id;
+    V9fsString client_id;
+} V9fsFlock;
+
+typedef struct V9fsGetlock
+{
+    uint8_t type;
+    uint64_t start; /* absolute offset */
+    uint64_t length;
+    uint32_t proc_id;
+    V9fsString client_id;
+} V9fsGetlock;
+
+extern int open_fd_hw;
+extern int total_open_fd;
+
+static inline void v9fs_path_write_lock(V9fsState *s)
+{
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        qemu_co_rwlock_wrlock(&s->rename_lock);
+    }
+}
+
+static inline void v9fs_path_read_lock(V9fsState *s)
+{
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        qemu_co_rwlock_rdlock(&s->rename_lock);
+    }
+}
+
+static inline void v9fs_path_unlock(V9fsState *s)
+{
+    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
+        qemu_co_rwlock_unlock(&s->rename_lock);
+    }
+}
+
+static inline uint8_t v9fs_request_cancelled(V9fsPDU *pdu)
+{
+    return pdu->cancelled;
+}
+
+extern void v9fs_reclaim_fd(V9fsPDU *pdu);
+extern void v9fs_path_init(V9fsPath *path);
+extern void v9fs_path_free(V9fsPath *path);
+extern void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs);
+extern int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
+                             const char *name, V9fsPath *path);
+extern int v9fs_device_realize_common(V9fsState *s, Error **errp);
+extern void v9fs_device_unrealize_common(V9fsState *s, Error **errp);
+
+ssize_t pdu_marshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...);
+ssize_t pdu_unmarshal(V9fsPDU *pdu, size_t offset, const char *fmt, ...);
+V9fsPDU *pdu_alloc(V9fsState *s);
+void pdu_free(V9fsPDU *pdu);
+void pdu_submit(V9fsPDU *pdu);
+
+#endif
diff --git a/hw/9pfs/Makefile.objs b/hw/9pfs/Makefile.objs
index 1e9b595..da0ae0c 100644
--- a/hw/9pfs/Makefile.objs
+++ b/hw/9pfs/Makefile.objs
@@ -1,9 +1,9 @@
-common-obj-y  = virtio-9p.o
-common-obj-y += virtio-9p-local.o virtio-9p-xattr.o
-common-obj-y += virtio-9p-xattr-user.o virtio-9p-posix-acl.o
-common-obj-y += virtio-9p-coth.o cofs.o codir.o cofile.o
-common-obj-y += coxattr.o virtio-9p-synth.o
-common-obj-$(CONFIG_OPEN_BY_HANDLE) +=  virtio-9p-handle.o
-common-obj-y += virtio-9p-proxy.o
+common-obj-y  = 9p.o
+common-obj-y += 9p-local.o 9p-xattr.o
+common-obj-y += 9p-xattr-user.o 9p-posix-acl.o
+common-obj-y += coth.o cofs.o codir.o cofile.o
+common-obj-y += coxattr.o 9p-synth.o
+common-obj-$(CONFIG_OPEN_BY_HANDLE) +=  9p-handle.o
+common-obj-y += 9p-proxy.o
 
 obj-y += virtio-9p-device.o
diff --git a/hw/9pfs/codir.c b/hw/9pfs/codir.c
index 65ad329..d91f9ad 100644
--- a/hw/9pfs/codir.c
+++ b/hw/9pfs/codir.c
@@ -1,6 +1,5 @@
-
 /*
- * Virtio 9p backend
+ * 9p backend
  *
  * Copyright IBM, Corp. 2011
  *
@@ -12,13 +11,13 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
-int v9fs_co_readdir_r(V9fsPDU *pdu, V9fsFidState *fidp, struct dirent *dent,
-                      struct dirent **result)
+int v9fs_co_readdir(V9fsPDU *pdu, V9fsFidState *fidp, struct dirent **dent)
 {
     int err;
     V9fsState *s = pdu->s;
@@ -28,11 +27,14 @@
     }
     v9fs_co_run_in_worker(
         {
+            struct dirent *entry;
+
             errno = 0;
-            err = s->ops->readdir_r(&s->ctx, &fidp->fs, dent, result);
-            if (!*result && errno) {
+            entry = s->ops->readdir(&s->ctx, &fidp->fs);
+            if (!entry && errno) {
                 err = -errno;
             } else {
+                *dent = entry;
                 err = 0;
             }
         });
diff --git a/hw/9pfs/cofile.c b/hw/9pfs/cofile.c
index 2efebf3..10343c0 100644
--- a/hw/9pfs/cofile.c
+++ b/hw/9pfs/cofile.c
@@ -1,6 +1,5 @@
-
 /*
- * Virtio 9p backend
+ * 9p backend
  *
  * Copyright IBM, Corp. 2011
  *
@@ -12,10 +11,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t st_mode,
                    V9fsStatDotl *v9stat)
diff --git a/hw/9pfs/cofs.c b/hw/9pfs/cofs.c
index 42ee614..70f584f 100644
--- a/hw/9pfs/cofs.c
+++ b/hw/9pfs/cofs.c
@@ -1,6 +1,5 @@
-
 /*
- * Virtio 9p backend
+ * 9p backend
  *
  * Copyright IBM, Corp. 2011
  *
@@ -12,10 +11,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 static ssize_t __readlink(V9fsState *s, V9fsPath *path, V9fsString *buf)
 {
diff --git a/hw/9pfs/coth.c b/hw/9pfs/coth.c
new file mode 100644
index 0000000..89018de
--- /dev/null
+++ b/hw/9pfs/coth.c
@@ -0,0 +1,41 @@
+/*
+ * 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Harsh Prateek Bora <harsh@linux.vnet.ibm.com>
+ *  Venkateswararao Jujjuri(JV) <jvrao@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "block/thread-pool.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
+
+/* Called from QEMU I/O thread.  */
+static void coroutine_enter_cb(void *opaque, int ret)
+{
+    Coroutine *co = opaque;
+    qemu_coroutine_enter(co);
+}
+
+/* Called from worker thread.  */
+static int coroutine_enter_func(void *arg)
+{
+    Coroutine *co = arg;
+    qemu_coroutine_enter(co);
+    return 0;
+}
+
+void co_run_in_worker_bh(void *opaque)
+{
+    Coroutine *co = opaque;
+    thread_pool_submit_aio(aio_get_thread_pool(qemu_get_aio_context()),
+                           coroutine_enter_func, co, coroutine_enter_cb, co);
+}
diff --git a/hw/9pfs/coth.h b/hw/9pfs/coth.h
new file mode 100644
index 0000000..3c7424e
--- /dev/null
+++ b/hw/9pfs/coth.h
@@ -0,0 +1,98 @@
+/*
+ * 9p backend
+ *
+ * Copyright IBM, Corp. 2010
+ *
+ * Authors:
+ *  Harsh Prateek Bora <harsh@linux.vnet.ibm.com>
+ *  Venkateswararao Jujjuri(JV) <jvrao@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_9P_COTH_H
+#define QEMU_9P_COTH_H
+
+#include "qemu/thread.h"
+#include "qemu/coroutine.h"
+#include "qemu/main-loop.h"
+#include "9p.h"
+
+/*
+ * we want to use bottom half because we want to make sure the below
+ * sequence of events.
+ *
+ *   1. Yield the coroutine in the QEMU thread.
+ *   2. Submit the coroutine to a worker thread.
+ *   3. Enter the coroutine in the worker thread.
+ * we cannot swap step 1 and 2, because that would imply worker thread
+ * can enter coroutine while step1 is still running
+ */
+#define v9fs_co_run_in_worker(code_block)                               \
+    do {                                                                \
+        QEMUBH *co_bh;                                                  \
+        co_bh = qemu_bh_new(co_run_in_worker_bh,                        \
+                            qemu_coroutine_self());                     \
+        qemu_bh_schedule(co_bh);                                        \
+        /*                                                              \
+         * yield in qemu thread and re-enter back                       \
+         * in worker thread                                             \
+         */                                                             \
+        qemu_coroutine_yield();                                         \
+        qemu_bh_delete(co_bh);                                          \
+        code_block;                                                     \
+        /* re-enter back to qemu thread */                              \
+        qemu_coroutine_yield();                                         \
+    } while (0)
+
+extern void co_run_in_worker_bh(void *);
+extern int v9fs_co_readlink(V9fsPDU *, V9fsPath *, V9fsString *);
+extern int v9fs_co_readdir(V9fsPDU *, V9fsFidState *, struct dirent **);
+extern off_t v9fs_co_telldir(V9fsPDU *, V9fsFidState *);
+extern void v9fs_co_seekdir(V9fsPDU *, V9fsFidState *, off_t);
+extern void v9fs_co_rewinddir(V9fsPDU *, V9fsFidState *);
+extern int v9fs_co_statfs(V9fsPDU *, V9fsPath *, struct statfs *);
+extern int v9fs_co_lstat(V9fsPDU *, V9fsPath *, struct stat *);
+extern int v9fs_co_chmod(V9fsPDU *, V9fsPath *, mode_t);
+extern int v9fs_co_utimensat(V9fsPDU *, V9fsPath *, struct timespec [2]);
+extern int v9fs_co_chown(V9fsPDU *, V9fsPath *, uid_t, gid_t);
+extern int v9fs_co_truncate(V9fsPDU *, V9fsPath *, off_t);
+extern int v9fs_co_llistxattr(V9fsPDU *, V9fsPath *, void *, size_t);
+extern int v9fs_co_lgetxattr(V9fsPDU *, V9fsPath *,
+                             V9fsString *, void *, size_t);
+extern int v9fs_co_mknod(V9fsPDU *, V9fsFidState *, V9fsString *, uid_t,
+                         gid_t, dev_t, mode_t, struct stat *);
+extern int v9fs_co_mkdir(V9fsPDU *, V9fsFidState *, V9fsString *,
+                         mode_t, uid_t, gid_t, struct stat *);
+extern int v9fs_co_remove(V9fsPDU *, V9fsPath *);
+extern int v9fs_co_rename(V9fsPDU *, V9fsPath *, V9fsPath *);
+extern int v9fs_co_unlinkat(V9fsPDU *, V9fsPath *, V9fsString *, int flags);
+extern int v9fs_co_renameat(V9fsPDU *, V9fsPath *, V9fsString *,
+                            V9fsPath *, V9fsString *);
+extern int v9fs_co_fstat(V9fsPDU *, V9fsFidState *, struct stat *);
+extern int v9fs_co_opendir(V9fsPDU *, V9fsFidState *);
+extern int v9fs_co_open(V9fsPDU *, V9fsFidState *, int);
+extern int v9fs_co_open2(V9fsPDU *, V9fsFidState *, V9fsString *,
+                         gid_t, int, int, struct stat *);
+extern int v9fs_co_lsetxattr(V9fsPDU *, V9fsPath *, V9fsString *,
+                             void *, size_t, int);
+extern int v9fs_co_lremovexattr(V9fsPDU *, V9fsPath *, V9fsString *);
+extern int v9fs_co_closedir(V9fsPDU *, V9fsFidOpenState *);
+extern int v9fs_co_close(V9fsPDU *, V9fsFidOpenState *);
+extern int v9fs_co_fsync(V9fsPDU *, V9fsFidState *, int);
+extern int v9fs_co_symlink(V9fsPDU *, V9fsFidState *, V9fsString *,
+                           const char *, gid_t, struct stat *);
+extern int v9fs_co_link(V9fsPDU *, V9fsFidState *,
+                        V9fsFidState *, V9fsString *);
+extern int v9fs_co_pwritev(V9fsPDU *, V9fsFidState *,
+                           struct iovec *, int, int64_t);
+extern int v9fs_co_preadv(V9fsPDU *, V9fsFidState *,
+                          struct iovec *, int, int64_t);
+extern int v9fs_co_name_to_path(V9fsPDU *, V9fsPath *,
+                                const char *, V9fsPath *);
+extern int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t,
+                          V9fsStatDotl *v9stat);
+
+#endif
diff --git a/hw/9pfs/coxattr.c b/hw/9pfs/coxattr.c
index 18ee08d..133c4ea 100644
--- a/hw/9pfs/coxattr.c
+++ b/hw/9pfs/coxattr.c
@@ -1,6 +1,5 @@
-
 /*
- * Virtio 9p backend
+ * 9p backend
  *
  * Copyright IBM, Corp. 2011
  *
@@ -12,10 +11,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "fsdev/qemu-fsdev.h"
 #include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
+#include "qemu/coroutine.h"
+#include "coth.h"
 
 int v9fs_co_llistxattr(V9fsPDU *pdu, V9fsPath *path, void *value, size_t size)
 {
diff --git a/hw/9pfs/trace-events b/hw/9pfs/trace-events
new file mode 100644
index 0000000..48d3d8a
--- /dev/null
+++ b/hw/9pfs/trace-events
@@ -0,0 +1,47 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/9pfs/virtio-9p.c
+v9fs_rerror(uint16_t tag, uint8_t id, int err) "tag %d id %d err %d"
+v9fs_version(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
+v9fs_version_return(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
+v9fs_attach(uint16_t tag, uint8_t id, int32_t fid, int32_t afid, char* uname, char* aname) "tag %u id %u fid %d afid %d uname %s aname %s"
+v9fs_attach_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d type %d version %d path %"PRId64
+v9fs_stat(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
+v9fs_stat_return(uint16_t tag, uint8_t id, int32_t mode, int32_t atime, int32_t mtime, int64_t length) "tag %d id %d stat={mode %d atime %d mtime %d length %"PRId64"}"
+v9fs_getattr(uint16_t tag, uint8_t id, int32_t fid, uint64_t request_mask) "tag %d id %d fid %d request_mask %"PRIu64
+v9fs_getattr_return(uint16_t tag, uint8_t id, uint64_t result_mask, uint32_t mode, uint32_t uid, uint32_t gid) "tag %d id %d getattr={result_mask %"PRId64" mode %u uid %u gid %u}"
+v9fs_walk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, uint16_t nwnames) "tag %d id %d fid %d newfid %d nwnames %d"
+v9fs_walk_return(uint16_t tag, uint8_t id, uint16_t nwnames, void* qids) "tag %d id %d nwnames %d qids %p"
+v9fs_open(uint16_t tag, uint8_t id, int32_t fid, int32_t mode) "tag %d id %d fid %d mode %d"
+v9fs_open_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int iounit) "tag %d id %d qid={type %d version %d path %"PRId64"} iounit %d"
+v9fs_lcreate(uint16_t tag, uint8_t id, int32_t dfid, int32_t flags, int32_t mode, uint32_t gid) "tag %d id %d dfid %d flags %d mode %d gid %u"
+v9fs_lcreate_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int32_t iounit) "tag %d id %d qid={type %d version %d path %"PRId64"} iounit %d"
+v9fs_fsync(uint16_t tag, uint8_t id, int32_t fid, int datasync) "tag %d id %d fid %d datasync %d"
+v9fs_clunk(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
+v9fs_read(uint16_t tag, uint8_t id, int32_t fid, uint64_t off, uint32_t max_count) "tag %d id %d fid %d off %"PRIu64" max_count %u"
+v9fs_read_return(uint16_t tag, uint8_t id, int32_t count, ssize_t err) "tag %d id %d count %d err %zd"
+v9fs_readdir(uint16_t tag, uint8_t id, int32_t fid, uint64_t offset, uint32_t max_count) "tag %d id %d fid %d offset %"PRIu64" max_count %u"
+v9fs_readdir_return(uint16_t tag, uint8_t id, uint32_t count, ssize_t retval) "tag %d id %d count %u retval %zd"
+v9fs_write(uint16_t tag, uint8_t id, int32_t fid, uint64_t off, uint32_t count, int cnt) "tag %d id %d fid %d off %"PRIu64" count %u cnt %d"
+v9fs_write_return(uint16_t tag, uint8_t id, int32_t total, ssize_t err) "tag %d id %d total %d err %zd"
+v9fs_create(uint16_t tag, uint8_t id, int32_t fid, char* name, int32_t perm, int8_t mode) "tag %d id %d fid %d name %s perm %d mode %d"
+v9fs_create_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int iounit) "tag %d id %d qid={type %d version %d path %"PRId64"} iounit %d"
+v9fs_symlink(uint16_t tag, uint8_t id, int32_t fid,  char* name, char* symname, uint32_t gid) "tag %d id %d fid %d name %s symname %s gid %u"
+v9fs_symlink_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d qid={type %d version %d path %"PRId64"}"
+v9fs_flush(uint16_t tag, uint8_t id, int16_t flush_tag) "tag %d id %d flush_tag %d"
+v9fs_link(uint16_t tag, uint8_t id, int32_t dfid, int32_t oldfid, char* name) "tag %d id %d dfid %d oldfid %d name %s"
+v9fs_remove(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
+v9fs_wstat(uint16_t tag, uint8_t id, int32_t fid, int32_t mode, int32_t atime, int32_t mtime) "tag %u id %u fid %d stat={mode %d atime %d mtime %d}"
+v9fs_mknod(uint16_t tag, uint8_t id, int32_t fid, int mode, int major, int minor) "tag %d id %d fid %d mode %d major %d minor %d"
+v9fs_mknod_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d qid={type %d version %d path %"PRId64"}"
+v9fs_lock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length) "tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64
+v9fs_lock_return(uint16_t tag, uint8_t id, int8_t status) "tag %d id %d status %d"
+v9fs_getlock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length)"tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64
+v9fs_getlock_return(uint16_t tag, uint8_t id, uint8_t type, uint64_t start, uint64_t length, uint32_t proc_id) "tag %d id %d type %d start %"PRIu64" length %"PRIu64" proc_id %u"
+v9fs_mkdir(uint16_t tag, uint8_t id, int32_t fid, char* name, int mode, uint32_t gid) "tag %u id %u fid %d name %s mode %d gid %u"
+v9fs_mkdir_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int err) "tag %u id %u qid={type %d version %d path %"PRId64"} err %d"
+v9fs_xattrwalk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, char* name) "tag %d id %d fid %d newfid %d name %s"
+v9fs_xattrwalk_return(uint16_t tag, uint8_t id, int64_t size) "tag %d id %d size %"PRId64
+v9fs_xattrcreate(uint16_t tag, uint8_t id, int32_t fid, char* name, int64_t size, int flags) "tag %d id %d fid %d name %s size %"PRId64" flags %d"
+v9fs_readlink(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
+v9fs_readlink_return(uint16_t tag, uint8_t id, char* target) "tag %d id %d name %s"
diff --git a/hw/9pfs/virtio-9p-coth.c b/hw/9pfs/virtio-9p-coth.c
deleted file mode 100644
index ae6cde8..0000000
--- a/hw/9pfs/virtio-9p-coth.c
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Virtio 9p backend
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Harsh Prateek Bora <harsh@linux.vnet.ibm.com>
- *  Venkateswararao Jujjuri(JV) <jvrao@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "fsdev/qemu-fsdev.h"
-#include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p-coth.h"
-
-/* v9fs glib thread pool */
-static V9fsThPool v9fs_pool;
-
-void co_run_in_worker_bh(void *opaque)
-{
-    Coroutine *co = opaque;
-    g_thread_pool_push(v9fs_pool.pool, co, NULL);
-}
-
-static void v9fs_qemu_process_req_done(void *arg)
-{
-    char byte;
-    ssize_t len;
-    Coroutine *co;
-
-    do {
-        len = read(v9fs_pool.rfd, &byte, sizeof(byte));
-    } while (len == -1 &&  errno == EINTR);
-
-    while ((co = g_async_queue_try_pop(v9fs_pool.completed)) != NULL) {
-        qemu_coroutine_enter(co, NULL);
-    }
-}
-
-static void v9fs_thread_routine(gpointer data, gpointer user_data)
-{
-    ssize_t len;
-    char byte = 0;
-    Coroutine *co = data;
-
-    qemu_coroutine_enter(co, NULL);
-
-    g_async_queue_push(v9fs_pool.completed, co);
-    do {
-        len = write(v9fs_pool.wfd, &byte, sizeof(byte));
-    } while (len == -1 && errno == EINTR);
-}
-
-int v9fs_init_worker_threads(void)
-{
-    int ret = 0;
-    int notifier_fds[2];
-    V9fsThPool *p = &v9fs_pool;
-    sigset_t set, oldset;
-
-    sigfillset(&set);
-    /* Leave signal handling to the iothread.  */
-    pthread_sigmask(SIG_SETMASK, &set, &oldset);
-
-    if (qemu_pipe(notifier_fds) == -1) {
-        ret = -1;
-        goto err_out;
-    }
-    p->pool = g_thread_pool_new(v9fs_thread_routine, p, -1, FALSE, NULL);
-    if (!p->pool) {
-        ret = -1;
-        goto err_out;
-    }
-    p->completed = g_async_queue_new();
-    if (!p->completed) {
-        /*
-         * We are going to terminate.
-         * So don't worry about cleanup
-         */
-        ret = -1;
-        goto err_out;
-    }
-    p->rfd = notifier_fds[0];
-    p->wfd = notifier_fds[1];
-
-    fcntl(p->rfd, F_SETFL, O_NONBLOCK);
-    fcntl(p->wfd, F_SETFL, O_NONBLOCK);
-
-    qemu_set_fd_handler(p->rfd, v9fs_qemu_process_req_done, NULL, NULL);
-err_out:
-    pthread_sigmask(SIG_SETMASK, &oldset, NULL);
-    return ret;
-}
diff --git a/hw/9pfs/virtio-9p-coth.h b/hw/9pfs/virtio-9p-coth.h
deleted file mode 100644
index 86d5ed4..0000000
--- a/hw/9pfs/virtio-9p-coth.h
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Virtio 9p backend
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Harsh Prateek Bora <harsh@linux.vnet.ibm.com>
- *  Venkateswararao Jujjuri(JV) <jvrao@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef _QEMU_VIRTIO_9P_COTH_H
-#define _QEMU_VIRTIO_9P_COTH_H
-
-#include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "virtio-9p.h"
-#include <glib.h>
-
-typedef struct V9fsThPool {
-    int rfd;
-    int wfd;
-    GThreadPool *pool;
-    GAsyncQueue *completed;
-} V9fsThPool;
-
-/*
- * we want to use bottom half because we want to make sure the below
- * sequence of events.
- *
- *   1. Yield the coroutine in the QEMU thread.
- *   2. Submit the coroutine to a worker thread.
- *   3. Enter the coroutine in the worker thread.
- * we cannot swap step 1 and 2, because that would imply worker thread
- * can enter coroutine while step1 is still running
- */
-#define v9fs_co_run_in_worker(code_block)                               \
-    do {                                                                \
-        QEMUBH *co_bh;                                                  \
-        co_bh = qemu_bh_new(co_run_in_worker_bh,                        \
-                            qemu_coroutine_self());                     \
-        qemu_bh_schedule(co_bh);                                        \
-        /*                                                              \
-         * yield in qemu thread and re-enter back                       \
-         * in glib worker thread                                        \
-         */                                                             \
-        qemu_coroutine_yield();                                         \
-        qemu_bh_delete(co_bh);                                          \
-        code_block;                                                     \
-        /* re-enter back to qemu thread */                              \
-        qemu_coroutine_yield();                                         \
-    } while (0)
-
-extern void co_run_in_worker_bh(void *);
-extern int v9fs_init_worker_threads(void);
-extern int v9fs_co_readlink(V9fsPDU *, V9fsPath *, V9fsString *);
-extern int v9fs_co_readdir_r(V9fsPDU *, V9fsFidState *,
-                           struct dirent *, struct dirent **result);
-extern off_t v9fs_co_telldir(V9fsPDU *, V9fsFidState *);
-extern void v9fs_co_seekdir(V9fsPDU *, V9fsFidState *, off_t);
-extern void v9fs_co_rewinddir(V9fsPDU *, V9fsFidState *);
-extern int v9fs_co_statfs(V9fsPDU *, V9fsPath *, struct statfs *);
-extern int v9fs_co_lstat(V9fsPDU *, V9fsPath *, struct stat *);
-extern int v9fs_co_chmod(V9fsPDU *, V9fsPath *, mode_t);
-extern int v9fs_co_utimensat(V9fsPDU *, V9fsPath *, struct timespec [2]);
-extern int v9fs_co_chown(V9fsPDU *, V9fsPath *, uid_t, gid_t);
-extern int v9fs_co_truncate(V9fsPDU *, V9fsPath *, off_t);
-extern int v9fs_co_llistxattr(V9fsPDU *, V9fsPath *, void *, size_t);
-extern int v9fs_co_lgetxattr(V9fsPDU *, V9fsPath *,
-                             V9fsString *, void *, size_t);
-extern int v9fs_co_mknod(V9fsPDU *, V9fsFidState *, V9fsString *, uid_t,
-                         gid_t, dev_t, mode_t, struct stat *);
-extern int v9fs_co_mkdir(V9fsPDU *, V9fsFidState *, V9fsString *,
-                         mode_t, uid_t, gid_t, struct stat *);
-extern int v9fs_co_remove(V9fsPDU *, V9fsPath *);
-extern int v9fs_co_rename(V9fsPDU *, V9fsPath *, V9fsPath *);
-extern int v9fs_co_unlinkat(V9fsPDU *, V9fsPath *, V9fsString *, int flags);
-extern int v9fs_co_renameat(V9fsPDU *, V9fsPath *, V9fsString *,
-                            V9fsPath *, V9fsString *);
-extern int v9fs_co_fstat(V9fsPDU *, V9fsFidState *, struct stat *);
-extern int v9fs_co_opendir(V9fsPDU *, V9fsFidState *);
-extern int v9fs_co_open(V9fsPDU *, V9fsFidState *, int);
-extern int v9fs_co_open2(V9fsPDU *, V9fsFidState *, V9fsString *,
-                         gid_t, int, int, struct stat *);
-extern int v9fs_co_lsetxattr(V9fsPDU *, V9fsPath *, V9fsString *,
-                             void *, size_t, int);
-extern int v9fs_co_lremovexattr(V9fsPDU *, V9fsPath *, V9fsString *);
-extern int v9fs_co_closedir(V9fsPDU *, V9fsFidOpenState *);
-extern int v9fs_co_close(V9fsPDU *, V9fsFidOpenState *);
-extern int v9fs_co_fsync(V9fsPDU *, V9fsFidState *, int);
-extern int v9fs_co_symlink(V9fsPDU *, V9fsFidState *, V9fsString *,
-                           const char *, gid_t, struct stat *);
-extern int v9fs_co_link(V9fsPDU *, V9fsFidState *,
-                        V9fsFidState *, V9fsString *);
-extern int v9fs_co_pwritev(V9fsPDU *, V9fsFidState *,
-                           struct iovec *, int, int64_t);
-extern int v9fs_co_preadv(V9fsPDU *, V9fsFidState *,
-                          struct iovec *, int, int64_t);
-extern int v9fs_co_name_to_path(V9fsPDU *, V9fsPath *,
-                                const char *, V9fsPath *);
-extern int v9fs_co_st_gen(V9fsPDU *pdu, V9fsPath *path, mode_t,
-                          V9fsStatDotl *v9stat);
-
-#endif
diff --git a/hw/9pfs/virtio-9p-device.c b/hw/9pfs/virtio-9p-device.c
index 2572747..009b43f 100644
--- a/hw/9pfs/virtio-9p-device.c
+++ b/hw/9pfs/virtio-9p-device.c
@@ -11,19 +11,73 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/virtio.h"
-#include "hw/virtio/virtio-9p.h"
-#include "hw/i386/pc.h"
 #include "qemu/sockets.h"
 #include "virtio-9p.h"
 #include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-xattr.h"
-#include "virtio-9p-coth.h"
+#include "coth.h"
 #include "hw/virtio/virtio-access.h"
+#include "qemu/iov.h"
 
-static uint32_t virtio_9p_get_features(VirtIODevice *vdev, uint32_t features)
+void virtio_9p_push_and_notify(V9fsPDU *pdu)
 {
-    features |= 1 << VIRTIO_9P_MOUNT_TAG;
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    /* push onto queue and notify */
+    virtqueue_push(v->vq, elem, pdu->size);
+    g_free(elem);
+    v->elems[pdu->idx] = NULL;
+
+    /* FIXME: we should batch these completions */
+    virtio_notify(VIRTIO_DEVICE(v), v->vq);
+}
+
+static void handle_9p_output(VirtIODevice *vdev, VirtQueue *vq)
+{
+    V9fsVirtioState *v = (V9fsVirtioState *)vdev;
+    V9fsState *s = &v->state;
+    V9fsPDU *pdu;
+    ssize_t len;
+
+    while ((pdu = pdu_alloc(s))) {
+        struct {
+            uint32_t size_le;
+            uint8_t id;
+            uint16_t tag_le;
+        } QEMU_PACKED out;
+        VirtQueueElement *elem;
+
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            pdu_free(pdu);
+            break;
+        }
+
+        BUG_ON(elem->out_num == 0 || elem->in_num == 0);
+        QEMU_BUILD_BUG_ON(sizeof out != 7);
+
+        v->elems[pdu->idx] = elem;
+        len = iov_to_buf(elem->out_sg, elem->out_num, 0,
+                         &out, sizeof out);
+        BUG_ON(len != sizeof out);
+
+        pdu->size = le32_to_cpu(out.size_le);
+
+        pdu->id = out.id;
+        pdu->tag = le16_to_cpu(out.tag_le);
+
+        qemu_co_queue_init(&pdu->complete);
+        pdu_submit(pdu);
+    }
+}
+
+static uint64_t virtio_9p_get_features(VirtIODevice *vdev, uint64_t features,
+                                       Error **errp)
+{
+    virtio_add_feature(&features, VIRTIO_9P_MOUNT_TAG);
     return features;
 }
 
@@ -31,116 +85,94 @@
 {
     int len;
     struct virtio_9p_config *cfg;
-    V9fsState *s = VIRTIO_9P(vdev);
+    V9fsVirtioState *v = VIRTIO_9P(vdev);
+    V9fsState *s = &v->state;
 
     len = strlen(s->tag);
     cfg = g_malloc0(sizeof(struct virtio_9p_config) + len);
     virtio_stw_p(vdev, &cfg->tag_len, len);
     /* We don't copy the terminating null to config space */
     memcpy(cfg->tag, s->tag, len);
-    memcpy(config, cfg, s->config_size);
+    memcpy(config, cfg, v->config_size);
     g_free(cfg);
 }
 
+static int virtio_9p_load(QEMUFile *f, void *opaque, size_t size)
+{
+    return virtio_load(VIRTIO_DEVICE(opaque), f, 1);
+}
+
 static void virtio_9p_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
-    V9fsState *s = VIRTIO_9P(dev);
-    int i, len;
-    struct stat stat;
-    FsDriverEntry *fse;
-    V9fsPath path;
+    V9fsVirtioState *v = VIRTIO_9P(dev);
+    V9fsState *s = &v->state;
 
-    virtio_init(vdev, "virtio-9p", VIRTIO_ID_9P,
-                sizeof(struct virtio_9p_config) + MAX_TAG_LEN);
-
-    /* initialize pdu allocator */
-    QLIST_INIT(&s->free_list);
-    QLIST_INIT(&s->active_list);
-    for (i = 0; i < (MAX_REQ - 1); i++) {
-        QLIST_INSERT_HEAD(&s->free_list, &s->pdus[i], next);
-    }
-
-    s->vq = virtio_add_queue(vdev, MAX_REQ, handle_9p_output);
-
-    v9fs_path_init(&path);
-
-    fse = get_fsdev_fsentry(s->fsconf.fsdev_id);
-
-    if (!fse) {
-        /* We don't have a fsdev identified by fsdev_id */
-        error_setg(errp, "Virtio-9p device couldn't find fsdev with the "
-                   "id = %s",
-                   s->fsconf.fsdev_id ? s->fsconf.fsdev_id : "NULL");
+    if (v9fs_device_realize_common(s, errp)) {
         goto out;
     }
 
-    if (!s->fsconf.tag) {
-        /* we haven't specified a mount_tag */
-        error_setg(errp, "fsdev with id %s needs mount_tag arguments",
-                   s->fsconf.fsdev_id);
-        goto out;
-    }
+    v->config_size = sizeof(struct virtio_9p_config) + strlen(s->fsconf.tag);
+    virtio_init(vdev, "virtio-9p", VIRTIO_ID_9P, v->config_size);
+    v->vq = virtio_add_queue(vdev, MAX_REQ, handle_9p_output);
 
-    s->ctx.export_flags = fse->export_flags;
-    s->ctx.fs_root = g_strdup(fse->path);
-    s->ctx.exops.get_st_gen = NULL;
-    len = strlen(s->fsconf.tag);
-    if (len > MAX_TAG_LEN - 1) {
-        error_setg(errp, "mount tag '%s' (%d bytes) is longer than "
-                   "maximum (%d bytes)", s->fsconf.tag, len, MAX_TAG_LEN - 1);
-        goto out;
-    }
-
-    s->tag = g_strdup(s->fsconf.tag);
-    s->ctx.uid = -1;
-
-    s->ops = fse->ops;
-    s->config_size = sizeof(struct virtio_9p_config) + len;
-    s->fid_list = NULL;
-    qemu_co_rwlock_init(&s->rename_lock);
-
-    if (s->ops->init(&s->ctx) < 0) {
-        error_setg(errp, "Virtio-9p Failed to initialize fs-driver with id:%s"
-                   " and export path:%s", s->fsconf.fsdev_id, s->ctx.fs_root);
-        goto out;
-    }
-    if (v9fs_init_worker_threads() < 0) {
-        error_setg(errp, "worker thread initialization failed");
-        goto out;
-    }
-
-    /*
-     * Check details of export path, We need to use fs driver
-     * call back to do that. Since we are in the init path, we don't
-     * use co-routines here.
-     */
-    if (s->ops->name_to_path(&s->ctx, NULL, "/", &path) < 0) {
-        error_setg(errp,
-                   "error in converting name to path %s", strerror(errno));
-        goto out;
-    }
-    if (s->ops->lstat(&s->ctx, &path, &stat)) {
-        error_setg(errp, "share path %s does not exist", fse->path);
-        goto out;
-    } else if (!S_ISDIR(stat.st_mode)) {
-        error_setg(errp, "share path %s is not a directory", fse->path);
-        goto out;
-    }
-    v9fs_path_free(&path);
-
-    return;
 out:
-    g_free(s->ctx.fs_root);
-    g_free(s->tag);
+    return;
+}
+
+static void virtio_9p_device_unrealize(DeviceState *dev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    V9fsVirtioState *v = VIRTIO_9P(dev);
+    V9fsState *s = &v->state;
+
     virtio_cleanup(vdev);
-    v9fs_path_free(&path);
+    v9fs_device_unrealize_common(s, errp);
+}
+
+ssize_t virtio_pdu_vmarshal(V9fsPDU *pdu, size_t offset,
+                            const char *fmt, va_list ap)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    return v9fs_iov_vmarshal(elem->in_sg, elem->in_num, offset, 1, fmt, ap);
+}
+
+ssize_t virtio_pdu_vunmarshal(V9fsPDU *pdu, size_t offset,
+                              const char *fmt, va_list ap)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    return v9fs_iov_vunmarshal(elem->out_sg, elem->out_num, offset, 1, fmt, ap);
+}
+
+void virtio_init_iov_from_pdu(V9fsPDU *pdu, struct iovec **piov,
+                              unsigned int *pniov, bool is_write)
+{
+    V9fsState *s = pdu->s;
+    V9fsVirtioState *v = container_of(s, V9fsVirtioState, state);
+    VirtQueueElement *elem = v->elems[pdu->idx];
+
+    if (is_write) {
+        *piov = elem->out_sg;
+        *pniov = elem->out_num;
+    } else {
+        *piov = elem->in_sg;
+        *pniov = elem->in_num;
+    }
 }
 
 /* virtio-9p device */
 
+VMSTATE_VIRTIO_DEVICE(9p, 1, virtio_9p_load, virtio_vmstate_save);
+
 static Property virtio_9p_properties[] = {
-    DEFINE_VIRTIO_9P_PROPERTIES(V9fsState, fsconf),
+    DEFINE_PROP_STRING("mount_tag", V9fsVirtioState, state.fsconf.tag),
+    DEFINE_PROP_STRING("fsdev", V9fsVirtioState, state.fsconf.fsdev_id),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -150,8 +182,10 @@
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 
     dc->props = virtio_9p_properties;
+    dc->vmsd = &vmstate_virtio_9p;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     vdc->realize = virtio_9p_device_realize;
+    vdc->unrealize = virtio_9p_device_unrealize;
     vdc->get_features = virtio_9p_get_features;
     vdc->get_config = virtio_9p_get_config;
 }
@@ -159,7 +193,7 @@
 static const TypeInfo virtio_device_info = {
     .name = TYPE_VIRTIO_9P,
     .parent = TYPE_VIRTIO_DEVICE,
-    .instance_size = sizeof(V9fsState),
+    .instance_size = sizeof(V9fsVirtioState),
     .class_init = virtio_9p_class_init,
 };
 
diff --git a/hw/9pfs/virtio-9p-handle.c b/hw/9pfs/virtio-9p-handle.c
deleted file mode 100644
index 4b79cef..0000000
--- a/hw/9pfs/virtio-9p-handle.c
+++ /dev/null
@@ -1,708 +0,0 @@
-/*
- * Virtio 9p handle callback
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *    Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "virtio-9p-xattr.h"
-#include <arpa/inet.h>
-#include <pwd.h>
-#include <grp.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include "qemu/xattr.h"
-#include <unistd.h>
-#include <linux/fs.h>
-#ifdef CONFIG_LINUX_MAGIC_H
-#include <linux/magic.h>
-#endif
-#include <sys/ioctl.h>
-
-#ifndef XFS_SUPER_MAGIC
-#define XFS_SUPER_MAGIC  0x58465342
-#endif
-#ifndef EXT2_SUPER_MAGIC
-#define EXT2_SUPER_MAGIC 0xEF53
-#endif
-#ifndef REISERFS_SUPER_MAGIC
-#define REISERFS_SUPER_MAGIC 0x52654973
-#endif
-#ifndef BTRFS_SUPER_MAGIC
-#define BTRFS_SUPER_MAGIC 0x9123683E
-#endif
-
-struct handle_data {
-    int mountfd;
-    int handle_bytes;
-};
-
-static inline int name_to_handle(int dirfd, const char *name,
-                                 struct file_handle *fh, int *mnt_id, int flags)
-{
-    return name_to_handle_at(dirfd, name, fh, mnt_id, flags);
-}
-
-static inline int open_by_handle(int mountfd, const char *fh, int flags)
-{
-    return open_by_handle_at(mountfd, (struct file_handle *)fh, flags);
-}
-
-static int handle_update_file_cred(int dirfd, const char *name, FsCred *credp)
-{
-    int fd, ret;
-    fd = openat(dirfd, name, O_NONBLOCK | O_NOFOLLOW);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fchownat(fd, "", credp->fc_uid, credp->fc_gid, AT_EMPTY_PATH);
-    if (ret < 0) {
-        goto err_out;
-    }
-    ret = fchmod(fd, credp->fc_mode & 07777);
-err_out:
-    close(fd);
-    return ret;
-}
-
-
-static int handle_lstat(FsContext *fs_ctx, V9fsPath *fs_path,
-                        struct stat *stbuf)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_PATH);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fstatat(fd, "", stbuf, AT_EMPTY_PATH);
-    close(fd);
-    return ret;
-}
-
-static ssize_t handle_readlink(FsContext *fs_ctx, V9fsPath *fs_path,
-                               char *buf, size_t bufsz)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_PATH);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = readlinkat(fd, "", buf, bufsz);
-    close(fd);
-    return ret;
-}
-
-static int handle_close(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return close(fs->fd);
-}
-
-static int handle_closedir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return closedir(fs->dir);
-}
-
-static int handle_open(FsContext *ctx, V9fsPath *fs_path,
-                       int flags, V9fsFidOpenState *fs)
-{
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fs->fd = open_by_handle(data->mountfd, fs_path->data, flags);
-    return fs->fd;
-}
-
-static int handle_opendir(FsContext *ctx,
-                          V9fsPath *fs_path, V9fsFidOpenState *fs)
-{
-    int ret;
-    ret = handle_open(ctx, fs_path, O_DIRECTORY, fs);
-    if (ret < 0) {
-        return -1;
-    }
-    fs->dir = fdopendir(ret);
-    if (!fs->dir) {
-        return -1;
-    }
-    return 0;
-}
-
-static void handle_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return rewinddir(fs->dir);
-}
-
-static off_t handle_telldir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return telldir(fs->dir);
-}
-
-static int handle_readdir_r(FsContext *ctx, V9fsFidOpenState *fs,
-                            struct dirent *entry,
-                            struct dirent **result)
-{
-    return readdir_r(fs->dir, entry, result);
-}
-
-static void handle_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
-{
-    return seekdir(fs->dir, off);
-}
-
-static ssize_t handle_preadv(FsContext *ctx, V9fsFidOpenState *fs,
-                             const struct iovec *iov,
-                             int iovcnt, off_t offset)
-{
-#ifdef CONFIG_PREADV
-    return preadv(fs->fd, iov, iovcnt, offset);
-#else
-    int err = lseek(fs->fd, offset, SEEK_SET);
-    if (err == -1) {
-        return err;
-    } else {
-        return readv(fs->fd, iov, iovcnt);
-    }
-#endif
-}
-
-static ssize_t handle_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
-                              const struct iovec *iov,
-                              int iovcnt, off_t offset)
-{
-    ssize_t ret;
-#ifdef CONFIG_PREADV
-    ret = pwritev(fs->fd, iov, iovcnt, offset);
-#else
-    int err = lseek(fs->fd, offset, SEEK_SET);
-    if (err == -1) {
-        return err;
-    } else {
-        ret = writev(fs->fd, iov, iovcnt);
-    }
-#endif
-#ifdef CONFIG_SYNC_FILE_RANGE
-    if (ret > 0 && ctx->export_flags & V9FS_IMMEDIATE_WRITEOUT) {
-        /*
-         * Initiate a writeback. This is not a data integrity sync.
-         * We want to ensure that we don't leave dirty pages in the cache
-         * after write when writeout=immediate is sepcified.
-         */
-        sync_file_range(fs->fd, offset, ret,
-                        SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
-    }
-#endif
-    return ret;
-}
-
-static int handle_chmod(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fchmod(fd, credp->fc_mode);
-    close(fd);
-    return ret;
-}
-
-static int handle_mknod(FsContext *fs_ctx, V9fsPath *dir_path,
-                       const char *name, FsCred *credp)
-{
-    int dirfd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
-    if (dirfd < 0) {
-        return dirfd;
-    }
-    ret = mknodat(dirfd, name, credp->fc_mode, credp->fc_rdev);
-    if (!ret) {
-        ret = handle_update_file_cred(dirfd, name, credp);
-    }
-    close(dirfd);
-    return ret;
-}
-
-static int handle_mkdir(FsContext *fs_ctx, V9fsPath *dir_path,
-                       const char *name, FsCred *credp)
-{
-    int dirfd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
-    if (dirfd < 0) {
-        return dirfd;
-    }
-    ret = mkdirat(dirfd, name, credp->fc_mode);
-    if (!ret) {
-        ret = handle_update_file_cred(dirfd, name, credp);
-    }
-    close(dirfd);
-    return ret;
-}
-
-static int handle_fstat(FsContext *fs_ctx, int fid_type,
-                        V9fsFidOpenState *fs, struct stat *stbuf)
-{
-    int fd;
-
-    if (fid_type == P9_FID_DIR) {
-        fd = dirfd(fs->dir);
-    } else {
-        fd = fs->fd;
-    }
-    return fstat(fd, stbuf);
-}
-
-static int handle_open2(FsContext *fs_ctx, V9fsPath *dir_path, const char *name,
-                        int flags, FsCred *credp, V9fsFidOpenState *fs)
-{
-    int ret;
-    int dirfd, fd;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
-    if (dirfd < 0) {
-        return dirfd;
-    }
-    fd = openat(dirfd, name, flags | O_NOFOLLOW, credp->fc_mode);
-    if (fd >= 0) {
-        ret = handle_update_file_cred(dirfd, name, credp);
-        if (ret < 0) {
-            close(fd);
-            fd = ret;
-        } else {
-            fs->fd = fd;
-        }
-    }
-    close(dirfd);
-    return fd;
-}
-
-
-static int handle_symlink(FsContext *fs_ctx, const char *oldpath,
-                          V9fsPath *dir_path, const char *name, FsCred *credp)
-{
-    int fd, dirfd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
-    if (dirfd < 0) {
-        return dirfd;
-    }
-    ret = symlinkat(oldpath, dirfd, name);
-    if (!ret) {
-        fd = openat(dirfd, name, O_PATH | O_NOFOLLOW);
-        if (fd < 0) {
-            ret = fd;
-            goto err_out;
-        }
-        ret = fchownat(fd, "", credp->fc_uid, credp->fc_gid, AT_EMPTY_PATH);
-        close(fd);
-    }
-err_out:
-    close(dirfd);
-    return ret;
-}
-
-static int handle_link(FsContext *ctx, V9fsPath *oldpath,
-                       V9fsPath *dirpath, const char *name)
-{
-    int oldfd, newdirfd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    oldfd = open_by_handle(data->mountfd, oldpath->data, O_PATH);
-    if (oldfd < 0) {
-        return oldfd;
-    }
-    newdirfd = open_by_handle(data->mountfd, dirpath->data, O_PATH);
-    if (newdirfd < 0) {
-        close(oldfd);
-        return newdirfd;
-    }
-    ret = linkat(oldfd, "", newdirfd, name, AT_EMPTY_PATH);
-    close(newdirfd);
-    close(oldfd);
-    return ret;
-}
-
-static int handle_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK | O_WRONLY);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = ftruncate(fd, size);
-    close(fd);
-    return ret;
-}
-
-static int handle_rename(FsContext *ctx, const char *oldpath,
-                         const char *newpath)
-{
-    errno = EOPNOTSUPP;
-    return -1;
-}
-
-static int handle_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)fs_ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_PATH);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fchownat(fd, "", credp->fc_uid, credp->fc_gid, AT_EMPTY_PATH);
-    close(fd);
-    return ret;
-}
-
-static int handle_utimensat(FsContext *ctx, V9fsPath *fs_path,
-                            const struct timespec *buf)
-{
-    int ret;
-#ifdef CONFIG_UTIMENSAT
-    int fd;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = futimens(fd, buf);
-    close(fd);
-#else
-    ret = -1;
-    errno = ENOSYS;
-#endif
-    return ret;
-}
-
-static int handle_remove(FsContext *ctx, const char *path)
-{
-    errno = EOPNOTSUPP;
-    return -1;
-}
-
-static int handle_fsync(FsContext *ctx, int fid_type,
-                        V9fsFidOpenState *fs, int datasync)
-{
-    int fd;
-
-    if (fid_type == P9_FID_DIR) {
-        fd = dirfd(fs->dir);
-    } else {
-        fd = fs->fd;
-    }
-
-    if (datasync) {
-        return qemu_fdatasync(fd);
-    } else {
-        return fsync(fd);
-    }
-}
-
-static int handle_statfs(FsContext *ctx, V9fsPath *fs_path,
-                         struct statfs *stbuf)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fstatfs(fd, stbuf);
-    close(fd);
-    return ret;
-}
-
-static ssize_t handle_lgetxattr(FsContext *ctx, V9fsPath *fs_path,
-                                const char *name, void *value, size_t size)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fgetxattr(fd, name, value, size);
-    close(fd);
-    return ret;
-}
-
-static ssize_t handle_llistxattr(FsContext *ctx, V9fsPath *fs_path,
-                                 void *value, size_t size)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = flistxattr(fd, value, size);
-    close(fd);
-    return ret;
-}
-
-static int handle_lsetxattr(FsContext *ctx, V9fsPath *fs_path, const char *name,
-                            void *value, size_t size, int flags)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fsetxattr(fd, name, value, size, flags);
-    close(fd);
-    return ret;
-}
-
-static int handle_lremovexattr(FsContext *ctx, V9fsPath *fs_path,
-                               const char *name)
-{
-    int fd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    fd = open_by_handle(data->mountfd, fs_path->data, O_NONBLOCK);
-    if (fd < 0) {
-        return fd;
-    }
-    ret = fremovexattr(fd, name);
-    close(fd);
-    return ret;
-}
-
-static int handle_name_to_path(FsContext *ctx, V9fsPath *dir_path,
-                              const char *name, V9fsPath *target)
-{
-    char *buffer;
-    struct file_handle *fh;
-    int dirfd, ret, mnt_id;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    /* "." and ".." are not allowed */
-    if (!strcmp(name, ".") || !strcmp(name, "..")) {
-        errno = EINVAL;
-        return -1;
-
-    }
-    if (dir_path) {
-        dirfd = open_by_handle(data->mountfd, dir_path->data, O_PATH);
-    } else {
-        /* relative to export root */
-        buffer = rpath(ctx, ".");
-        dirfd = open(buffer, O_DIRECTORY);
-        g_free(buffer);
-    }
-    if (dirfd < 0) {
-        return dirfd;
-    }
-    fh = g_malloc(sizeof(struct file_handle) + data->handle_bytes);
-    fh->handle_bytes = data->handle_bytes;
-    /* add a "./" at the beginning of the path */
-    buffer = g_strdup_printf("./%s", name);
-    /* flag = 0 imply don't follow symlink */
-    ret = name_to_handle(dirfd, buffer, fh, &mnt_id, 0);
-    if (!ret) {
-        target->data = (char *)fh;
-        target->size = sizeof(struct file_handle) + data->handle_bytes;
-    } else {
-        g_free(fh);
-    }
-    close(dirfd);
-    g_free(buffer);
-    return ret;
-}
-
-static int handle_renameat(FsContext *ctx, V9fsPath *olddir,
-                           const char *old_name, V9fsPath *newdir,
-                           const char *new_name)
-{
-    int olddirfd, newdirfd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-
-    olddirfd = open_by_handle(data->mountfd, olddir->data, O_PATH);
-    if (olddirfd < 0) {
-        return olddirfd;
-    }
-    newdirfd = open_by_handle(data->mountfd, newdir->data, O_PATH);
-    if (newdirfd < 0) {
-        close(olddirfd);
-        return newdirfd;
-    }
-    ret = renameat(olddirfd, old_name, newdirfd, new_name);
-    close(newdirfd);
-    close(olddirfd);
-    return ret;
-}
-
-static int handle_unlinkat(FsContext *ctx, V9fsPath *dir,
-                           const char *name, int flags)
-{
-    int dirfd, ret;
-    struct handle_data *data = (struct handle_data *)ctx->private;
-    int rflags;
-
-    dirfd = open_by_handle(data->mountfd, dir->data, O_PATH);
-    if (dirfd < 0) {
-        return dirfd;
-    }
-
-    rflags = 0;
-    if (flags & P9_DOTL_AT_REMOVEDIR) {
-        rflags |= AT_REMOVEDIR;
-    }
-
-    ret = unlinkat(dirfd, name, rflags);
-
-    close(dirfd);
-    return ret;
-}
-
-static int handle_ioc_getversion(FsContext *ctx, V9fsPath *path,
-                                 mode_t st_mode, uint64_t *st_gen)
-{
-#ifdef FS_IOC_GETVERSION
-    int err;
-    V9fsFidOpenState fid_open;
-
-    /*
-     * Do not try to open special files like device nodes, fifos etc
-     * We can get fd for regular files and directories only
-     */
-    if (!S_ISREG(st_mode) && !S_ISDIR(st_mode)) {
-        errno = ENOTTY;
-        return -1;
-    }
-    err = handle_open(ctx, path, O_RDONLY, &fid_open);
-    if (err < 0) {
-        return err;
-    }
-    err = ioctl(fid_open.fd, FS_IOC_GETVERSION, st_gen);
-    handle_close(ctx, &fid_open);
-    return err;
-#else
-    errno = ENOTTY;
-    return -1;
-#endif
-}
-
-static int handle_init(FsContext *ctx)
-{
-    int ret, mnt_id;
-    struct statfs stbuf;
-    struct file_handle fh;
-    struct handle_data *data = g_malloc(sizeof(struct handle_data));
-
-    data->mountfd = open(ctx->fs_root, O_DIRECTORY);
-    if (data->mountfd < 0) {
-        ret = data->mountfd;
-        goto err_out;
-    }
-    ret = statfs(ctx->fs_root, &stbuf);
-    if (!ret) {
-        switch (stbuf.f_type) {
-        case EXT2_SUPER_MAGIC:
-        case BTRFS_SUPER_MAGIC:
-        case REISERFS_SUPER_MAGIC:
-        case XFS_SUPER_MAGIC:
-            ctx->exops.get_st_gen = handle_ioc_getversion;
-            break;
-        }
-    }
-    memset(&fh, 0, sizeof(struct file_handle));
-    ret = name_to_handle(data->mountfd, ".", &fh, &mnt_id, 0);
-    if (ret && errno == EOVERFLOW) {
-        data->handle_bytes = fh.handle_bytes;
-        ctx->private = data;
-        ret = 0;
-        goto out;
-    }
-    /* we got 0 byte handle ? */
-    ret = -1;
-    close(data->mountfd);
-err_out:
-    g_free(data);
-out:
-    return ret;
-}
-
-static int handle_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
-{
-    const char *sec_model = qemu_opt_get(opts, "security_model");
-    const char *path = qemu_opt_get(opts, "path");
-
-    if (sec_model) {
-        fprintf(stderr, "Invalid argument security_model specified with handle fsdriver\n");
-        return -1;
-    }
-
-    if (!path) {
-        fprintf(stderr, "fsdev: No path specified.\n");
-        return -1;
-    }
-    fse->path = g_strdup(path);
-    return 0;
-
-}
-
-FileOperations handle_ops = {
-    .parse_opts   = handle_parse_opts,
-    .init         = handle_init,
-    .lstat        = handle_lstat,
-    .readlink     = handle_readlink,
-    .close        = handle_close,
-    .closedir     = handle_closedir,
-    .open         = handle_open,
-    .opendir      = handle_opendir,
-    .rewinddir    = handle_rewinddir,
-    .telldir      = handle_telldir,
-    .readdir_r    = handle_readdir_r,
-    .seekdir      = handle_seekdir,
-    .preadv       = handle_preadv,
-    .pwritev      = handle_pwritev,
-    .chmod        = handle_chmod,
-    .mknod        = handle_mknod,
-    .mkdir        = handle_mkdir,
-    .fstat        = handle_fstat,
-    .open2        = handle_open2,
-    .symlink      = handle_symlink,
-    .link         = handle_link,
-    .truncate     = handle_truncate,
-    .rename       = handle_rename,
-    .chown        = handle_chown,
-    .utimensat    = handle_utimensat,
-    .remove       = handle_remove,
-    .fsync        = handle_fsync,
-    .statfs       = handle_statfs,
-    .lgetxattr    = handle_lgetxattr,
-    .llistxattr   = handle_llistxattr,
-    .lsetxattr    = handle_lsetxattr,
-    .lremovexattr = handle_lremovexattr,
-    .name_to_path = handle_name_to_path,
-    .renameat     = handle_renameat,
-    .unlinkat     = handle_unlinkat,
-};
diff --git a/hw/9pfs/virtio-9p-local.c b/hw/9pfs/virtio-9p-local.c
deleted file mode 100644
index a183eee..0000000
--- a/hw/9pfs/virtio-9p-local.c
+++ /dev/null
@@ -1,1295 +0,0 @@
-/*
- * Virtio 9p Posix callback
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "virtio-9p-xattr.h"
-#include "fsdev/qemu-fsdev.h"   /* local_ops */
-#include <arpa/inet.h>
-#include <pwd.h>
-#include <grp.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include "qemu/xattr.h"
-#include <libgen.h>
-#include <linux/fs.h>
-#ifdef CONFIG_LINUX_MAGIC_H
-#include <linux/magic.h>
-#endif
-#include <sys/ioctl.h>
-
-#ifndef XFS_SUPER_MAGIC
-#define XFS_SUPER_MAGIC  0x58465342
-#endif
-#ifndef EXT2_SUPER_MAGIC
-#define EXT2_SUPER_MAGIC 0xEF53
-#endif
-#ifndef REISERFS_SUPER_MAGIC
-#define REISERFS_SUPER_MAGIC 0x52654973
-#endif
-#ifndef BTRFS_SUPER_MAGIC
-#define BTRFS_SUPER_MAGIC 0x9123683E
-#endif
-
-#define VIRTFS_META_DIR ".virtfs_metadata"
-
-static char *local_mapped_attr_path(FsContext *ctx, const char *path)
-{
-    char *dir_name;
-    char *tmp_path = g_strdup(path);
-    char *base_name = basename(tmp_path);
-    char *buffer;
-
-    /* NULL terminate the directory */
-    dir_name = tmp_path;
-    *(base_name - 1) = '\0';
-
-    buffer = g_strdup_printf("%s/%s/%s/%s",
-             ctx->fs_root, dir_name, VIRTFS_META_DIR, base_name);
-    g_free(tmp_path);
-    return buffer;
-}
-
-static FILE *local_fopen(const char *path, const char *mode)
-{
-    int fd, o_mode = 0;
-    FILE *fp;
-    int flags = O_NOFOLLOW;
-    /*
-     * only supports two modes
-     */
-    if (mode[0] == 'r') {
-        flags |= O_RDONLY;
-    } else if (mode[0] == 'w') {
-        flags |= O_WRONLY | O_TRUNC | O_CREAT;
-        o_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
-    } else {
-        return NULL;
-    }
-    fd = open(path, flags, o_mode);
-    if (fd == -1) {
-        return NULL;
-    }
-    fp = fdopen(fd, mode);
-    if (!fp) {
-        close(fd);
-    }
-    return fp;
-}
-
-#define ATTR_MAX 100
-static void local_mapped_file_attr(FsContext *ctx, const char *path,
-                                   struct stat *stbuf)
-{
-    FILE *fp;
-    char buf[ATTR_MAX];
-    char *attr_path;
-
-    attr_path = local_mapped_attr_path(ctx, path);
-    fp = local_fopen(attr_path, "r");
-    g_free(attr_path);
-    if (!fp) {
-        return;
-    }
-    memset(buf, 0, ATTR_MAX);
-    while (fgets(buf, ATTR_MAX, fp)) {
-        if (!strncmp(buf, "virtfs.uid", 10)) {
-            stbuf->st_uid = atoi(buf+11);
-        } else if (!strncmp(buf, "virtfs.gid", 10)) {
-            stbuf->st_gid = atoi(buf+11);
-        } else if (!strncmp(buf, "virtfs.mode", 11)) {
-            stbuf->st_mode = atoi(buf+12);
-        } else if (!strncmp(buf, "virtfs.rdev", 11)) {
-            stbuf->st_rdev = atoi(buf+12);
-        }
-        memset(buf, 0, ATTR_MAX);
-    }
-    fclose(fp);
-}
-
-static int local_lstat(FsContext *fs_ctx, V9fsPath *fs_path, struct stat *stbuf)
-{
-    int err;
-    char *buffer;
-    char *path = fs_path->data;
-
-    buffer = rpath(fs_ctx, path);
-    err =  lstat(buffer, stbuf);
-    if (err) {
-        goto err_out;
-    }
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        /* Actual credentials are part of extended attrs */
-        uid_t tmp_uid;
-        gid_t tmp_gid;
-        mode_t tmp_mode;
-        dev_t tmp_dev;
-        if (getxattr(buffer, "user.virtfs.uid", &tmp_uid, sizeof(uid_t)) > 0) {
-            stbuf->st_uid = le32_to_cpu(tmp_uid);
-        }
-        if (getxattr(buffer, "user.virtfs.gid", &tmp_gid, sizeof(gid_t)) > 0) {
-            stbuf->st_gid = le32_to_cpu(tmp_gid);
-        }
-        if (getxattr(buffer, "user.virtfs.mode",
-                    &tmp_mode, sizeof(mode_t)) > 0) {
-            stbuf->st_mode = le32_to_cpu(tmp_mode);
-        }
-        if (getxattr(buffer, "user.virtfs.rdev", &tmp_dev, sizeof(dev_t)) > 0) {
-            stbuf->st_rdev = le64_to_cpu(tmp_dev);
-        }
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        local_mapped_file_attr(fs_ctx, path, stbuf);
-    }
-
-err_out:
-    g_free(buffer);
-    return err;
-}
-
-static int local_create_mapped_attr_dir(FsContext *ctx, const char *path)
-{
-    int err;
-    char *attr_dir;
-    char *tmp_path = g_strdup(path);
-
-    attr_dir = g_strdup_printf("%s/%s/%s",
-             ctx->fs_root, dirname(tmp_path), VIRTFS_META_DIR);
-
-    err = mkdir(attr_dir, 0700);
-    if (err < 0 && errno == EEXIST) {
-        err = 0;
-    }
-    g_free(attr_dir);
-    g_free(tmp_path);
-    return err;
-}
-
-static int local_set_mapped_file_attr(FsContext *ctx,
-                                      const char *path, FsCred *credp)
-{
-    FILE *fp;
-    int ret = 0;
-    char buf[ATTR_MAX];
-    char *attr_path;
-    int uid = -1, gid = -1, mode = -1, rdev = -1;
-
-    attr_path = local_mapped_attr_path(ctx, path);
-    fp = local_fopen(attr_path, "r");
-    if (!fp) {
-        goto create_map_file;
-    }
-    memset(buf, 0, ATTR_MAX);
-    while (fgets(buf, ATTR_MAX, fp)) {
-        if (!strncmp(buf, "virtfs.uid", 10)) {
-            uid = atoi(buf+11);
-        } else if (!strncmp(buf, "virtfs.gid", 10)) {
-            gid = atoi(buf+11);
-        } else if (!strncmp(buf, "virtfs.mode", 11)) {
-            mode = atoi(buf+12);
-        } else if (!strncmp(buf, "virtfs.rdev", 11)) {
-            rdev = atoi(buf+12);
-        }
-        memset(buf, 0, ATTR_MAX);
-    }
-    fclose(fp);
-    goto update_map_file;
-
-create_map_file:
-    ret = local_create_mapped_attr_dir(ctx, path);
-    if (ret < 0) {
-        goto err_out;
-    }
-
-update_map_file:
-    fp = local_fopen(attr_path, "w");
-    if (!fp) {
-        ret = -1;
-        goto err_out;
-    }
-
-    if (credp->fc_uid != -1) {
-        uid = credp->fc_uid;
-    }
-    if (credp->fc_gid != -1) {
-        gid = credp->fc_gid;
-    }
-    if (credp->fc_mode != -1) {
-        mode = credp->fc_mode;
-    }
-    if (credp->fc_rdev != -1) {
-        rdev = credp->fc_rdev;
-    }
-
-
-    if (uid != -1) {
-        fprintf(fp, "virtfs.uid=%d\n", uid);
-    }
-    if (gid != -1) {
-        fprintf(fp, "virtfs.gid=%d\n", gid);
-    }
-    if (mode != -1) {
-        fprintf(fp, "virtfs.mode=%d\n", mode);
-    }
-    if (rdev != -1) {
-        fprintf(fp, "virtfs.rdev=%d\n", rdev);
-    }
-    fclose(fp);
-
-err_out:
-    g_free(attr_path);
-    return ret;
-}
-
-static int local_set_xattr(const char *path, FsCred *credp)
-{
-    int err;
-
-    if (credp->fc_uid != -1) {
-        uint32_t tmp_uid = cpu_to_le32(credp->fc_uid);
-        err = setxattr(path, "user.virtfs.uid", &tmp_uid, sizeof(uid_t), 0);
-        if (err) {
-            return err;
-        }
-    }
-    if (credp->fc_gid != -1) {
-        uint32_t tmp_gid = cpu_to_le32(credp->fc_gid);
-        err = setxattr(path, "user.virtfs.gid", &tmp_gid, sizeof(gid_t), 0);
-        if (err) {
-            return err;
-        }
-    }
-    if (credp->fc_mode != -1) {
-        uint32_t tmp_mode = cpu_to_le32(credp->fc_mode);
-        err = setxattr(path, "user.virtfs.mode", &tmp_mode, sizeof(mode_t), 0);
-        if (err) {
-            return err;
-        }
-    }
-    if (credp->fc_rdev != -1) {
-        uint64_t tmp_rdev = cpu_to_le64(credp->fc_rdev);
-        err = setxattr(path, "user.virtfs.rdev", &tmp_rdev, sizeof(dev_t), 0);
-        if (err) {
-            return err;
-        }
-    }
-    return 0;
-}
-
-static int local_post_create_passthrough(FsContext *fs_ctx, const char *path,
-                                         FsCred *credp)
-{
-    char *buffer;
-
-    buffer = rpath(fs_ctx, path);
-    if (lchown(buffer, credp->fc_uid, credp->fc_gid) < 0) {
-        /*
-         * If we fail to change ownership and if we are
-         * using security model none. Ignore the error
-         */
-        if ((fs_ctx->export_flags & V9FS_SEC_MASK) != V9FS_SM_NONE) {
-            goto err;
-        }
-    }
-
-    if (chmod(buffer, credp->fc_mode & 07777) < 0) {
-        goto err;
-    }
-
-    g_free(buffer);
-    return 0;
-err:
-    g_free(buffer);
-    return -1;
-}
-
-static ssize_t local_readlink(FsContext *fs_ctx, V9fsPath *fs_path,
-                              char *buf, size_t bufsz)
-{
-    ssize_t tsize = -1;
-    char *buffer;
-    char *path = fs_path->data;
-
-    if ((fs_ctx->export_flags & V9FS_SM_MAPPED) ||
-        (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE)) {
-        int fd;
-        buffer = rpath(fs_ctx, path);
-        fd = open(buffer, O_RDONLY | O_NOFOLLOW);
-        g_free(buffer);
-        if (fd == -1) {
-            return -1;
-        }
-        do {
-            tsize = read(fd, (void *)buf, bufsz);
-        } while (tsize == -1 && errno == EINTR);
-        close(fd);
-        return tsize;
-    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-               (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, path);
-        tsize = readlink(buffer, buf, bufsz);
-        g_free(buffer);
-    }
-    return tsize;
-}
-
-static int local_close(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return close(fs->fd);
-}
-
-static int local_closedir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return closedir(fs->dir);
-}
-
-static int local_open(FsContext *ctx, V9fsPath *fs_path,
-                      int flags, V9fsFidOpenState *fs)
-{
-    char *buffer;
-    char *path = fs_path->data;
-
-    buffer = rpath(ctx, path);
-    fs->fd = open(buffer, flags | O_NOFOLLOW);
-    g_free(buffer);
-    return fs->fd;
-}
-
-static int local_opendir(FsContext *ctx,
-                         V9fsPath *fs_path, V9fsFidOpenState *fs)
-{
-    char *buffer;
-    char *path = fs_path->data;
-
-    buffer = rpath(ctx, path);
-    fs->dir = opendir(buffer);
-    g_free(buffer);
-    if (!fs->dir) {
-        return -1;
-    }
-    return 0;
-}
-
-static void local_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return rewinddir(fs->dir);
-}
-
-static off_t local_telldir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return telldir(fs->dir);
-}
-
-static int local_readdir_r(FsContext *ctx, V9fsFidOpenState *fs,
-                           struct dirent *entry,
-                           struct dirent **result)
-{
-    int ret;
-
-again:
-    ret = readdir_r(fs->dir, entry, result);
-    if (ctx->export_flags & V9FS_SM_MAPPED) {
-        entry->d_type = DT_UNKNOWN;
-    } else if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        if (!ret && *result != NULL &&
-            !strcmp(entry->d_name, VIRTFS_META_DIR)) {
-            /* skp the meta data directory */
-            goto again;
-        }
-        entry->d_type = DT_UNKNOWN;
-    }
-    return ret;
-}
-
-static void local_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
-{
-    return seekdir(fs->dir, off);
-}
-
-static ssize_t local_preadv(FsContext *ctx, V9fsFidOpenState *fs,
-                            const struct iovec *iov,
-                            int iovcnt, off_t offset)
-{
-#ifdef CONFIG_PREADV
-    return preadv(fs->fd, iov, iovcnt, offset);
-#else
-    int err = lseek(fs->fd, offset, SEEK_SET);
-    if (err == -1) {
-        return err;
-    } else {
-        return readv(fs->fd, iov, iovcnt);
-    }
-#endif
-}
-
-static ssize_t local_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
-                             const struct iovec *iov,
-                             int iovcnt, off_t offset)
-{
-    ssize_t ret
-;
-#ifdef CONFIG_PREADV
-    ret = pwritev(fs->fd, iov, iovcnt, offset);
-#else
-    int err = lseek(fs->fd, offset, SEEK_SET);
-    if (err == -1) {
-        return err;
-    } else {
-        ret = writev(fs->fd, iov, iovcnt);
-    }
-#endif
-#ifdef CONFIG_SYNC_FILE_RANGE
-    if (ret > 0 && ctx->export_flags & V9FS_IMMEDIATE_WRITEOUT) {
-        /*
-         * Initiate a writeback. This is not a data integrity sync.
-         * We want to ensure that we don't leave dirty pages in the cache
-         * after write when writeout=immediate is sepcified.
-         */
-        sync_file_range(fs->fd, offset, ret,
-                        SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
-    }
-#endif
-    return ret;
-}
-
-static int local_chmod(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
-{
-    char *buffer;
-    int ret = -1;
-    char *path = fs_path->data;
-
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        buffer = rpath(fs_ctx, path);
-        ret = local_set_xattr(buffer, credp);
-        g_free(buffer);
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        return local_set_mapped_file_attr(fs_ctx, path, credp);
-    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-               (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, path);
-        ret = chmod(buffer, credp->fc_mode);
-        g_free(buffer);
-    }
-    return ret;
-}
-
-static int local_mknod(FsContext *fs_ctx, V9fsPath *dir_path,
-                       const char *name, FsCred *credp)
-{
-    char *path;
-    int err = -1;
-    int serrno = 0;
-    V9fsString fullname;
-    char *buffer;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-    path = fullname.data;
-
-    /* Determine the security model */
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        buffer = rpath(fs_ctx, path);
-        err = mknod(buffer, SM_LOCAL_MODE_BITS|S_IFREG, 0);
-        if (err == -1) {
-            g_free(buffer);
-            goto out;
-        }
-        err = local_set_xattr(buffer, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-
-        buffer = rpath(fs_ctx, path);
-        err = mknod(buffer, SM_LOCAL_MODE_BITS|S_IFREG, 0);
-        if (err == -1) {
-            g_free(buffer);
-            goto out;
-        }
-        err = local_set_mapped_file_attr(fs_ctx, path, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-               (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, path);
-        err = mknod(buffer, credp->fc_mode, credp->fc_rdev);
-        if (err == -1) {
-            g_free(buffer);
-            goto out;
-        }
-        err = local_post_create_passthrough(fs_ctx, path, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    }
-    goto out;
-
-err_end:
-    remove(buffer);
-    errno = serrno;
-    g_free(buffer);
-out:
-    v9fs_string_free(&fullname);
-    return err;
-}
-
-static int local_mkdir(FsContext *fs_ctx, V9fsPath *dir_path,
-                       const char *name, FsCred *credp)
-{
-    char *path;
-    int err = -1;
-    int serrno = 0;
-    V9fsString fullname;
-    char *buffer;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-    path = fullname.data;
-
-    /* Determine the security model */
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        buffer = rpath(fs_ctx, path);
-        err = mkdir(buffer, SM_LOCAL_DIR_MODE_BITS);
-        if (err == -1) {
-            g_free(buffer);
-            goto out;
-        }
-        credp->fc_mode = credp->fc_mode|S_IFDIR;
-        err = local_set_xattr(buffer, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        buffer = rpath(fs_ctx, path);
-        err = mkdir(buffer, SM_LOCAL_DIR_MODE_BITS);
-        if (err == -1) {
-            g_free(buffer);
-            goto out;
-        }
-        credp->fc_mode = credp->fc_mode|S_IFDIR;
-        err = local_set_mapped_file_attr(fs_ctx, path, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-               (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, path);
-        err = mkdir(buffer, credp->fc_mode);
-        if (err == -1) {
-            g_free(buffer);
-            goto out;
-        }
-        err = local_post_create_passthrough(fs_ctx, path, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    }
-    goto out;
-
-err_end:
-    remove(buffer);
-    errno = serrno;
-    g_free(buffer);
-out:
-    v9fs_string_free(&fullname);
-    return err;
-}
-
-static int local_fstat(FsContext *fs_ctx, int fid_type,
-                       V9fsFidOpenState *fs, struct stat *stbuf)
-{
-    int err, fd;
-
-    if (fid_type == P9_FID_DIR) {
-        fd = dirfd(fs->dir);
-    } else {
-        fd = fs->fd;
-    }
-
-    err = fstat(fd, stbuf);
-    if (err) {
-        return err;
-    }
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        /* Actual credentials are part of extended attrs */
-        uid_t tmp_uid;
-        gid_t tmp_gid;
-        mode_t tmp_mode;
-        dev_t tmp_dev;
-
-        if (fgetxattr(fd, "user.virtfs.uid", &tmp_uid, sizeof(uid_t)) > 0) {
-            stbuf->st_uid = le32_to_cpu(tmp_uid);
-        }
-        if (fgetxattr(fd, "user.virtfs.gid", &tmp_gid, sizeof(gid_t)) > 0) {
-            stbuf->st_gid = le32_to_cpu(tmp_gid);
-        }
-        if (fgetxattr(fd, "user.virtfs.mode", &tmp_mode, sizeof(mode_t)) > 0) {
-            stbuf->st_mode = le32_to_cpu(tmp_mode);
-        }
-        if (fgetxattr(fd, "user.virtfs.rdev", &tmp_dev, sizeof(dev_t)) > 0) {
-            stbuf->st_rdev = le64_to_cpu(tmp_dev);
-        }
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        errno = EOPNOTSUPP;
-        return -1;
-    }
-    return err;
-}
-
-static int local_open2(FsContext *fs_ctx, V9fsPath *dir_path, const char *name,
-                       int flags, FsCred *credp, V9fsFidOpenState *fs)
-{
-    char *path;
-    int fd = -1;
-    int err = -1;
-    int serrno = 0;
-    V9fsString fullname;
-    char *buffer;
-
-    /*
-     * Mark all the open to not follow symlinks
-     */
-    flags |= O_NOFOLLOW;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-    path = fullname.data;
-
-    /* Determine the security model */
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        buffer = rpath(fs_ctx, path);
-        fd = open(buffer, flags, SM_LOCAL_MODE_BITS);
-        if (fd == -1) {
-            g_free(buffer);
-            err = fd;
-            goto out;
-        }
-        credp->fc_mode = credp->fc_mode|S_IFREG;
-        /* Set cleint credentials in xattr */
-        err = local_set_xattr(buffer, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        buffer = rpath(fs_ctx, path);
-        fd = open(buffer, flags, SM_LOCAL_MODE_BITS);
-        if (fd == -1) {
-            g_free(buffer);
-            err = fd;
-            goto out;
-        }
-        credp->fc_mode = credp->fc_mode|S_IFREG;
-        /* Set client credentials in .virtfs_metadata directory files */
-        err = local_set_mapped_file_attr(fs_ctx, path, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-               (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, path);
-        fd = open(buffer, flags, credp->fc_mode);
-        if (fd == -1) {
-            g_free(buffer);
-            err = fd;
-            goto out;
-        }
-        err = local_post_create_passthrough(fs_ctx, path, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    }
-    err = fd;
-    fs->fd = fd;
-    goto out;
-
-err_end:
-    close(fd);
-    remove(buffer);
-    errno = serrno;
-    g_free(buffer);
-out:
-    v9fs_string_free(&fullname);
-    return err;
-}
-
-
-static int local_symlink(FsContext *fs_ctx, const char *oldpath,
-                         V9fsPath *dir_path, const char *name, FsCred *credp)
-{
-    int err = -1;
-    int serrno = 0;
-    char *newpath;
-    V9fsString fullname;
-    char *buffer;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-    newpath = fullname.data;
-
-    /* Determine the security model */
-    if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        int fd;
-        ssize_t oldpath_size, write_size;
-        buffer = rpath(fs_ctx, newpath);
-        fd = open(buffer, O_CREAT|O_EXCL|O_RDWR|O_NOFOLLOW, SM_LOCAL_MODE_BITS);
-        if (fd == -1) {
-            g_free(buffer);
-            err = fd;
-            goto out;
-        }
-        /* Write the oldpath (target) to the file. */
-        oldpath_size = strlen(oldpath);
-        do {
-            write_size = write(fd, (void *)oldpath, oldpath_size);
-        } while (write_size == -1 && errno == EINTR);
-
-        if (write_size != oldpath_size) {
-            serrno = errno;
-            close(fd);
-            err = -1;
-            goto err_end;
-        }
-        close(fd);
-        /* Set cleint credentials in symlink's xattr */
-        credp->fc_mode = credp->fc_mode|S_IFLNK;
-        err = local_set_xattr(buffer, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        int fd;
-        ssize_t oldpath_size, write_size;
-        buffer = rpath(fs_ctx, newpath);
-        fd = open(buffer, O_CREAT|O_EXCL|O_RDWR|O_NOFOLLOW, SM_LOCAL_MODE_BITS);
-        if (fd == -1) {
-            g_free(buffer);
-            err = fd;
-            goto out;
-        }
-        /* Write the oldpath (target) to the file. */
-        oldpath_size = strlen(oldpath);
-        do {
-            write_size = write(fd, (void *)oldpath, oldpath_size);
-        } while (write_size == -1 && errno == EINTR);
-
-        if (write_size != oldpath_size) {
-            serrno = errno;
-            close(fd);
-            err = -1;
-            goto err_end;
-        }
-        close(fd);
-        /* Set cleint credentials in symlink's xattr */
-        credp->fc_mode = credp->fc_mode|S_IFLNK;
-        err = local_set_mapped_file_attr(fs_ctx, newpath, credp);
-        if (err == -1) {
-            serrno = errno;
-            goto err_end;
-        }
-    } else if ((fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-               (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, newpath);
-        err = symlink(oldpath, buffer);
-        if (err) {
-            g_free(buffer);
-            goto out;
-        }
-        err = lchown(buffer, credp->fc_uid, credp->fc_gid);
-        if (err == -1) {
-            /*
-             * If we fail to change ownership and if we are
-             * using security model none. Ignore the error
-             */
-            if ((fs_ctx->export_flags & V9FS_SEC_MASK) != V9FS_SM_NONE) {
-                serrno = errno;
-                goto err_end;
-            } else
-                err = 0;
-        }
-    }
-    goto out;
-
-err_end:
-    remove(buffer);
-    errno = serrno;
-    g_free(buffer);
-out:
-    v9fs_string_free(&fullname);
-    return err;
-}
-
-static int local_link(FsContext *ctx, V9fsPath *oldpath,
-                      V9fsPath *dirpath, const char *name)
-{
-    int ret;
-    V9fsString newpath;
-    char *buffer, *buffer1;
-
-    v9fs_string_init(&newpath);
-    v9fs_string_sprintf(&newpath, "%s/%s", dirpath->data, name);
-
-    buffer = rpath(ctx, oldpath->data);
-    buffer1 = rpath(ctx, newpath.data);
-    ret = link(buffer, buffer1);
-    g_free(buffer);
-    g_free(buffer1);
-
-    /* now link the virtfs_metadata files */
-    if (!ret && (ctx->export_flags & V9FS_SM_MAPPED_FILE)) {
-        /* Link the .virtfs_metadata files. Create the metada directory */
-        ret = local_create_mapped_attr_dir(ctx, newpath.data);
-        if (ret < 0) {
-            goto err_out;
-        }
-        buffer = local_mapped_attr_path(ctx, oldpath->data);
-        buffer1 = local_mapped_attr_path(ctx, newpath.data);
-        ret = link(buffer, buffer1);
-        g_free(buffer);
-        g_free(buffer1);
-        if (ret < 0 && errno != ENOENT) {
-            goto err_out;
-        }
-    }
-err_out:
-    v9fs_string_free(&newpath);
-    return ret;
-}
-
-static int local_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size)
-{
-    char *buffer;
-    int ret;
-    char *path = fs_path->data;
-
-    buffer = rpath(ctx, path);
-    ret = truncate(buffer, size);
-    g_free(buffer);
-    return ret;
-}
-
-static int local_rename(FsContext *ctx, const char *oldpath,
-                        const char *newpath)
-{
-    int err;
-    char *buffer, *buffer1;
-
-    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        err = local_create_mapped_attr_dir(ctx, newpath);
-        if (err < 0) {
-            return err;
-        }
-        /* rename the .virtfs_metadata files */
-        buffer = local_mapped_attr_path(ctx, oldpath);
-        buffer1 = local_mapped_attr_path(ctx, newpath);
-        err = rename(buffer, buffer1);
-        g_free(buffer);
-        g_free(buffer1);
-        if (err < 0 && errno != ENOENT) {
-            return err;
-        }
-    }
-
-    buffer = rpath(ctx, oldpath);
-    buffer1 = rpath(ctx, newpath);
-    err = rename(buffer, buffer1);
-    g_free(buffer);
-    g_free(buffer1);
-    return err;
-}
-
-static int local_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
-{
-    char *buffer;
-    int ret = -1;
-    char *path = fs_path->data;
-
-    if ((credp->fc_uid == -1 && credp->fc_gid == -1) ||
-        (fs_ctx->export_flags & V9FS_SM_PASSTHROUGH) ||
-        (fs_ctx->export_flags & V9FS_SM_NONE)) {
-        buffer = rpath(fs_ctx, path);
-        ret = lchown(buffer, credp->fc_uid, credp->fc_gid);
-        g_free(buffer);
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED) {
-        buffer = rpath(fs_ctx, path);
-        ret = local_set_xattr(buffer, credp);
-        g_free(buffer);
-    } else if (fs_ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        return local_set_mapped_file_attr(fs_ctx, path, credp);
-    }
-    return ret;
-}
-
-static int local_utimensat(FsContext *s, V9fsPath *fs_path,
-                           const struct timespec *buf)
-{
-    char *buffer;
-    int ret;
-    char *path = fs_path->data;
-
-    buffer = rpath(s, path);
-    ret = qemu_utimens(buffer, buf);
-    g_free(buffer);
-    return ret;
-}
-
-static int local_remove(FsContext *ctx, const char *path)
-{
-    int err;
-    struct stat stbuf;
-    char *buffer;
-
-    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        buffer = rpath(ctx, path);
-        err =  lstat(buffer, &stbuf);
-        g_free(buffer);
-        if (err) {
-            goto err_out;
-        }
-        /*
-         * If directory remove .virtfs_metadata contained in the
-         * directory
-         */
-        if (S_ISDIR(stbuf.st_mode)) {
-            buffer = g_strdup_printf("%s/%s/%s", ctx->fs_root,
-                                     path, VIRTFS_META_DIR);
-            err = remove(buffer);
-            g_free(buffer);
-            if (err < 0 && errno != ENOENT) {
-                /*
-                 * We didn't had the .virtfs_metadata file. May be file created
-                 * in non-mapped mode ?. Ignore ENOENT.
-                 */
-                goto err_out;
-            }
-        }
-        /*
-         * Now remove the name from parent directory
-         * .virtfs_metadata directory
-         */
-        buffer = local_mapped_attr_path(ctx, path);
-        err = remove(buffer);
-        g_free(buffer);
-        if (err < 0 && errno != ENOENT) {
-            /*
-             * We didn't had the .virtfs_metadata file. May be file created
-             * in non-mapped mode ?. Ignore ENOENT.
-             */
-            goto err_out;
-        }
-    }
-
-    buffer = rpath(ctx, path);
-    err = remove(buffer);
-    g_free(buffer);
-err_out:
-    return err;
-}
-
-static int local_fsync(FsContext *ctx, int fid_type,
-                       V9fsFidOpenState *fs, int datasync)
-{
-    int fd;
-
-    if (fid_type == P9_FID_DIR) {
-        fd = dirfd(fs->dir);
-    } else {
-        fd = fs->fd;
-    }
-
-    if (datasync) {
-        return qemu_fdatasync(fd);
-    } else {
-        return fsync(fd);
-    }
-}
-
-static int local_statfs(FsContext *s, V9fsPath *fs_path, struct statfs *stbuf)
-{
-    char *buffer;
-    int ret;
-    char *path = fs_path->data;
-
-    buffer = rpath(s, path);
-    ret = statfs(buffer, stbuf);
-    g_free(buffer);
-    return ret;
-}
-
-static ssize_t local_lgetxattr(FsContext *ctx, V9fsPath *fs_path,
-                               const char *name, void *value, size_t size)
-{
-    char *path = fs_path->data;
-
-    return v9fs_get_xattr(ctx, path, name, value, size);
-}
-
-static ssize_t local_llistxattr(FsContext *ctx, V9fsPath *fs_path,
-                                void *value, size_t size)
-{
-    char *path = fs_path->data;
-
-    return v9fs_list_xattr(ctx, path, value, size);
-}
-
-static int local_lsetxattr(FsContext *ctx, V9fsPath *fs_path, const char *name,
-                           void *value, size_t size, int flags)
-{
-    char *path = fs_path->data;
-
-    return v9fs_set_xattr(ctx, path, name, value, size, flags);
-}
-
-static int local_lremovexattr(FsContext *ctx, V9fsPath *fs_path,
-                              const char *name)
-{
-    char *path = fs_path->data;
-
-    return v9fs_remove_xattr(ctx, path, name);
-}
-
-static int local_name_to_path(FsContext *ctx, V9fsPath *dir_path,
-                              const char *name, V9fsPath *target)
-{
-    if (dir_path) {
-        v9fs_string_sprintf((V9fsString *)target, "%s/%s",
-                            dir_path->data, name);
-    } else {
-        v9fs_string_sprintf((V9fsString *)target, "%s", name);
-    }
-    /* Bump the size for including terminating NULL */
-    target->size++;
-    return 0;
-}
-
-static int local_renameat(FsContext *ctx, V9fsPath *olddir,
-                          const char *old_name, V9fsPath *newdir,
-                          const char *new_name)
-{
-    int ret;
-    V9fsString old_full_name, new_full_name;
-
-    v9fs_string_init(&old_full_name);
-    v9fs_string_init(&new_full_name);
-
-    v9fs_string_sprintf(&old_full_name, "%s/%s", olddir->data, old_name);
-    v9fs_string_sprintf(&new_full_name, "%s/%s", newdir->data, new_name);
-
-    ret = local_rename(ctx, old_full_name.data, new_full_name.data);
-    v9fs_string_free(&old_full_name);
-    v9fs_string_free(&new_full_name);
-    return ret;
-}
-
-static int local_unlinkat(FsContext *ctx, V9fsPath *dir,
-                          const char *name, int flags)
-{
-    int ret;
-    V9fsString fullname;
-    char *buffer;
-
-    v9fs_string_init(&fullname);
-
-    v9fs_string_sprintf(&fullname, "%s/%s", dir->data, name);
-    if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        if (flags == AT_REMOVEDIR) {
-            /*
-             * If directory remove .virtfs_metadata contained in the
-             * directory
-             */
-            buffer = g_strdup_printf("%s/%s/%s", ctx->fs_root,
-                                     fullname.data, VIRTFS_META_DIR);
-            ret = remove(buffer);
-            g_free(buffer);
-            if (ret < 0 && errno != ENOENT) {
-                /*
-                 * We didn't had the .virtfs_metadata file. May be file created
-                 * in non-mapped mode ?. Ignore ENOENT.
-                 */
-                goto err_out;
-            }
-        }
-        /*
-         * Now remove the name from parent directory
-         * .virtfs_metadata directory.
-         */
-        buffer = local_mapped_attr_path(ctx, fullname.data);
-        ret = remove(buffer);
-        g_free(buffer);
-        if (ret < 0 && errno != ENOENT) {
-            /*
-             * We didn't had the .virtfs_metadata file. May be file created
-             * in non-mapped mode ?. Ignore ENOENT.
-             */
-            goto err_out;
-        }
-    }
-    /* Remove the name finally */
-    buffer = rpath(ctx, fullname.data);
-    ret = remove(buffer);
-    g_free(buffer);
-
-err_out:
-    v9fs_string_free(&fullname);
-    return ret;
-}
-
-static int local_ioc_getversion(FsContext *ctx, V9fsPath *path,
-                                mode_t st_mode, uint64_t *st_gen)
-{
-#ifdef FS_IOC_GETVERSION
-    int err;
-    V9fsFidOpenState fid_open;
-
-    /*
-     * Do not try to open special files like device nodes, fifos etc
-     * We can get fd for regular files and directories only
-     */
-    if (!S_ISREG(st_mode) && !S_ISDIR(st_mode)) {
-        errno = ENOTTY;
-        return -1;
-    }
-    err = local_open(ctx, path, O_RDONLY, &fid_open);
-    if (err < 0) {
-        return err;
-    }
-    err = ioctl(fid_open.fd, FS_IOC_GETVERSION, st_gen);
-    local_close(ctx, &fid_open);
-    return err;
-#else
-    errno = ENOTTY;
-    return -1;
-#endif
-}
-
-static int local_init(FsContext *ctx)
-{
-    int err = 0;
-    struct statfs stbuf;
-
-    if (ctx->export_flags & V9FS_SM_PASSTHROUGH) {
-        ctx->xops = passthrough_xattr_ops;
-    } else if (ctx->export_flags & V9FS_SM_MAPPED) {
-        ctx->xops = mapped_xattr_ops;
-    } else if (ctx->export_flags & V9FS_SM_NONE) {
-        ctx->xops = none_xattr_ops;
-    } else if (ctx->export_flags & V9FS_SM_MAPPED_FILE) {
-        /*
-         * xattr operation for mapped-file and passthrough
-         * remain same.
-         */
-        ctx->xops = passthrough_xattr_ops;
-    }
-    ctx->export_flags |= V9FS_PATHNAME_FSCONTEXT;
-#ifdef FS_IOC_GETVERSION
-    /*
-     * use ioc_getversion only if the iocl is definied
-     */
-    err = statfs(ctx->fs_root, &stbuf);
-    if (!err) {
-        switch (stbuf.f_type) {
-        case EXT2_SUPER_MAGIC:
-        case BTRFS_SUPER_MAGIC:
-        case REISERFS_SUPER_MAGIC:
-        case XFS_SUPER_MAGIC:
-            ctx->exops.get_st_gen = local_ioc_getversion;
-            break;
-        }
-    }
-#endif
-    return err;
-}
-
-static int local_parse_opts(QemuOpts *opts, struct FsDriverEntry *fse)
-{
-    const char *sec_model = qemu_opt_get(opts, "security_model");
-    const char *path = qemu_opt_get(opts, "path");
-
-    if (!sec_model) {
-        fprintf(stderr, "security model not specified, "
-                "local fs needs security model\nvalid options are:"
-                "\tsecurity_model=[passthrough|mapped|none]\n");
-        return -1;
-    }
-
-    if (!strcmp(sec_model, "passthrough")) {
-        fse->export_flags |= V9FS_SM_PASSTHROUGH;
-    } else if (!strcmp(sec_model, "mapped") ||
-               !strcmp(sec_model, "mapped-xattr")) {
-        fse->export_flags |= V9FS_SM_MAPPED;
-    } else if (!strcmp(sec_model, "none")) {
-        fse->export_flags |= V9FS_SM_NONE;
-    } else if (!strcmp(sec_model, "mapped-file")) {
-        fse->export_flags |= V9FS_SM_MAPPED_FILE;
-    } else {
-        fprintf(stderr, "Invalid security model %s specified, valid options are"
-                "\n\t [passthrough|mapped-xattr|mapped-file|none]\n",
-                sec_model);
-        return -1;
-    }
-
-    if (!path) {
-        fprintf(stderr, "fsdev: No path specified.\n");
-        return -1;
-    }
-    fse->path = g_strdup(path);
-
-    return 0;
-}
-
-FileOperations local_ops = {
-    .parse_opts = local_parse_opts,
-    .init  = local_init,
-    .lstat = local_lstat,
-    .readlink = local_readlink,
-    .close = local_close,
-    .closedir = local_closedir,
-    .open = local_open,
-    .opendir = local_opendir,
-    .rewinddir = local_rewinddir,
-    .telldir = local_telldir,
-    .readdir_r = local_readdir_r,
-    .seekdir = local_seekdir,
-    .preadv = local_preadv,
-    .pwritev = local_pwritev,
-    .chmod = local_chmod,
-    .mknod = local_mknod,
-    .mkdir = local_mkdir,
-    .fstat = local_fstat,
-    .open2 = local_open2,
-    .symlink = local_symlink,
-    .link = local_link,
-    .truncate = local_truncate,
-    .rename = local_rename,
-    .chown = local_chown,
-    .utimensat = local_utimensat,
-    .remove = local_remove,
-    .fsync = local_fsync,
-    .statfs = local_statfs,
-    .lgetxattr = local_lgetxattr,
-    .llistxattr = local_llistxattr,
-    .lsetxattr = local_lsetxattr,
-    .lremovexattr = local_lremovexattr,
-    .name_to_path = local_name_to_path,
-    .renameat  = local_renameat,
-    .unlinkat = local_unlinkat,
-};
diff --git a/hw/9pfs/virtio-9p-posix-acl.c b/hw/9pfs/virtio-9p-posix-acl.c
deleted file mode 100644
index 803d9d9..0000000
--- a/hw/9pfs/virtio-9p-posix-acl.c
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Virtio 9p system.posix* xattr callback
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include <sys/types.h>
-#include "qemu/xattr.h"
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "fsdev/file-op-9p.h"
-#include "virtio-9p-xattr.h"
-
-#define MAP_ACL_ACCESS "user.virtfs.system.posix_acl_access"
-#define MAP_ACL_DEFAULT "user.virtfs.system.posix_acl_default"
-#define ACL_ACCESS "system.posix_acl_access"
-#define ACL_DEFAULT "system.posix_acl_default"
-
-static ssize_t mp_pacl_getxattr(FsContext *ctx, const char *path,
-                                const char *name, void *value, size_t size)
-{
-    char *buffer;
-    ssize_t ret;
-
-    buffer = rpath(ctx, path);
-    ret = lgetxattr(buffer, MAP_ACL_ACCESS, value, size);
-    g_free(buffer);
-    return ret;
-}
-
-static ssize_t mp_pacl_listxattr(FsContext *ctx, const char *path,
-                                 char *name, void *value, size_t osize)
-{
-    ssize_t len = sizeof(ACL_ACCESS);
-
-    if (!value) {
-        return len;
-    }
-
-    if (osize < len) {
-        errno = ERANGE;
-        return -1;
-    }
-
-    /* len includes the trailing NUL */
-    memcpy(value, ACL_ACCESS, len);
-    return 0;
-}
-
-static int mp_pacl_setxattr(FsContext *ctx, const char *path, const char *name,
-                            void *value, size_t size, int flags)
-{
-    char *buffer;
-    int ret;
-
-    buffer = rpath(ctx, path);
-    ret = lsetxattr(buffer, MAP_ACL_ACCESS, value, size, flags);
-    g_free(buffer);
-    return ret;
-}
-
-static int mp_pacl_removexattr(FsContext *ctx,
-                               const char *path, const char *name)
-{
-    int ret;
-    char *buffer;
-
-    buffer = rpath(ctx, path);
-    ret  = lremovexattr(buffer, MAP_ACL_ACCESS);
-    if (ret == -1 && errno == ENODATA) {
-        /*
-         * We don't get ENODATA error when trying to remove a
-         * posix acl that is not present. So don't throw the error
-         * even in case of mapped security model
-         */
-        errno = 0;
-        ret = 0;
-    }
-    g_free(buffer);
-    return ret;
-}
-
-static ssize_t mp_dacl_getxattr(FsContext *ctx, const char *path,
-                                const char *name, void *value, size_t size)
-{
-    char *buffer;
-    ssize_t ret;
-
-    buffer = rpath(ctx, path);
-    ret = lgetxattr(buffer, MAP_ACL_DEFAULT, value, size);
-    g_free(buffer);
-    return ret;
-}
-
-static ssize_t mp_dacl_listxattr(FsContext *ctx, const char *path,
-                                 char *name, void *value, size_t osize)
-{
-    ssize_t len = sizeof(ACL_DEFAULT);
-
-    if (!value) {
-        return len;
-    }
-
-    if (osize < len) {
-        errno = ERANGE;
-        return -1;
-    }
-
-    /* len includes the trailing NUL */
-    memcpy(value, ACL_ACCESS, len);
-    return 0;
-}
-
-static int mp_dacl_setxattr(FsContext *ctx, const char *path, const char *name,
-                            void *value, size_t size, int flags)
-{
-    char *buffer;
-    int ret;
-
-    buffer = rpath(ctx, path);
-    ret = lsetxattr(buffer, MAP_ACL_DEFAULT, value, size, flags);
-    g_free(buffer);
-    return ret;
-}
-
-static int mp_dacl_removexattr(FsContext *ctx,
-                               const char *path, const char *name)
-{
-    int ret;
-    char *buffer;
-
-    buffer = rpath(ctx, path);
-    ret  = lremovexattr(buffer, MAP_ACL_DEFAULT);
-    if (ret == -1 && errno == ENODATA) {
-        /*
-         * We don't get ENODATA error when trying to remove a
-         * posix acl that is not present. So don't throw the error
-         * even in case of mapped security model
-         */
-        errno = 0;
-        ret = 0;
-    }
-    g_free(buffer);
-    return ret;
-}
-
-
-XattrOperations mapped_pacl_xattr = {
-    .name = "system.posix_acl_access",
-    .getxattr = mp_pacl_getxattr,
-    .setxattr = mp_pacl_setxattr,
-    .listxattr = mp_pacl_listxattr,
-    .removexattr = mp_pacl_removexattr,
-};
-
-XattrOperations mapped_dacl_xattr = {
-    .name = "system.posix_acl_default",
-    .getxattr = mp_dacl_getxattr,
-    .setxattr = mp_dacl_setxattr,
-    .listxattr = mp_dacl_listxattr,
-    .removexattr = mp_dacl_removexattr,
-};
-
-XattrOperations passthrough_acl_xattr = {
-    .name = "system.posix_acl_",
-    .getxattr = pt_getxattr,
-    .setxattr = pt_setxattr,
-    .listxattr = pt_listxattr,
-    .removexattr = pt_removexattr,
-};
-
-XattrOperations none_acl_xattr = {
-    .name = "system.posix_acl_",
-    .getxattr = notsup_getxattr,
-    .setxattr = notsup_setxattr,
-    .listxattr = notsup_listxattr,
-    .removexattr = notsup_removexattr,
-};
diff --git a/hw/9pfs/virtio-9p-proxy.c b/hw/9pfs/virtio-9p-proxy.c
deleted file mode 100644
index 59c7445..0000000
--- a/hw/9pfs/virtio-9p-proxy.c
+++ /dev/null
@@ -1,1217 +0,0 @@
-/*
- * Virtio 9p Proxy callback
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- * M. Mohan Kumar <mohan@in.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- */
-#include <sys/socket.h>
-#include <sys/un.h>
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "qemu/error-report.h"
-#include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-proxy.h"
-
-typedef struct V9fsProxy {
-    int sockfd;
-    QemuMutex mutex;
-    struct iovec in_iovec;
-    struct iovec out_iovec;
-} V9fsProxy;
-
-/*
- * Return received file descriptor on success in *status.
- * errno is also returned on *status (which will be < 0)
- * return < 0 on transport error.
- */
-static int v9fs_receivefd(int sockfd, int *status)
-{
-    struct iovec iov;
-    struct msghdr msg;
-    struct cmsghdr *cmsg;
-    int retval, data, fd;
-    union MsgControl msg_control;
-
-    iov.iov_base = &data;
-    iov.iov_len = sizeof(data);
-
-    memset(&msg, 0, sizeof(msg));
-    msg.msg_iov = &iov;
-    msg.msg_iovlen = 1;
-    msg.msg_control = &msg_control;
-    msg.msg_controllen = sizeof(msg_control);
-
-    do {
-        retval = recvmsg(sockfd, &msg, 0);
-    } while (retval < 0 && errno == EINTR);
-    if (retval <= 0) {
-        return retval;
-    }
-    /*
-     * data is set to V9FS_FD_VALID, if ancillary data is sent.  If this
-     * request doesn't need ancillary data (fd) or an error occurred,
-     * data is set to negative errno value.
-     */
-    if (data != V9FS_FD_VALID) {
-        *status = data;
-        return 0;
-    }
-    /*
-     * File descriptor (fd) is sent in the ancillary data. Check if we
-     * indeed received it. One of the reasons to fail to receive it is if
-     * we exceeded the maximum number of file descriptors!
-     */
-    for (cmsg = CMSG_FIRSTHDR(&msg); cmsg; cmsg = CMSG_NXTHDR(&msg, cmsg)) {
-        if (cmsg->cmsg_len != CMSG_LEN(sizeof(int)) ||
-            cmsg->cmsg_level != SOL_SOCKET ||
-            cmsg->cmsg_type != SCM_RIGHTS) {
-            continue;
-        }
-        fd = *((int *)CMSG_DATA(cmsg));
-        *status = fd;
-        return 0;
-    }
-    *status = -ENFILE;  /* Ancillary data sent but not received */
-    return 0;
-}
-
-static ssize_t socket_read(int sockfd, void *buff, size_t size)
-{
-    ssize_t retval, total = 0;
-
-    while (size) {
-        retval = read(sockfd, buff, size);
-        if (retval == 0) {
-            return -EIO;
-        }
-        if (retval < 0) {
-            if (errno == EINTR) {
-                continue;
-            }
-            return -errno;
-        }
-        size -= retval;
-        buff += retval;
-        total += retval;
-    }
-    return total;
-}
-
-/* Converts proxy_statfs to VFS statfs structure */
-static void prstatfs_to_statfs(struct statfs *stfs, ProxyStatFS *prstfs)
-{
-    memset(stfs, 0, sizeof(*stfs));
-    stfs->f_type = prstfs->f_type;
-    stfs->f_bsize = prstfs->f_bsize;
-    stfs->f_blocks = prstfs->f_blocks;
-    stfs->f_bfree = prstfs->f_bfree;
-    stfs->f_bavail = prstfs->f_bavail;
-    stfs->f_files = prstfs->f_files;
-    stfs->f_ffree = prstfs->f_ffree;
-    stfs->f_fsid.__val[0] = prstfs->f_fsid[0] & 0xFFFFFFFFU;
-    stfs->f_fsid.__val[1] = prstfs->f_fsid[1] >> 32 & 0xFFFFFFFFU;
-    stfs->f_namelen = prstfs->f_namelen;
-    stfs->f_frsize = prstfs->f_frsize;
-}
-
-/* Converts proxy_stat structure to VFS stat structure */
-static void prstat_to_stat(struct stat *stbuf, ProxyStat *prstat)
-{
-   memset(stbuf, 0, sizeof(*stbuf));
-   stbuf->st_dev = prstat->st_dev;
-   stbuf->st_ino = prstat->st_ino;
-   stbuf->st_nlink = prstat->st_nlink;
-   stbuf->st_mode = prstat->st_mode;
-   stbuf->st_uid = prstat->st_uid;
-   stbuf->st_gid = prstat->st_gid;
-   stbuf->st_rdev = prstat->st_rdev;
-   stbuf->st_size = prstat->st_size;
-   stbuf->st_blksize = prstat->st_blksize;
-   stbuf->st_blocks = prstat->st_blocks;
-   stbuf->st_atim.tv_sec = prstat->st_atim_sec;
-   stbuf->st_atim.tv_nsec = prstat->st_atim_nsec;
-   stbuf->st_mtime = prstat->st_mtim_sec;
-   stbuf->st_mtim.tv_nsec = prstat->st_mtim_nsec;
-   stbuf->st_ctime = prstat->st_ctim_sec;
-   stbuf->st_ctim.tv_nsec = prstat->st_ctim_nsec;
-}
-
-/*
- * Response contains two parts
- * {header, data}
- * header.type == T_ERROR, data -> -errno
- * header.type == T_SUCCESS, data -> response
- * size of errno/response is given by header.size
- * returns < 0, on transport error. response is
- * valid only if status >= 0.
- */
-static int v9fs_receive_response(V9fsProxy *proxy, int type,
-                                 int *status, void *response)
-{
-    int retval;
-    ProxyHeader header;
-    struct iovec *reply = &proxy->in_iovec;
-
-    *status = 0;
-    reply->iov_len = 0;
-    retval = socket_read(proxy->sockfd, reply->iov_base, PROXY_HDR_SZ);
-    if (retval < 0) {
-        return retval;
-    }
-    reply->iov_len = PROXY_HDR_SZ;
-    proxy_unmarshal(reply, 0, "dd", &header.type, &header.size);
-    /*
-     * if response size > PROXY_MAX_IO_SZ, read the response but ignore it and
-     * return -ENOBUFS
-     */
-    if (header.size > PROXY_MAX_IO_SZ) {
-        int count;
-        while (header.size > 0) {
-            count = MIN(PROXY_MAX_IO_SZ, header.size);
-            count = socket_read(proxy->sockfd, reply->iov_base, count);
-            if (count < 0) {
-                return count;
-            }
-            header.size -= count;
-        }
-        *status = -ENOBUFS;
-        return 0;
-    }
-
-    retval = socket_read(proxy->sockfd,
-                         reply->iov_base + PROXY_HDR_SZ, header.size);
-    if (retval < 0) {
-        return retval;
-    }
-    reply->iov_len += header.size;
-    /* there was an error during processing request */
-    if (header.type == T_ERROR) {
-        int ret;
-        ret = proxy_unmarshal(reply, PROXY_HDR_SZ, "d", status);
-        if (ret < 0) {
-            *status = ret;
-        }
-        return 0;
-    }
-
-    switch (type) {
-    case T_LSTAT: {
-        ProxyStat prstat;
-        retval = proxy_unmarshal(reply, PROXY_HDR_SZ,
-                                 "qqqdddqqqqqqqqqq", &prstat.st_dev,
-                                 &prstat.st_ino, &prstat.st_nlink,
-                                 &prstat.st_mode, &prstat.st_uid,
-                                 &prstat.st_gid, &prstat.st_rdev,
-                                 &prstat.st_size, &prstat.st_blksize,
-                                 &prstat.st_blocks,
-                                 &prstat.st_atim_sec, &prstat.st_atim_nsec,
-                                 &prstat.st_mtim_sec, &prstat.st_mtim_nsec,
-                                 &prstat.st_ctim_sec, &prstat.st_ctim_nsec);
-        prstat_to_stat(response, &prstat);
-        break;
-    }
-    case T_STATFS: {
-        ProxyStatFS prstfs;
-        retval = proxy_unmarshal(reply, PROXY_HDR_SZ,
-                                 "qqqqqqqqqqq", &prstfs.f_type,
-                                 &prstfs.f_bsize, &prstfs.f_blocks,
-                                 &prstfs.f_bfree, &prstfs.f_bavail,
-                                 &prstfs.f_files, &prstfs.f_ffree,
-                                 &prstfs.f_fsid[0], &prstfs.f_fsid[1],
-                                 &prstfs.f_namelen, &prstfs.f_frsize);
-        prstatfs_to_statfs(response, &prstfs);
-        break;
-    }
-    case T_READLINK: {
-        V9fsString target;
-        v9fs_string_init(&target);
-        retval = proxy_unmarshal(reply, PROXY_HDR_SZ, "s", &target);
-        strcpy(response, target.data);
-        v9fs_string_free(&target);
-        break;
-    }
-    case T_LGETXATTR:
-    case T_LLISTXATTR: {
-        V9fsString xattr;
-        v9fs_string_init(&xattr);
-        retval = proxy_unmarshal(reply, PROXY_HDR_SZ, "s", &xattr);
-        memcpy(response, xattr.data, xattr.size);
-        v9fs_string_free(&xattr);
-        break;
-    }
-    case T_GETVERSION:
-        proxy_unmarshal(reply, PROXY_HDR_SZ, "q", response);
-        break;
-    default:
-        return -1;
-    }
-    if (retval < 0) {
-        *status  = retval;
-    }
-    return 0;
-}
-
-/*
- * return < 0 on transport error.
- * *status is valid only if return >= 0
- */
-static int v9fs_receive_status(V9fsProxy *proxy,
-                               struct iovec *reply, int *status)
-{
-    int retval;
-    ProxyHeader header;
-
-    *status = 0;
-    reply->iov_len = 0;
-    retval = socket_read(proxy->sockfd, reply->iov_base, PROXY_HDR_SZ);
-    if (retval < 0) {
-        return retval;
-    }
-    reply->iov_len = PROXY_HDR_SZ;
-    proxy_unmarshal(reply, 0, "dd", &header.type, &header.size);
-    if (header.size != sizeof(int)) {
-        *status = -ENOBUFS;
-        return 0;
-    }
-    retval = socket_read(proxy->sockfd,
-                         reply->iov_base + PROXY_HDR_SZ, header.size);
-    if (retval < 0) {
-        return retval;
-    }
-    reply->iov_len += header.size;
-    proxy_unmarshal(reply, PROXY_HDR_SZ, "d", status);
-    return 0;
-}
-
-/*
- * Proxy->header and proxy->request written to socket by QEMU process.
- * This request read by proxy helper process
- * returns 0 on success and -errno on error
- */
-static int v9fs_request(V9fsProxy *proxy, int type,
-                        void *response, const char *fmt, ...)
-{
-    dev_t rdev;
-    va_list ap;
-    int size = 0;
-    int retval = 0;
-    uint64_t offset;
-    ProxyHeader header = { 0, 0};
-    struct timespec spec[2];
-    int flags, mode, uid, gid;
-    V9fsString *name, *value;
-    V9fsString *path, *oldpath;
-    struct iovec *iovec = NULL, *reply = NULL;
-
-    qemu_mutex_lock(&proxy->mutex);
-
-    if (proxy->sockfd == -1) {
-        retval = -EIO;
-        goto err_out;
-    }
-    iovec = &proxy->out_iovec;
-    reply = &proxy->in_iovec;
-    va_start(ap, fmt);
-    switch (type) {
-    case T_OPEN:
-        path = va_arg(ap, V9fsString *);
-        flags = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sd", path, flags);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_OPEN;
-        }
-        break;
-    case T_CREATE:
-        path = va_arg(ap, V9fsString *);
-        flags = va_arg(ap, int);
-        mode = va_arg(ap, int);
-        uid = va_arg(ap, int);
-        gid = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sdddd", path,
-                                    flags, mode, uid, gid);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_CREATE;
-        }
-        break;
-    case T_MKNOD:
-        path = va_arg(ap, V9fsString *);
-        mode = va_arg(ap, int);
-        rdev = va_arg(ap, long int);
-        uid = va_arg(ap, int);
-        gid = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ddsdq",
-                                    uid, gid, path, mode, rdev);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_MKNOD;
-        }
-        break;
-    case T_MKDIR:
-        path = va_arg(ap, V9fsString *);
-        mode = va_arg(ap, int);
-        uid = va_arg(ap, int);
-        gid = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ddsd",
-                                    uid, gid, path, mode);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_MKDIR;
-        }
-        break;
-    case T_SYMLINK:
-        oldpath = va_arg(ap, V9fsString *);
-        path = va_arg(ap, V9fsString *);
-        uid = va_arg(ap, int);
-        gid = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ddss",
-                                    uid, gid, oldpath, path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_SYMLINK;
-        }
-        break;
-    case T_LINK:
-        oldpath = va_arg(ap, V9fsString *);
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ss",
-                                    oldpath, path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_LINK;
-        }
-        break;
-    case T_LSTAT:
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_LSTAT;
-        }
-        break;
-    case T_READLINK:
-        path = va_arg(ap, V9fsString *);
-        size = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sd", path, size);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_READLINK;
-        }
-        break;
-    case T_STATFS:
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_STATFS;
-        }
-        break;
-    case T_CHMOD:
-        path = va_arg(ap, V9fsString *);
-        mode = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sd", path, mode);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_CHMOD;
-        }
-        break;
-    case T_CHOWN:
-        path = va_arg(ap, V9fsString *);
-        uid = va_arg(ap, int);
-        gid = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sdd", path, uid, gid);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_CHOWN;
-        }
-        break;
-    case T_TRUNCATE:
-        path = va_arg(ap, V9fsString *);
-        offset = va_arg(ap, uint64_t);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sq", path, offset);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_TRUNCATE;
-        }
-        break;
-    case T_UTIME:
-        path = va_arg(ap, V9fsString *);
-        spec[0].tv_sec = va_arg(ap, long);
-        spec[0].tv_nsec = va_arg(ap, long);
-        spec[1].tv_sec = va_arg(ap, long);
-        spec[1].tv_nsec = va_arg(ap, long);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sqqqq", path,
-                                    spec[0].tv_sec, spec[1].tv_nsec,
-                                    spec[1].tv_sec, spec[1].tv_nsec);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_UTIME;
-        }
-        break;
-    case T_RENAME:
-        oldpath = va_arg(ap, V9fsString *);
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ss", oldpath, path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_RENAME;
-        }
-        break;
-    case T_REMOVE:
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_REMOVE;
-        }
-        break;
-    case T_LGETXATTR:
-        size = va_arg(ap, int);
-        path = va_arg(ap, V9fsString *);
-        name = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ,
-                                    "dss", size, path, name);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_LGETXATTR;
-        }
-        break;
-    case T_LLISTXATTR:
-        size = va_arg(ap, int);
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ds", size, path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_LLISTXATTR;
-        }
-        break;
-    case T_LSETXATTR:
-        path = va_arg(ap, V9fsString *);
-        name = va_arg(ap, V9fsString *);
-        value = va_arg(ap, V9fsString *);
-        size = va_arg(ap, int);
-        flags = va_arg(ap, int);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "sssdd",
-                                    path, name, value, size, flags);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_LSETXATTR;
-        }
-        break;
-    case T_LREMOVEXATTR:
-        path = va_arg(ap, V9fsString *);
-        name = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "ss", path, name);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_LREMOVEXATTR;
-        }
-        break;
-    case T_GETVERSION:
-        path = va_arg(ap, V9fsString *);
-        retval = proxy_marshal(iovec, PROXY_HDR_SZ, "s", path);
-        if (retval > 0) {
-            header.size = retval;
-            header.type = T_GETVERSION;
-        }
-        break;
-    default:
-        error_report("Invalid type %d", type);
-        retval = -EINVAL;
-        break;
-    }
-    va_end(ap);
-
-    if (retval < 0) {
-        goto err_out;
-    }
-
-    /* marshal the header details */
-    proxy_marshal(iovec, 0, "dd", header.type, header.size);
-    header.size += PROXY_HDR_SZ;
-
-    retval = qemu_write_full(proxy->sockfd, iovec->iov_base, header.size);
-    if (retval != header.size) {
-        goto close_error;
-    }
-
-    switch (type) {
-    case T_OPEN:
-    case T_CREATE:
-        /*
-         * A file descriptor is returned as response for
-         * T_OPEN,T_CREATE on success
-         */
-        if (v9fs_receivefd(proxy->sockfd, &retval) < 0) {
-            goto close_error;
-        }
-        break;
-    case T_MKNOD:
-    case T_MKDIR:
-    case T_SYMLINK:
-    case T_LINK:
-    case T_CHMOD:
-    case T_CHOWN:
-    case T_RENAME:
-    case T_TRUNCATE:
-    case T_UTIME:
-    case T_REMOVE:
-    case T_LSETXATTR:
-    case T_LREMOVEXATTR:
-        if (v9fs_receive_status(proxy, reply, &retval) < 0) {
-            goto close_error;
-        }
-        break;
-    case T_LSTAT:
-    case T_READLINK:
-    case T_STATFS:
-    case T_GETVERSION:
-        if (v9fs_receive_response(proxy, type, &retval, response) < 0) {
-            goto close_error;
-        }
-        break;
-    case T_LGETXATTR:
-    case T_LLISTXATTR:
-        if (!size) {
-            if (v9fs_receive_status(proxy, reply, &retval) < 0) {
-                goto close_error;
-            }
-        } else {
-            if (v9fs_receive_response(proxy, type, &retval, response) < 0) {
-                goto close_error;
-            }
-        }
-        break;
-    }
-
-err_out:
-    qemu_mutex_unlock(&proxy->mutex);
-    return retval;
-
-close_error:
-    close(proxy->sockfd);
-    proxy->sockfd = -1;
-    qemu_mutex_unlock(&proxy->mutex);
-    return -EIO;
-}
-
-static int proxy_lstat(FsContext *fs_ctx, V9fsPath *fs_path, struct stat *stbuf)
-{
-    int retval;
-    retval = v9fs_request(fs_ctx->private, T_LSTAT, stbuf, "s", fs_path);
-    if (retval < 0) {
-        errno = -retval;
-        return -1;
-    }
-    return retval;
-}
-
-static ssize_t proxy_readlink(FsContext *fs_ctx, V9fsPath *fs_path,
-                              char *buf, size_t bufsz)
-{
-    int retval;
-    retval = v9fs_request(fs_ctx->private, T_READLINK, buf, "sd",
-                          fs_path, bufsz);
-    if (retval < 0) {
-        errno = -retval;
-        return -1;
-    }
-    return strlen(buf);
-}
-
-static int proxy_close(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return close(fs->fd);
-}
-
-static int proxy_closedir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return closedir(fs->dir);
-}
-
-static int proxy_open(FsContext *ctx, V9fsPath *fs_path,
-                      int flags, V9fsFidOpenState *fs)
-{
-    fs->fd = v9fs_request(ctx->private, T_OPEN, NULL, "sd", fs_path, flags);
-    if (fs->fd < 0) {
-        errno = -fs->fd;
-        fs->fd = -1;
-    }
-    return fs->fd;
-}
-
-static int proxy_opendir(FsContext *ctx,
-                         V9fsPath *fs_path, V9fsFidOpenState *fs)
-{
-    int serrno, fd;
-
-    fs->dir = NULL;
-    fd = v9fs_request(ctx->private, T_OPEN, NULL, "sd", fs_path, O_DIRECTORY);
-    if (fd < 0) {
-        errno = -fd;
-        return -1;
-    }
-    fs->dir = fdopendir(fd);
-    if (!fs->dir) {
-        serrno = errno;
-        close(fd);
-        errno = serrno;
-        return -1;
-    }
-    return 0;
-}
-
-static void proxy_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return rewinddir(fs->dir);
-}
-
-static off_t proxy_telldir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    return telldir(fs->dir);
-}
-
-static int proxy_readdir_r(FsContext *ctx, V9fsFidOpenState *fs,
-                           struct dirent *entry,
-                           struct dirent **result)
-{
-    return readdir_r(fs->dir, entry, result);
-}
-
-static void proxy_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
-{
-    return seekdir(fs->dir, off);
-}
-
-static ssize_t proxy_preadv(FsContext *ctx, V9fsFidOpenState *fs,
-                            const struct iovec *iov,
-                            int iovcnt, off_t offset)
-{
-#ifdef CONFIG_PREADV
-    return preadv(fs->fd, iov, iovcnt, offset);
-#else
-    int err = lseek(fs->fd, offset, SEEK_SET);
-    if (err == -1) {
-        return err;
-    } else {
-        return readv(fs->fd, iov, iovcnt);
-    }
-#endif
-}
-
-static ssize_t proxy_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
-                             const struct iovec *iov,
-                             int iovcnt, off_t offset)
-{
-    ssize_t ret;
-
-#ifdef CONFIG_PREADV
-    ret = pwritev(fs->fd, iov, iovcnt, offset);
-#else
-    int err = lseek(fs->fd, offset, SEEK_SET);
-    if (err == -1) {
-        return err;
-    } else {
-        ret = writev(fs->fd, iov, iovcnt);
-    }
-#endif
-#ifdef CONFIG_SYNC_FILE_RANGE
-    if (ret > 0 && ctx->export_flags & V9FS_IMMEDIATE_WRITEOUT) {
-        /*
-         * Initiate a writeback. This is not a data integrity sync.
-         * We want to ensure that we don't leave dirty pages in the cache
-         * after write when writeout=immediate is sepcified.
-         */
-        sync_file_range(fs->fd, offset, ret,
-                        SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE);
-    }
-#endif
-    return ret;
-}
-
-static int proxy_chmod(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
-{
-    int retval;
-    retval = v9fs_request(fs_ctx->private, T_CHMOD, NULL, "sd",
-                          fs_path, credp->fc_mode);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_mknod(FsContext *fs_ctx, V9fsPath *dir_path,
-                       const char *name, FsCred *credp)
-{
-    int retval;
-    V9fsString fullname;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-
-    retval = v9fs_request(fs_ctx->private, T_MKNOD, NULL, "sdqdd",
-                          &fullname, credp->fc_mode, credp->fc_rdev,
-                          credp->fc_uid, credp->fc_gid);
-    v9fs_string_free(&fullname);
-    if (retval < 0) {
-        errno = -retval;
-        retval = -1;
-    }
-    return retval;
-}
-
-static int proxy_mkdir(FsContext *fs_ctx, V9fsPath *dir_path,
-                       const char *name, FsCred *credp)
-{
-    int retval;
-    V9fsString fullname;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-
-    retval = v9fs_request(fs_ctx->private, T_MKDIR, NULL, "sddd", &fullname,
-                          credp->fc_mode, credp->fc_uid, credp->fc_gid);
-    v9fs_string_free(&fullname);
-    if (retval < 0) {
-        errno = -retval;
-        retval = -1;
-    }
-    v9fs_string_free(&fullname);
-    return retval;
-}
-
-static int proxy_fstat(FsContext *fs_ctx, int fid_type,
-                       V9fsFidOpenState *fs, struct stat *stbuf)
-{
-    int fd;
-
-    if (fid_type == P9_FID_DIR) {
-        fd = dirfd(fs->dir);
-    } else {
-        fd = fs->fd;
-    }
-    return fstat(fd, stbuf);
-}
-
-static int proxy_open2(FsContext *fs_ctx, V9fsPath *dir_path, const char *name,
-                       int flags, FsCred *credp, V9fsFidOpenState *fs)
-{
-    V9fsString fullname;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-
-    fs->fd = v9fs_request(fs_ctx->private, T_CREATE, NULL, "sdddd",
-                          &fullname, flags, credp->fc_mode,
-                          credp->fc_uid, credp->fc_gid);
-    v9fs_string_free(&fullname);
-    if (fs->fd < 0) {
-        errno = -fs->fd;
-        fs->fd = -1;
-    }
-    return fs->fd;
-}
-
-static int proxy_symlink(FsContext *fs_ctx, const char *oldpath,
-                         V9fsPath *dir_path, const char *name, FsCred *credp)
-{
-    int retval;
-    V9fsString fullname, target;
-
-    v9fs_string_init(&fullname);
-    v9fs_string_init(&target);
-
-    v9fs_string_sprintf(&fullname, "%s/%s", dir_path->data, name);
-    v9fs_string_sprintf(&target, "%s", oldpath);
-
-    retval = v9fs_request(fs_ctx->private, T_SYMLINK, NULL, "ssdd",
-                          &target, &fullname, credp->fc_uid, credp->fc_gid);
-    v9fs_string_free(&fullname);
-    v9fs_string_free(&target);
-    if (retval < 0) {
-        errno = -retval;
-        retval = -1;
-    }
-    return retval;
-}
-
-static int proxy_link(FsContext *ctx, V9fsPath *oldpath,
-                      V9fsPath *dirpath, const char *name)
-{
-    int retval;
-    V9fsString newpath;
-
-    v9fs_string_init(&newpath);
-    v9fs_string_sprintf(&newpath, "%s/%s", dirpath->data, name);
-
-    retval = v9fs_request(ctx->private, T_LINK, NULL, "ss", oldpath, &newpath);
-    v9fs_string_free(&newpath);
-    if (retval < 0) {
-        errno = -retval;
-        retval = -1;
-    }
-    return retval;
-}
-
-static int proxy_truncate(FsContext *ctx, V9fsPath *fs_path, off_t size)
-{
-    int retval;
-
-    retval = v9fs_request(ctx->private, T_TRUNCATE, NULL, "sq", fs_path, size);
-    if (retval < 0) {
-        errno = -retval;
-        return -1;
-    }
-    return 0;
-}
-
-static int proxy_rename(FsContext *ctx, const char *oldpath,
-                        const char *newpath)
-{
-    int retval;
-    V9fsString oldname, newname;
-
-    v9fs_string_init(&oldname);
-    v9fs_string_init(&newname);
-
-    v9fs_string_sprintf(&oldname, "%s", oldpath);
-    v9fs_string_sprintf(&newname, "%s", newpath);
-    retval = v9fs_request(ctx->private, T_RENAME, NULL, "ss",
-                          &oldname, &newname);
-    v9fs_string_free(&oldname);
-    v9fs_string_free(&newname);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_chown(FsContext *fs_ctx, V9fsPath *fs_path, FsCred *credp)
-{
-    int retval;
-    retval = v9fs_request(fs_ctx->private, T_CHOWN, NULL, "sdd",
-                          fs_path, credp->fc_uid, credp->fc_gid);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_utimensat(FsContext *s, V9fsPath *fs_path,
-                           const struct timespec *buf)
-{
-    int retval;
-    retval = v9fs_request(s->private, T_UTIME, NULL, "sqqqq",
-                          fs_path,
-                          buf[0].tv_sec, buf[0].tv_nsec,
-                          buf[1].tv_sec, buf[1].tv_nsec);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_remove(FsContext *ctx, const char *path)
-{
-    int retval;
-    V9fsString name;
-    v9fs_string_init(&name);
-    v9fs_string_sprintf(&name, "%s", path);
-    retval = v9fs_request(ctx->private, T_REMOVE, NULL, "s", &name);
-    v9fs_string_free(&name);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_fsync(FsContext *ctx, int fid_type,
-                       V9fsFidOpenState *fs, int datasync)
-{
-    int fd;
-
-    if (fid_type == P9_FID_DIR) {
-        fd = dirfd(fs->dir);
-    } else {
-        fd = fs->fd;
-    }
-
-    if (datasync) {
-        return qemu_fdatasync(fd);
-    } else {
-        return fsync(fd);
-    }
-}
-
-static int proxy_statfs(FsContext *s, V9fsPath *fs_path, struct statfs *stbuf)
-{
-    int retval;
-    retval = v9fs_request(s->private, T_STATFS, stbuf, "s", fs_path);
-    if (retval < 0) {
-        errno = -retval;
-        return -1;
-    }
-    return retval;
-}
-
-static ssize_t proxy_lgetxattr(FsContext *ctx, V9fsPath *fs_path,
-                               const char *name, void *value, size_t size)
-{
-    int retval;
-    V9fsString xname;
-
-    v9fs_string_init(&xname);
-    v9fs_string_sprintf(&xname, "%s", name);
-    retval = v9fs_request(ctx->private, T_LGETXATTR, value, "dss", size,
-                          fs_path, &xname);
-    v9fs_string_free(&xname);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static ssize_t proxy_llistxattr(FsContext *ctx, V9fsPath *fs_path,
-                                void *value, size_t size)
-{
-    int retval;
-    retval = v9fs_request(ctx->private, T_LLISTXATTR, value, "ds", size,
-                        fs_path);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_lsetxattr(FsContext *ctx, V9fsPath *fs_path, const char *name,
-                           void *value, size_t size, int flags)
-{
-    int retval;
-    V9fsString xname, xvalue;
-
-    v9fs_string_init(&xname);
-    v9fs_string_sprintf(&xname, "%s", name);
-
-    v9fs_string_init(&xvalue);
-    xvalue.size = size;
-    xvalue.data = g_malloc(size);
-    memcpy(xvalue.data, value, size);
-
-    retval = v9fs_request(ctx->private, T_LSETXATTR, value, "sssdd",
-                          fs_path, &xname, &xvalue, size, flags);
-    v9fs_string_free(&xname);
-    v9fs_string_free(&xvalue);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_lremovexattr(FsContext *ctx, V9fsPath *fs_path,
-                              const char *name)
-{
-    int retval;
-    V9fsString xname;
-
-    v9fs_string_init(&xname);
-    v9fs_string_sprintf(&xname, "%s", name);
-    retval = v9fs_request(ctx->private, T_LREMOVEXATTR, NULL, "ss",
-                          fs_path, &xname);
-    v9fs_string_free(&xname);
-    if (retval < 0) {
-        errno = -retval;
-    }
-    return retval;
-}
-
-static int proxy_name_to_path(FsContext *ctx, V9fsPath *dir_path,
-                              const char *name, V9fsPath *target)
-{
-    if (dir_path) {
-        v9fs_string_sprintf((V9fsString *)target, "%s/%s",
-                            dir_path->data, name);
-    } else {
-        v9fs_string_sprintf((V9fsString *)target, "%s", name);
-    }
-    /* Bump the size for including terminating NULL */
-    target->size++;
-    return 0;
-}
-
-static int proxy_renameat(FsContext *ctx, V9fsPath *olddir,
-                          const char *old_name, V9fsPath *newdir,
-                          const char *new_name)
-{
-    int ret;
-    V9fsString old_full_name, new_full_name;
-
-    v9fs_string_init(&old_full_name);
-    v9fs_string_init(&new_full_name);
-
-    v9fs_string_sprintf(&old_full_name, "%s/%s", olddir->data, old_name);
-    v9fs_string_sprintf(&new_full_name, "%s/%s", newdir->data, new_name);
-
-    ret = proxy_rename(ctx, old_full_name.data, new_full_name.data);
-    v9fs_string_free(&old_full_name);
-    v9fs_string_free(&new_full_name);
-    return ret;
-}
-
-static int proxy_unlinkat(FsContext *ctx, V9fsPath *dir,
-                          const char *name, int flags)
-{
-    int ret;
-    V9fsString fullname;
-    v9fs_string_init(&fullname);
-
-    v9fs_string_sprintf(&fullname, "%s/%s", dir->data, name);
-    ret = proxy_remove(ctx, fullname.data);
-    v9fs_string_free(&fullname);
-
-    return ret;
-}
-
-static int proxy_ioc_getversion(FsContext *fs_ctx, V9fsPath *path,
-                                mode_t st_mode, uint64_t *st_gen)
-{
-    int err;
-
-    /* Do not try to open special files like device nodes, fifos etc
-     * we can get fd for regular files and directories only
-     */
-    if (!S_ISREG(st_mode) && !S_ISDIR(st_mode)) {
-        errno = ENOTTY;
-        return -1;
-    }
-    err = v9fs_request(fs_ctx->private, T_GETVERSION, st_gen, "s", path);
-    if (err < 0) {
-        errno = -err;
-        err = -1;
-    }
-    return err;
-}
-
-static int connect_namedsocket(const char *path)
-{
-    int sockfd, size;
-    struct sockaddr_un helper;
-
-    sockfd = socket(AF_UNIX, SOCK_STREAM, 0);
-    if (sockfd < 0) {
-        fprintf(stderr, "failed to create socket: %s\n", strerror(errno));
-        return -1;
-    }
-    strcpy(helper.sun_path, path);
-    helper.sun_family = AF_UNIX;
-    size = strlen(helper.sun_path) + sizeof(helper.sun_family);
-    if (connect(sockfd, (struct sockaddr *)&helper, size) < 0) {
-        fprintf(stderr, "failed to connect to %s: %s\n", path, strerror(errno));
-        close(sockfd);
-        return -1;
-    }
-
-    /* remove the socket for security reasons */
-    unlink(path);
-    return sockfd;
-}
-
-static int proxy_parse_opts(QemuOpts *opts, struct FsDriverEntry *fs)
-{
-    const char *socket = qemu_opt_get(opts, "socket");
-    const char *sock_fd = qemu_opt_get(opts, "sock_fd");
-
-    if (!socket && !sock_fd) {
-        fprintf(stderr, "socket and sock_fd none of the option specified\n");
-        return -1;
-    }
-    if (socket && sock_fd) {
-        fprintf(stderr, "Both socket and sock_fd options specified\n");
-        return -1;
-    }
-    if (socket) {
-        fs->path = g_strdup(socket);
-        fs->export_flags = V9FS_PROXY_SOCK_NAME;
-    } else {
-        fs->path = g_strdup(sock_fd);
-        fs->export_flags = V9FS_PROXY_SOCK_FD;
-    }
-    return 0;
-}
-
-static int proxy_init(FsContext *ctx)
-{
-    V9fsProxy *proxy = g_malloc(sizeof(V9fsProxy));
-    int sock_id;
-
-    if (ctx->export_flags & V9FS_PROXY_SOCK_NAME) {
-        sock_id = connect_namedsocket(ctx->fs_root);
-    } else {
-        sock_id = atoi(ctx->fs_root);
-        if (sock_id < 0) {
-            fprintf(stderr, "socket descriptor not initialized\n");
-        }
-    }
-    if (sock_id < 0) {
-        g_free(proxy);
-        return -1;
-    }
-    g_free(ctx->fs_root);
-    ctx->fs_root = NULL;
-
-    proxy->in_iovec.iov_base  = g_malloc(PROXY_MAX_IO_SZ + PROXY_HDR_SZ);
-    proxy->in_iovec.iov_len   = PROXY_MAX_IO_SZ + PROXY_HDR_SZ;
-    proxy->out_iovec.iov_base = g_malloc(PROXY_MAX_IO_SZ + PROXY_HDR_SZ);
-    proxy->out_iovec.iov_len  = PROXY_MAX_IO_SZ + PROXY_HDR_SZ;
-
-    ctx->private = proxy;
-    proxy->sockfd = sock_id;
-    qemu_mutex_init(&proxy->mutex);
-
-    ctx->export_flags |= V9FS_PATHNAME_FSCONTEXT;
-    ctx->exops.get_st_gen = proxy_ioc_getversion;
-    return 0;
-}
-
-FileOperations proxy_ops = {
-    .parse_opts   = proxy_parse_opts,
-    .init         = proxy_init,
-    .lstat        = proxy_lstat,
-    .readlink     = proxy_readlink,
-    .close        = proxy_close,
-    .closedir     = proxy_closedir,
-    .open         = proxy_open,
-    .opendir      = proxy_opendir,
-    .rewinddir    = proxy_rewinddir,
-    .telldir      = proxy_telldir,
-    .readdir_r    = proxy_readdir_r,
-    .seekdir      = proxy_seekdir,
-    .preadv       = proxy_preadv,
-    .pwritev      = proxy_pwritev,
-    .chmod        = proxy_chmod,
-    .mknod        = proxy_mknod,
-    .mkdir        = proxy_mkdir,
-    .fstat        = proxy_fstat,
-    .open2        = proxy_open2,
-    .symlink      = proxy_symlink,
-    .link         = proxy_link,
-    .truncate     = proxy_truncate,
-    .rename       = proxy_rename,
-    .chown        = proxy_chown,
-    .utimensat    = proxy_utimensat,
-    .remove       = proxy_remove,
-    .fsync        = proxy_fsync,
-    .statfs       = proxy_statfs,
-    .lgetxattr    = proxy_lgetxattr,
-    .llistxattr   = proxy_llistxattr,
-    .lsetxattr    = proxy_lsetxattr,
-    .lremovexattr = proxy_lremovexattr,
-    .name_to_path = proxy_name_to_path,
-    .renameat     = proxy_renameat,
-    .unlinkat     = proxy_unlinkat,
-};
diff --git a/hw/9pfs/virtio-9p-proxy.h b/hw/9pfs/virtio-9p-proxy.h
deleted file mode 100644
index 005c1ad..0000000
--- a/hw/9pfs/virtio-9p-proxy.h
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Virtio 9p Proxy callback
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- * M. Mohan Kumar <mohan@in.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- */
-#ifndef _QEMU_VIRTIO_9P_PROXY_H
-#define _QEMU_VIRTIO_9P_PROXY_H
-
-#define PROXY_MAX_IO_SZ (64 * 1024)
-#define V9FS_FD_VALID INT_MAX
-
-/*
- * proxy iovec only support one element and
- * marsha/unmarshal doesn't do little endian conversion.
- */
-#define proxy_unmarshal(in_sg, offset, fmt, args...) \
-    v9fs_unmarshal(in_sg, 1, offset, 0, fmt, ##args)
-#define proxy_marshal(out_sg, offset, fmt, args...) \
-    v9fs_marshal(out_sg, 1, offset, 0, fmt, ##args)
-
-union MsgControl {
-    struct cmsghdr cmsg;
-    char control[CMSG_SPACE(sizeof(int))];
-};
-
-typedef struct {
-    uint32_t type;
-    uint32_t size;
-} ProxyHeader;
-
-#define PROXY_HDR_SZ (sizeof(ProxyHeader))
-
-enum {
-    T_SUCCESS = 0,
-    T_ERROR,
-    T_OPEN,
-    T_CREATE,
-    T_MKNOD,
-    T_MKDIR,
-    T_SYMLINK,
-    T_LINK,
-    T_LSTAT,
-    T_READLINK,
-    T_STATFS,
-    T_CHMOD,
-    T_CHOWN,
-    T_TRUNCATE,
-    T_UTIME,
-    T_RENAME,
-    T_REMOVE,
-    T_LGETXATTR,
-    T_LLISTXATTR,
-    T_LSETXATTR,
-    T_LREMOVEXATTR,
-    T_GETVERSION,
-};
-
-typedef struct {
-    uint64_t st_dev;
-    uint64_t st_ino;
-    uint64_t st_nlink;
-    uint32_t st_mode;
-    uint32_t st_uid;
-    uint32_t st_gid;
-    uint64_t st_rdev;
-    uint64_t st_size;
-    uint64_t st_blksize;
-    uint64_t st_blocks;
-    uint64_t st_atim_sec;
-    uint64_t st_atim_nsec;
-    uint64_t st_mtim_sec;
-    uint64_t st_mtim_nsec;
-    uint64_t st_ctim_sec;
-    uint64_t st_ctim_nsec;
-} ProxyStat;
-
-typedef struct {
-    uint64_t f_type;
-    uint64_t f_bsize;
-    uint64_t f_blocks;
-    uint64_t f_bfree;
-    uint64_t f_bavail;
-    uint64_t f_files;
-    uint64_t f_ffree;
-    uint64_t f_fsid[2];
-    uint64_t f_namelen;
-    uint64_t f_frsize;
-} ProxyStatFS;
-#endif
diff --git a/hw/9pfs/virtio-9p-synth.c b/hw/9pfs/virtio-9p-synth.c
deleted file mode 100644
index 71262bc..0000000
--- a/hw/9pfs/virtio-9p-synth.c
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- * Virtio 9p synthetic file system support
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Malahal Naineni <malahal@us.ibm.com>
- *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "virtio-9p-xattr.h"
-#include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-synth.h"
-
-#include <sys/stat.h>
-
-/* Root node for synth file system */
-static V9fsSynthNode v9fs_synth_root = {
-    .name = "/",
-    .actual_attr = {
-        .mode = 0555 | S_IFDIR,
-        .nlink = 1,
-    },
-    .attr = &v9fs_synth_root.actual_attr,
-};
-
-static QemuMutex  v9fs_synth_mutex;
-static int v9fs_synth_node_count;
-/* set to 1 when the synth fs is ready */
-static int v9fs_synth_fs;
-
-static V9fsSynthNode *v9fs_add_dir_node(V9fsSynthNode *parent, int mode,
-                                        const char *name,
-                                        V9fsSynthNodeAttr *attr, int inode)
-{
-    V9fsSynthNode *node;
-
-    /* Add directory type and remove write bits */
-    mode = ((mode & 0777) | S_IFDIR) & ~(S_IWUSR | S_IWGRP | S_IWOTH);
-    node = g_malloc0(sizeof(V9fsSynthNode));
-    if (attr) {
-        /* We are adding .. or . entries */
-        node->attr = attr;
-        node->attr->nlink++;
-    } else {
-        node->attr = &node->actual_attr;
-        node->attr->inode = inode;
-        node->attr->nlink = 1;
-        /* We don't allow write to directories */
-        node->attr->mode   = mode;
-        node->attr->write = NULL;
-        node->attr->read  = NULL;
-    }
-    node->private = node;
-    pstrcpy(node->name, sizeof(node->name), name);
-    QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
-    return node;
-}
-
-int qemu_v9fs_synth_mkdir(V9fsSynthNode *parent, int mode,
-                          const char *name, V9fsSynthNode **result)
-{
-    int ret;
-    V9fsSynthNode *node, *tmp;
-
-    if (!v9fs_synth_fs) {
-        return EAGAIN;
-    }
-    if (!name || (strlen(name) >= NAME_MAX)) {
-        return EINVAL;
-    }
-    if (!parent) {
-        parent = &v9fs_synth_root;
-    }
-    qemu_mutex_lock(&v9fs_synth_mutex);
-    QLIST_FOREACH(tmp, &parent->child, sibling) {
-        if (!strcmp(tmp->name, name)) {
-            ret = EEXIST;
-            goto err_out;
-        }
-    }
-    /* Add the name */
-    node = v9fs_add_dir_node(parent, mode, name, NULL, v9fs_synth_node_count++);
-    v9fs_add_dir_node(node, parent->attr->mode, "..",
-                      parent->attr, parent->attr->inode);
-    v9fs_add_dir_node(node, node->attr->mode, ".",
-                      node->attr, node->attr->inode);
-    *result = node;
-    ret = 0;
-err_out:
-    qemu_mutex_unlock(&v9fs_synth_mutex);
-    return ret;
-}
-
-int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
-                             const char *name, v9fs_synth_read read,
-                             v9fs_synth_write write, void *arg)
-{
-    int ret;
-    V9fsSynthNode *node, *tmp;
-
-    if (!v9fs_synth_fs) {
-        return EAGAIN;
-    }
-    if (!name || (strlen(name) >= NAME_MAX)) {
-        return EINVAL;
-    }
-    if (!parent) {
-        parent = &v9fs_synth_root;
-    }
-
-    qemu_mutex_lock(&v9fs_synth_mutex);
-    QLIST_FOREACH(tmp, &parent->child, sibling) {
-        if (!strcmp(tmp->name, name)) {
-            ret = EEXIST;
-            goto err_out;
-        }
-    }
-    /* Add file type and remove write bits */
-    mode = ((mode & 0777) | S_IFREG);
-    node = g_malloc0(sizeof(V9fsSynthNode));
-    node->attr         = &node->actual_attr;
-    node->attr->inode  = v9fs_synth_node_count++;
-    node->attr->nlink  = 1;
-    node->attr->read   = read;
-    node->attr->write  = write;
-    node->attr->mode   = mode;
-    node->private      = arg;
-    pstrcpy(node->name, sizeof(node->name), name);
-    QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
-    ret = 0;
-err_out:
-    qemu_mutex_unlock(&v9fs_synth_mutex);
-    return ret;
-}
-
-static void v9fs_synth_fill_statbuf(V9fsSynthNode *node, struct stat *stbuf)
-{
-    stbuf->st_dev = 0;
-    stbuf->st_ino = node->attr->inode;
-    stbuf->st_mode = node->attr->mode;
-    stbuf->st_nlink = node->attr->nlink;
-    stbuf->st_uid = 0;
-    stbuf->st_gid = 0;
-    stbuf->st_rdev = 0;
-    stbuf->st_size = 0;
-    stbuf->st_blksize = 0;
-    stbuf->st_blocks = 0;
-    stbuf->st_atime = 0;
-    stbuf->st_mtime = 0;
-    stbuf->st_ctime = 0;
-}
-
-static int v9fs_synth_lstat(FsContext *fs_ctx,
-                            V9fsPath *fs_path, struct stat *stbuf)
-{
-    V9fsSynthNode *node = *(V9fsSynthNode **)fs_path->data;
-
-    v9fs_synth_fill_statbuf(node, stbuf);
-    return 0;
-}
-
-static int v9fs_synth_fstat(FsContext *fs_ctx, int fid_type,
-                            V9fsFidOpenState *fs, struct stat *stbuf)
-{
-    V9fsSynthOpenState *synth_open = fs->private;
-    v9fs_synth_fill_statbuf(synth_open->node, stbuf);
-    return 0;
-}
-
-static int v9fs_synth_opendir(FsContext *ctx,
-                             V9fsPath *fs_path, V9fsFidOpenState *fs)
-{
-    V9fsSynthOpenState *synth_open;
-    V9fsSynthNode *node = *(V9fsSynthNode **)fs_path->data;
-
-    synth_open = g_malloc(sizeof(*synth_open));
-    synth_open->node = node;
-    node->open_count++;
-    fs->private = synth_open;
-    return 0;
-}
-
-static int v9fs_synth_closedir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    V9fsSynthOpenState *synth_open = fs->private;
-    V9fsSynthNode *node = synth_open->node;
-
-    node->open_count--;
-    g_free(synth_open);
-    fs->private = NULL;
-    return 0;
-}
-
-static off_t v9fs_synth_telldir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    V9fsSynthOpenState *synth_open = fs->private;
-    return synth_open->offset;
-}
-
-static void v9fs_synth_seekdir(FsContext *ctx, V9fsFidOpenState *fs, off_t off)
-{
-    V9fsSynthOpenState *synth_open = fs->private;
-    synth_open->offset = off;
-}
-
-static void v9fs_synth_rewinddir(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    v9fs_synth_seekdir(ctx, fs, 0);
-}
-
-static void v9fs_synth_direntry(V9fsSynthNode *node,
-                                struct dirent *entry, off_t off)
-{
-    strcpy(entry->d_name, node->name);
-    entry->d_ino = node->attr->inode;
-    entry->d_off = off + 1;
-}
-
-static int v9fs_synth_get_dentry(V9fsSynthNode *dir, struct dirent *entry,
-                                 struct dirent **result, off_t off)
-{
-    int i = 0;
-    V9fsSynthNode *node;
-
-    rcu_read_lock();
-    QLIST_FOREACH(node, &dir->child, sibling) {
-        /* This is the off child of the directory */
-        if (i == off) {
-            break;
-        }
-        i++;
-    }
-    rcu_read_unlock();
-    if (!node) {
-        /* end of directory */
-        *result = NULL;
-        return 0;
-    }
-    v9fs_synth_direntry(node, entry, off);
-    *result = entry;
-    return 0;
-}
-
-static int v9fs_synth_readdir_r(FsContext *ctx, V9fsFidOpenState *fs,
-                                struct dirent *entry, struct dirent **result)
-{
-    int ret;
-    V9fsSynthOpenState *synth_open = fs->private;
-    V9fsSynthNode *node = synth_open->node;
-    ret = v9fs_synth_get_dentry(node, entry, result, synth_open->offset);
-    if (!ret && *result != NULL) {
-        synth_open->offset++;
-    }
-    return ret;
-}
-
-static int v9fs_synth_open(FsContext *ctx, V9fsPath *fs_path,
-                           int flags, V9fsFidOpenState *fs)
-{
-    V9fsSynthOpenState *synth_open;
-    V9fsSynthNode *node = *(V9fsSynthNode **)fs_path->data;
-
-    synth_open = g_malloc(sizeof(*synth_open));
-    synth_open->node = node;
-    node->open_count++;
-    fs->private = synth_open;
-    return 0;
-}
-
-static int v9fs_synth_open2(FsContext *fs_ctx, V9fsPath *dir_path,
-                            const char *name, int flags,
-                            FsCred *credp, V9fsFidOpenState *fs)
-{
-    errno = ENOSYS;
-    return -1;
-}
-
-static int v9fs_synth_close(FsContext *ctx, V9fsFidOpenState *fs)
-{
-    V9fsSynthOpenState *synth_open = fs->private;
-    V9fsSynthNode *node = synth_open->node;
-
-    node->open_count--;
-    g_free(synth_open);
-    fs->private = NULL;
-    return 0;
-}
-
-static ssize_t v9fs_synth_pwritev(FsContext *ctx, V9fsFidOpenState *fs,
-                                  const struct iovec *iov,
-                                  int iovcnt, off_t offset)
-{
-    int i, count = 0, wcount;
-    V9fsSynthOpenState *synth_open = fs->private;
-    V9fsSynthNode *node = synth_open->node;
-    if (!node->attr->write) {
-        errno = EPERM;
-        return -1;
-    }
-    for (i = 0; i < iovcnt; i++) {
-        wcount = node->attr->write(iov[i].iov_base, iov[i].iov_len,
-                                   offset, node->private);
-        offset += wcount;
-        count  += wcount;
-        /* If we wrote less than requested. we are done */
-        if (wcount < iov[i].iov_len) {
-            break;
-        }
-    }
-    return count;
-}
-
-static ssize_t v9fs_synth_preadv(FsContext *ctx, V9fsFidOpenState *fs,
-                                 const struct iovec *iov,
-                                 int iovcnt, off_t offset)
-{
-    int i, count = 0, rcount;
-    V9fsSynthOpenState *synth_open = fs->private;
-    V9fsSynthNode *node = synth_open->node;
-    if (!node->attr->read) {
-        errno = EPERM;
-        return -1;
-    }
-    for (i = 0; i < iovcnt; i++) {
-        rcount = node->attr->read(iov[i].iov_base, iov[i].iov_len,
-                                  offset, node->private);
-        offset += rcount;
-        count  += rcount;
-        /* If we read less than requested. we are done */
-        if (rcount < iov[i].iov_len) {
-            break;
-        }
-    }
-    return count;
-}
-
-static int v9fs_synth_truncate(FsContext *ctx, V9fsPath *path, off_t offset)
-{
-    errno = ENOSYS;
-    return -1;
-}
-
-static int v9fs_synth_chmod(FsContext *fs_ctx, V9fsPath *path, FsCred *credp)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_mknod(FsContext *fs_ctx, V9fsPath *path,
-                       const char *buf, FsCred *credp)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_mkdir(FsContext *fs_ctx, V9fsPath *path,
-                       const char *buf, FsCred *credp)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static ssize_t v9fs_synth_readlink(FsContext *fs_ctx, V9fsPath *path,
-                                   char *buf, size_t bufsz)
-{
-    errno = ENOSYS;
-    return -1;
-}
-
-static int v9fs_synth_symlink(FsContext *fs_ctx, const char *oldpath,
-                              V9fsPath *newpath, const char *buf, FsCred *credp)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_link(FsContext *fs_ctx, V9fsPath *oldpath,
-                           V9fsPath *newpath, const char *buf)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_rename(FsContext *ctx, const char *oldpath,
-                             const char *newpath)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_chown(FsContext *fs_ctx, V9fsPath *path, FsCred *credp)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_utimensat(FsContext *fs_ctx, V9fsPath *path,
-                                const struct timespec *buf)
-{
-    errno = EPERM;
-    return 0;
-}
-
-static int v9fs_synth_remove(FsContext *ctx, const char *path)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_fsync(FsContext *ctx, int fid_type,
-                            V9fsFidOpenState *fs, int datasync)
-{
-    errno = ENOSYS;
-    return 0;
-}
-
-static int v9fs_synth_statfs(FsContext *s, V9fsPath *fs_path,
-                             struct statfs *stbuf)
-{
-    stbuf->f_type = 0xABCD;
-    stbuf->f_bsize = 512;
-    stbuf->f_blocks = 0;
-    stbuf->f_files = v9fs_synth_node_count;
-    stbuf->f_namelen = NAME_MAX;
-    return 0;
-}
-
-static ssize_t v9fs_synth_lgetxattr(FsContext *ctx, V9fsPath *path,
-                                    const char *name, void *value, size_t size)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-static ssize_t v9fs_synth_llistxattr(FsContext *ctx, V9fsPath *path,
-                                     void *value, size_t size)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-static int v9fs_synth_lsetxattr(FsContext *ctx, V9fsPath *path,
-                                const char *name, void *value,
-                                size_t size, int flags)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-static int v9fs_synth_lremovexattr(FsContext *ctx,
-                                   V9fsPath *path, const char *name)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-static int v9fs_synth_name_to_path(FsContext *ctx, V9fsPath *dir_path,
-                                   const char *name, V9fsPath *target)
-{
-    V9fsSynthNode *node;
-    V9fsSynthNode *dir_node;
-
-    /* "." and ".." are not allowed */
-    if (!strcmp(name, ".") || !strcmp(name, "..")) {
-        errno = EINVAL;
-        return -1;
-
-    }
-    if (!dir_path) {
-        dir_node = &v9fs_synth_root;
-    } else {
-        dir_node = *(V9fsSynthNode **)dir_path->data;
-    }
-    if (!strcmp(name, "/")) {
-        node = dir_node;
-        goto out;
-    }
-    /* search for the name in the childern */
-    rcu_read_lock();
-    QLIST_FOREACH(node, &dir_node->child, sibling) {
-        if (!strcmp(node->name, name)) {
-            break;
-        }
-    }
-    rcu_read_unlock();
-
-    if (!node) {
-        errno = ENOENT;
-        return -1;
-    }
-out:
-    /* Copy the node pointer to fid */
-    target->data = g_malloc(sizeof(void *));
-    memcpy(target->data, &node, sizeof(void *));
-    target->size = sizeof(void *);
-    return 0;
-}
-
-static int v9fs_synth_renameat(FsContext *ctx, V9fsPath *olddir,
-                               const char *old_name, V9fsPath *newdir,
-                               const char *new_name)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_unlinkat(FsContext *ctx, V9fsPath *dir,
-                               const char *name, int flags)
-{
-    errno = EPERM;
-    return -1;
-}
-
-static int v9fs_synth_init(FsContext *ctx)
-{
-    QLIST_INIT(&v9fs_synth_root.child);
-    qemu_mutex_init(&v9fs_synth_mutex);
-
-    /* Add "." and ".." entries for root */
-    v9fs_add_dir_node(&v9fs_synth_root, v9fs_synth_root.attr->mode,
-                      "..", v9fs_synth_root.attr, v9fs_synth_root.attr->inode);
-    v9fs_add_dir_node(&v9fs_synth_root, v9fs_synth_root.attr->mode,
-                      ".", v9fs_synth_root.attr, v9fs_synth_root.attr->inode);
-
-    /* Mark the subsystem is ready for use */
-    v9fs_synth_fs = 1;
-    return 0;
-}
-
-FileOperations synth_ops = {
-    .init         = v9fs_synth_init,
-    .lstat        = v9fs_synth_lstat,
-    .readlink     = v9fs_synth_readlink,
-    .close        = v9fs_synth_close,
-    .closedir     = v9fs_synth_closedir,
-    .open         = v9fs_synth_open,
-    .opendir      = v9fs_synth_opendir,
-    .rewinddir    = v9fs_synth_rewinddir,
-    .telldir      = v9fs_synth_telldir,
-    .readdir_r    = v9fs_synth_readdir_r,
-    .seekdir      = v9fs_synth_seekdir,
-    .preadv       = v9fs_synth_preadv,
-    .pwritev      = v9fs_synth_pwritev,
-    .chmod        = v9fs_synth_chmod,
-    .mknod        = v9fs_synth_mknod,
-    .mkdir        = v9fs_synth_mkdir,
-    .fstat        = v9fs_synth_fstat,
-    .open2        = v9fs_synth_open2,
-    .symlink      = v9fs_synth_symlink,
-    .link         = v9fs_synth_link,
-    .truncate     = v9fs_synth_truncate,
-    .rename       = v9fs_synth_rename,
-    .chown        = v9fs_synth_chown,
-    .utimensat    = v9fs_synth_utimensat,
-    .remove       = v9fs_synth_remove,
-    .fsync        = v9fs_synth_fsync,
-    .statfs       = v9fs_synth_statfs,
-    .lgetxattr    = v9fs_synth_lgetxattr,
-    .llistxattr   = v9fs_synth_llistxattr,
-    .lsetxattr    = v9fs_synth_lsetxattr,
-    .lremovexattr = v9fs_synth_lremovexattr,
-    .name_to_path = v9fs_synth_name_to_path,
-    .renameat     = v9fs_synth_renameat,
-    .unlinkat     = v9fs_synth_unlinkat,
-};
diff --git a/hw/9pfs/virtio-9p-synth.h b/hw/9pfs/virtio-9p-synth.h
deleted file mode 100644
index ab05a8e..0000000
--- a/hw/9pfs/virtio-9p-synth.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Virtio 9p
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-#ifndef HW_9PFS_VIRTIO9P_SYNTH_H
-#define HW_9PFS_VIRTIO9P_SYNTH_H 1
-
-#include <unistd.h>
-#include <sys/types.h>
-#include <limits.h>
-
-typedef struct V9fsSynthNode V9fsSynthNode;
-typedef ssize_t (*v9fs_synth_read)(void *buf, int len, off_t offset,
-                                   void *arg);
-typedef ssize_t (*v9fs_synth_write)(void *buf, int len, off_t offset,
-                                    void *arg);
-typedef struct V9fsSynthNodeAttr {
-    int mode;
-    int inode;
-    int nlink;
-    v9fs_synth_read read;
-    v9fs_synth_write write;
-} V9fsSynthNodeAttr;
-
-struct V9fsSynthNode {
-    QLIST_HEAD(, V9fsSynthNode) child;
-    QLIST_ENTRY(V9fsSynthNode) sibling;
-    char name[NAME_MAX];
-    V9fsSynthNodeAttr *attr;
-    V9fsSynthNodeAttr actual_attr;
-    void *private;
-    int open_count;
-};
-
-typedef struct V9fsSynthOpenState {
-    off_t offset;
-    V9fsSynthNode *node;
-} V9fsSynthOpenState;
-
-extern int qemu_v9fs_synth_mkdir(V9fsSynthNode *parent, int mode,
-                                 const char *name, V9fsSynthNode **result);
-extern int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
-                                    const char *name, v9fs_synth_read read,
-                                    v9fs_synth_write write, void *arg);
-
-#endif
diff --git a/hw/9pfs/virtio-9p-xattr-user.c b/hw/9pfs/virtio-9p-xattr-user.c
deleted file mode 100644
index 46133e0..0000000
--- a/hw/9pfs/virtio-9p-xattr-user.c
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Virtio 9p user. xattr callback
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include <sys/types.h>
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "fsdev/file-op-9p.h"
-#include "virtio-9p-xattr.h"
-
-
-static ssize_t mp_user_getxattr(FsContext *ctx, const char *path,
-                                const char *name, void *value, size_t size)
-{
-    char *buffer;
-    ssize_t ret;
-
-    if (strncmp(name, "user.virtfs.", 12) == 0) {
-        /*
-         * Don't allow fetch of user.virtfs namesapce
-         * in case of mapped security
-         */
-        errno = ENOATTR;
-        return -1;
-    }
-    buffer = rpath(ctx, path);
-    ret = lgetxattr(buffer, name, value, size);
-    g_free(buffer);
-    return ret;
-}
-
-static ssize_t mp_user_listxattr(FsContext *ctx, const char *path,
-                                 char *name, void *value, size_t size)
-{
-    int name_size = strlen(name) + 1;
-    if (strncmp(name, "user.virtfs.", 12) == 0) {
-
-        /*  check if it is a mapped posix acl */
-        if (strncmp(name, "user.virtfs.system.posix_acl_", 29) == 0) {
-            /* adjust the name and size */
-            name += 12;
-            name_size -= 12;
-        } else {
-            /*
-             * Don't allow fetch of user.virtfs namesapce
-             * in case of mapped security
-             */
-            return 0;
-        }
-    }
-    if (!value) {
-        return name_size;
-    }
-
-    if (size < name_size) {
-        errno = ERANGE;
-        return -1;
-    }
-
-    /* name_size includes the trailing NUL. */
-    memcpy(value, name, name_size);
-    return name_size;
-}
-
-static int mp_user_setxattr(FsContext *ctx, const char *path, const char *name,
-                            void *value, size_t size, int flags)
-{
-    char *buffer;
-    int ret;
-
-    if (strncmp(name, "user.virtfs.", 12) == 0) {
-        /*
-         * Don't allow fetch of user.virtfs namesapce
-         * in case of mapped security
-         */
-        errno = EACCES;
-        return -1;
-    }
-    buffer = rpath(ctx, path);
-    ret = lsetxattr(buffer, name, value, size, flags);
-    g_free(buffer);
-    return ret;
-}
-
-static int mp_user_removexattr(FsContext *ctx,
-                               const char *path, const char *name)
-{
-    char *buffer;
-    int ret;
-
-    if (strncmp(name, "user.virtfs.", 12) == 0) {
-        /*
-         * Don't allow fetch of user.virtfs namesapce
-         * in case of mapped security
-         */
-        errno = EACCES;
-        return -1;
-    }
-    buffer = rpath(ctx, path);
-    ret = lremovexattr(buffer, name);
-    g_free(buffer);
-    return ret;
-}
-
-XattrOperations mapped_user_xattr = {
-    .name = "user.",
-    .getxattr = mp_user_getxattr,
-    .setxattr = mp_user_setxattr,
-    .listxattr = mp_user_listxattr,
-    .removexattr = mp_user_removexattr,
-};
-
-XattrOperations passthrough_user_xattr = {
-    .name = "user.",
-    .getxattr = pt_getxattr,
-    .setxattr = pt_setxattr,
-    .listxattr = pt_listxattr,
-    .removexattr = pt_removexattr,
-};
diff --git a/hw/9pfs/virtio-9p-xattr.c b/hw/9pfs/virtio-9p-xattr.c
deleted file mode 100644
index 0718388..0000000
--- a/hw/9pfs/virtio-9p-xattr.c
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Virtio 9p  xattr callback
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- * Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "hw/virtio/virtio.h"
-#include "virtio-9p.h"
-#include "fsdev/file-op-9p.h"
-#include "virtio-9p-xattr.h"
-
-
-static XattrOperations *get_xattr_operations(XattrOperations **h,
-                                             const char *name)
-{
-    XattrOperations *xops;
-    for (xops = *(h)++; xops != NULL; xops = *(h)++) {
-        if (!strncmp(name, xops->name, strlen(xops->name))) {
-            return xops;
-        }
-    }
-    return NULL;
-}
-
-ssize_t v9fs_get_xattr(FsContext *ctx, const char *path,
-                       const char *name, void *value, size_t size)
-{
-    XattrOperations *xops = get_xattr_operations(ctx->xops, name);
-    if (xops) {
-        return xops->getxattr(ctx, path, name, value, size);
-    }
-    errno = EOPNOTSUPP;
-    return -1;
-}
-
-ssize_t pt_listxattr(FsContext *ctx, const char *path,
-                     char *name, void *value, size_t size)
-{
-    int name_size = strlen(name) + 1;
-    if (!value) {
-        return name_size;
-    }
-
-    if (size < name_size) {
-        errno = ERANGE;
-        return -1;
-    }
-
-    /* no need for strncpy: name_size is strlen(name)+1 */
-    memcpy(value, name, name_size);
-    return name_size;
-}
-
-
-/*
- * Get the list and pass to each layer to find out whether
- * to send the data or not
- */
-ssize_t v9fs_list_xattr(FsContext *ctx, const char *path,
-                        void *value, size_t vsize)
-{
-    ssize_t size = 0;
-    char *buffer;
-    void *ovalue = value;
-    XattrOperations *xops;
-    char *orig_value, *orig_value_start;
-    ssize_t xattr_len, parsed_len = 0, attr_len;
-
-    /* Get the actual len */
-    buffer = rpath(ctx, path);
-    xattr_len = llistxattr(buffer, value, 0);
-    if (xattr_len <= 0) {
-        g_free(buffer);
-        return xattr_len;
-    }
-
-    /* Now fetch the xattr and find the actual size */
-    orig_value = g_malloc(xattr_len);
-    xattr_len = llistxattr(buffer, orig_value, xattr_len);
-    g_free(buffer);
-
-    /* store the orig pointer */
-    orig_value_start = orig_value;
-    while (xattr_len > parsed_len) {
-        xops = get_xattr_operations(ctx->xops, orig_value);
-        if (!xops) {
-            goto next_entry;
-        }
-
-        if (!value) {
-            size += xops->listxattr(ctx, path, orig_value, value, vsize);
-        } else {
-            size = xops->listxattr(ctx, path, orig_value, value, vsize);
-            if (size < 0) {
-                goto err_out;
-            }
-            value += size;
-            vsize -= size;
-        }
-next_entry:
-        /* Got the next entry */
-        attr_len = strlen(orig_value) + 1;
-        parsed_len += attr_len;
-        orig_value += attr_len;
-    }
-    if (value) {
-        size = value - ovalue;
-    }
-
-err_out:
-    g_free(orig_value_start);
-    return size;
-}
-
-int v9fs_set_xattr(FsContext *ctx, const char *path, const char *name,
-                   void *value, size_t size, int flags)
-{
-    XattrOperations *xops = get_xattr_operations(ctx->xops, name);
-    if (xops) {
-        return xops->setxattr(ctx, path, name, value, size, flags);
-    }
-    errno = EOPNOTSUPP;
-    return -1;
-
-}
-
-int v9fs_remove_xattr(FsContext *ctx,
-                      const char *path, const char *name)
-{
-    XattrOperations *xops = get_xattr_operations(ctx->xops, name);
-    if (xops) {
-        return xops->removexattr(ctx, path, name);
-    }
-    errno = EOPNOTSUPP;
-    return -1;
-
-}
-
-XattrOperations *mapped_xattr_ops[] = {
-    &mapped_user_xattr,
-    &mapped_pacl_xattr,
-    &mapped_dacl_xattr,
-    NULL,
-};
-
-XattrOperations *passthrough_xattr_ops[] = {
-    &passthrough_user_xattr,
-    &passthrough_acl_xattr,
-    NULL,
-};
-
-/* for .user none model should be same as passthrough */
-XattrOperations *none_xattr_ops[] = {
-    &passthrough_user_xattr,
-    &none_acl_xattr,
-    NULL,
-};
diff --git a/hw/9pfs/virtio-9p-xattr.h b/hw/9pfs/virtio-9p-xattr.h
deleted file mode 100644
index 327b32b..0000000
--- a/hw/9pfs/virtio-9p-xattr.h
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Virtio 9p
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-#ifndef _QEMU_VIRTIO_9P_XATTR_H
-#define _QEMU_VIRTIO_9P_XATTR_H
-
-#include "qemu/xattr.h"
-
-typedef struct xattr_operations
-{
-    const char *name;
-    ssize_t (*getxattr)(FsContext *ctx, const char *path,
-                        const char *name, void *value, size_t size);
-    ssize_t (*listxattr)(FsContext *ctx, const char *path,
-                         char *name, void *value, size_t size);
-    int (*setxattr)(FsContext *ctx, const char *path, const char *name,
-                    void *value, size_t size, int flags);
-    int (*removexattr)(FsContext *ctx,
-                       const char *path, const char *name);
-} XattrOperations;
-
-
-extern XattrOperations mapped_user_xattr;
-extern XattrOperations passthrough_user_xattr;
-
-extern XattrOperations mapped_pacl_xattr;
-extern XattrOperations mapped_dacl_xattr;
-extern XattrOperations passthrough_acl_xattr;
-extern XattrOperations none_acl_xattr;
-
-extern XattrOperations *mapped_xattr_ops[];
-extern XattrOperations *passthrough_xattr_ops[];
-extern XattrOperations *none_xattr_ops[];
-
-ssize_t v9fs_get_xattr(FsContext *ctx, const char *path, const char *name,
-                       void *value, size_t size);
-ssize_t v9fs_list_xattr(FsContext *ctx, const char *path, void *value,
-                        size_t vsize);
-int v9fs_set_xattr(FsContext *ctx, const char *path, const char *name,
-                          void *value, size_t size, int flags);
-int v9fs_remove_xattr(FsContext *ctx, const char *path, const char *name);
-ssize_t pt_listxattr(FsContext *ctx, const char *path, char *name, void *value,
-                     size_t size);
-
-static inline ssize_t pt_getxattr(FsContext *ctx, const char *path,
-                                  const char *name, void *value, size_t size)
-{
-    char *buffer;
-    ssize_t ret;
-
-    buffer = rpath(ctx, path);
-    ret = lgetxattr(buffer, name, value, size);
-    g_free(buffer);
-    return ret;
-}
-
-static inline int pt_setxattr(FsContext *ctx, const char *path,
-                              const char *name, void *value,
-                              size_t size, int flags)
-{
-    char *buffer;
-    int ret;
-
-    buffer = rpath(ctx, path);
-    ret = lsetxattr(buffer, name, value, size, flags);
-    g_free(buffer);
-    return ret;
-}
-
-static inline int pt_removexattr(FsContext *ctx,
-                                 const char *path, const char *name)
-{
-    char *buffer;
-    int ret;
-
-    buffer = rpath(ctx, path);
-    ret = lremovexattr(path, name);
-    g_free(buffer);
-    return ret;
-}
-
-static inline ssize_t notsup_getxattr(FsContext *ctx, const char *path,
-                                      const char *name, void *value,
-                                      size_t size)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-static inline int notsup_setxattr(FsContext *ctx, const char *path,
-                                  const char *name, void *value,
-                                  size_t size, int flags)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-static inline ssize_t notsup_listxattr(FsContext *ctx, const char *path,
-                                       char *name, void *value, size_t size)
-{
-    return 0;
-}
-
-static inline int notsup_removexattr(FsContext *ctx,
-                                     const char *path, const char *name)
-{
-    errno = ENOTSUP;
-    return -1;
-}
-
-#endif
diff --git a/hw/9pfs/virtio-9p.c b/hw/9pfs/virtio-9p.c
deleted file mode 100644
index 5861a5b..0000000
--- a/hw/9pfs/virtio-9p.c
+++ /dev/null
@@ -1,3287 +0,0 @@
-/*
- * Virtio 9p backend
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "hw/virtio/virtio.h"
-#include "hw/i386/pc.h"
-#include "qemu/sockets.h"
-#include "virtio-9p.h"
-#include "fsdev/qemu-fsdev.h"
-#include "virtio-9p-xattr.h"
-#include "virtio-9p-coth.h"
-#include "trace.h"
-#include "migration/migration.h"
-
-int open_fd_hw;
-int total_open_fd;
-static int open_fd_rc;
-
-enum {
-    Oread   = 0x00,
-    Owrite  = 0x01,
-    Ordwr   = 0x02,
-    Oexec   = 0x03,
-    Oexcl   = 0x04,
-    Otrunc  = 0x10,
-    Orexec  = 0x20,
-    Orclose = 0x40,
-    Oappend = 0x80,
-};
-
-static int omode_to_uflags(int8_t mode)
-{
-    int ret = 0;
-
-    switch (mode & 3) {
-    case Oread:
-        ret = O_RDONLY;
-        break;
-    case Ordwr:
-        ret = O_RDWR;
-        break;
-    case Owrite:
-        ret = O_WRONLY;
-        break;
-    case Oexec:
-        ret = O_RDONLY;
-        break;
-    }
-
-    if (mode & Otrunc) {
-        ret |= O_TRUNC;
-    }
-
-    if (mode & Oappend) {
-        ret |= O_APPEND;
-    }
-
-    if (mode & Oexcl) {
-        ret |= O_EXCL;
-    }
-
-    return ret;
-}
-
-struct dotl_openflag_map {
-    int dotl_flag;
-    int open_flag;
-};
-
-static int dotl_to_open_flags(int flags)
-{
-    int i;
-    /*
-     * We have same bits for P9_DOTL_READONLY, P9_DOTL_WRONLY
-     * and P9_DOTL_NOACCESS
-     */
-    int oflags = flags & O_ACCMODE;
-
-    struct dotl_openflag_map dotl_oflag_map[] = {
-        { P9_DOTL_CREATE, O_CREAT },
-        { P9_DOTL_EXCL, O_EXCL },
-        { P9_DOTL_NOCTTY , O_NOCTTY },
-        { P9_DOTL_TRUNC, O_TRUNC },
-        { P9_DOTL_APPEND, O_APPEND },
-        { P9_DOTL_NONBLOCK, O_NONBLOCK } ,
-        { P9_DOTL_DSYNC, O_DSYNC },
-        { P9_DOTL_FASYNC, FASYNC },
-        { P9_DOTL_DIRECT, O_DIRECT },
-        { P9_DOTL_LARGEFILE, O_LARGEFILE },
-        { P9_DOTL_DIRECTORY, O_DIRECTORY },
-        { P9_DOTL_NOFOLLOW, O_NOFOLLOW },
-        { P9_DOTL_NOATIME, O_NOATIME },
-        { P9_DOTL_SYNC, O_SYNC },
-    };
-
-    for (i = 0; i < ARRAY_SIZE(dotl_oflag_map); i++) {
-        if (flags & dotl_oflag_map[i].dotl_flag) {
-            oflags |= dotl_oflag_map[i].open_flag;
-        }
-    }
-
-    return oflags;
-}
-
-void cred_init(FsCred *credp)
-{
-    credp->fc_uid = -1;
-    credp->fc_gid = -1;
-    credp->fc_mode = -1;
-    credp->fc_rdev = -1;
-}
-
-static int get_dotl_openflags(V9fsState *s, int oflags)
-{
-    int flags;
-    /*
-     * Filter the client open flags
-     */
-    flags = dotl_to_open_flags(oflags);
-    flags &= ~(O_NOCTTY | O_ASYNC | O_CREAT);
-    /*
-     * Ignore direct disk access hint until the server supports it.
-     */
-    flags &= ~O_DIRECT;
-    return flags;
-}
-
-void v9fs_path_init(V9fsPath *path)
-{
-    path->data = NULL;
-    path->size = 0;
-}
-
-void v9fs_path_free(V9fsPath *path)
-{
-    g_free(path->data);
-    path->data = NULL;
-    path->size = 0;
-}
-
-void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs)
-{
-    v9fs_path_free(lhs);
-    lhs->data = g_malloc(rhs->size);
-    memcpy(lhs->data, rhs->data, rhs->size);
-    lhs->size = rhs->size;
-}
-
-int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
-                      const char *name, V9fsPath *path)
-{
-    int err;
-    err = s->ops->name_to_path(&s->ctx, dirpath, name, path);
-    if (err < 0) {
-        err = -errno;
-    }
-    return err;
-}
-
-/*
- * Return TRUE if s1 is an ancestor of s2.
- *
- * E.g. "a/b" is an ancestor of "a/b/c" but not of "a/bc/d".
- * As a special case, We treat s1 as ancestor of s2 if they are same!
- */
-static int v9fs_path_is_ancestor(V9fsPath *s1, V9fsPath *s2)
-{
-    if (!strncmp(s1->data, s2->data, s1->size - 1)) {
-        if (s2->data[s1->size - 1] == '\0' || s2->data[s1->size - 1] == '/') {
-            return 1;
-        }
-    }
-    return 0;
-}
-
-static size_t v9fs_string_size(V9fsString *str)
-{
-    return str->size;
-}
-
-/*
- * returns 0 if fid got re-opened, 1 if not, < 0 on error */
-static int v9fs_reopen_fid(V9fsPDU *pdu, V9fsFidState *f)
-{
-    int err = 1;
-    if (f->fid_type == P9_FID_FILE) {
-        if (f->fs.fd == -1) {
-            do {
-                err = v9fs_co_open(pdu, f, f->open_flags);
-            } while (err == -EINTR && !pdu->cancelled);
-        }
-    } else if (f->fid_type == P9_FID_DIR) {
-        if (f->fs.dir == NULL) {
-            do {
-                err = v9fs_co_opendir(pdu, f);
-            } while (err == -EINTR && !pdu->cancelled);
-        }
-    }
-    return err;
-}
-
-static V9fsFidState *get_fid(V9fsPDU *pdu, int32_t fid)
-{
-    int err;
-    V9fsFidState *f;
-    V9fsState *s = pdu->s;
-
-    for (f = s->fid_list; f; f = f->next) {
-        BUG_ON(f->clunked);
-        if (f->fid == fid) {
-            /*
-             * Update the fid ref upfront so that
-             * we don't get reclaimed when we yield
-             * in open later.
-             */
-            f->ref++;
-            /*
-             * check whether we need to reopen the
-             * file. We might have closed the fd
-             * while trying to free up some file
-             * descriptors.
-             */
-            err = v9fs_reopen_fid(pdu, f);
-            if (err < 0) {
-                f->ref--;
-                return NULL;
-            }
-            /*
-             * Mark the fid as referenced so that the LRU
-             * reclaim won't close the file descriptor
-             */
-            f->flags |= FID_REFERENCED;
-            return f;
-        }
-    }
-    return NULL;
-}
-
-static V9fsFidState *alloc_fid(V9fsState *s, int32_t fid)
-{
-    V9fsFidState *f;
-
-    for (f = s->fid_list; f; f = f->next) {
-        /* If fid is already there return NULL */
-        BUG_ON(f->clunked);
-        if (f->fid == fid) {
-            return NULL;
-        }
-    }
-    f = g_malloc0(sizeof(V9fsFidState));
-    f->fid = fid;
-    f->fid_type = P9_FID_NONE;
-    f->ref = 1;
-    /*
-     * Mark the fid as referenced so that the LRU
-     * reclaim won't close the file descriptor
-     */
-    f->flags |= FID_REFERENCED;
-    f->next = s->fid_list;
-    s->fid_list = f;
-
-    return f;
-}
-
-static int v9fs_xattr_fid_clunk(V9fsPDU *pdu, V9fsFidState *fidp)
-{
-    int retval = 0;
-
-    if (fidp->fs.xattr.copied_len == -1) {
-        /* getxattr/listxattr fid */
-        goto free_value;
-    }
-    /*
-     * if this is fid for setxattr. clunk should
-     * result in setxattr localcall
-     */
-    if (fidp->fs.xattr.len != fidp->fs.xattr.copied_len) {
-        /* clunk after partial write */
-        retval = -EINVAL;
-        goto free_out;
-    }
-    if (fidp->fs.xattr.len) {
-        retval = v9fs_co_lsetxattr(pdu, &fidp->path, &fidp->fs.xattr.name,
-                                   fidp->fs.xattr.value,
-                                   fidp->fs.xattr.len,
-                                   fidp->fs.xattr.flags);
-    } else {
-        retval = v9fs_co_lremovexattr(pdu, &fidp->path, &fidp->fs.xattr.name);
-    }
-free_out:
-    v9fs_string_free(&fidp->fs.xattr.name);
-free_value:
-    g_free(fidp->fs.xattr.value);
-    return retval;
-}
-
-static int free_fid(V9fsPDU *pdu, V9fsFidState *fidp)
-{
-    int retval = 0;
-
-    if (fidp->fid_type == P9_FID_FILE) {
-        /* If we reclaimed the fd no need to close */
-        if (fidp->fs.fd != -1) {
-            retval = v9fs_co_close(pdu, &fidp->fs);
-        }
-    } else if (fidp->fid_type == P9_FID_DIR) {
-        if (fidp->fs.dir != NULL) {
-            retval = v9fs_co_closedir(pdu, &fidp->fs);
-        }
-    } else if (fidp->fid_type == P9_FID_XATTR) {
-        retval = v9fs_xattr_fid_clunk(pdu, fidp);
-    }
-    v9fs_path_free(&fidp->path);
-    g_free(fidp);
-    return retval;
-}
-
-static int put_fid(V9fsPDU *pdu, V9fsFidState *fidp)
-{
-    BUG_ON(!fidp->ref);
-    fidp->ref--;
-    /*
-     * Don't free the fid if it is in reclaim list
-     */
-    if (!fidp->ref && fidp->clunked) {
-        if (fidp->fid == pdu->s->root_fid) {
-            /*
-             * if the clunked fid is root fid then we
-             * have unmounted the fs on the client side.
-             * delete the migration blocker. Ideally, this
-             * should be hooked to transport close notification
-             */
-            if (pdu->s->migration_blocker) {
-                migrate_del_blocker(pdu->s->migration_blocker);
-                error_free(pdu->s->migration_blocker);
-                pdu->s->migration_blocker = NULL;
-            }
-        }
-        return free_fid(pdu, fidp);
-    }
-    return 0;
-}
-
-static V9fsFidState *clunk_fid(V9fsState *s, int32_t fid)
-{
-    V9fsFidState **fidpp, *fidp;
-
-    for (fidpp = &s->fid_list; *fidpp; fidpp = &(*fidpp)->next) {
-        if ((*fidpp)->fid == fid) {
-            break;
-        }
-    }
-    if (*fidpp == NULL) {
-        return NULL;
-    }
-    fidp = *fidpp;
-    *fidpp = fidp->next;
-    fidp->clunked = 1;
-    return fidp;
-}
-
-void v9fs_reclaim_fd(V9fsPDU *pdu)
-{
-    int reclaim_count = 0;
-    V9fsState *s = pdu->s;
-    V9fsFidState *f, *reclaim_list = NULL;
-
-    for (f = s->fid_list; f; f = f->next) {
-        /*
-         * Unlink fids cannot be reclaimed. Check
-         * for them and skip them. Also skip fids
-         * currently being operated on.
-         */
-        if (f->ref || f->flags & FID_NON_RECLAIMABLE) {
-            continue;
-        }
-        /*
-         * if it is a recently referenced fid
-         * we leave the fid untouched and clear the
-         * reference bit. We come back to it later
-         * in the next iteration. (a simple LRU without
-         * moving list elements around)
-         */
-        if (f->flags & FID_REFERENCED) {
-            f->flags &= ~FID_REFERENCED;
-            continue;
-        }
-        /*
-         * Add fids to reclaim list.
-         */
-        if (f->fid_type == P9_FID_FILE) {
-            if (f->fs.fd != -1) {
-                /*
-                 * Up the reference count so that
-                 * a clunk request won't free this fid
-                 */
-                f->ref++;
-                f->rclm_lst = reclaim_list;
-                reclaim_list = f;
-                f->fs_reclaim.fd = f->fs.fd;
-                f->fs.fd = -1;
-                reclaim_count++;
-            }
-        } else if (f->fid_type == P9_FID_DIR) {
-            if (f->fs.dir != NULL) {
-                /*
-                 * Up the reference count so that
-                 * a clunk request won't free this fid
-                 */
-                f->ref++;
-                f->rclm_lst = reclaim_list;
-                reclaim_list = f;
-                f->fs_reclaim.dir = f->fs.dir;
-                f->fs.dir = NULL;
-                reclaim_count++;
-            }
-        }
-        if (reclaim_count >= open_fd_rc) {
-            break;
-        }
-    }
-    /*
-     * Now close the fid in reclaim list. Free them if they
-     * are already clunked.
-     */
-    while (reclaim_list) {
-        f = reclaim_list;
-        reclaim_list = f->rclm_lst;
-        if (f->fid_type == P9_FID_FILE) {
-            v9fs_co_close(pdu, &f->fs_reclaim);
-        } else if (f->fid_type == P9_FID_DIR) {
-            v9fs_co_closedir(pdu, &f->fs_reclaim);
-        }
-        f->rclm_lst = NULL;
-        /*
-         * Now drop the fid reference, free it
-         * if clunked.
-         */
-        put_fid(pdu, f);
-    }
-}
-
-static int v9fs_mark_fids_unreclaim(V9fsPDU *pdu, V9fsPath *path)
-{
-    int err;
-    V9fsState *s = pdu->s;
-    V9fsFidState *fidp, head_fid;
-
-    head_fid.next = s->fid_list;
-    for (fidp = s->fid_list; fidp; fidp = fidp->next) {
-        if (fidp->path.size != path->size) {
-            continue;
-        }
-        if (!memcmp(fidp->path.data, path->data, path->size)) {
-            /* Mark the fid non reclaimable. */
-            fidp->flags |= FID_NON_RECLAIMABLE;
-
-            /* reopen the file/dir if already closed */
-            err = v9fs_reopen_fid(pdu, fidp);
-            if (err < 0) {
-                return -1;
-            }
-            /*
-             * Go back to head of fid list because
-             * the list could have got updated when
-             * switched to the worker thread
-             */
-            if (err == 0) {
-                fidp = &head_fid;
-            }
-        }
-    }
-    return 0;
-}
-
-static void virtfs_reset(V9fsPDU *pdu)
-{
-    V9fsState *s = pdu->s;
-    V9fsFidState *fidp = NULL;
-
-    /* Free all fids */
-    while (s->fid_list) {
-        fidp = s->fid_list;
-        s->fid_list = fidp->next;
-
-        if (fidp->ref) {
-            fidp->clunked = 1;
-        } else {
-            free_fid(pdu, fidp);
-        }
-    }
-    if (fidp) {
-        /* One or more unclunked fids found... */
-        error_report("9pfs:%s: One or more uncluncked fids "
-                     "found during reset", __func__);
-    }
-}
-
-#define P9_QID_TYPE_DIR         0x80
-#define P9_QID_TYPE_SYMLINK     0x02
-
-#define P9_STAT_MODE_DIR        0x80000000
-#define P9_STAT_MODE_APPEND     0x40000000
-#define P9_STAT_MODE_EXCL       0x20000000
-#define P9_STAT_MODE_MOUNT      0x10000000
-#define P9_STAT_MODE_AUTH       0x08000000
-#define P9_STAT_MODE_TMP        0x04000000
-#define P9_STAT_MODE_SYMLINK    0x02000000
-#define P9_STAT_MODE_LINK       0x01000000
-#define P9_STAT_MODE_DEVICE     0x00800000
-#define P9_STAT_MODE_NAMED_PIPE 0x00200000
-#define P9_STAT_MODE_SOCKET     0x00100000
-#define P9_STAT_MODE_SETUID     0x00080000
-#define P9_STAT_MODE_SETGID     0x00040000
-#define P9_STAT_MODE_SETVTX     0x00010000
-
-#define P9_STAT_MODE_TYPE_BITS (P9_STAT_MODE_DIR |          \
-                                P9_STAT_MODE_SYMLINK |      \
-                                P9_STAT_MODE_LINK |         \
-                                P9_STAT_MODE_DEVICE |       \
-                                P9_STAT_MODE_NAMED_PIPE |   \
-                                P9_STAT_MODE_SOCKET)
-
-/* This is the algorithm from ufs in spfs */
-static void stat_to_qid(const struct stat *stbuf, V9fsQID *qidp)
-{
-    size_t size;
-
-    memset(&qidp->path, 0, sizeof(qidp->path));
-    size = MIN(sizeof(stbuf->st_ino), sizeof(qidp->path));
-    memcpy(&qidp->path, &stbuf->st_ino, size);
-    qidp->version = stbuf->st_mtime ^ (stbuf->st_size << 8);
-    qidp->type = 0;
-    if (S_ISDIR(stbuf->st_mode)) {
-        qidp->type |= P9_QID_TYPE_DIR;
-    }
-    if (S_ISLNK(stbuf->st_mode)) {
-        qidp->type |= P9_QID_TYPE_SYMLINK;
-    }
-}
-
-static int fid_to_qid(V9fsPDU *pdu, V9fsFidState *fidp, V9fsQID *qidp)
-{
-    struct stat stbuf;
-    int err;
-
-    err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
-    if (err < 0) {
-        return err;
-    }
-    stat_to_qid(&stbuf, qidp);
-    return 0;
-}
-
-static V9fsPDU *alloc_pdu(V9fsState *s)
-{
-    V9fsPDU *pdu = NULL;
-
-    if (!QLIST_EMPTY(&s->free_list)) {
-        pdu = QLIST_FIRST(&s->free_list);
-        QLIST_REMOVE(pdu, next);
-        QLIST_INSERT_HEAD(&s->active_list, pdu, next);
-    }
-    return pdu;
-}
-
-static void free_pdu(V9fsState *s, V9fsPDU *pdu)
-{
-    if (pdu) {
-        /*
-         * Cancelled pdu are added back to the freelist
-         * by flush request .
-         */
-        if (!pdu->cancelled) {
-            QLIST_REMOVE(pdu, next);
-            QLIST_INSERT_HEAD(&s->free_list, pdu, next);
-        }
-    }
-}
-
-/*
- * We don't do error checking for pdu_marshal/unmarshal here
- * because we always expect to have enough space to encode
- * error details
- */
-static void complete_pdu(V9fsState *s, V9fsPDU *pdu, ssize_t len)
-{
-    int8_t id = pdu->id + 1; /* Response */
-
-    if (len < 0) {
-        int err = -len;
-        len = 7;
-
-        if (s->proto_version != V9FS_PROTO_2000L) {
-            V9fsString str;
-
-            str.data = strerror(err);
-            str.size = strlen(str.data);
-
-            len += pdu_marshal(pdu, len, "s", &str);
-            id = P9_RERROR;
-        }
-
-        len += pdu_marshal(pdu, len, "d", err);
-
-        if (s->proto_version == V9FS_PROTO_2000L) {
-            id = P9_RLERROR;
-        }
-        trace_v9fs_rerror(pdu->tag, pdu->id, err); /* Trace ERROR */
-    }
-
-    /* fill out the header */
-    pdu_marshal(pdu, 0, "dbw", (int32_t)len, id, pdu->tag);
-
-    /* keep these in sync */
-    pdu->size = len;
-    pdu->id = id;
-
-    /* push onto queue and notify */
-    virtqueue_push(s->vq, &pdu->elem, len);
-
-    /* FIXME: we should batch these completions */
-    virtio_notify(VIRTIO_DEVICE(s), s->vq);
-
-    /* Now wakeup anybody waiting in flush for this request */
-    qemu_co_queue_next(&pdu->complete);
-
-    free_pdu(s, pdu);
-}
-
-static mode_t v9mode_to_mode(uint32_t mode, V9fsString *extension)
-{
-    mode_t ret;
-
-    ret = mode & 0777;
-    if (mode & P9_STAT_MODE_DIR) {
-        ret |= S_IFDIR;
-    }
-
-    if (mode & P9_STAT_MODE_SYMLINK) {
-        ret |= S_IFLNK;
-    }
-    if (mode & P9_STAT_MODE_SOCKET) {
-        ret |= S_IFSOCK;
-    }
-    if (mode & P9_STAT_MODE_NAMED_PIPE) {
-        ret |= S_IFIFO;
-    }
-    if (mode & P9_STAT_MODE_DEVICE) {
-        if (extension->size && extension->data[0] == 'c') {
-            ret |= S_IFCHR;
-        } else {
-            ret |= S_IFBLK;
-        }
-    }
-
-    if (!(ret&~0777)) {
-        ret |= S_IFREG;
-    }
-
-    if (mode & P9_STAT_MODE_SETUID) {
-        ret |= S_ISUID;
-    }
-    if (mode & P9_STAT_MODE_SETGID) {
-        ret |= S_ISGID;
-    }
-    if (mode & P9_STAT_MODE_SETVTX) {
-        ret |= S_ISVTX;
-    }
-
-    return ret;
-}
-
-static int donttouch_stat(V9fsStat *stat)
-{
-    if (stat->type == -1 &&
-        stat->dev == -1 &&
-        stat->qid.type == -1 &&
-        stat->qid.version == -1 &&
-        stat->qid.path == -1 &&
-        stat->mode == -1 &&
-        stat->atime == -1 &&
-        stat->mtime == -1 &&
-        stat->length == -1 &&
-        !stat->name.size &&
-        !stat->uid.size &&
-        !stat->gid.size &&
-        !stat->muid.size &&
-        stat->n_uid == -1 &&
-        stat->n_gid == -1 &&
-        stat->n_muid == -1) {
-        return 1;
-    }
-
-    return 0;
-}
-
-static void v9fs_stat_init(V9fsStat *stat)
-{
-    v9fs_string_init(&stat->name);
-    v9fs_string_init(&stat->uid);
-    v9fs_string_init(&stat->gid);
-    v9fs_string_init(&stat->muid);
-    v9fs_string_init(&stat->extension);
-}
-
-static void v9fs_stat_free(V9fsStat *stat)
-{
-    v9fs_string_free(&stat->name);
-    v9fs_string_free(&stat->uid);
-    v9fs_string_free(&stat->gid);
-    v9fs_string_free(&stat->muid);
-    v9fs_string_free(&stat->extension);
-}
-
-static uint32_t stat_to_v9mode(const struct stat *stbuf)
-{
-    uint32_t mode;
-
-    mode = stbuf->st_mode & 0777;
-    if (S_ISDIR(stbuf->st_mode)) {
-        mode |= P9_STAT_MODE_DIR;
-    }
-
-    if (S_ISLNK(stbuf->st_mode)) {
-        mode |= P9_STAT_MODE_SYMLINK;
-    }
-
-    if (S_ISSOCK(stbuf->st_mode)) {
-        mode |= P9_STAT_MODE_SOCKET;
-    }
-
-    if (S_ISFIFO(stbuf->st_mode)) {
-        mode |= P9_STAT_MODE_NAMED_PIPE;
-    }
-
-    if (S_ISBLK(stbuf->st_mode) || S_ISCHR(stbuf->st_mode)) {
-        mode |= P9_STAT_MODE_DEVICE;
-    }
-
-    if (stbuf->st_mode & S_ISUID) {
-        mode |= P9_STAT_MODE_SETUID;
-    }
-
-    if (stbuf->st_mode & S_ISGID) {
-        mode |= P9_STAT_MODE_SETGID;
-    }
-
-    if (stbuf->st_mode & S_ISVTX) {
-        mode |= P9_STAT_MODE_SETVTX;
-    }
-
-    return mode;
-}
-
-static int stat_to_v9stat(V9fsPDU *pdu, V9fsPath *name,
-                            const struct stat *stbuf,
-                            V9fsStat *v9stat)
-{
-    int err;
-    const char *str;
-
-    memset(v9stat, 0, sizeof(*v9stat));
-
-    stat_to_qid(stbuf, &v9stat->qid);
-    v9stat->mode = stat_to_v9mode(stbuf);
-    v9stat->atime = stbuf->st_atime;
-    v9stat->mtime = stbuf->st_mtime;
-    v9stat->length = stbuf->st_size;
-
-    v9fs_string_null(&v9stat->uid);
-    v9fs_string_null(&v9stat->gid);
-    v9fs_string_null(&v9stat->muid);
-
-    v9stat->n_uid = stbuf->st_uid;
-    v9stat->n_gid = stbuf->st_gid;
-    v9stat->n_muid = 0;
-
-    v9fs_string_null(&v9stat->extension);
-
-    if (v9stat->mode & P9_STAT_MODE_SYMLINK) {
-        err = v9fs_co_readlink(pdu, name, &v9stat->extension);
-        if (err < 0) {
-            return err;
-        }
-    } else if (v9stat->mode & P9_STAT_MODE_DEVICE) {
-        v9fs_string_sprintf(&v9stat->extension, "%c %u %u",
-                S_ISCHR(stbuf->st_mode) ? 'c' : 'b',
-                major(stbuf->st_rdev), minor(stbuf->st_rdev));
-    } else if (S_ISDIR(stbuf->st_mode) || S_ISREG(stbuf->st_mode)) {
-        v9fs_string_sprintf(&v9stat->extension, "%s %lu",
-                "HARDLINKCOUNT", (unsigned long)stbuf->st_nlink);
-    }
-
-    str = strrchr(name->data, '/');
-    if (str) {
-        str += 1;
-    } else {
-        str = name->data;
-    }
-
-    v9fs_string_sprintf(&v9stat->name, "%s", str);
-
-    v9stat->size = 61 +
-        v9fs_string_size(&v9stat->name) +
-        v9fs_string_size(&v9stat->uid) +
-        v9fs_string_size(&v9stat->gid) +
-        v9fs_string_size(&v9stat->muid) +
-        v9fs_string_size(&v9stat->extension);
-    return 0;
-}
-
-#define P9_STATS_MODE          0x00000001ULL
-#define P9_STATS_NLINK         0x00000002ULL
-#define P9_STATS_UID           0x00000004ULL
-#define P9_STATS_GID           0x00000008ULL
-#define P9_STATS_RDEV          0x00000010ULL
-#define P9_STATS_ATIME         0x00000020ULL
-#define P9_STATS_MTIME         0x00000040ULL
-#define P9_STATS_CTIME         0x00000080ULL
-#define P9_STATS_INO           0x00000100ULL
-#define P9_STATS_SIZE          0x00000200ULL
-#define P9_STATS_BLOCKS        0x00000400ULL
-
-#define P9_STATS_BTIME         0x00000800ULL
-#define P9_STATS_GEN           0x00001000ULL
-#define P9_STATS_DATA_VERSION  0x00002000ULL
-
-#define P9_STATS_BASIC         0x000007ffULL /* Mask for fields up to BLOCKS */
-#define P9_STATS_ALL           0x00003fffULL /* Mask for All fields above */
-
-
-static void stat_to_v9stat_dotl(V9fsState *s, const struct stat *stbuf,
-                                V9fsStatDotl *v9lstat)
-{
-    memset(v9lstat, 0, sizeof(*v9lstat));
-
-    v9lstat->st_mode = stbuf->st_mode;
-    v9lstat->st_nlink = stbuf->st_nlink;
-    v9lstat->st_uid = stbuf->st_uid;
-    v9lstat->st_gid = stbuf->st_gid;
-    v9lstat->st_rdev = stbuf->st_rdev;
-    v9lstat->st_size = stbuf->st_size;
-    v9lstat->st_blksize = stbuf->st_blksize;
-    v9lstat->st_blocks = stbuf->st_blocks;
-    v9lstat->st_atime_sec = stbuf->st_atime;
-    v9lstat->st_atime_nsec = stbuf->st_atim.tv_nsec;
-    v9lstat->st_mtime_sec = stbuf->st_mtime;
-    v9lstat->st_mtime_nsec = stbuf->st_mtim.tv_nsec;
-    v9lstat->st_ctime_sec = stbuf->st_ctime;
-    v9lstat->st_ctime_nsec = stbuf->st_ctim.tv_nsec;
-    /* Currently we only support BASIC fields in stat */
-    v9lstat->st_result_mask = P9_STATS_BASIC;
-
-    stat_to_qid(stbuf, &v9lstat->qid);
-}
-
-static void print_sg(struct iovec *sg, int cnt)
-{
-    int i;
-
-    printf("sg[%d]: {", cnt);
-    for (i = 0; i < cnt; i++) {
-        if (i) {
-            printf(", ");
-        }
-        printf("(%p, %zd)", sg[i].iov_base, sg[i].iov_len);
-    }
-    printf("}\n");
-}
-
-/* Will call this only for path name based fid */
-static void v9fs_fix_path(V9fsPath *dst, V9fsPath *src, int len)
-{
-    V9fsPath str;
-    v9fs_path_init(&str);
-    v9fs_path_copy(&str, dst);
-    v9fs_string_sprintf((V9fsString *)dst, "%s%s", src->data, str.data+len);
-    v9fs_path_free(&str);
-    /* +1 to include terminating NULL */
-    dst->size++;
-}
-
-static inline bool is_ro_export(FsContext *ctx)
-{
-    return ctx->export_flags & V9FS_RDONLY;
-}
-
-static void v9fs_version(void *opaque)
-{
-    ssize_t err;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-    V9fsString version;
-    size_t offset = 7;
-
-    v9fs_string_init(&version);
-    err = pdu_unmarshal(pdu, offset, "ds", &s->msize, &version);
-    if (err < 0) {
-        offset = err;
-        goto out;
-    }
-    trace_v9fs_version(pdu->tag, pdu->id, s->msize, version.data);
-
-    virtfs_reset(pdu);
-
-    if (!strcmp(version.data, "9P2000.u")) {
-        s->proto_version = V9FS_PROTO_2000U;
-    } else if (!strcmp(version.data, "9P2000.L")) {
-        s->proto_version = V9FS_PROTO_2000L;
-    } else {
-        v9fs_string_sprintf(&version, "unknown");
-    }
-
-    err = pdu_marshal(pdu, offset, "ds", s->msize, &version);
-    if (err < 0) {
-        offset = err;
-        goto out;
-    }
-    offset += err;
-    trace_v9fs_version_return(pdu->tag, pdu->id, s->msize, version.data);
-out:
-    complete_pdu(s, pdu, offset);
-    v9fs_string_free(&version);
-}
-
-static void v9fs_attach(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-    int32_t fid, afid, n_uname;
-    V9fsString uname, aname;
-    V9fsFidState *fidp;
-    size_t offset = 7;
-    V9fsQID qid;
-    ssize_t err;
-
-    v9fs_string_init(&uname);
-    v9fs_string_init(&aname);
-    err = pdu_unmarshal(pdu, offset, "ddssd", &fid,
-                        &afid, &uname, &aname, &n_uname);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_attach(pdu->tag, pdu->id, fid, afid, uname.data, aname.data);
-
-    fidp = alloc_fid(s, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    fidp->uid = n_uname;
-    err = v9fs_co_name_to_path(pdu, NULL, "/", &fidp->path);
-    if (err < 0) {
-        err = -EINVAL;
-        clunk_fid(s, fid);
-        goto out;
-    }
-    err = fid_to_qid(pdu, fidp, &qid);
-    if (err < 0) {
-        err = -EINVAL;
-        clunk_fid(s, fid);
-        goto out;
-    }
-    err = pdu_marshal(pdu, offset, "Q", &qid);
-    if (err < 0) {
-        clunk_fid(s, fid);
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_attach_return(pdu->tag, pdu->id,
-                             qid.type, qid.version, qid.path);
-    /*
-     * disable migration if we haven't done already.
-     * attach could get called multiple times for the same export.
-     */
-    if (!s->migration_blocker) {
-        s->root_fid = fid;
-        error_setg(&s->migration_blocker,
-                   "Migration is disabled when VirtFS export path '%s' is mounted in the guest using mount_tag '%s'",
-                   s->ctx.fs_root ? s->ctx.fs_root : "NULL", s->tag);
-        migrate_add_blocker(s->migration_blocker);
-    }
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&uname);
-    v9fs_string_free(&aname);
-}
-
-static void v9fs_stat(void *opaque)
-{
-    int32_t fid;
-    V9fsStat v9stat;
-    ssize_t err = 0;
-    size_t offset = 7;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "d", &fid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_stat(pdu->tag, pdu->id, fid);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    err = stat_to_v9stat(pdu, &fidp->path, &stbuf, &v9stat);
-    if (err < 0) {
-        goto out;
-    }
-    err = pdu_marshal(pdu, offset, "wS", 0, &v9stat);
-    if (err < 0) {
-        v9fs_stat_free(&v9stat);
-        goto out;
-    }
-    trace_v9fs_stat_return(pdu->tag, pdu->id, v9stat.mode,
-                           v9stat.atime, v9stat.mtime, v9stat.length);
-    err += offset;
-    v9fs_stat_free(&v9stat);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-}
-
-static void v9fs_getattr(void *opaque)
-{
-    int32_t fid;
-    size_t offset = 7;
-    ssize_t retval = 0;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    uint64_t request_mask;
-    V9fsStatDotl v9stat_dotl;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    retval = pdu_unmarshal(pdu, offset, "dq", &fid, &request_mask);
-    if (retval < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_getattr(pdu->tag, pdu->id, fid, request_mask);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        retval = -ENOENT;
-        goto out_nofid;
-    }
-    /*
-     * Currently we only support BASIC fields in stat, so there is no
-     * need to look at request_mask.
-     */
-    retval = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
-    if (retval < 0) {
-        goto out;
-    }
-    stat_to_v9stat_dotl(s, &stbuf, &v9stat_dotl);
-
-    /*  fill st_gen if requested and supported by underlying fs */
-    if (request_mask & P9_STATS_GEN) {
-        retval = v9fs_co_st_gen(pdu, &fidp->path, stbuf.st_mode, &v9stat_dotl);
-        switch (retval) {
-        case 0:
-            /* we have valid st_gen: update result mask */
-            v9stat_dotl.st_result_mask |= P9_STATS_GEN;
-            break;
-        case -EINTR:
-            /* request cancelled, e.g. by Tflush */
-            goto out;
-        default:
-            /* failed to get st_gen: not fatal, ignore */
-            break;
-        }
-    }
-    retval = pdu_marshal(pdu, offset, "A", &v9stat_dotl);
-    if (retval < 0) {
-        goto out;
-    }
-    retval += offset;
-    trace_v9fs_getattr_return(pdu->tag, pdu->id, v9stat_dotl.st_result_mask,
-                              v9stat_dotl.st_mode, v9stat_dotl.st_uid,
-                              v9stat_dotl.st_gid);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, retval);
-}
-
-/* Attribute flags */
-#define P9_ATTR_MODE       (1 << 0)
-#define P9_ATTR_UID        (1 << 1)
-#define P9_ATTR_GID        (1 << 2)
-#define P9_ATTR_SIZE       (1 << 3)
-#define P9_ATTR_ATIME      (1 << 4)
-#define P9_ATTR_MTIME      (1 << 5)
-#define P9_ATTR_CTIME      (1 << 6)
-#define P9_ATTR_ATIME_SET  (1 << 7)
-#define P9_ATTR_MTIME_SET  (1 << 8)
-
-#define P9_ATTR_MASK    127
-
-static void v9fs_setattr(void *opaque)
-{
-    int err = 0;
-    int32_t fid;
-    V9fsFidState *fidp;
-    size_t offset = 7;
-    V9fsIattr v9iattr;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "dI", &fid, &v9iattr);
-    if (err < 0) {
-        goto out_nofid;
-    }
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    if (v9iattr.valid & P9_ATTR_MODE) {
-        err = v9fs_co_chmod(pdu, &fidp->path, v9iattr.mode);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    if (v9iattr.valid & (P9_ATTR_ATIME | P9_ATTR_MTIME)) {
-        struct timespec times[2];
-        if (v9iattr.valid & P9_ATTR_ATIME) {
-            if (v9iattr.valid & P9_ATTR_ATIME_SET) {
-                times[0].tv_sec = v9iattr.atime_sec;
-                times[0].tv_nsec = v9iattr.atime_nsec;
-            } else {
-                times[0].tv_nsec = UTIME_NOW;
-            }
-        } else {
-            times[0].tv_nsec = UTIME_OMIT;
-        }
-        if (v9iattr.valid & P9_ATTR_MTIME) {
-            if (v9iattr.valid & P9_ATTR_MTIME_SET) {
-                times[1].tv_sec = v9iattr.mtime_sec;
-                times[1].tv_nsec = v9iattr.mtime_nsec;
-            } else {
-                times[1].tv_nsec = UTIME_NOW;
-            }
-        } else {
-            times[1].tv_nsec = UTIME_OMIT;
-        }
-        err = v9fs_co_utimensat(pdu, &fidp->path, times);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    /*
-     * If the only valid entry in iattr is ctime we can call
-     * chown(-1,-1) to update the ctime of the file
-     */
-    if ((v9iattr.valid & (P9_ATTR_UID | P9_ATTR_GID)) ||
-        ((v9iattr.valid & P9_ATTR_CTIME)
-         && !((v9iattr.valid & P9_ATTR_MASK) & ~P9_ATTR_CTIME))) {
-        if (!(v9iattr.valid & P9_ATTR_UID)) {
-            v9iattr.uid = -1;
-        }
-        if (!(v9iattr.valid & P9_ATTR_GID)) {
-            v9iattr.gid = -1;
-        }
-        err = v9fs_co_chown(pdu, &fidp->path, v9iattr.uid,
-                            v9iattr.gid);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    if (v9iattr.valid & (P9_ATTR_SIZE)) {
-        err = v9fs_co_truncate(pdu, &fidp->path, v9iattr.size);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    err = offset;
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-}
-
-static int v9fs_walk_marshal(V9fsPDU *pdu, uint16_t nwnames, V9fsQID *qids)
-{
-    int i;
-    ssize_t err;
-    size_t offset = 7;
-
-    err = pdu_marshal(pdu, offset, "w", nwnames);
-    if (err < 0) {
-        return err;
-    }
-    offset += err;
-    for (i = 0; i < nwnames; i++) {
-        err = pdu_marshal(pdu, offset, "Q", &qids[i]);
-        if (err < 0) {
-            return err;
-        }
-        offset += err;
-    }
-    return offset;
-}
-
-static void v9fs_walk(void *opaque)
-{
-    int name_idx;
-    V9fsQID *qids = NULL;
-    int i, err = 0;
-    V9fsPath dpath, path;
-    uint16_t nwnames;
-    struct stat stbuf;
-    size_t offset = 7;
-    int32_t fid, newfid;
-    V9fsString *wnames = NULL;
-    V9fsFidState *fidp;
-    V9fsFidState *newfidp = NULL;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "ddw", &fid, &newfid, &nwnames);
-    if (err < 0) {
-        complete_pdu(s, pdu, err);
-        return ;
-    }
-    offset += err;
-
-    trace_v9fs_walk(pdu->tag, pdu->id, fid, newfid, nwnames);
-
-    if (nwnames && nwnames <= P9_MAXWELEM) {
-        wnames = g_malloc0(sizeof(wnames[0]) * nwnames);
-        qids   = g_malloc0(sizeof(qids[0]) * nwnames);
-        for (i = 0; i < nwnames; i++) {
-            err = pdu_unmarshal(pdu, offset, "s", &wnames[i]);
-            if (err < 0) {
-                goto out_nofid;
-            }
-            offset += err;
-        }
-    } else if (nwnames > P9_MAXWELEM) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    v9fs_path_init(&dpath);
-    v9fs_path_init(&path);
-    /*
-     * Both dpath and path initially poin to fidp.
-     * Needed to handle request with nwnames == 0
-     */
-    v9fs_path_copy(&dpath, &fidp->path);
-    v9fs_path_copy(&path, &fidp->path);
-    for (name_idx = 0; name_idx < nwnames; name_idx++) {
-        err = v9fs_co_name_to_path(pdu, &dpath, wnames[name_idx].data, &path);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_lstat(pdu, &path, &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        stat_to_qid(&stbuf, &qids[name_idx]);
-        v9fs_path_copy(&dpath, &path);
-    }
-    if (fid == newfid) {
-        BUG_ON(fidp->fid_type != P9_FID_NONE);
-        v9fs_path_copy(&fidp->path, &path);
-    } else {
-        newfidp = alloc_fid(s, newfid);
-        if (newfidp == NULL) {
-            err = -EINVAL;
-            goto out;
-        }
-        newfidp->uid = fidp->uid;
-        v9fs_path_copy(&newfidp->path, &path);
-    }
-    err = v9fs_walk_marshal(pdu, nwnames, qids);
-    trace_v9fs_walk_return(pdu->tag, pdu->id, nwnames, qids);
-out:
-    put_fid(pdu, fidp);
-    if (newfidp) {
-        put_fid(pdu, newfidp);
-    }
-    v9fs_path_free(&dpath);
-    v9fs_path_free(&path);
-out_nofid:
-    complete_pdu(s, pdu, err);
-    if (nwnames && nwnames <= P9_MAXWELEM) {
-        for (name_idx = 0; name_idx < nwnames; name_idx++) {
-            v9fs_string_free(&wnames[name_idx]);
-        }
-        g_free(wnames);
-        g_free(qids);
-    }
-}
-
-static int32_t get_iounit(V9fsPDU *pdu, V9fsPath *path)
-{
-    struct statfs stbuf;
-    int32_t iounit = 0;
-    V9fsState *s = pdu->s;
-
-    /*
-     * iounit should be multiples of f_bsize (host filesystem block size
-     * and as well as less than (client msize - P9_IOHDRSZ))
-     */
-    if (!v9fs_co_statfs(pdu, path, &stbuf)) {
-        iounit = stbuf.f_bsize;
-        iounit *= (s->msize - P9_IOHDRSZ)/stbuf.f_bsize;
-    }
-    if (!iounit) {
-        iounit = s->msize - P9_IOHDRSZ;
-    }
-    return iounit;
-}
-
-static void v9fs_open(void *opaque)
-{
-    int flags;
-    int32_t fid;
-    int32_t mode;
-    V9fsQID qid;
-    int iounit = 0;
-    ssize_t err = 0;
-    size_t offset = 7;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    if (s->proto_version == V9FS_PROTO_2000L) {
-        err = pdu_unmarshal(pdu, offset, "dd", &fid, &mode);
-    } else {
-        uint8_t modebyte;
-        err = pdu_unmarshal(pdu, offset, "db", &fid, &modebyte);
-        mode = modebyte;
-    }
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_open(pdu->tag, pdu->id, fid, mode);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    BUG_ON(fidp->fid_type != P9_FID_NONE);
-
-    err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    stat_to_qid(&stbuf, &qid);
-    if (S_ISDIR(stbuf.st_mode)) {
-        err = v9fs_co_opendir(pdu, fidp);
-        if (err < 0) {
-            goto out;
-        }
-        fidp->fid_type = P9_FID_DIR;
-        err = pdu_marshal(pdu, offset, "Qd", &qid, 0);
-        if (err < 0) {
-            goto out;
-        }
-        err += offset;
-    } else {
-        if (s->proto_version == V9FS_PROTO_2000L) {
-            flags = get_dotl_openflags(s, mode);
-        } else {
-            flags = omode_to_uflags(mode);
-        }
-        if (is_ro_export(&s->ctx)) {
-            if (mode & O_WRONLY || mode & O_RDWR ||
-                mode & O_APPEND || mode & O_TRUNC) {
-                err = -EROFS;
-                goto out;
-            }
-        }
-        err = v9fs_co_open(pdu, fidp, flags);
-        if (err < 0) {
-            goto out;
-        }
-        fidp->fid_type = P9_FID_FILE;
-        fidp->open_flags = flags;
-        if (flags & O_EXCL) {
-            /*
-             * We let the host file system do O_EXCL check
-             * We should not reclaim such fd
-             */
-            fidp->flags |= FID_NON_RECLAIMABLE;
-        }
-        iounit = get_iounit(pdu, &fidp->path);
-        err = pdu_marshal(pdu, offset, "Qd", &qid, iounit);
-        if (err < 0) {
-            goto out;
-        }
-        err += offset;
-    }
-    trace_v9fs_open_return(pdu->tag, pdu->id,
-                           qid.type, qid.version, qid.path, iounit);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-}
-
-static void v9fs_lcreate(void *opaque)
-{
-    int32_t dfid, flags, mode;
-    gid_t gid;
-    ssize_t err = 0;
-    ssize_t offset = 7;
-    V9fsString name;
-    V9fsFidState *fidp;
-    struct stat stbuf;
-    V9fsQID qid;
-    int32_t iounit;
-    V9fsPDU *pdu = opaque;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dsddd", &dfid,
-                        &name, &flags, &mode, &gid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_lcreate(pdu->tag, pdu->id, dfid, flags, mode, gid);
-
-    fidp = get_fid(pdu, dfid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-
-    flags = get_dotl_openflags(pdu->s, flags);
-    err = v9fs_co_open2(pdu, fidp, &name, gid,
-                        flags | O_CREAT, mode, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    fidp->fid_type = P9_FID_FILE;
-    fidp->open_flags = flags;
-    if (flags & O_EXCL) {
-        /*
-         * We let the host file system do O_EXCL check
-         * We should not reclaim such fd
-         */
-        fidp->flags |= FID_NON_RECLAIMABLE;
-    }
-    iounit =  get_iounit(pdu, &fidp->path);
-    stat_to_qid(&stbuf, &qid);
-    err = pdu_marshal(pdu, offset, "Qd", &qid, iounit);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_lcreate_return(pdu->tag, pdu->id,
-                              qid.type, qid.version, qid.path, iounit);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(pdu->s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-static void v9fs_fsync(void *opaque)
-{
-    int err;
-    int32_t fid;
-    int datasync;
-    size_t offset = 7;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "dd", &fid, &datasync);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_fsync(pdu->tag, pdu->id, fid, datasync);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    err = v9fs_co_fsync(pdu, fidp, datasync);
-    if (!err) {
-        err = offset;
-    }
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-}
-
-static void v9fs_clunk(void *opaque)
-{
-    int err;
-    int32_t fid;
-    size_t offset = 7;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "d", &fid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_clunk(pdu->tag, pdu->id, fid);
-
-    fidp = clunk_fid(s, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    /*
-     * Bump the ref so that put_fid will
-     * free the fid.
-     */
-    fidp->ref++;
-    err = put_fid(pdu, fidp);
-    if (!err) {
-        err = offset;
-    }
-out_nofid:
-    complete_pdu(s, pdu, err);
-}
-
-static int v9fs_xattr_read(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
-                           uint64_t off, uint32_t max_count)
-{
-    ssize_t err;
-    size_t offset = 7;
-    int read_count;
-    int64_t xattr_len;
-
-    xattr_len = fidp->fs.xattr.len;
-    read_count = xattr_len - off;
-    if (read_count > max_count) {
-        read_count = max_count;
-    } else if (read_count < 0) {
-        /*
-         * read beyond XATTR value
-         */
-        read_count = 0;
-    }
-    err = pdu_marshal(pdu, offset, "d", read_count);
-    if (err < 0) {
-        return err;
-    }
-    offset += err;
-    err = v9fs_pack(pdu->elem.in_sg, pdu->elem.in_num, offset,
-                    ((char *)fidp->fs.xattr.value) + off,
-                    read_count);
-    if (err < 0) {
-        return err;
-    }
-    offset += err;
-    return offset;
-}
-
-static int v9fs_do_readdir_with_stat(V9fsPDU *pdu,
-                                     V9fsFidState *fidp, uint32_t max_count)
-{
-    V9fsPath path;
-    V9fsStat v9stat;
-    int len, err = 0;
-    int32_t count = 0;
-    struct stat stbuf;
-    off_t saved_dir_pos;
-    struct dirent *dent, *result;
-
-    /* save the directory position */
-    saved_dir_pos = v9fs_co_telldir(pdu, fidp);
-    if (saved_dir_pos < 0) {
-        return saved_dir_pos;
-    }
-
-    dent = g_malloc(sizeof(struct dirent));
-
-    while (1) {
-        v9fs_path_init(&path);
-        err = v9fs_co_readdir_r(pdu, fidp, dent, &result);
-        if (err || !result) {
-            break;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, dent->d_name, &path);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_lstat(pdu, &path, &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        err = stat_to_v9stat(pdu, &path, &stbuf, &v9stat);
-        if (err < 0) {
-            goto out;
-        }
-        /* 11 = 7 + 4 (7 = start offset, 4 = space for storing count) */
-        len = pdu_marshal(pdu, 11 + count, "S", &v9stat);
-        if ((len != (v9stat.size + 2)) || ((count + len) > max_count)) {
-            /* Ran out of buffer. Set dir back to old position and return */
-            v9fs_co_seekdir(pdu, fidp, saved_dir_pos);
-            v9fs_stat_free(&v9stat);
-            v9fs_path_free(&path);
-            g_free(dent);
-            return count;
-        }
-        count += len;
-        v9fs_stat_free(&v9stat);
-        v9fs_path_free(&path);
-        saved_dir_pos = dent->d_off;
-    }
-out:
-    g_free(dent);
-    v9fs_path_free(&path);
-    if (err < 0) {
-        return err;
-    }
-    return count;
-}
-
-/*
- * Create a QEMUIOVector for a sub-region of PDU iovecs
- *
- * @qiov:       uninitialized QEMUIOVector
- * @skip:       number of bytes to skip from beginning of PDU
- * @size:       number of bytes to include
- * @is_write:   true - write, false - read
- *
- * The resulting QEMUIOVector has heap-allocated iovecs and must be cleaned up
- * with qemu_iovec_destroy().
- */
-static void v9fs_init_qiov_from_pdu(QEMUIOVector *qiov, V9fsPDU *pdu,
-                                    size_t skip, size_t size,
-                                    bool is_write)
-{
-    QEMUIOVector elem;
-    struct iovec *iov;
-    unsigned int niov;
-
-    if (is_write) {
-        iov = pdu->elem.out_sg;
-        niov = pdu->elem.out_num;
-    } else {
-        iov = pdu->elem.in_sg;
-        niov = pdu->elem.in_num;
-    }
-
-    qemu_iovec_init_external(&elem, iov, niov);
-    qemu_iovec_init(qiov, niov);
-    qemu_iovec_concat(qiov, &elem, skip, size);
-}
-
-static void v9fs_read(void *opaque)
-{
-    int32_t fid;
-    uint64_t off;
-    ssize_t err = 0;
-    int32_t count = 0;
-    size_t offset = 7;
-    uint32_t max_count;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "dqd", &fid, &off, &max_count);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_read(pdu->tag, pdu->id, fid, off, max_count);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    if (fidp->fid_type == P9_FID_DIR) {
-
-        if (off == 0) {
-            v9fs_co_rewinddir(pdu, fidp);
-        }
-        count = v9fs_do_readdir_with_stat(pdu, fidp, max_count);
-        if (count < 0) {
-            err = count;
-            goto out;
-        }
-        err = pdu_marshal(pdu, offset, "d", count);
-        if (err < 0) {
-            goto out;
-        }
-        err += offset + count;
-    } else if (fidp->fid_type == P9_FID_FILE) {
-        QEMUIOVector qiov_full;
-        QEMUIOVector qiov;
-        int32_t len;
-
-        v9fs_init_qiov_from_pdu(&qiov_full, pdu, offset + 4, max_count, false);
-        qemu_iovec_init(&qiov, qiov_full.niov);
-        do {
-            qemu_iovec_reset(&qiov);
-            qemu_iovec_concat(&qiov, &qiov_full, count, qiov_full.size - count);
-            if (0) {
-                print_sg(qiov.iov, qiov.niov);
-            }
-            /* Loop in case of EINTR */
-            do {
-                len = v9fs_co_preadv(pdu, fidp, qiov.iov, qiov.niov, off);
-                if (len >= 0) {
-                    off   += len;
-                    count += len;
-                }
-            } while (len == -EINTR && !pdu->cancelled);
-            if (len < 0) {
-                /* IO error return the error */
-                err = len;
-                goto out;
-            }
-        } while (count < max_count && len > 0);
-        err = pdu_marshal(pdu, offset, "d", count);
-        if (err < 0) {
-            goto out;
-        }
-        err += offset + count;
-        qemu_iovec_destroy(&qiov);
-        qemu_iovec_destroy(&qiov_full);
-    } else if (fidp->fid_type == P9_FID_XATTR) {
-        err = v9fs_xattr_read(s, pdu, fidp, off, max_count);
-    } else {
-        err = -EINVAL;
-    }
-    trace_v9fs_read_return(pdu->tag, pdu->id, count, err);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-}
-
-static size_t v9fs_readdir_data_size(V9fsString *name)
-{
-    /*
-     * Size of each dirent on the wire: size of qid (13) + size of offset (8)
-     * size of type (1) + size of name.size (2) + strlen(name.data)
-     */
-    return 24 + v9fs_string_size(name);
-}
-
-static int v9fs_do_readdir(V9fsPDU *pdu,
-                           V9fsFidState *fidp, int32_t max_count)
-{
-    size_t size;
-    V9fsQID qid;
-    V9fsString name;
-    int len, err = 0;
-    int32_t count = 0;
-    off_t saved_dir_pos;
-    struct dirent *dent, *result;
-
-    /* save the directory position */
-    saved_dir_pos = v9fs_co_telldir(pdu, fidp);
-    if (saved_dir_pos < 0) {
-        return saved_dir_pos;
-    }
-
-    dent = g_malloc(sizeof(struct dirent));
-
-    while (1) {
-        err = v9fs_co_readdir_r(pdu, fidp, dent, &result);
-        if (err || !result) {
-            break;
-        }
-        v9fs_string_init(&name);
-        v9fs_string_sprintf(&name, "%s", dent->d_name);
-        if ((count + v9fs_readdir_data_size(&name)) > max_count) {
-            /* Ran out of buffer. Set dir back to old position and return */
-            v9fs_co_seekdir(pdu, fidp, saved_dir_pos);
-            v9fs_string_free(&name);
-            g_free(dent);
-            return count;
-        }
-        /*
-         * Fill up just the path field of qid because the client uses
-         * only that. To fill the entire qid structure we will have
-         * to stat each dirent found, which is expensive
-         */
-        size = MIN(sizeof(dent->d_ino), sizeof(qid.path));
-        memcpy(&qid.path, &dent->d_ino, size);
-        /* Fill the other fields with dummy values */
-        qid.type = 0;
-        qid.version = 0;
-
-        /* 11 = 7 + 4 (7 = start offset, 4 = space for storing count) */
-        len = pdu_marshal(pdu, 11 + count, "Qqbs",
-                          &qid, dent->d_off,
-                          dent->d_type, &name);
-        if (len < 0) {
-            v9fs_co_seekdir(pdu, fidp, saved_dir_pos);
-            v9fs_string_free(&name);
-            g_free(dent);
-            return len;
-        }
-        count += len;
-        v9fs_string_free(&name);
-        saved_dir_pos = dent->d_off;
-    }
-    g_free(dent);
-    if (err < 0) {
-        return err;
-    }
-    return count;
-}
-
-static void v9fs_readdir(void *opaque)
-{
-    int32_t fid;
-    V9fsFidState *fidp;
-    ssize_t retval = 0;
-    size_t offset = 7;
-    uint64_t initial_offset;
-    int32_t count;
-    uint32_t max_count;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    retval = pdu_unmarshal(pdu, offset, "dqd", &fid,
-                           &initial_offset, &max_count);
-    if (retval < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_readdir(pdu->tag, pdu->id, fid, initial_offset, max_count);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        retval = -EINVAL;
-        goto out_nofid;
-    }
-    if (!fidp->fs.dir) {
-        retval = -EINVAL;
-        goto out;
-    }
-    if (initial_offset == 0) {
-        v9fs_co_rewinddir(pdu, fidp);
-    } else {
-        v9fs_co_seekdir(pdu, fidp, initial_offset);
-    }
-    count = v9fs_do_readdir(pdu, fidp, max_count);
-    if (count < 0) {
-        retval = count;
-        goto out;
-    }
-    retval = pdu_marshal(pdu, offset, "d", count);
-    if (retval < 0) {
-        goto out;
-    }
-    retval += count + offset;
-    trace_v9fs_readdir_return(pdu->tag, pdu->id, count, retval);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, retval);
-}
-
-static int v9fs_xattr_write(V9fsState *s, V9fsPDU *pdu, V9fsFidState *fidp,
-                            uint64_t off, uint32_t count,
-                            struct iovec *sg, int cnt)
-{
-    int i, to_copy;
-    ssize_t err = 0;
-    int write_count;
-    int64_t xattr_len;
-    size_t offset = 7;
-
-
-    xattr_len = fidp->fs.xattr.len;
-    write_count = xattr_len - off;
-    if (write_count > count) {
-        write_count = count;
-    } else if (write_count < 0) {
-        /*
-         * write beyond XATTR value len specified in
-         * xattrcreate
-         */
-        err = -ENOSPC;
-        goto out;
-    }
-    err = pdu_marshal(pdu, offset, "d", write_count);
-    if (err < 0) {
-        return err;
-    }
-    err += offset;
-    fidp->fs.xattr.copied_len += write_count;
-    /*
-     * Now copy the content from sg list
-     */
-    for (i = 0; i < cnt; i++) {
-        if (write_count > sg[i].iov_len) {
-            to_copy = sg[i].iov_len;
-        } else {
-            to_copy = write_count;
-        }
-        memcpy((char *)fidp->fs.xattr.value + off, sg[i].iov_base, to_copy);
-        /* updating vs->off since we are not using below */
-        off += to_copy;
-        write_count -= to_copy;
-    }
-out:
-    return err;
-}
-
-static void v9fs_write(void *opaque)
-{
-    ssize_t err;
-    int32_t fid;
-    uint64_t off;
-    uint32_t count;
-    int32_t len = 0;
-    int32_t total = 0;
-    size_t offset = 7;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-    QEMUIOVector qiov_full;
-    QEMUIOVector qiov;
-
-    err = pdu_unmarshal(pdu, offset, "dqd", &fid, &off, &count);
-    if (err < 0) {
-        return complete_pdu(s, pdu, err);
-    }
-    offset += err;
-    v9fs_init_qiov_from_pdu(&qiov_full, pdu, offset, count, true);
-    trace_v9fs_write(pdu->tag, pdu->id, fid, off, count, qiov_full.niov);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    if (fidp->fid_type == P9_FID_FILE) {
-        if (fidp->fs.fd == -1) {
-            err = -EINVAL;
-            goto out;
-        }
-    } else if (fidp->fid_type == P9_FID_XATTR) {
-        /*
-         * setxattr operation
-         */
-        err = v9fs_xattr_write(s, pdu, fidp, off, count,
-                               qiov_full.iov, qiov_full.niov);
-        goto out;
-    } else {
-        err = -EINVAL;
-        goto out;
-    }
-    qemu_iovec_init(&qiov, qiov_full.niov);
-    do {
-        qemu_iovec_reset(&qiov);
-        qemu_iovec_concat(&qiov, &qiov_full, total, qiov_full.size - total);
-        if (0) {
-            print_sg(qiov.iov, qiov.niov);
-        }
-        /* Loop in case of EINTR */
-        do {
-            len = v9fs_co_pwritev(pdu, fidp, qiov.iov, qiov.niov, off);
-            if (len >= 0) {
-                off   += len;
-                total += len;
-            }
-        } while (len == -EINTR && !pdu->cancelled);
-        if (len < 0) {
-            /* IO error return the error */
-            err = len;
-            goto out_qiov;
-        }
-    } while (total < count && len > 0);
-
-    offset = 7;
-    err = pdu_marshal(pdu, offset, "d", total);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_write_return(pdu->tag, pdu->id, total, err);
-out_qiov:
-    qemu_iovec_destroy(&qiov);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    qemu_iovec_destroy(&qiov_full);
-    complete_pdu(s, pdu, err);
-}
-
-static void v9fs_create(void *opaque)
-{
-    int32_t fid;
-    int err = 0;
-    size_t offset = 7;
-    V9fsFidState *fidp;
-    V9fsQID qid;
-    int32_t perm;
-    int8_t mode;
-    V9fsPath path;
-    struct stat stbuf;
-    V9fsString name;
-    V9fsString extension;
-    int iounit;
-    V9fsPDU *pdu = opaque;
-
-    v9fs_path_init(&path);
-    v9fs_string_init(&name);
-    v9fs_string_init(&extension);
-    err = pdu_unmarshal(pdu, offset, "dsdbs", &fid, &name,
-                        &perm, &mode, &extension);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_create(pdu->tag, pdu->id, fid, name.data, perm, mode);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    if (perm & P9_STAT_MODE_DIR) {
-        err = v9fs_co_mkdir(pdu, fidp, &name, perm & 0777,
-                            fidp->uid, -1, &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
-        if (err < 0) {
-            goto out;
-        }
-        v9fs_path_copy(&fidp->path, &path);
-        err = v9fs_co_opendir(pdu, fidp);
-        if (err < 0) {
-            goto out;
-        }
-        fidp->fid_type = P9_FID_DIR;
-    } else if (perm & P9_STAT_MODE_SYMLINK) {
-        err = v9fs_co_symlink(pdu, fidp, &name,
-                              extension.data, -1 , &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
-        if (err < 0) {
-            goto out;
-        }
-        v9fs_path_copy(&fidp->path, &path);
-    } else if (perm & P9_STAT_MODE_LINK) {
-        int32_t ofid = atoi(extension.data);
-        V9fsFidState *ofidp = get_fid(pdu, ofid);
-        if (ofidp == NULL) {
-            err = -EINVAL;
-            goto out;
-        }
-        err = v9fs_co_link(pdu, ofidp, fidp, &name);
-        put_fid(pdu, ofidp);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
-        if (err < 0) {
-            fidp->fid_type = P9_FID_NONE;
-            goto out;
-        }
-        v9fs_path_copy(&fidp->path, &path);
-        err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
-        if (err < 0) {
-            fidp->fid_type = P9_FID_NONE;
-            goto out;
-        }
-    } else if (perm & P9_STAT_MODE_DEVICE) {
-        char ctype;
-        uint32_t major, minor;
-        mode_t nmode = 0;
-
-        if (sscanf(extension.data, "%c %u %u", &ctype, &major, &minor) != 3) {
-            err = -errno;
-            goto out;
-        }
-
-        switch (ctype) {
-        case 'c':
-            nmode = S_IFCHR;
-            break;
-        case 'b':
-            nmode = S_IFBLK;
-            break;
-        default:
-            err = -EIO;
-            goto out;
-        }
-
-        nmode |= perm & 0777;
-        err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, -1,
-                            makedev(major, minor), nmode, &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
-        if (err < 0) {
-            goto out;
-        }
-        v9fs_path_copy(&fidp->path, &path);
-    } else if (perm & P9_STAT_MODE_NAMED_PIPE) {
-        err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, -1,
-                            0, S_IFIFO | (perm & 0777), &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
-        if (err < 0) {
-            goto out;
-        }
-        v9fs_path_copy(&fidp->path, &path);
-    } else if (perm & P9_STAT_MODE_SOCKET) {
-        err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, -1,
-                            0, S_IFSOCK | (perm & 0777), &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        err = v9fs_co_name_to_path(pdu, &fidp->path, name.data, &path);
-        if (err < 0) {
-            goto out;
-        }
-        v9fs_path_copy(&fidp->path, &path);
-    } else {
-        err = v9fs_co_open2(pdu, fidp, &name, -1,
-                            omode_to_uflags(mode)|O_CREAT, perm, &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        fidp->fid_type = P9_FID_FILE;
-        fidp->open_flags = omode_to_uflags(mode);
-        if (fidp->open_flags & O_EXCL) {
-            /*
-             * We let the host file system do O_EXCL check
-             * We should not reclaim such fd
-             */
-            fidp->flags |= FID_NON_RECLAIMABLE;
-        }
-    }
-    iounit = get_iounit(pdu, &fidp->path);
-    stat_to_qid(&stbuf, &qid);
-    err = pdu_marshal(pdu, offset, "Qd", &qid, iounit);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_create_return(pdu->tag, pdu->id,
-                             qid.type, qid.version, qid.path, iounit);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-   complete_pdu(pdu->s, pdu, err);
-   v9fs_string_free(&name);
-   v9fs_string_free(&extension);
-   v9fs_path_free(&path);
-}
-
-static void v9fs_symlink(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    V9fsString name;
-    V9fsString symname;
-    V9fsFidState *dfidp;
-    V9fsQID qid;
-    struct stat stbuf;
-    int32_t dfid;
-    int err = 0;
-    gid_t gid;
-    size_t offset = 7;
-
-    v9fs_string_init(&name);
-    v9fs_string_init(&symname);
-    err = pdu_unmarshal(pdu, offset, "dssd", &dfid, &name, &symname, &gid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_symlink(pdu->tag, pdu->id, dfid, name.data, symname.data, gid);
-
-    dfidp = get_fid(pdu, dfid);
-    if (dfidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    err = v9fs_co_symlink(pdu, dfidp, &name, symname.data, gid, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    stat_to_qid(&stbuf, &qid);
-    err =  pdu_marshal(pdu, offset, "Q", &qid);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_symlink_return(pdu->tag, pdu->id,
-                              qid.type, qid.version, qid.path);
-out:
-    put_fid(pdu, dfidp);
-out_nofid:
-    complete_pdu(pdu->s, pdu, err);
-    v9fs_string_free(&name);
-    v9fs_string_free(&symname);
-}
-
-static void v9fs_flush(void *opaque)
-{
-    ssize_t err;
-    int16_t tag;
-    size_t offset = 7;
-    V9fsPDU *cancel_pdu;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    err = pdu_unmarshal(pdu, offset, "w", &tag);
-    if (err < 0) {
-        complete_pdu(s, pdu, err);
-        return;
-    }
-    trace_v9fs_flush(pdu->tag, pdu->id, tag);
-
-    QLIST_FOREACH(cancel_pdu, &s->active_list, next) {
-        if (cancel_pdu->tag == tag) {
-            break;
-        }
-    }
-    if (cancel_pdu) {
-        cancel_pdu->cancelled = 1;
-        /*
-         * Wait for pdu to complete.
-         */
-        qemu_co_queue_wait(&cancel_pdu->complete);
-        cancel_pdu->cancelled = 0;
-        free_pdu(pdu->s, cancel_pdu);
-    }
-    complete_pdu(s, pdu, 7);
-}
-
-static void v9fs_link(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-    int32_t dfid, oldfid;
-    V9fsFidState *dfidp, *oldfidp;
-    V9fsString name;
-    size_t offset = 7;
-    int err = 0;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dds", &dfid, &oldfid, &name);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_link(pdu->tag, pdu->id, dfid, oldfid, name.data);
-
-    dfidp = get_fid(pdu, dfid);
-    if (dfidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-
-    oldfidp = get_fid(pdu, oldfid);
-    if (oldfidp == NULL) {
-        err = -ENOENT;
-        goto out;
-    }
-    err = v9fs_co_link(pdu, oldfidp, dfidp, &name);
-    if (!err) {
-        err = offset;
-    }
-out:
-    put_fid(pdu, dfidp);
-out_nofid:
-    v9fs_string_free(&name);
-    complete_pdu(s, pdu, err);
-}
-
-/* Only works with path name based fid */
-static void v9fs_remove(void *opaque)
-{
-    int32_t fid;
-    int err = 0;
-    size_t offset = 7;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-
-    err = pdu_unmarshal(pdu, offset, "d", &fid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_remove(pdu->tag, pdu->id, fid);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    /* if fs driver is not path based, return EOPNOTSUPP */
-    if (!(pdu->s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT)) {
-        err = -EOPNOTSUPP;
-        goto out_err;
-    }
-    /*
-     * IF the file is unlinked, we cannot reopen
-     * the file later. So don't reclaim fd
-     */
-    err = v9fs_mark_fids_unreclaim(pdu, &fidp->path);
-    if (err < 0) {
-        goto out_err;
-    }
-    err = v9fs_co_remove(pdu, &fidp->path);
-    if (!err) {
-        err = offset;
-    }
-out_err:
-    /* For TREMOVE we need to clunk the fid even on failed remove */
-    clunk_fid(pdu->s, fidp->fid);
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(pdu->s, pdu, err);
-}
-
-static void v9fs_unlinkat(void *opaque)
-{
-    int err = 0;
-    V9fsString name;
-    int32_t dfid, flags;
-    size_t offset = 7;
-    V9fsPath path;
-    V9fsFidState *dfidp;
-    V9fsPDU *pdu = opaque;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dsd", &dfid, &name, &flags);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    dfidp = get_fid(pdu, dfid);
-    if (dfidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    /*
-     * IF the file is unlinked, we cannot reopen
-     * the file later. So don't reclaim fd
-     */
-    v9fs_path_init(&path);
-    err = v9fs_co_name_to_path(pdu, &dfidp->path, name.data, &path);
-    if (err < 0) {
-        goto out_err;
-    }
-    err = v9fs_mark_fids_unreclaim(pdu, &path);
-    if (err < 0) {
-        goto out_err;
-    }
-    err = v9fs_co_unlinkat(pdu, &dfidp->path, &name, flags);
-    if (!err) {
-        err = offset;
-    }
-out_err:
-    put_fid(pdu, dfidp);
-    v9fs_path_free(&path);
-out_nofid:
-    complete_pdu(pdu->s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-
-/* Only works with path name based fid */
-static int v9fs_complete_rename(V9fsPDU *pdu, V9fsFidState *fidp,
-                                int32_t newdirfid, V9fsString *name)
-{
-    char *end;
-    int err = 0;
-    V9fsPath new_path;
-    V9fsFidState *tfidp;
-    V9fsState *s = pdu->s;
-    V9fsFidState *dirfidp = NULL;
-    char *old_name, *new_name;
-
-    v9fs_path_init(&new_path);
-    if (newdirfid != -1) {
-        dirfidp = get_fid(pdu, newdirfid);
-        if (dirfidp == NULL) {
-            err = -ENOENT;
-            goto out_nofid;
-        }
-        BUG_ON(dirfidp->fid_type != P9_FID_NONE);
-        v9fs_co_name_to_path(pdu, &dirfidp->path, name->data, &new_path);
-    } else {
-        old_name = fidp->path.data;
-        end = strrchr(old_name, '/');
-        if (end) {
-            end++;
-        } else {
-            end = old_name;
-        }
-        new_name = g_malloc0(end - old_name + name->size + 1);
-        strncat(new_name, old_name, end - old_name);
-        strncat(new_name + (end - old_name), name->data, name->size);
-        v9fs_co_name_to_path(pdu, NULL, new_name, &new_path);
-        g_free(new_name);
-    }
-    err = v9fs_co_rename(pdu, &fidp->path, &new_path);
-    if (err < 0) {
-        goto out;
-    }
-    /*
-     * Fixup fid's pointing to the old name to
-     * start pointing to the new name
-     */
-    for (tfidp = s->fid_list; tfidp; tfidp = tfidp->next) {
-        if (v9fs_path_is_ancestor(&fidp->path, &tfidp->path)) {
-            /* replace the name */
-            v9fs_fix_path(&tfidp->path, &new_path, strlen(fidp->path.data));
-        }
-    }
-out:
-    if (dirfidp) {
-        put_fid(pdu, dirfidp);
-    }
-    v9fs_path_free(&new_path);
-out_nofid:
-    return err;
-}
-
-/* Only works with path name based fid */
-static void v9fs_rename(void *opaque)
-{
-    int32_t fid;
-    ssize_t err = 0;
-    size_t offset = 7;
-    V9fsString name;
-    int32_t newdirfid;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dds", &fid, &newdirfid, &name);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    BUG_ON(fidp->fid_type != P9_FID_NONE);
-    /* if fs driver is not path based, return EOPNOTSUPP */
-    if (!(pdu->s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT)) {
-        err = -EOPNOTSUPP;
-        goto out;
-    }
-    v9fs_path_write_lock(s);
-    err = v9fs_complete_rename(pdu, fidp, newdirfid, &name);
-    v9fs_path_unlock(s);
-    if (!err) {
-        err = offset;
-    }
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-static void v9fs_fix_fid_paths(V9fsPDU *pdu, V9fsPath *olddir,
-                               V9fsString *old_name, V9fsPath *newdir,
-                               V9fsString *new_name)
-{
-    V9fsFidState *tfidp;
-    V9fsPath oldpath, newpath;
-    V9fsState *s = pdu->s;
-
-
-    v9fs_path_init(&oldpath);
-    v9fs_path_init(&newpath);
-    v9fs_co_name_to_path(pdu, olddir, old_name->data, &oldpath);
-    v9fs_co_name_to_path(pdu, newdir, new_name->data, &newpath);
-
-    /*
-     * Fixup fid's pointing to the old name to
-     * start pointing to the new name
-     */
-    for (tfidp = s->fid_list; tfidp; tfidp = tfidp->next) {
-        if (v9fs_path_is_ancestor(&oldpath, &tfidp->path)) {
-            /* replace the name */
-            v9fs_fix_path(&tfidp->path, &newpath, strlen(oldpath.data));
-        }
-    }
-    v9fs_path_free(&oldpath);
-    v9fs_path_free(&newpath);
-}
-
-static int v9fs_complete_renameat(V9fsPDU *pdu, int32_t olddirfid,
-                                  V9fsString *old_name, int32_t newdirfid,
-                                  V9fsString *new_name)
-{
-    int err = 0;
-    V9fsState *s = pdu->s;
-    V9fsFidState *newdirfidp = NULL, *olddirfidp = NULL;
-
-    olddirfidp = get_fid(pdu, olddirfid);
-    if (olddirfidp == NULL) {
-        err = -ENOENT;
-        goto out;
-    }
-    if (newdirfid != -1) {
-        newdirfidp = get_fid(pdu, newdirfid);
-        if (newdirfidp == NULL) {
-            err = -ENOENT;
-            goto out;
-        }
-    } else {
-        newdirfidp = get_fid(pdu, olddirfid);
-    }
-
-    err = v9fs_co_renameat(pdu, &olddirfidp->path, old_name,
-                           &newdirfidp->path, new_name);
-    if (err < 0) {
-        goto out;
-    }
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        /* Only for path based fid  we need to do the below fixup */
-        v9fs_fix_fid_paths(pdu, &olddirfidp->path, old_name,
-                           &newdirfidp->path, new_name);
-    }
-out:
-    if (olddirfidp) {
-        put_fid(pdu, olddirfidp);
-    }
-    if (newdirfidp) {
-        put_fid(pdu, newdirfidp);
-    }
-    return err;
-}
-
-static void v9fs_renameat(void *opaque)
-{
-    ssize_t err = 0;
-    size_t offset = 7;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-    int32_t olddirfid, newdirfid;
-    V9fsString old_name, new_name;
-
-    v9fs_string_init(&old_name);
-    v9fs_string_init(&new_name);
-    err = pdu_unmarshal(pdu, offset, "dsds", &olddirfid,
-                        &old_name, &newdirfid, &new_name);
-    if (err < 0) {
-        goto out_err;
-    }
-
-    v9fs_path_write_lock(s);
-    err = v9fs_complete_renameat(pdu, olddirfid,
-                                 &old_name, newdirfid, &new_name);
-    v9fs_path_unlock(s);
-    if (!err) {
-        err = offset;
-    }
-
-out_err:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&old_name);
-    v9fs_string_free(&new_name);
-}
-
-static void v9fs_wstat(void *opaque)
-{
-    int32_t fid;
-    int err = 0;
-    int16_t unused;
-    V9fsStat v9stat;
-    size_t offset = 7;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    v9fs_stat_init(&v9stat);
-    err = pdu_unmarshal(pdu, offset, "dwS", &fid, &unused, &v9stat);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_wstat(pdu->tag, pdu->id, fid,
-                     v9stat.mode, v9stat.atime, v9stat.mtime);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    /* do we need to sync the file? */
-    if (donttouch_stat(&v9stat)) {
-        err = v9fs_co_fsync(pdu, fidp, 0);
-        goto out;
-    }
-    if (v9stat.mode != -1) {
-        uint32_t v9_mode;
-        err = v9fs_co_lstat(pdu, &fidp->path, &stbuf);
-        if (err < 0) {
-            goto out;
-        }
-        v9_mode = stat_to_v9mode(&stbuf);
-        if ((v9stat.mode & P9_STAT_MODE_TYPE_BITS) !=
-            (v9_mode & P9_STAT_MODE_TYPE_BITS)) {
-            /* Attempting to change the type */
-            err = -EIO;
-            goto out;
-        }
-        err = v9fs_co_chmod(pdu, &fidp->path,
-                            v9mode_to_mode(v9stat.mode,
-                                           &v9stat.extension));
-        if (err < 0) {
-            goto out;
-        }
-    }
-    if (v9stat.mtime != -1 || v9stat.atime != -1) {
-        struct timespec times[2];
-        if (v9stat.atime != -1) {
-            times[0].tv_sec = v9stat.atime;
-            times[0].tv_nsec = 0;
-        } else {
-            times[0].tv_nsec = UTIME_OMIT;
-        }
-        if (v9stat.mtime != -1) {
-            times[1].tv_sec = v9stat.mtime;
-            times[1].tv_nsec = 0;
-        } else {
-            times[1].tv_nsec = UTIME_OMIT;
-        }
-        err = v9fs_co_utimensat(pdu, &fidp->path, times);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    if (v9stat.n_gid != -1 || v9stat.n_uid != -1) {
-        err = v9fs_co_chown(pdu, &fidp->path, v9stat.n_uid, v9stat.n_gid);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    if (v9stat.name.size != 0) {
-        err = v9fs_complete_rename(pdu, fidp, -1, &v9stat.name);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    if (v9stat.length != -1) {
-        err = v9fs_co_truncate(pdu, &fidp->path, v9stat.length);
-        if (err < 0) {
-            goto out;
-        }
-    }
-    err = offset;
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    v9fs_stat_free(&v9stat);
-    complete_pdu(s, pdu, err);
-}
-
-static int v9fs_fill_statfs(V9fsState *s, V9fsPDU *pdu, struct statfs *stbuf)
-{
-    uint32_t f_type;
-    uint32_t f_bsize;
-    uint64_t f_blocks;
-    uint64_t f_bfree;
-    uint64_t f_bavail;
-    uint64_t f_files;
-    uint64_t f_ffree;
-    uint64_t fsid_val;
-    uint32_t f_namelen;
-    size_t offset = 7;
-    int32_t bsize_factor;
-
-    /*
-     * compute bsize factor based on host file system block size
-     * and client msize
-     */
-    bsize_factor = (s->msize - P9_IOHDRSZ)/stbuf->f_bsize;
-    if (!bsize_factor) {
-        bsize_factor = 1;
-    }
-    f_type  = stbuf->f_type;
-    f_bsize = stbuf->f_bsize;
-    f_bsize *= bsize_factor;
-    /*
-     * f_bsize is adjusted(multiplied) by bsize factor, so we need to
-     * adjust(divide) the number of blocks, free blocks and available
-     * blocks by bsize factor
-     */
-    f_blocks = stbuf->f_blocks/bsize_factor;
-    f_bfree  = stbuf->f_bfree/bsize_factor;
-    f_bavail = stbuf->f_bavail/bsize_factor;
-    f_files  = stbuf->f_files;
-    f_ffree  = stbuf->f_ffree;
-    fsid_val = (unsigned int) stbuf->f_fsid.__val[0] |
-               (unsigned long long)stbuf->f_fsid.__val[1] << 32;
-    f_namelen = stbuf->f_namelen;
-
-    return pdu_marshal(pdu, offset, "ddqqqqqqd",
-                       f_type, f_bsize, f_blocks, f_bfree,
-                       f_bavail, f_files, f_ffree,
-                       fsid_val, f_namelen);
-}
-
-static void v9fs_statfs(void *opaque)
-{
-    int32_t fid;
-    ssize_t retval = 0;
-    size_t offset = 7;
-    V9fsFidState *fidp;
-    struct statfs stbuf;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    retval = pdu_unmarshal(pdu, offset, "d", &fid);
-    if (retval < 0) {
-        goto out_nofid;
-    }
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        retval = -ENOENT;
-        goto out_nofid;
-    }
-    retval = v9fs_co_statfs(pdu, &fidp->path, &stbuf);
-    if (retval < 0) {
-        goto out;
-    }
-    retval = v9fs_fill_statfs(s, pdu, &stbuf);
-    if (retval < 0) {
-        goto out;
-    }
-    retval += offset;
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, retval);
-}
-
-static void v9fs_mknod(void *opaque)
-{
-
-    int mode;
-    gid_t gid;
-    int32_t fid;
-    V9fsQID qid;
-    int err = 0;
-    int major, minor;
-    size_t offset = 7;
-    V9fsString name;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dsdddd", &fid, &name, &mode,
-                        &major, &minor, &gid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_mknod(pdu->tag, pdu->id, fid, mode, major, minor);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    err = v9fs_co_mknod(pdu, fidp, &name, fidp->uid, gid,
-                        makedev(major, minor), mode, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    stat_to_qid(&stbuf, &qid);
-    err = pdu_marshal(pdu, offset, "Q", &qid);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_mknod_return(pdu->tag, pdu->id,
-                            qid.type, qid.version, qid.path);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-/*
- * Implement posix byte range locking code
- * Server side handling of locking code is very simple, because 9p server in
- * QEMU can handle only one client. And most of the lock handling
- * (like conflict, merging) etc is done by the VFS layer itself, so no need to
- * do any thing in * qemu 9p server side lock code path.
- * So when a TLOCK request comes, always return success
- */
-static void v9fs_lock(void *opaque)
-{
-    int8_t status;
-    V9fsFlock flock;
-    size_t offset = 7;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    int32_t fid, err = 0;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    status = P9_LOCK_ERROR;
-    v9fs_string_init(&flock.client_id);
-    err = pdu_unmarshal(pdu, offset, "dbdqqds", &fid, &flock.type,
-                        &flock.flags, &flock.start, &flock.length,
-                        &flock.proc_id, &flock.client_id);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_lock(pdu->tag, pdu->id, fid,
-                    flock.type, flock.start, flock.length);
-
-
-    /* We support only block flag now (that too ignored currently) */
-    if (flock.flags & ~P9_LOCK_FLAGS_BLOCK) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    err = v9fs_co_fstat(pdu, fidp, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    status = P9_LOCK_SUCCESS;
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    err = pdu_marshal(pdu, offset, "b", status);
-    if (err > 0) {
-        err += offset;
-    }
-    trace_v9fs_lock_return(pdu->tag, pdu->id, status);
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&flock.client_id);
-}
-
-/*
- * When a TGETLOCK request comes, always return success because all lock
- * handling is done by client's VFS layer.
- */
-static void v9fs_getlock(void *opaque)
-{
-    size_t offset = 7;
-    struct stat stbuf;
-    V9fsFidState *fidp;
-    V9fsGetlock glock;
-    int32_t fid, err = 0;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    v9fs_string_init(&glock.client_id);
-    err = pdu_unmarshal(pdu, offset, "dbqqds", &fid, &glock.type,
-                        &glock.start, &glock.length, &glock.proc_id,
-                        &glock.client_id);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_getlock(pdu->tag, pdu->id, fid,
-                       glock.type, glock.start, glock.length);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    err = v9fs_co_fstat(pdu, fidp, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    glock.type = P9_LOCK_TYPE_UNLCK;
-    err = pdu_marshal(pdu, offset, "bqqds", glock.type,
-                          glock.start, glock.length, glock.proc_id,
-                          &glock.client_id);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_getlock_return(pdu->tag, pdu->id, glock.type, glock.start,
-                              glock.length, glock.proc_id);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&glock.client_id);
-}
-
-static void v9fs_mkdir(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    size_t offset = 7;
-    int32_t fid;
-    struct stat stbuf;
-    V9fsQID qid;
-    V9fsString name;
-    V9fsFidState *fidp;
-    gid_t gid;
-    int mode;
-    int err = 0;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dsdd", &fid, &name, &mode, &gid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_mkdir(pdu->tag, pdu->id, fid, name.data, mode, gid);
-
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    err = v9fs_co_mkdir(pdu, fidp, &name, mode, fidp->uid, gid, &stbuf);
-    if (err < 0) {
-        goto out;
-    }
-    stat_to_qid(&stbuf, &qid);
-    err = pdu_marshal(pdu, offset, "Q", &qid);
-    if (err < 0) {
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_mkdir_return(pdu->tag, pdu->id,
-                            qid.type, qid.version, qid.path, err);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(pdu->s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-static void v9fs_xattrwalk(void *opaque)
-{
-    int64_t size;
-    V9fsString name;
-    ssize_t err = 0;
-    size_t offset = 7;
-    int32_t fid, newfid;
-    V9fsFidState *file_fidp;
-    V9fsFidState *xattr_fidp = NULL;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dds", &fid, &newfid, &name);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_xattrwalk(pdu->tag, pdu->id, fid, newfid, name.data);
-
-    file_fidp = get_fid(pdu, fid);
-    if (file_fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-    xattr_fidp = alloc_fid(s, newfid);
-    if (xattr_fidp == NULL) {
-        err = -EINVAL;
-        goto out;
-    }
-    v9fs_path_copy(&xattr_fidp->path, &file_fidp->path);
-    if (name.data == NULL) {
-        /*
-         * listxattr request. Get the size first
-         */
-        size = v9fs_co_llistxattr(pdu, &xattr_fidp->path, NULL, 0);
-        if (size < 0) {
-            err = size;
-            clunk_fid(s, xattr_fidp->fid);
-            goto out;
-        }
-        /*
-         * Read the xattr value
-         */
-        xattr_fidp->fs.xattr.len = size;
-        xattr_fidp->fid_type = P9_FID_XATTR;
-        xattr_fidp->fs.xattr.copied_len = -1;
-        if (size) {
-            xattr_fidp->fs.xattr.value = g_malloc(size);
-            err = v9fs_co_llistxattr(pdu, &xattr_fidp->path,
-                                     xattr_fidp->fs.xattr.value,
-                                     xattr_fidp->fs.xattr.len);
-            if (err < 0) {
-                clunk_fid(s, xattr_fidp->fid);
-                goto out;
-            }
-        }
-        err = pdu_marshal(pdu, offset, "q", size);
-        if (err < 0) {
-            goto out;
-        }
-        err += offset;
-    } else {
-        /*
-         * specific xattr fid. We check for xattr
-         * presence also collect the xattr size
-         */
-        size = v9fs_co_lgetxattr(pdu, &xattr_fidp->path,
-                                 &name, NULL, 0);
-        if (size < 0) {
-            err = size;
-            clunk_fid(s, xattr_fidp->fid);
-            goto out;
-        }
-        /*
-         * Read the xattr value
-         */
-        xattr_fidp->fs.xattr.len = size;
-        xattr_fidp->fid_type = P9_FID_XATTR;
-        xattr_fidp->fs.xattr.copied_len = -1;
-        if (size) {
-            xattr_fidp->fs.xattr.value = g_malloc(size);
-            err = v9fs_co_lgetxattr(pdu, &xattr_fidp->path,
-                                    &name, xattr_fidp->fs.xattr.value,
-                                    xattr_fidp->fs.xattr.len);
-            if (err < 0) {
-                clunk_fid(s, xattr_fidp->fid);
-                goto out;
-            }
-        }
-        err = pdu_marshal(pdu, offset, "q", size);
-        if (err < 0) {
-            goto out;
-        }
-        err += offset;
-    }
-    trace_v9fs_xattrwalk_return(pdu->tag, pdu->id, size);
-out:
-    put_fid(pdu, file_fidp);
-    if (xattr_fidp) {
-        put_fid(pdu, xattr_fidp);
-    }
-out_nofid:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-static void v9fs_xattrcreate(void *opaque)
-{
-    int flags;
-    int32_t fid;
-    int64_t size;
-    ssize_t err = 0;
-    V9fsString name;
-    size_t offset = 7;
-    V9fsFidState *file_fidp;
-    V9fsFidState *xattr_fidp;
-    V9fsPDU *pdu = opaque;
-    V9fsState *s = pdu->s;
-
-    v9fs_string_init(&name);
-    err = pdu_unmarshal(pdu, offset, "dsqd", &fid, &name, &size, &flags);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_xattrcreate(pdu->tag, pdu->id, fid, name.data, size, flags);
-
-    file_fidp = get_fid(pdu, fid);
-    if (file_fidp == NULL) {
-        err = -EINVAL;
-        goto out_nofid;
-    }
-    /* Make the file fid point to xattr */
-    xattr_fidp = file_fidp;
-    xattr_fidp->fid_type = P9_FID_XATTR;
-    xattr_fidp->fs.xattr.copied_len = 0;
-    xattr_fidp->fs.xattr.len = size;
-    xattr_fidp->fs.xattr.flags = flags;
-    v9fs_string_init(&xattr_fidp->fs.xattr.name);
-    v9fs_string_copy(&xattr_fidp->fs.xattr.name, &name);
-    xattr_fidp->fs.xattr.value = g_malloc(size);
-    err = offset;
-    put_fid(pdu, file_fidp);
-out_nofid:
-    complete_pdu(s, pdu, err);
-    v9fs_string_free(&name);
-}
-
-static void v9fs_readlink(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    size_t offset = 7;
-    V9fsString target;
-    int32_t fid;
-    int err = 0;
-    V9fsFidState *fidp;
-
-    err = pdu_unmarshal(pdu, offset, "d", &fid);
-    if (err < 0) {
-        goto out_nofid;
-    }
-    trace_v9fs_readlink(pdu->tag, pdu->id, fid);
-    fidp = get_fid(pdu, fid);
-    if (fidp == NULL) {
-        err = -ENOENT;
-        goto out_nofid;
-    }
-
-    v9fs_string_init(&target);
-    err = v9fs_co_readlink(pdu, &fidp->path, &target);
-    if (err < 0) {
-        goto out;
-    }
-    err = pdu_marshal(pdu, offset, "s", &target);
-    if (err < 0) {
-        v9fs_string_free(&target);
-        goto out;
-    }
-    err += offset;
-    trace_v9fs_readlink_return(pdu->tag, pdu->id, target.data);
-    v9fs_string_free(&target);
-out:
-    put_fid(pdu, fidp);
-out_nofid:
-    complete_pdu(pdu->s, pdu, err);
-}
-
-static CoroutineEntry *pdu_co_handlers[] = {
-    [P9_TREADDIR] = v9fs_readdir,
-    [P9_TSTATFS] = v9fs_statfs,
-    [P9_TGETATTR] = v9fs_getattr,
-    [P9_TSETATTR] = v9fs_setattr,
-    [P9_TXATTRWALK] = v9fs_xattrwalk,
-    [P9_TXATTRCREATE] = v9fs_xattrcreate,
-    [P9_TMKNOD] = v9fs_mknod,
-    [P9_TRENAME] = v9fs_rename,
-    [P9_TLOCK] = v9fs_lock,
-    [P9_TGETLOCK] = v9fs_getlock,
-    [P9_TRENAMEAT] = v9fs_renameat,
-    [P9_TREADLINK] = v9fs_readlink,
-    [P9_TUNLINKAT] = v9fs_unlinkat,
-    [P9_TMKDIR] = v9fs_mkdir,
-    [P9_TVERSION] = v9fs_version,
-    [P9_TLOPEN] = v9fs_open,
-    [P9_TATTACH] = v9fs_attach,
-    [P9_TSTAT] = v9fs_stat,
-    [P9_TWALK] = v9fs_walk,
-    [P9_TCLUNK] = v9fs_clunk,
-    [P9_TFSYNC] = v9fs_fsync,
-    [P9_TOPEN] = v9fs_open,
-    [P9_TREAD] = v9fs_read,
-#if 0
-    [P9_TAUTH] = v9fs_auth,
-#endif
-    [P9_TFLUSH] = v9fs_flush,
-    [P9_TLINK] = v9fs_link,
-    [P9_TSYMLINK] = v9fs_symlink,
-    [P9_TCREATE] = v9fs_create,
-    [P9_TLCREATE] = v9fs_lcreate,
-    [P9_TWRITE] = v9fs_write,
-    [P9_TWSTAT] = v9fs_wstat,
-    [P9_TREMOVE] = v9fs_remove,
-};
-
-static void v9fs_op_not_supp(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    complete_pdu(pdu->s, pdu, -EOPNOTSUPP);
-}
-
-static void v9fs_fs_ro(void *opaque)
-{
-    V9fsPDU *pdu = opaque;
-    complete_pdu(pdu->s, pdu, -EROFS);
-}
-
-static inline bool is_read_only_op(V9fsPDU *pdu)
-{
-    switch (pdu->id) {
-    case P9_TREADDIR:
-    case P9_TSTATFS:
-    case P9_TGETATTR:
-    case P9_TXATTRWALK:
-    case P9_TLOCK:
-    case P9_TGETLOCK:
-    case P9_TREADLINK:
-    case P9_TVERSION:
-    case P9_TLOPEN:
-    case P9_TATTACH:
-    case P9_TSTAT:
-    case P9_TWALK:
-    case P9_TCLUNK:
-    case P9_TFSYNC:
-    case P9_TOPEN:
-    case P9_TREAD:
-    case P9_TAUTH:
-    case P9_TFLUSH:
-        return 1;
-    default:
-        return 0;
-    }
-}
-
-static void submit_pdu(V9fsState *s, V9fsPDU *pdu)
-{
-    Coroutine *co;
-    CoroutineEntry *handler;
-
-    if (pdu->id >= ARRAY_SIZE(pdu_co_handlers) ||
-        (pdu_co_handlers[pdu->id] == NULL)) {
-        handler = v9fs_op_not_supp;
-    } else {
-        handler = pdu_co_handlers[pdu->id];
-    }
-
-    if (is_ro_export(&s->ctx) && !is_read_only_op(pdu)) {
-        handler = v9fs_fs_ro;
-    }
-    co = qemu_coroutine_create(handler);
-    qemu_coroutine_enter(co, pdu);
-}
-
-void handle_9p_output(VirtIODevice *vdev, VirtQueue *vq)
-{
-    V9fsState *s = (V9fsState *)vdev;
-    V9fsPDU *pdu;
-    ssize_t len;
-
-    while ((pdu = alloc_pdu(s)) &&
-            (len = virtqueue_pop(vq, &pdu->elem)) != 0) {
-        uint8_t *ptr;
-        pdu->s = s;
-        BUG_ON(pdu->elem.out_num == 0 || pdu->elem.in_num == 0);
-        BUG_ON(pdu->elem.out_sg[0].iov_len < 7);
-
-        ptr = pdu->elem.out_sg[0].iov_base;
-
-        pdu->size = le32_to_cpu(*(uint32_t *)ptr);
-        pdu->id = ptr[4];
-        pdu->tag = le16_to_cpu(*(uint16_t *)(ptr + 5));
-        qemu_co_queue_init(&pdu->complete);
-        submit_pdu(s, pdu);
-    }
-    free_pdu(s, pdu);
-}
-
-static void __attribute__((__constructor__)) virtio_9p_set_fd_limit(void)
-{
-    struct rlimit rlim;
-    if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
-        fprintf(stderr, "Failed to get the resource limit\n");
-        exit(1);
-    }
-    open_fd_hw = rlim.rlim_cur - MIN(400, rlim.rlim_cur/3);
-    open_fd_rc = rlim.rlim_cur/2;
-}
diff --git a/hw/9pfs/virtio-9p.h b/hw/9pfs/virtio-9p.h
index 2c3603a..7586b79 100644
--- a/hw/9pfs/virtio-9p.h
+++ b/hw/9pfs/virtio-9p.h
@@ -1,412 +1,31 @@
-#ifndef _QEMU_VIRTIO_9P_H
-#define _QEMU_VIRTIO_9P_H
+#ifndef QEMU_VIRTIO_9P_H
+#define QEMU_VIRTIO_9P_H
 
-#include <sys/types.h>
-#include <dirent.h>
-#include <sys/time.h>
-#include <utime.h>
-#include <sys/resource.h>
-#include <glib.h>
+#include "standard-headers/linux/virtio_9p.h"
 #include "hw/virtio/virtio.h"
-#include "fsdev/file-op-9p.h"
-#include "fsdev/virtio-9p-marshal.h"
-#include "qemu/thread.h"
-#include "block/coroutine.h"
+#include "9p.h"
 
-/* The feature bitmap for virtio 9P */
-/* The mount point is specified in a config variable */
-#define VIRTIO_9P_MOUNT_TAG 0
-
-enum {
-    P9_TLERROR = 6,
-    P9_RLERROR,
-    P9_TSTATFS = 8,
-    P9_RSTATFS,
-    P9_TLOPEN = 12,
-    P9_RLOPEN,
-    P9_TLCREATE = 14,
-    P9_RLCREATE,
-    P9_TSYMLINK = 16,
-    P9_RSYMLINK,
-    P9_TMKNOD = 18,
-    P9_RMKNOD,
-    P9_TRENAME = 20,
-    P9_RRENAME,
-    P9_TREADLINK = 22,
-    P9_RREADLINK,
-    P9_TGETATTR = 24,
-    P9_RGETATTR,
-    P9_TSETATTR = 26,
-    P9_RSETATTR,
-    P9_TXATTRWALK = 30,
-    P9_RXATTRWALK,
-    P9_TXATTRCREATE = 32,
-    P9_RXATTRCREATE,
-    P9_TREADDIR = 40,
-    P9_RREADDIR,
-    P9_TFSYNC = 50,
-    P9_RFSYNC,
-    P9_TLOCK = 52,
-    P9_RLOCK,
-    P9_TGETLOCK = 54,
-    P9_RGETLOCK,
-    P9_TLINK = 70,
-    P9_RLINK,
-    P9_TMKDIR = 72,
-    P9_RMKDIR,
-    P9_TRENAMEAT = 74,
-    P9_RRENAMEAT,
-    P9_TUNLINKAT = 76,
-    P9_RUNLINKAT,
-    P9_TVERSION = 100,
-    P9_RVERSION,
-    P9_TAUTH = 102,
-    P9_RAUTH,
-    P9_TATTACH = 104,
-    P9_RATTACH,
-    P9_TERROR = 106,
-    P9_RERROR,
-    P9_TFLUSH = 108,
-    P9_RFLUSH,
-    P9_TWALK = 110,
-    P9_RWALK,
-    P9_TOPEN = 112,
-    P9_ROPEN,
-    P9_TCREATE = 114,
-    P9_RCREATE,
-    P9_TREAD = 116,
-    P9_RREAD,
-    P9_TWRITE = 118,
-    P9_RWRITE,
-    P9_TCLUNK = 120,
-    P9_RCLUNK,
-    P9_TREMOVE = 122,
-    P9_RREMOVE,
-    P9_TSTAT = 124,
-    P9_RSTAT,
-    P9_TWSTAT = 126,
-    P9_RWSTAT,
-};
-
-
-/* qid.types */
-enum {
-    P9_QTDIR = 0x80,
-    P9_QTAPPEND = 0x40,
-    P9_QTEXCL = 0x20,
-    P9_QTMOUNT = 0x10,
-    P9_QTAUTH = 0x08,
-    P9_QTTMP = 0x04,
-    P9_QTSYMLINK = 0x02,
-    P9_QTLINK = 0x01,
-    P9_QTFILE = 0x00,
-};
-
-enum p9_proto_version {
-    V9FS_PROTO_2000U = 0x01,
-    V9FS_PROTO_2000L = 0x02,
-};
-
-#define P9_NOTAG    (u16)(~0)
-#define P9_NOFID    (u32)(~0)
-#define P9_MAXWELEM 16
-
-#define FID_REFERENCED          0x1
-#define FID_NON_RECLAIMABLE     0x2
-static inline char *rpath(FsContext *ctx, const char *path)
-{
-    return g_strdup_printf("%s/%s", ctx->fs_root, path);
-}
-
-/*
- * ample room for Twrite/Rread header
- * size[4] Tread/Twrite tag[2] fid[4] offset[8] count[4]
- */
-#define P9_IOHDRSZ 24
-
-typedef struct V9fsPDU V9fsPDU;
-struct V9fsState;
-
-struct V9fsPDU
-{
-    uint32_t size;
-    uint16_t tag;
-    uint8_t id;
-    uint8_t cancelled;
-    CoQueue complete;
-    VirtQueueElement elem;
-    struct V9fsState *s;
-    QLIST_ENTRY(V9fsPDU) next;
-};
-
-
-/* FIXME
- * 1) change user needs to set groups and stuff
- */
-
-/* from Linux's linux/virtio_9p.h */
-
-/* The ID for virtio console */
-#define VIRTIO_ID_9P    9
-#define MAX_REQ         128
-#define MAX_TAG_LEN     32
-
-#define BUG_ON(cond) assert(!(cond))
-
-typedef struct V9fsFidState V9fsFidState;
-
-enum {
-    P9_FID_NONE = 0,
-    P9_FID_FILE,
-    P9_FID_DIR,
-    P9_FID_XATTR,
-};
-
-typedef struct V9fsXattr
-{
-    int64_t copied_len;
-    int64_t len;
-    void *value;
-    V9fsString name;
-    int flags;
-} V9fsXattr;
-
-/*
- * Filled by fs driver on open and other
- * calls.
- */
-union V9fsFidOpenState {
-    int fd;
-    DIR *dir;
-    V9fsXattr xattr;
-    /*
-     * private pointer for fs drivers, that
-     * have its own internal representation of
-     * open files.
-     */
-    void *private;
-};
-
-struct V9fsFidState
-{
-    int fid_type;
-    int32_t fid;
-    V9fsPath path;
-    V9fsFidOpenState fs;
-    V9fsFidOpenState fs_reclaim;
-    int flags;
-    int open_flags;
-    uid_t uid;
-    int ref;
-    int clunked;
-    V9fsFidState *next;
-    V9fsFidState *rclm_lst;
-};
-
-typedef struct V9fsState
+typedef struct V9fsVirtioState
 {
     VirtIODevice parent_obj;
     VirtQueue *vq;
-    V9fsPDU pdus[MAX_REQ];
-    QLIST_HEAD(, V9fsPDU) free_list;
-    QLIST_HEAD(, V9fsPDU) active_list;
-    V9fsFidState *fid_list;
-    FileOperations *ops;
-    FsContext ctx;
-    char *tag;
     size_t config_size;
-    enum p9_proto_version proto_version;
-    int32_t msize;
-    /*
-     * lock ensuring atomic path update
-     * on rename.
-     */
-    CoRwlock rename_lock;
-    int32_t root_fid;
-    Error *migration_blocker;
-    V9fsConf fsconf;
-} V9fsState;
+    V9fsPDU pdus[MAX_REQ];
+    VirtQueueElement *elems[MAX_REQ];
+    V9fsState state;
+} V9fsVirtioState;
 
-typedef struct V9fsStatState {
-    V9fsPDU *pdu;
-    size_t offset;
-    V9fsStat v9stat;
-    V9fsFidState *fidp;
-    struct stat stbuf;
-} V9fsStatState;
+extern void virtio_9p_push_and_notify(V9fsPDU *pdu);
 
-typedef struct V9fsOpenState {
-    V9fsPDU *pdu;
-    size_t offset;
-    int32_t mode;
-    V9fsFidState *fidp;
-    V9fsQID qid;
-    struct stat stbuf;
-    int iounit;
-} V9fsOpenState;
-
-typedef struct V9fsReadState {
-    V9fsPDU *pdu;
-    size_t offset;
-    int32_t count;
-    int32_t total;
-    int64_t off;
-    V9fsFidState *fidp;
-    struct iovec iov[128]; /* FIXME: bad, bad, bad */
-    struct iovec *sg;
-    off_t dir_pos;
-    struct dirent *dent;
-    struct stat stbuf;
-    V9fsString name;
-    V9fsStat v9stat;
-    int32_t len;
-    int32_t cnt;
-    int32_t max_count;
-} V9fsReadState;
-
-typedef struct V9fsWriteState {
-    V9fsPDU *pdu;
-    size_t offset;
-    int32_t len;
-    int32_t count;
-    int32_t total;
-    int64_t off;
-    V9fsFidState *fidp;
-    struct iovec iov[128]; /* FIXME: bad, bad, bad */
-    struct iovec *sg;
-    int cnt;
-} V9fsWriteState;
-
-struct virtio_9p_config
-{
-    /* number of characters in tag */
-    uint16_t tag_len;
-    /* Variable size tag name */
-    uint8_t tag[0];
-} QEMU_PACKED;
-
-typedef struct V9fsMkState {
-    V9fsPDU *pdu;
-    size_t offset;
-    V9fsQID qid;
-    struct stat stbuf;
-    V9fsString name;
-    V9fsString fullname;
-} V9fsMkState;
-
-/* 9p2000.L open flags */
-#define P9_DOTL_RDONLY        00000000
-#define P9_DOTL_WRONLY        00000001
-#define P9_DOTL_RDWR          00000002
-#define P9_DOTL_NOACCESS      00000003
-#define P9_DOTL_CREATE        00000100
-#define P9_DOTL_EXCL          00000200
-#define P9_DOTL_NOCTTY        00000400
-#define P9_DOTL_TRUNC         00001000
-#define P9_DOTL_APPEND        00002000
-#define P9_DOTL_NONBLOCK      00004000
-#define P9_DOTL_DSYNC         00010000
-#define P9_DOTL_FASYNC        00020000
-#define P9_DOTL_DIRECT        00040000
-#define P9_DOTL_LARGEFILE     00100000
-#define P9_DOTL_DIRECTORY     00200000
-#define P9_DOTL_NOFOLLOW      00400000
-#define P9_DOTL_NOATIME       01000000
-#define P9_DOTL_CLOEXEC       02000000
-#define P9_DOTL_SYNC          04000000
-
-/* 9p2000.L at flags */
-#define P9_DOTL_AT_REMOVEDIR         0x200
-
-/* 9P2000.L lock type */
-#define P9_LOCK_TYPE_RDLCK 0
-#define P9_LOCK_TYPE_WRLCK 1
-#define P9_LOCK_TYPE_UNLCK 2
-
-#define P9_LOCK_SUCCESS 0
-#define P9_LOCK_BLOCKED 1
-#define P9_LOCK_ERROR 2
-#define P9_LOCK_GRACE 3
-
-#define P9_LOCK_FLAGS_BLOCK 1
-#define P9_LOCK_FLAGS_RECLAIM 2
-
-typedef struct V9fsFlock
-{
-    uint8_t type;
-    uint32_t flags;
-    uint64_t start; /* absolute offset */
-    uint64_t length;
-    uint32_t proc_id;
-    V9fsString client_id;
-} V9fsFlock;
-
-typedef struct V9fsGetlock
-{
-    uint8_t type;
-    uint64_t start; /* absolute offset */
-    uint64_t length;
-    uint32_t proc_id;
-    V9fsString client_id;
-} V9fsGetlock;
-
-extern int open_fd_hw;
-extern int total_open_fd;
-
-size_t pdu_packunpack(void *addr, struct iovec *sg, int sg_count,
-                      size_t offset, size_t size, int pack);
-
-static inline size_t do_pdu_unpack(void *dst, struct iovec *sg, int sg_count,
-                        size_t offset, size_t size)
-{
-    return pdu_packunpack(dst, sg, sg_count, offset, size, 0);
-}
-
-static inline void v9fs_path_write_lock(V9fsState *s)
-{
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        qemu_co_rwlock_wrlock(&s->rename_lock);
-    }
-}
-
-static inline void v9fs_path_read_lock(V9fsState *s)
-{
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        qemu_co_rwlock_rdlock(&s->rename_lock);
-    }
-}
-
-static inline void v9fs_path_unlock(V9fsState *s)
-{
-    if (s->ctx.export_flags & V9FS_PATHNAME_FSCONTEXT) {
-        qemu_co_rwlock_unlock(&s->rename_lock);
-    }
-}
-
-static inline uint8_t v9fs_request_cancelled(V9fsPDU *pdu)
-{
-    return pdu->cancelled;
-}
-
-extern void handle_9p_output(VirtIODevice *vdev, VirtQueue *vq);
-extern void v9fs_reclaim_fd(V9fsPDU *pdu);
-extern void v9fs_path_init(V9fsPath *path);
-extern void v9fs_path_free(V9fsPath *path);
-extern void v9fs_path_copy(V9fsPath *lhs, V9fsPath *rhs);
-extern int v9fs_name_to_path(V9fsState *s, V9fsPath *dirpath,
-                             const char *name, V9fsPath *path);
-
-#define pdu_marshal(pdu, offset, fmt, args...)  \
-    v9fs_marshal(pdu->elem.in_sg, pdu->elem.in_num, offset, 1, fmt, ##args)
-#define pdu_unmarshal(pdu, offset, fmt, args...)  \
-    v9fs_unmarshal(pdu->elem.out_sg, pdu->elem.out_num, offset, 1, fmt, ##args)
+ssize_t virtio_pdu_vmarshal(V9fsPDU *pdu, size_t offset,
+                            const char *fmt, va_list ap);
+ssize_t virtio_pdu_vunmarshal(V9fsPDU *pdu, size_t offset,
+                              const char *fmt, va_list ap);
+void virtio_init_iov_from_pdu(V9fsPDU *pdu, struct iovec **piov,
+                              unsigned int *pniov, bool is_write);
 
 #define TYPE_VIRTIO_9P "virtio-9p-device"
 #define VIRTIO_9P(obj) \
-        OBJECT_CHECK(V9fsState, (obj), TYPE_VIRTIO_9P)
-
-#define DEFINE_VIRTIO_9P_PROPERTIES(_state, _field)             \
-        DEFINE_PROP_STRING("mount_tag", _state, _field.tag),    \
-        DEFINE_PROP_STRING("fsdev", _state, _field.fsdev_id)
+        OBJECT_CHECK(V9fsVirtioState, (obj), TYPE_VIRTIO_9P)
 
 #endif
diff --git a/hw/Makefile.objs b/hw/Makefile.objs
index 52a1464..4a07ed4 100644
--- a/hw/Makefile.objs
+++ b/hw/Makefile.objs
@@ -13,6 +13,7 @@
 devices-dirs-$(CONFIG_SOFTMMU) += input/
 devices-dirs-$(CONFIG_SOFTMMU) += intc/
 devices-dirs-$(CONFIG_IPACK) += ipack/
+devices-dirs-$(CONFIG_IPMI) += ipmi/
 devices-dirs-$(CONFIG_SOFTMMU) += isa/
 devices-dirs-$(CONFIG_SOFTMMU) += misc/
 devices-dirs-$(CONFIG_SOFTMMU) += net/
@@ -26,10 +27,12 @@
 devices-dirs-$(CONFIG_SOFTMMU) += timer/
 devices-dirs-$(CONFIG_TPM) += tpm/
 devices-dirs-$(CONFIG_SOFTMMU) += usb/
+devices-dirs-$(CONFIG_SOFTMMU) += vfio/
 devices-dirs-$(CONFIG_VIRTIO) += virtio/
 devices-dirs-$(CONFIG_SOFTMMU) += watchdog/
 devices-dirs-$(CONFIG_SOFTMMU) += xen/
 devices-dirs-$(CONFIG_MEM_HOTPLUG) += mem/
+devices-dirs-$(CONFIG_SMBIOS) += smbios/
 devices-dirs-y += core/
 common-obj-y += $(devices-dirs-y)
 obj-y += $(devices-dirs-y)
diff --git a/hw/acpi/Makefile.objs b/hw/acpi/Makefile.objs
index acd2389..4b7da66 100644
--- a/hw/acpi/Makefile.objs
+++ b/hw/acpi/Makefile.objs
@@ -1,3 +1,10 @@
-common-obj-$(CONFIG_ACPI) += core.o piix4.o ich9.o pcihp.o cpu_hotplug.o
-common-obj-$(CONFIG_ACPI) += memory_hotplug.o
+common-obj-$(CONFIG_ACPI_X86) += core.o piix4.o pcihp.o
+common-obj-$(CONFIG_ACPI_X86_ICH) += ich9.o tco.o
+common-obj-$(CONFIG_ACPI_CPU_HOTPLUG) += cpu_hotplug.o
+common-obj-$(CONFIG_ACPI_MEMORY_HOTPLUG) += memory_hotplug.o memory_hotplug_acpi_table.o
+common-obj-$(CONFIG_ACPI_CPU_HOTPLUG) += cpu.o
+obj-$(CONFIG_ACPI_NVDIMM) += nvdimm.o
 common-obj-$(CONFIG_ACPI) += acpi_interface.o
+common-obj-$(CONFIG_ACPI) += bios-linker-loader.o
+common-obj-$(CONFIG_ACPI) += aml-build.o
+common-obj-$(call land,$(CONFIG_ACPI),$(CONFIG_IPMI)) += ipmi.o
diff --git a/hw/acpi/acpi_interface.c b/hw/acpi/acpi_interface.c
index c181bb2..6583917 100644
--- a/hw/acpi/acpi_interface.c
+++ b/hw/acpi/acpi_interface.c
@@ -1,6 +1,16 @@
+#include "qemu/osdep.h"
 #include "hw/acpi/acpi_dev_interface.h"
 #include "qemu/module.h"
 
+void acpi_send_event(DeviceState *dev, AcpiEventStatusBits event)
+{
+    AcpiDeviceIfClass *adevc = ACPI_DEVICE_IF_GET_CLASS(dev);
+    if (adevc->send_event) {
+        AcpiDeviceIf *adev = ACPI_DEVICE_IF(dev);
+        adevc->send_event(adev, event);
+    }
+}
+
 static void register_types(void)
 {
     static const TypeInfo acpi_dev_if_info = {
diff --git a/hw/acpi/aml-build.c b/hw/acpi/aml-build.c
new file mode 100644
index 0000000..db3e914
--- /dev/null
+++ b/hw/acpi/aml-build.c
@@ -0,0 +1,1609 @@
+/* Support for generating ACPI tables and passing them to Guests
+ *
+ * Copyright (C) 2015 Red Hat Inc
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ * Author: Igor Mammedov <imammedo@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include <glib/gprintf.h>
+#include "hw/acpi/aml-build.h"
+#include "qemu/bswap.h"
+#include "qemu/bitops.h"
+
+static GArray *build_alloc_array(void)
+{
+    return g_array_new(false, true /* clear */, 1);
+}
+
+static void build_free_array(GArray *array)
+{
+    g_array_free(array, true);
+}
+
+static void build_prepend_byte(GArray *array, uint8_t val)
+{
+    g_array_prepend_val(array, val);
+}
+
+static void build_append_byte(GArray *array, uint8_t val)
+{
+    g_array_append_val(array, val);
+}
+
+static void build_append_array(GArray *array, GArray *val)
+{
+    g_array_append_vals(array, val->data, val->len);
+}
+
+#define ACPI_NAMESEG_LEN 4
+
+static void
+build_append_nameseg(GArray *array, const char *seg)
+{
+    int len;
+
+    len = strlen(seg);
+    assert(len <= ACPI_NAMESEG_LEN);
+
+    g_array_append_vals(array, seg, len);
+    /* Pad up to ACPI_NAMESEG_LEN characters if necessary. */
+    g_array_append_vals(array, "____", ACPI_NAMESEG_LEN - len);
+}
+
+static void GCC_FMT_ATTR(2, 0)
+build_append_namestringv(GArray *array, const char *format, va_list ap)
+{
+    char *s;
+    char **segs;
+    char **segs_iter;
+    int seg_count = 0;
+
+    s = g_strdup_vprintf(format, ap);
+    segs = g_strsplit(s, ".", 0);
+    g_free(s);
+
+    /* count segments */
+    segs_iter = segs;
+    while (*segs_iter) {
+        ++segs_iter;
+        ++seg_count;
+    }
+    /*
+     * ACPI 5.0 spec: 20.2.2 Name Objects Encoding:
+     * "SegCount can be from 1 to 255"
+     */
+    assert(seg_count > 0 && seg_count <= 255);
+
+    /* handle RootPath || PrefixPath */
+    s = *segs;
+    while (*s == '\\' || *s == '^') {
+        build_append_byte(array, *s);
+        ++s;
+    }
+
+    switch (seg_count) {
+    case 1:
+        if (!*s) {
+            build_append_byte(array, 0x00); /* NullName */
+        } else {
+            build_append_nameseg(array, s);
+        }
+        break;
+
+    case 2:
+        build_append_byte(array, 0x2E); /* DualNamePrefix */
+        build_append_nameseg(array, s);
+        build_append_nameseg(array, segs[1]);
+        break;
+    default:
+        build_append_byte(array, 0x2F); /* MultiNamePrefix */
+        build_append_byte(array, seg_count);
+
+        /* handle the 1st segment manually due to prefix/root path */
+        build_append_nameseg(array, s);
+
+        /* add the rest of segments */
+        segs_iter = segs + 1;
+        while (*segs_iter) {
+            build_append_nameseg(array, *segs_iter);
+            ++segs_iter;
+        }
+        break;
+    }
+    g_strfreev(segs);
+}
+
+GCC_FMT_ATTR(2, 3)
+static void build_append_namestring(GArray *array, const char *format, ...)
+{
+    va_list ap;
+
+    va_start(ap, format);
+    build_append_namestringv(array, format, ap);
+    va_end(ap);
+}
+
+/* 5.4 Definition Block Encoding */
+enum {
+    PACKAGE_LENGTH_1BYTE_SHIFT = 6, /* Up to 63 - use extra 2 bits. */
+    PACKAGE_LENGTH_2BYTE_SHIFT = 4,
+    PACKAGE_LENGTH_3BYTE_SHIFT = 12,
+    PACKAGE_LENGTH_4BYTE_SHIFT = 20,
+};
+
+static void
+build_prepend_package_length(GArray *package, unsigned length, bool incl_self)
+{
+    uint8_t byte;
+    unsigned length_bytes;
+
+    if (length + 1 < (1 << PACKAGE_LENGTH_1BYTE_SHIFT)) {
+        length_bytes = 1;
+    } else if (length + 2 < (1 << PACKAGE_LENGTH_3BYTE_SHIFT)) {
+        length_bytes = 2;
+    } else if (length + 3 < (1 << PACKAGE_LENGTH_4BYTE_SHIFT)) {
+        length_bytes = 3;
+    } else {
+        length_bytes = 4;
+    }
+
+    /*
+     * NamedField uses PkgLength encoding but it doesn't include length
+     * of PkgLength itself.
+     */
+    if (incl_self) {
+        /*
+         * PkgLength is the length of the inclusive length of the data
+         * and PkgLength's length itself when used for terms with
+         * explitit length.
+         */
+        length += length_bytes;
+    }
+
+    switch (length_bytes) {
+    case 1:
+        byte = length;
+        build_prepend_byte(package, byte);
+        return;
+    case 4:
+        byte = length >> PACKAGE_LENGTH_4BYTE_SHIFT;
+        build_prepend_byte(package, byte);
+        length &= (1 << PACKAGE_LENGTH_4BYTE_SHIFT) - 1;
+        /* fall through */
+    case 3:
+        byte = length >> PACKAGE_LENGTH_3BYTE_SHIFT;
+        build_prepend_byte(package, byte);
+        length &= (1 << PACKAGE_LENGTH_3BYTE_SHIFT) - 1;
+        /* fall through */
+    case 2:
+        byte = length >> PACKAGE_LENGTH_2BYTE_SHIFT;
+        build_prepend_byte(package, byte);
+        length &= (1 << PACKAGE_LENGTH_2BYTE_SHIFT) - 1;
+        /* fall through */
+    }
+    /*
+     * Most significant two bits of byte zero indicate how many following bytes
+     * are in PkgLength encoding.
+     */
+    byte = ((length_bytes - 1) << PACKAGE_LENGTH_1BYTE_SHIFT) | length;
+    build_prepend_byte(package, byte);
+}
+
+static void
+build_append_pkg_length(GArray *array, unsigned length, bool incl_self)
+{
+    GArray *tmp = build_alloc_array();
+
+    build_prepend_package_length(tmp, length, incl_self);
+    build_append_array(array, tmp);
+    build_free_array(tmp);
+}
+
+static void build_package(GArray *package, uint8_t op)
+{
+    build_prepend_package_length(package, package->len, true);
+    build_prepend_byte(package, op);
+}
+
+static void build_extop_package(GArray *package, uint8_t op)
+{
+    build_package(package, op);
+    build_prepend_byte(package, 0x5B); /* ExtOpPrefix */
+}
+
+static void build_append_int_noprefix(GArray *table, uint64_t value, int size)
+{
+    int i;
+
+    for (i = 0; i < size; ++i) {
+        build_append_byte(table, value & 0xFF);
+        value = value >> 8;
+    }
+}
+
+static void build_append_int(GArray *table, uint64_t value)
+{
+    if (value == 0x00) {
+        build_append_byte(table, 0x00); /* ZeroOp */
+    } else if (value == 0x01) {
+        build_append_byte(table, 0x01); /* OneOp */
+    } else if (value <= 0xFF) {
+        build_append_byte(table, 0x0A); /* BytePrefix */
+        build_append_int_noprefix(table, value, 1);
+    } else if (value <= 0xFFFF) {
+        build_append_byte(table, 0x0B); /* WordPrefix */
+        build_append_int_noprefix(table, value, 2);
+    } else if (value <= 0xFFFFFFFF) {
+        build_append_byte(table, 0x0C); /* DWordPrefix */
+        build_append_int_noprefix(table, value, 4);
+    } else {
+        build_append_byte(table, 0x0E); /* QWordPrefix */
+        build_append_int_noprefix(table, value, 8);
+    }
+}
+
+/*
+ * Build NAME(XXXX, 0x00000000) where 0x00000000 is encoded as a dword,
+ * and return the offset to 0x00000000 for runtime patching.
+ *
+ * Warning: runtime patching is best avoided. Only use this as
+ * a replacement for DataTableRegion (for guests that don't
+ * support it).
+ */
+int
+build_append_named_dword(GArray *array, const char *name_format, ...)
+{
+    int offset;
+    va_list ap;
+
+    build_append_byte(array, 0x08); /* NameOp */
+    va_start(ap, name_format);
+    build_append_namestringv(array, name_format, ap);
+    va_end(ap);
+
+    build_append_byte(array, 0x0C); /* DWordPrefix */
+
+    offset = array->len;
+    build_append_int_noprefix(array, 0x00000000, 4);
+    assert(array->len == offset + 4);
+
+    return offset;
+}
+
+static GPtrArray *alloc_list;
+
+static Aml *aml_alloc(void)
+{
+    Aml *var = g_new0(typeof(*var), 1);
+
+    g_ptr_array_add(alloc_list, var);
+    var->block_flags = AML_NO_OPCODE;
+    var->buf = build_alloc_array();
+    return var;
+}
+
+static Aml *aml_opcode(uint8_t op)
+{
+    Aml *var = aml_alloc();
+
+    var->op  = op;
+    var->block_flags = AML_OPCODE;
+    return var;
+}
+
+static Aml *aml_bundle(uint8_t op, AmlBlockFlags flags)
+{
+    Aml *var = aml_alloc();
+
+    var->op  = op;
+    var->block_flags = flags;
+    return var;
+}
+
+static void aml_free(gpointer data, gpointer user_data)
+{
+    Aml *var = data;
+    build_free_array(var->buf);
+    g_free(var);
+}
+
+Aml *init_aml_allocator(void)
+{
+    assert(!alloc_list);
+    alloc_list = g_ptr_array_new();
+    return aml_alloc();
+}
+
+void free_aml_allocator(void)
+{
+    g_ptr_array_foreach(alloc_list, aml_free, NULL);
+    g_ptr_array_free(alloc_list, true);
+    alloc_list = 0;
+}
+
+/* pack data with DefBuffer encoding */
+static void build_buffer(GArray *array, uint8_t op)
+{
+    GArray *data = build_alloc_array();
+
+    build_append_int(data, array->len);
+    g_array_prepend_vals(array, data->data, data->len);
+    build_free_array(data);
+    build_package(array, op);
+}
+
+void aml_append(Aml *parent_ctx, Aml *child)
+{
+    GArray *buf = build_alloc_array();
+    build_append_array(buf, child->buf);
+
+    switch (child->block_flags) {
+    case AML_OPCODE:
+        build_append_byte(parent_ctx->buf, child->op);
+        break;
+    case AML_EXT_PACKAGE:
+        build_extop_package(buf, child->op);
+        break;
+    case AML_PACKAGE:
+        build_package(buf, child->op);
+        break;
+    case AML_RES_TEMPLATE:
+        build_append_byte(buf, 0x79); /* EndTag */
+        /*
+         * checksum operations are treated as succeeded if checksum
+         * field is zero. [ACPI Spec 1.0b, 6.4.2.8 End Tag]
+         */
+        build_append_byte(buf, 0);
+        /* fall through, to pack resources in buffer */
+    case AML_BUFFER:
+        build_buffer(buf, child->op);
+        break;
+    case AML_NO_OPCODE:
+        break;
+    default:
+        assert(0);
+        break;
+    }
+    build_append_array(parent_ctx->buf, buf);
+    build_free_array(buf);
+}
+
+/* ACPI 1.0b: 16.2.5.1 Namespace Modifier Objects Encoding: DefScope */
+Aml *aml_scope(const char *name_format, ...)
+{
+    va_list ap;
+    Aml *var = aml_bundle(0x10 /* ScopeOp */, AML_PACKAGE);
+    va_start(ap, name_format);
+    build_append_namestringv(var->buf, name_format, ap);
+    va_end(ap);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefReturn */
+Aml *aml_return(Aml *val)
+{
+    Aml *var = aml_opcode(0xA4 /* ReturnOp */);
+    aml_append(var, val);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.6.3 Debug Objects Encoding: DebugObj */
+Aml *aml_debug(void)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x31); /* DebugOp */
+    return var;
+}
+
+/*
+ * ACPI 1.0b: 16.2.3 Data Objects Encoding:
+ * encodes: ByteConst, WordConst, DWordConst, QWordConst, ZeroOp, OneOp
+ */
+Aml *aml_int(const uint64_t val)
+{
+    Aml *var = aml_alloc();
+    build_append_int(var->buf, val);
+    return var;
+}
+
+/*
+ * helper to construct NameString, which returns Aml object
+ * for using with aml_append or other aml_* terms
+ */
+Aml *aml_name(const char *name_format, ...)
+{
+    va_list ap;
+    Aml *var = aml_alloc();
+    va_start(ap, name_format);
+    build_append_namestringv(var->buf, name_format, ap);
+    va_end(ap);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.1 Namespace Modifier Objects Encoding: DefName */
+Aml *aml_name_decl(const char *name, Aml *val)
+{
+    Aml *var = aml_opcode(0x08 /* NameOp */);
+    build_append_namestring(var->buf, "%s", name);
+    aml_append(var, val);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.6.1 Arg Objects Encoding */
+Aml *aml_arg(int pos)
+{
+    uint8_t op = 0x68 /* ARG0 op */ + pos;
+
+    assert(pos <= 6);
+    return aml_opcode(op);
+}
+
+/* ACPI 2.0a: 17.2.4.4 Type 2 Opcodes Encoding: DefToInteger */
+Aml *aml_to_integer(Aml *arg)
+{
+    Aml *var = aml_opcode(0x99 /* ToIntegerOp */);
+    aml_append(var, arg);
+    build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    return var;
+}
+
+/* ACPI 2.0a: 17.2.4.4 Type 2 Opcodes Encoding: DefToHexString */
+Aml *aml_to_hexstring(Aml *src, Aml *dst)
+{
+    Aml *var = aml_opcode(0x98 /* ToHexStringOp */);
+    aml_append(var, src);
+    if (dst) {
+        aml_append(var, dst);
+    } else {
+        build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    }
+    return var;
+}
+
+/* ACPI 2.0a: 17.2.4.4 Type 2 Opcodes Encoding: DefToBuffer */
+Aml *aml_to_buffer(Aml *src, Aml *dst)
+{
+    Aml *var = aml_opcode(0x96 /* ToBufferOp */);
+    aml_append(var, src);
+    if (dst) {
+        aml_append(var, dst);
+    } else {
+        build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    }
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefStore */
+Aml *aml_store(Aml *val, Aml *target)
+{
+    Aml *var = aml_opcode(0x70 /* StoreOp */);
+    aml_append(var, val);
+    aml_append(var, target);
+    return var;
+}
+
+/**
+ * build_opcode_2arg_dst:
+ * @op: 1-byte opcode
+ * @arg1: 1st operand
+ * @arg2: 2nd operand
+ * @dst: optional target to store to, set to NULL if it's not required
+ *
+ * An internal helper to compose AML terms that have
+ *   "Op Operand Operand Target"
+ * pattern.
+ *
+ * Returns: The newly allocated and composed according to patter Aml object.
+ */
+static Aml *
+build_opcode_2arg_dst(uint8_t op, Aml *arg1, Aml *arg2, Aml *dst)
+{
+    Aml *var = aml_opcode(op);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    if (dst) {
+        aml_append(var, dst);
+    } else {
+        build_append_byte(var->buf, 0x00 /* NullNameOp */);
+    }
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAnd */
+Aml *aml_and(Aml *arg1, Aml *arg2, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x7B /* AndOp */, arg1, arg2, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefOr */
+Aml *aml_or(Aml *arg1, Aml *arg2, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x7D /* OrOp */, arg1, arg2, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLOr */
+Aml *aml_lor(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x91 /* LOrOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefShiftLeft */
+Aml *aml_shiftleft(Aml *arg1, Aml *count)
+{
+    return build_opcode_2arg_dst(0x79 /* ShiftLeftOp */, arg1, count, NULL);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefShiftRight */
+Aml *aml_shiftright(Aml *arg1, Aml *count, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x7A /* ShiftRightOp */, arg1, count, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLLess */
+Aml *aml_lless(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x95 /* LLessOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAdd */
+Aml *aml_add(Aml *arg1, Aml *arg2, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x72 /* AddOp */, arg1, arg2, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefSubtract */
+Aml *aml_subtract(Aml *arg1, Aml *arg2, Aml *dst)
+{
+    return build_opcode_2arg_dst(0x74 /* SubtractOp */, arg1, arg2, dst);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefIncrement */
+Aml *aml_increment(Aml *arg)
+{
+    Aml *var = aml_opcode(0x75 /* IncrementOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefDecrement */
+Aml *aml_decrement(Aml *arg)
+{
+    Aml *var = aml_opcode(0x76 /* DecrementOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefIndex */
+Aml *aml_index(Aml *arg1, Aml *idx)
+{
+    return build_opcode_2arg_dst(0x88 /* IndexOp */, arg1, idx, NULL);
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefNotify */
+Aml *aml_notify(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x86 /* NotifyOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* helper to call method with 1 argument */
+Aml *aml_call0(const char *method)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    return var;
+}
+
+/* helper to call method with 1 argument */
+Aml *aml_call1(const char *method, Aml *arg1)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    aml_append(var, arg1);
+    return var;
+}
+
+/* helper to call method with 2 arguments */
+Aml *aml_call2(const char *method, Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* helper to call method with 3 arguments */
+Aml *aml_call3(const char *method, Aml *arg1, Aml *arg2, Aml *arg3)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    aml_append(var, arg3);
+    return var;
+}
+
+/* helper to call method with 4 arguments */
+Aml *aml_call4(const char *method, Aml *arg1, Aml *arg2, Aml *arg3, Aml *arg4)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    aml_append(var, arg3);
+    aml_append(var, arg4);
+    return var;
+}
+
+/* helper to call method with 5 arguments */
+Aml *aml_call5(const char *method, Aml *arg1, Aml *arg2, Aml *arg3, Aml *arg4,
+               Aml *arg5)
+{
+    Aml *var = aml_alloc();
+    build_append_namestring(var->buf, "%s", method);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    aml_append(var, arg3);
+    aml_append(var, arg4);
+    aml_append(var, arg5);
+    return var;
+}
+
+/*
+ * ACPI 5.0: 6.4.3.8.1 GPIO Connection Descriptor
+ * Type 1, Large Item Name 0xC
+ */
+
+static Aml *aml_gpio_connection(AmlGpioConnectionType type,
+                                AmlConsumerAndProducer con_and_pro,
+                                uint8_t flags, AmlPinConfig pin_config,
+                                uint16_t output_drive,
+                                uint16_t debounce_timeout,
+                                const uint32_t pin_list[], uint32_t pin_count,
+                                const char *resource_source_name,
+                                const uint8_t *vendor_data,
+                                uint16_t vendor_data_len)
+{
+    Aml *var = aml_alloc();
+    const uint16_t min_desc_len = 0x16;
+    uint16_t resource_source_name_len, length;
+    uint16_t pin_table_offset, resource_source_name_offset, vendor_data_offset;
+    uint32_t i;
+
+    assert(resource_source_name);
+    resource_source_name_len = strlen(resource_source_name) + 1;
+    length = min_desc_len + resource_source_name_len + vendor_data_len;
+    pin_table_offset = min_desc_len + 1;
+    resource_source_name_offset = pin_table_offset + pin_count * 2;
+    vendor_data_offset = resource_source_name_offset + resource_source_name_len;
+
+    build_append_byte(var->buf, 0x8C);  /* GPIO Connection Descriptor */
+    build_append_int_noprefix(var->buf, length, 2); /* Length */
+    build_append_byte(var->buf, 1);     /* Revision ID */
+    build_append_byte(var->buf, type);  /* GPIO Connection Type */
+    /* General Flags (2 bytes) */
+    build_append_int_noprefix(var->buf, con_and_pro, 2);
+    /* Interrupt and IO Flags (2 bytes) */
+    build_append_int_noprefix(var->buf, flags, 2);
+    /* Pin Configuration 0 = Default 1 = Pull-up 2 = Pull-down 3 = No Pull */
+    build_append_byte(var->buf, pin_config);
+    /* Output Drive Strength (2 bytes) */
+    build_append_int_noprefix(var->buf, output_drive, 2);
+    /* Debounce Timeout (2 bytes) */
+    build_append_int_noprefix(var->buf, debounce_timeout, 2);
+    /* Pin Table Offset (2 bytes) */
+    build_append_int_noprefix(var->buf, pin_table_offset, 2);
+    build_append_byte(var->buf, 0);     /* Resource Source Index */
+    /* Resource Source Name Offset (2 bytes) */
+    build_append_int_noprefix(var->buf, resource_source_name_offset, 2);
+    /* Vendor Data Offset (2 bytes) */
+    build_append_int_noprefix(var->buf, vendor_data_offset, 2);
+    /* Vendor Data Length (2 bytes) */
+    build_append_int_noprefix(var->buf, vendor_data_len, 2);
+    /* Pin Number (2n bytes)*/
+    for (i = 0; i < pin_count; i++) {
+        build_append_int_noprefix(var->buf, pin_list[i], 2);
+    }
+
+    /* Resource Source Name */
+    build_append_namestring(var->buf, "%s", resource_source_name);
+    build_append_byte(var->buf, '\0');
+
+    /* Vendor-defined Data */
+    if (vendor_data != NULL) {
+        g_array_append_vals(var->buf, vendor_data, vendor_data_len);
+    }
+
+    return var;
+}
+
+/*
+ * ACPI 5.0: 19.5.53
+ * GpioInt(GPIO Interrupt Connection Resource Descriptor Macro)
+ */
+Aml *aml_gpio_int(AmlConsumerAndProducer con_and_pro,
+                  AmlLevelAndEdge edge_level,
+                  AmlActiveHighAndLow active_level, AmlShared shared,
+                  AmlPinConfig pin_config, uint16_t debounce_timeout,
+                  const uint32_t pin_list[], uint32_t pin_count,
+                  const char *resource_source_name,
+                  const uint8_t *vendor_data, uint16_t vendor_data_len)
+{
+    uint8_t flags = edge_level | (active_level << 1) | (shared << 3);
+
+    return aml_gpio_connection(AML_INTERRUPT_CONNECTION, con_and_pro, flags,
+                               pin_config, 0, debounce_timeout, pin_list,
+                               pin_count, resource_source_name, vendor_data,
+                               vendor_data_len);
+}
+
+/*
+ * ACPI 1.0b: 6.4.3.4 32-Bit Fixed Location Memory Range Descriptor
+ * (Type 1, Large Item Name 0x6)
+ */
+Aml *aml_memory32_fixed(uint32_t addr, uint32_t size,
+                        AmlReadAndWrite read_and_write)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x86); /* Memory32Fixed Resource Descriptor */
+    build_append_byte(var->buf, 9);    /* Length, bits[7:0] value = 9 */
+    build_append_byte(var->buf, 0);    /* Length, bits[15:8] value = 0 */
+    build_append_byte(var->buf, read_and_write); /* Write status, 1 rw 0 ro */
+
+    /* Range base address */
+    build_append_byte(var->buf, extract32(addr, 0, 8));  /* bits[7:0] */
+    build_append_byte(var->buf, extract32(addr, 8, 8));  /* bits[15:8] */
+    build_append_byte(var->buf, extract32(addr, 16, 8)); /* bits[23:16] */
+    build_append_byte(var->buf, extract32(addr, 24, 8)); /* bits[31:24] */
+
+    /* Range length */
+    build_append_byte(var->buf, extract32(size, 0, 8));  /* bits[7:0] */
+    build_append_byte(var->buf, extract32(size, 8, 8));  /* bits[15:8] */
+    build_append_byte(var->buf, extract32(size, 16, 8)); /* bits[23:16] */
+    build_append_byte(var->buf, extract32(size, 24, 8)); /* bits[31:24] */
+    return var;
+}
+
+/*
+ * ACPI 5.0: 6.4.3.6 Extended Interrupt Descriptor
+ * Type 1, Large Item Name 0x9
+ */
+Aml *aml_interrupt(AmlConsumerAndProducer con_and_pro,
+                   AmlLevelAndEdge level_and_edge,
+                   AmlActiveHighAndLow high_and_low, AmlShared shared,
+                   uint32_t *irq_list, uint8_t irq_count)
+{
+    int i;
+    Aml *var = aml_alloc();
+    uint8_t irq_flags = con_and_pro | (level_and_edge << 1)
+                        | (high_and_low << 2) | (shared << 3);
+    const int header_bytes_in_len = 2;
+    uint16_t len = header_bytes_in_len + irq_count * sizeof(uint32_t);
+
+    assert(irq_count > 0);
+
+    build_append_byte(var->buf, 0x89); /* Extended irq descriptor */
+    build_append_byte(var->buf, len & 0xFF); /* Length, bits[7:0] */
+    build_append_byte(var->buf, len >> 8); /* Length, bits[15:8] */
+    build_append_byte(var->buf, irq_flags); /* Interrupt Vector Information. */
+    build_append_byte(var->buf, irq_count);   /* Interrupt table length */
+
+    /* Interrupt Number List */
+    for (i = 0; i < irq_count; i++) {
+        build_append_int_noprefix(var->buf, irq_list[i], 4);
+    }
+    return var;
+}
+
+/* ACPI 1.0b: 6.4.2.5 I/O Port Descriptor */
+Aml *aml_io(AmlIODecode dec, uint16_t min_base, uint16_t max_base,
+            uint8_t aln, uint8_t len)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x47); /* IO port descriptor */
+    build_append_byte(var->buf, dec);
+    build_append_byte(var->buf, min_base & 0xff);
+    build_append_byte(var->buf, (min_base >> 8) & 0xff);
+    build_append_byte(var->buf, max_base & 0xff);
+    build_append_byte(var->buf, (max_base >> 8) & 0xff);
+    build_append_byte(var->buf, aln);
+    build_append_byte(var->buf, len);
+    return var;
+}
+
+/*
+ * ACPI 1.0b: 6.4.2.1.1 ASL Macro for IRQ Descriptor
+ *
+ * More verbose description at:
+ * ACPI 5.0: 19.5.64 IRQNoFlags (Interrupt Resource Descriptor Macro)
+ *           6.4.2.1 IRQ Descriptor
+ */
+Aml *aml_irq_no_flags(uint8_t irq)
+{
+    uint16_t irq_mask;
+    Aml *var = aml_alloc();
+
+    assert(irq < 16);
+    build_append_byte(var->buf, 0x22); /* IRQ descriptor 2 byte form */
+
+    irq_mask = 1U << irq;
+    build_append_byte(var->buf, irq_mask & 0xFF); /* IRQ mask bits[7:0] */
+    build_append_byte(var->buf, irq_mask >> 8); /* IRQ mask bits[15:8] */
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLNot */
+Aml *aml_lnot(Aml *arg)
+{
+    Aml *var = aml_opcode(0x92 /* LNotOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLEqual */
+Aml *aml_equal(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x93 /* LequalOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLGreater */
+Aml *aml_lgreater(Aml *arg1, Aml *arg2)
+{
+    Aml *var = aml_opcode(0x94 /* LGreaterOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefLGreaterEqual */
+Aml *aml_lgreater_equal(Aml *arg1, Aml *arg2)
+{
+    /* LGreaterEqualOp := LNotOp LLessOp */
+    Aml *var = aml_opcode(0x92 /* LNotOp */);
+    build_append_byte(var->buf, 0x95 /* LLessOp */);
+    aml_append(var, arg1);
+    aml_append(var, arg2);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefIfElse */
+Aml *aml_if(Aml *predicate)
+{
+    Aml *var = aml_bundle(0xA0 /* IfOp */, AML_PACKAGE);
+    aml_append(var, predicate);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefElse */
+Aml *aml_else(void)
+{
+    Aml *var = aml_bundle(0xA1 /* ElseOp */, AML_PACKAGE);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefWhile */
+Aml *aml_while(Aml *predicate)
+{
+    Aml *var = aml_bundle(0xA2 /* WhileOp */, AML_PACKAGE);
+    aml_append(var, predicate);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefMethod */
+Aml *aml_method(const char *name, int arg_count, AmlSerializeFlag sflag)
+{
+    Aml *var = aml_bundle(0x14 /* MethodOp */, AML_PACKAGE);
+    int methodflags;
+
+    /*
+     * MethodFlags:
+     *   bit 0-2: ArgCount (0-7)
+     *   bit 3: SerializeFlag
+     *     0: NotSerialized
+     *     1: Serialized
+     *   bit 4-7: reserved (must be 0)
+     */
+    assert(arg_count < 8);
+    methodflags = arg_count | (sflag << 3);
+
+    build_append_namestring(var->buf, "%s", name);
+    build_append_byte(var->buf, methodflags); /* MethodFlags: ArgCount */
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefDevice */
+Aml *aml_device(const char *name_format, ...)
+{
+    va_list ap;
+    Aml *var = aml_bundle(0x82 /* DeviceOp */, AML_EXT_PACKAGE);
+    va_start(ap, name_format);
+    build_append_namestringv(var->buf, name_format, ap);
+    va_end(ap);
+    return var;
+}
+
+/* ACPI 1.0b: 6.4.1 ASL Macros for Resource Descriptors */
+Aml *aml_resource_template(void)
+{
+    /* ResourceTemplate is a buffer of Resources with EndTag at the end */
+    Aml *var = aml_bundle(0x11 /* BufferOp */, AML_RES_TEMPLATE);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefBuffer
+ * Pass byte_list as NULL to request uninitialized buffer to reserve space.
+ */
+Aml *aml_buffer(int buffer_size, uint8_t *byte_list)
+{
+    int i;
+    Aml *var = aml_bundle(0x11 /* BufferOp */, AML_BUFFER);
+
+    for (i = 0; i < buffer_size; i++) {
+        if (byte_list == NULL) {
+            build_append_byte(var->buf, 0x0);
+        } else {
+            build_append_byte(var->buf, byte_list[i]);
+        }
+    }
+
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefPackage */
+Aml *aml_package(uint8_t num_elements)
+{
+    Aml *var = aml_bundle(0x12 /* PackageOp */, AML_PACKAGE);
+    build_append_byte(var->buf, num_elements);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefOpRegion */
+Aml *aml_operation_region(const char *name, AmlRegionSpace rs,
+                          Aml *offset, uint32_t len)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x80); /* OpRegionOp */
+    build_append_namestring(var->buf, "%s", name);
+    build_append_byte(var->buf, rs);
+    aml_append(var, offset);
+    build_append_int(var->buf, len);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: NamedField */
+Aml *aml_named_field(const char *name, unsigned length)
+{
+    Aml *var = aml_alloc();
+    build_append_nameseg(var->buf, name);
+    build_append_pkg_length(var->buf, length, false);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: ReservedField */
+Aml *aml_reserved_field(unsigned length)
+{
+    Aml *var = aml_alloc();
+    /* ReservedField  := 0x00 PkgLength */
+    build_append_byte(var->buf, 0x00);
+    build_append_pkg_length(var->buf, length, false);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefField */
+Aml *aml_field(const char *name, AmlAccessType type, AmlLockRule lock,
+               AmlUpdateRule rule)
+{
+    Aml *var = aml_bundle(0x81 /* FieldOp */, AML_EXT_PACKAGE);
+    uint8_t flags = rule << 5 | type;
+
+    flags |= lock << 4; /* LockRule at 4 bit offset */
+
+    build_append_namestring(var->buf, "%s", name);
+    build_append_byte(var->buf, flags);
+    return var;
+}
+
+static
+Aml *create_field_common(int opcode, Aml *srcbuf, Aml *index, const char *name)
+{
+    Aml *var = aml_opcode(opcode);
+    aml_append(var, srcbuf);
+    aml_append(var, index);
+    build_append_namestring(var->buf, "%s", name);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefCreateField */
+Aml *aml_create_field(Aml *srcbuf, Aml *bit_index, Aml *num_bits,
+                      const char *name)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x13); /* CreateFieldOp */
+    aml_append(var, srcbuf);
+    aml_append(var, bit_index);
+    aml_append(var, num_bits);
+    build_append_namestring(var->buf, "%s", name);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefCreateDWordField */
+Aml *aml_create_dword_field(Aml *srcbuf, Aml *index, const char *name)
+{
+    return create_field_common(0x8A /* CreateDWordFieldOp */,
+                               srcbuf, index, name);
+}
+
+/* ACPI 2.0a: 17.2.4.2 Named Objects Encoding: DefCreateQWordField */
+Aml *aml_create_qword_field(Aml *srcbuf, Aml *index, const char *name)
+{
+    return create_field_common(0x8F /* CreateQWordFieldOp */,
+                               srcbuf, index, name);
+}
+
+/* ACPI 1.0b: 16.2.3 Data Objects Encoding: String */
+Aml *aml_string(const char *name_format, ...)
+{
+    Aml *var = aml_opcode(0x0D /* StringPrefix */);
+    va_list ap;
+    char *s;
+    int len;
+
+    va_start(ap, name_format);
+    len = g_vasprintf(&s, name_format, ap);
+    va_end(ap);
+
+    g_array_append_vals(var->buf, s, len + 1);
+    g_free(s);
+
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.6.2 Local Objects Encoding */
+Aml *aml_local(int num)
+{
+    uint8_t op = 0x60 /* Local0Op */ + num;
+
+    assert(num <= 7);
+    return aml_opcode(op);
+}
+
+/* ACPI 2.0a: 17.2.2 Data Objects Encoding: DefVarPackage */
+Aml *aml_varpackage(uint32_t num_elements)
+{
+    Aml *var = aml_bundle(0x13 /* VarPackageOp */, AML_PACKAGE);
+    build_append_int(var->buf, num_elements);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefProcessor */
+Aml *aml_processor(uint8_t proc_id, uint32_t pblk_addr, uint8_t pblk_len,
+                   const char *name_format, ...)
+{
+    va_list ap;
+    Aml *var = aml_bundle(0x83 /* ProcessorOp */, AML_EXT_PACKAGE);
+    va_start(ap, name_format);
+    build_append_namestringv(var->buf, name_format, ap);
+    va_end(ap);
+    build_append_byte(var->buf, proc_id); /* ProcID */
+    build_append_int_noprefix(var->buf, pblk_addr, sizeof(pblk_addr));
+    build_append_byte(var->buf, pblk_len); /* PblkLen */
+    return var;
+}
+
+static uint8_t Hex2Digit(char c)
+{
+    if (c >= 'A') {
+        return c - 'A' + 10;
+    }
+
+    return c - '0';
+}
+
+/* ACPI 1.0b: 15.2.3.6.4.1 EISAID Macro - Convert EISA ID String To Integer */
+Aml *aml_eisaid(const char *str)
+{
+    Aml *var = aml_alloc();
+    uint32_t id;
+
+    g_assert(strlen(str) == 7);
+    id = (str[0] - 0x40) << 26 |
+    (str[1] - 0x40) << 21 |
+    (str[2] - 0x40) << 16 |
+    Hex2Digit(str[3]) << 12 |
+    Hex2Digit(str[4]) << 8 |
+    Hex2Digit(str[5]) << 4 |
+    Hex2Digit(str[6]);
+
+    build_append_byte(var->buf, 0x0C); /* DWordPrefix */
+    build_append_int_noprefix(var->buf, bswap32(id), sizeof(id));
+    return var;
+}
+
+/* ACPI 1.0b: 6.4.3.5.5 Word Address Space Descriptor: bytes 3-5 */
+static Aml *aml_as_desc_header(AmlResourceType type, AmlMinFixed min_fixed,
+                               AmlMaxFixed max_fixed, AmlDecode dec,
+                               uint8_t type_flags)
+{
+    uint8_t flags = max_fixed | min_fixed | dec;
+    Aml *var = aml_alloc();
+
+    build_append_byte(var->buf, type);
+    build_append_byte(var->buf, flags);
+    build_append_byte(var->buf, type_flags); /* Type Specific Flags */
+    return var;
+}
+
+/* ACPI 1.0b: 6.4.3.5.5 Word Address Space Descriptor */
+static Aml *aml_word_as_desc(AmlResourceType type, AmlMinFixed min_fixed,
+                             AmlMaxFixed max_fixed, AmlDecode dec,
+                             uint16_t addr_gran, uint16_t addr_min,
+                             uint16_t addr_max, uint16_t addr_trans,
+                             uint16_t len, uint8_t type_flags)
+{
+    Aml *var = aml_alloc();
+
+    build_append_byte(var->buf, 0x88); /* Word Address Space Descriptor */
+    /* minimum length since we do not encode optional fields */
+    build_append_byte(var->buf, 0x0D);
+    build_append_byte(var->buf, 0x0);
+
+    aml_append(var,
+        aml_as_desc_header(type, min_fixed, max_fixed, dec, type_flags));
+    build_append_int_noprefix(var->buf, addr_gran, sizeof(addr_gran));
+    build_append_int_noprefix(var->buf, addr_min, sizeof(addr_min));
+    build_append_int_noprefix(var->buf, addr_max, sizeof(addr_max));
+    build_append_int_noprefix(var->buf, addr_trans, sizeof(addr_trans));
+    build_append_int_noprefix(var->buf, len, sizeof(len));
+    return var;
+}
+
+/* ACPI 1.0b: 6.4.3.5.3 DWord Address Space Descriptor */
+static Aml *aml_dword_as_desc(AmlResourceType type, AmlMinFixed min_fixed,
+                              AmlMaxFixed max_fixed, AmlDecode dec,
+                              uint32_t addr_gran, uint32_t addr_min,
+                              uint32_t addr_max, uint32_t addr_trans,
+                              uint32_t len, uint8_t type_flags)
+{
+    Aml *var = aml_alloc();
+
+    build_append_byte(var->buf, 0x87); /* DWord Address Space Descriptor */
+    /* minimum length since we do not encode optional fields */
+    build_append_byte(var->buf, 23);
+    build_append_byte(var->buf, 0x0);
+
+
+    aml_append(var,
+        aml_as_desc_header(type, min_fixed, max_fixed, dec, type_flags));
+    build_append_int_noprefix(var->buf, addr_gran, sizeof(addr_gran));
+    build_append_int_noprefix(var->buf, addr_min, sizeof(addr_min));
+    build_append_int_noprefix(var->buf, addr_max, sizeof(addr_max));
+    build_append_int_noprefix(var->buf, addr_trans, sizeof(addr_trans));
+    build_append_int_noprefix(var->buf, len, sizeof(len));
+    return var;
+}
+
+/* ACPI 1.0b: 6.4.3.5.1 QWord Address Space Descriptor */
+static Aml *aml_qword_as_desc(AmlResourceType type, AmlMinFixed min_fixed,
+                              AmlMaxFixed max_fixed, AmlDecode dec,
+                              uint64_t addr_gran, uint64_t addr_min,
+                              uint64_t addr_max, uint64_t addr_trans,
+                              uint64_t len, uint8_t type_flags)
+{
+    Aml *var = aml_alloc();
+
+    build_append_byte(var->buf, 0x8A); /* QWord Address Space Descriptor */
+    /* minimum length since we do not encode optional fields */
+    build_append_byte(var->buf, 0x2B);
+    build_append_byte(var->buf, 0x0);
+
+    aml_append(var,
+        aml_as_desc_header(type, min_fixed, max_fixed, dec, type_flags));
+    build_append_int_noprefix(var->buf, addr_gran, sizeof(addr_gran));
+    build_append_int_noprefix(var->buf, addr_min, sizeof(addr_min));
+    build_append_int_noprefix(var->buf, addr_max, sizeof(addr_max));
+    build_append_int_noprefix(var->buf, addr_trans, sizeof(addr_trans));
+    build_append_int_noprefix(var->buf, len, sizeof(len));
+    return var;
+}
+
+/*
+ * ACPI 1.0b: 6.4.3.5.6 ASL Macros for WORD Address Descriptor
+ *
+ * More verbose description at:
+ * ACPI 5.0: 19.5.141 WordBusNumber (Word Bus Number Resource Descriptor Macro)
+ */
+Aml *aml_word_bus_number(AmlMinFixed min_fixed, AmlMaxFixed max_fixed,
+                         AmlDecode dec, uint16_t addr_gran,
+                         uint16_t addr_min, uint16_t addr_max,
+                         uint16_t addr_trans, uint16_t len)
+
+{
+    return aml_word_as_desc(AML_BUS_NUMBER_RANGE, min_fixed, max_fixed, dec,
+                            addr_gran, addr_min, addr_max, addr_trans, len, 0);
+}
+
+/*
+ * ACPI 1.0b: 6.4.3.5.6 ASL Macros for WORD Address Descriptor
+ *
+ * More verbose description at:
+ * ACPI 5.0: 19.5.142 WordIO (Word IO Resource Descriptor Macro)
+ */
+Aml *aml_word_io(AmlMinFixed min_fixed, AmlMaxFixed max_fixed,
+                 AmlDecode dec, AmlISARanges isa_ranges,
+                 uint16_t addr_gran, uint16_t addr_min,
+                 uint16_t addr_max, uint16_t addr_trans,
+                 uint16_t len)
+
+{
+    return aml_word_as_desc(AML_IO_RANGE, min_fixed, max_fixed, dec,
+                            addr_gran, addr_min, addr_max, addr_trans, len,
+                            isa_ranges);
+}
+
+/*
+ * ACPI 1.0b: 6.4.3.5.4 ASL Macros for DWORD Address Descriptor
+ *
+ * More verbose description at:
+ * ACPI 5.0: 19.5.33 DWordIO (DWord IO Resource Descriptor Macro)
+ */
+Aml *aml_dword_io(AmlMinFixed min_fixed, AmlMaxFixed max_fixed,
+                 AmlDecode dec, AmlISARanges isa_ranges,
+                 uint32_t addr_gran, uint32_t addr_min,
+                 uint32_t addr_max, uint32_t addr_trans,
+                 uint32_t len)
+
+{
+    return aml_dword_as_desc(AML_IO_RANGE, min_fixed, max_fixed, dec,
+                            addr_gran, addr_min, addr_max, addr_trans, len,
+                            isa_ranges);
+}
+
+/*
+ * ACPI 1.0b: 6.4.3.5.4 ASL Macros for DWORD Address Space Descriptor
+ *
+ * More verbose description at:
+ * ACPI 5.0: 19.5.34 DWordMemory (DWord Memory Resource Descriptor Macro)
+ */
+Aml *aml_dword_memory(AmlDecode dec, AmlMinFixed min_fixed,
+                      AmlMaxFixed max_fixed, AmlCacheable cacheable,
+                      AmlReadAndWrite read_and_write,
+                      uint32_t addr_gran, uint32_t addr_min,
+                      uint32_t addr_max, uint32_t addr_trans,
+                      uint32_t len)
+{
+    uint8_t flags = read_and_write | (cacheable << 1);
+
+    return aml_dword_as_desc(AML_MEMORY_RANGE, min_fixed, max_fixed,
+                             dec, addr_gran, addr_min, addr_max,
+                             addr_trans, len, flags);
+}
+
+/*
+ * ACPI 1.0b: 6.4.3.5.2 ASL Macros for QWORD Address Space Descriptor
+ *
+ * More verbose description at:
+ * ACPI 5.0: 19.5.102 QWordMemory (QWord Memory Resource Descriptor Macro)
+ */
+Aml *aml_qword_memory(AmlDecode dec, AmlMinFixed min_fixed,
+                      AmlMaxFixed max_fixed, AmlCacheable cacheable,
+                      AmlReadAndWrite read_and_write,
+                      uint64_t addr_gran, uint64_t addr_min,
+                      uint64_t addr_max, uint64_t addr_trans,
+                      uint64_t len)
+{
+    uint8_t flags = read_and_write | (cacheable << 1);
+
+    return aml_qword_as_desc(AML_MEMORY_RANGE, min_fixed, max_fixed,
+                             dec, addr_gran, addr_min, addr_max,
+                             addr_trans, len, flags);
+}
+
+/* ACPI 1.0b: 6.4.2.2 DMA Format/6.4.2.2.1 ASL Macro for DMA Descriptor */
+Aml *aml_dma(AmlDmaType typ, AmlDmaBusMaster bm, AmlTransferSize sz,
+             uint8_t channel)
+{
+    Aml *var = aml_alloc();
+    uint8_t flags = sz | bm << 2 | typ << 5;
+
+    assert(channel < 8);
+    build_append_byte(var->buf, 0x2A);    /* Byte 0: DMA Descriptor */
+    build_append_byte(var->buf, 1U << channel); /* Byte 1: _DMA - DmaChannel */
+    build_append_byte(var->buf, flags);   /* Byte 2 */
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefSleep */
+Aml *aml_sleep(uint64_t msec)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x22); /* SleepOp */
+    aml_append(var, aml_int(msec));
+    return var;
+}
+
+static uint8_t Hex2Byte(const char *src)
+{
+    int hi, lo;
+
+    hi = Hex2Digit(src[0]);
+    assert(hi >= 0);
+    assert(hi <= 15);
+
+    lo = Hex2Digit(src[1]);
+    assert(lo >= 0);
+    assert(lo <= 15);
+    return (hi << 4) | lo;
+}
+
+/*
+ * ACPI 3.0: 17.5.124 ToUUID (Convert String to UUID Macro)
+ * e.g. UUID: aabbccdd-eeff-gghh-iijj-kkllmmnnoopp
+ * call aml_touuid("aabbccdd-eeff-gghh-iijj-kkllmmnnoopp");
+ */
+Aml *aml_touuid(const char *uuid)
+{
+    Aml *var = aml_bundle(0x11 /* BufferOp */, AML_BUFFER);
+
+    assert(strlen(uuid) == 36);
+    assert(uuid[8] == '-');
+    assert(uuid[13] == '-');
+    assert(uuid[18] == '-');
+    assert(uuid[23] == '-');
+
+    build_append_byte(var->buf, Hex2Byte(uuid + 6));  /* dd - at offset 00 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 4));  /* cc - at offset 01 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 2));  /* bb - at offset 02 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 0));  /* aa - at offset 03 */
+
+    build_append_byte(var->buf, Hex2Byte(uuid + 11)); /* ff - at offset 04 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 9));  /* ee - at offset 05 */
+
+    build_append_byte(var->buf, Hex2Byte(uuid + 16)); /* hh - at offset 06 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 14)); /* gg - at offset 07 */
+
+    build_append_byte(var->buf, Hex2Byte(uuid + 19)); /* ii - at offset 08 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 21)); /* jj - at offset 09 */
+
+    build_append_byte(var->buf, Hex2Byte(uuid + 24)); /* kk - at offset 10 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 26)); /* ll - at offset 11 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 28)); /* mm - at offset 12 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 30)); /* nn - at offset 13 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 32)); /* oo - at offset 14 */
+    build_append_byte(var->buf, Hex2Byte(uuid + 34)); /* pp - at offset 15 */
+
+    return var;
+}
+
+/*
+ * ACPI 2.0b: 16.2.3.6.4.3  Unicode Macro (Convert Ascii String To Unicode)
+ */
+Aml *aml_unicode(const char *str)
+{
+    int i = 0;
+    Aml *var = aml_bundle(0x11 /* BufferOp */, AML_BUFFER);
+
+    do {
+        build_append_byte(var->buf, str[i]);
+        build_append_byte(var->buf, 0);
+        i++;
+    } while (i <= strlen(str));
+
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefRefOf */
+Aml *aml_refof(Aml *arg)
+{
+    Aml *var = aml_opcode(0x71 /* RefOfOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefDerefOf */
+Aml *aml_derefof(Aml *arg)
+{
+    Aml *var = aml_opcode(0x83 /* DerefOfOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefSizeOf */
+Aml *aml_sizeof(Aml *arg)
+{
+    Aml *var = aml_opcode(0x87 /* SizeOfOp */);
+    aml_append(var, arg);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: DefMutex */
+Aml *aml_mutex(const char *name, uint8_t sync_level)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x01); /* MutexOp */
+    build_append_namestring(var->buf, "%s", name);
+    assert(!(sync_level & 0xF0));
+    build_append_byte(var->buf, sync_level);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefAcquire */
+Aml *aml_acquire(Aml *mutex, uint16_t timeout)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x23); /* AcquireOp */
+    aml_append(var, mutex);
+    build_append_int_noprefix(var->buf, timeout, sizeof(timeout));
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.3 Type 1 Opcodes Encoding: DefRelease */
+Aml *aml_release(Aml *mutex)
+{
+    Aml *var = aml_alloc();
+    build_append_byte(var->buf, 0x5B); /* ExtOpPrefix */
+    build_append_byte(var->buf, 0x27); /* ReleaseOp */
+    aml_append(var, mutex);
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.1 Name Space Modifier Objects Encoding: DefAlias */
+Aml *aml_alias(const char *source_object, const char *alias_object)
+{
+    Aml *var = aml_opcode(0x06 /* AliasOp */);
+    aml_append(var, aml_name("%s", source_object));
+    aml_append(var, aml_name("%s", alias_object));
+    return var;
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefConcat */
+Aml *aml_concatenate(Aml *source1, Aml *source2, Aml *target)
+{
+    return build_opcode_2arg_dst(0x73 /* ConcatOp */, source1, source2,
+                                 target);
+}
+
+/* ACPI 1.0b: 16.2.5.4 Type 2 Opcodes Encoding: DefObjectType */
+Aml *aml_object_type(Aml *object)
+{
+    Aml *var = aml_opcode(0x8E /* ObjectTypeOp */);
+    aml_append(var, object);
+    return var;
+}
+
+void
+build_header(BIOSLinker *linker, GArray *table_data,
+             AcpiTableHeader *h, const char *sig, int len, uint8_t rev,
+             const char *oem_id, const char *oem_table_id)
+{
+    unsigned tbl_offset = (char *)h - table_data->data;
+    unsigned checksum_offset = (char *)&h->checksum - table_data->data;
+    memcpy(&h->signature, sig, 4);
+    h->length = cpu_to_le32(len);
+    h->revision = rev;
+
+    if (oem_id) {
+        strncpy((char *)h->oem_id, oem_id, sizeof h->oem_id);
+    } else {
+        memcpy(h->oem_id, ACPI_BUILD_APPNAME6, 6);
+    }
+
+    if (oem_table_id) {
+        strncpy((char *)h->oem_table_id, oem_table_id, sizeof(h->oem_table_id));
+    } else {
+        memcpy(h->oem_table_id, ACPI_BUILD_APPNAME4, 4);
+        memcpy(h->oem_table_id + 4, sig, 4);
+    }
+
+    h->oem_revision = cpu_to_le32(1);
+    memcpy(h->asl_compiler_id, ACPI_BUILD_APPNAME4, 4);
+    h->asl_compiler_revision = cpu_to_le32(1);
+    /* Checksum to be filled in by Guest linker */
+    bios_linker_loader_add_checksum(linker, ACPI_BUILD_TABLE_FILE,
+        tbl_offset, len, checksum_offset);
+}
+
+void *acpi_data_push(GArray *table_data, unsigned size)
+{
+    unsigned off = table_data->len;
+    g_array_set_size(table_data, off + size);
+    return table_data->data + off;
+}
+
+unsigned acpi_data_len(GArray *table)
+{
+    assert(g_array_get_element_size(table) == 1);
+    return table->len;
+}
+
+void acpi_add_table(GArray *table_offsets, GArray *table_data)
+{
+    uint32_t offset = table_data->len;
+    g_array_append_val(table_offsets, offset);
+}
+
+void acpi_build_tables_init(AcpiBuildTables *tables)
+{
+    tables->rsdp = g_array_new(false, true /* clear */, 1);
+    tables->table_data = g_array_new(false, true /* clear */, 1);
+    tables->tcpalog = g_array_new(false, true /* clear */, 1);
+    tables->linker = bios_linker_loader_init();
+}
+
+void acpi_build_tables_cleanup(AcpiBuildTables *tables, bool mfre)
+{
+    bios_linker_loader_cleanup(tables->linker);
+    g_array_free(tables->rsdp, true);
+    g_array_free(tables->table_data, true);
+    g_array_free(tables->tcpalog, mfre);
+}
+
+/* Build rsdt table */
+void
+build_rsdt(GArray *table_data, BIOSLinker *linker, GArray *table_offsets,
+           const char *oem_id, const char *oem_table_id)
+{
+    int i;
+    unsigned rsdt_entries_offset;
+    AcpiRsdtDescriptorRev1 *rsdt;
+    const unsigned table_data_len = (sizeof(uint32_t) * table_offsets->len);
+    const unsigned rsdt_entry_size = sizeof(rsdt->table_offset_entry[0]);
+    const size_t rsdt_len = sizeof(*rsdt) + table_data_len;
+
+    rsdt = acpi_data_push(table_data, rsdt_len);
+    rsdt_entries_offset = (char *)rsdt->table_offset_entry - table_data->data;
+    for (i = 0; i < table_offsets->len; ++i) {
+        uint32_t ref_tbl_offset = g_array_index(table_offsets, uint32_t, i);
+        uint32_t rsdt_entry_offset = rsdt_entries_offset + rsdt_entry_size * i;
+
+        /* rsdt->table_offset_entry to be filled by Guest linker */
+        bios_linker_loader_add_pointer(linker,
+            ACPI_BUILD_TABLE_FILE, rsdt_entry_offset, rsdt_entry_size,
+            ACPI_BUILD_TABLE_FILE, ref_tbl_offset);
+    }
+    build_header(linker, table_data,
+                 (void *)rsdt, "RSDT", rsdt_len, 1, oem_id, oem_table_id);
+}
+
+void build_srat_memory(AcpiSratMemoryAffinity *numamem, uint64_t base,
+                       uint64_t len, int node, MemoryAffinityFlags flags)
+{
+    numamem->type = ACPI_SRAT_MEMORY;
+    numamem->length = sizeof(*numamem);
+    numamem->proximity = cpu_to_le32(node);
+    numamem->flags = cpu_to_le32(flags);
+    numamem->base_addr = cpu_to_le64(base);
+    numamem->range_length = cpu_to_le64(len);
+}
diff --git a/hw/acpi/bios-linker-loader.c b/hw/acpi/bios-linker-loader.c
new file mode 100644
index 0000000..d963ebe
--- /dev/null
+++ b/hw/acpi/bios-linker-loader.c
@@ -0,0 +1,280 @@
+/* Dynamic linker/loader of ACPI tables
+ *
+ * Copyright (C) 2013 Red Hat Inc
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "hw/acpi/bios-linker-loader.h"
+#include "hw/nvram/fw_cfg.h"
+
+#include "qemu/bswap.h"
+
+/*
+ * Linker/loader is a paravirtualized interface that passes commands to guest.
+ * The commands can be used to request guest to
+ * - allocate memory chunks and initialize them from QEMU FW CFG files
+ * - link allocated chunks by storing pointer to one chunk into another
+ * - calculate ACPI checksum of part of the chunk and store into same chunk
+ */
+#define BIOS_LINKER_LOADER_FILESZ FW_CFG_MAX_FILE_PATH
+
+struct BiosLinkerLoaderEntry {
+    uint32_t command;
+    union {
+        /*
+         * COMMAND_ALLOCATE - allocate a table from @alloc.file
+         * subject to @alloc.align alignment (must be power of 2)
+         * and @alloc.zone (can be HIGH or FSEG) requirements.
+         *
+         * Must appear exactly once for each file, and before
+         * this file is referenced by any other command.
+         */
+        struct {
+            char file[BIOS_LINKER_LOADER_FILESZ];
+            uint32_t align;
+            uint8_t zone;
+        } alloc;
+
+        /*
+         * COMMAND_ADD_POINTER - patch the table (originating from
+         * @dest_file) at @pointer.offset, by adding a pointer to the table
+         * originating from @src_file. 1,2,4 or 8 byte unsigned
+         * addition is used depending on @pointer.size.
+         */
+        struct {
+            char dest_file[BIOS_LINKER_LOADER_FILESZ];
+            char src_file[BIOS_LINKER_LOADER_FILESZ];
+            uint32_t offset;
+            uint8_t size;
+        } pointer;
+
+        /*
+         * COMMAND_ADD_CHECKSUM - calculate checksum of the range specified by
+         * @cksum_start and @cksum_length fields,
+         * and then add the value at @cksum.offset.
+         * Checksum simply sums -X for each byte X in the range
+         * using 8-bit math.
+         */
+        struct {
+            char file[BIOS_LINKER_LOADER_FILESZ];
+            uint32_t offset;
+            uint32_t start;
+            uint32_t length;
+        } cksum;
+
+        /* padding */
+        char pad[124];
+    };
+} QEMU_PACKED;
+typedef struct BiosLinkerLoaderEntry BiosLinkerLoaderEntry;
+
+enum {
+    BIOS_LINKER_LOADER_COMMAND_ALLOCATE     = 0x1,
+    BIOS_LINKER_LOADER_COMMAND_ADD_POINTER  = 0x2,
+    BIOS_LINKER_LOADER_COMMAND_ADD_CHECKSUM = 0x3,
+};
+
+enum {
+    BIOS_LINKER_LOADER_ALLOC_ZONE_HIGH = 0x1,
+    BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG = 0x2,
+};
+
+/*
+ * BiosLinkerFileEntry:
+ *
+ * An internal type used for book-keeping file entries
+ */
+typedef struct BiosLinkerFileEntry {
+    char *name; /* file name */
+    GArray *blob; /* data accosiated with @name */
+} BiosLinkerFileEntry;
+
+/*
+ * bios_linker_loader_init: allocate a new linker object instance.
+ *
+ * After initialization, linker commands can be added, and will
+ * be stored in the linker.cmd_blob array.
+ */
+BIOSLinker *bios_linker_loader_init(void)
+{
+    BIOSLinker *linker = g_new(BIOSLinker, 1);
+
+    linker->cmd_blob = g_array_new(false, true /* clear */, 1);
+    linker->file_list = g_array_new(false, true /* clear */,
+                                    sizeof(BiosLinkerFileEntry));
+    return linker;
+}
+
+/* Free linker wrapper */
+void bios_linker_loader_cleanup(BIOSLinker *linker)
+{
+    int i;
+    BiosLinkerFileEntry *entry;
+
+    g_array_free(linker->cmd_blob, true);
+
+    for (i = 0; i < linker->file_list->len; i++) {
+        entry = &g_array_index(linker->file_list, BiosLinkerFileEntry, i);
+        g_free(entry->name);
+    }
+    g_array_free(linker->file_list, true);
+    g_free(linker);
+}
+
+static const BiosLinkerFileEntry *
+bios_linker_find_file(const BIOSLinker *linker, const char *name)
+{
+    int i;
+    BiosLinkerFileEntry *entry;
+
+    for (i = 0; i < linker->file_list->len; i++) {
+        entry = &g_array_index(linker->file_list, BiosLinkerFileEntry, i);
+        if (!strcmp(entry->name, name)) {
+            return entry;
+        }
+    }
+    return NULL;
+}
+
+/*
+ * bios_linker_loader_alloc: ask guest to load file into guest memory.
+ *
+ * @linker: linker object instance
+ * @file_name: name of the file blob to be loaded
+ * @file_blob: pointer to blob corresponding to @file_name
+ * @alloc_align: required minimal alignment in bytes. Must be a power of 2.
+ * @alloc_fseg: request allocation in FSEG zone (useful for the RSDP ACPI table)
+ *
+ * Note: this command must precede any other linker command using this file.
+ */
+void bios_linker_loader_alloc(BIOSLinker *linker,
+                              const char *file_name,
+                              GArray *file_blob,
+                              uint32_t alloc_align,
+                              bool alloc_fseg)
+{
+    BiosLinkerLoaderEntry entry;
+    BiosLinkerFileEntry file = { g_strdup(file_name), file_blob};
+
+    assert(!(alloc_align & (alloc_align - 1)));
+
+    assert(!bios_linker_find_file(linker, file_name));
+    g_array_append_val(linker->file_list, file);
+
+    memset(&entry, 0, sizeof entry);
+    strncpy(entry.alloc.file, file_name, sizeof entry.alloc.file - 1);
+    entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ALLOCATE);
+    entry.alloc.align = cpu_to_le32(alloc_align);
+    entry.alloc.zone = alloc_fseg ? BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG :
+                                    BIOS_LINKER_LOADER_ALLOC_ZONE_HIGH;
+
+    /* Alloc entries must come first, so prepend them */
+    g_array_prepend_vals(linker->cmd_blob, &entry, sizeof entry);
+}
+
+/*
+ * bios_linker_loader_add_checksum: ask guest to add checksum of ACPI
+ * table in the specified file at the specified offset.
+ *
+ * Checksum calculation simply sums -X for each byte X in the range
+ * using 8-bit math (i.e. ACPI checksum).
+ *
+ * @linker: linker object instance
+ * @file: file that includes the checksum to be calculated
+ *        and the data to be checksummed
+ * @start_offset, @size: range of data in the file to checksum,
+ *                       relative to the start of file blob
+ * @checksum_offset: location of the checksum to be patched within file blob,
+ *                   relative to the start of file blob
+ */
+void bios_linker_loader_add_checksum(BIOSLinker *linker, const char *file_name,
+                                     unsigned start_offset, unsigned size,
+                                     unsigned checksum_offset)
+{
+    BiosLinkerLoaderEntry entry;
+    const BiosLinkerFileEntry *file = bios_linker_find_file(linker, file_name);
+
+    assert(file);
+    assert(start_offset < file->blob->len);
+    assert(start_offset + size <= file->blob->len);
+    assert(checksum_offset >= start_offset);
+    assert(checksum_offset + 1 <= start_offset + size);
+
+    *(file->blob->data + checksum_offset) = 0;
+    memset(&entry, 0, sizeof entry);
+    strncpy(entry.cksum.file, file_name, sizeof entry.cksum.file - 1);
+    entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ADD_CHECKSUM);
+    entry.cksum.offset = cpu_to_le32(checksum_offset);
+    entry.cksum.start = cpu_to_le32(start_offset);
+    entry.cksum.length = cpu_to_le32(size);
+
+    g_array_append_vals(linker->cmd_blob, &entry, sizeof entry);
+}
+
+/*
+ * bios_linker_loader_add_pointer: ask guest to patch address in
+ * destination file with a pointer to source file
+ *
+ * @linker: linker object instance
+ * @dest_file: destination file that must be changed
+ * @dst_patched_offset: location within destination file blob to be patched
+ *                      with the pointer to @src_file+@src_offset (i.e. source
+ *                      blob allocated in guest memory + @src_offset), in bytes
+ * @dst_patched_offset_size: size of the pointer to be patched
+ *                      at @dst_patched_offset in @dest_file blob, in bytes
+ * @src_file: source file who's address must be taken
+ * @src_offset: location within source file blob to which
+ *              @dest_file+@dst_patched_offset will point to after
+ *              firmware's executed ADD_POINTER command
+ */
+void bios_linker_loader_add_pointer(BIOSLinker *linker,
+                                    const char *dest_file,
+                                    uint32_t dst_patched_offset,
+                                    uint8_t dst_patched_size,
+                                    const char *src_file,
+                                    uint32_t src_offset)
+{
+    uint64_t le_src_offset;
+    BiosLinkerLoaderEntry entry;
+    const BiosLinkerFileEntry *dst_file =
+        bios_linker_find_file(linker, dest_file);
+    const BiosLinkerFileEntry *source_file =
+        bios_linker_find_file(linker, src_file);
+
+    assert(dst_patched_offset < dst_file->blob->len);
+    assert(dst_patched_offset + dst_patched_size <= dst_file->blob->len);
+    assert(src_offset < source_file->blob->len);
+
+    memset(&entry, 0, sizeof entry);
+    strncpy(entry.pointer.dest_file, dest_file,
+            sizeof entry.pointer.dest_file - 1);
+    strncpy(entry.pointer.src_file, src_file,
+            sizeof entry.pointer.src_file - 1);
+    entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ADD_POINTER);
+    entry.pointer.offset = cpu_to_le32(dst_patched_offset);
+    entry.pointer.size = dst_patched_size;
+    assert(dst_patched_size == 1 || dst_patched_size == 2 ||
+           dst_patched_size == 4 || dst_patched_size == 8);
+
+    le_src_offset = cpu_to_le64(src_offset);
+    memcpy(dst_file->blob->data + dst_patched_offset,
+           &le_src_offset, dst_patched_size);
+
+    g_array_append_vals(linker->cmd_blob, &entry, sizeof entry);
+}
diff --git a/hw/acpi/core.c b/hw/acpi/core.c
index 51913d6..e890a5d 100644
--- a/hw/acpi/core.c
+++ b/hw/acpi/core.c
@@ -18,13 +18,14 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/acpi/acpi.h"
+#include "hw/nvram/fw_cfg.h"
 #include "qemu/config-file.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/dealloc-visitor.h"
 #include "qapi-visit.h"
 #include "qapi-event.h"
 
@@ -66,7 +67,7 @@
     qemu_add_opts(&qemu_acpi_opts);
 }
 
-machine_init(acpi_register_config);
+opts_init(acpi_register_config);
 
 static int acpi_checksum(const uint8_t *data, int len)
 {
@@ -238,11 +239,11 @@
     char unsigned *blob = NULL;
 
     {
-        OptsVisitor *ov;
+        Visitor *v;
 
-        ov = opts_visitor_new(opts);
-        visit_type_AcpiTableOptions(opts_get_visitor(ov), &hdrs, NULL, &err);
-        opts_visitor_cleanup(ov);
+        v = opts_visitor_new(opts);
+        visit_type_AcpiTableOptions(v, NULL, &hdrs, &err);
+        visit_free(v);
     }
 
     if (err) {
@@ -295,15 +296,7 @@
 out:
     g_free(blob);
     g_strfreev(pathnames);
-
-    if (hdrs != NULL) {
-        QapiDeallocVisitor *dv;
-
-        dv = qapi_dealloc_visitor_new();
-        visit_type_AcpiTableOptions(qapi_dealloc_get_visitor(dv), &hdrs, NULL,
-                                    NULL);
-        qapi_dealloc_visitor_cleanup(dv);
-    }
+    qapi_free_AcpiTableOptions(hdrs);
 
     error_propagate(errp, err);
 }
@@ -348,6 +341,22 @@
     }
 }
 
+int acpi_get_slic_oem(AcpiSlicOem *oem)
+{
+    uint8_t *u;
+
+    for (u = acpi_table_first(); u; u = acpi_table_next(u)) {
+        struct acpi_table_header *hdr = (void *)(u - sizeof(hdr->_length));
+
+        if (memcmp(hdr->sig, "SLIC", 4) == 0) {
+            oem->id = hdr->oem_id;
+            oem->table_id = hdr->oem_table_id;
+            return 0;
+        }
+    }
+    return -1;
+}
+
 static void acpi_notify_wakeup(Notifier *notifier, void *data)
 {
     ACPIREGS *ar = container_of(notifier, ACPIREGS, wakeup);
@@ -380,7 +389,7 @@
        acpi_pm_tmr_update function uses ns for setting the timer. */
     int64_t d = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     if (d >= muldiv64(ar->tmr.overflow_time,
-                      get_ticks_per_sec(), PM_TIMER_FREQUENCY)) {
+                      NANOSECONDS_PER_SECOND, PM_TIMER_FREQUENCY)) {
         ar->pm1.evt.sts |= ACPI_BITMASK_TIMER_STATUS;
     }
     return ar->pm1.evt.sts;
@@ -474,7 +483,7 @@
 
     /* schedule a timer interruption if needed */
     if (enable) {
-        expire_time = muldiv64(ar->tmr.overflow_time, get_ticks_per_sec(),
+        expire_time = muldiv64(ar->tmr.overflow_time, NANOSECONDS_PER_SECOND,
                                PM_TIMER_FREQUENCY);
         timer_mod(ar->tmr.timer, expire_time);
     } else {
@@ -482,6 +491,12 @@
     }
 }
 
+static inline int64_t acpi_pm_tmr_get_clock(void)
+{
+    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), PM_TIMER_FREQUENCY,
+                    NANOSECONDS_PER_SECOND);
+}
+
 void acpi_pm_tmr_calc_overflow_time(ACPIREGS *ar)
 {
     int64_t d = acpi_pm_tmr_get_clock();
@@ -592,14 +607,26 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent, uint8_t s4_val)
+void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent,
+                       bool disable_s3, bool disable_s4, uint8_t s4_val)
 {
+    FWCfgState *fw_cfg;
+
     ar->pm1.cnt.s4_val = s4_val;
     ar->wakeup.notify = acpi_notify_wakeup;
     qemu_register_wakeup_notifier(&ar->wakeup);
     memory_region_init_io(&ar->pm1.cnt.io, memory_region_owner(parent),
                           &acpi_pm_cnt_ops, ar, "acpi-cnt", 2);
     memory_region_add_subregion(parent, 4, &ar->pm1.cnt.io);
+
+    fw_cfg = fw_cfg_find();
+    if (fw_cfg) {
+        uint8_t suspend[6] = {128, 0, 0, 129, 128, 128};
+        suspend[3] = 1 | ((!disable_s3) << 7);
+        suspend[4] = s4_val | ((!disable_s4) << 7);
+
+        fw_cfg_add_file(fw_cfg, "etc/system-states", g_memdup(suspend, 6), 6);
+    }
 }
 
 void acpi_pm1_cnt_reset(ACPIREGS *ar)
@@ -611,8 +638,12 @@
 void acpi_gpe_init(ACPIREGS *ar, uint8_t len)
 {
     ar->gpe.len = len;
-    ar->gpe.sts = g_malloc0(len / 2);
-    ar->gpe.en = g_malloc0(len / 2);
+    /* Only first len / 2 bytes are ever used,
+     * but the caller in ich9.c migrates full len bytes.
+     * TODO: fix ich9.c and drop the extra allocation.
+     */
+    ar->gpe.sts = g_malloc0(len);
+    ar->gpe.en = g_malloc0(len);
 }
 
 void acpi_gpe_reset(ACPIREGS *ar)
@@ -666,6 +697,13 @@
     return val;
 }
 
+void acpi_send_gpe_event(ACPIREGS *ar, qemu_irq irq,
+                         AcpiEventStatusBits status)
+{
+    ar->gpe.sts[0] |= status;
+    acpi_update_sci(ar, irq);
+}
+
 void acpi_update_sci(ACPIREGS *regs, qemu_irq irq)
 {
     int sci_level, pm1a_sts;
diff --git a/hw/acpi/cpu.c b/hw/acpi/cpu.c
new file mode 100644
index 0000000..c13b65c
--- /dev/null
+++ b/hw/acpi/cpu.c
@@ -0,0 +1,561 @@
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "hw/acpi/cpu.h"
+#include "qapi/error.h"
+#include "qapi-event.h"
+#include "trace.h"
+
+#define ACPI_CPU_HOTPLUG_REG_LEN 12
+#define ACPI_CPU_SELECTOR_OFFSET_WR 0
+#define ACPI_CPU_FLAGS_OFFSET_RW 4
+#define ACPI_CPU_CMD_OFFSET_WR 5
+#define ACPI_CPU_CMD_DATA_OFFSET_RW 8
+
+enum {
+    CPHP_GET_NEXT_CPU_WITH_EVENT_CMD = 0,
+    CPHP_OST_EVENT_CMD = 1,
+    CPHP_OST_STATUS_CMD = 2,
+    CPHP_CMD_MAX
+};
+
+static ACPIOSTInfo *acpi_cpu_device_status(int idx, AcpiCpuStatus *cdev)
+{
+    ACPIOSTInfo *info = g_new0(ACPIOSTInfo, 1);
+
+    info->slot_type = ACPI_SLOT_TYPE_CPU;
+    info->slot = g_strdup_printf("%d", idx);
+    info->source = cdev->ost_event;
+    info->status = cdev->ost_status;
+    if (cdev->cpu) {
+        DeviceState *dev = DEVICE(cdev->cpu);
+        if (dev->id) {
+            info->device = g_strdup(dev->id);
+            info->has_device = true;
+        }
+    }
+    return info;
+}
+
+void acpi_cpu_ospm_status(CPUHotplugState *cpu_st, ACPIOSTInfoList ***list)
+{
+    int i;
+
+    for (i = 0; i < cpu_st->dev_count; i++) {
+        ACPIOSTInfoList *elem = g_new0(ACPIOSTInfoList, 1);
+        elem->value = acpi_cpu_device_status(i, &cpu_st->devs[i]);
+        elem->next = NULL;
+        **list = elem;
+        *list = &elem->next;
+    }
+}
+
+static uint64_t cpu_hotplug_rd(void *opaque, hwaddr addr, unsigned size)
+{
+    uint64_t val = 0;
+    CPUHotplugState *cpu_st = opaque;
+    AcpiCpuStatus *cdev;
+
+    if (cpu_st->selector >= cpu_st->dev_count) {
+        return val;
+    }
+
+    cdev = &cpu_st->devs[cpu_st->selector];
+    switch (addr) {
+    case ACPI_CPU_FLAGS_OFFSET_RW: /* pack and return is_* fields */
+        val |= cdev->cpu ? 1 : 0;
+        val |= cdev->is_inserting ? 2 : 0;
+        val |= cdev->is_removing  ? 4 : 0;
+        trace_cpuhp_acpi_read_flags(cpu_st->selector, val);
+        break;
+    case ACPI_CPU_CMD_DATA_OFFSET_RW:
+        switch (cpu_st->command) {
+        case CPHP_GET_NEXT_CPU_WITH_EVENT_CMD:
+           val = cpu_st->selector;
+           break;
+        default:
+           break;
+        }
+        trace_cpuhp_acpi_read_cmd_data(cpu_st->selector, val);
+        break;
+    default:
+        break;
+    }
+    return val;
+}
+
+static void cpu_hotplug_wr(void *opaque, hwaddr addr, uint64_t data,
+                           unsigned int size)
+{
+    CPUHotplugState *cpu_st = opaque;
+    AcpiCpuStatus *cdev;
+    ACPIOSTInfo *info;
+
+    assert(cpu_st->dev_count);
+
+    if (addr) {
+        if (cpu_st->selector >= cpu_st->dev_count) {
+            trace_cpuhp_acpi_invalid_idx_selected(cpu_st->selector);
+            return;
+        }
+    }
+
+    switch (addr) {
+    case ACPI_CPU_SELECTOR_OFFSET_WR: /* current CPU selector */
+        cpu_st->selector = data;
+        trace_cpuhp_acpi_write_idx(cpu_st->selector);
+        break;
+    case ACPI_CPU_FLAGS_OFFSET_RW: /* set is_* fields  */
+        cdev = &cpu_st->devs[cpu_st->selector];
+        if (data & 2) { /* clear insert event */
+            cdev->is_inserting = false;
+            trace_cpuhp_acpi_clear_inserting_evt(cpu_st->selector);
+        } else if (data & 4) { /* clear remove event */
+            cdev->is_removing = false;
+            trace_cpuhp_acpi_clear_remove_evt(cpu_st->selector);
+        } else if (data & 8) {
+            DeviceState *dev = NULL;
+            HotplugHandler *hotplug_ctrl = NULL;
+
+            if (!cdev->cpu) {
+                trace_cpuhp_acpi_ejecting_invalid_cpu(cpu_st->selector);
+                break;
+            }
+
+            trace_cpuhp_acpi_ejecting_cpu(cpu_st->selector);
+            dev = DEVICE(cdev->cpu);
+            hotplug_ctrl = qdev_get_hotplug_handler(dev);
+            hotplug_handler_unplug(hotplug_ctrl, dev, NULL);
+        }
+        break;
+    case ACPI_CPU_CMD_OFFSET_WR:
+        trace_cpuhp_acpi_write_cmd(cpu_st->selector, data);
+        if (data < CPHP_CMD_MAX) {
+            cpu_st->command = data;
+            if (cpu_st->command == CPHP_GET_NEXT_CPU_WITH_EVENT_CMD) {
+                uint32_t iter = cpu_st->selector;
+
+                do {
+                    cdev = &cpu_st->devs[iter];
+                    if (cdev->is_inserting || cdev->is_removing) {
+                        cpu_st->selector = iter;
+                        trace_cpuhp_acpi_cpu_has_events(cpu_st->selector,
+                            cdev->is_inserting, cdev->is_removing);
+                        break;
+                    }
+                    iter = iter + 1 < cpu_st->dev_count ? iter + 1 : 0;
+                } while (iter != cpu_st->selector);
+            }
+        }
+        break;
+    case ACPI_CPU_CMD_DATA_OFFSET_RW:
+        switch (cpu_st->command) {
+        case CPHP_OST_EVENT_CMD: {
+           cdev = &cpu_st->devs[cpu_st->selector];
+           cdev->ost_event = data;
+           trace_cpuhp_acpi_write_ost_ev(cpu_st->selector, cdev->ost_event);
+           break;
+        }
+        case CPHP_OST_STATUS_CMD: {
+           cdev = &cpu_st->devs[cpu_st->selector];
+           cdev->ost_status = data;
+           info = acpi_cpu_device_status(cpu_st->selector, cdev);
+           qapi_event_send_acpi_device_ost(info, &error_abort);
+           qapi_free_ACPIOSTInfo(info);
+           trace_cpuhp_acpi_write_ost_status(cpu_st->selector,
+                                             cdev->ost_status);
+           break;
+        }
+        default:
+           break;
+        }
+        break;
+    default:
+        break;
+    }
+}
+
+static const MemoryRegionOps cpu_hotplug_ops = {
+    .read = cpu_hotplug_rd,
+    .write = cpu_hotplug_wr,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+};
+
+void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner,
+                         CPUHotplugState *state, hwaddr base_addr)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    CPUArchIdList *id_list;
+    int i;
+
+    assert(mc->possible_cpu_arch_ids);
+    id_list = mc->possible_cpu_arch_ids(machine);
+    state->dev_count = id_list->len;
+    state->devs = g_new0(typeof(*state->devs), state->dev_count);
+    for (i = 0; i < id_list->len; i++) {
+        state->devs[i].cpu =  id_list->cpus[i].cpu;
+        state->devs[i].arch_id = id_list->cpus[i].arch_id;
+    }
+    g_free(id_list);
+    memory_region_init_io(&state->ctrl_reg, owner, &cpu_hotplug_ops, state,
+                          "acpi-mem-hotplug", ACPI_CPU_HOTPLUG_REG_LEN);
+    memory_region_add_subregion(as, base_addr, &state->ctrl_reg);
+}
+
+static AcpiCpuStatus *get_cpu_status(CPUHotplugState *cpu_st, DeviceState *dev)
+{
+    CPUClass *k = CPU_GET_CLASS(dev);
+    uint64_t cpu_arch_id = k->get_arch_id(CPU(dev));
+    int i;
+
+    for (i = 0; i < cpu_st->dev_count; i++) {
+        if (cpu_arch_id == cpu_st->devs[i].arch_id) {
+            return &cpu_st->devs[i];
+        }
+    }
+    return NULL;
+}
+
+void acpi_cpu_plug_cb(HotplugHandler *hotplug_dev,
+                      CPUHotplugState *cpu_st, DeviceState *dev, Error **errp)
+{
+    AcpiCpuStatus *cdev;
+
+    cdev = get_cpu_status(cpu_st, dev);
+    if (!cdev) {
+        return;
+    }
+
+    cdev->cpu = CPU(dev);
+    if (dev->hotplugged) {
+        cdev->is_inserting = true;
+        acpi_send_event(DEVICE(hotplug_dev), ACPI_CPU_HOTPLUG_STATUS);
+    }
+}
+
+void acpi_cpu_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                CPUHotplugState *cpu_st,
+                                DeviceState *dev, Error **errp)
+{
+    AcpiCpuStatus *cdev;
+
+    cdev = get_cpu_status(cpu_st, dev);
+    if (!cdev) {
+        return;
+    }
+
+    cdev->is_removing = true;
+    acpi_send_event(DEVICE(hotplug_dev), ACPI_CPU_HOTPLUG_STATUS);
+}
+
+void acpi_cpu_unplug_cb(CPUHotplugState *cpu_st,
+                        DeviceState *dev, Error **errp)
+{
+    AcpiCpuStatus *cdev;
+
+    cdev = get_cpu_status(cpu_st, dev);
+    if (!cdev) {
+        return;
+    }
+
+    cdev->cpu = NULL;
+}
+
+static const VMStateDescription vmstate_cpuhp_sts = {
+    .name = "CPU hotplug device state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(is_inserting, AcpiCpuStatus),
+        VMSTATE_BOOL(is_removing, AcpiCpuStatus),
+        VMSTATE_UINT32(ost_event, AcpiCpuStatus),
+        VMSTATE_UINT32(ost_status, AcpiCpuStatus),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+const VMStateDescription vmstate_cpu_hotplug = {
+    .name = "CPU hotplug state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT32(selector, CPUHotplugState),
+        VMSTATE_UINT8(command, CPUHotplugState),
+        VMSTATE_STRUCT_VARRAY_POINTER_UINT32(devs, CPUHotplugState, dev_count,
+                                             vmstate_cpuhp_sts, AcpiCpuStatus),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define CPU_NAME_FMT      "C%.03X"
+#define CPUHP_RES_DEVICE  "PRES"
+#define CPU_LOCK          "CPLK"
+#define CPU_STS_METHOD    "CSTA"
+#define CPU_SCAN_METHOD   "CSCN"
+#define CPU_NOTIFY_METHOD "CTFY"
+#define CPU_EJECT_METHOD  "CEJ0"
+#define CPU_OST_METHOD    "COST"
+
+#define CPU_ENABLED       "CPEN"
+#define CPU_SELECTOR      "CSEL"
+#define CPU_COMMAND       "CCMD"
+#define CPU_DATA          "CDAT"
+#define CPU_INSERT_EVENT  "CINS"
+#define CPU_REMOVE_EVENT  "CRMV"
+#define CPU_EJECT_EVENT   "CEJ0"
+
+void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts,
+                    hwaddr io_base,
+                    const char *res_root,
+                    const char *event_handler_method)
+{
+    Aml *ifctx;
+    Aml *field;
+    Aml *method;
+    Aml *cpu_ctrl_dev;
+    Aml *cpus_dev;
+    Aml *zero = aml_int(0);
+    Aml *one = aml_int(1);
+    Aml *sb_scope = aml_scope("_SB");
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    CPUArchIdList *arch_ids = mc->possible_cpu_arch_ids(machine);
+    char *cphp_res_path = g_strdup_printf("%s." CPUHP_RES_DEVICE, res_root);
+    Object *obj = object_resolve_path_type("", TYPE_ACPI_DEVICE_IF, NULL);
+    AcpiDeviceIfClass *adevc = ACPI_DEVICE_IF_GET_CLASS(obj);
+    AcpiDeviceIf *adev = ACPI_DEVICE_IF(obj);
+
+    cpu_ctrl_dev = aml_device("%s", cphp_res_path);
+    {
+        Aml *crs;
+
+        aml_append(cpu_ctrl_dev,
+            aml_name_decl("_HID", aml_eisaid("PNP0A06")));
+        aml_append(cpu_ctrl_dev,
+            aml_name_decl("_UID", aml_string("CPU Hotplug resources")));
+        aml_append(cpu_ctrl_dev, aml_mutex(CPU_LOCK, 0));
+
+        crs = aml_resource_template();
+        aml_append(crs, aml_io(AML_DECODE16, io_base, io_base, 1,
+                               ACPI_CPU_HOTPLUG_REG_LEN));
+        aml_append(cpu_ctrl_dev, aml_name_decl("_CRS", crs));
+
+        /* declare CPU hotplug MMIO region with related access fields */
+        aml_append(cpu_ctrl_dev,
+            aml_operation_region("PRST", AML_SYSTEM_IO, aml_int(io_base),
+                                 ACPI_CPU_HOTPLUG_REG_LEN));
+
+        field = aml_field("PRST", AML_BYTE_ACC, AML_NOLOCK,
+                          AML_WRITE_AS_ZEROS);
+        aml_append(field, aml_reserved_field(ACPI_CPU_FLAGS_OFFSET_RW * 8));
+        /* 1 if enabled, read only */
+        aml_append(field, aml_named_field(CPU_ENABLED, 1));
+        /* (read) 1 if has a insert event. (write) 1 to clear event */
+        aml_append(field, aml_named_field(CPU_INSERT_EVENT, 1));
+        /* (read) 1 if has a remove event. (write) 1 to clear event */
+        aml_append(field, aml_named_field(CPU_REMOVE_EVENT, 1));
+        /* initiates device eject, write only */
+        aml_append(field, aml_named_field(CPU_EJECT_EVENT, 1));
+        aml_append(field, aml_reserved_field(4));
+        aml_append(field, aml_named_field(CPU_COMMAND, 8));
+        aml_append(cpu_ctrl_dev, field);
+
+        field = aml_field("PRST", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+        /* CPU selector, write only */
+        aml_append(field, aml_named_field(CPU_SELECTOR, 32));
+        /* flags + cmd + 2byte align */
+        aml_append(field, aml_reserved_field(4 * 8));
+        aml_append(field, aml_named_field(CPU_DATA, 32));
+        aml_append(cpu_ctrl_dev, field);
+
+        if (opts.has_legacy_cphp) {
+            method = aml_method("_INI", 0, AML_SERIALIZED);
+            /* switch off legacy CPU hotplug HW and use new one,
+             * on reboot system is in new mode and writing 0
+             * in CPU_SELECTOR selects BSP, which is NOP at
+             * the time _INI is called */
+            aml_append(method, aml_store(zero, aml_name(CPU_SELECTOR)));
+            aml_append(cpu_ctrl_dev, method);
+        }
+    }
+    aml_append(sb_scope, cpu_ctrl_dev);
+
+    cpus_dev = aml_device("\\_SB.CPUS");
+    {
+        int i;
+        Aml *ctrl_lock = aml_name("%s.%s", cphp_res_path, CPU_LOCK);
+        Aml *cpu_selector = aml_name("%s.%s", cphp_res_path, CPU_SELECTOR);
+        Aml *is_enabled = aml_name("%s.%s", cphp_res_path, CPU_ENABLED);
+        Aml *cpu_cmd = aml_name("%s.%s", cphp_res_path, CPU_COMMAND);
+        Aml *cpu_data = aml_name("%s.%s", cphp_res_path, CPU_DATA);
+        Aml *ins_evt = aml_name("%s.%s", cphp_res_path, CPU_INSERT_EVENT);
+        Aml *rm_evt = aml_name("%s.%s", cphp_res_path, CPU_REMOVE_EVENT);
+        Aml *ej_evt = aml_name("%s.%s", cphp_res_path, CPU_EJECT_EVENT);
+
+        aml_append(cpus_dev, aml_name_decl("_HID", aml_string("ACPI0010")));
+        aml_append(cpus_dev, aml_name_decl("_CID", aml_eisaid("PNP0A05")));
+
+        method = aml_method(CPU_NOTIFY_METHOD, 2, AML_NOTSERIALIZED);
+        for (i = 0; i < arch_ids->len; i++) {
+            Aml *cpu = aml_name(CPU_NAME_FMT, i);
+            Aml *uid = aml_arg(0);
+            Aml *event = aml_arg(1);
+
+            ifctx = aml_if(aml_equal(uid, aml_int(i)));
+            {
+                aml_append(ifctx, aml_notify(cpu, event));
+            }
+            aml_append(method, ifctx);
+        }
+        aml_append(cpus_dev, method);
+
+        method = aml_method(CPU_STS_METHOD, 1, AML_SERIALIZED);
+        {
+            Aml *idx = aml_arg(0);
+            Aml *sta = aml_local(0);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(idx, cpu_selector));
+            aml_append(method, aml_store(zero, sta));
+            ifctx = aml_if(aml_equal(is_enabled, one));
+            {
+                aml_append(ifctx, aml_store(aml_int(0xF), sta));
+            }
+            aml_append(method, ifctx);
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(sta));
+        }
+        aml_append(cpus_dev, method);
+
+        method = aml_method(CPU_EJECT_METHOD, 1, AML_SERIALIZED);
+        {
+            Aml *idx = aml_arg(0);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(idx, cpu_selector));
+            aml_append(method, aml_store(one, ej_evt));
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(cpus_dev, method);
+
+        method = aml_method(CPU_SCAN_METHOD, 0, AML_SERIALIZED);
+        {
+            Aml *else_ctx;
+            Aml *while_ctx;
+            Aml *has_event = aml_local(0);
+            Aml *dev_chk = aml_int(1);
+            Aml *eject_req = aml_int(3);
+            Aml *next_cpu_cmd = aml_int(CPHP_GET_NEXT_CPU_WITH_EVENT_CMD);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(one, has_event));
+            while_ctx = aml_while(aml_equal(has_event, one));
+            {
+                 /* clear loop exit condition, ins_evt/rm_evt checks
+                  * will set it to 1 while next_cpu_cmd returns a CPU
+                  * with events */
+                 aml_append(while_ctx, aml_store(zero, has_event));
+                 aml_append(while_ctx, aml_store(next_cpu_cmd, cpu_cmd));
+                 ifctx = aml_if(aml_equal(ins_evt, one));
+                 {
+                     aml_append(ifctx,
+                         aml_call2(CPU_NOTIFY_METHOD, cpu_data, dev_chk));
+                     aml_append(ifctx, aml_store(one, ins_evt));
+                     aml_append(ifctx, aml_store(one, has_event));
+                 }
+                 aml_append(while_ctx, ifctx);
+                 else_ctx = aml_else();
+                 ifctx = aml_if(aml_equal(rm_evt, one));
+                 {
+                     aml_append(ifctx,
+                         aml_call2(CPU_NOTIFY_METHOD, cpu_data, eject_req));
+                     aml_append(ifctx, aml_store(one, rm_evt));
+                     aml_append(ifctx, aml_store(one, has_event));
+                 }
+                 aml_append(else_ctx, ifctx);
+                 aml_append(while_ctx, else_ctx);
+            }
+            aml_append(method, while_ctx);
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(cpus_dev, method);
+
+        method = aml_method(CPU_OST_METHOD, 4, AML_SERIALIZED);
+        {
+            Aml *uid = aml_arg(0);
+            Aml *ev_cmd = aml_int(CPHP_OST_EVENT_CMD);
+            Aml *st_cmd = aml_int(CPHP_OST_STATUS_CMD);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(uid, cpu_selector));
+            aml_append(method, aml_store(ev_cmd, cpu_cmd));
+            aml_append(method, aml_store(aml_arg(1), cpu_data));
+            aml_append(method, aml_store(st_cmd, cpu_cmd));
+            aml_append(method, aml_store(aml_arg(2), cpu_data));
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(cpus_dev, method);
+
+        /* build Processor object for each processor */
+        for (i = 0; i < arch_ids->len; i++) {
+            Aml *dev;
+            Aml *uid = aml_int(i);
+            GArray *madt_buf = g_array_new(0, 1, 1);
+            int arch_id = arch_ids->cpus[i].arch_id;
+
+            if (opts.apci_1_compatible && arch_id < 255) {
+                dev = aml_processor(i, 0, 0, CPU_NAME_FMT, i);
+            } else {
+                dev = aml_device(CPU_NAME_FMT, i);
+                aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0007")));
+                aml_append(dev, aml_name_decl("_UID", uid));
+            }
+
+            method = aml_method("_STA", 0, AML_SERIALIZED);
+            aml_append(method, aml_return(aml_call1(CPU_STS_METHOD, uid)));
+            aml_append(dev, method);
+
+            /* build _MAT object */
+            assert(adevc && adevc->madt_cpu);
+            adevc->madt_cpu(adev, i, arch_ids, madt_buf);
+            switch (madt_buf->data[0]) {
+            case ACPI_APIC_PROCESSOR: {
+                AcpiMadtProcessorApic *apic = (void *)madt_buf->data;
+                apic->flags = cpu_to_le32(1);
+                break;
+            }
+            default:
+                assert(0);
+            }
+            aml_append(dev, aml_name_decl("_MAT",
+                aml_buffer(madt_buf->len, (uint8_t *)madt_buf->data)));
+            g_array_free(madt_buf, true);
+
+            method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+            aml_append(method, aml_call1(CPU_EJECT_METHOD, uid));
+            aml_append(dev, method);
+
+            method = aml_method("_OST", 3, AML_SERIALIZED);
+            aml_append(method,
+                aml_call4(CPU_OST_METHOD, uid, aml_arg(0),
+                          aml_arg(1), aml_arg(2))
+            );
+            aml_append(dev, method);
+            aml_append(cpus_dev, dev);
+        }
+    }
+    aml_append(sb_scope, cpus_dev);
+    aml_append(table, sb_scope);
+
+    method = aml_method(event_handler_method, 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_call0("\\_SB.CPUS." CPU_SCAN_METHOD));
+    aml_append(table, method);
+
+    g_free(cphp_res_path);
+    g_free(arch_ids);
+}
diff --git a/hw/acpi/cpu_hotplug.c b/hw/acpi/cpu_hotplug.c
index b8ebfad..e19d902 100644
--- a/hw/acpi/cpu_hotplug.c
+++ b/hw/acpi/cpu_hotplug.c
@@ -9,8 +9,19 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/acpi/cpu_hotplug.h"
+#include "qapi/error.h"
+#include "qom/cpu.h"
+#include "hw/i386/pc.h"
+
+#define CPU_EJECT_METHOD "CPEJ"
+#define CPU_MAT_METHOD "CPMA"
+#define CPU_ON_BITMAP "CPON"
+#define CPU_STATUS_METHOD "CPST"
+#define CPU_STATUS_MAP "PRS"
+#define CPU_SCAN_METHOD "PRSC"
 
 static uint64_t cpu_status_read(void *opaque, hwaddr addr, unsigned int size)
 {
@@ -23,7 +34,15 @@
 static void cpu_status_write(void *opaque, hwaddr addr, uint64_t data,
                              unsigned int size)
 {
-    /* TODO: implement VCPU removal on guest signal that CPU can be removed */
+    /* firmware never used to write in CPU present bitmap so use
+       this fact as means to switch QEMU into modern CPU hotplug
+       mode by writing 0 at the beginning of legacy CPU bitmap
+     */
+    if (addr == 0 && data == 0) {
+        AcpiCpuHotplug *cpus = opaque;
+        object_property_set_bool(cpus->device, false, "cpu-hotplug-legacy",
+                                 &error_abort);
+    }
 }
 
 static const MemoryRegionOps AcpiCpuHotplug_ops = {
@@ -51,20 +70,18 @@
     g->sts[cpu_id / 8] |= (1 << (cpu_id % 8));
 }
 
-void acpi_cpu_plug_cb(ACPIREGS *ar, qemu_irq irq,
-                      AcpiCpuHotplug *g, DeviceState *dev, Error **errp)
+void legacy_acpi_cpu_plug_cb(HotplugHandler *hotplug_dev,
+                             AcpiCpuHotplug *g, DeviceState *dev, Error **errp)
 {
     acpi_set_cpu_present_bit(g, CPU(dev), errp);
     if (*errp != NULL) {
         return;
     }
-
-    ar->gpe.sts[0] |= ACPI_CPU_HOTPLUG_STATUS;
-    acpi_update_sci(ar, irq);
+    acpi_send_event(DEVICE(hotplug_dev), ACPI_CPU_HOTPLUG_STATUS);
 }
 
-void acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
-                           AcpiCpuHotplug *gpe_cpu, uint16_t base)
+void legacy_acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
+                                  AcpiCpuHotplug *gpe_cpu, uint16_t base)
 {
     CPUState *cpu;
 
@@ -74,4 +91,242 @@
     memory_region_init_io(&gpe_cpu->io, owner, &AcpiCpuHotplug_ops,
                           gpe_cpu, "acpi-cpu-hotplug", ACPI_GPE_PROC_LEN);
     memory_region_add_subregion(parent, base, &gpe_cpu->io);
+    gpe_cpu->device = owner;
+}
+
+void acpi_switch_to_modern_cphp(AcpiCpuHotplug *gpe_cpu,
+                                CPUHotplugState *cpuhp_state,
+                                uint16_t io_port)
+{
+    MemoryRegion *parent = pci_address_space_io(PCI_DEVICE(gpe_cpu->device));
+
+    memory_region_del_subregion(parent, &gpe_cpu->io);
+    cpu_hotplug_hw_init(parent, gpe_cpu->device, cpuhp_state, io_port);
+}
+
+void build_legacy_cpu_hotplug_aml(Aml *ctx, MachineState *machine,
+                                  uint16_t io_base)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *pkg;
+    Aml *field;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *else_ctx;
+    int i, apic_idx;
+    Aml *sb_scope = aml_scope("_SB");
+    uint8_t madt_tmpl[8] = {0x00, 0x08, 0x00, 0x00, 0x00, 0, 0, 0};
+    Aml *cpu_id = aml_arg(1);
+    Aml *apic_id = aml_arg(0);
+    Aml *cpu_on = aml_local(0);
+    Aml *madt = aml_local(1);
+    Aml *cpus_map = aml_name(CPU_ON_BITMAP);
+    Aml *zero = aml_int(0);
+    Aml *one = aml_int(1);
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    CPUArchIdList *apic_ids = mc->possible_cpu_arch_ids(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
+
+    /*
+     * _MAT method - creates an madt apic buffer
+     * apic_id = Arg0 = Local APIC ID
+     * cpu_id  = Arg1 = Processor ID
+     * cpu_on = Local0 = CPON flag for this cpu
+     * madt = Local1 = Buffer (in madt apic form) to return
+     */
+    method = aml_method(CPU_MAT_METHOD, 2, AML_NOTSERIALIZED);
+    aml_append(method,
+        aml_store(aml_derefof(aml_index(cpus_map, apic_id)), cpu_on));
+    aml_append(method,
+        aml_store(aml_buffer(sizeof(madt_tmpl), madt_tmpl), madt));
+    /* Update the processor id, lapic id, and enable/disable status */
+    aml_append(method, aml_store(cpu_id, aml_index(madt, aml_int(2))));
+    aml_append(method, aml_store(apic_id, aml_index(madt, aml_int(3))));
+    aml_append(method, aml_store(cpu_on, aml_index(madt, aml_int(4))));
+    aml_append(method, aml_return(madt));
+    aml_append(sb_scope, method);
+
+    /*
+     * _STA method - return ON status of cpu
+     * apic_id = Arg0 = Local APIC ID
+     * cpu_on = Local0 = CPON flag for this cpu
+     */
+    method = aml_method(CPU_STATUS_METHOD, 1, AML_NOTSERIALIZED);
+    aml_append(method,
+        aml_store(aml_derefof(aml_index(cpus_map, apic_id)), cpu_on));
+    if_ctx = aml_if(cpu_on);
+    {
+        aml_append(if_ctx, aml_return(aml_int(0xF)));
+    }
+    aml_append(method, if_ctx);
+    else_ctx = aml_else();
+    {
+        aml_append(else_ctx, aml_return(zero));
+    }
+    aml_append(method, else_ctx);
+    aml_append(sb_scope, method);
+
+    method = aml_method(CPU_EJECT_METHOD, 2, AML_NOTSERIALIZED);
+    aml_append(method, aml_sleep(200));
+    aml_append(sb_scope, method);
+
+    method = aml_method(CPU_SCAN_METHOD, 0, AML_NOTSERIALIZED);
+    {
+        Aml *while_ctx, *if_ctx2, *else_ctx2;
+        Aml *bus_check_evt = aml_int(1);
+        Aml *remove_evt = aml_int(3);
+        Aml *status_map = aml_local(5); /* Local5 = active cpu bitmap */
+        Aml *byte = aml_local(2); /* Local2 = last read byte from bitmap */
+        Aml *idx = aml_local(0); /* Processor ID / APIC ID iterator */
+        Aml *is_cpu_on = aml_local(1); /* Local1 = CPON flag for cpu */
+        Aml *status = aml_local(3); /* Local3 = active state for cpu */
+
+        aml_append(method, aml_store(aml_name(CPU_STATUS_MAP), status_map));
+        aml_append(method, aml_store(zero, byte));
+        aml_append(method, aml_store(zero, idx));
+
+        /* While (idx < SizeOf(CPON)) */
+        while_ctx = aml_while(aml_lless(idx, aml_sizeof(cpus_map)));
+        aml_append(while_ctx,
+            aml_store(aml_derefof(aml_index(cpus_map, idx)), is_cpu_on));
+
+        if_ctx = aml_if(aml_and(idx, aml_int(0x07), NULL));
+        {
+            /* Shift down previously read bitmap byte */
+            aml_append(if_ctx, aml_shiftright(byte, one, byte));
+        }
+        aml_append(while_ctx, if_ctx);
+
+        else_ctx = aml_else();
+        {
+            /* Read next byte from cpu bitmap */
+            aml_append(else_ctx, aml_store(aml_derefof(aml_index(status_map,
+                       aml_shiftright(idx, aml_int(3), NULL))), byte));
+        }
+        aml_append(while_ctx, else_ctx);
+
+        aml_append(while_ctx, aml_store(aml_and(byte, one, NULL), status));
+        if_ctx = aml_if(aml_lnot(aml_equal(is_cpu_on, status)));
+        {
+            /* State change - update CPON with new state */
+            aml_append(if_ctx, aml_store(status, aml_index(cpus_map, idx)));
+            if_ctx2 = aml_if(aml_equal(status, one));
+            {
+                aml_append(if_ctx2,
+                    aml_call2(AML_NOTIFY_METHOD, idx, bus_check_evt));
+            }
+            aml_append(if_ctx, if_ctx2);
+            else_ctx2 = aml_else();
+            {
+                aml_append(else_ctx2,
+                    aml_call2(AML_NOTIFY_METHOD, idx, remove_evt));
+            }
+        }
+        aml_append(if_ctx, else_ctx2);
+        aml_append(while_ctx, if_ctx);
+
+        aml_append(while_ctx, aml_increment(idx)); /* go to next cpu */
+        aml_append(method, while_ctx);
+    }
+    aml_append(sb_scope, method);
+
+    /* The current AML generator can cover the APIC ID range [0..255],
+     * inclusive, for VCPU hotplug. */
+    QEMU_BUILD_BUG_ON(ACPI_CPU_HOTPLUG_ID_LIMIT > 256);
+    g_assert(pcms->apic_id_limit <= ACPI_CPU_HOTPLUG_ID_LIMIT);
+
+    /* create PCI0.PRES device and its _CRS to reserve CPU hotplug MMIO */
+    dev = aml_device("PCI0." stringify(CPU_HOTPLUG_RESOURCE_DEVICE));
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A06")));
+    aml_append(dev,
+        aml_name_decl("_UID", aml_string("CPU Hotplug resources"))
+    );
+    /* device present, functioning, decoding, not shown in UI */
+    aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+    crs = aml_resource_template();
+    aml_append(crs,
+        aml_io(AML_DECODE16, io_base, io_base, 1, ACPI_GPE_PROC_LEN)
+    );
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(sb_scope, dev);
+    /* declare CPU hotplug MMIO region and PRS field to access it */
+    aml_append(sb_scope, aml_operation_region(
+        "PRST", AML_SYSTEM_IO, aml_int(io_base), ACPI_GPE_PROC_LEN));
+    field = aml_field("PRST", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("PRS", 256));
+    aml_append(sb_scope, field);
+
+    /* build Processor object for each processor */
+    for (i = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
+        assert(apic_id < ACPI_CPU_HOTPLUG_ID_LIMIT);
+
+        dev = aml_processor(i, 0, 0, "CP%.02X", apic_id);
+
+        method = aml_method("_MAT", 0, AML_NOTSERIALIZED);
+        aml_append(method,
+            aml_return(aml_call2(CPU_MAT_METHOD, aml_int(apic_id), aml_int(i))
+        ));
+        aml_append(dev, method);
+
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        aml_append(method,
+            aml_return(aml_call1(CPU_STATUS_METHOD, aml_int(apic_id))));
+        aml_append(dev, method);
+
+        method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+        aml_append(method,
+            aml_return(aml_call2(CPU_EJECT_METHOD, aml_int(apic_id),
+                aml_arg(0)))
+        );
+        aml_append(dev, method);
+
+        aml_append(sb_scope, dev);
+    }
+
+    /* build this code:
+     *   Method(NTFY, 2) {If (LEqual(Arg0, 0x00)) {Notify(CP00, Arg1)} ...}
+     */
+    /* Arg0 = APIC ID */
+    method = aml_method(AML_NOTIFY_METHOD, 2, AML_NOTSERIALIZED);
+    for (i = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
+        if_ctx = aml_if(aml_equal(aml_arg(0), aml_int(apic_id)));
+        aml_append(if_ctx,
+            aml_notify(aml_name("CP%.02X", apic_id), aml_arg(1))
+        );
+        aml_append(method, if_ctx);
+    }
+    aml_append(sb_scope, method);
+
+    /* build "Name(CPON, Package() { One, One, ..., Zero, Zero, ... })"
+     *
+     * Note: The ability to create variable-sized packages was first
+     * introduced in ACPI 2.0. ACPI 1.0 only allowed fixed-size packages
+     * ith up to 255 elements. Windows guests up to win2k8 fail when
+     * VarPackageOp is used.
+     */
+    pkg = pcms->apic_id_limit <= 255 ? aml_package(pcms->apic_id_limit) :
+                                       aml_varpackage(pcms->apic_id_limit);
+
+    for (i = 0, apic_idx = 0; i < apic_ids->len; i++) {
+        int apic_id = apic_ids->cpus[i].arch_id;
+
+        for (; apic_idx < apic_id; apic_idx++) {
+            aml_append(pkg, aml_int(0));
+        }
+        aml_append(pkg, aml_int(apic_ids->cpus[i].cpu ? 1 : 0));
+        apic_idx = apic_id + 1;
+    }
+    aml_append(sb_scope, aml_name_decl(CPU_ON_BITMAP, pkg));
+    g_free(apic_ids);
+
+    aml_append(ctx, sb_scope);
+
+    method = aml_method("\\_GPE._E02", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_call0("\\_SB." CPU_SCAN_METHOD));
+    aml_append(ctx, method);
 }
diff --git a/hw/acpi/ich9.c b/hw/acpi/ich9.c
index 5647961..e5a3c18 100644
--- a/hw/acpi/ich9.c
+++ b/hw/acpi/ich9.c
@@ -23,15 +23,17 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "hw/acpi/acpi.h"
+#include "hw/acpi/tco.h"
 #include "sysemu/kvm.h"
-#include "sysemu/hax.h"
 #include "exec/address-spaces.h"
 
 #include "hw/i386/ich9.h"
@@ -93,9 +95,18 @@
                             unsigned width)
 {
     ICH9LPCPMRegs *pm = opaque;
+    TCOIORegs *tr = &pm->tco_regs;
+    uint64_t tco_en;
+
     switch (addr) {
     case 0:
-        pm->smi_en = val;
+        tco_en = pm->smi_en & ICH9_PMIO_SMI_EN_TCO_EN;
+        /* once TCO_LOCK bit is set, TCO_EN bit cannot be overwritten */
+        if (tr->tco.cnt1 & TCO_LOCK) {
+            val = (val & ~ICH9_PMIO_SMI_EN_TCO_EN) | tco_en;
+        }
+        pm->smi_en &= ~pm->smi_en_wmask;
+        pm->smi_en |= (val & pm->smi_en_wmask);
         break;
     }
 }
@@ -152,12 +163,59 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
+    .needed = vmstate_test_use_memhp,
     .fields      = (VMStateField[]) {
         VMSTATE_MEMORY_HOTPLUG(acpi_memory_hotplug, ICH9LPCPMRegs),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static bool vmstate_test_use_tco(void *opaque)
+{
+    ICH9LPCPMRegs *s = opaque;
+    return s->enable_tco;
+}
+
+static const VMStateDescription vmstate_tco_io_state = {
+    .name = "ich9_pm/tco",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .needed = vmstate_test_use_tco,
+    .fields      = (VMStateField[]) {
+        VMSTATE_STRUCT(tco_regs, ICH9LPCPMRegs, 1, vmstate_tco_io_sts,
+                       TCOIORegs),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool vmstate_test_use_cpuhp(void *opaque)
+{
+    ICH9LPCPMRegs *s = opaque;
+    return !s->cpu_hotplug_legacy;
+}
+
+static int vmstate_cpuhp_pre_load(void *opaque)
+{
+    ICH9LPCPMRegs *s = opaque;
+    Object *obj = OBJECT(s->gpe_cpu.device);
+    object_property_set_bool(obj, false, "cpu-hotplug-legacy", &error_abort);
+    return 0;
+}
+
+static const VMStateDescription vmstate_cpuhp_state = {
+    .name = "ich9_pm/cpuhp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .needed = vmstate_test_use_cpuhp,
+    .pre_load = vmstate_cpuhp_pre_load,
+    .fields      = (VMStateField[]) {
+        VMSTATE_CPU_HOTPLUG(cpuhp_state, ICH9LPCPMRegs),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_ich9_pm = {
     .name = "ich9_pm",
     .version_id = 1,
@@ -167,7 +225,7 @@
         VMSTATE_UINT16(acpi_regs.pm1.evt.sts, ICH9LPCPMRegs),
         VMSTATE_UINT16(acpi_regs.pm1.evt.en, ICH9LPCPMRegs),
         VMSTATE_UINT16(acpi_regs.pm1.cnt.cnt, ICH9LPCPMRegs),
-        VMSTATE_TIMER(acpi_regs.tmr.timer, ICH9LPCPMRegs),
+        VMSTATE_TIMER_PTR(acpi_regs.tmr.timer, ICH9LPCPMRegs),
         VMSTATE_INT64(acpi_regs.tmr.overflow_time, ICH9LPCPMRegs),
         VMSTATE_GPE_ARRAY(acpi_regs.gpe.sts, ICH9LPCPMRegs),
         VMSTATE_GPE_ARRAY(acpi_regs.gpe.en, ICH9LPCPMRegs),
@@ -175,12 +233,11 @@
         VMSTATE_UINT32(smi_sts, ICH9LPCPMRegs),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_memhp_state,
-            .needed = vmstate_test_use_memhp,
-        },
-        VMSTATE_END_OF_LIST()
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_memhp_state,
+        &vmstate_tco_io_state,
+        &vmstate_cpuhp_state,
+        NULL
     }
 };
 
@@ -194,11 +251,12 @@
     acpi_pm_tmr_reset(&pm->acpi_regs);
     acpi_gpe_reset(&pm->acpi_regs);
 
-    if (kvm_enabled() || hax_enabled()) {
-        /* Mark SMM as already inited to prevent SMM from running. KVM does not
-         * support SMM mode. */
+    pm->smi_en = 0;
+    if (!pm->smm_enabled) {
+        /* Mark SMM as already inited to prevent SMM from running. */
         pm->smi_en |= ICH9_PMIO_SMI_EN_APMC_EN;
     }
+    pm->smi_en_wmask = ~0;
 
     acpi_update_sci(&pm->acpi_regs, pm->irq);
 }
@@ -211,6 +269,7 @@
 }
 
 void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
+                  bool smm_enabled,
                   qemu_irq sci_irq)
 {
     memory_region_init(&pm->io, OBJECT(lpc_pci), "ich9-pm", ICH9_PMIO_SIZE);
@@ -220,7 +279,8 @@
 
     acpi_pm_tmr_init(&pm->acpi_regs, ich9_pm_update_sci_fn, &pm->io);
     acpi_pm1_evt_init(&pm->acpi_regs, ich9_pm_update_sci_fn, &pm->io);
-    acpi_pm1_cnt_init(&pm->acpi_regs, &pm->io, 2);
+    acpi_pm1_cnt_init(&pm->acpi_regs, &pm->io, pm->disable_s3, pm->disable_s4,
+                      pm->s4_val);
 
     acpi_gpe_init(&pm->acpi_regs, ICH9_PMIO_GPE0_LEN);
     memory_region_init_io(&pm->io_gpe, OBJECT(lpc_pci), &ich9_gpe_ops, pm,
@@ -231,13 +291,18 @@
                           "acpi-smi", 8);
     memory_region_add_subregion(&pm->io, ICH9_PMIO_SMI_EN, &pm->io_smi);
 
+    pm->smm_enabled = smm_enabled;
+
+    pm->enable_tco = true;
+    acpi_pm_tco_init(&pm->tco_regs, &pm->io);
+
     pm->irq = sci_irq;
     qemu_register_reset(pm_reset, pm);
     pm->powerdown_notifier.notify = pm_powerdown_req;
     qemu_register_powerdown_notifier(&pm->powerdown_notifier);
 
-    acpi_cpu_hotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
-                          &pm->gpe_cpu, ICH9_CPU_HOTPLUG_IO_BASE);
+    legacy_acpi_cpu_hotplug_init(pci_address_space_io(lpc_pci),
+        OBJECT(lpc_pci), &pm->gpe_cpu, ICH9_CPU_HOTPLUG_IO_BASE);
 
     if (pm->acpi_memory_hotplug.is_enabled) {
         acpi_memory_hotplug_init(pci_address_space_io(lpc_pci), OBJECT(lpc_pci),
@@ -245,14 +310,13 @@
     }
 }
 
-static void ich9_pm_get_gpe0_blk(Object *obj, Visitor *v,
-                                 void *opaque, const char *name,
-                                 Error **errp)
+static void ich9_pm_get_gpe0_blk(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     ICH9LPCPMRegs *pm = opaque;
     uint32_t value = pm->pm_io_base + ICH9_PMIO_GPE0_STS;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static bool ich9_pm_get_memory_hotplug_support(Object *obj, Error **errp)
@@ -270,10 +334,121 @@
     s->pm.acpi_memory_hotplug.is_enabled = value;
 }
 
+static bool ich9_pm_get_cpu_hotplug_legacy(Object *obj, Error **errp)
+{
+    ICH9LPCState *s = ICH9_LPC_DEVICE(obj);
+
+    return s->pm.cpu_hotplug_legacy;
+}
+
+static void ich9_pm_set_cpu_hotplug_legacy(Object *obj, bool value,
+                                           Error **errp)
+{
+    ICH9LPCState *s = ICH9_LPC_DEVICE(obj);
+
+    assert(!value);
+    if (s->pm.cpu_hotplug_legacy && value == false) {
+        acpi_switch_to_modern_cphp(&s->pm.gpe_cpu, &s->pm.cpuhp_state,
+                                   ICH9_CPU_HOTPLUG_IO_BASE);
+    }
+    s->pm.cpu_hotplug_legacy = value;
+}
+
+static void ich9_pm_get_disable_s3(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint8_t value = pm->disable_s3;
+
+    visit_type_uint8(v, name, &value, errp);
+}
+
+static void ich9_pm_set_disable_s3(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    Error *local_err = NULL;
+    uint8_t value;
+
+    visit_type_uint8(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    pm->disable_s3 = value;
+out:
+    error_propagate(errp, local_err);
+}
+
+static void ich9_pm_get_disable_s4(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint8_t value = pm->disable_s4;
+
+    visit_type_uint8(v, name, &value, errp);
+}
+
+static void ich9_pm_set_disable_s4(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    Error *local_err = NULL;
+    uint8_t value;
+
+    visit_type_uint8(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    pm->disable_s4 = value;
+out:
+    error_propagate(errp, local_err);
+}
+
+static void ich9_pm_get_s4_val(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint8_t value = pm->s4_val;
+
+    visit_type_uint8(v, name, &value, errp);
+}
+
+static void ich9_pm_set_s4_val(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    Error *local_err = NULL;
+    uint8_t value;
+
+    visit_type_uint8(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    pm->s4_val = value;
+out:
+    error_propagate(errp, local_err);
+}
+
+static bool ich9_pm_get_enable_tco(Object *obj, Error **errp)
+{
+    ICH9LPCState *s = ICH9_LPC_DEVICE(obj);
+    return s->pm.enable_tco;
+}
+
+static void ich9_pm_set_enable_tco(Object *obj, bool value, Error **errp)
+{
+    ICH9LPCState *s = ICH9_LPC_DEVICE(obj);
+    s->pm.enable_tco = value;
+}
+
 void ich9_pm_add_properties(Object *obj, ICH9LPCPMRegs *pm, Error **errp)
 {
     static const uint32_t gpe0_len = ICH9_PMIO_GPE0_LEN;
     pm->acpi_memory_hotplug.is_enabled = true;
+    pm->cpu_hotplug_legacy = true;
+    pm->disable_s3 = 0;
+    pm->disable_s4 = 0;
+    pm->s4_val = 2;
 
     object_property_add_uint32_ptr(obj, ACPI_PM_PROP_PM_IO_BASE,
                                    &pm->pm_io_base, errp);
@@ -286,25 +461,92 @@
                              ich9_pm_get_memory_hotplug_support,
                              ich9_pm_set_memory_hotplug_support,
                              NULL);
+    object_property_add_bool(obj, "cpu-hotplug-legacy",
+                             ich9_pm_get_cpu_hotplug_legacy,
+                             ich9_pm_set_cpu_hotplug_legacy,
+                             NULL);
+    object_property_add(obj, ACPI_PM_PROP_S3_DISABLED, "uint8",
+                        ich9_pm_get_disable_s3,
+                        ich9_pm_set_disable_s3,
+                        NULL, pm, NULL);
+    object_property_add(obj, ACPI_PM_PROP_S4_DISABLED, "uint8",
+                        ich9_pm_get_disable_s4,
+                        ich9_pm_set_disable_s4,
+                        NULL, pm, NULL);
+    object_property_add(obj, ACPI_PM_PROP_S4_VAL, "uint8",
+                        ich9_pm_get_s4_val,
+                        ich9_pm_set_s4_val,
+                        NULL, pm, NULL);
+    object_property_add_bool(obj, ACPI_PM_PROP_TCO_ENABLED,
+                             ich9_pm_get_enable_tco,
+                             ich9_pm_set_enable_tco,
+                             NULL);
 }
 
-void ich9_pm_device_plug_cb(ICH9LPCPMRegs *pm, DeviceState *dev, Error **errp)
+void ich9_pm_device_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
+                            Error **errp)
 {
-    if (pm->acpi_memory_hotplug.is_enabled &&
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(hotplug_dev);
+
+    if (lpc->pm.acpi_memory_hotplug.is_enabled &&
         object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
-        acpi_memory_plug_cb(&pm->acpi_regs, pm->irq, &pm->acpi_memory_hotplug,
+        acpi_memory_plug_cb(hotplug_dev, &lpc->pm.acpi_memory_hotplug,
                             dev, errp);
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
-        acpi_cpu_plug_cb(&pm->acpi_regs, pm->irq, &pm->gpe_cpu, dev, errp);
+        if (lpc->pm.cpu_hotplug_legacy) {
+            legacy_acpi_cpu_plug_cb(hotplug_dev, &lpc->pm.gpe_cpu, dev, errp);
+        } else {
+            acpi_cpu_plug_cb(hotplug_dev, &lpc->pm.cpuhp_state, dev, errp);
+        }
     } else {
         error_setg(errp, "acpi: device plug request for not supported device"
                    " type: %s", object_get_typename(OBJECT(dev)));
     }
 }
 
+void ich9_pm_device_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(hotplug_dev);
+
+    if (lpc->pm.acpi_memory_hotplug.is_enabled &&
+        object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        acpi_memory_unplug_request_cb(hotplug_dev,
+                                      &lpc->pm.acpi_memory_hotplug, dev,
+                                      errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) &&
+               !lpc->pm.cpu_hotplug_legacy) {
+        acpi_cpu_unplug_request_cb(hotplug_dev, &lpc->pm.cpuhp_state,
+                                   dev, errp);
+    } else {
+        error_setg(errp, "acpi: device unplug request for not supported device"
+                   " type: %s", object_get_typename(OBJECT(dev)));
+    }
+}
+
+void ich9_pm_device_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
+                              Error **errp)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(hotplug_dev);
+
+    if (lpc->pm.acpi_memory_hotplug.is_enabled &&
+        object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        acpi_memory_unplug_cb(&lpc->pm.acpi_memory_hotplug, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) &&
+               !lpc->pm.cpu_hotplug_legacy) {
+        acpi_cpu_unplug_cb(&lpc->pm.cpuhp_state, dev, errp);
+    } else {
+        error_setg(errp, "acpi: device unplug for not supported device"
+                   " type: %s", object_get_typename(OBJECT(dev)));
+    }
+}
+
 void ich9_pm_ospm_status(AcpiDeviceIf *adev, ACPIOSTInfoList ***list)
 {
     ICH9LPCState *s = ICH9_LPC_DEVICE(adev);
 
     acpi_memory_ospm_status(&s->pm.acpi_memory_hotplug, list);
+    if (!s->pm.cpu_hotplug_legacy) {
+        acpi_cpu_ospm_status(&s->pm.cpuhp_state, list);
+    }
 }
diff --git a/hw/acpi/ipmi.c b/hw/acpi/ipmi.c
new file mode 100644
index 0000000..7e74ce4
--- /dev/null
+++ b/hw/acpi/ipmi.c
@@ -0,0 +1,105 @@
+/*
+ * IPMI ACPI firmware handling
+ *
+ * Copyright (c) 2015,2016 Corey Minyard, MontaVista Software, LLC
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/ipmi/ipmi.h"
+#include "hw/acpi/aml-build.h"
+#include "hw/acpi/acpi.h"
+#include "hw/acpi/ipmi.h"
+
+static Aml *aml_ipmi_crs(IPMIFwInfo *info)
+{
+    Aml *crs = aml_resource_template();
+
+    /*
+     * The base address is fixed and cannot change.  That may be different
+     * if someone does PCI, but we aren't there yet.
+     */
+    switch (info->memspace) {
+    case IPMI_MEMSPACE_IO:
+        aml_append(crs, aml_io(AML_DECODE16, info->base_address,
+                               info->base_address + info->register_length - 1,
+                               info->register_spacing, info->register_length));
+        break;
+    case IPMI_MEMSPACE_MEM32:
+        aml_append(crs,
+                   aml_dword_memory(AML_POS_DECODE,
+                            AML_MIN_FIXED, AML_MAX_FIXED,
+                            AML_NON_CACHEABLE, AML_READ_WRITE,
+                            0xffffffff,
+                            info->base_address,
+                            info->base_address + info->register_length - 1,
+                            info->register_spacing, info->register_length));
+        break;
+    case IPMI_MEMSPACE_MEM64:
+        aml_append(crs,
+                   aml_qword_memory(AML_POS_DECODE,
+                            AML_MIN_FIXED, AML_MAX_FIXED,
+                            AML_NON_CACHEABLE, AML_READ_WRITE,
+                            0xffffffffffffffffULL,
+                            info->base_address,
+                            info->base_address + info->register_length - 1,
+                            info->register_spacing, info->register_length));
+        break;
+    case IPMI_MEMSPACE_SMBUS:
+        aml_append(crs, aml_return(aml_int(info->base_address)));
+        break;
+    default:
+        abort();
+    }
+
+    if (info->interrupt_number) {
+        aml_append(crs, aml_irq_no_flags(info->interrupt_number));
+    }
+
+    return crs;
+}
+
+static Aml *aml_ipmi_device(IPMIFwInfo *info)
+{
+    Aml *dev;
+    uint16_t version = ((info->ipmi_spec_major_revision << 8)
+                        | (info->ipmi_spec_minor_revision << 4));
+
+    assert(info->ipmi_spec_minor_revision <= 15);
+
+    dev = aml_device("MI%d", info->uuid);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("IPI0001")));
+    aml_append(dev, aml_name_decl("_STR", aml_string("ipmi_%s",
+                                                     info->interface_name)));
+    aml_append(dev, aml_name_decl("_UID", aml_int(info->uuid)));
+    aml_append(dev, aml_name_decl("_CRS", aml_ipmi_crs(info)));
+    aml_append(dev, aml_name_decl("_IFT", aml_int(info->interface_type)));
+    aml_append(dev, aml_name_decl("_SRV", aml_int(version)));
+
+    return dev;
+}
+
+void build_acpi_ipmi_devices(Aml *scope, BusState *bus)
+{
+
+    BusChild *kid;
+
+    QTAILQ_FOREACH(kid, &bus->children,  sibling) {
+        IPMIInterface *ii;
+        IPMIInterfaceClass *iic;
+        IPMIFwInfo info;
+        Object *obj = object_dynamic_cast(OBJECT(kid->child),
+                                          TYPE_IPMI_INTERFACE);
+
+        if (!obj) {
+            continue;
+        }
+
+        ii = IPMI_INTERFACE(obj);
+        iic = IPMI_INTERFACE_GET_CLASS(obj);
+        iic->get_fwinfo(ii, &info);
+        aml_append(scope, aml_ipmi_device(&info));
+    }
+}
diff --git a/hw/acpi/memory_hotplug.c b/hw/acpi/memory_hotplug.c
index ed39241..ec4e64b 100644
--- a/hw/acpi/memory_hotplug.c
+++ b/hw/acpi/memory_hotplug.c
@@ -1,7 +1,9 @@
+#include "qemu/osdep.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "hw/acpi/pc-hotplug.h"
 #include "hw/mem/pc-dimm.h"
 #include "hw/boards.h"
+#include "hw/qdev-core.h"
 #include "trace.h"
 #include "qapi-event.h"
 
@@ -75,6 +77,7 @@
     case 0x14: /* pack and return is_* fields */
         val |= mdev->is_enabled   ? 1 : 0;
         val |= mdev->is_inserting ? 2 : 0;
+        val |= mdev->is_removing  ? 4 : 0;
         trace_mhp_acpi_read_flags(mem_st->selector, val);
         break;
     default:
@@ -90,6 +93,9 @@
     MemHotplugState *mem_st = opaque;
     MemStatus *mdev;
     ACPIOSTInfo *info;
+    DeviceState *dev = NULL;
+    HotplugHandler *hotplug_ctrl = NULL;
+    Error *local_err = NULL;
 
     if (!mem_st->dev_count) {
         return;
@@ -127,13 +133,37 @@
         qapi_event_send_acpi_device_ost(info, &error_abort);
         qapi_free_ACPIOSTInfo(info);
         break;
-    case 0x14:
+    case 0x14: /* set is_* fields  */
         mdev = &mem_st->devs[mem_st->selector];
         if (data & 2) { /* clear insert event */
             mdev->is_inserting  = false;
             trace_mhp_acpi_clear_insert_evt(mem_st->selector);
+        } else if (data & 4) {
+            mdev->is_removing = false;
+            trace_mhp_acpi_clear_remove_evt(mem_st->selector);
+        } else if (data & 8) {
+            if (!mdev->is_enabled) {
+                trace_mhp_acpi_ejecting_invalid_slot(mem_st->selector);
+                break;
+            }
+
+            dev = DEVICE(mdev->dimm);
+            hotplug_ctrl = qdev_get_hotplug_handler(dev);
+            /* call pc-dimm unplug cb */
+            hotplug_handler_unplug(hotplug_ctrl, dev, &local_err);
+            if (local_err) {
+                trace_mhp_acpi_pc_dimm_delete_failed(mem_st->selector);
+                qapi_event_send_mem_unplug_error(dev->id,
+                                                 error_get_pretty(local_err),
+                                                 &error_abort);
+                error_free(local_err);
+                break;
+            }
+            trace_mhp_acpi_pc_dimm_deleted(mem_st->selector);
         }
         break;
+    default:
+        break;
     }
 
 }
@@ -163,36 +193,91 @@
     memory_region_add_subregion(as, ACPI_MEMORY_HOTPLUG_BASE, &state->io);
 }
 
-void acpi_memory_plug_cb(ACPIREGS *ar, qemu_irq irq, MemHotplugState *mem_st,
-                         DeviceState *dev, Error **errp)
+/**
+ * acpi_memory_slot_status:
+ * @mem_st: memory hotplug state
+ * @dev: device
+ * @errp: set in case of an error
+ *
+ * Obtain a single memory slot status.
+ *
+ * This function will be called by memory unplug request cb and unplug cb.
+ */
+static MemStatus *
+acpi_memory_slot_status(MemHotplugState *mem_st,
+                        DeviceState *dev, Error **errp)
 {
-    MemStatus *mdev;
     Error *local_err = NULL;
-    int slot = object_property_get_int(OBJECT(dev), "slot", &local_err);
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                       &local_err);
 
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        return NULL;
     }
 
     if (slot >= mem_st->dev_count) {
         char *dev_path = object_get_canonical_path(OBJECT(dev));
-        error_setg(errp, "acpi_memory_plug_cb: "
+        error_setg(errp, "acpi_memory_slot_status: "
                    "device [%s] returned invalid memory slot[%d]",
                     dev_path, slot);
         g_free(dev_path);
+        return NULL;
+    }
+
+    return &mem_st->devs[slot];
+}
+
+void acpi_memory_plug_cb(HotplugHandler *hotplug_dev, MemHotplugState *mem_st,
+                         DeviceState *dev, Error **errp)
+{
+    MemStatus *mdev;
+    DeviceClass *dc = DEVICE_GET_CLASS(dev);
+
+    if (!dc->hotpluggable) {
         return;
     }
 
-    mdev = &mem_st->devs[slot];
+    mdev = acpi_memory_slot_status(mem_st, dev, errp);
+    if (!mdev) {
+        return;
+    }
+
     mdev->dimm = dev;
     mdev->is_enabled = true;
-    mdev->is_inserting = true;
+    if (dev->hotplugged) {
+        mdev->is_inserting = true;
+        acpi_send_event(DEVICE(hotplug_dev), ACPI_MEMORY_HOTPLUG_STATUS);
+    }
+}
 
-    /* do ACPI magic */
-    ar->gpe.sts[0] |= ACPI_MEMORY_HOTPLUG_STATUS;
-    acpi_update_sci(ar, irq);
-    return;
+void acpi_memory_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                   MemHotplugState *mem_st,
+                                   DeviceState *dev, Error **errp)
+{
+    MemStatus *mdev;
+
+    mdev = acpi_memory_slot_status(mem_st, dev, errp);
+    if (!mdev) {
+        return;
+    }
+
+    mdev->is_removing = true;
+    acpi_send_event(DEVICE(hotplug_dev), ACPI_MEMORY_HOTPLUG_STATUS);
+}
+
+void acpi_memory_unplug_cb(MemHotplugState *mem_st,
+                           DeviceState *dev, Error **errp)
+{
+    MemStatus *mdev;
+
+    mdev = acpi_memory_slot_status(mem_st, dev, errp);
+    if (!mdev) {
+        return;
+    }
+
+    mdev->is_enabled = false;
+    mdev->dimm = NULL;
 }
 
 static const VMStateDescription vmstate_memhp_sts = {
diff --git a/hw/acpi/memory_hotplug_acpi_table.c b/hw/acpi/memory_hotplug_acpi_table.c
new file mode 100644
index 0000000..c756602
--- /dev/null
+++ b/hw/acpi/memory_hotplug_acpi_table.c
@@ -0,0 +1,262 @@
+/*
+ * Memory hotplug AML code of DSDT ACPI table
+ *
+ * Copyright (C) 2015 Red Hat Inc
+ *
+ * Author: Igor Mammedov <imammedo@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/acpi/memory_hotplug.h"
+#include "include/hw/acpi/pc-hotplug.h"
+#include "hw/boards.h"
+
+void build_memory_hotplug_aml(Aml *ctx, uint32_t nr_mem,
+                              uint16_t io_base, uint16_t io_len)
+{
+    Aml *ifctx;
+    Aml *method;
+    Aml *pci_scope;
+    Aml *mem_ctrl_dev;
+
+    /* scope for memory hotplug controller device node */
+    pci_scope = aml_scope("_SB.PCI0");
+    mem_ctrl_dev = aml_device(MEMORY_HOTPLUG_DEVICE);
+    {
+        Aml *one = aml_int(1);
+        Aml *zero = aml_int(0);
+        Aml *ret_val = aml_local(0);
+        Aml *slot_arg0 = aml_arg(0);
+        Aml *slots_nr = aml_name(MEMORY_SLOTS_NUMBER);
+        Aml *ctrl_lock = aml_name(MEMORY_SLOT_LOCK);
+        Aml *slot_selector = aml_name(MEMORY_SLOT_SLECTOR);
+
+        aml_append(mem_ctrl_dev, aml_name_decl("_HID", aml_string("PNP0A06")));
+        aml_append(mem_ctrl_dev,
+            aml_name_decl("_UID", aml_string("Memory hotplug resources")));
+
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        ifctx = aml_if(aml_equal(slots_nr, zero));
+        {
+            aml_append(ifctx, aml_return(zero));
+        }
+        aml_append(method, ifctx);
+        /* present, functioning, decoding, not shown in UI */
+        aml_append(method, aml_return(aml_int(0xB)));
+        aml_append(mem_ctrl_dev, method);
+
+        aml_append(mem_ctrl_dev, aml_mutex(MEMORY_SLOT_LOCK, 0));
+
+        method = aml_method(MEMORY_SLOT_SCAN_METHOD, 0, AML_NOTSERIALIZED);
+        {
+            Aml *else_ctx;
+            Aml *while_ctx;
+            Aml *idx = aml_local(0);
+            Aml *eject_req = aml_int(3);
+            Aml *dev_chk = aml_int(1);
+
+            ifctx = aml_if(aml_equal(slots_nr, zero));
+            {
+                aml_append(ifctx, aml_return(zero));
+            }
+            aml_append(method, ifctx);
+
+            aml_append(method, aml_store(zero, idx));
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            /* build AML that:
+             * loops over all slots and Notifies DIMMs with
+             * Device Check or Eject Request notifications if
+             * slot has corresponding status bit set and clears
+             * slot status.
+             */
+            while_ctx = aml_while(aml_lless(idx, slots_nr));
+            {
+                Aml *ins_evt = aml_name(MEMORY_SLOT_INSERT_EVENT);
+                Aml *rm_evt = aml_name(MEMORY_SLOT_REMOVE_EVENT);
+
+                aml_append(while_ctx, aml_store(idx, slot_selector));
+                ifctx = aml_if(aml_equal(ins_evt, one));
+                {
+                    aml_append(ifctx,
+                               aml_call2(MEMORY_SLOT_NOTIFY_METHOD,
+                                         idx, dev_chk));
+                    aml_append(ifctx, aml_store(one, ins_evt));
+                }
+                aml_append(while_ctx, ifctx);
+
+                else_ctx = aml_else();
+                ifctx = aml_if(aml_equal(rm_evt, one));
+                {
+                    aml_append(ifctx,
+                        aml_call2(MEMORY_SLOT_NOTIFY_METHOD,
+                                  idx, eject_req));
+                    aml_append(ifctx, aml_store(one, rm_evt));
+                }
+                aml_append(else_ctx, ifctx);
+                aml_append(while_ctx, else_ctx);
+
+                aml_append(while_ctx, aml_add(idx, one, idx));
+            }
+            aml_append(method, while_ctx);
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(one));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_STATUS_METHOD, 1, AML_NOTSERIALIZED);
+        {
+            Aml *slot_enabled = aml_name(MEMORY_SLOT_ENABLED);
+
+            aml_append(method, aml_store(zero, ret_val));
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method,
+                aml_store(aml_to_integer(slot_arg0), slot_selector));
+
+            ifctx = aml_if(aml_equal(slot_enabled, one));
+            {
+                aml_append(ifctx, aml_store(aml_int(0xF), ret_val));
+            }
+            aml_append(method, ifctx);
+
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(ret_val));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_CRS_METHOD, 1, AML_SERIALIZED);
+        {
+            Aml *mr64 = aml_name("MR64");
+            Aml *mr32 = aml_name("MR32");
+            Aml *crs_tmpl = aml_resource_template();
+            Aml *minl = aml_name("MINL");
+            Aml *minh = aml_name("MINH");
+            Aml *maxl =  aml_name("MAXL");
+            Aml *maxh =  aml_name("MAXH");
+            Aml *lenl = aml_name("LENL");
+            Aml *lenh = aml_name("LENH");
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+
+            aml_append(crs_tmpl,
+                aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                                 AML_CACHEABLE, AML_READ_WRITE,
+                                 0, 0x0, 0xFFFFFFFFFFFFFFFEULL, 0,
+                                 0xFFFFFFFFFFFFFFFFULL));
+            aml_append(method, aml_name_decl("MR64", crs_tmpl));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(14), "MINL"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(18), "MINH"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(38), "LENL"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(42), "LENH"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(22), "MAXL"));
+            aml_append(method,
+                aml_create_dword_field(mr64, aml_int(26), "MAXH"));
+
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_ADDR_HIGH), minh));
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_ADDR_LOW), minl));
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_SIZE_HIGH), lenh));
+            aml_append(method,
+                aml_store(aml_name(MEMORY_SLOT_SIZE_LOW), lenl));
+
+            /* 64-bit math: MAX = MIN + LEN - 1 */
+            aml_append(method, aml_add(minl, lenl, maxl));
+            aml_append(method, aml_add(minh, lenh, maxh));
+            ifctx = aml_if(aml_lless(maxl, minl));
+            {
+                aml_append(ifctx, aml_add(maxh, one, maxh));
+            }
+            aml_append(method, ifctx);
+            ifctx = aml_if(aml_lless(maxl, one));
+            {
+                aml_append(ifctx, aml_subtract(maxh, one, maxh));
+            }
+            aml_append(method, ifctx);
+            aml_append(method, aml_subtract(maxl, one, maxl));
+
+            /* return 32-bit _CRS if addr/size is in low mem */
+            /* TODO: remove it since all hotplugged DIMMs are in high mem */
+            ifctx = aml_if(aml_equal(maxh, zero));
+            {
+                crs_tmpl = aml_resource_template();
+                aml_append(crs_tmpl,
+                    aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                                     AML_MAX_FIXED, AML_CACHEABLE,
+                                     AML_READ_WRITE,
+                                     0, 0x0, 0xFFFFFFFE, 0,
+                                     0xFFFFFFFF));
+                aml_append(ifctx, aml_name_decl("MR32", crs_tmpl));
+                aml_append(ifctx,
+                    aml_create_dword_field(mr32, aml_int(10), "MIN"));
+                aml_append(ifctx,
+                    aml_create_dword_field(mr32, aml_int(14), "MAX"));
+                aml_append(ifctx,
+                    aml_create_dword_field(mr32, aml_int(22), "LEN"));
+                aml_append(ifctx, aml_store(minl, aml_name("MIN")));
+                aml_append(ifctx, aml_store(maxl, aml_name("MAX")));
+                aml_append(ifctx, aml_store(lenl, aml_name("LEN")));
+
+                aml_append(ifctx, aml_release(ctrl_lock));
+                aml_append(ifctx, aml_return(mr32));
+            }
+            aml_append(method, ifctx);
+
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(mr64));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_PROXIMITY_METHOD, 1,
+                            AML_NOTSERIALIZED);
+        {
+            Aml *proximity = aml_name(MEMORY_SLOT_PROXIMITY);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+            aml_append(method, aml_store(proximity, ret_val));
+            aml_append(method, aml_release(ctrl_lock));
+            aml_append(method, aml_return(ret_val));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_OST_METHOD, 4, AML_NOTSERIALIZED);
+        {
+            Aml *ost_evt = aml_name(MEMORY_SLOT_OST_EVENT);
+            Aml *ost_status = aml_name(MEMORY_SLOT_OST_STATUS);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+            aml_append(method, aml_store(aml_arg(1), ost_evt));
+            aml_append(method, aml_store(aml_arg(2), ost_status));
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(mem_ctrl_dev, method);
+
+        method = aml_method(MEMORY_SLOT_EJECT_METHOD, 2, AML_NOTSERIALIZED);
+        {
+            Aml *eject = aml_name(MEMORY_SLOT_EJECT);
+
+            aml_append(method, aml_acquire(ctrl_lock, 0xFFFF));
+            aml_append(method, aml_store(aml_to_integer(slot_arg0),
+                                         slot_selector));
+            aml_append(method, aml_store(one, eject));
+            aml_append(method, aml_release(ctrl_lock));
+        }
+        aml_append(mem_ctrl_dev, method);
+    }
+    aml_append(pci_scope, mem_ctrl_dev);
+    aml_append(ctx, pci_scope);
+}
diff --git a/hw/acpi/nvdimm.c b/hw/acpi/nvdimm.c
new file mode 100644
index 0000000..e486128
--- /dev/null
+++ b/hw/acpi/nvdimm.c
@@ -0,0 +1,1042 @@
+/*
+ * NVDIMM ACPI Implementation
+ *
+ * Copyright(C) 2015 Intel Corporation.
+ *
+ * Author:
+ *  Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * NFIT is defined in ACPI 6.0: 5.2.25 NVDIMM Firmware Interface Table (NFIT)
+ * and the DSM specification can be found at:
+ *       http://pmem.io/documents/NVDIMM_DSM_Interface_Example.pdf
+ *
+ * Currently, it only supports PMEM Virtualization.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu/osdep.h"
+#include "hw/acpi/acpi.h"
+#include "hw/acpi/aml-build.h"
+#include "hw/acpi/bios-linker-loader.h"
+#include "hw/nvram/fw_cfg.h"
+#include "hw/mem/nvdimm.h"
+
+static int nvdimm_plugged_device_list(Object *obj, void *opaque)
+{
+    GSList **list = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_NVDIMM)) {
+        DeviceState *dev = DEVICE(obj);
+
+        if (dev->realized) { /* only realized NVDIMMs matter */
+            *list = g_slist_append(*list, DEVICE(obj));
+        }
+    }
+
+    object_child_foreach(obj, nvdimm_plugged_device_list, opaque);
+    return 0;
+}
+
+/*
+ * inquire plugged NVDIMM devices and link them into the list which is
+ * returned to the caller.
+ *
+ * Note: it is the caller's responsibility to free the list to avoid
+ * memory leak.
+ */
+static GSList *nvdimm_get_plugged_device_list(void)
+{
+    GSList *list = NULL;
+
+    object_child_foreach(qdev_get_machine(), nvdimm_plugged_device_list,
+                         &list);
+    return list;
+}
+
+#define NVDIMM_UUID_LE(a, b, c, d0, d1, d2, d3, d4, d5, d6, d7)             \
+   { (a) & 0xff, ((a) >> 8) & 0xff, ((a) >> 16) & 0xff, ((a) >> 24) & 0xff, \
+     (b) & 0xff, ((b) >> 8) & 0xff, (c) & 0xff, ((c) >> 8) & 0xff,          \
+     (d0), (d1), (d2), (d3), (d4), (d5), (d6), (d7) }
+
+/*
+ * define Byte Addressable Persistent Memory (PM) Region according to
+ * ACPI 6.0: 5.2.25.1 System Physical Address Range Structure.
+ */
+static const uint8_t nvdimm_nfit_spa_uuid[] =
+      NVDIMM_UUID_LE(0x66f0d379, 0xb4f3, 0x4074, 0xac, 0x43, 0x0d, 0x33,
+                     0x18, 0xb7, 0x8c, 0xdb);
+
+/*
+ * NVDIMM Firmware Interface Table
+ * @signature: "NFIT"
+ *
+ * It provides information that allows OSPM to enumerate NVDIMM present in
+ * the platform and associate system physical address ranges created by the
+ * NVDIMMs.
+ *
+ * It is defined in ACPI 6.0: 5.2.25 NVDIMM Firmware Interface Table (NFIT)
+ */
+struct NvdimmNfitHeader {
+    ACPI_TABLE_HEADER_DEF
+    uint32_t reserved;
+} QEMU_PACKED;
+typedef struct NvdimmNfitHeader NvdimmNfitHeader;
+
+/*
+ * define NFIT structures according to ACPI 6.0: 5.2.25 NVDIMM Firmware
+ * Interface Table (NFIT).
+ */
+
+/*
+ * System Physical Address Range Structure
+ *
+ * It describes the system physical address ranges occupied by NVDIMMs and
+ * the types of the regions.
+ */
+struct NvdimmNfitSpa {
+    uint16_t type;
+    uint16_t length;
+    uint16_t spa_index;
+    uint16_t flags;
+    uint32_t reserved;
+    uint32_t proximity_domain;
+    uint8_t type_guid[16];
+    uint64_t spa_base;
+    uint64_t spa_length;
+    uint64_t mem_attr;
+} QEMU_PACKED;
+typedef struct NvdimmNfitSpa NvdimmNfitSpa;
+
+/*
+ * Memory Device to System Physical Address Range Mapping Structure
+ *
+ * It enables identifying each NVDIMM region and the corresponding SPA
+ * describing the memory interleave
+ */
+struct NvdimmNfitMemDev {
+    uint16_t type;
+    uint16_t length;
+    uint32_t nfit_handle;
+    uint16_t phys_id;
+    uint16_t region_id;
+    uint16_t spa_index;
+    uint16_t dcr_index;
+    uint64_t region_len;
+    uint64_t region_offset;
+    uint64_t region_dpa;
+    uint16_t interleave_index;
+    uint16_t interleave_ways;
+    uint16_t flags;
+    uint16_t reserved;
+} QEMU_PACKED;
+typedef struct NvdimmNfitMemDev NvdimmNfitMemDev;
+
+/*
+ * NVDIMM Control Region Structure
+ *
+ * It describes the NVDIMM and if applicable, Block Control Window.
+ */
+struct NvdimmNfitControlRegion {
+    uint16_t type;
+    uint16_t length;
+    uint16_t dcr_index;
+    uint16_t vendor_id;
+    uint16_t device_id;
+    uint16_t revision_id;
+    uint16_t sub_vendor_id;
+    uint16_t sub_device_id;
+    uint16_t sub_revision_id;
+    uint8_t reserved[6];
+    uint32_t serial_number;
+    uint16_t fic;
+    uint16_t num_bcw;
+    uint64_t bcw_size;
+    uint64_t cmd_offset;
+    uint64_t cmd_size;
+    uint64_t status_offset;
+    uint64_t status_size;
+    uint16_t flags;
+    uint8_t reserved2[6];
+} QEMU_PACKED;
+typedef struct NvdimmNfitControlRegion NvdimmNfitControlRegion;
+
+/*
+ * Module serial number is a unique number for each device. We use the
+ * slot id of NVDIMM device to generate this number so that each device
+ * associates with a different number.
+ *
+ * 0x123456 is a magic number we arbitrarily chose.
+ */
+static uint32_t nvdimm_slot_to_sn(int slot)
+{
+    return 0x123456 + slot;
+}
+
+/*
+ * handle is used to uniquely associate nfit_memdev structure with NVDIMM
+ * ACPI device - nfit_memdev.nfit_handle matches with the value returned
+ * by ACPI device _ADR method.
+ *
+ * We generate the handle with the slot id of NVDIMM device and reserve
+ * 0 for NVDIMM root device.
+ */
+static uint32_t nvdimm_slot_to_handle(int slot)
+{
+    return slot + 1;
+}
+
+/*
+ * index uniquely identifies the structure, 0 is reserved which indicates
+ * that the structure is not valid or the associated structure is not
+ * present.
+ *
+ * Each NVDIMM device needs two indexes, one for nfit_spa and another for
+ * nfit_dc which are generated by the slot id of NVDIMM device.
+ */
+static uint16_t nvdimm_slot_to_spa_index(int slot)
+{
+    return (slot + 1) << 1;
+}
+
+/* See the comments of nvdimm_slot_to_spa_index(). */
+static uint32_t nvdimm_slot_to_dcr_index(int slot)
+{
+    return nvdimm_slot_to_spa_index(slot) + 1;
+}
+
+static NVDIMMDevice *nvdimm_get_device_by_handle(uint32_t handle)
+{
+    NVDIMMDevice *nvdimm = NULL;
+    GSList *list, *device_list = nvdimm_get_plugged_device_list();
+
+    for (list = device_list; list; list = list->next) {
+        NVDIMMDevice *nvd = list->data;
+        int slot = object_property_get_int(OBJECT(nvd), PC_DIMM_SLOT_PROP,
+                                           NULL);
+
+        if (nvdimm_slot_to_handle(slot) == handle) {
+            nvdimm = nvd;
+            break;
+        }
+    }
+
+    g_slist_free(device_list);
+    return nvdimm;
+}
+
+/* ACPI 6.0: 5.2.25.1 System Physical Address Range Structure */
+static void
+nvdimm_build_structure_spa(GArray *structures, DeviceState *dev)
+{
+    NvdimmNfitSpa *nfit_spa;
+    uint64_t addr = object_property_get_int(OBJECT(dev), PC_DIMM_ADDR_PROP,
+                                            NULL);
+    uint64_t size = object_property_get_int(OBJECT(dev), PC_DIMM_SIZE_PROP,
+                                            NULL);
+    uint32_t node = object_property_get_int(OBJECT(dev), PC_DIMM_NODE_PROP,
+                                            NULL);
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                            NULL);
+
+    nfit_spa = acpi_data_push(structures, sizeof(*nfit_spa));
+
+    nfit_spa->type = cpu_to_le16(0 /* System Physical Address Range
+                                      Structure */);
+    nfit_spa->length = cpu_to_le16(sizeof(*nfit_spa));
+    nfit_spa->spa_index = cpu_to_le16(nvdimm_slot_to_spa_index(slot));
+
+    /*
+     * Control region is strict as all the device info, such as SN, index,
+     * is associated with slot id.
+     */
+    nfit_spa->flags = cpu_to_le16(1 /* Control region is strictly for
+                                       management during hot add/online
+                                       operation */ |
+                                  2 /* Data in Proximity Domain field is
+                                       valid*/);
+
+    /* NUMA node. */
+    nfit_spa->proximity_domain = cpu_to_le32(node);
+    /* the region reported as PMEM. */
+    memcpy(nfit_spa->type_guid, nvdimm_nfit_spa_uuid,
+           sizeof(nvdimm_nfit_spa_uuid));
+
+    nfit_spa->spa_base = cpu_to_le64(addr);
+    nfit_spa->spa_length = cpu_to_le64(size);
+
+    /* It is the PMEM and can be cached as writeback. */
+    nfit_spa->mem_attr = cpu_to_le64(0x8ULL /* EFI_MEMORY_WB */ |
+                                     0x8000ULL /* EFI_MEMORY_NV */);
+}
+
+/*
+ * ACPI 6.0: 5.2.25.2 Memory Device to System Physical Address Range Mapping
+ * Structure
+ */
+static void
+nvdimm_build_structure_memdev(GArray *structures, DeviceState *dev)
+{
+    NvdimmNfitMemDev *nfit_memdev;
+    uint64_t addr = object_property_get_int(OBJECT(dev), PC_DIMM_ADDR_PROP,
+                                            NULL);
+    uint64_t size = object_property_get_int(OBJECT(dev), PC_DIMM_SIZE_PROP,
+                                            NULL);
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                            NULL);
+    uint32_t handle = nvdimm_slot_to_handle(slot);
+
+    nfit_memdev = acpi_data_push(structures, sizeof(*nfit_memdev));
+
+    nfit_memdev->type = cpu_to_le16(1 /* Memory Device to System Address
+                                         Range Map Structure*/);
+    nfit_memdev->length = cpu_to_le16(sizeof(*nfit_memdev));
+    nfit_memdev->nfit_handle = cpu_to_le32(handle);
+
+    /*
+     * associate memory device with System Physical Address Range
+     * Structure.
+     */
+    nfit_memdev->spa_index = cpu_to_le16(nvdimm_slot_to_spa_index(slot));
+    /* associate memory device with Control Region Structure. */
+    nfit_memdev->dcr_index = cpu_to_le16(nvdimm_slot_to_dcr_index(slot));
+
+    /* The memory region on the device. */
+    nfit_memdev->region_len = cpu_to_le64(size);
+    nfit_memdev->region_dpa = cpu_to_le64(addr);
+
+    /* Only one interleave for PMEM. */
+    nfit_memdev->interleave_ways = cpu_to_le16(1);
+}
+
+/*
+ * ACPI 6.0: 5.2.25.5 NVDIMM Control Region Structure.
+ */
+static void nvdimm_build_structure_dcr(GArray *structures, DeviceState *dev)
+{
+    NvdimmNfitControlRegion *nfit_dcr;
+    int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                       NULL);
+    uint32_t sn = nvdimm_slot_to_sn(slot);
+
+    nfit_dcr = acpi_data_push(structures, sizeof(*nfit_dcr));
+
+    nfit_dcr->type = cpu_to_le16(4 /* NVDIMM Control Region Structure */);
+    nfit_dcr->length = cpu_to_le16(sizeof(*nfit_dcr));
+    nfit_dcr->dcr_index = cpu_to_le16(nvdimm_slot_to_dcr_index(slot));
+
+    /* vendor: Intel. */
+    nfit_dcr->vendor_id = cpu_to_le16(0x8086);
+    nfit_dcr->device_id = cpu_to_le16(1);
+
+    /* The _DSM method is following Intel's DSM specification. */
+    nfit_dcr->revision_id = cpu_to_le16(1 /* Current Revision supported
+                                             in ACPI 6.0 is 1. */);
+    nfit_dcr->serial_number = cpu_to_le32(sn);
+    nfit_dcr->fic = cpu_to_le16(0x201 /* Format Interface Code. See Chapter
+                                         2: NVDIMM Device Specific Method
+                                         (DSM) in DSM Spec Rev1.*/);
+}
+
+static GArray *nvdimm_build_device_structure(GSList *device_list)
+{
+    GArray *structures = g_array_new(false, true /* clear */, 1);
+
+    for (; device_list; device_list = device_list->next) {
+        DeviceState *dev = device_list->data;
+
+        /* build System Physical Address Range Structure. */
+        nvdimm_build_structure_spa(structures, dev);
+
+        /*
+         * build Memory Device to System Physical Address Range Mapping
+         * Structure.
+         */
+        nvdimm_build_structure_memdev(structures, dev);
+
+        /* build NVDIMM Control Region Structure. */
+        nvdimm_build_structure_dcr(structures, dev);
+    }
+
+    return structures;
+}
+
+static void nvdimm_build_nfit(GSList *device_list, GArray *table_offsets,
+                              GArray *table_data, BIOSLinker *linker)
+{
+    GArray *structures = nvdimm_build_device_structure(device_list);
+    unsigned int header;
+
+    acpi_add_table(table_offsets, table_data);
+
+    /* NFIT header. */
+    header = table_data->len;
+    acpi_data_push(table_data, sizeof(NvdimmNfitHeader));
+    /* NVDIMM device structures. */
+    g_array_append_vals(table_data, structures->data, structures->len);
+
+    build_header(linker, table_data,
+                 (void *)(table_data->data + header), "NFIT",
+                 sizeof(NvdimmNfitHeader) + structures->len, 1, NULL, NULL);
+    g_array_free(structures, true);
+}
+
+struct NvdimmDsmIn {
+    uint32_t handle;
+    uint32_t revision;
+    uint32_t function;
+    /* the remaining size in the page is used by arg3. */
+    union {
+        uint8_t arg3[4084];
+    };
+} QEMU_PACKED;
+typedef struct NvdimmDsmIn NvdimmDsmIn;
+QEMU_BUILD_BUG_ON(sizeof(NvdimmDsmIn) != 4096);
+
+struct NvdimmDsmOut {
+    /* the size of buffer filled by QEMU. */
+    uint32_t len;
+    uint8_t data[4092];
+} QEMU_PACKED;
+typedef struct NvdimmDsmOut NvdimmDsmOut;
+QEMU_BUILD_BUG_ON(sizeof(NvdimmDsmOut) != 4096);
+
+struct NvdimmDsmFunc0Out {
+    /* the size of buffer filled by QEMU. */
+     uint32_t len;
+     uint32_t supported_func;
+} QEMU_PACKED;
+typedef struct NvdimmDsmFunc0Out NvdimmDsmFunc0Out;
+
+struct NvdimmDsmFuncNoPayloadOut {
+    /* the size of buffer filled by QEMU. */
+     uint32_t len;
+     uint32_t func_ret_status;
+} QEMU_PACKED;
+typedef struct NvdimmDsmFuncNoPayloadOut NvdimmDsmFuncNoPayloadOut;
+
+struct NvdimmFuncGetLabelSizeOut {
+    /* the size of buffer filled by QEMU. */
+    uint32_t len;
+    uint32_t func_ret_status; /* return status code. */
+    uint32_t label_size; /* the size of label data area. */
+    /*
+     * Maximum size of the namespace label data length supported by
+     * the platform in Get/Set Namespace Label Data functions.
+     */
+    uint32_t max_xfer;
+} QEMU_PACKED;
+typedef struct NvdimmFuncGetLabelSizeOut NvdimmFuncGetLabelSizeOut;
+QEMU_BUILD_BUG_ON(sizeof(NvdimmFuncGetLabelSizeOut) > 4096);
+
+struct NvdimmFuncGetLabelDataIn {
+    uint32_t offset; /* the offset in the namespace label data area. */
+    uint32_t length; /* the size of data is to be read via the function. */
+} QEMU_PACKED;
+typedef struct NvdimmFuncGetLabelDataIn NvdimmFuncGetLabelDataIn;
+QEMU_BUILD_BUG_ON(sizeof(NvdimmFuncGetLabelDataIn) +
+                  offsetof(NvdimmDsmIn, arg3) > 4096);
+
+struct NvdimmFuncGetLabelDataOut {
+    /* the size of buffer filled by QEMU. */
+    uint32_t len;
+    uint32_t func_ret_status; /* return status code. */
+    uint8_t out_buf[0]; /* the data got via Get Namesapce Label function. */
+} QEMU_PACKED;
+typedef struct NvdimmFuncGetLabelDataOut NvdimmFuncGetLabelDataOut;
+QEMU_BUILD_BUG_ON(sizeof(NvdimmFuncGetLabelDataOut) > 4096);
+
+struct NvdimmFuncSetLabelDataIn {
+    uint32_t offset; /* the offset in the namespace label data area. */
+    uint32_t length; /* the size of data is to be written via the function. */
+    uint8_t in_buf[0]; /* the data written to label data area. */
+} QEMU_PACKED;
+typedef struct NvdimmFuncSetLabelDataIn NvdimmFuncSetLabelDataIn;
+QEMU_BUILD_BUG_ON(sizeof(NvdimmFuncSetLabelDataIn) +
+                  offsetof(NvdimmDsmIn, arg3) > 4096);
+
+static void
+nvdimm_dsm_function0(uint32_t supported_func, hwaddr dsm_mem_addr)
+{
+    NvdimmDsmFunc0Out func0 = {
+        .len = cpu_to_le32(sizeof(func0)),
+        .supported_func = cpu_to_le32(supported_func),
+    };
+    cpu_physical_memory_write(dsm_mem_addr, &func0, sizeof(func0));
+}
+
+static void
+nvdimm_dsm_no_payload(uint32_t func_ret_status, hwaddr dsm_mem_addr)
+{
+    NvdimmDsmFuncNoPayloadOut out = {
+        .len = cpu_to_le32(sizeof(out)),
+        .func_ret_status = cpu_to_le32(func_ret_status),
+    };
+    cpu_physical_memory_write(dsm_mem_addr, &out, sizeof(out));
+}
+
+static void nvdimm_dsm_root(NvdimmDsmIn *in, hwaddr dsm_mem_addr)
+{
+    /*
+     * function 0 is called to inquire which functions are supported by
+     * OSPM
+     */
+    if (!in->function) {
+        nvdimm_dsm_function0(0 /* No function supported other than
+                                  function 0 */, dsm_mem_addr);
+        return;
+    }
+
+    /* No function except function 0 is supported yet. */
+    nvdimm_dsm_no_payload(1 /* Not Supported */, dsm_mem_addr);
+}
+
+/*
+ * the max transfer size is the max size transferred by both a
+ * 'Get Namespace Label Data' function and a 'Set Namespace Label Data'
+ * function.
+ */
+static uint32_t nvdimm_get_max_xfer_label_size(void)
+{
+    uint32_t max_get_size, max_set_size, dsm_memory_size = 4096;
+
+    /*
+     * the max data ACPI can read one time which is transferred by
+     * the response of 'Get Namespace Label Data' function.
+     */
+    max_get_size = dsm_memory_size - sizeof(NvdimmFuncGetLabelDataOut);
+
+    /*
+     * the max data ACPI can write one time which is transferred by
+     * 'Set Namespace Label Data' function.
+     */
+    max_set_size = dsm_memory_size - offsetof(NvdimmDsmIn, arg3) -
+                   sizeof(NvdimmFuncSetLabelDataIn);
+
+    return MIN(max_get_size, max_set_size);
+}
+
+/*
+ * DSM Spec Rev1 4.4 Get Namespace Label Size (Function Index 4).
+ *
+ * It gets the size of Namespace Label data area and the max data size
+ * that Get/Set Namespace Label Data functions can transfer.
+ */
+static void nvdimm_dsm_label_size(NVDIMMDevice *nvdimm, hwaddr dsm_mem_addr)
+{
+    NvdimmFuncGetLabelSizeOut label_size_out = {
+        .len = cpu_to_le32(sizeof(label_size_out)),
+    };
+    uint32_t label_size, mxfer;
+
+    label_size = nvdimm->label_size;
+    mxfer = nvdimm_get_max_xfer_label_size();
+
+    nvdimm_debug("label_size %#x, max_xfer %#x.\n", label_size, mxfer);
+
+    label_size_out.func_ret_status = cpu_to_le32(0 /* Success */);
+    label_size_out.label_size = cpu_to_le32(label_size);
+    label_size_out.max_xfer = cpu_to_le32(mxfer);
+
+    cpu_physical_memory_write(dsm_mem_addr, &label_size_out,
+                              sizeof(label_size_out));
+}
+
+static uint32_t nvdimm_rw_label_data_check(NVDIMMDevice *nvdimm,
+                                           uint32_t offset, uint32_t length)
+{
+    uint32_t ret = 3 /* Invalid Input Parameters */;
+
+    if (offset + length < offset) {
+        nvdimm_debug("offset %#x + length %#x is overflow.\n", offset,
+                     length);
+        return ret;
+    }
+
+    if (nvdimm->label_size < offset + length) {
+        nvdimm_debug("position %#x is beyond label data (len = %" PRIx64 ").\n",
+                     offset + length, nvdimm->label_size);
+        return ret;
+    }
+
+    if (length > nvdimm_get_max_xfer_label_size()) {
+        nvdimm_debug("length (%#x) is larger than max_xfer (%#x).\n",
+                     length, nvdimm_get_max_xfer_label_size());
+        return ret;
+    }
+
+    return 0 /* Success */;
+}
+
+/*
+ * DSM Spec Rev1 4.5 Get Namespace Label Data (Function Index 5).
+ */
+static void nvdimm_dsm_get_label_data(NVDIMMDevice *nvdimm, NvdimmDsmIn *in,
+                                      hwaddr dsm_mem_addr)
+{
+    NVDIMMClass *nvc = NVDIMM_GET_CLASS(nvdimm);
+    NvdimmFuncGetLabelDataIn *get_label_data;
+    NvdimmFuncGetLabelDataOut *get_label_data_out;
+    uint32_t status;
+    int size;
+
+    get_label_data = (NvdimmFuncGetLabelDataIn *)in->arg3;
+    le32_to_cpus(&get_label_data->offset);
+    le32_to_cpus(&get_label_data->length);
+
+    nvdimm_debug("Read Label Data: offset %#x length %#x.\n",
+                 get_label_data->offset, get_label_data->length);
+
+    status = nvdimm_rw_label_data_check(nvdimm, get_label_data->offset,
+                                        get_label_data->length);
+    if (status != 0 /* Success */) {
+        nvdimm_dsm_no_payload(status, dsm_mem_addr);
+        return;
+    }
+
+    size = sizeof(*get_label_data_out) + get_label_data->length;
+    assert(size <= 4096);
+    get_label_data_out = g_malloc(size);
+
+    get_label_data_out->len = cpu_to_le32(size);
+    get_label_data_out->func_ret_status = cpu_to_le32(0 /* Success */);
+    nvc->read_label_data(nvdimm, get_label_data_out->out_buf,
+                         get_label_data->length, get_label_data->offset);
+
+    cpu_physical_memory_write(dsm_mem_addr, get_label_data_out, size);
+    g_free(get_label_data_out);
+}
+
+/*
+ * DSM Spec Rev1 4.6 Set Namespace Label Data (Function Index 6).
+ */
+static void nvdimm_dsm_set_label_data(NVDIMMDevice *nvdimm, NvdimmDsmIn *in,
+                                      hwaddr dsm_mem_addr)
+{
+    NVDIMMClass *nvc = NVDIMM_GET_CLASS(nvdimm);
+    NvdimmFuncSetLabelDataIn *set_label_data;
+    uint32_t status;
+
+    set_label_data = (NvdimmFuncSetLabelDataIn *)in->arg3;
+
+    le32_to_cpus(&set_label_data->offset);
+    le32_to_cpus(&set_label_data->length);
+
+    nvdimm_debug("Write Label Data: offset %#x length %#x.\n",
+                 set_label_data->offset, set_label_data->length);
+
+    status = nvdimm_rw_label_data_check(nvdimm, set_label_data->offset,
+                                        set_label_data->length);
+    if (status != 0 /* Success */) {
+        nvdimm_dsm_no_payload(status, dsm_mem_addr);
+        return;
+    }
+
+    assert(sizeof(*in) + sizeof(*set_label_data) + set_label_data->length <=
+           4096);
+
+    nvc->write_label_data(nvdimm, set_label_data->in_buf,
+                          set_label_data->length, set_label_data->offset);
+    nvdimm_dsm_no_payload(0 /* Success */, dsm_mem_addr);
+}
+
+static void nvdimm_dsm_device(NvdimmDsmIn *in, hwaddr dsm_mem_addr)
+{
+    NVDIMMDevice *nvdimm = nvdimm_get_device_by_handle(in->handle);
+
+    /* See the comments in nvdimm_dsm_root(). */
+    if (!in->function) {
+        uint32_t supported_func = 0;
+
+        if (nvdimm && nvdimm->label_size) {
+            supported_func |= 0x1 /* Bit 0 indicates whether there is
+                                     support for any functions other
+                                     than function 0. */ |
+                              1 << 4 /* Get Namespace Label Size */ |
+                              1 << 5 /* Get Namespace Label Data */ |
+                              1 << 6 /* Set Namespace Label Data */;
+        }
+        nvdimm_dsm_function0(supported_func, dsm_mem_addr);
+        return;
+    }
+
+    if (!nvdimm) {
+        nvdimm_dsm_no_payload(2 /* Non-Existing Memory Device */,
+                              dsm_mem_addr);
+        return;
+    }
+
+    /* Encode DSM function according to DSM Spec Rev1. */
+    switch (in->function) {
+    case 4 /* Get Namespace Label Size */:
+        if (nvdimm->label_size) {
+            nvdimm_dsm_label_size(nvdimm, dsm_mem_addr);
+            return;
+        }
+        break;
+    case 5 /* Get Namespace Label Data */:
+        if (nvdimm->label_size) {
+            nvdimm_dsm_get_label_data(nvdimm, in, dsm_mem_addr);
+            return;
+        }
+        break;
+    case 0x6 /* Set Namespace Label Data */:
+        if (nvdimm->label_size) {
+            nvdimm_dsm_set_label_data(nvdimm, in, dsm_mem_addr);
+            return;
+        }
+        break;
+    }
+
+    nvdimm_dsm_no_payload(1 /* Not Supported */, dsm_mem_addr);
+}
+
+static uint64_t
+nvdimm_dsm_read(void *opaque, hwaddr addr, unsigned size)
+{
+    nvdimm_debug("BUG: we never read _DSM IO Port.\n");
+    return 0;
+}
+
+static void
+nvdimm_dsm_write(void *opaque, hwaddr addr, uint64_t val, unsigned size)
+{
+    NvdimmDsmIn *in;
+    hwaddr dsm_mem_addr = val;
+
+    nvdimm_debug("dsm memory address %#" HWADDR_PRIx ".\n", dsm_mem_addr);
+
+    /*
+     * The DSM memory is mapped to guest address space so an evil guest
+     * can change its content while we are doing DSM emulation. Avoid
+     * this by copying DSM memory to QEMU local memory.
+     */
+    in = g_new(NvdimmDsmIn, 1);
+    cpu_physical_memory_read(dsm_mem_addr, in, sizeof(*in));
+
+    le32_to_cpus(&in->revision);
+    le32_to_cpus(&in->function);
+    le32_to_cpus(&in->handle);
+
+    nvdimm_debug("Revision %#x Handler %#x Function %#x.\n", in->revision,
+                 in->handle, in->function);
+
+    if (in->revision != 0x1 /* Currently we only support DSM Spec Rev1. */) {
+        nvdimm_debug("Revision %#x is not supported, expect %#x.\n",
+                     in->revision, 0x1);
+        nvdimm_dsm_no_payload(1 /* Not Supported */, dsm_mem_addr);
+        goto exit;
+    }
+
+     /* Handle 0 is reserved for NVDIMM Root Device. */
+    if (!in->handle) {
+        nvdimm_dsm_root(in, dsm_mem_addr);
+        goto exit;
+    }
+
+    nvdimm_dsm_device(in, dsm_mem_addr);
+
+exit:
+    g_free(in);
+}
+
+static const MemoryRegionOps nvdimm_dsm_ops = {
+    .read = nvdimm_dsm_read,
+    .write = nvdimm_dsm_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+void nvdimm_init_acpi_state(AcpiNVDIMMState *state, MemoryRegion *io,
+                            FWCfgState *fw_cfg, Object *owner)
+{
+    memory_region_init_io(&state->io_mr, owner, &nvdimm_dsm_ops, state,
+                          "nvdimm-acpi-io", NVDIMM_ACPI_IO_LEN);
+    memory_region_add_subregion(io, NVDIMM_ACPI_IO_BASE, &state->io_mr);
+
+    state->dsm_mem = g_array_new(false, true /* clear */, 1);
+    acpi_data_push(state->dsm_mem, sizeof(NvdimmDsmIn));
+    fw_cfg_add_file(fw_cfg, NVDIMM_DSM_MEM_FILE, state->dsm_mem->data,
+                    state->dsm_mem->len);
+}
+
+#define NVDIMM_COMMON_DSM      "NCAL"
+#define NVDIMM_ACPI_MEM_ADDR   "MEMA"
+
+static void nvdimm_build_common_dsm(Aml *dev)
+{
+    Aml *method, *ifctx, *function, *handle, *uuid, *dsm_mem, *result_size;
+    Aml *elsectx, *unsupport, *unpatched, *expected_uuid, *uuid_invalid;
+    Aml *pckg, *pckg_index, *pckg_buf;
+    uint8_t byte_list[1];
+
+    method = aml_method(NVDIMM_COMMON_DSM, 5, AML_SERIALIZED);
+    uuid = aml_arg(0);
+    function = aml_arg(2);
+    handle = aml_arg(4);
+    dsm_mem = aml_name(NVDIMM_ACPI_MEM_ADDR);
+
+    /*
+     * do not support any method if DSM memory address has not been
+     * patched.
+     */
+    unpatched = aml_equal(dsm_mem, aml_int(0x0));
+
+    expected_uuid = aml_local(0);
+
+    ifctx = aml_if(aml_equal(handle, aml_int(0x0)));
+    aml_append(ifctx, aml_store(
+               aml_touuid("2F10E7A4-9E91-11E4-89D3-123B93F75CBA")
+               /* UUID for NVDIMM Root Device */, expected_uuid));
+    aml_append(method, ifctx);
+    elsectx = aml_else();
+    aml_append(elsectx, aml_store(
+               aml_touuid("4309AC30-0D11-11E4-9191-0800200C9A66")
+               /* UUID for NVDIMM Devices */, expected_uuid));
+    aml_append(method, elsectx);
+
+    uuid_invalid = aml_lnot(aml_equal(uuid, expected_uuid));
+
+    unsupport = aml_if(aml_or(unpatched, uuid_invalid, NULL));
+
+    /*
+     * function 0 is called to inquire what functions are supported by
+     * OSPM
+     */
+    ifctx = aml_if(aml_equal(function, aml_int(0)));
+    byte_list[0] = 0 /* No function Supported */;
+    aml_append(ifctx, aml_return(aml_buffer(1, byte_list)));
+    aml_append(unsupport, ifctx);
+
+    /* No function is supported yet. */
+    byte_list[0] = 1 /* Not Supported */;
+    aml_append(unsupport, aml_return(aml_buffer(1, byte_list)));
+    aml_append(method, unsupport);
+
+    /*
+     * The HDLE indicates the DSM function is issued from which device,
+     * it reserves 0 for root device and is the handle for NVDIMM devices.
+     * See the comments in nvdimm_slot_to_handle().
+     */
+    aml_append(method, aml_store(handle, aml_name("HDLE")));
+    aml_append(method, aml_store(aml_arg(1), aml_name("REVS")));
+    aml_append(method, aml_store(aml_arg(2), aml_name("FUNC")));
+
+    /*
+     * The fourth parameter (Arg3) of _DSM is a package which contains
+     * a buffer, the layout of the buffer is specified by UUID (Arg0),
+     * Revision ID (Arg1) and Function Index (Arg2) which are documented
+     * in the DSM Spec.
+     */
+    pckg = aml_arg(3);
+    ifctx = aml_if(aml_and(aml_equal(aml_object_type(pckg),
+                   aml_int(4 /* Package */)) /* It is a Package? */,
+                   aml_equal(aml_sizeof(pckg), aml_int(1)) /* 1 element? */,
+                   NULL));
+
+    pckg_index = aml_local(2);
+    pckg_buf = aml_local(3);
+    aml_append(ifctx, aml_store(aml_index(pckg, aml_int(0)), pckg_index));
+    aml_append(ifctx, aml_store(aml_derefof(pckg_index), pckg_buf));
+    aml_append(ifctx, aml_store(pckg_buf, aml_name("ARG3")));
+    aml_append(method, ifctx);
+
+    /*
+     * tell QEMU about the real address of DSM memory, then QEMU
+     * gets the control and fills the result in DSM memory.
+     */
+    aml_append(method, aml_store(dsm_mem, aml_name("NTFI")));
+
+    result_size = aml_local(1);
+    aml_append(method, aml_store(aml_name("RLEN"), result_size));
+    aml_append(method, aml_store(aml_shiftleft(result_size, aml_int(3)),
+                                 result_size));
+    aml_append(method, aml_create_field(aml_name("ODAT"), aml_int(0),
+                                        result_size, "OBUF"));
+    aml_append(method, aml_concatenate(aml_buffer(0, NULL), aml_name("OBUF"),
+                                       aml_arg(6)));
+    aml_append(method, aml_return(aml_arg(6)));
+    aml_append(dev, method);
+}
+
+static void nvdimm_build_device_dsm(Aml *dev, uint32_t handle)
+{
+    Aml *method;
+
+    method = aml_method("_DSM", 4, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_call5(NVDIMM_COMMON_DSM, aml_arg(0),
+                                  aml_arg(1), aml_arg(2), aml_arg(3),
+                                  aml_int(handle))));
+    aml_append(dev, method);
+}
+
+static void nvdimm_build_nvdimm_devices(GSList *device_list, Aml *root_dev)
+{
+    for (; device_list; device_list = device_list->next) {
+        DeviceState *dev = device_list->data;
+        int slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP,
+                                           NULL);
+        uint32_t handle = nvdimm_slot_to_handle(slot);
+        Aml *nvdimm_dev;
+
+        nvdimm_dev = aml_device("NV%02X", slot);
+
+        /*
+         * ACPI 6.0: 9.20 NVDIMM Devices:
+         *
+         * _ADR object that is used to supply OSPM with unique address
+         * of the NVDIMM device. This is done by returning the NFIT Device
+         * handle that is used to identify the associated entries in ACPI
+         * table NFIT or _FIT.
+         */
+        aml_append(nvdimm_dev, aml_name_decl("_ADR", aml_int(handle)));
+
+        nvdimm_build_device_dsm(nvdimm_dev, handle);
+        aml_append(root_dev, nvdimm_dev);
+    }
+}
+
+static void nvdimm_build_ssdt(GSList *device_list, GArray *table_offsets,
+                              GArray *table_data, BIOSLinker *linker,
+                              GArray *dsm_dma_arrea)
+{
+    Aml *ssdt, *sb_scope, *dev, *field;
+    int mem_addr_offset, nvdimm_ssdt;
+
+    acpi_add_table(table_offsets, table_data);
+
+    ssdt = init_aml_allocator();
+    acpi_data_push(ssdt->buf, sizeof(AcpiTableHeader));
+
+    sb_scope = aml_scope("\\_SB");
+
+    dev = aml_device("NVDR");
+
+    /*
+     * ACPI 6.0: 9.20 NVDIMM Devices:
+     *
+     * The ACPI Name Space device uses _HID of ACPI0012 to identify the root
+     * NVDIMM interface device. Platform firmware is required to contain one
+     * such device in _SB scope if NVDIMMs support is exposed by platform to
+     * OSPM.
+     * For each NVDIMM present or intended to be supported by platform,
+     * platform firmware also exposes an ACPI Namespace Device under the
+     * root device.
+     */
+    aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0012")));
+
+    /* map DSM memory and IO into ACPI namespace. */
+    aml_append(dev, aml_operation_region("NPIO", AML_SYSTEM_IO,
+               aml_int(NVDIMM_ACPI_IO_BASE), NVDIMM_ACPI_IO_LEN));
+    aml_append(dev, aml_operation_region("NRAM", AML_SYSTEM_MEMORY,
+               aml_name(NVDIMM_ACPI_MEM_ADDR), sizeof(NvdimmDsmIn)));
+
+    /*
+     * DSM notifier:
+     * NTFI: write the address of DSM memory and notify QEMU to emulate
+     *       the access.
+     *
+     * It is the IO port so that accessing them will cause VM-exit, the
+     * control will be transferred to QEMU.
+     */
+    field = aml_field("NPIO", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("NTFI",
+               sizeof(uint32_t) * BITS_PER_BYTE));
+    aml_append(dev, field);
+
+    /*
+     * DSM input:
+     * HDLE: store device's handle, it's zero if the _DSM call happens
+     *       on NVDIMM Root Device.
+     * REVS: store the Arg1 of _DSM call.
+     * FUNC: store the Arg2 of _DSM call.
+     * ARG3: store the Arg3 of _DSM call.
+     *
+     * They are RAM mapping on host so that these accesses never cause
+     * VM-EXIT.
+     */
+    field = aml_field("NRAM", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("HDLE",
+               sizeof(typeof_field(NvdimmDsmIn, handle)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("REVS",
+               sizeof(typeof_field(NvdimmDsmIn, revision)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("FUNC",
+               sizeof(typeof_field(NvdimmDsmIn, function)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("ARG3",
+               (sizeof(NvdimmDsmIn) - offsetof(NvdimmDsmIn, arg3)) * BITS_PER_BYTE));
+    aml_append(dev, field);
+
+    /*
+     * DSM output:
+     * RLEN: the size of the buffer filled by QEMU.
+     * ODAT: the buffer QEMU uses to store the result.
+     *
+     * Since the page is reused by both input and out, the input data
+     * will be lost after storing new result into ODAT so we should fetch
+     * all the input data before writing the result.
+     */
+    field = aml_field("NRAM", AML_DWORD_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("RLEN",
+               sizeof(typeof_field(NvdimmDsmOut, len)) * BITS_PER_BYTE));
+    aml_append(field, aml_named_field("ODAT",
+               (sizeof(NvdimmDsmOut) - offsetof(NvdimmDsmOut, data)) * BITS_PER_BYTE));
+    aml_append(dev, field);
+
+    nvdimm_build_common_dsm(dev);
+
+    /* 0 is reserved for root device. */
+    nvdimm_build_device_dsm(dev, 0);
+
+    nvdimm_build_nvdimm_devices(device_list, dev);
+
+    aml_append(sb_scope, dev);
+    aml_append(ssdt, sb_scope);
+
+    nvdimm_ssdt = table_data->len;
+
+    /* copy AML table into ACPI tables blob and patch header there */
+    g_array_append_vals(table_data, ssdt->buf->data, ssdt->buf->len);
+    mem_addr_offset = build_append_named_dword(table_data,
+                                               NVDIMM_ACPI_MEM_ADDR);
+
+    bios_linker_loader_alloc(linker,
+                             NVDIMM_DSM_MEM_FILE, dsm_dma_arrea,
+                             sizeof(NvdimmDsmIn), false /* high memory */);
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_TABLE_FILE, mem_addr_offset, sizeof(uint32_t),
+        NVDIMM_DSM_MEM_FILE, 0);
+    build_header(linker, table_data,
+        (void *)(table_data->data + nvdimm_ssdt),
+        "SSDT", table_data->len - nvdimm_ssdt, 1, NULL, "NVDIMM");
+    free_aml_allocator();
+}
+
+void nvdimm_build_acpi(GArray *table_offsets, GArray *table_data,
+                       BIOSLinker *linker, GArray *dsm_dma_arrea)
+{
+    GSList *device_list;
+
+    /* no NVDIMM device is plugged. */
+    device_list = nvdimm_get_plugged_device_list();
+    if (!device_list) {
+        return;
+    }
+    nvdimm_build_nfit(device_list, table_offsets, table_data, linker);
+    nvdimm_build_ssdt(device_list, table_offsets, table_data, linker,
+                      dsm_dma_arrea);
+    g_slist_free(device_list);
+}
diff --git a/hw/acpi/pcihp.c b/hw/acpi/pcihp.c
index 34dedf1..d957d1e 100644
--- a/hw/acpi/pcihp.c
+++ b/hw/acpi/pcihp.c
@@ -24,6 +24,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/acpi/pcihp.h"
 
 #include "hw/hw.h"
@@ -31,10 +32,10 @@
 #include "hw/pci/pci.h"
 #include "hw/acpi/acpi.h"
 #include "sysemu/sysemu.h"
-#include "qemu/range.h"
 #include "exec/ioport.h"
 #include "exec/address-spaces.h"
 #include "hw/pci/pci_bus.h"
+#include "qapi/error.h"
 #include "qom/qom-qobject.h"
 #include "qapi/qmp/qint.h"
 
@@ -46,7 +47,6 @@
 # define ACPI_PCIHP_DPRINTF(format, ...)     do { } while (0)
 #endif
 
-#define ACPI_PCI_HOTPLUG_STATUS 2
 #define ACPI_PCIHP_ADDR 0xae00
 #define ACPI_PCIHP_SIZE 0x0014
 #define ACPI_PCIHP_LEGACY_SIZE 0x000f
@@ -120,7 +120,7 @@
 static void acpi_pcihp_eject_slot(AcpiPciHpState *s, unsigned bsel, unsigned slots)
 {
     BusChild *kid, *next;
-    int slot = ffs(slots) - 1;
+    int slot = ctz32(slots);
     PCIBus *bus = acpi_pcihp_find_hotplug_bus(s, bsel);
 
     if (!bus) {
@@ -182,7 +182,7 @@
     acpi_pcihp_update(s);
 }
 
-void acpi_pcihp_device_plug_cb(ACPIREGS *ar, qemu_irq irq, AcpiPciHpState *s,
+void acpi_pcihp_device_plug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s,
                                DeviceState *dev, Error **errp)
 {
     PCIDevice *pdev = PCI_DEVICE(dev);
@@ -202,12 +202,10 @@
     }
 
     s->acpi_pcihp_pci_status[bsel].up |= (1U << slot);
-
-    ar->gpe.sts[0] |= ACPI_PCI_HOTPLUG_STATUS;
-    acpi_update_sci(ar, irq);
+    acpi_send_event(DEVICE(hotplug_dev), ACPI_PCI_HOTPLUG_STATUS);
 }
 
-void acpi_pcihp_device_unplug_cb(ACPIREGS *ar, qemu_irq irq, AcpiPciHpState *s,
+void acpi_pcihp_device_unplug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s,
                                  DeviceState *dev, Error **errp)
 {
     PCIDevice *pdev = PCI_DEVICE(dev);
@@ -220,9 +218,7 @@
     }
 
     s->acpi_pcihp_pci_status[bsel].down |= (1U << slot);
-
-    ar->gpe.sts[0] |= ACPI_PCI_HOTPLUG_STATUS;
-    acpi_update_sci(ar, irq);
+    acpi_send_event(DEVICE(hotplug_dev), ACPI_PCI_HOTPLUG_STATUS);
 }
 
 static uint64_t pci_read(void *opaque, hwaddr addr, unsigned int size)
@@ -297,10 +293,11 @@
     },
 };
 
-void acpi_pcihp_init(AcpiPciHpState *s, PCIBus *root_bus,
+void acpi_pcihp_init(Object *owner, AcpiPciHpState *s, PCIBus *root_bus,
                      MemoryRegion *address_space_io, bool bridges_enabled)
 {
-    uint16_t io_size = ACPI_PCIHP_SIZE;
+    s->io_len = ACPI_PCIHP_SIZE;
+    s->io_base = ACPI_PCIHP_ADDR;
 
     s->root= root_bus;
     s->legacy_piix = !bridges_enabled;
@@ -308,16 +305,21 @@
     if (s->legacy_piix) {
         unsigned *bus_bsel = g_malloc(sizeof *bus_bsel);
 
-        io_size = ACPI_PCIHP_LEGACY_SIZE;
+        s->io_len = ACPI_PCIHP_LEGACY_SIZE;
 
         *bus_bsel = ACPI_PCIHP_BSEL_DEFAULT;
         object_property_add_uint32_ptr(OBJECT(root_bus), ACPI_PCIHP_PROP_BSEL,
                                        bus_bsel, NULL);
     }
 
-    memory_region_init_io(&s->io, NULL, &acpi_pcihp_io_ops, s,
-                          "acpi-pci-hotplug", io_size);
-    memory_region_add_subregion(address_space_io, ACPI_PCIHP_ADDR, &s->io);
+    memory_region_init_io(&s->io, owner, &acpi_pcihp_io_ops, s,
+                          "acpi-pci-hotplug", s->io_len);
+    memory_region_add_subregion(address_space_io, s->io_base, &s->io);
+
+    object_property_add_uint16_ptr(owner, ACPI_PCIHP_IO_BASE_PROP, &s->io_base,
+                                   &error_abort);
+    object_property_add_uint16_ptr(owner, ACPI_PCIHP_IO_LEN_PROP, &s->io_len,
+                                   &error_abort);
 }
 
 const VMStateDescription vmstate_acpi_pcihp_pci_status = {
diff --git a/hw/acpi/piix4.c b/hw/acpi/piix4.c
index dedfa92..2adc246 100644
--- a/hw/acpi/piix4.c
+++ b/hw/acpi/piix4.c
@@ -18,6 +18,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/apm.h"
@@ -25,7 +26,7 @@
 #include "hw/pci/pci.h"
 #include "hw/acpi/acpi.h"
 #include "sysemu/sysemu.h"
-#include "sysemu/hax.h"
+#include "qapi/error.h"
 #include "qemu/range.h"
 #include "exec/ioport.h"
 #include "hw/nvram/fw_cfg.h"
@@ -33,11 +34,13 @@
 #include "hw/acpi/piix4.h"
 #include "hw/acpi/pcihp.h"
 #include "hw/acpi/cpu_hotplug.h"
+#include "hw/acpi/cpu.h"
 #include "hw/hotplug.h"
 #include "hw/mem/pc-dimm.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "hw/acpi/acpi_dev_interface.h"
 #include "hw/xen/xen.h"
+#include "qom/cpu.h"
 
 //#define DEBUG
 
@@ -73,7 +76,7 @@
 
     qemu_irq irq;
     qemu_irq smi_irq;
-    int kvm_enabled;
+    int smm_enabled;
     Notifier machine_ready;
     Notifier powerdown_notifier;
 
@@ -84,7 +87,9 @@
     uint8_t disable_s4;
     uint8_t s4_val;
 
+    bool cpu_hotplug_legacy;
     AcpiCpuHotplug gpe_cpu;
+    CPUHotplugState cpuhp_state;
 
     MemHotplugState acpi_memory_hotplug;
 } PIIX4PMState;
@@ -113,6 +118,9 @@
 
     /* ACPI specs 3.0, 4.7.2.5 */
     acpi_pm1_cnt_update(&s->ar, val == ACPI_ENABLE, val == ACPI_DISABLE);
+    if (val == ACPI_ENABLE || val == ACPI_DISABLE) {
+        return;
+    }
 
     if (d->config[0x5b] & (1 << 1)) {
         if (s->smi_irq) {
@@ -261,12 +269,39 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
+    .needed = vmstate_test_use_memhp,
     .fields      = (VMStateField[]) {
         VMSTATE_MEMORY_HOTPLUG(acpi_memory_hotplug, PIIX4PMState),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static bool vmstate_test_use_cpuhp(void *opaque)
+{
+    PIIX4PMState *s = opaque;
+    return !s->cpu_hotplug_legacy;
+}
+
+static int vmstate_cpuhp_pre_load(void *opaque)
+{
+    Object *obj = OBJECT(opaque);
+    object_property_set_bool(obj, false, "cpu-hotplug-legacy", &error_abort);
+    return 0;
+}
+
+static const VMStateDescription vmstate_cpuhp_state = {
+    .name = "piix4_pm/cpuhp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .needed = vmstate_test_use_cpuhp,
+    .pre_load = vmstate_cpuhp_pre_load,
+    .fields      = (VMStateField[]) {
+        VMSTATE_CPU_HOTPLUG(cpuhp_state, PIIX4PMState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 /* qemu-kvm 1.2 uses version 3 but advertised as 2
  * To support incoming qemu-kvm 1.2 migration, change version_id
  * and minimum_version_id to 2 below (which breaks migration from
@@ -286,7 +321,7 @@
         VMSTATE_UINT16(ar.pm1.evt.en, PIIX4PMState),
         VMSTATE_UINT16(ar.pm1.cnt.cnt, PIIX4PMState),
         VMSTATE_STRUCT(apm, PIIX4PMState, 0, vmstate_apm, APMState),
-        VMSTATE_TIMER(ar.tmr.timer, PIIX4PMState),
+        VMSTATE_TIMER_PTR(ar.tmr.timer, PIIX4PMState),
         VMSTATE_INT64(ar.tmr.overflow_time, PIIX4PMState),
         VMSTATE_STRUCT(ar.gpe, PIIX4PMState, 2, vmstate_gpe, ACPIGPE),
         VMSTATE_STRUCT_TEST(
@@ -299,12 +334,10 @@
                             vmstate_test_use_acpi_pci_hotplug),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_memhp_state,
-            .needed = vmstate_test_use_memhp,
-        },
-        VMSTATE_END_OF_LIST()
+    .subsections = (const VMStateDescription*[]) {
+         &vmstate_memhp_state,
+         &vmstate_cpuhp_state,
+         NULL
     }
 };
 
@@ -322,7 +355,7 @@
     pci_conf[0x40] = 0x01; /* PM io base read only bit */
     pci_conf[0x80] = 0;
 
-    if (s->kvm_enabled || hax_enabled()) {
+    if (!s->smm_enabled) {
         /* Mark SMM as already inited (until KVM supports SMM). */
         pci_conf[0x5B] = 0x02;
     }
@@ -345,12 +378,15 @@
 
     if (s->acpi_memory_hotplug.is_enabled &&
         object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
-        acpi_memory_plug_cb(&s->ar, s->irq, &s->acpi_memory_hotplug, dev, errp);
+        acpi_memory_plug_cb(hotplug_dev, &s->acpi_memory_hotplug, dev, errp);
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
-        acpi_pcihp_device_plug_cb(&s->ar, s->irq, &s->acpi_pci_hotplug, dev,
-                                  errp);
+        acpi_pcihp_device_plug_cb(hotplug_dev, &s->acpi_pci_hotplug, dev, errp);
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
-        acpi_cpu_plug_cb(&s->ar, s->irq, &s->gpe_cpu, dev, errp);
+        if (s->cpu_hotplug_legacy) {
+            legacy_acpi_cpu_plug_cb(hotplug_dev, &s->gpe_cpu, dev, errp);
+        } else {
+            acpi_cpu_plug_cb(hotplug_dev, &s->cpuhp_state, dev, errp);
+        }
     } else {
         error_setg(errp, "acpi: device plug request for not supported device"
                    " type: %s", object_get_typename(OBJECT(dev)));
@@ -362,15 +398,39 @@
 {
     PIIX4PMState *s = PIIX4_PM(hotplug_dev);
 
-    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
-        acpi_pcihp_device_unplug_cb(&s->ar, s->irq, &s->acpi_pci_hotplug, dev,
+    if (s->acpi_memory_hotplug.is_enabled &&
+        object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        acpi_memory_unplug_request_cb(hotplug_dev, &s->acpi_memory_hotplug,
+                                      dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+        acpi_pcihp_device_unplug_cb(hotplug_dev, &s->acpi_pci_hotplug, dev,
                                     errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) &&
+               !s->cpu_hotplug_legacy) {
+        acpi_cpu_unplug_request_cb(hotplug_dev, &s->cpuhp_state, dev, errp);
     } else {
         error_setg(errp, "acpi: device unplug request for not supported device"
                    " type: %s", object_get_typename(OBJECT(dev)));
     }
 }
 
+static void piix4_device_unplug_cb(HotplugHandler *hotplug_dev,
+                                   DeviceState *dev, Error **errp)
+{
+    PIIX4PMState *s = PIIX4_PM(hotplug_dev);
+
+    if (s->acpi_memory_hotplug.is_enabled &&
+        object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        acpi_memory_unplug_cb(&s->acpi_memory_hotplug, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU) &&
+               !s->cpu_hotplug_legacy) {
+        acpi_cpu_unplug_cb(&s->cpuhp_state, dev, errp);
+    } else {
+        error_setg(errp, "acpi: device unplug for not supported device"
+                   " type: %s", object_get_typename(OBJECT(dev)));
+    }
+}
+
 static void piix4_update_bus_hotplug(PCIBus *pci_bus, void *opaque)
 {
     PIIX4PMState *s = opaque;
@@ -421,7 +481,7 @@
                                   &s->io_base, NULL);
 }
 
-static int piix4_pm_initfn(PCIDevice *dev)
+static void piix4_pm_realize(PCIDevice *dev, Error **errp)
 {
     PIIX4PMState *s = PIIX4_PM(dev);
     uint8_t *pci_conf;
@@ -435,7 +495,7 @@
     /* APM */
     apm_init(dev, &s->apm, apm_ctrl_changed, s);
 
-    if (s->kvm_enabled || hax_enabled()) {
+    if (!s->smm_enabled) {
         /* Mark SMM as already inited to prevent SMM from running.  KVM does not
          * support SMM mode. */
         pci_conf[0x5B] = 0x02;
@@ -458,7 +518,7 @@
 
     acpi_pm_tmr_init(&s->ar, pm_tmr_timer, &s->io);
     acpi_pm1_evt_init(&s->ar, pm_tmr_timer, &s->io);
-    acpi_pm1_cnt_init(&s->ar, &s->io, s->s4_val);
+    acpi_pm1_cnt_init(&s->ar, &s->io, s->disable_s3, s->disable_s4, s->s4_val);
     acpi_gpe_init(&s->ar, GPE_LEN);
 
     s->powerdown_notifier.notify = piix4_pm_powerdown_req;
@@ -471,7 +531,6 @@
     piix4_acpi_system_hot_add_init(pci_address_space_io(dev), dev->bus, s);
 
     piix4_pm_add_propeties(s);
-    return 0;
 }
 
 Object *piix4_pm_find(void)
@@ -487,8 +546,7 @@
 
 I2CBus *piix4_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
                       qemu_irq sci_irq, qemu_irq smi_irq,
-                      int kvm_enabled, FWCfgState *fw_cfg,
-                      DeviceState **piix4_pm)
+                      int smm_enabled, DeviceState **piix4_pm)
 {
     DeviceState *dev;
     PIIX4PMState *s;
@@ -502,21 +560,13 @@
     s = PIIX4_PM(dev);
     s->irq = sci_irq;
     s->smi_irq = smi_irq;
-    s->kvm_enabled = kvm_enabled;
+    s->smm_enabled = smm_enabled;
     if (xen_enabled()) {
         s->use_acpi_pci_hotplug = false;
     }
 
     qdev_init_nofail(dev);
 
-    if (fw_cfg) {
-        uint8_t suspend[6] = {128, 0, 0, 129, 128, 128};
-        suspend[3] = 1 | ((!s->disable_s3) << 7);
-        suspend[4] = s->s4_val | ((!s->disable_s4) << 7);
-
-        fw_cfg_add_file(fw_cfg, "etc/system-states", g_memdup(suspend, 6), 6);
-    }
-
     return s->smb.smbus;
 }
 
@@ -550,6 +600,26 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+
+static bool piix4_get_cpu_hotplug_legacy(Object *obj, Error **errp)
+{
+    PIIX4PMState *s = PIIX4_PM(obj);
+
+    return s->cpu_hotplug_legacy;
+}
+
+static void piix4_set_cpu_hotplug_legacy(Object *obj, bool value, Error **errp)
+{
+    PIIX4PMState *s = PIIX4_PM(obj);
+
+    assert(!value);
+    if (s->cpu_hotplug_legacy && value == false) {
+        acpi_switch_to_modern_cphp(&s->gpe_cpu, &s->cpuhp_state,
+                                   PIIX4_CPU_HOTPLUG_IO_BASE);
+    }
+    s->cpu_hotplug_legacy = value;
+}
+
 static void piix4_acpi_system_hot_add_init(MemoryRegion *parent,
                                            PCIBus *bus, PIIX4PMState *s)
 {
@@ -557,11 +627,16 @@
                           "acpi-gpe0", GPE_LEN);
     memory_region_add_subregion(parent, GPE_BASE, &s->io_gpe);
 
-    acpi_pcihp_init(&s->acpi_pci_hotplug, bus, parent,
+    acpi_pcihp_init(OBJECT(s), &s->acpi_pci_hotplug, bus, parent,
                     s->use_acpi_pci_hotplug);
 
-    acpi_cpu_hotplug_init(parent, OBJECT(s), &s->gpe_cpu,
-                          PIIX4_CPU_HOTPLUG_IO_BASE);
+    s->cpu_hotplug_legacy = true;
+    object_property_add_bool(OBJECT(s), "cpu-hotplug-legacy",
+                             piix4_get_cpu_hotplug_legacy,
+                             piix4_set_cpu_hotplug_legacy,
+                             NULL);
+    legacy_acpi_cpu_hotplug_init(parent, OBJECT(s), &s->gpe_cpu,
+                                 PIIX4_CPU_HOTPLUG_IO_BASE);
 
     if (s->acpi_memory_hotplug.is_enabled) {
         acpi_memory_hotplug_init(parent, OBJECT(s), &s->acpi_memory_hotplug);
@@ -573,6 +648,16 @@
     PIIX4PMState *s = PIIX4_PM(adev);
 
     acpi_memory_ospm_status(&s->acpi_memory_hotplug, list);
+    if (!s->cpu_hotplug_legacy) {
+        acpi_cpu_ospm_status(&s->cpuhp_state, list);
+    }
+}
+
+static void piix4_send_gpe(AcpiDeviceIf *adev, AcpiEventStatusBits ev)
+{
+    PIIX4PMState *s = PIIX4_PM(adev);
+
+    acpi_send_gpe_event(&s->ar, s->irq, ev);
 }
 
 static Property piix4_pm_properties[] = {
@@ -594,7 +679,7 @@
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
     AcpiDeviceIfClass *adevc = ACPI_DEVICE_IF_CLASS(klass);
 
-    k->init = piix4_pm_initfn;
+    k->realize = piix4_pm_realize;
     k->config_write = pm_write_config;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82371AB_3;
@@ -611,7 +696,10 @@
     dc->hotpluggable = false;
     hc->plug = piix4_device_plug_cb;
     hc->unplug_request = piix4_device_unplug_request_cb;
+    hc->unplug = piix4_device_unplug_cb;
     adevc->ospm_status = piix4_ospm_status;
+    adevc->send_event = piix4_send_gpe;
+    adevc->madt_cpu = pc_madt_cpu_entry;
 }
 
 static const TypeInfo piix4_pm_info = {
diff --git a/hw/acpi/tco.c b/hw/acpi/tco.c
new file mode 100644
index 0000000..8ce7daf
--- /dev/null
+++ b/hw/acpi/tco.c
@@ -0,0 +1,265 @@
+/*
+ * QEMU ICH9 TCO emulation
+ *
+ * Copyright (c) 2015 Paulo Alcantara <pcacjr@zytor.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/watchdog.h"
+#include "hw/i386/ich9.h"
+
+#include "hw/acpi/tco.h"
+
+//#define DEBUG
+
+#ifdef DEBUG
+#define TCO_DEBUG(fmt, ...)                                     \
+    do {                                                        \
+        fprintf(stderr, "%s "fmt, __func__, ## __VA_ARGS__);    \
+    } while (0)
+#else
+#define TCO_DEBUG(fmt, ...) do { } while (0)
+#endif
+
+enum {
+    TCO_RLD_DEFAULT         = 0x0000,
+    TCO_DAT_IN_DEFAULT      = 0x00,
+    TCO_DAT_OUT_DEFAULT     = 0x00,
+    TCO1_STS_DEFAULT        = 0x0000,
+    TCO2_STS_DEFAULT        = 0x0000,
+    TCO1_CNT_DEFAULT        = 0x0000,
+    TCO2_CNT_DEFAULT        = 0x0008,
+    TCO_MESSAGE1_DEFAULT    = 0x00,
+    TCO_MESSAGE2_DEFAULT    = 0x00,
+    TCO_WDCNT_DEFAULT       = 0x00,
+    TCO_TMR_DEFAULT         = 0x0004,
+    SW_IRQ_GEN_DEFAULT      = 0x03,
+};
+
+static inline void tco_timer_reload(TCOIORegs *tr)
+{
+    tr->expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+        ((int64_t)(tr->tco.tmr & TCO_TMR_MASK) * TCO_TICK_NSEC);
+    timer_mod(tr->tco_timer, tr->expire_time);
+}
+
+static inline void tco_timer_stop(TCOIORegs *tr)
+{
+    tr->expire_time = -1;
+}
+
+static void tco_timer_expired(void *opaque)
+{
+    TCOIORegs *tr = opaque;
+    ICH9LPCPMRegs *pm = container_of(tr, ICH9LPCPMRegs, tco_regs);
+    ICH9LPCState *lpc = container_of(pm, ICH9LPCState, pm);
+    uint32_t gcs = pci_get_long(lpc->chip_config + ICH9_CC_GCS);
+
+    tr->tco.rld = 0;
+    tr->tco.sts1 |= TCO_TIMEOUT;
+    if (++tr->timeouts_no == 2) {
+        tr->tco.sts2 |= TCO_SECOND_TO_STS;
+        tr->tco.sts2 |= TCO_BOOT_STS;
+        tr->timeouts_no = 0;
+
+        if (!lpc->pin_strap.spkr_hi && !(gcs & ICH9_CC_GCS_NO_REBOOT)) {
+            watchdog_perform_action();
+            tco_timer_stop(tr);
+            return;
+        }
+    }
+
+    if (pm->smi_en & ICH9_PMIO_SMI_EN_TCO_EN) {
+        ich9_generate_smi();
+    } else {
+        ich9_generate_nmi();
+    }
+    tr->tco.rld = tr->tco.tmr;
+    tco_timer_reload(tr);
+}
+
+/* NOTE: values of 0 or 1 will be ignored by ICH */
+static inline int can_start_tco_timer(TCOIORegs *tr)
+{
+    return !(tr->tco.cnt1 & TCO_TMR_HLT) && tr->tco.tmr > 1;
+}
+
+static uint32_t tco_ioport_readw(TCOIORegs *tr, uint32_t addr)
+{
+    uint16_t rld;
+
+    switch (addr) {
+    case TCO_RLD:
+        if (tr->expire_time != -1) {
+            int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+            int64_t elapsed = (tr->expire_time - now) / TCO_TICK_NSEC;
+            rld = (uint16_t)elapsed | (tr->tco.rld & ~TCO_RLD_MASK);
+        } else {
+            rld = tr->tco.rld;
+        }
+        return rld;
+    case TCO_DAT_IN:
+        return tr->tco.din;
+    case TCO_DAT_OUT:
+        return tr->tco.dout;
+    case TCO1_STS:
+        return tr->tco.sts1;
+    case TCO2_STS:
+        return tr->tco.sts2;
+    case TCO1_CNT:
+        return tr->tco.cnt1;
+    case TCO2_CNT:
+        return tr->tco.cnt2;
+    case TCO_MESSAGE1:
+        return tr->tco.msg1;
+    case TCO_MESSAGE2:
+        return tr->tco.msg2;
+    case TCO_WDCNT:
+        return tr->tco.wdcnt;
+    case TCO_TMR:
+        return tr->tco.tmr;
+    case SW_IRQ_GEN:
+        return tr->sw_irq_gen;
+    }
+    return 0;
+}
+
+static void tco_ioport_writew(TCOIORegs *tr, uint32_t addr, uint32_t val)
+{
+    switch (addr) {
+    case TCO_RLD:
+        tr->timeouts_no = 0;
+        if (can_start_tco_timer(tr)) {
+            tr->tco.rld = tr->tco.tmr;
+            tco_timer_reload(tr);
+        } else {
+            tr->tco.rld = val;
+        }
+        break;
+    case TCO_DAT_IN:
+        tr->tco.din = val;
+        tr->tco.sts1 |= SW_TCO_SMI;
+        ich9_generate_smi();
+        break;
+    case TCO_DAT_OUT:
+        tr->tco.dout = val;
+        tr->tco.sts1 |= TCO_INT_STS;
+        /* TODO: cause an interrupt, as selected by the TCO_INT_SEL bits */
+        break;
+    case TCO1_STS:
+        tr->tco.sts1 = val & TCO1_STS_MASK;
+        break;
+    case TCO2_STS:
+        tr->tco.sts2 = val & TCO2_STS_MASK;
+        break;
+    case TCO1_CNT:
+        val &= TCO1_CNT_MASK;
+        /*
+         * once TCO_LOCK bit is set, it can not be cleared by software. a reset
+         * is required to change this bit from 1 to 0 -- it defaults to 0.
+         */
+        tr->tco.cnt1 = val | (tr->tco.cnt1 & TCO_LOCK);
+        if (can_start_tco_timer(tr)) {
+            tr->tco.rld = tr->tco.tmr;
+            tco_timer_reload(tr);
+        } else {
+            tco_timer_stop(tr);
+        }
+        break;
+    case TCO2_CNT:
+        tr->tco.cnt2 = val;
+        break;
+    case TCO_MESSAGE1:
+        tr->tco.msg1 = val;
+        break;
+    case TCO_MESSAGE2:
+        tr->tco.msg2 = val;
+        break;
+    case TCO_WDCNT:
+        tr->tco.wdcnt = val;
+        break;
+    case TCO_TMR:
+        tr->tco.tmr = val;
+        break;
+    case SW_IRQ_GEN:
+        tr->sw_irq_gen = val;
+        break;
+    }
+}
+
+static uint64_t tco_io_readw(void *opaque, hwaddr addr, unsigned width)
+{
+    TCOIORegs *tr = opaque;
+    return tco_ioport_readw(tr, addr);
+}
+
+static void tco_io_writew(void *opaque, hwaddr addr, uint64_t val,
+                          unsigned width)
+{
+    TCOIORegs *tr = opaque;
+    tco_ioport_writew(tr, addr, val);
+}
+
+static const MemoryRegionOps tco_io_ops = {
+    .read = tco_io_readw,
+    .write = tco_io_writew,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void acpi_pm_tco_init(TCOIORegs *tr, MemoryRegion *parent)
+{
+    *tr = (TCOIORegs) {
+        .tco = {
+            .rld      = TCO_RLD_DEFAULT,
+            .din      = TCO_DAT_IN_DEFAULT,
+            .dout     = TCO_DAT_OUT_DEFAULT,
+            .sts1     = TCO1_STS_DEFAULT,
+            .sts2     = TCO2_STS_DEFAULT,
+            .cnt1     = TCO1_CNT_DEFAULT,
+            .cnt2     = TCO2_CNT_DEFAULT,
+            .msg1     = TCO_MESSAGE1_DEFAULT,
+            .msg2     = TCO_MESSAGE2_DEFAULT,
+            .wdcnt    = TCO_WDCNT_DEFAULT,
+            .tmr      = TCO_TMR_DEFAULT,
+        },
+        .sw_irq_gen    = SW_IRQ_GEN_DEFAULT,
+        .tco_timer     = timer_new_ns(QEMU_CLOCK_VIRTUAL, tco_timer_expired, tr),
+        .expire_time   = -1,
+        .timeouts_no   = 0,
+    };
+    memory_region_init_io(&tr->io, memory_region_owner(parent),
+                          &tco_io_ops, tr, "sm-tco", ICH9_PMIO_TCO_LEN);
+    memory_region_add_subregion(parent, ICH9_PMIO_TCO_RLD, &tr->io);
+}
+
+const VMStateDescription vmstate_tco_io_sts = {
+    .name = "tco io device status",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT16(tco.rld, TCOIORegs),
+        VMSTATE_UINT8(tco.din, TCOIORegs),
+        VMSTATE_UINT8(tco.dout, TCOIORegs),
+        VMSTATE_UINT16(tco.sts1, TCOIORegs),
+        VMSTATE_UINT16(tco.sts2, TCOIORegs),
+        VMSTATE_UINT16(tco.cnt1, TCOIORegs),
+        VMSTATE_UINT16(tco.cnt2, TCOIORegs),
+        VMSTATE_UINT8(tco.msg1, TCOIORegs),
+        VMSTATE_UINT8(tco.msg2, TCOIORegs),
+        VMSTATE_UINT8(tco.wdcnt, TCOIORegs),
+        VMSTATE_UINT16(tco.tmr, TCOIORegs),
+        VMSTATE_UINT8(sw_irq_gen, TCOIORegs),
+        VMSTATE_TIMER_PTR(tco_timer, TCOIORegs),
+        VMSTATE_INT64(expire_time, TCOIORegs),
+        VMSTATE_UINT8(timeouts_no, TCOIORegs),
+        VMSTATE_END_OF_LIST()
+    }
+};
diff --git a/hw/acpi/trace-events b/hw/acpi/trace-events
new file mode 100644
index 0000000..c379607
--- /dev/null
+++ b/hw/acpi/trace-events
@@ -0,0 +1,32 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/acpi/memory_hotplug.c
+mhp_acpi_invalid_slot_selected(uint32_t slot) "0x%"PRIx32
+mhp_acpi_ejecting_invalid_slot(uint32_t slot) "0x%"PRIx32
+mhp_acpi_read_addr_lo(uint32_t slot, uint32_t addr) "slot[0x%"PRIx32"] addr lo: 0x%"PRIx32
+mhp_acpi_read_addr_hi(uint32_t slot, uint32_t addr) "slot[0x%"PRIx32"] addr hi: 0x%"PRIx32
+mhp_acpi_read_size_lo(uint32_t slot, uint32_t size) "slot[0x%"PRIx32"] size lo: 0x%"PRIx32
+mhp_acpi_read_size_hi(uint32_t slot, uint32_t size) "slot[0x%"PRIx32"] size hi: 0x%"PRIx32
+mhp_acpi_read_pxm(uint32_t slot, uint32_t pxm) "slot[0x%"PRIx32"] proximity: 0x%"PRIx32
+mhp_acpi_read_flags(uint32_t slot, uint32_t flags) "slot[0x%"PRIx32"] flags: 0x%"PRIx32
+mhp_acpi_write_slot(uint32_t slot) "set active slot: 0x%"PRIx32
+mhp_acpi_write_ost_ev(uint32_t slot, uint32_t ev) "slot[0x%"PRIx32"] OST EVENT: 0x%"PRIx32
+mhp_acpi_write_ost_status(uint32_t slot, uint32_t st) "slot[0x%"PRIx32"] OST STATUS: 0x%"PRIx32
+mhp_acpi_clear_insert_evt(uint32_t slot) "slot[0x%"PRIx32"] clear insert event"
+mhp_acpi_clear_remove_evt(uint32_t slot) "slot[0x%"PRIx32"] clear remove event"
+mhp_acpi_pc_dimm_deleted(uint32_t slot) "slot[0x%"PRIx32"] pc-dimm deleted"
+mhp_acpi_pc_dimm_delete_failed(uint32_t slot) "slot[0x%"PRIx32"] pc-dimm delete failed"
+
+# hw/acpi/cpu.c
+cpuhp_acpi_invalid_idx_selected(uint32_t idx) "0x%"PRIx32
+cpuhp_acpi_read_flags(uint32_t idx, uint8_t flags) "idx[0x%"PRIx32"] flags: 0x%"PRIx8
+cpuhp_acpi_write_idx(uint32_t idx) "set active cpu idx: 0x%"PRIx32
+cpuhp_acpi_write_cmd(uint32_t idx, uint8_t cmd) "idx[0x%"PRIx32"] cmd: 0x%"PRIx8
+cpuhp_acpi_read_cmd_data(uint32_t idx, uint32_t data) "idx[0x%"PRIx32"] data: 0x%"PRIx32
+cpuhp_acpi_cpu_has_events(uint32_t idx, bool ins, bool rm) "idx[0x%"PRIx32"] inserting: %d, removing: %d"
+cpuhp_acpi_clear_inserting_evt(uint32_t idx) "idx[0x%"PRIx32"]"
+cpuhp_acpi_clear_remove_evt(uint32_t idx) "idx[0x%"PRIx32"]"
+cpuhp_acpi_ejecting_invalid_cpu(uint32_t idx) "0x%"PRIx32
+cpuhp_acpi_ejecting_cpu(uint32_t idx) "0x%"PRIx32
+cpuhp_acpi_write_ost_ev(uint32_t slot, uint32_t ev) "idx[0x%"PRIx32"] OST EVENT: 0x%"PRIx32
+cpuhp_acpi_write_ost_status(uint32_t slot, uint32_t st) "idx[0x%"PRIx32"] OST STATUS: 0x%"PRIx32
diff --git a/hw/alpha/alpha_sys.h b/hw/alpha/alpha_sys.h
index e11025b..ed911f2 100644
--- a/hw/alpha/alpha_sys.h
+++ b/hw/alpha/alpha_sys.h
@@ -1,8 +1,9 @@
 /* Alpha cores and system support chips.  */
 
-#ifndef HW_ALPHA_H
-#define HW_ALPHA_H 1
+#ifndef HW_ALPHA_SYS_H
+#define HW_ALPHA_SYS_H
 
+#include "target-alpha/cpu-qom.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
 #include "hw/ide.h"
diff --git a/hw/alpha/dp264.c b/hw/alpha/dp264.c
index 84a55e4..f1267b5 100644
--- a/hw/alpha/dp264.c
+++ b/hw/alpha/dp264.c
@@ -6,16 +6,21 @@
  * that we need to emulate as well.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "elf.h"
 #include "hw/loader.h"
 #include "hw/boards.h"
 #include "alpha_sys.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "hw/timer/mc146818rtc.h"
 #include "hw/ide.h"
 #include "hw/timer/i8254.h"
 #include "hw/char/serial.h"
+#include "qemu/cutils.h"
 
 #define MAX_IDE_BUS 2
 
@@ -55,7 +60,7 @@
     ISABus *isa_bus;
     qemu_irq rtc_irq;
     long size, i;
-    const char *palcode_filename;
+    char *palcode_filename;
     uint64_t palcode_entry, palcode_low, palcode_high;
     uint64_t kernel_entry, kernel_low, kernel_high;
 
@@ -83,11 +88,7 @@
     pci_vga_init(pci_bus);
 
     /* Serial code setup.  */
-    for (i = 0; i < MAX_SERIAL_PORTS; ++i) {
-        if (serial_hds[i]) {
-            serial_isa_init(isa_bus, i, serial_hds[i]);
-        }
-    }
+    serial_hds_isa_init(isa_bus, MAX_SERIAL_PORTS);
 
     /* Network setup.  e1000 is good enough, failing Tulip support.  */
     for (i = 0; i < nb_nics; i++) {
@@ -105,19 +106,20 @@
     /* Load PALcode.  Given that this is not "real" cpu palcode,
        but one explicitly written for the emulation, we might as
        well load it directly from and ELF image.  */
-    palcode_filename = (bios_name ? bios_name : "palcode-clipper");
-    palcode_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, palcode_filename);
+    palcode_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS,
+                                bios_name ? bios_name : "palcode-clipper");
     if (palcode_filename == NULL) {
-        hw_error("no palcode provided\n");
+        error_report("no palcode provided");
         exit(1);
     }
     size = load_elf(palcode_filename, cpu_alpha_superpage_to_phys,
                     NULL, &palcode_entry, &palcode_low, &palcode_high,
-                    0, EM_ALPHA, 0);
+                    0, EM_ALPHA, 0, 0);
     if (size < 0) {
-        hw_error("could not load palcode '%s'\n", palcode_filename);
+        error_report("could not load palcode '%s'", palcode_filename);
         exit(1);
     }
+    g_free(palcode_filename);
 
     /* Start all cpus at the PALcode RESET entry point.  */
     for (i = 0; i < smp_cpus; ++i) {
@@ -132,9 +134,9 @@
 
         size = load_elf(kernel_filename, cpu_alpha_superpage_to_phys,
                         NULL, &kernel_entry, &kernel_low, &kernel_high,
-                        0, EM_ALPHA, 0);
+                        0, EM_ALPHA, 0, 0);
         if (size < 0) {
-            hw_error("could not load kernel '%s'\n", kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
 
@@ -151,8 +153,8 @@
 
             initrd_size = get_image_size(initrd_filename);
             if (initrd_size < 0) {
-                hw_error("could not load initial ram disk '%s'\n",
-                         initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
 
@@ -161,24 +163,22 @@
             load_image_targphys(initrd_filename, initrd_base,
                                 ram_size - initrd_base);
 
-            stq_phys(&address_space_memory,
-                     param_offset + 0x100, initrd_base + 0xfffffc0000000000ULL);
-            stq_phys(&address_space_memory, param_offset + 0x108, initrd_size);
+            address_space_stq(&address_space_memory, param_offset + 0x100,
+                              initrd_base + 0xfffffc0000000000ULL,
+                              MEMTXATTRS_UNSPECIFIED,
+                              NULL);
+            address_space_stq(&address_space_memory, param_offset + 0x108,
+                              initrd_size, MEMTXATTRS_UNSPECIFIED, NULL);
         }
     }
 }
 
-static QEMUMachine clipper_machine = {
-    .name = "clipper",
-    .desc = "Alpha DP264/CLIPPER",
-    .init = clipper_init,
-    .max_cpus = 4,
-    .is_default = 1,
-};
-
-static void clipper_machine_init(void)
+static void clipper_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&clipper_machine);
+    mc->desc = "Alpha DP264/CLIPPER";
+    mc->init = clipper_init;
+    mc->max_cpus = 4;
+    mc->is_default = 1;
 }
 
-machine_init(clipper_machine_init);
+DEFINE_MACHINE("clipper", clipper_machine_init)
diff --git a/hw/alpha/pci.c b/hw/alpha/pci.c
index d839dd5..8dde637 100644
--- a/hw/alpha/pci.c
+++ b/hw/alpha/pci.c
@@ -6,10 +6,12 @@
  * ??? Sparse memory access not implemented.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "alpha_sys.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
+#include "trace.h"
 
 
 /* Fallback for unassigned PCI I/O operations.  Avoids MCHK.  */
@@ -73,7 +75,7 @@
 static void special_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
-    qemu_log("pci: special write cycle");
+    trace_alpha_pci_iack_write();
 }
 
 const MemoryRegionOps alpha_pci_iack_ops = {
diff --git a/hw/alpha/trace-events b/hw/alpha/trace-events
new file mode 100644
index 0000000..e44ff01
--- /dev/null
+++ b/hw/alpha/trace-events
@@ -0,0 +1,4 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/alpha/pci.c
+alpha_pci_iack_write(void) ""
diff --git a/hw/alpha/typhoon.c b/hw/alpha/typhoon.c
index 5310006..883db13 100644
--- a/hw/alpha/typhoon.c
+++ b/hw/alpha/typhoon.c
@@ -6,6 +6,8 @@
  * This work is licensed under the GNU GPL license version 2 or later.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
@@ -613,7 +615,8 @@
    translation, given the address of the PTE.  */
 static bool pte_translate(hwaddr pte_addr, IOMMUTLBEntry *ret)
 {
-    uint64_t pte = ldq_phys(&address_space_memory, pte_addr);
+    uint64_t pte = address_space_ldq(&address_space_memory, pte_addr,
+                                     MEMTXATTRS_UNSPECIFIED, NULL);
 
     /* Check valid bit.  */
     if ((pte & 1) == 0) {
@@ -821,7 +824,6 @@
     int i;
 
     dev = qdev_create(NULL, TYPE_TYPHOON_PCI_HOST_BRIDGE);
-    qdev_init_nofail(dev);
 
     s = TYPHOON_PCI_HOST_BRIDGE(dev);
     phb = PCI_HOST_BRIDGE(dev);
@@ -840,13 +842,12 @@
         }
     }
 
-    *p_rtc_irq = *qemu_allocate_irqs(typhoon_set_timer_irq, s, 1);
+    *p_rtc_irq = qemu_allocate_irq(typhoon_set_timer_irq, s, 0);
 
     /* Main memory region, 0x00.0000.0000.  Real hardware supports 32GB,
        but the address space hole reserved at this point is 8TB.  */
-    memory_region_init_ram(&s->ram_region, OBJECT(s), "ram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(&s->ram_region);
+    memory_region_allocate_system_memory(&s->ram_region, OBJECT(s), "ram",
+                                         ram_size);
     memory_region_add_subregion(addr_space, 0, &s->ram_region);
 
     /* TIGbus, 0x801.0000.0000, 1GB.  */
@@ -887,6 +888,7 @@
                          &s->pchip.reg_mem, &s->pchip.reg_io,
                          0, 64, TYPE_PCI_BUS);
     phb->bus = b;
+    qdev_init_nofail(dev);
 
     /* Host memory as seen from the PCI side, via the IOMMU.  */
     memory_region_init_iommu(&s->pchip.iommu, OBJECT(s), &typhoon_iommu_ops,
@@ -918,11 +920,12 @@
     /* Init the ISA bus.  */
     /* ??? Technically there should be a cy82c693ub pci-isa bridge.  */
     {
-        qemu_irq isa_pci_irq, *isa_irqs;
+        qemu_irq *isa_irqs;
 
-        *isa_bus = isa_bus_new(NULL, &s->pchip.reg_io);
-        isa_pci_irq = *qemu_allocate_irqs(typhoon_set_isa_irq, s, 1);
-        isa_irqs = i8259_init(*isa_bus, isa_pci_irq);
+        *isa_bus = isa_bus_new(NULL, get_system_memory(), &s->pchip.reg_io,
+                               &error_abort);
+        isa_irqs = i8259_init(*isa_bus,
+                              qemu_allocate_irq(typhoon_set_isa_irq, s, 0));
         isa_bus_irqs(*isa_bus, isa_irqs);
     }
 
diff --git a/hw/arm/Makefile.objs b/hw/arm/Makefile.objs
index 71b2a0a..de266aa 100644
--- a/hw/arm/Makefile.objs
+++ b/hw/arm/Makefile.objs
@@ -1,11 +1,22 @@
 obj-y += boot.o collie.o exynos4_boards.o gumstix.o highbank.o
 obj-$(CONFIG_DIGIC) += digic_boards.o
-obj-y += integratorcp.o kzm.o mainstone.o musicpal.o nseries.o
+obj-y += integratorcp.o mainstone.o musicpal.o nseries.o
 obj-y += omap_sx1.o palm.o realview.o spitz.o stellaris.o
 obj-y += tosa.o versatilepb.o vexpress.o virt.o xilinx_zynq.o z2.o
+obj-$(CONFIG_ACPI) += virt-acpi-build.o
+obj-y += netduino2.o
+obj-y += sysbus-fdt.o
+obj-y += ranchu.o
 
 obj-y += armv7m.o exynos4210.o pxa2xx.o pxa2xx_gpio.o pxa2xx_pic.o
 obj-$(CONFIG_DIGIC) += digic.o
 obj-y += omap1.o omap2.o strongarm.o
 obj-$(CONFIG_ALLWINNER_A10) += allwinner-a10.o cubieboard.o
-obj-$(CONFIG_ANDROID) += ranchu.o
+obj-$(CONFIG_RASPI) += bcm2835_peripherals.o bcm2836.o raspi.o
+obj-$(CONFIG_STM32F205_SOC) += stm32f205_soc.o
+obj-$(CONFIG_XLNX_ZYNQMP) += xlnx-zynqmp.o xlnx-ep108.o
+obj-$(CONFIG_FSL_IMX25) += fsl-imx25.o imx25_pdk.o
+obj-$(CONFIG_FSL_IMX31) += fsl-imx31.o kzm.o
+obj-$(CONFIG_FSL_IMX6) += fsl-imx6.o sabrelite.o
+obj-$(CONFIG_ASPEED_SOC) += ast2400.o palmetto-bmc.o
+
diff --git a/hw/arm/allwinner-a10.c b/hw/arm/allwinner-a10.c
index 01206f2..ca15d1c 100644
--- a/hw/arm/allwinner-a10.c
+++ b/hw/arm/allwinner-a10.c
@@ -15,6 +15,10 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/arm/allwinner-a10.h"
@@ -34,10 +38,14 @@
 
     object_initialize(&s->emac, sizeof(s->emac), TYPE_AW_EMAC);
     qdev_set_parent_bus(DEVICE(&s->emac), sysbus_get_default());
+    /* FIXME use qdev NIC properties instead of nd_table[] */
     if (nd_table[0].used) {
         qemu_check_nic_model(&nd_table[0], TYPE_AW_EMAC);
         qdev_set_nic_properties(DEVICE(&s->emac), &nd_table[0]);
     }
+
+    object_initialize(&s->sata, sizeof(s->sata), TYPE_ALLWINNER_AHCI);
+    qdev_set_parent_bus(DEVICE(&s->sata), sysbus_get_default());
 }
 
 static void aw_a10_realize(DeviceState *dev, Error **errp)
@@ -92,6 +100,15 @@
     sysbus_mmio_map(sysbusdev, 0, AW_A10_EMAC_BASE);
     sysbus_connect_irq(sysbusdev, 0, s->irq[55]);
 
+    object_property_set_bool(OBJECT(&s->sata), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->sata), 0, AW_A10_SATA_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sata), 0, s->irq[56]);
+
+    /* FIXME use a qdev chardev prop instead of serial_hds[] */
     serial_mm_init(get_system_memory(), AW_A10_UART0_REG_BASE, 2, s->irq[1],
                    115200, serial_hds[0], DEVICE_NATIVE_ENDIAN);
 }
@@ -101,6 +118,12 @@
     DeviceClass *dc = DEVICE_CLASS(oc);
 
     dc->realize = aw_a10_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo aw_a10_type_info = {
diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c
index ef24ca4..49d3078 100644
--- a/hw/arm/armv7m.c
+++ b/hw/arm/armv7m.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/loader.h"
@@ -128,14 +132,14 @@
     uint32_t base;
 } BitBandState;
 
-static int bitband_init(SysBusDevice *dev)
+static void bitband_init(Object *obj)
 {
-    BitBandState *s = BITBAND(dev);
+    BitBandState *s = BITBAND(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &bitband_ops, &s->base,
+    memory_region_init_io(&s->iomem, obj, &bitband_ops, &s->base,
                           "bitband", 0x02000000);
     sysbus_init_mmio(dev, &s->iomem);
-    return 0;
 }
 
 static void armv7m_bitband_init(void)
@@ -163,30 +167,21 @@
 }
 
 /* Init CPU and memory for a v7-M based board.
-   flash_size and sram_size are in kb.
+   mem_size is in bytes.
    Returns the NVIC array.  */
 
-qemu_irq *armv7m_init(MemoryRegion *system_memory,
-                      int flash_size, int sram_size,
+DeviceState *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
                       const char *kernel_filename, const char *cpu_model)
 {
     ARMCPU *cpu;
     CPUARMState *env;
     DeviceState *nvic;
-    /* FIXME: make this local state.  */
-    static qemu_irq pic[64];
     int image_size;
     uint64_t entry;
     uint64_t lowaddr;
-    int i;
     int big_endian;
-    MemoryRegion *sram = g_new(MemoryRegion, 1);
-    MemoryRegion *flash = g_new(MemoryRegion, 1);
     MemoryRegion *hack = g_new(MemoryRegion, 1);
 
-    flash_size *= 1024;
-    sram_size *= 1024;
-
     if (cpu_model == NULL) {
 	cpu_model = "cortex-m3";
     }
@@ -197,37 +192,14 @@
     }
     env = &cpu->env;
 
-#if 0
-    /* > 32Mb SRAM gets complicated because it overlaps the bitband area.
-       We don't have proper commandline options, so allocate half of memory
-       as SRAM, up to a maximum of 32Mb, and the rest as code.  */
-    if (ram_size > (512 + 32) * 1024 * 1024)
-        ram_size = (512 + 32) * 1024 * 1024;
-    sram_size = (ram_size / 2) & TARGET_PAGE_MASK;
-    if (sram_size > 32 * 1024 * 1024)
-        sram_size = 32 * 1024 * 1024;
-    code_size = ram_size - sram_size;
-#endif
-
-    /* Flash programming is done via the SCU, so pretend it is ROM.  */
-    memory_region_init_ram(flash, NULL, "armv7m.flash", flash_size,
-                           &error_abort);
-    vmstate_register_ram_global(flash);
-    memory_region_set_readonly(flash, true);
-    memory_region_add_subregion(system_memory, 0, flash);
-    memory_region_init_ram(sram, NULL, "armv7m.sram", sram_size, &error_abort);
-    vmstate_register_ram_global(sram);
-    memory_region_add_subregion(system_memory, 0x20000000, sram);
     armv7m_bitband_init();
 
     nvic = qdev_create(NULL, "armv7m_nvic");
+    qdev_prop_set_uint32(nvic, "num-irq", num_irq);
     env->nvic = nvic;
     qdev_init_nofail(nvic);
     sysbus_connect_irq(SYS_BUS_DEVICE(nvic), 0,
                        qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ));
-    for (i = 0; i < 64; i++) {
-        pic[i] = qdev_get_gpio_in(nvic, i);
-    }
 
 #ifdef TARGET_WORDS_BIGENDIAN
     big_endian = 1;
@@ -242,9 +214,9 @@
 
     if (kernel_filename) {
         image_size = load_elf(kernel_filename, NULL, NULL, &entry, &lowaddr,
-                              NULL, big_endian, ELF_MACHINE, 1);
+                              NULL, big_endian, EM_ARM, 1, 0);
         if (image_size < 0) {
-            image_size = load_image_targphys(kernel_filename, 0, flash_size);
+            image_size = load_image_targphys(kernel_filename, 0, mem_size);
             lowaddr = 0;
         }
         if (image_size < 0) {
@@ -256,12 +228,12 @@
     /* Hack to map an additional page of ram at the top of the address
        space.  This stops qemu complaining about executing code outside RAM
        when returning from an exception.  */
-    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_abort);
+    memory_region_init_ram(hack, NULL, "armv7m.hack", 0x1000, &error_fatal);
     vmstate_register_ram_global(hack);
     memory_region_add_subregion(system_memory, 0xfffff000, hack);
 
     qemu_register_reset(armv7m_reset, cpu);
-    return pic;
+    return nvic;
 }
 
 static Property bitband_properties[] = {
@@ -272,9 +244,7 @@
 static void bitband_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = bitband_init;
     dc->props = bitband_properties;
 }
 
@@ -282,6 +252,7 @@
     .name          = TYPE_BITBAND,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(BitBandState),
+    .instance_init = bitband_init,
     .class_init    = bitband_class_init,
 };
 
diff --git a/hw/arm/ast2400.c b/hw/arm/ast2400.c
new file mode 100644
index 0000000..326fdb3
--- /dev/null
+++ b/hw/arm/ast2400.c
@@ -0,0 +1,214 @@
+/*
+ * AST2400 SoC
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ * Jeremy Kerr <jk@ozlabs.org>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/address-spaces.h"
+#include "hw/arm/ast2400.h"
+#include "hw/char/serial.h"
+#include "qemu/log.h"
+#include "hw/i2c/aspeed_i2c.h"
+
+#define AST2400_UART_5_BASE      0x00184000
+#define AST2400_IOMEM_SIZE       0x00200000
+#define AST2400_IOMEM_BASE       0x1E600000
+#define AST2400_SMC_BASE         AST2400_IOMEM_BASE /* Legacy SMC */
+#define AST2400_FMC_BASE         0X1E620000
+#define AST2400_SPI_BASE         0X1E630000
+#define AST2400_VIC_BASE         0x1E6C0000
+#define AST2400_SCU_BASE         0x1E6E2000
+#define AST2400_TIMER_BASE       0x1E782000
+#define AST2400_I2C_BASE         0x1E78A000
+
+#define AST2400_FMC_FLASH_BASE   0x20000000
+#define AST2400_SPI_FLASH_BASE   0x30000000
+
+static const int uart_irqs[] = { 9, 32, 33, 34, 10 };
+static const int timer_irqs[] = { 16, 17, 18, 35, 36, 37, 38, 39, };
+
+/*
+ * IO handlers: simply catch any reads/writes to IO addresses that aren't
+ * handled by a device mapping.
+ */
+
+static uint64_t ast2400_io_read(void *p, hwaddr offset, unsigned size)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: 0x%" HWADDR_PRIx " [%u]\n",
+                  __func__, offset, size);
+    return 0;
+}
+
+static void ast2400_io_write(void *opaque, hwaddr offset, uint64_t value,
+                unsigned size)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: 0x%" HWADDR_PRIx " <- 0x%" PRIx64 " [%u]\n",
+                  __func__, offset, value, size);
+}
+
+static const MemoryRegionOps ast2400_io_ops = {
+    .read = ast2400_io_read,
+    .write = ast2400_io_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ast2400_init(Object *obj)
+{
+    AST2400State *s = AST2400(obj);
+
+    s->cpu = cpu_arm_init("arm926");
+
+    object_initialize(&s->vic, sizeof(s->vic), TYPE_ASPEED_VIC);
+    object_property_add_child(obj, "vic", OBJECT(&s->vic), NULL);
+    qdev_set_parent_bus(DEVICE(&s->vic), sysbus_get_default());
+
+    object_initialize(&s->timerctrl, sizeof(s->timerctrl), TYPE_ASPEED_TIMER);
+    object_property_add_child(obj, "timerctrl", OBJECT(&s->timerctrl), NULL);
+    qdev_set_parent_bus(DEVICE(&s->timerctrl), sysbus_get_default());
+
+    object_initialize(&s->i2c, sizeof(s->i2c), TYPE_ASPEED_I2C);
+    object_property_add_child(obj, "i2c", OBJECT(&s->i2c), NULL);
+    qdev_set_parent_bus(DEVICE(&s->i2c), sysbus_get_default());
+
+    object_initialize(&s->scu, sizeof(s->scu), TYPE_ASPEED_SCU);
+    object_property_add_child(obj, "scu", OBJECT(&s->scu), NULL);
+    qdev_set_parent_bus(DEVICE(&s->scu), sysbus_get_default());
+    qdev_prop_set_uint32(DEVICE(&s->scu), "silicon-rev",
+                         AST2400_A0_SILICON_REV);
+    object_property_add_alias(obj, "hw-strap1", OBJECT(&s->scu),
+                              "hw-strap1", &error_abort);
+    object_property_add_alias(obj, "hw-strap2", OBJECT(&s->scu),
+                              "hw-strap2", &error_abort);
+
+    object_initialize(&s->smc, sizeof(s->smc), "aspeed.smc.fmc");
+    object_property_add_child(obj, "smc", OBJECT(&s->smc), NULL);
+    qdev_set_parent_bus(DEVICE(&s->smc), sysbus_get_default());
+
+    object_initialize(&s->spi, sizeof(s->spi), "aspeed.smc.spi");
+    object_property_add_child(obj, "spi", OBJECT(&s->spi), NULL);
+    qdev_set_parent_bus(DEVICE(&s->spi), sysbus_get_default());
+}
+
+static void ast2400_realize(DeviceState *dev, Error **errp)
+{
+    int i;
+    AST2400State *s = AST2400(dev);
+    Error *err = NULL, *local_err = NULL;
+
+    /* IO space */
+    memory_region_init_io(&s->iomem, NULL, &ast2400_io_ops, NULL,
+            "ast2400.io", AST2400_IOMEM_SIZE);
+    memory_region_add_subregion_overlap(get_system_memory(), AST2400_IOMEM_BASE,
+            &s->iomem, -1);
+
+    /* VIC */
+    object_property_set_bool(OBJECT(&s->vic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->vic), 0, AST2400_VIC_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->vic), 0,
+                       qdev_get_gpio_in(DEVICE(s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->vic), 1,
+                       qdev_get_gpio_in(DEVICE(s->cpu), ARM_CPU_FIQ));
+
+    /* Timer */
+    object_property_set_bool(OBJECT(&s->timerctrl), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->timerctrl), 0, AST2400_TIMER_BASE);
+    for (i = 0; i < ARRAY_SIZE(timer_irqs); i++) {
+        qemu_irq irq = qdev_get_gpio_in(DEVICE(&s->vic), timer_irqs[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->timerctrl), i, irq);
+    }
+
+    /* SCU */
+    object_property_set_bool(OBJECT(&s->scu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->scu), 0, AST2400_SCU_BASE);
+
+    /* UART - attach an 8250 to the IO space as our UART5 */
+    if (serial_hds[0]) {
+        qemu_irq uart5 = qdev_get_gpio_in(DEVICE(&s->vic), uart_irqs[4]);
+        serial_mm_init(&s->iomem, AST2400_UART_5_BASE, 2,
+                       uart5, 38400, serial_hds[0], DEVICE_LITTLE_ENDIAN);
+    }
+
+    /* I2C */
+    object_property_set_bool(OBJECT(&s->i2c), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c), 0, AST2400_I2C_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c), 0,
+                       qdev_get_gpio_in(DEVICE(&s->vic), 12));
+
+    /* SMC */
+    object_property_set_int(OBJECT(&s->smc), 1, "num-cs", &err);
+    object_property_set_bool(OBJECT(&s->smc), true, "realized", &local_err);
+    error_propagate(&err, local_err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->smc), 0, AST2400_FMC_BASE);
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->smc), 1, AST2400_FMC_FLASH_BASE);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->smc), 0,
+                       qdev_get_gpio_in(DEVICE(&s->vic), 19));
+
+    /* SPI */
+    object_property_set_int(OBJECT(&s->spi), 1, "num-cs", &err);
+    object_property_set_bool(OBJECT(&s->spi), true, "realized", &local_err);
+    error_propagate(&err, local_err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->spi), 0, AST2400_SPI_BASE);
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->spi), 1, AST2400_SPI_FLASH_BASE);
+}
+
+static void ast2400_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = ast2400_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo ast2400_type_info = {
+    .name = TYPE_AST2400,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AST2400State),
+    .instance_init = ast2400_init,
+    .class_init = ast2400_class_init,
+};
+
+static void ast2400_register_types(void)
+{
+    type_register_static(&ast2400_type_info);
+}
+
+type_init(ast2400_register_types)
diff --git a/hw/arm/bcm2835_peripherals.c b/hw/arm/bcm2835_peripherals.c
new file mode 100644
index 0000000..2e641a3
--- /dev/null
+++ b/hw/arm/bcm2835_peripherals.c
@@ -0,0 +1,302 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/arm/bcm2835_peripherals.h"
+#include "hw/misc/bcm2835_mbox_defs.h"
+#include "hw/arm/raspi_platform.h"
+#include "sysemu/char.h"
+#include "sysemu/sysemu.h"
+
+/* Peripheral base address on the VC (GPU) system bus */
+#define BCM2835_VC_PERI_BASE 0x7e000000
+
+/* Capabilities for SD controller: no DMA, high-speed, default clocks etc. */
+#define BCM2835_SDHC_CAPAREG 0x52034b4
+
+static void bcm2835_peripherals_init(Object *obj)
+{
+    BCM2835PeripheralState *s = BCM2835_PERIPHERALS(obj);
+
+    /* Memory region for peripheral devices, which we export to our parent */
+    memory_region_init(&s->peri_mr, obj,"bcm2835-peripherals", 0x1000000);
+    object_property_add_child(obj, "peripheral-io", OBJECT(&s->peri_mr), NULL);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->peri_mr);
+
+    /* Internal memory region for peripheral bus addresses (not exported) */
+    memory_region_init(&s->gpu_bus_mr, obj, "bcm2835-gpu", (uint64_t)1 << 32);
+    object_property_add_child(obj, "gpu-bus", OBJECT(&s->gpu_bus_mr), NULL);
+
+    /* Internal memory region for request/response communication with
+     * mailbox-addressable peripherals (not exported)
+     */
+    memory_region_init(&s->mbox_mr, obj, "bcm2835-mbox",
+                       MBOX_CHAN_COUNT << MBOX_AS_CHAN_SHIFT);
+
+    /* Interrupt Controller */
+    object_initialize(&s->ic, sizeof(s->ic), TYPE_BCM2835_IC);
+    object_property_add_child(obj, "ic", OBJECT(&s->ic), NULL);
+    qdev_set_parent_bus(DEVICE(&s->ic), sysbus_get_default());
+
+    /* UART0 */
+    s->uart0 = SYS_BUS_DEVICE(object_new("pl011"));
+    object_property_add_child(obj, "uart0", OBJECT(s->uart0), NULL);
+    qdev_set_parent_bus(DEVICE(s->uart0), sysbus_get_default());
+
+    /* AUX / UART1 */
+    object_initialize(&s->aux, sizeof(s->aux), TYPE_BCM2835_AUX);
+    object_property_add_child(obj, "aux", OBJECT(&s->aux), NULL);
+    qdev_set_parent_bus(DEVICE(&s->aux), sysbus_get_default());
+
+    /* Mailboxes */
+    object_initialize(&s->mboxes, sizeof(s->mboxes), TYPE_BCM2835_MBOX);
+    object_property_add_child(obj, "mbox", OBJECT(&s->mboxes), NULL);
+    qdev_set_parent_bus(DEVICE(&s->mboxes), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->mboxes), "mbox-mr",
+                                   OBJECT(&s->mbox_mr), &error_abort);
+
+    /* Framebuffer */
+    object_initialize(&s->fb, sizeof(s->fb), TYPE_BCM2835_FB);
+    object_property_add_child(obj, "fb", OBJECT(&s->fb), NULL);
+    object_property_add_alias(obj, "vcram-size", OBJECT(&s->fb), "vcram-size",
+                              &error_abort);
+    qdev_set_parent_bus(DEVICE(&s->fb), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->fb), "dma-mr",
+                                   OBJECT(&s->gpu_bus_mr), &error_abort);
+
+    /* Property channel */
+    object_initialize(&s->property, sizeof(s->property), TYPE_BCM2835_PROPERTY);
+    object_property_add_child(obj, "property", OBJECT(&s->property), NULL);
+    object_property_add_alias(obj, "board-rev", OBJECT(&s->property),
+                              "board-rev", &error_abort);
+    qdev_set_parent_bus(DEVICE(&s->property), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->property), "fb",
+                                   OBJECT(&s->fb), &error_abort);
+    object_property_add_const_link(OBJECT(&s->property), "dma-mr",
+                                   OBJECT(&s->gpu_bus_mr), &error_abort);
+
+    /* Extended Mass Media Controller */
+    object_initialize(&s->sdhci, sizeof(s->sdhci), TYPE_SYSBUS_SDHCI);
+    object_property_add_child(obj, "sdhci", OBJECT(&s->sdhci), NULL);
+    qdev_set_parent_bus(DEVICE(&s->sdhci), sysbus_get_default());
+
+    /* DMA Channels */
+    object_initialize(&s->dma, sizeof(s->dma), TYPE_BCM2835_DMA);
+    object_property_add_child(obj, "dma", OBJECT(&s->dma), NULL);
+    qdev_set_parent_bus(DEVICE(&s->dma), sysbus_get_default());
+
+    object_property_add_const_link(OBJECT(&s->dma), "dma-mr",
+                                   OBJECT(&s->gpu_bus_mr), &error_abort);
+}
+
+static void bcm2835_peripherals_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835PeripheralState *s = BCM2835_PERIPHERALS(dev);
+    Object *obj;
+    MemoryRegion *ram;
+    Error *err = NULL;
+    uint32_t ram_size, vcram_size;
+    int n;
+
+    obj = object_property_get_link(OBJECT(dev), "ram", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required ram link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    ram = MEMORY_REGION(obj);
+    ram_size = memory_region_size(ram);
+
+    /* Map peripherals and RAM into the GPU address space. */
+    memory_region_init_alias(&s->peri_mr_alias, OBJECT(s),
+                             "bcm2835-peripherals", &s->peri_mr, 0,
+                             memory_region_size(&s->peri_mr));
+
+    memory_region_add_subregion_overlap(&s->gpu_bus_mr, BCM2835_VC_PERI_BASE,
+                                        &s->peri_mr_alias, 1);
+
+    /* RAM is aliased four times (different cache configurations) on the GPU */
+    for (n = 0; n < 4; n++) {
+        memory_region_init_alias(&s->ram_alias[n], OBJECT(s),
+                                 "bcm2835-gpu-ram-alias[*]", ram, 0, ram_size);
+        memory_region_add_subregion_overlap(&s->gpu_bus_mr, (hwaddr)n << 30,
+                                            &s->ram_alias[n], 0);
+    }
+
+    /* Interrupt Controller */
+    object_property_set_bool(OBJECT(&s->ic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, ARMCTRL_IC_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->ic), 0));
+    sysbus_pass_irq(SYS_BUS_DEVICE(s), SYS_BUS_DEVICE(&s->ic));
+
+    /* UART0 */
+    qdev_prop_set_chr(DEVICE(s->uart0), "chardev", serial_hds[0]);
+    object_property_set_bool(OBJECT(s->uart0), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, UART0_OFFSET,
+                                sysbus_mmio_get_region(s->uart0, 0));
+    sysbus_connect_irq(s->uart0, 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_GPU_IRQ,
+                               INTERRUPT_UART));
+    /* AUX / UART1 */
+    qdev_prop_set_chr(DEVICE(&s->aux), "chardev", serial_hds[1]);
+
+    object_property_set_bool(OBJECT(&s->aux), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, UART1_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->aux), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->aux), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_GPU_IRQ,
+                               INTERRUPT_AUX));
+
+    /* Mailboxes */
+    object_property_set_bool(OBJECT(&s->mboxes), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, ARMCTRL_0_SBM_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->mboxes), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->mboxes), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_ARM_IRQ,
+                               INTERRUPT_ARM_MAILBOX));
+
+    /* Framebuffer */
+    vcram_size = (uint32_t)object_property_get_int(OBJECT(s), "vcram-size",
+                                                   &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_int(OBJECT(&s->fb), ram_size - vcram_size,
+                            "vcram-base", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->fb), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->mbox_mr, MBOX_CHAN_FB << MBOX_AS_CHAN_SHIFT,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->fb), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->fb), 0,
+                       qdev_get_gpio_in(DEVICE(&s->mboxes), MBOX_CHAN_FB));
+
+    /* Property channel */
+    object_property_set_bool(OBJECT(&s->property), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->mbox_mr,
+                MBOX_CHAN_PROPERTY << MBOX_AS_CHAN_SHIFT,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->property), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->property), 0,
+                      qdev_get_gpio_in(DEVICE(&s->mboxes), MBOX_CHAN_PROPERTY));
+
+    /* Extended Mass Media Controller */
+    object_property_set_int(OBJECT(&s->sdhci), BCM2835_SDHC_CAPAREG, "capareg",
+                            &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->sdhci), true, "pending-insert-quirk",
+                             &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->sdhci), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, EMMC_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->sdhci), 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sdhci), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->ic), BCM2835_IC_GPU_IRQ,
+                               INTERRUPT_ARASANSDIO));
+    object_property_add_alias(OBJECT(s), "sd-bus", OBJECT(&s->sdhci), "sd-bus",
+                              &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    /* DMA Channels */
+    object_property_set_bool(OBJECT(&s->dma), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->peri_mr, DMA_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->dma), 0));
+    memory_region_add_subregion(&s->peri_mr, DMA15_OFFSET,
+                sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->dma), 1));
+
+    for (n = 0; n <= 12; n++) {
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->dma), n,
+                           qdev_get_gpio_in_named(DEVICE(&s->ic),
+                                                  BCM2835_IC_GPU_IRQ,
+                                                  INTERRUPT_DMA0 + n));
+    }
+}
+
+static void bcm2835_peripherals_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = bcm2835_peripherals_realize;
+}
+
+static const TypeInfo bcm2835_peripherals_type_info = {
+    .name = TYPE_BCM2835_PERIPHERALS,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835PeripheralState),
+    .instance_init = bcm2835_peripherals_init,
+    .class_init = bcm2835_peripherals_class_init,
+};
+
+static void bcm2835_peripherals_register_types(void)
+{
+    type_register_static(&bcm2835_peripherals_type_info);
+}
+
+type_init(bcm2835_peripherals_register_types)
diff --git a/hw/arm/bcm2836.c b/hw/arm/bcm2836.c
new file mode 100644
index 0000000..8451190
--- /dev/null
+++ b/hw/arm/bcm2836.c
@@ -0,0 +1,184 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/bcm2836.h"
+#include "hw/arm/raspi_platform.h"
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+
+/* Peripheral base address seen by the CPU */
+#define BCM2836_PERI_BASE       0x3F000000
+
+/* "QA7" (Pi2) interrupt controller and mailboxes etc. */
+#define BCM2836_CONTROL_BASE    0x40000000
+
+static void bcm2836_init(Object *obj)
+{
+    BCM2836State *s = BCM2836(obj);
+    int n;
+
+    for (n = 0; n < BCM2836_NCPUS; n++) {
+        object_initialize(&s->cpus[n], sizeof(s->cpus[n]),
+                          "cortex-a15-" TYPE_ARM_CPU);
+        object_property_add_child(obj, "cpu[*]", OBJECT(&s->cpus[n]),
+                                  &error_abort);
+    }
+
+    object_initialize(&s->control, sizeof(s->control), TYPE_BCM2836_CONTROL);
+    object_property_add_child(obj, "control", OBJECT(&s->control), NULL);
+    qdev_set_parent_bus(DEVICE(&s->control), sysbus_get_default());
+
+    object_initialize(&s->peripherals, sizeof(s->peripherals),
+                      TYPE_BCM2835_PERIPHERALS);
+    object_property_add_child(obj, "peripherals", OBJECT(&s->peripherals),
+                              &error_abort);
+    object_property_add_alias(obj, "board-rev", OBJECT(&s->peripherals),
+                              "board-rev", &error_abort);
+    object_property_add_alias(obj, "vcram-size", OBJECT(&s->peripherals),
+                              "vcram-size", &error_abort);
+    qdev_set_parent_bus(DEVICE(&s->peripherals), sysbus_get_default());
+}
+
+static void bcm2836_realize(DeviceState *dev, Error **errp)
+{
+    BCM2836State *s = BCM2836(dev);
+    Object *obj;
+    Error *err = NULL;
+    int n;
+
+    /* common peripherals from bcm2835 */
+
+    obj = object_property_get_link(OBJECT(dev), "ram", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required ram link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    object_property_add_const_link(OBJECT(&s->peripherals), "ram", obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->peripherals), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_add_alias(OBJECT(s), "sd-bus", OBJECT(&s->peripherals),
+                              "sd-bus", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map_overlap(SYS_BUS_DEVICE(&s->peripherals), 0,
+                            BCM2836_PERI_BASE, 1);
+
+    /* bcm2836 interrupt controller (and mailboxes, etc.) */
+    object_property_set_bool(OBJECT(&s->control), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->control), 0, BCM2836_CONTROL_BASE);
+
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->peripherals), 0,
+        qdev_get_gpio_in_named(DEVICE(&s->control), "gpu-irq", 0));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->peripherals), 1,
+        qdev_get_gpio_in_named(DEVICE(&s->control), "gpu-fiq", 0));
+
+    for (n = 0; n < BCM2836_NCPUS; n++) {
+        /* Mirror bcm2836, which has clusterid set to 0xf
+         * TODO: this should be converted to a property of ARM_CPU
+         */
+        s->cpus[n].mp_affinity = 0xF00 | n;
+
+        /* set periphbase/CBAR value for CPU-local registers */
+        object_property_set_int(OBJECT(&s->cpus[n]),
+                                BCM2836_PERI_BASE + MCORE_OFFSET,
+                                "reset-cbar", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        /* start powered off if not enabled */
+        object_property_set_bool(OBJECT(&s->cpus[n]), n >= s->enabled_cpus,
+                                 "start-powered-off", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        object_property_set_bool(OBJECT(&s->cpus[n]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        /* Connect irq/fiq outputs from the interrupt controller. */
+        qdev_connect_gpio_out_named(DEVICE(&s->control), "irq", n,
+                qdev_get_gpio_in(DEVICE(&s->cpus[n]), ARM_CPU_IRQ));
+        qdev_connect_gpio_out_named(DEVICE(&s->control), "fiq", n,
+                qdev_get_gpio_in(DEVICE(&s->cpus[n]), ARM_CPU_FIQ));
+
+        /* Connect timers from the CPU to the interrupt controller */
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_PHYS,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cntpnsirq", n));
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_VIRT,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cntvirq", n));
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_HYP,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cnthpirq", n));
+        qdev_connect_gpio_out(DEVICE(&s->cpus[n]), GTIMER_SEC,
+                qdev_get_gpio_in_named(DEVICE(&s->control), "cntpsirq", n));
+    }
+}
+
+static Property bcm2836_props[] = {
+    DEFINE_PROP_UINT32("enabled-cpus", BCM2836State, enabled_cpus, BCM2836_NCPUS),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void bcm2836_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->props = bcm2836_props;
+    dc->realize = bcm2836_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo bcm2836_type_info = {
+    .name = TYPE_BCM2836,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2836State),
+    .instance_init = bcm2836_init,
+    .class_init = bcm2836_class_init,
+};
+
+static void bcm2836_register_types(void)
+{
+    type_register_static(&bcm2836_type_info);
+}
+
+type_init(bcm2836_register_types)
diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 0014c34..1b913a4 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -7,10 +7,14 @@
  * This code is licensed under the GPL.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
+#include "hw/arm/linux-boot-if.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/numa.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "elf.h"
@@ -27,14 +31,15 @@
 #define KERNEL64_LOAD_ADDR 0x00080000
 
 typedef enum {
-    FIXUP_NONE = 0,   /* do nothing */
-    FIXUP_TERMINATOR, /* end of insns */
-    FIXUP_BOARDID,    /* overwrite with board ID number */
-    FIXUP_ARGPTR,     /* overwrite with pointer to kernel args */
-    FIXUP_ENTRYPOINT, /* overwrite with kernel entry point */
-    FIXUP_GIC_CPU_IF, /* overwrite with GIC CPU interface address */
-    FIXUP_BOOTREG,    /* overwrite with boot register address */
-    FIXUP_DSB,        /* overwrite with correct DSB insn for cpu */
+    FIXUP_NONE = 0,     /* do nothing */
+    FIXUP_TERMINATOR,   /* end of insns */
+    FIXUP_BOARDID,      /* overwrite with board ID number */
+    FIXUP_BOARD_SETUP,  /* overwrite with board specific setup code address */
+    FIXUP_ARGPTR,       /* overwrite with pointer to kernel args */
+    FIXUP_ENTRYPOINT,   /* overwrite with kernel entry point */
+    FIXUP_GIC_CPU_IF,   /* overwrite with GIC CPU interface address */
+    FIXUP_BOOTREG,      /* overwrite with boot register address */
+    FIXUP_DSB,          /* overwrite with correct DSB insn for cpu */
     FIXUP_MAX,
 } FixupType;
 
@@ -57,8 +62,17 @@
     { 0, FIXUP_TERMINATOR }
 };
 
-/* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
+/* A very small bootloader: call the board-setup code (if needed),
+ * set r0-r2, then jump to the kernel.
+ * If we're not calling boot setup code then we don't copy across
+ * the first BOOTLOADER_NO_BOARD_SETUP_OFFSET insns in this array.
+ */
+
 static const ARMInsnFixup bootloader[] = {
+    { 0xe28fe004 }, /* add     lr, pc, #4 */
+    { 0xe51ff004 }, /* ldr     pc, [pc, #-4] */
+    { 0, FIXUP_BOARD_SETUP },
+#define BOOTLOADER_NO_BOARD_SETUP_OFFSET 3
     { 0xe3a00000 }, /* mov     r0, #0 */
     { 0xe59f1004 }, /* ldr     r1, [pc, #4] */
     { 0xe59f2004 }, /* ldr     r2, [pc, #4] */
@@ -130,6 +144,7 @@
         case FIXUP_NONE:
             break;
         case FIXUP_BOARDID:
+        case FIXUP_BOARD_SETUP:
         case FIXUP_ARGPTR:
         case FIXUP_ENTRYPOINT:
         case FIXUP_GIC_CPU_IF:
@@ -165,13 +180,65 @@
                      smpboot, fixupcontext);
 }
 
+void arm_write_secure_board_setup_dummy_smc(ARMCPU *cpu,
+                                            const struct arm_boot_info *info,
+                                            hwaddr mvbar_addr)
+{
+    int n;
+    uint32_t mvbar_blob[] = {
+        /* mvbar_addr: secure monitor vectors
+         * Default unimplemented and unused vectors to spin. Makes it
+         * easier to debug (as opposed to the CPU running away).
+         */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xe1b0f00e, /* movs pc, lr ;SMC exception return */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+        0xeafffffe, /* (spin) */
+    };
+    uint32_t board_setup_blob[] = {
+        /* board setup addr */
+        0xe3a00e00 + (mvbar_addr >> 4), /* mov r0, #mvbar_addr */
+        0xee0c0f30, /* mcr     p15, 0, r0, c12, c0, 1 ;set MVBAR */
+        0xee110f11, /* mrc     p15, 0, r0, c1 , c1, 0 ;read SCR */
+        0xe3800031, /* orr     r0, #0x31              ;enable AW, FW, NS */
+        0xee010f11, /* mcr     p15, 0, r0, c1, c1, 0  ;write SCR */
+        0xe1a0100e, /* mov     r1, lr                 ;save LR across SMC */
+        0xe1600070, /* smc     #0                     ;call monitor to flush SCR */
+        0xe1a0f001, /* mov     pc, r1                 ;return */
+    };
+
+    /* check that mvbar_addr is correctly aligned and relocatable (using MOV) */
+    assert((mvbar_addr & 0x1f) == 0 && (mvbar_addr >> 4) < 0x100);
+
+    /* check that these blobs don't overlap */
+    assert((mvbar_addr + sizeof(mvbar_blob) <= info->board_setup_addr)
+          || (info->board_setup_addr + sizeof(board_setup_blob) <= mvbar_addr));
+
+    for (n = 0; n < ARRAY_SIZE(mvbar_blob); n++) {
+        mvbar_blob[n] = tswap32(mvbar_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup-mvbar", mvbar_blob, sizeof(mvbar_blob),
+                       mvbar_addr);
+
+    for (n = 0; n < ARRAY_SIZE(board_setup_blob); n++) {
+        board_setup_blob[n] = tswap32(board_setup_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup", board_setup_blob,
+                       sizeof(board_setup_blob), info->board_setup_addr);
+}
+
 static void default_reset_secondary(ARMCPU *cpu,
                                     const struct arm_boot_info *info)
 {
-    CPUARMState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
 
-    stl_phys_notdirty(&address_space_memory, info->smp_bootreg_addr, 0);
-    env->regs[15] = info->smp_loader_start;
+    address_space_stl_notdirty(&address_space_memory, info->smp_bootreg_addr,
+                               0, MEMTXATTRS_UNSPECIFIED, NULL);
+    cpu_set_pc(cs, info->smp_loader_start);
 }
 
 static inline bool have_dtb(const struct arm_boot_info *info)
@@ -180,7 +247,8 @@
 }
 
 #define WRITE_WORD(p, value) do { \
-    stl_phys_notdirty(&address_space_memory, p, value);  \
+    address_space_stl_notdirty(&address_space_memory, p, value, \
+                               MEMTXATTRS_UNSPECIFIED, NULL);  \
     p += 4;                       \
 } while (0)
 
@@ -329,6 +397,8 @@
  * Returns: the size of the device tree image on success,
  *          0 if the image size exceeds the limit,
  *          -1 on errors.
+ *
+ * Note: Must not be called unless have_dtb(binfo) is true.
  */
 static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo,
                     hwaddr addr_limit)
@@ -336,6 +406,9 @@
     void *fdt = NULL;
     int size, rc;
     uint32_t acells, scells;
+    char *nodename;
+    unsigned int i;
+    hwaddr mem_base, mem_len;
 
     if (binfo->dtb_filename) {
         char *filename;
@@ -352,7 +425,7 @@
             goto fail;
         }
         g_free(filename);
-    } else if (binfo->get_dtb) {
+    } else {
         fdt = binfo->get_dtb(binfo, &size);
         if (!fdt) {
             fprintf(stderr, "Board was unable to create a dtb blob\n");
@@ -369,8 +442,10 @@
         return 0;
     }
 
-    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells");
-    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells");
+    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells",
+                                   NULL, &error_fatal);
+    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells",
+                                   NULL, &error_fatal);
     if (acells == 0 || scells == 0) {
         fprintf(stderr, "dtb file invalid (#address-cells or #size-cells 0)\n");
         goto fail;
@@ -385,12 +460,39 @@
         goto fail;
     }
 
-    rc = qemu_fdt_setprop_sized_cells(fdt, "/memory", "reg",
-                                      acells, binfo->loader_start,
-                                      scells, binfo->ram_size);
-    if (rc < 0) {
-        fprintf(stderr, "couldn't set /memory/reg\n");
-        goto fail;
+    if (nb_numa_nodes > 0) {
+        /*
+         * Turn the /memory node created before into a NOP node, then create
+         * /memory@addr nodes for all numa nodes respectively.
+         */
+        qemu_fdt_nop_node(fdt, "/memory");
+        mem_base = binfo->loader_start;
+        for (i = 0; i < nb_numa_nodes; i++) {
+            mem_len = numa_info[i].node_mem;
+            nodename = g_strdup_printf("/memory@%" PRIx64, mem_base);
+            qemu_fdt_add_subnode(fdt, nodename);
+            qemu_fdt_setprop_string(fdt, nodename, "device_type", "memory");
+            rc = qemu_fdt_setprop_sized_cells(fdt, nodename, "reg",
+                                              acells, mem_base,
+                                              scells, mem_len);
+            if (rc < 0) {
+                fprintf(stderr, "couldn't set %s/reg for node %d\n", nodename,
+                        i);
+                goto fail;
+            }
+
+            qemu_fdt_setprop_cell(fdt, nodename, "numa-node-id", i);
+            mem_base += mem_len;
+            g_free(nodename);
+        }
+    } else {
+        rc = qemu_fdt_setprop_sized_cells(fdt, "/memory", "reg",
+                                          acells, binfo->loader_start,
+                                          scells, binfo->ram_size);
+        if (rc < 0) {
+            fprintf(stderr, "couldn't set /memory/reg\n");
+            goto fail;
+        }
     }
 
     if (binfo->kernel_cmdline && *binfo->kernel_cmdline) {
@@ -441,26 +543,80 @@
 static void do_cpu_reset(void *opaque)
 {
     ARMCPU *cpu = opaque;
+    CPUState *cs = CPU(cpu);
     CPUARMState *env = &cpu->env;
     const struct arm_boot_info *info = env->boot_info;
 
-    cpu_reset(CPU(cpu));
+    cpu_reset(cs);
     if (info) {
         if (!info->is_linux) {
+            int i;
             /* Jump to the entry point.  */
-            if (env->aarch64) {
-                env->pc = info->entry;
-            } else {
-                env->regs[15] = info->entry & 0xfffffffe;
-                env->thumb = info->entry & 1;
-            }
-        } else {
-            if (CPU(cpu) == first_cpu) {
-                if (env->aarch64) {
-                    env->pc = info->loader_start;
-                } else {
-                    env->regs[15] = info->loader_start;
+            uint64_t entry = info->entry;
+
+            switch (info->endianness) {
+            case ARM_ENDIANNESS_LE:
+                env->cp15.sctlr_el[1] &= ~SCTLR_E0E;
+                for (i = 1; i < 4; ++i) {
+                    env->cp15.sctlr_el[i] &= ~SCTLR_EE;
                 }
+                env->uncached_cpsr &= ~CPSR_E;
+                break;
+            case ARM_ENDIANNESS_BE8:
+                env->cp15.sctlr_el[1] |= SCTLR_E0E;
+                for (i = 1; i < 4; ++i) {
+                    env->cp15.sctlr_el[i] |= SCTLR_EE;
+                }
+                env->uncached_cpsr |= CPSR_E;
+                break;
+            case ARM_ENDIANNESS_BE32:
+                env->cp15.sctlr_el[1] |= SCTLR_B;
+                break;
+            case ARM_ENDIANNESS_UNKNOWN:
+                break; /* Board's decision */
+            default:
+                g_assert_not_reached();
+            }
+
+            if (!env->aarch64) {
+                env->thumb = info->entry & 1;
+                entry &= 0xfffffffe;
+            }
+            cpu_set_pc(cs, entry);
+        } else {
+            /* If we are booting Linux then we need to check whether we are
+             * booting into secure or non-secure state and adjust the state
+             * accordingly.  Out of reset, ARM is defined to be in secure state
+             * (SCR.NS = 0), we change that here if non-secure boot has been
+             * requested.
+             */
+            if (arm_feature(env, ARM_FEATURE_EL3)) {
+                /* AArch64 is defined to come out of reset into EL3 if enabled.
+                 * If we are booting Linux then we need to adjust our EL as
+                 * Linux expects us to be in EL2 or EL1.  AArch32 resets into
+                 * SVC, which Linux expects, so no privilege/exception level to
+                 * adjust.
+                 */
+                if (env->aarch64) {
+                    env->cp15.scr_el3 |= SCR_RW;
+                    if (arm_feature(env, ARM_FEATURE_EL2)) {
+                        env->cp15.hcr_el2 |= HCR_RW;
+                        env->pstate = PSTATE_MODE_EL2h;
+                    } else {
+                        env->pstate = PSTATE_MODE_EL1h;
+                    }
+                }
+
+                /* Set to non-secure if not a secure boot */
+                if (!info->secure_boot &&
+                    (cs != first_cpu || !info->secure_board_setup)) {
+                    /* Linux expects non-secure state */
+                    env->cp15.scr_el3 |= SCR_NS;
+                }
+            }
+
+            if (cs == first_cpu) {
+                cpu_set_pc(cs, info->loader_start);
 
                 if (!have_dtb(info)) {
                     if (old_param) {
@@ -476,7 +632,126 @@
     }
 }
 
-void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
+/**
+ * load_image_to_fw_cfg() - Load an image file into an fw_cfg entry identified
+ *                          by key.
+ * @fw_cfg:         The firmware config instance to store the data in.
+ * @size_key:       The firmware config key to store the size of the loaded
+ *                  data under, with fw_cfg_add_i32().
+ * @data_key:       The firmware config key to store the loaded data under,
+ *                  with fw_cfg_add_bytes().
+ * @image_name:     The name of the image file to load. If it is NULL, the
+ *                  function returns without doing anything.
+ * @try_decompress: Whether the image should be decompressed (gunzipped) before
+ *                  adding it to fw_cfg. If decompression fails, the image is
+ *                  loaded as-is.
+ *
+ * In case of failure, the function prints an error message to stderr and the
+ * process exits with status 1.
+ */
+static void load_image_to_fw_cfg(FWCfgState *fw_cfg, uint16_t size_key,
+                                 uint16_t data_key, const char *image_name,
+                                 bool try_decompress)
+{
+    size_t size = -1;
+    uint8_t *data;
+
+    if (image_name == NULL) {
+        return;
+    }
+
+    if (try_decompress) {
+        size = load_image_gzipped_buffer(image_name,
+                                         LOAD_IMAGE_MAX_GUNZIP_BYTES, &data);
+    }
+
+    if (size == (size_t)-1) {
+        gchar *contents;
+        gsize length;
+
+        if (!g_file_get_contents(image_name, &contents, &length, NULL)) {
+            fprintf(stderr, "failed to load \"%s\"\n", image_name);
+            exit(1);
+        }
+        size = length;
+        data = (uint8_t *)contents;
+    }
+
+    fw_cfg_add_i32(fw_cfg, size_key, size);
+    fw_cfg_add_bytes(fw_cfg, data_key, data, size);
+}
+
+static int do_arm_linux_init(Object *obj, void *opaque)
+{
+    if (object_dynamic_cast(obj, TYPE_ARM_LINUX_BOOT_IF)) {
+        ARMLinuxBootIf *albif = ARM_LINUX_BOOT_IF(obj);
+        ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_GET_CLASS(obj);
+        struct arm_boot_info *info = opaque;
+
+        if (albifc->arm_linux_init) {
+            albifc->arm_linux_init(albif, info->secure_boot);
+        }
+    }
+    return 0;
+}
+
+static uint64_t arm_load_elf(struct arm_boot_info *info, uint64_t *pentry,
+                             uint64_t *lowaddr, uint64_t *highaddr,
+                             int elf_machine)
+{
+    bool elf_is64;
+    union {
+        Elf32_Ehdr h32;
+        Elf64_Ehdr h64;
+    } elf_header;
+    int data_swab = 0;
+    bool big_endian;
+    uint64_t ret = -1;
+    Error *err = NULL;
+
+
+    load_elf_hdr(info->kernel_filename, &elf_header, &elf_is64, &err);
+    if (err) {
+        return ret;
+    }
+
+    if (elf_is64) {
+        big_endian = elf_header.h64.e_ident[EI_DATA] == ELFDATA2MSB;
+        info->endianness = big_endian ? ARM_ENDIANNESS_BE8
+                                      : ARM_ENDIANNESS_LE;
+    } else {
+        big_endian = elf_header.h32.e_ident[EI_DATA] == ELFDATA2MSB;
+        if (big_endian) {
+            if (bswap32(elf_header.h32.e_flags) & EF_ARM_BE8) {
+                info->endianness = ARM_ENDIANNESS_BE8;
+            } else {
+                info->endianness = ARM_ENDIANNESS_BE32;
+                /* In BE32, the CPU has a different view of the per-byte
+                 * address map than the rest of the system. BE32 ELF files
+                 * are organised such that they can be programmed through
+                 * the CPU's per-word byte-reversed view of the world. QEMU
+                 * however loads ELF files independently of the CPU. So
+                 * tell the ELF loader to byte reverse the data for us.
+                 */
+                data_swab = 2;
+            }
+        } else {
+            info->endianness = ARM_ENDIANNESS_LE;
+        }
+    }
+
+    ret = load_elf(info->kernel_filename, NULL, NULL,
+                   pentry, lowaddr, highaddr, big_endian, elf_machine,
+                   1, data_swab);
+    if (ret <= 0) {
+        /* The header loaded but the image didn't */
+        exit(1);
+    }
+
+    return ret;
+}
+
+static void arm_load_kernel_notify(Notifier *notifier, void *data)
 {
     CPUState *cs;
     int kernel_size;
@@ -485,32 +760,62 @@
     uint64_t elf_entry, elf_low_addr, elf_high_addr;
     int elf_machine;
     hwaddr entry, kernel_load_offset;
-    int big_endian;
     static const ARMInsnFixup *primary_loader;
+    ArmLoadKernelNotifier *n = DO_UPCAST(ArmLoadKernelNotifier,
+                                         notifier, notifier);
+    ARMCPU *cpu = n->cpu;
+    struct arm_boot_info *info =
+        container_of(n, struct arm_boot_info, load_kernel_notifier);
 
-    /* CPU objects (unlike devices) are not automatically reset on system
-     * reset, so we must always register a handler to do so. If we're
-     * actually loading a kernel, the handler is also responsible for
-     * arranging that we start it correctly.
+    /* The board code is not supposed to set secure_board_setup unless
+     * running its code in secure mode is actually possible, and KVM
+     * doesn't support secure.
      */
-    for (cs = CPU(cpu); cs; cs = CPU_NEXT(cs)) {
-        qemu_register_reset(do_cpu_reset, ARM_CPU(cs));
-    }
+    assert(!(info->secure_board_setup && kvm_enabled()));
 
     /* Load the kernel.  */
-    if (!info->kernel_filename) {
+    if (!info->kernel_filename || info->firmware_loaded) {
 
         if (have_dtb(info)) {
-            /* If we have a device tree blob, but no kernel to supply it to,
-             * copy it to the base of RAM for a bootloader to pick up.
+            /* If we have a device tree blob, but no kernel to supply it to (or
+             * the kernel is supposed to be loaded by the bootloader), copy the
+             * DTB to the base of RAM for the bootloader to pick up.
              */
             if (load_dtb(info->loader_start, info, 0) < 0) {
                 exit(1);
             }
         }
 
-        /* If no kernel specified, do nothing; we will start from address 0
-         * (typically a boot ROM image) in the same way as hardware.
+        if (info->kernel_filename) {
+            FWCfgState *fw_cfg;
+            bool try_decompressing_kernel;
+
+            fw_cfg = fw_cfg_find();
+            try_decompressing_kernel = arm_feature(&cpu->env,
+                                                   ARM_FEATURE_AARCH64);
+
+            /* Expose the kernel, the command line, and the initrd in fw_cfg.
+             * We don't process them here at all, it's all left to the
+             * firmware.
+             */
+            load_image_to_fw_cfg(fw_cfg,
+                                 FW_CFG_KERNEL_SIZE, FW_CFG_KERNEL_DATA,
+                                 info->kernel_filename,
+                                 try_decompressing_kernel);
+            load_image_to_fw_cfg(fw_cfg,
+                                 FW_CFG_INITRD_SIZE, FW_CFG_INITRD_DATA,
+                                 info->initrd_filename, false);
+
+            if (info->kernel_cmdline) {
+                fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
+                               strlen(info->kernel_cmdline) + 1);
+                fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA,
+                                  info->kernel_cmdline);
+            }
+        }
+
+        /* We will start from address 0 (typically a boot ROM image) in the
+         * same way as hardware.
          */
         return;
     }
@@ -521,6 +826,9 @@
         elf_machine = EM_AARCH64;
     } else {
         primary_loader = bootloader;
+        if (!info->write_board_setup) {
+            primary_loader += BOOTLOADER_NO_BOARD_SETUP_OFFSET;
+        }
         kernel_load_offset = KERNEL_LOAD_ADDR;
         elf_machine = EM_ARM;
     }
@@ -537,12 +845,6 @@
     if (info->nb_cpus == 0)
         info->nb_cpus = 1;
 
-#ifdef TARGET_WORDS_BIGENDIAN
-    big_endian = 1;
-#else
-    big_endian = 0;
-#endif
-
     /* We want to put the initrd far enough into RAM that when the
      * kernel is uncompressed it will not clobber the initrd. However
      * on boards without much RAM we must ensure that we still leave
@@ -557,9 +859,8 @@
         MIN(info->ram_size / 2, 128 * 1024 * 1024);
 
     /* Assume that raw images are linux kernels, and ELF images are not.  */
-    kernel_size = load_elf(info->kernel_filename, NULL, NULL, &elf_entry,
-                           &elf_low_addr, &elf_high_addr, big_endian,
-                           elf_machine, 1);
+    kernel_size = arm_load_elf(info, &elf_entry, &elf_low_addr,
+                               &elf_high_addr, elf_machine);
     if (kernel_size > 0 && have_dtb(info)) {
         /* If there is still some room left at the base of RAM, try and put
          * the DTB there like we do for images loaded with -bios or -pflash.
@@ -626,17 +927,34 @@
         info->initrd_size = initrd_size;
 
         fixupcontext[FIXUP_BOARDID] = info->board_id;
+        fixupcontext[FIXUP_BOARD_SETUP] = info->board_setup_addr;
 
         /* for device tree boot, we pass the DTB directly in r2. Otherwise
          * we point to the kernel args.
          */
         if (have_dtb(info)) {
-            /* Place the DTB after the initrd in memory. Note that some
-             * kernels will trash anything in the 4K page the initrd
-             * ends in, so make sure the DTB isn't caught up in that.
-             */
-            hwaddr dtb_start = QEMU_ALIGN_UP(info->initrd_start + initrd_size,
-                                             4096);
+            hwaddr align;
+            hwaddr dtb_start;
+
+            if (elf_machine == EM_AARCH64) {
+                /*
+                 * Some AArch64 kernels on early bootup map the fdt region as
+                 *
+                 *   [ ALIGN_DOWN(fdt, 2MB) ... ALIGN_DOWN(fdt, 2MB) + 2MB ]
+                 *
+                 * Let's play safe and prealign it to 2MB to give us some space.
+                 */
+                align = 2 * 1024 * 1024;
+            } else {
+                /*
+                 * Some 32bit kernels will trash anything in the 4K page the
+                 * initrd ends in, so make sure the DTB isn't caught up in that.
+                 */
+                align = 4096;
+            }
+
+            /* Place the DTB after the initrd in memory with alignment. */
+            dtb_start = QEMU_ALIGN_UP(info->initrd_start + initrd_size, align);
             if (load_dtb(dtb_start, info, 0) < 0) {
                 exit(1);
             }
@@ -658,6 +976,15 @@
         if (info->nb_cpus > 1) {
             info->write_secondary_boot(cpu, info);
         }
+        if (info->write_board_setup) {
+            info->write_board_setup(cpu, info);
+        }
+
+        /* Notify devices which need to fake up firmware initialization
+         * that we're doing a direct kernel boot.
+         */
+        object_child_foreach_recursive(object_get_root(),
+                                       do_arm_linux_init, info);
     }
     info->is_linux = is_linux;
 
@@ -665,3 +992,34 @@
         ARM_CPU(cs)->env.boot_info = info;
     }
 }
+
+void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
+{
+    CPUState *cs;
+
+    info->load_kernel_notifier.cpu = cpu;
+    info->load_kernel_notifier.notifier.notify = arm_load_kernel_notify;
+    qemu_add_machine_init_done_notifier(&info->load_kernel_notifier.notifier);
+
+    /* CPU objects (unlike devices) are not automatically reset on system
+     * reset, so we must always register a handler to do so. If we're
+     * actually loading a kernel, the handler is also responsible for
+     * arranging that we start it correctly.
+     */
+    for (cs = CPU(cpu); cs; cs = CPU_NEXT(cs)) {
+        qemu_register_reset(do_cpu_reset, ARM_CPU(cs));
+    }
+}
+
+static const TypeInfo arm_linux_boot_if_info = {
+    .name = TYPE_ARM_LINUX_BOOT_IF,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(ARMLinuxBootIfClass),
+};
+
+static void arm_linux_boot_register_types(void)
+{
+    type_register_static(&arm_linux_boot_if_info);
+}
+
+type_init(arm_linux_boot_register_types)
diff --git a/hw/arm/collie.c b/hw/arm/collie.c
index 6c9b82f..2e69531 100644
--- a/hw/arm/collie.c
+++ b/hw/arm/collie.c
@@ -8,6 +8,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/boards.h"
@@ -17,6 +18,7 @@
 #include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
+#include "qom/cpu.h"
 
 static struct arm_boot_info collie_binfo = {
     .loader_start = SA_SDCS0,
@@ -58,15 +60,10 @@
     arm_load_kernel(s->cpu, &collie_binfo);
 }
 
-static QEMUMachine collie_machine = {
-    .name = "collie",
-    .desc = "Collie PDA (SA-1110)",
-    .init = collie_init,
-};
-
-static void collie_machine_init(void)
+static void collie_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&collie_machine);
+    mc->desc = "Sharp SL-5500 (Collie) PDA (SA-1110)";
+    mc->init = collie_init;
 }
 
-machine_init(collie_machine_init)
+DEFINE_MACHINE("collie", collie_machine_init)
diff --git a/hw/arm/cubieboard.c b/hw/arm/cubieboard.c
index d1e53be..fbd78ed 100644
--- a/hw/arm/cubieboard.c
+++ b/hw/arm/cubieboard.c
@@ -15,6 +15,10 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
@@ -39,33 +43,31 @@
 
     object_property_set_int(OBJECT(&s->a10->emac), 1, "phy-addr", &err);
     if (err != NULL) {
-        error_report("Couldn't set phy address: %s", error_get_pretty(err));
+        error_reportf_err(err, "Couldn't set phy address: ");
         exit(1);
     }
 
     object_property_set_int(OBJECT(&s->a10->timer), 32768, "clk0-freq", &err);
     if (err != NULL) {
-        error_report("Couldn't set clk0 frequency: %s", error_get_pretty(err));
+        error_reportf_err(err, "Couldn't set clk0 frequency: ");
         exit(1);
     }
 
     object_property_set_int(OBJECT(&s->a10->timer), 24000000, "clk1-freq",
                             &err);
     if (err != NULL) {
-        error_report("Couldn't set clk1 frequency: %s", error_get_pretty(err));
+        error_reportf_err(err, "Couldn't set clk1 frequency: ");
         exit(1);
     }
 
     object_property_set_bool(OBJECT(s->a10), true, "realized", &err);
     if (err != NULL) {
-        error_report("Couldn't realize Allwinner A10: %s",
-                     error_get_pretty(err));
+        error_reportf_err(err, "Couldn't realize Allwinner A10: ");
         exit(1);
     }
 
-    memory_region_init_ram(&s->sdram, NULL, "cubieboard.ram",
-                           machine->ram_size, &error_abort);
-    vmstate_register_ram_global(&s->sdram);
+    memory_region_allocate_system_memory(&s->sdram, NULL, "cubieboard.ram",
+                                         machine->ram_size);
     memory_region_add_subregion(get_system_memory(), AW_A10_SDRAM_BASE,
                                 &s->sdram);
 
@@ -75,16 +77,10 @@
     arm_load_kernel(&s->a10->cpu, &cubieboard_binfo);
 }
 
-static QEMUMachine cubieboard_machine = {
-    .name = "cubieboard",
-    .desc = "cubietech cubieboard",
-    .init = cubieboard_init,
-};
-
-
-static void cubieboard_machine_init(void)
+static void cubieboard_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&cubieboard_machine);
+    mc->desc = "cubietech cubieboard";
+    mc->init = cubieboard_init;
 }
 
-machine_init(cubieboard_machine_init)
+DEFINE_MACHINE("cubieboard", cubieboard_machine_init)
diff --git a/hw/arm/digic.c b/hw/arm/digic.c
index ec8c330..d60ea39 100644
--- a/hw/arm/digic.c
+++ b/hw/arm/digic.c
@@ -20,7 +20,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/arm/digic.h"
+#include "sysemu/sysemu.h"
 
 #define DIGIC4_TIMER_BASE(n)    (0xc0210000 + (n) * 0x100)
 
@@ -82,6 +85,7 @@
         sysbus_mmio_map(sbd, 0, DIGIC4_TIMER_BASE(i));
     }
 
+    qdev_prop_set_chr(DEVICE(&s->uart), "chardev", serial_hds[0]);
     object_property_set_bool(OBJECT(&s->uart), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
@@ -97,6 +101,12 @@
     DeviceClass *dc = DEVICE_CLASS(oc);
 
     dc->realize = digic_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo digic_type_info = {
diff --git a/hw/arm/digic_boards.c b/hw/arm/digic_boards.c
index 2a4b872..520c8e9 100644
--- a/hw/arm/digic_boards.c
+++ b/hw/arm/digic_boards.c
@@ -23,6 +23,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
@@ -51,9 +55,8 @@
 
 static void digic4_board_setup_ram(DigicBoardState *s, hwaddr ram_size)
 {
-    memory_region_init_ram(&s->ram, NULL, "ram", ram_size, &error_abort);
+    memory_region_allocate_system_memory(&s->ram, NULL, "ram", ram_size);
     memory_region_add_subregion(get_system_memory(), 0, &s->ram);
-    vmstate_register_ram_global(&s->ram);
 }
 
 static void digic4_board_init(DigicBoard *board)
@@ -65,8 +68,7 @@
     s->digic = DIGIC(object_new(TYPE_DIGIC));
     object_property_set_bool(OBJECT(s->digic), true, "realized", &err);
     if (err != NULL) {
-        error_report("Couldn't realize DIGIC SoC: %s\n",
-                     error_get_pretty(err));
+        error_reportf_err(err, "Couldn't realize DIGIC SoC: ");
         exit(1);
     }
 
@@ -104,15 +106,16 @@
         char *fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, filename);
 
         if (!fn) {
-            error_report("Couldn't find rom image '%s'.\n", filename);
+            error_report("Couldn't find rom image '%s'.", filename);
             exit(1);
         }
 
         rom_size = load_image_targphys(fn, addr, max_size);
         if (rom_size < 0 || rom_size > max_size) {
-            error_report("Couldn't load rom image '%s'.\n", filename);
+            error_report("Couldn't load rom image '%s'.", filename);
             exit(1);
         }
+        g_free(fn);
     }
 }
 
@@ -148,15 +151,10 @@
     digic4_board_init(&digic4_board_canon_a1100);
 }
 
-static QEMUMachine canon_a1100 = {
-    .name = "canon-a1100",
-    .desc = "Canon PowerShot A1100 IS",
-    .init = &canon_a1100_init,
-};
-
-static void digic_register_machines(void)
+static void canon_a1100_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&canon_a1100);
+    mc->desc = "Canon PowerShot A1100 IS";
+    mc->init = &canon_a1100_init;
 }
 
-machine_init(digic_register_machines)
+DEFINE_MACHINE("canon-a1100", canon_a1100_machine_init)
diff --git a/hw/arm/exynos4210.c b/hw/arm/exynos4210.c
index 582794c..be3c96d 100644
--- a/hw/arm/exynos4210.c
+++ b/hw/arm/exynos4210.c
@@ -21,6 +21,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
@@ -150,16 +154,18 @@
 
     for (n = 0; n < EXYNOS4210_NCPUS; n++) {
         Object *cpuobj = object_new(object_class_get_name(cpu_oc));
-        Error *err = NULL;
+
+        /* By default A9 CPUs have EL3 enabled.  This board does not currently
+         * support EL3 so the CPU EL3 property is disabled before realization.
+         */
+        if (object_property_find(cpuobj, "has_el3", NULL)) {
+            object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
+        }
 
         s->cpu[n] = ARM_CPU(cpuobj);
         object_property_set_int(cpuobj, EXYNOS4210_SMP_PRIVATE_BASE_ADDR,
                                 "reset-cbar", &error_abort);
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report("%s", error_get_pretty(err));
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
     }
 
     /*** IRQs ***/
@@ -248,7 +254,7 @@
 
     /* Internal ROM */
     memory_region_init_ram(&s->irom_mem, NULL, "exynos4210.irom",
-                           EXYNOS4210_IROM_SIZE, &error_abort);
+                           EXYNOS4210_IROM_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->irom_mem);
     memory_region_set_readonly(&s->irom_mem, true);
     memory_region_add_subregion(system_mem, EXYNOS4210_IROM_BASE_ADDR,
@@ -264,7 +270,7 @@
 
     /* Internal RAM */
     memory_region_init_ram(&s->iram_mem, NULL, "exynos4210.iram",
-                           EXYNOS4210_IRAM_SIZE, &error_abort);
+                           EXYNOS4210_IRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->iram_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_IRAM_BASE_ADDR,
                                 &s->iram_mem);
@@ -273,14 +279,14 @@
     mem_size = ram_size;
     if (mem_size > EXYNOS4210_DRAM_MAX_SIZE) {
         memory_region_init_ram(&s->dram1_mem, NULL, "exynos4210.dram1",
-                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_abort);
+                mem_size - EXYNOS4210_DRAM_MAX_SIZE, &error_fatal);
         vmstate_register_ram_global(&s->dram1_mem);
         memory_region_add_subregion(system_mem, EXYNOS4210_DRAM1_BASE_ADDR,
                 &s->dram1_mem);
         mem_size = EXYNOS4210_DRAM_MAX_SIZE;
     }
     memory_region_init_ram(&s->dram0_mem, NULL, "exynos4210.dram0", mem_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->dram0_mem);
     memory_region_add_subregion(system_mem, EXYNOS4210_DRAM0_BASE_ADDR,
             &s->dram0_mem);
diff --git a/hw/arm/exynos4_boards.c b/hw/arm/exynos4_boards.c
index d644db1..0efa194 100644
--- a/hw/arm/exynos4_boards.c
+++ b/hw/arm/exynos4_boards.c
@@ -21,6 +21,9 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/qtest.h"
 #include "hw/sysbus.h"
@@ -74,8 +77,6 @@
     .write_secondary_boot = exynos4210_write_secondary,
 };
 
-static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS];
-
 static void lan9215_init(uint32_t base, qemu_irq irq)
 {
     DeviceState *dev;
@@ -97,11 +98,12 @@
 static Exynos4210State *exynos4_boards_init_common(MachineState *machine,
                                                    Exynos4BoardType board_type)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+
     if (smp_cpus != EXYNOS4210_NCPUS && !qtest_enabled()) {
         fprintf(stderr, "%s board supports only %d CPU cores. Ignoring smp_cpus"
                 " value.\n",
-                exynos4_machines[board_type].name,
-                exynos4_machines[board_type].max_cpus);
+                mc->name, EXYNOS4210_NCPUS);
     }
 
     exynos4_board_binfo.ram_size = exynos4_board_ram_size[board_type];
@@ -145,25 +147,40 @@
     arm_load_kernel(ARM_CPU(first_cpu), &exynos4_board_binfo);
 }
 
-static QEMUMachine exynos4_machines[EXYNOS4_NUM_OF_BOARDS] = {
-    [EXYNOS4_BOARD_NURI] = {
-        .name = "nuri",
-        .desc = "Samsung NURI board (Exynos4210)",
-        .init = nuri_init,
-        .max_cpus = EXYNOS4210_NCPUS,
-    },
-    [EXYNOS4_BOARD_SMDKC210] = {
-        .name = "smdkc210",
-        .desc = "Samsung SMDKC210 board (Exynos4210)",
-        .init = smdkc210_init,
-        .max_cpus = EXYNOS4210_NCPUS,
-    },
-};
-
-static void exynos4_machine_init(void)
+static void nuri_class_init(ObjectClass *oc, void *data)
 {
-    qemu_register_machine(&exynos4_machines[EXYNOS4_BOARD_NURI]);
-    qemu_register_machine(&exynos4_machines[EXYNOS4_BOARD_SMDKC210]);
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Samsung NURI board (Exynos4210)";
+    mc->init = nuri_init;
+    mc->max_cpus = EXYNOS4210_NCPUS;
 }
 
-machine_init(exynos4_machine_init);
+static const TypeInfo nuri_type = {
+    .name = MACHINE_TYPE_NAME("nuri"),
+    .parent = TYPE_MACHINE,
+    .class_init = nuri_class_init,
+};
+
+static void smdkc210_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Samsung SMDKC210 board (Exynos4210)";
+    mc->init = smdkc210_init;
+    mc->max_cpus = EXYNOS4210_NCPUS;
+}
+
+static const TypeInfo smdkc210_type = {
+    .name = MACHINE_TYPE_NAME("smdkc210"),
+    .parent = TYPE_MACHINE,
+    .class_init = smdkc210_class_init,
+};
+
+static void exynos4_machines_init(void)
+{
+    type_register_static(&nuri_type);
+    type_register_static(&smdkc210_type);
+}
+
+type_init(exynos4_machines_init)
diff --git a/hw/arm/fsl-imx25.c b/hw/arm/fsl-imx25.c
new file mode 100644
index 0000000..b4e358d
--- /dev/null
+++ b/hw/arm/fsl-imx25.c
@@ -0,0 +1,314 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX25 SOC emulation.
+ *
+ * Based on hw/arm/xlnx-zynqmp.c
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx25.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "sysemu/char.h"
+
+static void fsl_imx25_init(Object *obj)
+{
+    FslIMX25State *s = FSL_IMX25(obj);
+    int i;
+
+    object_initialize(&s->cpu, sizeof(s->cpu), "arm926-" TYPE_ARM_CPU);
+
+    object_initialize(&s->avic, sizeof(s->avic), TYPE_IMX_AVIC);
+    qdev_set_parent_bus(DEVICE(&s->avic), sysbus_get_default());
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX25_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX25_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_GPTS; i++) {
+        object_initialize(&s->gpt[i], sizeof(s->gpt[i]), TYPE_IMX25_GPT);
+        qdev_set_parent_bus(DEVICE(&s->gpt[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->fec, sizeof(s->fec), TYPE_IMX_FEC);
+    qdev_set_parent_bus(DEVICE(&s->fec), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX25_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX25_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+    }
+}
+
+static void fsl_imx25_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX25State *s = FSL_IMX25(dev);
+    uint8_t i;
+    Error *err = NULL;
+
+    object_property_set_bool(OBJECT(&s->cpu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->avic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->avic), 0, FSL_IMX25_AVIC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 0,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 1,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_FIQ));
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX25_CCM_ADDR);
+
+    /* Initialize all UARTs */
+    for (i = 0; i < FSL_IMX25_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX25_NUM_UARTS] = {
+            { FSL_IMX25_UART1_ADDR, FSL_IMX25_UART1_IRQ },
+            { FSL_IMX25_UART2_ADDR, FSL_IMX25_UART2_IRQ },
+            { FSL_IMX25_UART3_ADDR, FSL_IMX25_UART3_IRQ },
+            { FSL_IMX25_UART4_ADDR, FSL_IMX25_UART4_IRQ },
+            { FSL_IMX25_UART5_ADDR, FSL_IMX25_UART5_IRQ }
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char label[20];
+                snprintf(label, sizeof(label), "imx31.uart%d", i);
+                chr = qemu_chr_new(label, "null", NULL);
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            serial_table[i].irq));
+    }
+
+    /* Initialize all GPT timers */
+    for (i = 0; i < FSL_IMX25_NUM_GPTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpt_table[FSL_IMX25_NUM_GPTS] = {
+            { FSL_IMX25_GPT1_ADDR, FSL_IMX25_GPT1_IRQ },
+            { FSL_IMX25_GPT2_ADDR, FSL_IMX25_GPT2_IRQ },
+            { FSL_IMX25_GPT3_ADDR, FSL_IMX25_GPT3_IRQ },
+            { FSL_IMX25_GPT4_ADDR, FSL_IMX25_GPT4_IRQ }
+        };
+
+        s->gpt[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->gpt[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt[i]), 0, gpt_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpt_table[i].irq));
+    }
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX25_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX25_NUM_EPITS] = {
+            { FSL_IMX25_EPIT1_ADDR, FSL_IMX25_EPIT1_IRQ },
+            { FSL_IMX25_EPIT2_ADDR, FSL_IMX25_EPIT2_IRQ }
+        };
+
+        s->epit[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            epit_table[i].irq));
+    }
+
+    qdev_set_nic_properties(DEVICE(&s->fec), &nd_table[0]);
+
+    object_property_set_bool(OBJECT(&s->fec), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->fec), 0, FSL_IMX25_FEC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->fec), 0,
+                       qdev_get_gpio_in(DEVICE(&s->avic), FSL_IMX25_FEC_IRQ));
+
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX25_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX25_NUM_I2CS] = {
+            { FSL_IMX25_I2C1_ADDR, FSL_IMX25_I2C1_IRQ },
+            { FSL_IMX25_I2C2_ADDR, FSL_IMX25_I2C2_IRQ },
+            { FSL_IMX25_I2C3_ADDR, FSL_IMX25_I2C3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX25_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpio_table[FSL_IMX25_NUM_GPIOS] = {
+            { FSL_IMX25_GPIO1_ADDR, FSL_IMX25_GPIO1_IRQ },
+            { FSL_IMX25_GPIO2_ADDR, FSL_IMX25_GPIO2_IRQ },
+            { FSL_IMX25_GPIO3_ADDR, FSL_IMX25_GPIO3_IRQ },
+            { FSL_IMX25_GPIO4_ADDR, FSL_IMX25_GPIO4_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        /* Connect GPIO IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpio_table[i].irq));
+    }
+
+    /* initialize 2 x 16 KB ROM */
+    memory_region_init_rom(&s->rom[0], NULL,
+                           "imx25.rom0", FSL_IMX25_ROM0_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_ROM0_ADDR,
+                                &s->rom[0]);
+    memory_region_init_rom(&s->rom[1], NULL,
+                           "imx25.rom1", FSL_IMX25_ROM1_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_ROM1_ADDR,
+                                &s->rom[1]);
+
+    /* initialize internal RAM (128 KB) */
+    memory_region_init_ram(&s->iram, NULL, "imx25.iram", FSL_IMX25_IRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_IRAM_ADDR,
+                                &s->iram);
+    vmstate_register_ram_global(&s->iram);
+
+    /* internal RAM (128 KB) is aliased over 128 MB - 128 KB */
+    memory_region_init_alias(&s->iram_alias, NULL, "imx25.iram_alias",
+                             &s->iram, 0, FSL_IMX25_IRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_IRAM_ALIAS_ADDR,
+                                &s->iram_alias);
+}
+
+static void fsl_imx25_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx25_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+    dc->desc = "i.MX25 SOC";
+}
+
+static const TypeInfo fsl_imx25_type_info = {
+    .name = TYPE_FSL_IMX25,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX25State),
+    .instance_init = fsl_imx25_init,
+    .class_init = fsl_imx25_class_init,
+};
+
+static void fsl_imx25_register_types(void)
+{
+    type_register_static(&fsl_imx25_type_info);
+}
+
+type_init(fsl_imx25_register_types)
diff --git a/hw/arm/fsl-imx31.c b/hw/arm/fsl-imx31.c
new file mode 100644
index 0000000..fe204ac
--- /dev/null
+++ b/hw/arm/fsl-imx31.c
@@ -0,0 +1,286 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX31 SOC emulation.
+ *
+ * Based on hw/arm/fsl-imx31.c
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx31.h"
+#include "sysemu/sysemu.h"
+#include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "sysemu/char.h"
+
+static void fsl_imx31_init(Object *obj)
+{
+    FslIMX31State *s = FSL_IMX31(obj);
+    int i;
+
+    object_initialize(&s->cpu, sizeof(s->cpu), "arm1136-" TYPE_ARM_CPU);
+
+    object_initialize(&s->avic, sizeof(s->avic), TYPE_IMX_AVIC);
+    qdev_set_parent_bus(DEVICE(&s->avic), sysbus_get_default());
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX31_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX31_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->gpt, sizeof(s->gpt), TYPE_IMX31_GPT);
+    qdev_set_parent_bus(DEVICE(&s->gpt), sysbus_get_default());
+
+    for (i = 0; i < FSL_IMX31_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX31_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < FSL_IMX31_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+    }
+}
+
+static void fsl_imx31_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX31State *s = FSL_IMX31(dev);
+    uint16_t i;
+    Error *err = NULL;
+
+    object_property_set_bool(OBJECT(&s->cpu), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    object_property_set_bool(OBJECT(&s->avic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->avic), 0, FSL_IMX31_AVIC_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 0,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->avic), 1,
+                       qdev_get_gpio_in(DEVICE(&s->cpu), ARM_CPU_FIQ));
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX31_CCM_ADDR);
+
+    /* Initialize all UARTS */
+    for (i = 0; i < FSL_IMX31_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX31_NUM_UARTS] = {
+            { FSL_IMX31_UART1_ADDR, FSL_IMX31_UART1_IRQ },
+            { FSL_IMX31_UART2_ADDR, FSL_IMX31_UART2_IRQ },
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char label[20];
+                snprintf(label, sizeof(label), "imx31.uart%d", i);
+                chr = qemu_chr_new(label, "null", NULL);
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            serial_table[i].irq));
+    }
+
+    s->gpt.ccm = IMX_CCM(&s->ccm);
+
+    object_property_set_bool(OBJECT(&s->gpt), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt), 0, FSL_IMX31_GPT_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt), 0,
+                       qdev_get_gpio_in(DEVICE(&s->avic), FSL_IMX31_GPT_IRQ));
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX31_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX31_NUM_EPITS] = {
+            { FSL_IMX31_EPIT1_ADDR, FSL_IMX31_EPIT1_IRQ },
+            { FSL_IMX31_EPIT2_ADDR, FSL_IMX31_EPIT2_IRQ },
+        };
+
+        s->epit[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            epit_table[i].irq));
+    }
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX31_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX31_NUM_I2CS] = {
+            { FSL_IMX31_I2C1_ADDR, FSL_IMX31_I2C1_IRQ },
+            { FSL_IMX31_I2C2_ADDR, FSL_IMX31_I2C2_IRQ },
+            { FSL_IMX31_I2C3_ADDR, FSL_IMX31_I2C3_IRQ }
+        };
+
+        /* Initialize the I2C */
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        /* Map I2C memory */
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        /* Connect I2C IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX31_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } gpio_table[FSL_IMX31_NUM_GPIOS] = {
+            { FSL_IMX31_GPIO1_ADDR, FSL_IMX31_GPIO1_IRQ },
+            { FSL_IMX31_GPIO2_ADDR, FSL_IMX31_GPIO2_IRQ },
+            { FSL_IMX31_GPIO3_ADDR, FSL_IMX31_GPIO3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), false, "has-edge-sel",
+                                 &error_abort);
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        /* Connect GPIO IRQ to PIC */
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->avic),
+                                            gpio_table[i].irq));
+    }
+
+    /* On a real system, the first 16k is a `secure boot rom' */
+    memory_region_init_rom(&s->secure_rom, NULL, "imx31.secure_rom",
+                           FSL_IMX31_SECURE_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_SECURE_ROM_ADDR,
+                                &s->secure_rom);
+
+    /* There is also a 16k ROM */
+    memory_region_init_rom(&s->rom, NULL, "imx31.rom",
+                           FSL_IMX31_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_ROM_ADDR,
+                                &s->rom);
+
+    /* initialize internal RAM (16 KB) */
+    memory_region_init_ram(&s->iram, NULL, "imx31.iram", FSL_IMX31_IRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_IRAM_ADDR,
+                                &s->iram);
+    vmstate_register_ram_global(&s->iram);
+
+    /* internal RAM (16 KB) is aliased over 256 MB - 16 KB */
+    memory_region_init_alias(&s->iram_alias, NULL, "imx31.iram_alias",
+                             &s->iram, 0, FSL_IMX31_IRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_IRAM_ALIAS_ADDR,
+                                &s->iram_alias);
+}
+
+static void fsl_imx31_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx31_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+    dc->desc = "i.MX31 SOC";
+}
+
+static const TypeInfo fsl_imx31_type_info = {
+    .name = TYPE_FSL_IMX31,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX31State),
+    .instance_init = fsl_imx31_init,
+    .class_init = fsl_imx31_class_init,
+};
+
+static void fsl_imx31_register_types(void)
+{
+    type_register_static(&fsl_imx31_type_info);
+}
+
+type_init(fsl_imx31_register_types)
diff --git a/hw/arm/fsl-imx6.c b/hw/arm/fsl-imx6.c
new file mode 100644
index 0000000..6a1bf26
--- /dev/null
+++ b/hw/arm/fsl-imx6.c
@@ -0,0 +1,466 @@
+/*
+ * Copyright (c) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * i.MX6 SOC emulation.
+ *
+ * Based on hw/arm/fsl-imx31.c
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "hw/arm/fsl-imx6.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/char.h"
+#include "qemu/error-report.h"
+
+#define NAME_SIZE 20
+
+static void fsl_imx6_init(Object *obj)
+{
+    FslIMX6State *s = FSL_IMX6(obj);
+    char name[NAME_SIZE];
+    int i;
+
+    if (smp_cpus > FSL_IMX6_NUM_CPUS) {
+        error_report("%s: Only %d CPUs are supported (%d requested)",
+                     TYPE_FSL_IMX6, FSL_IMX6_NUM_CPUS, smp_cpus);
+        exit(1);
+    }
+
+    for (i = 0; i < smp_cpus; i++) {
+        object_initialize(&s->cpu[i], sizeof(s->cpu[i]),
+                          "cortex-a9-" TYPE_ARM_CPU);
+        snprintf(name, NAME_SIZE, "cpu%d", i);
+        object_property_add_child(obj, name, OBJECT(&s->cpu[i]), NULL);
+    }
+
+    object_initialize(&s->a9mpcore, sizeof(s->a9mpcore), TYPE_A9MPCORE_PRIV);
+    qdev_set_parent_bus(DEVICE(&s->a9mpcore), sysbus_get_default());
+    object_property_add_child(obj, "a9mpcore", OBJECT(&s->a9mpcore), NULL);
+
+    object_initialize(&s->ccm, sizeof(s->ccm), TYPE_IMX6_CCM);
+    qdev_set_parent_bus(DEVICE(&s->ccm), sysbus_get_default());
+    object_property_add_child(obj, "ccm", OBJECT(&s->ccm), NULL);
+
+    object_initialize(&s->src, sizeof(s->src), TYPE_IMX6_SRC);
+    qdev_set_parent_bus(DEVICE(&s->src), sysbus_get_default());
+    object_property_add_child(obj, "src", OBJECT(&s->src), NULL);
+
+    for (i = 0; i < FSL_IMX6_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_IMX_SERIAL);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+        snprintf(name, NAME_SIZE, "uart%d", i + 1);
+        object_property_add_child(obj, name, OBJECT(&s->uart[i]), NULL);
+    }
+
+    object_initialize(&s->gpt, sizeof(s->gpt), TYPE_IMX6_GPT);
+    qdev_set_parent_bus(DEVICE(&s->gpt), sysbus_get_default());
+    object_property_add_child(obj, "gpt", OBJECT(&s->gpt), NULL);
+
+    for (i = 0; i < FSL_IMX6_NUM_EPITS; i++) {
+        object_initialize(&s->epit[i], sizeof(s->epit[i]), TYPE_IMX_EPIT);
+        qdev_set_parent_bus(DEVICE(&s->epit[i]), sysbus_get_default());
+        snprintf(name, NAME_SIZE, "epit%d", i + 1);
+        object_property_add_child(obj, name, OBJECT(&s->epit[i]), NULL);
+    }
+
+    for (i = 0; i < FSL_IMX6_NUM_I2CS; i++) {
+        object_initialize(&s->i2c[i], sizeof(s->i2c[i]), TYPE_IMX_I2C);
+        qdev_set_parent_bus(DEVICE(&s->i2c[i]), sysbus_get_default());
+        snprintf(name, NAME_SIZE, "i2c%d", i + 1);
+        object_property_add_child(obj, name, OBJECT(&s->i2c[i]), NULL);
+    }
+
+    for (i = 0; i < FSL_IMX6_NUM_GPIOS; i++) {
+        object_initialize(&s->gpio[i], sizeof(s->gpio[i]), TYPE_IMX_GPIO);
+        qdev_set_parent_bus(DEVICE(&s->gpio[i]), sysbus_get_default());
+        snprintf(name, NAME_SIZE, "gpio%d", i + 1);
+        object_property_add_child(obj, name, OBJECT(&s->gpio[i]), NULL);
+    }
+
+    for (i = 0; i < FSL_IMX6_NUM_ESDHCS; i++) {
+        object_initialize(&s->esdhc[i], sizeof(s->esdhc[i]), TYPE_SYSBUS_SDHCI);
+        qdev_set_parent_bus(DEVICE(&s->esdhc[i]), sysbus_get_default());
+        snprintf(name, NAME_SIZE, "sdhc%d", i + 1);
+        object_property_add_child(obj, name, OBJECT(&s->esdhc[i]), NULL);
+    }
+
+    for (i = 0; i < FSL_IMX6_NUM_ECSPIS; i++) {
+        object_initialize(&s->spi[i], sizeof(s->spi[i]), TYPE_IMX_SPI);
+        qdev_set_parent_bus(DEVICE(&s->spi[i]), sysbus_get_default());
+        snprintf(name, NAME_SIZE, "spi%d", i + 1);
+        object_property_add_child(obj, name, OBJECT(&s->spi[i]), NULL);
+    }
+
+    object_initialize(&s->eth, sizeof(s->eth), TYPE_IMX_ENET);
+    qdev_set_parent_bus(DEVICE(&s->eth), sysbus_get_default());
+    object_property_add_child(obj, "eth", OBJECT(&s->eth), NULL);
+}
+
+static void fsl_imx6_realize(DeviceState *dev, Error **errp)
+{
+    FslIMX6State *s = FSL_IMX6(dev);
+    uint16_t i;
+    Error *err = NULL;
+
+    for (i = 0; i < smp_cpus; i++) {
+
+        /* On uniprocessor, the CBAR is set to 0 */
+        if (smp_cpus > 1) {
+            object_property_set_int(OBJECT(&s->cpu[i]), FSL_IMX6_A9MPCORE_ADDR,
+                                    "reset-cbar", &error_abort);
+        }
+
+        /* All CPU but CPU 0 start in power off mode */
+        if (i) {
+            object_property_set_bool(OBJECT(&s->cpu[i]), true,
+                                     "start-powered-off", &error_abort);
+        }
+
+        object_property_set_bool(OBJECT(&s->cpu[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    }
+
+    object_property_set_int(OBJECT(&s->a9mpcore), smp_cpus, "num-cpu",
+                            &error_abort);
+
+    object_property_set_int(OBJECT(&s->a9mpcore),
+                            FSL_IMX6_MAX_IRQ + GIC_INTERNAL, "num-irq",
+                            &error_abort);
+
+    object_property_set_bool(OBJECT(&s->a9mpcore), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->a9mpcore), 0, FSL_IMX6_A9MPCORE_ADDR);
+
+    for (i = 0; i < smp_cpus; i++) {
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->a9mpcore), i,
+                           qdev_get_gpio_in(DEVICE(&s->cpu[i]), ARM_CPU_IRQ));
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->a9mpcore), i + smp_cpus,
+                           qdev_get_gpio_in(DEVICE(&s->cpu[i]), ARM_CPU_FIQ));
+    }
+
+    object_property_set_bool(OBJECT(&s->ccm), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->ccm), 0, FSL_IMX6_CCM_ADDR);
+
+    object_property_set_bool(OBJECT(&s->src), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->src), 0, FSL_IMX6_SRC_ADDR);
+
+    /* Initialize all UARTs */
+    for (i = 0; i < FSL_IMX6_NUM_UARTS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } serial_table[FSL_IMX6_NUM_UARTS] = {
+            { FSL_IMX6_UART1_ADDR, FSL_IMX6_UART1_IRQ },
+            { FSL_IMX6_UART2_ADDR, FSL_IMX6_UART2_IRQ },
+            { FSL_IMX6_UART3_ADDR, FSL_IMX6_UART3_IRQ },
+            { FSL_IMX6_UART4_ADDR, FSL_IMX6_UART4_IRQ },
+            { FSL_IMX6_UART5_ADDR, FSL_IMX6_UART5_IRQ },
+        };
+
+        if (i < MAX_SERIAL_PORTS) {
+            CharDriverState *chr;
+
+            chr = serial_hds[i];
+
+            if (!chr) {
+                char *label = g_strdup_printf("imx6.uart%d", i + 1);
+                chr = qemu_chr_new(label, "null", NULL);
+                g_free(label);
+                serial_hds[i] = chr;
+            }
+
+            qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", chr);
+        }
+
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, serial_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            serial_table[i].irq));
+    }
+
+    s->gpt.ccm = IMX_CCM(&s->ccm);
+
+    object_property_set_bool(OBJECT(&s->gpt), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpt), 0, FSL_IMX6_GPT_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpt), 0,
+                       qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                        FSL_IMX6_GPT_IRQ));
+
+    /* Initialize all EPIT timers */
+    for (i = 0; i < FSL_IMX6_NUM_EPITS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } epit_table[FSL_IMX6_NUM_EPITS] = {
+            { FSL_IMX6_EPIT1_ADDR, FSL_IMX6_EPIT1_IRQ },
+            { FSL_IMX6_EPIT2_ADDR, FSL_IMX6_EPIT2_IRQ },
+        };
+
+        s->epit[i].ccm = IMX_CCM(&s->ccm);
+
+        object_property_set_bool(OBJECT(&s->epit[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->epit[i]), 0, epit_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->epit[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            epit_table[i].irq));
+    }
+
+    /* Initialize all I2C */
+    for (i = 0; i < FSL_IMX6_NUM_I2CS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } i2c_table[FSL_IMX6_NUM_I2CS] = {
+            { FSL_IMX6_I2C1_ADDR, FSL_IMX6_I2C1_IRQ },
+            { FSL_IMX6_I2C2_ADDR, FSL_IMX6_I2C2_IRQ },
+            { FSL_IMX6_I2C3_ADDR, FSL_IMX6_I2C3_IRQ }
+        };
+
+        object_property_set_bool(OBJECT(&s->i2c[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->i2c[i]), 0, i2c_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->i2c[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            i2c_table[i].irq));
+    }
+
+    /* Initialize all GPIOs */
+    for (i = 0; i < FSL_IMX6_NUM_GPIOS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq_low;
+            unsigned int irq_high;
+        } gpio_table[FSL_IMX6_NUM_GPIOS] = {
+            {
+                FSL_IMX6_GPIO1_ADDR,
+                FSL_IMX6_GPIO1_LOW_IRQ,
+                FSL_IMX6_GPIO1_HIGH_IRQ
+            },
+            {
+                FSL_IMX6_GPIO2_ADDR,
+                FSL_IMX6_GPIO2_LOW_IRQ,
+                FSL_IMX6_GPIO2_HIGH_IRQ
+            },
+            {
+                FSL_IMX6_GPIO3_ADDR,
+                FSL_IMX6_GPIO3_LOW_IRQ,
+                FSL_IMX6_GPIO3_HIGH_IRQ
+            },
+            {
+                FSL_IMX6_GPIO4_ADDR,
+                FSL_IMX6_GPIO4_LOW_IRQ,
+                FSL_IMX6_GPIO4_HIGH_IRQ
+            },
+            {
+                FSL_IMX6_GPIO5_ADDR,
+                FSL_IMX6_GPIO5_LOW_IRQ,
+                FSL_IMX6_GPIO5_HIGH_IRQ
+            },
+            {
+                FSL_IMX6_GPIO6_ADDR,
+                FSL_IMX6_GPIO6_LOW_IRQ,
+                FSL_IMX6_GPIO6_HIGH_IRQ
+            },
+            {
+                FSL_IMX6_GPIO7_ADDR,
+                FSL_IMX6_GPIO7_LOW_IRQ,
+                FSL_IMX6_GPIO7_HIGH_IRQ
+            },
+        };
+
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "has-edge-sel",
+                                 &error_abort);
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "has-upper-pin-irq",
+                                 &error_abort);
+        object_property_set_bool(OBJECT(&s->gpio[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gpio[i]), 0, gpio_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            gpio_table[i].irq_low));
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gpio[i]), 1,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            gpio_table[i].irq_high));
+    }
+
+    /* Initialize all SDHC */
+    for (i = 0; i < FSL_IMX6_NUM_ESDHCS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } esdhc_table[FSL_IMX6_NUM_ESDHCS] = {
+            { FSL_IMX6_uSDHC1_ADDR, FSL_IMX6_uSDHC1_IRQ },
+            { FSL_IMX6_uSDHC2_ADDR, FSL_IMX6_uSDHC2_IRQ },
+            { FSL_IMX6_uSDHC3_ADDR, FSL_IMX6_uSDHC3_IRQ },
+            { FSL_IMX6_uSDHC4_ADDR, FSL_IMX6_uSDHC4_IRQ },
+        };
+
+        object_property_set_bool(OBJECT(&s->esdhc[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->esdhc[i]), 0, esdhc_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->esdhc[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            esdhc_table[i].irq));
+    }
+
+    /* Initialize all ECSPI */
+    for (i = 0; i < FSL_IMX6_NUM_ECSPIS; i++) {
+        static const struct {
+            hwaddr addr;
+            unsigned int irq;
+        } spi_table[FSL_IMX6_NUM_ECSPIS] = {
+            { FSL_IMX6_eCSPI1_ADDR, FSL_IMX6_ECSPI1_IRQ },
+            { FSL_IMX6_eCSPI2_ADDR, FSL_IMX6_ECSPI2_IRQ },
+            { FSL_IMX6_eCSPI3_ADDR, FSL_IMX6_ECSPI3_IRQ },
+            { FSL_IMX6_eCSPI4_ADDR, FSL_IMX6_ECSPI4_IRQ },
+            { FSL_IMX6_eCSPI5_ADDR, FSL_IMX6_ECSPI5_IRQ },
+        };
+
+        /* Initialize the SPI */
+        object_property_set_bool(OBJECT(&s->spi[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->spi[i]), 0, spi_table[i].addr);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->spi[i]), 0,
+                           qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                            spi_table[i].irq));
+    }
+
+    object_property_set_bool(OBJECT(&s->eth), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->eth), 0, FSL_IMX6_ENET_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->eth), 0,
+                       qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                        FSL_IMX6_ENET_MAC_IRQ));
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->eth), 1,
+                       qdev_get_gpio_in(DEVICE(&s->a9mpcore),
+                                        FSL_IMX6_ENET_MAC_1588_IRQ));
+
+    /* ROM memory */
+    memory_region_init_rom(&s->rom, NULL, "imx6.rom",
+                           FSL_IMX6_ROM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX6_ROM_ADDR,
+                                &s->rom);
+
+    /* CAAM memory */
+    memory_region_init_rom(&s->caam, NULL, "imx6.caam",
+                           FSL_IMX6_CAAM_MEM_SIZE, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX6_CAAM_MEM_ADDR,
+                                &s->caam);
+
+    /* OCRAM memory */
+    memory_region_init_ram(&s->ocram, NULL, "imx6.ocram", FSL_IMX6_OCRAM_SIZE,
+                           &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    memory_region_add_subregion(get_system_memory(), FSL_IMX6_OCRAM_ADDR,
+                                &s->ocram);
+    vmstate_register_ram_global(&s->ocram);
+
+    /* internal OCRAM (256 KB) is aliased over 1 MB */
+    memory_region_init_alias(&s->ocram_alias, NULL, "imx6.ocram_alias",
+                             &s->ocram, 0, FSL_IMX6_OCRAM_ALIAS_SIZE);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX6_OCRAM_ALIAS_ADDR,
+                                &s->ocram_alias);
+}
+
+static void fsl_imx6_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = fsl_imx6_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+    dc->desc = "i.MX6 SOC";
+}
+
+static const TypeInfo fsl_imx6_type_info = {
+    .name = TYPE_FSL_IMX6,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(FslIMX6State),
+    .instance_init = fsl_imx6_init,
+    .class_init = fsl_imx6_class_init,
+};
+
+static void fsl_imx6_register_types(void)
+{
+    type_register_static(&fsl_imx6_type_info);
+}
+
+type_init(fsl_imx6_register_types)
diff --git a/hw/arm/gumstix.c b/hw/arm/gumstix.c
index 8103278..d59d9ba 100644
--- a/hw/arm/gumstix.c
+++ b/hw/arm/gumstix.c
@@ -34,6 +34,7 @@
  * # qemu-system-arm -M verdex -pflash flash -monitor null -nographic -m 289
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "net/net.h"
@@ -121,22 +122,38 @@
                     qdev_get_gpio_in(cpu->gpio, 99));
 }
 
-static QEMUMachine connex_machine = {
-    .name = "connex",
-    .desc = "Gumstix Connex (PXA255)",
-    .init = connex_init,
+static void connex_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Gumstix Connex (PXA255)";
+    mc->init = connex_init;
+}
+
+static const TypeInfo connex_type = {
+    .name = MACHINE_TYPE_NAME("connex"),
+    .parent = TYPE_MACHINE,
+    .class_init = connex_class_init,
 };
 
-static QEMUMachine verdex_machine = {
-    .name = "verdex",
-    .desc = "Gumstix Verdex (PXA270)",
-    .init = verdex_init,
+static void verdex_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Gumstix Verdex (PXA270)";
+    mc->init = verdex_init;
+}
+
+static const TypeInfo verdex_type = {
+    .name = MACHINE_TYPE_NAME("verdex"),
+    .parent = TYPE_MACHINE,
+    .class_init = verdex_class_init,
 };
 
 static void gumstix_machine_init(void)
 {
-    qemu_register_machine(&connex_machine);
-    qemu_register_machine(&verdex_machine);
+    type_register_static(&connex_type);
+    type_register_static(&verdex_type);
 }
 
-machine_init(gumstix_machine_init);
+type_init(gumstix_machine_init)
diff --git a/hw/arm/highbank.c b/hw/arm/highbank.c
index 30f744a..80e5fd4 100644
--- a/hw/arm/highbank.c
+++ b/hw/arm/highbank.c
@@ -17,25 +17,38 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
 #include "hw/loader.h"
 #include "net/net.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
+#include "hw/char/pl011.h"
 
 #define SMP_BOOT_ADDR           0x100
 #define SMP_BOOT_REG            0x40
 #define MPCORE_PERIPHBASE       0xfff10000
 
+#define MVBAR_ADDR              0x200
+#define BOARD_SETUP_ADDR        (MVBAR_ADDR + 8 * sizeof(uint32_t))
+
 #define NIRQ_GIC                160
 
 /* Board init.  */
 
+static void hb_write_board_setup(ARMCPU *cpu,
+                                 const struct arm_boot_info *info)
+{
+    arm_write_secure_board_setup_dummy_smc(cpu, info, MVBAR_ADDR);
+}
+
 static void hb_write_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
 {
     int n;
@@ -69,11 +82,17 @@
 
     switch (info->nb_cpus) {
     case 4:
-        stl_phys_notdirty(&address_space_memory, SMP_BOOT_REG + 0x30, 0);
+        address_space_stl_notdirty(&address_space_memory,
+                                   SMP_BOOT_REG + 0x30, 0,
+                                   MEMTXATTRS_UNSPECIFIED, NULL);
     case 3:
-        stl_phys_notdirty(&address_space_memory, SMP_BOOT_REG + 0x20, 0);
+        address_space_stl_notdirty(&address_space_memory,
+                                   SMP_BOOT_REG + 0x20, 0,
+                                   MEMTXATTRS_UNSPECIFIED, NULL);
     case 2:
-        stl_phys_notdirty(&address_space_memory, SMP_BOOT_REG + 0x10, 0);
+        address_space_stl_notdirty(&address_space_memory,
+                                   SMP_BOOT_REG + 0x10, 0,
+                                   MEMTXATTRS_UNSPECIFIED, NULL);
         env->regs[15] = SMP_BOOT_ADDR;
         break;
     default:
@@ -150,23 +169,20 @@
     s->regs[0x43] = 0x05F40121;
 }
 
-static int highbank_regs_init(SysBusDevice *dev)
+static void highbank_regs_init(Object *obj)
 {
-    HighbankRegsState *s = HIGHBANK_REGISTERS(dev);
+    HighbankRegsState *s = HIGHBANK_REGISTERS(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &hb_mem_ops, s->regs,
+    memory_region_init_io(&s->iomem, obj, &hb_mem_ops, s->regs,
                           "highbank_regs", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void highbank_regs_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    sbc->init = highbank_regs_init;
     dc->desc = "Calxeda Highbank registers";
     dc->vmsd = &vmstate_highbank_regs;
     dc->reset = highbank_regs_reset;
@@ -176,6 +192,7 @@
     .name          = TYPE_HIGHBANK_REGISTERS,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(HighbankRegsState),
+    .instance_init = highbank_regs_init,
     .class_init    = highbank_regs_class_init,
 };
 
@@ -211,67 +228,68 @@
     qemu_irq pic[128];
     int n;
     qemu_irq cpu_irq[4];
+    qemu_irq cpu_fiq[4];
     MemoryRegion *sysram;
     MemoryRegion *dram;
     MemoryRegion *sysmem;
     char *sysboot_filename;
 
-    if (!cpu_model) {
-        switch (machine_id) {
-        case CALXEDA_HIGHBANK:
-            cpu_model = "cortex-a9";
-            break;
-        case CALXEDA_MIDWAY:
-            cpu_model = "cortex-a15";
-            break;
-        }
+    switch (machine_id) {
+    case CALXEDA_HIGHBANK:
+        cpu_model = "cortex-a9";
+        break;
+    case CALXEDA_MIDWAY:
+        cpu_model = "cortex-a15";
+        break;
     }
 
     for (n = 0; n < smp_cpus; n++) {
         ObjectClass *oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
         Object *cpuobj;
         ARMCPU *cpu;
-        Error *err = NULL;
-
-        if (!oc) {
-            error_report("Unable to find CPU definition");
-            exit(1);
-        }
 
         cpuobj = object_new(object_class_get_name(oc));
         cpu = ARM_CPU(cpuobj);
 
+        object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_SMC,
+                                "psci-conduit", &error_abort);
+
+        if (n) {
+            /* Secondary CPUs start in PSCI powered-down state */
+            object_property_set_bool(cpuobj, true,
+                                     "start-powered-off", &error_abort);
+        }
+
         if (object_property_find(cpuobj, "reset-cbar", NULL)) {
             object_property_set_int(cpuobj, MPCORE_PERIPHBASE,
                                     "reset-cbar", &error_abort);
         }
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report("%s", error_get_pretty(err));
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
         cpu_irq[n] = qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ);
+        cpu_fiq[n] = qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_FIQ);
     }
 
     sysmem = get_system_memory();
     dram = g_new(MemoryRegion, 1);
-    memory_region_init_ram(dram, NULL, "highbank.dram", ram_size, &error_abort);
+    memory_region_allocate_system_memory(dram, NULL, "highbank.dram", ram_size);
     /* SDRAM at address zero.  */
     memory_region_add_subregion(sysmem, 0, dram);
 
     sysram = g_new(MemoryRegion, 1);
     memory_region_init_ram(sysram, NULL, "highbank.sysram", 0x8000,
-                           &error_abort);
+                           &error_fatal);
     memory_region_add_subregion(sysmem, 0xfff88000, sysram);
     if (bios_name != NULL) {
         sysboot_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (sysboot_filename != NULL) {
-            uint32_t filesize = get_image_size(sysboot_filename);
-            if (load_image_targphys("sysram.bin", 0xfff88000, filesize) < 0) {
-                hw_error("Unable to load %s\n", bios_name);
+            if (load_image_targphys(sysboot_filename, 0xfff88000, 0x8000) < 0) {
+                error_report("Unable to load %s", bios_name);
+                exit(1);
             }
+            g_free(sysboot_filename);
         } else {
-           hw_error("Unable to find %s\n", bios_name);
+            error_report("Unable to find %s", bios_name);
+            exit(1);
         }
     }
 
@@ -295,6 +313,7 @@
     sysbus_mmio_map(busdev, 0, MPCORE_PERIPHBASE);
     for (n = 0; n < smp_cpus; n++) {
         sysbus_connect_irq(busdev, n, cpu_irq[n]);
+        sysbus_connect_irq(busdev, n + smp_cpus, cpu_fiq[n]);
     }
 
     for (n = 0; n < 128; n++) {
@@ -308,7 +327,7 @@
     busdev = SYS_BUS_DEVICE(dev);
     sysbus_mmio_map(busdev, 0, 0xfff34000);
     sysbus_connect_irq(busdev, 0, pic[18]);
-    sysbus_create_simple("pl011", 0xfff36000, pic[20]);
+    pl011_create(0xfff36000, pic[20], serial_hds[0]);
 
     dev = qdev_create(NULL, "highbank-regs");
     qdev_init_nofail(dev);
@@ -357,6 +376,16 @@
     highbank_binfo.loader_start = 0;
     highbank_binfo.write_secondary_boot = hb_write_secondary;
     highbank_binfo.secondary_cpu_reset_hook = hb_reset_secondary;
+    if (!kvm_enabled()) {
+        highbank_binfo.board_setup_addr = BOARD_SETUP_ADDR;
+        highbank_binfo.write_board_setup = hb_write_board_setup;
+        highbank_binfo.secure_board_setup = true;
+    } else {
+        error_report("WARNING: cannot load built-in Monitor support "
+                     "if KVM is enabled. Some guests (such as Linux) "
+                     "may not boot.");
+    }
+
     arm_load_kernel(ARM_CPU(first_cpu), &highbank_binfo);
 }
 
@@ -370,26 +399,42 @@
     calxeda_init(machine, CALXEDA_MIDWAY);
 }
 
-static QEMUMachine highbank_machine = {
-    .name = "highbank",
-    .desc = "Calxeda Highbank (ECX-1000)",
-    .init = highbank_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void highbank_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Calxeda Highbank (ECX-1000)";
+    mc->init = highbank_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo highbank_type = {
+    .name = MACHINE_TYPE_NAME("highbank"),
+    .parent = TYPE_MACHINE,
+    .class_init = highbank_class_init,
 };
 
-static QEMUMachine midway_machine = {
-    .name = "midway",
-    .desc = "Calxeda Midway (ECX-2000)",
-    .init = midway_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void midway_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Calxeda Midway (ECX-2000)";
+    mc->init = midway_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo midway_type = {
+    .name = MACHINE_TYPE_NAME("midway"),
+    .parent = TYPE_MACHINE,
+    .class_init = midway_class_init,
 };
 
 static void calxeda_machines_init(void)
 {
-    qemu_register_machine(&highbank_machine);
-    qemu_register_machine(&midway_machine);
+    type_register_static(&highbank_type);
+    type_register_static(&midway_type);
 }
 
-machine_init(calxeda_machines_init);
+type_init(calxeda_machines_init)
diff --git a/hw/arm/imx25_pdk.c b/hw/arm/imx25_pdk.c
new file mode 100644
index 0000000..025b608
--- /dev/null
+++ b/hw/arm/imx25_pdk.c
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2013 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * PDK Board System emulation.
+ *
+ * Based on hw/arm/kzm.c
+ *
+ * Copyright (c) 2008 OKL and 2011 NICTA
+ * Written by Hans at OK-Labs
+ * Updated by Peter Chubb.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx25.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
+#include "exec/address-spaces.h"
+#include "sysemu/qtest.h"
+#include "hw/i2c/i2c.h"
+
+/* Memory map for PDK Emulation Baseboard:
+ * 0x00000000-0x7fffffff See i.MX25 SOC fr support
+ * 0x80000000-0x87ffffff RAM + Alias          EMULATED
+ * 0x90000000-0x9fffffff RAM + Alias          EMULATED
+ * 0xa0000000-0xa7ffffff Flash                IGNORED
+ * 0xa8000000-0xafffffff Flash                IGNORED
+ * 0xb0000000-0xb1ffffff SRAM                 IGNORED
+ * 0xb2000000-0xb3ffffff SRAM                 IGNORED
+ * 0xb4000000-0xb5ffffff CS4                  IGNORED
+ * 0xb6000000-0xb8000fff Reserved             IGNORED
+ * 0xb8001000-0xb8001fff SDRAM CTRL reg       IGNORED
+ * 0xb8002000-0xb8002fff WEIM CTRL reg        IGNORED
+ * 0xb8003000-0xb8003fff M3IF CTRL reg        IGNORED
+ * 0xb8004000-0xb8004fff EMI CTRL reg         IGNORED
+ * 0xb8005000-0xbaffffff Reserved             IGNORED
+ * 0xbb000000-0xbb000fff NAND flash area buf  IGNORED
+ * 0xbb001000-0xbb0011ff NAND flash reserved  IGNORED
+ * 0xbb001200-0xbb001dff Reserved             IGNORED
+ * 0xbb001e00-0xbb001fff NAN flash CTRL reg   IGNORED
+ * 0xbb012000-0xbfffffff Reserved             IGNORED
+ * 0xc0000000-0xffffffff Reserved             IGNORED
+ */
+
+typedef struct IMX25PDK {
+    FslIMX25State soc;
+    MemoryRegion ram;
+    MemoryRegion ram_alias;
+} IMX25PDK;
+
+static struct arm_boot_info imx25_pdk_binfo;
+
+static void imx25_pdk_init(MachineState *machine)
+{
+    IMX25PDK *s = g_new0(IMX25PDK, 1);
+    unsigned int ram_size;
+    unsigned int alias_offset;
+    int i;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX25);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_fatal);
+
+    /* We need to initialize our memory */
+    if (machine->ram_size > (FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE)) {
+        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
+                     "reduced to %x", machine->ram_size,
+                     FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE);
+        machine->ram_size = FSL_IMX25_SDRAM0_SIZE + FSL_IMX25_SDRAM1_SIZE;
+    }
+
+    memory_region_allocate_system_memory(&s->ram, NULL, "imx25.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX25_SDRAM0_ADDR,
+                                &s->ram);
+
+    /* initialize the alias memory if any */
+    for (i = 0, ram_size = machine->ram_size, alias_offset = 0;
+         (i < 2) && ram_size; i++) {
+        unsigned int size;
+        static const struct {
+            hwaddr addr;
+            unsigned int size;
+        } ram[2] = {
+            { FSL_IMX25_SDRAM0_ADDR, FSL_IMX25_SDRAM0_SIZE },
+            { FSL_IMX25_SDRAM1_ADDR, FSL_IMX25_SDRAM1_SIZE },
+        };
+
+        size = MIN(ram_size, ram[i].size);
+
+        ram_size -= size;
+
+        if (size < ram[i].size) {
+            memory_region_init_alias(&s->ram_alias, NULL, "ram.alias",
+                                     &s->ram, alias_offset, ram[i].size - size);
+            memory_region_add_subregion(get_system_memory(),
+                                        ram[i].addr + size, &s->ram_alias);
+        }
+
+        alias_offset += ram[i].size;
+    }
+
+    imx25_pdk_binfo.ram_size = machine->ram_size;
+    imx25_pdk_binfo.kernel_filename = machine->kernel_filename;
+    imx25_pdk_binfo.kernel_cmdline = machine->kernel_cmdline;
+    imx25_pdk_binfo.initrd_filename = machine->initrd_filename;
+    imx25_pdk_binfo.loader_start = FSL_IMX25_SDRAM0_ADDR;
+    imx25_pdk_binfo.board_id = 1771,
+    imx25_pdk_binfo.nb_cpus = 1;
+
+    /*
+     * We test explicitly for qtest here as it is not done (yet?) in
+     * arm_load_kernel(). Without this the "make check" command would
+     * fail.
+     */
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu, &imx25_pdk_binfo);
+    } else {
+        /*
+         * This I2C device doesn't exist on the real board.
+         * We add it here (only on qtest usage) to be able to do a bit
+         * of simple qtest. See "make check" for details.
+         */
+        i2c_create_slave((I2CBus *)qdev_get_child_bus(DEVICE(&s->soc.i2c[0]),
+                                                      "i2c"),
+                         "ds1338", 0x68);
+    }
+}
+
+static void imx25_pdk_machine_init(MachineClass *mc)
+{
+    mc->desc = "ARM i.MX25 PDK board (ARM926)";
+    mc->init = imx25_pdk_init;
+}
+
+DEFINE_MACHINE("imx25-pdk", imx25_pdk_machine_init)
diff --git a/hw/arm/integratorcp.c b/hw/arm/integratorcp.c
index 266ec18..96dc150 100644
--- a/hw/arm/integratorcp.c
+++ b/hw/arm/integratorcp.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/boards.h"
@@ -15,6 +19,8 @@
 #include "net/net.h"
 #include "exec/address-spaces.h"
 #include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
+#include "hw/char/pl011.h"
 
 #define TYPE_INTEGRATOR_CM "integrator_core"
 #define INTEGRATOR_CM(obj) \
@@ -237,9 +243,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int integratorcm_init(SysBusDevice *dev)
+static void integratorcm_init(Object *obj)
 {
-    IntegratorCMState *s = INTEGRATOR_CM(dev);
+    IntegratorCMState *s = INTEGRATOR_CM(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     s->cm_osc = 0x01000048;
     /* ??? What should the high bits of this value be?  */
@@ -264,17 +271,16 @@
     s->cm_init = 0x00000112;
     s->cm_refcnt_offset = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24,
                                    1000);
-    memory_region_init_ram(&s->flash, OBJECT(s), "integrator.flash", 0x100000,
-                           &error_abort);
+    memory_region_init_ram(&s->flash, obj, "integrator.flash", 0x100000,
+                           &error_fatal);
     vmstate_register_ram_global(&s->flash);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &integratorcm_ops, s,
+    memory_region_init_io(&s->iomem, obj, &integratorcm_ops, s,
                           "integratorcm", 0x00800000);
     sysbus_init_mmio(dev, &s->iomem);
 
     integratorcm_do_remap(s);
     /* ??? Save/restore.  */
-    return 0;
 }
 
 /* Integrator/CP hardware emulation.  */
@@ -389,32 +395,55 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int icp_pic_init(SysBusDevice *sbd)
+static void icp_pic_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    icp_pic_state *s = INTEGRATOR_PIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    icp_pic_state *s = INTEGRATOR_PIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     qdev_init_gpio_in(dev, icp_pic_set_irq, 32);
     sysbus_init_irq(sbd, &s->parent_irq);
     sysbus_init_irq(sbd, &s->parent_fiq);
-    memory_region_init_io(&s->iomem, OBJECT(s), &icp_pic_ops, s,
+    memory_region_init_io(&s->iomem, obj, &icp_pic_ops, s,
                           "icp-pic", 0x00800000);
     sysbus_init_mmio(sbd, &s->iomem);
-    return 0;
 }
 
 /* CP control registers.  */
 
+#define TYPE_ICP_CONTROL_REGS "icp-ctrl-regs"
+#define ICP_CONTROL_REGS(obj) \
+    OBJECT_CHECK(ICPCtrlRegsState, (obj), TYPE_ICP_CONTROL_REGS)
+
+typedef struct ICPCtrlRegsState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion iomem;
+
+    qemu_irq mmc_irq;
+    uint32_t intreg_state;
+} ICPCtrlRegsState;
+
+#define ICP_GPIO_MMC_WPROT      "mmc-wprot"
+#define ICP_GPIO_MMC_CARDIN     "mmc-cardin"
+
+#define ICP_INTREG_WPROT        (1 << 0)
+#define ICP_INTREG_CARDIN       (1 << 3)
+
 static uint64_t icp_control_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
+    ICPCtrlRegsState *s = opaque;
+
     switch (offset >> 2) {
     case 0: /* CP_IDFIELD */
         return 0x41034003;
     case 1: /* CP_FLASHPROG */
         return 0;
     case 2: /* CP_INTREG */
-        return 0;
+        return s->intreg_state;
     case 3: /* CP_DECODE */
         return 0x11;
     default:
@@ -426,9 +455,14 @@
 static void icp_control_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
+    ICPCtrlRegsState *s = opaque;
+
     switch (offset >> 2) {
-    case 1: /* CP_FLASHPROG */
     case 2: /* CP_INTREG */
+        s->intreg_state &= ~(value & ICP_INTREG_CARDIN);
+        qemu_set_irq(s->mmc_irq, !!(s->intreg_state & ICP_INTREG_CARDIN));
+        break;
+    case 1: /* CP_FLASHPROG */
     case 3: /* CP_DECODE */
         /* Nothing interesting implemented yet.  */
         break;
@@ -443,15 +477,41 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void icp_control_init(hwaddr base)
+static void icp_control_mmc_wprot(void *opaque, int line, int level)
 {
-    MemoryRegion *io;
+    ICPCtrlRegsState *s = opaque;
 
-    io = (MemoryRegion *)g_malloc0(sizeof(MemoryRegion));
-    memory_region_init_io(io, NULL, &icp_control_ops, NULL,
-                          "control", 0x00800000);
-    memory_region_add_subregion(get_system_memory(), base, io);
-    /* ??? Save/restore.  */
+    s->intreg_state &= ~ICP_INTREG_WPROT;
+    if (level) {
+        s->intreg_state |= ICP_INTREG_WPROT;
+    }
+}
+
+static void icp_control_mmc_cardin(void *opaque, int line, int level)
+{
+    ICPCtrlRegsState *s = opaque;
+
+    /* line is released by writing to CP_INTREG */
+    if (level) {
+        s->intreg_state |= ICP_INTREG_CARDIN;
+        qemu_set_irq(s->mmc_irq, 1);
+    }
+}
+
+static void icp_control_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    ICPCtrlRegsState *s = ICP_CONTROL_REGS(obj);
+    DeviceState *dev = DEVICE(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &icp_control_ops, s,
+                          "icp_ctrl_regs", 0x00800000);
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    qdev_init_gpio_in_named(dev, icp_control_mmc_wprot, ICP_GPIO_MMC_WPROT, 1);
+    qdev_init_gpio_in_named(dev, icp_control_mmc_cardin,
+                            ICP_GPIO_MMC_CARDIN, 1);
+    sysbus_init_irq(sbd, &s->mmc_irq);
 }
 
 
@@ -469,25 +529,42 @@
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
+    ObjectClass *cpu_oc;
+    Object *cpuobj;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *ram_alias = g_new(MemoryRegion, 1);
     qemu_irq pic[32];
-    DeviceState *dev;
+    DeviceState *dev, *sic, *icp;
     int i;
 
     if (!cpu_model) {
         cpu_model = "arm926";
     }
-    cpu = cpu_arm_init(cpu_model);
-    if (!cpu) {
+
+    cpu_oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
+    if (!cpu_oc) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
 
-    memory_region_init_ram(ram, NULL, "integrator.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    cpuobj = object_new(object_class_get_name(cpu_oc));
+
+    /* By default ARM1176 CPUs have EL3 enabled.  This board does not
+     * currently support EL3 so the CPU EL3 property is disabled before
+     * realization.
+     */
+    if (object_property_find(cpuobj, "has_el3", NULL)) {
+        object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
+    }
+
+    object_property_set_bool(cpuobj, true, "realized", &error_fatal);
+
+    cpu = ARM_CPU(cpuobj);
+
+    memory_region_allocate_system_memory(ram, NULL, "integrator.ram",
+                                         ram_size);
     /* ??? On a real system the first 1Mb is mapped as SSRAM or boot flash.  */
     /* ??? RAM should repeat to fill physical memory space.  */
     /* SDRAM at address zero*/
@@ -508,17 +585,24 @@
     for (i = 0; i < 32; i++) {
         pic[i] = qdev_get_gpio_in(dev, i);
     }
-    sysbus_create_simple(TYPE_INTEGRATOR_PIC, 0xca000000, pic[26]);
+    sic = sysbus_create_simple(TYPE_INTEGRATOR_PIC, 0xca000000, pic[26]);
     sysbus_create_varargs("integrator_pit", 0x13000000,
                           pic[5], pic[6], pic[7], NULL);
     sysbus_create_simple("pl031", 0x15000000, pic[8]);
-    sysbus_create_simple("pl011", 0x16000000, pic[1]);
-    sysbus_create_simple("pl011", 0x17000000, pic[2]);
-    icp_control_init(0xcb000000);
+    pl011_create(0x16000000, pic[1], serial_hds[0]);
+    pl011_create(0x17000000, pic[2], serial_hds[1]);
+    icp = sysbus_create_simple(TYPE_ICP_CONTROL_REGS, 0xcb000000,
+                               qdev_get_gpio_in(sic, 3));
     sysbus_create_simple("pl050_keyboard", 0x18000000, pic[3]);
     sysbus_create_simple("pl050_mouse", 0x19000000, pic[4]);
     sysbus_create_simple(TYPE_INTEGRATOR_DEBUG, 0x1a000000, 0);
-    sysbus_create_varargs("pl181", 0x1c000000, pic[23], pic[24], NULL);
+
+    dev = sysbus_create_varargs("pl181", 0x1c000000, pic[23], pic[24], NULL);
+    qdev_connect_gpio_out(dev, 0,
+                          qdev_get_gpio_in_named(icp, ICP_GPIO_MMC_WPROT, 0));
+    qdev_connect_gpio_out(dev, 1,
+                          qdev_get_gpio_in_named(icp, ICP_GPIO_MMC_CARDIN, 0));
+
     if (nd_table[0].used)
         smc91c111_init(&nd_table[0], 0xc8000000, pic[27]);
 
@@ -531,18 +615,13 @@
     arm_load_kernel(cpu, &integrator_binfo);
 }
 
-static QEMUMachine integratorcp_machine = {
-    .name = "integratorcp",
-    .desc = "ARM Integrator/CP (ARM926EJ-S)",
-    .init = integratorcp_init,
-};
-
-static void integratorcp_machine_init(void)
+static void integratorcp_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&integratorcp_machine);
+    mc->desc = "ARM Integrator/CP (ARM926EJ-S)";
+    mc->init = integratorcp_init;
 }
 
-machine_init(integratorcp_machine_init);
+DEFINE_MACHINE("integratorcp", integratorcp_machine_init)
 
 static Property core_properties[] = {
     DEFINE_PROP_UINT32("memsz", IntegratorCMState, memsz, 0),
@@ -552,9 +631,7 @@
 static void core_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = integratorcm_init;
     dc->props = core_properties;
 }
 
@@ -562,27 +639,29 @@
     .name          = TYPE_INTEGRATOR_CM,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IntegratorCMState),
+    .instance_init = integratorcm_init,
     .class_init    = core_class_init,
 };
 
-static void icp_pic_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
-
-    sdc->init = icp_pic_init;
-}
-
 static const TypeInfo icp_pic_info = {
     .name          = TYPE_INTEGRATOR_PIC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pic_state),
-    .class_init    = icp_pic_class_init,
+    .instance_init = icp_pic_init,
+};
+
+static const TypeInfo icp_ctrl_regs_info = {
+    .name          = TYPE_ICP_CONTROL_REGS,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(ICPCtrlRegsState),
+    .instance_init = icp_control_init,
 };
 
 static void integratorcp_register_types(void)
 {
     type_register_static(&icp_pic_info);
     type_register_static(&core_info);
+    type_register_static(&icp_ctrl_regs_info);
 }
 
 type_init(integratorcp_register_types)
diff --git a/hw/arm/kzm.c b/hw/arm/kzm.c
index 94ceab6..2c96ee3 100644
--- a/hw/arm/kzm.c
+++ b/hw/arm/kzm.c
@@ -13,142 +13,135 @@
  * i.MX31 SoC
  */
 
-#include "hw/sysbus.h"
-#include "exec/address-spaces.h"
-#include "hw/hw.h"
-#include "hw/arm/arm.h"
-#include "hw/devices.h"
-#include "net/net.h"
-#include "sysemu/sysemu.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/fsl-imx31.h"
 #include "hw/boards.h"
+#include "qemu/error-report.h"
+#include "exec/address-spaces.h"
+#include "net/net.h"
+#include "hw/devices.h"
 #include "hw/char/serial.h"
-#include "hw/arm/imx.h"
+#include "sysemu/qtest.h"
 
-    /* Memory map for Kzm Emulation Baseboard:
-     * 0x00000000-0x00003fff 16k secure ROM       IGNORED
-     * 0x00004000-0x00407fff Reserved             IGNORED
-     * 0x00404000-0x00407fff ROM                  IGNORED
-     * 0x00408000-0x0fffffff Reserved             IGNORED
-     * 0x10000000-0x1fffbfff RAM aliasing         IGNORED
-     * 0x1fffc000-0x1fffffff RAM                  EMULATED
-     * 0x20000000-0x2fffffff Reserved             IGNORED
-     * 0x30000000-0x7fffffff I.MX31 Internal Register Space
-     *   0x43f00000 IO_AREA0
-     *   0x43f90000 UART1                         EMULATED
-     *   0x43f94000 UART2                         EMULATED
-     *   0x68000000 AVIC                          EMULATED
-     *   0x53f80000 CCM                           EMULATED
-     *   0x53f94000 PIT 1                         EMULATED
-     *   0x53f98000 PIT 2                         EMULATED
-     *   0x53f90000 GPT                           EMULATED
-     * 0x80000000-0x87ffffff RAM                  EMULATED
-     * 0x88000000-0x8fffffff RAM Aliasing         EMULATED
-     * 0xa0000000-0xafffffff NAND Flash           IGNORED
-     * 0xb0000000-0xb3ffffff Unavailable          IGNORED
-     * 0xb4000000-0xb4000fff 8-bit free space     IGNORED
-     * 0xb4001000-0xb400100f Board control        IGNORED
-     *  0xb4001003           DIP switch
-     * 0xb4001010-0xb400101f 7-segment LED        IGNORED
-     * 0xb4001020-0xb400102f LED                  IGNORED
-     * 0xb4001030-0xb400103f LED                  IGNORED
-     * 0xb4001040-0xb400104f FPGA, UART           EMULATED
-     * 0xb4001050-0xb400105f FPGA, UART           EMULATED
-     * 0xb4001060-0xb40fffff FPGA                 IGNORED
-     * 0xb6000000-0xb61fffff LAN controller       EMULATED
-     * 0xb6200000-0xb62fffff FPGA NAND Controller IGNORED
-     * 0xb6300000-0xb7ffffff Free                 IGNORED
-     * 0xb8000000-0xb8004fff Memory control registers IGNORED
-     * 0xc0000000-0xc3ffffff PCMCIA/CF            IGNORED
-     * 0xc4000000-0xffffffff Reserved             IGNORED
-     */
+/* Memory map for Kzm Emulation Baseboard:
+ * 0x00000000-0x7fffffff See i.MX31 SOC for support
+ * 0x80000000-0x8fffffff RAM                  EMULATED
+ * 0x90000000-0x9fffffff RAM                  EMULATED
+ * 0xa0000000-0xafffffff Flash                IGNORED
+ * 0xb0000000-0xb3ffffff Unavailable          IGNORED
+ * 0xb4000000-0xb4000fff 8-bit free space     IGNORED
+ * 0xb4001000-0xb400100f Board control        IGNORED
+ *  0xb4001003           DIP switch
+ * 0xb4001010-0xb400101f 7-segment LED        IGNORED
+ * 0xb4001020-0xb400102f LED                  IGNORED
+ * 0xb4001030-0xb400103f LED                  IGNORED
+ * 0xb4001040-0xb400104f FPGA, UART           EMULATED
+ * 0xb4001050-0xb400105f FPGA, UART           EMULATED
+ * 0xb4001060-0xb40fffff FPGA                 IGNORED
+ * 0xb6000000-0xb61fffff LAN controller       EMULATED
+ * 0xb6200000-0xb62fffff FPGA NAND Controller IGNORED
+ * 0xb6300000-0xb7ffffff Free                 IGNORED
+ * 0xb8000000-0xb8004fff Memory control registers IGNORED
+ * 0xc0000000-0xc3ffffff PCMCIA/CF            IGNORED
+ * 0xc4000000-0xffffffff Reserved             IGNORED
+ */
 
-#define KZM_RAMADDRESS (0x80000000)
-#define KZM_FPGA       (0xb4001040)
+typedef struct IMX31KZM {
+    FslIMX31State soc;
+    MemoryRegion ram;
+    MemoryRegion ram_alias;
+} IMX31KZM;
+
+#define KZM_RAM_ADDR            (FSL_IMX31_SDRAM0_ADDR)
+#define KZM_FPGA_ADDR           (FSL_IMX31_CS4_ADDR + 0x1040)
+#define KZM_LAN9118_ADDR        (FSL_IMX31_CS5_ADDR)
 
 static struct arm_boot_info kzm_binfo = {
-    .loader_start = KZM_RAMADDRESS,
+    .loader_start = KZM_RAM_ADDR,
     .board_id = 1722,
 };
 
 static void kzm_init(MachineState *machine)
 {
-    ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
-    const char *kernel_filename = machine->kernel_filename;
-    const char *kernel_cmdline = machine->kernel_cmdline;
-    const char *initrd_filename = machine->initrd_filename;
-    ARMCPU *cpu;
-    MemoryRegion *address_space_mem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    MemoryRegion *sram = g_new(MemoryRegion, 1);
-    MemoryRegion *ram_alias = g_new(MemoryRegion, 1);
-    DeviceState *dev;
-    DeviceState *ccm;
+    IMX31KZM *s = g_new0(IMX31KZM, 1);
+    unsigned int ram_size;
+    unsigned int alias_offset;
+    unsigned int i;
 
-    if (!cpu_model) {
-        cpu_model = "arm1136";
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX31);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_fatal);
+
+    /* Check the amount of memory is compatible with the SOC */
+    if (machine->ram_size > (FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE)) {
+        error_report("WARNING: RAM size " RAM_ADDR_FMT " above max supported, "
+                     "reduced to %x", machine->ram_size,
+                     FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE);
+        machine->ram_size = FSL_IMX31_SDRAM0_SIZE + FSL_IMX31_SDRAM1_SIZE;
     }
 
-    cpu = cpu_arm_init(cpu_model);
-    if (!cpu) {
-        fprintf(stderr, "Unable to find CPU definition\n");
-        exit(1);
+    memory_region_allocate_system_memory(&s->ram, NULL, "kzm.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX31_SDRAM0_ADDR,
+                                &s->ram);
+
+    /* initialize the alias memory if any */
+    for (i = 0, ram_size = machine->ram_size, alias_offset = 0;
+         (i < 2) && ram_size; i++) {
+        unsigned int size;
+        static const struct {
+            hwaddr addr;
+            unsigned int size;
+        } ram[2] = {
+            { FSL_IMX31_SDRAM0_ADDR, FSL_IMX31_SDRAM0_SIZE },
+            { FSL_IMX31_SDRAM1_ADDR, FSL_IMX31_SDRAM1_SIZE },
+        };
+
+        size = MIN(ram_size, ram[i].size);
+
+        ram_size -= size;
+
+        if (size < ram[i].size) {
+            memory_region_init_alias(&s->ram_alias, NULL, "ram.alias",
+                                     &s->ram, alias_offset, ram[i].size - size);
+            memory_region_add_subregion(get_system_memory(),
+                                        ram[i].addr + size, &s->ram_alias);
+        }
+
+        alias_offset += ram[i].size;
     }
 
-    /* On a real system, the first 16k is a `secure boot rom' */
-
-    memory_region_init_ram(ram, NULL, "kzm.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(address_space_mem, KZM_RAMADDRESS, ram);
-
-    memory_region_init_alias(ram_alias, NULL, "ram.alias", ram, 0, ram_size);
-    memory_region_add_subregion(address_space_mem, 0x88000000, ram_alias);
-
-    memory_region_init_ram(sram, NULL, "kzm.sram", 0x4000, &error_abort);
-    memory_region_add_subregion(address_space_mem, 0x1FFFC000, sram);
-
-    dev = sysbus_create_varargs("imx_avic", 0x68000000,
-                                qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_IRQ),
-                                qdev_get_gpio_in(DEVICE(cpu), ARM_CPU_FIQ),
-                                NULL);
-
-    imx_serial_create(0, 0x43f90000, qdev_get_gpio_in(dev, 45));
-    imx_serial_create(1, 0x43f94000, qdev_get_gpio_in(dev, 32));
-
-    ccm = sysbus_create_simple("imx_ccm", 0x53f80000, NULL);
-
-    imx_timerp_create(0x53f94000, qdev_get_gpio_in(dev, 28), ccm);
-    imx_timerp_create(0x53f98000, qdev_get_gpio_in(dev, 27), ccm);
-    imx_timerg_create(0x53f90000, qdev_get_gpio_in(dev, 29), ccm);
-
     if (nd_table[0].used) {
-        lan9118_init(&nd_table[0], 0xb6000000, qdev_get_gpio_in(dev, 52));
+        lan9118_init(&nd_table[0], KZM_LAN9118_ADDR,
+                     qdev_get_gpio_in(DEVICE(&s->soc.avic), 52));
     }
 
     if (serial_hds[2]) { /* touchscreen */
-        serial_mm_init(address_space_mem, KZM_FPGA+0x10, 0,
-                       qdev_get_gpio_in(dev, 52),
-                       14745600, serial_hds[2],
-                       DEVICE_NATIVE_ENDIAN);
+        serial_mm_init(get_system_memory(), KZM_FPGA_ADDR+0x10, 0,
+                       qdev_get_gpio_in(DEVICE(&s->soc.avic), 52),
+                       14745600, serial_hds[2], DEVICE_NATIVE_ENDIAN);
     }
 
-    kzm_binfo.ram_size = ram_size;
-    kzm_binfo.kernel_filename = kernel_filename;
-    kzm_binfo.kernel_cmdline = kernel_cmdline;
-    kzm_binfo.initrd_filename = initrd_filename;
+    kzm_binfo.ram_size = machine->ram_size;
+    kzm_binfo.kernel_filename = machine->kernel_filename;
+    kzm_binfo.kernel_cmdline = machine->kernel_cmdline;
+    kzm_binfo.initrd_filename = machine->initrd_filename;
     kzm_binfo.nb_cpus = 1;
-    arm_load_kernel(cpu, &kzm_binfo);
+
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu, &kzm_binfo);
+    }
 }
 
-static QEMUMachine kzm_machine = {
-    .name = "kzm",
-    .desc = "ARM KZM Emulation Baseboard (ARM1136)",
-    .init = kzm_init,
-};
-
-static void kzm_machine_init(void)
+static void kzm_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&kzm_machine);
+    mc->desc = "ARM KZM Emulation Baseboard (ARM1136)";
+    mc->init = kzm_init;
 }
 
-machine_init(kzm_machine_init)
+DEFINE_MACHINE("kzm", kzm_machine_init)
diff --git a/hw/arm/mainstone.c b/hw/arm/mainstone.c
index 0da02a6..454acc5 100644
--- a/hw/arm/mainstone.c
+++ b/hw/arm/mainstone.c
@@ -11,6 +11,8 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
@@ -124,7 +126,7 @@
     /* Setup CPU & memory */
     mpu = pxa270_init(address_space_mem, mainstone_binfo.ram_size, cpu_model);
     memory_region_init_ram(rom, NULL, "mainstone.rom", MAINSTONE_ROM,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -188,15 +190,10 @@
     mainstone_common_init(get_system_memory(), machine, mainstone, 0x196);
 }
 
-static QEMUMachine mainstone2_machine = {
-    .name = "mainstone",
-    .desc = "Mainstone II (PXA27x)",
-    .init = mainstone_init,
-};
-
-static void mainstone_machine_init(void)
+static void mainstone2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mainstone2_machine);
+    mc->desc = "Mainstone II (PXA27x)";
+    mc->init = mainstone_init;
 }
 
-machine_init(mainstone_machine_init);
+DEFINE_MACHINE("mainstone", mainstone2_machine_init)
diff --git a/hw/arm/musicpal.c b/hw/arm/musicpal.c
index 3712de6..cc50ace 100644
--- a/hw/arm/musicpal.c
+++ b/hw/arm/musicpal.c
@@ -9,6 +9,10 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
@@ -187,11 +191,6 @@
     le32_to_cpus(&desc->next);
 }
 
-static int eth_can_receive(NetClientState *nc)
-{
-    return 1;
-}
-
 static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     mv88w8618_eth_state *s = qemu_get_nic_opaque(nc);
@@ -379,9 +378,8 @@
 }
 
 static NetClientInfo net_mv88w8618_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = eth_can_receive,
     .receive = eth_receive,
     .cleanup = eth_cleanup,
 };
@@ -1600,13 +1598,12 @@
     }
 
     /* For now we use a fixed - the original - RAM size */
-    memory_region_init_ram(ram, NULL, "musicpal.ram", MP_RAM_DEFAULT_SIZE,
-                           &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "musicpal.ram",
+                                         MP_RAM_DEFAULT_SIZE);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     memory_region_init_ram(sram, NULL, "musicpal.sram", MP_SRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, MP_SRAM_BASE, sram);
 
@@ -1716,18 +1713,13 @@
     arm_load_kernel(cpu, &musicpal_binfo);
 }
 
-static QEMUMachine musicpal_machine = {
-    .name = "musicpal",
-    .desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)",
-    .init = musicpal_init,
-};
-
-static void musicpal_machine_init(void)
+static void musicpal_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&musicpal_machine);
+    mc->desc = "Marvell 88w8618 / MusicPal (ARM926EJ-S)";
+    mc->init = musicpal_init;
 }
 
-machine_init(musicpal_machine_init);
+DEFINE_MACHINE("musicpal", musicpal_machine_init)
 
 static void mv88w8618_wlan_class_init(ObjectClass *klass, void *data)
 {
diff --git a/hw/arm/netduino2.c b/hw/arm/netduino2.c
new file mode 100644
index 0000000..23d7928
--- /dev/null
+++ b/hw/arm/netduino2.c
@@ -0,0 +1,49 @@
+/*
+ * Netduino 2 Machine Model
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
+#include "hw/arm/stm32f205_soc.h"
+
+static void netduino2_init(MachineState *machine)
+{
+    DeviceState *dev;
+
+    dev = qdev_create(NULL, TYPE_STM32F205_SOC);
+    if (machine->kernel_filename) {
+        qdev_prop_set_string(dev, "kernel-filename", machine->kernel_filename);
+    }
+    qdev_prop_set_string(dev, "cpu-model", "cortex-m3");
+    object_property_set_bool(OBJECT(dev), true, "realized", &error_fatal);
+}
+
+static void netduino2_machine_init(MachineClass *mc)
+{
+    mc->desc = "Netduino 2 Machine";
+    mc->init = netduino2_init;
+}
+
+DEFINE_MACHINE("netduino2", netduino2_machine_init)
diff --git a/hw/arm/nseries.c b/hw/arm/nseries.c
index c7ebaa6..fea911e 100644
--- a/hw/arm/nseries.c
+++ b/hw/arm/nseries.c
@@ -18,7 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu/cutils.h"
+#include "qemu/bswap.h"
 #include "sysemu/sysemu.h"
 #include "hw/arm/omap.h"
 #include "hw/arm/arm.h"
@@ -33,6 +37,7 @@
 #include "hw/loader.h"
 #include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
+#include "qemu/log.h"
 #include "exec/address-spaces.h"
 
 /* Nokia N8x0 support */
@@ -133,9 +138,8 @@
 
 static void n8x0_gpio_setup(struct n800_s *s)
 {
-    qemu_irq *mmc_cs = qemu_allocate_irqs(n800_mmc_cs_cb, s->mpu->mmc, 1);
-    qdev_connect_gpio_out(s->mpu->gpio, N8X0_MMC_CS_GPIO, mmc_cs[0]);
-
+    qdev_connect_gpio_out(s->mpu->gpio, N8X0_MMC_CS_GPIO,
+                          qemu_allocate_irq(n800_mmc_cs_cb, s->mpu->mmc, 0));
     qemu_irq_lower(qdev_get_gpio_in(s->mpu->gpio, N800_BAT_COVER_GPIO));
 }
 
@@ -173,8 +177,8 @@
     qdev_prop_set_int32(s->nand, "shift", 1);
     dinfo = drive_get(IF_MTD, 0, 0);
     if (dinfo) {
-        qdev_prop_set_drive_nofail(s->nand, "drive",
-                                   blk_by_legacy_dinfo(dinfo));
+        qdev_prop_set_drive(s->nand, "drive", blk_by_legacy_dinfo(dinfo),
+                            &error_fatal);
     }
     qdev_init_nofail(s->nand);
     sysbus_connect_irq(SYS_BUS_DEVICE(s->nand), 0,
@@ -579,7 +583,10 @@
 
     case 0x26:	/* GAMSET */
         if (!s->pm) {
-            s->gamma = ffs(s->param[0] & 0xf) - 1;
+            s->gamma = ctz32(s->param[0] & 0xf);
+            if (s->gamma == 32) {
+                s->gamma = -1; /* XXX: should this be 0? */
+            }
         } else if (s->pm < 0) {
             s->pm = 1;
         }
@@ -1273,7 +1280,7 @@
     strcpy((void *) w, "hw-build");		/* char component[12] */
     w += 6;
     strcpy((void *) w, "QEMU ");
-    pstrcat((void *) w, 12, qemu_get_version()); /* char version[12] */
+    pstrcat((void *) w, 12, qemu_hw_version()); /* char version[12] */
     w += 6;
 
     tag = (model == 810) ? "1.1.10-qemu" : "1.1.6-qemu";
@@ -1344,7 +1351,7 @@
     n8x0_dss_setup(s);
     n8x0_cbus_setup(s);
     n8x0_uart_setup(s);
-    if (usb_enabled(false)) {
+    if (machine_usb(machine)) {
         n8x0_usb_setup(s);
     }
 
@@ -1360,7 +1367,7 @@
 
     if (option_rom[0].name &&
         (machine->boot_order[0] == 'n' || !machine->kernel_filename)) {
-        uint8_t nolo_tags[0x10000];
+        uint8_t *nolo_tags = g_new(uint8_t, 0x10000);
         /* No, wait, better start at the ROM.  */
         s->mpu->cpu->env.regs[15] = OMAP2_Q2_BASE + 0x400000;
 
@@ -1379,6 +1386,7 @@
 
         n800_setup_nolo_tags(nolo_tags);
         cpu_physical_memory_write(OMAP2_SRAM_BASE, nolo_tags, 0x10000);
+        g_free(nolo_tags);
     }
 }
 
@@ -1403,32 +1411,48 @@
 
 static void n800_init(MachineState *machine)
 {
-    return n8x0_init(machine, &n800_binfo, 800);
+    n8x0_init(machine, &n800_binfo, 800);
 }
 
 static void n810_init(MachineState *machine)
 {
-    return n8x0_init(machine, &n810_binfo, 810);
+    n8x0_init(machine, &n810_binfo, 810);
 }
 
-static QEMUMachine n800_machine = {
-    .name = "n800",
-    .desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)",
-    .init = n800_init,
-    .default_boot_order = "",
+static void n800_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Nokia N800 tablet aka. RX-34 (OMAP2420)";
+    mc->init = n800_init;
+    mc->default_boot_order = "";
+}
+
+static const TypeInfo n800_type = {
+    .name = MACHINE_TYPE_NAME("n800"),
+    .parent = TYPE_MACHINE,
+    .class_init = n800_class_init,
 };
 
-static QEMUMachine n810_machine = {
-    .name = "n810",
-    .desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)",
-    .init = n810_init,
-    .default_boot_order = "",
+static void n810_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Nokia N810 tablet aka. RX-44 (OMAP2420)";
+    mc->init = n810_init;
+    mc->default_boot_order = "";
+}
+
+static const TypeInfo n810_type = {
+    .name = MACHINE_TYPE_NAME("n810"),
+    .parent = TYPE_MACHINE,
+    .class_init = n810_class_init,
 };
 
 static void nseries_machine_init(void)
 {
-    qemu_register_machine(&n800_machine);
-    qemu_register_machine(&n810_machine);
+    type_register_static(&n800_type);
+    type_register_static(&n810_type);
 }
 
-machine_init(nseries_machine_init);
+type_init(nseries_machine_init)
diff --git a/hw/arm/omap1.c b/hw/arm/omap1.c
index abb183c..b3cf0ec 100644
--- a/hw/arm/omap1.c
+++ b/hw/arm/omap1.c
@@ -16,6 +16,12 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/boards.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/omap.h"
@@ -25,6 +31,8 @@
 #include "sysemu/blockdev.h"
 #include "qemu/range.h"
 #include "hw/sysbus.h"
+#include "qemu/cutils.h"
+#include "qemu/bcd.h"
 
 /* Should signal the TCMI/GPMC */
 uint32_t omap_badwidth_read8(void *opaque, hwaddr addr)
@@ -104,7 +112,7 @@
 
     if (timer->st && timer->enable && timer->rate)
         return timer->val - muldiv64(distance >> (timer->ptv + 1),
-                                     timer->rate, get_ticks_per_sec());
+                                     timer->rate, NANOSECONDS_PER_SECOND);
     else
         return timer->val;
 }
@@ -122,7 +130,7 @@
     if (timer->enable && timer->st && timer->rate) {
         timer->val = timer->reset_val;	/* Should skip this on clk enable */
         expires = muldiv64((uint64_t) timer->val << (timer->ptv + 1),
-                           get_ticks_per_sec(), timer->rate);
+                           NANOSECONDS_PER_SECOND, timer->rate);
 
         /* If timer expiry would be sooner than in about 1 ms and
          * auto-reload isn't set, then fire immediately.  This is a hack
@@ -130,10 +138,11 @@
          * sets the interval to a very low value and polls the status bit
          * in a busy loop when it wants to sleep just a couple of CPU
          * ticks.  */
-        if (expires > (get_ticks_per_sec() >> 10) || timer->ar)
+        if (expires > (NANOSECONDS_PER_SECOND >> 10) || timer->ar) {
             timer_mod(timer->timer, timer->time + expires);
-        else
+        } else {
             qemu_bh_schedule(timer->tick);
+        }
     } else
         timer_del(timer->timer);
 }
@@ -207,7 +216,8 @@
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -255,8 +265,7 @@
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *)
-            g_malloc0(sizeof(struct omap_mpu_timer_s));
+    struct omap_mpu_timer_s *s = g_new0(struct omap_mpu_timer_s, 1);
 
     s->irq = irq;
     s->clk = clk;
@@ -314,7 +323,8 @@
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *) opaque;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -384,8 +394,7 @@
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *)
-            g_malloc0(sizeof(struct omap_watchdog_timer_s));
+    struct omap_watchdog_timer_s *s = g_new0(struct omap_watchdog_timer_s, 1);
 
     s->timer.irq = irq;
     s->timer.clk = clk;
@@ -440,7 +449,8 @@
     int offset = addr & OMAP_MPUI_REG_MASK;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -490,8 +500,7 @@
                 hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
-    struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *)
-            g_malloc0(sizeof(struct omap_32khz_timer_s));
+    struct omap_32khz_timer_s *s = g_new0(struct omap_32khz_timer_s, 1);
 
     s->timer.irq = irq;
     s->timer.clk = clk;
@@ -585,7 +594,8 @@
     uint16_t diff;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -609,14 +619,14 @@
                 now -= s->ulpd_gauge_start;
 
                 /* 32-kHz ticks */
-                ticks = muldiv64(now, 32768, get_ticks_per_sec());
+                ticks = muldiv64(now, 32768, NANOSECONDS_PER_SECOND);
                 s->ulpd_pm_regs[0x00 >> 2] = (ticks >>  0) & 0xffff;
                 s->ulpd_pm_regs[0x04 >> 2] = (ticks >> 16) & 0xffff;
                 if (ticks >> 32)	/* OVERFLOW_32K */
                     s->ulpd_pm_regs[0x14 >> 2] |= 1 << 2;
 
                 /* High frequency ticks */
-                ticks = muldiv64(now, 12000000, get_ticks_per_sec());
+                ticks = muldiv64(now, 12000000, NANOSECONDS_PER_SECOND);
                 s->ulpd_pm_regs[0x08 >> 2] = (ticks >>  0) & 0xffff;
                 s->ulpd_pm_regs[0x0c >> 2] = (ticks >> 16) & 0xffff;
                 if (ticks >> 32)	/* OVERFLOW_HI_FREQ */
@@ -857,7 +867,8 @@
     uint32_t diff;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1012,7 +1023,8 @@
                           uint64_t value, unsigned size)
 {
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     OMAP_BAD_REG(addr);
@@ -1081,7 +1093,8 @@
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1175,7 +1188,8 @@
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *) opaque;
 
     if (size < 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1226,8 +1240,7 @@
     MemoryRegion *memory, hwaddr base,
     qemu_irq abort_irq, omap_clk clk)
 {
-    struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *)
-            g_malloc0(sizeof(struct omap_tipb_bridge_s));
+    struct omap_tipb_bridge_s *s = g_new0(struct omap_tipb_bridge_s, 1);
 
     s->abort = abort_irq;
     omap_tipb_bridge_reset(s);
@@ -1284,7 +1297,8 @@
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1379,7 +1393,8 @@
     int div, mult;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     if (addr == 0x00) {	/* CTL_REG */
@@ -1647,7 +1662,8 @@
     };
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1775,7 +1791,8 @@
     uint16_t diff;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1982,15 +1999,15 @@
     int ln;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
     case 0x04:	/* OUTPUT_REG */
         diff = (s->outputs ^ value) & ~s->dir;
         s->outputs = value;
-        while ((ln = ffs(diff))) {
-            ln --;
+        while ((ln = ctz32(diff)) != 32) {
             if (s->handler[ln])
                 qemu_set_irq(s->handler[ln], (value >> ln) & 1);
             diff &= ~(1 << ln);
@@ -2002,8 +2019,7 @@
         s->dir = value;
 
         value = s->outputs & ~s->dir;
-        while ((ln = ffs(diff))) {
-            ln --;
+        while ((ln = ctz32(diff)) != 32) {
             if (s->handler[ln])
                 qemu_set_irq(s->handler[ln], (value >> ln) & 1);
             diff &= ~(1 << ln);
@@ -2086,8 +2102,7 @@
                 qemu_irq kbd_int, qemu_irq gpio_int, qemu_irq wakeup,
                 omap_clk clk)
 {
-    struct omap_mpuio_s *s = (struct omap_mpuio_s *)
-            g_malloc0(sizeof(struct omap_mpuio_s));
+    struct omap_mpuio_s *s = g_new0(struct omap_mpuio_s, 1);
 
     s->irq = gpio_int;
     s->kbd_irq = kbd_int;
@@ -2210,7 +2225,8 @@
     int offset = addr & OMAP_MPUI_REG_MASK;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -2278,8 +2294,7 @@
                                             qemu_irq dma,
                                             omap_clk clk)
 {
-    struct omap_uwire_s *s = (struct omap_uwire_s *)
-            g_malloc0(sizeof(struct omap_uwire_s));
+    struct omap_uwire_s *s = g_new0(struct omap_uwire_s, 1);
 
     s->txirq = txirq;
     s->rxirq = rxirq;
@@ -2349,7 +2364,8 @@
     int offset = addr & OMAP_MPUI_REG_MASK;
 
     if (size != 1) {
-        return omap_badwidth_write8(opaque, addr, value);
+        omap_badwidth_write8(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -2444,7 +2460,8 @@
     int offset = addr & OMAP_MPUI_REG_MASK;
 
     if (size != 1) {
-        return omap_badwidth_write8(opaque, addr, value);
+        omap_badwidth_write8(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -2637,7 +2654,8 @@
     time_t ti[2];
 
     if (size != 1) {
-        return omap_badwidth_write8(opaque, addr, value);
+        omap_badwidth_write8(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -2915,8 +2933,7 @@
                                         qemu_irq timerirq, qemu_irq alarmirq,
                                         omap_clk clk)
 {
-    struct omap_rtc_s *s = (struct omap_rtc_s *)
-            g_malloc0(sizeof(struct omap_rtc_s));
+    struct omap_rtc_s *s = g_new0(struct omap_rtc_s, 1);
 
     s->irq = timerirq;
     s->alarm = alarmirq;
@@ -3015,7 +3032,7 @@
 
     omap_mcbsp_rx_newdata(s);
     timer_mod(s->source_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec());
+                   NANOSECONDS_PER_SECOND);
 }
 
 static void omap_mcbsp_rx_start(struct omap_mcbsp_s *s)
@@ -3061,7 +3078,7 @@
 
     omap_mcbsp_tx_newdata(s);
     timer_mod(s->sink_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec());
+                   NANOSECONDS_PER_SECOND);
 }
 
 static void omap_mcbsp_tx_start(struct omap_mcbsp_s *s)
@@ -3410,9 +3427,14 @@
                              uint64_t value, unsigned size)
 {
     switch (size) {
-    case 2: return omap_mcbsp_writeh(opaque, addr, value);
-    case 4: return omap_mcbsp_writew(opaque, addr, value);
-    default: return omap_badwidth_write16(opaque, addr, value);
+    case 2:
+        omap_mcbsp_writeh(opaque, addr, value);
+        break;
+    case 4:
+        omap_mcbsp_writew(opaque, addr, value);
+        break;
+    default:
+        omap_badwidth_write16(opaque, addr, value);
     }
 }
 
@@ -3446,8 +3468,7 @@
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq *dma, omap_clk clk)
 {
-    struct omap_mcbsp_s *s = (struct omap_mcbsp_s *)
-            g_malloc0(sizeof(struct omap_mcbsp_s));
+    struct omap_mcbsp_s *s = g_new0(struct omap_mcbsp_s, 1);
 
     s->txirq = txirq;
     s->rxirq = rxirq;
@@ -3586,7 +3607,8 @@
     int offset = addr & OMAP_MPUI_REG_MASK;
 
     if (size != 1) {
-        return omap_badwidth_write8(opaque, addr, value);
+        omap_badwidth_write8(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -3625,8 +3647,7 @@
 static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
                                         hwaddr base, omap_clk clk)
 {
-    struct omap_lpg_s *s = (struct omap_lpg_s *)
-            g_malloc0(sizeof(struct omap_lpg_s));
+    struct omap_lpg_s *s = g_new0(struct omap_lpg_s, 1);
 
     s->tm = timer_new_ms(QEMU_CLOCK_VIRTUAL, omap_lpg_tick, s);
 
@@ -3830,8 +3851,7 @@
                 const char *core)
 {
     int i;
-    struct omap_mpu_state_s *s = (struct omap_mpu_state_s *)
-            g_malloc0(sizeof(struct omap_mpu_state_s));
+    struct omap_mpu_state_s *s = g_new0(struct omap_mpu_state_s, 1);
     qemu_irq dma_irqs[6];
     DriveInfo *dinfo;
     SysBusDevice *busdev;
@@ -3855,12 +3875,11 @@
     omap_clk_init(s);
 
     /* Memory-mapped stuff */
-    memory_region_init_ram(&s->emiff_ram, NULL, "omap1.dram", s->sdram_size,
-                           &error_abort);
-    vmstate_register_ram_global(&s->emiff_ram);
+    memory_region_allocate_system_memory(&s->emiff_ram, NULL, "omap1.dram",
+                                         s->sdram_size);
     memory_region_add_subregion(system_memory, OMAP_EMIFF_BASE, &s->emiff_ram);
     memory_region_init_ram(&s->imif_ram, NULL, "omap1.sram", s->sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->imif_ram);
     memory_region_add_subregion(system_memory, OMAP_IMIF_BASE, &s->imif_ram);
 
diff --git a/hw/arm/omap2.c b/hw/arm/omap2.c
index b083ebe..3a0d777 100644
--- a/hw/arm/omap2.c
+++ b/hw/arm/omap2.c
@@ -18,8 +18,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
+#include "hw/boards.h"
 #include "hw/hw.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/omap.h"
@@ -447,7 +452,8 @@
     struct omap_eac_s *s = (struct omap_eac_s *) opaque;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -594,8 +600,7 @@
 static struct omap_eac_s *omap_eac_init(struct omap_target_agent_s *ta,
                 qemu_irq irq, qemu_irq *drq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_eac_s *s = (struct omap_eac_s *)
-            g_malloc0(sizeof(struct omap_eac_s));
+    struct omap_eac_s *s = g_new0(struct omap_eac_s, 1);
 
     s->irq = irq;
     s->codec.rxdrq = *drq ++;
@@ -692,7 +697,8 @@
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -757,7 +763,8 @@
     uint8_t byte = value;
 
     if (size != 1) {
-        return omap_badwidth_write8(opaque, addr, size);
+        omap_badwidth_write8(opaque, addr, size);
+        return;
     }
 
     if (ch == STI_TRACE_CONTROL_CHANNEL) {
@@ -784,8 +791,7 @@
                 hwaddr channel_base, qemu_irq irq, omap_clk clk,
                 CharDriverState *chr)
 {
-    struct omap_sti_s *s = (struct omap_sti_s *)
-            g_malloc0(sizeof(struct omap_sti_s));
+    struct omap_sti_s *s = g_new0(struct omap_sti_s, 1);
 
     s->irq = irq;
     omap_sti_reset(s);
@@ -1359,7 +1365,8 @@
     struct omap_prcm_s *s = (struct omap_prcm_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1801,8 +1808,7 @@
                 qemu_irq mpu_int, qemu_irq dsp_int, qemu_irq iva_int,
                 struct omap_mpu_state_s *mpu)
 {
-    struct omap_prcm_s *s = (struct omap_prcm_s *)
-            g_malloc0(sizeof(struct omap_prcm_s));
+    struct omap_prcm_s *s = g_new0(struct omap_prcm_s, 1);
 
     s->irq[0] = mpu_int;
     s->irq[1] = dsp_int;
@@ -2180,8 +2186,7 @@
 static struct omap_sysctl_s *omap_sysctl_init(struct omap_target_agent_s *ta,
                 omap_clk iclk, struct omap_mpu_state_s *mpu)
 {
-    struct omap_sysctl_s *s = (struct omap_sysctl_s *)
-            g_malloc0(sizeof(struct omap_sysctl_s));
+    struct omap_sysctl_s *s = g_new0(struct omap_sysctl_s, 1);
 
     s->mpu = mpu;
     omap_sysctl_reset(s);
@@ -2243,8 +2248,7 @@
                 unsigned long sdram_size,
                 const char *core)
 {
-    struct omap_mpu_state_s *s = (struct omap_mpu_state_s *)
-            g_malloc0(sizeof(struct omap_mpu_state_s));
+    struct omap_mpu_state_s *s = g_new0(struct omap_mpu_state_s, 1);
     qemu_irq dma_irqs[4];
     DriveInfo *dinfo;
     int i;
@@ -2267,12 +2271,11 @@
     omap_clk_init(s);
 
     /* Memory-mapped stuff */
-    memory_region_init_ram(&s->sdram, NULL, "omap2.dram", s->sdram_size,
-                           &error_abort);
-    vmstate_register_ram_global(&s->sdram);
+    memory_region_allocate_system_memory(&s->sdram, NULL, "omap2.dram",
+                                         s->sdram_size);
     memory_region_add_subregion(sysmem, OMAP2_Q2_BASE, &s->sdram);
     memory_region_init_ram(&s->sram, NULL, "omap2.sram", s->sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sram);
     memory_region_add_subregion(sysmem, OMAP2_SRAM_BASE, &s->sram);
 
diff --git a/hw/arm/omap_sx1.c b/hw/arm/omap_sx1.c
index 671e02c..5d74026 100644
--- a/hw/arm/omap_sx1.c
+++ b/hw/arm/omap_sx1.c
@@ -25,6 +25,8 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/omap.h"
@@ -103,7 +105,6 @@
     struct omap_mpu_state_s *mpu;
     MemoryRegion *address_space = get_system_memory();
     MemoryRegion *flash = g_new(MemoryRegion, 1);
-    MemoryRegion *flash_1 = g_new(MemoryRegion, 1);
     MemoryRegion *cs = g_new(MemoryRegion, 4);
     static uint32_t cs0val = 0x00213090;
     static uint32_t cs1val = 0x00215070;
@@ -123,7 +124,7 @@
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "omap_sx1.flash0-0", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space, OMAP_CS0_BASE, flash);
@@ -165,8 +166,9 @@
 
     if ((version == 1) &&
             (dinfo = drive_get(IF_PFLASH, 0, fl_idx)) != NULL) {
+        MemoryRegion *flash_1 = g_new(MemoryRegion, 1);
         memory_region_init_ram(flash_1, NULL, "omap_sx1.flash1-0", flash1_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(flash_1);
         memory_region_set_readonly(flash_1, true);
         memory_region_add_subregion(address_space, OMAP_CS1_BASE, flash_1);
@@ -217,22 +219,38 @@
     sx1_init(machine, 2);
 }
 
-static QEMUMachine sx1_machine_v2 = {
-    .name = "sx1",
-    .desc = "Siemens SX1 (OMAP310) V2",
-    .init = sx1_init_v2,
+static void sx1_machine_v2_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Siemens SX1 (OMAP310) V2";
+    mc->init = sx1_init_v2;
+}
+
+static const TypeInfo sx1_machine_v2_type = {
+    .name = MACHINE_TYPE_NAME("sx1"),
+    .parent = TYPE_MACHINE,
+    .class_init = sx1_machine_v2_class_init,
 };
 
-static QEMUMachine sx1_machine_v1 = {
-    .name = "sx1-v1",
-    .desc = "Siemens SX1 (OMAP310) V1",
-    .init = sx1_init_v1,
+static void sx1_machine_v1_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Siemens SX1 (OMAP310) V1";
+    mc->init = sx1_init_v1;
+}
+
+static const TypeInfo sx1_machine_v1_type = {
+    .name = MACHINE_TYPE_NAME("sx1-v1"),
+    .parent = TYPE_MACHINE,
+    .class_init = sx1_machine_v1_class_init,
 };
 
 static void sx1_machine_init(void)
 {
-    qemu_register_machine(&sx1_machine_v2);
-    qemu_register_machine(&sx1_machine_v1);
+    type_register_static(&sx1_machine_v1_type);
+    type_register_static(&sx1_machine_v2_type);
 }
 
-machine_init(sx1_machine_init);
+type_init(sx1_machine_init)
diff --git a/hw/arm/palm.c b/hw/arm/palm.c
index 7f1cfb8..7f46073 100644
--- a/hw/arm/palm.c
+++ b/hw/arm/palm.c
@@ -16,6 +16,8 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "sysemu/sysemu.h"
@@ -213,7 +215,7 @@
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "palmte.flash", flash_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(flash);
     memory_region_set_readonly(flash, true);
     memory_region_add_subregion(address_space_mem, OMAP_CS0_BASE, flash);
@@ -269,15 +271,10 @@
     arm_load_kernel(mpu->cpu, &palmte_binfo);
 }
 
-static QEMUMachine palmte_machine = {
-    .name = "cheetah",
-    .desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)",
-    .init = palmte_init,
-};
-
-static void palmte_machine_init(void)
+static void palmte_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&palmte_machine);
+    mc->desc = "Palm Tungsten|E aka. Cheetah PDA (OMAP310)";
+    mc->init = palmte_init;
 }
 
-machine_init(palmte_machine_init);
+DEFINE_MACHINE("cheetah", palmte_machine_init)
diff --git a/hw/arm/palmetto-bmc.c b/hw/arm/palmetto-bmc.c
new file mode 100644
index 0000000..54e29a8
--- /dev/null
+++ b/hw/arm/palmetto-bmc.c
@@ -0,0 +1,102 @@
+/*
+ * OpenPOWER Palmetto BMC
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/address-spaces.h"
+#include "hw/arm/arm.h"
+#include "hw/arm/ast2400.h"
+#include "hw/boards.h"
+#include "qemu/log.h"
+#include "sysemu/block-backend.h"
+#include "sysemu/blockdev.h"
+
+static struct arm_boot_info palmetto_bmc_binfo = {
+    .loader_start = AST2400_SDRAM_BASE,
+    .board_id = 0,
+    .nb_cpus = 1,
+};
+
+typedef struct PalmettoBMCState {
+    AST2400State soc;
+    MemoryRegion ram;
+} PalmettoBMCState;
+
+static void palmetto_bmc_init_flashes(AspeedSMCState *s, const char *flashtype,
+                                      Error **errp)
+{
+    int i ;
+
+    for (i = 0; i < s->num_cs; ++i) {
+        AspeedSMCFlash *fl = &s->flashes[i];
+        DriveInfo *dinfo = drive_get_next(IF_MTD);
+        qemu_irq cs_line;
+
+        /*
+         * FIXME: check that we are not using a flash module exceeding
+         * the controller segment size
+         */
+        fl->flash = ssi_create_slave_no_init(s->spi, flashtype);
+        if (dinfo) {
+            qdev_prop_set_drive(fl->flash, "drive", blk_by_legacy_dinfo(dinfo),
+                                errp);
+        }
+        qdev_init_nofail(fl->flash);
+
+        cs_line = qdev_get_gpio_in_named(fl->flash, SSI_GPIO_CS, 0);
+        sysbus_connect_irq(SYS_BUS_DEVICE(s), i + 1, cs_line);
+    }
+}
+
+static void palmetto_bmc_init(MachineState *machine)
+{
+    PalmettoBMCState *bmc;
+
+    bmc = g_new0(PalmettoBMCState, 1);
+    object_initialize(&bmc->soc, (sizeof(bmc->soc)), TYPE_AST2400);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&bmc->soc),
+                              &error_abort);
+
+    memory_region_allocate_system_memory(&bmc->ram, NULL, "ram", ram_size);
+    memory_region_add_subregion(get_system_memory(), AST2400_SDRAM_BASE,
+                                &bmc->ram);
+    object_property_add_const_link(OBJECT(&bmc->soc), "ram", OBJECT(&bmc->ram),
+                                   &error_abort);
+    object_property_set_int(OBJECT(&bmc->soc), 0x120CE416, "hw-strap1",
+                            &error_abort);
+    object_property_set_bool(OBJECT(&bmc->soc), true, "realized",
+                             &error_abort);
+
+    palmetto_bmc_init_flashes(&bmc->soc.smc, "n25q256a", &error_abort);
+    palmetto_bmc_init_flashes(&bmc->soc.spi, "mx25l25635e", &error_abort);
+
+    palmetto_bmc_binfo.kernel_filename = machine->kernel_filename;
+    palmetto_bmc_binfo.initrd_filename = machine->initrd_filename;
+    palmetto_bmc_binfo.kernel_cmdline = machine->kernel_cmdline;
+    palmetto_bmc_binfo.ram_size = ram_size;
+    arm_load_kernel(ARM_CPU(first_cpu), &palmetto_bmc_binfo);
+}
+
+static void palmetto_bmc_machine_init(MachineClass *mc)
+{
+    mc->desc = "OpenPOWER Palmetto BMC";
+    mc->init = palmetto_bmc_init;
+    mc->max_cpus = 1;
+    mc->no_sdcard = 1;
+    mc->no_floppy = 1;
+    mc->no_cdrom = 1;
+    mc->no_sdcard = 1;
+    mc->no_parallel = 1;
+}
+
+DEFINE_MACHINE("palmetto-bmc", palmetto_bmc_machine_init);
diff --git a/hw/arm/pxa2xx.c b/hw/arm/pxa2xx.c
index 693dfec..cb55704 100644
--- a/hw/arm/pxa2xx.c
+++ b/hw/arm/pxa2xx.c
@@ -7,15 +7,20 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/pxa.h"
 #include "sysemu/sysemu.h"
 #include "hw/char/serial.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "sysemu/char.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
+#include "qemu/cutils.h"
 
 static struct {
     hwaddr io_base;
@@ -273,10 +278,10 @@
     case 3:
         s->cpu->env.uncached_cpsr = ARM_CPU_MODE_SVC;
         s->cpu->env.daif = PSTATE_A | PSTATE_F | PSTATE_I;
-        s->cpu->env.cp15.c1_sys = 0;
-        s->cpu->env.cp15.c1_coproc = 0;
-        s->cpu->env.cp15.ttbr0_el1 = 0;
-        s->cpu->env.cp15.c3 = 0;
+        s->cpu->env.cp15.sctlr_ns = 0;
+        s->cpu->env.cp15.cpacr_el1 = 0;
+        s->cpu->env.cp15.ttbr0_el[1] = 0;
+        s->cpu->env.cp15.dacr_ns = 0;
         s->pm_regs[PSSR >> 2] |= 0x8; /* Set STS */
         s->pm_regs[RCSR >> 2] |= 0x8; /* Set GPR */
 
@@ -334,10 +339,10 @@
 static const ARMCPRegInfo pxa_cp_reginfo[] = {
     /* cp14 crm==1: perf registers */
     { .name = "CPPMNC", .cp = 14, .crn = 0, .crm = 1, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_IO,
       .readfn = pxa2xx_cppmnc_read, .writefn = pxa2xx_cppmnc_write },
     { .name = "CPCCNT", .cp = 14, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_IO,
       .readfn = pxa2xx_cpccnt_read, .writefn = arm_cp_write_ignore },
     { .name = "CPINTEN", .cp = 14, .crn = 4, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
@@ -356,11 +361,11 @@
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
     /* cp14 crn==6: CLKCFG */
     { .name = "CLKCFG", .cp = 14, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_IO,
       .readfn = pxa2xx_clkcfg_read, .writefn = pxa2xx_clkcfg_write },
     /* cp14 crn==7: PWRMODE */
     { .name = "PWRMODE", .cp = 14, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW,
+      .access = PL1_RW, .type = ARM_CP_IO,
       .readfn = arm_cp_read_zero, .writefn = pxa2xx_pwrmode_write },
     REGINFO_SENTINEL
 };
@@ -457,7 +462,7 @@
 
     MemoryRegion iomem;
     qemu_irq irq;
-    int enable;
+    uint32_t enable;
     SSIBus *bus;
 
     uint32_t sscr[2];
@@ -470,10 +475,39 @@
     uint8_t ssacd;
 
     uint32_t rx_fifo[16];
-    int rx_level;
-    int rx_start;
+    uint32_t rx_level;
+    uint32_t rx_start;
 } PXA2xxSSPState;
 
+static bool pxa2xx_ssp_vmstate_validate(void *opaque, int version_id)
+{
+    PXA2xxSSPState *s = opaque;
+
+    return s->rx_start < sizeof(s->rx_fifo);
+}
+
+static const VMStateDescription vmstate_pxa2xx_ssp = {
+    .name = "pxa2xx-ssp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(enable, PXA2xxSSPState),
+        VMSTATE_UINT32_ARRAY(sscr, PXA2xxSSPState, 2),
+        VMSTATE_UINT32(sspsp, PXA2xxSSPState),
+        VMSTATE_UINT32(ssto, PXA2xxSSPState),
+        VMSTATE_UINT32(ssitr, PXA2xxSSPState),
+        VMSTATE_UINT32(sssr, PXA2xxSSPState),
+        VMSTATE_UINT8(sstsa, PXA2xxSSPState),
+        VMSTATE_UINT8(ssrsa, PXA2xxSSPState),
+        VMSTATE_UINT8(ssacd, PXA2xxSSPState),
+        VMSTATE_UINT32(rx_level, PXA2xxSSPState),
+        VMSTATE_UINT32(rx_start, PXA2xxSSPState),
+        VMSTATE_VALIDATE("fifo is 16 bytes", pxa2xx_ssp_vmstate_validate),
+        VMSTATE_UINT32_ARRAY(rx_fifo, PXA2xxSSPState, 16),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 #define SSCR0	0x00	/* SSP Control register 0 */
 #define SSCR1	0x04	/* SSP Control register 1 */
 #define SSSR	0x08	/* SSP Status register */
@@ -705,55 +739,20 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pxa2xx_ssp_save(QEMUFile *f, void *opaque)
+static void pxa2xx_ssp_reset(DeviceState *d)
 {
-    PXA2xxSSPState *s = (PXA2xxSSPState *) opaque;
-    int i;
+    PXA2xxSSPState *s = PXA2XX_SSP(d);
 
-    qemu_put_be32(f, s->enable);
-
-    qemu_put_be32s(f, &s->sscr[0]);
-    qemu_put_be32s(f, &s->sscr[1]);
-    qemu_put_be32s(f, &s->sspsp);
-    qemu_put_be32s(f, &s->ssto);
-    qemu_put_be32s(f, &s->ssitr);
-    qemu_put_be32s(f, &s->sssr);
-    qemu_put_8s(f, &s->sstsa);
-    qemu_put_8s(f, &s->ssrsa);
-    qemu_put_8s(f, &s->ssacd);
-
-    qemu_put_byte(f, s->rx_level);
-    for (i = 0; i < s->rx_level; i ++)
-        qemu_put_byte(f, s->rx_fifo[(s->rx_start + i) & 0xf]);
-}
-
-static int pxa2xx_ssp_load(QEMUFile *f, void *opaque, int version_id)
-{
-    PXA2xxSSPState *s = (PXA2xxSSPState *) opaque;
-    int i, v;
-
-    s->enable = qemu_get_be32(f);
-
-    qemu_get_be32s(f, &s->sscr[0]);
-    qemu_get_be32s(f, &s->sscr[1]);
-    qemu_get_be32s(f, &s->sspsp);
-    qemu_get_be32s(f, &s->ssto);
-    qemu_get_be32s(f, &s->ssitr);
-    qemu_get_be32s(f, &s->sssr);
-    qemu_get_8s(f, &s->sstsa);
-    qemu_get_8s(f, &s->ssrsa);
-    qemu_get_8s(f, &s->ssacd);
-
-    v = qemu_get_byte(f);
-    if (v < 0 || v > ARRAY_SIZE(s->rx_fifo)) {
-        return -EINVAL;
-    }
-    s->rx_level = v;
-    s->rx_start = 0;
-    for (i = 0; i < s->rx_level; i ++)
-        s->rx_fifo[i] = qemu_get_byte(f);
-
-    return 0;
+    s->enable = 0;
+    s->sscr[0] = s->sscr[1] = 0;
+    s->sspsp = 0;
+    s->ssto = 0;
+    s->ssitr = 0;
+    s->sssr = 0;
+    s->sstsa = 0;
+    s->ssrsa = 0;
+    s->ssacd = 0;
+    s->rx_start = s->rx_level = 0;
 }
 
 static int pxa2xx_ssp_init(SysBusDevice *sbd)
@@ -766,8 +765,6 @@
     memory_region_init_io(&s->iomem, OBJECT(s), &pxa2xx_ssp_ops, s,
                           "pxa2xx-ssp", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
-    register_savevm(dev, "pxa2xx_ssp", -1, 0,
-                    pxa2xx_ssp_save, pxa2xx_ssp_load, s);
 
     s->bus = ssi_create_bus(dev, "ssi");
     return 0;
@@ -1110,9 +1107,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int pxa2xx_rtc_init(SysBusDevice *dev)
+static void pxa2xx_rtc_init(Object *obj)
 {
-    PXA2xxRTCState *s = PXA2XX_RTC(dev);
+    PXA2xxRTCState *s = PXA2XX_RTC(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     struct tm tm;
     int wom;
 
@@ -1141,11 +1139,9 @@
 
     sysbus_init_irq(dev, &s->rtc_irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pxa2xx_rtc_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_rtc_ops, s,
                           "pxa2xx-rtc", 0x10000);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void pxa2xx_rtc_pre_save(void *opaque)
@@ -1198,9 +1194,7 @@
 static void pxa2xx_rtc_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pxa2xx_rtc_init;
     dc->desc = "PXA2xx RTC Controller";
     dc->vmsd = &vmstate_pxa2xx_rtc_regs;
 }
@@ -1209,6 +1203,7 @@
     .name          = TYPE_PXA2XX_RTC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxRTCState),
+    .instance_init = pxa2xx_rtc_init,
     .class_init    = pxa2xx_rtc_sysbus_class_init,
 };
 
@@ -1504,19 +1499,18 @@
     return s;
 }
 
-static int pxa2xx_i2c_initfn(SysBusDevice *sbd)
+static void pxa2xx_i2c_initfn(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    PXA2xxI2CState *s = PXA2XX_I2C(dev);
+    DeviceState *dev = DEVICE(obj);
+    PXA2xxI2CState *s = PXA2XX_I2C(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     s->bus = i2c_init_bus(dev, "i2c");
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pxa2xx_i2c_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_i2c_ops, s,
                           "pxa2xx-i2c", s->region_size);
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq);
-
-    return 0;
 }
 
 I2CBus *pxa2xx_i2c_bus(PXA2xxI2CState *s)
@@ -1533,9 +1527,7 @@
 static void pxa2xx_i2c_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pxa2xx_i2c_initfn;
     dc->desc = "PXA2xx I2C Bus Controller";
     dc->vmsd = &vmstate_pxa2xx_i2c;
     dc->props = pxa2xx_i2c_properties;
@@ -1545,6 +1537,7 @@
     .name          = TYPE_PXA2XX_I2C,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxI2CState),
+    .instance_init = pxa2xx_i2c_initfn,
     .class_init    = pxa2xx_i2c_class_init,
 };
 
@@ -1739,8 +1732,7 @@
                 hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma)
 {
-    PXA2xxI2SState *s = (PXA2xxI2SState *)
-            g_malloc0(sizeof(PXA2xxI2SState));
+    PXA2xxI2SState *s = g_new0(PXA2xxI2SState, 1);
 
     s->irq = irq;
     s->rx_dma = rx_dma;
@@ -1759,24 +1751,33 @@
 }
 
 /* PXA Fast Infra-red Communications Port */
+#define TYPE_PXA2XX_FIR "pxa2xx-fir"
+#define PXA2XX_FIR(obj) OBJECT_CHECK(PXA2xxFIrState, (obj), TYPE_PXA2XX_FIR)
+
 struct PXA2xxFIrState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
     MemoryRegion iomem;
     qemu_irq irq;
     qemu_irq rx_dma;
     qemu_irq tx_dma;
-    int enable;
+    uint32_t enable;
     CharDriverState *chr;
 
     uint8_t control[3];
     uint8_t status[2];
 
-    int rx_len;
-    int rx_start;
+    uint32_t rx_len;
+    uint32_t rx_start;
     uint8_t rx_fifo[64];
 };
 
-static void pxa2xx_fir_reset(PXA2xxFIrState *s)
+static void pxa2xx_fir_reset(DeviceState *d)
 {
+    PXA2xxFIrState *s = PXA2XX_FIR(d);
+
     s->control[0] = 0x00;
     s->control[1] = 0x00;
     s->control[2] = 0x00;
@@ -1953,73 +1954,94 @@
 {
 }
 
-static void pxa2xx_fir_save(QEMUFile *f, void *opaque)
+static void pxa2xx_fir_instance_init(Object *obj)
 {
-    PXA2xxFIrState *s = (PXA2xxFIrState *) opaque;
-    int i;
+    PXA2xxFIrState *s = PXA2XX_FIR(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    qemu_put_be32(f, s->enable);
-
-    qemu_put_8s(f, &s->control[0]);
-    qemu_put_8s(f, &s->control[1]);
-    qemu_put_8s(f, &s->control[2]);
-    qemu_put_8s(f, &s->status[0]);
-    qemu_put_8s(f, &s->status[1]);
-
-    qemu_put_byte(f, s->rx_len);
-    for (i = 0; i < s->rx_len; i ++)
-        qemu_put_byte(f, s->rx_fifo[(s->rx_start + i) & 63]);
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_fir_ops, s,
+                          "pxa2xx-fir", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    sysbus_init_irq(sbd, &s->rx_dma);
+    sysbus_init_irq(sbd, &s->tx_dma);
 }
 
-static int pxa2xx_fir_load(QEMUFile *f, void *opaque, int version_id)
+static void pxa2xx_fir_realize(DeviceState *dev, Error **errp)
 {
-    PXA2xxFIrState *s = (PXA2xxFIrState *) opaque;
-    int i;
+    PXA2xxFIrState *s = PXA2XX_FIR(dev);
 
-    s->enable = qemu_get_be32(f);
-
-    qemu_get_8s(f, &s->control[0]);
-    qemu_get_8s(f, &s->control[1]);
-    qemu_get_8s(f, &s->control[2]);
-    qemu_get_8s(f, &s->status[0]);
-    qemu_get_8s(f, &s->status[1]);
-
-    s->rx_len = qemu_get_byte(f);
-    s->rx_start = 0;
-    for (i = 0; i < s->rx_len; i ++)
-        s->rx_fifo[i] = qemu_get_byte(f);
-
-    return 0;
-}
-
-static PXA2xxFIrState *pxa2xx_fir_init(MemoryRegion *sysmem,
-                hwaddr base,
-                qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma,
-                CharDriverState *chr)
-{
-    PXA2xxFIrState *s = (PXA2xxFIrState *)
-            g_malloc0(sizeof(PXA2xxFIrState));
-
-    s->irq = irq;
-    s->rx_dma = rx_dma;
-    s->tx_dma = tx_dma;
-    s->chr = chr;
-
-    pxa2xx_fir_reset(s);
-
-    memory_region_init_io(&s->iomem, NULL, &pxa2xx_fir_ops, s, "pxa2xx-fir", 0x1000);
-    memory_region_add_subregion(sysmem, base, &s->iomem);
-
-    if (chr) {
-        qemu_chr_fe_claim_no_fail(chr);
-        qemu_chr_add_handlers(chr, pxa2xx_fir_is_empty,
+    if (s->chr) {
+        qemu_chr_fe_claim_no_fail(s->chr);
+        qemu_chr_add_handlers(s->chr, pxa2xx_fir_is_empty,
                         pxa2xx_fir_rx, pxa2xx_fir_event, s);
     }
+}
 
-    register_savevm(NULL, "pxa2xx_fir", 0, 0, pxa2xx_fir_save,
-                    pxa2xx_fir_load, s);
+static bool pxa2xx_fir_vmstate_validate(void *opaque, int version_id)
+{
+    PXA2xxFIrState *s = opaque;
 
-    return s;
+    return s->rx_start < ARRAY_SIZE(s->rx_fifo);
+}
+
+static const VMStateDescription pxa2xx_fir_vmsd = {
+    .name = "pxa2xx-fir",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(enable, PXA2xxFIrState),
+        VMSTATE_UINT8_ARRAY(control, PXA2xxFIrState, 3),
+        VMSTATE_UINT8_ARRAY(status, PXA2xxFIrState, 2),
+        VMSTATE_UINT32(rx_len, PXA2xxFIrState),
+        VMSTATE_UINT32(rx_start, PXA2xxFIrState),
+        VMSTATE_VALIDATE("fifo is 64 bytes", pxa2xx_fir_vmstate_validate),
+        VMSTATE_UINT8_ARRAY(rx_fifo, PXA2xxFIrState, 64),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property pxa2xx_fir_properties[] = {
+    DEFINE_PROP_CHR("chardev", PXA2xxFIrState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void pxa2xx_fir_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = pxa2xx_fir_realize;
+    dc->vmsd = &pxa2xx_fir_vmsd;
+    dc->props = pxa2xx_fir_properties;
+    dc->reset = pxa2xx_fir_reset;
+}
+
+static const TypeInfo pxa2xx_fir_info = {
+    .name = TYPE_PXA2XX_FIR,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(PXA2xxFIrState),
+    .class_init = pxa2xx_fir_class_init,
+    .instance_init = pxa2xx_fir_instance_init,
+};
+
+static PXA2xxFIrState *pxa2xx_fir_init(MemoryRegion *sysmem,
+                                       hwaddr base,
+                                       qemu_irq irq, qemu_irq rx_dma,
+                                       qemu_irq tx_dma,
+                                       CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *sbd;
+
+    dev = qdev_create(NULL, TYPE_PXA2XX_FIR);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sbd = SYS_BUS_DEVICE(dev);
+    sysbus_mmio_map(sbd, 0, base);
+    sysbus_connect_irq(sbd, 0, irq);
+    sysbus_connect_irq(sbd, 1, rx_dma);
+    sysbus_connect_irq(sbd, 2, tx_dma);
+    return PXA2XX_FIR(dev);
 }
 
 static void pxa2xx_reset(void *opaque, int line, int level)
@@ -2039,7 +2061,7 @@
     PXA2xxState *s;
     int i;
     DriveInfo *dinfo;
-    s = (PXA2xxState *) g_malloc0(sizeof(PXA2xxState));
+    s = g_new0(PXA2xxState, 1);
 
     if (revision && strncmp(revision, "pxa27", 5)) {
         fprintf(stderr, "Machine requires a PXA27x processor.\n");
@@ -2057,11 +2079,11 @@
 
     /* SDRAM & Internal Memory Storage */
     memory_region_init_ram(&s->sdram, NULL, "pxa270.sdram", sdram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa270.internal", 0x40000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2135,7 +2157,7 @@
     vmstate_register(NULL, 0, &vmstate_pxa2xx_pm, s);
 
     for (i = 0; pxa27x_ssp[i].io_base; i ++);
-    s->ssp = (SSIBus **)g_malloc0(sizeof(SSIBus *) * i);
+    s->ssp = g_new0(SSIBus *, i);
     for (i = 0; pxa27x_ssp[i].io_base; i ++) {
         DeviceState *dev;
         dev = sysbus_create_simple(TYPE_PXA2XX_SSP, pxa27x_ssp[i].io_base,
@@ -2143,10 +2165,8 @@
         s->ssp[i] = (SSIBus *)qdev_get_child_bus(dev, "ssi");
     }
 
-    if (usb_enabled(false)) {
-        sysbus_create_simple("sysbus-ohci", 0x4c000000,
-                        qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
-    }
+    sysbus_create_simple("sysbus-ohci", 0x4c000000,
+                         qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
 
     s->pcmcia[0] = pxa2xx_pcmcia_init(address_space, 0x20000000);
     s->pcmcia[1] = pxa2xx_pcmcia_init(address_space, 0x30000000);
@@ -2180,7 +2200,7 @@
     int i;
     DriveInfo *dinfo;
 
-    s = (PXA2xxState *) g_malloc0(sizeof(PXA2xxState));
+    s = g_new0(PXA2xxState, 1);
 
     s->cpu = cpu_arm_init("pxa255");
     if (s->cpu == NULL) {
@@ -2191,11 +2211,11 @@
 
     /* SDRAM & Internal Memory Storage */
     memory_region_init_ram(&s->sdram, NULL, "pxa255.sdram", sdram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(address_space, PXA2XX_SDRAM_BASE, &s->sdram);
     memory_region_init_ram(&s->internal, NULL, "pxa255.internal",
-                           PXA2XX_INTERNAL_SIZE, &error_abort);
+                           PXA2XX_INTERNAL_SIZE, &error_fatal);
     vmstate_register_ram_global(&s->internal);
     memory_region_add_subregion(address_space, PXA2XX_INTERNAL_BASE,
                                 &s->internal);
@@ -2268,7 +2288,7 @@
     vmstate_register(NULL, 0, &vmstate_pxa2xx_pm, s);
 
     for (i = 0; pxa255_ssp[i].io_base; i ++);
-    s->ssp = (SSIBus **)g_malloc0(sizeof(SSIBus *) * i);
+    s->ssp = g_new0(SSIBus *, i);
     for (i = 0; pxa255_ssp[i].io_base; i ++) {
         DeviceState *dev;
         dev = sysbus_create_simple(TYPE_PXA2XX_SSP, pxa255_ssp[i].io_base,
@@ -2276,10 +2296,8 @@
         s->ssp[i] = (SSIBus *)qdev_get_child_bus(dev, "ssi");
     }
 
-    if (usb_enabled(false)) {
-        sysbus_create_simple("sysbus-ohci", 0x4c000000,
-                        qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
-    }
+    sysbus_create_simple("sysbus-ohci", 0x4c000000,
+                         qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
 
     s->pcmcia[0] = pxa2xx_pcmcia_init(address_space, 0x20000000);
     s->pcmcia[1] = pxa2xx_pcmcia_init(address_space, 0x30000000);
@@ -2306,8 +2324,11 @@
 static void pxa2xx_ssp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sdc->init = pxa2xx_ssp_init;
+    dc->reset = pxa2xx_ssp_reset;
+    dc->vmsd = &vmstate_pxa2xx_ssp;
 }
 
 static const TypeInfo pxa2xx_ssp_info = {
@@ -2323,6 +2344,7 @@
     type_register_static(&pxa2xx_ssp_info);
     type_register_static(&pxa2xx_i2c_info);
     type_register_static(&pxa2xx_rtc_sysbus_info);
+    type_register_static(&pxa2xx_fir_info);
 }
 
 type_init(pxa2xx_register_types)
diff --git a/hw/arm/pxa2xx_gpio.c b/hw/arm/pxa2xx_gpio.c
index 354ccf1..576a8eb 100644
--- a/hw/arm/pxa2xx_gpio.c
+++ b/hw/arm/pxa2xx_gpio.c
@@ -7,9 +7,12 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/arm/pxa.h"
+#include "qemu/log.h"
 
 #define PXA2XX_GPIO_BANKS	4
 
@@ -137,7 +140,7 @@
         level = s->olevel[i] & s->dir[i];
 
         for (diff = s->prev_level[i] ^ level; diff; diff ^= 1 << bit) {
-            bit = ffs(diff) - 1;
+            bit = ctz32(diff);
             line = bit + 32 * i;
             qemu_set_irq(s->handler[line], (level >> bit) & 1);
         }
diff --git a/hw/arm/pxa2xx_pic.c b/hw/arm/pxa2xx_pic.c
index 9cfc714..b516ced 100644
--- a/hw/arm/pxa2xx_pic.c
+++ b/hw/arm/pxa2xx_pic.c
@@ -8,6 +8,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/sysbus.h"
@@ -232,7 +235,7 @@
 
 #define REGINFO_FOR_PIC_CP(NAME, CRN) \
     { .name = NAME, .cp = 6, .crn = CRN, .crm = 0, .opc1 = 0, .opc2 = 0, \
-      .access = PL1_RW, \
+      .access = PL1_RW, .type = ARM_CP_IO, \
       .readfn = pxa2xx_pic_cp_read, .writefn = pxa2xx_pic_cp_write }
 
 static const ARMCPRegInfo pxa_pic_cp_reginfo[] = {
@@ -307,17 +310,10 @@
     },
 };
 
-static int pxa2xx_pic_initfn(SysBusDevice *dev)
-{
-    return 0;
-}
-
 static void pxa2xx_pic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pxa2xx_pic_initfn;
     dc->desc = "PXA2xx PIC";
     dc->vmsd = &vmstate_pxa2xx_pic_regs;
 }
diff --git a/hw/arm/ranchu.c b/hw/arm/ranchu.c
index 8450a9a..f4dd8ce 100644
--- a/hw/arm/ranchu.c
+++ b/hw/arm/ranchu.c
@@ -23,6 +23,7 @@
  * We only support 64-bit ARM CPUs.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
@@ -38,8 +39,7 @@
 #include "qemu/config-file.h"
 #include "sysemu/char.h"
 #include "monitor/monitor.h"
-
-#include "android/android.h"
+#include "qapi/error.h"
 
 #define NUM_VIRTIO_TRANSPORTS 32
 
@@ -64,11 +64,12 @@
     RANCHU_GIC_DIST,
     RANCHU_GIC_CPU,
     RANCHU_UART,
-    RANCHU_GF_FB,
-    RANCHU_GF_BATTERY,
-    RANCHU_GF_AUDIO,
-    RANCHU_GF_EVDEV,
-    RANCHU_ANDROID_PIPE,
+    RANCHU_GOLDFISH_FB,
+    RANCHU_GOLDFISH_BATTERY,
+    RANCHU_GOLDFISH_AUDIO,
+    RANCHU_GOLDFISH_EVDEV,
+    RANCHU_GOLDFISH_PIPE,
+    RANCHU_GOLDFISH_SYNC,
     RANCHU_MMIO,
 };
 
@@ -108,12 +109,12 @@
     [RANCHU_GIC_DIST] = { 0x8000000, 0x10000 },
     [RANCHU_GIC_CPU] = { 0x8010000, 0x10000 },
     [RANCHU_UART] = { 0x9000000, 0x1000 },
-    [RANCHU_GF_FB] = { 0x9010000, 0x100 },
-    [RANCHU_GF_BATTERY] = { 0x9020000, 0x1000 },
-    [RANCHU_GF_AUDIO] = { 0x9030000, 0x100 },
-    [RANCHU_GF_EVDEV] = { 0x9040000, 0x1000 },
+    [RANCHU_GOLDFISH_FB] = { 0x9010000, 0x100 },
+    [RANCHU_GOLDFISH_BATTERY] = { 0x9020000, 0x1000 },
+    [RANCHU_GOLDFISH_AUDIO] = { 0x9030000, 0x100 },
+    [RANCHU_GOLDFISH_EVDEV] = { 0x9040000, 0x1000 },
     [RANCHU_MMIO] = { 0xa000000, 0x200 },
-    [RANCHU_ANDROID_PIPE] = {0xa010000, 0x2000 },
+    [RANCHU_GOLDFISH_PIPE] = {0xa010000, 0x2000 },
     /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */
     /* 0x10000000 .. 0x40000000 reserved for PCI */
     [RANCHU_MEM] = { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
@@ -121,11 +122,11 @@
 
 static const int irqmap[] = {
     [RANCHU_UART] = 1,
-    [RANCHU_GF_FB] = 2,
-    [RANCHU_GF_BATTERY] = 3,
-    [RANCHU_GF_AUDIO] = 4,
-    [RANCHU_GF_EVDEV] = 5,
-    [RANCHU_ANDROID_PIPE] = 6,
+    [RANCHU_GOLDFISH_FB] = 2,
+    [RANCHU_GOLDFISH_BATTERY] = 3,
+    [RANCHU_GOLDFISH_AUDIO] = 4,
+    [RANCHU_GOLDFISH_EVDEV] = 5,
+    [RANCHU_GOLDFISH_PIPE] = 6,
     [RANCHU_MMIO] = 16, /* ...to 16 + NUM_VIRTIO_TRANSPORTS - 1 */
 };
 
@@ -488,16 +489,18 @@
     create_gic(vbi, pic);
     create_simple_device(vbi, pic, RANCHU_UART, "pl011",
                          "arm,pl011\0arm,primecell", 2, "uartclk\0apb_pclk", 2);
-    create_simple_device(vbi, pic, RANCHU_GF_FB, "goldfish_fb",
+    create_simple_device(vbi, pic, RANCHU_GOLDFISH_FB, "goldfish_fb",
                          "generic,goldfish-fb", 1, 0, 0);
-    create_simple_device(vbi, pic, RANCHU_GF_BATTERY, "goldfish_battery",
+    create_simple_device(vbi, pic, RANCHU_GOLDFISH_BATTERY, "goldfish_battery",
                          "generic,goldfish-battery", 1, 0, 0);
-    create_simple_device(vbi, pic, RANCHU_GF_AUDIO, "goldfish_audio",
+    create_simple_device(vbi, pic, RANCHU_GOLDFISH_AUDIO, "goldfish_audio",
                          "generic,goldfish-audio", 1, 0, 0);
-    create_simple_device(vbi, pic, RANCHU_GF_EVDEV, "goldfish-events",
+    create_simple_device(vbi, pic, RANCHU_GOLDFISH_EVDEV, "goldfish-events",
                          "generic,goldfish-events-keypad", 1, 0, 0);
-    create_simple_device(vbi, pic, RANCHU_ANDROID_PIPE, "android_pipe",
+    create_simple_device(vbi, pic, RANCHU_GOLDFISH_PIPE, "goldfish_pipe",
                          "generic,android-pipe", 1, 0, 0);
+    create_simple_device(vbi, pic, RANCHU_GOLDFISH_SYNC, "goldfish-sync",
+                         "generic,goldfish-sync", 1, 0, 0);
 
     /* Create mmio transports, so the user can create virtio backends
      * (which will be automatically plugged in to the transports). If
@@ -516,16 +519,12 @@
     arm_load_kernel(ARM_CPU(first_cpu), &vbi->bootinfo);
 }
 
-static QEMUMachine ranchu_machine = {
-    .name = "ranchu",
-    .desc = "Ranchu Virtual Machine for Android Emulator",
-    .init = ranchu_init,
-    .max_cpus = 1,
-};
-
-static void ranchu_machine_init(void)
+static void ranchu_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ranchu_machine);
+    mc->desc = "Android/ARM ranchu";
+    mc->init = ranchu_init;
+    mc->max_cpus = 16;
+    mc->is_default = 1;
 }
 
-machine_init(ranchu_machine_init);
+DEFINE_MACHINE("ranchu", ranchu_machine_init)
diff --git a/hw/arm/raspi.c b/hw/arm/raspi.c
new file mode 100644
index 0000000..2b295f1
--- /dev/null
+++ b/hw/arm/raspi.c
@@ -0,0 +1,172 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/bcm2836.h"
+#include "qemu/error-report.h"
+#include "hw/boards.h"
+#include "hw/loader.h"
+#include "hw/arm/arm.h"
+#include "sysemu/sysemu.h"
+
+#define SMPBOOT_ADDR    0x300 /* this should leave enough space for ATAGS */
+#define MVBAR_ADDR      0x400 /* secure vectors */
+#define BOARDSETUP_ADDR (MVBAR_ADDR + 0x20) /* board setup code */
+#define FIRMWARE_ADDR   0x8000 /* Pi loads kernel.img here by default */
+
+/* Table of Linux board IDs for different Pi versions */
+static const int raspi_boardid[] = {[1] = 0xc42, [2] = 0xc43};
+
+typedef struct RasPiState {
+    BCM2836State soc;
+    MemoryRegion ram;
+} RasPiState;
+
+static void write_smpboot(ARMCPU *cpu, const struct arm_boot_info *info)
+{
+    static const uint32_t smpboot[] = {
+        0xe1a0e00f, /*    mov     lr, pc */
+        0xe3a0fe00 + (BOARDSETUP_ADDR >> 4), /* mov pc, BOARDSETUP_ADDR */
+        0xee100fb0, /*    mrc     p15, 0, r0, c0, c0, 5;get core ID */
+        0xe7e10050, /*    ubfx    r0, r0, #0, #2       ;extract LSB */
+        0xe59f5014, /*    ldr     r5, =0x400000CC      ;load mbox base */
+        0xe320f001, /* 1: yield */
+        0xe7953200, /*    ldr     r3, [r5, r0, lsl #4] ;read mbox for our core*/
+        0xe3530000, /*    cmp     r3, #0               ;spin while zero */
+        0x0afffffb, /*    beq     1b */
+        0xe7853200, /*    str     r3, [r5, r0, lsl #4] ;clear mbox */
+        0xe12fff13, /*    bx      r3                   ;jump to target */
+        0x400000cc, /* (constant: mailbox 3 read/clear base) */
+    };
+
+    /* check that we don't overrun board setup vectors */
+    QEMU_BUILD_BUG_ON(SMPBOOT_ADDR + sizeof(smpboot) > MVBAR_ADDR);
+    /* check that board setup address is correctly relocated */
+    QEMU_BUILD_BUG_ON((BOARDSETUP_ADDR & 0xf) != 0
+                      || (BOARDSETUP_ADDR >> 4) >= 0x100);
+
+    rom_add_blob_fixed("raspi_smpboot", smpboot, sizeof(smpboot),
+                       info->smp_loader_start);
+}
+
+static void write_board_setup(ARMCPU *cpu, const struct arm_boot_info *info)
+{
+    arm_write_secure_board_setup_dummy_smc(cpu, info, MVBAR_ADDR);
+}
+
+static void reset_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
+{
+    CPUState *cs = CPU(cpu);
+    cpu_set_pc(cs, info->smp_loader_start);
+}
+
+static void setup_boot(MachineState *machine, int version, size_t ram_size)
+{
+    static struct arm_boot_info binfo;
+    int r;
+
+    binfo.board_id = raspi_boardid[version];
+    binfo.ram_size = ram_size;
+    binfo.nb_cpus = smp_cpus;
+    binfo.board_setup_addr = BOARDSETUP_ADDR;
+    binfo.write_board_setup = write_board_setup;
+    binfo.secure_board_setup = true;
+    binfo.secure_boot = true;
+
+    /* Pi2 requires SMP setup */
+    if (version == 2) {
+        binfo.smp_loader_start = SMPBOOT_ADDR;
+        binfo.write_secondary_boot = write_smpboot;
+        binfo.secondary_cpu_reset_hook = reset_secondary;
+    }
+
+    /* If the user specified a "firmware" image (e.g. UEFI), we bypass
+     * the normal Linux boot process
+     */
+    if (machine->firmware) {
+        /* load the firmware image (typically kernel.img) */
+        r = load_image_targphys(machine->firmware, FIRMWARE_ADDR,
+                                ram_size - FIRMWARE_ADDR);
+        if (r < 0) {
+            error_report("Failed to load firmware from %s", machine->firmware);
+            exit(1);
+        }
+
+        binfo.entry = FIRMWARE_ADDR;
+        binfo.firmware_loaded = true;
+    } else {
+        binfo.kernel_filename = machine->kernel_filename;
+        binfo.kernel_cmdline = machine->kernel_cmdline;
+        binfo.initrd_filename = machine->initrd_filename;
+    }
+
+    arm_load_kernel(ARM_CPU(first_cpu), &binfo);
+}
+
+static void raspi2_init(MachineState *machine)
+{
+    RasPiState *s = g_new0(RasPiState, 1);
+    uint32_t vcram_size;
+    DriveInfo *di;
+    BlockBackend *blk;
+    BusState *bus;
+    DeviceState *carddev;
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_BCM2836);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    /* Allocate and map RAM */
+    memory_region_allocate_system_memory(&s->ram, OBJECT(machine), "ram",
+                                         machine->ram_size);
+    /* FIXME: Remove when we have custom CPU address space support */
+    memory_region_add_subregion_overlap(get_system_memory(), 0, &s->ram, 0);
+
+    /* Setup the SOC */
+    object_property_add_const_link(OBJECT(&s->soc), "ram", OBJECT(&s->ram),
+                                   &error_abort);
+    object_property_set_int(OBJECT(&s->soc), smp_cpus, "enabled-cpus",
+                            &error_abort);
+    object_property_set_int(OBJECT(&s->soc), 0xa21041, "board-rev",
+                            &error_abort);
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_abort);
+
+    /* Create and plug in the SD cards */
+    di = drive_get_next(IF_SD);
+    blk = di ? blk_by_legacy_dinfo(di) : NULL;
+    bus = qdev_get_child_bus(DEVICE(&s->soc), "sd-bus");
+    if (bus == NULL) {
+        error_report("No SD bus found in SOC object");
+        exit(1);
+    }
+    carddev = qdev_create(bus, TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &error_fatal);
+
+    vcram_size = object_property_get_int(OBJECT(&s->soc), "vcram-size",
+                                         &error_abort);
+    setup_boot(machine, 2, machine->ram_size - vcram_size);
+}
+
+static void raspi2_machine_init(MachineClass *mc)
+{
+    mc->desc = "Raspberry Pi 2";
+    mc->init = raspi2_init;
+    mc->block_default_type = IF_SD;
+    mc->no_parallel = 1;
+    mc->no_floppy = 1;
+    mc->no_cdrom = 1;
+    mc->max_cpus = BCM2836_NCPUS;
+    mc->default_ram_size = 1024 * 1024 * 1024;
+};
+DEFINE_MACHINE("raspi2", raspi2_machine_init)
diff --git a/hw/arm/realview.c b/hw/arm/realview.c
index af65aa4..8eafcca 100644
--- a/hw/arm/realview.c
+++ b/hw/arm/realview.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
@@ -19,6 +23,7 @@
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
+#include "hw/char/pl011.h"
 
 #define SMP_BOOT_ADDR 0xe0000000
 #define SMP_BOOTREG_ADDR 0x10000030
@@ -52,7 +57,7 @@
     CPUARMState *env;
     ObjectClass *cpu_oc;
     MemoryRegion *sysmem = get_system_memory();
-    MemoryRegion *ram_lo = g_new(MemoryRegion, 1);
+    MemoryRegion *ram_lo;
     MemoryRegion *ram_hi = g_new(MemoryRegion, 1);
     MemoryRegion *ram_alias = g_new(MemoryRegion, 1);
     MemoryRegion *ram_hack = g_new(MemoryRegion, 1);
@@ -99,21 +104,21 @@
 
     for (n = 0; n < smp_cpus; n++) {
         Object *cpuobj = object_new(object_class_get_name(cpu_oc));
-        Error *err = NULL;
+
+        /* By default A9,A15 and ARM1176 CPUs have EL3 enabled.  This board
+         * does not currently support EL3 so the CPU EL3 property is disabled
+         * before realization.
+         */
+        if (object_property_find(cpuobj, "has_el3", NULL)) {
+            object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
+        }
 
         if (is_pb && is_mpcore) {
-            object_property_set_int(cpuobj, periphbase, "reset-cbar", &err);
-            if (err) {
-                error_report("%s", error_get_pretty(err));
-                exit(1);
-            }
+            object_property_set_int(cpuobj, periphbase, "reset-cbar",
+                                    &error_fatal);
         }
 
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report("%s", error_get_pretty(err));
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
 
         cpu_irq[n] = qdev_get_gpio_in(DEVICE(cpuobj), ARM_CPU_IRQ);
     }
@@ -135,16 +140,17 @@
 
     if (is_pb && ram_size > 0x20000000) {
         /* Core tile RAM.  */
+        ram_lo = g_new(MemoryRegion, 1);
         low_ram_size = ram_size - 0x20000000;
         ram_size = 0x20000000;
         memory_region_init_ram(ram_lo, NULL, "realview.lowmem", low_ram_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(ram_lo);
         memory_region_add_subregion(sysmem, 0x20000000, ram_lo);
     }
 
     memory_region_init_ram(ram_hi, NULL, "realview.highmem", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_hi);
     low_ram_size = ram_size;
     if (low_ram_size > 0x10000000)
@@ -197,10 +203,10 @@
     sysbus_create_simple("pl050_keyboard", 0x10006000, pic[20]);
     sysbus_create_simple("pl050_mouse", 0x10007000, pic[21]);
 
-    sysbus_create_simple("pl011", 0x10009000, pic[12]);
-    sysbus_create_simple("pl011", 0x1000a000, pic[13]);
-    sysbus_create_simple("pl011", 0x1000b000, pic[14]);
-    sysbus_create_simple("pl011", 0x1000c000, pic[15]);
+    pl011_create(0x10009000, pic[12], serial_hds[0]);
+    pl011_create(0x1000a000, pic[13], serial_hds[1]);
+    pl011_create(0x1000b000, pic[14], serial_hds[2]);
+    pl011_create(0x1000c000, pic[15], serial_hds[3]);
 
     /* DMA controller is optional, apparently.  */
     sysbus_create_simple("pl081", 0x10030000, pic[24]);
@@ -248,7 +254,7 @@
         sysbus_connect_irq(busdev, 2, pic[50]);
         sysbus_connect_irq(busdev, 3, pic[51]);
         pci_bus = (PCIBus *)qdev_get_child_bus(dev, "pci");
-        if (usb_enabled(false)) {
+        if (machine_usb(machine)) {
             pci_create_simple(pci_bus, -1, "pci-ohci");
         }
         n = drive_get_max_bus(IF_SCSI);
@@ -340,7 +346,7 @@
        BootROM happens to be in ROM/flash or in memory that isn't clobbered
        until after Linux boots the secondary CPUs.  */
     memory_region_init_ram(ram_hack, NULL, "realview.hack", 0x1000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_hack);
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
@@ -386,41 +392,73 @@
     realview_init(machine, BOARD_PBX_A9);
 }
 
-static QEMUMachine realview_eb_machine = {
-    .name = "realview-eb",
-    .desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)",
-    .init = realview_eb_init,
-    .block_default_type = IF_SCSI,
+static void realview_eb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Emulation Baseboard (ARM926EJ-S)";
+    mc->init = realview_eb_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo realview_eb_type = {
+    .name = MACHINE_TYPE_NAME("realview-eb"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_eb_class_init,
 };
 
-static QEMUMachine realview_eb_mpcore_machine = {
-    .name = "realview-eb-mpcore",
-    .desc = "ARM RealView Emulation Baseboard (ARM11MPCore)",
-    .init = realview_eb_mpcore_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void realview_eb_mpcore_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Emulation Baseboard (ARM11MPCore)";
+    mc->init = realview_eb_mpcore_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo realview_eb_mpcore_type = {
+    .name = MACHINE_TYPE_NAME("realview-eb-mpcore"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_eb_mpcore_class_init,
 };
 
-static QEMUMachine realview_pb_a8_machine = {
-    .name = "realview-pb-a8",
-    .desc = "ARM RealView Platform Baseboard for Cortex-A8",
-    .init = realview_pb_a8_init,
+static void realview_pb_a8_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Platform Baseboard for Cortex-A8";
+    mc->init = realview_pb_a8_init;
+}
+
+static const TypeInfo realview_pb_a8_type = {
+    .name = MACHINE_TYPE_NAME("realview-pb-a8"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_pb_a8_class_init,
 };
 
-static QEMUMachine realview_pbx_a9_machine = {
-    .name = "realview-pbx-a9",
-    .desc = "ARM RealView Platform Baseboard Explore for Cortex-A9",
-    .init = realview_pbx_a9_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void realview_pbx_a9_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM RealView Platform Baseboard Explore for Cortex-A9";
+    mc->init = realview_pbx_a9_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo realview_pbx_a9_type = {
+    .name = MACHINE_TYPE_NAME("realview-pbx-a9"),
+    .parent = TYPE_MACHINE,
+    .class_init = realview_pbx_a9_class_init,
 };
 
 static void realview_machine_init(void)
 {
-    qemu_register_machine(&realview_eb_machine);
-    qemu_register_machine(&realview_eb_mpcore_machine);
-    qemu_register_machine(&realview_pb_a8_machine);
-    qemu_register_machine(&realview_pbx_a9_machine);
+    type_register_static(&realview_eb_type);
+    type_register_static(&realview_eb_mpcore_type);
+    type_register_static(&realview_pb_a8_type);
+    type_register_static(&realview_pbx_a9_type);
 }
 
-machine_init(realview_machine_init);
+type_init(realview_machine_init)
diff --git a/hw/arm/sabrelite.c b/hw/arm/sabrelite.c
new file mode 100644
index 0000000..4e7ac8c
--- /dev/null
+++ b/hw/arm/sabrelite.c
@@ -0,0 +1,127 @@
+/*
+ * SABRELITE Board System emulation.
+ *
+ * Copyright (c) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This code is licensed under the GPL, version 2 or later.
+ * See the file `COPYING' in the top level directory.
+ *
+ * It (partially) emulates a sabrelite board, with a Freescale
+ * i.MX6 SoC
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "hw/arm/fsl-imx6.h"
+#include "hw/boards.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
+#include "sysemu/qtest.h"
+
+typedef struct IMX6Sabrelite {
+    FslIMX6State soc;
+    MemoryRegion ram;
+} IMX6Sabrelite;
+
+static struct arm_boot_info sabrelite_binfo = {
+    /* DDR memory start */
+    .loader_start = FSL_IMX6_MMDC_ADDR,
+    /* No board ID, we boot from DT tree */
+    .board_id = -1,
+};
+
+/* No need to do any particular setup for secondary boot */
+static void sabrelite_write_secondary(ARMCPU *cpu,
+                                      const struct arm_boot_info *info)
+{
+}
+
+/* Secondary cores are reset through SRC device */
+static void sabrelite_reset_secondary(ARMCPU *cpu,
+                                      const struct arm_boot_info *info)
+{
+}
+
+static void sabrelite_init(MachineState *machine)
+{
+    IMX6Sabrelite *s = g_new0(IMX6Sabrelite, 1);
+    Error *err = NULL;
+
+    /* Check the amount of memory is compatible with the SOC */
+    if (machine->ram_size > FSL_IMX6_MMDC_SIZE) {
+        error_report("RAM size " RAM_ADDR_FMT " above max supported (%08x)",
+                     machine->ram_size, FSL_IMX6_MMDC_SIZE);
+        exit(1);
+    }
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_FSL_IMX6);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &err);
+    if (err != NULL) {
+        error_report("%s", error_get_pretty(err));
+        exit(1);
+    }
+
+    memory_region_allocate_system_memory(&s->ram, NULL, "sabrelite.ram",
+                                         machine->ram_size);
+    memory_region_add_subregion(get_system_memory(), FSL_IMX6_MMDC_ADDR,
+                                &s->ram);
+
+    {
+        /*
+         * TODO: Ideally we would expose the chip select and spi bus on the
+         * SoC object using alias properties; then we would not need to
+         * directly access the underlying spi device object.
+         */
+        /* Add the sst25vf016b NOR FLASH memory to first SPI */
+        Object *spi_dev;
+
+        spi_dev = object_resolve_path_component(OBJECT(&s->soc), "spi1");
+        if (spi_dev) {
+            SSIBus *spi_bus;
+
+            spi_bus = (SSIBus *)qdev_get_child_bus(DEVICE(spi_dev), "spi");
+            if (spi_bus) {
+                DeviceState *flash_dev;
+                qemu_irq cs_line;
+                DriveInfo *dinfo = drive_get_next(IF_MTD);
+
+                flash_dev = ssi_create_slave_no_init(spi_bus, "sst25vf016b");
+                if (dinfo) {
+                    qdev_prop_set_drive(flash_dev, "drive",
+                                        blk_by_legacy_dinfo(dinfo),
+                                        &error_fatal);
+                }
+                qdev_init_nofail(flash_dev);
+
+                cs_line = qdev_get_gpio_in_named(flash_dev, SSI_GPIO_CS, 0);
+                sysbus_connect_irq(SYS_BUS_DEVICE(spi_dev), 1, cs_line);
+            }
+        }
+    }
+
+    sabrelite_binfo.ram_size = machine->ram_size;
+    sabrelite_binfo.kernel_filename = machine->kernel_filename;
+    sabrelite_binfo.kernel_cmdline = machine->kernel_cmdline;
+    sabrelite_binfo.initrd_filename = machine->initrd_filename;
+    sabrelite_binfo.nb_cpus = smp_cpus;
+    sabrelite_binfo.secure_boot = true;
+    sabrelite_binfo.write_secondary_boot = sabrelite_write_secondary;
+    sabrelite_binfo.secondary_cpu_reset_hook = sabrelite_reset_secondary;
+
+    if (!qtest_enabled()) {
+        arm_load_kernel(&s->soc.cpu[0], &sabrelite_binfo);
+    }
+}
+
+static void sabrelite_machine_init(MachineClass *mc)
+{
+    mc->desc = "Freescale i.MX6 Quad SABRE Lite Board (Cortex A9)";
+    mc->init = sabrelite_init;
+    mc->max_cpus = FSL_IMX6_NUM_CPUS;
+}
+
+DEFINE_MACHINE("sabrelite", sabrelite_machine_init)
diff --git a/hw/arm/spitz.c b/hw/arm/spitz.c
index a16831c..41cc2ee 100644
--- a/hw/arm/spitz.c
+++ b/hw/arm/spitz.c
@@ -10,13 +10,15 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
 #include "sysemu/sysemu.h"
 #include "hw/pcmcia.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "hw/block/flash.h"
 #include "qemu/timer.h"
 #include "hw/devices.h"
@@ -162,20 +164,20 @@
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, FLASH_BASE);
 }
 
-static int sl_nand_init(SysBusDevice *dev)
+static void sl_nand_init(Object *obj)
 {
-    SLNANDState *s = SL_NAND(dev);
+    SLNANDState *s = SL_NAND(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     DriveInfo *nand;
 
     s->ctl = 0;
+    /* FIXME use a qdev drive property instead of drive_get() */
     nand = drive_get(IF_MTD, 0, 0);
     s->nand = nand_init(nand ? blk_by_legacy_dinfo(nand) : NULL,
                         s->manf_id, s->chip_id);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &sl_ops, s, "sl", 0x40);
+    memory_region_init_io(&s->iomem, obj, &sl_ops, s, "sl", 0x40);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 /* Spitz Keyboard */
@@ -402,7 +404,7 @@
     }
 
     timer_mod(s->kbdtimer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / 32);
+                   NANOSECONDS_PER_SECOND / 32);
 }
 
 static void spitz_keyboard_pre_map(SpitzKeyboardState *s)
@@ -498,10 +500,10 @@
     qemu_add_kbd_event_handler(spitz_keyboard_handler, s);
 }
 
-static int spitz_keyboard_init(SysBusDevice *sbd)
+static void spitz_keyboard_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    SpitzKeyboardState *s = SPITZ_KEYBOARD(dev);
+    DeviceState *dev = DEVICE(obj);
+    SpitzKeyboardState *s = SPITZ_KEYBOARD(obj);
     int i, j;
 
     for (i = 0; i < 0x80; i ++)
@@ -516,8 +518,6 @@
     s->kbdtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL, spitz_keyboard_tick, s);
     qdev_init_gpio_in(dev, spitz_keyboard_strobe, SPITZ_KEY_STROBE_NUM);
     qdev_init_gpio_out(dev, s->sense, SPITZ_KEY_SENSE_NUM);
-
-    return 0;
 }
 
 /* LCD backlight controller */
@@ -598,15 +598,13 @@
     return 0;
 }
 
-static int spitz_lcdtg_init(SSISlave *dev)
+static void spitz_lcdtg_realize(SSISlave *dev, Error **errp)
 {
     SpitzLCDTG *s = FROM_SSI_SLAVE(SpitzLCDTG, dev);
 
     spitz_lcdtg = s;
     s->bl_power = 0;
     s->bl_intensity = 0x20;
-
-    return 0;
 }
 
 /* SSP devices */
@@ -666,7 +664,7 @@
         max111x_set_input(max1111, MAX1111_BATT_TEMP, 0);
 }
 
-static int corgi_ssp_init(SSISlave *d)
+static void corgi_ssp_realize(SSISlave *d, Error **errp)
 {
     DeviceState *dev = DEVICE(d);
     CorgiSSPState *s = FROM_SSI_SLAVE(CorgiSSPState, d);
@@ -675,8 +673,6 @@
     s->bus[0] = ssi_create_bus(dev, "ssi0");
     s->bus[1] = ssi_create_bus(dev, "ssi1");
     s->bus[2] = ssi_create_bus(dev, "ssi2");
-
-    return 0;
 }
 
 static void spitz_ssp_attach(PXA2xxState *cpu)
@@ -912,7 +908,7 @@
 
     sl_flash_register(mpu, (model == spitz) ? FLASH_128M : FLASH_1024M);
 
-    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_abort);
+    memory_region_init_ram(rom, NULL, "spitz.rom", SPITZ_ROM, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -971,39 +967,71 @@
     spitz_common_init(machine, terrier, 0x33f);
 }
 
-static QEMUMachine akitapda_machine = {
-    .name = "akita",
-    .desc = "Akita PDA (PXA270)",
-    .init = akita_init,
+static void akitapda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C1000 (Akita) PDA (PXA270)";
+    mc->init = akita_init;
+}
+
+static const TypeInfo akitapda_type = {
+    .name = MACHINE_TYPE_NAME("akita"),
+    .parent = TYPE_MACHINE,
+    .class_init = akitapda_class_init,
 };
 
-static QEMUMachine spitzpda_machine = {
-    .name = "spitz",
-    .desc = "Spitz PDA (PXA270)",
-    .init = spitz_init,
+static void spitzpda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3000 (Spitz) PDA (PXA270)";
+    mc->init = spitz_init;
+}
+
+static const TypeInfo spitzpda_type = {
+    .name = MACHINE_TYPE_NAME("spitz"),
+    .parent = TYPE_MACHINE,
+    .class_init = spitzpda_class_init,
 };
 
-static QEMUMachine borzoipda_machine = {
-    .name = "borzoi",
-    .desc = "Borzoi PDA (PXA270)",
-    .init = borzoi_init,
+static void borzoipda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3100 (Borzoi) PDA (PXA270)";
+    mc->init = borzoi_init;
+}
+
+static const TypeInfo borzoipda_type = {
+    .name = MACHINE_TYPE_NAME("borzoi"),
+    .parent = TYPE_MACHINE,
+    .class_init = borzoipda_class_init,
 };
 
-static QEMUMachine terrierpda_machine = {
-    .name = "terrier",
-    .desc = "Terrier PDA (PXA270)",
-    .init = terrier_init,
+static void terrierpda_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sharp SL-C3200 (Terrier) PDA (PXA270)";
+    mc->init = terrier_init;
+}
+
+static const TypeInfo terrierpda_type = {
+    .name = MACHINE_TYPE_NAME("terrier"),
+    .parent = TYPE_MACHINE,
+    .class_init = terrierpda_class_init,
 };
 
 static void spitz_machine_init(void)
 {
-    qemu_register_machine(&akitapda_machine);
-    qemu_register_machine(&spitzpda_machine);
-    qemu_register_machine(&borzoipda_machine);
-    qemu_register_machine(&terrierpda_machine);
+    type_register_static(&akitapda_type);
+    type_register_static(&spitzpda_type);
+    type_register_static(&borzoipda_type);
+    type_register_static(&terrierpda_type);
 }
 
-machine_init(spitz_machine_init);
+type_init(spitz_machine_init)
 
 static bool is_version_0(void *opaque, int version_id)
 {
@@ -1030,17 +1058,18 @@
 static void sl_nand_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = sl_nand_init;
     dc->vmsd = &vmstate_sl_nand_info;
     dc->props = sl_nand_properties;
+    /* Reason: init() method uses drive_get() */
+    dc->cannot_instantiate_with_device_add_yet = true;
 }
 
 static const TypeInfo sl_nand_info = {
     .name          = TYPE_SL_NAND,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLNANDState),
+    .instance_init = sl_nand_init,
     .class_init    = sl_nand_class_init,
 };
 
@@ -1057,24 +1086,18 @@
     },
 };
 
-static Property spitz_keyboard_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void spitz_keyboard_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = spitz_keyboard_init;
     dc->vmsd = &vmstate_spitz_kbd;
-    dc->props = spitz_keyboard_properties;
 }
 
 static const TypeInfo spitz_keyboard_info = {
     .name          = TYPE_SPITZ_KEYBOARD,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SpitzKeyboardState),
+    .instance_init = spitz_keyboard_init,
     .class_init    = spitz_keyboard_class_init,
 };
 
@@ -1094,7 +1117,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = corgi_ssp_init;
+    k->realize = corgi_ssp_realize;
     k->transfer = corgi_ssp_transfer;
     dc->vmsd = &vmstate_corgi_ssp_regs;
 }
@@ -1123,7 +1146,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = spitz_lcdtg_init;
+    k->realize = spitz_lcdtg_realize;
     k->transfer = spitz_lcdtg_transfer;
     dc->vmsd = &vmstate_spitz_lcdtg_regs;
 }
diff --git a/hw/arm/stellaris.c b/hw/arm/stellaris.c
index 64bd4b4..794a3ad 100644
--- a/hw/arm/stellaris.c
+++ b/hw/arm/stellaris.c
@@ -7,15 +7,20 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
 #include "qemu/timer.h"
 #include "hw/i2c/i2c.h"
 #include "net/net.h"
 #include "hw/boards.h"
+#include "qemu/log.h"
 #include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
+#include "hw/char/pl011.h"
 
 #define GPIO_A 0
 #define GPIO_B 1
@@ -29,6 +34,8 @@
 #define BP_OLED_SSI  0x02
 #define BP_GAMEPAD   0x04
 
+#define NUM_IRQ_LINES 64
+
 typedef const struct {
     const char *name;
     uint32_t did0;
@@ -96,7 +103,7 @@
         tick += (int64_t)count * system_clock_scale;
     } else if (s->config == 1) {
         /* 32-bit RTC.  1Hz tick.  */
-        tick += get_ticks_per_sec();
+        tick += NANOSECONDS_PER_SECOND;
     } else if (s->mode[n] == 0xa) {
         /* PWM mode.  Not implemented.  */
     } else {
@@ -306,28 +313,27 @@
         VMSTATE_UINT32_ARRAY(match_prescale, gptm_state, 2),
         VMSTATE_UINT32(rtc, gptm_state),
         VMSTATE_INT64_ARRAY(tick, gptm_state, 2),
-        VMSTATE_TIMER_ARRAY(timer, gptm_state, 2),
+        VMSTATE_TIMER_PTR_ARRAY(timer, gptm_state, 2),
         VMSTATE_END_OF_LIST()
     }
 };
 
-static int stellaris_gptm_init(SysBusDevice *sbd)
+static void stellaris_gptm_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    gptm_state *s = STELLARIS_GPTM(dev);
+    DeviceState *dev = DEVICE(obj);
+    gptm_state *s = STELLARIS_GPTM(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     sysbus_init_irq(sbd, &s->irq);
     qdev_init_gpio_out(dev, &s->trigger, 1);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &gptm_ops, s,
+    memory_region_init_io(&s->iomem, obj, &gptm_ops, s,
                           "gptm", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
 
     s->opaque[0] = s->opaque[1] = s;
     s->timer[0] = timer_new_ns(QEMU_CLOCK_VIRTUAL, gptm_tick, &s->opaque[0]);
     s->timer[1] = timer_new_ns(QEMU_CLOCK_VIRTUAL, gptm_tick, &s->opaque[1]);
-    vmstate_register(dev, -1, &vmstate_stellaris_gptm, s);
-    return 0;
 }
 
 
@@ -673,7 +679,7 @@
 {
     ssys_state *s;
 
-    s = (ssys_state *)g_malloc0(sizeof(ssys_state));
+    s = g_new0(ssys_state, 1);
     s->irq = irq;
     s->board = board;
     /* Most devices come preprogrammed with a MAC address in the user data. */
@@ -868,23 +874,22 @@
     }
 };
 
-static int stellaris_i2c_init(SysBusDevice *sbd)
+static void stellaris_i2c_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    stellaris_i2c_state *s = STELLARIS_I2C(dev);
+    DeviceState *dev = DEVICE(obj);
+    stellaris_i2c_state *s = STELLARIS_I2C(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     I2CBus *bus;
 
     sysbus_init_irq(sbd, &s->irq);
     bus = i2c_init_bus(dev, "i2c");
     s->bus = bus;
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &stellaris_i2c_ops, s,
+    memory_region_init_io(&s->iomem, obj, &stellaris_i2c_ops, s,
                           "i2c", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     /* ??? For now we only implement the master interface.  */
     stellaris_i2c_reset(s);
-    vmstate_register(dev, -1, &vmstate_stellaris_i2c, s);
-    return 0;
 }
 
 /* Analogue to Digital Converter.  This is only partially implemented,
@@ -1155,23 +1160,30 @@
     }
 };
 
-static int stellaris_adc_init(SysBusDevice *sbd)
+static void stellaris_adc_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    stellaris_adc_state *s = STELLARIS_ADC(dev);
+    DeviceState *dev = DEVICE(obj);
+    stellaris_adc_state *s = STELLARIS_ADC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     int n;
 
     for (n = 0; n < 4; n++) {
         sysbus_init_irq(sbd, &s->irq[n]);
     }
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &stellaris_adc_ops, s,
+    memory_region_init_io(&s->iomem, obj, &stellaris_adc_ops, s,
                           "adc", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     stellaris_adc_reset(s);
     qdev_init_gpio_in(dev, stellaris_adc_trigger, 1);
-    vmstate_register(dev, -1, &vmstate_stellaris_adc, s);
-    return 0;
+}
+
+static
+void do_sys_reset(void *opaque, int n, int level)
+{
+    if (level) {
+        qemu_system_reset_request();
+    }
 }
 
 /* Board init.  */
@@ -1208,8 +1220,7 @@
         0x40024000, 0x40025000, 0x40026000};
     static const int gpio_irq[7] = {0, 1, 2, 3, 4, 30, 31};
 
-    qemu_irq *pic;
-    DeviceState *gpio_dev[7];
+    DeviceState *gpio_dev[7], *nvic;
     qemu_irq gpio_in[7][8];
     qemu_irq gpio_out[7][8];
     qemu_irq adc;
@@ -1220,14 +1231,38 @@
     int i;
     int j;
 
-    flash_size = ((board->dc0 & 0xffff) + 1) << 1;
-    sram_size = (board->dc0 >> 18) + 1;
-    pic = armv7m_init(get_system_memory(),
-                      flash_size, sram_size, kernel_filename, cpu_model);
+    MemoryRegion *sram = g_new(MemoryRegion, 1);
+    MemoryRegion *flash = g_new(MemoryRegion, 1);
+    MemoryRegion *system_memory = get_system_memory();
+
+    flash_size = (((board->dc0 & 0xffff) + 1) << 1) * 1024;
+    sram_size = ((board->dc0 >> 18) + 1) * 1024;
+
+    /* Flash programming is done via the SCU, so pretend it is ROM.  */
+    memory_region_init_ram(flash, NULL, "stellaris.flash", flash_size,
+                           &error_fatal);
+    vmstate_register_ram_global(flash);
+    memory_region_set_readonly(flash, true);
+    memory_region_add_subregion(system_memory, 0, flash);
+
+    memory_region_init_ram(sram, NULL, "stellaris.sram", sram_size,
+                           &error_fatal);
+    vmstate_register_ram_global(sram);
+    memory_region_add_subregion(system_memory, 0x20000000, sram);
+
+    nvic = armv7m_init(system_memory, flash_size, NUM_IRQ_LINES,
+                      kernel_filename, cpu_model);
+
+    qdev_connect_gpio_out_named(nvic, "SYSRESETREQ", 0,
+                                qemu_allocate_irq(&do_sys_reset, NULL, 0));
 
     if (board->dc1 & (1 << 16)) {
         dev = sysbus_create_varargs(TYPE_STELLARIS_ADC, 0x40038000,
-                                    pic[14], pic[15], pic[16], pic[17], NULL);
+                                    qdev_get_gpio_in(nvic, 14),
+                                    qdev_get_gpio_in(nvic, 15),
+                                    qdev_get_gpio_in(nvic, 16),
+                                    qdev_get_gpio_in(nvic, 17),
+                                    NULL);
         adc = qdev_get_gpio_in(dev, 0);
     } else {
         adc = NULL;
@@ -1236,19 +1271,21 @@
         if (board->dc2 & (0x10000 << i)) {
             dev = sysbus_create_simple(TYPE_STELLARIS_GPTM,
                                        0x40030000 + i * 0x1000,
-                                       pic[timer_irq[i]]);
+                                       qdev_get_gpio_in(nvic, timer_irq[i]));
             /* TODO: This is incorrect, but we get away with it because
                the ADC output is only ever pulsed.  */
             qdev_connect_gpio_out(dev, 0, adc);
         }
     }
 
-    stellaris_sys_init(0x400fe000, pic[28], board, nd_table[0].macaddr.a);
+    stellaris_sys_init(0x400fe000, qdev_get_gpio_in(nvic, 28),
+                       board, nd_table[0].macaddr.a);
 
     for (i = 0; i < 7; i++) {
         if (board->dc4 & (1 << i)) {
             gpio_dev[i] = sysbus_create_simple("pl061_luminary", gpio_addr[i],
-                                               pic[gpio_irq[i]]);
+                                               qdev_get_gpio_in(nvic,
+                                                                gpio_irq[i]));
             for (j = 0; j < 8; j++) {
                 gpio_in[i][j] = qdev_get_gpio_in(gpio_dev[i], j);
                 gpio_out[i][j] = NULL;
@@ -1257,7 +1294,8 @@
     }
 
     if (board->dc2 & (1 << 12)) {
-        dev = sysbus_create_simple(TYPE_STELLARIS_I2C, 0x40020000, pic[8]);
+        dev = sysbus_create_simple(TYPE_STELLARIS_I2C, 0x40020000,
+                                   qdev_get_gpio_in(nvic, 8));
         i2c = (I2CBus *)qdev_get_child_bus(dev, "i2c");
         if (board->peripherals & BP_OLED_I2C) {
             i2c_create_slave(i2c, "ssd0303", 0x3d);
@@ -1266,12 +1304,14 @@
 
     for (i = 0; i < 4; i++) {
         if (board->dc2 & (1 << i)) {
-            sysbus_create_simple("pl011_luminary", 0x4000c000 + i * 0x1000,
-                                 pic[uart_irq[i]]);
+            pl011_luminary_create(0x4000c000 + i * 0x1000,
+                                  qdev_get_gpio_in(nvic, uart_irq[i]),
+                                  serial_hds[i]);
         }
     }
     if (board->dc2 & (1 << 4)) {
-        dev = sysbus_create_simple("pl022", 0x40008000, pic[7]);
+        dev = sysbus_create_simple("pl022", 0x40008000,
+                                   qdev_get_gpio_in(nvic, 7));
         if (board->peripherals & BP_OLED_SSI) {
             void *bus;
             DeviceState *sddev;
@@ -1307,7 +1347,7 @@
         qdev_set_nic_properties(enet, &nd_table[0]);
         qdev_init_nofail(enet);
         sysbus_mmio_map(SYS_BUS_DEVICE(enet), 0, 0x40048000);
-        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, pic[42]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(enet), 0, qdev_get_gpio_in(nvic, 42));
     }
     if (board->peripherals & BP_GAMEPAD) {
         qemu_irq gpad_irq[5];
@@ -1347,65 +1387,84 @@
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[1]);
 }
 
-static QEMUMachine lm3s811evb_machine = {
-    .name = "lm3s811evb",
-    .desc = "Stellaris LM3S811EVB",
-    .init = lm3s811evb_init,
+static void lm3s811evb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Stellaris LM3S811EVB";
+    mc->init = lm3s811evb_init;
+}
+
+static const TypeInfo lm3s811evb_type = {
+    .name = MACHINE_TYPE_NAME("lm3s811evb"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm3s811evb_class_init,
 };
 
-static QEMUMachine lm3s6965evb_machine = {
-    .name = "lm3s6965evb",
-    .desc = "Stellaris LM3S6965EVB",
-    .init = lm3s6965evb_init,
+static void lm3s6965evb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Stellaris LM3S6965EVB";
+    mc->init = lm3s6965evb_init;
+}
+
+static const TypeInfo lm3s6965evb_type = {
+    .name = MACHINE_TYPE_NAME("lm3s6965evb"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm3s6965evb_class_init,
 };
 
 static void stellaris_machine_init(void)
 {
-    qemu_register_machine(&lm3s811evb_machine);
-    qemu_register_machine(&lm3s6965evb_machine);
+    type_register_static(&lm3s811evb_type);
+    type_register_static(&lm3s6965evb_type);
 }
 
-machine_init(stellaris_machine_init);
+type_init(stellaris_machine_init)
 
 static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
-    sdc->init = stellaris_i2c_init;
+    dc->vmsd = &vmstate_stellaris_i2c;
 }
 
 static const TypeInfo stellaris_i2c_info = {
     .name          = TYPE_STELLARIS_I2C,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_i2c_state),
+    .instance_init = stellaris_i2c_init,
     .class_init    = stellaris_i2c_class_init,
 };
 
 static void stellaris_gptm_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
-    sdc->init = stellaris_gptm_init;
+    dc->vmsd = &vmstate_stellaris_gptm;
 }
 
 static const TypeInfo stellaris_gptm_info = {
     .name          = TYPE_STELLARIS_GPTM,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(gptm_state),
+    .instance_init = stellaris_gptm_init,
     .class_init    = stellaris_gptm_class_init,
 };
 
 static void stellaris_adc_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
-    sdc->init = stellaris_adc_init;
+    dc->vmsd = &vmstate_stellaris_adc;
 }
 
 static const TypeInfo stellaris_adc_info = {
     .name          = TYPE_STELLARIS_ADC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(stellaris_adc_state),
+    .instance_init = stellaris_adc_init,
     .class_init    = stellaris_adc_class_init,
 };
 
diff --git a/hw/arm/stm32f205_soc.c b/hw/arm/stm32f205_soc.c
new file mode 100644
index 0000000..de26b8c
--- /dev/null
+++ b/hw/arm/stm32f205_soc.c
@@ -0,0 +1,165 @@
+/*
+ * STM32F205 SoC
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "hw/arm/arm.h"
+#include "exec/address-spaces.h"
+#include "hw/arm/stm32f205_soc.h"
+
+/* At the moment only Timer 2 to 5 are modelled */
+static const uint32_t timer_addr[STM_NUM_TIMERS] = { 0x40000000, 0x40000400,
+    0x40000800, 0x40000C00 };
+static const uint32_t usart_addr[STM_NUM_USARTS] = { 0x40011000, 0x40004400,
+    0x40004800, 0x40004C00, 0x40005000, 0x40011400 };
+
+static const int timer_irq[STM_NUM_TIMERS] = {28, 29, 30, 50};
+static const int usart_irq[STM_NUM_USARTS] = {37, 38, 39, 52, 53, 71};
+
+static void stm32f205_soc_initfn(Object *obj)
+{
+    STM32F205State *s = STM32F205_SOC(obj);
+    int i;
+
+    object_initialize(&s->syscfg, sizeof(s->syscfg), TYPE_STM32F2XX_SYSCFG);
+    qdev_set_parent_bus(DEVICE(&s->syscfg), sysbus_get_default());
+
+    for (i = 0; i < STM_NUM_USARTS; i++) {
+        object_initialize(&s->usart[i], sizeof(s->usart[i]),
+                          TYPE_STM32F2XX_USART);
+        qdev_set_parent_bus(DEVICE(&s->usart[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < STM_NUM_TIMERS; i++) {
+        object_initialize(&s->timer[i], sizeof(s->timer[i]),
+                          TYPE_STM32F2XX_TIMER);
+        qdev_set_parent_bus(DEVICE(&s->timer[i]), sysbus_get_default());
+    }
+}
+
+static void stm32f205_soc_realize(DeviceState *dev_soc, Error **errp)
+{
+    STM32F205State *s = STM32F205_SOC(dev_soc);
+    DeviceState *syscfgdev, *usartdev, *timerdev, *nvic;
+    SysBusDevice *syscfgbusdev, *usartbusdev, *timerbusdev;
+    Error *err = NULL;
+    int i;
+
+    MemoryRegion *system_memory = get_system_memory();
+    MemoryRegion *sram = g_new(MemoryRegion, 1);
+    MemoryRegion *flash = g_new(MemoryRegion, 1);
+    MemoryRegion *flash_alias = g_new(MemoryRegion, 1);
+
+    memory_region_init_ram(flash, NULL, "STM32F205.flash", FLASH_SIZE,
+                           &error_fatal);
+    memory_region_init_alias(flash_alias, NULL, "STM32F205.flash.alias",
+                             flash, 0, FLASH_SIZE);
+
+    vmstate_register_ram_global(flash);
+
+    memory_region_set_readonly(flash, true);
+    memory_region_set_readonly(flash_alias, true);
+
+    memory_region_add_subregion(system_memory, FLASH_BASE_ADDRESS, flash);
+    memory_region_add_subregion(system_memory, 0, flash_alias);
+
+    memory_region_init_ram(sram, NULL, "STM32F205.sram", SRAM_SIZE,
+                           &error_fatal);
+    vmstate_register_ram_global(sram);
+    memory_region_add_subregion(system_memory, SRAM_BASE_ADDRESS, sram);
+
+    nvic = armv7m_init(get_system_memory(), FLASH_SIZE, 96,
+                       s->kernel_filename, s->cpu_model);
+
+    /* System configuration controller */
+    syscfgdev = DEVICE(&s->syscfg);
+    object_property_set_bool(OBJECT(&s->syscfg), true, "realized", &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
+    syscfgbusdev = SYS_BUS_DEVICE(syscfgdev);
+    sysbus_mmio_map(syscfgbusdev, 0, 0x40013800);
+    sysbus_connect_irq(syscfgbusdev, 0, qdev_get_gpio_in(nvic, 71));
+
+    /* Attach UART (uses USART registers) and USART controllers */
+    for (i = 0; i < STM_NUM_USARTS; i++) {
+        usartdev = DEVICE(&(s->usart[i]));
+        qdev_prop_set_chr(usartdev, "chardev", i < MAX_SERIAL_PORTS ? serial_hds[i] : NULL);
+        object_property_set_bool(OBJECT(&s->usart[i]), true, "realized", &err);
+        if (err != NULL) {
+            error_propagate(errp, err);
+            return;
+        }
+        usartbusdev = SYS_BUS_DEVICE(usartdev);
+        sysbus_mmio_map(usartbusdev, 0, usart_addr[i]);
+        sysbus_connect_irq(usartbusdev, 0,
+                           qdev_get_gpio_in(nvic, usart_irq[i]));
+    }
+
+    /* Timer 2 to 5 */
+    for (i = 0; i < STM_NUM_TIMERS; i++) {
+        timerdev = DEVICE(&(s->timer[i]));
+        qdev_prop_set_uint64(timerdev, "clock-frequency", 1000000000);
+        object_property_set_bool(OBJECT(&s->timer[i]), true, "realized", &err);
+        if (err != NULL) {
+            error_propagate(errp, err);
+            return;
+        }
+        timerbusdev = SYS_BUS_DEVICE(timerdev);
+        sysbus_mmio_map(timerbusdev, 0, timer_addr[i]);
+        sysbus_connect_irq(timerbusdev, 0,
+                           qdev_get_gpio_in(nvic, timer_irq[i]));
+    }
+}
+
+static Property stm32f205_soc_properties[] = {
+    DEFINE_PROP_STRING("kernel-filename", STM32F205State, kernel_filename),
+    DEFINE_PROP_STRING("cpu-model", STM32F205State, cpu_model),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void stm32f205_soc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = stm32f205_soc_realize;
+    dc->props = stm32f205_soc_properties;
+}
+
+static const TypeInfo stm32f205_soc_info = {
+    .name          = TYPE_STM32F205_SOC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(STM32F205State),
+    .instance_init = stm32f205_soc_initfn,
+    .class_init    = stm32f205_soc_class_init,
+};
+
+static void stm32f205_soc_types(void)
+{
+    type_register_static(&stm32f205_soc_info);
+}
+
+type_init(stm32f205_soc_types)
diff --git a/hw/arm/strongarm.c b/hw/arm/strongarm.c
index 3206345..f1b2c6c 100644
--- a/hw/arm/strongarm.c
+++ b/hw/arm/strongarm.c
@@ -26,13 +26,19 @@
  *  Contributions after 2012-01-13 are licensed under the terms of the
  *  GNU GPL, version 2 or (at your option) any later version.
  */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "hw/boards.h"
 #include "hw/sysbus.h"
 #include "strongarm.h"
 #include "qemu/error-report.h"
 #include "hw/arm/arm.h"
 #include "sysemu/char.h"
 #include "sysemu/sysemu.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
+#include "qemu/cutils.h"
+#include "qemu/log.h"
 
 //#define DEBUG
 
@@ -174,19 +180,18 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int strongarm_pic_initfn(SysBusDevice *sbd)
+static void strongarm_pic_initfn(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    StrongARMPICState *s = STRONGARM_PIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    StrongARMPICState *s = STRONGARM_PIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     qdev_init_gpio_in(dev, strongarm_pic_set_irq, SA_PIC_SRCS);
-    memory_region_init_io(&s->iomem, OBJECT(s), &strongarm_pic_ops, s,
+    memory_region_init_io(&s->iomem, obj, &strongarm_pic_ops, s,
                           "pic", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq);
     sysbus_init_irq(sbd, &s->fiq);
-
-    return 0;
 }
 
 static int strongarm_pic_post_load(void *opaque, int version_id)
@@ -212,9 +217,7 @@
 static void strongarm_pic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = strongarm_pic_initfn;
     dc->desc = "StrongARM PIC";
     dc->vmsd = &vmstate_strongarm_pic_regs;
 }
@@ -223,6 +226,7 @@
     .name          = TYPE_STRONGARM_PIC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMPICState),
+    .instance_init = strongarm_pic_initfn,
     .class_init    = strongarm_pic_class_init,
 };
 
@@ -376,9 +380,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int strongarm_rtc_init(SysBusDevice *dev)
+static void strongarm_rtc_init(Object *obj)
 {
-    StrongARMRTCState *s = STRONGARM_RTC(dev);
+    StrongARMRTCState *s = STRONGARM_RTC(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     struct tm tm;
 
     s->rttr = 0x0;
@@ -395,11 +400,9 @@
     sysbus_init_irq(dev, &s->rtc_irq);
     sysbus_init_irq(dev, &s->rtc_hz_irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &strongarm_rtc_ops, s,
+    memory_region_init_io(&s->iomem, obj, &strongarm_rtc_ops, s,
                           "rtc", 0x10000);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void strongarm_rtc_pre_save(void *opaque)
@@ -438,9 +441,7 @@
 static void strongarm_rtc_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = strongarm_rtc_init;
     dc->desc = "StrongARM RTC Controller";
     dc->vmsd = &vmstate_strongarm_rtc_regs;
 }
@@ -449,6 +450,7 @@
     .name          = TYPE_STRONGARM_RTC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMRTCState),
+    .instance_init = strongarm_rtc_init,
     .class_init    = strongarm_rtc_sysbus_class_init,
 };
 
@@ -526,7 +528,7 @@
     level = s->olevel & s->dir;
 
     for (diff = s->prev_level ^ level; diff; diff ^= 1 << bit) {
-        bit = ffs(diff) - 1;
+        bit = ctz32(diff);
         qemu_set_irq(s->handler[bit], (level >> bit) & 1);
     }
 
@@ -641,16 +643,17 @@
     return dev;
 }
 
-static int strongarm_gpio_initfn(SysBusDevice *sbd)
+static void strongarm_gpio_initfn(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    StrongARMGPIOInfo *s = STRONGARM_GPIO(dev);
+    DeviceState *dev = DEVICE(obj);
+    StrongARMGPIOInfo *s = STRONGARM_GPIO(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     int i;
 
     qdev_init_gpio_in(dev, strongarm_gpio_set, 28);
     qdev_init_gpio_out(dev, s->handler, 28);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &strongarm_gpio_ops, s,
+    memory_region_init_io(&s->iomem, obj, &strongarm_gpio_ops, s,
                           "gpio", 0x1000);
 
     sysbus_init_mmio(sbd, &s->iomem);
@@ -658,8 +661,6 @@
         sysbus_init_irq(sbd, &s->irqs[i]);
     }
     sysbus_init_irq(sbd, &s->irqX);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_strongarm_gpio_regs = {
@@ -682,9 +683,7 @@
 static void strongarm_gpio_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = strongarm_gpio_initfn;
     dc->desc = "StrongARM GPIO controller";
     dc->vmsd = &vmstate_strongarm_gpio_regs;
 }
@@ -693,6 +692,7 @@
     .name          = TYPE_STRONGARM_GPIO,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMGPIOInfo),
+    .instance_init = strongarm_gpio_initfn,
     .class_init    = strongarm_gpio_class_init,
 };
 
@@ -743,7 +743,7 @@
     level = s->olevel & s->dir;
 
     for (diff = s->prev_level ^ level; diff; diff ^= 1 << bit) {
-        bit = ffs(diff) - 1;
+        bit = ctz32(diff);
         qemu_set_irq(s->handler[bit], (level >> bit) & 1);
     }
 
@@ -819,20 +819,19 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int strongarm_ppc_init(SysBusDevice *sbd)
+static void strongarm_ppc_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    StrongARMPPCInfo *s = STRONGARM_PPC(dev);
+    DeviceState *dev = DEVICE(obj);
+    StrongARMPPCInfo *s = STRONGARM_PPC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     qdev_init_gpio_in(dev, strongarm_ppc_set, 22);
     qdev_init_gpio_out(dev, s->handler, 22);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &strongarm_ppc_ops, s,
+    memory_region_init_io(&s->iomem, obj, &strongarm_ppc_ops, s,
                           "ppc", 0x1000);
 
     sysbus_init_mmio(sbd, &s->iomem);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_strongarm_ppc_regs = {
@@ -854,9 +853,7 @@
 static void strongarm_ppc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = strongarm_ppc_init;
     dc->desc = "StrongARM PPC controller";
     dc->vmsd = &vmstate_strongarm_ppc_regs;
 }
@@ -865,6 +862,7 @@
     .name          = TYPE_STRONGARM_PPC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMPPCInfo),
+    .instance_init = strongarm_ppc_init,
     .class_init    = strongarm_ppc_class_init,
 };
 
@@ -1021,7 +1019,7 @@
     ssp.parity = parity;
     ssp.data_bits = data_bits;
     ssp.stop_bits = stop_bits;
-    s->char_transmit_time =  (get_ticks_per_sec() / speed) * frame_size;
+    s->char_transmit_time =  (NANOSECONDS_PER_SECOND / speed) * frame_size;
     if (s->chr) {
         qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
     }
@@ -1226,11 +1224,12 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int strongarm_uart_init(SysBusDevice *dev)
+static void strongarm_uart_init(Object *obj)
 {
-    StrongARMUARTState *s = STRONGARM_UART(dev);
+    StrongARMUARTState *s = STRONGARM_UART(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &strongarm_uart_ops, s,
+    memory_region_init_io(&s->iomem, obj, &strongarm_uart_ops, s,
                           "uart", 0x10000);
     sysbus_init_mmio(dev, &s->iomem);
     sysbus_init_irq(dev, &s->irq);
@@ -1245,8 +1244,6 @@
                         strongarm_uart_event,
                         s);
     }
-
-    return 0;
 }
 
 static void strongarm_uart_reset(DeviceState *dev)
@@ -1316,9 +1313,7 @@
 static void strongarm_uart_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = strongarm_uart_init;
     dc->desc = "StrongARM UART controller";
     dc->reset = strongarm_uart_reset;
     dc->vmsd = &vmstate_strongarm_uart_regs;
@@ -1329,6 +1324,7 @@
     .name          = TYPE_STRONGARM_UART,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(StrongARMUARTState),
+    .instance_init = strongarm_uart_init,
     .class_init    = strongarm_uart_class_init,
 };
 
@@ -1586,7 +1582,7 @@
     StrongARMState *s;
     int i;
 
-    s = g_malloc0(sizeof(StrongARMState));
+    s = g_new0(StrongARMState, 1);
 
     if (!rev) {
         rev = "sa1110-b5";
@@ -1604,9 +1600,8 @@
         exit(1);
     }
 
-    memory_region_init_ram(&s->sdram, NULL, "strongarm.sdram", sdram_size,
-                           &error_abort);
-    vmstate_register_ram_global(&s->sdram);
+    memory_region_allocate_system_memory(&s->sdram, NULL, "strongarm.sdram",
+                                         sdram_size);
     memory_region_add_subregion(sysmem, SA_SDCS0, &s->sdram);
 
     s->pic = sysbus_create_varargs("strongarm_pic", 0x90050000,
diff --git a/hw/arm/strongarm.h b/hw/arm/strongarm.h
index 2893f94..1470eac 100644
--- a/hw/arm/strongarm.h
+++ b/hw/arm/strongarm.h
@@ -1,7 +1,8 @@
-#ifndef _STRONGARM_H
-#define _STRONGARM_H
+#ifndef STRONGARM_H
+#define STRONGARM_H
 
 #include "exec/memory.h"
+#include "target-arm/cpu-qom.h"
 
 #define SA_CS0          0x00000000
 #define SA_CS1          0x08000000
diff --git a/hw/arm/sysbus-fdt.c b/hw/arm/sysbus-fdt.c
new file mode 100644
index 0000000..5debb33
--- /dev/null
+++ b/hw/arm/sysbus-fdt.c
@@ -0,0 +1,542 @@
+/*
+ * ARM Platform Bus device tree generation helpers
+ *
+ * Copyright (c) 2014 Linaro Limited
+ *
+ * Authors:
+ *  Alex Graf <agraf@suse.de>
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include <libfdt.h>
+#include "qemu-common.h"
+#ifdef CONFIG_LINUX
+#include <linux/vfio.h>
+#endif
+#include "hw/arm/sysbus-fdt.h"
+#include "qemu/error-report.h"
+#include "sysemu/device_tree.h"
+#include "hw/platform-bus.h"
+#include "sysemu/sysemu.h"
+#include "hw/vfio/vfio-platform.h"
+#include "hw/vfio/vfio-calxeda-xgmac.h"
+#include "hw/vfio/vfio-amd-xgbe.h"
+#include "hw/arm/fdt.h"
+
+/*
+ * internal struct that contains the information to create dynamic
+ * sysbus device node
+ */
+typedef struct PlatformBusFDTData {
+    void *fdt; /* device tree handle */
+    int irq_start; /* index of the first IRQ usable by platform bus devices */
+    const char *pbus_node_name; /* name of the platform bus node */
+    PlatformBusDevice *pbus;
+} PlatformBusFDTData;
+
+/*
+ * struct used when calling the machine init done notifier
+ * that constructs the fdt nodes of platform bus devices
+ */
+typedef struct PlatformBusFDTNotifierParams {
+    Notifier notifier;
+    ARMPlatformBusFDTParams *fdt_params;
+} PlatformBusFDTNotifierParams;
+
+/* struct that associates a device type name and a node creation function */
+typedef struct NodeCreationPair {
+    const char *typename;
+    int (*add_fdt_node_fn)(SysBusDevice *sbdev, void *opaque);
+} NodeCreationPair;
+
+/* helpers */
+
+typedef struct HostProperty {
+    const char *name;
+    bool optional;
+} HostProperty;
+
+#ifdef CONFIG_LINUX
+
+/**
+ * copy_properties_from_host
+ *
+ * copies properties listed in an array from host device tree to
+ * guest device tree. If a non optional property is not found, the
+ * function asserts. An optional property is ignored if not found
+ * in the host device tree.
+ * @props: array of HostProperty to copy
+ * @nb_props: number of properties in the array
+ * @host_dt: host device tree blob
+ * @guest_dt: guest device tree blob
+ * @node_path: host dt node path where the property is supposed to be
+              found
+ * @nodename: guest node name the properties should be added to
+ */
+static void copy_properties_from_host(HostProperty *props, int nb_props,
+                                      void *host_fdt, void *guest_fdt,
+                                      char *node_path, char *nodename)
+{
+    int i, prop_len;
+    const void *r;
+    Error *err = NULL;
+
+    for (i = 0; i < nb_props; i++) {
+        r = qemu_fdt_getprop(host_fdt, node_path,
+                             props[i].name,
+                             &prop_len,
+                             props[i].optional ? &err : &error_fatal);
+        if (r) {
+            qemu_fdt_setprop(guest_fdt, nodename,
+                             props[i].name, r, prop_len);
+        } else {
+            if (prop_len != -FDT_ERR_NOTFOUND) {
+                /* optional property not returned although property exists */
+                error_report_err(err);
+            } else {
+                error_free(err);
+            }
+        }
+    }
+}
+
+/* clock properties whose values are copied/pasted from host */
+static HostProperty clock_copied_properties[] = {
+    {"compatible", false},
+    {"#clock-cells", false},
+    {"clock-frequency", true},
+    {"clock-output-names", true},
+};
+
+/**
+ * fdt_build_clock_node
+ *
+ * Build a guest clock node, used as a dependency from a passthrough'ed
+ * device. Most information are retrieved from the host clock node.
+ * Also check the host clock is a fixed one.
+ *
+ * @host_fdt: host device tree blob from which info are retrieved
+ * @guest_fdt: guest device tree blob where the clock node is added
+ * @host_phandle: phandle of the clock in host device tree
+ * @guest_phandle: phandle to assign to the guest node
+ */
+static void fdt_build_clock_node(void *host_fdt, void *guest_fdt,
+                                uint32_t host_phandle,
+                                uint32_t guest_phandle)
+{
+    char *node_path = NULL;
+    char *nodename;
+    const void *r;
+    int ret, node_offset, prop_len, path_len = 16;
+
+    node_offset = fdt_node_offset_by_phandle(host_fdt, host_phandle);
+    if (node_offset <= 0) {
+        error_setg(&error_fatal,
+                   "not able to locate clock handle %d in host device tree",
+                   host_phandle);
+    }
+    node_path = g_malloc(path_len);
+    while ((ret = fdt_get_path(host_fdt, node_offset, node_path, path_len))
+            == -FDT_ERR_NOSPACE) {
+        path_len += 16;
+        node_path = g_realloc(node_path, path_len);
+    }
+    if (ret < 0) {
+        error_setg(&error_fatal,
+                   "not able to retrieve node path for clock handle %d",
+                   host_phandle);
+    }
+
+    r = qemu_fdt_getprop(host_fdt, node_path, "compatible", &prop_len,
+                         &error_fatal);
+    if (strcmp(r, "fixed-clock")) {
+        error_setg(&error_fatal,
+                   "clock handle %d is not a fixed clock", host_phandle);
+    }
+
+    nodename = strrchr(node_path, '/');
+    qemu_fdt_add_subnode(guest_fdt, nodename);
+
+    copy_properties_from_host(clock_copied_properties,
+                              ARRAY_SIZE(clock_copied_properties),
+                              host_fdt, guest_fdt,
+                              node_path, nodename);
+
+    qemu_fdt_setprop_cell(guest_fdt, nodename, "phandle", guest_phandle);
+
+    g_free(node_path);
+}
+
+/**
+ * sysfs_to_dt_name: convert the name found in sysfs into the node name
+ * for instance e0900000.xgmac is converted into xgmac@e0900000
+ * @sysfs_name: directory name in sysfs
+ *
+ * returns the device tree name upon success or NULL in case the sysfs name
+ * does not match the expected format
+ */
+static char *sysfs_to_dt_name(const char *sysfs_name)
+{
+    gchar **substrings =  g_strsplit(sysfs_name, ".", 2);
+    char *dt_name = NULL;
+
+    if (!substrings || !substrings[0] || !substrings[1]) {
+        goto out;
+    }
+    dt_name = g_strdup_printf("%s@%s", substrings[1], substrings[0]);
+out:
+    g_strfreev(substrings);
+    return dt_name;
+}
+
+/* Device Specific Code */
+
+/**
+ * add_calxeda_midway_xgmac_fdt_node
+ *
+ * Generates a simple node with following properties:
+ * compatible string, regs, interrupts, dma-coherent
+ */
+static int add_calxeda_midway_xgmac_fdt_node(SysBusDevice *sbdev, void *opaque)
+{
+    PlatformBusFDTData *data = opaque;
+    PlatformBusDevice *pbus = data->pbus;
+    void *fdt = data->fdt;
+    const char *parent_node = data->pbus_node_name;
+    int compat_str_len, i;
+    char *nodename;
+    uint32_t *irq_attr, *reg_attr;
+    uint64_t mmio_base, irq_number;
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIODevice *vbasedev = &vdev->vbasedev;
+
+    mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, 0);
+    nodename = g_strdup_printf("%s/%s@%" PRIx64, parent_node,
+                               vbasedev->name, mmio_base);
+    qemu_fdt_add_subnode(fdt, nodename);
+
+    compat_str_len = strlen(vdev->compat) + 1;
+    qemu_fdt_setprop(fdt, nodename, "compatible",
+                          vdev->compat, compat_str_len);
+
+    qemu_fdt_setprop(fdt, nodename, "dma-coherent", "", 0);
+
+    reg_attr = g_new(uint32_t, vbasedev->num_regions * 2);
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, i);
+        reg_attr[2 * i] = cpu_to_be32(mmio_base);
+        reg_attr[2 * i + 1] = cpu_to_be32(
+                                memory_region_size(vdev->regions[i]->mem));
+    }
+    qemu_fdt_setprop(fdt, nodename, "reg", reg_attr,
+                     vbasedev->num_regions * 2 * sizeof(uint32_t));
+
+    irq_attr = g_new(uint32_t, vbasedev->num_irqs * 3);
+    for (i = 0; i < vbasedev->num_irqs; i++) {
+        irq_number = platform_bus_get_irqn(pbus, sbdev , i)
+                         + data->irq_start;
+        irq_attr[3 * i] = cpu_to_be32(GIC_FDT_IRQ_TYPE_SPI);
+        irq_attr[3 * i + 1] = cpu_to_be32(irq_number);
+        irq_attr[3 * i + 2] = cpu_to_be32(GIC_FDT_IRQ_FLAGS_LEVEL_HI);
+    }
+    qemu_fdt_setprop(fdt, nodename, "interrupts",
+                     irq_attr, vbasedev->num_irqs * 3 * sizeof(uint32_t));
+    g_free(irq_attr);
+    g_free(reg_attr);
+    g_free(nodename);
+    return 0;
+}
+
+/* AMD xgbe properties whose values are copied/pasted from host */
+static HostProperty amd_xgbe_copied_properties[] = {
+    {"compatible", false},
+    {"dma-coherent", true},
+    {"amd,per-channel-interrupt", true},
+    {"phy-mode", false},
+    {"mac-address", true},
+    {"amd,speed-set", false},
+    {"amd,serdes-blwc", true},
+    {"amd,serdes-cdr-rate", true},
+    {"amd,serdes-pq-skew", true},
+    {"amd,serdes-tx-amp", true},
+    {"amd,serdes-dfe-tap-config", true},
+    {"amd,serdes-dfe-tap-enable", true},
+    {"clock-names", false},
+};
+
+/**
+ * add_amd_xgbe_fdt_node
+ *
+ * Generates the combined xgbe/phy node following kernel >=4.2
+ * binding documentation:
+ * Documentation/devicetree/bindings/net/amd-xgbe.txt:
+ * Also 2 clock nodes are created (dma and ptp)
+ *
+ * Asserts in case of error
+ */
+static int add_amd_xgbe_fdt_node(SysBusDevice *sbdev, void *opaque)
+{
+    PlatformBusFDTData *data = opaque;
+    PlatformBusDevice *pbus = data->pbus;
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIODevice *vbasedev = &vdev->vbasedev;
+    VFIOINTp *intp;
+    const char *parent_node = data->pbus_node_name;
+    char **node_path, *nodename, *dt_name;
+    void *guest_fdt = data->fdt, *host_fdt;
+    const void *r;
+    int i, prop_len;
+    uint32_t *irq_attr, *reg_attr, *host_clock_phandles;
+    uint64_t mmio_base, irq_number;
+    uint32_t guest_clock_phandles[2];
+
+    host_fdt = load_device_tree_from_sysfs();
+
+    dt_name = sysfs_to_dt_name(vbasedev->name);
+    if (!dt_name) {
+        error_setg(&error_fatal, "%s incorrect sysfs device name %s",
+                    __func__, vbasedev->name);
+    }
+    node_path = qemu_fdt_node_path(host_fdt, dt_name, vdev->compat,
+                                   &error_fatal);
+    if (!node_path || !node_path[0]) {
+        error_setg(&error_fatal, "%s unable to retrieve node path for %s/%s",
+                   __func__, dt_name, vdev->compat);
+    }
+
+    if (node_path[1]) {
+        error_setg(&error_fatal, "%s more than one node matching %s/%s!",
+                   __func__, dt_name, vdev->compat);
+    }
+
+    g_free(dt_name);
+
+    if (vbasedev->num_regions != 5) {
+        error_setg(&error_fatal, "%s Does the host dt node combine XGBE/PHY?",
+                   __func__);
+    }
+
+    /* generate nodes for DMA_CLK and PTP_CLK */
+    r = qemu_fdt_getprop(host_fdt, node_path[0], "clocks",
+                         &prop_len, &error_fatal);
+    if (prop_len != 8) {
+        error_setg(&error_fatal, "%s clocks property should contain 2 handles",
+                   __func__);
+    }
+    host_clock_phandles = (uint32_t *)r;
+    guest_clock_phandles[0] = qemu_fdt_alloc_phandle(guest_fdt);
+    guest_clock_phandles[1] = qemu_fdt_alloc_phandle(guest_fdt);
+
+    /**
+     * clock handles fetched from host dt are in be32 layout whereas
+     * rest of the code uses cpu layout. Also guest clock handles are
+     * in cpu layout.
+     */
+    fdt_build_clock_node(host_fdt, guest_fdt,
+                         be32_to_cpu(host_clock_phandles[0]),
+                         guest_clock_phandles[0]);
+
+    fdt_build_clock_node(host_fdt, guest_fdt,
+                         be32_to_cpu(host_clock_phandles[1]),
+                         guest_clock_phandles[1]);
+
+    /* combined XGBE/PHY node */
+    mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, 0);
+    nodename = g_strdup_printf("%s/%s@%" PRIx64, parent_node,
+                               vbasedev->name, mmio_base);
+    qemu_fdt_add_subnode(guest_fdt, nodename);
+
+    copy_properties_from_host(amd_xgbe_copied_properties,
+                       ARRAY_SIZE(amd_xgbe_copied_properties),
+                       host_fdt, guest_fdt,
+                       node_path[0], nodename);
+
+    qemu_fdt_setprop_cells(guest_fdt, nodename, "clocks",
+                           guest_clock_phandles[0],
+                           guest_clock_phandles[1]);
+
+    reg_attr = g_new(uint32_t, vbasedev->num_regions * 2);
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        mmio_base = platform_bus_get_mmio_addr(pbus, sbdev, i);
+        reg_attr[2 * i] = cpu_to_be32(mmio_base);
+        reg_attr[2 * i + 1] = cpu_to_be32(
+                                memory_region_size(vdev->regions[i]->mem));
+    }
+    qemu_fdt_setprop(guest_fdt, nodename, "reg", reg_attr,
+                     vbasedev->num_regions * 2 * sizeof(uint32_t));
+
+    irq_attr = g_new(uint32_t, vbasedev->num_irqs * 3);
+    for (i = 0; i < vbasedev->num_irqs; i++) {
+        irq_number = platform_bus_get_irqn(pbus, sbdev , i)
+                         + data->irq_start;
+        irq_attr[3 * i] = cpu_to_be32(GIC_FDT_IRQ_TYPE_SPI);
+        irq_attr[3 * i + 1] = cpu_to_be32(irq_number);
+        /*
+          * General device interrupt and PCS auto-negotiation interrupts are
+          * level-sensitive while the 4 per-channel interrupts are edge
+          * sensitive
+          */
+        QLIST_FOREACH(intp, &vdev->intp_list, next) {
+            if (intp->pin == i) {
+                break;
+            }
+        }
+        if (intp->flags & VFIO_IRQ_INFO_AUTOMASKED) {
+            irq_attr[3 * i + 2] = cpu_to_be32(GIC_FDT_IRQ_FLAGS_LEVEL_HI);
+        } else {
+            irq_attr[3 * i + 2] = cpu_to_be32(GIC_FDT_IRQ_FLAGS_EDGE_LO_HI);
+        }
+    }
+    qemu_fdt_setprop(guest_fdt, nodename, "interrupts",
+                     irq_attr, vbasedev->num_irqs * 3 * sizeof(uint32_t));
+
+    g_free(host_fdt);
+    g_strfreev(node_path);
+    g_free(irq_attr);
+    g_free(reg_attr);
+    g_free(nodename);
+    return 0;
+}
+
+#endif /* CONFIG_LINUX */
+
+/* list of supported dynamic sysbus devices */
+static const NodeCreationPair add_fdt_node_functions[] = {
+#ifdef CONFIG_LINUX
+    {TYPE_VFIO_CALXEDA_XGMAC, add_calxeda_midway_xgmac_fdt_node},
+    {TYPE_VFIO_AMD_XGBE, add_amd_xgbe_fdt_node},
+#endif
+    {"", NULL}, /* last element */
+};
+
+/* Generic Code */
+
+/**
+ * add_fdt_node - add the device tree node of a dynamic sysbus device
+ *
+ * @sbdev: handle to the sysbus device
+ * @opaque: handle to the PlatformBusFDTData
+ *
+ * Checks the sysbus type belongs to the list of device types that
+ * are dynamically instantiable and if so call the node creation
+ * function.
+ */
+static int add_fdt_node(SysBusDevice *sbdev, void *opaque)
+{
+    int i, ret;
+
+    for (i = 0; i < ARRAY_SIZE(add_fdt_node_functions); i++) {
+        if (!strcmp(object_get_typename(OBJECT(sbdev)),
+                    add_fdt_node_functions[i].typename)) {
+            ret = add_fdt_node_functions[i].add_fdt_node_fn(sbdev, opaque);
+            assert(!ret);
+            return 0;
+        }
+    }
+    error_report("Device %s can not be dynamically instantiated",
+                     qdev_fw_name(DEVICE(sbdev)));
+    exit(1);
+}
+
+/**
+ * add_all_platform_bus_fdt_nodes - create all the platform bus nodes
+ *
+ * builds the parent platform bus node and all the nodes of dynamic
+ * sysbus devices attached to it.
+ */
+static void add_all_platform_bus_fdt_nodes(ARMPlatformBusFDTParams *fdt_params)
+{
+    const char platcomp[] = "qemu,platform\0simple-bus";
+    PlatformBusDevice *pbus;
+    DeviceState *dev;
+    gchar *node;
+    uint64_t addr, size;
+    int irq_start, dtb_size;
+    struct arm_boot_info *info = fdt_params->binfo;
+    const ARMPlatformBusSystemParams *params = fdt_params->system_params;
+    const char *intc = fdt_params->intc;
+    void *fdt = info->get_dtb(info, &dtb_size);
+
+    /*
+     * If the user provided a dtb, we assume the dynamic sysbus nodes
+     * already are integrated there. This corresponds to a use case where
+     * the dynamic sysbus nodes are complex and their generation is not yet
+     * supported. In that case the user can take charge of the guest dt
+     * while qemu takes charge of the qom stuff.
+     */
+    if (info->dtb_filename) {
+        return;
+    }
+
+    assert(fdt);
+
+    node = g_strdup_printf("/platform@%"PRIx64, params->platform_bus_base);
+    addr = params->platform_bus_base;
+    size = params->platform_bus_size;
+    irq_start = params->platform_bus_first_irq;
+
+    /* Create a /platform node that we can put all devices into */
+    qemu_fdt_add_subnode(fdt, node);
+    qemu_fdt_setprop(fdt, node, "compatible", platcomp, sizeof(platcomp));
+
+    /* Our platform bus region is less than 32bits, so 1 cell is enough for
+     * address and size
+     */
+    qemu_fdt_setprop_cells(fdt, node, "#size-cells", 1);
+    qemu_fdt_setprop_cells(fdt, node, "#address-cells", 1);
+    qemu_fdt_setprop_cells(fdt, node, "ranges", 0, addr >> 32, addr, size);
+
+    qemu_fdt_setprop_phandle(fdt, node, "interrupt-parent", intc);
+
+    dev = qdev_find_recursive(sysbus_get_default(), TYPE_PLATFORM_BUS_DEVICE);
+    pbus = PLATFORM_BUS_DEVICE(dev);
+
+    /* We can only create dt nodes for dynamic devices when they're ready */
+    assert(pbus->done_gathering);
+
+    PlatformBusFDTData data = {
+        .fdt = fdt,
+        .irq_start = irq_start,
+        .pbus_node_name = node,
+        .pbus = pbus,
+    };
+
+    /* Loop through all dynamic sysbus devices and create their node */
+    foreach_dynamic_sysbus_device(add_fdt_node, &data);
+
+    g_free(node);
+}
+
+static void platform_bus_fdt_notify(Notifier *notifier, void *data)
+{
+    PlatformBusFDTNotifierParams *p = DO_UPCAST(PlatformBusFDTNotifierParams,
+                                                notifier, notifier);
+
+    add_all_platform_bus_fdt_nodes(p->fdt_params);
+    g_free(p->fdt_params);
+    g_free(p);
+}
+
+void arm_register_platform_bus_fdt_creator(ARMPlatformBusFDTParams *fdt_params)
+{
+    PlatformBusFDTNotifierParams *p = g_new(PlatformBusFDTNotifierParams, 1);
+
+    p->fdt_params = fdt_params;
+    p->notifier.notify = platform_bus_fdt_notify;
+    qemu_add_machine_init_done_notifier(&p->notifier);
+}
diff --git a/hw/arm/tosa.c b/hw/arm/tosa.c
index 73572eb..2db6650 100644
--- a/hw/arm/tosa.c
+++ b/hw/arm/tosa.c
@@ -11,6 +11,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
@@ -19,7 +21,7 @@
 #include "hw/pcmcia.h"
 #include "hw/boards.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "sysemu/block-backend.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
@@ -125,10 +127,9 @@
     return 0;
 }
 
-static int tosa_ssp_init(SSISlave *dev)
+static void tosa_ssp_realize(SSISlave *dev, Error **errp)
 {
     /* Nothing to do.  */
-    return 0;
 }
 
 #define TYPE_TOSA_DAC "tosa_dac"
@@ -227,7 +228,7 @@
 
     mpu = pxa255_init(address_space_mem, tosa_binfo.ram_size);
 
-    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_abort);
+    memory_region_init_ram(rom, NULL, "tosa.rom", TOSA_ROM, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(address_space_mem, 0, rom);
@@ -252,18 +253,13 @@
     sl_bootparam_write(SL_PXA_PARAM_BASE);
 }
 
-static QEMUMachine tosapda_machine = {
-    .name = "tosa",
-    .desc = "Tosa PDA (PXA255)",
-    .init = tosa_init,
-};
-
-static void tosapda_machine_init(void)
+static void tosapda_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&tosapda_machine);
+    mc->desc = "Sharp SL-6000 (Tosa) PDA (PXA255)";
+    mc->init = tosa_init;
 }
 
-machine_init(tosapda_machine_init);
+DEFINE_MACHINE("tosa", tosapda_machine_init)
 
 static void tosa_dac_class_init(ObjectClass *klass, void *data)
 {
@@ -286,7 +282,7 @@
 {
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = tosa_ssp_init;
+    k->realize = tosa_ssp_realize;
     k->transfer = tosa_ssp_tansfer;
 }
 
diff --git a/hw/arm/trace-events b/hw/arm/trace-events
new file mode 100644
index 0000000..d5f33a2
--- /dev/null
+++ b/hw/arm/trace-events
@@ -0,0 +1,4 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/arm/virt-acpi-build.c
+virt_acpi_setup(void) "No fw cfg or ACPI disabled. Bailing out."
diff --git a/hw/arm/versatilepb.c b/hw/arm/versatilepb.c
index e6ef0a2..8ae5392 100644
--- a/hw/arm/versatilepb.c
+++ b/hw/arm/versatilepb.c
@@ -7,6 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
@@ -18,6 +22,8 @@
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "hw/block/flash.h"
+#include "qemu/error-report.h"
+#include "hw/char/pl011.h"
 
 #define VERSATILE_FLASH_ADDR 0x34000000
 #define VERSATILE_FLASH_SIZE (64 * 1024 * 1024)
@@ -148,10 +154,11 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int vpb_sic_init(SysBusDevice *sbd)
+static void vpb_sic_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    vpb_sic_state *s = VERSATILE_PB_SIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    vpb_sic_state *s = VERSATILE_PB_SIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     int i;
 
     qdev_init_gpio_in(dev, vpb_sic_set_irq, 32);
@@ -159,10 +166,9 @@
         sysbus_init_irq(sbd, &s->parent[i]);
     }
     s->irq = 31;
-    memory_region_init_io(&s->iomem, OBJECT(s), &vpb_sic_ops, s,
+    memory_region_init_io(&s->iomem, obj, &vpb_sic_ops, s,
                           "vpb-sic", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
-    return 0;
 }
 
 /* Board init.  */
@@ -175,6 +181,8 @@
 
 static void versatile_init(MachineState *machine, int board_id)
 {
+    ObjectClass *cpu_oc;
+    Object *cpuobj;
     ARMCPU *cpu;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -193,14 +201,29 @@
     if (!machine->cpu_model) {
         machine->cpu_model = "arm926";
     }
-    cpu = cpu_arm_init(machine->cpu_model);
-    if (!cpu) {
+
+    cpu_oc = cpu_class_by_name(TYPE_ARM_CPU, machine->cpu_model);
+    if (!cpu_oc) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-    memory_region_init_ram(ram, NULL, "versatile.ram", machine->ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ram);
+
+    cpuobj = object_new(object_class_get_name(cpu_oc));
+
+    /* By default ARM1176 CPUs have EL3 enabled.  This board does not
+     * currently support EL3 so the CPU EL3 property is disabled before
+     * realization.
+     */
+    if (object_property_find(cpuobj, "has_el3", NULL)) {
+        object_property_set_bool(cpuobj, false, "has_el3", &error_fatal);
+    }
+
+    object_property_set_bool(cpuobj, true, "realized", &error_fatal);
+
+    cpu = ARM_CPU(cpuobj);
+
+    memory_region_allocate_system_memory(ram, NULL, "versatile.ram",
+                                         machine->ram_size);
     /* ??? RAM should repeat to fill physical memory space.  */
     /* SDRAM at address zero.  */
     memory_region_add_subregion(sysmem, 0, ram);
@@ -253,7 +276,7 @@
             pci_nic_init_nofail(nd, pci_bus, "rtl8139", NULL);
         }
     }
-    if (usb_enabled(false)) {
+    if (machine_usb(machine)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
     n = drive_get_max_bus(IF_SCSI);
@@ -262,10 +285,10 @@
         n--;
     }
 
-    sysbus_create_simple("pl011", 0x101f1000, pic[12]);
-    sysbus_create_simple("pl011", 0x101f2000, pic[13]);
-    sysbus_create_simple("pl011", 0x101f3000, pic[14]);
-    sysbus_create_simple("pl011", 0x10009000, sic[6]);
+    pl011_create(0x101f1000, pic[12], serial_hds[0]);
+    pl011_create(0x101f2000, pic[13], serial_hds[1]);
+    pl011_create(0x101f3000, pic[14], serial_hds[2]);
+    pl011_create(0x10009000, sic[6], serial_hds[3]);
 
     sysbus_create_simple("pl080", 0x10130000, pic[17]);
     sysbus_create_simple("sp804", 0x101e2000, pic[4]);
@@ -364,34 +387,48 @@
     versatile_init(machine, 0x25e);
 }
 
-static QEMUMachine versatilepb_machine = {
-    .name = "versatilepb",
-    .desc = "ARM Versatile/PB (ARM926EJ-S)",
-    .init = vpb_init,
-    .block_default_type = IF_SCSI,
+static void versatilepb_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile/PB (ARM926EJ-S)";
+    mc->init = vpb_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo versatilepb_type = {
+    .name = MACHINE_TYPE_NAME("versatilepb"),
+    .parent = TYPE_MACHINE,
+    .class_init = versatilepb_class_init,
 };
 
-static QEMUMachine versatileab_machine = {
-    .name = "versatileab",
-    .desc = "ARM Versatile/AB (ARM926EJ-S)",
-    .init = vab_init,
-    .block_default_type = IF_SCSI,
+static void versatileab_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile/AB (ARM926EJ-S)";
+    mc->init = vab_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo versatileab_type = {
+    .name = MACHINE_TYPE_NAME("versatileab"),
+    .parent = TYPE_MACHINE,
+    .class_init = versatileab_class_init,
 };
 
 static void versatile_machine_init(void)
 {
-    qemu_register_machine(&versatilepb_machine);
-    qemu_register_machine(&versatileab_machine);
+    type_register_static(&versatilepb_type);
+    type_register_static(&versatileab_type);
 }
 
-machine_init(versatile_machine_init);
+type_init(versatile_machine_init)
 
 static void vpb_sic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = vpb_sic_init;
     dc->vmsd = &vmstate_vpb_sic;
 }
 
@@ -399,6 +436,7 @@
     .name          = TYPE_VERSATILE_PB_SIC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(vpb_sic_state),
+    .instance_init = vpb_sic_init,
     .class_init    = vpb_sic_class_init,
 };
 
diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c
index 7cbd13f..58760f4 100644
--- a/hw/arm/vexpress.c
+++ b/hw/arm/vexpress.c
@@ -21,6 +21,10 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
@@ -35,6 +39,7 @@
 #include "sysemu/device_tree.h"
 #include "qemu/error-report.h"
 #include <libfdt.h>
+#include "hw/char/pl011.h"
 
 #define VEXPRESS_BOARD_ID 0x8e0
 #define VEXPRESS_FLASH_SIZE (64 * 1024 * 1024)
@@ -157,7 +162,27 @@
 
 typedef struct VEDBoardInfo VEDBoardInfo;
 
-typedef void DBoardInitFn(const VEDBoardInfo *daughterboard,
+typedef struct {
+    MachineClass parent;
+    VEDBoardInfo *daughterboard;
+} VexpressMachineClass;
+
+typedef struct {
+    MachineState parent;
+    bool secure;
+} VexpressMachineState;
+
+#define TYPE_VEXPRESS_MACHINE   "vexpress"
+#define TYPE_VEXPRESS_A9_MACHINE   MACHINE_TYPE_NAME("vexpress-a9")
+#define TYPE_VEXPRESS_A15_MACHINE   MACHINE_TYPE_NAME("vexpress-a15")
+#define VEXPRESS_MACHINE(obj) \
+    OBJECT_CHECK(VexpressMachineState, (obj), TYPE_VEXPRESS_MACHINE)
+#define VEXPRESS_MACHINE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(VexpressMachineClass, obj, TYPE_VEXPRESS_MACHINE)
+#define VEXPRESS_MACHINE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(VexpressMachineClass, klass, TYPE_VEXPRESS_MACHINE)
+
+typedef void DBoardInitFn(const VexpressMachineState *machine,
                           ram_addr_t ram_size,
                           const char *cpu_model,
                           qemu_irq *pic);
@@ -176,7 +201,7 @@
 };
 
 static void init_cpus(const char *cpu_model, const char *privdev,
-                      hwaddr periphbase, qemu_irq *pic)
+                      hwaddr periphbase, qemu_irq *pic, bool secure)
 {
     ObjectClass *cpu_oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
     DeviceState *dev;
@@ -191,17 +216,16 @@
     /* Create the actual CPUs */
     for (n = 0; n < smp_cpus; n++) {
         Object *cpuobj = object_new(object_class_get_name(cpu_oc));
-        Error *err = NULL;
+
+        if (!secure) {
+            object_property_set_bool(cpuobj, false, "has_el3", NULL);
+        }
 
         if (object_property_find(cpuobj, "reset-cbar", NULL)) {
             object_property_set_int(cpuobj, periphbase,
                                     "reset-cbar", &error_abort);
         }
-        object_property_set_bool(cpuobj, true, "realized", &err);
-        if (err) {
-            error_report("%s", error_get_pretty(err));
-            exit(1);
-        }
+        object_property_set_bool(cpuobj, true, "realized", &error_fatal);
     }
 
     /* Create the private peripheral devices (including the GIC);
@@ -229,10 +253,12 @@
         DeviceState *cpudev = DEVICE(qemu_get_cpu(n));
 
         sysbus_connect_irq(busdev, n, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ));
+        sysbus_connect_irq(busdev, n + smp_cpus,
+                           qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
     }
 }
 
-static void a9_daughterboard_init(const VEDBoardInfo *daughterboard,
+static void a9_daughterboard_init(const VexpressMachineState *vms,
                                   ram_addr_t ram_size,
                                   const char *cpu_model,
                                   qemu_irq *pic)
@@ -252,9 +278,8 @@
         exit(1);
     }
 
-    memory_region_init_ram(ram, NULL, "vexpress.highmem", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "vexpress.highmem",
+                                         ram_size);
     low_ram_size = ram_size;
     if (low_ram_size > 0x4000000) {
         low_ram_size = 0x4000000;
@@ -268,7 +293,7 @@
     memory_region_add_subregion(sysmem, 0x60000000, ram);
 
     /* 0x1e000000 A9MPCore (SCU) private memory region */
-    init_cpus(cpu_model, "a9mpcore_priv", 0x1e000000, pic);
+    init_cpus(cpu_model, "a9mpcore_priv", 0x1e000000, pic, vms->secure);
 
     /* Daughterboard peripherals : 0x10020000 .. 0x20000000 */
 
@@ -322,7 +347,7 @@
     .init = a9_daughterboard_init,
 };
 
-static void a15_daughterboard_init(const VEDBoardInfo *daughterboard,
+static void a15_daughterboard_init(const VexpressMachineState *vms,
                                    ram_addr_t ram_size,
                                    const char *cpu_model,
                                    qemu_irq *pic)
@@ -347,14 +372,13 @@
         }
     }
 
-    memory_region_init_ram(ram, NULL, "vexpress.highmem", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "vexpress.highmem",
+                                         ram_size);
     /* RAM is from 0x80000000 upwards; there is no low-memory alias for it. */
     memory_region_add_subregion(sysmem, 0x80000000, ram);
 
     /* 0x2c000000 A15MPCore private memory region (GIC) */
-    init_cpus(cpu_model, "a15mpcore_priv", 0x2c000000, pic);
+    init_cpus(cpu_model, "a15mpcore_priv", 0x2c000000, pic, vms->secure);
 
     /* A15 daughterboard peripherals: */
 
@@ -367,7 +391,7 @@
     /* 0x2b0a0000: PL341 dynamic memory controller: not modelled */
     /* 0x2e000000: system SRAM */
     memory_region_init_ram(sram, NULL, "vexpress.a15sram", 0x10000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0x2e000000, sram);
 
@@ -458,8 +482,10 @@
     uint32_t acells, scells, intc;
     const VEDBoardInfo *daughterboard = (const VEDBoardInfo *)info;
 
-    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells");
-    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells");
+    acells = qemu_fdt_getprop_cell(fdt, "/", "#address-cells",
+                                   NULL, &error_fatal);
+    scells = qemu_fdt_getprop_cell(fdt, "/", "#size-cells",
+                                   NULL, &error_fatal);
     intc = find_int_controller(fdt);
     if (!intc) {
         /* Not fatal, we just won't provide virtio. This will
@@ -491,9 +517,9 @@
 {
     DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
 
-    if (di && qdev_prop_set_drive(dev, "drive",
-                                  blk_by_legacy_dinfo(di))) {
-        abort();
+    if (di) {
+        qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(di),
+                            &error_abort);
     }
 
     qdev_prop_set_uint32(dev, "num-blocks",
@@ -501,7 +527,7 @@
     qdev_prop_set_uint64(dev, "sector-length", VEXPRESS_FLASH_SECT_SIZE);
     qdev_prop_set_uint8(dev, "width", 4);
     qdev_prop_set_uint8(dev, "device-width", 2);
-    qdev_prop_set_uint8(dev, "big-endian", 0);
+    qdev_prop_set_bit(dev, "big-endian", false);
     qdev_prop_set_uint16(dev, "id0", 0x89);
     qdev_prop_set_uint16(dev, "id1", 0x18);
     qdev_prop_set_uint16(dev, "id2", 0x00);
@@ -513,9 +539,11 @@
     return OBJECT_CHECK(pflash_t, (dev), "cfi.pflash01");
 }
 
-static void vexpress_common_init(VEDBoardInfo *daughterboard,
-                                 MachineState *machine)
+static void vexpress_common_init(MachineState *machine)
 {
+    VexpressMachineState *vms = VEXPRESS_MACHINE(machine);
+    VexpressMachineClass *vmc = VEXPRESS_MACHINE_GET_CLASS(machine);
+    VEDBoardInfo *daughterboard = vmc->daughterboard;
     DeviceState *dev, *sysctl, *pl041;
     qemu_irq pic[64];
     uint32_t sys_id;
@@ -530,14 +558,14 @@
     const hwaddr *map = daughterboard->motherboard_map;
     int i;
 
-    daughterboard->init(daughterboard, machine->ram_size, machine->cpu_model,
-                        pic);
+    daughterboard->init(vms, machine->ram_size, machine->cpu_model, pic);
 
     /*
      * If a bios file was provided, attempt to map it into memory
      */
     if (bios_name) {
-        const char *fn;
+        char *fn;
+        int image_size;
 
         if (drive_get(IF_PFLASH, 0, 0)) {
             error_report("The contents of the first flash device may be "
@@ -546,8 +574,14 @@
             exit(1);
         }
         fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
-        if (!fn || load_image_targphys(fn, map[VE_NORFLASH0],
-                                       VEXPRESS_FLASH_SIZE) < 0) {
+        if (!fn) {
+            error_report("Could not find ROM image '%s'", bios_name);
+            exit(1);
+        }
+        image_size = load_image_targphys(fn, map[VE_NORFLASH0],
+                                         VEXPRESS_FLASH_SIZE);
+        g_free(fn);
+        if (image_size < 0) {
             error_report("Could not load ROM image '%s'", bios_name);
             exit(1);
         }
@@ -598,10 +632,10 @@
     sysbus_create_simple("pl050_keyboard", map[VE_KMI0], pic[12]);
     sysbus_create_simple("pl050_mouse", map[VE_KMI1], pic[13]);
 
-    sysbus_create_simple("pl011", map[VE_UART0], pic[5]);
-    sysbus_create_simple("pl011", map[VE_UART1], pic[6]);
-    sysbus_create_simple("pl011", map[VE_UART2], pic[7]);
-    sysbus_create_simple("pl011", map[VE_UART3], pic[8]);
+    pl011_create(map[VE_UART0], pic[5], serial_hds[0]);
+    pl011_create(map[VE_UART1], pic[6], serial_hds[1]);
+    pl011_create(map[VE_UART2], pic[7], serial_hds[2]);
+    pl011_create(map[VE_UART3], pic[8], serial_hds[3]);
 
     sysbus_create_simple("sp804", map[VE_TIMER01], pic[2]);
     sysbus_create_simple("sp804", map[VE_TIMER23], pic[3]);
@@ -639,13 +673,13 @@
 
     sram_size = 0x2000000;
     memory_region_init_ram(sram, NULL, "vexpress.sram", sram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, map[VE_SRAM], sram);
 
     vram_size = 0x800000;
     memory_region_init_ram(vram, NULL, "vexpress.vram", vram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(vram);
     memory_region_add_subregion(sysmem, map[VE_VIDEORAM], vram);
 
@@ -678,39 +712,96 @@
     daughterboard->bootinfo.smp_bootreg_addr = map[VE_SYSREGS] + 0x30;
     daughterboard->bootinfo.gic_cpu_if_addr = daughterboard->gic_cpu_if_addr;
     daughterboard->bootinfo.modify_dtb = vexpress_modify_dtb;
+    /* Indicate that when booting Linux we should be in secure state */
+    daughterboard->bootinfo.secure_boot = true;
     arm_load_kernel(ARM_CPU(first_cpu), &daughterboard->bootinfo);
 }
 
-static void vexpress_a9_init(MachineState *machine)
+static bool vexpress_get_secure(Object *obj, Error **errp)
 {
-    vexpress_common_init(&a9_daughterboard, machine);
+    VexpressMachineState *vms = VEXPRESS_MACHINE(obj);
+
+    return vms->secure;
 }
 
-static void vexpress_a15_init(MachineState *machine)
+static void vexpress_set_secure(Object *obj, bool value, Error **errp)
 {
-    vexpress_common_init(&a15_daughterboard, machine);
+    VexpressMachineState *vms = VEXPRESS_MACHINE(obj);
+
+    vms->secure = value;
 }
 
-static QEMUMachine vexpress_a9_machine = {
-    .name = "vexpress-a9",
-    .desc = "ARM Versatile Express for Cortex-A9",
-    .init = vexpress_a9_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static void vexpress_instance_init(Object *obj)
+{
+    VexpressMachineState *vms = VEXPRESS_MACHINE(obj);
+
+    /* EL3 is enabled by default on vexpress */
+    vms->secure = true;
+    object_property_add_bool(obj, "secure", vexpress_get_secure,
+                             vexpress_set_secure, NULL);
+    object_property_set_description(obj, "secure",
+                                    "Set on/off to enable/disable the ARM "
+                                    "Security Extensions (TrustZone)",
+                                    NULL);
+}
+
+static void vexpress_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile Express";
+    mc->init = vexpress_common_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+}
+
+static void vexpress_a9_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+    VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile Express for Cortex-A9";
+
+    vmc->daughterboard = &a9_daughterboard;
+}
+
+static void vexpress_a15_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+    VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
+
+    mc->desc = "ARM Versatile Express for Cortex-A15";
+
+    vmc->daughterboard = &a15_daughterboard;
+}
+
+static const TypeInfo vexpress_info = {
+    .name = TYPE_VEXPRESS_MACHINE,
+    .parent = TYPE_MACHINE,
+    .abstract = true,
+    .instance_size = sizeof(VexpressMachineState),
+    .instance_init = vexpress_instance_init,
+    .class_size = sizeof(VexpressMachineClass),
+    .class_init = vexpress_class_init,
 };
 
-static QEMUMachine vexpress_a15_machine = {
-    .name = "vexpress-a15",
-    .desc = "ARM Versatile Express for Cortex-A15",
-    .init = vexpress_a15_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
+static const TypeInfo vexpress_a9_info = {
+    .name = TYPE_VEXPRESS_A9_MACHINE,
+    .parent = TYPE_VEXPRESS_MACHINE,
+    .class_init = vexpress_a9_class_init,
+};
+
+static const TypeInfo vexpress_a15_info = {
+    .name = TYPE_VEXPRESS_A15_MACHINE,
+    .parent = TYPE_VEXPRESS_MACHINE,
+    .class_init = vexpress_a15_class_init,
 };
 
 static void vexpress_machine_init(void)
 {
-    qemu_register_machine(&vexpress_a9_machine);
-    qemu_register_machine(&vexpress_a15_machine);
+    type_register_static(&vexpress_info);
+    type_register_static(&vexpress_a9_info);
+    type_register_static(&vexpress_a15_info);
 }
 
-machine_init(vexpress_machine_init);
+type_init(vexpress_machine_init);
diff --git a/hw/arm/virt-acpi-build.c b/hw/arm/virt-acpi-build.c
new file mode 100644
index 0000000..28fc59c
--- /dev/null
+++ b/hw/arm/virt-acpi-build.c
@@ -0,0 +1,813 @@
+/* Support for generating ACPI tables and passing them to Guests
+ *
+ * ARM virt ACPI generation
+ *
+ * Copyright (C) 2008-2010  Kevin O'Connor <kevin@koconnor.net>
+ * Copyright (C) 2006 Fabrice Bellard
+ * Copyright (C) 2013 Red Hat Inc
+ *
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO.,LTD.
+ *
+ * Author: Shannon Zhao <zhaoshenglong@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "hw/arm/virt-acpi-build.h"
+#include "qemu/bitmap.h"
+#include "trace.h"
+#include "qom/cpu.h"
+#include "target-arm/cpu.h"
+#include "hw/acpi/acpi-defs.h"
+#include "hw/acpi/acpi.h"
+#include "hw/nvram/fw_cfg.h"
+#include "hw/acpi/bios-linker-loader.h"
+#include "hw/loader.h"
+#include "hw/hw.h"
+#include "hw/acpi/aml-build.h"
+#include "hw/pci/pcie_host.h"
+#include "hw/pci/pci.h"
+#include "sysemu/numa.h"
+
+#define ARM_SPI_BASE 32
+#define ACPI_POWER_BUTTON_DEVICE "PWRB"
+
+static void acpi_dsdt_add_cpus(Aml *scope, int smp_cpus)
+{
+    uint16_t i;
+
+    for (i = 0; i < smp_cpus; i++) {
+        Aml *dev = aml_device("C%03x", i);
+        aml_append(dev, aml_name_decl("_HID", aml_string("ACPI0007")));
+        aml_append(dev, aml_name_decl("_UID", aml_int(i)));
+        aml_append(scope, dev);
+    }
+}
+
+static void acpi_dsdt_add_uart(Aml *scope, const MemMapEntry *uart_memmap,
+                                           uint32_t uart_irq)
+{
+    Aml *dev = aml_device("COM0");
+    aml_append(dev, aml_name_decl("_HID", aml_string("ARMH0011")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+
+    Aml *crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(uart_memmap->base,
+                                       uart_memmap->size, AML_READ_WRITE));
+    aml_append(crs,
+               aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                             AML_EXCLUSIVE, &uart_irq, 1));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    /* The _ADR entry is used to link this device to the UART described
+     * in the SPCR table, i.e. SPCR.base_address.address == _ADR.
+     */
+    aml_append(dev, aml_name_decl("_ADR", aml_int(uart_memmap->base)));
+
+    aml_append(scope, dev);
+}
+
+static void acpi_dsdt_add_fw_cfg(Aml *scope, const MemMapEntry *fw_cfg_memmap)
+{
+    Aml *dev = aml_device("FWCF");
+    aml_append(dev, aml_name_decl("_HID", aml_string("QEMU0002")));
+    /* device present, functioning, decoding, not shown in UI */
+    aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+
+    Aml *crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(fw_cfg_memmap->base,
+                                       fw_cfg_memmap->size, AML_READ_WRITE));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(scope, dev);
+}
+
+static void acpi_dsdt_add_flash(Aml *scope, const MemMapEntry *flash_memmap)
+{
+    Aml *dev, *crs;
+    hwaddr base = flash_memmap->base;
+    hwaddr size = flash_memmap->size / 2;
+
+    dev = aml_device("FLS0");
+    aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0015")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(base, size, AML_READ_WRITE));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(scope, dev);
+
+    dev = aml_device("FLS1");
+    aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0015")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(1)));
+    crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(base + size, size, AML_READ_WRITE));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(scope, dev);
+}
+
+static void acpi_dsdt_add_virtio(Aml *scope,
+                                 const MemMapEntry *virtio_mmio_memmap,
+                                 uint32_t mmio_irq, int num)
+{
+    hwaddr base = virtio_mmio_memmap->base;
+    hwaddr size = virtio_mmio_memmap->size;
+    int i;
+
+    for (i = 0; i < num; i++) {
+        uint32_t irq = mmio_irq + i;
+        Aml *dev = aml_device("VR%02u", i);
+        aml_append(dev, aml_name_decl("_HID", aml_string("LNRO0005")));
+        aml_append(dev, aml_name_decl("_UID", aml_int(i)));
+
+        Aml *crs = aml_resource_template();
+        aml_append(crs, aml_memory32_fixed(base, size, AML_READ_WRITE));
+        aml_append(crs,
+                   aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                 AML_EXCLUSIVE, &irq, 1));
+        aml_append(dev, aml_name_decl("_CRS", crs));
+        aml_append(scope, dev);
+        base += size;
+    }
+}
+
+static void acpi_dsdt_add_pci(Aml *scope, const MemMapEntry *memmap,
+                              uint32_t irq, bool use_highmem)
+{
+    Aml *method, *crs, *ifctx, *UUID, *ifctx1, *elsectx, *buf;
+    int i, bus_no;
+    hwaddr base_mmio = memmap[VIRT_PCIE_MMIO].base;
+    hwaddr size_mmio = memmap[VIRT_PCIE_MMIO].size;
+    hwaddr base_pio = memmap[VIRT_PCIE_PIO].base;
+    hwaddr size_pio = memmap[VIRT_PCIE_PIO].size;
+    hwaddr base_ecam = memmap[VIRT_PCIE_ECAM].base;
+    hwaddr size_ecam = memmap[VIRT_PCIE_ECAM].size;
+    int nr_pcie_buses = size_ecam / PCIE_MMCFG_SIZE_MIN;
+
+    Aml *dev = aml_device("%s", "PCI0");
+    aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A08")));
+    aml_append(dev, aml_name_decl("_CID", aml_string("PNP0A03")));
+    aml_append(dev, aml_name_decl("_SEG", aml_int(0)));
+    aml_append(dev, aml_name_decl("_BBN", aml_int(0)));
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+    aml_append(dev, aml_name_decl("_UID", aml_string("PCI0")));
+    aml_append(dev, aml_name_decl("_STR", aml_unicode("PCIe 0 Device")));
+    aml_append(dev, aml_name_decl("_CCA", aml_int(1)));
+
+    /* Declare the PCI Routing Table. */
+    Aml *rt_pkg = aml_package(nr_pcie_buses * PCI_NUM_PINS);
+    for (bus_no = 0; bus_no < nr_pcie_buses; bus_no++) {
+        for (i = 0; i < PCI_NUM_PINS; i++) {
+            int gsi = (i + bus_no) % PCI_NUM_PINS;
+            Aml *pkg = aml_package(4);
+            aml_append(pkg, aml_int((bus_no << 16) | 0xFFFF));
+            aml_append(pkg, aml_int(i));
+            aml_append(pkg, aml_name("GSI%d", gsi));
+            aml_append(pkg, aml_int(0));
+            aml_append(rt_pkg, pkg);
+        }
+    }
+    aml_append(dev, aml_name_decl("_PRT", rt_pkg));
+
+    /* Create GSI link device */
+    for (i = 0; i < PCI_NUM_PINS; i++) {
+        uint32_t irqs =  irq + i;
+        Aml *dev_gsi = aml_device("GSI%d", i);
+        aml_append(dev_gsi, aml_name_decl("_HID", aml_string("PNP0C0F")));
+        aml_append(dev_gsi, aml_name_decl("_UID", aml_int(0)));
+        crs = aml_resource_template();
+        aml_append(crs,
+                   aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                 AML_EXCLUSIVE, &irqs, 1));
+        aml_append(dev_gsi, aml_name_decl("_PRS", crs));
+        crs = aml_resource_template();
+        aml_append(crs,
+                   aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                 AML_EXCLUSIVE, &irqs, 1));
+        aml_append(dev_gsi, aml_name_decl("_CRS", crs));
+        method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+        aml_append(dev_gsi, method);
+        aml_append(dev, dev_gsi);
+    }
+
+    method = aml_method("_CBA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_int(base_ecam)));
+    aml_append(dev, method);
+
+    method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+    Aml *rbuf = aml_resource_template();
+    aml_append(rbuf,
+        aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
+                            0x0000, 0x0000, nr_pcie_buses - 1, 0x0000,
+                            nr_pcie_buses));
+    aml_append(rbuf,
+        aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                         AML_NON_CACHEABLE, AML_READ_WRITE, 0x0000, base_mmio,
+                         base_mmio + size_mmio - 1, 0x0000, size_mmio));
+    aml_append(rbuf,
+        aml_dword_io(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
+                     AML_ENTIRE_RANGE, 0x0000, 0x0000, size_pio - 1, base_pio,
+                     size_pio));
+
+    if (use_highmem) {
+        hwaddr base_mmio_high = memmap[VIRT_PCIE_MMIO_HIGH].base;
+        hwaddr size_mmio_high = memmap[VIRT_PCIE_MMIO_HIGH].size;
+
+        aml_append(rbuf,
+            aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                             AML_NON_CACHEABLE, AML_READ_WRITE, 0x0000,
+                             base_mmio_high,
+                             base_mmio_high + size_mmio_high - 1, 0x0000,
+                             size_mmio_high));
+    }
+
+    aml_append(method, aml_name_decl("RBUF", rbuf));
+    aml_append(method, aml_return(rbuf));
+    aml_append(dev, method);
+
+    /* Declare an _OSC (OS Control Handoff) method */
+    aml_append(dev, aml_name_decl("SUPP", aml_int(0)));
+    aml_append(dev, aml_name_decl("CTRL", aml_int(0)));
+    method = aml_method("_OSC", 4, AML_NOTSERIALIZED);
+    aml_append(method,
+        aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1"));
+
+    /* PCI Firmware Specification 3.0
+     * 4.5.1. _OSC Interface for PCI Host Bridge Devices
+     * The _OSC interface for a PCI/PCI-X/PCI Express hierarchy is
+     * identified by the Universal Unique IDentifier (UUID)
+     * 33DB4D5B-1FF7-401C-9657-7441C03DD766
+     */
+    UUID = aml_touuid("33DB4D5B-1FF7-401C-9657-7441C03DD766");
+    ifctx = aml_if(aml_equal(aml_arg(0), UUID));
+    aml_append(ifctx,
+        aml_create_dword_field(aml_arg(3), aml_int(4), "CDW2"));
+    aml_append(ifctx,
+        aml_create_dword_field(aml_arg(3), aml_int(8), "CDW3"));
+    aml_append(ifctx, aml_store(aml_name("CDW2"), aml_name("SUPP")));
+    aml_append(ifctx, aml_store(aml_name("CDW3"), aml_name("CTRL")));
+    aml_append(ifctx, aml_store(aml_and(aml_name("CTRL"), aml_int(0x1D), NULL),
+                                aml_name("CTRL")));
+
+    ifctx1 = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(0x1))));
+    aml_append(ifctx1, aml_store(aml_or(aml_name("CDW1"), aml_int(0x08), NULL),
+                                 aml_name("CDW1")));
+    aml_append(ifctx, ifctx1);
+
+    ifctx1 = aml_if(aml_lnot(aml_equal(aml_name("CDW3"), aml_name("CTRL"))));
+    aml_append(ifctx1, aml_store(aml_or(aml_name("CDW1"), aml_int(0x10), NULL),
+                                 aml_name("CDW1")));
+    aml_append(ifctx, ifctx1);
+
+    aml_append(ifctx, aml_store(aml_name("CTRL"), aml_name("CDW3")));
+    aml_append(ifctx, aml_return(aml_arg(3)));
+    aml_append(method, ifctx);
+
+    elsectx = aml_else();
+    aml_append(elsectx, aml_store(aml_or(aml_name("CDW1"), aml_int(4), NULL),
+                                  aml_name("CDW1")));
+    aml_append(elsectx, aml_return(aml_arg(3)));
+    aml_append(method, elsectx);
+    aml_append(dev, method);
+
+    method = aml_method("_DSM", 4, AML_NOTSERIALIZED);
+
+    /* PCI Firmware Specification 3.0
+     * 4.6.1. _DSM for PCI Express Slot Information
+     * The UUID in _DSM in this context is
+     * {E5C937D0-3553-4D7A-9117-EA4D19C3434D}
+     */
+    UUID = aml_touuid("E5C937D0-3553-4D7A-9117-EA4D19C3434D");
+    ifctx = aml_if(aml_equal(aml_arg(0), UUID));
+    ifctx1 = aml_if(aml_equal(aml_arg(2), aml_int(0)));
+    uint8_t byte_list[1] = {1};
+    buf = aml_buffer(1, byte_list);
+    aml_append(ifctx1, aml_return(buf));
+    aml_append(ifctx, ifctx1);
+    aml_append(method, ifctx);
+
+    byte_list[0] = 0;
+    buf = aml_buffer(1, byte_list);
+    aml_append(method, aml_return(buf));
+    aml_append(dev, method);
+
+    Aml *dev_rp0 = aml_device("%s", "RP0");
+    aml_append(dev_rp0, aml_name_decl("_ADR", aml_int(0)));
+    aml_append(dev, dev_rp0);
+    aml_append(scope, dev);
+}
+
+static void acpi_dsdt_add_gpio(Aml *scope, const MemMapEntry *gpio_memmap,
+                                           uint32_t gpio_irq)
+{
+    Aml *dev = aml_device("GPO0");
+    aml_append(dev, aml_name_decl("_HID", aml_string("ARMH0061")));
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+
+    Aml *crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(gpio_memmap->base, gpio_memmap->size,
+                                       AML_READ_WRITE));
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                  AML_EXCLUSIVE, &gpio_irq, 1));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    Aml *aei = aml_resource_template();
+    /* Pin 3 for power button */
+    const uint32_t pin_list[1] = {3};
+    aml_append(aei, aml_gpio_int(AML_CONSUMER, AML_EDGE, AML_ACTIVE_HIGH,
+                                 AML_EXCLUSIVE, AML_PULL_UP, 0, pin_list, 1,
+                                 "GPO0", NULL, 0));
+    aml_append(dev, aml_name_decl("_AEI", aei));
+
+    /* _E03 is handle for power button */
+    Aml *method = aml_method("_E03", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_notify(aml_name(ACPI_POWER_BUTTON_DEVICE),
+                                  aml_int(0x80)));
+    aml_append(dev, method);
+    aml_append(scope, dev);
+}
+
+static void acpi_dsdt_add_power_button(Aml *scope)
+{
+    Aml *dev = aml_device(ACPI_POWER_BUTTON_DEVICE);
+    aml_append(dev, aml_name_decl("_HID", aml_string("PNP0C0C")));
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+    aml_append(dev, aml_name_decl("_UID", aml_int(0)));
+    aml_append(scope, dev);
+}
+
+/* RSDP */
+static GArray *
+build_rsdp(GArray *rsdp_table, BIOSLinker *linker, unsigned rsdt_tbl_offset)
+{
+    AcpiRsdpDescriptor *rsdp = acpi_data_push(rsdp_table, sizeof *rsdp);
+    unsigned rsdt_pa_size = sizeof(rsdp->rsdt_physical_address);
+    unsigned rsdt_pa_offset =
+        (char *)&rsdp->rsdt_physical_address - rsdp_table->data;
+
+    bios_linker_loader_alloc(linker, ACPI_BUILD_RSDP_FILE, rsdp_table, 16,
+                             true /* fseg memory */);
+
+    memcpy(&rsdp->signature, "RSD PTR ", sizeof(rsdp->signature));
+    memcpy(rsdp->oem_id, ACPI_BUILD_APPNAME6, sizeof(rsdp->oem_id));
+    rsdp->length = cpu_to_le32(sizeof(*rsdp));
+    rsdp->revision = 0x02;
+
+    /* Address to be filled by Guest linker */
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_RSDP_FILE, rsdt_pa_offset, rsdt_pa_size,
+        ACPI_BUILD_TABLE_FILE, rsdt_tbl_offset);
+
+    /* Checksum to be filled by Guest linker */
+    bios_linker_loader_add_checksum(linker, ACPI_BUILD_RSDP_FILE,
+        (char *)rsdp - rsdp_table->data, sizeof *rsdp,
+        (char *)&rsdp->checksum - rsdp_table->data);
+
+    return rsdp_table;
+}
+
+static void
+build_spcr(GArray *table_data, BIOSLinker *linker, VirtGuestInfo *guest_info)
+{
+    AcpiSerialPortConsoleRedirection *spcr;
+    const MemMapEntry *uart_memmap = &guest_info->memmap[VIRT_UART];
+    int irq = guest_info->irqmap[VIRT_UART] + ARM_SPI_BASE;
+
+    spcr = acpi_data_push(table_data, sizeof(*spcr));
+
+    spcr->interface_type = 0x3;    /* ARM PL011 UART */
+
+    spcr->base_address.space_id = AML_SYSTEM_MEMORY;
+    spcr->base_address.bit_width = 8;
+    spcr->base_address.bit_offset = 0;
+    spcr->base_address.access_width = 1;
+    spcr->base_address.address = cpu_to_le64(uart_memmap->base);
+
+    spcr->interrupt_types = (1 << 3); /* Bit[3] ARMH GIC interrupt */
+    spcr->gsi = cpu_to_le32(irq);  /* Global System Interrupt */
+
+    spcr->baud = 3;                /* Baud Rate: 3 = 9600 */
+    spcr->parity = 0;              /* No Parity */
+    spcr->stopbits = 1;            /* 1 Stop bit */
+    spcr->flowctrl = (1 << 1);     /* Bit[1] = RTS/CTS hardware flow control */
+    spcr->term_type = 0;           /* Terminal Type: 0 = VT100 */
+
+    spcr->pci_device_id = 0xffff;  /* PCI Device ID: not a PCI device */
+    spcr->pci_vendor_id = 0xffff;  /* PCI Vendor ID: not a PCI device */
+
+    build_header(linker, table_data, (void *)spcr, "SPCR", sizeof(*spcr), 2,
+                 NULL, NULL);
+}
+
+static void
+build_srat(GArray *table_data, BIOSLinker *linker, VirtGuestInfo *guest_info)
+{
+    AcpiSystemResourceAffinityTable *srat;
+    AcpiSratProcessorGiccAffinity *core;
+    AcpiSratMemoryAffinity *numamem;
+    int i, j, srat_start;
+    uint64_t mem_base;
+    uint32_t *cpu_node = g_malloc0(guest_info->smp_cpus * sizeof(uint32_t));
+
+    for (i = 0; i < guest_info->smp_cpus; i++) {
+        for (j = 0; j < nb_numa_nodes; j++) {
+            if (test_bit(i, numa_info[j].node_cpu)) {
+                cpu_node[i] = j;
+                break;
+            }
+        }
+    }
+
+    srat_start = table_data->len;
+    srat = acpi_data_push(table_data, sizeof(*srat));
+    srat->reserved1 = cpu_to_le32(1);
+
+    for (i = 0; i < guest_info->smp_cpus; ++i) {
+        core = acpi_data_push(table_data, sizeof(*core));
+        core->type = ACPI_SRAT_PROCESSOR_GICC;
+        core->length = sizeof(*core);
+        core->proximity = cpu_to_le32(cpu_node[i]);
+        core->acpi_processor_uid = cpu_to_le32(i);
+        core->flags = cpu_to_le32(1);
+    }
+    g_free(cpu_node);
+
+    mem_base = guest_info->memmap[VIRT_MEM].base;
+    for (i = 0; i < nb_numa_nodes; ++i) {
+        numamem = acpi_data_push(table_data, sizeof(*numamem));
+        build_srat_memory(numamem, mem_base, numa_info[i].node_mem, i,
+                          MEM_AFFINITY_ENABLED);
+        mem_base += numa_info[i].node_mem;
+    }
+
+    build_header(linker, table_data, (void *)srat, "SRAT",
+                 table_data->len - srat_start, 3, NULL, NULL);
+}
+
+static void
+build_mcfg(GArray *table_data, BIOSLinker *linker, VirtGuestInfo *guest_info)
+{
+    AcpiTableMcfg *mcfg;
+    const MemMapEntry *memmap = guest_info->memmap;
+    int len = sizeof(*mcfg) + sizeof(mcfg->allocation[0]);
+
+    mcfg = acpi_data_push(table_data, len);
+    mcfg->allocation[0].address = cpu_to_le64(memmap[VIRT_PCIE_ECAM].base);
+
+    /* Only a single allocation so no need to play with segments */
+    mcfg->allocation[0].pci_segment = cpu_to_le16(0);
+    mcfg->allocation[0].start_bus_number = 0;
+    mcfg->allocation[0].end_bus_number = (memmap[VIRT_PCIE_ECAM].size
+                                          / PCIE_MMCFG_SIZE_MIN) - 1;
+
+    build_header(linker, table_data, (void *)mcfg, "MCFG", len, 1, NULL, NULL);
+}
+
+/* GTDT */
+static void
+build_gtdt(GArray *table_data, BIOSLinker *linker)
+{
+    int gtdt_start = table_data->len;
+    AcpiGenericTimerTable *gtdt;
+
+    gtdt = acpi_data_push(table_data, sizeof *gtdt);
+    /* The interrupt values are the same with the device tree when adding 16 */
+    gtdt->secure_el1_interrupt = ARCH_TIMER_S_EL1_IRQ + 16;
+    gtdt->secure_el1_flags = ACPI_EDGE_SENSITIVE;
+
+    gtdt->non_secure_el1_interrupt = ARCH_TIMER_NS_EL1_IRQ + 16;
+    gtdt->non_secure_el1_flags = ACPI_EDGE_SENSITIVE | ACPI_GTDT_ALWAYS_ON;
+
+    gtdt->virtual_timer_interrupt = ARCH_TIMER_VIRT_IRQ + 16;
+    gtdt->virtual_timer_flags = ACPI_EDGE_SENSITIVE;
+
+    gtdt->non_secure_el2_interrupt = ARCH_TIMER_NS_EL2_IRQ + 16;
+    gtdt->non_secure_el2_flags = ACPI_EDGE_SENSITIVE;
+
+    build_header(linker, table_data,
+                 (void *)(table_data->data + gtdt_start), "GTDT",
+                 table_data->len - gtdt_start, 2, NULL, NULL);
+}
+
+/* MADT */
+static void
+build_madt(GArray *table_data, BIOSLinker *linker, VirtGuestInfo *guest_info)
+{
+    int madt_start = table_data->len;
+    const MemMapEntry *memmap = guest_info->memmap;
+    const int *irqmap = guest_info->irqmap;
+    AcpiMultipleApicTable *madt;
+    AcpiMadtGenericDistributor *gicd;
+    AcpiMadtGenericMsiFrame *gic_msi;
+    int i;
+
+    madt = acpi_data_push(table_data, sizeof *madt);
+
+    gicd = acpi_data_push(table_data, sizeof *gicd);
+    gicd->type = ACPI_APIC_GENERIC_DISTRIBUTOR;
+    gicd->length = sizeof(*gicd);
+    gicd->base_address = memmap[VIRT_GIC_DIST].base;
+    gicd->version = guest_info->gic_version;
+
+    for (i = 0; i < guest_info->smp_cpus; i++) {
+        AcpiMadtGenericInterrupt *gicc = acpi_data_push(table_data,
+                                                     sizeof *gicc);
+        ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(i));
+
+        gicc->type = ACPI_APIC_GENERIC_INTERRUPT;
+        gicc->length = sizeof(*gicc);
+        if (guest_info->gic_version == 2) {
+            gicc->base_address = memmap[VIRT_GIC_CPU].base;
+        }
+        gicc->cpu_interface_number = i;
+        gicc->arm_mpidr = armcpu->mp_affinity;
+        gicc->uid = i;
+        gicc->flags = cpu_to_le32(ACPI_GICC_ENABLED);
+
+        if (armcpu->has_pmu) {
+            gicc->performance_interrupt = cpu_to_le32(PPI(VIRTUAL_PMU_IRQ));
+        }
+    }
+
+    if (guest_info->gic_version == 3) {
+        AcpiMadtGenericRedistributor *gicr = acpi_data_push(table_data,
+                                                         sizeof *gicr);
+
+        gicr->type = ACPI_APIC_GENERIC_REDISTRIBUTOR;
+        gicr->length = sizeof(*gicr);
+        gicr->base_address = cpu_to_le64(memmap[VIRT_GIC_REDIST].base);
+        gicr->range_length = cpu_to_le32(memmap[VIRT_GIC_REDIST].size);
+    } else {
+        gic_msi = acpi_data_push(table_data, sizeof *gic_msi);
+        gic_msi->type = ACPI_APIC_GENERIC_MSI_FRAME;
+        gic_msi->length = sizeof(*gic_msi);
+        gic_msi->gic_msi_frame_id = 0;
+        gic_msi->base_address = cpu_to_le64(memmap[VIRT_GIC_V2M].base);
+        gic_msi->flags = cpu_to_le32(1);
+        gic_msi->spi_count = cpu_to_le16(NUM_GICV2M_SPIS);
+        gic_msi->spi_base = cpu_to_le16(irqmap[VIRT_GIC_V2M] + ARM_SPI_BASE);
+    }
+
+    build_header(linker, table_data,
+                 (void *)(table_data->data + madt_start), "APIC",
+                 table_data->len - madt_start, 3, NULL, NULL);
+}
+
+/* FADT */
+static void
+build_fadt(GArray *table_data, BIOSLinker *linker, unsigned dsdt_tbl_offset)
+{
+    AcpiFadtDescriptorRev5_1 *fadt = acpi_data_push(table_data, sizeof(*fadt));
+    unsigned dsdt_entry_offset = (char *)&fadt->dsdt - table_data->data;
+
+    /* Hardware Reduced = 1 and use PSCI 0.2+ and with HVC */
+    fadt->flags = cpu_to_le32(1 << ACPI_FADT_F_HW_REDUCED_ACPI);
+    fadt->arm_boot_flags = cpu_to_le16((1 << ACPI_FADT_ARM_USE_PSCI_G_0_2) |
+                                       (1 << ACPI_FADT_ARM_PSCI_USE_HVC));
+
+    /* ACPI v5.1 (fadt->revision.fadt->minor_revision) */
+    fadt->minor_revision = 0x1;
+
+    /* DSDT address to be filled by Guest linker */
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_TABLE_FILE, dsdt_entry_offset, sizeof(fadt->dsdt),
+        ACPI_BUILD_TABLE_FILE, dsdt_tbl_offset);
+
+    build_header(linker, table_data,
+                 (void *)fadt, "FACP", sizeof(*fadt), 5, NULL, NULL);
+}
+
+/* DSDT */
+static void
+build_dsdt(GArray *table_data, BIOSLinker *linker, VirtGuestInfo *guest_info)
+{
+    Aml *scope, *dsdt;
+    const MemMapEntry *memmap = guest_info->memmap;
+    const int *irqmap = guest_info->irqmap;
+
+    dsdt = init_aml_allocator();
+    /* Reserve space for header */
+    acpi_data_push(dsdt->buf, sizeof(AcpiTableHeader));
+
+    /* When booting the VM with UEFI, UEFI takes ownership of the RTC hardware.
+     * While UEFI can use libfdt to disable the RTC device node in the DTB that
+     * it passes to the OS, it cannot modify AML. Therefore, we won't generate
+     * the RTC ACPI device at all when using UEFI.
+     */
+    scope = aml_scope("\\_SB");
+    acpi_dsdt_add_cpus(scope, guest_info->smp_cpus);
+    acpi_dsdt_add_uart(scope, &memmap[VIRT_UART],
+                       (irqmap[VIRT_UART] + ARM_SPI_BASE));
+    acpi_dsdt_add_flash(scope, &memmap[VIRT_FLASH]);
+    acpi_dsdt_add_fw_cfg(scope, &memmap[VIRT_FW_CFG]);
+    acpi_dsdt_add_virtio(scope, &memmap[VIRT_MMIO],
+                    (irqmap[VIRT_MMIO] + ARM_SPI_BASE), NUM_VIRTIO_TRANSPORTS);
+    acpi_dsdt_add_pci(scope, memmap, (irqmap[VIRT_PCIE] + ARM_SPI_BASE),
+                      guest_info->use_highmem);
+    acpi_dsdt_add_gpio(scope, &memmap[VIRT_GPIO],
+                       (irqmap[VIRT_GPIO] + ARM_SPI_BASE));
+    acpi_dsdt_add_power_button(scope);
+
+    aml_append(dsdt, scope);
+
+    /* copy AML table into ACPI tables blob and patch header there */
+    g_array_append_vals(table_data, dsdt->buf->data, dsdt->buf->len);
+    build_header(linker, table_data,
+        (void *)(table_data->data + table_data->len - dsdt->buf->len),
+        "DSDT", dsdt->buf->len, 2, NULL, NULL);
+    free_aml_allocator();
+}
+
+typedef
+struct AcpiBuildState {
+    /* Copy of table in RAM (for patching). */
+    MemoryRegion *table_mr;
+    MemoryRegion *rsdp_mr;
+    MemoryRegion *linker_mr;
+    /* Is table patched? */
+    bool patched;
+    VirtGuestInfo *guest_info;
+} AcpiBuildState;
+
+static
+void virt_acpi_build(VirtGuestInfo *guest_info, AcpiBuildTables *tables)
+{
+    GArray *table_offsets;
+    unsigned dsdt, rsdt;
+    GArray *tables_blob = tables->table_data;
+
+    table_offsets = g_array_new(false, true /* clear */,
+                                        sizeof(uint32_t));
+
+    bios_linker_loader_alloc(tables->linker,
+                             ACPI_BUILD_TABLE_FILE, tables_blob,
+                             64, false /* high memory */);
+
+    /*
+     * The ACPI v5.1 tables for Hardware-reduced ACPI platform are:
+     * RSDP
+     * RSDT
+     * FADT
+     * GTDT
+     * MADT
+     * MCFG
+     * DSDT
+     */
+
+    /* DSDT is pointed to by FADT */
+    dsdt = tables_blob->len;
+    build_dsdt(tables_blob, tables->linker, guest_info);
+
+    /* FADT MADT GTDT MCFG SPCR pointed to by RSDT */
+    acpi_add_table(table_offsets, tables_blob);
+    build_fadt(tables_blob, tables->linker, dsdt);
+
+    acpi_add_table(table_offsets, tables_blob);
+    build_madt(tables_blob, tables->linker, guest_info);
+
+    acpi_add_table(table_offsets, tables_blob);
+    build_gtdt(tables_blob, tables->linker);
+
+    acpi_add_table(table_offsets, tables_blob);
+    build_mcfg(tables_blob, tables->linker, guest_info);
+
+    acpi_add_table(table_offsets, tables_blob);
+    build_spcr(tables_blob, tables->linker, guest_info);
+
+    if (nb_numa_nodes > 0) {
+        acpi_add_table(table_offsets, tables_blob);
+        build_srat(tables_blob, tables->linker, guest_info);
+    }
+
+    /* RSDT is pointed to by RSDP */
+    rsdt = tables_blob->len;
+    build_rsdt(tables_blob, tables->linker, table_offsets, NULL, NULL);
+
+    /* RSDP is in FSEG memory, so allocate it separately */
+    build_rsdp(tables->rsdp, tables->linker, rsdt);
+
+    /* Cleanup memory that's no longer used. */
+    g_array_free(table_offsets, true);
+}
+
+static void acpi_ram_update(MemoryRegion *mr, GArray *data)
+{
+    uint32_t size = acpi_data_len(data);
+
+    /* Make sure RAM size is correct - in case it got changed
+     * e.g. by migration */
+    memory_region_ram_resize(mr, size, &error_abort);
+
+    memcpy(memory_region_get_ram_ptr(mr), data->data, size);
+    memory_region_set_dirty(mr, 0, size);
+}
+
+static void virt_acpi_build_update(void *build_opaque)
+{
+    AcpiBuildState *build_state = build_opaque;
+    AcpiBuildTables tables;
+
+    /* No state to update or already patched? Nothing to do. */
+    if (!build_state || build_state->patched) {
+        return;
+    }
+    build_state->patched = true;
+
+    acpi_build_tables_init(&tables);
+
+    virt_acpi_build(build_state->guest_info, &tables);
+
+    acpi_ram_update(build_state->table_mr, tables.table_data);
+    acpi_ram_update(build_state->rsdp_mr, tables.rsdp);
+    acpi_ram_update(build_state->linker_mr, tables.linker->cmd_blob);
+
+
+    acpi_build_tables_cleanup(&tables, true);
+}
+
+static void virt_acpi_build_reset(void *build_opaque)
+{
+    AcpiBuildState *build_state = build_opaque;
+    build_state->patched = false;
+}
+
+static MemoryRegion *acpi_add_rom_blob(AcpiBuildState *build_state,
+                                       GArray *blob, const char *name,
+                                       uint64_t max_size)
+{
+    return rom_add_blob(name, blob->data, acpi_data_len(blob), max_size, -1,
+                        name, virt_acpi_build_update, build_state);
+}
+
+static const VMStateDescription vmstate_virt_acpi_build = {
+    .name = "virt_acpi_build",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(patched, AcpiBuildState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+void virt_acpi_setup(VirtGuestInfo *guest_info)
+{
+    AcpiBuildTables tables;
+    AcpiBuildState *build_state;
+
+    if (!guest_info->fw_cfg) {
+        trace_virt_acpi_setup();
+        return;
+    }
+
+    if (!acpi_enabled) {
+        trace_virt_acpi_setup();
+        return;
+    }
+
+    build_state = g_malloc0(sizeof *build_state);
+    build_state->guest_info = guest_info;
+
+    acpi_build_tables_init(&tables);
+    virt_acpi_build(build_state->guest_info, &tables);
+
+    /* Now expose it all to Guest */
+    build_state->table_mr = acpi_add_rom_blob(build_state, tables.table_data,
+                                               ACPI_BUILD_TABLE_FILE,
+                                               ACPI_BUILD_TABLE_MAX_SIZE);
+    assert(build_state->table_mr != NULL);
+
+    build_state->linker_mr =
+        acpi_add_rom_blob(build_state, tables.linker->cmd_blob,
+                          "etc/table-loader", 0);
+
+    fw_cfg_add_file(guest_info->fw_cfg, ACPI_BUILD_TPMLOG_FILE,
+                    tables.tcpalog->data, acpi_data_len(tables.tcpalog));
+
+    build_state->rsdp_mr = acpi_add_rom_blob(build_state, tables.rsdp,
+                                              ACPI_BUILD_RSDP_FILE, 0);
+
+    qemu_register_reset(virt_acpi_build_reset, build_state);
+    virt_acpi_build_reset(build_state);
+    vmstate_register(NULL, 0, &vmstate_virt_acpi_build, build_state);
+
+    /* Cleanup tables but don't free the memory: we track it
+     * in build_state.
+     */
+    acpi_build_tables_cleanup(&tables, false);
+}
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index 314e55b..a193b5a 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -28,52 +28,43 @@
  * This is essentially the same approach kvmtool uses.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "hw/arm/primecell.h"
+#include "hw/arm/virt.h"
 #include "hw/devices.h"
 #include "net/net.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/device_tree.h"
+#include "sysemu/numa.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "hw/boards.h"
+#include "hw/compat.h"
 #include "hw/loader.h"
 #include "exec/address-spaces.h"
 #include "qemu/bitops.h"
 #include "qemu/error-report.h"
-
-#define NUM_VIRTIO_TRANSPORTS 32
+#include "hw/pci-host/gpex.h"
+#include "hw/arm/virt-acpi-build.h"
+#include "hw/arm/sysbus-fdt.h"
+#include "hw/platform-bus.h"
+#include "hw/arm/fdt.h"
+#include "hw/intc/arm_gic.h"
+#include "hw/intc/arm_gicv3_common.h"
+#include "kvm_arm.h"
+#include "hw/smbios/smbios.h"
+#include "qapi/visitor.h"
+#include "standard-headers/linux/input.h"
 
 /* Number of external interrupt lines to configure the GIC with */
-#define NUM_IRQS 128
+#define NUM_IRQS 256
 
-#define GIC_FDT_IRQ_TYPE_SPI 0
-#define GIC_FDT_IRQ_TYPE_PPI 1
+#define PLATFORM_BUS_NUM_IRQS 64
 
-#define GIC_FDT_IRQ_FLAGS_EDGE_LO_HI 1
-#define GIC_FDT_IRQ_FLAGS_EDGE_HI_LO 2
-#define GIC_FDT_IRQ_FLAGS_LEVEL_HI 4
-#define GIC_FDT_IRQ_FLAGS_LEVEL_LO 8
-
-#define GIC_FDT_IRQ_PPI_CPU_START 8
-#define GIC_FDT_IRQ_PPI_CPU_WIDTH 8
-
-enum {
-    VIRT_FLASH,
-    VIRT_MEM,
-    VIRT_CPUPERIPHS,
-    VIRT_GIC_DIST,
-    VIRT_GIC_CPU,
-    VIRT_UART,
-    VIRT_MMIO,
-    VIRT_RTC,
-};
-
-typedef struct MemMapEntry {
-    hwaddr base;
-    hwaddr size;
-} MemMapEntry;
+static ARMPlatformBusSystemParams platform_bus_params;
 
 typedef struct VirtBoardInfo {
     struct arm_boot_info bootinfo;
@@ -84,8 +75,79 @@
     void *fdt;
     int fdt_size;
     uint32_t clock_phandle;
+    uint32_t gic_phandle;
+    uint32_t v2m_phandle;
+    bool using_psci;
 } VirtBoardInfo;
 
+typedef struct {
+    MachineClass parent;
+    VirtBoardInfo *daughterboard;
+    bool disallow_affinity_adjustment;
+} VirtMachineClass;
+
+typedef struct {
+    MachineState parent;
+    bool secure;
+    bool highmem;
+    int32_t gic_version;
+} VirtMachineState;
+
+#define TYPE_VIRT_MACHINE   MACHINE_TYPE_NAME("virt")
+#define VIRT_MACHINE(obj) \
+    OBJECT_CHECK(VirtMachineState, (obj), TYPE_VIRT_MACHINE)
+#define VIRT_MACHINE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(VirtMachineClass, obj, TYPE_VIRT_MACHINE)
+#define VIRT_MACHINE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(VirtMachineClass, klass, TYPE_VIRT_MACHINE)
+
+
+#define DEFINE_VIRT_MACHINE_LATEST(major, minor, latest) \
+    static void virt_##major##_##minor##_class_init(ObjectClass *oc, \
+                                                    void *data) \
+    { \
+        MachineClass *mc = MACHINE_CLASS(oc); \
+        virt_machine_##major##_##minor##_options(mc); \
+        mc->desc = "QEMU " # major "." # minor " ARM Virtual Machine"; \
+        if (latest) { \
+            mc->alias = "virt"; \
+        } \
+    } \
+    static const TypeInfo machvirt_##major##_##minor##_info = { \
+        .name = MACHINE_TYPE_NAME("virt-" # major "." # minor), \
+        .parent = TYPE_VIRT_MACHINE, \
+        .instance_init = virt_##major##_##minor##_instance_init, \
+        .class_init = virt_##major##_##minor##_class_init, \
+    }; \
+    static void machvirt_machine_##major##_##minor##_init(void) \
+    { \
+        type_register_static(&machvirt_##major##_##minor##_info); \
+    } \
+    type_init(machvirt_machine_##major##_##minor##_init);
+
+#define DEFINE_VIRT_MACHINE_AS_LATEST(major, minor) \
+    DEFINE_VIRT_MACHINE_LATEST(major, minor, true)
+#define DEFINE_VIRT_MACHINE(major, minor) \
+    DEFINE_VIRT_MACHINE_LATEST(major, minor, false)
+
+
+/* RAM limit in GB. Since VIRT_MEM starts at the 1GB mark, this means
+ * RAM can go up to the 256GB mark, leaving 256GB of the physical
+ * address space unallocated and free for future use between 256G and 512G.
+ * If we need to provide more RAM to VMs in the future then we need to:
+ *  * allocate a second bank of RAM starting at 2TB and working up
+ *  * fix the DT and ACPI table generation code in QEMU to correctly
+ *    report two split lumps of RAM to the guest
+ *  * fix KVM in the host kernel to allow guests with >40 bit address spaces
+ * (We don't want to fill all the way up to 512GB with RAM because
+ * we might want it for non-RAM purposes later. Conversely it seems
+ * reasonable to assume that anybody configuring a VM with a quarter
+ * of a terabyte of RAM will be doing it on a host with more than a
+ * terabyte of physical address space.)
+ */
+#define RAMLIMIT_GB 255
+#define RAMLIMIT_BYTES (RAMLIMIT_GB * 1024ULL * 1024 * 1024)
+
 /* Addresses and sizes of our components.
  * 0..128MB is space for a flash device so we can run bootrom code such as UEFI.
  * 128MB..256MB is used for miscellaneous device I/O.
@@ -100,23 +162,42 @@
  */
 static const MemMapEntry a15memmap[] = {
     /* Space up to 0x8000000 is reserved for a boot ROM */
-    [VIRT_FLASH] =      {          0, 0x08000000 },
-    [VIRT_CPUPERIPHS] = { 0x08000000, 0x00020000 },
+    [VIRT_FLASH] =              {          0, 0x08000000 },
+    [VIRT_CPUPERIPHS] =         { 0x08000000, 0x00020000 },
     /* GIC distributor and CPU interfaces sit inside the CPU peripheral space */
-    [VIRT_GIC_DIST] =   { 0x08000000, 0x00010000 },
-    [VIRT_GIC_CPU] =    { 0x08010000, 0x00010000 },
-    [VIRT_UART] =       { 0x09000000, 0x00001000 },
-    [VIRT_RTC] =        { 0x09010000, 0x00001000 },
-    [VIRT_MMIO] =       { 0x0a000000, 0x00000200 },
+    [VIRT_GIC_DIST] =           { 0x08000000, 0x00010000 },
+    [VIRT_GIC_CPU] =            { 0x08010000, 0x00010000 },
+    [VIRT_GIC_V2M] =            { 0x08020000, 0x00001000 },
+    /* The space in between here is reserved for GICv3 CPU/vCPU/HYP */
+    [VIRT_GIC_ITS] =            { 0x08080000, 0x00020000 },
+    /* This redistributor space allows up to 2*64kB*123 CPUs */
+    [VIRT_GIC_REDIST] =         { 0x080A0000, 0x00F60000 },
+    [VIRT_UART] =               { 0x09000000, 0x00001000 },
+    [VIRT_RTC] =                { 0x09010000, 0x00001000 },
+    [VIRT_FW_CFG] =             { 0x09020000, 0x00000018 },
+    [VIRT_GPIO] =               { 0x09030000, 0x00001000 },
+    [VIRT_SECURE_UART] =        { 0x09040000, 0x00001000 },
+    [VIRT_MMIO] =               { 0x0a000000, 0x00000200 },
     /* ...repeating for a total of NUM_VIRTIO_TRANSPORTS, each of that size */
-    /* 0x10000000 .. 0x40000000 reserved for PCI */
-    [VIRT_MEM] =        { 0x40000000, 30ULL * 1024 * 1024 * 1024 },
+    [VIRT_PLATFORM_BUS] =       { 0x0c000000, 0x02000000 },
+    [VIRT_SECURE_MEM] =         { 0x0e000000, 0x01000000 },
+    [VIRT_PCIE_MMIO] =          { 0x10000000, 0x2eff0000 },
+    [VIRT_PCIE_PIO] =           { 0x3eff0000, 0x00010000 },
+    [VIRT_PCIE_ECAM] =          { 0x3f000000, 0x01000000 },
+    [VIRT_MEM] =                { 0x40000000, RAMLIMIT_BYTES },
+    /* Second PCIe window, 512GB wide at the 512GB boundary */
+    [VIRT_PCIE_MMIO_HIGH] =   { 0x8000000000ULL, 0x8000000000ULL },
 };
 
 static const int a15irqmap[] = {
     [VIRT_UART] = 1,
     [VIRT_RTC] = 2,
+    [VIRT_PCIE] = 3, /* ... to 6 */
+    [VIRT_GPIO] = 7,
+    [VIRT_SECURE_UART] = 8,
     [VIRT_MMIO] = 16, /* ...to 16 + NUM_VIRTIO_TRANSPORTS - 1 */
+    [VIRT_GIC_V2M] = 48, /* ...to 48 + NUM_GICV2M_SPIS - 1 */
+    [VIRT_PLATFORM_BUS] = 112, /* ...to 112 + PLATFORM_BUS_NUM_IRQS -1 */
 };
 
 static VirtBoardInfo machines[] = {
@@ -126,6 +207,11 @@
         .irqmap = a15irqmap,
     },
     {
+        .cpu_model = "cortex-a53",
+        .memmap = a15memmap,
+        .irqmap = a15irqmap,
+    },
+    {
         .cpu_model = "cortex-a57",
         .memmap = a15memmap,
         .irqmap = a15irqmap,
@@ -198,6 +284,10 @@
     void *fdt = vbi->fdt;
     ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(0));
 
+    if (!vbi->using_psci) {
+        return;
+    }
+
     qemu_fdt_add_subnode(fdt, "/psci");
     if (armcpu->psci_version == 2) {
         const char comp[] = "arm,psci-0.2\0arm,psci";
@@ -235,7 +325,7 @@
     qemu_fdt_setprop_cell(fdt, "/psci", "migrate", migrate_fn);
 }
 
-static void fdt_add_timer_nodes(const VirtBoardInfo *vbi)
+static void fdt_add_timer_nodes(const VirtBoardInfo *vbi, int gictype)
 {
     /* Note that on A15 h/w these interrupts are level-triggered,
      * but for the GIC implementation provided by both QEMU and KVM
@@ -244,8 +334,11 @@
     ARMCPU *armcpu;
     uint32_t irqflags = GIC_FDT_IRQ_FLAGS_EDGE_LO_HI;
 
-    irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
-                         GIC_FDT_IRQ_PPI_CPU_WIDTH, (1 << vbi->smp_cpus) - 1);
+    if (gictype == 2) {
+        irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
+                             GIC_FDT_IRQ_PPI_CPU_WIDTH,
+                             (1 << vbi->smp_cpus) - 1);
+    }
 
     qemu_fdt_add_subnode(vbi->fdt, "/timer");
 
@@ -258,19 +351,44 @@
         qemu_fdt_setprop_string(vbi->fdt, "/timer", "compatible",
                                 "arm,armv7-timer");
     }
+    qemu_fdt_setprop(vbi->fdt, "/timer", "always-on", NULL, 0);
     qemu_fdt_setprop_cells(vbi->fdt, "/timer", "interrupts",
-                               GIC_FDT_IRQ_TYPE_PPI, 13, irqflags,
-                               GIC_FDT_IRQ_TYPE_PPI, 14, irqflags,
-                               GIC_FDT_IRQ_TYPE_PPI, 11, irqflags,
-                               GIC_FDT_IRQ_TYPE_PPI, 10, irqflags);
+                       GIC_FDT_IRQ_TYPE_PPI, ARCH_TIMER_S_EL1_IRQ, irqflags,
+                       GIC_FDT_IRQ_TYPE_PPI, ARCH_TIMER_NS_EL1_IRQ, irqflags,
+                       GIC_FDT_IRQ_TYPE_PPI, ARCH_TIMER_VIRT_IRQ, irqflags,
+                       GIC_FDT_IRQ_TYPE_PPI, ARCH_TIMER_NS_EL2_IRQ, irqflags);
 }
 
 static void fdt_add_cpu_nodes(const VirtBoardInfo *vbi)
 {
     int cpu;
+    int addr_cells = 1;
+    unsigned int i;
+
+    /*
+     * From Documentation/devicetree/bindings/arm/cpus.txt
+     *  On ARM v8 64-bit systems value should be set to 2,
+     *  that corresponds to the MPIDR_EL1 register size.
+     *  If MPIDR_EL1[63:32] value is equal to 0 on all CPUs
+     *  in the system, #address-cells can be set to 1, since
+     *  MPIDR_EL1[63:32] bits are not used for CPUs
+     *  identification.
+     *
+     *  Here we actually don't know whether our system is 32- or 64-bit one.
+     *  The simplest way to go is to examine affinity IDs of all our CPUs. If
+     *  at least one of them has Aff3 populated, we set #address-cells to 2.
+     */
+    for (cpu = 0; cpu < vbi->smp_cpus; cpu++) {
+        ARMCPU *armcpu = ARM_CPU(qemu_get_cpu(cpu));
+
+        if (armcpu->mp_affinity & ARM_AFF3_MASK) {
+            addr_cells = 2;
+            break;
+        }
+    }
 
     qemu_fdt_add_subnode(vbi->fdt, "/cpus");
-    qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", 0x1);
+    qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#address-cells", addr_cells);
     qemu_fdt_setprop_cell(vbi->fdt, "/cpus", "#size-cells", 0x0);
 
     for (cpu = vbi->smp_cpus - 1; cpu >= 0; cpu--) {
@@ -282,60 +400,153 @@
         qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible",
                                     armcpu->dtb_compatible);
 
-        if (vbi->smp_cpus > 1) {
+        if (vbi->using_psci && vbi->smp_cpus > 1) {
             qemu_fdt_setprop_string(vbi->fdt, nodename,
                                         "enable-method", "psci");
         }
 
-        qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg", cpu);
+        if (addr_cells == 2) {
+            qemu_fdt_setprop_u64(vbi->fdt, nodename, "reg",
+                                 armcpu->mp_affinity);
+        } else {
+            qemu_fdt_setprop_cell(vbi->fdt, nodename, "reg",
+                                  armcpu->mp_affinity);
+        }
+
+        for (i = 0; i < nb_numa_nodes; i++) {
+            if (test_bit(cpu, numa_info[i].node_cpu)) {
+                qemu_fdt_setprop_cell(vbi->fdt, nodename, "numa-node-id", i);
+            }
+        }
+
         g_free(nodename);
     }
 }
 
-static void fdt_add_gic_node(const VirtBoardInfo *vbi)
+static void fdt_add_v2m_gic_node(VirtBoardInfo *vbi)
 {
-    uint32_t gic_phandle;
-
-    gic_phandle = qemu_fdt_alloc_phandle(vbi->fdt);
-    qemu_fdt_setprop_cell(vbi->fdt, "/", "interrupt-parent", gic_phandle);
-
-    qemu_fdt_add_subnode(vbi->fdt, "/intc");
-    /* 'cortex-a15-gic' means 'GIC v2' */
-    qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
-                            "arm,cortex-a15-gic");
-    qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#interrupt-cells", 3);
-    qemu_fdt_setprop(vbi->fdt, "/intc", "interrupt-controller", NULL, 0);
-    qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
-                                     2, vbi->memmap[VIRT_GIC_DIST].base,
-                                     2, vbi->memmap[VIRT_GIC_DIST].size,
-                                     2, vbi->memmap[VIRT_GIC_CPU].base,
-                                     2, vbi->memmap[VIRT_GIC_CPU].size);
-    qemu_fdt_setprop_cell(vbi->fdt, "/intc", "phandle", gic_phandle);
+    vbi->v2m_phandle = qemu_fdt_alloc_phandle(vbi->fdt);
+    qemu_fdt_add_subnode(vbi->fdt, "/intc/v2m");
+    qemu_fdt_setprop_string(vbi->fdt, "/intc/v2m", "compatible",
+                            "arm,gic-v2m-frame");
+    qemu_fdt_setprop(vbi->fdt, "/intc/v2m", "msi-controller", NULL, 0);
+    qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc/v2m", "reg",
+                                 2, vbi->memmap[VIRT_GIC_V2M].base,
+                                 2, vbi->memmap[VIRT_GIC_V2M].size);
+    qemu_fdt_setprop_cell(vbi->fdt, "/intc/v2m", "phandle", vbi->v2m_phandle);
 }
 
-static void create_gic(const VirtBoardInfo *vbi, qemu_irq *pic)
+static void fdt_add_gic_node(VirtBoardInfo *vbi, int type)
 {
-    /* We create a standalone GIC v2 */
-    DeviceState *gicdev;
-    SysBusDevice *gicbusdev;
-    const char *gictype = "arm_gic";
-    int i;
+    vbi->gic_phandle = qemu_fdt_alloc_phandle(vbi->fdt);
+    qemu_fdt_setprop_cell(vbi->fdt, "/", "interrupt-parent", vbi->gic_phandle);
 
-    if (kvm_irqchip_in_kernel()) {
-        gictype = "kvm-arm-gic";
+    qemu_fdt_add_subnode(vbi->fdt, "/intc");
+    qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#interrupt-cells", 3);
+    qemu_fdt_setprop(vbi->fdt, "/intc", "interrupt-controller", NULL, 0);
+    qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#address-cells", 0x2);
+    qemu_fdt_setprop_cell(vbi->fdt, "/intc", "#size-cells", 0x2);
+    qemu_fdt_setprop(vbi->fdt, "/intc", "ranges", NULL, 0);
+    if (type == 3) {
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,gic-v3");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                     2, vbi->memmap[VIRT_GIC_DIST].base,
+                                     2, vbi->memmap[VIRT_GIC_DIST].size,
+                                     2, vbi->memmap[VIRT_GIC_REDIST].base,
+                                     2, vbi->memmap[VIRT_GIC_REDIST].size);
+    } else {
+        /* 'cortex-a15-gic' means 'GIC v2' */
+        qemu_fdt_setprop_string(vbi->fdt, "/intc", "compatible",
+                                "arm,cortex-a15-gic");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, "/intc", "reg",
+                                      2, vbi->memmap[VIRT_GIC_DIST].base,
+                                      2, vbi->memmap[VIRT_GIC_DIST].size,
+                                      2, vbi->memmap[VIRT_GIC_CPU].base,
+                                      2, vbi->memmap[VIRT_GIC_CPU].size);
     }
 
+    qemu_fdt_setprop_cell(vbi->fdt, "/intc", "phandle", vbi->gic_phandle);
+}
+
+static void fdt_add_pmu_nodes(const VirtBoardInfo *vbi, int gictype)
+{
+    CPUState *cpu;
+    ARMCPU *armcpu;
+    uint32_t irqflags = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
+
+    CPU_FOREACH(cpu) {
+        armcpu = ARM_CPU(cpu);
+        if (!armcpu->has_pmu ||
+            !kvm_arm_pmu_create(cpu, PPI(VIRTUAL_PMU_IRQ))) {
+            return;
+        }
+    }
+
+    if (gictype == 2) {
+        irqflags = deposit32(irqflags, GIC_FDT_IRQ_PPI_CPU_START,
+                             GIC_FDT_IRQ_PPI_CPU_WIDTH,
+                             (1 << vbi->smp_cpus) - 1);
+    }
+
+    armcpu = ARM_CPU(qemu_get_cpu(0));
+    qemu_fdt_add_subnode(vbi->fdt, "/pmu");
+    if (arm_feature(&armcpu->env, ARM_FEATURE_V8)) {
+        const char compat[] = "arm,armv8-pmuv3";
+        qemu_fdt_setprop(vbi->fdt, "/pmu", "compatible",
+                         compat, sizeof(compat));
+        qemu_fdt_setprop_cells(vbi->fdt, "/pmu", "interrupts",
+                               GIC_FDT_IRQ_TYPE_PPI, VIRTUAL_PMU_IRQ, irqflags);
+    }
+}
+
+static void create_v2m(VirtBoardInfo *vbi, qemu_irq *pic)
+{
+    int i;
+    int irq = vbi->irqmap[VIRT_GIC_V2M];
+    DeviceState *dev;
+
+    dev = qdev_create(NULL, "arm-gicv2m");
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, vbi->memmap[VIRT_GIC_V2M].base);
+    qdev_prop_set_uint32(dev, "base-spi", irq);
+    qdev_prop_set_uint32(dev, "num-spi", NUM_GICV2M_SPIS);
+    qdev_init_nofail(dev);
+
+    for (i = 0; i < NUM_GICV2M_SPIS; i++) {
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, pic[irq + i]);
+    }
+
+    fdt_add_v2m_gic_node(vbi);
+}
+
+static void create_gic(VirtBoardInfo *vbi, qemu_irq *pic, int type, bool secure)
+{
+    /* We create a standalone GIC */
+    DeviceState *gicdev;
+    SysBusDevice *gicbusdev;
+    const char *gictype;
+    int i;
+
+    gictype = (type == 3) ? gicv3_class_name() : gic_class_name();
+
     gicdev = qdev_create(NULL, gictype);
-    qdev_prop_set_uint32(gicdev, "revision", 2);
+    qdev_prop_set_uint32(gicdev, "revision", type);
     qdev_prop_set_uint32(gicdev, "num-cpu", smp_cpus);
     /* Note that the num-irq property counts both internal and external
      * interrupts; there are always 32 of the former (mandated by GIC spec).
      */
     qdev_prop_set_uint32(gicdev, "num-irq", NUM_IRQS + 32);
+    if (!kvm_irqchip_in_kernel()) {
+        qdev_prop_set_bit(gicdev, "has-security-extensions", secure);
+    }
     qdev_init_nofail(gicdev);
     gicbusdev = SYS_BUS_DEVICE(gicdev);
     sysbus_mmio_map(gicbusdev, 0, vbi->memmap[VIRT_GIC_DIST].base);
-    sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    if (type == 3) {
+        sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_REDIST].base);
+    } else {
+        sysbus_mmio_map(gicbusdev, 1, vbi->memmap[VIRT_GIC_CPU].base);
+    }
 
     /* Wire the outputs from each CPU's generic timer to the
      * appropriate GIC PPI inputs, and the GIC's IRQ output to
@@ -343,36 +554,57 @@
      */
     for (i = 0; i < smp_cpus; i++) {
         DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
-        int ppibase = NUM_IRQS + i * 32;
-        /* physical timer; we wire it up to the non-secure timer's ID,
-         * since a real A15 always has TrustZone but QEMU doesn't.
+        int ppibase = NUM_IRQS + i * GIC_INTERNAL + GIC_NR_SGIS;
+        int irq;
+        /* Mapping from the output timer irq lines from the CPU to the
+         * GIC PPI inputs we use for the virt board.
          */
-        qdev_connect_gpio_out(cpudev, 0,
-                              qdev_get_gpio_in(gicdev, ppibase + 30));
-        /* virtual timer */
-        qdev_connect_gpio_out(cpudev, 1,
-                              qdev_get_gpio_in(gicdev, ppibase + 27));
+        const int timer_irq[] = {
+            [GTIMER_PHYS] = ARCH_TIMER_NS_EL1_IRQ,
+            [GTIMER_VIRT] = ARCH_TIMER_VIRT_IRQ,
+            [GTIMER_HYP]  = ARCH_TIMER_NS_EL2_IRQ,
+            [GTIMER_SEC]  = ARCH_TIMER_S_EL1_IRQ,
+        };
+
+        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
+            qdev_connect_gpio_out(cpudev, irq,
+                                  qdev_get_gpio_in(gicdev,
+                                                   ppibase + timer_irq[irq]));
+        }
 
         sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ));
+        sysbus_connect_irq(gicbusdev, i + smp_cpus,
+                           qdev_get_gpio_in(cpudev, ARM_CPU_FIQ));
     }
 
     for (i = 0; i < NUM_IRQS; i++) {
         pic[i] = qdev_get_gpio_in(gicdev, i);
     }
 
-    fdt_add_gic_node(vbi);
+    fdt_add_gic_node(vbi, type);
+
+    if (type == 2) {
+        create_v2m(vbi, pic);
+    }
 }
 
-static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic)
+static void create_uart(const VirtBoardInfo *vbi, qemu_irq *pic, int uart,
+                        MemoryRegion *mem, CharDriverState *chr)
 {
     char *nodename;
-    hwaddr base = vbi->memmap[VIRT_UART].base;
-    hwaddr size = vbi->memmap[VIRT_UART].size;
-    int irq = vbi->irqmap[VIRT_UART];
+    hwaddr base = vbi->memmap[uart].base;
+    hwaddr size = vbi->memmap[uart].size;
+    int irq = vbi->irqmap[uart];
     const char compat[] = "arm,pl011\0arm,primecell";
     const char clocknames[] = "uartclk\0apb_pclk";
+    DeviceState *dev = qdev_create(NULL, "pl011");
+    SysBusDevice *s = SYS_BUS_DEVICE(dev);
 
-    sysbus_create_simple("pl011", base, pic[irq]);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    memory_region_add_subregion(mem, base,
+                                sysbus_mmio_get_region(s, 0));
+    sysbus_connect_irq(s, 0, pic[irq]);
 
     nodename = g_strdup_printf("/pl011@%" PRIx64, base);
     qemu_fdt_add_subnode(vbi->fdt, nodename);
@@ -389,7 +621,14 @@
     qemu_fdt_setprop(vbi->fdt, nodename, "clock-names",
                          clocknames, sizeof(clocknames));
 
-    qemu_fdt_setprop_string(vbi->fdt, "/chosen", "stdout-path", nodename);
+    if (uart == VIRT_UART) {
+        qemu_fdt_setprop_string(vbi->fdt, "/chosen", "stdout-path", nodename);
+    } else {
+        /* Mark as not usable by the normal world */
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "status", "disabled");
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "secure-status", "okay");
+    }
+
     g_free(nodename);
 }
 
@@ -416,15 +655,95 @@
     g_free(nodename);
 }
 
+static DeviceState *gpio_key_dev;
+static void virt_powerdown_req(Notifier *n, void *opaque)
+{
+    /* use gpio Pin 3 for power button event */
+    qemu_set_irq(qdev_get_gpio_in(gpio_key_dev, 0), 1);
+}
+
+static Notifier virt_system_powerdown_notifier = {
+    .notify = virt_powerdown_req
+};
+
+static void create_gpio(const VirtBoardInfo *vbi, qemu_irq *pic)
+{
+    char *nodename;
+    DeviceState *pl061_dev;
+    hwaddr base = vbi->memmap[VIRT_GPIO].base;
+    hwaddr size = vbi->memmap[VIRT_GPIO].size;
+    int irq = vbi->irqmap[VIRT_GPIO];
+    const char compat[] = "arm,pl061\0arm,primecell";
+
+    pl061_dev = sysbus_create_simple("pl061", base, pic[irq]);
+
+    uint32_t phandle = qemu_fdt_alloc_phandle(vbi->fdt);
+    nodename = g_strdup_printf("/pl061@%" PRIx64, base);
+    qemu_fdt_add_subnode(vbi->fdt, nodename);
+    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                 2, base, 2, size);
+    qemu_fdt_setprop(vbi->fdt, nodename, "compatible", compat, sizeof(compat));
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "#gpio-cells", 2);
+    qemu_fdt_setprop(vbi->fdt, nodename, "gpio-controller", NULL, 0);
+    qemu_fdt_setprop_cells(vbi->fdt, nodename, "interrupts",
+                           GIC_FDT_IRQ_TYPE_SPI, irq,
+                           GIC_FDT_IRQ_FLAGS_LEVEL_HI);
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "clocks", vbi->clock_phandle);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "clock-names", "apb_pclk");
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "phandle", phandle);
+
+    gpio_key_dev = sysbus_create_simple("gpio-key", -1,
+                                        qdev_get_gpio_in(pl061_dev, 3));
+    qemu_fdt_add_subnode(vbi->fdt, "/gpio-keys");
+    qemu_fdt_setprop_string(vbi->fdt, "/gpio-keys", "compatible", "gpio-keys");
+    qemu_fdt_setprop_cell(vbi->fdt, "/gpio-keys", "#size-cells", 0);
+    qemu_fdt_setprop_cell(vbi->fdt, "/gpio-keys", "#address-cells", 1);
+
+    qemu_fdt_add_subnode(vbi->fdt, "/gpio-keys/poweroff");
+    qemu_fdt_setprop_string(vbi->fdt, "/gpio-keys/poweroff",
+                            "label", "GPIO Key Poweroff");
+    qemu_fdt_setprop_cell(vbi->fdt, "/gpio-keys/poweroff", "linux,code",
+                          KEY_POWER);
+    qemu_fdt_setprop_cells(vbi->fdt, "/gpio-keys/poweroff",
+                           "gpios", phandle, 3, 0);
+
+    /* connect powerdown request */
+    qemu_register_powerdown_notifier(&virt_system_powerdown_notifier);
+
+    g_free(nodename);
+}
+
 static void create_virtio_devices(const VirtBoardInfo *vbi, qemu_irq *pic)
 {
     int i;
     hwaddr size = vbi->memmap[VIRT_MMIO].size;
 
-    /* Note that we have to create the transports in forwards order
-     * so that command line devices are inserted lowest address first,
-     * and then add dtb nodes in reverse order so that they appear in
-     * the finished device tree lowest address first.
+    /* We create the transports in forwards order. Since qbus_realize()
+     * prepends (not appends) new child buses, the incrementing loop below will
+     * create a list of virtio-mmio buses with decreasing base addresses.
+     *
+     * When a -device option is processed from the command line,
+     * qbus_find_recursive() picks the next free virtio-mmio bus in forwards
+     * order. The upshot is that -device options in increasing command line
+     * order are mapped to virtio-mmio buses with decreasing base addresses.
+     *
+     * When this code was originally written, that arrangement ensured that the
+     * guest Linux kernel would give the lowest "name" (/dev/vda, eth0, etc) to
+     * the first -device on the command line. (The end-to-end order is a
+     * function of this loop, qbus_realize(), qbus_find_recursive(), and the
+     * guest kernel's name-to-address assignment strategy.)
+     *
+     * Meanwhile, the kernel's traversal seems to have been reversed; see eg.
+     * the message, if not necessarily the code, of commit 70161ff336.
+     * Therefore the loop now establishes the inverse of the original intent.
+     *
+     * Unfortunately, we can't counteract the kernel change by reversing the
+     * loop; it would break existing command lines.
+     *
+     * In any case, the kernel makes no guarantee about the stability of
+     * enumeration order of virtio devices (as demonstrated by it changing
+     * between kernel versions). For reliable and stable identification
+     * of disks users must use UUIDs or similar mechanisms.
      */
     for (i = 0; i < NUM_VIRTIO_TRANSPORTS; i++) {
         int irq = vbi->irqmap[VIRT_MMIO] + i;
@@ -433,6 +752,13 @@
         sysbus_create_simple("virtio-mmio", base, pic[irq]);
     }
 
+    /* We add dtb nodes in reverse order so that they appear in the finished
+     * device tree lowest address first.
+     *
+     * Note that this mapping is independent of the loop above. The previous
+     * loop influences virtio device to virtio transport assignment, whereas
+     * this loop controls how virtio transports are laid out in the dtb.
+     */
     for (i = NUM_VIRTIO_TRANSPORTS - 1; i >= 0; i--) {
         char *nodename;
         int irq = vbi->irqmap[VIRT_MMIO] + i;
@@ -452,25 +778,27 @@
 }
 
 static void create_one_flash(const char *name, hwaddr flashbase,
-                             hwaddr flashsize)
+                             hwaddr flashsize, const char *file,
+                             MemoryRegion *sysmem)
 {
     /* Create and map a single flash device. We use the same
      * parameters as the flash devices on the Versatile Express board.
      */
     DriveInfo *dinfo = drive_get_next(IF_PFLASH);
     DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
     const uint64_t sectorlength = 256 * 1024;
 
-    if (dinfo && qdev_prop_set_drive(dev, "drive",
-                                     blk_by_legacy_dinfo(dinfo))) {
-        abort();
+    if (dinfo) {
+        qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                            &error_abort);
     }
 
     qdev_prop_set_uint32(dev, "num-blocks", flashsize / sectorlength);
     qdev_prop_set_uint64(dev, "sector-length", sectorlength);
     qdev_prop_set_uint8(dev, "width", 4);
     qdev_prop_set_uint8(dev, "device-width", 2);
-    qdev_prop_set_uint8(dev, "big-endian", 0);
+    qdev_prop_set_bit(dev, "big-endian", false);
     qdev_prop_set_uint16(dev, "id0", 0x89);
     qdev_prop_set_uint16(dev, "id1", 0x18);
     qdev_prop_set_uint16(dev, "id2", 0x00);
@@ -478,20 +806,12 @@
     qdev_prop_set_string(dev, "name", name);
     qdev_init_nofail(dev);
 
-    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, flashbase);
-}
+    memory_region_add_subregion(sysmem, flashbase,
+                                sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0));
 
-static void create_flash(const VirtBoardInfo *vbi)
-{
-    /* Create two flash devices to fill the VIRT_FLASH space in the memmap.
-     * Any file passed via -bios goes in the first of these.
-     */
-    hwaddr flashsize = vbi->memmap[VIRT_FLASH].size / 2;
-    hwaddr flashbase = vbi->memmap[VIRT_FLASH].base;
-    char *nodename;
-
-    if (bios_name) {
-        const char *fn;
+    if (file) {
+        char *fn;
+        int image_size;
 
         if (drive_get(IF_PFLASH, 0, 0)) {
             error_report("The contents of the first flash device may be "
@@ -499,23 +819,302 @@
                          "but you cannot use both options at once");
             exit(1);
         }
-        fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
-        if (!fn || load_image_targphys(fn, flashbase, flashsize) < 0) {
-            error_report("Could not load ROM image '%s'", bios_name);
+        fn = qemu_find_file(QEMU_FILE_TYPE_BIOS, file);
+        if (!fn) {
+            error_report("Could not find ROM image '%s'", file);
+            exit(1);
+        }
+        image_size = load_image_mr(fn, sysbus_mmio_get_region(sbd, 0));
+        g_free(fn);
+        if (image_size < 0) {
+            error_report("Could not load ROM image '%s'", file);
             exit(1);
         }
     }
+}
 
-    create_one_flash("virt.flash0", flashbase, flashsize);
-    create_one_flash("virt.flash1", flashbase + flashsize, flashsize);
+static void create_flash(const VirtBoardInfo *vbi,
+                         MemoryRegion *sysmem,
+                         MemoryRegion *secure_sysmem)
+{
+    /* Create two flash devices to fill the VIRT_FLASH space in the memmap.
+     * Any file passed via -bios goes in the first of these.
+     * sysmem is the system memory space. secure_sysmem is the secure view
+     * of the system, and the first flash device should be made visible only
+     * there. The second flash device is visible to both secure and nonsecure.
+     * If sysmem == secure_sysmem this means there is no separate Secure
+     * address space and both flash devices are generally visible.
+     */
+    hwaddr flashsize = vbi->memmap[VIRT_FLASH].size / 2;
+    hwaddr flashbase = vbi->memmap[VIRT_FLASH].base;
+    char *nodename;
 
-    nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
+    create_one_flash("virt.flash0", flashbase, flashsize,
+                     bios_name, secure_sysmem);
+    create_one_flash("virt.flash1", flashbase + flashsize, flashsize,
+                     NULL, sysmem);
+
+    if (sysmem == secure_sysmem) {
+        /* Report both flash devices as a single node in the DT */
+        nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
+        qemu_fdt_add_subnode(vbi->fdt, nodename);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                     2, flashbase, 2, flashsize,
+                                     2, flashbase + flashsize, 2, flashsize);
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+        g_free(nodename);
+    } else {
+        /* Report the devices as separate nodes so we can mark one as
+         * only visible to the secure world.
+         */
+        nodename = g_strdup_printf("/secflash@%" PRIx64, flashbase);
+        qemu_fdt_add_subnode(vbi->fdt, nodename);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                     2, flashbase, 2, flashsize);
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "status", "disabled");
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "secure-status", "okay");
+        g_free(nodename);
+
+        nodename = g_strdup_printf("/flash@%" PRIx64, flashbase);
+        qemu_fdt_add_subnode(vbi->fdt, nodename);
+        qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                     2, flashbase + flashsize, 2, flashsize);
+        qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+        g_free(nodename);
+    }
+}
+
+static void create_fw_cfg(const VirtBoardInfo *vbi, AddressSpace *as)
+{
+    hwaddr base = vbi->memmap[VIRT_FW_CFG].base;
+    hwaddr size = vbi->memmap[VIRT_FW_CFG].size;
+    char *nodename;
+
+    fw_cfg_init_mem_wide(base + 8, base, 8, base + 16, as);
+
+    nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base);
     qemu_fdt_add_subnode(vbi->fdt, nodename);
-    qemu_fdt_setprop_string(vbi->fdt, nodename, "compatible", "cfi-flash");
+    qemu_fdt_setprop_string(vbi->fdt, nodename,
+                            "compatible", "qemu,fw-cfg-mmio");
     qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
-                                 2, flashbase, 2, flashsize,
-                                 2, flashbase + flashsize, 2, flashsize);
-    qemu_fdt_setprop_cell(vbi->fdt, nodename, "bank-width", 4);
+                                 2, base, 2, size);
+    g_free(nodename);
+}
+
+static void create_pcie_irq_map(const VirtBoardInfo *vbi, uint32_t gic_phandle,
+                                int first_irq, const char *nodename)
+{
+    int devfn, pin;
+    uint32_t full_irq_map[4 * 4 * 10] = { 0 };
+    uint32_t *irq_map = full_irq_map;
+
+    for (devfn = 0; devfn <= 0x18; devfn += 0x8) {
+        for (pin = 0; pin < 4; pin++) {
+            int irq_type = GIC_FDT_IRQ_TYPE_SPI;
+            int irq_nr = first_irq + ((pin + PCI_SLOT(devfn)) % PCI_NUM_PINS);
+            int irq_level = GIC_FDT_IRQ_FLAGS_LEVEL_HI;
+            int i;
+
+            uint32_t map[] = {
+                devfn << 8, 0, 0,                           /* devfn */
+                pin + 1,                                    /* PCI pin */
+                gic_phandle, 0, 0, irq_type, irq_nr, irq_level }; /* GIC irq */
+
+            /* Convert map to big endian */
+            for (i = 0; i < 10; i++) {
+                irq_map[i] = cpu_to_be32(map[i]);
+            }
+            irq_map += 10;
+        }
+    }
+
+    qemu_fdt_setprop(vbi->fdt, nodename, "interrupt-map",
+                     full_irq_map, sizeof(full_irq_map));
+
+    qemu_fdt_setprop_cells(vbi->fdt, nodename, "interrupt-map-mask",
+                           0x1800, 0, 0, /* devfn (PCI_SLOT(3)) */
+                           0x7           /* PCI irq */);
+}
+
+static void create_pcie(const VirtBoardInfo *vbi, qemu_irq *pic,
+                        bool use_highmem)
+{
+    hwaddr base_mmio = vbi->memmap[VIRT_PCIE_MMIO].base;
+    hwaddr size_mmio = vbi->memmap[VIRT_PCIE_MMIO].size;
+    hwaddr base_mmio_high = vbi->memmap[VIRT_PCIE_MMIO_HIGH].base;
+    hwaddr size_mmio_high = vbi->memmap[VIRT_PCIE_MMIO_HIGH].size;
+    hwaddr base_pio = vbi->memmap[VIRT_PCIE_PIO].base;
+    hwaddr size_pio = vbi->memmap[VIRT_PCIE_PIO].size;
+    hwaddr base_ecam = vbi->memmap[VIRT_PCIE_ECAM].base;
+    hwaddr size_ecam = vbi->memmap[VIRT_PCIE_ECAM].size;
+    hwaddr base = base_mmio;
+    int nr_pcie_buses = size_ecam / PCIE_MMCFG_SIZE_MIN;
+    int irq = vbi->irqmap[VIRT_PCIE];
+    MemoryRegion *mmio_alias;
+    MemoryRegion *mmio_reg;
+    MemoryRegion *ecam_alias;
+    MemoryRegion *ecam_reg;
+    DeviceState *dev;
+    char *nodename;
+    int i;
+    PCIHostState *pci;
+
+    dev = qdev_create(NULL, TYPE_GPEX_HOST);
+    qdev_init_nofail(dev);
+
+    /* Map only the first size_ecam bytes of ECAM space */
+    ecam_alias = g_new0(MemoryRegion, 1);
+    ecam_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 0);
+    memory_region_init_alias(ecam_alias, OBJECT(dev), "pcie-ecam",
+                             ecam_reg, 0, size_ecam);
+    memory_region_add_subregion(get_system_memory(), base_ecam, ecam_alias);
+
+    /* Map the MMIO window into system address space so as to expose
+     * the section of PCI MMIO space which starts at the same base address
+     * (ie 1:1 mapping for that part of PCI MMIO space visible through
+     * the window).
+     */
+    mmio_alias = g_new0(MemoryRegion, 1);
+    mmio_reg = sysbus_mmio_get_region(SYS_BUS_DEVICE(dev), 1);
+    memory_region_init_alias(mmio_alias, OBJECT(dev), "pcie-mmio",
+                             mmio_reg, base_mmio, size_mmio);
+    memory_region_add_subregion(get_system_memory(), base_mmio, mmio_alias);
+
+    if (use_highmem) {
+        /* Map high MMIO space */
+        MemoryRegion *high_mmio_alias = g_new0(MemoryRegion, 1);
+
+        memory_region_init_alias(high_mmio_alias, OBJECT(dev), "pcie-mmio-high",
+                                 mmio_reg, base_mmio_high, size_mmio_high);
+        memory_region_add_subregion(get_system_memory(), base_mmio_high,
+                                    high_mmio_alias);
+    }
+
+    /* Map IO port space */
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 2, base_pio);
+
+    for (i = 0; i < GPEX_NUM_IRQS; i++) {
+        sysbus_connect_irq(SYS_BUS_DEVICE(dev), i, pic[irq + i]);
+    }
+
+    pci = PCI_HOST_BRIDGE(dev);
+    if (pci->bus) {
+        for (i = 0; i < nb_nics; i++) {
+            NICInfo *nd = &nd_table[i];
+
+            if (!nd->model) {
+                nd->model = g_strdup("virtio");
+            }
+
+            pci_nic_init_nofail(nd, pci->bus, nd->model, NULL);
+        }
+    }
+
+    nodename = g_strdup_printf("/pcie@%" PRIx64, base);
+    qemu_fdt_add_subnode(vbi->fdt, nodename);
+    qemu_fdt_setprop_string(vbi->fdt, nodename,
+                            "compatible", "pci-host-ecam-generic");
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "device_type", "pci");
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "#address-cells", 3);
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "#size-cells", 2);
+    qemu_fdt_setprop_cells(vbi->fdt, nodename, "bus-range", 0,
+                           nr_pcie_buses - 1);
+    qemu_fdt_setprop(vbi->fdt, nodename, "dma-coherent", NULL, 0);
+
+    if (vbi->v2m_phandle) {
+        qemu_fdt_setprop_cells(vbi->fdt, nodename, "msi-parent",
+                               vbi->v2m_phandle);
+    }
+
+    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg",
+                                 2, base_ecam, 2, size_ecam);
+
+    if (use_highmem) {
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
+                                     1, FDT_PCI_RANGE_IOPORT, 2, 0,
+                                     2, base_pio, 2, size_pio,
+                                     1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
+                                     2, base_mmio, 2, size_mmio,
+                                     1, FDT_PCI_RANGE_MMIO_64BIT,
+                                     2, base_mmio_high,
+                                     2, base_mmio_high, 2, size_mmio_high);
+    } else {
+        qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "ranges",
+                                     1, FDT_PCI_RANGE_IOPORT, 2, 0,
+                                     2, base_pio, 2, size_pio,
+                                     1, FDT_PCI_RANGE_MMIO, 2, base_mmio,
+                                     2, base_mmio, 2, size_mmio);
+    }
+
+    qemu_fdt_setprop_cell(vbi->fdt, nodename, "#interrupt-cells", 1);
+    create_pcie_irq_map(vbi, vbi->gic_phandle, irq, nodename);
+
+    g_free(nodename);
+}
+
+static void create_platform_bus(VirtBoardInfo *vbi, qemu_irq *pic)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+    int i;
+    ARMPlatformBusFDTParams *fdt_params = g_new(ARMPlatformBusFDTParams, 1);
+    MemoryRegion *sysmem = get_system_memory();
+
+    platform_bus_params.platform_bus_base = vbi->memmap[VIRT_PLATFORM_BUS].base;
+    platform_bus_params.platform_bus_size = vbi->memmap[VIRT_PLATFORM_BUS].size;
+    platform_bus_params.platform_bus_first_irq = vbi->irqmap[VIRT_PLATFORM_BUS];
+    platform_bus_params.platform_bus_num_irqs = PLATFORM_BUS_NUM_IRQS;
+
+    fdt_params->system_params = &platform_bus_params;
+    fdt_params->binfo = &vbi->bootinfo;
+    fdt_params->intc = "/intc";
+    /*
+     * register a machine init done notifier that creates the device tree
+     * nodes of the platform bus and its children dynamic sysbus devices
+     */
+    arm_register_platform_bus_fdt_creator(fdt_params);
+
+    dev = qdev_create(NULL, TYPE_PLATFORM_BUS_DEVICE);
+    dev->id = TYPE_PLATFORM_BUS_DEVICE;
+    qdev_prop_set_uint32(dev, "num_irqs",
+        platform_bus_params.platform_bus_num_irqs);
+    qdev_prop_set_uint32(dev, "mmio_size",
+        platform_bus_params.platform_bus_size);
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+
+    for (i = 0; i < platform_bus_params.platform_bus_num_irqs; i++) {
+        int irqn = platform_bus_params.platform_bus_first_irq + i;
+        sysbus_connect_irq(s, i, pic[irqn]);
+    }
+
+    memory_region_add_subregion(sysmem,
+                                platform_bus_params.platform_bus_base,
+                                sysbus_mmio_get_region(s, 0));
+}
+
+static void create_secure_ram(VirtBoardInfo *vbi, MemoryRegion *secure_sysmem)
+{
+    MemoryRegion *secram = g_new(MemoryRegion, 1);
+    char *nodename;
+    hwaddr base = vbi->memmap[VIRT_SECURE_MEM].base;
+    hwaddr size = vbi->memmap[VIRT_SECURE_MEM].size;
+
+    memory_region_init_ram(secram, NULL, "virt.secure-ram", size, &error_fatal);
+    vmstate_register_ram_global(secram);
+    memory_region_add_subregion(secure_sysmem, base, secram);
+
+    nodename = g_strdup_printf("/secram@%" PRIx64, base);
+    qemu_fdt_add_subnode(vbi->fdt, nodename);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "device_type", "memory");
+    qemu_fdt_setprop_sized_cells(vbi->fdt, nodename, "reg", 2, base, 2, size);
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "status", "disabled");
+    qemu_fdt_setprop_string(vbi->fdt, nodename, "secure-status", "okay");
+
     g_free(nodename);
 }
 
@@ -527,51 +1126,195 @@
     return board->fdt;
 }
 
+static void virt_build_smbios(VirtGuestInfo *guest_info)
+{
+    FWCfgState *fw_cfg = guest_info->fw_cfg;
+    uint8_t *smbios_tables, *smbios_anchor;
+    size_t smbios_tables_len, smbios_anchor_len;
+    const char *product = "QEMU Virtual Machine";
+
+    if (!fw_cfg) {
+        return;
+    }
+
+    if (kvm_enabled()) {
+        product = "KVM Virtual Machine";
+    }
+
+    smbios_set_defaults("QEMU", product,
+                        "1.0", false, true, SMBIOS_ENTRY_POINT_30);
+
+    smbios_get_tables(NULL, 0, &smbios_tables, &smbios_tables_len,
+                      &smbios_anchor, &smbios_anchor_len);
+
+    if (smbios_anchor) {
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
+                        smbios_tables, smbios_tables_len);
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
+                        smbios_anchor, smbios_anchor_len);
+    }
+}
+
+static
+void virt_guest_info_machine_done(Notifier *notifier, void *data)
+{
+    VirtGuestInfoState *guest_info_state = container_of(notifier,
+                                              VirtGuestInfoState, machine_done);
+    virt_acpi_setup(&guest_info_state->info);
+    virt_build_smbios(&guest_info_state->info);
+}
+
 static void machvirt_init(MachineState *machine)
 {
+    VirtMachineState *vms = VIRT_MACHINE(machine);
+    VirtMachineClass *vmc = VIRT_MACHINE_GET_CLASS(machine);
     qemu_irq pic[NUM_IRQS];
     MemoryRegion *sysmem = get_system_memory();
-    int n;
+    MemoryRegion *secure_sysmem = NULL;
+    int gic_version = vms->gic_version;
+    int n, virt_max_cpus;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     const char *cpu_model = machine->cpu_model;
     VirtBoardInfo *vbi;
+    VirtGuestInfoState *guest_info_state = g_malloc0(sizeof *guest_info_state);
+    VirtGuestInfo *guest_info = &guest_info_state->info;
+    char **cpustr;
+    ObjectClass *oc;
+    const char *typename;
+    CPUClass *cc;
+    Error *err = NULL;
+    bool firmware_loaded = bios_name || drive_get(IF_PFLASH, 0, 0);
+    uint8_t clustersz;
 
     if (!cpu_model) {
         cpu_model = "cortex-a15";
     }
 
-    vbi = find_machine_info(cpu_model);
+    /* We can probe only here because during property set
+     * KVM is not available yet
+     */
+    if (!gic_version) {
+        if (!kvm_enabled()) {
+            error_report("gic-version=host requires KVM");
+            exit(1);
+        }
+
+        gic_version = kvm_arm_vgic_probe();
+        if (!gic_version) {
+            error_report("Unable to determine GIC version supported by host");
+            exit(1);
+        }
+    }
+
+    /* Separate the actual CPU model name from any appended features */
+    cpustr = g_strsplit(cpu_model, ",", 2);
+
+    vbi = find_machine_info(cpustr[0]);
 
     if (!vbi) {
-        error_report("mach-virt: CPU %s not supported", cpu_model);
+        error_report("mach-virt: CPU %s not supported", cpustr[0]);
+        exit(1);
+    }
+
+    /* If we have an EL3 boot ROM then the assumption is that it will
+     * implement PSCI itself, so disable QEMU's internal implementation
+     * so it doesn't get in the way. Instead of starting secondary
+     * CPUs in PSCI powerdown state we will start them all running and
+     * let the boot ROM sort them out.
+     * The usual case is that we do use QEMU's PSCI implementation.
+     */
+    vbi->using_psci = !(vms->secure && firmware_loaded);
+
+    /* The maximum number of CPUs depends on the GIC version, or on how
+     * many redistributors we can fit into the memory map.
+     */
+    if (gic_version == 3) {
+        virt_max_cpus = vbi->memmap[VIRT_GIC_REDIST].size / 0x20000;
+        clustersz = GICV3_TARGETLIST_BITS;
+    } else {
+        virt_max_cpus = GIC_NCPU;
+        clustersz = GIC_TARGETLIST_BITS;
+    }
+
+    if (max_cpus > virt_max_cpus) {
+        error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
+                     "supported by machine 'mach-virt' (%d)",
+                     max_cpus, virt_max_cpus);
         exit(1);
     }
 
     vbi->smp_cpus = smp_cpus;
 
     if (machine->ram_size > vbi->memmap[VIRT_MEM].size) {
-        error_report("mach-virt: cannot model more than 30GB RAM");
+        error_report("mach-virt: cannot model more than %dGB RAM", RAMLIMIT_GB);
         exit(1);
     }
 
+    if (vms->secure) {
+        if (kvm_enabled()) {
+            error_report("mach-virt: KVM does not support Security extensions");
+            exit(1);
+        }
+
+        /* The Secure view of the world is the same as the NonSecure,
+         * but with a few extra devices. Create it as a container region
+         * containing the system memory at low priority; any secure-only
+         * devices go in at higher priority and take precedence.
+         */
+        secure_sysmem = g_new(MemoryRegion, 1);
+        memory_region_init(secure_sysmem, OBJECT(machine), "secure-memory",
+                           UINT64_MAX);
+        memory_region_add_subregion_overlap(secure_sysmem, 0, sysmem, -1);
+    }
+
     create_fdt(vbi);
 
+    oc = cpu_class_by_name(TYPE_ARM_CPU, cpustr[0]);
+    if (!oc) {
+        error_report("Unable to find CPU definition");
+        exit(1);
+    }
+    typename = object_class_get_name(oc);
+
+    /* convert -smp CPU options specified by the user into global props */
+    cc = CPU_CLASS(oc);
+    cc->parse_features(typename, cpustr[1], &err);
+    g_strfreev(cpustr);
+    if (err) {
+        error_report_err(err);
+        exit(1);
+    }
+
     for (n = 0; n < smp_cpus; n++) {
-        ObjectClass *oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
-        Object *cpuobj;
-
-        if (!oc) {
-            fprintf(stderr, "Unable to find CPU definition\n");
-            exit(1);
+        Object *cpuobj = object_new(typename);
+        if (!vmc->disallow_affinity_adjustment) {
+            /* Adjust MPIDR like 64-bit KVM hosts, which incorporate the
+             * GIC's target-list limitations. 32-bit KVM hosts currently
+             * always create clusters of 4 CPUs, but that is expected to
+             * change when they gain support for gicv3. When KVM is enabled
+             * it will override the changes we make here, therefore our
+             * purposes are to make TCG consistent (with 64-bit KVM hosts)
+             * and to improve SGI efficiency.
+             */
+            uint8_t aff1 = n / clustersz;
+            uint8_t aff0 = n % clustersz;
+            object_property_set_int(cpuobj, (aff1 << ARM_AFF1_SHIFT) | aff0,
+                                    "mp-affinity", NULL);
         }
-        cpuobj = object_new(object_class_get_name(oc));
 
-        object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_HVC, "psci-conduit",
-                                NULL);
+        if (!vms->secure) {
+            object_property_set_bool(cpuobj, false, "has_el3", NULL);
+        }
 
-        /* Secondary CPUs start in PSCI powered-down state */
-        if (n > 0) {
-            object_property_set_bool(cpuobj, true, "start-powered-off", NULL);
+        if (vbi->using_psci) {
+            object_property_set_int(cpuobj, QEMU_PSCI_CONDUIT_HVC,
+                                    "psci-conduit", NULL);
+
+            /* Secondary CPUs start in PSCI powered-down state */
+            if (n > 0) {
+                object_property_set_bool(cpuobj, true,
+                                         "start-powered-off", NULL);
+            }
         }
 
         if (object_property_find(cpuobj, "reset-cbar", NULL)) {
@@ -579,31 +1322,60 @@
                                     "reset-cbar", &error_abort);
         }
 
+        object_property_set_link(cpuobj, OBJECT(sysmem), "memory",
+                                 &error_abort);
+        if (vms->secure) {
+            object_property_set_link(cpuobj, OBJECT(secure_sysmem),
+                                     "secure-memory", &error_abort);
+        }
+
         object_property_set_bool(cpuobj, true, "realized", NULL);
     }
-    fdt_add_timer_nodes(vbi);
+    fdt_add_timer_nodes(vbi, gic_version);
     fdt_add_cpu_nodes(vbi);
     fdt_add_psci_node(vbi);
 
-    memory_region_init_ram(ram, NULL, "mach-virt.ram", machine->ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "mach-virt.ram",
+                                         machine->ram_size);
     memory_region_add_subregion(sysmem, vbi->memmap[VIRT_MEM].base, ram);
 
-    create_flash(vbi);
+    create_flash(vbi, sysmem, secure_sysmem ? secure_sysmem : sysmem);
 
-    create_gic(vbi, pic);
+    create_gic(vbi, pic, gic_version, vms->secure);
 
-    create_uart(vbi, pic);
+    fdt_add_pmu_nodes(vbi, gic_version);
+
+    create_uart(vbi, pic, VIRT_UART, sysmem, serial_hds[0]);
+
+    if (vms->secure) {
+        create_secure_ram(vbi, secure_sysmem);
+        create_uart(vbi, pic, VIRT_SECURE_UART, secure_sysmem, serial_hds[1]);
+    }
 
     create_rtc(vbi, pic);
 
+    create_pcie(vbi, pic, vms->highmem);
+
+    create_gpio(vbi, pic);
+
     /* Create mmio transports, so the user can create virtio backends
      * (which will be automatically plugged in to the transports). If
      * no backend is created the transport will just sit harmlessly idle.
      */
     create_virtio_devices(vbi, pic);
 
+    create_fw_cfg(vbi, &address_space_memory);
+    rom_set_fw(fw_cfg_find());
+
+    guest_info->smp_cpus = smp_cpus;
+    guest_info->fw_cfg = fw_cfg_find();
+    guest_info->memmap = vbi->memmap;
+    guest_info->irqmap = vbi->irqmap;
+    guest_info->use_highmem = vms->highmem;
+    guest_info->gic_version = gic_version;
+    guest_info_state->machine_done.notify = virt_guest_info_machine_done;
+    qemu_add_machine_init_done_notifier(&guest_info_state->machine_done);
+
     vbi->bootinfo.ram_size = machine->ram_size;
     vbi->bootinfo.kernel_filename = machine->kernel_filename;
     vbi->bootinfo.kernel_cmdline = machine->kernel_cmdline;
@@ -612,19 +1384,153 @@
     vbi->bootinfo.board_id = -1;
     vbi->bootinfo.loader_start = vbi->memmap[VIRT_MEM].base;
     vbi->bootinfo.get_dtb = machvirt_dtb;
+    vbi->bootinfo.firmware_loaded = firmware_loaded;
     arm_load_kernel(ARM_CPU(first_cpu), &vbi->bootinfo);
+
+    /*
+     * arm_load_kernel machine init done notifier registration must
+     * happen before the platform_bus_create call. In this latter,
+     * another notifier is registered which adds platform bus nodes.
+     * Notifiers are executed in registration reverse order.
+     */
+    create_platform_bus(vbi, pic);
 }
 
-static QEMUMachine machvirt_a15_machine = {
-    .name = "virt",
-    .desc = "ARM Virtual Machine",
-    .init = machvirt_init,
-    .max_cpus = 8,
+static bool virt_get_secure(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    return vms->secure;
+}
+
+static void virt_set_secure(Object *obj, bool value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    vms->secure = value;
+}
+
+static bool virt_get_highmem(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    return vms->highmem;
+}
+
+static void virt_set_highmem(Object *obj, bool value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    vms->highmem = value;
+}
+
+static char *virt_get_gic_version(Object *obj, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+    const char *val = vms->gic_version == 3 ? "3" : "2";
+
+    return g_strdup(val);
+}
+
+static void virt_set_gic_version(Object *obj, const char *value, Error **errp)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    if (!strcmp(value, "3")) {
+        vms->gic_version = 3;
+    } else if (!strcmp(value, "2")) {
+        vms->gic_version = 2;
+    } else if (!strcmp(value, "host")) {
+        vms->gic_version = 0; /* Will probe later */
+    } else {
+        error_setg(errp, "Invalid gic-version value");
+        error_append_hint(errp, "Valid values are 3, 2, host.\n");
+    }
+}
+
+static void virt_machine_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->init = machvirt_init;
+    /* Start max_cpus at the maximum QEMU supports. We'll further restrict
+     * it later in machvirt_init, where we have more information about the
+     * configuration of the particular instance.
+     */
+    mc->max_cpus = MAX_CPUMASK_BITS;
+    mc->has_dynamic_sysbus = true;
+    mc->block_default_type = IF_VIRTIO;
+    mc->no_cdrom = 1;
+    mc->pci_allow_0_address = true;
+}
+
+static const TypeInfo virt_machine_info = {
+    .name          = TYPE_VIRT_MACHINE,
+    .parent        = TYPE_MACHINE,
+    .abstract      = true,
+    .instance_size = sizeof(VirtMachineState),
+    .class_size    = sizeof(VirtMachineClass),
+    .class_init    = virt_machine_class_init,
 };
 
 static void machvirt_machine_init(void)
 {
-    qemu_register_machine(&machvirt_a15_machine);
+    type_register_static(&virt_machine_info);
+}
+type_init(machvirt_machine_init);
+
+static void virt_2_7_instance_init(Object *obj)
+{
+    VirtMachineState *vms = VIRT_MACHINE(obj);
+
+    /* EL3 is disabled by default on virt: this makes us consistent
+     * between KVM and TCG for this board, and it also allows us to
+     * boot UEFI blobs which assume no TrustZone support.
+     */
+    vms->secure = false;
+    object_property_add_bool(obj, "secure", virt_get_secure,
+                             virt_set_secure, NULL);
+    object_property_set_description(obj, "secure",
+                                    "Set on/off to enable/disable the ARM "
+                                    "Security Extensions (TrustZone)",
+                                    NULL);
+
+    /* High memory is enabled by default */
+    vms->highmem = true;
+    object_property_add_bool(obj, "highmem", virt_get_highmem,
+                             virt_set_highmem, NULL);
+    object_property_set_description(obj, "highmem",
+                                    "Set on/off to enable/disable using "
+                                    "physical address space above 32 bits",
+                                    NULL);
+    /* Default GIC type is v2 */
+    vms->gic_version = 2;
+    object_property_add_str(obj, "gic-version", virt_get_gic_version,
+                        virt_set_gic_version, NULL);
+    object_property_set_description(obj, "gic-version",
+                                    "Set GIC version. "
+                                    "Valid values are 2, 3 and host", NULL);
 }
 
-machine_init(machvirt_machine_init);
+static void virt_machine_2_7_options(MachineClass *mc)
+{
+}
+DEFINE_VIRT_MACHINE_AS_LATEST(2, 7)
+
+#define VIRT_COMPAT_2_6 \
+    HW_COMPAT_2_6
+
+static void virt_2_6_instance_init(Object *obj)
+{
+    virt_2_7_instance_init(obj);
+}
+
+static void virt_machine_2_6_options(MachineClass *mc)
+{
+    VirtMachineClass *vmc = VIRT_MACHINE_CLASS(OBJECT_CLASS(mc));
+
+    virt_machine_2_7_options(mc);
+    SET_MACHINE_COMPAT(mc, VIRT_COMPAT_2_6);
+    vmc->disallow_affinity_adjustment = true;
+}
+DEFINE_VIRT_MACHINE(2, 6)
diff --git a/hw/arm/xilinx_zynq.c b/hw/arm/xilinx_zynq.c
index b590392..7dac20d 100644
--- a/hw/arm/xilinx_zynq.c
+++ b/hw/arm/xilinx_zynq.c
@@ -15,6 +15,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/arm/arm.h"
 #include "net/net.h"
@@ -24,8 +28,11 @@
 #include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
 #include "hw/loader.h"
-#include "hw/ssi.h"
+#include "hw/misc/zynq-xadc.h"
+#include "hw/ssi/ssi.h"
 #include "qemu/error-report.h"
+#include "hw/sd/sd.h"
+#include "hw/char/cadence_uart.h"
 
 #define NUM_SPI_FLASHES 4
 #define NUM_QSPI_FLASHES 2
@@ -43,6 +50,45 @@
     46, 47, 48, 49, 72, 73, 74, 75
 };
 
+#define BOARD_SETUP_ADDR        0x100
+
+#define SLCR_LOCK_OFFSET        0x004
+#define SLCR_UNLOCK_OFFSET      0x008
+#define SLCR_ARM_PLL_OFFSET     0x100
+
+#define SLCR_XILINX_UNLOCK_KEY  0xdf0d
+#define SLCR_XILINX_LOCK_KEY    0x767b
+
+#define ARMV7_IMM16(x) (extract32((x),  0, 12) | \
+                        extract32((x), 12,  4) << 16)
+
+/* Write immediate val to address r0 + addr. r0 should contain base offset
+ * of the SLCR block. Clobbers r1.
+ */
+
+#define SLCR_WRITE(addr, val) \
+    0xe3001000 + ARMV7_IMM16(extract32((val),  0, 16)), /* movw r1 ... */ \
+    0xe3401000 + ARMV7_IMM16(extract32((val), 16, 16)), /* movt r1 ... */ \
+    0xe5801000 + (addr)
+
+static void zynq_write_board_setup(ARMCPU *cpu,
+                                   const struct arm_boot_info *info)
+{
+    int n;
+    uint32_t board_setup_blob[] = {
+        0xe3a004f8, /* mov r0, #0xf8000000 */
+        SLCR_WRITE(SLCR_UNLOCK_OFFSET, SLCR_XILINX_UNLOCK_KEY),
+        SLCR_WRITE(SLCR_ARM_PLL_OFFSET, 0x00014008),
+        SLCR_WRITE(SLCR_LOCK_OFFSET, SLCR_XILINX_LOCK_KEY),
+        0xe12fff1e, /* bx lr */
+    };
+    for (n = 0; n < ARRAY_SIZE(board_setup_blob); n++) {
+        board_setup_blob[n] = tswap32(board_setup_blob[n]);
+    }
+    rom_add_blob_fixed("board-setup", board_setup_blob,
+                       sizeof(board_setup_blob), BOARD_SETUP_ADDR);
+}
+
 static struct arm_boot_info zynq_binfo = {};
 
 static void gem_init(NICInfo *nd, uint32_t base, qemu_irq irq)
@@ -92,7 +138,13 @@
         spi = (SSIBus *)qdev_get_child_bus(dev, bus_name);
 
         for (j = 0; j < num_ss; ++j) {
-            flash_dev = ssi_create_slave(spi, "n25q128");
+            DriveInfo *dinfo = drive_get_next(IF_MTD);
+            flash_dev = ssi_create_slave_no_init(spi, "n25q128");
+            if (dinfo) {
+                qdev_prop_set_drive(flash_dev, "drive",
+                                    blk_by_legacy_dinfo(dinfo), &error_fatal);
+            }
+            qdev_init_nofail(flash_dev);
 
             cs_line = qdev_get_gpio_in_named(flash_dev, SSI_GPIO_CS, 0);
             sysbus_connect_irq(busdev, i * num_ss + j + 1, cs_line);
@@ -113,10 +165,11 @@
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ext_ram = g_new(MemoryRegion, 1);
     MemoryRegion *ocm_ram = g_new(MemoryRegion, 1);
-    DeviceState *dev;
+    DeviceState *dev, *carddev;
     SysBusDevice *busdev;
+    DriveInfo *di;
+    BlockBackend *blk;
     qemu_irq pic[64];
-    Error *err = NULL;
     int n;
 
     if (!cpu_model) {
@@ -126,22 +179,19 @@
 
     cpu = ARM_CPU(object_new(object_class_get_name(cpu_oc)));
 
-    object_property_set_int(OBJECT(cpu), ZYNQ_BOARD_MIDR, "midr", &err);
-    if (err) {
-        error_report("%s", error_get_pretty(err));
-        exit(1);
+    /* By default A9 CPUs have EL3 enabled.  This board does not
+     * currently support EL3 so the CPU EL3 property is disabled before
+     * realization.
+     */
+    if (object_property_find(OBJECT(cpu), "has_el3", NULL)) {
+        object_property_set_bool(OBJECT(cpu), false, "has_el3", &error_fatal);
     }
 
-    object_property_set_int(OBJECT(cpu), MPCORE_PERIPHBASE, "reset-cbar", &err);
-    if (err) {
-        error_report("%s", error_get_pretty(err));
-        exit(1);
-    }
-    object_property_set_bool(OBJECT(cpu), true, "realized", &err);
-    if (err) {
-        error_report("%s", error_get_pretty(err));
-        exit(1);
-    }
+    object_property_set_int(OBJECT(cpu), ZYNQ_BOARD_MIDR, "midr",
+                            &error_fatal);
+    object_property_set_int(OBJECT(cpu), MPCORE_PERIPHBASE, "reset-cbar",
+                            &error_fatal);
+    object_property_set_bool(OBJECT(cpu), true, "realized", &error_fatal);
 
     /* max 2GB ram */
     if (ram_size > 0x80000000) {
@@ -149,14 +199,13 @@
     }
 
     /* DDR remapped to address zero.  */
-    memory_region_init_ram(ext_ram, NULL, "zynq.ext_ram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ext_ram);
+    memory_region_allocate_system_memory(ext_ram, NULL, "zynq.ext_ram",
+                                         ram_size);
     memory_region_add_subregion(address_space_mem, 0, ext_ram);
 
     /* 256K of on-chip memory */
     memory_region_init_ram(ocm_ram, NULL, "zynq.ocm_ram", 256 << 10,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ocm_ram);
     memory_region_add_subregion(address_space_mem, 0xFFFC0000, ocm_ram);
 
@@ -193,8 +242,8 @@
     sysbus_create_simple("xlnx,ps7-usb", 0xE0002000, pic[53-IRQ_OFFSET]);
     sysbus_create_simple("xlnx,ps7-usb", 0xE0003000, pic[76-IRQ_OFFSET]);
 
-    sysbus_create_simple("cadence_uart", 0xE0000000, pic[59-IRQ_OFFSET]);
-    sysbus_create_simple("cadence_uart", 0xE0001000, pic[82-IRQ_OFFSET]);
+    cadence_uart_create(0xE0000000, pic[59 - IRQ_OFFSET], serial_hds[0]);
+    cadence_uart_create(0xE0001000, pic[82 - IRQ_OFFSET], serial_hds[1]);
 
     sysbus_create_varargs("cadence_ttc", 0xF8001000,
             pic[42-IRQ_OFFSET], pic[43-IRQ_OFFSET], pic[44-IRQ_OFFSET], NULL);
@@ -209,11 +258,28 @@
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xE0100000);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[56-IRQ_OFFSET]);
 
+    di = drive_get_next(IF_SD);
+    blk = di ? blk_by_legacy_dinfo(di) : NULL;
+    carddev = qdev_create(qdev_get_child_bus(dev, "sd-bus"), TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &error_fatal);
+
     dev = qdev_create(NULL, "generic-sdhci");
     qdev_init_nofail(dev);
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xE0101000);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[79-IRQ_OFFSET]);
 
+    di = drive_get_next(IF_SD);
+    blk = di ? blk_by_legacy_dinfo(di) : NULL;
+    carddev = qdev_create(qdev_get_child_bus(dev, "sd-bus"), TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &error_fatal);
+
+    dev = qdev_create(NULL, TYPE_ZYNQ_XADC);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, 0xF8007100);
+    sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, pic[39-IRQ_OFFSET]);
+
     dev = qdev_create(NULL, "pl330");
     qdev_prop_set_uint8(dev, "num_chnls",  8);
     qdev_prop_set_uint8(dev, "num_periph_req",  4);
@@ -234,6 +300,12 @@
         sysbus_connect_irq(busdev, n + 1, pic[dma_irqs[n] - IRQ_OFFSET]);
     }
 
+    dev = qdev_create(NULL, "xlnx.ps7-dev-cfg");
+    qdev_init_nofail(dev);
+    busdev = SYS_BUS_DEVICE(dev);
+    sysbus_connect_irq(busdev, 0, pic[40 - IRQ_OFFSET]);
+    sysbus_mmio_map(busdev, 0, 0xF8007000);
+
     zynq_binfo.ram_size = ram_size;
     zynq_binfo.kernel_filename = kernel_filename;
     zynq_binfo.kernel_cmdline = kernel_cmdline;
@@ -241,21 +313,19 @@
     zynq_binfo.nb_cpus = 1;
     zynq_binfo.board_id = 0xd32;
     zynq_binfo.loader_start = 0;
+    zynq_binfo.board_setup_addr = BOARD_SETUP_ADDR;
+    zynq_binfo.write_board_setup = zynq_write_board_setup;
+
     arm_load_kernel(ARM_CPU(first_cpu), &zynq_binfo);
 }
 
-static QEMUMachine zynq_machine = {
-    .name = "xilinx-zynq-a9",
-    .desc = "Xilinx Zynq Platform Baseboard for Cortex-A9",
-    .init = zynq_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 1,
-    .no_sdcard = 1,
-};
-
-static void zynq_machine_init(void)
+static void zynq_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&zynq_machine);
+    mc->desc = "Xilinx Zynq Platform Baseboard for Cortex-A9";
+    mc->init = zynq_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 1;
+    mc->no_sdcard = 1;
 }
 
-machine_init(zynq_machine_init);
+DEFINE_MACHINE("xilinx-zynq-a9", zynq_machine_init)
diff --git a/hw/arm/xlnx-ep108.c b/hw/arm/xlnx-ep108.c
new file mode 100644
index 0000000..4ec590a
--- /dev/null
+++ b/hw/arm/xlnx-ep108.c
@@ -0,0 +1,131 @@
+/*
+ * Xilinx ZynqMP EP108 board
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/xlnx-zynqmp.h"
+#include "hw/boards.h"
+#include "qemu/error-report.h"
+#include "exec/address-spaces.h"
+#include "qemu/log.h"
+
+typedef struct XlnxEP108 {
+    XlnxZynqMPState soc;
+    MemoryRegion ddr_ram;
+} XlnxEP108;
+
+static struct arm_boot_info xlnx_ep108_binfo;
+
+static void xlnx_ep108_init(MachineState *machine)
+{
+    XlnxEP108 *s = g_new0(XlnxEP108, 1);
+    int i;
+    uint64_t ram_size = machine->ram_size;
+
+    /* Create the memory region to pass to the SoC */
+    if (ram_size > XLNX_ZYNQMP_MAX_RAM_SIZE) {
+        error_report("ERROR: RAM size 0x%" PRIx64 " above max supported of "
+                     "0x%llx", ram_size,
+                     XLNX_ZYNQMP_MAX_RAM_SIZE);
+        exit(1);
+    }
+
+    if (ram_size < 0x08000000) {
+        qemu_log("WARNING: RAM size 0x%" PRIx64 " is small for EP108",
+                 ram_size);
+    }
+
+    memory_region_allocate_system_memory(&s->ddr_ram, NULL, "ddr-ram",
+                                         ram_size);
+
+    object_initialize(&s->soc, sizeof(s->soc), TYPE_XLNX_ZYNQMP);
+    object_property_add_child(OBJECT(machine), "soc", OBJECT(&s->soc),
+                              &error_abort);
+
+    object_property_set_link(OBJECT(&s->soc), OBJECT(&s->ddr_ram),
+                         "ddr-ram", &error_abort);
+
+    object_property_set_bool(OBJECT(&s->soc), true, "realized", &error_fatal);
+
+    /* Create and plug in the SD cards */
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        BusState *bus;
+        DriveInfo *di = drive_get_next(IF_SD);
+        BlockBackend *blk = di ? blk_by_legacy_dinfo(di) : NULL;
+        DeviceState *carddev;
+        char *bus_name;
+
+        bus_name = g_strdup_printf("sd-bus%d", i);
+        bus = qdev_get_child_bus(DEVICE(&s->soc), bus_name);
+        g_free(bus_name);
+        if (!bus) {
+            error_report("No SD bus found for SD card %d", i);
+            exit(1);
+        }
+        carddev = qdev_create(bus, TYPE_SD_CARD);
+        qdev_prop_set_drive(carddev, "drive", blk, &error_fatal);
+        object_property_set_bool(OBJECT(carddev), true, "realized",
+                                 &error_fatal);
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SPIS; i++) {
+        SSIBus *spi_bus;
+        DeviceState *flash_dev;
+        qemu_irq cs_line;
+        DriveInfo *dinfo = drive_get_next(IF_MTD);
+        gchar *bus_name = g_strdup_printf("spi%d", i);
+
+        spi_bus = (SSIBus *)qdev_get_child_bus(DEVICE(&s->soc), bus_name);
+        g_free(bus_name);
+
+        flash_dev = ssi_create_slave_no_init(spi_bus, "sst25wf080");
+        if (dinfo) {
+            qdev_prop_set_drive(flash_dev, "drive", blk_by_legacy_dinfo(dinfo),
+                                &error_fatal);
+        }
+        qdev_init_nofail(flash_dev);
+
+        cs_line = qdev_get_gpio_in_named(flash_dev, SSI_GPIO_CS, 0);
+
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->soc.spi[i]), 1, cs_line);
+    }
+
+    xlnx_ep108_binfo.ram_size = ram_size;
+    xlnx_ep108_binfo.kernel_filename = machine->kernel_filename;
+    xlnx_ep108_binfo.kernel_cmdline = machine->kernel_cmdline;
+    xlnx_ep108_binfo.initrd_filename = machine->initrd_filename;
+    xlnx_ep108_binfo.loader_start = 0;
+    arm_load_kernel(s->soc.boot_cpu_ptr, &xlnx_ep108_binfo);
+}
+
+static void xlnx_ep108_machine_init(MachineClass *mc)
+{
+    mc->desc = "Xilinx ZynqMP EP108 board";
+    mc->init = xlnx_ep108_init;
+}
+
+DEFINE_MACHINE("xlnx-ep108", xlnx_ep108_machine_init)
+
+static void xlnx_zcu102_machine_init(MachineClass *mc)
+{
+    mc->desc = "Xilinx ZynqMP ZCU102 board";
+    mc->init = xlnx_ep108_init;
+}
+
+DEFINE_MACHINE("xlnx-zcu102", xlnx_zcu102_machine_init)
diff --git a/hw/arm/xlnx-zynqmp.c b/hw/arm/xlnx-zynqmp.c
new file mode 100644
index 0000000..23c7199
--- /dev/null
+++ b/hw/arm/xlnx-zynqmp.c
@@ -0,0 +1,459 @@
+/*
+ * Xilinx Zynq MPSoC emulation
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "hw/arm/xlnx-zynqmp.h"
+#include "hw/intc/arm_gic_common.h"
+#include "exec/address-spaces.h"
+#include "sysemu/kvm.h"
+#include "kvm_arm.h"
+
+#define GIC_NUM_SPI_INTR 160
+
+#define ARM_PHYS_TIMER_PPI  30
+#define ARM_VIRT_TIMER_PPI  27
+
+#define GIC_BASE_ADDR       0xf9000000
+#define GIC_DIST_ADDR       0xf9010000
+#define GIC_CPU_ADDR        0xf9020000
+
+#define SATA_INTR           133
+#define SATA_ADDR           0xFD0C0000
+#define SATA_NUM_PORTS      2
+
+#define DP_ADDR             0xfd4a0000
+#define DP_IRQ              113
+
+#define DPDMA_ADDR          0xfd4c0000
+#define DPDMA_IRQ           116
+
+static const uint64_t gem_addr[XLNX_ZYNQMP_NUM_GEMS] = {
+    0xFF0B0000, 0xFF0C0000, 0xFF0D0000, 0xFF0E0000,
+};
+
+static const int gem_intr[XLNX_ZYNQMP_NUM_GEMS] = {
+    57, 59, 61, 63,
+};
+
+static const uint64_t uart_addr[XLNX_ZYNQMP_NUM_UARTS] = {
+    0xFF000000, 0xFF010000,
+};
+
+static const int uart_intr[XLNX_ZYNQMP_NUM_UARTS] = {
+    21, 22,
+};
+
+static const uint64_t sdhci_addr[XLNX_ZYNQMP_NUM_SDHCI] = {
+    0xFF160000, 0xFF170000,
+};
+
+static const int sdhci_intr[XLNX_ZYNQMP_NUM_SDHCI] = {
+    48, 49,
+};
+
+static const uint64_t spi_addr[XLNX_ZYNQMP_NUM_SPIS] = {
+    0xFF040000, 0xFF050000,
+};
+
+static const int spi_intr[XLNX_ZYNQMP_NUM_SPIS] = {
+    19, 20,
+};
+
+typedef struct XlnxZynqMPGICRegion {
+    int region_index;
+    uint32_t address;
+} XlnxZynqMPGICRegion;
+
+static const XlnxZynqMPGICRegion xlnx_zynqmp_gic_regions[] = {
+    { .region_index = 0, .address = GIC_DIST_ADDR, },
+    { .region_index = 1, .address = GIC_CPU_ADDR,  },
+};
+
+static inline int arm_gic_ppi_index(int cpu_nr, int ppi_index)
+{
+    return GIC_NUM_SPI_INTR + cpu_nr * GIC_INTERNAL + ppi_index;
+}
+
+static void xlnx_zynqmp_create_rpu(XlnxZynqMPState *s, const char *boot_cpu,
+                                   Error **errp)
+{
+    Error *err = NULL;
+    int i;
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_RPU_CPUS; i++) {
+        char *name;
+
+        object_initialize(&s->rpu_cpu[i], sizeof(s->rpu_cpu[i]),
+                          "cortex-r5-" TYPE_ARM_CPU);
+        object_property_add_child(OBJECT(s), "rpu-cpu[*]",
+                                  OBJECT(&s->rpu_cpu[i]), &error_abort);
+
+        name = object_get_canonical_path_component(OBJECT(&s->rpu_cpu[i]));
+        if (strcmp(name, boot_cpu)) {
+            /* Secondary CPUs start in PSCI powered-down state */
+            object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true,
+                                     "start-powered-off", &error_abort);
+        } else {
+            s->boot_cpu_ptr = &s->rpu_cpu[i];
+        }
+        g_free(name);
+
+        object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true, "reset-hivecs",
+                                 &error_abort);
+        object_property_set_bool(OBJECT(&s->rpu_cpu[i]), true, "realized",
+                                 &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    }
+}
+
+static void xlnx_zynqmp_init(Object *obj)
+{
+    XlnxZynqMPState *s = XLNX_ZYNQMP(obj);
+    int i;
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_APU_CPUS; i++) {
+        object_initialize(&s->apu_cpu[i], sizeof(s->apu_cpu[i]),
+                          "cortex-a53-" TYPE_ARM_CPU);
+        object_property_add_child(obj, "apu-cpu[*]", OBJECT(&s->apu_cpu[i]),
+                                  &error_abort);
+    }
+
+    object_property_add_link(obj, "ddr-ram", TYPE_MEMORY_REGION,
+                             (Object **)&s->ddr_ram,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
+
+    object_initialize(&s->gic, sizeof(s->gic), gic_class_name());
+    qdev_set_parent_bus(DEVICE(&s->gic), sysbus_get_default());
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_GEMS; i++) {
+        object_initialize(&s->gem[i], sizeof(s->gem[i]), TYPE_CADENCE_GEM);
+        qdev_set_parent_bus(DEVICE(&s->gem[i]), sysbus_get_default());
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_UARTS; i++) {
+        object_initialize(&s->uart[i], sizeof(s->uart[i]), TYPE_CADENCE_UART);
+        qdev_set_parent_bus(DEVICE(&s->uart[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->sata, sizeof(s->sata), TYPE_SYSBUS_AHCI);
+    qdev_set_parent_bus(DEVICE(&s->sata), sysbus_get_default());
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        object_initialize(&s->sdhci[i], sizeof(s->sdhci[i]),
+                          TYPE_SYSBUS_SDHCI);
+        qdev_set_parent_bus(DEVICE(&s->sdhci[i]),
+                            sysbus_get_default());
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SPIS; i++) {
+        object_initialize(&s->spi[i], sizeof(s->spi[i]),
+                          TYPE_XILINX_SPIPS);
+        qdev_set_parent_bus(DEVICE(&s->spi[i]), sysbus_get_default());
+    }
+
+    object_initialize(&s->dp, sizeof(s->dp), TYPE_XLNX_DP);
+    qdev_set_parent_bus(DEVICE(&s->dp), sysbus_get_default());
+
+    object_initialize(&s->dpdma, sizeof(s->dpdma), TYPE_XLNX_DPDMA);
+    qdev_set_parent_bus(DEVICE(&s->dpdma), sysbus_get_default());
+}
+
+static void xlnx_zynqmp_realize(DeviceState *dev, Error **errp)
+{
+    XlnxZynqMPState *s = XLNX_ZYNQMP(dev);
+    MemoryRegion *system_memory = get_system_memory();
+    uint8_t i;
+    uint64_t ram_size;
+    const char *boot_cpu = s->boot_cpu ? s->boot_cpu : "apu-cpu[0]";
+    ram_addr_t ddr_low_size, ddr_high_size;
+    qemu_irq gic_spi[GIC_NUM_SPI_INTR];
+    Error *err = NULL;
+
+    ram_size = memory_region_size(s->ddr_ram);
+
+    /* Create the DDR Memory Regions. User friendly checks should happen at
+     * the board level
+     */
+    if (ram_size > XLNX_ZYNQMP_MAX_LOW_RAM_SIZE) {
+        /* The RAM size is above the maximum available for the low DDR.
+         * Create the high DDR memory region as well.
+         */
+        assert(ram_size <= XLNX_ZYNQMP_MAX_RAM_SIZE);
+        ddr_low_size = XLNX_ZYNQMP_MAX_LOW_RAM_SIZE;
+        ddr_high_size = ram_size - XLNX_ZYNQMP_MAX_LOW_RAM_SIZE;
+
+        memory_region_init_alias(&s->ddr_ram_high, NULL,
+                                 "ddr-ram-high", s->ddr_ram,
+                                  ddr_low_size, ddr_high_size);
+        memory_region_add_subregion(get_system_memory(),
+                                    XLNX_ZYNQMP_HIGH_RAM_START,
+                                    &s->ddr_ram_high);
+    } else {
+        /* RAM must be non-zero */
+        assert(ram_size);
+        ddr_low_size = ram_size;
+    }
+
+    memory_region_init_alias(&s->ddr_ram_low, NULL,
+                             "ddr-ram-low", s->ddr_ram,
+                              0, ddr_low_size);
+    memory_region_add_subregion(get_system_memory(), 0, &s->ddr_ram_low);
+
+    /* Create the four OCM banks */
+    for (i = 0; i < XLNX_ZYNQMP_NUM_OCM_BANKS; i++) {
+        char *ocm_name = g_strdup_printf("zynqmp.ocm_ram_bank_%d", i);
+
+        memory_region_init_ram(&s->ocm_ram[i], NULL, ocm_name,
+                               XLNX_ZYNQMP_OCM_RAM_SIZE, &error_fatal);
+        vmstate_register_ram_global(&s->ocm_ram[i]);
+        memory_region_add_subregion(get_system_memory(),
+                                    XLNX_ZYNQMP_OCM_RAM_0_ADDRESS +
+                                        i * XLNX_ZYNQMP_OCM_RAM_SIZE,
+                                    &s->ocm_ram[i]);
+
+        g_free(ocm_name);
+    }
+
+    qdev_prop_set_uint32(DEVICE(&s->gic), "num-irq", GIC_NUM_SPI_INTR + 32);
+    qdev_prop_set_uint32(DEVICE(&s->gic), "revision", 2);
+    qdev_prop_set_uint32(DEVICE(&s->gic), "num-cpu", XLNX_ZYNQMP_NUM_APU_CPUS);
+
+    /* Realize APUs before realizing the GIC. KVM requires this.  */
+    for (i = 0; i < XLNX_ZYNQMP_NUM_APU_CPUS; i++) {
+        char *name;
+
+        object_property_set_int(OBJECT(&s->apu_cpu[i]), QEMU_PSCI_CONDUIT_SMC,
+                                "psci-conduit", &error_abort);
+
+        name = object_get_canonical_path_component(OBJECT(&s->apu_cpu[i]));
+        if (strcmp(name, boot_cpu)) {
+            /* Secondary CPUs start in PSCI powered-down state */
+            object_property_set_bool(OBJECT(&s->apu_cpu[i]), true,
+                                     "start-powered-off", &error_abort);
+        } else {
+            s->boot_cpu_ptr = &s->apu_cpu[i];
+        }
+        g_free(name);
+
+        object_property_set_bool(OBJECT(&s->apu_cpu[i]),
+                                 s->secure, "has_el3", NULL);
+        object_property_set_int(OBJECT(&s->apu_cpu[i]), GIC_BASE_ADDR,
+                                "reset-cbar", &error_abort);
+        object_property_set_bool(OBJECT(&s->apu_cpu[i]), true, "realized",
+                                 &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    }
+
+    object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    assert(ARRAY_SIZE(xlnx_zynqmp_gic_regions) == XLNX_ZYNQMP_GIC_REGIONS);
+    for (i = 0; i < XLNX_ZYNQMP_GIC_REGIONS; i++) {
+        SysBusDevice *gic = SYS_BUS_DEVICE(&s->gic);
+        const XlnxZynqMPGICRegion *r = &xlnx_zynqmp_gic_regions[i];
+        MemoryRegion *mr = sysbus_mmio_get_region(gic, r->region_index);
+        uint32_t addr = r->address;
+        int j;
+
+        sysbus_mmio_map(gic, r->region_index, addr);
+
+        for (j = 0; j < XLNX_ZYNQMP_GIC_ALIASES; j++) {
+            MemoryRegion *alias = &s->gic_mr[i][j];
+
+            addr += XLNX_ZYNQMP_GIC_REGION_SIZE;
+            memory_region_init_alias(alias, OBJECT(s), "zynqmp-gic-alias", mr,
+                                     0, XLNX_ZYNQMP_GIC_REGION_SIZE);
+            memory_region_add_subregion(system_memory, addr, alias);
+        }
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_APU_CPUS; i++) {
+        qemu_irq irq;
+
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gic), i,
+                           qdev_get_gpio_in(DEVICE(&s->apu_cpu[i]),
+                                            ARM_CPU_IRQ));
+        irq = qdev_get_gpio_in(DEVICE(&s->gic),
+                               arm_gic_ppi_index(i, ARM_PHYS_TIMER_PPI));
+        qdev_connect_gpio_out(DEVICE(&s->apu_cpu[i]), 0, irq);
+        irq = qdev_get_gpio_in(DEVICE(&s->gic),
+                               arm_gic_ppi_index(i, ARM_VIRT_TIMER_PPI));
+        qdev_connect_gpio_out(DEVICE(&s->apu_cpu[i]), 1, irq);
+    }
+
+    if (s->has_rpu) {
+        xlnx_zynqmp_create_rpu(s, boot_cpu, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    }
+
+    if (!s->boot_cpu_ptr) {
+        error_setg(errp, "ZynqMP Boot cpu %s not found", boot_cpu);
+        return;
+    }
+
+    for (i = 0; i < GIC_NUM_SPI_INTR; i++) {
+        gic_spi[i] = qdev_get_gpio_in(DEVICE(&s->gic), i);
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_GEMS; i++) {
+        NICInfo *nd = &nd_table[i];
+
+        if (nd->used) {
+            qemu_check_nic_model(nd, TYPE_CADENCE_GEM);
+            qdev_set_nic_properties(DEVICE(&s->gem[i]), nd);
+        }
+        object_property_set_bool(OBJECT(&s->gem[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->gem[i]), 0, gem_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->gem[i]), 0,
+                           gic_spi[gem_intr[i]]);
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_UARTS; i++) {
+        qdev_prop_set_chr(DEVICE(&s->uart[i]), "chardev", serial_hds[i]);
+        object_property_set_bool(OBJECT(&s->uart[i]), true, "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->uart[i]), 0, uart_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->uart[i]), 0,
+                           gic_spi[uart_intr[i]]);
+    }
+
+    object_property_set_int(OBJECT(&s->sata), SATA_NUM_PORTS, "num-ports",
+                            &error_abort);
+    object_property_set_bool(OBJECT(&s->sata), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->sata), 0, SATA_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->sata), 0, gic_spi[SATA_INTR]);
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SDHCI; i++) {
+        char *bus_name;
+
+        object_property_set_bool(OBJECT(&s->sdhci[i]), true,
+                                 "realized", &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->sdhci[i]), 0,
+                        sdhci_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->sdhci[i]), 0,
+                           gic_spi[sdhci_intr[i]]);
+        /* Alias controller SD bus to the SoC itself */
+        bus_name = g_strdup_printf("sd-bus%d", i);
+        object_property_add_alias(OBJECT(s), bus_name,
+                                  OBJECT(&s->sdhci[i]), "sd-bus",
+                                  &error_abort);
+        g_free(bus_name);
+    }
+
+    for (i = 0; i < XLNX_ZYNQMP_NUM_SPIS; i++) {
+        gchar *bus_name;
+
+        object_property_set_bool(OBJECT(&s->spi[i]), true, "realized", &err);
+
+        sysbus_mmio_map(SYS_BUS_DEVICE(&s->spi[i]), 0, spi_addr[i]);
+        sysbus_connect_irq(SYS_BUS_DEVICE(&s->spi[i]), 0,
+                           gic_spi[spi_intr[i]]);
+
+        /* Alias controller SPI bus to the SoC itself */
+        bus_name = g_strdup_printf("spi%d", i);
+        object_property_add_alias(OBJECT(s), bus_name,
+                                  OBJECT(&s->spi[i]), "spi0",
+                                  &error_abort);
+        g_free(bus_name);
+    }
+
+    object_property_set_bool(OBJECT(&s->dp), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->dp), 0, DP_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->dp), 0, gic_spi[DP_IRQ]);
+
+    object_property_set_bool(OBJECT(&s->dpdma), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    object_property_set_link(OBJECT(&s->dp), OBJECT(&s->dpdma), "dpdma",
+                             &error_abort);
+    sysbus_mmio_map(SYS_BUS_DEVICE(&s->dpdma), 0, DPDMA_ADDR);
+    sysbus_connect_irq(SYS_BUS_DEVICE(&s->dpdma), 0, gic_spi[DPDMA_IRQ]);
+}
+
+static Property xlnx_zynqmp_props[] = {
+    DEFINE_PROP_STRING("boot-cpu", XlnxZynqMPState, boot_cpu),
+    DEFINE_PROP_BOOL("secure", XlnxZynqMPState, secure, false),
+    DEFINE_PROP_BOOL("has_rpu", XlnxZynqMPState, has_rpu, false),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void xlnx_zynqmp_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->props = xlnx_zynqmp_props;
+    dc->realize = xlnx_zynqmp_realize;
+
+    /*
+     * Reason: creates an ARM CPU, thus use after free(), see
+     * arm_cpu_class_init()
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo xlnx_zynqmp_type_info = {
+    .name = TYPE_XLNX_ZYNQMP,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(XlnxZynqMPState),
+    .instance_init = xlnx_zynqmp_init,
+    .class_init = xlnx_zynqmp_class_init,
+};
+
+static void xlnx_zynqmp_register_types(void)
+{
+    type_register_static(&xlnx_zynqmp_type_info);
+}
+
+type_init(xlnx_zynqmp_register_types)
diff --git a/hw/arm/z2.c b/hw/arm/z2.c
index 1735547..68a92f3 100644
--- a/hw/arm/z2.c
+++ b/hw/arm/z2.c
@@ -11,12 +11,13 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/arm/arm.h"
 #include "hw/devices.h"
 #include "hw/i2c/i2c.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "hw/boards.h"
 #include "sysemu/sysemu.h"
 #include "hw/block/flash.h"
@@ -150,14 +151,12 @@
     z2_lcd->selected = !level;
 }
 
-static int zipit_lcd_init(SSISlave *dev)
+static void zipit_lcd_realize(SSISlave *dev, Error **errp)
 {
     ZipitLCD *z = FROM_SSI_SLAVE(ZipitLCD, dev);
     z->selected = 0;
     z->enabled = 0;
     z->pos = 0;
-
-    return 0;
 }
 
 static VMStateDescription vmstate_zipit_lcd_state = {
@@ -180,7 +179,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = zipit_lcd_init;
+    k->realize = zipit_lcd_realize;
     k->transfer = zipit_lcd_transfer;
     dc->vmsd = &vmstate_zipit_lcd_state;
 }
@@ -372,15 +371,10 @@
     arm_load_kernel(mpu->cpu, &z2_binfo);
 }
 
-static QEMUMachine z2_machine = {
-    .name = "z2",
-    .desc = "Zipit Z2 (PXA27x)",
-    .init = z2_init,
-};
-
-static void z2_machine_init(void)
+static void z2_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&z2_machine);
+    mc->desc = "Zipit Z2 (PXA27x)";
+    mc->init = z2_init;
 }
 
-machine_init(z2_machine_init);
+DEFINE_MACHINE("z2", z2_machine_init)
diff --git a/hw/audio/ac97.c b/hw/audio/ac97.c
index 111ec0e..cbd959e 100644
--- a/hw/audio/ac97.c
+++ b/hw/audio/ac97.c
@@ -17,6 +17,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -1337,7 +1338,7 @@
     mixer_reset (s);
 }
 
-static int ac97_initfn (PCIDevice *dev)
+static void ac97_realize(PCIDevice *dev, Error **errp)
 {
     AC97LinkState *s = DO_UPCAST (AC97LinkState, dev, dev);
     uint8_t *c = s->dev.config;
@@ -1384,7 +1385,6 @@
     pci_register_bar (&s->dev, 1, PCI_BASE_ADDRESS_SPACE_IO, &s->io_nabm);
     AUD_register_card ("ac97", &s->card);
     ac97_on_reset (&s->dev.qdev);
-    return 0;
 }
 
 static int ac97_init (PCIBus *bus)
@@ -1403,7 +1403,7 @@
     DeviceClass *dc = DEVICE_CLASS (klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS (klass);
 
-    k->init = ac97_initfn;
+    k->realize = ac97_realize;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82801AA_5;
     k->revision = 0x01;
diff --git a/hw/audio/adlib.c b/hw/audio/adlib.c
index 656eb37..7836446 100644
--- a/hw/audio/adlib.c
+++ b/hw/audio/adlib.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -57,11 +59,6 @@
 #define SHIFT 1
 #endif
 
-#define IO_READ_PROTO(name) \
-    uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name) \
-    void name (void *opaque, uint32_t nport, uint32_t val)
-
 #define TYPE_ADLIB "adlib"
 #define ADLIB(obj) OBJECT_CHECK(AdlibState, (obj), TYPE_ADLIB)
 
@@ -124,7 +121,7 @@
     }
 }
 
-static IO_WRITE_PROTO (adlib_write)
+static void adlib_write(void *opaque, uint32_t nport, uint32_t val)
 {
     AdlibState *s = opaque;
     int a = nport & 3;
@@ -141,7 +138,7 @@
 #endif
 }
 
-static IO_READ_PROTO (adlib_read)
+static uint32_t adlib_read(void *opaque, uint32_t nport)
 {
     AdlibState *s = opaque;
     uint8_t data;
@@ -173,7 +170,7 @@
 
     s->ticking[n] = 1;
 #ifdef DEBUG
-    interval = get_ticks_per_sec () * interval_Sec;
+    interval = NANOSECONDS_PER_SECOND * interval_Sec;
     exp = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + interval;
     s->exp[n] = exp;
 #endif
diff --git a/hw/audio/cs4231.c b/hw/audio/cs4231.c
index 6325a8c..30690f9 100644
--- a/hw/audio/cs4231.c
+++ b/hw/audio/cs4231.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 
@@ -144,16 +145,15 @@
     }
 };
 
-static int cs4231_init1(SysBusDevice *dev)
+static void cs4231_init(Object *obj)
 {
-    CSState *s = CS4231(dev);
+    CSState *s = CS4231(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &cs_mem_ops, s, "cs4321",
+    memory_region_init_io(&s->iomem, obj, &cs_mem_ops, s, "cs4321",
                           CS_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
     sysbus_init_irq(dev, &s->irq);
-
-    return 0;
 }
 
 static Property cs4231_properties[] = {
@@ -163,9 +163,7 @@
 static void cs4231_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = cs4231_init1;
     dc->reset = cs_reset;
     dc->vmsd = &vmstate_cs4231;
     dc->props = cs4231_properties;
@@ -175,6 +173,7 @@
     .name          = TYPE_CS4231,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(CSState),
+    .instance_init = cs4231_init,
     .class_init    = cs4231_class_init,
 };
 
diff --git a/hw/audio/cs4231a.c b/hw/audio/cs4231a.c
index f96f561..3ecd058 100644
--- a/hw/audio/cs4231a.c
+++ b/hw/audio/cs4231a.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -69,6 +70,7 @@
     uint32_t irq;
     uint32_t dma;
     uint32_t port;
+    IsaDma *isa_dma;
     int shift;
     int dma_running;
     int audio_free;
@@ -264,6 +266,7 @@
 {
     int xtal;
     struct audsettings as;
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
 
 #ifdef DEBUG_XLAW
     if (val == 0 || val == 32)
@@ -327,7 +330,7 @@
 
     if (s->dregs[Interface_Configuration] & PEN) {
         if (!s->dma_running) {
-            DMA_hold_DREQ (s->dma);
+            k->hold_DREQ(s->isa_dma, s->dma);
             AUD_set_active_out (s->voice, 1);
             s->transferred = 0;
         }
@@ -335,7 +338,7 @@
     }
     else {
         if (s->dma_running) {
-            DMA_release_DREQ (s->dma);
+            k->release_DREQ(s->isa_dma, s->dma);
             AUD_set_active_out (s->voice, 0);
         }
         s->dma_running = 0;
@@ -344,7 +347,7 @@
 
  error:
     if (s->dma_running) {
-        DMA_release_DREQ (s->dma);
+        k->release_DREQ(s->isa_dma, s->dma);
         AUD_set_active_out (s->voice, 0);
     }
 }
@@ -452,7 +455,8 @@
             }
             else {
                 if (s->dma_running) {
-                    DMA_release_DREQ (s->dma);
+                    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
+                    k->release_DREQ(s->isa_dma, s->dma);
                     AUD_set_active_out (s->voice, 0);
                     s->dma_running = 0;
                 }
@@ -517,6 +521,7 @@
 {
     int temp, net;
     uint8_t tmpbuf[4096];
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
 
     temp = len;
     net = 0;
@@ -531,7 +536,7 @@
             to_copy = sizeof (tmpbuf);
         }
 
-        copied = DMA_read_memory (nchan, tmpbuf, dma_pos, to_copy);
+        copied = k->read_memory(s->isa_dma, nchan, tmpbuf, dma_pos, to_copy);
         if (s->tab) {
             int i;
             int16_t linbuf[4096];
@@ -599,7 +604,8 @@
     CSState *s = opaque;
 
     if (s->dma_running) {
-        DMA_release_DREQ (s->dma);
+        IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
+        k->release_DREQ(s->isa_dma, s->dma);
         AUD_set_active_out (s->voice, 0);
     }
     s->dma_running = 0;
@@ -655,13 +661,15 @@
 {
     ISADevice *d = ISA_DEVICE (dev);
     CSState *s = CS4231A (dev);
+    IsaDmaClass *k;
 
     isa_init_irq (d, &s->pic, s->irq);
+    s->isa_dma = isa_get_dma(isa_bus_from_device(d), s->dma);
+    k = ISADMA_GET_CLASS(s->isa_dma);
+    k->register_channel(s->isa_dma, s->dma, cs_dma_read, s);
 
     isa_register_ioport (d, &s->ioports, s->port);
 
-    DMA_register_channel (s->dma, cs_dma_read, s);
-
     AUD_register_card ("cs4231a", &s->card);
 }
 
diff --git a/hw/audio/es1370.c b/hw/audio/es1370.c
index e67d1ea..8449b5f 100644
--- a/hw/audio/es1370.c
+++ b/hw/audio/es1370.c
@@ -26,6 +26,7 @@
 /* #define VERBOSE_ES1370 */
 #define SILENT_ES1370
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -157,11 +158,6 @@
 #define DAC2_CHANNEL 1
 #define ADC_CHANNEL 2
 
-#define IO_READ_PROTO(n) \
-static uint32_t n (void *opaque, uint32_t addr)
-#define IO_WRITE_PROTO(n) \
-static void n (void *opaque, uint32_t addr, uint32_t val)
-
 static void es1370_dac1_callback (void *opaque, int free);
 static void es1370_dac2_callback (void *opaque, int free);
 static void es1370_adc_callback (void *opaque, int avail);
@@ -293,6 +289,10 @@
                        uint32_t *old_freq, uint32_t *new_freq);
 };
 
+#define TYPE_ES1370 "ES1370"
+#define ES1370(obj) \
+    OBJECT_CHECK(ES1370State, (obj), TYPE_ES1370)
+
 static void es1370_dac1_calc_freq (ES1370State *s, uint32_t ctl,
                                    uint32_t *old_freq, uint32_t *new_freq);
 static void es1370_dac2_and_adc_calc_freq (ES1370State *s, uint32_t ctl,
@@ -474,7 +474,7 @@
     return addr;
 }
 
-IO_WRITE_PROTO (es1370_writeb)
+static void es1370_writeb(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     uint32_t shift, mask;
@@ -512,7 +512,7 @@
     }
 }
 
-IO_WRITE_PROTO (es1370_writew)
+static void es1370_writew(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     addr = es1370_fixup (s, addr);
@@ -549,7 +549,7 @@
     }
 }
 
-IO_WRITE_PROTO (es1370_writel)
+static void es1370_writel(void *opaque, uint32_t addr, uint32_t val)
 {
     ES1370State *s = opaque;
     struct chan *d = &s->chan[0];
@@ -615,7 +615,7 @@
     }
 }
 
-IO_READ_PROTO (es1370_readb)
+static uint32_t es1370_readb(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     uint32_t val;
@@ -650,7 +650,7 @@
     return val;
 }
 
-IO_READ_PROTO (es1370_readw)
+static uint32_t es1370_readw(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     struct chan *d = &s->chan[0];
@@ -692,7 +692,7 @@
     return val;
 }
 
-IO_READ_PROTO (es1370_readl)
+static uint32_t es1370_readl(void *opaque, uint32_t addr)
 {
     ES1370State *s = opaque;
     uint32_t val;
@@ -1016,9 +1016,9 @@
     es1370_reset (s);
 }
 
-static int es1370_initfn (PCIDevice *dev)
+static void es1370_realize(PCIDevice *dev, Error **errp)
 {
-    ES1370State *s = DO_UPCAST (ES1370State, dev, dev);
+    ES1370State *s = ES1370(dev);
     uint8_t *c = s->dev.config;
 
     c[PCI_STATUS + 1] = PCI_STATUS_DEVSEL_SLOW >> 8;
@@ -1039,12 +1039,11 @@
 
     AUD_register_card ("es1370", &s->card);
     es1370_reset (s);
-    return 0;
 }
 
 static int es1370_init (PCIBus *bus)
 {
-    pci_create_simple (bus, -1, "ES1370");
+    pci_create_simple (bus, -1, TYPE_ES1370);
     return 0;
 }
 
@@ -1053,7 +1052,7 @@
     DeviceClass *dc = DEVICE_CLASS (klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS (klass);
 
-    k->init = es1370_initfn;
+    k->realize = es1370_realize;
     k->vendor_id = PCI_VENDOR_ID_ENSONIQ;
     k->device_id = PCI_DEVICE_ID_ENSONIQ_ES1370;
     k->class_id = PCI_CLASS_MULTIMEDIA_AUDIO;
@@ -1065,7 +1064,7 @@
 }
 
 static const TypeInfo es1370_info = {
-    .name          = "ES1370",
+    .name          = TYPE_ES1370,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof (ES1370State),
     .class_init    = es1370_class_init,
diff --git a/hw/audio/fmopl.c b/hw/audio/fmopl.c
index adcef2d..731110f 100644
--- a/hw/audio/fmopl.c
+++ b/hw/audio/fmopl.c
@@ -32,10 +32,7 @@
 
 #define HAS_YM3812	1
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 #include <math.h>
 //#include "driver.h"		/* use M.A.M.E. */
 #include "fmopl.h"
@@ -1177,7 +1174,7 @@
 	OPLWriteReg(OPL,0x03,0); /* Timer2 */
 	OPLWriteReg(OPL,0x04,0); /* IRQ mask clear */
 	for(i = 0xff ; i >= 0x20 ; i-- ) OPLWriteReg(OPL,i,0);
-	/* reset OPerator paramater */
+	/* reset operator parameter */
 	for( c = 0 ; c < OPL->max_ch ; c++ )
 	{
 		OPL_CH *CH = &OPL->P_CH[c];
diff --git a/hw/audio/fmopl.h b/hw/audio/fmopl.h
index 24ba5f4..fdda7f9 100644
--- a/hw/audio/fmopl.h
+++ b/hw/audio/fmopl.h
@@ -1,5 +1,5 @@
-#ifndef __FMOPL_H_
-#define __FMOPL_H_
+#ifndef FMOPL_H
+#define FMOPL_H
 
 /* --- select emulation chips --- */
 #define BUILD_YM3812 (HAS_YM3812)
diff --git a/hw/audio/goldfish_audio.c b/hw/audio/goldfish_audio.c
index 521fee9..59dc111 100644
--- a/hw/audio/goldfish_audio.c
+++ b/hw/audio/goldfish_audio.c
@@ -9,6 +9,7 @@
 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ** GNU General Public License for more details.
 */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "hw/sysbus.h"
@@ -57,7 +58,7 @@
 struct goldfish_audio_buff {
     uint64_t  address;
     uint32_t  length;
-    uint8*    data;
+    uint8_t*  data;
     uint32_t  capacity;
     uint32_t  offset;
 };
diff --git a/hw/audio/gus.c b/hw/audio/gus.c
index 4a43ce7..6c02646 100644
--- a/hw/audio/gus.c
+++ b/hw/audio/gus.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -41,11 +43,6 @@
 #define GUS_ENDIANNESS 0
 #endif
 
-#define IO_READ_PROTO(name) \
-    static uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name) \
-    static void name (void *opaque, uint32_t nport, uint32_t val)
-
 #define TYPE_GUS "gus"
 #define GUS(obj) OBJECT_CHECK (GUSState, (obj), TYPE_GUS)
 
@@ -62,36 +59,23 @@
     SWVoiceOut *voice;
     int64_t last_ticks;
     qemu_irq pic;
+    IsaDma *isa_dma;
 } GUSState;
 
-IO_READ_PROTO (gus_readb)
+static uint32_t gus_readb(void *opaque, uint32_t nport)
 {
     GUSState *s = opaque;
 
     return gus_read (&s->emu, nport, 1);
 }
 
-IO_READ_PROTO (gus_readw)
-{
-    GUSState *s = opaque;
-
-    return gus_read (&s->emu, nport, 2);
-}
-
-IO_WRITE_PROTO (gus_writeb)
+static void gus_writeb(void *opaque, uint32_t nport, uint32_t val)
 {
     GUSState *s = opaque;
 
     gus_write (&s->emu, nport, 1, val);
 }
 
-IO_WRITE_PROTO (gus_writew)
-{
-    GUSState *s = opaque;
-
-    gus_write (&s->emu, nport, 2, val);
-}
-
 static int write_audio (GUSState *s, int samples)
 {
     int net = 0;
@@ -160,7 +144,7 @@
     s->left = samples;
 
  reset:
-    gus_irqgen (&s->emu, muldiv64 (net, 1000000, s->freq));
+    gus_irqgen (&s->emu, (uint64_t)net * 1000000 / s->freq);
 }
 
 int GUS_irqrequest (GUSEmuState *emu, int hwirq, int n)
@@ -186,34 +170,36 @@
 #endif
 }
 
-void GUS_dmarequest (GUSEmuState *der)
+void GUS_dmarequest (GUSEmuState *emu)
 {
-    /* GUSState *s = (GUSState *) der; */
+    GUSState *s = emu->opaque;
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
     ldebug ("dma request %d\n", der->gusdma);
-    DMA_hold_DREQ (der->gusdma);
+    k->hold_DREQ(s->isa_dma, s->emu.gusdma);
 }
 
 static int GUS_read_DMA (void *opaque, int nchan, int dma_pos, int dma_len)
 {
     GUSState *s = opaque;
+    IsaDmaClass *k = ISADMA_GET_CLASS(s->isa_dma);
     char tmpbuf[4096];
     int pos = dma_pos, mode, left = dma_len - dma_pos;
 
     ldebug ("read DMA %#x %d\n", dma_pos, dma_len);
-    mode = DMA_get_channel_mode (s->emu.gusdma);
+    mode = k->has_autoinitialization(s->isa_dma, s->emu.gusdma);
     while (left) {
         int to_copy = audio_MIN ((size_t) left, sizeof (tmpbuf));
         int copied;
 
         ldebug ("left=%d to_copy=%d pos=%d\n", left, to_copy, pos);
-        copied = DMA_read_memory (nchan, tmpbuf, pos, to_copy);
+        copied = k->read_memory(s->isa_dma, nchan, tmpbuf, pos, to_copy);
         gus_dma_transferdata (&s->emu, tmpbuf, copied, left == copied);
         left -= copied;
         pos += copied;
     }
 
     if (((mode >> 4) & 1) == 0) {
-        DMA_release_DREQ (s->emu.gusdma);
+        k->release_DREQ(s->isa_dma, s->emu.gusdma);
     }
     return dma_len;
 }
@@ -236,17 +222,13 @@
 
 static const MemoryRegionPortio gus_portio_list1[] = {
     {0x000,  1, 1, .write = gus_writeb },
-    {0x000,  1, 2, .write = gus_writew },
     {0x006, 10, 1, .read = gus_readb, .write = gus_writeb },
-    {0x006, 10, 2, .read = gus_readw, .write = gus_writew },
     {0x100,  8, 1, .read = gus_readb, .write = gus_writeb },
-    {0x100,  8, 2, .read = gus_readw, .write = gus_writew },
     PORTIO_END_OF_LIST (),
 };
 
 static const MemoryRegionPortio gus_portio_list2[] = {
-    {0, 1, 1, .read = gus_readb },
-    {0, 1, 2, .read = gus_readw },
+    {0, 2, 1, .read = gus_readb },
     PORTIO_END_OF_LIST (),
 };
 
@@ -254,6 +236,7 @@
 {
     ISADevice *d = ISA_DEVICE(dev);
     GUSState *s = GUS (dev);
+    IsaDmaClass *k;
     struct audsettings as;
 
     AUD_register_card ("gus", &s->card);
@@ -286,7 +269,9 @@
     isa_register_portio_list (d, (s->port + 0x100) & 0xf00,
                               gus_portio_list2, s, "gus");
 
-    DMA_register_channel (s->emu.gusdma, GUS_read_DMA, s);
+    s->isa_dma = isa_get_dma(isa_bus_from_device(d), s->emu.gusdma);
+    k = ISADMA_GET_CLASS(s->isa_dma);
+    k->register_channel(s->isa_dma, s->emu.gusdma, GUS_read_DMA, s);
     s->emu.himemaddr = s->himem;
     s->emu.gusdatapos = s->emu.himemaddr + 1024 * 1024 + 32;
     s->emu.opaque = s;
diff --git a/hw/audio/gusemu.h b/hw/audio/gusemu.h
index 331bb6f..9aec7bf 100644
--- a/hw/audio/gusemu.h
+++ b/hw/audio/gusemu.h
@@ -34,7 +34,6 @@
  typedef signed char GUSchar;
  typedef signed short GUSsample;
 #else
- #include <stdint.h>
  typedef int8_t GUSchar;
  typedef uint8_t GUSbyte;
  typedef uint16_t GUSword;
@@ -102,4 +101,4 @@
 /* lower values won´t provide any benefit at all, higher values can cause audible timing delays */
 /* note: masked timers are also calculated by this function, thus it might be needed even without any IRQs in use! */
 
-#endif  /* gusemu.h */
+#endif /* GUSEMU_H */
diff --git a/hw/audio/gusemu_hal.c b/hw/audio/gusemu_hal.c
index 6096690..973d6b9 100644
--- a/hw/audio/gusemu_hal.c
+++ b/hw/audio/gusemu_hal.c
@@ -26,6 +26,7 @@
  * TODO: check mixer: see 7.20 of sdk for panning pos (applies to all gus models?)?
  */
 
+#include "qemu/osdep.h"
 #include "gustate.h"
 #include "gusemu.h"
 
diff --git a/hw/audio/gusemu_mixer.c b/hw/audio/gusemu_mixer.c
index 6d8d9ce..701e8fb 100644
--- a/hw/audio/gusemu_mixer.c
+++ b/hw/audio/gusemu_mixer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "gusemu.h"
 #include "gustate.h"
 
diff --git a/hw/audio/gustate.h b/hw/audio/gustate.h
index ece903a..d162971 100644
--- a/hw/audio/gustate.h
+++ b/hw/audio/gustate.h
@@ -129,4 +129,4 @@
 
 #define gusdataend (VSRegsEnd+4)
 
-#endif  /* gustate.h */
+#endif /* GUSTATE_H */
diff --git a/hw/audio/hda-codec.c b/hw/audio/hda-codec.c
index 11fdcba..437eefe 100644
--- a/hw/audio/hda-codec.c
+++ b/hw/audio/hda-codec.c
@@ -17,6 +17,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "intel-hda.h"
diff --git a/hw/audio/intel-hda.c b/hw/audio/intel-hda.c
index 2885231..cd95340 100644
--- a/hw/audio/intel-hda.c
+++ b/hw/audio/intel-hda.c
@@ -17,6 +17,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/msi.h"
@@ -25,6 +26,7 @@
 #include "intel-hda.h"
 #include "intel-hda-defs.h"
 #include "sysemu/dma.h"
+#include "qapi/error.h"
 
 /* --------------------------------------------------------------------- */
 /* hda bus                                                               */
@@ -49,25 +51,28 @@
     bus->xfer = xfer;
 }
 
-static int hda_codec_dev_init(DeviceState *qdev)
+static void hda_codec_dev_realize(DeviceState *qdev, Error **errp)
 {
-    HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, qdev->parent_bus);
-    HDACodecDevice *dev = DO_UPCAST(HDACodecDevice, qdev, qdev);
+    HDACodecBus *bus = HDA_BUS(qdev->parent_bus);
+    HDACodecDevice *dev = HDA_CODEC_DEVICE(qdev);
     HDACodecDeviceClass *cdc = HDA_CODEC_DEVICE_GET_CLASS(dev);
 
     if (dev->cad == -1) {
         dev->cad = bus->next_cad;
     }
     if (dev->cad >= 15) {
-        return -1;
+        error_setg(errp, "HDA audio codec address is full");
+        return;
     }
     bus->next_cad = dev->cad + 1;
-    return cdc->init(dev);
+    if (cdc->init(dev) != 0) {
+        error_setg(errp, "HDA audio init failed");
+    }
 }
 
 static int hda_codec_dev_exit(DeviceState *qdev)
 {
-    HDACodecDevice *dev = DO_UPCAST(HDACodecDevice, qdev, qdev);
+    HDACodecDevice *dev = HDA_CODEC_DEVICE(qdev);
     HDACodecDeviceClass *cdc = HDA_CODEC_DEVICE_GET_CLASS(dev);
 
     if (cdc->exit) {
@@ -83,7 +88,7 @@
 
     QTAILQ_FOREACH(kid, &bus->qbus.children, sibling) {
         DeviceState *qdev = kid->child;
-        cdev = DO_UPCAST(HDACodecDevice, qdev, qdev);
+        cdev = HDA_CODEC_DEVICE(qdev);
         if (cdev->cad == cad) {
             return cdev;
         }
@@ -93,14 +98,14 @@
 
 void hda_codec_response(HDACodecDevice *dev, bool solicited, uint32_t response)
 {
-    HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
+    HDACodecBus *bus = HDA_BUS(dev->qdev.parent_bus);
     bus->response(dev, solicited, response);
 }
 
 bool hda_codec_xfer(HDACodecDevice *dev, uint32_t stnr, bool output,
                     uint8_t *buf, uint32_t len)
 {
-    HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
+    HDACodecBus *bus = HDA_BUS(dev->qdev.parent_bus);
     return bus->xfer(dev, stnr, output, buf, len);
 }
 
@@ -186,7 +191,7 @@
 
     /* properties */
     uint32_t debug;
-    uint32_t msi;
+    OnOffAuto msi;
     bool old_msi_addr;
 };
 
@@ -214,10 +219,7 @@
 
 static hwaddr intel_hda_addr(uint32_t lbase, uint32_t ubase)
 {
-    hwaddr addr;
-
-    addr = ((uint64_t)ubase << 32) | lbase;
-    return addr;
+    return ((uint64_t)ubase << 32) | lbase;
 }
 
 static void intel_hda_update_int_sts(IntelHDAState *d)
@@ -254,7 +256,7 @@
 
 static void intel_hda_update_irq(IntelHDAState *d)
 {
-    int msi = d->msi && msi_enabled(&d->pci);
+    bool msi = msi_enabled(&d->pci);
     int level;
 
     intel_hda_update_int_sts(d);
@@ -336,7 +338,7 @@
 
 static void intel_hda_response(HDACodecDevice *dev, bool solicited, uint32_t response)
 {
-    HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
+    HDACodecBus *bus = HDA_BUS(dev->qdev.parent_bus);
     IntelHDAState *d = container_of(bus, IntelHDAState, codecs);
     hwaddr addr;
     uint32_t wp, ex;
@@ -385,7 +387,7 @@
 static bool intel_hda_xfer(HDACodecDevice *dev, uint32_t stnr, bool output,
                            uint8_t *buf, uint32_t len)
 {
-    HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
+    HDACodecBus *bus = HDA_BUS(dev->qdev.parent_bus);
     IntelHDAState *d = container_of(bus, IntelHDAState, codecs);
     hwaddr addr;
     uint32_t s, copy, left;
@@ -492,7 +494,7 @@
         DeviceState *qdev = kid->child;
         HDACodecDeviceClass *cdc;
 
-        cdev = DO_UPCAST(HDACodecDevice, qdev, qdev);
+        cdev = HDA_CODEC_DEVICE(qdev);
         cdc = HDA_CODEC_DEVICE_GET_CLASS(cdev);
         if (cdc->stream) {
             cdc->stream(cdev, stream, running, output);
@@ -1119,17 +1121,19 @@
     /* reset codecs */
     QTAILQ_FOREACH(kid, &d->codecs.qbus.children, sibling) {
         DeviceState *qdev = kid->child;
-        cdev = DO_UPCAST(HDACodecDevice, qdev, qdev);
+        cdev = HDA_CODEC_DEVICE(qdev);
         device_reset(DEVICE(cdev));
         d->state_sts |= (1 << cdev->cad);
     }
     intel_hda_update_irq(d);
 }
 
-static int intel_hda_init(PCIDevice *pci)
+static void intel_hda_realize(PCIDevice *pci, Error **errp)
 {
     IntelHDAState *d = INTEL_HDA(pci);
     uint8_t *conf = d->pci.config;
+    Error *err = NULL;
+    int ret;
 
     d->name = object_get_typename(OBJECT(d));
 
@@ -1138,17 +1142,30 @@
     /* HDCTL off 0x40 bit 0 selects signaling mode (1-HDA, 0 - Ac97) 18.1.19 */
     conf[0x40] = 0x01;
 
+    if (d->msi != ON_OFF_AUTO_OFF) {
+        ret = msi_init(&d->pci, d->old_msi_addr ? 0x50 : 0x60,
+                       1, true, false, &err);
+        /* Any error other than -ENOTSUP(board's MSI support is broken)
+         * is a programming error */
+        assert(!ret || ret == -ENOTSUP);
+        if (ret && d->msi == ON_OFF_AUTO_ON) {
+            /* Can't satisfy user's explicit msi=on request, fail */
+            error_append_hint(&err, "You have to use msi=auto (default) or "
+                    "msi=off with this machine type.\n");
+            error_propagate(errp, err);
+            return;
+        }
+        assert(!err || d->msi == ON_OFF_AUTO_AUTO);
+        /* With msi=auto, we fall back to MSI off silently */
+        error_free(err);
+    }
+
     memory_region_init_io(&d->mmio, OBJECT(d), &intel_hda_mmio_ops, d,
                           "intel-hda", 0x4000);
     pci_register_bar(&d->pci, 0, 0, &d->mmio);
-    if (d->msi) {
-        msi_init(&d->pci, d->old_msi_addr ? 0x50 : 0x60, 1, true, false);
-    }
 
     hda_codec_bus_init(DEVICE(pci), &d->codecs, sizeof(d->codecs),
                        intel_hda_response, intel_hda_xfer);
-
-    return 0;
 }
 
 static void intel_hda_exit(PCIDevice *pci)
@@ -1235,7 +1252,7 @@
 
 static Property intel_hda_properties[] = {
     DEFINE_PROP_UINT32("debug", IntelHDAState, debug, 0),
-    DEFINE_PROP_UINT32("msi", IntelHDAState, msi, 1),
+    DEFINE_PROP_ON_OFF_AUTO("msi", IntelHDAState, msi, ON_OFF_AUTO_AUTO),
     DEFINE_PROP_BOOL("old_msi_addr", IntelHDAState, old_msi_addr, false),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -1245,7 +1262,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = intel_hda_init;
+    k->realize = intel_hda_realize;
     k->exit = intel_hda_exit;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->class_id = PCI_CLASS_MULTIMEDIA_HD_AUDIO;
@@ -1299,7 +1316,7 @@
 static void hda_codec_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *k = DEVICE_CLASS(klass);
-    k->init = hda_codec_dev_init;
+    k->realize = hda_codec_dev_realize;
     k->exit = hda_codec_dev_exit;
     set_bit(DEVICE_CATEGORY_SOUND, k->categories);
     k->bus_type = TYPE_HDA_BUS;
diff --git a/hw/audio/lm4549.c b/hw/audio/lm4549.c
index 380ef60..a46f2301 100644
--- a/hw/audio/lm4549.c
+++ b/hw/audio/lm4549.c
@@ -13,6 +13,7 @@
  * It supports only one playback voice and no record voice.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "lm4549.h"
@@ -33,7 +34,6 @@
 #endif
 
 #if defined(LM4549_DUMP_DAC_INPUT)
-#include <stdio.h>
 static FILE *fp_dac_input;
 #endif
 
diff --git a/hw/audio/lm4549.h b/hw/audio/lm4549.h
index 812a7a4..74c3ee8 100644
--- a/hw/audio/lm4549.h
+++ b/hw/audio/lm4549.h
@@ -40,4 +40,4 @@
 void lm4549_write(lm4549_state *s, hwaddr offset, uint32_t value);
 uint32_t lm4549_write_samples(lm4549_state *s, uint32_t left, uint32_t right);
 
-#endif /* #ifndef HW_LM4549_H */
+#endif /* HW_LM4549_H */
diff --git a/hw/audio/marvell_88w8618.c b/hw/audio/marvell_88w8618.c
index 8699267..a6ca180 100644
--- a/hw/audio/marvell_88w8618.c
+++ b/hw/audio/marvell_88w8618.c
@@ -9,6 +9,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
diff --git a/hw/audio/milkymist-ac97.c b/hw/audio/milkymist-ac97.c
index 28f55e8..bc8db71 100644
--- a/hw/audio/milkymist-ac97.c
+++ b/hw/audio/milkymist-ac97.c
@@ -18,9 +18,10 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/ac97.pdf
+ *   http://milkymist.walle.cc/socdoc/ac97.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -283,16 +284,26 @@
     return 0;
 }
 
-static int milkymist_ac97_init(SysBusDevice *dev)
+static void milkymist_ac97_init(Object *obj)
 {
-    MilkymistAC97State *s = MILKYMIST_AC97(dev);
+    MilkymistAC97State *s = MILKYMIST_AC97(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    struct audsettings as;
     sysbus_init_irq(dev, &s->crrequest_irq);
     sysbus_init_irq(dev, &s->crreply_irq);
     sysbus_init_irq(dev, &s->dmar_irq);
     sysbus_init_irq(dev, &s->dmaw_irq);
 
+    memory_region_init_io(&s->regs_region, obj, &ac97_mmio_ops, s,
+            "milkymist-ac97", R_MAX * 4);
+    sysbus_init_mmio(dev, &s->regs_region);
+}
+
+static void milkymist_ac97_realize(DeviceState *dev, Error **errp)
+{
+    MilkymistAC97State *s = MILKYMIST_AC97(dev);
+    struct audsettings as;
+
     AUD_register_card("Milkymist AC'97", &s->card);
 
     as.freq = 48000;
@@ -304,12 +315,6 @@
             "mm_ac97.in", s, ac97_in_cb, &as);
     s->voice_out = AUD_open_out(&s->card, s->voice_out,
             "mm_ac97.out", s, ac97_out_cb, &as);
-
-    memory_region_init_io(&s->regs_region, OBJECT(s), &ac97_mmio_ops, s,
-            "milkymist-ac97", R_MAX * 4);
-    sysbus_init_mmio(dev, &s->regs_region);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_milkymist_ac97 = {
@@ -326,9 +331,8 @@
 static void milkymist_ac97_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = milkymist_ac97_init;
+    dc->realize = milkymist_ac97_realize;
     dc->reset = milkymist_ac97_reset;
     dc->vmsd = &vmstate_milkymist_ac97;
 }
@@ -337,6 +341,7 @@
     .name          = TYPE_MILKYMIST_AC97,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistAC97State),
+    .instance_init = milkymist_ac97_init,
     .class_init    = milkymist_ac97_class_init,
 };
 
diff --git a/hw/audio/pcspk.c b/hw/audio/pcspk.c
index 1d81bbe..42a6f48 100644
--- a/hw/audio/pcspk.c
+++ b/hw/audio/pcspk.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -30,11 +31,12 @@
 #include "qemu/timer.h"
 #include "hw/timer/i8254.h"
 #include "hw/audio/pcspk.h"
+#include "qapi/error.h"
 
 #define PCSPK_BUF_LEN 1792
 #define PCSPK_SAMPLE_RATE 32000
 #define PCSPK_MAX_FREQ (PCSPK_SAMPLE_RATE >> 1)
-#define PCSPK_MIN_COUNT ((PIT_FREQ + PCSPK_MAX_FREQ - 1) / PCSPK_MAX_FREQ)
+#define PCSPK_MIN_COUNT DIV_ROUND_UP(PIT_FREQ, PCSPK_MAX_FREQ)
 
 #define PC_SPEAKER(obj) OBJECT_CHECK(PCSpkState, (obj), TYPE_PC_SPEAKER)
 
@@ -167,7 +169,12 @@
 {
     PCSpkState *s = PC_SPEAKER(obj);
 
-    memory_region_init_io(&s->ioport, OBJECT(s), &pcspk_io_ops, s, "elcr", 1);
+    memory_region_init_io(&s->ioport, OBJECT(s), &pcspk_io_ops, s, "pcspk", 1);
+
+    object_property_add_link(obj, "pit", TYPE_PIT_COMMON,
+                             (Object **)&s->pit,
+                             qdev_prop_allow_set_link_before_realize,
+                             0, &error_abort);
 }
 
 static void pcspk_realizefn(DeviceState *dev, Error **errp)
@@ -182,7 +189,6 @@
 
 static Property pcspk_properties[] = {
     DEFINE_PROP_UINT32("iobase", PCSpkState, iobase,  -1),
-    DEFINE_PROP_PTR("pit", PCSpkState, pit),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -193,7 +199,7 @@
     dc->realize = pcspk_realizefn;
     set_bit(DEVICE_CATEGORY_SOUND, dc->categories);
     dc->props = pcspk_properties;
-    /* Reason: pointer property "pit", realize sets global pcspk_state */
+    /* Reason: realize sets global pcspk_state */
     dc->cannot_instantiate_with_device_add_yet = true;
 }
 
diff --git a/hw/audio/pl041.c b/hw/audio/pl041.c
index 19982f2..6e9c104 100644
--- a/hw/audio/pl041.c
+++ b/hw/audio/pl041.c
@@ -20,7 +20,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qemu/log.h"
 
 #include "pl041.h"
 #include "lm4549.h"
diff --git a/hw/audio/pl041.h b/hw/audio/pl041.h
index 427ab6d..515db47 100644
--- a/hw/audio/pl041.h
+++ b/hw/audio/pl041.h
@@ -132,4 +132,4 @@
 #define RXTOFEC3     (1 << 11)
 #define RXTOFEC4     (1 << 12)
 
-#endif /* #ifndef HW_PL041_H */
+#endif /* HW_PL041_H */
diff --git a/hw/audio/sb16.c b/hw/audio/sb16.c
index bda26d0..3a4a57ac 100644
--- a/hw/audio/sb16.c
+++ b/hw/audio/sb16.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/audio/audio.h"
 #include "audio/audio.h"
@@ -40,11 +41,6 @@
 #define ldebug(...)
 #endif
 
-#define IO_READ_PROTO(name)                             \
-    uint32_t name (void *opaque, uint32_t nport)
-#define IO_WRITE_PROTO(name)                                    \
-    void name (void *opaque, uint32_t nport, uint32_t val)
-
 static const char e3[] = "COPYRIGHT (C) CREATIVE TECHNOLOGY LTD, 1992.";
 
 #define TYPE_SB16 "sb16"
@@ -60,6 +56,8 @@
     uint32_t hdma;
     uint32_t port;
     uint32_t ver;
+    IsaDma *isa_dma;
+    IsaDma *isa_hdma;
 
     int in_index;
     int out_data_len;
@@ -170,16 +168,18 @@
 static void control (SB16State *s, int hold)
 {
     int dma = s->use_hdma ? s->hdma : s->dma;
+    IsaDma *isa_dma = s->use_hdma ? s->isa_hdma : s->isa_dma;
+    IsaDmaClass *k = ISADMA_GET_CLASS(isa_dma);
     s->dma_running = hold;
 
     ldebug ("hold %d high %d dma %d\n", hold, s->use_hdma, dma);
 
     if (hold) {
-        DMA_hold_DREQ (dma);
+        k->hold_DREQ(isa_dma, dma);
         AUD_set_active_out (s->voice, 1);
     }
     else {
-        DMA_release_DREQ (dma);
+        k->release_DREQ(isa_dma, dma);
         AUD_set_active_out (s->voice, 0);
     }
 }
@@ -762,8 +762,8 @@
                 freq = s->freq > 0 ? s->freq : 11025;
                 samples = dsp_get_lohi (s) + 1;
                 bytes = samples << s->fmt_stereo << (s->fmt_bits == 16);
-                ticks = muldiv64 (bytes, get_ticks_per_sec (), freq);
-                if (ticks < get_ticks_per_sec () / 1024) {
+                ticks = muldiv64(bytes, NANOSECONDS_PER_SECOND, freq);
+                if (ticks < NANOSECONDS_PER_SECOND / 1024) {
                     qemu_irq_raise (s->pic);
                 }
                 else {
@@ -881,7 +881,7 @@
     legacy_reset (s);
 }
 
-static IO_WRITE_PROTO (dsp_write)
+static void dsp_write(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
     int iport;
@@ -959,7 +959,7 @@
     }
 }
 
-static IO_READ_PROTO (dsp_read)
+static uint32_t dsp_read(void *opaque, uint32_t nport)
 {
     SB16State *s = opaque;
     int iport, retval, ack = 0;
@@ -999,7 +999,7 @@
         retval = (!s->out_data_len || s->highspeed) ? 0 : 0x80;
         if (s->mixer_regs[0x82] & 1) {
             ack = 1;
-            s->mixer_regs[0x82] &= 1;
+            s->mixer_regs[0x82] &= ~1;
             qemu_irq_lower (s->pic);
         }
         break;
@@ -1008,7 +1008,7 @@
         retval = 0xff;
         if (s->mixer_regs[0x82] & 2) {
             ack = 1;
-            s->mixer_regs[0x82] &= 2;
+            s->mixer_regs[0x82] &= ~2;
             qemu_irq_lower (s->pic);
         }
         break;
@@ -1058,14 +1058,14 @@
     }
 }
 
-static IO_WRITE_PROTO (mixer_write_indexb)
+static void mixer_write_indexb(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
     (void) nport;
     s->mixer_nreg = val;
 }
 
-static IO_WRITE_PROTO (mixer_write_datab)
+static void mixer_write_datab(void *opaque, uint32_t nport, uint32_t val)
 {
     SB16State *s = opaque;
 
@@ -1121,13 +1121,7 @@
     s->mixer_regs[s->mixer_nreg] = val;
 }
 
-static IO_WRITE_PROTO (mixer_write_indexw)
-{
-    mixer_write_indexb (opaque, nport, val & 0xff);
-    mixer_write_datab (opaque, nport, (val >> 8) & 0xff);
-}
-
-static IO_READ_PROTO (mixer_read)
+static uint32_t mixer_read(void *opaque, uint32_t nport)
 {
     SB16State *s = opaque;
 
@@ -1147,6 +1141,8 @@
 static int write_audio (SB16State *s, int nchan, int dma_pos,
                         int dma_len, int len)
 {
+    IsaDma *isa_dma = nchan == s->dma ? s->isa_dma : s->isa_hdma;
+    IsaDmaClass *k = ISADMA_GET_CLASS(isa_dma);
     int temp, net;
     uint8_t tmpbuf[4096];
 
@@ -1163,7 +1159,7 @@
             to_copy = sizeof (tmpbuf);
         }
 
-        copied = DMA_read_memory (nchan, tmpbuf, dma_pos, to_copy);
+        copied = k->read_memory(isa_dma, nchan, tmpbuf, dma_pos, to_copy);
         copied = AUD_write (s->voice, tmpbuf, copied);
 
         temp -= copied;
@@ -1345,7 +1341,6 @@
 
 static const MemoryRegionPortio sb16_ioport_list[] = {
     {  4, 1, 1, .write = mixer_write_indexb },
-    {  4, 1, 2, .write = mixer_write_indexw },
     {  5, 1, 1, .read = mixer_read, .write = mixer_write_datab },
     {  6, 1, 1, .read = dsp_read, .write = dsp_write },
     { 10, 1, 1, .read = dsp_read },
@@ -1366,6 +1361,7 @@
 {
     ISADevice *isadev = ISA_DEVICE (dev);
     SB16State *s = SB16 (dev);
+    IsaDmaClass *k;
 
     isa_init_irq (isadev, &s->pic, s->irq);
 
@@ -1384,8 +1380,14 @@
 
     isa_register_portio_list (isadev, s->port, sb16_ioport_list, s, "sb16");
 
-    DMA_register_channel (s->hdma, SB_read_DMA, s);
-    DMA_register_channel (s->dma, SB_read_DMA, s);
+    s->isa_hdma = isa_get_dma(isa_bus_from_device(isadev), s->hdma);
+    k = ISADMA_GET_CLASS(s->isa_hdma);
+    k->register_channel(s->isa_hdma, s->hdma, SB_read_DMA, s);
+
+    s->isa_dma = isa_get_dma(isa_bus_from_device(isadev), s->dma);
+    k = ISADMA_GET_CLASS(s->isa_dma);
+    k->register_channel(s->isa_dma, s->dma, SB_read_DMA, s);
+
     s->can_write = 1;
 
     AUD_register_card ("sb16", &s->card);
diff --git a/hw/audio/trace-events b/hw/audio/trace-events
new file mode 100644
index 0000000..8d99368
--- /dev/null
+++ b/hw/audio/trace-events
@@ -0,0 +1,26 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/audio/cs4231.c
+cs4231_mem_readl_dreg(uint32_t reg, uint32_t ret) "read dreg %d: 0x%02x"
+cs4231_mem_readl_reg(uint32_t reg, uint32_t ret) "read reg %d: 0x%08x"
+cs4231_mem_writel_reg(uint32_t reg, uint32_t old, uint32_t val) "write reg %d: 0x%08x -> 0x%08x"
+cs4231_mem_writel_dreg(uint32_t reg, uint32_t old, uint32_t val) "write dreg %d: 0x%02x -> 0x%02x"
+
+# hw/audio/milkymist-ac97.c
+milkymist_ac97_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_ac97_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_ac97_pulse_irq_crrequest(void) "Pulse IRQ CR request"
+milkymist_ac97_pulse_irq_crreply(void) "Pulse IRQ CR reply"
+milkymist_ac97_pulse_irq_dmaw(void) "Pulse IRQ DMA write"
+milkymist_ac97_pulse_irq_dmar(void) "Pulse IRQ DMA read"
+milkymist_ac97_in_cb(int avail, uint32_t remaining) "avail %d remaining %u"
+milkymist_ac97_in_cb_transferred(int transferred) "transferred %d"
+milkymist_ac97_out_cb(int free, uint32_t remaining) "free %d remaining %u"
+milkymist_ac97_out_cb_transferred(int transferred) "transferred %d"
+
+# hw/audio/goldfish_audio.c
+goldfish_audio_memory_read(const char *regname, uint32_t value) "%s returns %d"
+goldfish_audio_memory_write(const char *regname, uint32_t value) "%s %08x"
+goldfish_audio_buff_recv(int size, int read) "AUD_read (%d) returned %d"
+goldfish_audio_buff_send(int size, int buffer) "sent %5d bytes to audio output (buffer %d)"
+goldfish_audio_buff_full(int available) "AUDIO_INT_READ_BUFFER_FULL available=%d"
diff --git a/hw/audio/wm8750.c b/hw/audio/wm8750.c
index b50b331..0c6500e 100644
--- a/hw/audio/wm8750.c
+++ b/hw/audio/wm8750.c
@@ -7,6 +7,7 @@
  * This file is licensed under GNU GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "audio/audio.h"
diff --git a/hw/block/block.c b/hw/block/block.c
index a625773..8dc9d84 100644
--- a/hw/block/block.c
+++ b/hw/block/block.c
@@ -7,9 +7,11 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/block-backend.h"
 #include "hw/block/block.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 
 void blkconf_serial(BlockConf *conf, char **serial)
@@ -25,6 +27,58 @@
     }
 }
 
+void blkconf_blocksizes(BlockConf *conf)
+{
+    BlockBackend *blk = conf->blk;
+    BlockSizes blocksizes;
+    int backend_ret;
+
+    backend_ret = blk_probe_blocksizes(blk, &blocksizes);
+    /* fill in detected values if they are not defined via qemu command line */
+    if (!conf->physical_block_size) {
+        if (!backend_ret) {
+           conf->physical_block_size = blocksizes.phys;
+        } else {
+            conf->physical_block_size = BDRV_SECTOR_SIZE;
+        }
+    }
+    if (!conf->logical_block_size) {
+        if (!backend_ret) {
+            conf->logical_block_size = blocksizes.log;
+        } else {
+            conf->logical_block_size = BDRV_SECTOR_SIZE;
+        }
+    }
+}
+
+void blkconf_apply_backend_options(BlockConf *conf)
+{
+    BlockBackend *blk = conf->blk;
+    BlockdevOnError rerror, werror;
+    bool wce;
+
+    switch (conf->wce) {
+    case ON_OFF_AUTO_ON:    wce = true; break;
+    case ON_OFF_AUTO_OFF:   wce = false; break;
+    case ON_OFF_AUTO_AUTO:  wce = blk_enable_write_cache(blk); break;
+    default:
+        abort();
+    }
+
+    rerror = conf->rerror;
+    if (rerror == BLOCKDEV_ON_ERROR_AUTO) {
+        rerror = blk_get_on_error(blk, true);
+    }
+
+    werror = conf->werror;
+    if (werror == BLOCKDEV_ON_ERROR_AUTO) {
+        werror = blk_get_on_error(blk, false);
+    }
+
+    blk_set_enable_write_cache(blk, wce);
+    blk_set_on_error(blk, rerror, werror);
+}
+
 void blkconf_geometry(BlockConf *conf, int *ptrans,
                       unsigned cyls_max, unsigned heads_max, unsigned secs_max,
                       Error **errp)
diff --git a/hw/block/cdrom.c b/hw/block/cdrom.c
index 4e1019c..da937fe 100644
--- a/hw/block/cdrom.c
+++ b/hw/block/cdrom.c
@@ -25,6 +25,7 @@
 /* ??? Most of the ATAPI emulation is still in ide.c.  It should be moved
    here.  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/scsi/scsi.h"
 
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index 1222a37..704a763 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -12,11 +12,13 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "trace.h"
 #include "qemu/iov.h"
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
-#include "hw/virtio/dataplane/vring.h"
+#include "hw/virtio/virtio-access.h"
 #include "sysemu/block-backend.h"
 #include "hw/virtio/virtio-blk.h"
 #include "virtio-blk.h"
@@ -25,17 +27,13 @@
 #include "qom/object_interfaces.h"
 
 struct VirtIOBlockDataPlane {
-    bool started;
     bool starting;
     bool stopping;
-    bool disabled;
 
     VirtIOBlkConf *conf;
-
     VirtIODevice *vdev;
-    Vring vring;                    /* virtqueue vring */
-    EventNotifier *guest_notifier;  /* irq */
     QEMUBH *bh;                     /* bh for guest notification */
+    unsigned long *batch_notify_vqs;
 
     /* Note that these EventNotifiers are assigned by value.  This is
      * fine as long as you do not call event_notifier_cleanup on them
@@ -43,97 +41,40 @@
      * use it).
      */
     IOThread *iothread;
-    IOThread internal_iothread_obj;
     AioContext *ctx;
-    EventNotifier host_notifier;    /* doorbell */
-
-    /* Operation blocker on BDS */
-    Error *blocker;
-    void (*saved_complete_request)(struct VirtIOBlockReq *req,
-                                   unsigned char status);
 };
 
 /* Raise an interrupt to signal guest, if necessary */
-static void notify_guest(VirtIOBlockDataPlane *s)
+void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq)
 {
-    if (!vring_should_notify(s->vdev, &s->vring)) {
-        return;
-    }
-
-    event_notifier_set(s->guest_notifier);
+    set_bit(virtio_get_queue_index(vq), s->batch_notify_vqs);
+    qemu_bh_schedule(s->bh);
 }
 
 static void notify_guest_bh(void *opaque)
 {
     VirtIOBlockDataPlane *s = opaque;
+    unsigned nvqs = s->conf->num_queues;
+    unsigned long bitmap[BITS_TO_LONGS(nvqs)];
+    unsigned j;
 
-    notify_guest(s);
-}
+    memcpy(bitmap, s->batch_notify_vqs, sizeof(bitmap));
+    memset(s->batch_notify_vqs, 0, sizeof(bitmap));
 
-static void complete_request_vring(VirtIOBlockReq *req, unsigned char status)
-{
-    VirtIOBlockDataPlane *s = req->dev->dataplane;
-    stb_p(&req->in->status, status);
+    for (j = 0; j < nvqs; j += BITS_PER_LONG) {
+        unsigned long bits = bitmap[j];
 
-    vring_push(&req->dev->dataplane->vring, &req->elem,
-               req->qiov.size + sizeof(*req->in));
+        while (bits != 0) {
+            unsigned i = j + ctzl(bits);
+            VirtQueue *vq = virtio_get_queue(s->vdev, i);
 
-    /* Suppress notification to guest by BH and its scheduled
-     * flag because requests are completed as a batch after io
-     * plug & unplug is introduced, and the BH can still be
-     * executed in dataplane aio context even after it is
-     * stopped, so needn't worry about notification loss with BH.
-     */
-    qemu_bh_schedule(s->bh);
-}
-
-static void handle_notify(EventNotifier *e)
-{
-    VirtIOBlockDataPlane *s = container_of(e, VirtIOBlockDataPlane,
-                                           host_notifier);
-    VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
-
-    event_notifier_test_and_clear(&s->host_notifier);
-    blk_io_plug(s->conf->conf.blk);
-    for (;;) {
-        MultiReqBuffer mrb = {
-            .num_writes = 0,
-        };
-        int ret;
-
-        /* Disable guest->host notifies to avoid unnecessary vmexits */
-        vring_disable_notification(s->vdev, &s->vring);
-
-        for (;;) {
-            VirtIOBlockReq *req = virtio_blk_alloc_request(vblk);
-
-            ret = vring_pop(s->vdev, &s->vring, &req->elem);
-            if (ret < 0) {
-                virtio_blk_free_request(req);
-                break; /* no more requests */
+            if (virtio_should_notify(s->vdev, vq)) {
+                event_notifier_set(virtio_queue_get_guest_notifier(vq));
             }
 
-            trace_virtio_blk_data_plane_process_request(s, req->elem.out_num,
-                                                        req->elem.in_num,
-                                                        req->elem.index);
-
-            virtio_blk_handle_request(req, &mrb);
-        }
-
-        virtio_submit_multiwrite(s->conf->conf.blk, &mrb);
-
-        if (likely(ret == -EAGAIN)) { /* vring emptied */
-            /* Re-enable guest->host notifies and stop processing the vring.
-             * But if the guest has snuck in more descriptors, keep processing.
-             */
-            if (vring_enable_notification(s->vdev, &s->vring)) {
-                break;
-            }
-        } else { /* fatal error */
-            break;
+            bits &= bits - 1; /* clear right-most bit */
         }
     }
-    blk_io_unplug(s->conf->conf.blk);
 }
 
 /* Context: QEMU global mutex held */
@@ -142,20 +83,19 @@
                                   Error **errp)
 {
     VirtIOBlockDataPlane *s;
-    Error *local_err = NULL;
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 
     *dataplane = NULL;
 
-    if (!conf->data_plane && !conf->iothread) {
+    if (!conf->iothread) {
         return;
     }
 
     /* Don't try if transport does not support notifiers. */
-    if (!k->set_guest_notifiers || !k->set_host_notifier) {
+    if (!k->set_guest_notifiers || !k->ioeventfd_started) {
         error_setg(errp,
-                   "device is incompatible with x-data-plane "
+                   "device is incompatible with dataplane "
                    "(transport does not support notifiers)");
         return;
     }
@@ -163,11 +103,8 @@
     /* If dataplane is (re-)enabled while the guest is running there could be
      * block jobs that can conflict.
      */
-    if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE,
-                          &local_err)) {
-        error_setg(errp, "cannot start dataplane thread: %s",
-                   error_get_pretty(local_err));
-        error_free(local_err);
+    if (blk_op_is_blocked(conf->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
+        error_prepend(errp, "cannot start dataplane thread: ");
         return;
     }
 
@@ -175,32 +112,11 @@
     s->vdev = vdev;
     s->conf = conf;
 
-    if (conf->iothread) {
-        s->iothread = conf->iothread;
-        object_ref(OBJECT(s->iothread));
-    } else {
-        /* Create per-device IOThread if none specified.  This is for
-         * x-data-plane option compatibility.  If x-data-plane is removed we
-         * can drop this.
-         */
-        object_initialize(&s->internal_iothread_obj,
-                          sizeof(s->internal_iothread_obj),
-                          TYPE_IOTHREAD);
-        user_creatable_complete(OBJECT(&s->internal_iothread_obj), &error_abort);
-        s->iothread = &s->internal_iothread_obj;
-    }
+    s->iothread = conf->iothread;
+    object_ref(OBJECT(s->iothread));
     s->ctx = iothread_get_aio_context(s->iothread);
     s->bh = aio_bh_new(s->ctx, notify_guest_bh, s);
-
-    error_setg(&s->blocker, "block device is in use by data plane");
-    blk_op_block_all(conf->conf.blk, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_RESIZE, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_DRIVE_DEL, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_BACKUP_SOURCE, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_COMMIT, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_MIRROR, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_STREAM, s->blocker);
-    blk_op_unblock(conf->conf.blk, BLOCK_OP_TYPE_REPLACE, s->blocker);
+    s->batch_notify_vqs = bitmap_new(conf->num_queues);
 
     *dataplane = s;
 }
@@ -213,79 +129,87 @@
     }
 
     virtio_blk_data_plane_stop(s);
-    blk_op_unblock_all(s->conf->conf.blk, s->blocker);
-    error_free(s->blocker);
-    object_unref(OBJECT(s->iothread));
+    g_free(s->batch_notify_vqs);
     qemu_bh_delete(s->bh);
+    object_unref(OBJECT(s->iothread));
     g_free(s);
 }
 
+static void virtio_blk_data_plane_handle_output(VirtIODevice *vdev,
+                                                VirtQueue *vq)
+{
+    VirtIOBlock *s = (VirtIOBlock *)vdev;
+
+    assert(s->dataplane);
+    assert(s->dataplane_started);
+
+    virtio_blk_handle_vq(s, vq);
+}
+
 /* Context: QEMU global mutex held */
 void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s)
 {
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
-    VirtQueue *vq;
+    unsigned i;
+    unsigned nvqs = s->conf->num_queues;
     int r;
 
-    if (s->started || s->disabled) {
-        return;
-    }
-
-    if (s->starting) {
+    if (vblk->dataplane_started || s->starting) {
         return;
     }
 
     s->starting = true;
 
-    vq = virtio_get_queue(s->vdev, 0);
-    if (!vring_setup(&s->vring, s->vdev, 0)) {
-        goto fail_vring;
-    }
-
     /* Set up guest notifier (irq) */
-    r = k->set_guest_notifiers(qbus->parent, 1, true);
+    r = k->set_guest_notifiers(qbus->parent, nvqs, true);
     if (r != 0) {
         fprintf(stderr, "virtio-blk failed to set guest notifier (%d), "
                 "ensure -enable-kvm is set\n", r);
         goto fail_guest_notifiers;
     }
-    s->guest_notifier = virtio_queue_get_guest_notifier(vq);
 
     /* Set up virtqueue notify */
-    r = k->set_host_notifier(qbus->parent, 0, true);
-    if (r != 0) {
-        fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
-        goto fail_host_notifier;
+    for (i = 0; i < nvqs; i++) {
+        r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, true);
+        if (r != 0) {
+            fprintf(stderr, "virtio-blk failed to set host notifier (%d)\n", r);
+            while (i--) {
+                virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
+            }
+            goto fail_guest_notifiers;
+        }
     }
-    s->host_notifier = *virtio_queue_get_host_notifier(vq);
-
-    s->saved_complete_request = vblk->complete_request;
-    vblk->complete_request = complete_request_vring;
 
     s->starting = false;
-    s->started = true;
+    vblk->dataplane_started = true;
     trace_virtio_blk_data_plane_start(s);
 
     blk_set_aio_context(s->conf->conf.blk, s->ctx);
 
     /* Kick right away to begin processing requests already in vring */
-    event_notifier_set(virtio_queue_get_host_notifier(vq));
+    for (i = 0; i < nvqs; i++) {
+        VirtQueue *vq = virtio_get_queue(s->vdev, i);
+
+        event_notifier_set(virtio_queue_get_host_notifier(vq));
+    }
 
     /* Get this show started by hooking up our callbacks */
     aio_context_acquire(s->ctx);
-    aio_set_event_notifier(s->ctx, &s->host_notifier, handle_notify);
+    for (i = 0; i < nvqs; i++) {
+        VirtQueue *vq = virtio_get_queue(s->vdev, i);
+
+        virtio_queue_aio_set_host_notifier_handler(vq, s->ctx,
+                virtio_blk_data_plane_handle_output);
+    }
     aio_context_release(s->ctx);
     return;
 
-  fail_host_notifier:
-    k->set_guest_notifiers(qbus->parent, 1, false);
   fail_guest_notifiers:
-    vring_teardown(&s->vring, s->vdev, 0);
-    s->disabled = true;
-  fail_vring:
+    vblk->dataplane_disabled = true;
     s->starting = false;
+    vblk->dataplane_started = true;
 }
 
 /* Context: QEMU global mutex held */
@@ -294,40 +218,43 @@
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s->vdev)));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtIOBlock *vblk = VIRTIO_BLK(s->vdev);
+    unsigned i;
+    unsigned nvqs = s->conf->num_queues;
 
-
-    /* Better luck next time. */
-    if (s->disabled) {
-        s->disabled = false;
+    if (!vblk->dataplane_started || s->stopping) {
         return;
     }
-    if (!s->started || s->stopping) {
+
+    /* Better luck next time. */
+    if (vblk->dataplane_disabled) {
+        vblk->dataplane_disabled = false;
+        vblk->dataplane_started = false;
         return;
     }
     s->stopping = true;
-    vblk->complete_request = s->saved_complete_request;
     trace_virtio_blk_data_plane_stop(s);
 
     aio_context_acquire(s->ctx);
 
     /* Stop notifications for new requests from guest */
-    aio_set_event_notifier(s->ctx, &s->host_notifier, NULL);
+    for (i = 0; i < nvqs; i++) {
+        VirtQueue *vq = virtio_get_queue(s->vdev, i);
+
+        virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, NULL);
+    }
 
     /* Drain and switch bs back to the QEMU main loop */
     blk_set_aio_context(s->conf->conf.blk, qemu_get_aio_context());
 
     aio_context_release(s->ctx);
 
-    /* Sync vring state back to virtqueue so that non-dataplane request
-     * processing can continue when we disable the host notifier below.
-     */
-    vring_teardown(&s->vring, s->vdev, 0);
-
-    k->set_host_notifier(qbus->parent, 0, false);
+    for (i = 0; i < nvqs; i++) {
+        virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
+    }
 
     /* Clean up guest notifier (irq) */
-    k->set_guest_notifiers(qbus->parent, 1, false);
+    k->set_guest_notifiers(qbus->parent, nvqs, false);
 
-    s->started = false;
+    vblk->dataplane_started = false;
     s->stopping = false;
 }
diff --git a/hw/block/dataplane/virtio-blk.h b/hw/block/dataplane/virtio-blk.h
index c88d40e..b1f0b95 100644
--- a/hw/block/dataplane/virtio-blk.h
+++ b/hw/block/dataplane/virtio-blk.h
@@ -26,5 +26,6 @@
 void virtio_blk_data_plane_start(VirtIOBlockDataPlane *s);
 void virtio_blk_data_plane_stop(VirtIOBlockDataPlane *s);
 void virtio_blk_data_plane_drain(VirtIOBlockDataPlane *s);
+void virtio_blk_data_plane_notify(VirtIOBlockDataPlane *s, VirtQueue *vq);
 
 #endif /* HW_DATAPLANE_VIRTIO_BLK_H */
diff --git a/hw/block/ecc.c b/hw/block/ecc.c
index 10bb233..48311d2 100644
--- a/hw/block/ecc.c
+++ b/hw/block/ecc.c
@@ -11,6 +11,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 
diff --git a/hw/block/fdc.c b/hw/block/fdc.c
index 1168e8c..f73af7d 100644
--- a/hw/block/fdc.c
+++ b/hw/block/fdc.c
@@ -27,8 +27,10 @@
  * way. There are changes in DOR register and DMA is not available.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/fdc.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "hw/isa/isa.h"
@@ -40,14 +42,15 @@
 
 /********************************************************/
 /* debug Floppy devices */
-//#define DEBUG_FLOPPY
 
-#ifdef DEBUG_FLOPPY
+#define DEBUG_FLOPPY 0
+
 #define FLOPPY_DPRINTF(fmt, ...)                                \
-    do { printf("FLOPPY: " fmt , ## __VA_ARGS__); } while (0)
-#else
-#define FLOPPY_DPRINTF(fmt, ...)
-#endif
+    do {                                                        \
+        if (DEBUG_FLOPPY) {                                     \
+            fprintf(stderr, "FLOPPY: " fmt , ## __VA_ARGS__);   \
+        }                                                       \
+    } while (0)
 
 /********************************************************/
 /* Floppy drive emulation                               */
@@ -59,104 +62,82 @@
     FDRIVE_RATE_1M   = 0x03,  /*   1 Mbps */
 } FDriveRate;
 
+typedef enum FDriveSize {
+    FDRIVE_SIZE_UNKNOWN,
+    FDRIVE_SIZE_350,
+    FDRIVE_SIZE_525,
+} FDriveSize;
+
 typedef struct FDFormat {
-    FDriveType drive;
+    FloppyDriveType drive;
     uint8_t last_sect;
     uint8_t max_track;
     uint8_t max_head;
     FDriveRate rate;
 } FDFormat;
 
+/* In many cases, the total sector size of a format is enough to uniquely
+ * identify it. However, there are some total sector collisions between
+ * formats of different physical size, and these are noted below by
+ * highlighting the total sector size for entries with collisions. */
 static const FDFormat fd_formats[] = {
     /* First entry is default format */
     /* 1.44 MB 3"1/2 floppy disks */
-    { FDRIVE_DRV_144, 18, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 20, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 21, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 21, 82, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 21, 83, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 22, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 23, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_144, 24, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 18, 80, 1, FDRIVE_RATE_500K, }, /* 3.5" 2880 */
+    { FLOPPY_DRIVE_TYPE_144, 20, 80, 1, FDRIVE_RATE_500K, }, /* 3.5" 3200 */
+    { FLOPPY_DRIVE_TYPE_144, 21, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 21, 82, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 21, 83, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 22, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 23, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_144, 24, 80, 1, FDRIVE_RATE_500K, },
     /* 2.88 MB 3"1/2 floppy disks */
-    { FDRIVE_DRV_288, 36, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 39, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 40, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 44, 80, 1, FDRIVE_RATE_1M, },
-    { FDRIVE_DRV_288, 48, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 36, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 39, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 40, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 44, 80, 1, FDRIVE_RATE_1M, },
+    { FLOPPY_DRIVE_TYPE_288, 48, 80, 1, FDRIVE_RATE_1M, },
     /* 720 kB 3"1/2 floppy disks */
-    { FDRIVE_DRV_144,  9, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 10, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 10, 82, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 10, 83, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 13, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_144, 14, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144,  9, 80, 1, FDRIVE_RATE_250K, }, /* 3.5" 1440 */
+    { FLOPPY_DRIVE_TYPE_144, 10, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 10, 82, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 10, 83, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 13, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144, 14, 80, 1, FDRIVE_RATE_250K, },
     /* 1.2 MB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120, 15, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 18, 80, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 18, 82, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 18, 83, 1, FDRIVE_RATE_500K, },
-    { FDRIVE_DRV_120, 20, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 15, 80, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 18, 80, 1, FDRIVE_RATE_500K, }, /* 5.25" 2880 */
+    { FLOPPY_DRIVE_TYPE_120, 18, 82, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 18, 83, 1, FDRIVE_RATE_500K, },
+    { FLOPPY_DRIVE_TYPE_120, 20, 80, 1, FDRIVE_RATE_500K, }, /* 5.25" 3200 */
     /* 720 kB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120,  9, 80, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_120, 11, 80, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_120,  9, 80, 1, FDRIVE_RATE_250K, }, /* 5.25" 1440 */
+    { FLOPPY_DRIVE_TYPE_120, 11, 80, 1, FDRIVE_RATE_250K, },
     /* 360 kB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120,  9, 40, 1, FDRIVE_RATE_300K, },
-    { FDRIVE_DRV_120,  9, 40, 0, FDRIVE_RATE_300K, },
-    { FDRIVE_DRV_120, 10, 41, 1, FDRIVE_RATE_300K, },
-    { FDRIVE_DRV_120, 10, 42, 1, FDRIVE_RATE_300K, },
+    { FLOPPY_DRIVE_TYPE_120,  9, 40, 1, FDRIVE_RATE_300K, }, /* 5.25" 720 */
+    { FLOPPY_DRIVE_TYPE_120,  9, 40, 0, FDRIVE_RATE_300K, },
+    { FLOPPY_DRIVE_TYPE_120, 10, 41, 1, FDRIVE_RATE_300K, },
+    { FLOPPY_DRIVE_TYPE_120, 10, 42, 1, FDRIVE_RATE_300K, },
     /* 320 kB 5"1/4 floppy disks */
-    { FDRIVE_DRV_120,  8, 40, 1, FDRIVE_RATE_250K, },
-    { FDRIVE_DRV_120,  8, 40, 0, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_120,  8, 40, 1, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_120,  8, 40, 0, FDRIVE_RATE_250K, },
     /* 360 kB must match 5"1/4 better than 3"1/2... */
-    { FDRIVE_DRV_144,  9, 80, 0, FDRIVE_RATE_250K, },
+    { FLOPPY_DRIVE_TYPE_144,  9, 80, 0, FDRIVE_RATE_250K, }, /* 3.5" 720 */
     /* end */
-    { FDRIVE_DRV_NONE, -1, -1, 0, 0, },
+    { FLOPPY_DRIVE_TYPE_NONE, -1, -1, 0, 0, },
 };
 
-static void pick_geometry(BlockBackend *blk, int *nb_heads,
-                          int *max_track, int *last_sect,
-                          FDriveType drive_in, FDriveType *drive,
-                          FDriveRate *rate)
+static FDriveSize drive_size(FloppyDriveType drive)
 {
-    const FDFormat *parse;
-    uint64_t nb_sectors, size;
-    int i, first_match, match;
-
-    blk_get_geometry(blk, &nb_sectors);
-    match = -1;
-    first_match = -1;
-    for (i = 0; ; i++) {
-        parse = &fd_formats[i];
-        if (parse->drive == FDRIVE_DRV_NONE) {
-            break;
-        }
-        if (drive_in == parse->drive ||
-            drive_in == FDRIVE_DRV_NONE) {
-            size = (parse->max_head + 1) * parse->max_track *
-                parse->last_sect;
-            if (nb_sectors == size) {
-                match = i;
-                break;
-            }
-            if (first_match == -1) {
-                first_match = i;
-            }
-        }
+    switch (drive) {
+    case FLOPPY_DRIVE_TYPE_120:
+        return FDRIVE_SIZE_525;
+    case FLOPPY_DRIVE_TYPE_144:
+    case FLOPPY_DRIVE_TYPE_288:
+        return FDRIVE_SIZE_350;
+    default:
+        return FDRIVE_SIZE_UNKNOWN;
     }
-    if (match == -1) {
-        if (first_match == -1) {
-            match = 1;
-        } else {
-            match = first_match;
-        }
-        parse = &fd_formats[match];
-    }
-    *nb_heads = parse->max_head + 1;
-    *max_track = parse->max_track;
-    *last_sect = parse->last_sect;
-    *drive = parse->drive;
-    *rate = parse->rate;
 }
 
 #define GET_CUR_DRV(fdctrl) ((fdctrl)->cur_drv)
@@ -178,13 +159,14 @@
     FDCtrl *fdctrl;
     BlockBackend *blk;
     /* Drive status */
-    FDriveType drive;
+    FloppyDriveType drive;    /* CMOS drive type        */
     uint8_t perpendicular;    /* 2.88 MB access mode    */
     /* Position */
     uint8_t head;
     uint8_t track;
     uint8_t sect;
     /* Media */
+    FloppyDriveType disk;     /* Current disk type      */
     FDiskFlags flags;
     uint8_t last_sect;        /* Nb sector per track    */
     uint8_t max_track;        /* Nb of tracks           */
@@ -192,16 +174,38 @@
     uint8_t ro;               /* Is read-only           */
     uint8_t media_changed;    /* Is media changed       */
     uint8_t media_rate;       /* Data rate of medium    */
+
+    bool media_validated;     /* Have we validated the media? */
 } FDrive;
 
+
+static FloppyDriveType get_fallback_drive_type(FDrive *drv);
+
+/* Hack: FD_SEEK is expected to work on empty drives. However, QEMU
+ * currently goes through some pains to keep seeks within the bounds
+ * established by last_sect and max_track. Correcting this is difficult,
+ * as refactoring FDC code tends to expose nasty bugs in the Linux kernel.
+ *
+ * For now: allow empty drives to have large bounds so we can seek around,
+ * with the understanding that when a diskette is inserted, the bounds will
+ * properly tighten to match the geometry of that inserted medium.
+ */
+static void fd_empty_seek_hack(FDrive *drv)
+{
+    drv->last_sect = 0xFF;
+    drv->max_track = 0xFF;
+}
+
 static void fd_init(FDrive *drv)
 {
     /* Drive */
-    drv->drive = FDRIVE_DRV_NONE;
     drv->perpendicular = 0;
     /* Disk */
+    drv->disk = FLOPPY_DRIVE_TYPE_NONE;
     drv->last_sect = 0;
     drv->max_track = 0;
+    drv->ro = true;
+    drv->media_changed = 1;
 }
 
 #define NUM_SIDES(drv) ((drv)->flags & FDISK_DBL_SIDES ? 2 : 1)
@@ -219,6 +223,13 @@
                           NUM_SIDES(drv));
 }
 
+/* Returns current position, in bytes, for given drive */
+static int fd_offset(FDrive *drv)
+{
+    g_assert(fd_sector(drv) < INT_MAX >> BDRV_SECTOR_BITS);
+    return fd_sector(drv) << BDRV_SECTOR_BITS;
+}
+
 /* Seek to a new position:
  * returns 0 if already on right track
  * returns 1 if track changed
@@ -284,39 +295,149 @@
     fd_seek(drv, 0, 0, 1, 1);
 }
 
+/**
+ * Determine geometry based on inserted diskette.
+ * Will not operate on an empty drive.
+ *
+ * @return: 0 on success, -1 if the drive is empty.
+ */
+static int pick_geometry(FDrive *drv)
+{
+    BlockBackend *blk = drv->blk;
+    const FDFormat *parse;
+    uint64_t nb_sectors, size;
+    int i;
+    int match, size_match, type_match;
+    bool magic = drv->drive == FLOPPY_DRIVE_TYPE_AUTO;
+
+    /* We can only pick a geometry if we have a diskette. */
+    if (!drv->blk || !blk_is_inserted(drv->blk) ||
+        drv->drive == FLOPPY_DRIVE_TYPE_NONE)
+    {
+        return -1;
+    }
+
+    /* We need to determine the likely geometry of the inserted medium.
+     * In order of preference, we look for:
+     * (1) The same drive type and number of sectors,
+     * (2) The same diskette size and number of sectors,
+     * (3) The same drive type.
+     *
+     * In all cases, matches that occur higher in the drive table will take
+     * precedence over matches that occur later in the table.
+     */
+    blk_get_geometry(blk, &nb_sectors);
+    match = size_match = type_match = -1;
+    for (i = 0; ; i++) {
+        parse = &fd_formats[i];
+        if (parse->drive == FLOPPY_DRIVE_TYPE_NONE) {
+            break;
+        }
+        size = (parse->max_head + 1) * parse->max_track * parse->last_sect;
+        if (nb_sectors == size) {
+            if (magic || parse->drive == drv->drive) {
+                /* (1) perfect match -- nb_sectors and drive type */
+                goto out;
+            } else if (drive_size(parse->drive) == drive_size(drv->drive)) {
+                /* (2) size match -- nb_sectors and physical medium size */
+                match = (match == -1) ? i : match;
+            } else {
+                /* This is suspicious -- Did the user misconfigure? */
+                size_match = (size_match == -1) ? i : size_match;
+            }
+        } else if (type_match == -1) {
+            if ((parse->drive == drv->drive) ||
+                (magic && (parse->drive == get_fallback_drive_type(drv)))) {
+                /* (3) type match -- nb_sectors mismatch, but matches the type
+                 *     specified explicitly by the user, or matches the fallback
+                 *     default type when using the drive autodetect mechanism */
+                type_match = i;
+            }
+        }
+    }
+
+    /* No exact match found */
+    if (match == -1) {
+        if (size_match != -1) {
+            parse = &fd_formats[size_match];
+            FLOPPY_DPRINTF("User requested floppy drive type '%s', "
+                           "but inserted medium appears to be a "
+                           "%"PRId64" sector '%s' type\n",
+                           FloppyDriveType_lookup[drv->drive],
+                           nb_sectors,
+                           FloppyDriveType_lookup[parse->drive]);
+        }
+        match = type_match;
+    }
+
+    /* No match of any kind found -- fd_format is misconfigured, abort. */
+    if (match == -1) {
+        error_setg(&error_abort, "No candidate geometries present in table "
+                   " for floppy drive type '%s'",
+                   FloppyDriveType_lookup[drv->drive]);
+    }
+
+    parse = &(fd_formats[match]);
+
+ out:
+    if (parse->max_head == 0) {
+        drv->flags &= ~FDISK_DBL_SIDES;
+    } else {
+        drv->flags |= FDISK_DBL_SIDES;
+    }
+    drv->max_track = parse->max_track;
+    drv->last_sect = parse->last_sect;
+    drv->disk = parse->drive;
+    drv->media_rate = parse->rate;
+    return 0;
+}
+
+static void pick_drive_type(FDrive *drv)
+{
+    if (drv->drive != FLOPPY_DRIVE_TYPE_AUTO) {
+        return;
+    }
+
+    if (pick_geometry(drv) == 0) {
+        drv->drive = drv->disk;
+    } else {
+        drv->drive = get_fallback_drive_type(drv);
+    }
+
+    g_assert(drv->drive != FLOPPY_DRIVE_TYPE_AUTO);
+}
+
 /* Revalidate a disk drive after a disk change */
 static void fd_revalidate(FDrive *drv)
 {
-    int nb_heads, max_track, last_sect, ro;
-    FDriveType drive;
-    FDriveRate rate;
+    int rc;
 
     FLOPPY_DPRINTF("revalidate\n");
     if (drv->blk != NULL) {
-        ro = blk_is_read_only(drv->blk);
-        pick_geometry(drv->blk, &nb_heads, &max_track,
-                      &last_sect, drv->drive, &drive, &rate);
+        drv->ro = blk_is_read_only(drv->blk);
         if (!blk_is_inserted(drv->blk)) {
             FLOPPY_DPRINTF("No disk in drive\n");
-        } else {
-            FLOPPY_DPRINTF("Floppy disk (%d h %d t %d s) %s\n", nb_heads,
-                           max_track, last_sect, ro ? "ro" : "rw");
+            drv->disk = FLOPPY_DRIVE_TYPE_NONE;
+            fd_empty_seek_hack(drv);
+        } else if (!drv->media_validated) {
+            rc = pick_geometry(drv);
+            if (rc) {
+                FLOPPY_DPRINTF("Could not validate floppy drive media");
+            } else {
+                drv->media_validated = true;
+                FLOPPY_DPRINTF("Floppy disk (%d h %d t %d s) %s\n",
+                               (drv->flags & FDISK_DBL_SIDES) ? 2 : 1,
+                               drv->max_track, drv->last_sect,
+                               drv->ro ? "ro" : "rw");
+            }
         }
-        if (nb_heads == 1) {
-            drv->flags &= ~FDISK_DBL_SIDES;
-        } else {
-            drv->flags |= FDISK_DBL_SIDES;
-        }
-        drv->max_track = max_track;
-        drv->last_sect = last_sect;
-        drv->ro = ro;
-        drv->drive = drive;
-        drv->media_rate = rate;
     } else {
         FLOPPY_DPRINTF("No drive connected\n");
         drv->last_sect = 0;
         drv->max_track = 0;
         drv->flags &= ~FDISK_DBL_SIDES;
+        drv->drive = FLOPPY_DRIVE_TYPE_NONE;
+        drv->disk = FLOPPY_DRIVE_TYPE_NONE;
     }
 }
 
@@ -324,7 +445,7 @@
 /* Intel 82078 floppy disk controller emulation          */
 
 static void fdctrl_reset(FDCtrl *fdctrl, int do_irq);
-static void fdctrl_reset_fifo(FDCtrl *fdctrl);
+static void fdctrl_to_command_phase(FDCtrl *fdctrl);
 static int fdctrl_transfer_handler (void *opaque, int nchan,
                                     int dma_pos, int dma_len);
 static void fdctrl_raise_irq(FDCtrl *fdctrl);
@@ -495,6 +616,33 @@
     FD_DIR_DSKCHG   = 0x80,
 };
 
+/*
+ * See chapter 5.0 "Controller phases" of the spec:
+ *
+ * Command phase:
+ * The host writes a command and its parameters into the FIFO. The command
+ * phase is completed when all parameters for the command have been supplied,
+ * and execution phase is entered.
+ *
+ * Execution phase:
+ * Data transfers, either DMA or non-DMA. For non-DMA transfers, the FIFO
+ * contains the payload now, otherwise it's unused. When all bytes of the
+ * required data have been transferred, the state is switched to either result
+ * phase (if the command produces status bytes) or directly back into the
+ * command phase for the next command.
+ *
+ * Result phase:
+ * The host reads out the FIFO, which contains one or more result bytes now.
+ */
+enum {
+    /* Only for migration: reconstruct phase from registers like qemu 2.3 */
+    FD_PHASE_RECONSTRUCT    = 0,
+
+    FD_PHASE_COMMAND        = 1,
+    FD_PHASE_EXECUTION      = 2,
+    FD_PHASE_RESULT         = 3,
+};
+
 #define FD_MULTI_TRACK(state) ((state) & FD_STATE_MULTI)
 #define FD_FORMAT_CMD(state) ((state) & FD_STATE_FORMAT)
 
@@ -504,6 +652,8 @@
     /* Controller state */
     QEMUTimer *result_timer;
     int dma_chann;
+    uint8_t phase;
+    IsaDma *dma;
     /* Controller's identification */
     uint8_t version;
     /* HW */
@@ -535,16 +685,20 @@
     uint8_t pwrd;
     /* Floppy drives */
     uint8_t num_floppies;
-    /* Sun4m quirks? */
-    int sun4m;
     FDrive drives[MAX_FD];
     int reset_sensei;
     uint32_t check_media_rate;
+    FloppyDriveType fallback; /* type=auto failure fallback */
     /* Timers state */
     uint8_t timer0;
     uint8_t timer1;
 };
 
+static FloppyDriveType get_fallback_drive_type(FDrive *drv)
+{
+    return drv->fdctrl->fallback;
+}
+
 #define TYPE_SYSBUS_FDC "base-sysbus-fdc"
 #define SYSBUS_FDC(obj) OBJECT_CHECK(FDCtrlSysBus, (obj), TYPE_SYSBUS_FDC)
 
@@ -673,6 +827,7 @@
     .name = "fdrive/media_changed",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fdrive_media_changed_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(media_changed, FDrive),
         VMSTATE_END_OF_LIST()
@@ -690,6 +845,7 @@
     .name = "fdrive/media_rate",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fdrive_media_rate_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(media_rate, FDrive),
         VMSTATE_END_OF_LIST()
@@ -707,6 +863,7 @@
     .name = "fdrive/perpendicular",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fdrive_perpendicular_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(perpendicular, FDrive),
         VMSTATE_END_OF_LIST()
@@ -730,22 +887,36 @@
         VMSTATE_UINT8(sect, FDrive),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_fdrive_media_changed,
-            .needed = &fdrive_media_changed_needed,
-        } , {
-            .vmsd = &vmstate_fdrive_media_rate,
-            .needed = &fdrive_media_rate_needed,
-        } , {
-            .vmsd = &vmstate_fdrive_perpendicular,
-            .needed = &fdrive_perpendicular_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fdrive_media_changed,
+        &vmstate_fdrive_media_rate,
+        &vmstate_fdrive_perpendicular,
+        NULL
     }
 };
 
+/*
+ * Reconstructs the phase from register values according to the logic that was
+ * implemented in qemu 2.3. This is the default value that is used if the phase
+ * subsection is not present on migration.
+ *
+ * Don't change this function to reflect newer qemu versions, it is part of
+ * the migration ABI.
+ */
+static int reconstruct_phase(FDCtrl *fdctrl)
+{
+    if (fdctrl->msr & FD_MSR_NONDMA) {
+        return FD_PHASE_EXECUTION;
+    } else if ((fdctrl->msr & FD_MSR_RQM) == 0) {
+        /* qemu 2.3 disabled RQM only during DMA transfers */
+        return FD_PHASE_EXECUTION;
+    } else if (fdctrl->msr & FD_MSR_DIO) {
+        return FD_PHASE_RESULT;
+    } else {
+        return FD_PHASE_COMMAND;
+    }
+}
+
 static void fdc_pre_save(void *opaque)
 {
     FDCtrl *s = opaque;
@@ -753,12 +924,24 @@
     s->dor_vmstate = s->dor | GET_CUR_DRV(s);
 }
 
+static int fdc_pre_load(void *opaque)
+{
+    FDCtrl *s = opaque;
+    s->phase = FD_PHASE_RECONSTRUCT;
+    return 0;
+}
+
 static int fdc_post_load(void *opaque, int version_id)
 {
     FDCtrl *s = opaque;
 
     SET_CUR_DRV(s, s->dor_vmstate & FD_DOR_SELMASK);
     s->dor = s->dor_vmstate & ~FD_DOR_SELMASK;
+
+    if (s->phase == FD_PHASE_RECONSTRUCT) {
+        s->phase = reconstruct_phase(s);
+    }
+
     return 0;
 }
 
@@ -773,6 +956,7 @@
     .name = "fdc/reset_sensei",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fdc_reset_sensei_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(reset_sensei, FDCtrl),
         VMSTATE_END_OF_LIST()
@@ -790,8 +974,27 @@
     .name = "fdc/result_timer",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fdc_result_timer_needed,
     .fields = (VMStateField[]) {
-        VMSTATE_TIMER(result_timer, FDCtrl),
+        VMSTATE_TIMER_PTR(result_timer, FDCtrl),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool fdc_phase_needed(void *opaque)
+{
+    FDCtrl *fdctrl = opaque;
+
+    return reconstruct_phase(fdctrl) != fdctrl->phase;
+}
+
+static const VMStateDescription vmstate_fdc_phase = {
+    .name = "fdc/phase",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = fdc_phase_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(phase, FDCtrl),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -801,6 +1004,7 @@
     .version_id = 2,
     .minimum_version_id = 2,
     .pre_save = fdc_pre_save,
+    .pre_load = fdc_pre_load,
     .post_load = fdc_post_load,
     .fields = (VMStateField[]) {
         /* Controller State */
@@ -833,16 +1037,11 @@
                              vmstate_fdrive, FDrive),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_fdc_reset_sensei,
-            .needed = fdc_reset_sensei_needed,
-        } , {
-            .vmsd = &vmstate_fdc_result_timer,
-            .needed = fdc_result_timer_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fdc_reset_sensei,
+        &vmstate_fdc_result_timer,
+        &vmstate_fdc_phase,
+        NULL
     }
 };
 
@@ -885,13 +1084,6 @@
 
 static void fdctrl_raise_irq(FDCtrl *fdctrl)
 {
-    /* Sparc mutation */
-    if (fdctrl->sun4m && (fdctrl->msr & FD_MSR_CMDBUSY)) {
-        /* XXX: not sure */
-        fdctrl->msr &= ~FD_MSR_CMDBUSY;
-        fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
-        return;
-    }
     if (!(fdctrl->sra & FD_SRA_INTPEND)) {
         qemu_set_irq(fdctrl->irq, 1);
         fdctrl->sra |= FD_SRA_INTPEND;
@@ -927,7 +1119,7 @@
     fdctrl->data_dir = FD_DIR_WRITE;
     for (i = 0; i < MAX_FD; i++)
         fd_recalibrate(&fdctrl->drives[i]);
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
     if (do_irq) {
         fdctrl->status0 |= FD_SR0_RDYCHG;
         fdctrl_raise_irq(fdctrl);
@@ -1080,12 +1272,6 @@
     fdctrl->dsr &= ~FD_DSR_PWRDOWN;
     fdctrl->dor |= FD_DOR_nRESET;
 
-    /* Sparc mutation */
-    if (fdctrl->sun4m) {
-        retval |= FD_MSR_DIO;
-        fdctrl_reset_irq(fdctrl);
-    };
-
     FLOPPY_DPRINTF("main status register: 0x%02x\n", retval);
 
     return retval;
@@ -1149,17 +1335,22 @@
     return retval;
 }
 
-/* FIFO state control */
-static void fdctrl_reset_fifo(FDCtrl *fdctrl)
+/* Clear the FIFO and update the state for receiving the next command */
+static void fdctrl_to_command_phase(FDCtrl *fdctrl)
 {
+    fdctrl->phase = FD_PHASE_COMMAND;
     fdctrl->data_dir = FD_DIR_WRITE;
     fdctrl->data_pos = 0;
+    fdctrl->data_len = 1; /* Accept command byte, adjust for params later */
     fdctrl->msr &= ~(FD_MSR_CMDBUSY | FD_MSR_DIO);
+    fdctrl->msr |= FD_MSR_RQM;
 }
 
-/* Set FIFO status for the host to read */
-static void fdctrl_set_fifo(FDCtrl *fdctrl, int fifo_len)
+/* Update the state to allow the guest to read out the command status.
+ * @fifo_len is the number of result bytes to be read out. */
+static void fdctrl_to_result_phase(FDCtrl *fdctrl, int fifo_len)
 {
+    fdctrl->phase = FD_PHASE_RESULT;
     fdctrl->data_dir = FD_DIR_READ;
     fdctrl->data_len = fifo_len;
     fdctrl->data_pos = 0;
@@ -1172,7 +1363,7 @@
     qemu_log_mask(LOG_UNIMP, "fdc: unimplemented command 0x%02x\n",
                   fdctrl->fifo[0]);
     fdctrl->fifo[0] = FD_SR0_INVCMD;
-    fdctrl_set_fifo(fdctrl, 1);
+    fdctrl_to_result_phase(fdctrl, 1);
 }
 
 /* Seek to next sector
@@ -1248,12 +1439,13 @@
     fdctrl->fifo[6] = FD_SECTOR_SC;
     fdctrl->data_dir = FD_DIR_READ;
     if (!(fdctrl->msr & FD_MSR_NONDMA)) {
-        DMA_release_DREQ(fdctrl->dma_chann);
+        IsaDmaClass *k = ISADMA_GET_CLASS(fdctrl->dma);
+        k->release_DREQ(fdctrl->dma, fdctrl->dma_chann);
     }
     fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
     fdctrl->msr &= ~FD_MSR_NONDMA;
 
-    fdctrl_set_fifo(fdctrl, 7);
+    fdctrl_to_result_phase(fdctrl, 7);
     fdctrl_raise_irq(fdctrl);
 }
 
@@ -1334,27 +1526,43 @@
     }
     fdctrl->eot = fdctrl->fifo[6];
     if (fdctrl->dor & FD_DOR_DMAEN) {
-        int dma_mode;
+        IsaDmaTransferMode dma_mode;
+        IsaDmaClass *k = ISADMA_GET_CLASS(fdctrl->dma);
+        bool dma_mode_ok;
         /* DMA transfer are enabled. Check if DMA channel is well programmed */
-        dma_mode = DMA_get_channel_mode(fdctrl->dma_chann);
-        dma_mode = (dma_mode >> 2) & 3;
+        dma_mode = k->get_transfer_mode(fdctrl->dma, fdctrl->dma_chann);
         FLOPPY_DPRINTF("dma_mode=%d direction=%d (%d - %d)\n",
                        dma_mode, direction,
                        (128 << fdctrl->fifo[5]) *
                        (cur_drv->last_sect - ks + 1), fdctrl->data_len);
-        if (((direction == FD_DIR_SCANE || direction == FD_DIR_SCANL ||
-              direction == FD_DIR_SCANH) && dma_mode == 0) ||
-            (direction == FD_DIR_WRITE && dma_mode == 2) ||
-            (direction == FD_DIR_READ && dma_mode == 1) ||
-            (direction == FD_DIR_VERIFY)) {
+        switch (direction) {
+        case FD_DIR_SCANE:
+        case FD_DIR_SCANL:
+        case FD_DIR_SCANH:
+            dma_mode_ok = (dma_mode == ISADMA_TRANSFER_VERIFY);
+            break;
+        case FD_DIR_WRITE:
+            dma_mode_ok = (dma_mode == ISADMA_TRANSFER_WRITE);
+            break;
+        case FD_DIR_READ:
+            dma_mode_ok = (dma_mode == ISADMA_TRANSFER_READ);
+            break;
+        case FD_DIR_VERIFY:
+            dma_mode_ok = true;
+            break;
+        default:
+            dma_mode_ok = false;
+            break;
+        }
+        if (dma_mode_ok) {
             /* No access is allowed until DMA transfer has completed */
             fdctrl->msr &= ~FD_MSR_RQM;
             if (direction != FD_DIR_VERIFY) {
                 /* Now, we just have to wait for the DMA controller to
                  * recall us...
                  */
-                DMA_hold_DREQ(fdctrl->dma_chann);
-                DMA_schedule(fdctrl->dma_chann);
+                k->hold_DREQ(fdctrl->dma, fdctrl->dma_chann);
+                k->schedule(fdctrl->dma);
             } else {
                 /* Start transfer */
                 fdctrl_transfer_handler(fdctrl, fdctrl->dma_chann, 0,
@@ -1367,7 +1575,7 @@
         }
     }
     FLOPPY_DPRINTF("start non-DMA transfer\n");
-    fdctrl->msr |= FD_MSR_NONDMA;
+    fdctrl->msr |= FD_MSR_NONDMA | FD_MSR_RQM;
     if (direction != FD_DIR_WRITE)
         fdctrl->msr |= FD_MSR_DIO;
     /* IO based transfer: calculate len */
@@ -1393,12 +1601,14 @@
     FDrive *cur_drv;
     int len, start_pos, rel_pos;
     uint8_t status0 = 0x00, status1 = 0x00, status2 = 0x00;
+    IsaDmaClass *k;
 
     fdctrl = opaque;
     if (fdctrl->msr & FD_MSR_RQM) {
         FLOPPY_DPRINTF("Not in DMA transfer mode !\n");
         return 0;
     }
+    k = ISADMA_GET_CLASS(fdctrl->dma);
     cur_drv = get_cur_drv(fdctrl);
     if (fdctrl->data_dir == FD_DIR_SCANE || fdctrl->data_dir == FD_DIR_SCANL ||
         fdctrl->data_dir == FD_DIR_SCANH)
@@ -1426,8 +1636,8 @@
         if (fdctrl->data_dir != FD_DIR_WRITE ||
             len < FD_SECTOR_LEN || rel_pos != 0) {
             /* READ & SCAN commands and realign to a sector for WRITE */
-            if (blk_read(cur_drv->blk, fd_sector(cur_drv),
-                         fdctrl->fifo, 1) < 0) {
+            if (blk_pread(cur_drv->blk, fd_offset(cur_drv),
+                          fdctrl->fifo, BDRV_SECTOR_SIZE) < 0) {
                 FLOPPY_DPRINTF("Floppy: error getting sector %d\n",
                                fd_sector(cur_drv));
                 /* Sure, image size is too small... */
@@ -1437,8 +1647,8 @@
         switch (fdctrl->data_dir) {
         case FD_DIR_READ:
             /* READ commands */
-            DMA_write_memory (nchan, fdctrl->fifo + rel_pos,
-                              fdctrl->data_pos, len);
+            k->write_memory(fdctrl->dma, nchan, fdctrl->fifo + rel_pos,
+                            fdctrl->data_pos, len);
             break;
         case FD_DIR_WRITE:
             /* WRITE commands */
@@ -1452,10 +1662,10 @@
                 goto transfer_error;
             }
 
-            DMA_read_memory (nchan, fdctrl->fifo + rel_pos,
-                             fdctrl->data_pos, len);
-            if (blk_write(cur_drv->blk, fd_sector(cur_drv),
-                          fdctrl->fifo, 1) < 0) {
+            k->read_memory(fdctrl->dma, nchan, fdctrl->fifo + rel_pos,
+                           fdctrl->data_pos, len);
+            if (blk_pwrite(cur_drv->blk, fd_offset(cur_drv),
+                           fdctrl->fifo, BDRV_SECTOR_SIZE, 0) < 0) {
                 FLOPPY_DPRINTF("error writing sector %d\n",
                                fd_sector(cur_drv));
                 fdctrl_stop_transfer(fdctrl, FD_SR0_ABNTERM | FD_SR0_SEEK, 0x00, 0x00);
@@ -1470,7 +1680,8 @@
             {
                 uint8_t tmpbuf[FD_SECTOR_LEN];
                 int ret;
-                DMA_read_memory (nchan, tmpbuf, fdctrl->data_pos, len);
+                k->read_memory(fdctrl->dma, nchan, tmpbuf, fdctrl->data_pos,
+                               len);
                 ret = memcmp(tmpbuf, fdctrl->fifo + rel_pos, len);
                 if (ret == 0) {
                     status2 = FD_SR2_SEH;
@@ -1520,9 +1731,16 @@
         FLOPPY_DPRINTF("error: controller not ready for reading\n");
         return 0;
     }
+
+    /* If data_len spans multiple sectors, the current position in the FIFO
+     * wraps around while fdctrl->data_pos is the real position in the whole
+     * request. */
     pos = fdctrl->data_pos;
     pos %= FD_SECTOR_LEN;
-    if (fdctrl->msr & FD_MSR_NONDMA) {
+
+    switch (fdctrl->phase) {
+    case FD_PHASE_EXECUTION:
+        assert(fdctrl->msr & FD_MSR_NONDMA);
         if (pos == 0) {
             if (fdctrl->data_pos != 0)
                 if (!fdctrl_seek_to_next_sect(fdctrl, cur_drv)) {
@@ -1530,7 +1748,8 @@
                                    fd_sector(cur_drv));
                     return 0;
                 }
-            if (blk_read(cur_drv->blk, fd_sector(cur_drv), fdctrl->fifo, 1)
+            if (blk_pread(cur_drv->blk, fd_offset(cur_drv), fdctrl->fifo,
+                          BDRV_SECTOR_SIZE)
                 < 0) {
                 FLOPPY_DPRINTF("error getting sector %d\n",
                                fd_sector(cur_drv));
@@ -1538,20 +1757,28 @@
                 memset(fdctrl->fifo, 0, FD_SECTOR_LEN);
             }
         }
-    }
-    retval = fdctrl->fifo[pos];
-    if (++fdctrl->data_pos == fdctrl->data_len) {
-        fdctrl->data_pos = 0;
-        /* Switch from transfer mode to status mode
-         * then from status mode to command mode
-         */
-        if (fdctrl->msr & FD_MSR_NONDMA) {
+
+        if (++fdctrl->data_pos == fdctrl->data_len) {
+            fdctrl->msr &= ~FD_MSR_RQM;
             fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
-        } else {
-            fdctrl_reset_fifo(fdctrl);
+        }
+        break;
+
+    case FD_PHASE_RESULT:
+        assert(!(fdctrl->msr & FD_MSR_NONDMA));
+        if (++fdctrl->data_pos == fdctrl->data_len) {
+            fdctrl->msr &= ~FD_MSR_RQM;
+            fdctrl_to_command_phase(fdctrl);
             fdctrl_reset_irq(fdctrl);
         }
+        break;
+
+    case FD_PHASE_COMMAND:
+    default:
+        abort();
     }
+
+    retval = fdctrl->fifo[pos];
     FLOPPY_DPRINTF("data register: 0x%02x\n", retval);
 
     return retval;
@@ -1601,7 +1828,8 @@
     }
     memset(fdctrl->fifo, 0, FD_SECTOR_LEN);
     if (cur_drv->blk == NULL ||
-        blk_write(cur_drv->blk, fd_sector(cur_drv), fdctrl->fifo, 1) < 0) {
+        blk_pwrite(cur_drv->blk, fd_offset(cur_drv), fdctrl->fifo,
+                   BDRV_SECTOR_SIZE, 0) < 0) {
         FLOPPY_DPRINTF("error formatting sector %d\n", fd_sector(cur_drv));
         fdctrl_stop_transfer(fdctrl, FD_SR0_ABNTERM | FD_SR0_SEEK, 0x00, 0x00);
     } else {
@@ -1621,7 +1849,7 @@
 {
     fdctrl->lock = (fdctrl->fifo[0] & 0x80) ? 1 : 0;
     fdctrl->fifo[0] = fdctrl->lock << 4;
-    fdctrl_set_fifo(fdctrl, 1);
+    fdctrl_to_result_phase(fdctrl, 1);
 }
 
 static void fdctrl_handle_dumpreg(FDCtrl *fdctrl, int direction)
@@ -1646,20 +1874,20 @@
         (cur_drv->perpendicular << 2);
     fdctrl->fifo[8] = fdctrl->config;
     fdctrl->fifo[9] = fdctrl->precomp_trk;
-    fdctrl_set_fifo(fdctrl, 10);
+    fdctrl_to_result_phase(fdctrl, 10);
 }
 
 static void fdctrl_handle_version(FDCtrl *fdctrl, int direction)
 {
     /* Controller's version */
     fdctrl->fifo[0] = fdctrl->version;
-    fdctrl_set_fifo(fdctrl, 1);
+    fdctrl_to_result_phase(fdctrl, 1);
 }
 
 static void fdctrl_handle_partid(FDCtrl *fdctrl, int direction)
 {
     fdctrl->fifo[0] = 0x41; /* Stepping 1 */
-    fdctrl_set_fifo(fdctrl, 1);
+    fdctrl_to_result_phase(fdctrl, 1);
 }
 
 static void fdctrl_handle_restore(FDCtrl *fdctrl, int direction)
@@ -1682,7 +1910,7 @@
     fdctrl->config = fdctrl->fifo[11];
     fdctrl->precomp_trk = fdctrl->fifo[12];
     fdctrl->pwrd = fdctrl->fifo[13];
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
 }
 
 static void fdctrl_handle_save(FDCtrl *fdctrl, int direction)
@@ -1712,7 +1940,7 @@
     fdctrl->fifo[12] = fdctrl->pwrd;
     fdctrl->fifo[13] = 0;
     fdctrl->fifo[14] = 0;
-    fdctrl_set_fifo(fdctrl, 15);
+    fdctrl_to_result_phase(fdctrl, 15);
 }
 
 static void fdctrl_handle_readid(FDCtrl *fdctrl, int direction)
@@ -1720,8 +1948,8 @@
     FDrive *cur_drv = get_cur_drv(fdctrl);
 
     cur_drv->head = (fdctrl->fifo[1] >> 2) & 1;
-    timer_mod(fdctrl->result_timer,
-                   qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() / 50));
+    timer_mod(fdctrl->result_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+             (NANOSECONDS_PER_SECOND / 50));
 }
 
 static void fdctrl_handle_format_track(FDCtrl *fdctrl, int direction)
@@ -1761,7 +1989,7 @@
     else
         fdctrl->dor |= FD_DOR_DMAEN;
     /* No result back */
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
 }
 
 static void fdctrl_handle_sense_drive_status(FDCtrl *fdctrl, int direction)
@@ -1777,7 +2005,7 @@
         (cur_drv->head << 2) |
         GET_CUR_DRV(fdctrl) |
         0x28;
-    fdctrl_set_fifo(fdctrl, 1);
+    fdctrl_to_result_phase(fdctrl, 1);
 }
 
 static void fdctrl_handle_recalibrate(FDCtrl *fdctrl, int direction)
@@ -1787,7 +2015,7 @@
     SET_CUR_DRV(fdctrl, fdctrl->fifo[1] & FD_DOR_SELMASK);
     cur_drv = get_cur_drv(fdctrl);
     fd_recalibrate(cur_drv);
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
     /* Raise Interrupt */
     fdctrl->status0 |= FD_SR0_SEEK;
     fdctrl_raise_irq(fdctrl);
@@ -1803,7 +2031,7 @@
         fdctrl->reset_sensei--;
     } else if (!(fdctrl->sra & FD_SRA_INTPEND)) {
         fdctrl->fifo[0] = FD_SR0_INVCMD;
-        fdctrl_set_fifo(fdctrl, 1);
+        fdctrl_to_result_phase(fdctrl, 1);
         return;
     } else {
         fdctrl->fifo[0] =
@@ -1812,7 +2040,7 @@
     }
 
     fdctrl->fifo[1] = cur_drv->track;
-    fdctrl_set_fifo(fdctrl, 2);
+    fdctrl_to_result_phase(fdctrl, 2);
     fdctrl_reset_irq(fdctrl);
     fdctrl->status0 = FD_SR0_RDYCHG;
 }
@@ -1823,7 +2051,7 @@
 
     SET_CUR_DRV(fdctrl, fdctrl->fifo[1] & FD_DOR_SELMASK);
     cur_drv = get_cur_drv(fdctrl);
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
     /* The seek command just sends step pulses to the drive and doesn't care if
      * there is a medium inserted of if it's banging the head against the drive.
      */
@@ -1840,7 +2068,7 @@
     if (fdctrl->fifo[1] & 0x80)
         cur_drv->perpendicular = fdctrl->fifo[1] & 0x7;
     /* No result back */
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
 }
 
 static void fdctrl_handle_configure(FDCtrl *fdctrl, int direction)
@@ -1848,20 +2076,20 @@
     fdctrl->config = fdctrl->fifo[2];
     fdctrl->precomp_trk =  fdctrl->fifo[3];
     /* No result back */
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
 }
 
 static void fdctrl_handle_powerdown_mode(FDCtrl *fdctrl, int direction)
 {
     fdctrl->pwrd = fdctrl->fifo[1];
     fdctrl->fifo[0] = fdctrl->fifo[1];
-    fdctrl_set_fifo(fdctrl, 1);
+    fdctrl_to_result_phase(fdctrl, 1);
 }
 
 static void fdctrl_handle_option(FDCtrl *fdctrl, int direction)
 {
     /* No result back */
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
 }
 
 static void fdctrl_handle_drive_specification_command(FDCtrl *fdctrl, int direction)
@@ -1877,15 +2105,15 @@
             fdctrl->fifo[0] = fdctrl->fifo[1];
             fdctrl->fifo[2] = 0;
             fdctrl->fifo[3] = 0;
-            fdctrl_set_fifo(fdctrl, 4);
+            fdctrl_to_result_phase(fdctrl, 4);
         } else {
-            fdctrl_reset_fifo(fdctrl);
+            fdctrl_to_command_phase(fdctrl);
         }
     } else if (fdctrl->data_len > 7) {
         /* ERROR */
         fdctrl->fifo[0] = 0x80 |
             (cur_drv->head << 2) | GET_CUR_DRV(fdctrl);
-        fdctrl_set_fifo(fdctrl, 1);
+        fdctrl_to_result_phase(fdctrl, 1);
     }
 }
 
@@ -1902,7 +2130,7 @@
         fd_seek(cur_drv, cur_drv->head,
                 cur_drv->track + fdctrl->fifo[2], cur_drv->sect, 1);
     }
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
     /* Raise Interrupt */
     fdctrl->status0 |= FD_SR0_SEEK;
     fdctrl_raise_irq(fdctrl);
@@ -1920,20 +2148,25 @@
         fd_seek(cur_drv, cur_drv->head,
                 cur_drv->track - fdctrl->fifo[2], cur_drv->sect, 1);
     }
-    fdctrl_reset_fifo(fdctrl);
+    fdctrl_to_command_phase(fdctrl);
     /* Raise Interrupt */
     fdctrl->status0 |= FD_SR0_SEEK;
     fdctrl_raise_irq(fdctrl);
 }
 
-static const struct {
+/*
+ * Handlers for the execution phase of each command
+ */
+typedef struct FDCtrlCommand {
     uint8_t value;
     uint8_t mask;
     const char* name;
     int parameters;
     void (*handler)(FDCtrl *fdctrl, int direction);
     int direction;
-} handlers[] = {
+} FDCtrlCommand;
+
+static const FDCtrlCommand handlers[] = {
     { FD_CMD_READ, 0x1f, "READ", 8, fdctrl_start_transfer, FD_DIR_READ },
     { FD_CMD_WRITE, 0x3f, "WRITE", 8, fdctrl_start_transfer, FD_DIR_WRITE },
     { FD_CMD_SEEK, 0xff, "SEEK", 2, fdctrl_handle_seek },
@@ -1970,9 +2203,19 @@
 /* Associate command to an index in the 'handlers' array */
 static uint8_t command_to_handler[256];
 
+static const FDCtrlCommand *get_command(uint8_t cmd)
+{
+    int idx;
+
+    idx = command_to_handler[cmd];
+    FLOPPY_DPRINTF("%s command\n", handlers[idx].name);
+    return &handlers[idx];
+}
+
 static void fdctrl_write_data(FDCtrl *fdctrl, uint32_t value)
 {
     FDrive *cur_drv;
+    const FDCtrlCommand *cmd;
     uint32_t pos;
 
     /* Reset mode */
@@ -1985,58 +2228,82 @@
         return;
     }
     fdctrl->dsr &= ~FD_DSR_PWRDOWN;
-    /* Is it write command time ? */
-    if (fdctrl->msr & FD_MSR_NONDMA) {
+
+    FLOPPY_DPRINTF("%s: %02x\n", __func__, value);
+
+    /* If data_len spans multiple sectors, the current position in the FIFO
+     * wraps around while fdctrl->data_pos is the real position in the whole
+     * request. */
+    pos = fdctrl->data_pos++;
+    pos %= FD_SECTOR_LEN;
+    fdctrl->fifo[pos] = value;
+
+    if (fdctrl->data_pos == fdctrl->data_len) {
+        fdctrl->msr &= ~FD_MSR_RQM;
+    }
+
+    switch (fdctrl->phase) {
+    case FD_PHASE_EXECUTION:
+        /* For DMA requests, RQM should be cleared during execution phase, so
+         * we would have errored out above. */
+        assert(fdctrl->msr & FD_MSR_NONDMA);
+
         /* FIFO data write */
-        pos = fdctrl->data_pos++;
-        pos %= FD_SECTOR_LEN;
-        fdctrl->fifo[pos] = value;
         if (pos == FD_SECTOR_LEN - 1 ||
             fdctrl->data_pos == fdctrl->data_len) {
             cur_drv = get_cur_drv(fdctrl);
-            if (blk_write(cur_drv->blk, fd_sector(cur_drv), fdctrl->fifo, 1)
-                < 0) {
+            if (blk_pwrite(cur_drv->blk, fd_offset(cur_drv), fdctrl->fifo,
+                           BDRV_SECTOR_SIZE, 0) < 0) {
                 FLOPPY_DPRINTF("error writing sector %d\n",
                                fd_sector(cur_drv));
-                return;
+                break;
             }
             if (!fdctrl_seek_to_next_sect(fdctrl, cur_drv)) {
                 FLOPPY_DPRINTF("error seeking to next sector %d\n",
                                fd_sector(cur_drv));
-                return;
+                break;
             }
         }
-        /* Switch from transfer mode to status mode
-         * then from status mode to command mode
-         */
-        if (fdctrl->data_pos == fdctrl->data_len)
-            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
-        return;
-    }
-    if (fdctrl->data_pos == 0) {
-        /* Command */
-        pos = command_to_handler[value & 0xff];
-        FLOPPY_DPRINTF("%s command\n", handlers[pos].name);
-        fdctrl->data_len = handlers[pos].parameters + 1;
-        fdctrl->msr |= FD_MSR_CMDBUSY;
-    }
 
-    FLOPPY_DPRINTF("%s: %02x\n", __func__, value);
-    pos = fdctrl->data_pos++;
-    pos %= FD_SECTOR_LEN;
-    fdctrl->fifo[pos] = value;
-    if (fdctrl->data_pos == fdctrl->data_len) {
-        /* We now have all parameters
-         * and will be able to treat the command
-         */
-        if (fdctrl->data_state & FD_STATE_FORMAT) {
-            fdctrl_format_sector(fdctrl);
-            return;
+        /* Switch to result phase when done with the transfer */
+        if (fdctrl->data_pos == fdctrl->data_len) {
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
+        }
+        break;
+
+    case FD_PHASE_COMMAND:
+        assert(!(fdctrl->msr & FD_MSR_NONDMA));
+        assert(fdctrl->data_pos < FD_SECTOR_LEN);
+
+        if (pos == 0) {
+            /* The first byte specifies the command. Now we start reading
+             * as many parameters as this command requires. */
+            cmd = get_command(value);
+            fdctrl->data_len = cmd->parameters + 1;
+            if (cmd->parameters) {
+                fdctrl->msr |= FD_MSR_RQM;
+            }
+            fdctrl->msr |= FD_MSR_CMDBUSY;
         }
 
-        pos = command_to_handler[fdctrl->fifo[0] & 0xff];
-        FLOPPY_DPRINTF("treat %s command\n", handlers[pos].name);
-        (*handlers[pos].handler)(fdctrl, handlers[pos].direction);
+        if (fdctrl->data_pos == fdctrl->data_len) {
+            /* We have all parameters now, execute the command */
+            fdctrl->phase = FD_PHASE_EXECUTION;
+
+            if (fdctrl->data_state & FD_STATE_FORMAT) {
+                fdctrl_format_sector(fdctrl);
+                break;
+            }
+
+            cmd = get_command(fdctrl->fifo[0]);
+            FLOPPY_DPRINTF("Calling handler for '%s'\n", cmd->name);
+            cmd->handler(fdctrl, cmd->direction);
+        }
+        break;
+
+    case FD_PHASE_RESULT:
+    default:
+        abort();
     }
 }
 
@@ -2068,6 +2335,7 @@
     FDrive *drive = opaque;
 
     drive->media_changed = 1;
+    drive->media_validated = false;
     fd_revalidate(drive);
 }
 
@@ -2097,10 +2365,11 @@
         }
 
         fd_init(drive);
-        fdctrl_change_cb(drive, 0);
         if (drive->blk) {
             blk_set_dev_ops(drive->blk, &fdctrl_block_ops, drive);
+            pick_drive_type(drive);
         }
+        fd_revalidate(drive);
     }
 }
 
@@ -2116,10 +2385,12 @@
     dev = DEVICE(isadev);
 
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "driveA", blk_by_legacy_dinfo(fds[0]));
+        qdev_prop_set_drive(dev, "driveA", blk_by_legacy_dinfo(fds[0]),
+                            &error_fatal);
     }
     if (fds[1]) {
-        qdev_prop_set_drive_nofail(dev, "driveB", blk_by_legacy_dinfo(fds[1]));
+        qdev_prop_set_drive(dev, "driveB", blk_by_legacy_dinfo(fds[1]),
+                            &error_fatal);
     }
     qdev_init_nofail(dev);
 
@@ -2139,10 +2410,12 @@
     fdctrl = &sys->state;
     fdctrl->dma_chann = dma_chann; /* FIXME */
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "driveA", blk_by_legacy_dinfo(fds[0]));
+        qdev_prop_set_drive(dev, "driveA", blk_by_legacy_dinfo(fds[0]),
+                            &error_fatal);
     }
     if (fds[1]) {
-        qdev_prop_set_drive_nofail(dev, "driveB", blk_by_legacy_dinfo(fds[1]));
+        qdev_prop_set_drive(dev, "driveB", blk_by_legacy_dinfo(fds[1]),
+                            &error_fatal);
     }
     qdev_init_nofail(dev);
     sbd = SYS_BUS_DEVICE(dev);
@@ -2158,7 +2431,8 @@
 
     dev = qdev_create(NULL, "SUNW,fdtwo");
     if (fds[0]) {
-        qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(fds[0]));
+        qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(fds[0]),
+                            &error_fatal);
     }
     qdev_init_nofail(dev);
     sys = SYSBUS_FDC(dev);
@@ -2172,6 +2446,10 @@
     int i, j;
     static int command_tables_inited = 0;
 
+    if (fdctrl->fallback == FLOPPY_DRIVE_TYPE_AUTO) {
+        error_setg(errp, "Cannot choose a fallback FDrive type of 'auto'");
+    }
+
     /* Fill 'command_to_handler' lookup table */
     if (!command_tables_inited) {
         command_tables_inited = 1;
@@ -2195,7 +2473,11 @@
     fdctrl->num_floppies = MAX_FD;
 
     if (fdctrl->dma_chann != -1) {
-        DMA_register_channel(fdctrl->dma_chann, &fdctrl_transfer_handler, fdctrl);
+        IsaDmaClass *k;
+        assert(fdctrl->dma);
+        k = ISADMA_GET_CLASS(fdctrl->dma);
+        k->register_channel(fdctrl->dma, fdctrl->dma_chann,
+                            &fdctrl_transfer_handler, fdctrl);
     }
     fdctrl_connect_drives(fdctrl, errp);
 }
@@ -2218,6 +2500,10 @@
 
     isa_init_irq(isadev, &fdctrl->irq, isa->irq);
     fdctrl->dma_chann = isa->dma;
+    if (fdctrl->dma_chann != -1) {
+        fdctrl->dma = isa_get_dma(isa_bus_from_device(isadev), isa->dma);
+        assert(fdctrl->dma);
+    }
 
     qdev_set_legacy_instance_id(dev, isa->iobase, 2);
     fdctrl_realize_common(fdctrl, &err);
@@ -2246,7 +2532,7 @@
     FDCtrlSysBus *sys = SYSBUS_FDC(obj);
     FDCtrl *fdctrl = &sys->state;
 
-    fdctrl->sun4m = 1;
+    fdctrl->dma_chann = -1;
 
     memory_region_init_io(&fdctrl->iomem, obj, &fdctrl_mem_strict_ops,
                           fdctrl, "fdctrl", 0x08);
@@ -2274,13 +2560,36 @@
     fdctrl_realize_common(fdctrl, errp);
 }
 
-FDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i)
+FloppyDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i)
 {
     FDCtrlISABus *isa = ISA_FDC(fdc);
 
     return isa->state.drives[i].drive;
 }
 
+void isa_fdc_get_drive_max_chs(FloppyDriveType type,
+                               uint8_t *maxc, uint8_t *maxh, uint8_t *maxs)
+{
+    const FDFormat *fdf;
+
+    *maxc = *maxh = *maxs = 0;
+    for (fdf = fd_formats; fdf->drive != FLOPPY_DRIVE_TYPE_NONE; fdf++) {
+        if (fdf->drive != type) {
+            continue;
+        }
+        if (*maxc < fdf->max_track) {
+            *maxc = fdf->max_track;
+        }
+        if (*maxh < fdf->max_head) {
+            *maxh = fdf->max_head;
+        }
+        if (*maxs < fdf->last_sect) {
+            *maxs = fdf->last_sect;
+        }
+    }
+    (*maxc)--;
+}
+
 static const VMStateDescription vmstate_isa_fdc ={
     .name = "fdc",
     .version_id = 2,
@@ -2299,6 +2608,15 @@
     DEFINE_PROP_DRIVE("driveB", FDCtrlISABus, state.drives[1].blk),
     DEFINE_PROP_BIT("check_media_rate", FDCtrlISABus, state.check_media_rate,
                     0, true),
+    DEFINE_PROP_DEFAULT("fdtypeA", FDCtrlISABus, state.drives[0].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fdtypeB", FDCtrlISABus, state.drives[1].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fallback", FDCtrlISABus, state.fallback,
+                        FLOPPY_DRIVE_TYPE_288, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2347,6 +2665,15 @@
 static Property sysbus_fdc_properties[] = {
     DEFINE_PROP_DRIVE("driveA", FDCtrlSysBus, state.drives[0].blk),
     DEFINE_PROP_DRIVE("driveB", FDCtrlSysBus, state.drives[1].blk),
+    DEFINE_PROP_DEFAULT("fdtypeA", FDCtrlSysBus, state.drives[0].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fdtypeB", FDCtrlSysBus, state.drives[1].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fallback", FDCtrlISABus, state.fallback,
+                        FLOPPY_DRIVE_TYPE_144, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2367,6 +2694,12 @@
 
 static Property sun4m_fdc_properties[] = {
     DEFINE_PROP_DRIVE("drive", FDCtrlSysBus, state.drives[0].blk),
+    DEFINE_PROP_DEFAULT("fdtype", FDCtrlSysBus, state.drives[0].drive,
+                        FLOPPY_DRIVE_TYPE_AUTO, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
+    DEFINE_PROP_DEFAULT("fallback", FDCtrlISABus, state.fallback,
+                        FLOPPY_DRIVE_TYPE_144, qdev_prop_fdc_drive_type,
+                        FloppyDriveType),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/block/hd-geometry.c b/hw/block/hd-geometry.c
index 6fcf74d..57ad501 100644
--- a/hw/block/hd-geometry.c
+++ b/hw/block/hd-geometry.c
@@ -30,7 +30,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
+#include "qemu/bswap.h"
 #include "hw/block/block.h"
 #include "trace.h"
 
@@ -65,7 +67,7 @@
      * but also in async I/O mode. So the I/O throttling function has to
      * be disabled temporarily here, not permanently.
      */
-    if (blk_read_unthrottled(blk, 0, buf, 1) < 0) {
+    if (blk_pread_unthrottled(blk, 0, buf, BDRV_SECTOR_SIZE) < 0) {
         return -1;
     }
     /* test msdos magic */
@@ -121,8 +123,16 @@
                        int *ptrans)
 {
     int cylinders, heads, secs, translation;
+    HDGeometry geo;
 
-    if (guess_disk_lchs(blk, &cylinders, &heads, &secs) < 0) {
+    /* Try to probe the backing device geometry, otherwise fallback
+       to the old logic. (as of 12/2014 probing only succeeds on DASDs) */
+    if (blk_probe_geometry(blk, &geo) == 0) {
+        *pcyls = geo.cylinders;
+        *psecs = geo.sectors;
+        *pheads = geo.heads;
+        translation = BIOS_ATA_TRANSLATION_NONE;
+    } else if (guess_disk_lchs(blk, &cylinders, &heads, &secs) < 0) {
         /* no LCHS guess: use a standard physical disk geometry  */
         guess_chs_for_size(blk, pcyls, pheads, psecs);
         translation = hd_bios_chs_auto_trans(*pcyls, *pheads, *psecs);
diff --git a/hw/block/m25p80.c b/hw/block/m25p80.c
index ff1106b..9828ee6 100644
--- a/hw/block/m25p80.c
+++ b/hw/block/m25p80.c
@@ -21,10 +21,14 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
+#include "qemu/bitops.h"
+#include "qemu/log.h"
+#include "qapi/error.h"
 
 #ifndef M25P80_ERR_DEBUG
 #define M25P80_ERR_DEBUG 0
@@ -45,14 +49,22 @@
 /* set to allow the page program command to write 0s back to 1. Useful for
  * modelling EEPROM with SPI flash command set
  */
-#define WR_1 0x100
+#define EEPROM 0x100
+
+/* 16 MiB max in 3 byte address mode */
+#define MAX_3BYTES_SIZE 0x1000000
+
+#define SPI_NOR_MAX_ID_LEN 6
 
 typedef struct FlashPartInfo {
     const char *part_name;
-    /* jedec code. (jedec >> 16) & 0xff is the 1st byte, >> 8 the 2nd etc */
-    uint32_t jedec;
-    /* extended jedec code */
-    uint16_t ext_jedec;
+    /*
+     * This array stores the ID bytes.
+     * The first three bytes are the JEDIC ID.
+     * JEDEC ID zero means "no ID" (mostly older chips).
+     */
+    uint8_t id[SPI_NOR_MAX_ID_LEN];
+    uint8_t id_len;
     /* there is confusion between manufacturers as to what a sector is. In this
      * device model, a "sector" is the size that is erased by the ERASE_SECTOR
      * command (opcode 0xd8).
@@ -60,15 +72,37 @@
     uint32_t sector_size;
     uint32_t n_sectors;
     uint32_t page_size;
-    uint8_t flags;
+    uint16_t flags;
 } FlashPartInfo;
 
 /* adapted from linux */
+/* Used when the "_ext_id" is two bytes at most */
+#define INFO(_part_name, _jedec_id, _ext_id, _sector_size, _n_sectors, _flags)\
+    .part_name = _part_name,\
+    .id = {\
+        ((_jedec_id) >> 16) & 0xff,\
+        ((_jedec_id) >> 8) & 0xff,\
+        (_jedec_id) & 0xff,\
+        ((_ext_id) >> 8) & 0xff,\
+        (_ext_id) & 0xff,\
+          },\
+    .id_len = (!(_jedec_id) ? 0 : (3 + ((_ext_id) ? 2 : 0))),\
+    .sector_size = (_sector_size),\
+    .n_sectors = (_n_sectors),\
+    .page_size = 256,\
+    .flags = (_flags),
 
-#define INFO(_part_name, _jedec, _ext_jedec, _sector_size, _n_sectors, _flags)\
-    .part_name = (_part_name),\
-    .jedec = (_jedec),\
-    .ext_jedec = (_ext_jedec),\
+#define INFO6(_part_name, _jedec_id, _ext_id, _sector_size, _n_sectors, _flags)\
+    .part_name = _part_name,\
+    .id = {\
+        ((_jedec_id) >> 16) & 0xff,\
+        ((_jedec_id) >> 8) & 0xff,\
+        (_jedec_id) & 0xff,\
+        ((_ext_id) >> 16) & 0xff,\
+        ((_ext_id) >> 8) & 0xff,\
+        (_ext_id) & 0xff,\
+          },\
+    .id_len = 6,\
     .sector_size = (_sector_size),\
     .n_sectors = (_n_sectors),\
     .page_size = 256,\
@@ -78,6 +112,45 @@
 #define JEDEC_WINBOND 0xEF
 #define JEDEC_SPANSION 0x01
 
+/* Numonyx (Micron) Configuration register macros */
+#define VCFG_DUMMY 0x1
+#define VCFG_WRAP_SEQUENTIAL 0x2
+#define NVCFG_XIP_MODE_DISABLED (7 << 9)
+#define NVCFG_XIP_MODE_MASK (7 << 9)
+#define VCFG_XIP_MODE_ENABLED (1 << 3)
+#define CFG_DUMMY_CLK_LEN 4
+#define NVCFG_DUMMY_CLK_POS 12
+#define VCFG_DUMMY_CLK_POS 4
+#define EVCFG_OUT_DRIVER_STRENGHT_DEF 7
+#define EVCFG_VPP_ACCELERATOR (1 << 3)
+#define EVCFG_RESET_HOLD_ENABLED (1 << 4)
+#define NVCFG_DUAL_IO_MASK (1 << 2)
+#define EVCFG_DUAL_IO_ENABLED (1 << 6)
+#define NVCFG_QUAD_IO_MASK (1 << 3)
+#define EVCFG_QUAD_IO_ENABLED (1 << 7)
+#define NVCFG_4BYTE_ADDR_MASK (1 << 0)
+#define NVCFG_LOWER_SEGMENT_MASK (1 << 1)
+
+/* Numonyx (Micron) Flag Status Register macros */
+#define FSR_4BYTE_ADDR_MODE_ENABLED 0x1
+#define FSR_FLASH_READY (1 << 7)
+
+/* Spansion configuration registers macros. */
+#define SPANSION_QUAD_CFG_POS 0
+#define SPANSION_QUAD_CFG_LEN 1
+#define SPANSION_DUMMY_CLK_POS 0
+#define SPANSION_DUMMY_CLK_LEN 4
+#define SPANSION_ADDR_LEN_POS 7
+#define SPANSION_ADDR_LEN_LEN 1
+
+/*
+ * Spansion read mode command length in bytes,
+ * the mode is currently not supported.
+*/
+
+#define SPANSION_CONTINUOUS_READ_MODE_CMD_LEN 1
+#define WINBOND_CONTINUOUS_READ_MODE_CMD_LEN 1
+
 static const FlashPartInfo known_devices[] = {
     /* Atmel -- some are (confusingly) marketed as "DataFlash" */
     { INFO("at25fs010",   0x1f6601,      0,  32 << 10,   4, ER_4K) },
@@ -94,6 +167,12 @@
 
     { INFO("at45db081d",  0x1f2500,      0,  64 << 10,  16, ER_4K) },
 
+    /* Atmel EEPROMS - it is assumed, that don't care bit in command
+     * is set to 0. Block protection is not supported.
+     */
+    { INFO("at25128a-nonjedec", 0x0,     0,         1, 131072, EEPROM) },
+    { INFO("at25256a-nonjedec", 0x0,     0,         1, 262144, EEPROM) },
+
     /* EON -- en25xxx */
     { INFO("en25f32",     0x1c3116,      0,  64 << 10,  64, ER_4K) },
     { INFO("en25p32",     0x1c2016,      0,  64 << 10,  64, 0) },
@@ -122,6 +201,8 @@
     { INFO("mx25l12855e", 0xc22618,      0,  64 << 10, 256, 0) },
     { INFO("mx25l25635e", 0xc22019,      0,  64 << 10, 512, 0) },
     { INFO("mx25l25655e", 0xc22619,      0,  64 << 10, 512, 0) },
+    { INFO("mx66u51235f", 0xc2253a,      0,  64 << 10, 1024, ER_4K | ER_32K) },
+    { INFO("mx66u1g45g",  0xc2253b,      0,  64 << 10, 2048, ER_4K | ER_32K) },
 
     /* Micron */
     { INFO("n25q032a11",  0x20bb16,      0,  64 << 10,  64, ER_4K) },
@@ -132,6 +213,11 @@
     { INFO("n25q128a13",  0x20ba18,      0,  64 << 10, 256, ER_4K) },
     { INFO("n25q256a11",  0x20bb19,      0,  64 << 10, 512, ER_4K) },
     { INFO("n25q256a13",  0x20ba19,      0,  64 << 10, 512, ER_4K) },
+    { INFO("n25q128",     0x20ba18,      0,  64 << 10, 256, 0) },
+    { INFO("n25q256a",    0x20ba19,      0,  64 << 10, 512, ER_4K) },
+    { INFO("n25q512a",    0x20ba20,      0,  64 << 10, 1024, ER_4K) },
+    { INFO("mt25ql01g",   0x20ba21,      0,  64 << 10, 2048, ER_4K) },
+    { INFO("mt25qu01g",   0x20bb21,      0,  64 << 10, 2048, ER_4K) },
 
     /* Spansion -- single (large) sector size only, at least
      * for the chips listed here (without boot sectors).
@@ -140,8 +226,8 @@
     { INFO("s25sl064p",   0x010216, 0x4d00,  64 << 10, 128, ER_4K) },
     { INFO("s25fl256s0",  0x010219, 0x4d00, 256 << 10, 128, 0) },
     { INFO("s25fl256s1",  0x010219, 0x4d01,  64 << 10, 512, 0) },
-    { INFO("s25fl512s",   0x010220, 0x4d00, 256 << 10, 256, 0) },
-    { INFO("s70fl01gs",   0x010221, 0x4d00, 256 << 10, 256, 0) },
+    { INFO6("s25fl512s",  0x010220, 0x4d0080, 256 << 10, 256, 0) },
+    { INFO6("s70fl01gs",  0x010221, 0x4d0080, 256 << 10, 512, 0) },
     { INFO("s25sl12800",  0x012018, 0x0300, 256 << 10,  64, 0) },
     { INFO("s25sl12801",  0x012018, 0x0301,  64 << 10, 256, 0) },
     { INFO("s25fl129p0",  0x012018, 0x4d00, 256 << 10,  64, 0) },
@@ -154,6 +240,10 @@
     { INFO("s25fl016k",   0xef4015,      0,  64 << 10,  32, ER_4K | ER_32K) },
     { INFO("s25fl064k",   0xef4017,      0,  64 << 10, 128, ER_4K | ER_32K) },
 
+    /* Spansion --  boot sectors support  */
+    { INFO6("s25fs512s",    0x010220, 0x4d0081, 256 << 10, 256, 0) },
+    { INFO6("s70fs01gs",    0x010221, 0x4d0081, 256 << 10, 512, 0) },
+
     /* SST -- large erase sizes are "overlays", "sectors" are 4<< 10 */
     { INFO("sst25vf040b", 0xbf258d,      0,  64 << 10,   8, ER_4K) },
     { INFO("sst25vf080b", 0xbf258e,      0,  64 << 10,  16, ER_4K) },
@@ -163,6 +253,7 @@
     { INFO("sst25wf010",  0xbf2502,      0,  64 << 10,   2, ER_4K) },
     { INFO("sst25wf020",  0xbf2503,      0,  64 << 10,   4, ER_4K) },
     { INFO("sst25wf040",  0xbf2504,      0,  64 << 10,   8, ER_4K) },
+    { INFO("sst25wf080",  0xbf2505,      0,  64 << 10,  16, ER_4K) },
 
     /* ST Microelectronics -- newer production may have feature updates */
     { INFO("m25p05",      0x202010,      0,  32 << 10,   2, 0) },
@@ -203,9 +294,6 @@
     { INFO("w25q80",      0xef5014,      0,  64 << 10,  16, ER_4K) },
     { INFO("w25q80bl",    0xef4014,      0,  64 << 10,  16, ER_4K) },
     { INFO("w25q256",     0xef4019,      0,  64 << 10, 512, ER_4K) },
-
-    /* Numonyx -- n25q128 */
-    { INFO("n25q128",      0x20ba18,      0,  64 << 10, 256, 0) },
 };
 
 typedef enum {
@@ -216,21 +304,59 @@
     WREN = 0x6,
     JEDEC_READ = 0x9f,
     BULK_ERASE = 0xc7,
+    READ_FSR = 0x70,
+    RDCR = 0x15,
 
-    READ = 0x3,
-    FAST_READ = 0xb,
+    READ = 0x03,
+    READ4 = 0x13,
+    FAST_READ = 0x0b,
+    FAST_READ4 = 0x0c,
     DOR = 0x3b,
+    DOR4 = 0x3c,
     QOR = 0x6b,
+    QOR4 = 0x6c,
     DIOR = 0xbb,
+    DIOR4 = 0xbc,
     QIOR = 0xeb,
+    QIOR4 = 0xec,
 
-    PP = 0x2,
+    PP = 0x02,
+    PP4 = 0x12,
+    PP4_4 = 0x3e,
     DPP = 0xa2,
     QPP = 0x32,
 
     ERASE_4K = 0x20,
+    ERASE4_4K = 0x21,
     ERASE_32K = 0x52,
+    ERASE4_32K = 0x5c,
     ERASE_SECTOR = 0xd8,
+    ERASE4_SECTOR = 0xdc,
+
+    EN_4BYTE_ADDR = 0xB7,
+    EX_4BYTE_ADDR = 0xE9,
+
+    EXTEND_ADDR_READ = 0xC8,
+    EXTEND_ADDR_WRITE = 0xC5,
+
+    RESET_ENABLE = 0x66,
+    RESET_MEMORY = 0x99,
+
+    /*
+     * Micron: 0x35 - enable QPI
+     * Spansion: 0x35 - read control register
+     */
+    RDCR_EQIO = 0x35,
+    RSTQIO = 0xf5,
+
+    RNVCR = 0xB5,
+    WNVCR = 0xB1,
+
+    RVCR = 0x85,
+    WVCR = 0x81,
+
+    REVCR = 0x65,
+    WEVCR = 0x61,
 } FlashCMD;
 
 typedef enum {
@@ -238,14 +364,21 @@
     STATE_PAGE_PROGRAM,
     STATE_READ,
     STATE_COLLECTING_DATA,
+    STATE_COLLECTING_VAR_LEN_DATA,
     STATE_READING_DATA,
 } CMDState;
 
+typedef enum {
+    MAN_SPANSION,
+    MAN_MACRONIX,
+    MAN_NUMONYX,
+    MAN_WINBOND,
+    MAN_GENERIC,
+} Manufacturer;
+
 typedef struct Flash {
     SSISlave parent_obj;
 
-    uint32_t r;
-
     BlockBackend *blk;
 
     uint8_t *storage;
@@ -258,8 +391,25 @@
     uint32_t pos;
     uint8_t needed_bytes;
     uint8_t cmd_in_progress;
-    uint64_t cur_addr;
+    uint32_t cur_addr;
+    uint32_t nonvolatile_cfg;
+    /* Configuration register for Macronix */
+    uint32_t volatile_cfg;
+    uint32_t enh_volatile_cfg;
+    /* Spansion cfg registers. */
+    uint8_t spansion_cr1nv;
+    uint8_t spansion_cr2nv;
+    uint8_t spansion_cr3nv;
+    uint8_t spansion_cr4nv;
+    uint8_t spansion_cr1v;
+    uint8_t spansion_cr2v;
+    uint8_t spansion_cr3v;
+    uint8_t spansion_cr4v;
     bool write_enable;
+    bool four_bytes_address_mode;
+    bool reset_enable;
+    bool quad_enable;
+    uint8_t ear;
 
     int64_t dirty_page;
 
@@ -280,8 +430,29 @@
 #define M25P80_GET_CLASS(obj) \
      OBJECT_GET_CLASS(M25P80Class, (obj), TYPE_M25P80)
 
+static inline Manufacturer get_man(Flash *s)
+{
+    switch (s->pi->id[0]) {
+    case 0x20:
+        return MAN_NUMONYX;
+    case 0xEF:
+        return MAN_WINBOND;
+    case 0x01:
+        return MAN_SPANSION;
+    case 0xC2:
+        return MAN_MACRONIX;
+    default:
+        return MAN_GENERIC;
+    }
+}
+
 static void blk_sync_complete(void *opaque, int ret)
 {
+    QEMUIOVector *iov = opaque;
+
+    qemu_iovec_destroy(iov);
+    g_free(iov);
+
     /* do nothing. Masters do not directly interact with the backing store,
      * only the working copy so no mutexing required.
      */
@@ -289,39 +460,33 @@
 
 static void flash_sync_page(Flash *s, int page)
 {
-    int blk_sector, nb_sectors;
-    QEMUIOVector iov;
+    QEMUIOVector *iov;
 
     if (!s->blk || blk_is_read_only(s->blk)) {
         return;
     }
 
-    blk_sector = (page * s->pi->page_size) / BDRV_SECTOR_SIZE;
-    nb_sectors = DIV_ROUND_UP(s->pi->page_size, BDRV_SECTOR_SIZE);
-    qemu_iovec_init(&iov, 1);
-    qemu_iovec_add(&iov, s->storage + blk_sector * BDRV_SECTOR_SIZE,
-                   nb_sectors * BDRV_SECTOR_SIZE);
-    blk_aio_writev(s->blk, blk_sector, &iov, nb_sectors, blk_sync_complete,
-                   NULL);
+    iov = g_new(QEMUIOVector, 1);
+    qemu_iovec_init(iov, 1);
+    qemu_iovec_add(iov, s->storage + page * s->pi->page_size,
+                   s->pi->page_size);
+    blk_aio_pwritev(s->blk, page * s->pi->page_size, iov, 0,
+                    blk_sync_complete, iov);
 }
 
 static inline void flash_sync_area(Flash *s, int64_t off, int64_t len)
 {
-    int64_t start, end, nb_sectors;
-    QEMUIOVector iov;
+    QEMUIOVector *iov;
 
     if (!s->blk || blk_is_read_only(s->blk)) {
         return;
     }
 
     assert(!(len % BDRV_SECTOR_SIZE));
-    start = off / BDRV_SECTOR_SIZE;
-    end = (off + len) / BDRV_SECTOR_SIZE;
-    nb_sectors = end - start;
-    qemu_iovec_init(&iov, 1);
-    qemu_iovec_add(&iov, s->storage + (start * BDRV_SECTOR_SIZE),
-                                        nb_sectors * BDRV_SECTOR_SIZE);
-    blk_aio_writev(s->blk, start, &iov, nb_sectors, blk_sync_complete, NULL);
+    iov = g_new(QEMUIOVector, 1);
+    qemu_iovec_init(iov, 1);
+    qemu_iovec_add(iov, s->storage + off, len);
+    blk_aio_pwritev(s->blk, off, iov, 0, blk_sync_complete, iov);
 }
 
 static void flash_erase(Flash *s, int offset, FlashCMD cmd)
@@ -331,14 +496,17 @@
 
     switch (cmd) {
     case ERASE_4K:
+    case ERASE4_4K:
         len = 4 << 10;
         capa_to_assert = ER_4K;
         break;
     case ERASE_32K:
+    case ERASE4_32K:
         len = 32 << 10;
         capa_to_assert = ER_32K;
         break;
     case ERASE_SECTOR:
+    case ERASE4_SECTOR:
         len = s->pi->sector_size;
         break;
     case BULK_ERASE:
@@ -371,9 +539,9 @@
 }
 
 static inline
-void flash_write8(Flash *s, uint64_t addr, uint8_t data)
+void flash_write8(Flash *s, uint32_t addr, uint8_t data)
 {
-    int64_t page = addr / s->pi->page_size;
+    uint32_t page = addr / s->pi->page_size;
     uint8_t prev = s->storage[s->cur_addr];
 
     if (!s->write_enable) {
@@ -381,11 +549,11 @@
     }
 
     if ((prev ^ data) & data) {
-        DB_PRINT_L(1, "programming zero to one! addr=%" PRIx64 "  %" PRIx8
+        DB_PRINT_L(1, "programming zero to one! addr=%" PRIx32 "  %" PRIx8
                    " -> %" PRIx8 "\n", addr, prev, data);
     }
 
-    if (s->pi->flags & WR_1) {
+    if (s->pi->flags & EEPROM) {
         s->storage[s->cur_addr] = data;
     } else {
         s->storage[s->cur_addr] &= data;
@@ -395,11 +563,43 @@
     s->dirty_page = page;
 }
 
+static inline int get_addr_length(Flash *s)
+{
+   /* check if eeprom is in use */
+    if (s->pi->flags == EEPROM) {
+        return 2;
+    }
+
+   switch (s->cmd_in_progress) {
+   case PP4:
+   case PP4_4:
+   case READ4:
+   case QIOR4:
+   case ERASE4_4K:
+   case ERASE4_32K:
+   case ERASE4_SECTOR:
+   case FAST_READ4:
+   case DOR4:
+   case QOR4:
+   case DIOR4:
+       return 4;
+   default:
+       return s->four_bytes_address_mode ? 4 : 3;
+   }
+}
+
 static void complete_collecting_data(Flash *s)
 {
-    s->cur_addr = s->data[0] << 16;
-    s->cur_addr |= s->data[1] << 8;
-    s->cur_addr |= s->data[2];
+    int i, n;
+
+    n = get_addr_length(s);
+    s->cur_addr = (n == 3 ? s->ear : 0);
+    for (i = 0; i < n; ++i) {
+        s->cur_addr <<= 8;
+        s->cur_addr |= s->data[i];
+    }
+
+    s->cur_addr &= s->size - 1;
 
     s->state = STATE_IDLE;
 
@@ -407,96 +607,316 @@
     case DPP:
     case QPP:
     case PP:
+    case PP4:
+    case PP4_4:
         s->state = STATE_PAGE_PROGRAM;
         break;
     case READ:
+    case READ4:
     case FAST_READ:
+    case FAST_READ4:
     case DOR:
+    case DOR4:
     case QOR:
+    case QOR4:
     case DIOR:
+    case DIOR4:
     case QIOR:
+    case QIOR4:
         s->state = STATE_READ;
         break;
     case ERASE_4K:
+    case ERASE4_4K:
     case ERASE_32K:
+    case ERASE4_32K:
     case ERASE_SECTOR:
+    case ERASE4_SECTOR:
         flash_erase(s, s->cur_addr, s->cmd_in_progress);
         break;
     case WRSR:
+        switch (get_man(s)) {
+        case MAN_SPANSION:
+            s->quad_enable = !!(s->data[1] & 0x02);
+            break;
+        case MAN_MACRONIX:
+            s->quad_enable = extract32(s->data[0], 6, 1);
+            if (s->len > 1) {
+                s->four_bytes_address_mode = extract32(s->data[1], 5, 1);
+            }
+            break;
+        default:
+            break;
+        }
         if (s->write_enable) {
             s->write_enable = false;
         }
         break;
+    case EXTEND_ADDR_WRITE:
+        s->ear = s->data[0];
+        break;
+    case WNVCR:
+        s->nonvolatile_cfg = s->data[0] | (s->data[1] << 8);
+        break;
+    case WVCR:
+        s->volatile_cfg = s->data[0];
+        break;
+    case WEVCR:
+        s->enh_volatile_cfg = s->data[0];
+        break;
     default:
         break;
     }
 }
 
+static void reset_memory(Flash *s)
+{
+    s->cmd_in_progress = NOP;
+    s->cur_addr = 0;
+    s->ear = 0;
+    s->four_bytes_address_mode = false;
+    s->len = 0;
+    s->needed_bytes = 0;
+    s->pos = 0;
+    s->state = STATE_IDLE;
+    s->write_enable = false;
+    s->reset_enable = false;
+    s->quad_enable = false;
+
+    switch (get_man(s)) {
+    case MAN_NUMONYX:
+        s->volatile_cfg = 0;
+        s->volatile_cfg |= VCFG_DUMMY;
+        s->volatile_cfg |= VCFG_WRAP_SEQUENTIAL;
+        if ((s->nonvolatile_cfg & NVCFG_XIP_MODE_MASK)
+                                != NVCFG_XIP_MODE_DISABLED) {
+            s->volatile_cfg |= VCFG_XIP_MODE_ENABLED;
+        }
+        s->volatile_cfg |= deposit32(s->volatile_cfg,
+                            VCFG_DUMMY_CLK_POS,
+                            CFG_DUMMY_CLK_LEN,
+                            extract32(s->nonvolatile_cfg,
+                                        NVCFG_DUMMY_CLK_POS,
+                                        CFG_DUMMY_CLK_LEN)
+                            );
+
+        s->enh_volatile_cfg = 0;
+        s->enh_volatile_cfg |= EVCFG_OUT_DRIVER_STRENGHT_DEF;
+        s->enh_volatile_cfg |= EVCFG_VPP_ACCELERATOR;
+        s->enh_volatile_cfg |= EVCFG_RESET_HOLD_ENABLED;
+        if (s->nonvolatile_cfg & NVCFG_DUAL_IO_MASK) {
+            s->enh_volatile_cfg |= EVCFG_DUAL_IO_ENABLED;
+        }
+        if (s->nonvolatile_cfg & NVCFG_QUAD_IO_MASK) {
+            s->enh_volatile_cfg |= EVCFG_QUAD_IO_ENABLED;
+        }
+        if (!(s->nonvolatile_cfg & NVCFG_4BYTE_ADDR_MASK)) {
+            s->four_bytes_address_mode = true;
+        }
+        if (!(s->nonvolatile_cfg & NVCFG_LOWER_SEGMENT_MASK)) {
+            s->ear = s->size / MAX_3BYTES_SIZE - 1;
+        }
+        break;
+    case MAN_MACRONIX:
+        s->volatile_cfg = 0x7;
+        break;
+    case MAN_SPANSION:
+        s->spansion_cr1v = s->spansion_cr1nv;
+        s->spansion_cr2v = s->spansion_cr2nv;
+        s->spansion_cr3v = s->spansion_cr3nv;
+        s->spansion_cr4v = s->spansion_cr4nv;
+        s->quad_enable = extract32(s->spansion_cr1v,
+                                   SPANSION_QUAD_CFG_POS,
+                                   SPANSION_QUAD_CFG_LEN
+                                   );
+        s->four_bytes_address_mode = extract32(s->spansion_cr2v,
+                SPANSION_ADDR_LEN_POS,
+                SPANSION_ADDR_LEN_LEN
+                );
+        break;
+    default:
+        break;
+    }
+
+    DB_PRINT_L(0, "Reset done.\n");
+}
+
+static void decode_fast_read_cmd(Flash *s)
+{
+    s->needed_bytes = get_addr_length(s);
+    switch (get_man(s)) {
+    /* Dummy cycles - modeled with bytes writes instead of bits */
+    case MAN_WINBOND:
+        s->needed_bytes += 8;
+        break;
+    case MAN_NUMONYX:
+        s->needed_bytes += extract32(s->volatile_cfg, 4, 4);
+        break;
+    case MAN_MACRONIX:
+        if (extract32(s->volatile_cfg, 6, 2) == 1) {
+            s->needed_bytes += 6;
+        } else {
+            s->needed_bytes += 8;
+        }
+        break;
+    case MAN_SPANSION:
+        s->needed_bytes += extract32(s->spansion_cr2v,
+                                    SPANSION_DUMMY_CLK_POS,
+                                    SPANSION_DUMMY_CLK_LEN
+                                    );
+        break;
+    default:
+        break;
+    }
+    s->pos = 0;
+    s->len = 0;
+    s->state = STATE_COLLECTING_DATA;
+}
+
+static void decode_dio_read_cmd(Flash *s)
+{
+    s->needed_bytes = get_addr_length(s);
+    /* Dummy cycles modeled with bytes writes instead of bits */
+    switch (get_man(s)) {
+    case MAN_WINBOND:
+        s->needed_bytes += WINBOND_CONTINUOUS_READ_MODE_CMD_LEN;
+        break;
+    case MAN_SPANSION:
+        s->needed_bytes += SPANSION_CONTINUOUS_READ_MODE_CMD_LEN;
+        s->needed_bytes += extract32(s->spansion_cr2v,
+                                    SPANSION_DUMMY_CLK_POS,
+                                    SPANSION_DUMMY_CLK_LEN
+                                    );
+        break;
+    case MAN_NUMONYX:
+        s->needed_bytes += extract32(s->volatile_cfg, 4, 4);
+        break;
+    case MAN_MACRONIX:
+        switch (extract32(s->volatile_cfg, 6, 2)) {
+        case 1:
+            s->needed_bytes += 6;
+            break;
+        case 2:
+            s->needed_bytes += 8;
+            break;
+        default:
+            s->needed_bytes += 4;
+            break;
+        }
+        break;
+    default:
+        break;
+    }
+    s->pos = 0;
+    s->len = 0;
+    s->state = STATE_COLLECTING_DATA;
+}
+
+static void decode_qio_read_cmd(Flash *s)
+{
+    s->needed_bytes = get_addr_length(s);
+    /* Dummy cycles modeled with bytes writes instead of bits */
+    switch (get_man(s)) {
+    case MAN_WINBOND:
+        s->needed_bytes += WINBOND_CONTINUOUS_READ_MODE_CMD_LEN;
+        s->needed_bytes += 4;
+        break;
+    case MAN_SPANSION:
+        s->needed_bytes += SPANSION_CONTINUOUS_READ_MODE_CMD_LEN;
+        s->needed_bytes += extract32(s->spansion_cr2v,
+                                    SPANSION_DUMMY_CLK_POS,
+                                    SPANSION_DUMMY_CLK_LEN
+                                    );
+        break;
+    case MAN_NUMONYX:
+        s->needed_bytes += extract32(s->volatile_cfg, 4, 4);
+        break;
+    case MAN_MACRONIX:
+        switch (extract32(s->volatile_cfg, 6, 2)) {
+        case 1:
+            s->needed_bytes += 4;
+            break;
+        case 2:
+            s->needed_bytes += 8;
+            break;
+        default:
+            s->needed_bytes += 6;
+            break;
+        }
+        break;
+    default:
+        break;
+    }
+    s->pos = 0;
+    s->len = 0;
+    s->state = STATE_COLLECTING_DATA;
+}
+
 static void decode_new_cmd(Flash *s, uint32_t value)
 {
     s->cmd_in_progress = value;
+    int i;
     DB_PRINT_L(0, "decoded new command:%x\n", value);
 
+    if (value != RESET_MEMORY) {
+        s->reset_enable = false;
+    }
+
     switch (value) {
 
     case ERASE_4K:
+    case ERASE4_4K:
     case ERASE_32K:
+    case ERASE4_32K:
     case ERASE_SECTOR:
+    case ERASE4_SECTOR:
     case READ:
+    case READ4:
     case DPP:
     case QPP:
     case PP:
-        s->needed_bytes = 3;
+    case PP4:
+    case PP4_4:
+        s->needed_bytes = get_addr_length(s);
         s->pos = 0;
         s->len = 0;
         s->state = STATE_COLLECTING_DATA;
         break;
 
     case FAST_READ:
+    case FAST_READ4:
     case DOR:
+    case DOR4:
     case QOR:
-        s->needed_bytes = 4;
-        s->pos = 0;
-        s->len = 0;
-        s->state = STATE_COLLECTING_DATA;
+    case QOR4:
+        decode_fast_read_cmd(s);
         break;
 
     case DIOR:
-        switch ((s->pi->jedec >> 16) & 0xFF) {
-        case JEDEC_WINBOND:
-        case JEDEC_SPANSION:
-            s->needed_bytes = 4;
-            break;
-        case JEDEC_NUMONYX:
-        default:
-            s->needed_bytes = 5;
-        }
-        s->pos = 0;
-        s->len = 0;
-        s->state = STATE_COLLECTING_DATA;
+    case DIOR4:
+        decode_dio_read_cmd(s);
         break;
 
     case QIOR:
-        switch ((s->pi->jedec >> 16) & 0xFF) {
-        case JEDEC_WINBOND:
-        case JEDEC_SPANSION:
-            s->needed_bytes = 6;
-            break;
-        case JEDEC_NUMONYX:
-        default:
-            s->needed_bytes = 8;
-        }
-        s->pos = 0;
-        s->len = 0;
-        s->state = STATE_COLLECTING_DATA;
+    case QIOR4:
+        decode_qio_read_cmd(s);
         break;
 
     case WRSR:
         if (s->write_enable) {
-            s->needed_bytes = 1;
+            switch (get_man(s)) {
+            case MAN_SPANSION:
+                s->needed_bytes = 2;
+                s->state = STATE_COLLECTING_DATA;
+                break;
+            case MAN_MACRONIX:
+                s->needed_bytes = 2;
+                s->state = STATE_COLLECTING_VAR_LEN_DATA;
+                break;
+            default:
+                s->needed_bytes = 1;
+                s->state = STATE_COLLECTING_DATA;
+            }
             s->pos = 0;
-            s->len = 0;
-            s->state = STATE_COLLECTING_DATA;
         }
         break;
 
@@ -509,6 +929,19 @@
 
     case RDSR:
         s->data[0] = (!!s->write_enable) << 1;
+        if (get_man(s) == MAN_MACRONIX) {
+            s->data[0] |= (!!s->quad_enable) << 6;
+        }
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+
+    case READ_FSR:
+        s->data[0] = FSR_FLASH_READY;
+        if (s->four_bytes_address_mode) {
+            s->data[0] |= FSR_4BYTE_ADDR_MODE_ENABLED;
+        }
         s->pos = 0;
         s->len = 1;
         s->state = STATE_READING_DATA;
@@ -516,20 +949,23 @@
 
     case JEDEC_READ:
         DB_PRINT_L(0, "populated jedec code\n");
-        s->data[0] = (s->pi->jedec >> 16) & 0xff;
-        s->data[1] = (s->pi->jedec >> 8) & 0xff;
-        s->data[2] = s->pi->jedec & 0xff;
-        if (s->pi->ext_jedec) {
-            s->data[3] = (s->pi->ext_jedec >> 8) & 0xff;
-            s->data[4] = s->pi->ext_jedec & 0xff;
-            s->len = 5;
-        } else {
-            s->len = 3;
+        for (i = 0; i < s->pi->id_len; i++) {
+            s->data[i] = s->pi->id[i];
         }
+
+        s->len = s->pi->id_len;
         s->pos = 0;
         s->state = STATE_READING_DATA;
         break;
 
+    case RDCR:
+        s->data[0] = s->volatile_cfg & 0xFF;
+        s->data[0] |= (!!s->four_bytes_address_mode) << 5;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+
     case BULK_ERASE:
         if (s->write_enable) {
             DB_PRINT_L(0, "chip erase\n");
@@ -541,6 +977,95 @@
         break;
     case NOP:
         break;
+    case EN_4BYTE_ADDR:
+        s->four_bytes_address_mode = true;
+        break;
+    case EX_4BYTE_ADDR:
+        s->four_bytes_address_mode = false;
+        break;
+    case EXTEND_ADDR_READ:
+        s->data[0] = s->ear;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+    case EXTEND_ADDR_WRITE:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case RNVCR:
+        s->data[0] = s->nonvolatile_cfg & 0xFF;
+        s->data[1] = (s->nonvolatile_cfg >> 8) & 0xFF;
+        s->pos = 0;
+        s->len = 2;
+        s->state = STATE_READING_DATA;
+        break;
+    case WNVCR:
+        if (s->write_enable && get_man(s) == MAN_NUMONYX) {
+            s->needed_bytes = 2;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case RVCR:
+        s->data[0] = s->volatile_cfg & 0xFF;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+    case WVCR:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case REVCR:
+        s->data[0] = s->enh_volatile_cfg & 0xFF;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+    case WEVCR:
+        if (s->write_enable) {
+            s->needed_bytes = 1;
+            s->pos = 0;
+            s->len = 0;
+            s->state = STATE_COLLECTING_DATA;
+        }
+        break;
+    case RESET_ENABLE:
+        s->reset_enable = true;
+        break;
+    case RESET_MEMORY:
+        if (s->reset_enable) {
+            reset_memory(s);
+        }
+        break;
+    case RDCR_EQIO:
+        switch (get_man(s)) {
+        case MAN_SPANSION:
+            s->data[0] = (!!s->quad_enable) << 1;
+            s->pos = 0;
+            s->len = 1;
+            s->state = STATE_READING_DATA;
+            break;
+        case MAN_MACRONIX:
+            s->quad_enable = true;
+            break;
+        default:
+            break;
+        }
+        break;
+    case RSTQIO:
+        s->quad_enable = false;
+        break;
     default:
         qemu_log_mask(LOG_GUEST_ERROR, "M25P80: Unknown cmd %x\n", value);
         break;
@@ -552,6 +1077,9 @@
     Flash *s = M25P80(ss);
 
     if (select) {
+        if (s->state == STATE_COLLECTING_VAR_LEN_DATA) {
+            complete_collecting_data(s);
+        }
         s->len = 0;
         s->pos = 0;
         s->state = STATE_IDLE;
@@ -571,20 +1099,21 @@
     switch (s->state) {
 
     case STATE_PAGE_PROGRAM:
-        DB_PRINT_L(1, "page program cur_addr=%#" PRIx64 " data=%" PRIx8 "\n",
+        DB_PRINT_L(1, "page program cur_addr=%#" PRIx32 " data=%" PRIx8 "\n",
                    s->cur_addr, (uint8_t)tx);
         flash_write8(s, s->cur_addr, (uint8_t)tx);
-        s->cur_addr++;
+        s->cur_addr = (s->cur_addr + 1) & (s->size - 1);
         break;
 
     case STATE_READ:
         r = s->storage[s->cur_addr];
-        DB_PRINT_L(1, "READ 0x%" PRIx64 "=%" PRIx8 "\n", s->cur_addr,
+        DB_PRINT_L(1, "READ 0x%" PRIx32 "=%" PRIx8 "\n", s->cur_addr,
                    (uint8_t)r);
-        s->cur_addr = (s->cur_addr + 1) % s->size;
+        s->cur_addr = (s->cur_addr + 1) & (s->size - 1);
         break;
 
     case STATE_COLLECTING_DATA:
+    case STATE_COLLECTING_VAR_LEN_DATA:
         s->data[s->len] = (uint8_t)tx;
         s->len++;
 
@@ -611,9 +1140,8 @@
     return r;
 }
 
-static int m25p80_init(SSISlave *ss)
+static void m25p80_realize(SSISlave *ss, Error **errp)
 {
-    DriveInfo *dinfo;
     Flash *s = M25P80(ss);
     M25P80Class *mc = M25P80_GET_CLASS(s);
 
@@ -621,26 +1149,27 @@
 
     s->size = s->pi->sector_size * s->pi->n_sectors;
     s->dirty_page = -1;
-    s->storage = blk_blockalign(s->blk, s->size);
 
-    dinfo = drive_get_next(IF_MTD);
-
-    if (dinfo) {
+    if (s->blk) {
         DB_PRINT_L(0, "Binding to IF_MTD drive\n");
-        s->blk = blk_by_legacy_dinfo(dinfo);
+        s->storage = blk_blockalign(s->blk, s->size);
 
-        /* FIXME: Move to late init */
-        if (blk_read(s->blk, 0, s->storage,
-                     DIV_ROUND_UP(s->size, BDRV_SECTOR_SIZE))) {
-            fprintf(stderr, "Failed to initialize SPI flash!\n");
-            return 1;
+        if (blk_pread(s->blk, 0, s->storage, s->size) != s->size) {
+            error_setg(errp, "failed to read the initial flash content");
+            return;
         }
     } else {
         DB_PRINT_L(0, "No BDRV - binding to RAM\n");
+        s->storage = blk_blockalign(NULL, s->size);
         memset(s->storage, 0xFF, s->size);
     }
+}
 
-    return 0;
+static void m25p80_reset(DeviceState *d)
+{
+    Flash *s = M25P80(d);
+
+    reset_memory(s);
 }
 
 static void m25p80_pre_save(void *opaque)
@@ -648,9 +1177,20 @@
     flash_sync_dirty((Flash *)opaque, -1);
 }
 
+static Property m25p80_properties[] = {
+    /* This is default value for Micron flash */
+    DEFINE_PROP_UINT32("nonvolatile-cfg", Flash, nonvolatile_cfg, 0x8FFF),
+    DEFINE_PROP_UINT8("spansion-cr1nv", Flash, spansion_cr1nv, 0x0),
+    DEFINE_PROP_UINT8("spansion-cr2nv", Flash, spansion_cr2nv, 0x8),
+    DEFINE_PROP_UINT8("spansion-cr3nv", Flash, spansion_cr3nv, 0x2),
+    DEFINE_PROP_UINT8("spansion-cr4nv", Flash, spansion_cr4nv, 0x10),
+    DEFINE_PROP_DRIVE("drive", Flash, blk),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static const VMStateDescription vmstate_m25p80 = {
     .name = "xilinx_spi",
-    .version_id = 1,
+    .version_id = 3,
     .minimum_version_id = 1,
     .pre_save = m25p80_pre_save,
     .fields = (VMStateField[]) {
@@ -660,8 +1200,20 @@
         VMSTATE_UINT32(pos, Flash),
         VMSTATE_UINT8(needed_bytes, Flash),
         VMSTATE_UINT8(cmd_in_progress, Flash),
-        VMSTATE_UINT64(cur_addr, Flash),
+        VMSTATE_UNUSED(4),
+        VMSTATE_UINT32(cur_addr, Flash),
         VMSTATE_BOOL(write_enable, Flash),
+        VMSTATE_BOOL_V(reset_enable, Flash, 2),
+        VMSTATE_UINT8_V(ear, Flash, 2),
+        VMSTATE_BOOL_V(four_bytes_address_mode, Flash, 2),
+        VMSTATE_UINT32_V(nonvolatile_cfg, Flash, 2),
+        VMSTATE_UINT32_V(volatile_cfg, Flash, 2),
+        VMSTATE_UINT32_V(enh_volatile_cfg, Flash, 2),
+        VMSTATE_BOOL_V(quad_enable, Flash, 3),
+        VMSTATE_UINT8_V(spansion_cr1nv, Flash, 3),
+        VMSTATE_UINT8_V(spansion_cr2nv, Flash, 3),
+        VMSTATE_UINT8_V(spansion_cr3nv, Flash, 3),
+        VMSTATE_UINT8_V(spansion_cr4nv, Flash, 3),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -672,11 +1224,13 @@
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
     M25P80Class *mc = M25P80_CLASS(klass);
 
-    k->init = m25p80_init;
+    k->realize = m25p80_realize;
     k->transfer = m25p80_transfer8;
     k->set_cs = m25p80_cs;
     k->cs_polarity = SSI_CS_LOW;
     dc->vmsd = &vmstate_m25p80;
+    dc->props = m25p80_properties;
+    dc->reset = m25p80_reset;
     mc->pi = data;
 }
 
diff --git a/hw/block/nand.c b/hw/block/nand.c
index 1882a0c..c69e675 100644
--- a/hw/block/nand.c
+++ b/hw/block/nand.c
@@ -18,10 +18,12 @@
 
 #ifndef NAND_IO
 
-# include "hw/hw.h"
-# include "hw/block/flash.h"
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 
 # define NAND_CMD_READ0		0x00
@@ -393,7 +395,7 @@
         nand_init_2048(s);
         break;
     default:
-        error_setg(errp, "Unsupported NAND block size %#x\n",
+        error_setg(errp, "Unsupported NAND block size %#x",
                    1 << s->page_shift);
         return;
     }
@@ -522,8 +524,8 @@
 
     if (s->ale) {
         unsigned int shift = s->addrlen * 8;
-        unsigned int mask = ~(0xff << shift);
-        unsigned int v = value << shift;
+        uint64_t mask = ~(0xffull << shift);
+        uint64_t v = (uint64_t)value << shift;
 
         s->addr = (s->addr & mask) | v;
         s->addrlen ++;
@@ -635,7 +637,7 @@
     qdev_prop_set_uint8(dev, "manufacturer_id", manf_id);
     qdev_prop_set_uint8(dev, "chip_id", chip_id);
     if (blk) {
-        qdev_prop_set_drive_nofail(dev, "drive", blk);
+        qdev_prop_set_drive(dev, "drive", blk, &error_fatal);
     }
 
     qdev_init_nofail(dev);
@@ -661,7 +663,8 @@
         sector = SECTOR(s->addr);
         off = (s->addr & PAGE_MASK) + s->offset;
         soff = SECTOR_OFFSET(s->addr);
-        if (blk_read(s->blk, sector, iobuf, PAGE_SECTORS) < 0) {
+        if (blk_pread(s->blk, sector << BDRV_SECTOR_BITS, iobuf,
+                      PAGE_SECTORS << BDRV_SECTOR_BITS) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
@@ -673,21 +676,24 @@
                             MIN(OOB_SIZE, off + s->iolen - PAGE_SIZE));
         }
 
-        if (blk_write(s->blk, sector, iobuf, PAGE_SECTORS) < 0) {
+        if (blk_pwrite(s->blk, sector << BDRV_SECTOR_BITS, iobuf,
+                       PAGE_SECTORS << BDRV_SECTOR_BITS, 0) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
         }
     } else {
         off = PAGE_START(s->addr) + (s->addr & PAGE_MASK) + s->offset;
         sector = off >> 9;
         soff = off & 0x1ff;
-        if (blk_read(s->blk, sector, iobuf, PAGE_SECTORS + 2) < 0) {
+        if (blk_pread(s->blk, sector << BDRV_SECTOR_BITS, iobuf,
+                      (PAGE_SECTORS + 2) << BDRV_SECTOR_BITS) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
 
         mem_and(iobuf + soff, s->io, s->iolen);
 
-        if (blk_write(s->blk, sector, iobuf, PAGE_SECTORS + 2) < 0) {
+        if (blk_pwrite(s->blk, sector << BDRV_SECTOR_BITS, iobuf,
+                       (PAGE_SECTORS + 2) << BDRV_SECTOR_BITS, 0) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
         }
     }
@@ -712,19 +718,22 @@
         memset(s->storage + (PAGE(addr) << OOB_SHIFT),
                         0xff, OOB_SIZE << s->erase_shift);
         i = SECTOR(addr);
-        page = SECTOR(addr + (ADDR_SHIFT + s->erase_shift));
+        page = SECTOR(addr + (1 << (ADDR_SHIFT + s->erase_shift)));
         for (; i < page; i ++)
-            if (blk_write(s->blk, i, iobuf, 1) < 0) {
+            if (blk_pwrite(s->blk, i << BDRV_SECTOR_BITS, iobuf,
+                           BDRV_SECTOR_SIZE, 0) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n", __func__, i);
             }
     } else {
         addr = PAGE_START(addr);
         page = addr >> 9;
-        if (blk_read(s->blk, page, iobuf, 1) < 0) {
+        if (blk_pread(s->blk, page << BDRV_SECTOR_BITS, iobuf,
+                      BDRV_SECTOR_SIZE) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
         }
         memset(iobuf + (addr & 0x1ff), 0xff, (~addr & 0x1ff) + 1);
-        if (blk_write(s->blk, page, iobuf, 1) < 0) {
+        if (blk_pwrite(s->blk, page << BDRV_SECTOR_BITS, iobuf,
+                       BDRV_SECTOR_SIZE, 0) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
         }
 
@@ -732,18 +741,20 @@
         i = (addr & ~0x1ff) + 0x200;
         for (addr += ((PAGE_SIZE + OOB_SIZE) << s->erase_shift) - 0x200;
                         i < addr; i += 0x200) {
-            if (blk_write(s->blk, i >> 9, iobuf, 1) < 0) {
+            if (blk_pwrite(s->blk, i, iobuf, BDRV_SECTOR_SIZE, 0) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n",
                        __func__, i >> 9);
             }
         }
 
         page = i >> 9;
-        if (blk_read(s->blk, page, iobuf, 1) < 0) {
+        if (blk_pread(s->blk, page << BDRV_SECTOR_BITS, iobuf,
+                      BDRV_SECTOR_SIZE) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
         }
         memset(iobuf, 0xff, ((addr - 1) & 0x1ff) + 1);
-        if (blk_write(s->blk, page, iobuf, 1) < 0) {
+        if (blk_pwrite(s->blk, page << BDRV_SECTOR_BITS, iobuf,
+                       BDRV_SECTOR_SIZE, 0) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
         }
     }
@@ -758,7 +769,8 @@
 
     if (s->blk) {
         if (s->mem_oob) {
-            if (blk_read(s->blk, SECTOR(addr), s->io, PAGE_SECTORS) < 0) {
+            if (blk_pread(s->blk, SECTOR(addr) << BDRV_SECTOR_BITS, s->io,
+                          PAGE_SECTORS << BDRV_SECTOR_BITS) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, SECTOR(addr));
             }
@@ -767,8 +779,8 @@
                             OOB_SIZE);
             s->ioaddr = s->io + SECTOR_OFFSET(s->addr) + offset;
         } else {
-            if (blk_read(s->blk, PAGE_START(addr) >> 9,
-                         s->io, (PAGE_SECTORS + 2)) < 0) {
+            if (blk_pread(s->blk, PAGE_START(addr), s->io,
+                          (PAGE_SECTORS + 2) << BDRV_SECTOR_BITS) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, PAGE_START(addr) >> 9);
             }
diff --git a/hw/block/nvme.c b/hw/block/nvme.c
index 1327658..cef3bb4 100644
--- a/hw/block/nvme.c
+++ b/hw/block/nvme.c
@@ -20,11 +20,13 @@
  *      -device nvme,drive=<drive_id>,serial=<serial>,id=<id[optional]>
  */
 
-#include <hw/block/block.h>
-#include <hw/hw.h>
-#include <hw/pci/msix.h>
-#include <hw/pci/pci.h>
+#include "qemu/osdep.h"
+#include "hw/block/block.h"
+#include "hw/hw.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/pci.h"
 #include "sysemu/sysemu.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 #include "sysemu/block-backend.h"
 
@@ -154,6 +156,7 @@
         qemu_sglist_destroy(&qsg);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
+    qemu_sglist_destroy(&qsg);
     return NVME_SUCCESS;
 }
 
@@ -200,17 +203,30 @@
     NvmeCtrl *n = sq->ctrl;
     NvmeCQueue *cq = n->cq[sq->cqid];
 
-    block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
     if (!ret) {
+        block_acct_done(blk_get_stats(n->conf.blk), &req->acct);
         req->status = NVME_SUCCESS;
     } else {
+        block_acct_failed(blk_get_stats(n->conf.blk), &req->acct);
         req->status = NVME_INTERNAL_DEV_ERROR;
     }
-
-    qemu_sglist_destroy(&req->qsg);
+    if (req->has_sg) {
+        qemu_sglist_destroy(&req->qsg);
+    }
     nvme_enqueue_req_completion(cq, req);
 }
 
+static uint16_t nvme_flush(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
+    NvmeRequest *req)
+{
+    req->has_sg = false;
+    block_acct_start(blk_get_stats(n->conf.blk), &req->acct, 0,
+         BLOCK_ACCT_FLUSH);
+    req->aiocb = blk_aio_flush(n->conf.blk, nvme_rw_cb, req);
+
+    return NVME_NO_COMPLETE;
+}
+
 static uint16_t nvme_rw(NvmeCtrl *n, NvmeNamespace *ns, NvmeCmd *cmd,
     NvmeRequest *req)
 {
@@ -222,23 +238,28 @@
 
     uint8_t lba_index  = NVME_ID_NS_FLBAS_INDEX(ns->id_ns.flbas);
     uint8_t data_shift = ns->id_ns.lbaf[lba_index].ds;
-    uint64_t data_size = nlb << data_shift;
-    uint64_t aio_slba  = slba << (data_shift - BDRV_SECTOR_BITS);
+    uint64_t data_size = (uint64_t)nlb << data_shift;
+    uint64_t data_offset = slba << data_shift;
     int is_write = rw->opcode == NVME_CMD_WRITE ? 1 : 0;
+    enum BlockAcctType acct = is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ;
 
     if ((slba + nlb) > ns->id_ns.nsze) {
+        block_acct_invalid(blk_get_stats(n->conf.blk), acct);
         return NVME_LBA_RANGE | NVME_DNR;
     }
+
     if (nvme_map_prp(&req->qsg, prp1, prp2, data_size, n)) {
+        block_acct_invalid(blk_get_stats(n->conf.blk), acct);
         return NVME_INVALID_FIELD | NVME_DNR;
     }
+
     assert((nlb << data_shift) == req->qsg.size);
 
-    dma_acct_start(n->conf.blk, &req->acct, &req->qsg,
-                   is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
+    req->has_sg = true;
+    dma_acct_start(n->conf.blk, &req->acct, &req->qsg, acct);
     req->aiocb = is_write ?
-        dma_blk_write(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req) :
-        dma_blk_read(n->conf.blk, &req->qsg, aio_slba, nvme_rw_cb, req);
+        dma_blk_write(n->conf.blk, &req->qsg, data_offset, nvme_rw_cb, req) :
+        dma_blk_read(n->conf.blk, &req->qsg, data_offset, nvme_rw_cb, req);
 
     return NVME_NO_COMPLETE;
 }
@@ -255,7 +276,7 @@
     ns = &n->namespaces[nsid - 1];
     switch (cmd->opcode) {
     case NVME_CMD_FLUSH:
-        return NVME_SUCCESS;
+        return nvme_flush(n, ns, cmd, req);
     case NVME_CMD_WRITE:
     case NVME_CMD_READ:
         return nvme_rw(n, ns, cmd, req);
@@ -448,19 +469,22 @@
     return NVME_SUCCESS;
 }
 
-static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
+static uint16_t nvme_identify_ctrl(NvmeCtrl *n, NvmeIdentify *c)
+{
+    uint64_t prp1 = le64_to_cpu(c->prp1);
+    uint64_t prp2 = le64_to_cpu(c->prp2);
+
+    return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
+        prp1, prp2);
+}
+
+static uint16_t nvme_identify_ns(NvmeCtrl *n, NvmeIdentify *c)
 {
     NvmeNamespace *ns;
-    NvmeIdentify *c = (NvmeIdentify *)cmd;
-    uint32_t cns  = le32_to_cpu(c->cns);
     uint32_t nsid = le32_to_cpu(c->nsid);
     uint64_t prp1 = le64_to_cpu(c->prp1);
     uint64_t prp2 = le64_to_cpu(c->prp2);
 
-    if (cns) {
-        return nvme_dma_read_prp(n, (uint8_t *)&n->id_ctrl, sizeof(n->id_ctrl),
-            prp1, prp2);
-    }
     if (nsid == 0 || nsid > n->num_namespaces) {
         return NVME_INVALID_NSID | NVME_DNR;
     }
@@ -470,27 +494,80 @@
         prp1, prp2);
 }
 
+static uint16_t nvme_identify_nslist(NvmeCtrl *n, NvmeIdentify *c)
+{
+    static const int data_len = 4096;
+    uint32_t min_nsid = le32_to_cpu(c->nsid);
+    uint64_t prp1 = le64_to_cpu(c->prp1);
+    uint64_t prp2 = le64_to_cpu(c->prp2);
+    uint32_t *list;
+    uint16_t ret;
+    int i, j = 0;
+
+    list = g_malloc0(data_len);
+    for (i = 0; i < n->num_namespaces; i++) {
+        if (i < min_nsid) {
+            continue;
+        }
+        list[j++] = cpu_to_le32(i + 1);
+        if (j == data_len / sizeof(uint32_t)) {
+            break;
+        }
+    }
+    ret = nvme_dma_read_prp(n, (uint8_t *)list, data_len, prp1, prp2);
+    g_free(list);
+    return ret;
+}
+
+
+static uint16_t nvme_identify(NvmeCtrl *n, NvmeCmd *cmd)
+{
+    NvmeIdentify *c = (NvmeIdentify *)cmd;
+
+    switch (le32_to_cpu(c->cns)) {
+    case 0x00:
+        return nvme_identify_ns(n, c);
+    case 0x01:
+        return nvme_identify_ctrl(n, c);
+    case 0x02:
+        return nvme_identify_nslist(n, c);
+    default:
+        return NVME_INVALID_FIELD | NVME_DNR;
+    }
+}
+
 static uint16_t nvme_get_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 {
     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
+    uint32_t result;
 
     switch (dw10) {
+    case NVME_VOLATILE_WRITE_CACHE:
+        result = blk_enable_write_cache(n->conf.blk);
+        break;
     case NVME_NUMBER_OF_QUEUES:
-        req->cqe.result = cpu_to_le32(n->num_queues);
+        result = cpu_to_le32((n->num_queues - 1) | ((n->num_queues - 1) << 16));
         break;
     default:
         return NVME_INVALID_FIELD | NVME_DNR;
     }
+
+    req->cqe.result = result;
     return NVME_SUCCESS;
 }
 
 static uint16_t nvme_set_feature(NvmeCtrl *n, NvmeCmd *cmd, NvmeRequest *req)
 {
     uint32_t dw10 = le32_to_cpu(cmd->cdw10);
+    uint32_t dw11 = le32_to_cpu(cmd->cdw11);
 
     switch (dw10) {
+    case NVME_VOLATILE_WRITE_CACHE:
+        blk_set_enable_write_cache(n->conf.blk, dw11 & 1);
+        break;
     case NVME_NUMBER_OF_QUEUES:
-        req->cqe.result = cpu_to_le32(n->num_queues);
+        req->cqe.result =
+            cpu_to_le32((n->num_queues - 1) | ((n->num_queues - 1) << 16));
         break;
     default:
         return NVME_INVALID_FIELD | NVME_DNR;
@@ -613,6 +690,13 @@
         n->bar.intmc = n->bar.intms;
         break;
     case 0x14:
+        /* Windows first sends data, then sends enable bit */
+        if (!NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc) &&
+            !NVME_CC_SHN(data) && !NVME_CC_SHN(n->bar.cc))
+        {
+            n->bar.cc = data;
+        }
+
         if (NVME_CC_EN(data) && !NVME_CC_EN(n->bar.cc)) {
             n->bar.cc = data;
             if (nvme_start_ctrl(n)) {
@@ -763,6 +847,8 @@
     if (!n->serial) {
         return -1;
     }
+    blkconf_blocksizes(&n->conf);
+    blkconf_apply_backend_options(&n->conf);
 
     pci_conf = pci_dev->config;
     pci_conf[PCI_INTERRUPT_PIN] = 1;
@@ -772,7 +858,7 @@
 
     n->num_namespaces = 1;
     n->num_queues = 64;
-    n->reg_size = 1 << qemu_fls(0x1004 + 2 * (n->num_queues + 1) * 4);
+    n->reg_size = pow2ceil(0x1004 + 2 * (n->num_queues + 1) * 4);
     n->ns_size = bs_size / (uint64_t)n->num_namespaces;
 
     n->namespaces = g_new0(NvmeNamespace, n->num_namespaces);
@@ -804,6 +890,9 @@
     id->psd[0].mp = cpu_to_le16(0x9c4);
     id->psd[0].enlat = cpu_to_le32(0x10);
     id->psd[0].exlat = cpu_to_le32(0x4);
+    if (blk_enable_write_cache(n->conf.blk)) {
+        id->vwc = 1;
+    }
 
     n->bar.cap = 0;
     NVME_CAP_SET_MQES(n->bar.cap, 0x7ff);
@@ -811,8 +900,9 @@
     NVME_CAP_SET_AMS(n->bar.cap, 1);
     NVME_CAP_SET_TO(n->bar.cap, 0xf);
     NVME_CAP_SET_CSS(n->bar.cap, 1);
+    NVME_CAP_SET_MPSMAX(n->bar.cap, 4);
 
-    n->bar.vs = 0x00010001;
+    n->bar.vs = 0x00010100;
     n->bar.intmc = n->bar.intms = 0;
 
     for (i = 0; i < n->num_namespaces; i++) {
@@ -864,7 +954,7 @@
     pc->class_id = PCI_CLASS_STORAGE_EXPRESS;
     pc->vendor_id = PCI_VENDOR_ID_INTEL;
     pc->device_id = 0x5845;
-    pc->revision = 1;
+    pc->revision = 2;
     pc->is_express = 1;
 
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
@@ -873,45 +963,13 @@
     dc->vmsd = &nvme_vmstate;
 }
 
-static void nvme_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
-{
-    NvmeCtrl *s = NVME(obj);
-
-    visit_type_int32(v, &s->conf.bootindex, name, errp);
-}
-
-static void nvme_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
-{
-    NvmeCtrl *s = NVME(obj);
-    int32_t boot_index;
-    Error *local_err = NULL;
-
-    visit_type_int32(v, &boot_index, name, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    /* check whether bootindex is present in fw_boot_order list  */
-    check_boot_index(boot_index, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    /* change bootindex to a new one */
-    s->conf.bootindex = boot_index;
-
-out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-}
-
 static void nvme_instance_init(Object *obj)
 {
-    object_property_add(obj, "bootindex", "int32",
-                        nvme_get_bootindex,
-                        nvme_set_bootindex, NULL, NULL, NULL);
-    object_property_set_int(obj, -1, "bootindex", NULL);
+    NvmeCtrl *s = NVME(obj);
+
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/namespace@1,0",
+                                  DEVICE(obj), &error_abort);
 }
 
 static const TypeInfo nvme_info = {
diff --git a/hw/block/nvme.h b/hw/block/nvme.h
index 993c511..8fb0c10 100644
--- a/hw/block/nvme.h
+++ b/hw/block/nvme.h
@@ -1,5 +1,6 @@
 #ifndef HW_NVME_H
 #define HW_NVME_H
+#include "qemu/cutils.h"
 
 typedef struct NvmeBar {
     uint64_t    cap;
@@ -638,6 +639,7 @@
     struct NvmeSQueue       *sq;
     BlockAIOCB              *aiocb;
     uint16_t                status;
+    bool                    has_sg;
     NvmeCqe                 cqe;
     BlockAcctCookie         acct;
     QEMUSGList              qsg;
@@ -688,7 +690,7 @@
     NvmeBar      bar;
     BlockConf    conf;
 
-    uint16_t    page_size;
+    uint32_t    page_size;
     uint16_t    page_bits;
     uint16_t    max_prp_ents;
     uint16_t    cqe_size;
diff --git a/hw/block/onenand.c b/hw/block/onenand.c
index 348630d..8d84227 100644
--- a/hw/block/onenand.c
+++ b/hw/block/onenand.c
@@ -18,6 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
@@ -222,7 +224,8 @@
         /* Lock the whole flash */
         memset(s->blockwp, ONEN_LOCK_LOCKED, s->blocks);
 
-        if (s->blk_cur && blk_read(s->blk_cur, 0, s->boot[0], 8) < 0) {
+        if (s->blk_cur && blk_pread(s->blk_cur, 0, s->boot[0],
+                                    8 << BDRV_SECTOR_BITS) < 0) {
             hw_error("%s: Loading the BootRAM failed.\n", __func__);
         }
     }
@@ -238,8 +241,11 @@
 static inline int onenand_load_main(OneNANDState *s, int sec, int secn,
                 void *dest)
 {
+    assert(UINT32_MAX >> BDRV_SECTOR_BITS > sec);
+    assert(UINT32_MAX >> BDRV_SECTOR_BITS > secn);
     if (s->blk_cur) {
-        return blk_read(s->blk_cur, sec, dest, secn) < 0;
+        return blk_pread(s->blk_cur, sec << BDRV_SECTOR_BITS, dest,
+                         secn << BDRV_SECTOR_BITS) < 0;
     } else if (sec + secn > s->secs_cur) {
         return 1;
     }
@@ -255,19 +261,22 @@
     int result = 0;
 
     if (secn > 0) {
-        uint32_t size = (uint32_t)secn * 512;
+        uint32_t size = secn << BDRV_SECTOR_BITS;
+        uint32_t offset = sec << BDRV_SECTOR_BITS;
+        assert(UINT32_MAX >> BDRV_SECTOR_BITS > sec);
+        assert(UINT32_MAX >> BDRV_SECTOR_BITS > secn);
         const uint8_t *sp = (const uint8_t *)src;
         uint8_t *dp = 0;
         if (s->blk_cur) {
             dp = g_malloc(size);
-            if (!dp || blk_read(s->blk_cur, sec, dp, secn) < 0) {
+            if (!dp || blk_pread(s->blk_cur, offset, dp, size) < 0) {
                 result = 1;
             }
         } else {
             if (sec + secn > s->secs_cur) {
                 result = 1;
             } else {
-                dp = (uint8_t *)s->current + (sec << 9);
+                dp = (uint8_t *)s->current + offset;
             }
         }
         if (!result) {
@@ -276,7 +285,7 @@
                 dp[i] &= sp[i];
             }
             if (s->blk_cur) {
-                result = blk_write(s->blk_cur, sec, dp, secn) < 0;
+                result = blk_pwrite(s->blk_cur, offset, dp, size, 0) < 0;
             }
         }
         if (dp && s->blk_cur) {
@@ -293,7 +302,8 @@
     uint8_t buf[512];
 
     if (s->blk_cur) {
-        if (blk_read(s->blk_cur, s->secs_cur + (sec >> 5), buf, 1) < 0) {
+        uint32_t offset = (s->secs_cur + (sec >> 5)) << BDRV_SECTOR_BITS;
+        if (blk_pread(s->blk_cur, offset, buf, BDRV_SECTOR_SIZE) < 0) {
             return 1;
         }
         memcpy(dest, buf + ((sec & 31) << 4), secn << 4);
@@ -302,7 +312,7 @@
     } else {
         memcpy(dest, s->current + (s->secs_cur << 9) + (sec << 4), secn << 4);
     }
- 
+
     return 0;
 }
 
@@ -313,10 +323,12 @@
     if (secn > 0) {
         const uint8_t *sp = (const uint8_t *)src;
         uint8_t *dp = 0, *dpp = 0;
+        uint32_t offset = (s->secs_cur + (sec >> 5)) << BDRV_SECTOR_BITS;
+        assert(UINT32_MAX >> BDRV_SECTOR_BITS > s->secs_cur + (sec >> 5));
         if (s->blk_cur) {
             dp = g_malloc(512);
             if (!dp
-                || blk_read(s->blk_cur, s->secs_cur + (sec >> 5), dp, 1) < 0) {
+                || blk_pread(s->blk_cur, offset, dp, BDRV_SECTOR_SIZE) < 0) {
                 result = 1;
             } else {
                 dpp = dp + ((sec & 31) << 4);
@@ -334,8 +346,8 @@
                 dpp[i] &= sp[i];
             }
             if (s->blk_cur) {
-                result = blk_write(s->blk_cur, s->secs_cur + (sec >> 5),
-                                   dp, 1) < 0;
+                result = blk_pwrite(s->blk_cur, offset, dp,
+                                    BDRV_SECTOR_SIZE, 0) < 0;
             }
         }
         g_free(dp);
@@ -346,27 +358,24 @@
 static inline int onenand_erase(OneNANDState *s, int sec, int num)
 {
     uint8_t *blankbuf, *tmpbuf;
+
     blankbuf = g_malloc(512);
-    if (!blankbuf) {
-        return 1;
-    }
     tmpbuf = g_malloc(512);
-    if (!tmpbuf) {
-        g_free(blankbuf);
-        return 1;
-    }
     memset(blankbuf, 0xff, 512);
     for (; num > 0; num--, sec++) {
         if (s->blk_cur) {
             int erasesec = s->secs_cur + (sec >> 5);
-            if (blk_write(s->blk_cur, sec, blankbuf, 1) < 0) {
+            if (blk_pwrite(s->blk_cur, sec << BDRV_SECTOR_BITS, blankbuf,
+                           BDRV_SECTOR_SIZE, 0) < 0) {
                 goto fail;
             }
-            if (blk_read(s->blk_cur, erasesec, tmpbuf, 1) < 0) {
+            if (blk_pread(s->blk_cur, erasesec << BDRV_SECTOR_BITS, tmpbuf,
+                          BDRV_SECTOR_SIZE) < 0) {
                 goto fail;
             }
             memcpy(tmpbuf + ((sec & 31) << 4), blankbuf, 1 << 4);
-            if (blk_write(s->blk_cur, erasesec, tmpbuf, 1) < 0) {
+            if (blk_pwrite(s->blk_cur, erasesec << BDRV_SECTOR_BITS, tmpbuf,
+                           BDRV_SECTOR_SIZE, 0) < 0) {
                 goto fail;
             }
         } else {
@@ -792,7 +801,7 @@
     s->otp = memset(g_malloc((64 + 2) << PAGE_SHIFT),
                     0xff, (64 + 2) << PAGE_SHIFT);
     memory_region_init_ram(&s->ram, OBJECT(s), "onenand.ram",
-                           0xc000 << s->shift, &error_abort);
+                           0xc000 << s->shift, &error_fatal);
     vmstate_register_ram_global(&s->ram);
     ram = memory_region_get_ram_ptr(&s->ram);
     s->boot[0] = ram + (0x0000 << s->shift);
diff --git a/hw/block/pflash_cfi01.c b/hw/block/pflash_cfi01.c
index 89d380e..62d7a56 100644
--- a/hw/block/pflash_cfi01.c
+++ b/hw/block/pflash_cfi01.c
@@ -36,14 +36,18 @@
  * It does not implement much more ...
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 #include "sysemu/block-backend.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
 #include "exec/address-spaces.h"
 #include "qemu/host-utils.h"
+#include "qemu/log.h"
 #include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
 
 #define PFLASH_BUG(fmt, ...) \
 do { \
@@ -61,9 +65,11 @@
 #define DPRINTF(fmt, ...) do { } while (0)
 #endif
 
-#define TYPE_CFI_PFLASH01 "cfi.pflash01"
 #define CFI_PFLASH01(obj) OBJECT_CHECK(pflash_t, (obj), TYPE_CFI_PFLASH01)
 
+#define PFLASH_BE          0
+#define PFLASH_SECURE      1
+
 struct pflash_t {
     /*< private >*/
     SysBusDevice parent_obj;
@@ -75,7 +81,7 @@
     uint8_t bank_width;
     uint8_t device_width; /* If 0, device width not specified. */
     uint8_t max_device_width;  /* max device width in bytes */
-    uint8_t be;
+    uint32_t features;
     uint8_t wcycle; /* if 0, the flash is read normally */
     int ro;
     uint8_t cmd;
@@ -92,6 +98,7 @@
     MemoryRegion mem;
     char *name;
     void *storage;
+    VMChangeStateEntry *vmstate;
 };
 
 static int pflash_post_load(void *opaque, int version_id);
@@ -235,12 +242,57 @@
     return resp;
 }
 
+static uint32_t pflash_data_read(pflash_t *pfl, hwaddr offset,
+                                 int width, int be)
+{
+    uint8_t *p;
+    uint32_t ret;
+
+    p = pfl->storage;
+    switch (width) {
+    case 1:
+        ret = p[offset];
+        DPRINTF("%s: data offset " TARGET_FMT_plx " %02x\n",
+                __func__, offset, ret);
+        break;
+    case 2:
+        if (be) {
+            ret = p[offset] << 8;
+            ret |= p[offset + 1];
+        } else {
+            ret = p[offset];
+            ret |= p[offset + 1] << 8;
+        }
+        DPRINTF("%s: data offset " TARGET_FMT_plx " %04x\n",
+                __func__, offset, ret);
+        break;
+    case 4:
+        if (be) {
+            ret = p[offset] << 24;
+            ret |= p[offset + 1] << 16;
+            ret |= p[offset + 2] << 8;
+            ret |= p[offset + 3];
+        } else {
+            ret = p[offset];
+            ret |= p[offset + 1] << 8;
+            ret |= p[offset + 2] << 16;
+            ret |= p[offset + 3] << 24;
+        }
+        DPRINTF("%s: data offset " TARGET_FMT_plx " %08x\n",
+                __func__, offset, ret);
+        break;
+    default:
+        DPRINTF("BUG in %s\n", __func__);
+        abort();
+    }
+    return ret;
+}
+
 static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
                              int width, int be)
 {
     hwaddr boff;
     uint32_t ret;
-    uint8_t *p;
 
     ret = -1;
 
@@ -257,43 +309,7 @@
         /* fall through to read code */
     case 0x00:
         /* Flash area read */
-        p = pfl->storage;
-        switch (width) {
-        case 1:
-            ret = p[offset];
-            DPRINTF("%s: data offset " TARGET_FMT_plx " %02x\n",
-                    __func__, offset, ret);
-            break;
-        case 2:
-            if (be) {
-                ret = p[offset] << 8;
-                ret |= p[offset + 1];
-            } else {
-                ret = p[offset];
-                ret |= p[offset + 1] << 8;
-            }
-            DPRINTF("%s: data offset " TARGET_FMT_plx " %04x\n",
-                    __func__, offset, ret);
-            break;
-        case 4:
-            if (be) {
-                ret = p[offset] << 24;
-                ret |= p[offset + 1] << 16;
-                ret |= p[offset + 2] << 8;
-                ret |= p[offset + 3];
-            } else {
-                ret = p[offset];
-                ret |= p[offset + 1] << 8;
-                ret |= p[offset + 2] << 16;
-                ret |= p[offset + 3] << 24;
-            }
-            DPRINTF("%s: data offset " TARGET_FMT_plx " %08x\n",
-                    __func__, offset, ret);
-            break;
-        default:
-            DPRINTF("BUG in %s\n", __func__);
-        }
-
+        ret = pflash_data_read(pfl, offset, width, be);
         break;
     case 0x10: /* Single byte program */
     case 0x20: /* Block erase */
@@ -397,11 +413,11 @@
     int offset_end;
     if (pfl->blk) {
         offset_end = offset + size;
-        /* round to sectors */
-        offset = offset >> 9;
-        offset_end = (offset_end + 511) >> 9;
-        blk_write(pfl->blk, offset, pfl->storage + (offset << 9),
-                  offset_end - offset);
+        /* widen to sector boundaries */
+        offset = QEMU_ALIGN_DOWN(offset, BDRV_SECTOR_SIZE);
+        offset_end = QEMU_ALIGN_UP(offset_end, BDRV_SECTOR_SIZE);
+        blk_pwrite(pfl->blk, offset, pfl->storage + offset,
+                   offset_end - offset, 0);
     }
 }
 
@@ -648,101 +664,37 @@
 }
 
 
-static uint32_t pflash_readb_be(void *opaque, hwaddr addr)
-{
-    return pflash_read(opaque, addr, 1, 1);
-}
-
-static uint32_t pflash_readb_le(void *opaque, hwaddr addr)
-{
-    return pflash_read(opaque, addr, 1, 0);
-}
-
-static uint32_t pflash_readw_be(void *opaque, hwaddr addr)
+static MemTxResult pflash_mem_read_with_attrs(void *opaque, hwaddr addr, uint64_t *value,
+                                              unsigned len, MemTxAttrs attrs)
 {
     pflash_t *pfl = opaque;
+    bool be = !!(pfl->features & (1 << PFLASH_BE));
 
-    return pflash_read(pfl, addr, 2, 1);
+    if ((pfl->features & (1 << PFLASH_SECURE)) && !attrs.secure) {
+        *value = pflash_data_read(opaque, addr, len, be);
+    } else {
+        *value = pflash_read(opaque, addr, len, be);
+    }
+    return MEMTX_OK;
 }
 
-static uint32_t pflash_readw_le(void *opaque, hwaddr addr)
+static MemTxResult pflash_mem_write_with_attrs(void *opaque, hwaddr addr, uint64_t value,
+                                               unsigned len, MemTxAttrs attrs)
 {
     pflash_t *pfl = opaque;
+    bool be = !!(pfl->features & (1 << PFLASH_BE));
 
-    return pflash_read(pfl, addr, 2, 0);
+    if ((pfl->features & (1 << PFLASH_SECURE)) && !attrs.secure) {
+        return MEMTX_ERROR;
+    } else {
+        pflash_write(opaque, addr, value, len, be);
+        return MEMTX_OK;
+    }
 }
 
-static uint32_t pflash_readl_be(void *opaque, hwaddr addr)
-{
-    pflash_t *pfl = opaque;
-
-    return pflash_read(pfl, addr, 4, 1);
-}
-
-static uint32_t pflash_readl_le(void *opaque, hwaddr addr)
-{
-    pflash_t *pfl = opaque;
-
-    return pflash_read(pfl, addr, 4, 0);
-}
-
-static void pflash_writeb_be(void *opaque, hwaddr addr,
-                             uint32_t value)
-{
-    pflash_write(opaque, addr, value, 1, 1);
-}
-
-static void pflash_writeb_le(void *opaque, hwaddr addr,
-                             uint32_t value)
-{
-    pflash_write(opaque, addr, value, 1, 0);
-}
-
-static void pflash_writew_be(void *opaque, hwaddr addr,
-                             uint32_t value)
-{
-    pflash_t *pfl = opaque;
-
-    pflash_write(pfl, addr, value, 2, 1);
-}
-
-static void pflash_writew_le(void *opaque, hwaddr addr,
-                             uint32_t value)
-{
-    pflash_t *pfl = opaque;
-
-    pflash_write(pfl, addr, value, 2, 0);
-}
-
-static void pflash_writel_be(void *opaque, hwaddr addr,
-                             uint32_t value)
-{
-    pflash_t *pfl = opaque;
-
-    pflash_write(pfl, addr, value, 4, 1);
-}
-
-static void pflash_writel_le(void *opaque, hwaddr addr,
-                             uint32_t value)
-{
-    pflash_t *pfl = opaque;
-
-    pflash_write(pfl, addr, value, 4, 0);
-}
-
-static const MemoryRegionOps pflash_cfi01_ops_be = {
-    .old_mmio = {
-        .read = { pflash_readb_be, pflash_readw_be, pflash_readl_be, },
-        .write = { pflash_writeb_be, pflash_writew_be, pflash_writel_be, },
-    },
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static const MemoryRegionOps pflash_cfi01_ops_le = {
-    .old_mmio = {
-        .read = { pflash_readb_le, pflash_readw_le, pflash_readl_le, },
-        .write = { pflash_writeb_le, pflash_writew_le, pflash_writel_le, },
-    },
+static const MemoryRegionOps pflash_cfi01_ops = {
+    .read_with_attrs = pflash_mem_read_with_attrs,
+    .write_with_attrs = pflash_mem_write_with_attrs,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
@@ -773,7 +725,8 @@
 
     memory_region_init_rom_device(
         &pfl->mem, OBJECT(dev),
-        pfl->be ? &pflash_cfi01_ops_be : &pflash_cfi01_ops_le, pfl,
+        &pflash_cfi01_ops,
+        pfl,
         pfl->name, total_len, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
@@ -786,7 +739,7 @@
 
     if (pfl->blk) {
         /* read the initial flash content */
-        ret = blk_read(pfl->blk, 0, pfl->storage, total_len >> 9);
+        ret = blk_pread(pfl->blk, 0, pfl->storage, total_len);
 
         if (ret < 0) {
             vmstate_unregister_ram(&pfl->mem, DEVICE(pfl));
@@ -925,7 +878,8 @@
     DEFINE_PROP_UINT8("width", struct pflash_t, bank_width, 0),
     DEFINE_PROP_UINT8("device-width", struct pflash_t, device_width, 0),
     DEFINE_PROP_UINT8("max-device-width", struct pflash_t, max_device_width, 0),
-    DEFINE_PROP_UINT8("big-endian", struct pflash_t, be, 0),
+    DEFINE_PROP_BIT("big-endian", struct pflash_t, features, PFLASH_BE, 0),
+    DEFINE_PROP_BIT("secure", struct pflash_t, features, PFLASH_SECURE, 0),
     DEFINE_PROP_UINT16("id0", struct pflash_t, ident0, 0),
     DEFINE_PROP_UINT16("id1", struct pflash_t, ident1, 0),
     DEFINE_PROP_UINT16("id2", struct pflash_t, ident2, 0),
@@ -969,13 +923,13 @@
 {
     DeviceState *dev = qdev_create(NULL, TYPE_CFI_PFLASH01);
 
-    if (blk && qdev_prop_set_drive(dev, "drive", blk)) {
-        abort();
+    if (blk) {
+        qdev_prop_set_drive(dev, "drive", blk, &error_abort);
     }
     qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
     qdev_prop_set_uint64(dev, "sector-length", sector_len);
     qdev_prop_set_uint8(dev, "width", bank_width);
-    qdev_prop_set_uint8(dev, "big-endian", !!be);
+    qdev_prop_set_bit(dev, "big-endian", !!be);
     qdev_prop_set_uint16(dev, "id0", id0);
     qdev_prop_set_uint16(dev, "id1", id1);
     qdev_prop_set_uint16(dev, "id2", id2);
@@ -992,13 +946,25 @@
     return &fl->mem;
 }
 
+static void postload_update_cb(void *opaque, int running, RunState state)
+{
+    pflash_t *pfl = opaque;
+
+    /* This is called after bdrv_invalidate_cache_all.  */
+    qemu_del_vm_change_state_handler(pfl->vmstate);
+    pfl->vmstate = NULL;
+
+    DPRINTF("%s: updating bdrv for %s\n", __func__, pfl->name);
+    pflash_update(pfl, 0, pfl->sector_len * pfl->nb_blocs);
+}
+
 static int pflash_post_load(void *opaque, int version_id)
 {
     pflash_t *pfl = opaque;
 
     if (!pfl->ro) {
-        DPRINTF("%s: updating bdrv for %s\n", __func__, pfl->name);
-        pflash_update(pfl, 0, pfl->sector_len * pfl->nb_blocs);
+        pfl->vmstate = qemu_add_vm_change_state_handler(postload_update_cb,
+                                                        pfl);
     }
     return 0;
 }
diff --git a/hw/block/pflash_cfi02.c b/hw/block/pflash_cfi02.c
index 8513a17..4f6105c 100644
--- a/hw/block/pflash_cfi02.c
+++ b/hw/block/pflash_cfi02.c
@@ -35,8 +35,10 @@
  * It does not implement multiple sectors erase
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
@@ -55,7 +57,6 @@
 
 #define PFLASH_LAZY_ROMD_THRESHOLD 42
 
-#define TYPE_CFI_PFLASH02 "cfi.pflash02"
 #define CFI_PFLASH02(obj) OBJECT_CHECK(pflash_t, (obj), TYPE_CFI_PFLASH02)
 
 struct pflash_t {
@@ -251,11 +252,11 @@
     int offset_end;
     if (pfl->blk) {
         offset_end = offset + size;
-        /* round to sectors */
-        offset = offset >> 9;
-        offset_end = (offset_end + 511) >> 9;
-        blk_write(pfl->blk, offset, pfl->storage + (offset << 9),
-                  offset_end - offset);
+        /* widen to sector boundaries */
+        offset = QEMU_ALIGN_DOWN(offset, BDRV_SECTOR_SIZE);
+        offset_end = QEMU_ALIGN_UP(offset_end, BDRV_SECTOR_SIZE);
+        blk_pwrite(pfl->blk, offset, pfl->storage + offset,
+                   offset_end - offset, 0);
     }
 }
 
@@ -430,8 +431,8 @@
             }
             pfl->status = 0x00;
             /* Let's wait 5 seconds before chip erase is done */
-            timer_mod(pfl->timer,
-                           qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() * 5));
+            timer_mod(pfl->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      (NANOSECONDS_PER_SECOND * 5));
             break;
         case 0x30:
             /* Sector erase */
@@ -445,8 +446,8 @@
             }
             pfl->status = 0x00;
             /* Let's wait 1/2 second before sector erase is done */
-            timer_mod(pfl->timer,
-                           qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() / 2));
+            timer_mod(pfl->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      (NANOSECONDS_PER_SECOND / 2));
             break;
         default:
             DPRINTF("%s: invalid command %02x (wc 5)\n", __func__, cmd);
@@ -620,7 +621,7 @@
     pfl->chip_len = chip_len;
     if (pfl->blk) {
         /* read the initial flash content */
-        ret = blk_read(pfl->blk, 0, pfl->storage, chip_len >> 9);
+        ret = blk_pread(pfl->blk, 0, pfl->storage, chip_len);
         if (ret < 0) {
             vmstate_unregister_ram(&pfl->orig_mem, DEVICE(pfl));
             error_setg(errp, "failed to read the initial flash content");
@@ -744,6 +745,7 @@
 
     dc->realize = pflash_cfi02_realize;
     dc->props = pflash_cfi02_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo pflash_cfi02_info = {
@@ -772,8 +774,8 @@
 {
     DeviceState *dev = qdev_create(NULL, TYPE_CFI_PFLASH02);
 
-    if (blk && qdev_prop_set_drive(dev, "drive", blk)) {
-        abort();
+    if (blk) {
+        qdev_prop_set_drive(dev, "drive", blk, &error_abort);
     }
     qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
     qdev_prop_set_uint32(dev, "sector-length", sector_len);
diff --git a/hw/block/tc58128.c b/hw/block/tc58128.c
index 728f1c3..1d9f7ee 100644
--- a/hw/block/tc58128.c
+++ b/hw/block/tc58128.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "hw/loader.h"
@@ -44,7 +45,7 @@
             }
 	} else {
 	    /* Build first block with number of blocks */
-	    blocks = (ret + 528 * 32 - 1) / (528 * 32);
+            blocks = DIV_ROUND_UP(ret, 528 * 32);
 	    dev->flash_contents[0] = blocks & 0xff;
 	    dev->flash_contents[1] = (blocks >> 8) & 0xff;
 	    dev->flash_contents[2] = (blocks >> 16) & 0xff;
diff --git a/hw/block/trace-events b/hw/block/trace-events
new file mode 100644
index 0000000..d0dd94f
--- /dev/null
+++ b/hw/block/trace-events
@@ -0,0 +1,17 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/block/virtio-blk.c
+virtio_blk_req_complete(void *req, int status) "req %p status %d"
+virtio_blk_rw_complete(void *req, int ret) "req %p ret %d"
+virtio_blk_handle_write(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
+virtio_blk_handle_read(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
+virtio_blk_submit_multireq(void *mrb, int start, int num_reqs, uint64_t offset, size_t size, bool is_write) "mrb %p start %d num_reqs %d offset %"PRIu64" size %zu is_write %d"
+
+# hw/block/dataplane/virtio-blk.c
+virtio_blk_data_plane_start(void *s) "dataplane %p"
+virtio_blk_data_plane_stop(void *s) "dataplane %p"
+virtio_blk_data_plane_process_request(void *s, unsigned int out_num, unsigned int in_num, unsigned int head) "dataplane %p out_num %u in_num %u head %u"
+
+# hw/block/hd-geometry.c
+hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d"
+hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "blk %p CHS %u %u %u trans %d"
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index b19b102..331d766 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -11,6 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/iov.h"
 #include "qemu/error-report.h"
@@ -20,7 +22,6 @@
 #include "sysemu/blockdev.h"
 #include "hw/virtio/virtio-blk.h"
 #include "dataplane/virtio-blk.h"
-#include "migration/migration.h"
 #include "block/scsi.h"
 #ifdef __linux__
 # include <scsi/sg.h>
@@ -28,24 +29,25 @@
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
 
-VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s)
+void virtio_blk_init_request(VirtIOBlock *s, VirtQueue *vq,
+                             VirtIOBlockReq *req)
 {
-    VirtIOBlockReq *req = g_slice_new(VirtIOBlockReq);
     req->dev = s;
+    req->vq = vq;
     req->qiov.size = 0;
+    req->in_len = 0;
     req->next = NULL;
-    return req;
+    req->mr_next = NULL;
 }
 
 void virtio_blk_free_request(VirtIOBlockReq *req)
 {
     if (req) {
-        g_slice_free(VirtIOBlockReq, req);
+        g_free(req);
     }
 }
 
-static void virtio_blk_complete_request(VirtIOBlockReq *req,
-                                        unsigned char status)
+static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
 {
     VirtIOBlock *s = req->dev;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
@@ -53,13 +55,12 @@
     trace_virtio_blk_req_complete(req, status);
 
     stb_p(&req->in->status, status);
-    virtqueue_push(s->vq, &req->elem, req->qiov.size + sizeof(*req->in));
-    virtio_notify(vdev, s->vq);
-}
-
-static void virtio_blk_req_complete(VirtIOBlockReq *req, unsigned char status)
-{
-    req->dev->complete_request(req, status);
+    virtqueue_push(req->vq, &req->elem, req->in_len);
+    if (s->dataplane_started && !s->dataplane_disabled) {
+        virtio_blk_data_plane_notify(s->dataplane, req->vq);
+    } else {
+        virtio_notify(vdev, req->vq);
+    }
 }
 
 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
@@ -70,11 +71,14 @@
     VirtIOBlock *s = req->dev;
 
     if (action == BLOCK_ERROR_ACTION_STOP) {
+        /* Break the link as the next request is going to be parsed from the
+         * ring again. Otherwise we may end up doing a double completion! */
+        req->mr_next = NULL;
         req->next = s->rq;
         s->rq = req;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
-        block_acct_done(blk_get_stats(s->blk), &req->acct);
+        block_acct_failed(blk_get_stats(s->blk), &req->acct);
         virtio_blk_free_request(req);
     }
 
@@ -84,20 +88,40 @@
 
 static void virtio_blk_rw_complete(void *opaque, int ret)
 {
-    VirtIOBlockReq *req = opaque;
+    VirtIOBlockReq *next = opaque;
 
-    trace_virtio_blk_rw_complete(req, ret);
+    while (next) {
+        VirtIOBlockReq *req = next;
+        next = req->mr_next;
+        trace_virtio_blk_rw_complete(req, ret);
 
-    if (ret) {
-        int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
-        bool is_read = !(p & VIRTIO_BLK_T_OUT);
-        if (virtio_blk_handle_rw_error(req, -ret, is_read))
-            return;
+        if (req->qiov.nalloc != -1) {
+            /* If nalloc is != 1 req->qiov is a local copy of the original
+             * external iovec. It was allocated in submit_merged_requests
+             * to be able to merge requests. */
+            qemu_iovec_destroy(&req->qiov);
+        }
+
+        if (ret) {
+            int p = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
+            bool is_read = !(p & VIRTIO_BLK_T_OUT);
+            /* Note that memory may be dirtied on read failure.  If the
+             * virtio request is not completed here, as is the case for
+             * BLOCK_ERROR_ACTION_STOP, the memory may not be copied
+             * correctly during live migration.  While this is ugly,
+             * it is acceptable because the device is free to write to
+             * the memory until the request is completed (which will
+             * happen on the other side of the migration).
+             */
+            if (virtio_blk_handle_rw_error(req, -ret, is_read)) {
+                continue;
+            }
+        }
+
+        virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
+        block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
+        virtio_blk_free_request(req);
     }
-
-    virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
-    block_acct_done(blk_get_stats(req->dev->blk), &req->acct);
-    virtio_blk_free_request(req);
 }
 
 static void virtio_blk_flush_complete(void *opaque, int ret)
@@ -115,28 +139,78 @@
     virtio_blk_free_request(req);
 }
 
-static VirtIOBlockReq *virtio_blk_get_request(VirtIOBlock *s)
-{
-    VirtIOBlockReq *req = virtio_blk_alloc_request(s);
+#ifdef __linux__
 
-    if (!virtqueue_pop(s->vq, &req->elem)) {
-        virtio_blk_free_request(req);
-        return NULL;
+typedef struct {
+    VirtIOBlockReq *req;
+    struct sg_io_hdr hdr;
+} VirtIOBlockIoctlReq;
+
+static void virtio_blk_ioctl_complete(void *opaque, int status)
+{
+    VirtIOBlockIoctlReq *ioctl_req = opaque;
+    VirtIOBlockReq *req = ioctl_req->req;
+    VirtIODevice *vdev = VIRTIO_DEVICE(req->dev);
+    struct virtio_scsi_inhdr *scsi;
+    struct sg_io_hdr *hdr;
+
+    scsi = (void *)req->elem.in_sg[req->elem.in_num - 2].iov_base;
+
+    if (status) {
+        status = VIRTIO_BLK_S_UNSUPP;
+        virtio_stl_p(vdev, &scsi->errors, 255);
+        goto out;
     }
 
+    hdr = &ioctl_req->hdr;
+    /*
+     * From SCSI-Generic-HOWTO: "Some lower level drivers (e.g. ide-scsi)
+     * clear the masked_status field [hence status gets cleared too, see
+     * block/scsi_ioctl.c] even when a CHECK_CONDITION or COMMAND_TERMINATED
+     * status has occurred.  However they do set DRIVER_SENSE in driver_status
+     * field. Also a (sb_len_wr > 0) indicates there is a sense buffer.
+     */
+    if (hdr->status == 0 && hdr->sb_len_wr > 0) {
+        hdr->status = CHECK_CONDITION;
+    }
+
+    virtio_stl_p(vdev, &scsi->errors,
+                 hdr->status | (hdr->msg_status << 8) |
+                 (hdr->host_status << 16) | (hdr->driver_status << 24));
+    virtio_stl_p(vdev, &scsi->residual, hdr->resid);
+    virtio_stl_p(vdev, &scsi->sense_len, hdr->sb_len_wr);
+    virtio_stl_p(vdev, &scsi->data_len, hdr->dxfer_len);
+
+out:
+    virtio_blk_req_complete(req, status);
+    virtio_blk_free_request(req);
+    g_free(ioctl_req);
+}
+
+#endif
+
+static VirtIOBlockReq *virtio_blk_get_request(VirtIOBlock *s, VirtQueue *vq)
+{
+    VirtIOBlockReq *req = virtqueue_pop(vq, sizeof(VirtIOBlockReq));
+
+    if (req) {
+        virtio_blk_init_request(s, vq, req);
+    }
     return req;
 }
 
-int virtio_blk_handle_scsi_req(VirtIOBlock *blk,
-                               VirtQueueElement *elem)
+static int virtio_blk_handle_scsi_req(VirtIOBlockReq *req)
 {
     int status = VIRTIO_BLK_S_OK;
     struct virtio_scsi_inhdr *scsi = NULL;
-    VirtIODevice *vdev = VIRTIO_DEVICE(blk);
+    VirtIODevice *vdev = VIRTIO_DEVICE(req->dev);
+    VirtQueueElement *elem = &req->elem;
+    VirtIOBlock *blk = req->dev;
 
 #ifdef __linux__
     int i;
-    struct sg_io_hdr hdr;
+    VirtIOBlockIoctlReq *ioctl_req;
+    BlockAIOCB *acb;
 #endif
 
     /*
@@ -171,71 +245,57 @@
     }
 
 #ifdef __linux__
-    memset(&hdr, 0, sizeof(struct sg_io_hdr));
-    hdr.interface_id = 'S';
-    hdr.cmd_len = elem->out_sg[1].iov_len;
-    hdr.cmdp = elem->out_sg[1].iov_base;
-    hdr.dxfer_len = 0;
+    ioctl_req = g_new0(VirtIOBlockIoctlReq, 1);
+    ioctl_req->req = req;
+    ioctl_req->hdr.interface_id = 'S';
+    ioctl_req->hdr.cmd_len = elem->out_sg[1].iov_len;
+    ioctl_req->hdr.cmdp = elem->out_sg[1].iov_base;
+    ioctl_req->hdr.dxfer_len = 0;
 
     if (elem->out_num > 2) {
         /*
          * If there are more than the minimally required 2 output segments
          * there is write payload starting from the third iovec.
          */
-        hdr.dxfer_direction = SG_DXFER_TO_DEV;
-        hdr.iovec_count = elem->out_num - 2;
+        ioctl_req->hdr.dxfer_direction = SG_DXFER_TO_DEV;
+        ioctl_req->hdr.iovec_count = elem->out_num - 2;
 
-        for (i = 0; i < hdr.iovec_count; i++)
-            hdr.dxfer_len += elem->out_sg[i + 2].iov_len;
+        for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
+            ioctl_req->hdr.dxfer_len += elem->out_sg[i + 2].iov_len;
+        }
 
-        hdr.dxferp = elem->out_sg + 2;
+        ioctl_req->hdr.dxferp = elem->out_sg + 2;
 
     } else if (elem->in_num > 3) {
         /*
          * If we have more than 3 input segments the guest wants to actually
          * read data.
          */
-        hdr.dxfer_direction = SG_DXFER_FROM_DEV;
-        hdr.iovec_count = elem->in_num - 3;
-        for (i = 0; i < hdr.iovec_count; i++)
-            hdr.dxfer_len += elem->in_sg[i].iov_len;
+        ioctl_req->hdr.dxfer_direction = SG_DXFER_FROM_DEV;
+        ioctl_req->hdr.iovec_count = elem->in_num - 3;
+        for (i = 0; i < ioctl_req->hdr.iovec_count; i++) {
+            ioctl_req->hdr.dxfer_len += elem->in_sg[i].iov_len;
+        }
 
-        hdr.dxferp = elem->in_sg;
+        ioctl_req->hdr.dxferp = elem->in_sg;
     } else {
         /*
          * Some SCSI commands don't actually transfer any data.
          */
-        hdr.dxfer_direction = SG_DXFER_NONE;
+        ioctl_req->hdr.dxfer_direction = SG_DXFER_NONE;
     }
 
-    hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base;
-    hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len;
+    ioctl_req->hdr.sbp = elem->in_sg[elem->in_num - 3].iov_base;
+    ioctl_req->hdr.mx_sb_len = elem->in_sg[elem->in_num - 3].iov_len;
 
-    status = blk_ioctl(blk->blk, SG_IO, &hdr);
-    if (status) {
+    acb = blk_aio_ioctl(blk->blk, SG_IO, &ioctl_req->hdr,
+                        virtio_blk_ioctl_complete, ioctl_req);
+    if (!acb) {
+        g_free(ioctl_req);
         status = VIRTIO_BLK_S_UNSUPP;
         goto fail;
     }
-
-    /*
-     * From SCSI-Generic-HOWTO: "Some lower level drivers (e.g. ide-scsi)
-     * clear the masked_status field [hence status gets cleared too, see
-     * block/scsi_ioctl.c] even when a CHECK_CONDITION or COMMAND_TERMINATED
-     * status has occurred.  However they do set DRIVER_SENSE in driver_status
-     * field. Also a (sb_len_wr > 0) indicates there is a sense buffer.
-     */
-    if (hdr.status == 0 && hdr.sb_len_wr > 0) {
-        hdr.status = CHECK_CONDITION;
-    }
-
-    virtio_stl_p(vdev, &scsi->errors,
-                 hdr.status | (hdr.msg_status << 8) |
-                 (hdr.host_status << 16) | (hdr.driver_status << 24));
-    virtio_stl_p(vdev, &scsi->residual, hdr.resid);
-    virtio_stl_p(vdev, &scsi->sense_len, hdr.sb_len_wr);
-    virtio_stl_p(vdev, &scsi->data_len, hdr.dxfer_len);
-
-    return status;
+    return -EINPROGRESS;
 #else
     abort();
 #endif
@@ -252,29 +312,124 @@
 {
     int status;
 
-    status = virtio_blk_handle_scsi_req(req->dev, &req->elem);
-    virtio_blk_req_complete(req, status);
-    virtio_blk_free_request(req);
+    status = virtio_blk_handle_scsi_req(req);
+    if (status != -EINPROGRESS) {
+        virtio_blk_req_complete(req, status);
+        virtio_blk_free_request(req);
+    }
 }
 
-void virtio_submit_multiwrite(BlockBackend *blk, MultiReqBuffer *mrb)
+static inline void submit_requests(BlockBackend *blk, MultiReqBuffer *mrb,
+                                   int start, int num_reqs, int niov)
 {
-    int i, ret;
+    QEMUIOVector *qiov = &mrb->reqs[start]->qiov;
+    int64_t sector_num = mrb->reqs[start]->sector_num;
+    bool is_write = mrb->is_write;
 
-    if (!mrb->num_writes) {
+    if (num_reqs > 1) {
+        int i;
+        struct iovec *tmp_iov = qiov->iov;
+        int tmp_niov = qiov->niov;
+
+        /* mrb->reqs[start]->qiov was initialized from external so we can't
+         * modify it here. We need to initialize it locally and then add the
+         * external iovecs. */
+        qemu_iovec_init(qiov, niov);
+
+        for (i = 0; i < tmp_niov; i++) {
+            qemu_iovec_add(qiov, tmp_iov[i].iov_base, tmp_iov[i].iov_len);
+        }
+
+        for (i = start + 1; i < start + num_reqs; i++) {
+            qemu_iovec_concat(qiov, &mrb->reqs[i]->qiov, 0,
+                              mrb->reqs[i]->qiov.size);
+            mrb->reqs[i - 1]->mr_next = mrb->reqs[i];
+        }
+
+        trace_virtio_blk_submit_multireq(mrb, start, num_reqs,
+                                         sector_num << BDRV_SECTOR_BITS,
+                                         qiov->size, is_write);
+        block_acct_merge_done(blk_get_stats(blk),
+                              is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ,
+                              num_reqs - 1);
+    }
+
+    if (is_write) {
+        blk_aio_pwritev(blk, sector_num << BDRV_SECTOR_BITS, qiov, 0,
+                        virtio_blk_rw_complete, mrb->reqs[start]);
+    } else {
+        blk_aio_preadv(blk, sector_num << BDRV_SECTOR_BITS, qiov, 0,
+                       virtio_blk_rw_complete, mrb->reqs[start]);
+    }
+}
+
+static int multireq_compare(const void *a, const void *b)
+{
+    const VirtIOBlockReq *req1 = *(VirtIOBlockReq **)a,
+                         *req2 = *(VirtIOBlockReq **)b;
+
+    /*
+     * Note that we can't simply subtract sector_num1 from sector_num2
+     * here as that could overflow the return value.
+     */
+    if (req1->sector_num > req2->sector_num) {
+        return 1;
+    } else if (req1->sector_num < req2->sector_num) {
+        return -1;
+    } else {
+        return 0;
+    }
+}
+
+void virtio_blk_submit_multireq(BlockBackend *blk, MultiReqBuffer *mrb)
+{
+    int i = 0, start = 0, num_reqs = 0, niov = 0, nb_sectors = 0;
+    uint32_t max_transfer;
+    int64_t sector_num = 0;
+
+    if (mrb->num_reqs == 1) {
+        submit_requests(blk, mrb, 0, 1, -1);
+        mrb->num_reqs = 0;
         return;
     }
 
-    ret = blk_aio_multiwrite(blk, mrb->blkreq, mrb->num_writes);
-    if (ret != 0) {
-        for (i = 0; i < mrb->num_writes; i++) {
-            if (mrb->blkreq[i].error) {
-                virtio_blk_rw_complete(mrb->blkreq[i].opaque, -EIO);
+    max_transfer = blk_get_max_transfer(mrb->reqs[0]->dev->blk);
+
+    qsort(mrb->reqs, mrb->num_reqs, sizeof(*mrb->reqs),
+          &multireq_compare);
+
+    for (i = 0; i < mrb->num_reqs; i++) {
+        VirtIOBlockReq *req = mrb->reqs[i];
+        if (num_reqs > 0) {
+            /*
+             * NOTE: We cannot merge the requests in below situations:
+             * 1. requests are not sequential
+             * 2. merge would exceed maximum number of IOVs
+             * 3. merge would exceed maximum transfer length of backend device
+             */
+            if (sector_num + nb_sectors != req->sector_num ||
+                niov > blk_get_max_iov(blk) - req->qiov.niov ||
+                req->qiov.size > max_transfer ||
+                nb_sectors > (max_transfer -
+                              req->qiov.size) / BDRV_SECTOR_SIZE) {
+                submit_requests(blk, mrb, start, num_reqs, niov);
+                num_reqs = 0;
             }
         }
+
+        if (num_reqs == 0) {
+            sector_num = req->sector_num;
+            nb_sectors = niov = 0;
+            start = i;
+        }
+
+        nb_sectors += req->qiov.size / BDRV_SECTOR_SIZE;
+        niov += req->qiov.niov;
+        num_reqs++;
     }
 
-    mrb->num_writes = 0;
+    submit_requests(blk, mrb, start, num_reqs, niov);
+    mrb->num_reqs = 0;
 }
 
 static void virtio_blk_handle_flush(VirtIOBlockReq *req, MultiReqBuffer *mrb)
@@ -285,7 +440,9 @@
     /*
      * Make sure all outstanding writes are posted to the backing device.
      */
-    virtio_submit_multiwrite(req->dev->blk, mrb);
+    if (mrb->is_write && mrb->num_reqs > 0) {
+        virtio_blk_submit_multireq(req->dev->blk, mrb);
+    }
     blk_aio_flush(req->dev->blk, virtio_blk_flush_complete, req);
 }
 
@@ -295,6 +452,9 @@
     uint64_t nb_sectors = size >> BDRV_SECTOR_BITS;
     uint64_t total_sectors;
 
+    if (nb_sectors > BDRV_REQUEST_MAX_SECTORS) {
+        return false;
+    }
     if (sector & dev->sector_mask) {
         return false;
     }
@@ -308,60 +468,6 @@
     return true;
 }
 
-static void virtio_blk_handle_write(VirtIOBlockReq *req, MultiReqBuffer *mrb)
-{
-    BlockRequest *blkreq;
-    uint64_t sector;
-
-    sector = virtio_ldq_p(VIRTIO_DEVICE(req->dev), &req->out.sector);
-
-    trace_virtio_blk_handle_write(req, sector, req->qiov.size / 512);
-
-    if (!virtio_blk_sect_range_ok(req->dev, sector, req->qiov.size)) {
-        virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
-        virtio_blk_free_request(req);
-        return;
-    }
-
-    block_acct_start(blk_get_stats(req->dev->blk), &req->acct, req->qiov.size,
-                     BLOCK_ACCT_WRITE);
-
-    if (mrb->num_writes == 32) {
-        virtio_submit_multiwrite(req->dev->blk, mrb);
-    }
-
-    blkreq = &mrb->blkreq[mrb->num_writes];
-    blkreq->sector = sector;
-    blkreq->nb_sectors = req->qiov.size / BDRV_SECTOR_SIZE;
-    blkreq->qiov = &req->qiov;
-    blkreq->cb = virtio_blk_rw_complete;
-    blkreq->opaque = req;
-    blkreq->error = 0;
-
-    mrb->num_writes++;
-}
-
-static void virtio_blk_handle_read(VirtIOBlockReq *req)
-{
-    uint64_t sector;
-
-    sector = virtio_ldq_p(VIRTIO_DEVICE(req->dev), &req->out.sector);
-
-    trace_virtio_blk_handle_read(req, sector, req->qiov.size / 512);
-
-    if (!virtio_blk_sect_range_ok(req->dev, sector, req->qiov.size)) {
-        virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
-        virtio_blk_free_request(req);
-        return;
-    }
-
-    block_acct_start(blk_get_stats(req->dev->blk), &req->acct, req->qiov.size,
-                     BLOCK_ACCT_READ);
-    blk_aio_readv(req->dev->blk, sector, &req->qiov,
-                  req->qiov.size / BDRV_SECTOR_SIZE,
-                  virtio_blk_rw_complete, req);
-}
-
 void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb)
 {
     uint32_t type;
@@ -383,12 +489,13 @@
 
     iov_discard_front(&iov, &out_num, sizeof(req->out));
 
-    if (in_num < 1 ||
-        in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
+    if (in_iov[in_num - 1].iov_len < sizeof(struct virtio_blk_inhdr)) {
         error_report("virtio-blk request inhdr too short");
         exit(1);
     }
 
+    /* We always touch the last byte, so just see how big in_iov is.  */
+    req->in_len = iov_size(in_iov, in_num);
     req->in = (void *)in_iov[in_num - 1].iov_base
               + in_iov[in_num - 1].iov_len
               - sizeof(struct virtio_blk_inhdr);
@@ -396,11 +503,60 @@
 
     type = virtio_ldl_p(VIRTIO_DEVICE(req->dev), &req->out.type);
 
-    if (type & VIRTIO_BLK_T_FLUSH) {
+    /* VIRTIO_BLK_T_OUT defines the command direction. VIRTIO_BLK_T_BARRIER
+     * is an optional flag. Although a guest should not send this flag if
+     * not negotiated we ignored it in the past. So keep ignoring it. */
+    switch (type & ~(VIRTIO_BLK_T_OUT | VIRTIO_BLK_T_BARRIER)) {
+    case VIRTIO_BLK_T_IN:
+    {
+        bool is_write = type & VIRTIO_BLK_T_OUT;
+        req->sector_num = virtio_ldq_p(VIRTIO_DEVICE(req->dev),
+                                       &req->out.sector);
+
+        if (is_write) {
+            qemu_iovec_init_external(&req->qiov, iov, out_num);
+            trace_virtio_blk_handle_write(req, req->sector_num,
+                                          req->qiov.size / BDRV_SECTOR_SIZE);
+        } else {
+            qemu_iovec_init_external(&req->qiov, in_iov, in_num);
+            trace_virtio_blk_handle_read(req, req->sector_num,
+                                         req->qiov.size / BDRV_SECTOR_SIZE);
+        }
+
+        if (!virtio_blk_sect_range_ok(req->dev, req->sector_num,
+                                      req->qiov.size)) {
+            virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
+            block_acct_invalid(blk_get_stats(req->dev->blk),
+                               is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
+            virtio_blk_free_request(req);
+            return;
+        }
+
+        block_acct_start(blk_get_stats(req->dev->blk),
+                         &req->acct, req->qiov.size,
+                         is_write ? BLOCK_ACCT_WRITE : BLOCK_ACCT_READ);
+
+        /* merge would exceed maximum number of requests or IO direction
+         * changes */
+        if (mrb->num_reqs > 0 && (mrb->num_reqs == VIRTIO_BLK_MAX_MERGE_REQS ||
+                                  is_write != mrb->is_write ||
+                                  !req->dev->conf.request_merging)) {
+            virtio_blk_submit_multireq(req->dev->blk, mrb);
+        }
+
+        assert(mrb->num_reqs < VIRTIO_BLK_MAX_MERGE_REQS);
+        mrb->reqs[mrb->num_reqs++] = req;
+        mrb->is_write = is_write;
+        break;
+    }
+    case VIRTIO_BLK_T_FLUSH:
         virtio_blk_handle_flush(req, mrb);
-    } else if (type & VIRTIO_BLK_T_SCSI_CMD) {
+        break;
+    case VIRTIO_BLK_T_SCSI_CMD:
         virtio_blk_handle_scsi(req);
-    } else if (type & VIRTIO_BLK_T_GET_ID) {
+        break;
+    case VIRTIO_BLK_T_GET_ID:
+    {
         VirtIOBlock *s = req->dev;
 
         /*
@@ -414,55 +570,53 @@
         iov_from_buf(in_iov, in_num, 0, serial, size);
         virtio_blk_req_complete(req, VIRTIO_BLK_S_OK);
         virtio_blk_free_request(req);
-    } else if (type & VIRTIO_BLK_T_OUT) {
-        qemu_iovec_init_external(&req->qiov, iov, out_num);
-        virtio_blk_handle_write(req, mrb);
-    } else if (type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_BARRIER) {
-        /* VIRTIO_BLK_T_IN is 0, so we can't just & it. */
-        qemu_iovec_init_external(&req->qiov, in_iov, in_num);
-        virtio_blk_handle_read(req);
-    } else {
+        break;
+    }
+    default:
         virtio_blk_req_complete(req, VIRTIO_BLK_S_UNSUPP);
         virtio_blk_free_request(req);
     }
 }
 
-static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq)
 {
-    VirtIOBlock *s = VIRTIO_BLK(vdev);
     VirtIOBlockReq *req;
-    MultiReqBuffer mrb = {
-        .num_writes = 0,
-    };
+    MultiReqBuffer mrb = {};
 
-    /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
-     * dataplane here instead of waiting for .set_status().
-     */
-    if (s->dataplane) {
-        virtio_blk_data_plane_start(s->dataplane);
-        return;
-    }
+    blk_io_plug(s->blk);
 
-    while ((req = virtio_blk_get_request(s))) {
+    while ((req = virtio_blk_get_request(s, vq))) {
         virtio_blk_handle_request(req, &mrb);
     }
 
-    virtio_submit_multiwrite(s->blk, &mrb);
+    if (mrb.num_reqs) {
+        virtio_blk_submit_multireq(s->blk, &mrb);
+    }
 
-    /*
-     * FIXME: Want to check for completions before returning to guest mode,
-     * so cached reads and writes are reported as quickly as possible. But
-     * that should be done in the generic block layer.
-     */
+    blk_io_unplug(s->blk);
+}
+
+static void virtio_blk_handle_output(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOBlock *s = (VirtIOBlock *)vdev;
+
+    if (s->dataplane) {
+        /* Some guests kick before setting VIRTIO_CONFIG_S_DRIVER_OK so start
+         * dataplane here instead of waiting for .set_status().
+         */
+        virtio_blk_data_plane_start(s->dataplane);
+        if (!s->dataplane_disabled) {
+            return;
+        }
+    }
+    virtio_blk_handle_vq(s, vq);
 }
 
 static void virtio_blk_dma_restart_bh(void *opaque)
 {
     VirtIOBlock *s = opaque;
     VirtIOBlockReq *req = s->rq;
-    MultiReqBuffer mrb = {
-        .num_writes = 0,
-    };
+    MultiReqBuffer mrb = {};
 
     qemu_bh_delete(s->bh);
     s->bh = NULL;
@@ -475,7 +629,9 @@
         req = next;
     }
 
-    virtio_submit_multiwrite(s->blk, &mrb);
+    if (mrb.num_reqs) {
+        virtio_blk_submit_multireq(s->blk, &mrb);
+    }
 }
 
 static void virtio_blk_dma_restart_cb(void *opaque, int running,
@@ -497,16 +653,26 @@
 static void virtio_blk_reset(VirtIODevice *vdev)
 {
     VirtIOBlock *s = VIRTIO_BLK(vdev);
+    AioContext *ctx;
+    VirtIOBlockReq *req;
+
+    ctx = blk_get_aio_context(s->blk);
+    aio_context_acquire(ctx);
+    blk_drain(s->blk);
+
+    /* We drop queued requests after blk_drain() because blk_drain() itself can
+     * produce them. */
+    while (s->rq) {
+        req = s->rq;
+        s->rq = req->next;
+        virtio_blk_free_request(req);
+    }
 
     if (s->dataplane) {
         virtio_blk_data_plane_stop(s->dataplane);
     }
+    aio_context_release(ctx);
 
-    /*
-     * This should cancel pending requests, but can't do nicely until there
-     * are per-device request lists.
-     */
-    blk_drain_all();
     blk_set_enable_write_cache(s->blk, s->original_wce);
 }
 
@@ -524,11 +690,11 @@
     memset(&blkcfg, 0, sizeof(blkcfg));
     virtio_stq_p(vdev, &blkcfg.capacity, capacity);
     virtio_stl_p(vdev, &blkcfg.seg_max, 128 - 2);
-    virtio_stw_p(vdev, &blkcfg.cylinders, conf->cyls);
+    virtio_stw_p(vdev, &blkcfg.geometry.cylinders, conf->cyls);
     virtio_stl_p(vdev, &blkcfg.blk_size, blk_size);
     virtio_stw_p(vdev, &blkcfg.min_io_size, conf->min_io_size / blk_size);
     virtio_stw_p(vdev, &blkcfg.opt_io_size, conf->opt_io_size / blk_size);
-    blkcfg.heads = conf->heads;
+    blkcfg.geometry.heads = conf->heads;
     /*
      * We must ensure that the block device capacity is a multiple of
      * the logical block size. If that is not the case, let's use
@@ -541,14 +707,15 @@
      * per track (cylinder).
      */
     if (blk_getlength(s->blk) /  conf->heads / conf->secs % blk_size) {
-        blkcfg.sectors = conf->secs & ~s->sector_mask;
+        blkcfg.geometry.sectors = conf->secs & ~s->sector_mask;
     } else {
-        blkcfg.sectors = conf->secs;
+        blkcfg.geometry.sectors = conf->secs;
     }
     blkcfg.size_max = 0;
     blkcfg.physical_block_exp = get_physical_block_exp(conf);
     blkcfg.alignment_offset = 0;
     blkcfg.wce = blk_enable_write_cache(s->blk);
+    virtio_stw_p(vdev, &blkcfg.num_queues, s->conf.num_queues);
     memcpy(config, &blkcfg, sizeof(struct virtio_blk_config));
 }
 
@@ -564,24 +731,36 @@
     aio_context_release(blk_get_aio_context(s->blk));
 }
 
-static uint32_t virtio_blk_get_features(VirtIODevice *vdev, uint32_t features)
+static uint64_t virtio_blk_get_features(VirtIODevice *vdev, uint64_t features,
+                                        Error **errp)
 {
     VirtIOBlock *s = VIRTIO_BLK(vdev);
 
-    features |= (1 << VIRTIO_BLK_F_SEG_MAX);
-    features |= (1 << VIRTIO_BLK_F_GEOMETRY);
-    features |= (1 << VIRTIO_BLK_F_TOPOLOGY);
-    features |= (1 << VIRTIO_BLK_F_BLK_SIZE);
-    features |= (1 << VIRTIO_BLK_F_SCSI);
+    virtio_add_feature(&features, VIRTIO_BLK_F_SEG_MAX);
+    virtio_add_feature(&features, VIRTIO_BLK_F_GEOMETRY);
+    virtio_add_feature(&features, VIRTIO_BLK_F_TOPOLOGY);
+    virtio_add_feature(&features, VIRTIO_BLK_F_BLK_SIZE);
+    if (virtio_has_feature(features, VIRTIO_F_VERSION_1)) {
+        if (s->conf.scsi) {
+            error_setg(errp, "Please set scsi=off for virtio-blk devices in order to use virtio 1.0");
+            return 0;
+        }
+    } else {
+        virtio_clear_feature(&features, VIRTIO_F_ANY_LAYOUT);
+        virtio_add_feature(&features, VIRTIO_BLK_F_SCSI);
+    }
 
     if (s->conf.config_wce) {
-        features |= (1 << VIRTIO_BLK_F_CONFIG_WCE);
+        virtio_add_feature(&features, VIRTIO_BLK_F_CONFIG_WCE);
     }
     if (blk_enable_write_cache(s->blk)) {
-        features |= (1 << VIRTIO_BLK_F_WCE);
+        virtio_add_feature(&features, VIRTIO_BLK_F_WCE);
     }
     if (blk_is_read_only(s->blk)) {
-        features |= 1 << VIRTIO_BLK_F_RO;
+        virtio_add_feature(&features, VIRTIO_BLK_F_RO);
+    }
+    if (s->conf.num_queues > 1) {
+        virtio_add_feature(&features, VIRTIO_BLK_F_MQ);
     }
 
     return features;
@@ -590,7 +769,6 @@
 static void virtio_blk_set_status(VirtIODevice *vdev, uint8_t status)
 {
     VirtIOBlock *s = VIRTIO_BLK(vdev);
-    uint32_t features;
 
     if (s->dataplane && !(status & (VIRTIO_CONFIG_S_DRIVER |
                                     VIRTIO_CONFIG_S_DRIVER_OK))) {
@@ -601,8 +779,6 @@
         return;
     }
 
-    features = vdev->guest_features;
-
     /* A guest that supports VIRTIO_BLK_F_CONFIG_WCE must be able to send
      * cache flushes.  Thus, the "auto writethrough" behavior is never
      * necessary for guests that support the VIRTIO_BLK_F_CONFIG_WCE feature.
@@ -618,15 +794,16 @@
      *
      * s->blk would erroneously be placed in writethrough mode.
      */
-    if (!(features & (1 << VIRTIO_BLK_F_CONFIG_WCE))) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE)) {
         aio_context_acquire(blk_get_aio_context(s->blk));
         blk_set_enable_write_cache(s->blk,
-                                   !!(features & (1 << VIRTIO_BLK_F_WCE)));
+                                   virtio_vdev_has_feature(vdev,
+                                                           VIRTIO_BLK_F_WCE));
         aio_context_release(blk_get_aio_context(s->blk));
     }
 }
 
-static void virtio_blk_save(QEMUFile *f, void *opaque)
+static void virtio_blk_save(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
 
@@ -640,22 +817,23 @@
 
     while (req) {
         qemu_put_sbyte(f, 1);
-        qemu_put_buffer(f, (unsigned char *)&req->elem,
-                        sizeof(VirtQueueElement));
+
+        if (s->conf.num_queues > 1) {
+            qemu_put_be32(f, virtio_get_queue_index(req->vq));
+        }
+
+        qemu_put_virtqueue_element(f, &req->elem);
         req = req->next;
     }
     qemu_put_sbyte(f, 0);
 }
 
-static int virtio_blk_load(QEMUFile *f, void *opaque, int version_id)
+static int virtio_blk_load(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIOBlock *s = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
-    if (version_id != 2)
-        return -EINVAL;
-
-    return virtio_load(vdev, f, version_id);
+    return virtio_load(vdev, f, 2);
 }
 
 static int virtio_blk_load_device(VirtIODevice *vdev, QEMUFile *f,
@@ -664,16 +842,24 @@
     VirtIOBlock *s = VIRTIO_BLK(vdev);
 
     while (qemu_get_sbyte(f)) {
-        VirtIOBlockReq *req = virtio_blk_alloc_request(s);
-        qemu_get_buffer(f, (unsigned char *)&req->elem,
-                        sizeof(VirtQueueElement));
+        unsigned nvqs = s->conf.num_queues;
+        unsigned vq_idx = 0;
+        VirtIOBlockReq *req;
+
+        if (nvqs > 1) {
+            vq_idx = qemu_get_be32(f);
+
+            if (vq_idx >= nvqs) {
+                error_report("Invalid virtqueue index in request list: %#x",
+                             vq_idx);
+                return -EINVAL;
+            }
+        }
+
+        req = qemu_get_virtqueue_element(f, sizeof(VirtIOBlockReq));
+        virtio_blk_init_request(s, virtio_get_queue(vdev, vq_idx), req);
         req->next = s->rq;
         s->rq = req;
-
-        virtqueue_map_sg(req->elem.in_sg, req->elem.in_addr,
-            req->elem.in_num, 1);
-        virtqueue_map_sg(req->elem.out_sg, req->elem.out_addr,
-            req->elem.out_num, 0);
     }
 
     return 0;
@@ -690,44 +876,13 @@
     .resize_cb = virtio_blk_resize,
 };
 
-/* Disable dataplane thread during live migration since it does not
- * update the dirty memory bitmap yet.
- */
-static void virtio_blk_migration_state_changed(Notifier *notifier, void *data)
-{
-    VirtIOBlock *s = container_of(notifier, VirtIOBlock,
-                                  migration_state_notifier);
-    MigrationState *mig = data;
-    Error *err = NULL;
-
-    if (migration_in_setup(mig)) {
-        if (!s->dataplane) {
-            return;
-        }
-        virtio_blk_data_plane_destroy(s->dataplane);
-        s->dataplane = NULL;
-    } else if (migration_has_finished(mig) ||
-               migration_has_failed(mig)) {
-        if (s->dataplane) {
-            return;
-        }
-        blk_drain_all(); /* complete in-flight non-dataplane requests */
-        virtio_blk_data_plane_create(VIRTIO_DEVICE(s), &s->conf,
-                                     &s->dataplane, &err);
-        if (err != NULL) {
-            error_report("%s", error_get_pretty(err));
-            error_free(err);
-        }
-    }
-}
-
 static void virtio_blk_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
     VirtIOBlkConf *conf = &s->conf;
     Error *err = NULL;
-    static int virtio_blk_id;
+    unsigned i;
 
     if (!conf->conf.blk) {
         error_setg(errp, "drive property not set");
@@ -737,14 +892,20 @@
         error_setg(errp, "Device needs media, but drive is empty");
         return;
     }
+    if (!conf->num_queues) {
+        error_setg(errp, "num-queues property must be larger than 0");
+        return;
+    }
 
     blkconf_serial(&conf->conf, &conf->serial);
+    blkconf_apply_backend_options(&conf->conf);
     s->original_wce = blk_enable_write_cache(conf->conf.blk);
     blkconf_geometry(&conf->conf, NULL, 65535, 255, 255, &err);
     if (err) {
         error_propagate(errp, err);
         return;
     }
+    blkconf_blocksizes(&conf->conf);
 
     virtio_init(vdev, "virtio-blk", VIRTIO_ID_BLOCK,
                 sizeof(struct virtio_blk_config));
@@ -753,20 +914,17 @@
     s->rq = NULL;
     s->sector_mask = (s->conf.conf.logical_block_size / BDRV_SECTOR_SIZE) - 1;
 
-    s->vq = virtio_add_queue(vdev, 128, virtio_blk_handle_output);
-    s->complete_request = virtio_blk_complete_request;
+    for (i = 0; i < conf->num_queues; i++) {
+        virtio_add_queue_aio(vdev, 128, virtio_blk_handle_output);
+    }
     virtio_blk_data_plane_create(vdev, conf, &s->dataplane, &err);
     if (err != NULL) {
         error_propagate(errp, err);
         virtio_cleanup(vdev);
         return;
     }
-    s->migration_state_notifier.notify = virtio_blk_migration_state_changed;
-    add_migration_state_change_notifier(&s->migration_state_notifier);
 
     s->change = qemu_add_vm_change_state_handler(virtio_blk_dma_restart_cb, s);
-    register_savevm(dev, "virtio-blk", virtio_blk_id++, 2,
-                    virtio_blk_save, virtio_blk_load, s);
     blk_set_dev_ops(s->blk, &virtio_block_ops, s);
     blk_set_guest_block_size(s->blk, s->conf.conf.logical_block_size);
 
@@ -778,11 +936,9 @@
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOBlock *s = VIRTIO_BLK(dev);
 
-    remove_migration_state_change_notifier(&s->migration_state_notifier);
     virtio_blk_data_plane_destroy(s->dataplane);
     s->dataplane = NULL;
     qemu_del_vm_change_state_handler(s->change);
-    unregister_savevm(dev, "virtio-blk", s);
     blockdev_mark_auto_del(s->blk);
     virtio_cleanup(vdev);
 }
@@ -800,15 +956,20 @@
                                   DEVICE(obj), NULL);
 }
 
+VMSTATE_VIRTIO_DEVICE(blk, 2, virtio_blk_load, virtio_blk_save);
+
 static Property virtio_blk_properties[] = {
     DEFINE_BLOCK_PROPERTIES(VirtIOBlock, conf.conf),
+    DEFINE_BLOCK_ERROR_PROPERTIES(VirtIOBlock, conf.conf),
     DEFINE_BLOCK_CHS_PROPERTIES(VirtIOBlock, conf.conf),
     DEFINE_PROP_STRING("serial", VirtIOBlock, conf.serial),
     DEFINE_PROP_BIT("config-wce", VirtIOBlock, conf.config_wce, 0, true),
 #ifdef __linux__
-    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, true),
+    DEFINE_PROP_BIT("scsi", VirtIOBlock, conf.scsi, 0, false),
 #endif
-    DEFINE_PROP_BIT("x-data-plane", VirtIOBlock, conf.data_plane, 0, false),
+    DEFINE_PROP_BIT("request-merging", VirtIOBlock, conf.request_merging, 0,
+                    true),
+    DEFINE_PROP_UINT16("num-queues", VirtIOBlock, conf.num_queues, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -818,6 +979,7 @@
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 
     dc->props = virtio_blk_properties;
+    dc->vmsd = &vmstate_virtio_blk;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     vdc->realize = virtio_blk_device_realize;
     vdc->unrealize = virtio_blk_device_unrealize;
diff --git a/hw/block/xen_blkif.h b/hw/block/xen_blkif.h
index 711b692..3300b6f 100644
--- a/hw/block/xen_blkif.h
+++ b/hw/block/xen_blkif.h
@@ -1,35 +1,45 @@
-#ifndef __XEN_BLKIF_H__
-#define __XEN_BLKIF_H__
+#ifndef XEN_BLKIF_H
+#define XEN_BLKIF_H
 
 #include <xen/io/ring.h>
 #include <xen/io/blkif.h>
 #include <xen/io/protocols.h>
 
-/* Not a real protocol.  Used to generate ring structs which contain
+/*
+ * Not a real protocol.  Used to generate ring structs which contain
  * the elements common to all protocols only.  This way we get a
  * compiler-checkable way to use common struct elements, so we can
- * avoid using switch(protocol) in a number of places.  */
+ * avoid using switch(protocol) in a number of places.
+ */
 struct blkif_common_request {
-	char dummy;
+    char dummy;
 };
 struct blkif_common_response {
-	char dummy;
+    char dummy;
 };
 
 /* i386 protocol version */
 #pragma pack(push, 4)
 struct blkif_x86_32_request {
-	uint8_t        operation;    /* BLKIF_OP_???                         */
-	uint8_t        nr_segments;  /* number of segments                   */
-	blkif_vdev_t   handle;       /* only for read/write requests         */
-	uint64_t       id;           /* private guest value, echoed in resp  */
-	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
-	struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    uint8_t        operation;        /* BLKIF_OP_???                         */
+    uint8_t        nr_segments;      /* number of segments                   */
+    blkif_vdev_t   handle;           /* only for read/write requests         */
+    uint64_t       id;               /* private guest value, echoed in resp  */
+    blkif_sector_t sector_number;    /* start sector idx on disk (r/w only)  */
+    struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+};
+struct blkif_x86_32_request_discard {
+    uint8_t        operation;        /* BLKIF_OP_DISCARD                     */
+    uint8_t        flag;             /* nr_segments in request struct        */
+    blkif_vdev_t   handle;           /* only for read/write requests         */
+    uint64_t       id;               /* private guest value, echoed in resp  */
+    blkif_sector_t sector_number;    /* start sector idx on disk (r/w only)  */
+    uint64_t       nr_sectors;       /* # of contiguous sectors to discard   */
 };
 struct blkif_x86_32_response {
-	uint64_t        id;              /* copied from request */
-	uint8_t         operation;       /* copied from request */
-	int16_t         status;          /* BLKIF_RSP_???       */
+    uint64_t        id;              /* copied from request */
+    uint8_t         operation;       /* copied from request */
+    int16_t         status;          /* BLKIF_RSP_???       */
 };
 typedef struct blkif_x86_32_request blkif_x86_32_request_t;
 typedef struct blkif_x86_32_response blkif_x86_32_response_t;
@@ -37,79 +47,100 @@
 
 /* x86_64 protocol version */
 struct blkif_x86_64_request {
-	uint8_t        operation;    /* BLKIF_OP_???                         */
-	uint8_t        nr_segments;  /* number of segments                   */
-	blkif_vdev_t   handle;       /* only for read/write requests         */
-	uint64_t       __attribute__((__aligned__(8))) id;
-	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
-	struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+    uint8_t        operation;        /* BLKIF_OP_???                         */
+    uint8_t        nr_segments;      /* number of segments                   */
+    blkif_vdev_t   handle;           /* only for read/write requests         */
+    uint64_t       __attribute__((__aligned__(8))) id;
+    blkif_sector_t sector_number;    /* start sector idx on disk (r/w only)  */
+    struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
+};
+struct blkif_x86_64_request_discard {
+    uint8_t        operation;        /* BLKIF_OP_DISCARD                     */
+    uint8_t        flag;             /* nr_segments in request struct        */
+    blkif_vdev_t   handle;           /* only for read/write requests         */
+    uint64_t       __attribute__((__aligned__(8))) id;
+    blkif_sector_t sector_number;    /* start sector idx on disk (r/w only)  */
+    uint64_t       nr_sectors;       /* # of contiguous sectors to discard   */
 };
 struct blkif_x86_64_response {
-	uint64_t       __attribute__((__aligned__(8))) id;
-	uint8_t         operation;       /* copied from request */
-	int16_t         status;          /* BLKIF_RSP_???       */
+    uint64_t       __attribute__((__aligned__(8))) id;
+    uint8_t         operation;       /* copied from request */
+    int16_t         status;          /* BLKIF_RSP_???       */
 };
 typedef struct blkif_x86_64_request blkif_x86_64_request_t;
 typedef struct blkif_x86_64_response blkif_x86_64_response_t;
 
-DEFINE_RING_TYPES(blkif_common, struct blkif_common_request, struct blkif_common_response);
-DEFINE_RING_TYPES(blkif_x86_32, struct blkif_x86_32_request, struct blkif_x86_32_response);
-DEFINE_RING_TYPES(blkif_x86_64, struct blkif_x86_64_request, struct blkif_x86_64_response);
+DEFINE_RING_TYPES(blkif_common, struct blkif_common_request,
+                  struct blkif_common_response);
+DEFINE_RING_TYPES(blkif_x86_32, struct blkif_x86_32_request,
+                  struct blkif_x86_32_response);
+DEFINE_RING_TYPES(blkif_x86_64, struct blkif_x86_64_request,
+                  struct blkif_x86_64_response);
 
 union blkif_back_rings {
-	blkif_back_ring_t        native;
-	blkif_common_back_ring_t common;
-        blkif_x86_32_back_ring_t x86_32_part;
-        blkif_x86_64_back_ring_t x86_64_part;
+    blkif_back_ring_t        native;
+    blkif_common_back_ring_t common;
+    blkif_x86_32_back_ring_t x86_32_part;
+    blkif_x86_64_back_ring_t x86_64_part;
 };
 typedef union blkif_back_rings blkif_back_rings_t;
 
 enum blkif_protocol {
-	BLKIF_PROTOCOL_NATIVE = 1,
-	BLKIF_PROTOCOL_X86_32 = 2,
-	BLKIF_PROTOCOL_X86_64 = 3,
+    BLKIF_PROTOCOL_NATIVE = 1,
+    BLKIF_PROTOCOL_X86_32 = 2,
+    BLKIF_PROTOCOL_X86_64 = 3,
 };
 
-static inline void blkif_get_x86_32_req(blkif_request_t *dst, blkif_x86_32_request_t *src)
+static inline void blkif_get_x86_32_req(blkif_request_t *dst,
+                                        blkif_x86_32_request_t *src)
 {
-	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
+    int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
 
-	dst->operation = src->operation;
-	dst->nr_segments = src->nr_segments;
-	dst->handle = src->handle;
-	dst->id = src->id;
-	dst->sector_number = src->sector_number;
-	if (src->operation == BLKIF_OP_DISCARD) {
-		struct blkif_request_discard *s = (void *)src;
-		struct blkif_request_discard *d = (void *)dst;
-		d->nr_sectors = s->nr_sectors;
-		return;
-	}
-	if (n > src->nr_segments)
-		n = src->nr_segments;
-	for (i = 0; i < n; i++)
-		dst->seg[i] = src->seg[i];
+    dst->operation = src->operation;
+    dst->nr_segments = src->nr_segments;
+    dst->handle = src->handle;
+    dst->id = src->id;
+    dst->sector_number = src->sector_number;
+    /* Prevent the compiler from using src->... instead. */
+    barrier();
+    if (dst->operation == BLKIF_OP_DISCARD) {
+        struct blkif_x86_32_request_discard *s = (void *)src;
+        struct blkif_request_discard *d = (void *)dst;
+        d->nr_sectors = s->nr_sectors;
+        return;
+    }
+    if (n > dst->nr_segments) {
+        n = dst->nr_segments;
+    }
+    for (i = 0; i < n; i++) {
+        dst->seg[i] = src->seg[i];
+    }
 }
 
-static inline void blkif_get_x86_64_req(blkif_request_t *dst, blkif_x86_64_request_t *src)
+static inline void blkif_get_x86_64_req(blkif_request_t *dst,
+                                        blkif_x86_64_request_t *src)
 {
-	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
+    int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
 
-	dst->operation = src->operation;
-	dst->nr_segments = src->nr_segments;
-	dst->handle = src->handle;
-	dst->id = src->id;
-	dst->sector_number = src->sector_number;
-	if (src->operation == BLKIF_OP_DISCARD) {
-		struct blkif_request_discard *s = (void *)src;
-		struct blkif_request_discard *d = (void *)dst;
-		d->nr_sectors = s->nr_sectors;
-		return;
-	}
-	if (n > src->nr_segments)
-		n = src->nr_segments;
-	for (i = 0; i < n; i++)
-		dst->seg[i] = src->seg[i];
+    dst->operation = src->operation;
+    dst->nr_segments = src->nr_segments;
+    dst->handle = src->handle;
+    dst->id = src->id;
+    dst->sector_number = src->sector_number;
+    /* Prevent the compiler from using src->... instead. */
+    barrier();
+    if (dst->operation == BLKIF_OP_DISCARD) {
+        struct blkif_x86_64_request_discard *s = (void *)src;
+        struct blkif_request_discard *d = (void *)dst;
+        d->nr_sectors = s->nr_sectors;
+        return;
+    }
+    if (n > dst->nr_segments) {
+        n = dst->nr_segments;
+    }
+    for (i = 0; i < n; i++) {
+        dst->seg[i] = src->seg[i];
+    }
 }
 
-#endif /* __XEN_BLKIF_H__ */
+#endif /* XEN_BLKIF_H */
diff --git a/hw/block/xen_disk.c b/hw/block/xen_disk.c
index 21842a0..3b8ad33 100644
--- a/hw/block/xen_disk.c
+++ b/hw/block/xen_disk.c
@@ -19,20 +19,8 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <signal.h>
-#include <inttypes.h>
-#include <time.h>
-#include <fcntl.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
 #include <sys/uio.h>
 
 #include "hw/hw.h"
@@ -40,6 +28,9 @@
 #include "xen_blkif.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/block-backend.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/qstring.h"
 
 /* ------------------------------------------------------------- */
 
@@ -74,7 +65,6 @@
     off_t               start;
     QEMUIOVector        v;
     int                 presync;
-    int                 postsync;
     uint8_t             mapped;
 
     /* grant mapping */
@@ -143,7 +133,6 @@
     ioreq->status = 0;
     ioreq->start = 0;
     ioreq->presync = 0;
-    ioreq->postsync = 0;
     ioreq->mapped = 0;
 
     memset(ioreq->domids, 0, sizeof(ioreq->domids));
@@ -172,11 +161,11 @@
 static void destroy_grant(gpointer pgnt)
 {
     PersistentGrant *grant = pgnt;
-    XenGnttab gnt = grant->blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = grant->blkdev->xendev.gnttabdev;
 
-    if (xc_gnttab_munmap(gnt, grant->page, 1) != 0) {
+    if (xengnttab_unmap(gnt, grant->page, 1) != 0) {
         xen_be_printf(&grant->blkdev->xendev, 0,
-                      "xc_gnttab_munmap failed: %s\n",
+                      "xengnttab_unmap failed: %s\n",
                       strerror(errno));
     }
     grant->blkdev->persistent_gnt_count--;
@@ -189,11 +178,11 @@
 {
     PersistentRegion *region = data;
     struct XenBlkDev *blkdev = dev;
-    XenGnttab gnt = blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = blkdev->xendev.gnttabdev;
 
-    if (xc_gnttab_munmap(gnt, region->addr, region->num) != 0) {
+    if (xengnttab_unmap(gnt, region->addr, region->num) != 0) {
         xen_be_printf(&blkdev->xendev, 0,
-                      "xc_gnttab_munmap region %p failed: %s\n",
+                      "xengnttab_unmap region %p failed: %s\n",
                       region->addr, strerror(errno));
     }
     xen_be_printf(&blkdev->xendev, 3,
@@ -328,7 +317,7 @@
 
 static void ioreq_unmap(struct ioreq *ioreq)
 {
-    XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = ioreq->blkdev->xendev.gnttabdev;
     int i;
 
     if (ioreq->num_unmap == 0 || ioreq->mapped == 0) {
@@ -338,8 +327,9 @@
         if (!ioreq->pages) {
             return;
         }
-        if (xc_gnttab_munmap(gnt, ioreq->pages, ioreq->num_unmap) != 0) {
-            xen_be_printf(&ioreq->blkdev->xendev, 0, "xc_gnttab_munmap failed: %s\n",
+        if (xengnttab_unmap(gnt, ioreq->pages, ioreq->num_unmap) != 0) {
+            xen_be_printf(&ioreq->blkdev->xendev, 0,
+                          "xengnttab_unmap failed: %s\n",
                           strerror(errno));
         }
         ioreq->blkdev->cnt_map -= ioreq->num_unmap;
@@ -349,8 +339,9 @@
             if (!ioreq->page[i]) {
                 continue;
             }
-            if (xc_gnttab_munmap(gnt, ioreq->page[i], 1) != 0) {
-                xen_be_printf(&ioreq->blkdev->xendev, 0, "xc_gnttab_munmap failed: %s\n",
+            if (xengnttab_unmap(gnt, ioreq->page[i], 1) != 0) {
+                xen_be_printf(&ioreq->blkdev->xendev, 0,
+                              "xengnttab_unmap failed: %s\n",
                               strerror(errno));
             }
             ioreq->blkdev->cnt_map--;
@@ -362,7 +353,7 @@
 
 static int ioreq_map(struct ioreq *ioreq)
 {
-    XenGnttab gnt = ioreq->blkdev->xendev.gnttabdev;
+    xengnttab_handle *gnt = ioreq->blkdev->xendev.gnttabdev;
     uint32_t domids[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     uint32_t refs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
     void *page[BLKIF_MAX_SEGMENTS_PER_REQUEST];
@@ -413,7 +404,7 @@
     }
 
     if (batch_maps && new_maps) {
-        ioreq->pages = xc_gnttab_map_grant_refs
+        ioreq->pages = xengnttab_map_grant_refs
             (gnt, new_maps, domids, refs, ioreq->prot);
         if (ioreq->pages == NULL) {
             xen_be_printf(&ioreq->blkdev->xendev, 0,
@@ -429,7 +420,7 @@
         ioreq->blkdev->cnt_map += new_maps;
     } else if (new_maps)  {
         for (i = 0; i < new_maps; i++) {
-            ioreq->page[i] = xc_gnttab_map_grant_ref
+            ioreq->page[i] = xengnttab_map_grant_ref
                 (gnt, domids[i], refs[i], ioreq->prot);
             if (ioreq->page[i] == NULL) {
                 xen_be_printf(&ioreq->blkdev->xendev, 0,
@@ -519,12 +510,6 @@
     if (ioreq->aio_inflight > 0) {
         return;
     }
-    if (ioreq->postsync) {
-        ioreq->postsync = 0;
-        ioreq->aio_inflight++;
-        blk_aio_flush(ioreq->blkdev->blk, qemu_aio_complete, ioreq);
-        return;
-    }
 
     ioreq->status = ioreq->aio_errors ? BLKIF_RSP_ERROR : BLKIF_RSP_OKAY;
     ioreq_unmap(ioreq);
@@ -536,7 +521,11 @@
             break;
         }
     case BLKIF_OP_READ:
-        block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        if (ioreq->status == BLKIF_RSP_OKAY) {
+            block_acct_done(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        } else {
+            block_acct_failed(blk_get_stats(ioreq->blkdev->blk), &ioreq->acct);
+        }
         break;
     case BLKIF_OP_DISCARD:
     default:
@@ -564,9 +553,8 @@
         block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
                          ioreq->v.size, BLOCK_ACCT_READ);
         ioreq->aio_inflight++;
-        blk_aio_readv(blkdev->blk, ioreq->start / BLOCK_SIZE,
-                      &ioreq->v, ioreq->v.size / BLOCK_SIZE,
-                      qemu_aio_complete, ioreq);
+        blk_aio_preadv(blkdev->blk, ioreq->start, &ioreq->v, 0,
+                       qemu_aio_complete, ioreq);
         break;
     case BLKIF_OP_WRITE:
     case BLKIF_OP_FLUSH_DISKCACHE:
@@ -575,19 +563,21 @@
         }
 
         block_acct_start(blk_get_stats(blkdev->blk), &ioreq->acct,
-                         ioreq->v.size, BLOCK_ACCT_WRITE);
+                         ioreq->v.size,
+                         ioreq->req.operation == BLKIF_OP_WRITE ?
+                         BLOCK_ACCT_WRITE : BLOCK_ACCT_FLUSH);
         ioreq->aio_inflight++;
-        blk_aio_writev(blkdev->blk, ioreq->start / BLOCK_SIZE,
-                       &ioreq->v, ioreq->v.size / BLOCK_SIZE,
-                       qemu_aio_complete, ioreq);
+        blk_aio_pwritev(blkdev->blk, ioreq->start, &ioreq->v, 0,
+                        qemu_aio_complete, ioreq);
         break;
     case BLKIF_OP_DISCARD:
     {
         struct blkif_request_discard *discard_req = (void *)&ioreq->req;
         ioreq->aio_inflight++;
-        blk_aio_discard(blkdev->blk,
-                        discard_req->sector_number, discard_req->nr_sectors,
-                        qemu_aio_complete, ioreq);
+        blk_aio_pdiscard(blkdev->blk,
+                         discard_req->sector_number << BDRV_SECTOR_BITS,
+                         discard_req->nr_sectors << BDRV_SECTOR_BITS,
+                         qemu_aio_complete, ioreq);
         break;
     }
     default:
@@ -689,6 +679,8 @@
                              RING_GET_REQUEST(&blkdev->rings.x86_64_part, rc));
         break;
     }
+    /* Prevent the compiler from accessing the on-ring fields instead. */
+    barrier();
     return 0;
 }
 
@@ -719,6 +711,23 @@
 
         /* parse them */
         if (ioreq_parse(ioreq) != 0) {
+
+            switch (ioreq->req.operation) {
+            case BLKIF_OP_READ:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_READ);
+                break;
+            case BLKIF_OP_WRITE:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_WRITE);
+                break;
+            case BLKIF_OP_FLUSH_DISKCACHE:
+                block_acct_invalid(blk_get_stats(blkdev->blk),
+                                   BLOCK_ACCT_FLUSH);
+            default:
+                break;
+            };
+
             if (blk_send_response_one(ioreq)) {
                 xen_be_send_notify(&blkdev->xendev);
             }
@@ -762,9 +771,9 @@
     if (xen_mode != XEN_EMULATE) {
         batch_maps = 1;
     }
-    if (xc_gnttab_set_max_grants(xendev->gnttabdev,
+    if (xengnttab_set_max_grants(xendev->gnttabdev,
             MAX_GRANTS(max_requests, BLKIF_MAX_SEGMENTS_PER_REQUEST)) < 0) {
-        xen_be_printf(xendev, 0, "xc_gnttab_set_max_grants failed: %s\n",
+        xen_be_printf(xendev, 0, "xengnttab_set_max_grants failed: %s\n",
                       strerror(errno));
     }
 }
@@ -809,6 +818,9 @@
     if (!strcmp("aio", blkdev->fileproto)) {
         blkdev->fileproto = "raw";
     }
+    if (!strcmp("vhd", blkdev->fileproto)) {
+        blkdev->fileproto = "vpc";
+    }
     if (blkdev->mode == NULL) {
         blkdev->mode = xenstore_read_be_str(&blkdev->xendev, "mode");
     }
@@ -877,12 +889,14 @@
     struct XenBlkDev *blkdev = container_of(xendev, struct XenBlkDev, xendev);
     int pers, index, qflags;
     bool readonly = true;
+    bool writethrough = true;
 
     /* read-only ? */
     if (blkdev->directiosafe) {
         qflags = BDRV_O_NOCACHE | BDRV_O_NATIVE_AIO;
     } else {
-        qflags = BDRV_O_CACHE_WB;
+        qflags = 0;
+        writethrough = false;
     }
     if (strcmp(blkdev->mode, "w") == 0) {
         qflags |= BDRV_O_RDWR;
@@ -897,30 +911,24 @@
     blkdev->dinfo = drive_get(IF_XEN, 0, index);
     if (!blkdev->dinfo) {
         Error *local_err = NULL;
-        BlockBackend *blk;
-        BlockDriver *drv;
-        BlockDriverState *bs;
+        QDict *options = NULL;
+
+        if (strcmp(blkdev->fileproto, "<unset>")) {
+            options = qdict_new();
+            qdict_put(options, "driver", qstring_from_str(blkdev->fileproto));
+        }
 
         /* setup via xenbus -> create new block driver instance */
         xen_be_printf(&blkdev->xendev, 2, "create new bdrv (xenbus setup)\n");
-        blk = blk_new_with_bs(blkdev->dev, NULL);
-        if (!blk) {
-            return -1;
-        }
-        blkdev->blk = blk;
-
-        bs = blk_bs(blk);
-        drv = bdrv_find_whitelisted_format(blkdev->fileproto, readonly);
-        if (bdrv_open(&bs, blkdev->filename, NULL, NULL, qflags,
-                      drv, &local_err) != 0) {
+        blkdev->blk = blk_new_open(blkdev->filename, NULL, options,
+                                   qflags, &local_err);
+        if (!blkdev->blk) {
             xen_be_printf(&blkdev->xendev, 0, "error: %s\n",
                           error_get_pretty(local_err));
             error_free(local_err);
-            blk_unref(blk);
-            blkdev->blk = NULL;
             return -1;
         }
-        assert(bs == blk_bs(blk));
+        blk_set_enable_write_cache(blkdev->blk, !writethrough);
     } else {
         /* setup via qemu cmdline -> already setup for us */
         xen_be_printf(&blkdev->xendev, 2, "get configured bdrv (cmdline setup)\n");
@@ -937,9 +945,11 @@
     blk_attach_dev_nofail(blkdev->blk, blkdev);
     blkdev->file_size = blk_getlength(blkdev->blk);
     if (blkdev->file_size < 0) {
+        BlockDriverState *bs = blk_bs(blkdev->blk);
+        const char *drv_name = bs ? bdrv_get_format_name(bs) : NULL;
         xen_be_printf(&blkdev->xendev, 1, "blk_getlength: %d (%s) | drv %s\n",
                       (int)blkdev->file_size, strerror(-blkdev->file_size),
-                      bdrv_get_format_name(blk_bs(blkdev->blk)) ?: "-");
+                      drv_name ?: "-");
         blkdev->file_size = 0;
     }
 
@@ -976,7 +986,7 @@
         }
     }
 
-    blkdev->sring = xc_gnttab_map_grant_ref(blkdev->xendev.gnttabdev,
+    blkdev->sring = xengnttab_map_grant_ref(blkdev->xendev.gnttabdev,
                                             blkdev->xendev.dom,
                                             blkdev->ring_ref,
                                             PROT_READ | PROT_WRITE);
@@ -1041,7 +1051,7 @@
     xen_be_unbind_evtchn(&blkdev->xendev);
 
     if (blkdev->sring) {
-        xc_gnttab_munmap(blkdev->xendev.gnttabdev, blkdev->sring, 1);
+        xengnttab_unmap(blkdev->xendev.gnttabdev, blkdev->sring, 1);
         blkdev->cnt_map--;
         blkdev->sring = NULL;
     }
diff --git a/hw/bt/core.c b/hw/bt/core.c
index 0ffc948..615f0af 100644
--- a/hw/bt/core.c
+++ b/hw/bt/core.c
@@ -17,6 +17,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
diff --git a/hw/bt/hci-csr.c b/hw/bt/hci-csr.c
index 7b9b916..d688372 100644
--- a/hw/bt/hci-csr.c
+++ b/hw/bt/hci-csr.c
@@ -18,9 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/char.h"
 #include "qemu/timer.h"
+#include "qemu/bswap.h"
 #include "hw/irq.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
@@ -37,9 +39,14 @@
     int out_size;
     uint8_t outfifo[FIFO_LEN * 2];
     uint8_t inpkt[FIFO_LEN];
+    enum {
+        CSR_HDR_LEN,
+        CSR_DATA_LEN,
+        CSR_DATA
+    } in_state;
     int in_len;
     int in_hdr;
-    int in_data;
+    int in_needed;
     QEMUTimer *out_tm;
     int64_t baud_delay;
 
@@ -294,38 +301,60 @@
     exit(-1);
 }
 
+static void csrhci_ready_for_next_inpkt(struct csrhci_s *s)
+{
+    s->in_state = CSR_HDR_LEN;
+    s->in_len = 0;
+    s->in_needed = 2;
+    s->in_hdr = INT_MAX;
+}
+
 static int csrhci_write(struct CharDriverState *chr,
                 const uint8_t *buf, int len)
 {
     struct csrhci_s *s = (struct csrhci_s *) chr->opaque;
-    int plen = s->in_len;
+    int total = 0;
 
     if (!s->enable)
         return 0;
 
-    s->in_len += len;
-    memcpy(s->inpkt + plen, buf, len);
+    for (;;) {
+        int cnt = MIN(len, s->in_needed - s->in_len);
+        if (cnt) {
+            memcpy(s->inpkt + s->in_len, buf, cnt);
+            s->in_len += cnt;
+            buf += cnt;
+            len -= cnt;
+            total += cnt;
+        }
 
-    while (1) {
-        if (s->in_len >= 2 && plen < 2)
-            s->in_hdr = csrhci_header_len(s->inpkt) + 1;
-
-        if (s->in_len >= s->in_hdr && plen < s->in_hdr)
-            s->in_data = csrhci_data_len(s->inpkt) + s->in_hdr;
-
-        if (s->in_len >= s->in_data) {
-            csrhci_in_packet(s, s->inpkt);
-
-            memmove(s->inpkt, s->inpkt + s->in_len, s->in_len - s->in_data);
-            s->in_len -= s->in_data;
-            s->in_hdr = INT_MAX;
-            s->in_data = INT_MAX;
-            plen = 0;
-        } else
+        if (s->in_len < s->in_needed) {
             break;
+        }
+
+        if (s->in_state == CSR_HDR_LEN) {
+            s->in_hdr = csrhci_header_len(s->inpkt) + 1;
+            assert(s->in_hdr >= s->in_needed);
+            s->in_needed = s->in_hdr;
+            s->in_state = CSR_DATA_LEN;
+            continue;
+        }
+
+        if (s->in_state == CSR_DATA_LEN) {
+            s->in_needed += csrhci_data_len(s->inpkt);
+            /* hci_acl_hdr could specify more than 4096 bytes, so assert.  */
+            assert(s->in_needed <= sizeof(s->inpkt));
+            s->in_state = CSR_DATA;
+            continue;
+        }
+
+        if (s->in_state == CSR_DATA) {
+            csrhci_in_packet(s, s->inpkt);
+            csrhci_ready_for_next_inpkt(s);
+        }
     }
 
-    return len;
+    return total;
 }
 
 static void csrhci_out_hci_packet_event(void *opaque,
@@ -362,7 +391,7 @@
     switch (cmd) {
     case CHR_IOCTL_SERIAL_SET_PARAMS:
         ssp = (QEMUSerialSetParams *) arg;
-        s->baud_delay = get_ticks_per_sec() / ssp->speed;
+        s->baud_delay = NANOSECONDS_PER_SECOND / ssp->speed;
         /* Moments later... (but shorter than 100ms) */
         s->modem_state |= CHR_TIOCM_CTS;
         break;
@@ -387,11 +416,9 @@
 {
     s->out_len = 0;
     s->out_size = FIFO_LEN;
-    s->in_len = 0;
-    s->baud_delay = get_ticks_per_sec();
+    csrhci_ready_for_next_inpkt(s);
+    s->baud_delay = NANOSECONDS_PER_SECOND;
     s->enable = 0;
-    s->in_hdr = INT_MAX;
-    s->in_data = INT_MAX;
 
     s->modem_state = 0;
     /* After a while... (but sooner than 10ms) */
diff --git a/hw/bt/hci.c b/hw/bt/hci.c
index 7ea3dc6..351123f 100644
--- a/hw/bt/hci.c
+++ b/hw/bt/hci.c
@@ -18,11 +18,16 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
 #include "sysemu/bt.h"
 #include "hw/bt.h"
+#include "qapi/qmp/qerror.h"
+#include "sysemu/replay.h"
+#include "qemu/cutils.h"
 
 struct bt_hci_s {
     uint8_t *(*evt_packet)(void *opaque);
@@ -72,6 +77,8 @@
 
     struct HCIInfo info;
     struct bt_device_s device;
+
+    Error *replay_blocker;
 };
 
 #define DEFAULT_RSSI_DBM	20
@@ -419,11 +426,7 @@
  * be continuously allocated.  We do it though, to preserve similar
  * behaviour between hosts.  Some things, like the BD_ADDR cannot be
  * preserved though (for example if a real hci is used).  */
-#ifdef HOST_WORDS_BIGENDIAN
-# define HNDL(raw)	bswap16(raw)
-#else
-# define HNDL(raw)	(raw)
-#endif
+#define HNDL(raw) cpu_to_le16(raw)
 
 static const uint8_t bt_event_reserved_mask[8] = {
     0xff, 0x9f, 0xfb, 0xff, 0x07, 0x18, 0x00, 0x00,
@@ -595,7 +598,7 @@
 static void bt_hci_mod_timer_1280ms(QEMUTimer *timer, int period)
 {
     timer_mod(timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(period << 7, get_ticks_per_sec(), 100));
+                     (uint64_t)(period << 7) * 10000000);
 }
 
 static void bt_hci_inquiry_start(struct bt_hci_s *hci, int length)
@@ -1099,7 +1102,7 @@
     bt_hci_event_status(hci, HCI_SUCCESS);
 
     timer_mod(link->acl_mode_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(interval * 625, get_ticks_per_sec(), 1000000));
+                                    ((uint64_t)interval * 625) * 1000);
     bt_hci_lmp_mode_change_master(hci, link->link, mode, interval);
 
     return 0;
@@ -1151,8 +1154,7 @@
     hci->event_mask[7] = 0x00;
     hci->device.inquiry_scan = 0;
     hci->device.page_scan = 0;
-    if (hci->device.lmp_name)
-        g_free((void *) hci->device.lmp_name);
+    g_free((void *) hci->device.lmp_name);
     hci->device.lmp_name = NULL;
     hci->device.class[0] = 0x00;
     hci->device.class[1] = 0x00;
@@ -1498,8 +1500,8 @@
         return;
 
 #define PARAM(cmd, param)	(((cmd##_cp *) data)->param)
-#define PARAM16(cmd, param)	le16_to_cpup(&PARAM(cmd, param))
-#define PARAMHANDLE(cmd)	HNDL(PARAM(cmd, handle))
+#define PARAM16(cmd, param) lduw_le_p(&PARAM(cmd, param))
+#define PARAMHANDLE(cmd) PARAM16(cmd, handle)
 #define LENGTH_CHECK(cmd)	if (length < sizeof(cmd##_cp)) goto short_hci
     /* Note: the supported commands bitmask in bt_hci_read_local_commands_rp
      * needs to be updated every time a command is implemented here!  */
@@ -1829,8 +1831,7 @@
     case cmd_opcode_pack(OGF_HOST_CTL, OCF_CHANGE_LOCAL_NAME):
         LENGTH_CHECK(change_local_name);
 
-        if (hci->device.lmp_name)
-            g_free((void *) hci->device.lmp_name);
+        g_free((void *) hci->device.lmp_name);
         hci->device.lmp_name = g_strndup(PARAM(change_local_name, name),
                         sizeof(PARAM(change_local_name, name)));
         bt_hci_event_complete_status(hci, HCI_SUCCESS);
@@ -2191,6 +2192,9 @@
 
     s->device.handle_destroy = bt_hci_destroy;
 
+    error_setg(&s->replay_blocker, QERR_REPLAY_NOT_SUPPORTED, "-bt hci");
+    replay_add_blocker(s->replay_blocker);
+
     return &s->info;
 }
 
@@ -2231,8 +2235,7 @@
 
     bt_device_done(&hci->device);
 
-    if (hci->device.lmp_name)
-        g_free((void *) hci->device.lmp_name);
+    g_free((void *) hci->device.lmp_name);
 
     /* Be gentle and send DISCONNECT to all connected peers and those
      * currently waiting for us to accept or reject a connection request.
diff --git a/hw/bt/hid.c b/hw/bt/hid.c
index af494e1..f6affbb 100644
--- a/hw/bt/hid.c
+++ b/hw/bt/hid.c
@@ -18,6 +18,7 @@
  * with this program; if not, if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "ui/console.h"
diff --git a/hw/bt/l2cap.c b/hw/bt/l2cap.c
index 591e047..e342045 100644
--- a/hw/bt/l2cap.c
+++ b/hw/bt/l2cap.c
@@ -17,8 +17,10 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/bswap.h"
 #include "hw/bt.h"
 
 #define L2CAP_CID_MAX	0x100	/* Between 0x40 and 0x10000 */
@@ -524,9 +526,9 @@
             }
 
             /* MTU */
-            val = le16_to_cpup((void *) opt->val);
+            val = lduw_le_p(opt->val);
             if (val < ch->min_mtu) {
-                cpu_to_le16w((void *) opt->val, ch->min_mtu);
+                stw_le_p(opt->val, ch->min_mtu);
                 result = L2CAP_CONF_UNACCEPT;
                 break;
             }
@@ -541,7 +543,7 @@
             }
 
             /* Flush Timeout */
-            val = le16_to_cpup((void *) opt->val);
+            val = lduw_le_p(opt->val);
             if (val < 0x0001) {
                 opt->val[0] = 0xff;
                 opt->val[1] = 0xff;
@@ -985,7 +987,7 @@
 static void l2cap_iframe_in(struct l2cap_chan_s *ch, uint16_t cid,
                 const l2cap_hdr *hdr, int len)
 {
-    uint16_t fcs = le16_to_cpup((void *) (hdr->data + len - 2));
+    uint16_t fcs = lduw_le_p(hdr->data + len - 2);
 
     if (len < 4)
         goto len_error;
@@ -1000,7 +1002,7 @@
             /* TODO: Signal an error? */
             return;
         }
-        l2cap_sframe_in(ch, le16_to_cpup((void *) hdr->data));
+        l2cap_sframe_in(ch, lduw_le_p(hdr->data));
         return;
     }
 
@@ -1020,7 +1022,7 @@
         if (len - 6 > ch->mps)
             goto len_error;
 
-        ch->len_total = le16_to_cpup((void *) (hdr->data + 2));
+        ch->len_total = lduw_le_p(hdr->data + 2);
         if (len >= 6 + ch->len_total)
             goto seg_error;
 
diff --git a/hw/bt/sdp.c b/hw/bt/sdp.c
index 218e075..f67b3b8 100644
--- a/hw/bt/sdp.c
+++ b/hw/bt/sdp.c
@@ -17,7 +17,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/host-utils.h"
 #include "hw/bt.h"
 
 struct bt_l2cap_sdp_state_s {
@@ -42,7 +44,7 @@
 
 static ssize_t sdp_datalen(const uint8_t **element, ssize_t *left)
 {
-    size_t len = *(*element) ++ & SDP_DSIZE_MASK;
+    uint32_t len = *(*element) ++ & SDP_DSIZE_MASK;
 
     if (!*left)
         return -1;
@@ -150,12 +152,14 @@
         if (seqlen < 3 || len < seqlen)
             return -SDP_INVALID_SYNTAX;
         len -= seqlen;
-
         while (seqlen)
             if (sdp_svc_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 3)
         return -SDP_INVALID_SYNTAX;
@@ -278,8 +282,11 @@
         while (seqlen)
             if (sdp_attr_match(record, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_attr_match(record, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_attr_match(record, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 1)
         return -SDP_INVALID_SYNTAX;
@@ -393,8 +400,11 @@
         while (seqlen)
             if (sdp_svc_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 3)
         return -SDP_INVALID_SYNTAX;
@@ -413,8 +423,11 @@
         while (seqlen)
             if (sdp_svc_attr_match(sdp, &req, &seqlen))
                 return -SDP_INVALID_SYNTAX;
-    } else if (sdp_svc_attr_match(sdp, &req, &seqlen))
-        return -SDP_INVALID_SYNTAX;
+    } else {
+        if (sdp_svc_attr_match(sdp, &req, &len)) {
+            return -SDP_INVALID_SYNTAX;
+        }
+    }
 
     if (len < 1)
         return -SDP_INVALID_SYNTAX;
@@ -707,7 +720,7 @@
         len += sdp_attr_max_size(&def->attributes[record->attributes ++].data,
                         &record->uuids);
     }
-    record->uuids = 1 << ffs(record->uuids - 1);
+    record->uuids = pow2ceil(record->uuids);
     record->attribute_list =
             g_malloc0(record->attributes * sizeof(*record->attribute_list));
     record->uuid =
diff --git a/hw/char/Makefile.objs b/hw/char/Makefile.objs
index 6fc2d11..9563bb8 100644
--- a/hw/char/Makefile.objs
+++ b/hw/char/Makefile.objs
@@ -5,7 +5,6 @@
 common-obj-$(CONFIG_SERIAL) += serial.o serial-isa.o
 common-obj-$(CONFIG_SERIAL_PCI) += serial-pci.o
 common-obj-$(CONFIG_VIRTIO) += virtio-console.o
-common-obj-$(CONFIG_VIRTIO) += virtio-pingpong.o
 common-obj-$(CONFIG_XILINX) += xilinx_uartlite.o
 common-obj-$(CONFIG_XEN_BACKEND) += xen_console.o
 common-obj-$(CONFIG_CADENCE) += cadence_uart.o
@@ -16,6 +15,8 @@
 obj-$(CONFIG_SH4) += sh_serial.o
 obj-$(CONFIG_PSERIES) += spapr_vty.o
 obj-$(CONFIG_DIGIC) += digic-uart.o
+obj-$(CONFIG_STM32F2XX_USART) += stm32f2xx_usart.o
+obj-$(CONFIG_RASPI) += bcm2835_aux.o
 obj-$(CONFIG_GOLDFISH) += goldfish_tty.o
 
 common-obj-$(CONFIG_ETRAXFS) += etraxfs_ser.o
diff --git a/hw/char/bcm2835_aux.c b/hw/char/bcm2835_aux.c
new file mode 100644
index 0000000..319f165
--- /dev/null
+++ b/hw/char/bcm2835_aux.c
@@ -0,0 +1,317 @@
+/*
+ * BCM2835 (Raspberry Pi / Pi 2) Aux block (mini UART and SPI).
+ * Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ * Based on pl011.c, copyright terms below:
+ *
+ * Arm PrimeCell PL011 UART
+ *
+ * Copyright (c) 2006 CodeSourcery.
+ * Written by Paul Brook
+ *
+ * This code is licensed under the GPL.
+ *
+ * At present only the core UART functions (data path for tx/rx) are
+ * implemented. The following features/registers are unimplemented:
+ *  - Line/modem control
+ *  - Scratch register
+ *  - Extra control
+ *  - Baudrate
+ *  - SPI interfaces
+ */
+
+#include "qemu/osdep.h"
+#include "hw/char/bcm2835_aux.h"
+#include "qemu/log.h"
+
+#define AUX_IRQ         0x0
+#define AUX_ENABLES     0x4
+#define AUX_MU_IO_REG   0x40
+#define AUX_MU_IER_REG  0x44
+#define AUX_MU_IIR_REG  0x48
+#define AUX_MU_LCR_REG  0x4c
+#define AUX_MU_MCR_REG  0x50
+#define AUX_MU_LSR_REG  0x54
+#define AUX_MU_MSR_REG  0x58
+#define AUX_MU_SCRATCH  0x5c
+#define AUX_MU_CNTL_REG 0x60
+#define AUX_MU_STAT_REG 0x64
+#define AUX_MU_BAUD_REG 0x68
+
+/* bits in IER/IIR registers */
+#define TX_INT  0x1
+#define RX_INT  0x2
+
+static void bcm2835_aux_update(BCM2835AuxState *s)
+{
+    /* signal an interrupt if either:
+     * 1. rx interrupt is enabled and we have a non-empty rx fifo, or
+     * 2. the tx interrupt is enabled (since we instantly drain the tx fifo)
+     */
+    s->iir = 0;
+    if ((s->ier & RX_INT) && s->read_count != 0) {
+        s->iir |= RX_INT;
+    }
+    if (s->ier & TX_INT) {
+        s->iir |= TX_INT;
+    }
+    qemu_set_irq(s->irq, s->iir != 0);
+}
+
+static uint64_t bcm2835_aux_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835AuxState *s = opaque;
+    uint32_t c, res;
+
+    switch (offset) {
+    case AUX_IRQ:
+        return s->iir != 0;
+
+    case AUX_ENABLES:
+        return 1; /* mini UART permanently enabled */
+
+    case AUX_MU_IO_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        c = s->read_fifo[s->read_pos];
+        if (s->read_count > 0) {
+            s->read_count--;
+            if (++s->read_pos == BCM2835_AUX_RX_FIFO_LEN) {
+                s->read_pos = 0;
+            }
+        }
+        if (s->chr) {
+            qemu_chr_accept_input(s->chr);
+        }
+        bcm2835_aux_update(s);
+        return c;
+
+    case AUX_MU_IER_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        return 0xc0 | s->ier; /* FIFO enables always read 1 */
+
+    case AUX_MU_IIR_REG:
+        res = 0xc0; /* FIFO enables */
+        /* The spec is unclear on what happens when both tx and rx
+         * interrupts are active, besides that this cannot occur. At
+         * present, we choose to prioritise the rx interrupt, since
+         * the tx fifo is always empty. */
+        if (s->read_count != 0) {
+            res |= 0x4;
+        } else {
+            res |= 0x2;
+        }
+        if (s->iir == 0) {
+            res |= 0x1;
+        }
+        return res;
+
+    case AUX_MU_LCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_LCR_REG unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_MCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_MCR_REG unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_LSR_REG:
+        res = 0x60; /* tx idle, empty */
+        if (s->read_count != 0) {
+            res |= 0x1;
+        }
+        return res;
+
+    case AUX_MU_MSR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_MSR_REG unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_SCRATCH:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_SCRATCH unsupported\n", __func__);
+        return 0;
+
+    case AUX_MU_CNTL_REG:
+        return 0x3; /* tx, rx enabled */
+
+    case AUX_MU_STAT_REG:
+        res = 0x30e; /* space in the output buffer, empty tx fifo, idle tx/rx */
+        if (s->read_count > 0) {
+            res |= 0x1; /* data in input buffer */
+            assert(s->read_count < BCM2835_AUX_RX_FIFO_LEN);
+            res |= ((uint32_t)s->read_count) << 16; /* rx fifo fill level */
+        }
+        return res;
+
+    case AUX_MU_BAUD_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_BAUD_REG unsupported\n", __func__);
+        return 0;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+}
+
+static void bcm2835_aux_write(void *opaque, hwaddr offset, uint64_t value,
+                              unsigned size)
+{
+    BCM2835AuxState *s = opaque;
+    unsigned char ch;
+
+    switch (offset) {
+    case AUX_ENABLES:
+        if (value != 1) {
+            qemu_log_mask(LOG_UNIMP, "%s: unsupported attempt to enable SPI "
+                          "or disable UART\n", __func__);
+        }
+        break;
+
+    case AUX_MU_IO_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        ch = value;
+        if (s->chr) {
+            qemu_chr_fe_write(s->chr, &ch, 1);
+        }
+        break;
+
+    case AUX_MU_IER_REG:
+        /* "DLAB bit set means access baudrate register" is NYI */
+        s->ier = value & (TX_INT | RX_INT);
+        bcm2835_aux_update(s);
+        break;
+
+    case AUX_MU_IIR_REG:
+        if (value & 0x2) {
+            s->read_count = 0;
+        }
+        break;
+
+    case AUX_MU_LCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_LCR_REG unsupported\n", __func__);
+        break;
+
+    case AUX_MU_MCR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_MCR_REG unsupported\n", __func__);
+        break;
+
+    case AUX_MU_SCRATCH:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_SCRATCH unsupported\n", __func__);
+        break;
+
+    case AUX_MU_CNTL_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_CNTL_REG unsupported\n", __func__);
+        break;
+
+    case AUX_MU_BAUD_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: AUX_MU_BAUD_REG unsupported\n", __func__);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+    }
+
+    bcm2835_aux_update(s);
+}
+
+static int bcm2835_aux_can_receive(void *opaque)
+{
+    BCM2835AuxState *s = opaque;
+
+    return s->read_count < BCM2835_AUX_RX_FIFO_LEN;
+}
+
+static void bcm2835_aux_put_fifo(void *opaque, uint8_t value)
+{
+    BCM2835AuxState *s = opaque;
+    int slot;
+
+    slot = s->read_pos + s->read_count;
+    if (slot >= BCM2835_AUX_RX_FIFO_LEN) {
+        slot -= BCM2835_AUX_RX_FIFO_LEN;
+    }
+    s->read_fifo[slot] = value;
+    s->read_count++;
+    if (s->read_count == BCM2835_AUX_RX_FIFO_LEN) {
+        /* buffer full */
+    }
+    bcm2835_aux_update(s);
+}
+
+static void bcm2835_aux_receive(void *opaque, const uint8_t *buf, int size)
+{
+    bcm2835_aux_put_fifo(opaque, *buf);
+}
+
+static const MemoryRegionOps bcm2835_aux_ops = {
+    .read = bcm2835_aux_read,
+    .write = bcm2835_aux_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_aux = {
+    .name = TYPE_BCM2835_AUX,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8_ARRAY(read_fifo, BCM2835AuxState,
+                            BCM2835_AUX_RX_FIFO_LEN),
+        VMSTATE_UINT8(read_pos, BCM2835AuxState),
+        VMSTATE_UINT8(read_count, BCM2835AuxState),
+        VMSTATE_UINT8(ier, BCM2835AuxState),
+        VMSTATE_UINT8(iir, BCM2835AuxState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_aux_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    BCM2835AuxState *s = BCM2835_AUX(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &bcm2835_aux_ops, s,
+                          TYPE_BCM2835_AUX, 0x100);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+}
+
+static void bcm2835_aux_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835AuxState *s = BCM2835_AUX(dev);
+
+    if (s->chr) {
+        qemu_chr_add_handlers(s->chr, bcm2835_aux_can_receive,
+                              bcm2835_aux_receive, NULL, s);
+    }
+}
+
+static Property bcm2835_aux_props[] = {
+    DEFINE_PROP_CHR("chardev", BCM2835AuxState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void bcm2835_aux_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = bcm2835_aux_realize;
+    dc->vmsd = &vmstate_bcm2835_aux;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
+    dc->props = bcm2835_aux_props;
+}
+
+static const TypeInfo bcm2835_aux_info = {
+    .name          = TYPE_BCM2835_AUX,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835AuxState),
+    .instance_init = bcm2835_aux_init,
+    .class_init    = bcm2835_aux_class_init,
+};
+
+static void bcm2835_aux_register_types(void)
+{
+    type_register_static(&bcm2835_aux_info);
+}
+
+type_init(bcm2835_aux_register_types)
diff --git a/hw/char/cadence_uart.c b/hw/char/cadence_uart.c
index 303ba4a..e3bc52f 100644
--- a/hw/char/cadence_uart.c
+++ b/hw/char/cadence_uart.c
@@ -16,9 +16,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 #include "qemu/timer.h"
+#include "qemu/log.h"
+#include "hw/char/cadence_uart.h"
 
 #ifdef CADENCE_UART_ERR_DEBUG
 #define DB_PRINT(...) do { \
@@ -85,8 +88,6 @@
 #define LOCAL_LOOPBACK         (0x2 << UART_MR_CHMODE_SH)
 #define REMOTE_LOOPBACK        (0x3 << UART_MR_CHMODE_SH)
 
-#define RX_FIFO_SIZE           16
-#define TX_FIFO_SIZE           16
 #define UART_INPUT_CLK         50000000
 
 #define R_CR       (0x00/4)
@@ -108,38 +109,18 @@
 #define R_PWID     (0x40/4)
 #define R_TTRIG    (0x44/4)
 
-#define R_MAX (R_TTRIG + 1)
 
-#define TYPE_CADENCE_UART "cadence_uart"
-#define CADENCE_UART(obj) OBJECT_CHECK(UartState, (obj), TYPE_CADENCE_UART)
-
-typedef struct {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    MemoryRegion iomem;
-    uint32_t r[R_MAX];
-    uint8_t rx_fifo[RX_FIFO_SIZE];
-    uint8_t tx_fifo[TX_FIFO_SIZE];
-    uint32_t rx_wpos;
-    uint32_t rx_count;
-    uint32_t tx_count;
-    uint64_t char_tx_time;
-    CharDriverState *chr;
-    qemu_irq irq;
-    QEMUTimer *fifo_trigger_handle;
-} UartState;
-
-static void uart_update_status(UartState *s)
+static void uart_update_status(CadenceUARTState *s)
 {
     s->r[R_SR] = 0;
 
-    s->r[R_SR] |= s->rx_count == RX_FIFO_SIZE ? UART_SR_INTR_RFUL : 0;
+    s->r[R_SR] |= s->rx_count == CADENCE_UART_RX_FIFO_SIZE ? UART_SR_INTR_RFUL
+                                                           : 0;
     s->r[R_SR] |= !s->rx_count ? UART_SR_INTR_REMPTY : 0;
     s->r[R_SR] |= s->rx_count >= s->r[R_RTRIG] ? UART_SR_INTR_RTRIG : 0;
 
-    s->r[R_SR] |= s->tx_count == TX_FIFO_SIZE ? UART_SR_INTR_TFUL : 0;
+    s->r[R_SR] |= s->tx_count == CADENCE_UART_TX_FIFO_SIZE ? UART_SR_INTR_TFUL
+                                                           : 0;
     s->r[R_SR] |= !s->tx_count ? UART_SR_INTR_TEMPTY : 0;
     s->r[R_SR] |= s->tx_count >= s->r[R_TTRIG] ? UART_SR_TTRIG : 0;
 
@@ -150,14 +131,14 @@
 
 static void fifo_trigger_update(void *opaque)
 {
-    UartState *s = (UartState *)opaque;
+    CadenceUARTState *s = opaque;
 
     s->r[R_CISR] |= UART_INTR_TIMEOUT;
 
     uart_update_status(s);
 }
 
-static void uart_rx_reset(UartState *s)
+static void uart_rx_reset(CadenceUARTState *s)
 {
     s->rx_wpos = 0;
     s->rx_count = 0;
@@ -166,12 +147,12 @@
     }
 }
 
-static void uart_tx_reset(UartState *s)
+static void uart_tx_reset(CadenceUARTState *s)
 {
     s->tx_count = 0;
 }
 
-static void uart_send_breaks(UartState *s)
+static void uart_send_breaks(CadenceUARTState *s)
 {
     int break_enabled = 1;
 
@@ -181,7 +162,7 @@
     }
 }
 
-static void uart_parameters_setup(UartState *s)
+static void uart_parameters_setup(CadenceUARTState *s)
 {
     QEMUSerialSetParams ssp;
     unsigned int baud_rate, packet_size;
@@ -228,7 +209,7 @@
     }
 
     packet_size += ssp.data_bits + ssp.stop_bits;
-    s->char_tx_time = (get_ticks_per_sec() / ssp.speed) * packet_size;
+    s->char_tx_time = (NANOSECONDS_PER_SECOND / ssp.speed) * packet_size;
     if (s->chr) {
         qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
     }
@@ -236,20 +217,20 @@
 
 static int uart_can_receive(void *opaque)
 {
-    UartState *s = (UartState *)opaque;
-    int ret = MAX(RX_FIFO_SIZE, TX_FIFO_SIZE);
+    CadenceUARTState *s = opaque;
+    int ret = MAX(CADENCE_UART_RX_FIFO_SIZE, CADENCE_UART_TX_FIFO_SIZE);
     uint32_t ch_mode = s->r[R_MR] & UART_MR_CHMODE;
 
     if (ch_mode == NORMAL_MODE || ch_mode == ECHO_MODE) {
-        ret = MIN(ret, RX_FIFO_SIZE - s->rx_count);
+        ret = MIN(ret, CADENCE_UART_RX_FIFO_SIZE - s->rx_count);
     }
     if (ch_mode == REMOTE_LOOPBACK || ch_mode == ECHO_MODE) {
-        ret = MIN(ret, TX_FIFO_SIZE - s->tx_count);
+        ret = MIN(ret, CADENCE_UART_TX_FIFO_SIZE - s->tx_count);
     }
     return ret;
 }
 
-static void uart_ctrl_update(UartState *s)
+static void uart_ctrl_update(CadenceUARTState *s)
 {
     if (s->r[R_CR] & UART_CR_TXRST) {
         uart_tx_reset(s);
@@ -268,7 +249,7 @@
 
 static void uart_write_rx_fifo(void *opaque, const uint8_t *buf, int size)
 {
-    UartState *s = (UartState *)opaque;
+    CadenceUARTState *s = opaque;
     uint64_t new_rx_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     int i;
 
@@ -276,12 +257,12 @@
         return;
     }
 
-    if (s->rx_count == RX_FIFO_SIZE) {
+    if (s->rx_count == CADENCE_UART_RX_FIFO_SIZE) {
         s->r[R_CISR] |= UART_INTR_ROVR;
     } else {
         for (i = 0; i < size; i++) {
             s->rx_fifo[s->rx_wpos] = buf[i];
-            s->rx_wpos = (s->rx_wpos + 1) % RX_FIFO_SIZE;
+            s->rx_wpos = (s->rx_wpos + 1) % CADENCE_UART_RX_FIFO_SIZE;
             s->rx_count++;
         }
         timer_mod(s->fifo_trigger_handle, new_rx_time +
@@ -293,7 +274,7 @@
 static gboolean cadence_uart_xmit(GIOChannel *chan, GIOCondition cond,
                                   void *opaque)
 {
-    UartState *s = opaque;
+    CadenceUARTState *s = opaque;
     int ret;
 
     /* instant drain the fifo when there's no back-end */
@@ -307,28 +288,34 @@
     }
 
     ret = qemu_chr_fe_write(s->chr, s->tx_fifo, s->tx_count);
-    s->tx_count -= ret;
-    memmove(s->tx_fifo, s->tx_fifo + ret, s->tx_count);
+
+    if (ret >= 0) {
+        s->tx_count -= ret;
+        memmove(s->tx_fifo, s->tx_fifo + ret, s->tx_count);
+    }
 
     if (s->tx_count) {
-        int r = qemu_chr_fe_add_watch(s->chr, G_IO_OUT|G_IO_HUP,
-                                      cadence_uart_xmit, s);
-        assert(r);
-        (void)r;
+        guint r = qemu_chr_fe_add_watch(s->chr, G_IO_OUT|G_IO_HUP,
+                                        cadence_uart_xmit, s);
+        if (!r) {
+            s->tx_count = 0;
+            return FALSE;
+        }
     }
 
     uart_update_status(s);
     return FALSE;
 }
 
-static void uart_write_tx_fifo(UartState *s, const uint8_t *buf, int size)
+static void uart_write_tx_fifo(CadenceUARTState *s, const uint8_t *buf,
+                               int size)
 {
     if ((s->r[R_CR] & UART_CR_TX_DIS) || !(s->r[R_CR] & UART_CR_TX_EN)) {
         return;
     }
 
-    if (size > TX_FIFO_SIZE - s->tx_count) {
-        size = TX_FIFO_SIZE - s->tx_count;
+    if (size > CADENCE_UART_TX_FIFO_SIZE - s->tx_count) {
+        size = CADENCE_UART_TX_FIFO_SIZE - s->tx_count;
         /*
          * This can only be a guest error via a bad tx fifo register push,
          * as can_receive() should stop remote loop and echo modes ever getting
@@ -346,7 +333,7 @@
 
 static void uart_receive(void *opaque, const uint8_t *buf, int size)
 {
-    UartState *s = (UartState *)opaque;
+    CadenceUARTState *s = opaque;
     uint32_t ch_mode = s->r[R_MR] & UART_MR_CHMODE;
 
     if (ch_mode == NORMAL_MODE || ch_mode == ECHO_MODE) {
@@ -359,7 +346,7 @@
 
 static void uart_event(void *opaque, int event)
 {
-    UartState *s = (UartState *)opaque;
+    CadenceUARTState *s = opaque;
     uint8_t buf = '\0';
 
     if (event == CHR_EVENT_BREAK) {
@@ -369,15 +356,15 @@
     uart_update_status(s);
 }
 
-static void uart_read_rx_fifo(UartState *s, uint32_t *c)
+static void uart_read_rx_fifo(CadenceUARTState *s, uint32_t *c)
 {
     if ((s->r[R_CR] & UART_CR_RX_DIS) || !(s->r[R_CR] & UART_CR_RX_EN)) {
         return;
     }
 
     if (s->rx_count) {
-        uint32_t rx_rpos =
-                (RX_FIFO_SIZE + s->rx_wpos - s->rx_count) % RX_FIFO_SIZE;
+        uint32_t rx_rpos = (CADENCE_UART_RX_FIFO_SIZE + s->rx_wpos -
+                            s->rx_count) % CADENCE_UART_RX_FIFO_SIZE;
         *c = s->rx_fifo[rx_rpos];
         s->rx_count--;
 
@@ -394,10 +381,13 @@
 static void uart_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
-    UartState *s = (UartState *)opaque;
+    CadenceUARTState *s = opaque;
 
     DB_PRINT(" offset:%x data:%08x\n", (unsigned)offset, (unsigned)value);
     offset >>= 2;
+    if (offset >= CADENCE_UART_R_MAX) {
+        return;
+    }
     switch (offset) {
     case R_IER: /* ier (wts imr) */
         s->r[R_IMR] |= value;
@@ -438,11 +428,11 @@
 static uint64_t uart_read(void *opaque, hwaddr offset,
         unsigned size)
 {
-    UartState *s = (UartState *)opaque;
+    CadenceUARTState *s = opaque;
     uint32_t c = 0;
 
     offset >>= 2;
-    if (offset >= R_MAX) {
+    if (offset >= CADENCE_UART_R_MAX) {
         c = 0;
     } else if (offset == R_TX_RX) {
         uart_read_rx_fifo(s, &c);
@@ -462,7 +452,7 @@
 
 static void cadence_uart_reset(DeviceState *dev)
 {
-    UartState *s = CADENCE_UART(dev);
+    CadenceUARTState *s = CADENCE_UART(dev);
 
     s->r[R_CR] = 0x00000128;
     s->r[R_IMR] = 0;
@@ -477,32 +467,34 @@
     uart_update_status(s);
 }
 
-static int cadence_uart_init(SysBusDevice *dev)
+static void cadence_uart_realize(DeviceState *dev, Error **errp)
 {
-    UartState *s = CADENCE_UART(dev);
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &uart_ops, s, "uart", 0x1000);
-    sysbus_init_mmio(dev, &s->iomem);
-    sysbus_init_irq(dev, &s->irq);
+    CadenceUARTState *s = CADENCE_UART(dev);
 
     s->fifo_trigger_handle = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-            (QEMUTimerCB *)fifo_trigger_update, s);
-
-    s->char_tx_time = (get_ticks_per_sec() / 9600) * 10;
-
-    s->chr = qemu_char_get_next_serial();
+                                          fifo_trigger_update, s);
 
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, uart_can_receive, uart_receive,
                               uart_event, s);
     }
+}
 
-    return 0;
+static void cadence_uart_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    CadenceUARTState *s = CADENCE_UART(obj);
+
+    memory_region_init_io(&s->iomem, obj, &uart_ops, s, "uart", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+
+    s->char_tx_time = (NANOSECONDS_PER_SECOND / 9600) * 10;
 }
 
 static int cadence_uart_post_load(void *opaque, int version_id)
 {
-    UartState *s = opaque;
+    CadenceUARTState *s = opaque;
 
     uart_parameters_setup(s);
     uart_update_status(s);
@@ -515,31 +507,39 @@
     .minimum_version_id = 2,
     .post_load = cadence_uart_post_load,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT32_ARRAY(r, UartState, R_MAX),
-        VMSTATE_UINT8_ARRAY(rx_fifo, UartState, RX_FIFO_SIZE),
-        VMSTATE_UINT8_ARRAY(tx_fifo, UartState, RX_FIFO_SIZE),
-        VMSTATE_UINT32(rx_count, UartState),
-        VMSTATE_UINT32(tx_count, UartState),
-        VMSTATE_UINT32(rx_wpos, UartState),
-        VMSTATE_TIMER(fifo_trigger_handle, UartState),
+        VMSTATE_UINT32_ARRAY(r, CadenceUARTState, CADENCE_UART_R_MAX),
+        VMSTATE_UINT8_ARRAY(rx_fifo, CadenceUARTState,
+                            CADENCE_UART_RX_FIFO_SIZE),
+        VMSTATE_UINT8_ARRAY(tx_fifo, CadenceUARTState,
+                            CADENCE_UART_TX_FIFO_SIZE),
+        VMSTATE_UINT32(rx_count, CadenceUARTState),
+        VMSTATE_UINT32(tx_count, CadenceUARTState),
+        VMSTATE_UINT32(rx_wpos, CadenceUARTState),
+        VMSTATE_TIMER_PTR(fifo_trigger_handle, CadenceUARTState),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static Property cadence_uart_properties[] = {
+    DEFINE_PROP_CHR("chardev", CadenceUARTState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void cadence_uart_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
 
-    sdc->init = cadence_uart_init;
+    dc->realize = cadence_uart_realize;
     dc->vmsd = &vmstate_cadence_uart;
     dc->reset = cadence_uart_reset;
-}
+    dc->props = cadence_uart_properties;
+  }
 
 static const TypeInfo cadence_uart_info = {
     .name          = TYPE_CADENCE_UART,
     .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(UartState),
+    .instance_size = sizeof(CadenceUARTState),
+    .instance_init = cadence_uart_init,
     .class_init    = cadence_uart_class_init,
 };
 
diff --git a/hw/char/debugcon.c b/hw/char/debugcon.c
index 36f1c4a..e7f025e 100644
--- a/hw/char/debugcon.c
+++ b/hw/char/debugcon.c
@@ -24,6 +24,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/char.h"
 #include "hw/isa/isa.h"
diff --git a/hw/char/digic-uart.c b/hw/char/digic-uart.c
index 8abe944..c7604e6 100644
--- a/hw/char/digic-uart.c
+++ b/hw/char/digic-uart.c
@@ -26,9 +26,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
+#include "qemu/log.h"
 
 #include "hw/char/digic-uart.h"
 
@@ -143,7 +145,6 @@
 {
     DigicUartState *s = DIGIC_UART(dev);
 
-    s->chr = qemu_char_get_next_serial();
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, uart_can_rx, uart_rx, uart_event, s);
     }
@@ -169,6 +170,11 @@
     }
 };
 
+static Property digic_uart_properties[] = {
+    DEFINE_PROP_CHR("chardev", DigicUartState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void digic_uart_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -176,6 +182,7 @@
     dc->realize = digic_uart_realize;
     dc->reset = digic_uart_reset;
     dc->vmsd = &vmstate_digic_uart;
+    dc->props = digic_uart_properties;
 }
 
 static const TypeInfo digic_uart_info = {
diff --git a/hw/char/escc.c b/hw/char/escc.c
index ba653ef..31a5f90 100644
--- a/hw/char/escc.c
+++ b/hw/char/escc.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/char/escc.h"
@@ -714,7 +715,7 @@
     return &d->mmio;
 }
 
-static const uint8_t qcode_to_keycode[Q_KEY_CODE_MAX] = {
+static const uint8_t qcode_to_keycode[Q_KEY_CODE__MAX] = {
     [Q_KEY_CODE_SHIFT]         = 99,
     [Q_KEY_CODE_SHIFT_R]       = 110,
     [Q_KEY_CODE_ALT]           = 19,
@@ -841,14 +842,16 @@
 {
     ChannelState *s = (ChannelState *)dev;
     int qcode, keycode;
+    InputKeyEvent *key;
 
-    assert(evt->kind == INPUT_EVENT_KIND_KEY);
-    qcode = qemu_input_key_value_to_qcode(evt->key->key);
+    assert(evt->type == INPUT_EVENT_KIND_KEY);
+    key = evt->u.key.data;
+    qcode = qemu_input_key_value_to_qcode(key->key);
     trace_escc_sunkbd_event_in(qcode, QKeyCode_lookup[qcode],
-                               evt->key->down);
+                               key->down);
 
     if (qcode == Q_KEY_CODE_CAPS_LOCK) {
-        if (evt->key->down) {
+        if (key->down) {
             s->caps_lock_mode ^= 1;
             if (s->caps_lock_mode == 2) {
                 return; /* Drop second press */
@@ -862,7 +865,7 @@
     }
 
     if (qcode == Q_KEY_CODE_NUM_LOCK) {
-        if (evt->key->down) {
+        if (key->down) {
             s->num_lock_mode ^= 1;
             if (s->num_lock_mode == 2) {
                 return; /* Drop second press */
@@ -876,7 +879,7 @@
     }
 
     keycode = qcode_to_keycode[qcode];
-    if (!evt->key->down) {
+    if (!key->down) {
         keycode |= 0x80;
     }
     trace_escc_sunkbd_event_out(keycode);
@@ -980,28 +983,40 @@
     sysbus_mmio_map(s, 0, base);
 }
 
-static int escc_init1(SysBusDevice *dev)
+static void escc_init1(Object *obj)
+{
+    ESCCState *s = ESCC(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
+    unsigned int i;
+
+    for (i = 0; i < 2; i++) {
+        sysbus_init_irq(dev, &s->chn[i].irq);
+        s->chn[i].chn = 1 - i;
+    }
+    s->chn[0].otherchn = &s->chn[1];
+    s->chn[1].otherchn = &s->chn[0];
+
+    sysbus_init_mmio(dev, &s->mmio);
+}
+
+static void escc_realize(DeviceState *dev, Error **errp)
 {
     ESCCState *s = ESCC(dev);
     unsigned int i;
 
     s->chn[0].disabled = s->disabled;
     s->chn[1].disabled = s->disabled;
+
+    memory_region_init_io(&s->mmio, OBJECT(dev), &escc_mem_ops, s, "escc",
+                          ESCC_SIZE << s->it_shift);
+
     for (i = 0; i < 2; i++) {
-        sysbus_init_irq(dev, &s->chn[i].irq);
-        s->chn[i].chn = 1 - i;
-        s->chn[i].clock = s->frequency / 2;
         if (s->chn[i].chr) {
+            s->chn[i].clock = s->frequency / 2;
             qemu_chr_add_handlers(s->chn[i].chr, serial_can_receive,
                                   serial_receive1, serial_event, &s->chn[i]);
         }
     }
-    s->chn[0].otherchn = &s->chn[1];
-    s->chn[1].otherchn = &s->chn[0];
-
-    memory_region_init_io(&s->mmio, OBJECT(s), &escc_mem_ops, s, "escc",
-                          ESCC_SIZE << s->it_shift);
-    sysbus_init_mmio(dev, &s->mmio);
 
     if (s->chn[0].type == mouse) {
         qemu_add_mouse_event_handler(sunmouse_event, &s->chn[0], 0,
@@ -1011,8 +1026,6 @@
         s->chn[1].hs = qemu_input_handler_register((DeviceState *)(&s->chn[1]),
                                                    &sunkbd_handler);
     }
-
-    return 0;
 }
 
 static Property escc_properties[] = {
@@ -1029,18 +1042,19 @@
 static void escc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = escc_init1;
     dc->reset = escc_reset;
+    dc->realize = escc_realize;
     dc->vmsd = &vmstate_escc;
     dc->props = escc_properties;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo escc_info = {
     .name          = TYPE_ESCC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ESCCState),
+    .instance_init = escc_init1,
     .class_init    = escc_class_init,
 };
 
diff --git a/hw/char/etraxfs_ser.c b/hw/char/etraxfs_ser.c
index 460094e..04ca04f 100644
--- a/hw/char/etraxfs_ser.c
+++ b/hw/char/etraxfs_ser.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 #include "qemu/log.h"
@@ -158,6 +159,11 @@
     }
 };
 
+static Property etraxfs_ser_properties[] = {
+    DEFINE_PROP_CHR("chardev", ETRAXSerial, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void serial_receive(void *opaque, const uint8_t *buf, int size)
 {
     ETRAXSerial *s = opaque;
@@ -165,7 +171,7 @@
 
     /* Got a byte.  */
     if (s->rx_fifo_len >= 16) {
-        qemu_log("WARNING: UART dropped char.\n");
+        D(qemu_log("WARNING: UART dropped char.\n"));
         return;
     }
 
@@ -182,15 +188,13 @@
 static int serial_can_receive(void *opaque)
 {
     ETRAXSerial *s = opaque;
-    int r;
 
     /* Is the receiver enabled?  */
     if (!(s->regs[RW_REC_CTRL] & (1 << 3))) {
         return 0;
     }
 
-    r = sizeof(s->rx_fifo) - s->rx_fifo_len;
-    return r;
+    return sizeof(s->rx_fifo) - s->rx_fifo_len;
 }
 
 static void serial_event(void *opaque, int event)
@@ -210,37 +214,42 @@
 
 }
 
-static int etraxfs_ser_init(SysBusDevice *dev)
+static void etraxfs_ser_init(Object *obj)
+{
+    ETRAXSerial *s = ETRAX_SERIAL(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
+
+    sysbus_init_irq(dev, &s->irq);
+    memory_region_init_io(&s->mmio, obj, &ser_ops, s,
+                          "etraxfs-serial", R_MAX * 4);
+    sysbus_init_mmio(dev, &s->mmio);
+}
+
+static void etraxfs_ser_realize(DeviceState *dev, Error **errp)
 {
     ETRAXSerial *s = ETRAX_SERIAL(dev);
 
-    sysbus_init_irq(dev, &s->irq);
-    memory_region_init_io(&s->mmio, OBJECT(s), &ser_ops, s,
-                          "etraxfs-serial", R_MAX * 4);
-    sysbus_init_mmio(dev, &s->mmio);
-
-    s->chr = qemu_char_get_next_serial();
     if (s->chr) {
         qemu_chr_add_handlers(s->chr,
                               serial_can_receive, serial_receive,
                               serial_event, s);
     }
-    return 0;
 }
 
 static void etraxfs_ser_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = etraxfs_ser_init;
     dc->reset = etraxfs_ser_reset;
+    dc->props = etraxfs_ser_properties;
+    dc->realize = etraxfs_ser_realize;
 }
 
 static const TypeInfo etraxfs_ser_info = {
     .name          = TYPE_ETRAX_FS_SERIAL,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ETRAXSerial),
+    .instance_init = etraxfs_ser_init,
     .class_init    = etraxfs_ser_class_init,
 };
 
diff --git a/hw/char/exynos4210_uart.c b/hw/char/exynos4210_uart.c
index 7614e58..885ecc0 100644
--- a/hw/char/exynos4210_uart.c
+++ b/hw/char/exynos4210_uart.c
@@ -19,7 +19,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
 
@@ -234,10 +236,8 @@
 
 static void fifo_reset(Exynos4210UartFIFO *q)
 {
-    if (q->data != NULL) {
-        g_free(q->data);
-        q->data = NULL;
-    }
+    g_free(q->data);
+    q->data = NULL;
 
     q->data = (uint8_t *)g_malloc0(q->size);
 
@@ -597,15 +597,17 @@
 
     if (!chr) {
         if (channel >= MAX_SERIAL_PORTS) {
-            hw_error("Only %d serial ports are supported by QEMU.\n",
-                     MAX_SERIAL_PORTS);
+            error_report("Only %d serial ports are supported by QEMU",
+                         MAX_SERIAL_PORTS);
+            exit(1);
         }
         chr = serial_hds[channel];
         if (!chr) {
             snprintf(label, ARRAY_SIZE(label), "%s%d", chr_name, channel);
             chr = qemu_chr_new(label, "null", NULL);
             if (!(chr)) {
-                hw_error("Can't assign serial port to UART%d.\n", channel);
+                error_report("Can't assign serial port to UART%d", channel);
+                exit(1);
             }
         }
     }
diff --git a/hw/char/goldfish_tty.c b/hw/char/goldfish_tty.c
index 7223c60..be9cae2 100644
--- a/hw/char/goldfish_tty.c
+++ b/hw/char/goldfish_tty.c
@@ -9,6 +9,11 @@
 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ** GNU General Public License for more details.
 */
+#include "qemu/osdep.h"
+#include "qemu/rcu.h"
+#include "cpu.h"
+#include "exec/cpu-all.h"
+#include "exec/memory.h"
 #include "exec/ram_addr.h"
 #include "migration/qemu-file.h"
 #include "sysemu/char.h"
@@ -86,7 +91,7 @@
     s->ready      = qemu_get_byte(f);
     s->data_count = qemu_get_byte(f);
 
-    if (qemu_get_buffer(f, s->data, s->data_count) < 0)
+    if (qemu_get_buffer(f, s->data, s->data_count) < s->data_count)
         return -1;
 
     qemu_set_irq(s->irq, s->ready && s->data_count > 0);
diff --git a/hw/char/grlib_apbuart.c b/hw/char/grlib_apbuart.c
index 35ef661..871524c 100644
--- a/hw/char/grlib_apbuart.c
+++ b/hw/char/grlib_apbuart.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 
diff --git a/hw/char/imx_serial.c b/hw/char/imx_serial.c
index f3fbc77..44856d6 100644
--- a/hw/char/imx_serial.c
+++ b/hw/char/imx_serial.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2008 OKL
  * Originally Written by Hans Jiang
  * Copyright (c) 2011 NICTA Pty Ltd.
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -17,63 +18,26 @@
  *     is a real serial device.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
+#include "qemu/osdep.h"
+#include "hw/char/imx_serial.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
-#include "hw/arm/imx.h"
+#include "qemu/log.h"
 
-//#define DEBUG_SERIAL 1
-#ifdef DEBUG_SERIAL
+#ifndef DEBUG_IMX_UART
+#define DEBUG_IMX_UART 0
+#endif
+
 #define DPRINTF(fmt, args...) \
-do { printf("imx_serial: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-//#define DEBUG_IMPLEMENTATION 1
-#ifdef DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-    do  { fprintf(stderr, "imx_serial: " fmt, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define TYPE_IMX_SERIAL "imx-serial"
-#define IMX_SERIAL(obj) OBJECT_CHECK(IMXSerialState, (obj), TYPE_IMX_SERIAL)
-
-typedef struct IMXSerialState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    int32_t readbuff;
-
-    uint32_t usr1;
-    uint32_t usr2;
-    uint32_t ucr1;
-    uint32_t ucr2;
-    uint32_t uts1;
-
-    /*
-     * The registers below are implemented just so that the
-     * guest OS sees what it has written
-     */
-    uint32_t onems;
-    uint32_t ufcr;
-    uint32_t ubmr;
-    uint32_t ubrc;
-    uint32_t ucr3;
-
-    qemu_irq irq;
-    CharDriverState *chr;
-} IMXSerialState;
+    do { \
+        if (DEBUG_IMX_UART) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_SERIAL, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static const VMStateDescription vmstate_imx_serial = {
-    .name = "imx-serial",
+    .name = TYPE_IMX_SERIAL,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -91,61 +55,14 @@
     },
 };
 
-
-#define URXD_CHARRDY    (1<<15)   /* character read is valid */
-#define URXD_ERR        (1<<14)   /* Character has error */
-#define URXD_BRK        (1<<11)   /* Break received */
-
-#define USR1_PARTYER    (1<<15)   /* Parity Error */
-#define USR1_RTSS       (1<<14)   /* RTS pin status */
-#define USR1_TRDY       (1<<13)   /* Tx ready */
-#define USR1_RTSD       (1<<12)   /* RTS delta: pin changed state */
-#define USR1_ESCF       (1<<11)   /* Escape sequence interrupt */
-#define USR1_FRAMERR    (1<<10)   /* Framing error  */
-#define USR1_RRDY       (1<<9)    /* receiver ready */
-#define USR1_AGTIM      (1<<8)    /* Aging timer interrupt */
-#define USR1_DTRD       (1<<7)    /* DTR changed */
-#define USR1_RXDS       (1<<6)    /* Receiver is idle */
-#define USR1_AIRINT     (1<<5)    /* Aysnch IR interrupt */
-#define USR1_AWAKE      (1<<4)    /* Falling edge detected on RXd pin */
-
-#define USR2_ADET       (1<<15)   /* Autobaud complete */
-#define USR2_TXFE       (1<<14)   /* Transmit FIFO empty */
-#define USR2_DTRF       (1<<13)   /* DTR/DSR transition */
-#define USR2_IDLE       (1<<12)   /* UART has been idle for too long */
-#define USR2_ACST       (1<<11)   /* Autobaud counter stopped */
-#define USR2_RIDELT     (1<<10)   /* Ring Indicator delta */
-#define USR2_RIIN       (1<<9)    /* Ring Indicator Input */
-#define USR2_IRINT      (1<<8)    /* Serial Infrared Interrupt */
-#define USR2_WAKE       (1<<7)    /* Start bit detected */
-#define USR2_DCDDELT    (1<<6)    /* Data Carrier Detect delta */
-#define USR2_DCDIN      (1<<5)    /* Data Carrier Detect Input */
-#define USR2_RTSF       (1<<4)    /* RTS transition */
-#define USR2_TXDC       (1<<3)    /* Transmission complete */
-#define USR2_BRCD       (1<<2)    /* Break condition detected */
-#define USR2_ORE        (1<<1)    /* Overrun error */
-#define USR2_RDR        (1<<0)    /* Receive data ready */
-
-#define UCR1_TRDYEN     (1<<13)   /* Tx Ready Interrupt Enable */
-#define UCR1_RRDYEN     (1<<9)    /* Rx Ready Interrupt Enable */
-#define UCR1_TXMPTYEN   (1<<6)    /* Tx Empty Interrupt Enable */
-#define UCR1_UARTEN     (1<<0)    /* UART Enable */
-
-#define UCR2_TXEN       (1<<2)    /* Transmitter enable */
-#define UCR2_RXEN       (1<<1)    /* Receiver enable */
-#define UCR2_SRST       (1<<0)    /* Reset complete */
-
-#define UTS1_TXEMPTY    (1<<6)
-#define UTS1_RXEMPTY    (1<<5)
-#define UTS1_TXFULL     (1<<4)
-#define UTS1_RXFULL     (1<<3)
-
 static void imx_update(IMXSerialState *s)
 {
     uint32_t flags;
 
     flags = (s->usr1 & s->ucr1) & (USR1_TRDY|USR1_RRDY);
-    if (!(s->ucr1 & UCR1_TXMPTYEN)) {
+    if (s->ucr1 & UCR1_TXMPTYEN) {
+        flags |= (s->uts1 & UTS1_TXEMPTY);
+    } else {
         flags &= ~USR1_TRDY;
     }
 
@@ -192,7 +109,8 @@
     IMXSerialState *s = (IMXSerialState *)opaque;
     uint32_t c;
 
-    DPRINTF("read(offset=%x)\n", offset >> 2);
+    DPRINTF("read(offset=0x%" HWADDR_PRIx ")\n", offset);
+
     switch (offset >> 2) {
     case 0x0: /* URXD */
         c = s->readbuff;
@@ -203,7 +121,9 @@
             s->usr2 &= ~USR2_RDR;
             s->uts1 |= UTS1_RXEMPTY;
             imx_update(s);
-            qemu_chr_accept_input(s->chr);
+            if (s->chr) {
+                qemu_chr_accept_input(s->chr);
+            }
         }
         return c;
 
@@ -242,20 +162,20 @@
         return 0x0; /* TODO */
 
     default:
-        IPRINTF("imx_serial_read: bad offset: 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
         return 0;
     }
 }
 
 static void imx_serial_write(void *opaque, hwaddr offset,
-                      uint64_t value, unsigned size)
+                             uint64_t value, unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
     unsigned char ch;
 
-    DPRINTF("write(offset=%x, value = %x) to %s\n",
-            offset >> 2,
-            (unsigned int)value, s->chr ? s->chr->label : "NODEV");
+    DPRINTF("write(offset=0x%" HWADDR_PRIx ", value = 0x%x) to %s\n",
+            offset, (unsigned int)value, s->chr ? s->chr->label : "NODEV");
 
     switch (offset >> 2) {
     case 0x10: /* UTXD */
@@ -273,7 +193,9 @@
 
     case 0x20: /* UCR1 */
         s->ucr1 = value & 0xffff;
+
         DPRINTF("write(ucr1=%x)\n", (unsigned int)value);
+
         imx_update(s);
         break;
 
@@ -290,7 +212,9 @@
         }
         if (value & UCR2_RXEN) {
             if (!(s->ucr2 & UCR2_RXEN)) {
-                qemu_chr_accept_input(s->chr);
+                if (s->chr) {
+                    qemu_chr_accept_input(s->chr);
+                }
             }
         }
         s->ucr2 = value & 0xffff;
@@ -298,25 +222,25 @@
 
     case 0x25: /* USR1 */
         value &= USR1_AWAKE | USR1_AIRINT | USR1_DTRD | USR1_AGTIM |
-            USR1_FRAMERR | USR1_ESCF | USR1_RTSD | USR1_PARTYER;
+                 USR1_FRAMERR | USR1_ESCF | USR1_RTSD | USR1_PARTYER;
         s->usr1 &= ~value;
         break;
 
     case 0x26: /* USR2 */
-       /*
-        * Writing 1 to some bits clears them; all other
-        * values are ignored
-        */
+        /*
+         * Writing 1 to some bits clears them; all other
+         * values are ignored
+         */
         value &= USR2_ADET | USR2_DTRF | USR2_IDLE | USR2_ACST |
-            USR2_RIDELT | USR2_IRINT | USR2_WAKE |
-            USR2_DCDDELT | USR2_RTSF | USR2_BRCD | USR2_ORE;
+                 USR2_RIDELT | USR2_IRINT | USR2_WAKE |
+                 USR2_DCDDELT | USR2_RTSF | USR2_BRCD | USR2_ORE;
         s->usr2 &= ~value;
         break;
 
-        /*
-         * Linux expects to see what it writes to these registers
-         * We don't currently alter the baud rate
-         */
+    /*
+     * Linux expects to see what it writes to these registers
+     * We don't currently alter the baud rate
+     */
     case 0x29: /* UBIR */
         s->ubrc = value & 0xffff;
         break;
@@ -339,12 +263,14 @@
 
     case 0x2d: /* UTS1 */
     case 0x23: /* UCR4 */
-        IPRINTF("Unimplemented Register %x written to\n", offset >> 2);
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: Unimplemented reg 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
         /* TODO */
         break;
 
     default:
-        IPRINTF("imx_serial_write: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SERIAL, __func__, offset);
     }
 }
 
@@ -357,7 +283,9 @@
 static void imx_put_data(void *opaque, uint32_t value)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
+
     DPRINTF("received char\n");
+
     s->usr1 |= USR1_RRDY;
     s->usr2 |= USR2_RDR;
     s->uts1 &= ~UTS1_RXEMPTY;
@@ -384,62 +312,30 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int imx_serial_init(SysBusDevice *dev)
+static void imx_serial_realize(DeviceState *dev, Error **errp)
 {
     IMXSerialState *s = IMX_SERIAL(dev);
 
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &imx_serial_ops, s,
-                          "imx-serial", 0x1000);
-    sysbus_init_mmio(dev, &s->iomem);
-    sysbus_init_irq(dev, &s->irq);
-
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, imx_can_receive, imx_receive,
                               imx_event, s);
     } else {
-        DPRINTF("No char dev for uart at 0x%lx\n",
-                (unsigned long)s->iomem.ram_addr);
+        DPRINTF("No char dev for uart\n");
     }
-
-    return 0;
 }
 
-void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq)
+static void imx_serial_init(Object *obj)
 {
-    DeviceState *dev;
-    SysBusDevice *bus;
-    CharDriverState *chr;
-    const char chr_name[] = "serial";
-    char label[ARRAY_SIZE(chr_name) + 1];
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    IMXSerialState *s = IMX_SERIAL(obj);
 
-    dev = qdev_create(NULL, TYPE_IMX_SERIAL);
-
-    if (uart >= MAX_SERIAL_PORTS) {
-        hw_error("Cannot assign uart %d: QEMU supports only %d ports\n",
-                 uart, MAX_SERIAL_PORTS);
-    }
-    chr = serial_hds[uart];
-    if (!chr) {
-        snprintf(label, ARRAY_SIZE(label), "%s%d", chr_name, uart);
-        chr = qemu_chr_new(label, "null", NULL);
-        if (!(chr)) {
-            hw_error("Can't assign serial port to imx-uart%d.\n", uart);
-        }
-    }
-
-    qdev_prop_set_chr(dev, "chardev", chr);
-    bus = SYS_BUS_DEVICE(dev);
-    qdev_init_nofail(dev);
-    if (addr != (hwaddr)-1) {
-        sysbus_mmio_map(bus, 0, addr);
-    }
-    sysbus_connect_irq(bus, 0, irq);
-
+    memory_region_init_io(&s->iomem, obj, &imx_serial_ops, s,
+                          TYPE_IMX_SERIAL, 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
 }
 
-
-static Property imx32_serial_properties[] = {
+static Property imx_serial_properties[] = {
     DEFINE_PROP_CHR("chardev", IMXSerialState, chr),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -447,21 +343,21 @@
 static void imx_serial_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = imx_serial_init;
+    dc->realize = imx_serial_realize;
     dc->vmsd = &vmstate_imx_serial;
     dc->reset = imx_serial_reset_at_boot;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
     dc->desc = "i.MX series UART";
-    dc->props = imx32_serial_properties;
+    dc->props = imx_serial_properties;
 }
 
 static const TypeInfo imx_serial_info = {
-    .name = TYPE_IMX_SERIAL,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(IMXSerialState),
-    .class_init = imx_serial_class_init,
+    .name           = TYPE_IMX_SERIAL,
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(IMXSerialState),
+    .instance_init  = imx_serial_init,
+    .class_init     = imx_serial_class_init,
 };
 
 static void imx_serial_register_types(void)
diff --git a/hw/char/ipoctal232.c b/hw/char/ipoctal232.c
index c8d5cdb..9ead32a 100644
--- a/hw/char/ipoctal232.c
+++ b/hw/char/ipoctal232.c
@@ -2,12 +2,13 @@
  * QEMU GE IP-Octal 232 IndustryPack emulation
  *
  * Copyright (C) 2012 Igalia, S.L.
- * Author: Alberto Garcia <agarcia@igalia.com>
+ * Author: Alberto Garcia <berto@igalia.com>
  *
  * This code is licensed under the GNU GPL v2 or (at your option) any
  * later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/ipack/ipack.h"
 #include "qemu/bitops.h"
 #include "sysemu/char.h"
diff --git a/hw/char/lm32_juart.c b/hw/char/lm32_juart.c
index 628a86f..28c2cf7 100644
--- a/hw/char/lm32_juart.c
+++ b/hw/char/lm32_juart.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -113,16 +114,13 @@
     s->jrx = 0;
 }
 
-static int lm32_juart_init(SysBusDevice *dev)
+static void lm32_juart_realize(DeviceState *dev, Error **errp)
 {
     LM32JuartState *s = LM32_JUART(dev);
 
-    s->chr = qemu_char_get_next_serial();
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, juart_can_rx, juart_rx, juart_event, s);
     }
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_lm32_juart = {
@@ -136,14 +134,19 @@
     }
 };
 
+static Property lm32_juart_properties[] = {
+    DEFINE_PROP_CHR("chardev", LM32JuartState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void lm32_juart_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = lm32_juart_init;
     dc->reset = juart_reset;
     dc->vmsd = &vmstate_lm32_juart;
+    dc->props = lm32_juart_properties;
+    dc->realize = lm32_juart_realize;
 }
 
 static const TypeInfo lm32_juart_info = {
diff --git a/hw/char/lm32_uart.c b/hw/char/lm32_uart.c
index 4f20966..b5c760d 100644
--- a/hw/char/lm32_uart.c
+++ b/hw/char/lm32_uart.c
@@ -22,6 +22,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -248,22 +249,25 @@
     s->regs[R_LSR] = LSR_THRE | LSR_TEMT;
 }
 
-static int lm32_uart_init(SysBusDevice *dev)
+static void lm32_uart_init(Object *obj)
 {
-    LM32UartState *s = LM32_UART(dev);
+    LM32UartState *s = LM32_UART(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     sysbus_init_irq(dev, &s->irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &uart_ops, s,
+    memory_region_init_io(&s->iomem, obj, &uart_ops, s,
                           "uart", R_MAX * 4);
     sysbus_init_mmio(dev, &s->iomem);
+}
 
-    s->chr = qemu_char_get_next_serial();
+static void lm32_uart_realize(DeviceState *dev, Error **errp)
+{
+    LM32UartState *s = LM32_UART(dev);
+
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, uart_can_rx, uart_rx, uart_event, s);
     }
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_lm32_uart = {
@@ -276,20 +280,26 @@
     }
 };
 
+static Property lm32_uart_properties[] = {
+    DEFINE_PROP_CHR("chardev", LM32UartState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void lm32_uart_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = lm32_uart_init;
     dc->reset = uart_reset;
     dc->vmsd = &vmstate_lm32_uart;
+    dc->props = lm32_uart_properties;
+    dc->realize = lm32_uart_realize;
 }
 
 static const TypeInfo lm32_uart_info = {
     .name          = TYPE_LM32_UART,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32UartState),
+    .instance_init = lm32_uart_init,
     .class_init    = lm32_uart_class_init,
 };
 
diff --git a/hw/char/mcf_uart.c b/hw/char/mcf_uart.c
index 98fd44e..3c0438f 100644
--- a/hw/char/mcf_uart.c
+++ b/hw/char/mcf_uart.c
@@ -5,6 +5,7 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "sysemu/char.h"
@@ -126,7 +127,7 @@
 static void mcf_do_command(mcf_uart_state *s, uint8_t cmd)
 {
     /* Misc command.  */
-    switch ((cmd >> 4) & 3) {
+    switch ((cmd >> 4) & 7) {
     case 0: /* No-op.  */
         break;
     case 1: /* Reset mode register pointer.  */
diff --git a/hw/char/milkymist-uart.c b/hw/char/milkymist-uart.c
index d05b825..baddb37 100644
--- a/hw/char/milkymist-uart.c
+++ b/hw/char/milkymist-uart.c
@@ -18,9 +18,10 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/uart.pdf
+ *   http://milkymist.walle.cc/socdoc/uart.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -199,7 +200,6 @@
 {
     MilkymistUartState *s = MILKYMIST_UART(dev);
 
-    s->chr = qemu_char_get_next_serial();
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, uart_can_rx, uart_rx, uart_event, s);
     }
@@ -227,6 +227,11 @@
     }
 };
 
+static Property milkymist_uart_properties[] = {
+    DEFINE_PROP_CHR("chardev", MilkymistUartState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void milkymist_uart_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -234,6 +239,7 @@
     dc->realize = milkymist_uart_realize;
     dc->reset = milkymist_uart_reset;
     dc->vmsd = &vmstate_milkymist_uart;
+    dc->props = milkymist_uart_properties;
 }
 
 static const TypeInfo milkymist_uart_info = {
diff --git a/hw/char/omap_uart.c b/hw/char/omap_uart.c
index 0b91693..415bec5 100644
--- a/hw/char/omap_uart.c
+++ b/hw/char/omap_uart.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
@@ -55,8 +56,7 @@
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
 {
-    struct omap_uart_s *s = (struct omap_uart_s *)
-            g_malloc0(sizeof(struct omap_uart_s));
+    struct omap_uart_s *s = g_new0(struct omap_uart_s, 1);
 
     s->base = base;
     s->fclk = fclk;
@@ -112,7 +112,8 @@
     struct omap_uart_s *s = (struct omap_uart_s *) opaque;
 
     if (size == 4) {
-        return omap_badwidth_write8(opaque, addr, value);
+        omap_badwidth_write8(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
diff --git a/hw/char/parallel.c b/hw/char/parallel.c
index c2b553f..11c78fe 100644
--- a/hw/char/parallel.c
+++ b/hw/char/parallel.c
@@ -22,6 +22,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/char.h"
 #include "hw/isa/isa.h"
diff --git a/hw/char/pl011.c b/hw/char/pl011.c
index 0a45115..c0fbf8a 100644
--- a/hw/char/pl011.c
+++ b/hw/char/pl011.c
@@ -7,8 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
+#include "qemu/log.h"
 
 #define TYPE_PL011 "pl011"
 #define PL011(obj) OBJECT_CHECK(PL011State, (obj), TYPE_PL011)
@@ -272,6 +274,11 @@
     }
 };
 
+static Property pl011_properties[] = {
+    DEFINE_PROP_CHR("chardev", PL011State, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void pl011_init(Object *obj)
 {
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
@@ -293,8 +300,6 @@
 {
     PL011State *s = PL011(dev);
 
-    s->chr = qemu_char_get_next_serial();
-
     if (s->chr) {
         qemu_chr_add_handlers(s->chr, pl011_can_receive, pl011_receive,
                               pl011_event, s);
@@ -307,6 +312,7 @@
 
     dc->realize = pl011_realize;
     dc->vmsd = &vmstate_pl011;
+    dc->props = pl011_properties;
 }
 
 static const TypeInfo pl011_arm_info = {
diff --git a/hw/char/sclpconsole-lm.c b/hw/char/sclpconsole-lm.c
index a9f5e62..a22ad8d 100644
--- a/hw/char/sclpconsole-lm.c
+++ b/hw/char/sclpconsole-lm.c
@@ -13,6 +13,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
@@ -43,6 +44,10 @@
     uint8_t buf[SIZE_CONSOLE_BUFFER];
 } SCLPConsoleLM;
 
+#define TYPE_SCLPLM_CONSOLE "sclplmconsole"
+#define SCLPLM_CONSOLE(obj) \
+    OBJECT_CHECK(SCLPConsoleLM, (obj), TYPE_SCLPLM_CONSOLE)
+
 /*
 *  Character layer call-back functions
  *
@@ -115,7 +120,7 @@
 {
     int len;
 
-    SCLPConsoleLM *cons = DO_UPCAST(SCLPConsoleLM, event, event);
+    SCLPConsoleLM *cons = SCLPLM_CONSOLE(event);
 
     len = cons->length;
     /* data need to fit into provided SCLP buffer */
@@ -189,7 +194,7 @@
     int ret = 0;
     const uint8_t *buf_offset;
 
-    SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+    SCLPConsoleLM *scon = SCLPLM_CONSOLE(event);
 
     if (!scon->chr) {
         /* If there's no backend, we can just say we consumed all data. */
@@ -243,7 +248,7 @@
     int errors = 0;
     MDBO *mdbo;
     SclpMsg *data = (SclpMsg *) ebh;
-    SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+    SCLPConsoleLM *scon = SCLPLM_CONSOLE(event);
 
     len = be16_to_cpu(data->mdb.header.length);
     if (len < sizeof(data->mdb.header)) {
@@ -312,7 +317,7 @@
 {
     static bool console_available;
 
-    SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+    SCLPConsoleLM *scon = SCLPLM_CONSOLE(event);
 
     if (console_available) {
         error_report("Multiple line-mode operator consoles are not supported");
@@ -335,7 +340,7 @@
 static void console_reset(DeviceState *dev)
 {
    SCLPEvent *event = SCLP_EVENT(dev);
-   SCLPConsoleLM *scon = DO_UPCAST(SCLPConsoleLM, event, event);
+   SCLPConsoleLM *scon = SCLPLM_CONSOLE(event);
 
    event->event_pending = false;
    scon->length = 0;
@@ -364,6 +369,7 @@
     ec->can_handle_event = can_handle_event;
     ec->read_event_data = read_event_data;
     ec->write_event_data = write_event_data;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo sclp_console_info = {
diff --git a/hw/char/sclpconsole.c b/hw/char/sclpconsole.c
index 79891df..d224648 100644
--- a/hw/char/sclpconsole.c
+++ b/hw/char/sclpconsole.c
@@ -12,7 +12,8 @@
  *
  */
 
-#include <hw/qdev.h>
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
 #include "qemu/thread.h"
 #include "qemu/error-report.h"
 
@@ -39,6 +40,10 @@
     bool notify;            /* qemu_notify_event() req'd if true           */
 } SCLPConsole;
 
+#define TYPE_SCLP_CONSOLE "sclpconsole"
+#define SCLP_CONSOLE(obj) \
+    OBJECT_CHECK(SCLPConsole, (obj), TYPE_SCLP_CONSOLE)
+
 /* character layer call-back functions */
 
 /* Return number of bytes that fit into iov buffer */
@@ -94,7 +99,7 @@
 static void get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
                              int avail)
 {
-    SCLPConsole *cons = DO_UPCAST(SCLPConsole, event, event);
+    SCLPConsole *cons = SCLP_CONSOLE(event);
 
     /* first byte is hex 0 saying an ascii string follows */
     *buf++ = '\0';
@@ -156,7 +161,7 @@
 static ssize_t write_console_data(SCLPEvent *event, const uint8_t *buf,
                                   size_t len)
 {
-    SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+    SCLPConsole *scon = SCLP_CONSOLE(event);
 
     if (!scon->chr) {
         /* If there's no backend, we can just say we consumed all data. */
@@ -213,7 +218,7 @@
 {
     static bool console_available;
 
-    SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+    SCLPConsole *scon = SCLP_CONSOLE(event);
 
     if (console_available) {
         error_report("Multiple VT220 operator consoles are not supported");
@@ -231,7 +236,7 @@
 static void console_reset(DeviceState *dev)
 {
    SCLPEvent *event = SCLP_EVENT(dev);
-   SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+   SCLPConsole *scon = SCLP_CONSOLE(event);
 
    event->event_pending = false;
    scon->iov_sclp = 0;
@@ -266,6 +271,7 @@
     ec->can_handle_event = can_handle_event;
     ec->read_event_data = read_event_data;
     ec->write_event_data = write_event_data;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo sclp_console_info = {
diff --git a/hw/char/serial-isa.c b/hw/char/serial-isa.c
index c9fcb27..1594ec4 100644
--- a/hw/char/serial-isa.c
+++ b/hw/char/serial-isa.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/char/serial.h"
 #include "hw/isa/isa.h"
 
@@ -119,20 +121,27 @@
 
 type_init(serial_register_types)
 
-bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr)
+static void serial_isa_init(ISABus *bus, int index, CharDriverState *chr)
 {
     DeviceState *dev;
     ISADevice *isadev;
 
-    isadev = isa_try_create(bus, TYPE_ISA_SERIAL);
-    if (!isadev) {
-        return false;
-    }
+    isadev = isa_create(bus, TYPE_ISA_SERIAL);
     dev = DEVICE(isadev);
     qdev_prop_set_uint32(dev, "index", index);
     qdev_prop_set_chr(dev, "chardev", chr);
-    if (qdev_init(dev) < 0) {
-        return false;
+    qdev_init_nofail(dev);
+}
+
+void serial_hds_isa_init(ISABus *bus, int n)
+{
+    int i;
+
+    assert(n <= MAX_SERIAL_PORTS);
+
+    for (i = 0; i < n; ++i) {
+        if (serial_hds[i]) {
+            serial_isa_init(bus, i, serial_hds[i]);
+        }
     }
-    return true;
 }
diff --git a/hw/char/serial-pci.c b/hw/char/serial-pci.c
index f05c9b4..303104d 100644
--- a/hw/char/serial-pci.c
+++ b/hw/char/serial-pci.c
@@ -25,9 +25,10 @@
 
 /* see docs/specs/pci-serial.txt */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/char/serial.h"
 #include "hw/pci/pci.h"
-#include "qapi/qmp/qerror.h"
 
 #define PCI_SERIAL_MAX_PORTS 4
 
@@ -48,7 +49,9 @@
     uint8_t      prog_if;
 } PCIMultiSerialState;
 
-static int serial_pci_init(PCIDevice *dev)
+static void multi_serial_pci_exit(PCIDevice *dev);
+
+static void serial_pci_realize(PCIDevice *dev, Error **errp)
 {
     PCISerialState *pci = DO_UPCAST(PCISerialState, dev, dev);
     SerialState *s = &pci->state;
@@ -57,9 +60,8 @@
     s->baudbase = 115200;
     serial_realize_core(s, &err);
     if (err != NULL) {
-        qerror_report_err(err);
-        error_free(err);
-        return -1;
+        error_propagate(errp, err);
+        return;
     }
 
     pci->dev.config[PCI_CLASS_PROG] = pci->prog_if;
@@ -68,7 +70,6 @@
 
     memory_region_init_io(&s->io, OBJECT(pci), &serial_io_ops, s, "serial", 8);
     pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io);
-    return 0;
 }
 
 static void multi_serial_irq_mux(void *opaque, int n, int level)
@@ -85,48 +86,48 @@
     pci_set_irq(&pci->dev, pending);
 }
 
-static int multi_serial_pci_init(PCIDevice *dev)
+static void multi_serial_pci_realize(PCIDevice *dev, Error **errp)
 {
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
     PCIMultiSerialState *pci = DO_UPCAST(PCIMultiSerialState, dev, dev);
     SerialState *s;
     Error *err = NULL;
-    int i;
+    int i, nr_ports = 0;
 
     switch (pc->device_id) {
     case 0x0003:
-        pci->ports = 2;
+        nr_ports = 2;
         break;
     case 0x0004:
-        pci->ports = 4;
+        nr_ports = 4;
         break;
     }
-    assert(pci->ports > 0);
-    assert(pci->ports <= PCI_SERIAL_MAX_PORTS);
+    assert(nr_ports > 0);
+    assert(nr_ports <= PCI_SERIAL_MAX_PORTS);
 
     pci->dev.config[PCI_CLASS_PROG] = pci->prog_if;
     pci->dev.config[PCI_INTERRUPT_PIN] = 0x01;
-    memory_region_init(&pci->iobar, OBJECT(pci), "multiserial", 8 * pci->ports);
+    memory_region_init(&pci->iobar, OBJECT(pci), "multiserial", 8 * nr_ports);
     pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &pci->iobar);
     pci->irqs = qemu_allocate_irqs(multi_serial_irq_mux, pci,
-                                   pci->ports);
+                                   nr_ports);
 
-    for (i = 0; i < pci->ports; i++) {
+    for (i = 0; i < nr_ports; i++) {
         s = pci->state + i;
         s->baudbase = 115200;
         serial_realize_core(s, &err);
         if (err != NULL) {
-            qerror_report_err(err);
-            error_free(err);
-            return -1;
+            error_propagate(errp, err);
+            multi_serial_pci_exit(dev);
+            return;
         }
         s->irq = pci->irqs[i];
         pci->name[i] = g_strdup_printf("uart #%d", i+1);
         memory_region_init_io(&s->io, OBJECT(pci), &serial_io_ops, s,
                               pci->name[i], 8);
         memory_region_add_subregion(&pci->iobar, 8 * i, &s->io);
+        pci->ports++;
     }
-    return 0;
 }
 
 static void serial_pci_exit(PCIDevice *dev)
@@ -203,7 +204,7 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
-    pc->init = serial_pci_init;
+    pc->realize = serial_pci_realize;
     pc->exit = serial_pci_exit;
     pc->vendor_id = PCI_VENDOR_ID_REDHAT;
     pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL;
@@ -218,7 +219,7 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
-    pc->init = multi_serial_pci_init;
+    pc->realize = multi_serial_pci_realize;
     pc->exit = multi_serial_pci_exit;
     pc->vendor_id = PCI_VENDOR_ID_REDHAT;
     pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL2;
@@ -233,7 +234,7 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
-    pc->init = multi_serial_pci_init;
+    pc->realize = multi_serial_pci_realize;
     pc->exit = multi_serial_pci_exit;
     pc->vendor_id = PCI_VENDOR_ID_REDHAT;
     pc->device_id = PCI_DEVICE_ID_REDHAT_SERIAL4;
diff --git a/hw/char/serial.c b/hw/char/serial.c
index ebcacdc..3442f47 100644
--- a/hw/char/serial.c
+++ b/hw/char/serial.c
@@ -23,8 +23,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/char/serial.h"
 #include "sysemu/char.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
@@ -104,6 +106,7 @@
 #endif
 
 static void serial_receive1(void *opaque, const uint8_t *buf, int size);
+static void serial_xmit(SerialState *s);
 
 static inline void recv_fifo_put(SerialState *s, uint8_t chr)
 {
@@ -177,7 +180,7 @@
     ssp.parity = parity;
     ssp.data_bits = data_bits;
     ssp.stop_bits = stop_bits;
-    s->char_transmit_time =  (get_ticks_per_sec() / speed) * frame_size;
+    s->char_transmit_time =  (NANOSECONDS_PER_SECOND / speed) * frame_size;
     qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
 
     DPRINTF("speed=%d parity=%c data=%d stop=%d\n",
@@ -215,63 +218,67 @@
     /* The real 16550A apparently has a 250ns response latency to line status changes.
        We'll be lazy and poll only every 10ms, and only poll it at all if MSI interrupts are turned on */
 
-    if (s->poll_msl)
-        timer_mod(s->modem_status_poll, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + get_ticks_per_sec() / 100);
+    if (s->poll_msl) {
+        timer_mod(s->modem_status_poll, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  NANOSECONDS_PER_SECOND / 100);
+    }
 }
 
-static gboolean serial_xmit(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean serial_watch_cb(GIOChannel *chan, GIOCondition cond,
+                                void *opaque)
 {
     SerialState *s = opaque;
+    s->watch_tag = 0;
+    serial_xmit(s);
+    return FALSE;
+}
 
+static void serial_xmit(SerialState *s)
+{
     do {
-        if (s->tsr_retry <= 0) {
+        assert(!(s->lsr & UART_LSR_TEMT));
+        if (s->tsr_retry == 0) {
+            assert(!(s->lsr & UART_LSR_THRE));
+
             if (s->fcr & UART_FCR_FE) {
-                if (fifo8_is_empty(&s->xmit_fifo)) {
-                    return FALSE;
-                }
+                assert(!fifo8_is_empty(&s->xmit_fifo));
                 s->tsr = fifo8_pop(&s->xmit_fifo);
                 if (!s->xmit_fifo.num) {
                     s->lsr |= UART_LSR_THRE;
                 }
-            } else if ((s->lsr & UART_LSR_THRE)) {
-                return FALSE;
             } else {
                 s->tsr = s->thr;
                 s->lsr |= UART_LSR_THRE;
-                s->lsr &= ~UART_LSR_TEMT;
+            }
+            if ((s->lsr & UART_LSR_THRE) && !s->thr_ipending) {
+                s->thr_ipending = 1;
+                serial_update_irq(s);
             }
         }
 
         if (s->mcr & UART_MCR_LOOP) {
             /* in loopback mode, say that we just received a char */
             serial_receive1(s, &s->tsr, 1);
-        } else if (qemu_chr_fe_write(s->chr, &s->tsr, 1) != 1) {
-            if (s->tsr_retry >= 0 && s->tsr_retry < MAX_XMIT_RETRY &&
-                qemu_chr_fe_add_watch(s->chr, G_IO_OUT|G_IO_HUP,
-                                      serial_xmit, s) > 0) {
+        } else if (qemu_chr_fe_write(s->chr, &s->tsr, 1) != 1 &&
+                   s->tsr_retry < MAX_XMIT_RETRY) {
+            assert(s->watch_tag == 0);
+            s->watch_tag = qemu_chr_fe_add_watch(s->chr, G_IO_OUT|G_IO_HUP,
+                                                 serial_watch_cb, s);
+            if (s->watch_tag > 0) {
                 s->tsr_retry++;
-                return FALSE;
+                return;
             }
-            s->tsr_retry = 0;
-        } else {
-            s->tsr_retry = 0;
         }
+        s->tsr_retry = 0;
+
         /* Transmit another byte if it is already available. It is only
            possible when FIFO is enabled and not empty. */
-    } while ((s->fcr & UART_FCR_FE) && !fifo8_is_empty(&s->xmit_fifo));
+    } while (!(s->lsr & UART_LSR_THRE));
 
     s->last_xmit_ts = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-
-    if (s->lsr & UART_LSR_THRE) {
-        s->lsr |= UART_LSR_TEMT;
-        s->thr_ipending = 1;
-        serial_update_irq(s);
-    }
-
-    return FALSE;
+    s->lsr |= UART_LSR_TEMT;
 }
 
-
 /* Setter for FCR.
    is_load flag means, that value is set while loading VM state
    and interrupt should not be invoked */
@@ -323,13 +330,13 @@
                     fifo8_pop(&s->xmit_fifo);
                 }
                 fifo8_push(&s->xmit_fifo, s->thr);
-                s->lsr &= ~UART_LSR_TEMT;
             }
             s->thr_ipending = 0;
             s->lsr &= ~UART_LSR_THRE;
+            s->lsr &= ~UART_LSR_TEMT;
             serial_update_irq(s);
-            if (s->tsr_retry <= 0) {
-                serial_xmit(NULL, G_IO_OUT, s);
+            if (s->tsr_retry == 0) {
+                serial_xmit(s);
             }
         }
         break;
@@ -338,10 +345,12 @@
             s->divider = (s->divider & 0x00ff) | (val << 8);
             serial_update_parameters(s);
         } else {
+            uint8_t changed = (s->ier ^ val) & 0x0f;
             s->ier = val & 0x0f;
             /* If the backend device is a real serial port, turn polling of the modem
-               status lines on physical port on or off depending on UART_IER_MSI state */
-            if (s->poll_msl >= 0) {
+             * status lines on physical port on or off depending on UART_IER_MSI state.
+             */
+            if ((changed & UART_IER_MSI) && s->poll_msl >= 0) {
                 if (s->ier & UART_IER_MSI) {
                      s->poll_msl = 1;
                      serial_update_msl(s);
@@ -350,8 +359,27 @@
                      s->poll_msl = 0;
                 }
             }
-            if (s->lsr & UART_LSR_THRE) {
-                s->thr_ipending = 1;
+
+            /* Turning on the THRE interrupt on IER can trigger the interrupt
+             * if LSR.THRE=1, even if it had been masked before by reading IIR.
+             * This is not in the datasheet, but Windows relies on it.  It is
+             * unclear if THRE has to be resampled every time THRI becomes
+             * 1, or only on the rising edge.  Bochs does the latter, and Windows
+             * always toggles IER to all zeroes and back to all ones, so do the
+             * same.
+             *
+             * If IER.THRI is zero, thr_ipending is not used.  Set it to zero
+             * so that the thr_ipending subsection is not migrated.
+             */
+            if (changed & UART_IER_THRI) {
+                if ((s->ier & UART_IER_THRI) && (s->lsr & UART_LSR_THRE)) {
+                    s->thr_ipending = 1;
+                } else {
+                    s->thr_ipending = 0;
+                }
+            }
+
+            if (changed) {
                 serial_update_irq(s);
             }
         }
@@ -365,12 +393,15 @@
         /* FIFO clear */
 
         if (val & UART_FCR_RFR) {
+            s->lsr &= ~(UART_LSR_DR | UART_LSR_BI);
             timer_del(s->fifo_timeout_timer);
             s->timeout_ipending = 0;
             fifo8_reset(&s->recv_fifo);
         }
 
         if (val & UART_FCR_XFR) {
+            s->lsr |= UART_LSR_THRE;
+            s->thr_ipending = 1;
             fifo8_reset(&s->xmit_fifo);
         }
 
@@ -613,6 +644,31 @@
     if (s->thr_ipending == -1) {
         s->thr_ipending = ((s->iir & UART_IIR_ID) == UART_IIR_THRI);
     }
+
+    if (s->tsr_retry > 0) {
+        /* tsr_retry > 0 implies LSR.TEMT = 0 (transmitter not empty).  */
+        if (s->lsr & UART_LSR_TEMT) {
+            error_report("inconsistent state in serial device "
+                         "(tsr empty, tsr_retry=%d", s->tsr_retry);
+            return -1;
+        }
+
+        if (s->tsr_retry > MAX_XMIT_RETRY) {
+            s->tsr_retry = MAX_XMIT_RETRY;
+        }
+
+        assert(s->watch_tag == 0);
+        s->watch_tag = qemu_chr_fe_add_watch(s->chr, G_IO_OUT|G_IO_HUP,
+                                             serial_watch_cb, s);
+    } else {
+        /* tsr_retry == 0 implies LSR.TEMT = 1 (transmitter empty).  */
+        if (!(s->lsr & UART_LSR_TEMT)) {
+            error_report("inconsistent state in serial device "
+                         "(tsr not empty, tsr_retry=0");
+            return -1;
+        }
+    }
+
     s->last_break_enable = (s->lcr >> 6) & 1;
     /* Initialize fcr via setter to perform essential side-effects */
     serial_write_fcr(s, s->fcr_vmstate);
@@ -623,14 +679,24 @@
 static bool serial_thr_ipending_needed(void *opaque)
 {
     SerialState *s = opaque;
-    bool expected_value = ((s->iir & UART_IIR_ID) == UART_IIR_THRI);
-    return s->thr_ipending != expected_value;
+
+    if (s->ier & UART_IER_THRI) {
+        bool expected_value = ((s->iir & UART_IIR_ID) == UART_IIR_THRI);
+        return s->thr_ipending != expected_value;
+    } else {
+        /* LSR.THRE will be sampled again when the interrupt is
+         * enabled.  thr_ipending is not used in this case, do
+         * not migrate it.
+         */
+        return false;
+    }
 }
 
-const VMStateDescription vmstate_serial_thr_ipending = {
+static const VMStateDescription vmstate_serial_thr_ipending = {
     .name = "serial/thr_ipending",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = serial_thr_ipending_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(thr_ipending, SerialState),
         VMSTATE_END_OF_LIST()
@@ -643,12 +709,13 @@
     return s->tsr_retry != 0;
 }
 
-const VMStateDescription vmstate_serial_tsr = {
+static const VMStateDescription vmstate_serial_tsr = {
     .name = "serial/tsr",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = serial_tsr_needed,
     .fields = (VMStateField[]) {
-        VMSTATE_INT32(tsr_retry, SerialState),
+        VMSTATE_UINT32(tsr_retry, SerialState),
         VMSTATE_UINT8(thr, SerialState),
         VMSTATE_UINT8(tsr, SerialState),
         VMSTATE_END_OF_LIST()
@@ -662,10 +729,11 @@
 
 }
 
-const VMStateDescription vmstate_serial_recv_fifo = {
+static const VMStateDescription vmstate_serial_recv_fifo = {
     .name = "serial/recv_fifo",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = serial_recv_fifo_needed,
     .fields = (VMStateField[]) {
         VMSTATE_STRUCT(recv_fifo, SerialState, 1, vmstate_fifo8, Fifo8),
         VMSTATE_END_OF_LIST()
@@ -678,10 +746,11 @@
     return !fifo8_is_empty(&s->xmit_fifo);
 }
 
-const VMStateDescription vmstate_serial_xmit_fifo = {
+static const VMStateDescription vmstate_serial_xmit_fifo = {
     .name = "serial/xmit_fifo",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = serial_xmit_fifo_needed,
     .fields = (VMStateField[]) {
         VMSTATE_STRUCT(xmit_fifo, SerialState, 1, vmstate_fifo8, Fifo8),
         VMSTATE_END_OF_LIST()
@@ -694,12 +763,13 @@
     return timer_pending(s->fifo_timeout_timer);
 }
 
-const VMStateDescription vmstate_serial_fifo_timeout_timer = {
+static const VMStateDescription vmstate_serial_fifo_timeout_timer = {
     .name = "serial/fifo_timeout_timer",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = serial_fifo_timeout_timer_needed,
     .fields = (VMStateField[]) {
-        VMSTATE_TIMER(fifo_timeout_timer, SerialState),
+        VMSTATE_TIMER_PTR(fifo_timeout_timer, SerialState),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -710,10 +780,11 @@
     return s->timeout_ipending != 0;
 }
 
-const VMStateDescription vmstate_serial_timeout_ipending = {
+static const VMStateDescription vmstate_serial_timeout_ipending = {
     .name = "serial/timeout_ipending",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = serial_timeout_ipending_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(timeout_ipending, SerialState),
         VMSTATE_END_OF_LIST()
@@ -726,13 +797,14 @@
     return s->poll_msl >= 0;
 }
 
-const VMStateDescription vmstate_serial_poll = {
+static const VMStateDescription vmstate_serial_poll = {
     .name = "serial/poll",
     .version_id = 1,
+    .needed = serial_poll_needed,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(poll_msl, SerialState),
-        VMSTATE_TIMER(modem_status_poll, SerialState),
+        VMSTATE_TIMER_PTR(modem_status_poll, SerialState),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -757,31 +829,15 @@
         VMSTATE_UINT8_V(fcr_vmstate, SerialState, 3),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_serial_thr_ipending,
-            .needed = &serial_thr_ipending_needed,
-        } , {
-            .vmsd = &vmstate_serial_tsr,
-            .needed = &serial_tsr_needed,
-        } , {
-            .vmsd = &vmstate_serial_recv_fifo,
-            .needed = &serial_recv_fifo_needed,
-        } , {
-            .vmsd = &vmstate_serial_xmit_fifo,
-            .needed = &serial_xmit_fifo_needed,
-        } , {
-            .vmsd = &vmstate_serial_fifo_timeout_timer,
-            .needed = &serial_fifo_timeout_timer_needed,
-        } , {
-            .vmsd = &vmstate_serial_timeout_ipending,
-            .needed = &serial_timeout_ipending_needed,
-        } , {
-            .vmsd = &vmstate_serial_poll,
-            .needed = &serial_poll_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_serial_thr_ipending,
+        &vmstate_serial_tsr,
+        &vmstate_serial_recv_fifo,
+        &vmstate_serial_xmit_fifo,
+        &vmstate_serial_fifo_timeout_timer,
+        &vmstate_serial_timeout_ipending,
+        &vmstate_serial_poll,
+        NULL
     }
 };
 
@@ -789,6 +845,11 @@
 {
     SerialState *s = opaque;
 
+    if (s->watch_tag > 0) {
+        g_source_remove(s->watch_tag);
+        s->watch_tag = 0;
+    }
+
     s->rbr = 0;
     s->ier = 0;
     s->iir = UART_IIR_NO_INT;
@@ -800,7 +861,7 @@
     s->mcr = UART_MCR_OUT2;
     s->scr = 0;
     s->tsr_retry = 0;
-    s->char_transmit_time = (get_ticks_per_sec() / 9600) * 10;
+    s->char_transmit_time = (NANOSECONDS_PER_SECOND / 9600) * 10;
     s->poll_msl = 0;
 
     s->timeout_ipending = 0;
@@ -866,19 +927,13 @@
                          CharDriverState *chr, MemoryRegion *system_io)
 {
     SerialState *s;
-    Error *err = NULL;
 
     s = g_malloc0(sizeof(SerialState));
 
     s->irq = irq;
     s->baudbase = baudbase;
     s->chr = chr;
-    serial_realize_core(s, &err);
-    if (err != NULL) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
-        exit(1);
-    }
+    serial_realize_core(s, &error_fatal);
 
     vmstate_register(NULL, base, &vmstate_serial, s);
 
@@ -928,7 +983,6 @@
                             CharDriverState *chr, enum device_endian end)
 {
     SerialState *s;
-    Error *err = NULL;
 
     s = g_malloc0(sizeof(SerialState));
 
@@ -937,12 +991,7 @@
     s->baudbase = baudbase;
     s->chr = chr;
 
-    serial_realize_core(s, &err);
-    if (err != NULL) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
-        exit(1);
-    }
+    serial_realize_core(s, &error_fatal);
     vmstate_register(NULL, base, &vmstate_serial, s);
 
     memory_region_init_io(&s->io, NULL, &serial_mm_ops[end], s,
diff --git a/hw/char/sh_serial.c b/hw/char/sh_serial.c
index 9328dd1..4c55dcb 100644
--- a/hw/char/sh_serial.c
+++ b/hw/char/sh_serial.c
@@ -24,6 +24,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sysemu/char.h"
diff --git a/hw/char/spapr_vty.c b/hw/char/spapr_vty.c
index 0adf096..3498d7b 100644
--- a/hw/char/spapr_vty.c
+++ b/hw/char/spapr_vty.c
@@ -1,3 +1,7 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/qdev.h"
 #include "sysemu/char.h"
 #include "hw/ppc/spapr.h"
@@ -60,23 +64,21 @@
     qemu_chr_fe_write(dev->chardev, buf, len);
 }
 
-static int spapr_vty_init(VIOsPAPRDevice *sdev)
+static void spapr_vty_realize(VIOsPAPRDevice *sdev, Error **errp)
 {
     VIOsPAPRVTYDevice *dev = VIO_SPAPR_VTY_DEVICE(sdev);
 
     if (!dev->chardev) {
-        fprintf(stderr, "spapr-vty: Can't create vty without a chardev!\n");
-        exit(1);
+        error_setg(errp, "chardev property not set");
+        return;
     }
 
     qemu_chr_add_handlers(dev->chardev, vty_can_receive,
                           vty_receive, NULL, dev);
-
-    return 0;
 }
 
 /* Forward declaration */
-static target_ulong h_put_term_char(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_put_term_char(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -103,7 +105,7 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_get_term_char(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_get_term_char(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -163,7 +165,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VIOsPAPRDeviceClass *k = VIO_SPAPR_DEVICE_CLASS(klass);
 
-    k->init = spapr_vty_init;
+    k->realize = spapr_vty_realize;
     k->dt_name = "vty";
     k->dt_type = "serial";
     k->dt_compatible = "hvterm1";
@@ -195,7 +197,7 @@
         DeviceState *iter = kid->child;
 
         /* Only look at VTY devices */
-        if (!object_dynamic_cast(OBJECT(iter), "spapr-vty")) {
+        if (!object_dynamic_cast(OBJECT(iter), TYPE_VIO_SPAPR_VTY_DEVICE)) {
             continue;
         }
 
@@ -216,7 +218,7 @@
     return selected;
 }
 
-VIOsPAPRDevice *vty_lookup(sPAPREnvironment *spapr, target_ulong reg)
+VIOsPAPRDevice *vty_lookup(sPAPRMachineState *spapr, target_ulong reg)
 {
     VIOsPAPRDevice *sdev;
 
@@ -230,6 +232,10 @@
         return spapr_vty_get_default(spapr->vio_bus);
     }
 
+    if (!object_dynamic_cast(OBJECT(sdev), TYPE_VIO_SPAPR_VTY_DEVICE)) {
+        return NULL;
+    }
+
     return sdev;
 }
 
diff --git a/hw/char/stm32f2xx_usart.c b/hw/char/stm32f2xx_usart.c
new file mode 100644
index 0000000..15657ab
--- /dev/null
+++ b/hw/char/stm32f2xx_usart.c
@@ -0,0 +1,241 @@
+/*
+ * STM32F2XX USART
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/char/stm32f2xx_usart.h"
+#include "qemu/log.h"
+
+#ifndef STM_USART_ERR_DEBUG
+#define STM_USART_ERR_DEBUG 0
+#endif
+
+#define DB_PRINT_L(lvl, fmt, args...) do { \
+    if (STM_USART_ERR_DEBUG >= lvl) { \
+        qemu_log("%s: " fmt, __func__, ## args); \
+    } \
+} while (0);
+
+#define DB_PRINT(fmt, args...) DB_PRINT_L(1, fmt, ## args)
+
+static int stm32f2xx_usart_can_receive(void *opaque)
+{
+    STM32F2XXUsartState *s = opaque;
+
+    if (!(s->usart_sr & USART_SR_RXNE)) {
+        return 1;
+    }
+
+    return 0;
+}
+
+static void stm32f2xx_usart_receive(void *opaque, const uint8_t *buf, int size)
+{
+    STM32F2XXUsartState *s = opaque;
+
+    s->usart_dr = *buf;
+
+    if (!(s->usart_cr1 & USART_CR1_UE && s->usart_cr1 & USART_CR1_RE)) {
+        /* USART not enabled - drop the chars */
+        DB_PRINT("Dropping the chars\n");
+        return;
+    }
+
+    s->usart_sr |= USART_SR_RXNE;
+
+    if (s->usart_cr1 & USART_CR1_RXNEIE) {
+        qemu_set_irq(s->irq, 1);
+    }
+
+    DB_PRINT("Receiving: %c\n", s->usart_dr);
+}
+
+static void stm32f2xx_usart_reset(DeviceState *dev)
+{
+    STM32F2XXUsartState *s = STM32F2XX_USART(dev);
+
+    s->usart_sr = USART_SR_RESET;
+    s->usart_dr = 0x00000000;
+    s->usart_brr = 0x00000000;
+    s->usart_cr1 = 0x00000000;
+    s->usart_cr2 = 0x00000000;
+    s->usart_cr3 = 0x00000000;
+    s->usart_gtpr = 0x00000000;
+
+    qemu_set_irq(s->irq, 0);
+}
+
+static uint64_t stm32f2xx_usart_read(void *opaque, hwaddr addr,
+                                       unsigned int size)
+{
+    STM32F2XXUsartState *s = opaque;
+    uint64_t retvalue;
+
+    DB_PRINT("Read 0x%"HWADDR_PRIx"\n", addr);
+
+    switch (addr) {
+    case USART_SR:
+        retvalue = s->usart_sr;
+        s->usart_sr &= ~USART_SR_TC;
+        if (s->chr) {
+            qemu_chr_accept_input(s->chr);
+        }
+        return retvalue;
+    case USART_DR:
+        DB_PRINT("Value: 0x%" PRIx32 ", %c\n", s->usart_dr, (char) s->usart_dr);
+        s->usart_sr |= USART_SR_TXE;
+        s->usart_sr &= ~USART_SR_RXNE;
+        if (s->chr) {
+            qemu_chr_accept_input(s->chr);
+        }
+        qemu_set_irq(s->irq, 0);
+        return s->usart_dr & 0x3FF;
+    case USART_BRR:
+        return s->usart_brr;
+    case USART_CR1:
+        return s->usart_cr1;
+    case USART_CR2:
+        return s->usart_cr2;
+    case USART_CR3:
+        return s->usart_cr3;
+    case USART_GTPR:
+        return s->usart_gtpr;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%"HWADDR_PRIx"\n", __func__, addr);
+        return 0;
+    }
+
+    return 0;
+}
+
+static void stm32f2xx_usart_write(void *opaque, hwaddr addr,
+                                  uint64_t val64, unsigned int size)
+{
+    STM32F2XXUsartState *s = opaque;
+    uint32_t value = val64;
+    unsigned char ch;
+
+    DB_PRINT("Write 0x%" PRIx32 ", 0x%"HWADDR_PRIx"\n", value, addr);
+
+    switch (addr) {
+    case USART_SR:
+        if (value <= 0x3FF) {
+            s->usart_sr = value;
+        } else {
+            s->usart_sr &= value;
+        }
+        if (!(s->usart_sr & USART_SR_RXNE)) {
+            qemu_set_irq(s->irq, 0);
+        }
+        return;
+    case USART_DR:
+        if (value < 0xF000) {
+            ch = value;
+            if (s->chr) {
+                qemu_chr_fe_write_all(s->chr, &ch, 1);
+            }
+            s->usart_sr |= USART_SR_TC;
+            s->usart_sr &= ~USART_SR_TXE;
+        }
+        return;
+    case USART_BRR:
+        s->usart_brr = value;
+        return;
+    case USART_CR1:
+        s->usart_cr1 = value;
+            if (s->usart_cr1 & USART_CR1_RXNEIE &&
+                s->usart_sr & USART_SR_RXNE) {
+                qemu_set_irq(s->irq, 1);
+            }
+        return;
+    case USART_CR2:
+        s->usart_cr2 = value;
+        return;
+    case USART_CR3:
+        s->usart_cr3 = value;
+        return;
+    case USART_GTPR:
+        s->usart_gtpr = value;
+        return;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%"HWADDR_PRIx"\n", __func__, addr);
+    }
+}
+
+static const MemoryRegionOps stm32f2xx_usart_ops = {
+    .read = stm32f2xx_usart_read,
+    .write = stm32f2xx_usart_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static Property stm32f2xx_usart_properties[] = {
+    DEFINE_PROP_CHR("chardev", STM32F2XXUsartState, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void stm32f2xx_usart_init(Object *obj)
+{
+    STM32F2XXUsartState *s = STM32F2XX_USART(obj);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(obj), &s->irq);
+
+    memory_region_init_io(&s->mmio, obj, &stm32f2xx_usart_ops, s,
+                          TYPE_STM32F2XX_USART, 0x2000);
+    sysbus_init_mmio(SYS_BUS_DEVICE(obj), &s->mmio);
+}
+
+static void stm32f2xx_usart_realize(DeviceState *dev, Error **errp)
+{
+    STM32F2XXUsartState *s = STM32F2XX_USART(dev);
+
+    if (s->chr) {
+        qemu_chr_add_handlers(s->chr, stm32f2xx_usart_can_receive,
+                              stm32f2xx_usart_receive, NULL, s);
+    }
+}
+
+static void stm32f2xx_usart_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = stm32f2xx_usart_reset;
+    dc->props = stm32f2xx_usart_properties;
+    dc->realize = stm32f2xx_usart_realize;
+}
+
+static const TypeInfo stm32f2xx_usart_info = {
+    .name          = TYPE_STM32F2XX_USART,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(STM32F2XXUsartState),
+    .instance_init = stm32f2xx_usart_init,
+    .class_init    = stm32f2xx_usart_class_init,
+};
+
+static void stm32f2xx_usart_register_types(void)
+{
+    type_register_static(&stm32f2xx_usart_info);
+}
+
+type_init(stm32f2xx_usart_register_types)
diff --git a/hw/char/trace-events b/hw/char/trace-events
new file mode 100644
index 0000000..d53577c
--- /dev/null
+++ b/hw/char/trace-events
@@ -0,0 +1,49 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/char/virtio-serial-bus.c
+virtio_serial_send_control_event(unsigned int port, uint16_t event, uint16_t value) "port %u, event %u, value %u"
+virtio_serial_throttle_port(unsigned int port, bool throttle) "port %u, throttle %d"
+virtio_serial_handle_control_message(uint16_t event, uint16_t value) "event %u, value %u"
+virtio_serial_handle_control_message_port(unsigned int port) "port %u"
+
+# hw/char/virtio-console.c
+virtio_console_flush_buf(unsigned int port, size_t len, ssize_t ret) "port %u, in_len %zu, out_len %zd"
+virtio_console_chr_read(unsigned int port, int size) "port %u, size %d"
+virtio_console_chr_event(unsigned int port, int event) "port %u, event %d"
+
+# hw/char/grlib_apbuart.c
+grlib_apbuart_event(int event) "event:%d"
+grlib_apbuart_writel_unknown(uint64_t addr, uint32_t value) "addr 0x%"PRIx64" value 0x%x"
+grlib_apbuart_readl_unknown(uint64_t addr) "addr 0x%"PRIx64
+
+# hw/char/lm32_juart.c
+lm32_juart_get_jtx(uint32_t value) "jtx 0x%08x"
+lm32_juart_set_jtx(uint32_t value) "jtx 0x%08x"
+lm32_juart_get_jrx(uint32_t value) "jrx 0x%08x"
+lm32_juart_set_jrx(uint32_t value) "jrx 0x%08x"
+
+# hw/char/lm32_uart.c
+lm32_uart_memory_write(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
+lm32_uart_memory_read(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
+lm32_uart_irq_state(int level) "irq state %d"
+
+# hw/char/milkymist-uart.c
+milkymist_uart_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_uart_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_uart_raise_irq(void) "Raise IRQ"
+milkymist_uart_lower_irq(void) "Lower IRQ"
+
+# hw/char/escc.c
+escc_put_queue(char channel, int b) "channel %c put: 0x%02x"
+escc_get_queue(char channel, int val) "channel %c get 0x%02x"
+escc_update_irq(int irq) "IRQ = %d"
+escc_update_parameters(char channel, int speed, int parity, int data_bits, int stop_bits) "channel %c: speed=%d parity=%c data=%d stop=%d"
+escc_mem_writeb_ctrl(char channel, uint32_t reg, uint32_t val) "Write channel %c, reg[%d] = %2.2x"
+escc_mem_writeb_data(char channel, uint32_t val) "Write channel %c, ch %d"
+escc_mem_readb_ctrl(char channel, uint32_t reg, uint8_t val) "Read channel %c, reg[%d] = %2.2x"
+escc_mem_readb_data(char channel, uint32_t ret) "Read channel %c, ch %d"
+escc_serial_receive_byte(char channel, int ch) "channel %c put ch %d"
+escc_sunkbd_event_in(int ch, const char *name, int down) "QKeyCode 0x%2.2x [%s], down %d"
+escc_sunkbd_event_out(int ch) "Translated keycode 0x%2.2x"
+escc_kbd_command(int val) "Command %d"
+escc_sunmouse_event(int dx, int dy, int buttons_state) "dx=%d dy=%d buttons=%01x"
diff --git a/hw/char/virtio-console.c b/hw/char/virtio-console.c
index 752ed2c..4f0e03d 100644
--- a/hw/char/virtio-console.c
+++ b/hw/char/virtio-console.c
@@ -10,6 +10,7 @@
  * the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
 #include "qemu/error-report.h"
 #include "trace.h"
@@ -84,8 +85,9 @@
 {
     VirtConsole *vcon = VIRTIO_CONSOLE(port);
     DeviceState *dev = DEVICE(port);
+    VirtIOSerialPortClass *k = VIRTIO_SERIAL_PORT_GET_CLASS(port);
 
-    if (vcon->chr) {
+    if (vcon->chr && !k->is_console) {
         qemu_chr_fe_set_open(vcon->chr, guest_connected);
     }
 
@@ -95,6 +97,15 @@
     }
 }
 
+static void guest_writable(VirtIOSerialPort *port)
+{
+    VirtConsole *vcon = VIRTIO_CONSOLE(port);
+
+    if (vcon->chr) {
+        qemu_chr_accept_input(vcon->chr);
+    }
+}
+
 /* Readiness of the guest to accept data on a port */
 static int chr_can_read(void *opaque)
 {
@@ -146,9 +157,25 @@
     }
 
     if (vcon->chr) {
-        vcon->chr->explicit_fe_open = 1;
-        qemu_chr_add_handlers(vcon->chr, chr_can_read, chr_read, chr_event,
-                              vcon);
+        /*
+         * For consoles we don't block guest data transfer just
+         * because nothing is connected - we'll just let it go
+         * whetherever the chardev wants - /dev/null probably.
+         *
+         * For serial ports we need 100% reliable data transfer
+         * so we use the opened/closed signals from chardev to
+         * trigger open/close of the device
+         */
+        if (k->is_console) {
+            vcon->chr->explicit_fe_open = 0;
+            qemu_chr_add_handlers(vcon->chr, chr_can_read, chr_read,
+                                  NULL, vcon);
+            virtio_serial_open(port);
+        } else {
+            vcon->chr->explicit_fe_open = 1;
+            qemu_chr_add_handlers(vcon->chr, chr_can_read, chr_read,
+                                  chr_event, vcon);
+        }
     }
 }
 
@@ -188,6 +215,7 @@
     k->unrealize = virtconsole_unrealize;
     k->have_data = flush_buf;
     k->set_guest_connected = set_guest_connected;
+    k->guest_writable = guest_writable;
     dc->props = virtserialport_properties;
 }
 
diff --git a/hw/char/virtio-pingpong.c b/hw/char/virtio-pingpong.c
deleted file mode 100644
index 31674c4..0000000
--- a/hw/char/virtio-pingpong.c
+++ /dev/null
@@ -1,114 +0,0 @@
-/* virtio-pingpong: virtio-serial backend that just sends all
- * data it receives back to the guest.
- *
- * Copyright (c) 2013 Linaro Limited
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2 or later, as published by the Free Software Foundation.
- */
-
-#include "sysemu/char.h"
-#include "qemu/error-report.h"
-#include "trace.h"
-#include "hw/virtio/virtio-serial.h"
-
-#define TYPE_VIRTIO_PINGPONG "virtpingpong"
-#define VIRTIO_PINGPONG(obj) \
-    OBJECT_CHECK(VirtPingpong, (obj), TYPE_VIRTIO_PINGPONG)
-
-typedef struct VirtPingpong {
-    VirtIOSerialPort parent_obj;
-    /* Any per-backend data goes here */
-} VirtPingpong;
-
-
-/* Callback function that's called when the guest sends us data */
-static ssize_t virtpingpong_have_data(VirtIOSerialPort *port,
-                                      const uint8_t *buf, ssize_t len)
-{
-    //VirtPingpong *vpp = VIRTIO_PINGPONG(port);
-    ssize_t ret;
-
-    printf("virtpingpong_have_data: %zd bytes\n", len);
-
-    ret = virtio_serial_write(port, buf, len);
-
-    printf("virtpingpong_have_data: passed %zd bytes back\n", ret);
-
-    if (ret < len) {
-        /* We didn't manage to send all the data: throttle the
-         * port so we don't get any more for now.
-         */
-        virtio_serial_throttle_port(port, true);
-    }
-
-    return ret;
-}
-
-static void virtpingpong_guest_writable(VirtIOSerialPort *port)
-{
-    printf("virtpingpong_guest_writable\n");
-    /* Just unthrottle the other end of the port so it can send us more data */
-    virtio_serial_throttle_port(port, false);
-}
-
-/* Callback function that's called when the guest opens/closes the port */
-static void virtpingpong_set_guest_connected(VirtIOSerialPort *port,
-                                             int guest_connected)
-{
-    //VirtPingpong *vpp = VIRTIO_PINGPONG(port);
-
-    printf("virtpingpong: guest %sconnected\n", guest_connected ? "" : "dis");
-}
-
-static void virtpingpong_realize(DeviceState *dev, Error **errp)
-{
-    VirtIOSerialPort *port = VIRTIO_SERIAL_PORT(dev);
-    //VirtPingpong *vpp = VIRTIO_PINGPONG(dev);
-    VirtIOSerialPortClass *k = VIRTIO_SERIAL_PORT_GET_CLASS(dev);
-
-    if (port->id == 0 && !k->is_console) {
-        error_setg(errp, "Port number 0 on virtio-serial devices reserved "
-                   "for virtconsole devices for backward compatibility.");
-        return;
-    }
-
-    /* The host end of this connection is always open. */
-    virtio_serial_open(port);
-}
-
-static void virtpingpong_unrealize(DeviceState *dev, Error **errp)
-{
-    // VirtPingpong *vpp = VIRTIO_PINGPONG(dev);
-
-    /* Nothing for us to do here */
-}
-
-static void virtpingpong_class_init(ObjectClass *klass, void *data)
-{
-    VirtIOSerialPortClass *k = VIRTIO_SERIAL_PORT_CLASS(klass);
-
-    k->realize = virtpingpong_realize;
-    k->unrealize = virtpingpong_unrealize;
-    k->have_data = virtpingpong_have_data;
-    k->guest_writable = virtpingpong_guest_writable;
-    k->set_guest_connected = virtpingpong_set_guest_connected;
-    /* If we have any device state for save/load we should
-     * register a vmstate here.
-     */
-}
-
-static const TypeInfo virtpingpong_info = {
-    .name          = TYPE_VIRTIO_PINGPONG,
-    .parent        = TYPE_VIRTIO_SERIAL_PORT,
-    .instance_size = sizeof(VirtPingpong),
-    .class_init    = virtpingpong_class_init,
-};
-
-static void virtpingpong_register_types(void)
-{
-    type_register_static(&virtpingpong_info);
-}
-
-type_init(virtpingpong_register_types)
diff --git a/hw/char/virtio-serial-bus.c b/hw/char/virtio-serial-bus.c
index 1070302..db57a38 100644
--- a/hw/char/virtio-serial-bus.c
+++ b/hw/char/virtio-serial-bus.c
@@ -18,15 +18,18 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
 #include "monitor/monitor.h"
+#include "qemu/error-report.h"
 #include "qemu/queue.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 #include "hw/virtio/virtio-serial.h"
 #include "hw/virtio/virtio-access.h"
 
-struct VirtIOSerialDevices {
+static struct VirtIOSerialDevices {
     QLIST_HEAD(, VirtIOSerial) devices;
 } vserdevices;
 
@@ -64,7 +67,7 @@
         VirtIOSerialPort *port;
 
         QTAILQ_FOREACH(port, &vser->ports, next) {
-            if (!strcmp(port->name, name)) {
+            if (port->name && !strcmp(port->name, name)) {
                 return port;
             }
         }
@@ -75,13 +78,13 @@
 static bool use_multiport(VirtIOSerial *vser)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(vser);
-    return vdev->guest_features & (1 << VIRTIO_CONSOLE_F_MULTIPORT);
+    return virtio_vdev_has_feature(vdev, VIRTIO_CONSOLE_F_MULTIPORT);
 }
 
 static size_t write_to_port(VirtIOSerialPort *port,
                             const uint8_t *buf, size_t size)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     VirtQueue *vq;
     size_t offset;
 
@@ -94,15 +97,17 @@
     while (offset < size) {
         size_t len;
 
-        if (!virtqueue_pop(vq, &elem)) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
             break;
         }
 
-        len = iov_from_buf(elem.in_sg, elem.in_num, 0,
+        len = iov_from_buf(elem->in_sg, elem->in_num, 0,
                            buf + offset, size - offset);
         offset += len;
 
-        virtqueue_push(vq, &elem, len);
+        virtqueue_push(vq, elem, len);
+        g_free(elem);
     }
 
     virtio_notify(VIRTIO_DEVICE(port->vser), vq);
@@ -111,13 +116,18 @@
 
 static void discard_vq_data(VirtQueue *vq, VirtIODevice *vdev)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
 
     if (!virtio_queue_ready(vq)) {
         return;
     }
-    while (virtqueue_pop(vq, &elem)) {
-        virtqueue_push(vq, &elem, 0);
+    for (;;) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+        virtqueue_push(vq, elem, 0);
+        g_free(elem);
     }
     virtio_notify(vdev, vq);
 }
@@ -136,21 +146,22 @@
         unsigned int i;
 
         /* Pop an elem only if we haven't left off a previous one mid-way */
-        if (!port->elem.out_num) {
-            if (!virtqueue_pop(vq, &port->elem)) {
+        if (!port->elem) {
+            port->elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+            if (!port->elem) {
                 break;
             }
             port->iov_idx = 0;
             port->iov_offset = 0;
         }
 
-        for (i = port->iov_idx; i < port->elem.out_num; i++) {
+        for (i = port->iov_idx; i < port->elem->out_num; i++) {
             size_t buf_size;
             ssize_t ret;
 
-            buf_size = port->elem.out_sg[i].iov_len - port->iov_offset;
+            buf_size = port->elem->out_sg[i].iov_len - port->iov_offset;
             ret = vsc->have_data(port,
-                                  port->elem.out_sg[i].iov_base
+                                  port->elem->out_sg[i].iov_base
                                   + port->iov_offset,
                                   buf_size);
             if (port->throttled) {
@@ -165,8 +176,9 @@
         if (port->throttled) {
             break;
         }
-        virtqueue_push(vq, &port->elem, 0);
-        port->elem.out_num = 0;
+        virtqueue_push(vq, port->elem, 0);
+        g_free(port->elem);
+        port->elem = NULL;
     }
     virtio_notify(vdev, vq);
 }
@@ -183,21 +195,26 @@
 
 static size_t send_control_msg(VirtIOSerial *vser, void *buf, size_t len)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     VirtQueue *vq;
 
     vq = vser->c_ivq;
     if (!virtio_queue_ready(vq)) {
         return 0;
     }
-    if (!virtqueue_pop(vq, &elem)) {
+
+    elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+    if (!elem) {
         return 0;
     }
 
-    memcpy(elem.in_sg[0].iov_base, buf, len);
+    /* TODO: detect a buffer that's too short, set NEEDS_RESET */
+    iov_from_buf(elem->in_sg, elem->in_num, 0, buf, len);
 
-    virtqueue_push(vq, &elem, len);
+    virtqueue_push(vq, elem, len);
     virtio_notify(VIRTIO_DEVICE(vser), vq);
+    g_free(elem);
+
     return len;
 }
 
@@ -411,7 +428,7 @@
 
 static void control_out(VirtIODevice *vdev, VirtQueue *vq)
 {
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     VirtIOSerial *vser;
     uint8_t *buf;
     size_t len;
@@ -420,10 +437,15 @@
 
     len = 0;
     buf = NULL;
-    while (virtqueue_pop(vq, &elem)) {
+    for (;;) {
         size_t cur_len;
 
-        cur_len = iov_size(elem.out_sg, elem.out_num);
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        cur_len = iov_size(elem->out_sg, elem->out_num);
         /*
          * Allocate a new buf only if we didn't have one previously or
          * if the size of the buf differs
@@ -434,10 +456,11 @@
             buf = g_malloc(cur_len);
             len = cur_len;
         }
-        iov_to_buf(elem.out_sg, elem.out_num, 0, buf, cur_len);
+        iov_to_buf(elem->out_sg, elem->out_num, 0, buf, cur_len);
 
         handle_control_message(vser, buf, cur_len);
-        virtqueue_push(vq, &elem, 0);
+        virtqueue_push(vq, elem, 0);
+        g_free(elem);
     }
     g_free(buf);
     virtio_notify(vdev, vq);
@@ -470,7 +493,11 @@
      * writable again -- i.e. if a vq had stuff queued up and the
      * guest wasn't reading at all, the host would not be able to
      * write to the vq anymore.  Once the guest reads off something,
-     * we can start queueing things up again.
+     * we can start queueing things up again.  However, this call is
+     * made for each buffer addition by the guest -- even though free
+     * buffers existed prior to the current buffer addition.  This is
+     * done so as not to maintain previous state, which will need
+     * additional live-migration-related changes.
      */
     VirtIOSerial *vser;
     VirtIOSerialPort *port;
@@ -488,20 +515,21 @@
      * If guest_connected is false, this call is being made by the
      * early-boot queueing up of descriptors, which is just noise for
      * the host apps -- don't disturb them in that case.
-    */
+     */
     if (port->guest_connected && port->host_connected && vsc->guest_writable) {
         vsc->guest_writable(port);
     }
 }
 
-static uint32_t get_features(VirtIODevice *vdev, uint32_t features)
+static uint64_t get_features(VirtIODevice *vdev, uint64_t features,
+                             Error **errp)
 {
     VirtIOSerial *vser;
 
     vser = VIRTIO_SERIAL(vdev);
 
     if (vser->bus.max_nr_ports > 1) {
-        features |= (1 << VIRTIO_CONSOLE_F_MULTIPORT);
+        virtio_add_feature(&features, VIRTIO_CONSOLE_F_MULTIPORT);
     }
     return features;
 }
@@ -509,10 +537,14 @@
 /* Guest requested config info */
 static void get_config(VirtIODevice *vdev, uint8_t *config_data)
 {
-    VirtIOSerial *vser;
+    VirtIOSerial *vser = VIRTIO_SERIAL(vdev);
+    struct virtio_console_config *config =
+        (struct virtio_console_config *)config_data;
 
-    vser = VIRTIO_SERIAL(vdev);
-    memcpy(config_data, &vser->config, sizeof(struct virtio_console_config));
+    config->cols = 0;
+    config->rows = 0;
+    config->max_nr_ports = virtio_tswap32(vdev,
+                                          vser->serial.max_virtserial_ports);
 }
 
 static void guest_reset(VirtIOSerial *vser)
@@ -560,16 +592,6 @@
 
     vser = VIRTIO_SERIAL(vdev);
     guest_reset(vser);
-
-    /* In case we have switched endianness */
-    vser->config.max_nr_ports =
-        virtio_tswap32(vdev, vser->serial.max_virtserial_ports);
-}
-
-static void virtio_serial_save(QEMUFile *f, void *opaque)
-{
-    /* The virtio device */
-    virtio_save(VIRTIO_DEVICE(opaque), f);
 }
 
 static void virtio_serial_save_device(VirtIODevice *vdev, QEMUFile *f)
@@ -578,15 +600,16 @@
     VirtIOSerialPort *port;
     uint32_t nr_active_ports;
     unsigned int i, max_nr_ports;
+    struct virtio_console_config config;
 
-    /* The config space */
-    qemu_put_be16s(f, &s->config.cols);
-    qemu_put_be16s(f, &s->config.rows);
-
-    qemu_put_be32s(f, &s->config.max_nr_ports);
+    /* The config space (ignored on the far end in current versions) */
+    get_config(vdev, (uint8_t *)&config);
+    qemu_put_be16s(f, &config.cols);
+    qemu_put_be16s(f, &config.rows);
+    qemu_put_be32s(f, &config.max_nr_ports);
 
     /* The ports map */
-    max_nr_ports = virtio_tswap32(vdev, s->config.max_nr_ports);
+    max_nr_ports = s->serial.max_virtserial_ports;
     for (i = 0; i < (max_nr_ports + 31) / 32; i++) {
         qemu_put_be32s(f, &s->ports_map[i]);
     }
@@ -611,16 +634,14 @@
         qemu_put_byte(f, port->host_connected);
 
 	elem_popped = 0;
-        if (port->elem.out_num) {
+        if (port->elem) {
             elem_popped = 1;
         }
         qemu_put_be32s(f, &elem_popped);
         if (elem_popped) {
             qemu_put_be32s(f, &port->iov_idx);
             qemu_put_be64s(f, &port->iov_offset);
-
-            qemu_put_buffer(f, (unsigned char *)&port->elem,
-                            sizeof(port->elem));
+            qemu_put_virtqueue_element(f, port->elem);
         }
     }
 }
@@ -658,7 +679,7 @@
     s->post_load = NULL;
 }
 
-static int fetch_active_ports_list(QEMUFile *f, int version_id,
+static int fetch_active_ports_list(QEMUFile *f,
                                    VirtIOSerial *s, uint32_t nr_active_ports)
 {
     uint32_t i;
@@ -675,6 +696,7 @@
     /* Items in struct VirtIOSerialPort */
     for (i = 0; i < nr_active_ports; i++) {
         VirtIOSerialPort *port;
+        uint32_t elem_popped;
         uint32_t id;
 
         id = qemu_get_be32(f);
@@ -687,41 +709,29 @@
         s->post_load->connected[i].port = port;
         s->post_load->connected[i].host_connected = qemu_get_byte(f);
 
-        if (version_id > 2) {
-            uint32_t elem_popped;
+        qemu_get_be32s(f, &elem_popped);
+        if (elem_popped) {
+            qemu_get_be32s(f, &port->iov_idx);
+            qemu_get_be64s(f, &port->iov_offset);
 
-            qemu_get_be32s(f, &elem_popped);
-            if (elem_popped) {
-                qemu_get_be32s(f, &port->iov_idx);
-                qemu_get_be64s(f, &port->iov_offset);
+            port->elem =
+                qemu_get_virtqueue_element(f, sizeof(VirtQueueElement));
 
-                qemu_get_buffer(f, (unsigned char *)&port->elem,
-                                sizeof(port->elem));
-                virtqueue_map_sg(port->elem.in_sg, port->elem.in_addr,
-                                 port->elem.in_num, 1);
-                virtqueue_map_sg(port->elem.out_sg, port->elem.out_addr,
-                                 port->elem.out_num, 1);
-
-                /*
-                 *  Port was throttled on source machine.  Let's
-                 *  unthrottle it here so data starts flowing again.
-                 */
-                virtio_serial_throttle_port(port, false);
-            }
+            /*
+             *  Port was throttled on source machine.  Let's
+             *  unthrottle it here so data starts flowing again.
+             */
+            virtio_serial_throttle_port(port, false);
         }
     }
     timer_mod(s->post_load->timer, 1);
     return 0;
 }
 
-static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
+static int virtio_serial_load(QEMUFile *f, void *opaque, size_t size)
 {
-    if (version_id > 3) {
-        return -EINVAL;
-    }
-
     /* The virtio device */
-    return virtio_load(VIRTIO_DEVICE(opaque), f, version_id);
+    return virtio_load(VIRTIO_DEVICE(opaque), f, 3);
 }
 
 static int virtio_serial_load_device(VirtIODevice *vdev, QEMUFile *f,
@@ -733,22 +743,12 @@
     int ret;
     uint32_t tmp;
 
-    if (version_id < 2) {
-        return 0;
-    }
-
     /* Unused */
     qemu_get_be16s(f, (uint16_t *) &tmp);
     qemu_get_be16s(f, (uint16_t *) &tmp);
     qemu_get_be32s(f, &tmp);
 
-    /* Note: this is the only location where we use tswap32() instead of
-     * virtio_tswap32() because:
-     * - virtio_tswap32() only makes sense when the device is fully restored
-     * - the target endianness that was used to populate s->config is
-     *   necessarly the default one
-     */
-    max_nr_ports = tswap32(s->config.max_nr_ports);
+    max_nr_ports = s->serial.max_virtserial_ports;
     for (i = 0; i < (max_nr_ports + 31) / 32; i++) {
         qemu_get_be32s(f, &ports_map);
 
@@ -764,7 +764,7 @@
     qemu_get_be32s(f, &nr_active_ports);
 
     if (nr_active_ports) {
-        ret = fetch_active_ports_list(f, version_id, s, nr_active_ports);
+        ret = fetch_active_ports_list(f, s, nr_active_ports);
         if (ret) {
             return ret;
         }
@@ -799,7 +799,7 @@
 
 static void virtser_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent)
 {
-    VirtIOSerialPort *port = DO_UPCAST(VirtIOSerialPort, dev, qdev);
+    VirtIOSerialPort *port = VIRTIO_SERIAL_PORT(qdev);
 
     monitor_printf(mon, "%*sport %d, guest %s, host %s, throttle %s\n",
                    indent, "", port->id,
@@ -811,17 +811,16 @@
 /* This function is only used if a port id is not provided by the user */
 static uint32_t find_free_port_id(VirtIOSerial *vser)
 {
-    VirtIODevice *vdev = VIRTIO_DEVICE(vser);
     unsigned int i, max_nr_ports;
 
-    max_nr_ports = virtio_tswap32(vdev, vser->config.max_nr_ports);
+    max_nr_ports = vser->serial.max_virtserial_ports;
     for (i = 0; i < (max_nr_ports + 31) / 32; i++) {
-        uint32_t map, bit;
+        uint32_t map, zeroes;
 
         map = vser->ports_map[i];
-        bit = ffs(~map);
-        if (bit) {
-            return (bit - 1) + i * 32;
+        zeroes = ctz32(~map);
+        if (zeroes != 32) {
+            return zeroes + i * 32;
         }
     }
     return VIRTIO_CONSOLE_BAD_ID;
@@ -875,7 +874,6 @@
     VirtIOSerialPort *port = VIRTIO_SERIAL_PORT(dev);
     VirtIOSerialPortClass *vsc = VIRTIO_SERIAL_PORT_GET_CLASS(port);
     VirtIOSerialBus *bus = VIRTIO_SERIAL_BUS(qdev_get_parent_bus(dev));
-    VirtIODevice *vdev = VIRTIO_DEVICE(bus->vser);
     int max_nr_ports;
     bool plugging_port0;
     Error *err = NULL;
@@ -917,7 +915,7 @@
         }
     }
 
-    max_nr_ports = virtio_tswap32(vdev, port->vser->config.max_nr_ports);
+    max_nr_ports = port->vser->serial.max_virtserial_ports;
     if (port->id >= max_nr_ports) {
         error_setg(errp, "virtio-serial-bus: Out-of-range port id specified, "
                          "max. allowed: %u", max_nr_ports - 1);
@@ -930,7 +928,7 @@
         return;
     }
 
-    port->elem.out_num = 0;
+    port->elem = NULL;
 }
 
 static void virtser_port_device_plug(HotplugHandler *hotplug_dev,
@@ -976,15 +974,17 @@
     }
 
     /* Each port takes 2 queues, and one pair is for the control queue */
-    max_supported_ports = VIRTIO_PCI_QUEUE_MAX / 2 - 1;
+    max_supported_ports = VIRTIO_QUEUE_MAX / 2 - 1;
 
     if (vser->serial.max_virtserial_ports > max_supported_ports) {
         error_setg(errp, "maximum ports supported: %u", max_supported_ports);
         return;
     }
 
+    /* We don't support emergency write, skip it for now. */
+    /* TODO: cleaner fix, depending on host features. */
     virtio_init(vdev, "virtio-serial", VIRTIO_ID_CONSOLE,
-                sizeof(struct virtio_console_config));
+                offsetof(struct virtio_console_config, emerg_wr));
 
     /* Spawn a new virtio-serial bus on which the ports will ride as devices */
     qbus_create_inplace(&vser->bus, sizeof(vser->bus), TYPE_VIRTIO_SERIAL_BUS,
@@ -1022,8 +1022,6 @@
         vser->ovqs[i] = virtio_add_queue(vdev, 128, handle_output);
     }
 
-    vser->config.max_nr_ports =
-        virtio_tswap32(vdev, vser->serial.max_virtserial_ports);
     vser->ports_map = g_malloc0(((vser->serial.max_virtserial_ports + 31) / 32)
         * sizeof(vser->ports_map[0]));
     /*
@@ -1034,13 +1032,6 @@
 
     vser->post_load = NULL;
 
-    /*
-     * Register for the savevm section with the virtio-console name
-     * to preserve backward compat
-     */
-    register_savevm(dev, "virtio-console", -1, 3, virtio_serial_save,
-                    virtio_serial_load, vser);
-
     QLIST_INSERT_HEAD(&vserdevices.devices, vser, next);
 }
 
@@ -1071,8 +1062,6 @@
 
     QLIST_REMOVE(vser, next);
 
-    unregister_savevm(dev, "virtio-console", vser);
-
     g_free(vser->ivqs);
     g_free(vser->ovqs);
     g_free(vser->ports_map);
@@ -1085,8 +1074,12 @@
     virtio_cleanup(vdev);
 }
 
+/* Note: 'console' is used for backwards compatibility */
+VMSTATE_VIRTIO_DEVICE(console, 3, virtio_serial_load, virtio_vmstate_save);
+
 static Property virtio_serial_properties[] = {
-    DEFINE_VIRTIO_SERIAL_PROPERTIES(VirtIOSerial, serial),
+    DEFINE_PROP_UINT32("max_ports", VirtIOSerial, serial.max_virtserial_ports,
+                                                  31),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1099,6 +1092,7 @@
     QLIST_INIT(&vserdevices.devices);
 
     dc->props = virtio_serial_properties;
+    dc->vmsd = &vmstate_virtio_console;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
     vdc->realize = virtio_serial_device_realize;
     vdc->unrealize = virtio_serial_device_unrealize;
diff --git a/hw/char/xen_console.c b/hw/char/xen_console.c
index eb7f450..83108b0 100644
--- a/hw/char/xen_console.c
+++ b/hw/char/xen_console.c
@@ -19,15 +19,9 @@
  *  with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdlib.h>
-#include <errno.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include <sys/select.h>
-#include <fcntl.h>
-#include <unistd.h>
 #include <termios.h>
-#include <stdarg.h>
-#include <sys/mman.h>
 
 #include "hw/hw.h"
 #include "sysemu/char.h"
@@ -228,12 +222,12 @@
 	con->buffer.max_capacity = limit;
 
     if (!xendev->dev) {
-        con->sring = xc_map_foreign_range(xen_xc, con->xendev.dom,
-                                          XC_PAGE_SIZE,
+        xen_pfn_t mfn = con->ring_ref;
+        con->sring = xenforeignmemory_map(xen_fmem, con->xendev.dom,
                                           PROT_READ|PROT_WRITE,
-                                          con->ring_ref);
+                                          1, &mfn, NULL);
     } else {
-        con->sring = xc_gnttab_map_grant_ref(xendev->gnttabdev, con->xendev.dom,
+        con->sring = xengnttab_map_grant_ref(xendev->gnttabdev, con->xendev.dom,
                                              con->ring_ref,
                                              PROT_READ|PROT_WRITE);
     }
@@ -265,9 +259,6 @@
 {
     struct XenConsole *con = container_of(xendev, struct XenConsole, xendev);
 
-    if (!xendev->dev) {
-        return;
-    }
     if (con->chr) {
         qemu_chr_add_handlers(con->chr, NULL, NULL, NULL, NULL);
         qemu_chr_fe_release(con->chr);
@@ -275,12 +266,12 @@
     xen_be_unbind_evtchn(&con->xendev);
 
     if (con->sring) {
-        if (!xendev->gnttabdev) {
-            munmap(con->sring, XC_PAGE_SIZE);
+        if (!xendev->dev) {
+            xenforeignmemory_unmap(xen_fmem, con->sring, 1);
         } else {
-            xc_gnttab_munmap(xendev->gnttabdev, con->sring, 1);
+            xengnttab_unmap(xendev->gnttabdev, con->sring, 1);
         }
-	con->sring = NULL;
+        con->sring = NULL;
     }
 }
 
diff --git a/hw/char/xilinx_uartlite.c b/hw/char/xilinx_uartlite.c
index f7c3cae..4847efb 100644
--- a/hw/char/xilinx_uartlite.c
+++ b/hw/char/xilinx_uartlite.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 
@@ -171,6 +172,11 @@
     }
 };
 
+static Property xilinx_uartlite_properties[] = {
+    DEFINE_PROP_CHR("chardev", XilinxUARTLite, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void uart_rx(void *opaque, const uint8_t *buf, int size)
 {
     XilinxUARTLite *s = opaque;
@@ -205,7 +211,6 @@
 {
     XilinxUARTLite *s = XILINX_UARTLITE(dev);
 
-    s->chr = qemu_char_get_next_serial();
     if (s->chr)
         qemu_chr_add_handlers(s->chr, uart_can_rx, uart_rx, uart_event, s);
 }
@@ -227,6 +232,7 @@
 
     dc->reset = xilinx_uartlite_reset;
     dc->realize = xilinx_uartlite_realize;
+    dc->props = xilinx_uartlite_properties;
 }
 
 static const TypeInfo xilinx_uartlite_info = {
diff --git a/hw/core/Makefile.objs b/hw/core/Makefile.objs
index 9dce1bc..cfd4840 100644
--- a/hw/core/Makefile.objs
+++ b/hw/core/Makefile.objs
@@ -1,10 +1,11 @@
 # core qdev-related obj files, also used by *-user:
 common-obj-y += qdev.o qdev-properties.o
+common-obj-y += bus.o
 common-obj-y += fw-path-provider.o
 # irq.o needed for qdev GPIO handling:
 common-obj-y += irq.o
 common-obj-y += hotplug.o
-common-obj-y += nmi.o
+obj-y += nmi.o
 
 common-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
 common-obj-$(CONFIG_XILINX_AXI) += stream.o
@@ -14,4 +15,5 @@
 common-obj-$(CONFIG_SOFTMMU) += null-machine.o
 common-obj-$(CONFIG_SOFTMMU) += loader.o
 common-obj-$(CONFIG_SOFTMMU) += qdev-properties-system.o
-common-obj-$(CONFIG_SOFTMMU) += platform-bus.o
+common-obj-$(CONFIG_SOFTMMU) += register.o
+common-obj-$(CONFIG_PLATFORM_BUS) += platform-bus.o
diff --git a/hw/core/bus.c b/hw/core/bus.c
new file mode 100644
index 0000000..21cee55
--- /dev/null
+++ b/hw/core/bus.c
@@ -0,0 +1,254 @@
+/*
+ *  Dynamic device configuration and creation -- buses.
+ *
+ *  Copyright (c) 2009 CodeSourcery
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "hw/qdev.h"
+#include "qapi/error.h"
+
+static void qbus_set_hotplug_handler_internal(BusState *bus, Object *handler,
+                                              Error **errp)
+{
+
+    object_property_set_link(OBJECT(bus), OBJECT(handler),
+                             QDEV_HOTPLUG_HANDLER_PROPERTY, errp);
+}
+
+void qbus_set_hotplug_handler(BusState *bus, DeviceState *handler, Error **errp)
+{
+    qbus_set_hotplug_handler_internal(bus, OBJECT(handler), errp);
+}
+
+void qbus_set_bus_hotplug_handler(BusState *bus, Error **errp)
+{
+    qbus_set_hotplug_handler_internal(bus, OBJECT(bus), errp);
+}
+
+int qbus_walk_children(BusState *bus,
+                       qdev_walkerfn *pre_devfn, qbus_walkerfn *pre_busfn,
+                       qdev_walkerfn *post_devfn, qbus_walkerfn *post_busfn,
+                       void *opaque)
+{
+    BusChild *kid;
+    int err;
+
+    if (pre_busfn) {
+        err = pre_busfn(bus, opaque);
+        if (err) {
+            return err;
+        }
+    }
+
+    QTAILQ_FOREACH(kid, &bus->children, sibling) {
+        err = qdev_walk_children(kid->child,
+                                 pre_devfn, pre_busfn,
+                                 post_devfn, post_busfn, opaque);
+        if (err < 0) {
+            return err;
+        }
+    }
+
+    if (post_busfn) {
+        err = post_busfn(bus, opaque);
+        if (err) {
+            return err;
+        }
+    }
+
+    return 0;
+}
+
+static void qbus_realize(BusState *bus, DeviceState *parent, const char *name)
+{
+    const char *typename = object_get_typename(OBJECT(bus));
+    BusClass *bc;
+    char *buf;
+    int i, len, bus_id;
+
+    bus->parent = parent;
+
+    if (name) {
+        bus->name = g_strdup(name);
+    } else if (bus->parent && bus->parent->id) {
+        /* parent device has id -> use it plus parent-bus-id for bus name */
+        bus_id = bus->parent->num_child_bus;
+
+        len = strlen(bus->parent->id) + 16;
+        buf = g_malloc(len);
+        snprintf(buf, len, "%s.%d", bus->parent->id, bus_id);
+        bus->name = buf;
+    } else {
+        /* no id -> use lowercase bus type plus global bus-id for bus name */
+        bc = BUS_GET_CLASS(bus);
+        bus_id = bc->automatic_ids++;
+
+        len = strlen(typename) + 16;
+        buf = g_malloc(len);
+        len = snprintf(buf, len, "%s.%d", typename, bus_id);
+        for (i = 0; i < len; i++) {
+            buf[i] = qemu_tolower(buf[i]);
+        }
+        bus->name = buf;
+    }
+
+    if (bus->parent) {
+        QLIST_INSERT_HEAD(&bus->parent->child_bus, bus, sibling);
+        bus->parent->num_child_bus++;
+        object_property_add_child(OBJECT(bus->parent), bus->name, OBJECT(bus), NULL);
+        object_unref(OBJECT(bus));
+    } else if (bus != sysbus_get_default()) {
+        /* TODO: once all bus devices are qdevified,
+           only reset handler for main_system_bus should be registered here. */
+        qemu_register_reset(qbus_reset_all_fn, bus);
+    }
+}
+
+static void bus_unparent(Object *obj)
+{
+    BusState *bus = BUS(obj);
+    BusChild *kid;
+
+    while ((kid = QTAILQ_FIRST(&bus->children)) != NULL) {
+        DeviceState *dev = kid->child;
+        object_unparent(OBJECT(dev));
+    }
+    if (bus->parent) {
+        QLIST_REMOVE(bus, sibling);
+        bus->parent->num_child_bus--;
+        bus->parent = NULL;
+    } else {
+        assert(bus != sysbus_get_default()); /* main_system_bus is never freed */
+        qemu_unregister_reset(qbus_reset_all_fn, bus);
+    }
+}
+
+void qbus_create_inplace(void *bus, size_t size, const char *typename,
+                         DeviceState *parent, const char *name)
+{
+    object_initialize(bus, size, typename);
+    qbus_realize(bus, parent, name);
+}
+
+BusState *qbus_create(const char *typename, DeviceState *parent, const char *name)
+{
+    BusState *bus;
+
+    bus = BUS(object_new(typename));
+    if (!bus) {
+        hw_error("Can't create bus.");
+    }
+    qbus_realize(bus, parent, name);
+
+    return bus;
+}
+
+static bool bus_get_realized(Object *obj, Error **errp)
+{
+    BusState *bus = BUS(obj);
+
+    return bus->realized;
+}
+
+static void bus_set_realized(Object *obj, bool value, Error **errp)
+{
+    BusState *bus = BUS(obj);
+    BusClass *bc = BUS_GET_CLASS(bus);
+    BusChild *kid;
+    Error *local_err = NULL;
+
+    if (value && !bus->realized) {
+        if (bc->realize) {
+            bc->realize(bus, &local_err);
+        }
+
+        /* TODO: recursive realization */
+    } else if (!value && bus->realized) {
+        QTAILQ_FOREACH(kid, &bus->children, sibling) {
+            DeviceState *dev = kid->child;
+            object_property_set_bool(OBJECT(dev), false, "realized",
+                                     &local_err);
+            if (local_err != NULL) {
+                break;
+            }
+        }
+        if (bc->unrealize && local_err == NULL) {
+            bc->unrealize(bus, &local_err);
+        }
+    }
+
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    bus->realized = value;
+}
+
+static void qbus_initfn(Object *obj)
+{
+    BusState *bus = BUS(obj);
+
+    QTAILQ_INIT(&bus->children);
+    object_property_add_link(obj, QDEV_HOTPLUG_HANDLER_PROPERTY,
+                             TYPE_HOTPLUG_HANDLER,
+                             (Object **)&bus->hotplug_handler,
+                             object_property_allow_set_link,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             NULL);
+    object_property_add_bool(obj, "realized",
+                             bus_get_realized, bus_set_realized, NULL);
+}
+
+static char *default_bus_get_fw_dev_path(DeviceState *dev)
+{
+    return g_strdup(object_get_typename(OBJECT(dev)));
+}
+
+static void bus_class_init(ObjectClass *class, void *data)
+{
+    BusClass *bc = BUS_CLASS(class);
+
+    class->unparent = bus_unparent;
+    bc->get_fw_dev_path = default_bus_get_fw_dev_path;
+}
+
+static void qbus_finalize(Object *obj)
+{
+    BusState *bus = BUS(obj);
+
+    g_free((char *)bus->name);
+}
+
+static const TypeInfo bus_info = {
+    .name = TYPE_BUS,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(BusState),
+    .abstract = true,
+    .class_size = sizeof(BusClass),
+    .instance_init = qbus_initfn,
+    .instance_finalize = qbus_finalize,
+    .class_init = bus_class_init,
+};
+
+static void bus_register_types(void)
+{
+    type_register_static(&bus_info);
+}
+
+type_init(bus_register_types)
diff --git a/hw/core/empty_slot.c b/hw/core/empty_slot.c
index 612b109..c1b9c2b 100644
--- a/hw/core/empty_slot.c
+++ b/hw/core/empty_slot.c
@@ -9,6 +9,7 @@
  * version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/empty_slot.h"
diff --git a/hw/core/fw-path-provider.c b/hw/core/fw-path-provider.c
index 1290c3e..33b9983 100644
--- a/hw/core/fw-path-provider.c
+++ b/hw/core/fw-path-provider.c
@@ -1,5 +1,5 @@
 /*
- *  Firmware patch provider class and helpers.
+ *  Firmware path provider class and helpers.
  *
  *  This program is free software; you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
@@ -15,6 +15,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/fw-path-provider.h"
 
 char *fw_path_provider_get_dev_path(FWPathProvider *p, BusState *bus,
diff --git a/hw/core/hotplug.c b/hw/core/hotplug.c
index 4e01074..17ac986 100644
--- a/hw/core/hotplug.c
+++ b/hw/core/hotplug.c
@@ -9,9 +9,21 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hotplug.h"
 #include "qemu/module.h"
 
+void hotplug_handler_pre_plug(HotplugHandler *plug_handler,
+                              DeviceState *plugged_dev,
+                              Error **errp)
+{
+    HotplugHandlerClass *hdc = HOTPLUG_HANDLER_GET_CLASS(plug_handler);
+
+    if (hdc->pre_plug) {
+        hdc->pre_plug(plug_handler, plugged_dev, errp);
+    }
+}
+
 void hotplug_handler_plug(HotplugHandler *plug_handler,
                           DeviceState *plugged_dev,
                           Error **errp)
diff --git a/hw/core/irq.c b/hw/core/irq.c
index 8a62a36..49ff2e6 100644
--- a/hw/core/irq.c
+++ b/hw/core/irq.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/irq.h"
 #include "qom/object.h"
diff --git a/hw/core/loader.c b/hw/core/loader.c
index 7527fd3..53e0e41 100644
--- a/hw/core/loader.c
+++ b/hw/core/loader.c
@@ -42,6 +42,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "disas/disas.h"
 #include "monitor/monitor.h"
@@ -51,12 +53,11 @@
 #include "hw/nvram/fw_cfg.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
+#include "hw/boards.h"
+#include "qemu/cutils.h"
 
 #include <zlib.h>
 
-bool option_rom_has_mr = false;
-bool rom_file_has_mr = true;
-
 static int roms_loaded;
 
 /* return the size or -1 if error */
@@ -148,6 +149,28 @@
     return size;
 }
 
+int load_image_mr(const char *filename, MemoryRegion *mr)
+{
+    int size;
+
+    if (!memory_access_is_direct(mr, false)) {
+        /* Can only load an image into RAM or ROM */
+        return -1;
+    }
+
+    size = get_image_size(filename);
+
+    if (size > memory_region_size(mr)) {
+        return -1;
+    }
+    if (size > 0) {
+        if (rom_add_file_mr(filename, mr, -1) < 0) {
+            return -1;
+        }
+    }
+    return size;
+}
+
 void pstrcpy_targphys(const char *name, hwaddr dest, int buf_size,
                       const char *source)
 {
@@ -267,7 +290,7 @@
 
 /* ELF loader */
 
-static void *load_at(int fd, int offset, int size)
+static void *load_at(int fd, off_t offset, size_t size)
 {
     void *ptr;
     if (lseek(fd, offset, SEEK_SET) < 0)
@@ -297,6 +320,7 @@
 #undef elf_phdr
 #undef elf_shdr
 #undef elf_sym
+#undef elf_rela
 #undef elf_note
 #undef elf_word
 #undef elf_sword
@@ -307,6 +331,7 @@
 #define elf_note	elf64_note
 #define elf_shdr	elf64_shdr
 #define elf_sym		elf64_sym
+#define elf_rela        elf64_rela
 #define elf_word        uint64_t
 #define elf_sword        int64_t
 #define bswapSZs	bswap64s
@@ -331,10 +356,66 @@
     }
 }
 
+void load_elf_hdr(const char *filename, void *hdr, bool *is64, Error **errp)
+{
+    int fd;
+    uint8_t e_ident_local[EI_NIDENT];
+    uint8_t *e_ident;
+    size_t hdr_size, off;
+    bool is64l;
+
+    if (!hdr) {
+        hdr = e_ident_local;
+    }
+    e_ident = hdr;
+
+    fd = open(filename, O_RDONLY | O_BINARY);
+    if (fd < 0) {
+        error_setg_errno(errp, errno, "Failed to open file: %s", filename);
+        return;
+    }
+    if (read(fd, hdr, EI_NIDENT) != EI_NIDENT) {
+        error_setg_errno(errp, errno, "Failed to read file: %s", filename);
+        goto fail;
+    }
+    if (e_ident[0] != ELFMAG0 ||
+        e_ident[1] != ELFMAG1 ||
+        e_ident[2] != ELFMAG2 ||
+        e_ident[3] != ELFMAG3) {
+        error_setg(errp, "Bad ELF magic");
+        goto fail;
+    }
+
+    is64l = e_ident[EI_CLASS] == ELFCLASS64;
+    hdr_size = is64l ? sizeof(Elf64_Ehdr) : sizeof(Elf32_Ehdr);
+    if (is64) {
+        *is64 = is64l;
+    }
+
+    off = EI_NIDENT;
+    while (hdr != e_ident_local && off < hdr_size) {
+        size_t br = read(fd, hdr + off, hdr_size - off);
+        switch (br) {
+        case 0:
+            error_setg(errp, "File too short: %s", filename);
+            goto fail;
+        case -1:
+            error_setg_errno(errp, errno, "Failed to read file: %s",
+                             filename);
+            goto fail;
+        }
+        off += br;
+    }
+
+fail:
+    close(fd);
+}
+
 /* return < 0 if error, otherwise the number of bytes loaded in memory */
 int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
              void *translate_opaque, uint64_t *pentry, uint64_t *lowaddr,
-             uint64_t *highaddr, int big_endian, int elf_machine, int clear_lsb)
+             uint64_t *highaddr, int big_endian, int elf_machine,
+             int clear_lsb, int data_swab)
 {
     int fd, data_order, target_data_order, must_swab, ret = ELF_LOAD_FAILED;
     uint8_t e_ident[EI_NIDENT];
@@ -373,10 +454,12 @@
     lseek(fd, 0, SEEK_SET);
     if (e_ident[EI_CLASS] == ELFCLASS64) {
         ret = load_elf64(filename, fd, translate_fn, translate_opaque, must_swab,
-                         pentry, lowaddr, highaddr, elf_machine, clear_lsb);
+                         pentry, lowaddr, highaddr, elf_machine, clear_lsb,
+                         data_swab);
     } else {
         ret = load_elf32(filename, fd, translate_fn, translate_opaque, must_swab,
-                         pentry, lowaddr, highaddr, elf_machine, clear_lsb);
+                         pentry, lowaddr, highaddr, elf_machine, clear_lsb,
+                         data_swab);
     }
 
  fail:
@@ -592,8 +675,7 @@
     ret = hdr->ih_size;
 
 out:
-    if (data)
-        g_free(data);
+    g_free(data);
     close(fd);
     return ret;
 }
@@ -614,14 +696,9 @@
                             NULL, NULL);
 }
 
-/* This simply prevents g_malloc in the function below from allocating
- * a huge amount of memory, by placing a limit on the maximum
- * uncompressed image size that load_image_gzipped will read.
- */
-#define LOAD_IMAGE_MAX_GUNZIP_BYTES (256 << 20)
-
-/* Load a gzip-compressed kernel. */
-int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz)
+/* Load a gzip-compressed kernel to a dynamically allocated buffer. */
+int load_image_gzipped_buffer(const char *filename, uint64_t max_sz,
+                              uint8_t **buffer)
 {
     uint8_t *compressed_data = NULL;
     uint8_t *data = NULL;
@@ -653,8 +730,11 @@
         goto out;
     }
 
-    rom_add_blob_fixed(filename, data, bytes, addr);
+    /* trim to actual size and return to caller */
+    *buffer = g_realloc(data, bytes);
     ret = bytes;
+    /* ownership has been transferred to caller */
+    data = NULL;
 
  out:
     g_free(compressed_data);
@@ -662,6 +742,20 @@
     return ret;
 }
 
+/* Load a gzip-compressed kernel. */
+int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz)
+{
+    int bytes;
+    uint8_t *data;
+
+    bytes = load_image_gzipped_buffer(filename, max_sz, &data);
+    if (bytes != -1) {
+        rom_add_blob_fixed(filename, data, bytes, addr);
+        g_free(data);
+    }
+    return bytes;
+}
+
 /*
  * Functions for reboot-persistent memory regions.
  *  - used for vga bios and option roms.
@@ -712,12 +806,22 @@
     QTAILQ_INSERT_TAIL(&roms, rom, next);
 }
 
+static void fw_cfg_resized(const char *id, uint64_t length, void *host)
+{
+    if (fw_cfg) {
+        fw_cfg_modify_file(fw_cfg, id + strlen("/rom@"), host, length);
+    }
+}
+
 static void *rom_set_mr(Rom *rom, Object *owner, const char *name)
 {
     void *data;
 
     rom->mr = g_malloc(sizeof(*rom->mr));
-    memory_region_init_ram(rom->mr, owner, name, rom->datasize, &error_abort);
+    memory_region_init_resizeable_ram(rom->mr, owner, name,
+                                      rom->datasize, rom->romsize,
+                                      fw_cfg_resized,
+                                      &error_fatal);
     memory_region_set_readonly(rom->mr, true);
     vmstate_register_ram_global(rom->mr);
 
@@ -729,8 +833,9 @@
 
 int rom_add_file(const char *file, const char *fw_dir,
                  hwaddr addr, int32_t bootindex,
-                 bool option_rom)
+                 bool option_rom, MemoryRegion *mr)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
     Rom *rom;
     int rc, fd = -1;
     char devpath[100];
@@ -787,7 +892,7 @@
                  basename);
         snprintf(devpath, sizeof(devpath), "/rom@%s", fw_file_name);
 
-        if ((!option_rom || option_rom_has_mr) && rom_file_has_mr) {
+        if ((!option_rom || mc->option_rom_has_mr) && mc->rom_file_has_mr) {
             data = rom_set_mr(rom, OBJECT(fw_cfg), devpath);
         } else {
             data = rom->data;
@@ -795,7 +900,12 @@
 
         fw_cfg_add_file(fw_cfg, fw_file_name, data, rom->romsize);
     } else {
-        snprintf(devpath, sizeof(devpath), "/rom@" TARGET_FMT_plx, addr);
+        if (mr) {
+            rom->mr = mr;
+            snprintf(devpath, sizeof(devpath), "/rom@%s", file);
+        } else {
+            snprintf(devpath, sizeof(devpath), "/rom@" TARGET_FMT_plx, addr);
+        }
     }
 
     add_boot_device_path(bootindex, NULL, devpath);
@@ -804,24 +914,31 @@
 err:
     if (fd != -1)
         close(fd);
+
     g_free(rom->data);
     g_free(rom->path);
     g_free(rom->name);
+    if (fw_dir) {
+        g_free(rom->fw_dir);
+        g_free(rom->fw_file);
+    }
     g_free(rom);
+
     return -1;
 }
 
-ram_addr_t rom_add_blob(const char *name, const void *blob, size_t len,
-                   hwaddr addr, const char *fw_file_name,
+MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len,
+                   size_t max_len, hwaddr addr, const char *fw_file_name,
                    FWCfgReadCallback fw_callback, void *callback_opaque)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
     Rom *rom;
-    ram_addr_t ret = RAM_ADDR_MAX;
+    MemoryRegion *mr = NULL;
 
     rom           = g_malloc0(sizeof(*rom));
     rom->name     = g_strdup(name);
     rom->addr     = addr;
-    rom->romsize  = len;
+    rom->romsize  = max_len ? max_len : len;
     rom->datasize = len;
     rom->data     = g_malloc0(rom->datasize);
     memcpy(rom->data, blob, len);
@@ -832,18 +949,18 @@
 
         snprintf(devpath, sizeof(devpath), "/rom@%s", fw_file_name);
 
-        if (rom_file_has_mr) {
+        if (mc->rom_file_has_mr) {
             data = rom_set_mr(rom, OBJECT(fw_cfg), devpath);
-            ret = memory_region_get_ram_addr(rom->mr);
+            mr = rom->mr;
         } else {
             data = rom->data;
         }
 
         fw_cfg_add_file_callback(fw_cfg, fw_file_name,
                                  fw_callback, callback_opaque,
-                                 data, rom->romsize);
+                                 data, rom->datasize);
     }
-    return ret;
+    return mr;
 }
 
 /* This function is specific for elf program because we don't need to allocate
@@ -868,12 +985,12 @@
 
 int rom_add_vga(const char *file)
 {
-    return rom_add_file(file, "vgaroms", 0, -1, true);
+    return rom_add_file(file, "vgaroms", 0, -1, true, NULL);
 }
 
 int rom_add_option(const char *file, int32_t bootindex)
 {
-    return rom_add_file(file, "genroms", 0, bootindex, true);
+    return rom_add_file(file, "genroms", 0, bootindex, true, NULL);
 }
 
 static void rom_reset(void *unused)
@@ -909,7 +1026,7 @@
     }
 }
 
-int rom_load_all(void)
+int rom_check_and_register_reset(void)
 {
     hwaddr addr = 0;
     MemoryRegionSection section;
@@ -933,12 +1050,8 @@
         memory_region_unref(section.mr);
     }
     qemu_register_reset(rom_reset, NULL);
-    return 0;
-}
-
-void rom_load_done(void)
-{
     roms_loaded = 1;
+    return 0;
 }
 
 void rom_set_fw(FWCfgState *f)
@@ -946,6 +1059,20 @@
     fw_cfg = f;
 }
 
+void rom_set_order_override(int order)
+{
+    if (!fw_cfg)
+        return;
+    fw_cfg_set_order_override(fw_cfg, order);
+}
+
+void rom_reset_order_override(void)
+{
+    if (!fw_cfg)
+        return;
+    fw_cfg_reset_order_override(fw_cfg);
+}
+
 static Rom *find_rom(hwaddr addr)
 {
     Rom *rom;
@@ -1040,7 +1167,7 @@
     return rom->data + (addr - rom->addr);
 }
 
-void do_info_roms(Monitor *mon, const QDict *qdict)
+void hmp_info_roms(Monitor *mon, const QDict *qdict)
 {
     Rom *rom;
 
diff --git a/hw/core/machine.c b/hw/core/machine.c
index 19d3e3a..e5a456f 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -10,11 +10,15 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "hw/boards.h"
+#include "qapi/error.h"
+#include "qapi-visit.h"
 #include "qapi/visitor.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
+#include "qemu/cutils.h"
 
 static char *machine_get_accel(Object *obj, Error **errp)
 {
@@ -31,39 +35,63 @@
     ms->accel = g_strdup(value);
 }
 
-static bool machine_get_kernel_irqchip(Object *obj, Error **errp)
+static void machine_set_kernel_irqchip(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
 {
+    Error *err = NULL;
     MachineState *ms = MACHINE(obj);
+    OnOffSplit mode;
 
-    return ms->kernel_irqchip;
-}
-
-static void machine_set_kernel_irqchip(Object *obj, bool value, Error **errp)
-{
-    MachineState *ms = MACHINE(obj);
-
-    ms->kernel_irqchip = value;
+    visit_type_OnOffSplit(v, name, &mode, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    } else {
+        switch (mode) {
+        case ON_OFF_SPLIT_ON:
+            ms->kernel_irqchip_allowed = true;
+            ms->kernel_irqchip_required = true;
+            ms->kernel_irqchip_split = false;
+            break;
+        case ON_OFF_SPLIT_OFF:
+            ms->kernel_irqchip_allowed = false;
+            ms->kernel_irqchip_required = false;
+            ms->kernel_irqchip_split = false;
+            break;
+        case ON_OFF_SPLIT_SPLIT:
+            ms->kernel_irqchip_allowed = true;
+            ms->kernel_irqchip_required = true;
+            ms->kernel_irqchip_split = true;
+            break;
+        default:
+            /* The value was checked in visit_type_OnOffSplit() above. If
+             * we get here, then something is wrong in QEMU.
+             */
+            abort();
+        }
+    }
 }
 
 static void machine_get_kvm_shadow_mem(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
+                                       const char *name, void *opaque,
                                        Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     int64_t value = ms->kvm_shadow_mem;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void machine_set_kvm_shadow_mem(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
+                                       const char *name, void *opaque,
                                        Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -148,24 +176,24 @@
 }
 
 static void machine_get_phandle_start(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
-                                       Error **errp)
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     int64_t value = ms->phandle_start;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void machine_set_phandle_start(Object *obj, Visitor *v,
-                                       void *opaque, const char *name,
-                                       Error **errp)
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     MachineState *ms = MACHINE(obj);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -229,6 +257,35 @@
     MachineState *ms = MACHINE(obj);
 
     ms->usb = value;
+    ms->usb_disabled = !value;
+}
+
+static bool machine_get_graphics(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->enable_graphics;
+}
+
+static void machine_set_graphics(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->enable_graphics = value;
+}
+
+static bool machine_get_igd_gfx_passthru(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->igd_gfx_passthru;
+}
+
+static void machine_set_igd_gfx_passthru(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->igd_gfx_passthru = value;
 }
 
 static char *machine_get_firmware(Object *obj, Error **errp)
@@ -246,18 +303,33 @@
     ms->firmware = g_strdup(value);
 }
 
-static bool machine_get_iommu(Object *obj, Error **errp)
+static void machine_set_suppress_vmdesc(Object *obj, bool value, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
-    return ms->iommu;
+    ms->suppress_vmdesc = value;
 }
 
-static void machine_set_iommu(Object *obj, bool value, Error **errp)
+static bool machine_get_suppress_vmdesc(Object *obj, Error **errp)
 {
     MachineState *ms = MACHINE(obj);
 
-    ms->iommu = value;
+    return ms->suppress_vmdesc;
+}
+
+static void machine_set_enforce_config_section(Object *obj, bool value,
+                                             Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->enforce_config_section = value;
+}
+
+static bool machine_get_enforce_config_section(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->enforce_config_section;
 }
 
 static int error_on_sysbus_device(SysBusDevice *sbdev, void *opaque)
@@ -285,54 +357,143 @@
     foreach_dynamic_sysbus_device(error_on_sysbus_device, NULL);
 }
 
+static void machine_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    /* Default 128 MB as guest ram size */
+    mc->default_ram_size = 128 * M_BYTE;
+    mc->rom_file_has_mr = true;
+}
+
+static void machine_class_base_init(ObjectClass *oc, void *data)
+{
+    if (!object_class_is_abstract(oc)) {
+        MachineClass *mc = MACHINE_CLASS(oc);
+        const char *cname = object_class_get_name(oc);
+        assert(g_str_has_suffix(cname, TYPE_MACHINE_SUFFIX));
+        mc->name = g_strndup(cname,
+                            strlen(cname) - strlen(TYPE_MACHINE_SUFFIX));
+    }
+}
+
 static void machine_initfn(Object *obj)
 {
     MachineState *ms = MACHINE(obj);
 
+    ms->kernel_irqchip_allowed = true;
+    ms->kvm_shadow_mem = -1;
+    ms->dump_guest_core = true;
+    ms->mem_merge = true;
+    ms->enable_graphics = true;
+
     object_property_add_str(obj, "accel",
                             machine_get_accel, machine_set_accel, NULL);
-    object_property_add_bool(obj, "kernel-irqchip",
-                             machine_get_kernel_irqchip,
-                             machine_set_kernel_irqchip,
-                             NULL);
+    object_property_set_description(obj, "accel",
+                                    "Accelerator list",
+                                    NULL);
+    object_property_add(obj, "kernel-irqchip", "OnOffSplit",
+                        NULL,
+                        machine_set_kernel_irqchip,
+                        NULL, NULL, NULL);
+    object_property_set_description(obj, "kernel-irqchip",
+                                    "Configure KVM in-kernel irqchip",
+                                    NULL);
     object_property_add(obj, "kvm-shadow-mem", "int",
                         machine_get_kvm_shadow_mem,
                         machine_set_kvm_shadow_mem,
                         NULL, NULL, NULL);
+    object_property_set_description(obj, "kvm-shadow-mem",
+                                    "KVM shadow MMU size",
+                                    NULL);
     object_property_add_str(obj, "kernel",
                             machine_get_kernel, machine_set_kernel, NULL);
+    object_property_set_description(obj, "kernel",
+                                    "Linux kernel image file",
+                                    NULL);
     object_property_add_str(obj, "initrd",
                             machine_get_initrd, machine_set_initrd, NULL);
+    object_property_set_description(obj, "initrd",
+                                    "Linux initial ramdisk file",
+                                    NULL);
     object_property_add_str(obj, "append",
                             machine_get_append, machine_set_append, NULL);
+    object_property_set_description(obj, "append",
+                                    "Linux kernel command line",
+                                    NULL);
     object_property_add_str(obj, "dtb",
                             machine_get_dtb, machine_set_dtb, NULL);
+    object_property_set_description(obj, "dtb",
+                                    "Linux kernel device tree file",
+                                    NULL);
     object_property_add_str(obj, "dumpdtb",
                             machine_get_dumpdtb, machine_set_dumpdtb, NULL);
+    object_property_set_description(obj, "dumpdtb",
+                                    "Dump current dtb to a file and quit",
+                                    NULL);
     object_property_add(obj, "phandle-start", "int",
                         machine_get_phandle_start,
                         machine_set_phandle_start,
                         NULL, NULL, NULL);
+    object_property_set_description(obj, "phandle-start",
+                                    "The first phandle ID we may generate dynamically",
+                                    NULL);
     object_property_add_str(obj, "dt-compatible",
                             machine_get_dt_compatible,
                             machine_set_dt_compatible,
                             NULL);
+    object_property_set_description(obj, "dt-compatible",
+                                    "Overrides the \"compatible\" property of the dt root node",
+                                    NULL);
     object_property_add_bool(obj, "dump-guest-core",
                              machine_get_dump_guest_core,
                              machine_set_dump_guest_core,
                              NULL);
+    object_property_set_description(obj, "dump-guest-core",
+                                    "Include guest memory in  a core dump",
+                                    NULL);
     object_property_add_bool(obj, "mem-merge",
                              machine_get_mem_merge,
                              machine_set_mem_merge, NULL);
+    object_property_set_description(obj, "mem-merge",
+                                    "Enable/disable memory merge support",
+                                    NULL);
     object_property_add_bool(obj, "usb",
                              machine_get_usb,
                              machine_set_usb, NULL);
+    object_property_set_description(obj, "usb",
+                                    "Set on/off to enable/disable usb",
+                                    NULL);
+    object_property_add_bool(obj, "graphics",
+                             machine_get_graphics,
+                             machine_set_graphics, NULL);
+    object_property_set_description(obj, "graphics",
+                                    "Set on/off to enable/disable graphics emulation",
+                                    NULL);
+    object_property_add_bool(obj, "igd-passthru",
+                             machine_get_igd_gfx_passthru,
+                             machine_set_igd_gfx_passthru, NULL);
+    object_property_set_description(obj, "igd-passthru",
+                                    "Set on/off to enable/disable igd passthrou",
+                                    NULL);
     object_property_add_str(obj, "firmware",
                             machine_get_firmware,
                             machine_set_firmware, NULL);
-    object_property_add_bool(obj, "iommu",
-                             machine_get_iommu,
-                             machine_set_iommu, NULL);
+    object_property_set_description(obj, "firmware",
+                                    "Firmware image",
+                                    NULL);
+    object_property_add_bool(obj, "suppress-vmdesc",
+                             machine_get_suppress_vmdesc,
+                             machine_set_suppress_vmdesc, NULL);
+    object_property_set_description(obj, "suppress-vmdesc",
+                                    "Set on to disable self-describing migration",
+                                    NULL);
+    object_property_add_bool(obj, "enforce-config-section",
+                             machine_get_enforce_config_section,
+                             machine_set_enforce_config_section, NULL);
+    object_property_set_description(obj, "enforce-config-section",
+                                    "Set on to enforce configuration section migration",
+                                    NULL);
 
     /* Register notifier when init is done for sysbus sanity checks */
     ms->sysbus_notifier.notify = machine_init_notify;
@@ -353,11 +514,81 @@
     g_free(ms->firmware);
 }
 
+bool machine_usb(MachineState *machine)
+{
+    return machine->usb;
+}
+
+bool machine_kernel_irqchip_allowed(MachineState *machine)
+{
+    return machine->kernel_irqchip_allowed;
+}
+
+bool machine_kernel_irqchip_required(MachineState *machine)
+{
+    return machine->kernel_irqchip_required;
+}
+
+bool machine_kernel_irqchip_split(MachineState *machine)
+{
+    return machine->kernel_irqchip_split;
+}
+
+int machine_kvm_shadow_mem(MachineState *machine)
+{
+    return machine->kvm_shadow_mem;
+}
+
+int machine_phandle_start(MachineState *machine)
+{
+    return machine->phandle_start;
+}
+
+bool machine_dump_guest_core(MachineState *machine)
+{
+    return machine->dump_guest_core;
+}
+
+bool machine_mem_merge(MachineState *machine)
+{
+    return machine->mem_merge;
+}
+
+static void machine_class_finalize(ObjectClass *klass, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(klass);
+
+    if (mc->compat_props) {
+        g_array_free(mc->compat_props, true);
+    }
+}
+
+void machine_register_compat_props(MachineState *machine)
+{
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    int i;
+    GlobalProperty *p;
+
+    if (!mc->compat_props) {
+        return;
+    }
+
+    for (i = 0; i < mc->compat_props->len; i++) {
+        p = g_array_index(mc->compat_props, GlobalProperty *, i);
+        /* Machine compat_props must never cause errors: */
+        p->errp = &error_abort;
+        qdev_prop_register_global(p);
+    }
+}
+
 static const TypeInfo machine_info = {
     .name = TYPE_MACHINE,
     .parent = TYPE_OBJECT,
     .abstract = true,
     .class_size = sizeof(MachineClass),
+    .class_init    = machine_class_init,
+    .class_base_init = machine_class_base_init,
+    .class_finalize = machine_class_finalize,
     .instance_size = sizeof(MachineState),
     .instance_init = machine_initfn,
     .instance_finalize = machine_finalize,
diff --git a/hw/core/nmi.c b/hw/core/nmi.c
index 3dff020..bfd0896 100644
--- a/hw/core/nmi.c
+++ b/hw/core/nmi.c
@@ -19,12 +19,15 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/nmi.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
+#include "monitor/monitor.h"
 
 struct do_nmi_s {
     int cpu_index;
-    Error *errp;
+    Error *err;
     bool handled;
 };
 
@@ -39,8 +42,8 @@
         NMIClass *nc = NMI_GET_CLASS(n);
 
         ns->handled = true;
-        nc->nmi_monitor_handler(n, ns->cpu_index, &ns->errp);
-        if (ns->errp) {
+        nc->nmi_monitor_handler(n, ns->cpu_index, &ns->err);
+        if (ns->err) {
             return -1;
         }
     }
@@ -58,15 +61,15 @@
 {
     struct do_nmi_s ns = {
         .cpu_index = cpu_index,
-        .errp = NULL,
+        .err = NULL,
         .handled = false
     };
 
     nmi_children(object_get_root(), &ns);
     if (ns.handled) {
-        error_propagate(errp, ns.errp);
+        error_propagate(errp, ns.err);
     } else {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
     }
 }
 
diff --git a/hw/core/null-machine.c b/hw/core/null-machine.c
index 1ec7c3b..0351ba7 100644
--- a/hw/core/null-machine.c
+++ b/hw/core/null-machine.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
@@ -19,17 +20,11 @@
 {
 }
 
-static QEMUMachine machine_none = {
-    .name = "none",
-    .desc = "empty machine",
-    .init = machine_none_init,
-    .max_cpus = 0,
-};
-
-static void register_machines(void)
+static void machine_none_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&machine_none);
+    mc->desc = "empty machine";
+    mc->init = machine_none_init;
+    mc->max_cpus = 0;
 }
 
-machine_init(register_machines);
-
+DEFINE_MACHINE("none", machine_none_machine_init)
diff --git a/hw/core/platform-bus.c b/hw/core/platform-bus.c
index 0f052b3..36f84ab 100644
--- a/hw/core/platform-bus.c
+++ b/hw/core/platform-bus.c
@@ -19,9 +19,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/platform-bus.h"
-#include "monitor/monitor.h"
 #include "exec/address-spaces.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 
 
@@ -107,31 +108,29 @@
     pbus->done_gathering = true;
 }
 
-static int platform_bus_map_irq(PlatformBusDevice *pbus, SysBusDevice *sbdev,
-                                int n)
+static void platform_bus_map_irq(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                 int n)
 {
     int max_irqs = pbus->num_irqs;
     int irqn;
 
     if (sysbus_is_irq_connected(sbdev, n)) {
         /* IRQ is already mapped, nothing to do */
-        return 0;
+        return;
     }
 
     irqn = find_first_zero_bit(pbus->used_irqs, max_irqs);
     if (irqn >= max_irqs) {
-        hw_error("Platform Bus: Can not fit IRQ line");
-        return -1;
+        error_report("Platform Bus: Can not fit IRQ line");
+        exit(1);
     }
 
     set_bit(irqn, pbus->used_irqs);
     sysbus_connect_irq(sbdev, n, pbus->irqs[irqn]);
-
-    return 0;
 }
 
-static int platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
-                                 int n)
+static void platform_bus_map_mmio(PlatformBusDevice *pbus, SysBusDevice *sbdev,
+                                  int n)
 {
     MemoryRegion *sbdev_mr = sysbus_mmio_get_region(sbdev, n);
     uint64_t size = memory_region_size(sbdev_mr);
@@ -141,7 +140,7 @@
 
     if (memory_region_is_mapped(sbdev_mr)) {
         /* Region is already mapped, nothing to do */
-        return 0;
+        return;
     }
 
     /*
@@ -156,13 +155,13 @@
     }
 
     if (!found_region) {
-        hw_error("Platform Bus: Can not fit MMIO region of size %"PRIx64, size);
+        error_report("Platform Bus: Can not fit MMIO region of size %"PRIx64,
+                     size);
+        exit(1);
     }
 
     /* Map the device's region into our Platform Bus MMIO space */
     memory_region_add_subregion(&pbus->mmio, off, sbdev_mr);
-
-    return 0;
 }
 
 /*
diff --git a/hw/core/ptimer.c b/hw/core/ptimer.c
index 466e543..30829ee 100644
--- a/hw/core/ptimer.c
+++ b/hw/core/ptimer.c
@@ -5,10 +5,12 @@
  *
  * This code is licensed under the GNU LGPL.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
 #include "qemu/host-utils.h"
+#include "sysemu/replay.h"
 
 struct ptimer_state
 {
@@ -27,12 +29,15 @@
 static void ptimer_trigger(ptimer_state *s)
 {
     if (s->bh) {
-        qemu_bh_schedule(s->bh);
+        replay_bh_schedule_event(s->bh);
     }
 }
 
 static void ptimer_reload(ptimer_state *s)
 {
+    uint32_t period_frac = s->period_frac;
+    uint64_t period = s->period;
+
     if (s->delta == 0) {
         ptimer_trigger(s);
         s->delta = s->limit;
@@ -43,10 +48,24 @@
         return;
     }
 
+    /*
+     * Artificially limit timeout rate to something
+     * achievable under QEMU.  Otherwise, QEMU spends all
+     * its time generating timer interrupts, and there
+     * is no forward progress.
+     * About ten microseconds is the fastest that really works
+     * on the current generation of host machines.
+     */
+
+    if (s->enabled == 1 && (s->delta * period < 10000) && !use_icount) {
+        period = 10000 / s->delta;
+        period_frac = 0;
+    }
+
     s->last_event = s->next_event;
-    s->next_event = s->last_event + s->delta * s->period;
-    if (s->period_frac) {
-        s->next_event += ((int64_t)s->period_frac * s->delta) >> 32;
+    s->next_event = s->last_event + s->delta * period;
+    if (period_frac) {
+        s->next_event += ((int64_t)period_frac * s->delta) >> 32;
     }
     timer_mod(s->timer, s->next_event);
 }
@@ -65,14 +84,16 @@
 
 uint64_t ptimer_get_count(ptimer_state *s)
 {
-    int64_t now;
     uint64_t counter;
 
     if (s->enabled) {
-        now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        int64_t next = s->next_event;
+        bool expired = (now - next >= 0);
+        bool oneshot = (s->enabled == 2);
+
         /* Figure out the current counter value.  */
-        if (now - s->next_event > 0
-            || s->period == 0) {
+        if (expired) {
             /* Prevent timer underflowing if it should already have
                triggered.  */
             counter = 0;
@@ -81,6 +102,13 @@
             uint64_t div;
             int clz1, clz2;
             int shift;
+            uint32_t period_frac = s->period_frac;
+            uint64_t period = s->period;
+
+            if (!oneshot && (s->delta * period < 10000) && !use_icount) {
+                period = 10000 / s->delta;
+                period_frac = 0;
+            }
 
             /* We need to divide time by period, where time is stored in
                rem (64-bit integer) and period is stored in period/period_frac
@@ -92,8 +120,8 @@
                backwards.
             */
 
-            rem = s->next_event - now;
-            div = s->period;
+            rem = next - now;
+            div = period;
 
             clz1 = clz64(rem);
             clz2 = clz64(div);
@@ -102,13 +130,13 @@
             rem <<= shift;
             div <<= shift;
             if (shift >= 32) {
-                div |= ((uint64_t)s->period_frac << (shift - 32));
+                div |= ((uint64_t)period_frac << (shift - 32));
             } else {
                 if (shift != 0)
-                    div |= (s->period_frac >> (32 - shift));
+                    div |= (period_frac >> (32 - shift));
                 /* Look at remaining bits of period_frac and round div up if 
                    necessary.  */
-                if ((uint32_t)(s->period_frac << shift))
+                if ((uint32_t)(period_frac << shift))
                     div += 1;
             }
             counter = rem / div;
@@ -130,16 +158,17 @@
 
 void ptimer_run(ptimer_state *s, int oneshot)
 {
-    if (s->enabled) {
-        return;
-    }
-    if (s->period == 0) {
+    bool was_disabled = !s->enabled;
+
+    if (was_disabled && s->period == 0) {
         fprintf(stderr, "Timer with period zero, disabling\n");
         return;
     }
     s->enabled = oneshot ? 2 : 1;
-    s->next_event = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    ptimer_reload(s);
+    if (was_disabled) {
+        s->next_event = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        ptimer_reload(s);
+    }
 }
 
 /* Pause a timer.  Note that this may cause it to "lose" time, even if it
@@ -157,6 +186,7 @@
 /* Set counter increment interval in nanoseconds.  */
 void ptimer_set_period(ptimer_state *s, int64_t period)
 {
+    s->delta = ptimer_get_count(s);
     s->period = period;
     s->period_frac = 0;
     if (s->enabled) {
@@ -168,6 +198,7 @@
 /* Set counter frequency in Hz.  */
 void ptimer_set_freq(ptimer_state *s, uint32_t freq)
 {
+    s->delta = ptimer_get_count(s);
     s->period = 1000000000ll / freq;
     s->period_frac = (1000000000ll << 32) / freq;
     if (s->enabled) {
@@ -180,19 +211,6 @@
    count = limit.  */
 void ptimer_set_limit(ptimer_state *s, uint64_t limit, int reload)
 {
-    /*
-     * Artificially limit timeout rate to something
-     * achievable under QEMU.  Otherwise, QEMU spends all
-     * its time generating timer interrupts, and there
-     * is no forward progress.
-     * About ten microseconds is the fastest that really works
-     * on the current generation of host machines.
-     */
-
-    if (limit * s->period < 10000 && s->period) {
-        limit = 10000 / s->period;
-    }
-
     s->limit = limit;
     if (reload)
         s->delta = limit;
@@ -202,6 +220,11 @@
     }
 }
 
+uint64_t ptimer_get_limit(ptimer_state *s)
+{
+    return s->limit;
+}
+
 const VMStateDescription vmstate_ptimer = {
     .name = "ptimer",
     .version_id = 1,
@@ -214,7 +237,7 @@
         VMSTATE_INT64(period, ptimer_state),
         VMSTATE_INT64(last_event, ptimer_state),
         VMSTATE_INT64(next_event, ptimer_state),
-        VMSTATE_TIMER(timer, ptimer_state),
+        VMSTATE_TIMER_PTR(timer, ptimer_state),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/core/qdev-properties-system.c b/hw/core/qdev-properties-system.c
index 65901ef..e55afe6 100644
--- a/hw/core/qdev-properties-system.c
+++ b/hw/core/qdev-properties-system.c
@@ -1,5 +1,5 @@
 /*
- * qdev property parsing and global properties
+ * qdev property parsing
  * (parts specific for qemu-system-*)
  *
  * This file is based on code from hw/qdev-properties.c from
@@ -10,8 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
@@ -30,27 +32,27 @@
     char *p;
 
     p = *ptr ? print(*ptr) : g_strdup("");
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
     g_free(p);
 }
 
 static void set_pointer(Object *obj, Visitor *v, Property *prop,
-                        int (*parse)(DeviceState *dev, const char *str,
-                                     void **ptr),
+                        void (*parse)(DeviceState *dev, const char *str,
+                                      void **ptr, const char *propname,
+                                      Error **errp),
                         const char *name, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Error *local_err = NULL;
     void **ptr = qdev_get_prop_ptr(dev, prop);
     char *str;
-    int ret;
 
     if (dev->realized) {
         qdev_prop_set_after_realize(dev, name, errp);
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -60,26 +62,54 @@
         *ptr = NULL;
         return;
     }
-    ret = parse(dev, str, ptr);
-    error_set_from_qdev_prop_error(errp, ret, dev, prop, str);
+    parse(dev, str, ptr, prop->name, errp);
     g_free(str);
 }
 
 /* --- drive --- */
 
-static int parse_drive(DeviceState *dev, const char *str, void **ptr)
+static void parse_drive(DeviceState *dev, const char *str, void **ptr,
+                        const char *propname, Error **errp)
 {
     BlockBackend *blk;
+    bool blk_created = false;
 
     blk = blk_by_name(str);
     if (!blk) {
-        return -ENOENT;
+        BlockDriverState *bs = bdrv_lookup_bs(NULL, str, NULL);
+        if (bs) {
+            blk = blk_new();
+            blk_insert_bs(blk, bs);
+            blk_created = true;
+        }
+    }
+    if (!blk) {
+        error_setg(errp, "Property '%s.%s' can't find value '%s'",
+                   object_get_typename(OBJECT(dev)), propname, str);
+        goto fail;
     }
     if (blk_attach_dev(blk, dev) < 0) {
-        return -EEXIST;
+        DriveInfo *dinfo = blk_legacy_dinfo(blk);
+
+        if (dinfo && dinfo->type != IF_NONE) {
+            error_setg(errp, "Drive '%s' is already in use because "
+                       "it has been automatically connected to another "
+                       "device (did you need 'if=none' in the drive options?)",
+                       str);
+        } else {
+            error_setg(errp, "Drive '%s' is already in use by another device",
+                       str);
+        }
+        goto fail;
     }
+
     *ptr = blk;
-    return 0;
+
+fail:
+    if (blk_created) {
+        /* If we need to keep a reference, blk_attach_dev() took it */
+        blk_unref(blk);
+    }
 }
 
 static void release_drive(Object *obj, const char *name, void *opaque)
@@ -89,31 +119,40 @@
     BlockBackend **ptr = qdev_get_prop_ptr(dev, prop);
 
     if (*ptr) {
-        blk_detach_dev(*ptr, dev);
         blockdev_auto_del(*ptr);
+        blk_detach_dev(*ptr, dev);
     }
 }
 
 static char *print_drive(void *ptr)
 {
-    return g_strdup(blk_name(ptr));
+    const char *name;
+
+    name = blk_name(ptr);
+    if (!*name) {
+        BlockDriverState *bs = blk_bs(ptr);
+        if (bs) {
+            name = bdrv_get_node_name(bs);
+        }
+    }
+    return g_strdup(name);
 }
 
-static void get_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void get_drive(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     get_pointer(obj, v, opaque, print_drive, name, errp);
 }
 
-static void set_drive(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void set_drive(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     set_pointer(obj, v, opaque, parse_drive, name, errp);
 }
 
 PropertyInfo qdev_prop_drive = {
     .name  = "str",
-    .description = "ID of a drive to use as a backend",
+    .description = "Node name or ID of a block device to use as a backend",
     .get   = get_drive,
     .set   = set_drive,
     .release = release_drive,
@@ -121,17 +160,21 @@
 
 /* --- character device --- */
 
-static int parse_chr(DeviceState *dev, const char *str, void **ptr)
+static void parse_chr(DeviceState *dev, const char *str, void **ptr,
+                      const char *propname, Error **errp)
 {
     CharDriverState *chr = qemu_chr_find(str);
     if (chr == NULL) {
-        return -ENOENT;
+        error_setg(errp, "Property '%s.%s' can't find value '%s'",
+                   object_get_typename(OBJECT(dev)), propname, str);
+        return;
     }
     if (qemu_chr_fe_claim(chr) != 0) {
-        return -EEXIST;
+        error_setg(errp, "Property '%s.%s' can't take value '%s', it's in use",
+                  object_get_typename(OBJECT(dev)), propname, str);
+        return;
     }
     *ptr = chr;
-    return 0;
 }
 
 static void release_chr(Object *obj, const char *name, void *opaque)
@@ -156,14 +199,14 @@
     return g_strdup(val);
 }
 
-static void get_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void get_chr(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     get_pointer(obj, v, opaque, print_chr, name, errp);
 }
 
-static void set_chr(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void set_chr(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     set_pointer(obj, v, opaque, parse_chr, name, errp);
 }
@@ -177,42 +220,69 @@
 };
 
 /* --- netdev device --- */
-
-static int parse_netdev(DeviceState *dev, const char *str, void **ptr)
+static void get_netdev(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
-    NICPeers *peers_ptr = (NICPeers *)ptr;
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    NICPeers *peers_ptr = qdev_get_prop_ptr(dev, prop);
+    char *p = g_strdup(peers_ptr->ncs[0] ? peers_ptr->ncs[0]->name : "");
+
+    visit_type_str(v, name, &p, errp);
+    g_free(p);
+}
+
+static void set_netdev(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    NICPeers *peers_ptr = qdev_get_prop_ptr(dev, prop);
     NetClientState **ncs = peers_ptr->ncs;
     NetClientState *peers[MAX_QUEUE_NUM];
-    int queues, i = 0;
-    int ret;
+    Error *local_err = NULL;
+    int queues, err = 0, i = 0;
+    char *str;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_str(v, name, &str, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
 
     queues = qemu_find_net_clients_except(str, peers,
-                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          NET_CLIENT_DRIVER_NIC,
                                           MAX_QUEUE_NUM);
     if (queues == 0) {
-        ret = -ENOENT;
-        goto err;
+        err = -ENOENT;
+        goto out;
     }
 
     if (queues > MAX_QUEUE_NUM) {
-        ret = -E2BIG;
-        goto err;
+        error_setg(errp, "queues of backend '%s'(%d) exceeds QEMU limitation(%d)",
+                   str, queues, MAX_QUEUE_NUM);
+        goto out;
     }
 
     for (i = 0; i < queues; i++) {
         if (peers[i] == NULL) {
-            ret = -ENOENT;
-            goto err;
+            err = -ENOENT;
+            goto out;
         }
 
         if (peers[i]->peer) {
-            ret = -EEXIST;
-            goto err;
+            err = -EEXIST;
+            goto out;
         }
 
         if (ncs[i]) {
-            ret = -EINVAL;
-            goto err;
+            err = -EINVAL;
+            goto out;
         }
 
         ncs[i] = peers[i];
@@ -221,30 +291,9 @@
 
     peers_ptr->queues = queues;
 
-    return 0;
-
-err:
-    return ret;
-}
-
-static char *print_netdev(void *ptr)
-{
-    NetClientState *netdev = ptr;
-    const char *val = netdev->name ? netdev->name : "";
-
-    return g_strdup(val);
-}
-
-static void get_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
-{
-    get_pointer(obj, v, opaque, print_netdev, name, errp);
-}
-
-static void set_netdev(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
-{
-    set_pointer(obj, v, opaque, parse_netdev, name, errp);
+out:
+    error_set_from_qdev_prop_error(errp, err, dev, prop, str);
+    g_free(str);
 }
 
 PropertyInfo qdev_prop_netdev = {
@@ -270,8 +319,8 @@
     return snprintf(dest, len, "<null>");
 }
 
-static void get_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_vlan(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -285,11 +334,11 @@
         }
     }
 
-    visit_type_int32(v, &id, name, errp);
+    visit_type_int32(v, name, &id, errp);
 }
 
-static void set_vlan(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_vlan(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -304,7 +353,7 @@
         return;
     }
 
-    visit_type_int32(v, &id, name, &local_err);
+    visit_type_int32(v, name, &id, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -320,8 +369,8 @@
 
     hubport = net_hub_port_find(id);
     if (!hubport) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  name, prop->info->name);
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   name, prop->info->name);
         return;
     }
     *ptr = hubport;
@@ -335,27 +384,24 @@
     .set   = set_vlan,
 };
 
-int qdev_prop_set_drive(DeviceState *dev, const char *name,
-                        BlockBackend *value)
+void qdev_prop_set_drive(DeviceState *dev, const char *name,
+                         BlockBackend *value, Error **errp)
 {
-    Error *err = NULL;
-    object_property_set_str(OBJECT(dev),
-                            value ? blk_name(value) : "", name, &err);
-    if (err) {
-        qerror_report_err(err);
-        error_free(err);
-        return -1;
+    const char *ref = "";
+
+    if (value) {
+        ref = blk_name(value);
+        if (!*ref) {
+            BlockDriverState *bs = blk_bs(value);
+            if (bs) {
+                ref = bdrv_get_node_name(bs);
+            }
+        }
     }
-    return 0;
+
+    object_property_set_str(OBJECT(dev), ref, name, errp);
 }
 
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
-                                BlockBackend *value)
-{
-    if (qdev_prop_set_drive(dev, name, value) < 0) {
-        exit(1);
-    }
-}
 void qdev_prop_set_chr(DeviceState *dev, const char *name,
                        CharDriverState *value)
 {
@@ -384,21 +430,3 @@
     }
     nd->instantiated = 1;
 }
-
-static int qdev_add_one_global(QemuOpts *opts, void *opaque)
-{
-    GlobalProperty *g;
-
-    g = g_malloc0(sizeof(*g));
-    g->driver   = qemu_opt_get(opts, "driver");
-    g->property = qemu_opt_get(opts, "property");
-    g->value    = qemu_opt_get(opts, "value");
-    g->user_provided = true;
-    qdev_prop_register_global(g);
-    return 0;
-}
-
-void qemu_add_globals(void)
-{
-    qemu_opts_foreach(qemu_find_opts("global"), qdev_add_one_global, NULL, 0);
-}
diff --git a/hw/core/qdev-properties.c b/hw/core/qdev-properties.c
index 3d8ed31..e31d30c 100644
--- a/hw/core/qdev-properties.c
+++ b/hw/core/qdev-properties.c
@@ -1,6 +1,10 @@
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
+#include "hw/pci/pci.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "hw/block/block.h"
 #include "net/hub.h"
@@ -40,19 +44,18 @@
     return ptr;
 }
 
-static void get_enum(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_enum(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     int *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_enum(v, ptr, prop->info->enum_table,
-                    prop->info->name, prop->name, errp);
+    visit_type_enum(v, prop->name, ptr, prop->info->enum_table, errp);
 }
 
-static void set_enum(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_enum(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -63,8 +66,7 @@
         return;
     }
 
-    visit_type_enum(v, ptr, prop->info->enum_table,
-                    prop->info->name, prop->name, errp);
+    visit_type_enum(v, prop->name, ptr, prop->info->enum_table, errp);
 }
 
 /* Bit */
@@ -86,19 +88,19 @@
     }
 }
 
-static void prop_get_bit(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void prop_get_bit(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint32_t *p = qdev_get_prop_ptr(dev, prop);
     bool value = (*p & qdev_get_prop_mask(prop)) != 0;
 
-    visit_type_bool(v, &value, name, errp);
+    visit_type_bool(v, name, &value, errp);
 }
 
-static void prop_set_bit(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void prop_set_bit(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -110,7 +112,7 @@
         return;
     }
 
-    visit_type_bool(v, &value, name, &local_err);
+    visit_type_bool(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -125,20 +127,78 @@
     .set   = prop_set_bit,
 };
 
+/* Bit64 */
+
+static uint64_t qdev_get_prop_mask64(Property *prop)
+{
+    assert(prop->info == &qdev_prop_bit64);
+    return 0x1ull << prop->bitnr;
+}
+
+static void bit64_prop_set(DeviceState *dev, Property *props, bool val)
+{
+    uint64_t *p = qdev_get_prop_ptr(dev, props);
+    uint64_t mask = qdev_get_prop_mask64(props);
+    if (val) {
+        *p |= mask;
+    } else {
+        *p &= ~mask;
+    }
+}
+
+static void prop_get_bit64(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    uint64_t *p = qdev_get_prop_ptr(dev, prop);
+    bool value = (*p & qdev_get_prop_mask64(prop)) != 0;
+
+    visit_type_bool(v, name, &value, errp);
+}
+
+static void prop_set_bit64(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    Error *local_err = NULL;
+    bool value;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_bool(v, name, &value, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    bit64_prop_set(dev, prop, value);
+}
+
+PropertyInfo qdev_prop_bit64 = {
+    .name  = "bool",
+    .description = "on/off",
+    .get   = prop_get_bit64,
+    .set   = prop_set_bit64,
+};
+
 /* --- bool --- */
 
-static void get_bool(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_bool(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     bool *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_bool(v, ptr, name, errp);
+    visit_type_bool(v, name, ptr, errp);
 }
 
-static void set_bool(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_bool(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -149,7 +209,7 @@
         return;
     }
 
-    visit_type_bool(v, ptr, name, errp);
+    visit_type_bool(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_bool = {
@@ -160,18 +220,18 @@
 
 /* --- 8bit integer --- */
 
-static void get_uint8(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void get_uint8(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint8_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint8(v, ptr, name, errp);
+    visit_type_uint8(v, name, ptr, errp);
 }
 
-static void set_uint8(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void set_uint8(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -182,7 +242,7 @@
         return;
     }
 
-    visit_type_uint8(v, ptr, name, errp);
+    visit_type_uint8(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint8 = {
@@ -193,18 +253,18 @@
 
 /* --- 16bit integer --- */
 
-static void get_uint16(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_uint16(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint16_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint16(v, ptr, name, errp);
+    visit_type_uint16(v, name, ptr, errp);
 }
 
-static void set_uint16(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_uint16(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -215,7 +275,7 @@
         return;
     }
 
-    visit_type_uint16(v, ptr, name, errp);
+    visit_type_uint16(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint16 = {
@@ -226,18 +286,18 @@
 
 /* --- 32bit integer --- */
 
-static void get_uint32(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_uint32(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint32(v, ptr, name, errp);
+    visit_type_uint32(v, name, ptr, errp);
 }
 
-static void set_uint32(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_uint32(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -248,21 +308,21 @@
         return;
     }
 
-    visit_type_uint32(v, ptr, name, errp);
+    visit_type_uint32(v, name, ptr, errp);
 }
 
-static void get_int32(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void get_int32(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     int32_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_int32(v, ptr, name, errp);
+    visit_type_int32(v, name, ptr, errp);
 }
 
-static void set_int32(Object *obj, Visitor *v, void *opaque,
-                      const char *name, Error **errp)
+static void set_int32(Object *obj, Visitor *v, const char *name, void *opaque,
+                      Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -273,7 +333,7 @@
         return;
     }
 
-    visit_type_int32(v, ptr, name, errp);
+    visit_type_int32(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint32 = {
@@ -290,18 +350,18 @@
 
 /* --- 64bit integer --- */
 
-static void get_uint64(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_uint64(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_uint64(v, ptr, name, errp);
+    visit_type_uint64(v, name, ptr, errp);
 }
 
-static void set_uint64(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_uint64(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -312,7 +372,7 @@
         return;
     }
 
-    visit_type_uint64(v, ptr, name, errp);
+    visit_type_uint64(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_uint64 = {
@@ -329,8 +389,8 @@
     g_free(*(char **)qdev_get_prop_ptr(DEVICE(obj), prop));
 }
 
-static void get_string(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void get_string(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -338,14 +398,14 @@
 
     if (!*ptr) {
         char *str = (char *)"";
-        visit_type_str(v, &str, name, errp);
+        visit_type_str(v, name, &str, errp);
     } else {
-        visit_type_str(v, ptr, name, errp);
+        visit_type_str(v, name, ptr, errp);
     }
 }
 
-static void set_string(Object *obj, Visitor *v, void *opaque,
-                       const char *name, Error **errp)
+static void set_string(Object *obj, Visitor *v, const char *name,
+                       void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -358,14 +418,12 @@
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
-    if (*ptr) {
-        g_free(*ptr);
-    }
+    g_free(*ptr);
     *ptr = str;
 }
 
@@ -390,8 +448,8 @@
  *   01:02:03:04:05:06
  *   01-02-03-04-05-06
  */
-static void get_mac(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void get_mac(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -403,11 +461,11 @@
              mac->a[0], mac->a[1], mac->a[2],
              mac->a[3], mac->a[4], mac->a[5]);
 
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
 }
 
-static void set_mac(Object *obj, Visitor *v, void *opaque,
-                    const char *name, Error **errp)
+static void set_mac(Object *obj, Visitor *v, const char *name, void *opaque,
+                    Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -421,7 +479,7 @@
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -460,6 +518,16 @@
     .set   = set_mac,
 };
 
+/* --- on/off/auto --- */
+
+PropertyInfo qdev_prop_on_off_auto = {
+    .name = "OnOffAuto",
+    .description = "on/off/auto",
+    .enum_table = OnOffAuto_lookup,
+    .get = get_enum,
+    .set = set_enum,
+};
+
 /* --- lost tick policy --- */
 
 QEMU_BUILD_BUG_ON(sizeof(LostTickPolicy) != sizeof(int));
@@ -471,6 +539,19 @@
     .set   = set_enum,
 };
 
+/* --- Block device error handling policy --- */
+
+QEMU_BUILD_BUG_ON(sizeof(BlockdevOnError) != sizeof(int));
+
+PropertyInfo qdev_prop_blockdev_on_error = {
+    .name = "BlockdevOnError",
+    .description = "Error handling policy, "
+                   "report/ignore/enospc/stop/auto",
+    .enum_table = BlockdevOnError_lookup,
+    .get = get_enum,
+    .set = set_enum,
+};
+
 /* --- BIOS CHS translation */
 
 QEMU_BUILD_BUG_ON(sizeof(BiosAtaTranslation) != sizeof(int));
@@ -484,13 +565,24 @@
     .set = set_enum,
 };
 
+/* --- FDC default drive types */
+
+PropertyInfo qdev_prop_fdc_drive_type = {
+    .name = "FdcDriveType",
+    .description = "FDC drive type, "
+                   "144/288/120/none/auto",
+    .enum_table = FloppyDriveType_lookup,
+    .get = get_enum,
+    .set = set_enum
+};
+
 /* --- pci address --- */
 
 /*
  * bus-local address, i.e. "$slot" or "$slot.$fn"
  */
-static void set_pci_devfn(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void set_pci_devfn(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -504,16 +596,16 @@
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_free(local_err);
         local_err = NULL;
-        visit_type_int32(v, &value, name, &local_err);
+        visit_type_int32(v, name, &value, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
         } else if (value < -1 || value > 255) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "pci_devfn");
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                       name ? name : "null", "pci_devfn");
         } else {
             *ptr = value;
         }
@@ -560,8 +652,8 @@
 
 /* --- blocksize --- */
 
-static void set_blocksize(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void set_blocksize(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -575,14 +667,15 @@
         return;
     }
 
-    visit_type_uint16(v, &value, name, &local_err);
+    visit_type_uint16(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
-    if (value < min || value > max) {
-        error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,
-                  dev->id?:"", name, (int64_t)value, min, max);
+    /* value of 0 means "unset" */
+    if (value && (value < min || value > max)) {
+        error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,
+                   dev->id ? : "", name, (int64_t)value, min, max);
         return;
     }
 
@@ -606,8 +699,8 @@
 
 /* --- pci host address --- */
 
-static void get_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void get_pci_host_devaddr(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -621,15 +714,15 @@
     assert(rc == sizeof(buffer) - 1);
     (void)rc;
 
-    visit_type_str(v, &p, name, errp);
+    visit_type_str(v, name, &p, errp);
 }
 
 /*
  * Parse [<domain>:]<bus>:<slot>.<func>
  *   if <domain> is not supplied, it's assumed to be 0.
  */
-static void set_pci_host_devaddr(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void set_pci_host_devaddr(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -646,7 +739,7 @@
         return;
     }
 
-    visit_type_str(v, &str, name, &local_err);
+    visit_type_str(v, name, &str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -742,8 +835,8 @@
     g_free(p);
 }
 
-static void set_prop_arraylen(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void set_prop_arraylen(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     /* Setter for the property which defines the length of a
      * variable-sized property array. As well as actually setting the
@@ -768,7 +861,7 @@
                    name);
         return;
     }
-    visit_type_uint32(v, alenptr, name, &local_err);
+    visit_type_uint32(v, name, alenptr, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -865,8 +958,8 @@
         break;
     default:
     case -EINVAL:
-        error_set(errp, QERR_PROPERTY_VALUE_BAD,
-                  object_get_typename(OBJECT(dev)), prop->name, value);
+        error_setg(errp, QERR_PROPERTY_VALUE_BAD,
+                   object_get_typename(OBJECT(dev)), prop->name, value);
         break;
     case -ENOENT:
         error_setg(errp, "Property '%s.%s' can't find value '%s'",
@@ -941,12 +1034,11 @@
     *ptr = value;
 }
 
-static QTAILQ_HEAD(, GlobalProperty) global_props =
-        QTAILQ_HEAD_INITIALIZER(global_props);
+static GList *global_props;
 
 void qdev_prop_register_global(GlobalProperty *prop)
 {
-    QTAILQ_INSERT_TAIL(&global_props, prop, next);
+    global_props = g_list_append(global_props, prop);
 }
 
 void qdev_prop_register_global_list(GlobalProperty *props)
@@ -960,10 +1052,11 @@
 
 int qdev_prop_check_globals(void)
 {
-    GlobalProperty *prop;
+    GList *l;
     int ret = 0;
 
-    QTAILQ_FOREACH(prop, &global_props, next) {
+    for (l = global_props; l; l = l->next) {
+        GlobalProperty *prop = l->data;
         ObjectClass *oc;
         DeviceClass *dc;
         if (prop->used) {
@@ -991,12 +1084,13 @@
     return ret;
 }
 
-void qdev_prop_set_globals_for_type(DeviceState *dev, const char *typename,
-                                    Error **errp)
+static void qdev_prop_set_globals_for_type(DeviceState *dev,
+                                           const char *typename)
 {
-    GlobalProperty *prop;
+    GList *l;
 
-    QTAILQ_FOREACH(prop, &global_props, next) {
+    for (l = global_props; l; l = l->next) {
+        GlobalProperty *prop = l->data;
         Error *err = NULL;
 
         if (strcmp(typename, prop->driver) != 0) {
@@ -1005,49 +1099,48 @@
         prop->used = true;
         object_property_parse(OBJECT(dev), prop->value, prop->property, &err);
         if (err != NULL) {
-            error_propagate(errp, err);
-            return;
+            error_prepend(&err, "can't apply global %s.%s=%s: ",
+                          prop->driver, prop->property, prop->value);
+            if (!dev->hotplugged && prop->errp) {
+                error_propagate(prop->errp, err);
+            } else {
+                assert(prop->user_provided);
+                error_reportf_err(err, "Warning: ");
+            }
         }
     }
 }
 
-void qdev_prop_set_globals(DeviceState *dev, Error **errp)
+void qdev_prop_set_globals(DeviceState *dev)
 {
     ObjectClass *class = object_get_class(OBJECT(dev));
 
     do {
-        Error *err = NULL;
-
-        qdev_prop_set_globals_for_type(dev, object_class_get_name(class),
-                                       &err);
-        if (err != NULL) {
-            error_propagate(errp, err);
-            return;
-        }
+        qdev_prop_set_globals_for_type(dev, object_class_get_name(class));
         class = object_class_get_parent(class);
     } while (class);
 }
 
 /* --- 64bit unsigned int 'size' type --- */
 
-static void get_size(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void get_size(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_size(v, ptr, name, errp);
+    visit_type_size(v, name, ptr, errp);
 }
 
-static void set_size(Object *obj, Visitor *v, void *opaque,
-                     const char *name, Error **errp)
+static void set_size(Object *obj, Visitor *v, const char *name, void *opaque,
+                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
     uint64_t *ptr = qdev_get_prop_ptr(dev, prop);
 
-    visit_type_size(v, ptr, name, errp);
+    visit_type_size(v, name, ptr, errp);
 }
 
 PropertyInfo qdev_prop_size = {
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 447be39..5783442 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -25,15 +25,17 @@
    inherit from a particular bus (e.g. PCI or I2C) rather than
    this API directly.  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "hw/fw-path-provider.h"
 #include "sysemu/sysemu.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/visitor.h"
 #include "qapi/qmp/qjson.h"
+#include "qemu/error-report.h"
 #include "hw/hotplug.h"
 #include "hw/boards.h"
+#include "hw/sysbus.h"
 #include "qapi-event.h"
 
 int qdev_hotplug = 0;
@@ -57,9 +59,6 @@
     return object_get_typename(OBJECT(dev));
 }
 
-static void qdev_property_add_legacy(DeviceState *dev, Property *prop,
-                                     Error **errp);
-
 static void bus_remove_child(BusState *bus, DeviceState *child)
 {
     BusChild *kid;
@@ -108,27 +107,9 @@
     bus_add_child(bus, dev);
 }
 
-static void qbus_set_hotplug_handler_internal(BusState *bus, Object *handler,
-                                              Error **errp)
-{
-
-    object_property_set_link(OBJECT(bus), OBJECT(handler),
-                             QDEV_HOTPLUG_HANDLER_PROPERTY, errp);
-}
-
-void qbus_set_hotplug_handler(BusState *bus, DeviceState *handler, Error **errp)
-{
-    qbus_set_hotplug_handler_internal(bus, OBJECT(handler), errp);
-}
-
-void qbus_set_bus_hotplug_handler(BusState *bus, Error **errp)
-{
-    qbus_set_hotplug_handler_internal(bus, OBJECT(bus), errp);
-}
-
-/* Create a new device.  This only initializes the device state structure
-   and allows properties to be set.  qdev_init should be called to
-   initialize the actual device emulation.  */
+/* Create a new device.  This only initializes the device state
+   structure and allows properties to be set.  The device still needs
+   to be realized.  See qdev-core.h.  */
 DeviceState *qdev_create(BusState *bus, const char *name)
 {
     DeviceState *dev;
@@ -160,6 +141,12 @@
     }
 
     if (!bus) {
+        /* Assert that the device really is a SysBusDevice before
+         * we put it onto the sysbus. Non-sysbus devices which aren't
+         * being put onto a bus should be created with object_new(TYPE_FOO),
+         * not qdev_create(NULL, TYPE_FOO).
+         */
+        g_assert(object_dynamic_cast(OBJECT(dev), TYPE_SYS_BUS_DEVICE));
         bus = sysbus_get_default();
     }
 
@@ -168,27 +155,56 @@
     return dev;
 }
 
-/* Initialize a device.  Device properties should be set before calling
-   this function.  IRQs and MMIO regions should be connected/mapped after
-   calling this function.
-   On failure, destroy the device and return negative value.
-   Return 0 on success.  */
-int qdev_init(DeviceState *dev)
+static QTAILQ_HEAD(device_listeners, DeviceListener) device_listeners
+    = QTAILQ_HEAD_INITIALIZER(device_listeners);
+
+enum ListenerDirection { Forward, Reverse };
+
+#define DEVICE_LISTENER_CALL(_callback, _direction, _args...)     \
+    do {                                                          \
+        DeviceListener *_listener;                                \
+                                                                  \
+        switch (_direction) {                                     \
+        case Forward:                                             \
+            QTAILQ_FOREACH(_listener, &device_listeners, link) {  \
+                if (_listener->_callback) {                       \
+                    _listener->_callback(_listener, ##_args);     \
+                }                                                 \
+            }                                                     \
+            break;                                                \
+        case Reverse:                                             \
+            QTAILQ_FOREACH_REVERSE(_listener, &device_listeners,  \
+                                   device_listeners, link) {      \
+                if (_listener->_callback) {                       \
+                    _listener->_callback(_listener, ##_args);     \
+                }                                                 \
+            }                                                     \
+            break;                                                \
+        default:                                                  \
+            abort();                                              \
+        }                                                         \
+    } while (0)
+
+static int device_listener_add(DeviceState *dev, void *opaque)
 {
-    Error *local_err = NULL;
+    DEVICE_LISTENER_CALL(realize, Forward, dev);
 
-    assert(!dev->realized);
-
-    object_property_set_bool(OBJECT(dev), true, "realized", &local_err);
-    if (local_err != NULL) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        object_unparent(OBJECT(dev));
-        return -1;
-    }
     return 0;
 }
 
+void device_listener_register(DeviceListener *listener)
+{
+    QTAILQ_INSERT_TAIL(&device_listeners, listener, link);
+
+    qbus_walk_children(sysbus_get_default(), NULL, NULL, device_listener_add,
+                       NULL, NULL);
+}
+
+void device_listener_unregister(DeviceListener *listener)
+{
+    QTAILQ_REMOVE(&device_listeners, listener, link);
+}
+
 static void device_realize(DeviceState *dev, Error **errp)
 {
     DeviceClass *dc = DEVICE_GET_CLASS(dev);
@@ -223,7 +239,7 @@
     dev->alias_required_for_version = required_for_version;
 }
 
-static HotplugHandler *qdev_get_hotplug_handler(DeviceState *dev)
+HotplugHandler *qdev_get_hotplug_handler(DeviceState *dev)
 {
     HotplugHandler *hotplug_ctrl = NULL;
 
@@ -247,13 +263,13 @@
     HotplugHandlerClass *hdc;
 
     if (dev->parent_bus && !qbus_is_hotpluggable(dev->parent_bus)) {
-        error_set(errp, QERR_BUS_NO_HOTPLUG, dev->parent_bus->name);
+        error_setg(errp, QERR_BUS_NO_HOTPLUG, dev->parent_bus->name);
         return;
     }
 
     if (!dc->hotpluggable) {
-        error_set(errp, QERR_DEVICE_NO_HOTPLUG,
-                  object_get_typename(OBJECT(dev)));
+        error_setg(errp, QERR_DEVICE_NO_HOTPLUG,
+                   object_get_typename(OBJECT(dev)));
         return;
     }
 
@@ -295,6 +311,11 @@
     qdev_walk_children(dev, NULL, NULL, qdev_reset_one, qbus_reset_one, NULL);
 }
 
+void qdev_reset_all_fn(void *opaque)
+{
+    qdev_reset_all(DEVICE(opaque));
+}
+
 void qbus_reset_all(BusState *bus)
 {
     qbus_walk_children(bus, NULL, NULL, qdev_reset_one, qbus_reset_one, NULL);
@@ -314,21 +335,33 @@
     object_unparent(OBJECT(dev));
 }
 
-/* Like qdev_init(), but terminate program via error_report() instead of
-   returning an error value.  This is okay during machine creation.
-   Don't use for hotplug, because there callers need to recover from
-   failure.  Exception: if you know the device's init() callback can't
-   fail, then qdev_init_nofail() can't fail either, and is therefore
-   usable even then.  But relying on the device implementation that
-   way is somewhat unclean, and best avoided.  */
+/*
+ * Realize @dev.
+ * Device properties should be set before calling this function.  IRQs
+ * and MMIO regions should be connected/mapped after calling this
+ * function.
+ * On failure, report an error with error_report() and terminate the
+ * program.  This is okay during machine creation.  Don't use for
+ * hotplug, because there callers need to recover from failure.
+ * Exception: if you know the device's init() callback can't fail,
+ * then qdev_init_nofail() can't fail either, and is therefore usable
+ * even then.  But relying on the device implementation that way is
+ * somewhat unclean, and best avoided.
+ */
 void qdev_init_nofail(DeviceState *dev)
 {
-    const char *typename = object_get_typename(OBJECT(dev));
+    Error *err = NULL;
 
-    if (qdev_init(dev) < 0) {
-        error_report("Initialization of device %s failed", typename);
+    assert(!dev->realized);
+
+    object_ref(OBJECT(dev));
+    object_property_set_bool(OBJECT(dev), true, "realized", &err);
+    if (err) {
+        error_reportf_err(err, "Initialization of device %s failed: ",
+                          object_get_typename(OBJECT(dev)));
         exit(1);
     }
+    object_unref(OBJECT(dev));
 }
 
 void qdev_machine_creation_done(void)
@@ -376,17 +409,21 @@
 {
     int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-in");
 
     assert(gpio_list->num_out == 0 || !name);
     gpio_list->in = qemu_extend_irqs(gpio_list->in, gpio_list->num_in, handler,
                                      dev, n);
 
+    if (!name) {
+        name = "unnamed-gpio-in";
+    }
     for (i = gpio_list->num_in; i < gpio_list->num_in + n; i++) {
+        gchar *propname = g_strdup_printf("%s[%u]", name, i);
+
         object_property_add_child(OBJECT(dev), propname,
                                   OBJECT(gpio_list->in[i]), &error_abort);
+        g_free(propname);
     }
-    g_free(propname);
 
     gpio_list->num_in += n;
 }
@@ -401,20 +438,25 @@
 {
     int i;
     NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
-    char *propname = g_strdup_printf("%s[*]", name ? name : "unnamed-gpio-out");
 
     assert(gpio_list->num_in == 0 || !name);
-    gpio_list->num_out += n;
 
+    if (!name) {
+        name = "unnamed-gpio-out";
+    }
+    memset(pins, 0, sizeof(*pins) * n);
     for (i = 0; i < n; ++i) {
-        memset(&pins[i], 0, sizeof(*pins));
+        gchar *propname = g_strdup_printf("%s[%u]", name,
+                                          gpio_list->num_out + i);
+
         object_property_add_link(OBJECT(dev), propname, TYPE_IRQ,
                                  (Object **)&pins[i],
                                  object_property_allow_set_link,
                                  OBJ_PROP_LINK_UNREF_ON_RELEASE,
                                  &error_abort);
+        g_free(propname);
     }
-    g_free(propname);
+    gpio_list->num_out += n;
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
@@ -446,8 +488,9 @@
          * with an error without doing anything.  If it has none, it will
          * never fail.  So we can just call it with a NULL Error pointer.
          */
-        object_property_add_child(qdev_get_machine(), "non-qdev-gpio[*]",
-                                  OBJECT(pin), NULL);
+        object_property_add_child(container_get(qdev_get_machine(),
+                                                "/unattached"),
+                                  "non-qdev-gpio[*]", OBJECT(pin), NULL);
     }
     object_property_set_link(OBJECT(dev), OBJECT(pin), propname, &error_abort);
     g_free(propname);
@@ -464,7 +507,7 @@
     return ret;
 }
 
-/* disconnect a GPIO ouput, returning the disconnected input (if any) */
+/* disconnect a GPIO output, returning the disconnected input (if any) */
 
 static qemu_irq qdev_disconnect_gpio_out_named(DeviceState *dev,
                                                const char *name, int n)
@@ -507,6 +550,7 @@
         object_property_add_alias(OBJECT(container), propname,
                                   OBJECT(dev), propname,
                                   &error_abort);
+        g_free(propname);
     }
     for (i = 0; i < ngl->num_out; i++) {
         const char *nm = ngl->name ? ngl->name : "unnamed-gpio-out";
@@ -515,6 +559,7 @@
         object_property_add_alias(OBJECT(container), propname,
                                   OBJECT(dev), propname,
                                   &error_abort);
+        g_free(propname);
     }
     QLIST_REMOVE(ngl, node);
     QLIST_INSERT_HEAD(&container->gpios, ngl, node);
@@ -523,6 +568,12 @@
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name)
 {
     BusState *bus;
+    Object *child = object_resolve_path_component(OBJECT(dev), name);
+
+    bus = (BusState *)object_dynamic_cast(child, TYPE_BUS);
+    if (bus) {
+        return bus;
+    }
 
     QLIST_FOREACH(bus, &dev->child_bus, sibling) {
         if (strcmp(name, bus->name) == 0) {
@@ -532,40 +583,6 @@
     return NULL;
 }
 
-int qbus_walk_children(BusState *bus,
-                       qdev_walkerfn *pre_devfn, qbus_walkerfn *pre_busfn,
-                       qdev_walkerfn *post_devfn, qbus_walkerfn *post_busfn,
-                       void *opaque)
-{
-    BusChild *kid;
-    int err;
-
-    if (pre_busfn) {
-        err = pre_busfn(bus, opaque);
-        if (err) {
-            return err;
-        }
-    }
-
-    QTAILQ_FOREACH(kid, &bus->children, sibling) {
-        err = qdev_walk_children(kid->child,
-                                 pre_devfn, pre_busfn,
-                                 post_devfn, post_busfn, opaque);
-        if (err < 0) {
-            return err;
-        }
-    }
-
-    if (post_busfn) {
-        err = post_busfn(bus, opaque);
-        if (err) {
-            return err;
-        }
-    }
-
-    return 0;
-}
-
 int qdev_walk_children(DeviceState *dev,
                        qdev_walkerfn *pre_devfn, qbus_walkerfn *pre_busfn,
                        qdev_walkerfn *post_devfn, qbus_walkerfn *post_busfn,
@@ -622,132 +639,6 @@
     return NULL;
 }
 
-static void qbus_realize(BusState *bus, DeviceState *parent, const char *name)
-{
-    const char *typename = object_get_typename(OBJECT(bus));
-    BusClass *bc;
-    char *buf;
-    int i, len, bus_id;
-
-    bus->parent = parent;
-
-    if (name) {
-        bus->name = g_strdup(name);
-    } else if (bus->parent && bus->parent->id) {
-        /* parent device has id -> use it plus parent-bus-id for bus name */
-        bus_id = bus->parent->num_child_bus;
-
-        len = strlen(bus->parent->id) + 16;
-        buf = g_malloc(len);
-        snprintf(buf, len, "%s.%d", bus->parent->id, bus_id);
-        bus->name = buf;
-    } else {
-        /* no id -> use lowercase bus type plus global bus-id for bus name */
-        bc = BUS_GET_CLASS(bus);
-        bus_id = bc->automatic_ids++;
-
-        len = strlen(typename) + 16;
-        buf = g_malloc(len);
-        len = snprintf(buf, len, "%s.%d", typename, bus_id);
-        for (i = 0; i < len; i++) {
-            buf[i] = qemu_tolower(buf[i]);
-        }
-        bus->name = buf;
-    }
-
-    if (bus->parent) {
-        QLIST_INSERT_HEAD(&bus->parent->child_bus, bus, sibling);
-        bus->parent->num_child_bus++;
-        object_property_add_child(OBJECT(bus->parent), bus->name, OBJECT(bus), NULL);
-        object_unref(OBJECT(bus));
-    } else if (bus != sysbus_get_default()) {
-        /* TODO: once all bus devices are qdevified,
-           only reset handler for main_system_bus should be registered here. */
-        qemu_register_reset(qbus_reset_all_fn, bus);
-    }
-}
-
-static void bus_unparent(Object *obj)
-{
-    BusState *bus = BUS(obj);
-    BusChild *kid;
-
-    while ((kid = QTAILQ_FIRST(&bus->children)) != NULL) {
-        DeviceState *dev = kid->child;
-        object_unparent(OBJECT(dev));
-    }
-    if (bus->parent) {
-        QLIST_REMOVE(bus, sibling);
-        bus->parent->num_child_bus--;
-        bus->parent = NULL;
-    } else {
-        assert(bus != sysbus_get_default()); /* main_system_bus is never freed */
-        qemu_unregister_reset(qbus_reset_all_fn, bus);
-    }
-}
-
-static bool bus_get_realized(Object *obj, Error **errp)
-{
-    BusState *bus = BUS(obj);
-
-    return bus->realized;
-}
-
-static void bus_set_realized(Object *obj, bool value, Error **errp)
-{
-    BusState *bus = BUS(obj);
-    BusClass *bc = BUS_GET_CLASS(bus);
-    BusChild *kid;
-    Error *local_err = NULL;
-
-    if (value && !bus->realized) {
-        if (bc->realize) {
-            bc->realize(bus, &local_err);
-        }
-
-        /* TODO: recursive realization */
-    } else if (!value && bus->realized) {
-        QTAILQ_FOREACH(kid, &bus->children, sibling) {
-            DeviceState *dev = kid->child;
-            object_property_set_bool(OBJECT(dev), false, "realized",
-                                     &local_err);
-            if (local_err != NULL) {
-                break;
-            }
-        }
-        if (bc->unrealize && local_err == NULL) {
-            bc->unrealize(bus, &local_err);
-        }
-    }
-
-    if (local_err != NULL) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    bus->realized = value;
-}
-
-void qbus_create_inplace(void *bus, size_t size, const char *typename,
-                         DeviceState *parent, const char *name)
-{
-    object_initialize(bus, size, typename);
-    qbus_realize(bus, parent, name);
-}
-
-BusState *qbus_create(const char *typename, DeviceState *parent, const char *name)
-{
-    BusState *bus;
-
-    bus = BUS(object_new(typename));
-    if (!bus) {
-        hw_error("Can't create bus.");
-    }
-    qbus_realize(bus, parent, name);
-
-    return bus;
-}
-
 static char *bus_get_fw_dev_path(BusState *bus, DeviceState *dev)
 {
     BusClass *bc = BUS_GET_CLASS(bus);
@@ -771,6 +662,13 @@
     return d;
 }
 
+char *qdev_get_own_fw_dev_path_from_handler(BusState *bus, DeviceState *dev)
+{
+    Object *obj = OBJECT(dev);
+
+    return fw_path_provider_try_get_dev_path(obj, bus, dev);
+}
+
 static int qdev_get_fw_dev_path_helper(DeviceState *dev, char *p, int size)
 {
     int l = 0;
@@ -826,8 +724,9 @@
  * Legacy property handling
  */
 
-static void qdev_get_legacy_property(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void qdev_get_legacy_property(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
@@ -836,17 +735,24 @@
     char *ptr = buffer;
 
     prop->info->print(dev, prop, buffer, sizeof(buffer));
-    visit_type_str(v, &ptr, name, errp);
+    visit_type_str(v, name, &ptr, errp);
 }
 
 /**
- * @qdev_add_legacy_property - adds a legacy property
+ * qdev_property_add_legacy:
+ * @dev: Device to add the property to.
+ * @prop: The qdev property definition.
+ * @errp: location to store error information.
  *
- * Do not use this is new code!  Properties added through this interface will
- * be given names and types in the "legacy" namespace.
+ * Add a legacy QOM property to @dev for qdev property @prop.
+ * On error, store error in @errp.
  *
- * Legacy properties are string versions of other OOM properties.  The format
- * of the string depends on the property type.
+ * Legacy properties are string versions of QOM properties.  The format of
+ * the string depends on the property type.  Legacy properties are only
+ * needed for "info qtree".
+ *
+ * Do not use this is new code!  QOM Properties added through this interface
+ * will be given names in the "legacy" namespace.
  */
 static void qdev_property_add_legacy(DeviceState *dev, Property *prop,
                                      Error **errp)
@@ -869,10 +775,14 @@
 }
 
 /**
- * @qdev_property_add_static - add a @Property to a device.
+ * qdev_property_add_static:
+ * @dev: Device to add the property to.
+ * @prop: The qdev property definition.
+ * @errp: location to store error information.
  *
- * Static properties access data in a struct.  The actual type of the
- * property and the field depends on the property type.
+ * Add a static QOM property to @dev for qdev property @prop.
+ * On error, store error in @errp.  Static properties access data in a struct.
+ * The type of the QOM property is derived from prop->info.
  */
 void qdev_property_add_static(DeviceState *dev, Property *prop,
                               Error **errp)
@@ -941,7 +851,12 @@
 static int qdev_add_hotpluggable_device(Object *obj, void *opaque)
 {
     GSList **list = opaque;
-    DeviceState *dev = DEVICE(obj);
+    DeviceState *dev = (DeviceState *)object_dynamic_cast(OBJECT(obj),
+                                                          TYPE_DEVICE);
+
+    if (dev == NULL) {
+        return 0;
+    }
 
     if (dev->realized && object_property_get_bool(obj, "hotpluggable", NULL)) {
         *list = g_slist_append(*list, dev);
@@ -972,23 +887,33 @@
     HotplugHandler *hotplug_ctrl;
     BusState *bus;
     Error *local_err = NULL;
+    bool unattached_parent = false;
+    static int unattached_count;
 
     if (dev->hotplugged && !dc->hotpluggable) {
-        error_set(errp, QERR_DEVICE_NO_HOTPLUG, object_get_typename(obj));
+        error_setg(errp, QERR_DEVICE_NO_HOTPLUG, object_get_typename(obj));
         return;
     }
 
     if (value && !dev->realized) {
         if (!obj->parent) {
-            static int unattached_count;
             gchar *name = g_strdup_printf("device[%d]", unattached_count++);
 
             object_property_add_child(container_get(qdev_get_machine(),
                                                     "/unattached"),
                                       name, obj, &error_abort);
+            unattached_parent = true;
             g_free(name);
         }
 
+        hotplug_ctrl = qdev_get_hotplug_handler(dev);
+        if (hotplug_ctrl) {
+            hotplug_handler_pre_plug(hotplug_ctrl, dev, &local_err);
+            if (local_err != NULL) {
+                goto fail;
+            }
+        }
+
         if (dc->realize) {
             dc->realize(dev, &local_err);
         }
@@ -997,7 +922,8 @@
             goto fail;
         }
 
-        hotplug_ctrl = qdev_get_hotplug_handler(dev);
+        DEVICE_LISTENER_CALL(realize, Forward, dev);
+
         if (hotplug_ctrl) {
             hotplug_handler_plug(hotplug_ctrl, dev, &local_err);
         }
@@ -1038,6 +964,7 @@
             dc->unrealize(dev, local_errp);
         }
         dev->pending_deleted_event = true;
+        DEVICE_LISTENER_CALL(unrealize, Reverse, dev);
     }
 
     if (local_err != NULL) {
@@ -1064,7 +991,10 @@
 
 fail:
     error_propagate(errp, local_err);
-    return;
+    if (unattached_parent) {
+        object_unparent(OBJECT(dev));
+        unattached_count--;
+    }
 }
 
 static bool device_get_hotpluggable(Object *obj, Error **errp)
@@ -1129,13 +1059,7 @@
 
 static void device_post_init(Object *obj)
 {
-    Error *err = NULL;
-    qdev_prop_set_globals(DEVICE(obj), &err);
-    if (err) {
-        qerror_report_err(err);
-        error_free(err);
-        exit(EXIT_FAILURE);
-    }
+    qdev_prop_set_globals(DEVICE(obj));
 }
 
 /* Unlink device from bus and free the structure.  */
@@ -1144,9 +1068,6 @@
     NamedGPIOList *ngl, *next;
 
     DeviceState *dev = DEVICE(obj);
-    if (dev->opts) {
-        qemu_opts_del(dev->opts);
-    }
 
     QLIST_FOREACH_SAFE(ngl, &dev->gpios, node, next) {
         QLIST_REMOVE(ngl, node);
@@ -1194,6 +1115,9 @@
         qapi_event_send_device_deleted(!!dev->id, dev->id, path, &error_abort);
         g_free(path);
     }
+
+    qemu_opts_del(dev->opts);
+    dev->opts = NULL;
 }
 
 static void device_class_init(ObjectClass *class, void *data)
@@ -1246,55 +1170,8 @@
     .class_size = sizeof(DeviceClass),
 };
 
-static void qbus_initfn(Object *obj)
-{
-    BusState *bus = BUS(obj);
-
-    QTAILQ_INIT(&bus->children);
-    object_property_add_link(obj, QDEV_HOTPLUG_HANDLER_PROPERTY,
-                             TYPE_HOTPLUG_HANDLER,
-                             (Object **)&bus->hotplug_handler,
-                             object_property_allow_set_link,
-                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
-                             NULL);
-    object_property_add_bool(obj, "realized",
-                             bus_get_realized, bus_set_realized, NULL);
-}
-
-static char *default_bus_get_fw_dev_path(DeviceState *dev)
-{
-    return g_strdup(object_get_typename(OBJECT(dev)));
-}
-
-static void bus_class_init(ObjectClass *class, void *data)
-{
-    BusClass *bc = BUS_CLASS(class);
-
-    class->unparent = bus_unparent;
-    bc->get_fw_dev_path = default_bus_get_fw_dev_path;
-}
-
-static void qbus_finalize(Object *obj)
-{
-    BusState *bus = BUS(obj);
-
-    g_free((char *)bus->name);
-}
-
-static const TypeInfo bus_info = {
-    .name = TYPE_BUS,
-    .parent = TYPE_OBJECT,
-    .instance_size = sizeof(BusState),
-    .abstract = true,
-    .class_size = sizeof(BusClass),
-    .instance_init = qbus_initfn,
-    .instance_finalize = qbus_finalize,
-    .class_init = bus_class_init,
-};
-
 static void qdev_register_types(void)
 {
-    type_register_static(&bus_info);
     type_register_static(&device_type_info);
 }
 
diff --git a/hw/core/register.c b/hw/core/register.c
new file mode 100644
index 0000000..4bfbc50
--- /dev/null
+++ b/hw/core/register.c
@@ -0,0 +1,287 @@
+/*
+ * Register Definition API
+ *
+ * Copyright (c) 2016 Xilinx Inc.
+ * Copyright (c) 2013 Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/register.h"
+#include "hw/qdev.h"
+#include "qemu/log.h"
+
+static inline void register_write_val(RegisterInfo *reg, uint64_t val)
+{
+    g_assert(reg->data);
+
+    switch (reg->data_size) {
+    case 1:
+        *(uint8_t *)reg->data = val;
+        break;
+    case 2:
+        *(uint16_t *)reg->data = val;
+        break;
+    case 4:
+        *(uint32_t *)reg->data = val;
+        break;
+    case 8:
+        *(uint64_t *)reg->data = val;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static inline uint64_t register_read_val(RegisterInfo *reg)
+{
+    switch (reg->data_size) {
+    case 1:
+        return *(uint8_t *)reg->data;
+    case 2:
+        return *(uint16_t *)reg->data;
+    case 4:
+        return *(uint32_t *)reg->data;
+    case 8:
+        return *(uint64_t *)reg->data;
+    default:
+        g_assert_not_reached();
+    }
+    return 0; /* unreachable */
+}
+
+void register_write(RegisterInfo *reg, uint64_t val, uint64_t we,
+                    const char *prefix, bool debug)
+{
+    uint64_t old_val, new_val, test, no_w_mask;
+    const RegisterAccessInfo *ac;
+
+    assert(reg);
+
+    ac = reg->access;
+
+    if (!ac || !ac->name) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: write to undefined device state "
+                      "(written value: %#" PRIx64 ")\n", prefix, val);
+        return;
+    }
+
+    old_val = reg->data ? register_read_val(reg) : ac->reset;
+
+    test = (old_val ^ val) & ac->rsvd;
+    if (test) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: change of value in reserved bit"
+                      "fields: %#" PRIx64 ")\n", prefix, test);
+    }
+
+    test = val & ac->unimp;
+    if (test) {
+        qemu_log_mask(LOG_UNIMP,
+                      "%s:%s writing %#" PRIx64 " to unimplemented bits:" \
+                      " %#" PRIx64 "",
+                      prefix, reg->access->name, val, ac->unimp);
+    }
+
+    /* Create the no write mask based on the read only, write to clear and
+     * reserved bit masks.
+     */
+    no_w_mask = ac->ro | ac->w1c | ac->rsvd | ~we;
+    new_val = (val & ~no_w_mask) | (old_val & no_w_mask);
+    new_val &= ~(val & ac->w1c);
+
+    if (ac->pre_write) {
+        new_val = ac->pre_write(reg, new_val);
+    }
+
+    if (debug) {
+        qemu_log("%s:%s: write of value %#" PRIx64 "\n", prefix, ac->name,
+                 new_val);
+    }
+
+    register_write_val(reg, new_val);
+
+    if (ac->post_write) {
+        ac->post_write(reg, new_val);
+    }
+}
+
+uint64_t register_read(RegisterInfo *reg, uint64_t re, const char* prefix,
+                       bool debug)
+{
+    uint64_t ret;
+    const RegisterAccessInfo *ac;
+
+    assert(reg);
+
+    ac = reg->access;
+    if (!ac || !ac->name) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: read from undefined device state\n",
+                      prefix);
+        return 0;
+    }
+
+    ret = reg->data ? register_read_val(reg) : ac->reset;
+
+    register_write_val(reg, ret & ~(ac->cor & re));
+
+    /* Mask based on the read enable size */
+    ret &= re;
+
+    if (ac->post_read) {
+        ret = ac->post_read(reg, ret);
+    }
+
+    if (debug) {
+        qemu_log("%s:%s: read of value %#" PRIx64 "\n", prefix,
+                 ac->name, ret);
+    }
+
+    return ret;
+}
+
+void register_reset(RegisterInfo *reg)
+{
+    g_assert(reg);
+
+    if (!reg->data || !reg->access) {
+        return;
+    }
+
+    register_write_val(reg, reg->access->reset);
+}
+
+void register_init(RegisterInfo *reg)
+{
+    assert(reg);
+
+    if (!reg->data || !reg->access) {
+        return;
+    }
+
+    object_initialize((void *)reg, sizeof(*reg), TYPE_REGISTER);
+}
+
+void register_write_memory(void *opaque, hwaddr addr,
+                           uint64_t value, unsigned size)
+{
+    RegisterInfoArray *reg_array = opaque;
+    RegisterInfo *reg = NULL;
+    uint64_t we;
+    int i;
+
+    for (i = 0; i < reg_array->num_elements; i++) {
+        if (reg_array->r[i]->access->addr == addr) {
+            reg = reg_array->r[i];
+            break;
+        }
+    }
+
+    if (!reg) {
+        qemu_log_mask(LOG_GUEST_ERROR, "Write to unimplemented register at " \
+                      "address: %#" PRIx64 "\n", addr);
+        return;
+    }
+
+    /* Generate appropriate write enable mask */
+    if (reg->data_size < size) {
+        we = MAKE_64BIT_MASK(0, reg->data_size * 8);
+    } else {
+        we = MAKE_64BIT_MASK(0, size * 8);
+    }
+
+    register_write(reg, value, we, reg_array->prefix,
+                   reg_array->debug);
+}
+
+uint64_t register_read_memory(void *opaque, hwaddr addr,
+                              unsigned size)
+{
+    RegisterInfoArray *reg_array = opaque;
+    RegisterInfo *reg = NULL;
+    uint64_t read_val;
+    int i;
+
+    for (i = 0; i < reg_array->num_elements; i++) {
+        if (reg_array->r[i]->access->addr == addr) {
+            reg = reg_array->r[i];
+            break;
+        }
+    }
+
+    if (!reg) {
+        qemu_log_mask(LOG_GUEST_ERROR, "Read to unimplemented register at " \
+                      "address: %#" PRIx64 "\n", addr);
+        return 0;
+    }
+
+    read_val = register_read(reg, size * 8, reg_array->prefix,
+                             reg_array->debug);
+
+    return extract64(read_val, 0, size * 8);
+}
+
+RegisterInfoArray *register_init_block32(DeviceState *owner,
+                                         const RegisterAccessInfo *rae,
+                                         int num, RegisterInfo *ri,
+                                         uint32_t *data,
+                                         const MemoryRegionOps *ops,
+                                         bool debug_enabled,
+                                         uint64_t memory_size)
+{
+    const char *device_prefix = object_get_typename(OBJECT(owner));
+    RegisterInfoArray *r_array = g_new0(RegisterInfoArray, 1);
+    int i;
+
+    r_array->r = g_new0(RegisterInfo *, num);
+    r_array->num_elements = num;
+    r_array->debug = debug_enabled;
+    r_array->prefix = device_prefix;
+
+    for (i = 0; i < num; i++) {
+        int index = rae[i].addr / 4;
+        RegisterInfo *r = &ri[index];
+
+        *r = (RegisterInfo) {
+            .data = &data[index],
+            .data_size = sizeof(uint32_t),
+            .access = &rae[i],
+            .opaque = owner,
+        };
+        register_init(r);
+
+        r_array->r[i] = r;
+    }
+
+    memory_region_init_io(&r_array->mem, OBJECT(owner), ops, r_array,
+                          device_prefix, memory_size);
+
+    return r_array;
+}
+
+void register_finalize_block(RegisterInfoArray *r_array)
+{
+    object_unparent(OBJECT(&r_array->mem));
+    g_free(r_array->r);
+    g_free(r_array);
+}
+
+static const TypeInfo register_info = {
+    .name  = TYPE_REGISTER,
+    .parent = TYPE_DEVICE,
+};
+
+static void register_register_types(void)
+{
+    type_register_static(&register_info);
+}
+
+type_init(register_register_types)
diff --git a/hw/core/stream.c b/hw/core/stream.c
index e6a05a5..4439ecd 100644
--- a/hw/core/stream.c
+++ b/hw/core/stream.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/stream.h"
 
 size_t
diff --git a/hw/core/sysbus.c b/hw/core/sysbus.c
index 34b040d..838f4a7 100644
--- a/hw/core/sysbus.c
+++ b/hw/core/sysbus.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "monitor/monitor.h"
 #include "exec/address-spaces.h"
@@ -91,6 +92,8 @@
     ObjectProperty *r;
 
     r = object_property_find(OBJECT(dev), prop, NULL);
+    g_free(prop);
+
     return (r != NULL);
 }
 
@@ -107,7 +110,13 @@
 
 void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq)
 {
+    SysBusDeviceClass *sbd = SYS_BUS_DEVICE_GET_CLASS(dev);
+
     qdev_connect_gpio_out_named(DEVICE(dev), SYSBUS_DEVICE_GPIO_IRQ, n, irq);
+
+    if (sbd->connect_irq_notifier) {
+        sbd->connect_irq_notifier(dev, irq);
+    }
 }
 
 /* Check whether an MMIO region exists */
@@ -182,9 +191,9 @@
     return dev->mmio[n].memory;
 }
 
-void sysbus_init_ioports(SysBusDevice *dev, pio_addr_t ioport, pio_addr_t size)
+void sysbus_init_ioports(SysBusDevice *dev, uint32_t ioport, uint32_t size)
 {
-    pio_addr_t i;
+    uint32_t i;
 
     for (i = 0; i < size; i++) {
         assert(dev->num_pio < QDEV_MAX_PIO);
@@ -280,19 +289,26 @@
 static char *sysbus_get_fw_dev_path(DeviceState *dev)
 {
     SysBusDevice *s = SYS_BUS_DEVICE(dev);
-    char path[40];
-    int off;
-
-    off = snprintf(path, sizeof(path), "%s", qdev_fw_name(dev));
+    SysBusDeviceClass *sbc = SYS_BUS_DEVICE_GET_CLASS(s);
+    /* for the explicit unit address fallback case: */
+    char *addr, *fw_dev_path;
 
     if (s->num_mmio) {
-        snprintf(path + off, sizeof(path) - off, "@"TARGET_FMT_plx,
-                 s->mmio[0].addr);
-    } else if (s->num_pio) {
-        snprintf(path + off, sizeof(path) - off, "@i%04x", s->pio[0]);
+        return g_strdup_printf("%s@" TARGET_FMT_plx, qdev_fw_name(dev),
+                               s->mmio[0].addr);
     }
-
-    return g_strdup(path);
+    if (s->num_pio) {
+        return g_strdup_printf("%s@i%04x", qdev_fw_name(dev), s->pio[0]);
+    }
+    if (sbc->explicit_ofw_unit_address) {
+        addr = sbc->explicit_ofw_unit_address(s);
+        if (addr) {
+            fw_dev_path = g_strdup_printf("%s@%s", qdev_fw_name(dev), addr);
+            g_free(addr);
+            return fw_dev_path;
+        }
+    }
+    return g_strdup(qdev_fw_name(dev));
 }
 
 void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
diff --git a/hw/core/uboot_image.h b/hw/core/uboot_image.h
index 9fc2760..34c11a7 100644
--- a/hw/core/uboot_image.h
+++ b/hw/core/uboot_image.h
@@ -26,8 +26,8 @@
  ********************************************************************
  */
 
-#ifndef __UBOOT_IMAGE_H__
-#define __UBOOT_IMAGE_H__
+#ifndef UBOOT_IMAGE_H
+#define UBOOT_IMAGE_H
 
 /*
  * Operating System Codes
@@ -155,4 +155,4 @@
 } uboot_image_header_t;
 
 
-#endif	/* __IMAGE_H__ */
+#endif /* UBOOT_IMAGE_H */
diff --git a/hw/cpu/Makefile.objs b/hw/cpu/Makefile.objs
index 6381238..942a4bb 100644
--- a/hw/cpu/Makefile.objs
+++ b/hw/cpu/Makefile.objs
@@ -2,5 +2,5 @@
 obj-$(CONFIG_REALVIEW) += realview_mpcore.o
 obj-$(CONFIG_A9MPCORE) += a9mpcore.o
 obj-$(CONFIG_A15MPCORE) += a15mpcore.o
-obj-$(CONFIG_ICC_BUS) += icc_bus.o
+obj-y += core.o
 
diff --git a/hw/cpu/a15mpcore.c b/hw/cpu/a15mpcore.c
index acc419e..bc05152 100644
--- a/hw/cpu/a15mpcore.c
+++ b/hw/cpu/a15mpcore.c
@@ -18,8 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/a15mpcore.h"
 #include "sysemu/kvm.h"
+#include "kvm_arm.h"
 
 static void a15mp_priv_set_irq(void *opaque, int irq, int level)
 {
@@ -33,16 +36,11 @@
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     A15MPPrivState *s = A15MPCORE_PRIV(obj);
     DeviceState *gicdev;
-    const char *gictype = "arm_gic";
-
-    if (kvm_irqchip_in_kernel()) {
-        gictype = "kvm-arm-gic";
-    }
 
     memory_region_init(&s->container, obj, "a15mp-priv-container", 0x8000);
     sysbus_init_mmio(sbd, &s->container);
 
-    object_initialize(&s->gic, sizeof(s->gic), gictype);
+    object_initialize(&s->gic, sizeof(s->gic), gic_class_name());
     gicdev = DEVICE(&s->gic);
     qdev_set_parent_bus(gicdev, sysbus_get_default());
     qdev_prop_set_uint32(gicdev, "revision", 2);
@@ -56,10 +54,23 @@
     SysBusDevice *busdev;
     int i;
     Error *err = NULL;
+    bool has_el3;
+    Object *cpuobj;
 
     gicdev = DEVICE(&s->gic);
     qdev_prop_set_uint32(gicdev, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(gicdev, "num-irq", s->num_irq);
+
+    if (!kvm_irqchip_in_kernel()) {
+        /* Make the GIC's TZ support match the CPUs. We assume that
+         * either all the CPUs have TZ, or none do.
+         */
+        cpuobj = OBJECT(qemu_get_cpu(0));
+        has_el3 = object_property_find(cpuobj, "has_el3", NULL) &&
+            object_property_get_bool(cpuobj, "has_el3", &error_abort);
+        qdev_prop_set_bit(gicdev, "has-security-extensions", has_el3);
+    }
+
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
@@ -79,20 +90,27 @@
     for (i = 0; i < s->num_cpu; i++) {
         DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
         int ppibase = s->num_irq - 32 + i * 32;
-        /* physical timer; we wire it up to the non-secure timer's ID,
-         * since a real A15 always has TrustZone but QEMU doesn't.
+        int irq;
+        /* Mapping from the output timer irq lines from the CPU to the
+         * GIC PPI inputs used on the A15:
          */
-        qdev_connect_gpio_out(cpudev, 0,
-                              qdev_get_gpio_in(gicdev, ppibase + 30));
-        /* virtual timer */
-        qdev_connect_gpio_out(cpudev, 1,
-                              qdev_get_gpio_in(gicdev, ppibase + 27));
+        const int timer_irq[] = {
+            [GTIMER_PHYS] = 30,
+            [GTIMER_VIRT] = 27,
+            [GTIMER_HYP]  = 26,
+            [GTIMER_SEC]  = 29,
+        };
+        for (irq = 0; irq < ARRAY_SIZE(timer_irq); irq++) {
+            qdev_connect_gpio_out(cpudev, irq,
+                                  qdev_get_gpio_in(gicdev,
+                                                   ppibase + timer_irq[irq]));
+        }
     }
 
     /* Memory map (addresses are offsets from PERIPHBASE):
      *  0x0000-0x0fff -- reserved
      *  0x1000-0x1fff -- GIC Distributor
-     *  0x2000-0x2fff -- GIC CPU interface
+     *  0x2000-0x3fff -- GIC CPU interface
      *  0x4000-0x4fff -- GIC virtual interface control (not modelled)
      *  0x5000-0x5fff -- GIC virtual interface control (not modelled)
      *  0x6000-0x7fff -- GIC virtual CPU interface (not modelled)
diff --git a/hw/cpu/a9mpcore.c b/hw/cpu/a9mpcore.c
index c09358c..f17f292 100644
--- a/hw/cpu/a9mpcore.c
+++ b/hw/cpu/a9mpcore.c
@@ -8,7 +8,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/a9mpcore.h"
+#include "qom/cpu.h"
 
 static void a9mp_priv_set_irq(void *opaque, int irq, int level)
 {
@@ -49,6 +52,8 @@
                  *wdtbusdev;
     Error *err = NULL;
     int i;
+    bool has_el3;
+    Object *cpuobj;
 
     scudev = DEVICE(&s->scu);
     qdev_prop_set_uint32(scudev, "num-cpu", s->num_cpu);
@@ -62,6 +67,15 @@
     gicdev = DEVICE(&s->gic);
     qdev_prop_set_uint32(gicdev, "num-cpu", s->num_cpu);
     qdev_prop_set_uint32(gicdev, "num-irq", s->num_irq);
+
+    /* Make the GIC's TZ support match the CPUs. We assume that
+     * either all the CPUs have TZ, or none do.
+     */
+    cpuobj = OBJECT(qemu_get_cpu(0));
+    has_el3 = object_property_find(cpuobj, "has_el3", NULL) &&
+        object_property_get_bool(cpuobj, "has_el3", &error_abort);
+    qdev_prop_set_bit(gicdev, "has-security-extensions", has_el3);
+
     object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
     if (err != NULL) {
         error_propagate(errp, err);
diff --git a/hw/cpu/arm11mpcore.c b/hw/cpu/arm11mpcore.c
index 717d3e4..eb24465 100644
--- a/hw/cpu/arm11mpcore.c
+++ b/hw/cpu/arm11mpcore.c
@@ -7,6 +7,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/arm11mpcore.h"
 #include "hw/intc/realview_gic.h"
 
diff --git a/hw/cpu/core.c b/hw/cpu/core.c
new file mode 100644
index 0000000..eff90c1
--- /dev/null
+++ b/hw/cpu/core.c
@@ -0,0 +1,88 @@
+/*
+ * CPU core abstract device
+ *
+ * Copyright (C) 2016 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "hw/cpu/core.h"
+#include "qapi/visitor.h"
+#include "qapi/error.h"
+#include "sysemu/cpus.h"
+
+static void core_prop_get_core_id(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    int64_t value = core->core_id;
+
+    visit_type_int(v, name, &value, errp);
+}
+
+static void core_prop_set_core_id(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    Error *local_err = NULL;
+    int64_t value;
+
+    visit_type_int(v, name, &value, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    core->core_id = value;
+}
+
+static void core_prop_get_nr_threads(Object *obj, Visitor *v, const char *name,
+                                     void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    int64_t value = core->nr_threads;
+
+    visit_type_int(v, name, &value, errp);
+}
+
+static void core_prop_set_nr_threads(Object *obj, Visitor *v, const char *name,
+                                     void *opaque, Error **errp)
+{
+    CPUCore *core = CPU_CORE(obj);
+    Error *local_err = NULL;
+    int64_t value;
+
+    visit_type_int(v, name, &value, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    core->nr_threads = value;
+}
+
+static void cpu_core_instance_init(Object *obj)
+{
+    CPUCore *core = CPU_CORE(obj);
+
+    object_property_add(obj, "core-id", "int", core_prop_get_core_id,
+                        core_prop_set_core_id, NULL, NULL, NULL);
+    object_property_add(obj, "nr-threads", "int", core_prop_get_nr_threads,
+                        core_prop_set_nr_threads, NULL, NULL, NULL);
+    core->nr_threads = smp_threads;
+}
+
+static const TypeInfo cpu_core_type_info = {
+    .name = TYPE_CPU_CORE,
+    .parent = TYPE_DEVICE,
+    .abstract = true,
+    .instance_size = sizeof(CPUCore),
+    .instance_init = cpu_core_instance_init,
+};
+
+static void cpu_core_register_types(void)
+{
+    type_register_static(&cpu_core_type_info);
+}
+
+type_init(cpu_core_register_types)
diff --git a/hw/cpu/icc_bus.c b/hw/cpu/icc_bus.c
deleted file mode 100644
index 6646ea2..0000000
--- a/hw/cpu/icc_bus.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/* icc_bus.c
- * emulate x86 ICC (Interrupt Controller Communications) bus
- *
- * Copyright (c) 2013 Red Hat, Inc
- *
- * Authors:
- *     Igor Mammedov <imammedo@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-#include "hw/cpu/icc_bus.h"
-#include "hw/sysbus.h"
-
-/* icc-bridge implementation */
-
-static const TypeInfo icc_bus_info = {
-    .name = TYPE_ICC_BUS,
-    .parent = TYPE_BUS,
-    .instance_size = sizeof(ICCBus),
-};
-
-
-/* icc-device implementation */
-
-static void icc_device_realize(DeviceState *dev, Error **errp)
-{
-    ICCDeviceClass *idc = ICC_DEVICE_GET_CLASS(dev);
-
-    /* convert to QOM */
-    if (idc->realize) {
-        idc->realize(dev, errp);
-    }
-
-}
-
-static void icc_device_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    dc->realize = icc_device_realize;
-    dc->bus_type = TYPE_ICC_BUS;
-}
-
-static const TypeInfo icc_device_info = {
-    .name = TYPE_ICC_DEVICE,
-    .parent = TYPE_DEVICE,
-    .abstract = true,
-    .instance_size = sizeof(ICCDevice),
-    .class_size = sizeof(ICCDeviceClass),
-    .class_init = icc_device_class_init,
-};
-
-
-/*  icc-bridge implementation */
-
-typedef struct ICCBridgeState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    ICCBus icc_bus;
-    MemoryRegion apic_container;
-} ICCBridgeState;
-
-#define ICC_BRIDGE(obj) OBJECT_CHECK(ICCBridgeState, (obj), TYPE_ICC_BRIDGE)
-
-static void icc_bridge_init(Object *obj)
-{
-    ICCBridgeState *s = ICC_BRIDGE(obj);
-    SysBusDevice *sb = SYS_BUS_DEVICE(obj);
-
-    qbus_create_inplace(&s->icc_bus, sizeof(s->icc_bus), TYPE_ICC_BUS,
-                        DEVICE(s), "icc");
-
-    /* Do not change order of registering regions,
-     * APIC must be first registered region, board maps it by 0 index
-     */
-    memory_region_init(&s->apic_container, obj, "icc-apic-container",
-                       APIC_SPACE_SIZE);
-    sysbus_init_mmio(sb, &s->apic_container);
-    s->icc_bus.apic_address_space = &s->apic_container;
-}
-
-static void icc_bridge_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-}
-
-static const TypeInfo icc_bridge_info = {
-    .name  = TYPE_ICC_BRIDGE,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_init  = icc_bridge_init,
-    .instance_size  = sizeof(ICCBridgeState),
-    .class_init = icc_bridge_class_init,
-};
-
-
-static void icc_bus_register_types(void)
-{
-    type_register_static(&icc_bus_info);
-    type_register_static(&icc_device_info);
-    type_register_static(&icc_bridge_info);
-}
-
-type_init(icc_bus_register_types)
diff --git a/hw/cpu/realview_mpcore.c b/hw/cpu/realview_mpcore.c
index c39a2da..39d4ebe 100644
--- a/hw/cpu/realview_mpcore.c
+++ b/hw/cpu/realview_mpcore.c
@@ -8,6 +8,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/cpu/arm11mpcore.h"
 #include "hw/intc/realview_gic.h"
 
diff --git a/hw/cris/axis_dev88.c b/hw/cris/axis_dev88.c
index 0479196..60df887 100644
--- a/hw/cris/axis_dev88.c
+++ b/hw/cris/axis_dev88.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/block/flash.h"
@@ -33,6 +37,7 @@
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
+#include "sysemu/sysemu.h"
 
 #define D(x)
 #define DNAND(x)
@@ -138,7 +143,7 @@
                     s->count = 16;
 
                     if ((s->regs[0] & 0xff) == 0) {
-                        /* 25 degrees celcius.  */
+                        /* 25 degrees celsius.  */
                         s->shiftreg = 0x0b9f;
                     } else if ((s->regs[0] & 0xff) == 0xff) {
                         /* Sensor ID, 0x8100 LM70.  */
@@ -270,15 +275,14 @@
     env = &cpu->env;
 
     /* allocate RAM */
-    memory_region_init_ram(phys_ram, NULL, "axisdev88.ram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(phys_ram);
+    memory_region_allocate_system_memory(phys_ram, NULL, "axisdev88.ram",
+                                         ram_size);
     memory_region_add_subregion(address_space_mem, 0x40000000, phys_ram);
 
     /* The ETRAX-FS has 128Kb on chip ram, the docs refer to it as the 
        internal memory.  */
     memory_region_init_ram(phys_intmem, NULL, "axisdev88.chipram", INTMEM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_intmem);
     memory_region_add_subregion(address_space_mem, 0x38000000, phys_intmem);
 
@@ -338,8 +342,7 @@
     sysbus_create_varargs("etraxfs,timer", 0x3005e000, irq[0x1b], nmi[1], NULL);
 
     for (i = 0; i < 4; i++) {
-        sysbus_create_simple("etraxfs,serial", 0x30026000 + i * 0x2000,
-                             irq[0x14 + i]);
+        etraxfs_ser_create(0x30026000 + i * 0x2000, irq[0x14 + i], serial_hds[i]);
     }
 
     if (kernel_filename) {
@@ -352,16 +355,11 @@
     }
 }
 
-static QEMUMachine axisdev88_machine = {
-    .name = "axis-dev88",
-    .desc = "AXIS devboard 88",
-    .init = axisdev88_init,
-    .is_default = 1,
-};
-
-static void axisdev88_machine_init(void)
+static void axisdev88_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&axisdev88_machine);
+    mc->desc = "AXIS devboard 88";
+    mc->init = axisdev88_init;
+    mc->is_default = 1;
 }
 
-machine_init(axisdev88_machine_init);
+DEFINE_MACHINE("axis-dev88", axisdev88_machine_init)
diff --git a/hw/cris/boot.c b/hw/cris/boot.c
index 622f353..f896ed7 100644
--- a/hw/cris/boot.c
+++ b/hw/cris/boot.c
@@ -22,10 +22,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "boot.h"
+#include "qemu/cutils.h"
 
 static void main_cpu_reset(void *opaque)
 {
@@ -72,7 +76,7 @@
     /* Boots a kernel elf binary, os/linux-2.6/vmlinux from the axis 
        devboard SDK.  */
     image_size = load_elf(li->image_filename, translate_kernel_address, NULL,
-                          &entry, NULL, &high, 0, ELF_MACHINE, 0);
+                          &entry, NULL, &high, 0, EM_CRIS, 0, 0);
     li->entry = entry;
     if (image_size < 0) {
         /* Takes a kimage from the axis devboard SDK.  */
diff --git a/hw/cris/boot.h b/hw/cris/boot.h
index c4d3fa6..218854e 100644
--- a/hw/cris/boot.h
+++ b/hw/cris/boot.h
@@ -1,5 +1,5 @@
-#ifndef _CRIS_BOOT_H
-#define HW_CRIS_BOOT_H 1
+#ifndef HW_CRIS_BOOT_H
+#define HW_CRIS_BOOT_H
 
 struct cris_load_info
 {
diff --git a/hw/display/Makefile.objs b/hw/display/Makefile.objs
index 2456da6..aeff3fd 100644
--- a/hw/display/Makefile.objs
+++ b/hw/display/Makefile.objs
@@ -14,19 +14,21 @@
 
 common-obj-$(CONFIG_BLIZZARD) += blizzard.o
 common-obj-$(CONFIG_EXYNOS4) += exynos4210_fimd.o
-common-obj-$(CONFIG_FRAMEBUFFER) += framebuffer.o
+common-obj-$(call lor,$(CONFIG_FRAMEBUFFER),$(CONFIG_GOLDFISH)) += framebuffer.o
 common-obj-$(CONFIG_GOLDFISH) += goldfish_fb.o
 common-obj-$(CONFIG_MILKYMIST) += milkymist-vgafb.o
 common-obj-$(CONFIG_ZAURUS) += tc6393xb.o
 
 ifeq ($(CONFIG_MILKYMIST_TMU2),y)
 common-obj-y += milkymist-tmu2.o
-libs_softmmu += $(GLX_LIBS)
+milkymist-tmu2.o-cflags := $(OPENGL_CFLAGS)
+milkymist-tmu2.o-libs += $(OPENGL_LIBS)
 endif
 
 obj-$(CONFIG_OMAP) += omap_dss.o
 obj-$(CONFIG_OMAP) += omap_lcdc.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_lcd.o
+obj-$(CONFIG_RASPI) += bcm2835_fb.o
 obj-$(CONFIG_SM501) += sm501.o
 obj-$(CONFIG_TCX) += tcx.o
 obj-$(CONFIG_CG3) += cg3.o
@@ -34,3 +36,13 @@
 obj-$(CONFIG_VGA) += vga.o
 
 common-obj-$(CONFIG_QXL) += qxl.o qxl-logger.o qxl-render.o
+
+obj-$(CONFIG_VIRTIO) += virtio-gpu.o virtio-gpu-3d.o
+obj-$(CONFIG_VIRTIO_PCI) += virtio-gpu-pci.o
+obj-$(CONFIG_VIRTIO_VGA) += virtio-vga.o
+virtio-gpu.o-cflags := $(VIRGL_CFLAGS)
+virtio-gpu.o-libs += $(VIRGL_LIBS)
+virtio-gpu-3d.o-cflags := $(VIRGL_CFLAGS)
+virtio-gpu-3d.o-libs += $(VIRGL_LIBS)
+obj-$(CONFIG_DPCD) += dpcd.o
+obj-$(CONFIG_XLNX_ZYNQMP) += xlnx_dp.o
diff --git a/hw/display/ads7846.c b/hw/display/ads7846.c
index 3f35369..166edad 100644
--- a/hw/display/ads7846.c
+++ b/hw/display/ads7846.c
@@ -10,7 +10,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 #include "ui/console.h"
 
 typedef struct {
@@ -132,7 +133,7 @@
     }
 };
 
-static int ads7846_init(SSISlave *d)
+static void ads7846_realize(SSISlave *d, Error **errp)
 {
     DeviceState *dev = DEVICE(d);
     ADS7846State *s = FROM_SSI_SLAVE(ADS7846State, d);
@@ -151,14 +152,13 @@
     ads7846_int_update(s);
 
     vmstate_register(NULL, -1, &vmstate_ads7846, s);
-    return 0;
 }
 
 static void ads7846_class_init(ObjectClass *klass, void *data)
 {
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = ads7846_init;
+    k->realize = ads7846_realize;
     k->transfer = ads7846_transfer;
 }
 
diff --git a/hw/display/bcm2835_fb.c b/hw/display/bcm2835_fb.c
new file mode 100644
index 0000000..7eab927
--- /dev/null
+++ b/hw/display/bcm2835_fb.c
@@ -0,0 +1,426 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Refactoring for Pi2 Copyright (c) 2015, Microsoft. Written by Andrew Baumann.
+ * This code is licensed under the GNU GPLv2 and later.
+ *
+ * Heavily based on milkymist-vgafb.c, copyright terms below:
+ *  QEMU model of the Milkymist VGA framebuffer.
+ *
+ *  Copyright (c) 2010-2012 Michael Walle <michael@walle.cc>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/display/bcm2835_fb.h"
+#include "hw/display/framebuffer.h"
+#include "ui/pixel_ops.h"
+#include "hw/misc/bcm2835_mbox_defs.h"
+#include "qemu/log.h"
+
+#define DEFAULT_VCRAM_SIZE 0x4000000
+#define BCM2835_FB_OFFSET  0x00100000
+
+static void fb_invalidate_display(void *opaque)
+{
+    BCM2835FBState *s = BCM2835_FB(opaque);
+
+    s->invalidate = true;
+}
+
+static void draw_line_src16(void *opaque, uint8_t *dst, const uint8_t *src,
+                            int width, int deststep)
+{
+    BCM2835FBState *s = opaque;
+    uint16_t rgb565;
+    uint32_t rgb888;
+    uint8_t r, g, b;
+    DisplaySurface *surface = qemu_console_surface(s->con);
+    int bpp = surface_bits_per_pixel(surface);
+
+    while (width--) {
+        switch (s->bpp) {
+        case 8:
+            /* lookup palette starting at video ram base
+             * TODO: cache translation, rather than doing this each time!
+             */
+            rgb888 = ldl_le_phys(&s->dma_as, s->vcram_base + (*src << 2));
+            r = (rgb888 >> 0) & 0xff;
+            g = (rgb888 >> 8) & 0xff;
+            b = (rgb888 >> 16) & 0xff;
+            src++;
+            break;
+        case 16:
+            rgb565 = lduw_le_p(src);
+            r = ((rgb565 >> 11) & 0x1f) << 3;
+            g = ((rgb565 >>  5) & 0x3f) << 2;
+            b = ((rgb565 >>  0) & 0x1f) << 3;
+            src += 2;
+            break;
+        case 24:
+            rgb888 = ldl_le_p(src);
+            r = (rgb888 >> 0) & 0xff;
+            g = (rgb888 >> 8) & 0xff;
+            b = (rgb888 >> 16) & 0xff;
+            src += 3;
+            break;
+        case 32:
+            rgb888 = ldl_le_p(src);
+            r = (rgb888 >> 0) & 0xff;
+            g = (rgb888 >> 8) & 0xff;
+            b = (rgb888 >> 16) & 0xff;
+            src += 4;
+            break;
+        default:
+            r = 0;
+            g = 0;
+            b = 0;
+            break;
+        }
+
+        if (s->pixo == 0) {
+            /* swap to BGR pixel format */
+            uint8_t tmp = r;
+            r = b;
+            b = tmp;
+        }
+
+        switch (bpp) {
+        case 8:
+            *dst++ = rgb_to_pixel8(r, g, b);
+            break;
+        case 15:
+            *(uint16_t *)dst = rgb_to_pixel15(r, g, b);
+            dst += 2;
+            break;
+        case 16:
+            *(uint16_t *)dst = rgb_to_pixel16(r, g, b);
+            dst += 2;
+            break;
+        case 24:
+            rgb888 = rgb_to_pixel24(r, g, b);
+            *dst++ = rgb888 & 0xff;
+            *dst++ = (rgb888 >> 8) & 0xff;
+            *dst++ = (rgb888 >> 16) & 0xff;
+            break;
+        case 32:
+            *(uint32_t *)dst = rgb_to_pixel32(r, g, b);
+            dst += 4;
+            break;
+        default:
+            return;
+        }
+    }
+}
+
+static void fb_update_display(void *opaque)
+{
+    BCM2835FBState *s = opaque;
+    DisplaySurface *surface = qemu_console_surface(s->con);
+    int first = 0;
+    int last = 0;
+    int src_width = 0;
+    int dest_width = 0;
+
+    if (s->lock || !s->xres) {
+        return;
+    }
+
+    src_width = s->xres * (s->bpp >> 3);
+    dest_width = s->xres;
+
+    switch (surface_bits_per_pixel(surface)) {
+    case 0:
+        return;
+    case 8:
+        break;
+    case 15:
+        dest_width *= 2;
+        break;
+    case 16:
+        dest_width *= 2;
+        break;
+    case 24:
+        dest_width *= 3;
+        break;
+    case 32:
+        dest_width *= 4;
+        break;
+    default:
+        hw_error("bcm2835_fb: bad color depth\n");
+        break;
+    }
+
+    if (s->invalidate) {
+        framebuffer_update_memory_section(&s->fbsection, s->dma_mr, s->base,
+                                          s->yres, src_width);
+    }
+
+    framebuffer_update_display(surface, &s->fbsection, s->xres, s->yres,
+                               src_width, dest_width, 0, s->invalidate,
+                               draw_line_src16, s, &first, &last);
+
+    if (first >= 0) {
+        dpy_gfx_update(s->con, 0, first, s->xres, last - first + 1);
+    }
+
+    s->invalidate = false;
+}
+
+static void bcm2835_fb_mbox_push(BCM2835FBState *s, uint32_t value)
+{
+    value &= ~0xf;
+
+    s->lock = true;
+
+    s->xres = ldl_le_phys(&s->dma_as, value);
+    s->yres = ldl_le_phys(&s->dma_as, value + 4);
+    s->xres_virtual = ldl_le_phys(&s->dma_as, value + 8);
+    s->yres_virtual = ldl_le_phys(&s->dma_as, value + 12);
+    s->bpp = ldl_le_phys(&s->dma_as, value + 20);
+    s->xoffset = ldl_le_phys(&s->dma_as, value + 24);
+    s->yoffset = ldl_le_phys(&s->dma_as, value + 28);
+
+    s->base = s->vcram_base | (value & 0xc0000000);
+    s->base += BCM2835_FB_OFFSET;
+
+    /* TODO - Manage properly virtual resolution */
+
+    s->pitch = s->xres * (s->bpp >> 3);
+    s->size = s->yres * s->pitch;
+
+    stl_le_phys(&s->dma_as, value + 16, s->pitch);
+    stl_le_phys(&s->dma_as, value + 32, s->base);
+    stl_le_phys(&s->dma_as, value + 36, s->size);
+
+    s->invalidate = true;
+    qemu_console_resize(s->con, s->xres, s->yres);
+    s->lock = false;
+}
+
+void bcm2835_fb_reconfigure(BCM2835FBState *s, uint32_t *xres, uint32_t *yres,
+                            uint32_t *xoffset, uint32_t *yoffset, uint32_t *bpp,
+                            uint32_t *pixo, uint32_t *alpha)
+{
+    s->lock = true;
+
+    /* TODO: input validation! */
+    if (xres) {
+        s->xres = *xres;
+    }
+    if (yres) {
+        s->yres = *yres;
+    }
+    if (xoffset) {
+        s->xoffset = *xoffset;
+    }
+    if (yoffset) {
+        s->yoffset = *yoffset;
+    }
+    if (bpp) {
+        s->bpp = *bpp;
+    }
+    if (pixo) {
+        s->pixo = *pixo;
+    }
+    if (alpha) {
+        s->alpha = *alpha;
+    }
+
+    /* TODO - Manage properly virtual resolution */
+
+    s->pitch = s->xres * (s->bpp >> 3);
+    s->size = s->yres * s->pitch;
+
+    s->invalidate = true;
+    qemu_console_resize(s->con, s->xres, s->yres);
+    s->lock = false;
+}
+
+static uint64_t bcm2835_fb_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835FBState *s = opaque;
+    uint32_t res = 0;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        res = MBOX_CHAN_FB;
+        s->pending = false;
+        qemu_set_irq(s->mbox_irq, 0);
+        break;
+
+    case MBOX_AS_PENDING:
+        res = s->pending;
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    return res;
+}
+
+static void bcm2835_fb_write(void *opaque, hwaddr offset, uint64_t value,
+                             unsigned size)
+{
+    BCM2835FBState *s = opaque;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        /* bcm2835_mbox should check our pending status before pushing */
+        assert(!s->pending);
+        s->pending = true;
+        bcm2835_fb_mbox_push(s, value);
+        qemu_set_irq(s->mbox_irq, 1);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+}
+
+static const MemoryRegionOps bcm2835_fb_ops = {
+    .read = bcm2835_fb_read,
+    .write = bcm2835_fb_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_fb = {
+    .name = TYPE_BCM2835_FB,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(lock, BCM2835FBState),
+        VMSTATE_BOOL(invalidate, BCM2835FBState),
+        VMSTATE_BOOL(pending, BCM2835FBState),
+        VMSTATE_UINT32(xres, BCM2835FBState),
+        VMSTATE_UINT32(yres, BCM2835FBState),
+        VMSTATE_UINT32(xres_virtual, BCM2835FBState),
+        VMSTATE_UINT32(yres_virtual, BCM2835FBState),
+        VMSTATE_UINT32(xoffset, BCM2835FBState),
+        VMSTATE_UINT32(yoffset, BCM2835FBState),
+        VMSTATE_UINT32(bpp, BCM2835FBState),
+        VMSTATE_UINT32(base, BCM2835FBState),
+        VMSTATE_UINT32(pitch, BCM2835FBState),
+        VMSTATE_UINT32(size, BCM2835FBState),
+        VMSTATE_UINT32(pixo, BCM2835FBState),
+        VMSTATE_UINT32(alpha, BCM2835FBState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const GraphicHwOps vgafb_ops = {
+    .invalidate  = fb_invalidate_display,
+    .gfx_update  = fb_update_display,
+};
+
+static void bcm2835_fb_init(Object *obj)
+{
+    BCM2835FBState *s = BCM2835_FB(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2835_fb_ops, s, TYPE_BCM2835_FB,
+                          0x10);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->mbox_irq);
+}
+
+static void bcm2835_fb_reset(DeviceState *dev)
+{
+    BCM2835FBState *s = BCM2835_FB(dev);
+
+    s->pending = false;
+
+    s->xres_virtual = s->xres;
+    s->yres_virtual = s->yres;
+    s->xoffset = 0;
+    s->yoffset = 0;
+    s->base = s->vcram_base + BCM2835_FB_OFFSET;
+    s->pitch = s->xres * (s->bpp >> 3);
+    s->size = s->yres * s->pitch;
+
+    s->invalidate = true;
+    s->lock = false;
+}
+
+static void bcm2835_fb_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835FBState *s = BCM2835_FB(dev);
+    Error *err = NULL;
+    Object *obj;
+
+    if (s->vcram_base == 0) {
+        error_setg(errp, "%s: required vcram-base property not set", __func__);
+        return;
+    }
+
+    obj = object_property_get_link(OBJECT(dev), "dma-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required dma-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->dma_mr = MEMORY_REGION(obj);
+    address_space_init(&s->dma_as, s->dma_mr, NULL);
+
+    bcm2835_fb_reset(dev);
+
+    s->con = graphic_console_init(dev, 0, &vgafb_ops, s);
+    qemu_console_resize(s->con, s->xres, s->yres);
+}
+
+static Property bcm2835_fb_props[] = {
+    DEFINE_PROP_UINT32("vcram-base", BCM2835FBState, vcram_base, 0),/*required*/
+    DEFINE_PROP_UINT32("vcram-size", BCM2835FBState, vcram_size,
+                       DEFAULT_VCRAM_SIZE),
+    DEFINE_PROP_UINT32("xres", BCM2835FBState, xres, 640),
+    DEFINE_PROP_UINT32("yres", BCM2835FBState, yres, 480),
+    DEFINE_PROP_UINT32("bpp", BCM2835FBState, bpp, 16),
+    DEFINE_PROP_UINT32("pixo", BCM2835FBState, pixo, 1), /* 1=RGB, 0=BGR */
+    DEFINE_PROP_UINT32("alpha", BCM2835FBState, alpha, 2), /* alpha ignored */
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void bcm2835_fb_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = bcm2835_fb_props;
+    dc->realize = bcm2835_fb_realize;
+    dc->reset = bcm2835_fb_reset;
+    dc->vmsd = &vmstate_bcm2835_fb;
+}
+
+static TypeInfo bcm2835_fb_info = {
+    .name          = TYPE_BCM2835_FB,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835FBState),
+    .class_init    = bcm2835_fb_class_init,
+    .instance_init = bcm2835_fb_init,
+};
+
+static void bcm2835_fb_register_types(void)
+{
+    type_register_static(&bcm2835_fb_info);
+}
+
+type_init(bcm2835_fb_register_types)
diff --git a/hw/display/blizzard.c b/hw/display/blizzard.c
index 92b1fac..cbf07d1 100644
--- a/hw/display/blizzard.c
+++ b/hw/display/blizzard.c
@@ -18,10 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "hw/devices.h"
-#include "vga_int.h"
 #include "ui/pixel_ops.h"
 
 typedef void (*blizzard_fn_t)(uint8_t *, const uint8_t *, unsigned int);
@@ -925,16 +925,83 @@
     s->my[1] = 0;
 }
 
-#define DEPTH 8
-#include "blizzard_template.h"
-#define DEPTH 15
-#include "blizzard_template.h"
-#define DEPTH 16
-#include "blizzard_template.h"
-#define DEPTH 24
-#include "blizzard_template.h"
-#define DEPTH 32
-#include "blizzard_template.h"
+static void blizzard_draw_line16_32(uint32_t *dest,
+                                    const uint16_t *src, unsigned int width)
+{
+    uint16_t data;
+    unsigned int r, g, b;
+    const uint16_t *end = (const void *) src + width;
+    while (src < end) {
+        data = *src ++;
+        b = (data & 0x1f) << 3;
+        data >>= 5;
+        g = (data & 0x3f) << 2;
+        data >>= 6;
+        r = (data & 0x1f) << 3;
+        data >>= 5;
+        *dest++ = rgb_to_pixel32(r, g, b);
+    }
+}
+
+static void blizzard_draw_line24mode1_32(uint32_t *dest,
+                                         const uint8_t *src, unsigned int width)
+{
+    /* TODO: check if SDL 24-bit planes are not in the same format and
+     * if so, use memcpy */
+    unsigned int r[2], g[2], b[2];
+    const uint8_t *end = src + width;
+    while (src < end) {
+        g[0] = *src ++;
+        r[0] = *src ++;
+        r[1] = *src ++;
+        b[0] = *src ++;
+        *dest++ = rgb_to_pixel32(r[0], g[0], b[0]);
+        b[1] = *src ++;
+        g[1] = *src ++;
+        *dest++ = rgb_to_pixel32(r[1], g[1], b[1]);
+    }
+}
+
+static void blizzard_draw_line24mode2_32(uint32_t *dest,
+                                         const uint8_t *src, unsigned int width)
+{
+    unsigned int r, g, b;
+    const uint8_t *end = src + width;
+    while (src < end) {
+        r = *src ++;
+        src ++;
+        b = *src ++;
+        g = *src ++;
+        *dest++ = rgb_to_pixel32(r, g, b);
+    }
+}
+
+/* No rotation */
+static blizzard_fn_t blizzard_draw_fn_32[0x10] = {
+    NULL,
+    /* RGB 5:6:5*/
+    (blizzard_fn_t) blizzard_draw_line16_32,
+    /* RGB 6:6:6 mode 1 */
+    (blizzard_fn_t) blizzard_draw_line24mode1_32,
+    /* RGB 8:8:8 mode 1 */
+    (blizzard_fn_t) blizzard_draw_line24mode1_32,
+    NULL, NULL,
+    /* RGB 6:6:6 mode 2 */
+    (blizzard_fn_t) blizzard_draw_line24mode2_32,
+    /* RGB 8:8:8 mode 2 */
+    (blizzard_fn_t) blizzard_draw_line24mode2_32,
+    /* YUV 4:2:2 */
+    NULL,
+    /* YUV 4:2:0 */
+    NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL,
+};
+
+/* 90deg, 180deg and 270deg rotation */
+static blizzard_fn_t blizzard_draw_fn_r_32[0x10] = {
+    /* TODO */
+    [0 ... 0xf] = NULL,
+};
 
 static const GraphicHwOps blizzard_ops = {
     .invalidate  = blizzard_invalidate_display,
@@ -951,35 +1018,10 @@
     s->con = graphic_console_init(NULL, 0, &blizzard_ops, s);
     surface = qemu_console_surface(s->con);
 
-    switch (surface_bits_per_pixel(surface)) {
-    case 0:
-        s->line_fn_tab[0] = s->line_fn_tab[1] =
-                g_malloc0(sizeof(blizzard_fn_t) * 0x10);
-        break;
-    case 8:
-        s->line_fn_tab[0] = blizzard_draw_fn_8;
-        s->line_fn_tab[1] = blizzard_draw_fn_r_8;
-        break;
-    case 15:
-        s->line_fn_tab[0] = blizzard_draw_fn_15;
-        s->line_fn_tab[1] = blizzard_draw_fn_r_15;
-        break;
-    case 16:
-        s->line_fn_tab[0] = blizzard_draw_fn_16;
-        s->line_fn_tab[1] = blizzard_draw_fn_r_16;
-        break;
-    case 24:
-        s->line_fn_tab[0] = blizzard_draw_fn_24;
-        s->line_fn_tab[1] = blizzard_draw_fn_r_24;
-        break;
-    case 32:
-        s->line_fn_tab[0] = blizzard_draw_fn_32;
-        s->line_fn_tab[1] = blizzard_draw_fn_r_32;
-        break;
-    default:
-        fprintf(stderr, "%s: Bad color depth\n", __FUNCTION__);
-        exit(1);
-    }
+    assert(surface_bits_per_pixel(surface) == 32);
+
+    s->line_fn_tab[0] = blizzard_draw_fn_32;
+    s->line_fn_tab[1] = blizzard_draw_fn_r_32;
 
     blizzard_reset(s);
 
diff --git a/hw/display/blizzard_template.h b/hw/display/blizzard_template.h
deleted file mode 100644
index b7ef27c..0000000
--- a/hw/display/blizzard_template.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * QEMU Epson S1D13744/S1D13745 templates
- *
- * Copyright (C) 2008 Nokia Corporation
- * Written by Andrzej Zaborowski <andrew@openedhand.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 or
- * (at your option) version 3 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#define SKIP_PIXEL(to)         (to += deststep)
-#if DEPTH == 8
-# define PIXEL_TYPE            uint8_t
-# define COPY_PIXEL(to, from)  do { *to = from; SKIP_PIXEL(to); } while (0)
-# define COPY_PIXEL1(to, from) (*to++ = from)
-#elif DEPTH == 15 || DEPTH == 16
-# define PIXEL_TYPE            uint16_t
-# define COPY_PIXEL(to, from)  do { *to = from; SKIP_PIXEL(to); } while (0)
-# define COPY_PIXEL1(to, from) (*to++ = from)
-#elif DEPTH == 24
-# define PIXEL_TYPE            uint8_t
-# define COPY_PIXEL(to, from) \
-    do {                      \
-        to[0] = from;         \
-        to[1] = (from) >> 8;  \
-        to[2] = (from) >> 16; \
-        SKIP_PIXEL(to);       \
-    } while (0)
-
-# define COPY_PIXEL1(to, from) \
-    do {                       \
-        *to++ = from;          \
-        *to++ = (from) >> 8;   \
-        *to++ = (from) >> 16;  \
-    } while (0)
-#elif DEPTH == 32
-# define PIXEL_TYPE            uint32_t
-# define COPY_PIXEL(to, from)  do { *to = from; SKIP_PIXEL(to); } while (0)
-# define COPY_PIXEL1(to, from) (*to++ = from)
-#else
-# error unknown bit depth
-#endif
-
-#ifdef HOST_WORDS_BIGENDIAN
-# define SWAP_WORDS	1
-#endif
-
-static void glue(blizzard_draw_line16_, DEPTH)(PIXEL_TYPE *dest,
-                const uint16_t *src, unsigned int width)
-{
-#if !defined(SWAP_WORDS) && DEPTH == 16
-    memcpy(dest, src, width);
-#else
-    uint16_t data;
-    unsigned int r, g, b;
-    const uint16_t *end = (const void *) src + width;
-    while (src < end) {
-        data = *src ++;
-        b = (data & 0x1f) << 3;
-        data >>= 5;
-        g = (data & 0x3f) << 2;
-        data >>= 6;
-        r = (data & 0x1f) << 3;
-        data >>= 5;
-        COPY_PIXEL1(dest, glue(rgb_to_pixel, DEPTH)(r, g, b));
-    }
-#endif
-}
-
-static void glue(blizzard_draw_line24mode1_, DEPTH)(PIXEL_TYPE *dest,
-                const uint8_t *src, unsigned int width)
-{
-    /* TODO: check if SDL 24-bit planes are not in the same format and
-     * if so, use memcpy */
-    unsigned int r[2], g[2], b[2];
-    const uint8_t *end = src + width;
-    while (src < end) {
-        g[0] = *src ++;
-        r[0] = *src ++;
-        r[1] = *src ++;
-        b[0] = *src ++;
-        COPY_PIXEL1(dest, glue(rgb_to_pixel, DEPTH)(r[0], g[0], b[0]));
-        b[1] = *src ++;
-        g[1] = *src ++;
-        COPY_PIXEL1(dest, glue(rgb_to_pixel, DEPTH)(r[1], g[1], b[1]));
-    }
-}
-
-static void glue(blizzard_draw_line24mode2_, DEPTH)(PIXEL_TYPE *dest,
-                const uint8_t *src, unsigned int width)
-{
-    unsigned int r, g, b;
-    const uint8_t *end = src + width;
-    while (src < end) {
-        r = *src ++;
-        src ++;
-        b = *src ++;
-        g = *src ++;
-        COPY_PIXEL1(dest, glue(rgb_to_pixel, DEPTH)(r, g, b));
-    }
-}
-
-/* No rotation */
-static blizzard_fn_t glue(blizzard_draw_fn_, DEPTH)[0x10] = {
-    NULL,
-    /* RGB 5:6:5*/
-    (blizzard_fn_t) glue(blizzard_draw_line16_, DEPTH),
-    /* RGB 6:6:6 mode 1 */
-    (blizzard_fn_t) glue(blizzard_draw_line24mode1_, DEPTH),
-    /* RGB 8:8:8 mode 1 */
-    (blizzard_fn_t) glue(blizzard_draw_line24mode1_, DEPTH),
-    NULL, NULL,
-    /* RGB 6:6:6 mode 2 */
-    (blizzard_fn_t) glue(blizzard_draw_line24mode2_, DEPTH),
-    /* RGB 8:8:8 mode 2 */
-    (blizzard_fn_t) glue(blizzard_draw_line24mode2_, DEPTH),
-    /* YUV 4:2:2 */
-    NULL,
-    /* YUV 4:2:0 */
-    NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL,
-};
-
-/* 90deg, 180deg and 270deg rotation */
-static blizzard_fn_t glue(blizzard_draw_fn_r_, DEPTH)[0x10] = {
-    /* TODO */
-    [0 ... 0xf] = NULL,
-};
-
-#undef DEPTH
-#undef SKIP_PIXEL
-#undef COPY_PIXEL
-#undef COPY_PIXEL1
-#undef PIXEL_TYPE
-
-#undef SWAP_WORDS
diff --git a/hw/display/cg3.c b/hw/display/cg3.c
index 1e6ff2b..1174220 100644
--- a/hw/display/cg3.c
+++ b/hw/display/cg3.c
@@ -23,11 +23,15 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/error-report.h"
 #include "ui/console.h"
 #include "hw/sysbus.h"
 #include "hw/loader.h"
+#include "qemu/log.h"
 
 /* Change to 1 to enable debugging */
 #define DEBUG_CG3 0
@@ -106,6 +110,7 @@
     pix = memory_region_get_ram_ptr(&s->vram_mem);
     data = (uint32_t *)surface_data(surface);
 
+    memory_region_sync_dirty_bitmap(&s->vram_mem);
     for (y = 0; y < height; y++) {
         int update = s->full_update;
 
@@ -279,12 +284,12 @@
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     CG3State *s = CG3(obj);
 
-    memory_region_init_ram(&s->rom, NULL, "cg3.prom", FCODE_MAX_ROM_SIZE,
-                           &error_abort);
+    memory_region_init_ram(&s->rom, obj, "cg3.prom", FCODE_MAX_ROM_SIZE,
+                           &error_fatal);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
-    memory_region_init_io(&s->reg, NULL, &cg3_reg_ops, s, "cg3.reg",
+    memory_region_init_io(&s->reg, obj, &cg3_reg_ops, s, "cg3.reg",
                           CG3_REG_SIZE);
     sysbus_init_mmio(sbd, &s->reg);
 }
@@ -302,13 +307,15 @@
     if (fcode_filename) {
         ret = load_image_targphys(fcode_filename, s->prom_addr,
                                   FCODE_MAX_ROM_SIZE);
+        g_free(fcode_filename);
         if (ret < 0 || ret > FCODE_MAX_ROM_SIZE) {
             error_report("cg3: could not load prom '%s'", CG3_ROM_FILE);
         }
     }
 
     memory_region_init_ram(&s->vram_mem, NULL, "cg3.vram", s->vram_size,
-                           &error_abort);
+                           &error_fatal);
+    memory_region_set_log(&s->vram_mem, true, DIRTY_MEMORY_VGA);
     vmstate_register_ram_global(&s->vram_mem);
     sysbus_init_mmio(sbd, &s->vram_mem);
 
diff --git a/hw/display/cirrus_vga.c b/hw/display/cirrus_vga.c
index 2725264..3d712d5 100644
--- a/hw/display/cirrus_vga.c
+++ b/hw/display/cirrus_vga.c
@@ -26,6 +26,8 @@
  * Reference: Finn Thogersons' VGADOC4b
  *   available at http://home.worldonline.dk/~finth/
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "ui/console.h"
@@ -202,8 +204,6 @@
     uint32_t cirrus_bank_base[2];
     uint32_t cirrus_bank_limit[2];
     uint8_t cirrus_hidden_palette[48];
-    uint32_t hw_cursor_x;
-    uint32_t hw_cursor_y;
     int cirrus_blt_pixelwidth;
     int cirrus_blt_width;
     int cirrus_blt_height;
@@ -237,6 +237,10 @@
     CirrusVGAState cirrus_vga;
 } PCICirrusVGAState;
 
+#define TYPE_PCI_CIRRUS_VGA "cirrus-vga"
+#define PCI_CIRRUS_VGA(obj) \
+    OBJECT_CHECK(PCICirrusVGAState, (obj), TYPE_PCI_CIRRUS_VGA)
+
 #define TYPE_ISA_CIRRUS_VGA "isa-cirrus-vga"
 #define ISA_CIRRUS_VGA(obj) \
     OBJECT_CHECK(ISACirrusVGAState, (obj), TYPE_ISA_CIRRUS_VGA)
@@ -273,14 +277,14 @@
             + ((int64_t)s->cirrus_blt_height-1) * pitch;
         int32_t max = addr
             + s->cirrus_blt_width;
-        if (min < 0 || max >= s->vga.vram_size) {
+        if (min < 0 || max > s->vga.vram_size) {
             return true;
         }
     } else {
         int64_t max = addr
             + ((int64_t)s->cirrus_blt_height-1) * pitch
             + s->cirrus_blt_width;
-        if (max >= s->vga.vram_size) {
+        if (max > s->vga.vram_size) {
             return true;
         }
     }
@@ -1328,7 +1332,7 @@
     case 0xd0:
     case 0xf0:			// Graphics Cursor X
 	s->vga.sr[0x10] = val;
-	s->hw_cursor_x = (val << 3) | (s->vga.sr_index >> 5);
+        s->vga.hw_cursor_x = (val << 3) | (s->vga.sr_index >> 5);
 	break;
     case 0x11:
     case 0x31:
@@ -1339,7 +1343,7 @@
     case 0xd1:
     case 0xf1:			// Graphics Cursor Y
 	s->vga.sr[0x11] = val;
-	s->hw_cursor_y = (val << 3) | (s->vga.sr_index >> 5);
+        s->vga.hw_cursor_y = (val << 3) | (s->vga.sr_index >> 5);
 	break;
     case 0x07:			// Extended Sequencer Mode
     cirrus_update_memory_access(s);
@@ -1351,7 +1355,6 @@
     case 0x0d:			// VCLK 2
     case 0x0e:			// VCLK 3
     case 0x0f:			// DRAM Control
-    case 0x12:			// Graphics Cursor Attribute
     case 0x13:			// Graphics Cursor Pattern Address
     case 0x14:			// Scratch Register 2
     case 0x15:			// Scratch Register 3
@@ -1370,6 +1373,14 @@
 	       s->vga.sr_index, val);
 #endif
 	break;
+    case 0x12:			// Graphics Cursor Attribute
+	s->vga.sr[0x12] = val;
+        s->vga.force_shadow = !!(val & CIRRUS_CURSOR_SHOW);
+#ifdef DEBUG_CIRRUS
+        printf("cirrus: cursor ctl SR12=%02x (force shadow: %d)\n",
+               val, s->vga.force_shadow);
+#endif
+        break;
     case 0x17:			// Configuration Readback and Extended Control
 	s->vga.sr[s->vga.sr_index] = (s->vga.sr[s->vga.sr_index] & 0x38)
                                    | (val & 0xc7);
@@ -2188,14 +2199,14 @@
     }
     /* invalidate last cursor and new cursor if any change */
     if (s->last_hw_cursor_size != size ||
-        s->last_hw_cursor_x != s->hw_cursor_x ||
-        s->last_hw_cursor_y != s->hw_cursor_y) {
+        s->last_hw_cursor_x != s->vga.hw_cursor_x ||
+        s->last_hw_cursor_y != s->vga.hw_cursor_y) {
 
         invalidate_cursor1(s);
 
         s->last_hw_cursor_size = size;
-        s->last_hw_cursor_x = s->hw_cursor_x;
-        s->last_hw_cursor_y = s->hw_cursor_y;
+        s->last_hw_cursor_x = s->vga.hw_cursor_x;
+        s->last_hw_cursor_y = s->vga.hw_cursor_y;
         /* compute the real cursor min and max y */
         cirrus_cursor_compute_yrange(s);
         invalidate_cursor1(s);
@@ -2252,14 +2263,15 @@
     } else {
         h = 32;
     }
-    if (scr_y < s->hw_cursor_y ||
-        scr_y >= (s->hw_cursor_y + h))
+    if (scr_y < s->vga.hw_cursor_y ||
+        scr_y >= (s->vga.hw_cursor_y + h)) {
         return;
+    }
 
     src = s->vga.vram_ptr + s->real_vram_size - 16 * 1024;
     if (s->vga.sr[0x12] & CIRRUS_CURSOR_LARGE) {
         src += (s->vga.sr[0x13] & 0x3c) * 256;
-        src += (scr_y - s->hw_cursor_y) * 16;
+        src += (scr_y - s->vga.hw_cursor_y) * 16;
         poffset = 8;
         content = ((uint32_t *)src)[0] |
             ((uint32_t *)src)[1] |
@@ -2267,7 +2279,7 @@
             ((uint32_t *)src)[3];
     } else {
         src += (s->vga.sr[0x13] & 0x3f) * 256;
-        src += (scr_y - s->hw_cursor_y) * 4;
+        src += (scr_y - s->vga.hw_cursor_y) * 4;
 
 
         poffset = 128;
@@ -2279,10 +2291,10 @@
         return;
     w = h;
 
-    x1 = s->hw_cursor_x;
+    x1 = s->vga.hw_cursor_x;
     if (x1 >= s->vga.last_scr_width)
         return;
-    x2 = s->hw_cursor_x + w;
+    x2 = s->vga.hw_cursor_x + w;
     if (x2 > s->vga.last_scr_width)
         x2 = s->vga.last_scr_width;
     w = x2 - x1;
@@ -2771,8 +2783,8 @@
         VMSTATE_INT32(vga.bank_offset, CirrusVGAState),
         VMSTATE_UINT8(cirrus_hidden_dac_lockindex, CirrusVGAState),
         VMSTATE_UINT8(cirrus_hidden_dac_data, CirrusVGAState),
-        VMSTATE_UINT32(hw_cursor_x, CirrusVGAState),
-        VMSTATE_UINT32(hw_cursor_y, CirrusVGAState),
+        VMSTATE_UINT32(vga.hw_cursor_x, CirrusVGAState),
+        VMSTATE_UINT32(vga.hw_cursor_y, CirrusVGAState),
         /* XXX: we do not save the bitblt state - we assume we do not save
            the state when the blitter is active */
         VMSTATE_END_OF_LIST()
@@ -2901,7 +2913,7 @@
                                             bank, 1);
     }
     memory_region_add_subregion_overlap(system_memory,
-                                        isa_mem_base + 0x000a0000,
+                                        0x000a0000,
                                         &s->low_mem_container,
                                         1);
     memory_region_set_coalescing(&s->low_mem);
@@ -3000,9 +3012,9 @@
  *
  ***************************************/
 
-static int pci_cirrus_vga_initfn(PCIDevice *dev)
+static void pci_cirrus_vga_realize(PCIDevice *dev, Error **errp)
 {
-     PCICirrusVGAState *d = DO_UPCAST(PCICirrusVGAState, dev, dev);
+     PCICirrusVGAState *d = PCI_CIRRUS_VGA(dev);
      CirrusVGAState *s = &d->cirrus_vga;
      PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
      int16_t device_id = pc->device_id;
@@ -3011,9 +3023,9 @@
        Also accept 8 MB/16 MB for backward compatibility. */
      if (s->vga.vram_size_mb != 4 && s->vga.vram_size_mb != 8 &&
          s->vga.vram_size_mb != 16) {
-         error_report("Invalid cirrus_vga ram size '%u'",
-                      s->vga.vram_size_mb);
-         return -1;
+         error_setg(errp, "Invalid cirrus_vga ram size '%u'",
+                    s->vga.vram_size_mb);
+         return;
      }
      /* setup VGA */
      vga_common_init(&s->vga, OBJECT(dev), true);
@@ -3038,7 +3050,6 @@
      if (device_id == CIRRUS_ID_CLGD5446) {
          pci_register_bar(&d->dev, 1, 0, &s->cirrus_mmio_io);
      }
-     return 0;
 }
 
 static Property pci_vga_cirrus_properties[] = {
@@ -3052,7 +3063,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_cirrus_vga_initfn;
+    k->realize = pci_cirrus_vga_realize;
     k->romfile = VGABIOS_CIRRUS_FILENAME;
     k->vendor_id = PCI_VENDOR_ID_CIRRUS;
     k->device_id = CIRRUS_ID_CLGD5446;
@@ -3065,7 +3076,7 @@
 }
 
 static const TypeInfo cirrus_vga_info = {
-    .name          = "cirrus-vga",
+    .name          = TYPE_PCI_CIRRUS_VGA,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCICirrusVGAState),
     .class_init    = cirrus_vga_class_init,
diff --git a/hw/display/dpcd.c b/hw/display/dpcd.c
new file mode 100644
index 0000000..ce92ff6
--- /dev/null
+++ b/hw/display/dpcd.c
@@ -0,0 +1,173 @@
+/*
+ * dpcd.c
+ *
+ *  Copyright (C) 2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option)any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * This is a simple AUX slave which emulates a connected screen.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "hw/misc/auxbus.h"
+#include "hw/display/dpcd.h"
+
+#ifndef DEBUG_DPCD
+#define DEBUG_DPCD 0
+#endif
+
+#define DPRINTF(fmt, ...) do {                                                 \
+    if (DEBUG_DPCD) {                                                          \
+        qemu_log("dpcd: " fmt, ## __VA_ARGS__);                                \
+    }                                                                          \
+} while (0);
+
+#define DPCD_READABLE_AREA                      0x600
+
+struct DPCDState {
+    /*< private >*/
+    AUXSlave parent_obj;
+
+    /*< public >*/
+    /*
+     * The DCPD is 0x7FFFF length but read as 0 after offset 0x5FF.
+     */
+    uint8_t dpcd_info[DPCD_READABLE_AREA];
+
+    MemoryRegion iomem;
+};
+
+static uint64_t dpcd_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint8_t ret;
+    DPCDState *e = DPCD(opaque);
+
+    if (offset < DPCD_READABLE_AREA) {
+        ret = e->dpcd_info[offset];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "dpcd: Bad offset 0x%" HWADDR_PRIX "\n",
+                                       offset);
+        ret = 0;
+    }
+
+    DPRINTF("read 0x%" PRIX8 " @0x%" HWADDR_PRIX "\n", ret, offset);
+    return ret;
+}
+
+static void dpcd_write(void *opaque, hwaddr offset, uint64_t value,
+                       unsigned size)
+{
+    DPCDState *e = DPCD(opaque);
+
+    DPRINTF("write 0x%" PRIX8 " @0x%" HWADDR_PRIX "\n", (uint8_t)value, offset);
+
+    if (offset < DPCD_READABLE_AREA) {
+        e->dpcd_info[offset] = value;
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "dpcd: Bad offset 0x%" HWADDR_PRIX "\n",
+                                       offset);
+    }
+}
+
+static const MemoryRegionOps aux_ops = {
+    .read = dpcd_read,
+    .write = dpcd_write,
+    .valid = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+static void dpcd_reset(DeviceState *dev)
+{
+    DPCDState *s = DPCD(dev);
+
+    memset(&(s->dpcd_info), 0, sizeof(s->dpcd_info));
+
+    s->dpcd_info[DPCD_REVISION] = DPCD_REV_1_0;
+    s->dpcd_info[DPCD_MAX_LINK_RATE] = DPCD_5_4GBPS;
+    s->dpcd_info[DPCD_MAX_LANE_COUNT] = DPCD_FOUR_LANES;
+    s->dpcd_info[DPCD_RECEIVE_PORT0_CAP_0] = DPCD_EDID_PRESENT;
+    /* buffer size */
+    s->dpcd_info[DPCD_RECEIVE_PORT0_CAP_1] = 0xFF;
+
+    s->dpcd_info[DPCD_LANE0_1_STATUS] = DPCD_LANE0_CR_DONE
+                                      | DPCD_LANE0_CHANNEL_EQ_DONE
+                                      | DPCD_LANE0_SYMBOL_LOCKED
+                                      | DPCD_LANE1_CR_DONE
+                                      | DPCD_LANE1_CHANNEL_EQ_DONE
+                                      | DPCD_LANE1_SYMBOL_LOCKED;
+    s->dpcd_info[DPCD_LANE2_3_STATUS] = DPCD_LANE2_CR_DONE
+                                      | DPCD_LANE2_CHANNEL_EQ_DONE
+                                      | DPCD_LANE2_SYMBOL_LOCKED
+                                      | DPCD_LANE3_CR_DONE
+                                      | DPCD_LANE3_CHANNEL_EQ_DONE
+                                      | DPCD_LANE3_SYMBOL_LOCKED;
+
+    s->dpcd_info[DPCD_LANE_ALIGN_STATUS_UPDATED] = DPCD_INTERLANE_ALIGN_DONE;
+    s->dpcd_info[DPCD_SINK_STATUS] = DPCD_RECEIVE_PORT_0_STATUS;
+}
+
+static void dpcd_init(Object *obj)
+{
+    DPCDState *s = DPCD(obj);
+
+    memory_region_init_io(&s->iomem, obj, &aux_ops, s, TYPE_DPCD, 0x7FFFF);
+    aux_init_mmio(AUX_SLAVE(obj), &s->iomem);
+}
+
+static const VMStateDescription vmstate_dpcd = {
+    .name = TYPE_DPCD,
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8_ARRAY_V(dpcd_info, DPCDState, DPCD_READABLE_AREA, 0),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void dpcd_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->reset = dpcd_reset;
+    dc->vmsd = &vmstate_dpcd;
+}
+
+static const TypeInfo dpcd_info = {
+    .name          = TYPE_DPCD,
+    .parent        = TYPE_AUX_SLAVE,
+    .instance_size = sizeof(DPCDState),
+    .class_init    = dpcd_class_init,
+    .instance_init = dpcd_init,
+};
+
+static void dpcd_register_types(void)
+{
+    type_register_static(&dpcd_info);
+}
+
+type_init(dpcd_register_types)
diff --git a/hw/display/exynos4210_fimd.c b/hw/display/exynos4210_fimd.c
index 45c62af..e5be713 100644
--- a/hw/display/exynos4210_fimd.c
+++ b/hw/display/exynos4210_fimd.c
@@ -22,6 +22,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/sysbus.h"
 #include "ui/console.h"
@@ -337,7 +338,7 @@
     if (swap_ctl & FIMD_WINCON_SWAP_BITS) {
         res = 0;
         for (i = 0; i < 64; i++) {
-            if (x & (1ULL << (64 - i))) {
+            if (x & (1ULL << (63 - i))) {
                 res |= (1ULL << i);
             }
         }
@@ -1109,6 +1110,12 @@
     }
 }
 
+static void exynos4210_fimd_invalidate(void *opaque)
+{
+    Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
+    s->invalidate = true;
+}
+
 /* Updates specified window's MemorySection based on values of WINCON,
  * VIDOSDA, VIDOSDB, VIDWADDx and SHADOWCON registers */
 static void fimd_update_memory_section(Exynos4210fimdState *s, unsigned win)
@@ -1136,7 +1143,11 @@
     /* TODO: add .exit and unref the region there.  Not needed yet since sysbus
      * does not support hot-unplug.
      */
-    memory_region_unref(w->mem_section.mr);
+    if (w->mem_section.mr) {
+        memory_region_set_log(w->mem_section.mr, false, DIRTY_MEMORY_VGA);
+        memory_region_unref(w->mem_section.mr);
+    }
+
     w->mem_section = memory_region_find(sysbus_address_space(sbd),
                                         fb_start_addr, w->fb_len);
     assert(w->mem_section.mr);
@@ -1162,6 +1173,8 @@
         cpu_physical_memory_unmap(w->host_fb_addr, fb_mapped_len, 0, 0);
         goto error_return;
     }
+    memory_region_set_log(w->mem_section.mr, true, DIRTY_MEMORY_VGA);
+    exynos4210_fimd_invalidate(s);
     return;
 
 error_return:
@@ -1224,12 +1237,6 @@
     }
 }
 
-static void exynos4210_fimd_invalidate(void *opaque)
-{
-    Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
-    s->invalidate = true;
-}
-
 static void exynos4210_update_resolution(Exynos4210fimdState *s)
 {
     DisplaySurface *surface = qemu_console_surface(s->console);
@@ -1348,9 +1355,7 @@
         fimd_update_get_alpha(s, w);
     }
 
-    if (s->ifb != NULL) {
-        g_free(s->ifb);
-    }
+    g_free(s->ifb);
     s->ifb = NULL;
 
     exynos4210_fimd_invalidate(s);
@@ -1904,9 +1909,10 @@
     .gfx_update  = exynos4210_fimd_update,
 };
 
-static int exynos4210_fimd_init(SysBusDevice *dev)
+static void exynos4210_fimd_init(Object *obj)
 {
-    Exynos4210fimdState *s = EXYNOS4210_FIMD(dev);
+    Exynos4210fimdState *s = EXYNOS4210_FIMD(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     s->ifb = NULL;
 
@@ -1914,28 +1920,32 @@
     sysbus_init_irq(dev, &s->irq[1]);
     sysbus_init_irq(dev, &s->irq[2]);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_fimd_mmio_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_fimd_mmio_ops, s,
             "exynos4210.fimd", FIMD_REGS_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-    s->console = graphic_console_init(DEVICE(dev), 0, &exynos4210_fimd_ops, s);
+}
 
-    return 0;
+static void exynos4210_fimd_realize(DeviceState *dev, Error **errp)
+{
+    Exynos4210fimdState *s = EXYNOS4210_FIMD(dev);
+
+    s->console = graphic_console_init(dev, 0, &exynos4210_fimd_ops, s);
 }
 
 static void exynos4210_fimd_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
     dc->vmsd = &exynos4210_fimd_vmstate;
     dc->reset = exynos4210_fimd_reset;
-    k->init = exynos4210_fimd_init;
+    dc->realize = exynos4210_fimd_realize;
 }
 
 static const TypeInfo exynos4210_fimd_info = {
     .name = TYPE_EXYNOS4210_FIMD,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210fimdState),
+    .instance_init = exynos4210_fimd_init,
     .class_init = exynos4210_fimd_class_init,
 };
 
diff --git a/hw/display/framebuffer.c b/hw/display/framebuffer.c
index 8082f72..df51358 100644
--- a/hw/display/framebuffer.c
+++ b/hw/display/framebuffer.c
@@ -17,16 +17,45 @@
    - Remove all DisplayState knowledge from devices.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "framebuffer.h"
 
+void framebuffer_update_memory_section(
+    MemoryRegionSection *mem_section,
+    MemoryRegion *root,
+    hwaddr base,
+    unsigned rows,
+    unsigned src_width)
+{
+    hwaddr src_len = (hwaddr)rows * src_width;
+
+    if (mem_section->mr) {
+        memory_region_set_log(mem_section->mr, false, DIRTY_MEMORY_VGA);
+        memory_region_unref(mem_section->mr);
+        mem_section->mr = NULL;
+    }
+
+    *mem_section = memory_region_find(root, base, src_len);
+    if (!mem_section->mr) {
+        return;
+    }
+
+    if (int128_get64(mem_section->size) < src_len ||
+            !memory_region_is_ram(mem_section->mr)) {
+        memory_region_unref(mem_section->mr);
+        mem_section->mr = NULL;
+        return;
+    }
+
+    memory_region_set_log(mem_section->mr, true, DIRTY_MEMORY_VGA);
+}
+
 /* Render an image from a shared memory framebuffer.  */
-   
 void framebuffer_update_display(
     DisplaySurface *ds,
-    MemoryRegion *address_space,
-    hwaddr base,
+    MemoryRegionSection *mem_section,
     int cols, /* Width in pixels.  */
     int rows, /* Height in pixels.  */
     int src_width, /* Length of source line, in bytes.  */
@@ -41,52 +70,33 @@
     hwaddr src_len;
     uint8_t *dest;
     uint8_t *src;
-    uint8_t *src_base;
     int first, last = 0;
     int dirty;
     int i;
     ram_addr_t addr;
-    MemoryRegionSection mem_section;
     MemoryRegion *mem;
 
     i = *first_row;
     *first_row = -1;
     src_len = src_width * rows;
 
-    mem_section = memory_region_find(address_space, base, src_len);
-    mem = mem_section.mr;
-    if (int128_get64(mem_section.size) != src_len ||
-            !memory_region_is_ram(mem_section.mr)) {
-        goto out;
+    mem = mem_section->mr;
+    if (!mem) {
+        return;
     }
-    assert(mem);
-    assert(mem_section.offset_within_address_space == base);
-
     memory_region_sync_dirty_bitmap(mem);
-    src_base = cpu_physical_memory_map(base, &src_len, 0);
-    /* If we can't map the framebuffer then bail.  We could try harder,
-       but it's not really worth it as dirty flag tracking will probably
-       already have failed above.  */
-    if (!src_base)
-        goto out;
-    if (src_len != src_width * rows) {
-        cpu_physical_memory_unmap(src_base, src_len, 0, 0);
-        goto out;
-    }
-    src = src_base;
-    dest = surface_data(ds);
 
-    /* Make sure we start in the right place if we are striding
-       backwards */
+    addr = mem_section->offset_within_region;
+    src = memory_region_get_ram_ptr(mem) + addr;
+
+    dest = surface_data(ds);
     if (dest_col_pitch < 0) {
         dest -= dest_col_pitch * (cols - 1);
     }
     if (dest_row_pitch < 0) {
         dest -= dest_row_pitch * (rows - 1);
     }
-
     first = -1;
-    addr = mem_section.offset_within_region;
 
     addr += i * src_width;
     src += i * src_width;
@@ -105,14 +115,11 @@
         src += src_width;
         dest += dest_row_pitch;
     }
-    cpu_physical_memory_unmap(src_base, src_len, 0, 0);
     if (first < 0) {
-        goto out;
+        return;
     }
-    memory_region_reset_dirty(mem, mem_section.offset_within_region, src_len,
+    memory_region_reset_dirty(mem, mem_section->offset_within_region, src_len,
                               DIRTY_MEMORY_VGA);
     *first_row = first;
     *last_row = last;
-out:
-    memory_region_unref(mem);
 }
diff --git a/hw/display/framebuffer.h b/hw/display/framebuffer.h
index 6eae035..38fa0dc 100644
--- a/hw/display/framebuffer.h
+++ b/hw/display/framebuffer.h
@@ -7,10 +7,50 @@
 
 typedef void (*drawfn)(void *, uint8_t *, const uint8_t *, int, int);
 
+/* framebuffer_update_memory_section: Update framebuffer
+ * #MemoryRegionSection, for example if the framebuffer is switched to
+ * a different memory area.
+ *
+ * @mem_section: Output #MemoryRegionSection, to be passed to
+ * framebuffer_update_display().
+ * @root: #MemoryRegion within which the framebuffer lies
+ * @base: Base address of the framebuffer within @root.
+ * @rows: Height of the screen.
+ * @src_width: Number of bytes in framebuffer memory between two rows.
+ */
+void framebuffer_update_memory_section(
+    MemoryRegionSection *mem_section,
+    MemoryRegion *root,
+    hwaddr base,
+    unsigned rows,
+    unsigned src_width);
+
+/* framebuffer_update_display: Draw the framebuffer on a surface.
+ *
+ * @ds: #DisplaySurface to draw to.
+ * @mem_section: #MemoryRegionSection provided by
+ * framebuffer_update_memory_section().
+ * @cols: Width the screen.
+ * @rows: Height of the screen.
+ * @src_width: Number of bytes in framebuffer memory between two rows.
+ * @dest_row_pitch: Number of bytes in the surface data between two rows.
+ * Negative if the framebuffer is stored in the opposite order (e.g.
+ * bottom-to-top) compared to the framebuffer.
+ * @dest_col_pitch: Number of bytes in the surface data between two pixels.
+ * Negative if the framebuffer is stored in the opposite order (e.g.
+ * right-to-left) compared to the framebuffer.
+ * @invalidate: True if the function should redraw the whole screen
+ * without checking the DIRTY_MEMORY_VGA dirty bitmap.
+ * @fn: Drawing function to be called for each row that has to be drawn.
+ * @opaque: Opaque pointer passed to @fn.
+ * @first_row: Pointer to an integer, receives the number of the first row
+ * that was drawn (either the first dirty row, or 0 if @invalidate is true).
+ * @last_row: Pointer to an integer, receives the number of the last row that
+ * was drawn (either the last dirty row, or @rows-1 if @invalidate is true).
+ */
 void framebuffer_update_display(
     DisplaySurface *ds,
-    MemoryRegion *address_space,
-    hwaddr base,
+    MemoryRegionSection *mem_section,
     int cols,
     int rows,
     int src_width,
diff --git a/hw/display/g364fb.c b/hw/display/g364fb.c
index 46f7b41..70ef2c7 100644
--- a/hw/display/g364fb.c
+++ b/hw/display/g364fb.c
@@ -17,7 +17,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qemu/error-report.h"
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
 #include "trace.h"
@@ -260,6 +262,7 @@
         qemu_console_resize(s->con, s->width, s->height);
     }
 
+    memory_region_sync_dirty_bitmap(&s->mem_vram);
     if (s->ctla & CTLA_FORCE_BLANK) {
         g364fb_draw_blank(s);
     } else if (s->depth == 8) {
@@ -489,7 +492,7 @@
     memory_region_init_ram_ptr(&s->mem_vram, NULL, "vram",
                                s->vram_size, s->vram);
     vmstate_register_ram(&s->mem_vram, dev);
-    memory_region_set_coalescing(&s->mem_vram);
+    memory_region_set_log(&s->mem_vram, true, DIRTY_MEMORY_VGA);
 }
 
 #define TYPE_G364 "sysbus-g364"
diff --git a/hw/display/goldfish_fb.c b/hw/display/goldfish_fb.c
index be6be5a..3c0820a 100644
--- a/hw/display/goldfish_fb.c
+++ b/hw/display/goldfish_fb.c
@@ -9,12 +9,14 @@
 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ** GNU General Public License for more details.
 */
+#include "qemu/osdep.h"
 #include "framebuffer.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
 #include "trace.h"
+#include "exec/address-spaces.h"
 #include "hw/display/goldfish_fb.h"
 
 #include <inttypes.h>
@@ -107,6 +109,8 @@
     int      rotation;   /* 0, 1, 2 or 3 */
     int      dpi;
     int      format;
+
+    MemoryRegionSection fbsection;
 };
 
 #define  GOLDFISH_FB_SAVE_VERSION  3
@@ -269,8 +273,6 @@
     }
     else
     {
-        SysBusDevice *dev = SYS_BUS_DEVICE(opaque);
-        MemoryRegion *address_space = sysbus_address_space(dev);
         int src_width, src_height;
         int dest_row_pitch, dest_col_pitch;
         drawfn fn;
@@ -347,7 +349,12 @@
         // CPU time on OSX; saving on other platforms may differ.
         if (s_use_host_gpu) return;
 
-        framebuffer_update_display(ds, address_space, s->fb_base,
+        if (s->need_update) {
+            framebuffer_update_memory_section(
+                    &s->fbsection, get_system_memory(), s->fb_base,
+                    src_height, src_width * source_bytes_per_pixel);
+        }
+        framebuffer_update_display(ds, &s->fbsection,
                                    src_width, src_height,
                                    src_width * source_bytes_per_pixel,
                                    dest_row_pitch, dest_col_pitch,
diff --git a/hw/display/jazz_led.c b/hw/display/jazz_led.c
index 12b1707..b72fdb1 100644
--- a/hw/display/jazz_led.c
+++ b/hw/display/jazz_led.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
@@ -233,8 +234,10 @@
 
     /* TODO: draw the segments */
     snprintf(buf, 2, "%02hhx\n", s->segments);
-    console_write_ch(chardata++, 0x00200100 | buf[0]);
-    console_write_ch(chardata++, 0x00200100 | buf[1]);
+    console_write_ch(chardata++, ATTR2CHTYPE(buf[0], QEMU_COLOR_BLUE,
+                                             QEMU_COLOR_BLACK, 1));
+    console_write_ch(chardata++, ATTR2CHTYPE(buf[1], QEMU_COLOR_BLUE,
+                                             QEMU_COLOR_BLACK, 1));
 
     dpy_text_update(s->con, 0, 0, 2, 1);
 }
@@ -264,16 +267,20 @@
     .text_update = jazz_led_text_update,
 };
 
-static int jazz_led_init(SysBusDevice *dev)
+static void jazz_led_init(Object *obj)
+{
+    LedState *s = JAZZ_LED(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
+
+    memory_region_init_io(&s->iomem, obj, &led_ops, s, "led", 1);
+    sysbus_init_mmio(dev, &s->iomem);
+}
+
+static void jazz_led_realize(DeviceState *dev, Error **errp)
 {
     LedState *s = JAZZ_LED(dev);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &led_ops, s, "led", 1);
-    sysbus_init_mmio(dev, &s->iomem);
-
-    s->con = graphic_console_init(DEVICE(dev), 0, &jazz_led_ops, s);
-
-    return 0;
+    s->con = graphic_console_init(dev, 0, &jazz_led_ops, s);
 }
 
 static void jazz_led_reset(DeviceState *d)
@@ -288,18 +295,18 @@
 static void jazz_led_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = jazz_led_init;
     dc->desc = "Jazz LED display",
     dc->vmsd = &vmstate_jazz_led;
     dc->reset = jazz_led_reset;
+    dc->realize = jazz_led_realize;
 }
 
 static const TypeInfo jazz_led_info = {
     .name          = TYPE_JAZZ_LED,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LedState),
+    .instance_init = jazz_led_init,
     .class_init    = jazz_led_class_init,
 };
 
diff --git a/hw/display/milkymist-tmu2.c b/hw/display/milkymist-tmu2.c
index 3e1d0b9..9c00184 100644
--- a/hw/display/milkymist-tmu2.c
+++ b/hw/display/milkymist-tmu2.c
@@ -20,18 +20,20 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/tmu2.pdf
+ *   http://milkymist.walle.cc/socdoc/tmu2.pdf
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 
 #include <X11/Xlib.h>
-#include <GL/gl.h>
-#include <GL/glx.h>
+#include <epoxy/gl.h>
+#include <epoxy/glx.h>
 
 enum {
     R_CTL = 0,
@@ -442,21 +444,25 @@
     }
 }
 
-static int milkymist_tmu2_init(SysBusDevice *dev)
+static void milkymist_tmu2_init(Object *obj)
+{
+    MilkymistTMU2State *s = MILKYMIST_TMU2(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
+
+    sysbus_init_irq(dev, &s->irq);
+
+    memory_region_init_io(&s->regs_region, obj, &tmu2_mmio_ops, s,
+            "milkymist-tmu2", R_MAX * 4);
+    sysbus_init_mmio(dev, &s->regs_region);
+}
+
+static void milkymist_tmu2_realize(DeviceState *dev, Error **errp)
 {
     MilkymistTMU2State *s = MILKYMIST_TMU2(dev);
 
     if (tmu2_glx_init(s)) {
-        return 1;
+        error_setg(errp, "tmu2_glx_init failed");
     }
-
-    sysbus_init_irq(dev, &s->irq);
-
-    memory_region_init_io(&s->regs_region, OBJECT(s), &tmu2_mmio_ops, s,
-            "milkymist-tmu2", R_MAX * 4);
-    sysbus_init_mmio(dev, &s->regs_region);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_milkymist_tmu2 = {
@@ -472,9 +478,8 @@
 static void milkymist_tmu2_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = milkymist_tmu2_init;
+    dc->realize = milkymist_tmu2_realize;
     dc->reset = milkymist_tmu2_reset;
     dc->vmsd = &vmstate_milkymist_tmu2;
 }
@@ -483,6 +488,7 @@
     .name          = TYPE_MILKYMIST_TMU2,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistTMU2State),
+    .instance_init = milkymist_tmu2_init,
     .class_init    = milkymist_tmu2_class_init,
 };
 
diff --git a/hw/display/milkymist-vgafb.c b/hw/display/milkymist-vgafb.c
index 9b35e76..177fdac 100644
--- a/hw/display/milkymist-vgafb.c
+++ b/hw/display/milkymist-vgafb.c
@@ -19,9 +19,10 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/vgafb.pdf
+ *   http://milkymist.walle.cc/socdoc/vgafb.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -71,6 +72,7 @@
     SysBusDevice parent_obj;
 
     MemoryRegion regs_region;
+    MemoryRegionSection fbsection;
     QemuConsole *con;
 
     int invalidate;
@@ -91,6 +93,7 @@
     MilkymistVgafbState *s = opaque;
     SysBusDevice *sbd;
     DisplaySurface *surface = qemu_console_surface(s->con);
+    int src_width;
     int first = 0;
     int last = 0;
     drawfn fn;
@@ -129,11 +132,18 @@
         break;
     }
 
-    framebuffer_update_display(surface, sysbus_address_space(sbd),
-                               s->regs[R_BASEADDRESS] + s->fb_offset,
+    src_width = s->regs[R_HRES] * 2;
+    if (s->invalidate) {
+        framebuffer_update_memory_section(&s->fbsection,
+                                          sysbus_address_space(sbd),
+                                          s->regs[R_BASEADDRESS] + s->fb_offset,
+                                          s->regs[R_VRES], src_width);
+    }
+
+    framebuffer_update_display(surface, &s->fbsection,
                                s->regs[R_HRES],
                                s->regs[R_VRES],
-                               s->regs[R_HRES] * 2,
+                               src_width,
                                dest_width,
                                0,
                                s->invalidate,
@@ -282,17 +292,21 @@
     .gfx_update  = vgafb_update_display,
 };
 
-static int milkymist_vgafb_init(SysBusDevice *dev)
+static void milkymist_vgafb_init(Object *obj)
 {
-    MilkymistVgafbState *s = MILKYMIST_VGAFB(dev);
+    MilkymistVgafbState *s = MILKYMIST_VGAFB(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     memory_region_init_io(&s->regs_region, OBJECT(s), &vgafb_mmio_ops, s,
             "milkymist-vgafb", R_MAX * 4);
     sysbus_init_mmio(dev, &s->regs_region);
+}
 
-    s->con = graphic_console_init(DEVICE(dev), 0, &vgafb_ops, s);
+static void milkymist_vgafb_realize(DeviceState *dev, Error **errp)
+{
+    MilkymistVgafbState *s = MILKYMIST_VGAFB(dev);
 
-    return 0;
+    s->con = graphic_console_init(dev, 0, &vgafb_ops, s);
 }
 
 static int vgafb_post_load(void *opaque, int version_id)
@@ -321,18 +335,18 @@
 static void milkymist_vgafb_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = milkymist_vgafb_init;
     dc->reset = milkymist_vgafb_reset;
     dc->vmsd = &vmstate_milkymist_vgafb;
     dc->props = milkymist_vgafb_properties;
+    dc->realize = milkymist_vgafb_realize;
 }
 
 static const TypeInfo milkymist_vgafb_info = {
     .name          = TYPE_MILKYMIST_VGAFB,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistVgafbState),
+    .instance_init = milkymist_vgafb_init,
     .class_init    = milkymist_vgafb_class_init,
 };
 
diff --git a/hw/display/omap_dss.c b/hw/display/omap_dss.c
index 24ccbcc..783e9e1 100644
--- a/hw/display/omap_dss.c
+++ b/hw/display/omap_dss.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/omap.h"
@@ -212,7 +213,8 @@
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -377,7 +379,8 @@
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -736,7 +739,8 @@
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -928,7 +932,8 @@
                             uint64_t value, unsigned size)
 {
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, size);
+        omap_badwidth_write32(opaque, addr, size);
+        return;
     }
 
     switch (addr) {
@@ -1016,7 +1021,8 @@
                            uint64_t value, unsigned size)
 {
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1046,8 +1052,7 @@
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2)
 {
-    struct omap_dss_s *s = (struct omap_dss_s *)
-            g_malloc0(sizeof(struct omap_dss_s));
+    struct omap_dss_s *s = g_new0(struct omap_dss_s, 1);
 
     s->irq = irq;
     s->drq = drq;
diff --git a/hw/display/omap_lcd_template.h b/hw/display/omap_lcd_template.h
index e5dd447..1025ff3 100644
--- a/hw/display/omap_lcd_template.h
+++ b/hw/display/omap_lcd_template.h
@@ -27,13 +27,7 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#if DEPTH == 8
-# define BPP 1
-# define PIXEL_TYPE uint8_t
-#elif DEPTH == 15 || DEPTH == 16
-# define BPP 2
-# define PIXEL_TYPE uint16_t
-#elif DEPTH == 32
+#if DEPTH == 32
 # define BPP 4
 # define PIXEL_TYPE uint32_t
 #else
@@ -136,7 +130,7 @@
     uint8_t r, g, b;
 
     do {
-        v = lduw_p((void *) s);
+        v = lduw_le_p((void *) s);
         r = (v >> 4) & 0xf0;
         g = v & 0xf0;
         b = (v << 4) & 0xf0;
@@ -152,14 +146,14 @@
 static void glue(draw_line16_, DEPTH)(void *opaque,
                 uint8_t *d, const uint8_t *s, int width, int deststep)
 {
-#if DEPTH == 16 && defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
+#if defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
     memcpy(d, s, width * 2);
 #else
     uint16_t v;
     uint8_t r, g, b;
 
     do {
-        v = lduw_p((void *) s);
+        v = lduw_le_p((void *) s);
         r = (v >> 8) & 0xf8;
         g = (v >> 3) & 0xfc;
         b = (v << 3) & 0xf8;
diff --git a/hw/display/omap_lcdc.c b/hw/display/omap_lcdc.c
index fda81ba..07a5eff 100644
--- a/hw/display/omap_lcdc.c
+++ b/hw/display/omap_lcdc.c
@@ -16,6 +16,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/omap.h"
@@ -25,6 +26,7 @@
 struct omap_lcd_panel_s {
     MemoryRegion *sysmem;
     MemoryRegion iomem;
+    MemoryRegionSection fbsection;
     qemu_irq irq;
     QemuConsole *con;
 
@@ -69,47 +71,9 @@
 
 #define draw_line_func drawfn
 
-#define DEPTH 8
-#include "omap_lcd_template.h"
-#define DEPTH 15
-#include "omap_lcd_template.h"
-#define DEPTH 16
-#include "omap_lcd_template.h"
 #define DEPTH 32
 #include "omap_lcd_template.h"
 
-static draw_line_func draw_line_table2[33] = {
-    [0 ... 32]	= NULL,
-    [8]		= draw_line2_8,
-    [15]	= draw_line2_15,
-    [16]	= draw_line2_16,
-    [32]	= draw_line2_32,
-}, draw_line_table4[33] = {
-    [0 ... 32]	= NULL,
-    [8]		= draw_line4_8,
-    [15]	= draw_line4_15,
-    [16]	= draw_line4_16,
-    [32]	= draw_line4_32,
-}, draw_line_table8[33] = {
-    [0 ... 32]	= NULL,
-    [8]		= draw_line8_8,
-    [15]	= draw_line8_15,
-    [16]	= draw_line8_16,
-    [32]	= draw_line8_32,
-}, draw_line_table12[33] = {
-    [0 ... 32]	= NULL,
-    [8]		= draw_line12_8,
-    [15]	= draw_line12_15,
-    [16]	= draw_line12_16,
-    [32]	= draw_line12_32,
-}, draw_line_table16[33] = {
-    [0 ... 32]	= NULL,
-    [8]		= draw_line16_8,
-    [15]	= draw_line16_15,
-    [16]	= draw_line16_16,
-    [32]	= draw_line16_32,
-};
-
 static void omap_update_display(void *opaque)
 {
     struct omap_lcd_panel_s *omap_lcd = (struct omap_lcd_panel_s *) opaque;
@@ -141,25 +105,25 @@
     /* Colour depth */
     switch ((omap_lcd->palette[0] >> 12) & 7) {
     case 1:
-        draw_line = draw_line_table2[surface_bits_per_pixel(surface)];
+        draw_line = draw_line2_32;
         bpp = 2;
         break;
 
     case 2:
-        draw_line = draw_line_table4[surface_bits_per_pixel(surface)];
+        draw_line = draw_line4_32;
         bpp = 4;
         break;
 
     case 3:
-        draw_line = draw_line_table8[surface_bits_per_pixel(surface)];
+        draw_line = draw_line8_32;
         bpp = 8;
         break;
 
     case 4 ... 7:
         if (!omap_lcd->tft)
-            draw_line = draw_line_table12[surface_bits_per_pixel(surface)];
+            draw_line = draw_line12_32;
         else
-            draw_line = draw_line_table16[surface_bits_per_pixel(surface)];
+            draw_line = draw_line16_32;
         bpp = 16;
         break;
 
@@ -215,12 +179,19 @@
 
     step = width * bpp >> 3;
     linesize = surface_stride(surface);
-    framebuffer_update_display(surface, omap_lcd->sysmem,
-                               frame_base, width, height,
+    if (omap_lcd->invalidate) {
+        framebuffer_update_memory_section(&omap_lcd->fbsection,
+                                          omap_lcd->sysmem, frame_base,
+                                          height, step);
+    }
+
+    framebuffer_update_display(surface, &omap_lcd->fbsection,
+                               width, height,
                                step, linesize, 0,
                                omap_lcd->invalidate,
                                draw_line, omap_lcd->palette,
                                &first, &last);
+
     if (first >= 0) {
         dpy_gfx_update(omap_lcd->con, 0, first, width, last - first + 1);
     }
@@ -395,8 +366,7 @@
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk)
 {
-    struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *)
-            g_malloc0(sizeof(struct omap_lcd_panel_s));
+    struct omap_lcd_panel_s *s = g_new0(struct omap_lcd_panel_s, 1);
 
     s->irq = irq;
     s->dma = dma;
diff --git a/hw/display/pl110.c b/hw/display/pl110.c
index c574cf1..c069c0b 100644
--- a/hw/display/pl110.c
+++ b/hw/display/pl110.c
@@ -7,10 +7,12 @@
  * This code is licensed under the GNU LGPL
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "ui/console.h"
 #include "framebuffer.h"
 #include "ui/pixel_ops.h"
+#include "qemu/log.h"
 
 #define PL110_CR_EN   0x001
 #define PL110_CR_BGR  0x100
@@ -46,6 +48,7 @@
     SysBusDevice parent_obj;
 
     MemoryRegion iomem;
+    MemoryRegionSection fbsection;
     QemuConsole *con;
 
     int version;
@@ -238,12 +241,20 @@
     }
     dest_width *= s->cols;
     first = 0;
-    framebuffer_update_display(surface, sysbus_address_space(sbd),
-                               s->upbase, s->cols, s->rows,
+    if (s->invalidate) {
+        framebuffer_update_memory_section(&s->fbsection,
+                                          sysbus_address_space(sbd),
+                                          s->upbase,
+                                          s->rows, src_width);
+    }
+
+    framebuffer_update_display(surface, &s->fbsection,
+                               s->cols, s->rows,
                                src_width, dest_width, 0,
                                s->invalidate,
                                fn, s->palette,
                                &first, &last);
+
     if (first >= 0) {
         dpy_gfx_update(s->con, 0, first, s->cols, last - first + 1);
     }
diff --git a/hw/display/pxa2xx_lcd.c b/hw/display/pxa2xx_lcd.c
index ac3c018..845521c 100644
--- a/hw/display/pxa2xx_lcd.c
+++ b/hw/display/pxa2xx_lcd.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/arm/pxa.h"
@@ -35,6 +36,7 @@
 struct PXA2xxLCDState {
     MemoryRegion *sysmem;
     MemoryRegion iomem;
+    MemoryRegionSection fbsection;
     qemu_irq irq;
     int irqlevel;
 
@@ -308,10 +310,10 @@
         }
 
         cpu_physical_memory_read(descptr, &desc, sizeof(desc));
-        s->dma_ch[i].descriptor = tswap32(desc.fdaddr);
-        s->dma_ch[i].source = tswap32(desc.fsaddr);
-        s->dma_ch[i].id = tswap32(desc.fidr);
-        s->dma_ch[i].command = tswap32(desc.ldcmd);
+        s->dma_ch[i].descriptor = le32_to_cpu(desc.fdaddr);
+        s->dma_ch[i].source = le32_to_cpu(desc.fsaddr);
+        s->dma_ch[i].id = le32_to_cpu(desc.fidr);
+        s->dma_ch[i].command = le32_to_cpu(desc.ldcmd);
     }
 }
 
@@ -687,8 +689,11 @@
 
     dest_width = s->xres * s->dest_width;
     *miny = 0;
-    framebuffer_update_display(surface, s->sysmem,
-                               addr, s->xres, s->yres,
+    if (s->invalidated) {
+        framebuffer_update_memory_section(&s->fbsection, s->sysmem,
+                                          addr, s->yres, src_width);
+    }
+    framebuffer_update_display(surface, &s->fbsection, s->xres, s->yres,
                                src_width, dest_width, s->dest_width,
                                s->invalidated,
                                fn, s->dma_ch[0].palette, miny, maxy);
@@ -715,8 +720,11 @@
 
     dest_width = s->yres * s->dest_width;
     *miny = 0;
-    framebuffer_update_display(surface, s->sysmem,
-                               addr, s->xres, s->yres,
+    if (s->invalidated) {
+        framebuffer_update_memory_section(&s->fbsection, s->sysmem,
+                                          addr, s->yres, src_width);
+    }
+    framebuffer_update_display(surface, &s->fbsection, s->xres, s->yres,
                                src_width, s->dest_width, -dest_width,
                                s->invalidated,
                                fn, s->dma_ch[0].palette,
@@ -747,8 +755,11 @@
 
     dest_width = s->xres * s->dest_width;
     *miny = 0;
-    framebuffer_update_display(surface, s->sysmem,
-                               addr, s->xres, s->yres,
+    if (s->invalidated) {
+        framebuffer_update_memory_section(&s->fbsection, s->sysmem,
+                                          addr, s->yres, src_width);
+    }
+    framebuffer_update_display(surface, &s->fbsection, s->xres, s->yres,
                                src_width, -dest_width, -s->dest_width,
                                s->invalidated,
                                fn, s->dma_ch[0].palette, miny, maxy);
@@ -778,8 +789,11 @@
 
     dest_width = s->yres * s->dest_width;
     *miny = 0;
-    framebuffer_update_display(surface, s->sysmem,
-                               addr, s->xres, s->yres,
+    if (s->invalidated) {
+        framebuffer_update_memory_section(&s->fbsection, s->sysmem,
+                                          addr, s->yres, src_width);
+    }
+    framebuffer_update_display(surface, &s->fbsection, s->xres, s->yres,
                                src_width, -s->dest_width, dest_width,
                                s->invalidated,
                                fn, s->dma_ch[0].palette,
diff --git a/hw/display/qxl-logger.c b/hw/display/qxl-logger.c
index c900c2c..2ec6d8f 100644
--- a/hw/display/qxl-logger.c
+++ b/hw/display/qxl-logger.c
@@ -19,10 +19,11 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/timer.h"
 #include "qxl.h"
 
-static const char *qxl_type[] = {
+static const char *const qxl_type[] = {
     [ QXL_CMD_NOP ]     = "nop",
     [ QXL_CMD_DRAW ]    = "draw",
     [ QXL_CMD_UPDATE ]  = "update",
@@ -31,7 +32,7 @@
     [ QXL_CMD_SURFACE ] = "surface",
 };
 
-static const char *qxl_draw_type[] = {
+static const char *const qxl_draw_type[] = {
     [ QXL_DRAW_NOP         ] = "nop",
     [ QXL_DRAW_FILL        ] = "fill",
     [ QXL_DRAW_OPAQUE      ] = "opaque",
@@ -48,7 +49,7 @@
     [ QXL_DRAW_ALPHA_BLEND ] = "alpha-blend",
 };
 
-static const char *qxl_draw_effect[] = {
+static const char *const qxl_draw_effect[] = {
     [ QXL_EFFECT_BLEND            ] = "blend",
     [ QXL_EFFECT_OPAQUE           ] = "opaque",
     [ QXL_EFFECT_REVERT_ON_DUP    ] = "revert-on-dup",
@@ -59,12 +60,12 @@
     [ QXL_EFFECT_OPAQUE_BRUSH     ] = "opaque-brush",
 };
 
-static const char *qxl_surface_cmd[] = {
+static const char *const qxl_surface_cmd[] = {
    [ QXL_SURFACE_CMD_CREATE  ] = "create",
    [ QXL_SURFACE_CMD_DESTROY ] = "destroy",
 };
 
-static const char *spice_surface_fmt[] = {
+static const char *const spice_surface_fmt[] = {
    [ SPICE_SURFACE_FMT_INVALID  ] = "invalid",
    [ SPICE_SURFACE_FMT_1_A      ] = "alpha/1",
    [ SPICE_SURFACE_FMT_8_A      ] = "alpha/8",
@@ -74,14 +75,14 @@
    [ SPICE_SURFACE_FMT_32_ARGB  ] = "ARGB/32",
 };
 
-static const char *qxl_cursor_cmd[] = {
+static const char *const qxl_cursor_cmd[] = {
    [ QXL_CURSOR_SET   ] = "set",
    [ QXL_CURSOR_MOVE  ] = "move",
    [ QXL_CURSOR_HIDE  ] = "hide",
    [ QXL_CURSOR_TRAIL ] = "trail",
 };
 
-static const char *spice_cursor_type[] = {
+static const char *const spice_cursor_type[] = {
    [ SPICE_CURSOR_TYPE_ALPHA   ] = "alpha",
    [ SPICE_CURSOR_TYPE_MONO    ] = "mono",
    [ SPICE_CURSOR_TYPE_COLOR4  ] = "color4",
@@ -91,7 +92,7 @@
    [ SPICE_CURSOR_TYPE_COLOR32 ] = "color32",
 };
 
-static const char *qxl_v2n(const char *n[], size_t l, int v)
+static const char *qxl_v2n(const char *const n[], size_t l, int v)
 {
     if (v >= l || !n[v]) {
         return "???";
diff --git a/hw/display/qxl-render.c b/hw/display/qxl-render.c
index e812ddd..9ad9d9e 100644
--- a/hw/display/qxl-render.c
+++ b/hw/display/qxl-render.c
@@ -19,6 +19,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qxl.h"
 #include "trace.h"
 
@@ -159,7 +160,7 @@
 /*
  * use ssd.lock to protect render_update_cookie_num.
  * qxl_render_update is called by io thread or vcpu thread, and the completion
- * callbacks are called by spice_server thread, defering to bh called from the
+ * callbacks are called by spice_server thread, deferring to bh called from the
  * io thread.
  */
 void qxl_render_update(PCIQXLDevice *qxl)
@@ -283,12 +284,14 @@
         qxl->ssd.mouse_x = cmd->u.set.position.x;
         qxl->ssd.mouse_y = cmd->u.set.position.y;
         qemu_mutex_unlock(&qxl->ssd.lock);
+        qemu_bh_schedule(qxl->ssd.cursor_bh);
         break;
     case QXL_CURSOR_MOVE:
         qemu_mutex_lock(&qxl->ssd.lock);
         qxl->ssd.mouse_x = cmd->u.position.x;
         qxl->ssd.mouse_y = cmd->u.position.y;
         qemu_mutex_unlock(&qxl->ssd.lock);
+        qemu_bh_schedule(qxl->ssd.cursor_bh);
         break;
     }
     return 0;
diff --git a/hw/display/qxl.c b/hw/display/qxl.c
index b540dd6..0e2682d 100644
--- a/hw/display/qxl.c
+++ b/hw/display/qxl.c
@@ -18,14 +18,13 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <zlib.h>
-#include <stdint.h>
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "qemu/queue.h"
 #include "qemu/atomic.h"
-#include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
 #include "trace.h"
 
@@ -120,9 +119,12 @@
     QXL_MODE_EX(2560, 2048),
     QXL_MODE_EX(2800, 2100),
     QXL_MODE_EX(3200, 2400),
+    /* these modes need more than 32 MB video memory */
     QXL_MODE_EX(3840, 2160), /* 4k mainstream */
     QXL_MODE_EX(4096, 2160), /* 4k            */
+    /* these modes need more than 64 MB video memory */
     QXL_MODE_EX(7680, 4320), /* 8k mainstream */
+    /* these modes need more than 128 MB video memory */
     QXL_MODE_EX(8192, 4320), /* 8k            */
 };
 
@@ -269,6 +271,11 @@
                     QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
                     0));
     } else {
+#if SPICE_SERVER_VERSION >= 0x000c06 /* release 0.12.6 */
+        if (qxl->max_outputs) {
+            spice_qxl_set_max_monitors(&qxl->ssd.qxl, qxl->max_outputs);
+        }
+#endif
         qxl->guest_monitors_config = qxl->ram->monitors_config;
         spice_qxl_monitors_config_async(&qxl->ssd.qxl,
                 qxl->ram->monitors_config,
@@ -297,19 +304,6 @@
     qxl->ssd.cursor = cursor_builtin_hidden();
 }
 
-
-static inline uint32_t msb_mask(uint32_t val)
-{
-    uint32_t mask;
-
-    do {
-        mask = ~(val - 1) & val;
-        val &= ~mask;
-    } while (mask < val);
-
-    return mask;
-}
-
 static ram_addr_t qxl_rom_size(void)
 {
     uint32_t required_rom_size = sizeof(QXLRom) + sizeof(QXLModes) +
@@ -367,6 +361,8 @@
     num_pages         -= surface0_area_size;
     num_pages          = num_pages / QXL_PAGE_SIZE;
 
+    assert(ram_header_size + surface0_area_size <= d->vga.vram_size);
+
     rom->draw_area_offset   = cpu_to_le32(0);
     rom->surface0_area_size = cpu_to_le32(surface0_area_size);
     rom->pages_offset       = cpu_to_le32(surface0_area_size);
@@ -508,15 +504,21 @@
     qxl_rom_set_dirty(qxl);
 }
 
+#if SPICE_NEEDS_SET_MM_TIME
 static void interface_set_mm_time(QXLInstance *sin, uint32_t mm_time)
 {
     PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
 
+    if (!qemu_spice_display_is_running(&qxl->ssd)) {
+        return;
+    }
+
     trace_qxl_interface_set_mm_time(qxl->id, mm_time);
     qxl->shadow_rom.mm_clock = cpu_to_le32(mm_time);
     qxl->rom->mm_clock = cpu_to_le32(mm_time);
     qxl_rom_set_dirty(qxl);
 }
+#endif
 
 static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
 {
@@ -704,7 +706,7 @@
 
 /* called from spice server thread context only */
 static void interface_release_resource(QXLInstance *sin,
-                                       struct QXLReleaseInfoExt ext)
+                                       QXLReleaseInfoExt ext)
 {
     PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
     QXLReleaseRing *ring;
@@ -893,7 +895,8 @@
     int qxl_i;
 
     qemu_mutex_lock(&qxl->ssd.lock);
-    if (surface_id != 0 || !qxl->render_update_cookie_num) {
+    if (surface_id != 0 || !num_updated_rects ||
+        !qxl->render_update_cookie_num) {
         qemu_mutex_unlock(&qxl->ssd.lock);
         return;
     }
@@ -996,6 +999,7 @@
     PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
     QXLRom *rom = memory_region_get_ram_ptr(&qxl->rom_bar);
     int i;
+    unsigned max_outputs = ARRAY_SIZE(rom->client_monitors_config.heads);
 
     if (qxl->revision < 4) {
         trace_qxl_client_monitors_config_unsupported_by_device(qxl->id,
@@ -1018,17 +1022,23 @@
     if (!monitors_config) {
         return 1;
     }
+
+#if SPICE_SERVER_VERSION >= 0x000c06 /* release 0.12.6 */
+    /* limit number of outputs based on setting limit */
+    if (qxl->max_outputs && qxl->max_outputs <= max_outputs) {
+        max_outputs = qxl->max_outputs;
+    }
+#endif
+
     memset(&rom->client_monitors_config, 0,
            sizeof(rom->client_monitors_config));
     rom->client_monitors_config.count = monitors_config->num_of_monitors;
     /* monitors_config->flags ignored */
-    if (rom->client_monitors_config.count >=
-            ARRAY_SIZE(rom->client_monitors_config.heads)) {
+    if (rom->client_monitors_config.count >= max_outputs) {
         trace_qxl_client_monitors_config_capped(qxl->id,
                                 monitors_config->num_of_monitors,
-                                ARRAY_SIZE(rom->client_monitors_config.heads));
-        rom->client_monitors_config.count =
-            ARRAY_SIZE(rom->client_monitors_config.heads);
+                                max_outputs);
+        rom->client_monitors_config.count = max_outputs;
     }
     for (i = 0 ; i < rom->client_monitors_config.count ; ++i) {
         VDAgentMonConfig *monitor = &monitors_config->monitors[i];
@@ -1061,7 +1071,9 @@
 
     .attache_worker          = interface_attach_worker,
     .set_compression_level   = interface_set_compression_level,
+#if SPICE_NEEDS_SET_MM_TIME
     .set_mm_time             = interface_set_mm_time,
+#endif
     .get_init_info           = interface_get_init_info,
 
     /* the callbacks below are called from spice server thread context */
@@ -1092,6 +1104,7 @@
     spice_qxl_driver_unload(&d->ssd.qxl);
 #endif
     graphic_console_set_hwops(d->ssd.dcl.con, d->vga.hw_ops, &d->vga);
+    update_displaychangelistener(&d->ssd.dcl, GUI_REFRESH_INTERVAL_DEFAULT);
     qemu_spice_create_host_primary(&d->ssd);
     d->mode = QXL_MODE_VGA;
     vga_dirty_log_start(&d->vga);
@@ -1105,6 +1118,7 @@
     }
     trace_qxl_exit_vga_mode(d->id);
     graphic_console_set_hwops(d->ssd.dcl.con, &qxl_ops, d);
+    update_displaychangelistener(&d->ssd.dcl, GUI_REFRESH_INTERVAL_IDLE);
     vga_dirty_log_stop(&d->vga);
     qxl_destroy_primary(d, QXL_SYNC);
 }
@@ -1147,7 +1161,9 @@
     trace_qxl_soft_reset(d->id);
     qxl_check_state(d);
     qxl_clear_guest_bug(d);
+    qemu_mutex_lock(&d->async_lock);
     d->current_async = QXL_UNDEFINED_IO;
+    qemu_mutex_unlock(&d->async_lock);
 
     if (d->id == 0) {
         qxl_enter_vga_mode(d);
@@ -1187,7 +1203,7 @@
 
 static void qxl_reset_handler(DeviceState *dev)
 {
-    PCIQXLDevice *d = DO_UPCAST(PCIQXLDevice, pci.qdev, dev);
+    PCIQXLDevice *d = PCI_QXL(PCI_DEVICE(dev));
 
     qxl_hard_reset(d, 0);
 }
@@ -1232,6 +1248,7 @@
     int pci_region;
     pcibus_t pci_start;
     pcibus_t pci_end;
+    MemoryRegion *mr;
     intptr_t virt_start;
     QXLDevMemSlot memslot;
     int i;
@@ -1278,11 +1295,11 @@
 
     switch (pci_region) {
     case QXL_RAM_RANGE_INDEX:
-        virt_start = (intptr_t)memory_region_get_ram_ptr(&d->vga.vram);
+        mr = &d->vga.vram;
         break;
     case QXL_VRAM_RANGE_INDEX:
     case 4 /* vram 64bit */:
-        virt_start = (intptr_t)memory_region_get_ram_ptr(&d->vram_bar);
+        mr = &d->vram_bar;
         break;
     default:
         /* should not happen */
@@ -1290,6 +1307,7 @@
         return 1;
     }
 
+    virt_start = (intptr_t)memory_region_get_ram_ptr(mr);
     memslot.slot_id = slot_id;
     memslot.slot_group_id = MEMSLOT_GROUP_GUEST; /* guest group */
     memslot.virt_start = virt_start + (guest_start - pci_start);
@@ -1299,7 +1317,8 @@
     qxl_rom_set_dirty(d);
 
     qemu_spice_add_memslot(&d->ssd, &memslot, async);
-    d->guest_slots[slot_id].ptr = (void*)memslot.virt_start;
+    d->guest_slots[slot_id].mr = mr;
+    d->guest_slots[slot_id].offset = memslot.virt_start - virt_start;
     d->guest_slots[slot_id].size = memslot.virt_end - memslot.virt_start;
     d->guest_slots[slot_id].delta = delta;
     d->guest_slots[slot_id].active = 1;
@@ -1326,39 +1345,60 @@
 }
 
 /* can be also called from spice server thread context */
-void *qxl_phys2virt(PCIQXLDevice *qxl, QXLPHYSICAL pqxl, int group_id)
+static bool qxl_get_check_slot_offset(PCIQXLDevice *qxl, QXLPHYSICAL pqxl,
+                                      uint32_t *s, uint64_t *o)
 {
     uint64_t phys   = le64_to_cpu(pqxl);
     uint32_t slot   = (phys >> (64 -  8)) & 0xff;
     uint64_t offset = phys & 0xffffffffffff;
 
-    switch (group_id) {
-    case MEMSLOT_GROUP_HOST:
-        return (void *)(intptr_t)offset;
-    case MEMSLOT_GROUP_GUEST:
-        if (slot >= NUM_MEMSLOTS) {
-            qxl_set_guest_bug(qxl, "slot too large %d >= %d", slot,
-                              NUM_MEMSLOTS);
-            return NULL;
-        }
-        if (!qxl->guest_slots[slot].active) {
-            qxl_set_guest_bug(qxl, "inactive slot %d\n", slot);
-            return NULL;
-        }
-        if (offset < qxl->guest_slots[slot].delta) {
-            qxl_set_guest_bug(qxl,
+    if (slot >= NUM_MEMSLOTS) {
+        qxl_set_guest_bug(qxl, "slot too large %d >= %d", slot,
+                          NUM_MEMSLOTS);
+        return false;
+    }
+    if (!qxl->guest_slots[slot].active) {
+        qxl_set_guest_bug(qxl, "inactive slot %d\n", slot);
+        return false;
+    }
+    if (offset < qxl->guest_slots[slot].delta) {
+        qxl_set_guest_bug(qxl,
                           "slot %d offset %"PRIu64" < delta %"PRIu64"\n",
                           slot, offset, qxl->guest_slots[slot].delta);
-            return NULL;
-        }
-        offset -= qxl->guest_slots[slot].delta;
-        if (offset > qxl->guest_slots[slot].size) {
-            qxl_set_guest_bug(qxl,
+        return false;
+    }
+    offset -= qxl->guest_slots[slot].delta;
+    if (offset > qxl->guest_slots[slot].size) {
+        qxl_set_guest_bug(qxl,
                           "slot %d offset %"PRIu64" > size %"PRIu64"\n",
                           slot, offset, qxl->guest_slots[slot].size);
+        return false;
+    }
+
+    *s = slot;
+    *o = offset;
+    return true;
+}
+
+/* can be also called from spice server thread context */
+void *qxl_phys2virt(PCIQXLDevice *qxl, QXLPHYSICAL pqxl, int group_id)
+{
+    uint64_t offset;
+    uint32_t slot;
+    void *ptr;
+
+    switch (group_id) {
+    case MEMSLOT_GROUP_HOST:
+        offset = le64_to_cpu(pqxl) & 0xffffffffffff;
+        return (void *)(intptr_t)offset;
+    case MEMSLOT_GROUP_GUEST:
+        if (!qxl_get_check_slot_offset(qxl, pqxl, &slot, &offset)) {
             return NULL;
         }
-        return qxl->guest_slots[slot].ptr + offset;
+        ptr = memory_region_get_ram_ptr(qxl->guest_slots[slot].mr);
+        ptr += qxl->guest_slots[slot].offset;
+        ptr += offset;
+        return ptr;
     }
     return NULL;
 }
@@ -1773,9 +1813,24 @@
     qxl_render_update(qxl);
 }
 
+static void qxl_dirty_one_surface(PCIQXLDevice *qxl, QXLPHYSICAL pqxl,
+                                  uint32_t height, int32_t stride)
+{
+    uint64_t offset, size;
+    uint32_t slot;
+    bool rc;
+
+    rc = qxl_get_check_slot_offset(qxl, pqxl, &slot, &offset);
+    assert(rc == true);
+    size = (uint64_t)height * abs(stride);
+    trace_qxl_surfaces_dirty(qxl->id, offset, size);
+    qxl_set_dirty(qxl->guest_slots[slot].mr,
+                  qxl->guest_slots[slot].offset + offset,
+                  qxl->guest_slots[slot].offset + offset + size);
+}
+
 static void qxl_dirty_surfaces(PCIQXLDevice *qxl)
 {
-    uintptr_t vram_start;
     int i;
 
     if (qxl->mode != QXL_MODE_NATIVE && qxl->mode != QXL_MODE_COMPAT) {
@@ -1783,16 +1838,13 @@
     }
 
     /* dirty the primary surface */
-    qxl_set_dirty(&qxl->vga.vram, qxl->shadow_rom.draw_area_offset,
-                  qxl->shadow_rom.surface0_area_size);
-
-    vram_start = (uintptr_t)memory_region_get_ram_ptr(&qxl->vram_bar);
+    qxl_dirty_one_surface(qxl, qxl->guest_primary.surface.mem,
+                          qxl->guest_primary.surface.height,
+                          qxl->guest_primary.surface.stride);
 
     /* dirty the off-screen surfaces */
     for (i = 0; i < qxl->ssd.num_surfaces; i++) {
         QXLSurfaceCmd *cmd;
-        intptr_t surface_offset;
-        int surface_size;
 
         if (qxl->guest_surfaces.cmds[i] == 0) {
             continue;
@@ -1802,15 +1854,9 @@
                             MEMSLOT_GROUP_GUEST);
         assert(cmd);
         assert(cmd->type == QXL_SURFACE_CMD_CREATE);
-        surface_offset = (intptr_t)qxl_phys2virt(qxl,
-                                                 cmd->u.surface_create.data,
-                                                 MEMSLOT_GROUP_GUEST);
-        assert(surface_offset);
-        surface_offset -= vram_start;
-        surface_size = cmd->u.surface_create.height *
-                       abs(cmd->u.surface_create.stride);
-        trace_qxl_surfaces_dirty(qxl->id, i, (int)surface_offset, surface_size);
-        qxl_set_dirty(&qxl->vram_bar, surface_offset, surface_size);
+        qxl_dirty_one_surface(qxl, cmd->u.surface_create.data,
+                              cmd->u.surface_create.height,
+                              cmd->u.surface_create.stride);
     }
 }
 
@@ -1861,10 +1907,6 @@
 
     if (qxl->mode == QXL_MODE_VGA) {
         qemu_spice_display_refresh(&qxl->ssd);
-    } else {
-        qemu_mutex_lock(&qxl->ssd.lock);
-        qemu_spice_cursor_refresh_unlocked(&qxl->ssd);
-        qemu_mutex_unlock(&qxl->ssd.lock);
     }
 }
 
@@ -1881,6 +1923,12 @@
     if (qxl->vgamem_size_mb < 8) {
         qxl->vgamem_size_mb = 8;
     }
+    /* XXX: we round vgamem_size_mb up to a nearest power of two and it must be
+     * less than vga_common_init()'s maximum on qxl->vga.vram_size (512 now).
+     */
+    if (qxl->vgamem_size_mb > 256) {
+        qxl->vgamem_size_mb = 256;
+    }
     qxl->vgamem_size = qxl->vgamem_size_mb * 1024 * 1024;
 
     /* vga ram (bar 0, total) */
@@ -1901,7 +1949,7 @@
 
     /* vram (surfaces, 64bit, bar 4+5) */
     if (qxl->vram_size_mb != -1) {
-        qxl->vram_size = qxl->vram_size_mb * 1024 * 1024;
+        qxl->vram_size = (uint64_t)qxl->vram_size_mb * 1024 * 1024;
     }
     if (qxl->vram_size < qxl->vram32_size) {
         qxl->vram_size = qxl->vram32_size;
@@ -1911,13 +1959,13 @@
         qxl->vram32_size = 4096;
         qxl->vram_size = 4096;
     }
-    qxl->vgamem_size = msb_mask(qxl->vgamem_size * 2 - 1);
-    qxl->vga.vram_size = msb_mask(qxl->vga.vram_size * 2 - 1);
-    qxl->vram32_size = msb_mask(qxl->vram32_size * 2 - 1);
-    qxl->vram_size = msb_mask(qxl->vram_size * 2 - 1);
+    qxl->vgamem_size = pow2ceil(qxl->vgamem_size);
+    qxl->vga.vram_size = pow2ceil(qxl->vga.vram_size);
+    qxl->vram32_size = pow2ceil(qxl->vram32_size);
+    qxl->vram_size = pow2ceil(qxl->vram_size);
 }
 
-static int qxl_init_common(PCIQXLDevice *qxl)
+static void qxl_realize_common(PCIQXLDevice *qxl, Error **errp)
 {
     uint8_t* config = qxl->pci.config;
     uint32_t pci_device_rev;
@@ -1946,12 +1994,12 @@
         break;
     case 4: /* qxl-4 */
         pci_device_rev = QXL_REVISION_STABLE_V12;
-        io_size = msb_mask(QXL_IO_RANGE_SIZE * 2 - 1);
+        io_size = pow2ceil(QXL_IO_RANGE_SIZE);
         break;
     default:
-        error_report("Invalid revision %d for qxl device (max %d)",
-                     qxl->revision, QXL_DEFAULT_REVISION);
-        return -1;
+        error_setg(errp, "Invalid revision %d for qxl device (max %d)",
+                   qxl->revision, QXL_DEFAULT_REVISION);
+        return;
     }
 
     pci_set_byte(&config[PCI_REVISION_ID], pci_device_rev);
@@ -1959,14 +2007,14 @@
 
     qxl->rom_size = qxl_rom_size();
     memory_region_init_ram(&qxl->rom_bar, OBJECT(qxl), "qxl.vrom",
-                           qxl->rom_size, &error_abort);
+                           qxl->rom_size, &error_fatal);
     vmstate_register_ram(&qxl->rom_bar, &qxl->pci.qdev);
     init_qxl_rom(qxl);
     init_qxl_ram(qxl);
 
     qxl->guest_surfaces.cmds = g_new0(QXLPHYSICAL, qxl->ssd.num_surfaces);
     memory_region_init_ram(&qxl->vram_bar, OBJECT(qxl), "qxl.vram",
-                           qxl->vram_size, &error_abort);
+                           qxl->vram_size, &error_fatal);
     vmstate_register_ram(&qxl->vram_bar, &qxl->pci.qdev);
     memory_region_init_alias(&qxl->vram32_bar, OBJECT(qxl), "qxl.vram32",
                              &qxl->vram_bar, 0, qxl->vram32_size);
@@ -2007,17 +2055,17 @@
     dprint(qxl, 1, "ram/%s: %d MB [region 0]\n",
            qxl->id == 0 ? "pri" : "sec",
            qxl->vga.vram_size / (1024*1024));
-    dprint(qxl, 1, "vram/32: %d MB [region 1]\n",
+    dprint(qxl, 1, "vram/32: %" PRIx64 "d MB [region 1]\n",
            qxl->vram32_size / (1024*1024));
-    dprint(qxl, 1, "vram/64: %d MB %s\n",
+    dprint(qxl, 1, "vram/64: %" PRIx64 "d MB %s\n",
            qxl->vram_size / (1024*1024),
            qxl->vram32_size < qxl->vram_size ? "[region 4]" : "[unmapped]");
 
     qxl->ssd.qxl.base.sif = &qxl_interface.base;
     if (qemu_spice_add_display_interface(&qxl->ssd.qxl, qxl->vga.con) != 0) {
-        error_report("qxl interface %d.%d not supported by spice-server",
-                     SPICE_INTERFACE_QXL_MAJOR, SPICE_INTERFACE_QXL_MINOR);
-        return -1;
+        error_setg(errp, "qxl interface %d.%d not supported by spice-server",
+                   SPICE_INTERFACE_QXL_MAJOR, SPICE_INTERFACE_QXL_MINOR);
+        return;
     }
     qemu_add_vm_change_state_handler(qxl_vm_change_state_handler, qxl);
 
@@ -2025,15 +2073,14 @@
     qxl_reset_state(qxl);
 
     qxl->update_area_bh = qemu_bh_new(qxl_render_update_area_bh, qxl);
-
-    return 0;
+    qxl->ssd.cursor_bh = qemu_bh_new(qemu_spice_cursor_refresh_bh, &qxl->ssd);
 }
 
-static int qxl_init_primary(PCIDevice *dev)
+static void qxl_realize_primary(PCIDevice *dev, Error **errp)
 {
-    PCIQXLDevice *qxl = DO_UPCAST(PCIQXLDevice, pci, dev);
+    PCIQXLDevice *qxl = PCI_QXL(dev);
     VGACommonState *vga = &qxl->vga;
-    int rc;
+    Error *local_err = NULL;
 
     qxl->id = 0;
     qxl_init_ramsize(qxl);
@@ -2050,31 +2097,31 @@
     vga->con = graphic_console_init(DEVICE(dev), 0, &qxl_ops, qxl);
     qemu_spice_display_init_common(&qxl->ssd);
 
-    rc = qxl_init_common(qxl);
-    if (rc != 0) {
-        return rc;
+    qxl_realize_common(qxl, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
     qxl->ssd.dcl.ops = &display_listener_ops;
     qxl->ssd.dcl.con = vga->con;
     register_displaychangelistener(&qxl->ssd.dcl);
-    return rc;
 }
 
-static int qxl_init_secondary(PCIDevice *dev)
+static void qxl_realize_secondary(PCIDevice *dev, Error **errp)
 {
     static int device_id = 1;
-    PCIQXLDevice *qxl = DO_UPCAST(PCIQXLDevice, pci, dev);
+    PCIQXLDevice *qxl = PCI_QXL(dev);
 
     qxl->id = device_id++;
     qxl_init_ramsize(qxl);
     memory_region_init_ram(&qxl->vga.vram, OBJECT(dev), "qxl.vgavram",
-                           qxl->vga.vram_size, &error_abort);
+                           qxl->vga.vram_size, &error_fatal);
     vmstate_register_ram(&qxl->vga.vram, &qxl->pci.qdev);
     qxl->vga.vram_ptr = memory_region_get_ram_ptr(&qxl->vga.vram);
     qxl->vga.con = graphic_console_init(DEVICE(dev), 0, &qxl_ops, qxl);
 
-    return qxl_init_common(qxl);
+    qxl_realize_common(qxl, errp);
 }
 
 static void qxl_pre_save(void *opaque)
@@ -2146,7 +2193,7 @@
         qxl_create_guest_primary(d, 1, QXL_SYNC);
 
         /* replay surface-create and cursor-set commands */
-        cmds = g_malloc0(sizeof(QXLCommandExt) * (d->ssd.num_surfaces + 1));
+        cmds = g_new0(QXLCommandExt, d->ssd.num_surfaces + 1);
         for (in = 0, out = 0; in < d->ssd.num_surfaces; in++) {
             if (d->guest_surfaces.cmds[in] == 0) {
                 continue;
@@ -2221,6 +2268,7 @@
     .name               = "qxl/monitors-config",
     .version_id         = 1,
     .minimum_version_id = 1,
+    .needed = qxl_monitors_config_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(guest_monitors_config, PCIQXLDevice),
         VMSTATE_END_OF_LIST()
@@ -2254,20 +2302,16 @@
         VMSTATE_UINT64(guest_cursor, PCIQXLDevice),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &qxl_vmstate_monitors_config,
-            .needed = qxl_monitors_config_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &qxl_vmstate_monitors_config,
+        NULL
     }
 };
 
 static Property qxl_properties[] = {
         DEFINE_PROP_UINT32("ram_size", PCIQXLDevice, vga.vram_size,
                            64 * 1024 * 1024),
-        DEFINE_PROP_UINT32("vram_size", PCIQXLDevice, vram32_size,
+        DEFINE_PROP_UINT64("vram_size", PCIQXLDevice, vram32_size,
                            64 * 1024 * 1024),
         DEFINE_PROP_UINT32("revision", PCIQXLDevice, revision,
                            QXL_DEFAULT_REVISION),
@@ -2279,31 +2323,48 @@
         DEFINE_PROP_UINT32("vram64_size_mb", PCIQXLDevice, vram_size_mb, -1),
         DEFINE_PROP_UINT32("vgamem_mb", PCIQXLDevice, vgamem_size_mb, 16),
         DEFINE_PROP_INT32("surfaces", PCIQXLDevice, ssd.num_surfaces, 1024),
+#if SPICE_SERVER_VERSION >= 0x000c06 /* release 0.12.6 */
+        DEFINE_PROP_UINT16("max_outputs", PCIQXLDevice, max_outputs, 0),
+#endif
         DEFINE_PROP_END_OF_LIST(),
 };
 
+static void qxl_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->vendor_id = REDHAT_PCI_VENDOR_ID;
+    k->device_id = QXL_DEVICE_ID_STABLE;
+    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
+    dc->reset = qxl_reset_handler;
+    dc->vmsd = &qxl_vmstate;
+    dc->props = qxl_properties;
+}
+
+static const TypeInfo qxl_pci_type_info = {
+    .name = TYPE_PCI_QXL,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIQXLDevice),
+    .abstract = true,
+    .class_init = qxl_pci_class_init,
+};
+
 static void qxl_primary_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = qxl_init_primary;
+    k->realize = qxl_realize_primary;
     k->romfile = "vgabios-qxl.bin";
-    k->vendor_id = REDHAT_PCI_VENDOR_ID;
-    k->device_id = QXL_DEVICE_ID_STABLE;
     k->class_id = PCI_CLASS_DISPLAY_VGA;
-    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
     dc->desc = "Spice QXL GPU (primary, vga compatible)";
-    dc->reset = qxl_reset_handler;
-    dc->vmsd = &qxl_vmstate;
-    dc->props = qxl_properties;
     dc->hotpluggable = false;
 }
 
 static const TypeInfo qxl_primary_info = {
     .name          = "qxl-vga",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PCIQXLDevice),
+    .parent        = TYPE_PCI_QXL,
     .class_init    = qxl_primary_class_init,
 };
 
@@ -2312,26 +2373,20 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = qxl_init_secondary;
-    k->vendor_id = REDHAT_PCI_VENDOR_ID;
-    k->device_id = QXL_DEVICE_ID_STABLE;
+    k->realize = qxl_realize_secondary;
     k->class_id = PCI_CLASS_DISPLAY_OTHER;
-    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
     dc->desc = "Spice QXL GPU (secondary)";
-    dc->reset = qxl_reset_handler;
-    dc->vmsd = &qxl_vmstate;
-    dc->props = qxl_properties;
 }
 
 static const TypeInfo qxl_secondary_info = {
     .name          = "qxl",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PCIQXLDevice),
+    .parent        = TYPE_PCI_QXL,
     .class_init    = qxl_secondary_class_init,
 };
 
 static void qxl_register_types(void)
 {
+    type_register_static(&qxl_pci_type_info);
     type_register_static(&qxl_primary_info);
     type_register_static(&qxl_secondary_info);
 }
diff --git a/hw/display/qxl.h b/hw/display/qxl.h
index 412e346..d2d49dd 100644
--- a/hw/display/qxl.h
+++ b/hw/display/qxl.h
@@ -1,5 +1,5 @@
 #ifndef HW_QXL_H
-#define HW_QXL_H 1
+#define HW_QXL_H
 
 #include "qemu-common.h"
 
@@ -53,7 +53,8 @@
 
     struct guest_slots {
         QXLMemSlot     slot;
-        void           *ptr;
+        MemoryRegion   *mr;
+        uint64_t       offset;
         uint64_t       size;
         uint64_t       delta;
         uint32_t       active;
@@ -99,11 +100,14 @@
     QXLModes           *modes;
     uint32_t           rom_size;
     MemoryRegion       rom_bar;
+#if SPICE_SERVER_VERSION >= 0x000c06 /* release 0.12.6 */
+    uint16_t           max_outputs;
+#endif
 
     /* vram pci bar */
-    uint32_t           vram_size;
+    uint64_t           vram_size;
     MemoryRegion       vram_bar;
-    uint32_t           vram32_size;
+    uint64_t           vram32_size;
     MemoryRegion       vram32_bar;
 
     /* io bar */
@@ -122,6 +126,9 @@
     QEMUBH            *update_area_bh;
 } PCIQXLDevice;
 
+#define TYPE_PCI_QXL "pci-qxl"
+#define PCI_QXL(obj) OBJECT_CHECK(PCIQXLDevice, (obj), TYPE_PCI_QXL)
+
 #define PANIC_ON(x) if ((x)) {                         \
     printf("%s: PANIC %s failed\n", __FUNCTION__, #x); \
     abort();                                           \
diff --git a/hw/display/sm501.c b/hw/display/sm501.c
index c72154b..5f71012 100644
--- a/hw/display/sm501.c
+++ b/hw/display/sm501.c
@@ -22,7 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/char/serial.h"
 #include "ui/console.h"
@@ -1322,6 +1325,7 @@
     }
 
     /* draw each line according to conditions */
+    memory_region_sync_dirty_bitmap(&s->local_mem_region);
     for (y = 0; y < height; y++) {
 	int update_hwc = draw_hwc_line ? within_hwc_y_range(s, y, 1) : 0;
 	int update = full_update || update_hwc;
@@ -1410,8 +1414,9 @@
 
     /* allocate local memory */
     memory_region_init_ram(&s->local_mem_region, NULL, "sm501.local",
-                           local_mem_bytes, &error_abort);
+                           local_mem_bytes, &error_fatal);
     vmstate_register_ram_global(&s->local_mem_region);
+    memory_region_set_log(&s->local_mem_region, true, DIRTY_MEMORY_VGA);
     s->local_mem = memory_region_get_ram_ptr(&s->local_mem_region);
     memory_region_add_subregion(address_space_mem, base, &s->local_mem_region);
 
diff --git a/hw/display/ssd0303.c b/hw/display/ssd0303.c
index f6804fb..d301756 100644
--- a/hw/display/ssd0303.c
+++ b/hw/display/ssd0303.c
@@ -10,6 +10,7 @@
 /* The controller can support a variety of different displays, but we only
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
+#include "qemu/osdep.h"
 #include "hw/i2c/i2c.h"
 #include "ui/console.h"
 
diff --git a/hw/display/ssd0323.c b/hw/display/ssd0323.c
index 9727007..6d1faf4 100644
--- a/hw/display/ssd0323.c
+++ b/hw/display/ssd0323.c
@@ -10,7 +10,8 @@
 /* The controller can support a variety of different displays, but we only
    implement one.  Most of the commends relating to brightness and geometry
    setup are ignored. */
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 #include "ui/console.h"
 
 //#define DEBUG_SSD0323 1
@@ -360,7 +361,7 @@
     .gfx_update  = ssd0323_update_display,
 };
 
-static int ssd0323_init(SSISlave *d)
+static void ssd0323_realize(SSISlave *d, Error **errp)
 {
     DeviceState *dev = DEVICE(d);
     ssd0323_state *s = FROM_SSI_SLAVE(ssd0323_state, d);
@@ -374,14 +375,13 @@
 
     register_savevm(dev, "ssd0323_oled", -1, 1,
                     ssd0323_save, ssd0323_load, s);
-    return 0;
 }
 
 static void ssd0323_class_init(ObjectClass *klass, void *data)
 {
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = ssd0323_init;
+    k->realize = ssd0323_realize;
     k->transfer = ssd0323_transfer;
     k->cs_polarity = SSI_CS_HIGH;
 }
diff --git a/hw/display/tc6393xb.c b/hw/display/tc6393xb.c
index 4306adc..92f7120 100644
--- a/hw/display/tc6393xb.c
+++ b/hw/display/tc6393xb.c
@@ -10,6 +10,9 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/host-utils.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
 #include "hw/block/flash.h"
@@ -171,7 +174,7 @@
     level = s->gpio_level & s->gpio_dir;
 
     for (diff = s->prev_level ^ level; diff; diff ^= 1 << bit) {
-        bit = ffs(diff) - 1;
+        bit = ctz32(diff);
         qemu_set_irq(s->handler[bit], (level >> bit) & 1);
     }
 
@@ -571,7 +574,7 @@
     s->irq = irq;
     s->gpio_in = qemu_allocate_irqs(tc6393xb_gpio_set, s, TC6393XB_GPIOS);
 
-    s->l3v = *qemu_allocate_irqs(tc6393xb_l3v, s, 1);
+    s->l3v = qemu_allocate_irq(tc6393xb_l3v, s, 0);
     s->blanked = 1;
 
     s->sub_irqs = qemu_allocate_irqs(tc6393xb_sub_irq, s, TC6393XB_NR_IRQS);
@@ -584,7 +587,7 @@
     memory_region_add_subregion(sysmem, base, &s->iomem);
 
     memory_region_init_ram(&s->vram, NULL, "tc6393xb.vram", 0x100000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
     memory_region_add_subregion(sysmem, base + 0x100000, &s->vram);
diff --git a/hw/display/tcx.c b/hw/display/tcx.c
index a9f9f66..8e26aae 100644
--- a/hw/display/tcx.c
+++ b/hw/display/tcx.c
@@ -22,11 +22,15 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "cpu.h" /* FIXME shouldn't use TARGET_PAGE_SIZE */
 #include "ui/console.h"
 #include "ui/pixel_ops.h"
 #include "hw/loader.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
 
 #define TCX_ROM_FILE "QEMU,tcx.bin"
 #define FCODE_MAX_ROM_SIZE 0x10000
@@ -353,6 +357,7 @@
         return;
     }
 
+    memory_region_sync_dirty_bitmap(&ts->vram_mem);
     for (y = 0; y < ts->height; page += TARGET_PAGE_SIZE) {
         if (memory_region_get_dirty(&ts->vram_mem, page, TARGET_PAGE_SIZE,
                                     DIRTY_MEMORY_VGA)) {
@@ -446,6 +451,7 @@
     dd = surface_stride(surface);
     ds = 1024;
 
+    memory_region_sync_dirty_bitmap(&ts->vram_mem);
     for (y = 0; y < ts->height; page += TARGET_PAGE_SIZE,
             page24 += TARGET_PAGE_SIZE, cpage += TARGET_PAGE_SIZE) {
         if (tcx24_check_dirty(ts, page, page24, cpage)) {
@@ -941,57 +947,55 @@
     SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     TCXState *s = TCX(obj);
 
-    memory_region_init_ram(&s->rom, NULL, "tcx.prom", FCODE_MAX_ROM_SIZE,
-                           &error_abort);
+    memory_region_init_ram(&s->rom, obj, "tcx.prom", FCODE_MAX_ROM_SIZE,
+                           &error_fatal);
     memory_region_set_readonly(&s->rom, true);
     sysbus_init_mmio(sbd, &s->rom);
 
     /* 2/STIP : Stippler */
-    memory_region_init_io(&s->stip, OBJECT(s), &tcx_stip_ops, s, "tcx.stip",
+    memory_region_init_io(&s->stip, obj, &tcx_stip_ops, s, "tcx.stip",
                           TCX_STIP_NREGS);
     sysbus_init_mmio(sbd, &s->stip);
 
     /* 3/BLIT : Blitter */
-    memory_region_init_io(&s->blit, OBJECT(s), &tcx_blit_ops, s, "tcx.blit",
+    memory_region_init_io(&s->blit, obj, &tcx_blit_ops, s, "tcx.blit",
                           TCX_BLIT_NREGS);
     sysbus_init_mmio(sbd, &s->blit);
 
     /* 5/RSTIP : Raw Stippler */
-    memory_region_init_io(&s->rstip, OBJECT(s), &tcx_rstip_ops, s, "tcx.rstip",
+    memory_region_init_io(&s->rstip, obj, &tcx_rstip_ops, s, "tcx.rstip",
                           TCX_RSTIP_NREGS);
     sysbus_init_mmio(sbd, &s->rstip);
 
     /* 6/RBLIT : Raw Blitter */
-    memory_region_init_io(&s->rblit, OBJECT(s), &tcx_rblit_ops, s, "tcx.rblit",
+    memory_region_init_io(&s->rblit, obj, &tcx_rblit_ops, s, "tcx.rblit",
                           TCX_RBLIT_NREGS);
     sysbus_init_mmio(sbd, &s->rblit);
 
     /* 7/TEC : ??? */
-    memory_region_init_io(&s->tec, OBJECT(s), &tcx_dummy_ops, s,
-                          "tcx.tec", TCX_TEC_NREGS);
+    memory_region_init_io(&s->tec, obj, &tcx_dummy_ops, s, "tcx.tec",
+                          TCX_TEC_NREGS);
     sysbus_init_mmio(sbd, &s->tec);
 
     /* 8/CMAP : DAC */
-    memory_region_init_io(&s->dac, OBJECT(s), &tcx_dac_ops, s,
-                          "tcx.dac", TCX_DAC_NREGS);
+    memory_region_init_io(&s->dac, obj, &tcx_dac_ops, s, "tcx.dac",
+                          TCX_DAC_NREGS);
     sysbus_init_mmio(sbd, &s->dac);
 
     /* 9/THC : Cursor */
-    memory_region_init_io(&s->thc, OBJECT(s), &tcx_thc_ops, s, "tcx.thc",
+    memory_region_init_io(&s->thc, obj, &tcx_thc_ops, s, "tcx.thc",
                           TCX_THC_NREGS);
     sysbus_init_mmio(sbd, &s->thc);
 
     /* 11/DHC : ??? */
-    memory_region_init_io(&s->dhc, OBJECT(s), &tcx_dummy_ops, s, "tcx.dhc",
+    memory_region_init_io(&s->dhc, obj, &tcx_dummy_ops, s, "tcx.dhc",
                           TCX_DHC_NREGS);
     sysbus_init_mmio(sbd, &s->dhc);
 
     /* 12/ALT : ??? */
-    memory_region_init_io(&s->alt, OBJECT(s), &tcx_dummy_ops, s, "tcx.alt",
+    memory_region_init_io(&s->alt, obj, &tcx_dummy_ops, s, "tcx.alt",
                           TCX_ALT_NREGS);
     sysbus_init_mmio(sbd, &s->alt);
-
-    return;
 }
 
 static void tcx_realizefn(DeviceState *dev, Error **errp)
@@ -1004,8 +1008,9 @@
     char *fcode_filename;
 
     memory_region_init_ram(&s->vram_mem, OBJECT(s), "tcx.vram",
-                           s->vram_size * (1 + 4 + 4), &error_abort);
+                           s->vram_size * (1 + 4 + 4), &error_fatal);
     vmstate_register_ram_global(&s->vram_mem);
+    memory_region_set_log(&s->vram_mem, true, DIRTY_MEMORY_VGA);
     vram_base = memory_region_get_ram_ptr(&s->vram_mem);
 
     /* 10/ROM : FCode ROM */
@@ -1014,6 +1019,7 @@
     if (fcode_filename) {
         ret = load_image_targphys(fcode_filename, s->prom_addr,
                                   FCODE_MAX_ROM_SIZE);
+        g_free(fcode_filename);
         if (ret < 0 || ret > FCODE_MAX_ROM_SIZE) {
             error_report("tcx: could not load prom '%s'", TCX_ROM_FILE);
         }
diff --git a/hw/display/trace-events b/hw/display/trace-events
new file mode 100644
index 0000000..33a5eba
--- /dev/null
+++ b/hw/display/trace-events
@@ -0,0 +1,128 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/display/jazz_led.c
+jazz_led_read(uint64_t addr, uint8_t val) "read addr=0x%"PRIx64": 0x%x"
+jazz_led_write(uint64_t addr, uint8_t new) "write addr=0x%"PRIx64": 0x%x"
+
+# hw/display/xenfb.c
+xenfb_mouse_event(void *opaque, int dx, int dy, int dz, int button_state, int abs_pointer_wanted) "%p x %d y %d z %d bs %#x abs %d"
+xenfb_input_connected(void *xendev, int abs_pointer_wanted) "%p abs %d"
+
+# hw/display/g364fb.c
+g364fb_read(uint64_t addr, uint32_t val) "read addr=0x%"PRIx64": 0x%x"
+g364fb_write(uint64_t addr, uint32_t new) "write addr=0x%"PRIx64": 0x%x"
+
+# hw/display/milkymist-tmu2.c
+milkymist_tmu2_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_tmu2_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_tmu2_start(void) "Start TMU"
+milkymist_tmu2_pulse_irq(void) "Pulse IRQ"
+
+# hw/display/milkymist-vgafb.c
+milkymist_vgafb_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_vgafb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+
+# hw/display/vmware_vga.c
+vmware_value_read(uint32_t index, uint32_t value) "index %d, value 0x%x"
+vmware_value_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
+vmware_palette_read(uint32_t index, uint32_t value) "index %d, value 0x%x"
+vmware_palette_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
+vmware_scratch_read(uint32_t index, uint32_t value) "index %d, value 0x%x"
+vmware_scratch_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
+vmware_setmode(uint32_t w, uint32_t h, uint32_t bpp) "%dx%d @ %d bpp"
+
+# hw/display/virtio-gpu.c
+virtio_gpu_features(bool virgl) "virgl %d"
+virtio_gpu_cmd_get_display_info(void) ""
+virtio_gpu_cmd_get_caps(void) ""
+virtio_gpu_cmd_set_scanout(uint32_t id, uint32_t res, uint32_t w, uint32_t h, uint32_t x, uint32_t y) "id %d, res 0x%x, w %d, h %d, x %d, y %d"
+virtio_gpu_cmd_res_create_2d(uint32_t res, uint32_t fmt, uint32_t w, uint32_t h) "res 0x%x, fmt 0x%x, w %d, h %d"
+virtio_gpu_cmd_res_create_3d(uint32_t res, uint32_t fmt, uint32_t w, uint32_t h, uint32_t d) "res 0x%x, fmt 0x%x, w %d, h %d, d %d"
+virtio_gpu_cmd_res_unref(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_back_attach(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_back_detach(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_toh_2d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_toh_3d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_xfer_fromh_3d(uint32_t res) "res 0x%x"
+virtio_gpu_cmd_res_flush(uint32_t res, uint32_t w, uint32_t h, uint32_t x, uint32_t y) "res 0x%x, w %d, h %d, x %d, y %d"
+virtio_gpu_cmd_ctx_create(uint32_t ctx, const char *name) "ctx 0x%x, name %s"
+virtio_gpu_cmd_ctx_destroy(uint32_t ctx) "ctx 0x%x"
+virtio_gpu_cmd_ctx_res_attach(uint32_t ctx, uint32_t res) "ctx 0x%x, res 0x%x"
+virtio_gpu_cmd_ctx_res_detach(uint32_t ctx, uint32_t res) "ctx 0x%x, res 0x%x"
+virtio_gpu_cmd_ctx_submit(uint32_t ctx, uint32_t size) "ctx 0x%x, size %d"
+virtio_gpu_update_cursor(uint32_t scanout, uint32_t x, uint32_t y, const char *type, uint32_t res) "scanout %d, x %d, y %d, %s, res 0x%x"
+virtio_gpu_fence_ctrl(uint64_t fence, uint32_t type) "fence 0x%" PRIx64 ", type 0x%x"
+virtio_gpu_fence_resp(uint64_t fence) "fence 0x%" PRIx64
+
+# hw/display/qxl.c
+disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
+disable qxl_io_write_vga(int qid, const char *mode, uint32_t addr, uint32_t val) "%d %s addr=%u val=%u"
+qxl_create_guest_primary(int qid, uint32_t width, uint32_t height, uint64_t mem, uint32_t format, uint32_t position) "%d %ux%u mem=%" PRIx64 " %u,%u"
+qxl_create_guest_primary_rest(int qid, int32_t stride, uint32_t type, uint32_t flags) "%d %d,%d,%d"
+qxl_destroy_primary(int qid) "%d"
+qxl_enter_vga_mode(int qid) "%d"
+qxl_exit_vga_mode(int qid) "%d"
+qxl_hard_reset(int qid, int64_t loadvm) "%d loadvm=%"PRId64
+qxl_interface_async_complete_io(int qid, uint32_t current_async, void *cookie) "%d current=%d cookie=%p"
+qxl_interface_attach_worker(int qid) "%d"
+qxl_interface_get_init_info(int qid) "%d"
+qxl_interface_set_compression_level(int qid, int64_t level) "%d %"PRId64
+qxl_interface_update_area_complete(int qid, uint32_t surface_id, uint32_t dirty_left, uint32_t dirty_right, uint32_t dirty_top, uint32_t dirty_bottom) "%d surface=%d [%d,%d,%d,%d]"
+qxl_interface_update_area_complete_rest(int qid, uint32_t num_updated_rects) "%d #=%d"
+qxl_interface_update_area_complete_overflow(int qid, int max) "%d max=%d"
+qxl_interface_update_area_complete_schedule_bh(int qid, uint32_t num_dirty) "%d #dirty=%d"
+qxl_io_destroy_primary_ignored(int qid, const char *mode) "%d %s"
+qxl_io_log(int qid, const uint8_t *log_buf) "%d %s"
+qxl_io_read_unexpected(int qid) "%d"
+qxl_io_unexpected_vga_mode(int qid, uint64_t addr, uint64_t val, const char *desc) "%d 0x%"PRIx64"=%"PRIu64" (%s)"
+qxl_io_write(int qid, const char *mode, uint64_t addr, const char *aname, uint64_t val, unsigned size, int async) "%d %s addr=%"PRIu64 " (%s) val=%"PRIu64" size=%u async=%d"
+qxl_memslot_add_guest(int qid, uint32_t slot_id, uint64_t guest_start, uint64_t guest_end) "%d %u: guest phys 0x%"PRIx64 " - 0x%" PRIx64
+qxl_post_load(int qid, const char *mode) "%d %s"
+qxl_pre_load(int qid) "%d"
+qxl_pre_save(int qid) "%d"
+qxl_reset_surfaces(int qid) "%d"
+qxl_ring_command_check(int qid, const char *mode) "%d %s"
+qxl_ring_command_get(int qid, const char *mode) "%d %s"
+qxl_ring_command_req_notification(int qid) "%d"
+qxl_ring_cursor_check(int qid, const char *mode) "%d %s"
+qxl_ring_cursor_get(int qid, const char *mode) "%d %s"
+qxl_ring_cursor_req_notification(int qid) "%d"
+qxl_ring_res_push(int qid, const char *mode, uint32_t surface_count, uint32_t free_res, void *last_release, const char *notify) "%d %s s#=%d res#=%d last=%p notify=%s"
+qxl_ring_res_push_rest(int qid, uint32_t ring_has, uint32_t ring_size, uint32_t prod, uint32_t cons) "%d ring %d/%d [%d,%d]"
+qxl_ring_res_put(int qid, uint32_t free_res) "%d #res=%d"
+qxl_set_mode(int qid, int modenr, uint32_t x_res, uint32_t y_res, uint32_t bits, uint64_t devmem) "%d mode=%d [ x=%d y=%d @ bpp=%d devmem=0x%" PRIx64 " ]"
+qxl_soft_reset(int qid) "%d"
+qxl_spice_destroy_surfaces_complete(int qid) "%d"
+qxl_spice_destroy_surfaces(int qid, int async) "%d async=%d"
+qxl_spice_destroy_surface_wait_complete(int qid, uint32_t id) "%d sid=%d"
+qxl_spice_destroy_surface_wait(int qid, uint32_t id, int async) "%d sid=%d async=%d"
+qxl_spice_flush_surfaces_async(int qid, uint32_t surface_count, uint32_t num_free_res) "%d s#=%d, res#=%d"
+qxl_spice_monitors_config(int qid) "%d"
+qxl_spice_loadvm_commands(int qid, void *ext, uint32_t count) "%d ext=%p count=%d"
+qxl_spice_oom(int qid) "%d"
+qxl_spice_reset_cursor(int qid) "%d"
+qxl_spice_reset_image_cache(int qid) "%d"
+qxl_spice_reset_memslots(int qid) "%d"
+qxl_spice_update_area(int qid, uint32_t surface_id, uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "%d sid=%d [%d,%d,%d,%d]"
+qxl_spice_update_area_rest(int qid, uint32_t num_dirty_rects, uint32_t clear_dirty_region) "%d #d=%d clear=%d"
+qxl_surfaces_dirty(int qid, uint64_t offset, uint64_t size) "%d offset=0x%"PRIx64" size=0x%"PRIx64
+qxl_send_events(int qid, uint32_t events) "%d %d"
+qxl_send_events_vm_stopped(int qid, uint32_t events) "%d %d"
+qxl_set_guest_bug(int qid) "%d"
+qxl_interrupt_client_monitors_config(int qid, int num_heads, void *heads) "%d %d %p"
+qxl_client_monitors_config_unsupported_by_guest(int qid, uint32_t int_mask, void *client_monitors_config) "%d %X %p"
+qxl_client_monitors_config_unsupported_by_device(int qid, int revision) "%d revision=%d"
+qxl_client_monitors_config_capped(int qid, int requested, int limit) "%d %d %d"
+qxl_client_monitors_config_crc(int qid, unsigned size, uint32_t crc32) "%d %u %u"
+qxl_set_client_capabilities_unsupported_by_revision(int qid, int revision) "%d revision=%d"
+
+# hw/display/qxl-render.c
+qxl_render_blit(int32_t stride, int32_t left, int32_t right, int32_t top, int32_t bottom) "stride=%d [%d, %d, %d, %d]"
+qxl_render_guest_primary_resized(int32_t width, int32_t height, int32_t stride, int32_t bytes_pp, int32_t bits_pp) "%dx%d, stride %d, bpp %d, depth %d"
+qxl_render_update_area_done(void *cookie) "%p"
+
+# hw/display/goldfish_fb.c
+goldfish_fb_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+goldfish_fb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+goldfish_fb_update_display(int y, int h, int x, int w) "y:%d,h:%d,x=%d,w=%d"
+goldfish_fb_update_stats(float peak, float total) "peak %.2f %%  total %.2f %%"
diff --git a/hw/display/vga-isa-mm.c b/hw/display/vga-isa-mm.c
index 4efc222..51ccbcc 100644
--- a/hw/display/vga-isa-mm.c
+++ b/hw/display/vga-isa-mm.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/i386/pc.h"
diff --git a/hw/display/vga-isa.c b/hw/display/vga-isa.c
index 2b480bd..f5aff1c 100644
--- a/hw/display/vga-isa.c
+++ b/hw/display/vga-isa.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/i386/pc.h"
@@ -64,7 +65,7 @@
         isa_register_portio_list(isadev, 0x1ce, vbe_ports, s, "vbe");
     }
     memory_region_add_subregion_overlap(isa_address_space(isadev),
-                                        isa_mem_base + 0x000a0000,
+                                        0x000a0000,
                                         vga_io_memory, 1);
     memory_region_set_coalescing(vga_io_memory);
     s->con = graphic_console_init(DEVICE(dev), 0, s->hw_ops, s);
diff --git a/hw/display/vga-pci.c b/hw/display/vga-pci.c
index db922f1..ac9a764 100644
--- a/hw/display/vga-pci.c
+++ b/hw/display/vga-pci.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/pci/pci.h"
@@ -54,11 +55,12 @@
     VGACommonState vga;
     uint32_t flags;
     MemoryRegion mmio;
-    MemoryRegion ioport;
-    MemoryRegion bochs;
-    MemoryRegion qext;
+    MemoryRegion mrs[3];
 } PCIVGAState;
 
+#define TYPE_PCI_VGA "pci-vga"
+#define PCI_VGA(obj) OBJECT_CHECK(PCIVGAState, (obj), TYPE_PCI_VGA)
+
 static const VMStateDescription vmstate_vga_pci = {
     .name = "vga",
     .version_id = 2,
@@ -73,16 +75,16 @@
 static uint64_t pci_vga_ioport_read(void *ptr, hwaddr addr,
                                     unsigned size)
 {
-    PCIVGAState *d = ptr;
+    VGACommonState *s = ptr;
     uint64_t ret = 0;
 
     switch (size) {
     case 1:
-        ret = vga_ioport_read(&d->vga, addr);
+        ret = vga_ioport_read(s, addr + 0x3c0);
         break;
     case 2:
-        ret  = vga_ioport_read(&d->vga, addr);
-        ret |= vga_ioport_read(&d->vga, addr+1) << 8;
+        ret  = vga_ioport_read(s, addr + 0x3c0);
+        ret |= vga_ioport_read(s, addr + 0x3c1) << 8;
         break;
     }
     return ret;
@@ -91,11 +93,11 @@
 static void pci_vga_ioport_write(void *ptr, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
-    PCIVGAState *d = ptr;
+    VGACommonState *s = ptr;
 
     switch (size) {
     case 1:
-        vga_ioport_write(&d->vga, addr + 0x3c0, val);
+        vga_ioport_write(s, addr + 0x3c0, val);
         break;
     case 2:
         /*
@@ -103,8 +105,8 @@
          * indexed registers with a single word write because the
          * index byte is updated first.
          */
-        vga_ioport_write(&d->vga, addr + 0x3c0, val & 0xff);
-        vga_ioport_write(&d->vga, addr + 0x3c1, (val >> 8) & 0xff);
+        vga_ioport_write(s, addr + 0x3c0, val & 0xff);
+        vga_ioport_write(s, addr + 0x3c1, (val >> 8) & 0xff);
         break;
     }
 }
@@ -122,21 +124,21 @@
 static uint64_t pci_vga_bochs_read(void *ptr, hwaddr addr,
                                    unsigned size)
 {
-    PCIVGAState *d = ptr;
+    VGACommonState *s = ptr;
     int index = addr >> 1;
 
-    vbe_ioport_write_index(&d->vga, 0, index);
-    return vbe_ioport_read_data(&d->vga, 0);
+    vbe_ioport_write_index(s, 0, index);
+    return vbe_ioport_read_data(s, 0);
 }
 
 static void pci_vga_bochs_write(void *ptr, hwaddr addr,
                                 uint64_t val, unsigned size)
 {
-    PCIVGAState *d = ptr;
+    VGACommonState *s = ptr;
     int index = addr >> 1;
 
-    vbe_ioport_write_index(&d->vga, 0, index);
-    vbe_ioport_write_data(&d->vga, 0, val);
+    vbe_ioport_write_index(s, 0, index);
+    vbe_ioport_write_data(s, 0, val);
 }
 
 static const MemoryRegionOps pci_vga_bochs_ops = {
@@ -151,13 +153,13 @@
 
 static uint64_t pci_vga_qext_read(void *ptr, hwaddr addr, unsigned size)
 {
-    PCIVGAState *d = ptr;
+    VGACommonState *s = ptr;
 
     switch (addr) {
     case PCI_VGA_QEXT_REG_SIZE:
         return PCI_VGA_QEXT_SIZE;
     case PCI_VGA_QEXT_REG_BYTEORDER:
-        return d->vga.big_endian_fb ?
+        return s->big_endian_fb ?
             PCI_VGA_QEXT_BIG_ENDIAN : PCI_VGA_QEXT_LITTLE_ENDIAN;
     default:
         return 0;
@@ -167,20 +169,34 @@
 static void pci_vga_qext_write(void *ptr, hwaddr addr,
                                uint64_t val, unsigned size)
 {
-    PCIVGAState *d = ptr;
+    VGACommonState *s = ptr;
 
     switch (addr) {
     case PCI_VGA_QEXT_REG_BYTEORDER:
         if (val == PCI_VGA_QEXT_BIG_ENDIAN) {
-            d->vga.big_endian_fb = true;
+            s->big_endian_fb = true;
         }
         if (val == PCI_VGA_QEXT_LITTLE_ENDIAN) {
-            d->vga.big_endian_fb = false;
+            s->big_endian_fb = false;
         }
         break;
     }
 }
 
+static bool vga_get_big_endian_fb(Object *obj, Error **errp)
+{
+    PCIVGAState *d = PCI_VGA(PCI_DEVICE(obj));
+
+    return d->vga.big_endian_fb;
+}
+
+static void vga_set_big_endian_fb(Object *obj, bool value, Error **errp)
+{
+    PCIVGAState *d = PCI_VGA(PCI_DEVICE(obj));
+
+    d->vga.big_endian_fb = value;
+}
+
 static const MemoryRegionOps pci_vga_qext_ops = {
     .read = pci_vga_qext_read,
     .write = pci_vga_qext_write,
@@ -189,10 +205,34 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int pci_std_vga_initfn(PCIDevice *dev)
+void pci_std_vga_mmio_region_init(VGACommonState *s,
+                                  MemoryRegion *parent,
+                                  MemoryRegion *subs,
+                                  bool qext)
 {
-    PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
+    memory_region_init_io(&subs[0], NULL, &pci_vga_ioport_ops, s,
+                          "vga ioports remapped", PCI_VGA_IOPORT_SIZE);
+    memory_region_add_subregion(parent, PCI_VGA_IOPORT_OFFSET,
+                                &subs[0]);
+
+    memory_region_init_io(&subs[1], NULL, &pci_vga_bochs_ops, s,
+                          "bochs dispi interface", PCI_VGA_BOCHS_SIZE);
+    memory_region_add_subregion(parent, PCI_VGA_BOCHS_OFFSET,
+                                &subs[1]);
+
+    if (qext) {
+        memory_region_init_io(&subs[2], NULL, &pci_vga_qext_ops, s,
+                              "qemu extended regs", PCI_VGA_QEXT_SIZE);
+        memory_region_add_subregion(parent, PCI_VGA_QEXT_OFFSET,
+                                    &subs[2]);
+    }
+}
+
+static void pci_std_vga_realize(PCIDevice *dev, Error **errp)
+{
+    PCIVGAState *d = PCI_VGA(dev);
     VGACommonState *s = &d->vga;
+    bool qext = false;
 
     /* vga + console init */
     vga_common_init(s, OBJECT(dev), true);
@@ -207,23 +247,12 @@
     /* mmio bar for vga register access */
     if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_MMIO)) {
         memory_region_init(&d->mmio, NULL, "vga.mmio", 4096);
-        memory_region_init_io(&d->ioport, NULL, &pci_vga_ioport_ops, d,
-                              "vga ioports remapped", PCI_VGA_IOPORT_SIZE);
-        memory_region_init_io(&d->bochs, NULL, &pci_vga_bochs_ops, d,
-                              "bochs dispi interface", PCI_VGA_BOCHS_SIZE);
-
-        memory_region_add_subregion(&d->mmio, PCI_VGA_IOPORT_OFFSET,
-                                    &d->ioport);
-        memory_region_add_subregion(&d->mmio, PCI_VGA_BOCHS_OFFSET,
-                                    &d->bochs);
 
         if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_QEXT)) {
-            memory_region_init_io(&d->qext, NULL, &pci_vga_qext_ops, d,
-                                  "qemu extended regs", PCI_VGA_QEXT_SIZE);
-            memory_region_add_subregion(&d->mmio, PCI_VGA_QEXT_OFFSET,
-                                        &d->qext);
+            qext = true;
             pci_set_byte(&d->dev.config[PCI_REVISION_ID], 2);
         }
+        pci_std_vga_mmio_region_init(s, &d->mmio, d->mrs, qext);
 
         pci_register_bar(&d->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &d->mmio);
     }
@@ -232,14 +261,20 @@
         /* compatibility with pc-0.13 and older */
         vga_init_vbe(s, OBJECT(dev), pci_address_space(dev));
     }
-
-    return 0;
 }
 
-static int pci_secondary_vga_initfn(PCIDevice *dev)
+static void pci_std_vga_init(Object *obj)
 {
-    PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
+    /* Expose framebuffer byteorder via QOM */
+    object_property_add_bool(obj, "big-endian-framebuffer",
+                             vga_get_big_endian_fb, vga_set_big_endian_fb, NULL);
+}
+
+static void pci_secondary_vga_realize(PCIDevice *dev, Error **errp)
+{
+    PCIVGAState *d = PCI_VGA(dev);
     VGACommonState *s = &d->vga;
+    bool qext = false;
 
     /* vga + console init */
     vga_common_init(s, OBJECT(dev), false);
@@ -247,34 +282,27 @@
 
     /* mmio bar */
     memory_region_init(&d->mmio, OBJECT(dev), "vga.mmio", 4096);
-    memory_region_init_io(&d->ioport, OBJECT(dev), &pci_vga_ioport_ops, d,
-                          "vga ioports remapped", PCI_VGA_IOPORT_SIZE);
-    memory_region_init_io(&d->bochs, OBJECT(dev), &pci_vga_bochs_ops, d,
-                          "bochs dispi interface", PCI_VGA_BOCHS_SIZE);
-
-    memory_region_add_subregion(&d->mmio, PCI_VGA_IOPORT_OFFSET,
-                                &d->ioport);
-    memory_region_add_subregion(&d->mmio, PCI_VGA_BOCHS_OFFSET,
-                                &d->bochs);
 
     if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_QEXT)) {
-        memory_region_init_io(&d->qext, NULL, &pci_vga_qext_ops, d,
-                              "qemu extended regs", PCI_VGA_QEXT_SIZE);
-        memory_region_add_subregion(&d->mmio, PCI_VGA_QEXT_OFFSET,
-                                    &d->qext);
+        qext = true;
         pci_set_byte(&d->dev.config[PCI_REVISION_ID], 2);
     }
+    pci_std_vga_mmio_region_init(s, &d->mmio, d->mrs, qext);
 
     pci_register_bar(&d->dev, 0, PCI_BASE_ADDRESS_MEM_PREFETCH, &s->vram);
     pci_register_bar(&d->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &d->mmio);
+}
 
-    return 0;
+static void pci_secondary_vga_init(Object *obj)
+{
+    /* Expose framebuffer byteorder via QOM */
+    object_property_add_bool(obj, "big-endian-framebuffer",
+                             vga_get_big_endian_fb, vga_set_big_endian_fb, NULL);
 }
 
 static void pci_secondary_vga_reset(DeviceState *dev)
 {
-    PCIVGAState *d = DO_UPCAST(PCIVGAState, dev.qdev, dev);
-
+    PCIVGAState *d = PCI_VGA(PCI_DEVICE(dev));
     vga_common_reset(&d->vga);
 }
 
@@ -293,20 +321,35 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void vga_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->vendor_id = PCI_VENDOR_ID_QEMU;
+    k->device_id = PCI_DEVICE_ID_QEMU_VGA;
+    dc->vmsd = &vmstate_vga_pci;
+    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
+}
+
+static const TypeInfo vga_pci_type_info = {
+    .name = TYPE_PCI_VGA,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIVGAState),
+    .abstract = true,
+    .class_init = vga_pci_class_init,
+};
+
 static void vga_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_std_vga_initfn;
+    k->realize = pci_std_vga_realize;
     k->romfile = "vgabios-stdvga.bin";
-    k->vendor_id = PCI_VENDOR_ID_QEMU;
-    k->device_id = PCI_DEVICE_ID_QEMU_VGA;
     k->class_id = PCI_CLASS_DISPLAY_VGA;
-    dc->vmsd = &vmstate_vga_pci;
     dc->props = vga_pci_properties;
     dc->hotpluggable = false;
-    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
 }
 
 static void secondary_class_init(ObjectClass *klass, void *data)
@@ -314,31 +357,29 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_secondary_vga_initfn;
-    k->vendor_id = PCI_VENDOR_ID_QEMU;
-    k->device_id = PCI_DEVICE_ID_QEMU_VGA;
+    k->realize = pci_secondary_vga_realize;
     k->class_id = PCI_CLASS_DISPLAY_OTHER;
-    dc->vmsd = &vmstate_vga_pci;
     dc->props = secondary_pci_properties;
     dc->reset = pci_secondary_vga_reset;
 }
 
 static const TypeInfo vga_info = {
     .name          = "VGA",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PCIVGAState),
+    .parent        = TYPE_PCI_VGA,
+    .instance_init = pci_std_vga_init,
     .class_init    = vga_class_init,
 };
 
 static const TypeInfo secondary_info = {
     .name          = "secondary-vga",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PCIVGAState),
+    .parent        = TYPE_PCI_VGA,
+    .instance_init = pci_secondary_vga_init,
     .class_init    = secondary_class_init,
 };
 
 static void vga_register_types(void)
 {
+    type_register_static(&vga_pci_type_info);
     type_register_static(&vga_info);
     type_register_static(&secondary_info);
 }
diff --git a/hw/display/vga.c b/hw/display/vga.c
index 52eaf05..2a88b3c 100644
--- a/hw/display/vga.c
+++ b/hw/display/vga.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "vga.h"
 #include "ui/console.h"
@@ -140,6 +142,18 @@
 static uint16_t expand2[256];
 static uint8_t expand4to8[16];
 
+static void vbe_update_vgaregs(VGACommonState *s);
+
+static inline bool vbe_enabled(VGACommonState *s)
+{
+    return s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED;
+}
+
+static inline uint8_t sr(VGACommonState *s, int idx)
+{
+    return vbe_enabled(s) ? s->sr_vbe[idx] : s->sr[idx];
+}
+
 static void vga_update_memory_access(VGACommonState *s)
 {
     hwaddr base, offset, size;
@@ -154,8 +168,8 @@
         s->has_chain4_alias = false;
         s->plane_updated = 0xf;
     }
-    if ((s->sr[VGA_SEQ_PLANE_WRITE] & VGA_SR02_ALL_PLANES) ==
-        VGA_SR02_ALL_PLANES && s->sr[VGA_SEQ_MEMORY_MODE] & VGA_SR04_CHN_4M) {
+    if ((sr(s, VGA_SEQ_PLANE_WRITE) & VGA_SR02_ALL_PLANES) ==
+        VGA_SR02_ALL_PLANES && sr(s, VGA_SEQ_MEMORY_MODE) & VGA_SR04_CHN_4M) {
         offset = 0;
         switch ((s->gr[VGA_GFX_MISC] >> 2) & 3) {
         case 0:
@@ -177,7 +191,7 @@
             size = 0x8000;
             break;
         }
-        base += isa_mem_base;
+        assert(offset + size <= s->vram_size);
         memory_region_init_alias(&s->chain4_alias, memory_region_owner(&s->vram),
                                  "vga.chain4", &s->vram, offset, size);
         memory_region_add_subregion_overlap(s->legacy_address_space, base,
@@ -225,7 +239,7 @@
           ((s->cr[VGA_CRTC_OVERFLOW] >> 6) & 2)) << 8);
     vretr_end_line = s->cr[VGA_CRTC_V_SYNC_END] & 0xf;
 
-    clocking_mode = (s->sr[VGA_SEQ_CLOCK_MODE] >> 3) & 1;
+    clocking_mode = (sr(s, VGA_SEQ_CLOCK_MODE) >> 3) & 1;
     clock_sel = (s->msr >> 2) & 3;
     dots = (s->msr & 1) ? 8 : 9;
 
@@ -235,9 +249,9 @@
 
     r->total_chars = vtotal_lines * htotal_chars;
     if (r->freq) {
-        r->ticks_per_char = get_ticks_per_sec() / (r->total_chars * r->freq);
+        r->ticks_per_char = NANOSECONDS_PER_SECOND / (r->total_chars * r->freq);
     } else {
-        r->ticks_per_char = get_ticks_per_sec() / chars_per_sec;
+        r->ticks_per_char = NANOSECONDS_PER_SECOND / chars_per_sec;
     }
 
     r->vstart = vretr_start_line;
@@ -265,7 +279,7 @@
         "dots = %d\n"
         "ticks/char = %" PRId64 "\n"
         "\n",
-        (double) get_ticks_per_sec() / (r->ticks_per_char * r->total_chars),
+        (double) NANOSECONDS_PER_SECOND / (r->ticks_per_char * r->total_chars),
         htotal_chars,
         hretr_start_char,
         hretr_skew_chars,
@@ -508,6 +522,7 @@
         printf("vga: write GR%x = 0x%02x\n", s->gr_index, val);
 #endif
         s->gr[s->gr_index] = val & gr_mask[s->gr_index];
+        vbe_update_vgaregs(s);
         vga_update_memory_access(s);
         break;
     case VGA_CRT_IM:
@@ -526,10 +541,12 @@
             if (s->cr_index == VGA_CRTC_OVERFLOW) {
                 s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x10) |
                     (val & 0x10);
+                vbe_update_vgaregs(s);
             }
             return;
         }
         s->cr[s->cr_index] = val;
+        vbe_update_vgaregs(s);
 
         switch(s->cr_index) {
         case VGA_CRTC_H_TOTAL:
@@ -562,7 +579,7 @@
     uint16_t *r = s->vbe_regs;
     uint32_t bits, linelength, maxy, offset;
 
-    if (!(r[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
+    if (!vbe_enabled(s)) {
         /* vbe is turned off -- nothing to do */
         return;
     }
@@ -637,12 +654,53 @@
     s->vbe_start_addr  = offset / 4;
 }
 
+/* we initialize the VGA graphic mode */
+static void vbe_update_vgaregs(VGACommonState *s)
+{
+    int h, shift_control;
+
+    if (!vbe_enabled(s)) {
+        /* vbe is turned off -- nothing to do */
+        return;
+    }
+
+    /* graphic mode + memory map 1 */
+    s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
+        VGA_GR06_GRAPHICS_MODE;
+    s->cr[VGA_CRTC_MODE] |= 3; /* no CGA modes */
+    s->cr[VGA_CRTC_OFFSET] = s->vbe_line_offset >> 3;
+    /* width */
+    s->cr[VGA_CRTC_H_DISP] =
+        (s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 3) - 1;
+    /* height (only meaningful if < 1024) */
+    h = s->vbe_regs[VBE_DISPI_INDEX_YRES] - 1;
+    s->cr[VGA_CRTC_V_DISP_END] = h;
+    s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x42) |
+        ((h >> 7) & 0x02) | ((h >> 3) & 0x40);
+    /* line compare to 1023 */
+    s->cr[VGA_CRTC_LINE_COMPARE] = 0xff;
+    s->cr[VGA_CRTC_OVERFLOW] |= 0x10;
+    s->cr[VGA_CRTC_MAX_SCAN] |= 0x40;
+
+    if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
+        shift_control = 0;
+        s->sr_vbe[VGA_SEQ_CLOCK_MODE] &= ~8; /* no double line */
+    } else {
+        shift_control = 2;
+        /* set chain 4 mode */
+        s->sr_vbe[VGA_SEQ_MEMORY_MODE] |= VGA_SR04_CHN_4M;
+        /* activate all planes */
+        s->sr_vbe[VGA_SEQ_PLANE_WRITE] |= VGA_SR02_ALL_PLANES;
+    }
+    s->gr[VGA_GFX_MODE] = (s->gr[VGA_GFX_MODE] & ~0x60) |
+        (shift_control << 5);
+    s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
+}
+
 static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
-    uint32_t val;
-    val = s->vbe_index;
-    return val;
+    return s->vbe_index;
 }
 
 uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
@@ -713,13 +771,10 @@
         case VBE_DISPI_INDEX_Y_OFFSET:
             s->vbe_regs[s->vbe_index] = val;
             vbe_fixup_regs(s);
+            vbe_update_vgaregs(s);
             break;
         case VBE_DISPI_INDEX_BANK:
-            if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
-              val &= (s->vbe_bank_mask >> 2);
-            } else {
-              val &= s->vbe_bank_mask;
-            }
+            val &= s->vbe_bank_mask;
             s->vbe_regs[s->vbe_index] = val;
             s->bank_offset = (val << 16);
             vga_update_memory_access(s);
@@ -727,52 +782,19 @@
         case VBE_DISPI_INDEX_ENABLE:
             if ((val & VBE_DISPI_ENABLED) &&
                 !(s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED)) {
-                int h, shift_control;
 
                 s->vbe_regs[VBE_DISPI_INDEX_VIRT_WIDTH] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_X_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_Y_OFFSET] = 0;
                 s->vbe_regs[VBE_DISPI_INDEX_ENABLE] |= VBE_DISPI_ENABLED;
                 vbe_fixup_regs(s);
+                vbe_update_vgaregs(s);
 
                 /* clear the screen */
                 if (!(val & VBE_DISPI_NOCLEARMEM)) {
                     memset(s->vram_ptr, 0,
                            s->vbe_regs[VBE_DISPI_INDEX_YRES] * s->vbe_line_offset);
                 }
-
-                /* we initialize the VGA graphic mode */
-                /* graphic mode + memory map 1 */
-                s->gr[VGA_GFX_MISC] = (s->gr[VGA_GFX_MISC] & ~0x0c) | 0x04 |
-                    VGA_GR06_GRAPHICS_MODE;
-                s->cr[VGA_CRTC_MODE] |= 3; /* no CGA modes */
-                s->cr[VGA_CRTC_OFFSET] = s->vbe_line_offset >> 3;
-                /* width */
-                s->cr[VGA_CRTC_H_DISP] =
-                    (s->vbe_regs[VBE_DISPI_INDEX_XRES] >> 3) - 1;
-                /* height (only meaningful if < 1024) */
-                h = s->vbe_regs[VBE_DISPI_INDEX_YRES] - 1;
-                s->cr[VGA_CRTC_V_DISP_END] = h;
-                s->cr[VGA_CRTC_OVERFLOW] = (s->cr[VGA_CRTC_OVERFLOW] & ~0x42) |
-                    ((h >> 7) & 0x02) | ((h >> 3) & 0x40);
-                /* line compare to 1023 */
-                s->cr[VGA_CRTC_LINE_COMPARE] = 0xff;
-                s->cr[VGA_CRTC_OVERFLOW] |= 0x10;
-                s->cr[VGA_CRTC_MAX_SCAN] |= 0x40;
-
-                if (s->vbe_regs[VBE_DISPI_INDEX_BPP] == 4) {
-                    shift_control = 0;
-                    s->sr[VGA_SEQ_CLOCK_MODE] &= ~8; /* no double line */
-                } else {
-                    shift_control = 2;
-                    /* set chain 4 mode */
-                    s->sr[VGA_SEQ_MEMORY_MODE] |= VGA_SR04_CHN_4M;
-                    /* activate all planes */
-                    s->sr[VGA_SEQ_PLANE_WRITE] |= VGA_SR02_ALL_PLANES;
-                }
-                s->gr[VGA_GFX_MODE] = (s->gr[VGA_GFX_MODE] & ~0x60) |
-                    (shift_control << 5);
-                s->cr[VGA_CRTC_MAX_SCAN] &= ~0x9f; /* no double scan */
             } else {
                 s->bank_offset = 0;
             }
@@ -816,15 +838,23 @@
         break;
     }
 
-    if (s->sr[VGA_SEQ_MEMORY_MODE] & VGA_SR04_CHN_4M) {
+    if (sr(s, VGA_SEQ_MEMORY_MODE) & VGA_SR04_CHN_4M) {
         /* chain 4 mode : simplest access */
+        assert(addr < s->vram_size);
         ret = s->vram_ptr[addr];
     } else if (s->gr[VGA_GFX_MODE] & 0x10) {
         /* odd/even mode (aka text mode mapping) */
         plane = (s->gr[VGA_GFX_PLANE_READ] & 2) | (addr & 1);
-        ret = s->vram_ptr[((addr & ~1) << 1) | plane];
+        addr = ((addr & ~1) << 1) | plane;
+        if (addr >= s->vram_size) {
+            return 0xff;
+        }
+        ret = s->vram_ptr[addr];
     } else {
         /* standard VGA latched access */
+        if (addr * sizeof(uint32_t) >= s->vram_size) {
+            return 0xff;
+        }
         s->latch = ((uint32_t *)s->vram_ptr)[addr];
 
         if (!(s->gr[VGA_GFX_MODE] & 0x08)) {
@@ -876,11 +906,12 @@
         break;
     }
 
-    if (s->sr[VGA_SEQ_MEMORY_MODE] & VGA_SR04_CHN_4M) {
+    if (sr(s, VGA_SEQ_MEMORY_MODE) & VGA_SR04_CHN_4M) {
         /* chain 4 mode : simplest access */
         plane = addr & 3;
         mask = (1 << plane);
-        if (s->sr[VGA_SEQ_PLANE_WRITE] & mask) {
+        if (sr(s, VGA_SEQ_PLANE_WRITE) & mask) {
+            assert(addr < s->vram_size);
             s->vram_ptr[addr] = val;
 #ifdef DEBUG_VGA_MEM
             printf("vga: chain4: [0x" TARGET_FMT_plx "]\n", addr);
@@ -892,8 +923,11 @@
         /* odd/even mode (aka text mode mapping) */
         plane = (s->gr[VGA_GFX_PLANE_READ] & 2) | (addr & 1);
         mask = (1 << plane);
-        if (s->sr[VGA_SEQ_PLANE_WRITE] & mask) {
+        if (sr(s, VGA_SEQ_PLANE_WRITE) & mask) {
             addr = ((addr & ~1) << 1) | plane;
+            if (addr >= s->vram_size) {
+                return;
+            }
             s->vram_ptr[addr] = val;
 #ifdef DEBUG_VGA_MEM
             printf("vga: odd/even: [0x" TARGET_FMT_plx "]\n", addr);
@@ -964,9 +998,12 @@
 
     do_write:
         /* mask data according to sr[2] */
-        mask = s->sr[VGA_SEQ_PLANE_WRITE];
+        mask = sr(s, VGA_SEQ_PLANE_WRITE);
         s->plane_updated |= mask; /* only used to detect font change */
         write_mask = mask16[mask];
+        if (addr * sizeof(uint32_t) >= s->vram_size) {
+            return;
+        }
         ((uint32_t *)s->vram_ptr)[addr] =
             (((uint32_t *)s->vram_ptr)[addr] & ~write_mask) |
             (val & write_mask);
@@ -1045,7 +1082,7 @@
 {
     uint32_t start_addr, line_offset, line_compare;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         line_offset = s->vbe_line_offset;
         start_addr = s->vbe_start_addr;
         line_compare = 65535;
@@ -1117,10 +1154,10 @@
     /* total width & height */
     cheight = (s->cr[VGA_CRTC_MAX_SCAN] & 0x1f) + 1;
     cwidth = 8;
-    if (!(s->sr[VGA_SEQ_CLOCK_MODE] & VGA_SR01_CHAR_CLK_8DOTS)) {
+    if (!(sr(s, VGA_SEQ_CLOCK_MODE) & VGA_SR01_CHAR_CLK_8DOTS)) {
         cwidth = 9;
     }
-    if (s->sr[VGA_SEQ_CLOCK_MODE] & 0x08) {
+    if (sr(s, VGA_SEQ_CLOCK_MODE) & 0x08) {
         cwidth = 16; /* NOTE: no 18 pixel wide */
     }
     width = (s->cr[VGA_CRTC_H_DISP] + 1);
@@ -1162,7 +1199,7 @@
     int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL);
 
     /* compute font data address (in plane 2) */
-    v = s->sr[VGA_SEQ_CHARACTER_MAP];
+    v = sr(s, VGA_SEQ_CHARACTER_MAP);
     offset = (((v >> 4) & 1) | ((v << 1) & 6)) * 8192 * 4 + 2;
     if (offset != s->font_offsets[0]) {
         s->font_offsets[0] = offset;
@@ -1370,7 +1407,7 @@
 {
     int ret;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         ret = s->vbe_regs[VBE_DISPI_INDEX_BPP];
     } else {
         ret = 0;
@@ -1382,7 +1419,7 @@
 {
     int width, height;
 
-    if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
+    if (vbe_enabled(s)) {
         width = s->vbe_regs[VBE_DISPI_INDEX_XRES];
         height = s->vbe_regs[VBE_DISPI_INDEX_YRES];
     } else {
@@ -1436,6 +1473,8 @@
     uint8_t *d;
     uint32_t v, addr1, addr;
     vga_draw_line_func *vga_draw_line = NULL;
+    bool share_surface;
+    pixman_format_code_t format;
 #ifdef HOST_WORDS_BIGENDIAN
     bool byteswap = !s->big_endian_fb;
 #else
@@ -1469,31 +1508,52 @@
     }
 
     if (shift_control == 0) {
-        if (s->sr[VGA_SEQ_CLOCK_MODE] & 8) {
+        if (sr(s, VGA_SEQ_CLOCK_MODE) & 8) {
             disp_width <<= 1;
         }
     } else if (shift_control == 1) {
-        if (s->sr[VGA_SEQ_CLOCK_MODE] & 8) {
+        if (sr(s, VGA_SEQ_CLOCK_MODE) & 8) {
             disp_width <<= 1;
         }
     }
 
     depth = s->get_bpp(s);
+
+    /*
+     * Check whether we can share the surface with the backend
+     * or whether we need a shadow surface. We share native
+     * endian surfaces for 15bpp and above and byteswapped
+     * surfaces for 24bpp and above.
+     */
+    format = qemu_default_pixman_format(depth, !byteswap);
+    if (format) {
+        share_surface = dpy_gfx_check_format(s->con, format)
+            && !s->force_shadow;
+    } else {
+        share_surface = false;
+    }
     if (s->line_offset != s->last_line_offset ||
         disp_width != s->last_width ||
         height != s->last_height ||
         s->last_depth != depth ||
-        s->last_byteswap != byteswap) {
-        if (depth == 32 || (depth == 16 && !byteswap)) {
-            pixman_format_code_t format =
-                qemu_default_pixman_format(depth, !byteswap);
+        s->last_byteswap != byteswap ||
+        share_surface != is_buffer_shared(surface)) {
+        if (share_surface) {
             surface = qemu_create_displaysurface_from(disp_width,
                     height, format, s->line_offset,
                     s->vram_ptr + (s->start_addr * 4));
             dpy_gfx_replace_surface(s->con, surface);
+#ifdef DEBUG_VGA
+            printf("VGA: Using shared surface for depth=%d swap=%d\n",
+                   depth, byteswap);
+#endif
         } else {
             qemu_console_resize(s->con, disp_width, height);
             surface = qemu_console_surface(s->con);
+#ifdef DEBUG_VGA
+            printf("VGA: Using shadow surface for depth=%d swap=%d\n",
+                   depth, byteswap);
+#endif
         }
         s->last_scr_width = disp_width;
         s->last_scr_height = height;
@@ -1516,7 +1576,7 @@
 
     if (shift_control == 0) {
         full_update |= update_palette16(s);
-        if (s->sr[VGA_SEQ_CLOCK_MODE] & 8) {
+        if (sr(s, VGA_SEQ_CLOCK_MODE) & 8) {
             v = VGA_DRAW_LINE4D2;
         } else {
             v = VGA_DRAW_LINE4;
@@ -1524,7 +1584,7 @@
         bits = 4;
     } else if (shift_control == 1) {
         full_update |= update_palette16(s);
-        if (s->sr[VGA_SEQ_CLOCK_MODE] & 8) {
+        if (sr(s, VGA_SEQ_CLOCK_MODE) & 8) {
             v = VGA_DRAW_LINE2D2;
         } else {
             v = VGA_DRAW_LINE2;
@@ -1571,7 +1631,7 @@
 #if 0
     printf("w=%d h=%d v=%d line_offset=%d cr[0x09]=0x%02x cr[0x17]=0x%02x linecmp=%d sr[0x01]=0x%02x\n",
            width, height, v, line_offset, s->cr[9], s->cr[VGA_CRTC_MODE],
-           s->line_compare, s->sr[VGA_SEQ_CLOCK_MODE]);
+           s->line_compare, sr(s, VGA_SEQ_CLOCK_MODE));
 #endif
     addr1 = (s->start_addr * 4);
     bwidth = (width * bits + 7) / 8;
@@ -1723,6 +1783,7 @@
 {
     s->sr_index = 0;
     memset(s->sr, '\0', sizeof(s->sr));
+    memset(s->sr_vbe, '\0', sizeof(s->sr_vbe));
     s->gr_index = 0;
     memset(s->gr, '\0', sizeof(s->gr));
     s->ar_index = 0;
@@ -1825,10 +1886,10 @@
         /* total width & height */
         cheight = (s->cr[VGA_CRTC_MAX_SCAN] & 0x1f) + 1;
         cw = 8;
-        if (!(s->sr[VGA_SEQ_CLOCK_MODE] & VGA_SR01_CHAR_CLK_8DOTS)) {
+        if (!(sr(s, VGA_SEQ_CLOCK_MODE) & VGA_SR01_CHAR_CLK_8DOTS)) {
             cw = 9;
         }
-        if (s->sr[VGA_SEQ_CLOCK_MODE] & 0x08) {
+        if (sr(s, VGA_SEQ_CLOCK_MODE) & 0x08) {
             cw = 16; /* NOTE: no 18 pixel wide */
         }
         width = (s->cr[VGA_CRTC_H_DISP] + 1);
@@ -1957,7 +2018,8 @@
     width = (s->last_width - size) / 2;
     dst = chardata + s->last_width + width;
     for (i = 0; i < size; i ++)
-        console_write_ch(dst ++, 0x00200100 | msg_buffer[i]);
+        console_write_ch(dst ++, ATTR2CHTYPE(msg_buffer[i], QEMU_COLOR_BLUE,
+                                             QEMU_COLOR_BLACK, 1));
 
     dpy_text_update(s->con, 0, 0, s->last_width, height);
 }
@@ -1975,7 +2037,7 @@
 {
     VGACommonState *s = opaque;
 
-    return vga_mem_writeb(s, addr, data);
+    vga_mem_writeb(s, addr, data);
 }
 
 const MemoryRegionOps vga_mem_ops = {
@@ -1994,6 +2056,7 @@
 
     /* force refresh */
     s->graphic_mode = -1;
+    vbe_update_vgaregs(s);
     return 0;
 }
 
@@ -2009,10 +2072,11 @@
     return s->default_endian_fb != s->big_endian_fb;
 }
 
-const VMStateDescription vmstate_vga_endian = {
+static const VMStateDescription vmstate_vga_endian = {
     .name = "vga.endian",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = vga_endian_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_BOOL(big_endian_fb, VGACommonState),
         VMSTATE_END_OF_LIST()
@@ -2056,13 +2120,9 @@
         VMSTATE_UINT32(vbe_bank_mask, VGACommonState),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_vga_endian,
-            .needed = vga_endian_state_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_vga_endian,
+        NULL
     }
 };
 
@@ -2072,6 +2132,17 @@
     .text_update = vga_update_text,
 };
 
+static inline uint32_t uint_clamp(uint32_t val, uint32_t vmin, uint32_t vmax)
+{
+    if (val < vmin) {
+        return vmin;
+    }
+    if (val > vmax) {
+        return vmax;
+    }
+    return val;
+}
+
 void vga_common_init(VGACommonState *s, Object *obj, bool global_vmstate)
 {
     int i, j, v, b;
@@ -2099,20 +2170,17 @@
         expand4to8[i] = v;
     }
 
-    /* valid range: 1 MB -> 256 MB */
-    s->vram_size = 1024 * 1024;
-    while (s->vram_size < (s->vram_size_mb << 20) &&
-           s->vram_size < (256 << 20)) {
-        s->vram_size <<= 1;
-    }
-    s->vram_size_mb = s->vram_size >> 20;
+    s->vram_size_mb = uint_clamp(s->vram_size_mb, 1, 512);
+    s->vram_size_mb = pow2ceil(s->vram_size_mb);
+    s->vram_size = s->vram_size_mb << 20;
+
     if (!s->vbe_size) {
         s->vbe_size = s->vram_size;
     }
 
     s->is_vbe_vmstate = 1;
     memory_region_init_ram(&s->vram, obj, "vga.vram", s->vram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram(&s->vram, global_vmstate ? NULL : DEVICE(obj));
     xen_register_framebuffer(&s->vram);
     s->vram_ptr = memory_region_get_ram_ptr(&s->vram);
@@ -2195,7 +2263,7 @@
 
     vga_io_memory = vga_init_io(s, obj, &vga_ports, &vbe_ports);
     memory_region_add_subregion_overlap(address_space,
-                                        isa_mem_base + 0x000a0000,
+                                        0x000a0000,
                                         vga_io_memory,
                                         1);
     memory_region_set_coalescing(vga_io_memory);
diff --git a/hw/display/vga.h b/hw/display/vga.h
index d917046..16886f5 100644
--- a/hw/display/vga.h
+++ b/hw/display/vga.h
@@ -14,8 +14,8 @@
  *
  */
 
-#ifndef __linux_video_vga_h__
-#define __linux_video_vga_h__
+#ifndef LINUX_VIDEO_VGA_H
+#define LINUX_VIDEO_VGA_H
 
 /* Some of the code below is taken from SVGAlib.  The original,
    unmodified copyright notice for that code is below. */
@@ -156,4 +156,4 @@
 /* VGA graphics controller bit masks */
 #define VGA_GR06_GRAPHICS_MODE  0x01
 
-#endif /* __linux_video_vga_h__ */
+#endif /* LINUX_VIDEO_VGA_H */
diff --git a/hw/display/vga_int.h b/hw/display/vga_int.h
index ed69e06..dd6c958 100644
--- a/hw/display/vga_int.h
+++ b/hw/display/vga_int.h
@@ -21,11 +21,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef HW_VGA_INT_H
-#define HW_VGA_INT_H 1
 
-#include <hw/hw.h>
-#include "qapi/error.h"
+#ifndef HW_VGA_INT_H
+#define HW_VGA_INT_H
+
+#include "hw/hw.h"
 #include "exec/memory.h"
 
 #define ST01_V_RETRACE      0x08
@@ -99,6 +99,7 @@
     MemoryRegion chain4_alias;
     uint8_t sr_index;
     uint8_t sr[256];
+    uint8_t sr_vbe[256];
     uint8_t gr_index;
     uint8_t gr[256];
     uint8_t ar_index;
@@ -151,6 +152,7 @@
     uint32_t last_scr_width, last_scr_height; /* in pixels */
     uint32_t last_depth; /* in bits */
     bool last_byteswap;
+    bool force_shadow;
     uint8_t cursor_start, cursor_end;
     bool cursor_visible_phase;
     int64_t cursor_blink_time;
@@ -162,6 +164,8 @@
     bool default_endian_fb;
     /* hardware mouse cursor support */
     uint32_t invalidated_y_table[VGA_MAX_HEIGHT / 32];
+    uint32_t hw_cursor_x;
+    uint32_t hw_cursor_y;
     void (*cursor_invalidate)(struct VGACommonState *s);
     void (*cursor_draw_line)(struct VGACommonState *s, uint8_t *d, int y);
     /* tell for each page if it has been updated since the last time */
@@ -216,4 +220,10 @@
 
 extern const MemoryRegionOps vga_mem_ops;
 
+/* vga-pci.c */
+void pci_std_vga_mmio_region_init(VGACommonState *s,
+                                  MemoryRegion *parent,
+                                  MemoryRegion *subs,
+                                  bool qext);
+
 #endif
diff --git a/hw/display/virtio-gpu-3d.c b/hw/display/virtio-gpu-3d.c
new file mode 100644
index 0000000..758d33a
--- /dev/null
+++ b/hw/display/virtio-gpu-3d.c
@@ -0,0 +1,608 @@
+/*
+ * Virtio GPU Device
+ *
+ * Copyright Red Hat, Inc. 2013-2014
+ *
+ * Authors:
+ *     Dave Airlie <airlied@redhat.com>
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/iov.h"
+#include "trace.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-gpu.h"
+#include "qapi/error.h"
+
+#ifdef CONFIG_VIRGL
+
+#include <virglrenderer.h>
+
+static struct virgl_renderer_callbacks virtio_gpu_3d_cbs;
+
+static void virgl_cmd_create_resource_2d(VirtIOGPU *g,
+                                         struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_create_2d c2d;
+    struct virgl_renderer_resource_create_args args;
+
+    VIRTIO_GPU_FILL_CMD(c2d);
+    trace_virtio_gpu_cmd_res_create_2d(c2d.resource_id, c2d.format,
+                                       c2d.width, c2d.height);
+
+    args.handle = c2d.resource_id;
+    args.target = 2;
+    args.format = c2d.format;
+    args.bind = (1 << 1);
+    args.width = c2d.width;
+    args.height = c2d.height;
+    args.depth = 1;
+    args.array_size = 1;
+    args.last_level = 0;
+    args.nr_samples = 0;
+    args.flags = VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP;
+    virgl_renderer_resource_create(&args, NULL, 0);
+}
+
+static void virgl_cmd_create_resource_3d(VirtIOGPU *g,
+                                         struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_create_3d c3d;
+    struct virgl_renderer_resource_create_args args;
+
+    VIRTIO_GPU_FILL_CMD(c3d);
+    trace_virtio_gpu_cmd_res_create_3d(c3d.resource_id, c3d.format,
+                                       c3d.width, c3d.height, c3d.depth);
+
+    args.handle = c3d.resource_id;
+    args.target = c3d.target;
+    args.format = c3d.format;
+    args.bind = c3d.bind;
+    args.width = c3d.width;
+    args.height = c3d.height;
+    args.depth = c3d.depth;
+    args.array_size = c3d.array_size;
+    args.last_level = c3d.last_level;
+    args.nr_samples = c3d.nr_samples;
+    args.flags = c3d.flags;
+    virgl_renderer_resource_create(&args, NULL, 0);
+}
+
+static void virgl_cmd_resource_unref(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_unref unref;
+
+    VIRTIO_GPU_FILL_CMD(unref);
+    trace_virtio_gpu_cmd_res_unref(unref.resource_id);
+
+    virgl_renderer_resource_unref(unref.resource_id);
+}
+
+static void virgl_cmd_context_create(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_create cc;
+
+    VIRTIO_GPU_FILL_CMD(cc);
+    trace_virtio_gpu_cmd_ctx_create(cc.hdr.ctx_id,
+                                    cc.debug_name);
+
+    virgl_renderer_context_create(cc.hdr.ctx_id, cc.nlen,
+                                  cc.debug_name);
+}
+
+static void virgl_cmd_context_destroy(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_destroy cd;
+
+    VIRTIO_GPU_FILL_CMD(cd);
+    trace_virtio_gpu_cmd_ctx_destroy(cd.hdr.ctx_id);
+
+    virgl_renderer_context_destroy(cd.hdr.ctx_id);
+}
+
+static void virtio_gpu_rect_update(VirtIOGPU *g, int idx, int x, int y,
+                                int width, int height)
+{
+    if (!g->scanout[idx].con) {
+        return;
+    }
+
+    dpy_gl_update(g->scanout[idx].con, x, y, width, height);
+}
+
+static void virgl_cmd_resource_flush(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_flush rf;
+    int i;
+
+    VIRTIO_GPU_FILL_CMD(rf);
+    trace_virtio_gpu_cmd_res_flush(rf.resource_id,
+                                   rf.r.width, rf.r.height, rf.r.x, rf.r.y);
+
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        if (g->scanout[i].resource_id != rf.resource_id) {
+            continue;
+        }
+        virtio_gpu_rect_update(g, i, rf.r.x, rf.r.y, rf.r.width, rf.r.height);
+    }
+}
+
+static void virgl_cmd_set_scanout(VirtIOGPU *g,
+                                  struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_set_scanout ss;
+    struct virgl_renderer_resource_info info;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(ss);
+    trace_virtio_gpu_cmd_set_scanout(ss.scanout_id, ss.resource_id,
+                                     ss.r.width, ss.r.height, ss.r.x, ss.r.y);
+
+    if (ss.scanout_id >= g->conf.max_outputs) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal scanout id specified %d",
+                      __func__, ss.scanout_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_SCANOUT_ID;
+        return;
+    }
+    g->enable = 1;
+
+    memset(&info, 0, sizeof(info));
+
+    if (ss.resource_id && ss.r.width && ss.r.height) {
+        ret = virgl_renderer_resource_get_info(ss.resource_id, &info);
+        if (ret == -1) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "%s: illegal resource specified %d\n",
+                          __func__, ss.resource_id);
+            cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+            return;
+        }
+        qemu_console_resize(g->scanout[ss.scanout_id].con,
+                            ss.r.width, ss.r.height);
+        virgl_renderer_force_ctx_0();
+        dpy_gl_scanout(g->scanout[ss.scanout_id].con, info.tex_id,
+                       info.flags & 1 /* FIXME: Y_0_TOP */,
+                       info.width, info.height,
+                       ss.r.x, ss.r.y, ss.r.width, ss.r.height);
+    } else {
+        if (ss.scanout_id != 0) {
+            dpy_gfx_replace_surface(g->scanout[ss.scanout_id].con, NULL);
+        }
+        dpy_gl_scanout(g->scanout[ss.scanout_id].con, 0, false,
+                       0, 0, 0, 0, 0, 0);
+    }
+    g->scanout[ss.scanout_id].resource_id = ss.resource_id;
+}
+
+static void virgl_cmd_submit_3d(VirtIOGPU *g,
+                                struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_cmd_submit cs;
+    void *buf;
+    size_t s;
+
+    VIRTIO_GPU_FILL_CMD(cs);
+    trace_virtio_gpu_cmd_ctx_submit(cs.hdr.ctx_id, cs.size);
+
+    buf = g_malloc(cs.size);
+    s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
+                   sizeof(cs), buf, cs.size);
+    if (s != cs.size) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: size mismatch (%zd/%d)",
+                      __func__, s, cs.size);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        goto out;
+    }
+
+    if (virtio_gpu_stats_enabled(g->conf)) {
+        g->stats.req_3d++;
+        g->stats.bytes_3d += cs.size;
+    }
+
+    virgl_renderer_submit_cmd(buf, cs.hdr.ctx_id, cs.size / 4);
+
+out:
+    g_free(buf);
+}
+
+static void virgl_cmd_transfer_to_host_2d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_to_host_2d t2d;
+    struct virtio_gpu_box box;
+
+    VIRTIO_GPU_FILL_CMD(t2d);
+    trace_virtio_gpu_cmd_res_xfer_toh_2d(t2d.resource_id);
+
+    box.x = t2d.r.x;
+    box.y = t2d.r.y;
+    box.z = 0;
+    box.w = t2d.r.width;
+    box.h = t2d.r.height;
+    box.d = 1;
+
+    virgl_renderer_transfer_write_iov(t2d.resource_id,
+                                      0,
+                                      0,
+                                      0,
+                                      0,
+                                      (struct virgl_box *)&box,
+                                      t2d.offset, NULL, 0);
+}
+
+static void virgl_cmd_transfer_to_host_3d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_host_3d t3d;
+
+    VIRTIO_GPU_FILL_CMD(t3d);
+    trace_virtio_gpu_cmd_res_xfer_toh_3d(t3d.resource_id);
+
+    virgl_renderer_transfer_write_iov(t3d.resource_id,
+                                      t3d.hdr.ctx_id,
+                                      t3d.level,
+                                      t3d.stride,
+                                      t3d.layer_stride,
+                                      (struct virgl_box *)&t3d.box,
+                                      t3d.offset, NULL, 0);
+}
+
+static void
+virgl_cmd_transfer_from_host_3d(VirtIOGPU *g,
+                                struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_transfer_host_3d tf3d;
+
+    VIRTIO_GPU_FILL_CMD(tf3d);
+    trace_virtio_gpu_cmd_res_xfer_fromh_3d(tf3d.resource_id);
+
+    virgl_renderer_transfer_read_iov(tf3d.resource_id,
+                                     tf3d.hdr.ctx_id,
+                                     tf3d.level,
+                                     tf3d.stride,
+                                     tf3d.layer_stride,
+                                     (struct virgl_box *)&tf3d.box,
+                                     tf3d.offset, NULL, 0);
+}
+
+
+static void virgl_resource_attach_backing(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_attach_backing att_rb;
+    struct iovec *res_iovs;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(att_rb);
+    trace_virtio_gpu_cmd_res_back_attach(att_rb.resource_id);
+
+    ret = virtio_gpu_create_mapping_iov(&att_rb, cmd, NULL, &res_iovs);
+    if (ret != 0) {
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        return;
+    }
+
+    virgl_renderer_resource_attach_iov(att_rb.resource_id,
+                                       res_iovs, att_rb.nr_entries);
+}
+
+static void virgl_resource_detach_backing(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resource_detach_backing detach_rb;
+    struct iovec *res_iovs = NULL;
+    int num_iovs = 0;
+
+    VIRTIO_GPU_FILL_CMD(detach_rb);
+    trace_virtio_gpu_cmd_res_back_detach(detach_rb.resource_id);
+
+    virgl_renderer_resource_detach_iov(detach_rb.resource_id,
+                                       &res_iovs,
+                                       &num_iovs);
+    if (res_iovs == NULL || num_iovs == 0) {
+        return;
+    }
+    virtio_gpu_cleanup_mapping_iov(res_iovs, num_iovs);
+}
+
+
+static void virgl_cmd_ctx_attach_resource(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_resource att_res;
+
+    VIRTIO_GPU_FILL_CMD(att_res);
+    trace_virtio_gpu_cmd_ctx_res_attach(att_res.hdr.ctx_id,
+                                        att_res.resource_id);
+
+    virgl_renderer_ctx_attach_resource(att_res.hdr.ctx_id, att_res.resource_id);
+}
+
+static void virgl_cmd_ctx_detach_resource(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_ctx_resource det_res;
+
+    VIRTIO_GPU_FILL_CMD(det_res);
+    trace_virtio_gpu_cmd_ctx_res_detach(det_res.hdr.ctx_id,
+                                        det_res.resource_id);
+
+    virgl_renderer_ctx_detach_resource(det_res.hdr.ctx_id, det_res.resource_id);
+}
+
+static void virgl_cmd_get_capset_info(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_get_capset_info info;
+    struct virtio_gpu_resp_capset_info resp;
+
+    VIRTIO_GPU_FILL_CMD(info);
+
+    if (info.capset_index == 0) {
+        resp.capset_id = VIRTIO_GPU_CAPSET_VIRGL;
+        virgl_renderer_get_cap_set(resp.capset_id,
+                                   &resp.capset_max_version,
+                                   &resp.capset_max_size);
+    } else {
+        resp.capset_max_version = 0;
+        resp.capset_max_size = 0;
+    }
+    resp.hdr.type = VIRTIO_GPU_RESP_OK_CAPSET_INFO;
+    virtio_gpu_ctrl_response(g, cmd, &resp.hdr, sizeof(resp));
+}
+
+static void virgl_cmd_get_capset(VirtIOGPU *g,
+                                 struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_get_capset gc;
+    struct virtio_gpu_resp_capset *resp;
+    uint32_t max_ver, max_size;
+    VIRTIO_GPU_FILL_CMD(gc);
+
+    virgl_renderer_get_cap_set(gc.capset_id, &max_ver,
+                               &max_size);
+    resp = g_malloc(sizeof(*resp) + max_size);
+
+    resp->hdr.type = VIRTIO_GPU_RESP_OK_CAPSET;
+    virgl_renderer_fill_caps(gc.capset_id,
+                             gc.capset_version,
+                             (void *)resp->capset_data);
+    virtio_gpu_ctrl_response(g, cmd, &resp->hdr, sizeof(*resp) + max_size);
+    g_free(resp);
+}
+
+void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    VIRTIO_GPU_FILL_CMD(cmd->cmd_hdr);
+
+    cmd->waiting = g->renderer_blocked;
+    if (cmd->waiting) {
+        return;
+    }
+
+    virgl_renderer_force_ctx_0();
+    switch (cmd->cmd_hdr.type) {
+    case VIRTIO_GPU_CMD_CTX_CREATE:
+        virgl_cmd_context_create(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_DESTROY:
+        virgl_cmd_context_destroy(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_2D:
+        virgl_cmd_create_resource_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_3D:
+        virgl_cmd_create_resource_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SUBMIT_3D:
+        virgl_cmd_submit_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D:
+        virgl_cmd_transfer_to_host_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D:
+        virgl_cmd_transfer_to_host_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D:
+        virgl_cmd_transfer_from_host_3d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING:
+        virgl_resource_attach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING:
+        virgl_resource_detach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SET_SCANOUT:
+        virgl_cmd_set_scanout(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_FLUSH:
+        virgl_cmd_resource_flush(g, cmd);
+       break;
+    case VIRTIO_GPU_CMD_RESOURCE_UNREF:
+        virgl_cmd_resource_unref(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE:
+        /* TODO add security */
+        virgl_cmd_ctx_attach_resource(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE:
+        /* TODO add security */
+        virgl_cmd_ctx_detach_resource(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_GET_CAPSET_INFO:
+        virgl_cmd_get_capset_info(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_GET_CAPSET:
+        virgl_cmd_get_capset(g, cmd);
+        break;
+
+    case VIRTIO_GPU_CMD_GET_DISPLAY_INFO:
+        virtio_gpu_get_display_info(g, cmd);
+        break;
+    default:
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        break;
+    }
+
+    if (cmd->finished) {
+        return;
+    }
+    if (cmd->error) {
+        fprintf(stderr, "%s: ctrl 0x%x, error 0x%x\n", __func__,
+                cmd->cmd_hdr.type, cmd->error);
+        virtio_gpu_ctrl_response_nodata(g, cmd, cmd->error);
+        return;
+    }
+    if (!(cmd->cmd_hdr.flags & VIRTIO_GPU_FLAG_FENCE)) {
+        virtio_gpu_ctrl_response_nodata(g, cmd, VIRTIO_GPU_RESP_OK_NODATA);
+        return;
+    }
+
+    trace_virtio_gpu_fence_ctrl(cmd->cmd_hdr.fence_id, cmd->cmd_hdr.type);
+    virgl_renderer_create_fence(cmd->cmd_hdr.fence_id, cmd->cmd_hdr.type);
+}
+
+static void virgl_write_fence(void *opaque, uint32_t fence)
+{
+    VirtIOGPU *g = opaque;
+    struct virtio_gpu_ctrl_command *cmd, *tmp;
+
+    QTAILQ_FOREACH_SAFE(cmd, &g->fenceq, next, tmp) {
+        /*
+	 * the guest can end up emitting fences out of order
+	 * so we should check all fenced cmds not just the first one.
+	 */
+        if (cmd->cmd_hdr.fence_id > fence) {
+            continue;
+        }
+        trace_virtio_gpu_fence_resp(cmd->cmd_hdr.fence_id);
+        virtio_gpu_ctrl_response_nodata(g, cmd, VIRTIO_GPU_RESP_OK_NODATA);
+        QTAILQ_REMOVE(&g->fenceq, cmd, next);
+        g_free(cmd);
+        g->inflight--;
+        if (virtio_gpu_stats_enabled(g->conf)) {
+            fprintf(stderr, "inflight: %3d (-)\r", g->inflight);
+        }
+    }
+}
+
+static virgl_renderer_gl_context
+virgl_create_context(void *opaque, int scanout_idx,
+                     struct virgl_renderer_gl_ctx_param *params)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext ctx;
+    QEMUGLParams qparams;
+
+    qparams.major_ver = params->major_ver;
+    qparams.minor_ver = params->minor_ver;
+
+    ctx = dpy_gl_ctx_create(g->scanout[scanout_idx].con, &qparams);
+    return (virgl_renderer_gl_context)ctx;
+}
+
+static void virgl_destroy_context(void *opaque, virgl_renderer_gl_context ctx)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext qctx = (QEMUGLContext)ctx;
+
+    dpy_gl_ctx_destroy(g->scanout[0].con, qctx);
+}
+
+static int virgl_make_context_current(void *opaque, int scanout_idx,
+                                      virgl_renderer_gl_context ctx)
+{
+    VirtIOGPU *g = opaque;
+    QEMUGLContext qctx = (QEMUGLContext)ctx;
+
+    return dpy_gl_ctx_make_current(g->scanout[scanout_idx].con, qctx);
+}
+
+static struct virgl_renderer_callbacks virtio_gpu_3d_cbs = {
+    .version             = 1,
+    .write_fence         = virgl_write_fence,
+    .create_gl_context   = virgl_create_context,
+    .destroy_gl_context  = virgl_destroy_context,
+    .make_current        = virgl_make_context_current,
+};
+
+static void virtio_gpu_print_stats(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+
+    if (g->stats.requests) {
+        fprintf(stderr, "stats: vq req %4d, %3d -- 3D %4d (%5d)\n",
+                g->stats.requests,
+                g->stats.max_inflight,
+                g->stats.req_3d,
+                g->stats.bytes_3d);
+        g->stats.requests     = 0;
+        g->stats.max_inflight = 0;
+        g->stats.req_3d       = 0;
+        g->stats.bytes_3d     = 0;
+    } else {
+        fprintf(stderr, "stats: idle\r");
+    }
+    timer_mod(g->print_stats, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 1000);
+}
+
+static void virtio_gpu_fence_poll(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+
+    virgl_renderer_poll();
+    virtio_gpu_process_cmdq(g);
+    if (!QTAILQ_EMPTY(&g->cmdq) || !QTAILQ_EMPTY(&g->fenceq)) {
+        timer_mod(g->fence_poll, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 10);
+    }
+}
+
+void virtio_gpu_virgl_fence_poll(VirtIOGPU *g)
+{
+    virtio_gpu_fence_poll(g);
+}
+
+void virtio_gpu_virgl_reset(VirtIOGPU *g)
+{
+    int i;
+
+    /* virgl_renderer_reset() ??? */
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        if (i != 0) {
+            dpy_gfx_replace_surface(g->scanout[i].con, NULL);
+        }
+        dpy_gl_scanout(g->scanout[i].con, 0, false, 0, 0, 0, 0, 0, 0);
+    }
+}
+
+int virtio_gpu_virgl_init(VirtIOGPU *g)
+{
+    int ret;
+
+    ret = virgl_renderer_init(g, 0, &virtio_gpu_3d_cbs);
+    if (ret != 0) {
+        return ret;
+    }
+
+    g->fence_poll = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                 virtio_gpu_fence_poll, g);
+
+    if (virtio_gpu_stats_enabled(g->conf)) {
+        g->print_stats = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                      virtio_gpu_print_stats, g);
+        timer_mod(g->print_stats, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 1000);
+    }
+    return 0;
+}
+
+#endif /* CONFIG_VIRGL */
diff --git a/hw/display/virtio-gpu-pci.c b/hw/display/virtio-gpu-pci.c
new file mode 100644
index 0000000..34a724c
--- /dev/null
+++ b/hw/display/virtio-gpu-pci.c
@@ -0,0 +1,75 @@
+/*
+ * Virtio video device
+ *
+ * Copyright Red Hat
+ *
+ * Authors:
+ *  Dave Airlie
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include "qemu/osdep.h"
+#include "hw/pci/pci.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-bus.h"
+#include "hw/virtio/virtio-pci.h"
+#include "hw/virtio/virtio-gpu.h"
+
+static Property virtio_gpu_pci_properties[] = {
+    DEFINE_VIRTIO_GPU_PCI_PROPERTIES(VirtIOPCIProxy),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_gpu_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
+{
+    VirtIOGPUPCI *vgpu = VIRTIO_GPU_PCI(vpci_dev);
+    VirtIOGPU *g = &vgpu->vdev;
+    DeviceState *vdev = DEVICE(&vgpu->vdev);
+    int i;
+
+    qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
+    virtio_pci_force_virtio_1(vpci_dev);
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
+
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        object_property_set_link(OBJECT(g->scanout[i].con),
+                                 OBJECT(vpci_dev),
+                                 "device", errp);
+    }
+}
+
+static void virtio_gpu_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
+    PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
+    dc->props = virtio_gpu_pci_properties;
+    k->realize = virtio_gpu_pci_realize;
+    pcidev_k->class_id = PCI_CLASS_DISPLAY_OTHER;
+}
+
+static void virtio_gpu_initfn(Object *obj)
+{
+    VirtIOGPUPCI *dev = VIRTIO_GPU_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_GPU);
+}
+
+static const TypeInfo virtio_gpu_pci_info = {
+    .name = TYPE_VIRTIO_GPU_PCI,
+    .parent = TYPE_VIRTIO_PCI,
+    .instance_size = sizeof(VirtIOGPUPCI),
+    .instance_init = virtio_gpu_initfn,
+    .class_init = virtio_gpu_pci_class_init,
+};
+
+static void virtio_gpu_pci_register_types(void)
+{
+    type_register_static(&virtio_gpu_pci_info);
+}
+type_init(virtio_gpu_pci_register_types)
diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c
new file mode 100644
index 0000000..7fe6ed8
--- /dev/null
+++ b/hw/display/virtio-gpu.c
@@ -0,0 +1,1290 @@
+/*
+ * Virtio GPU Device
+ *
+ * Copyright Red Hat, Inc. 2013-2014
+ *
+ * Authors:
+ *     Dave Airlie <airlied@redhat.com>
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/iov.h"
+#include "ui/console.h"
+#include "trace.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-gpu.h"
+#include "hw/virtio/virtio-bus.h"
+#include "migration/migration.h"
+#include "qemu/log.h"
+#include "qapi/error.h"
+
+#define VIRTIO_GPU_VM_VERSION 1
+
+static struct virtio_gpu_simple_resource*
+virtio_gpu_find_resource(VirtIOGPU *g, uint32_t resource_id);
+
+#ifdef CONFIG_VIRGL
+#include <virglrenderer.h>
+#define VIRGL(_g, _virgl, _simple, ...)                     \
+    do {                                                    \
+        if (_g->use_virgl_renderer) {                       \
+            _virgl(__VA_ARGS__);                            \
+        } else {                                            \
+            _simple(__VA_ARGS__);                           \
+        }                                                   \
+    } while (0)
+#else
+#define VIRGL(_g, _virgl, _simple, ...)                 \
+    do {                                                \
+        _simple(__VA_ARGS__);                           \
+    } while (0)
+#endif
+
+static void update_cursor_data_simple(VirtIOGPU *g,
+                                      struct virtio_gpu_scanout *s,
+                                      uint32_t resource_id)
+{
+    struct virtio_gpu_simple_resource *res;
+    uint32_t pixels;
+
+    res = virtio_gpu_find_resource(g, resource_id);
+    if (!res) {
+        return;
+    }
+
+    if (pixman_image_get_width(res->image)  != s->current_cursor->width ||
+        pixman_image_get_height(res->image) != s->current_cursor->height) {
+        return;
+    }
+
+    pixels = s->current_cursor->width * s->current_cursor->height;
+    memcpy(s->current_cursor->data,
+           pixman_image_get_data(res->image),
+           pixels * sizeof(uint32_t));
+}
+
+#ifdef CONFIG_VIRGL
+
+static void update_cursor_data_virgl(VirtIOGPU *g,
+                                     struct virtio_gpu_scanout *s,
+                                     uint32_t resource_id)
+{
+    uint32_t width, height;
+    uint32_t pixels, *data;
+
+    data = virgl_renderer_get_cursor_data(resource_id, &width, &height);
+    if (!data) {
+        return;
+    }
+
+    if (width != s->current_cursor->width ||
+        height != s->current_cursor->height) {
+        return;
+    }
+
+    pixels = s->current_cursor->width * s->current_cursor->height;
+    memcpy(s->current_cursor->data, data, pixels * sizeof(uint32_t));
+    free(data);
+}
+
+#endif
+
+static void update_cursor(VirtIOGPU *g, struct virtio_gpu_update_cursor *cursor)
+{
+    struct virtio_gpu_scanout *s;
+    bool move = cursor->hdr.type == VIRTIO_GPU_CMD_MOVE_CURSOR;
+
+    if (cursor->pos.scanout_id >= g->conf.max_outputs) {
+        return;
+    }
+    s = &g->scanout[cursor->pos.scanout_id];
+
+    trace_virtio_gpu_update_cursor(cursor->pos.scanout_id,
+                                   cursor->pos.x,
+                                   cursor->pos.y,
+                                   move ? "move" : "update",
+                                   cursor->resource_id);
+
+    if (!move) {
+        if (!s->current_cursor) {
+            s->current_cursor = cursor_alloc(64, 64);
+        }
+
+        s->current_cursor->hot_x = cursor->hot_x;
+        s->current_cursor->hot_y = cursor->hot_y;
+
+        if (cursor->resource_id > 0) {
+            VIRGL(g, update_cursor_data_virgl, update_cursor_data_simple,
+                  g, s, cursor->resource_id);
+        }
+        dpy_cursor_define(s->con, s->current_cursor);
+
+        s->cursor = *cursor;
+    } else {
+        s->cursor.pos.x = cursor->pos.x;
+        s->cursor.pos.y = cursor->pos.y;
+    }
+    dpy_mouse_set(s->con, cursor->pos.x, cursor->pos.y,
+                  cursor->resource_id ? 1 : 0);
+}
+
+static void virtio_gpu_get_config(VirtIODevice *vdev, uint8_t *config)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    memcpy(config, &g->virtio_config, sizeof(g->virtio_config));
+}
+
+static void virtio_gpu_set_config(VirtIODevice *vdev, const uint8_t *config)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    struct virtio_gpu_config vgconfig;
+
+    memcpy(&vgconfig, config, sizeof(g->virtio_config));
+
+    if (vgconfig.events_clear) {
+        g->virtio_config.events_read &= ~vgconfig.events_clear;
+    }
+}
+
+static uint64_t virtio_gpu_get_features(VirtIODevice *vdev, uint64_t features,
+                                        Error **errp)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        features |= (1 << VIRTIO_GPU_F_VIRGL);
+    }
+    return features;
+}
+
+static void virtio_gpu_set_features(VirtIODevice *vdev, uint64_t features)
+{
+    static const uint32_t virgl = (1 << VIRTIO_GPU_F_VIRGL);
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+
+    g->use_virgl_renderer = ((features & virgl) == virgl);
+    trace_virtio_gpu_features(g->use_virgl_renderer);
+}
+
+static void virtio_gpu_notify_event(VirtIOGPU *g, uint32_t event_type)
+{
+    g->virtio_config.events_read |= event_type;
+    virtio_notify_config(&g->parent_obj);
+}
+
+static struct virtio_gpu_simple_resource *
+virtio_gpu_find_resource(VirtIOGPU *g, uint32_t resource_id)
+{
+    struct virtio_gpu_simple_resource *res;
+
+    QTAILQ_FOREACH(res, &g->reslist, next) {
+        if (res->resource_id == resource_id) {
+            return res;
+        }
+    }
+    return NULL;
+}
+
+void virtio_gpu_ctrl_response(VirtIOGPU *g,
+                              struct virtio_gpu_ctrl_command *cmd,
+                              struct virtio_gpu_ctrl_hdr *resp,
+                              size_t resp_len)
+{
+    size_t s;
+
+    if (cmd->cmd_hdr.flags & VIRTIO_GPU_FLAG_FENCE) {
+        resp->flags |= VIRTIO_GPU_FLAG_FENCE;
+        resp->fence_id = cmd->cmd_hdr.fence_id;
+        resp->ctx_id = cmd->cmd_hdr.ctx_id;
+    }
+    s = iov_from_buf(cmd->elem.in_sg, cmd->elem.in_num, 0, resp, resp_len);
+    if (s != resp_len) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: response size incorrect %zu vs %zu\n",
+                      __func__, s, resp_len);
+    }
+    virtqueue_push(cmd->vq, &cmd->elem, s);
+    virtio_notify(VIRTIO_DEVICE(g), cmd->vq);
+    cmd->finished = true;
+}
+
+void virtio_gpu_ctrl_response_nodata(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd,
+                                     enum virtio_gpu_ctrl_type type)
+{
+    struct virtio_gpu_ctrl_hdr resp;
+
+    memset(&resp, 0, sizeof(resp));
+    resp.type = type;
+    virtio_gpu_ctrl_response(g, cmd, &resp, sizeof(resp));
+}
+
+static void
+virtio_gpu_fill_display_info(VirtIOGPU *g,
+                             struct virtio_gpu_resp_display_info *dpy_info)
+{
+    int i;
+
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        if (g->enabled_output_bitmask & (1 << i)) {
+            dpy_info->pmodes[i].enabled = 1;
+            dpy_info->pmodes[i].r.width = g->req_state[i].width;
+            dpy_info->pmodes[i].r.height = g->req_state[i].height;
+        }
+    }
+}
+
+void virtio_gpu_get_display_info(VirtIOGPU *g,
+                                 struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_resp_display_info display_info;
+
+    trace_virtio_gpu_cmd_get_display_info();
+    memset(&display_info, 0, sizeof(display_info));
+    display_info.hdr.type = VIRTIO_GPU_RESP_OK_DISPLAY_INFO;
+    virtio_gpu_fill_display_info(g, &display_info);
+    virtio_gpu_ctrl_response(g, cmd, &display_info.hdr,
+                             sizeof(display_info));
+}
+
+static pixman_format_code_t get_pixman_format(uint32_t virtio_gpu_format)
+{
+    switch (virtio_gpu_format) {
+#ifdef HOST_WORDS_BIGENDIAN
+    case VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM:
+        return PIXMAN_b8g8r8x8;
+    case VIRTIO_GPU_FORMAT_B8G8R8A8_UNORM:
+        return PIXMAN_b8g8r8a8;
+    case VIRTIO_GPU_FORMAT_X8R8G8B8_UNORM:
+        return PIXMAN_x8r8g8b8;
+    case VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM:
+        return PIXMAN_a8r8g8b8;
+    case VIRTIO_GPU_FORMAT_R8G8B8X8_UNORM:
+        return PIXMAN_r8g8b8x8;
+    case VIRTIO_GPU_FORMAT_R8G8B8A8_UNORM:
+        return PIXMAN_r8g8b8a8;
+    case VIRTIO_GPU_FORMAT_X8B8G8R8_UNORM:
+        return PIXMAN_x8b8g8r8;
+    case VIRTIO_GPU_FORMAT_A8B8G8R8_UNORM:
+        return PIXMAN_a8b8g8r8;
+#else
+    case VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM:
+        return PIXMAN_x8r8g8b8;
+    case VIRTIO_GPU_FORMAT_B8G8R8A8_UNORM:
+        return PIXMAN_a8r8g8b8;
+    case VIRTIO_GPU_FORMAT_X8R8G8B8_UNORM:
+        return PIXMAN_b8g8r8x8;
+    case VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM:
+        return PIXMAN_b8g8r8a8;
+    case VIRTIO_GPU_FORMAT_R8G8B8X8_UNORM:
+        return PIXMAN_x8b8g8r8;
+    case VIRTIO_GPU_FORMAT_R8G8B8A8_UNORM:
+        return PIXMAN_a8b8g8r8;
+    case VIRTIO_GPU_FORMAT_X8B8G8R8_UNORM:
+        return PIXMAN_r8g8b8x8;
+    case VIRTIO_GPU_FORMAT_A8B8G8R8_UNORM:
+        return PIXMAN_r8g8b8a8;
+#endif
+    default:
+        return 0;
+    }
+}
+
+static void virtio_gpu_resource_create_2d(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    pixman_format_code_t pformat;
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_resource_create_2d c2d;
+
+    VIRTIO_GPU_FILL_CMD(c2d);
+    trace_virtio_gpu_cmd_res_create_2d(c2d.resource_id, c2d.format,
+                                       c2d.width, c2d.height);
+
+    if (c2d.resource_id == 0) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: resource id 0 is not allowed\n",
+                      __func__);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+
+    res = virtio_gpu_find_resource(g, c2d.resource_id);
+    if (res) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: resource already exists %d\n",
+                      __func__, c2d.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+
+    res = g_new0(struct virtio_gpu_simple_resource, 1);
+
+    res->width = c2d.width;
+    res->height = c2d.height;
+    res->format = c2d.format;
+    res->resource_id = c2d.resource_id;
+
+    pformat = get_pixman_format(c2d.format);
+    if (!pformat) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: host couldn't handle guest format %d\n",
+                      __func__, c2d.format);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        return;
+    }
+    res->image = pixman_image_create_bits(pformat,
+                                          c2d.width,
+                                          c2d.height,
+                                          NULL, 0);
+
+    if (!res->image) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: resource creation failed %d %d %d\n",
+                      __func__, c2d.resource_id, c2d.width, c2d.height);
+        g_free(res);
+        cmd->error = VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY;
+        return;
+    }
+
+    QTAILQ_INSERT_HEAD(&g->reslist, res, next);
+}
+
+static void virtio_gpu_resource_destroy(VirtIOGPU *g,
+                                        struct virtio_gpu_simple_resource *res)
+{
+    pixman_image_unref(res->image);
+    QTAILQ_REMOVE(&g->reslist, res, next);
+    g_free(res);
+}
+
+static void virtio_gpu_resource_unref(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_resource_unref unref;
+
+    VIRTIO_GPU_FILL_CMD(unref);
+    trace_virtio_gpu_cmd_res_unref(unref.resource_id);
+
+    res = virtio_gpu_find_resource(g, unref.resource_id);
+    if (!res) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal resource specified %d\n",
+                      __func__, unref.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+    virtio_gpu_resource_destroy(g, res);
+}
+
+static void virtio_gpu_transfer_to_host_2d(VirtIOGPU *g,
+                                           struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_simple_resource *res;
+    int h;
+    uint32_t src_offset, dst_offset, stride;
+    int bpp;
+    pixman_format_code_t format;
+    struct virtio_gpu_transfer_to_host_2d t2d;
+
+    VIRTIO_GPU_FILL_CMD(t2d);
+    trace_virtio_gpu_cmd_res_xfer_toh_2d(t2d.resource_id);
+
+    res = virtio_gpu_find_resource(g, t2d.resource_id);
+    if (!res || !res->iov) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal resource specified %d\n",
+                      __func__, t2d.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+
+    if (t2d.r.x > res->width ||
+        t2d.r.y > res->height ||
+        t2d.r.width > res->width ||
+        t2d.r.height > res->height ||
+        t2d.r.x + t2d.r.width > res->width ||
+        t2d.r.y + t2d.r.height > res->height) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: transfer bounds outside resource"
+                      " bounds for resource %d: %d %d %d %d vs %d %d\n",
+                      __func__, t2d.resource_id, t2d.r.x, t2d.r.y,
+                      t2d.r.width, t2d.r.height, res->width, res->height);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        return;
+    }
+
+    format = pixman_image_get_format(res->image);
+    bpp = (PIXMAN_FORMAT_BPP(format) + 7) / 8;
+    stride = pixman_image_get_stride(res->image);
+
+    if (t2d.offset || t2d.r.x || t2d.r.y ||
+        t2d.r.width != pixman_image_get_width(res->image)) {
+        void *img_data = pixman_image_get_data(res->image);
+        for (h = 0; h < t2d.r.height; h++) {
+            src_offset = t2d.offset + stride * h;
+            dst_offset = (t2d.r.y + h) * stride + (t2d.r.x * bpp);
+
+            iov_to_buf(res->iov, res->iov_cnt, src_offset,
+                       (uint8_t *)img_data
+                       + dst_offset, t2d.r.width * bpp);
+        }
+    } else {
+        iov_to_buf(res->iov, res->iov_cnt, 0,
+                   pixman_image_get_data(res->image),
+                   pixman_image_get_stride(res->image)
+                   * pixman_image_get_height(res->image));
+    }
+}
+
+static void virtio_gpu_resource_flush(VirtIOGPU *g,
+                                      struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_resource_flush rf;
+    pixman_region16_t flush_region;
+    int i;
+
+    VIRTIO_GPU_FILL_CMD(rf);
+    trace_virtio_gpu_cmd_res_flush(rf.resource_id,
+                                   rf.r.width, rf.r.height, rf.r.x, rf.r.y);
+
+    res = virtio_gpu_find_resource(g, rf.resource_id);
+    if (!res) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal resource specified %d\n",
+                      __func__, rf.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+
+    if (rf.r.x > res->width ||
+        rf.r.y > res->height ||
+        rf.r.width > res->width ||
+        rf.r.height > res->height ||
+        rf.r.x + rf.r.width > res->width ||
+        rf.r.y + rf.r.height > res->height) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: flush bounds outside resource"
+                      " bounds for resource %d: %d %d %d %d vs %d %d\n",
+                      __func__, rf.resource_id, rf.r.x, rf.r.y,
+                      rf.r.width, rf.r.height, res->width, res->height);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        return;
+    }
+
+    pixman_region_init_rect(&flush_region,
+                            rf.r.x, rf.r.y, rf.r.width, rf.r.height);
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        struct virtio_gpu_scanout *scanout;
+        pixman_region16_t region, finalregion;
+        pixman_box16_t *extents;
+
+        if (!(res->scanout_bitmask & (1 << i))) {
+            continue;
+        }
+        scanout = &g->scanout[i];
+
+        pixman_region_init(&finalregion);
+        pixman_region_init_rect(&region, scanout->x, scanout->y,
+                                scanout->width, scanout->height);
+
+        pixman_region_intersect(&finalregion, &flush_region, &region);
+        pixman_region_translate(&finalregion, -scanout->x, -scanout->y);
+        extents = pixman_region_extents(&finalregion);
+        /* work out the area we need to update for each console */
+        dpy_gfx_update(g->scanout[i].con,
+                       extents->x1, extents->y1,
+                       extents->x2 - extents->x1,
+                       extents->y2 - extents->y1);
+
+        pixman_region_fini(&region);
+        pixman_region_fini(&finalregion);
+    }
+    pixman_region_fini(&flush_region);
+}
+
+static void virtio_unref_resource(pixman_image_t *image, void *data)
+{
+    pixman_image_unref(data);
+}
+
+static void virtio_gpu_set_scanout(VirtIOGPU *g,
+                                   struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_scanout *scanout;
+    pixman_format_code_t format;
+    uint32_t offset;
+    int bpp;
+    struct virtio_gpu_set_scanout ss;
+
+    VIRTIO_GPU_FILL_CMD(ss);
+    trace_virtio_gpu_cmd_set_scanout(ss.scanout_id, ss.resource_id,
+                                     ss.r.width, ss.r.height, ss.r.x, ss.r.y);
+
+    if (ss.scanout_id >= g->conf.max_outputs) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal scanout id specified %d",
+                      __func__, ss.scanout_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_SCANOUT_ID;
+        return;
+    }
+
+    g->enable = 1;
+    if (ss.resource_id == 0) {
+        scanout = &g->scanout[ss.scanout_id];
+        if (scanout->resource_id) {
+            res = virtio_gpu_find_resource(g, scanout->resource_id);
+            if (res) {
+                res->scanout_bitmask &= ~(1 << ss.scanout_id);
+            }
+        }
+        if (ss.scanout_id == 0) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "%s: illegal scanout id specified %d",
+                          __func__, ss.scanout_id);
+            cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_SCANOUT_ID;
+            return;
+        }
+        dpy_gfx_replace_surface(g->scanout[ss.scanout_id].con, NULL);
+        scanout->ds = NULL;
+        scanout->width = 0;
+        scanout->height = 0;
+        return;
+    }
+
+    /* create a surface for this scanout */
+    res = virtio_gpu_find_resource(g, ss.resource_id);
+    if (!res) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal resource specified %d\n",
+                      __func__, ss.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+
+    if (ss.r.x > res->width ||
+        ss.r.y > res->height ||
+        ss.r.width > res->width ||
+        ss.r.height > res->height ||
+        ss.r.x + ss.r.width > res->width ||
+        ss.r.y + ss.r.height > res->height) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal scanout %d bounds for"
+                      " resource %d, (%d,%d)+%d,%d vs %d %d\n",
+                      __func__, ss.scanout_id, ss.resource_id, ss.r.x, ss.r.y,
+                      ss.r.width, ss.r.height, res->width, res->height);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER;
+        return;
+    }
+
+    scanout = &g->scanout[ss.scanout_id];
+
+    format = pixman_image_get_format(res->image);
+    bpp = (PIXMAN_FORMAT_BPP(format) + 7) / 8;
+    offset = (ss.r.x * bpp) + ss.r.y * pixman_image_get_stride(res->image);
+    if (!scanout->ds || surface_data(scanout->ds)
+        != ((uint8_t *)pixman_image_get_data(res->image) + offset) ||
+        scanout->width != ss.r.width ||
+        scanout->height != ss.r.height) {
+        pixman_image_t *rect;
+        void *ptr = (uint8_t *)pixman_image_get_data(res->image) + offset;
+        rect = pixman_image_create_bits(format, ss.r.width, ss.r.height, ptr,
+                                        pixman_image_get_stride(res->image));
+        pixman_image_ref(res->image);
+        pixman_image_set_destroy_function(rect, virtio_unref_resource,
+                                          res->image);
+        /* realloc the surface ptr */
+        scanout->ds = qemu_create_displaysurface_pixman(rect);
+        if (!scanout->ds) {
+            cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+            return;
+        }
+        dpy_gfx_replace_surface(g->scanout[ss.scanout_id].con, scanout->ds);
+    }
+
+    res->scanout_bitmask |= (1 << ss.scanout_id);
+    scanout->resource_id = ss.resource_id;
+    scanout->x = ss.r.x;
+    scanout->y = ss.r.y;
+    scanout->width = ss.r.width;
+    scanout->height = ss.r.height;
+}
+
+int virtio_gpu_create_mapping_iov(struct virtio_gpu_resource_attach_backing *ab,
+                                  struct virtio_gpu_ctrl_command *cmd,
+                                  uint64_t **addr, struct iovec **iov)
+{
+    struct virtio_gpu_mem_entry *ents;
+    size_t esize, s;
+    int i;
+
+    if (ab->nr_entries > 16384) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: nr_entries is too big (%d > 16384)\n",
+                      __func__, ab->nr_entries);
+        return -1;
+    }
+
+    esize = sizeof(*ents) * ab->nr_entries;
+    ents = g_malloc(esize);
+    s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num,
+                   sizeof(*ab), ents, esize);
+    if (s != esize) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: command data size incorrect %zu vs %zu\n",
+                      __func__, s, esize);
+        g_free(ents);
+        return -1;
+    }
+
+    *iov = g_malloc0(sizeof(struct iovec) * ab->nr_entries);
+    if (addr) {
+        *addr = g_malloc0(sizeof(uint64_t) * ab->nr_entries);
+    }
+    for (i = 0; i < ab->nr_entries; i++) {
+        hwaddr len = ents[i].length;
+        (*iov)[i].iov_len = ents[i].length;
+        (*iov)[i].iov_base = cpu_physical_memory_map(ents[i].addr, &len, 1);
+        if (addr) {
+            (*addr)[i] = ents[i].addr;
+        }
+        if (!(*iov)[i].iov_base || len != ents[i].length) {
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: failed to map MMIO memory for"
+                          " resource %d element %d\n",
+                          __func__, ab->resource_id, i);
+            virtio_gpu_cleanup_mapping_iov(*iov, i);
+            g_free(ents);
+            *iov = NULL;
+            if (addr) {
+                g_free(*addr);
+                *addr = NULL;
+            }
+            return -1;
+        }
+    }
+    g_free(ents);
+    return 0;
+}
+
+void virtio_gpu_cleanup_mapping_iov(struct iovec *iov, uint32_t count)
+{
+    int i;
+
+    for (i = 0; i < count; i++) {
+        cpu_physical_memory_unmap(iov[i].iov_base, iov[i].iov_len, 1,
+                                  iov[i].iov_len);
+    }
+    g_free(iov);
+}
+
+static void virtio_gpu_cleanup_mapping(struct virtio_gpu_simple_resource *res)
+{
+    virtio_gpu_cleanup_mapping_iov(res->iov, res->iov_cnt);
+    res->iov = NULL;
+    res->iov_cnt = 0;
+    g_free(res->addrs);
+    res->addrs = NULL;
+}
+
+static void
+virtio_gpu_resource_attach_backing(VirtIOGPU *g,
+                                   struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_resource_attach_backing ab;
+    int ret;
+
+    VIRTIO_GPU_FILL_CMD(ab);
+    trace_virtio_gpu_cmd_res_back_attach(ab.resource_id);
+
+    res = virtio_gpu_find_resource(g, ab.resource_id);
+    if (!res) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal resource specified %d\n",
+                      __func__, ab.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+
+    ret = virtio_gpu_create_mapping_iov(&ab, cmd, &res->addrs, &res->iov);
+    if (ret != 0) {
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        return;
+    }
+
+    res->iov_cnt = ab.nr_entries;
+}
+
+static void
+virtio_gpu_resource_detach_backing(VirtIOGPU *g,
+                                   struct virtio_gpu_ctrl_command *cmd)
+{
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_resource_detach_backing detach;
+
+    VIRTIO_GPU_FILL_CMD(detach);
+    trace_virtio_gpu_cmd_res_back_detach(detach.resource_id);
+
+    res = virtio_gpu_find_resource(g, detach.resource_id);
+    if (!res || !res->iov) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: illegal resource specified %d\n",
+                      __func__, detach.resource_id);
+        cmd->error = VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID;
+        return;
+    }
+    virtio_gpu_cleanup_mapping(res);
+}
+
+static void virtio_gpu_simple_process_cmd(VirtIOGPU *g,
+                                          struct virtio_gpu_ctrl_command *cmd)
+{
+    VIRTIO_GPU_FILL_CMD(cmd->cmd_hdr);
+
+    switch (cmd->cmd_hdr.type) {
+    case VIRTIO_GPU_CMD_GET_DISPLAY_INFO:
+        virtio_gpu_get_display_info(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_CREATE_2D:
+        virtio_gpu_resource_create_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_UNREF:
+        virtio_gpu_resource_unref(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_FLUSH:
+        virtio_gpu_resource_flush(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D:
+        virtio_gpu_transfer_to_host_2d(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_SET_SCANOUT:
+        virtio_gpu_set_scanout(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING:
+        virtio_gpu_resource_attach_backing(g, cmd);
+        break;
+    case VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING:
+        virtio_gpu_resource_detach_backing(g, cmd);
+        break;
+    default:
+        cmd->error = VIRTIO_GPU_RESP_ERR_UNSPEC;
+        break;
+    }
+    if (!cmd->finished) {
+        virtio_gpu_ctrl_response_nodata(g, cmd, cmd->error ? cmd->error :
+                                        VIRTIO_GPU_RESP_OK_NODATA);
+    }
+}
+
+static void virtio_gpu_handle_ctrl_cb(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    qemu_bh_schedule(g->ctrl_bh);
+}
+
+static void virtio_gpu_handle_cursor_cb(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    qemu_bh_schedule(g->cursor_bh);
+}
+
+void virtio_gpu_process_cmdq(VirtIOGPU *g)
+{
+    struct virtio_gpu_ctrl_command *cmd;
+
+    while (!QTAILQ_EMPTY(&g->cmdq)) {
+        cmd = QTAILQ_FIRST(&g->cmdq);
+
+        /* process command */
+        VIRGL(g, virtio_gpu_virgl_process_cmd, virtio_gpu_simple_process_cmd,
+              g, cmd);
+        if (cmd->waiting) {
+            break;
+        }
+        QTAILQ_REMOVE(&g->cmdq, cmd, next);
+        if (virtio_gpu_stats_enabled(g->conf)) {
+            g->stats.requests++;
+        }
+
+        if (!cmd->finished) {
+            QTAILQ_INSERT_TAIL(&g->fenceq, cmd, next);
+            g->inflight++;
+            if (virtio_gpu_stats_enabled(g->conf)) {
+                if (g->stats.max_inflight < g->inflight) {
+                    g->stats.max_inflight = g->inflight;
+                }
+                fprintf(stderr, "inflight: %3d (+)\r", g->inflight);
+            }
+        } else {
+            g_free(cmd);
+        }
+    }
+}
+
+static void virtio_gpu_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    struct virtio_gpu_ctrl_command *cmd;
+
+    if (!virtio_queue_ready(vq)) {
+        return;
+    }
+
+#ifdef CONFIG_VIRGL
+    if (!g->renderer_inited && g->use_virgl_renderer) {
+        virtio_gpu_virgl_init(g);
+        g->renderer_inited = true;
+    }
+#endif
+
+    cmd = virtqueue_pop(vq, sizeof(struct virtio_gpu_ctrl_command));
+    while (cmd) {
+        cmd->vq = vq;
+        cmd->error = 0;
+        cmd->finished = false;
+        cmd->waiting = false;
+        QTAILQ_INSERT_TAIL(&g->cmdq, cmd, next);
+        cmd = virtqueue_pop(vq, sizeof(struct virtio_gpu_ctrl_command));
+    }
+
+    virtio_gpu_process_cmdq(g);
+
+#ifdef CONFIG_VIRGL
+    if (g->use_virgl_renderer) {
+        virtio_gpu_virgl_fence_poll(g);
+    }
+#endif
+}
+
+static void virtio_gpu_ctrl_bh(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+    virtio_gpu_handle_ctrl(&g->parent_obj, g->ctrl_vq);
+}
+
+static void virtio_gpu_handle_cursor(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    VirtQueueElement *elem;
+    size_t s;
+    struct virtio_gpu_update_cursor cursor_info;
+
+    if (!virtio_queue_ready(vq)) {
+        return;
+    }
+    for (;;) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        s = iov_to_buf(elem->out_sg, elem->out_num, 0,
+                       &cursor_info, sizeof(cursor_info));
+        if (s != sizeof(cursor_info)) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "%s: cursor size incorrect %zu vs %zu\n",
+                          __func__, s, sizeof(cursor_info));
+        } else {
+            update_cursor(g, &cursor_info);
+        }
+        virtqueue_push(vq, elem, 0);
+        virtio_notify(vdev, vq);
+        g_free(elem);
+    }
+}
+
+static void virtio_gpu_cursor_bh(void *opaque)
+{
+    VirtIOGPU *g = opaque;
+    virtio_gpu_handle_cursor(&g->parent_obj, g->cursor_vq);
+}
+
+static void virtio_gpu_invalidate_display(void *opaque)
+{
+}
+
+static void virtio_gpu_update_display(void *opaque)
+{
+}
+
+static void virtio_gpu_text_update(void *opaque, console_ch_t *chardata)
+{
+}
+
+static int virtio_gpu_ui_info(void *opaque, uint32_t idx, QemuUIInfo *info)
+{
+    VirtIOGPU *g = opaque;
+
+    if (idx >= g->conf.max_outputs) {
+        return -1;
+    }
+
+    g->req_state[idx].x = info->xoff;
+    g->req_state[idx].y = info->yoff;
+    g->req_state[idx].width = info->width;
+    g->req_state[idx].height = info->height;
+
+    if (info->width && info->height) {
+        g->enabled_output_bitmask |= (1 << idx);
+    } else {
+        g->enabled_output_bitmask &= ~(1 << idx);
+    }
+
+    /* send event to guest */
+    virtio_gpu_notify_event(g, VIRTIO_GPU_EVENT_DISPLAY);
+    return 0;
+}
+
+static void virtio_gpu_gl_block(void *opaque, bool block)
+{
+    VirtIOGPU *g = opaque;
+
+    if (block) {
+        g->renderer_blocked++;
+    } else {
+        g->renderer_blocked--;
+    }
+    assert(g->renderer_blocked >= 0);
+
+    if (g->renderer_blocked == 0) {
+        virtio_gpu_process_cmdq(g);
+    }
+}
+
+const GraphicHwOps virtio_gpu_ops = {
+    .invalidate = virtio_gpu_invalidate_display,
+    .gfx_update = virtio_gpu_update_display,
+    .text_update = virtio_gpu_text_update,
+    .ui_info = virtio_gpu_ui_info,
+    .gl_block = virtio_gpu_gl_block,
+};
+
+static const VMStateDescription vmstate_virtio_gpu_scanout = {
+    .name = "virtio-gpu-one-scanout",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(resource_id, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(width, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(height, struct virtio_gpu_scanout),
+        VMSTATE_INT32(x, struct virtio_gpu_scanout),
+        VMSTATE_INT32(y, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(cursor.resource_id, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(cursor.hot_x, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(cursor.hot_y, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(cursor.pos.x, struct virtio_gpu_scanout),
+        VMSTATE_UINT32(cursor.pos.y, struct virtio_gpu_scanout),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static const VMStateDescription vmstate_virtio_gpu_scanouts = {
+    .name = "virtio-gpu-scanouts",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(enable, struct VirtIOGPU),
+        VMSTATE_UINT32_EQUAL(conf.max_outputs, struct VirtIOGPU),
+        VMSTATE_STRUCT_VARRAY_UINT32(scanout, struct VirtIOGPU,
+                                     conf.max_outputs, 1,
+                                     vmstate_virtio_gpu_scanout,
+                                     struct virtio_gpu_scanout),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static void virtio_gpu_save(QEMUFile *f, void *opaque, size_t size)
+{
+    VirtIOGPU *g = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(g);
+    struct virtio_gpu_simple_resource *res;
+    int i;
+
+    virtio_save(vdev, f);
+
+    /* in 2d mode we should never find unprocessed commands here */
+    assert(QTAILQ_EMPTY(&g->cmdq));
+
+    QTAILQ_FOREACH(res, &g->reslist, next) {
+        qemu_put_be32(f, res->resource_id);
+        qemu_put_be32(f, res->width);
+        qemu_put_be32(f, res->height);
+        qemu_put_be32(f, res->format);
+        qemu_put_be32(f, res->iov_cnt);
+        for (i = 0; i < res->iov_cnt; i++) {
+            qemu_put_be64(f, res->addrs[i]);
+            qemu_put_be32(f, res->iov[i].iov_len);
+        }
+        qemu_put_buffer(f, (void *)pixman_image_get_data(res->image),
+                        pixman_image_get_stride(res->image) * res->height);
+    }
+    qemu_put_be32(f, 0); /* end of list */
+
+    vmstate_save_state(f, &vmstate_virtio_gpu_scanouts, g, NULL);
+}
+
+static int virtio_gpu_load(QEMUFile *f, void *opaque, size_t size)
+{
+    VirtIOGPU *g = opaque;
+    VirtIODevice *vdev = VIRTIO_DEVICE(g);
+    struct virtio_gpu_simple_resource *res;
+    struct virtio_gpu_scanout *scanout;
+    uint32_t resource_id, pformat;
+    int i, ret;
+
+    ret = virtio_load(vdev, f, VIRTIO_GPU_VM_VERSION);
+    if (ret) {
+        return ret;
+    }
+
+    resource_id = qemu_get_be32(f);
+    while (resource_id != 0) {
+        res = g_new0(struct virtio_gpu_simple_resource, 1);
+        res->resource_id = resource_id;
+        res->width = qemu_get_be32(f);
+        res->height = qemu_get_be32(f);
+        res->format = qemu_get_be32(f);
+        res->iov_cnt = qemu_get_be32(f);
+
+        /* allocate */
+        pformat = get_pixman_format(res->format);
+        if (!pformat) {
+            return -EINVAL;
+        }
+        res->image = pixman_image_create_bits(pformat,
+                                              res->width, res->height,
+                                              NULL, 0);
+        if (!res->image) {
+            return -EINVAL;
+        }
+
+        res->addrs = g_new(uint64_t, res->iov_cnt);
+        res->iov = g_new(struct iovec, res->iov_cnt);
+
+        /* read data */
+        for (i = 0; i < res->iov_cnt; i++) {
+            res->addrs[i] = qemu_get_be64(f);
+            res->iov[i].iov_len = qemu_get_be32(f);
+        }
+        qemu_get_buffer(f, (void *)pixman_image_get_data(res->image),
+                        pixman_image_get_stride(res->image) * res->height);
+
+        /* restore mapping */
+        for (i = 0; i < res->iov_cnt; i++) {
+            hwaddr len = res->iov[i].iov_len;
+            res->iov[i].iov_base =
+                cpu_physical_memory_map(res->addrs[i], &len, 1);
+            if (!res->iov[i].iov_base || len != res->iov[i].iov_len) {
+                return -EINVAL;
+            }
+        }
+
+        QTAILQ_INSERT_HEAD(&g->reslist, res, next);
+
+        resource_id = qemu_get_be32(f);
+    }
+
+    /* load & apply scanout state */
+    vmstate_load_state(f, &vmstate_virtio_gpu_scanouts, g, 1);
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        scanout = &g->scanout[i];
+        if (!scanout->resource_id) {
+            continue;
+        }
+        res = virtio_gpu_find_resource(g, scanout->resource_id);
+        if (!res) {
+            return -EINVAL;
+        }
+        scanout->ds = qemu_create_displaysurface_pixman(res->image);
+        if (!scanout->ds) {
+            return -EINVAL;
+        }
+
+        dpy_gfx_replace_surface(scanout->con, scanout->ds);
+        dpy_gfx_update(scanout->con, 0, 0, scanout->width, scanout->height);
+        update_cursor(g, &scanout->cursor);
+        res->scanout_bitmask |= (1 << i);
+    }
+
+    return 0;
+}
+
+static void virtio_gpu_device_realize(DeviceState *qdev, Error **errp)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(qdev);
+    VirtIOGPU *g = VIRTIO_GPU(qdev);
+    bool have_virgl;
+    int i;
+
+    if (g->conf.max_outputs > VIRTIO_GPU_MAX_SCANOUTS) {
+        error_setg(errp, "invalid max_outputs > %d", VIRTIO_GPU_MAX_SCANOUTS);
+        return;
+    }
+
+    g->config_size = sizeof(struct virtio_gpu_config);
+    g->virtio_config.num_scanouts = g->conf.max_outputs;
+    virtio_init(VIRTIO_DEVICE(g), "virtio-gpu", VIRTIO_ID_GPU,
+                g->config_size);
+
+    g->req_state[0].width = 1024;
+    g->req_state[0].height = 768;
+
+    g->use_virgl_renderer = false;
+#if !defined(CONFIG_VIRGL) || defined(HOST_WORDS_BIGENDIAN)
+    have_virgl = false;
+#else
+    have_virgl = display_opengl;
+#endif
+    if (!have_virgl) {
+        g->conf.flags &= ~(1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED);
+    }
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        /* use larger control queue in 3d mode */
+        g->ctrl_vq   = virtio_add_queue(vdev, 256, virtio_gpu_handle_ctrl_cb);
+        g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+        g->virtio_config.num_capsets = 1;
+    } else {
+        g->ctrl_vq   = virtio_add_queue(vdev, 64, virtio_gpu_handle_ctrl_cb);
+        g->cursor_vq = virtio_add_queue(vdev, 16, virtio_gpu_handle_cursor_cb);
+    }
+
+    g->ctrl_bh = qemu_bh_new(virtio_gpu_ctrl_bh, g);
+    g->cursor_bh = qemu_bh_new(virtio_gpu_cursor_bh, g);
+    QTAILQ_INIT(&g->reslist);
+    QTAILQ_INIT(&g->cmdq);
+    QTAILQ_INIT(&g->fenceq);
+
+    g->enabled_output_bitmask = 1;
+    g->qdev = qdev;
+
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        g->scanout[i].con =
+            graphic_console_init(DEVICE(g), i, &virtio_gpu_ops, g);
+        if (i > 0) {
+            dpy_gfx_replace_surface(g->scanout[i].con, NULL);
+        }
+    }
+
+    if (virtio_gpu_virgl_enabled(g->conf)) {
+        error_setg(&g->migration_blocker, "virgl is not yet migratable");
+        migrate_add_blocker(g->migration_blocker);
+    }
+}
+
+static void virtio_gpu_device_unrealize(DeviceState *qdev, Error **errp)
+{
+    VirtIOGPU *g = VIRTIO_GPU(qdev);
+    if (g->migration_blocker) {
+        migrate_del_blocker(g->migration_blocker);
+        error_free(g->migration_blocker);
+    }
+}
+
+static void virtio_gpu_instance_init(Object *obj)
+{
+}
+
+static void virtio_gpu_reset(VirtIODevice *vdev)
+{
+    VirtIOGPU *g = VIRTIO_GPU(vdev);
+    struct virtio_gpu_simple_resource *res, *tmp;
+    int i;
+
+    g->enable = 0;
+
+    QTAILQ_FOREACH_SAFE(res, &g->reslist, next, tmp) {
+        virtio_gpu_resource_destroy(g, res);
+    }
+    for (i = 0; i < g->conf.max_outputs; i++) {
+#if 0
+        g->req_state[i].x = 0;
+        g->req_state[i].y = 0;
+        if (i == 0) {
+            g->req_state[0].width = 1024;
+            g->req_state[0].height = 768;
+        } else {
+            g->req_state[i].width = 0;
+            g->req_state[i].height = 0;
+        }
+#endif
+        g->scanout[i].resource_id = 0;
+        g->scanout[i].width = 0;
+        g->scanout[i].height = 0;
+        g->scanout[i].x = 0;
+        g->scanout[i].y = 0;
+        g->scanout[i].ds = NULL;
+    }
+    g->enabled_output_bitmask = 1;
+
+#ifdef CONFIG_VIRGL
+    if (g->use_virgl_renderer) {
+        virtio_gpu_virgl_reset(g);
+        g->use_virgl_renderer = 0;
+    }
+#endif
+}
+
+VMSTATE_VIRTIO_DEVICE(gpu, VIRTIO_GPU_VM_VERSION, virtio_gpu_load,
+                      virtio_gpu_save);
+
+static Property virtio_gpu_properties[] = {
+    DEFINE_PROP_UINT32("max_outputs", VirtIOGPU, conf.max_outputs, 1),
+#ifdef CONFIG_VIRGL
+    DEFINE_PROP_BIT("virgl", VirtIOGPU, conf.flags,
+                    VIRTIO_GPU_FLAG_VIRGL_ENABLED, true),
+    DEFINE_PROP_BIT("stats", VirtIOGPU, conf.flags,
+                    VIRTIO_GPU_FLAG_STATS_ENABLED, false),
+#endif
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_gpu_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
+
+    vdc->realize = virtio_gpu_device_realize;
+    vdc->unrealize = virtio_gpu_device_unrealize;
+    vdc->get_config = virtio_gpu_get_config;
+    vdc->set_config = virtio_gpu_set_config;
+    vdc->get_features = virtio_gpu_get_features;
+    vdc->set_features = virtio_gpu_set_features;
+
+    vdc->reset = virtio_gpu_reset;
+
+    dc->props = virtio_gpu_properties;
+    dc->vmsd = &vmstate_virtio_gpu;
+}
+
+static const TypeInfo virtio_gpu_info = {
+    .name = TYPE_VIRTIO_GPU,
+    .parent = TYPE_VIRTIO_DEVICE,
+    .instance_size = sizeof(VirtIOGPU),
+    .instance_init = virtio_gpu_instance_init,
+    .class_init = virtio_gpu_class_init,
+};
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_gpu_info);
+}
+
+type_init(virtio_register_types)
+
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctrl_hdr)                != 24);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_update_cursor)           != 56);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_unref)          != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_create_2d)      != 40);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_set_scanout)             != 48);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_flush)          != 48);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_transfer_to_host_2d)     != 56);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_mem_entry)               != 16);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_attach_backing) != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_detach_backing) != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_display_info)       != 408);
+
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_transfer_host_3d)        != 72);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resource_create_3d)      != 72);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_create)              != 96);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_destroy)             != 24);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_ctx_resource)            != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_cmd_submit)              != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_get_capset_info)         != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_capset_info)        != 40);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_get_capset)              != 32);
+QEMU_BUILD_BUG_ON(sizeof(struct virtio_gpu_resp_capset)             != 24);
diff --git a/hw/display/virtio-vga.c b/hw/display/virtio-vga.c
new file mode 100644
index 0000000..5b510a1
--- /dev/null
+++ b/hw/display/virtio-vga.c
@@ -0,0 +1,209 @@
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "ui/console.h"
+#include "vga_int.h"
+#include "hw/virtio/virtio-pci.h"
+#include "qapi/error.h"
+
+/*
+ * virtio-vga: This extends VirtioPCIProxy.
+ */
+#define TYPE_VIRTIO_VGA "virtio-vga"
+#define VIRTIO_VGA(obj) \
+        OBJECT_CHECK(VirtIOVGA, (obj), TYPE_VIRTIO_VGA)
+
+typedef struct VirtIOVGA {
+    VirtIOPCIProxy parent_obj;
+    VirtIOGPU      vdev;
+    VGACommonState vga;
+    MemoryRegion   vga_mrs[3];
+} VirtIOVGA;
+
+static void virtio_vga_invalidate_display(void *opaque)
+{
+    VirtIOVGA *vvga = opaque;
+
+    if (vvga->vdev.enable) {
+        virtio_gpu_ops.invalidate(&vvga->vdev);
+    } else {
+        vvga->vga.hw_ops->invalidate(&vvga->vga);
+    }
+}
+
+static void virtio_vga_update_display(void *opaque)
+{
+    VirtIOVGA *vvga = opaque;
+
+    if (vvga->vdev.enable) {
+        virtio_gpu_ops.gfx_update(&vvga->vdev);
+    } else {
+        vvga->vga.hw_ops->gfx_update(&vvga->vga);
+    }
+}
+
+static void virtio_vga_text_update(void *opaque, console_ch_t *chardata)
+{
+    VirtIOVGA *vvga = opaque;
+
+    if (vvga->vdev.enable) {
+        if (virtio_gpu_ops.text_update) {
+            virtio_gpu_ops.text_update(&vvga->vdev, chardata);
+        }
+    } else {
+        if (vvga->vga.hw_ops->text_update) {
+            vvga->vga.hw_ops->text_update(&vvga->vga, chardata);
+        }
+    }
+}
+
+static int virtio_vga_ui_info(void *opaque, uint32_t idx, QemuUIInfo *info)
+{
+    VirtIOVGA *vvga = opaque;
+
+    if (virtio_gpu_ops.ui_info) {
+        return virtio_gpu_ops.ui_info(&vvga->vdev, idx, info);
+    }
+    return -1;
+}
+
+static void virtio_vga_gl_block(void *opaque, bool block)
+{
+    VirtIOVGA *vvga = opaque;
+
+    if (virtio_gpu_ops.gl_block) {
+        virtio_gpu_ops.gl_block(&vvga->vdev, block);
+    }
+}
+
+static const GraphicHwOps virtio_vga_ops = {
+    .invalidate = virtio_vga_invalidate_display,
+    .gfx_update = virtio_vga_update_display,
+    .text_update = virtio_vga_text_update,
+    .ui_info = virtio_vga_ui_info,
+    .gl_block = virtio_vga_gl_block,
+};
+
+static const VMStateDescription vmstate_virtio_vga = {
+    .name = "virtio-vga",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        /* no pci stuff here, saving the virtio device will handle that */
+        VMSTATE_STRUCT(vga, VirtIOVGA, 0, vmstate_vga_common, VGACommonState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* VGA device wrapper around PCI device around virtio GPU */
+static void virtio_vga_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
+{
+    VirtIOVGA *vvga = VIRTIO_VGA(vpci_dev);
+    VirtIOGPU *g = &vvga->vdev;
+    VGACommonState *vga = &vvga->vga;
+    Error *err = NULL;
+    uint32_t offset;
+    int i;
+
+    /* init vga compat bits */
+    vga->vram_size_mb = 8;
+    vga_common_init(vga, OBJECT(vpci_dev), false);
+    vga_init(vga, OBJECT(vpci_dev), pci_address_space(&vpci_dev->pci_dev),
+             pci_address_space_io(&vpci_dev->pci_dev), true);
+    pci_register_bar(&vpci_dev->pci_dev, 0,
+                     PCI_BASE_ADDRESS_MEM_PREFETCH, &vga->vram);
+
+    /*
+     * Configure virtio bar and regions
+     *
+     * We use bar #2 for the mmio regions, to be compatible with stdvga.
+     * virtio regions are moved to the end of bar #2, to make room for
+     * the stdvga mmio registers at the start of bar #2.
+     */
+    vpci_dev->modern_mem_bar = 2;
+    vpci_dev->msix_bar = 4;
+    offset = memory_region_size(&vpci_dev->modern_bar);
+    offset -= vpci_dev->notify.size;
+    vpci_dev->notify.offset = offset;
+    offset -= vpci_dev->device.size;
+    vpci_dev->device.offset = offset;
+    offset -= vpci_dev->isr.size;
+    vpci_dev->isr.offset = offset;
+    offset -= vpci_dev->common.size;
+    vpci_dev->common.offset = offset;
+
+    /* init virtio bits */
+    qdev_set_parent_bus(DEVICE(g), BUS(&vpci_dev->bus));
+    virtio_pci_force_virtio_1(vpci_dev);
+    object_property_set_bool(OBJECT(g), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    /* add stdvga mmio regions */
+    pci_std_vga_mmio_region_init(vga, &vpci_dev->modern_bar,
+                                 vvga->vga_mrs, true);
+
+    vga->con = g->scanout[0].con;
+    graphic_console_set_hwops(vga->con, &virtio_vga_ops, vvga);
+
+    for (i = 0; i < g->conf.max_outputs; i++) {
+        object_property_set_link(OBJECT(g->scanout[i].con),
+                                 OBJECT(vpci_dev),
+                                 "device", errp);
+    }
+}
+
+static void virtio_vga_reset(DeviceState *dev)
+{
+    VirtIOVGA *vvga = VIRTIO_VGA(dev);
+    vvga->vdev.enable = 0;
+
+    vga_dirty_log_start(&vvga->vga);
+}
+
+static Property virtio_vga_properties[] = {
+    DEFINE_VIRTIO_GPU_PCI_PROPERTIES(VirtIOPCIProxy),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_vga_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
+    PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_DISPLAY, dc->categories);
+    dc->props = virtio_vga_properties;
+    dc->reset = virtio_vga_reset;
+    dc->vmsd = &vmstate_virtio_vga;
+    dc->hotpluggable = false;
+
+    k->realize = virtio_vga_realize;
+    pcidev_k->romfile = "vgabios-virtio.bin";
+    pcidev_k->class_id = PCI_CLASS_DISPLAY_VGA;
+}
+
+static void virtio_vga_inst_initfn(Object *obj)
+{
+    VirtIOVGA *dev = VIRTIO_VGA(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_GPU);
+}
+
+static TypeInfo virtio_vga_info = {
+    .name          = TYPE_VIRTIO_VGA,
+    .parent        = TYPE_VIRTIO_PCI,
+    .instance_size = sizeof(struct VirtIOVGA),
+    .instance_init = virtio_vga_inst_initfn,
+    .class_init    = virtio_vga_class_init,
+};
+
+static void virtio_vga_register_types(void)
+{
+    type_register_static(&virtio_vga_info);
+}
+
+type_init(virtio_vga_register_types)
diff --git a/hw/display/vmware_vga.c b/hw/display/vmware_vga.c
index 1751f19..e51a05e 100644
--- a/hw/display/vmware_vga.c
+++ b/hw/display/vmware_vga.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "trace.h"
@@ -64,17 +66,11 @@
     uint8_t *fifo_ptr;
     unsigned int fifo_size;
 
-    union {
-        uint32_t *fifo;
-        struct QEMU_PACKED {
-            uint32_t min;
-            uint32_t max;
-            uint32_t next_cmd;
-            uint32_t stop;
-            /* Add registers here when adding capabilities.  */
-            uint32_t fifo[0];
-        } *cmd;
-    };
+    uint32_t *fifo;
+    uint32_t fifo_min;
+    uint32_t fifo_max;
+    uint32_t fifo_next;
+    uint32_t fifo_stop;
 
 #define REDRAW_FIFO_LEN  512
     struct vmsvga_rect_s {
@@ -196,7 +192,7 @@
      */
     SVGA_FIFO_MIN = 0,
     SVGA_FIFO_MAX,      /* The distance from MIN to MAX must be at least 10K */
-    SVGA_FIFO_NEXT_CMD,
+    SVGA_FIFO_NEXT,
     SVGA_FIFO_STOP,
 
     /*
@@ -488,10 +484,10 @@
 #endif
 
 struct vmsvga_cursor_definition_s {
-    int width;
-    int height;
+    uint32_t width;
+    uint32_t height;
     int id;
-    int bpp;
+    uint32_t bpp;
     int hot_x;
     int hot_y;
     uint32_t mask[1024];
@@ -544,8 +540,6 @@
 }
 #endif
 
-#define CMD(f)  le32_to_cpu(s->cmd->f)
-
 static inline int vmsvga_fifo_length(struct vmsvga_state_s *s)
 {
     int num;
@@ -553,21 +547,45 @@
     if (!s->config || !s->enable) {
         return 0;
     }
-    num = CMD(next_cmd) - CMD(stop);
+
+    s->fifo_min  = le32_to_cpu(s->fifo[SVGA_FIFO_MIN]);
+    s->fifo_max  = le32_to_cpu(s->fifo[SVGA_FIFO_MAX]);
+    s->fifo_next = le32_to_cpu(s->fifo[SVGA_FIFO_NEXT]);
+    s->fifo_stop = le32_to_cpu(s->fifo[SVGA_FIFO_STOP]);
+
+    /* Check range and alignment.  */
+    if ((s->fifo_min | s->fifo_max | s->fifo_next | s->fifo_stop) & 3) {
+        return 0;
+    }
+    if (s->fifo_min < sizeof(uint32_t) * 4) {
+        return 0;
+    }
+    if (s->fifo_max > SVGA_FIFO_SIZE ||
+        s->fifo_min >= SVGA_FIFO_SIZE ||
+        s->fifo_stop >= SVGA_FIFO_SIZE ||
+        s->fifo_next >= SVGA_FIFO_SIZE) {
+        return 0;
+    }
+    if (s->fifo_max < s->fifo_min + 10 * 1024) {
+        return 0;
+    }
+
+    num = s->fifo_next - s->fifo_stop;
     if (num < 0) {
-        num += CMD(max) - CMD(min);
+        num += s->fifo_max - s->fifo_min;
     }
     return num >> 2;
 }
 
 static inline uint32_t vmsvga_fifo_read_raw(struct vmsvga_state_s *s)
 {
-    uint32_t cmd = s->fifo[CMD(stop) >> 2];
+    uint32_t cmd = s->fifo[s->fifo_stop >> 2];
 
-    s->cmd->stop = cpu_to_le32(CMD(stop) + 4);
-    if (CMD(stop) >= CMD(max)) {
-        s->cmd->stop = s->cmd->min;
+    s->fifo_stop += 4;
+    if (s->fifo_stop >= s->fifo_max) {
+        s->fifo_stop = s->fifo_min;
     }
+    s->fifo[SVGA_FIFO_STOP] = cpu_to_le32(s->fifo_stop);
     return cmd;
 }
 
@@ -579,15 +597,15 @@
 static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 {
     uint32_t cmd, colour;
-    int args, len;
+    int args, len, maxloop = 1024;
     int x, y, dx, dy, width, height;
     struct vmsvga_cursor_definition_s cursor;
     uint32_t cmd_start;
 
     len = vmsvga_fifo_length(s);
-    while (len > 0) {
+    while (len > 0 && --maxloop > 0) {
         /* May need to go back to the start of the command if incomplete */
-        cmd_start = s->cmd->stop;
+        cmd_start = s->fifo_stop;
 
         switch (cmd = vmsvga_fifo_read(s)) {
         case SVGA_CMD_UPDATE:
@@ -658,7 +676,10 @@
             cursor.bpp = vmsvga_fifo_read(s);
 
             args = SVGA_BITMAP_SIZE(x, y) + SVGA_PIXMAP_SIZE(x, y, cursor.bpp);
-            if (SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
+            if (cursor.width > 256 ||
+                cursor.height > 256 ||
+                cursor.bpp > 32 ||
+                SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
                 SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image) {
                     goto badcmd;
             }
@@ -743,7 +764,8 @@
             break;
 
         rewind:
-            s->cmd->stop = cmd_start;
+            s->fifo_stop = cmd_start;
+            s->fifo[SVGA_FIFO_STOP] = cpu_to_le32(s->fifo_stop);
             break;
         }
     }
@@ -1000,19 +1022,6 @@
     case SVGA_REG_CONFIG_DONE:
         if (value) {
             s->fifo = (uint32_t *) s->fifo_ptr;
-            /* Check range and alignment.  */
-            if ((CMD(min) | CMD(max) | CMD(next_cmd) | CMD(stop)) & 3) {
-                break;
-            }
-            if (CMD(min) < (uint8_t *) s->cmd->fifo - (uint8_t *) s->fifo) {
-                break;
-            }
-            if (CMD(max) > SVGA_FIFO_SIZE) {
-                break;
-            }
-            if (CMD(max) < CMD(min) + 10 * 1024) {
-                break;
-            }
             vga_dirty_log_stop(&s->vga);
         }
         s->config = !!value;
@@ -1124,7 +1133,7 @@
      * Is it more efficient to look at vram VGA-dirty bits or wait
      * for the driver to issue SVGA_CMD_UPDATE?
      */
-    if (memory_region_is_logging(&s->vga.vram)) {
+    if (memory_region_is_logging(&s->vga.vram, DIRTY_MEMORY_VGA)) {
         vga_sync_dirty_bitmap(&s->vga);
         dirty = memory_region_get_dirty(&s->vga.vram, 0,
             surface_stride(surface) * surface_height(surface),
@@ -1244,7 +1253,7 @@
 
     s->fifo_size = SVGA_FIFO_SIZE;
     memory_region_init_ram(&s->fifo_ram, NULL, "vmsvga.fifo", s->fifo_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->fifo_ram);
     s->fifo_ptr = memory_region_get_ram_ptr(&s->fifo_ram);
 
@@ -1298,7 +1307,7 @@
     },
 };
 
-static int pci_vmsvga_initfn(PCIDevice *dev)
+static void pci_vmsvga_realize(PCIDevice *dev, Error **errp)
 {
     struct pci_vmsvga_state_s *s = VMWARE_SVGA(dev);
 
@@ -1323,8 +1332,6 @@
         /* compatibility with pc-0.13 and older */
         vga_init_vbe(&s->chip.vga, OBJECT(dev), pci_address_space(dev));
     }
-
-    return 0;
 }
 
 static Property vga_vmware_properties[] = {
@@ -1338,7 +1345,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_vmsvga_initfn;
+    k->realize = pci_vmsvga_realize;
     k->romfile = "vgabios-vmware.bin";
     k->vendor_id = PCI_VENDOR_ID_VMWARE;
     k->device_id = SVGA_PCI_DEVICE_ID;
diff --git a/hw/display/xenfb.c b/hw/display/xenfb.c
index 8a61e95..46b7d5e 100644
--- a/hw/display/xenfb.c
+++ b/hw/display/xenfb.c
@@ -24,16 +24,7 @@
  *  with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "ui/console.h"
@@ -45,6 +36,8 @@
 #include <xen/io/kbdif.h>
 #include <xen/io/protocols.h>
 
+#include "trace.h"
+
 #ifndef BTN_LEFT
 #define BTN_LEFT 0x110 /* from <linux/input.h> */
 #endif
@@ -93,23 +86,24 @@
 
 static int common_bind(struct common *c)
 {
-    uint64_t mfn;
+    uint64_t val;
+    xen_pfn_t mfn;
 
-    if (xenstore_read_fe_uint64(&c->xendev, "page-ref", &mfn) == -1)
+    if (xenstore_read_fe_uint64(&c->xendev, "page-ref", &val) == -1)
 	return -1;
-    assert(mfn == (xen_pfn_t)mfn);
+    mfn = (xen_pfn_t)val;
+    assert(val == mfn);
 
     if (xenstore_read_fe_int(&c->xendev, "event-channel", &c->xendev.remote_port) == -1)
 	return -1;
 
-    c->page = xc_map_foreign_range(xen_xc, c->xendev.dom,
-				   XC_PAGE_SIZE,
-				   PROT_READ | PROT_WRITE, mfn);
+    c->page = xenforeignmemory_map(xen_fmem, c->xendev.dom,
+                                   PROT_READ | PROT_WRITE, 1, &mfn, NULL);
     if (c->page == NULL)
 	return -1;
 
     xen_be_bind_evtchn(&c->xendev);
-    xen_be_printf(&c->xendev, 1, "ring mfn %"PRIx64", remote-port %d, local-port %d\n",
+    xen_be_printf(&c->xendev, 1, "ring mfn %"PRI_xen_pfn", remote-port %d, local-port %d\n",
 		  mfn, c->xendev.remote_port, c->xendev.local_port);
 
     return 0;
@@ -119,7 +113,7 @@
 {
     xen_be_unbind_evtchn(&c->xendev);
     if (c->page) {
-	munmap(c->page, XC_PAGE_SIZE);
+        xenforeignmemory_unmap(xen_fmem, c->page, 1);
 	c->page = NULL;
     }
 }
@@ -246,9 +240,7 @@
     event.type = XENKBD_TYPE_MOTION;
     event.motion.rel_x = rel_x;
     event.motion.rel_y = rel_y;
-#if __XEN_LATEST_INTERFACE_VERSION__ >= 0x00030207
     event.motion.rel_z = rel_z;
-#endif
 
     return xenfb_kbd_event(xenfb, &event);
 }
@@ -263,12 +255,7 @@
     event.type = XENKBD_TYPE_POS;
     event.pos.abs_x = abs_x;
     event.pos.abs_y = abs_y;
-#if __XEN_LATEST_INTERFACE_VERSION__ == 0x00030207
-    event.pos.abs_z = z;
-#endif
-#if __XEN_LATEST_INTERFACE_VERSION__ >= 0x00030208
     event.pos.rel_z = z;
-#endif
 
     return xenfb_kbd_event(xenfb, &event);
 }
@@ -324,6 +311,8 @@
     int dh = surface_height(surface);
     int i;
 
+    trace_xenfb_mouse_event(opaque, dx, dy, dz, button_state,
+                            xenfb->abs_pointer_wanted);
     if (xenfb->abs_pointer_wanted)
 	xenfb_send_position(xenfb,
 			    dx * (dw - 1) / 0x7fff,
@@ -380,6 +369,7 @@
     if (in->qmouse) {
         qemu_remove_mouse_event_handler(in->qmouse);
     }
+    trace_xenfb_input_connected(xendev, in->abs_pointer_wanted);
     in->qmouse = qemu_add_mouse_event_handler(xenfb_mouse_event, in,
 					      in->abs_pointer_wanted,
 					      "Xen PVFB Mouse");
@@ -481,23 +471,23 @@
         xenfb->pixels = NULL;
     }
 
-    xenfb->fbpages = (xenfb->fb_len + (XC_PAGE_SIZE - 1)) / XC_PAGE_SIZE;
+    xenfb->fbpages = DIV_ROUND_UP(xenfb->fb_len, XC_PAGE_SIZE);
     n_fbdirs = xenfb->fbpages * mode / 8;
-    n_fbdirs = (n_fbdirs + (XC_PAGE_SIZE - 1)) / XC_PAGE_SIZE;
+    n_fbdirs = DIV_ROUND_UP(n_fbdirs, XC_PAGE_SIZE);
 
     pgmfns = g_malloc0(sizeof(xen_pfn_t) * n_fbdirs);
     fbmfns = g_malloc0(sizeof(xen_pfn_t) * xenfb->fbpages);
 
     xenfb_copy_mfns(mode, n_fbdirs, pgmfns, pd);
-    map = xc_map_foreign_pages(xen_xc, xenfb->c.xendev.dom,
-			       PROT_READ, pgmfns, n_fbdirs);
+    map = xenforeignmemory_map(xen_fmem, xenfb->c.xendev.dom,
+                               PROT_READ, n_fbdirs, pgmfns, NULL);
     if (map == NULL)
 	goto out;
     xenfb_copy_mfns(mode, xenfb->fbpages, fbmfns, map);
-    munmap(map, n_fbdirs * XC_PAGE_SIZE);
+    xenforeignmemory_unmap(xen_fmem, map, n_fbdirs);
 
-    xenfb->pixels = xc_map_foreign_pages(xen_xc, xenfb->c.xendev.dom,
-            PROT_READ, fbmfns, xenfb->fbpages);
+    xenfb->pixels = xenforeignmemory_map(xen_fmem, xenfb->c.xendev.dom,
+            PROT_READ, xenfb->fbpages, fbmfns, NULL);
     if (xenfb->pixels == NULL)
 	goto out;
 
@@ -779,18 +769,21 @@
 
 static void xenfb_handle_events(struct XenFB *xenfb)
 {
-    uint32_t prod, cons;
+    uint32_t prod, cons, out_cons;
     struct xenfb_page *page = xenfb->c.page;
 
     prod = page->out_prod;
-    if (prod == page->out_cons)
-	return;
+    out_cons = page->out_cons;
+    if (prod - out_cons > XENFB_OUT_RING_LEN) {
+        return;
+    }
     xen_rmb();		/* ensure we see ring contents up to prod */
-    for (cons = page->out_cons; cons != prod; cons++) {
+    for (cons = out_cons; cons != prod; cons++) {
 	union xenfb_out_event *event = &XENFB_OUT_RING_REF(page, cons);
+        uint8_t type = event->type;
 	int x, y, w, h;
 
-	switch (event->type) {
+	switch (type) {
 	case XENFB_TYPE_UPDATE:
 	    if (xenfb->up_count == UP_QUEUE)
 		xenfb->up_fullscreen = 1;
@@ -904,6 +897,7 @@
      *   Replacing the framebuffer with anonymous shared memory
      *   instead.  This releases the guest pages and keeps qemu happy.
      */
+    xenforeignmemory_unmap(xen_fmem, fb->pixels, fb->fbpages);
     fb->pixels = mmap(fb->pixels, fb->fbpages * XC_PAGE_SIZE,
                       PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANON,
                       -1, 0);
diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c
new file mode 100644
index 0000000..f43eb09
--- /dev/null
+++ b/hw/display/xlnx_dp.c
@@ -0,0 +1,1338 @@
+/*
+ * xlnx_dp.c
+ *
+ *  Copyright (C) 2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option)any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "hw/display/xlnx_dp.h"
+
+#ifndef DEBUG_DP
+#define DEBUG_DP 0
+#endif
+
+#define DPRINTF(fmt, ...) do {                                                 \
+    if (DEBUG_DP) {                                                            \
+        qemu_log("xlnx_dp: " fmt , ## __VA_ARGS__);                            \
+    }                                                                          \
+} while (0);
+
+/*
+ * Register offset for DP.
+ */
+#define DP_LINK_BW_SET                      (0x0000 >> 2)
+#define DP_LANE_COUNT_SET                   (0x0004 >> 2)
+#define DP_ENHANCED_FRAME_EN                (0x0008 >> 2)
+#define DP_TRAINING_PATTERN_SET             (0x000C >> 2)
+#define DP_LINK_QUAL_PATTERN_SET            (0x0010 >> 2)
+#define DP_SCRAMBLING_DISABLE               (0x0014 >> 2)
+#define DP_DOWNSPREAD_CTRL                  (0x0018 >> 2)
+#define DP_SOFTWARE_RESET                   (0x001C >> 2)
+#define DP_TRANSMITTER_ENABLE               (0x0080 >> 2)
+#define DP_MAIN_STREAM_ENABLE               (0x0084 >> 2)
+#define DP_FORCE_SCRAMBLER_RESET            (0x00C0 >> 2)
+#define DP_VERSION_REGISTER                 (0x00F8 >> 2)
+#define DP_CORE_ID                          (0x00FC >> 2)
+
+#define DP_AUX_COMMAND_REGISTER             (0x0100 >> 2)
+#define AUX_ADDR_ONLY_MASK                  (0x1000)
+#define AUX_COMMAND_MASK                    (0x0F00)
+#define AUX_COMMAND_SHIFT                   (8)
+#define AUX_COMMAND_NBYTES                  (0x000F)
+
+#define DP_AUX_WRITE_FIFO                   (0x0104 >> 2)
+#define DP_AUX_ADDRESS                      (0x0108 >> 2)
+#define DP_AUX_CLOCK_DIVIDER                (0x010C >> 2)
+#define DP_TX_USER_FIFO_OVERFLOW            (0x0110 >> 2)
+#define DP_INTERRUPT_SIGNAL_STATE           (0x0130 >> 2)
+#define DP_AUX_REPLY_DATA                   (0x0134 >> 2)
+#define DP_AUX_REPLY_CODE                   (0x0138 >> 2)
+#define DP_AUX_REPLY_COUNT                  (0x013C >> 2)
+#define DP_REPLY_DATA_COUNT                 (0x0148 >> 2)
+#define DP_REPLY_STATUS                     (0x014C >> 2)
+#define DP_HPD_DURATION                     (0x0150 >> 2)
+#define DP_MAIN_STREAM_HTOTAL               (0x0180 >> 2)
+#define DP_MAIN_STREAM_VTOTAL               (0x0184 >> 2)
+#define DP_MAIN_STREAM_POLARITY             (0x0188 >> 2)
+#define DP_MAIN_STREAM_HSWIDTH              (0x018C >> 2)
+#define DP_MAIN_STREAM_VSWIDTH              (0x0190 >> 2)
+#define DP_MAIN_STREAM_HRES                 (0x0194 >> 2)
+#define DP_MAIN_STREAM_VRES                 (0x0198 >> 2)
+#define DP_MAIN_STREAM_HSTART               (0x019C >> 2)
+#define DP_MAIN_STREAM_VSTART               (0x01A0 >> 2)
+#define DP_MAIN_STREAM_MISC0                (0x01A4 >> 2)
+#define DP_MAIN_STREAM_MISC1                (0x01A8 >> 2)
+#define DP_MAIN_STREAM_M_VID                (0x01AC >> 2)
+#define DP_MSA_TRANSFER_UNIT_SIZE           (0x01B0 >> 2)
+#define DP_MAIN_STREAM_N_VID                (0x01B4 >> 2)
+#define DP_USER_DATA_COUNT_PER_LANE         (0x01BC >> 2)
+#define DP_MIN_BYTES_PER_TU                 (0x01C4 >> 2)
+#define DP_FRAC_BYTES_PER_TU                (0x01C8 >> 2)
+#define DP_INIT_WAIT                        (0x01CC >> 2)
+#define DP_PHY_RESET                        (0x0200 >> 2)
+#define DP_PHY_VOLTAGE_DIFF_LANE_0          (0x0220 >> 2)
+#define DP_PHY_VOLTAGE_DIFF_LANE_1          (0x0224 >> 2)
+#define DP_TRANSMIT_PRBS7                   (0x0230 >> 2)
+#define DP_PHY_CLOCK_SELECT                 (0x0234 >> 2)
+#define DP_TX_PHY_POWER_DOWN                (0x0238 >> 2)
+#define DP_PHY_PRECURSOR_LANE_0             (0x023C >> 2)
+#define DP_PHY_PRECURSOR_LANE_1             (0x0240 >> 2)
+#define DP_PHY_POSTCURSOR_LANE_0            (0x024C >> 2)
+#define DP_PHY_POSTCURSOR_LANE_1            (0x0250 >> 2)
+#define DP_PHY_STATUS                       (0x0280 >> 2)
+
+#define DP_TX_AUDIO_CONTROL                 (0x0300 >> 2)
+#define DP_TX_AUD_CTRL                      (1)
+
+#define DP_TX_AUDIO_CHANNELS                (0x0304 >> 2)
+#define DP_TX_AUDIO_INFO_DATA(n)            ((0x0308 + 4 * n) >> 2)
+#define DP_TX_M_AUD                         (0x0328 >> 2)
+#define DP_TX_N_AUD                         (0x032C >> 2)
+#define DP_TX_AUDIO_EXT_DATA(n)             ((0x0330 + 4 * n) >> 2)
+#define DP_INT_STATUS                       (0x03A0 >> 2)
+#define DP_INT_MASK                         (0x03A4 >> 2)
+#define DP_INT_EN                           (0x03A8 >> 2)
+#define DP_INT_DS                           (0x03AC >> 2)
+
+/*
+ * Registers offset for Audio Video Buffer configuration.
+ */
+#define V_BLEND_OFFSET                      (0xA000)
+#define V_BLEND_BG_CLR_0                    (0x0000 >> 2)
+#define V_BLEND_BG_CLR_1                    (0x0004 >> 2)
+#define V_BLEND_BG_CLR_2                    (0x0008 >> 2)
+#define V_BLEND_SET_GLOBAL_ALPHA_REG        (0x000C >> 2)
+#define V_BLEND_OUTPUT_VID_FORMAT           (0x0014 >> 2)
+#define V_BLEND_LAYER0_CONTROL              (0x0018 >> 2)
+#define V_BLEND_LAYER1_CONTROL              (0x001C >> 2)
+
+#define V_BLEND_RGB2YCBCR_COEFF(n)          ((0x0020 + 4 * n) >> 2)
+#define V_BLEND_IN1CSC_COEFF(n)             ((0x0044 + 4 * n) >> 2)
+
+#define V_BLEND_LUMA_IN1CSC_OFFSET          (0x0068 >> 2)
+#define V_BLEND_CR_IN1CSC_OFFSET            (0x006C >> 2)
+#define V_BLEND_CB_IN1CSC_OFFSET            (0x0070 >> 2)
+#define V_BLEND_LUMA_OUTCSC_OFFSET          (0x0074 >> 2)
+#define V_BLEND_CR_OUTCSC_OFFSET            (0x0078 >> 2)
+#define V_BLEND_CB_OUTCSC_OFFSET            (0x007C >> 2)
+
+#define V_BLEND_IN2CSC_COEFF(n)             ((0x0080 + 4 * n) >> 2)
+
+#define V_BLEND_LUMA_IN2CSC_OFFSET          (0x00A4 >> 2)
+#define V_BLEND_CR_IN2CSC_OFFSET            (0x00A8 >> 2)
+#define V_BLEND_CB_IN2CSC_OFFSET            (0x00AC >> 2)
+#define V_BLEND_CHROMA_KEY_ENABLE           (0x01D0 >> 2)
+#define V_BLEND_CHROMA_KEY_COMP1            (0x01D4 >> 2)
+#define V_BLEND_CHROMA_KEY_COMP2            (0x01D8 >> 2)
+#define V_BLEND_CHROMA_KEY_COMP3            (0x01DC >> 2)
+
+/*
+ * Registers offset for Audio Video Buffer configuration.
+ */
+#define AV_BUF_MANAGER_OFFSET               (0xB000)
+#define AV_BUF_FORMAT                       (0x0000 >> 2)
+#define AV_BUF_NON_LIVE_LATENCY             (0x0008 >> 2)
+#define AV_CHBUF0                           (0x0010 >> 2)
+#define AV_CHBUF1                           (0x0014 >> 2)
+#define AV_CHBUF2                           (0x0018 >> 2)
+#define AV_CHBUF3                           (0x001C >> 2)
+#define AV_CHBUF4                           (0x0020 >> 2)
+#define AV_CHBUF5                           (0x0024 >> 2)
+#define AV_BUF_STC_CONTROL                  (0x002C >> 2)
+#define AV_BUF_STC_INIT_VALUE0              (0x0030 >> 2)
+#define AV_BUF_STC_INIT_VALUE1              (0x0034 >> 2)
+#define AV_BUF_STC_ADJ                      (0x0038 >> 2)
+#define AV_BUF_STC_VIDEO_VSYNC_TS_REG0      (0x003C >> 2)
+#define AV_BUF_STC_VIDEO_VSYNC_TS_REG1      (0x0040 >> 2)
+#define AV_BUF_STC_EXT_VSYNC_TS_REG0        (0x0044 >> 2)
+#define AV_BUF_STC_EXT_VSYNC_TS_REG1        (0x0048 >> 2)
+#define AV_BUF_STC_CUSTOM_EVENT_TS_REG0     (0x004C >> 2)
+#define AV_BUF_STC_CUSTOM_EVENT_TS_REG1     (0x0050 >> 2)
+#define AV_BUF_STC_CUSTOM_EVENT2_TS_REG0    (0x0054 >> 2)
+#define AV_BUF_STC_CUSTOM_EVENT2_TS_REG1    (0x0058 >> 2)
+#define AV_BUF_STC_SNAPSHOT0                (0x0060 >> 2)
+#define AV_BUF_STC_SNAPSHOT1                (0x0064 >> 2)
+#define AV_BUF_OUTPUT_AUDIO_VIDEO_SELECT    (0x0070 >> 2)
+#define AV_BUF_HCOUNT_VCOUNT_INT0           (0x0074 >> 2)
+#define AV_BUF_HCOUNT_VCOUNT_INT1           (0x0078 >> 2)
+#define AV_BUF_DITHER_CONFIG                (0x007C >> 2)
+#define AV_BUF_DITHER_CONFIG_MAX            (0x008C >> 2)
+#define AV_BUF_DITHER_CONFIG_MIN            (0x0090 >> 2)
+#define AV_BUF_PATTERN_GEN_SELECT           (0x0100 >> 2)
+#define AV_BUF_AUD_VID_CLK_SOURCE           (0x0120 >> 2)
+#define AV_BUF_SRST_REG                     (0x0124 >> 2)
+#define AV_BUF_AUDIO_RDY_INTERVAL           (0x0128 >> 2)
+#define AV_BUF_AUDIO_CH_CONFIG              (0x012C >> 2)
+
+#define AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(n)((0x0200 + 4 * n) >> 2)
+
+#define AV_BUF_VIDEO_COMP_SCALE_FACTOR(n)   ((0x020C + 4 * n) >> 2)
+
+#define AV_BUF_LIVE_VIDEO_COMP_SF(n)        ((0x0218 + 4 * n) >> 2)
+
+#define AV_BUF_LIVE_VID_CONFIG              (0x0224 >> 2)
+
+#define AV_BUF_LIVE_GFX_COMP_SF(n)          ((0x0228 + 4 * n) >> 2)
+
+#define AV_BUF_LIVE_GFX_CONFIG              (0x0234 >> 2)
+
+#define AUDIO_MIXER_REGISTER_OFFSET         (0xC000)
+#define AUDIO_MIXER_VOLUME_CONTROL          (0x0000 >> 2)
+#define AUDIO_MIXER_META_DATA               (0x0004 >> 2)
+#define AUD_CH_STATUS_REG(n)                ((0x0008 + 4 * n) >> 2)
+#define AUD_CH_A_DATA_REG(n)                ((0x0020 + 4 * n) >> 2)
+#define AUD_CH_B_DATA_REG(n)                ((0x0038 + 4 * n) >> 2)
+
+#define DP_AUDIO_DMA_CHANNEL(n)             (4 + n)
+#define DP_GRAPHIC_DMA_CHANNEL              (3)
+#define DP_VIDEO_DMA_CHANNEL                (0)
+
+enum DPGraphicFmt {
+    DP_GRAPHIC_RGBA8888 = 0 << 8,
+    DP_GRAPHIC_ABGR8888 = 1 << 8,
+    DP_GRAPHIC_RGB888 = 2 << 8,
+    DP_GRAPHIC_BGR888 = 3 << 8,
+    DP_GRAPHIC_RGBA5551 = 4 << 8,
+    DP_GRAPHIC_RGBA4444 = 5 << 8,
+    DP_GRAPHIC_RGB565 = 6 << 8,
+    DP_GRAPHIC_8BPP = 7 << 8,
+    DP_GRAPHIC_4BPP = 8 << 8,
+    DP_GRAPHIC_2BPP = 9 << 8,
+    DP_GRAPHIC_1BPP = 10 << 8,
+    DP_GRAPHIC_MASK = 0xF << 8
+};
+
+enum DPVideoFmt {
+    DP_NL_VID_CB_Y0_CR_Y1 = 0,
+    DP_NL_VID_CR_Y0_CB_Y1 = 1,
+    DP_NL_VID_Y0_CR_Y1_CB = 2,
+    DP_NL_VID_Y0_CB_Y1_CR = 3,
+    DP_NL_VID_YV16 = 4,
+    DP_NL_VID_YV24 = 5,
+    DP_NL_VID_YV16CL = 6,
+    DP_NL_VID_MONO = 7,
+    DP_NL_VID_YV16CL2 = 8,
+    DP_NL_VID_YUV444 = 9,
+    DP_NL_VID_RGB888 = 10,
+    DP_NL_VID_RGBA8880 = 11,
+    DP_NL_VID_RGB888_10BPC = 12,
+    DP_NL_VID_YUV444_10BPC = 13,
+    DP_NL_VID_YV16CL2_10BPC = 14,
+    DP_NL_VID_YV16CL_10BPC = 15,
+    DP_NL_VID_YV16_10BPC = 16,
+    DP_NL_VID_YV24_10BPC = 17,
+    DP_NL_VID_Y_ONLY_10BPC = 18,
+    DP_NL_VID_YV16_420 = 19,
+    DP_NL_VID_YV16CL_420 = 20,
+    DP_NL_VID_YV16CL2_420 = 21,
+    DP_NL_VID_YV16_420_10BPC = 22,
+    DP_NL_VID_YV16CL_420_10BPC = 23,
+    DP_NL_VID_YV16CL2_420_10BPC = 24,
+    DP_NL_VID_FMT_MASK = 0x1F
+};
+
+typedef enum DPGraphicFmt DPGraphicFmt;
+typedef enum DPVideoFmt DPVideoFmt;
+
+static const VMStateDescription vmstate_dp = {
+    .name = TYPE_XLNX_DP,
+    .version_id = 1,
+    .fields = (VMStateField[]){
+        VMSTATE_UINT32_ARRAY(core_registers, XlnxDPState,
+                             DP_CORE_REG_ARRAY_SIZE),
+        VMSTATE_UINT32_ARRAY(avbufm_registers, XlnxDPState,
+                             DP_AVBUF_REG_ARRAY_SIZE),
+        VMSTATE_UINT32_ARRAY(vblend_registers, XlnxDPState,
+                             DP_VBLEND_REG_ARRAY_SIZE),
+        VMSTATE_UINT32_ARRAY(audio_registers, XlnxDPState,
+                             DP_AUDIO_REG_ARRAY_SIZE),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void xlnx_dp_update_irq(XlnxDPState *s);
+
+static uint64_t xlnx_dp_audio_read(void *opaque, hwaddr offset, unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    offset = offset >> 2;
+    return s->audio_registers[offset];
+}
+
+static void xlnx_dp_audio_write(void *opaque, hwaddr offset, uint64_t value,
+                                unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    offset = offset >> 2;
+
+    switch (offset) {
+    case AUDIO_MIXER_META_DATA:
+        s->audio_registers[offset] = value & 0x00000001;
+        break;
+    default:
+        s->audio_registers[offset] = value;
+        break;
+    }
+}
+
+static const MemoryRegionOps audio_ops = {
+    .read = xlnx_dp_audio_read,
+    .write = xlnx_dp_audio_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static inline uint32_t xlnx_dp_audio_get_volume(XlnxDPState *s,
+                                                uint8_t channel)
+{
+    switch (channel) {
+    case 0:
+        return extract32(s->audio_registers[AUDIO_MIXER_VOLUME_CONTROL], 0, 16);
+    case 1:
+        return extract32(s->audio_registers[AUDIO_MIXER_VOLUME_CONTROL], 16,
+                                                                         16);
+    default:
+        return 0;
+    }
+}
+
+static inline void xlnx_dp_audio_activate(XlnxDPState *s)
+{
+    bool activated = ((s->core_registers[DP_TX_AUDIO_CONTROL]
+                   & DP_TX_AUD_CTRL) != 0);
+    AUD_set_active_out(s->amixer_output_stream, activated);
+    xlnx_dpdma_set_host_data_location(s->dpdma, DP_AUDIO_DMA_CHANNEL(0),
+                                      &s->audio_buffer_0);
+    xlnx_dpdma_set_host_data_location(s->dpdma, DP_AUDIO_DMA_CHANNEL(1),
+                                      &s->audio_buffer_1);
+}
+
+static inline void xlnx_dp_audio_mix_buffer(XlnxDPState *s)
+{
+    /*
+     * Audio packets are signed and have this shape:
+     * | 16 | 16 | 16 | 16 | 16 | 16 | 16 | 16 |
+     * | R3 | L3 | R2 | L2 | R1 | L1 | R0 | L0 |
+     *
+     * Output audio is 16bits saturated.
+     */
+    int i;
+
+    if ((s->audio_data_available[0]) && (xlnx_dp_audio_get_volume(s, 0))) {
+        for (i = 0; i < s->audio_data_available[0] / 2; i++) {
+            s->temp_buffer[i] = (int64_t)(s->audio_buffer_0[i])
+                              * xlnx_dp_audio_get_volume(s, 0) / 8192;
+        }
+        s->byte_left = s->audio_data_available[0];
+    } else {
+        memset(s->temp_buffer, 0, s->audio_data_available[1] / 2);
+    }
+
+    if ((s->audio_data_available[1]) && (xlnx_dp_audio_get_volume(s, 1))) {
+        if ((s->audio_data_available[0] == 0)
+        || (s->audio_data_available[1] == s->audio_data_available[0])) {
+            for (i = 0; i < s->audio_data_available[1] / 2; i++) {
+                s->temp_buffer[i] += (int64_t)(s->audio_buffer_1[i])
+                                   * xlnx_dp_audio_get_volume(s, 1) / 8192;
+            }
+            s->byte_left = s->audio_data_available[1];
+        }
+    }
+
+    for (i = 0; i < s->byte_left / 2; i++) {
+        s->out_buffer[i] = MAX(-32767, MIN(s->temp_buffer[i], 32767));
+    }
+
+    s->data_ptr = 0;
+}
+
+static void xlnx_dp_audio_callback(void *opaque, int avail)
+{
+    /*
+     * Get some data from the DPDMA and compute these datas.
+     * Then wait for QEMU's audio subsystem to call this callback.
+     */
+    XlnxDPState *s = XLNX_DP(opaque);
+    size_t written = 0;
+
+    /* If there are already some data don't get more data. */
+    if (s->byte_left == 0) {
+        s->audio_data_available[0] = xlnx_dpdma_start_operation(s->dpdma, 4,
+                                                                  true);
+        s->audio_data_available[1] = xlnx_dpdma_start_operation(s->dpdma, 5,
+                                                                  true);
+        xlnx_dp_audio_mix_buffer(s);
+    }
+
+    /* Send the buffer through the audio. */
+    if (s->byte_left <= MAX_QEMU_BUFFER_SIZE) {
+        if (s->byte_left != 0) {
+            written = AUD_write(s->amixer_output_stream,
+                                &s->out_buffer[s->data_ptr], s->byte_left);
+        } else {
+            /*
+             * There is nothing to play.. We don't have any data! Fill the
+             * buffer with zero's and send it.
+             */
+            written = 0;
+            memset(s->out_buffer, 0, 1024);
+            AUD_write(s->amixer_output_stream, s->out_buffer, 1024);
+        }
+    } else {
+        written = AUD_write(s->amixer_output_stream,
+                            &s->out_buffer[s->data_ptr], MAX_QEMU_BUFFER_SIZE);
+    }
+    s->byte_left -= written;
+    s->data_ptr += written;
+}
+
+/*
+ * AUX channel related function.
+ */
+static void xlnx_dp_aux_clear_rx_fifo(XlnxDPState *s)
+{
+    fifo8_reset(&s->rx_fifo);
+}
+
+static void xlnx_dp_aux_push_rx_fifo(XlnxDPState *s, uint8_t *buf, size_t len)
+{
+    DPRINTF("Push %u data in rx_fifo\n", (unsigned)len);
+    fifo8_push_all(&s->rx_fifo, buf, len);
+}
+
+static uint8_t xlnx_dp_aux_pop_rx_fifo(XlnxDPState *s)
+{
+    uint8_t ret;
+
+    if (fifo8_is_empty(&s->rx_fifo)) {
+        DPRINTF("rx_fifo underflow..\n");
+        abort();
+    }
+    ret = fifo8_pop(&s->rx_fifo);
+    DPRINTF("pop 0x%" PRIX8 " from rx_fifo.\n", ret);
+    return ret;
+}
+
+static void xlnx_dp_aux_clear_tx_fifo(XlnxDPState *s)
+{
+    fifo8_reset(&s->tx_fifo);
+}
+
+static void xlnx_dp_aux_push_tx_fifo(XlnxDPState *s, uint8_t *buf, size_t len)
+{
+    DPRINTF("Push %u data in tx_fifo\n", (unsigned)len);
+    fifo8_push_all(&s->tx_fifo, buf, len);
+}
+
+static uint8_t xlnx_dp_aux_pop_tx_fifo(XlnxDPState *s)
+{
+    uint8_t ret;
+
+    if (fifo8_is_empty(&s->tx_fifo)) {
+        DPRINTF("tx_fifo underflow..\n");
+        abort();
+    }
+    ret = fifo8_pop(&s->tx_fifo);
+    DPRINTF("pop 0x%2.2X from tx_fifo.\n", ret);
+    return ret;
+}
+
+static uint32_t xlnx_dp_aux_get_address(XlnxDPState *s)
+{
+    return s->core_registers[DP_AUX_ADDRESS];
+}
+
+/*
+ * Get command from the register.
+ */
+static void xlnx_dp_aux_set_command(XlnxDPState *s, uint32_t value)
+{
+    bool address_only = (value & AUX_ADDR_ONLY_MASK) != 0;
+    AUXCommand cmd = (value & AUX_COMMAND_MASK) >> AUX_COMMAND_SHIFT;
+    uint8_t nbytes = (value & AUX_COMMAND_NBYTES) + 1;
+    uint8_t buf[16];
+    int i;
+
+    /*
+     * When an address_only command is executed nothing happen to the fifo, so
+     * just make nbytes = 0.
+     */
+    if (address_only) {
+        nbytes = 0;
+    }
+
+    switch (cmd) {
+    case READ_AUX:
+    case READ_I2C:
+    case READ_I2C_MOT:
+        s->core_registers[DP_AUX_REPLY_CODE] = aux_request(s->aux_bus, cmd,
+                                               xlnx_dp_aux_get_address(s),
+                                               nbytes, buf);
+        s->core_registers[DP_REPLY_DATA_COUNT] = nbytes;
+
+        if (s->core_registers[DP_AUX_REPLY_CODE] == AUX_I2C_ACK) {
+            xlnx_dp_aux_push_rx_fifo(s, buf, nbytes);
+        }
+        break;
+    case WRITE_AUX:
+    case WRITE_I2C:
+    case WRITE_I2C_MOT:
+        for (i = 0; i < nbytes; i++) {
+            buf[i] = xlnx_dp_aux_pop_tx_fifo(s);
+        }
+        s->core_registers[DP_AUX_REPLY_CODE] = aux_request(s->aux_bus, cmd,
+                                               xlnx_dp_aux_get_address(s),
+                                               nbytes, buf);
+        xlnx_dp_aux_clear_tx_fifo(s);
+        break;
+    case WRITE_I2C_STATUS:
+        qemu_log_mask(LOG_UNIMP, "xlnx_dp: Write i2c status not implemented\n");
+        break;
+    default:
+        abort();
+    }
+
+    s->core_registers[DP_INTERRUPT_SIGNAL_STATE] |= 0x04;
+}
+
+static void xlnx_dp_set_dpdma(Object *obj, const char *name, Object *val,
+                              Error **errp)
+{
+    XlnxDPState *s = XLNX_DP(obj);
+    if (s->console) {
+        DisplaySurface *surface = qemu_console_surface(s->console);
+        XlnxDPDMAState *dma = XLNX_DPDMA(val);
+        xlnx_dpdma_set_host_data_location(dma, DP_GRAPHIC_DMA_CHANNEL,
+                                          surface_data(surface));
+    }
+}
+
+static inline uint8_t xlnx_dp_global_alpha_value(XlnxDPState *s)
+{
+    return (s->vblend_registers[V_BLEND_SET_GLOBAL_ALPHA_REG] & 0x1FE) >> 1;
+}
+
+static inline bool xlnx_dp_global_alpha_enabled(XlnxDPState *s)
+{
+    /*
+     * If the alpha is totally opaque (255) we consider the alpha is disabled to
+     * reduce CPU consumption.
+     */
+    return ((xlnx_dp_global_alpha_value(s) != 0xFF) &&
+           ((s->vblend_registers[V_BLEND_SET_GLOBAL_ALPHA_REG] & 0x01) != 0));
+}
+
+static void xlnx_dp_recreate_surface(XlnxDPState *s)
+{
+    /*
+     * Two possibilities, if blending is enabled the console displays
+     * bout_plane, if not g_plane is displayed.
+     */
+    uint16_t width = s->core_registers[DP_MAIN_STREAM_HRES];
+    uint16_t height = s->core_registers[DP_MAIN_STREAM_VRES];
+    DisplaySurface *current_console_surface = qemu_console_surface(s->console);
+
+    if ((width != 0) && (height != 0)) {
+        /*
+         * As dpy_gfx_replace_surface calls qemu_free_displaysurface on the
+         * surface we need to be carefull and don't free the surface associated
+         * to the console or double free will happen.
+         */
+        if (s->bout_plane.surface != current_console_surface) {
+            qemu_free_displaysurface(s->bout_plane.surface);
+        }
+        if (s->v_plane.surface != current_console_surface) {
+            qemu_free_displaysurface(s->v_plane.surface);
+        }
+        if (s->g_plane.surface != current_console_surface) {
+            qemu_free_displaysurface(s->g_plane.surface);
+        }
+
+        s->g_plane.surface
+                = qemu_create_displaysurface_from(width, height,
+                                                  s->g_plane.format, 0, NULL);
+        s->v_plane.surface
+                = qemu_create_displaysurface_from(width, height,
+                                                  s->v_plane.format, 0, NULL);
+        if (xlnx_dp_global_alpha_enabled(s)) {
+            s->bout_plane.surface =
+                            qemu_create_displaysurface_from(width,
+                                                            height,
+                                                            s->g_plane.format,
+                                                            0, NULL);
+            dpy_gfx_replace_surface(s->console, s->bout_plane.surface);
+        } else {
+            s->bout_plane.surface = NULL;
+            dpy_gfx_replace_surface(s->console, s->g_plane.surface);
+        }
+
+        xlnx_dpdma_set_host_data_location(s->dpdma, DP_GRAPHIC_DMA_CHANNEL,
+                                            surface_data(s->g_plane.surface));
+        xlnx_dpdma_set_host_data_location(s->dpdma, DP_VIDEO_DMA_CHANNEL,
+                                            surface_data(s->v_plane.surface));
+    }
+}
+
+/*
+ * Change the graphic format of the surface.
+ */
+static void xlnx_dp_change_graphic_fmt(XlnxDPState *s)
+{
+    switch (s->avbufm_registers[AV_BUF_FORMAT] & DP_GRAPHIC_MASK) {
+    case DP_GRAPHIC_RGBA8888:
+        s->g_plane.format = PIXMAN_r8g8b8a8;
+        break;
+    case DP_GRAPHIC_ABGR8888:
+        s->g_plane.format = PIXMAN_a8b8g8r8;
+        break;
+    case DP_GRAPHIC_RGB565:
+        s->g_plane.format = PIXMAN_r5g6b5;
+        break;
+    case DP_GRAPHIC_RGB888:
+        s->g_plane.format = PIXMAN_r8g8b8;
+        break;
+    case DP_GRAPHIC_BGR888:
+        s->g_plane.format = PIXMAN_b8g8r8;
+        break;
+    default:
+        DPRINTF("error: unsupported graphic format %u.\n",
+                s->avbufm_registers[AV_BUF_FORMAT] & DP_GRAPHIC_MASK);
+        abort();
+    }
+
+    switch (s->avbufm_registers[AV_BUF_FORMAT] & DP_NL_VID_FMT_MASK) {
+    case 0:
+        s->v_plane.format = PIXMAN_x8b8g8r8;
+        break;
+    case DP_NL_VID_RGBA8880:
+        s->v_plane.format = PIXMAN_x8b8g8r8;
+        break;
+    default:
+        DPRINTF("error: unsupported video format %u.\n",
+                s->avbufm_registers[AV_BUF_FORMAT] & DP_NL_VID_FMT_MASK);
+        abort();
+    }
+
+    xlnx_dp_recreate_surface(s);
+}
+
+static void xlnx_dp_update_irq(XlnxDPState *s)
+{
+    uint32_t flags;
+
+    flags = s->core_registers[DP_INT_STATUS] & ~s->core_registers[DP_INT_MASK];
+    DPRINTF("update IRQ value = %" PRIx32 "\n", flags);
+    qemu_set_irq(s->irq, flags != 0);
+}
+
+static uint64_t xlnx_dp_read(void *opaque, hwaddr offset, unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+    uint64_t ret = 0;
+
+    offset = offset >> 2;
+
+    switch (offset) {
+    case DP_TX_USER_FIFO_OVERFLOW:
+        /* This register is cleared after a read */
+        ret = s->core_registers[DP_TX_USER_FIFO_OVERFLOW];
+        s->core_registers[DP_TX_USER_FIFO_OVERFLOW] = 0;
+        break;
+    case DP_AUX_REPLY_DATA:
+        ret = xlnx_dp_aux_pop_rx_fifo(s);
+        break;
+    case DP_INTERRUPT_SIGNAL_STATE:
+        /*
+         * XXX: Not sure it is the right thing to do actually.
+         * The register is not written by the device driver so it's stuck
+         * to 0x04.
+         */
+        ret = s->core_registers[DP_INTERRUPT_SIGNAL_STATE];
+        s->core_registers[DP_INTERRUPT_SIGNAL_STATE] &= ~0x04;
+        break;
+    case DP_AUX_WRITE_FIFO:
+    case DP_TX_AUDIO_INFO_DATA(0):
+    case DP_TX_AUDIO_INFO_DATA(1):
+    case DP_TX_AUDIO_INFO_DATA(2):
+    case DP_TX_AUDIO_INFO_DATA(3):
+    case DP_TX_AUDIO_INFO_DATA(4):
+    case DP_TX_AUDIO_INFO_DATA(5):
+    case DP_TX_AUDIO_INFO_DATA(6):
+    case DP_TX_AUDIO_INFO_DATA(7):
+    case DP_TX_AUDIO_EXT_DATA(0):
+    case DP_TX_AUDIO_EXT_DATA(1):
+    case DP_TX_AUDIO_EXT_DATA(2):
+    case DP_TX_AUDIO_EXT_DATA(3):
+    case DP_TX_AUDIO_EXT_DATA(4):
+    case DP_TX_AUDIO_EXT_DATA(5):
+    case DP_TX_AUDIO_EXT_DATA(6):
+    case DP_TX_AUDIO_EXT_DATA(7):
+    case DP_TX_AUDIO_EXT_DATA(8):
+        /* write only registers */
+        ret = 0;
+        break;
+    default:
+        assert(offset <= (0x3AC >> 2));
+        ret = s->core_registers[offset];
+        break;
+    }
+
+    DPRINTF("core read @%" PRIx64 " = 0x%8.8" PRIX64 "\n", offset << 2, ret);
+    return ret;
+}
+
+static void xlnx_dp_write(void *opaque, hwaddr offset, uint64_t value,
+                          unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    DPRINTF("core write @%" PRIx64 " = 0x%8.8" PRIX64 "\n", offset, value);
+
+    offset = offset >> 2;
+
+    switch (offset) {
+    /*
+     * Only special write case are handled.
+     */
+    case DP_LINK_BW_SET:
+        s->core_registers[offset] = value & 0x000000FF;
+        break;
+    case DP_LANE_COUNT_SET:
+    case DP_MAIN_STREAM_MISC0:
+        s->core_registers[offset] = value & 0x0000000F;
+        break;
+    case DP_TRAINING_PATTERN_SET:
+    case DP_LINK_QUAL_PATTERN_SET:
+    case DP_MAIN_STREAM_POLARITY:
+    case DP_PHY_VOLTAGE_DIFF_LANE_0:
+    case DP_PHY_VOLTAGE_DIFF_LANE_1:
+        s->core_registers[offset] = value & 0x00000003;
+        break;
+    case DP_ENHANCED_FRAME_EN:
+    case DP_SCRAMBLING_DISABLE:
+    case DP_DOWNSPREAD_CTRL:
+    case DP_MAIN_STREAM_ENABLE:
+    case DP_TRANSMIT_PRBS7:
+        s->core_registers[offset] = value & 0x00000001;
+        break;
+    case DP_PHY_CLOCK_SELECT:
+        s->core_registers[offset] = value & 0x00000007;
+        break;
+    case DP_SOFTWARE_RESET:
+        /*
+         * No need to update this bit as it's read '0'.
+         */
+        /*
+         * TODO: reset IP.
+         */
+        break;
+    case DP_TRANSMITTER_ENABLE:
+        s->core_registers[offset] = value & 0x01;
+        break;
+    case DP_FORCE_SCRAMBLER_RESET:
+        /*
+         * No need to update this bit as it's read '0'.
+         */
+        /*
+         * TODO: force a scrambler reset??
+         */
+        break;
+    case DP_AUX_COMMAND_REGISTER:
+        s->core_registers[offset] = value & 0x00001F0F;
+        xlnx_dp_aux_set_command(s, s->core_registers[offset]);
+        break;
+    case DP_MAIN_STREAM_HTOTAL:
+    case DP_MAIN_STREAM_VTOTAL:
+    case DP_MAIN_STREAM_HSTART:
+    case DP_MAIN_STREAM_VSTART:
+        s->core_registers[offset] = value & 0x0000FFFF;
+        break;
+    case DP_MAIN_STREAM_HRES:
+    case DP_MAIN_STREAM_VRES:
+        s->core_registers[offset] = value & 0x0000FFFF;
+        xlnx_dp_recreate_surface(s);
+        break;
+    case DP_MAIN_STREAM_HSWIDTH:
+    case DP_MAIN_STREAM_VSWIDTH:
+        s->core_registers[offset] = value & 0x00007FFF;
+        break;
+    case DP_MAIN_STREAM_MISC1:
+        s->core_registers[offset] = value & 0x00000086;
+        break;
+    case DP_MAIN_STREAM_M_VID:
+    case DP_MAIN_STREAM_N_VID:
+        s->core_registers[offset] = value & 0x00FFFFFF;
+        break;
+    case DP_MSA_TRANSFER_UNIT_SIZE:
+    case DP_MIN_BYTES_PER_TU:
+    case DP_INIT_WAIT:
+        s->core_registers[offset] = value & 0x00000007;
+        break;
+    case DP_USER_DATA_COUNT_PER_LANE:
+        s->core_registers[offset] = value & 0x0003FFFF;
+        break;
+    case DP_FRAC_BYTES_PER_TU:
+        s->core_registers[offset] = value & 0x000003FF;
+        break;
+    case DP_PHY_RESET:
+        s->core_registers[offset] = value & 0x00010003;
+        /*
+         * TODO: Reset something?
+         */
+        break;
+    case DP_TX_PHY_POWER_DOWN:
+        s->core_registers[offset] = value & 0x0000000F;
+        /*
+         * TODO: Power down things?
+         */
+        break;
+    case DP_AUX_WRITE_FIFO: {
+        uint8_t c = value;
+        xlnx_dp_aux_push_tx_fifo(s, &c, 1);
+        break;
+    }
+    case DP_AUX_CLOCK_DIVIDER:
+        break;
+    case DP_AUX_REPLY_COUNT:
+        /*
+         * Writing to this register clear the counter.
+         */
+        s->core_registers[offset] = 0x00000000;
+        break;
+    case DP_AUX_ADDRESS:
+        s->core_registers[offset] = value & 0x000FFFFF;
+        break;
+    case DP_VERSION_REGISTER:
+    case DP_CORE_ID:
+    case DP_TX_USER_FIFO_OVERFLOW:
+    case DP_AUX_REPLY_DATA:
+    case DP_AUX_REPLY_CODE:
+    case DP_REPLY_DATA_COUNT:
+    case DP_REPLY_STATUS:
+    case DP_HPD_DURATION:
+        /*
+         * Write to read only location..
+         */
+        break;
+    case DP_TX_AUDIO_CONTROL:
+        s->core_registers[offset] = value & 0x00000001;
+        xlnx_dp_audio_activate(s);
+        break;
+    case DP_TX_AUDIO_CHANNELS:
+        s->core_registers[offset] = value & 0x00000007;
+        xlnx_dp_audio_activate(s);
+        break;
+    case DP_INT_STATUS:
+        s->core_registers[DP_INT_STATUS] &= ~value;
+        xlnx_dp_update_irq(s);
+        break;
+    case DP_INT_EN:
+        s->core_registers[DP_INT_MASK] &= ~value;
+        xlnx_dp_update_irq(s);
+        break;
+    case DP_INT_DS:
+        s->core_registers[DP_INT_MASK] |= ~value;
+        xlnx_dp_update_irq(s);
+        break;
+    default:
+        assert(offset <= (0x504C >> 2));
+        s->core_registers[offset] = value;
+        break;
+    }
+}
+
+static const MemoryRegionOps dp_ops = {
+    .read = xlnx_dp_read,
+    .write = xlnx_dp_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+/*
+ * This is to handle Read/Write to the Video Blender.
+ */
+static void xlnx_dp_vblend_write(void *opaque, hwaddr offset,
+                                 uint64_t value, unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+    bool alpha_was_enabled;
+
+    DPRINTF("vblend: write @0x%" HWADDR_PRIX " = 0x%" PRIX32 "\n", offset,
+                                                               (uint32_t)value);
+    offset = offset >> 2;
+
+    switch (offset) {
+    case V_BLEND_BG_CLR_0:
+    case V_BLEND_BG_CLR_1:
+    case V_BLEND_BG_CLR_2:
+        s->vblend_registers[offset] = value & 0x00000FFF;
+        break;
+    case V_BLEND_SET_GLOBAL_ALPHA_REG:
+        /*
+         * A write to this register can enable or disable blending. Thus we need
+         * to recreate the surfaces.
+         */
+        alpha_was_enabled = xlnx_dp_global_alpha_enabled(s);
+        s->vblend_registers[offset] = value & 0x000001FF;
+        if (xlnx_dp_global_alpha_enabled(s) != alpha_was_enabled) {
+            xlnx_dp_recreate_surface(s);
+        }
+        break;
+    case V_BLEND_OUTPUT_VID_FORMAT:
+        s->vblend_registers[offset] = value & 0x00000017;
+        break;
+    case V_BLEND_LAYER0_CONTROL:
+    case V_BLEND_LAYER1_CONTROL:
+        s->vblend_registers[offset] = value & 0x00000103;
+        break;
+    case V_BLEND_RGB2YCBCR_COEFF(0):
+    case V_BLEND_RGB2YCBCR_COEFF(1):
+    case V_BLEND_RGB2YCBCR_COEFF(2):
+    case V_BLEND_RGB2YCBCR_COEFF(3):
+    case V_BLEND_RGB2YCBCR_COEFF(4):
+    case V_BLEND_RGB2YCBCR_COEFF(5):
+    case V_BLEND_RGB2YCBCR_COEFF(6):
+    case V_BLEND_RGB2YCBCR_COEFF(7):
+    case V_BLEND_RGB2YCBCR_COEFF(8):
+    case V_BLEND_IN1CSC_COEFF(0):
+    case V_BLEND_IN1CSC_COEFF(1):
+    case V_BLEND_IN1CSC_COEFF(2):
+    case V_BLEND_IN1CSC_COEFF(3):
+    case V_BLEND_IN1CSC_COEFF(4):
+    case V_BLEND_IN1CSC_COEFF(5):
+    case V_BLEND_IN1CSC_COEFF(6):
+    case V_BLEND_IN1CSC_COEFF(7):
+    case V_BLEND_IN1CSC_COEFF(8):
+    case V_BLEND_IN2CSC_COEFF(0):
+    case V_BLEND_IN2CSC_COEFF(1):
+    case V_BLEND_IN2CSC_COEFF(2):
+    case V_BLEND_IN2CSC_COEFF(3):
+    case V_BLEND_IN2CSC_COEFF(4):
+    case V_BLEND_IN2CSC_COEFF(5):
+    case V_BLEND_IN2CSC_COEFF(6):
+    case V_BLEND_IN2CSC_COEFF(7):
+    case V_BLEND_IN2CSC_COEFF(8):
+        s->vblend_registers[offset] = value & 0x0000FFFF;
+        break;
+    case V_BLEND_LUMA_IN1CSC_OFFSET:
+    case V_BLEND_CR_IN1CSC_OFFSET:
+    case V_BLEND_CB_IN1CSC_OFFSET:
+    case V_BLEND_LUMA_IN2CSC_OFFSET:
+    case V_BLEND_CR_IN2CSC_OFFSET:
+    case V_BLEND_CB_IN2CSC_OFFSET:
+    case V_BLEND_LUMA_OUTCSC_OFFSET:
+    case V_BLEND_CR_OUTCSC_OFFSET:
+    case V_BLEND_CB_OUTCSC_OFFSET:
+        s->vblend_registers[offset] = value & 0x3FFF7FFF;
+        break;
+    case V_BLEND_CHROMA_KEY_ENABLE:
+        s->vblend_registers[offset] = value & 0x00000003;
+        break;
+    case V_BLEND_CHROMA_KEY_COMP1:
+    case V_BLEND_CHROMA_KEY_COMP2:
+    case V_BLEND_CHROMA_KEY_COMP3:
+        s->vblend_registers[offset] = value & 0x0FFF0FFF;
+        break;
+    default:
+        s->vblend_registers[offset] = value;
+        break;
+    }
+}
+
+static uint64_t xlnx_dp_vblend_read(void *opaque, hwaddr offset,
+                                    unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    DPRINTF("vblend: read @0x%" HWADDR_PRIX " = 0x%" PRIX32 "\n", offset,
+            s->vblend_registers[offset >> 2]);
+    return s->vblend_registers[offset >> 2];
+}
+
+static const MemoryRegionOps vblend_ops = {
+    .read = xlnx_dp_vblend_read,
+    .write = xlnx_dp_vblend_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+/*
+ * This is to handle Read/Write to the Audio Video buffer manager.
+ */
+static void xlnx_dp_avbufm_write(void *opaque, hwaddr offset, uint64_t value,
+                                 unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    DPRINTF("avbufm: write @0x%" HWADDR_PRIX " = 0x%" PRIX32 "\n", offset,
+                                                               (uint32_t)value);
+    offset = offset >> 2;
+
+    switch (offset) {
+    case AV_BUF_FORMAT:
+        s->avbufm_registers[offset] = value & 0x00000FFF;
+        xlnx_dp_change_graphic_fmt(s);
+        break;
+    case AV_CHBUF0:
+    case AV_CHBUF1:
+    case AV_CHBUF2:
+    case AV_CHBUF3:
+    case AV_CHBUF4:
+    case AV_CHBUF5:
+        s->avbufm_registers[offset] = value & 0x0000007F;
+        break;
+    case AV_BUF_OUTPUT_AUDIO_VIDEO_SELECT:
+        s->avbufm_registers[offset] = value & 0x0000007F;
+        break;
+    case AV_BUF_DITHER_CONFIG:
+        s->avbufm_registers[offset] = value & 0x000007FF;
+        break;
+    case AV_BUF_DITHER_CONFIG_MAX:
+    case AV_BUF_DITHER_CONFIG_MIN:
+        s->avbufm_registers[offset] = value & 0x00000FFF;
+        break;
+    case AV_BUF_PATTERN_GEN_SELECT:
+        s->avbufm_registers[offset] = value & 0xFFFFFF03;
+        break;
+    case AV_BUF_AUD_VID_CLK_SOURCE:
+        s->avbufm_registers[offset] = value & 0x00000007;
+        break;
+    case AV_BUF_SRST_REG:
+        s->avbufm_registers[offset] = value & 0x00000002;
+        break;
+    case AV_BUF_AUDIO_CH_CONFIG:
+        s->avbufm_registers[offset] = value & 0x00000003;
+        break;
+    case AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(0):
+    case AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(1):
+    case AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(2):
+    case AV_BUF_VIDEO_COMP_SCALE_FACTOR(0):
+    case AV_BUF_VIDEO_COMP_SCALE_FACTOR(1):
+    case AV_BUF_VIDEO_COMP_SCALE_FACTOR(2):
+        s->avbufm_registers[offset] = value & 0x0000FFFF;
+        break;
+    case AV_BUF_LIVE_VIDEO_COMP_SF(0):
+    case AV_BUF_LIVE_VIDEO_COMP_SF(1):
+    case AV_BUF_LIVE_VIDEO_COMP_SF(2):
+    case AV_BUF_LIVE_VID_CONFIG:
+    case AV_BUF_LIVE_GFX_COMP_SF(0):
+    case AV_BUF_LIVE_GFX_COMP_SF(1):
+    case AV_BUF_LIVE_GFX_COMP_SF(2):
+    case AV_BUF_LIVE_GFX_CONFIG:
+    case AV_BUF_NON_LIVE_LATENCY:
+    case AV_BUF_STC_CONTROL:
+    case AV_BUF_STC_INIT_VALUE0:
+    case AV_BUF_STC_INIT_VALUE1:
+    case AV_BUF_STC_ADJ:
+    case AV_BUF_STC_VIDEO_VSYNC_TS_REG0:
+    case AV_BUF_STC_VIDEO_VSYNC_TS_REG1:
+    case AV_BUF_STC_EXT_VSYNC_TS_REG0:
+    case AV_BUF_STC_EXT_VSYNC_TS_REG1:
+    case AV_BUF_STC_CUSTOM_EVENT_TS_REG0:
+    case AV_BUF_STC_CUSTOM_EVENT_TS_REG1:
+    case AV_BUF_STC_CUSTOM_EVENT2_TS_REG0:
+    case AV_BUF_STC_CUSTOM_EVENT2_TS_REG1:
+    case AV_BUF_STC_SNAPSHOT0:
+    case AV_BUF_STC_SNAPSHOT1:
+    case AV_BUF_HCOUNT_VCOUNT_INT0:
+    case AV_BUF_HCOUNT_VCOUNT_INT1:
+        qemu_log_mask(LOG_UNIMP, "avbufm: unimplmented");
+        break;
+    default:
+        s->avbufm_registers[offset] = value;
+        break;
+    }
+}
+
+static uint64_t xlnx_dp_avbufm_read(void *opaque, hwaddr offset,
+                                    unsigned size)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    offset = offset >> 2;
+    return s->avbufm_registers[offset];
+}
+
+static const MemoryRegionOps avbufm_ops = {
+    .read = xlnx_dp_avbufm_read,
+    .write = xlnx_dp_avbufm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+/*
+ * This is a global alpha blending using pixman.
+ * Both graphic and video planes are multiplied with the global alpha
+ * coefficient and added.
+ */
+static inline void xlnx_dp_blend_surface(XlnxDPState *s)
+{
+    pixman_fixed_t alpha1[] = { pixman_double_to_fixed(1),
+                                pixman_double_to_fixed(1),
+                                pixman_double_to_fixed(1.0) };
+    pixman_fixed_t alpha2[] = { pixman_double_to_fixed(1),
+                                pixman_double_to_fixed(1),
+                                pixman_double_to_fixed(1.0) };
+
+    if ((surface_width(s->g_plane.surface)
+         != surface_width(s->v_plane.surface)) ||
+        (surface_height(s->g_plane.surface)
+         != surface_height(s->v_plane.surface))) {
+        return;
+    }
+
+    alpha1[2] = pixman_double_to_fixed((double)(xlnx_dp_global_alpha_value(s))
+                                       / 256.0);
+    alpha2[2] = pixman_double_to_fixed((255.0
+                                    - (double)xlnx_dp_global_alpha_value(s))
+                                       / 256.0);
+
+    pixman_image_set_filter(s->g_plane.surface->image,
+                            PIXMAN_FILTER_CONVOLUTION, alpha1, 3);
+    pixman_image_composite(PIXMAN_OP_SRC, s->g_plane.surface->image, 0,
+                           s->bout_plane.surface->image, 0, 0, 0, 0, 0, 0,
+                           surface_width(s->g_plane.surface),
+                           surface_height(s->g_plane.surface));
+    pixman_image_set_filter(s->v_plane.surface->image,
+                            PIXMAN_FILTER_CONVOLUTION, alpha2, 3);
+    pixman_image_composite(PIXMAN_OP_ADD, s->v_plane.surface->image, 0,
+                           s->bout_plane.surface->image, 0, 0, 0, 0, 0, 0,
+                           surface_width(s->g_plane.surface),
+                           surface_height(s->g_plane.surface));
+}
+
+static void xlnx_dp_update_display(void *opaque)
+{
+    XlnxDPState *s = XLNX_DP(opaque);
+
+    if ((s->core_registers[DP_TRANSMITTER_ENABLE] & 0x01) == 0) {
+        return;
+    }
+
+    s->core_registers[DP_INT_STATUS] |= (1 << 13);
+    xlnx_dp_update_irq(s);
+
+    xlnx_dpdma_trigger_vsync_irq(s->dpdma);
+
+    /*
+     * Trigger the DMA channel.
+     */
+    if (!xlnx_dpdma_start_operation(s->dpdma, 3, false)) {
+        /*
+         * An error occured don't do anything with the data..
+         * Trigger an underflow interrupt.
+         */
+        s->core_registers[DP_INT_STATUS] |= (1 << 21);
+        xlnx_dp_update_irq(s);
+        return;
+    }
+
+    if (xlnx_dp_global_alpha_enabled(s)) {
+        if (!xlnx_dpdma_start_operation(s->dpdma, 0, false)) {
+            s->core_registers[DP_INT_STATUS] |= (1 << 21);
+            xlnx_dp_update_irq(s);
+            return;
+        }
+        xlnx_dp_blend_surface(s);
+    }
+
+    /*
+     * XXX: We might want to update only what changed.
+     */
+    dpy_gfx_update(s->console, 0, 0, surface_width(s->g_plane.surface),
+                                     surface_height(s->g_plane.surface));
+}
+
+static const GraphicHwOps xlnx_dp_gfx_ops = {
+    .gfx_update  = xlnx_dp_update_display,
+};
+
+static void xlnx_dp_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    XlnxDPState *s = XLNX_DP(obj);
+
+    memory_region_init(&s->container, obj, TYPE_XLNX_DP, 0xC050);
+
+    memory_region_init_io(&s->core_iomem, obj, &dp_ops, s, TYPE_XLNX_DP
+                          ".core", 0x3AF);
+    memory_region_add_subregion(&s->container, 0x0000, &s->core_iomem);
+
+    memory_region_init_io(&s->vblend_iomem, obj, &vblend_ops, s, TYPE_XLNX_DP
+                          ".v_blend", 0x1DF);
+    memory_region_add_subregion(&s->container, 0xA000, &s->vblend_iomem);
+
+    memory_region_init_io(&s->avbufm_iomem, obj, &avbufm_ops, s, TYPE_XLNX_DP
+                          ".av_buffer_manager", 0x238);
+    memory_region_add_subregion(&s->container, 0xB000, &s->avbufm_iomem);
+
+    memory_region_init_io(&s->audio_iomem, obj, &audio_ops, s, TYPE_XLNX_DP
+                          ".audio", sizeof(s->audio_registers));
+    memory_region_add_subregion(&s->container, 0xC000, &s->audio_iomem);
+
+    sysbus_init_mmio(sbd, &s->container);
+    sysbus_init_irq(sbd, &s->irq);
+
+    object_property_add_link(obj, "dpdma", TYPE_XLNX_DPDMA,
+                             (Object **) &s->dpdma,
+                             xlnx_dp_set_dpdma,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+
+    /*
+     * Initialize AUX Bus.
+     */
+    s->aux_bus = aux_init_bus(DEVICE(obj), "aux");
+
+    /*
+     * Initialize DPCD and EDID..
+     */
+    s->dpcd = DPCD(aux_create_slave(s->aux_bus, "dpcd", 0x00000));
+    s->edid = I2CDDC(qdev_create(BUS(aux_get_i2c_bus(s->aux_bus)), "i2c-ddc"));
+    i2c_set_slave_address(I2C_SLAVE(s->edid), 0x50);
+
+    fifo8_create(&s->rx_fifo, 16);
+    fifo8_create(&s->tx_fifo, 16);
+}
+
+static void xlnx_dp_realize(DeviceState *dev, Error **errp)
+{
+    XlnxDPState *s = XLNX_DP(dev);
+    DisplaySurface *surface;
+    struct audsettings as;
+
+    s->console = graphic_console_init(dev, 0, &xlnx_dp_gfx_ops, s);
+    surface = qemu_console_surface(s->console);
+    xlnx_dpdma_set_host_data_location(s->dpdma, DP_GRAPHIC_DMA_CHANNEL,
+                                      surface_data(surface));
+
+    as.freq = 44100;
+    as.nchannels = 2;
+    as.fmt = AUD_FMT_S16;
+    as.endianness = 0;
+
+    AUD_register_card("xlnx_dp.audio", &s->aud_card);
+
+    s->amixer_output_stream = AUD_open_out(&s->aud_card,
+                                           s->amixer_output_stream,
+                                           "xlnx_dp.audio.out",
+                                           s,
+                                           xlnx_dp_audio_callback,
+                                           &as);
+    AUD_set_volume_out(s->amixer_output_stream, 0, 255, 255);
+    xlnx_dp_audio_activate(s);
+}
+
+static void xlnx_dp_reset(DeviceState *dev)
+{
+    XlnxDPState *s = XLNX_DP(dev);
+
+    memset(s->core_registers, 0, sizeof(s->core_registers));
+    s->core_registers[DP_VERSION_REGISTER] = 0x04010000;
+    s->core_registers[DP_CORE_ID] = 0x01020000;
+    s->core_registers[DP_REPLY_STATUS] = 0x00000010;
+    s->core_registers[DP_MSA_TRANSFER_UNIT_SIZE] = 0x00000040;
+    s->core_registers[DP_INIT_WAIT] = 0x00000020;
+    s->core_registers[DP_PHY_RESET] = 0x00010003;
+    s->core_registers[DP_INT_MASK] = 0xFFFFF03F;
+    s->core_registers[DP_PHY_STATUS] = 0x00000043;
+    s->core_registers[DP_INTERRUPT_SIGNAL_STATE] = 0x00000001;
+
+    s->vblend_registers[V_BLEND_RGB2YCBCR_COEFF(0)] = 0x00001000;
+    s->vblend_registers[V_BLEND_RGB2YCBCR_COEFF(4)] = 0x00001000;
+    s->vblend_registers[V_BLEND_RGB2YCBCR_COEFF(8)] = 0x00001000;
+    s->vblend_registers[V_BLEND_IN1CSC_COEFF(0)] = 0x00001000;
+    s->vblend_registers[V_BLEND_IN1CSC_COEFF(4)] = 0x00001000;
+    s->vblend_registers[V_BLEND_IN1CSC_COEFF(8)] = 0x00001000;
+    s->vblend_registers[V_BLEND_IN2CSC_COEFF(0)] = 0x00001000;
+    s->vblend_registers[V_BLEND_IN2CSC_COEFF(4)] = 0x00001000;
+    s->vblend_registers[V_BLEND_IN2CSC_COEFF(8)] = 0x00001000;
+
+    s->avbufm_registers[AV_BUF_NON_LIVE_LATENCY] = 0x00000180;
+    s->avbufm_registers[AV_BUF_OUTPUT_AUDIO_VIDEO_SELECT] = 0x00000008;
+    s->avbufm_registers[AV_BUF_DITHER_CONFIG_MAX] = 0x00000FFF;
+    s->avbufm_registers[AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(0)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(1)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_GRAPHICS_COMP_SCALE_FACTOR(2)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_VIDEO_COMP_SCALE_FACTOR(0)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_VIDEO_COMP_SCALE_FACTOR(1)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_VIDEO_COMP_SCALE_FACTOR(2)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_LIVE_VIDEO_COMP_SF(0)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_LIVE_VIDEO_COMP_SF(1)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_LIVE_VIDEO_COMP_SF(2)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_LIVE_GFX_COMP_SF(0)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_LIVE_GFX_COMP_SF(1)] = 0x00010101;
+    s->avbufm_registers[AV_BUF_LIVE_GFX_COMP_SF(2)] = 0x00010101;
+
+    memset(s->audio_registers, 0, sizeof(s->audio_registers));
+    s->byte_left = 0;
+
+    xlnx_dp_aux_clear_rx_fifo(s);
+    xlnx_dp_change_graphic_fmt(s);
+    xlnx_dp_update_irq(s);
+}
+
+static void xlnx_dp_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = xlnx_dp_realize;
+    dc->vmsd = &vmstate_dp;
+    dc->reset = xlnx_dp_reset;
+}
+
+static const TypeInfo xlnx_dp_info = {
+    .name          = TYPE_XLNX_DP,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(XlnxDPState),
+    .instance_init = xlnx_dp_init,
+    .class_init    = xlnx_dp_class_init,
+};
+
+static void xlnx_dp_register_types(void)
+{
+    type_register_static(&xlnx_dp_info);
+}
+
+type_init(xlnx_dp_register_types)
diff --git a/hw/dma/Makefile.objs b/hw/dma/Makefile.objs
index 0e65ed0..087c8e6 100644
--- a/hw/dma/Makefile.objs
+++ b/hw/dma/Makefile.objs
@@ -5,9 +5,12 @@
 common-obj-$(CONFIG_I82374) += i82374.o
 common-obj-$(CONFIG_I8257) += i8257.o
 common-obj-$(CONFIG_XILINX_AXI) += xilinx_axidma.o
+common-obj-$(CONFIG_ZYNQ_DEVCFG) += xlnx-zynq-devcfg.o
 common-obj-$(CONFIG_ETRAXFS) += etraxfs_dma.o
 common-obj-$(CONFIG_STP2000) += sparc32_dma.o
 common-obj-$(CONFIG_SUN4M) += sun4m_iommu.o
+obj-$(CONFIG_XLNX_ZYNQMP) += xlnx_dpdma.o
 
 obj-$(CONFIG_OMAP) += omap_dma.o soc_dma.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_dma.o
+obj-$(CONFIG_RASPI) += bcm2835_dma.o
diff --git a/hw/dma/bcm2835_dma.c b/hw/dma/bcm2835_dma.c
new file mode 100644
index 0000000..5d144a2
--- /dev/null
+++ b/hw/dma/bcm2835_dma.c
@@ -0,0 +1,410 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/dma/bcm2835_dma.h"
+#include "qemu/log.h"
+
+/* DMA CS Control and Status bits */
+#define BCM2708_DMA_ACTIVE      (1 << 0)
+#define BCM2708_DMA_END         (1 << 1) /* GE */
+#define BCM2708_DMA_INT         (1 << 2)
+#define BCM2708_DMA_ISPAUSED    (1 << 4)  /* Pause requested or not active */
+#define BCM2708_DMA_ISHELD      (1 << 5)  /* Is held by DREQ flow control */
+#define BCM2708_DMA_ERR         (1 << 8)
+#define BCM2708_DMA_ABORT       (1 << 30) /* stop current CB, go to next, WO */
+#define BCM2708_DMA_RESET       (1 << 31) /* WO, self clearing */
+
+/* DMA control block "info" field bits */
+#define BCM2708_DMA_INT_EN      (1 << 0)
+#define BCM2708_DMA_TDMODE      (1 << 1)
+#define BCM2708_DMA_WAIT_RESP   (1 << 3)
+#define BCM2708_DMA_D_INC       (1 << 4)
+#define BCM2708_DMA_D_WIDTH     (1 << 5)
+#define BCM2708_DMA_D_DREQ      (1 << 6)
+#define BCM2708_DMA_D_IGNORE    (1 << 7)
+#define BCM2708_DMA_S_INC       (1 << 8)
+#define BCM2708_DMA_S_WIDTH     (1 << 9)
+#define BCM2708_DMA_S_DREQ      (1 << 10)
+#define BCM2708_DMA_S_IGNORE    (1 << 11)
+
+/* Register offsets */
+#define BCM2708_DMA_CS          0x00 /* Control and Status */
+#define BCM2708_DMA_ADDR        0x04 /* Control block address */
+/* the current control block appears in the following registers - read only */
+#define BCM2708_DMA_INFO        0x08
+#define BCM2708_DMA_SOURCE_AD   0x0c
+#define BCM2708_DMA_DEST_AD     0x10
+#define BCM2708_DMA_TXFR_LEN    0x14
+#define BCM2708_DMA_STRIDE      0x18
+#define BCM2708_DMA_NEXTCB      0x1C
+#define BCM2708_DMA_DEBUG       0x20
+
+#define BCM2708_DMA_INT_STATUS  0xfe0 /* Interrupt status of each channel */
+#define BCM2708_DMA_ENABLE      0xff0 /* Global enable bits for each channel */
+
+#define BCM2708_DMA_CS_RW_MASK  0x30ff0001 /* All RW bits in DMA_CS */
+
+static void bcm2835_dma_update(BCM2835DMAState *s, unsigned c)
+{
+    BCM2835DMAChan *ch = &s->chan[c];
+    uint32_t data, xlen, ylen;
+    int16_t dst_stride, src_stride;
+
+    if (!(s->enable & (1 << c))) {
+        return;
+    }
+
+    while ((s->enable & (1 << c)) && (ch->conblk_ad != 0)) {
+        /* CB fetch */
+        ch->ti = ldl_le_phys(&s->dma_as, ch->conblk_ad);
+        ch->source_ad = ldl_le_phys(&s->dma_as, ch->conblk_ad + 4);
+        ch->dest_ad = ldl_le_phys(&s->dma_as, ch->conblk_ad + 8);
+        ch->txfr_len = ldl_le_phys(&s->dma_as, ch->conblk_ad + 12);
+        ch->stride = ldl_le_phys(&s->dma_as, ch->conblk_ad + 16);
+        ch->nextconbk = ldl_le_phys(&s->dma_as, ch->conblk_ad + 20);
+
+        if (ch->ti & BCM2708_DMA_TDMODE) {
+            /* 2D transfer mode */
+            ylen = (ch->txfr_len >> 16) & 0x3fff;
+            xlen = ch->txfr_len & 0xffff;
+            dst_stride = ch->stride >> 16;
+            src_stride = ch->stride & 0xffff;
+        } else {
+            ylen = 1;
+            xlen = ch->txfr_len;
+            dst_stride = 0;
+            src_stride = 0;
+        }
+
+        while (ylen != 0) {
+            /* Normal transfer mode */
+            while (xlen != 0) {
+                if (ch->ti & BCM2708_DMA_S_IGNORE) {
+                    /* Ignore reads */
+                    data = 0;
+                } else {
+                    data = ldl_le_phys(&s->dma_as, ch->source_ad);
+                }
+                if (ch->ti & BCM2708_DMA_S_INC) {
+                    ch->source_ad += 4;
+                }
+
+                if (ch->ti & BCM2708_DMA_D_IGNORE) {
+                    /* Ignore writes */
+                } else {
+                    stl_le_phys(&s->dma_as, ch->dest_ad, data);
+                }
+                if (ch->ti & BCM2708_DMA_D_INC) {
+                    ch->dest_ad += 4;
+                }
+
+                /* update remaining transfer length */
+                xlen -= 4;
+                if (ch->ti & BCM2708_DMA_TDMODE) {
+                    ch->txfr_len = (ylen << 16) | xlen;
+                } else {
+                    ch->txfr_len = xlen;
+                }
+            }
+
+            if (--ylen != 0) {
+                ch->source_ad += src_stride;
+                ch->dest_ad += dst_stride;
+            }
+        }
+        ch->cs |= BCM2708_DMA_END;
+        if (ch->ti & BCM2708_DMA_INT_EN) {
+            ch->cs |= BCM2708_DMA_INT;
+            s->int_status |= (1 << c);
+            qemu_set_irq(ch->irq, 1);
+        }
+
+        /* Process next CB */
+        ch->conblk_ad = ch->nextconbk;
+    }
+
+    ch->cs &= ~BCM2708_DMA_ACTIVE;
+    ch->cs |= BCM2708_DMA_ISPAUSED;
+}
+
+static void bcm2835_dma_chan_reset(BCM2835DMAChan *ch)
+{
+    ch->cs = 0;
+    ch->conblk_ad = 0;
+}
+
+static uint64_t bcm2835_dma_read(BCM2835DMAState *s, hwaddr offset,
+                                 unsigned size, unsigned c)
+{
+    BCM2835DMAChan *ch;
+    uint32_t res = 0;
+
+    assert(size == 4);
+    assert(c < BCM2835_DMA_NCHANS);
+
+    ch = &s->chan[c];
+
+    switch (offset) {
+    case BCM2708_DMA_CS:
+        res = ch->cs;
+        break;
+    case BCM2708_DMA_ADDR:
+        res = ch->conblk_ad;
+        break;
+    case BCM2708_DMA_INFO:
+        res = ch->ti;
+        break;
+    case BCM2708_DMA_SOURCE_AD:
+        res = ch->source_ad;
+        break;
+    case BCM2708_DMA_DEST_AD:
+        res = ch->dest_ad;
+        break;
+    case BCM2708_DMA_TXFR_LEN:
+        res = ch->txfr_len;
+        break;
+    case BCM2708_DMA_STRIDE:
+        res = ch->stride;
+        break;
+    case BCM2708_DMA_NEXTCB:
+        res = ch->nextconbk;
+        break;
+    case BCM2708_DMA_DEBUG:
+        res = ch->debug;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        break;
+    }
+    return res;
+}
+
+static void bcm2835_dma_write(BCM2835DMAState *s, hwaddr offset,
+                              uint64_t value, unsigned size, unsigned c)
+{
+    BCM2835DMAChan *ch;
+    uint32_t oldcs;
+
+    assert(size == 4);
+    assert(c < BCM2835_DMA_NCHANS);
+
+    ch = &s->chan[c];
+
+    switch (offset) {
+    case BCM2708_DMA_CS:
+        oldcs = ch->cs;
+        if (value & BCM2708_DMA_RESET) {
+            bcm2835_dma_chan_reset(ch);
+        }
+        if (value & BCM2708_DMA_ABORT) {
+            /* abort is a no-op, since we always run to completion */
+        }
+        if (value & BCM2708_DMA_END) {
+            ch->cs &= ~BCM2708_DMA_END;
+        }
+        if (value & BCM2708_DMA_INT) {
+            ch->cs &= ~BCM2708_DMA_INT;
+            s->int_status &= ~(1 << c);
+            qemu_set_irq(ch->irq, 0);
+        }
+        ch->cs &= ~BCM2708_DMA_CS_RW_MASK;
+        ch->cs |= (value & BCM2708_DMA_CS_RW_MASK);
+        if (!(oldcs & BCM2708_DMA_ACTIVE) && (ch->cs & BCM2708_DMA_ACTIVE)) {
+            bcm2835_dma_update(s, c);
+        }
+        break;
+    case BCM2708_DMA_ADDR:
+        ch->conblk_ad = value;
+        break;
+    case BCM2708_DMA_DEBUG:
+        ch->debug = value;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        break;
+    }
+}
+
+static uint64_t bcm2835_dma0_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835DMAState *s = opaque;
+
+    if (offset < 0xf00) {
+        return bcm2835_dma_read(s, (offset & 0xff), size, (offset >> 8) & 0xf);
+    } else {
+        switch (offset) {
+        case BCM2708_DMA_INT_STATUS:
+            return s->int_status;
+        case BCM2708_DMA_ENABLE:
+            return s->enable;
+        default:
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                          __func__, offset);
+            return 0;
+        }
+    }
+}
+
+static uint64_t bcm2835_dma15_read(void *opaque, hwaddr offset, unsigned size)
+{
+    return bcm2835_dma_read(opaque, (offset & 0xff), size, 15);
+}
+
+static void bcm2835_dma0_write(void *opaque, hwaddr offset, uint64_t value,
+                               unsigned size)
+{
+    BCM2835DMAState *s = opaque;
+
+    if (offset < 0xf00) {
+        bcm2835_dma_write(s, (offset & 0xff), value, size, (offset >> 8) & 0xf);
+    } else {
+        switch (offset) {
+        case BCM2708_DMA_INT_STATUS:
+            break;
+        case BCM2708_DMA_ENABLE:
+            s->enable = (value & 0xffff);
+            break;
+        default:
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                          __func__, offset);
+        }
+    }
+
+}
+
+static void bcm2835_dma15_write(void *opaque, hwaddr offset, uint64_t value,
+                                unsigned size)
+{
+    bcm2835_dma_write(opaque, (offset & 0xff), value, size, 15);
+}
+
+static const MemoryRegionOps bcm2835_dma0_ops = {
+    .read = bcm2835_dma0_read,
+    .write = bcm2835_dma0_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const MemoryRegionOps bcm2835_dma15_ops = {
+    .read = bcm2835_dma15_read,
+    .write = bcm2835_dma15_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_dma_chan = {
+    .name = TYPE_BCM2835_DMA "-chan",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(cs, BCM2835DMAChan),
+        VMSTATE_UINT32(conblk_ad, BCM2835DMAChan),
+        VMSTATE_UINT32(ti, BCM2835DMAChan),
+        VMSTATE_UINT32(source_ad, BCM2835DMAChan),
+        VMSTATE_UINT32(dest_ad, BCM2835DMAChan),
+        VMSTATE_UINT32(txfr_len, BCM2835DMAChan),
+        VMSTATE_UINT32(stride, BCM2835DMAChan),
+        VMSTATE_UINT32(nextconbk, BCM2835DMAChan),
+        VMSTATE_UINT32(debug, BCM2835DMAChan),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_bcm2835_dma = {
+    .name = TYPE_BCM2835_DMA,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_ARRAY(chan, BCM2835DMAState, BCM2835_DMA_NCHANS, 1,
+                             vmstate_bcm2835_dma_chan, BCM2835DMAChan),
+        VMSTATE_UINT32(int_status, BCM2835DMAState),
+        VMSTATE_UINT32(enable, BCM2835DMAState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_dma_init(Object *obj)
+{
+    BCM2835DMAState *s = BCM2835_DMA(obj);
+    int n;
+
+    /* DMA channels 0-14 occupy a contiguous block of IO memory, along
+     * with the global enable and interrupt status bits. Channel 15
+     * has the same register map, but is mapped at a discontiguous
+     * address in a separate IO block.
+     */
+    memory_region_init_io(&s->iomem0, OBJECT(s), &bcm2835_dma0_ops, s,
+                          TYPE_BCM2835_DMA, 0x1000);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem0);
+
+    memory_region_init_io(&s->iomem15, OBJECT(s), &bcm2835_dma15_ops, s,
+                          TYPE_BCM2835_DMA "-chan15", 0x100);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem15);
+
+    for (n = 0; n < 16; n++) {
+        sysbus_init_irq(SYS_BUS_DEVICE(s), &s->chan[n].irq);
+    }
+}
+
+static void bcm2835_dma_reset(DeviceState *dev)
+{
+    BCM2835DMAState *s = BCM2835_DMA(dev);
+    int n;
+
+    s->enable = 0xffff;
+    s->int_status = 0;
+    for (n = 0; n < BCM2835_DMA_NCHANS; n++) {
+        bcm2835_dma_chan_reset(&s->chan[n]);
+    }
+}
+
+static void bcm2835_dma_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835DMAState *s = BCM2835_DMA(dev);
+    Error *err = NULL;
+    Object *obj;
+
+    obj = object_property_get_link(OBJECT(dev), "dma-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required dma-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->dma_mr = MEMORY_REGION(obj);
+    address_space_init(&s->dma_as, s->dma_mr, NULL);
+
+    bcm2835_dma_reset(dev);
+}
+
+static void bcm2835_dma_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = bcm2835_dma_realize;
+    dc->reset = bcm2835_dma_reset;
+    dc->vmsd = &vmstate_bcm2835_dma;
+}
+
+static TypeInfo bcm2835_dma_info = {
+    .name          = TYPE_BCM2835_DMA,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835DMAState),
+    .class_init    = bcm2835_dma_class_init,
+    .instance_init = bcm2835_dma_init,
+};
+
+static void bcm2835_dma_register_types(void)
+{
+    type_register_static(&bcm2835_dma_info);
+}
+
+type_init(bcm2835_dma_register_types)
diff --git a/hw/dma/etraxfs_dma.c b/hw/dma/etraxfs_dma.c
index 3599513..d5650eb 100644
--- a/hw/dma/etraxfs_dma.c
+++ b/hw/dma/etraxfs_dma.c
@@ -21,8 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdio.h>
-#include <sys/time.h>
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "exec/address-spaces.h"
 #include "qemu-common.h"
@@ -441,13 +440,16 @@
 		D(printf("channel %d pushes %x %u bytes eop=%u\n", c,
 		         saved_data_buf, len, out_eop));
 
-		if (ctrl->channels[c].client->client.push)
-			ctrl->channels[c].client->client.push(
-				ctrl->channels[c].client->client.opaque,
-				buf, len, out_eop);
-		else
+		if (ctrl->channels[c].client->client.push) {
+                        if (len > 0) {
+				ctrl->channels[c].client->client.push(
+					ctrl->channels[c].client->client.opaque,
+					buf, len, out_eop);
+			}
+		} else {
 			printf("WARNING: DMA ch%d dataloss,"
 			       " no attached client.\n", c);
+		}
 
 		saved_data_buf += len;
 
diff --git a/hw/dma/i82374.c b/hw/dma/i82374.c
index b8ad2e6..6c0f975 100644
--- a/hw/dma/i82374.c
+++ b/hw/dma/i82374.c
@@ -22,8 +22,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/isa/isa.h"
 
+#define TYPE_I82374 "i82374"
+#define I82374(obj) OBJECT_CHECK(I82374State, (obj), TYPE_I82374)
+
 //#define DEBUG_I82374
 
 #ifdef DEBUG_I82374
@@ -37,8 +41,10 @@
 do { fprintf(stderr, "i82374 ERROR: " fmt , ## __VA_ARGS__); } while (0)
 
 typedef struct I82374State {
+    ISADevice parent_obj;
+
+    uint32_t iobase;
     uint8_t commands[8];
-    qemu_irq out;
     PortioList port_list;
 } I82374State;
 
@@ -99,32 +105,6 @@
     return val;
 }
 
-static void i82374_realize(I82374State *s, Error **errp)
-{
-    DMA_init(1, &s->out);
-    memset(s->commands, 0, sizeof(s->commands));
-}
-
-#define TYPE_I82374 "i82374"
-#define I82374(obj) OBJECT_CHECK(ISAi82374State, (obj), TYPE_I82374)
-
-typedef struct ISAi82374State {
-    ISADevice parent_obj;
-
-    uint32_t iobase;
-    I82374State state;
-} ISAi82374State;
-
-static const VMStateDescription vmstate_isa_i82374 = {
-    .name = "isa-i82374",
-    .version_id = 0,
-    .minimum_version_id = 0,
-    .fields = (VMStateField[]) {
-        VMSTATE_STRUCT(state, ISAi82374State, 0, vmstate_i82374, I82374State),
-        VMSTATE_END_OF_LIST()
-    },
-};
-
 static const MemoryRegionPortio i82374_portio_list[] = {
     { 0x0A, 1, 1, .read = i82374_read_isr, },
     { 0x10, 8, 1, .write = i82374_write_command, },
@@ -134,23 +114,21 @@
     PORTIO_END_OF_LIST(),
 };
 
-static void i82374_isa_realize(DeviceState *dev, Error **errp)
+static void i82374_realize(DeviceState *dev, Error **errp)
 {
-    ISAi82374State *isa = I82374(dev);
-    I82374State *s = &isa->state;
+    I82374State *s = I82374(dev);
 
-    portio_list_init(&s->port_list, OBJECT(isa), i82374_portio_list, s,
+    portio_list_init(&s->port_list, OBJECT(s), i82374_portio_list, s,
                      "i82374");
-    portio_list_add(&s->port_list, isa_address_space_io(&isa->parent_obj),
-                    isa->iobase);
+    portio_list_add(&s->port_list, isa_address_space_io(&s->parent_obj),
+                    s->iobase);
 
-    i82374_realize(s, errp);
-
-    qdev_init_gpio_out(dev, &s->out, 1);
+    DMA_init(isa_bus_from_device(ISA_DEVICE(dev)), 1);
+    memset(s->commands, 0, sizeof(s->commands));
 }
 
 static Property i82374_properties[] = {
-    DEFINE_PROP_UINT32("iobase", ISAi82374State, iobase, 0x400),
+    DEFINE_PROP_UINT32("iobase", I82374State, iobase, 0x400),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -158,21 +136,21 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     
-    dc->realize = i82374_isa_realize;
-    dc->vmsd = &vmstate_isa_i82374;
+    dc->realize = i82374_realize;
+    dc->vmsd = &vmstate_i82374;
     dc->props = i82374_properties;
 }
 
-static const TypeInfo i82374_isa_info = {
+static const TypeInfo i82374_info = {
     .name  = TYPE_I82374,
     .parent = TYPE_ISA_DEVICE,
-    .instance_size  = sizeof(ISAi82374State),
+    .instance_size  = sizeof(I82374State),
     .class_init = i82374_class_init,
 };
 
 static void i82374_register_types(void)
 {
-    type_register_static(&i82374_isa_info);
+    type_register_static(&i82374_info);
 }
 
 type_init(i82374_register_types)
diff --git a/hw/dma/i8257.c b/hw/dma/i8257.c
index a414029..f345c54 100644
--- a/hw/dma/i8257.c
+++ b/hw/dma/i8257.c
@@ -21,11 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
+#include "hw/isa/i8257.h"
 #include "qemu/main-loop.h"
 #include "trace.h"
 
+#define I8257(obj) \
+    OBJECT_CHECK(I8257State, (obj), TYPE_I8257)
+
 /* #define DEBUG_DMA */
 
 #define dolog(...) fprintf (stderr, "dma: " __VA_ARGS__)
@@ -37,33 +42,9 @@
 #define ldebug(...)
 #endif
 
-struct dma_regs {
-    int now[2];
-    uint16_t base[2];
-    uint8_t mode;
-    uint8_t page;
-    uint8_t pageh;
-    uint8_t dack;
-    uint8_t eop;
-    DMA_transfer_handler transfer_handler;
-    void *opaque;
-};
-
 #define ADDR 0
 #define COUNT 1
 
-static struct dma_cont {
-    uint8_t status;
-    uint8_t command;
-    uint8_t mask;
-    uint8_t flip_flop;
-    int dshift;
-    struct dma_regs regs[4];
-    qemu_irq *cpu_request_exit;
-    MemoryRegion channel_io;
-    MemoryRegion cont_io;
-} dma_controllers[2];
-
 enum {
     CMD_MEMORY_TO_MEMORY = 0x01,
     CMD_FIXED_ADDRESS    = 0x02,
@@ -79,13 +60,13 @@
 
 };
 
-static void DMA_run (void);
+static void i8257_dma_run(void *opaque);
 
-static int channels[8] = {-1, 2, 3, 1, -1, -1, -1, 0};
+static const int channels[8] = {-1, 2, 3, 1, -1, -1, -1, 0};
 
-static void write_page (void *opaque, uint32_t nport, uint32_t data)
+static void i8257_write_page(void *opaque, uint32_t nport, uint32_t data)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -96,9 +77,9 @@
     d->regs[ichan].page = data;
 }
 
-static void write_pageh (void *opaque, uint32_t nport, uint32_t data)
+static void i8257_write_pageh(void *opaque, uint32_t nport, uint32_t data)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -109,9 +90,9 @@
     d->regs[ichan].pageh = data;
 }
 
-static uint32_t read_page (void *opaque, uint32_t nport)
+static uint32_t i8257_read_page(void *opaque, uint32_t nport)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -122,9 +103,9 @@
     return d->regs[ichan].page;
 }
 
-static uint32_t read_pageh (void *opaque, uint32_t nport)
+static uint32_t i8257_read_pageh(void *opaque, uint32_t nport)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan;
 
     ichan = channels[nport & 7];
@@ -135,16 +116,16 @@
     return d->regs[ichan].pageh;
 }
 
-static inline void init_chan (struct dma_cont *d, int ichan)
+static inline void i8257_init_chan(I8257State *d, int ichan)
 {
-    struct dma_regs *r;
+    I8257Regs *r;
 
     r = d->regs + ichan;
     r->now[ADDR] = r->base[ADDR] << d->dshift;
     r->now[COUNT] = 0;
 }
 
-static inline int getff (struct dma_cont *d)
+static inline int i8257_getff(I8257State *d)
 {
     int ff;
 
@@ -153,11 +134,11 @@
     return ff;
 }
 
-static uint64_t read_chan(void *opaque, hwaddr nport, unsigned size)
+static uint64_t i8257_read_chan(void *opaque, hwaddr nport, unsigned size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int ichan, nreg, iport, ff, val, dir;
-    struct dma_regs *r;
+    I8257Regs *r;
 
     iport = (nport >> d->dshift) & 0x0f;
     ichan = iport >> 1;
@@ -165,7 +146,7 @@
     r = d->regs + ichan;
 
     dir = ((r->mode >> 5) & 1) ? -1 : 1;
-    ff = getff (d);
+    ff = i8257_getff(d);
     if (nreg)
         val = (r->base[COUNT] << d->dshift) - r->now[COUNT];
     else
@@ -175,29 +156,29 @@
     return (val >> (d->dshift + (ff << 3))) & 0xff;
 }
 
-static void write_chan(void *opaque, hwaddr nport, uint64_t data,
-                       unsigned size)
+static void i8257_write_chan(void *opaque, hwaddr nport, uint64_t data,
+                             unsigned int size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int iport, ichan, nreg;
-    struct dma_regs *r;
+    I8257Regs *r;
 
     iport = (nport >> d->dshift) & 0x0f;
     ichan = iport >> 1;
     nreg = iport & 1;
     r = d->regs + ichan;
-    if (getff (d)) {
+    if (i8257_getff(d)) {
         r->base[nreg] = (r->base[nreg] & 0xff) | ((data << 8) & 0xff00);
-        init_chan (d, ichan);
+        i8257_init_chan(d, ichan);
     } else {
         r->base[nreg] = (r->base[nreg] & 0xff00) | (data & 0xff);
     }
 }
 
-static void write_cont(void *opaque, hwaddr nport, uint64_t data,
-                       unsigned size)
+static void i8257_write_cont(void *opaque, hwaddr nport, uint64_t data,
+                             unsigned int size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int iport, ichan = 0;
 
     iport = (nport >> d->dshift) & 0x0f;
@@ -219,7 +200,7 @@
             d->status &= ~(1 << (ichan + 4));
         }
         d->status &= ~(1 << ichan);
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     case 0x02:                  /* single mask */
@@ -227,7 +208,7 @@
             d->mask |= 1 << (data & 3);
         else
             d->mask &= ~(1 << (data & 3));
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     case 0x03:                  /* mode */
@@ -262,12 +243,12 @@
 
     case 0x06:                  /* clear mask for all channels */
         d->mask = 0;
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     case 0x07:                  /* write mask for all channels */
         d->mask = data;
-        DMA_run();
+        i8257_dma_run(d);
         break;
 
     default:
@@ -283,9 +264,9 @@
 #endif
 }
 
-static uint64_t read_cont(void *opaque, hwaddr nport, unsigned size)
+static uint64_t i8257_read_cont(void *opaque, hwaddr nport, unsigned size)
 {
-    struct dma_cont *d = opaque;
+    I8257State *d = opaque;
     int iport, val;
 
     iport = (nport >> d->dshift) & 0x0f;
@@ -306,37 +287,43 @@
     return val;
 }
 
-int DMA_get_channel_mode (int nchan)
+static IsaDmaTransferMode i8257_dma_get_transfer_mode(IsaDma *obj, int nchan)
 {
-    return dma_controllers[nchan > 3].regs[nchan & 3].mode;
+    I8257State *d = I8257(obj);
+    return (d->regs[nchan & 3].mode >> 2) & 3;
 }
 
-void DMA_hold_DREQ (int nchan)
+static bool i8257_dma_has_autoinitialization(IsaDma *obj, int nchan)
 {
-    int ncont, ichan;
+    I8257State *d = I8257(obj);
+    return (d->regs[nchan & 3].mode >> 4) & 1;
+}
 
-    ncont = nchan > 3;
+static void i8257_dma_hold_DREQ(IsaDma *obj, int nchan)
+{
+    I8257State *d = I8257(obj);
+    int ichan;
+
     ichan = nchan & 3;
-    linfo ("held cont=%d chan=%d\n", ncont, ichan);
-    dma_controllers[ncont].status |= 1 << (ichan + 4);
-    DMA_run();
+    d->status |= 1 << (ichan + 4);
+    i8257_dma_run(d);
 }
 
-void DMA_release_DREQ (int nchan)
+static void i8257_dma_release_DREQ(IsaDma *obj, int nchan)
 {
-    int ncont, ichan;
+    I8257State *d = I8257(obj);
+    int ichan;
 
-    ncont = nchan > 3;
     ichan = nchan & 3;
-    linfo ("released cont=%d chan=%d\n", ncont, ichan);
-    dma_controllers[ncont].status &= ~(1 << (ichan + 4));
-    DMA_run();
+    d->status &= ~(1 << (ichan + 4));
+    i8257_dma_run(d);
 }
 
-static void channel_run (int ncont, int ichan)
+static void i8257_channel_run(I8257State *d, int ichan)
 {
+    int ncont = d->dshift;
     int n;
-    struct dma_regs *r = &dma_controllers[ncont].regs[ichan];
+    I8257Regs *r = &d->regs[ichan];
 #ifdef DEBUG_DMA
     int dir, opmode;
 
@@ -355,68 +342,64 @@
                              r->now[COUNT], (r->base[COUNT] + 1) << ncont);
     r->now[COUNT] = n;
     ldebug ("dma_pos %d size %d\n", n, (r->base[COUNT] + 1) << ncont);
+    if (n == (r->base[COUNT] + 1) << ncont) {
+        ldebug("transfer done\n");
+        d->status |= (1 << ichan);
+    }
 }
 
-static QEMUBH *dma_bh;
-
-static void DMA_run (void)
+static void i8257_dma_run(void *opaque)
 {
-    struct dma_cont *d;
-    int icont, ichan;
+    I8257State *d = opaque;
+    int ichan;
     int rearm = 0;
-    static int running = 0;
 
-    if (running) {
+    if (d->running) {
         rearm = 1;
         goto out;
     } else {
-        running = 1;
+        d->running = 1;
     }
 
-    d = dma_controllers;
+    for (ichan = 0; ichan < 4; ichan++) {
+        int mask;
 
-    for (icont = 0; icont < 2; icont++, d++) {
-        for (ichan = 0; ichan < 4; ichan++) {
-            int mask;
+        mask = 1 << ichan;
 
-            mask = 1 << ichan;
-
-            if ((0 == (d->mask & mask)) && (0 != (d->status & (mask << 4)))) {
-                channel_run (icont, ichan);
-                rearm = 1;
-            }
+        if ((0 == (d->mask & mask)) && (0 != (d->status & (mask << 4)))) {
+            i8257_channel_run(d, ichan);
+            rearm = 1;
         }
     }
 
-    running = 0;
+    d->running = 0;
 out:
-    if (rearm)
-        qemu_bh_schedule_idle(dma_bh);
+    if (rearm) {
+        qemu_bh_schedule_idle(d->dma_bh);
+        d->dma_bh_scheduled = true;
+    }
 }
 
-static void DMA_run_bh(void *unused)
+static void i8257_dma_register_channel(IsaDma *obj, int nchan,
+                                       IsaDmaTransferHandler transfer_handler,
+                                       void *opaque)
 {
-    DMA_run();
-}
+    I8257State *d = I8257(obj);
+    I8257Regs *r;
+    int ichan;
 
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque)
-{
-    struct dma_regs *r;
-    int ichan, ncont;
-
-    ncont = nchan > 3;
     ichan = nchan & 3;
 
-    r = dma_controllers[ncont].regs + ichan;
+    r = d->regs + ichan;
     r->transfer_handler = transfer_handler;
     r->opaque = opaque;
 }
 
-int DMA_read_memory (int nchan, void *buf, int pos, int len)
+static int i8257_dma_read_memory(IsaDma *obj, int nchan, void *buf, int pos,
+                                 int len)
 {
-    struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
+    I8257State *d = I8257(obj);
+    I8257Regs *r = &d->regs[nchan & 3];
     hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
@@ -436,9 +419,11 @@
     return len;
 }
 
-int DMA_write_memory (int nchan, void *buf, int pos, int len)
+static int i8257_dma_write_memory(IsaDma *obj, int nchan, void *buf, int pos,
+                                 int len)
 {
-    struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
+    I8257State *s = I8257(obj);
+    I8257Regs *r = &s->regs[nchan & 3];
     hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
@@ -458,21 +443,25 @@
     return len;
 }
 
-/* request the emulator to transfer a new DMA memory block ASAP */
-void DMA_schedule(int nchan)
+/* request the emulator to transfer a new DMA memory block ASAP (even
+ * if the idle bottom half would not have exited the iothread yet).
+ */
+static void i8257_dma_schedule(IsaDma *obj)
 {
-    struct dma_cont *d = &dma_controllers[nchan > 3];
-
-    qemu_irq_pulse(*d->cpu_request_exit);
+    I8257State *d = I8257(obj);
+    if (d->dma_bh_scheduled) {
+        qemu_notify_event();
+    }
 }
 
-static void dma_reset(void *opaque)
+static void i8257_reset(DeviceState *dev)
 {
-    struct dma_cont *d = opaque;
-    write_cont(d, (0x05 << d->dshift), 0, 1);
+    I8257State *d = I8257(dev);
+    i8257_write_cont(d, (0x05 << d->dshift), 0, 1);
 }
 
-static int dma_phony_handler (void *opaque, int nchan, int dma_pos, int dma_len)
+static int i8257_phony_handler(void *opaque, int nchan, int dma_pos,
+                               int dma_len)
 {
     trace_i8257_unregistered_dma(nchan, dma_pos, dma_len);
     return dma_pos;
@@ -480,8 +469,8 @@
 
 
 static const MemoryRegionOps channel_io_ops = {
-    .read = read_chan,
-    .write = write_chan,
+    .read = i8257_read_chan,
+    .write = i8257_write_chan,
     .endianness = DEVICE_NATIVE_ENDIAN,
     .impl = {
         .min_access_size = 1,
@@ -491,21 +480,21 @@
 
 /* IOport from page_base */
 static const MemoryRegionPortio page_portio_list[] = {
-    { 0x01, 3, 1, .write = write_page, .read = read_page, },
-    { 0x07, 1, 1, .write = write_page, .read = read_page, },
+    { 0x01, 3, 1, .write = i8257_write_page, .read = i8257_read_page, },
+    { 0x07, 1, 1, .write = i8257_write_page, .read = i8257_read_page, },
     PORTIO_END_OF_LIST(),
 };
 
 /* IOport from pageh_base */
 static const MemoryRegionPortio pageh_portio_list[] = {
-    { 0x01, 3, 1, .write = write_pageh, .read = read_pageh, },
-    { 0x07, 3, 1, .write = write_pageh, .read = read_pageh, },
+    { 0x01, 3, 1, .write = i8257_write_pageh, .read = i8257_read_pageh, },
+    { 0x07, 3, 1, .write = i8257_write_pageh, .read = i8257_read_pageh, },
     PORTIO_END_OF_LIST(),
 };
 
 static const MemoryRegionOps cont_io_ops = {
-    .read = read_cont,
-    .write = write_cont,
+    .read = i8257_read_cont,
+    .write = i8257_write_cont,
     .endianness = DEVICE_NATIVE_ENDIAN,
     .impl = {
         .min_access_size = 1,
@@ -513,86 +502,142 @@
     },
 };
 
-/* dshift = 0: 8 bit DMA, 1 = 16 bit DMA */
-static void dma_init2(struct dma_cont *d, int base, int dshift,
-                      int page_base, int pageh_base,
-                      qemu_irq *cpu_request_exit)
-{
-    int i;
-
-    d->dshift = dshift;
-    d->cpu_request_exit = cpu_request_exit;
-
-    memory_region_init_io(&d->channel_io, NULL, &channel_io_ops, d,
-                          "dma-chan", 8 << d->dshift);
-    memory_region_add_subregion(isa_address_space_io(NULL),
-                                base, &d->channel_io);
-
-    isa_register_portio_list(NULL, page_base, page_portio_list, d,
-                             "dma-page");
-    if (pageh_base >= 0) {
-        isa_register_portio_list(NULL, pageh_base, pageh_portio_list, d,
-                                 "dma-pageh");
-    }
-
-    memory_region_init_io(&d->cont_io, NULL, &cont_io_ops, d, "dma-cont",
-                          8 << d->dshift);
-    memory_region_add_subregion(isa_address_space_io(NULL),
-                                base + (8 << d->dshift), &d->cont_io);
-
-    qemu_register_reset(dma_reset, d);
-    dma_reset(d);
-    for (i = 0; i < ARRAY_SIZE (d->regs); ++i) {
-        d->regs[i].transfer_handler = dma_phony_handler;
-    }
-}
-
-static const VMStateDescription vmstate_dma_regs = {
+static const VMStateDescription vmstate_i8257_regs = {
     .name = "dma_regs",
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_INT32_ARRAY(now, struct dma_regs, 2),
-        VMSTATE_UINT16_ARRAY(base, struct dma_regs, 2),
-        VMSTATE_UINT8(mode, struct dma_regs),
-        VMSTATE_UINT8(page, struct dma_regs),
-        VMSTATE_UINT8(pageh, struct dma_regs),
-        VMSTATE_UINT8(dack, struct dma_regs),
-        VMSTATE_UINT8(eop, struct dma_regs),
+        VMSTATE_INT32_ARRAY(now, I8257Regs, 2),
+        VMSTATE_UINT16_ARRAY(base, I8257Regs, 2),
+        VMSTATE_UINT8(mode, I8257Regs),
+        VMSTATE_UINT8(page, I8257Regs),
+        VMSTATE_UINT8(pageh, I8257Regs),
+        VMSTATE_UINT8(dack, I8257Regs),
+        VMSTATE_UINT8(eop, I8257Regs),
         VMSTATE_END_OF_LIST()
     }
 };
 
-static int dma_post_load(void *opaque, int version_id)
+static int i8257_post_load(void *opaque, int version_id)
 {
-    DMA_run();
+    I8257State *d = opaque;
+    i8257_dma_run(d);
 
     return 0;
 }
 
-static const VMStateDescription vmstate_dma = {
+static const VMStateDescription vmstate_i8257 = {
     .name = "dma",
     .version_id = 1,
     .minimum_version_id = 1,
-    .post_load = dma_post_load,
+    .post_load = i8257_post_load,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT8(command, struct dma_cont),
-        VMSTATE_UINT8(mask, struct dma_cont),
-        VMSTATE_UINT8(flip_flop, struct dma_cont),
-        VMSTATE_INT32(dshift, struct dma_cont),
-        VMSTATE_STRUCT_ARRAY(regs, struct dma_cont, 4, 1, vmstate_dma_regs, struct dma_regs),
+        VMSTATE_UINT8(command, I8257State),
+        VMSTATE_UINT8(mask, I8257State),
+        VMSTATE_UINT8(flip_flop, I8257State),
+        VMSTATE_INT32(dshift, I8257State),
+        VMSTATE_STRUCT_ARRAY(regs, I8257State, 4, 1, vmstate_i8257_regs,
+                             I8257Regs),
         VMSTATE_END_OF_LIST()
     }
 };
 
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+static void i8257_realize(DeviceState *dev, Error **errp)
 {
-    dma_init2(&dma_controllers[0], 0x00, 0, 0x80,
-              high_page_enable ? 0x480 : -1, cpu_request_exit);
-    dma_init2(&dma_controllers[1], 0xc0, 1, 0x88,
-              high_page_enable ? 0x488 : -1, cpu_request_exit);
-    vmstate_register (NULL, 0, &vmstate_dma, &dma_controllers[0]);
-    vmstate_register (NULL, 1, &vmstate_dma, &dma_controllers[1]);
+    ISADevice *isa = ISA_DEVICE(dev);
+    I8257State *d = I8257(dev);
+    int i;
 
-    dma_bh = qemu_bh_new(DMA_run_bh, NULL);
+    memory_region_init_io(&d->channel_io, NULL, &channel_io_ops, d,
+                          "dma-chan", 8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(isa),
+                                d->base, &d->channel_io);
+
+    isa_register_portio_list(isa, d->page_base, page_portio_list, d,
+                             "dma-page");
+    if (d->pageh_base >= 0) {
+        isa_register_portio_list(isa, d->pageh_base, pageh_portio_list, d,
+                                 "dma-pageh");
+    }
+
+    memory_region_init_io(&d->cont_io, OBJECT(isa), &cont_io_ops, d,
+                          "dma-cont", 8 << d->dshift);
+    memory_region_add_subregion(isa_address_space_io(isa),
+                                d->base + (8 << d->dshift), &d->cont_io);
+
+    for (i = 0; i < ARRAY_SIZE(d->regs); ++i) {
+        d->regs[i].transfer_handler = i8257_phony_handler;
+    }
+
+    d->dma_bh = qemu_bh_new(i8257_dma_run, d);
+}
+
+static Property i8257_properties[] = {
+    DEFINE_PROP_INT32("base", I8257State, base, 0x00),
+    DEFINE_PROP_INT32("page-base", I8257State, page_base, 0x80),
+    DEFINE_PROP_INT32("pageh-base", I8257State, pageh_base, 0x480),
+    DEFINE_PROP_INT32("dshift", I8257State, dshift, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void i8257_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IsaDmaClass *idc = ISADMA_CLASS(klass);
+
+    dc->realize = i8257_realize;
+    dc->reset = i8257_reset;
+    dc->vmsd = &vmstate_i8257;
+    dc->props = i8257_properties;
+
+    idc->get_transfer_mode = i8257_dma_get_transfer_mode;
+    idc->has_autoinitialization = i8257_dma_has_autoinitialization;
+    idc->read_memory = i8257_dma_read_memory;
+    idc->write_memory = i8257_dma_write_memory;
+    idc->hold_DREQ = i8257_dma_hold_DREQ;
+    idc->release_DREQ = i8257_dma_release_DREQ;
+    idc->schedule = i8257_dma_schedule;
+    idc->register_channel = i8257_dma_register_channel;
+}
+
+static const TypeInfo i8257_info = {
+    .name = TYPE_I8257,
+    .parent = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(I8257State),
+    .class_init = i8257_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_ISADMA },
+        { }
+    }
+};
+
+static void i8257_register_types(void)
+{
+    type_register_static(&i8257_info);
+}
+
+type_init(i8257_register_types)
+
+void DMA_init(ISABus *bus, int high_page_enable)
+{
+    ISADevice *isa1, *isa2;
+    DeviceState *d;
+
+    isa1 = isa_create(bus, TYPE_I8257);
+    d = DEVICE(isa1);
+    qdev_prop_set_int32(d, "base", 0x00);
+    qdev_prop_set_int32(d, "page-base", 0x80);
+    qdev_prop_set_int32(d, "pageh-base", high_page_enable ? 0x480 : -1);
+    qdev_prop_set_int32(d, "dshift", 0);
+    qdev_init_nofail(d);
+
+    isa2 = isa_create(bus, TYPE_I8257);
+    d = DEVICE(isa2);
+    qdev_prop_set_int32(d, "base", 0xc0);
+    qdev_prop_set_int32(d, "page-base", 0x88);
+    qdev_prop_set_int32(d, "pageh-base", high_page_enable ? 0x488 : -1);
+    qdev_prop_set_int32(d, "dshift", 1);
+    qdev_init_nofail(d);
+
+    isa_bus_dma(bus, ISADMA(isa1), ISADMA(isa2));
 }
diff --git a/hw/dma/omap_dma.c b/hw/dma/omap_dma.c
index 756a87a..700cd6b 100644
--- a/hw/dma/omap_dma.c
+++ b/hw/dma/omap_dma.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/arm/omap.h"
@@ -136,7 +137,7 @@
 
 static inline void omap_dma_interrupts_update(struct omap_dma_s *s)
 {
-    return s->intr_update(s);
+    s->intr_update(s);
 }
 
 static void omap_dma_channel_load(struct omap_dma_channel_s *ch)
@@ -1502,7 +1503,8 @@
     int reg, ch;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -1625,8 +1627,7 @@
                 enum omap_dma_model model)
 {
     int num_irqs, memsize, i;
-    struct omap_dma_s *s = (struct omap_dma_s *)
-            g_malloc0(sizeof(struct omap_dma_s));
+    struct omap_dma_s *s = g_new0(struct omap_dma_s, 1);
 
     if (model <= omap_dma_3_1) {
         num_irqs = 6;
@@ -1857,7 +1858,8 @@
     struct omap_dma_channel_s *ch;
 
     if (size == 1) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -2059,8 +2061,7 @@
                 int chans, omap_clk iclk, omap_clk fclk)
 {
     int i;
-    struct omap_dma_s *s = (struct omap_dma_s *)
-            g_malloc0(sizeof(struct omap_dma_s));
+    struct omap_dma_s *s = g_new0(struct omap_dma_s, 1);
 
     s->model = omap_dma_4;
     s->chans = chans;
diff --git a/hw/dma/pl080.c b/hw/dma/pl080.c
index 741dd20..3bed5c3 100644
--- a/hw/dma/pl080.c
+++ b/hw/dma/pl080.c
@@ -7,8 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
+#include "qemu/log.h"
 
 #define PL080_MAX_CHANNELS 8
 #define PL080_CONF_E    0x1
@@ -205,10 +207,22 @@
             if (size == 0) {
                 /* Transfer complete.  */
                 if (ch->lli) {
-                    ch->src = ldl_le_phys(&address_space_memory, ch->lli);
-                    ch->dest = ldl_le_phys(&address_space_memory, ch->lli + 4);
-                    ch->ctrl = ldl_le_phys(&address_space_memory, ch->lli + 12);
-                    ch->lli = ldl_le_phys(&address_space_memory, ch->lli + 8);
+                    ch->src = address_space_ldl_le(&address_space_memory,
+                                                   ch->lli,
+                                                   MEMTXATTRS_UNSPECIFIED,
+                                                   NULL);
+                    ch->dest = address_space_ldl_le(&address_space_memory,
+                                                    ch->lli + 4,
+                                                    MEMTXATTRS_UNSPECIFIED,
+                                                    NULL);
+                    ch->ctrl = address_space_ldl_le(&address_space_memory,
+                                                    ch->lli + 12,
+                                                    MEMTXATTRS_UNSPECIFIED,
+                                                    NULL);
+                    ch->lli = address_space_ldl_le(&address_space_memory,
+                                                   ch->lli + 8,
+                                                   MEMTXATTRS_UNSPECIFIED,
+                                                   NULL);
                 } else {
                     ch->conf &= ~PL080_CCONF_E;
                 }
diff --git a/hw/dma/pl330.c b/hw/dma/pl330.c
index 6b6eaae..c0bd9fe 100644
--- a/hw/dma/pl330.c
+++ b/hw/dma/pl330.c
@@ -14,9 +14,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/dma.h"
+#include "qemu/log.h"
 
 #ifndef PL330_ERR_DEBUG
 #define PL330_ERR_DEBUG 0
@@ -286,7 +289,7 @@
                        PL330Queue),
         VMSTATE_STRUCT(write_queue, PL330State, 0, vmstate_pl330_queue,
                        PL330Queue),
-        VMSTATE_TIMER(timer, PL330State),
+        VMSTATE_TIMER_PTR(timer, PL330State),
         VMSTATE_UINT32(inten, PL330State),
         VMSTATE_UINT32(int_status, PL330State),
         VMSTATE_UINT32(ev_status, PL330State),
@@ -1566,7 +1569,7 @@
         s->cfg[1] |= 5;
         break;
     default:
-        error_setg(errp, "Bad value for i-cache_len property: %" PRIx8 "\n",
+        error_setg(errp, "Bad value for i-cache_len property: %" PRIx8,
                    s->i_cache_len);
         return;
     }
@@ -1601,7 +1604,7 @@
         s->cfg[CFG_CRD] |= 0x4;
         break;
     default:
-        error_setg(errp, "Bad value for data_width property: %" PRIx8 "\n",
+        error_setg(errp, "Bad value for data_width property: %" PRIx8,
                    s->data_width);
         return;
     }
diff --git a/hw/dma/puv3_dma.c b/hw/dma/puv3_dma.c
index 101bd7f..b97a6c1 100644
--- a/hw/dma/puv3_dma.c
+++ b/hw/dma/puv3_dma.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/hw/dma/pxa2xx_dma.c b/hw/dma/pxa2xx_dma.c
index d4501fb..634a432 100644
--- a/hw/dma/pxa2xx_dma.c
+++ b/hw/dma/pxa2xx_dma.c
@@ -8,9 +8,11 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 
 #define PXA255_DMA_NUM_CHANNELS 16
 #define PXA27X_DMA_NUM_CHANNELS 32
@@ -449,32 +451,36 @@
     }
 }
 
-static int pxa2xx_dma_init(SysBusDevice *sbd)
+static void pxa2xx_dma_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    PXA2xxDMAState *s = PXA2XX_DMA(dev);
-    int i;
-
-    if (s->channels <= 0) {
-        return -1;
-    }
-
-    s->chan = g_malloc0(sizeof(PXA2xxDMAChannel) * s->channels);
-
-    memset(s->chan, 0, sizeof(PXA2xxDMAChannel) * s->channels);
-    for (i = 0; i < s->channels; i ++)
-        s->chan[i].state = DCSR_STOPINTR;
+    DeviceState *dev = DEVICE(obj);
+    PXA2xxDMAState *s = PXA2XX_DMA(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     memset(s->req, 0, sizeof(uint8_t) * PXA2XX_DMA_NUM_REQUESTS);
 
     qdev_init_gpio_in(dev, pxa2xx_dma_request, PXA2XX_DMA_NUM_REQUESTS);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pxa2xx_dma_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_dma_ops, s,
                           "pxa2xx.dma", 0x00010000);
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq);
+}
 
-    return 0;
+static void pxa2xx_dma_realize(DeviceState *dev, Error **errp)
+{
+    PXA2xxDMAState *s = PXA2XX_DMA(dev);
+    int i;
+
+    if (s->channels <= 0) {
+        error_setg(errp, "channels value invalid");
+        return;
+    }
+
+    s->chan = g_new0(PXA2xxDMAChannel, s->channels);
+
+    for (i = 0; i < s->channels; i ++)
+        s->chan[i].state = DCSR_STOPINTR;
 }
 
 DeviceState *pxa27x_dma_init(hwaddr base, qemu_irq irq)
@@ -553,18 +559,18 @@
 static void pxa2xx_dma_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pxa2xx_dma_init;
     dc->desc = "PXA2xx DMA controller";
     dc->vmsd = &vmstate_pxa2xx_dma;
     dc->props = pxa2xx_dma_properties;
+    dc->realize = pxa2xx_dma_realize;
 }
 
 static const TypeInfo pxa2xx_dma_info = {
     .name          = TYPE_PXA2XX_DMA,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxDMAState),
+    .instance_init = pxa2xx_dma_init,
     .class_init    = pxa2xx_dma_class_init,
 };
 
diff --git a/hw/dma/rc4030.c b/hw/dma/rc4030.c
index af26632..2f2576f 100644
--- a/hw/dma/rc4030.c
+++ b/hw/dma/rc4030.c
@@ -1,7 +1,7 @@
 /*
  * QEMU JAZZ RC4030 chipset
  *
- * Copyright (c) 2007-2009 Herve Poussineau
+ * Copyright (c) 2007-2013 Hervé Poussineau
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,31 +22,20 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
+#include "hw/sysbus.h"
 #include "qemu/timer.h"
-
-/********************************************************/
-/* debug rc4030 */
-
-//#define DEBUG_RC4030
-//#define DEBUG_RC4030_DMA
-
-#ifdef DEBUG_RC4030
-#define DPRINTF(fmt, ...) \
-do { printf("rc4030: " fmt , ## __VA_ARGS__); } while (0)
-static const char* irq_names[] = { "parallel", "floppy", "sound", "video",
-            "network", "scsi", "keyboard", "mouse", "serial0", "serial1" };
-#else
-#define DPRINTF(fmt, ...)
-#endif
-
-#define RC4030_ERROR(fmt, ...) \
-do { fprintf(stderr, "rc4030 ERROR: %s: " fmt, __func__ , ## __VA_ARGS__); } while (0)
+#include "qemu/log.h"
+#include "exec/address-spaces.h"
+#include "trace.h"
 
 /********************************************************/
 /* rc4030 emulation                                     */
 
+#define MAX_TL_ENTRIES 512
+
 typedef struct dma_pagetable_entry {
     int32_t frame;
     int32_t owner;
@@ -63,8 +52,14 @@
 #define DMA_FLAG_MEM_INTR   0x0200
 #define DMA_FLAG_ADDR_INTR  0x0400
 
+#define TYPE_RC4030 "rc4030"
+#define RC4030(obj) \
+    OBJECT_CHECK(rc4030State, (obj), TYPE_RC4030)
+
 typedef struct rc4030State
 {
+    SysBusDevice parent;
+
     uint32_t config; /* 0x0000: RC4030 config register */
     uint32_t revision; /* 0x0008: RC4030 Revision register */
     uint32_t invalid_address_register; /* 0x0010: Invalid Address register */
@@ -83,7 +78,7 @@
     uint32_t cache_bmask; /* 0x0058: I/O Cache Byte Mask */
 
     uint32_t nmi_interrupt; /* 0x0200: interrupt source */
-    uint32_t offset210;
+    uint32_t memory_refresh_rate; /* 0x0210: memory refresh rate */
     uint32_t nvram_protect; /* 0x0220: NV ram protect register */
     uint32_t rem_speed[16];
     uint32_t imr_jazz; /* Local bus int enable mask */
@@ -96,6 +91,16 @@
     qemu_irq timer_irq;
     qemu_irq jazz_bus_irq;
 
+    /* biggest translation table */
+    MemoryRegion dma_tt;
+    /* translation table memory region alias, added to system RAM */
+    MemoryRegion dma_tt_alias;
+    /* whole DMA memory region, root of DMA address space */
+    MemoryRegion dma_mr;
+    /* translation table entry aliases, added to DMA memory region */
+    MemoryRegion dma_mrs[MAX_TL_ENTRIES];
+    AddressSpace dma_as;
+
     MemoryRegion iomem_chipset;
     MemoryRegion iomem_jazzio;
 } rc4030State;
@@ -108,11 +113,11 @@
     tm_hz = 1000 / (s->itr + 1);
 
     timer_mod(s->periodic_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   get_ticks_per_sec() / tm_hz);
+                   NANOSECONDS_PER_SECOND / tm_hz);
 }
 
 /* called for accesses to rc4030 */
-static uint32_t rc4030_readl(void *opaque, hwaddr addr)
+static uint64_t rc4030_read(void *opaque, hwaddr addr, unsigned int size)
 {
     rc4030State *s = opaque;
     uint32_t val;
@@ -220,9 +225,9 @@
     case 0x0208:
         val = 0;
         break;
-    /* Offset 0x0210 */
+    /* Memory refresh rate */
     case 0x0210:
-        val = s->offset210;
+        val = s->memory_refresh_rate;
         break;
     /* NV ram protect register */
     case 0x0220:
@@ -238,39 +243,117 @@
         val = 7; /* FIXME: should be read from EISA controller */
         break;
     default:
-        RC4030_ERROR("invalid read [" TARGET_FMT_plx "]\n", addr);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "rc4030: invalid read at 0x%x", (int)addr);
         val = 0;
         break;
     }
 
     if ((addr & ~3) != 0x230) {
-        DPRINTF("read 0x%02x at " TARGET_FMT_plx "\n", val, addr);
+        trace_rc4030_read(addr, val);
     }
 
     return val;
 }
 
-static uint32_t rc4030_readw(void *opaque, hwaddr addr)
+static void rc4030_dma_as_update_one(rc4030State *s, int index, uint32_t frame)
 {
-    uint32_t v = rc4030_readl(opaque, addr & ~0x3);
-    if (addr & 0x2)
-        return v >> 16;
-    else
-        return v & 0xffff;
+    if (index < MAX_TL_ENTRIES) {
+        memory_region_set_enabled(&s->dma_mrs[index], false);
+    }
+
+    if (!frame) {
+        return;
+    }
+
+    if (index >= MAX_TL_ENTRIES) {
+        qemu_log_mask(LOG_UNIMP,
+                      "rc4030: trying to use too high "
+                      "translation table entry %d (max allowed=%d)",
+                      index, MAX_TL_ENTRIES);
+        return;
+    }
+    memory_region_set_alias_offset(&s->dma_mrs[index], frame);
+    memory_region_set_enabled(&s->dma_mrs[index], true);
 }
 
-static uint32_t rc4030_readb(void *opaque, hwaddr addr)
-{
-    uint32_t v = rc4030_readl(opaque, addr & ~0x3);
-    return (v >> (8 * (addr & 0x3))) & 0xff;
-}
-
-static void rc4030_writel(void *opaque, hwaddr addr, uint32_t val)
+static void rc4030_dma_tt_write(void *opaque, hwaddr addr, uint64_t data,
+                                unsigned int size)
 {
     rc4030State *s = opaque;
+
+    /* write memory */
+    memcpy(memory_region_get_ram_ptr(&s->dma_tt) + addr, &data, size);
+
+    /* update dma address space (only if frame field has been written) */
+    if (addr % sizeof(dma_pagetable_entry) == 0) {
+        int index = addr / sizeof(dma_pagetable_entry);
+        memory_region_transaction_begin();
+        rc4030_dma_as_update_one(s, index, (uint32_t)data);
+        memory_region_transaction_commit();
+    }
+}
+
+static const MemoryRegionOps rc4030_dma_tt_ops = {
+    .write = rc4030_dma_tt_write,
+    .impl.min_access_size = 4,
+    .impl.max_access_size = 4,
+};
+
+static void rc4030_dma_tt_update(rc4030State *s, uint32_t new_tl_base,
+                                 uint32_t new_tl_limit)
+{
+    int entries, i;
+    dma_pagetable_entry *dma_tl_contents;
+
+    if (s->dma_tl_limit) {
+        /* write old dma tl table to physical memory */
+        memory_region_del_subregion(get_system_memory(), &s->dma_tt_alias);
+        cpu_physical_memory_write(s->dma_tl_limit & 0x7fffffff,
+                                  memory_region_get_ram_ptr(&s->dma_tt),
+                                  memory_region_size(&s->dma_tt_alias));
+    }
+    object_unparent(OBJECT(&s->dma_tt_alias));
+
+    s->dma_tl_base = new_tl_base;
+    s->dma_tl_limit = new_tl_limit;
+    new_tl_base &= 0x7fffffff;
+
+    if (s->dma_tl_limit) {
+        uint64_t dma_tt_size;
+        if (s->dma_tl_limit <= memory_region_size(&s->dma_tt)) {
+            dma_tt_size = s->dma_tl_limit;
+        } else {
+            dma_tt_size = memory_region_size(&s->dma_tt);
+        }
+        memory_region_init_alias(&s->dma_tt_alias, OBJECT(s),
+                                 "dma-table-alias",
+                                 &s->dma_tt, 0, dma_tt_size);
+        dma_tl_contents = memory_region_get_ram_ptr(&s->dma_tt);
+        cpu_physical_memory_read(new_tl_base, dma_tl_contents, dma_tt_size);
+
+        memory_region_transaction_begin();
+        entries = dma_tt_size / sizeof(dma_pagetable_entry);
+        for (i = 0; i < entries; i++) {
+            rc4030_dma_as_update_one(s, i, dma_tl_contents[i].frame);
+        }
+        memory_region_add_subregion(get_system_memory(), new_tl_base,
+                                    &s->dma_tt_alias);
+        memory_region_transaction_commit();
+    } else {
+        memory_region_init(&s->dma_tt_alias, OBJECT(s),
+                           "dma-table-alias", 0);
+    }
+}
+
+static void rc4030_write(void *opaque, hwaddr addr, uint64_t data,
+                         unsigned int size)
+{
+    rc4030State *s = opaque;
+    uint32_t val = data;
     addr &= 0x3fff;
 
-    DPRINTF("write 0x%02x at " TARGET_FMT_plx "\n", val, addr);
+    trace_rc4030_write(addr, val);
 
     switch (addr & ~0x3) {
     /* Global config register */
@@ -279,11 +362,11 @@
         break;
     /* DMA transl. table base */
     case 0x0018:
-        s->dma_tl_base = val;
+        rc4030_dma_tt_update(s, val, s->dma_tl_limit);
         break;
     /* DMA transl. table limit */
     case 0x0020:
-        s->dma_tl_limit = val;
+        rc4030_dma_tt_update(s, s->dma_tl_base, val);
         break;
     /* DMA transl. table invalidated */
     case 0x0028:
@@ -371,9 +454,9 @@
             s->dma_regs[entry][idx] = val;
         }
         break;
-    /* Offset 0x0210 */
+    /* Memory refresh rate */
     case 0x0210:
-        s->offset210 = val;
+        s->memory_refresh_rate = val;
         break;
     /* Interval timer reload */
     case 0x0228:
@@ -385,48 +468,18 @@
     case 0x0238:
         break;
     default:
-        RC4030_ERROR("invalid write of 0x%02x at [" TARGET_FMT_plx "]\n", val, addr);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "rc4030: invalid write of 0x%02x at 0x%x",
+                      val, (int)addr);
         break;
     }
 }
 
-static void rc4030_writew(void *opaque, hwaddr addr, uint32_t val)
-{
-    uint32_t old_val = rc4030_readl(opaque, addr & ~0x3);
-
-    if (addr & 0x2)
-        val = (val << 16) | (old_val & 0x0000ffff);
-    else
-        val = val | (old_val & 0xffff0000);
-    rc4030_writel(opaque, addr & ~0x3, val);
-}
-
-static void rc4030_writeb(void *opaque, hwaddr addr, uint32_t val)
-{
-    uint32_t old_val = rc4030_readl(opaque, addr & ~0x3);
-
-    switch (addr & 3) {
-    case 0:
-        val = val | (old_val & 0xffffff00);
-        break;
-    case 1:
-        val = (val << 8) | (old_val & 0xffff00ff);
-        break;
-    case 2:
-        val = (val << 16) | (old_val & 0xff00ffff);
-        break;
-    case 3:
-        val = (val << 24) | (old_val & 0x00ffffff);
-        break;
-    }
-    rc4030_writel(opaque, addr & ~0x3, val);
-}
-
 static const MemoryRegionOps rc4030_ops = {
-    .old_mmio = {
-        .read = { rc4030_readb, rc4030_readw, rc4030_readl, },
-        .write = { rc4030_writeb, rc4030_writew, rc4030_writel, },
-    },
+    .read = rc4030_read,
+    .write = rc4030_write,
+    .impl.min_access_size = 4,
+    .impl.max_access_size = 4,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
@@ -436,22 +489,6 @@
 
     pending = s->isr_jazz & s->imr_jazz;
 
-#ifdef DEBUG_RC4030
-    if (s->isr_jazz != 0) {
-        uint32_t irq = 0;
-        DPRINTF("pending irqs:");
-        for (irq = 0; irq < ARRAY_SIZE(irq_names); irq++) {
-            if (s->isr_jazz & (1 << irq)) {
-                printf(" %s", irq_names[irq]);
-                if (!(s->imr_jazz & (1 << irq))) {
-                    printf("(ignored)");
-                }
-            }
-        }
-        printf("\n");
-    }
-#endif
-
     if (pending != 0)
         qemu_irq_raise(s->jazz_bus_irq);
     else
@@ -479,7 +516,7 @@
     qemu_irq_raise(s->timer_irq);
 }
 
-static uint32_t jazzio_readw(void *opaque, hwaddr addr)
+static uint64_t jazzio_read(void *opaque, hwaddr addr, unsigned int size)
 {
     rc4030State *s = opaque;
     uint32_t val;
@@ -494,7 +531,6 @@
         irq = 0;
         while (pending) {
             if (pending & 1) {
-                DPRINTF("returning irq %s\n", irq_names[irq]);
                 val = (irq + 1) << 2;
                 break;
             }
@@ -508,36 +544,25 @@
         val = s->imr_jazz;
         break;
     default:
-        RC4030_ERROR("(jazz io controller) invalid read [" TARGET_FMT_plx "]\n", addr);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "rc4030/jazzio: invalid read at 0x%x", (int)addr);
         val = 0;
+        break;
     }
 
-    DPRINTF("(jazz io controller) read 0x%04x at " TARGET_FMT_plx "\n", val, addr);
+    trace_jazzio_read(addr, val);
 
     return val;
 }
 
-static uint32_t jazzio_readb(void *opaque, hwaddr addr)
-{
-    uint32_t v;
-    v = jazzio_readw(opaque, addr & ~0x1);
-    return (v >> (8 * (addr & 0x1))) & 0xff;
-}
-
-static uint32_t jazzio_readl(void *opaque, hwaddr addr)
-{
-    uint32_t v;
-    v = jazzio_readw(opaque, addr);
-    v |= jazzio_readw(opaque, addr + 2) << 16;
-    return v;
-}
-
-static void jazzio_writew(void *opaque, hwaddr addr, uint32_t val)
+static void jazzio_write(void *opaque, hwaddr addr, uint64_t data,
+                         unsigned int size)
 {
     rc4030State *s = opaque;
+    uint32_t val = data;
     addr &= 0xfff;
 
-    DPRINTF("(jazz io controller) write 0x%04x at " TARGET_FMT_plx "\n", val, addr);
+    trace_jazzio_write(addr, val);
 
     switch (addr) {
     /* Local bus int enable mask */
@@ -546,43 +571,24 @@
         update_jazz_irq(s);
         break;
     default:
-        RC4030_ERROR("(jazz io controller) invalid write of 0x%04x at [" TARGET_FMT_plx "]\n", val, addr);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "rc4030/jazzio: invalid write of 0x%02x at 0x%x",
+                      val, (int)addr);
         break;
     }
 }
 
-static void jazzio_writeb(void *opaque, hwaddr addr, uint32_t val)
-{
-    uint32_t old_val = jazzio_readw(opaque, addr & ~0x1);
-
-    switch (addr & 1) {
-    case 0:
-        val = val | (old_val & 0xff00);
-        break;
-    case 1:
-        val = (val << 8) | (old_val & 0x00ff);
-        break;
-    }
-    jazzio_writew(opaque, addr & ~0x1, val);
-}
-
-static void jazzio_writel(void *opaque, hwaddr addr, uint32_t val)
-{
-    jazzio_writew(opaque, addr, val & 0xffff);
-    jazzio_writew(opaque, addr + 2, (val >> 16) & 0xffff);
-}
-
 static const MemoryRegionOps jazzio_ops = {
-    .old_mmio = {
-        .read = { jazzio_readb, jazzio_readw, jazzio_readl, },
-        .write = { jazzio_writeb, jazzio_writew, jazzio_writel, },
-    },
+    .read = jazzio_read,
+    .write = jazzio_write,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void rc4030_reset(void *opaque)
+static void rc4030_reset(DeviceState *dev)
 {
-    rc4030State *s = opaque;
+    rc4030State *s = RC4030(dev);
     int i;
 
     s->config = 0x410; /* some boards seem to accept 0x104 too */
@@ -590,14 +596,14 @@
     s->invalid_address_register = 0;
 
     memset(s->dma_regs, 0, sizeof(s->dma_regs));
-    s->dma_tl_base = s->dma_tl_limit = 0;
+    rc4030_dma_tt_update(s, 0, 0);
 
     s->remote_failed_address = s->memory_failed_address = 0;
     s->cache_maint = 0;
     s->cache_ptag = s->cache_ltag = 0;
     s->cache_bmask = 0;
 
-    s->offset210 = 0x18186;
+    s->memory_refresh_rate = 0x18186;
     s->nvram_protect = 7;
     for (i = 0; i < 15; i++)
         s->rem_speed[i] = 7;
@@ -631,7 +637,7 @@
     s->cache_ptag = qemu_get_be32(f);
     s->cache_ltag = qemu_get_be32(f);
     s->cache_bmask = qemu_get_be32(f);
-    s->offset210 = qemu_get_be32(f);
+    s->memory_refresh_rate = qemu_get_be32(f);
     s->nvram_protect = qemu_get_be32(f);
     for (i = 0; i < 15; i++)
         s->rem_speed[i] = qemu_get_be32(f);
@@ -663,7 +669,7 @@
     qemu_put_be32(f, s->cache_ptag);
     qemu_put_be32(f, s->cache_ltag);
     qemu_put_be32(f, s->cache_bmask);
-    qemu_put_be32(f, s->offset210);
+    qemu_put_be32(f, s->memory_refresh_rate);
     qemu_put_be32(f, s->nvram_protect);
     for (i = 0; i < 15; i++)
         qemu_put_be32(f, s->rem_speed[i]);
@@ -672,44 +678,6 @@
     qemu_put_be32(f, s->itr);
 }
 
-void rc4030_dma_memory_rw(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write)
-{
-    rc4030State *s = opaque;
-    hwaddr entry_addr;
-    hwaddr phys_addr;
-    dma_pagetable_entry entry;
-    int index;
-    int ncpy, i;
-
-    i = 0;
-    for (;;) {
-        if (i == len) {
-            break;
-        }
-
-        ncpy = DMA_PAGESIZE - (addr & (DMA_PAGESIZE - 1));
-        if (ncpy > len - i)
-            ncpy = len - i;
-
-        /* Get DMA translation table entry */
-        index = addr / DMA_PAGESIZE;
-        if (index >= s->dma_tl_limit / sizeof(dma_pagetable_entry)) {
-            break;
-        }
-        entry_addr = s->dma_tl_base + index * sizeof(dma_pagetable_entry);
-        /* XXX: not sure. should we really use only lowest bits? */
-        entry_addr &= 0x7fffffff;
-        cpu_physical_memory_read(entry_addr, &entry, sizeof(entry));
-
-        /* Read/write data at right place */
-        phys_addr = entry.frame + (addr & (DMA_PAGESIZE - 1));
-        cpu_physical_memory_rw(phys_addr, &buf[i], ncpy, is_write);
-
-        i += ncpy;
-        addr += ncpy;
-    }
-}
-
 static void rc4030_do_dma(void *opaque, int n, uint8_t *buf, int len, int is_write)
 {
     rc4030State *s = opaque;
@@ -733,32 +701,11 @@
     dma_addr = s->dma_regs[n][DMA_REG_ADDRESS];
 
     /* Read/write data at right place */
-    rc4030_dma_memory_rw(opaque, dma_addr, buf, len, is_write);
+    address_space_rw(&s->dma_as, dma_addr, MEMTXATTRS_UNSPECIFIED,
+                     buf, len, is_write);
 
     s->dma_regs[n][DMA_REG_ENABLE] |= DMA_FLAG_TC_INTR;
     s->dma_regs[n][DMA_REG_COUNT] -= len;
-
-#ifdef DEBUG_RC4030_DMA
-    {
-        int i, j;
-        printf("rc4030 dma: Copying %d bytes %s host %p\n",
-            len, is_write ? "from" : "to", buf);
-        for (i = 0; i < len; i += 16) {
-            int n = 16;
-            if (n > len - i) {
-                n = len - i;
-            }
-            for (j = 0; j < n; j++)
-                printf("%02x ", buf[i + j]);
-            while (j++ < 16)
-                printf("   ");
-            printf("| ");
-            for (j = 0; j < n; j++)
-                printf("%c", isprint(buf[i + j]) ? buf[i + j] : '.');
-            printf("\n");
-        }
-    }
-#endif
 }
 
 struct rc4030DMAState {
@@ -795,31 +742,102 @@
     return s;
 }
 
-void *rc4030_init(qemu_irq timer, qemu_irq jazz_bus,
-                  qemu_irq **irqs, rc4030_dma **dmas,
-                  MemoryRegion *sysmem)
+static void rc4030_initfn(Object *obj)
 {
-    rc4030State *s;
+    DeviceState *dev = DEVICE(obj);
+    rc4030State *s = RC4030(obj);
+    SysBusDevice *sysbus = SYS_BUS_DEVICE(obj);
 
-    s = g_malloc0(sizeof(rc4030State));
+    qdev_init_gpio_in(dev, rc4030_irq_jazz_request, 16);
 
-    *irqs = qemu_allocate_irqs(rc4030_irq_jazz_request, s, 16);
-    *dmas = rc4030_allocate_dmas(s, 4);
+    sysbus_init_irq(sysbus, &s->timer_irq);
+    sysbus_init_irq(sysbus, &s->jazz_bus_irq);
 
-    s->periodic_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, rc4030_periodic_timer, s);
-    s->timer_irq = timer;
-    s->jazz_bus_irq = jazz_bus;
-
-    qemu_register_reset(rc4030_reset, s);
     register_savevm(NULL, "rc4030", 0, 2, rc4030_save, rc4030_load, s);
-    rc4030_reset(s);
+
+    sysbus_init_mmio(sysbus, &s->iomem_chipset);
+    sysbus_init_mmio(sysbus, &s->iomem_jazzio);
+}
+
+static void rc4030_realize(DeviceState *dev, Error **errp)
+{
+    rc4030State *s = RC4030(dev);
+    Object *o = OBJECT(dev);
+    int i;
+
+    s->periodic_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                     rc4030_periodic_timer, s);
 
     memory_region_init_io(&s->iomem_chipset, NULL, &rc4030_ops, s,
                           "rc4030.chipset", 0x300);
-    memory_region_add_subregion(sysmem, 0x80000000, &s->iomem_chipset);
     memory_region_init_io(&s->iomem_jazzio, NULL, &jazzio_ops, s,
                           "rc4030.jazzio", 0x00001000);
-    memory_region_add_subregion(sysmem, 0xf0000000, &s->iomem_jazzio);
 
-    return s;
+    memory_region_init_rom_device(&s->dma_tt, o,
+                                  &rc4030_dma_tt_ops, s, "dma-table",
+                                  MAX_TL_ENTRIES * sizeof(dma_pagetable_entry),
+                                  NULL);
+    memory_region_init(&s->dma_tt_alias, o, "dma-table-alias", 0);
+    memory_region_init(&s->dma_mr, o, "dma", INT32_MAX);
+    for (i = 0; i < MAX_TL_ENTRIES; ++i) {
+        memory_region_init_alias(&s->dma_mrs[i], o, "dma-alias",
+                                 get_system_memory(), 0, DMA_PAGESIZE);
+        memory_region_set_enabled(&s->dma_mrs[i], false);
+        memory_region_add_subregion(&s->dma_mr, i * DMA_PAGESIZE,
+                                    &s->dma_mrs[i]);
+    }
+    address_space_init(&s->dma_as, &s->dma_mr, "rc4030-dma");
+}
+
+static void rc4030_unrealize(DeviceState *dev, Error **errp)
+{
+    rc4030State *s = RC4030(dev);
+    int i;
+
+    timer_free(s->periodic_timer);
+
+    address_space_destroy(&s->dma_as);
+    object_unparent(OBJECT(&s->dma_tt));
+    object_unparent(OBJECT(&s->dma_tt_alias));
+    object_unparent(OBJECT(&s->dma_mr));
+    for (i = 0; i < MAX_TL_ENTRIES; ++i) {
+        memory_region_del_subregion(&s->dma_mr, &s->dma_mrs[i]);
+        object_unparent(OBJECT(&s->dma_mrs[i]));
+    }
+}
+
+static void rc4030_class_init(ObjectClass *klass, void *class_data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = rc4030_realize;
+    dc->unrealize = rc4030_unrealize;
+    dc->reset = rc4030_reset;
+}
+
+static const TypeInfo rc4030_info = {
+    .name = TYPE_RC4030,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(rc4030State),
+    .instance_init = rc4030_initfn,
+    .class_init = rc4030_class_init,
+};
+
+static void rc4030_register_types(void)
+{
+    type_register_static(&rc4030_info);
+}
+
+type_init(rc4030_register_types)
+
+DeviceState *rc4030_init(rc4030_dma **dmas, MemoryRegion **dma_mr)
+{
+    DeviceState *dev;
+
+    dev = qdev_create(NULL, TYPE_RC4030);
+    qdev_init_nofail(dev);
+
+    *dmas = rc4030_allocate_dmas(dev, 4);
+    *dma_mr = &RC4030(dev)->dma_mr;
+    return dev;
 }
diff --git a/hw/dma/soc_dma.c b/hw/dma/soc_dma.c
index c06aabb..9bb499b 100644
--- a/hw/dma/soc_dma.c
+++ b/hw/dma/soc_dma.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/arm/soc_dma.h"
@@ -269,11 +270,10 @@
         if (entry->type == soc_dma_port_mem) {
             if (entry->addr <= virt_base &&
                             entry->addr + entry->u.mem.size > virt_base) {
-                fprintf(stderr, "%s: FIFO at " TARGET_FMT_lx
-                                " collides with RAM region at " TARGET_FMT_lx
-                                "-" TARGET_FMT_lx "\n", __FUNCTION__,
-                                (target_ulong) virt_base,
-                                (target_ulong) entry->addr, (target_ulong)
+                fprintf(stderr, "%s: FIFO at %"PRIx64
+                                " collides with RAM region at %"PRIx64
+                                "-%"PRIx64 "\n", __func__,
+                                virt_base, entry->addr,
                                 (entry->addr + entry->u.mem.size));
                 exit(-1);
             }
@@ -284,10 +284,9 @@
             while (entry < dma->memmap + dma->memmap_size &&
                             entry->addr <= virt_base) {
                 if (entry->addr == virt_base && entry->u.fifo.out == out) {
-                    fprintf(stderr, "%s: FIFO at " TARGET_FMT_lx
-                                    " collides FIFO at " TARGET_FMT_lx "\n",
-                                    __FUNCTION__, (target_ulong) virt_base,
-                                    (target_ulong) entry->addr);
+                    fprintf(stderr, "%s: FIFO at %"PRIx64
+                                    " collides FIFO at %"PRIx64 "\n",
+                                    __func__, virt_base, entry->addr);
                     exit(-1);
                 }
 
@@ -322,13 +321,11 @@
             if ((entry->addr >= virt_base && entry->addr < virt_base + size) ||
                             (entry->addr <= virt_base &&
                              entry->addr + entry->u.mem.size > virt_base)) {
-                fprintf(stderr, "%s: RAM at " TARGET_FMT_lx "-" TARGET_FMT_lx
-                                " collides with RAM region at " TARGET_FMT_lx
-                                "-" TARGET_FMT_lx "\n", __FUNCTION__,
-                                (target_ulong) virt_base,
-                                (target_ulong) (virt_base + size),
-                                (target_ulong) entry->addr, (target_ulong)
-                                (entry->addr + entry->u.mem.size));
+                fprintf(stderr, "%s: RAM at %"PRIx64 "-%"PRIx64
+                                " collides with RAM region at %"PRIx64
+                                "-%"PRIx64 "\n", __func__,
+                                virt_base, virt_base + size,
+                                entry->addr, entry->addr + entry->u.mem.size);
                 exit(-1);
             }
 
@@ -337,12 +334,11 @@
         } else {
             if (entry->addr >= virt_base &&
                             entry->addr < virt_base + size) {
-                fprintf(stderr, "%s: RAM at " TARGET_FMT_lx "-" TARGET_FMT_lx
-                                " collides with FIFO at " TARGET_FMT_lx
-                                "\n", __FUNCTION__,
-                                (target_ulong) virt_base,
-                                (target_ulong) (virt_base + size),
-                                (target_ulong) entry->addr);
+                fprintf(stderr, "%s: RAM at %"PRIx64 "-%"PRIx64
+                                " collides with FIFO at %"PRIx64
+                                "\n", __func__,
+                                virt_base, virt_base + size,
+                                entry->addr);
                 exit(-1);
             }
 
diff --git a/hw/dma/sparc32_dma.c b/hw/dma/sparc32_dma.c
index e6a453c..9d545e4 100644
--- a/hw/dma/sparc32_dma.c
+++ b/hw/dma/sparc32_dma.c
@@ -25,6 +25,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sparc/sparc32_dma.h"
 #include "hw/sparc/sun4m.h"
diff --git a/hw/dma/sun4m_iommu.c b/hw/dma/sun4m_iommu.c
index ec7c2ef..b3cbc54 100644
--- a/hw/dma/sun4m_iommu.c
+++ b/hw/dma/sun4m_iommu.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sparc/sun4m.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
@@ -263,7 +264,8 @@
     iopte = s->regs[IOMMU_BASE] << 4;
     addr &= ~s->iostart;
     iopte += (addr >> (IOMMU_PAGE_SHIFT - 2)) & ~3;
-    ret = ldl_be_phys(&address_space_memory, iopte);
+    ret = address_space_ldl_be(&address_space_memory, iopte,
+                               MEMTXATTRS_UNSPECIFIED, NULL);
     trace_sun4m_iommu_page_get_flags(pa, iopte, ret);
     return ret;
 }
diff --git a/hw/dma/trace-events b/hw/dma/trace-events
new file mode 100644
index 0000000..22878df
--- /dev/null
+++ b/hw/dma/trace-events
@@ -0,0 +1,32 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/dma/rc4030.c
+jazzio_read(uint64_t addr, uint32_t ret) "read reg[0x%"PRIx64"] = 0x%x"
+jazzio_write(uint64_t addr, uint32_t val) "write reg[0x%"PRIx64"] = 0x%x"
+rc4030_read(uint64_t addr, uint32_t ret) "read reg[0x%"PRIx64"] = 0x%x"
+rc4030_write(uint64_t addr, uint32_t val) "write reg[0x%"PRIx64"] = 0x%x"
+
+# hw/dma/sparc32_dma.c
+ledma_memory_read(uint64_t addr) "DMA read addr 0x%"PRIx64
+ledma_memory_write(uint64_t addr) "DMA write addr 0x%"PRIx64
+sparc32_dma_set_irq_raise(void) "Raise IRQ"
+sparc32_dma_set_irq_lower(void) "Lower IRQ"
+espdma_memory_read(uint32_t addr) "DMA read addr 0x%08x"
+espdma_memory_write(uint32_t addr) "DMA write addr 0x%08x"
+sparc32_dma_mem_readl(uint64_t addr, uint32_t ret) "read dmareg %"PRIx64": 0x%08x"
+sparc32_dma_mem_writel(uint64_t addr, uint32_t old, uint32_t val) "write dmareg %"PRIx64": 0x%08x -> 0x%08x"
+sparc32_dma_enable_raise(void) "Raise DMA enable"
+sparc32_dma_enable_lower(void) "Lower DMA enable"
+
+# hw/dma/sun4m_iommu.c
+sun4m_iommu_mem_readl(uint64_t addr, uint32_t ret) "read reg[%"PRIx64"] = %x"
+sun4m_iommu_mem_writel(uint64_t addr, uint32_t val) "write reg[%"PRIx64"] = %x"
+sun4m_iommu_mem_writel_ctrl(uint64_t iostart) "iostart = %"PRIx64
+sun4m_iommu_mem_writel_tlbflush(uint32_t val) "tlb flush %x"
+sun4m_iommu_mem_writel_pgflush(uint32_t val) "page flush %x"
+sun4m_iommu_page_get_flags(uint64_t pa, uint64_t iopte, uint32_t ret) "get flags addr %"PRIx64" => pte %"PRIx64", *pte = %x"
+sun4m_iommu_translate_pa(uint64_t addr, uint64_t pa, uint32_t iopte) "xlate dva %"PRIx64" => pa %"PRIx64" iopte = %x"
+sun4m_iommu_bad_addr(uint64_t addr) "bad addr %"PRIx64
+
+# hw/dma/i8257.c
+i8257_unregistered_dma(int nchan, int dma_pos, int dma_len) "unregistered DMA channel used nchan=%d dma_pos=%d dma_len=%d"
diff --git a/hw/dma/xilinx_axidma.c b/hw/dma/xilinx_axidma.c
index d06002d..a4753e5 100644
--- a/hw/dma/xilinx_axidma.c
+++ b/hw/dma/xilinx_axidma.c
@@ -22,11 +22,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
 #include "qemu/log.h"
-#include "qapi/qmp/qerror.h"
 #include "qemu/main-loop.h"
 
 #include "hw/stream.h"
@@ -134,7 +135,7 @@
 };
 
 /*
- * Helper calls to extract info from desriptors and other trivial
+ * Helper calls to extract info from descriptors and other trivial
  * state from regs.
  */
 static inline int stream_desc_sof(struct SDesc *d)
@@ -178,16 +179,6 @@
     return sid;
 }
 
-#ifdef DEBUG_ENET
-static void stream_desc_show(struct SDesc *d)
-{
-    qemu_log("buffer_addr  = " PRIx64 "\n", d->buffer_address);
-    qemu_log("nxtdesc      = " PRIx64 "\n", d->nxtdesc);
-    qemu_log("control      = %x\n", d->control);
-    qemu_log("status       = %x\n", d->status);
-}
-#endif
-
 static void stream_desc_load(struct Stream *s, hwaddr addr)
 {
     struct SDesc *d = &s->desc;
diff --git a/hw/dma/xlnx-zynq-devcfg.c b/hw/dma/xlnx-zynq-devcfg.c
new file mode 100644
index 0000000..3b10523
--- /dev/null
+++ b/hw/dma/xlnx-zynq-devcfg.c
@@ -0,0 +1,400 @@
+/*
+ * QEMU model of the Xilinx Zynq Devcfg Interface
+ *
+ * (C) 2011 PetaLogix Pty Ltd
+ * (C) 2014 Xilinx Inc.
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/dma/xlnx-zynq-devcfg.h"
+#include "qemu/bitops.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
+#include "qemu/log.h"
+
+#define FREQ_HZ 900000000
+
+#define BTT_MAX 0x400
+
+#ifndef XLNX_ZYNQ_DEVCFG_ERR_DEBUG
+#define XLNX_ZYNQ_DEVCFG_ERR_DEBUG 0
+#endif
+
+#define DB_PRINT(fmt, args...) do { \
+    if (XLNX_ZYNQ_DEVCFG_ERR_DEBUG) { \
+        qemu_log("%s: " fmt, __func__, ## args); \
+    } \
+} while (0);
+
+REG32(CTRL, 0x00)
+    FIELD(CTRL,     FORCE_RST,          31,  1) /* Not supported, wr ignored */
+    FIELD(CTRL,     PCAP_PR,            27,  1) /* Forced to 0 on bad unlock */
+    FIELD(CTRL,     PCAP_MODE,          26,  1)
+    FIELD(CTRL,     MULTIBOOT_EN,       24,  1)
+    FIELD(CTRL,     USER_MODE,          15,  1)
+    FIELD(CTRL,     PCFG_AES_FUSE,      12,  1)
+    FIELD(CTRL,     PCFG_AES_EN,         9,  3)
+    FIELD(CTRL,     SEU_EN,              8,  1)
+    FIELD(CTRL,     SEC_EN,              7,  1)
+    FIELD(CTRL,     SPNIDEN,             6,  1)
+    FIELD(CTRL,     SPIDEN,              5,  1)
+    FIELD(CTRL,     NIDEN,               4,  1)
+    FIELD(CTRL,     DBGEN,               3,  1)
+    FIELD(CTRL,     DAP_EN,              0,  3)
+
+REG32(LOCK, 0x04)
+#define AES_FUSE_LOCK        4
+#define AES_EN_LOCK          3
+#define SEU_LOCK             2
+#define SEC_LOCK             1
+#define DBG_LOCK             0
+
+/* mapping bits in R_LOCK to what they lock in R_CTRL */
+static const uint32_t lock_ctrl_map[] = {
+    [AES_FUSE_LOCK] = R_CTRL_PCFG_AES_FUSE_MASK,
+    [AES_EN_LOCK]   = R_CTRL_PCFG_AES_EN_MASK,
+    [SEU_LOCK]      = R_CTRL_SEU_EN_MASK,
+    [SEC_LOCK]      = R_CTRL_SEC_EN_MASK,
+    [DBG_LOCK]      = R_CTRL_SPNIDEN_MASK | R_CTRL_SPIDEN_MASK |
+                      R_CTRL_NIDEN_MASK   | R_CTRL_DBGEN_MASK  |
+                      R_CTRL_DAP_EN_MASK,
+};
+
+REG32(CFG, 0x08)
+    FIELD(CFG,      RFIFO_TH,           10,  2)
+    FIELD(CFG,      WFIFO_TH,            8,  2)
+    FIELD(CFG,      RCLK_EDGE,           7,  1)
+    FIELD(CFG,      WCLK_EDGE,           6,  1)
+    FIELD(CFG,      DISABLE_SRC_INC,     5,  1)
+    FIELD(CFG,      DISABLE_DST_INC,     4,  1)
+#define R_CFG_RESET 0x50B
+
+REG32(INT_STS, 0x0C)
+    FIELD(INT_STS,  PSS_GTS_USR_B,      31,  1)
+    FIELD(INT_STS,  PSS_FST_CFG_B,      30,  1)
+    FIELD(INT_STS,  PSS_CFG_RESET_B,    27,  1)
+    FIELD(INT_STS,  RX_FIFO_OV,         18,  1)
+    FIELD(INT_STS,  WR_FIFO_LVL,        17,  1)
+    FIELD(INT_STS,  RD_FIFO_LVL,        16,  1)
+    FIELD(INT_STS,  DMA_CMD_ERR,        15,  1)
+    FIELD(INT_STS,  DMA_Q_OV,           14,  1)
+    FIELD(INT_STS,  DMA_DONE,           13,  1)
+    FIELD(INT_STS,  DMA_P_DONE,         12,  1)
+    FIELD(INT_STS,  P2D_LEN_ERR,        11,  1)
+    FIELD(INT_STS,  PCFG_DONE,           2,  1)
+#define R_INT_STS_RSVD       ((0x7 << 24) | (0x1 << 19) | (0xF < 7))
+
+REG32(INT_MASK, 0x10)
+
+REG32(STATUS, 0x14)
+    FIELD(STATUS,   DMA_CMD_Q_F,        31,  1)
+    FIELD(STATUS,   DMA_CMD_Q_E,        30,  1)
+    FIELD(STATUS,   DMA_DONE_CNT,       28,  2)
+    FIELD(STATUS,   RX_FIFO_LVL,        20,  5)
+    FIELD(STATUS,   TX_FIFO_LVL,        12,  7)
+    FIELD(STATUS,   PSS_GTS_USR_B,      11,  1)
+    FIELD(STATUS,   PSS_FST_CFG_B,      10,  1)
+    FIELD(STATUS,   PSS_CFG_RESET_B,     5,  1)
+
+REG32(DMA_SRC_ADDR, 0x18)
+REG32(DMA_DST_ADDR, 0x1C)
+REG32(DMA_SRC_LEN, 0x20)
+REG32(DMA_DST_LEN, 0x24)
+REG32(ROM_SHADOW, 0x28)
+REG32(SW_ID, 0x30)
+REG32(UNLOCK, 0x34)
+
+#define R_UNLOCK_MAGIC 0x757BDF0D
+
+REG32(MCTRL, 0x80)
+    FIELD(MCTRL,    PS_VERSION,         28,  4)
+    FIELD(MCTRL,    PCFG_POR_B,          8,  1)
+    FIELD(MCTRL,    INT_PCAP_LPBK,       4,  1)
+    FIELD(MCTRL,    QEMU,                3,  1)
+
+static void xlnx_zynq_devcfg_update_ixr(XlnxZynqDevcfg *s)
+{
+    qemu_set_irq(s->irq, ~s->regs[R_INT_MASK] & s->regs[R_INT_STS]);
+}
+
+static void xlnx_zynq_devcfg_reset(DeviceState *dev)
+{
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(dev);
+    int i;
+
+    for (i = 0; i < XLNX_ZYNQ_DEVCFG_R_MAX; ++i) {
+        register_reset(&s->regs_info[i]);
+    }
+}
+
+static void xlnx_zynq_devcfg_dma_go(XlnxZynqDevcfg *s)
+{
+    do {
+        uint8_t buf[BTT_MAX];
+        XlnxZynqDevcfgDMACmd *dmah = s->dma_cmd_fifo;
+        uint32_t btt = BTT_MAX;
+        bool loopback = s->regs[R_MCTRL] & R_MCTRL_INT_PCAP_LPBK_MASK;
+
+        btt = MIN(btt, dmah->src_len);
+        if (loopback) {
+            btt = MIN(btt, dmah->dest_len);
+        }
+        DB_PRINT("reading %x bytes from %x\n", btt, dmah->src_addr);
+        dma_memory_read(&address_space_memory, dmah->src_addr, buf, btt);
+        dmah->src_len -= btt;
+        dmah->src_addr += btt;
+        if (loopback && (dmah->src_len || dmah->dest_len)) {
+            DB_PRINT("writing %x bytes from %x\n", btt, dmah->dest_addr);
+            dma_memory_write(&address_space_memory, dmah->dest_addr, buf, btt);
+            dmah->dest_len -= btt;
+            dmah->dest_addr += btt;
+        }
+        if (!dmah->src_len && !dmah->dest_len) {
+            DB_PRINT("dma operation finished\n");
+            s->regs[R_INT_STS] |= R_INT_STS_DMA_DONE_MASK |
+                                  R_INT_STS_DMA_P_DONE_MASK;
+            s->dma_cmd_fifo_num--;
+            memmove(s->dma_cmd_fifo, &s->dma_cmd_fifo[1],
+                    sizeof(s->dma_cmd_fifo) - sizeof(s->dma_cmd_fifo[0]));
+        }
+        xlnx_zynq_devcfg_update_ixr(s);
+    } while (s->dma_cmd_fifo_num);
+}
+
+static void r_ixr_post_write(RegisterInfo *reg, uint64_t val)
+{
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(reg->opaque);
+
+    xlnx_zynq_devcfg_update_ixr(s);
+}
+
+static uint64_t r_ctrl_pre_write(RegisterInfo *reg, uint64_t val)
+{
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(reg->opaque);
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(lock_ctrl_map); ++i) {
+        if (s->regs[R_LOCK] & 1 << i) {
+            val &= ~lock_ctrl_map[i];
+            val |= lock_ctrl_map[i] & s->regs[R_CTRL];
+        }
+    }
+    return val;
+}
+
+static void r_ctrl_post_write(RegisterInfo *reg, uint64_t val)
+{
+    const char *device_prefix = object_get_typename(OBJECT(reg->opaque));
+    uint32_t aes_en = FIELD_EX32(val, CTRL, PCFG_AES_EN);
+
+    if (aes_en != 0 && aes_en != 7) {
+        qemu_log_mask(LOG_UNIMP, "%s: warning, aes-en bits inconsistent,"
+                      "unimplemented security reset should happen!\n",
+                      device_prefix);
+    }
+}
+
+static void r_unlock_post_write(RegisterInfo *reg, uint64_t val)
+{
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(reg->opaque);
+    const char *device_prefix = object_get_typename(OBJECT(s));
+
+    if (val == R_UNLOCK_MAGIC) {
+        DB_PRINT("successful unlock\n");
+        s->regs[R_CTRL] |= R_CTRL_PCAP_PR_MASK;
+        s->regs[R_CTRL] |= R_CTRL_PCFG_AES_EN_MASK;
+        memory_region_set_enabled(&s->iomem, true);
+    } else { /* bad unlock attempt */
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: failed unlock\n", device_prefix);
+        s->regs[R_CTRL] &= ~R_CTRL_PCAP_PR_MASK;
+        s->regs[R_CTRL] &= ~R_CTRL_PCFG_AES_EN_MASK;
+        /* core becomes inaccessible */
+        memory_region_set_enabled(&s->iomem, false);
+    }
+}
+
+static uint64_t r_lock_pre_write(RegisterInfo *reg, uint64_t val)
+{
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(reg->opaque);
+
+    /* once bits are locked they stay locked */
+    return s->regs[R_LOCK] | val;
+}
+
+static void r_dma_dst_len_post_write(RegisterInfo *reg, uint64_t val)
+{
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(reg->opaque);
+
+    s->dma_cmd_fifo[s->dma_cmd_fifo_num] = (XlnxZynqDevcfgDMACmd) {
+            .src_addr = s->regs[R_DMA_SRC_ADDR] & ~0x3UL,
+            .dest_addr = s->regs[R_DMA_DST_ADDR] & ~0x3UL,
+            .src_len = s->regs[R_DMA_SRC_LEN] << 2,
+            .dest_len = s->regs[R_DMA_DST_LEN] << 2,
+    };
+    s->dma_cmd_fifo_num++;
+    DB_PRINT("dma transfer started; %d total transfers pending\n",
+             s->dma_cmd_fifo_num);
+    xlnx_zynq_devcfg_dma_go(s);
+}
+
+static const RegisterAccessInfo xlnx_zynq_devcfg_regs_info[] = {
+    {   .name = "CTRL",                 .addr = A_CTRL,
+        .reset = R_CTRL_PCAP_PR_MASK | R_CTRL_PCAP_MODE_MASK | 0x3 << 13,
+        .rsvd = 0x1 << 28 | 0x3ff << 13 | 0x3 << 13,
+        .pre_write = r_ctrl_pre_write,
+        .post_write = r_ctrl_post_write,
+    },
+    {   .name = "LOCK",                 .addr = A_LOCK,
+        .rsvd = MAKE_64BIT_MASK(5, 64 - 5),
+        .pre_write = r_lock_pre_write,
+    },
+    {   .name = "CFG",                  .addr = A_CFG,
+        .reset = R_CFG_RESET,
+        .rsvd = 0xfffff00f,
+    },
+    {   .name = "INT_STS",              .addr = A_INT_STS,
+        .w1c = ~R_INT_STS_RSVD,
+        .reset = R_INT_STS_PSS_GTS_USR_B_MASK   |
+                 R_INT_STS_PSS_CFG_RESET_B_MASK |
+                 R_INT_STS_WR_FIFO_LVL_MASK,
+        .rsvd = R_INT_STS_RSVD,
+        .post_write = r_ixr_post_write,
+    },
+    {   .name = "INT_MASK",            .addr = A_INT_MASK,
+        .reset = ~0,
+        .rsvd = R_INT_STS_RSVD,
+        .post_write = r_ixr_post_write,
+    },
+    {   .name = "STATUS",               .addr = A_STATUS,
+        .reset = R_STATUS_DMA_CMD_Q_E_MASK      |
+                 R_STATUS_PSS_GTS_USR_B_MASK    |
+                 R_STATUS_PSS_CFG_RESET_B_MASK,
+        .ro = ~0,
+    },
+    {   .name = "DMA_SRC_ADDR",         .addr = A_DMA_SRC_ADDR, },
+    {   .name = "DMA_DST_ADDR",         .addr = A_DMA_DST_ADDR, },
+    {   .name = "DMA_SRC_LEN",          .addr = A_DMA_SRC_LEN,
+        .ro = MAKE_64BIT_MASK(27, 64 - 27) },
+    {   .name = "DMA_DST_LEN",          .addr = A_DMA_DST_LEN,
+        .ro = MAKE_64BIT_MASK(27, 64 - 27),
+        .post_write = r_dma_dst_len_post_write,
+    },
+    {   .name = "ROM_SHADOW",           .addr = A_ROM_SHADOW,
+        .rsvd = ~0ull,
+    },
+    {   .name = "SW_ID",                .addr = A_SW_ID, },
+    {   .name = "UNLOCK",               .addr = A_UNLOCK,
+        .post_write = r_unlock_post_write,
+    },
+    {   .name = "MCTRL",                .addr = R_MCTRL * 4,
+       /* Silicon 3.0 for version field, the mysterious reserved bit 23
+        * and QEMU platform identifier.
+        */
+       .reset = 0x2 << R_MCTRL_PS_VERSION_SHIFT | 1 << 23 | R_MCTRL_QEMU_MASK,
+       .ro = ~R_MCTRL_INT_PCAP_LPBK_MASK,
+       .rsvd = 0x00f00303,
+    },
+};
+
+static const MemoryRegionOps xlnx_zynq_devcfg_reg_ops = {
+    .read = register_read_memory,
+    .write = register_write_memory,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static const VMStateDescription vmstate_xlnx_zynq_devcfg_dma_cmd = {
+    .name = "xlnx_zynq_devcfg_dma_cmd",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(src_addr, XlnxZynqDevcfgDMACmd),
+        VMSTATE_UINT32(dest_addr, XlnxZynqDevcfgDMACmd),
+        VMSTATE_UINT32(src_len, XlnxZynqDevcfgDMACmd),
+        VMSTATE_UINT32(dest_len, XlnxZynqDevcfgDMACmd),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_xlnx_zynq_devcfg = {
+    .name = "xlnx_zynq_devcfg",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_ARRAY(dma_cmd_fifo, XlnxZynqDevcfg,
+                             XLNX_ZYNQ_DEVCFG_DMA_CMD_FIFO_LEN, 0,
+                             vmstate_xlnx_zynq_devcfg_dma_cmd,
+                             XlnxZynqDevcfgDMACmd),
+        VMSTATE_UINT8(dma_cmd_fifo_num, XlnxZynqDevcfg),
+        VMSTATE_UINT32_ARRAY(regs, XlnxZynqDevcfg, XLNX_ZYNQ_DEVCFG_R_MAX),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void xlnx_zynq_devcfg_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    XlnxZynqDevcfg *s = XLNX_ZYNQ_DEVCFG(obj);
+    RegisterInfoArray *reg_array;
+
+    sysbus_init_irq(sbd, &s->irq);
+
+    memory_region_init(&s->iomem, obj, "devcfg", XLNX_ZYNQ_DEVCFG_R_MAX * 4);
+    reg_array =
+        register_init_block32(DEVICE(obj), xlnx_zynq_devcfg_regs_info,
+                              ARRAY_SIZE(xlnx_zynq_devcfg_regs_info),
+                              s->regs_info, s->regs,
+                              &xlnx_zynq_devcfg_reg_ops,
+                              XLNX_ZYNQ_DEVCFG_ERR_DEBUG,
+                              XLNX_ZYNQ_DEVCFG_R_MAX);
+    memory_region_add_subregion(&s->iomem,
+                                A_CTRL,
+                                &reg_array->mem);
+
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static void xlnx_zynq_devcfg_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = xlnx_zynq_devcfg_reset;
+    dc->vmsd = &vmstate_xlnx_zynq_devcfg;
+}
+
+static const TypeInfo xlnx_zynq_devcfg_info = {
+    .name           = TYPE_XLNX_ZYNQ_DEVCFG,
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(XlnxZynqDevcfg),
+    .instance_init  = xlnx_zynq_devcfg_init,
+    .class_init     = xlnx_zynq_devcfg_class_init,
+};
+
+static void xlnx_zynq_devcfg_register_types(void)
+{
+    type_register_static(&xlnx_zynq_devcfg_info);
+}
+
+type_init(xlnx_zynq_devcfg_register_types)
diff --git a/hw/dma/xlnx_dpdma.c b/hw/dma/xlnx_dpdma.c
new file mode 100644
index 0000000..8ceb21d
--- /dev/null
+++ b/hw/dma/xlnx_dpdma.c
@@ -0,0 +1,786 @@
+/*
+ * xlnx_dpdma.c
+ *
+ *  Copyright (C) 2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "hw/dma/xlnx_dpdma.h"
+
+#ifndef DEBUG_DPDMA
+#define DEBUG_DPDMA 0
+#endif
+
+#define DPRINTF(fmt, ...) do {                                                 \
+    if (DEBUG_DPDMA) {                                                         \
+        qemu_log("xlnx_dpdma: " fmt , ## __VA_ARGS__);                         \
+    }                                                                          \
+} while (0);
+
+/*
+ * Registers offset for DPDMA.
+ */
+#define DPDMA_ERR_CTRL                        (0x0000)
+#define DPDMA_ISR                             (0x0004 >> 2)
+#define DPDMA_IMR                             (0x0008 >> 2)
+#define DPDMA_IEN                             (0x000C >> 2)
+#define DPDMA_IDS                             (0x0010 >> 2)
+#define DPDMA_EISR                            (0x0014 >> 2)
+#define DPDMA_EIMR                            (0x0018 >> 2)
+#define DPDMA_EIEN                            (0x001C >> 2)
+#define DPDMA_EIDS                            (0x0020 >> 2)
+#define DPDMA_CNTL                            (0x0100 >> 2)
+
+#define DPDMA_GBL                             (0x0104 >> 2)
+#define DPDMA_GBL_TRG_CH(n)                   (1 << n)
+#define DPDMA_GBL_RTRG_CH(n)                  (1 << 6 << n)
+
+#define DPDMA_ALC0_CNTL                       (0x0108 >> 2)
+#define DPDMA_ALC0_STATUS                     (0x010C >> 2)
+#define DPDMA_ALC0_MAX                        (0x0110 >> 2)
+#define DPDMA_ALC0_MIN                        (0x0114 >> 2)
+#define DPDMA_ALC0_ACC                        (0x0118 >> 2)
+#define DPDMA_ALC0_ACC_TRAN                   (0x011C >> 2)
+#define DPDMA_ALC1_CNTL                       (0x0120 >> 2)
+#define DPDMA_ALC1_STATUS                     (0x0124 >> 2)
+#define DPDMA_ALC1_MAX                        (0x0128 >> 2)
+#define DPDMA_ALC1_MIN                        (0x012C >> 2)
+#define DPDMA_ALC1_ACC                        (0x0130 >> 2)
+#define DPDMA_ALC1_ACC_TRAN                   (0x0134 >> 2)
+
+#define DPDMA_DSCR_STRT_ADDRE_CH(n)           ((0x0200 + n * 0x100) >> 2)
+#define DPDMA_DSCR_STRT_ADDR_CH(n)            ((0x0204 + n * 0x100) >> 2)
+#define DPDMA_DSCR_NEXT_ADDRE_CH(n)           ((0x0208 + n * 0x100) >> 2)
+#define DPDMA_DSCR_NEXT_ADDR_CH(n)            ((0x020C + n * 0x100) >> 2)
+#define DPDMA_PYLD_CUR_ADDRE_CH(n)            ((0x0210 + n * 0x100) >> 2)
+#define DPDMA_PYLD_CUR_ADDR_CH(n)             ((0x0214 + n * 0x100) >> 2)
+
+#define DPDMA_CNTL_CH(n)                      ((0x0218 + n * 0x100) >> 2)
+#define DPDMA_CNTL_CH_EN                      (1)
+#define DPDMA_CNTL_CH_PAUSED                  (1 << 1)
+
+#define DPDMA_STATUS_CH(n)                    ((0x021C + n * 0x100) >> 2)
+#define DPDMA_STATUS_BURST_TYPE               (1 << 4)
+#define DPDMA_STATUS_MODE                     (1 << 5)
+#define DPDMA_STATUS_EN_CRC                   (1 << 6)
+#define DPDMA_STATUS_LAST_DSCR                (1 << 7)
+#define DPDMA_STATUS_LDSCR_FRAME              (1 << 8)
+#define DPDMA_STATUS_IGNR_DONE                (1 << 9)
+#define DPDMA_STATUS_DSCR_DONE                (1 << 10)
+#define DPDMA_STATUS_EN_DSCR_UP               (1 << 11)
+#define DPDMA_STATUS_EN_DSCR_INTR             (1 << 12)
+#define DPDMA_STATUS_PREAMBLE_OFF             (13)
+
+#define DPDMA_VDO_CH(n)                       ((0x0220 + n * 0x100) >> 2)
+#define DPDMA_PYLD_SZ_CH(n)                   ((0x0224 + n * 0x100) >> 2)
+#define DPDMA_DSCR_ID_CH(n)                   ((0x0228 + n * 0x100) >> 2)
+
+/*
+ * Descriptor control field.
+ */
+#define CONTROL_PREAMBLE_VALUE                0xA5
+
+#define DSCR_CTRL_PREAMBLE                    0xFF
+#define DSCR_CTRL_EN_DSCR_DONE_INTR           (1 << 8)
+#define DSCR_CTRL_EN_DSCR_UPDATE              (1 << 9)
+#define DSCR_CTRL_IGNORE_DONE                 (1 << 10)
+#define DSCR_CTRL_AXI_BURST_TYPE              (1 << 11)
+#define DSCR_CTRL_AXCACHE                     (0x0F << 12)
+#define DSCR_CTRL_AXPROT                      (0x2 << 16)
+#define DSCR_CTRL_DESCRIPTOR_MODE             (1 << 18)
+#define DSCR_CTRL_LAST_DESCRIPTOR             (1 << 19)
+#define DSCR_CTRL_ENABLE_CRC                  (1 << 20)
+#define DSCR_CTRL_LAST_DESCRIPTOR_OF_FRAME    (1 << 21)
+
+/*
+ * Descriptor timestamp field.
+ */
+#define STATUS_DONE                           (1 << 31)
+
+#define DPDMA_FRAG_MAX_SZ                     (4096)
+
+enum DPDMABurstType {
+    DPDMA_INCR = 0,
+    DPDMA_FIXED = 1
+};
+
+enum DPDMAMode {
+    DPDMA_CONTIGOUS = 0,
+    DPDMA_FRAGMENTED = 1
+};
+
+struct DPDMADescriptor {
+    uint32_t control;
+    uint32_t descriptor_id;
+    /* transfer size in byte. */
+    uint32_t xfer_size;
+    uint32_t line_size_stride;
+    uint32_t timestamp_lsb;
+    uint32_t timestamp_msb;
+    /* contains extension for both descriptor and source. */
+    uint32_t address_extension;
+    uint32_t next_descriptor;
+    uint32_t source_address;
+    uint32_t address_extension_23;
+    uint32_t address_extension_45;
+    uint32_t source_address2;
+    uint32_t source_address3;
+    uint32_t source_address4;
+    uint32_t source_address5;
+    uint32_t crc;
+};
+
+typedef enum DPDMABurstType DPDMABurstType;
+typedef enum DPDMAMode DPDMAMode;
+typedef struct DPDMADescriptor DPDMADescriptor;
+
+static bool xlnx_dpdma_desc_is_last(DPDMADescriptor *desc)
+{
+    return ((desc->control & DSCR_CTRL_LAST_DESCRIPTOR) != 0);
+}
+
+static bool xlnx_dpdma_desc_is_last_of_frame(DPDMADescriptor *desc)
+{
+    return ((desc->control & DSCR_CTRL_LAST_DESCRIPTOR_OF_FRAME) != 0);
+}
+
+static uint64_t xlnx_dpdma_desc_get_source_address(DPDMADescriptor *desc,
+                                                     uint8_t frag)
+{
+    uint64_t addr = 0;
+    assert(frag < 5);
+
+    switch (frag) {
+    case 0:
+        addr = desc->source_address
+            + (extract32(desc->address_extension, 16, 12) << 20);
+        break;
+    case 1:
+        addr = desc->source_address2
+            + (extract32(desc->address_extension_23, 0, 12) << 8);
+        break;
+    case 2:
+        addr = desc->source_address3
+            + (extract32(desc->address_extension_23, 16, 12) << 20);
+        break;
+    case 3:
+        addr = desc->source_address4
+            + (extract32(desc->address_extension_45, 0, 12) << 8);
+        break;
+    case 4:
+        addr = desc->source_address5
+            + (extract32(desc->address_extension_45, 16, 12) << 20);
+        break;
+    default:
+        addr = 0;
+        break;
+    }
+
+    return addr;
+}
+
+static uint32_t xlnx_dpdma_desc_get_transfer_size(DPDMADescriptor *desc)
+{
+    return desc->xfer_size;
+}
+
+static uint32_t xlnx_dpdma_desc_get_line_size(DPDMADescriptor *desc)
+{
+    return extract32(desc->line_size_stride, 0, 18);
+}
+
+static uint32_t xlnx_dpdma_desc_get_line_stride(DPDMADescriptor *desc)
+{
+    return extract32(desc->line_size_stride, 18, 14) * 16;
+}
+
+static inline bool xlnx_dpdma_desc_crc_enabled(DPDMADescriptor *desc)
+{
+    return (desc->control & DSCR_CTRL_ENABLE_CRC) != 0;
+}
+
+static inline bool xlnx_dpdma_desc_check_crc(DPDMADescriptor *desc)
+{
+    uint32_t *p = (uint32_t *)desc;
+    uint32_t crc = 0;
+    uint8_t i;
+
+    /*
+     * CRC is calculated on the whole descriptor except the last 32bits word
+     * using 32bits addition.
+     */
+    for (i = 0; i < 15; i++) {
+        crc += p[i];
+    }
+
+    return crc == desc->crc;
+}
+
+static inline bool xlnx_dpdma_desc_completion_interrupt(DPDMADescriptor *desc)
+{
+    return (desc->control & DSCR_CTRL_EN_DSCR_DONE_INTR) != 0;
+}
+
+static inline bool xlnx_dpdma_desc_is_valid(DPDMADescriptor *desc)
+{
+    return (desc->control & DSCR_CTRL_PREAMBLE) == CONTROL_PREAMBLE_VALUE;
+}
+
+static inline bool xlnx_dpdma_desc_is_contiguous(DPDMADescriptor *desc)
+{
+    return (desc->control & DSCR_CTRL_DESCRIPTOR_MODE) == 0;
+}
+
+static inline bool xlnx_dpdma_desc_update_enabled(DPDMADescriptor *desc)
+{
+    return (desc->control & DSCR_CTRL_EN_DSCR_UPDATE) != 0;
+}
+
+static inline void xlnx_dpdma_desc_set_done(DPDMADescriptor *desc)
+{
+    desc->timestamp_msb |= STATUS_DONE;
+}
+
+static inline bool xlnx_dpdma_desc_is_already_done(DPDMADescriptor *desc)
+{
+    return (desc->timestamp_msb & STATUS_DONE) != 0;
+}
+
+static inline bool xlnx_dpdma_desc_ignore_done_bit(DPDMADescriptor *desc)
+{
+    return (desc->control & DSCR_CTRL_IGNORE_DONE) != 0;
+}
+
+static const VMStateDescription vmstate_xlnx_dpdma = {
+    .name = TYPE_XLNX_DPDMA,
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(registers, XlnxDPDMAState,
+                             XLNX_DPDMA_REG_ARRAY_SIZE),
+        VMSTATE_BOOL_ARRAY(operation_finished, XlnxDPDMAState, 6),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void xlnx_dpdma_update_irq(XlnxDPDMAState *s)
+{
+    bool flags;
+
+    flags = ((s->registers[DPDMA_ISR] & (~s->registers[DPDMA_IMR]))
+          || (s->registers[DPDMA_EISR] & (~s->registers[DPDMA_EIMR])));
+    qemu_set_irq(s->irq, flags);
+}
+
+static uint64_t xlnx_dpdma_descriptor_start_address(XlnxDPDMAState *s,
+                                                      uint8_t channel)
+{
+    return (s->registers[DPDMA_DSCR_STRT_ADDRE_CH(channel)] << 16)
+          + s->registers[DPDMA_DSCR_STRT_ADDR_CH(channel)];
+}
+
+static uint64_t xlnx_dpdma_descriptor_next_address(XlnxDPDMAState *s,
+                                                     uint8_t channel)
+{
+    return ((uint64_t)s->registers[DPDMA_DSCR_NEXT_ADDRE_CH(channel)] << 32)
+           + s->registers[DPDMA_DSCR_NEXT_ADDR_CH(channel)];
+}
+
+static bool xlnx_dpdma_is_channel_enabled(XlnxDPDMAState *s,
+                                            uint8_t channel)
+{
+    return (s->registers[DPDMA_CNTL_CH(channel)] & DPDMA_CNTL_CH_EN) != 0;
+}
+
+static bool xlnx_dpdma_is_channel_paused(XlnxDPDMAState *s,
+                                           uint8_t channel)
+{
+    return (s->registers[DPDMA_CNTL_CH(channel)] & DPDMA_CNTL_CH_PAUSED) != 0;
+}
+
+static inline bool xlnx_dpdma_is_channel_retriggered(XlnxDPDMAState *s,
+                                                       uint8_t channel)
+{
+    /* Clear the retriggered bit after reading it. */
+    bool channel_is_retriggered = s->registers[DPDMA_GBL]
+                                & DPDMA_GBL_RTRG_CH(channel);
+    s->registers[DPDMA_GBL] &= ~DPDMA_GBL_RTRG_CH(channel);
+    return channel_is_retriggered;
+}
+
+static inline bool xlnx_dpdma_is_channel_triggered(XlnxDPDMAState *s,
+                                                     uint8_t channel)
+{
+    return s->registers[DPDMA_GBL] & DPDMA_GBL_TRG_CH(channel);
+}
+
+static void xlnx_dpdma_update_desc_info(XlnxDPDMAState *s, uint8_t channel,
+                                          DPDMADescriptor *desc)
+{
+    s->registers[DPDMA_DSCR_NEXT_ADDRE_CH(channel)] =
+                                extract32(desc->address_extension, 0, 16);
+    s->registers[DPDMA_DSCR_NEXT_ADDR_CH(channel)] = desc->next_descriptor;
+    s->registers[DPDMA_PYLD_CUR_ADDRE_CH(channel)] =
+                                extract32(desc->address_extension, 16, 16);
+    s->registers[DPDMA_PYLD_CUR_ADDR_CH(channel)] = desc->source_address;
+    s->registers[DPDMA_VDO_CH(channel)] =
+                                extract32(desc->line_size_stride, 18, 14)
+                                + (extract32(desc->line_size_stride, 0, 18)
+                                  << 14);
+    s->registers[DPDMA_PYLD_SZ_CH(channel)] = desc->xfer_size;
+    s->registers[DPDMA_DSCR_ID_CH(channel)] = desc->descriptor_id;
+
+    /* Compute the status register with the descriptor information. */
+    s->registers[DPDMA_STATUS_CH(channel)] =
+                                extract32(desc->control, 0, 8) << 13;
+    if ((desc->control & DSCR_CTRL_EN_DSCR_DONE_INTR) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_EN_DSCR_INTR;
+    }
+    if ((desc->control & DSCR_CTRL_EN_DSCR_UPDATE) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_EN_DSCR_UP;
+    }
+    if ((desc->timestamp_msb & STATUS_DONE) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_DSCR_DONE;
+    }
+    if ((desc->control & DSCR_CTRL_IGNORE_DONE) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_IGNR_DONE;
+    }
+    if ((desc->control & DSCR_CTRL_LAST_DESCRIPTOR_OF_FRAME) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_LDSCR_FRAME;
+    }
+    if ((desc->control & DSCR_CTRL_LAST_DESCRIPTOR) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_LAST_DSCR;
+    }
+    if ((desc->control & DSCR_CTRL_ENABLE_CRC) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_EN_CRC;
+    }
+    if ((desc->control & DSCR_CTRL_DESCRIPTOR_MODE) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_MODE;
+    }
+    if ((desc->control & DSCR_CTRL_AXI_BURST_TYPE) != 0) {
+        s->registers[DPDMA_STATUS_CH(channel)] |= DPDMA_STATUS_BURST_TYPE;
+    }
+}
+
+static void xlnx_dpdma_dump_descriptor(DPDMADescriptor *desc)
+{
+    if (DEBUG_DPDMA) {
+        qemu_log("DUMP DESCRIPTOR:\n");
+        qemu_hexdump((char *)desc, stdout, "", sizeof(DPDMADescriptor));
+    }
+}
+
+static uint64_t xlnx_dpdma_read(void *opaque, hwaddr offset,
+                                unsigned size)
+{
+    XlnxDPDMAState *s = XLNX_DPDMA(opaque);
+
+    DPRINTF("read @%" HWADDR_PRIx "\n", offset);
+    offset = offset >> 2;
+
+    switch (offset) {
+    /*
+     * Trying to read a write only register.
+     */
+    case DPDMA_GBL:
+        return 0;
+    default:
+        assert(offset <= (0xFFC >> 2));
+        return s->registers[offset];
+    }
+    return 0;
+}
+
+static void xlnx_dpdma_write(void *opaque, hwaddr offset,
+                               uint64_t value, unsigned size)
+{
+    XlnxDPDMAState *s = XLNX_DPDMA(opaque);
+
+    DPRINTF("write @%" HWADDR_PRIx " = %" PRIx64 "\n", offset, value);
+    offset = offset >> 2;
+
+    switch (offset) {
+    case DPDMA_ISR:
+        s->registers[DPDMA_ISR] &= ~value;
+        xlnx_dpdma_update_irq(s);
+        break;
+    case DPDMA_IEN:
+        s->registers[DPDMA_IMR] &= ~value;
+        break;
+    case DPDMA_IDS:
+        s->registers[DPDMA_IMR] |= value;
+        break;
+    case DPDMA_EISR:
+        s->registers[DPDMA_EISR] &= ~value;
+        xlnx_dpdma_update_irq(s);
+        break;
+    case DPDMA_EIEN:
+        s->registers[DPDMA_EIMR] &= ~value;
+        break;
+    case DPDMA_EIDS:
+        s->registers[DPDMA_EIMR] |= value;
+        break;
+    case DPDMA_IMR:
+    case DPDMA_EIMR:
+    case DPDMA_DSCR_NEXT_ADDRE_CH(0):
+    case DPDMA_DSCR_NEXT_ADDRE_CH(1):
+    case DPDMA_DSCR_NEXT_ADDRE_CH(2):
+    case DPDMA_DSCR_NEXT_ADDRE_CH(3):
+    case DPDMA_DSCR_NEXT_ADDRE_CH(4):
+    case DPDMA_DSCR_NEXT_ADDRE_CH(5):
+    case DPDMA_DSCR_NEXT_ADDR_CH(0):
+    case DPDMA_DSCR_NEXT_ADDR_CH(1):
+    case DPDMA_DSCR_NEXT_ADDR_CH(2):
+    case DPDMA_DSCR_NEXT_ADDR_CH(3):
+    case DPDMA_DSCR_NEXT_ADDR_CH(4):
+    case DPDMA_DSCR_NEXT_ADDR_CH(5):
+    case DPDMA_PYLD_CUR_ADDRE_CH(0):
+    case DPDMA_PYLD_CUR_ADDRE_CH(1):
+    case DPDMA_PYLD_CUR_ADDRE_CH(2):
+    case DPDMA_PYLD_CUR_ADDRE_CH(3):
+    case DPDMA_PYLD_CUR_ADDRE_CH(4):
+    case DPDMA_PYLD_CUR_ADDRE_CH(5):
+    case DPDMA_PYLD_CUR_ADDR_CH(0):
+    case DPDMA_PYLD_CUR_ADDR_CH(1):
+    case DPDMA_PYLD_CUR_ADDR_CH(2):
+    case DPDMA_PYLD_CUR_ADDR_CH(3):
+    case DPDMA_PYLD_CUR_ADDR_CH(4):
+    case DPDMA_PYLD_CUR_ADDR_CH(5):
+    case DPDMA_STATUS_CH(0):
+    case DPDMA_STATUS_CH(1):
+    case DPDMA_STATUS_CH(2):
+    case DPDMA_STATUS_CH(3):
+    case DPDMA_STATUS_CH(4):
+    case DPDMA_STATUS_CH(5):
+    case DPDMA_VDO_CH(0):
+    case DPDMA_VDO_CH(1):
+    case DPDMA_VDO_CH(2):
+    case DPDMA_VDO_CH(3):
+    case DPDMA_VDO_CH(4):
+    case DPDMA_VDO_CH(5):
+    case DPDMA_PYLD_SZ_CH(0):
+    case DPDMA_PYLD_SZ_CH(1):
+    case DPDMA_PYLD_SZ_CH(2):
+    case DPDMA_PYLD_SZ_CH(3):
+    case DPDMA_PYLD_SZ_CH(4):
+    case DPDMA_PYLD_SZ_CH(5):
+    case DPDMA_DSCR_ID_CH(0):
+    case DPDMA_DSCR_ID_CH(1):
+    case DPDMA_DSCR_ID_CH(2):
+    case DPDMA_DSCR_ID_CH(3):
+    case DPDMA_DSCR_ID_CH(4):
+    case DPDMA_DSCR_ID_CH(5):
+        /*
+         * Trying to write to a read only register..
+         */
+        break;
+    case DPDMA_GBL:
+        /*
+         * This is a write only register so it's read as zero in the read
+         * callback.
+         * We store the value anyway so we can know if the channel is
+         * enabled.
+         */
+        s->registers[offset] |= value & 0x00000FFF;
+        break;
+    case DPDMA_DSCR_STRT_ADDRE_CH(0):
+    case DPDMA_DSCR_STRT_ADDRE_CH(1):
+    case DPDMA_DSCR_STRT_ADDRE_CH(2):
+    case DPDMA_DSCR_STRT_ADDRE_CH(3):
+    case DPDMA_DSCR_STRT_ADDRE_CH(4):
+    case DPDMA_DSCR_STRT_ADDRE_CH(5):
+        value &= 0x0000FFFF;
+        s->registers[offset] = value;
+        break;
+    case DPDMA_CNTL_CH(0):
+        s->registers[DPDMA_GBL] &= ~DPDMA_GBL_TRG_CH(0);
+        value &= 0x3FFFFFFF;
+        s->registers[offset] = value;
+        break;
+    case DPDMA_CNTL_CH(1):
+        s->registers[DPDMA_GBL] &= ~DPDMA_GBL_TRG_CH(1);
+        value &= 0x3FFFFFFF;
+        s->registers[offset] = value;
+        break;
+    case DPDMA_CNTL_CH(2):
+        s->registers[DPDMA_GBL] &= ~DPDMA_GBL_TRG_CH(2);
+        value &= 0x3FFFFFFF;
+        s->registers[offset] = value;
+        break;
+    case DPDMA_CNTL_CH(3):
+        s->registers[DPDMA_GBL] &= ~DPDMA_GBL_TRG_CH(3);
+        value &= 0x3FFFFFFF;
+        s->registers[offset] = value;
+        break;
+    case DPDMA_CNTL_CH(4):
+        s->registers[DPDMA_GBL] &= ~DPDMA_GBL_TRG_CH(4);
+        value &= 0x3FFFFFFF;
+        s->registers[offset] = value;
+        break;
+    case DPDMA_CNTL_CH(5):
+        s->registers[DPDMA_GBL] &= ~DPDMA_GBL_TRG_CH(5);
+        value &= 0x3FFFFFFF;
+        s->registers[offset] = value;
+        break;
+    default:
+        assert(offset <= (0xFFC >> 2));
+        s->registers[offset] = value;
+        break;
+    }
+}
+
+static const MemoryRegionOps dma_ops = {
+    .read = xlnx_dpdma_read,
+    .write = xlnx_dpdma_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static void xlnx_dpdma_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    XlnxDPDMAState *s = XLNX_DPDMA(obj);
+
+    memory_region_init_io(&s->iomem, obj, &dma_ops, s,
+                          TYPE_XLNX_DPDMA, 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+}
+
+static void xlnx_dpdma_reset(DeviceState *dev)
+{
+    XlnxDPDMAState *s = XLNX_DPDMA(dev);
+    size_t i;
+
+    memset(s->registers, 0, sizeof(s->registers));
+    s->registers[DPDMA_IMR] =  0x07FFFFFF;
+    s->registers[DPDMA_EIMR] = 0xFFFFFFFF;
+    s->registers[DPDMA_ALC0_MIN] = 0x0000FFFF;
+    s->registers[DPDMA_ALC1_MIN] = 0x0000FFFF;
+
+    for (i = 0; i < 6; i++) {
+        s->data[i] = NULL;
+        s->operation_finished[i] = true;
+    }
+}
+
+static void xlnx_dpdma_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->vmsd = &vmstate_xlnx_dpdma;
+    dc->reset = xlnx_dpdma_reset;
+}
+
+static const TypeInfo xlnx_dpdma_info = {
+    .name          = TYPE_XLNX_DPDMA,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(XlnxDPDMAState),
+    .instance_init = xlnx_dpdma_init,
+    .class_init    = xlnx_dpdma_class_init,
+};
+
+static void xlnx_dpdma_register_types(void)
+{
+    type_register_static(&xlnx_dpdma_info);
+}
+
+size_t xlnx_dpdma_start_operation(XlnxDPDMAState *s, uint8_t channel,
+                                    bool one_desc)
+{
+    uint64_t desc_addr;
+    uint64_t source_addr[6];
+    DPDMADescriptor desc;
+    bool done = false;
+    size_t ptr = 0;
+
+    assert(channel <= 5);
+
+    DPRINTF("start dpdma channel 0x%" PRIX8 "\n", channel);
+
+    if (!xlnx_dpdma_is_channel_triggered(s, channel)) {
+        DPRINTF("Channel isn't triggered..\n");
+        return 0;
+    }
+
+    if (!xlnx_dpdma_is_channel_enabled(s, channel)) {
+        DPRINTF("Channel isn't enabled..\n");
+        return 0;
+    }
+
+    if (xlnx_dpdma_is_channel_paused(s, channel)) {
+        DPRINTF("Channel is paused..\n");
+        return 0;
+    }
+
+    do {
+        if ((s->operation_finished[channel])
+          || xlnx_dpdma_is_channel_retriggered(s, channel)) {
+            desc_addr = xlnx_dpdma_descriptor_start_address(s, channel);
+            s->operation_finished[channel] = false;
+        } else {
+            desc_addr = xlnx_dpdma_descriptor_next_address(s, channel);
+        }
+
+        if (dma_memory_read(&address_space_memory, desc_addr, &desc,
+                            sizeof(DPDMADescriptor))) {
+            s->registers[DPDMA_EISR] |= ((1 << 1) << channel);
+            xlnx_dpdma_update_irq(s);
+            s->operation_finished[channel] = true;
+            DPRINTF("Can't get the descriptor.\n");
+            break;
+        }
+
+        xlnx_dpdma_update_desc_info(s, channel, &desc);
+
+#ifdef DEBUG_DPDMA
+        xlnx_dpdma_dump_descriptor(&desc);
+#endif
+
+        DPRINTF("location of the descriptor: %" PRIx64 "\n", desc_addr);
+        if (!xlnx_dpdma_desc_is_valid(&desc)) {
+            s->registers[DPDMA_EISR] |= ((1 << 7) << channel);
+            xlnx_dpdma_update_irq(s);
+            s->operation_finished[channel] = true;
+            DPRINTF("Invalid descriptor..\n");
+            break;
+        }
+
+        if (xlnx_dpdma_desc_crc_enabled(&desc)
+            && !xlnx_dpdma_desc_check_crc(&desc)) {
+            s->registers[DPDMA_EISR] |= ((1 << 13) << channel);
+            xlnx_dpdma_update_irq(s);
+            s->operation_finished[channel] = true;
+            DPRINTF("Bad CRC for descriptor..\n");
+            break;
+        }
+
+        if (xlnx_dpdma_desc_is_already_done(&desc)
+            && !xlnx_dpdma_desc_ignore_done_bit(&desc)) {
+            /* We are trying to process an already processed descriptor. */
+            s->registers[DPDMA_EISR] |= ((1 << 25) << channel);
+            xlnx_dpdma_update_irq(s);
+            s->operation_finished[channel] = true;
+            DPRINTF("Already processed descriptor..\n");
+            break;
+        }
+
+        done = xlnx_dpdma_desc_is_last(&desc)
+             || xlnx_dpdma_desc_is_last_of_frame(&desc);
+
+        s->operation_finished[channel] = done;
+        if (s->data[channel]) {
+            int64_t transfer_len = xlnx_dpdma_desc_get_transfer_size(&desc);
+            uint32_t line_size = xlnx_dpdma_desc_get_line_size(&desc);
+            uint32_t line_stride = xlnx_dpdma_desc_get_line_stride(&desc);
+            if (xlnx_dpdma_desc_is_contiguous(&desc)) {
+                source_addr[0] = xlnx_dpdma_desc_get_source_address(&desc, 0);
+                while (transfer_len != 0) {
+                    if (dma_memory_read(&address_space_memory,
+                                        source_addr[0],
+                                        &s->data[channel][ptr],
+                                        line_size)) {
+                        s->registers[DPDMA_ISR] |= ((1 << 12) << channel);
+                        xlnx_dpdma_update_irq(s);
+                        DPRINTF("Can't get data.\n");
+                        break;
+                    }
+                    ptr += line_size;
+                    transfer_len -= line_size;
+                    source_addr[0] += line_stride;
+                }
+            } else {
+                DPRINTF("Source address:\n");
+                int frag;
+                for (frag = 0; frag < 5; frag++) {
+                    source_addr[frag] =
+                          xlnx_dpdma_desc_get_source_address(&desc, frag);
+                    DPRINTF("Fragment %u: %" PRIx64 "\n", frag + 1,
+                            source_addr[frag]);
+                }
+
+                frag = 0;
+                while ((transfer_len < 0) && (frag < 5)) {
+                    size_t fragment_len = DPDMA_FRAG_MAX_SZ
+                                    - (source_addr[frag] % DPDMA_FRAG_MAX_SZ);
+
+                    if (dma_memory_read(&address_space_memory,
+                                        source_addr[frag],
+                                        &(s->data[channel][ptr]),
+                                        fragment_len)) {
+                        s->registers[DPDMA_ISR] |= ((1 << 12) << channel);
+                        xlnx_dpdma_update_irq(s);
+                        DPRINTF("Can't get data.\n");
+                        break;
+                    }
+                    ptr += fragment_len;
+                    transfer_len -= fragment_len;
+                    frag += 1;
+                }
+            }
+        }
+
+        if (xlnx_dpdma_desc_update_enabled(&desc)) {
+            /* The descriptor need to be updated when it's completed. */
+            DPRINTF("update the descriptor with the done flag set.\n");
+            xlnx_dpdma_desc_set_done(&desc);
+            dma_memory_write(&address_space_memory, desc_addr, &desc,
+                             sizeof(DPDMADescriptor));
+        }
+
+        if (xlnx_dpdma_desc_completion_interrupt(&desc)) {
+            DPRINTF("completion interrupt enabled!\n");
+            s->registers[DPDMA_ISR] |= (1 << channel);
+            xlnx_dpdma_update_irq(s);
+        }
+
+    } while (!done && !one_desc);
+
+    return ptr;
+}
+
+void xlnx_dpdma_set_host_data_location(XlnxDPDMAState *s, uint8_t channel,
+                                         void *p)
+{
+    if (!s) {
+        qemu_log_mask(LOG_UNIMP, "DPDMA client not attached to valid DPDMA"
+                      " instance\n");
+        return;
+    }
+
+    assert(channel <= 5);
+    s->data[channel] = p;
+}
+
+void xlnx_dpdma_trigger_vsync_irq(XlnxDPDMAState *s)
+{
+    s->registers[DPDMA_ISR] |= (1 << 27);
+    xlnx_dpdma_update_irq(s);
+}
+
+type_init(xlnx_dpdma_register_types)
diff --git a/hw/gpio/Makefile.objs b/hw/gpio/Makefile.objs
index 1abcf17..a43c7cf 100644
--- a/hw/gpio/Makefile.objs
+++ b/hw/gpio/Makefile.objs
@@ -3,5 +3,7 @@
 common-obj-$(CONFIG_PUV3) += puv3_gpio.o
 common-obj-$(CONFIG_ZAURUS) += zaurus.o
 common-obj-$(CONFIG_E500) += mpc8xxx.o
+common-obj-$(CONFIG_GPIO_KEY) += gpio_key.o
 
 obj-$(CONFIG_OMAP) += omap_gpio.o
+obj-$(CONFIG_IMX) += imx_gpio.o
diff --git a/hw/gpio/gpio_key.c b/hw/gpio/gpio_key.c
new file mode 100644
index 0000000..b34aa49
--- /dev/null
+++ b/hw/gpio/gpio_key.c
@@ -0,0 +1,105 @@
+/*
+ * GPIO key
+ *
+ * Copyright (c) 2016 Linaro Limited
+ *
+ * Author: Shannon Zhao <shannon.zhao@linaro.org>
+ *
+ * Emulate a (human) keypress -- when the key is triggered by
+ * setting the incoming gpio line, the outbound irq line is
+ * raised for 100ms before being dropped again.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+#include "qemu/timer.h"
+
+#define TYPE_GPIOKEY "gpio-key"
+#define GPIOKEY(obj) OBJECT_CHECK(GPIOKEYState, (obj), TYPE_GPIOKEY)
+#define GPIO_KEY_LATENCY 100 /* 100ms */
+
+typedef struct GPIOKEYState {
+    SysBusDevice parent_obj;
+
+    QEMUTimer *timer;
+    qemu_irq irq;
+} GPIOKEYState;
+
+static const VMStateDescription vmstate_gpio_key = {
+    .name = "gpio-key",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_TIMER_PTR(timer, GPIOKEYState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void gpio_key_reset(DeviceState *dev)
+{
+    GPIOKEYState *s = GPIOKEY(dev);
+
+    timer_del(s->timer);
+}
+
+static void gpio_key_timer_expired(void *opaque)
+{
+    GPIOKEYState *s = (GPIOKEYState *)opaque;
+
+    qemu_set_irq(s->irq, 0);
+    timer_del(s->timer);
+}
+
+static void gpio_key_set_irq(void *opaque, int irq, int level)
+{
+    GPIOKEYState *s = (GPIOKEYState *)opaque;
+
+    qemu_set_irq(s->irq, 1);
+    timer_mod(s->timer,
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + GPIO_KEY_LATENCY);
+}
+
+static void gpio_key_realize(DeviceState *dev, Error **errp)
+{
+    GPIOKEYState *s = GPIOKEY(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_in(dev, gpio_key_set_irq, 1);
+    s->timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, gpio_key_timer_expired, s);
+}
+
+static void gpio_key_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = gpio_key_realize;
+    dc->vmsd = &vmstate_gpio_key;
+    dc->reset = &gpio_key_reset;
+}
+
+static const TypeInfo gpio_key_info = {
+    .name          = TYPE_GPIOKEY,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(GPIOKEYState),
+    .class_init    = gpio_key_class_init,
+};
+
+static void gpio_key_register_types(void)
+{
+    type_register_static(&gpio_key_info);
+}
+
+type_init(gpio_key_register_types)
diff --git a/hw/gpio/imx_gpio.c b/hw/gpio/imx_gpio.c
new file mode 100644
index 0000000..f3574aa
--- /dev/null
+++ b/hw/gpio/imx_gpio.c
@@ -0,0 +1,351 @@
+/*
+ * i.MX processors GPIO emulation.
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/gpio/imx_gpio.h"
+#include "qemu/log.h"
+
+#ifndef DEBUG_IMX_GPIO
+#define DEBUG_IMX_GPIO 0
+#endif
+
+typedef enum IMXGPIOLevel {
+    IMX_GPIO_LEVEL_LOW = 0,
+    IMX_GPIO_LEVEL_HIGH = 1,
+} IMXGPIOLevel;
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_GPIO) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_GPIO, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_gpio_reg_name(uint32_t reg)
+{
+    switch (reg) {
+    case DR_ADDR:
+        return "DR";
+    case GDIR_ADDR:
+        return "GDIR";
+    case PSR_ADDR:
+        return "PSR";
+    case ICR1_ADDR:
+        return "ICR1";
+    case ICR2_ADDR:
+        return "ICR2";
+    case IMR_ADDR:
+        return "IMR";
+    case ISR_ADDR:
+        return "ISR";
+    case EDGE_SEL_ADDR:
+        return "EDGE_SEL";
+    default:
+        return "[?]";
+    }
+}
+
+static void imx_gpio_update_int(IMXGPIOState *s)
+{
+    if (s->has_upper_pin_irq) {
+        qemu_set_irq(s->irq[0], (s->isr & s->imr & 0x0000FFFF) ? 1 : 0);
+        qemu_set_irq(s->irq[1], (s->isr & s->imr & 0xFFFF0000) ? 1 : 0);
+    } else {
+        qemu_set_irq(s->irq[0], (s->isr & s->imr) ? 1 : 0);
+    }
+}
+
+static void imx_gpio_set_int_line(IMXGPIOState *s, int line, IMXGPIOLevel level)
+{
+    /* if this signal isn't configured as an input signal, nothing to do */
+    if (!extract32(s->gdir, line, 1)) {
+        return;
+    }
+
+    /* When set, EDGE_SEL overrides the ICR config */
+    if (extract32(s->edge_sel, line, 1)) {
+        /* we detect interrupt on rising and falling edge */
+        if (extract32(s->psr, line, 1) != level) {
+            /* level changed */
+            s->isr = deposit32(s->isr, line, 1, 1);
+        }
+    } else if (extract64(s->icr, 2*line + 1, 1)) {
+        /* interrupt is edge sensitive */
+        if (extract32(s->psr, line, 1) != level) {
+            /* level changed */
+            if (extract64(s->icr, 2*line, 1) != level) {
+                s->isr = deposit32(s->isr, line, 1, 1);
+            }
+        }
+    } else {
+        /* interrupt is level sensitive */
+        if (extract64(s->icr, 2*line, 1) == level) {
+            s->isr = deposit32(s->isr, line, 1, 1);
+        }
+    }
+}
+
+static void imx_gpio_set(void *opaque, int line, int level)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+    IMXGPIOLevel imx_level = level ? IMX_GPIO_LEVEL_HIGH : IMX_GPIO_LEVEL_LOW;
+
+    imx_gpio_set_int_line(s, line, imx_level);
+
+    /* this is an input signal, so set PSR */
+    s->psr = deposit32(s->psr, line, 1, imx_level);
+
+    imx_gpio_update_int(s);
+}
+
+static void imx_gpio_set_all_int_lines(IMXGPIOState *s)
+{
+    int i;
+
+    for (i = 0; i < IMX_GPIO_PIN_COUNT; i++) {
+        IMXGPIOLevel imx_level = extract32(s->psr, i, 1);
+        imx_gpio_set_int_line(s, i, imx_level);
+    }
+
+    imx_gpio_update_int(s);
+}
+
+static inline void imx_gpio_set_all_output_lines(IMXGPIOState *s)
+{
+    int i;
+
+    for (i = 0; i < IMX_GPIO_PIN_COUNT; i++) {
+        /*
+         * if the line is set as output, then forward the line
+         * level to its user.
+         */
+        if (extract32(s->gdir, i, 1) && s->output[i]) {
+            qemu_set_irq(s->output[i], extract32(s->dr, i, 1));
+        }
+    }
+}
+
+static uint64_t imx_gpio_read(void *opaque, hwaddr offset, unsigned size)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+    uint32_t reg_value = 0;
+
+    switch (offset) {
+    case DR_ADDR:
+        /*
+         * depending on the "line" configuration, the bit values
+         * are coming either from DR or PSR
+         */
+        reg_value = (s->dr & s->gdir) | (s->psr & ~s->gdir);
+        break;
+
+    case GDIR_ADDR:
+        reg_value = s->gdir;
+        break;
+
+    case PSR_ADDR:
+        reg_value = s->psr & ~s->gdir;
+        break;
+
+    case ICR1_ADDR:
+        reg_value = extract64(s->icr, 0, 32);
+        break;
+
+    case ICR2_ADDR:
+        reg_value = extract64(s->icr, 32, 32);
+        break;
+
+    case IMR_ADDR:
+        reg_value = s->imr;
+        break;
+
+    case ISR_ADDR:
+        reg_value = s->isr;
+        break;
+
+    case EDGE_SEL_ADDR:
+        if (s->has_edge_sel) {
+            reg_value = s->edge_sel;
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: EDGE_SEL register not "
+                          "present on this version of GPIO device\n",
+                          TYPE_IMX_GPIO, __func__);
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPIO, __func__, offset);
+        break;
+    }
+
+    DPRINTF("(%s) = 0x%" PRIx32 "\n", imx_gpio_reg_name(offset), reg_value);
+
+    return reg_value;
+}
+
+static void imx_gpio_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    IMXGPIOState *s = IMX_GPIO(opaque);
+
+    DPRINTF("(%s, value = 0x%" PRIx32 ")\n", imx_gpio_reg_name(offset),
+            (uint32_t)value);
+
+    switch (offset) {
+    case DR_ADDR:
+        s->dr = value;
+        imx_gpio_set_all_output_lines(s);
+        break;
+
+    case GDIR_ADDR:
+        s->gdir = value;
+        imx_gpio_set_all_output_lines(s);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case ICR1_ADDR:
+        s->icr = deposit64(s->icr, 0, 32, value);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case ICR2_ADDR:
+        s->icr = deposit64(s->icr, 32, 32, value);
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case IMR_ADDR:
+        s->imr = value;
+        imx_gpio_update_int(s);
+        break;
+
+    case ISR_ADDR:
+        s->isr |= ~value;
+        imx_gpio_set_all_int_lines(s);
+        break;
+
+    case EDGE_SEL_ADDR:
+        if (s->has_edge_sel) {
+            s->edge_sel = value;
+            imx_gpio_set_all_int_lines(s);
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: EDGE_SEL register not "
+                          "present on this version of GPIO device\n",
+                          TYPE_IMX_GPIO, __func__);
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPIO, __func__, offset);
+        break;
+    }
+
+    return;
+}
+
+static const MemoryRegionOps imx_gpio_ops = {
+    .read = imx_gpio_read,
+    .write = imx_gpio_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription vmstate_imx_gpio = {
+    .name = TYPE_IMX_GPIO,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(dr, IMXGPIOState),
+        VMSTATE_UINT32(gdir, IMXGPIOState),
+        VMSTATE_UINT32(psr, IMXGPIOState),
+        VMSTATE_UINT64(icr, IMXGPIOState),
+        VMSTATE_UINT32(imr, IMXGPIOState),
+        VMSTATE_UINT32(isr, IMXGPIOState),
+        VMSTATE_BOOL(has_edge_sel, IMXGPIOState),
+        VMSTATE_UINT32(edge_sel, IMXGPIOState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property imx_gpio_properties[] = {
+    DEFINE_PROP_BOOL("has-edge-sel", IMXGPIOState, has_edge_sel, true),
+    DEFINE_PROP_BOOL("has-upper-pin-irq", IMXGPIOState, has_upper_pin_irq,
+                     false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void imx_gpio_reset(DeviceState *dev)
+{
+    IMXGPIOState *s = IMX_GPIO(dev);
+
+    s->dr       = 0;
+    s->gdir     = 0;
+    s->psr      = 0;
+    s->icr      = 0;
+    s->imr      = 0;
+    s->isr      = 0;
+    s->edge_sel = 0;
+
+    imx_gpio_set_all_output_lines(s);
+    imx_gpio_update_int(s);
+}
+
+static void imx_gpio_realize(DeviceState *dev, Error **errp)
+{
+    IMXGPIOState *s = IMX_GPIO(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &imx_gpio_ops, s,
+                          TYPE_IMX_GPIO, IMX_GPIO_MEM_SIZE);
+
+    qdev_init_gpio_in(DEVICE(s), imx_gpio_set, IMX_GPIO_PIN_COUNT);
+    qdev_init_gpio_out(DEVICE(s), s->output, IMX_GPIO_PIN_COUNT);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq[0]);
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq[1]);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+}
+
+static void imx_gpio_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = imx_gpio_realize;
+    dc->reset = imx_gpio_reset;
+    dc->props = imx_gpio_properties;
+    dc->vmsd = &vmstate_imx_gpio;
+    dc->desc = "i.MX GPIO controller";
+}
+
+static const TypeInfo imx_gpio_info = {
+    .name = TYPE_IMX_GPIO,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXGPIOState),
+    .class_init = imx_gpio_class_init,
+};
+
+static void imx_gpio_register_types(void)
+{
+    type_register_static(&imx_gpio_info);
+}
+
+type_init(imx_gpio_register_types)
diff --git a/hw/gpio/max7310.c b/hw/gpio/max7310.c
index 7fbf313..1bd5eaf 100644
--- a/hw/gpio/max7310.c
+++ b/hw/gpio/max7310.c
@@ -7,6 +7,7 @@
  * This file is licensed under GNU GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/i2c/i2c.h"
 
 #define TYPE_MAX7310 "max7310"
@@ -96,7 +97,7 @@
     case 0x01:	/* Output port */
         for (diff = (data ^ s->level) & ~s->direction; diff;
                         diff &= ~(1 << line)) {
-            line = ffs(diff) - 1;
+            line = ctz32(diff);
             if (s->handler[line])
                 qemu_set_irq(s->handler[line], (data >> line) & 1);
         }
diff --git a/hw/gpio/mpc8xxx.c b/hw/gpio/mpc8xxx.c
index 1aeaaaa..d149719 100644
--- a/hw/gpio/mpc8xxx.c
+++ b/hw/gpio/mpc8xxx.c
@@ -19,6 +19,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #define TYPE_MPC8XXX_GPIO "mpc8xxx_gpio"
diff --git a/hw/gpio/omap_gpio.c b/hw/gpio/omap_gpio.c
index 938782a..dabef4a 100644
--- a/hw/gpio/omap_gpio.c
+++ b/hw/gpio/omap_gpio.c
@@ -18,9 +18,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
 
 struct omap_gpio_s {
     qemu_irq irq;
@@ -113,7 +116,8 @@
     int ln;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, addr, value);
+        omap_badwidth_write16(opaque, addr, value);
+        return;
     }
 
     switch (offset) {
@@ -124,8 +128,7 @@
     case 0x04:	/* DATA_OUTPUT */
         diff = (s->outputs ^ value) & ~s->dir;
         s->outputs = value;
-        while ((ln = ffs(diff))) {
-            ln --;
+        while ((ln = ctz32(diff)) != 32) {
             if (s->handler[ln])
                 qemu_set_irq(s->handler[ln], (value >> ln) & 1);
             diff &= ~(1 << ln);
@@ -137,8 +140,7 @@
         s->dir = value;
 
         value = s->outputs & ~s->dir;
-        while ((ln = ffs(diff))) {
-            ln --;
+        while ((ln = ctz32(diff)) != 32) {
             if (s->handler[ln])
                 qemu_set_irq(s->handler[ln], (value >> ln) & 1);
             diff &= ~(1 << ln);
@@ -252,8 +254,7 @@
 
     s->outputs ^= diff;
     diff &= ~s->dir;
-    while ((ln = ffs(diff))) {
-        ln --;
+    while ((ln = ctz32(diff)) != 32) {
         qemu_set_irq(s->handler[ln], (s->outputs >> ln) & 1);
         diff &= ~(1 << ln);
     }
@@ -441,8 +442,8 @@
         s->dir = value;
 
         value = s->outputs & ~s->dir;
-        while ((ln = ffs(diff))) {
-            diff &= ~(1 <<-- ln);
+        while ((ln = ctz32(diff)) != 32) {
+            diff &= ~(1 << ln);
             qemu_set_irq(s->handler[ln], (value >> ln) & 1);
         }
 
@@ -678,59 +679,72 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int omap_gpio_init(SysBusDevice *sbd)
+static void omap_gpio_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    struct omap_gpif_s *s = OMAP1_GPIO(dev);
+    DeviceState *dev = DEVICE(obj);
+    struct omap_gpif_s *s = OMAP1_GPIO(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    if (!s->clk) {
-        hw_error("omap-gpio: clk not connected\n");
-    }
     qdev_init_gpio_in(dev, omap_gpio_set, 16);
     qdev_init_gpio_out(dev, s->omap1.handler, 16);
     sysbus_init_irq(sbd, &s->omap1.irq);
-    memory_region_init_io(&s->iomem, OBJECT(s), &omap_gpio_ops, &s->omap1,
+    memory_region_init_io(&s->iomem, obj, &omap_gpio_ops, &s->omap1,
                           "omap.gpio", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
-    return 0;
 }
 
-static int omap2_gpio_init(SysBusDevice *sbd)
+static void omap_gpio_realize(DeviceState *dev, Error **errp)
 {
-    DeviceState *dev = DEVICE(sbd);
+    struct omap_gpif_s *s = OMAP1_GPIO(dev);
+
+    if (!s->clk) {
+        error_setg(errp, "omap-gpio: clk not connected");
+    }
+}
+
+static void omap2_gpio_realize(DeviceState *dev, Error **errp)
+{
     struct omap2_gpif_s *s = OMAP2_GPIO(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
     int i;
 
     if (!s->iclk) {
-        hw_error("omap2-gpio: iclk not connected\n");
+        error_setg(errp, "omap2-gpio: iclk not connected");
+        return;
     }
+
+    s->modulecount = s->mpu_model < omap2430 ? 4
+        : s->mpu_model < omap3430 ? 5
+        : 6;
+
     if (s->mpu_model < omap3430) {
-        s->modulecount = (s->mpu_model < omap2430) ? 4 : 5;
-        memory_region_init_io(&s->iomem, OBJECT(s), &omap2_gpif_top_ops, s,
+        memory_region_init_io(&s->iomem, OBJECT(dev), &omap2_gpif_top_ops, s,
                               "omap2.gpio", 0x1000);
         sysbus_init_mmio(sbd, &s->iomem);
-    } else {
-        s->modulecount = 6;
     }
-    s->modules = g_malloc0(s->modulecount * sizeof(struct omap2_gpio_s));
-    s->handler = g_malloc0(s->modulecount * 32 * sizeof(qemu_irq));
+
+    s->modules = g_new0(struct omap2_gpio_s, s->modulecount);
+    s->handler = g_new0(qemu_irq, s->modulecount * 32);
     qdev_init_gpio_in(dev, omap2_gpio_set, s->modulecount * 32);
     qdev_init_gpio_out(dev, s->handler, s->modulecount * 32);
+
     for (i = 0; i < s->modulecount; i++) {
         struct omap2_gpio_s *m = &s->modules[i];
+
         if (!s->fclk[i]) {
-            hw_error("omap2-gpio: fclk%d not connected\n", i);
+            error_setg(errp, "omap2-gpio: fclk%d not connected", i);
+            return;
         }
+
         m->revision = (s->mpu_model < omap3430) ? 0x18 : 0x25;
         m->handler = &s->handler[i * 32];
         sysbus_init_irq(sbd, &m->irq[0]); /* mpu irq */
         sysbus_init_irq(sbd, &m->irq[1]); /* dsp irq */
         sysbus_init_irq(sbd, &m->wkup);
-        memory_region_init_io(&m->iomem, OBJECT(s), &omap2_gpio_module_ops, m,
+        memory_region_init_io(&m->iomem, OBJECT(dev), &omap2_gpio_module_ops, m,
                               "omap.gpio-module", 0x1000);
         sysbus_init_mmio(sbd, &m->iomem);
     }
-    return 0;
 }
 
 /* Using qdev pointer properties for the clocks is not ideal.
@@ -754,9 +768,8 @@
 static void omap_gpio_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = omap_gpio_init;
+    dc->realize = omap_gpio_realize;
     dc->reset = omap_gpif_reset;
     dc->props = omap_gpio_properties;
     /* Reason: pointer property "clk" */
@@ -767,6 +780,7 @@
     .name          = TYPE_OMAP1_GPIO,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct omap_gpif_s),
+    .instance_init = omap_gpio_init,
     .class_init    = omap_gpio_class_init,
 };
 
@@ -785,9 +799,8 @@
 static void omap2_gpio_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = omap2_gpio_init;
+    dc->realize = omap2_gpio_realize;
     dc->reset = omap2_gpif_reset;
     dc->props = omap2_gpio_properties;
     /* Reason: pointer properties "iclk", "fclk0", ..., "fclk5" */
diff --git a/hw/gpio/pl061.c b/hw/gpio/pl061.c
index bd03e99..4ae2aa1 100644
--- a/hw/gpio/pl061.c
+++ b/hw/gpio/pl061.c
@@ -8,7 +8,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qemu/log.h"
 
 //#define DEBUG_PL061 1
 
@@ -55,17 +57,17 @@
     uint32_t slr;
     uint32_t den;
     uint32_t cr;
-    uint32_t float_high;
     uint32_t amsel;
     qemu_irq irq;
     qemu_irq out[8];
     const unsigned char *id;
+    uint32_t rsvd_start; /* reserved area: [rsvd_start, 0xfcc] */
 } PL061State;
 
 static const VMStateDescription vmstate_pl061 = {
     .name = "pl061",
-    .version_id = 3,
-    .minimum_version_id = 3,
+    .version_id = 4,
+    .minimum_version_id = 4,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(locked, PL061State),
         VMSTATE_UINT32(data, PL061State),
@@ -87,7 +89,6 @@
         VMSTATE_UINT32(slr, PL061State),
         VMSTATE_UINT32(den, PL061State),
         VMSTATE_UINT32(cr, PL061State),
-        VMSTATE_UINT32(float_high, PL061State),
         VMSTATE_UINT32_V(amsel, PL061State, 2),
         VMSTATE_END_OF_LIST()
     }
@@ -153,12 +154,15 @@
 {
     PL061State *s = (PL061State *)opaque;
 
-    if (offset >= 0xfd0 && offset < 0x1000) {
-        return s->id[(offset - 0xfd0) >> 2];
-    }
     if (offset < 0x400) {
         return s->data & (offset >> 2);
     }
+    if (offset >= s->rsvd_start && offset <= 0xfcc) {
+        goto err_out;
+    }
+    if (offset >= 0xfd0 && offset < 0x1000) {
+        return s->id[(offset - 0xfd0) >> 2];
+    }
     switch (offset) {
     case 0x400: /* Direction */
         return s->dir;
@@ -173,7 +177,7 @@
     case 0x414: /* Raw interrupt status */
         return s->istate;
     case 0x418: /* Masked interrupt status */
-        return s->istate | s->im;
+        return s->istate & s->im;
     case 0x420: /* Alternate function select */
         return s->afsel;
     case 0x500: /* 2mA drive */
@@ -199,10 +203,12 @@
     case 0x528: /* Analog mode select */
         return s->amsel;
     default:
-        qemu_log_mask(LOG_GUEST_ERROR,
-                      "pl061_read: Bad offset %x\n", (int)offset);
-        return 0;
+        break;
     }
+err_out:
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "pl061_read: Bad offset %x\n", (int)offset);
+    return 0;
 }
 
 static void pl061_write(void *opaque, hwaddr offset,
@@ -217,6 +223,9 @@
         pl061_update(s);
         return;
     }
+    if (offset >= s->rsvd_start) {
+        goto err_out;
+    }
     switch (offset) {
     case 0x400: /* Direction */
         s->dir = value & 0xff;
@@ -275,16 +284,41 @@
         s->amsel = value & 0xff;
         break;
     default:
-        qemu_log_mask(LOG_GUEST_ERROR,
-                      "pl061_write: Bad offset %x\n", (int)offset);
+        goto err_out;
     }
     pl061_update(s);
+    return;
+err_out:
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "pl061_write: Bad offset %x\n", (int)offset);
 }
 
-static void pl061_reset(PL061State *s)
+static void pl061_reset(DeviceState *dev)
 {
-  s->locked = 1;
-  s->cr = 0xff;
+    PL061State *s = PL061(dev);
+
+    /* reset values from PL061 TRM, Stellaris LM3S5P31 & LM3S8962 Data Sheet */
+    s->data = 0;
+    s->old_out_data = 0;
+    s->old_in_data = 0;
+    s->dir = 0;
+    s->isense = 0;
+    s->ibe = 0;
+    s->iev = 0;
+    s->im = 0;
+    s->istate = 0;
+    s->afsel = 0;
+    s->dr2r = 0xff;
+    s->dr4r = 0;
+    s->dr8r = 0;
+    s->odr = 0;
+    s->pur = 0;
+    s->pdr = 0;
+    s->slr = 0;
+    s->den = 0;
+    s->locked = 1;
+    s->cr = 0xff;
+    s->amsel = 0;
 }
 
 static void pl061_set_irq(void * opaque, int irq, int level)
@@ -307,41 +341,36 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int pl061_initfn(SysBusDevice *sbd)
-{
-    DeviceState *dev = DEVICE(sbd);
-    PL061State *s = PL061(dev);
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &pl061_ops, s, "pl061", 0x1000);
-    sysbus_init_mmio(sbd, &s->iomem);
-    sysbus_init_irq(sbd, &s->irq);
-    qdev_init_gpio_in(dev, pl061_set_irq, 8);
-    qdev_init_gpio_out(dev, s->out, 8);
-    pl061_reset(s);
-    return 0;
-}
-
 static void pl061_luminary_init(Object *obj)
 {
     PL061State *s = PL061(obj);
 
     s->id = pl061_id_luminary;
+    s->rsvd_start = 0x52c;
 }
 
 static void pl061_init(Object *obj)
 {
     PL061State *s = PL061(obj);
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     s->id = pl061_id;
+    s->rsvd_start = 0x424;
+
+    memory_region_init_io(&s->iomem, obj, &pl061_ops, s, "pl061", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_in(dev, pl061_set_irq, 8);
+    qdev_init_gpio_out(dev, s->out, 8);
 }
 
 static void pl061_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pl061_initfn;
     dc->vmsd = &vmstate_pl061;
+    dc->reset = &pl061_reset;
 }
 
 static const TypeInfo pl061_info = {
diff --git a/hw/gpio/puv3_gpio.c b/hw/gpio/puv3_gpio.c
index 39840aa..445afcc 100644
--- a/hw/gpio/puv3_gpio.c
+++ b/hw/gpio/puv3_gpio.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/hw/gpio/zaurus.c b/hw/gpio/zaurus.c
index 9408342..15865e1 100644
--- a/hw/gpio/zaurus.c
+++ b/hw/gpio/zaurus.c
@@ -15,6 +15,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/sharpsl.h"
 #include "hw/sysbus.h"
@@ -65,7 +66,7 @@
     level = s->gpio_level & s->gpio_dir;
 
     for (diff = s->prev_level ^ level; diff; diff ^= 1 << bit) {
-        bit = ffs(diff) - 1;
+        bit = ctz32(diff);
         qemu_set_irq(s->handler[bit], (level >> bit) & 1);
     }
 
@@ -166,19 +167,18 @@
         s->gpio_level &= ~(1 << line);
 }
 
-static int scoop_init(SysBusDevice *sbd)
+static void scoop_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    ScoopInfo *s = SCOOP(dev);
+    DeviceState *dev = DEVICE(obj);
+    ScoopInfo *s = SCOOP(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     s->status = 0x02;
     qdev_init_gpio_out(dev, s->handler, 16);
     qdev_init_gpio_in(dev, scoop_gpio_set, 16);
-    memory_region_init_io(&s->iomem, OBJECT(s), &scoop_ops, s, "scoop", 0x1000);
+    memory_region_init_io(&s->iomem, obj, &scoop_ops, s, "scoop", 0x1000);
 
     sysbus_init_mmio(sbd, &s->iomem);
-
-    return 0;
 }
 
 static int scoop_post_load(void *opaque, int version_id)
@@ -235,25 +235,19 @@
     },
 };
 
-static Property scoop_sysbus_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void scoop_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = scoop_init;
     dc->desc = "Scoop2 Sharp custom ASIC";
     dc->vmsd = &vmstate_scoop_regs;
-    dc->props = scoop_sysbus_properties;
 }
 
 static const TypeInfo scoop_sysbus_info = {
     .name          = TYPE_SCOOP,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(ScoopInfo),
+    .instance_init = scoop_init,
     .class_init    = scoop_sysbus_class_init,
 };
 
diff --git a/hw/i2c/Makefile.objs b/hw/i2c/Makefile.objs
index 648278e..a081b8e 100644
--- a/hw/i2c/Makefile.objs
+++ b/hw/i2c/Makefile.objs
@@ -1,7 +1,10 @@
 common-obj-y += core.o smbus.o smbus_eeprom.o
+common-obj-$(CONFIG_DDC) += i2c-ddc.o
 common-obj-$(CONFIG_VERSATILE_I2C) += versatile_i2c.o
-common-obj-$(CONFIG_ACPI) += smbus_ich9.o
+common-obj-$(CONFIG_ACPI_X86) += smbus_ich9.o
 common-obj-$(CONFIG_APM) += pm_smbus.o
 common-obj-$(CONFIG_BITBANG_I2C) += bitbang_i2c.o
 common-obj-$(CONFIG_EXYNOS4) += exynos4210_i2c.o
+common-obj-$(CONFIG_IMX_I2C) += imx_i2c.o
+common-obj-$(CONFIG_ASPEED_SOC) += aspeed_i2c.o
 obj-$(CONFIG_OMAP) += omap_i2c.o
diff --git a/hw/i2c/aspeed_i2c.c b/hw/i2c/aspeed_i2c.c
new file mode 100644
index 0000000..ce5b1f0
--- /dev/null
+++ b/hw/i2c/aspeed_i2c.c
@@ -0,0 +1,440 @@
+/*
+ * ARM Aspeed I2C controller
+ *
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+#include "qemu/log.h"
+#include "hw/i2c/aspeed_i2c.h"
+
+/* I2C Global Register */
+
+#define I2C_CTRL_STATUS         0x00        /* Device Interrupt Status */
+#define I2C_CTRL_ASSIGN         0x08        /* Device Interrupt Target
+                                               Assignment */
+
+/* I2C Device (Bus) Register */
+
+#define I2CD_FUN_CTRL_REG       0x00       /* I2CD Function Control  */
+#define   I2CD_BUFF_SEL_MASK               (0x7 << 20)
+#define   I2CD_BUFF_SEL(x)                 (x << 20)
+#define   I2CD_M_SDA_LOCK_EN               (0x1 << 16)
+#define   I2CD_MULTI_MASTER_DIS            (0x1 << 15)
+#define   I2CD_M_SCL_DRIVE_EN              (0x1 << 14)
+#define   I2CD_MSB_STS                     (0x1 << 9)
+#define   I2CD_SDA_DRIVE_1T_EN             (0x1 << 8)
+#define   I2CD_M_SDA_DRIVE_1T_EN           (0x1 << 7)
+#define   I2CD_M_HIGH_SPEED_EN             (0x1 << 6)
+#define   I2CD_DEF_ADDR_EN                 (0x1 << 5)
+#define   I2CD_DEF_ALERT_EN                (0x1 << 4)
+#define   I2CD_DEF_ARP_EN                  (0x1 << 3)
+#define   I2CD_DEF_GCALL_EN                (0x1 << 2)
+#define   I2CD_SLAVE_EN                    (0x1 << 1)
+#define   I2CD_MASTER_EN                   (0x1)
+
+#define I2CD_AC_TIMING_REG1     0x04       /* Clock and AC Timing Control #1 */
+#define I2CD_AC_TIMING_REG2     0x08       /* Clock and AC Timing Control #1 */
+#define I2CD_INTR_CTRL_REG      0x0c       /* I2CD Interrupt Control */
+#define I2CD_INTR_STS_REG       0x10       /* I2CD Interrupt Status */
+#define   I2CD_INTR_SDA_DL_TIMEOUT         (0x1 << 14)
+#define   I2CD_INTR_BUS_RECOVER_DONE       (0x1 << 13)
+#define   I2CD_INTR_SMBUS_ALERT            (0x1 << 12) /* Bus [0-3] only */
+#define   I2CD_INTR_SMBUS_ARP_ADDR         (0x1 << 11) /* Removed */
+#define   I2CD_INTR_SMBUS_DEV_ALERT_ADDR   (0x1 << 10) /* Removed */
+#define   I2CD_INTR_SMBUS_DEF_ADDR         (0x1 << 9)  /* Removed */
+#define   I2CD_INTR_GCALL_ADDR             (0x1 << 8)  /* Removed */
+#define   I2CD_INTR_SLAVE_MATCH            (0x1 << 7)  /* use RX_DONE */
+#define   I2CD_INTR_SCL_TIMEOUT            (0x1 << 6)
+#define   I2CD_INTR_ABNORMAL               (0x1 << 5)
+#define   I2CD_INTR_NORMAL_STOP            (0x1 << 4)
+#define   I2CD_INTR_ARBIT_LOSS             (0x1 << 3)
+#define   I2CD_INTR_RX_DONE                (0x1 << 2)
+#define   I2CD_INTR_TX_NAK                 (0x1 << 1)
+#define   I2CD_INTR_TX_ACK                 (0x1 << 0)
+
+#define I2CD_CMD_REG            0x14       /* I2CD Command/Status */
+#define   I2CD_SDA_OE                      (0x1 << 28)
+#define   I2CD_SDA_O                       (0x1 << 27)
+#define   I2CD_SCL_OE                      (0x1 << 26)
+#define   I2CD_SCL_O                       (0x1 << 25)
+#define   I2CD_TX_TIMING                   (0x1 << 24)
+#define   I2CD_TX_STATUS                   (0x1 << 23)
+
+#define   I2CD_TX_STATE_SHIFT              19 /* Tx State Machine */
+#define   I2CD_TX_STATE_MASK                  0xf
+#define     I2CD_IDLE                         0x0
+#define     I2CD_MACTIVE                      0x8
+#define     I2CD_MSTART                       0x9
+#define     I2CD_MSTARTR                      0xa
+#define     I2CD_MSTOP                        0xb
+#define     I2CD_MTXD                         0xc
+#define     I2CD_MRXACK                       0xd
+#define     I2CD_MRXD                         0xe
+#define     I2CD_MTXACK                       0xf
+#define     I2CD_SWAIT                        0x1
+#define     I2CD_SRXD                         0x4
+#define     I2CD_STXACK                       0x5
+#define     I2CD_STXD                         0x6
+#define     I2CD_SRXACK                       0x7
+#define     I2CD_RECOVER                      0x3
+
+#define   I2CD_SCL_LINE_STS                (0x1 << 18)
+#define   I2CD_SDA_LINE_STS                (0x1 << 17)
+#define   I2CD_BUS_BUSY_STS                (0x1 << 16)
+#define   I2CD_SDA_OE_OUT_DIR              (0x1 << 15)
+#define   I2CD_SDA_O_OUT_DIR               (0x1 << 14)
+#define   I2CD_SCL_OE_OUT_DIR              (0x1 << 13)
+#define   I2CD_SCL_O_OUT_DIR               (0x1 << 12)
+#define   I2CD_BUS_RECOVER_CMD_EN          (0x1 << 11)
+#define   I2CD_S_ALT_EN                    (0x1 << 10)
+#define   I2CD_RX_DMA_ENABLE               (0x1 << 9)
+#define   I2CD_TX_DMA_ENABLE               (0x1 << 8)
+
+/* Command Bit */
+#define   I2CD_M_STOP_CMD                  (0x1 << 5)
+#define   I2CD_M_S_RX_CMD_LAST             (0x1 << 4)
+#define   I2CD_M_RX_CMD                    (0x1 << 3)
+#define   I2CD_S_TX_CMD                    (0x1 << 2)
+#define   I2CD_M_TX_CMD                    (0x1 << 1)
+#define   I2CD_M_START_CMD                 (0x1)
+
+#define I2CD_DEV_ADDR_REG       0x18       /* Slave Device Address */
+#define I2CD_BUF_CTRL_REG       0x1c       /* Pool Buffer Control */
+#define I2CD_BYTE_BUF_REG       0x20       /* Transmit/Receive Byte Buffer */
+#define   I2CD_BYTE_BUF_TX_SHIFT           0
+#define   I2CD_BYTE_BUF_TX_MASK            0xff
+#define   I2CD_BYTE_BUF_RX_SHIFT           8
+#define   I2CD_BYTE_BUF_RX_MASK            0xff
+
+
+static inline bool aspeed_i2c_bus_is_master(AspeedI2CBus *bus)
+{
+    return bus->ctrl & I2CD_MASTER_EN;
+}
+
+static inline bool aspeed_i2c_bus_is_enabled(AspeedI2CBus *bus)
+{
+    return bus->ctrl & (I2CD_MASTER_EN | I2CD_SLAVE_EN);
+}
+
+static inline void aspeed_i2c_bus_raise_interrupt(AspeedI2CBus *bus)
+{
+    bus->intr_status &= bus->intr_ctrl;
+    if (bus->intr_status) {
+        bus->controller->intr_status |= 1 << bus->id;
+        qemu_irq_raise(bus->controller->irq);
+    }
+}
+
+static uint64_t aspeed_i2c_bus_read(void *opaque, hwaddr offset,
+                                    unsigned size)
+{
+    AspeedI2CBus *bus = opaque;
+
+    switch (offset) {
+    case I2CD_FUN_CTRL_REG:
+        return bus->ctrl;
+    case I2CD_AC_TIMING_REG1:
+        return bus->timing[0];
+    case I2CD_AC_TIMING_REG2:
+        return bus->timing[1];
+    case I2CD_INTR_CTRL_REG:
+        return bus->intr_ctrl;
+    case I2CD_INTR_STS_REG:
+        return bus->intr_status;
+    case I2CD_BYTE_BUF_REG:
+        return bus->buf;
+    case I2CD_CMD_REG:
+        return bus->cmd | (i2c_bus_busy(bus->bus) << 16);
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%" HWADDR_PRIx "\n", __func__, offset);
+        return -1;
+    }
+}
+
+static void aspeed_i2c_bus_handle_cmd(AspeedI2CBus *bus, uint64_t value)
+{
+    bus->cmd |= value & 0xFFFF;
+    bus->intr_status = 0;
+
+    if (bus->cmd & I2CD_M_START_CMD) {
+        if (i2c_start_transfer(bus->bus, extract32(bus->buf, 1, 7),
+                               extract32(bus->buf, 0, 1))) {
+            bus->intr_status |= I2CD_INTR_TX_NAK;
+        } else {
+            bus->intr_status |= I2CD_INTR_TX_ACK;
+        }
+
+    } else if (bus->cmd & I2CD_M_TX_CMD) {
+        if (i2c_send(bus->bus, bus->buf)) {
+            bus->intr_status |= (I2CD_INTR_TX_NAK | I2CD_INTR_ABNORMAL);
+            i2c_end_transfer(bus->bus);
+        } else {
+            bus->intr_status |= I2CD_INTR_TX_ACK;
+        }
+
+    } else if (bus->cmd & I2CD_M_RX_CMD) {
+        int ret = i2c_recv(bus->bus);
+        if (ret < 0) {
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: read failed\n", __func__);
+            ret = 0xff;
+        } else {
+            bus->intr_status |= I2CD_INTR_RX_DONE;
+        }
+        bus->buf = (ret & I2CD_BYTE_BUF_RX_MASK) << I2CD_BYTE_BUF_RX_SHIFT;
+    }
+
+    if (bus->cmd & (I2CD_M_STOP_CMD | I2CD_M_S_RX_CMD_LAST)) {
+        if (!i2c_bus_busy(bus->bus)) {
+            bus->intr_status |= I2CD_INTR_ABNORMAL;
+        } else {
+            i2c_end_transfer(bus->bus);
+            bus->intr_status |= I2CD_INTR_NORMAL_STOP;
+        }
+    }
+
+    /* command is handled, reset it and check for interrupts  */
+    bus->cmd &= ~0xFFFF;
+    aspeed_i2c_bus_raise_interrupt(bus);
+}
+
+static void aspeed_i2c_bus_write(void *opaque, hwaddr offset,
+                                 uint64_t value, unsigned size)
+{
+    AspeedI2CBus *bus = opaque;
+
+    switch (offset) {
+    case I2CD_FUN_CTRL_REG:
+        if (value & I2CD_SLAVE_EN) {
+            qemu_log_mask(LOG_UNIMP, "%s: slave mode not implemented\n",
+                          __func__);
+            break;
+        }
+        bus->ctrl = value & 0x0071C3FF;
+        break;
+    case I2CD_AC_TIMING_REG1:
+        bus->timing[0] = value & 0xFFFFF0F;
+        break;
+    case I2CD_AC_TIMING_REG2:
+        bus->timing[1] = value & 0x7;
+        break;
+    case I2CD_INTR_CTRL_REG:
+        bus->intr_ctrl = value & 0x7FFF;
+        break;
+    case I2CD_INTR_STS_REG:
+        bus->intr_status &= ~(value & 0x7FFF);
+        bus->controller->intr_status &= ~(1 << bus->id);
+        qemu_irq_lower(bus->controller->irq);
+        break;
+    case I2CD_DEV_ADDR_REG:
+        qemu_log_mask(LOG_UNIMP, "%s: slave mode not implemented\n",
+                      __func__);
+        break;
+    case I2CD_BYTE_BUF_REG:
+        bus->buf = (value & I2CD_BYTE_BUF_TX_MASK) << I2CD_BYTE_BUF_TX_SHIFT;
+        break;
+    case I2CD_CMD_REG:
+        if (!aspeed_i2c_bus_is_enabled(bus)) {
+            break;
+        }
+
+        if (!aspeed_i2c_bus_is_master(bus)) {
+            qemu_log_mask(LOG_UNIMP, "%s: slave mode not implemented\n",
+                          __func__);
+            break;
+        }
+
+        aspeed_i2c_bus_handle_cmd(bus, value);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+    }
+}
+
+static uint64_t aspeed_i2c_ctrl_read(void *opaque, hwaddr offset,
+                                   unsigned size)
+{
+    AspeedI2CState *s = opaque;
+
+    switch (offset) {
+    case I2C_CTRL_STATUS:
+        return s->intr_status;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        break;
+    }
+
+    return -1;
+}
+
+static void aspeed_i2c_ctrl_write(void *opaque, hwaddr offset,
+                                  uint64_t value, unsigned size)
+{
+    switch (offset) {
+    case I2C_CTRL_STATUS:
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        break;
+    }
+}
+
+static const MemoryRegionOps aspeed_i2c_bus_ops = {
+    .read = aspeed_i2c_bus_read,
+    .write = aspeed_i2c_bus_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static const MemoryRegionOps aspeed_i2c_ctrl_ops = {
+    .read = aspeed_i2c_ctrl_read,
+    .write = aspeed_i2c_ctrl_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static const VMStateDescription aspeed_i2c_bus_vmstate = {
+    .name = TYPE_ASPEED_I2C,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(id, AspeedI2CBus),
+        VMSTATE_UINT32(ctrl, AspeedI2CBus),
+        VMSTATE_UINT32_ARRAY(timing, AspeedI2CBus, 2),
+        VMSTATE_UINT32(intr_ctrl, AspeedI2CBus),
+        VMSTATE_UINT32(intr_status, AspeedI2CBus),
+        VMSTATE_UINT32(cmd, AspeedI2CBus),
+        VMSTATE_UINT32(buf, AspeedI2CBus),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription aspeed_i2c_vmstate = {
+    .name = TYPE_ASPEED_I2C,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(intr_status, AspeedI2CState),
+        VMSTATE_STRUCT_ARRAY(busses, AspeedI2CState,
+                             ASPEED_I2C_NR_BUSSES, 1, aspeed_i2c_bus_vmstate,
+                             AspeedI2CBus),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void aspeed_i2c_reset(DeviceState *dev)
+{
+    int i;
+    AspeedI2CState *s = ASPEED_I2C(dev);
+
+    s->intr_status = 0;
+
+    for (i = 0; i < ASPEED_I2C_NR_BUSSES; i++) {
+        s->busses[i].intr_ctrl = 0;
+        s->busses[i].intr_status = 0;
+        s->busses[i].cmd = 0;
+        s->busses[i].buf = 0;
+        i2c_end_transfer(s->busses[i].bus);
+    }
+}
+
+/*
+ * Address Definitions
+ *
+ *   0x000 ... 0x03F: Global Register
+ *   0x040 ... 0x07F: Device 1
+ *   0x080 ... 0x0BF: Device 2
+ *   0x0C0 ... 0x0FF: Device 3
+ *   0x100 ... 0x13F: Device 4
+ *   0x140 ... 0x17F: Device 5
+ *   0x180 ... 0x1BF: Device 6
+ *   0x1C0 ... 0x1FF: Device 7
+ *   0x200 ... 0x2FF: Buffer Pool  (unused in linux driver)
+ *   0x300 ... 0x33F: Device 8
+ *   0x340 ... 0x37F: Device 9
+ *   0x380 ... 0x3BF: Device 10
+ *   0x3C0 ... 0x3FF: Device 11
+ *   0x400 ... 0x43F: Device 12
+ *   0x440 ... 0x47F: Device 13
+ *   0x480 ... 0x4BF: Device 14
+ *   0x800 ... 0xFFF: Buffer Pool  (unused in linux driver)
+ */
+static void aspeed_i2c_realize(DeviceState *dev, Error **errp)
+{
+    int i;
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedI2CState *s = ASPEED_I2C(dev);
+
+    sysbus_init_irq(sbd, &s->irq);
+    memory_region_init_io(&s->iomem, OBJECT(s), &aspeed_i2c_ctrl_ops, s,
+                          "aspeed.i2c", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    for (i = 0; i < ASPEED_I2C_NR_BUSSES; i++) {
+        char name[16];
+        int offset = i < 7 ? 1 : 5;
+        snprintf(name, sizeof(name), "aspeed.i2c.%d", i);
+        s->busses[i].controller = s;
+        s->busses[i].id = i;
+        s->busses[i].bus = i2c_init_bus(dev, name);
+        memory_region_init_io(&s->busses[i].mr, OBJECT(dev),
+                              &aspeed_i2c_bus_ops, &s->busses[i], name, 0x40);
+        memory_region_add_subregion(&s->iomem, 0x40 * (i + offset),
+                                    &s->busses[i].mr);
+    }
+}
+
+static void aspeed_i2c_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &aspeed_i2c_vmstate;
+    dc->reset = aspeed_i2c_reset;
+    dc->realize = aspeed_i2c_realize;
+    dc->desc = "Aspeed I2C Controller";
+}
+
+static const TypeInfo aspeed_i2c_info = {
+    .name          = TYPE_ASPEED_I2C,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AspeedI2CState),
+    .class_init    = aspeed_i2c_class_init,
+};
+
+static void aspeed_i2c_register_types(void)
+{
+    type_register_static(&aspeed_i2c_info);
+}
+
+type_init(aspeed_i2c_register_types)
+
+
+I2CBus *aspeed_i2c_get_bus(DeviceState *dev, int busnr)
+{
+    AspeedI2CState *s = ASPEED_I2C(dev);
+    I2CBus *bus = NULL;
+
+    if (busnr >= 0 && busnr < ASPEED_I2C_NR_BUSSES) {
+        bus = s->busses[busnr].bus;
+    }
+
+    return bus;
+}
diff --git a/hw/i2c/bitbang_i2c.c b/hw/i2c/bitbang_i2c.c
index 6d1bb03..d3a2989 100644
--- a/hw/i2c/bitbang_i2c.c
+++ b/hw/i2c/bitbang_i2c.c
@@ -9,6 +9,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "bitbang_i2c.h"
 #include "hw/sysbus.h"
@@ -209,13 +210,14 @@
     }
 }
 
-static int gpio_i2c_init(SysBusDevice *sbd)
+static void gpio_i2c_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    GPIOI2CState *s = GPIO_I2C(dev);
+    DeviceState *dev = DEVICE(obj);
+    GPIOI2CState *s = GPIO_I2C(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     I2CBus *bus;
 
-    memory_region_init(&s->dummy_iomem, OBJECT(s), "gpio_i2c", 0);
+    memory_region_init(&s->dummy_iomem, obj, "gpio_i2c", 0);
     sysbus_init_mmio(sbd, &s->dummy_iomem);
 
     bus = i2c_init_bus(dev, "i2c");
@@ -223,16 +225,12 @@
 
     qdev_init_gpio_in(dev, bitbang_i2c_gpio_set, 2);
     qdev_init_gpio_out(dev, &s->out, 1);
-
-    return 0;
 }
 
 static void gpio_i2c_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = gpio_i2c_init;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
     dc->desc = "Virtual GPIO to I2C bridge";
 }
@@ -241,6 +239,7 @@
     .name          = TYPE_GPIO_I2C,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(GPIOI2CState),
+    .instance_init = gpio_i2c_init,
     .class_init    = gpio_i2c_class_init,
 };
 
diff --git a/hw/i2c/core.c b/hw/i2c/core.c
index 5a64026..4afbe0b 100644
--- a/hw/i2c/core.c
+++ b/hw/i2c/core.c
@@ -7,14 +7,24 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/i2c/i2c.h"
 
+typedef struct I2CNode I2CNode;
+
+struct I2CNode {
+    I2CSlave *elt;
+    QLIST_ENTRY(I2CNode) next;
+};
+
+#define I2C_BROADCAST 0x00
+
 struct I2CBus
 {
     BusState qbus;
-    I2CSlave *current_dev;
-    I2CSlave *dev;
+    QLIST_HEAD(, I2CNode) current_devs;
     uint8_t saved_address;
+    bool broadcast;
 };
 
 static Property i2c_props[] = {
@@ -35,17 +45,14 @@
 {
     I2CBus *bus = opaque;
 
-    bus->saved_address = bus->current_dev ? bus->current_dev->address : -1;
-}
-
-static int i2c_bus_post_load(void *opaque, int version_id)
-{
-    I2CBus *bus = opaque;
-
-    /* The bus is loaded before attached devices, so load and save the
-       current device id.  Devices will check themselves as loaded.  */
-    bus->current_dev = NULL;
-    return 0;
+    bus->saved_address = -1;
+    if (!QLIST_EMPTY(&bus->current_devs)) {
+        if (!bus->broadcast) {
+            bus->saved_address = QLIST_FIRST(&bus->current_devs)->elt->address;
+        } else {
+            bus->saved_address = I2C_BROADCAST;
+        }
+    }
 }
 
 static const VMStateDescription vmstate_i2c_bus = {
@@ -53,7 +60,6 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .pre_save = i2c_bus_pre_save,
-    .post_load = i2c_bus_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(saved_address, I2CBus),
         VMSTATE_END_OF_LIST()
@@ -66,6 +72,7 @@
     I2CBus *bus;
 
     bus = I2C_BUS(qbus_create(TYPE_I2C_BUS, parent, name));
+    QLIST_INIT(&bus->current_devs);
     vmstate_register(NULL, -1, &vmstate_i2c_bus, bus);
     return bus;
 }
@@ -78,7 +85,7 @@
 /* Return nonzero if bus is busy.  */
 int i2c_bus_busy(I2CBus *bus)
 {
-    return bus->current_dev != NULL;
+    return !QLIST_EMPTY(&bus->current_devs);
 }
 
 /* Returns non-zero if the address is not valid.  */
@@ -86,95 +93,127 @@
 int i2c_start_transfer(I2CBus *bus, uint8_t address, int recv)
 {
     BusChild *kid;
-    I2CSlave *slave = NULL;
     I2CSlaveClass *sc;
+    I2CNode *node;
+
+    if (address == I2C_BROADCAST) {
+        /*
+         * This is a broadcast, the current_devs will be all the devices of the
+         * bus.
+         */
+        bus->broadcast = true;
+    }
 
     QTAILQ_FOREACH(kid, &bus->qbus.children, sibling) {
         DeviceState *qdev = kid->child;
         I2CSlave *candidate = I2C_SLAVE(qdev);
-        if (candidate->address == address) {
-            slave = candidate;
-            break;
+        if ((candidate->address == address) || (bus->broadcast)) {
+            node = g_malloc(sizeof(struct I2CNode));
+            node->elt = candidate;
+            QLIST_INSERT_HEAD(&bus->current_devs, node, next);
+            if (!bus->broadcast) {
+                break;
+            }
         }
     }
 
-    if (!slave) {
+    if (QLIST_EMPTY(&bus->current_devs)) {
         return 1;
     }
 
-    sc = I2C_SLAVE_GET_CLASS(slave);
-    /* If the bus is already busy, assume this is a repeated
-       start condition.  */
-    bus->current_dev = slave;
-    if (sc->event) {
-        sc->event(slave, recv ? I2C_START_RECV : I2C_START_SEND);
+    QLIST_FOREACH(node, &bus->current_devs, next) {
+        sc = I2C_SLAVE_GET_CLASS(node->elt);
+        /* If the bus is already busy, assume this is a repeated
+           start condition.  */
+        if (sc->event) {
+            sc->event(node->elt, recv ? I2C_START_RECV : I2C_START_SEND);
+        }
     }
     return 0;
 }
 
 void i2c_end_transfer(I2CBus *bus)
 {
-    I2CSlave *dev = bus->current_dev;
     I2CSlaveClass *sc;
+    I2CNode *node, *next;
 
-    if (!dev) {
+    if (QLIST_EMPTY(&bus->current_devs)) {
         return;
     }
 
-    sc = I2C_SLAVE_GET_CLASS(dev);
-    if (sc->event) {
-        sc->event(dev, I2C_FINISH);
+    QLIST_FOREACH_SAFE(node, &bus->current_devs, next, next) {
+        sc = I2C_SLAVE_GET_CLASS(node->elt);
+        if (sc->event) {
+            sc->event(node->elt, I2C_FINISH);
+        }
+        QLIST_REMOVE(node, next);
+        g_free(node);
     }
+    bus->broadcast = false;
+}
 
-    bus->current_dev = NULL;
+int i2c_send_recv(I2CBus *bus, uint8_t *data, bool send)
+{
+    I2CSlaveClass *sc;
+    I2CNode *node;
+    int ret = 0;
+
+    if (send) {
+        QLIST_FOREACH(node, &bus->current_devs, next) {
+            sc = I2C_SLAVE_GET_CLASS(node->elt);
+            if (sc->send) {
+                ret = ret || sc->send(node->elt, *data);
+            } else {
+                ret = -1;
+            }
+        }
+        return ret ? -1 : 0;
+    } else {
+        if ((QLIST_EMPTY(&bus->current_devs)) || (bus->broadcast)) {
+            return -1;
+        }
+
+        sc = I2C_SLAVE_GET_CLASS(QLIST_FIRST(&bus->current_devs)->elt);
+        if (sc->recv) {
+            ret = sc->recv(QLIST_FIRST(&bus->current_devs)->elt);
+            if (ret < 0) {
+                return ret;
+            } else {
+                *data = ret;
+                return 0;
+            }
+        }
+        return -1;
+    }
 }
 
 int i2c_send(I2CBus *bus, uint8_t data)
 {
-    I2CSlave *dev = bus->current_dev;
-    I2CSlaveClass *sc;
-
-    if (!dev) {
-        return -1;
-    }
-
-    sc = I2C_SLAVE_GET_CLASS(dev);
-    if (sc->send) {
-        return sc->send(dev, data);
-    }
-
-    return -1;
+    return i2c_send_recv(bus, &data, true);
 }
 
 int i2c_recv(I2CBus *bus)
 {
-    I2CSlave *dev = bus->current_dev;
-    I2CSlaveClass *sc;
+    uint8_t data;
+    int ret = i2c_send_recv(bus, &data, false);
 
-    if (!dev) {
-        return -1;
-    }
-
-    sc = I2C_SLAVE_GET_CLASS(dev);
-    if (sc->recv) {
-        return sc->recv(dev);
-    }
-
-    return -1;
+    return ret < 0 ? ret : data;
 }
 
 void i2c_nack(I2CBus *bus)
 {
-    I2CSlave *dev = bus->current_dev;
     I2CSlaveClass *sc;
+    I2CNode *node;
 
-    if (!dev) {
+    if (QLIST_EMPTY(&bus->current_devs)) {
         return;
     }
 
-    sc = I2C_SLAVE_GET_CLASS(dev);
-    if (sc->event) {
-        sc->event(dev, I2C_NACK);
+    QLIST_FOREACH(node, &bus->current_devs, next) {
+        sc = I2C_SLAVE_GET_CLASS(node->elt);
+        if (sc->event) {
+            sc->event(node->elt, I2C_NACK);
+        }
     }
 }
 
@@ -182,9 +221,14 @@
 {
     I2CSlave *dev = opaque;
     I2CBus *bus;
+    I2CNode *node;
+
     bus = I2C_BUS(qdev_get_parent_bus(DEVICE(dev)));
-    if (bus->saved_address == dev->address) {
-        bus->current_dev = dev;
+    if ((bus->saved_address == dev->address) ||
+        (bus->saved_address == I2C_BROADCAST)) {
+        node = g_malloc(sizeof(struct I2CNode));
+        node->elt = dev;
+        QLIST_INSERT_HEAD(&bus->current_devs, node, next);
     }
     return 0;
 }
diff --git a/hw/i2c/exynos4210_i2c.c b/hw/i2c/exynos4210_i2c.c
index fb99dfd..c96fa7d 100644
--- a/hw/i2c/exynos4210_i2c.c
+++ b/hw/i2c/exynos4210_i2c.c
@@ -20,6 +20,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu/timer.h"
 #include "hw/sysbus.h"
 #include "hw/i2c/i2c.h"
@@ -298,33 +299,32 @@
     s->scl_free = true;
 }
 
-static int exynos4210_i2c_realize(SysBusDevice *sbd)
+static void exynos4210_i2c_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    Exynos4210I2CState *s = EXYNOS4_I2C(dev);
+    DeviceState *dev = DEVICE(obj);
+    Exynos4210I2CState *s = EXYNOS4_I2C(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_i2c_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_i2c_ops, s,
                           TYPE_EXYNOS4_I2C, EXYNOS4_I2C_MEM_SIZE);
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq);
     s->bus = i2c_init_bus(dev, "i2c");
-    return 0;
 }
 
 static void exynos4210_i2c_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *sbdc = SYS_BUS_DEVICE_CLASS(klass);
 
     dc->vmsd = &exynos4210_i2c_vmstate;
     dc->reset = exynos4210_i2c_reset;
-    sbdc->init = exynos4210_i2c_realize;
 }
 
 static const TypeInfo exynos4210_i2c_type_info = {
     .name = TYPE_EXYNOS4_I2C,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210I2CState),
+    .instance_init = exynos4210_i2c_init,
     .class_init = exynos4210_i2c_class_init,
 };
 
diff --git a/hw/i2c/i2c-ddc.c b/hw/i2c/i2c-ddc.c
new file mode 100644
index 0000000..1227212
--- /dev/null
+++ b/hw/i2c/i2c-ddc.c
@@ -0,0 +1,308 @@
+/* A simple I2C slave for returning monitor EDID data via DDC.
+ *
+ * Copyright (c) 2011 Linaro Limited
+ * Written by Peter Maydell
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 2 as
+ *  published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "hw/i2c/i2c.h"
+#include "hw/i2c/i2c-ddc.h"
+
+#ifndef DEBUG_I2CDDC
+#define DEBUG_I2CDDC 0
+#endif
+
+#define DPRINTF(fmt, ...) do {                                                 \
+    if (DEBUG_I2CDDC) {                                                        \
+        qemu_log("i2c-ddc: " fmt , ## __VA_ARGS__);                            \
+    }                                                                          \
+} while (0);
+
+/* Structure defining a monitor's characteristics in a
+ * readable format: this should be passed to build_edid_blob()
+ * to convert it into the 128 byte binary EDID blob.
+ * Not all bits of the EDID are customisable here.
+ */
+struct EDIDData {
+    char manuf_id[3]; /* three upper case letters */
+    uint16_t product_id;
+    uint32_t serial_no;
+    uint8_t manuf_week;
+    int manuf_year;
+    uint8_t h_cm;
+    uint8_t v_cm;
+    uint8_t gamma;
+    char monitor_name[14];
+    char serial_no_string[14];
+    /* Range limits */
+    uint8_t vmin; /* Hz */
+    uint8_t vmax; /* Hz */
+    uint8_t hmin; /* kHz */
+    uint8_t hmax; /* kHz */
+    uint8_t pixclock; /* MHz / 10 */
+    uint8_t timing_data[18];
+};
+
+typedef struct EDIDData EDIDData;
+
+/* EDID data for a simple LCD monitor */
+static const EDIDData lcd_edid = {
+    /* The manuf_id ought really to be an assigned EISA ID */
+    .manuf_id = "QMU",
+    .product_id = 0,
+    .serial_no = 1,
+    .manuf_week = 1,
+    .manuf_year = 2011,
+    .h_cm = 40,
+    .v_cm = 30,
+    .gamma = 0x78,
+    .monitor_name = "QEMU monitor",
+    .serial_no_string = "1",
+    .vmin = 40,
+    .vmax = 120,
+    .hmin = 30,
+    .hmax = 100,
+    .pixclock = 18,
+    .timing_data = {
+        /* Borrowed from a 21" LCD */
+        0x48, 0x3f, 0x40, 0x30, 0x62, 0xb0, 0x32, 0x40, 0x40,
+        0xc0, 0x13, 0x00, 0x98, 0x32, 0x11, 0x00, 0x00, 0x1e
+    }
+};
+
+static uint8_t manuf_char_to_int(char c)
+{
+    return (c - 'A') & 0x1f;
+}
+
+static void write_ascii_descriptor_block(uint8_t *descblob, uint8_t blocktype,
+                                         const char *string)
+{
+    /* Write an EDID Descriptor Block of the "ascii string" type */
+    int i;
+    descblob[0] = descblob[1] = descblob[2] = descblob[4] = 0;
+    descblob[3] = blocktype;
+    /* The rest is 13 bytes of ASCII; if less then the rest must
+     * be filled with newline then spaces
+     */
+    for (i = 5; i < 19; i++) {
+        descblob[i] = string[i - 5];
+        if (!descblob[i]) {
+            break;
+        }
+    }
+    if (i < 19) {
+        descblob[i++] = '\n';
+    }
+    for ( ; i < 19; i++) {
+        descblob[i] = ' ';
+    }
+}
+
+static void write_range_limits_descriptor(const EDIDData *edid,
+                                          uint8_t *descblob)
+{
+    int i;
+    descblob[0] = descblob[1] = descblob[2] = descblob[4] = 0;
+    descblob[3] = 0xfd;
+    descblob[5] = edid->vmin;
+    descblob[6] = edid->vmax;
+    descblob[7] = edid->hmin;
+    descblob[8] = edid->hmax;
+    descblob[9] = edid->pixclock;
+    descblob[10] = 0;
+    descblob[11] = 0xa;
+    for (i = 12; i < 19; i++) {
+        descblob[i] = 0x20;
+    }
+}
+
+static void build_edid_blob(const EDIDData *edid, uint8_t *blob)
+{
+    /* Write an EDID 1.3 format blob (128 bytes) based
+     * on the EDIDData structure.
+     */
+    int i;
+    uint8_t cksum;
+
+    /* 00-07 : header */
+    blob[0] = blob[7] = 0;
+    for (i = 1 ; i < 7; i++) {
+        blob[i] = 0xff;
+    }
+    /* 08-09 : manufacturer ID */
+    blob[8] = (manuf_char_to_int(edid->manuf_id[0]) << 2)
+        | (manuf_char_to_int(edid->manuf_id[1]) >> 3);
+    blob[9] = (manuf_char_to_int(edid->manuf_id[1]) << 5)
+        | manuf_char_to_int(edid->manuf_id[2]);
+    /* 10-11 : product ID code */
+    blob[10] = edid->product_id;
+    blob[11] = edid->product_id >> 8;
+    blob[12] = edid->serial_no;
+    blob[13] = edid->serial_no >> 8;
+    blob[14] = edid->serial_no >> 16;
+    blob[15] = edid->serial_no >> 24;
+    /* 16 : week of manufacture */
+    blob[16] = edid->manuf_week;
+    /* 17 : year of manufacture - 1990 */
+    blob[17] = edid->manuf_year - 1990;
+    /* 18, 19 : EDID version and revision */
+    blob[18] = 1;
+    blob[19] = 3;
+    /* 20 - 24 : basic display parameters */
+    /* We are always a digital display */
+    blob[20] = 0x80;
+    /* 21, 22 : max h/v size in cm */
+    blob[21] = edid->h_cm;
+    blob[22] = edid->v_cm;
+    /* 23 : gamma (divide by 100 then add 1 for actual value) */
+    blob[23] = edid->gamma;
+    /* 24 feature support: no power management, RGB, preferred timing mode,
+     * standard colour space
+     */
+    blob[24] = 0x0e;
+    /* 25 - 34 : chromaticity coordinates. These are the
+     * standard sRGB chromaticity values
+     */
+    blob[25] = 0xee;
+    blob[26] = 0x91;
+    blob[27] = 0xa3;
+    blob[28] = 0x54;
+    blob[29] = 0x4c;
+    blob[30] = 0x99;
+    blob[31] = 0x26;
+    blob[32] = 0x0f;
+    blob[33] = 0x50;
+    blob[34] = 0x54;
+    /* 35, 36 : Established timings: claim to support everything */
+    blob[35] = blob[36] = 0xff;
+    /* 37 : manufacturer's reserved timing: none */
+    blob[37] = 0;
+    /* 38 - 53 : standard timing identification
+     * don't claim anything beyond what the 'established timings'
+     * already provide. Unused slots must be (0x1, 0x1)
+     */
+    for (i = 38; i < 54; i++) {
+        blob[i] = 0x1;
+    }
+    /* 54 - 71 : descriptor block 1 : must be preferred timing data */
+    memcpy(blob + 54, edid->timing_data, 18);
+    /* 72 - 89, 90 - 107, 108 - 125 : descriptor block 2, 3, 4
+     * Order not important, but we must have a monitor name and a
+     * range limits descriptor.
+     */
+    write_range_limits_descriptor(edid, blob + 72);
+    write_ascii_descriptor_block(blob + 90, 0xfc, edid->monitor_name);
+    write_ascii_descriptor_block(blob + 108, 0xff, edid->serial_no_string);
+
+    /* 126 : extension flag */
+    blob[126] = 0;
+
+    cksum = 0;
+    for (i = 0; i < 127; i++) {
+        cksum += blob[i];
+    }
+    /* 127 : checksum */
+    blob[127] = -cksum;
+    if (DEBUG_I2CDDC) {
+        qemu_hexdump((char *)blob, stdout, "", 128);
+    }
+}
+
+static void i2c_ddc_reset(DeviceState *ds)
+{
+    I2CDDCState *s = I2CDDC(ds);
+
+    s->firstbyte = false;
+    s->reg = 0;
+}
+
+static void i2c_ddc_event(I2CSlave *i2c, enum i2c_event event)
+{
+    I2CDDCState *s = I2CDDC(i2c);
+
+    if (event == I2C_START_SEND) {
+        s->firstbyte = true;
+    }
+}
+
+static int i2c_ddc_rx(I2CSlave *i2c)
+{
+    I2CDDCState *s = I2CDDC(i2c);
+
+    int value;
+    value = s->edid_blob[s->reg];
+    s->reg++;
+    return value;
+}
+
+static int i2c_ddc_tx(I2CSlave *i2c, uint8_t data)
+{
+    I2CDDCState *s = I2CDDC(i2c);
+    if (s->firstbyte) {
+        s->reg = data;
+        s->firstbyte = false;
+        DPRINTF("[EDID] Written new pointer: %u\n", data);
+        return 1;
+    }
+
+    /* Ignore all writes */
+    s->reg++;
+    return 1;
+}
+
+static void i2c_ddc_init(Object *obj)
+{
+    I2CDDCState *s = I2CDDC(obj);
+    build_edid_blob(&lcd_edid, s->edid_blob);
+}
+
+static const VMStateDescription vmstate_i2c_ddc = {
+    .name = TYPE_I2CDDC,
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(firstbyte, I2CDDCState),
+        VMSTATE_UINT8(reg, I2CDDCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void i2c_ddc_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    I2CSlaveClass *isc = I2C_SLAVE_CLASS(oc);
+
+    dc->reset = i2c_ddc_reset;
+    dc->vmsd = &vmstate_i2c_ddc;
+    isc->event = i2c_ddc_event;
+    isc->recv = i2c_ddc_rx;
+    isc->send = i2c_ddc_tx;
+}
+
+static TypeInfo i2c_ddc_info = {
+    .name = TYPE_I2CDDC,
+    .parent = TYPE_I2C_SLAVE,
+    .instance_size = sizeof(I2CDDCState),
+    .instance_init = i2c_ddc_init,
+    .class_init = i2c_ddc_class_init
+};
+
+static void ddc_register_devices(void)
+{
+    type_register_static(&i2c_ddc_info);
+}
+
+type_init(ddc_register_devices);
diff --git a/hw/i2c/imx_i2c.c b/hw/i2c/imx_i2c.c
new file mode 100644
index 0000000..37e5a62
--- /dev/null
+++ b/hw/i2c/imx_i2c.c
@@ -0,0 +1,338 @@
+/*
+ *  i.MX I2C Bus Serial Interface Emulation
+ *
+ *  Copyright (C) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/i2c/i2c.h"
+#include "qemu/log.h"
+
+#ifndef DEBUG_IMX_I2C
+#define DEBUG_IMX_I2C 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_I2C) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_I2C, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_i2c_get_regname(unsigned offset)
+{
+    switch (offset) {
+    case IADR_ADDR:
+        return "IADR";
+    case IFDR_ADDR:
+        return "IFDR";
+    case I2CR_ADDR:
+        return "I2CR";
+    case I2SR_ADDR:
+        return "I2SR";
+    case I2DR_ADDR:
+        return "I2DR";
+    default:
+        return "[?]";
+    }
+}
+
+static inline bool imx_i2c_is_enabled(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_IEN;
+}
+
+static inline bool imx_i2c_interrupt_is_enabled(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_IIEN;
+}
+
+static inline bool imx_i2c_is_master(IMXI2CState *s)
+{
+    return s->i2cr & I2CR_MSTA;
+}
+
+static void imx_i2c_reset(DeviceState *dev)
+{
+    IMXI2CState *s = IMX_I2C(dev);
+
+    if (s->address != ADDR_RESET) {
+        i2c_end_transfer(s->bus);
+    }
+
+    s->address    = ADDR_RESET;
+    s->iadr       = IADR_RESET;
+    s->ifdr       = IFDR_RESET;
+    s->i2cr       = I2CR_RESET;
+    s->i2sr       = I2SR_RESET;
+    s->i2dr_read  = I2DR_RESET;
+    s->i2dr_write = I2DR_RESET;
+}
+
+static inline void imx_i2c_raise_interrupt(IMXI2CState *s)
+{
+    /*
+     * raise an interrupt if the device is enabled and it is configured
+     * to generate some interrupts.
+     */
+    if (imx_i2c_is_enabled(s) && imx_i2c_interrupt_is_enabled(s)) {
+        s->i2sr |= I2SR_IIF;
+        qemu_irq_raise(s->irq);
+    }
+}
+
+static uint64_t imx_i2c_read(void *opaque, hwaddr offset,
+                             unsigned size)
+{
+    uint16_t value;
+    IMXI2CState *s = IMX_I2C(opaque);
+
+    switch (offset) {
+    case IADR_ADDR:
+        value = s->iadr;
+        break;
+    case IFDR_ADDR:
+        value = s->ifdr;
+        break;
+    case I2CR_ADDR:
+        value = s->i2cr;
+        break;
+    case I2SR_ADDR:
+        value = s->i2sr;
+        break;
+    case I2DR_ADDR:
+        value = s->i2dr_read;
+
+        if (imx_i2c_is_master(s)) {
+            int ret = 0xff;
+
+            if (s->address == ADDR_RESET) {
+                /* something is wrong as the address is not set */
+                qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read "
+                              "without specifying the slave address\n",
+                              TYPE_IMX_I2C, __func__);
+            } else if (s->i2cr & I2CR_MTX) {
+                qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read "
+                              "but MTX is set\n", TYPE_IMX_I2C, __func__);
+            } else {
+                /* get the next byte */
+                ret = i2c_recv(s->bus);
+
+                if (ret >= 0) {
+                    imx_i2c_raise_interrupt(s);
+                } else {
+                    qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: read failed "
+                                  "for device 0x%02x\n", TYPE_IMX_I2C,
+                                  __func__, s->address);
+                    ret = 0xff;
+                }
+            }
+
+            s->i2dr_read = ret;
+        } else {
+            qemu_log_mask(LOG_UNIMP, "[%s]%s: slave mode not implemented\n",
+                          TYPE_IMX_I2C, __func__);
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_I2C, __func__, offset);
+        value = 0;
+        break;
+    }
+
+    DPRINTF("read %s [0x%" HWADDR_PRIx "] -> 0x%02x\n",
+            imx_i2c_get_regname(offset), offset, value);
+
+    return (uint64_t)value;
+}
+
+static void imx_i2c_write(void *opaque, hwaddr offset,
+                          uint64_t value, unsigned size)
+{
+    IMXI2CState *s = IMX_I2C(opaque);
+
+    DPRINTF("write %s [0x%" HWADDR_PRIx "] <- 0x%02x\n",
+            imx_i2c_get_regname(offset), offset, (int)value);
+
+    value &= 0xff;
+
+    switch (offset) {
+    case IADR_ADDR:
+        s->iadr = value & IADR_MASK;
+        /* i2c_set_slave_address(s->bus, (uint8_t)s->iadr); */
+        break;
+    case IFDR_ADDR:
+        s->ifdr = value & IFDR_MASK;
+        break;
+    case I2CR_ADDR:
+        if (imx_i2c_is_enabled(s) && ((value & I2CR_IEN) == 0)) {
+            /* This is a soft reset. IADR is preserved during soft resets */
+            uint16_t iadr = s->iadr;
+            imx_i2c_reset(DEVICE(s));
+            s->iadr = iadr;
+        } else { /* normal write */
+            s->i2cr = value & I2CR_MASK;
+
+            if (imx_i2c_is_master(s)) {
+                /* set the bus to busy */
+                s->i2sr |= I2SR_IBB;
+            } else { /* slave mode */
+                /* bus is not busy anymore */
+                s->i2sr &= ~I2SR_IBB;
+
+                /*
+                 * if we unset the master mode then it ends the ongoing
+                 * transfer if any
+                 */
+                if (s->address != ADDR_RESET) {
+                    i2c_end_transfer(s->bus);
+                    s->address = ADDR_RESET;
+                }
+            }
+
+            if (s->i2cr & I2CR_RSTA) { /* Restart */
+                /* if this is a restart then it ends the ongoing transfer */
+                if (s->address != ADDR_RESET) {
+                    i2c_end_transfer(s->bus);
+                    s->address = ADDR_RESET;
+                    s->i2cr &= ~I2CR_RSTA;
+                }
+            }
+        }
+        break;
+    case I2SR_ADDR:
+        /*
+         * if the user writes 0 to IIF then lower the interrupt and
+         * reset the bit
+         */
+        if ((s->i2sr & I2SR_IIF) && !(value & I2SR_IIF)) {
+            s->i2sr &= ~I2SR_IIF;
+            qemu_irq_lower(s->irq);
+        }
+
+        /*
+         * if the user writes 0 to IAL, reset the bit
+         */
+        if ((s->i2sr & I2SR_IAL) && !(value & I2SR_IAL)) {
+            s->i2sr &= ~I2SR_IAL;
+        }
+
+        break;
+    case I2DR_ADDR:
+        /* if the device is not enabled, nothing to do */
+        if (!imx_i2c_is_enabled(s)) {
+            break;
+        }
+
+        s->i2dr_write = value & I2DR_MASK;
+
+        if (imx_i2c_is_master(s)) {
+            /* If this is the first write cycle then it is the slave addr */
+            if (s->address == ADDR_RESET) {
+                if (i2c_start_transfer(s->bus, extract32(s->i2dr_write, 1, 7),
+                                       extract32(s->i2dr_write, 0, 1))) {
+                    /* if non zero is returned, the address is not valid */
+                    s->i2sr |= I2SR_RXAK;
+                } else {
+                    s->address = s->i2dr_write;
+                    s->i2sr &= ~I2SR_RXAK;
+                    imx_i2c_raise_interrupt(s);
+                }
+            } else { /* This is a normal data write */
+                if (i2c_send(s->bus, s->i2dr_write)) {
+                    /* if the target return non zero then end the transfer */
+                    s->i2sr |= I2SR_RXAK;
+                    s->address = ADDR_RESET;
+                    i2c_end_transfer(s->bus);
+                } else {
+                    s->i2sr &= ~I2SR_RXAK;
+                    imx_i2c_raise_interrupt(s);
+                }
+            }
+        } else {
+            qemu_log_mask(LOG_UNIMP, "[%s]%s: slave mode not implemented\n",
+                          TYPE_IMX_I2C, __func__);
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_I2C, __func__, offset);
+        break;
+    }
+}
+
+static const MemoryRegionOps imx_i2c_ops = {
+    .read = imx_i2c_read,
+    .write = imx_i2c_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 2,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription imx_i2c_vmstate = {
+    .name = TYPE_IMX_I2C,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(address, IMXI2CState),
+        VMSTATE_UINT16(iadr, IMXI2CState),
+        VMSTATE_UINT16(ifdr, IMXI2CState),
+        VMSTATE_UINT16(i2cr, IMXI2CState),
+        VMSTATE_UINT16(i2sr, IMXI2CState),
+        VMSTATE_UINT16(i2dr_read, IMXI2CState),
+        VMSTATE_UINT16(i2dr_write, IMXI2CState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void imx_i2c_realize(DeviceState *dev, Error **errp)
+{
+    IMXI2CState *s = IMX_I2C(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &imx_i2c_ops, s, TYPE_IMX_I2C,
+                          IMX_I2C_MEM_SIZE);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq);
+    s->bus = i2c_init_bus(DEVICE(dev), "i2c");
+}
+
+static void imx_i2c_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &imx_i2c_vmstate;
+    dc->reset = imx_i2c_reset;
+    dc->realize = imx_i2c_realize;
+    dc->desc = "i.MX I2C Controller";
+}
+
+static const TypeInfo imx_i2c_type_info = {
+    .name = TYPE_IMX_I2C,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXI2CState),
+    .class_init = imx_i2c_class_init,
+};
+
+static void imx_i2c_register_types(void)
+{
+    type_register_static(&imx_i2c_type_info);
+}
+
+type_init(imx_i2c_register_types)
diff --git a/hw/i2c/omap_i2c.c b/hw/i2c/omap_i2c.c
index d63278d..f7c92ea 100644
--- a/hw/i2c/omap_i2c.c
+++ b/hw/i2c/omap_i2c.c
@@ -16,10 +16,13 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "hw/arm/omap.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
 
 #define TYPE_OMAP_I2C "omap_i2c"
 #define OMAP_I2C(obj) OBJECT_CHECK(OMAPI2CState, (obj), TYPE_OMAP_I2C)
@@ -171,9 +174,13 @@
     case 0x0c:	/* I2C_IV */
         if (s->revision >= OMAP2_INTR_REV)
             break;
-        ret = ffs(s->stat & s->mask);
-        if (ret)
-            s->stat ^= 1 << (ret - 1);
+        ret = ctz32(s->stat & s->mask);
+        if (ret != 32) {
+            s->stat ^= 1 << ret;
+            ret++;
+        } else {
+            ret = 0;
+        }
         omap_i2c_interrupts_update(s);
         return ret;
 
@@ -439,26 +446,35 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int omap_i2c_init(SysBusDevice *sbd)
+static void omap_i2c_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    OMAPI2CState *s = OMAP_I2C(dev);
+    DeviceState *dev = DEVICE(obj);
+    OMAPI2CState *s = OMAP_I2C(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    if (!s->fclk) {
-        hw_error("omap_i2c: fclk not connected\n");
-    }
-    if (s->revision >= OMAP2_INTR_REV && !s->iclk) {
-        /* Note that OMAP1 doesn't have a separate interface clock */
-        hw_error("omap_i2c: iclk not connected\n");
-    }
     sysbus_init_irq(sbd, &s->irq);
     sysbus_init_irq(sbd, &s->drq[0]);
     sysbus_init_irq(sbd, &s->drq[1]);
-    memory_region_init_io(&s->iomem, OBJECT(s), &omap_i2c_ops, s, "omap.i2c",
-                          (s->revision < OMAP2_INTR_REV) ? 0x800 : 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     s->bus = i2c_init_bus(dev, NULL);
-    return 0;
+}
+
+static void omap_i2c_realize(DeviceState *dev, Error **errp)
+{
+    OMAPI2CState *s = OMAP_I2C(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &omap_i2c_ops, s, "omap.i2c",
+                          (s->revision < OMAP2_INTR_REV) ? 0x800 : 0x1000);
+
+    if (!s->fclk) {
+        error_setg(errp, "omap_i2c: fclk not connected");
+        return;
+    }
+    if (s->revision >= OMAP2_INTR_REV && !s->iclk) {
+        /* Note that OMAP1 doesn't have a separate interface clock */
+        error_setg(errp, "omap_i2c: iclk not connected");
+        return;
+    }
 }
 
 static Property omap_i2c_properties[] = {
@@ -471,18 +487,19 @@
 static void omap_i2c_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-    k->init = omap_i2c_init;
+
     dc->props = omap_i2c_properties;
     dc->reset = omap_i2c_reset;
     /* Reason: pointer properties "iclk", "fclk" */
     dc->cannot_instantiate_with_device_add_yet = true;
+    dc->realize = omap_i2c_realize;
 }
 
 static const TypeInfo omap_i2c_info = {
     .name = TYPE_OMAP_I2C,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(OMAPI2CState),
+    .instance_init = omap_i2c_init,
     .class_init = omap_i2c_class_init,
 };
 
diff --git a/hw/i2c/pm_smbus.c b/hw/i2c/pm_smbus.c
index ce1713d..6fc3923 100644
--- a/hw/i2c/pm_smbus.c
+++ b/hw/i2c/pm_smbus.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see
  * <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/i2c/pm_smbus.h"
diff --git a/hw/i2c/smbus.c b/hw/i2c/smbus.c
index 6e27ae8..3979b3d 100644
--- a/hw/i2c/smbus.c
+++ b/hw/i2c/smbus.c
@@ -9,6 +9,7 @@
 
 /* TODO: Implement PEC.  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "hw/i2c/smbus.h"
diff --git a/hw/i2c/smbus_eeprom.c b/hw/i2c/smbus_eeprom.c
index 72c09cb..5b7bd89 100644
--- a/hw/i2c/smbus_eeprom.c
+++ b/hw/i2c/smbus_eeprom.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "hw/i2c/smbus.h"
diff --git a/hw/i2c/smbus_ich9.c b/hw/i2c/smbus_ich9.c
index 0803dc4..48fab22 100644
--- a/hw/i2c/smbus_ich9.c
+++ b/hw/i2c/smbus_ich9.c
@@ -24,6 +24,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/i2c/pm_smbus.h"
@@ -34,7 +35,6 @@
 
 #include "hw/i386/ich9.h"
 
-#define TYPE_ICH9_SMB_DEVICE "ICH9 SMB"
 #define ICH9_SMB_DEVICE(obj) \
      OBJECT_CHECK(ICH9SMBState, (obj), TYPE_ICH9_SMB_DEVICE)
 
@@ -71,7 +71,7 @@
     }
 }
 
-static int ich9_smbus_initfn(PCIDevice *d)
+static void ich9_smbus_realize(PCIDevice *d, Error **errp)
 {
     ICH9SMBState *s = ICH9_SMB_DEVICE(d);
 
@@ -84,7 +84,6 @@
     pm_smbus_init(&d->qdev, &s->smb);
     pci_register_bar(d, ICH9_SMB_SMB_BASE_BAR, PCI_BASE_ADDRESS_SPACE_IO,
                      &s->smb.io);
-    return 0;
 }
 
 static void ich9_smb_class_init(ObjectClass *klass, void *data)
@@ -98,7 +97,7 @@
     k->class_id = PCI_CLASS_SERIAL_SMBUS;
     dc->vmsd = &vmstate_ich9_smbus;
     dc->desc = "ICH9 SMBUS Bridge";
-    k->init = ich9_smbus_initfn;
+    k->realize = ich9_smbus_realize;
     k->config_write = ich9_smbus_write_config;
     /*
      * Reason: part of ICH9 southbridge, needs to be wired up by
diff --git a/hw/i2c/versatile_i2c.c b/hw/i2c/versatile_i2c.c
index 3c0c2c1..da9f298 100644
--- a/hw/i2c/versatile_i2c.c
+++ b/hw/i2c/versatile_i2c.c
@@ -21,8 +21,10 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "bitbang_i2c.h"
+#include "qemu/log.h"
 
 #define TYPE_VERSATILE_I2C "versatile_i2c"
 #define VERSATILE_I2C(obj) \
@@ -77,32 +79,25 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int versatile_i2c_init(SysBusDevice *sbd)
+static void versatile_i2c_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    VersatileI2CState *s = VERSATILE_I2C(dev);
+    DeviceState *dev = DEVICE(obj);
+    VersatileI2CState *s = VERSATILE_I2C(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     I2CBus *bus;
 
     bus = i2c_init_bus(dev, "i2c");
     s->bitbang = bitbang_i2c_init(bus);
-    memory_region_init_io(&s->iomem, OBJECT(s), &versatile_i2c_ops, s,
+    memory_region_init_io(&s->iomem, obj, &versatile_i2c_ops, s,
                           "versatile_i2c", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
-    return 0;
-}
-
-static void versatile_i2c_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-
-    k->init = versatile_i2c_init;
 }
 
 static const TypeInfo versatile_i2c_info = {
     .name          = TYPE_VERSATILE_I2C,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(VersatileI2CState),
-    .class_init    = versatile_i2c_class_init,
+    .instance_init = versatile_i2c_init,
 };
 
 static void versatile_i2c_register_types(void)
diff --git a/hw/i386/Makefile.objs b/hw/i386/Makefile.objs
index 9d419ad..90e94ff 100644
--- a/hw/i386/Makefile.objs
+++ b/hw/i386/Makefile.objs
@@ -1,37 +1,10 @@
 obj-$(CONFIG_KVM) += kvm/
-obj-y += multiboot.o smbios.o
+obj-y += multiboot.o
 obj-y += pc.o pc_piix.o pc_q35.o
 obj-y += pc_sysfw.o
-obj-y += intel_iommu.o
+obj-y += x86-iommu.o intel_iommu.o
 obj-$(CONFIG_XEN) += ../xenpv/ xen/
 
 obj-y += kvmvapic.o
 obj-y += acpi-build.o
-obj-y += bios-linker-loader.o
-hw/i386/acpi-build.o: hw/i386/acpi-build.c hw/i386/acpi-dsdt.hex \
-	hw/i386/ssdt-proc.hex hw/i386/ssdt-pcihp.hex hw/i386/ssdt-misc.hex \
-	hw/i386/acpi-dsdt.hex hw/i386/q35-acpi-dsdt.hex \
-	hw/i386/q35-acpi-dsdt.hex hw/i386/ssdt-mem.hex \
-	hw/i386/ssdt-tpm.hex
-
-iasl-option=$(shell if test -z "`$(1) $(2) 2>&1 > /dev/null`" \
-    ; then echo "$(2)"; else echo "$(3)"; fi ;)
-
-ifdef IASL
-#IASL Present. Generate hex files from .dsl
-hw/i386/%.hex: $(SRC_PATH)/hw/i386/%.dsl $(SRC_PATH)/scripts/acpi_extract_preprocess.py $(SRC_PATH)/scripts/acpi_extract.py
-	$(call quiet-command, $(CPP) -x c -P $(QEMU_DGFLAGS) $(QEMU_INCLUDES) $< -o $*.dsl.i.orig, "  CPP $(TARGET_DIR)$*.dsl.i.orig")
-	$(call quiet-command, $(PYTHON) $(SRC_PATH)/scripts/acpi_extract_preprocess.py $*.dsl.i.orig > $*.dsl.i, "  ACPI_PREPROCESS $(TARGET_DIR)$*.dsl.i")
-	$(call quiet-command, $(IASL) $(call iasl-option,$(IASL),-Pn,) -vs -l -tc -p $* $*.dsl.i $(if $(V), , > /dev/null) 2>&1 ,"  IASL $(TARGET_DIR)$*.dsl.i")
-	$(call quiet-command, $(PYTHON) $(SRC_PATH)/scripts/acpi_extract.py $*.lst > $*.off, "  ACPI_EXTRACT $(TARGET_DIR)$*.off")
-	$(call quiet-command, cat $*.off > $@, "  CAT $(TARGET_DIR)$@")
-else
-#IASL Not present. Restore pre-generated hex files.
-hw/i386/%.hex: $(SRC_PATH)/hw/i386/%.hex.generated
-	$(call quiet-command, cp -f $< $@, "  CP $(TARGET_DIR)$@")
-endif
-
-.PHONY: cleanhex
-cleanhex:
-	rm -f hw/i386/*hex
-clean: cleanhex
+obj-y += pci-assign-load-rom.o
diff --git a/hw/i386/acpi-build.c b/hw/i386/acpi-build.c
index b37a397..41fdeec 100644
--- a/hw/i386/acpi-build.c
+++ b/hw/i386/acpi-build.c
@@ -20,28 +20,31 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "acpi-build.h"
-#include <stddef.h>
-#include <glib.h>
 #include "qemu-common.h"
 #include "qemu/bitmap.h"
-#include "qemu/osdep.h"
-#include "qemu/range.h"
 #include "qemu/error-report.h"
 #include "hw/pci/pci.h"
 #include "qom/cpu.h"
 #include "hw/i386/pc.h"
 #include "target-i386/cpu.h"
 #include "hw/timer/hpet.h"
-#include "hw/i386/acpi-defs.h"
+#include "hw/acpi/acpi-defs.h"
 #include "hw/acpi/acpi.h"
+#include "hw/acpi/cpu.h"
 #include "hw/nvram/fw_cfg.h"
-#include "bios-linker-loader.h"
+#include "hw/acpi/bios-linker-loader.h"
 #include "hw/loader.h"
 #include "hw/isa/isa.h"
+#include "hw/block/fdc.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "sysemu/tpm.h"
 #include "hw/acpi/tpm.h"
+#include "sysemu/tpm_backend.h"
+#include "hw/timer/mc146818rtc_regs.h"
+#include "sysemu/numa.h"
 
 /* Supported chipsets: */
 #include "hw/acpi/piix4.h"
@@ -49,14 +52,20 @@
 #include "hw/i386/ich9.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci-host/q35.h"
-#include "hw/i386/intel_iommu.h"
+#include "hw/i386/x86-iommu.h"
+#include "hw/timer/hpet.h"
 
-#include "hw/i386/q35-acpi-dsdt.hex"
-#include "hw/i386/acpi-dsdt.hex"
+#include "hw/acpi/aml-build.h"
 
 #include "qapi/qmp/qint.h"
 #include "qom/qom-qobject.h"
-#include "exec/ram_addr.h"
+#include "hw/i386/x86-iommu.h"
+
+#include "hw/acpi/ipmi.h"
+
+#ifdef CONFIG_ANDROID
+#include "hw/acpi/goldfish_defs.h"
+#endif
 
 /* These are used to size the ACPI tables for -M pc-i440fx-1.7 and
  * -M pc-i440fx-2.0.  Even if the actual amount of AML generated grows
@@ -68,9 +77,16 @@
 
 #define ACPI_BUILD_TABLE_SIZE             0x20000
 
-typedef struct AcpiCpuInfo {
-    DECLARE_BITMAP(found_cpus, ACPI_CPU_HOTPLUG_ID_LIMIT);
-} AcpiCpuInfo;
+/* #define DEBUG_ACPI_BUILD */
+#ifdef DEBUG_ACPI_BUILD
+#define ACPI_BUILD_DPRINTF(fmt, ...)        \
+    do {printf("ACPI_BUILD: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define ACPI_BUILD_DPRINTF(fmt, ...)
+#endif
+
+/* Default IOAPIC ID */
+#define ACPI_BUILD_IOAPIC_ID 0x0
 
 typedef struct AcpiMcfgInfo {
     uint64_t mcfg_base;
@@ -88,15 +104,21 @@
     uint32_t gpe0_blk;
     uint32_t gpe0_blk_len;
     uint32_t io_base;
+    uint16_t cpu_hp_io_base;
+    uint16_t mem_hp_io_base;
+    uint16_t mem_hp_io_len;
+    uint16_t pcihp_io_base;
+    uint16_t pcihp_io_len;
 } AcpiPmInfo;
 
 typedef struct AcpiMiscInfo {
+    bool is_piix4;
     bool has_hpet;
-    bool has_tpm;
-    DECLARE_BITMAP(slot_hotplug_enable, PCI_SLOT_MAX);
+    TPMVersion tpm_version;
     const unsigned char *dsdt_code;
     unsigned dsdt_size;
     uint16_t pvpanic_port;
+    uint16_t applesmc_io_base;
 } AcpiMiscInfo;
 
 typedef struct AcpiBuildPciBusHotplugState {
@@ -106,54 +128,6 @@
     bool pcihp_bridge_en;
 } AcpiBuildPciBusHotplugState;
 
-static void acpi_get_dsdt(AcpiMiscInfo *info)
-{
-    uint16_t *applesmc_sta;
-    Object *piix = piix4_pm_find();
-    Object *lpc = ich9_lpc_find();
-    assert(!!piix != !!lpc);
-
-    if (piix) {
-        info->dsdt_code = AcpiDsdtAmlCode;
-        info->dsdt_size = sizeof AcpiDsdtAmlCode;
-        applesmc_sta = piix_dsdt_applesmc_sta;
-    }
-    if (lpc) {
-        info->dsdt_code = Q35AcpiDsdtAmlCode;
-        info->dsdt_size = sizeof Q35AcpiDsdtAmlCode;
-        applesmc_sta = q35_dsdt_applesmc_sta;
-    }
-
-    /* Patch in appropriate value for AppleSMC _STA */
-    *(uint8_t *)(info->dsdt_code + *applesmc_sta) =
-        applesmc_find() ? 0x0b : 0x00;
-}
-
-static
-int acpi_add_cpu_info(Object *o, void *opaque)
-{
-    AcpiCpuInfo *cpu = opaque;
-    uint64_t apic_id;
-
-    if (object_dynamic_cast(o, TYPE_CPU)) {
-        apic_id = object_property_get_int(o, "apic-id", NULL);
-        assert(apic_id < ACPI_CPU_HOTPLUG_ID_LIMIT);
-
-        set_bit(apic_id, cpu->found_cpus);
-    }
-
-    object_child_foreach(o, acpi_add_cpu_info, opaque);
-    return 0;
-}
-
-static void acpi_get_cpu_info(AcpiCpuInfo *cpu)
-{
-    Object *root = object_get_root();
-
-    memset(cpu->found_cpus, 0, sizeof cpu->found_cpus);
-    object_child_foreach(root, acpi_add_cpu_info, cpu);
-}
-
 static void acpi_get_pm_info(AcpiPmInfo *pm)
 {
     Object *piix = piix4_pm_find();
@@ -161,14 +135,26 @@
     Object *obj = NULL;
     QObject *o;
 
+    pm->cpu_hp_io_base = 0;
+    pm->pcihp_io_base = 0;
+    pm->pcihp_io_len = 0;
     if (piix) {
         obj = piix;
+        pm->cpu_hp_io_base = PIIX4_CPU_HOTPLUG_IO_BASE;
+        pm->pcihp_io_base =
+            object_property_get_int(obj, ACPI_PCIHP_IO_BASE_PROP, NULL);
+        pm->pcihp_io_len =
+            object_property_get_int(obj, ACPI_PCIHP_IO_LEN_PROP, NULL);
     }
     if (lpc) {
         obj = lpc;
+        pm->cpu_hp_io_base = ICH9_CPU_HOTPLUG_IO_BASE;
     }
     assert(obj);
 
+    pm->mem_hp_io_base = ACPI_MEMORY_HOTPLUG_BASE;
+    pm->mem_hp_io_len = ACPI_MEMORY_HOTPLUG_IO_LEN;
+
     /* Fill in optional s3/s4 related properties */
     o = object_property_get_qobject(obj, ACPI_PM_PROP_S3_DISABLED, NULL);
     if (o) {
@@ -214,278 +200,68 @@
 
 static void acpi_get_misc_info(AcpiMiscInfo *info)
 {
+    Object *piix = piix4_pm_find();
+    Object *lpc = ich9_lpc_find();
+    assert(!!piix != !!lpc);
+
+    if (piix) {
+        info->is_piix4 = true;
+    }
+    if (lpc) {
+        info->is_piix4 = false;
+    }
+
     info->has_hpet = hpet_find();
-    info->has_tpm = tpm_find();
+    info->tpm_version = tpm_get_version();
     info->pvpanic_port = pvpanic_port();
+    info->applesmc_io_base = applesmc_port();
 }
 
-static void acpi_get_pci_info(PcPciInfo *info)
+/*
+ * Because of the PXB hosts we cannot simply query TYPE_PCI_HOST_BRIDGE.
+ * On i386 arch we only have two pci hosts, so we can look only for them.
+ */
+static Object *acpi_get_i386_pci_host(void)
+{
+    PCIHostState *host;
+
+    host = OBJECT_CHECK(PCIHostState,
+                        object_resolve_path("/machine/i440fx", NULL),
+                        TYPE_PCI_HOST_BRIDGE);
+    if (!host) {
+        host = OBJECT_CHECK(PCIHostState,
+                            object_resolve_path("/machine/q35", NULL),
+                            TYPE_PCI_HOST_BRIDGE);
+    }
+
+    return OBJECT(host);
+}
+
+static void acpi_get_pci_holes(Range *hole, Range *hole64)
 {
     Object *pci_host;
-    bool ambiguous;
 
-    pci_host = object_resolve_path_type("", TYPE_PCI_HOST_BRIDGE, &ambiguous);
-    g_assert(!ambiguous);
+    pci_host = acpi_get_i386_pci_host();
     g_assert(pci_host);
 
-    info->w32.begin = object_property_get_int(pci_host,
+    range_set_bounds1(hole,
+                      object_property_get_int(pci_host,
                                               PCI_HOST_PROP_PCI_HOLE_START,
-                                              NULL);
-    info->w32.end = object_property_get_int(pci_host,
-                                            PCI_HOST_PROP_PCI_HOLE_END,
-                                            NULL);
-    info->w64.begin = object_property_get_int(pci_host,
+                                              NULL),
+                      object_property_get_int(pci_host,
+                                              PCI_HOST_PROP_PCI_HOLE_END,
+                                              NULL));
+    range_set_bounds1(hole64,
+                      object_property_get_int(pci_host,
                                               PCI_HOST_PROP_PCI_HOLE64_START,
-                                              NULL);
-    info->w64.end = object_property_get_int(pci_host,
-                                            PCI_HOST_PROP_PCI_HOLE64_END,
-                                            NULL);
+                                              NULL),
+                      object_property_get_int(pci_host,
+                                              PCI_HOST_PROP_PCI_HOLE64_END,
+                                              NULL));
 }
 
-#define ACPI_BUILD_APPNAME  "Bochs"
-#define ACPI_BUILD_APPNAME6 "BOCHS "
-#define ACPI_BUILD_APPNAME4 "BXPC"
-
-#define ACPI_BUILD_DPRINTF(level, fmt, ...) do {} while (0)
-
-#define ACPI_BUILD_TABLE_FILE "etc/acpi/tables"
-#define ACPI_BUILD_RSDP_FILE "etc/acpi/rsdp"
-#define ACPI_BUILD_TPMLOG_FILE "etc/tpm/log"
-
-static void
-build_header(GArray *linker, GArray *table_data,
-             AcpiTableHeader *h, const char *sig, int len, uint8_t rev)
-{
-    memcpy(&h->signature, sig, 4);
-    h->length = cpu_to_le32(len);
-    h->revision = rev;
-    memcpy(h->oem_id, ACPI_BUILD_APPNAME6, 6);
-    memcpy(h->oem_table_id, ACPI_BUILD_APPNAME4, 4);
-    memcpy(h->oem_table_id + 4, sig, 4);
-    h->oem_revision = cpu_to_le32(1);
-    memcpy(h->asl_compiler_id, ACPI_BUILD_APPNAME4, 4);
-    h->asl_compiler_revision = cpu_to_le32(1);
-    h->checksum = 0;
-    /* Checksum to be filled in by Guest linker */
-    bios_linker_loader_add_checksum(linker, ACPI_BUILD_TABLE_FILE,
-                                    table_data->data, h, len, &h->checksum);
-}
-
-static inline GArray *build_alloc_array(void)
-{
-        return g_array_new(false, true /* clear */, 1);
-}
-
-static inline void build_free_array(GArray *array)
-{
-        g_array_free(array, true);
-}
-
-static inline void build_prepend_byte(GArray *array, uint8_t val)
-{
-    g_array_prepend_val(array, val);
-}
-
-static inline void build_append_byte(GArray *array, uint8_t val)
-{
-    g_array_append_val(array, val);
-}
-
-static inline void build_append_array(GArray *array, GArray *val)
-{
-    g_array_append_vals(array, val->data, val->len);
-}
-
-static void GCC_FMT_ATTR(2, 3)
-build_append_nameseg(GArray *array, const char *format, ...)
-{
-    /* It would be nicer to use g_string_vprintf but it's only there in 2.22 */
-    char s[] = "XXXX";
-    int len;
-    va_list args;
-
-    va_start(args, format);
-    len = vsnprintf(s, sizeof s, format, args);
-    va_end(args);
-
-    assert(len == 4);
-    g_array_append_vals(array, s, len);
-}
-
-/* 5.4 Definition Block Encoding */
-enum {
-    PACKAGE_LENGTH_1BYTE_SHIFT = 6, /* Up to 63 - use extra 2 bits. */
-    PACKAGE_LENGTH_2BYTE_SHIFT = 4,
-    PACKAGE_LENGTH_3BYTE_SHIFT = 12,
-    PACKAGE_LENGTH_4BYTE_SHIFT = 20,
-};
-
-static void build_prepend_package_length(GArray *package, unsigned min_bytes)
-{
-    uint8_t byte;
-    unsigned length = package->len;
-    unsigned length_bytes;
-
-    if (length + 1 < (1 << PACKAGE_LENGTH_1BYTE_SHIFT)) {
-        length_bytes = 1;
-    } else if (length + 2 < (1 << PACKAGE_LENGTH_3BYTE_SHIFT)) {
-        length_bytes = 2;
-    } else if (length + 3 < (1 << PACKAGE_LENGTH_4BYTE_SHIFT)) {
-        length_bytes = 3;
-    } else {
-        length_bytes = 4;
-    }
-
-    /* Force length to at least min_bytes.
-     * This wastes memory but that's how bios did it.
-     */
-    length_bytes = MAX(length_bytes, min_bytes);
-
-    /* PkgLength is the length of the inclusive length of the data. */
-    length += length_bytes;
-
-    switch (length_bytes) {
-    case 1:
-        byte = length;
-        build_prepend_byte(package, byte);
-        return;
-    case 4:
-        byte = length >> PACKAGE_LENGTH_4BYTE_SHIFT;
-        build_prepend_byte(package, byte);
-        length &= (1 << PACKAGE_LENGTH_4BYTE_SHIFT) - 1;
-        /* fall through */
-    case 3:
-        byte = length >> PACKAGE_LENGTH_3BYTE_SHIFT;
-        build_prepend_byte(package, byte);
-        length &= (1 << PACKAGE_LENGTH_3BYTE_SHIFT) - 1;
-        /* fall through */
-    case 2:
-        byte = length >> PACKAGE_LENGTH_2BYTE_SHIFT;
-        build_prepend_byte(package, byte);
-        length &= (1 << PACKAGE_LENGTH_2BYTE_SHIFT) - 1;
-        /* fall through */
-    }
-    /*
-     * Most significant two bits of byte zero indicate how many following bytes
-     * are in PkgLength encoding.
-     */
-    byte = ((length_bytes - 1) << PACKAGE_LENGTH_1BYTE_SHIFT) | length;
-    build_prepend_byte(package, byte);
-}
-
-static void build_package(GArray *package, uint8_t op, unsigned min_bytes)
-{
-    build_prepend_package_length(package, min_bytes);
-    build_prepend_byte(package, op);
-}
-
-static void build_extop_package(GArray *package, uint8_t op)
-{
-    build_package(package, op, 1);
-    build_prepend_byte(package, 0x5B); /* ExtOpPrefix */
-}
-
-static void build_append_value(GArray *table, uint32_t value, int size)
-{
-    uint8_t prefix;
-    int i;
-
-    switch (size) {
-    case 1:
-        prefix = 0x0A; /* BytePrefix */
-        break;
-    case 2:
-        prefix = 0x0B; /* WordPrefix */
-        break;
-    case 4:
-        prefix = 0x0C; /* DWordPrefix */
-        break;
-    default:
-        assert(0);
-        return;
-    }
-    build_append_byte(table, prefix);
-    for (i = 0; i < size; ++i) {
-        build_append_byte(table, value & 0xFF);
-        value = value >> 8;
-    }
-}
-
-static void build_append_int(GArray *table, uint32_t value)
-{
-    if (value == 0x00) {
-        build_append_byte(table, 0x00); /* ZeroOp */
-    } else if (value == 0x01) {
-        build_append_byte(table, 0x01); /* OneOp */
-    } else if (value <= 0xFF) {
-        build_append_value(table, value, 1);
-    } else if (value <= 0xFFFF) {
-        build_append_value(table, value, 2);
-    } else {
-        build_append_value(table, value, 4);
-    }
-}
-
-static GArray *build_alloc_method(const char *name, uint8_t arg_count)
-{
-    GArray *method = build_alloc_array();
-
-    build_append_nameseg(method, "%s", name);
-    build_append_byte(method, arg_count); /* MethodFlags: ArgCount */
-
-    return method;
-}
-
-static void build_append_and_cleanup_method(GArray *device, GArray *method)
-{
-    uint8_t op = 0x14; /* MethodOp */
-
-    build_package(method, op, 0);
-
-    build_append_array(device, method);
-    build_free_array(method);
-}
-
-static void build_append_notify_target_ifequal(GArray *method,
-                                               GArray *target_name,
-                                               uint32_t value, int size)
-{
-    GArray *notify = build_alloc_array();
-    uint8_t op = 0xA0; /* IfOp */
-
-    build_append_byte(notify, 0x93); /* LEqualOp */
-    build_append_byte(notify, 0x68); /* Arg0Op */
-    build_append_value(notify, value, size);
-    build_append_byte(notify, 0x86); /* NotifyOp */
-    build_append_array(notify, target_name);
-    build_append_byte(notify, 0x69); /* Arg1Op */
-
-    /* Pack it up */
-    build_package(notify, op, 1);
-
-    build_append_array(method, notify);
-
-    build_free_array(notify);
-}
-
-/* End here */
 #define ACPI_PORT_SMI_CMD           0x00b2 /* TODO: this is APM_CNT_IOPORT */
 
-static inline void *acpi_data_push(GArray *table_data, unsigned size)
-{
-    unsigned off = table_data->len;
-    g_array_set_size(table_data, off + size);
-    return table_data->data + off;
-}
-
-static unsigned acpi_data_len(GArray *table)
-{
-#if GLIB_CHECK_VERSION(2, 22, 0)
-    assert(g_array_get_element_size(table) == 1);
-#endif
-    return table->len;
-}
-
 static void acpi_align_size(GArray *blob, unsigned align)
 {
     /* Align size to multiple of given size. This reduces the chance
@@ -494,33 +270,9 @@
     g_array_set_size(blob, ROUND_UP(acpi_data_len(blob), align));
 }
 
-/* Set a value within table in a safe manner */
-#define ACPI_BUILD_SET_LE(table, size, off, bits, val) \
-    do { \
-        uint64_t ACPI_BUILD_SET_LE_val = cpu_to_le64(val); \
-        memcpy(acpi_data_get_ptr(table, size, off, \
-                                 (bits) / BITS_PER_BYTE), \
-               &ACPI_BUILD_SET_LE_val, \
-               (bits) / BITS_PER_BYTE); \
-    } while (0)
-
-static inline void *acpi_data_get_ptr(uint8_t *table_data, unsigned table_size,
-                                      unsigned off, unsigned size)
-{
-    assert(off + size > off);
-    assert(off + size <= table_size);
-    return table_data + off;
-}
-
-static inline void acpi_add_table(GArray *table_offsets, GArray *table_data)
-{
-    uint32_t offset = cpu_to_le32(table_data->len);
-    g_array_append_val(table_offsets, offset);
-}
-
 /* FACS */
 static void
-build_facs(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
+build_facs(GArray *table_data, BIOSLinker *linker)
 {
     AcpiFacsDescriptorRev1 *facs = acpi_data_push(table_data, sizeof *facs);
     memcpy(&facs->signature, "FACS", 4);
@@ -559,41 +311,67 @@
     if (max_cpus > 8) {
         fadt->flags |= cpu_to_le32(1 << ACPI_FADT_F_FORCE_APIC_CLUSTER_MODEL);
     }
+    fadt->century = RTC_CENTURY;
 }
 
 
 /* FADT */
 static void
-build_fadt(GArray *table_data, GArray *linker, AcpiPmInfo *pm,
-           unsigned facs, unsigned dsdt)
+build_fadt(GArray *table_data, BIOSLinker *linker, AcpiPmInfo *pm,
+           unsigned facs_tbl_offset, unsigned dsdt_tbl_offset,
+           const char *oem_id, const char *oem_table_id)
 {
     AcpiFadtDescriptorRev1 *fadt = acpi_data_push(table_data, sizeof(*fadt));
+    unsigned fw_ctrl_offset = (char *)&fadt->firmware_ctrl - table_data->data;
+    unsigned dsdt_entry_offset = (char *)&fadt->dsdt - table_data->data;
 
-    fadt->firmware_ctrl = cpu_to_le32(facs);
     /* FACS address to be filled by Guest linker */
-    bios_linker_loader_add_pointer(linker, ACPI_BUILD_TABLE_FILE,
-                                   ACPI_BUILD_TABLE_FILE,
-                                   table_data, &fadt->firmware_ctrl,
-                                   sizeof fadt->firmware_ctrl);
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_TABLE_FILE, fw_ctrl_offset, sizeof(fadt->firmware_ctrl),
+        ACPI_BUILD_TABLE_FILE, facs_tbl_offset);
 
-    fadt->dsdt = cpu_to_le32(dsdt);
     /* DSDT address to be filled by Guest linker */
-    bios_linker_loader_add_pointer(linker, ACPI_BUILD_TABLE_FILE,
-                                   ACPI_BUILD_TABLE_FILE,
-                                   table_data, &fadt->dsdt,
-                                   sizeof fadt->dsdt);
-
     fadt_setup(fadt, pm);
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_TABLE_FILE, dsdt_entry_offset, sizeof(fadt->dsdt),
+        ACPI_BUILD_TABLE_FILE, dsdt_tbl_offset);
 
     build_header(linker, table_data,
-                 (void *)fadt, "FACP", sizeof(*fadt), 1);
+                 (void *)fadt, "FACP", sizeof(*fadt), 1, oem_id, oem_table_id);
+}
+
+void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid,
+                       CPUArchIdList *apic_ids, GArray *entry)
+{
+    int apic_id;
+    AcpiMadtProcessorApic *apic = acpi_data_push(entry, sizeof *apic);
+
+    apic_id = apic_ids->cpus[uid].arch_id;
+    apic->type = ACPI_APIC_PROCESSOR;
+    apic->length = sizeof(*apic);
+    apic->processor_id = uid;
+    apic->local_apic_id = apic_id;
+    if (apic_ids->cpus[uid].cpu != NULL) {
+        apic->flags = cpu_to_le32(1);
+    } else {
+        /* ACPI spec says that LAPIC entry for non present
+         * CPU may be omitted from MADT or it must be marked
+         * as disabled. However omitting non present CPU from
+         * MADT breaks hotplug on linux. So possible CPUs
+         * should be put in MADT but kept disabled.
+         */
+        apic->flags = cpu_to_le32(0);
+    }
 }
 
 static void
-build_madt(GArray *table_data, GArray *linker, AcpiCpuInfo *cpu,
-           PcGuestInfo *guest_info)
+build_madt(GArray *table_data, BIOSLinker *linker, PCMachineState *pcms)
 {
+    MachineClass *mc = MACHINE_GET_CLASS(pcms);
+    CPUArchIdList *apic_ids = mc->possible_cpu_arch_ids(MACHINE(pcms));
     int madt_start = table_data->len;
+    AcpiDeviceIfClass *adevc = ACPI_DEVICE_IF_GET_CLASS(pcms->acpi_dev);
+    AcpiDeviceIf *adev = ACPI_DEVICE_IF(pcms->acpi_dev);
 
     AcpiMultipleApicTable *madt;
     AcpiMadtIoApic *io_apic;
@@ -605,27 +383,19 @@
     madt->local_apic_address = cpu_to_le32(APIC_DEFAULT_ADDRESS);
     madt->flags = cpu_to_le32(1);
 
-    for (i = 0; i < guest_info->apic_id_limit; i++) {
-        AcpiMadtProcessorApic *apic = acpi_data_push(table_data, sizeof *apic);
-        apic->type = ACPI_APIC_PROCESSOR;
-        apic->length = sizeof(*apic);
-        apic->processor_id = i;
-        apic->local_apic_id = i;
-        if (test_bit(i, cpu->found_cpus)) {
-            apic->flags = cpu_to_le32(1);
-        } else {
-            apic->flags = cpu_to_le32(0);
-        }
+    for (i = 0; i < apic_ids->len; i++) {
+        adevc->madt_cpu(adev, i, apic_ids, table_data);
     }
+    g_free(apic_ids);
+
     io_apic = acpi_data_push(table_data, sizeof *io_apic);
     io_apic->type = ACPI_APIC_IO;
     io_apic->length = sizeof(*io_apic);
-#define ACPI_BUILD_IOAPIC_ID 0x0
     io_apic->io_apic_id = ACPI_BUILD_IOAPIC_ID;
     io_apic->address = cpu_to_le32(IO_APIC_DEFAULT_ADDRESS);
     io_apic->interrupt = cpu_to_le32(0);
 
-    if (guest_info->apic_xrupt_override) {
+    if (pcms->apic_xrupt_override) {
         intsrcovr = acpi_data_push(table_data, sizeof *intsrcovr);
         intsrcovr->type   = ACPI_APIC_XRUPT_OVERRIDE;
         intsrcovr->length = sizeof(*intsrcovr);
@@ -656,116 +426,7 @@
 
     build_header(linker, table_data,
                  (void *)(table_data->data + madt_start), "APIC",
-                 table_data->len - madt_start, 1);
-}
-
-/* Encode a hex value */
-static inline char acpi_get_hex(uint32_t val)
-{
-    val &= 0x0f;
-    return (val <= 9) ? ('0' + val) : ('A' + val - 10);
-}
-
-#include "hw/i386/ssdt-proc.hex"
-
-/* 0x5B 0x83 ProcessorOp PkgLength NameString ProcID */
-#define ACPI_PROC_OFFSET_CPUHEX (*ssdt_proc_name - *ssdt_proc_start + 2)
-#define ACPI_PROC_OFFSET_CPUID1 (*ssdt_proc_name - *ssdt_proc_start + 4)
-#define ACPI_PROC_OFFSET_CPUID2 (*ssdt_proc_id - *ssdt_proc_start)
-#define ACPI_PROC_SIZEOF (*ssdt_proc_end - *ssdt_proc_start)
-#define ACPI_PROC_AML (ssdp_proc_aml + *ssdt_proc_start)
-
-/* 0x5B 0x82 DeviceOp PkgLength NameString */
-#define ACPI_PCIHP_OFFSET_HEX (*ssdt_pcihp_name - *ssdt_pcihp_start + 1)
-#define ACPI_PCIHP_OFFSET_ID (*ssdt_pcihp_id - *ssdt_pcihp_start)
-#define ACPI_PCIHP_OFFSET_ADR (*ssdt_pcihp_adr - *ssdt_pcihp_start)
-#define ACPI_PCIHP_OFFSET_EJ0 (*ssdt_pcihp_ej0 - *ssdt_pcihp_start)
-#define ACPI_PCIHP_SIZEOF (*ssdt_pcihp_end - *ssdt_pcihp_start)
-#define ACPI_PCIHP_AML (ssdp_pcihp_aml + *ssdt_pcihp_start)
-
-#define ACPI_PCINOHP_OFFSET_HEX (*ssdt_pcinohp_name - *ssdt_pcinohp_start + 1)
-#define ACPI_PCINOHP_OFFSET_ADR (*ssdt_pcinohp_adr - *ssdt_pcinohp_start)
-#define ACPI_PCINOHP_SIZEOF (*ssdt_pcinohp_end - *ssdt_pcinohp_start)
-#define ACPI_PCINOHP_AML (ssdp_pcihp_aml + *ssdt_pcinohp_start)
-
-#define ACPI_PCIVGA_OFFSET_HEX (*ssdt_pcivga_name - *ssdt_pcivga_start + 1)
-#define ACPI_PCIVGA_OFFSET_ADR (*ssdt_pcivga_adr - *ssdt_pcivga_start)
-#define ACPI_PCIVGA_SIZEOF (*ssdt_pcivga_end - *ssdt_pcivga_start)
-#define ACPI_PCIVGA_AML (ssdp_pcihp_aml + *ssdt_pcivga_start)
-
-#define ACPI_PCIQXL_OFFSET_HEX (*ssdt_pciqxl_name - *ssdt_pciqxl_start + 1)
-#define ACPI_PCIQXL_OFFSET_ADR (*ssdt_pciqxl_adr - *ssdt_pciqxl_start)
-#define ACPI_PCIQXL_SIZEOF (*ssdt_pciqxl_end - *ssdt_pciqxl_start)
-#define ACPI_PCIQXL_AML (ssdp_pcihp_aml + *ssdt_pciqxl_start)
-
-#include "hw/i386/ssdt-mem.hex"
-
-/* 0x5B 0x82 DeviceOp PkgLength NameString DimmID */
-#define ACPI_MEM_OFFSET_HEX (*ssdt_mem_name - *ssdt_mem_start + 2)
-#define ACPI_MEM_OFFSET_ID (*ssdt_mem_id - *ssdt_mem_start + 7)
-#define ACPI_MEM_SIZEOF (*ssdt_mem_end - *ssdt_mem_start)
-#define ACPI_MEM_AML (ssdm_mem_aml + *ssdt_mem_start)
-
-#define ACPI_SSDT_SIGNATURE 0x54445353 /* SSDT */
-#define ACPI_SSDT_HEADER_LENGTH 36
-
-#include "hw/i386/ssdt-misc.hex"
-#include "hw/i386/ssdt-pcihp.hex"
-#include "hw/i386/ssdt-tpm.hex"
-
-static void
-build_append_notify_method(GArray *device, const char *name,
-                           const char *format, int count)
-{
-    int i;
-    GArray *method = build_alloc_method(name, 2);
-
-    for (i = 0; i < count; i++) {
-        GArray *target = build_alloc_array();
-        build_append_nameseg(target, format, i);
-        assert(i < 256); /* Fits in 1 byte */
-        build_append_notify_target_ifequal(method, target, i, 1);
-        build_free_array(target);
-    }
-
-    build_append_and_cleanup_method(device, method);
-}
-
-static void patch_pcihp(int slot, uint8_t *ssdt_ptr)
-{
-    unsigned devfn = PCI_DEVFN(slot, 0);
-
-    ssdt_ptr[ACPI_PCIHP_OFFSET_HEX] = acpi_get_hex(devfn >> 4);
-    ssdt_ptr[ACPI_PCIHP_OFFSET_HEX + 1] = acpi_get_hex(devfn);
-    ssdt_ptr[ACPI_PCIHP_OFFSET_ID] = slot;
-    ssdt_ptr[ACPI_PCIHP_OFFSET_ADR + 2] = slot;
-}
-
-static void patch_pcinohp(int slot, uint8_t *ssdt_ptr)
-{
-    unsigned devfn = PCI_DEVFN(slot, 0);
-
-    ssdt_ptr[ACPI_PCINOHP_OFFSET_HEX] = acpi_get_hex(devfn >> 4);
-    ssdt_ptr[ACPI_PCINOHP_OFFSET_HEX + 1] = acpi_get_hex(devfn);
-    ssdt_ptr[ACPI_PCINOHP_OFFSET_ADR + 2] = slot;
-}
-
-static void patch_pcivga(int slot, uint8_t *ssdt_ptr)
-{
-    unsigned devfn = PCI_DEVFN(slot, 0);
-
-    ssdt_ptr[ACPI_PCIVGA_OFFSET_HEX] = acpi_get_hex(devfn >> 4);
-    ssdt_ptr[ACPI_PCIVGA_OFFSET_HEX + 1] = acpi_get_hex(devfn);
-    ssdt_ptr[ACPI_PCIVGA_OFFSET_ADR + 2] = slot;
-}
-
-static void patch_pciqxl(int slot, uint8_t *ssdt_ptr)
-{
-    unsigned devfn = PCI_DEVFN(slot, 0);
-
-    ssdt_ptr[ACPI_PCIQXL_OFFSET_HEX] = acpi_get_hex(devfn >> 4);
-    ssdt_ptr[ACPI_PCIQXL_OFFSET_HEX + 1] = acpi_get_hex(devfn);
-    ssdt_ptr[ACPI_PCIQXL_OFFSET_ADR + 2] = slot;
+                 table_data->len - madt_start, 1, NULL, NULL);
 }
 
 /* Assign BSEL property to all buses.  In the future, this can be changed
@@ -798,410 +459,1954 @@
     }
 }
 
-static void build_pci_bus_state_init(AcpiBuildPciBusHotplugState *state,
-                                     AcpiBuildPciBusHotplugState *parent,
-                                     bool pcihp_bridge_en)
+static void build_append_pcihp_notify_entry(Aml *method, int slot)
 {
-    state->parent = parent;
-    state->device_table = build_alloc_array();
-    state->notify_table = build_alloc_array();
-    state->pcihp_bridge_en = pcihp_bridge_en;
+    Aml *if_ctx;
+    int32_t devfn = PCI_DEVFN(slot, 0);
+
+    if_ctx = aml_if(aml_and(aml_arg(0), aml_int(0x1U << slot), NULL));
+    aml_append(if_ctx, aml_notify(aml_name("S%.02X", devfn), aml_arg(1)));
+    aml_append(method, if_ctx);
 }
 
-static void build_pci_bus_state_cleanup(AcpiBuildPciBusHotplugState *state)
+static void build_append_pci_bus_devices(Aml *parent_scope, PCIBus *bus,
+                                         bool pcihp_bridge_en)
 {
-    build_free_array(state->device_table);
-    build_free_array(state->notify_table);
-}
-
-static void *build_pci_bus_begin(PCIBus *bus, void *parent_state)
-{
-    AcpiBuildPciBusHotplugState *parent = parent_state;
-    AcpiBuildPciBusHotplugState *child = g_malloc(sizeof *child);
-
-    build_pci_bus_state_init(child, parent, parent->pcihp_bridge_en);
-
-    return child;
-}
-
-static void build_pci_bus_end(PCIBus *bus, void *bus_state)
-{
-    AcpiBuildPciBusHotplugState *child = bus_state;
-    AcpiBuildPciBusHotplugState *parent = child->parent;
-    GArray *bus_table = build_alloc_array();
-    DECLARE_BITMAP(slot_hotplug_enable, PCI_SLOT_MAX);
-    DECLARE_BITMAP(slot_device_present, PCI_SLOT_MAX);
-    DECLARE_BITMAP(slot_device_system, PCI_SLOT_MAX);
-    DECLARE_BITMAP(slot_device_vga, PCI_SLOT_MAX);
-    DECLARE_BITMAP(slot_device_qxl, PCI_SLOT_MAX);
-    uint8_t op;
-    int i;
+    Aml *dev, *notify_method, *method;
     QObject *bsel;
-    GArray *method;
-    bool bus_hotplug_support = false;
-
-    /*
-     * Skip bridge subtree creation if bridge hotplug is disabled
-     * to make acpi tables compatible with legacy machine types.
-     */
-    if (!child->pcihp_bridge_en && bus->parent_dev) {
-        return;
-    }
-
-    if (bus->parent_dev) {
-        op = 0x82; /* DeviceOp */
-        build_append_nameseg(bus_table, "S%.02X_",
-                             bus->parent_dev->devfn);
-        build_append_byte(bus_table, 0x08); /* NameOp */
-        build_append_nameseg(bus_table, "_SUN");
-        build_append_value(bus_table, PCI_SLOT(bus->parent_dev->devfn), 1);
-        build_append_byte(bus_table, 0x08); /* NameOp */
-        build_append_nameseg(bus_table, "_ADR");
-        build_append_value(bus_table, (PCI_SLOT(bus->parent_dev->devfn) << 16) |
-                           PCI_FUNC(bus->parent_dev->devfn), 4);
-    } else {
-        op = 0x10; /* ScopeOp */;
-        build_append_nameseg(bus_table, "PCI0");
-    }
+    PCIBus *sec;
+    int i;
 
     bsel = object_property_get_qobject(OBJECT(bus), ACPI_PCIHP_PROP_BSEL, NULL);
     if (bsel) {
-        build_append_byte(bus_table, 0x08); /* NameOp */
-        build_append_nameseg(bus_table, "BSEL");
-        build_append_int(bus_table, qint_get_int(qobject_to_qint(bsel)));
-        memset(slot_hotplug_enable, 0xff, sizeof slot_hotplug_enable);
-    } else {
-        /* No bsel - no slots are hot-pluggable */
-        memset(slot_hotplug_enable, 0x00, sizeof slot_hotplug_enable);
-    }
+        int64_t bsel_val = qint_get_int(qobject_to_qint(bsel));
 
-    memset(slot_device_present, 0x00, sizeof slot_device_present);
-    memset(slot_device_system, 0x00, sizeof slot_device_present);
-    memset(slot_device_vga, 0x00, sizeof slot_device_vga);
-    memset(slot_device_qxl, 0x00, sizeof slot_device_qxl);
+        aml_append(parent_scope, aml_name_decl("BSEL", aml_int(bsel_val)));
+        notify_method = aml_method("DVNT", 2, AML_NOTSERIALIZED);
+    }
 
     for (i = 0; i < ARRAY_SIZE(bus->devices); i += PCI_FUNC_MAX) {
         DeviceClass *dc;
         PCIDeviceClass *pc;
         PCIDevice *pdev = bus->devices[i];
         int slot = PCI_SLOT(i);
+        bool hotplug_enabled_dev;
         bool bridge_in_acpi;
 
         if (!pdev) {
+            if (bsel) { /* add hotplug slots for non present devices */
+                dev = aml_device("S%.02X", PCI_DEVFN(slot, 0));
+                aml_append(dev, aml_name_decl("_SUN", aml_int(slot)));
+                aml_append(dev, aml_name_decl("_ADR", aml_int(slot << 16)));
+                method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+                aml_append(method,
+                    aml_call2("PCEJ", aml_name("BSEL"), aml_name("_SUN"))
+                );
+                aml_append(dev, method);
+                aml_append(parent_scope, dev);
+
+                build_append_pcihp_notify_entry(notify_method, slot);
+            }
             continue;
         }
 
-        set_bit(slot, slot_device_present);
         pc = PCI_DEVICE_GET_CLASS(pdev);
         dc = DEVICE_GET_CLASS(pdev);
 
         /* When hotplug for bridges is enabled, bridges are
          * described in ACPI separately (see build_pci_bus_end).
          * In this case they aren't themselves hot-pluggable.
+         * Hotplugged bridges *are* hot-pluggable.
          */
-        bridge_in_acpi = pc->is_bridge && child->pcihp_bridge_en;
+        bridge_in_acpi = pc->is_bridge && pcihp_bridge_en &&
+            !DEVICE(pdev)->hotplugged;
 
-        if (pc->class_id == PCI_CLASS_BRIDGE_ISA || bridge_in_acpi) {
-            set_bit(slot, slot_device_system);
+        hotplug_enabled_dev = bsel && dc->hotpluggable && !bridge_in_acpi;
+
+        if (pc->class_id == PCI_CLASS_BRIDGE_ISA) {
+            continue;
         }
 
+        /* start to compose PCI slot descriptor */
+        dev = aml_device("S%.02X", PCI_DEVFN(slot, 0));
+        aml_append(dev, aml_name_decl("_ADR", aml_int(slot << 16)));
+
         if (pc->class_id == PCI_CLASS_DISPLAY_VGA) {
-            set_bit(slot, slot_device_vga);
+            /* add VGA specific AML methods */
+            int s3d;
 
             if (object_dynamic_cast(OBJECT(pdev), "qxl-vga")) {
-                set_bit(slot, slot_device_qxl);
+                s3d = 3;
+            } else {
+                s3d = 0;
             }
-        }
 
-        if (!dc->hotpluggable || bridge_in_acpi) {
-            clear_bit(slot, slot_hotplug_enable);
-        }
-    }
+            method = aml_method("_S1D", 0, AML_NOTSERIALIZED);
+            aml_append(method, aml_return(aml_int(0)));
+            aml_append(dev, method);
 
-    /* Append Device object for each slot */
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        bool can_eject = test_bit(i, slot_hotplug_enable);
-        bool present = test_bit(i, slot_device_present);
-        bool vga = test_bit(i, slot_device_vga);
-        bool qxl = test_bit(i, slot_device_qxl);
-        bool system = test_bit(i, slot_device_system);
-        if (can_eject) {
-            void *pcihp = acpi_data_push(bus_table,
-                                         ACPI_PCIHP_SIZEOF);
-            memcpy(pcihp, ACPI_PCIHP_AML, ACPI_PCIHP_SIZEOF);
-            patch_pcihp(i, pcihp);
-            bus_hotplug_support = true;
-        } else if (qxl) {
-            void *pcihp = acpi_data_push(bus_table,
-                                         ACPI_PCIQXL_SIZEOF);
-            memcpy(pcihp, ACPI_PCIQXL_AML, ACPI_PCIQXL_SIZEOF);
-            patch_pciqxl(i, pcihp);
-        } else if (vga) {
-            void *pcihp = acpi_data_push(bus_table,
-                                         ACPI_PCIVGA_SIZEOF);
-            memcpy(pcihp, ACPI_PCIVGA_AML, ACPI_PCIVGA_SIZEOF);
-            patch_pcivga(i, pcihp);
-        } else if (system) {
-            /* Nothing to do: system devices are in DSDT or in SSDT above. */
-        } else if (present) {
-            void *pcihp = acpi_data_push(bus_table,
-                                         ACPI_PCINOHP_SIZEOF);
-            memcpy(pcihp, ACPI_PCINOHP_AML, ACPI_PCINOHP_SIZEOF);
-            patch_pcinohp(i, pcihp);
+            method = aml_method("_S2D", 0, AML_NOTSERIALIZED);
+            aml_append(method, aml_return(aml_int(0)));
+            aml_append(dev, method);
+
+            method = aml_method("_S3D", 0, AML_NOTSERIALIZED);
+            aml_append(method, aml_return(aml_int(s3d)));
+            aml_append(dev, method);
+        } else if (hotplug_enabled_dev) {
+            /* add _SUN/_EJ0 to make slot hotpluggable  */
+            aml_append(dev, aml_name_decl("_SUN", aml_int(slot)));
+
+            method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+            aml_append(method,
+                aml_call2("PCEJ", aml_name("BSEL"), aml_name("_SUN"))
+            );
+            aml_append(dev, method);
+
+            if (bsel) {
+                build_append_pcihp_notify_entry(notify_method, slot);
+            }
+        } else if (bridge_in_acpi) {
+            /*
+             * device is coldplugged bridge,
+             * add child device descriptions into its scope
+             */
+            PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev));
+
+            build_append_pci_bus_devices(dev, sec_bus, pcihp_bridge_en);
         }
+        /* slot descriptor has been composed, add it into parent context */
+        aml_append(parent_scope, dev);
     }
 
     if (bsel) {
-        method = build_alloc_method("DVNT", 2);
-
-        for (i = 0; i < PCI_SLOT_MAX; i++) {
-            GArray *notify;
-            uint8_t op;
-
-            if (!test_bit(i, slot_hotplug_enable)) {
-                continue;
-            }
-
-            notify = build_alloc_array();
-            op = 0xA0; /* IfOp */
-
-            build_append_byte(notify, 0x7B); /* AndOp */
-            build_append_byte(notify, 0x68); /* Arg0Op */
-            build_append_int(notify, 0x1U << i);
-            build_append_byte(notify, 0x00); /* NullName */
-            build_append_byte(notify, 0x86); /* NotifyOp */
-            build_append_nameseg(notify, "S%.02X_", PCI_DEVFN(i, 0));
-            build_append_byte(notify, 0x69); /* Arg1Op */
-
-            /* Pack it up */
-            build_package(notify, op, 0);
-
-            build_append_array(method, notify);
-
-            build_free_array(notify);
-        }
-
-        build_append_and_cleanup_method(bus_table, method);
+        aml_append(parent_scope, notify_method);
     }
 
     /* Append PCNT method to notify about events on local and child buses.
      * Add unconditionally for root since DSDT expects it.
      */
-    if (bus_hotplug_support || child->notify_table->len || !bus->parent_dev) {
-        method = build_alloc_method("PCNT", 0);
+    method = aml_method("PCNT", 0, AML_NOTSERIALIZED);
 
-        /* If bus supports hotplug select it and notify about local events */
-        if (bsel) {
-            build_append_byte(method, 0x70); /* StoreOp */
-            build_append_int(method, qint_get_int(qobject_to_qint(bsel)));
-            build_append_nameseg(method, "BNUM");
-            build_append_nameseg(method, "DVNT");
-            build_append_nameseg(method, "PCIU");
-            build_append_int(method, 1); /* Device Check */
-            build_append_nameseg(method, "DVNT");
-            build_append_nameseg(method, "PCID");
-            build_append_int(method, 3); /* Eject Request */
-        }
-
-        /* Notify about child bus events in any case */
-        build_append_array(method, child->notify_table);
-
-        build_append_and_cleanup_method(bus_table, method);
-
-        /* Append description of child buses */
-        build_append_array(bus_table, child->device_table);
-
-        /* Pack it up */
-        if (bus->parent_dev) {
-            build_extop_package(bus_table, op);
-        } else {
-            build_package(bus_table, op, 0);
-        }
-
-        /* Append our bus description to parent table */
-        build_append_array(parent->device_table, bus_table);
-
-        /* Also tell parent how to notify us, invoking PCNT method.
-         * At the moment this is not needed for root as we have a single root.
-         */
-        if (bus->parent_dev) {
-            build_append_byte(parent->notify_table, '^'); /* ParentPrefixChar */
-            build_append_byte(parent->notify_table, 0x2E); /* DualNamePrefix */
-            build_append_nameseg(parent->notify_table, "S%.02X_",
-                                 bus->parent_dev->devfn);
-            build_append_nameseg(parent->notify_table, "PCNT");
-        }
+    /* If bus supports hotplug select it and notify about local events */
+    if (bsel) {
+        int64_t bsel_val = qint_get_int(qobject_to_qint(bsel));
+        aml_append(method, aml_store(aml_int(bsel_val), aml_name("BNUM")));
+        aml_append(method,
+            aml_call2("DVNT", aml_name("PCIU"), aml_int(1) /* Device Check */)
+        );
+        aml_append(method,
+            aml_call2("DVNT", aml_name("PCID"), aml_int(3)/* Eject Request */)
+        );
     }
 
+    /* Notify about child bus events in any case */
+    if (pcihp_bridge_en) {
+        QLIST_FOREACH(sec, &bus->child, sibling) {
+            int32_t devfn = sec->parent_dev->devfn;
+
+            if (pci_bus_is_root(sec) || pci_bus_is_express(sec)) {
+                continue;
+            }
+
+            aml_append(method, aml_name("^S%.02X.PCNT", devfn));
+        }
+    }
+    aml_append(parent_scope, method);
     qobject_decref(bsel);
-    build_free_array(bus_table);
-    build_pci_bus_state_cleanup(child);
-    g_free(child);
 }
 
-static void patch_pci_windows(PcPciInfo *pci, uint8_t *start, unsigned size)
+/**
+ * build_prt_entry:
+ * @link_name: link name for PCI route entry
+ *
+ * build AML package containing a PCI route entry for @link_name
+ */
+static Aml *build_prt_entry(const char *link_name)
 {
-    ACPI_BUILD_SET_LE(start, size, acpi_pci32_start[0], 32, pci->w32.begin);
+    Aml *a_zero = aml_int(0);
+    Aml *pkg = aml_package(4);
+    aml_append(pkg, a_zero);
+    aml_append(pkg, a_zero);
+    aml_append(pkg, aml_name("%s", link_name));
+    aml_append(pkg, a_zero);
+    return pkg;
+}
 
-    ACPI_BUILD_SET_LE(start, size, acpi_pci32_end[0], 32, pci->w32.end - 1);
+/*
+ * initialize_route - Initialize the interrupt routing rule
+ * through a specific LINK:
+ *  if (lnk_idx == idx)
+ *      route using link 'link_name'
+ */
+static Aml *initialize_route(Aml *route, const char *link_name,
+                             Aml *lnk_idx, int idx)
+{
+    Aml *if_ctx = aml_if(aml_equal(lnk_idx, aml_int(idx)));
+    Aml *pkg = build_prt_entry(link_name);
 
-    if (pci->w64.end || pci->w64.begin) {
-        ACPI_BUILD_SET_LE(start, size, acpi_pci64_valid[0], 8, 1);
-        ACPI_BUILD_SET_LE(start, size, acpi_pci64_start[0], 64, pci->w64.begin);
-        ACPI_BUILD_SET_LE(start, size, acpi_pci64_end[0], 64, pci->w64.end - 1);
-        ACPI_BUILD_SET_LE(start, size, acpi_pci64_length[0], 64, pci->w64.end - pci->w64.begin);
-    } else {
-        ACPI_BUILD_SET_LE(start, size, acpi_pci64_valid[0], 8, 0);
+    aml_append(if_ctx, aml_store(pkg, route));
+
+    return if_ctx;
+}
+
+/*
+ * build_prt - Define interrupt rounting rules
+ *
+ * Returns an array of 128 routes, one for each device,
+ * based on device location.
+ * The main goal is to equaly distribute the interrupts
+ * over the 4 existing ACPI links (works only for i440fx).
+ * The hash function is  (slot + pin) & 3 -> "LNK[D|A|B|C]".
+ *
+ */
+static Aml *build_prt(bool is_pci0_prt)
+{
+    Aml *method, *while_ctx, *pin, *res;
+
+    method = aml_method("_PRT", 0, AML_NOTSERIALIZED);
+    res = aml_local(0);
+    pin = aml_local(1);
+    aml_append(method, aml_store(aml_package(128), res));
+    aml_append(method, aml_store(aml_int(0), pin));
+
+    /* while (pin < 128) */
+    while_ctx = aml_while(aml_lless(pin, aml_int(128)));
+    {
+        Aml *slot = aml_local(2);
+        Aml *lnk_idx = aml_local(3);
+        Aml *route = aml_local(4);
+
+        /* slot = pin >> 2 */
+        aml_append(while_ctx,
+                   aml_store(aml_shiftright(pin, aml_int(2), NULL), slot));
+        /* lnk_idx = (slot + pin) & 3 */
+        aml_append(while_ctx,
+            aml_store(aml_and(aml_add(pin, slot, NULL), aml_int(3), NULL),
+                      lnk_idx));
+
+        /* route[2] = "LNK[D|A|B|C]", selection based on pin % 3  */
+        aml_append(while_ctx, initialize_route(route, "LNKD", lnk_idx, 0));
+        if (is_pci0_prt) {
+            Aml *if_device_1, *if_pin_4, *else_pin_4;
+
+            /* device 1 is the power-management device, needs SCI */
+            if_device_1 = aml_if(aml_equal(lnk_idx, aml_int(1)));
+            {
+                if_pin_4 = aml_if(aml_equal(pin, aml_int(4)));
+                {
+                    aml_append(if_pin_4,
+                        aml_store(build_prt_entry("LNKS"), route));
+                }
+                aml_append(if_device_1, if_pin_4);
+                else_pin_4 = aml_else();
+                {
+                    aml_append(else_pin_4,
+                        aml_store(build_prt_entry("LNKA"), route));
+                }
+                aml_append(if_device_1, else_pin_4);
+            }
+            aml_append(while_ctx, if_device_1);
+        } else {
+            aml_append(while_ctx, initialize_route(route, "LNKA", lnk_idx, 1));
+        }
+        aml_append(while_ctx, initialize_route(route, "LNKB", lnk_idx, 2));
+        aml_append(while_ctx, initialize_route(route, "LNKC", lnk_idx, 3));
+
+        /* route[0] = 0x[slot]FFFF */
+        aml_append(while_ctx,
+            aml_store(aml_or(aml_shiftleft(slot, aml_int(16)), aml_int(0xFFFF),
+                             NULL),
+                      aml_index(route, aml_int(0))));
+        /* route[1] = pin & 3 */
+        aml_append(while_ctx,
+            aml_store(aml_and(pin, aml_int(3), NULL),
+                      aml_index(route, aml_int(1))));
+        /* res[pin] = route */
+        aml_append(while_ctx, aml_store(route, aml_index(res, pin)));
+        /* pin++ */
+        aml_append(while_ctx, aml_increment(pin));
     }
+    aml_append(method, while_ctx);
+    /* return res*/
+    aml_append(method, aml_return(res));
+
+    return method;
+}
+
+typedef struct CrsRangeEntry {
+    uint64_t base;
+    uint64_t limit;
+} CrsRangeEntry;
+
+static void crs_range_insert(GPtrArray *ranges, uint64_t base, uint64_t limit)
+{
+    CrsRangeEntry *entry;
+
+    entry = g_malloc(sizeof(*entry));
+    entry->base = base;
+    entry->limit = limit;
+
+    g_ptr_array_add(ranges, entry);
+}
+
+static void crs_range_free(gpointer data)
+{
+    CrsRangeEntry *entry = (CrsRangeEntry *)data;
+    g_free(entry);
+}
+
+typedef struct CrsRangeSet {
+    GPtrArray *io_ranges;
+    GPtrArray *mem_ranges;
+    GPtrArray *mem_64bit_ranges;
+ } CrsRangeSet;
+
+static void crs_range_set_init(CrsRangeSet *range_set)
+{
+    range_set->io_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    range_set->mem_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    range_set->mem_64bit_ranges =
+            g_ptr_array_new_with_free_func(crs_range_free);
+}
+
+static void crs_range_set_free(CrsRangeSet *range_set)
+{
+    g_ptr_array_free(range_set->io_ranges, true);
+    g_ptr_array_free(range_set->mem_ranges, true);
+    g_ptr_array_free(range_set->mem_64bit_ranges, true);
+}
+
+static gint crs_range_compare(gconstpointer a, gconstpointer b)
+{
+     CrsRangeEntry *entry_a = *(CrsRangeEntry **)a;
+     CrsRangeEntry *entry_b = *(CrsRangeEntry **)b;
+
+     return (int64_t)entry_a->base - (int64_t)entry_b->base;
+}
+
+/*
+ * crs_replace_with_free_ranges - given the 'used' ranges within [start - end]
+ * interval, computes the 'free' ranges from the same interval.
+ * Example: If the input array is { [a1 - a2],[b1 - b2] }, the function
+ * will return { [base - a1], [a2 - b1], [b2 - limit] }.
+ */
+static void crs_replace_with_free_ranges(GPtrArray *ranges,
+                                         uint64_t start, uint64_t end)
+{
+    GPtrArray *free_ranges = g_ptr_array_new_with_free_func(crs_range_free);
+    uint64_t free_base = start;
+    int i;
+
+    g_ptr_array_sort(ranges, crs_range_compare);
+    for (i = 0; i < ranges->len; i++) {
+        CrsRangeEntry *used = g_ptr_array_index(ranges, i);
+
+        if (free_base < used->base) {
+            crs_range_insert(free_ranges, free_base, used->base - 1);
+        }
+
+        free_base = used->limit + 1;
+    }
+
+    if (free_base < end) {
+        crs_range_insert(free_ranges, free_base, end);
+    }
+
+    g_ptr_array_set_size(ranges, 0);
+    for (i = 0; i < free_ranges->len; i++) {
+        g_ptr_array_add(ranges, g_ptr_array_index(free_ranges, i));
+    }
+
+    g_ptr_array_free(free_ranges, false);
+}
+
+/*
+ * crs_range_merge - merges adjacent ranges in the given array.
+ * Array elements are deleted and replaced with the merged ranges.
+ */
+static void crs_range_merge(GPtrArray *range)
+{
+    GPtrArray *tmp =  g_ptr_array_new_with_free_func(crs_range_free);
+    CrsRangeEntry *entry;
+    uint64_t range_base, range_limit;
+    int i;
+
+    if (!range->len) {
+        return;
+    }
+
+    g_ptr_array_sort(range, crs_range_compare);
+
+    entry = g_ptr_array_index(range, 0);
+    range_base = entry->base;
+    range_limit = entry->limit;
+    for (i = 1; i < range->len; i++) {
+        entry = g_ptr_array_index(range, i);
+        if (entry->base - 1 == range_limit) {
+            range_limit = entry->limit;
+        } else {
+            crs_range_insert(tmp, range_base, range_limit);
+            range_base = entry->base;
+            range_limit = entry->limit;
+        }
+    }
+    crs_range_insert(tmp, range_base, range_limit);
+
+    g_ptr_array_set_size(range, 0);
+    for (i = 0; i < tmp->len; i++) {
+        entry = g_ptr_array_index(tmp, i);
+        crs_range_insert(range, entry->base, entry->limit);
+    }
+    g_ptr_array_free(tmp, true);
+}
+
+static Aml *build_crs(PCIHostState *host, CrsRangeSet *range_set)
+{
+    Aml *crs = aml_resource_template();
+    CrsRangeSet temp_range_set;
+    CrsRangeEntry *entry;
+    uint8_t max_bus = pci_bus_num(host->bus);
+    uint8_t type;
+    int devfn;
+    int i;
+
+    crs_range_set_init(&temp_range_set);
+    for (devfn = 0; devfn < ARRAY_SIZE(host->bus->devices); devfn++) {
+        uint64_t range_base, range_limit;
+        PCIDevice *dev = host->bus->devices[devfn];
+
+        if (!dev) {
+            continue;
+        }
+
+        for (i = 0; i < PCI_NUM_REGIONS; i++) {
+            PCIIORegion *r = &dev->io_regions[i];
+
+            range_base = r->addr;
+            range_limit = r->addr + r->size - 1;
+
+            /*
+             * Work-around for old bioses
+             * that do not support multiple root buses
+             */
+            if (!range_base || range_base > range_limit) {
+                continue;
+            }
+
+            if (r->type & PCI_BASE_ADDRESS_SPACE_IO) {
+                crs_range_insert(temp_range_set.io_ranges,
+                                 range_base, range_limit);
+            } else { /* "memory" */
+                crs_range_insert(temp_range_set.mem_ranges,
+                                 range_base, range_limit);
+            }
+        }
+
+        type = dev->config[PCI_HEADER_TYPE] & ~PCI_HEADER_TYPE_MULTI_FUNCTION;
+        if (type == PCI_HEADER_TYPE_BRIDGE) {
+            uint8_t subordinate = dev->config[PCI_SUBORDINATE_BUS];
+            if (subordinate > max_bus) {
+                max_bus = subordinate;
+            }
+
+            range_base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_SPACE_IO);
+            range_limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_SPACE_IO);
+
+            /*
+             * Work-around for old bioses
+             * that do not support multiple root buses
+             */
+            if (range_base && range_base <= range_limit) {
+                crs_range_insert(temp_range_set.io_ranges,
+                                 range_base, range_limit);
+            }
+
+            range_base =
+                pci_bridge_get_base(dev, PCI_BASE_ADDRESS_SPACE_MEMORY);
+            range_limit =
+                pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_SPACE_MEMORY);
+
+            /*
+             * Work-around for old bioses
+             * that do not support multiple root buses
+             */
+            if (range_base && range_base <= range_limit) {
+                uint64_t length = range_limit - range_base + 1;
+                if (range_limit <= UINT32_MAX && length <= UINT32_MAX) {
+                    crs_range_insert(temp_range_set.mem_ranges,
+                                     range_base, range_limit);
+                } else {
+                    crs_range_insert(temp_range_set.mem_64bit_ranges,
+                                     range_base, range_limit);
+                }
+            }
+
+            range_base =
+                pci_bridge_get_base(dev, PCI_BASE_ADDRESS_MEM_PREFETCH);
+            range_limit =
+                pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_MEM_PREFETCH);
+
+            /*
+             * Work-around for old bioses
+             * that do not support multiple root buses
+             */
+            if (range_base && range_base <= range_limit) {
+                uint64_t length = range_limit - range_base + 1;
+                if (range_limit <= UINT32_MAX && length <= UINT32_MAX) {
+                    crs_range_insert(temp_range_set.mem_ranges,
+                                     range_base, range_limit);
+                } else {
+                    crs_range_insert(temp_range_set.mem_64bit_ranges,
+                                     range_base, range_limit);
+                }
+            }
+        }
+    }
+
+    crs_range_merge(temp_range_set.io_ranges);
+    for (i = 0; i < temp_range_set.io_ranges->len; i++) {
+        entry = g_ptr_array_index(temp_range_set.io_ranges, i);
+        aml_append(crs,
+                   aml_word_io(AML_MIN_FIXED, AML_MAX_FIXED,
+                               AML_POS_DECODE, AML_ENTIRE_RANGE,
+                               0, entry->base, entry->limit, 0,
+                               entry->limit - entry->base + 1));
+        crs_range_insert(range_set->io_ranges, entry->base, entry->limit);
+    }
+
+    crs_range_merge(temp_range_set.mem_ranges);
+    for (i = 0; i < temp_range_set.mem_ranges->len; i++) {
+        entry = g_ptr_array_index(temp_range_set.mem_ranges, i);
+        aml_append(crs,
+                   aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                                    AML_MAX_FIXED, AML_NON_CACHEABLE,
+                                    AML_READ_WRITE,
+                                    0, entry->base, entry->limit, 0,
+                                    entry->limit - entry->base + 1));
+        crs_range_insert(range_set->mem_ranges, entry->base, entry->limit);
+    }
+
+    crs_range_merge(temp_range_set.mem_64bit_ranges);
+    for (i = 0; i < temp_range_set.mem_64bit_ranges->len; i++) {
+        entry = g_ptr_array_index(temp_range_set.mem_64bit_ranges, i);
+        aml_append(crs,
+                   aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                                    AML_MAX_FIXED, AML_NON_CACHEABLE,
+                                    AML_READ_WRITE,
+                                    0, entry->base, entry->limit, 0,
+                                    entry->limit - entry->base + 1));
+        crs_range_insert(range_set->mem_64bit_ranges,
+                         entry->base, entry->limit);
+    }
+
+    crs_range_set_free(&temp_range_set);
+
+    aml_append(crs,
+        aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
+                            0,
+                            pci_bus_num(host->bus),
+                            max_bus,
+                            0,
+                            max_bus - pci_bus_num(host->bus) + 1));
+
+    return crs;
+}
+
+static void build_memory_devices(Aml *sb_scope, int nr_mem,
+                                 uint16_t io_base, uint16_t io_len)
+{
+    int i;
+    Aml *scope;
+    Aml *crs;
+    Aml *field;
+    Aml *dev;
+    Aml *method;
+    Aml *ifctx;
+
+    /* build memory devices */
+    assert(nr_mem <= ACPI_MAX_RAM_SLOTS);
+    scope = aml_scope("\\_SB.PCI0." MEMORY_HOTPLUG_DEVICE);
+    aml_append(scope,
+        aml_name_decl(MEMORY_SLOTS_NUMBER, aml_int(nr_mem))
+    );
+
+    crs = aml_resource_template();
+    aml_append(crs,
+        aml_io(AML_DECODE16, io_base, io_base, 0, io_len)
+    );
+    aml_append(scope, aml_name_decl("_CRS", crs));
+
+    aml_append(scope, aml_operation_region(
+        MEMORY_HOTPLUG_IO_REGION, AML_SYSTEM_IO,
+        aml_int(io_base), io_len)
+    );
+
+    field = aml_field(MEMORY_HOTPLUG_IO_REGION, AML_DWORD_ACC,
+                      AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_ADDR_LOW, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_ADDR_HIGH, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_SIZE_LOW, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_SIZE_HIGH, 32));
+    aml_append(field, /* read only */
+        aml_named_field(MEMORY_SLOT_PROXIMITY, 32));
+    aml_append(scope, field);
+
+    field = aml_field(MEMORY_HOTPLUG_IO_REGION, AML_BYTE_ACC,
+                      AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_reserved_field(160 /* bits, Offset(20) */));
+    aml_append(field, /* 1 if enabled, read only */
+        aml_named_field(MEMORY_SLOT_ENABLED, 1));
+    aml_append(field,
+        /*(read) 1 if has a insert event. (write) 1 to clear event */
+        aml_named_field(MEMORY_SLOT_INSERT_EVENT, 1));
+    aml_append(field,
+        /* (read) 1 if has a remove event. (write) 1 to clear event */
+        aml_named_field(MEMORY_SLOT_REMOVE_EVENT, 1));
+    aml_append(field,
+        /* initiates device eject, write only */
+        aml_named_field(MEMORY_SLOT_EJECT, 1));
+    aml_append(scope, field);
+
+    field = aml_field(MEMORY_HOTPLUG_IO_REGION, AML_DWORD_ACC,
+                      AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, /* DIMM selector, write only */
+        aml_named_field(MEMORY_SLOT_SLECTOR, 32));
+    aml_append(field, /* _OST event code, write only */
+        aml_named_field(MEMORY_SLOT_OST_EVENT, 32));
+    aml_append(field, /* _OST status code, write only */
+        aml_named_field(MEMORY_SLOT_OST_STATUS, 32));
+    aml_append(scope, field);
+    aml_append(sb_scope, scope);
+
+    for (i = 0; i < nr_mem; i++) {
+        #define BASEPATH "\\_SB.PCI0." MEMORY_HOTPLUG_DEVICE "."
+        const char *s;
+
+        dev = aml_device("MP%02X", i);
+        aml_append(dev, aml_name_decl("_UID", aml_string("0x%02X", i)));
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C80")));
+
+        method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_CRS_METHOD;
+        aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
+        aml_append(dev, method);
+
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_STATUS_METHOD;
+        aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
+        aml_append(dev, method);
+
+        method = aml_method("_PXM", 0, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_PROXIMITY_METHOD;
+        aml_append(method, aml_return(aml_call1(s, aml_name("_UID"))));
+        aml_append(dev, method);
+
+        method = aml_method("_OST", 3, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_OST_METHOD;
+
+        aml_append(method, aml_return(aml_call4(
+            s, aml_name("_UID"), aml_arg(0), aml_arg(1), aml_arg(2)
+        )));
+        aml_append(dev, method);
+
+        method = aml_method("_EJ0", 1, AML_NOTSERIALIZED);
+        s = BASEPATH MEMORY_SLOT_EJECT_METHOD;
+        aml_append(method, aml_return(aml_call2(
+                   s, aml_name("_UID"), aml_arg(0))));
+        aml_append(dev, method);
+
+        aml_append(sb_scope, dev);
+    }
+
+    /* build Method(MEMORY_SLOT_NOTIFY_METHOD, 2) {
+     *     If (LEqual(Arg0, 0x00)) {Notify(MP00, Arg1)} ... }
+     */
+    method = aml_method(MEMORY_SLOT_NOTIFY_METHOD, 2, AML_NOTSERIALIZED);
+    for (i = 0; i < nr_mem; i++) {
+        ifctx = aml_if(aml_equal(aml_arg(0), aml_int(i)));
+        aml_append(ifctx,
+            aml_notify(aml_name("MP%.02X", i), aml_arg(1))
+        );
+        aml_append(method, ifctx);
+    }
+    aml_append(sb_scope, method);
+}
+
+static void build_hpet_aml(Aml *table)
+{
+    Aml *crs;
+    Aml *field;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *scope = aml_scope("_SB");
+    Aml *dev = aml_device("HPET");
+    Aml *zero = aml_int(0);
+    Aml *id = aml_local(0);
+    Aml *period = aml_local(1);
+
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0103")));
+    aml_append(dev, aml_name_decl("_UID", zero));
+
+    aml_append(dev,
+        aml_operation_region("HPTM", AML_SYSTEM_MEMORY, aml_int(HPET_BASE),
+                             HPET_LEN));
+    field = aml_field("HPTM", AML_DWORD_ACC, AML_LOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("VEND", 32));
+    aml_append(field, aml_named_field("PRD", 32));
+    aml_append(dev, field);
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_store(aml_name("VEND"), id));
+    aml_append(method, aml_store(aml_name("PRD"), period));
+    aml_append(method, aml_shiftright(id, aml_int(16), id));
+    if_ctx = aml_if(aml_lor(aml_equal(id, zero),
+                            aml_equal(id, aml_int(0xffff))));
+    {
+        aml_append(if_ctx, aml_return(zero));
+    }
+    aml_append(method, if_ctx);
+
+    if_ctx = aml_if(aml_lor(aml_equal(period, zero),
+                            aml_lgreater(period, aml_int(100000000))));
+    {
+        aml_append(if_ctx, aml_return(zero));
+    }
+    aml_append(method, if_ctx);
+
+    aml_append(method, aml_return(aml_int(0x0F)));
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(HPET_BASE, HPET_LEN, AML_READ_ONLY));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+#ifdef CONFIG_ANDROID
+static void build_goldfish_device_aml(Aml *scope,
+                                      const char *dev_name,
+                                      const char *hid_name,
+                                      const char *str_name,
+                                      uint32_t iomem_base,
+                                      uint32_t iomem_size,
+                                      uint32_t irq_number)
+{
+    Aml *dev = aml_device(dev_name);
+    Aml *crs;
+
+    aml_append(dev, aml_name_decl("_HID", aml_string(hid_name)));
+    aml_append(dev, aml_name_decl("_STR", aml_unicode(str_name)));
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_memory32_fixed(iomem_base, iomem_size, AML_READ_WRITE));
+    aml_append(crs, aml_interrupt(AML_CONSUMER_PRODUCER, AML_EDGE,
+                                  AML_ACTIVE_HIGH, AML_SHARED, &irq_number, 1));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(scope, dev);
+}
+
+static void build_goldfish_aml(Aml *table)
+{
+    Aml *scope = aml_scope("_SB");
+
+    build_goldfish_device_aml(scope, "GFBY", "GFSH0001", "goldfish battery",
+                              GOLDFISH_BATTERY_IOMEM_BASE,
+                              GOLDFISH_BATTERY_IOMEM_SIZE,
+                              GOLDFISH_BATTERY_IRQ);
+
+    build_goldfish_device_aml(scope, "GFEV", "GFSH0002", "goldfish events",
+                              GOLDFISH_EVENTS_IOMEM_BASE,
+                              GOLDFISH_EVENTS_IOMEM_SIZE,
+                              GOLDFISH_EVENTS_IRQ);
+
+    build_goldfish_device_aml(scope, "GFPP", "GFSH0003", "goldfish pipe",
+                              GOLDFISH_PIPE_IOMEM_BASE,
+                              GOLDFISH_PIPE_IOMEM_SIZE,
+                              GOLDFISH_PIPE_IRQ);
+
+    build_goldfish_device_aml(scope, "GFFB", "GFSH0004", "goldfish framebuffer",
+                              GOLDFISH_FB_IOMEM_BASE,
+                              GOLDFISH_FB_IOMEM_SIZE,
+                              GOLDFISH_FB_IRQ);
+
+    build_goldfish_device_aml(scope, "GFAU", "GFSH0005", "goldfish audio",
+                              GOLDFISH_AUDIO_IOMEM_BASE,
+                              GOLDFISH_AUDIO_IOMEM_SIZE,
+                              GOLDFISH_AUDIO_IRQ);
+
+    build_goldfish_device_aml(scope, "GFSK", "GFSH0006", "goldfish sync",
+                              GOLDFISH_SYNC_IOMEM_BASE,
+                              GOLDFISH_SYNC_IOMEM_SIZE,
+                              GOLDFISH_SYNC_IRQ);
+
+    build_goldfish_device_aml(scope, "GFRT", "GFSH0007", "goldfish rtc",
+                              GOLDFISH_RTC_IOMEM_BASE,
+                              GOLDFISH_RTC_IOMEM_SIZE,
+                              GOLDFISH_RTC_IRQ);
+
+    aml_append(table, scope);
+}
+#endif  /* CONFIG_ANDROID */
+
+static Aml *build_fdinfo_aml(int idx, FloppyDriveType type)
+{
+    Aml *dev, *fdi;
+    uint8_t maxc, maxh, maxs;
+
+    isa_fdc_get_drive_max_chs(type, &maxc, &maxh, &maxs);
+
+    dev = aml_device("FLP%c", 'A' + idx);
+
+    aml_append(dev, aml_name_decl("_ADR", aml_int(idx)));
+
+    fdi = aml_package(16);
+    aml_append(fdi, aml_int(idx));  /* Drive Number */
+    aml_append(fdi,
+        aml_int(cmos_get_fd_drive_type(type)));  /* Device Type */
+    /*
+     * the values below are the limits of the drive, and are thus independent
+     * of the inserted media
+     */
+    aml_append(fdi, aml_int(maxc));  /* Maximum Cylinder Number */
+    aml_append(fdi, aml_int(maxs));  /* Maximum Sector Number */
+    aml_append(fdi, aml_int(maxh));  /* Maximum Head Number */
+    /*
+     * SeaBIOS returns the below values for int 0x13 func 0x08 regardless of
+     * the drive type, so shall we
+     */
+    aml_append(fdi, aml_int(0xAF));  /* disk_specify_1 */
+    aml_append(fdi, aml_int(0x02));  /* disk_specify_2 */
+    aml_append(fdi, aml_int(0x25));  /* disk_motor_wait */
+    aml_append(fdi, aml_int(0x02));  /* disk_sector_siz */
+    aml_append(fdi, aml_int(0x12));  /* disk_eot */
+    aml_append(fdi, aml_int(0x1B));  /* disk_rw_gap */
+    aml_append(fdi, aml_int(0xFF));  /* disk_dtl */
+    aml_append(fdi, aml_int(0x6C));  /* disk_formt_gap */
+    aml_append(fdi, aml_int(0xF6));  /* disk_fill */
+    aml_append(fdi, aml_int(0x0F));  /* disk_head_sttl */
+    aml_append(fdi, aml_int(0x08));  /* disk_motor_strt */
+
+    aml_append(dev, aml_name_decl("_FDI", fdi));
+    return dev;
+}
+
+static Aml *build_fdc_device_aml(ISADevice *fdc)
+{
+    int i;
+    Aml *dev;
+    Aml *crs;
+
+#define ACPI_FDE_MAX_FD 4
+    uint32_t fde_buf[5] = {
+        0, 0, 0, 0,     /* presence of floppy drives #0 - #3 */
+        cpu_to_le32(2)  /* tape presence (2 == never present) */
+    };
+
+    dev = aml_device("FDC0");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0700")));
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x03F2, 0x03F2, 0x00, 0x04));
+    aml_append(crs, aml_io(AML_DECODE16, 0x03F7, 0x03F7, 0x00, 0x01));
+    aml_append(crs, aml_irq_no_flags(6));
+    aml_append(crs,
+        aml_dma(AML_COMPATIBILITY, AML_NOTBUSMASTER, AML_TRANSFER8, 2));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    for (i = 0; i < MIN(MAX_FD, ACPI_FDE_MAX_FD); i++) {
+        FloppyDriveType type = isa_fdc_get_drive_type(fdc, i);
+
+        if (type < FLOPPY_DRIVE_TYPE_NONE) {
+            fde_buf[i] = cpu_to_le32(1);  /* drive present */
+            aml_append(dev, build_fdinfo_aml(i, type));
+        }
+    }
+    aml_append(dev, aml_name_decl("_FDE",
+               aml_buffer(sizeof(fde_buf), (uint8_t *)fde_buf)));
+
+    return dev;
+}
+
+static Aml *build_rtc_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+
+    dev = aml_device("RTC");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0B00")));
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x0070, 0x0070, 0x10, 0x02));
+    aml_append(crs, aml_irq_no_flags(8));
+    aml_append(crs, aml_io(AML_DECODE16, 0x0072, 0x0072, 0x02, 0x06));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_kbd_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+
+    dev = aml_device("KBD");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0303")));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_int(0x0f)));
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x0060, 0x0060, 0x01, 0x01));
+    aml_append(crs, aml_io(AML_DECODE16, 0x0064, 0x0064, 0x01, 0x01));
+    aml_append(crs, aml_irq_no_flags(1));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_mouse_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+
+    dev = aml_device("MOU");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0F13")));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_int(0x0f)));
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_irq_no_flags(12));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_lpt_device_aml(void)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *else_ctx;
+    Aml *zero = aml_int(0);
+    Aml *is_present = aml_local(0);
+
+    dev = aml_device("LPT");
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0400")));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_store(aml_name("LPEN"), is_present));
+    if_ctx = aml_if(aml_equal(is_present, zero));
+    {
+        aml_append(if_ctx, aml_return(aml_int(0x00)));
+    }
+    aml_append(method, if_ctx);
+    else_ctx = aml_else();
+    {
+        aml_append(else_ctx, aml_return(aml_int(0x0f)));
+    }
+    aml_append(method, else_ctx);
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, 0x0378, 0x0378, 0x08, 0x08));
+    aml_append(crs, aml_irq_no_flags(7));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static Aml *build_com_device_aml(uint8_t uid)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    Aml *if_ctx;
+    Aml *else_ctx;
+    Aml *zero = aml_int(0);
+    Aml *is_present = aml_local(0);
+    const char *enabled_field = "CAEN";
+    uint8_t irq = 4;
+    uint16_t io_port = 0x03F8;
+
+    assert(uid == 1 || uid == 2);
+    if (uid == 2) {
+        enabled_field = "CBEN";
+        irq = 3;
+        io_port = 0x02F8;
+    }
+
+    dev = aml_device("COM%d", uid);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0501")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(uid)));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_store(aml_name("%s", enabled_field), is_present));
+    if_ctx = aml_if(aml_equal(is_present, zero));
+    {
+        aml_append(if_ctx, aml_return(aml_int(0x00)));
+    }
+    aml_append(method, if_ctx);
+    else_ctx = aml_else();
+    {
+        aml_append(else_ctx, aml_return(aml_int(0x0f)));
+    }
+    aml_append(method, else_ctx);
+    aml_append(dev, method);
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_io(AML_DECODE16, io_port, io_port, 0x00, 0x08));
+    aml_append(crs, aml_irq_no_flags(irq));
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    return dev;
+}
+
+static void build_isa_devices_aml(Aml *table)
+{
+    ISADevice *fdc = pc_find_fdc0();
+    bool ambiguous;
+
+    Aml *scope = aml_scope("_SB.PCI0.ISA");
+    Object *obj = object_resolve_path_type("", TYPE_ISA_BUS, &ambiguous);
+
+    aml_append(scope, build_rtc_device_aml());
+    aml_append(scope, build_kbd_device_aml());
+    aml_append(scope, build_mouse_device_aml());
+    if (fdc) {
+        aml_append(scope, build_fdc_device_aml(fdc));
+    }
+    aml_append(scope, build_lpt_device_aml());
+    aml_append(scope, build_com_device_aml(1));
+    aml_append(scope, build_com_device_aml(2));
+
+    if (ambiguous) {
+        error_report("Multiple ISA busses, unable to define IPMI ACPI data");
+    } else if (!obj) {
+        error_report("No ISA bus, unable to define IPMI ACPI data");
+    } else {
+        build_acpi_ipmi_devices(scope, BUS(obj));
+    }
+
+    aml_append(table, scope);
+}
+
+static void build_dbg_aml(Aml *table)
+{
+    Aml *field;
+    Aml *method;
+    Aml *while_ctx;
+    Aml *scope = aml_scope("\\");
+    Aml *buf = aml_local(0);
+    Aml *len = aml_local(1);
+    Aml *idx = aml_local(2);
+
+    aml_append(scope,
+       aml_operation_region("DBG", AML_SYSTEM_IO, aml_int(0x0402), 0x01));
+    field = aml_field("DBG", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("DBGB", 8));
+    aml_append(scope, field);
+
+    method = aml_method("DBUG", 1, AML_NOTSERIALIZED);
+
+    aml_append(method, aml_to_hexstring(aml_arg(0), buf));
+    aml_append(method, aml_to_buffer(buf, buf));
+    aml_append(method, aml_subtract(aml_sizeof(buf), aml_int(1), len));
+    aml_append(method, aml_store(aml_int(0), idx));
+
+    while_ctx = aml_while(aml_lless(idx, len));
+    aml_append(while_ctx,
+        aml_store(aml_derefof(aml_index(buf, idx)), aml_name("DBGB")));
+    aml_append(while_ctx, aml_increment(idx));
+    aml_append(method, while_ctx);
+
+    aml_append(method, aml_store(aml_int(0x0A), aml_name("DBGB")));
+    aml_append(scope, method);
+
+    aml_append(table, scope);
+}
+
+static Aml *build_link_dev(const char *name, uint8_t uid, Aml *reg)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    uint32_t irqs[] = {5, 10, 11};
+
+    dev = aml_device("%s", name);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C0F")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(uid)));
+
+    crs = aml_resource_template();
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                  AML_SHARED, irqs, ARRAY_SIZE(irqs)));
+    aml_append(dev, aml_name_decl("_PRS", crs));
+
+    method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_call1("IQST", reg)));
+    aml_append(dev, method);
+
+    method = aml_method("_DIS", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_or(reg, aml_int(0x80), reg));
+    aml_append(dev, method);
+
+    method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+    aml_append(method, aml_return(aml_call1("IQCR", reg)));
+    aml_append(dev, method);
+
+    method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+    aml_append(method, aml_create_dword_field(aml_arg(0), aml_int(5), "PRRI"));
+    aml_append(method, aml_store(aml_name("PRRI"), reg));
+    aml_append(dev, method);
+
+    return dev;
+ }
+
+static Aml *build_gsi_link_dev(const char *name, uint8_t uid, uint8_t gsi)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *method;
+    uint32_t irqs;
+
+    dev = aml_device("%s", name);
+    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C0F")));
+    aml_append(dev, aml_name_decl("_UID", aml_int(uid)));
+
+    crs = aml_resource_template();
+    irqs = gsi;
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL, AML_ACTIVE_HIGH,
+                                  AML_SHARED, &irqs, 1));
+    aml_append(dev, aml_name_decl("_PRS", crs));
+
+    aml_append(dev, aml_name_decl("_CRS", crs));
+
+    /*
+     * _DIS can be no-op because the interrupt cannot be disabled.
+     */
+    method = aml_method("_DIS", 0, AML_NOTSERIALIZED);
+    aml_append(dev, method);
+
+    method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+    aml_append(dev, method);
+
+    return dev;
+}
+
+/* _CRS method - get current settings */
+static Aml *build_iqcr_method(bool is_piix4)
+{
+    Aml *if_ctx;
+    uint32_t irqs;
+    Aml *method = aml_method("IQCR", 1, AML_SERIALIZED);
+    Aml *crs = aml_resource_template();
+
+    irqs = 0;
+    aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL,
+                                  AML_ACTIVE_HIGH, AML_SHARED, &irqs, 1));
+    aml_append(method, aml_name_decl("PRR0", crs));
+
+    aml_append(method,
+        aml_create_dword_field(aml_name("PRR0"), aml_int(5), "PRRI"));
+
+    if (is_piix4) {
+        if_ctx = aml_if(aml_lless(aml_arg(0), aml_int(0x80)));
+        aml_append(if_ctx, aml_store(aml_arg(0), aml_name("PRRI")));
+        aml_append(method, if_ctx);
+    } else {
+        aml_append(method,
+            aml_store(aml_and(aml_arg(0), aml_int(0xF), NULL),
+                      aml_name("PRRI")));
+    }
+
+    aml_append(method, aml_return(aml_name("PRR0")));
+    return method;
+}
+
+/* _STA method - get status */
+static Aml *build_irq_status_method(void)
+{
+    Aml *if_ctx;
+    Aml *method = aml_method("IQST", 1, AML_NOTSERIALIZED);
+
+    if_ctx = aml_if(aml_and(aml_int(0x80), aml_arg(0), NULL));
+    aml_append(if_ctx, aml_return(aml_int(0x09)));
+    aml_append(method, if_ctx);
+    aml_append(method, aml_return(aml_int(0x0B)));
+    return method;
+}
+
+static void build_piix4_pci0_int(Aml *table)
+{
+    Aml *dev;
+    Aml *crs;
+    Aml *field;
+    Aml *method;
+    uint32_t irqs;
+    Aml *sb_scope = aml_scope("_SB");
+    Aml *pci0_scope = aml_scope("PCI0");
+
+    aml_append(pci0_scope, build_prt(true));
+    aml_append(sb_scope, pci0_scope);
+
+    field = aml_field("PCI0.ISA.P40C", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("PRQ0", 8));
+    aml_append(field, aml_named_field("PRQ1", 8));
+    aml_append(field, aml_named_field("PRQ2", 8));
+    aml_append(field, aml_named_field("PRQ3", 8));
+    aml_append(sb_scope, field);
+
+    aml_append(sb_scope, build_irq_status_method());
+    aml_append(sb_scope, build_iqcr_method(true));
+
+    aml_append(sb_scope, build_link_dev("LNKA", 0, aml_name("PRQ0")));
+    aml_append(sb_scope, build_link_dev("LNKB", 1, aml_name("PRQ1")));
+    aml_append(sb_scope, build_link_dev("LNKC", 2, aml_name("PRQ2")));
+    aml_append(sb_scope, build_link_dev("LNKD", 3, aml_name("PRQ3")));
+
+    dev = aml_device("LNKS");
+    {
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C0F")));
+        aml_append(dev, aml_name_decl("_UID", aml_int(4)));
+
+        crs = aml_resource_template();
+        irqs = 9;
+        aml_append(crs, aml_interrupt(AML_CONSUMER, AML_LEVEL,
+                                      AML_ACTIVE_HIGH, AML_SHARED,
+                                      &irqs, 1));
+        aml_append(dev, aml_name_decl("_PRS", crs));
+
+        /* The SCI cannot be disabled and is always attached to GSI 9,
+         * so these are no-ops.  We only need this link to override the
+         * polarity to active high and match the content of the MADT.
+         */
+        method = aml_method("_STA", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_return(aml_int(0x0b)));
+        aml_append(dev, method);
+
+        method = aml_method("_DIS", 0, AML_NOTSERIALIZED);
+        aml_append(dev, method);
+
+        method = aml_method("_CRS", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_return(aml_name("_PRS")));
+        aml_append(dev, method);
+
+        method = aml_method("_SRS", 1, AML_NOTSERIALIZED);
+        aml_append(dev, method);
+    }
+    aml_append(sb_scope, dev);
+
+    aml_append(table, sb_scope);
+}
+
+static void append_q35_prt_entry(Aml *ctx, uint32_t nr, const char *name)
+{
+    int i;
+    int head;
+    Aml *pkg;
+    char base = name[3] < 'E' ? 'A' : 'E';
+    char *s = g_strdup(name);
+    Aml *a_nr = aml_int((nr << 16) | 0xffff);
+
+    assert(strlen(s) == 4);
+
+    head = name[3] - base;
+    for (i = 0; i < 4; i++) {
+        if (head + i > 3) {
+            head = i * -1;
+        }
+        s[3] = base + head + i;
+        pkg = aml_package(4);
+        aml_append(pkg, a_nr);
+        aml_append(pkg, aml_int(i));
+        aml_append(pkg, aml_name("%s", s));
+        aml_append(pkg, aml_int(0));
+        aml_append(ctx, pkg);
+    }
+    g_free(s);
+}
+
+static Aml *build_q35_routing_table(const char *str)
+{
+    int i;
+    Aml *pkg;
+    char *name = g_strdup_printf("%s ", str);
+
+    pkg = aml_package(128);
+    for (i = 0; i < 0x18; i++) {
+            name[3] = 'E' + (i & 0x3);
+            append_q35_prt_entry(pkg, i, name);
+    }
+
+    name[3] = 'E';
+    append_q35_prt_entry(pkg, 0x18, name);
+
+    /* INTA -> PIRQA for slot 25 - 31, see the default value of D<N>IR */
+    for (i = 0x0019; i < 0x1e; i++) {
+        name[3] = 'A';
+        append_q35_prt_entry(pkg, i, name);
+    }
+
+    /* PCIe->PCI bridge. use PIRQ[E-H] */
+    name[3] = 'E';
+    append_q35_prt_entry(pkg, 0x1e, name);
+    name[3] = 'A';
+    append_q35_prt_entry(pkg, 0x1f, name);
+
+    g_free(name);
+    return pkg;
+}
+
+static void build_q35_pci0_int(Aml *table)
+{
+    Aml *field;
+    Aml *method;
+    Aml *sb_scope = aml_scope("_SB");
+    Aml *pci0_scope = aml_scope("PCI0");
+
+    /* Zero => PIC mode, One => APIC Mode */
+    aml_append(table, aml_name_decl("PICF", aml_int(0)));
+    method = aml_method("_PIC", 1, AML_NOTSERIALIZED);
+    {
+        aml_append(method, aml_store(aml_arg(0), aml_name("PICF")));
+    }
+    aml_append(table, method);
+
+    aml_append(pci0_scope,
+        aml_name_decl("PRTP", build_q35_routing_table("LNK")));
+    aml_append(pci0_scope,
+        aml_name_decl("PRTA", build_q35_routing_table("GSI")));
+
+    method = aml_method("_PRT", 0, AML_NOTSERIALIZED);
+    {
+        Aml *if_ctx;
+        Aml *else_ctx;
+
+        /* PCI IRQ routing table, example from ACPI 2.0a specification,
+           section 6.2.8.1 */
+        /* Note: we provide the same info as the PCI routing
+           table of the Bochs BIOS */
+        if_ctx = aml_if(aml_equal(aml_name("PICF"), aml_int(0)));
+        aml_append(if_ctx, aml_return(aml_name("PRTP")));
+        aml_append(method, if_ctx);
+        else_ctx = aml_else();
+        aml_append(else_ctx, aml_return(aml_name("PRTA")));
+        aml_append(method, else_ctx);
+    }
+    aml_append(pci0_scope, method);
+    aml_append(sb_scope, pci0_scope);
+
+    field = aml_field("PCI0.ISA.PIRQ", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("PRQA", 8));
+    aml_append(field, aml_named_field("PRQB", 8));
+    aml_append(field, aml_named_field("PRQC", 8));
+    aml_append(field, aml_named_field("PRQD", 8));
+    aml_append(field, aml_reserved_field(0x20));
+    aml_append(field, aml_named_field("PRQE", 8));
+    aml_append(field, aml_named_field("PRQF", 8));
+    aml_append(field, aml_named_field("PRQG", 8));
+    aml_append(field, aml_named_field("PRQH", 8));
+    aml_append(sb_scope, field);
+
+    aml_append(sb_scope, build_irq_status_method());
+    aml_append(sb_scope, build_iqcr_method(false));
+
+    aml_append(sb_scope, build_link_dev("LNKA", 0, aml_name("PRQA")));
+    aml_append(sb_scope, build_link_dev("LNKB", 1, aml_name("PRQB")));
+    aml_append(sb_scope, build_link_dev("LNKC", 2, aml_name("PRQC")));
+    aml_append(sb_scope, build_link_dev("LNKD", 3, aml_name("PRQD")));
+    aml_append(sb_scope, build_link_dev("LNKE", 4, aml_name("PRQE")));
+    aml_append(sb_scope, build_link_dev("LNKF", 5, aml_name("PRQF")));
+    aml_append(sb_scope, build_link_dev("LNKG", 6, aml_name("PRQG")));
+    aml_append(sb_scope, build_link_dev("LNKH", 7, aml_name("PRQH")));
+
+    aml_append(sb_scope, build_gsi_link_dev("GSIA", 0x10, 0x10));
+    aml_append(sb_scope, build_gsi_link_dev("GSIB", 0x11, 0x11));
+    aml_append(sb_scope, build_gsi_link_dev("GSIC", 0x12, 0x12));
+    aml_append(sb_scope, build_gsi_link_dev("GSID", 0x13, 0x13));
+    aml_append(sb_scope, build_gsi_link_dev("GSIE", 0x14, 0x14));
+    aml_append(sb_scope, build_gsi_link_dev("GSIF", 0x15, 0x15));
+    aml_append(sb_scope, build_gsi_link_dev("GSIG", 0x16, 0x16));
+    aml_append(sb_scope, build_gsi_link_dev("GSIH", 0x17, 0x17));
+
+    aml_append(table, sb_scope);
+}
+
+static void build_q35_isa_bridge(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+    Aml *field;
+
+    scope =  aml_scope("_SB.PCI0");
+    dev = aml_device("ISA");
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0x001F0000)));
+
+    /* ICH9 PCI to ISA irq remapping */
+    aml_append(dev, aml_operation_region("PIRQ", AML_PCI_CONFIG,
+                                         aml_int(0x60), 0x0C));
+
+    aml_append(dev, aml_operation_region("LPCD", AML_PCI_CONFIG,
+                                         aml_int(0x80), 0x02));
+    field = aml_field("LPCD", AML_ANY_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("COMA", 3));
+    aml_append(field, aml_reserved_field(1));
+    aml_append(field, aml_named_field("COMB", 3));
+    aml_append(field, aml_reserved_field(1));
+    aml_append(field, aml_named_field("LPTD", 2));
+    aml_append(dev, field);
+
+    aml_append(dev, aml_operation_region("LPCE", AML_PCI_CONFIG,
+                                         aml_int(0x82), 0x02));
+    /* enable bits */
+    field = aml_field("LPCE", AML_ANY_ACC, AML_NOLOCK, AML_PRESERVE);
+    aml_append(field, aml_named_field("CAEN", 1));
+    aml_append(field, aml_named_field("CBEN", 1));
+    aml_append(field, aml_named_field("LPEN", 1));
+    aml_append(dev, field);
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static void build_piix4_pm(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+
+    scope =  aml_scope("_SB.PCI0");
+    dev = aml_device("PX13");
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0x00010003)));
+
+    aml_append(dev, aml_operation_region("P13C", AML_PCI_CONFIG,
+                                         aml_int(0x00), 0xff));
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static void build_piix4_isa_bridge(Aml *table)
+{
+    Aml *dev;
+    Aml *scope;
+    Aml *field;
+
+    scope =  aml_scope("_SB.PCI0");
+    dev = aml_device("ISA");
+    aml_append(dev, aml_name_decl("_ADR", aml_int(0x00010000)));
+
+    /* PIIX PCI to ISA irq remapping */
+    aml_append(dev, aml_operation_region("P40C", AML_PCI_CONFIG,
+                                         aml_int(0x60), 0x04));
+    /* enable bits */
+    field = aml_field("^PX13.P13C", AML_ANY_ACC, AML_NOLOCK, AML_PRESERVE);
+    /* Offset(0x5f),, 7, */
+    aml_append(field, aml_reserved_field(0x2f8));
+    aml_append(field, aml_reserved_field(7));
+    aml_append(field, aml_named_field("LPEN", 1));
+    /* Offset(0x67),, 3, */
+    aml_append(field, aml_reserved_field(0x38));
+    aml_append(field, aml_reserved_field(3));
+    aml_append(field, aml_named_field("CAEN", 1));
+    aml_append(field, aml_reserved_field(3));
+    aml_append(field, aml_named_field("CBEN", 1));
+    aml_append(dev, field);
+
+    aml_append(scope, dev);
+    aml_append(table, scope);
+}
+
+static void build_piix4_pci_hotplug(Aml *table)
+{
+    Aml *scope;
+    Aml *field;
+    Aml *method;
+
+    scope =  aml_scope("_SB.PCI0");
+
+    aml_append(scope,
+        aml_operation_region("PCST", AML_SYSTEM_IO, aml_int(0xae00), 0x08));
+    field = aml_field("PCST", AML_DWORD_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_named_field("PCIU", 32));
+    aml_append(field, aml_named_field("PCID", 32));
+    aml_append(scope, field);
+
+    aml_append(scope,
+        aml_operation_region("SEJ", AML_SYSTEM_IO, aml_int(0xae08), 0x04));
+    field = aml_field("SEJ", AML_DWORD_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_named_field("B0EJ", 32));
+    aml_append(scope, field);
+
+    aml_append(scope,
+        aml_operation_region("BNMR", AML_SYSTEM_IO, aml_int(0xae10), 0x04));
+    field = aml_field("BNMR", AML_DWORD_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+    aml_append(field, aml_named_field("BNUM", 32));
+    aml_append(scope, field);
+
+    aml_append(scope, aml_mutex("BLCK", 0));
+
+    method = aml_method("PCEJ", 2, AML_NOTSERIALIZED);
+    aml_append(method, aml_acquire(aml_name("BLCK"), 0xFFFF));
+    aml_append(method, aml_store(aml_arg(0), aml_name("BNUM")));
+    aml_append(method,
+        aml_store(aml_shiftleft(aml_int(1), aml_arg(1)), aml_name("B0EJ")));
+    aml_append(method, aml_release(aml_name("BLCK")));
+    aml_append(method, aml_return(aml_int(0)));
+    aml_append(scope, method);
+
+    aml_append(table, scope);
+}
+
+static Aml *build_q35_osc_method(void)
+{
+    Aml *if_ctx;
+    Aml *if_ctx2;
+    Aml *else_ctx;
+    Aml *method;
+    Aml *a_cwd1 = aml_name("CDW1");
+    Aml *a_ctrl = aml_name("CTRL");
+
+    method = aml_method("_OSC", 4, AML_NOTSERIALIZED);
+    aml_append(method, aml_create_dword_field(aml_arg(3), aml_int(0), "CDW1"));
+
+    if_ctx = aml_if(aml_equal(
+        aml_arg(0), aml_touuid("33DB4D5B-1FF7-401C-9657-7441C03DD766")));
+    aml_append(if_ctx, aml_create_dword_field(aml_arg(3), aml_int(4), "CDW2"));
+    aml_append(if_ctx, aml_create_dword_field(aml_arg(3), aml_int(8), "CDW3"));
+
+    aml_append(if_ctx, aml_store(aml_name("CDW2"), aml_name("SUPP")));
+    aml_append(if_ctx, aml_store(aml_name("CDW3"), a_ctrl));
+
+    /*
+     * Always allow native PME, AER (no dependencies)
+     * Never allow SHPC (no SHPC controller in this system)
+     */
+    aml_append(if_ctx, aml_and(a_ctrl, aml_int(0x1D), a_ctrl));
+
+    if_ctx2 = aml_if(aml_lnot(aml_equal(aml_arg(1), aml_int(1))));
+    /* Unknown revision */
+    aml_append(if_ctx2, aml_or(a_cwd1, aml_int(0x08), a_cwd1));
+    aml_append(if_ctx, if_ctx2);
+
+    if_ctx2 = aml_if(aml_lnot(aml_equal(aml_name("CDW3"), a_ctrl)));
+    /* Capabilities bits were masked */
+    aml_append(if_ctx2, aml_or(a_cwd1, aml_int(0x10), a_cwd1));
+    aml_append(if_ctx, if_ctx2);
+
+    /* Update DWORD3 in the buffer */
+    aml_append(if_ctx, aml_store(a_ctrl, aml_name("CDW3")));
+    aml_append(method, if_ctx);
+
+    else_ctx = aml_else();
+    /* Unrecognized UUID */
+    aml_append(else_ctx, aml_or(a_cwd1, aml_int(4), a_cwd1));
+    aml_append(method, else_ctx);
+
+    aml_append(method, aml_return(aml_arg(3)));
+    return method;
 }
 
 static void
-build_ssdt(GArray *table_data, GArray *linker,
-           AcpiCpuInfo *cpu, AcpiPmInfo *pm, AcpiMiscInfo *misc,
-           PcPciInfo *pci, PcGuestInfo *guest_info)
+build_dsdt(GArray *table_data, BIOSLinker *linker,
+           AcpiPmInfo *pm, AcpiMiscInfo *misc,
+           Range *pci_hole, Range *pci_hole64, MachineState *machine)
 {
-    MachineState *machine = MACHINE(qdev_get_machine());
+    CrsRangeEntry *entry;
+    Aml *dsdt, *sb_scope, *scope, *dev, *method, *field, *pkg, *crs;
+    CrsRangeSet crs_range_set;
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(machine);
     uint32_t nr_mem = machine->ram_slots;
-    unsigned acpi_cpus = guest_info->apic_id_limit;
-    int ssdt_start = table_data->len;
-    uint8_t *ssdt_ptr;
+    int root_bus_limit = 0xFF;
+    PCIBus *bus = NULL;
     int i;
 
-    /* The current AML generator can cover the APIC ID range [0..255],
-     * inclusive, for VCPU hotplug. */
-    QEMU_BUILD_BUG_ON(ACPI_CPU_HOTPLUG_ID_LIMIT > 256);
-    g_assert(acpi_cpus <= ACPI_CPU_HOTPLUG_ID_LIMIT);
+    dsdt = init_aml_allocator();
 
-    /* Copy header and patch values in the S3_ / S4_ / S5_ packages */
-    ssdt_ptr = acpi_data_push(table_data, sizeof(ssdp_misc_aml));
-    memcpy(ssdt_ptr, ssdp_misc_aml, sizeof(ssdp_misc_aml));
-    if (pm->s3_disabled) {
-        ssdt_ptr[acpi_s3_name[0]] = 'X';
-    }
-    if (pm->s4_disabled) {
-        ssdt_ptr[acpi_s4_name[0]] = 'X';
+    /* Reserve space for header */
+    acpi_data_push(dsdt->buf, sizeof(AcpiTableHeader));
+
+    build_dbg_aml(dsdt);
+    if (misc->is_piix4) {
+        sb_scope = aml_scope("_SB");
+        dev = aml_device("PCI0");
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A03")));
+        aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+        aml_append(dev, aml_name_decl("_UID", aml_int(1)));
+        aml_append(sb_scope, dev);
+        aml_append(dsdt, sb_scope);
+
+        build_hpet_aml(dsdt);
+        build_piix4_pm(dsdt);
+        build_piix4_isa_bridge(dsdt);
+        build_isa_devices_aml(dsdt);
+        build_piix4_pci_hotplug(dsdt);
+        build_piix4_pci0_int(dsdt);
     } else {
-        ssdt_ptr[acpi_s4_pkg[0] + 1] = ssdt_ptr[acpi_s4_pkg[0] + 3] =
-            pm->s4_val;
+        sb_scope = aml_scope("_SB");
+        aml_append(sb_scope,
+            aml_operation_region("PCST", AML_SYSTEM_IO, aml_int(0xae00), 0x0c));
+        aml_append(sb_scope,
+            aml_operation_region("PCSB", AML_SYSTEM_IO, aml_int(0xae0c), 0x01));
+        field = aml_field("PCSB", AML_ANY_ACC, AML_NOLOCK, AML_WRITE_AS_ZEROS);
+        aml_append(field, aml_named_field("PCIB", 8));
+        aml_append(sb_scope, field);
+        aml_append(dsdt, sb_scope);
+
+        sb_scope = aml_scope("_SB");
+        dev = aml_device("PCI0");
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A08")));
+        aml_append(dev, aml_name_decl("_CID", aml_eisaid("PNP0A03")));
+        aml_append(dev, aml_name_decl("_ADR", aml_int(0)));
+        aml_append(dev, aml_name_decl("_UID", aml_int(1)));
+        aml_append(dev, aml_name_decl("SUPP", aml_int(0)));
+        aml_append(dev, aml_name_decl("CTRL", aml_int(0)));
+        aml_append(dev, build_q35_osc_method());
+        aml_append(sb_scope, dev);
+        aml_append(dsdt, sb_scope);
+
+        build_hpet_aml(dsdt);
+        build_q35_isa_bridge(dsdt);
+        build_isa_devices_aml(dsdt);
+        build_q35_pci0_int(dsdt);
+    }
+#ifdef CONFIG_ANDROID
+    build_goldfish_aml(dsdt);
+#endif
+
+    if (pcmc->legacy_cpu_hotplug) {
+        build_legacy_cpu_hotplug_aml(dsdt, machine, pm->cpu_hp_io_base);
+    } else {
+        CPUHotplugFeatures opts = {
+            .apci_1_compatible = true, .has_legacy_cphp = true
+        };
+        build_cpus_aml(dsdt, machine, opts, pm->cpu_hp_io_base,
+                       "\\_SB.PCI0", "\\_GPE._E02");
+    }
+    build_memory_hotplug_aml(dsdt, nr_mem, pm->mem_hp_io_base,
+                             pm->mem_hp_io_len);
+
+    scope =  aml_scope("_GPE");
+    {
+        aml_append(scope, aml_name_decl("_HID", aml_string("ACPI0006")));
+
+        if (misc->is_piix4) {
+            method = aml_method("_E01", 0, AML_NOTSERIALIZED);
+            aml_append(method,
+                aml_acquire(aml_name("\\_SB.PCI0.BLCK"), 0xFFFF));
+            aml_append(method, aml_call0("\\_SB.PCI0.PCNT"));
+            aml_append(method, aml_release(aml_name("\\_SB.PCI0.BLCK")));
+            aml_append(scope, method);
+        }
+
+        method = aml_method("_E03", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_call0(MEMORY_HOTPLUG_HANDLER_PATH));
+        aml_append(scope, method);
+    }
+    aml_append(dsdt, scope);
+
+    crs_range_set_init(&crs_range_set);
+    bus = PC_MACHINE(machine)->bus;
+    if (bus) {
+        QLIST_FOREACH(bus, &bus->child, sibling) {
+            uint8_t bus_num = pci_bus_num(bus);
+            uint8_t numa_node = pci_bus_numa_node(bus);
+
+            /* look only for expander root buses */
+            if (!pci_bus_is_root(bus)) {
+                continue;
+            }
+
+            if (bus_num < root_bus_limit) {
+                root_bus_limit = bus_num - 1;
+            }
+
+            scope = aml_scope("\\_SB");
+            dev = aml_device("PC%.02X", bus_num);
+            aml_append(dev, aml_name_decl("_UID", aml_int(bus_num)));
+            aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0A03")));
+            aml_append(dev, aml_name_decl("_BBN", aml_int(bus_num)));
+
+            if (numa_node != NUMA_NODE_UNASSIGNED) {
+                aml_append(dev, aml_name_decl("_PXM", aml_int(numa_node)));
+            }
+
+            aml_append(dev, build_prt(false));
+            crs = build_crs(PCI_HOST_BRIDGE(BUS(bus)->parent), &crs_range_set);
+            aml_append(dev, aml_name_decl("_CRS", crs));
+            aml_append(scope, dev);
+            aml_append(dsdt, scope);
+        }
     }
 
-    patch_pci_windows(pci, ssdt_ptr, sizeof(ssdp_misc_aml));
+    scope = aml_scope("\\_SB.PCI0");
+    /* build PCI0._CRS */
+    crs = aml_resource_template();
+    aml_append(crs,
+        aml_word_bus_number(AML_MIN_FIXED, AML_MAX_FIXED, AML_POS_DECODE,
+                            0x0000, 0x0, root_bus_limit,
+                            0x0000, root_bus_limit + 1));
+    aml_append(crs, aml_io(AML_DECODE16, 0x0CF8, 0x0CF8, 0x01, 0x08));
 
-    ACPI_BUILD_SET_LE(ssdt_ptr, sizeof(ssdp_misc_aml),
-                      ssdt_isa_pest[0], 16, misc->pvpanic_port);
+    aml_append(crs,
+        aml_word_io(AML_MIN_FIXED, AML_MAX_FIXED,
+                    AML_POS_DECODE, AML_ENTIRE_RANGE,
+                    0x0000, 0x0000, 0x0CF7, 0x0000, 0x0CF8));
 
-    ACPI_BUILD_SET_LE(ssdt_ptr, sizeof(ssdp_misc_aml),
-                      ssdt_mctrl_nr_slots[0], 32, nr_mem);
+    crs_replace_with_free_ranges(crs_range_set.io_ranges, 0x0D00, 0xFFFF);
+    for (i = 0; i < crs_range_set.io_ranges->len; i++) {
+        entry = g_ptr_array_index(crs_range_set.io_ranges, i);
+        aml_append(crs,
+            aml_word_io(AML_MIN_FIXED, AML_MAX_FIXED,
+                        AML_POS_DECODE, AML_ENTIRE_RANGE,
+                        0x0000, entry->base, entry->limit,
+                        0x0000, entry->limit - entry->base + 1));
+    }
 
+    aml_append(crs,
+        aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                         AML_CACHEABLE, AML_READ_WRITE,
+                         0, 0x000A0000, 0x000BFFFF, 0, 0x00020000));
+
+    crs_replace_with_free_ranges(crs_range_set.mem_ranges,
+                                 range_lob(pci_hole),
+                                 range_upb(pci_hole));
+    for (i = 0; i < crs_range_set.mem_ranges->len; i++) {
+        entry = g_ptr_array_index(crs_range_set.mem_ranges, i);
+        aml_append(crs,
+            aml_dword_memory(AML_POS_DECODE, AML_MIN_FIXED, AML_MAX_FIXED,
+                             AML_NON_CACHEABLE, AML_READ_WRITE,
+                             0, entry->base, entry->limit,
+                             0, entry->limit - entry->base + 1));
+    }
+
+    if (!range_is_empty(pci_hole64)) {
+        crs_replace_with_free_ranges(crs_range_set.mem_64bit_ranges,
+                                     range_lob(pci_hole64),
+                                     range_upb(pci_hole64));
+        for (i = 0; i < crs_range_set.mem_64bit_ranges->len; i++) {
+            entry = g_ptr_array_index(crs_range_set.mem_64bit_ranges, i);
+            aml_append(crs,
+                       aml_qword_memory(AML_POS_DECODE, AML_MIN_FIXED,
+                                        AML_MAX_FIXED,
+                                        AML_CACHEABLE, AML_READ_WRITE,
+                                        0, entry->base, entry->limit,
+                                        0, entry->limit - entry->base + 1));
+        }
+    }
+
+    if (misc->tpm_version != TPM_VERSION_UNSPEC) {
+        aml_append(crs, aml_memory32_fixed(TPM_TIS_ADDR_BASE,
+                   TPM_TIS_ADDR_SIZE, AML_READ_WRITE));
+    }
+    aml_append(scope, aml_name_decl("_CRS", crs));
+
+    /* reserve GPE0 block resources */
+    dev = aml_device("GPE0");
+    aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A06")));
+    aml_append(dev, aml_name_decl("_UID", aml_string("GPE0 resources")));
+    /* device present, functioning, decoding, not shown in UI */
+    aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+    crs = aml_resource_template();
+    aml_append(crs,
+        aml_io(AML_DECODE16, pm->gpe0_blk, pm->gpe0_blk, 1, pm->gpe0_blk_len)
+    );
+    aml_append(dev, aml_name_decl("_CRS", crs));
+    aml_append(scope, dev);
+
+    crs_range_set_free(&crs_range_set);
+
+    /* reserve PCIHP resources */
+    if (pm->pcihp_io_len) {
+        dev = aml_device("PHPR");
+        aml_append(dev, aml_name_decl("_HID", aml_string("PNP0A06")));
+        aml_append(dev,
+            aml_name_decl("_UID", aml_string("PCI Hotplug resources")));
+        /* device present, functioning, decoding, not shown in UI */
+        aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+        crs = aml_resource_template();
+        aml_append(crs,
+            aml_io(AML_DECODE16, pm->pcihp_io_base, pm->pcihp_io_base, 1,
+                   pm->pcihp_io_len)
+        );
+        aml_append(dev, aml_name_decl("_CRS", crs));
+        aml_append(scope, dev);
+    }
+    aml_append(dsdt, scope);
+
+    /*  create S3_ / S4_ / S5_ packages if necessary */
+    scope = aml_scope("\\");
+    if (!pm->s3_disabled) {
+        pkg = aml_package(4);
+        aml_append(pkg, aml_int(1)); /* PM1a_CNT.SLP_TYP */
+        aml_append(pkg, aml_int(1)); /* PM1b_CNT.SLP_TYP, FIXME: not impl. */
+        aml_append(pkg, aml_int(0)); /* reserved */
+        aml_append(pkg, aml_int(0)); /* reserved */
+        aml_append(scope, aml_name_decl("_S3", pkg));
+    }
+
+    if (!pm->s4_disabled) {
+        pkg = aml_package(4);
+        aml_append(pkg, aml_int(pm->s4_val)); /* PM1a_CNT.SLP_TYP */
+        /* PM1b_CNT.SLP_TYP, FIXME: not impl. */
+        aml_append(pkg, aml_int(pm->s4_val));
+        aml_append(pkg, aml_int(0)); /* reserved */
+        aml_append(pkg, aml_int(0)); /* reserved */
+        aml_append(scope, aml_name_decl("_S4", pkg));
+    }
+
+    pkg = aml_package(4);
+    aml_append(pkg, aml_int(0)); /* PM1a_CNT.SLP_TYP */
+    aml_append(pkg, aml_int(0)); /* PM1b_CNT.SLP_TYP not impl. */
+    aml_append(pkg, aml_int(0)); /* reserved */
+    aml_append(pkg, aml_int(0)); /* reserved */
+    aml_append(scope, aml_name_decl("_S5", pkg));
+    aml_append(dsdt, scope);
+
+    /* create fw_cfg node, unconditionally */
     {
-        GArray *sb_scope = build_alloc_array();
-        uint8_t op = 0x10; /* ScopeOp */
+        /* when using port i/o, the 8-bit data register *always* overlaps
+         * with half of the 16-bit control register. Hence, the total size
+         * of the i/o region used is FW_CFG_CTL_SIZE; when using DMA, the
+         * DMA control register is located at FW_CFG_DMA_IO_BASE + 4 */
+        uint8_t io_size = object_property_get_bool(OBJECT(pcms->fw_cfg),
+                                                   "dma_enabled", NULL) ?
+                          ROUND_UP(FW_CFG_CTL_SIZE, 4) + sizeof(dma_addr_t) :
+                          FW_CFG_CTL_SIZE;
 
-        build_append_nameseg(sb_scope, "_SB_");
+        scope = aml_scope("\\_SB.PCI0");
+        dev = aml_device("FWCF");
 
-        /* build Processor object for each processor */
-        for (i = 0; i < acpi_cpus; i++) {
-            uint8_t *proc = acpi_data_push(sb_scope, ACPI_PROC_SIZEOF);
-            memcpy(proc, ACPI_PROC_AML, ACPI_PROC_SIZEOF);
-            proc[ACPI_PROC_OFFSET_CPUHEX] = acpi_get_hex(i >> 4);
-            proc[ACPI_PROC_OFFSET_CPUHEX+1] = acpi_get_hex(i);
-            proc[ACPI_PROC_OFFSET_CPUID1] = i;
-            proc[ACPI_PROC_OFFSET_CPUID2] = i;
-        }
+        aml_append(dev, aml_name_decl("_HID", aml_string("QEMU0002")));
 
-        /* build this code:
-         *   Method(NTFY, 2) {If (LEqual(Arg0, 0x00)) {Notify(CP00, Arg1)} ...}
-         */
-        /* Arg0 = Processor ID = APIC ID */
-        build_append_notify_method(sb_scope, "NTFY", "CP%0.02X", acpi_cpus);
+        /* device present, functioning, decoding, not shown in UI */
+        aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
 
-        /* build "Name(CPON, Package() { One, One, ..., Zero, Zero, ... })" */
-        build_append_byte(sb_scope, 0x08); /* NameOp */
-        build_append_nameseg(sb_scope, "CPON");
+        crs = aml_resource_template();
+        aml_append(crs,
+            aml_io(AML_DECODE16, FW_CFG_IO_BASE, FW_CFG_IO_BASE, 0x01, io_size)
+        );
+        aml_append(dev, aml_name_decl("_CRS", crs));
+
+        aml_append(scope, dev);
+        aml_append(dsdt, scope);
+    }
+
+    if (misc->applesmc_io_base) {
+        scope = aml_scope("\\_SB.PCI0.ISA");
+        dev = aml_device("SMC");
+
+        aml_append(dev, aml_name_decl("_HID", aml_eisaid("APP0001")));
+        /* device present, functioning, decoding, not shown in UI */
+        aml_append(dev, aml_name_decl("_STA", aml_int(0xB)));
+
+        crs = aml_resource_template();
+        aml_append(crs,
+            aml_io(AML_DECODE16, misc->applesmc_io_base, misc->applesmc_io_base,
+                   0x01, APPLESMC_MAX_DATA_LENGTH)
+        );
+        aml_append(crs, aml_irq_no_flags(6));
+        aml_append(dev, aml_name_decl("_CRS", crs));
+
+        aml_append(scope, dev);
+        aml_append(dsdt, scope);
+    }
+
+    if (misc->pvpanic_port) {
+        scope = aml_scope("\\_SB.PCI0.ISA");
+
+        dev = aml_device("PEVT");
+        aml_append(dev, aml_name_decl("_HID", aml_string("QEMU0001")));
+
+        crs = aml_resource_template();
+        aml_append(crs,
+            aml_io(AML_DECODE16, misc->pvpanic_port, misc->pvpanic_port, 1, 1)
+        );
+        aml_append(dev, aml_name_decl("_CRS", crs));
+
+        aml_append(dev, aml_operation_region("PEOR", AML_SYSTEM_IO,
+                                              aml_int(misc->pvpanic_port), 1));
+        field = aml_field("PEOR", AML_BYTE_ACC, AML_NOLOCK, AML_PRESERVE);
+        aml_append(field, aml_named_field("PEPT", 8));
+        aml_append(dev, field);
+
+        /* device present, functioning, decoding, shown in UI */
+        aml_append(dev, aml_name_decl("_STA", aml_int(0xF)));
+
+        method = aml_method("RDPT", 0, AML_NOTSERIALIZED);
+        aml_append(method, aml_store(aml_name("PEPT"), aml_local(0)));
+        aml_append(method, aml_return(aml_local(0)));
+        aml_append(dev, method);
+
+        method = aml_method("WRPT", 1, AML_NOTSERIALIZED);
+        aml_append(method, aml_store(aml_arg(0), aml_name("PEPT")));
+        aml_append(dev, method);
+
+        aml_append(scope, dev);
+        aml_append(dsdt, scope);
+    }
+
+    sb_scope = aml_scope("\\_SB");
+    {
+        build_memory_devices(sb_scope, nr_mem, pm->mem_hp_io_base,
+                             pm->mem_hp_io_len);
 
         {
-            GArray *package = build_alloc_array();
-            uint8_t op;
-
-            /*
-             * Note: The ability to create variable-sized packages was first introduced in ACPI 2.0. ACPI 1.0 only
-             * allowed fixed-size packages with up to 255 elements.
-             * Windows guests up to win2k8 fail when VarPackageOp is used.
-             */
-            if (acpi_cpus <= 255) {
-                op = 0x12; /* PackageOp */
-                build_append_byte(package, acpi_cpus); /* NumElements */
-            } else {
-                op = 0x13; /* VarPackageOp */
-                build_append_int(package, acpi_cpus); /* VarNumElements */
-            }
-
-            for (i = 0; i < acpi_cpus; i++) {
-                uint8_t b = test_bit(i, cpu->found_cpus) ? 0x01 : 0x00;
-                build_append_byte(package, b);
-            }
-
-            build_package(package, op, 2);
-            build_append_array(sb_scope, package);
-            build_free_array(package);
-        }
-
-        if (nr_mem) {
-            assert(nr_mem <= ACPI_MAX_RAM_SLOTS);
-            /* build memory devices */
-            for (i = 0; i < nr_mem; i++) {
-                char id[3];
-                uint8_t *mem = acpi_data_push(sb_scope, ACPI_MEM_SIZEOF);
-
-                snprintf(id, sizeof(id), "%02X", i);
-                memcpy(mem, ACPI_MEM_AML, ACPI_MEM_SIZEOF);
-                memcpy(mem + ACPI_MEM_OFFSET_HEX, id, 2);
-                memcpy(mem + ACPI_MEM_OFFSET_ID, id, 2);
-            }
-
-            /* build Method(MEMORY_SLOT_NOTIFY_METHOD, 2) {
-             *     If (LEqual(Arg0, 0x00)) {Notify(MP00, Arg1)} ...
-             */
-            build_append_notify_method(sb_scope,
-                                       stringify(MEMORY_SLOT_NOTIFY_METHOD),
-                                       "MP%0.02X", nr_mem);
-        }
-
-        {
-            AcpiBuildPciBusHotplugState hotplug_state;
             Object *pci_host;
             PCIBus *bus = NULL;
-            bool ambiguous;
 
-            pci_host = object_resolve_path_type("", TYPE_PCI_HOST_BRIDGE, &ambiguous);
-            if (!ambiguous && pci_host) {
+            pci_host = acpi_get_i386_pci_host();
+            if (pci_host) {
                 bus = PCI_HOST_BRIDGE(pci_host)->bus;
             }
 
-            build_pci_bus_state_init(&hotplug_state, NULL, pm->pcihp_bridge_en);
-
             if (bus) {
+                Aml *scope = aml_scope("PCI0");
                 /* Scan all PCI buses. Generate tables to support hotplug. */
-                pci_for_each_bus_depth_first(bus, build_pci_bus_begin,
-                                             build_pci_bus_end, &hotplug_state);
+                build_append_pci_bus_devices(scope, bus, pm->pcihp_bridge_en);
+
+                if (misc->tpm_version != TPM_VERSION_UNSPEC) {
+                    dev = aml_device("ISA.TPM");
+                    aml_append(dev, aml_name_decl("_HID", aml_eisaid("PNP0C31")));
+                    aml_append(dev, aml_name_decl("_STA", aml_int(0xF)));
+                    crs = aml_resource_template();
+                    aml_append(crs, aml_memory32_fixed(TPM_TIS_ADDR_BASE,
+                               TPM_TIS_ADDR_SIZE, AML_READ_WRITE));
+                    /*
+                        FIXME: TPM_TIS_IRQ=5 conflicts with PNP0C0F irqs,
+                        Rewrite to take IRQ from TPM device model and
+                        fix default IRQ value there to use some unused IRQ
+                     */
+                    /* aml_append(crs, aml_irq_no_flags(TPM_TIS_IRQ)); */
+                    aml_append(dev, aml_name_decl("_CRS", crs));
+                    aml_append(scope, dev);
+                }
+
+                aml_append(sb_scope, scope);
             }
-
-            build_append_array(sb_scope, hotplug_state.device_table);
-            build_pci_bus_state_cleanup(&hotplug_state);
         }
-
-        build_package(sb_scope, op, 3);
-        build_append_array(table_data, sb_scope);
-        build_free_array(sb_scope);
+        aml_append(dsdt, sb_scope);
     }
 
+    /* copy AML table into ACPI tables blob and patch header there */
+    g_array_append_vals(table_data, dsdt->buf->data, dsdt->buf->len);
     build_header(linker, table_data,
-                 (void *)(table_data->data + ssdt_start),
-                 "SSDT", table_data->len - ssdt_start, 1);
+        (void *)(table_data->data + table_data->len - dsdt->buf->len),
+        "DSDT", dsdt->buf->len, 1, NULL, NULL);
+    free_aml_allocator();
 }
 
 static void
-build_hpet(GArray *table_data, GArray *linker)
+build_hpet(GArray *table_data, BIOSLinker *linker)
 {
     Acpi20Hpet *hpet;
 
@@ -1212,75 +2417,61 @@
     hpet->timer_block_id = cpu_to_le32(0x8086a201);
     hpet->addr.address = cpu_to_le64(HPET_BASE);
     build_header(linker, table_data,
-                 (void *)hpet, "HPET", sizeof(*hpet), 1);
+                 (void *)hpet, "HPET", sizeof(*hpet), 1, NULL, NULL);
 }
 
 static void
-build_tpm_tcpa(GArray *table_data, GArray *linker, GArray *tcpalog)
+build_tpm_tcpa(GArray *table_data, BIOSLinker *linker, GArray *tcpalog)
 {
     Acpi20Tcpa *tcpa = acpi_data_push(table_data, sizeof *tcpa);
-    uint64_t log_area_start_address = acpi_data_len(tcpalog);
+    unsigned log_addr_size = sizeof(tcpa->log_area_start_address);
+    unsigned log_addr_offset =
+        (char *)&tcpa->log_area_start_address - table_data->data;
 
     tcpa->platform_class = cpu_to_le16(TPM_TCPA_ACPI_CLASS_CLIENT);
     tcpa->log_area_minimum_length = cpu_to_le32(TPM_LOG_AREA_MINIMUM_SIZE);
-    tcpa->log_area_start_address = cpu_to_le64(log_area_start_address);
+    acpi_data_push(tcpalog, le32_to_cpu(tcpa->log_area_minimum_length));
 
-    bios_linker_loader_alloc(linker, ACPI_BUILD_TPMLOG_FILE, 1,
+    bios_linker_loader_alloc(linker, ACPI_BUILD_TPMLOG_FILE, tcpalog, 1,
                              false /* high memory */);
 
     /* log area start address to be filled by Guest linker */
-    bios_linker_loader_add_pointer(linker, ACPI_BUILD_TABLE_FILE,
-                                   ACPI_BUILD_TPMLOG_FILE,
-                                   table_data, &tcpa->log_area_start_address,
-                                   sizeof(tcpa->log_area_start_address));
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_TABLE_FILE, log_addr_offset, log_addr_size,
+        ACPI_BUILD_TPMLOG_FILE, 0);
 
     build_header(linker, table_data,
-                 (void *)tcpa, "TCPA", sizeof(*tcpa), 2);
-
-    acpi_data_push(tcpalog, TPM_LOG_AREA_MINIMUM_SIZE);
+                 (void *)tcpa, "TCPA", sizeof(*tcpa), 2, NULL, NULL);
 }
 
 static void
-build_tpm_ssdt(GArray *table_data, GArray *linker)
+build_tpm2(GArray *table_data, BIOSLinker *linker)
 {
-    void *tpm_ptr;
+    Acpi20TPM2 *tpm2_ptr;
 
-    tpm_ptr = acpi_data_push(table_data, sizeof(ssdt_tpm_aml));
-    memcpy(tpm_ptr, ssdt_tpm_aml, sizeof(ssdt_tpm_aml));
-}
+    tpm2_ptr = acpi_data_push(table_data, sizeof *tpm2_ptr);
 
-typedef enum {
-    MEM_AFFINITY_NOFLAGS      = 0,
-    MEM_AFFINITY_ENABLED      = (1 << 0),
-    MEM_AFFINITY_HOTPLUGGABLE = (1 << 1),
-    MEM_AFFINITY_NON_VOLATILE = (1 << 2),
-} MemoryAffinityFlags;
+    tpm2_ptr->platform_class = cpu_to_le16(TPM2_ACPI_CLASS_CLIENT);
+    tpm2_ptr->control_area_address = cpu_to_le64(0);
+    tpm2_ptr->start_method = cpu_to_le32(TPM2_START_METHOD_MMIO);
 
-static void
-acpi_build_srat_memory(AcpiSratMemoryAffinity *numamem, uint64_t base,
-                       uint64_t len, int node, MemoryAffinityFlags flags)
-{
-    numamem->type = ACPI_SRAT_MEMORY;
-    numamem->length = sizeof(*numamem);
-    memset(numamem->proximity, 0, 4);
-    numamem->proximity[0] = node;
-    numamem->flags = cpu_to_le32(flags);
-    numamem->base_addr = cpu_to_le64(base);
-    numamem->range_length = cpu_to_le64(len);
+    build_header(linker, table_data,
+                 (void *)tpm2_ptr, "TPM2", sizeof(*tpm2_ptr), 4, NULL, NULL);
 }
 
 static void
-build_srat(GArray *table_data, GArray *linker, PcGuestInfo *guest_info)
+build_srat(GArray *table_data, BIOSLinker *linker, MachineState *machine)
 {
     AcpiSystemResourceAffinityTable *srat;
     AcpiSratProcessorAffinity *core;
     AcpiSratMemoryAffinity *numamem;
 
     int i;
-    uint64_t curnode;
     int srat_start, numa_start, slots;
     uint64_t mem_len, mem_base, next_base;
-    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    CPUArchIdList *apic_ids = mc->possible_cpu_arch_ids(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
     ram_addr_t hotplugabble_address_space_size =
         object_property_get_int(OBJECT(pcms), PC_MACHINE_MEMHP_REGION_SIZE,
                                 NULL);
@@ -1289,15 +2480,21 @@
 
     srat = acpi_data_push(table_data, sizeof *srat);
     srat->reserved1 = cpu_to_le32(1);
-    core = (void *)(srat + 1);
 
-    for (i = 0; i < guest_info->apic_id_limit; ++i) {
+    for (i = 0; i < apic_ids->len; i++) {
+        int j;
+        int apic_id = apic_ids->cpus[i].arch_id;
+
         core = acpi_data_push(table_data, sizeof *core);
-        core->type = ACPI_SRAT_PROCESSOR;
+        core->type = ACPI_SRAT_PROCESSOR_APIC;
         core->length = sizeof(*core);
-        core->local_apic_id = i;
-        curnode = guest_info->node_cpu[i];
-        core->proximity_lo = curnode;
+        core->local_apic_id = apic_id;
+        for (j = 0; j < nb_numa_nodes; j++) {
+            if (test_bit(i, numa_info[j].node_cpu)) {
+                core->proximity_lo = j;
+                break;
+            }
+        }
         memset(core->proximity_hi, 0, 3);
         core->local_sapic_eid = 0;
         core->flags = cpu_to_le32(1);
@@ -1311,37 +2508,37 @@
     numa_start = table_data->len;
 
     numamem = acpi_data_push(table_data, sizeof *numamem);
-    acpi_build_srat_memory(numamem, 0, 640*1024, 0, MEM_AFFINITY_ENABLED);
+    build_srat_memory(numamem, 0, 640 * 1024, 0, MEM_AFFINITY_ENABLED);
     next_base = 1024 * 1024;
-    for (i = 1; i < guest_info->numa_nodes + 1; ++i) {
+    for (i = 1; i < pcms->numa_nodes + 1; ++i) {
         mem_base = next_base;
-        mem_len = guest_info->node_mem[i - 1];
+        mem_len = pcms->node_mem[i - 1];
         if (i == 1) {
             mem_len -= 1024 * 1024;
         }
         next_base = mem_base + mem_len;
 
         /* Cut out the ACPI_PCI hole */
-        if (mem_base <= guest_info->ram_size_below_4g &&
-            next_base > guest_info->ram_size_below_4g) {
-            mem_len -= next_base - guest_info->ram_size_below_4g;
+        if (mem_base <= pcms->below_4g_mem_size &&
+            next_base > pcms->below_4g_mem_size) {
+            mem_len -= next_base - pcms->below_4g_mem_size;
             if (mem_len > 0) {
                 numamem = acpi_data_push(table_data, sizeof *numamem);
-                acpi_build_srat_memory(numamem, mem_base, mem_len, i - 1,
-                                       MEM_AFFINITY_ENABLED);
+                build_srat_memory(numamem, mem_base, mem_len, i - 1,
+                                  MEM_AFFINITY_ENABLED);
             }
             mem_base = 1ULL << 32;
-            mem_len = next_base - guest_info->ram_size_below_4g;
-            next_base += (1ULL << 32) - guest_info->ram_size_below_4g;
+            mem_len = next_base - pcms->below_4g_mem_size;
+            next_base += (1ULL << 32) - pcms->below_4g_mem_size;
         }
         numamem = acpi_data_push(table_data, sizeof *numamem);
-        acpi_build_srat_memory(numamem, mem_base, mem_len, i - 1,
-                               MEM_AFFINITY_ENABLED);
+        build_srat_memory(numamem, mem_base, mem_len, i - 1,
+                          MEM_AFFINITY_ENABLED);
     }
     slots = (table_data->len - numa_start) / sizeof *numamem;
-    for (; slots < guest_info->numa_nodes + 2; slots++) {
+    for (; slots < pcms->numa_nodes + 2; slots++) {
         numamem = acpi_data_push(table_data, sizeof *numamem);
-        acpi_build_srat_memory(numamem, 0, 0, 0, MEM_AFFINITY_NOFLAGS);
+        build_srat_memory(numamem, 0, 0, 0, MEM_AFFINITY_NOFLAGS);
     }
 
     /*
@@ -1351,20 +2548,20 @@
      */
     if (hotplugabble_address_space_size) {
         numamem = acpi_data_push(table_data, sizeof *numamem);
-        acpi_build_srat_memory(numamem, pcms->hotplug_memory_base,
-                               hotplugabble_address_space_size, 0,
-                               MEM_AFFINITY_HOTPLUGGABLE |
-                               MEM_AFFINITY_ENABLED);
+        build_srat_memory(numamem, pcms->hotplug_memory.base,
+                          hotplugabble_address_space_size, 0,
+                          MEM_AFFINITY_HOTPLUGGABLE | MEM_AFFINITY_ENABLED);
     }
 
     build_header(linker, table_data,
                  (void *)(table_data->data + srat_start),
                  "SRAT",
-                 table_data->len - srat_start, 1);
+                 table_data->len - srat_start, 1, NULL, NULL);
+    g_free(apic_ids);
 }
 
 static void
-build_mcfg_q35(GArray *table_data, GArray *linker, AcpiMcfgInfo *info)
+build_mcfg_q35(GArray *table_data, BIOSLinker *linker, AcpiMcfgInfo *info)
 {
     AcpiTableMcfg *mcfg;
     const char *sig;
@@ -1389,139 +2586,99 @@
     } else {
         sig = "MCFG";
     }
-    build_header(linker, table_data, (void *)mcfg, sig, len, 1);
+    build_header(linker, table_data, (void *)mcfg, sig, len, 1, NULL, NULL);
 }
 
+/*
+ * VT-d spec 8.1 DMA Remapping Reporting Structure
+ * (version Oct. 2014 or later)
+ */
 static void
-build_dmar_q35(GArray *table_data, GArray *linker)
+build_dmar_q35(GArray *table_data, BIOSLinker *linker)
 {
     int dmar_start = table_data->len;
 
     AcpiTableDmar *dmar;
     AcpiDmarHardwareUnit *drhd;
+    uint8_t dmar_flags = 0;
+    X86IOMMUState *iommu = x86_iommu_get_default();
+    AcpiDmarDeviceScope *scope = NULL;
+    /* Root complex IOAPIC use one path[0] only */
+    size_t ioapic_scope_size = sizeof(*scope) + sizeof(scope->path[0]);
+
+    assert(iommu);
+    if (iommu->intr_supported) {
+        dmar_flags |= 0x1;      /* Flags: 0x1: INT_REMAP */
+    }
 
     dmar = acpi_data_push(table_data, sizeof(*dmar));
     dmar->host_address_width = VTD_HOST_ADDRESS_WIDTH - 1;
-    dmar->flags = 0;    /* No intr_remap for now */
+    dmar->flags = dmar_flags;
 
     /* DMAR Remapping Hardware Unit Definition structure */
-    drhd = acpi_data_push(table_data, sizeof(*drhd));
+    drhd = acpi_data_push(table_data, sizeof(*drhd) + ioapic_scope_size);
     drhd->type = cpu_to_le16(ACPI_DMAR_TYPE_HARDWARE_UNIT);
-    drhd->length = cpu_to_le16(sizeof(*drhd));   /* No device scope now */
+    drhd->length = cpu_to_le16(sizeof(*drhd) + ioapic_scope_size);
     drhd->flags = ACPI_DMAR_INCLUDE_PCI_ALL;
     drhd->pci_segment = cpu_to_le16(0);
     drhd->address = cpu_to_le64(Q35_HOST_BRIDGE_IOMMU_ADDR);
 
+    /* Scope definition for the root-complex IOAPIC. See VT-d spec
+     * 8.3.1 (version Oct. 2014 or later). */
+    scope = &drhd->scope[0];
+    scope->entry_type = 0x03;   /* Type: 0x03 for IOAPIC */
+    scope->length = ioapic_scope_size;
+    scope->enumeration_id = ACPI_BUILD_IOAPIC_ID;
+    scope->bus = Q35_PSEUDO_BUS_PLATFORM;
+    scope->path[0] = cpu_to_le16(Q35_PSEUDO_DEVFN_IOAPIC);
+
     build_header(linker, table_data, (void *)(table_data->data + dmar_start),
-                 "DMAR", table_data->len - dmar_start, 1);
-}
-
-static void
-build_dsdt(GArray *table_data, GArray *linker, AcpiMiscInfo *misc)
-{
-    AcpiTableHeader *dsdt;
-
-    assert(misc->dsdt_code && misc->dsdt_size);
-
-    dsdt = acpi_data_push(table_data, misc->dsdt_size);
-    memcpy(dsdt, misc->dsdt_code, misc->dsdt_size);
-
-    memset(dsdt, 0, sizeof *dsdt);
-    build_header(linker, table_data, dsdt, "DSDT",
-                 misc->dsdt_size, 1);
-}
-
-/* Build final rsdt table */
-static void
-build_rsdt(GArray *table_data, GArray *linker, GArray *table_offsets)
-{
-    AcpiRsdtDescriptorRev1 *rsdt;
-    size_t rsdt_len;
-    int i;
-
-    rsdt_len = sizeof(*rsdt) + sizeof(uint32_t) * table_offsets->len;
-    rsdt = acpi_data_push(table_data, rsdt_len);
-    memcpy(rsdt->table_offset_entry, table_offsets->data,
-           sizeof(uint32_t) * table_offsets->len);
-    for (i = 0; i < table_offsets->len; ++i) {
-        /* rsdt->table_offset_entry to be filled by Guest linker */
-        bios_linker_loader_add_pointer(linker,
-                                       ACPI_BUILD_TABLE_FILE,
-                                       ACPI_BUILD_TABLE_FILE,
-                                       table_data, &rsdt->table_offset_entry[i],
-                                       sizeof(uint32_t));
-    }
-    build_header(linker, table_data,
-                 (void *)rsdt, "RSDT", rsdt_len, 1);
+                 "DMAR", table_data->len - dmar_start, 1, NULL, NULL);
 }
 
 static GArray *
-build_rsdp(GArray *rsdp_table, GArray *linker, unsigned rsdt)
+build_rsdp(GArray *rsdp_table, BIOSLinker *linker, unsigned rsdt_tbl_offset)
 {
     AcpiRsdpDescriptor *rsdp = acpi_data_push(rsdp_table, sizeof *rsdp);
+    unsigned rsdt_pa_size = sizeof(rsdp->rsdt_physical_address);
+    unsigned rsdt_pa_offset =
+        (char *)&rsdp->rsdt_physical_address - rsdp_table->data;
 
-    bios_linker_loader_alloc(linker, ACPI_BUILD_RSDP_FILE, 16,
+    bios_linker_loader_alloc(linker, ACPI_BUILD_RSDP_FILE, rsdp_table, 16,
                              true /* fseg memory */);
 
     memcpy(&rsdp->signature, "RSD PTR ", 8);
     memcpy(rsdp->oem_id, ACPI_BUILD_APPNAME6, 6);
-    rsdp->rsdt_physical_address = cpu_to_le32(rsdt);
     /* Address to be filled by Guest linker */
-    bios_linker_loader_add_pointer(linker, ACPI_BUILD_RSDP_FILE,
-                                   ACPI_BUILD_TABLE_FILE,
-                                   rsdp_table, &rsdp->rsdt_physical_address,
-                                   sizeof rsdp->rsdt_physical_address);
-    rsdp->checksum = 0;
+    bios_linker_loader_add_pointer(linker,
+        ACPI_BUILD_RSDP_FILE, rsdt_pa_offset, rsdt_pa_size,
+        ACPI_BUILD_TABLE_FILE, rsdt_tbl_offset);
+
     /* Checksum to be filled by Guest linker */
     bios_linker_loader_add_checksum(linker, ACPI_BUILD_RSDP_FILE,
-                                    rsdp, rsdp, sizeof *rsdp, &rsdp->checksum);
+        (char *)rsdp - rsdp_table->data, sizeof *rsdp,
+        (char *)&rsdp->checksum - rsdp_table->data);
 
     return rsdp_table;
 }
 
 typedef
-struct AcpiBuildTables {
-    GArray *table_data;
-    GArray *rsdp;
-    GArray *tcpalog;
-    GArray *linker;
-} AcpiBuildTables;
-
-static inline void acpi_build_tables_init(AcpiBuildTables *tables)
-{
-    tables->rsdp = g_array_new(false, true /* clear */, 1);
-    tables->table_data = g_array_new(false, true /* clear */, 1);
-    tables->tcpalog = g_array_new(false, true /* clear */, 1);
-    tables->linker = bios_linker_loader_init();
-}
-
-static inline void acpi_build_tables_cleanup(AcpiBuildTables *tables, bool mfre)
-{
-    void *linker_data = bios_linker_loader_cleanup(tables->linker);
-    g_free(linker_data);
-    g_array_free(tables->rsdp, mfre);
-    g_array_free(tables->table_data, true);
-    g_array_free(tables->tcpalog, mfre);
-}
-
-typedef
 struct AcpiBuildState {
     /* Copy of table in RAM (for patching). */
-    ram_addr_t table_ram;
-    uint32_t table_size;
+    MemoryRegion *table_mr;
     /* Is table patched? */
     uint8_t patched;
-    PcGuestInfo *guest_info;
+    void *rsdp;
+    MemoryRegion *rsdp_mr;
+    MemoryRegion *linker_mr;
 } AcpiBuildState;
 
 static bool acpi_get_mcfg(AcpiMcfgInfo *mcfg)
 {
     Object *pci_host;
     QObject *o;
-    bool ambiguous;
 
-    pci_host = object_resolve_path_type("", TYPE_PCI_HOST_BRIDGE, &ambiguous);
-    g_assert(!ambiguous);
+    pci_host = acpi_get_i386_pci_host();
     g_assert(pci_host);
 
     o = object_property_get_qobject(pci_host, PCIE_HOST_MCFG_BASE, NULL);
@@ -1540,38 +2697,36 @@
 
 static bool acpi_has_iommu(void)
 {
-    bool ambiguous;
-    Object *intel_iommu;
-
-    intel_iommu = object_resolve_path_type("", TYPE_INTEL_IOMMU_DEVICE,
-                                           &ambiguous);
-    return intel_iommu && !ambiguous;
+    return !!x86_iommu_get_default();
 }
 
 static
-void acpi_build(PcGuestInfo *guest_info, AcpiBuildTables *tables)
+void acpi_build(AcpiBuildTables *tables, MachineState *machine)
 {
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     GArray *table_offsets;
-    unsigned facs, ssdt, dsdt, rsdt;
-    AcpiCpuInfo cpu;
+    unsigned facs, dsdt, rsdt, fadt;
     AcpiPmInfo pm;
     AcpiMiscInfo misc;
     AcpiMcfgInfo mcfg;
-    PcPciInfo pci;
+    Range pci_hole, pci_hole64;
     uint8_t *u;
     size_t aml_len = 0;
+    GArray *tables_blob = tables->table_data;
+    AcpiSlicOem slic_oem = { .id = NULL, .table_id = NULL };
 
-    acpi_get_cpu_info(&cpu);
     acpi_get_pm_info(&pm);
-    acpi_get_dsdt(&misc);
     acpi_get_misc_info(&misc);
-    acpi_get_pci_info(&pci);
+    acpi_get_pci_holes(&pci_hole, &pci_hole64);
+    acpi_get_slic_oem(&slic_oem);
 
     table_offsets = g_array_new(false, true /* clear */,
                                         sizeof(uint32_t));
-    ACPI_BUILD_DPRINTF(3, "init ACPI tables\n");
+    ACPI_BUILD_DPRINTF("init ACPI tables\n");
 
-    bios_linker_loader_alloc(tables->linker, ACPI_BUILD_TABLE_FILE,
+    bios_linker_loader_alloc(tables->linker,
+                             ACPI_BUILD_TABLE_FILE, tables_blob,
                              64 /* Ensure FACS is aligned */,
                              false /* high memory */);
 
@@ -1580,74 +2735,77 @@
      * We place it first since it's the only table that has alignment
      * requirements.
      */
-    facs = tables->table_data->len;
-    build_facs(tables->table_data, tables->linker, guest_info);
+    facs = tables_blob->len;
+    build_facs(tables_blob, tables->linker);
 
     /* DSDT is pointed to by FADT */
-    dsdt = tables->table_data->len;
-    build_dsdt(tables->table_data, tables->linker, &misc);
+    dsdt = tables_blob->len;
+    build_dsdt(tables_blob, tables->linker, &pm, &misc,
+               &pci_hole, &pci_hole64, machine);
 
     /* Count the size of the DSDT and SSDT, we will need it for legacy
      * sizing of ACPI tables.
      */
-    aml_len += tables->table_data->len - dsdt;
+    aml_len += tables_blob->len - dsdt;
 
     /* ACPI tables pointed to by RSDT */
-    acpi_add_table(table_offsets, tables->table_data);
-    build_fadt(tables->table_data, tables->linker, &pm, facs, dsdt);
+    fadt = tables_blob->len;
+    acpi_add_table(table_offsets, tables_blob);
+    build_fadt(tables_blob, tables->linker, &pm, facs, dsdt,
+               slic_oem.id, slic_oem.table_id);
+    aml_len += tables_blob->len - fadt;
 
-    ssdt = tables->table_data->len;
-    acpi_add_table(table_offsets, tables->table_data);
-    build_ssdt(tables->table_data, tables->linker, &cpu, &pm, &misc, &pci,
-               guest_info);
-    aml_len += tables->table_data->len - ssdt;
-
-    acpi_add_table(table_offsets, tables->table_data);
-    build_madt(tables->table_data, tables->linker, &cpu, guest_info);
+    acpi_add_table(table_offsets, tables_blob);
+    build_madt(tables_blob, tables->linker, pcms);
 
     if (misc.has_hpet) {
-        acpi_add_table(table_offsets, tables->table_data);
-        build_hpet(tables->table_data, tables->linker);
+        acpi_add_table(table_offsets, tables_blob);
+        build_hpet(tables_blob, tables->linker);
     }
-    if (misc.has_tpm) {
-        acpi_add_table(table_offsets, tables->table_data);
-        build_tpm_tcpa(tables->table_data, tables->linker, tables->tcpalog);
+    if (misc.tpm_version != TPM_VERSION_UNSPEC) {
+        acpi_add_table(table_offsets, tables_blob);
+        build_tpm_tcpa(tables_blob, tables->linker, tables->tcpalog);
 
-        acpi_add_table(table_offsets, tables->table_data);
-        build_tpm_ssdt(tables->table_data, tables->linker);
+        if (misc.tpm_version == TPM_VERSION_2_0) {
+            acpi_add_table(table_offsets, tables_blob);
+            build_tpm2(tables_blob, tables->linker);
+        }
     }
-    if (guest_info->numa_nodes) {
-        acpi_add_table(table_offsets, tables->table_data);
-        build_srat(tables->table_data, tables->linker, guest_info);
+    if (pcms->numa_nodes) {
+        acpi_add_table(table_offsets, tables_blob);
+        build_srat(tables_blob, tables->linker, machine);
     }
     if (acpi_get_mcfg(&mcfg)) {
-        acpi_add_table(table_offsets, tables->table_data);
-        build_mcfg_q35(tables->table_data, tables->linker, &mcfg);
+        acpi_add_table(table_offsets, tables_blob);
+        build_mcfg_q35(tables_blob, tables->linker, &mcfg);
     }
     if (acpi_has_iommu()) {
-        acpi_add_table(table_offsets, tables->table_data);
-        build_dmar_q35(tables->table_data, tables->linker);
+        acpi_add_table(table_offsets, tables_blob);
+        build_dmar_q35(tables_blob, tables->linker);
+    }
+    if (pcms->acpi_nvdimm_state.is_enabled) {
+        nvdimm_build_acpi(table_offsets, tables_blob, tables->linker,
+                          pcms->acpi_nvdimm_state.dsm_mem);
     }
 
     /* Add tables supplied by user (if any) */
     for (u = acpi_table_first(); u; u = acpi_table_next(u)) {
         unsigned len = acpi_table_len(u);
 
-        acpi_add_table(table_offsets, tables->table_data);
-        g_array_append_vals(tables->table_data, u, len);
+        acpi_add_table(table_offsets, tables_blob);
+        g_array_append_vals(tables_blob, u, len);
     }
 
     /* RSDT is pointed to by RSDP */
-    rsdt = tables->table_data->len;
-    build_rsdt(tables->table_data, tables->linker, table_offsets);
+    rsdt = tables_blob->len;
+    build_rsdt(tables_blob, tables->linker, table_offsets,
+               slic_oem.id, slic_oem.table_id);
 
     /* RSDP is in FSEG memory, so allocate it separately */
     build_rsdp(tables->rsdp, tables->linker, rsdt);
 
     /* We'll expose it all to Guest so we want to reduce
      * chance of size changes.
-     * RSDP is small so it's easy to keep it immutable, no need to
-     * bother with alignment.
      *
      * We used to align the tables to 4k, but of course this would
      * too simple to be enough.  4k turned out to be too small an
@@ -1663,40 +2821,51 @@
      *
      * All this is for PIIX4, since QEMU 2.0 didn't support Q35 migration.
      */
-    if (guest_info->legacy_acpi_table_size) {
+    if (pcmc->legacy_acpi_table_size) {
         /* Subtracting aml_len gives the size of fixed tables.  Then add the
          * size of the PIIX4 DSDT/SSDT in QEMU 2.0.
          */
         int legacy_aml_len =
-            guest_info->legacy_acpi_table_size +
+            pcmc->legacy_acpi_table_size +
             ACPI_BUILD_LEGACY_CPU_AML_SIZE * max_cpus;
         int legacy_table_size =
-            ROUND_UP(tables->table_data->len - aml_len + legacy_aml_len,
+            ROUND_UP(tables_blob->len - aml_len + legacy_aml_len,
                      ACPI_BUILD_ALIGN_SIZE);
-        if (tables->table_data->len > legacy_table_size) {
+        if (tables_blob->len > legacy_table_size) {
             /* Should happen only with PCI bridges and -M pc-i440fx-2.0.  */
             error_report("Warning: migration may not work.");
         }
-        g_array_set_size(tables->table_data, legacy_table_size);
+        g_array_set_size(tables_blob, legacy_table_size);
     } else {
         /* Make sure we have a buffer in case we need to resize the tables. */
-        if (tables->table_data->len > ACPI_BUILD_TABLE_SIZE / 2) {
+        if (tables_blob->len > ACPI_BUILD_TABLE_SIZE / 2) {
             /* As of QEMU 2.1, this fires with 160 VCPUs and 255 memory slots.  */
             error_report("Warning: ACPI tables are larger than 64k.");
             error_report("Warning: migration may not work.");
             error_report("Warning: please remove CPUs, NUMA nodes, "
                          "memory slots or PCI bridges.");
         }
-        acpi_align_size(tables->table_data, ACPI_BUILD_TABLE_SIZE);
+        acpi_align_size(tables_blob, ACPI_BUILD_TABLE_SIZE);
     }
 
-    acpi_align_size(tables->linker, ACPI_BUILD_ALIGN_SIZE);
+    acpi_align_size(tables->linker->cmd_blob, ACPI_BUILD_ALIGN_SIZE);
 
     /* Cleanup memory that's no longer used. */
     g_array_free(table_offsets, true);
 }
 
-static void acpi_build_update(void *build_opaque, uint32_t offset)
+static void acpi_ram_update(MemoryRegion *mr, GArray *data)
+{
+    uint32_t size = acpi_data_len(data);
+
+    /* Make sure RAM size is correct - in case it got changed e.g. by migration */
+    memory_region_ram_resize(mr, size, &error_abort);
+
+    memcpy(memory_region_get_ram_ptr(mr), data->data, size);
+    memory_region_set_dirty(mr, 0, size);
+}
+
+static void acpi_build_update(void *build_opaque)
 {
     AcpiBuildState *build_state = build_opaque;
     AcpiBuildTables tables;
@@ -1709,15 +2878,17 @@
 
     acpi_build_tables_init(&tables);
 
-    acpi_build(build_state->guest_info, &tables);
+    acpi_build(&tables, MACHINE(qdev_get_machine()));
 
-    assert(acpi_data_len(tables.table_data) == build_state->table_size);
-    memcpy(qemu_get_ram_ptr(build_state->table_ram), tables.table_data->data,
-           build_state->table_size);
+    acpi_ram_update(build_state->table_mr, tables.table_data);
 
-    cpu_physical_memory_set_dirty_range_nocode(build_state->table_ram,
-                                               build_state->table_size);
+    if (build_state->rsdp) {
+        memcpy(build_state->rsdp, tables.rsdp->data, acpi_data_len(tables.rsdp));
+    } else {
+        acpi_ram_update(build_state->rsdp_mr, tables.rsdp);
+    }
 
+    acpi_ram_update(build_state->linker_mr, tables.linker->cmd_blob);
     acpi_build_tables_cleanup(&tables, true);
 }
 
@@ -1727,11 +2898,12 @@
     build_state->patched = 0;
 }
 
-static ram_addr_t acpi_add_rom_blob(AcpiBuildState *build_state, GArray *blob,
-                               const char *name)
+static MemoryRegion *acpi_add_rom_blob(AcpiBuildState *build_state,
+                                       GArray *blob, const char *name,
+                                       uint64_t max_size)
 {
-    return rom_add_blob(name, blob->data, acpi_data_len(blob), -1, name,
-                        acpi_build_update, build_state);
+    return rom_add_blob(name, blob->data, acpi_data_len(blob), max_size, -1,
+                        name, acpi_build_update, build_state);
 }
 
 static const VMStateDescription vmstate_acpi_build = {
@@ -1744,52 +2916,66 @@
     },
 };
 
-void acpi_setup(PcGuestInfo *guest_info)
+void acpi_setup(void)
 {
+    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     AcpiBuildTables tables;
     AcpiBuildState *build_state;
 
-    if (!guest_info->fw_cfg) {
-        ACPI_BUILD_DPRINTF(3, "No fw cfg. Bailing out.\n");
+    if (!pcms->fw_cfg) {
+        ACPI_BUILD_DPRINTF("No fw cfg. Bailing out.\n");
         return;
     }
 
-    if (!guest_info->has_acpi_build) {
-        ACPI_BUILD_DPRINTF(3, "ACPI build disabled. Bailing out.\n");
+    if (!pcmc->has_acpi_build) {
+        ACPI_BUILD_DPRINTF("ACPI build disabled. Bailing out.\n");
         return;
     }
 
     if (!acpi_enabled) {
-        ACPI_BUILD_DPRINTF(3, "ACPI disabled. Bailing out.\n");
+        ACPI_BUILD_DPRINTF("ACPI disabled. Bailing out.\n");
         return;
     }
 
     build_state = g_malloc0(sizeof *build_state);
 
-    build_state->guest_info = guest_info;
-
     acpi_set_pci_info();
 
     acpi_build_tables_init(&tables);
-    acpi_build(build_state->guest_info, &tables);
+    acpi_build(&tables, MACHINE(pcms));
 
     /* Now expose it all to Guest */
-    build_state->table_ram = acpi_add_rom_blob(build_state, tables.table_data,
-                                               ACPI_BUILD_TABLE_FILE);
-    assert(build_state->table_ram != RAM_ADDR_MAX);
-    build_state->table_size = acpi_data_len(tables.table_data);
+    build_state->table_mr = acpi_add_rom_blob(build_state, tables.table_data,
+                                               ACPI_BUILD_TABLE_FILE,
+                                               ACPI_BUILD_TABLE_MAX_SIZE);
+    assert(build_state->table_mr != NULL);
 
-    acpi_add_rom_blob(NULL, tables.linker, "etc/table-loader");
+    build_state->linker_mr =
+        acpi_add_rom_blob(build_state, tables.linker->cmd_blob,
+                          "etc/table-loader", 0);
 
-    fw_cfg_add_file(guest_info->fw_cfg, ACPI_BUILD_TPMLOG_FILE,
+    fw_cfg_add_file(pcms->fw_cfg, ACPI_BUILD_TPMLOG_FILE,
                     tables.tcpalog->data, acpi_data_len(tables.tcpalog));
 
-    /*
-     * RSDP is small so it's easy to keep it immutable, no need to
-     * bother with ROM blobs.
-     */
-    fw_cfg_add_file(guest_info->fw_cfg, ACPI_BUILD_RSDP_FILE,
-                    tables.rsdp->data, acpi_data_len(tables.rsdp));
+    if (!pcmc->rsdp_in_ram) {
+        /*
+         * Keep for compatibility with old machine types.
+         * Though RSDP is small, its contents isn't immutable, so
+         * we'll update it along with the rest of tables on guest access.
+         */
+        uint32_t rsdp_size = acpi_data_len(tables.rsdp);
+
+        build_state->rsdp = g_memdup(tables.rsdp->data, rsdp_size);
+        fw_cfg_add_file_callback(pcms->fw_cfg, ACPI_BUILD_RSDP_FILE,
+                                 acpi_build_update, build_state,
+                                 build_state->rsdp, rsdp_size);
+        build_state->rsdp_mr = NULL;
+    } else {
+        build_state->rsdp = NULL;
+        build_state->rsdp_mr = acpi_add_rom_blob(build_state, tables.rsdp,
+                                                  ACPI_BUILD_RSDP_FILE, 0);
+    }
 
     qemu_register_reset(acpi_build_reset, build_state);
     acpi_build_reset(build_state);
diff --git a/hw/i386/acpi-build.h b/hw/i386/acpi-build.h
index e57b1aa..007332e 100644
--- a/hw/i386/acpi-build.h
+++ b/hw/i386/acpi-build.h
@@ -2,8 +2,6 @@
 #ifndef HW_I386_ACPI_BUILD_H
 #define HW_I386_ACPI_BUILD_H
 
-#include "qemu/typedefs.h"
-
-void acpi_setup(PcGuestInfo *);
+void acpi_setup(void);
 
 #endif
diff --git a/hw/i386/acpi-defs.h b/hw/i386/acpi-defs.h
deleted file mode 100644
index c4468f8..0000000
--- a/hw/i386/acpi-defs.h
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef QEMU_ACPI_DEFS_H
-#define QEMU_ACPI_DEFS_H
-
-enum {
-    ACPI_FADT_F_WBINVD,
-    ACPI_FADT_F_WBINVD_FLUSH,
-    ACPI_FADT_F_PROC_C1,
-    ACPI_FADT_F_P_LVL2_UP,
-    ACPI_FADT_F_PWR_BUTTON,
-    ACPI_FADT_F_SLP_BUTTON,
-    ACPI_FADT_F_FIX_RTC,
-    ACPI_FADT_F_RTC_S4,
-    ACPI_FADT_F_TMR_VAL_EXT,
-    ACPI_FADT_F_DCK_CAP,
-    ACPI_FADT_F_RESET_REG_SUP,
-    ACPI_FADT_F_SEALED_CASE,
-    ACPI_FADT_F_HEADLESS,
-    ACPI_FADT_F_CPU_SW_SLP,
-    ACPI_FADT_F_PCI_EXP_WAK,
-    ACPI_FADT_F_USE_PLATFORM_CLOCK,
-    ACPI_FADT_F_S4_RTC_STS_VALID,
-    ACPI_FADT_F_REMOTE_POWER_ON_CAPABLE,
-    ACPI_FADT_F_FORCE_APIC_CLUSTER_MODEL,
-    ACPI_FADT_F_FORCE_APIC_PHYSICAL_DESTINATION_MODE,
-    ACPI_FADT_F_HW_REDUCED_ACPI,
-    ACPI_FADT_F_LOW_POWER_S0_IDLE_CAPABLE,
-};
-
-/*
- * ACPI 2.0 Generic Address Space definition.
- */
-struct Acpi20GenericAddress {
-    uint8_t  address_space_id;
-    uint8_t  register_bit_width;
-    uint8_t  register_bit_offset;
-    uint8_t  reserved;
-    uint64_t address;
-} QEMU_PACKED;
-typedef struct Acpi20GenericAddress Acpi20GenericAddress;
-
-struct AcpiRsdpDescriptor {        /* Root System Descriptor Pointer */
-    uint64_t signature;              /* ACPI signature, contains "RSD PTR " */
-    uint8_t  checksum;               /* To make sum of struct == 0 */
-    uint8_t  oem_id [6];             /* OEM identification */
-    uint8_t  revision;               /* Must be 0 for 1.0, 2 for 2.0 */
-    uint32_t rsdt_physical_address;  /* 32-bit physical address of RSDT */
-    uint32_t length;                 /* XSDT Length in bytes including hdr */
-    uint64_t xsdt_physical_address;  /* 64-bit physical address of XSDT */
-    uint8_t  extended_checksum;      /* Checksum of entire table */
-    uint8_t  reserved [3];           /* Reserved field must be 0 */
-} QEMU_PACKED;
-typedef struct AcpiRsdpDescriptor AcpiRsdpDescriptor;
-
-/* Table structure from Linux kernel (the ACPI tables are under the
-   BSD license) */
-
-
-#define ACPI_TABLE_HEADER_DEF   /* ACPI common table header */ \
-    uint32_t signature;          /* ACPI signature (4 ASCII characters) */ \
-    uint32_t length;                 /* Length of table, in bytes, including header */ \
-    uint8_t  revision;               /* ACPI Specification minor version # */ \
-    uint8_t  checksum;               /* To make sum of entire table == 0 */ \
-    uint8_t  oem_id [6];             /* OEM identification */ \
-    uint8_t  oem_table_id [8];       /* OEM table identification */ \
-    uint32_t oem_revision;           /* OEM revision number */ \
-    uint8_t  asl_compiler_id [4];    /* ASL compiler vendor ID */ \
-    uint32_t asl_compiler_revision;  /* ASL compiler revision number */
-
-
-struct AcpiTableHeader         /* ACPI common table header */
-{
-    ACPI_TABLE_HEADER_DEF
-} QEMU_PACKED;
-typedef struct AcpiTableHeader AcpiTableHeader;
-
-/*
- * ACPI 1.0 Fixed ACPI Description Table (FADT)
- */
-struct AcpiFadtDescriptorRev1
-{
-    ACPI_TABLE_HEADER_DEF     /* ACPI common table header */
-    uint32_t firmware_ctrl;          /* Physical address of FACS */
-    uint32_t dsdt;                   /* Physical address of DSDT */
-    uint8_t  model;                  /* System Interrupt Model */
-    uint8_t  reserved1;              /* Reserved */
-    uint16_t sci_int;                /* System vector of SCI interrupt */
-    uint32_t smi_cmd;                /* Port address of SMI command port */
-    uint8_t  acpi_enable;            /* Value to write to smi_cmd to enable ACPI */
-    uint8_t  acpi_disable;           /* Value to write to smi_cmd to disable ACPI */
-    uint8_t  S4bios_req;             /* Value to write to SMI CMD to enter S4BIOS state */
-    uint8_t  reserved2;              /* Reserved - must be zero */
-    uint32_t pm1a_evt_blk;           /* Port address of Power Mgt 1a acpi_event Reg Blk */
-    uint32_t pm1b_evt_blk;           /* Port address of Power Mgt 1b acpi_event Reg Blk */
-    uint32_t pm1a_cnt_blk;           /* Port address of Power Mgt 1a Control Reg Blk */
-    uint32_t pm1b_cnt_blk;           /* Port address of Power Mgt 1b Control Reg Blk */
-    uint32_t pm2_cnt_blk;            /* Port address of Power Mgt 2 Control Reg Blk */
-    uint32_t pm_tmr_blk;             /* Port address of Power Mgt Timer Ctrl Reg Blk */
-    uint32_t gpe0_blk;               /* Port addr of General Purpose acpi_event 0 Reg Blk */
-    uint32_t gpe1_blk;               /* Port addr of General Purpose acpi_event 1 Reg Blk */
-    uint8_t  pm1_evt_len;            /* Byte length of ports at pm1_x_evt_blk */
-    uint8_t  pm1_cnt_len;            /* Byte length of ports at pm1_x_cnt_blk */
-    uint8_t  pm2_cnt_len;            /* Byte Length of ports at pm2_cnt_blk */
-    uint8_t  pm_tmr_len;             /* Byte Length of ports at pm_tm_blk */
-    uint8_t  gpe0_blk_len;           /* Byte Length of ports at gpe0_blk */
-    uint8_t  gpe1_blk_len;           /* Byte Length of ports at gpe1_blk */
-    uint8_t  gpe1_base;              /* Offset in gpe model where gpe1 events start */
-    uint8_t  reserved3;              /* Reserved */
-    uint16_t plvl2_lat;              /* Worst case HW latency to enter/exit C2 state */
-    uint16_t plvl3_lat;              /* Worst case HW latency to enter/exit C3 state */
-    uint16_t flush_size;             /* Size of area read to flush caches */
-    uint16_t flush_stride;           /* Stride used in flushing caches */
-    uint8_t  duty_offset;            /* Bit location of duty cycle field in p_cnt reg */
-    uint8_t  duty_width;             /* Bit width of duty cycle field in p_cnt reg */
-    uint8_t  day_alrm;               /* Index to day-of-month alarm in RTC CMOS RAM */
-    uint8_t  mon_alrm;               /* Index to month-of-year alarm in RTC CMOS RAM */
-    uint8_t  century;                /* Index to century in RTC CMOS RAM */
-    uint8_t  reserved4;              /* Reserved */
-    uint8_t  reserved4a;             /* Reserved */
-    uint8_t  reserved4b;             /* Reserved */
-    uint32_t flags;
-} QEMU_PACKED;
-typedef struct AcpiFadtDescriptorRev1 AcpiFadtDescriptorRev1;
-
-/*
- * ACPI 1.0 Root System Description Table (RSDT)
- */
-struct AcpiRsdtDescriptorRev1
-{
-    ACPI_TABLE_HEADER_DEF       /* ACPI common table header */
-    uint32_t table_offset_entry[0];  /* Array of pointers to other */
-    /* ACPI tables */
-} QEMU_PACKED;
-typedef struct AcpiRsdtDescriptorRev1 AcpiRsdtDescriptorRev1;
-
-/*
- * ACPI 1.0 Firmware ACPI Control Structure (FACS)
- */
-struct AcpiFacsDescriptorRev1
-{
-    uint32_t signature;           /* ACPI Signature */
-    uint32_t length;                 /* Length of structure, in bytes */
-    uint32_t hardware_signature;     /* Hardware configuration signature */
-    uint32_t firmware_waking_vector; /* ACPI OS waking vector */
-    uint32_t global_lock;            /* Global Lock */
-    uint32_t flags;
-    uint8_t  resverved3 [40];        /* Reserved - must be zero */
-} QEMU_PACKED;
-typedef struct AcpiFacsDescriptorRev1 AcpiFacsDescriptorRev1;
-
-/*
- * Differentiated System Description Table (DSDT)
- */
-
-/*
- * MADT values and structures
- */
-
-/* Values for MADT PCATCompat */
-
-#define ACPI_DUAL_PIC                0
-#define ACPI_MULTIPLE_APIC           1
-
-/* Master MADT */
-
-struct AcpiMultipleApicTable
-{
-    ACPI_TABLE_HEADER_DEF     /* ACPI common table header */
-    uint32_t local_apic_address;     /* Physical address of local APIC */
-    uint32_t flags;
-} QEMU_PACKED;
-typedef struct AcpiMultipleApicTable AcpiMultipleApicTable;
-
-/* Values for Type in APIC sub-headers */
-
-#define ACPI_APIC_PROCESSOR          0
-#define ACPI_APIC_IO                 1
-#define ACPI_APIC_XRUPT_OVERRIDE     2
-#define ACPI_APIC_NMI                3
-#define ACPI_APIC_LOCAL_NMI          4
-#define ACPI_APIC_ADDRESS_OVERRIDE   5
-#define ACPI_APIC_IO_SAPIC           6
-#define ACPI_APIC_LOCAL_SAPIC        7
-#define ACPI_APIC_XRUPT_SOURCE       8
-#define ACPI_APIC_RESERVED           9           /* 9 and greater are reserved */
-
-/*
- * MADT sub-structures (Follow MULTIPLE_APIC_DESCRIPTION_TABLE)
- */
-#define ACPI_SUB_HEADER_DEF   /* Common ACPI sub-structure header */\
-    uint8_t  type;                               \
-    uint8_t  length;
-
-/* Sub-structures for MADT */
-
-struct AcpiMadtProcessorApic
-{
-    ACPI_SUB_HEADER_DEF
-    uint8_t  processor_id;           /* ACPI processor id */
-    uint8_t  local_apic_id;          /* Processor's local APIC id */
-    uint32_t flags;
-} QEMU_PACKED;
-typedef struct AcpiMadtProcessorApic AcpiMadtProcessorApic;
-
-struct AcpiMadtIoApic
-{
-    ACPI_SUB_HEADER_DEF
-    uint8_t  io_apic_id;             /* I/O APIC ID */
-    uint8_t  reserved;               /* Reserved - must be zero */
-    uint32_t address;                /* APIC physical address */
-    uint32_t interrupt;              /* Global system interrupt where INTI
-                                 * lines start */
-} QEMU_PACKED;
-typedef struct AcpiMadtIoApic AcpiMadtIoApic;
-
-struct AcpiMadtIntsrcovr {
-    ACPI_SUB_HEADER_DEF
-    uint8_t  bus;
-    uint8_t  source;
-    uint32_t gsi;
-    uint16_t flags;
-} QEMU_PACKED;
-typedef struct AcpiMadtIntsrcovr AcpiMadtIntsrcovr;
-
-struct AcpiMadtLocalNmi {
-    ACPI_SUB_HEADER_DEF
-    uint8_t  processor_id;           /* ACPI processor id */
-    uint16_t flags;                  /* MPS INTI flags */
-    uint8_t  lint;                   /* Local APIC LINT# */
-} QEMU_PACKED;
-typedef struct AcpiMadtLocalNmi AcpiMadtLocalNmi;
-
-/*
- * HPET Description Table
- */
-struct Acpi20Hpet {
-    ACPI_TABLE_HEADER_DEF                    /* ACPI common table header */
-    uint32_t           timer_block_id;
-    Acpi20GenericAddress addr;
-    uint8_t            hpet_number;
-    uint16_t           min_tick;
-    uint8_t            page_protect;
-} QEMU_PACKED;
-typedef struct Acpi20Hpet Acpi20Hpet;
-
-/*
- * SRAT (NUMA topology description) table
- */
-
-struct AcpiSystemResourceAffinityTable
-{
-    ACPI_TABLE_HEADER_DEF
-    uint32_t    reserved1;
-    uint32_t    reserved2[2];
-} QEMU_PACKED;
-typedef struct AcpiSystemResourceAffinityTable AcpiSystemResourceAffinityTable;
-
-#define ACPI_SRAT_PROCESSOR          0
-#define ACPI_SRAT_MEMORY             1
-
-struct AcpiSratProcessorAffinity
-{
-    ACPI_SUB_HEADER_DEF
-    uint8_t     proximity_lo;
-    uint8_t     local_apic_id;
-    uint32_t    flags;
-    uint8_t     local_sapic_eid;
-    uint8_t     proximity_hi[3];
-    uint32_t    reserved;
-} QEMU_PACKED;
-typedef struct AcpiSratProcessorAffinity AcpiSratProcessorAffinity;
-
-struct AcpiSratMemoryAffinity
-{
-    ACPI_SUB_HEADER_DEF
-    uint8_t     proximity[4];
-    uint16_t    reserved1;
-    uint64_t    base_addr;
-    uint64_t    range_length;
-    uint32_t    reserved2;
-    uint32_t    flags;
-    uint32_t    reserved3[2];
-} QEMU_PACKED;
-typedef struct AcpiSratMemoryAffinity AcpiSratMemoryAffinity;
-
-/* PCI fw r3.0 MCFG table. */
-/* Subtable */
-struct AcpiMcfgAllocation {
-    uint64_t address;                /* Base address, processor-relative */
-    uint16_t pci_segment;            /* PCI segment group number */
-    uint8_t start_bus_number;       /* Starting PCI Bus number */
-    uint8_t end_bus_number;         /* Final PCI Bus number */
-    uint32_t reserved;
-} QEMU_PACKED;
-typedef struct AcpiMcfgAllocation AcpiMcfgAllocation;
-
-struct AcpiTableMcfg {
-    ACPI_TABLE_HEADER_DEF;
-    uint8_t reserved[8];
-    AcpiMcfgAllocation allocation[0];
-} QEMU_PACKED;
-typedef struct AcpiTableMcfg AcpiTableMcfg;
-
-/*
- * TCPA Description Table
- */
-struct Acpi20Tcpa {
-    ACPI_TABLE_HEADER_DEF                    /* ACPI common table header */
-    uint16_t platform_class;
-    uint32_t log_area_minimum_length;
-    uint64_t log_area_start_address;
-} QEMU_PACKED;
-typedef struct Acpi20Tcpa Acpi20Tcpa;
-
-/* DMAR - DMA Remapping table r2.2 */
-struct AcpiTableDmar {
-    ACPI_TABLE_HEADER_DEF
-    uint8_t host_address_width; /* Maximum DMA physical addressability */
-    uint8_t flags;
-    uint8_t reserved[10];
-} QEMU_PACKED;
-typedef struct AcpiTableDmar AcpiTableDmar;
-
-/* Masks for Flags field above */
-#define ACPI_DMAR_INTR_REMAP        1
-#define ACPI_DMAR_X2APIC_OPT_OUT    (1 << 1)
-
-/* Values for sub-structure type for DMAR */
-enum {
-    ACPI_DMAR_TYPE_HARDWARE_UNIT = 0,       /* DRHD */
-    ACPI_DMAR_TYPE_RESERVED_MEMORY = 1,     /* RMRR */
-    ACPI_DMAR_TYPE_ATSR = 2,                /* ATSR */
-    ACPI_DMAR_TYPE_HARDWARE_AFFINITY = 3,   /* RHSR */
-    ACPI_DMAR_TYPE_ANDD = 4,                /* ANDD */
-    ACPI_DMAR_TYPE_RESERVED = 5             /* Reserved for furture use */
-};
-
-/*
- * Sub-structures for DMAR
- */
-/* Type 0: Hardware Unit Definition */
-struct AcpiDmarHardwareUnit {
-    uint16_t type;
-    uint16_t length;
-    uint8_t flags;
-    uint8_t reserved;
-    uint16_t pci_segment;   /* The PCI Segment associated with this unit */
-    uint64_t address;   /* Base address of remapping hardware register-set */
-} QEMU_PACKED;
-typedef struct AcpiDmarHardwareUnit AcpiDmarHardwareUnit;
-
-/* Masks for Flags field above */
-#define ACPI_DMAR_INCLUDE_PCI_ALL   1
-
-#endif
diff --git a/hw/i386/acpi-dsdt-cpu-hotplug.dsl b/hw/i386/acpi-dsdt-cpu-hotplug.dsl
deleted file mode 100644
index 34aab5a..0000000
--- a/hw/i386/acpi-dsdt-cpu-hotplug.dsl
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/****************************************************************
- * CPU hotplug
- ****************************************************************/
-#define CPU_HOTPLUG_RESOURCE_DEVICE PRES
-
-Scope(\_SB) {
-    /* Objects filled in by run-time generated SSDT */
-    External(NTFY, MethodObj)
-    External(CPON, PkgObj)
-
-    /* Methods called by run-time generated SSDT Processor objects */
-    Method(CPMA, 1, NotSerialized) {
-        // _MAT method - create an madt apic buffer
-        // Arg0 = Processor ID = Local APIC ID
-        // Local0 = CPON flag for this cpu
-        Store(DerefOf(Index(CPON, Arg0)), Local0)
-        // Local1 = Buffer (in madt apic form) to return
-        Store(Buffer(8) {0x00, 0x08, 0x00, 0x00, 0x00, 0, 0, 0}, Local1)
-        // Update the processor id, lapic id, and enable/disable status
-        Store(Arg0, Index(Local1, 2))
-        Store(Arg0, Index(Local1, 3))
-        Store(Local0, Index(Local1, 4))
-        Return (Local1)
-    }
-    Method(CPST, 1, NotSerialized) {
-        // _STA method - return ON status of cpu
-        // Arg0 = Processor ID = Local APIC ID
-        // Local0 = CPON flag for this cpu
-        Store(DerefOf(Index(CPON, Arg0)), Local0)
-        If (Local0) {
-            Return (0xF)
-        } Else {
-            Return (0x0)
-        }
-    }
-    Method(CPEJ, 2, NotSerialized) {
-        // _EJ0 method - eject callback
-        Sleep(200)
-    }
-
-#define CPU_STATUS_LEN ACPI_GPE_PROC_LEN
-    OperationRegion(PRST, SystemIO, CPU_STATUS_BASE, CPU_STATUS_LEN)
-    Field(PRST, ByteAcc, NoLock, Preserve) {
-        PRS, 256
-    }
-    Method(PRSC, 0) {
-        // Local5 = active cpu bitmap
-        Store(PRS, Local5)
-        // Local2 = last read byte from bitmap
-        Store(Zero, Local2)
-        // Local0 = Processor ID / APIC ID iterator
-        Store(Zero, Local0)
-        While (LLess(Local0, SizeOf(CPON))) {
-            // Local1 = CPON flag for this cpu
-            Store(DerefOf(Index(CPON, Local0)), Local1)
-            If (And(Local0, 0x07)) {
-                // Shift down previously read bitmap byte
-                ShiftRight(Local2, 1, Local2)
-            } Else {
-                // Read next byte from cpu bitmap
-                Store(DerefOf(Index(Local5, ShiftRight(Local0, 3))), Local2)
-            }
-            // Local3 = active state for this cpu
-            Store(And(Local2, 1), Local3)
-
-            If (LNotEqual(Local1, Local3)) {
-                // State change - update CPON with new state
-                Store(Local3, Index(CPON, Local0))
-                // Do CPU notify
-                If (LEqual(Local3, 1)) {
-                    NTFY(Local0, 1)
-                } Else {
-                    NTFY(Local0, 3)
-                }
-            }
-            Increment(Local0)
-        }
-    }
-
-    Device(CPU_HOTPLUG_RESOURCE_DEVICE) {
-        Name(_HID, EisaId("PNP0A06"))
-
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, CPU_STATUS_BASE, CPU_STATUS_BASE, 0, CPU_STATUS_LEN)
-        })
-
-        Name(_STA, 0xB) /* present, functioning, decoding, not shown in UI */
-    }
-}
diff --git a/hw/i386/acpi-dsdt-dbug.dsl b/hw/i386/acpi-dsdt-dbug.dsl
deleted file mode 100644
index 86230f7..0000000
--- a/hw/i386/acpi-dsdt-dbug.dsl
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/****************************************************************
- * Debugging
- ****************************************************************/
-
-Scope(\) {
-    /* Debug Output */
-    OperationRegion(DBG, SystemIO, 0x0402, 0x01)
-    Field(DBG, ByteAcc, NoLock, Preserve) {
-        DBGB,   8,
-    }
-
-    /* Debug method - use this method to send output to the QEMU
-     * BIOS debug port.  This method handles strings, integers,
-     * and buffers.  For example: DBUG("abc") DBUG(0x123) */
-    Method(DBUG, 1) {
-        ToHexString(Arg0, Local0)
-        ToBuffer(Local0, Local0)
-        Subtract(SizeOf(Local0), 1, Local1)
-        Store(Zero, Local2)
-        While (LLess(Local2, Local1)) {
-            Store(DerefOf(Index(Local0, Local2)), DBGB)
-            Increment(Local2)
-        }
-        Store(0x0A, DBGB)
-    }
-}
diff --git a/hw/i386/acpi-dsdt-hpet.dsl b/hw/i386/acpi-dsdt-hpet.dsl
deleted file mode 100644
index 44961b8..0000000
--- a/hw/i386/acpi-dsdt-hpet.dsl
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/****************************************************************
- * HPET
- ****************************************************************/
-
-Scope(\_SB) {
-    Device(HPET) {
-        Name(_HID, EISAID("PNP0103"))
-        Name(_UID, 0)
-        OperationRegion(HPTM, SystemMemory, 0xFED00000, 0x400)
-        Field(HPTM, DWordAcc, Lock, Preserve) {
-            VEND, 32,
-            PRD, 32,
-        }
-        Method(_STA, 0, NotSerialized) {
-            Store(VEND, Local0)
-            Store(PRD, Local1)
-            ShiftRight(Local0, 16, Local0)
-            If (LOr(LEqual(Local0, 0), LEqual(Local0, 0xffff))) {
-                Return (0x0)
-            }
-            If (LOr(LEqual(Local1, 0), LGreater(Local1, 100000000))) {
-                Return (0x0)
-            }
-            Return (0x0F)
-        }
-        Name(_CRS, ResourceTemplate() {
-            Memory32Fixed(ReadOnly,
-                0xFED00000,         // Address Base
-                0x00000400,         // Address Length
-                )
-        })
-    }
-}
diff --git a/hw/i386/acpi-dsdt-isa.dsl b/hw/i386/acpi-dsdt-isa.dsl
deleted file mode 100644
index deb37de..0000000
--- a/hw/i386/acpi-dsdt-isa.dsl
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* Common legacy ISA style devices. */
-Scope(\_SB.PCI0.ISA) {
-
-    Device (SMC) {
-        Name(_HID, EisaId("APP0001"))
-        /* _STA will be patched to 0x0B if AppleSMC is present */
-        ACPI_EXTRACT_NAME_BYTE_CONST DSDT_APPLESMC_STA
-        Name(_STA, 0xF0)
-        Name(_CRS, ResourceTemplate () {
-            IO (Decode16, 0x0300, 0x0300, 0x01, 0x20)
-            IRQNoFlags() { 6 }
-        })
-    }
-
-    Device(RTC) {
-        Name(_HID, EisaId("PNP0B00"))
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x0070, 0x0070, 0x10, 0x02)
-            IRQNoFlags() { 8 }
-            IO(Decode16, 0x0072, 0x0072, 0x02, 0x06)
-        })
-    }
-
-    Device(KBD) {
-        Name(_HID, EisaId("PNP0303"))
-        Method(_STA, 0, NotSerialized) {
-            Return (0x0f)
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x0060, 0x0060, 0x01, 0x01)
-            IO(Decode16, 0x0064, 0x0064, 0x01, 0x01)
-            IRQNoFlags() { 1 }
-        })
-    }
-
-    Device(MOU) {
-        Name(_HID, EisaId("PNP0F13"))
-        Method(_STA, 0, NotSerialized) {
-            Return (0x0f)
-        }
-        Name(_CRS, ResourceTemplate() {
-            IRQNoFlags() { 12 }
-        })
-    }
-
-    Device(FDC0) {
-        Name(_HID, EisaId("PNP0700"))
-        Method(_STA, 0, NotSerialized) {
-            Store(FDEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x03F2, 0x03F2, 0x00, 0x04)
-            IO(Decode16, 0x03F7, 0x03F7, 0x00, 0x01)
-            IRQNoFlags() { 6 }
-            DMA(Compatibility, NotBusMaster, Transfer8) { 2 }
-        })
-    }
-
-    Device(LPT) {
-        Name(_HID, EisaId("PNP0400"))
-        Method(_STA, 0, NotSerialized) {
-            Store(LPEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x0378, 0x0378, 0x08, 0x08)
-            IRQNoFlags() { 7 }
-        })
-    }
-
-    Device(COM1) {
-        Name(_HID, EisaId("PNP0501"))
-        Name(_UID, 0x01)
-        Method(_STA, 0, NotSerialized) {
-            Store(CAEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x03F8, 0x03F8, 0x00, 0x08)
-            IRQNoFlags() { 4 }
-        })
-    }
-
-    Device(COM2) {
-        Name(_HID, EisaId("PNP0501"))
-        Name(_UID, 0x02)
-        Method(_STA, 0, NotSerialized) {
-            Store(CBEN, Local0)
-            If (LEqual(Local0, 0)) {
-                Return (0x00)
-            } Else {
-                Return (0x0F)
-            }
-        }
-        Name(_CRS, ResourceTemplate() {
-            IO(Decode16, 0x02F8, 0x02F8, 0x00, 0x08)
-            IRQNoFlags() { 3 }
-        })
-    }
-}
diff --git a/hw/i386/acpi-dsdt-mem-hotplug.dsl b/hw/i386/acpi-dsdt-mem-hotplug.dsl
deleted file mode 100644
index 2a36c47..0000000
--- a/hw/i386/acpi-dsdt-mem-hotplug.dsl
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-    External(MEMORY_SLOT_NOTIFY_METHOD, MethodObj)
-
-    Scope(\_SB.PCI0) {
-        Device(MEMORY_HOTPLUG_DEVICE) {
-            Name(_HID, "PNP0A06")
-            Name(_UID, "Memory hotplug resources")
-            External(MEMORY_SLOTS_NUMBER, IntObj)
-
-            /* Memory hotplug IO registers */
-            OperationRegion(MEMORY_HOTPLUG_IO_REGION, SystemIO,
-                            ACPI_MEMORY_HOTPLUG_BASE,
-                            ACPI_MEMORY_HOTPLUG_IO_LEN)
-
-            Name(_CRS, ResourceTemplate() {
-                IO(Decode16, ACPI_MEMORY_HOTPLUG_BASE, ACPI_MEMORY_HOTPLUG_BASE,
-                   0, ACPI_MEMORY_HOTPLUG_IO_LEN, IO)
-            })
-
-            Method(_STA, 0) {
-                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
-                    Return(0x0)
-                }
-                /* present, functioning, decoding, not shown in UI */
-                Return(0xB)
-            }
-
-            Field(MEMORY_HOTPLUG_IO_REGION, DWordAcc, NoLock, Preserve) {
-                MEMORY_SLOT_ADDR_LOW, 32,  // read only
-                MEMORY_SLOT_ADDR_HIGH, 32, // read only
-                MEMORY_SLOT_SIZE_LOW, 32,  // read only
-                MEMORY_SLOT_SIZE_HIGH, 32, // read only
-                MEMORY_SLOT_PROXIMITY, 32, // read only
-            }
-            Field(MEMORY_HOTPLUG_IO_REGION, ByteAcc, NoLock, Preserve) {
-                Offset(20),
-                MEMORY_SLOT_ENABLED,  1, // 1 if enabled, read only
-                MEMORY_SLOT_INSERT_EVENT, 1, // (read) 1 if has a insert event. (write) 1 to clear event
-            }
-
-            Mutex (MEMORY_SLOT_LOCK, 0)
-            Field (MEMORY_HOTPLUG_IO_REGION, DWordAcc, NoLock, Preserve) {
-                MEMORY_SLOT_SLECTOR, 32,  // DIMM selector, write only
-                MEMORY_SLOT_OST_EVENT, 32,  // _OST event code, write only
-                MEMORY_SLOT_OST_STATUS, 32,  // _OST status code, write only
-            }
-
-            Method(MEMORY_SLOT_SCAN_METHOD, 0) {
-                If (LEqual(MEMORY_SLOTS_NUMBER, Zero)) {
-                     Return(Zero)
-                }
-
-                Store(Zero, Local0) // Mem devs iterrator
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                while (LLess(Local0, MEMORY_SLOTS_NUMBER)) {
-                    Store(Local0, MEMORY_SLOT_SLECTOR) // select Local0 DIMM
-                    If (LEqual(MEMORY_SLOT_INSERT_EVENT, One)) { // Memory device needs check
-                        MEMORY_SLOT_NOTIFY_METHOD(Local0, 1)
-                        Store(1, MEMORY_SLOT_INSERT_EVENT)
-                    }
-                    // TODO: handle memory eject request
-                    Add(Local0, One, Local0) // goto next DIMM
-                }
-                Release(MEMORY_SLOT_LOCK)
-                Return(One)
-            }
-
-            Method(MEMORY_SLOT_STATUS_METHOD, 1) {
-                Store(Zero, Local0)
-
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-
-                If (LEqual(MEMORY_SLOT_ENABLED, One)) {
-                    Store(0xF, Local0)
-                }
-
-                Release(MEMORY_SLOT_LOCK)
-                Return(Local0)
-            }
-
-            Method(MEMORY_SLOT_CRS_METHOD, 1, Serialized) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-
-                Name(MR64, ResourceTemplate() {
-                    QWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
-                    Cacheable, ReadWrite,
-                    0x0000000000000000,        // Address Space Granularity
-                    0x0000000000000000,        // Address Range Minimum
-                    0xFFFFFFFFFFFFFFFE,        // Address Range Maximum
-                    0x0000000000000000,        // Address Translation Offset
-                    0xFFFFFFFFFFFFFFFF,        // Address Length
-                    ,, MW64, AddressRangeMemory, TypeStatic)
-                })
-
-                CreateDWordField(MR64, 14, MINL)
-                CreateDWordField(MR64, 18, MINH)
-                CreateDWordField(MR64, 38, LENL)
-                CreateDWordField(MR64, 42, LENH)
-                CreateDWordField(MR64, 22, MAXL)
-                CreateDWordField(MR64, 26, MAXH)
-
-                Store(MEMORY_SLOT_ADDR_HIGH, MINH)
-                Store(MEMORY_SLOT_ADDR_LOW, MINL)
-                Store(MEMORY_SLOT_SIZE_HIGH, LENH)
-                Store(MEMORY_SLOT_SIZE_LOW, LENL)
-
-                // 64-bit math: MAX = MIN + LEN - 1
-                Add(MINL, LENL, MAXL)
-                Add(MINH, LENH, MAXH)
-                If (LLess(MAXL, MINL)) {
-                    Add(MAXH, One, MAXH)
-                }
-                If (LLess(MAXL, One)) {
-                    Subtract(MAXH, One, MAXH)
-                }
-                Subtract(MAXL, One, MAXL)
-
-                If (LEqual(MAXH, Zero)){
-                    Name(MR32, ResourceTemplate() {
-                        DWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed,
-                        Cacheable, ReadWrite,
-                        0x00000000,        // Address Space Granularity
-                        0x00000000,        // Address Range Minimum
-                        0xFFFFFFFE,        // Address Range Maximum
-                        0x00000000,        // Address Translation Offset
-                        0xFFFFFFFF,        // Address Length
-                        ,, MW32, AddressRangeMemory, TypeStatic)
-                    })
-                    CreateDWordField(MR32, MW32._MIN, MIN)
-                    CreateDWordField(MR32, MW32._MAX, MAX)
-                    CreateDWordField(MR32, MW32._LEN, LEN)
-                    Store(MINL, MIN)
-                    Store(MAXL, MAX)
-                    Store(LENL, LEN)
-
-                    Release(MEMORY_SLOT_LOCK)
-                    Return(MR32)
-                }
-
-                Release(MEMORY_SLOT_LOCK)
-                Return(MR64)
-            }
-
-            Method(MEMORY_SLOT_PROXIMITY_METHOD, 1) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(MEMORY_SLOT_PROXIMITY, Local0)
-                Release(MEMORY_SLOT_LOCK)
-                Return(Local0)
-            }
-
-            Method(MEMORY_SLOT_OST_METHOD, 4) {
-                Acquire(MEMORY_SLOT_LOCK, 0xFFFF)
-                Store(ToInteger(Arg0), MEMORY_SLOT_SLECTOR) // select DIMM
-                Store(Arg1, MEMORY_SLOT_OST_EVENT)
-                Store(Arg2, MEMORY_SLOT_OST_STATUS)
-                Release(MEMORY_SLOT_LOCK)
-            }
-        } // Device()
-    } // Scope()
diff --git a/hw/i386/acpi-dsdt-pci-crs.dsl b/hw/i386/acpi-dsdt-pci-crs.dsl
deleted file mode 100644
index 4648e90..0000000
--- a/hw/i386/acpi-dsdt-pci-crs.dsl
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* PCI CRS (current resources) definition. */
-Scope(\_SB.PCI0) {
-
-    Name(CRES, ResourceTemplate() {
-        WordBusNumber(ResourceProducer, MinFixed, MaxFixed, PosDecode,
-            0x0000,             // Address Space Granularity
-            0x0000,             // Address Range Minimum
-            0x00FF,             // Address Range Maximum
-            0x0000,             // Address Translation Offset
-            0x0100,             // Address Length
-            ,, )
-        IO(Decode16,
-            0x0CF8,             // Address Range Minimum
-            0x0CF8,             // Address Range Maximum
-            0x01,               // Address Alignment
-            0x08,               // Address Length
-            )
-        BOARD_SPECIFIC_PCI_RESOURSES
-        DWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite,
-            0x00000000,         // Address Space Granularity
-            0x000A0000,         // Address Range Minimum
-            0x000BFFFF,         // Address Range Maximum
-            0x00000000,         // Address Translation Offset
-            0x00020000,         // Address Length
-            ,, , AddressRangeMemory, TypeStatic)
-        DWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed, NonCacheable, ReadWrite,
-            0x00000000,         // Address Space Granularity
-            0xE0000000,         // Address Range Minimum
-            0xFEBFFFFF,         // Address Range Maximum
-            0x00000000,         // Address Translation Offset
-            0x1EC00000,         // Address Length
-            ,, PW32, AddressRangeMemory, TypeStatic)
-    })
-
-    Name(CR64, ResourceTemplate() {
-        QWordMemory(ResourceProducer, PosDecode, MinFixed, MaxFixed, Cacheable, ReadWrite,
-            0x00000000,          // Address Space Granularity
-            0x8000000000,        // Address Range Minimum
-            0xFFFFFFFFFF,        // Address Range Maximum
-            0x00000000,          // Address Translation Offset
-            0x8000000000,        // Address Length
-            ,, PW64, AddressRangeMemory, TypeStatic)
-    })
-
-    Method(_CRS, 0) {
-        /* Fields provided by dynamically created ssdt */
-        External(P0S, IntObj)
-        External(P0E, IntObj)
-        External(P1V, IntObj)
-        External(P1S, BuffObj)
-        External(P1E, BuffObj)
-        External(P1L, BuffObj)
-
-        /* fixup 32bit pci io window */
-        CreateDWordField(CRES, \_SB.PCI0.PW32._MIN, PS32)
-        CreateDWordField(CRES, \_SB.PCI0.PW32._MAX, PE32)
-        CreateDWordField(CRES, \_SB.PCI0.PW32._LEN, PL32)
-        Store(P0S, PS32)
-        Store(P0E, PE32)
-        Store(Add(Subtract(P0E, P0S), 1), PL32)
-
-        If (LEqual(P1V, Zero)) {
-            Return (CRES)
-        }
-
-        /* fixup 64bit pci io window */
-        CreateQWordField(CR64, \_SB.PCI0.PW64._MIN, PS64)
-        CreateQWordField(CR64, \_SB.PCI0.PW64._MAX, PE64)
-        CreateQWordField(CR64, \_SB.PCI0.PW64._LEN, PL64)
-        Store(P1S, PS64)
-        Store(P1E, PE64)
-        Store(P1L, PL64)
-        /* add window and return result */
-        ConcatenateResTemplate(CRES, CR64, Local0)
-        Return (Local0)
-    }
-}
diff --git a/hw/i386/acpi-dsdt.dsl b/hw/i386/acpi-dsdt.dsl
deleted file mode 100644
index 8326a96..0000000
--- a/hw/i386/acpi-dsdt.dsl
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Bochs/QEMU ACPI DSDT ASL definition
- *
- * Copyright (c) 2006 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2 as published by the Free Software Foundation.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- */
-
-ACPI_EXTRACT_ALL_CODE AcpiDsdtAmlCode
-
-DefinitionBlock (
-    "acpi-dsdt.aml",    // Output Filename
-    "DSDT",             // Signature
-    0x01,               // DSDT Compliance Revision
-    "BXPC",             // OEMID
-    "BXDSDT",           // TABLE ID
-    0x1                 // OEM Revision
-    )
-{
-
-#include "acpi-dsdt-dbug.dsl"
-
-/* Definitions for Android-specific virtual hardware */
-#include "acpi-dsdt-goldfish.dsl"
-
-
-/****************************************************************
- * PCI Bus definition
- ****************************************************************/
-#define BOARD_SPECIFIC_PCI_RESOURSES \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0x0000, \
-         0x0CF7, \
-         0x0000, \
-         0x0CF8, \
-         ,, , TypeStatic) \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0x0D00, \
-         0xADFF, \
-         0x0000, \
-         0xA100, \
-         ,, , TypeStatic) \
-     /* 0xae00-0xae0e hole for PCI hotplug, hw/acpi/piix4.c:PCI_HOTPLUG_ADDR */ \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0xAE0F, \
-         0xAEFF, \
-         0x0000, \
-         0x00F1, \
-         ,, , TypeStatic) \
-     /* 0xaf00-0xaf1f hole for CPU hotplug, hw/acpi/piix4.c:PIIX4_PROC_BASE */ \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0xAF20, \
-         0xAFDF, \
-         0x0000, \
-         0x00C0, \
-         ,, , TypeStatic) \
-     /* 0xafe0-0xafe3 hole for ACPI.GPE0, hw/acpi/piix4.c:GPE_BASE */ \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0xAFE4, \
-         0xFFFF, \
-         0x0000, \
-         0x501C, \
-         ,, , TypeStatic)
-
-    Scope(\_SB) {
-        Device(PCI0) {
-            Name(_HID, EisaId("PNP0A03"))
-            Name(_ADR, 0x00)
-            Name(_UID, 1)
-//#define PX13 S0B_
-//            External(PX13, DeviceObj)
-        }
-    }
-
-#include "acpi-dsdt-pci-crs.dsl"
-#include "acpi-dsdt-hpet.dsl"
-
-
-/****************************************************************
- * PIIX4 PM
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-        Device(PX13) {
-            Name(_ADR, 0x00010003)
-            OperationRegion(P13C, PCI_Config, 0x00, 0xff)
-        }
-    }
-
-
-/****************************************************************
- * PIIX3 ISA bridge
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-
-        External(ISA, DeviceObj)
-
-        Device(ISA) {
-            Name(_ADR, 0x00010000)
-
-            /* PIIX PCI to ISA irq remapping */
-            OperationRegion(P40C, PCI_Config, 0x60, 0x04)
-
-            /* enable bits */
-            Field(\_SB.PCI0.PX13.P13C, AnyAcc, NoLock, Preserve) {
-                Offset(0x5f),
-                , 7,
-                LPEN, 1,         // LPT
-                Offset(0x67),
-                , 3,
-                CAEN, 1,         // COM1
-                , 3,
-                CBEN, 1,         // COM2
-            }
-            Name(FDEN, 1)
-        }
-    }
-
-#define DSDT_APPLESMC_STA piix_dsdt_applesmc_sta
-#include "acpi-dsdt-isa.dsl"
-
-
-/****************************************************************
- * PCI hotplug
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-        OperationRegion(PCST, SystemIO, 0xae00, 0x08)
-        Field(PCST, DWordAcc, NoLock, WriteAsZeros) {
-            PCIU, 32,
-            PCID, 32,
-        }
-
-        OperationRegion(SEJ, SystemIO, 0xae08, 0x04)
-        Field(SEJ, DWordAcc, NoLock, WriteAsZeros) {
-            B0EJ, 32,
-        }
-
-        OperationRegion(BNMR, SystemIO, 0xae10, 0x04)
-        Field(BNMR, DWordAcc, NoLock, WriteAsZeros) {
-            BNUM, 32,
-        }
-
-        /* Lock to protect access to fields above. */
-        Mutex(BLCK, 0)
-
-        /* Methods called by bulk generated PCI devices below */
-
-        /* Methods called by hotplug devices */
-        Method(PCEJ, 2, NotSerialized) {
-            // _EJ0 method - eject callback
-            Acquire(BLCK, 0xFFFF)
-            Store(Arg0, BNUM)
-            Store(ShiftLeft(1, Arg1), B0EJ)
-            Release(BLCK)
-            Return (0x0)
-        }
-
-        /* Hotplug notification method supplied by SSDT */
-        External(\_SB.PCI0.PCNT, MethodObj)
-    }
-
-
-/****************************************************************
- * PCI IRQs
- ****************************************************************/
-
-    Scope(\_SB) {
-        Scope(PCI0) {
-            Method (_PRT, 0) {
-                Store(Package(128) {}, Local0)
-                Store(Zero, Local1)
-                While(LLess(Local1, 128)) {
-                    // slot = pin >> 2
-                    Store(ShiftRight(Local1, 2), Local2)
-
-                    // lnk = (slot + pin) & 3
-                    Store(And(Add(Local1, Local2), 3), Local3)
-                    If (LEqual(Local3, 0)) {
-                        Store(Package(4) { Zero, Zero, LNKD, Zero }, Local4)
-                    }
-                    If (LEqual(Local3, 1)) {
-                        // device 1 is the power-management device, needs SCI
-                        If (LEqual(Local1, 4)) {
-                            Store(Package(4) { Zero, Zero, LNKS, Zero }, Local4)
-                        } Else {
-                            Store(Package(4) { Zero, Zero, LNKA, Zero }, Local4)
-                        }
-                    }
-                    If (LEqual(Local3, 2)) {
-                        Store(Package(4) { Zero, Zero, LNKB, Zero }, Local4)
-                    }
-                    If (LEqual(Local3, 3)) {
-                        Store(Package(4) { Zero, Zero, LNKC, Zero }, Local4)
-                    }
-
-                    // Complete the interrupt routing entry:
-                    //    Package(4) { 0x[slot]FFFF, [pin], [link], 0) }
-
-                    Store(Or(ShiftLeft(Local2, 16), 0xFFFF), Index(Local4, 0))
-                    Store(And(Local1, 3),                    Index(Local4, 1))
-                    Store(Local4,                            Index(Local0, Local1))
-
-                    Increment(Local1)
-                }
-
-                Return(Local0)
-            }
-        }
-
-        Field(PCI0.ISA.P40C, ByteAcc, NoLock, Preserve) {
-            PRQ0,   8,
-            PRQ1,   8,
-            PRQ2,   8,
-            PRQ3,   8
-        }
-
-        Method(IQST, 1, NotSerialized) {
-            // _STA method - get status
-            If (And(0x80, Arg0)) {
-                Return (0x09)
-            }
-            Return (0x0B)
-        }
-        Method(IQCR, 1, Serialized) {
-            // _CRS method - get current settings
-            Name(PRR0, ResourceTemplate() {
-                Interrupt(, Level, ActiveHigh, Shared) { 0 }
-            })
-            CreateDWordField(PRR0, 0x05, PRRI)
-            If (LLess(Arg0, 0x80)) {
-                Store(Arg0, PRRI)
-            }
-            Return (PRR0)
-        }
-
-#define define_link(link, uid, reg)                             \
-        Device(link) {                                          \
-            Name(_HID, EISAID("PNP0C0F"))                       \
-            Name(_UID, uid)                                     \
-            Name(_PRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    5, 10, 11                                   \
-                }                                               \
-            })                                                  \
-            Method(_STA, 0, NotSerialized) {                    \
-                Return (IQST(reg))                              \
-            }                                                   \
-            Method(_DIS, 0, NotSerialized) {                    \
-                Or(reg, 0x80, reg)                              \
-            }                                                   \
-            Method(_CRS, 0, NotSerialized) {                    \
-                Return (IQCR(reg))                              \
-            }                                                   \
-            Method(_SRS, 1, NotSerialized) {                    \
-                CreateDWordField(Arg0, 0x05, PRRI)              \
-                Store(PRRI, reg)                                \
-            }                                                   \
-        }
-
-        define_link(LNKA, 0, PRQ0)
-        define_link(LNKB, 1, PRQ1)
-        define_link(LNKC, 2, PRQ2)
-        define_link(LNKD, 3, PRQ3)
-
-        Device(LNKS) {
-            Name(_HID, EISAID("PNP0C0F"))
-            Name(_UID, 4)
-            Name(_PRS, ResourceTemplate() {
-                Interrupt(, Level, ActiveHigh, Shared) { 9 }
-            })
-
-            // The SCI cannot be disabled and is always attached to GSI 9,
-            // so these are no-ops.  We only need this link to override the
-            // polarity to active high and match the content of the MADT.
-            Method(_STA, 0, NotSerialized) { Return (0x0b) }
-            Method(_DIS, 0, NotSerialized) { }
-            Method(_CRS, 0, NotSerialized) { Return (_PRS) }
-            Method(_SRS, 1, NotSerialized) { }
-        }
-    }
-
-#include "hw/acpi/pc-hotplug.h"
-#define CPU_STATUS_BASE PIIX4_CPU_HOTPLUG_IO_BASE
-#include "acpi-dsdt-cpu-hotplug.dsl"
-#include "acpi-dsdt-mem-hotplug.dsl"
-
-
-/****************************************************************
- * General purpose events
- ****************************************************************/
-    Scope(\_GPE) {
-        Name(_HID, "ACPI0006")
-
-        Method(_L00) {
-        }
-        Method(_E01) {
-            // PCI hotplug event
-            Acquire(\_SB.PCI0.BLCK, 0xFFFF)
-            \_SB.PCI0.PCNT()
-            Release(\_SB.PCI0.BLCK)
-        }
-        Method(_E02) {
-            // CPU hotplug event
-            \_SB.PRSC()
-        }
-        Method(_E03) {
-            // Memory hotplug event
-            \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
-        }
-        Method(_L04) {
-        }
-        Method(_L05) {
-        }
-        Method(_L06) {
-        }
-        Method(_L07) {
-        }
-        Method(_L08) {
-        }
-        Method(_L09) {
-        }
-        Method(_L0A) {
-        }
-        Method(_L0B) {
-        }
-        Method(_L0C) {
-        }
-        Method(_L0D) {
-        }
-        Method(_L0E) {
-        }
-        Method(_L0F) {
-        }
-    }
-}
diff --git a/hw/i386/acpi-dsdt.hex.generated b/hw/i386/acpi-dsdt.hex.generated
deleted file mode 100644
index e0a7142..0000000
--- a/hw/i386/acpi-dsdt.hex.generated
+++ /dev/null
@@ -1,4270 +0,0 @@
-static unsigned char AcpiDsdtAmlCode[] = {
-0x44,
-0x53,
-0x44,
-0x54,
-0xa9,
-0x10,
-0x0,
-0x0,
-0x1,
-0xcb,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x44,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x14,
-0x2,
-0x14,
-0x20,
-0x10,
-0x49,
-0x4,
-0x5c,
-0x0,
-0x5b,
-0x80,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0xb,
-0x2,
-0x4,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0x44,
-0x42,
-0x47,
-0x42,
-0x8,
-0x14,
-0x2c,
-0x44,
-0x42,
-0x55,
-0x47,
-0x1,
-0x98,
-0x68,
-0x60,
-0x96,
-0x60,
-0x60,
-0x74,
-0x87,
-0x60,
-0x1,
-0x61,
-0x70,
-0x0,
-0x62,
-0xa2,
-0x10,
-0x95,
-0x62,
-0x61,
-0x70,
-0x83,
-0x88,
-0x60,
-0x62,
-0x0,
-0x44,
-0x42,
-0x47,
-0x42,
-0x75,
-0x62,
-0x70,
-0xa,
-0xa,
-0x44,
-0x42,
-0x47,
-0x42,
-0x10,
-0x40,
-0x2a,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x40,
-0x6,
-0x47,
-0x46,
-0x42,
-0x59,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x31,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x25,
-0xa,
-0x22,
-0x67,
-0x0,
-0x6f,
-0x0,
-0x6c,
-0x0,
-0x64,
-0x0,
-0x66,
-0x0,
-0x69,
-0x0,
-0x73,
-0x0,
-0x68,
-0x0,
-0x20,
-0x0,
-0x62,
-0x0,
-0x61,
-0x0,
-0x74,
-0x0,
-0x74,
-0x0,
-0x65,
-0x0,
-0x72,
-0x0,
-0x79,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x0,
-0x1,
-0xff,
-0x0,
-0x10,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x10,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4e,
-0x5,
-0x47,
-0x46,
-0x45,
-0x56,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x32,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x23,
-0xa,
-0x20,
-0x67,
-0x0,
-0x6f,
-0x0,
-0x6c,
-0x0,
-0x64,
-0x0,
-0x66,
-0x0,
-0x69,
-0x0,
-0x73,
-0x0,
-0x68,
-0x0,
-0x20,
-0x0,
-0x65,
-0x0,
-0x76,
-0x0,
-0x65,
-0x0,
-0x6e,
-0x0,
-0x74,
-0x0,
-0x73,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x10,
-0x1,
-0xff,
-0x0,
-0x10,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x11,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x48,
-0x5,
-0x47,
-0x46,
-0x50,
-0x50,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x33,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x1d,
-0xa,
-0x1a,
-0x61,
-0x0,
-0x6e,
-0x0,
-0x64,
-0x0,
-0x72,
-0x0,
-0x6f,
-0x0,
-0x69,
-0x0,
-0x64,
-0x0,
-0x20,
-0x0,
-0x70,
-0x0,
-0x69,
-0x0,
-0x70,
-0x0,
-0x65,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x10,
-0x0,
-0xff,
-0x0,
-0x20,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x12,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x48,
-0x6,
-0x47,
-0x46,
-0x46,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x34,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x2d,
-0xa,
-0x2a,
-0x67,
-0x0,
-0x6f,
-0x0,
-0x6c,
-0x0,
-0x64,
-0x0,
-0x66,
-0x0,
-0x69,
-0x0,
-0x73,
-0x0,
-0x68,
-0x0,
-0x20,
-0x0,
-0x66,
-0x0,
-0x72,
-0x0,
-0x61,
-0x0,
-0x6d,
-0x0,
-0x65,
-0x0,
-0x62,
-0x0,
-0x75,
-0x0,
-0x66,
-0x0,
-0x66,
-0x0,
-0x65,
-0x0,
-0x72,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x20,
-0x1,
-0xff,
-0x0,
-0x1,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x13,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4c,
-0x5,
-0x47,
-0x46,
-0x41,
-0x55,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x35,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x21,
-0xa,
-0x1e,
-0x67,
-0x0,
-0x6f,
-0x0,
-0x6c,
-0x0,
-0x64,
-0x0,
-0x66,
-0x0,
-0x69,
-0x0,
-0x73,
-0x0,
-0x68,
-0x0,
-0x20,
-0x0,
-0x61,
-0x0,
-0x75,
-0x0,
-0x64,
-0x0,
-0x69,
-0x0,
-0x6f,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x30,
-0x1,
-0xff,
-0x0,
-0x1,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x14,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4a,
-0x5,
-0x47,
-0x46,
-0x53,
-0x4b,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x36,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x1f,
-0xa,
-0x1c,
-0x67,
-0x0,
-0x6f,
-0x0,
-0x6c,
-0x0,
-0x64,
-0x0,
-0x66,
-0x0,
-0x69,
-0x0,
-0x73,
-0x0,
-0x68,
-0x0,
-0x20,
-0x0,
-0x73,
-0x0,
-0x79,
-0x0,
-0x6e,
-0x0,
-0x63,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x40,
-0x1,
-0xff,
-0x0,
-0x20,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x15,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x48,
-0x5,
-0x47,
-0x46,
-0x52,
-0x54,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x47,
-0x46,
-0x53,
-0x48,
-0x30,
-0x30,
-0x30,
-0x37,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x52,
-0x11,
-0x1d,
-0xa,
-0x1a,
-0x67,
-0x0,
-0x6f,
-0x0,
-0x6c,
-0x0,
-0x64,
-0x0,
-0x66,
-0x0,
-0x69,
-0x0,
-0x73,
-0x0,
-0x68,
-0x0,
-0x20,
-0x0,
-0x72,
-0x0,
-0x74,
-0x0,
-0x63,
-0x0,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1a,
-0xa,
-0x17,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x60,
-0x1,
-0xff,
-0x0,
-0x10,
-0x0,
-0x0,
-0x89,
-0x6,
-0x0,
-0x3,
-0x1,
-0x16,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x10,
-0x22,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x1b,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x10,
-0x4e,
-0x18,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x43,
-0x52,
-0x45,
-0x53,
-0x11,
-0x42,
-0xa,
-0xa,
-0x9e,
-0x88,
-0xd,
-0x0,
-0x2,
-0xc,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x1,
-0x47,
-0x1,
-0xf8,
-0xc,
-0xf8,
-0xc,
-0x1,
-0x8,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0xf7,
-0xc,
-0x0,
-0x0,
-0xf8,
-0xc,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0xd,
-0xff,
-0xad,
-0x0,
-0x0,
-0x0,
-0xa1,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0xf,
-0xae,
-0xff,
-0xae,
-0x0,
-0x0,
-0xf1,
-0x0,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0x20,
-0xaf,
-0xdf,
-0xaf,
-0x0,
-0x0,
-0xc0,
-0x0,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0xe4,
-0xaf,
-0xff,
-0xff,
-0x0,
-0x0,
-0x1c,
-0x50,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0xff,
-0xff,
-0xb,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x2,
-0x0,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xe0,
-0xff,
-0xff,
-0xbf,
-0xfe,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xc0,
-0x1e,
-0x79,
-0x0,
-0x8,
-0x43,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x80,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x80,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x41,
-0xa,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0x8a,
-0x43,
-0x52,
-0x45,
-0x53,
-0xa,
-0x8c,
-0x50,
-0x53,
-0x33,
-0x32,
-0x8a,
-0x43,
-0x52,
-0x45,
-0x53,
-0xa,
-0x90,
-0x50,
-0x45,
-0x33,
-0x32,
-0x8a,
-0x43,
-0x52,
-0x45,
-0x53,
-0xa,
-0x98,
-0x50,
-0x4c,
-0x33,
-0x32,
-0x70,
-0x50,
-0x30,
-0x53,
-0x5f,
-0x50,
-0x53,
-0x33,
-0x32,
-0x70,
-0x50,
-0x30,
-0x45,
-0x5f,
-0x50,
-0x45,
-0x33,
-0x32,
-0x70,
-0x72,
-0x74,
-0x50,
-0x30,
-0x45,
-0x5f,
-0x50,
-0x30,
-0x53,
-0x5f,
-0x0,
-0x1,
-0x0,
-0x50,
-0x4c,
-0x33,
-0x32,
-0xa0,
-0xc,
-0x93,
-0x50,
-0x31,
-0x56,
-0x5f,
-0x0,
-0xa4,
-0x43,
-0x52,
-0x45,
-0x53,
-0x8f,
-0x43,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x50,
-0x53,
-0x36,
-0x34,
-0x8f,
-0x43,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x50,
-0x45,
-0x36,
-0x34,
-0x8f,
-0x43,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x50,
-0x4c,
-0x36,
-0x34,
-0x70,
-0x50,
-0x31,
-0x53,
-0x5f,
-0x50,
-0x53,
-0x36,
-0x34,
-0x70,
-0x50,
-0x31,
-0x45,
-0x5f,
-0x50,
-0x45,
-0x36,
-0x34,
-0x70,
-0x50,
-0x31,
-0x4c,
-0x5f,
-0x50,
-0x4c,
-0x36,
-0x34,
-0x84,
-0x43,
-0x52,
-0x45,
-0x53,
-0x43,
-0x52,
-0x36,
-0x34,
-0x60,
-0xa4,
-0x60,
-0x10,
-0x4d,
-0x8,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x45,
-0x8,
-0x48,
-0x50,
-0x45,
-0x54,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x1,
-0x3,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x0,
-0xc,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0xb,
-0x0,
-0x4,
-0x5b,
-0x81,
-0x10,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x13,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x20,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x20,
-0x14,
-0x36,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x60,
-0x70,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x61,
-0x7a,
-0x60,
-0xa,
-0x10,
-0x60,
-0xa0,
-0xc,
-0x91,
-0x93,
-0x60,
-0x0,
-0x93,
-0x60,
-0xb,
-0xff,
-0xff,
-0xa4,
-0x0,
-0xa0,
-0xe,
-0x91,
-0x93,
-0x61,
-0x0,
-0x94,
-0x61,
-0xc,
-0x0,
-0xe1,
-0xf5,
-0x5,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x11,
-0xa,
-0xe,
-0x86,
-0x9,
-0x0,
-0x0,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0x0,
-0x4,
-0x0,
-0x0,
-0x79,
-0x0,
-0x10,
-0x25,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x19,
-0x50,
-0x58,
-0x31,
-0x33,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x3,
-0x0,
-0x1,
-0x0,
-0x5b,
-0x80,
-0x50,
-0x31,
-0x33,
-0x43,
-0x2,
-0x0,
-0xa,
-0xff,
-0x10,
-0x46,
-0x5,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x49,
-0x4,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0x1,
-0x0,
-0x5b,
-0x80,
-0x50,
-0x34,
-0x30,
-0x43,
-0x2,
-0xa,
-0x60,
-0xa,
-0x4,
-0x5b,
-0x81,
-0x26,
-0x5e,
-0x2e,
-0x50,
-0x58,
-0x31,
-0x33,
-0x50,
-0x31,
-0x33,
-0x43,
-0x0,
-0x0,
-0x48,
-0x2f,
-0x0,
-0x7,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x1,
-0x0,
-0x38,
-0x0,
-0x3,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x1,
-0x0,
-0x3,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x1,
-0x8,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x1,
-0x10,
-0x4a,
-0x1e,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x5b,
-0x82,
-0x2c,
-0x53,
-0x4d,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x6,
-0x10,
-0x0,
-0x1,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x41,
-0xa,
-0xf0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0x0,
-0x3,
-0x0,
-0x3,
-0x1,
-0x20,
-0x22,
-0x40,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x2d,
-0x52,
-0x54,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xb,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x70,
-0x0,
-0x70,
-0x0,
-0x10,
-0x2,
-0x22,
-0x0,
-0x1,
-0x47,
-0x1,
-0x72,
-0x0,
-0x72,
-0x0,
-0x2,
-0x6,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x37,
-0x4b,
-0x42,
-0x44,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x3,
-0x3,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x60,
-0x0,
-0x60,
-0x0,
-0x1,
-0x1,
-0x47,
-0x1,
-0x64,
-0x0,
-0x64,
-0x0,
-0x1,
-0x1,
-0x22,
-0x2,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x27,
-0x4d,
-0x4f,
-0x55,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xf,
-0x13,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x8,
-0xa,
-0x5,
-0x22,
-0x0,
-0x10,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4a,
-0x4,
-0x46,
-0x44,
-0x43,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x7,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1b,
-0xa,
-0x18,
-0x47,
-0x1,
-0xf2,
-0x3,
-0xf2,
-0x3,
-0x0,
-0x4,
-0x47,
-0x1,
-0xf7,
-0x3,
-0xf7,
-0x3,
-0x0,
-0x1,
-0x22,
-0x40,
-0x0,
-0x2a,
-0x4,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x3e,
-0x4c,
-0x50,
-0x54,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x4,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0x78,
-0x3,
-0x78,
-0x3,
-0x8,
-0x8,
-0x22,
-0x80,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x31,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x3,
-0xf8,
-0x3,
-0x0,
-0x8,
-0x22,
-0x10,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x46,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x32,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x2,
-0xf8,
-0x2,
-0x0,
-0x8,
-0x22,
-0x8,
-0x0,
-0x79,
-0x0,
-0x10,
-0x48,
-0x8,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x80,
-0x50,
-0x43,
-0x53,
-0x54,
-0x1,
-0xb,
-0x0,
-0xae,
-0xa,
-0x8,
-0x5b,
-0x81,
-0x10,
-0x50,
-0x43,
-0x53,
-0x54,
-0x43,
-0x50,
-0x43,
-0x49,
-0x55,
-0x20,
-0x50,
-0x43,
-0x49,
-0x44,
-0x20,
-0x5b,
-0x80,
-0x53,
-0x45,
-0x4a,
-0x5f,
-0x1,
-0xb,
-0x8,
-0xae,
-0xa,
-0x4,
-0x5b,
-0x81,
-0xb,
-0x53,
-0x45,
-0x4a,
-0x5f,
-0x43,
-0x42,
-0x30,
-0x45,
-0x4a,
-0x20,
-0x5b,
-0x80,
-0x42,
-0x4e,
-0x4d,
-0x52,
-0x1,
-0xb,
-0x10,
-0xae,
-0xa,
-0x4,
-0x5b,
-0x81,
-0xb,
-0x42,
-0x4e,
-0x4d,
-0x52,
-0x43,
-0x42,
-0x4e,
-0x55,
-0x4d,
-0x20,
-0x5b,
-0x1,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x14,
-0x25,
-0x50,
-0x43,
-0x45,
-0x4a,
-0x2,
-0x5b,
-0x23,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x68,
-0x42,
-0x4e,
-0x55,
-0x4d,
-0x70,
-0x79,
-0x1,
-0x69,
-0x0,
-0x42,
-0x30,
-0x45,
-0x4a,
-0x5b,
-0x27,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x0,
-0x10,
-0x4e,
-0x36,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x10,
-0x4b,
-0xa,
-0x50,
-0x43,
-0x49,
-0x30,
-0x14,
-0x44,
-0xa,
-0x5f,
-0x50,
-0x52,
-0x54,
-0x0,
-0x70,
-0x12,
-0x2,
-0x80,
-0x60,
-0x70,
-0x0,
-0x61,
-0xa2,
-0x42,
-0x9,
-0x95,
-0x61,
-0xa,
-0x80,
-0x70,
-0x7a,
-0x61,
-0xa,
-0x2,
-0x0,
-0x62,
-0x70,
-0x7b,
-0x72,
-0x61,
-0x62,
-0x0,
-0xa,
-0x3,
-0x0,
-0x63,
-0xa0,
-0x10,
-0x93,
-0x63,
-0x0,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x64,
-0xa0,
-0x24,
-0x93,
-0x63,
-0x1,
-0xa0,
-0x11,
-0x93,
-0x61,
-0xa,
-0x4,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x53,
-0x0,
-0x64,
-0xa1,
-0xd,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x64,
-0xa0,
-0x11,
-0x93,
-0x63,
-0xa,
-0x2,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x64,
-0xa0,
-0x11,
-0x93,
-0x63,
-0xa,
-0x3,
-0x70,
-0x12,
-0x9,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x64,
-0x70,
-0x7d,
-0x79,
-0x62,
-0xa,
-0x10,
-0x0,
-0xb,
-0xff,
-0xff,
-0x0,
-0x88,
-0x64,
-0x0,
-0x0,
-0x70,
-0x7b,
-0x61,
-0xa,
-0x3,
-0x0,
-0x88,
-0x64,
-0x1,
-0x0,
-0x70,
-0x64,
-0x88,
-0x60,
-0x61,
-0x0,
-0x75,
-0x61,
-0xa4,
-0x60,
-0x5b,
-0x81,
-0x24,
-0x2f,
-0x3,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x50,
-0x34,
-0x30,
-0x43,
-0x1,
-0x50,
-0x52,
-0x51,
-0x30,
-0x8,
-0x50,
-0x52,
-0x51,
-0x31,
-0x8,
-0x50,
-0x52,
-0x51,
-0x32,
-0x8,
-0x50,
-0x52,
-0x51,
-0x33,
-0x8,
-0x14,
-0x13,
-0x49,
-0x51,
-0x53,
-0x54,
-0x1,
-0xa0,
-0x9,
-0x7b,
-0xa,
-0x80,
-0x68,
-0x0,
-0xa4,
-0xa,
-0x9,
-0xa4,
-0xa,
-0xb,
-0x14,
-0x36,
-0x49,
-0x51,
-0x43,
-0x52,
-0x9,
-0x8,
-0x50,
-0x52,
-0x52,
-0x30,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8a,
-0x50,
-0x52,
-0x52,
-0x30,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0xa0,
-0xb,
-0x95,
-0x68,
-0xa,
-0x80,
-0x70,
-0x68,
-0x50,
-0x52,
-0x52,
-0x49,
-0xa4,
-0x50,
-0x52,
-0x52,
-0x30,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x30,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x30,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x30,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x30,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x30,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x31,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x31,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x31,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x31,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x31,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x32,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x32,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x32,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x32,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x32,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x33,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x33,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x33,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x33,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x33,
-0x5b,
-0x82,
-0x4f,
-0x4,
-0x4c,
-0x4e,
-0x4b,
-0x53,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x4,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x9,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x14,
-0x6,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x14,
-0xb,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x10,
-0x42,
-0x11,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x14,
-0x35,
-0x43,
-0x50,
-0x4d,
-0x41,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0x70,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x61,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x2,
-0x0,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x3,
-0x0,
-0x70,
-0x60,
-0x88,
-0x61,
-0xa,
-0x4,
-0x0,
-0xa4,
-0x61,
-0x14,
-0x1a,
-0x43,
-0x50,
-0x53,
-0x54,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0xa0,
-0x5,
-0x60,
-0xa4,
-0xa,
-0xf,
-0xa1,
-0x3,
-0xa4,
-0x0,
-0x14,
-0xa,
-0x43,
-0x50,
-0x45,
-0x4a,
-0x2,
-0x5b,
-0x22,
-0xa,
-0xc8,
-0x5b,
-0x80,
-0x50,
-0x52,
-0x53,
-0x54,
-0x1,
-0xb,
-0x0,
-0xaf,
-0xa,
-0x20,
-0x5b,
-0x81,
-0xc,
-0x50,
-0x52,
-0x53,
-0x54,
-0x1,
-0x50,
-0x52,
-0x53,
-0x5f,
-0x40,
-0x10,
-0x14,
-0x4a,
-0x6,
-0x50,
-0x52,
-0x53,
-0x43,
-0x0,
-0x70,
-0x50,
-0x52,
-0x53,
-0x5f,
-0x65,
-0x70,
-0x0,
-0x62,
-0x70,
-0x0,
-0x60,
-0xa2,
-0x46,
-0x5,
-0x95,
-0x60,
-0x87,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0x61,
-0xa0,
-0xa,
-0x7b,
-0x60,
-0xa,
-0x7,
-0x0,
-0x7a,
-0x62,
-0x1,
-0x62,
-0xa1,
-0xc,
-0x70,
-0x83,
-0x88,
-0x65,
-0x7a,
-0x60,
-0xa,
-0x3,
-0x0,
-0x0,
-0x62,
-0x70,
-0x7b,
-0x62,
-0x1,
-0x0,
-0x63,
-0xa0,
-0x22,
-0x92,
-0x93,
-0x61,
-0x63,
-0x70,
-0x63,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0xa0,
-0xa,
-0x93,
-0x63,
-0x1,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0xa1,
-0x8,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0xa,
-0x3,
-0x75,
-0x60,
-0x5b,
-0x82,
-0x29,
-0x50,
-0x52,
-0x45,
-0x53,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x6,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xd,
-0xa,
-0xa,
-0x47,
-0x1,
-0x0,
-0xaf,
-0x0,
-0xaf,
-0x0,
-0x20,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x41,
-0xa,
-0xb,
-0x10,
-0x40,
-0x31,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x43,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x50,
-0x4e,
-0x50,
-0x30,
-0x41,
-0x30,
-0x36,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xd,
-0x4d,
-0x65,
-0x6d,
-0x6f,
-0x72,
-0x79,
-0x20,
-0x68,
-0x6f,
-0x74,
-0x70,
-0x6c,
-0x75,
-0x67,
-0x20,
-0x72,
-0x65,
-0x73,
-0x6f,
-0x75,
-0x72,
-0x63,
-0x65,
-0x73,
-0x0,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x1,
-0xb,
-0x0,
-0xa,
-0xa,
-0x18,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xd,
-0xa,
-0xa,
-0x47,
-0x1,
-0x0,
-0xa,
-0x0,
-0xa,
-0x0,
-0x18,
-0x79,
-0x0,
-0x14,
-0x13,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x5b,
-0x81,
-0x1f,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x3,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x20,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x20,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x20,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x20,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x20,
-0x5b,
-0x81,
-0x13,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x1,
-0x0,
-0x40,
-0xa,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x5b,
-0x1,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x5b,
-0x81,
-0x15,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x3,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x20,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x20,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x20,
-0x14,
-0x4a,
-0x4,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0xa2,
-0x25,
-0x95,
-0x60,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x70,
-0x60,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0x13,
-0x93,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x4d,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0x70,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x72,
-0x60,
-0x1,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x1,
-0x14,
-0x2d,
-0x4d,
-0x52,
-0x53,
-0x54,
-0x1,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0xb,
-0x93,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x70,
-0xa,
-0xf,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x41,
-0x18,
-0x4d,
-0x43,
-0x52,
-0x53,
-0x9,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x8,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x12,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x2a,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x1a,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x14,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x11,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0xa0,
-0x44,
-0x7,
-0x93,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x0,
-0x8,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x11,
-0x1f,
-0xa,
-0x1c,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xa,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xe,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0x16,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x70,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x14,
-0x24,
-0x4d,
-0x50,
-0x58,
-0x4d,
-0x1,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x28,
-0x4d,
-0x4f,
-0x53,
-0x54,
-0x4,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x69,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x70,
-0x6a,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x10,
-0x45,
-0xd,
-0x5f,
-0x47,
-0x50,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x41,
-0x43,
-0x50,
-0x49,
-0x30,
-0x30,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x30,
-0x0,
-0x14,
-0x39,
-0x5f,
-0x45,
-0x30,
-0x31,
-0x0,
-0x5b,
-0x23,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x50,
-0x43,
-0x4e,
-0x54,
-0x5b,
-0x27,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x42,
-0x4c,
-0x43,
-0x4b,
-0x14,
-0x10,
-0x5f,
-0x45,
-0x30,
-0x32,
-0x0,
-0x5c,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x43,
-0x14,
-0x19,
-0x5f,
-0x45,
-0x30,
-0x33,
-0x0,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x34,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x35,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x37,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x38,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x39,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x41,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x42,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x43,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x44,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x45,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x46,
-0x0
-};
-static unsigned short piix_dsdt_applesmc_sta[] = {
-0x5f4
-};
diff --git a/hw/i386/bios-linker-loader.c b/hw/i386/bios-linker-loader.c
deleted file mode 100644
index aa56184..0000000
--- a/hw/i386/bios-linker-loader.c
+++ /dev/null
@@ -1,157 +0,0 @@
-/* Dynamic linker/loader of ACPI tables
- *
- * Copyright (C) 2013 Red Hat Inc
- *
- * Author: Michael S. Tsirkin <mst@redhat.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "qemu-common.h"
-#include "bios-linker-loader.h"
-#include "hw/nvram/fw_cfg.h"
-
-#include "qemu/bswap.h"
-
-#define BIOS_LINKER_LOADER_FILESZ FW_CFG_MAX_FILE_PATH
-
-struct BiosLinkerLoaderEntry {
-    uint32_t command;
-    union {
-        /*
-         * COMMAND_ALLOCATE - allocate a table from @alloc.file
-         * subject to @alloc.align alignment (must be power of 2)
-         * and @alloc.zone (can be HIGH or FSEG) requirements.
-         *
-         * Must appear exactly once for each file, and before
-         * this file is referenced by any other command.
-         */
-        struct {
-            char file[BIOS_LINKER_LOADER_FILESZ];
-            uint32_t align;
-            uint8_t zone;
-        } alloc;
-
-        /*
-         * COMMAND_ADD_POINTER - patch the table (originating from
-         * @dest_file) at @pointer.offset, by adding a pointer to the table
-         * originating from @src_file. 1,2,4 or 8 byte unsigned
-         * addition is used depending on @pointer.size.
-         */
-        struct {
-            char dest_file[BIOS_LINKER_LOADER_FILESZ];
-            char src_file[BIOS_LINKER_LOADER_FILESZ];
-            uint32_t offset;
-            uint8_t size;
-        } pointer;
-
-        /*
-         * COMMAND_ADD_CHECKSUM - calculate checksum of the range specified by
-         * @cksum_start and @cksum_length fields,
-         * and then add the value at @cksum.offset.
-         * Checksum simply sums -X for each byte X in the range
-         * using 8-bit math.
-         */
-        struct {
-            char file[BIOS_LINKER_LOADER_FILESZ];
-            uint32_t offset;
-            uint32_t start;
-            uint32_t length;
-        } cksum;
-
-        /* padding */
-        char pad[124];
-    };
-} QEMU_PACKED;
-typedef struct BiosLinkerLoaderEntry BiosLinkerLoaderEntry;
-
-enum {
-    BIOS_LINKER_LOADER_COMMAND_ALLOCATE     = 0x1,
-    BIOS_LINKER_LOADER_COMMAND_ADD_POINTER  = 0x2,
-    BIOS_LINKER_LOADER_COMMAND_ADD_CHECKSUM = 0x3,
-};
-
-enum {
-    BIOS_LINKER_LOADER_ALLOC_ZONE_HIGH = 0x1,
-    BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG = 0x2,
-};
-
-GArray *bios_linker_loader_init(void)
-{
-    return g_array_new(false, true /* clear */, 1);
-}
-
-/* Free linker wrapper and return the linker array. */
-void *bios_linker_loader_cleanup(GArray *linker)
-{
-    return g_array_free(linker, false);
-}
-
-void bios_linker_loader_alloc(GArray *linker,
-                              const char *file,
-                              uint32_t alloc_align,
-                              bool alloc_fseg)
-{
-    BiosLinkerLoaderEntry entry;
-
-    memset(&entry, 0, sizeof entry);
-    strncpy(entry.alloc.file, file, sizeof entry.alloc.file - 1);
-    entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ALLOCATE);
-    entry.alloc.align = cpu_to_le32(alloc_align);
-    entry.alloc.zone = cpu_to_le32(alloc_fseg ?
-                                    BIOS_LINKER_LOADER_ALLOC_ZONE_FSEG :
-                                    BIOS_LINKER_LOADER_ALLOC_ZONE_HIGH);
-
-    /* Alloc entries must come first, so prepend them */
-    g_array_prepend_vals(linker, &entry, sizeof entry);
-}
-
-void bios_linker_loader_add_checksum(GArray *linker, const char *file,
-                                     void *table,
-                                     void *start, unsigned size,
-                                     uint8_t *checksum)
-{
-    BiosLinkerLoaderEntry entry;
-
-    memset(&entry, 0, sizeof entry);
-    strncpy(entry.cksum.file, file, sizeof entry.cksum.file - 1);
-    entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ADD_CHECKSUM);
-    entry.cksum.offset = cpu_to_le32(checksum - (uint8_t *)table);
-    entry.cksum.start = cpu_to_le32((uint8_t *)start - (uint8_t *)table);
-    entry.cksum.length = cpu_to_le32(size);
-
-    g_array_append_vals(linker, &entry, sizeof entry);
-}
-
-void bios_linker_loader_add_pointer(GArray *linker,
-                                    const char *dest_file,
-                                    const char *src_file,
-                                    GArray *table, void *pointer,
-                                    uint8_t pointer_size)
-{
-    BiosLinkerLoaderEntry entry;
-
-    memset(&entry, 0, sizeof entry);
-    strncpy(entry.pointer.dest_file, dest_file,
-            sizeof entry.pointer.dest_file - 1);
-    strncpy(entry.pointer.src_file, src_file,
-            sizeof entry.pointer.src_file - 1);
-    entry.command = cpu_to_le32(BIOS_LINKER_LOADER_COMMAND_ADD_POINTER);
-    entry.pointer.offset = cpu_to_le32((gchar *)pointer - table->data);
-    entry.pointer.size = pointer_size;
-    assert(pointer_size == 1 || pointer_size == 2 ||
-           pointer_size == 4 || pointer_size == 8);
-
-    g_array_append_vals(linker, &entry, sizeof entry);
-}
diff --git a/hw/i386/bios-linker-loader.h b/hw/i386/bios-linker-loader.h
deleted file mode 100644
index 498c0af..0000000
--- a/hw/i386/bios-linker-loader.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef BIOS_LINKER_LOADER_H
-#define BIOS_LINKER_LOADER_H
-
-#include <glib.h>
-#include <stdbool.h>
-#include <inttypes.h>
-
-GArray *bios_linker_loader_init(void);
-
-void bios_linker_loader_alloc(GArray *linker,
-                              const char *file,
-                              uint32_t alloc_align,
-                              bool alloc_fseg);
-
-void bios_linker_loader_add_checksum(GArray *linker, const char *file,
-                                     void *table,
-                                     void *start, unsigned size,
-                                     uint8_t *checksum);
-
-void bios_linker_loader_add_pointer(GArray *linker,
-                                    const char *dest_file,
-                                    const char *src_file,
-                                    GArray *table, void *pointer,
-                                    uint8_t pointer_size);
-
-void *bios_linker_loader_cleanup(GArray *linker);
-#endif
diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
index 0a4282a..28c31a2 100644
--- a/hw/i386/intel_iommu.c
+++ b/hw/i386/intel_iommu.c
@@ -19,15 +19,24 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 #include "intel_iommu_internal.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/i386/pc.h"
+#include "hw/boards.h"
+#include "hw/i386/x86-iommu.h"
+#include "hw/pci-host/q35.h"
+#include "sysemu/kvm.h"
 
 /*#define DEBUG_INTEL_IOMMU*/
 #ifdef DEBUG_INTEL_IOMMU
 enum {
     DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
-    DEBUG_CACHE,
+    DEBUG_CACHE, DEBUG_IR,
 };
 #define VTD_DBGBIT(x)   (1 << DEBUG_##x)
 static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
@@ -151,14 +160,27 @@
     return entry->domain_id == domain_id;
 }
 
+/* The shift of an addr for a certain level of paging structure */
+static inline uint32_t vtd_slpt_level_shift(uint32_t level)
+{
+    return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
+}
+
+static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
+{
+    return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
+}
+
 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
                                         gpointer user_data)
 {
     VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
     VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
-    uint64_t gfn = info->gfn & info->mask;
+    uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
+    uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
     return (entry->domain_id == info->domain_id) &&
-            ((entry->gfn & info->mask) == gfn);
+            (((entry->gfn & info->mask) == gfn) ||
+             (entry->gfn == gfn_tlb));
 }
 
 /* Reset all the gen of VTDAddressSpace to zero and set the gen of
@@ -166,19 +188,17 @@
  */
 static void vtd_reset_context_cache(IntelIOMMUState *s)
 {
-    VTDAddressSpace **pvtd_as;
     VTDAddressSpace *vtd_as;
-    uint32_t bus_it;
+    VTDBus *vtd_bus;
+    GHashTableIter bus_it;
     uint32_t devfn_it;
 
+    g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
+
     VTD_DPRINTF(CACHE, "global context_cache_gen=1");
-    for (bus_it = 0; bus_it < VTD_PCI_BUS_MAX; ++bus_it) {
-        pvtd_as = s->address_spaces[bus_it];
-        if (!pvtd_as) {
-            continue;
-        }
-        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
-            vtd_as = pvtd_as[devfn_it];
+    while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
+        for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
+            vtd_as = vtd_bus->dev_as[devfn_it];
             if (!vtd_as) {
                 continue;
             }
@@ -194,24 +214,46 @@
     g_hash_table_remove_all(s->iotlb);
 }
 
+static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint8_t source_id,
+                                  uint32_t level)
+{
+    return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
+           ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
+}
+
+static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
+{
+    return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
+}
+
 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
                                        hwaddr addr)
 {
+    VTDIOTLBEntry *entry;
     uint64_t key;
+    int level;
 
-    key = (addr >> VTD_PAGE_SHIFT_4K) |
-           ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT);
-    return g_hash_table_lookup(s->iotlb, &key);
+    for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
+        key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
+                                source_id, level);
+        entry = g_hash_table_lookup(s->iotlb, &key);
+        if (entry) {
+            goto out;
+        }
+    }
 
+out:
+    return entry;
 }
 
 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
                              uint16_t domain_id, hwaddr addr, uint64_t slpte,
-                             bool read_flags, bool write_flags)
+                             bool read_flags, bool write_flags,
+                             uint32_t level)
 {
     VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
     uint64_t *key = g_malloc(sizeof(*key));
-    uint64_t gfn = addr >> VTD_PAGE_SHIFT_4K;
+    uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
 
     VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
                 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte,
@@ -226,7 +268,8 @@
     entry->slpte = slpte;
     entry->read_flags = read_flags;
     entry->write_flags = write_flags;
-    *key = gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT);
+    entry->mask = vtd_slpt_level_page_mask(level);
+    *key = vtd_get_iotlb_key(gfn, source_id, level);
     g_hash_table_replace(s->iotlb, key, entry);
 }
 
@@ -246,7 +289,8 @@
     data = vtd_get_long_raw(s, mesg_data_reg);
 
     VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data);
-    stl_le_phys(&address_space_memory, addr, data);
+    address_space_stl_le(&address_space_memory, addr, data,
+                         MEMTXATTRS_UNSPECIFIED, NULL);
 }
 
 /* Generate a fault event to software via MSI if conditions are met.
@@ -500,12 +544,6 @@
     return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
 }
 
-/* The shift of an addr for a certain level of paging structure */
-static inline uint32_t vtd_slpt_level_shift(uint32_t level)
-{
-    return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
-}
-
 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
 {
     return slpte & VTD_SL_PT_BASE_ADDR_MASK;
@@ -745,19 +783,23 @@
 
 /* Map dev to context-entry then do a paging-structures walk to do a iommu
  * translation.
+ *
+ * Called from RCU critical section.
+ *
  * @bus_num: The bus number
  * @devfn: The devfn, which is the  combined of device and function number
  * @is_write: The access is a write operation
  * @entry: IOMMUTLBEntry that contain the addr to be translated and result
  */
-static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, uint8_t bus_num,
+static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
                                    uint8_t devfn, hwaddr addr, bool is_write,
                                    IOMMUTLBEntry *entry)
 {
     IntelIOMMUState *s = vtd_as->iommu_state;
     VTDContextEntry ce;
+    uint8_t bus_num = pci_bus_num(bus);
     VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
-    uint64_t slpte;
+    uint64_t slpte, page_mask;
     uint32_t level;
     uint16_t source_id = vtd_make_source_id(bus_num, devfn);
     int ret_fr;
@@ -797,6 +839,7 @@
         slpte = iotlb_entry->slpte;
         reads = iotlb_entry->read_flags;
         writes = iotlb_entry->write_flags;
+        page_mask = iotlb_entry->mask;
         goto out;
     }
     /* Try to fetch context-entry from cache first */
@@ -843,12 +886,13 @@
         return;
     }
 
+    page_mask = vtd_slpt_level_page_mask(level);
     vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
-                     reads, writes);
+                     reads, writes, level);
 out:
-    entry->iova = addr & VTD_PAGE_MASK_4K;
-    entry->translated_addr = vtd_get_slpte_addr(slpte) & VTD_PAGE_MASK_4K;
-    entry->addr_mask = ~VTD_PAGE_MASK_4K;
+    entry->iova = addr & page_mask;
+    entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask;
+    entry->addr_mask = ~page_mask;
     entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
 }
 
@@ -862,6 +906,27 @@
                 (s->root_extended ? "(extended)" : ""));
 }
 
+static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
+                               uint32_t index, uint32_t mask)
+{
+    x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
+}
+
+static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
+{
+    uint64_t value = 0;
+    value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
+    s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
+    s->intr_root = value & VTD_IRTA_ADDR_MASK;
+    s->intr_eime = value & VTD_IRTA_EIME;
+
+    /* Notify global invalidation */
+    vtd_iec_notify_all(s, true, 0, 0);
+
+    VTD_DPRINTF(CSR, "int remap table addr 0x%"PRIx64 " size %"PRIu32,
+                s->intr_root, s->intr_size);
+}
+
 static void vtd_context_global_invalidate(IntelIOMMUState *s)
 {
     s->context_cache_gen++;
@@ -870,6 +935,29 @@
     }
 }
 
+
+/* Find the VTD address space currently associated with a given bus number,
+ */
+static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
+{
+    VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
+    if (!vtd_bus) {
+        /* Iterate over the registered buses to find the one
+         * which currently hold this bus number, and update the bus_num lookup table:
+         */
+        GHashTableIter iter;
+
+        g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
+        while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
+            if (pci_bus_num(vtd_bus->bus) == bus_num) {
+                s->vtd_as_by_bus_num[bus_num] = vtd_bus;
+                return vtd_bus;
+            }
+        }
+    }
+    return vtd_bus;
+}
+
 /* Do a context-cache device-selective invalidation.
  * @func_mask: FM field after shifting
  */
@@ -878,7 +966,7 @@
                                           uint16_t func_mask)
 {
     uint16_t mask;
-    VTDAddressSpace **pvtd_as;
+    VTDBus *vtd_bus;
     VTDAddressSpace *vtd_as;
     uint16_t devfn;
     uint16_t devfn_it;
@@ -899,11 +987,11 @@
     }
     VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
                     " mask %"PRIu16, source_id, mask);
-    pvtd_as = s->address_spaces[VTD_SID_TO_BUS(source_id)];
-    if (pvtd_as) {
+    vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
+    if (vtd_bus) {
         devfn = VTD_SID_TO_DEVFN(source_id);
-        for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) {
-            vtd_as = pvtd_as[devfn_it];
+        for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
+            vtd_as = vtd_bus->dev_as[devfn_it];
             if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
                 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
                             devfn_it);
@@ -963,7 +1051,7 @@
 
     assert(am <= VTD_MAMV);
     info.domain_id = domain_id;
-    info.gfn = addr >> VTD_PAGE_SHIFT_4K;
+    info.addr = addr;
     info.mask = ~((1 << am) - 1);
     g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
 }
@@ -1077,6 +1165,16 @@
     vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
 }
 
+/* Set Interrupt Remap Table Pointer */
+static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
+{
+    VTD_DPRINTF(CSR, "set Interrupt Remap Table Pointer");
+
+    vtd_interrupt_remap_table_setup(s);
+    /* Ok - report back to driver */
+    vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
+}
+
 /* Handle Translation Enable/Disable */
 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
 {
@@ -1096,6 +1194,22 @@
     }
 }
 
+/* Handle Interrupt Remap Enable/Disable */
+static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
+{
+    VTD_DPRINTF(CSR, "Interrupt Remap Enable %s", (en ? "on" : "off"));
+
+    if (en) {
+        s->intr_enabled = true;
+        /* Ok - report back to driver */
+        vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
+    } else {
+        s->intr_enabled = false;
+        /* Ok - report back to driver */
+        vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
+    }
+}
+
 /* Handle write to Global Command Register */
 static void vtd_handle_gcmd_write(IntelIOMMUState *s)
 {
@@ -1116,6 +1230,14 @@
         /* Queued Invalidation Enable */
         vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
     }
+    if (val & VTD_GCMD_SIRTP) {
+        /* Set/update the interrupt remapping root-table pointer */
+        vtd_handle_gcmd_sirtp(s);
+    }
+    if (changed & VTD_GCMD_IRE) {
+        /* Interrupt remap enable/disable */
+        vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
+    }
 }
 
 /* Handle write to Context Command Register */
@@ -1301,6 +1423,21 @@
     return true;
 }
 
+static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
+                                     VTDInvDesc *inv_desc)
+{
+    VTD_DPRINTF(INV, "inv ir glob %d index %d mask %d",
+                inv_desc->iec.granularity,
+                inv_desc->iec.index,
+                inv_desc->iec.index_mask);
+
+    vtd_iec_notify_all(s, !inv_desc->iec.granularity,
+                       inv_desc->iec.index,
+                       inv_desc->iec.index_mask);
+
+    return true;
+}
+
 static bool vtd_process_inv_desc(IntelIOMMUState *s)
 {
     VTDInvDesc inv_desc;
@@ -1340,6 +1477,15 @@
         }
         break;
 
+    case VTD_INV_DESC_IEC:
+        VTD_DPRINTF(INV, "Invalidation Interrupt Entry Cache "
+                    "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
+                    inv_desc.hi, inv_desc.lo);
+        if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
+            return false;
+        }
+        break;
+
     default:
         VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type "
                     "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8,
@@ -1768,6 +1914,23 @@
         vtd_update_fsts_ppf(s);
         break;
 
+    case DMAR_IRTA_REG:
+        VTD_DPRINTF(IR, "DMAR_IRTA_REG write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        if (size == 4) {
+            vtd_set_long(s, addr, val);
+        } else {
+            vtd_set_quad(s, addr, val);
+        }
+        break;
+
+    case DMAR_IRTA_REG_HI:
+        VTD_DPRINTF(IR, "DMAR_IRTA_REG_HI write addr 0x%"PRIx64
+                    ", size %d, val 0x%"PRIx64, addr, size, val);
+        assert(size == 4);
+        vtd_set_long(s, addr, val);
+        break;
+
     default:
         VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64
                     ", size %d, val 0x%"PRIx64, addr, size, val);
@@ -1801,16 +1964,26 @@
         return ret;
     }
 
-    vtd_do_iommu_translate(vtd_as, vtd_as->bus_num, vtd_as->devfn, addr,
+    vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
                            is_write, &ret);
     VTD_DPRINTF(MMU,
                 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
-                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, vtd_as->bus_num,
+                " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
                 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
                 vtd_as->devfn, addr, ret.translated_addr);
     return ret;
 }
 
+static void vtd_iommu_notify_started(MemoryRegion *iommu)
+{
+    VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
+
+    hw_error("Device at bus %s addr %02x.%d requires iommu notifier which "
+             "is currently not supported by intel-iommu emulation",
+             vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn),
+             PCI_FUNC(vtd_as->devfn));
+}
+
 static const VMStateDescription vtd_vmstate = {
     .name = "iommu-intel",
     .unmigratable = 1,
@@ -1835,17 +2008,347 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
+/* Read IRTE entry with specific index */
+static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
+                        VTD_IR_TableEntry *entry, uint16_t sid)
+{
+    static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
+        {0xffff, 0xfffb, 0xfff9, 0xfff8};
+    dma_addr_t addr = 0x00;
+    uint16_t mask, source_id;
+    uint8_t bus, bus_max, bus_min;
+
+    addr = iommu->intr_root + index * sizeof(*entry);
+    if (dma_memory_read(&address_space_memory, addr, entry,
+                        sizeof(*entry))) {
+        VTD_DPRINTF(GENERAL, "error: fail to access IR root at 0x%"PRIx64
+                    " + %"PRIu16, iommu->intr_root, index);
+        return -VTD_FR_IR_ROOT_INVAL;
+    }
+
+    if (!entry->irte.present) {
+        VTD_DPRINTF(GENERAL, "error: present flag not set in IRTE"
+                    " entry index %u value 0x%"PRIx64 " 0x%"PRIx64,
+                    index, le64_to_cpu(entry->data[1]),
+                    le64_to_cpu(entry->data[0]));
+        return -VTD_FR_IR_ENTRY_P;
+    }
+
+    if (entry->irte.__reserved_0 || entry->irte.__reserved_1 ||
+        entry->irte.__reserved_2) {
+        VTD_DPRINTF(GENERAL, "error: IRTE entry index %"PRIu16
+                    " reserved fields non-zero: 0x%"PRIx64 " 0x%"PRIx64,
+                    index, le64_to_cpu(entry->data[1]),
+                    le64_to_cpu(entry->data[0]));
+        return -VTD_FR_IR_IRTE_RSVD;
+    }
+
+    if (sid != X86_IOMMU_SID_INVALID) {
+        /* Validate IRTE SID */
+        source_id = le32_to_cpu(entry->irte.source_id);
+        switch (entry->irte.sid_vtype) {
+        case VTD_SVT_NONE:
+            VTD_DPRINTF(IR, "No SID validation for IRTE index %d", index);
+            break;
+
+        case VTD_SVT_ALL:
+            mask = vtd_svt_mask[entry->irte.sid_q];
+            if ((source_id & mask) != (sid & mask)) {
+                VTD_DPRINTF(GENERAL, "SID validation for IRTE index "
+                            "%d failed (reqid 0x%04x sid 0x%04x)", index,
+                            sid, source_id);
+                return -VTD_FR_IR_SID_ERR;
+            }
+            break;
+
+        case VTD_SVT_BUS:
+            bus_max = source_id >> 8;
+            bus_min = source_id & 0xff;
+            bus = sid >> 8;
+            if (bus > bus_max || bus < bus_min) {
+                VTD_DPRINTF(GENERAL, "SID validation for IRTE index %d "
+                            "failed (bus %d outside %d-%d)", index, bus,
+                            bus_min, bus_max);
+                return -VTD_FR_IR_SID_ERR;
+            }
+            break;
+
+        default:
+            VTD_DPRINTF(GENERAL, "Invalid SVT bits (0x%x) in IRTE index "
+                        "%d", entry->irte.sid_vtype, index);
+            /* Take this as verification failure. */
+            return -VTD_FR_IR_SID_ERR;
+            break;
+        }
+    }
+
+    return 0;
+}
+
+/* Fetch IRQ information of specific IR index */
+static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
+                             VTDIrq *irq, uint16_t sid)
+{
+    VTD_IR_TableEntry irte = {};
+    int ret = 0;
+
+    ret = vtd_irte_get(iommu, index, &irte, sid);
+    if (ret) {
+        return ret;
+    }
+
+    irq->trigger_mode = irte.irte.trigger_mode;
+    irq->vector = irte.irte.vector;
+    irq->delivery_mode = irte.irte.delivery_mode;
+    irq->dest = le32_to_cpu(irte.irte.dest_id);
+    if (!iommu->intr_eime) {
+#define  VTD_IR_APIC_DEST_MASK         (0xff00ULL)
+#define  VTD_IR_APIC_DEST_SHIFT        (8)
+        irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
+            VTD_IR_APIC_DEST_SHIFT;
+    }
+    irq->dest_mode = irte.irte.dest_mode;
+    irq->redir_hint = irte.irte.redir_hint;
+
+    VTD_DPRINTF(IR, "remapping interrupt index %d: trig:%u,vec:%u,"
+                "deliver:%u,dest:%u,dest_mode:%u", index,
+                irq->trigger_mode, irq->vector, irq->delivery_mode,
+                irq->dest, irq->dest_mode);
+
+    return 0;
+}
+
+/* Generate one MSI message from VTDIrq info */
+static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out)
+{
+    VTD_MSIMessage msg = {};
+
+    /* Generate address bits */
+    msg.dest_mode = irq->dest_mode;
+    msg.redir_hint = irq->redir_hint;
+    msg.dest = irq->dest;
+    msg.__addr_head = cpu_to_le32(0xfee);
+    /* Keep this from original MSI address bits */
+    msg.__not_used = irq->msi_addr_last_bits;
+
+    /* Generate data bits */
+    msg.vector = irq->vector;
+    msg.delivery_mode = irq->delivery_mode;
+    msg.level = 1;
+    msg.trigger_mode = irq->trigger_mode;
+
+    msg_out->address = msg.msi_addr;
+    msg_out->data = msg.msi_data;
+}
+
+/* Interrupt remapping for MSI/MSI-X entry */
+static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
+                                   MSIMessage *origin,
+                                   MSIMessage *translated,
+                                   uint16_t sid)
+{
+    int ret = 0;
+    VTD_IR_MSIAddress addr;
+    uint16_t index;
+    VTDIrq irq = {};
+
+    assert(origin && translated);
+
+    if (!iommu || !iommu->intr_enabled) {
+        goto do_not_translate;
+    }
+
+    if (origin->address & VTD_MSI_ADDR_HI_MASK) {
+        VTD_DPRINTF(GENERAL, "error: MSI addr high 32 bits nonzero"
+                    " during interrupt remapping: 0x%"PRIx32,
+                    (uint32_t)((origin->address & VTD_MSI_ADDR_HI_MASK) >> \
+                    VTD_MSI_ADDR_HI_SHIFT));
+        return -VTD_FR_IR_REQ_RSVD;
+    }
+
+    addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
+    if (le16_to_cpu(addr.addr.__head) != 0xfee) {
+        VTD_DPRINTF(GENERAL, "error: MSI addr low 32 bits invalid: "
+                    "0x%"PRIx32, addr.data);
+        return -VTD_FR_IR_REQ_RSVD;
+    }
+
+    /* This is compatible mode. */
+    if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
+        goto do_not_translate;
+    }
+
+    index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l);
+
+#define  VTD_IR_MSI_DATA_SUBHANDLE       (0x0000ffff)
+#define  VTD_IR_MSI_DATA_RESERVED        (0xffff0000)
+
+    if (addr.addr.sub_valid) {
+        /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
+        index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
+    }
+
+    ret = vtd_remap_irq_get(iommu, index, &irq, sid);
+    if (ret) {
+        return ret;
+    }
+
+    if (addr.addr.sub_valid) {
+        VTD_DPRINTF(IR, "received MSI interrupt");
+        if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
+            VTD_DPRINTF(GENERAL, "error: MSI data bits non-zero for "
+                        "interrupt remappable entry: 0x%"PRIx32,
+                        origin->data);
+            return -VTD_FR_IR_REQ_RSVD;
+        }
+    } else {
+        uint8_t vector = origin->data & 0xff;
+        VTD_DPRINTF(IR, "received IOAPIC interrupt");
+        /* IOAPIC entry vector should be aligned with IRTE vector
+         * (see vt-d spec 5.1.5.1). */
+        if (vector != irq.vector) {
+            VTD_DPRINTF(GENERAL, "IOAPIC vector inconsistent: "
+                        "entry: %d, IRTE: %d, index: %d",
+                        vector, irq.vector, index);
+        }
+    }
+
+    /*
+     * We'd better keep the last two bits, assuming that guest OS
+     * might modify it. Keep it does not hurt after all.
+     */
+    irq.msi_addr_last_bits = addr.addr.__not_care;
+
+    /* Translate VTDIrq to MSI message */
+    vtd_generate_msi_message(&irq, translated);
+
+    VTD_DPRINTF(IR, "mapping MSI 0x%"PRIx64":0x%"PRIx32 " -> "
+                "0x%"PRIx64":0x%"PRIx32, origin->address, origin->data,
+                translated->address, translated->data);
+    return 0;
+
+do_not_translate:
+    memcpy(translated, origin, sizeof(*origin));
+    return 0;
+}
+
+static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src,
+                         MSIMessage *dst, uint16_t sid)
+{
+    return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
+                                   src, dst, sid);
+}
+
+static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
+                                   uint64_t *data, unsigned size,
+                                   MemTxAttrs attrs)
+{
+    return MEMTX_OK;
+}
+
+static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
+                                    uint64_t value, unsigned size,
+                                    MemTxAttrs attrs)
+{
+    int ret = 0;
+    MSIMessage from = {}, to = {};
+    uint16_t sid = X86_IOMMU_SID_INVALID;
+
+    from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
+    from.data = (uint32_t) value;
+
+    if (!attrs.unspecified) {
+        /* We have explicit Source ID */
+        sid = attrs.requester_id;
+    }
+
+    ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid);
+    if (ret) {
+        /* TODO: report error */
+        VTD_DPRINTF(GENERAL, "int remap fail for addr 0x%"PRIx64
+                    " data 0x%"PRIx32, from.address, from.data);
+        /* Drop this interrupt */
+        return MEMTX_ERROR;
+    }
+
+    VTD_DPRINTF(IR, "delivering MSI 0x%"PRIx64":0x%"PRIx32
+                " for device sid 0x%04x",
+                to.address, to.data, sid);
+
+    if (dma_memory_write(&address_space_memory, to.address,
+                         &to.data, size)) {
+        VTD_DPRINTF(GENERAL, "error: fail to write 0x%"PRIx64
+                    " value 0x%"PRIx32, to.address, to.data);
+    }
+
+    return MEMTX_OK;
+}
+
+static const MemoryRegionOps vtd_mem_ir_ops = {
+    .read_with_attrs = vtd_mem_ir_read,
+    .write_with_attrs = vtd_mem_ir_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
+{
+    uintptr_t key = (uintptr_t)bus;
+    VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
+    VTDAddressSpace *vtd_dev_as;
+
+    if (!vtd_bus) {
+        /* No corresponding free() */
+        vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
+                            X86_IOMMU_PCI_DEVFN_MAX);
+        vtd_bus->bus = bus;
+        key = (uintptr_t)bus;
+        g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus);
+    }
+
+    vtd_dev_as = vtd_bus->dev_as[devfn];
+
+    if (!vtd_dev_as) {
+        vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
+
+        vtd_dev_as->bus = bus;
+        vtd_dev_as->devfn = (uint8_t)devfn;
+        vtd_dev_as->iommu_state = s;
+        vtd_dev_as->context_cache_entry.context_cache_gen = 0;
+        memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
+                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
+        memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s),
+                              &vtd_mem_ir_ops, s, "intel_iommu_ir",
+                              VTD_INTERRUPT_ADDR_SIZE);
+        memory_region_add_subregion(&vtd_dev_as->iommu, VTD_INTERRUPT_ADDR_FIRST,
+                                    &vtd_dev_as->iommu_ir);
+        address_space_init(&vtd_dev_as->as,
+                           &vtd_dev_as->iommu, "intel_iommu");
+    }
+    return vtd_dev_as;
+}
+
 /* Do the initialization. It will also be called when reset, so pay
  * attention when adding new initialization stuff.
  */
 static void vtd_init(IntelIOMMUState *s)
 {
+    X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
+
     memset(s->csr, 0, DMAR_REG_SIZE);
     memset(s->wmask, 0, DMAR_REG_SIZE);
     memset(s->w1cmask, 0, DMAR_REG_SIZE);
     memset(s->womask, 0, DMAR_REG_SIZE);
 
     s->iommu_ops.translate = vtd_iommu_translate;
+    s->iommu_ops.notify_started = vtd_iommu_notify_started;
     s->root = 0;
     s->root_extended = false;
     s->dmar_enabled = false;
@@ -1857,9 +2360,13 @@
     s->iq_last_desc_type = VTD_INV_DESC_NONE;
     s->next_frcd_reg = 0;
     s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
-             VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI;
+             VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS;
     s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
 
+    if (x86_iommu->intr_supported) {
+        s->ecap |= VTD_ECAP_IR | VTD_ECAP_EIM | VTD_ECAP_MHMV;
+    }
+
     vtd_reset_context_cache(s);
     vtd_reset_iotlb(s);
 
@@ -1909,6 +2416,11 @@
     /* Fault Recording Registers, 128-bit */
     vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
     vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
+
+    /*
+     * Interrupt remapping registers.
+     */
+    vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0);
 }
 
 /* Should not reset address_spaces when reset because devices will still use
@@ -1922,34 +2434,65 @@
     vtd_init(s);
 }
 
+static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
+{
+    IntelIOMMUState *s = opaque;
+    VTDAddressSpace *vtd_as;
+
+    assert(0 <= devfn && devfn <= X86_IOMMU_PCI_DEVFN_MAX);
+
+    vtd_as = vtd_find_add_as(s, bus, devfn);
+    return &vtd_as->as;
+}
+
 static void vtd_realize(DeviceState *dev, Error **errp)
 {
+    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
+    PCIBus *bus = pcms->bus;
     IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
+    X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
 
     VTD_DPRINTF(GENERAL, "");
-    memset(s->address_spaces, 0, sizeof(s->address_spaces));
+    memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
     memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
                           "intel_iommu", DMAR_REG_SIZE);
     sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
     /* No corresponding destroy */
     s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
                                      g_free, g_free);
+    s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
+                                              g_free, g_free);
     vtd_init(s);
+    sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
+    pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
+    /* Pseudo address space under root PCI bus. */
+    pcms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC);
+
+    /* Currently Intel IOMMU IR only support "kernel-irqchip={off|split}" */
+    if (x86_iommu->intr_supported && kvm_irqchip_in_kernel() &&
+        !kvm_irqchip_is_split()) {
+        error_report("Intel Interrupt Remapping cannot work with "
+                     "kernel-irqchip=on, please use 'split|off'.");
+        exit(1);
+    }
 }
 
 static void vtd_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass);
 
     dc->reset = vtd_reset;
-    dc->realize = vtd_realize;
     dc->vmsd = &vtd_vmstate;
     dc->props = vtd_properties;
+    dc->hotpluggable = false;
+    x86_class->realize = vtd_realize;
+    x86_class->int_remap = vtd_int_remap;
 }
 
 static const TypeInfo vtd_info = {
     .name          = TYPE_INTEL_IOMMU_DEVICE,
-    .parent        = TYPE_SYS_BUS_DEVICE,
+    .parent        = TYPE_X86_IOMMU_DEVICE,
     .instance_size = sizeof(IntelIOMMUState),
     .class_init    = vtd_class_init,
 };
diff --git a/hw/i386/intel_iommu_internal.h b/hw/i386/intel_iommu_internal.h
index ba288ab..0829a50 100644
--- a/hw/i386/intel_iommu_internal.h
+++ b/hw/i386/intel_iommu_internal.h
@@ -110,9 +110,12 @@
 /* Interrupt Address Range */
 #define VTD_INTERRUPT_ADDR_FIRST    0xfee00000ULL
 #define VTD_INTERRUPT_ADDR_LAST     0xfeefffffULL
+#define VTD_INTERRUPT_ADDR_SIZE     (VTD_INTERRUPT_ADDR_LAST - \
+                                     VTD_INTERRUPT_ADDR_FIRST + 1)
 
 /* The shift of source_id in the key of IOTLB hash table */
 #define VTD_IOTLB_SID_SHIFT         36
+#define VTD_IOTLB_LVL_SHIFT         44
 #define VTD_IOTLB_MAX_SIZE          1024    /* Max size of the hash table */
 
 /* IOTLB_REG */
@@ -171,10 +174,19 @@
 #define VTD_RTADDR_RTT              (1ULL << 11)
 #define VTD_RTADDR_ADDR_MASK        (VTD_HAW_MASK ^ 0xfffULL)
 
+/* IRTA_REG */
+#define VTD_IRTA_ADDR_MASK          (VTD_HAW_MASK ^ 0xfffULL)
+#define VTD_IRTA_EIME               (1ULL << 11)
+#define VTD_IRTA_SIZE_MASK          (0xfULL)
+
 /* ECAP_REG */
 /* (offset >> 4) << 8 */
 #define VTD_ECAP_IRO                (DMAR_IOTLB_REG_OFFSET << 4)
 #define VTD_ECAP_QI                 (1ULL << 1)
+/* Interrupt Remapping support */
+#define VTD_ECAP_IR                 (1ULL << 3)
+#define VTD_ECAP_EIM                (1ULL << 4)
+#define VTD_ECAP_MHMV               (15ULL << 20)
 
 /* CAP_REG */
 /* (offset >> 4) << 24 */
@@ -185,9 +197,10 @@
 #define VTD_CAP_ND                  (((VTD_DOMAIN_ID_SHIFT - 4) / 2) & 7ULL)
 #define VTD_MGAW                    39  /* Maximum Guest Address Width */
 #define VTD_CAP_MGAW                (((VTD_MGAW - 1) & 0x3fULL) << 16)
-#define VTD_MAMV                    9ULL
+#define VTD_MAMV                    18ULL
 #define VTD_CAP_MAMV                (VTD_MAMV << 48)
 #define VTD_CAP_PSI                 (1ULL << 39)
+#define VTD_CAP_SLLPS               ((1ULL << 34) | (1ULL << 35))
 
 /* Supported Adjusted Guest Address Widths */
 #define VTD_CAP_SAGAW_SHIFT         8
@@ -263,6 +276,19 @@
      * context-entry.
      */
     VTD_FR_CONTEXT_ENTRY_TT,
+
+    /* Interrupt remapping transition faults */
+    VTD_FR_IR_REQ_RSVD = 0x20, /* One or more IR request reserved
+                                * fields set */
+    VTD_FR_IR_INDEX_OVER = 0x21, /* Index value greater than max */
+    VTD_FR_IR_ENTRY_P = 0x22,    /* Present (P) not set in IRTE */
+    VTD_FR_IR_ROOT_INVAL = 0x23, /* IR Root table invalid */
+    VTD_FR_IR_IRTE_RSVD = 0x24,  /* IRTE Rsvd field non-zero with
+                                  * Present flag set */
+    VTD_FR_IR_REQ_COMPAT = 0x25, /* Encountered compatible IR
+                                  * request while disabled */
+    VTD_FR_IR_SID_ERR = 0x26,   /* Invalid Source-ID */
+
     /* This is not a normal fault reason. We use this to indicate some faults
      * that are not referenced by the VT-d specification.
      * Fault event with such reason should not be recorded.
@@ -273,17 +299,35 @@
 
 #define VTD_CONTEXT_CACHE_GEN_MAX       0xffffffffUL
 
-/* Queued Invalidation Descriptor */
-struct VTDInvDesc {
-    uint64_t lo;
-    uint64_t hi;
+/* Interrupt Entry Cache Invalidation Descriptor: VT-d 6.5.2.7. */
+struct VTDInvDescIEC {
+    uint32_t type:4;            /* Should always be 0x4 */
+    uint32_t granularity:1;     /* If set, it's global IR invalidation */
+    uint32_t resved_1:22;
+    uint32_t index_mask:5;      /* 2^N for continuous int invalidation */
+    uint32_t index:16;          /* Start index to invalidate */
+    uint32_t reserved_2:16;
 };
-typedef struct VTDInvDesc VTDInvDesc;
+typedef struct VTDInvDescIEC VTDInvDescIEC;
+
+/* Queued Invalidation Descriptor */
+union VTDInvDesc {
+    struct {
+        uint64_t lo;
+        uint64_t hi;
+    };
+    union {
+        VTDInvDescIEC iec;
+    };
+};
+typedef union VTDInvDesc VTDInvDesc;
 
 /* Masks for struct VTDInvDesc */
 #define VTD_INV_DESC_TYPE               0xf
 #define VTD_INV_DESC_CC                 0x1 /* Context-cache Invalidate Desc */
 #define VTD_INV_DESC_IOTLB              0x2
+#define VTD_INV_DESC_IEC                0x4 /* Interrupt Entry Cache
+                                               Invalidate Descriptor */
 #define VTD_INV_DESC_WAIT               0x5 /* Invalidation Wait Descriptor */
 #define VTD_INV_DESC_NONE               0   /* Not an Invalidate Descriptor */
 
@@ -320,7 +364,7 @@
 /* Information about page-selective IOTLB invalidate */
 struct VTDIOTLBPageInvInfo {
     uint16_t domain_id;
-    uint64_t gfn;
+    uint64_t addr;
     uint8_t mask;
 };
 typedef struct VTDIOTLBPageInvInfo VTDIOTLBPageInvInfo;
diff --git a/hw/i386/kvm/apic.c b/hw/i386/kvm/apic.c
index 271e97f..2bd0de8 100644
--- a/hw/i386/kvm/apic.c
+++ b/hw/i386/kvm/apic.c
@@ -9,6 +9,9 @@
  * This work is licensed under the terms of the GNU GPL version 2.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/pci/msi.h"
 #include "sysemu/kvm.h"
@@ -171,26 +174,35 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
+static void kvm_apic_reset(APICCommonState *s)
+{
+    /* Not used by KVM, which uses the CPU mp_state instead.  */
+    s->wait_for_sipi = 0;
+}
+
 static void kvm_apic_realize(DeviceState *dev, Error **errp)
 {
     APICCommonState *s = APIC_COMMON(dev);
 
-    /* Not used by KVM, which uses the CPU mp_state instead.  */
-    s->wait_for_sipi = 0;
-
-    memory_region_init_io(&s->io_memory, NULL, &kvm_apic_io_ops, s, "kvm-apic-msi",
-                          APIC_SPACE_SIZE);
+    memory_region_init_io(&s->io_memory, OBJECT(s), &kvm_apic_io_ops, s,
+                          "kvm-apic-msi", APIC_SPACE_SIZE);
 
     if (kvm_has_gsi_routing()) {
-        msi_supported = true;
+        msi_nonbroken = true;
     }
 }
 
+static void kvm_apic_unrealize(DeviceState *dev, Error **errp)
+{
+}
+
 static void kvm_apic_class_init(ObjectClass *klass, void *data)
 {
     APICCommonClass *k = APIC_COMMON_CLASS(klass);
 
     k->realize = kvm_apic_realize;
+    k->unrealize = kvm_apic_unrealize;
+    k->reset = kvm_apic_reset;
     k->set_base = kvm_apic_set_base;
     k->set_tpr = kvm_apic_set_tpr;
     k->get_tpr = kvm_apic_get_tpr;
diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c
index 58be2bd..0f75dd3 100644
--- a/hw/i386/kvm/clock.c
+++ b/hw/i386/kvm/clock.c
@@ -13,11 +13,13 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
-#include "sysemu/cpus.h"
+#include "kvm_i386.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -88,7 +90,7 @@
     int ret;
 
     if (running) {
-        struct kvm_clock_data data;
+        struct kvm_clock_data data = {};
         uint64_t time_at_migration = kvmclock_current_nsec(s);
 
         s->clock_valid = false;
@@ -99,7 +101,6 @@
         }
 
         data.clock = s->clock;
-        data.flags = 0;
         ret = kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, &data);
         if (ret < 0) {
             fprintf(stderr, "KVM_SET_CLOCK failed: %s\n", strerror(ret));
@@ -126,21 +127,7 @@
             return;
         }
 
-        cpu_synchronize_all_states();
-        /* In theory, the cpu_synchronize_all_states() call above wouldn't
-         * affect the rest of the code, as the VCPU state inside CPUState
-         * is supposed to always match the VCPU state on the kernel side.
-         *
-         * In practice, calling cpu_synchronize_state() too soon will load the
-         * kernel-side APIC state into X86CPU.apic_state too early, APIC state
-         * won't be reloaded later because CPUState.vcpu_dirty==true, and
-         * outdated APIC state may be migrated to another host.
-         *
-         * The real fix would be to make sure outdated APIC state is read
-         * from the kernel again when necessary. While this is not fixed, we
-         * need the cpu_clean_all_dirty() call below.
-         */
-        cpu_clean_all_dirty();
+        kvm_synchronize_all_tsc();
 
         ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
         if (ret < 0) {
diff --git a/hw/i386/kvm/i8254.c b/hw/i386/kvm/i8254.c
index 472af81..521a584 100644
--- a/hw/i386/kvm/i8254.c
+++ b/hw/i386/kvm/i8254.c
@@ -22,6 +22,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
+#include "qemu/osdep.h"
+#include <linux/kvm.h>
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "hw/timer/i8254.h"
@@ -138,7 +142,7 @@
 static void kvm_pit_put(PITCommonState *pit)
 {
     KVMPITState *s = KVM_PIT(pit);
-    struct kvm_pit_state2 kpit;
+    struct kvm_pit_state2 kpit = {};
     struct kvm_pit_channel_state *kchan;
     struct PITChannelState *sc;
     int i, ret;
diff --git a/hw/i386/kvm/i8259.c b/hw/i386/kvm/i8259.c
index 53e3ca8..2b207de 100644
--- a/hw/i386/kvm/i8259.c
+++ b/hw/i386/kvm/i8259.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL version 2.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/isa/i8259_internal.h"
 #include "hw/i386/apic_internal.h"
 #include "sysemu/kvm.h"
diff --git a/hw/i386/kvm/ioapic.c b/hw/i386/kvm/ioapic.c
index d2a6c4c..8eb2c7a 100644
--- a/hw/i386/kvm/ioapic.c
+++ b/hw/i386/kvm/ioapic.c
@@ -10,6 +10,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "monitor/monitor.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/ioapic_internal.h"
 #include "hw/i386/apic_internal.h"
@@ -110,6 +112,15 @@
     }
 }
 
+void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict)
+{
+    IOAPICCommonState s;
+
+    kvm_ioapic_get(&s);
+
+    ioapic_print_redtbl(mon, &s);
+}
+
 static void kvm_ioapic_reset(DeviceState *dev)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
diff --git a/hw/i386/kvm/pci-assign.c b/hw/i386/kvm/pci-assign.c
index 249c62c..f92e972 100644
--- a/hw/i386/kvm/pci-assign.c
+++ b/hw/i386/kvm/pci-assign.c
@@ -20,12 +20,10 @@
  *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
  *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
  */
-#include <stdio.h>
-#include <unistd.h>
-#include <sys/io.h>
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <sys/stat.h>
+
+#include "qemu/osdep.h"
+#include <linux/kvm.h>
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "qemu/error-report.h"
@@ -37,11 +35,7 @@
 #include "hw/pci/pci.h"
 #include "hw/pci/msi.h"
 #include "kvm_i386.h"
-#ifdef CONFIG_ANDROID
-#include "android/utils/file_io.h"
-#endif  // CONFIG_ANDROID
-
-#define MSIX_PAGE_SIZE 0x1000
+#include "hw/pci/pci-assign.h"
 
 /* From linux/ioport.h */
 #define IORESOURCE_IO       0x00000100  /* Resource type */
@@ -51,17 +45,6 @@
 #define IORESOURCE_PREFETCH 0x00002000  /* No side effects */
 #define IORESOURCE_MEM_64   0x00100000
 
-//#define DEVICE_ASSIGNMENT_DEBUG
-
-#ifdef DEVICE_ASSIGNMENT_DEBUG
-#define DEBUG(fmt, ...)                                       \
-    do {                                                      \
-        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
-    } while (0)
-#else
-#define DEBUG(fmt, ...)
-#endif
-
 typedef struct PCIRegion {
     int type;           /* Memory or port I/O */
     int valid;
@@ -138,12 +121,16 @@
     int *msi_virq;
     MSIXTableEntry *msix_table;
     hwaddr msix_table_addr;
+    uint16_t msix_table_size;
     uint16_t msix_max;
     MemoryRegion mmio;
     char *configfd_name;
     int32_t bootindex;
 } AssignedDevice;
 
+#define TYPE_PCI_ASSIGN "kvm-pci-assign"
+#define PCI_ASSIGN(obj) OBJECT_CHECK(AssignedDevice, (obj), TYPE_PCI_ASSIGN)
+
 static void assigned_dev_update_irq_routing(PCIDevice *dev);
 
 static void assigned_dev_load_option_rom(AssignedDevice *dev);
@@ -260,7 +247,7 @@
 static void assigned_dev_iomem_setup(PCIDevice *pci_dev, int region_num,
                                      pcibus_t e_size)
 {
-    AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *r_dev = PCI_ASSIGN(pci_dev);
     AssignedDevRegion *region = &r_dev->v_addrs[region_num];
     PCIRegion *real_region = &r_dev->real_device.regions[region_num];
 
@@ -292,7 +279,7 @@
 static void assigned_dev_ioport_setup(PCIDevice *pci_dev, int region_num,
                                       pcibus_t size)
 {
-    AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *r_dev = PCI_ASSIGN(pci_dev);
     AssignedDevRegion *region = &r_dev->v_addrs[region_num];
 
     region->e_size = size;
@@ -306,7 +293,7 @@
 
 static uint32_t assigned_dev_pci_read(PCIDevice *d, int pos, int len)
 {
-    AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d);
+    AssignedDevice *pci_dev = PCI_ASSIGN(d);
     uint32_t val;
     ssize_t ret;
     int fd = pci_dev->real_device.config_fd;
@@ -331,7 +318,7 @@
 
 static void assigned_dev_pci_write(PCIDevice *d, int pos, uint32_t val, int len)
 {
-    AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d);
+    AssignedDevice *pci_dev = PCI_ASSIGN(d);
     ssize_t ret;
     int fd = pci_dev->real_device.config_fd;
 
@@ -555,9 +542,8 @@
     snprintf(name, sizeof(name), "%sconfig", dir);
 
     if (pci_dev->configfd_name && *pci_dev->configfd_name) {
-        dev->config_fd = monitor_handle_fd_param2(cur_mon,
-                                                  pci_dev->configfd_name,
-                                                  &local_err);
+        dev->config_fd = monitor_fd_param(cur_mon, pci_dev->configfd_name,
+                                          &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             return;
@@ -782,7 +768,7 @@
         "*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/"
         "pci-stub/bind\n"
         "*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub/remove_id\n"
-        "***",
+        "***\n",
         ns, dev->host.domain, dev->host.bus, dev->host.slot,
         dev->host.function, vendor_id, device_id,
         dev->host.domain, dev->host.bus, dev->host.slot, dev->host.function,
@@ -790,7 +776,7 @@
         dev->host.function, vendor_id, device_id);
 
 fail:
-    return g_strdup("Couldn't find out why.");
+    return g_strdup("Couldn't find out why.\n");
 }
 
 static void assign_device(AssignedDevice *dev, Error **errp)
@@ -824,8 +810,9 @@
             char *cause;
 
             cause = assign_failed_examine(dev);
-            error_setg_errno(errp, -r, "Failed to assign device \"%s\"\n%s",
-                             dev->dev.qdev.id, cause);
+            error_setg_errno(errp, -r, "Failed to assign device \"%s\"",
+                             dev->dev.qdev.id);
+            error_append_hint(errp, "%s", cause);
             g_free(cause);
             break;
         }
@@ -924,11 +911,10 @@
             dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK;
             goto retry;
         }
-        error_setg_errno(errp, -r,
-                         "Failed to assign irq for \"%s\"\n"
-                         "Perhaps you are assigning a device "
-                         "that shares an IRQ with another device?",
+        error_setg_errno(errp, -r, "Failed to assign irq for \"%s\"",
                          dev->dev.qdev.id);
+        error_append_hint(errp, "Perhaps you are assigning a device "
+                          "that shares an IRQ with another device?\n");
         return r;
     }
 
@@ -951,14 +937,13 @@
  */
 static void assigned_dev_update_irq_routing(PCIDevice *dev)
 {
-    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, dev);
+    AssignedDevice *assigned_dev = PCI_ASSIGN(dev);
     Error *err = NULL;
     int r;
 
     r = assign_intx(assigned_dev, &err);
     if (r < 0) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         err = NULL;
         qdev_unplug(&dev->qdev, &err);
         assert(!err);
@@ -967,7 +952,7 @@
 
 static void assigned_dev_update_msi(PCIDevice *pci_dev)
 {
-    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *assigned_dev = PCI_ASSIGN(pci_dev);
     uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap +
                                      PCI_MSI_FLAGS);
     int r;
@@ -990,10 +975,9 @@
     }
 
     if (ctrl_byte & PCI_MSI_FLAGS_ENABLE) {
-        MSIMessage msg = msi_get_message(pci_dev, 0);
         int virq;
 
-        virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        virq = kvm_irqchip_add_msi_route(kvm_state, 0, pci_dev);
         if (virq < 0) {
             perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route");
             return;
@@ -1014,15 +998,14 @@
 
         assign_intx(assigned_dev, &local_err);
         if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
+            error_report_err(local_err);
         }
     }
 }
 
 static void assigned_dev_update_msi_msg(PCIDevice *pci_dev)
 {
-    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *assigned_dev = PCI_ASSIGN(pci_dev);
     uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap +
                                      PCI_MSI_FLAGS);
 
@@ -1032,7 +1015,8 @@
     }
 
     kvm_irqchip_update_msi_route(kvm_state, assigned_dev->msi_virq[0],
-                                 msi_get_message(pci_dev, 0));
+                                 msi_get_message(pci_dev, 0), pci_dev);
+    kvm_irqchip_commit_routes(kvm_state);
 }
 
 static bool assigned_dev_msix_masked(MSIXTableEntry *entry)
@@ -1055,11 +1039,10 @@
 
 static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
 {
-    AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *adev = PCI_ASSIGN(pci_dev);
     uint16_t entries_nr = 0;
     int i, r = 0;
     MSIXTableEntry *entry = adev->msix_table;
-    MSIMessage msg;
 
     /* Get the usable entry number for allocating */
     for (i = 0; i < adev->msix_max; i++, entry++) {
@@ -1096,9 +1079,7 @@
             continue;
         }
 
-        msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32);
-        msg.data = entry->data;
-        r = kvm_irqchip_add_msi_route(kvm_state, msg);
+        r = kvm_irqchip_add_msi_route(kvm_state, i, pci_dev);
         if (r < 0) {
             return r;
         }
@@ -1120,7 +1101,7 @@
 
 static void assigned_dev_update_msix(PCIDevice *pci_dev)
 {
-    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *assigned_dev = PCI_ASSIGN(pci_dev);
     uint16_t ctrl_word = pci_get_word(pci_dev->config + pci_dev->msix_cap +
                                       PCI_MSIX_FLAGS);
     int r;
@@ -1162,8 +1143,7 @@
 
         assign_intx(assigned_dev, &local_err);
         if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
+            error_report_err(local_err);
         }
     }
 }
@@ -1171,7 +1151,7 @@
 static uint32_t assigned_dev_pci_read_config(PCIDevice *pci_dev,
                                              uint32_t address, int len)
 {
-    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *assigned_dev = PCI_ASSIGN(pci_dev);
     uint32_t virt_val = pci_default_read_config(pci_dev, address, len);
     uint32_t real_val, emulate_mask, full_emulation_mask;
 
@@ -1192,7 +1172,7 @@
 static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
                                           uint32_t val, int len)
 {
-    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *assigned_dev = PCI_ASSIGN(pci_dev);
     uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
     uint32_t emulate_mask, full_emulation_mask;
     int ret;
@@ -1252,7 +1232,7 @@
 
 static int assigned_device_pci_cap_init(PCIDevice *pci_dev, Error **errp)
 {
-    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *dev = PCI_ASSIGN(pci_dev);
     PCIRegion *pci_region = dev->real_device.regions;
     int ret, pos;
     Error *local_err = NULL;
@@ -1328,6 +1308,7 @@
         bar_nr = msix_table_entry & PCI_MSIX_FLAGS_BIRMASK;
         msix_table_entry &= ~PCI_MSIX_FLAGS_BIRMASK;
         dev->msix_table_addr = pci_region[bar_nr].base_addr + msix_table_entry;
+        dev->msix_table_size = msix_max * sizeof(MSIXTableEntry);
         dev->msix_max = msix_max;
     }
 
@@ -1499,7 +1480,7 @@
          * error bits, leave the rest. */
         status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS);
         status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN);
-        status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn;
+        status |= pci_get_bdf(pci_dev);
         status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL |
                     PCI_X_STATUS_SPL_ERR);
         pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status);
@@ -1618,10 +1599,12 @@
                 msg.data = entry->data;
 
                 ret = kvm_irqchip_update_msi_route(kvm_state,
-                                                   adev->msi_virq[i], msg);
+                                                   adev->msi_virq[i], msg,
+                                                   pdev);
                 if (ret) {
                     error_report("Error updating irq routing entry (%d)", ret);
                 }
+                kvm_irqchip_commit_routes(kvm_state);
             }
         }
     }
@@ -1650,7 +1633,7 @@
         return;
     }
 
-    memset(dev->msix_table, 0, MSIX_PAGE_SIZE);
+    memset(dev->msix_table, 0, dev->msix_table_size);
 
     for (i = 0, entry = dev->msix_table; i < dev->msix_max; i++, entry++) {
         entry->ctrl = cpu_to_le32(0x1); /* Masked */
@@ -1659,8 +1642,8 @@
 
 static void assigned_dev_register_msix_mmio(AssignedDevice *dev, Error **errp)
 {
-    dev->msix_table = mmap(NULL, MSIX_PAGE_SIZE, PROT_READ|PROT_WRITE,
-                           MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+    dev->msix_table = mmap(NULL, dev->msix_table_size, PROT_READ | PROT_WRITE,
+                           MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
     if (dev->msix_table == MAP_FAILED) {
         error_setg_errno(errp, errno, "failed to allocate msix_table");
         dev->msix_table = NULL;
@@ -1670,7 +1653,7 @@
     assigned_dev_msix_reset(dev);
 
     memory_region_init_io(&dev->mmio, OBJECT(dev), &assigned_dev_msix_mmio_ops,
-                          dev, "assigned-dev-msix", MSIX_PAGE_SIZE);
+                          dev, "assigned-dev-msix", dev->msix_table_size);
 }
 
 static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev)
@@ -1679,7 +1662,7 @@
         return;
     }
 
-    if (munmap(dev->msix_table, MSIX_PAGE_SIZE) == -1) {
+    if (munmap(dev->msix_table, dev->msix_table_size) == -1) {
         error_report("error unmapping msix_table! %s", strerror(errno));
     }
     dev->msix_table = NULL;
@@ -1692,8 +1675,8 @@
 
 static void reset_assigned_device(DeviceState *dev)
 {
-    PCIDevice *pci_dev = DO_UPCAST(PCIDevice, qdev, dev);
-    AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    PCIDevice *pci_dev = PCI_DEVICE(dev);
+    AssignedDevice *adev = PCI_ASSIGN(pci_dev);
     char reset_file[64];
     const char reset[] = "1";
     int fd, ret;
@@ -1746,9 +1729,9 @@
     assigned_dev_pci_write_config(pci_dev, PCI_COMMAND, 0, 1);
 }
 
-static int assigned_initfn(struct PCIDevice *pci_dev)
+static void assigned_realize(struct PCIDevice *pci_dev, Error **errp)
 {
-    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *dev = PCI_ASSIGN(pci_dev);
     uint8_t e_intx;
     int r;
     Error *local_err = NULL;
@@ -1829,7 +1812,7 @@
 
     assigned_dev_load_option_rom(dev);
 
-    return 0;
+    return;
 
 assigned_out:
     deassign_device(dev);
@@ -1839,14 +1822,12 @@
 
 exit_with_error:
     assert(local_err);
-    qerror_report_err(local_err);
-    error_free(local_err);
-    return -1;
+    error_propagate(errp, local_err);
 }
 
 static void assigned_exitfn(struct PCIDevice *pci_dev)
 {
-    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevice *dev = PCI_ASSIGN(pci_dev);
 
     deassign_device(dev);
     free_assigned_device(dev);
@@ -1855,7 +1836,7 @@
 static void assigned_dev_instance_init(Object *obj)
 {
     PCIDevice *pci_dev = PCI_DEVICE(obj);
-    AssignedDevice *d = DO_UPCAST(AssignedDevice, dev, PCI_DEVICE(obj));
+    AssignedDevice *d = PCI_ASSIGN(pci_dev);
 
     device_add_bootindex_property(obj, &d->bootindex,
                                   "bootindex", NULL,
@@ -1877,7 +1858,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init         = assigned_initfn;
+    k->realize      = assigned_realize;
     k->exit         = assigned_exitfn;
     k->config_read  = assigned_dev_pci_read_config;
     k->config_write = assigned_dev_pci_write_config;
@@ -1889,7 +1870,7 @@
 }
 
 static const TypeInfo assign_info = {
-    .name               = "kvm-pci-assign",
+    .name               = TYPE_PCI_ASSIGN,
     .parent             = TYPE_PCI_DEVICE,
     .instance_size      = sizeof(AssignedDevice),
     .class_init         = assign_class_init,
@@ -1903,77 +1884,11 @@
 
 type_init(assign_register_types)
 
-/*
- * Scan the assigned devices for the devices that have an option ROM, and then
- * load the corresponding ROM data to RAM. If an error occurs while loading an
- * option ROM, we just ignore that option ROM and continue with the next one.
- */
 static void assigned_dev_load_option_rom(AssignedDevice *dev)
 {
-    char name[32], rom_file[64];
-    FILE *fp;
-    uint8_t val;
-    struct stat st;
-    void *ptr;
+    int size = 0;
 
-    /* If loading ROM from file, pci handles it */
-    if (dev->dev.romfile || !dev->dev.rom_bar) {
-        return;
-    }
-
-    snprintf(rom_file, sizeof(rom_file),
-             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
-             dev->host.domain, dev->host.bus, dev->host.slot,
-             dev->host.function);
-
-#ifdef CONFIG_ANDROID
-    if (android_stat(rom_file, &st)) {
-#else  // !CONFIG_ANDROID
-    if (stat(rom_file, &st)) {
-#endif  // !CONFIG_ANDROID
-        return;
-    }
-
-    if (access(rom_file, F_OK)) {
-        error_report("pci-assign: Insufficient privileges for %s", rom_file);
-        return;
-    }
-
-    /* Write "1" to the ROM file to enable it */
-    fp = fopen(rom_file, "r+");
-    if (fp == NULL) {
-        return;
-    }
-    val = 1;
-    if (fwrite(&val, 1, 1, fp) != 1) {
-        goto close_rom;
-    }
-    fseek(fp, 0, SEEK_SET);
-
-    snprintf(name, sizeof(name), "%s.rom",
-            object_get_typename(OBJECT(dev)));
-    memory_region_init_ram(&dev->dev.rom, OBJECT(dev), name, st.st_size,
-                           &error_abort);
-    vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev);
-    ptr = memory_region_get_ram_ptr(&dev->dev.rom);
-    memset(ptr, 0xff, st.st_size);
-
-    if (!fread(ptr, 1, st.st_size, fp)) {
-        error_report("pci-assign: Cannot read from host %s", rom_file);
-        error_printf("Device option ROM contents are probably invalid "
-                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
-                     "or load from file with romfile=\n");
-        goto close_rom;
-    }
-
-    pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom);
-    dev->dev.has_rom = true;
-close_rom:
-    /* Write "0" to disable ROM */
-    fseek(fp, 0, SEEK_SET);
-    val = 0;
-    if (!fwrite(&val, 1, 1, fp)) {
-        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
-    }
-    fclose(fp);
+    pci_assign_dev_load_option_rom(&dev->dev, OBJECT(dev), &size,
+                                   dev->host.domain, dev->host.bus,
+                                   dev->host.slot, dev->host.function);
 }
diff --git a/hw/i386/kvmvapic.c b/hw/i386/kvmvapic.c
index c6d34b2..3bf1ddd 100644
--- a/hw/i386/kvmvapic.c
+++ b/hw/i386/kvmvapic.c
@@ -8,6 +8,10 @@
  * (at your option) any later version. See the COPYING file in the
  * top-level directory.
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
@@ -393,10 +397,10 @@
     CPUX86State *env = &cpu->env;
     VAPICHandlers *handlers;
     uint8_t opcode[2];
-    uint32_t imm32;
+    uint32_t imm32 = 0;
     target_ulong current_pc = 0;
     target_ulong current_cs_base = 0;
-    int current_flags = 0;
+    uint32_t current_flags = 0;
 
     if (smp_cpus == 1) {
         handlers = &s->rom_state.up;
@@ -445,9 +449,8 @@
     resume_all_vcpus();
 
     if (!kvm_enabled()) {
-        cs->current_tb = NULL;
         tb_gen_code(cs, current_pc, current_cs_base, current_flags, 1);
-        cpu_resume_from_signal(cs, NULL);
+        cpu_loop_exit_noexc(cs);
     }
 }
 
@@ -634,13 +637,18 @@
 static void vapic_write(void *opaque, hwaddr addr, uint64_t data,
                         unsigned int size)
 {
-    CPUState *cs = current_cpu;
-    X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    hwaddr rom_paddr;
     VAPICROMState *s = opaque;
+    X86CPU *cpu;
+    CPUX86State *env;
+    hwaddr rom_paddr;
 
-    cpu_synchronize_state(cs);
+    if (!current_cpu) {
+        return;
+    }
+
+    cpu_synchronize_state(current_cpu);
+    cpu = X86_CPU(current_cpu);
+    env = &cpu->env;
 
     /*
      * The VAPIC supports two PIO-based hypercalls, both via port 0x7E.
diff --git a/hw/i386/multiboot.c b/hw/i386/multiboot.c
index 985ca1e..387caa6 100644
--- a/hw/i386/multiboot.c
+++ b/hw/i386/multiboot.c
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/nvram/fw_cfg.h"
 #include "multiboot.h"
@@ -54,6 +57,7 @@
     MBI_MODS_COUNT  = 20,
     MBI_MODS_ADDR   = 24,
     MBI_MMAP_ADDR   = 48,
+    MBI_BOOTLOADER  = 64,
 
     MBI_SIZE        = 88,
 
@@ -74,6 +78,7 @@
     MULTIBOOT_FLAGS_CMDLINE     = 1 << 2,
     MULTIBOOT_FLAGS_MODULES     = 1 << 3,
     MULTIBOOT_FLAGS_MMAP        = 1 << 6,
+    MULTIBOOT_FLAGS_BOOTLOADER  = 1 << 9,
 };
 
 typedef struct {
@@ -87,6 +92,8 @@
     hwaddr offset_mbinfo;
     /* offset in buffer for cmdlines in bytes */
     hwaddr offset_cmdlines;
+    /* offset in buffer for bootloader name in bytes */
+    hwaddr offset_bootloader;
     /* offset of modules in bytes */
     hwaddr offset_mods;
     /* available slots for mb modules infos */
@@ -95,6 +102,8 @@
     int mb_mods_count;
 } MultibootState;
 
+const char *bootloader_name = "qemu";
+
 static uint32_t mb_add_cmdline(MultibootState *s, const char *cmdline)
 {
     hwaddr p = s->offset_cmdlines;
@@ -105,6 +114,16 @@
     return s->mb_buf_phys + p;
 }
 
+static uint32_t mb_add_bootloader(MultibootState *s, const char *bootloader)
+{
+    hwaddr p = s->offset_bootloader;
+    char *b = (char *)s->mb_buf + p;
+
+    memcpy(b, bootloader, strlen(bootloader) + 1);
+    s->offset_bootloader += strlen(b) + 1;
+    return s->mb_buf_phys + p;
+}
+
 static void mb_add_mod(MultibootState *s,
                        hwaddr start, hwaddr end,
                        hwaddr cmdline_phys)
@@ -140,6 +159,7 @@
     MultibootState mbs;
     uint8_t bootinfo[MBI_SIZE];
     uint8_t *mb_bootinfo_data;
+    uint32_t cmdline_len;
 
     /* Ok, let's see if it is a multiboot image.
        The header is 12x32bit long, so the latest entry may be 8192 - 48. */
@@ -178,7 +198,8 @@
         }
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               &elf_low, &elf_high, 0, ELF_MACHINE, 0);
+                               &elf_low, &elf_high, 0, I386_ELF_MACHINE,
+                               0, 0);
         if (kernel_size < 0) {
             fprintf(stderr, "Error while loading elf kernel\n");
             exit(1);
@@ -241,25 +262,29 @@
     mbs.mb_buf_size = TARGET_PAGE_ALIGN(mb_kernel_size);
     mbs.offset_mbinfo = mbs.mb_buf_size;
 
-    /* Calculate space for cmdlines and mb_mods */
-    mbs.mb_buf_size += strlen(kernel_filename) + 1;
-    mbs.mb_buf_size += strlen(kernel_cmdline) + 1;
+    /* Calculate space for cmdlines, bootloader name, and mb_mods */
+    cmdline_len = strlen(kernel_filename) + 1;
+    cmdline_len += strlen(kernel_cmdline) + 1;
     if (initrd_filename) {
         const char *r = initrd_filename;
-        mbs.mb_buf_size += strlen(r) + 1;
+        cmdline_len += strlen(r) + 1;
         mbs.mb_mods_avail = 1;
         while (*(r = get_opt_value(NULL, 0, r))) {
            mbs.mb_mods_avail++;
            r++;
         }
-        mbs.mb_buf_size += MB_MOD_SIZE * mbs.mb_mods_avail;
     }
 
+    mbs.mb_buf_size += cmdline_len;
+    mbs.mb_buf_size += MB_MOD_SIZE * mbs.mb_mods_avail;
+    mbs.mb_buf_size += strlen(bootloader_name) + 1;
+
     mbs.mb_buf_size = TARGET_PAGE_ALIGN(mbs.mb_buf_size);
 
-    /* enlarge mb_buf to hold cmdlines and mb-info structs */
-    mbs.mb_buf          = g_realloc(mbs.mb_buf, mbs.mb_buf_size);
-    mbs.offset_cmdlines = mbs.offset_mbinfo + mbs.mb_mods_avail * MB_MOD_SIZE;
+    /* enlarge mb_buf to hold cmdlines, bootloader, mb-info structs */
+    mbs.mb_buf            = g_realloc(mbs.mb_buf, mbs.mb_buf_size);
+    mbs.offset_cmdlines   = mbs.offset_mbinfo + mbs.mb_mods_avail * MB_MOD_SIZE;
+    mbs.offset_bootloader = mbs.offset_cmdlines + cmdline_len;
 
     if (initrd_filename) {
         char *next_initrd, not_last;
@@ -306,6 +331,8 @@
              kernel_filename, kernel_cmdline);
     stl_p(bootinfo + MBI_CMDLINE, mb_add_cmdline(&mbs, kcmdline));
 
+    stl_p(bootinfo + MBI_BOOTLOADER, mb_add_bootloader(&mbs, bootloader_name));
+
     stl_p(bootinfo + MBI_MODS_ADDR,  mbs.mb_buf_phys + mbs.offset_mbinfo);
     stl_p(bootinfo + MBI_MODS_COUNT, mbs.mb_mods_count); /* mods_count */
 
@@ -314,7 +341,8 @@
                                 | MULTIBOOT_FLAGS_BOOT_DEVICE
                                 | MULTIBOOT_FLAGS_CMDLINE
                                 | MULTIBOOT_FLAGS_MODULES
-                                | MULTIBOOT_FLAGS_MMAP);
+                                | MULTIBOOT_FLAGS_MMAP
+                                | MULTIBOOT_FLAGS_BOOTLOADER);
     stl_p(bootinfo + MBI_BOOT_DEVICE, 0x8000ffff); /* XXX: use the -boot switch? */
     stl_p(bootinfo + MBI_MMAP_ADDR,   ADDR_E820_MAP);
 
diff --git a/hw/i386/pc.c b/hw/i386/pc.c
index e3e6c82..fc37220 100644
--- a/hw/i386/pc.c
+++ b/hw/i386/pc.c
@@ -21,17 +21,20 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/char/serial.h"
 #include "hw/i386/apic.h"
+#include "hw/i386/topology.h"
+#include "sysemu/cpus.h"
 #include "hw/block/fdc.h"
 #include "hw/ide.h"
 #include "hw/pci/pci.h"
-#include "monitor/monitor.h"
+#include "hw/pci/pci_bus.h"
 #include "hw/nvram/fw_cfg.h"
 #include "hw/timer/hpet.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "multiboot.h"
@@ -41,7 +44,12 @@
 #include "hw/pci/msi.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/numa.h"
 #include "sysemu/kvm.h"
+#ifdef CONFIG_HAX
+#include "sysemu/hax.h"
+#endif
+#include "sysemu/qtest.h"
 #include "kvm_i386.h"
 #include "hw/xen/xen.h"
 #include "sysemu/block-backend.h"
@@ -52,20 +60,17 @@
 #include "sysemu/arch_init.h"
 #include "qemu/bitmap.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/cpu_hotplug.h"
-#include "hw/cpu/icc_bus.h"
 #include "hw/boards.h"
 #include "hw/pci/pci_host.h"
 #include "acpi-build.h"
 #include "hw/mem/pc-dimm.h"
-#include "trace.h"
 #include "qapi/visitor.h"
 #include "qapi-visit.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/error-messages.h"
-#endif  // CONFIG_ANDROID
+#include "qom/cpu.h"
+#include "hw/nmi.h"
 
 /* debug PC/ISA interrupts */
 //#define DEBUG_IRQ
@@ -77,16 +82,6 @@
 #define DPRINTF(fmt, ...)
 #endif
 
-/* Leave a chunk of memory at the top of RAM for the BIOS ACPI tables
- * (128K) and other BIOS datastructures (less than 4K reported to be used at
- * the moment, 32K should be enough for a while).  */
-static unsigned acpi_data_size = 0x20000 + 0x8000;
-void pc_set_legacy_acpi_data_size(void)
-{
-    acpi_data_size = 0x10000;
-}
-
-#define BIOS_CFG_IOPORT 0x510
 #define FW_CFG_ACPI_TABLES (FW_CFG_ARCH_LOCAL + 0)
 #define FW_CFG_SMBIOS_ENTRIES (FW_CFG_ARCH_LOCAL + 1)
 #define FW_CFG_IRQ0_OVERRIDE (FW_CFG_ARCH_LOCAL + 2)
@@ -163,27 +158,6 @@
     return cpu_get_ticks();
 }
 
-/* SMM support */
-
-static cpu_set_smm_t smm_set;
-static void *smm_arg;
-
-void cpu_smm_register(cpu_set_smm_t callback, void *arg)
-{
-    assert(smm_set == NULL);
-    assert(smm_arg == NULL);
-    smm_set = callback;
-    smm_arg = arg;
-}
-
-void cpu_smm_update(CPUX86State *env)
-{
-    if (smm_set && smm_arg && CPU(x86_env_get_cpu(env)) == first_cpu) {
-        smm_set(!!(env->hflags & HF_SMM_MASK), smm_arg);
-    }
-}
-
-
 /* IRQ handling */
 int cpu_get_pic_interrupt(CPUX86State *env)
 {
@@ -229,24 +203,24 @@
 
 #define REG_EQUIPMENT_BYTE          0x14
 
-static int cmos_get_fd_drive_type(FDriveType fd0)
+int cmos_get_fd_drive_type(FloppyDriveType fd0)
 {
     int val;
 
     switch (fd0) {
-    case FDRIVE_DRV_144:
+    case FLOPPY_DRIVE_TYPE_144:
         /* 1.44 Mb 3"5 drive */
         val = 4;
         break;
-    case FDRIVE_DRV_288:
+    case FLOPPY_DRIVE_TYPE_288:
         /* 2.88 Mb 3"5 drive */
         val = 5;
         break;
-    case FDRIVE_DRV_120:
+    case FLOPPY_DRIVE_TYPE_120:
         /* 1.2 Mb 5"5 drive */
         val = 2;
         break;
-    case FDRIVE_DRV_NONE:
+    case FLOPPY_DRIVE_TYPE_NONE:
     default:
         val = 0;
         break;
@@ -286,7 +260,7 @@
     return 0;
 }
 
-static int set_boot_dev(ISADevice *s, const char *boot_device)
+static void set_boot_dev(ISADevice *s, const char *boot_device, Error **errp)
 {
 #define PC_MAX_BOOT_DEVICES 3
     int nbds, bds[3] = { 0, };
@@ -294,25 +268,61 @@
 
     nbds = strlen(boot_device);
     if (nbds > PC_MAX_BOOT_DEVICES) {
-        error_report("Too many boot devices for PC");
-        return(1);
+        error_setg(errp, "Too many boot devices for PC");
+        return;
     }
     for (i = 0; i < nbds; i++) {
         bds[i] = boot_device2nibble(boot_device[i]);
         if (bds[i] == 0) {
-            error_report("Invalid boot device for PC: '%c'",
-                         boot_device[i]);
-            return(1);
+            error_setg(errp, "Invalid boot device for PC: '%c'",
+                       boot_device[i]);
+            return;
         }
     }
     rtc_set_memory(s, 0x3d, (bds[1] << 4) | bds[0]);
     rtc_set_memory(s, 0x38, (bds[2] << 4) | (fd_bootchk ? 0x0 : 0x1));
-    return(0);
 }
 
-static int pc_boot_set(void *opaque, const char *boot_device)
+static void pc_boot_set(void *opaque, const char *boot_device, Error **errp)
 {
-    return set_boot_dev(opaque, boot_device);
+    set_boot_dev(opaque, boot_device, errp);
+}
+
+static void pc_cmos_init_floppy(ISADevice *rtc_state, ISADevice *floppy)
+{
+    int val, nb, i;
+    FloppyDriveType fd_type[2] = { FLOPPY_DRIVE_TYPE_NONE,
+                                   FLOPPY_DRIVE_TYPE_NONE };
+
+    /* floppy type */
+    if (floppy) {
+        for (i = 0; i < 2; i++) {
+            fd_type[i] = isa_fdc_get_drive_type(floppy, i);
+        }
+    }
+    val = (cmos_get_fd_drive_type(fd_type[0]) << 4) |
+        cmos_get_fd_drive_type(fd_type[1]);
+    rtc_set_memory(rtc_state, 0x10, val);
+
+    val = rtc_get_memory(rtc_state, REG_EQUIPMENT_BYTE);
+    nb = 0;
+    if (fd_type[0] != FLOPPY_DRIVE_TYPE_NONE) {
+        nb++;
+    }
+    if (fd_type[1] != FLOPPY_DRIVE_TYPE_NONE) {
+        nb++;
+    }
+    switch (nb) {
+    case 0:
+        break;
+    case 1:
+        val |= 0x01; /* 1 drive, ready for boot */
+        break;
+    case 2:
+        val |= 0x41; /* 2 drives, ready for boot */
+        break;
+    }
+    rtc_set_memory(rtc_state, REG_EQUIPMENT_BYTE, val);
 }
 
 typedef struct pc_cmos_init_late_arg {
@@ -320,6 +330,66 @@
     BusState *idebus[2];
 } pc_cmos_init_late_arg;
 
+typedef struct check_fdc_state {
+    ISADevice *floppy;
+    bool multiple;
+} CheckFdcState;
+
+static int check_fdc(Object *obj, void *opaque)
+{
+    CheckFdcState *state = opaque;
+    Object *fdc;
+    uint32_t iobase;
+    Error *local_err = NULL;
+
+    fdc = object_dynamic_cast(obj, TYPE_ISA_FDC);
+    if (!fdc) {
+        return 0;
+    }
+
+    iobase = object_property_get_int(obj, "iobase", &local_err);
+    if (local_err || iobase != 0x3f0) {
+        error_free(local_err);
+        return 0;
+    }
+
+    if (state->floppy) {
+        state->multiple = true;
+    } else {
+        state->floppy = ISA_DEVICE(obj);
+    }
+    return 0;
+}
+
+static const char * const fdc_container_path[] = {
+    "/unattached", "/peripheral", "/peripheral-anon"
+};
+
+/*
+ * Locate the FDC at IO address 0x3f0, in order to configure the CMOS registers
+ * and ACPI objects.
+ */
+ISADevice *pc_find_fdc0(void)
+{
+    int i;
+    Object *container;
+    CheckFdcState state = { 0 };
+
+    for (i = 0; i < ARRAY_SIZE(fdc_container_path); i++) {
+        container = container_get(qdev_get_machine(), fdc_container_path[i]);
+        object_child_foreach(container, check_fdc, &state);
+    }
+
+    if (state.multiple) {
+        error_report("warning: multiple floppy disk controllers with "
+                     "iobase=0x3f0 have been found");
+        error_printf("the one being picked for CMOS setup might not reflect "
+                     "your intent\n");
+    }
+
+    return state.floppy;
+}
+
 static void pc_cmos_init_late(void *opaque)
 {
     pc_cmos_init_late_arg *arg = opaque;
@@ -357,29 +427,28 @@
     }
     rtc_set_memory(s, 0x39, val);
 
+    pc_cmos_init_floppy(s, pc_find_fdc0());
+
     qemu_unregister_reset(pc_cmos_init_late, opaque);
 }
 
-void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device, MachineState *machine,
-                  ISADevice *floppy, BusState *idebus0, BusState *idebus1,
+void pc_cmos_init(PCMachineState *pcms,
+                  BusState *idebus0, BusState *idebus1,
                   ISADevice *s)
 {
-    int val, nb, i;
-    FDriveType fd_type[2] = { FDRIVE_DRV_NONE, FDRIVE_DRV_NONE };
+    int val;
     static pc_cmos_init_late_arg arg;
-    PCMachineState *pc_machine = PC_MACHINE(machine);
 
     /* various important CMOS locations needed by PC/Bochs bios */
 
     /* memory size */
     /* base memory (first MiB) */
-    val = MIN(ram_size / 1024, 640);
+    val = MIN(pcms->below_4g_mem_size / 1024, 640);
     rtc_set_memory(s, 0x15, val);
     rtc_set_memory(s, 0x16, val >> 8);
     /* extended memory (next 64MiB) */
-    if (ram_size > 1024 * 1024) {
-        val = (ram_size - 1024 * 1024) / 1024;
+    if (pcms->below_4g_mem_size > 1024 * 1024) {
+        val = (pcms->below_4g_mem_size - 1024 * 1024) / 1024;
     } else {
         val = 0;
     }
@@ -390,8 +459,8 @@
     rtc_set_memory(s, 0x30, val);
     rtc_set_memory(s, 0x31, val >> 8);
     /* memory between 16MiB and 4GiB */
-    if (ram_size > 16 * 1024 * 1024) {
-        val = (ram_size - 16 * 1024 * 1024) / 65536;
+    if (pcms->below_4g_mem_size > 16 * 1024 * 1024) {
+        val = (pcms->below_4g_mem_size - 16 * 1024 * 1024) / 65536;
     } else {
         val = 0;
     }
@@ -400,59 +469,27 @@
     rtc_set_memory(s, 0x34, val);
     rtc_set_memory(s, 0x35, val >> 8);
     /* memory above 4GiB */
-    val = above_4g_mem_size / 65536;
+    val = pcms->above_4g_mem_size / 65536;
     rtc_set_memory(s, 0x5b, val);
     rtc_set_memory(s, 0x5c, val >> 8);
     rtc_set_memory(s, 0x5d, val >> 16);
 
-    /* set the number of CPU */
-    rtc_set_memory(s, 0x5f, smp_cpus - 1);
-
-    object_property_add_link(OBJECT(machine), "rtc_state",
+    object_property_add_link(OBJECT(pcms), "rtc_state",
                              TYPE_ISA_DEVICE,
-                             (Object **)&pc_machine->rtc,
+                             (Object **)&pcms->rtc,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
-    object_property_set_link(OBJECT(machine), OBJECT(s),
+    object_property_set_link(OBJECT(pcms), OBJECT(s),
                              "rtc_state", &error_abort);
 
-    if (set_boot_dev(s, boot_device)) {
-        exit(1);
-    }
-
-    /* floppy type */
-    if (floppy) {
-        for (i = 0; i < 2; i++) {
-            fd_type[i] = isa_fdc_get_drive_type(floppy, i);
-        }
-    }
-    val = (cmos_get_fd_drive_type(fd_type[0]) << 4) |
-        cmos_get_fd_drive_type(fd_type[1]);
-    rtc_set_memory(s, 0x10, val);
+    set_boot_dev(s, MACHINE(pcms)->boot_order, &error_fatal);
 
     val = 0;
-    nb = 0;
-    if (fd_type[0] < FDRIVE_DRV_NONE) {
-        nb++;
-    }
-    if (fd_type[1] < FDRIVE_DRV_NONE) {
-        nb++;
-    }
-    switch (nb) {
-    case 0:
-        break;
-    case 1:
-        val |= 0x01; /* 1 drive, ready for boot */
-        break;
-    case 2:
-        val |= 0x41; /* 2 drives, ready for boot */
-        break;
-    }
     val |= 0x02; /* FPU is there */
     val |= 0x04; /* PS/2 mouse installed */
     rtc_set_memory(s, REG_EQUIPMENT_BYTE, val);
 
-    /* hard drives */
+    /* hard drives and FDC */
     arg.rtc_state = s;
     arg.idebus[0] = idebus0;
     arg.idebus[1] = idebus1;
@@ -468,7 +505,7 @@
 
     MemoryRegion io;
     uint8_t outport;
-    qemu_irq *a20_out;
+    qemu_irq a20_out;
 } Port92State;
 
 static void port92_write(void *opaque, hwaddr addr, uint64_t val,
@@ -479,7 +516,7 @@
 
     DPRINTF("port92: write 0x%02" PRIx64 "\n", val);
     s->outport = val;
-    qemu_set_irq(*s->a20_out, (val >> 1) & 1);
+    qemu_set_irq(s->a20_out, (val >> 1) & 1);
     if ((val & 1) && !(oldval & 1)) {
         qemu_system_reset_request();
     }
@@ -498,9 +535,7 @@
 
 static void port92_init(ISADevice *dev, qemu_irq *a20_out)
 {
-    Port92State *s = PORT92(dev);
-
-    s->a20_out = a20_out;
+    qdev_connect_gpio_out_named(DEVICE(dev), PORT92_A20_LINE, 0, *a20_out);
 }
 
 static const VMStateDescription vmstate_port92_isa = {
@@ -537,6 +572,8 @@
     memory_region_init_io(&s->io, OBJECT(s), &port92_ops, s, "port92", 1);
 
     s->outport = 0;
+
+    qdev_init_gpio_out_named(DEVICE(obj), &s->a20_out, PORT92_A20_LINE, 1);
 }
 
 static void port92_realizefn(DeviceState *dev, Error **errp)
@@ -606,8 +643,7 @@
     }
 
     /* new "etc/e820" file -- include ram too */
-    e820_table = g_realloc(e820_table,
-                           sizeof(struct e820_entry) * (e820_entries+1));
+    e820_table = g_renew(struct e820_entry, e820_table, e820_entries + 1);
     e820_table[e820_entries].address = cpu_to_le64(address);
     e820_table[e820_entries].length = cpu_to_le64(length);
     e820_table[e820_entries].type = cpu_to_le32(type);
@@ -631,28 +667,84 @@
     return false;
 }
 
-/* Calculates the limit to CPU APIC ID values
- *
- * This function returns the limit for the APIC ID value, so that all
- * CPU APIC IDs are < pc_apic_id_limit().
- *
- * This is used for FW_CFG_MAX_CPUS. See comments on bochs_bios_init().
- */
-static unsigned int pc_apic_id_limit(unsigned int max_cpus)
+/* Enables contiguous-apic-ID mode, for compatibility */
+static bool compat_apic_id_mode;
+
+void enable_compat_apic_id_mode(void)
 {
-    return x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
+    compat_apic_id_mode = true;
 }
 
-static FWCfgState *bochs_bios_init(void)
+/* Calculates initial APIC ID for a specific CPU index
+ *
+ * Currently we need to be able to calculate the APIC ID from the CPU index
+ * alone (without requiring a CPU object), as the QEMU<->Seabios interfaces have
+ * no concept of "CPU index", and the NUMA tables on fw_cfg need the APIC ID of
+ * all CPUs up to max_cpus.
+ */
+static uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index)
 {
-    FWCfgState *fw_cfg;
+    uint32_t correct_id;
+    static bool warned;
+
+    correct_id = x86_apicid_from_cpu_idx(smp_cores, smp_threads, cpu_index);
+    if (compat_apic_id_mode) {
+        if (cpu_index != correct_id && !warned && !qtest_enabled()) {
+            error_report("APIC IDs set in compatibility mode, "
+                         "CPU topology won't match the configuration");
+            warned = true;
+        }
+        return cpu_index;
+    } else {
+        return correct_id;
+    }
+}
+
+static void pc_build_smbios(FWCfgState *fw_cfg)
+{
     uint8_t *smbios_tables, *smbios_anchor;
     size_t smbios_tables_len, smbios_anchor_len;
+    struct smbios_phys_mem_area *mem_array;
+    unsigned i, array_count;
+
+    smbios_tables = smbios_get_table_legacy(&smbios_tables_len);
+    if (smbios_tables) {
+        fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
+                         smbios_tables, smbios_tables_len);
+    }
+
+    /* build the array of physical mem area from e820 table */
+    mem_array = g_malloc0(sizeof(*mem_array) * e820_get_num_entries());
+    for (i = 0, array_count = 0; i < e820_get_num_entries(); i++) {
+        uint64_t addr, len;
+
+        if (e820_get_entry(i, E820_RAM, &addr, &len)) {
+            mem_array[array_count].address = addr;
+            mem_array[array_count].length = len;
+            array_count++;
+        }
+    }
+    smbios_get_tables(mem_array, array_count,
+                      &smbios_tables, &smbios_tables_len,
+                      &smbios_anchor, &smbios_anchor_len);
+    g_free(mem_array);
+
+    if (smbios_anchor) {
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
+                        smbios_tables, smbios_tables_len);
+        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
+                        smbios_anchor, smbios_anchor_len);
+    }
+}
+
+static FWCfgState *bochs_bios_init(AddressSpace *as, PCMachineState *pcms)
+{
+    FWCfgState *fw_cfg;
     uint64_t *numa_fw_cfg;
     int i, j;
-    unsigned int apic_id_limit = pc_apic_id_limit(max_cpus);
 
-    fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
+    fw_cfg = fw_cfg_init_io_dma(FW_CFG_IO_BASE, FW_CFG_IO_BASE + 4, as);
+
     /* FW_CFG_MAX_CPUS is a bit confusing/problematic on x86:
      *
      * SeaBIOS needs FW_CFG_MAX_CPUS for CPU hotplug, but the CPU hotplug
@@ -667,28 +759,12 @@
      * [1] The only kind of "CPU identifier" used between SeaBIOS and QEMU is
      *     the APIC ID, not the "CPU index"
      */
-    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)apic_id_limit);
-    fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)pcms->apic_id_limit);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_bytes(fw_cfg, FW_CFG_ACPI_TABLES,
                      acpi_tables, acpi_tables_len);
     fw_cfg_add_i32(fw_cfg, FW_CFG_IRQ0_OVERRIDE, kvm_allows_irq0_override());
 
-    smbios_tables = smbios_get_table_legacy(&smbios_tables_len);
-    if (smbios_tables) {
-        fw_cfg_add_bytes(fw_cfg, FW_CFG_SMBIOS_ENTRIES,
-                         smbios_tables, smbios_tables_len);
-    }
-
-    smbios_get_tables(&smbios_tables, &smbios_tables_len,
-                      &smbios_anchor, &smbios_anchor_len);
-    if (smbios_anchor) {
-        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-tables",
-                        smbios_tables, smbios_tables_len);
-        fw_cfg_add_file(fw_cfg, "etc/smbios/smbios-anchor",
-                        smbios_anchor, smbios_anchor_len);
-    }
-
     fw_cfg_add_bytes(fw_cfg, FW_CFG_E820_TABLE,
                      &e820_reserve, sizeof(e820_reserve));
     fw_cfg_add_file(fw_cfg, "etc/e820", e820_table,
@@ -699,11 +775,11 @@
      * of nodes, one word for each VCPU->node and one word for each node to
      * hold the amount of memory.
      */
-    numa_fw_cfg = g_new0(uint64_t, 1 + apic_id_limit + nb_numa_nodes);
+    numa_fw_cfg = g_new0(uint64_t, 1 + pcms->apic_id_limit + nb_numa_nodes);
     numa_fw_cfg[0] = cpu_to_le64(nb_numa_nodes);
     for (i = 0; i < max_cpus; i++) {
         unsigned int apic_id = x86_cpu_apic_id_from_index(i);
-        assert(apic_id < apic_id_limit);
+        assert(apic_id < pcms->apic_id_limit);
         for (j = 0; j < nb_numa_nodes; j++) {
             if (test_bit(i, numa_info[j].node_cpu)) {
                 numa_fw_cfg[apic_id + 1] = cpu_to_le64(j);
@@ -712,10 +788,11 @@
         }
     }
     for (i = 0; i < nb_numa_nodes; i++) {
-        numa_fw_cfg[apic_id_limit + 1 + i] = cpu_to_le64(numa_info[i].node_mem);
+        numa_fw_cfg[pcms->apic_id_limit + 1 + i] =
+            cpu_to_le64(numa_info[i].node_mem);
     }
     fw_cfg_add_bytes(fw_cfg, FW_CFG_NUMA, numa_fw_cfg,
-                     (1 + apic_id_limit + nb_numa_nodes) *
+                     (1 + pcms->apic_id_limit + nb_numa_nodes) *
                      sizeof(*numa_fw_cfg));
 
     return fw_cfg;
@@ -735,19 +812,38 @@
     return size;
 }
 
-static void load_linux(FWCfgState *fw_cfg,
-                       const char *kernel_filename,
-                       const char *initrd_filename,
-                       const char *kernel_cmdline,
-                       hwaddr max_ram_size)
+/* setup_data types */
+#define SETUP_NONE     0
+#define SETUP_E820_EXT 1
+#define SETUP_DTB      2
+#define SETUP_PCI      3
+#define SETUP_EFI      4
+
+struct setup_data {
+    uint64_t next;
+    uint32_t type;
+    uint32_t len;
+    uint8_t data[0];
+} __attribute__((packed));
+
+static void load_linux(PCMachineState *pcms,
+                       FWCfgState *fw_cfg)
 {
     uint16_t protocol;
     int setup_size, kernel_size, initrd_size = 0, cmdline_size;
+    int dtb_size, setup_data_offset;
     uint32_t initrd_max;
     uint8_t header[8192], *setup, *kernel, *initrd_data;
     hwaddr real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
     FILE *f;
     char *vmode;
+    MachineState *machine = MACHINE(pcms);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
+    struct setup_data *setup_data;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *dtb_filename = machine->dtb;
+    const char *kernel_cmdline = machine->kernel_cmdline;
 
     /* Align to 16 bytes as a paranoia measure */
     cmdline_size = (strlen(kernel_cmdline)+16) & ~15;
@@ -812,8 +908,8 @@
         initrd_max = 0x37ffffff;
     }
 
-    if (initrd_max >= max_ram_size - acpi_data_size) {
-        initrd_max = max_ram_size - acpi_data_size - 1;
+    if (initrd_max >= pcms->below_4g_mem_size - pcmc->acpi_data_size) {
+        initrd_max = pcms->below_4g_mem_size - pcmc->acpi_data_size - 1;
     }
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_ADDR, cmdline_addr);
@@ -891,6 +987,10 @@
         setup_size = 4;
     }
     setup_size = (setup_size+1)*512;
+    if (setup_size > kernel_size) {
+        fprintf(stderr, "qemu: invalid kernel header\n");
+        exit(1);
+    }
     kernel_size -= setup_size;
 
     setup  = g_malloc(setup_size);
@@ -905,6 +1005,35 @@
         exit(1);
     }
     fclose(f);
+
+    /* append dtb to kernel */
+    if (dtb_filename) {
+        if (protocol < 0x209) {
+            fprintf(stderr, "qemu: Linux kernel too old to load a dtb\n");
+            exit(1);
+        }
+
+        dtb_size = get_image_size(dtb_filename);
+        if (dtb_size <= 0) {
+            fprintf(stderr, "qemu: error reading dtb %s: %s\n",
+                    dtb_filename, strerror(errno));
+            exit(1);
+        }
+
+        setup_data_offset = QEMU_ALIGN_UP(kernel_size, 16);
+        kernel_size = setup_data_offset + sizeof(struct setup_data) + dtb_size;
+        kernel = g_realloc(kernel, kernel_size);
+
+        stq_p(header+0x250, prot_addr + setup_data_offset);
+
+        setup_data = (struct setup_data *)(kernel + setup_data_offset);
+        setup_data->next = 0;
+        setup_data->type = cpu_to_le32(SETUP_DTB);
+        setup_data->len = cpu_to_le32(dtb_size);
+
+        load_image_size(dtb_filename, setup_data->data, dtb_size);
+    }
+
     memcpy(setup, header, MIN(sizeof(header), setup_size));
 
     fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_ADDR, prot_addr);
@@ -915,8 +1044,13 @@
     fw_cfg_add_i32(fw_cfg, FW_CFG_SETUP_SIZE, setup_size);
     fw_cfg_add_bytes(fw_cfg, FW_CFG_SETUP_DATA, setup, setup_size);
 
-    option_rom[nb_option_roms].name = "linuxboot.bin";
-    option_rom[nb_option_roms].bootindex = 0;
+    if (fw_cfg_dma_enabled(fw_cfg)) {
+        option_rom[nb_option_roms].name = "linuxboot_dma.bin";
+        option_rom[nb_option_roms].bootindex = 0;
+    } else {
+        option_rom[nb_option_roms].name = "linuxboot.bin";
+        option_rom[nb_option_roms].bootindex = 0;
+    }
     nb_option_roms++;
 }
 
@@ -956,17 +1090,24 @@
     }
 }
 
-static X86CPU *pc_new_cpu(const char *cpu_model, int64_t apic_id,
-                          DeviceState *icc_bridge, Error **errp)
+static int pc_present_cpus_count(PCMachineState *pcms)
 {
-    X86CPU *cpu;
+    int i, boot_cpus = 0;
+    for (i = 0; i < pcms->possible_cpus->len; i++) {
+        if (pcms->possible_cpus->cpus[i].cpu) {
+            boot_cpus++;
+        }
+    }
+    return boot_cpus;
+}
+
+static X86CPU *pc_new_cpu(const char *typename, int64_t apic_id,
+                          Error **errp)
+{
+    X86CPU *cpu = NULL;
     Error *local_err = NULL;
 
-    cpu = cpu_x86_create(cpu_model, icc_bridge, &local_err);
-    if (local_err != NULL) {
-        error_propagate(errp, local_err);
-        return NULL;
-    }
+    cpu = X86_CPU(object_new(typename));
 
     object_property_set_int(OBJECT(cpu), apic_id, "apic-id", &local_err);
     object_property_set_bool(OBJECT(cpu), true, "realized", &local_err);
@@ -979,30 +1120,19 @@
     return cpu;
 }
 
-static const char *current_cpu_model;
-
 void pc_hot_add_cpu(const int64_t id, Error **errp)
 {
-    DeviceState *icc_bridge;
+    X86CPU *cpu;
+    ObjectClass *oc;
+    PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
     int64_t apic_id = x86_cpu_apic_id_from_index(id);
+    Error *local_err = NULL;
 
     if (id < 0) {
         error_setg(errp, "Invalid CPU id: %" PRIi64, id);
         return;
     }
 
-    if (cpu_exists(apic_id)) {
-        error_setg(errp, "Unable to add CPU: %" PRIi64
-                   ", it already exists", id);
-        return;
-    }
-
-    if (id >= max_cpus) {
-        error_setg(errp, "Unable to add CPU: %" PRIi64
-                   ", max allowed: %d", id, max_cpus - 1);
-        return;
-    }
-
     if (apic_id >= ACPI_CPU_HOTPLUG_ID_LIMIT) {
         error_setg(errp, "Unable to add CPU: %" PRIi64
                    ", resulting APIC ID (%" PRIi64 ") is too large",
@@ -1010,113 +1140,157 @@
         return;
     }
 
-    icc_bridge = DEVICE(object_resolve_path_type("icc-bridge",
-                                                 TYPE_ICC_BRIDGE, NULL));
-    pc_new_cpu(current_cpu_model, apic_id, icc_bridge, errp);
+    assert(pcms->possible_cpus->cpus[0].cpu); /* BSP is always present */
+    oc = OBJECT_CLASS(CPU_GET_CLASS(pcms->possible_cpus->cpus[0].cpu));
+    cpu = pc_new_cpu(object_class_get_name(oc), apic_id, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    object_unref(OBJECT(cpu));
 }
 
-void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge)
+void pc_cpus_init(PCMachineState *pcms)
 {
     int i;
+    CPUClass *cc;
+    ObjectClass *oc;
+    const char *typename;
+    gchar **model_pieces;
     X86CPU *cpu = NULL;
-    Error *error = NULL;
-    unsigned long apic_id_limit;
+    MachineState *machine = MACHINE(pcms);
 
     /* init CPUs */
-    if (cpu_model == NULL) {
+    if (machine->cpu_model == NULL) {
 #ifdef TARGET_X86_64
-        cpu_model = "qemu64";
+        machine->cpu_model = "qemu64";
 #else
-        cpu_model = "qemu32";
+        machine->cpu_model = "qemu32";
 #endif
     }
-    current_cpu_model = cpu_model;
 
-    apic_id_limit = pc_apic_id_limit(max_cpus);
-    if (apic_id_limit > ACPI_CPU_HOTPLUG_ID_LIMIT) {
-        error_report("max_cpus is too large. APIC ID of last CPU is %lu",
-                     apic_id_limit - 1);
+    model_pieces = g_strsplit(machine->cpu_model, ",", 2);
+    if (!model_pieces[0]) {
+        error_report("Invalid/empty CPU model name");
         exit(1);
     }
 
-    for (i = 0; i < smp_cpus; i++) {
-        cpu = pc_new_cpu(cpu_model, x86_cpu_apic_id_from_index(i),
-                         icc_bridge, &error);
-        if (error) {
-            error_report("%s", error_get_pretty(error));
-            error_free(error);
-            exit(1);
-        }
+    oc = cpu_class_by_name(TYPE_X86_CPU, model_pieces[0]);
+    if (oc == NULL) {
+        error_report("Unable to find CPU definition: %s", model_pieces[0]);
+        exit(1);
+    }
+    typename = object_class_get_name(oc);
+    cc = CPU_CLASS(oc);
+    cc->parse_features(typename, model_pieces[1], &error_fatal);
+    g_strfreev(model_pieces);
+
+    /* Calculates the limit to CPU APIC ID values
+     *
+     * Limit for the APIC ID value, so that all
+     * CPU APIC IDs are < pcms->apic_id_limit.
+     *
+     * This is used for FW_CFG_MAX_CPUS. See comments on bochs_bios_init().
+     */
+    pcms->apic_id_limit = x86_cpu_apic_id_from_index(max_cpus - 1) + 1;
+    if (pcms->apic_id_limit > ACPI_CPU_HOTPLUG_ID_LIMIT) {
+        error_report("max_cpus is too large. APIC ID of last CPU is %u",
+                     pcms->apic_id_limit - 1);
+        exit(1);
     }
 
-    /* map APIC MMIO area if CPU has APIC */
-    if (cpu && cpu->apic_state) {
-        /* XXX: what if the base changes? */
-        sysbus_mmio_map_overlap(SYS_BUS_DEVICE(icc_bridge), 0,
-                                APIC_DEFAULT_ADDRESS, 0x1000);
+    pcms->possible_cpus = g_malloc0(sizeof(CPUArchIdList) +
+                                    sizeof(CPUArchId) * max_cpus);
+    for (i = 0; i < max_cpus; i++) {
+        pcms->possible_cpus->cpus[i].arch_id = x86_cpu_apic_id_from_index(i);
+        pcms->possible_cpus->len++;
+        if (i < smp_cpus) {
+            cpu = pc_new_cpu(typename, x86_cpu_apic_id_from_index(i),
+                             &error_fatal);
+            object_unref(OBJECT(cpu));
+        }
     }
 
     /* tell smbios about cpuid version and features */
     smbios_set_cpuid(cpu->env.cpuid_version, cpu->env.features[FEAT_1_EDX]);
 }
 
-/* pci-info ROM file. Little endian format */
-typedef struct PcRomPciInfo {
-    uint64_t w32_min;
-    uint64_t w32_max;
-    uint64_t w64_min;
-    uint64_t w64_max;
-} PcRomPciInfo;
-
-typedef struct PcGuestInfoState {
-    PcGuestInfo info;
-    Notifier machine_done;
-} PcGuestInfoState;
-
-static
-void pc_guest_info_machine_done(Notifier *notifier, void *data)
+static void pc_build_feature_control_file(PCMachineState *pcms)
 {
-    PcGuestInfoState *guest_info_state = container_of(notifier,
-                                                      PcGuestInfoState,
-                                                      machine_done);
-    acpi_setup(&guest_info_state->info);
-}
+    X86CPU *cpu = X86_CPU(pcms->possible_cpus->cpus[0].cpu);
+    CPUX86State *env = &cpu->env;
+    uint32_t unused, ecx, edx;
+    uint64_t feature_control_bits = 0;
+    uint64_t *val;
 
-PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
-                                ram_addr_t above_4g_mem_size)
-{
-    PcGuestInfoState *guest_info_state = g_malloc0(sizeof *guest_info_state);
-    PcGuestInfo *guest_info = &guest_info_state->info;
-    int i, j;
-
-    guest_info->ram_size_below_4g = below_4g_mem_size;
-    guest_info->ram_size = below_4g_mem_size + above_4g_mem_size;
-    guest_info->apic_id_limit = pc_apic_id_limit(max_cpus);
-    guest_info->apic_xrupt_override = kvm_allows_irq0_override();
-    guest_info->numa_nodes = nb_numa_nodes;
-    guest_info->node_mem = g_malloc0(guest_info->numa_nodes *
-                                    sizeof *guest_info->node_mem);
-    for (i = 0; i < nb_numa_nodes; i++) {
-        guest_info->node_mem[i] = numa_info[i].node_mem;
+    cpu_x86_cpuid(env, 1, 0, &unused, &unused, &ecx, &edx);
+    if (ecx & CPUID_EXT_VMX) {
+        feature_control_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
     }
 
-    guest_info->node_cpu = g_malloc0(guest_info->apic_id_limit *
-                                     sizeof *guest_info->node_cpu);
+    if ((edx & (CPUID_EXT2_MCE | CPUID_EXT2_MCA)) ==
+        (CPUID_EXT2_MCE | CPUID_EXT2_MCA) &&
+        (env->mcg_cap & MCG_LMCE_P)) {
+        feature_control_bits |= FEATURE_CONTROL_LMCE;
+    }
 
-    for (i = 0; i < max_cpus; i++) {
-        unsigned int apic_id = x86_cpu_apic_id_from_index(i);
-        assert(apic_id < guest_info->apic_id_limit);
-        for (j = 0; j < nb_numa_nodes; j++) {
-            if (test_bit(i, numa_info[j].node_cpu)) {
-                guest_info->node_cpu[apic_id] = j;
-                break;
+    if (!feature_control_bits) {
+        return;
+    }
+
+    val = g_malloc(sizeof(*val));
+    *val = cpu_to_le64(feature_control_bits | FEATURE_CONTROL_LOCKED);
+    fw_cfg_add_file(pcms->fw_cfg, "etc/msr_feature_control", val, sizeof(*val));
+}
+
+static
+void pc_machine_done(Notifier *notifier, void *data)
+{
+    PCMachineState *pcms = container_of(notifier,
+                                        PCMachineState, machine_done);
+    PCIBus *bus = pcms->bus;
+
+    /* set the number of CPUs */
+    rtc_set_memory(pcms->rtc, 0x5f, pc_present_cpus_count(pcms) - 1);
+
+    if (bus) {
+        int extra_hosts = 0;
+
+        QLIST_FOREACH(bus, &bus->child, sibling) {
+            /* look for expander root buses */
+            if (pci_bus_is_root(bus)) {
+                extra_hosts++;
             }
         }
+        if (extra_hosts && pcms->fw_cfg) {
+            uint64_t *val = g_malloc(sizeof(*val));
+            *val = cpu_to_le64(extra_hosts);
+            fw_cfg_add_file(pcms->fw_cfg,
+                    "etc/extra-pci-roots", val, sizeof(*val));
+        }
     }
 
-    guest_info_state->machine_done.notify = pc_guest_info_machine_done;
-    qemu_add_machine_init_done_notifier(&guest_info_state->machine_done);
-    return guest_info;
+    acpi_setup();
+    if (pcms->fw_cfg) {
+        pc_build_smbios(pcms->fw_cfg);
+        pc_build_feature_control_file(pcms);
+    }
+}
+
+void pc_guest_info_init(PCMachineState *pcms)
+{
+    int i;
+
+    pcms->apic_xrupt_override = kvm_allows_irq0_override();
+    pcms->numa_nodes = nb_numa_nodes;
+    pcms->node_mem = g_malloc0(pcms->numa_nodes *
+                                    sizeof *pcms->node_mem);
+    for (i = 0; i < nb_numa_nodes; i++) {
+        pcms->node_mem[i] = numa_info[i].node_mem;
+    }
+
+    pcms->machine_done.notify = pc_machine_done;
+    qemu_add_machine_init_done_notifier(&pcms->machine_done);
 }
 
 /* setup pci memory address space mapping into system address space */
@@ -1141,67 +1315,55 @@
     if (filename == NULL) {
         fprintf(stderr, "WARNING: failed to find %s\n", default_dsdt);
     } else {
-        char *arg;
-        QemuOpts *opts;
+        QemuOpts *opts = qemu_opts_create(qemu_find_opts("acpi"), NULL, 0,
+                                          &error_abort);
         Error *err = NULL;
 
-        arg = g_strdup_printf("file=%s", filename);
-
-        /* creates a deep copy of "arg" */
-        opts = qemu_opts_parse(qemu_find_opts("acpi"), arg, 0);
-        g_assert(opts != NULL);
+        qemu_opt_set(opts, "file", filename, &error_abort);
 
         acpi_table_add_builtin(opts, &err);
         if (err) {
-            error_report("WARNING: failed to load %s: %s", filename,
-                         error_get_pretty(err));
-            error_free(err);
+            error_reportf_err(err, "WARNING: failed to load %s: ",
+                              filename);
         }
-        g_free(arg);
         g_free(filename);
     }
 }
 
-FWCfgState *xen_load_linux(const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           ram_addr_t below_4g_mem_size,
-                           PcGuestInfo *guest_info)
+void xen_load_linux(PCMachineState *pcms)
 {
     int i;
     FWCfgState *fw_cfg;
 
-    assert(kernel_filename != NULL);
+    assert(MACHINE(pcms)->kernel_filename != NULL);
 
-    fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
+    fw_cfg = fw_cfg_init_io(FW_CFG_IO_BASE);
     rom_set_fw(fw_cfg);
 
-    load_linux(fw_cfg, kernel_filename, initrd_filename,
-               kernel_cmdline, below_4g_mem_size);
+    load_linux(pcms, fw_cfg);
     for (i = 0; i < nb_option_roms; i++) {
         assert(!strcmp(option_rom[i].name, "linuxboot.bin") ||
+               !strcmp(option_rom[i].name, "linuxboot_dma.bin") ||
                !strcmp(option_rom[i].name, "multiboot.bin"));
         rom_add_option(option_rom[i].name, option_rom[i].bootindex);
     }
-    guest_info->fw_cfg = fw_cfg;
-    return fw_cfg;
+    pcms->fw_cfg = fw_cfg;
 }
 
-FWCfgState *pc_memory_init(MachineState *machine,
-                           MemoryRegion *system_memory,
-                           ram_addr_t below_4g_mem_size,
-                           ram_addr_t above_4g_mem_size,
-                           MemoryRegion *rom_memory,
-                           MemoryRegion **ram_memory,
-                           PcGuestInfo *guest_info)
+void pc_memory_init(PCMachineState *pcms,
+                    MemoryRegion *system_memory,
+                    MemoryRegion *rom_memory,
+                    MemoryRegion **ram_memory)
 {
     int linux_boot, i;
     MemoryRegion *ram, *option_rom_mr;
     MemoryRegion *ram_below_4g, *ram_above_4g;
     FWCfgState *fw_cfg;
-    PCMachineState *pcms = PC_MACHINE(machine);
+    MachineState *machine = MACHINE(pcms);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
 
-    assert(machine->ram_size == below_4g_mem_size + above_4g_mem_size);
+    assert(machine->ram_size == pcms->below_4g_mem_size +
+                                pcms->above_4g_mem_size);
 
     linux_boot = (machine->kernel_filename != NULL);
 
@@ -1212,27 +1374,27 @@
     ram = g_malloc(sizeof(*ram));
     memory_region_allocate_system_memory(ram, NULL, "pc.ram",
                                          machine->ram_size);
-#ifdef CONFIG_ANDROID
-    if (android_init_error_occurred()) {
-        return NULL;
+    if (error_during_init) {
+        g_free(ram);
+        return;
     }
-#endif  // CONFIG_ANDROID
     *ram_memory = ram;
     ram_below_4g = g_malloc(sizeof(*ram_below_4g));
     memory_region_init_alias(ram_below_4g, NULL, "ram-below-4g", ram,
-                             0, below_4g_mem_size);
+                             0, pcms->below_4g_mem_size);
     memory_region_add_subregion(system_memory, 0, ram_below_4g);
-    e820_add_entry(0, below_4g_mem_size, E820_RAM);
-    if (above_4g_mem_size > 0) {
+    e820_add_entry(0, pcms->below_4g_mem_size, E820_RAM);
+    if (pcms->above_4g_mem_size > 0) {
         ram_above_4g = g_malloc(sizeof(*ram_above_4g));
         memory_region_init_alias(ram_above_4g, NULL, "ram-above-4g", ram,
-                                 below_4g_mem_size, above_4g_mem_size);
+                                 pcms->below_4g_mem_size,
+                                 pcms->above_4g_mem_size);
         memory_region_add_subregion(system_memory, 0x100000000ULL,
                                     ram_above_4g);
-        e820_add_entry(0x100000000ULL, above_4g_mem_size, E820_RAM);
+        e820_add_entry(0x100000000ULL, pcms->above_4g_mem_size, E820_RAM);
     }
 
-    if (!guest_info->has_reserved_memory &&
+    if (!pcmc->has_reserved_memory &&
         (machine->ram_slots ||
          (machine->maxram_size > machine->ram_size))) {
         MachineClass *mc = MACHINE_GET_CLASS(machine);
@@ -1243,7 +1405,7 @@
     }
 
     /* initialize hotplug memory address space */
-    if (guest_info->has_reserved_memory &&
+    if (pcmc->has_reserved_memory &&
         (machine->ram_size < machine->maxram_size)) {
         ram_addr_t hotplug_mem_size =
             machine->maxram_size - machine->ram_size;
@@ -1254,69 +1416,85 @@
             exit(EXIT_FAILURE);
         }
 
-        pcms->hotplug_memory_base =
-            ROUND_UP(0x100000000ULL + above_4g_mem_size, 1ULL << 30);
+        if (QEMU_ALIGN_UP(machine->maxram_size,
+                          TARGET_PAGE_SIZE) != machine->maxram_size) {
+            error_report("maximum memory size must by aligned to multiple of "
+                         "%d bytes", TARGET_PAGE_SIZE);
+            exit(EXIT_FAILURE);
+        }
 
-        if (pcms->enforce_aligned_dimm) {
+        pcms->hotplug_memory.base =
+            ROUND_UP(0x100000000ULL + pcms->above_4g_mem_size, 1ULL << 30);
+
+        if (pcmc->enforce_aligned_dimm) {
             /* size hotplug region assuming 1G page max alignment per slot */
             hotplug_mem_size += (1ULL << 30) * machine->ram_slots;
         }
 
-        if ((pcms->hotplug_memory_base + hotplug_mem_size) <
+        if ((pcms->hotplug_memory.base + hotplug_mem_size) <
             hotplug_mem_size) {
             error_report("unsupported amount of maximum memory: " RAM_ADDR_FMT,
                          machine->maxram_size);
             exit(EXIT_FAILURE);
         }
 
-        memory_region_init(&pcms->hotplug_memory, OBJECT(pcms),
+        memory_region_init(&pcms->hotplug_memory.mr, OBJECT(pcms),
                            "hotplug-memory", hotplug_mem_size);
-        memory_region_add_subregion(system_memory, pcms->hotplug_memory_base,
-                                    &pcms->hotplug_memory);
+        memory_region_add_subregion(system_memory, pcms->hotplug_memory.base,
+                                    &pcms->hotplug_memory.mr);
     }
 
     /* Initialize PC system firmware */
-    pc_system_firmware_init(rom_memory, guest_info->isapc_ram_fw);
+    pc_system_firmware_init(rom_memory, !pcmc->pci_enabled);
 
     option_rom_mr = g_malloc(sizeof(*option_rom_mr));
     memory_region_init_ram(option_rom_mr, NULL, "pc.rom", PC_ROM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(option_rom_mr);
     memory_region_add_subregion_overlap(rom_memory,
                                         PC_ROM_MIN_VGA,
                                         option_rom_mr,
                                         1);
 
-    fw_cfg = bochs_bios_init();
+    fw_cfg = bochs_bios_init(&address_space_memory, pcms);
+
     rom_set_fw(fw_cfg);
 
-    if (guest_info->has_reserved_memory && pcms->hotplug_memory_base) {
+    if (pcmc->has_reserved_memory && pcms->hotplug_memory.base) {
         uint64_t *val = g_malloc(sizeof(*val));
-        *val = cpu_to_le64(ROUND_UP(pcms->hotplug_memory_base, 0x1ULL << 30));
+        PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
+        uint64_t res_mem_end = pcms->hotplug_memory.base;
+
+        if (!pcmc->broken_reserved_end) {
+            res_mem_end += memory_region_size(&pcms->hotplug_memory.mr);
+        }
+        *val = cpu_to_le64(ROUND_UP(res_mem_end, 0x1ULL << 30));
         fw_cfg_add_file(fw_cfg, "etc/reserved-memory-end", val, sizeof(*val));
     }
 
     if (linux_boot) {
-        load_linux(fw_cfg, machine->kernel_filename, machine->initrd_filename,
-                   machine->kernel_cmdline, below_4g_mem_size);
+        load_linux(pcms, fw_cfg);
     }
 
     for (i = 0; i < nb_option_roms; i++) {
         rom_add_option(option_rom[i].name, option_rom[i].bootindex);
     }
-    guest_info->fw_cfg = fw_cfg;
-    return fw_cfg;
+    pcms->fw_cfg = fw_cfg;
+
+    /* Init default IOAPIC address space */
+    pcms->ioapic_as = &address_space_memory;
 }
 
-qemu_irq *pc_allocate_cpu_irq(void)
+qemu_irq pc_allocate_cpu_irq(void)
 {
-    return qemu_allocate_irqs(pic_irq_request, NULL, 1);
+    return qemu_allocate_irq(pic_irq_request, NULL, 0);
 }
 
 DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus)
 {
     DeviceState *dev = NULL;
 
+    rom_set_order_override(FW_CFG_ORDER_OVERRIDE_VGA);
     if (pci_bus) {
         PCIDevice *pcidev = pci_vga_init(pci_bus);
         dev = pcidev ? &pcidev->qdev : NULL;
@@ -1324,18 +1502,10 @@
         ISADevice *isadev = isa_vga_init(isa_bus);
         dev = isadev ? DEVICE(isadev) : NULL;
     }
+    rom_reset_order_override();
     return dev;
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static const MemoryRegionOps ioport80_io_ops = {
     .write = ioport80_write,
     .read = ioport80_read,
@@ -1358,9 +1528,9 @@
 
 void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           ISADevice **rtc_state,
-                          ISADevice **floppy,
+                          bool create_fdctrl,
                           bool no_vmport,
-                          uint32 hpet_irqs)
+                          uint32_t hpet_irqs)
 {
     int i;
     DriveInfo *fd[MAX_FD];
@@ -1370,7 +1540,6 @@
     qemu_irq rtc_irq = NULL;
     qemu_irq *a20_line;
     ISADevice *i8042, *port92, *vmmouse, *pit = NULL;
-    qemu_irq *cpu_exit_irq;
     MemoryRegion *ioport80_io = g_new(MemoryRegion, 1);
     MemoryRegion *ioportF0_io = g_new(MemoryRegion, 1);
 
@@ -1415,7 +1584,7 @@
     qemu_register_boot_set(pc_boot_set, *rtc_state);
 
     if (!xen_enabled()) {
-        if (kvm_irqchip_in_kernel()) {
+        if (kvm_pit_in_kernel()) {
             pit = kvm_pit_init(isa_bus, 0x40);
         } else {
             pit = pit_init(isa_bus, 0x40, pit_isa_irq, pit_alt_irq);
@@ -1427,17 +1596,8 @@
         pcspk_init(isa_bus, pit);
     }
 
-    for(i = 0; i < MAX_SERIAL_PORTS; i++) {
-        if (serial_hds[i]) {
-            serial_isa_init(isa_bus, i, serial_hds[i]);
-        }
-    }
-
-    for(i = 0; i < MAX_PARALLEL_PORTS; i++) {
-        if (parallel_hds[i]) {
-            parallel_init(isa_bus, i, parallel_hds[i]);
-        }
-    }
+    serial_hds_isa_init(isa_bus, MAX_SERIAL_PORTS);
+    parallel_hds_isa_init(isa_bus, MAX_PARALLEL_PORTS);
 
     a20_line = qemu_allocate_irqs(handle_a20_line_change, first_cpu, 2);
     i8042 = isa_create_simple(isa_bus, "i8042");
@@ -1456,19 +1616,22 @@
     port92 = isa_create_simple(isa_bus, "port92");
     port92_init(port92, &a20_line[1]);
 
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
 
     for(i = 0; i < MAX_FD; i++) {
         fd[i] = drive_get(IF_FLOPPY, 0, i);
+        create_fdctrl |= !!fd[i];
     }
-    *floppy = fdctrl_init_isa(isa_bus, fd);
+    if (create_fdctrl) {
+        fdctrl_init_isa(isa_bus, fd);
+    }
 }
 
 void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus)
 {
     int i;
 
+    rom_set_order_override(FW_CFG_ORDER_OVERRIDE_NIC);
     for (i = 0; i < nb_nics; i++) {
         NICInfo *nd = &nd_table[i];
 
@@ -1478,6 +1641,7 @@
             pci_nic_init_nofail(nd, pci_bus, "e1000", NULL);
         }
     }
+    rom_reset_order_override();
 }
 
 void pc_pci_device_init(PCIBus *pci_bus)
@@ -1497,7 +1661,7 @@
     SysBusDevice *d;
     unsigned int i;
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_ioapic_in_kernel()) {
         dev = qdev_create(NULL, "kvm-ioapic");
     } else {
         dev = qdev_create(NULL, "ioapic");
@@ -1515,198 +1679,288 @@
     }
 }
 
-static void pc_generic_machine_class_init(ObjectClass *oc, void *data)
-{
-    MachineClass *mc = MACHINE_CLASS(oc);
-    QEMUMachine *qm = data;
-
-    mc->family = qm->family;
-    mc->name = qm->name;
-    mc->alias = qm->alias;
-    mc->desc = qm->desc;
-    mc->init = qm->init;
-    mc->reset = qm->reset;
-    mc->hot_add_cpu = qm->hot_add_cpu;
-    mc->kvm_type = qm->kvm_type;
-    mc->block_default_type = qm->block_default_type;
-    mc->units_per_default_bus = qm->units_per_default_bus;
-    mc->max_cpus = qm->max_cpus;
-    mc->no_serial = qm->no_serial;
-    mc->no_parallel = qm->no_parallel;
-    mc->use_virtcon = qm->use_virtcon;
-    mc->use_sclp = qm->use_sclp;
-    mc->no_floppy = qm->no_floppy;
-    mc->no_cdrom = qm->no_cdrom;
-    mc->no_sdcard = qm->no_sdcard;
-    mc->is_default = qm->is_default;
-    mc->default_machine_opts = qm->default_machine_opts;
-    mc->default_boot_order = qm->default_boot_order;
-    mc->default_display = qm->default_display;
-    mc->compat_props = qm->compat_props;
-    mc->hw_version = qm->hw_version;
-}
-
-void qemu_register_pc_machine(QEMUMachine *m)
-{
-    char *name = g_strconcat(m->name, TYPE_MACHINE_SUFFIX, NULL);
-    TypeInfo ti = {
-        .name       = name,
-        .parent     = TYPE_PC_MACHINE,
-        .class_init = pc_generic_machine_class_init,
-        .class_data = (void *)m,
-    };
-
-    type_register(&ti);
-    g_free(name);
-}
-
-static int pc_dimm_count(Object *obj, void *opaque)
-{
-    int *count = opaque;
-
-    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
-        (*count)++;
-    }
-
-    object_child_foreach(obj, pc_dimm_count, opaque);
-    return 0;
-}
-
-static int pc_existing_dimms_capacity(Object *obj, void *opaque)
-{
-    Error *local_err = NULL;
-    uint64_t *size = opaque;
-
-    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
-        (*size) += object_property_get_int(obj, PC_DIMM_SIZE_PROP, &local_err);
-
-        if (local_err) {
-            qerror_report_err(local_err);
-            error_free(local_err);
-            return 1;
-        }
-    }
-
-    object_child_foreach(obj, pc_dimm_count, opaque);
-    return 0;
-}
-
 static void pc_dimm_plug(HotplugHandler *hotplug_dev,
                          DeviceState *dev, Error **errp)
 {
-    int slot;
     HotplugHandlerClass *hhc;
     Error *local_err = NULL;
     PCMachineState *pcms = PC_MACHINE(hotplug_dev);
-    MachineState *machine = MACHINE(hotplug_dev);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     PCDIMMDevice *dimm = PC_DIMM(dev);
     PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
     MemoryRegion *mr = ddc->get_memory_region(dimm);
-    uint64_t existing_dimms_capacity = 0;
     uint64_t align = TARGET_PAGE_SIZE;
-    uint64_t addr;
 
-    addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP, &local_err);
-    if (local_err) {
-        goto out;
-    }
-
-    if (memory_region_get_alignment(mr) && pcms->enforce_aligned_dimm) {
+    if (memory_region_get_alignment(mr) && pcmc->enforce_aligned_dimm) {
         align = memory_region_get_alignment(mr);
     }
 
-    addr = pc_dimm_get_free_addr(pcms->hotplug_memory_base,
-                                 memory_region_size(&pcms->hotplug_memory),
-                                 !addr ? NULL : &addr, align,
-                                 memory_region_size(mr), &local_err);
-    if (local_err) {
-        goto out;
-    }
-
-    if (pc_existing_dimms_capacity(OBJECT(machine), &existing_dimms_capacity)) {
-        error_setg(&local_err, "failed to get total size of existing DIMMs");
-        goto out;
-    }
-
-    if (existing_dimms_capacity + memory_region_size(mr) >
-        machine->maxram_size - machine->ram_size) {
-        error_setg(&local_err, "not enough space, currently 0x%" PRIx64
-                   " in use of total 0x" RAM_ADDR_FMT,
-                   existing_dimms_capacity, machine->maxram_size);
-        goto out;
-    }
-
-    object_property_set_int(OBJECT(dev), addr, PC_DIMM_ADDR_PROP, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    trace_mhp_pc_dimm_assigned_address(addr);
-
-    slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP, &local_err);
-    if (local_err) {
-        goto out;
-    }
-
-    slot = pc_dimm_get_free_slot(slot == PC_DIMM_UNASSIGNED_SLOT ? NULL : &slot,
-                                 machine->ram_slots, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    object_property_set_int(OBJECT(dev), slot, PC_DIMM_SLOT_PROP, &local_err);
-    if (local_err) {
-        goto out;
-    }
-    trace_mhp_pc_dimm_assigned_slot(slot);
-
     if (!pcms->acpi_dev) {
         error_setg(&local_err,
                    "memory hotplug is not enabled: missing acpi device");
         goto out;
     }
 
-    if (kvm_enabled() && !kvm_has_free_slot(machine)) {
-        error_setg(&local_err, "hypervisor has no free memory slots left");
+    pc_dimm_memory_plug(dev, &pcms->hotplug_memory, mr, align, &local_err);
+    if (local_err) {
         goto out;
     }
 
-    memory_region_add_subregion(&pcms->hotplug_memory,
-                                addr - pcms->hotplug_memory_base, mr);
-    vmstate_register_ram(mr, dev);
-
     hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
-    hhc->plug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+    hhc->plug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &error_abort);
 out:
     error_propagate(errp, local_err);
 }
 
-static void pc_cpu_plug(HotplugHandler *hotplug_dev,
-                        DeviceState *dev, Error **errp)
+static void pc_dimm_unplug_request(HotplugHandler *hotplug_dev,
+                                   DeviceState *dev, Error **errp)
 {
     HotplugHandlerClass *hhc;
     Error *local_err = NULL;
     PCMachineState *pcms = PC_MACHINE(hotplug_dev);
 
-    if (!dev->hotplugged) {
-        goto out;
-    }
-
     if (!pcms->acpi_dev) {
         error_setg(&local_err,
-                   "cpu hotplug is not enabled: missing acpi device");
+                   "memory hotplug is not enabled: missing acpi device");
         goto out;
     }
 
     hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
-    hhc->plug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+    hhc->unplug_request(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void pc_dimm_unplug(HotplugHandler *hotplug_dev,
+                           DeviceState *dev, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+    MemoryRegion *mr = ddc->get_memory_region(dimm);
+    HotplugHandlerClass *hhc;
+    Error *local_err = NULL;
+
+    hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
+    hhc->unplug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+
     if (local_err) {
         goto out;
     }
 
-    /* increment the number of CPUs */
-    rtc_set_memory(pcms->rtc, 0x5f, rtc_get_memory(pcms->rtc, 0x5f) + 1);
+    pc_dimm_memory_unplug(dev, &pcms->hotplug_memory, mr);
+    object_unparent(OBJECT(dev));
+
+ out:
+    error_propagate(errp, local_err);
+}
+
+static int pc_apic_cmp(const void *a, const void *b)
+{
+   CPUArchId *apic_a = (CPUArchId *)a;
+   CPUArchId *apic_b = (CPUArchId *)b;
+
+   return apic_a->arch_id - apic_b->arch_id;
+}
+
+/* returns pointer to CPUArchId descriptor that matches CPU's apic_id
+ * in pcms->possible_cpus->cpus, if pcms->possible_cpus->cpus has no
+ * entry correponding to CPU's apic_id returns NULL.
+ */
+static CPUArchId *pc_find_cpu_slot(PCMachineState *pcms, CPUState *cpu,
+                                   int *idx)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+    CPUArchId apic_id, *found_cpu;
+
+    apic_id.arch_id = cc->get_arch_id(CPU(cpu));
+    found_cpu = bsearch(&apic_id, pcms->possible_cpus->cpus,
+        pcms->possible_cpus->len, sizeof(*pcms->possible_cpus->cpus),
+        pc_apic_cmp);
+    if (found_cpu && idx) {
+        *idx = found_cpu - pcms->possible_cpus->cpus;
+    }
+    return found_cpu;
+}
+
+static void pc_cpu_plug(HotplugHandler *hotplug_dev,
+                        DeviceState *dev, Error **errp)
+{
+    CPUArchId *found_cpu;
+    HotplugHandlerClass *hhc;
+    Error *local_err = NULL;
+    PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+
+    if (pcms->acpi_dev) {
+        hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
+        hhc->plug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+        if (local_err) {
+            goto out;
+        }
+    }
+
+    if (dev->hotplugged) {
+        /* increment the number of CPUs */
+        rtc_set_memory(pcms->rtc, 0x5f, rtc_get_memory(pcms->rtc, 0x5f) + 1);
+    }
+
+    found_cpu = pc_find_cpu_slot(pcms, CPU(dev), NULL);
+    found_cpu->cpu = CPU(dev);
 out:
     error_propagate(errp, local_err);
 }
+static void pc_cpu_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
+{
+    int idx = -1;
+    HotplugHandlerClass *hhc;
+    Error *local_err = NULL;
+    PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+
+    pc_find_cpu_slot(pcms, CPU(dev), &idx);
+    assert(idx != -1);
+    if (idx == 0) {
+        error_setg(&local_err, "Boot CPU is unpluggable");
+        goto out;
+    }
+
+    hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
+    hhc->unplug_request(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+
+    if (local_err) {
+        goto out;
+    }
+
+ out:
+    error_propagate(errp, local_err);
+
+}
+
+static void pc_cpu_unplug_cb(HotplugHandler *hotplug_dev,
+                             DeviceState *dev, Error **errp)
+{
+    CPUArchId *found_cpu;
+    HotplugHandlerClass *hhc;
+    Error *local_err = NULL;
+    PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+
+    hhc = HOTPLUG_HANDLER_GET_CLASS(pcms->acpi_dev);
+    hhc->unplug(HOTPLUG_HANDLER(pcms->acpi_dev), dev, &local_err);
+
+    if (local_err) {
+        goto out;
+    }
+
+    found_cpu = pc_find_cpu_slot(pcms, CPU(dev), NULL);
+    found_cpu->cpu = NULL;
+    object_unparent(OBJECT(dev));
+
+    rtc_set_memory(pcms->rtc, 0x5f, rtc_get_memory(pcms->rtc, 0x5f) - 1);
+ out:
+    error_propagate(errp, local_err);
+}
+
+static void pc_cpu_pre_plug(HotplugHandler *hotplug_dev,
+                            DeviceState *dev, Error **errp)
+{
+    int idx;
+    CPUState *cs;
+    CPUArchId *cpu_slot;
+    X86CPUTopoInfo topo;
+    X86CPU *cpu = X86_CPU(dev);
+    PCMachineState *pcms = PC_MACHINE(hotplug_dev);
+
+    /* if APIC ID is not set, set it based on socket/core/thread properties */
+    if (cpu->apic_id == UNASSIGNED_APIC_ID) {
+        int max_socket = (max_cpus - 1) / smp_threads / smp_cores;
+
+        if (cpu->socket_id < 0) {
+            error_setg(errp, "CPU socket-id is not set");
+            return;
+        } else if (cpu->socket_id > max_socket) {
+            error_setg(errp, "Invalid CPU socket-id: %u must be in range 0:%u",
+                       cpu->socket_id, max_socket);
+            return;
+        }
+        if (cpu->core_id < 0) {
+            error_setg(errp, "CPU core-id is not set");
+            return;
+        } else if (cpu->core_id > (smp_cores - 1)) {
+            error_setg(errp, "Invalid CPU core-id: %u must be in range 0:%u",
+                       cpu->core_id, smp_cores - 1);
+            return;
+        }
+        if (cpu->thread_id < 0) {
+            error_setg(errp, "CPU thread-id is not set");
+            return;
+        } else if (cpu->thread_id > (smp_threads - 1)) {
+            error_setg(errp, "Invalid CPU thread-id: %u must be in range 0:%u",
+                       cpu->thread_id, smp_threads - 1);
+            return;
+        }
+
+        topo.pkg_id = cpu->socket_id;
+        topo.core_id = cpu->core_id;
+        topo.smt_id = cpu->thread_id;
+        cpu->apic_id = apicid_from_topo_ids(smp_cores, smp_threads, &topo);
+    }
+
+    cpu_slot = pc_find_cpu_slot(pcms, CPU(dev), &idx);
+    if (!cpu_slot) {
+        x86_topo_ids_from_apicid(cpu->apic_id, smp_cores, smp_threads, &topo);
+        error_setg(errp, "Invalid CPU [socket: %u, core: %u, thread: %u] with"
+                  " APIC ID %" PRIu32 ", valid index range 0:%d",
+                   topo.pkg_id, topo.core_id, topo.smt_id, cpu->apic_id,
+                   pcms->possible_cpus->len - 1);
+        return;
+    }
+
+    if (cpu_slot->cpu) {
+        error_setg(errp, "CPU[%d] with APIC ID %" PRIu32 " exists",
+                   idx, cpu->apic_id);
+        return;
+    }
+
+    /* if 'address' properties socket-id/core-id/thread-id are not set, set them
+     * so that query_hotpluggable_cpus would show correct values
+     */
+    /* TODO: move socket_id/core_id/thread_id checks into x86_cpu_realizefn()
+     * once -smp refactoring is complete and there will be CPU private
+     * CPUState::nr_cores and CPUState::nr_threads fields instead of globals */
+    x86_topo_ids_from_apicid(cpu->apic_id, smp_cores, smp_threads, &topo);
+    if (cpu->socket_id != -1 && cpu->socket_id != topo.pkg_id) {
+        error_setg(errp, "property socket-id: %u doesn't match set apic-id:"
+            " 0x%x (socket-id: %u)", cpu->socket_id, cpu->apic_id, topo.pkg_id);
+        return;
+    }
+    cpu->socket_id = topo.pkg_id;
+
+    if (cpu->core_id != -1 && cpu->core_id != topo.core_id) {
+        error_setg(errp, "property core-id: %u doesn't match set apic-id:"
+            " 0x%x (core-id: %u)", cpu->core_id, cpu->apic_id, topo.core_id);
+        return;
+    }
+    cpu->core_id = topo.core_id;
+
+    if (cpu->thread_id != -1 && cpu->thread_id != topo.smt_id) {
+        error_setg(errp, "property thread-id: %u doesn't match set apic-id:"
+            " 0x%x (thread-id: %u)", cpu->thread_id, cpu->apic_id, topo.smt_id);
+        return;
+    }
+    cpu->thread_id = topo.smt_id;
+
+    cs = CPU(cpu);
+    cs->cpu_index = idx;
+}
+
+static void pc_machine_device_pre_plug_cb(HotplugHandler *hotplug_dev,
+                                          DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        pc_cpu_pre_plug(hotplug_dev, dev, errp);
+    }
+}
 
 static void pc_machine_device_plug_cb(HotplugHandler *hotplug_dev,
                                       DeviceState *dev, Error **errp)
@@ -1718,6 +1972,32 @@
     }
 }
 
+static void pc_machine_device_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                                DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        pc_dimm_unplug_request(hotplug_dev, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        pc_cpu_unplug_request_cb(hotplug_dev, dev, errp);
+    } else {
+        error_setg(errp, "acpi: device unplug request for not supported device"
+                   " type: %s", object_get_typename(OBJECT(dev)));
+    }
+}
+
+static void pc_machine_device_unplug_cb(HotplugHandler *hotplug_dev,
+                                        DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        pc_dimm_unplug(hotplug_dev, dev, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        pc_cpu_unplug_cb(hotplug_dev, dev, errp);
+    } else {
+        error_setg(errp, "acpi: device unplug for not supported device"
+                   " type: %s", object_get_typename(OBJECT(dev)));
+    }
+}
+
 static HotplugHandler *pc_get_hotpug_handler(MachineState *machine,
                                              DeviceState *dev)
 {
@@ -1733,42 +2013,43 @@
 }
 
 static void
-pc_machine_get_hotplug_memory_region_size(Object *obj, Visitor *v, void *opaque,
-                                          const char *name, Error **errp)
+pc_machine_get_hotplug_memory_region_size(Object *obj, Visitor *v,
+                                          const char *name, void *opaque,
+                                          Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
-    int64_t value = memory_region_size(&pcms->hotplug_memory);
+    int64_t value = memory_region_size(&pcms->hotplug_memory.mr);
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void pc_machine_get_max_ram_below_4g(Object *obj, Visitor *v,
-                                         void *opaque, const char *name,
-                                         Error **errp)
+                                            const char *name, void *opaque,
+                                            Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     uint64_t value = pcms->max_ram_below_4g;
 
-    visit_type_size(v, &value, name, errp);
+    visit_type_size(v, name, &value, errp);
 }
 
 static void pc_machine_set_max_ram_below_4g(Object *obj, Visitor *v,
-                                         void *opaque, const char *name,
-                                         Error **errp)
+                                            const char *name, void *opaque,
+                                            Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     Error *error = NULL;
     uint64_t value;
 
-    visit_type_size(v, &value, name, &error);
+    visit_type_size(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
     }
     if (value > (1ULL << 32)) {
-        error_set(&error, ERROR_CLASS_GENERIC_ERROR,
-                  "Machine option 'max-ram-below-4g=%"PRIu64
-                  "' expects size less than or equal to 4G", value);
+        error_setg(&error,
+                   "Machine option 'max-ram-below-4g=%"PRIu64
+                   "' expects size less than or equal to 4G", value);
         error_propagate(errp, error);
         return;
     }
@@ -1782,28 +2063,81 @@
     pcms->max_ram_below_4g = value;
 }
 
-static void pc_machine_get_vmport(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void pc_machine_get_vmport(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
     OnOffAuto vmport = pcms->vmport;
 
-    visit_type_OnOffAuto(v, &vmport, name, errp);
+    visit_type_OnOffAuto(v, name, &vmport, errp);
 }
 
-static void pc_machine_set_vmport(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void pc_machine_set_vmport(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
 
-    visit_type_OnOffAuto(v, &pcms->vmport, name, errp);
+    visit_type_OnOffAuto(v, name, &pcms->vmport, errp);
 }
 
-static bool pc_machine_get_aligned_dimm(Object *obj, Error **errp)
+bool pc_machine_is_smm_enabled(PCMachineState *pcms)
+{
+    bool smm_available = false;
+
+    if (pcms->smm == ON_OFF_AUTO_OFF) {
+        return false;
+    }
+
+    if (tcg_enabled() || qtest_enabled()) {
+        smm_available = true;
+    } else if (kvm_enabled()) {
+        smm_available = kvm_has_smm();
+#ifdef CONFIG_HAX
+    } else if (hax_enabled()) {
+        smm_available = false;
+#endif
+    }
+
+    if (smm_available) {
+        return true;
+    }
+
+    if (pcms->smm == ON_OFF_AUTO_ON) {
+        error_report("System Management Mode not supported by this hypervisor.");
+        exit(1);
+    }
+    return false;
+}
+
+static void pc_machine_get_smm(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+    OnOffAuto smm = pcms->smm;
+
+    visit_type_OnOffAuto(v, name, &smm, errp);
+}
+
+static void pc_machine_set_smm(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     PCMachineState *pcms = PC_MACHINE(obj);
 
-    return pcms->enforce_aligned_dimm;
+    visit_type_OnOffAuto(v, name, &pcms->smm, errp);
+}
+
+static bool pc_machine_get_nvdimm(Object *obj, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+
+    return pcms->acpi_nvdimm_state.is_enabled;
+}
+
+static void pc_machine_set_nvdimm(Object *obj, bool value, Error **errp)
+{
+    PCMachineState *pcms = PC_MACHINE(obj);
+
+    pcms->acpi_nvdimm_state.is_enabled = value;
 }
 
 static void pc_machine_initfn(Object *obj)
@@ -1812,23 +2146,137 @@
 
     object_property_add(obj, PC_MACHINE_MEMHP_REGION_SIZE, "int",
                         pc_machine_get_hotplug_memory_region_size,
-                        NULL, NULL, NULL, NULL);
-    pcms->max_ram_below_4g = 1ULL << 32; /* 4G */
+                        NULL, NULL, NULL, &error_abort);
+
+    pcms->max_ram_below_4g = 0; /* use default */
     object_property_add(obj, PC_MACHINE_MAX_RAM_BELOW_4G, "size",
                         pc_machine_get_max_ram_below_4g,
                         pc_machine_set_max_ram_below_4g,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
+    object_property_set_description(obj, PC_MACHINE_MAX_RAM_BELOW_4G,
+                                    "Maximum ram below the 4G boundary (32bit boundary)",
+                                    &error_abort);
+
+    pcms->smm = ON_OFF_AUTO_AUTO;
+    object_property_add(obj, PC_MACHINE_SMM, "OnOffAuto",
+                        pc_machine_get_smm,
+                        pc_machine_set_smm,
+                        NULL, NULL, &error_abort);
+    object_property_set_description(obj, PC_MACHINE_SMM,
+                                    "Enable SMM (pc & q35)",
+                                    &error_abort);
 
     pcms->vmport = ON_OFF_AUTO_AUTO;
     object_property_add(obj, PC_MACHINE_VMPORT, "OnOffAuto",
                         pc_machine_get_vmport,
                         pc_machine_set_vmport,
-                        NULL, NULL, NULL);
+                        NULL, NULL, &error_abort);
+    object_property_set_description(obj, PC_MACHINE_VMPORT,
+                                    "Enable vmport (pc & q35)",
+                                    &error_abort);
 
-    pcms->enforce_aligned_dimm = true;
-    object_property_add_bool(obj, PC_MACHINE_ENFORCE_ALIGNED_DIMM,
-                             pc_machine_get_aligned_dimm,
-                             NULL, NULL);
+    /* nvdimm is disabled on default. */
+    pcms->acpi_nvdimm_state.is_enabled = false;
+    object_property_add_bool(obj, PC_MACHINE_NVDIMM, pc_machine_get_nvdimm,
+                             pc_machine_set_nvdimm, &error_abort);
+}
+
+static void pc_machine_reset(void)
+{
+    CPUState *cs;
+    X86CPU *cpu;
+
+    qemu_devices_reset();
+
+    /* Reset APIC after devices have been reset to cancel
+     * any changes that qemu_devices_reset() might have done.
+     */
+    CPU_FOREACH(cs) {
+        cpu = X86_CPU(cs);
+
+        if (cpu->apic_state) {
+            device_reset(cpu->apic_state);
+        }
+    }
+}
+
+static unsigned pc_cpu_index_to_socket_id(unsigned cpu_index)
+{
+    X86CPUTopoInfo topo;
+    x86_topo_ids_from_idx(smp_cores, smp_threads, cpu_index,
+                          &topo);
+    return topo.pkg_id;
+}
+
+static CPUArchIdList *pc_possible_cpu_arch_ids(MachineState *machine)
+{
+    PCMachineState *pcms = PC_MACHINE(machine);
+    int len = sizeof(CPUArchIdList) +
+              sizeof(CPUArchId) * (pcms->possible_cpus->len);
+    CPUArchIdList *list = g_malloc(len);
+
+    memcpy(list, pcms->possible_cpus, len);
+    return list;
+}
+
+static HotpluggableCPUList *pc_query_hotpluggable_cpus(MachineState *machine)
+{
+    int i;
+    CPUState *cpu;
+    HotpluggableCPUList *head = NULL;
+    PCMachineState *pcms = PC_MACHINE(machine);
+    const char *cpu_type;
+
+    cpu = pcms->possible_cpus->cpus[0].cpu;
+    assert(cpu); /* BSP is always present */
+    cpu_type = object_class_get_name(OBJECT_CLASS(CPU_GET_CLASS(cpu)));
+
+    for (i = 0; i < pcms->possible_cpus->len; i++) {
+        X86CPUTopoInfo topo;
+        HotpluggableCPUList *list_item = g_new0(typeof(*list_item), 1);
+        HotpluggableCPU *cpu_item = g_new0(typeof(*cpu_item), 1);
+        CpuInstanceProperties *cpu_props = g_new0(typeof(*cpu_props), 1);
+        const uint32_t apic_id = pcms->possible_cpus->cpus[i].arch_id;
+
+        x86_topo_ids_from_apicid(apic_id, smp_cores, smp_threads, &topo);
+
+        cpu_item->type = g_strdup(cpu_type);
+        cpu_item->vcpus_count = 1;
+        cpu_props->has_socket_id = true;
+        cpu_props->socket_id = topo.pkg_id;
+        cpu_props->has_core_id = true;
+        cpu_props->core_id = topo.core_id;
+        cpu_props->has_thread_id = true;
+        cpu_props->thread_id = topo.smt_id;
+        cpu_item->props = cpu_props;
+
+        cpu = pcms->possible_cpus->cpus[i].cpu;
+        if (cpu) {
+            cpu_item->has_qom_path = true;
+            cpu_item->qom_path = object_get_canonical_path(OBJECT(cpu));
+        }
+
+        list_item->value = cpu_item;
+        list_item->next = head;
+        head = list_item;
+    }
+    return head;
+}
+
+static void x86_nmi(NMIState *n, int cpu_index, Error **errp)
+{
+    /* cpu index isn't used */
+    CPUState *cs;
+
+    CPU_FOREACH(cs) {
+        X86CPU *cpu = X86_CPU(cs);
+
+        if (!cpu->apic_state) {
+            cpu_interrupt(cs, CPU_INTERRUPT_NMI);
+        } else {
+            apic_deliver_nmi(cpu->apic_state);
+        }
+    }
 }
 
 static void pc_machine_class_init(ObjectClass *oc, void *data)
@@ -1836,10 +2284,35 @@
     MachineClass *mc = MACHINE_CLASS(oc);
     PCMachineClass *pcmc = PC_MACHINE_CLASS(oc);
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+    NMIClass *nc = NMI_CLASS(oc);
 
     pcmc->get_hotplug_handler = mc->get_hotplug_handler;
+    pcmc->pci_enabled = true;
+    pcmc->has_acpi_build = true;
+    pcmc->rsdp_in_ram = true;
+    pcmc->smbios_defaults = true;
+    pcmc->smbios_uuid_encoded = true;
+    pcmc->gigabyte_align = true;
+    pcmc->has_reserved_memory = true;
+    pcmc->kvmclock_enabled = true;
+    pcmc->enforce_aligned_dimm = true;
+    /* BIOS ACPI tables: 128K. Other BIOS datastructures: less than 4K reported
+     * to be used at the moment, 32K should be enough for a while.  */
+    pcmc->acpi_data_size = 0x20000 + 0x8000;
+    pcmc->save_tsc_khz = true;
     mc->get_hotplug_handler = pc_get_hotpug_handler;
+    mc->cpu_index_to_socket_id = pc_cpu_index_to_socket_id;
+    mc->possible_cpu_arch_ids = pc_possible_cpu_arch_ids;
+    mc->query_hotpluggable_cpus = pc_query_hotpluggable_cpus;
+    mc->default_boot_order = "cad";
+    mc->hot_add_cpu = pc_hot_add_cpu;
+    mc->max_cpus = 255;
+    mc->reset = pc_machine_reset;
+    hc->pre_plug = pc_machine_device_pre_plug_cb;
     hc->plug = pc_machine_device_plug_cb;
+    hc->unplug_request = pc_machine_device_unplug_request_cb;
+    hc->unplug = pc_machine_device_unplug_cb;
+    nc->nmi_monitor_handler = x86_nmi;
 }
 
 static const TypeInfo pc_machine_info = {
@@ -1852,6 +2325,7 @@
     .class_init = pc_machine_class_init,
     .interfaces = (InterfaceInfo[]) {
          { TYPE_HOTPLUG_HANDLER },
+         { TYPE_NMI },
          { }
     },
 };
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index 3be2c5f..a2694ff 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -22,13 +22,13 @@
  * THE SOFTWARE.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/usb.h"
@@ -39,7 +39,6 @@
 #include "hw/kvm/clock.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
-#include "hw/cpu/icc_bus.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/block-backend.h"
 #include "hw/i2c/smbus.h"
@@ -50,19 +49,15 @@
 #include "cpu.h"
 #include "qemu/error-report.h"
 #ifdef CONFIG_XEN
-#  include <xen/hvm/hvm_info_table.h>
+#include <xen/hvm/hvm_info_table.h>
+#include "hw/xen/xen_pt.h"
 #endif
-// android related header
+#include "migration/migration.h"
+#include "kvm_i386.h"
 
 #ifdef CONFIG_ANDROID
 #include "hw/acpi/goldfish_defs.h"
-
-#include "android/constants.h"
-#include "android/android.h"
-#include "android/error-messages.h"
-#endif  // CONFIG_ANDROID
-
-// end of android related header
+#endif
 
 #define MAX_IDE_BUS 2
 
@@ -70,100 +65,99 @@
 static const int ide_iobase2[MAX_IDE_BUS] = { 0x3f6, 0x376 };
 static const int ide_irq[MAX_IDE_BUS] = { 14, 15 };
 
-static bool has_acpi_build = true;
-static int legacy_acpi_table_size;
-static bool smbios_defaults = true;
-static bool smbios_legacy_mode;
-static bool smbios_uuid_encoded = true;
-/* Make sure that guest addresses aligned at 1Gbyte boundaries get mapped to
- * host addresses aligned at 1Gbyte boundaries.  This way we can use 1GByte
- * pages in the host.
- */
-static bool gigabyte_align = true;
-static bool has_reserved_memory = true;
-
 /* PC hardware initialisation */
 static void pc_init1(MachineState *machine,
-                     int pci_enabled,
-                     int kvmclock_enabled)
+                     const char *host_type, const char *pci_type)
 {
-    PCMachineState *pc_machine = PC_MACHINE(machine);
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     MemoryRegion *system_memory = get_system_memory();
     MemoryRegion *system_io = get_system_io();
     int i;
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     PCII440FXState *i440fx_state;
     int piix3_devfn = -1;
-    qemu_irq *cpu_irq;
     qemu_irq *gsi;
     qemu_irq *i8259;
-    qemu_irq *smi_irq;
+    qemu_irq smi_irq;
     GSIState *gsi_state;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     BusState *idebus[MAX_IDE_BUS];
     ISADevice *rtc_state;
-    ISADevice *floppy;
     MemoryRegion *ram_memory;
     MemoryRegion *pci_memory;
     MemoryRegion *rom_memory;
-    DeviceState *icc_bridge;
-    FWCfgState *fw_cfg = NULL;
-    PcGuestInfo *guest_info;
     ram_addr_t lowmem;
 
-    /* Check whether RAM fits below 4G (leaving 1/2 GByte for IO memory).
-     * If it doesn't, we need to split it in chunks below and above 4G.
-     * In any case, try to make sure that guest addresses aligned at
-     * 1G boundaries get mapped to host addresses aligned at 1G boundaries.
-     * For old machine types, use whatever split we used historically to avoid
-     * breaking migration.
+    /*
+     * Calculate ram split, for memory below and above 4G.  It's a bit
+     * complicated for backward compatibility reasons ...
+     *
+     *  - Traditional split is 3.5G (lowmem = 0xe0000000).  This is the
+     *    default value for max_ram_below_4g now.
+     *
+     *  - Then, to gigabyte align the memory, we move the split to 3G
+     *    (lowmem = 0xc0000000).  But only in case we have to split in
+     *    the first place, i.e. ram_size is larger than (traditional)
+     *    lowmem.  And for new machine types (gigabyte_align = true)
+     *    only, for live migration compatibility reasons.
+     *
+     *  - Next the max-ram-below-4g option was added, which allowed to
+     *    reduce lowmem to a smaller value, to allow a larger PCI I/O
+     *    window below 4G.  qemu doesn't enforce gigabyte alignment here,
+     *    but prints a warning.
+     *
+     *  - Finally max-ram-below-4g got updated to also allow raising lowmem,
+     *    so legacy non-PAE guests can get as much memory as possible in
+     *    the 32bit address space below 4G.
+     *
+     *  - Note that Xen has its own ram setp code in xen_ram_init(),
+     *    called via xen_hvm_init().
+     *
+     * Examples:
+     *    qemu -M pc-1.7 -m 4G    (old default)    -> 3584M low,  512M high
+     *    qemu -M pc -m 4G        (new default)    -> 3072M low, 1024M high
+     *    qemu -M pc,max-ram-below-4g=2G -m 4G     -> 2048M low, 2048M high
+     *    qemu -M pc,max-ram-below-4g=4G -m 3968M  -> 3968M low (=4G-128M)
      */
-    if (machine->ram_size >= 0xe0000000) {
-        lowmem = gigabyte_align ? 0xc0000000 : 0xe0000000;
+    if (xen_enabled()) {
+        xen_hvm_init(pcms, &ram_memory);
     } else {
-        lowmem = 0xe0000000;
-    }
+        if (!pcms->max_ram_below_4g) {
+            pcms->max_ram_below_4g = 0xe0000000; /* default: 3.5G */
+        }
+        lowmem = pcms->max_ram_below_4g;
+        if (machine->ram_size >= pcms->max_ram_below_4g) {
+            if (pcmc->gigabyte_align) {
+                if (lowmem > 0xc0000000) {
+                    lowmem = 0xc0000000;
+                }
+                if (lowmem & ((1ULL << 30) - 1)) {
+                    error_report("Warning: Large machine and max_ram_below_4g "
+                                 "(%" PRIu64 ") not a multiple of 1G; "
+                                 "possible bad performance.",
+                                 pcms->max_ram_below_4g);
+                }
+            }
+        }
 
-    /* Handle the machine opt max-ram-below-4g.  It is basically doing
-     * min(qemu limit, user limit).
-     */
-    if (lowmem > pc_machine->max_ram_below_4g) {
-        lowmem = pc_machine->max_ram_below_4g;
-        if (machine->ram_size - lowmem > lowmem &&
-            lowmem & ((1ULL << 30) - 1)) {
-            error_report("Warning: Large machine and max_ram_below_4g(%"PRIu64
-                         ") not a multiple of 1G; possible bad performance.",
-                         pc_machine->max_ram_below_4g);
+        if (machine->ram_size >= lowmem) {
+            pcms->above_4g_mem_size = machine->ram_size - lowmem;
+            pcms->below_4g_mem_size = lowmem;
+        } else {
+            pcms->above_4g_mem_size = 0;
+            pcms->below_4g_mem_size = machine->ram_size;
         }
     }
 
-    if (machine->ram_size >= lowmem) {
-        above_4g_mem_size = machine->ram_size - lowmem;
-        below_4g_mem_size = lowmem;
-    } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = machine->ram_size;
-    }
+    pc_cpus_init(pcms);
 
-    if (xen_enabled() && xen_hvm_init(&below_4g_mem_size, &above_4g_mem_size,
-                                      &ram_memory) != 0) {
-        fprintf(stderr, "xen hardware virtual machine initialisation failed\n");
-        exit(1);
-    }
-
-    icc_bridge = qdev_create(NULL, TYPE_ICC_BRIDGE);
-    object_property_add_child(qdev_get_machine(), "icc-bridge",
-                              OBJECT(icc_bridge), NULL);
-
-    pc_cpus_init(machine->cpu_model, icc_bridge);
-
-    if (kvm_enabled() && kvmclock_enabled) {
+    if (kvm_enabled() && pcmc->kvmclock_enabled) {
         kvmclock_create();
     }
 
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pci_memory = g_new(MemoryRegion, 1);
         memory_region_init(pci_memory, NULL, "pci", UINT64_MAX);
         rom_memory = pci_memory;
@@ -172,109 +166,105 @@
         rom_memory = system_memory;
     }
 
-    guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
+    pc_guest_info_init(pcms);
 
-    guest_info->has_acpi_build = has_acpi_build;
-    guest_info->legacy_acpi_table_size = legacy_acpi_table_size;
-
-    guest_info->isapc_ram_fw = !pci_enabled;
-    guest_info->has_reserved_memory = has_reserved_memory;
-
-    if (smbios_defaults) {
+    if (pcmc->smbios_defaults) {
         MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (i440FX + PIIX, 1996)",
-                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
+                            mc->name, pcmc->smbios_legacy_mode,
+                            pcmc->smbios_uuid_encoded,
+                            SMBIOS_ENTRY_POINT_21);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
-        fw_cfg = pc_memory_init(machine, system_memory,
-                                below_4g_mem_size, above_4g_mem_size,
-                                rom_memory, &ram_memory, guest_info);
+        pc_memory_init(pcms, system_memory,
+                       rom_memory, &ram_memory);
     } else if (machine->kernel_filename != NULL) {
         /* For xen HVM direct kernel boot, load linux here */
-        fw_cfg = xen_load_linux(machine->kernel_filename,
-                                machine->kernel_cmdline,
-                                machine->initrd_filename,
-                                below_4g_mem_size,
-                                guest_info);
+        xen_load_linux(pcms);
     }
-#ifdef CONFIG_ANDROID
-    if (android_init_error_occurred()) {
+
+    if (error_during_init != NULL) {
         return;
     }
-#endif  // CONFIG_ANDROID
 
     gsi_state = g_malloc0(sizeof(*gsi_state));
-    if (kvm_irqchip_in_kernel()) {
-        kvm_pc_setup_irq_routing(pci_enabled);
+    if (kvm_ioapic_in_kernel()) {
+        kvm_pc_setup_irq_routing(pcmc->pci_enabled);
         gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
                                  GSI_NUM_PINS);
     } else {
         gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
     }
 
-    if (pci_enabled) {
-        pci_bus = i440fx_init(&i440fx_state, &piix3_devfn, &isa_bus, gsi,
+    if (pcmc->pci_enabled) {
+        pci_bus = i440fx_init(host_type,
+                              pci_type,
+                              &i440fx_state, &piix3_devfn, &isa_bus, gsi,
                               system_memory, system_io, machine->ram_size,
-                              below_4g_mem_size,
-                              above_4g_mem_size,
+                              pcms->below_4g_mem_size,
+                              pcms->above_4g_mem_size,
                               pci_memory, ram_memory);
+        pcms->bus = pci_bus;
     } else {
         pci_bus = NULL;
         i440fx_state = NULL;
-        isa_bus = isa_bus_new(NULL, system_io);
+        isa_bus = isa_bus_new(NULL, get_system_memory(), system_io,
+                              &error_abort);
         no_hpet = 1;
     }
     isa_bus_irqs(isa_bus, gsi);
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_pic_in_kernel()) {
         i8259 = kvm_i8259_init(isa_bus);
     } else if (xen_enabled()) {
         i8259 = xen_interrupt_controller_init();
     } else {
-        cpu_irq = pc_allocate_cpu_irq();
-        i8259 = i8259_init(isa_bus, cpu_irq[0]);
+        i8259 = i8259_init(isa_bus, pc_allocate_cpu_irq());
     }
 
     for (i = 0; i < ISA_NUM_IRQS; i++) {
         gsi_state->i8259_irq[i] = i8259[i];
     }
-    if (pci_enabled) {
+    g_free(i8259);
+    if (pcmc->pci_enabled) {
         ioapic_init_gsi(gsi_state, "i440fx");
     }
-    qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
 #if defined(CONFIG_ANDROID)
-    sysbus_create_simple("goldfish_battery", GF_BATTERY_IOMEM_BASE,
-                         gsi[GF_BATTERY_IRQ]);
-    sysbus_create_simple("goldfish-events", GF_EVENTS_IOMEM_BASE,
-                         gsi[GF_EVENTS_IRQ]);
-    sysbus_create_simple("android_pipe", GF_PIPE_IOMEM_BASE, gsi[GF_PIPE_IRQ]);
-    sysbus_create_simple("goldfish_fb", GF_FB_IOMEM_BASE, gsi[GF_FB_IRQ]);
-    sysbus_create_simple("goldfish_audio", GF_AUDIO_IOMEM_BASE, gsi[GF_AUDIO_IRQ]);
-    sysbus_create_simple("goldfish_sync", GF_SYNC_IOMEM_BASE, gsi[GF_SYNC_IRQ]);
-    sysbus_create_simple("goldfish_rtc", GF_RTC_IOMEM_BASE, gsi[GF_RTC_IRQ]);
+    sysbus_create_simple("goldfish_battery", GOLDFISH_BATTERY_IOMEM_BASE,
+                         gsi[GOLDFISH_BATTERY_IRQ]);
+    sysbus_create_simple("goldfish-events", GOLDFISH_EVENTS_IOMEM_BASE,
+                         gsi[GOLDFISH_EVENTS_IRQ]);
+    sysbus_create_simple("goldfish_pipe", GOLDFISH_PIPE_IOMEM_BASE,
+                         gsi[GOLDFISH_PIPE_IRQ]);
+    sysbus_create_simple("goldfish_fb", GOLDFISH_FB_IOMEM_BASE,
+                         gsi[GOLDFISH_FB_IRQ]);
+    sysbus_create_simple("goldfish_audio", GOLDFISH_AUDIO_IOMEM_BASE,
+                         gsi[GOLDFISH_AUDIO_IRQ]);
+    sysbus_create_simple("goldfish_rtc", GOLDFISH_RTC_IOMEM_BASE,
+                         gsi[GOLDFISH_RTC_IRQ]);
 #endif  // CONFIG_ANDROID
 
-    pc_vga_init(isa_bus, pci_enabled ? pci_bus : NULL);
+    pc_vga_init(isa_bus, pcmc->pci_enabled ? pci_bus : NULL);
 
-    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
-    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
-        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    assert(pcms->vmport != ON_OFF_AUTO__MAX);
+    if (pcms->vmport == ON_OFF_AUTO_AUTO) {
+        pcms->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
     }
 
     /* init basic PC hardware */
-    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy,
-                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0x4);
+    pc_basic_device_init(isa_bus, gsi, &rtc_state, true,
+                         (pcms->vmport != ON_OFF_AUTO_ON), 0x4);
 
     pc_nic_init(isa_bus, pci_bus);
 
     ide_drive_get(hd, ARRAY_SIZE(hd));
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         PCIDevice *dev;
         if (xen_enabled()) {
             dev = pci_piix3_xen_ide_init(pci_bus, hd, piix3_devfn + 1);
@@ -299,56 +289,277 @@
         }
     }
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 machine, floppy, idebus[0], idebus[1], rtc_state);
+    pc_cmos_init(pcms, idebus[0], idebus[1], rtc_state);
 
-    if (pci_enabled && usb_enabled(false)) {
+    if (pcmc->pci_enabled && machine_usb(machine)) {
         pci_create_simple(pci_bus, piix3_devfn + 2, "piix3-usb-uhci");
     }
 
-    if (pci_enabled && acpi_enabled) {
+    if (pcmc->pci_enabled && acpi_enabled) {
         DeviceState *piix4_pm;
         I2CBus *smbus;
 
-        smi_irq = qemu_allocate_irqs(pc_acpi_smi_interrupt, first_cpu, 1);
+        smi_irq = qemu_allocate_irq(pc_acpi_smi_interrupt, first_cpu, 0);
         /* TODO: Populate SPD eeprom data.  */
         smbus = piix4_pm_init(pci_bus, piix3_devfn + 3, 0xb100,
-                              gsi[9], *smi_irq,
-                              kvm_enabled(), fw_cfg, &piix4_pm);
+                              gsi[9], smi_irq,
+                              pc_machine_is_smm_enabled(pcms),
+                              &piix4_pm);
         smbus_eeprom_init(smbus, 8, NULL, 0);
 
         object_property_add_link(OBJECT(machine), PC_MACHINE_ACPI_DEVICE_PROP,
                                  TYPE_HOTPLUG_HANDLER,
-                                 (Object **)&pc_machine->acpi_dev,
+                                 (Object **)&pcms->acpi_dev,
                                  object_property_allow_set_link,
                                  OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
         object_property_set_link(OBJECT(machine), OBJECT(piix4_pm),
                                  PC_MACHINE_ACPI_DEVICE_PROP, &error_abort);
     }
 
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pc_pci_device_init(pci_bus);
     }
+
+    if (pcms->acpi_nvdimm_state.is_enabled) {
+        nvdimm_init_acpi_state(&pcms->acpi_nvdimm_state, system_io,
+                               pcms->fw_cfg, OBJECT(pcms));
+    }
 }
 
-static void pc_init_pci(MachineState *machine)
+/* Looking for a pc_compat_2_4() function? It doesn't exist.
+ * pc_compat_*() functions that run on machine-init time and
+ * change global QEMU state are deprecated. Please don't create
+ * one, and implement any pc-*-2.4 (and newer) compat code in
+ * HW_COMPAT_*, PC_COMPAT_*, or * pc_*_machine_options().
+ */
+
+static void pc_compat_2_3(MachineState *machine)
 {
-    pc_init1(machine, 1, 1);
+    PCMachineState *pcms = PC_MACHINE(machine);
+    savevm_skip_section_footers();
+    if (kvm_enabled()) {
+        pcms->smm = ON_OFF_AUTO_OFF;
+    }
+    global_state_set_optional();
+    savevm_skip_configuration();
+}
+
+static void pc_compat_2_2(MachineState *machine)
+{
+    pc_compat_2_3(machine);
+    machine->suppress_vmdesc = true;
 }
 
 static void pc_compat_2_1(MachineState *machine)
 {
-    PCMachineState *pcms = PC_MACHINE(machine);
-    smbios_uuid_encoded = false;
-    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
-    pcms->enforce_aligned_dimm = false;
+    pc_compat_2_2(machine);
+    x86_cpu_change_kvm_default("svm", NULL);
 }
 
 static void pc_compat_2_0(MachineState *machine)
 {
     pc_compat_2_1(machine);
+}
+
+static void pc_compat_1_7(MachineState *machine)
+{
+    pc_compat_2_0(machine);
+    x86_cpu_change_kvm_default("x2apic", NULL);
+}
+
+static void pc_compat_1_6(MachineState *machine)
+{
+    pc_compat_1_7(machine);
+}
+
+static void pc_compat_1_5(MachineState *machine)
+{
+    pc_compat_1_6(machine);
+}
+
+static void pc_compat_1_4(MachineState *machine)
+{
+    pc_compat_1_5(machine);
+}
+
+static void pc_compat_1_3(MachineState *machine)
+{
+    pc_compat_1_4(machine);
+    enable_compat_apic_id_mode();
+}
+
+/* PC compat function for pc-0.14 to pc-1.2 */
+static void pc_compat_1_2(MachineState *machine)
+{
+    pc_compat_1_3(machine);
+    x86_cpu_change_kvm_default("kvm-pv-eoi", NULL);
+}
+
+/* PC compat function for pc-0.10 to pc-0.13 */
+static void pc_compat_0_13(MachineState *machine)
+{
+    pc_compat_1_2(machine);
+}
+
+static void pc_init_isa(MachineState *machine)
+{
+    if (!machine->cpu_model) {
+        machine->cpu_model = "486";
+    }
+    x86_cpu_change_kvm_default("kvm-pv-eoi", NULL);
+    enable_compat_apic_id_mode();
+    pc_init1(machine, TYPE_I440FX_PCI_HOST_BRIDGE, TYPE_I440FX_PCI_DEVICE);
+}
+
+#ifdef CONFIG_XEN
+static void pc_xen_hvm_init_pci(MachineState *machine)
+{
+    const char *pci_type = has_igd_gfx_passthru ?
+                TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE : TYPE_I440FX_PCI_DEVICE;
+
+    pc_init1(machine,
+             TYPE_I440FX_PCI_HOST_BRIDGE,
+             pci_type);
+}
+
+static void pc_xen_hvm_init(MachineState *machine)
+{
+    PCIBus *bus;
+
+    if (!xen_enabled()) {
+        error_report("xenfv machine requires the xen accelerator");
+        exit(1);
+    }
+
+    pc_xen_hvm_init_pci(machine);
+
+    bus = pci_find_primary_bus();
+    if (bus != NULL) {
+        pci_create_simple(bus, -1, "xen-platform");
+    }
+}
+#endif
+
+#define DEFINE_I440FX_MACHINE(suffix, name, compatfn, optionfn) \
+    static void pc_init_##suffix(MachineState *machine) \
+    { \
+        void (*compat)(MachineState *m) = (compatfn); \
+        if (compat) { \
+            compat(machine); \
+        } \
+        pc_init1(machine, TYPE_I440FX_PCI_HOST_BRIDGE, \
+                 TYPE_I440FX_PCI_DEVICE); \
+    } \
+    DEFINE_PC_MACHINE(suffix, name, pc_init_##suffix, optionfn)
+
+static void pc_i440fx_machine_options(MachineClass *m)
+{
+    m->family = "pc_piix";
+    m->desc = "Standard PC (i440FX + PIIX, 1996)";
+    m->hot_add_cpu = pc_hot_add_cpu;
+    m->default_machine_opts = "firmware=bios-256k.bin";
+    m->default_display = "std";
+}
+
+static void pc_i440fx_2_7_machine_options(MachineClass *m)
+{
+    pc_i440fx_machine_options(m);
+    m->alias = "pc";
+    m->is_default = 1;
+}
+
+DEFINE_I440FX_MACHINE(v2_7, "pc-i440fx-2.7", NULL,
+                      pc_i440fx_2_7_machine_options);
+
+
+static void pc_i440fx_2_6_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_7_machine_options(m);
+    m->is_default = 0;
+    m->alias = NULL;
+    pcmc->legacy_cpu_hotplug = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_6);
+}
+
+DEFINE_I440FX_MACHINE(v2_6, "pc-i440fx-2.6", NULL,
+                      pc_i440fx_2_6_machine_options);
+
+
+static void pc_i440fx_2_5_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_6_machine_options(m);
+    pcmc->save_tsc_khz = false;
+    m->legacy_fw_cfg_order = 1;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_5);
+}
+
+DEFINE_I440FX_MACHINE(v2_5, "pc-i440fx-2.5", NULL,
+                      pc_i440fx_2_5_machine_options);
+
+
+static void pc_i440fx_2_4_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_5_machine_options(m);
+    m->hw_version = "2.4.0";
+    pcmc->broken_reserved_end = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
+}
+
+DEFINE_I440FX_MACHINE(v2_4, "pc-i440fx-2.4", NULL,
+                      pc_i440fx_2_4_machine_options)
+
+
+static void pc_i440fx_2_3_machine_options(MachineClass *m)
+{
+    pc_i440fx_2_4_machine_options(m);
+    m->hw_version = "2.3.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_3);
+}
+
+DEFINE_I440FX_MACHINE(v2_3, "pc-i440fx-2.3", pc_compat_2_3,
+                      pc_i440fx_2_3_machine_options);
+
+
+static void pc_i440fx_2_2_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_3_machine_options(m);
+    m->hw_version = "2.2.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_2);
+    pcmc->rsdp_in_ram = false;
+}
+
+DEFINE_I440FX_MACHINE(v2_2, "pc-i440fx-2.2", pc_compat_2_2,
+                      pc_i440fx_2_2_machine_options);
+
+
+static void pc_i440fx_2_1_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_2_machine_options(m);
+    m->hw_version = "2.1.0";
+    m->default_display = NULL;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_1);
+    pcmc->smbios_uuid_encoded = false;
+    pcmc->enforce_aligned_dimm = false;
+}
+
+DEFINE_I440FX_MACHINE(v2_1, "pc-i440fx-2.1", pc_compat_2_1,
+                      pc_i440fx_2_1_machine_options);
+
+
+
+static void pc_i440fx_2_0_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_1_machine_options(m);
+    m->hw_version = "2.0.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_0);
+    pcmc->smbios_legacy_mode = true;
+    pcmc->has_reserved_memory = false;
     /* This value depends on the actual DSDT and SSDT compiled into
      * the source QEMU; unfortunately it depends on the binary and
      * not on the machine type, so we cannot make pc-i440fx-1.7 work on
@@ -365,236 +576,70 @@
      * 6652 is valid for QEMU 2.0, the right value for pc-i440fx-1.7 on
      * QEMU 1.7 it is 6414.  For RHEL/CentOS 7.0 it is 6418.
      */
-    legacy_acpi_table_size = 6652;
-    smbios_legacy_mode = true;
-    has_reserved_memory = false;
-    pc_set_legacy_acpi_data_size();
+    pcmc->legacy_acpi_table_size = 6652;
+    pcmc->acpi_data_size = 0x10000;
 }
 
-static void pc_compat_1_7(MachineState *machine)
+DEFINE_I440FX_MACHINE(v2_0, "pc-i440fx-2.0", pc_compat_2_0,
+                      pc_i440fx_2_0_machine_options);
+
+
+static void pc_i440fx_1_7_machine_options(MachineClass *m)
 {
-    pc_compat_2_0(machine);
-    smbios_defaults = false;
-    gigabyte_align = false;
-    option_rom_has_mr = true;
-    legacy_acpi_table_size = 6414;
-    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_2_0_machine_options(m);
+    m->hw_version = "1.7.0";
+    m->default_machine_opts = NULL;
+    m->option_rom_has_mr = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_7);
+    pcmc->smbios_defaults = false;
+    pcmc->gigabyte_align = false;
+    pcmc->legacy_acpi_table_size = 6414;
 }
 
-static void pc_compat_1_6(MachineState *machine)
+DEFINE_I440FX_MACHINE(v1_7, "pc-i440fx-1.7", pc_compat_1_7,
+                      pc_i440fx_1_7_machine_options);
+
+
+static void pc_i440fx_1_6_machine_options(MachineClass *m)
 {
-    pc_compat_1_7(machine);
-    rom_file_has_mr = false;
-    has_acpi_build = false;
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_1_7_machine_options(m);
+    m->hw_version = "1.6.0";
+    m->rom_file_has_mr = false;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_6);
+    pcmc->has_acpi_build = false;
 }
 
-static void pc_compat_1_5(MachineState *machine)
+DEFINE_I440FX_MACHINE(v1_6, "pc-i440fx-1.6", pc_compat_1_6,
+                      pc_i440fx_1_6_machine_options);
+
+
+static void pc_i440fx_1_5_machine_options(MachineClass *m)
 {
-    pc_compat_1_6(machine);
+    pc_i440fx_1_6_machine_options(m);
+    m->hw_version = "1.5.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_5);
 }
 
-static void pc_compat_1_4(MachineState *machine)
+DEFINE_I440FX_MACHINE(v1_5, "pc-i440fx-1.5", pc_compat_1_5,
+                      pc_i440fx_1_5_machine_options);
+
+
+static void pc_i440fx_1_4_machine_options(MachineClass *m)
 {
-    pc_compat_1_5(machine);
-    x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
+    pc_i440fx_1_5_machine_options(m);
+    m->hw_version = "1.4.0";
+    m->hot_add_cpu = NULL;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_4);
 }
 
-static void pc_compat_1_3(MachineState *machine)
-{
-    pc_compat_1_4(machine);
-    enable_compat_apic_id_mode();
-}
+DEFINE_I440FX_MACHINE(v1_4, "pc-i440fx-1.4", pc_compat_1_4,
+                      pc_i440fx_1_4_machine_options);
 
-/* PC compat function for pc-0.14 to pc-1.2 */
-static void pc_compat_1_2(MachineState *machine)
-{
-    pc_compat_1_3(machine);
-    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, KVM_FEATURE_PV_EOI);
-}
-
-static void pc_init_pci_2_1(MachineState *machine)
-{
-    pc_compat_2_1(machine);
-    pc_init_pci(machine);
-}
-
-static void pc_init_pci_2_0(MachineState *machine)
-{
-    pc_compat_2_0(machine);
-    pc_init_pci(machine);
-}
-
-static void pc_init_pci_1_7(MachineState *machine)
-{
-    pc_compat_1_7(machine);
-    pc_init_pci(machine);
-}
-
-static void pc_init_pci_1_6(MachineState *machine)
-{
-    pc_compat_1_6(machine);
-    pc_init_pci(machine);
-}
-
-static void pc_init_pci_1_5(MachineState *machine)
-{
-    pc_compat_1_5(machine);
-    pc_init_pci(machine);
-}
-
-static void pc_init_pci_1_4(MachineState *machine)
-{
-    pc_compat_1_4(machine);
-    pc_init_pci(machine);
-}
-
-static void pc_init_pci_1_3(MachineState *machine)
-{
-    pc_compat_1_3(machine);
-    pc_init_pci(machine);
-}
-
-/* PC machine init function for pc-0.14 to pc-1.2 */
-static void pc_init_pci_1_2(MachineState *machine)
-{
-    pc_compat_1_2(machine);
-    pc_init_pci(machine);
-}
-
-/* PC init function for pc-0.10 to pc-0.13 */
-static void pc_init_pci_no_kvmclock(MachineState *machine)
-{
-    pc_compat_1_2(machine);
-    pc_init1(machine, 1, 0);
-}
-
-static void pc_init_isa(MachineState *machine)
-{
-    has_acpi_build = false;
-    smbios_defaults = false;
-    gigabyte_align = false;
-    smbios_legacy_mode = true;
-    has_reserved_memory = false;
-    option_rom_has_mr = true;
-    rom_file_has_mr = false;
-    if (!machine->cpu_model) {
-        machine->cpu_model = "486";
-    }
-    x86_cpu_compat_kvm_no_autoenable(FEAT_KVM, KVM_FEATURE_PV_EOI);
-    enable_compat_apic_id_mode();
-    pc_init1(machine, 0, 1);
-}
-
-#ifdef CONFIG_XEN
-static void pc_xen_hvm_init(MachineState *machine)
-{
-    PCIBus *bus;
-
-    pc_init_pci(machine);
-
-    bus = pci_find_primary_bus();
-    if (bus != NULL) {
-        pci_create_simple(bus, -1, "xen-platform");
-    }
-}
-#endif
-
-#define PC_I440FX_MACHINE_OPTIONS \
-    PC_DEFAULT_MACHINE_OPTIONS, \
-    .family = "pc_piix", \
-    .desc = "Standard PC (i440FX + PIIX, 1996)", \
-    .hot_add_cpu = pc_hot_add_cpu
-
-#define PC_I440FX_2_2_MACHINE_OPTIONS                           \
-    PC_I440FX_MACHINE_OPTIONS,                                  \
-    .default_machine_opts = "firmware=bios-256k.bin",           \
-    .default_display = "std"
-
-static QEMUMachine pc_i440fx_machine_v2_2 = {
-    PC_I440FX_2_2_MACHINE_OPTIONS,
-    .name = "pc-i440fx-2.2",
-    .alias = "pc",
-    .init = pc_init_pci,
-    .is_default = 1,
-};
-
-#define PC_I440FX_2_1_MACHINE_OPTIONS                           \
-    PC_I440FX_MACHINE_OPTIONS,                                  \
-    .default_machine_opts = "firmware=bios-256k.bin"
-
-static QEMUMachine pc_i440fx_machine_v2_1 = {
-    PC_I440FX_2_1_MACHINE_OPTIONS,
-    .name = "pc-i440fx-2.1",
-    .init = pc_init_pci_2_1,
-    .compat_props = (GlobalProperty[]) {
-        HW_COMPAT_2_1,
-        { /* end of list */ }
-    },
-};
-
-#define PC_I440FX_2_0_MACHINE_OPTIONS PC_I440FX_2_1_MACHINE_OPTIONS
-
-static QEMUMachine pc_i440fx_machine_v2_0 = {
-    PC_I440FX_2_0_MACHINE_OPTIONS,
-    .name = "pc-i440fx-2.0",
-    .init = pc_init_pci_2_0,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_2_0,
-        { /* end of list */ }
-    },
-};
-
-#define PC_I440FX_1_7_MACHINE_OPTIONS PC_I440FX_MACHINE_OPTIONS
-
-static QEMUMachine pc_i440fx_machine_v1_7 = {
-    PC_I440FX_1_7_MACHINE_OPTIONS,
-    .name = "pc-i440fx-1.7",
-    .init = pc_init_pci_1_7,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_7,
-        { /* end of list */ }
-    },
-};
-
-#define PC_I440FX_1_6_MACHINE_OPTIONS PC_I440FX_MACHINE_OPTIONS
-
-static QEMUMachine pc_i440fx_machine_v1_6 = {
-    PC_I440FX_1_6_MACHINE_OPTIONS,
-    .name = "pc-i440fx-1.6",
-    .init = pc_init_pci_1_6,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_6,
-        { /* end of list */ }
-    },
-};
-
-static QEMUMachine pc_i440fx_machine_v1_5 = {
-    PC_I440FX_1_6_MACHINE_OPTIONS,
-    .name = "pc-i440fx-1.5",
-    .init = pc_init_pci_1_5,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_5,
-        { /* end of list */ }
-    },
-};
-
-#define PC_I440FX_1_4_MACHINE_OPTIONS \
-    PC_I440FX_1_6_MACHINE_OPTIONS, \
-    .hot_add_cpu = NULL
-
-static QEMUMachine pc_i440fx_machine_v1_4 = {
-    PC_I440FX_1_4_MACHINE_OPTIONS,
-    .name = "pc-i440fx-1.4",
-    .init = pc_init_pci_1_4,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_4,
-        { /* end of list */ }
-    },
-};
 
 #define PC_COMPAT_1_3 \
-	PC_COMPAT_1_4, \
+        PC_CPU_MODEL_IDS("1.3.0") \
         {\
             .driver   = "usb-tablet",\
             .property = "usb_version",\
@@ -611,20 +656,22 @@
             .driver   = "e1000",\
             .property = "autonegotiation",\
             .value    = "off",\
-        }
+        },
 
-static QEMUMachine pc_machine_v1_3 = {
-    PC_I440FX_1_4_MACHINE_OPTIONS,
-    .name = "pc-1.3",
-    .init = pc_init_pci_1_3,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_3,
-        { /* end of list */ }
-    },
-};
+
+static void pc_i440fx_1_3_machine_options(MachineClass *m)
+{
+    pc_i440fx_1_4_machine_options(m);
+    m->hw_version = "1.3.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_3);
+}
+
+DEFINE_I440FX_MACHINE(v1_3, "pc-1.3", pc_compat_1_3,
+                      pc_i440fx_1_3_machine_options);
+
 
 #define PC_COMPAT_1_2 \
-        PC_COMPAT_1_3,\
+        PC_CPU_MODEL_IDS("1.2.0") \
         {\
             .driver   = "nec-usb-xhci",\
             .property = "msi",\
@@ -649,23 +696,21 @@
             .driver   = "VGA",\
             .property = "mmio",\
             .value    = "off",\
-        }
+        },
 
-#define PC_I440FX_1_2_MACHINE_OPTIONS \
-    PC_I440FX_1_4_MACHINE_OPTIONS, \
-    .init = pc_init_pci_1_2
+static void pc_i440fx_1_2_machine_options(MachineClass *m)
+{
+    pc_i440fx_1_3_machine_options(m);
+    m->hw_version = "1.2.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_2);
+}
 
-static QEMUMachine pc_machine_v1_2 = {
-    PC_I440FX_1_2_MACHINE_OPTIONS,
-    .name = "pc-1.2",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_2,
-        { /* end of list */ }
-    },
-};
+DEFINE_I440FX_MACHINE(v1_2, "pc-1.2", pc_compat_1_2,
+                      pc_i440fx_1_2_machine_options);
+
 
 #define PC_COMPAT_1_1 \
-        PC_COMPAT_1_2,\
+        PC_CPU_MODEL_IDS("1.1.0") \
         {\
             .driver   = "virtio-scsi-pci",\
             .property = "hotplug",\
@@ -694,19 +739,21 @@
             .driver   = "virtio-blk-pci",\
             .property = "config-wce",\
             .value    = "off",\
-        }
+        },
 
-static QEMUMachine pc_machine_v1_1 = {
-    PC_I440FX_1_2_MACHINE_OPTIONS,
-    .name = "pc-1.1",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_1,
-        { /* end of list */ }
-    },
-};
+static void pc_i440fx_1_1_machine_options(MachineClass *m)
+{
+    pc_i440fx_1_2_machine_options(m);
+    m->hw_version = "1.1.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_1);
+}
+
+DEFINE_I440FX_MACHINE(v1_1, "pc-1.1", pc_compat_1_2,
+                      pc_i440fx_1_1_machine_options);
+
 
 #define PC_COMPAT_1_0 \
-        PC_COMPAT_1_1,\
+        PC_CPU_MODEL_IDS("1.0") \
         {\
             .driver   = TYPE_ISA_FDC,\
             .property = "check_media_rate",\
@@ -723,33 +770,35 @@
             .driver   = TYPE_USB_DEVICE,\
             .property = "full-path",\
             .value    = "no",\
-        }
+        },
 
-static QEMUMachine pc_machine_v1_0 = {
-    PC_I440FX_1_2_MACHINE_OPTIONS,
-    .name = "pc-1.0",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_0,
-        { /* end of list */ }
-    },
-    .hw_version = "1.0",
-};
+static void pc_i440fx_1_0_machine_options(MachineClass *m)
+{
+    pc_i440fx_1_1_machine_options(m);
+    m->hw_version = "1.0";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_1_0);
+}
+
+DEFINE_I440FX_MACHINE(v1_0, "pc-1.0", pc_compat_1_2,
+                      pc_i440fx_1_0_machine_options);
+
 
 #define PC_COMPAT_0_15 \
-        PC_COMPAT_1_0
+        PC_CPU_MODEL_IDS("0.15")
 
-static QEMUMachine pc_machine_v0_15 = {
-    PC_I440FX_1_2_MACHINE_OPTIONS,
-    .name = "pc-0.15",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_0_15,
-        { /* end of list */ }
-    },
-    .hw_version = "0.15",
-};
+static void pc_i440fx_0_15_machine_options(MachineClass *m)
+{
+    pc_i440fx_1_0_machine_options(m);
+    m->hw_version = "0.15";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_0_15);
+}
+
+DEFINE_I440FX_MACHINE(v0_15, "pc-0.15", pc_compat_1_2,
+                      pc_i440fx_0_15_machine_options);
+
 
 #define PC_COMPAT_0_14 \
-        PC_COMPAT_0_15,\
+        PC_CPU_MODEL_IDS("0.14") \
         {\
             .driver   = "virtio-blk-pci",\
             .property = "event_idx",\
@@ -766,29 +815,29 @@
             .driver   = "virtio-balloon-pci",\
             .property = "event_idx",\
             .value    = "off",\
-        }
-
-static QEMUMachine pc_machine_v0_14 = {
-    PC_I440FX_1_2_MACHINE_OPTIONS,
-    .name = "pc-0.14",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_0_14,
-        {
-            .driver   = "qxl",
-            .property = "revision",
-            .value    = stringify(2),
-        },{
-            .driver   = "qxl-vga",
-            .property = "revision",
-            .value    = stringify(2),
+        },{\
+            .driver   = "qxl",\
+            .property = "revision",\
+            .value    = stringify(2),\
+        },{\
+            .driver   = "qxl-vga",\
+            .property = "revision",\
+            .value    = stringify(2),\
         },
-        { /* end of list */ }
-    },
-    .hw_version = "0.14",
-};
+
+static void pc_i440fx_0_14_machine_options(MachineClass *m)
+{
+    pc_i440fx_0_15_machine_options(m);
+    m->hw_version = "0.14";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_0_14);
+}
+
+DEFINE_I440FX_MACHINE(v0_14, "pc-0.14", pc_compat_1_2,
+                      pc_i440fx_0_14_machine_options);
+
 
 #define PC_COMPAT_0_13 \
-        PC_COMPAT_0_14,\
+        PC_CPU_MODEL_IDS("0.13") \
         {\
             .driver   = TYPE_PCI_DEVICE,\
             .property = "command_serr_enable",\
@@ -797,37 +846,35 @@
             .driver   = "AC97",\
             .property = "use_broken_id",\
             .value    = stringify(1),\
-        }
-
-#define PC_I440FX_0_13_MACHINE_OPTIONS \
-    PC_I440FX_1_2_MACHINE_OPTIONS, \
-    .init = pc_init_pci_no_kvmclock
-
-static QEMUMachine pc_machine_v0_13 = {
-    PC_I440FX_0_13_MACHINE_OPTIONS,
-    .name = "pc-0.13",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_0_13,
-        {
-            .driver   = "virtio-9p-pci",
-            .property = "vectors",
-            .value    = stringify(0),
-        },{
-            .driver   = "VGA",
-            .property = "rombar",
-            .value    = stringify(0),
-        },{
-            .driver   = "vmware-svga",
-            .property = "rombar",
-            .value    = stringify(0),
+        },{\
+            .driver   = "virtio-9p-pci",\
+            .property = "vectors",\
+            .value    = stringify(0),\
+        },{\
+            .driver   = "VGA",\
+            .property = "rombar",\
+            .value    = stringify(0),\
+        },{\
+            .driver   = "vmware-svga",\
+            .property = "rombar",\
+            .value    = stringify(0),\
         },
-        { /* end of list */ }
-    },
-    .hw_version = "0.13",
-};
+
+static void pc_i440fx_0_13_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_i440fx_0_14_machine_options(m);
+    m->hw_version = "0.13";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_0_13);
+    pcmc->kvmclock_enabled = false;
+}
+
+DEFINE_I440FX_MACHINE(v0_13, "pc-0.13", pc_compat_0_13,
+                      pc_i440fx_0_13_machine_options);
+
 
 #define PC_COMPAT_0_12 \
-        PC_COMPAT_0_13,\
+        PC_CPU_MODEL_IDS("0.12") \
         {\
             .driver   = "virtio-serial-pci",\
             .property = "max_ports",\
@@ -848,29 +895,21 @@
             .driver   = "usb-kbd",\
             .property = "serial",\
             .value    = "1",\
-        }
-
-static QEMUMachine pc_machine_v0_12 = {
-    PC_I440FX_0_13_MACHINE_OPTIONS,
-    .name = "pc-0.12",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_0_12,
-        {
-            .driver   = "VGA",
-            .property = "rombar",
-            .value    = stringify(0),
-        },{
-            .driver   = "vmware-svga",
-            .property = "rombar",
-            .value    = stringify(0),
         },
-        { /* end of list */ }
-    },
-    .hw_version = "0.12",
-};
+
+static void pc_i440fx_0_12_machine_options(MachineClass *m)
+{
+    pc_i440fx_0_13_machine_options(m);
+    m->hw_version = "0.12";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_0_12);
+}
+
+DEFINE_I440FX_MACHINE(v0_12, "pc-0.12", pc_compat_0_13,
+                      pc_i440fx_0_12_machine_options);
+
 
 #define PC_COMPAT_0_11 \
-        PC_COMPAT_0_12,\
+        PC_CPU_MODEL_IDS("0.11") \
         {\
             .driver   = "virtio-blk-pci",\
             .property = "vectors",\
@@ -879,104 +918,202 @@
             .driver   = TYPE_PCI_DEVICE,\
             .property = "rombar",\
             .value    = stringify(0),\
-        }
-
-static QEMUMachine pc_machine_v0_11 = {
-    PC_I440FX_0_13_MACHINE_OPTIONS,
-    .name = "pc-0.11",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_0_11,
-        {
-            .driver   = "ide-drive",
-            .property = "ver",
-            .value    = "0.11",
-        },{
-            .driver   = "scsi-disk",
-            .property = "ver",
-            .value    = "0.11",
+        },{\
+            .driver   = "ide-drive",\
+            .property = "ver",\
+            .value    = "0.11",\
+        },{\
+            .driver   = "scsi-disk",\
+            .property = "ver",\
+            .value    = "0.11",\
         },
-        { /* end of list */ }
-    },
-    .hw_version = "0.11",
-};
 
-static QEMUMachine pc_machine_v0_10 = {
-    PC_I440FX_0_13_MACHINE_OPTIONS,
-    .name = "pc-0.10",
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_0_11,
-        {
-            .driver   = "virtio-blk-pci",
-            .property = "class",
-            .value    = stringify(PCI_CLASS_STORAGE_OTHER),
-        },{
-            .driver   = "virtio-serial-pci",
-            .property = "class",
-            .value    = stringify(PCI_CLASS_DISPLAY_OTHER),
-        },{
-            .driver   = "virtio-net-pci",
-            .property = "vectors",
-            .value    = stringify(0),
-        },{
-            .driver   = "ide-drive",
-            .property = "ver",
-            .value    = "0.10",
-        },{
-            .driver   = "scsi-disk",
-            .property = "ver",
-            .value    = "0.10",
-        },
-        { /* end of list */ }
-    },
-    .hw_version = "0.10",
-};
-
-static QEMUMachine isapc_machine = {
-    PC_COMMON_MACHINE_OPTIONS,
-    .name = "isapc",
-    .desc = "ISA-only PC",
-    .init = pc_init_isa,
-    .max_cpus = 1,
-    .compat_props = (GlobalProperty[]) {
-        { /* end of list */ }
-    },
-};
-
-#ifdef CONFIG_XEN
-static QEMUMachine xenfv_machine = {
-    PC_COMMON_MACHINE_OPTIONS,
-    .name = "xenfv",
-    .desc = "Xen Fully-virtualized PC",
-    .init = pc_xen_hvm_init,
-    .max_cpus = HVM_MAX_VCPUS,
-    .default_machine_opts = "accel=xen",
-    .hot_add_cpu = pc_hot_add_cpu,
-};
-#endif
-
-static void pc_machine_init(void)
+static void pc_i440fx_0_11_machine_options(MachineClass *m)
 {
-    qemu_register_pc_machine(&pc_i440fx_machine_v2_2);
-    qemu_register_pc_machine(&pc_i440fx_machine_v2_1);
-    qemu_register_pc_machine(&pc_i440fx_machine_v2_0);
-    qemu_register_pc_machine(&pc_i440fx_machine_v1_7);
-    qemu_register_pc_machine(&pc_i440fx_machine_v1_6);
-    qemu_register_pc_machine(&pc_i440fx_machine_v1_5);
-    qemu_register_pc_machine(&pc_i440fx_machine_v1_4);
-    qemu_register_pc_machine(&pc_machine_v1_3);
-    qemu_register_pc_machine(&pc_machine_v1_2);
-    qemu_register_pc_machine(&pc_machine_v1_1);
-    qemu_register_pc_machine(&pc_machine_v1_0);
-    qemu_register_pc_machine(&pc_machine_v0_15);
-    qemu_register_pc_machine(&pc_machine_v0_14);
-    qemu_register_pc_machine(&pc_machine_v0_13);
-    qemu_register_pc_machine(&pc_machine_v0_12);
-    qemu_register_pc_machine(&pc_machine_v0_11);
-    qemu_register_pc_machine(&pc_machine_v0_10);
-    qemu_register_pc_machine(&isapc_machine);
-#ifdef CONFIG_XEN
-    qemu_register_pc_machine(&xenfv_machine);
-#endif
+    pc_i440fx_0_12_machine_options(m);
+    m->hw_version = "0.11";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_0_11);
 }
 
-machine_init(pc_machine_init);
+DEFINE_I440FX_MACHINE(v0_11, "pc-0.11", pc_compat_0_13,
+                      pc_i440fx_0_11_machine_options);
+
+
+#define PC_COMPAT_0_10 \
+    PC_CPU_MODEL_IDS("0.10") \
+    {\
+        .driver   = "virtio-blk-pci",\
+        .property = "class",\
+        .value    = stringify(PCI_CLASS_STORAGE_OTHER),\
+    },{\
+        .driver   = "virtio-serial-pci",\
+        .property = "class",\
+        .value    = stringify(PCI_CLASS_DISPLAY_OTHER),\
+    },{\
+        .driver   = "virtio-net-pci",\
+        .property = "vectors",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-drive",\
+        .property = "ver",\
+        .value    = "0.10",\
+    },{\
+        .driver   = "scsi-disk",\
+        .property = "ver",\
+        .value    = "0.10",\
+    },
+
+static void pc_i440fx_0_10_machine_options(MachineClass *m)
+{
+    pc_i440fx_0_11_machine_options(m);
+    m->hw_version = "0.10";
+    SET_MACHINE_COMPAT(m, PC_COMPAT_0_10);
+}
+
+DEFINE_I440FX_MACHINE(v0_10, "pc-0.10", pc_compat_0_13,
+                      pc_i440fx_0_10_machine_options);
+
+typedef struct {
+    uint16_t gpu_device_id;
+    uint16_t pch_device_id;
+    uint8_t pch_revision_id;
+} IGDDeviceIDInfo;
+
+/* In real world different GPU should have different PCH. But actually
+ * the different PCH DIDs likely map to different PCH SKUs. We do the
+ * same thing for the GPU. For PCH, the different SKUs are going to be
+ * all the same silicon design and implementation, just different
+ * features turn on and off with fuses. The SW interfaces should be
+ * consistent across all SKUs in a given family (eg LPT). But just same
+ * features may not be supported.
+ *
+ * Most of these different PCH features probably don't matter to the
+ * Gfx driver, but obviously any difference in display port connections
+ * will so it should be fine with any PCH in case of passthrough.
+ *
+ * So currently use one PCH version, 0x8c4e, to cover all HSW(Haswell)
+ * scenarios, 0x9cc3 for BDW(Broadwell).
+ */
+static const IGDDeviceIDInfo igd_combo_id_infos[] = {
+    /* HSW Classic */
+    {0x0402, 0x8c4e, 0x04}, /* HSWGT1D, HSWD_w7 */
+    {0x0406, 0x8c4e, 0x04}, /* HSWGT1M, HSWM_w7 */
+    {0x0412, 0x8c4e, 0x04}, /* HSWGT2D, HSWD_w7 */
+    {0x0416, 0x8c4e, 0x04}, /* HSWGT2M, HSWM_w7 */
+    {0x041E, 0x8c4e, 0x04}, /* HSWGT15D, HSWD_w7 */
+    /* HSW ULT */
+    {0x0A06, 0x8c4e, 0x04}, /* HSWGT1UT, HSWM_w7 */
+    {0x0A16, 0x8c4e, 0x04}, /* HSWGT2UT, HSWM_w7 */
+    {0x0A26, 0x8c4e, 0x06}, /* HSWGT3UT, HSWM_w7 */
+    {0x0A2E, 0x8c4e, 0x04}, /* HSWGT3UT28W, HSWM_w7 */
+    {0x0A1E, 0x8c4e, 0x04}, /* HSWGT2UX, HSWM_w7 */
+    {0x0A0E, 0x8c4e, 0x04}, /* HSWGT1ULX, HSWM_w7 */
+    /* HSW CRW */
+    {0x0D26, 0x8c4e, 0x04}, /* HSWGT3CW, HSWM_w7 */
+    {0x0D22, 0x8c4e, 0x04}, /* HSWGT3CWDT, HSWD_w7 */
+    /* HSW Server */
+    {0x041A, 0x8c4e, 0x04}, /* HSWSVGT2, HSWD_w7 */
+    /* HSW SRVR */
+    {0x040A, 0x8c4e, 0x04}, /* HSWSVGT1, HSWD_w7 */
+    /* BSW */
+    {0x1606, 0x9cc3, 0x03}, /* BDWULTGT1, BDWM_w7 */
+    {0x1616, 0x9cc3, 0x03}, /* BDWULTGT2, BDWM_w7 */
+    {0x1626, 0x9cc3, 0x03}, /* BDWULTGT3, BDWM_w7 */
+    {0x160E, 0x9cc3, 0x03}, /* BDWULXGT1, BDWM_w7 */
+    {0x161E, 0x9cc3, 0x03}, /* BDWULXGT2, BDWM_w7 */
+    {0x1602, 0x9cc3, 0x03}, /* BDWHALOGT1, BDWM_w7 */
+    {0x1612, 0x9cc3, 0x03}, /* BDWHALOGT2, BDWM_w7 */
+    {0x1622, 0x9cc3, 0x03}, /* BDWHALOGT3, BDWM_w7 */
+    {0x162B, 0x9cc3, 0x03}, /* BDWHALO28W, BDWM_w7 */
+    {0x162A, 0x9cc3, 0x03}, /* BDWGT3WRKS, BDWM_w7 */
+    {0x162D, 0x9cc3, 0x03}, /* BDWGT3SRVR, BDWM_w7 */
+};
+
+static void isa_bridge_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    dc->desc        = "ISA bridge faked to support IGD PT";
+    k->vendor_id    = PCI_VENDOR_ID_INTEL;
+    k->class_id     = PCI_CLASS_BRIDGE_ISA;
+};
+
+static TypeInfo isa_bridge_info = {
+    .name          = "igd-passthrough-isa-bridge",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIDevice),
+    .class_init = isa_bridge_class_init,
+};
+
+static void pt_graphics_register_types(void)
+{
+    type_register_static(&isa_bridge_info);
+}
+type_init(pt_graphics_register_types)
+
+void igd_passthrough_isa_bridge_create(PCIBus *bus, uint16_t gpu_dev_id)
+{
+    struct PCIDevice *bridge_dev;
+    int i, num;
+    uint16_t pch_dev_id = 0xffff;
+    uint8_t pch_rev_id;
+
+    num = ARRAY_SIZE(igd_combo_id_infos);
+    for (i = 0; i < num; i++) {
+        if (gpu_dev_id == igd_combo_id_infos[i].gpu_device_id) {
+            pch_dev_id = igd_combo_id_infos[i].pch_device_id;
+            pch_rev_id = igd_combo_id_infos[i].pch_revision_id;
+        }
+    }
+
+    if (pch_dev_id == 0xffff) {
+        return;
+    }
+
+    /* Currently IGD drivers always need to access PCH by 1f.0. */
+    bridge_dev = pci_create_simple(bus, PCI_DEVFN(0x1f, 0),
+                                   "igd-passthrough-isa-bridge");
+
+    /*
+     * Note that vendor id is always PCI_VENDOR_ID_INTEL.
+     */
+    if (!bridge_dev) {
+        fprintf(stderr, "set igd-passthrough-isa-bridge failed!\n");
+        return;
+    }
+    pci_config_set_device_id(bridge_dev->config, pch_dev_id);
+    pci_config_set_revision(bridge_dev->config, pch_rev_id);
+}
+
+static void isapc_machine_options(MachineClass *m)
+{
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    m->desc = "ISA-only PC";
+    m->max_cpus = 1;
+    m->option_rom_has_mr = true;
+    m->rom_file_has_mr = false;
+    pcmc->pci_enabled = false;
+    pcmc->has_acpi_build = false;
+    pcmc->smbios_defaults = false;
+    pcmc->gigabyte_align = false;
+    pcmc->smbios_legacy_mode = true;
+    pcmc->has_reserved_memory = false;
+}
+
+DEFINE_PC_MACHINE(isapc, "isapc", pc_init_isa,
+                  isapc_machine_options);
+
+
+#ifdef CONFIG_XEN
+static void xenfv_machine_options(MachineClass *m)
+{
+    m->desc = "Xen Fully-virtualized PC";
+    m->max_cpus = HVM_MAX_VCPUS;
+    m->default_machine_opts = "accel=xen";
+    m->hot_add_cpu = pc_hot_add_cpu;
+}
+
+DEFINE_PC_MACHINE(xenfv, "xenfv", pc_xen_hvm_init,
+                  xenfv_machine_options);
+#endif
diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index 0262b5e..c0b9961 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -27,6 +27,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/loader.h"
 #include "sysemu/arch_init.h"
@@ -38,56 +39,44 @@
 #include "hw/kvm/clock.h"
 #include "hw/pci-host/q35.h"
 #include "exec/address-spaces.h"
+#include "hw/i386/pc.h"
 #include "hw/i386/ich9.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/ide/pci.h"
 #include "hw/ide/ahci.h"
 #include "hw/usb.h"
-#include "hw/cpu/icc_bus.h"
 #include "qemu/error-report.h"
+#include "migration/migration.h"
 
 /* ICH9 AHCI has 6 ports */
 #define MAX_SATA_PORTS     6
 
-static bool has_acpi_build = true;
-static bool smbios_defaults = true;
-static bool smbios_legacy_mode;
-static bool smbios_uuid_encoded = true;
-/* Make sure that guest addresses aligned at 1Gbyte boundaries get mapped to
- * host addresses aligned at 1Gbyte boundaries.  This way we can use 1GByte
- * pages in the host.
- */
-static bool gigabyte_align = true;
-static bool has_reserved_memory = true;
-
 /* PC hardware initialisation */
 static void pc_q35_init(MachineState *machine)
 {
-    PCMachineState *pc_machine = PC_MACHINE(machine);
-    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+    PCMachineState *pcms = PC_MACHINE(machine);
+    PCMachineClass *pcmc = PC_MACHINE_GET_CLASS(pcms);
     Q35PCIHost *q35_host;
     PCIHostState *phb;
     PCIBus *host_bus;
     PCIDevice *lpc;
+    DeviceState *lpc_dev;
     BusState *idebus[MAX_SATA_PORTS];
     ISADevice *rtc_state;
-    ISADevice *floppy;
+    MemoryRegion *system_io = get_system_io();
     MemoryRegion *pci_memory;
     MemoryRegion *rom_memory;
     MemoryRegion *ram_memory;
     GSIState *gsi_state;
     ISABus *isa_bus;
-    int pci_enabled = 1;
-    qemu_irq *cpu_irq;
     qemu_irq *gsi;
     qemu_irq *i8259;
     int i;
     ICH9LPCState *ich9_lpc;
     PCIDevice *ahci;
-    DeviceState *icc_bridge;
-    PcGuestInfo *guest_info;
     ram_addr_t lowmem;
     DriveInfo *hd[MAX_SATA_PORTS];
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
 
     /* Check whether RAM fits below 4G (leaving 1/2 GByte for IO memory
      * and 256 Mbytes for PCI Express Enhanced Configuration Access Mapping
@@ -95,11 +84,9 @@
      * If it doesn't, we need to split it in chunks below and above 4G.
      * In any case, try to make sure that guest addresses aligned at
      * 1G boundaries get mapped to host addresses aligned at 1G boundaries.
-     * For old machine types, use whatever split we used historically to avoid
-     * breaking migration.
      */
     if (machine->ram_size >= 0xb0000000) {
-        lowmem = gigabyte_align ? 0x80000000 : 0xb0000000;
+        lowmem = 0x80000000;
     } else {
         lowmem = 0xb0000000;
     }
@@ -107,41 +94,37 @@
     /* Handle the machine opt max-ram-below-4g.  It is basically doing
      * min(qemu limit, user limit).
      */
-    if (lowmem > pc_machine->max_ram_below_4g) {
-        lowmem = pc_machine->max_ram_below_4g;
+    if (!pcms->max_ram_below_4g) {
+        pcms->max_ram_below_4g = 1ULL << 32; /* default: 4G */;
+    }
+    if (lowmem > pcms->max_ram_below_4g) {
+        lowmem = pcms->max_ram_below_4g;
         if (machine->ram_size - lowmem > lowmem &&
             lowmem & ((1ULL << 30) - 1)) {
             error_report("Warning: Large machine and max_ram_below_4g(%"PRIu64
                          ") not a multiple of 1G; possible bad performance.",
-                         pc_machine->max_ram_below_4g);
+                         pcms->max_ram_below_4g);
         }
     }
 
     if (machine->ram_size >= lowmem) {
-        above_4g_mem_size = machine->ram_size - lowmem;
-        below_4g_mem_size = lowmem;
+        pcms->above_4g_mem_size = machine->ram_size - lowmem;
+        pcms->below_4g_mem_size = lowmem;
     } else {
-        above_4g_mem_size = 0;
-        below_4g_mem_size = machine->ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = machine->ram_size;
     }
 
-    if (xen_enabled() && xen_hvm_init(&below_4g_mem_size, &above_4g_mem_size,
-                                      &ram_memory) != 0) {
-        fprintf(stderr, "xen hardware virtual machine initialisation failed\n");
-        exit(1);
+    if (xen_enabled()) {
+        xen_hvm_init(pcms, &ram_memory);
     }
 
-    icc_bridge = qdev_create(NULL, TYPE_ICC_BRIDGE);
-    object_property_add_child(qdev_get_machine(), "icc-bridge",
-                              OBJECT(icc_bridge), NULL);
-
-    pc_cpus_init(machine->cpu_model, icc_bridge);
-    pc_acpi_init("q35-acpi-dsdt.aml");
+    pc_cpus_init(pcms);
 
     kvmclock_create();
 
     /* pci enabled */
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pci_memory = g_new(MemoryRegion, 1);
         memory_region_init(pci_memory, NULL, "pci", UINT64_MAX);
         rom_memory = pci_memory;
@@ -150,34 +133,26 @@
         rom_memory = get_system_memory();
     }
 
-    guest_info = pc_guest_info_init(below_4g_mem_size, above_4g_mem_size);
-    guest_info->isapc_ram_fw = false;
-    guest_info->has_acpi_build = has_acpi_build;
-    guest_info->has_reserved_memory = has_reserved_memory;
+    pc_guest_info_init(pcms);
 
-    /* Migration was not supported in 2.0 for Q35, so do not bother
-     * with this hack (see hw/i386/acpi-build.c).
-     */
-    guest_info->legacy_acpi_table_size = 0;
-
-    if (smbios_defaults) {
-        MachineClass *mc = MACHINE_GET_CLASS(machine);
+    if (pcmc->smbios_defaults) {
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (Q35 + ICH9, 2009)",
-                            mc->name, smbios_legacy_mode, smbios_uuid_encoded);
+                            mc->name, pcmc->smbios_legacy_mode,
+                            pcmc->smbios_uuid_encoded,
+                            SMBIOS_ENTRY_POINT_21);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
-        pc_memory_init(machine, get_system_memory(),
-                       below_4g_mem_size, above_4g_mem_size,
-                       rom_memory, &ram_memory, guest_info);
+        pc_memory_init(pcms, get_system_memory(),
+                       rom_memory, &ram_memory);
     }
 
     /* irq lines */
     gsi_state = g_malloc0(sizeof(*gsi_state));
-    if (kvm_irqchip_in_kernel()) {
-        kvm_pc_setup_irq_routing(pci_enabled);
+    if (kvm_ioapic_in_kernel()) {
+        kvm_pc_setup_irq_routing(pcmc->pci_enabled);
         gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
                                  GSI_NUM_PINS);
     } else {
@@ -188,13 +163,18 @@
     q35_host = Q35_HOST_DEVICE(qdev_create(NULL, TYPE_Q35_HOST_DEVICE));
 
     object_property_add_child(qdev_get_machine(), "q35", OBJECT(q35_host), NULL);
-    q35_host->mch.ram_memory = ram_memory;
-    q35_host->mch.pci_address_space = pci_memory;
-    q35_host->mch.system_memory = get_system_memory();
-    q35_host->mch.address_space_io = get_system_io();
-    q35_host->mch.below_4g_mem_size = below_4g_mem_size;
-    q35_host->mch.above_4g_mem_size = above_4g_mem_size;
-    q35_host->mch.guest_info = guest_info;
+    object_property_set_link(OBJECT(q35_host), OBJECT(ram_memory),
+                             MCH_HOST_PROP_RAM_MEM, NULL);
+    object_property_set_link(OBJECT(q35_host), OBJECT(pci_memory),
+                             MCH_HOST_PROP_PCI_MEM, NULL);
+    object_property_set_link(OBJECT(q35_host), OBJECT(get_system_memory()),
+                             MCH_HOST_PROP_SYSTEM_MEM, NULL);
+    object_property_set_link(OBJECT(q35_host), OBJECT(system_io),
+                             MCH_HOST_PROP_IO_MEM, NULL);
+    object_property_set_int(OBJECT(q35_host), pcms->below_4g_mem_size,
+                            PCI_HOST_BELOW_4G_MEM_SIZE, NULL);
+    object_property_set_int(OBJECT(q35_host), pcms->above_4g_mem_size,
+                            PCI_HOST_ABOVE_4G_MEM_SIZE, NULL);
     /* pci */
     qdev_init_nofail(DEVICE(q35_host));
     phb = PCI_HOST_BRIDGE(q35_host);
@@ -206,53 +186,50 @@
 
     object_property_add_link(OBJECT(machine), PC_MACHINE_ACPI_DEVICE_PROP,
                              TYPE_HOTPLUG_HANDLER,
-                             (Object **)&pc_machine->acpi_dev,
+                             (Object **)&pcms->acpi_dev,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE, &error_abort);
     object_property_set_link(OBJECT(machine), OBJECT(lpc),
                              PC_MACHINE_ACPI_DEVICE_PROP, &error_abort);
 
     ich9_lpc = ICH9_LPC_DEVICE(lpc);
-    ich9_lpc->pic = gsi;
-    ich9_lpc->ioapic = gsi_state->ioapic_irq;
+    lpc_dev = DEVICE(lpc);
+    for (i = 0; i < GSI_NUM_PINS; i++) {
+        qdev_connect_gpio_out_named(lpc_dev, ICH9_GPIO_GSI, i, gsi[i]);
+    }
     pci_bus_irqs(host_bus, ich9_lpc_set_irq, ich9_lpc_map_irq, ich9_lpc,
                  ICH9_LPC_NB_PIRQS);
     pci_bus_set_route_irq_fn(host_bus, ich9_route_intx_pin_to_irq);
     isa_bus = ich9_lpc->isa_bus;
 
-    /*end early*/
-    isa_bus_irqs(isa_bus, gsi);
-
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_pic_in_kernel()) {
         i8259 = kvm_i8259_init(isa_bus);
     } else if (xen_enabled()) {
         i8259 = xen_interrupt_controller_init();
     } else {
-        cpu_irq = pc_allocate_cpu_irq();
-        i8259 = i8259_init(isa_bus, cpu_irq[0]);
+        i8259 = i8259_init(isa_bus, pc_allocate_cpu_irq());
     }
 
     for (i = 0; i < ISA_NUM_IRQS; i++) {
         gsi_state->i8259_irq[i] = i8259[i];
     }
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         ioapic_init_gsi(gsi_state, "q35");
     }
-    qdev_init_nofail(icc_bridge);
 
     pc_register_ferr_irq(gsi[13]);
 
-    assert(pc_machine->vmport != ON_OFF_AUTO_MAX);
-    if (pc_machine->vmport == ON_OFF_AUTO_AUTO) {
-        pc_machine->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
+    assert(pcms->vmport != ON_OFF_AUTO__MAX);
+    if (pcms->vmport == ON_OFF_AUTO_AUTO) {
+        pcms->vmport = xen_enabled() ? ON_OFF_AUTO_OFF : ON_OFF_AUTO_ON;
     }
 
     /* init basic PC hardware */
-    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy,
-                         (pc_machine->vmport != ON_OFF_AUTO_ON), 0xff0104);
+    pc_basic_device_init(isa_bus, gsi, &rtc_state, !mc->no_floppy,
+                         (pcms->vmport != ON_OFF_AUTO_ON), 0xff0104);
 
     /* connect pm stuff to lpc */
-    ich9_lpc_pm_init(lpc);
+    ich9_lpc_pm_init(lpc, pc_machine_is_smm_enabled(pcms));
 
     /* ahci and SATA device, for q35 1 ahci controller is built-in */
     ahci = pci_create_simple_multifunction(host_bus,
@@ -265,7 +242,7 @@
     ide_drive_get(hd, ICH_AHCI(ahci)->ahci.ports);
     ahci_ide_create_devs(ahci, hd);
 
-    if (usb_enabled(false)) {
+    if (machine_usb(machine)) {
         /* Should we create 6 UHCI according to ich9 spec? */
         ehci_create_ich9_with_companions(host_bus, 0x1d);
     }
@@ -276,202 +253,86 @@
                                     0xb100),
                       8, NULL, 0);
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
-                 machine, floppy, idebus[0], idebus[1], rtc_state);
+    pc_cmos_init(pcms, idebus[0], idebus[1], rtc_state);
 
     /* the rest devices to which pci devfn is automatically assigned */
     pc_vga_init(isa_bus, host_bus);
     pc_nic_init(isa_bus, host_bus);
-    if (pci_enabled) {
+    if (pcmc->pci_enabled) {
         pc_pci_device_init(host_bus);
     }
+
+    if (pcms->acpi_nvdimm_state.is_enabled) {
+        nvdimm_init_acpi_state(&pcms->acpi_nvdimm_state, system_io,
+                               pcms->fw_cfg, OBJECT(pcms));
+    }
 }
 
-static void pc_compat_2_1(MachineState *machine)
-{
-    PCMachineState *pcms = PC_MACHINE(machine);
+#define DEFINE_Q35_MACHINE(suffix, name, compatfn, optionfn) \
+    static void pc_init_##suffix(MachineState *machine) \
+    { \
+        void (*compat)(MachineState *m) = (compatfn); \
+        if (compat) { \
+            compat(machine); \
+        } \
+        pc_q35_init(machine); \
+    } \
+    DEFINE_PC_MACHINE(suffix, name, pc_init_##suffix, optionfn)
 
-    pcms->enforce_aligned_dimm = false;
-    smbios_uuid_encoded = false;
-    x86_cpu_compat_set_features("coreduo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_set_features("core2duo", FEAT_1_ECX, CPUID_EXT_VMX, 0);
-    x86_cpu_compat_kvm_no_autodisable(FEAT_8000_0001_ECX, CPUID_EXT3_SVM);
+
+static void pc_q35_machine_options(MachineClass *m)
+{
+    m->family = "pc_q35";
+    m->desc = "Standard PC (Q35 + ICH9, 2009)";
+    m->hot_add_cpu = pc_hot_add_cpu;
+    m->units_per_default_bus = 1;
+    m->default_machine_opts = "firmware=bios-256k.bin";
+    m->default_display = "std";
+    m->no_floppy = 1;
+    m->has_dynamic_sysbus = true;
 }
 
-static void pc_compat_2_0(MachineState *machine)
+static void pc_q35_2_7_machine_options(MachineClass *m)
 {
-    pc_compat_2_1(machine);
-    smbios_legacy_mode = true;
-    has_reserved_memory = false;
-    pc_set_legacy_acpi_data_size();
+    pc_q35_machine_options(m);
+    m->alias = "q35";
 }
 
-static void pc_compat_1_7(MachineState *machine)
+DEFINE_Q35_MACHINE(v2_7, "pc-q35-2.7", NULL,
+                   pc_q35_2_7_machine_options);
+
+static void pc_q35_2_6_machine_options(MachineClass *m)
 {
-    pc_compat_2_0(machine);
-    smbios_defaults = false;
-    gigabyte_align = false;
-    option_rom_has_mr = true;
-    x86_cpu_compat_kvm_no_autoenable(FEAT_1_ECX, CPUID_EXT_X2APIC);
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_q35_2_7_machine_options(m);
+    m->alias = NULL;
+    pcmc->legacy_cpu_hotplug = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_6);
 }
 
-static void pc_compat_1_6(MachineState *machine)
+DEFINE_Q35_MACHINE(v2_6, "pc-q35-2.6", NULL,
+                   pc_q35_2_6_machine_options);
+
+static void pc_q35_2_5_machine_options(MachineClass *m)
 {
-    pc_compat_1_7(machine);
-    rom_file_has_mr = false;
-    has_acpi_build = false;
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_q35_2_6_machine_options(m);
+    pcmc->save_tsc_khz = false;
+    m->legacy_fw_cfg_order = 1;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_5);
 }
 
-static void pc_compat_1_5(MachineState *machine)
+DEFINE_Q35_MACHINE(v2_5, "pc-q35-2.5", NULL,
+                   pc_q35_2_5_machine_options);
+
+static void pc_q35_2_4_machine_options(MachineClass *m)
 {
-    pc_compat_1_6(machine);
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(m);
+    pc_q35_2_5_machine_options(m);
+    m->hw_version = "2.4.0";
+    pcmc->broken_reserved_end = true;
+    SET_MACHINE_COMPAT(m, PC_COMPAT_2_4);
 }
 
-static void pc_compat_1_4(MachineState *machine)
-{
-    pc_compat_1_5(machine);
-    x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
-    x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
-}
-
-static void pc_q35_init_2_1(MachineState *machine)
-{
-    pc_compat_2_1(machine);
-    pc_q35_init(machine);
-}
-
-static void pc_q35_init_2_0(MachineState *machine)
-{
-    pc_compat_2_0(machine);
-    pc_q35_init(machine);
-}
-
-static void pc_q35_init_1_7(MachineState *machine)
-{
-    pc_compat_1_7(machine);
-    pc_q35_init(machine);
-}
-
-static void pc_q35_init_1_6(MachineState *machine)
-{
-    pc_compat_1_6(machine);
-    pc_q35_init(machine);
-}
-
-static void pc_q35_init_1_5(MachineState *machine)
-{
-    pc_compat_1_5(machine);
-    pc_q35_init(machine);
-}
-
-static void pc_q35_init_1_4(MachineState *machine)
-{
-    pc_compat_1_4(machine);
-    pc_q35_init(machine);
-}
-
-#define PC_Q35_MACHINE_OPTIONS \
-    PC_DEFAULT_MACHINE_OPTIONS, \
-    .family = "pc_q35", \
-    .desc = "Standard PC (Q35 + ICH9, 2009)", \
-    .hot_add_cpu = pc_hot_add_cpu, \
-    .units_per_default_bus = 1
-
-#define PC_Q35_2_2_MACHINE_OPTIONS                      \
-    PC_Q35_MACHINE_OPTIONS,                             \
-    .default_machine_opts = "firmware=bios-256k.bin",   \
-    .default_display = "std"
-
-static QEMUMachine pc_q35_machine_v2_2 = {
-    PC_Q35_2_2_MACHINE_OPTIONS,
-    .name = "pc-q35-2.2",
-    .alias = "q35",
-    .init = pc_q35_init,
-};
-
-#define PC_Q35_2_1_MACHINE_OPTIONS                      \
-    PC_Q35_MACHINE_OPTIONS,                             \
-    .default_machine_opts = "firmware=bios-256k.bin"
-
-static QEMUMachine pc_q35_machine_v2_1 = {
-    PC_Q35_2_1_MACHINE_OPTIONS,
-    .name = "pc-q35-2.1",
-    .init = pc_q35_init_2_1,
-    .compat_props = (GlobalProperty[]) {
-        HW_COMPAT_2_1,
-        { /* end of list */ }
-    },
-};
-
-#define PC_Q35_2_0_MACHINE_OPTIONS PC_Q35_2_1_MACHINE_OPTIONS
-
-static QEMUMachine pc_q35_machine_v2_0 = {
-    PC_Q35_2_0_MACHINE_OPTIONS,
-    .name = "pc-q35-2.0",
-    .init = pc_q35_init_2_0,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_2_0,
-        { /* end of list */ }
-    },
-};
-
-#define PC_Q35_1_7_MACHINE_OPTIONS PC_Q35_MACHINE_OPTIONS
-
-static QEMUMachine pc_q35_machine_v1_7 = {
-    PC_Q35_1_7_MACHINE_OPTIONS,
-    .name = "pc-q35-1.7",
-    .init = pc_q35_init_1_7,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_7,
-        { /* end of list */ }
-    },
-};
-
-#define PC_Q35_1_6_MACHINE_OPTIONS PC_Q35_MACHINE_OPTIONS
-
-static QEMUMachine pc_q35_machine_v1_6 = {
-    PC_Q35_1_6_MACHINE_OPTIONS,
-    .name = "pc-q35-1.6",
-    .init = pc_q35_init_1_6,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_6,
-        { /* end of list */ }
-    },
-};
-
-static QEMUMachine pc_q35_machine_v1_5 = {
-    PC_Q35_1_6_MACHINE_OPTIONS,
-    .name = "pc-q35-1.5",
-    .init = pc_q35_init_1_5,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_5,
-        { /* end of list */ }
-    },
-};
-
-#define PC_Q35_1_4_MACHINE_OPTIONS \
-    PC_Q35_1_6_MACHINE_OPTIONS, \
-    .hot_add_cpu = NULL
-
-static QEMUMachine pc_q35_machine_v1_4 = {
-    PC_Q35_1_4_MACHINE_OPTIONS,
-    .name = "pc-q35-1.4",
-    .init = pc_q35_init_1_4,
-    .compat_props = (GlobalProperty[]) {
-        PC_COMPAT_1_4,
-        { /* end of list */ }
-    },
-};
-
-static void pc_q35_machine_init(void)
-{
-    qemu_register_pc_machine(&pc_q35_machine_v2_2);
-    qemu_register_pc_machine(&pc_q35_machine_v2_1);
-    qemu_register_pc_machine(&pc_q35_machine_v2_0);
-    qemu_register_pc_machine(&pc_q35_machine_v1_7);
-    qemu_register_pc_machine(&pc_q35_machine_v1_6);
-    qemu_register_pc_machine(&pc_q35_machine_v1_5);
-    qemu_register_pc_machine(&pc_q35_machine_v1_4);
-}
-
-machine_init(pc_q35_machine_init);
+DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL,
+                   pc_q35_2_4_machine_options);
diff --git a/hw/i386/pc_sysfw.c b/hw/i386/pc_sysfw.c
index 75913c5..f915ad0 100644
--- a/hw/i386/pc_sysfw.c
+++ b/hw/i386/pc_sysfw.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/block-backend.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
@@ -56,7 +58,7 @@
     isa_bios_size = MIN(flash_size, 128 * 1024);
     isa_bios = g_malloc(sizeof(*isa_bios));
     memory_region_init_ram(isa_bios, NULL, "isa-bios", isa_bios_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(isa_bios);
     memory_region_add_subregion_overlap(rom_memory,
                                         0x100000 - isa_bios_size,
@@ -193,7 +195,7 @@
         goto bios_error;
     }
     bios = g_malloc(sizeof(*bios));
-    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_abort);
+    memory_region_init_ram(bios, NULL, "pc.bios", bios_size, &error_fatal);
     vmstate_register_ram_global(bios);
     if (!isapc_ram_fw) {
         memory_region_set_readonly(bios, true);
@@ -204,9 +206,7 @@
         fprintf(stderr, "qemu: could not load PC BIOS '%s'\n", bios_name);
         exit(1);
     }
-    if (filename) {
-        g_free(filename);
-    }
+    g_free(filename);
 
     /* map the last 128KB of the BIOS in ISA space */
     isa_bios_size = bios_size;
diff --git a/hw/i386/pci-assign-load-rom.c b/hw/i386/pci-assign-load-rom.c
new file mode 100644
index 0000000..fb66caa
--- /dev/null
+++ b/hw/i386/pci-assign-load-rom.c
@@ -0,0 +1,88 @@
+/*
+ * This is splited from hw/i386/kvm/pci-assign.c
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "qemu/error-report.h"
+#include "ui/console.h"
+#include "hw/loader.h"
+#include "monitor/monitor.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci-assign.h"
+
+/*
+ * Scan the assigned devices for the devices that have an option ROM, and then
+ * load the corresponding ROM data to RAM. If an error occurs while loading an
+ * option ROM, we just ignore that option ROM and continue with the next one.
+ */
+void *pci_assign_dev_load_option_rom(PCIDevice *dev, struct Object *owner,
+                                     int *size, unsigned int domain,
+                                     unsigned int bus, unsigned int slot,
+                                     unsigned int function)
+{
+    char name[32], rom_file[64];
+    FILE *fp;
+    uint8_t val;
+    struct stat st;
+    void *ptr = NULL;
+
+    /* If loading ROM from file, pci handles it */
+    if (dev->romfile || !dev->rom_bar) {
+        return NULL;
+    }
+
+    snprintf(rom_file, sizeof(rom_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
+             domain, bus, slot, function);
+
+    if (qemu_stat(rom_file, &st)) {
+        if (errno != ENOENT) {
+            error_report("pci-assign: Invalid ROM.");
+        }
+        return NULL;
+    }
+
+    /* Write "1" to the ROM file to enable it */
+    fp = fopen(rom_file, "r+");
+    if (fp == NULL) {
+        error_report("pci-assign: Cannot open %s: %s", rom_file, strerror(errno));
+        return NULL;
+    }
+    val = 1;
+    if (fwrite(&val, 1, 1, fp) != 1) {
+        goto close_rom;
+    }
+    fseek(fp, 0, SEEK_SET);
+
+    snprintf(name, sizeof(name), "%s.rom", object_get_typename(owner));
+    memory_region_init_ram(&dev->rom, owner, name, st.st_size, &error_abort);
+    vmstate_register_ram(&dev->rom, &dev->qdev);
+    ptr = memory_region_get_ram_ptr(&dev->rom);
+    memset(ptr, 0xff, st.st_size);
+
+    if (!fread(ptr, 1, st.st_size, fp)) {
+        error_report("pci-assign: Cannot read from host %s", rom_file);
+        error_printf("Device option ROM contents are probably invalid "
+                     "(check dmesg).\nSkip option ROM probe with rombar=0, "
+                     "or load from file with romfile=\n");
+        goto close_rom;
+    }
+
+    pci_register_bar(dev, PCI_ROM_SLOT, 0, &dev->rom);
+    dev->has_rom = true;
+    *size = st.st_size;
+close_rom:
+    /* Write "0" to disable ROM */
+    fseek(fp, 0, SEEK_SET);
+    val = 0;
+    if (!fwrite(&val, 1, 1, fp)) {
+        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
+    }
+    fclose(fp);
+
+    return ptr;
+}
diff --git a/hw/i386/q35-acpi-dsdt.dsl b/hw/i386/q35-acpi-dsdt.dsl
deleted file mode 100644
index e1cee5d..0000000
--- a/hw/i386/q35-acpi-dsdt.dsl
+++ /dev/null
@@ -1,454 +0,0 @@
-/*
- * Bochs/QEMU ACPI DSDT ASL definition
- *
- * Copyright (c) 2006 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License version 2 as published by the Free Software Foundation.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
- */
-/*
- * Copyright (c) 2010 Isaku Yamahata
- *                    yamahata at valinux co jp
- * Based on acpi-dsdt.dsl, but heavily modified for q35 chipset.
- */
-
-ACPI_EXTRACT_ALL_CODE Q35AcpiDsdtAmlCode
-
-DefinitionBlock (
-    "q35-acpi-dsdt.aml",// Output Filename
-    "DSDT",             // Signature
-    0x01,               // DSDT Compliance Revision
-    "BXPC",             // OEMID
-    "BXDSDT",           // TABLE ID
-    0x2                 // OEM Revision
-    )
-{
-
-#include "acpi-dsdt-dbug.dsl"
-
-    Scope(\_SB) {
-        OperationRegion(PCST, SystemIO, 0xae00, 0x0c)
-        OperationRegion(PCSB, SystemIO, 0xae0c, 0x01)
-        Field(PCSB, AnyAcc, NoLock, WriteAsZeros) {
-            PCIB, 8,
-        }
-    }
-
-
-/****************************************************************
- * PCI Bus definition
- ****************************************************************/
-#define BOARD_SPECIFIC_PCI_RESOURSES \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0x0000, \
-         0x0CD7, \
-         0x0000, \
-         0x0CD8, \
-         ,, , TypeStatic) \
-     /* 0xcd8-0xcf7 hole for CPU hotplug, hw/acpi/ich9.c:ICH9_PROC_BASE */ \
-     WordIO(ResourceProducer, MinFixed, MaxFixed, PosDecode, EntireRange, \
-         0x0000, \
-         0x0D00, \
-         0xFFFF, \
-         0x0000, \
-         0xF300, \
-         ,, , TypeStatic)
-
-    Scope(\_SB) {
-        Device(PCI0) {
-            Name(_HID, EisaId("PNP0A08"))
-            Name(_CID, EisaId("PNP0A03"))
-            Name(_ADR, 0x00)
-            Name(_UID, 1)
-
-            External(ISA, DeviceObj)
-
-            // _OSC: based on sample of ACPI3.0b spec
-            Name(SUPP, 0) // PCI _OSC Support Field value
-            Name(CTRL, 0) // PCI _OSC Control Field value
-            Method(_OSC, 4) {
-                // Create DWORD-addressable fields from the Capabilities Buffer
-                CreateDWordField(Arg3, 0, CDW1)
-
-                // Check for proper UUID
-                If (LEqual(Arg0, ToUUID("33DB4D5B-1FF7-401C-9657-7441C03DD766"))) {
-                    // Create DWORD-addressable fields from the Capabilities Buffer
-                    CreateDWordField(Arg3, 4, CDW2)
-                    CreateDWordField(Arg3, 8, CDW3)
-
-                    // Save Capabilities DWORD2 & 3
-                    Store(CDW2, SUPP)
-                    Store(CDW3, CTRL)
-
-                    // Always allow native PME, AER (no dependencies)
-                    // Never allow SHPC (no SHPC controller in this system)
-                    And(CTRL, 0x1D, CTRL)
-
-#if 0 // For now, nothing to do
-                    If (Not(And(CDW1, 1))) { // Query flag clear?
-                        // Disable GPEs for features granted native control.
-                        If (And(CTRL, 0x01)) { // Hot plug control granted?
-                            Store(0, HPCE) // clear the hot plug SCI enable bit
-                            Store(1, HPCS) // clear the hot plug SCI status bit
-                        }
-                        If (And(CTRL, 0x04)) { // PME control granted?
-                            Store(0, PMCE) // clear the PME SCI enable bit
-                            Store(1, PMCS) // clear the PME SCI status bit
-                        }
-                        If (And(CTRL, 0x10)) { // OS restoring PCI Express cap structure?
-                            // Set status to not restore PCI Express cap structure
-                            // upon resume from S3
-                            Store(1, S3CR)
-                        }
-                    }
-#endif
-                    If (LNotEqual(Arg1, One)) {
-                        // Unknown revision
-                        Or(CDW1, 0x08, CDW1)
-                    }
-                    If (LNotEqual(CDW3, CTRL)) {
-                        // Capabilities bits were masked
-                        Or(CDW1, 0x10, CDW1)
-                    }
-                    // Update DWORD3 in the buffer
-                    Store(CTRL, CDW3)
-                } Else {
-                    Or(CDW1, 4, CDW1) // Unrecognized UUID
-                }
-                Return (Arg3)
-            }
-        }
-    }
-
-#include "acpi-dsdt-pci-crs.dsl"
-#include "acpi-dsdt-hpet.dsl"
-
-
-/****************************************************************
- * LPC ISA bridge
- ****************************************************************/
-
-    Scope(\_SB.PCI0) {
-        /* PCI D31:f0 LPC ISA bridge */
-        Device(ISA) {
-            Name (_ADR, 0x001F0000)  // _ADR: Address
-
-            /* ICH9 PCI to ISA irq remapping */
-            OperationRegion(PIRQ, PCI_Config, 0x60, 0x0C)
-
-            OperationRegion(LPCD, PCI_Config, 0x80, 0x2)
-            Field(LPCD, AnyAcc, NoLock, Preserve) {
-                COMA,   3,
-                    ,   1,
-                COMB,   3,
-
-                Offset(0x01),
-                LPTD,   2,
-                    ,   2,
-                FDCD,   2
-            }
-            OperationRegion(LPCE, PCI_Config, 0x82, 0x2)
-            Field(LPCE, AnyAcc, NoLock, Preserve) {
-                CAEN,   1,
-                CBEN,   1,
-                LPEN,   1,
-                FDEN,   1
-            }
-        }
-    }
-
-#define DSDT_APPLESMC_STA q35_dsdt_applesmc_sta
-#include "acpi-dsdt-isa.dsl"
-
-
-/****************************************************************
- * PCI IRQs
- ****************************************************************/
-
-    /* Zero => PIC mode, One => APIC Mode */
-    Name(\PICF, Zero)
-    Method(\_PIC, 1, NotSerialized) {
-        Store(Arg0, \PICF)
-    }
-
-    Scope(\_SB) {
-        Scope(PCI0) {
-#define prt_slot_lnk(nr, lnk0, lnk1, lnk2, lnk3)  \
-    Package() { nr##ffff, 0, lnk0, 0 },           \
-    Package() { nr##ffff, 1, lnk1, 0 },           \
-    Package() { nr##ffff, 2, lnk2, 0 },           \
-    Package() { nr##ffff, 3, lnk3, 0 }
-
-#define prt_slot_lnkA(nr) prt_slot_lnk(nr, LNKA, LNKB, LNKC, LNKD)
-#define prt_slot_lnkB(nr) prt_slot_lnk(nr, LNKB, LNKC, LNKD, LNKA)
-#define prt_slot_lnkC(nr) prt_slot_lnk(nr, LNKC, LNKD, LNKA, LNKB)
-#define prt_slot_lnkD(nr) prt_slot_lnk(nr, LNKD, LNKA, LNKB, LNKC)
-
-#define prt_slot_lnkE(nr) prt_slot_lnk(nr, LNKE, LNKF, LNKG, LNKH)
-#define prt_slot_lnkF(nr) prt_slot_lnk(nr, LNKF, LNKG, LNKH, LNKE)
-#define prt_slot_lnkG(nr) prt_slot_lnk(nr, LNKG, LNKH, LNKE, LNKF)
-#define prt_slot_lnkH(nr) prt_slot_lnk(nr, LNKH, LNKE, LNKF, LNKG)
-
-            Name(PRTP, package() {
-                prt_slot_lnkE(0x0000),
-                prt_slot_lnkF(0x0001),
-                prt_slot_lnkG(0x0002),
-                prt_slot_lnkH(0x0003),
-                prt_slot_lnkE(0x0004),
-                prt_slot_lnkF(0x0005),
-                prt_slot_lnkG(0x0006),
-                prt_slot_lnkH(0x0007),
-                prt_slot_lnkE(0x0008),
-                prt_slot_lnkF(0x0009),
-                prt_slot_lnkG(0x000a),
-                prt_slot_lnkH(0x000b),
-                prt_slot_lnkE(0x000c),
-                prt_slot_lnkF(0x000d),
-                prt_slot_lnkG(0x000e),
-                prt_slot_lnkH(0x000f),
-                prt_slot_lnkE(0x0010),
-                prt_slot_lnkF(0x0011),
-                prt_slot_lnkG(0x0012),
-                prt_slot_lnkH(0x0013),
-                prt_slot_lnkE(0x0014),
-                prt_slot_lnkF(0x0015),
-                prt_slot_lnkG(0x0016),
-                prt_slot_lnkH(0x0017),
-                prt_slot_lnkE(0x0018),
-
-                /* INTA -> PIRQA for slot 25 - 31
-                   see the default value of D<N>IR */
-                prt_slot_lnkA(0x0019),
-                prt_slot_lnkA(0x001a),
-                prt_slot_lnkA(0x001b),
-                prt_slot_lnkA(0x001c),
-                prt_slot_lnkA(0x001d),
-
-                /* PCIe->PCI bridge. use PIRQ[E-H] */
-                prt_slot_lnkE(0x001e),
-
-                prt_slot_lnkA(0x001f)
-            })
-
-#define prt_slot_gsi(nr, gsi0, gsi1, gsi2, gsi3)  \
-    Package() { nr##ffff, 0, gsi0, 0 },           \
-    Package() { nr##ffff, 1, gsi1, 0 },           \
-    Package() { nr##ffff, 2, gsi2, 0 },           \
-    Package() { nr##ffff, 3, gsi3, 0 }
-
-#define prt_slot_gsiA(nr) prt_slot_gsi(nr, GSIA, GSIB, GSIC, GSID)
-#define prt_slot_gsiB(nr) prt_slot_gsi(nr, GSIB, GSIC, GSID, GSIA)
-#define prt_slot_gsiC(nr) prt_slot_gsi(nr, GSIC, GSID, GSIA, GSIB)
-#define prt_slot_gsiD(nr) prt_slot_gsi(nr, GSID, GSIA, GSIB, GSIC)
-
-#define prt_slot_gsiE(nr) prt_slot_gsi(nr, GSIE, GSIF, GSIG, GSIH)
-#define prt_slot_gsiF(nr) prt_slot_gsi(nr, GSIF, GSIG, GSIH, GSIE)
-#define prt_slot_gsiG(nr) prt_slot_gsi(nr, GSIG, GSIH, GSIE, GSIF)
-#define prt_slot_gsiH(nr) prt_slot_gsi(nr, GSIH, GSIE, GSIF, GSIG)
-
-            Name(PRTA, package() {
-                prt_slot_gsiE(0x0000),
-                prt_slot_gsiF(0x0001),
-                prt_slot_gsiG(0x0002),
-                prt_slot_gsiH(0x0003),
-                prt_slot_gsiE(0x0004),
-                prt_slot_gsiF(0x0005),
-                prt_slot_gsiG(0x0006),
-                prt_slot_gsiH(0x0007),
-                prt_slot_gsiE(0x0008),
-                prt_slot_gsiF(0x0009),
-                prt_slot_gsiG(0x000a),
-                prt_slot_gsiH(0x000b),
-                prt_slot_gsiE(0x000c),
-                prt_slot_gsiF(0x000d),
-                prt_slot_gsiG(0x000e),
-                prt_slot_gsiH(0x000f),
-                prt_slot_gsiE(0x0010),
-                prt_slot_gsiF(0x0011),
-                prt_slot_gsiG(0x0012),
-                prt_slot_gsiH(0x0013),
-                prt_slot_gsiE(0x0014),
-                prt_slot_gsiF(0x0015),
-                prt_slot_gsiG(0x0016),
-                prt_slot_gsiH(0x0017),
-                prt_slot_gsiE(0x0018),
-
-                /* INTA -> PIRQA for slot 25 - 31, but 30
-                   see the default value of D<N>IR */
-                prt_slot_gsiA(0x0019),
-                prt_slot_gsiA(0x001a),
-                prt_slot_gsiA(0x001b),
-                prt_slot_gsiA(0x001c),
-                prt_slot_gsiA(0x001d),
-
-                /* PCIe->PCI bridge. use PIRQ[E-H] */
-                prt_slot_gsiE(0x001e),
-
-                prt_slot_gsiA(0x001f)
-            })
-
-            Method(_PRT, 0, NotSerialized) {
-                /* PCI IRQ routing table, example from ACPI 2.0a specification,
-                   section 6.2.8.1 */
-                /* Note: we provide the same info as the PCI routing
-                   table of the Bochs BIOS */
-                If (LEqual(\PICF, Zero)) {
-                    Return (PRTP)
-                } Else {
-                    Return (PRTA)
-                }
-            }
-        }
-
-        Field(PCI0.ISA.PIRQ, ByteAcc, NoLock, Preserve) {
-            PRQA,   8,
-            PRQB,   8,
-            PRQC,   8,
-            PRQD,   8,
-
-            Offset(0x08),
-            PRQE,   8,
-            PRQF,   8,
-            PRQG,   8,
-            PRQH,   8
-        }
-
-        Method(IQST, 1, NotSerialized) {
-            // _STA method - get status
-            If (And(0x80, Arg0)) {
-                Return (0x09)
-            }
-            Return (0x0B)
-        }
-        Method(IQCR, 1, Serialized) {
-            // _CRS method - get current settings
-            Name(PRR0, ResourceTemplate() {
-                Interrupt(, Level, ActiveHigh, Shared) { 0 }
-            })
-            CreateDWordField(PRR0, 0x05, PRRI)
-            Store(And(Arg0, 0x0F), PRRI)
-            Return (PRR0)
-        }
-
-#define define_link(link, uid, reg)                             \
-        Device(link) {                                          \
-            Name(_HID, EISAID("PNP0C0F"))                       \
-            Name(_UID, uid)                                     \
-            Name(_PRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    5, 10, 11                                   \
-                }                                               \
-            })                                                  \
-            Method(_STA, 0, NotSerialized) {                    \
-                Return (IQST(reg))                              \
-            }                                                   \
-            Method(_DIS, 0, NotSerialized) {                    \
-                Or(reg, 0x80, reg)                              \
-            }                                                   \
-            Method(_CRS, 0, NotSerialized) {                    \
-                Return (IQCR(reg))                              \
-            }                                                   \
-            Method(_SRS, 1, NotSerialized) {                    \
-                CreateDWordField(Arg0, 0x05, PRRI)              \
-                Store(PRRI, reg)                                \
-            }                                                   \
-        }
-
-        define_link(LNKA, 0, PRQA)
-        define_link(LNKB, 1, PRQB)
-        define_link(LNKC, 2, PRQC)
-        define_link(LNKD, 3, PRQD)
-        define_link(LNKE, 4, PRQE)
-        define_link(LNKF, 5, PRQF)
-        define_link(LNKG, 6, PRQG)
-        define_link(LNKH, 7, PRQH)
-
-#define define_gsi_link(link, uid, gsi)                         \
-        Device(link) {                                          \
-            Name(_HID, EISAID("PNP0C0F"))                       \
-            Name(_UID, uid)                                     \
-            Name(_PRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    gsi                                         \
-                }                                               \
-            })                                                  \
-            Name(_CRS, ResourceTemplate() {                     \
-                Interrupt(, Level, ActiveHigh, Shared) {        \
-                    gsi                                         \
-                }                                               \
-            })                                                  \
-            Method(_SRS, 1, NotSerialized) {                    \
-            }                                                   \
-        }
-
-        define_gsi_link(GSIA, 0, 0x10)
-        define_gsi_link(GSIB, 0, 0x11)
-        define_gsi_link(GSIC, 0, 0x12)
-        define_gsi_link(GSID, 0, 0x13)
-        define_gsi_link(GSIE, 0, 0x14)
-        define_gsi_link(GSIF, 0, 0x15)
-        define_gsi_link(GSIG, 0, 0x16)
-        define_gsi_link(GSIH, 0, 0x17)
-    }
-
-#include "hw/acpi/pc-hotplug.h"
-#define CPU_STATUS_BASE ICH9_CPU_HOTPLUG_IO_BASE
-#include "acpi-dsdt-cpu-hotplug.dsl"
-#include "acpi-dsdt-mem-hotplug.dsl"
-
-
-/****************************************************************
- * General purpose events
- ****************************************************************/
-    Scope(\_GPE) {
-        Name(_HID, "ACPI0006")
-
-        Method(_L00) {
-        }
-        Method(_L01) {
-        }
-        Method(_E02) {
-            // CPU hotplug event
-            \_SB.PRSC()
-        }
-        Method(_E03) {
-            // Memory hotplug event
-            \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_SCAN_METHOD()
-        }
-        Method(_L04) {
-        }
-        Method(_L05) {
-        }
-        Method(_L06) {
-        }
-        Method(_L07) {
-        }
-        Method(_L08) {
-        }
-        Method(_L09) {
-        }
-        Method(_L0A) {
-        }
-        Method(_L0B) {
-        }
-        Method(_L0C) {
-        }
-        Method(_L0D) {
-        }
-        Method(_L0E) {
-        }
-        Method(_L0F) {
-        }
-    }
-}
diff --git a/hw/i386/q35-acpi-dsdt.hex.generated b/hw/i386/q35-acpi-dsdt.hex.generated
deleted file mode 100644
index 4807bdf..0000000
--- a/hw/i386/q35-acpi-dsdt.hex.generated
+++ /dev/null
@@ -1,8187 +0,0 @@
-static unsigned char Q35AcpiDsdtAmlCode[] = {
-0x44,
-0x53,
-0x44,
-0x54,
-0xf6,
-0x1f,
-0x0,
-0x0,
-0x1,
-0x91,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x44,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x2,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x28,
-0x8,
-0x14,
-0x20,
-0x10,
-0x49,
-0x4,
-0x5c,
-0x0,
-0x5b,
-0x80,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0xb,
-0x2,
-0x4,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x44,
-0x42,
-0x47,
-0x5f,
-0x1,
-0x44,
-0x42,
-0x47,
-0x42,
-0x8,
-0x14,
-0x2c,
-0x44,
-0x42,
-0x55,
-0x47,
-0x1,
-0x98,
-0x68,
-0x60,
-0x96,
-0x60,
-0x60,
-0x74,
-0x87,
-0x60,
-0x1,
-0x61,
-0x70,
-0x0,
-0x62,
-0xa2,
-0x10,
-0x95,
-0x62,
-0x61,
-0x70,
-0x83,
-0x88,
-0x60,
-0x62,
-0x0,
-0x44,
-0x42,
-0x47,
-0x42,
-0x75,
-0x62,
-0x70,
-0xa,
-0xa,
-0x44,
-0x42,
-0x47,
-0x42,
-0x10,
-0x29,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x80,
-0x50,
-0x43,
-0x53,
-0x54,
-0x1,
-0xb,
-0x0,
-0xae,
-0xa,
-0xc,
-0x5b,
-0x80,
-0x50,
-0x43,
-0x53,
-0x42,
-0x1,
-0xb,
-0xc,
-0xae,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x50,
-0x43,
-0x53,
-0x42,
-0x40,
-0x50,
-0x43,
-0x49,
-0x42,
-0x8,
-0x10,
-0x4f,
-0xc,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x47,
-0xc,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x8,
-0x8,
-0x5f,
-0x43,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x8,
-0x53,
-0x55,
-0x50,
-0x50,
-0x0,
-0x8,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x0,
-0x14,
-0x44,
-0x9,
-0x5f,
-0x4f,
-0x53,
-0x43,
-0x4,
-0x8a,
-0x6b,
-0x0,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa0,
-0x46,
-0x7,
-0x93,
-0x68,
-0x11,
-0x13,
-0xa,
-0x10,
-0x5b,
-0x4d,
-0xdb,
-0x33,
-0xf7,
-0x1f,
-0x1c,
-0x40,
-0x96,
-0x57,
-0x74,
-0x41,
-0xc0,
-0x3d,
-0xd7,
-0x66,
-0x8a,
-0x6b,
-0xa,
-0x4,
-0x43,
-0x44,
-0x57,
-0x32,
-0x8a,
-0x6b,
-0xa,
-0x8,
-0x43,
-0x44,
-0x57,
-0x33,
-0x70,
-0x43,
-0x44,
-0x57,
-0x32,
-0x53,
-0x55,
-0x50,
-0x50,
-0x70,
-0x43,
-0x44,
-0x57,
-0x33,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x7b,
-0x43,
-0x54,
-0x52,
-0x4c,
-0xa,
-0x1d,
-0x43,
-0x54,
-0x52,
-0x4c,
-0xa0,
-0x10,
-0x92,
-0x93,
-0x69,
-0x1,
-0x7d,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa,
-0x8,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa0,
-0x16,
-0x92,
-0x93,
-0x43,
-0x44,
-0x57,
-0x33,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x7d,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa,
-0x10,
-0x43,
-0x44,
-0x57,
-0x31,
-0x70,
-0x43,
-0x54,
-0x52,
-0x4c,
-0x43,
-0x44,
-0x57,
-0x33,
-0xa1,
-0xc,
-0x7d,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa,
-0x4,
-0x43,
-0x44,
-0x57,
-0x31,
-0xa4,
-0x6b,
-0x10,
-0x4e,
-0x15,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x43,
-0x52,
-0x45,
-0x53,
-0x11,
-0x42,
-0x7,
-0xa,
-0x6e,
-0x88,
-0xd,
-0x0,
-0x2,
-0xc,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x1,
-0x47,
-0x1,
-0xf8,
-0xc,
-0xf8,
-0xc,
-0x1,
-0x8,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0xd7,
-0xc,
-0x0,
-0x0,
-0xd8,
-0xc,
-0x88,
-0xd,
-0x0,
-0x1,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0xd,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0xf3,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0xff,
-0xff,
-0xb,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x2,
-0x0,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xe0,
-0xff,
-0xff,
-0xbf,
-0xfe,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xc0,
-0x1e,
-0x79,
-0x0,
-0x8,
-0x43,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x80,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x80,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x41,
-0xa,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0x8a,
-0x43,
-0x52,
-0x45,
-0x53,
-0xa,
-0x5c,
-0x50,
-0x53,
-0x33,
-0x32,
-0x8a,
-0x43,
-0x52,
-0x45,
-0x53,
-0xa,
-0x60,
-0x50,
-0x45,
-0x33,
-0x32,
-0x8a,
-0x43,
-0x52,
-0x45,
-0x53,
-0xa,
-0x68,
-0x50,
-0x4c,
-0x33,
-0x32,
-0x70,
-0x50,
-0x30,
-0x53,
-0x5f,
-0x50,
-0x53,
-0x33,
-0x32,
-0x70,
-0x50,
-0x30,
-0x45,
-0x5f,
-0x50,
-0x45,
-0x33,
-0x32,
-0x70,
-0x72,
-0x74,
-0x50,
-0x30,
-0x45,
-0x5f,
-0x50,
-0x30,
-0x53,
-0x5f,
-0x0,
-0x1,
-0x0,
-0x50,
-0x4c,
-0x33,
-0x32,
-0xa0,
-0xc,
-0x93,
-0x50,
-0x31,
-0x56,
-0x5f,
-0x0,
-0xa4,
-0x43,
-0x52,
-0x45,
-0x53,
-0x8f,
-0x43,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x50,
-0x53,
-0x36,
-0x34,
-0x8f,
-0x43,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x50,
-0x45,
-0x36,
-0x34,
-0x8f,
-0x43,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x50,
-0x4c,
-0x36,
-0x34,
-0x70,
-0x50,
-0x31,
-0x53,
-0x5f,
-0x50,
-0x53,
-0x36,
-0x34,
-0x70,
-0x50,
-0x31,
-0x45,
-0x5f,
-0x50,
-0x45,
-0x36,
-0x34,
-0x70,
-0x50,
-0x31,
-0x4c,
-0x5f,
-0x50,
-0x4c,
-0x36,
-0x34,
-0x84,
-0x43,
-0x52,
-0x45,
-0x53,
-0x43,
-0x52,
-0x36,
-0x34,
-0x60,
-0xa4,
-0x60,
-0x10,
-0x4d,
-0x8,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x45,
-0x8,
-0x48,
-0x50,
-0x45,
-0x54,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x1,
-0x3,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x0,
-0xc,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0xb,
-0x0,
-0x4,
-0x5b,
-0x81,
-0x10,
-0x48,
-0x50,
-0x54,
-0x4d,
-0x13,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x20,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x20,
-0x14,
-0x36,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x56,
-0x45,
-0x4e,
-0x44,
-0x60,
-0x70,
-0x50,
-0x52,
-0x44,
-0x5f,
-0x61,
-0x7a,
-0x60,
-0xa,
-0x10,
-0x60,
-0xa0,
-0xc,
-0x91,
-0x93,
-0x60,
-0x0,
-0x93,
-0x60,
-0xb,
-0xff,
-0xff,
-0xa4,
-0x0,
-0xa0,
-0xe,
-0x91,
-0x93,
-0x61,
-0x0,
-0x94,
-0x61,
-0xc,
-0x0,
-0xe1,
-0xf5,
-0x5,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x11,
-0xa,
-0xe,
-0x86,
-0x9,
-0x0,
-0x0,
-0x0,
-0x0,
-0xd0,
-0xfe,
-0x0,
-0x4,
-0x0,
-0x0,
-0x79,
-0x0,
-0x10,
-0x4c,
-0x7,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x4f,
-0x6,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0x1f,
-0x0,
-0x5b,
-0x80,
-0x50,
-0x49,
-0x52,
-0x51,
-0x2,
-0xa,
-0x60,
-0xa,
-0xc,
-0x5b,
-0x80,
-0x4c,
-0x50,
-0x43,
-0x44,
-0x2,
-0xa,
-0x80,
-0xa,
-0x2,
-0x5b,
-0x81,
-0x20,
-0x4c,
-0x50,
-0x43,
-0x44,
-0x0,
-0x43,
-0x4f,
-0x4d,
-0x41,
-0x3,
-0x0,
-0x1,
-0x43,
-0x4f,
-0x4d,
-0x42,
-0x3,
-0x0,
-0x1,
-0x4c,
-0x50,
-0x54,
-0x44,
-0x2,
-0x0,
-0x2,
-0x46,
-0x44,
-0x43,
-0x44,
-0x2,
-0x5b,
-0x80,
-0x4c,
-0x50,
-0x43,
-0x45,
-0x2,
-0xa,
-0x82,
-0xa,
-0x2,
-0x5b,
-0x81,
-0x1a,
-0x4c,
-0x50,
-0x43,
-0x45,
-0x0,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x1,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x1,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x1,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x1,
-0x10,
-0x4a,
-0x1e,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x5b,
-0x82,
-0x2c,
-0x53,
-0x4d,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x6,
-0x10,
-0x0,
-0x1,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x41,
-0xa,
-0xf0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0x0,
-0x3,
-0x0,
-0x3,
-0x1,
-0x20,
-0x22,
-0x40,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x2d,
-0x52,
-0x54,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xb,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x70,
-0x0,
-0x70,
-0x0,
-0x10,
-0x2,
-0x22,
-0x0,
-0x1,
-0x47,
-0x1,
-0x72,
-0x0,
-0x72,
-0x0,
-0x2,
-0x6,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x37,
-0x4b,
-0x42,
-0x44,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x3,
-0x3,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x18,
-0xa,
-0x15,
-0x47,
-0x1,
-0x60,
-0x0,
-0x60,
-0x0,
-0x1,
-0x1,
-0x47,
-0x1,
-0x64,
-0x0,
-0x64,
-0x0,
-0x1,
-0x1,
-0x22,
-0x2,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x27,
-0x4d,
-0x4f,
-0x55,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xf,
-0x13,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x8,
-0xa,
-0x5,
-0x22,
-0x0,
-0x10,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x4a,
-0x4,
-0x46,
-0x44,
-0x43,
-0x30,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x7,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x46,
-0x44,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x1b,
-0xa,
-0x18,
-0x47,
-0x1,
-0xf2,
-0x3,
-0xf2,
-0x3,
-0x0,
-0x4,
-0x47,
-0x1,
-0xf7,
-0x3,
-0xf7,
-0x3,
-0x0,
-0x1,
-0x22,
-0x40,
-0x0,
-0x2a,
-0x4,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x3e,
-0x4c,
-0x50,
-0x54,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x4,
-0x0,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x4c,
-0x50,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0x78,
-0x3,
-0x78,
-0x3,
-0x8,
-0x8,
-0x22,
-0x80,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x31,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x41,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x3,
-0xf8,
-0x3,
-0x0,
-0x8,
-0x22,
-0x10,
-0x0,
-0x79,
-0x0,
-0x5b,
-0x82,
-0x46,
-0x4,
-0x43,
-0x4f,
-0x4d,
-0x32,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0x5,
-0x1,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x43,
-0x42,
-0x45,
-0x4e,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x10,
-0xa,
-0xd,
-0x47,
-0x1,
-0xf8,
-0x2,
-0xf8,
-0x2,
-0x0,
-0x8,
-0x22,
-0x8,
-0x0,
-0x79,
-0x0,
-0x8,
-0x50,
-0x49,
-0x43,
-0x46,
-0x0,
-0x14,
-0xc,
-0x5f,
-0x50,
-0x49,
-0x43,
-0x1,
-0x70,
-0x68,
-0x50,
-0x49,
-0x43,
-0x46,
-0x10,
-0x8e,
-0x55,
-0x1,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x10,
-0x43,
-0xea,
-0x50,
-0x43,
-0x49,
-0x30,
-0x8,
-0x50,
-0x52,
-0x54,
-0x50,
-0x12,
-0x4b,
-0x73,
-0x80,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x0,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x1,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x2,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x3,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x0,
-0x8,
-0x50,
-0x52,
-0x54,
-0x41,
-0x12,
-0x4b,
-0x73,
-0x80,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xb,
-0x4,
-0xb,
-0xff,
-0xff,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xc,
-0x4,
-0xb,
-0xff,
-0xff,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x2,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x3,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x4,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x5,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x6,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x7,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x8,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x9,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xa,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xb,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xc,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xd,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xe,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0xf,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x10,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x11,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x12,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x13,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x14,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x15,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x16,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x17,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x18,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x19,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1a,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1b,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1c,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1d,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x45,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x46,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x47,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1e,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x48,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x0,
-0x47,
-0x53,
-0x49,
-0x41,
-0x0,
-0x12,
-0xd,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0x1,
-0x47,
-0x53,
-0x49,
-0x42,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x2,
-0x47,
-0x53,
-0x49,
-0x43,
-0x0,
-0x12,
-0xe,
-0x4,
-0xc,
-0xff,
-0xff,
-0x1f,
-0x0,
-0xa,
-0x3,
-0x47,
-0x53,
-0x49,
-0x44,
-0x0,
-0x14,
-0x1a,
-0x5f,
-0x50,
-0x52,
-0x54,
-0x0,
-0xa0,
-0xc,
-0x93,
-0x50,
-0x49,
-0x43,
-0x46,
-0x0,
-0xa4,
-0x50,
-0x52,
-0x54,
-0x50,
-0xa1,
-0x6,
-0xa4,
-0x50,
-0x52,
-0x54,
-0x41,
-0x5b,
-0x81,
-0x3a,
-0x2f,
-0x3,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x50,
-0x49,
-0x52,
-0x51,
-0x1,
-0x50,
-0x52,
-0x51,
-0x41,
-0x8,
-0x50,
-0x52,
-0x51,
-0x42,
-0x8,
-0x50,
-0x52,
-0x51,
-0x43,
-0x8,
-0x50,
-0x52,
-0x51,
-0x44,
-0x8,
-0x0,
-0x20,
-0x50,
-0x52,
-0x51,
-0x45,
-0x8,
-0x50,
-0x52,
-0x51,
-0x46,
-0x8,
-0x50,
-0x52,
-0x51,
-0x47,
-0x8,
-0x50,
-0x52,
-0x51,
-0x48,
-0x8,
-0x14,
-0x13,
-0x49,
-0x51,
-0x53,
-0x54,
-0x1,
-0xa0,
-0x9,
-0x7b,
-0xa,
-0x80,
-0x68,
-0x0,
-0xa4,
-0xa,
-0x9,
-0xa4,
-0xa,
-0xb,
-0x14,
-0x34,
-0x49,
-0x51,
-0x43,
-0x52,
-0x9,
-0x8,
-0x50,
-0x52,
-0x52,
-0x30,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8a,
-0x50,
-0x52,
-0x52,
-0x30,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x7b,
-0x68,
-0xa,
-0xf,
-0x0,
-0x50,
-0x52,
-0x52,
-0x49,
-0xa4,
-0x50,
-0x52,
-0x52,
-0x30,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x41,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x41,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x41,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x41,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x41,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x41,
-0x5b,
-0x82,
-0x4c,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x1,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x42,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x42,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x42,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x42,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x42,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x43,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x2,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x43,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x43,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x43,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x43,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x43,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x3,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x44,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x44,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x44,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x44,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x44,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x4,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x45,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x45,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x45,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x45,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x45,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x46,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x5,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x46,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x46,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x46,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x46,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x46,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x47,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x6,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x47,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x47,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x47,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x47,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x47,
-0x5b,
-0x82,
-0x4d,
-0x7,
-0x4c,
-0x4e,
-0x4b,
-0x48,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xa,
-0x7,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0x16,
-0xa,
-0x13,
-0x89,
-0xe,
-0x0,
-0x9,
-0x3,
-0x5,
-0x0,
-0x0,
-0x0,
-0xa,
-0x0,
-0x0,
-0x0,
-0xb,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x53,
-0x54,
-0x50,
-0x52,
-0x51,
-0x48,
-0x14,
-0x11,
-0x5f,
-0x44,
-0x49,
-0x53,
-0x0,
-0x7d,
-0x50,
-0x52,
-0x51,
-0x48,
-0xa,
-0x80,
-0x50,
-0x52,
-0x51,
-0x48,
-0x14,
-0xf,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x49,
-0x51,
-0x43,
-0x52,
-0x50,
-0x52,
-0x51,
-0x48,
-0x14,
-0x17,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x8a,
-0x68,
-0xa,
-0x5,
-0x50,
-0x52,
-0x52,
-0x49,
-0x70,
-0x50,
-0x52,
-0x52,
-0x49,
-0x50,
-0x52,
-0x51,
-0x48,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x41,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x10,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x10,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x42,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x11,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x11,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x43,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x12,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x12,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x13,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x13,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x14,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x14,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x46,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x15,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x15,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x47,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x16,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x16,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x5b,
-0x82,
-0x45,
-0x4,
-0x47,
-0x53,
-0x49,
-0x48,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0xf,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x0,
-0x8,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x17,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xe,
-0xa,
-0xb,
-0x89,
-0x6,
-0x0,
-0x9,
-0x1,
-0x17,
-0x0,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x53,
-0x52,
-0x53,
-0x1,
-0x10,
-0x42,
-0x11,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x14,
-0x35,
-0x43,
-0x50,
-0x4d,
-0x41,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0x70,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x61,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x2,
-0x0,
-0x70,
-0x68,
-0x88,
-0x61,
-0xa,
-0x3,
-0x0,
-0x70,
-0x60,
-0x88,
-0x61,
-0xa,
-0x4,
-0x0,
-0xa4,
-0x61,
-0x14,
-0x1a,
-0x43,
-0x50,
-0x53,
-0x54,
-0x1,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x68,
-0x0,
-0x60,
-0xa0,
-0x5,
-0x60,
-0xa4,
-0xa,
-0xf,
-0xa1,
-0x3,
-0xa4,
-0x0,
-0x14,
-0xa,
-0x43,
-0x50,
-0x45,
-0x4a,
-0x2,
-0x5b,
-0x22,
-0xa,
-0xc8,
-0x5b,
-0x80,
-0x50,
-0x52,
-0x53,
-0x54,
-0x1,
-0xb,
-0xd8,
-0xc,
-0xa,
-0x20,
-0x5b,
-0x81,
-0xc,
-0x50,
-0x52,
-0x53,
-0x54,
-0x1,
-0x50,
-0x52,
-0x53,
-0x5f,
-0x40,
-0x10,
-0x14,
-0x4a,
-0x6,
-0x50,
-0x52,
-0x53,
-0x43,
-0x0,
-0x70,
-0x50,
-0x52,
-0x53,
-0x5f,
-0x65,
-0x70,
-0x0,
-0x62,
-0x70,
-0x0,
-0x60,
-0xa2,
-0x46,
-0x5,
-0x95,
-0x60,
-0x87,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x70,
-0x83,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0x61,
-0xa0,
-0xa,
-0x7b,
-0x60,
-0xa,
-0x7,
-0x0,
-0x7a,
-0x62,
-0x1,
-0x62,
-0xa1,
-0xc,
-0x70,
-0x83,
-0x88,
-0x65,
-0x7a,
-0x60,
-0xa,
-0x3,
-0x0,
-0x0,
-0x62,
-0x70,
-0x7b,
-0x62,
-0x1,
-0x0,
-0x63,
-0xa0,
-0x22,
-0x92,
-0x93,
-0x61,
-0x63,
-0x70,
-0x63,
-0x88,
-0x43,
-0x50,
-0x4f,
-0x4e,
-0x60,
-0x0,
-0xa0,
-0xa,
-0x93,
-0x63,
-0x1,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0xa1,
-0x8,
-0x4e,
-0x54,
-0x46,
-0x59,
-0x60,
-0xa,
-0x3,
-0x75,
-0x60,
-0x5b,
-0x82,
-0x29,
-0x50,
-0x52,
-0x45,
-0x53,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xa,
-0x6,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xd,
-0xa,
-0xa,
-0x47,
-0x1,
-0xd8,
-0xc,
-0xd8,
-0xc,
-0x0,
-0x20,
-0x79,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x54,
-0x41,
-0xa,
-0xb,
-0x10,
-0x40,
-0x31,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x43,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x50,
-0x4e,
-0x50,
-0x30,
-0x41,
-0x30,
-0x36,
-0x0,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xd,
-0x4d,
-0x65,
-0x6d,
-0x6f,
-0x72,
-0x79,
-0x20,
-0x68,
-0x6f,
-0x74,
-0x70,
-0x6c,
-0x75,
-0x67,
-0x20,
-0x72,
-0x65,
-0x73,
-0x6f,
-0x75,
-0x72,
-0x63,
-0x65,
-0x73,
-0x0,
-0x5b,
-0x80,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x1,
-0xb,
-0x0,
-0xa,
-0xa,
-0x18,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xd,
-0xa,
-0xa,
-0x47,
-0x1,
-0x0,
-0xa,
-0x0,
-0xa,
-0x0,
-0x18,
-0x79,
-0x0,
-0x14,
-0x13,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0xa4,
-0xa,
-0xb,
-0x5b,
-0x81,
-0x1f,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x3,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x20,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x20,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x20,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x20,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x20,
-0x5b,
-0x81,
-0x13,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x1,
-0x0,
-0x40,
-0xa,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x5b,
-0x1,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x0,
-0x5b,
-0x81,
-0x15,
-0x48,
-0x50,
-0x4d,
-0x52,
-0x3,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x20,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x20,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x20,
-0x14,
-0x4a,
-0x4,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x0,
-0xa0,
-0x9,
-0x93,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x0,
-0xa4,
-0x0,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0xa2,
-0x25,
-0x95,
-0x60,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0x70,
-0x60,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0x13,
-0x93,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x1,
-0x4d,
-0x54,
-0x46,
-0x59,
-0x60,
-0x1,
-0x70,
-0x1,
-0x4d,
-0x49,
-0x4e,
-0x53,
-0x72,
-0x60,
-0x1,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x1,
-0x14,
-0x2d,
-0x4d,
-0x52,
-0x53,
-0x54,
-0x1,
-0x70,
-0x0,
-0x60,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0xa0,
-0xb,
-0x93,
-0x4d,
-0x45,
-0x53,
-0x5f,
-0x1,
-0x70,
-0xa,
-0xf,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x41,
-0x18,
-0x4d,
-0x43,
-0x52,
-0x53,
-0x9,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x8,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x11,
-0x33,
-0xa,
-0x30,
-0x8a,
-0x2b,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0xe,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x12,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x26,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x2a,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x16,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x8a,
-0x4d,
-0x52,
-0x36,
-0x34,
-0xa,
-0x1a,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x48,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x42,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x70,
-0x4d,
-0x52,
-0x4c,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x72,
-0x4d,
-0x49,
-0x4e,
-0x48,
-0x4c,
-0x45,
-0x4e,
-0x48,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x14,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x72,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0xa0,
-0x11,
-0x95,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x74,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x1,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0xa0,
-0x44,
-0x7,
-0x93,
-0x4d,
-0x41,
-0x58,
-0x48,
-0x0,
-0x8,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x11,
-0x1f,
-0xa,
-0x1c,
-0x87,
-0x17,
-0x0,
-0x0,
-0xc,
-0x3,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0xfe,
-0xff,
-0xff,
-0xff,
-0x0,
-0x0,
-0x0,
-0x0,
-0xff,
-0xff,
-0xff,
-0xff,
-0x79,
-0x0,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xa,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0xe,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x8a,
-0x4d,
-0x52,
-0x33,
-0x32,
-0xa,
-0x16,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x49,
-0x4e,
-0x4c,
-0x4d,
-0x49,
-0x4e,
-0x5f,
-0x70,
-0x4d,
-0x41,
-0x58,
-0x4c,
-0x4d,
-0x41,
-0x58,
-0x5f,
-0x70,
-0x4c,
-0x45,
-0x4e,
-0x4c,
-0x4c,
-0x45,
-0x4e,
-0x5f,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x33,
-0x32,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x4d,
-0x52,
-0x36,
-0x34,
-0x14,
-0x24,
-0x4d,
-0x50,
-0x58,
-0x4d,
-0x1,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x4d,
-0x50,
-0x58,
-0x5f,
-0x60,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xa4,
-0x60,
-0x14,
-0x28,
-0x4d,
-0x4f,
-0x53,
-0x54,
-0x4,
-0x5b,
-0x23,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0xff,
-0xff,
-0x70,
-0x99,
-0x68,
-0x0,
-0x4d,
-0x53,
-0x45,
-0x4c,
-0x70,
-0x69,
-0x4d,
-0x4f,
-0x45,
-0x56,
-0x70,
-0x6a,
-0x4d,
-0x4f,
-0x53,
-0x43,
-0x5b,
-0x27,
-0x4d,
-0x4c,
-0x43,
-0x4b,
-0x10,
-0x42,
-0xa,
-0x5f,
-0x47,
-0x50,
-0x45,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x41,
-0x43,
-0x50,
-0x49,
-0x30,
-0x30,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x30,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x31,
-0x0,
-0x14,
-0x10,
-0x5f,
-0x45,
-0x30,
-0x32,
-0x0,
-0x5c,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x52,
-0x53,
-0x43,
-0x14,
-0x19,
-0x5f,
-0x45,
-0x30,
-0x33,
-0x0,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x53,
-0x43,
-0x4e,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x34,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x35,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x36,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x37,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x38,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x39,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x41,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x42,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x43,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x44,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x45,
-0x0,
-0x14,
-0x6,
-0x5f,
-0x4c,
-0x30,
-0x46,
-0x0
-};
-static unsigned short q35_dsdt_applesmc_sta[] = {
-0x3fa
-};
diff --git a/hw/i386/smbios.c b/hw/i386/smbios.c
deleted file mode 100644
index 024e594..0000000
--- a/hw/i386/smbios.c
+++ /dev/null
@@ -1,1094 +0,0 @@
-/*
- * SMBIOS Support
- *
- * Copyright (C) 2009 Hewlett-Packard Development Company, L.P.
- * Copyright (C) 2013 Red Hat, Inc.
- *
- * Authors:
- *  Alex Williamson <alex.williamson@hp.com>
- *  Markus Armbruster <armbru@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu/config-file.h"
-#include "qemu/error-report.h"
-#include "sysemu/sysemu.h"
-#include "sysemu/cpus.h"
-#include "hw/i386/pc.h"
-#include "hw/i386/smbios.h"
-#include "hw/loader.h"
-
-
-/* legacy structures and constants for <= 2.0 machines */
-struct smbios_header {
-    uint16_t length;
-    uint8_t type;
-} QEMU_PACKED;
-
-struct smbios_field {
-    struct smbios_header header;
-    uint8_t type;
-    uint16_t offset;
-    uint8_t data[];
-} QEMU_PACKED;
-
-struct smbios_table {
-    struct smbios_header header;
-    uint8_t data[];
-} QEMU_PACKED;
-
-#define SMBIOS_FIELD_ENTRY 0
-#define SMBIOS_TABLE_ENTRY 1
-
-static uint8_t *smbios_entries;
-static size_t smbios_entries_len;
-static bool smbios_legacy = true;
-static bool smbios_uuid_encoded = true;
-/* end: legacy structures & constants for <= 2.0 machines */
-
-
-static uint8_t *smbios_tables;
-static size_t smbios_tables_len;
-static unsigned smbios_table_max;
-static unsigned smbios_table_cnt;
-static struct smbios_entry_point ep;
-
-static int smbios_type4_count = 0;
-static bool smbios_immutable;
-static bool smbios_have_defaults;
-static uint32_t smbios_cpuid_version, smbios_cpuid_features, smbios_smp_sockets;
-
-static DECLARE_BITMAP(have_binfile_bitmap, SMBIOS_MAX_TYPE+1);
-static DECLARE_BITMAP(have_fields_bitmap, SMBIOS_MAX_TYPE+1);
-
-static struct {
-    const char *vendor, *version, *date;
-    bool have_major_minor, uefi;
-    uint8_t major, minor;
-} type0;
-
-static struct {
-    const char *manufacturer, *product, *version, *serial, *sku, *family;
-    /* uuid is in qemu_uuid[] */
-} type1;
-
-static struct {
-    const char *manufacturer, *product, *version, *serial, *asset, *location;
-} type2;
-
-static struct {
-    const char *manufacturer, *version, *serial, *asset, *sku;
-} type3;
-
-static struct {
-    const char *sock_pfx, *manufacturer, *version, *serial, *asset, *part;
-} type4;
-
-static struct {
-    const char *loc_pfx, *bank, *manufacturer, *serial, *asset, *part;
-} type17;
-
-static QemuOptsList qemu_smbios_opts = {
-    .name = "smbios",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_smbios_opts.head),
-    .desc = {
-        /*
-         * no elements => accept any params
-         * validation will happen later
-         */
-        { /* end of list */ }
-    }
-};
-
-static const QemuOptDesc qemu_smbios_file_opts[] = {
-    {
-        .name = "file",
-        .type = QEMU_OPT_STRING,
-        .help = "binary file containing an SMBIOS element",
-    },
-    { /* end of list */ }
-};
-
-static const QemuOptDesc qemu_smbios_type0_opts[] = {
-    {
-        .name = "type",
-        .type = QEMU_OPT_NUMBER,
-        .help = "SMBIOS element type",
-    },{
-        .name = "vendor",
-        .type = QEMU_OPT_STRING,
-        .help = "vendor name",
-    },{
-        .name = "version",
-        .type = QEMU_OPT_STRING,
-        .help = "version number",
-    },{
-        .name = "date",
-        .type = QEMU_OPT_STRING,
-        .help = "release date",
-    },{
-        .name = "release",
-        .type = QEMU_OPT_STRING,
-        .help = "revision number",
-    },{
-        .name = "uefi",
-        .type = QEMU_OPT_BOOL,
-        .help = "uefi support",
-    },
-    { /* end of list */ }
-};
-
-static const QemuOptDesc qemu_smbios_type1_opts[] = {
-    {
-        .name = "type",
-        .type = QEMU_OPT_NUMBER,
-        .help = "SMBIOS element type",
-    },{
-        .name = "manufacturer",
-        .type = QEMU_OPT_STRING,
-        .help = "manufacturer name",
-    },{
-        .name = "product",
-        .type = QEMU_OPT_STRING,
-        .help = "product name",
-    },{
-        .name = "version",
-        .type = QEMU_OPT_STRING,
-        .help = "version number",
-    },{
-        .name = "serial",
-        .type = QEMU_OPT_STRING,
-        .help = "serial number",
-    },{
-        .name = "uuid",
-        .type = QEMU_OPT_STRING,
-        .help = "UUID",
-    },{
-        .name = "sku",
-        .type = QEMU_OPT_STRING,
-        .help = "SKU number",
-    },{
-        .name = "family",
-        .type = QEMU_OPT_STRING,
-        .help = "family name",
-    },
-    { /* end of list */ }
-};
-
-static const QemuOptDesc qemu_smbios_type2_opts[] = {
-    {
-        .name = "type",
-        .type = QEMU_OPT_NUMBER,
-        .help = "SMBIOS element type",
-    },{
-        .name = "manufacturer",
-        .type = QEMU_OPT_STRING,
-        .help = "manufacturer name",
-    },{
-        .name = "product",
-        .type = QEMU_OPT_STRING,
-        .help = "product name",
-    },{
-        .name = "version",
-        .type = QEMU_OPT_STRING,
-        .help = "version number",
-    },{
-        .name = "serial",
-        .type = QEMU_OPT_STRING,
-        .help = "serial number",
-    },{
-        .name = "asset",
-        .type = QEMU_OPT_STRING,
-        .help = "asset tag number",
-    },{
-        .name = "location",
-        .type = QEMU_OPT_STRING,
-        .help = "location in chassis",
-    },
-    { /* end of list */ }
-};
-
-static const QemuOptDesc qemu_smbios_type3_opts[] = {
-    {
-        .name = "type",
-        .type = QEMU_OPT_NUMBER,
-        .help = "SMBIOS element type",
-    },{
-        .name = "manufacturer",
-        .type = QEMU_OPT_STRING,
-        .help = "manufacturer name",
-    },{
-        .name = "version",
-        .type = QEMU_OPT_STRING,
-        .help = "version number",
-    },{
-        .name = "serial",
-        .type = QEMU_OPT_STRING,
-        .help = "serial number",
-    },{
-        .name = "asset",
-        .type = QEMU_OPT_STRING,
-        .help = "asset tag number",
-    },{
-        .name = "sku",
-        .type = QEMU_OPT_STRING,
-        .help = "SKU number",
-    },
-    { /* end of list */ }
-};
-
-static const QemuOptDesc qemu_smbios_type4_opts[] = {
-    {
-        .name = "type",
-        .type = QEMU_OPT_NUMBER,
-        .help = "SMBIOS element type",
-    },{
-        .name = "sock_pfx",
-        .type = QEMU_OPT_STRING,
-        .help = "socket designation string prefix",
-    },{
-        .name = "manufacturer",
-        .type = QEMU_OPT_STRING,
-        .help = "manufacturer name",
-    },{
-        .name = "version",
-        .type = QEMU_OPT_STRING,
-        .help = "version number",
-    },{
-        .name = "serial",
-        .type = QEMU_OPT_STRING,
-        .help = "serial number",
-    },{
-        .name = "asset",
-        .type = QEMU_OPT_STRING,
-        .help = "asset tag number",
-    },{
-        .name = "part",
-        .type = QEMU_OPT_STRING,
-        .help = "part number",
-    },
-    { /* end of list */ }
-};
-
-static const QemuOptDesc qemu_smbios_type17_opts[] = {
-    {
-        .name = "type",
-        .type = QEMU_OPT_NUMBER,
-        .help = "SMBIOS element type",
-    },{
-        .name = "loc_pfx",
-        .type = QEMU_OPT_STRING,
-        .help = "device locator string prefix",
-    },{
-        .name = "bank",
-        .type = QEMU_OPT_STRING,
-        .help = "bank locator string",
-    },{
-        .name = "manufacturer",
-        .type = QEMU_OPT_STRING,
-        .help = "manufacturer name",
-    },{
-        .name = "serial",
-        .type = QEMU_OPT_STRING,
-        .help = "serial number",
-    },{
-        .name = "asset",
-        .type = QEMU_OPT_STRING,
-        .help = "asset tag number",
-    },{
-        .name = "part",
-        .type = QEMU_OPT_STRING,
-        .help = "part number",
-    },
-    { /* end of list */ }
-};
-
-static void smbios_register_config(void)
-{
-    qemu_add_opts(&qemu_smbios_opts);
-}
-
-machine_init(smbios_register_config);
-
-static void smbios_validate_table(void)
-{
-    uint32_t expect_t4_count = smbios_legacy ? smp_cpus : smbios_smp_sockets;
-
-    if (smbios_type4_count && smbios_type4_count != expect_t4_count) {
-        error_report("Expected %d SMBIOS Type 4 tables, got %d instead",
-                     expect_t4_count, smbios_type4_count);
-        exit(1);
-    }
-}
-
-
-/* legacy setup functions for <= 2.0 machines */
-static void smbios_add_field(int type, int offset, const void *data, size_t len)
-{
-    struct smbios_field *field;
-
-    if (!smbios_entries) {
-        smbios_entries_len = sizeof(uint16_t);
-        smbios_entries = g_malloc0(smbios_entries_len);
-    }
-    smbios_entries = g_realloc(smbios_entries, smbios_entries_len +
-                                                  sizeof(*field) + len);
-    field = (struct smbios_field *)(smbios_entries + smbios_entries_len);
-    field->header.type = SMBIOS_FIELD_ENTRY;
-    field->header.length = cpu_to_le16(sizeof(*field) + len);
-
-    field->type = type;
-    field->offset = cpu_to_le16(offset);
-    memcpy(field->data, data, len);
-
-    smbios_entries_len += sizeof(*field) + len;
-    (*(uint16_t *)smbios_entries) =
-            cpu_to_le16(le16_to_cpu(*(uint16_t *)smbios_entries) + 1);
-}
-
-static void smbios_maybe_add_str(int type, int offset, const char *data)
-{
-    if (data) {
-        smbios_add_field(type, offset, data, strlen(data) + 1);
-    }
-}
-
-static void smbios_build_type_0_fields(void)
-{
-    smbios_maybe_add_str(0, offsetof(struct smbios_type_0, vendor_str),
-                         type0.vendor);
-    smbios_maybe_add_str(0, offsetof(struct smbios_type_0, bios_version_str),
-                         type0.version);
-    smbios_maybe_add_str(0, offsetof(struct smbios_type_0,
-                                     bios_release_date_str),
-                         type0.date);
-    if (type0.have_major_minor) {
-        smbios_add_field(0, offsetof(struct smbios_type_0,
-                                     system_bios_major_release),
-                         &type0.major, 1);
-        smbios_add_field(0, offsetof(struct smbios_type_0,
-                                     system_bios_minor_release),
-                         &type0.minor, 1);
-    }
-}
-
-static void smbios_build_type_1_fields(void)
-{
-    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, manufacturer_str),
-                         type1.manufacturer);
-    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, product_name_str),
-                         type1.product);
-    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, version_str),
-                         type1.version);
-    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, serial_number_str),
-                         type1.serial);
-    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, sku_number_str),
-                         type1.sku);
-    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, family_str),
-                         type1.family);
-    if (qemu_uuid_set) {
-        /* We don't encode the UUID in the "wire format" here because this
-         * function is for legacy mode and needs to keep the guest ABI, and
-         * because we don't know what's the SMBIOS version advertised by the
-         * BIOS.
-         */
-        smbios_add_field(1, offsetof(struct smbios_type_1, uuid),
-                         qemu_uuid, 16);
-    }
-}
-
-uint8_t *smbios_get_table_legacy(size_t *length)
-{
-    if (!smbios_legacy) {
-        *length = 0;
-        return NULL;
-    }
-
-    if (!smbios_immutable) {
-        smbios_build_type_0_fields();
-        smbios_build_type_1_fields();
-        smbios_validate_table();
-        smbios_immutable = true;
-    }
-    *length = smbios_entries_len;
-    return smbios_entries;
-}
-/* end: legacy setup functions for <= 2.0 machines */
-
-
-static bool smbios_skip_table(uint8_t type, bool required_table)
-{
-    if (test_bit(type, have_binfile_bitmap)) {
-        return true; /* user provided their own binary blob(s) */
-    }
-    if (test_bit(type, have_fields_bitmap)) {
-        return false; /* user provided fields via command line */
-    }
-    if (smbios_have_defaults && required_table) {
-        return false; /* we're building tables, and this one's required */
-    }
-    return true;
-}
-
-#define SMBIOS_BUILD_TABLE_PRE(tbl_type, tbl_handle, tbl_required)        \
-    struct smbios_type_##tbl_type *t;                                     \
-    size_t t_off; /* table offset into smbios_tables */                   \
-    int str_index = 0;                                                    \
-    do {                                                                  \
-        /* should we skip building this table ? */                        \
-        if (smbios_skip_table(tbl_type, tbl_required)) {                  \
-            return;                                                       \
-        }                                                                 \
-                                                                          \
-        /* use offset of table t within smbios_tables */                  \
-        /* (pointer must be updated after each realloc) */                \
-        t_off = smbios_tables_len;                                        \
-        smbios_tables_len += sizeof(*t);                                  \
-        smbios_tables = g_realloc(smbios_tables, smbios_tables_len);      \
-        t = (struct smbios_type_##tbl_type *)(smbios_tables + t_off);     \
-                                                                          \
-        t->header.type = tbl_type;                                        \
-        t->header.length = sizeof(*t);                                    \
-        t->header.handle = cpu_to_le16(tbl_handle);                       \
-    } while (0)
-
-#define SMBIOS_TABLE_SET_STR(tbl_type, field, value)                      \
-    do {                                                                  \
-        int len = (value != NULL) ? strlen(value) + 1 : 0;                \
-        if (len > 1) {                                                    \
-            smbios_tables = g_realloc(smbios_tables,                      \
-                                      smbios_tables_len + len);           \
-            memcpy(smbios_tables + smbios_tables_len, value, len);        \
-            smbios_tables_len += len;                                     \
-            /* update pointer post-realloc */                             \
-            t = (struct smbios_type_##tbl_type *)(smbios_tables + t_off); \
-            t->field = ++str_index;                                       \
-        } else {                                                          \
-            t->field = 0;                                                 \
-        }                                                                 \
-    } while (0)
-
-#define SMBIOS_BUILD_TABLE_POST                                           \
-    do {                                                                  \
-        size_t term_cnt, t_size;                                          \
-                                                                          \
-        /* add '\0' terminator (add two if no strings defined) */         \
-        term_cnt = (str_index == 0) ? 2 : 1;                              \
-        smbios_tables = g_realloc(smbios_tables,                          \
-                                  smbios_tables_len + term_cnt);          \
-        memset(smbios_tables + smbios_tables_len, 0, term_cnt);           \
-        smbios_tables_len += term_cnt;                                    \
-                                                                          \
-        /* update smbios max. element size */                             \
-        t_size = smbios_tables_len - t_off;                               \
-        if (t_size > smbios_table_max) {                                  \
-            smbios_table_max = t_size;                                    \
-        }                                                                 \
-                                                                          \
-        /* update smbios element count */                                 \
-        smbios_table_cnt++;                                               \
-    } while (0)
-
-static void smbios_build_type_0_table(void)
-{
-    SMBIOS_BUILD_TABLE_PRE(0, 0x000, false); /* optional, leave up to BIOS */
-
-    SMBIOS_TABLE_SET_STR(0, vendor_str, type0.vendor);
-    SMBIOS_TABLE_SET_STR(0, bios_version_str, type0.version);
-
-    t->bios_starting_address_segment = cpu_to_le16(0xE800); /* from SeaBIOS */
-
-    SMBIOS_TABLE_SET_STR(0, bios_release_date_str, type0.date);
-
-    t->bios_rom_size = 0; /* hardcoded in SeaBIOS with FIXME comment */
-
-    t->bios_characteristics = cpu_to_le64(0x08); /* Not supported */
-    t->bios_characteristics_extension_bytes[0] = 0;
-    t->bios_characteristics_extension_bytes[1] = 0x14; /* TCD/SVVP | VM */
-    if (type0.uefi) {
-        t->bios_characteristics_extension_bytes[1] |= 0x08; /* |= UEFI */
-    }
-
-    if (type0.have_major_minor) {
-        t->system_bios_major_release = type0.major;
-        t->system_bios_minor_release = type0.minor;
-    } else {
-        t->system_bios_major_release = 0;
-        t->system_bios_minor_release = 0;
-    }
-
-    /* hardcoded in SeaBIOS */
-    t->embedded_controller_major_release = 0xFF;
-    t->embedded_controller_minor_release = 0xFF;
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-/* Encode UUID from the big endian encoding described on RFC4122 to the wire
- * format specified by SMBIOS version 2.6.
- */
-static void smbios_encode_uuid(struct smbios_uuid *uuid, const uint8_t *buf)
-{
-    memcpy(uuid, buf, 16);
-    if (smbios_uuid_encoded) {
-        uuid->time_low = bswap32(uuid->time_low);
-        uuid->time_mid = bswap16(uuid->time_mid);
-        uuid->time_hi_and_version = bswap16(uuid->time_hi_and_version);
-    }
-}
-
-static void smbios_build_type_1_table(void)
-{
-    SMBIOS_BUILD_TABLE_PRE(1, 0x100, true); /* required */
-
-    SMBIOS_TABLE_SET_STR(1, manufacturer_str, type1.manufacturer);
-    SMBIOS_TABLE_SET_STR(1, product_name_str, type1.product);
-    SMBIOS_TABLE_SET_STR(1, version_str, type1.version);
-    SMBIOS_TABLE_SET_STR(1, serial_number_str, type1.serial);
-    if (qemu_uuid_set) {
-        smbios_encode_uuid(&t->uuid, qemu_uuid);
-    } else {
-        memset(&t->uuid, 0, 16);
-    }
-    t->wake_up_type = 0x06; /* power switch */
-    SMBIOS_TABLE_SET_STR(1, sku_number_str, type1.sku);
-    SMBIOS_TABLE_SET_STR(1, family_str, type1.family);
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-static void smbios_build_type_2_table(void)
-{
-    SMBIOS_BUILD_TABLE_PRE(2, 0x200, false); /* optional */
-
-    SMBIOS_TABLE_SET_STR(2, manufacturer_str, type2.manufacturer);
-    SMBIOS_TABLE_SET_STR(2, product_str, type2.product);
-    SMBIOS_TABLE_SET_STR(2, version_str, type2.version);
-    SMBIOS_TABLE_SET_STR(2, serial_number_str, type2.serial);
-    SMBIOS_TABLE_SET_STR(2, asset_tag_number_str, type2.asset);
-    t->feature_flags = 0x01; /* Motherboard */
-    SMBIOS_TABLE_SET_STR(2, location_str, type2.location);
-    t->chassis_handle = cpu_to_le16(0x300); /* Type 3 (System enclosure) */
-    t->board_type = 0x0A; /* Motherboard */
-    t->contained_element_count = 0;
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-static void smbios_build_type_3_table(void)
-{
-    SMBIOS_BUILD_TABLE_PRE(3, 0x300, true); /* required */
-
-    SMBIOS_TABLE_SET_STR(3, manufacturer_str, type3.manufacturer);
-    t->type = 0x01; /* Other */
-    SMBIOS_TABLE_SET_STR(3, version_str, type3.version);
-    SMBIOS_TABLE_SET_STR(3, serial_number_str, type3.serial);
-    SMBIOS_TABLE_SET_STR(3, asset_tag_number_str, type3.asset);
-    t->boot_up_state = 0x03; /* Safe */
-    t->power_supply_state = 0x03; /* Safe */
-    t->thermal_state = 0x03; /* Safe */
-    t->security_status = 0x02; /* Unknown */
-    t->oem_defined = cpu_to_le32(0);
-    t->height = 0;
-    t->number_of_power_cords = 0;
-    t->contained_element_count = 0;
-    SMBIOS_TABLE_SET_STR(3, sku_number_str, type3.sku);
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-static void smbios_build_type_4_table(unsigned instance)
-{
-    char sock_str[128];
-
-    SMBIOS_BUILD_TABLE_PRE(4, 0x400 + instance, true); /* required */
-
-    snprintf(sock_str, sizeof(sock_str), "%s%2x", type4.sock_pfx, instance);
-    SMBIOS_TABLE_SET_STR(4, socket_designation_str, sock_str);
-    t->processor_type = 0x03; /* CPU */
-    t->processor_family = 0x01; /* Other */
-    SMBIOS_TABLE_SET_STR(4, processor_manufacturer_str, type4.manufacturer);
-    t->processor_id[0] = cpu_to_le32(smbios_cpuid_version);
-    t->processor_id[1] = cpu_to_le32(smbios_cpuid_features);
-    SMBIOS_TABLE_SET_STR(4, processor_version_str, type4.version);
-    t->voltage = 0;
-    t->external_clock = cpu_to_le16(0); /* Unknown */
-    t->max_speed = cpu_to_le16(0); /* Unknown */
-    t->current_speed = cpu_to_le16(0); /* Unknown */
-    t->status = 0x41; /* Socket populated, CPU enabled */
-    t->processor_upgrade = 0x01; /* Other */
-    t->l1_cache_handle = cpu_to_le16(0xFFFF); /* N/A */
-    t->l2_cache_handle = cpu_to_le16(0xFFFF); /* N/A */
-    t->l3_cache_handle = cpu_to_le16(0xFFFF); /* N/A */
-    SMBIOS_TABLE_SET_STR(4, serial_number_str, type4.serial);
-    SMBIOS_TABLE_SET_STR(4, asset_tag_number_str, type4.asset);
-    SMBIOS_TABLE_SET_STR(4, part_number_str, type4.part);
-    t->core_count = t->core_enabled = smp_cores;
-    t->thread_count = smp_threads;
-    t->processor_characteristics = cpu_to_le16(0x02); /* Unknown */
-    t->processor_family2 = cpu_to_le16(0x01); /* Other */
-
-    SMBIOS_BUILD_TABLE_POST;
-    smbios_type4_count++;
-}
-
-#define ONE_KB ((ram_addr_t)1 << 10)
-#define ONE_MB ((ram_addr_t)1 << 20)
-#define ONE_GB ((ram_addr_t)1 << 30)
-
-#define MAX_T16_STD_SZ 0x80000000 /* 2T in Kilobytes */
-
-static void smbios_build_type_16_table(unsigned dimm_cnt)
-{
-    uint64_t size_kb;
-
-    SMBIOS_BUILD_TABLE_PRE(16, 0x1000, true); /* required */
-
-    t->location = 0x01; /* Other */
-    t->use = 0x03; /* System memory */
-    t->error_correction = 0x06; /* Multi-bit ECC (for Microsoft, per SeaBIOS) */
-    size_kb = QEMU_ALIGN_UP(ram_size, ONE_KB) / ONE_KB;
-    if (size_kb < MAX_T16_STD_SZ) {
-        t->maximum_capacity = cpu_to_le32(size_kb);
-        t->extended_maximum_capacity = cpu_to_le64(0);
-    } else {
-        t->maximum_capacity = cpu_to_le32(MAX_T16_STD_SZ);
-        t->extended_maximum_capacity = cpu_to_le64(ram_size);
-    }
-    t->memory_error_information_handle = cpu_to_le16(0xFFFE); /* Not provided */
-    t->number_of_memory_devices = cpu_to_le16(dimm_cnt);
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-#define MAX_T17_STD_SZ 0x7FFF /* (32G - 1M), in Megabytes */
-#define MAX_T17_EXT_SZ 0x80000000 /* 2P, in Megabytes */
-
-static void smbios_build_type_17_table(unsigned instance, uint64_t size)
-{
-    char loc_str[128];
-    uint64_t size_mb;
-
-    SMBIOS_BUILD_TABLE_PRE(17, 0x1100 + instance, true); /* required */
-
-    t->physical_memory_array_handle = cpu_to_le16(0x1000); /* Type 16 above */
-    t->memory_error_information_handle = cpu_to_le16(0xFFFE); /* Not provided */
-    t->total_width = cpu_to_le16(0xFFFF); /* Unknown */
-    t->data_width = cpu_to_le16(0xFFFF); /* Unknown */
-    size_mb = QEMU_ALIGN_UP(size, ONE_MB) / ONE_MB;
-    if (size_mb < MAX_T17_STD_SZ) {
-        t->size = cpu_to_le16(size_mb);
-        t->extended_size = cpu_to_le32(0);
-    } else {
-        assert(size_mb < MAX_T17_EXT_SZ);
-        t->size = cpu_to_le16(MAX_T17_STD_SZ);
-        t->extended_size = cpu_to_le32(size_mb);
-    }
-    t->form_factor = 0x09; /* DIMM */
-    t->device_set = 0; /* Not in a set */
-    snprintf(loc_str, sizeof(loc_str), "%s %d", type17.loc_pfx, instance);
-    SMBIOS_TABLE_SET_STR(17, device_locator_str, loc_str);
-    SMBIOS_TABLE_SET_STR(17, bank_locator_str, type17.bank);
-    t->memory_type = 0x07; /* RAM */
-    t->type_detail = cpu_to_le16(0x02); /* Other */
-    t->speed = cpu_to_le16(0); /* Unknown */
-    SMBIOS_TABLE_SET_STR(17, manufacturer_str, type17.manufacturer);
-    SMBIOS_TABLE_SET_STR(17, serial_number_str, type17.serial);
-    SMBIOS_TABLE_SET_STR(17, asset_tag_number_str, type17.asset);
-    SMBIOS_TABLE_SET_STR(17, part_number_str, type17.part);
-    t->attributes = 0; /* Unknown */
-    t->configured_clock_speed = cpu_to_le16(0); /* Unknown */
-    t->minimum_voltage = cpu_to_le16(0); /* Unknown */
-    t->maximum_voltage = cpu_to_le16(0); /* Unknown */
-    t->configured_voltage = cpu_to_le16(0); /* Unknown */
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-static void smbios_build_type_19_table(unsigned instance,
-                                       uint64_t start, uint64_t size)
-{
-    uint64_t end, start_kb, end_kb;
-
-    SMBIOS_BUILD_TABLE_PRE(19, 0x1300 + instance, true); /* required */
-
-    end = start + size - 1;
-    assert(end > start);
-    start_kb = start / ONE_KB;
-    end_kb = end / ONE_KB;
-    if (start_kb < UINT32_MAX && end_kb < UINT32_MAX) {
-        t->starting_address = cpu_to_le32(start_kb);
-        t->ending_address = cpu_to_le32(end_kb);
-        t->extended_starting_address =
-            t->extended_ending_address = cpu_to_le64(0);
-    } else {
-        t->starting_address = t->ending_address = cpu_to_le32(UINT32_MAX);
-        t->extended_starting_address = cpu_to_le64(start);
-        t->extended_ending_address = cpu_to_le64(end);
-    }
-    t->memory_array_handle = cpu_to_le16(0x1000); /* Type 16 above */
-    t->partition_width = 1; /* One device per row */
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-static void smbios_build_type_32_table(void)
-{
-    SMBIOS_BUILD_TABLE_PRE(32, 0x2000, true); /* required */
-
-    memset(t->reserved, 0, 6);
-    t->boot_status = 0; /* No errors detected */
-
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-static void smbios_build_type_127_table(void)
-{
-    SMBIOS_BUILD_TABLE_PRE(127, 0x7F00, true); /* required */
-    SMBIOS_BUILD_TABLE_POST;
-}
-
-void smbios_set_cpuid(uint32_t version, uint32_t features)
-{
-    smbios_cpuid_version = version;
-    smbios_cpuid_features = features;
-}
-
-#define SMBIOS_SET_DEFAULT(field, value)                                  \
-    if (!field) {                                                         \
-        field = value;                                                    \
-    }
-
-void smbios_set_defaults(const char *manufacturer, const char *product,
-                         const char *version, bool legacy_mode,
-                         bool uuid_encoded)
-{
-    smbios_have_defaults = true;
-    smbios_legacy = legacy_mode;
-    smbios_uuid_encoded = uuid_encoded;
-
-    /* drop unwanted version of command-line file blob(s) */
-    if (smbios_legacy) {
-        g_free(smbios_tables);
-        /* in legacy mode, also complain if fields were given for types > 1 */
-        if (find_next_bit(have_fields_bitmap,
-                          SMBIOS_MAX_TYPE+1, 2) < SMBIOS_MAX_TYPE+1) {
-            error_report("can't process fields for smbios "
-                         "types > 1 on machine versions < 2.1!");
-            exit(1);
-        }
-    } else {
-        g_free(smbios_entries);
-    }
-
-    SMBIOS_SET_DEFAULT(type1.manufacturer, manufacturer);
-    SMBIOS_SET_DEFAULT(type1.product, product);
-    SMBIOS_SET_DEFAULT(type1.version, version);
-    SMBIOS_SET_DEFAULT(type2.manufacturer, manufacturer);
-    SMBIOS_SET_DEFAULT(type2.product, product);
-    SMBIOS_SET_DEFAULT(type2.version, version);
-    SMBIOS_SET_DEFAULT(type3.manufacturer, manufacturer);
-    SMBIOS_SET_DEFAULT(type3.version, version);
-    SMBIOS_SET_DEFAULT(type4.sock_pfx, "CPU");
-    SMBIOS_SET_DEFAULT(type4.manufacturer, manufacturer);
-    SMBIOS_SET_DEFAULT(type4.version, version);
-    SMBIOS_SET_DEFAULT(type17.loc_pfx, "DIMM");
-    SMBIOS_SET_DEFAULT(type17.manufacturer, manufacturer);
-}
-
-static void smbios_entry_point_setup(void)
-{
-    memcpy(ep.anchor_string, "_SM_", 4);
-    memcpy(ep.intermediate_anchor_string, "_DMI_", 5);
-    ep.length = sizeof(struct smbios_entry_point);
-    ep.entry_point_revision = 0; /* formatted_area reserved, per spec v2.1+ */
-    memset(ep.formatted_area, 0, 5);
-
-    /* compliant with smbios spec v2.8 */
-    ep.smbios_major_version = 2;
-    ep.smbios_minor_version = 8;
-    ep.smbios_bcd_revision = 0x28;
-
-    /* set during table construction, but BIOS may override: */
-    ep.structure_table_length = cpu_to_le16(smbios_tables_len);
-    ep.max_structure_size = cpu_to_le16(smbios_table_max);
-    ep.number_of_structures = cpu_to_le16(smbios_table_cnt);
-
-    /* BIOS must recalculate: */
-    ep.checksum = 0;
-    ep.intermediate_checksum = 0;
-    ep.structure_table_address = cpu_to_le32(0);
-}
-
-void smbios_get_tables(uint8_t **tables, size_t *tables_len,
-                       uint8_t **anchor, size_t *anchor_len)
-{
-    unsigned i, dimm_cnt, instance;
-
-    if (smbios_legacy) {
-        *tables = *anchor = NULL;
-        *tables_len = *anchor_len = 0;
-        return;
-    }
-
-    if (!smbios_immutable) {
-        smbios_build_type_0_table();
-        smbios_build_type_1_table();
-        smbios_build_type_2_table();
-        smbios_build_type_3_table();
-
-        smbios_smp_sockets = DIV_ROUND_UP(smp_cpus, smp_cores * smp_threads);
-        assert(smbios_smp_sockets >= 1);
-
-        for (i = 0; i < smbios_smp_sockets; i++) {
-            smbios_build_type_4_table(i);
-        }
-
-#define MAX_DIMM_SZ (16ll * ONE_GB)
-#define GET_DIMM_SZ ((i < dimm_cnt - 1) ? MAX_DIMM_SZ : ram_size % MAX_DIMM_SZ)
-
-        dimm_cnt = QEMU_ALIGN_UP(ram_size, MAX_DIMM_SZ) / MAX_DIMM_SZ;
-
-        smbios_build_type_16_table(dimm_cnt);
-
-        for (i = 0; i < dimm_cnt; i++) {
-            smbios_build_type_17_table(i, GET_DIMM_SZ);
-        }
-
-        for (i = 0, instance = 0; i < e820_get_num_entries(); i++) {
-            uint64_t address, length;
-            if (e820_get_entry(i, E820_RAM, &address, &length)) {
-                smbios_build_type_19_table(instance++, address, length);
-            }
-        }
-
-        smbios_build_type_32_table();
-        smbios_build_type_127_table();
-
-        smbios_validate_table();
-        smbios_entry_point_setup();
-        smbios_immutable = true;
-    }
-
-    /* return tables blob and entry point (anchor), and their sizes */
-    *tables = smbios_tables;
-    *tables_len = smbios_tables_len;
-    *anchor = (uint8_t *)&ep;
-    *anchor_len = sizeof(struct smbios_entry_point);
-}
-
-static void save_opt(const char **dest, QemuOpts *opts, const char *name)
-{
-    const char *val = qemu_opt_get(opts, name);
-
-    if (val) {
-        *dest = val;
-    }
-}
-
-void smbios_entry_add(QemuOpts *opts)
-{
-    Error *local_err = NULL;
-    const char *val;
-
-    assert(!smbios_immutable);
-
-    val = qemu_opt_get(opts, "file");
-    if (val) {
-        struct smbios_structure_header *header;
-        int size;
-        struct smbios_table *table; /* legacy mode only */
-
-        qemu_opts_validate(opts, qemu_smbios_file_opts, &local_err);
-        if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            exit(1);
-        }
-
-        size = get_image_size(val);
-        if (size == -1 || size < sizeof(struct smbios_structure_header)) {
-            error_report("Cannot read SMBIOS file %s", val);
-            exit(1);
-        }
-
-        /*
-         * NOTE: standard double '\0' terminator expected, per smbios spec.
-         * (except in legacy mode, where the second '\0' is implicit and
-         *  will be inserted by the BIOS).
-         */
-        smbios_tables = g_realloc(smbios_tables, smbios_tables_len + size);
-        header = (struct smbios_structure_header *)(smbios_tables +
-                                                    smbios_tables_len);
-
-        if (load_image(val, (uint8_t *)header) != size) {
-            error_report("Failed to load SMBIOS file %s", val);
-            exit(1);
-        }
-
-        if (test_bit(header->type, have_fields_bitmap)) {
-            error_report("can't load type %d struct, fields already specified!",
-                         header->type);
-            exit(1);
-        }
-        set_bit(header->type, have_binfile_bitmap);
-
-        if (header->type == 4) {
-            smbios_type4_count++;
-        }
-
-        smbios_tables_len += size;
-        if (size > smbios_table_max) {
-            smbios_table_max = size;
-        }
-        smbios_table_cnt++;
-
-        /* add a copy of the newly loaded blob to legacy smbios_entries */
-        /* NOTE: This code runs before smbios_set_defaults(), so we don't
-         *       yet know which mode (legacy vs. aggregate-table) will be
-         *       required. We therefore add the binary blob to both legacy
-         *       (smbios_entries) and aggregate (smbios_tables) tables, and
-         *       delete the one we don't need from smbios_set_defaults(),
-         *       once we know which machine version has been requested.
-         */
-        if (!smbios_entries) {
-            smbios_entries_len = sizeof(uint16_t);
-            smbios_entries = g_malloc0(smbios_entries_len);
-        }
-        smbios_entries = g_realloc(smbios_entries, smbios_entries_len +
-                                                   size + sizeof(*table));
-        table = (struct smbios_table *)(smbios_entries + smbios_entries_len);
-        table->header.type = SMBIOS_TABLE_ENTRY;
-        table->header.length = cpu_to_le16(sizeof(*table) + size);
-        memcpy(table->data, header, size);
-        smbios_entries_len += sizeof(*table) + size;
-        (*(uint16_t *)smbios_entries) =
-                cpu_to_le16(le16_to_cpu(*(uint16_t *)smbios_entries) + 1);
-        /* end: add a copy of the newly loaded blob to legacy smbios_entries */
-
-        return;
-    }
-
-    val = qemu_opt_get(opts, "type");
-    if (val) {
-        unsigned long type = strtoul(val, NULL, 0);
-
-        if (type > SMBIOS_MAX_TYPE) {
-            error_report("out of range!");
-            exit(1);
-        }
-
-        if (test_bit(type, have_binfile_bitmap)) {
-            error_report("can't add fields, binary file already loaded!");
-            exit(1);
-        }
-        set_bit(type, have_fields_bitmap);
-
-        switch (type) {
-        case 0:
-            qemu_opts_validate(opts, qemu_smbios_type0_opts, &local_err);
-            if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                exit(1);
-            }
-            save_opt(&type0.vendor, opts, "vendor");
-            save_opt(&type0.version, opts, "version");
-            save_opt(&type0.date, opts, "date");
-            type0.uefi = qemu_opt_get_bool(opts, "uefi", false);
-
-            val = qemu_opt_get(opts, "release");
-            if (val) {
-                if (sscanf(val, "%hhu.%hhu", &type0.major, &type0.minor) != 2) {
-                    error_report("Invalid release");
-                    exit(1);
-                }
-                type0.have_major_minor = true;
-            }
-            return;
-        case 1:
-            qemu_opts_validate(opts, qemu_smbios_type1_opts, &local_err);
-            if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                exit(1);
-            }
-            save_opt(&type1.manufacturer, opts, "manufacturer");
-            save_opt(&type1.product, opts, "product");
-            save_opt(&type1.version, opts, "version");
-            save_opt(&type1.serial, opts, "serial");
-            save_opt(&type1.sku, opts, "sku");
-            save_opt(&type1.family, opts, "family");
-
-            val = qemu_opt_get(opts, "uuid");
-            if (val) {
-                if (qemu_uuid_parse(val, qemu_uuid) != 0) {
-                    error_report("Invalid UUID");
-                    exit(1);
-                }
-                qemu_uuid_set = true;
-            }
-            return;
-        case 2:
-            qemu_opts_validate(opts, qemu_smbios_type2_opts, &local_err);
-            if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                exit(1);
-            }
-            save_opt(&type2.manufacturer, opts, "manufacturer");
-            save_opt(&type2.product, opts, "product");
-            save_opt(&type2.version, opts, "version");
-            save_opt(&type2.serial, opts, "serial");
-            save_opt(&type2.asset, opts, "asset");
-            save_opt(&type2.location, opts, "location");
-            return;
-        case 3:
-            qemu_opts_validate(opts, qemu_smbios_type3_opts, &local_err);
-            if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                exit(1);
-            }
-            save_opt(&type3.manufacturer, opts, "manufacturer");
-            save_opt(&type3.version, opts, "version");
-            save_opt(&type3.serial, opts, "serial");
-            save_opt(&type3.asset, opts, "asset");
-            save_opt(&type3.sku, opts, "sku");
-            return;
-        case 4:
-            qemu_opts_validate(opts, qemu_smbios_type4_opts, &local_err);
-            if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                exit(1);
-            }
-            save_opt(&type4.sock_pfx, opts, "sock_pfx");
-            save_opt(&type4.manufacturer, opts, "manufacturer");
-            save_opt(&type4.version, opts, "version");
-            save_opt(&type4.serial, opts, "serial");
-            save_opt(&type4.asset, opts, "asset");
-            save_opt(&type4.part, opts, "part");
-            return;
-        case 17:
-            qemu_opts_validate(opts, qemu_smbios_type17_opts, &local_err);
-            if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                exit(1);
-            }
-            save_opt(&type17.loc_pfx, opts, "loc_pfx");
-            save_opt(&type17.bank, opts, "bank");
-            save_opt(&type17.manufacturer, opts, "manufacturer");
-            save_opt(&type17.serial, opts, "serial");
-            save_opt(&type17.asset, opts, "asset");
-            save_opt(&type17.part, opts, "part");
-            return;
-        default:
-            error_report("Don't know how to build fields for SMBIOS type %ld",
-                         type);
-            exit(1);
-        }
-    }
-
-    error_report("Must specify type= or file=");
-    exit(1);
-}
diff --git a/hw/i386/ssdt-mem.dsl b/hw/i386/ssdt-mem.dsl
deleted file mode 100644
index 22ff5dd..0000000
--- a/hw/i386/ssdt-mem.dsl
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Memory hotplug ACPI DSDT static objects definitions
- *
- * Copyright ProfitBricks GmbH 2012
- * Copyright (C) 2013-2014 Red Hat Inc
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-
-/* This file is the basis for the ssdt_mem[] variable in src/acpi.c.
- * It defines the contents of the memory device object.  At
- * runtime, a dynamically generated SSDT will contain one copy of this
- * AML snippet for every possible memory device in the system.  The
- * objects will be placed in the \_SB_ namespace.
- *
- * In addition to the aml code generated from this file, the
- * src/acpi.c file creates a MTFY method with an entry for each memdevice:
- *     Method(MTFY, 2) {
- *         If (LEqual(Arg0, 0x00)) { Notify(MP00, Arg1) }
- *         If (LEqual(Arg0, 0x01)) { Notify(MP01, Arg1) }
- *         ...
- *     }
- */
-#include "hw/acpi/pc-hotplug.h"
-
-ACPI_EXTRACT_ALL_CODE ssdm_mem_aml
-
-DefinitionBlock ("ssdt-mem.aml", "SSDT", 0x02, "BXPC", "CSSDT", 0x1)
-{
-
-    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_CRS_METHOD, MethodObj)
-    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_STATUS_METHOD, MethodObj)
-    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_OST_METHOD, MethodObj)
-    External(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_PROXIMITY_METHOD, MethodObj)
-
-    Scope(\_SB) {
-/*  v------------------ DO NOT EDIT ------------------v */
-        ACPI_EXTRACT_DEVICE_START ssdt_mem_start
-        ACPI_EXTRACT_DEVICE_END ssdt_mem_end
-        ACPI_EXTRACT_DEVICE_STRING ssdt_mem_name
-        Device(MPAA) {
-            ACPI_EXTRACT_NAME_STRING ssdt_mem_id
-            Name(_UID, "0xAA")
-/*  ^------------------ DO NOT EDIT ------------------^
- * Don't change the above without also updating the C code.
- */
-            Name(_HID, EISAID("PNP0C80"))
-
-            Method(_CRS, 0) {
-                Return(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_CRS_METHOD(_UID))
-            }
-
-            Method(_STA, 0) {
-                Return(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_STATUS_METHOD(_UID))
-            }
-
-            Method(_PXM, 0) {
-                Return(\_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_PROXIMITY_METHOD(_UID))
-            }
-
-            Method(_OST, 3) {
-                \_SB.PCI0.MEMORY_HOTPLUG_DEVICE.MEMORY_SLOT_OST_METHOD(_UID, Arg0, Arg1, Arg2)
-            }
-        }
-    }
-}
diff --git a/hw/i386/ssdt-mem.hex.generated b/hw/i386/ssdt-mem.hex.generated
deleted file mode 100644
index b3bfbbd..0000000
--- a/hw/i386/ssdt-mem.hex.generated
+++ /dev/null
@@ -1,213 +0,0 @@
-static unsigned char ssdt_mem_id[] = {
-0x35
-};
-static unsigned char ssdm_mem_aml[] = {
-0x53,
-0x53,
-0x44,
-0x54,
-0xc7,
-0x0,
-0x0,
-0x0,
-0x2,
-0x66,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x43,
-0x53,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x0,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x28,
-0x8,
-0x14,
-0x20,
-0x10,
-0x42,
-0xa,
-0x5c,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x49,
-0x9,
-0x4d,
-0x50,
-0x41,
-0x41,
-0x8,
-0x5f,
-0x55,
-0x49,
-0x44,
-0xd,
-0x30,
-0x78,
-0x41,
-0x41,
-0x0,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0x80,
-0x14,
-0x1e,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x0,
-0xa4,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x43,
-0x52,
-0x53,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x14,
-0x1e,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x52,
-0x53,
-0x54,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x14,
-0x1e,
-0x5f,
-0x50,
-0x58,
-0x4d,
-0x0,
-0xa4,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x50,
-0x58,
-0x4d,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x14,
-0x20,
-0x5f,
-0x4f,
-0x53,
-0x54,
-0x3,
-0x5c,
-0x2f,
-0x4,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x4d,
-0x48,
-0x50,
-0x44,
-0x4d,
-0x4f,
-0x53,
-0x54,
-0x5f,
-0x55,
-0x49,
-0x44,
-0x68,
-0x69,
-0x6a
-};
-static unsigned char ssdt_mem_start[] = {
-0x2c
-};
-static unsigned char ssdt_mem_end[] = {
-0xc7
-};
-static unsigned char ssdt_mem_name[] = {
-0x30
-};
diff --git a/hw/i386/ssdt-misc.dsl b/hw/i386/ssdt-misc.dsl
deleted file mode 100644
index 1e3baae..0000000
--- a/hw/i386/ssdt-misc.dsl
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-#include "hw/acpi/pc-hotplug.h"
-
-ACPI_EXTRACT_ALL_CODE ssdp_misc_aml
-
-DefinitionBlock ("ssdt-misc.aml", "SSDT", 0x01, "BXPC", "BXSSDTSUSP", 0x1)
-{
-
-/****************************************************************
- * PCI memory ranges
- ****************************************************************/
-
-    Scope(\) {
-       ACPI_EXTRACT_NAME_DWORD_CONST acpi_pci32_start
-       Name(P0S, 0x12345678)
-       ACPI_EXTRACT_NAME_DWORD_CONST acpi_pci32_end
-       Name(P0E, 0x12345678)
-       ACPI_EXTRACT_NAME_BYTE_CONST acpi_pci64_valid
-       Name(P1V, 0x12)
-       ACPI_EXTRACT_NAME_BUFFER8 acpi_pci64_start
-       Name(P1S, Buffer() { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 })
-       ACPI_EXTRACT_NAME_BUFFER8 acpi_pci64_end
-       Name(P1E, Buffer() { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 })
-       ACPI_EXTRACT_NAME_BUFFER8 acpi_pci64_length
-       Name(P1L, Buffer() { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 })
-       ACPI_EXTRACT_NAME_DWORD_CONST ssdt_mctrl_nr_slots
-       Name(MEMORY_SLOTS_NUMBER, 0x12345678)
-    }
-
-
-/****************************************************************
- * Suspend
- ****************************************************************/
-
-    Scope(\) {
-    /*
-     * S3 (suspend-to-ram), S4 (suspend-to-disk) and S5 (power-off) type codes:
-     * must match piix4 emulation.
-     */
-
-        ACPI_EXTRACT_NAME_STRING acpi_s3_name
-        Name(_S3, Package(0x04) {
-            One,  /* PM1a_CNT.SLP_TYP */
-            One,  /* PM1b_CNT.SLP_TYP */
-            Zero,  /* reserved */
-            Zero   /* reserved */
-        })
-        ACPI_EXTRACT_NAME_STRING acpi_s4_name
-        ACPI_EXTRACT_PKG_START acpi_s4_pkg
-        Name(_S4, Package(0x04) {
-            0x2,  /* PM1a_CNT.SLP_TYP */
-            0x2,  /* PM1b_CNT.SLP_TYP */
-            Zero,  /* reserved */
-            Zero   /* reserved */
-        })
-        Name(_S5, Package(0x04) {
-            Zero,  /* PM1a_CNT.SLP_TYP */
-            Zero,  /* PM1b_CNT.SLP_TYP */
-            Zero,  /* reserved */
-            Zero   /* reserved */
-        })
-    }
-
-    External(\_SB.PCI0, DeviceObj)
-    External(\_SB.PCI0.ISA, DeviceObj)
-
-    Scope(\_SB.PCI0.ISA) {
-        Device(PEVT) {
-            Name(_HID, "QEMU0001")
-            /* PEST will be patched to be Zero if no such device */
-            ACPI_EXTRACT_NAME_WORD_CONST ssdt_isa_pest
-            Name(PEST, 0xFFFF)
-            OperationRegion(PEOR, SystemIO, PEST, 0x01)
-            Field(PEOR, ByteAcc, NoLock, Preserve) {
-                PEPT,   8,
-            }
-
-            Method(_STA, 0, NotSerialized) {
-                Store(PEST, Local0)
-                If (LEqual(Local0, Zero)) {
-                    Return (0x00)
-                } Else {
-                    Return (0x0F)
-                }
-            }
-
-            Method(RDPT, 0, NotSerialized) {
-                Store(PEPT, Local0)
-                Return (Local0)
-            }
-
-            Method(WRPT, 1, NotSerialized) {
-                Store(Arg0, PEPT)
-            }
-
-            Name(_CRS, ResourceTemplate() {
-                IO(Decode16, 0x00, 0x00, 0x01, 0x01, IO)
-            })
-
-            CreateWordField(_CRS, IO._MIN, IOMN)
-            CreateWordField(_CRS, IO._MAX, IOMX)
-
-            Method(_INI, 0, NotSerialized) {
-                Store(PEST, IOMN)
-                Store(PEST, IOMX)
-            }
-        }
-    }
-}
diff --git a/hw/i386/ssdt-misc.hex.generated b/hw/i386/ssdt-misc.hex.generated
deleted file mode 100644
index cbcf61d..0000000
--- a/hw/i386/ssdt-misc.hex.generated
+++ /dev/null
@@ -1,399 +0,0 @@
-static unsigned char acpi_pci64_length[] = {
-0x6f
-};
-static unsigned char acpi_s4_pkg[] = {
-0x99
-};
-static unsigned char ssdt_mctrl_nr_slots[] = {
-0x7d
-};
-static unsigned char acpi_s3_name[] = {
-0x86
-};
-static unsigned char acpi_pci32_start[] = {
-0x2f
-};
-static unsigned char acpi_pci64_valid[] = {
-0x43
-};
-static unsigned char ssdp_misc_aml[] = {
-0x53,
-0x53,
-0x44,
-0x54,
-0x6c,
-0x1,
-0x0,
-0x0,
-0x1,
-0x3,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x53,
-0x53,
-0x44,
-0x54,
-0x53,
-0x55,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x28,
-0x8,
-0x14,
-0x20,
-0x10,
-0x4c,
-0x5,
-0x5c,
-0x0,
-0x8,
-0x50,
-0x30,
-0x53,
-0x5f,
-0xc,
-0x78,
-0x56,
-0x34,
-0x12,
-0x8,
-0x50,
-0x30,
-0x45,
-0x5f,
-0xc,
-0x78,
-0x56,
-0x34,
-0x12,
-0x8,
-0x50,
-0x31,
-0x56,
-0x5f,
-0xa,
-0x12,
-0x8,
-0x50,
-0x31,
-0x53,
-0x5f,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x8,
-0x50,
-0x31,
-0x45,
-0x5f,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x8,
-0x50,
-0x31,
-0x4c,
-0x5f,
-0x11,
-0xb,
-0xa,
-0x8,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x8,
-0x4d,
-0x44,
-0x4e,
-0x52,
-0xc,
-0x78,
-0x56,
-0x34,
-0x12,
-0x10,
-0x29,
-0x5c,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x33,
-0x5f,
-0x12,
-0x6,
-0x4,
-0x1,
-0x1,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x34,
-0x5f,
-0x12,
-0x8,
-0x4,
-0xa,
-0x2,
-0xa,
-0x2,
-0x0,
-0x0,
-0x8,
-0x5f,
-0x53,
-0x35,
-0x5f,
-0x12,
-0x6,
-0x4,
-0x0,
-0x0,
-0x0,
-0x0,
-0x10,
-0x40,
-0xc,
-0x5c,
-0x2f,
-0x3,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x49,
-0x53,
-0x41,
-0x5f,
-0x5b,
-0x82,
-0x4d,
-0xa,
-0x50,
-0x45,
-0x56,
-0x54,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x51,
-0x45,
-0x4d,
-0x55,
-0x30,
-0x30,
-0x30,
-0x31,
-0x0,
-0x8,
-0x50,
-0x45,
-0x53,
-0x54,
-0xb,
-0xff,
-0xff,
-0x5b,
-0x80,
-0x50,
-0x45,
-0x4f,
-0x52,
-0x1,
-0x50,
-0x45,
-0x53,
-0x54,
-0x1,
-0x5b,
-0x81,
-0xb,
-0x50,
-0x45,
-0x4f,
-0x52,
-0x1,
-0x50,
-0x45,
-0x50,
-0x54,
-0x8,
-0x14,
-0x18,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0x70,
-0x50,
-0x45,
-0x53,
-0x54,
-0x60,
-0xa0,
-0x6,
-0x93,
-0x60,
-0x0,
-0xa4,
-0x0,
-0xa1,
-0x4,
-0xa4,
-0xa,
-0xf,
-0x14,
-0xe,
-0x52,
-0x44,
-0x50,
-0x54,
-0x0,
-0x70,
-0x50,
-0x45,
-0x50,
-0x54,
-0x60,
-0xa4,
-0x60,
-0x14,
-0xc,
-0x57,
-0x52,
-0x50,
-0x54,
-0x1,
-0x70,
-0x68,
-0x50,
-0x45,
-0x50,
-0x54,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0xd,
-0xa,
-0xa,
-0x47,
-0x1,
-0x0,
-0x0,
-0x0,
-0x0,
-0x1,
-0x1,
-0x79,
-0x0,
-0x8b,
-0x5f,
-0x43,
-0x52,
-0x53,
-0xa,
-0x2,
-0x49,
-0x4f,
-0x4d,
-0x4e,
-0x8b,
-0x5f,
-0x43,
-0x52,
-0x53,
-0xa,
-0x4,
-0x49,
-0x4f,
-0x4d,
-0x58,
-0x14,
-0x18,
-0x5f,
-0x49,
-0x4e,
-0x49,
-0x0,
-0x70,
-0x50,
-0x45,
-0x53,
-0x54,
-0x49,
-0x4f,
-0x4d,
-0x4e,
-0x70,
-0x50,
-0x45,
-0x53,
-0x54,
-0x49,
-0x4f,
-0x4d,
-0x58
-};
-static unsigned char ssdt_isa_pest[] = {
-0xda
-};
-static unsigned char acpi_s4_name[] = {
-0x92
-};
-static unsigned char acpi_pci64_start[] = {
-0x4d
-};
-static unsigned char acpi_pci64_end[] = {
-0x5e
-};
-static unsigned char acpi_pci32_end[] = {
-0x39
-};
diff --git a/hw/i386/ssdt-pcihp.dsl b/hw/i386/ssdt-pcihp.dsl
deleted file mode 100644
index ac91c05..0000000
--- a/hw/i386/ssdt-pcihp.dsl
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-ACPI_EXTRACT_ALL_CODE ssdp_pcihp_aml
-
-DefinitionBlock ("ssdt-pcihp.aml", "SSDT", 0x01, "BXPC", "BXSSDTPCIHP", 0x1)
-{
-
-/****************************************************************
- * PCI hotplug
- ****************************************************************/
-
-    /* Objects supplied by DSDT */
-    External(\_SB.PCI0, DeviceObj)
-    External(\_SB.PCI0.PCEJ, MethodObj)
-    External(BSEL, IntObj)
-
-    Scope(\_SB.PCI0) {
-
-        /* Bulk generated PCI hotplug devices */
-        ACPI_EXTRACT_DEVICE_START ssdt_pcihp_start
-        ACPI_EXTRACT_DEVICE_END ssdt_pcihp_end
-        ACPI_EXTRACT_DEVICE_STRING ssdt_pcihp_name
-
-        // Extract the offsets of the device name, address dword and the slot
-        // name byte - we fill them in for each device.
-        Device(SAA) {
-            ACPI_EXTRACT_NAME_BYTE_CONST ssdt_pcihp_id
-            Name(_SUN, 0xAA)
-            ACPI_EXTRACT_NAME_DWORD_CONST ssdt_pcihp_adr
-            Name(_ADR, 0xAA0000)
-            Method(_EJ0, 1) {
-                PCEJ(BSEL, _SUN)
-            }
-        }
-
-        ACPI_EXTRACT_DEVICE_START ssdt_pcinohp_start
-        ACPI_EXTRACT_DEVICE_END ssdt_pcinohp_end
-        ACPI_EXTRACT_DEVICE_STRING ssdt_pcinohp_name
-
-        // Extract the offsets of the device name, address dword and the slot
-        // name byte - we fill them in for each device.
-        Device(SBB) {
-            ACPI_EXTRACT_NAME_DWORD_CONST ssdt_pcinohp_adr
-            Name(_ADR, 0xAA0000)
-        }
-
-        ACPI_EXTRACT_DEVICE_START ssdt_pcivga_start
-        ACPI_EXTRACT_DEVICE_END ssdt_pcivga_end
-        ACPI_EXTRACT_DEVICE_STRING ssdt_pcivga_name
-
-        // Extract the offsets of the device name, address dword and the slot
-        // name byte - we fill them in for each device.
-        Device(SCC) {
-            ACPI_EXTRACT_NAME_DWORD_CONST ssdt_pcivga_adr
-            Name(_ADR, 0xAA0000)
-            Method(_S1D, 0, NotSerialized) {
-                Return (0x00)
-            }
-            Method(_S2D, 0, NotSerialized) {
-                Return (0x00)
-            }
-            Method(_S3D, 0, NotSerialized) {
-                Return (0x00)
-            }
-        }
-
-        ACPI_EXTRACT_DEVICE_START ssdt_pciqxl_start
-        ACPI_EXTRACT_DEVICE_END ssdt_pciqxl_end
-        ACPI_EXTRACT_DEVICE_STRING ssdt_pciqxl_name
-
-        // Extract the offsets of the device name, address dword and the slot
-        // name byte - we fill them in for each device.
-        Device(SDD) {
-            ACPI_EXTRACT_NAME_DWORD_CONST ssdt_pciqxl_adr
-            Name(_ADR, 0xAA0000)
-            Method(_S1D, 0, NotSerialized) {
-                Return (0x00)
-            }
-            Method(_S2D, 0, NotSerialized) {
-                Return (0x00)
-            }
-            Method(_S3D, 0, NotSerialized) {
-                Return (0x03)           // QXL
-            }
-        }
-    }
-}
diff --git a/hw/i386/ssdt-pcihp.hex.generated b/hw/i386/ssdt-pcihp.hex.generated
deleted file mode 100644
index 72ffa84..0000000
--- a/hw/i386/ssdt-pcihp.hex.generated
+++ /dev/null
@@ -1,251 +0,0 @@
-static unsigned char ssdt_pcihp_name[] = {
-0x34
-};
-static unsigned char ssdt_pcivga_end[] = {
-0x99
-};
-static unsigned char ssdt_pcivga_name[] = {
-0x70
-};
-static unsigned char ssdt_pcihp_adr[] = {
-0x45
-};
-static unsigned char ssdt_pcinohp_end[] = {
-0x6d
-};
-static unsigned char ssdt_pcihp_end[] = {
-0x5c
-};
-static unsigned char ssdt_pciqxl_start[] = {
-0x99
-};
-static unsigned char ssdt_pcinohp_name[] = {
-0x5f
-};
-static unsigned char ssdp_pcihp_aml[] = {
-0x53,
-0x53,
-0x44,
-0x54,
-0xc6,
-0x0,
-0x0,
-0x0,
-0x1,
-0x70,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x53,
-0x53,
-0x44,
-0x54,
-0x50,
-0x43,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x15,
-0x11,
-0x13,
-0x20,
-0x10,
-0x41,
-0xa,
-0x5c,
-0x2e,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x50,
-0x43,
-0x49,
-0x30,
-0x5b,
-0x82,
-0x29,
-0x53,
-0x41,
-0x41,
-0x5f,
-0x8,
-0x5f,
-0x53,
-0x55,
-0x4e,
-0xa,
-0xaa,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0xaa,
-0x0,
-0x14,
-0x12,
-0x5f,
-0x45,
-0x4a,
-0x30,
-0x1,
-0x50,
-0x43,
-0x45,
-0x4a,
-0x42,
-0x53,
-0x45,
-0x4c,
-0x5f,
-0x53,
-0x55,
-0x4e,
-0x5b,
-0x82,
-0xf,
-0x53,
-0x42,
-0x42,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0xaa,
-0x0,
-0x5b,
-0x82,
-0x2a,
-0x53,
-0x43,
-0x43,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0xaa,
-0x0,
-0x14,
-0x8,
-0x5f,
-0x53,
-0x31,
-0x44,
-0x0,
-0xa4,
-0x0,
-0x14,
-0x8,
-0x5f,
-0x53,
-0x32,
-0x44,
-0x0,
-0xa4,
-0x0,
-0x14,
-0x8,
-0x5f,
-0x53,
-0x33,
-0x44,
-0x0,
-0xa4,
-0x0,
-0x5b,
-0x82,
-0x2b,
-0x53,
-0x44,
-0x44,
-0x5f,
-0x8,
-0x5f,
-0x41,
-0x44,
-0x52,
-0xc,
-0x0,
-0x0,
-0xaa,
-0x0,
-0x14,
-0x8,
-0x5f,
-0x53,
-0x31,
-0x44,
-0x0,
-0xa4,
-0x0,
-0x14,
-0x8,
-0x5f,
-0x53,
-0x32,
-0x44,
-0x0,
-0xa4,
-0x0,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x33,
-0x44,
-0x0,
-0xa4,
-0xa,
-0x3
-};
-static unsigned char ssdt_pciqxl_adr[] = {
-0xa6
-};
-static unsigned char ssdt_pcinohp_adr[] = {
-0x69
-};
-static unsigned char ssdt_pcivga_adr[] = {
-0x7a
-};
-static unsigned char ssdt_pciqxl_name[] = {
-0x9c
-};
-static unsigned char ssdt_pcivga_start[] = {
-0x6d
-};
-static unsigned char ssdt_pciqxl_end[] = {
-0xc6
-};
-static unsigned char ssdt_pcihp_start[] = {
-0x31
-};
-static unsigned char ssdt_pcihp_id[] = {
-0x3e
-};
-static unsigned char ssdt_pcinohp_start[] = {
-0x5c
-};
diff --git a/hw/i386/ssdt-proc.dsl b/hw/i386/ssdt-proc.dsl
deleted file mode 100644
index 8229bfd..0000000
--- a/hw/i386/ssdt-proc.dsl
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* This file is the basis for the ssdt table generated in src/acpi.c.
- * It defines the contents of the per-cpu Processor() object.  At
- * runtime, a dynamically generated SSDT will contain one copy of this
- * AML snippet for every possible cpu in the system.  The objects will
- * be placed in the \_SB_ namespace.
- *
- * In addition to the aml code generated from this file, the
- * src/acpi.c file creates a NTFY method with an entry for each cpu:
- *     Method(NTFY, 2) {
- *         If (LEqual(Arg0, 0x00)) { Notify(CP00, Arg1) }
- *         If (LEqual(Arg0, 0x01)) { Notify(CP01, Arg1) }
- *         ...
- *     }
- * and a CPON array with the list of active and inactive cpus:
- *     Name(CPON, Package() { One, One, ..., Zero, Zero, ... })
- */
-
-ACPI_EXTRACT_ALL_CODE ssdp_proc_aml
-
-DefinitionBlock ("ssdt-proc.aml", "SSDT", 0x01, "BXPC", "BXSSDT", 0x1)
-{
-    ACPI_EXTRACT_PROCESSOR_START ssdt_proc_start
-    ACPI_EXTRACT_PROCESSOR_END ssdt_proc_end
-    ACPI_EXTRACT_PROCESSOR_STRING ssdt_proc_name
-    Processor(CPAA, 0xAA, 0x00000000, 0x0) {
-        ACPI_EXTRACT_NAME_BYTE_CONST ssdt_proc_id
-        Name(ID, 0xAA)
-/*
- * The src/acpi.c code requires the above ACP_EXTRACT tags so that it can update
- * CPAA and 0xAA with the appropriate CPU id (see
- * SD_OFFSET_CPUHEX/CPUID1/CPUID2).  Don't change the above without
- * also updating the C code.
- */
-        Name(_HID, "ACPI0007")
-        External(CPMA, MethodObj)
-        External(CPST, MethodObj)
-        External(CPEJ, MethodObj)
-        Method(_MAT, 0) {
-            Return (CPMA(ID))
-        }
-        Method(_STA, 0) {
-            Return (CPST(ID))
-        }
-        Method(_EJ0, 1, NotSerialized) {
-            CPEJ(ID, Arg0)
-        }
-    }
-}
diff --git a/hw/i386/ssdt-proc.hex.generated b/hw/i386/ssdt-proc.hex.generated
deleted file mode 100644
index 4df0734..0000000
--- a/hw/i386/ssdt-proc.hex.generated
+++ /dev/null
@@ -1,134 +0,0 @@
-static unsigned char ssdt_proc_name[] = {
-0x28
-};
-static unsigned char ssdp_proc_aml[] = {
-0x53,
-0x53,
-0x44,
-0x54,
-0x78,
-0x0,
-0x0,
-0x0,
-0x1,
-0x7d,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x53,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x15,
-0x11,
-0x13,
-0x20,
-0x5b,
-0x83,
-0x42,
-0x5,
-0x43,
-0x50,
-0x41,
-0x41,
-0xaa,
-0x0,
-0x0,
-0x0,
-0x0,
-0x0,
-0x8,
-0x49,
-0x44,
-0x5f,
-0x5f,
-0xa,
-0xaa,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xd,
-0x41,
-0x43,
-0x50,
-0x49,
-0x30,
-0x30,
-0x30,
-0x37,
-0x0,
-0x14,
-0xf,
-0x5f,
-0x4d,
-0x41,
-0x54,
-0x0,
-0xa4,
-0x43,
-0x50,
-0x4d,
-0x41,
-0x49,
-0x44,
-0x5f,
-0x5f,
-0x14,
-0xf,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0x43,
-0x50,
-0x53,
-0x54,
-0x49,
-0x44,
-0x5f,
-0x5f,
-0x14,
-0xf,
-0x5f,
-0x45,
-0x4a,
-0x30,
-0x1,
-0x43,
-0x50,
-0x45,
-0x4a,
-0x49,
-0x44,
-0x5f,
-0x5f,
-0x68
-};
-static unsigned char ssdt_proc_id[] = {
-0x38
-};
-static unsigned char ssdt_proc_end[] = {
-0x78
-};
-static unsigned char ssdt_proc_start[] = {
-0x24
-};
diff --git a/hw/i386/ssdt-tpm.dsl b/hw/i386/ssdt-tpm.dsl
deleted file mode 100644
index 75d9691..0000000
--- a/hw/i386/ssdt-tpm.dsl
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
-
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-#include "hw/acpi/tpm.h"
-
-ACPI_EXTRACT_ALL_CODE ssdt_tpm_aml
-
-DefinitionBlock (
-    "ssdt-tpm.aml",     // Output Filename
-    "SSDT",             // Signature
-    0x01,               // SSDT Compliance Revision
-    "BXPC",             // OEMID
-    "BXSSDT",           // TABLE ID
-    0x1                 // OEM Revision
-    )
-{
-    Scope(\_SB) {
-        /* TPM with emulated TPM TIS interface */
-        Device (TPM) {
-            Name (_HID, EisaID ("PNP0C31"))
-            Name (_CRS, ResourceTemplate ()
-            {
-                Memory32Fixed (ReadWrite, TPM_TIS_ADDR_BASE, TPM_TIS_ADDR_SIZE)
-                // older Linux tpm_tis drivers do not work with IRQ
-                //IRQNoFlags () {TPM_TIS_IRQ}
-            })
-            Method (_STA, 0, NotSerialized) {
-                Return (0x0F)
-            }
-        }
-    }
-}
diff --git a/hw/i386/ssdt-tpm.hex.generated b/hw/i386/ssdt-tpm.hex.generated
deleted file mode 100644
index 4a916a8..0000000
--- a/hw/i386/ssdt-tpm.hex.generated
+++ /dev/null
@@ -1,95 +0,0 @@
-static unsigned char ssdt_tpm_aml[] = {
-0x53,
-0x53,
-0x44,
-0x54,
-0x5d,
-0x0,
-0x0,
-0x0,
-0x1,
-0xf,
-0x42,
-0x58,
-0x50,
-0x43,
-0x0,
-0x0,
-0x42,
-0x58,
-0x53,
-0x53,
-0x44,
-0x54,
-0x0,
-0x0,
-0x1,
-0x0,
-0x0,
-0x0,
-0x49,
-0x4e,
-0x54,
-0x4c,
-0x15,
-0x11,
-0x13,
-0x20,
-0x10,
-0x38,
-0x5c,
-0x5f,
-0x53,
-0x42,
-0x5f,
-0x5b,
-0x82,
-0x30,
-0x54,
-0x50,
-0x4d,
-0x5f,
-0x8,
-0x5f,
-0x48,
-0x49,
-0x44,
-0xc,
-0x41,
-0xd0,
-0xc,
-0x31,
-0x8,
-0x5f,
-0x43,
-0x52,
-0x53,
-0x11,
-0x11,
-0xa,
-0xe,
-0x86,
-0x9,
-0x0,
-0x1,
-0x0,
-0x0,
-0xd4,
-0xfe,
-0x0,
-0x50,
-0x0,
-0x0,
-0x79,
-0x0,
-0x14,
-0x9,
-0x5f,
-0x53,
-0x54,
-0x41,
-0x0,
-0xa4,
-0xa,
-0xf
-};
diff --git a/hw/i386/trace-events b/hw/i386/trace-events
new file mode 100644
index 0000000..7735e46
--- /dev/null
+++ b/hw/i386/trace-events
@@ -0,0 +1,15 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/i386/xen/xen_platform.c
+xen_platform_log(char *s) "xen platform: %s"
+
+# hw/i386/xen/xen_pvdevice.c
+xen_pv_mmio_read(uint64_t addr) "WARNING: read from Xen PV Device MMIO space (address %"PRIx64")"
+xen_pv_mmio_write(uint64_t addr) "WARNING: write to Xen PV Device MMIO space (address %"PRIx64")"
+
+# hw/i386/pc.c
+mhp_pc_dimm_assigned_slot(int slot) "0x%d"
+mhp_pc_dimm_assigned_address(uint64_t addr) "0x%"PRIx64
+
+# hw/i386/x86-iommu.c
+x86_iommu_iec_notify(bool global, uint32_t index, uint32_t mask) "Notify IEC invalidation: global=%d index=%" PRIu32 " mask=%" PRIu32
diff --git a/hw/i386/x86-iommu.c b/hw/i386/x86-iommu.c
new file mode 100644
index 0000000..ce26b2a
--- /dev/null
+++ b/hw/i386/x86-iommu.c
@@ -0,0 +1,128 @@
+/*
+ * QEMU emulation of common X86 IOMMU
+ *
+ * Copyright (C) 2016 Peter Xu, Red Hat <peterx@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+#include "hw/boards.h"
+#include "hw/i386/x86-iommu.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+
+void x86_iommu_iec_register_notifier(X86IOMMUState *iommu,
+                                     iec_notify_fn fn, void *data)
+{
+    IEC_Notifier *notifier = g_new0(IEC_Notifier, 1);
+
+    notifier->iec_notify = fn;
+    notifier->private = data;
+
+    QLIST_INSERT_HEAD(&iommu->iec_notifiers, notifier, list);
+}
+
+void x86_iommu_iec_notify_all(X86IOMMUState *iommu, bool global,
+                              uint32_t index, uint32_t mask)
+{
+    IEC_Notifier *notifier;
+
+    trace_x86_iommu_iec_notify(global, index, mask);
+
+    QLIST_FOREACH(notifier, &iommu->iec_notifiers, list) {
+        if (notifier->iec_notify) {
+            notifier->iec_notify(notifier->private, global,
+                                 index, mask);
+        }
+    }
+}
+
+/* Default X86 IOMMU device */
+static X86IOMMUState *x86_iommu_default = NULL;
+
+static void x86_iommu_set_default(X86IOMMUState *x86_iommu)
+{
+    assert(x86_iommu);
+
+    if (x86_iommu_default) {
+        error_report("QEMU does not support multiple vIOMMUs "
+                     "for x86 yet.");
+        exit(1);
+    }
+
+    x86_iommu_default = x86_iommu;
+}
+
+X86IOMMUState *x86_iommu_get_default(void)
+{
+    return x86_iommu_default;
+}
+
+static void x86_iommu_realize(DeviceState *dev, Error **errp)
+{
+    X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
+    X86IOMMUClass *x86_class = X86_IOMMU_GET_CLASS(dev);
+    QLIST_INIT(&x86_iommu->iec_notifiers);
+    if (x86_class->realize) {
+        x86_class->realize(dev, errp);
+    }
+    x86_iommu_set_default(X86_IOMMU_DEVICE(dev));
+}
+
+static void x86_iommu_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->realize = x86_iommu_realize;
+}
+
+static bool x86_iommu_intremap_prop_get(Object *o, Error **errp)
+{
+    X86IOMMUState *s = X86_IOMMU_DEVICE(o);
+    return s->intr_supported;
+}
+
+static void x86_iommu_intremap_prop_set(Object *o, bool value, Error **errp)
+{
+    X86IOMMUState *s = X86_IOMMU_DEVICE(o);
+    s->intr_supported = value;
+}
+
+static void x86_iommu_instance_init(Object *o)
+{
+    X86IOMMUState *s = X86_IOMMU_DEVICE(o);
+
+    /* By default, do not support IR */
+    s->intr_supported = false;
+    object_property_add_bool(o, "intremap", x86_iommu_intremap_prop_get,
+                             x86_iommu_intremap_prop_set, NULL);
+}
+
+static const TypeInfo x86_iommu_info = {
+    .name          = TYPE_X86_IOMMU_DEVICE,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_init = x86_iommu_instance_init,
+    .instance_size = sizeof(X86IOMMUState),
+    .class_init    = x86_iommu_class_init,
+    .class_size    = sizeof(X86IOMMUClass),
+    .abstract      = true,
+};
+
+static void x86_iommu_register_types(void)
+{
+    type_register_static(&x86_iommu_info);
+}
+
+type_init(x86_iommu_register_types)
diff --git a/hw/i386/xen/xen_apic.c b/hw/i386/xen/xen_apic.c
index f5acd6a..21d68ee 100644
--- a/hw/i386/xen/xen_apic.c
+++ b/hw/i386/xen/xen_apic.c
@@ -9,6 +9,7 @@
  * This work is licensed under the terms of the GNU GPL version 2 or
  * later. See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/pci/msi.h"
 #include "hw/xen/xen.h"
@@ -43,11 +44,7 @@
     s->vapic_control = 0;
     memory_region_init_io(&s->io_memory, OBJECT(s), &xen_apic_io_ops, s,
                           "xen-apic-msi", APIC_SPACE_SIZE);
-
-#if defined(CONFIG_XEN_CTRL_INTERFACE_VERSION) \
-    && CONFIG_XEN_CTRL_INTERFACE_VERSION >= 420
-    msi_supported = true;
-#endif
+    msi_nonbroken = true;
 }
 
 static void xen_apic_set_base(APICCommonState *s, uint64_t val)
diff --git a/hw/i386/xen/xen_platform.c b/hw/i386/xen/xen_platform.c
index 28b324a..aa78393 100644
--- a/hw/i386/xen/xen_platform.c
+++ b/hw/i386/xen/xen_platform.c
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
-#include <assert.h>
-
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/ide.h"
@@ -35,6 +35,7 @@
 #include "trace.h"
 #include "exec/address-spaces.h"
 #include "sysemu/block-backend.h"
+#include "qemu/error-report.h"
 
 #include <xenguest.h>
 
@@ -384,11 +385,17 @@
     }
 };
 
-static int xen_platform_initfn(PCIDevice *dev)
+static void xen_platform_realize(PCIDevice *dev, Error **errp)
 {
     PCIXenPlatformState *d = XEN_PLATFORM(dev);
     uint8_t *pci_conf;
 
+    /* Device will crash on reset if xen is not initialized */
+    if (!xen_enabled()) {
+        error_setg(errp, "xen-platform device requires the Xen accelerator");
+        return;
+    }
+
     pci_conf = dev->config;
 
     pci_set_word(pci_conf + PCI_COMMAND, PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
@@ -406,8 +413,6 @@
                      &d->mmio_bar);
 
     platform_fixed_ioport_init(d);
-
-    return 0;
 }
 
 static void platform_reset(DeviceState *dev)
@@ -422,7 +427,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_platform_initfn;
+    k->realize = xen_platform_realize;
     k->vendor_id = PCI_VENDOR_ID_XEN;
     k->device_id = PCI_DEVICE_ID_XEN_PLATFORM;
     k->class_id = PCI_CLASS_OTHERS << 8 | 0x80;
diff --git a/hw/i386/xen/xen_pvdevice.c b/hw/i386/xen/xen_pvdevice.c
index c218947..c093b34 100644
--- a/hw/i386/xen/xen_pvdevice.c
+++ b/hw/i386/xen/xen_pvdevice.c
@@ -29,6 +29,8 @@
  * SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "trace.h"
@@ -69,14 +71,16 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int xen_pv_init(PCIDevice *pci_dev)
+static void xen_pv_realize(PCIDevice *pci_dev, Error **errp)
 {
     XenPVDevice *d = XEN_PV_DEVICE(pci_dev);
     uint8_t *pci_conf;
 
     /* device-id property must always be supplied */
-    if (d->device_id == 0xffff)
-	    return -1;
+    if (d->device_id == 0xffff) {
+        error_setg(errp, "Device ID invalid, it must always be supplied");
+        return;
+    }
 
     pci_conf = pci_dev->config;
 
@@ -97,8 +101,6 @@
 
     pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_MEM_PREFETCH,
                      &d->mmio);
-
-    return 0;
 }
 
 static Property xen_pv_props[] = {
@@ -114,7 +116,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_pv_init;
+    k->realize = xen_pv_realize;
     k->class_id = PCI_CLASS_SYSTEM_OTHER;
     dc->desc = "Xen PV Device";
     dc->props = xen_pv_props;
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 94f28e6..f3438ad 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -21,36 +21,39 @@
  *
  */
 
-#include <hw/hw.h>
-#include <hw/pci/msi.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/sysbus.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/msi.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
 
-#include "monitor/monitor.h"
+#include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
-#include "internal.h"
-#include <hw/ide/pci.h>
-#include <hw/ide/ahci.h>
+#include "hw/ide/internal.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
 
-/* #define DEBUG_AHCI */
+#define DEBUG_AHCI 0
 
-#ifdef DEBUG_AHCI
 #define DPRINTF(port, fmt, ...) \
-do { fprintf(stderr, "ahci: %s: [%d] ", __FUNCTION__, port); \
-     fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(port, fmt, ...) do {} while(0)
-#endif
+do { \
+    if (DEBUG_AHCI) { \
+        fprintf(stderr, "ahci: %s: [%d] ", __func__, port); \
+        fprintf(stderr, fmt, ## __VA_ARGS__); \
+    } \
+} while (0)
 
 static void check_cmd(AHCIState *s, int port);
-static int handle_cmd(AHCIState *s,int port,int slot);
+static int handle_cmd(AHCIState *s, int port, uint8_t slot);
 static void ahci_reset_port(AHCIState *s, int port);
-static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis);
+static bool ahci_write_fis_d2h(AHCIDevice *ad);
 static void ahci_init_d2h(AHCIDevice *ad);
-static int ahci_dma_prepare_buf(IDEDMA *dma, int is_write);
-static void ahci_commit_buf(IDEDMA *dma, uint32_t tx_bytes);
+static int ahci_dma_prepare_buf(IDEDMA *dma, int32_t limit);
+static bool ahci_map_clb_address(AHCIDevice *ad);
+static bool ahci_map_fis_address(AHCIDevice *ad);
+static void ahci_unmap_clb_address(AHCIDevice *ad);
+static void ahci_unmap_fis_address(AHCIDevice *ad);
 
 
 static uint32_t  ahci_port_read(AHCIState *s, int port, int offset)
@@ -102,8 +105,6 @@
         val = pr->scr_err;
         break;
     case PORT_SCR_ACT:
-        pr->scr_act &= ~s->dev[port].finished;
-        s->dev[port].finished = 0;
         val = pr->scr_act;
         break;
     case PORT_CMD_ISSUE:
@@ -120,9 +121,9 @@
 
 static void ahci_irq_raise(AHCIState *s, AHCIDevice *dev)
 {
-    AHCIPCIState *d = container_of(s, AHCIPCIState, ahci);
-    PCIDevice *pci_dev =
-        (PCIDevice *)object_dynamic_cast(OBJECT(d), TYPE_PCI_DEVICE);
+    DeviceState *dev_state = s->container;
+    PCIDevice *pci_dev = (PCIDevice *) object_dynamic_cast(OBJECT(dev_state),
+                                                           TYPE_PCI_DEVICE);
 
     DPRINTF(0, "raise irq\n");
 
@@ -135,9 +136,9 @@
 
 static void ahci_irq_lower(AHCIState *s, AHCIDevice *dev)
 {
-    AHCIPCIState *d = container_of(s, AHCIPCIState, ahci);
-    PCIDevice *pci_dev =
-        (PCIDevice *)object_dynamic_cast(OBJECT(d), TYPE_PCI_DEVICE);
+    DeviceState *dev_state = s->container;
+    PCIDevice *pci_dev = (PCIDevice *) object_dynamic_cast(OBJECT(dev_state),
+                                                           TYPE_PCI_DEVICE);
 
     DPRINTF(0, "lower irq\n");
 
@@ -194,6 +195,47 @@
     }
 }
 
+/**
+ * Check the cmd register to see if we should start or stop
+ * the DMA or FIS RX engines.
+ *
+ * @ad: Device to dis/engage.
+ *
+ * @return 0 on success, -1 on error.
+ */
+static int ahci_cond_start_engines(AHCIDevice *ad)
+{
+    AHCIPortRegs *pr = &ad->port_regs;
+    bool cmd_start = pr->cmd & PORT_CMD_START;
+    bool cmd_on    = pr->cmd & PORT_CMD_LIST_ON;
+    bool fis_start = pr->cmd & PORT_CMD_FIS_RX;
+    bool fis_on    = pr->cmd & PORT_CMD_FIS_ON;
+
+    if (cmd_start && !cmd_on) {
+        if (!ahci_map_clb_address(ad)) {
+            pr->cmd &= ~PORT_CMD_START;
+            error_report("AHCI: Failed to start DMA engine: "
+                         "bad command list buffer address");
+            return -1;
+        }
+    } else if (!cmd_start && cmd_on) {
+        ahci_unmap_clb_address(ad);
+    }
+
+    if (fis_start && !fis_on) {
+        if (!ahci_map_fis_address(ad)) {
+            pr->cmd &= ~PORT_CMD_FIS_RX;
+            error_report("AHCI: Failed to start FIS receive engine: "
+                         "bad FIS receive buffer address");
+            return -1;
+        }
+    } else if (!fis_start && fis_on) {
+        ahci_unmap_fis_address(ad);
+    }
+
+    return 0;
+}
+
 static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
 {
     AHCIPortRegs *pr = &s->dev[port].port_regs;
@@ -202,25 +244,15 @@
     switch (offset) {
         case PORT_LST_ADDR:
             pr->lst_addr = val;
-            map_page(s->as, &s->dev[port].lst,
-                     ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
-            s->dev[port].cur_cmd = NULL;
             break;
         case PORT_LST_ADDR_HI:
             pr->lst_addr_hi = val;
-            map_page(s->as, &s->dev[port].lst,
-                     ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
-            s->dev[port].cur_cmd = NULL;
             break;
         case PORT_FIS_ADDR:
             pr->fis_addr = val;
-            map_page(s->as, &s->dev[port].res_fis,
-                     ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
             break;
         case PORT_FIS_ADDR_HI:
             pr->fis_addr_hi = val;
-            map_page(s->as, &s->dev[port].res_fis,
-                     ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
             break;
         case PORT_IRQ_STAT:
             pr->irq_stat &= ~val;
@@ -231,15 +263,17 @@
             ahci_check_irq(s);
             break;
         case PORT_CMD:
-            pr->cmd = val & ~(PORT_CMD_LIST_ON | PORT_CMD_FIS_ON);
+            /* Block any Read-only fields from being set;
+             * including LIST_ON and FIS_ON.
+             * The spec requires to set ICC bits to zero after the ICC change
+             * is done. We don't support ICC state changes, therefore always
+             * force the ICC bits to zero.
+             */
+            pr->cmd = (pr->cmd & PORT_CMD_RO_MASK) |
+                      (val & ~(PORT_CMD_RO_MASK|PORT_CMD_ICC_MASK));
 
-            if (pr->cmd & PORT_CMD_START) {
-                pr->cmd |= PORT_CMD_LIST_ON;
-            }
-
-            if (pr->cmd & PORT_CMD_FIS_RX) {
-                pr->cmd |= PORT_CMD_FIS_ON;
-            }
+            /* Check FIS RX and CLB engines */
+            ahci_cond_start_engines(&s->dev[port]);
 
             /* XXX usually the FIS would be pending on the bus here and
                    issuing deferred until the OS enables FIS receival.
@@ -248,7 +282,6 @@
             if ((pr->cmd & PORT_CMD_FIS_ON) &&
                 !s->dev[port].init_d2h_sent) {
                 ahci_init_d2h(&s->dev[port]);
-                s->dev[port].init_d2h_sent = true;
             }
 
             check_cmd(s, port);
@@ -285,8 +318,7 @@
     }
 }
 
-static uint64_t ahci_mem_read(void *opaque, hwaddr addr,
-                              unsigned size)
+static uint64_t ahci_mem_read_32(void *opaque, hwaddr addr)
 {
     AHCIState *s = opaque;
     uint32_t val = 0;
@@ -322,12 +354,45 @@
 }
 
 
+/**
+ * AHCI 1.3 section 3 ("HBA Memory Registers")
+ * Support unaligned 8/16/32 bit reads, and 64 bit aligned reads.
+ * Caller is responsible for masking unwanted higher order bytes.
+ */
+static uint64_t ahci_mem_read(void *opaque, hwaddr addr, unsigned size)
+{
+    hwaddr aligned = addr & ~0x3;
+    int ofst = addr - aligned;
+    uint64_t lo = ahci_mem_read_32(opaque, aligned);
+    uint64_t hi;
+    uint64_t val;
+
+    /* if < 8 byte read does not cross 4 byte boundary */
+    if (ofst + size <= 4) {
+        val = lo >> (ofst * 8);
+    } else {
+        g_assert_cmpint(size, >, 1);
+
+        /* If the 64bit read is unaligned, we will produce undefined
+         * results. AHCI does not support unaligned 64bit reads. */
+        hi = ahci_mem_read_32(opaque, aligned + 4);
+        val = (hi << 32 | lo) >> (ofst * 8);
+    }
+
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    return val;
+}
+
 
 static void ahci_mem_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     AHCIState *s = opaque;
 
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+
     /* Only aligned reads are allowed on AHCI */
     if (addr & 3) {
         fprintf(stderr, "ahci: Mis-aligned write to addr 0x"
@@ -437,7 +502,7 @@
 static void check_cmd(AHCIState *s, int port)
 {
     AHCIPortRegs *pr = &s->dev[port].port_regs;
-    int slot;
+    uint8_t slot;
 
     if ((pr->cmd & PORT_CMD_START) && pr->cmd_issue) {
         for (slot = 0; (slot < 32) && pr->cmd_issue; slot++) {
@@ -468,20 +533,33 @@
 
 static void ahci_init_d2h(AHCIDevice *ad)
 {
-    uint8_t init_fis[20];
     IDEState *ide_state = &ad->port.ifs[0];
+    AHCIPortRegs *pr = &ad->port_regs;
 
-    memset(init_fis, 0, sizeof(init_fis));
-
-    init_fis[4] = 1;
-    init_fis[12] = 1;
-
-    if (ide_state->drive_kind == IDE_CD) {
-        init_fis[5] = ide_state->lcyl;
-        init_fis[6] = ide_state->hcyl;
+    if (ad->init_d2h_sent) {
+        return;
     }
 
-    ahci_write_fis_d2h(ad, init_fis);
+    if (ahci_write_fis_d2h(ad)) {
+        ad->init_d2h_sent = true;
+        /* We're emulating receiving the first Reg H2D Fis from the device;
+         * Update the SIG register, but otherwise proceed as normal. */
+        pr->sig = ((uint32_t)ide_state->hcyl << 24) |
+            (ide_state->lcyl << 16) |
+            (ide_state->sector << 8) |
+            (ide_state->nsector & 0xFF);
+    }
+}
+
+static void ahci_set_signature(AHCIDevice *ad, uint32_t sig)
+{
+    IDEState *s = &ad->port.ifs[0];
+    s->hcyl = sig >> 24 & 0xFF;
+    s->lcyl = sig >> 16 & 0xFF;
+    s->sector = sig >> 8 & 0xFF;
+    s->nsector = sig & 0xFF;
+
+    DPRINTF(ad->port_no, "set hcyl:lcyl:sect:nsect = 0x%08x\n", sig);
 }
 
 static void ahci_reset_port(AHCIState *s, int port)
@@ -512,6 +590,7 @@
     /* reset ncq queue */
     for (i = 0; i < AHCI_MAX_CMDS; i++) {
         NCQTransferState *ncq_tfs = &s->dev[port].ncq_tfs[i];
+        ncq_tfs->halt = false;
         if (!ncq_tfs->used) {
             continue;
         }
@@ -531,17 +610,11 @@
     }
 
     s->dev[port].port_state = STATE_RUN;
-    if (!ide_state->blk) {
-        pr->sig = 0;
-        ide_state->status = SEEK_STAT | WRERR_STAT;
-    } else if (ide_state->drive_kind == IDE_CD) {
-        pr->sig = SATA_SIGNATURE_CDROM;
-        ide_state->lcyl = 0x14;
-        ide_state->hcyl = 0xeb;
-        DPRINTF(port, "set lcyl = %d\n", ide_state->lcyl);
+    if (ide_state->drive_kind == IDE_CD) {
+        ahci_set_signature(d, SATA_SIGNATURE_CDROM);\
         ide_state->status = SEEK_STAT | WRERR_STAT | READY_STAT;
     } else {
-        pr->sig = SATA_SIGNATURE_DISK;
+        ahci_set_signature(d, SATA_SIGNATURE_DISK);
         ide_state->status = SEEK_STAT | WRERR_STAT;
     }
 
@@ -551,7 +624,7 @@
 
 static void debug_print_fis(uint8_t *fis, int cmd_len)
 {
-#ifdef DEBUG_AHCI
+#if DEBUG_AHCI
     int i;
 
     fprintf(stderr, "fis:");
@@ -565,14 +638,67 @@
 #endif
 }
 
-static void ahci_write_fis_sdb(AHCIState *s, int port, uint32_t finished)
+static bool ahci_map_fis_address(AHCIDevice *ad)
 {
-    AHCIDevice *ad = &s->dev[port];
+    AHCIPortRegs *pr = &ad->port_regs;
+    map_page(ad->hba->as, &ad->res_fis,
+             ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
+    if (ad->res_fis != NULL) {
+        pr->cmd |= PORT_CMD_FIS_ON;
+        return true;
+    }
+
+    pr->cmd &= ~PORT_CMD_FIS_ON;
+    return false;
+}
+
+static void ahci_unmap_fis_address(AHCIDevice *ad)
+{
+    if (ad->res_fis == NULL) {
+        DPRINTF(ad->port_no, "Attempt to unmap NULL FIS address\n");
+        return;
+    }
+    ad->port_regs.cmd &= ~PORT_CMD_FIS_ON;
+    dma_memory_unmap(ad->hba->as, ad->res_fis, 256,
+                     DMA_DIRECTION_FROM_DEVICE, 256);
+    ad->res_fis = NULL;
+}
+
+static bool ahci_map_clb_address(AHCIDevice *ad)
+{
+    AHCIPortRegs *pr = &ad->port_regs;
+    ad->cur_cmd = NULL;
+    map_page(ad->hba->as, &ad->lst,
+             ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
+    if (ad->lst != NULL) {
+        pr->cmd |= PORT_CMD_LIST_ON;
+        return true;
+    }
+
+    pr->cmd &= ~PORT_CMD_LIST_ON;
+    return false;
+}
+
+static void ahci_unmap_clb_address(AHCIDevice *ad)
+{
+    if (ad->lst == NULL) {
+        DPRINTF(ad->port_no, "Attempt to unmap NULL CLB address\n");
+        return;
+    }
+    ad->port_regs.cmd &= ~PORT_CMD_LIST_ON;
+    dma_memory_unmap(ad->hba->as, ad->lst, 1024,
+                     DMA_DIRECTION_FROM_DEVICE, 1024);
+    ad->lst = NULL;
+}
+
+static void ahci_write_fis_sdb(AHCIState *s, NCQTransferState *ncq_tfs)
+{
+    AHCIDevice *ad = ncq_tfs->drive;
     AHCIPortRegs *pr = &ad->port_regs;
     IDEState *ide_state;
     SDBFIS *sdb_fis;
 
-    if (!s->dev[port].res_fis ||
+    if (!ad->res_fis ||
         !(pr->cmd & PORT_CMD_FIS_RX)) {
         return;
     }
@@ -580,58 +706,40 @@
     sdb_fis = (SDBFIS *)&ad->res_fis[RES_FIS_SDBFIS];
     ide_state = &ad->port.ifs[0];
 
-    sdb_fis->type = 0xA1;
+    sdb_fis->type = SATA_FIS_TYPE_SDB;
     /* Interrupt pending & Notification bit */
-    sdb_fis->flags = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
+    sdb_fis->flags = 0x40; /* Interrupt bit, always 1 for NCQ */
     sdb_fis->status = ide_state->status & 0x77;
     sdb_fis->error = ide_state->error;
     /* update SAct field in SDB_FIS */
-    s->dev[port].finished |= finished;
     sdb_fis->payload = cpu_to_le32(ad->finished);
 
     /* Update shadow registers (except BSY 0x80 and DRQ 0x08) */
     pr->tfdata = (ad->port.ifs[0].error << 8) |
         (ad->port.ifs[0].status & 0x77) |
         (pr->tfdata & 0x88);
+    pr->scr_act &= ~ad->finished;
+    ad->finished = 0;
 
-    ahci_trigger_irq(s, ad, PORT_IRQ_SDB_FIS);
+    /* Trigger IRQ if interrupt bit is set (which currently, it always is) */
+    if (sdb_fis->flags & 0x40) {
+        ahci_trigger_irq(s, ad, PORT_IRQ_SDB_FIS);
+    }
 }
 
 static void ahci_write_fis_pio(AHCIDevice *ad, uint16_t len)
 {
     AHCIPortRegs *pr = &ad->port_regs;
-    uint8_t *pio_fis, *cmd_fis;
-    uint64_t tbl_addr;
-    dma_addr_t cmd_len = 0x80;
+    uint8_t *pio_fis;
     IDEState *s = &ad->port.ifs[0];
 
     if (!ad->res_fis || !(pr->cmd & PORT_CMD_FIS_RX)) {
         return;
     }
 
-    /* map cmd_fis */
-    tbl_addr = le64_to_cpu(ad->cur_cmd->tbl_addr);
-    cmd_fis = dma_memory_map(ad->hba->as, tbl_addr, &cmd_len,
-                             DMA_DIRECTION_TO_DEVICE);
-
-    if (cmd_fis == NULL) {
-        DPRINTF(ad->port_no, "dma_memory_map failed in ahci_write_fis_pio");
-        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_HBUS_ERR);
-        return;
-    }
-
-    if (cmd_len != 0x80) {
-        DPRINTF(ad->port_no,
-                "dma_memory_map mapped too few bytes in ahci_write_fis_pio");
-        dma_memory_unmap(ad->hba->as, cmd_fis, cmd_len,
-                         DMA_DIRECTION_TO_DEVICE, cmd_len);
-        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_HBUS_ERR);
-        return;
-    }
-
     pio_fis = &ad->res_fis[RES_FIS_PSFIS];
 
-    pio_fis[0] = 0x5f;
+    pio_fis[0] = SATA_FIS_TYPE_PIO_SETUP;
     pio_fis[1] = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
     pio_fis[2] = s->status;
     pio_fis[3] = s->error;
@@ -644,8 +752,8 @@
     pio_fis[9] = s->hob_lcyl;
     pio_fis[10] = s->hob_hcyl;
     pio_fis[11] = 0;
-    pio_fis[12] = cmd_fis[12];
-    pio_fis[13] = cmd_fis[13];
+    pio_fis[12] = s->nsector & 0xFF;
+    pio_fis[13] = (s->nsector >> 8) & 0xFF;
     pio_fis[14] = 0;
     pio_fis[15] = s->status;
     pio_fis[16] = len & 255;
@@ -662,35 +770,22 @@
     }
 
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_PIOS_FIS);
-
-    dma_memory_unmap(ad->hba->as, cmd_fis, cmd_len,
-                     DMA_DIRECTION_TO_DEVICE, cmd_len);
 }
 
-static void ahci_write_fis_d2h(AHCIDevice *ad, uint8_t *cmd_fis)
+static bool ahci_write_fis_d2h(AHCIDevice *ad)
 {
     AHCIPortRegs *pr = &ad->port_regs;
     uint8_t *d2h_fis;
     int i;
-    dma_addr_t cmd_len = 0x80;
-    int cmd_mapped = 0;
     IDEState *s = &ad->port.ifs[0];
 
     if (!ad->res_fis || !(pr->cmd & PORT_CMD_FIS_RX)) {
-        return;
-    }
-
-    if (!cmd_fis) {
-        /* map cmd_fis */
-        uint64_t tbl_addr = le64_to_cpu(ad->cur_cmd->tbl_addr);
-        cmd_fis = dma_memory_map(ad->hba->as, tbl_addr, &cmd_len,
-                                 DMA_DIRECTION_TO_DEVICE);
-        cmd_mapped = 1;
+        return false;
     }
 
     d2h_fis = &ad->res_fis[RES_FIS_RFIS];
 
-    d2h_fis[0] = 0x34;
+    d2h_fis[0] = SATA_FIS_TYPE_REGISTER_D2H;
     d2h_fis[1] = (ad->hba->control_regs.irqstatus ? (1 << 6) : 0);
     d2h_fis[2] = s->status;
     d2h_fis[3] = s->error;
@@ -703,8 +798,8 @@
     d2h_fis[9] = s->hob_lcyl;
     d2h_fis[10] = s->hob_hcyl;
     d2h_fis[11] = 0;
-    d2h_fis[12] = cmd_fis[12];
-    d2h_fis[13] = cmd_fis[13];
+    d2h_fis[12] = s->nsector & 0xFF;
+    d2h_fis[13] = (s->nsector >> 8) & 0xFF;
     for (i = 14; i < 20; i++) {
         d2h_fis[i] = 0;
     }
@@ -718,26 +813,36 @@
     }
 
     ahci_trigger_irq(ad->hba, ad, PORT_IRQ_D2H_REG_FIS);
-
-    if (cmd_mapped) {
-        dma_memory_unmap(ad->hba->as, cmd_fis, cmd_len,
-                         DMA_DIRECTION_TO_DEVICE, cmd_len);
-    }
+    return true;
 }
 
 static int prdt_tbl_entry_size(const AHCI_SG *tbl)
 {
+    /* flags_size is zero-based */
     return (le32_to_cpu(tbl->flags_size) & AHCI_PRDT_SIZE_MASK) + 1;
 }
 
+/**
+ * Fetch entries in a guest-provided PRDT and convert it into a QEMU SGlist.
+ * @ad: The AHCIDevice for whom we are building the SGList.
+ * @sglist: The SGList target to add PRD entries to.
+ * @cmd: The AHCI Command Header that describes where the PRDT is.
+ * @limit: The remaining size of the S/ATA transaction, in bytes.
+ * @offset: The number of bytes already transferred, in bytes.
+ *
+ * The AHCI PRDT can describe up to 256GiB. S/ATA only support transactions of
+ * up to 32MiB as of ATA8-ACS3 rev 1b, assuming a 512 byte sector size. We stop
+ * building the sglist from the PRDT as soon as we hit @limit bytes,
+ * which is <= INT32_MAX/2GiB.
+ */
 static int ahci_populate_sglist(AHCIDevice *ad, QEMUSGList *sglist,
-                                int32_t offset)
+                                AHCICmdHdr *cmd, int64_t limit, uint64_t offset)
 {
-    AHCICmdHdr *cmd = ad->cur_cmd;
-    uint32_t opts = le32_to_cpu(cmd->opts);
-    uint64_t prdt_addr = le64_to_cpu(cmd->tbl_addr) + 0x80;
-    int sglist_alloc_hint = opts >> AHCI_CMD_HDR_PRDT_LEN;
-    dma_addr_t prdt_len = (sglist_alloc_hint * sizeof(AHCI_SG));
+    uint16_t opts = le16_to_cpu(cmd->opts);
+    uint16_t prdtl = le16_to_cpu(cmd->prdtl);
+    uint64_t cfis_addr = le64_to_cpu(cmd->tbl_addr);
+    uint64_t prdt_addr = cfis_addr + 0x80;
+    dma_addr_t prdt_len = (prdtl * sizeof(AHCI_SG));
     dma_addr_t real_prdt_len = prdt_len;
     uint8_t *prdt;
     int i;
@@ -749,15 +854,7 @@
     IDEBus *bus = &ad->port;
     BusState *qbus = BUS(bus);
 
-    /*
-     * Note: AHCI PRDT can describe up to 256GiB. SATA/ATA only support
-     * transactions of up to 32MiB as of ATA8-ACS3 rev 1b, assuming a
-     * 512 byte sector size. We limit the PRDT in this implementation to
-     * a reasonably large 2GiB, which can accommodate the maximum transfer
-     * request for sector sizes up to 32K.
-     */
-
-    if (!sglist_alloc_hint) {
+    if (!prdtl) {
         DPRINTF(ad->port_no, "no sg list given by guest: 0x%08x\n", opts);
         return -1;
     }
@@ -776,13 +873,12 @@
     }
 
     /* Get entries in the PRDT, init a qemu sglist accordingly */
-    if (sglist_alloc_hint > 0) {
+    if (prdtl > 0) {
         AHCI_SG *tbl = (AHCI_SG *)prdt;
         sum = 0;
-        for (i = 0; i < sglist_alloc_hint; i++) {
-            /* flags_size is zero-based */
+        for (i = 0; i < prdtl; i++) {
             tbl_entry_size = prdt_tbl_entry_size(&tbl[i]);
-            if (offset <= (sum + tbl_entry_size)) {
+            if (offset < (sum + tbl_entry_size)) {
                 off_idx = i;
                 off_pos = offset - sum;
                 break;
@@ -797,22 +893,16 @@
             goto out;
         }
 
-        qemu_sglist_init(sglist, qbus->parent, (sglist_alloc_hint - off_idx),
+        qemu_sglist_init(sglist, qbus->parent, (prdtl - off_idx),
                          ad->hba->as);
-        qemu_sglist_add(sglist, le64_to_cpu(tbl[off_idx].addr + off_pos),
-                        prdt_tbl_entry_size(&tbl[off_idx]) - off_pos);
+        qemu_sglist_add(sglist, le64_to_cpu(tbl[off_idx].addr) + off_pos,
+                        MIN(prdt_tbl_entry_size(&tbl[off_idx]) - off_pos,
+                            limit));
 
-        for (i = off_idx + 1; i < sglist_alloc_hint; i++) {
-            /* flags_size is zero-based */
+        for (i = off_idx + 1; i < prdtl && sglist->size < limit; i++) {
             qemu_sglist_add(sglist, le64_to_cpu(tbl[i].addr),
-                            prdt_tbl_entry_size(&tbl[i]));
-            if (sglist->size > INT32_MAX) {
-                error_report("AHCI Physical Region Descriptor Table describes "
-                             "more than 2 GiB.\n");
-                qemu_sglist_destroy(sglist);
-                r = -1;
-                goto out;
-            }
+                            MIN(prdt_tbl_entry_size(&tbl[i]),
+                                limit - sglist->size));
         }
     }
 
@@ -822,6 +912,37 @@
     return r;
 }
 
+static void ncq_err(NCQTransferState *ncq_tfs)
+{
+    IDEState *ide_state = &ncq_tfs->drive->port.ifs[0];
+
+    ide_state->error = ABRT_ERR;
+    ide_state->status = READY_STAT | ERR_STAT;
+    ncq_tfs->drive->port_regs.scr_err |= (1 << ncq_tfs->tag);
+    qemu_sglist_destroy(&ncq_tfs->sglist);
+    ncq_tfs->used = 0;
+}
+
+static void ncq_finish(NCQTransferState *ncq_tfs)
+{
+    /* If we didn't error out, set our finished bit. Errored commands
+     * do not get a bit set for the SDB FIS ACT register, nor do they
+     * clear the outstanding bit in scr_act (PxSACT). */
+    if (!(ncq_tfs->drive->port_regs.scr_err & (1 << ncq_tfs->tag))) {
+        ncq_tfs->drive->finished |= (1 << ncq_tfs->tag);
+    }
+
+    ahci_write_fis_sdb(ncq_tfs->drive->hba, ncq_tfs);
+
+    DPRINTF(ncq_tfs->drive->port_no, "NCQ transfer tag %d finished\n",
+            ncq_tfs->tag);
+
+    block_acct_done(blk_get_stats(ncq_tfs->drive->port.ifs[0].blk),
+                    &ncq_tfs->acct);
+    qemu_sglist_destroy(&ncq_tfs->sglist);
+    ncq_tfs->used = 0;
+}
+
 static void ncq_cb(void *opaque, int ret)
 {
     NCQTransferState *ncq_tfs = (NCQTransferState *)opaque;
@@ -830,28 +951,25 @@
     if (ret == -ECANCELED) {
         return;
     }
-    /* Clear bit for this tag in SActive */
-    ncq_tfs->drive->port_regs.scr_act &= ~(1 << ncq_tfs->tag);
 
     if (ret < 0) {
-        /* error */
-        ide_state->error = ABRT_ERR;
-        ide_state->status = READY_STAT | ERR_STAT;
-        ncq_tfs->drive->port_regs.scr_err |= (1 << ncq_tfs->tag);
+        bool is_read = ncq_tfs->cmd == READ_FPDMA_QUEUED;
+        BlockErrorAction action = blk_get_error_action(ide_state->blk,
+                                                       is_read, -ret);
+        if (action == BLOCK_ERROR_ACTION_STOP) {
+            ncq_tfs->halt = true;
+            ide_state->bus->error_status = IDE_RETRY_HBA;
+        } else if (action == BLOCK_ERROR_ACTION_REPORT) {
+            ncq_err(ncq_tfs);
+        }
+        blk_error_action(ide_state->blk, action, is_read, -ret);
     } else {
         ide_state->status = READY_STAT | SEEK_STAT;
     }
 
-    ahci_write_fis_sdb(ncq_tfs->drive->hba, ncq_tfs->drive->port_no,
-                       (1 << ncq_tfs->tag));
-
-    DPRINTF(ncq_tfs->drive->port_no, "NCQ transfer tag %d finished\n",
-            ncq_tfs->tag);
-
-    block_acct_done(blk_get_stats(ncq_tfs->drive->port.ifs[0].blk),
-                    &ncq_tfs->acct);
-    qemu_sglist_destroy(&ncq_tfs->sglist);
-    ncq_tfs->used = 0;
+    if (!ncq_tfs->halt) {
+        ncq_finish(ncq_tfs);
+    }
 }
 
 static int is_ncq(uint8_t ata_cmd)
@@ -869,13 +987,61 @@
     }
 }
 
-static void process_ncq_command(AHCIState *s, int port, uint8_t *cmd_fis,
-                                int slot)
+static void execute_ncq_command(NCQTransferState *ncq_tfs)
 {
+    AHCIDevice *ad = ncq_tfs->drive;
+    IDEState *ide_state = &ad->port.ifs[0];
+    int port = ad->port_no;
+
+    g_assert(is_ncq(ncq_tfs->cmd));
+    ncq_tfs->halt = false;
+
+    switch (ncq_tfs->cmd) {
+    case READ_FPDMA_QUEUED:
+        DPRINTF(port, "NCQ reading %d sectors from LBA %"PRId64", tag %d\n",
+                ncq_tfs->sector_count, ncq_tfs->lba, ncq_tfs->tag);
+
+        DPRINTF(port, "tag %d aio read %"PRId64"\n",
+                ncq_tfs->tag, ncq_tfs->lba);
+
+        dma_acct_start(ide_state->blk, &ncq_tfs->acct,
+                       &ncq_tfs->sglist, BLOCK_ACCT_READ);
+        ncq_tfs->aiocb = dma_blk_read(ide_state->blk, &ncq_tfs->sglist,
+                                      ncq_tfs->lba << BDRV_SECTOR_BITS,
+                                      ncq_cb, ncq_tfs);
+        break;
+    case WRITE_FPDMA_QUEUED:
+        DPRINTF(port, "NCQ writing %d sectors to LBA %"PRId64", tag %d\n",
+                ncq_tfs->sector_count, ncq_tfs->lba, ncq_tfs->tag);
+
+        DPRINTF(port, "tag %d aio write %"PRId64"\n",
+                ncq_tfs->tag, ncq_tfs->lba);
+
+        dma_acct_start(ide_state->blk, &ncq_tfs->acct,
+                       &ncq_tfs->sglist, BLOCK_ACCT_WRITE);
+        ncq_tfs->aiocb = dma_blk_write(ide_state->blk, &ncq_tfs->sglist,
+                                       ncq_tfs->lba << BDRV_SECTOR_BITS,
+                                       ncq_cb, ncq_tfs);
+        break;
+    default:
+        DPRINTF(port, "error: unsupported NCQ command (0x%02x) received\n",
+                ncq_tfs->cmd);
+        ncq_err(ncq_tfs);
+    }
+}
+
+
+static void process_ncq_command(AHCIState *s, int port, uint8_t *cmd_fis,
+                                uint8_t slot)
+{
+    AHCIDevice *ad = &s->dev[port];
+    IDEState *ide_state = &ad->port.ifs[0];
     NCQFrame *ncq_fis = (NCQFrame*)cmd_fis;
     uint8_t tag = ncq_fis->tag >> 3;
-    NCQTransferState *ncq_tfs = &s->dev[port].ncq_tfs[tag];
+    NCQTransferState *ncq_tfs = &ad->ncq_tfs[tag];
+    size_t size;
 
+    g_assert(is_ncq(ncq_fis->command));
     if (ncq_tfs->used) {
         /* error - already in use */
         fprintf(stderr, "%s: tag %d already used\n", __FUNCTION__, tag);
@@ -883,75 +1049,81 @@
     }
 
     ncq_tfs->used = 1;
-    ncq_tfs->drive = &s->dev[port];
+    ncq_tfs->drive = ad;
     ncq_tfs->slot = slot;
+    ncq_tfs->cmdh = &((AHCICmdHdr *)ad->lst)[slot];
+    ncq_tfs->cmd = ncq_fis->command;
     ncq_tfs->lba = ((uint64_t)ncq_fis->lba5 << 40) |
                    ((uint64_t)ncq_fis->lba4 << 32) |
                    ((uint64_t)ncq_fis->lba3 << 24) |
                    ((uint64_t)ncq_fis->lba2 << 16) |
                    ((uint64_t)ncq_fis->lba1 << 8) |
                    (uint64_t)ncq_fis->lba0;
+    ncq_tfs->tag = tag;
 
-    /* Note: We calculate the sector count, but don't currently rely on it.
-     * The total size of the DMA buffer tells us the transfer size instead. */
-    ncq_tfs->sector_count = ((uint16_t)ncq_fis->sector_count_high << 8) |
-                                ncq_fis->sector_count_low;
+    /* Sanity-check the NCQ packet */
+    if (tag != slot) {
+        DPRINTF(port, "Warn: NCQ slot (%d) did not match the given tag (%d)\n",
+                slot, tag);
+    }
+
+    if (ncq_fis->aux0 || ncq_fis->aux1 || ncq_fis->aux2 || ncq_fis->aux3) {
+        DPRINTF(port, "Warn: Attempt to use NCQ auxiliary fields.\n");
+    }
+    if (ncq_fis->prio || ncq_fis->icc) {
+        DPRINTF(port, "Warn: Unsupported attempt to use PRIO/ICC fields\n");
+    }
+    if (ncq_fis->fua & NCQ_FIS_FUA_MASK) {
+        DPRINTF(port, "Warn: Unsupported attempt to use Force Unit Access\n");
+    }
+    if (ncq_fis->tag & NCQ_FIS_RARC_MASK) {
+        DPRINTF(port, "Warn: Unsupported attempt to use Rebuild Assist\n");
+    }
+
+    ncq_tfs->sector_count = ((ncq_fis->sector_count_high << 8) |
+                             ncq_fis->sector_count_low);
+    if (!ncq_tfs->sector_count) {
+        ncq_tfs->sector_count = 0x10000;
+    }
+    size = ncq_tfs->sector_count * 512;
+    ahci_populate_sglist(ad, &ncq_tfs->sglist, ncq_tfs->cmdh, size, 0);
+
+    if (ncq_tfs->sglist.size < size) {
+        error_report("ahci: PRDT length for NCQ command (0x%zx) "
+                     "is smaller than the requested size (0x%zx)",
+                     ncq_tfs->sglist.size, size);
+        ncq_err(ncq_tfs);
+        ahci_trigger_irq(ad->hba, ad, PORT_IRQ_OVERFLOW);
+        return;
+    } else if (ncq_tfs->sglist.size != size) {
+        DPRINTF(port, "Warn: PRDTL (0x%zx)"
+                " does not match requested size (0x%zx)",
+                ncq_tfs->sglist.size, size);
+    }
 
     DPRINTF(port, "NCQ transfer LBA from %"PRId64" to %"PRId64", "
             "drive max %"PRId64"\n",
-            ncq_tfs->lba, ncq_tfs->lba + ncq_tfs->sector_count - 2,
-            s->dev[port].port.ifs[0].nb_sectors - 1);
+            ncq_tfs->lba, ncq_tfs->lba + ncq_tfs->sector_count - 1,
+            ide_state->nb_sectors - 1);
 
-    ahci_populate_sglist(&s->dev[port], &ncq_tfs->sglist, 0);
-    ncq_tfs->tag = tag;
+    execute_ncq_command(ncq_tfs);
+}
 
-    switch(ncq_fis->command) {
-        case READ_FPDMA_QUEUED:
-            DPRINTF(port, "NCQ reading %d sectors from LBA %"PRId64", "
-                    "tag %d\n",
-                    ncq_tfs->sector_count-1, ncq_tfs->lba, ncq_tfs->tag);
-
-            DPRINTF(port, "tag %d aio read %"PRId64"\n",
-                    ncq_tfs->tag, ncq_tfs->lba);
-
-            dma_acct_start(ncq_tfs->drive->port.ifs[0].blk, &ncq_tfs->acct,
-                           &ncq_tfs->sglist, BLOCK_ACCT_READ);
-            ncq_tfs->aiocb = dma_blk_read(ncq_tfs->drive->port.ifs[0].blk,
-                                          &ncq_tfs->sglist, ncq_tfs->lba,
-                                          ncq_cb, ncq_tfs);
-            break;
-        case WRITE_FPDMA_QUEUED:
-            DPRINTF(port, "NCQ writing %d sectors to LBA %"PRId64", tag %d\n",
-                    ncq_tfs->sector_count-1, ncq_tfs->lba, ncq_tfs->tag);
-
-            DPRINTF(port, "tag %d aio write %"PRId64"\n",
-                    ncq_tfs->tag, ncq_tfs->lba);
-
-            dma_acct_start(ncq_tfs->drive->port.ifs[0].blk, &ncq_tfs->acct,
-                           &ncq_tfs->sglist, BLOCK_ACCT_WRITE);
-            ncq_tfs->aiocb = dma_blk_write(ncq_tfs->drive->port.ifs[0].blk,
-                                           &ncq_tfs->sglist, ncq_tfs->lba,
-                                           ncq_cb, ncq_tfs);
-            break;
-        default:
-            if (is_ncq(cmd_fis[2])) {
-                DPRINTF(port,
-                        "error: unsupported NCQ command (0x%02x) received\n",
-                        cmd_fis[2]);
-            } else {
-                DPRINTF(port,
-                        "error: tried to process non-NCQ command as NCQ\n");
-            }
-            qemu_sglist_destroy(&ncq_tfs->sglist);
+static AHCICmdHdr *get_cmd_header(AHCIState *s, uint8_t port, uint8_t slot)
+{
+    if (port >= s->ports || slot >= AHCI_MAX_CMDS) {
+        return NULL;
     }
+
+    return s->dev[port].lst ? &((AHCICmdHdr *)s->dev[port].lst)[slot] : NULL;
 }
 
 static void handle_reg_h2d_fis(AHCIState *s, int port,
-                               int slot, uint8_t *cmd_fis)
+                               uint8_t slot, uint8_t *cmd_fis)
 {
     IDEState *ide_state = &s->dev[port].port.ifs[0];
-    AHCICmdHdr *cmd = s->dev[port].cur_cmd;
-    uint32_t opts = le32_to_cpu(cmd->opts);
+    AHCICmdHdr *cmd = get_cmd_header(s, port, slot);
+    uint16_t opts = le16_to_cpu(cmd->opts);
 
     if (cmd_fis[1] & 0x0F) {
         DPRINTF(port, "Port Multiplier not supported."
@@ -1031,7 +1203,7 @@
     ide_exec_cmd(&s->dev[port].port, cmd_fis[2]);
 }
 
-static int handle_cmd(AHCIState *s, int port, int slot)
+static int handle_cmd(AHCIState *s, int port, uint8_t slot)
 {
     IDEState *ide_state;
     uint64_t tbl_addr;
@@ -1049,7 +1221,7 @@
         DPRINTF(port, "error: lst not given but cmd handled");
         return -1;
     }
-    cmd = &((AHCICmdHdr *)s->dev[port].lst)[slot];
+    cmd = get_cmd_header(s, port, slot);
     /* remember current slot handle for later */
     s->dev[port].cur_cmd = cmd;
 
@@ -1108,7 +1280,7 @@
     IDEState *s = &ad->port.ifs[0];
     uint32_t size = (uint32_t)(s->data_end - s->data_ptr);
     /* write == ram -> device */
-    uint32_t opts = le32_to_cpu(ad->cur_cmd->opts);
+    uint16_t opts = le16_to_cpu(ad->cur_cmd->opts);
     int is_write = opts & AHCI_CMD_WRITE;
     int is_atapi = opts & AHCI_CMD_ATAPI;
     int has_sglist = 0;
@@ -1120,7 +1292,7 @@
         goto out;
     }
 
-    if (ahci_dma_prepare_buf(dma, is_write)) {
+    if (ahci_dma_prepare_buf(dma, size)) {
         has_sglist = 1;
     }
 
@@ -1141,7 +1313,7 @@
     s->data_ptr = s->data_end;
 
     /* Update number of transferred bytes, destroy sglist */
-    ahci_commit_buf(dma, size);
+    dma_buf_commit(s, size);
 
     s->end_transfer_func(s);
 
@@ -1154,25 +1326,45 @@
 static void ahci_start_dma(IDEDMA *dma, IDEState *s,
                            BlockCompletionFunc *dma_cb)
 {
-#ifdef DEBUG_AHCI
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-#endif
     DPRINTF(ad->port_no, "\n");
     s->io_buffer_offset = 0;
     dma_cb(s, 0);
 }
 
+static void ahci_restart_dma(IDEDMA *dma)
+{
+    /* Nothing to do, ahci_start_dma already resets s->io_buffer_offset.  */
+}
+
 /**
- * Called in DMA R/W chains to read the PRDT, utilizing ahci_populate_sglist.
- * Not currently invoked by PIO R/W chains,
- * which invoke ahci_populate_sglist via ahci_start_transfer.
+ * IDE/PIO restarts are handled by the core layer, but NCQ commands
+ * need an extra kick from the AHCI HBA.
  */
-static int32_t ahci_dma_prepare_buf(IDEDMA *dma, int is_write)
+static void ahci_restart(IDEDMA *dma)
+{
+    AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
+    int i;
+
+    for (i = 0; i < AHCI_MAX_CMDS; i++) {
+        NCQTransferState *ncq_tfs = &ad->ncq_tfs[i];
+        if (ncq_tfs->halt) {
+            execute_ncq_command(ncq_tfs);
+        }
+    }
+}
+
+/**
+ * Called in DMA and PIO R/W chains to read the PRDT.
+ * Not shared with NCQ pathways.
+ */
+static int32_t ahci_dma_prepare_buf(IDEDMA *dma, int32_t limit)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
     IDEState *s = &ad->port.ifs[0];
 
-    if (ahci_populate_sglist(ad, &s->sg, s->io_buffer_offset) == -1) {
+    if (ahci_populate_sglist(ad, &s->sg, ad->cur_cmd,
+                             limit, s->io_buffer_offset) == -1) {
         DPRINTF(ad->port_no, "ahci_dma_prepare_buf failed.\n");
         return -1;
     }
@@ -1183,21 +1375,16 @@
 }
 
 /**
- * Destroys the scatter-gather list,
- * and updates the command header with a bytes-read value.
- * called explicitly via ahci_dma_rw_buf (ATAPI DMA),
- * and ahci_start_transfer (PIO R/W),
- * and called via callback from ide_dma_cb for DMA R/W paths.
+ * Updates the command header with a bytes-read value.
+ * Called via dma_buf_commit, for both DMA and PIO paths.
+ * sglist destruction is handled within dma_buf_commit.
  */
 static void ahci_commit_buf(IDEDMA *dma, uint32_t tx_bytes)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
-    IDEState *s = &ad->port.ifs[0];
 
     tx_bytes += le32_to_cpu(ad->cur_cmd->status);
     ad->cur_cmd->status = cpu_to_le32(tx_bytes);
-
-    qemu_sglist_destroy(&s->sg);
 }
 
 static int ahci_dma_rw_buf(IDEDMA *dma, int is_write)
@@ -1207,7 +1394,7 @@
     uint8_t *p = s->io_buffer + s->io_buffer_index;
     int l = s->io_buffer_size - s->io_buffer_index;
 
-    if (ahci_populate_sglist(ad, &s->sg, s->io_buffer_offset)) {
+    if (ahci_populate_sglist(ad, &s->sg, ad->cur_cmd, l, s->io_buffer_offset)) {
         return 0;
     }
 
@@ -1218,22 +1405,15 @@
     }
 
     /* free sglist, update byte count */
-    ahci_commit_buf(dma, l);
+    dma_buf_commit(s, l);
 
     s->io_buffer_index += l;
-    s->io_buffer_offset += l;
 
     DPRINTF(ad->port_no, "len=%#x\n", l);
 
     return 1;
 }
 
-static int ahci_dma_set_unit(IDEDMA *dma, int unit)
-{
-    /* only a single unit per link */
-    return 0;
-}
-
 static void ahci_cmd_done(IDEDMA *dma)
 {
     AHCIDevice *ad = DO_UPCAST(AHCIDevice, dma, dma);
@@ -1241,7 +1421,7 @@
     DPRINTF(ad->port_no, "cmd done\n");
 
     /* update d2h status */
-    ahci_write_fis_d2h(ad, NULL);
+    ahci_write_fis_d2h(ad);
 
     if (!ad->check_bh) {
         /* maybe we still have something to process, check later */
@@ -1254,22 +1434,28 @@
 {
 }
 
-static void ahci_dma_restart_cb(void *opaque, int running, RunState state)
-{
-}
-
 static const IDEDMAOps ahci_dma_ops = {
     .start_dma = ahci_start_dma,
+    .restart = ahci_restart,
+    .restart_dma = ahci_restart_dma,
     .start_transfer = ahci_start_transfer,
     .prepare_buf = ahci_dma_prepare_buf,
     .commit_buf = ahci_commit_buf,
     .rw_buf = ahci_dma_rw_buf,
-    .set_unit = ahci_dma_set_unit,
     .cmd_done = ahci_cmd_done,
-    .restart_cb = ahci_dma_restart_cb,
 };
 
-void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
+void ahci_init(AHCIState *s, DeviceState *qdev)
+{
+    s->container = qdev;
+    /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
+    memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
+                          "ahci", AHCI_MEM_BAR_SIZE);
+    memory_region_init_io(&s->idp, OBJECT(qdev), &ahci_idp_ops, s,
+                          "ahci-idp", 32);
+}
+
+void ahci_realize(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports)
 {
     qemu_irq *irqs;
     int i;
@@ -1278,14 +1464,7 @@
     s->ports = ports;
     s->dev = g_new0(AHCIDevice, ports);
     ahci_reg_init(s);
-    /* XXX BAR size should be 1k, but that breaks, so bump it to 4k for now */
-    memory_region_init_io(&s->mem, OBJECT(qdev), &ahci_mem_ops, s,
-                          "ahci", AHCI_MEM_BAR_SIZE);
-    memory_region_init_io(&s->idp, OBJECT(qdev), &ahci_idp_ops, s,
-                          "ahci-idp", 32);
-
     irqs = qemu_allocate_irqs(ahci_irq_set, s, s->ports);
-
     for (i = 0; i < s->ports; i++) {
         AHCIDevice *ad = &s->dev[i];
 
@@ -1296,7 +1475,9 @@
         ad->port_no = i;
         ad->port.dma = &ad->dma;
         ad->port.dma->ops = &ahci_dma_ops;
+        ide_register_restart_cb(&ad->port);
     }
+    g_free(irqs);
 }
 
 void ahci_uninit(AHCIState *s)
@@ -1330,11 +1511,27 @@
     }
 }
 
+static const VMStateDescription vmstate_ncq_tfs = {
+    .name = "ncq state",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(sector_count, NCQTransferState),
+        VMSTATE_UINT64(lba, NCQTransferState),
+        VMSTATE_UINT8(tag, NCQTransferState),
+        VMSTATE_UINT8(cmd, NCQTransferState),
+        VMSTATE_UINT8(slot, NCQTransferState),
+        VMSTATE_BOOL(used, NCQTransferState),
+        VMSTATE_BOOL(halt, NCQTransferState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static const VMStateDescription vmstate_ahci_device = {
     .name = "ahci port",
     .version_id = 1,
     .fields = (VMStateField[]) {
         VMSTATE_IDE_BUS(port, AHCIDevice),
+        VMSTATE_IDE_DRIVE(port.ifs[0], AHCIDevice),
         VMSTATE_UINT32(port_state, AHCIDevice),
         VMSTATE_UINT32(finished, AHCIDevice),
         VMSTATE_UINT32(port_regs.lst_addr, AHCIDevice),
@@ -1354,35 +1551,91 @@
         VMSTATE_BOOL(done_atapi_packet, AHCIDevice),
         VMSTATE_INT32(busy_slot, AHCIDevice),
         VMSTATE_BOOL(init_d2h_sent, AHCIDevice),
+        VMSTATE_STRUCT_ARRAY(ncq_tfs, AHCIDevice, AHCI_MAX_CMDS,
+                             1, vmstate_ncq_tfs, NCQTransferState),
         VMSTATE_END_OF_LIST()
     },
 };
 
 static int ahci_state_post_load(void *opaque, int version_id)
 {
-    int i;
+    int i, j;
     struct AHCIDevice *ad;
+    NCQTransferState *ncq_tfs;
+    AHCIPortRegs *pr;
     AHCIState *s = opaque;
 
     for (i = 0; i < s->ports; i++) {
         ad = &s->dev[i];
-        AHCIPortRegs *pr = &ad->port_regs;
+        pr = &ad->port_regs;
 
-        map_page(s->as, &ad->lst,
-                 ((uint64_t)pr->lst_addr_hi << 32) | pr->lst_addr, 1024);
-        map_page(s->as, &ad->res_fis,
-                 ((uint64_t)pr->fis_addr_hi << 32) | pr->fis_addr, 256);
-        /*
-         * All pending i/o should be flushed out on a migrate. However,
-         * we might not have cleared the busy_slot since this is done
-         * in a bh. Also, issue i/o against any slots that are pending.
-         */
-        if ((ad->busy_slot != -1) &&
-            !(ad->port.ifs[0].status & (BUSY_STAT|DRQ_STAT))) {
-            pr->cmd_issue &= ~(1 << ad->busy_slot);
-            ad->busy_slot = -1;
+        if (!(pr->cmd & PORT_CMD_START) && (pr->cmd & PORT_CMD_LIST_ON)) {
+            error_report("AHCI: DMA engine should be off, but status bit "
+                         "indicates it is still running.");
+            return -1;
         }
-        check_cmd(s, i);
+        if (!(pr->cmd & PORT_CMD_FIS_RX) && (pr->cmd & PORT_CMD_FIS_ON)) {
+            error_report("AHCI: FIS RX engine should be off, but status bit "
+                         "indicates it is still running.");
+            return -1;
+        }
+
+        /* After a migrate, the DMA/FIS engines are "off" and
+         * need to be conditionally restarted */
+        pr->cmd &= ~(PORT_CMD_LIST_ON | PORT_CMD_FIS_ON);
+        if (ahci_cond_start_engines(ad) != 0) {
+            return -1;
+        }
+
+        for (j = 0; j < AHCI_MAX_CMDS; j++) {
+            ncq_tfs = &ad->ncq_tfs[j];
+            ncq_tfs->drive = ad;
+
+            if (ncq_tfs->used != ncq_tfs->halt) {
+                return -1;
+            }
+            if (!ncq_tfs->halt) {
+                continue;
+            }
+            if (!is_ncq(ncq_tfs->cmd)) {
+                return -1;
+            }
+            if (ncq_tfs->slot != ncq_tfs->tag) {
+                return -1;
+            }
+            /* If ncq_tfs->halt is justly set, the engine should be engaged,
+             * and the command list buffer should be mapped. */
+            ncq_tfs->cmdh = get_cmd_header(s, i, ncq_tfs->slot);
+            if (!ncq_tfs->cmdh) {
+                return -1;
+            }
+            ahci_populate_sglist(ncq_tfs->drive, &ncq_tfs->sglist,
+                                 ncq_tfs->cmdh, ncq_tfs->sector_count * 512,
+                                 0);
+            if (ncq_tfs->sector_count != ncq_tfs->sglist.size >> 9) {
+                return -1;
+            }
+        }
+
+
+        /*
+         * If an error is present, ad->busy_slot will be valid and not -1.
+         * In this case, an operation is waiting to resume and will re-check
+         * for additional AHCI commands to execute upon completion.
+         *
+         * In the case where no error was present, busy_slot will be -1,
+         * and we should check to see if there are additional commands waiting.
+         */
+        if (ad->busy_slot == -1) {
+            check_cmd(s, i);
+        } else {
+            /* We are in the middle of a command, and may need to access
+             * the command header in guest memory again. */
+            if (ad->busy_slot < 0 || ad->busy_slot >= AHCI_MAX_CMDS) {
+                return -1;
+            }
+            ad->cur_cmd = get_cmd_header(s, i, ad->busy_slot);
+        }
     }
 
     return 0;
@@ -1406,21 +1659,8 @@
     },
 };
 
-#define TYPE_SYSBUS_AHCI "sysbus-ahci"
-#define SYSBUS_AHCI(obj) OBJECT_CHECK(SysbusAHCIState, (obj), TYPE_SYSBUS_AHCI)
-
-typedef struct SysbusAHCIState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    /*< public >*/
-
-    AHCIState ahci;
-    uint32_t num_ports;
-} SysbusAHCIState;
-
 static const VMStateDescription vmstate_sysbus_ahci = {
     .name = "sysbus-ahci",
-    .unmigratable = 1, /* Still buggy under I/O load */
     .fields = (VMStateField[]) {
         VMSTATE_AHCI(ahci, SysbusAHCIState),
         VMSTATE_END_OF_LIST()
@@ -1434,17 +1674,24 @@
     ahci_reset(&s->ahci);
 }
 
-static void sysbus_ahci_realize(DeviceState *dev, Error **errp)
+static void sysbus_ahci_init(Object *obj)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
-    SysbusAHCIState *s = SYSBUS_AHCI(dev);
+    SysbusAHCIState *s = SYSBUS_AHCI(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    ahci_init(&s->ahci, dev, &address_space_memory, s->num_ports);
+    ahci_init(&s->ahci, DEVICE(obj));
 
     sysbus_init_mmio(sbd, &s->ahci.mem);
     sysbus_init_irq(sbd, &s->ahci.irq);
 }
 
+static void sysbus_ahci_realize(DeviceState *dev, Error **errp)
+{
+    SysbusAHCIState *s = SYSBUS_AHCI(dev);
+
+    ahci_realize(&s->ahci, dev, &address_space_memory, s->num_ports);
+}
+
 static Property sysbus_ahci_properties[] = {
     DEFINE_PROP_UINT32("num-ports", SysbusAHCIState, num_ports, 1),
     DEFINE_PROP_END_OF_LIST(),
@@ -1465,12 +1712,108 @@
     .name          = TYPE_SYSBUS_AHCI,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SysbusAHCIState),
+    .instance_init = sysbus_ahci_init,
     .class_init    = sysbus_ahci_class_init,
 };
 
+#define ALLWINNER_AHCI_BISTAFR    ((0xa0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTCR     ((0xa4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTFCTR   ((0xa8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTSR     ((0xac - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_BISTDECR   ((0xb0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_DIAGNR0    ((0xb4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_DIAGNR1    ((0xb8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_OOBR       ((0xbc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS0R    ((0xc0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS1R    ((0xc4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PHYCS2R    ((0xc8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_TIMER1MS   ((0xe0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_GPARAM1R   ((0xe8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_GPARAM2R   ((0xec - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_PPARAMR    ((0xf0 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_TESTR      ((0xf4 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_VERSIONR   ((0xf8 - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_IDR        ((0xfc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+#define ALLWINNER_AHCI_RWCR       ((0xfc - ALLWINNER_AHCI_MMIO_OFF) / 4)
+
+static uint64_t allwinner_ahci_mem_read(void *opaque, hwaddr addr,
+                                        unsigned size)
+{
+    AllwinnerAHCIState *a = opaque;
+    uint64_t val = a->regs[addr/4];
+
+    switch (addr / 4) {
+    case ALLWINNER_AHCI_PHYCS0R:
+        val |= 0x2 << 28;
+        break;
+    case ALLWINNER_AHCI_PHYCS2R:
+        val &= ~(0x1 << 24);
+        break;
+    }
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    return  val;
+}
+
+static void allwinner_ahci_mem_write(void *opaque, hwaddr addr,
+                                     uint64_t val, unsigned size)
+{
+    AllwinnerAHCIState *a = opaque;
+
+    DPRINTF(-1, "addr=0x%" HWADDR_PRIx " val=0x%" PRIx64 ", size=%d\n",
+            addr, val, size);
+    a->regs[addr/4] = val;
+}
+
+static const MemoryRegionOps allwinner_ahci_mem_ops = {
+    .read = allwinner_ahci_mem_read,
+    .write = allwinner_ahci_mem_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void allwinner_ahci_init(Object *obj)
+{
+    SysbusAHCIState *s = SYSBUS_AHCI(obj);
+    AllwinnerAHCIState *a = ALLWINNER_AHCI(obj);
+
+    memory_region_init_io(&a->mmio, OBJECT(obj), &allwinner_ahci_mem_ops, a,
+                          "allwinner-ahci", ALLWINNER_AHCI_MMIO_SIZE);
+    memory_region_add_subregion(&s->ahci.mem, ALLWINNER_AHCI_MMIO_OFF,
+                                &a->mmio);
+}
+
+static const VMStateDescription vmstate_allwinner_ahci = {
+    .name = "allwinner-ahci",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, AllwinnerAHCIState,
+                             ALLWINNER_AHCI_MMIO_SIZE/4),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void allwinner_ahci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_allwinner_ahci;
+}
+
+static const TypeInfo allwinner_ahci_info = {
+    .name          = TYPE_ALLWINNER_AHCI,
+    .parent        = TYPE_SYSBUS_AHCI,
+    .instance_size = sizeof(AllwinnerAHCIState),
+    .instance_init = allwinner_ahci_init,
+    .class_init    = allwinner_ahci_class_init,
+};
+
 static void sysbus_ahci_register_types(void)
 {
     type_register_static(&sysbus_ahci_info);
+    type_register_static(&allwinner_ahci_info);
 }
 
 type_init(sysbus_ahci_register_types)
diff --git a/hw/ide/ahci.h b/hw/ide/ahci.h
deleted file mode 100644
index e0d2eb8..0000000
--- a/hw/ide/ahci.h
+++ /dev/null
@@ -1,348 +0,0 @@
-/*
- * QEMU AHCI Emulation
- *
- * Copyright (c) 2010 qiaochong@loongson.cn
- * Copyright (c) 2010 Roland Elek <elek.roland@gmail.com>
- * Copyright (c) 2010 Sebastian Herbszt <herbszt@gmx.de>
- * Copyright (c) 2010 Alexander Graf <agraf@suse.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- *
- */
-
-#ifndef HW_IDE_AHCI_H
-#define HW_IDE_AHCI_H
-
-#define AHCI_MEM_BAR_SIZE         0x1000
-#define AHCI_MAX_PORTS            32
-#define AHCI_MAX_SG               168 /* hardware max is 64K */
-#define AHCI_DMA_BOUNDARY         0xffffffff
-#define AHCI_USE_CLUSTERING       0
-#define AHCI_MAX_CMDS             32
-#define AHCI_CMD_SZ               32
-#define AHCI_CMD_SLOT_SZ          (AHCI_MAX_CMDS * AHCI_CMD_SZ)
-#define AHCI_RX_FIS_SZ            256
-#define AHCI_CMD_TBL_CDB          0x40
-#define AHCI_CMD_TBL_HDR_SZ       0x80
-#define AHCI_CMD_TBL_SZ           (AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16))
-#define AHCI_CMD_TBL_AR_SZ        (AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS)
-#define AHCI_PORT_PRIV_DMA_SZ     (AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ + \
-                                   AHCI_RX_FIS_SZ)
-
-#define AHCI_IRQ_ON_SG            (1U << 31)
-#define AHCI_CMD_ATAPI            (1 << 5)
-#define AHCI_CMD_WRITE            (1 << 6)
-#define AHCI_CMD_PREFETCH         (1 << 7)
-#define AHCI_CMD_RESET            (1 << 8)
-#define AHCI_CMD_CLR_BUSY         (1 << 10)
-
-#define RX_FIS_D2H_REG            0x40 /* offset of D2H Register FIS data */
-#define RX_FIS_SDB                0x58 /* offset of SDB FIS data */
-#define RX_FIS_UNK                0x60 /* offset of Unknown FIS data */
-
-/* global controller registers */
-#define HOST_CAP                  0x00 /* host capabilities */
-#define HOST_CTL                  0x04 /* global host control */
-#define HOST_IRQ_STAT             0x08 /* interrupt status */
-#define HOST_PORTS_IMPL           0x0c /* bitmap of implemented ports */
-#define HOST_VERSION              0x10 /* AHCI spec. version compliancy */
-
-/* HOST_CTL bits */
-#define HOST_CTL_RESET            (1 << 0)  /* reset controller; self-clear */
-#define HOST_CTL_IRQ_EN           (1 << 1)  /* global IRQ enable */
-#define HOST_CTL_AHCI_EN          (1U << 31) /* AHCI enabled */
-
-/* HOST_CAP bits */
-#define HOST_CAP_SSC              (1 << 14) /* Slumber capable */
-#define HOST_CAP_AHCI             (1 << 18) /* AHCI only */
-#define HOST_CAP_CLO              (1 << 24) /* Command List Override support */
-#define HOST_CAP_SSS              (1 << 27) /* Staggered Spin-up */
-#define HOST_CAP_NCQ              (1 << 30) /* Native Command Queueing */
-#define HOST_CAP_64               (1U << 31) /* PCI DAC (64-bit DMA) support */
-
-/* registers for each SATA port */
-#define PORT_LST_ADDR             0x00 /* command list DMA addr */
-#define PORT_LST_ADDR_HI          0x04 /* command list DMA addr hi */
-#define PORT_FIS_ADDR             0x08 /* FIS rx buf addr */
-#define PORT_FIS_ADDR_HI          0x0c /* FIS rx buf addr hi */
-#define PORT_IRQ_STAT             0x10 /* interrupt status */
-#define PORT_IRQ_MASK             0x14 /* interrupt enable/disable mask */
-#define PORT_CMD                  0x18 /* port command */
-#define PORT_TFDATA               0x20 /* taskfile data */
-#define PORT_SIG                  0x24 /* device TF signature */
-#define PORT_SCR_STAT             0x28 /* SATA phy register: SStatus */
-#define PORT_SCR_CTL              0x2c /* SATA phy register: SControl */
-#define PORT_SCR_ERR              0x30 /* SATA phy register: SError */
-#define PORT_SCR_ACT              0x34 /* SATA phy register: SActive */
-#define PORT_CMD_ISSUE            0x38 /* command issue */
-#define PORT_RESERVED             0x3c /* reserved */
-
-/* PORT_IRQ_{STAT,MASK} bits */
-#define PORT_IRQ_COLD_PRES        (1U << 31) /* cold presence detect */
-#define PORT_IRQ_TF_ERR           (1 << 30) /* task file error */
-#define PORT_IRQ_HBUS_ERR         (1 << 29) /* host bus fatal error */
-#define PORT_IRQ_HBUS_DATA_ERR    (1 << 28) /* host bus data error */
-#define PORT_IRQ_IF_ERR           (1 << 27) /* interface fatal error */
-#define PORT_IRQ_IF_NONFATAL      (1 << 26) /* interface non-fatal error */
-#define PORT_IRQ_OVERFLOW         (1 << 24) /* xfer exhausted available S/G */
-#define PORT_IRQ_BAD_PMP          (1 << 23) /* incorrect port multiplier */
-
-#define PORT_IRQ_PHYRDY           (1 << 22) /* PhyRdy changed */
-#define PORT_IRQ_DEV_ILCK         (1 << 7) /* device interlock */
-#define PORT_IRQ_CONNECT          (1 << 6) /* port connect change status */
-#define PORT_IRQ_SG_DONE          (1 << 5) /* descriptor processed */
-#define PORT_IRQ_UNK_FIS          (1 << 4) /* unknown FIS rx'd */
-#define PORT_IRQ_SDB_FIS          (1 << 3) /* Set Device Bits FIS rx'd */
-#define PORT_IRQ_DMAS_FIS         (1 << 2) /* DMA Setup FIS rx'd */
-#define PORT_IRQ_PIOS_FIS         (1 << 1) /* PIO Setup FIS rx'd */
-#define PORT_IRQ_D2H_REG_FIS      (1 << 0) /* D2H Register FIS rx'd */
-
-#define PORT_IRQ_FREEZE           (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR |   \
-                                   PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY |    \
-                                   PORT_IRQ_UNK_FIS)
-#define PORT_IRQ_ERROR            (PORT_IRQ_FREEZE | PORT_IRQ_TF_ERR |     \
-                                   PORT_IRQ_HBUS_DATA_ERR)
-#define DEF_PORT_IRQ              (PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |     \
-                                   PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |  \
-                                   PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
-
-/* PORT_CMD bits */
-#define PORT_CMD_ATAPI            (1 << 24) /* Device is ATAPI */
-#define PORT_CMD_LIST_ON          (1 << 15) /* cmd list DMA engine running */
-#define PORT_CMD_FIS_ON           (1 << 14) /* FIS DMA engine running */
-#define PORT_CMD_FIS_RX           (1 << 4) /* Enable FIS receive DMA engine */
-#define PORT_CMD_CLO              (1 << 3) /* Command list override */
-#define PORT_CMD_POWER_ON         (1 << 2) /* Power up device */
-#define PORT_CMD_SPIN_UP          (1 << 1) /* Spin up device */
-#define PORT_CMD_START            (1 << 0) /* Enable port DMA engine */
-
-#define PORT_CMD_ICC_MASK         (0xf << 28) /* i/f ICC state mask */
-#define PORT_CMD_ICC_ACTIVE       (0x1 << 28) /* Put i/f in active state */
-#define PORT_CMD_ICC_PARTIAL      (0x2 << 28) /* Put i/f in partial state */
-#define PORT_CMD_ICC_SLUMBER      (0x6 << 28) /* Put i/f in slumber state */
-
-/* ap->flags bits */
-#define AHCI_FLAG_NO_NCQ                  (1 << 24)
-#define AHCI_FLAG_IGN_IRQ_IF_ERR          (1 << 25) /* ignore IRQ_IF_ERR */
-#define AHCI_FLAG_HONOR_PI                (1 << 26) /* honor PORTS_IMPL */
-#define AHCI_FLAG_IGN_SERR_INTERNAL       (1 << 27) /* ignore SERR_INTERNAL */
-#define AHCI_FLAG_32BIT_ONLY              (1 << 28) /* force 32bit */
-
-#define ATA_SRST                          (1 << 2)  /* software reset */
-
-#define STATE_RUN                         0
-#define STATE_RESET                       1
-
-#define SATA_SCR_SSTATUS_DET_NODEV        0x0
-#define SATA_SCR_SSTATUS_DET_DEV_PRESENT_PHY_UP 0x3
-
-#define SATA_SCR_SSTATUS_SPD_NODEV        0x00
-#define SATA_SCR_SSTATUS_SPD_GEN1         0x10
-
-#define SATA_SCR_SSTATUS_IPM_NODEV        0x000
-#define SATA_SCR_SSTATUS_IPM_ACTIVE       0X100
-
-#define AHCI_SCR_SCTL_DET                 0xf
-
-#define SATA_FIS_TYPE_REGISTER_H2D        0x27
-#define SATA_FIS_REG_H2D_UPDATE_COMMAND_REGISTER 0x80
-
-#define AHCI_CMD_HDR_CMD_FIS_LEN           0x1f
-#define AHCI_CMD_HDR_PRDT_LEN              16
-
-#define SATA_SIGNATURE_CDROM               0xeb140000
-#define SATA_SIGNATURE_DISK                0x00000101
-
-#define AHCI_GENERIC_HOST_CONTROL_REGS_MAX_ADDR 0x20
-                                            /* Shouldn't this be 0x2c? */
-
-#define AHCI_PORT_REGS_START_ADDR          0x100
-#define AHCI_PORT_ADDR_OFFSET_MASK         0x7f
-#define AHCI_PORT_ADDR_OFFSET_LEN          0x80
-
-#define AHCI_NUM_COMMAND_SLOTS             31
-#define AHCI_SUPPORTED_SPEED               20
-#define AHCI_SUPPORTED_SPEED_GEN1          1
-#define AHCI_VERSION_1_0                   0x10000
-
-#define AHCI_PROGMODE_MAJOR_REV_1          1
-
-#define AHCI_COMMAND_TABLE_ACMD            0x40
-
-#define AHCI_PRDT_SIZE_MASK                0x3fffff
-
-#define IDE_FEATURE_DMA                    1
-
-#define READ_FPDMA_QUEUED                  0x60
-#define WRITE_FPDMA_QUEUED                 0x61
-#define NCQ_NON_DATA                       0x63
-#define RECEIVE_FPDMA_QUEUED               0x65
-#define SEND_FPDMA_QUEUED                  0x64
-
-#define RES_FIS_DSFIS                      0x00
-#define RES_FIS_PSFIS                      0x20
-#define RES_FIS_RFIS                       0x40
-#define RES_FIS_SDBFIS                     0x58
-#define RES_FIS_UFIS                       0x60
-
-#define SATA_CAP_SIZE           0x8
-#define SATA_CAP_REV            0x2
-#define SATA_CAP_BAR            0x4
-
-typedef struct AHCIControlRegs {
-    uint32_t    cap;
-    uint32_t    ghc;
-    uint32_t    irqstatus;
-    uint32_t    impl;
-    uint32_t    version;
-} AHCIControlRegs;
-
-typedef struct AHCIPortRegs {
-    uint32_t    lst_addr;
-    uint32_t    lst_addr_hi;
-    uint32_t    fis_addr;
-    uint32_t    fis_addr_hi;
-    uint32_t    irq_stat;
-    uint32_t    irq_mask;
-    uint32_t    cmd;
-    uint32_t    unused0;
-    uint32_t    tfdata;
-    uint32_t    sig;
-    uint32_t    scr_stat;
-    uint32_t    scr_ctl;
-    uint32_t    scr_err;
-    uint32_t    scr_act;
-    uint32_t    cmd_issue;
-    uint32_t    reserved;
-} AHCIPortRegs;
-
-typedef struct AHCICmdHdr {
-    uint32_t    opts;
-    uint32_t    status;
-    uint64_t    tbl_addr;
-    uint32_t    reserved[4];
-} QEMU_PACKED AHCICmdHdr;
-
-typedef struct AHCI_SG {
-    uint64_t    addr;
-    uint32_t    reserved;
-    uint32_t    flags_size;
-} QEMU_PACKED AHCI_SG;
-
-typedef struct AHCIDevice AHCIDevice;
-
-typedef struct NCQTransferState {
-    AHCIDevice *drive;
-    BlockAIOCB *aiocb;
-    QEMUSGList sglist;
-    BlockAcctCookie acct;
-    uint16_t sector_count;
-    uint64_t lba;
-    uint8_t tag;
-    int slot;
-    int used;
-} NCQTransferState;
-
-struct AHCIDevice {
-    IDEDMA dma;
-    IDEBus port;
-    int port_no;
-    uint32_t port_state;
-    uint32_t finished;
-    AHCIPortRegs port_regs;
-    struct AHCIState *hba;
-    QEMUBH *check_bh;
-    uint8_t *lst;
-    uint8_t *res_fis;
-    bool done_atapi_packet;
-    int32_t busy_slot;
-    bool init_d2h_sent;
-    AHCICmdHdr *cur_cmd;
-    NCQTransferState ncq_tfs[AHCI_MAX_CMDS];
-};
-
-typedef struct AHCIState {
-    AHCIDevice *dev;
-    AHCIControlRegs control_regs;
-    MemoryRegion mem;
-    MemoryRegion idp;       /* Index-Data Pair I/O port space */
-    unsigned idp_offset;    /* Offset of index in I/O port space */
-    uint32_t idp_index;     /* Current IDP index */
-    int32_t ports;
-    qemu_irq irq;
-    AddressSpace *as;
-} AHCIState;
-
-typedef struct AHCIPCIState {
-    /*< private >*/
-    PCIDevice parent_obj;
-    /*< public >*/
-
-    AHCIState ahci;
-} AHCIPCIState;
-
-#define TYPE_ICH9_AHCI "ich9-ahci"
-
-#define ICH_AHCI(obj) \
-    OBJECT_CHECK(AHCIPCIState, (obj), TYPE_ICH9_AHCI)
-
-extern const VMStateDescription vmstate_ahci;
-
-#define VMSTATE_AHCI(_field, _state) {                               \
-    .name       = (stringify(_field)),                               \
-    .size       = sizeof(AHCIState),                                 \
-    .vmsd       = &vmstate_ahci,                                     \
-    .flags      = VMS_STRUCT,                                        \
-    .offset     = vmstate_offset_value(_state, _field, AHCIState),   \
-}
-
-typedef struct NCQFrame {
-    uint8_t fis_type;
-    uint8_t c;
-    uint8_t command;
-    uint8_t sector_count_low;
-    uint8_t lba0;
-    uint8_t lba1;
-    uint8_t lba2;
-    uint8_t fua;
-    uint8_t lba3;
-    uint8_t lba4;
-    uint8_t lba5;
-    uint8_t sector_count_high;
-    uint8_t tag;
-    uint8_t reserved5;
-    uint8_t reserved6;
-    uint8_t control;
-    uint8_t reserved7;
-    uint8_t reserved8;
-    uint8_t reserved9;
-    uint8_t reserved10;
-} QEMU_PACKED NCQFrame;
-
-typedef struct SDBFIS {
-    uint8_t type;
-    uint8_t flags;
-    uint8_t status;
-    uint8_t error;
-    uint32_t payload;
-} QEMU_PACKED SDBFIS;
-
-void ahci_init(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
-void ahci_uninit(AHCIState *s);
-
-void ahci_reset(AHCIState *s);
-
-void ahci_ide_create_devs(PCIDevice *dev, DriveInfo **hd);
-
-#endif /* HW_IDE_AHCI_H */
diff --git a/hw/ide/atapi.c b/hw/ide/atapi.c
index c63b7e5..6189675 100644
--- a/hw/ide/atapi.c
+++ b/hw/ide/atapi.c
@@ -23,10 +23,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/ide/internal.h"
 #include "hw/scsi/scsi.h"
 #include "sysemu/block-backend.h"
 
+#define ATAPI_SECTOR_BITS (2 + BDRV_SECTOR_BITS)
+#define ATAPI_SECTOR_SIZE (1 << ATAPI_SECTOR_BITS)
+
 static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret);
 
 static void padstr8(uint8_t *buf, int buf_size, const char *src)
@@ -105,33 +109,99 @@
     memset(buf, 0, 288);
 }
 
-static int cd_read_sector(IDEState *s, int lba, uint8_t *buf, int sector_size)
+static int
+cd_read_sector_sync(IDEState *s)
 {
     int ret;
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     ATAPI_SECTOR_SIZE, BLOCK_ACCT_READ);
 
-    switch(sector_size) {
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector_sync: lba=%d\n", s->lba);
+#endif
+
+    switch (s->cd_sector_size) {
     case 2048:
-        block_acct_start(blk_get_stats(s->blk), &s->acct,
-                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        ret = blk_read(s->blk, (int64_t)lba << 2, buf, 4);
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        ret = blk_pread(s->blk, (int64_t)s->lba << ATAPI_SECTOR_BITS,
+                        s->io_buffer, ATAPI_SECTOR_SIZE);
         break;
     case 2352:
-        block_acct_start(blk_get_stats(s->blk), &s->acct,
-                         4 * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        ret = blk_read(s->blk, (int64_t)lba << 2, buf + 16, 4);
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
-        if (ret < 0)
-            return ret;
-        cd_data_to_raw(buf, lba);
+        ret = blk_pread(s->blk, (int64_t)s->lba << ATAPI_SECTOR_BITS,
+                        s->io_buffer + 16, ATAPI_SECTOR_SIZE);
+        if (ret >= 0) {
+            cd_data_to_raw(s->io_buffer, s->lba);
+        }
         break;
     default:
-        ret = -EIO;
-        break;
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
+        return -EIO;
     }
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        s->lba++;
+        s->io_buffer_index = 0;
+    }
+
     return ret;
 }
 
+static void cd_read_sector_cb(void *opaque, int ret)
+{
+    IDEState *s = opaque;
+
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector_cb: lba=%d ret=%d\n", s->lba, ret);
+#endif
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        ide_atapi_io_error(s, ret);
+        return;
+    }
+
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
+    if (s->cd_sector_size == 2352) {
+        cd_data_to_raw(s->io_buffer, s->lba);
+    }
+
+    s->lba++;
+    s->io_buffer_index = 0;
+    s->status &= ~BUSY_STAT;
+
+    ide_atapi_cmd_reply_end(s);
+}
+
+static int cd_read_sector(IDEState *s)
+{
+    if (s->cd_sector_size != 2048 && s->cd_sector_size != 2352) {
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
+        return -EINVAL;
+    }
+
+    s->iov.iov_base = (s->cd_sector_size == 2352) ?
+                      s->io_buffer + 16 : s->io_buffer;
+
+    s->iov.iov_len = ATAPI_SECTOR_SIZE;
+    qemu_iovec_init_external(&s->qiov, &s->iov, 1);
+
+#ifdef DEBUG_IDE_ATAPI
+    printf("cd_read_sector: lba=%d\n", s->lba);
+#endif
+
+    block_acct_start(blk_get_stats(s->blk), &s->acct,
+                     ATAPI_SECTOR_SIZE, BLOCK_ACCT_READ);
+
+    ide_buffered_readv(s, (int64_t)s->lba << 2, &s->qiov, 4,
+                       cd_read_sector_cb, s);
+
+    s->status |= BUSY_STAT;
+    return 0;
+}
+
 void ide_atapi_cmd_ok(IDEState *s)
 {
     s->error = 0;
@@ -167,6 +237,17 @@
     }
 }
 
+static uint16_t atapi_byte_count_limit(IDEState *s)
+{
+    uint16_t bcl;
+
+    bcl = s->lcyl | (s->hcyl << 8);
+    if (bcl == 0xffff) {
+        return 0xfffe;
+    }
+    return bcl;
+}
+
 /* The whole ATAPI transfer logic is handled in this function */
 void ide_atapi_cmd_reply_end(IDEState *s)
 {
@@ -182,18 +263,27 @@
         ide_atapi_cmd_ok(s);
         ide_set_irq(s->bus);
 #ifdef DEBUG_IDE_ATAPI
-        printf("status=0x%x\n", s->status);
+        printf("end of transfer, status=0x%x\n", s->status);
 #endif
     } else {
         /* see if a new sector must be read */
         if (s->lba != -1 && s->io_buffer_index >= s->cd_sector_size) {
-            ret = cd_read_sector(s, s->lba, s->io_buffer, s->cd_sector_size);
-            if (ret < 0) {
-                ide_atapi_io_error(s, ret);
+            if (!s->elementary_transfer_size) {
+                ret = cd_read_sector(s);
+                if (ret < 0) {
+                    ide_atapi_io_error(s, ret);
+                }
                 return;
+            } else {
+                /* rebuffering within an elementary transfer is
+                 * only possible with a sync request because we
+                 * end up with a race condition otherwise */
+                ret = cd_read_sector_sync(s);
+                if (ret < 0) {
+                    ide_atapi_io_error(s, ret);
+                    return;
+                }
             }
-            s->lba++;
-            s->io_buffer_index = 0;
         }
         if (s->elementary_transfer_size > 0) {
             /* there are some data left to transmit in this elementary
@@ -209,12 +299,10 @@
         } else {
             /* a new transfer is needed */
             s->nsector = (s->nsector & ~7) | ATAPI_INT_REASON_IO;
-            byte_count_limit = s->lcyl | (s->hcyl << 8);
+            byte_count_limit = atapi_byte_count_limit(s);
 #ifdef DEBUG_IDE_ATAPI
             printf("byte_count_limit=%d\n", byte_count_limit);
 #endif
-            if (byte_count_limit == 0xffff)
-                byte_count_limit--;
             size = s->packet_transfer_size;
             if (size > byte_count_limit) {
                 /* byte count limit must be even if this case */
@@ -252,7 +340,6 @@
     s->packet_transfer_size = size;
     s->io_buffer_size = size;    /* dma: send the reply data as one chunk */
     s->elementary_transfer_size = 0;
-    s->io_buffer_index = 0;
 
     if (s->atapi_dma) {
         block_acct_start(blk_get_stats(s->blk), &s->acct, size,
@@ -261,6 +348,7 @@
         ide_start_dma(s, ide_atapi_cmd_read_dma_cb);
     } else {
         s->status = READY_STAT | SEEK_STAT;
+        s->io_buffer_index = 0;
         ide_atapi_cmd_reply_end(s);
     }
 }
@@ -275,7 +363,6 @@
     s->io_buffer_index = sector_size;
     s->cd_sector_size = sector_size;
 
-    s->status = READY_STAT | SEEK_STAT;
     ide_atapi_cmd_reply_end(s);
 }
 
@@ -291,15 +378,19 @@
 }
 /* ATAPI DMA support */
 
-/* XXX: handle read errors */
 static void ide_atapi_cmd_read_dma_cb(void *opaque, int ret)
 {
     IDEState *s = opaque;
     int data_offset, n;
 
     if (ret < 0) {
-        ide_atapi_io_error(s, ret);
-        goto eot;
+        if (ide_handle_rw_error(s, -ret, ide_dma_cmd_to_retry(s->dma_cmd))) {
+            if (s->bus->error_status) {
+                s->bus->dma->aiocb = NULL;
+                return;
+            }
+            goto eot;
+        }
     }
 
     if (s->io_buffer_size > 0) {
@@ -348,16 +439,20 @@
 #endif
 
     s->bus->dma->iov.iov_base = (void *)(s->io_buffer + data_offset);
-    s->bus->dma->iov.iov_len = n * 4 * 512;
+    s->bus->dma->iov.iov_len = n * ATAPI_SECTOR_SIZE;
     qemu_iovec_init_external(&s->bus->dma->qiov, &s->bus->dma->iov, 1);
 
-    s->bus->dma->aiocb = blk_aio_readv(s->blk, (int64_t)s->lba << 2,
-                                       &s->bus->dma->qiov, n * 4,
-                                       ide_atapi_cmd_read_dma_cb, s);
+    s->bus->dma->aiocb = ide_buffered_readv(s, (int64_t)s->lba << 2,
+                                            &s->bus->dma->qiov, n * 4,
+                                            ide_atapi_cmd_read_dma_cb, s);
     return;
 
 eot:
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
     ide_set_inactive(s, false);
 }
 
@@ -368,7 +463,6 @@
 {
     s->lba = lba;
     s->packet_transfer_size = nb_sectors * sector_size;
-    s->io_buffer_index = 0;
     s->io_buffer_size = 0;
     s->cd_sector_size = sector_size;
 
@@ -394,6 +488,18 @@
     }
 }
 
+void ide_atapi_dma_restart(IDEState *s)
+{
+    /*
+     * At this point we can just re-evaluate the packet command and start over.
+     * The presence of ->dma_cb callback in the pre_save ensures that the packet
+     * command has been completely sent and we can safely restart command.
+     */
+    s->unit = s->bus->retry_unit;
+    s->bus->dma->ops->restart_dma(s->bus->dma);
+    ide_atapi_cmd(s);
+}
+
 static inline uint8_t ide_atapi_set_profile(uint8_t *buf, uint8_t *index,
                                             uint16_t profile)
 {
@@ -621,20 +727,106 @@
 
 static void cmd_inquiry(IDEState *s, uint8_t *buf)
 {
+    uint8_t page_code = buf[2];
     int max_len = buf[4];
 
-    buf[0] = 0x05; /* CD-ROM */
-    buf[1] = 0x80; /* removable */
-    buf[2] = 0x00; /* ISO */
-    buf[3] = 0x21; /* ATAPI-2 (XXX: put ATAPI-4 ?) */
-    buf[4] = 31; /* additional length */
-    buf[5] = 0; /* reserved */
-    buf[6] = 0; /* reserved */
-    buf[7] = 0; /* reserved */
-    padstr8(buf + 8, 8, "QEMU");
-    padstr8(buf + 16, 16, "QEMU DVD-ROM");
-    padstr8(buf + 32, 4, s->version);
-    ide_atapi_cmd_reply(s, 36, max_len);
+    unsigned idx = 0;
+    unsigned size_idx;
+    unsigned preamble_len;
+
+    /* If the EVPD (Enable Vital Product Data) bit is set in byte 1,
+     * we are being asked for a specific page of info indicated by byte 2. */
+    if (buf[1] & 0x01) {
+        preamble_len = 4;
+        size_idx = 3;
+
+        buf[idx++] = 0x05;      /* CD-ROM */
+        buf[idx++] = page_code; /* Page Code */
+        buf[idx++] = 0x00;      /* reserved */
+        idx++;                  /* length (set later) */
+
+        switch (page_code) {
+        case 0x00:
+            /* Supported Pages: List of supported VPD responses. */
+            buf[idx++] = 0x00; /* 0x00: Supported Pages, and: */
+            buf[idx++] = 0x83; /* 0x83: Device Identification. */
+            break;
+
+        case 0x83:
+            /* Device Identification. Each entry is optional, but the entries
+             * included here are modeled after libata's VPD responses.
+             * If the response is given, at least one entry must be present. */
+
+            /* Entry 1: Serial */
+            if (idx + 24 > max_len) {
+                /* Not enough room for even the first entry: */
+                /* 4 byte header + 20 byte string */
+                ide_atapi_cmd_error(s, ILLEGAL_REQUEST,
+                                    ASC_DATA_PHASE_ERROR);
+                return;
+            }
+            buf[idx++] = 0x02; /* Ascii */
+            buf[idx++] = 0x00; /* Vendor Specific */
+            buf[idx++] = 0x00;
+            buf[idx++] = 20;   /* Remaining length */
+            padstr8(buf + idx, 20, s->drive_serial_str);
+            idx += 20;
+
+            /* Entry 2: Drive Model and Serial */
+            if (idx + 72 > max_len) {
+                /* 4 (header) + 8 (vendor) + 60 (model & serial) */
+                goto out;
+            }
+            buf[idx++] = 0x02; /* Ascii */
+            buf[idx++] = 0x01; /* T10 Vendor */
+            buf[idx++] = 0x00;
+            buf[idx++] = 68;
+            padstr8(buf + idx, 8, "ATA"); /* Generic T10 vendor */
+            idx += 8;
+            padstr8(buf + idx, 40, s->drive_model_str);
+            idx += 40;
+            padstr8(buf + idx, 20, s->drive_serial_str);
+            idx += 20;
+
+            /* Entry 3: WWN */
+            if (s->wwn && (idx + 12 <= max_len)) {
+                /* 4 byte header + 8 byte wwn */
+                buf[idx++] = 0x01; /* Binary */
+                buf[idx++] = 0x03; /* NAA */
+                buf[idx++] = 0x00;
+                buf[idx++] = 0x08;
+                stq_be_p(&buf[idx], s->wwn);
+                idx += 8;
+            }
+            break;
+
+        default:
+            /* SPC-3, revision 23 sec. 6.4 */
+            ide_atapi_cmd_error(s, ILLEGAL_REQUEST,
+                                ASC_INV_FIELD_IN_CMD_PACKET);
+            return;
+        }
+    } else {
+        preamble_len = 5;
+        size_idx = 4;
+
+        buf[0] = 0x05; /* CD-ROM */
+        buf[1] = 0x80; /* removable */
+        buf[2] = 0x00; /* ISO */
+        buf[3] = 0x21; /* ATAPI-2 (XXX: put ATAPI-4 ?) */
+        /* buf[size_idx] set below. */
+        buf[5] = 0;    /* reserved */
+        buf[6] = 0;    /* reserved */
+        buf[7] = 0;    /* reserved */
+        padstr8(buf + 8, 8, "QEMU");
+        padstr8(buf + 16, 16, "QEMU DVD-ROM");
+        padstr8(buf + 32, 4, s->version);
+        idx = 36;
+    }
+
+ out:
+    buf[size_idx] = idx - preamble_len;
+    ide_atapi_cmd_reply(s, idx, max_len);
 }
 
 static void cmd_get_configuration(IDEState *s, uint8_t *buf)
@@ -880,6 +1072,7 @@
 
     if (pwrcnd) {
         /* eject/load only happens for power condition == 0 */
+        ide_atapi_cmd_ok(s);
         return;
     }
 
@@ -1065,20 +1258,28 @@
      * 4.1.8)
      */
     CHECK_READY = 0x02,
+
+    /*
+     * Commands flagged with NONDATA do not in any circumstances return
+     * any data via ide_atapi_cmd_reply. These commands are exempt from
+     * the normal byte_count_limit constraints.
+     * See ATA8-ACS3 "7.21.5 Byte Count Limit"
+     */
+    NONDATA = 0x04,
 };
 
-static const struct {
+static const struct AtapiCmd {
     void (*handler)(IDEState *s, uint8_t *buf);
     int flags;
 } atapi_cmd_table[0x100] = {
-    [ 0x00 ] = { cmd_test_unit_ready,               CHECK_READY },
+    [ 0x00 ] = { cmd_test_unit_ready,               CHECK_READY | NONDATA },
     [ 0x03 ] = { cmd_request_sense,                 ALLOW_UA },
     [ 0x12 ] = { cmd_inquiry,                       ALLOW_UA },
-    [ 0x1b ] = { cmd_start_stop_unit,               0 }, /* [1] */
-    [ 0x1e ] = { cmd_prevent_allow_medium_removal,  0 },
+    [ 0x1b ] = { cmd_start_stop_unit,               NONDATA }, /* [1] */
+    [ 0x1e ] = { cmd_prevent_allow_medium_removal,  NONDATA },
     [ 0x25 ] = { cmd_read_cdvd_capacity,            CHECK_READY },
     [ 0x28 ] = { cmd_read, /* (10) */               CHECK_READY },
-    [ 0x2b ] = { cmd_seek,                          CHECK_READY },
+    [ 0x2b ] = { cmd_seek,                          CHECK_READY | NONDATA },
     [ 0x43 ] = { cmd_read_toc_pma_atip,             CHECK_READY },
     [ 0x46 ] = { cmd_get_configuration,             ALLOW_UA },
     [ 0x4a ] = { cmd_get_event_status_notification, ALLOW_UA },
@@ -1086,7 +1287,7 @@
     [ 0x5a ] = { cmd_mode_sense, /* (10) */         0 },
     [ 0xa8 ] = { cmd_read, /* (12) */               CHECK_READY },
     [ 0xad ] = { cmd_read_dvd_structure,            CHECK_READY },
-    [ 0xbb ] = { cmd_set_speed,                     0 },
+    [ 0xbb ] = { cmd_set_speed,                     NONDATA },
     [ 0xbd ] = { cmd_mechanism_status,              0 },
     [ 0xbe ] = { cmd_read_cd,                       CHECK_READY },
     /* [1] handler detects and reports not ready condition itself */
@@ -1094,9 +1295,9 @@
 
 void ide_atapi_cmd(IDEState *s)
 {
-    uint8_t *buf;
+    uint8_t *buf = s->io_buffer;
+    const struct AtapiCmd *cmd = &atapi_cmd_table[s->io_buffer[0]];
 
-    buf = s->io_buffer;
 #ifdef DEBUG_IDE_ATAPI
     {
         int i;
@@ -1107,14 +1308,14 @@
         printf("\n");
     }
 #endif
+
     /*
      * If there's a UNIT_ATTENTION condition pending, only command flagged with
      * ALLOW_UA are allowed to complete. with other commands getting a CHECK
      * condition response unless a higher priority status, defined by the drive
      * here, is pending.
      */
-    if (s->sense_key == UNIT_ATTENTION &&
-        !(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA)) {
+    if (s->sense_key == UNIT_ATTENTION && !(cmd->flags & ALLOW_UA)) {
         ide_atapi_cmd_check_status(s);
         return;
     }
@@ -1125,7 +1326,7 @@
      * GET_EVENT_STATUS_NOTIFICATION to detect such tray open/close
      * states rely on this behavior.
      */
-    if (!(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA) &&
+    if (!(cmd->flags & ALLOW_UA) &&
         !s->tray_open && blk_is_inserted(s->blk) && s->cdrom_changed) {
 
         if (s->cdrom_changed == 1) {
@@ -1140,16 +1341,29 @@
     }
 
     /* Report a Not Ready condition if appropriate for the command */
-    if ((atapi_cmd_table[s->io_buffer[0]].flags & CHECK_READY) &&
+    if ((cmd->flags & CHECK_READY) &&
         (!media_present(s) || !blk_is_inserted(s->blk)))
     {
         ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
         return;
     }
 
+    /* Nondata commands permit the byte_count_limit to be 0.
+     * If this is a data-transferring PIO command and BCL is 0,
+     * we abort at the /ATA/ level, not the ATAPI level.
+     * See ATA8 ACS3 section 7.17.6.49 and 7.21.5 */
+    if (cmd->handler && !(cmd->flags & NONDATA)) {
+        /* TODO: Check IDENTIFY data word 125 for default BCL (currently 0) */
+        if (!(atapi_byte_count_limit(s) || s->atapi_dma)) {
+            /* TODO: Move abort back into core.c and make static inline again */
+            ide_abort_command(s);
+            return;
+        }
+    }
+
     /* Execute the command */
-    if (atapi_cmd_table[s->io_buffer[0]].handler) {
-        atapi_cmd_table[s->io_buffer[0]].handler(s, buf);
+    if (cmd->handler) {
+        cmd->handler(s, buf);
         return;
     }
 
diff --git a/hw/ide/cmd646.c b/hw/ide/cmd646.c
index c8b0322..9ebb8d4 100644
--- a/hw/ide/cmd646.c
+++ b/hw/ide/cmd646.c
@@ -22,15 +22,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
+#include "hw/isa/isa.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/pci.h>
+#include "hw/ide/pci.h"
 
 /* CMD646 specific */
 #define CFR		0x50
@@ -326,7 +327,7 @@
 }
 
 /* CMD646 PCI IDE controller */
-static int pci_cmd646_ide_initfn(PCIDevice *dev)
+static void pci_cmd646_ide_realize(PCIDevice *dev, Error **errp)
 {
     PCIIDEState *d = PCI_IDE(dev);
     uint8_t *pci_conf = dev->config;
@@ -368,13 +369,11 @@
 
         bmdma_init(&d->bus[i], &d->bmdma[i], d);
         d->bmdma[i].bus = &d->bus[i];
-        qemu_add_vm_change_state_handler(d->bus[i].dma->ops->restart_cb,
-                                         &d->bmdma[i].dma);
+        ide_register_restart_cb(&d->bus[i]);
     }
 
     vmstate_register(DEVICE(dev), 0, &vmstate_ide_pci, d);
     qemu_register_reset(cmd646_reset, d);
-    return 0;
 }
 
 static void pci_cmd646_ide_exitfn(PCIDevice *dev)
@@ -410,7 +409,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_cmd646_ide_initfn;
+    k->realize = pci_cmd646_ide_realize;
     k->exit = pci_cmd646_ide_exitfn;
     k->vendor_id = PCI_VENDOR_ID_CMD;
     k->device_id = PCI_DEVICE_ID_CMD_646;
@@ -419,6 +418,7 @@
     k->config_read = cmd646_pci_config_read;
     k->config_write = cmd646_pci_config_write;
     dc->props = cmd646_ide_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo cmd646_ide_info = {
diff --git a/hw/ide/core.c b/hw/ide/core.c
index d4af5e2..45b6df1 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -22,18 +22,20 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
+#include "hw/isa/isa.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 #include "hw/block/block.h"
 #include "sysemu/block-backend.h"
+#include "qemu/cutils.h"
 
-#include <hw/ide/internal.h>
+#include "hw/ide/internal.h"
 
 /* These values were based on a Seagate ST3500418AS but have been modified
    to make more sense in QEMU */
@@ -55,7 +57,6 @@
     { 190,  0x03, 0x00, 0x45, 0x45, 0x1f, 0x00, 0x1f, 0x1f, 0x00, 0x00, 0x32},
 };
 
-static int ide_handle_rw_error(IDEState *s, int error, int op);
 static void ide_dummy_transfer_stop(IDEState *s);
 
 static void padstr(char *str, const char *src, int len)
@@ -422,8 +423,10 @@
                 }
 
                 /* Got an entry! Submit and exit.  */
-                iocb->aiocb = blk_aio_discard(iocb->blk, sector, count,
-                                              ide_issue_trim_cb, opaque);
+                iocb->aiocb = blk_aio_pdiscard(iocb->blk,
+                                               sector << BDRV_SECTOR_BITS,
+                                               count << BDRV_SECTOR_BITS,
+                                               ide_issue_trim_cb, opaque);
                 return;
             }
 
@@ -440,13 +443,14 @@
     }
 }
 
-BlockAIOCB *ide_issue_trim(BlockBackend *blk,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *ide_issue_trim(
+        int64_t offset, QEMUIOVector *qiov,
+        BlockCompletionFunc *cb, void *cb_opaque, void *opaque)
 {
+    BlockBackend *blk = opaque;
     TrimAIOCB *iocb;
 
-    iocb = blk_aio_get(&trim_aiocb_info, blk, cb, opaque);
+    iocb = blk_aio_get(&trim_aiocb_info, blk, cb, cb_opaque);
     iocb->blk = blk;
     iocb->bh = qemu_bh_new(ide_trim_bh_cb, iocb);
     iocb->ret = 0;
@@ -457,13 +461,27 @@
     return &iocb->common;
 }
 
-static inline void ide_abort_command(IDEState *s)
+void ide_abort_command(IDEState *s)
 {
     ide_transfer_stop(s);
     s->status = READY_STAT | ERR_STAT;
     s->error = ABRT_ERR;
 }
 
+static void ide_set_retry(IDEState *s)
+{
+    s->bus->retry_unit = s->unit;
+    s->bus->retry_sector_num = ide_get_sector(s);
+    s->bus->retry_nsector = s->nsector;
+}
+
+static void ide_clear_retry(IDEState *s)
+{
+    s->bus->retry_unit = -1;
+    s->bus->retry_sector_num = 0;
+    s->bus->retry_nsector = 0;
+}
+
 /* prepare data transfer and tell what to do after */
 void ide_transfer_start(IDEState *s, uint8_t *buf, int size,
                         EndTransferFunc *end_transfer_func)
@@ -471,6 +489,7 @@
     s->end_transfer_func = end_transfer_func;
     s->data_ptr = buf;
     s->data_end = buf + size;
+    ide_set_retry(s);
     if (!(s->status & ERR_STAT)) {
         s->status |= DRQ_STAT;
     }
@@ -486,13 +505,27 @@
     }
 }
 
-void ide_transfer_stop(IDEState *s)
+static void ide_transfer_halt(IDEState *s,
+                              void(*end_transfer_func)(IDEState *),
+                              bool notify)
 {
-    s->end_transfer_func = ide_transfer_stop;
+    s->end_transfer_func = end_transfer_func;
     s->data_ptr = s->io_buffer;
     s->data_end = s->io_buffer;
     s->status &= ~DRQ_STAT;
-    ide_cmd_done(s);
+    if (notify) {
+        ide_cmd_done(s);
+    }
+}
+
+void ide_transfer_stop(IDEState *s)
+{
+    ide_transfer_halt(s, ide_transfer_stop, true);
+}
+
+static void ide_transfer_cancel(IDEState *s)
+{
+    ide_transfer_halt(s, ide_transfer_cancel, false);
 }
 
 int64_t ide_get_sector(IDEState *s)
@@ -561,6 +594,100 @@
     return true;
 }
 
+static void ide_buffered_readv_cb(void *opaque, int ret)
+{
+    IDEBufferedRequest *req = opaque;
+    if (!req->orphaned) {
+        if (!ret) {
+            qemu_iovec_from_buf(req->original_qiov, 0, req->iov.iov_base,
+                                req->original_qiov->size);
+        }
+        req->original_cb(req->original_opaque, ret);
+    }
+    QLIST_REMOVE(req, list);
+    qemu_vfree(req->iov.iov_base);
+    g_free(req);
+}
+
+#define MAX_BUFFERED_REQS 16
+
+BlockAIOCB *ide_buffered_readv(IDEState *s, int64_t sector_num,
+                               QEMUIOVector *iov, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque)
+{
+    BlockAIOCB *aioreq;
+    IDEBufferedRequest *req;
+    int c = 0;
+
+    QLIST_FOREACH(req, &s->buffered_requests, list) {
+        c++;
+    }
+    if (c > MAX_BUFFERED_REQS) {
+        return blk_abort_aio_request(s->blk, cb, opaque, -EIO);
+    }
+
+    req = g_new0(IDEBufferedRequest, 1);
+    req->original_qiov = iov;
+    req->original_cb = cb;
+    req->original_opaque = opaque;
+    req->iov.iov_base = qemu_blockalign(blk_bs(s->blk), iov->size);
+    req->iov.iov_len = iov->size;
+    qemu_iovec_init_external(&req->qiov, &req->iov, 1);
+
+    aioreq = blk_aio_preadv(s->blk, sector_num << BDRV_SECTOR_BITS,
+                            &req->qiov, 0, ide_buffered_readv_cb, req);
+
+    QLIST_INSERT_HEAD(&s->buffered_requests, req, list);
+    return aioreq;
+}
+
+/**
+ * Cancel all pending DMA requests.
+ * Any buffered DMA requests are instantly canceled,
+ * but any pending unbuffered DMA requests must be waited on.
+ */
+void ide_cancel_dma_sync(IDEState *s)
+{
+    IDEBufferedRequest *req;
+
+    /* First invoke the callbacks of all buffered requests
+     * and flag those requests as orphaned. Ideally there
+     * are no unbuffered (Scatter Gather DMA Requests or
+     * write requests) pending and we can avoid to drain. */
+    QLIST_FOREACH(req, &s->buffered_requests, list) {
+        if (!req->orphaned) {
+#ifdef DEBUG_IDE
+            printf("%s: invoking cb %p of buffered request %p with"
+                   " -ECANCELED\n", __func__, req->original_cb, req);
+#endif
+            req->original_cb(req->original_opaque, -ECANCELED);
+        }
+        req->orphaned = true;
+    }
+
+    /*
+     * We can't cancel Scatter Gather DMA in the middle of the
+     * operation or a partial (not full) DMA transfer would reach
+     * the storage so we wait for completion instead (we beahve
+     * like if the DMA was completed by the time the guest trying
+     * to cancel dma with bmdma_cmd_writeb with BM_CMD_START not
+     * set).
+     *
+     * In the future we'll be able to safely cancel the I/O if the
+     * whole DMA operation will be submitted to disk with a single
+     * aio operation with preadv/pwritev.
+     */
+    if (s->bus->dma->aiocb) {
+#ifdef DEBUG_IDE
+        printf("%s: draining all remaining requests", __func__);
+#endif
+        blk_drain(s->blk);
+        assert(s->bus->dma->aiocb == NULL);
+    }
+}
+
+static void ide_sector_read(IDEState *s);
+
 static void ide_sector_read_cb(void *opaque, int ret)
 {
     IDEState *s = opaque;
@@ -572,7 +699,6 @@
     if (ret == -ECANCELED) {
         return;
     }
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
     if (ret != 0) {
         if (ide_handle_rw_error(s, -ret, IDE_RETRY_PIO |
                                 IDE_RETRY_READ)) {
@@ -580,22 +706,21 @@
         }
     }
 
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
     n = s->nsector;
     if (n > s->req_nb_sectors) {
         n = s->req_nb_sectors;
     }
 
-    /* Allow the guest to read the io_buffer */
-    ide_transfer_start(s, s->io_buffer, n * BDRV_SECTOR_SIZE, ide_sector_read);
-
-    ide_set_irq(s->bus);
-
     ide_set_sector(s, ide_get_sector(s) + n);
     s->nsector -= n;
-    s->io_buffer_offset += 512 * n;
+    /* Allow the guest to read the io_buffer */
+    ide_transfer_start(s, s->io_buffer, n * BDRV_SECTOR_SIZE, ide_sector_read);
+    ide_set_irq(s->bus);
 }
 
-void ide_sector_read(IDEState *s)
+static void ide_sector_read(IDEState *s)
 {
     int64_t sector_num;
     int n;
@@ -622,6 +747,7 @@
 
     if (!ide_sect_range_ok(s, sector_num, n)) {
         ide_rw_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_READ);
         return;
     }
 
@@ -631,21 +757,23 @@
 
     block_acct_start(blk_get_stats(s->blk), &s->acct,
                      n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-    s->pio_aiocb = blk_aio_readv(s->blk, sector_num, &s->qiov, n,
-                                 ide_sector_read_cb, s);
+    s->pio_aiocb = ide_buffered_readv(s, sector_num, &s->qiov, n,
+                                      ide_sector_read_cb, s);
 }
 
-static void dma_buf_commit(IDEState *s, uint32_t tx_bytes)
+void dma_buf_commit(IDEState *s, uint32_t tx_bytes)
 {
     if (s->bus->dma->ops->commit_buf) {
         s->bus->dma->ops->commit_buf(s->bus->dma, tx_bytes);
     }
+    s->io_buffer_offset += tx_bytes;
     qemu_sglist_destroy(&s->sg);
 }
 
 void ide_set_inactive(IDEState *s, bool more)
 {
     s->bus->dma->aiocb = NULL;
+    ide_clear_retry(s);
     if (s->bus->dma->ops->set_inactive) {
         s->bus->dma->ops->set_inactive(s->bus->dma, more);
     }
@@ -660,17 +788,20 @@
     ide_set_irq(s->bus);
 }
 
-static int ide_handle_rw_error(IDEState *s, int error, int op)
+int ide_handle_rw_error(IDEState *s, int error, int op)
 {
     bool is_read = (op & IDE_RETRY_READ) != 0;
     BlockErrorAction action = blk_get_error_action(s->blk, is_read, error);
 
     if (action == BLOCK_ERROR_ACTION_STOP) {
-        s->bus->dma->ops->set_unit(s->bus->dma, s->unit);
+        assert(s->bus->retry_unit == s->unit);
         s->bus->error_status = op;
     } else if (action == BLOCK_ERROR_ACTION_REPORT) {
-        if (op & IDE_RETRY_DMA) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        if (IS_IDE_RETRY_DMA(op)) {
             ide_dma_error(s);
+        } else if (IS_IDE_RETRY_ATAPI(op)) {
+            ide_atapi_io_error(s, -error);
         } else {
             ide_rw_error(s);
         }
@@ -679,25 +810,21 @@
     return action != BLOCK_ERROR_ACTION_IGNORE;
 }
 
-void ide_dma_cb(void *opaque, int ret)
+static void ide_dma_cb(void *opaque, int ret)
 {
     IDEState *s = opaque;
     int n;
     int64_t sector_num;
+    uint64_t offset;
     bool stay_active = false;
 
     if (ret == -ECANCELED) {
         return;
     }
     if (ret < 0) {
-        int op = IDE_RETRY_DMA;
-
-        if (s->dma_cmd == IDE_DMA_READ)
-            op |= IDE_RETRY_READ;
-        else if (s->dma_cmd == IDE_DMA_TRIM)
-            op |= IDE_RETRY_TRIM;
-
-        if (ide_handle_rw_error(s, -ret, op)) {
+        if (ide_handle_rw_error(s, -ret, ide_dma_cmd_to_retry(s->dma_cmd))) {
+            s->bus->dma->aiocb = NULL;
+            dma_buf_commit(s, 0);
             return;
         }
     }
@@ -713,8 +840,8 @@
 
     sector_num = ide_get_sector(s);
     if (n > 0) {
-        assert(s->io_buffer_size == s->sg.size);
-        dma_buf_commit(s, s->io_buffer_size);
+        assert(n * 512 == s->sg.size);
+        dma_buf_commit(s, s->sg.size);
         sector_num += n;
         ide_set_sector(s, sector_num);
         s->nsector -= n;
@@ -731,7 +858,7 @@
     n = s->nsector;
     s->io_buffer_index = 0;
     s->io_buffer_size = n * 512;
-    if (s->bus->dma->ops->prepare_buf(s->bus->dma, ide_cmd_is_read(s)) < 512) {
+    if (s->bus->dma->ops->prepare_buf(s->bus->dma, s->io_buffer_size) < 512) {
         /* The PRDs were too short. Reset the Active bit, but don't raise an
          * interrupt. */
         s->status = READY_STAT | SEEK_STAT;
@@ -747,23 +874,28 @@
     if ((s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) &&
         !ide_sect_range_ok(s, sector_num, n)) {
         ide_dma_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), s->acct.type);
         return;
     }
 
+    offset = sector_num << BDRV_SECTOR_BITS;
     switch (s->dma_cmd) {
     case IDE_DMA_READ:
-        s->bus->dma->aiocb = dma_blk_read(s->blk, &s->sg, sector_num,
+        s->bus->dma->aiocb = dma_blk_read(s->blk, &s->sg, offset,
                                           ide_dma_cb, s);
         break;
     case IDE_DMA_WRITE:
-        s->bus->dma->aiocb = dma_blk_write(s->blk, &s->sg, sector_num,
+        s->bus->dma->aiocb = dma_blk_write(s->blk, &s->sg, offset,
                                            ide_dma_cb, s);
         break;
     case IDE_DMA_TRIM:
-        s->bus->dma->aiocb = dma_blk_io(s->blk, &s->sg, sector_num,
-                                        ide_issue_trim, ide_dma_cb, s,
+        s->bus->dma->aiocb = dma_blk_io(blk_get_aio_context(s->blk),
+                                        &s->sg, offset,
+                                        ide_issue_trim, s->blk, ide_dma_cb, s,
                                         DMA_DIRECTION_TO_DEVICE);
         break;
+    default:
+        abort();
     }
     return;
 
@@ -777,7 +909,6 @@
 static void ide_sector_start_dma(IDEState *s, enum ide_dma_cmd dma_cmd)
 {
     s->status = READY_STAT | SEEK_STAT | DRQ_STAT | BUSY_STAT;
-    s->io_buffer_index = 0;
     s->io_buffer_size = 0;
     s->dma_cmd = dma_cmd;
 
@@ -799,11 +930,15 @@
 
 void ide_start_dma(IDEState *s, BlockCompletionFunc *cb)
 {
+    s->io_buffer_index = 0;
+    ide_set_retry(s);
     if (s->bus->dma->ops->start_dma) {
         s->bus->dma->ops->start_dma(s->bus->dma, s, cb);
     }
 }
 
+static void ide_sector_write(IDEState *s);
+
 static void ide_sector_write_timer_cb(void *opaque)
 {
     IDEState *s = opaque;
@@ -818,7 +953,6 @@
     if (ret == -ECANCELED) {
         return;
     }
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
 
     s->pio_aiocb = NULL;
     s->status &= ~BUSY_STAT;
@@ -829,13 +963,15 @@
         }
     }
 
+    block_acct_done(blk_get_stats(s->blk), &s->acct);
+
     n = s->nsector;
     if (n > s->req_nb_sectors) {
         n = s->req_nb_sectors;
     }
     s->nsector -= n;
-    s->io_buffer_offset += 512 * n;
 
+    ide_set_sector(s, ide_get_sector(s) + n);
     if (s->nsector == 0) {
         /* no more sectors to write */
         ide_transfer_stop(s);
@@ -847,7 +983,6 @@
         ide_transfer_start(s, s->io_buffer, n1 * BDRV_SECTOR_SIZE,
                            ide_sector_write);
     }
-    ide_set_sector(s, ide_get_sector(s) + n);
 
     if (win2k_install_hack && ((++s->irq_count % 16) == 0)) {
         /* It seems there is a bug in the Windows 2000 installer HDD
@@ -856,14 +991,14 @@
            that at the expense of slower write performances. Use this
            option _only_ to install Windows 2000. You must disable it
            for normal use. */
-        timer_mod(s->sector_write_timer,
-                       qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() / 1000));
+        timer_mod(s->sector_write_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  (NANOSECONDS_PER_SECOND / 1000));
     } else {
         ide_set_irq(s->bus);
     }
 }
 
-void ide_sector_write(IDEState *s)
+static void ide_sector_write(IDEState *s)
 {
     int64_t sector_num;
     int n;
@@ -880,6 +1015,7 @@
 
     if (!ide_sect_range_ok(s, sector_num, n)) {
         ide_rw_error(s);
+        block_acct_invalid(blk_get_stats(s->blk), BLOCK_ACCT_WRITE);
         return;
     }
 
@@ -888,9 +1024,9 @@
     qemu_iovec_init_external(&s->qiov, &s->iov, 1);
 
     block_acct_start(blk_get_stats(s->blk), &s->acct,
-                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-    s->pio_aiocb = blk_aio_writev(s->blk, sector_num, &s->qiov, n,
-                                  ide_sector_write_cb, s);
+                     n * BDRV_SECTOR_SIZE, BLOCK_ACCT_WRITE);
+    s->pio_aiocb = blk_aio_pwritev(s->blk, sector_num << BDRV_SECTOR_BITS,
+                                   &s->qiov, 0, ide_sector_write_cb, s);
 }
 
 static void ide_flush_cb(void *opaque, int ret)
@@ -917,7 +1053,7 @@
     ide_set_irq(s->bus);
 }
 
-void ide_flush_cache(IDEState *s)
+static void ide_flush_cache(IDEState *s)
 {
     if (s->blk == NULL) {
         ide_flush_cb(s, 0);
@@ -925,6 +1061,7 @@
     }
 
     s->status |= BUSY_STAT;
+    ide_set_retry(s);
     block_acct_start(blk_get_stats(s->blk), &s->acct, 0, BLOCK_ACCT_FLUSH);
     s->pio_aiocb = blk_aio_flush(s->blk, ide_flush_cb, s);
 }
@@ -1114,11 +1251,86 @@
     }
 }
 
+static void ide_reset(IDEState *s)
+{
+#ifdef DEBUG_IDE
+    printf("ide: reset\n");
+#endif
+
+    if (s->pio_aiocb) {
+        blk_aio_cancel(s->pio_aiocb);
+        s->pio_aiocb = NULL;
+    }
+
+    if (s->drive_kind == IDE_CFATA)
+        s->mult_sectors = 0;
+    else
+        s->mult_sectors = MAX_MULT_SECTORS;
+    /* ide regs */
+    s->feature = 0;
+    s->error = 0;
+    s->nsector = 0;
+    s->sector = 0;
+    s->lcyl = 0;
+    s->hcyl = 0;
+
+    /* lba48 */
+    s->hob_feature = 0;
+    s->hob_sector = 0;
+    s->hob_nsector = 0;
+    s->hob_lcyl = 0;
+    s->hob_hcyl = 0;
+
+    s->select = 0xa0;
+    s->status = READY_STAT | SEEK_STAT;
+
+    s->lba48 = 0;
+
+    /* ATAPI specific */
+    s->sense_key = 0;
+    s->asc = 0;
+    s->cdrom_changed = 0;
+    s->packet_transfer_size = 0;
+    s->elementary_transfer_size = 0;
+    s->io_buffer_index = 0;
+    s->cd_sector_size = 0;
+    s->atapi_dma = 0;
+    s->tray_locked = 0;
+    s->tray_open = 0;
+    /* ATA DMA state */
+    s->io_buffer_size = 0;
+    s->req_nb_sectors = 0;
+
+    ide_set_signature(s);
+    /* init the transfer handler so that 0xffff is returned on data
+       accesses */
+    s->end_transfer_func = ide_dummy_transfer_stop;
+    ide_dummy_transfer_stop(s);
+    s->media_changed = 0;
+}
+
 static bool cmd_nop(IDEState *s, uint8_t cmd)
 {
     return true;
 }
 
+static bool cmd_device_reset(IDEState *s, uint8_t cmd)
+{
+    /* Halt PIO (in the DRQ phase), then DMA */
+    ide_transfer_cancel(s);
+    ide_cancel_dma_sync(s);
+
+    /* Reset any PIO commands, reset signature, etc */
+    ide_reset(s);
+
+    /* RESET: ATA8-ACS3 7.10.4 "Normal Outputs";
+     * ATA8-ACS3 Table 184 "Device Signatures for Normal Output" */
+    s->status = 0x00;
+
+    /* Do not overwrite status register */
+    return false;
+}
+
 static bool cmd_data_set_management(IDEState *s, uint8_t cmd)
 {
     switch (s->feature) {
@@ -1435,15 +1647,6 @@
     return false;
 }
 
-static bool cmd_device_reset(IDEState *s, uint8_t cmd)
-{
-    ide_set_signature(s);
-    s->status = 0x00; /* NOTE: READY is _not_ set */
-    s->error = 0x01;
-
-    return false;
-}
-
 static bool cmd_packet(IDEState *s, uint8_t cmd)
 {
     /* overlapping commands not supported */
@@ -1454,6 +1657,9 @@
 
     s->status = READY_STAT | SEEK_STAT;
     s->atapi_dma = s->feature & 1;
+    if (s->atapi_dma) {
+        s->dma_cmd = IDE_DMA_ATAPI;
+    }
     s->nsector = 1;
     ide_transfer_start(s, s->io_buffer, ATAPI_PACKET_SIZE,
                        ide_atapi_cmd);
@@ -1739,11 +1945,11 @@
 } ide_cmd_table[0x100] = {
     /* NOP not implemented, mandatory for CD */
     [CFA_REQ_EXT_ERROR_CODE]      = { cmd_cfa_req_ext_error_code, CFA_OK },
-    [WIN_DSM]                     = { cmd_data_set_management, ALL_OK },
+    [WIN_DSM]                     = { cmd_data_set_management, HD_CFA_OK },
     [WIN_DEVICE_RESET]            = { cmd_device_reset, CD_OK },
     [WIN_RECAL]                   = { cmd_nop, HD_CFA_OK | SET_DSC},
     [WIN_READ]                    = { cmd_read_pio, ALL_OK },
-    [WIN_READ_ONCE]               = { cmd_read_pio, ALL_OK },
+    [WIN_READ_ONCE]               = { cmd_read_pio, HD_CFA_OK },
     [WIN_READ_EXT]                = { cmd_read_pio, HD_CFA_OK },
     [WIN_READDMA_EXT]             = { cmd_read_dma, HD_CFA_OK },
     [WIN_READ_NATIVE_MAX_EXT]     = { cmd_read_native_max, HD_CFA_OK | SET_DSC },
@@ -1762,12 +1968,12 @@
     [CFA_TRANSLATE_SECTOR]        = { cmd_cfa_translate_sector, CFA_OK },
     [WIN_DIAGNOSE]                = { cmd_exec_dev_diagnostic, ALL_OK },
     [WIN_SPECIFY]                 = { cmd_nop, HD_CFA_OK | SET_DSC },
-    [WIN_STANDBYNOW2]             = { cmd_nop, ALL_OK },
-    [WIN_IDLEIMMEDIATE2]          = { cmd_nop, ALL_OK },
-    [WIN_STANDBY2]                = { cmd_nop, ALL_OK },
-    [WIN_SETIDLE2]                = { cmd_nop, ALL_OK },
-    [WIN_CHECKPOWERMODE2]         = { cmd_check_power_mode, ALL_OK | SET_DSC },
-    [WIN_SLEEPNOW2]               = { cmd_nop, ALL_OK },
+    [WIN_STANDBYNOW2]             = { cmd_nop, HD_CFA_OK },
+    [WIN_IDLEIMMEDIATE2]          = { cmd_nop, HD_CFA_OK },
+    [WIN_STANDBY2]                = { cmd_nop, HD_CFA_OK },
+    [WIN_SETIDLE2]                = { cmd_nop, HD_CFA_OK },
+    [WIN_CHECKPOWERMODE2]         = { cmd_check_power_mode, HD_CFA_OK | SET_DSC },
+    [WIN_SLEEPNOW2]               = { cmd_nop, HD_CFA_OK },
     [WIN_PACKETCMD]               = { cmd_packet, CD_OK },
     [WIN_PIDENTIFY]               = { cmd_identify_packet, CD_OK },
     [WIN_SMART]                   = { cmd_smart, HD_CFA_OK | SET_DSC },
@@ -1781,19 +1987,19 @@
     [WIN_WRITEDMA]                = { cmd_write_dma, HD_CFA_OK },
     [WIN_WRITEDMA_ONCE]           = { cmd_write_dma, HD_CFA_OK },
     [CFA_WRITE_MULTI_WO_ERASE]    = { cmd_write_multiple, CFA_OK },
-    [WIN_STANDBYNOW1]             = { cmd_nop, ALL_OK },
-    [WIN_IDLEIMMEDIATE]           = { cmd_nop, ALL_OK },
-    [WIN_STANDBY]                 = { cmd_nop, ALL_OK },
-    [WIN_SETIDLE1]                = { cmd_nop, ALL_OK },
-    [WIN_CHECKPOWERMODE1]         = { cmd_check_power_mode, ALL_OK | SET_DSC },
-    [WIN_SLEEPNOW1]               = { cmd_nop, ALL_OK },
+    [WIN_STANDBYNOW1]             = { cmd_nop, HD_CFA_OK },
+    [WIN_IDLEIMMEDIATE]           = { cmd_nop, HD_CFA_OK },
+    [WIN_STANDBY]                 = { cmd_nop, HD_CFA_OK },
+    [WIN_SETIDLE1]                = { cmd_nop, HD_CFA_OK },
+    [WIN_CHECKPOWERMODE1]         = { cmd_check_power_mode, HD_CFA_OK | SET_DSC },
+    [WIN_SLEEPNOW1]               = { cmd_nop, HD_CFA_OK },
     [WIN_FLUSH_CACHE]             = { cmd_flush_cache, ALL_OK },
     [WIN_FLUSH_CACHE_EXT]         = { cmd_flush_cache, HD_CFA_OK },
     [WIN_IDENTIFY]                = { cmd_identify, ALL_OK },
     [WIN_SETFEATURES]             = { cmd_set_features, ALL_OK | SET_DSC },
     [IBM_SENSE_CONDITION]         = { cmd_ibm_sense_condition, CFA_OK | SET_DSC },
     [CFA_WEAR_LEVEL]              = { cmd_cfa_erase_sectors, HD_CFA_OK | SET_DSC },
-    [WIN_READ_NATIVE_MAX]         = { cmd_read_native_max, ALL_OK | SET_DSC },
+    [WIN_READ_NATIVE_MAX]         = { cmd_read_native_max, HD_CFA_OK | SET_DSC },
 };
 
 static bool ide_cmd_permitted(IDEState *s, uint32_t cmd)
@@ -1816,9 +2022,13 @@
         return;
     }
 
-    /* Only DEVICE RESET is allowed while BSY or/and DRQ are set */
-    if ((s->status & (BUSY_STAT|DRQ_STAT)) && val != WIN_DEVICE_RESET)
-        return;
+    /* Only RESET is allowed while BSY and/or DRQ are set,
+     * and only to ATAPI devices. */
+    if (s->status & (BUSY_STAT|DRQ_STAT)) {
+        if (val != WIN_DEVICE_RESET || s->drive_kind != IDE_CD) {
+            return;
+        }
+    }
 
     if (!ide_cmd_permitted(s, val)) {
         ide_abort_command(s);
@@ -2013,11 +2223,17 @@
     }
 
     p = s->data_ptr;
+    if (p + 2 > s->data_end) {
+        return;
+    }
+
     *(uint16_t *)p = le16_to_cpu(val);
     p += 2;
     s->data_ptr = p;
-    if (p >= s->data_end)
+    if (p >= s->data_end) {
+        s->status &= ~DRQ_STAT;
         s->end_transfer_func(s);
+    }
 }
 
 uint32_t ide_data_readw(void *opaque, uint32_t addr)
@@ -2034,11 +2250,17 @@
     }
 
     p = s->data_ptr;
+    if (p + 2 > s->data_end) {
+        return 0;
+    }
+
     ret = cpu_to_le16(*(uint16_t *)p);
     p += 2;
     s->data_ptr = p;
-    if (p >= s->data_end)
+    if (p >= s->data_end) {
+        s->status &= ~DRQ_STAT;
         s->end_transfer_func(s);
+    }
     return ret;
 }
 
@@ -2055,11 +2277,17 @@
     }
 
     p = s->data_ptr;
+    if (p + 4 > s->data_end) {
+        return;
+    }
+
     *(uint32_t *)p = le32_to_cpu(val);
     p += 4;
     s->data_ptr = p;
-    if (p >= s->data_end)
+    if (p >= s->data_end) {
+        s->status &= ~DRQ_STAT;
         s->end_transfer_func(s);
+    }
 }
 
 uint32_t ide_data_readl(void *opaque, uint32_t addr)
@@ -2076,11 +2304,17 @@
     }
 
     p = s->data_ptr;
+    if (p + 4 > s->data_end) {
+        return 0;
+    }
+
     ret = cpu_to_le32(*(uint32_t *)p);
     p += 4;
     s->data_ptr = p;
-    if (p >= s->data_end)
+    if (p >= s->data_end) {
+        s->status &= ~DRQ_STAT;
         s->end_transfer_func(s);
+    }
     return ret;
 }
 
@@ -2094,64 +2328,6 @@
     s->io_buffer[3] = 0xff;
 }
 
-static void ide_reset(IDEState *s)
-{
-#ifdef DEBUG_IDE
-    printf("ide: reset\n");
-#endif
-
-    if (s->pio_aiocb) {
-        blk_aio_cancel(s->pio_aiocb);
-        s->pio_aiocb = NULL;
-    }
-
-    if (s->drive_kind == IDE_CFATA)
-        s->mult_sectors = 0;
-    else
-        s->mult_sectors = MAX_MULT_SECTORS;
-    /* ide regs */
-    s->feature = 0;
-    s->error = 0;
-    s->nsector = 0;
-    s->sector = 0;
-    s->lcyl = 0;
-    s->hcyl = 0;
-
-    /* lba48 */
-    s->hob_feature = 0;
-    s->hob_sector = 0;
-    s->hob_nsector = 0;
-    s->hob_lcyl = 0;
-    s->hob_hcyl = 0;
-
-    s->select = 0xa0;
-    s->status = READY_STAT | SEEK_STAT;
-
-    s->lba48 = 0;
-
-    /* ATAPI specific */
-    s->sense_key = 0;
-    s->asc = 0;
-    s->cdrom_changed = 0;
-    s->packet_transfer_size = 0;
-    s->elementary_transfer_size = 0;
-    s->io_buffer_index = 0;
-    s->cd_sector_size = 0;
-    s->atapi_dma = 0;
-    s->tray_locked = 0;
-    s->tray_open = 0;
-    /* ATA DMA state */
-    s->io_buffer_size = 0;
-    s->req_nb_sectors = 0;
-
-    ide_set_signature(s);
-    /* init the transfer handler so that 0xffff is returned on data
-       accesses */
-    s->end_transfer_func = ide_dummy_transfer_stop;
-    ide_dummy_transfer_stop(s);
-    s->media_changed = 0;
-}
-
 void ide_bus_reset(IDEBus *bus)
 {
     bus->unit = 0;
@@ -2281,7 +2457,7 @@
     if (version) {
         pstrcpy(s->version, sizeof(s->version), version);
     } else {
-        pstrcpy(s->version, sizeof(s->version), qemu_get_version());
+        pstrcpy(s->version, sizeof(s->version), qemu_hw_version());
     }
 
     ide_reset(s);
@@ -2314,22 +2490,102 @@
     return 0;
 }
 
-static int32_t ide_nop_int32(IDEDMA *dma, int x)
+static void ide_nop(IDEDMA *dma)
+{
+}
+
+static int32_t ide_nop_int32(IDEDMA *dma, int32_t l)
 {
     return 0;
 }
 
-static void ide_nop_restart(void *opaque, int x, RunState y)
-{
-}
-
 static const IDEDMAOps ide_dma_nop_ops = {
     .prepare_buf    = ide_nop_int32,
+    .restart_dma    = ide_nop,
     .rw_buf         = ide_nop_int,
-    .set_unit       = ide_nop_int,
-    .restart_cb     = ide_nop_restart,
 };
 
+static void ide_restart_dma(IDEState *s, enum ide_dma_cmd dma_cmd)
+{
+    s->unit = s->bus->retry_unit;
+    ide_set_sector(s, s->bus->retry_sector_num);
+    s->nsector = s->bus->retry_nsector;
+    s->bus->dma->ops->restart_dma(s->bus->dma);
+    s->io_buffer_size = 0;
+    s->dma_cmd = dma_cmd;
+    ide_start_dma(s, ide_dma_cb);
+}
+
+static void ide_restart_bh(void *opaque)
+{
+    IDEBus *bus = opaque;
+    IDEState *s;
+    bool is_read;
+    int error_status;
+
+    qemu_bh_delete(bus->bh);
+    bus->bh = NULL;
+
+    error_status = bus->error_status;
+    if (bus->error_status == 0) {
+        return;
+    }
+
+    s = idebus_active_if(bus);
+    is_read = (bus->error_status & IDE_RETRY_READ) != 0;
+
+    /* The error status must be cleared before resubmitting the request: The
+     * request may fail again, and this case can only be distinguished if the
+     * called function can set a new error status. */
+    bus->error_status = 0;
+
+    /* The HBA has generically asked to be kicked on retry */
+    if (error_status & IDE_RETRY_HBA) {
+        if (s->bus->dma->ops->restart) {
+            s->bus->dma->ops->restart(s->bus->dma);
+        }
+    } else if (IS_IDE_RETRY_DMA(error_status)) {
+        if (error_status & IDE_RETRY_TRIM) {
+            ide_restart_dma(s, IDE_DMA_TRIM);
+        } else {
+            ide_restart_dma(s, is_read ? IDE_DMA_READ : IDE_DMA_WRITE);
+        }
+    } else if (IS_IDE_RETRY_PIO(error_status)) {
+        if (is_read) {
+            ide_sector_read(s);
+        } else {
+            ide_sector_write(s);
+        }
+    } else if (error_status & IDE_RETRY_FLUSH) {
+        ide_flush_cache(s);
+    } else if (IS_IDE_RETRY_ATAPI(error_status)) {
+        assert(s->end_transfer_func == ide_atapi_cmd);
+        ide_atapi_dma_restart(s);
+    } else {
+        abort();
+    }
+}
+
+static void ide_restart_cb(void *opaque, int running, RunState state)
+{
+    IDEBus *bus = opaque;
+
+    if (!running)
+        return;
+
+    if (!bus->bh) {
+        bus->bh = qemu_bh_new(ide_restart_bh, bus);
+        qemu_bh_schedule(bus->bh);
+    }
+}
+
+void ide_register_restart_cb(IDEBus *bus)
+{
+    if (bus->dma->ops->restart_dma) {
+        qemu_add_vm_change_state_handler(ide_restart_cb, bus);
+    }
+}
+
 static IDEDMA ide_dma_nop = {
     .ops = &ide_dma_nop_ops,
     .aiocb = NULL,
@@ -2349,8 +2605,8 @@
 
 static const MemoryRegionPortio ide_portio_list[] = {
     { 0, 8, 1, .read = ide_ioport_read, .write = ide_ioport_write },
-    { 0, 2, 2, .read = ide_data_readw, .write = ide_data_writew },
-    { 0, 4, 4, .read = ide_data_readl, .write = ide_data_writel },
+    { 0, 1, 2, .read = ide_data_readw, .write = ide_data_writew },
+    { 0, 1, 4, .read = ide_data_readl, .write = ide_data_writel },
     PORTIO_END_OF_LIST(),
 };
 
@@ -2417,6 +2673,7 @@
     s->end_transfer_func = transfer_end_table[s->end_transfer_fn_idx];
     s->data_ptr = s->io_buffer + s->cur_io_buffer_offset;
     s->data_end = s->data_ptr + s->cur_io_buffer_len;
+    s->atapi_dma = s->feature & 1; /* as per cmd_packet */
 
     return 0;
 }
@@ -2473,6 +2730,7 @@
     .name ="ide_drive/atapi/gesn_state",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = ide_atapi_gesn_needed,
     .fields = (VMStateField[]) {
         VMSTATE_BOOL(events.new_media, IDEState),
         VMSTATE_BOOL(events.eject_request, IDEState),
@@ -2484,6 +2742,7 @@
     .name = "ide_drive/tray_state",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = ide_tray_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_BOOL(tray_open, IDEState),
         VMSTATE_BOOL(tray_locked, IDEState),
@@ -2497,6 +2756,7 @@
     .minimum_version_id = 1,
     .pre_save = ide_drive_pio_pre_save,
     .post_load = ide_drive_pio_post_load,
+    .needed = ide_drive_pio_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(req_nb_sectors, IDEState),
         VMSTATE_VARRAY_INT32(io_buffer, IDEState, io_buffer_total_len, 1,
@@ -2538,28 +2798,24 @@
         VMSTATE_UINT8_V(cdrom_changed, IDEState, 3),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_ide_drive_pio_state,
-            .needed = ide_drive_pio_state_needed,
-        }, {
-            .vmsd = &vmstate_ide_tray_state,
-            .needed = ide_tray_state_needed,
-        }, {
-            .vmsd = &vmstate_ide_atapi_gesn_state,
-            .needed = ide_atapi_gesn_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_ide_drive_pio_state,
+        &vmstate_ide_tray_state,
+        &vmstate_ide_atapi_gesn_state,
+        NULL
     }
 };
 
 static const VMStateDescription vmstate_ide_error_status = {
     .name ="ide_bus/error",
-    .version_id = 1,
+    .version_id = 2,
     .minimum_version_id = 1,
+    .needed = ide_error_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(error_status, IDEBus),
+        VMSTATE_INT64_V(retry_sector_num, IDEBus, 2),
+        VMSTATE_UINT32_V(retry_nsector, IDEBus, 2),
+        VMSTATE_UINT8_V(retry_unit, IDEBus, 2),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -2573,13 +2829,9 @@
         VMSTATE_UINT8(unit, IDEBus),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_ide_error_status,
-            .needed = ide_error_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_ide_error_status,
+        NULL
     }
 };
 
diff --git a/hw/ide/ich.c b/hw/ide/ich.c
index fb1d095..4599169 100644
--- a/hw/ide/ich.c
+++ b/hw/ide/ich.c
@@ -60,16 +60,16 @@
  *
  */
 
-#include <hw/hw.h>
-#include <hw/pci/msi.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/msi.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
+#include "hw/isa/isa.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
-
-#include <hw/ide/pci.h>
-#include <hw/ide/ahci.h>
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
 
 #define ICH9_MSI_CAP_OFFSET     0x80
 #define ICH9_SATA_CAP_OFFSET    0xA8
@@ -82,7 +82,6 @@
 
 static const VMStateDescription vmstate_ich9_ahci = {
     .name = "ich9_ahci",
-    .unmigratable = 1, /* Still buggy under I/O load */
     .version_id = 1,
     .fields = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(parent_obj, AHCIPCIState),
@@ -98,14 +97,22 @@
     ahci_reset(&d->ahci);
 }
 
-static int pci_ich9_ahci_init(PCIDevice *dev)
+static void pci_ich9_ahci_init(Object *obj)
+{
+    struct AHCIPCIState *d = ICH_AHCI(obj);
+
+    ahci_init(&d->ahci, DEVICE(obj));
+}
+
+static void pci_ich9_ahci_realize(PCIDevice *dev, Error **errp)
 {
     struct AHCIPCIState *d;
     int sata_cap_offset;
     uint8_t *sata_cap;
     d = ICH_AHCI(dev);
+    int ret;
 
-    ahci_init(&d->ahci, DEVICE(dev), pci_get_address_space(dev), 6);
+    ahci_realize(&d->ahci, DEVICE(dev), pci_get_address_space(dev), 6);
 
     pci_config_set_prog_interface(dev->config, AHCI_PROGMODE_MAJOR_REV_1);
 
@@ -123,10 +130,11 @@
     pci_register_bar(dev, ICH9_MEM_BAR, PCI_BASE_ADDRESS_SPACE_MEMORY,
                      &d->ahci.mem);
 
-    sata_cap_offset = pci_add_capability(dev, PCI_CAP_ID_SATA,
-                                         ICH9_SATA_CAP_OFFSET, SATA_CAP_SIZE);
+    sata_cap_offset = pci_add_capability2(dev, PCI_CAP_ID_SATA,
+                                          ICH9_SATA_CAP_OFFSET, SATA_CAP_SIZE,
+                                          errp);
     if (sata_cap_offset < 0) {
-        return sata_cap_offset;
+        return;
     }
 
     sata_cap = dev->config + sata_cap_offset;
@@ -138,9 +146,10 @@
     /* Although the AHCI 1.3 specification states that the first capability
      * should be PMCAP, the Intel ICH9 data sheet specifies that the ICH9
      * AHCI device puts the MSI capability first, pointing to 0x80. */
-    msi_init(dev, ICH9_MSI_CAP_OFFSET, 1, true, false);
-
-    return 0;
+    ret = msi_init(dev, ICH9_MSI_CAP_OFFSET, 1, true, false, NULL);
+    /* Any error other than -ENOTSUP(board's MSI support is broken)
+     * is a programming error.  Fall back to INTx silently on -ENOTSUP */
+    assert(!ret || ret == -ENOTSUP);
 }
 
 static void pci_ich9_uninit(PCIDevice *dev)
@@ -158,7 +167,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ich9_ahci_init;
+    k->realize = pci_ich9_ahci_realize;
     k->exit = pci_ich9_uninit;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82801IR;
@@ -173,6 +182,7 @@
     .name          = TYPE_ICH9_AHCI,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(AHCIPCIState),
+    .instance_init = pci_ich9_ahci_init,
     .class_init    = ich_ahci_class_init,
 };
 
diff --git a/hw/ide/internal.h b/hw/ide/internal.h
deleted file mode 100644
index 8a3eca4..0000000
--- a/hw/ide/internal.h
+++ /dev/null
@@ -1,574 +0,0 @@
-#ifndef HW_IDE_INTERNAL_H
-#define HW_IDE_INTERNAL_H
-
-/*
- * QEMU IDE Emulation -- internal header file
- * only files in hw/ide/ are supposed to include this file.
- * non-internal declarations are in hw/ide.h
- */
-#include <hw/ide.h>
-#include <hw/isa/isa.h>
-#include "sysemu/dma.h"
-#include "sysemu/sysemu.h"
-#include "hw/block/block.h"
-#include "block/scsi.h"
-
-/* debug IDE devices */
-//#define DEBUG_IDE
-//#define DEBUG_IDE_ATAPI
-//#define DEBUG_AIO
-#define USE_DMA_CDROM
-
-typedef struct IDEBus IDEBus;
-typedef struct IDEDevice IDEDevice;
-typedef struct IDEState IDEState;
-typedef struct IDEDMA IDEDMA;
-typedef struct IDEDMAOps IDEDMAOps;
-
-#define TYPE_IDE_BUS "IDE"
-#define IDE_BUS(obj) OBJECT_CHECK(IDEBus, (obj), TYPE_IDE_BUS)
-
-/* Bits of HD_STATUS */
-#define ERR_STAT		0x01
-#define INDEX_STAT		0x02
-#define ECC_STAT		0x04	/* Corrected error */
-#define DRQ_STAT		0x08
-#define SEEK_STAT		0x10
-#define SRV_STAT		0x10
-#define WRERR_STAT		0x20
-#define READY_STAT		0x40
-#define BUSY_STAT		0x80
-
-/* Bits for HD_ERROR */
-#define MARK_ERR		0x01	/* Bad address mark */
-#define TRK0_ERR		0x02	/* couldn't find track 0 */
-#define ABRT_ERR		0x04	/* Command aborted */
-#define MCR_ERR			0x08	/* media change request */
-#define ID_ERR			0x10	/* ID field not found */
-#define MC_ERR			0x20	/* media changed */
-#define ECC_ERR			0x40	/* Uncorrectable ECC error */
-#define BBD_ERR			0x80	/* pre-EIDE meaning:  block marked bad */
-#define ICRC_ERR		0x80	/* new meaning:  CRC error during transfer */
-
-/* Bits of HD_NSECTOR */
-#define CD			0x01
-#define IO			0x02
-#define REL			0x04
-#define TAG_MASK		0xf8
-
-#define IDE_CMD_RESET           0x04
-#define IDE_CMD_DISABLE_IRQ     0x02
-
-/* ACS-2 T13/2015-D Table B.2 Command codes */
-#define WIN_NOP				0x00
-/* reserved                             0x01..0x02 */
-#define CFA_REQ_EXT_ERROR_CODE		0x03 /* CFA Request Extended Error Code */
-/* reserved                             0x04..0x05 */
-#define WIN_DSM                         0x06
-/* reserved                             0x07 */
-#define WIN_DEVICE_RESET		0x08
-/* reserved                             0x09..0x0a */
-/* REQUEST SENSE DATA EXT               0x0B */
-/* reserved                             0x0C..0x0F */
-#define WIN_RECAL                       0x10 /* obsolete since ATA4 */
-/* obsolete since ATA3, retired in ATA4 0x11..0x1F */
-#define WIN_READ			0x20 /* 28-Bit */
-#define WIN_READ_ONCE                   0x21 /* 28-Bit w/o retries, obsolete since ATA5 */
-/* obsolete since ATA4                  0x22..0x23 */
-#define WIN_READ_EXT			0x24 /* 48-Bit */
-#define WIN_READDMA_EXT			0x25 /* 48-Bit */
-#define WIN_READDMA_QUEUED_EXT          0x26 /* 48-Bit, obsolete since ACS2 */
-#define WIN_READ_NATIVE_MAX_EXT		0x27 /* 48-Bit */
-/* reserved                             0x28 */
-#define WIN_MULTREAD_EXT		0x29 /* 48-Bit */
-/* READ STREAM DMA EXT                  0x2A */
-/* READ STREAM EXT                      0x2B */
-/* reserved                             0x2C..0x2E */
-/* READ LOG EXT                         0x2F */
-#define WIN_WRITE			0x30 /* 28-Bit */
-#define WIN_WRITE_ONCE                  0x31 /* 28-Bit w/o retries, obsolete since ATA5 */
-/* obsolete since ATA4                  0x32..0x33 */
-#define WIN_WRITE_EXT			0x34 /* 48-Bit */
-#define WIN_WRITEDMA_EXT		0x35 /* 48-Bit */
-#define WIN_WRITEDMA_QUEUED_EXT		0x36 /* 48-Bit */
-#define WIN_SET_MAX_EXT                 0x37 /* 48-Bit, obsolete since ACS2 */
-#define WIN_SET_MAX_EXT			0x37 /* 48-Bit */
-#define CFA_WRITE_SECT_WO_ERASE		0x38 /* CFA Write Sectors without erase */
-#define WIN_MULTWRITE_EXT		0x39 /* 48-Bit */
-/* WRITE STREAM DMA EXT                 0x3A */
-/* WRITE STREAM EXT                     0x3B */
-#define WIN_WRITE_VERIFY                0x3C /* 28-Bit, obsolete since ATA4 */
-/* WRITE DMA FUA EXT                    0x3D */
-/* obsolete since ACS2                  0x3E */
-/* WRITE LOG EXT                        0x3F */
-#define WIN_VERIFY			0x40 /* 28-Bit - Read Verify Sectors */
-#define WIN_VERIFY_ONCE                 0x41 /* 28-Bit - w/o retries, obsolete since ATA5 */
-#define WIN_VERIFY_EXT			0x42 /* 48-Bit */
-/* reserved                             0x43..0x44 */
-/* WRITE UNCORRECTABLE EXT              0x45 */
-/* reserved                             0x46 */
-/* READ LOG DMA EXT                     0x47 */
-/* reserved                             0x48..0x4F */
-/* obsolete since ATA4                  0x50 */
-/* CONFIGURE STREAM                     0x51 */
-/* reserved                             0x52..0x56 */
-/* WRITE LOG DMA EXT                    0x57 */
-/* reserved                             0x58..0x5A */
-/* TRUSTED NON DATA                     0x5B */
-/* TRUSTED RECEIVE                      0x5C */
-/* TRUSTED RECEIVE DMA                  0x5D */
-/* TRUSTED SEND                         0x5E */
-/* TRUSTED SEND DMA                     0x5F */
-/* READ FPDMA QUEUED                    0x60 */
-/* WRITE FPDMA QUEUED                   0x61 */
-/* reserved                             0x62->0x6F */
-#define WIN_SEEK                        0x70 /* obsolete since ATA7 */
-/* reserved                             0x71-0x7F */
-/* vendor specific                      0x80-0x86 */
-#define CFA_TRANSLATE_SECTOR		0x87 /* CFA Translate Sector */
-/* vendor specific                      0x88-0x8F */
-#define WIN_DIAGNOSE			0x90
-#define WIN_SPECIFY                     0x91 /* set drive geometry translation, obsolete since ATA6 */
-#define WIN_DOWNLOAD_MICROCODE		0x92
-/* DOWNLOAD MICROCODE DMA               0x93 */
-#define WIN_STANDBYNOW2                 0x94 /* retired in ATA4 */
-#define WIN_IDLEIMMEDIATE2              0x95 /* force drive to become "ready", retired in ATA4 */
-#define WIN_STANDBY2                    0x96 /* retired in ATA4 */
-#define WIN_SETIDLE2                    0x97 /* retired in ATA4 */
-#define WIN_CHECKPOWERMODE2             0x98 /* retired in ATA4 */
-#define WIN_SLEEPNOW2                   0x99 /* retired in ATA4 */
-/* vendor specific                      0x9A */
-/* reserved                             0x9B..0x9F */
-#define WIN_PACKETCMD			0xA0 /* Send a packet command. */
-#define WIN_PIDENTIFY			0xA1 /* identify ATAPI device	*/
-#define WIN_QUEUED_SERVICE              0xA2 /* obsolete since ACS2 */
-/* reserved                             0xA3..0xAF */
-#define WIN_SMART			0xB0 /* self-monitoring and reporting */
-/* Device Configuration Overlay         0xB1 */
-/* reserved                             0xB2..0xB3 */
-/* Sanitize Device                      0xB4 */
-/* reserved                             0xB5 */
-/* NV Cache                             0xB6 */
-/* reserved for CFA                     0xB7..0xBB */
-#define CFA_ACCESS_METADATA_STORAGE	0xB8
-/* reserved                             0xBC..0xBF */
-#define CFA_ERASE_SECTORS       	0xC0 /* microdrives implement as NOP */
-/* vendor specific                      0xC1..0xC3 */
-#define WIN_MULTREAD			0xC4 /* read sectors using multiple mode*/
-#define WIN_MULTWRITE			0xC5 /* write sectors using multiple mode */
-#define WIN_SETMULT			0xC6 /* enable/disable multiple mode */
-#define WIN_READDMA_QUEUED              0xC7 /* read sectors using Queued DMA transfers, obsolete since ACS2 */
-#define WIN_READDMA			0xC8 /* read sectors using DMA transfers */
-#define WIN_READDMA_ONCE                0xC9 /* 28-Bit - w/o retries, obsolete since ATA5 */
-#define WIN_WRITEDMA			0xCA /* write sectors using DMA transfers */
-#define WIN_WRITEDMA_ONCE               0xCB /* 28-Bit - w/o retries, obsolete since ATA5 */
-#define WIN_WRITEDMA_QUEUED		0xCC /* write sectors using Queued DMA transfers, obsolete since ACS2 */
-#define CFA_WRITE_MULTI_WO_ERASE	0xCD /* CFA Write multiple without erase */
-/* WRITE MULTIPLE FUA EXT               0xCE */
-/* reserved                             0xCF..0xDO */
-/* CHECK MEDIA CARD TYPE                0xD1 */
-/* reserved for media card pass through 0xD2..0xD4 */
-/* reserved                             0xD5..0xD9 */
-#define WIN_GETMEDIASTATUS              0xDA /* obsolete since ATA8 */
-/* obsolete since ATA3, retired in ATA4 0xDB..0xDD */
-#define WIN_DOORLOCK                    0xDE /* lock door on removable drives, obsolete since ATA8 */
-#define WIN_DOORUNLOCK                  0xDF /* unlock door on removable drives, obsolete since ATA8 */
-#define WIN_STANDBYNOW1			0xE0
-#define WIN_IDLEIMMEDIATE		0xE1 /* force drive to become "ready" */
-#define WIN_STANDBY             	0xE2 /* Set device in Standby Mode */
-#define WIN_SETIDLE1			0xE3
-#define WIN_READ_BUFFER			0xE4 /* force read only 1 sector */
-#define WIN_CHECKPOWERMODE1		0xE5
-#define WIN_SLEEPNOW1			0xE6
-#define WIN_FLUSH_CACHE			0xE7
-#define WIN_WRITE_BUFFER		0xE8 /* force write only 1 sector */
-/* READ BUFFER DMA                      0xE9 */
-#define WIN_FLUSH_CACHE_EXT		0xEA /* 48-Bit */
-/* WRITE BUFFER DMA                     0xEB */
-#define WIN_IDENTIFY			0xEC /* ask drive to identify itself	*/
-#define WIN_MEDIAEJECT                  0xED /* obsolete since ATA8 */
-/* obsolete since ATA4                  0xEE */
-#define WIN_SETFEATURES			0xEF /* set special drive features */
-#define IBM_SENSE_CONDITION             0xF0 /* measure disk temperature, vendor specific */
-#define WIN_SECURITY_SET_PASS		0xF1
-#define WIN_SECURITY_UNLOCK		0xF2
-#define WIN_SECURITY_ERASE_PREPARE	0xF3
-#define WIN_SECURITY_ERASE_UNIT		0xF4
-#define WIN_SECURITY_FREEZE_LOCK	0xF5
-#define CFA_WEAR_LEVEL                  0xF5 /* microdrives implement as NOP; not specified in T13! */
-#define WIN_SECURITY_DISABLE		0xF6
-/* vendor specific                      0xF7 */
-#define WIN_READ_NATIVE_MAX		0xF8 /* return the native maximum address */
-#define WIN_SET_MAX			0xF9
-/* vendor specific                      0xFA..0xFF */
-
-/* set to 1 set disable mult support */
-#define MAX_MULT_SECTORS 16
-
-#define IDE_DMA_BUF_SECTORS 256
-
-/* feature values for Data Set Management */
-#define DSM_TRIM                        0x01
-
-#if (IDE_DMA_BUF_SECTORS < MAX_MULT_SECTORS)
-#error "IDE_DMA_BUF_SECTORS must be bigger or equal to MAX_MULT_SECTORS"
-#endif
-
-/* ATAPI defines */
-
-#define ATAPI_PACKET_SIZE 12
-
-/* The generic packet command opcodes for CD/DVD Logical Units,
- * From Table 57 of the SFF8090 Ver. 3 (Mt. Fuji) draft standard. */
-#define GPCMD_BLANK			    0xa1
-#define GPCMD_CLOSE_TRACK		    0x5b
-#define GPCMD_FLUSH_CACHE		    0x35
-#define GPCMD_FORMAT_UNIT		    0x04
-#define GPCMD_GET_CONFIGURATION		    0x46
-#define GPCMD_GET_EVENT_STATUS_NOTIFICATION 0x4a
-#define GPCMD_GET_PERFORMANCE		    0xac
-#define GPCMD_INQUIRY			    0x12
-#define GPCMD_LOAD_UNLOAD		    0xa6
-#define GPCMD_MECHANISM_STATUS		    0xbd
-#define GPCMD_MODE_SELECT_10		    0x55
-#define GPCMD_MODE_SENSE_10		    0x5a
-#define GPCMD_PAUSE_RESUME		    0x4b
-#define GPCMD_PLAY_AUDIO_10		    0x45
-#define GPCMD_PLAY_AUDIO_MSF		    0x47
-#define GPCMD_PLAY_AUDIO_TI		    0x48
-#define GPCMD_PLAY_CD			    0xbc
-#define GPCMD_PREVENT_ALLOW_MEDIUM_REMOVAL  0x1e
-#define GPCMD_READ_10			    0x28
-#define GPCMD_READ_12			    0xa8
-#define GPCMD_READ_CDVD_CAPACITY	    0x25
-#define GPCMD_READ_CD			    0xbe
-#define GPCMD_READ_CD_MSF		    0xb9
-#define GPCMD_READ_DISC_INFO		    0x51
-#define GPCMD_READ_DVD_STRUCTURE	    0xad
-#define GPCMD_READ_FORMAT_CAPACITIES	    0x23
-#define GPCMD_READ_HEADER		    0x44
-#define GPCMD_READ_TRACK_RZONE_INFO	    0x52
-#define GPCMD_READ_SUBCHANNEL		    0x42
-#define GPCMD_READ_TOC_PMA_ATIP		    0x43
-#define GPCMD_REPAIR_RZONE_TRACK	    0x58
-#define GPCMD_REPORT_KEY		    0xa4
-#define GPCMD_REQUEST_SENSE		    0x03
-#define GPCMD_RESERVE_RZONE_TRACK	    0x53
-#define GPCMD_SCAN			    0xba
-#define GPCMD_SEEK			    0x2b
-#define GPCMD_SEND_DVD_STRUCTURE	    0xad
-#define GPCMD_SEND_EVENT		    0xa2
-#define GPCMD_SEND_KEY			    0xa3
-#define GPCMD_SEND_OPC			    0x54
-#define GPCMD_SET_READ_AHEAD		    0xa7
-#define GPCMD_SET_STREAMING		    0xb6
-#define GPCMD_START_STOP_UNIT		    0x1b
-#define GPCMD_STOP_PLAY_SCAN		    0x4e
-#define GPCMD_TEST_UNIT_READY		    0x00
-#define GPCMD_VERIFY_10			    0x2f
-#define GPCMD_WRITE_10			    0x2a
-#define GPCMD_WRITE_AND_VERIFY_10	    0x2e
-/* This is listed as optional in ATAPI 2.6, but is (curiously)
- * missing from Mt. Fuji, Table 57.  It _is_ mentioned in Mt. Fuji
- * Table 377 as an MMC command for SCSi devices though...  Most ATAPI
- * drives support it. */
-#define GPCMD_SET_SPEED			    0xbb
-/* This seems to be a SCSI specific CD-ROM opcode
- * to play data at track/index */
-#define GPCMD_PLAYAUDIO_TI		    0x48
-/*
- * From MS Media Status Notification Support Specification. For
- * older drives only.
- */
-#define GPCMD_GET_MEDIA_STATUS		    0xda
-#define GPCMD_MODE_SENSE_6		    0x1a
-
-#define ATAPI_INT_REASON_CD             0x01 /* 0 = data transfer */
-#define ATAPI_INT_REASON_IO             0x02 /* 1 = transfer to the host */
-#define ATAPI_INT_REASON_REL            0x04
-#define ATAPI_INT_REASON_TAG            0xf8
-
-/* same constants as bochs */
-#define ASC_ILLEGAL_OPCODE                   0x20
-#define ASC_LOGICAL_BLOCK_OOR                0x21
-#define ASC_INV_FIELD_IN_CMD_PACKET          0x24
-#define ASC_MEDIUM_MAY_HAVE_CHANGED          0x28
-#define ASC_INCOMPATIBLE_FORMAT              0x30
-#define ASC_MEDIUM_NOT_PRESENT               0x3a
-#define ASC_SAVING_PARAMETERS_NOT_SUPPORTED  0x39
-#define ASC_MEDIA_REMOVAL_PREVENTED          0x53
-
-#define CFA_NO_ERROR            0x00
-#define CFA_MISC_ERROR          0x09
-#define CFA_INVALID_COMMAND     0x20
-#define CFA_INVALID_ADDRESS     0x21
-#define CFA_ADDRESS_OVERFLOW    0x2f
-
-#define SMART_READ_DATA       0xd0
-#define SMART_READ_THRESH     0xd1
-#define SMART_ATTR_AUTOSAVE   0xd2
-#define SMART_SAVE_ATTR       0xd3
-#define SMART_EXECUTE_OFFLINE 0xd4
-#define SMART_READ_LOG        0xd5
-#define SMART_WRITE_LOG       0xd6
-#define SMART_ENABLE          0xd8
-#define SMART_DISABLE         0xd9
-#define SMART_STATUS          0xda
-
-typedef enum { IDE_HD, IDE_CD, IDE_CFATA } IDEDriveKind;
-
-typedef void EndTransferFunc(IDEState *);
-
-typedef void DMAStartFunc(IDEDMA *, IDEState *, BlockCompletionFunc *);
-typedef void DMAVoidFunc(IDEDMA *);
-typedef int DMAIntFunc(IDEDMA *, int);
-typedef int32_t DMAInt32Func(IDEDMA *, int);
-typedef void DMAu32Func(IDEDMA *, uint32_t);
-typedef void DMAStopFunc(IDEDMA *, bool);
-typedef void DMARestartFunc(void *, int, RunState);
-
-struct unreported_events {
-    bool eject_request;
-    bool new_media;
-};
-
-enum ide_dma_cmd {
-    IDE_DMA_READ,
-    IDE_DMA_WRITE,
-    IDE_DMA_TRIM,
-};
-
-#define ide_cmd_is_read(s) \
-	((s)->dma_cmd == IDE_DMA_READ)
-
-/* NOTE: IDEState represents in fact one drive */
-struct IDEState {
-    IDEBus *bus;
-    uint8_t unit;
-    /* ide config */
-    IDEDriveKind drive_kind;
-    int cylinders, heads, sectors, chs_trans;
-    int64_t nb_sectors;
-    int mult_sectors;
-    int identify_set;
-    uint8_t identify_data[512];
-    int drive_serial;
-    char drive_serial_str[21];
-    char drive_model_str[41];
-    uint64_t wwn;
-    /* ide regs */
-    uint8_t feature;
-    uint8_t error;
-    uint32_t nsector;
-    uint8_t sector;
-    uint8_t lcyl;
-    uint8_t hcyl;
-    /* other part of tf for lba48 support */
-    uint8_t hob_feature;
-    uint8_t hob_nsector;
-    uint8_t hob_sector;
-    uint8_t hob_lcyl;
-    uint8_t hob_hcyl;
-
-    uint8_t select;
-    uint8_t status;
-
-    /* set for lba48 access */
-    uint8_t lba48;
-    BlockBackend *blk;
-    char version[9];
-    /* ATAPI specific */
-    struct unreported_events events;
-    uint8_t sense_key;
-    uint8_t asc;
-    bool tray_open;
-    bool tray_locked;
-    uint8_t cdrom_changed;
-    int packet_transfer_size;
-    int elementary_transfer_size;
-    int32_t io_buffer_index;
-    int lba;
-    int cd_sector_size;
-    int atapi_dma; /* true if dma is requested for the packet cmd */
-    BlockAcctCookie acct;
-    BlockAIOCB *pio_aiocb;
-    struct iovec iov;
-    QEMUIOVector qiov;
-    /* ATA DMA state */
-    int32_t io_buffer_offset;
-    int32_t io_buffer_size;
-    QEMUSGList sg;
-    /* PIO transfer handling */
-    int req_nb_sectors; /* number of sectors per interrupt */
-    EndTransferFunc *end_transfer_func;
-    uint8_t *data_ptr;
-    uint8_t *data_end;
-    uint8_t *io_buffer;
-    /* PIO save/restore */
-    int32_t io_buffer_total_len;
-    int32_t cur_io_buffer_offset;
-    int32_t cur_io_buffer_len;
-    uint8_t end_transfer_fn_idx;
-    QEMUTimer *sector_write_timer; /* only used for win2k install hack */
-    uint32_t irq_count; /* counts IRQs when using win2k install hack */
-    /* CF-ATA extended error */
-    uint8_t ext_error;
-    /* CF-ATA metadata storage */
-    uint32_t mdata_size;
-    uint8_t *mdata_storage;
-    int media_changed;
-    enum ide_dma_cmd dma_cmd;
-    /* SMART */
-    uint8_t smart_enabled;
-    uint8_t smart_autosave;
-    int smart_errors;
-    uint8_t smart_selftest_count;
-    uint8_t *smart_selftest_data;
-    /* AHCI */
-    int ncq_queues;
-};
-
-struct IDEDMAOps {
-    DMAStartFunc *start_dma;
-    DMAVoidFunc *start_transfer;
-    DMAInt32Func *prepare_buf;
-    DMAu32Func *commit_buf;
-    DMAIntFunc *rw_buf;
-    DMAIntFunc *set_unit;
-    DMAStopFunc *set_inactive;
-    DMAVoidFunc *cmd_done;
-    DMARestartFunc *restart_cb;
-    DMAVoidFunc *reset;
-};
-
-struct IDEDMA {
-    const struct IDEDMAOps *ops;
-    struct iovec iov;
-    QEMUIOVector qiov;
-    BlockAIOCB *aiocb;
-};
-
-struct IDEBus {
-    BusState qbus;
-    IDEDevice *master;
-    IDEDevice *slave;
-    IDEState ifs[2];
-    int bus_id;
-    int max_units;
-    IDEDMA *dma;
-    uint8_t unit;
-    uint8_t cmd;
-    qemu_irq irq;
-
-    int error_status;
-};
-
-#define TYPE_IDE_DEVICE "ide-device"
-#define IDE_DEVICE(obj) \
-     OBJECT_CHECK(IDEDevice, (obj), TYPE_IDE_DEVICE)
-#define IDE_DEVICE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(IDEDeviceClass, (klass), TYPE_IDE_DEVICE)
-#define IDE_DEVICE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(IDEDeviceClass, (obj), TYPE_IDE_DEVICE)
-
-typedef struct IDEDeviceClass {
-    DeviceClass parent_class;
-    int (*init)(IDEDevice *dev);
-} IDEDeviceClass;
-
-struct IDEDevice {
-    DeviceState qdev;
-    uint32_t unit;
-    BlockConf conf;
-    int chs_trans;
-    char *version;
-    char *serial;
-    char *model;
-    uint64_t wwn;
-};
-
-/* These are used for the error_status field of IDEBus */
-#define IDE_RETRY_DMA  0x08
-#define IDE_RETRY_PIO  0x10
-#define IDE_RETRY_READ  0x20
-#define IDE_RETRY_FLUSH 0x40
-#define IDE_RETRY_TRIM 0x80
-
-static inline IDEState *idebus_active_if(IDEBus *bus)
-{
-    return bus->ifs + bus->unit;
-}
-
-static inline void ide_set_irq(IDEBus *bus)
-{
-    if (!(bus->cmd & IDE_CMD_DISABLE_IRQ)) {
-        qemu_irq_raise(bus->irq);
-    }
-}
-
-/* hw/ide/core.c */
-extern const VMStateDescription vmstate_ide_bus;
-
-#define VMSTATE_IDE_BUS(_field, _state)                          \
-    VMSTATE_STRUCT(_field, _state, 1, vmstate_ide_bus, IDEBus)
-
-#define VMSTATE_IDE_BUS_ARRAY(_field, _state, _num)              \
-    VMSTATE_STRUCT_ARRAY(_field, _state, _num, 1, vmstate_ide_bus, IDEBus)
-
-extern const VMStateDescription vmstate_ide_drive;
-
-#define VMSTATE_IDE_DRIVES(_field, _state) \
-    VMSTATE_STRUCT_ARRAY(_field, _state, 2, 3, vmstate_ide_drive, IDEState)
-
-void ide_bus_reset(IDEBus *bus);
-int64_t ide_get_sector(IDEState *s);
-void ide_set_sector(IDEState *s, int64_t sector_num);
-
-void ide_start_dma(IDEState *s, BlockCompletionFunc *cb);
-void ide_dma_error(IDEState *s);
-
-void ide_atapi_cmd_ok(IDEState *s);
-void ide_atapi_cmd_error(IDEState *s, int sense_key, int asc);
-void ide_atapi_io_error(IDEState *s, int ret);
-
-void ide_ioport_write(void *opaque, uint32_t addr, uint32_t val);
-uint32_t ide_ioport_read(void *opaque, uint32_t addr1);
-uint32_t ide_status_read(void *opaque, uint32_t addr);
-void ide_cmd_write(void *opaque, uint32_t addr, uint32_t val);
-void ide_data_writew(void *opaque, uint32_t addr, uint32_t val);
-uint32_t ide_data_readw(void *opaque, uint32_t addr);
-void ide_data_writel(void *opaque, uint32_t addr, uint32_t val);
-uint32_t ide_data_readl(void *opaque, uint32_t addr);
-
-int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
-                   const char *version, const char *serial, const char *model,
-                   uint64_t wwn,
-                   uint32_t cylinders, uint32_t heads, uint32_t secs,
-                   int chs_trans);
-void ide_init2(IDEBus *bus, qemu_irq irq);
-void ide_init_ioport(IDEBus *bus, ISADevice *isa, int iobase, int iobase2);
-
-void ide_exec_cmd(IDEBus *bus, uint32_t val);
-void ide_dma_cb(void *opaque, int ret);
-void ide_sector_write(IDEState *s);
-void ide_sector_read(IDEState *s);
-void ide_flush_cache(IDEState *s);
-
-void ide_transfer_start(IDEState *s, uint8_t *buf, int size,
-                        EndTransferFunc *end_transfer_func);
-void ide_transfer_stop(IDEState *s);
-void ide_set_inactive(IDEState *s, bool more);
-BlockAIOCB *ide_issue_trim(BlockBackend *blk,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockCompletionFunc *cb, void *opaque);
-
-/* hw/ide/atapi.c */
-void ide_atapi_cmd(IDEState *s);
-void ide_atapi_cmd_reply_end(IDEState *s);
-
-/* hw/ide/qdev.c */
-void ide_bus_new(IDEBus *idebus, size_t idebus_size, DeviceState *dev,
-                 int bus_id, int max_units);
-IDEDevice *ide_create_drive(IDEBus *bus, int unit, DriveInfo *drive);
-
-#endif /* HW_IDE_INTERNAL_H */
diff --git a/hw/ide/isa.c b/hw/ide/isa.c
index b084162..40213d6 100644
--- a/hw/ide/isa.c
+++ b/hw/ide/isa.c
@@ -22,13 +22,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/isa/isa.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/internal.h>
+#include "hw/ide/internal.h"
 
 /***********************************************************/
 /* ISA IDE definitions */
@@ -74,7 +75,8 @@
     isa_init_irq(isadev, &s->irq, s->isairq);
     ide_init2(&s->bus, s->irq);
     vmstate_register(dev, 0, &vmstate_ide_isa, s);
-};
+    ide_register_restart_cb(&s->bus);
+}
 
 ISADevice *isa_ide_init(ISABus *bus, int iobase, int iobase2, int isairq,
                         DriveInfo *hd0, DriveInfo *hd1)
@@ -88,9 +90,7 @@
     qdev_prop_set_uint32(dev, "iobase",  iobase);
     qdev_prop_set_uint32(dev, "iobase2", iobase2);
     qdev_prop_set_uint32(dev, "irq",     isairq);
-    if (qdev_init(dev) < 0) {
-        return NULL;
-    }
+    qdev_init_nofail(dev);
 
     s = ISA_IDE(dev);
     if (hd0) {
diff --git a/hw/ide/macio.c b/hw/ide/macio.c
index f6074f2..76f97c2 100644
--- a/hw/ide/macio.c
+++ b/hw/ide/macio.c
@@ -22,13 +22,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/ppc/mac_dbdma.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/internal.h>
+#include "hw/ide/internal.h"
 
 /* debug MACIO */
 // #define DEBUG_MACIO
@@ -51,18 +52,199 @@
 
 #define MACIO_PAGE_SIZE 4096
 
+/*
+ * Unaligned DMA read/write access functions required for OS X/Darwin which
+ * don't perform DMA transactions on sector boundaries. These functions are
+ * modelled on bdrv_co_preadv()/bdrv_co_pwritev() and so should be easy to
+ * remove if the unaligned block APIs are ever exposed.
+ */
+
+static void pmac_dma_read(BlockBackend *blk,
+                          int64_t offset, unsigned int bytes,
+                          void (*cb)(void *opaque, int ret), void *opaque)
+{
+    DBDMA_io *io = opaque;
+    MACIOIDEState *m = io->opaque;
+    IDEState *s = idebus_active_if(&m->bus);
+    dma_addr_t dma_addr;
+    int64_t sector_num;
+    int nsector;
+    uint64_t align = BDRV_SECTOR_SIZE;
+    size_t head_bytes, tail_bytes;
+
+    qemu_iovec_destroy(&io->iov);
+    qemu_iovec_init(&io->iov, io->len / MACIO_PAGE_SIZE + 1);
+
+    sector_num = (offset >> 9);
+    nsector = (io->len >> 9);
+
+    MACIO_DPRINTF("--- DMA read transfer (0x%" HWADDR_PRIx ",0x%x): "
+                  "sector_num: %" PRId64 ", nsector: %d\n", io->addr, io->len,
+                  sector_num, nsector);
+
+    dma_addr = io->addr;
+    io->dir = DMA_DIRECTION_FROM_DEVICE;
+    io->dma_len = io->len;
+    io->dma_mem = dma_memory_map(&address_space_memory, dma_addr, &io->dma_len,
+                                 io->dir);
+
+    if (offset & (align - 1)) {
+        head_bytes = offset & (align - 1);
+
+        MACIO_DPRINTF("--- DMA unaligned head: sector %" PRId64 ", "
+                      "discarding %zu bytes\n", sector_num, head_bytes);
+
+        qemu_iovec_add(&io->iov, &io->head_remainder, head_bytes);
+
+        bytes += offset & (align - 1);
+        offset = offset & ~(align - 1);
+    }
+
+    qemu_iovec_add(&io->iov, io->dma_mem, io->len);
+
+    if ((offset + bytes) & (align - 1)) {
+        tail_bytes = (offset + bytes) & (align - 1);
+
+        MACIO_DPRINTF("--- DMA unaligned tail: sector %" PRId64 ", "
+                      "discarding bytes %zu\n", sector_num, tail_bytes);
+
+        qemu_iovec_add(&io->iov, &io->tail_remainder, align - tail_bytes);
+        bytes = ROUND_UP(bytes, align);
+    }
+
+    s->io_buffer_size -= io->len;
+    s->io_buffer_index += io->len;
+
+    io->len = 0;
+
+    MACIO_DPRINTF("--- Block read transfer - sector_num: %" PRIx64 "  "
+                  "nsector: %x\n", (offset >> 9), (bytes >> 9));
+
+    s->bus->dma->aiocb = blk_aio_preadv(blk, offset, &io->iov, 0, cb, io);
+}
+
+static void pmac_dma_write(BlockBackend *blk,
+                         int64_t offset, int bytes,
+                         void (*cb)(void *opaque, int ret), void *opaque)
+{
+    DBDMA_io *io = opaque;
+    MACIOIDEState *m = io->opaque;
+    IDEState *s = idebus_active_if(&m->bus);
+    dma_addr_t dma_addr;
+    int64_t sector_num;
+    int nsector;
+    uint64_t align = BDRV_SECTOR_SIZE;
+    size_t head_bytes, tail_bytes;
+    bool unaligned_head = false, unaligned_tail = false;
+
+    qemu_iovec_destroy(&io->iov);
+    qemu_iovec_init(&io->iov, io->len / MACIO_PAGE_SIZE + 1);
+
+    sector_num = (offset >> 9);
+    nsector = (io->len >> 9);
+
+    MACIO_DPRINTF("--- DMA write transfer (0x%" HWADDR_PRIx ",0x%x): "
+                  "sector_num: %" PRId64 ", nsector: %d\n", io->addr, io->len,
+                  sector_num, nsector);
+
+    dma_addr = io->addr;
+    io->dir = DMA_DIRECTION_TO_DEVICE;
+    io->dma_len = io->len;
+    io->dma_mem = dma_memory_map(&address_space_memory, dma_addr, &io->dma_len,
+                                 io->dir);
+
+    if (offset & (align - 1)) {
+        head_bytes = offset & (align - 1);
+        sector_num = ((offset & ~(align - 1)) >> 9);
+
+        MACIO_DPRINTF("--- DMA unaligned head: pre-reading head sector %"
+                      PRId64 "\n", sector_num);
+
+        blk_pread(s->blk, (sector_num << 9), &io->head_remainder, align);
+
+        qemu_iovec_add(&io->iov, &io->head_remainder, head_bytes);
+        qemu_iovec_add(&io->iov, io->dma_mem, io->len);
+
+        bytes += offset & (align - 1);
+        offset = offset & ~(align - 1);
+
+        unaligned_head = true;
+    }
+
+    if ((offset + bytes) & (align - 1)) {
+        tail_bytes = (offset + bytes) & (align - 1);
+        sector_num = (((offset + bytes) & ~(align - 1)) >> 9);
+
+        MACIO_DPRINTF("--- DMA unaligned tail: pre-reading tail sector %"
+                      PRId64 "\n", sector_num);
+
+        blk_pread(s->blk, (sector_num << 9), &io->tail_remainder, align);
+
+        if (!unaligned_head) {
+            qemu_iovec_add(&io->iov, io->dma_mem, io->len);
+        }
+
+        qemu_iovec_add(&io->iov, &io->tail_remainder + tail_bytes,
+                       align - tail_bytes);
+
+        bytes = ROUND_UP(bytes, align);
+
+        unaligned_tail = true;
+    }
+
+    if (!unaligned_head && !unaligned_tail) {
+        qemu_iovec_add(&io->iov, io->dma_mem, io->len);
+    }
+
+    s->io_buffer_size -= io->len;
+    s->io_buffer_index += io->len;
+
+    io->len = 0;
+
+    MACIO_DPRINTF("--- Block write transfer - sector_num: %" PRIx64 "  "
+                  "nsector: %x\n", (offset >> 9), (bytes >> 9));
+
+    s->bus->dma->aiocb = blk_aio_pwritev(blk, offset, &io->iov, 0, cb, io);
+}
+
+static void pmac_dma_trim(BlockBackend *blk,
+                        int64_t offset, int bytes,
+                        void (*cb)(void *opaque, int ret), void *opaque)
+{
+    DBDMA_io *io = opaque;
+    MACIOIDEState *m = io->opaque;
+    IDEState *s = idebus_active_if(&m->bus);
+    dma_addr_t dma_addr;
+
+    qemu_iovec_destroy(&io->iov);
+    qemu_iovec_init(&io->iov, io->len / MACIO_PAGE_SIZE + 1);
+
+    dma_addr = io->addr;
+    io->dir = DMA_DIRECTION_TO_DEVICE;
+    io->dma_len = io->len;
+    io->dma_mem = dma_memory_map(&address_space_memory, dma_addr, &io->dma_len,
+                                 io->dir);
+
+    qemu_iovec_add(&io->iov, io->dma_mem, io->len);
+    s->io_buffer_size -= io->len;
+    s->io_buffer_index += io->len;
+    io->len = 0;
+
+    s->bus->dma->aiocb = ide_issue_trim(offset, &io->iov, cb, io, blk);
+}
+
 static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
 {
     DBDMA_io *io = opaque;
     MACIOIDEState *m = io->opaque;
     IDEState *s = idebus_active_if(&m->bus);
-    int unaligned;
+    int64_t offset;
+
+    MACIO_DPRINTF("pmac_ide_atapi_transfer_cb\n");
 
     if (ret < 0) {
-        m->aiocb = NULL;
-        qemu_sglist_destroy(&s->sg);
+        MACIO_DPRINTF("DMA error: %d\n", ret);
         ide_atapi_io_error(s, ret);
-        io->remainder_len = 0;
         goto done;
     }
 
@@ -74,104 +256,46 @@
         return;
     }
 
-    MACIO_DPRINTF("io_buffer_size = %#x\n", s->io_buffer_size);
-
-    if (s->io_buffer_size > 0) {
-        m->aiocb = NULL;
-        qemu_sglist_destroy(&s->sg);
-
-        s->packet_transfer_size -= s->io_buffer_size;
-
-        s->io_buffer_index += s->io_buffer_size;
-        s->lba += s->io_buffer_index >> 11;
-        s->io_buffer_index &= 0x7ff;
-    }
-
-    s->io_buffer_size = MIN(io->len, s->packet_transfer_size);
-
-    MACIO_DPRINTF("remainder: %d io->len: %d size: %d\n", io->remainder_len,
-                  io->len, s->packet_transfer_size);
-    if (io->remainder_len && io->len) {
-        /* guest wants the rest of its previous transfer */
-        int remainder_len = MIN(io->remainder_len, io->len);
-
-        MACIO_DPRINTF("copying remainder %d bytes\n", remainder_len);
-
-        cpu_physical_memory_write(io->addr, io->remainder + 0x200 -
-                                  remainder_len, remainder_len);
-
-        io->addr += remainder_len;
-        io->len -= remainder_len;
-        s->io_buffer_size = remainder_len;
-        io->remainder_len -= remainder_len;
-        /* treat remainder as individual transfer, start again */
-        qemu_sglist_init(&s->sg, DEVICE(m), io->len / MACIO_PAGE_SIZE + 1,
-                         &address_space_memory);
-        pmac_ide_atapi_transfer_cb(opaque, 0);
-        return;
-    }
-
-    if (!s->packet_transfer_size) {
-        MACIO_DPRINTF("end of transfer\n");
+    if (s->io_buffer_size <= 0) {
+        MACIO_DPRINTF("End of IDE transfer\n");
         ide_atapi_cmd_ok(s);
         m->dma_active = false;
-    }
-
-    if (io->len == 0) {
-        MACIO_DPRINTF("end of DMA\n");
         goto done;
     }
 
-    /* launch next transfer */
-
-    /* handle unaligned accesses first, get them over with and only do the
-       remaining bulk transfer using our async DMA helpers */
-    unaligned = io->len & 0x1ff;
-    if (unaligned) {
-        int sector_num = (s->lba << 2) + (s->io_buffer_index >> 9);
-        int nsector = io->len >> 9;
-
-        MACIO_DPRINTF("precopying unaligned %d bytes to %#" HWADDR_PRIx "\n",
-                      unaligned, io->addr + io->len - unaligned);
-
-        blk_read(s->blk, sector_num + nsector, io->remainder, 1);
-        cpu_physical_memory_write(io->addr + io->len - unaligned,
-                                  io->remainder, unaligned);
-
-        io->len -= unaligned;
+    if (io->len == 0) {
+        MACIO_DPRINTF("End of DMA transfer\n");
+        goto done;
     }
 
-    MACIO_DPRINTF("io->len = %#x\n", io->len);
-
-    qemu_sglist_init(&s->sg, DEVICE(m), io->len / MACIO_PAGE_SIZE + 1,
-                     &address_space_memory);
-    qemu_sglist_add(&s->sg, io->addr, io->len);
-    io->addr += s->io_buffer_size;
-    io->remainder_len = MIN(s->packet_transfer_size - s->io_buffer_size,
-                            (0x200 - unaligned) & 0x1ff);
-    MACIO_DPRINTF("set remainder to: %d\n", io->remainder_len);
-
-    /* We would read no data from the block layer, thus not get a callback.
-       Just fake completion manually. */
-    if (!io->len) {
-        pmac_ide_atapi_transfer_cb(opaque, 0);
-        return;
+    if (s->lba == -1) {
+        /* Non-block ATAPI transfer - just copy to RAM */
+        s->io_buffer_size = MIN(s->io_buffer_size, io->len);
+        dma_memory_write(&address_space_memory, io->addr, s->io_buffer,
+                         s->io_buffer_size);
+        io->len = 0;
+        ide_atapi_cmd_ok(s);
+        m->dma_active = false;
+        goto done;
     }
 
-    io->len = 0;
+    /* Calculate current offset */
+    offset = ((int64_t)s->lba << 11) + s->io_buffer_index;
 
-    MACIO_DPRINTF("sector_num=%d size=%d, cmd_cmd=%d\n",
-                  (s->lba << 2) + (s->io_buffer_index >> 9),
-                  s->packet_transfer_size, s->dma_cmd);
-
-    m->aiocb = dma_blk_read(s->blk, &s->sg,
-                            (int64_t)(s->lba << 2) + (s->io_buffer_index >> 9),
-                            pmac_ide_atapi_transfer_cb, io);
+    pmac_dma_read(s->blk, offset, io->len, pmac_ide_atapi_transfer_cb, io);
     return;
 
 done:
-    MACIO_DPRINTF("done DMA\n");
-    block_acct_done(blk_get_stats(s->blk), &s->acct);
+    dma_memory_unmap(&address_space_memory, io->dma_mem, io->dma_len,
+                     io->dir, io->dma_len);
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->blk), &s->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->blk), &s->acct);
+    }
+
+    ide_set_inactive(s, false);
     io->dma_end(opaque);
 }
 
@@ -180,24 +304,16 @@
     DBDMA_io *io = opaque;
     MACIOIDEState *m = io->opaque;
     IDEState *s = idebus_active_if(&m->bus);
-    int n = 0;
-    int64_t sector_num;
-    int unaligned;
+    int64_t offset;
+
+    MACIO_DPRINTF("pmac_ide_transfer_cb\n");
 
     if (ret < 0) {
-        MACIO_DPRINTF("DMA error\n");
-        m->aiocb = NULL;
-        qemu_sglist_destroy(&s->sg);
+        MACIO_DPRINTF("DMA error: %d\n", ret);
         ide_dma_error(s);
-        io->remainder_len = 0;
         goto done;
     }
 
-    if (--io->requests) {
-        /* More requests still in flight */
-        return;
-    }
-
     if (!m->dma_active) {
         MACIO_DPRINTF("waiting for data (%#x - %#x - %x)\n",
                       s->nsector, io->len, s->status);
@@ -206,155 +322,52 @@
         return;
     }
 
-    sector_num = ide_get_sector(s);
-    MACIO_DPRINTF("io_buffer_size = %#x\n", s->io_buffer_size);
-    if (s->io_buffer_size > 0) {
-        m->aiocb = NULL;
-        qemu_sglist_destroy(&s->sg);
-        n = (s->io_buffer_size + 0x1ff) >> 9;
-        sector_num += n;
-        ide_set_sector(s, sector_num);
-        s->nsector -= n;
-    }
-
-    if (io->finish_remain_read) {
-        /* Finish a stale read from the last iteration */
-        io->finish_remain_read = false;
-        cpu_physical_memory_write(io->finish_addr, io->remainder,
-                                  io->finish_len);
-    }
-
-    MACIO_DPRINTF("remainder: %d io->len: %d nsector: %d "
-                  "sector_num: %" PRId64 "\n",
-                  io->remainder_len, io->len, s->nsector, sector_num);
-    if (io->remainder_len && io->len) {
-        /* guest wants the rest of its previous transfer */
-        int remainder_len = MIN(io->remainder_len, io->len);
-        uint8_t *p = &io->remainder[0x200 - remainder_len];
-
-        MACIO_DPRINTF("copying remainder %d bytes at %#" HWADDR_PRIx "\n",
-                      remainder_len, io->addr);
-
-        switch (s->dma_cmd) {
-        case IDE_DMA_READ:
-            cpu_physical_memory_write(io->addr, p, remainder_len);
-            break;
-        case IDE_DMA_WRITE:
-            cpu_physical_memory_read(io->addr, p, remainder_len);
-            break;
-        case IDE_DMA_TRIM:
-            break;
-        }
-        io->addr += remainder_len;
-        io->len -= remainder_len;
-        io->remainder_len -= remainder_len;
-
-        if (s->dma_cmd == IDE_DMA_WRITE && !io->remainder_len) {
-            io->requests++;
-            qemu_iovec_reset(&io->iov);
-            qemu_iovec_add(&io->iov, io->remainder, 0x200);
-
-            m->aiocb = blk_aio_writev(s->blk, sector_num - 1, &io->iov, 1,
-                                      pmac_ide_transfer_cb, io);
-        }
-    }
-
-    if (s->nsector == 0 && !io->remainder_len) {
-        MACIO_DPRINTF("end of transfer\n");
+    if (s->io_buffer_size <= 0) {
+        MACIO_DPRINTF("End of IDE transfer\n");
         s->status = READY_STAT | SEEK_STAT;
         ide_set_irq(s->bus);
         m->dma_active = false;
-    }
-
-    if (io->len == 0) {
-        MACIO_DPRINTF("end of DMA\n");
         goto done;
     }
 
-    /* launch next transfer */
-
-    s->io_buffer_index = 0;
-    s->io_buffer_size = MIN(io->len, s->nsector * 512);
-
-    /* handle unaligned accesses first, get them over with and only do the
-       remaining bulk transfer using our async DMA helpers */
-    unaligned = io->len & 0x1ff;
-    if (unaligned) {
-        int nsector = io->len >> 9;
-
-        MACIO_DPRINTF("precopying unaligned %d bytes to %#" HWADDR_PRIx "\n",
-                      unaligned, io->addr + io->len - unaligned);
-
-        switch (s->dma_cmd) {
-        case IDE_DMA_READ:
-            io->requests++;
-            io->finish_addr = io->addr + io->len - unaligned;
-            io->finish_len = unaligned;
-            io->finish_remain_read = true;
-            qemu_iovec_reset(&io->iov);
-            qemu_iovec_add(&io->iov, io->remainder, 0x200);
-
-            m->aiocb = blk_aio_readv(s->blk, sector_num + nsector, &io->iov, 1,
-                                     pmac_ide_transfer_cb, io);
-            break;
-        case IDE_DMA_WRITE:
-            /* cache the contents in our io struct */
-            cpu_physical_memory_read(io->addr + io->len - unaligned,
-                                     io->remainder + io->remainder_len,
-                                     unaligned);
-            break;
-        case IDE_DMA_TRIM:
-            break;
-        }
+    if (io->len == 0) {
+        MACIO_DPRINTF("End of DMA transfer\n");
+        goto done;
     }
 
-    MACIO_DPRINTF("io->len = %#x\n", io->len);
-
-    qemu_sglist_init(&s->sg, DEVICE(m), io->len / MACIO_PAGE_SIZE + 1,
-                     &address_space_memory);
-    qemu_sglist_add(&s->sg, io->addr, io->len);
-    io->addr += io->len + unaligned;
-    io->remainder_len = (0x200 - unaligned) & 0x1ff;
-    MACIO_DPRINTF("set remainder to: %d\n", io->remainder_len);
-
-    /* Only subsector reads happening */
-    if (!io->len) {
-        if (!io->requests) {
-            io->requests++;
-            pmac_ide_transfer_cb(opaque, ret);
-        }
-        return;
-    }
-
-    io->len = 0;
-
-    MACIO_DPRINTF("sector_num=%" PRId64 " n=%d, nsector=%d, cmd_cmd=%d\n",
-                  sector_num, n, s->nsector, s->dma_cmd);
+    /* Calculate number of sectors */
+    offset = (ide_get_sector(s) << 9) + s->io_buffer_index;
 
     switch (s->dma_cmd) {
     case IDE_DMA_READ:
-        m->aiocb = dma_blk_read(s->blk, &s->sg, sector_num,
-                                pmac_ide_transfer_cb, io);
+        pmac_dma_read(s->blk, offset, io->len, pmac_ide_transfer_cb, io);
         break;
     case IDE_DMA_WRITE:
-        m->aiocb = dma_blk_write(s->blk, &s->sg, sector_num,
-                                 pmac_ide_transfer_cb, io);
+        pmac_dma_write(s->blk, offset, io->len, pmac_ide_transfer_cb, io);
         break;
     case IDE_DMA_TRIM:
-        m->aiocb = dma_blk_io(s->blk, &s->sg, sector_num,
-                              ide_issue_trim, pmac_ide_transfer_cb, io,
-                              DMA_DIRECTION_TO_DEVICE);
+        pmac_dma_trim(s->blk, offset, io->len, pmac_ide_transfer_cb, io);
         break;
+    default:
+        abort();
     }
 
-    io->requests++;
     return;
 
 done:
+    dma_memory_unmap(&address_space_memory, io->dma_mem, io->dma_len,
+                     io->dir, io->dma_len);
+
     if (s->dma_cmd == IDE_DMA_READ || s->dma_cmd == IDE_DMA_WRITE) {
-        block_acct_done(blk_get_stats(s->blk), &s->acct);
+        if (ret < 0) {
+            block_acct_failed(blk_get_stats(s->blk), &s->acct);
+        } else {
+            block_acct_done(blk_get_stats(s->blk), &s->acct);
+        }
     }
-    io->dma_end(io);
+
+    ide_set_inactive(s, false);
+    io->dma_end(opaque);
 }
 
 static void pmac_ide_transfer(DBDMA_io *io)
@@ -364,31 +377,10 @@
 
     MACIO_DPRINTF("\n");
 
-    s->io_buffer_size = 0;
     if (s->drive_kind == IDE_CD) {
-
-        /* Handle non-block ATAPI DMA transfers */
-        if (s->lba == -1) {
-            s->io_buffer_size = MIN(io->len, s->packet_transfer_size);
-            block_acct_start(blk_get_stats(s->blk), &s->acct, s->io_buffer_size,
-                             BLOCK_ACCT_READ);
-            MACIO_DPRINTF("non-block ATAPI DMA transfer size: %d\n",
-                          s->io_buffer_size);
-
-            /* Copy ATAPI buffer directly to RAM and finish */
-            cpu_physical_memory_write(io->addr, s->io_buffer,
-                                      s->io_buffer_size);
-            ide_atapi_cmd_ok(s);
-            m->dma_active = false;
-
-            MACIO_DPRINTF("end of non-block ATAPI DMA transfer\n");
-            block_acct_done(blk_get_stats(s->blk), &s->acct);
-            io->dma_end(io);
-            return;
-        }
-
         block_acct_start(blk_get_stats(s->blk), &s->acct, io->len,
                          BLOCK_ACCT_READ);
+
         pmac_ide_atapi_transfer_cb(io, 0);
         return;
     }
@@ -406,16 +398,16 @@
         break;
     }
 
-    io->requests++;
     pmac_ide_transfer_cb(io, 0);
 }
 
 static void pmac_ide_flush(DBDMA_io *io)
 {
     MACIOIDEState *m = io->opaque;
+    IDEState *s = idebus_active_if(&m->bus);
 
-    if (m->aiocb) {
-        blk_drain_all();
+    if (s->bus->dma->aiocb) {
+        blk_drain(s->blk);
     }
 }
 
@@ -532,11 +524,12 @@
 
 static const VMStateDescription vmstate_pmac = {
     .name = "ide",
-    .version_id = 3,
+    .version_id = 4,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
         VMSTATE_IDE_BUS(bus, MACIOIDEState),
         VMSTATE_IDE_DRIVES(bus.ifs, MACIOIDEState),
+        VMSTATE_BOOL(dma_active, MACIOIDEState),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -553,21 +546,29 @@
     return 0;
 }
 
-static int32_t ide_nop_int32(IDEDMA *dma, int x)
+static int32_t ide_nop_int32(IDEDMA *dma, int32_t l)
 {
     return 0;
 }
 
-static void ide_nop_restart(void *opaque, int x, RunState y)
-{
-}
-
 static void ide_dbdma_start(IDEDMA *dma, IDEState *s,
                             BlockCompletionFunc *cb)
 {
     MACIOIDEState *m = container_of(dma, MACIOIDEState, dma);
 
-    MACIO_DPRINTF("\n");
+    s->io_buffer_index = 0;
+    if (s->drive_kind == IDE_CD) {
+        s->io_buffer_size = s->packet_transfer_size;
+    } else {
+        s->io_buffer_size = s->nsector * BDRV_SECTOR_SIZE;
+    }
+
+    MACIO_DPRINTF("\n\n------------ IDE transfer\n");
+    MACIO_DPRINTF("buffer_size: %x   buffer_index: %x\n",
+                  s->io_buffer_size, s->io_buffer_index);
+    MACIO_DPRINTF("lba: %x    size: %x\n", s->lba, s->io_buffer_size);
+    MACIO_DPRINTF("-------------------------\n");
+
     m->dma_active = true;
     DBDMA_kick(m->dbdma);
 }
@@ -576,8 +577,6 @@
     .start_dma      = ide_dbdma_start,
     .prepare_buf    = ide_nop_int32,
     .rw_buf         = ide_nop_int,
-    .set_unit       = ide_nop_int,
-    .restart_cb     = ide_nop_restart,
 };
 
 static void macio_ide_realizefn(DeviceState *dev, Error **errp)
@@ -610,6 +609,7 @@
     dc->realize = macio_ide_realizefn;
     dc->reset = macio_ide_reset;
     dc->vmsd = &vmstate_pmac;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo macio_ide_type_info = {
diff --git a/hw/ide/microdrive.c b/hw/ide/microdrive.c
index 6639dd4..e3fd30e 100644
--- a/hw/ide/microdrive.c
+++ b/hw/ide/microdrive.c
@@ -22,13 +22,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/pcmcia.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/pcmcia.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/internal.h>
+#include "hw/ide/internal.h"
 
 #define TYPE_MICRODRIVE "microdrive"
 #define MICRODRIVE(obj) OBJECT_CHECK(MicroDriveState, (obj), TYPE_MICRODRIVE)
diff --git a/hw/ide/mmio.c b/hw/ide/mmio.c
index b6ce62a..6f12f45 100644
--- a/hw/ide/mmio.c
+++ b/hw/ide/mmio.c
@@ -22,12 +22,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/internal.h>
+#include "hw/ide/internal.h"
 
 /***********************************************************/
 /* MMIO based ide port
diff --git a/hw/ide/pci.c b/hw/ide/pci.c
index bee5ad3..3cfb510 100644
--- a/hw/ide/pci.c
+++ b/hw/ide/pci.c
@@ -22,14 +22,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
+#include "hw/isa/isa.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/dma.h"
 #include "qemu/error-report.h"
-#include <hw/ide/pci.h>
+#include "hw/ide/pci.h"
 
 #define BMDMA_PAGE_SIZE 4096
 
@@ -42,13 +43,10 @@
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
 
-    bm->unit = s->unit;
     bm->dma_cb = dma_cb;
     bm->cur_prd_last = 0;
     bm->cur_prd_addr = 0;
     bm->cur_prd_len = 0;
-    bm->sector_num = ide_get_sector(s);
-    bm->nsector = s->nsector;
 
     if (bm->status & BM_STATUS_DMAING) {
         bm->dma_cb(bmdma_active_if(bm), 0);
@@ -56,10 +54,14 @@
 }
 
 /**
- * Return the number of bytes successfully prepared.
- * -1 on error.
+ * Prepare an sglist based on available PRDs.
+ * @limit: How many bytes to prepare total.
+ *
+ * Returns the number of bytes prepared, -1 on error.
+ * IDEState.io_buffer_size will contain the number of bytes described
+ * by the PRDs, whether or not we added them to the sglist.
  */
-static int32_t bmdma_prepare_buf(IDEDMA *dma, int is_write)
+static int32_t bmdma_prepare_buf(IDEDMA *dma, int32_t limit)
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
     IDEState *s = bmdma_active_if(bm);
@@ -78,7 +80,7 @@
             /* end of table (with a fail safe of one page) */
             if (bm->cur_prd_last ||
                 (bm->cur_addr - bm->addr) >= BMDMA_PAGE_SIZE) {
-                return s->io_buffer_size;
+                return s->sg.size;
             }
             pci_dma_read(pci_dev, bm->cur_addr, &prd, 8);
             bm->cur_addr += 8;
@@ -93,15 +95,15 @@
         }
         l = bm->cur_prd_len;
         if (l > 0) {
-            qemu_sglist_add(&s->sg, bm->cur_prd_addr, l);
+            uint64_t sg_len;
 
-            /* Note: We limit the max transfer to be 2GiB.
-             * This should accommodate the largest ATA transaction
-             * for LBA48 (65,536 sectors) and 32K sector sizes. */
-            if (s->sg.size > INT32_MAX) {
-                error_report("IDE: sglist describes more than 2GiB.\n");
-                break;
+            /* Don't add extra bytes to the SGList; consume any remaining
+             * PRDs from the guest, but ignore them. */
+            sg_len = MIN(limit - s->sg.size, bm->cur_prd_len);
+            if (sg_len) {
+                qemu_sglist_add(&s->sg, bm->cur_prd_addr, sg_len);
             }
+
             bm->cur_prd_addr += l;
             bm->cur_prd_len -= l;
             s->io_buffer_size += l;
@@ -163,20 +165,11 @@
     return 1;
 }
 
-static int bmdma_set_unit(IDEDMA *dma, int unit)
-{
-    BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
-    bm->unit = unit;
-
-    return 0;
-}
-
 static void bmdma_set_inactive(IDEDMA *dma, bool more)
 {
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
 
     bm->dma_cb = NULL;
-    bm->unit = -1;
     if (more) {
         bm->status |= BM_STATUS_DMAING;
     } else {
@@ -184,72 +177,11 @@
     }
 }
 
-static void bmdma_restart_dma(BMDMAState *bm, enum ide_dma_cmd dma_cmd)
+static void bmdma_restart_dma(IDEDMA *dma)
 {
-    IDEState *s = bmdma_active_if(bm);
-
-    ide_set_sector(s, bm->sector_num);
-    s->io_buffer_index = 0;
-    s->io_buffer_size = 0;
-    s->nsector = bm->nsector;
-    s->dma_cmd = dma_cmd;
-    bm->cur_addr = bm->addr;
-    bm->dma_cb = ide_dma_cb;
-    bmdma_start_dma(&bm->dma, s, bm->dma_cb);
-}
-
-/* TODO This should be common IDE code */
-static void bmdma_restart_bh(void *opaque)
-{
-    BMDMAState *bm = opaque;
-    IDEBus *bus = bm->bus;
-    bool is_read;
-    int error_status;
-
-    qemu_bh_delete(bm->bh);
-    bm->bh = NULL;
-
-    if (bm->unit == (uint8_t) -1) {
-        return;
-    }
-
-    is_read = (bus->error_status & IDE_RETRY_READ) != 0;
-
-    /* The error status must be cleared before resubmitting the request: The
-     * request may fail again, and this case can only be distinguished if the
-     * called function can set a new error status. */
-    error_status = bus->error_status;
-    bus->error_status = 0;
-
-    if (error_status & IDE_RETRY_DMA) {
-        if (error_status & IDE_RETRY_TRIM) {
-            bmdma_restart_dma(bm, IDE_DMA_TRIM);
-        } else {
-            bmdma_restart_dma(bm, is_read ? IDE_DMA_READ : IDE_DMA_WRITE);
-        }
-    } else if (error_status & IDE_RETRY_PIO) {
-        if (is_read) {
-            ide_sector_read(bmdma_active_if(bm));
-        } else {
-            ide_sector_write(bmdma_active_if(bm));
-        }
-    } else if (error_status & IDE_RETRY_FLUSH) {
-        ide_flush_cache(bmdma_active_if(bm));
-    }
-}
-
-static void bmdma_restart_cb(void *opaque, int running, RunState state)
-{
-    IDEDMA *dma = opaque;
     BMDMAState *bm = DO_UPCAST(BMDMAState, dma, dma);
 
-    if (!running)
-        return;
-
-    if (!bm->bh) {
-        bm->bh = qemu_bh_new(bmdma_restart_bh, &bm->dma);
-        qemu_bh_schedule(bm->bh);
-    }
+    bm->cur_addr = bm->addr;
 }
 
 static void bmdma_cancel(BMDMAState *bm)
@@ -275,8 +207,6 @@
     bm->cur_prd_last = 0;
     bm->cur_prd_addr = 0;
     bm->cur_prd_len = 0;
-    bm->sector_num = 0;
-    bm->nsector = 0;
 }
 
 static void bmdma_irq(void *opaque, int n, int level)
@@ -304,22 +234,7 @@
     /* Ignore writes to SSBM if it keeps the old value */
     if ((val & BM_CMD_START) != (bm->cmd & BM_CMD_START)) {
         if (!(val & BM_CMD_START)) {
-            /*
-             * We can't cancel Scatter Gather DMA in the middle of the
-             * operation or a partial (not full) DMA transfer would reach
-             * the storage so we wait for completion instead (we beahve
-             * like if the DMA was completed by the time the guest trying
-             * to cancel dma with bmdma_cmd_writeb with BM_CMD_START not
-             * set).
-             *
-             * In the future we'll be able to safely cancel the I/O if the
-             * whole DMA operation will be submitted to disk with a single
-             * aio operation with preadv/pwritev.
-             */
-            if (bm->bus->dma->aiocb) {
-                blk_drain_all();
-                assert(bm->bus->dma->aiocb == NULL);
-            }
+            ide_cancel_dma_sync(idebus_active_if(bm->bus));
             bm->status &= ~BM_STATUS_DMAING;
         } else {
             bm->cur_addr = bm->addr;
@@ -393,6 +308,13 @@
     BMDMAState *bm = opaque;
     uint8_t abused_bits = BM_MIGRATION_COMPAT_STATUS_BITS;
 
+    if (!(bm->status & BM_STATUS_DMAING) && bm->dma_cb) {
+        bm->bus->error_status =
+            ide_dma_cmd_to_retry(bmdma_active_if(bm)->dma_cmd);
+    }
+    bm->migration_retry_unit = bm->bus->retry_unit;
+    bm->migration_retry_sector_num = bm->bus->retry_sector_num;
+    bm->migration_retry_nsector = bm->bus->retry_nsector;
     bm->migration_compat_status =
         (bm->status & ~abused_bits) | (bm->bus->error_status & abused_bits);
 }
@@ -409,6 +331,11 @@
         bm->status = bm->migration_compat_status & ~abused_bits;
         bm->bus->error_status |= bm->migration_compat_status & abused_bits;
     }
+    if (bm->bus->error_status) {
+        bm->bus->retry_sector_num = bm->migration_retry_sector_num;
+        bm->bus->retry_nsector = bm->migration_retry_nsector;
+        bm->bus->retry_unit = bm->migration_retry_unit;
+    }
 
     return 0;
 }
@@ -417,6 +344,7 @@
     .name = "ide bmdma_current",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = ide_bmdma_current_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(cur_addr, BMDMAState),
         VMSTATE_UINT32(cur_prd_last, BMDMAState),
@@ -430,6 +358,7 @@
     .name ="ide bmdma/status",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = ide_bmdma_status_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(status, BMDMAState),
         VMSTATE_END_OF_LIST()
@@ -445,21 +374,15 @@
         VMSTATE_UINT8(cmd, BMDMAState),
         VMSTATE_UINT8(migration_compat_status, BMDMAState),
         VMSTATE_UINT32(addr, BMDMAState),
-        VMSTATE_INT64(sector_num, BMDMAState),
-        VMSTATE_UINT32(nsector, BMDMAState),
-        VMSTATE_UINT8(unit, BMDMAState),
+        VMSTATE_INT64(migration_retry_sector_num, BMDMAState),
+        VMSTATE_UINT32(migration_retry_nsector, BMDMAState),
+        VMSTATE_UINT8(migration_retry_unit, BMDMAState),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_bmdma_current,
-            .needed = ide_bmdma_current_needed,
-        }, {
-            .vmsd = &vmstate_bmdma_status,
-            .needed = ide_bmdma_status_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_bmdma_current,
+        &vmstate_bmdma_status,
+        NULL
     }
 };
 
@@ -471,7 +394,7 @@
     for(i = 0; i < 2; i++) {
         /* current versions always store 0/1, but older version
            stored bigger values. We only need last bit */
-        d->bmdma[i].unit &= 1;
+        d->bmdma[i].migration_retry_unit &= 1;
         ide_bmdma_post_load(&d->bmdma[i], -1);
     }
 
@@ -512,16 +435,13 @@
     .start_dma = bmdma_start_dma,
     .prepare_buf = bmdma_prepare_buf,
     .rw_buf = bmdma_rw_buf,
-    .set_unit = bmdma_set_unit,
+    .restart_dma = bmdma_restart_dma,
     .set_inactive = bmdma_set_inactive,
-    .restart_cb = bmdma_restart_cb,
     .reset = bmdma_reset,
 };
 
 void bmdma_init(IDEBus *bus, BMDMAState *bm, PCIIDEState *d)
 {
-    qemu_irq *irq;
-
     if (bus->dma == &bm->dma) {
         return;
     }
@@ -529,8 +449,7 @@
     bm->dma.ops = &bmdma_ops;
     bus->dma = &bm->dma;
     bm->irq = bus->irq;
-    irq = qemu_allocate_irqs(bmdma_irq, bm, 1);
-    bus->irq = *irq;
+    bus->irq = qemu_allocate_irq(bmdma_irq, bm, 0);
     bm->pci_dev = d;
 }
 
diff --git a/hw/ide/pci.h b/hw/ide/pci.h
deleted file mode 100644
index 2e9314a..0000000
--- a/hw/ide/pci.h
+++ /dev/null
@@ -1,76 +0,0 @@
-#ifndef HW_IDE_PCI_H
-#define HW_IDE_PCI_H
-
-#include <hw/ide/internal.h>
-
-#define BM_STATUS_DMAING 0x01
-#define BM_STATUS_ERROR  0x02
-#define BM_STATUS_INT    0x04
-
-#define BM_CMD_START     0x01
-#define BM_CMD_READ      0x08
-
-typedef struct BMDMAState {
-    IDEDMA dma;
-    uint8_t cmd;
-    uint8_t status;
-    uint32_t addr;
-
-    IDEBus *bus;
-    /* current transfer state */
-    uint32_t cur_addr;
-    uint32_t cur_prd_last;
-    uint32_t cur_prd_addr;
-    uint32_t cur_prd_len;
-    uint8_t unit;
-    BlockCompletionFunc *dma_cb;
-    int64_t sector_num;
-    uint32_t nsector;
-    MemoryRegion addr_ioport;
-    MemoryRegion extra_io;
-    QEMUBH *bh;
-    qemu_irq irq;
-
-    /* Bit 0-2 and 7:   BM status register
-     * Bit 3-6:         bus->error_status */
-    uint8_t migration_compat_status;
-    struct PCIIDEState *pci_dev;
-} BMDMAState;
-
-typedef struct CMD646BAR {
-    MemoryRegion cmd;
-    MemoryRegion data;
-    IDEBus *bus;
-    struct PCIIDEState *pci_dev;
-} CMD646BAR;
-
-#define TYPE_PCI_IDE "pci-ide"
-#define PCI_IDE(obj) OBJECT_CHECK(PCIIDEState, (obj), TYPE_PCI_IDE)
-
-typedef struct PCIIDEState {
-    /*< private >*/
-    PCIDevice parent_obj;
-    /*< public >*/
-
-    IDEBus bus[2];
-    BMDMAState bmdma[2];
-    uint32_t secondary; /* used only for cmd646 */
-    MemoryRegion bmdma_bar;
-    CMD646BAR cmd646_bar[2]; /* used only for cmd646 */
-} PCIIDEState;
-
-
-static inline IDEState *bmdma_active_if(BMDMAState *bmdma)
-{
-    assert(bmdma->unit != (uint8_t)-1);
-    return bmdma->bus->ifs + bmdma->unit;
-}
-
-
-void bmdma_init(IDEBus *bus, BMDMAState *bm, PCIIDEState *d);
-void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val);
-extern MemoryRegionOps bmdma_addr_ioport_ops;
-void pci_ide_create_devs(PCIDevice *dev, DriveInfo **hd_table);
-
-extern const VMStateDescription vmstate_ide_pci;
-#endif
diff --git a/hw/ide/piix.c b/hw/ide/piix.c
index b0172fb..c190fca 100644
--- a/hw/ide/piix.c
+++ b/hw/ide/piix.c
@@ -23,15 +23,16 @@
  * THE SOFTWARE.
  */
 
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
+#include "hw/isa/isa.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/pci.h>
+#include "hw/ide/pci.h"
 
 static uint64_t bmdma_read(void *opaque, hwaddr addr, unsigned size)
 {
@@ -143,12 +144,11 @@
 
         bmdma_init(&d->bus[i], &d->bmdma[i], d);
         d->bmdma[i].bus = &d->bus[i];
-        qemu_add_vm_change_state_handler(d->bus[i].dma->ops->restart_cb,
-                                         &d->bmdma[i].dma);
+        ide_register_restart_cb(&d->bus[i]);
     }
 }
 
-static int pci_piix_ide_initfn(PCIDevice *dev)
+static void pci_piix_ide_realize(PCIDevice *dev, Error **errp)
 {
     PCIIDEState *d = PCI_IDE(dev);
     uint8_t *pci_conf = dev->config;
@@ -163,8 +163,6 @@
     vmstate_register(DEVICE(dev), 0, &vmstate_ide_pci, d);
 
     pci_piix_init_ports(d);
-
-    return 0;
 }
 
 int pci_piix3_xen_ide_unplug(DeviceState *dev)
@@ -172,6 +170,7 @@
     PCIIDEState *pci_ide;
     DriveInfo *di;
     int i;
+    IDEDevice *idedev;
 
     pci_ide = PCI_IDE(dev);
 
@@ -184,6 +183,13 @@
                 blk_detach_dev(blk, ds);
             }
             pci_ide->bus[di->bus].ifs[di->unit].blk = NULL;
+            if (!(i % 2)) {
+                idedev = pci_ide->bus[di->bus].master;
+            } else {
+                idedev = pci_ide->bus[di->bus].slave;
+            }
+            idedev->conf.blk = NULL;
+            monitor_remove_blk(blk);
             blk_unref(blk);
         }
     }
@@ -238,7 +244,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_piix_ide_initfn;
+    k->realize = pci_piix_ide_realize;
     k->exit = pci_piix_ide_exitfn;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82371SB_1;
@@ -253,22 +259,10 @@
     .class_init    = piix3_ide_class_init,
 };
 
-static void piix3_ide_xen_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = pci_piix_ide_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82371SB_1;
-    k->class_id = PCI_CLASS_STORAGE_IDE;
-    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-}
-
 static const TypeInfo piix3_ide_xen_info = {
     .name          = "piix3-ide-xen",
     .parent        = TYPE_PCI_IDE,
-    .class_init    = piix3_ide_xen_class_init,
+    .class_init    = piix3_ide_class_init,
 };
 
 static void piix4_ide_class_init(ObjectClass *klass, void *data)
@@ -276,7 +270,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_piix_ide_initfn;
+    k->realize = pci_piix_ide_realize;
     k->exit = pci_piix_ide_exitfn;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82371AB;
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index b4f096e..67c76bf 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -16,10 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <hw/hw.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
 #include "sysemu/dma.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
-#include <hw/ide/internal.h>
+#include "hw/ide/internal.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
@@ -118,7 +120,8 @@
 
     dev = qdev_create(&bus->qbus, drive->media_cd ? "ide-cd" : "ide-hd");
     qdev_prop_set_uint32(dev, "unit", unit);
-    qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(drive));
+    qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(drive),
+                        &error_fatal);
     qdev_init_nofail(dev);
     return DO_UPCAST(IDEDevice, qdev, dev);
 }
@@ -163,15 +166,21 @@
         return -1;
     }
 
+    blkconf_blocksizes(&dev->conf);
+    if (dev->conf.logical_block_size != 512) {
+        error_report("logical_block_size must be 512 for IDE");
+        return -1;
+    }
+
     blkconf_serial(&dev->conf, &dev->serial);
     if (kind != IDE_CD) {
-        blkconf_geometry(&dev->conf, &dev->chs_trans, 65536, 16, 255, &err);
+        blkconf_geometry(&dev->conf, &dev->chs_trans, 65535, 16, 255, &err);
         if (err) {
-            error_report("%s", error_get_pretty(err));
-            error_free(err);
+            error_report_err(err);
             return -1;
         }
     }
+    blkconf_apply_backend_options(&dev->conf);
 
     if (ide_init_drive(s, dev->conf.blk, kind,
                        dev->version, dev->serial, dev->model, dev->wwn,
@@ -193,22 +202,22 @@
     return 0;
 }
 
-static void ide_dev_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void ide_dev_get_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     IDEDevice *d = IDE_DEVICE(obj);
 
-    visit_type_int32(v, &d->conf.bootindex, name, errp);
+    visit_type_int32(v, name, &d->conf.bootindex, errp);
 }
 
-static void ide_dev_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void ide_dev_set_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     IDEDevice *d = IDE_DEVICE(obj);
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
@@ -225,9 +234,7 @@
                              d->unit ? "/disk@1" : "/disk@0");
     }
 out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
 static void ide_dev_instance_init(Object *obj)
@@ -257,6 +264,7 @@
 
 #define DEFINE_IDE_DEV_PROPERTIES()                     \
     DEFINE_BLOCK_PROPERTIES(IDEDrive, dev.conf),        \
+    DEFINE_BLOCK_ERROR_PROPERTIES(IDEDrive, dev.conf),  \
     DEFINE_PROP_STRING("ver",  IDEDrive, dev.version),  \
     DEFINE_PROP_UINT64("wwn",  IDEDrive, dev.wwn, 0),    \
     DEFINE_PROP_STRING("serial",  IDEDrive, dev.serial),\
diff --git a/hw/ide/via.c b/hw/ide/via.c
index 4d8089d..5b32ecb 100644
--- a/hw/ide/via.c
+++ b/hw/ide/via.c
@@ -23,15 +23,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <hw/hw.h>
-#include <hw/i386/pc.h>
-#include <hw/pci/pci.h>
-#include <hw/isa/isa.h>
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/i386/pc.h"
+#include "hw/pci/pci.h"
+#include "hw/isa/isa.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 
-#include <hw/ide/pci.h>
+#include "hw/ide/pci.h"
 
 static uint64_t bmdma_read(void *opaque, hwaddr addr,
                            unsigned size)
@@ -166,13 +167,12 @@
 
         bmdma_init(&d->bus[i], &d->bmdma[i], d);
         d->bmdma[i].bus = &d->bus[i];
-        qemu_add_vm_change_state_handler(d->bus[i].dma->ops->restart_cb,
-                                         &d->bmdma[i].dma);
+        ide_register_restart_cb(&d->bus[i]);
     }
 }
 
 /* via ide func */
-static int vt82c686b_ide_initfn(PCIDevice *dev)
+static void vt82c686b_ide_realize(PCIDevice *dev, Error **errp)
 {
     PCIIDEState *d = PCI_IDE(dev);
     uint8_t *pci_conf = dev->config;
@@ -187,8 +187,6 @@
     vmstate_register(DEVICE(dev), 0, &vmstate_ide_pci, d);
 
     vt82c686b_init_ports(d);
-
-    return 0;
 }
 
 static void vt82c686b_ide_exitfn(PCIDevice *dev)
@@ -215,7 +213,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = vt82c686b_ide_initfn;
+    k->realize = vt82c686b_ide_realize;
     k->exit = vt82c686b_ide_exitfn;
     k->vendor_id = PCI_VENDOR_ID_VIA;
     k->device_id = PCI_DEVICE_ID_VIA_IDE;
diff --git a/hw/input/Makefile.objs b/hw/input/Makefile.objs
index 978a9c4..4adb849 100644
--- a/hw/input/Makefile.objs
+++ b/hw/input/Makefile.objs
@@ -8,8 +8,13 @@
 common-obj-$(CONFIG_TSC2005) += tsc2005.o
 common-obj-$(CONFIG_VMMOUSE) += vmmouse.o
 
+common-obj-$(CONFIG_VIRTIO) += virtio-input.o
+common-obj-$(CONFIG_VIRTIO) += virtio-input-hid.o
+ifeq ($(CONFIG_LINUX),y)
+common-obj-$(CONFIG_VIRTIO) += virtio-input-host.o
+endif
+
 obj-$(CONFIG_MILKYMIST) += milkymist-softusb.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_keypad.o
 obj-$(CONFIG_TSC210X) += tsc210x.o
 obj-$(CONFIG_GOLDFISH) += goldfish_events.o
-obj-$(CONFIG_GOLDFISH) += goldfish_sensors.o
diff --git a/hw/input/adb.c b/hw/input/adb.c
index 34c8058..f0ad0d4 100644
--- a/hw/input/adb.c
+++ b/hw/input/adb.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/input/adb.h"
 #include "ui/console.h"
@@ -88,7 +89,7 @@
 }
 
 /* XXX: move that to cuda ? */
-int adb_poll(ADBBusState *s, uint8_t *obuf)
+int adb_poll(ADBBusState *s, uint8_t *obuf, uint16_t poll_mask)
 {
     ADBDevice *d;
     int olen, i;
@@ -99,13 +100,15 @@
         if (s->poll_index >= s->nb_devices)
             s->poll_index = 0;
         d = s->devices[s->poll_index];
-        buf[0] = ADB_READREG | (d->devaddr << 4);
-        olen = adb_request(s, obuf + 1, buf, 1);
-        /* if there is data, we poll again the same device */
-        if (olen > 0) {
-            obuf[0] = buf[0];
-            olen++;
-            break;
+        if ((1 << d->devaddr) & poll_mask) {
+            buf[0] = ADB_READREG | (d->devaddr << 4);
+            olen = adb_request(s, obuf + 1, buf, 1);
+            /* if there is data, we poll again the same device */
+            if (olen > 0) {
+                obuf[0] = buf[0];
+                olen++;
+                break;
+            }
         }
         s->poll_index++;
     }
@@ -118,6 +121,17 @@
     .instance_size = sizeof(ADBBusState),
 };
 
+static const VMStateDescription vmstate_adb_device = {
+    .name = "adb_device",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(devaddr, ADBDevice),
+        VMSTATE_INT32(handler, ADBDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void adb_device_realizefn(DeviceState *dev, Error **errp)
 {
     ADBDevice *d = ADB_DEVICE(dev);
@@ -301,9 +315,10 @@
 
 static const VMStateDescription vmstate_adb_kbd = {
     .name = "adb_kbd",
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
     .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(parent_obj, KBDState, 0, vmstate_adb_device, ADBDevice),
         VMSTATE_BUFFER(data, KBDState),
         VMSTATE_INT32(rptr, KBDState),
         VMSTATE_INT32(wptr, KBDState),
@@ -350,6 +365,7 @@
 
     akc->parent_realize = dc->realize;
     dc->realize = adb_kbd_realizefn;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 
     adc->devreq = adb_kbd_request;
     dc->reset = adb_kbd_reset;
@@ -515,9 +531,11 @@
 
 static const VMStateDescription vmstate_adb_mouse = {
     .name = "adb_mouse",
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
     .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(parent_obj, MouseState, 0, vmstate_adb_device,
+                       ADBDevice),
         VMSTATE_INT32(buttons_state, MouseState),
         VMSTATE_INT32(last_buttons_state, MouseState),
         VMSTATE_INT32(dx, MouseState),
@@ -552,6 +570,7 @@
 
     amc->parent_realize = dc->realize;
     dc->realize = adb_mouse_realizefn;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 
     adc->devreq = adb_mouse_request;
     dc->reset = adb_mouse_reset;
diff --git a/hw/input/goldfish_events.c b/hw/input/goldfish_events.c
index 76fc63f..96271f7 100644
--- a/hw/input/goldfish_events.c
+++ b/hw/input/goldfish_events.c
@@ -13,7 +13,8 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  */
-
+#include "qemu/osdep.h"
+#include "qemu/log.h"
 #include "hw/sysbus.h"
 #include "ui/input.h"
 #include "ui/console.h"
@@ -877,7 +878,7 @@
     }
 }
 
-static const int dpad_map[Q_KEY_CODE_MAX] = {
+static const int dpad_map[Q_KEY_CODE__MAX] = {
     [Q_KEY_CODE_KP_4] = LINUX_KEY_LEFT,
     [Q_KEY_CODE_KP_6] = LINUX_KEY_RIGHT,
     [Q_KEY_CODE_KP_8] = LINUX_KEY_UP,
@@ -893,15 +894,15 @@
     int lkey = 0;
     int mod;
 
-    assert(evt->kind == INPUT_EVENT_KIND_KEY);
+    assert(evt->type == INPUT_EVENT_KIND_KEY);
 
-    KeyValue* kv = evt->key->key;
+    KeyValue* kv = evt->u.key.data->key;
 
-    int qcode = kv->qcode;
+    int qcode = kv->u.qcode.data;
 
-    enqueue_event(s, EV_KEY, qcode, evt->key->down);
+    enqueue_event(s, EV_KEY, qcode, evt->u.key.data->down);
 
-    int qemu2_qcode = qemu_input_key_value_to_qcode(evt->key->key);
+    int qemu2_qcode = qemu_input_key_value_to_qcode(evt->u.key.data->key);
 
     /* Keep our modifier state up to date */
     switch (qemu2_qcode) {
@@ -923,7 +924,7 @@
     }
 
     if (mod) {
-        if (evt->key->down) {
+        if (evt->u.key.data->down) {
             s->modifier_state |= mod;
         } else {
             s->modifier_state &= ~mod;
@@ -935,7 +936,7 @@
     }
 
     if (lkey) {
-        enqueue_event(s, EV_KEY, lkey, evt->key->down);
+        enqueue_event(s, EV_KEY, lkey, evt->u.key.data->down);
     }
 }
 
diff --git a/hw/input/hid.c b/hw/input/hid.c
index 148c003..5e2850e 100644
--- a/hw/input/hid.c
+++ b/hw/input/hid.c
@@ -22,10 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "qemu/timer.h"
 #include "hw/input/hid.h"
+#include "trace.h"
 
 #define HID_USAGE_ERROR_ROLLOVER        0x01
 #define HID_USAGE_POSTFAIL              0x02
@@ -41,10 +43,10 @@
     0x07, 0x09, 0x0a, 0x0b, 0x0d, 0x0e, 0x0f, 0x33,
     0x34, 0x35, 0xe1, 0x31, 0x1d, 0x1b, 0x06, 0x19,
     0x05, 0x11, 0x10, 0x36, 0x37, 0x38, 0xe5, 0x55,
-    0xe2, 0x2c, 0x32, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e,
+    0xe2, 0x2c, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e,
     0x3f, 0x40, 0x41, 0x42, 0x43, 0x53, 0x47, 0x5f,
     0x60, 0x61, 0x56, 0x5c, 0x5d, 0x5e, 0x57, 0x59,
-    0x5a, 0x5b, 0x62, 0x63, 0x00, 0x00, 0x00, 0x44,
+    0x5a, 0x5b, 0x62, 0x63, 0x00, 0x00, 0x64, 0x44,
     0x45, 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e,
     0xe8, 0xe9, 0x71, 0x72, 0x73, 0x00, 0x00, 0x00,
     0x00, 0x00, 0x00, 0x85, 0x00, 0x00, 0x00, 0x00,
@@ -95,7 +97,7 @@
 {
     if (hs->idle) {
         uint64_t expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                               get_ticks_per_sec() * hs->idle * 4 / 1000;
+                               NANOSECONDS_PER_SECOND * hs->idle * 4 / 1000;
         if (!hs->idle_timer) {
             hs->idle_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, hid_idle_timer, hs);
         }
@@ -108,44 +110,49 @@
 static void hid_pointer_event(DeviceState *dev, QemuConsole *src,
                               InputEvent *evt)
 {
-    static const int bmap[INPUT_BUTTON_MAX] = {
+    static const int bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]   = 0x01,
         [INPUT_BUTTON_RIGHT]  = 0x02,
         [INPUT_BUTTON_MIDDLE] = 0x04,
     };
     HIDState *hs = (HIDState *)dev;
     HIDPointerEvent *e;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
 
     assert(hs->n < QUEUE_LENGTH);
     e = &hs->ptr.queue[(hs->head + hs->n) & QUEUE_MASK];
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_REL:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            e->xdx += evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            e->ydy += evt->rel->value;
+        move = evt->u.rel.data;
+        if (move->axis == INPUT_AXIS_X) {
+            e->xdx += move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            e->ydy += move->value;
         }
         break;
 
     case INPUT_EVENT_KIND_ABS:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            e->xdx = evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            e->ydy = evt->rel->value;
+        move = evt->u.abs.data;
+        if (move->axis == INPUT_AXIS_X) {
+            e->xdx = move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            e->ydy = move->value;
         }
         break;
 
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            e->buttons_state |= bmap[evt->btn->button];
-            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        btn = evt->u.btn.data;
+        if (btn->down) {
+            e->buttons_state |= bmap[btn->button];
+            if (btn->button == INPUT_BUTTON_WHEEL_UP) {
                 e->dz--;
-            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+            } else if (btn->button == INPUT_BUTTON_WHEEL_DOWN) {
                 e->dz++;
             }
         } else {
-            e->buttons_state &= ~bmap[evt->btn->button];
+            e->buttons_state &= ~bmap[btn->button];
         }
         break;
 
@@ -222,12 +229,13 @@
     HIDState *hs = (HIDState *)dev;
     int scancodes[3], i, count;
     int slot;
+    InputKeyEvent *key = evt->u.key.data;
 
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(key->key,
+                                             key->down,
                                              scancodes);
     if (hs->n + count > QUEUE_LENGTH) {
-        fprintf(stderr, "usb-kbd: warning: key event queue full\n");
+        trace_hid_kbd_queue_full();
         return;
     }
     for (i = 0; i < count; i++) {
@@ -239,7 +247,7 @@
 
 static void hid_keyboard_process_keycode(HIDState *hs)
 {
-    uint8_t hid_code, key;
+    uint8_t hid_code, index, key;
     int i, keycode, slot;
 
     if (hs->n == 0) {
@@ -249,7 +257,8 @@
     keycode = hs->kbd.keycodes[slot];
 
     key = keycode & 0x7f;
-    hid_code = hid_usage_keys[key | ((hs->kbd.modifiers >> 1) & (1 << 7))];
+    index = key | ((hs->kbd.modifiers & (1 << 8)) >> 1);
+    hid_code = hid_usage_keys[index];
     hs->kbd.modifiers &= ~(1 << 8);
 
     switch (hid_code) {
@@ -257,18 +266,41 @@
         return;
 
     case 0xe0:
+        assert(key == 0x1d);
         if (hs->kbd.modifiers & (1 << 9)) {
-            hs->kbd.modifiers ^= 3 << 8;
+            /* The hid_codes for the 0xe1/0x1d scancode sequence are 0xe9/0xe0.
+             * Here we're processing the second hid_code.  By dropping bit 9
+             * and setting bit 8, the scancode after 0x1d will access the
+             * second half of the table.
+             */
+            hs->kbd.modifiers ^= (1 << 8) | (1 << 9);
             return;
         }
+        /* fall through to process Ctrl_L */
     case 0xe1 ... 0xe7:
+        /* Ctrl_L/Ctrl_R, Shift_L/Shift_R, Alt_L/Alt_R, Win_L/Win_R.
+         * Handle releases here, or fall through to process presses.
+         */
         if (keycode & (1 << 7)) {
             hs->kbd.modifiers &= ~(1 << (hid_code & 0x0f));
             return;
         }
-    case 0xe8 ... 0xef:
+        /* fall through */
+    case 0xe8 ... 0xe9:
+        /* USB modifiers are just 1 byte long.  Bits 8 and 9 of
+         * hs->kbd.modifiers implement a state machine that detects the
+         * 0xe0 and 0xe1/0x1d sequences.  These bits do not follow the
+         * usual rules where bit 7 marks released keys; they are cleared
+         * elsewhere in the function as the state machine dictates.
+         */
         hs->kbd.modifiers |= 1 << (hid_code & 0x0f);
         return;
+
+    case 0xea ... 0xef:
+        abort();
+
+    default:
+        break;
     }
 
     if (keycode & (1 << 7)) {
@@ -514,6 +546,27 @@
     HIDState *s = opaque;
 
     hid_set_next_idle(s);
+
+    if (s->n == QUEUE_LENGTH && (s->kind == HID_TABLET ||
+                                 s->kind == HID_MOUSE)) {
+        /*
+         * Handle ptr device migration from old qemu with full queue.
+         *
+         * Throw away everything but the last event, so we propagate
+         * at least the current button state to the guest.  Also keep
+         * current position for the tablet, signal "no motion" for the
+         * mouse.
+         */
+        HIDPointerEvent evt;
+        evt = s->ptr.queue[(s->head+s->n) & QUEUE_MASK];
+        if (s->kind == HID_MOUSE) {
+            evt.xdx = 0;
+            evt.ydy = 0;
+        }
+        s->ptr.queue[0] = evt;
+        s->head = 0;
+        s->n = 1;
+    }
     return 0;
 }
 
diff --git a/hw/input/lm832x.c b/hw/input/lm832x.c
index 9eb68e8..539682c 100644
--- a/hw/input/lm832x.c
+++ b/hw/input/lm832x.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "qemu/timer.h"
@@ -455,7 +456,7 @@
         VMSTATE_UINT16_ARRAY(pwm.file, LM823KbdState, 256),
         VMSTATE_UINT8(pwm.faddr, LM823KbdState),
         VMSTATE_BUFFER(pwm.addr, LM823KbdState),
-        VMSTATE_TIMER_ARRAY(pwm.tm, LM823KbdState, 3),
+        VMSTATE_TIMER_PTR_ARRAY(pwm.tm, LM823KbdState, 3),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/input/milkymist-softusb.c b/hw/input/milkymist-softusb.c
index 5a427f0..40dfca1 100644
--- a/hw/input/milkymist-softusb.c
+++ b/hw/input/milkymist-softusb.c
@@ -21,6 +21,8 @@
  *   not available yet
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -194,10 +196,13 @@
         return;
     }
 
-    len = hid_keyboard_poll(hs, s->kbd_hid_buffer, sizeof(s->kbd_hid_buffer));
+    while (hid_has_events(hs)) {
+        len = hid_keyboard_poll(hs, s->kbd_hid_buffer,
+                sizeof(s->kbd_hid_buffer));
 
-    if (len == 8) {
-        softusb_kbd_changed(s);
+        if (len == 8) {
+            softusb_kbd_changed(s);
+        }
     }
 }
 
@@ -212,11 +217,13 @@
         return;
     }
 
-    len = hid_pointer_poll(hs, s->mouse_hid_buffer,
-            sizeof(s->mouse_hid_buffer));
+    while (hid_has_events(hs)) {
+        len = hid_pointer_poll(hs, s->mouse_hid_buffer,
+                sizeof(s->mouse_hid_buffer));
 
-    if (len == 4) {
-        softusb_mouse_changed(s);
+        if (len == 4) {
+            softusb_mouse_changed(s);
+        }
     }
 }
 
@@ -250,12 +257,12 @@
 
     /* register pmem and dmem */
     memory_region_init_ram(&s->pmem, OBJECT(s), "milkymist-softusb.pmem",
-                           s->pmem_size, &error_abort);
+                           s->pmem_size, &error_fatal);
     vmstate_register_ram_global(&s->pmem);
     s->pmem_ptr = memory_region_get_ram_ptr(&s->pmem);
     sysbus_init_mmio(dev, &s->pmem);
     memory_region_init_ram(&s->dmem, OBJECT(s), "milkymist-softusb.dmem",
-                           s->dmem_size, &error_abort);
+                           s->dmem_size, &error_fatal);
     vmstate_register_ram_global(&s->dmem);
     s->dmem_ptr = memory_region_get_ram_ptr(&s->dmem);
     sysbus_init_mmio(dev, &s->dmem);
diff --git a/hw/input/pckbd.c b/hw/input/pckbd.c
index 2b0cd3d..dc57e2c 100644
--- a/hw/input/pckbd.c
+++ b/hw/input/pckbd.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "hw/i386/pc.h"
@@ -101,6 +102,12 @@
 #define KBD_OUT_OBF             0x10    /* Keyboard output buffer full */
 #define KBD_OUT_MOUSE_OBF       0x20    /* Mouse output buffer full */
 
+/* OSes typically write 0xdd/0xdf to turn the A20 line off and on.
+ * We make the default value of the outport include these four bits,
+ * so that the subsection is rarely necessary.
+ */
+#define KBD_OUT_ONES            0xcc
+
 /* Mouse Commands */
 #define AUX_SET_SCALE11		0xE6	/* Set 1:1 scaling */
 #define AUX_SET_SCALE21		0xE7	/* Set 2:1 scaling */
@@ -139,7 +146,7 @@
 
     qemu_irq irq_kbd;
     qemu_irq irq_mouse;
-    qemu_irq *a20_out;
+    qemu_irq a20_out;
     hwaddr mask;
 } KBDState;
 
@@ -217,9 +224,7 @@
 {
     DPRINTF("kbd: write outport=0x%02x\n", val);
     s->outport = val;
-    if (s->a20_out) {
-        qemu_set_irq(*s->a20_out, (val >> 1) & 1);
-    }
+    qemu_set_irq(s->a20_out, (val >> 1) & 1);
     if (!(val & 1)) {
         qemu_system_reset_request();
     }
@@ -288,15 +293,11 @@
         kbd_queue(s, s->outport, 0);
         break;
     case KBD_CCMD_ENABLE_A20:
-        if (s->a20_out) {
-            qemu_irq_raise(*s->a20_out);
-        }
+        qemu_irq_raise(s->a20_out);
         s->outport |= KBD_OUT_A20;
         break;
     case KBD_CCMD_DISABLE_A20:
-        if (s->a20_out) {
-            qemu_irq_lower(*s->a20_out);
-        }
+        qemu_irq_lower(s->a20_out);
         s->outport &= ~KBD_OUT_A20;
         break;
     case KBD_CCMD_RESET:
@@ -367,13 +368,13 @@
 
     s->mode = KBD_MODE_KBD_INT | KBD_MODE_MOUSE_INT;
     s->status = KBD_STAT_CMD | KBD_STAT_UNLOCKED;
-    s->outport = KBD_OUT_RESET | KBD_OUT_A20;
+    s->outport = KBD_OUT_RESET | KBD_OUT_A20 | KBD_OUT_ONES;
     s->outport_present = false;
 }
 
 static uint8_t kbd_outport_default(KBDState *s)
 {
-    return KBD_OUT_RESET | KBD_OUT_A20
+    return KBD_OUT_RESET | KBD_OUT_A20 | KBD_OUT_ONES
            | (s->status & KBD_STAT_OBF ? KBD_OUT_OBF : 0)
            | (s->status & KBD_STAT_MOUSE_OBF ? KBD_OUT_MOUSE_OBF : 0);
 }
@@ -385,23 +386,24 @@
     return 0;
 }
 
-static const VMStateDescription vmstate_kbd_outport = {
-    .name = "pckbd_outport",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .post_load = kbd_outport_post_load,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT8(outport, KBDState),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
 static bool kbd_outport_needed(void *opaque)
 {
     KBDState *s = opaque;
     return s->outport != kbd_outport_default(s);
 }
 
+static const VMStateDescription vmstate_kbd_outport = {
+    .name = "pckbd_outport",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = kbd_outport_post_load,
+    .needed = kbd_outport_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(outport, KBDState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static int kbd_post_load(void *opaque, int version_id)
 {
     KBDState *s = opaque;
@@ -424,12 +426,9 @@
         VMSTATE_UINT8(pending, KBDState),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_kbd_outport,
-            .needed = kbd_outport_needed,
-        },
-        VMSTATE_END_OF_LIST()
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_kbd_outport,
+        NULL
     }
 };
 
@@ -502,10 +501,7 @@
 
 void i8042_setup_a20_line(ISADevice *dev, qemu_irq *a20_out)
 {
-    ISAKBDState *isa = I8042(dev);
-    KBDState *s = &isa->kbd;
-
-    s->a20_out = a20_out;
+    qdev_connect_gpio_out_named(DEVICE(dev), I8042_A20_LINE, 0, *a20_out);
 }
 
 static const VMStateDescription vmstate_kbd_isa = {
@@ -547,6 +543,8 @@
                           "i8042-data", 1);
     memory_region_init_io(isa_s->io + 1, obj, &i8042_cmd_ops, s,
                           "i8042-cmd", 1);
+
+    qdev_init_gpio_out_named(DEVICE(obj), &s->a20_out, I8042_A20_LINE, 1);
 }
 
 static void i8042_realizefn(DeviceState *dev, Error **errp)
diff --git a/hw/input/pl050.c b/hw/input/pl050.c
index c1b08d5..be9cd57 100644
--- a/hw/input/pl050.c
+++ b/hw/input/pl050.c
@@ -7,8 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/input/ps2.h"
+#include "qemu/log.h"
 
 #define TYPE_PL050 "pl050"
 #define PL050(obj) OBJECT_CHECK(PL050State, (obj), TYPE_PL050)
diff --git a/hw/input/ps2.c b/hw/input/ps2.c
index a466e25..a8aa36f 100644
--- a/hw/input/ps2.c
+++ b/hw/input/ps2.c
@@ -21,12 +21,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/input/ps2.h"
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/sysemu.h"
 
+#include "trace.h"
+
 /* debug PC keyboard */
 //#define DEBUG_KBD
 
@@ -158,6 +161,7 @@
 {
     PS2KbdState *s = opaque;
 
+    trace_ps2_put_keycode(opaque, keycode);
     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
     /* XXX: add support for scancode set 1 */
     if (!s->translate && keycode < 0xe0 && s->scancode_set > 1) {
@@ -178,10 +182,11 @@
 {
     PS2KbdState *s = (PS2KbdState *)dev;
     int scancodes[3], i, count;
+    InputKeyEvent *key = evt->u.key.data;
 
     qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(key->key,
+                                             key->down,
                                              scancodes);
     for (i = 0; i < count; i++) {
         ps2_put_keycode(s, scancodes[i]);
@@ -194,6 +199,7 @@
     PS2Queue *q;
     int val, index;
 
+    trace_ps2_read_data(opaque);
     q = &s->queue;
     if (q->count == 0) {
         /* NOTE: if no data left, we return the last keyboard one
@@ -218,12 +224,14 @@
 
 static void ps2_set_ledstate(PS2KbdState *s, int ledstate)
 {
+    trace_ps2_set_ledstate(s, ledstate);
     s->ledstate = ledstate;
     kbd_put_ledstate(ledstate);
 }
 
 static void ps2_reset_keyboard(PS2KbdState *s)
 {
+    trace_ps2_reset_keyboard(s);
     s->scan_enabled = 1;
     s->scancode_set = 2;
     ps2_set_ledstate(s, 0);
@@ -233,6 +241,7 @@
 {
     PS2KbdState *s = (PS2KbdState *)opaque;
 
+    trace_ps2_write_keyboard(opaque, val);
     switch(s->common.write_cmd) {
     default:
     case -1:
@@ -319,6 +328,7 @@
 void ps2_keyboard_set_translation(void *opaque, int mode)
 {
     PS2KbdState *s = (PS2KbdState *)opaque;
+    trace_ps2_keyboard_set_translation(opaque, mode);
     s->translate = mode;
 }
 
@@ -364,6 +374,7 @@
         break;
     }
 
+    trace_ps2_mouse_send_packet(s, dx1, dy1, dz1, b);
     /* update deltas */
     s->mouse_dx -= dx1;
     s->mouse_dy -= dy1;
@@ -373,36 +384,40 @@
 static void ps2_mouse_event(DeviceState *dev, QemuConsole *src,
                             InputEvent *evt)
 {
-    static const int bmap[INPUT_BUTTON_MAX] = {
+    static const int bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]   = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE] = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]  = MOUSE_EVENT_RBUTTON,
     };
     PS2MouseState *s = (PS2MouseState *)dev;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
 
     /* check if deltas are recorded when disabled */
     if (!(s->mouse_status & MOUSE_STATUS_ENABLED))
         return;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_REL:
-        if (evt->rel->axis == INPUT_AXIS_X) {
-            s->mouse_dx += evt->rel->value;
-        } else if (evt->rel->axis == INPUT_AXIS_Y) {
-            s->mouse_dy -= evt->rel->value;
+        move = evt->u.rel.data;
+        if (move->axis == INPUT_AXIS_X) {
+            s->mouse_dx += move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            s->mouse_dy -= move->value;
         }
         break;
 
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            s->mouse_buttons |= bmap[evt->btn->button];
-            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        btn = evt->u.btn.data;
+        if (btn->down) {
+            s->mouse_buttons |= bmap[btn->button];
+            if (btn->button == INPUT_BUTTON_WHEEL_UP) {
                 s->mouse_dz--;
-            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+            } else if (btn->button == INPUT_BUTTON_WHEEL_DOWN) {
                 s->mouse_dz++;
             }
         } else {
-            s->mouse_buttons &= ~bmap[evt->btn->button];
+            s->mouse_buttons &= ~bmap[btn->button];
         }
         break;
 
@@ -433,6 +448,7 @@
 void ps2_mouse_fake_event(void *opaque)
 {
     PS2MouseState *s = opaque;
+    trace_ps2_mouse_fake_event(opaque);
     s->mouse_dx++;
     ps2_mouse_sync(opaque);
 }
@@ -440,6 +456,8 @@
 void ps2_write_mouse(void *opaque, int val)
 {
     PS2MouseState *s = (PS2MouseState *)opaque;
+
+    trace_ps2_write_mouse(opaque, val);
 #ifdef DEBUG_MOUSE
     printf("kbd: write mouse 0x%02x\n", val);
 #endif
@@ -606,16 +624,18 @@
 {
     PS2KbdState *s = (PS2KbdState *) opaque;
 
+    trace_ps2_kbd_reset(opaque);
     ps2_common_reset(&s->common);
     s->scan_enabled = 0;
     s->translate = 0;
-    s->scancode_set = 0;
+    s->scancode_set = 2;
 }
 
 static void ps2_mouse_reset(void *opaque)
 {
     PS2MouseState *s = (PS2MouseState *) opaque;
 
+    trace_ps2_mouse_reset(opaque);
     ps2_common_reset(&s->common);
     s->mouse_status = 0;
     s->mouse_resolution = 0;
@@ -663,6 +683,7 @@
     .version_id = 3,
     .minimum_version_id = 2,
     .post_load = ps2_kbd_ledstate_post_load,
+    .needed = ps2_keyboard_ledstate_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(ledstate, PS2KbdState),
         VMSTATE_END_OF_LIST()
@@ -703,13 +724,9 @@
         VMSTATE_INT32_V(scancode_set, PS2KbdState,3),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_ps2_keyboard_ledstate,
-            .needed = ps2_keyboard_ledstate_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_ps2_keyboard_ledstate,
+        NULL
     }
 };
 
@@ -763,6 +780,7 @@
 {
     PS2KbdState *s = (PS2KbdState *)g_malloc0(sizeof(PS2KbdState));
 
+    trace_ps2_kbd_init(s);
     s->common.update_irq = update_irq;
     s->common.update_arg = update_arg;
     s->scancode_set = 2;
@@ -784,6 +802,7 @@
 {
     PS2MouseState *s = (PS2MouseState *)g_malloc0(sizeof(PS2MouseState));
 
+    trace_ps2_mouse_init(s);
     s->common.update_irq = update_irq;
     s->common.update_arg = update_arg;
     vmstate_register(NULL, 0, &vmstate_ps2_mouse, s);
diff --git a/hw/input/pxa2xx_keypad.c b/hw/input/pxa2xx_keypad.c
index 8501114..2b70bbb 100644
--- a/hw/input/pxa2xx_keypad.c
+++ b/hw/input/pxa2xx_keypad.c
@@ -11,6 +11,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/pxa.h"
 #include "ui/console.h"
diff --git a/hw/input/stellaris_input.c b/hw/input/stellaris_input.c
index 0609e80..99168bf 100644
--- a/hw/input/stellaris_input.c
+++ b/hw/input/stellaris_input.c
@@ -6,6 +6,7 @@
  *
  * This code is licensed under the GPL.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
 #include "ui/console.h"
@@ -69,14 +70,14 @@
     }
 };
 
-/* Returns an array 5 ouput slots.  */
+/* Returns an array of 5 output slots.  */
 void stellaris_gamepad_init(int n, qemu_irq *irq, const int *keycode)
 {
     gamepad_state *s;
     int i;
 
-    s = (gamepad_state *)g_malloc0(sizeof (gamepad_state));
-    s->buttons = (gamepad_button *)g_malloc0(n * sizeof (gamepad_button));
+    s = g_new0(gamepad_state, 1);
+    s->buttons = g_new0(gamepad_button, n);
     for (i = 0; i < n; i++) {
         s->buttons[i].irq = irq[i];
         s->buttons[i].keycode = keycode[i];
diff --git a/hw/input/trace-events b/hw/input/trace-events
new file mode 100644
index 0000000..8c4003f
--- /dev/null
+++ b/hw/input/trace-events
@@ -0,0 +1,31 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/input/ps2.c
+ps2_put_keycode(void *opaque, int keycode) "%p keycode %d"
+ps2_read_data(void *opaque) "%p"
+ps2_set_ledstate(void *s, int ledstate) "%p ledstate %d"
+ps2_reset_keyboard(void *s) "%p"
+ps2_write_keyboard(void *opaque, int val) "%p val %d"
+ps2_keyboard_set_translation(void *opaque, int mode) "%p mode %d"
+ps2_mouse_send_packet(void *s, int dx1, int dy1, int dz1, int b) "%p x %d y %d z %d bs %#x"
+ps2_mouse_event_disabled(void *opaque, int dx, int dy, int dz, int buttons_state, int mouse_dx, int mouse_dy, int mouse_dz) "%p x %d y %d z %d bs %#x mx %d my %d mz %d "
+ps2_mouse_event(void *opaque, int dx, int dy, int dz, int buttons_state, int mouse_dx, int mouse_dy, int mouse_dz) "%p x %d y %d z %d bs %#x mx %d my %d mz %d "
+ps2_mouse_fake_event(void *opaque) "%p"
+ps2_write_mouse(void *opaque, int val) "%p val %d"
+ps2_kbd_reset(void *opaque) "%p"
+ps2_mouse_reset(void *opaque) "%p"
+ps2_kbd_init(void *s) "%p"
+ps2_mouse_init(void *s) "%p"
+
+# hw/input/milkymist-softusb.c
+milkymist_softusb_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_softusb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_softusb_mevt(uint8_t m) "m %d"
+milkymist_softusb_kevt(uint8_t m) "m %d"
+milkymist_softusb_pulse_irq(void) "Pulse IRQ"
+
+# hw/input/hid.c
+hid_kbd_queue_full(void) "queue full"
+
+# hw/input/virtio
+virtio_input_queue_full(void) "queue full"
diff --git a/hw/input/tsc2005.c b/hw/input/tsc2005.c
index 21d4f4d..9b359aa 100644
--- a/hw/input/tsc2005.c
+++ b/hw/input/tsc2005.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "ui/console.h"
@@ -290,7 +291,8 @@
     s->precision = s->nextprecision;
     s->function = s->nextfunction;
     s->pdst = !s->pnd0;	/* Synchronised on internal clock */
-    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() >> 7);
+    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+        (NANOSECONDS_PER_SECOND >> 7);
     timer_mod(s->timer, expires);
 }
 
diff --git a/hw/input/tsc210x.c b/hw/input/tsc210x.c
index fae3385..93ca374 100644
--- a/hw/input/tsc210x.c
+++ b/hw/input/tsc210x.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "audio/audio.h"
 #include "qemu/timer.h"
@@ -834,7 +835,8 @@
     s->busy = 1;
     s->precision = s->nextprecision;
     s->function = s->nextfunction;
-    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + (get_ticks_per_sec() >> 10);
+    expires = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+        (NANOSECONDS_PER_SECOND >> 10);
     timer_mod(s->timer, expires);
 }
 
@@ -1086,9 +1088,7 @@
 {
     TSC210xState *s;
 
-    s = (TSC210xState *)
-            g_malloc0(sizeof(TSC210xState));
-    memset(s, 0, sizeof(TSC210xState));
+    s = g_new0(TSC210xState, 1);
     s->x = 160;
     s->y = 160;
     s->pressure = 0;
@@ -1135,9 +1135,7 @@
 {
     TSC210xState *s;
 
-    s = (TSC210xState *)
-            g_malloc0(sizeof(TSC210xState));
-    memset(s, 0, sizeof(TSC210xState));
+    s = g_new0(TSC210xState, 1);
     s->x = 400;
     s->y = 240;
     s->pressure = 0;
diff --git a/hw/input/virtio-input-hid.c b/hw/input/virtio-input-hid.c
new file mode 100644
index 0000000..3ee0c18
--- /dev/null
+++ b/hw/input/virtio-input-hid.c
@@ -0,0 +1,525 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/iov.h"
+
+#include "hw/qdev.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-input.h"
+
+#undef CONFIG_CURSES
+#include "ui/console.h"
+
+#include "standard-headers/linux/input.h"
+
+#define VIRTIO_ID_NAME_KEYBOARD "QEMU Virtio Keyboard"
+#define VIRTIO_ID_NAME_MOUSE    "QEMU Virtio Mouse"
+#define VIRTIO_ID_NAME_TABLET   "QEMU Virtio Tablet"
+
+/* ----------------------------------------------------------------- */
+
+static const unsigned int keymap_qcode[Q_KEY_CODE__MAX] = {
+    [Q_KEY_CODE_ESC]                 = KEY_ESC,
+    [Q_KEY_CODE_1]                   = KEY_1,
+    [Q_KEY_CODE_2]                   = KEY_2,
+    [Q_KEY_CODE_3]                   = KEY_3,
+    [Q_KEY_CODE_4]                   = KEY_4,
+    [Q_KEY_CODE_5]                   = KEY_5,
+    [Q_KEY_CODE_6]                   = KEY_6,
+    [Q_KEY_CODE_7]                   = KEY_7,
+    [Q_KEY_CODE_8]                   = KEY_8,
+    [Q_KEY_CODE_9]                   = KEY_9,
+    [Q_KEY_CODE_0]                   = KEY_0,
+    [Q_KEY_CODE_MINUS]               = KEY_MINUS,
+    [Q_KEY_CODE_EQUAL]               = KEY_EQUAL,
+    [Q_KEY_CODE_BACKSPACE]           = KEY_BACKSPACE,
+
+    [Q_KEY_CODE_TAB]                 = KEY_TAB,
+    [Q_KEY_CODE_Q]                   = KEY_Q,
+    [Q_KEY_CODE_W]                   = KEY_W,
+    [Q_KEY_CODE_E]                   = KEY_E,
+    [Q_KEY_CODE_R]                   = KEY_R,
+    [Q_KEY_CODE_T]                   = KEY_T,
+    [Q_KEY_CODE_Y]                   = KEY_Y,
+    [Q_KEY_CODE_U]                   = KEY_U,
+    [Q_KEY_CODE_I]                   = KEY_I,
+    [Q_KEY_CODE_O]                   = KEY_O,
+    [Q_KEY_CODE_P]                   = KEY_P,
+    [Q_KEY_CODE_BRACKET_LEFT]        = KEY_LEFTBRACE,
+    [Q_KEY_CODE_BRACKET_RIGHT]       = KEY_RIGHTBRACE,
+    [Q_KEY_CODE_RET]                 = KEY_ENTER,
+
+    [Q_KEY_CODE_CTRL]                = KEY_LEFTCTRL,
+    [Q_KEY_CODE_A]                   = KEY_A,
+    [Q_KEY_CODE_S]                   = KEY_S,
+    [Q_KEY_CODE_D]                   = KEY_D,
+    [Q_KEY_CODE_F]                   = KEY_F,
+    [Q_KEY_CODE_G]                   = KEY_G,
+    [Q_KEY_CODE_H]                   = KEY_H,
+    [Q_KEY_CODE_J]                   = KEY_J,
+    [Q_KEY_CODE_K]                   = KEY_K,
+    [Q_KEY_CODE_L]                   = KEY_L,
+    [Q_KEY_CODE_SEMICOLON]           = KEY_SEMICOLON,
+    [Q_KEY_CODE_APOSTROPHE]          = KEY_APOSTROPHE,
+    [Q_KEY_CODE_GRAVE_ACCENT]        = KEY_GRAVE,
+
+    [Q_KEY_CODE_SHIFT]               = KEY_LEFTSHIFT,
+    [Q_KEY_CODE_BACKSLASH]           = KEY_BACKSLASH,
+    [Q_KEY_CODE_LESS]                = KEY_102ND,
+    [Q_KEY_CODE_Z]                   = KEY_Z,
+    [Q_KEY_CODE_X]                   = KEY_X,
+    [Q_KEY_CODE_C]                   = KEY_C,
+    [Q_KEY_CODE_V]                   = KEY_V,
+    [Q_KEY_CODE_B]                   = KEY_B,
+    [Q_KEY_CODE_N]                   = KEY_N,
+    [Q_KEY_CODE_M]                   = KEY_M,
+    [Q_KEY_CODE_COMMA]               = KEY_COMMA,
+    [Q_KEY_CODE_DOT]                 = KEY_DOT,
+    [Q_KEY_CODE_SLASH]               = KEY_SLASH,
+    [Q_KEY_CODE_SHIFT_R]             = KEY_RIGHTSHIFT,
+
+    [Q_KEY_CODE_ALT]                 = KEY_LEFTALT,
+    [Q_KEY_CODE_SPC]                 = KEY_SPACE,
+    [Q_KEY_CODE_CAPS_LOCK]           = KEY_CAPSLOCK,
+
+    [Q_KEY_CODE_F1]                  = KEY_F1,
+    [Q_KEY_CODE_F2]                  = KEY_F2,
+    [Q_KEY_CODE_F3]                  = KEY_F3,
+    [Q_KEY_CODE_F4]                  = KEY_F4,
+    [Q_KEY_CODE_F5]                  = KEY_F5,
+    [Q_KEY_CODE_F6]                  = KEY_F6,
+    [Q_KEY_CODE_F7]                  = KEY_F7,
+    [Q_KEY_CODE_F8]                  = KEY_F8,
+    [Q_KEY_CODE_F9]                  = KEY_F9,
+    [Q_KEY_CODE_F10]                 = KEY_F10,
+    [Q_KEY_CODE_NUM_LOCK]            = KEY_NUMLOCK,
+    [Q_KEY_CODE_SCROLL_LOCK]         = KEY_SCROLLLOCK,
+
+    [Q_KEY_CODE_KP_0]                = KEY_KP0,
+    [Q_KEY_CODE_KP_1]                = KEY_KP1,
+    [Q_KEY_CODE_KP_2]                = KEY_KP2,
+    [Q_KEY_CODE_KP_3]                = KEY_KP3,
+    [Q_KEY_CODE_KP_4]                = KEY_KP4,
+    [Q_KEY_CODE_KP_5]                = KEY_KP5,
+    [Q_KEY_CODE_KP_6]                = KEY_KP6,
+    [Q_KEY_CODE_KP_7]                = KEY_KP7,
+    [Q_KEY_CODE_KP_8]                = KEY_KP8,
+    [Q_KEY_CODE_KP_9]                = KEY_KP9,
+    [Q_KEY_CODE_KP_SUBTRACT]         = KEY_KPMINUS,
+    [Q_KEY_CODE_KP_ADD]              = KEY_KPPLUS,
+    [Q_KEY_CODE_KP_DECIMAL]          = KEY_KPDOT,
+    [Q_KEY_CODE_KP_ENTER]            = KEY_KPENTER,
+    [Q_KEY_CODE_KP_DIVIDE]           = KEY_KPSLASH,
+    [Q_KEY_CODE_KP_MULTIPLY]         = KEY_KPASTERISK,
+
+    [Q_KEY_CODE_F11]                 = KEY_F11,
+    [Q_KEY_CODE_F12]                 = KEY_F12,
+
+    [Q_KEY_CODE_CTRL_R]              = KEY_RIGHTCTRL,
+    [Q_KEY_CODE_SYSRQ]               = KEY_SYSRQ,
+    [Q_KEY_CODE_PRINT]               = KEY_SYSRQ,
+    [Q_KEY_CODE_PAUSE]               = KEY_PAUSE,
+    [Q_KEY_CODE_ALT_R]               = KEY_RIGHTALT,
+
+    [Q_KEY_CODE_HOME]                = KEY_HOME,
+    [Q_KEY_CODE_UP]                  = KEY_UP,
+    [Q_KEY_CODE_PGUP]                = KEY_PAGEUP,
+    [Q_KEY_CODE_LEFT]                = KEY_LEFT,
+    [Q_KEY_CODE_RIGHT]               = KEY_RIGHT,
+    [Q_KEY_CODE_END]                 = KEY_END,
+    [Q_KEY_CODE_DOWN]                = KEY_DOWN,
+    [Q_KEY_CODE_PGDN]                = KEY_PAGEDOWN,
+    [Q_KEY_CODE_INSERT]              = KEY_INSERT,
+    [Q_KEY_CODE_DELETE]              = KEY_DELETE,
+
+    [Q_KEY_CODE_META_L]              = KEY_LEFTMETA,
+    [Q_KEY_CODE_META_R]              = KEY_RIGHTMETA,
+    [Q_KEY_CODE_MENU]                = KEY_MENU,
+};
+
+static const unsigned int keymap_button[INPUT_BUTTON__MAX] = {
+    [INPUT_BUTTON_LEFT]              = BTN_LEFT,
+    [INPUT_BUTTON_RIGHT]             = BTN_RIGHT,
+    [INPUT_BUTTON_MIDDLE]            = BTN_MIDDLE,
+    [INPUT_BUTTON_WHEEL_UP]          = BTN_GEAR_UP,
+    [INPUT_BUTTON_WHEEL_DOWN]        = BTN_GEAR_DOWN,
+};
+
+static const unsigned int axismap_rel[INPUT_AXIS__MAX] = {
+    [INPUT_AXIS_X]                   = REL_X,
+    [INPUT_AXIS_Y]                   = REL_Y,
+};
+
+static const unsigned int axismap_abs[INPUT_AXIS__MAX] = {
+    [INPUT_AXIS_X]                   = ABS_X,
+    [INPUT_AXIS_Y]                   = ABS_Y,
+};
+
+/* ----------------------------------------------------------------- */
+
+static void virtio_input_key_config(VirtIOInput *vinput,
+                                    const unsigned int *keymap,
+                                    size_t mapsize)
+{
+    virtio_input_config keys;
+    int i, bit, byte, bmax = 0;
+
+    memset(&keys, 0, sizeof(keys));
+    for (i = 0; i < mapsize; i++) {
+        bit = keymap[i];
+        if (!bit) {
+            continue;
+        }
+        byte = bit / 8;
+        bit  = bit % 8;
+        keys.u.bitmap[byte] |= (1 << bit);
+        if (bmax < byte+1) {
+            bmax = byte+1;
+        }
+    }
+    keys.select = VIRTIO_INPUT_CFG_EV_BITS;
+    keys.subsel = EV_KEY;
+    keys.size   = bmax;
+    virtio_input_add_config(vinput, &keys);
+}
+
+static void virtio_input_handle_event(DeviceState *dev, QemuConsole *src,
+                                      InputEvent *evt)
+{
+    VirtIOInput *vinput = VIRTIO_INPUT(dev);
+    virtio_input_event event;
+    int qcode;
+    InputKeyEvent *key;
+    InputMoveEvent *move;
+    InputBtnEvent *btn;
+
+    switch (evt->type) {
+    case INPUT_EVENT_KIND_KEY:
+        key = evt->u.key.data;
+        qcode = qemu_input_key_value_to_qcode(key->key);
+        if (qcode && keymap_qcode[qcode]) {
+            event.type  = cpu_to_le16(EV_KEY);
+            event.code  = cpu_to_le16(keymap_qcode[qcode]);
+            event.value = cpu_to_le32(key->down ? 1 : 0);
+            virtio_input_send(vinput, &event);
+        } else {
+            if (key->down) {
+                fprintf(stderr, "%s: unmapped key: %d [%s]\n", __func__,
+                        qcode, QKeyCode_lookup[qcode]);
+            }
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        btn = evt->u.btn.data;
+        if (keymap_button[btn->button]) {
+            event.type  = cpu_to_le16(EV_KEY);
+            event.code  = cpu_to_le16(keymap_button[btn->button]);
+            event.value = cpu_to_le32(btn->down ? 1 : 0);
+            virtio_input_send(vinput, &event);
+        } else {
+            if (btn->down) {
+                fprintf(stderr, "%s: unmapped button: %d [%s]\n", __func__,
+                        btn->button,
+                        InputButton_lookup[btn->button]);
+            }
+        }
+        break;
+    case INPUT_EVENT_KIND_REL:
+        move = evt->u.rel.data;
+        event.type  = cpu_to_le16(EV_REL);
+        event.code  = cpu_to_le16(axismap_rel[move->axis]);
+        event.value = cpu_to_le32(move->value);
+        virtio_input_send(vinput, &event);
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        move = evt->u.abs.data;
+        event.type  = cpu_to_le16(EV_ABS);
+        event.code  = cpu_to_le16(axismap_abs[move->axis]);
+        event.value = cpu_to_le32(move->value);
+        virtio_input_send(vinput, &event);
+        break;
+    default:
+        /* keep gcc happy */
+        break;
+    }
+}
+
+static void virtio_input_handle_sync(DeviceState *dev)
+{
+    VirtIOInput *vinput = VIRTIO_INPUT(dev);
+    virtio_input_event event = {
+        .type  = cpu_to_le16(EV_SYN),
+        .code  = cpu_to_le16(SYN_REPORT),
+        .value = 0,
+    };
+
+    virtio_input_send(vinput, &event);
+}
+
+static void virtio_input_hid_realize(DeviceState *dev, Error **errp)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(dev);
+
+    vhid->hs = qemu_input_handler_register(dev, vhid->handler);
+    if (vhid->display && vhid->hs) {
+        qemu_input_handler_bind(vhid->hs, vhid->display, vhid->head, NULL);
+    }
+}
+
+static void virtio_input_hid_unrealize(DeviceState *dev, Error **errp)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(dev);
+    qemu_input_handler_unregister(vhid->hs);
+}
+
+static void virtio_input_hid_change_active(VirtIOInput *vinput)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(vinput);
+
+    if (vinput->active) {
+        qemu_input_handler_activate(vhid->hs);
+    } else {
+        qemu_input_handler_deactivate(vhid->hs);
+    }
+}
+
+static void virtio_input_hid_handle_status(VirtIOInput *vinput,
+                                           virtio_input_event *event)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(vinput);
+    int ledbit = 0;
+
+    switch (le16_to_cpu(event->type)) {
+    case EV_LED:
+        if (event->code == LED_NUML) {
+            ledbit = QEMU_NUM_LOCK_LED;
+        } else if (event->code == LED_CAPSL) {
+            ledbit = QEMU_CAPS_LOCK_LED;
+        } else if (event->code == LED_SCROLLL) {
+            ledbit = QEMU_SCROLL_LOCK_LED;
+        }
+        if (event->value) {
+            vhid->ledstate |= ledbit;
+        } else {
+            vhid->ledstate &= ~ledbit;
+        }
+        kbd_put_ledstate(vhid->ledstate);
+        break;
+    default:
+        fprintf(stderr, "%s: unknown type %d\n", __func__,
+                le16_to_cpu(event->type));
+        break;
+    }
+}
+
+static Property virtio_input_hid_properties[] = {
+    DEFINE_PROP_STRING("display", VirtIOInputHID, display),
+    DEFINE_PROP_UINT32("head", VirtIOInputHID, head, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_input_hid_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOInputClass *vic = VIRTIO_INPUT_CLASS(klass);
+
+    dc->props          = virtio_input_hid_properties;
+    vic->realize       = virtio_input_hid_realize;
+    vic->unrealize     = virtio_input_hid_unrealize;
+    vic->change_active = virtio_input_hid_change_active;
+    vic->handle_status = virtio_input_hid_handle_status;
+}
+
+static const TypeInfo virtio_input_hid_info = {
+    .name          = TYPE_VIRTIO_INPUT_HID,
+    .parent        = TYPE_VIRTIO_INPUT,
+    .instance_size = sizeof(VirtIOInputHID),
+    .class_init    = virtio_input_hid_class_init,
+    .abstract      = true,
+};
+
+/* ----------------------------------------------------------------- */
+
+static QemuInputHandler virtio_keyboard_handler = {
+    .name  = VIRTIO_ID_NAME_KEYBOARD,
+    .mask  = INPUT_EVENT_MASK_KEY,
+    .event = virtio_input_handle_event,
+    .sync  = virtio_input_handle_sync,
+};
+
+static struct virtio_input_config virtio_keyboard_config[] = {
+    {
+        .select    = VIRTIO_INPUT_CFG_ID_NAME,
+        .size      = sizeof(VIRTIO_ID_NAME_KEYBOARD),
+        .u.string  = VIRTIO_ID_NAME_KEYBOARD,
+    },{
+        .select    = VIRTIO_INPUT_CFG_ID_DEVIDS,
+        .size      = sizeof(struct virtio_input_devids),
+        .u.ids     = {
+            .bustype = const_le16(BUS_VIRTUAL),
+            .vendor  = const_le16(0x0627), /* same we use for usb hid devices */
+            .product = const_le16(0x0001),
+            .version = const_le16(0x0001),
+        },
+    },{
+        .select    = VIRTIO_INPUT_CFG_EV_BITS,
+        .subsel    = EV_REP,
+        .size      = 1,
+    },{
+        .select    = VIRTIO_INPUT_CFG_EV_BITS,
+        .subsel    = EV_LED,
+        .size      = 1,
+        .u.bitmap  = {
+            (1 << LED_NUML) | (1 << LED_CAPSL) | (1 << LED_SCROLLL),
+        },
+    },
+    { /* end of list */ },
+};
+
+static void virtio_keyboard_init(Object *obj)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(obj);
+    VirtIOInput *vinput = VIRTIO_INPUT(obj);
+
+    vhid->handler = &virtio_keyboard_handler;
+    virtio_input_init_config(vinput, virtio_keyboard_config);
+    virtio_input_key_config(vinput, keymap_qcode,
+                            ARRAY_SIZE(keymap_qcode));
+}
+
+static const TypeInfo virtio_keyboard_info = {
+    .name          = TYPE_VIRTIO_KEYBOARD,
+    .parent        = TYPE_VIRTIO_INPUT_HID,
+    .instance_size = sizeof(VirtIOInputHID),
+    .instance_init = virtio_keyboard_init,
+};
+
+/* ----------------------------------------------------------------- */
+
+static QemuInputHandler virtio_mouse_handler = {
+    .name  = VIRTIO_ID_NAME_MOUSE,
+    .mask  = INPUT_EVENT_MASK_BTN | INPUT_EVENT_MASK_REL,
+    .event = virtio_input_handle_event,
+    .sync  = virtio_input_handle_sync,
+};
+
+static struct virtio_input_config virtio_mouse_config[] = {
+    {
+        .select    = VIRTIO_INPUT_CFG_ID_NAME,
+        .size      = sizeof(VIRTIO_ID_NAME_MOUSE),
+        .u.string  = VIRTIO_ID_NAME_MOUSE,
+    },{
+        .select    = VIRTIO_INPUT_CFG_ID_DEVIDS,
+        .size      = sizeof(struct virtio_input_devids),
+        .u.ids     = {
+            .bustype = const_le16(BUS_VIRTUAL),
+            .vendor  = const_le16(0x0627), /* same we use for usb hid devices */
+            .product = const_le16(0x0002),
+            .version = const_le16(0x0001),
+        },
+    },{
+        .select    = VIRTIO_INPUT_CFG_EV_BITS,
+        .subsel    = EV_REL,
+        .size      = 1,
+        .u.bitmap  = {
+            (1 << REL_X) | (1 << REL_Y),
+        },
+    },
+    { /* end of list */ },
+};
+
+static void virtio_mouse_init(Object *obj)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(obj);
+    VirtIOInput *vinput = VIRTIO_INPUT(obj);
+
+    vhid->handler = &virtio_mouse_handler;
+    virtio_input_init_config(vinput, virtio_mouse_config);
+    virtio_input_key_config(vinput, keymap_button,
+                            ARRAY_SIZE(keymap_button));
+}
+
+static const TypeInfo virtio_mouse_info = {
+    .name          = TYPE_VIRTIO_MOUSE,
+    .parent        = TYPE_VIRTIO_INPUT_HID,
+    .instance_size = sizeof(VirtIOInputHID),
+    .instance_init = virtio_mouse_init,
+};
+
+/* ----------------------------------------------------------------- */
+
+static QemuInputHandler virtio_tablet_handler = {
+    .name  = VIRTIO_ID_NAME_TABLET,
+    .mask  = INPUT_EVENT_MASK_BTN | INPUT_EVENT_MASK_ABS,
+    .event = virtio_input_handle_event,
+    .sync  = virtio_input_handle_sync,
+};
+
+static struct virtio_input_config virtio_tablet_config[] = {
+    {
+        .select    = VIRTIO_INPUT_CFG_ID_NAME,
+        .size      = sizeof(VIRTIO_ID_NAME_TABLET),
+        .u.string  = VIRTIO_ID_NAME_TABLET,
+    },{
+        .select    = VIRTIO_INPUT_CFG_ID_DEVIDS,
+        .size      = sizeof(struct virtio_input_devids),
+        .u.ids     = {
+            .bustype = const_le16(BUS_VIRTUAL),
+            .vendor  = const_le16(0x0627), /* same we use for usb hid devices */
+            .product = const_le16(0x0003),
+            .version = const_le16(0x0001),
+        },
+    },{
+        .select    = VIRTIO_INPUT_CFG_EV_BITS,
+        .subsel    = EV_ABS,
+        .size      = 1,
+        .u.bitmap  = {
+            (1 << ABS_X) | (1 << ABS_Y),
+        },
+    },{
+        .select    = VIRTIO_INPUT_CFG_ABS_INFO,
+        .subsel    = ABS_X,
+        .size      = sizeof(virtio_input_absinfo),
+        .u.abs.max = const_le32(INPUT_EVENT_ABS_SIZE - 1),
+    },{
+        .select    = VIRTIO_INPUT_CFG_ABS_INFO,
+        .subsel    = ABS_Y,
+        .size      = sizeof(virtio_input_absinfo),
+        .u.abs.max = const_le32(INPUT_EVENT_ABS_SIZE - 1),
+    },
+    { /* end of list */ },
+};
+
+static void virtio_tablet_init(Object *obj)
+{
+    VirtIOInputHID *vhid = VIRTIO_INPUT_HID(obj);
+    VirtIOInput *vinput = VIRTIO_INPUT(obj);
+
+    vhid->handler = &virtio_tablet_handler;
+    virtio_input_init_config(vinput, virtio_tablet_config);
+    virtio_input_key_config(vinput, keymap_button,
+                            ARRAY_SIZE(keymap_button));
+}
+
+static const TypeInfo virtio_tablet_info = {
+    .name          = TYPE_VIRTIO_TABLET,
+    .parent        = TYPE_VIRTIO_INPUT_HID,
+    .instance_size = sizeof(VirtIOInputHID),
+    .instance_init = virtio_tablet_init,
+};
+
+/* ----------------------------------------------------------------- */
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_input_hid_info);
+    type_register_static(&virtio_keyboard_info);
+    type_register_static(&virtio_mouse_info);
+    type_register_static(&virtio_tablet_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/input/virtio-input-host.c b/hw/input/virtio-input-host.c
new file mode 100644
index 0000000..cb79e80
--- /dev/null
+++ b/hw/input/virtio-input-host.c
@@ -0,0 +1,257 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/sockets.h"
+
+#include "hw/qdev.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-input.h"
+
+#include <sys/ioctl.h>
+#include "standard-headers/linux/input.h"
+
+/* ----------------------------------------------------------------- */
+
+static struct virtio_input_config virtio_input_host_config[] = {
+    { /* empty list */ },
+};
+
+static void virtio_input_host_event(void *opaque)
+{
+    VirtIOInputHost *vih = opaque;
+    VirtIOInput *vinput = VIRTIO_INPUT(vih);
+    struct virtio_input_event virtio;
+    struct input_event evdev;
+    int rc;
+
+    for (;;) {
+        rc = read(vih->fd, &evdev, sizeof(evdev));
+        if (rc != sizeof(evdev)) {
+            break;
+        }
+
+        virtio.type  = cpu_to_le16(evdev.type);
+        virtio.code  = cpu_to_le16(evdev.code);
+        virtio.value = cpu_to_le32(evdev.value);
+        virtio_input_send(vinput, &virtio);
+    }
+}
+
+static void virtio_input_bits_config(VirtIOInputHost *vih,
+                                     int type, int count)
+{
+    virtio_input_config bits;
+    int rc, i, size = 0;
+
+    memset(&bits, 0, sizeof(bits));
+    rc = ioctl(vih->fd, EVIOCGBIT(type, count/8), bits.u.bitmap);
+    if (rc < 0) {
+        return;
+    }
+
+    for (i = 0; i < count/8; i++) {
+        if (bits.u.bitmap[i]) {
+            size = i+1;
+        }
+    }
+    if (size == 0) {
+        return;
+    }
+
+    bits.select = VIRTIO_INPUT_CFG_EV_BITS;
+    bits.subsel = type;
+    bits.size   = size;
+    virtio_input_add_config(VIRTIO_INPUT(vih), &bits);
+}
+
+static void virtio_input_abs_config(VirtIOInputHost *vih, int axis)
+{
+    virtio_input_config config;
+    struct input_absinfo absinfo;
+    int rc;
+
+    rc = ioctl(vih->fd, EVIOCGABS(axis), &absinfo);
+    if (rc < 0) {
+        return;
+    }
+
+    memset(&config, 0, sizeof(config));
+    config.select = VIRTIO_INPUT_CFG_ABS_INFO;
+    config.subsel = axis;
+    config.size   = sizeof(virtio_input_absinfo);
+
+    config.u.abs.min  = cpu_to_le32(absinfo.minimum);
+    config.u.abs.max  = cpu_to_le32(absinfo.maximum);
+    config.u.abs.fuzz = cpu_to_le32(absinfo.fuzz);
+    config.u.abs.flat = cpu_to_le32(absinfo.flat);
+    config.u.abs.res  = cpu_to_le32(absinfo.resolution);
+
+    virtio_input_add_config(VIRTIO_INPUT(vih), &config);
+}
+
+static void virtio_input_host_realize(DeviceState *dev, Error **errp)
+{
+    VirtIOInputHost *vih = VIRTIO_INPUT_HOST(dev);
+    VirtIOInput *vinput = VIRTIO_INPUT(dev);
+    virtio_input_config id, *abs;
+    struct input_id ids;
+    int rc, ver, i, axis;
+    uint8_t byte;
+
+    if (!vih->evdev) {
+        error_setg(errp, "evdev property is required");
+        return;
+    }
+
+    vih->fd = open(vih->evdev, O_RDWR);
+    if (vih->fd < 0)  {
+        error_setg_file_open(errp, errno, vih->evdev);
+        return;
+    }
+    qemu_set_nonblock(vih->fd);
+
+    rc = ioctl(vih->fd, EVIOCGVERSION, &ver);
+    if (rc < 0) {
+        error_setg(errp, "%s: is not an evdev device", vih->evdev);
+        goto err_close;
+    }
+
+    rc = ioctl(vih->fd, EVIOCGRAB, 1);
+    if (rc < 0) {
+        error_setg_errno(errp, errno, "%s: failed to get exclusive access",
+                         vih->evdev);
+        goto err_close;
+    }
+
+    memset(&id, 0, sizeof(id));
+    ioctl(vih->fd, EVIOCGNAME(sizeof(id.u.string)-1), id.u.string);
+    id.select = VIRTIO_INPUT_CFG_ID_NAME;
+    id.size = strlen(id.u.string);
+    virtio_input_add_config(vinput, &id);
+
+    if (ioctl(vih->fd, EVIOCGID, &ids) == 0) {
+        memset(&id, 0, sizeof(id));
+        id.select = VIRTIO_INPUT_CFG_ID_DEVIDS;
+        id.size = sizeof(struct virtio_input_devids);
+        id.u.ids.bustype = cpu_to_le16(ids.bustype);
+        id.u.ids.vendor  = cpu_to_le16(ids.vendor);
+        id.u.ids.product = cpu_to_le16(ids.product);
+        id.u.ids.version = cpu_to_le16(ids.version);
+        virtio_input_add_config(vinput, &id);
+    }
+
+    virtio_input_bits_config(vih, EV_KEY, KEY_CNT);
+    virtio_input_bits_config(vih, EV_REL, REL_CNT);
+    virtio_input_bits_config(vih, EV_ABS, ABS_CNT);
+    virtio_input_bits_config(vih, EV_MSC, MSC_CNT);
+    virtio_input_bits_config(vih, EV_SW,  SW_CNT);
+    virtio_input_bits_config(vih, EV_LED, LED_CNT);
+
+    abs = virtio_input_find_config(VIRTIO_INPUT(vih),
+        VIRTIO_INPUT_CFG_EV_BITS, EV_ABS);
+    if (abs) {
+        for (i = 0; i < abs->size; i++) {
+            byte = abs->u.bitmap[i];
+            axis = 8 * i;
+            while (byte) {
+                if (byte & 1) {
+                    virtio_input_abs_config(vih, axis);
+                }
+                axis++;
+                byte >>= 1;
+            }
+        }
+    }
+
+    qemu_set_fd_handler(vih->fd, virtio_input_host_event, NULL, vih);
+    return;
+
+err_close:
+    close(vih->fd);
+    vih->fd = -1;
+    return;
+}
+
+static void virtio_input_host_unrealize(DeviceState *dev, Error **errp)
+{
+    VirtIOInputHost *vih = VIRTIO_INPUT_HOST(dev);
+
+    if (vih->fd > 0) {
+        qemu_set_fd_handler(vih->fd, NULL, NULL, NULL);
+        close(vih->fd);
+    }
+}
+
+static void virtio_input_host_handle_status(VirtIOInput *vinput,
+                                            virtio_input_event *event)
+{
+    VirtIOInputHost *vih = VIRTIO_INPUT_HOST(vinput);
+    struct input_event evdev;
+    int rc;
+
+    if (gettimeofday(&evdev.time, NULL)) {
+        perror("virtio_input_host_handle_status: gettimeofday");
+        return;
+    }
+
+    evdev.type = le16_to_cpu(event->type);
+    evdev.code = le16_to_cpu(event->code);
+    evdev.value = le32_to_cpu(event->value);
+
+    rc = write(vih->fd, &evdev, sizeof(evdev));
+    if (rc == -1) {
+        perror("virtio_input_host_handle_status: write");
+    }
+}
+
+static const VMStateDescription vmstate_virtio_input_host = {
+    .name = "virtio-input-host",
+    .unmigratable = 1,
+};
+
+static Property virtio_input_host_properties[] = {
+    DEFINE_PROP_STRING("evdev", VirtIOInputHost, evdev),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_input_host_class_init(ObjectClass *klass, void *data)
+{
+    VirtIOInputClass *vic = VIRTIO_INPUT_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd           = &vmstate_virtio_input_host;
+    dc->props          = virtio_input_host_properties;
+    vic->realize       = virtio_input_host_realize;
+    vic->unrealize     = virtio_input_host_unrealize;
+    vic->handle_status = virtio_input_host_handle_status;
+}
+
+static void virtio_input_host_init(Object *obj)
+{
+    VirtIOInput *vinput = VIRTIO_INPUT(obj);
+
+    virtio_input_init_config(vinput, virtio_input_host_config);
+}
+
+static const TypeInfo virtio_input_host_info = {
+    .name          = TYPE_VIRTIO_INPUT_HOST,
+    .parent        = TYPE_VIRTIO_INPUT,
+    .instance_size = sizeof(VirtIOInputHost),
+    .instance_init = virtio_input_host_init,
+    .class_init    = virtio_input_host_class_init,
+};
+
+/* ----------------------------------------------------------------- */
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_input_host_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/input/virtio-input.c b/hw/input/virtio-input.c
new file mode 100644
index 0000000..ccdf730
--- /dev/null
+++ b/hw/input/virtio-input.c
@@ -0,0 +1,341 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/iov.h"
+#include "trace.h"
+
+#include "hw/qdev.h"
+#include "hw/virtio/virtio.h"
+#include "hw/virtio/virtio-input.h"
+
+#include "standard-headers/linux/input.h"
+
+#define VIRTIO_INPUT_VM_VERSION 1
+
+/* ----------------------------------------------------------------- */
+
+void virtio_input_send(VirtIOInput *vinput, virtio_input_event *event)
+{
+    VirtQueueElement *elem;
+    unsigned have, need;
+    int i, len;
+
+    if (!vinput->active) {
+        return;
+    }
+
+    /* queue up events ... */
+    if (vinput->qindex == vinput->qsize) {
+        vinput->qsize++;
+        vinput->queue = realloc(vinput->queue, vinput->qsize *
+                                sizeof(virtio_input_event));
+    }
+    vinput->queue[vinput->qindex++] = *event;
+
+    /* ... until we see a report sync ... */
+    if (event->type != cpu_to_le16(EV_SYN) ||
+        event->code != cpu_to_le16(SYN_REPORT)) {
+        return;
+    }
+
+    /* ... then check available space ... */
+    need = sizeof(virtio_input_event) * vinput->qindex;
+    virtqueue_get_avail_bytes(vinput->evt, &have, NULL, need, 0);
+    if (have < need) {
+        vinput->qindex = 0;
+        trace_virtio_input_queue_full();
+        return;
+    }
+
+    /* ... and finally pass them to the guest */
+    for (i = 0; i < vinput->qindex; i++) {
+        elem = virtqueue_pop(vinput->evt, sizeof(VirtQueueElement));
+        if (!elem) {
+            /* should not happen, we've checked for space beforehand */
+            fprintf(stderr, "%s: Huh?  No vq elem available ...\n", __func__);
+            return;
+        }
+        len = iov_from_buf(elem->in_sg, elem->in_num,
+                           0, vinput->queue+i, sizeof(virtio_input_event));
+        virtqueue_push(vinput->evt, elem, len);
+        g_free(elem);
+    }
+    virtio_notify(VIRTIO_DEVICE(vinput), vinput->evt);
+    vinput->qindex = 0;
+}
+
+static void virtio_input_handle_evt(VirtIODevice *vdev, VirtQueue *vq)
+{
+    /* nothing */
+}
+
+static void virtio_input_handle_sts(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(vdev);
+    VirtIOInput *vinput = VIRTIO_INPUT(vdev);
+    virtio_input_event event;
+    VirtQueueElement *elem;
+    int len;
+
+    for (;;) {
+        elem = virtqueue_pop(vinput->sts, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        memset(&event, 0, sizeof(event));
+        len = iov_to_buf(elem->out_sg, elem->out_num,
+                         0, &event, sizeof(event));
+        if (vic->handle_status) {
+            vic->handle_status(vinput, &event);
+        }
+        virtqueue_push(vinput->sts, elem, len);
+        g_free(elem);
+    }
+    virtio_notify(vdev, vinput->sts);
+}
+
+virtio_input_config *virtio_input_find_config(VirtIOInput *vinput,
+                                              uint8_t select,
+                                              uint8_t subsel)
+{
+    VirtIOInputConfig *cfg;
+
+    QTAILQ_FOREACH(cfg, &vinput->cfg_list, node) {
+        if (select == cfg->config.select &&
+            subsel == cfg->config.subsel) {
+            return &cfg->config;
+        }
+    }
+    return NULL;
+}
+
+void virtio_input_add_config(VirtIOInput *vinput,
+                             virtio_input_config *config)
+{
+    VirtIOInputConfig *cfg;
+
+    if (virtio_input_find_config(vinput, config->select, config->subsel)) {
+        /* should not happen */
+        fprintf(stderr, "%s: duplicate config: %d/%d\n",
+                __func__, config->select, config->subsel);
+        abort();
+    }
+
+    cfg = g_new0(VirtIOInputConfig, 1);
+    cfg->config = *config;
+    QTAILQ_INSERT_TAIL(&vinput->cfg_list, cfg, node);
+}
+
+void virtio_input_init_config(VirtIOInput *vinput,
+                              virtio_input_config *config)
+{
+    int i = 0;
+
+    QTAILQ_INIT(&vinput->cfg_list);
+    while (config[i].select) {
+        virtio_input_add_config(vinput, config + i);
+        i++;
+    }
+}
+
+void virtio_input_idstr_config(VirtIOInput *vinput,
+                               uint8_t select, const char *string)
+{
+    virtio_input_config id;
+
+    if (!string) {
+        return;
+    }
+    memset(&id, 0, sizeof(id));
+    id.select = select;
+    id.size = snprintf(id.u.string, sizeof(id.u.string), "%s", string);
+    virtio_input_add_config(vinput, &id);
+}
+
+static void virtio_input_get_config(VirtIODevice *vdev, uint8_t *config_data)
+{
+    VirtIOInput *vinput = VIRTIO_INPUT(vdev);
+    virtio_input_config *config;
+
+    config = virtio_input_find_config(vinput, vinput->cfg_select,
+                                      vinput->cfg_subsel);
+    if (config) {
+        memcpy(config_data, config, vinput->cfg_size);
+    } else {
+        memset(config_data, 0, vinput->cfg_size);
+    }
+}
+
+static void virtio_input_set_config(VirtIODevice *vdev,
+                                    const uint8_t *config_data)
+{
+    VirtIOInput *vinput = VIRTIO_INPUT(vdev);
+    virtio_input_config *config = (virtio_input_config *)config_data;
+
+    vinput->cfg_select = config->select;
+    vinput->cfg_subsel = config->subsel;
+    virtio_notify_config(vdev);
+}
+
+static uint64_t virtio_input_get_features(VirtIODevice *vdev, uint64_t f,
+                                          Error **errp)
+{
+    return f;
+}
+
+static void virtio_input_set_status(VirtIODevice *vdev, uint8_t val)
+{
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(vdev);
+    VirtIOInput *vinput = VIRTIO_INPUT(vdev);
+
+    if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
+        if (!vinput->active) {
+            vinput->active = true;
+            if (vic->change_active) {
+                vic->change_active(vinput);
+            }
+        }
+    }
+}
+
+static void virtio_input_reset(VirtIODevice *vdev)
+{
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(vdev);
+    VirtIOInput *vinput = VIRTIO_INPUT(vdev);
+
+    if (vinput->active) {
+        vinput->active = false;
+        if (vic->change_active) {
+            vic->change_active(vinput);
+        }
+    }
+}
+
+static int virtio_input_load(QEMUFile *f, void *opaque, size_t size)
+{
+    VirtIOInput *vinput = opaque;
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(vinput);
+    VirtIODevice *vdev = VIRTIO_DEVICE(vinput);
+    int ret;
+
+    ret = virtio_load(vdev, f, VIRTIO_INPUT_VM_VERSION);
+    if (ret) {
+        return ret;
+    }
+
+    /* post_load() */
+    vinput->active = vdev->status & VIRTIO_CONFIG_S_DRIVER_OK;
+    if (vic->change_active) {
+        vic->change_active(vinput);
+    }
+    return 0;
+}
+
+static void virtio_input_device_realize(DeviceState *dev, Error **errp)
+{
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(dev);
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    VirtIOInput *vinput = VIRTIO_INPUT(dev);
+    VirtIOInputConfig *cfg;
+    Error *local_err = NULL;
+
+    if (vic->realize) {
+        vic->realize(dev, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+    virtio_input_idstr_config(vinput, VIRTIO_INPUT_CFG_ID_SERIAL,
+                              vinput->serial);
+
+    QTAILQ_FOREACH(cfg, &vinput->cfg_list, node) {
+        if (vinput->cfg_size < cfg->config.size) {
+            vinput->cfg_size = cfg->config.size;
+        }
+    }
+    vinput->cfg_size += 8;
+    assert(vinput->cfg_size <= sizeof(virtio_input_config));
+
+    virtio_init(vdev, "virtio-input", VIRTIO_ID_INPUT,
+                vinput->cfg_size);
+    vinput->evt = virtio_add_queue(vdev, 64, virtio_input_handle_evt);
+    vinput->sts = virtio_add_queue(vdev, 64, virtio_input_handle_sts);
+}
+
+static void virtio_input_finalize(Object *obj)
+{
+    VirtIOInput *vinput = VIRTIO_INPUT(obj);
+    VirtIOInputConfig *cfg, *next;
+
+    QTAILQ_FOREACH_SAFE(cfg, &vinput->cfg_list, node, next) {
+        QTAILQ_REMOVE(&vinput->cfg_list, cfg, node);
+        g_free(cfg);
+    }
+}
+static void virtio_input_device_unrealize(DeviceState *dev, Error **errp)
+{
+    VirtIOInputClass *vic = VIRTIO_INPUT_GET_CLASS(dev);
+    VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    Error *local_err = NULL;
+
+    if (vic->unrealize) {
+        vic->unrealize(dev, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+    virtio_cleanup(vdev);
+}
+
+VMSTATE_VIRTIO_DEVICE(input, VIRTIO_INPUT_VM_VERSION, virtio_input_load,
+                      virtio_vmstate_save);
+
+static Property virtio_input_properties[] = {
+    DEFINE_PROP_STRING("serial", VirtIOInput, serial),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_input_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
+
+    dc->props          = virtio_input_properties;
+    dc->vmsd           = &vmstate_virtio_input;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
+    vdc->realize      = virtio_input_device_realize;
+    vdc->unrealize    = virtio_input_device_unrealize;
+    vdc->get_config   = virtio_input_get_config;
+    vdc->set_config   = virtio_input_set_config;
+    vdc->get_features = virtio_input_get_features;
+    vdc->set_status   = virtio_input_set_status;
+    vdc->reset        = virtio_input_reset;
+}
+
+static const TypeInfo virtio_input_info = {
+    .name          = TYPE_VIRTIO_INPUT,
+    .parent        = TYPE_VIRTIO_DEVICE,
+    .instance_size = sizeof(VirtIOInput),
+    .class_size    = sizeof(VirtIOInputClass),
+    .class_init    = virtio_input_class_init,
+    .abstract      = true,
+    .instance_finalize = virtio_input_finalize,
+};
+
+/* ----------------------------------------------------------------- */
+
+static void virtio_register_types(void)
+{
+    type_register_static(&virtio_input_info);
+}
+
+type_init(virtio_register_types)
diff --git a/hw/input/vmmouse.c b/hw/input/vmmouse.c
index d7b1c76..6d15a88 100644
--- a/hw/input/vmmouse.c
+++ b/hw/input/vmmouse.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/input/ps2.h"
diff --git a/hw/intc/Makefile.objs b/hw/intc/Makefile.objs
index 9776083..fb13a4b 100644
--- a/hw/intc/Makefile.objs
+++ b/hw/intc/Makefile.objs
@@ -11,20 +11,31 @@
 common-obj-$(CONFIG_IOAPIC) += ioapic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic_common.o
 common-obj-$(CONFIG_ARM_GIC) += arm_gic.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv2m.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv3_common.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv3.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv3_dist.o
+common-obj-$(CONFIG_ARM_GIC) += arm_gicv3_redist.o
 common-obj-$(CONFIG_OPENPIC) += openpic.o
 
 obj-$(CONFIG_APIC) += apic.o apic_common.o
 obj-$(CONFIG_ARM_GIC_KVM) += arm_gic_kvm.o
+obj-$(call land,$(CONFIG_ARM_GIC_KVM),$(TARGET_AARCH64)) += arm_gicv3_kvm.o
 obj-$(CONFIG_STELLARIS) += armv7m_nvic.o
 obj-$(CONFIG_EXYNOS4) += exynos4210_gic.o exynos4210_combiner.o
 obj-$(CONFIG_GRLIB) += grlib_irqmp.o
 obj-$(CONFIG_IOAPIC) += ioapic.o
 obj-$(CONFIG_OMAP) += omap_intc.o
 obj-$(CONFIG_OPENPIC_KVM) += openpic_kvm.o
+obj-$(CONFIG_RASPI) += bcm2835_ic.o bcm2836_control.o
 obj-$(CONFIG_SH4) += sh_intc.o
 obj-$(CONFIG_XICS) += xics.o
+obj-$(CONFIG_XICS_SPAPR) += xics_spapr.o
 obj-$(CONFIG_XICS_KVM) += xics_kvm.o
 obj-$(CONFIG_ALLWINNER_A10_PIC) += allwinner-a10-pic.o
 obj-$(CONFIG_S390_FLIC) += s390_flic.o
 obj-$(CONFIG_S390_FLIC_KVM) += s390_flic_kvm.o
+obj-$(CONFIG_ASPEED_SOC) += aspeed_vic.o
+obj-$(CONFIG_ARM_GIC) += arm_gicv3_cpuif.o
+obj-$(CONFIG_MIPS_CPS) += mips_gic.o
 obj-$(CONFIG_GOLDFISH) += goldfish_pic.o
diff --git a/hw/intc/allwinner-a10-pic.c b/hw/intc/allwinner-a10-pic.c
index de820b9..11f1366 100644
--- a/hw/intc/allwinner-a10-pic.c
+++ b/hw/intc/allwinner-a10-pic.c
@@ -15,15 +15,17 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "sysemu/sysemu.h"
 #include "hw/intc/allwinner-a10-pic.h"
+#include "qemu/log.h"
 
 static void aw_a10_pic_update(AwA10PICState *s)
 {
     uint8_t i;
-    int irq = 0, fiq = 0, pending;
+    int irq = 0, fiq = 0, zeroes;
 
     s->vector = 0;
 
@@ -32,9 +34,9 @@
         fiq |= s->select[i] & s->irq_pending[i] & ~s->mask[i];
 
         if (!s->vector) {
-            pending = ffs(s->irq_pending[i] & ~s->mask[i]);
-            if (pending) {
-                s->vector = (i * 32 + pending - 1) * 4;
+            zeroes = ctz32(s->irq_pending[i] & ~s->mask[i]);
+            if (zeroes != 32) {
+                s->vector = (i * 32 + zeroes) * 4;
             }
         }
     }
diff --git a/hw/intc/apic.c b/hw/intc/apic.c
index 0f97b47..45887d9 100644
--- a/hw/intc/apic.c
+++ b/hw/intc/apic.c
@@ -16,6 +16,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/thread.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/i386/apic.h"
@@ -25,7 +28,9 @@
 #include "trace.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic-msidef.h"
+#include "qapi/error.h"
 
+#define MAX_APICS 255
 #define MAX_APIC_WORDS 8
 
 #define SYNC_FROM_VAPIC                 0x1
@@ -51,14 +56,6 @@
     return ctz32(value);
 }
 
-static inline void apic_set_bit(uint32_t *tab, int index)
-{
-    int i, mask;
-    i = index >> 5;
-    mask = 1 << (index & 0x1f);
-    tab[i] |= mask;
-}
-
 static inline void apic_reset_bit(uint32_t *tab, int index)
 {
     int i, mask;
@@ -67,14 +64,6 @@
     tab[i] &= ~mask;
 }
 
-static inline int apic_get_bit(uint32_t *tab, int index)
-{
-    int i, mask;
-    i = index >> 5;
-    mask = 1 << (index & 0x1f);
-    return !!(tab[i] & mask);
-}
-
 /* return -1 if no bit is set */
 static int get_highest_priority_int(uint32_t *tab)
 {
@@ -318,7 +307,7 @@
     return s->tpr >> 4;
 }
 
-static int apic_get_ppr(APICCommonState *s)
+int apic_get_ppr(APICCommonState *s)
 {
     int tpr, isrv, ppr;
 
@@ -370,13 +359,14 @@
 static void apic_update_irq(APICCommonState *s)
 {
     CPUState *cpu;
+    DeviceState *dev = (DeviceState *)s;
 
     cpu = CPU(s->cpu);
     if (!qemu_cpu_is_self(cpu)) {
         cpu_interrupt(cpu, CPU_INTERRUPT_POLL);
     } else if (apic_irq_pending(s) > 0) {
         cpu_interrupt(cpu, CPU_INTERRUPT_HARD);
-    } else if (!apic_accept_pic_intr(&s->busdev.qdev) || !pic_get_output(isa_pic)) {
+    } else if (!apic_accept_pic_intr(dev) || !pic_get_output(isa_pic)) {
         cpu_reset_interrupt(cpu, CPU_INTERRUPT_HARD);
     }
 }
@@ -431,7 +421,7 @@
     int i;
 
     if (apic && apic->id == dest)
-        return dest;  /* shortcut in case apic->id == apic->idx */
+        return dest;  /* shortcut in case apic->id == local_apics[dest]->id */
 
     for (i = 0; i < MAX_APICS; i++) {
         apic = local_apics[i];
@@ -514,14 +504,14 @@
         break;
     case 1:
         memset(deliver_bitmask, 0x00, sizeof(deliver_bitmask));
-        apic_set_bit(deliver_bitmask, s->idx);
+        apic_set_bit(deliver_bitmask, s->id);
         break;
     case 2:
         memset(deliver_bitmask, 0xff, sizeof(deliver_bitmask));
         break;
     case 3:
         memset(deliver_bitmask, 0xff, sizeof(deliver_bitmask));
-        apic_reset_bit(deliver_bitmask, s->idx);
+        apic_reset_bit(deliver_bitmask, s->id);
         break;
     }
 
@@ -549,10 +539,12 @@
 
 static bool apic_check_pic(APICCommonState *s)
 {
-    if (!apic_accept_pic_intr(&s->busdev.qdev) || !pic_get_output(isa_pic)) {
+    DeviceState *dev = (DeviceState *)s;
+
+    if (!apic_accept_pic_intr(dev) || !pic_get_output(isa_pic)) {
         return false;
     }
-    apic_deliver_pic_intr(&s->busdev.qdev, 1);
+    apic_deliver_pic_intr(dev, 1);
     return true;
 }
 
@@ -736,7 +728,7 @@
         val = s->divide_conf;
         break;
     default:
-        s->esr |= ESR_ILLEGAL_ADDRESS;
+        s->esr |= APIC_ESR_ILLEGAL_ADDRESS;
         val = 0;
         break;
     }
@@ -849,7 +841,7 @@
         }
         break;
     default:
-        s->esr |= ESR_ILLEGAL_ADDRESS;
+        s->esr |= APIC_ESR_ILLEGAL_ADDRESS;
         break;
     }
 }
@@ -880,13 +872,28 @@
 {
     APICCommonState *s = APIC_COMMON(dev);
 
+    if (s->id >= MAX_APICS) {
+        error_setg(errp, "%s initialization failed. APIC ID %d is invalid",
+                   object_get_typename(OBJECT(dev)), s->id);
+        return;
+    }
+
     memory_region_init_io(&s->io_memory, OBJECT(s), &apic_io_ops, s, "apic-msi",
                           APIC_SPACE_SIZE);
 
     s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, apic_timer, s);
-    local_apics[s->idx] = s;
+    local_apics[s->id] = s;
 
-    msi_supported = true;
+    msi_nonbroken = true;
+}
+
+static void apic_unrealize(DeviceState *dev, Error **errp)
+{
+    APICCommonState *s = APIC_COMMON(dev);
+
+    timer_del(s->timer);
+    timer_free(s->timer);
+    local_apics[s->id] = NULL;
 }
 
 static void apic_class_init(ObjectClass *klass, void *data)
@@ -894,6 +901,7 @@
     APICCommonClass *k = APIC_COMMON_CLASS(klass);
 
     k->realize = apic_realize;
+    k->unrealize = apic_unrealize;
     k->set_base = apic_set_base;
     k->set_tpr = apic_set_tpr;
     k->get_tpr = apic_get_tpr;
diff --git a/hw/intc/apic_common.c b/hw/intc/apic_common.c
index 6b5571f..cc77d80 100644
--- a/hw/intc/apic_common.c
+++ b/hw/intc/apic_common.c
@@ -17,6 +17,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/i386/apic.h"
 #include "hw/i386/apic_internal.h"
 #include "trace.h"
@@ -177,12 +181,14 @@
 
 void apic_init_reset(DeviceState *dev)
 {
-    APICCommonState *s = APIC_COMMON(dev);
+    APICCommonState *s;
+    APICCommonClass *info;
     int i;
 
-    if (!s) {
+    if (!dev) {
         return;
     }
+    s = APIC_COMMON(dev);
     s->tpr = 0;
     s->spurious_vec = 0xff;
     s->log_dest = 0;
@@ -206,41 +212,40 @@
         timer_del(s->timer);
     }
     s->timer_expiry = -1;
+
+    info = APIC_COMMON_GET_CLASS(s);
+    if (info->reset) {
+        info->reset(s);
+    }
 }
 
-void apic_designate_bsp(DeviceState *dev)
+void apic_designate_bsp(DeviceState *dev, bool bsp)
 {
     if (dev == NULL) {
         return;
     }
 
     APICCommonState *s = APIC_COMMON(dev);
-    s->apicbase |= MSR_IA32_APICBASE_BSP;
+    if (bsp) {
+        s->apicbase |= MSR_IA32_APICBASE_BSP;
+    } else {
+        s->apicbase &= ~MSR_IA32_APICBASE_BSP;
+    }
 }
 
 static void apic_reset_common(DeviceState *dev)
 {
     APICCommonState *s = APIC_COMMON(dev);
     APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
-    bool bsp;
+    uint32_t bsp;
 
-    bsp = cpu_is_bsp(s->cpu);
-    s->apicbase = APIC_DEFAULT_ADDRESS |
-        (bsp ? MSR_IA32_APICBASE_BSP : 0) | MSR_IA32_APICBASE_ENABLE;
+    bsp = s->apicbase & MSR_IA32_APICBASE_BSP;
+    s->apicbase = APIC_DEFAULT_ADDRESS | bsp | MSR_IA32_APICBASE_ENABLE;
 
     s->vapic_paddr = 0;
     info->vapic_base_update(s);
 
     apic_init_reset(dev);
-
-    if (bsp) {
-        /*
-         * LINT0 delivery mode on CPU #0 is set to ExtInt at initialization
-         * time typically by BIOS, so PIC interrupt can be delivered to the
-         * processor when local APIC is enabled.
-         */
-        s->lvt[APIC_LVT_LINT0] = 0x700;
-    }
 }
 
 /* This function is only used for old state version 1 and 2 */
@@ -289,42 +294,50 @@
     return 0;
 }
 
+static const VMStateDescription vmstate_apic_common;
+
 static void apic_common_realize(DeviceState *dev, Error **errp)
 {
     APICCommonState *s = APIC_COMMON(dev);
     APICCommonClass *info;
     static DeviceState *vapic;
-    static int apic_no;
-    static bool mmio_registered;
-
-    if (apic_no >= MAX_APICS) {
-        error_setg(errp, "%s initialization failed.",
-                   object_get_typename(OBJECT(dev)));
-        return;
-    }
-    s->idx = apic_no++;
+    int instance_id = s->id;
 
     info = APIC_COMMON_GET_CLASS(s);
     info->realize(dev, errp);
-    if (!mmio_registered) {
-        ICCBus *b = ICC_BUS(qdev_get_parent_bus(dev));
-        memory_region_add_subregion(b->apic_address_space, 0, &s->io_memory);
-        mmio_registered = true;
-    }
 
+    /* NOTE: Why this needs to be disabled for HAX exactly? */
 #ifndef CONFIG_HAX
     /* Note: We need at least 1M to map the VAPIC option ROM */
     if (!vapic && s->vapic_control & VAPIC_ENABLE_MASK &&
         ram_size >= 1024 * 1024) {
         vapic = sysbus_create_simple("kvmvapic", -1, NULL);
     }
-#endif
+#endif /* CONFIG_HAX */
 
     s->vapic = vapic;
     if (apic_report_tpr_access && info->enable_tpr_reporting) {
         info->enable_tpr_reporting(s, true);
     }
 
+    if (s->legacy_instance_id) {
+        instance_id = -1;
+    }
+    vmstate_register_with_alias_id(NULL, instance_id, &vmstate_apic_common,
+                                   s, -1, 0);
+}
+
+static void apic_common_unrealize(DeviceState *dev, Error **errp)
+{
+    APICCommonState *s = APIC_COMMON(dev);
+    APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
+
+    vmstate_unregister(NULL, &vmstate_apic_common, s);
+    info->unrealize(dev, errp);
+
+    if (apic_report_tpr_access && info->enable_tpr_reporting) {
+        info->enable_tpr_reporting(s, false);
+    }
 }
 
 static int apic_pre_load(void *opaque)
@@ -371,6 +384,7 @@
     .name = "apic_sipi",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = apic_common_sipi_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32(sipi_vector, APICCommonState),
         VMSTATE_INT32(wait_for_sipi, APICCommonState),
@@ -410,12 +424,9 @@
                       APICCommonState), /* open-coded timer state */
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_apic_common_sipi,
-            .needed = apic_common_sipi_needed,
-        },
-        VMSTATE_END_OF_LIST()
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_apic_common_sipi,
+        NULL
     }
 };
 
@@ -424,18 +435,19 @@
     DEFINE_PROP_UINT8("version", APICCommonState, version, 0x14),
     DEFINE_PROP_BIT("vapic", APICCommonState, vapic_control, VAPIC_ENABLE_BIT,
                     true),
+    DEFINE_PROP_BOOL("legacy-instance-id", APICCommonState, legacy_instance_id,
+                     false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
 static void apic_common_class_init(ObjectClass *klass, void *data)
 {
-    ICCDeviceClass *idc = ICC_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    dc->vmsd = &vmstate_apic_common;
     dc->reset = apic_reset_common;
     dc->props = apic_properties_common;
-    idc->realize = apic_common_realize;
+    dc->realize = apic_common_realize;
+    dc->unrealize = apic_common_unrealize;
     /*
      * Reason: APIC and CPU need to be wired up by
      * x86_cpu_apic_create()
@@ -445,7 +457,7 @@
 
 static const TypeInfo apic_common_type = {
     .name = TYPE_APIC_COMMON,
-    .parent = TYPE_ICC_DEVICE,
+    .parent = TYPE_DEVICE,
     .instance_size = sizeof(APICCommonState),
     .class_size = sizeof(APICCommonClass),
     .class_init = apic_common_class_init,
diff --git a/hw/intc/arm_gic.c b/hw/intc/arm_gic.c
index 270ce05..b30cc91 100644
--- a/hw/intc/arm_gic.c
+++ b/hw/intc/arm_gic.c
@@ -18,9 +18,13 @@
  *  armv7m_nvic device.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "gic_internal.h"
+#include "qapi/error.h"
 #include "qom/cpu.h"
+#include "qemu/log.h"
+#include "trace.h"
 
 //#define DEBUG_GIC
 
@@ -31,11 +35,17 @@
 #define DPRINTF(fmt, ...) do {} while(0)
 #endif
 
-static const uint8_t gic_id[] = {
-    0x90, 0x13, 0x04, 0x00, 0x0d, 0xf0, 0x05, 0xb1
+static const uint8_t gic_id_11mpcore[] = {
+    0x00, 0x00, 0x00, 0x00, 0x90, 0x13, 0x04, 0x00, 0x0d, 0xf0, 0x05, 0xb1
 };
 
-#define NUM_CPU(s) ((s)->num_cpu)
+static const uint8_t gic_id_gicv1[] = {
+    0x04, 0x00, 0x00, 0x00, 0x90, 0xb3, 0x1b, 0x00, 0x0d, 0xf0, 0x05, 0xb1
+};
+
+static const uint8_t gic_id_gicv2[] = {
+    0x04, 0x00, 0x00, 0x00, 0x90, 0xb4, 0x2b, 0x00, 0x0d, 0xf0, 0x05, 0xb1
+};
 
 static inline int gic_get_current_cpu(GICState *s)
 {
@@ -45,6 +55,14 @@
     return 0;
 }
 
+/* Return true if this GIC config has interrupt groups, which is
+ * true if we're a GICv2, or a GICv1 with the security extensions.
+ */
+static inline bool gic_has_groups(GICState *s)
+{
+    return s->revision == 2 || s->security_extn;
+}
+
 /* TODO: Many places that call this routine could be optimized.  */
 /* Update interrupt status after enabled or pending bits have been changed.  */
 void gic_update(GICState *s)
@@ -52,16 +70,18 @@
     int best_irq;
     int best_prio;
     int irq;
-    int level;
+    int irq_level, fiq_level;
     int cpu;
     int cm;
 
-    for (cpu = 0; cpu < NUM_CPU(s); cpu++) {
+    for (cpu = 0; cpu < s->num_cpu; cpu++) {
         cm = 1 << cpu;
         s->current_pending[cpu] = 1023;
-        if (!s->enabled || !s->cpu_enabled[cpu]) {
+        if (!(s->ctlr & (GICD_CTLR_EN_GRP0 | GICD_CTLR_EN_GRP1))
+            || !(s->cpu_ctlr[cpu] & (GICC_CTLR_EN_GRP0 | GICC_CTLR_EN_GRP1))) {
             qemu_irq_lower(s->parent_irq[cpu]);
-            return;
+            qemu_irq_lower(s->parent_fiq[cpu]);
+            continue;
         }
         best_prio = 0x100;
         best_irq = 1023;
@@ -74,15 +94,38 @@
                 }
             }
         }
-        level = 0;
+
+        if (best_irq != 1023) {
+            trace_gic_update_bestirq(cpu, best_irq, best_prio,
+                s->priority_mask[cpu], s->running_priority[cpu]);
+        }
+
+        irq_level = fiq_level = 0;
+
         if (best_prio < s->priority_mask[cpu]) {
             s->current_pending[cpu] = best_irq;
             if (best_prio < s->running_priority[cpu]) {
-                DPRINTF("Raised pending IRQ %d (cpu %d)\n", best_irq, cpu);
-                level = 1;
+                int group = GIC_TEST_GROUP(best_irq, cm);
+
+                if (extract32(s->ctlr, group, 1) &&
+                    extract32(s->cpu_ctlr[cpu], group, 1)) {
+                    if (group == 0 && s->cpu_ctlr[cpu] & GICC_CTLR_FIQ_EN) {
+                        DPRINTF("Raised pending FIQ %d (cpu %d)\n",
+                                best_irq, cpu);
+                        fiq_level = 1;
+                        trace_gic_update_set_irq(cpu, "fiq", fiq_level);
+                    } else {
+                        DPRINTF("Raised pending IRQ %d (cpu %d)\n",
+                                best_irq, cpu);
+                        irq_level = 1;
+                        trace_gic_update_set_irq(cpu, "irq", irq_level);
+                    }
+                }
             }
         }
-        qemu_set_irq(s->parent_irq[cpu], level);
+
+        qemu_set_irq(s->parent_irq[cpu], irq_level);
+        qemu_set_irq(s->parent_fiq[cpu], fiq_level);
     }
 }
 
@@ -163,32 +206,153 @@
     } else {
         gic_set_irq_generic(s, irq, level, cm, target);
     }
+    trace_gic_set_irq(irq, level, cm, target);
 
     gic_update(s);
 }
 
-static void gic_set_running_irq(GICState *s, int cpu, int irq)
+static uint16_t gic_get_current_pending_irq(GICState *s, int cpu,
+                                            MemTxAttrs attrs)
 {
-    s->running_irq[cpu] = irq;
-    if (irq == 1023) {
-        s->running_priority[cpu] = 0x100;
-    } else {
-        s->running_priority[cpu] = GIC_GET_PRIORITY(irq, cpu);
+    uint16_t pending_irq = s->current_pending[cpu];
+
+    if (pending_irq < GIC_MAXIRQ && gic_has_groups(s)) {
+        int group = GIC_TEST_GROUP(pending_irq, (1 << cpu));
+        /* On a GIC without the security extensions, reading this register
+         * behaves in the same way as a secure access to a GIC with them.
+         */
+        bool secure = !s->security_extn || attrs.secure;
+
+        if (group == 0 && !secure) {
+            /* Group0 interrupts hidden from Non-secure access */
+            return 1023;
+        }
+        if (group == 1 && secure && !(s->cpu_ctlr[cpu] & GICC_CTLR_ACK_CTL)) {
+            /* Group1 interrupts only seen by Secure access if
+             * AckCtl bit set.
+             */
+            return 1022;
+        }
     }
-    gic_update(s);
+    return pending_irq;
 }
 
-uint32_t gic_acknowledge_irq(GICState *s, int cpu)
+static int gic_get_group_priority(GICState *s, int cpu, int irq)
+{
+    /* Return the group priority of the specified interrupt
+     * (which is the top bits of its priority, with the number
+     * of bits masked determined by the applicable binary point register).
+     */
+    int bpr;
+    uint32_t mask;
+
+    if (gic_has_groups(s) &&
+        !(s->cpu_ctlr[cpu] & GICC_CTLR_CBPR) &&
+        GIC_TEST_GROUP(irq, (1 << cpu))) {
+        bpr = s->abpr[cpu];
+    } else {
+        bpr = s->bpr[cpu];
+    }
+
+    /* a BPR of 0 means the group priority bits are [7:1];
+     * a BPR of 1 means they are [7:2], and so on down to
+     * a BPR of 7 meaning no group priority bits at all.
+     */
+    mask = ~0U << ((bpr & 7) + 1);
+
+    return GIC_GET_PRIORITY(irq, cpu) & mask;
+}
+
+static void gic_activate_irq(GICState *s, int cpu, int irq)
+{
+    /* Set the appropriate Active Priority Register bit for this IRQ,
+     * and update the running priority.
+     */
+    int prio = gic_get_group_priority(s, cpu, irq);
+    int preemption_level = prio >> (GIC_MIN_BPR + 1);
+    int regno = preemption_level / 32;
+    int bitno = preemption_level % 32;
+
+    if (gic_has_groups(s) && GIC_TEST_GROUP(irq, (1 << cpu))) {
+        s->nsapr[regno][cpu] |= (1 << bitno);
+    } else {
+        s->apr[regno][cpu] |= (1 << bitno);
+    }
+
+    s->running_priority[cpu] = prio;
+    GIC_SET_ACTIVE(irq, 1 << cpu);
+}
+
+static int gic_get_prio_from_apr_bits(GICState *s, int cpu)
+{
+    /* Recalculate the current running priority for this CPU based
+     * on the set bits in the Active Priority Registers.
+     */
+    int i;
+    for (i = 0; i < GIC_NR_APRS; i++) {
+        uint32_t apr = s->apr[i][cpu] | s->nsapr[i][cpu];
+        if (!apr) {
+            continue;
+        }
+        return (i * 32 + ctz32(apr)) << (GIC_MIN_BPR + 1);
+    }
+    return 0x100;
+}
+
+static void gic_drop_prio(GICState *s, int cpu, int group)
+{
+    /* Drop the priority of the currently active interrupt in the
+     * specified group.
+     *
+     * Note that we can guarantee (because of the requirement to nest
+     * GICC_IAR reads [which activate an interrupt and raise priority]
+     * with GICC_EOIR writes [which drop the priority for the interrupt])
+     * that the interrupt we're being called for is the highest priority
+     * active interrupt, meaning that it has the lowest set bit in the
+     * APR registers.
+     *
+     * If the guest does not honour the ordering constraints then the
+     * behaviour of the GIC is UNPREDICTABLE, which for us means that
+     * the values of the APR registers might become incorrect and the
+     * running priority will be wrong, so interrupts that should preempt
+     * might not do so, and interrupts that should not preempt might do so.
+     */
+    int i;
+
+    for (i = 0; i < GIC_NR_APRS; i++) {
+        uint32_t *papr = group ? &s->nsapr[i][cpu] : &s->apr[i][cpu];
+        if (!*papr) {
+            continue;
+        }
+        /* Clear lowest set bit */
+        *papr &= *papr - 1;
+        break;
+    }
+
+    s->running_priority[cpu] = gic_get_prio_from_apr_bits(s, cpu);
+}
+
+uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs)
 {
     int ret, irq, src;
     int cm = 1 << cpu;
-    irq = s->current_pending[cpu];
-    if (irq == 1023
-            || GIC_GET_PRIORITY(irq, cpu) >= s->running_priority[cpu]) {
-        DPRINTF("ACK no pending IRQ\n");
+
+    /* gic_get_current_pending_irq() will return 1022 or 1023 appropriately
+     * for the case where this GIC supports grouping and the pending interrupt
+     * is in the wrong group.
+     */
+    irq = gic_get_current_pending_irq(s, cpu, attrs);
+    trace_gic_acknowledge_irq(cpu, irq);
+
+    if (irq >= GIC_MAXIRQ) {
+        DPRINTF("ACK, no pending interrupt or it is hidden: %d\n", irq);
+        return irq;
+    }
+
+    if (GIC_GET_PRIORITY(irq, cpu) >= s->running_priority[cpu]) {
+        DPRINTF("ACK, pending interrupt (%d) has insufficient priority\n", irq);
         return 1023;
     }
-    s->last_active[irq][cpu] = s->running_irq[cpu];
 
     if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
         /* Clear pending flags for both level and edge triggered interrupts.
@@ -219,13 +383,22 @@
         }
     }
 
-    gic_set_running_irq(s, cpu, irq);
+    gic_activate_irq(s, cpu, irq);
+    gic_update(s);
     DPRINTF("ACK %d\n", irq);
     return ret;
 }
 
-void gic_set_priority(GICState *s, int cpu, int irq, uint8_t val)
+void gic_set_priority(GICState *s, int cpu, int irq, uint8_t val,
+                      MemTxAttrs attrs)
 {
+    if (s->security_extn && !attrs.secure) {
+        if (!GIC_TEST_GROUP(irq, (1 << cpu))) {
+            return; /* Ignore Non-secure access of Group0 IRQ */
+        }
+        val = 0x80 | (val >> 1); /* Non-secure view */
+    }
+
     if (irq < GIC_INTERNAL) {
         s->priority1[irq][cpu] = val;
     } else {
@@ -233,10 +406,152 @@
     }
 }
 
-void gic_complete_irq(GICState *s, int cpu, int irq)
+static uint32_t gic_get_priority(GICState *s, int cpu, int irq,
+                                 MemTxAttrs attrs)
 {
-    int update = 0;
+    uint32_t prio = GIC_GET_PRIORITY(irq, cpu);
+
+    if (s->security_extn && !attrs.secure) {
+        if (!GIC_TEST_GROUP(irq, (1 << cpu))) {
+            return 0; /* Non-secure access cannot read priority of Group0 IRQ */
+        }
+        prio = (prio << 1) & 0xff; /* Non-secure view */
+    }
+    return prio;
+}
+
+static void gic_set_priority_mask(GICState *s, int cpu, uint8_t pmask,
+                                  MemTxAttrs attrs)
+{
+    if (s->security_extn && !attrs.secure) {
+        if (s->priority_mask[cpu] & 0x80) {
+            /* Priority Mask in upper half */
+            pmask = 0x80 | (pmask >> 1);
+        } else {
+            /* Non-secure write ignored if priority mask is in lower half */
+            return;
+        }
+    }
+    s->priority_mask[cpu] = pmask;
+}
+
+static uint32_t gic_get_priority_mask(GICState *s, int cpu, MemTxAttrs attrs)
+{
+    uint32_t pmask = s->priority_mask[cpu];
+
+    if (s->security_extn && !attrs.secure) {
+        if (pmask & 0x80) {
+            /* Priority Mask in upper half, return Non-secure view */
+            pmask = (pmask << 1) & 0xff;
+        } else {
+            /* Priority Mask in lower half, RAZ */
+            pmask = 0;
+        }
+    }
+    return pmask;
+}
+
+static uint32_t gic_get_cpu_control(GICState *s, int cpu, MemTxAttrs attrs)
+{
+    uint32_t ret = s->cpu_ctlr[cpu];
+
+    if (s->security_extn && !attrs.secure) {
+        /* Construct the NS banked view of GICC_CTLR from the correct
+         * bits of the S banked view. We don't need to move the bypass
+         * control bits because we don't implement that (IMPDEF) part
+         * of the GIC architecture.
+         */
+        ret = (ret & (GICC_CTLR_EN_GRP1 | GICC_CTLR_EOIMODE_NS)) >> 1;
+    }
+    return ret;
+}
+
+static void gic_set_cpu_control(GICState *s, int cpu, uint32_t value,
+                                MemTxAttrs attrs)
+{
+    uint32_t mask;
+
+    if (s->security_extn && !attrs.secure) {
+        /* The NS view can only write certain bits in the register;
+         * the rest are unchanged
+         */
+        mask = GICC_CTLR_EN_GRP1;
+        if (s->revision == 2) {
+            mask |= GICC_CTLR_EOIMODE_NS;
+        }
+        s->cpu_ctlr[cpu] &= ~mask;
+        s->cpu_ctlr[cpu] |= (value << 1) & mask;
+    } else {
+        if (s->revision == 2) {
+            mask = s->security_extn ? GICC_CTLR_V2_S_MASK : GICC_CTLR_V2_MASK;
+        } else {
+            mask = s->security_extn ? GICC_CTLR_V1_S_MASK : GICC_CTLR_V1_MASK;
+        }
+        s->cpu_ctlr[cpu] = value & mask;
+    }
+    DPRINTF("CPU Interface %d: Group0 Interrupts %sabled, "
+            "Group1 Interrupts %sabled\n", cpu,
+            (s->cpu_ctlr[cpu] & GICC_CTLR_EN_GRP0) ? "En" : "Dis",
+            (s->cpu_ctlr[cpu] & GICC_CTLR_EN_GRP1) ? "En" : "Dis");
+}
+
+static uint8_t gic_get_running_priority(GICState *s, int cpu, MemTxAttrs attrs)
+{
+    if (s->security_extn && !attrs.secure) {
+        if (s->running_priority[cpu] & 0x80) {
+            /* Running priority in upper half of range: return the Non-secure
+             * view of the priority.
+             */
+            return s->running_priority[cpu] << 1;
+        } else {
+            /* Running priority in lower half of range: RAZ */
+            return 0;
+        }
+    } else {
+        return s->running_priority[cpu];
+    }
+}
+
+/* Return true if we should split priority drop and interrupt deactivation,
+ * ie whether the relevant EOIMode bit is set.
+ */
+static bool gic_eoi_split(GICState *s, int cpu, MemTxAttrs attrs)
+{
+    if (s->revision != 2) {
+        /* Before GICv2 prio-drop and deactivate are not separable */
+        return false;
+    }
+    if (s->security_extn && !attrs.secure) {
+        return s->cpu_ctlr[cpu] & GICC_CTLR_EOIMODE_NS;
+    }
+    return s->cpu_ctlr[cpu] & GICC_CTLR_EOIMODE;
+}
+
+static void gic_deactivate_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
+{
     int cm = 1 << cpu;
+    int group = gic_has_groups(s) && GIC_TEST_GROUP(irq, cm);
+
+    if (!gic_eoi_split(s, cpu, attrs)) {
+        /* This is UNPREDICTABLE; we choose to ignore it */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_deactivate_irq: GICC_DIR write when EOIMode clear");
+        return;
+    }
+
+    if (s->security_extn && !attrs.secure && !group) {
+        DPRINTF("Non-secure DI for Group0 interrupt %d ignored\n", irq);
+        return;
+    }
+
+    GIC_CLEAR_ACTIVE(irq, cm);
+}
+
+void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs)
+{
+    int cm = 1 << cpu;
+    int group;
+
     DPRINTF("EOI %d\n", irq);
     if (irq >= s->num_irq) {
         /* This handles two cases:
@@ -249,8 +564,9 @@
          */
         return;
     }
-    if (s->running_irq[cpu] == 1023)
+    if (s->running_priority[cpu] == 0x100) {
         return; /* No active IRQ.  */
+    }
 
     if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
         /* Mark level triggered interrupts as pending if they are still
@@ -259,30 +575,31 @@
             && GIC_TEST_LEVEL(irq, cm) && (GIC_TARGET(irq) & cm) != 0) {
             DPRINTF("Set %d pending mask %x\n", irq, cm);
             GIC_SET_PENDING(irq, cm);
-            update = 1;
         }
     }
 
-    if (irq != s->running_irq[cpu]) {
-        /* Complete an IRQ that is not currently running.  */
-        int tmp = s->running_irq[cpu];
-        while (s->last_active[tmp][cpu] != 1023) {
-            if (s->last_active[tmp][cpu] == irq) {
-                s->last_active[tmp][cpu] = s->last_active[irq][cpu];
-                break;
-            }
-            tmp = s->last_active[tmp][cpu];
-        }
-        if (update) {
-            gic_update(s);
-        }
-    } else {
-        /* Complete the current running IRQ.  */
-        gic_set_running_irq(s, cpu, s->last_active[s->running_irq[cpu]][cpu]);
+    group = gic_has_groups(s) && GIC_TEST_GROUP(irq, cm);
+
+    if (s->security_extn && !attrs.secure && !group) {
+        DPRINTF("Non-secure EOI for Group0 interrupt %d ignored\n", irq);
+        return;
     }
+
+    /* Secure EOI with GICC_CTLR.AckCtl == 0 when the IRQ is a Group 1
+     * interrupt is UNPREDICTABLE. We choose to handle it as if AckCtl == 1,
+     * i.e. go ahead and complete the irq anyway.
+     */
+
+    gic_drop_prio(s, cpu, group);
+
+    /* In GICv2 the guest can choose to split priority-drop and deactivate */
+    if (!gic_eoi_split(s, cpu, attrs)) {
+        GIC_CLEAR_ACTIVE(irq, cm);
+    }
+    gic_update(s);
 }
 
-static uint32_t gic_dist_readb(void *opaque, hwaddr offset)
+static uint32_t gic_dist_readb(void *opaque, hwaddr offset, MemTxAttrs attrs)
 {
     GICState *s = (GICState *)opaque;
     uint32_t res;
@@ -295,15 +612,42 @@
     cpu = gic_get_current_cpu(s);
     cm = 1 << cpu;
     if (offset < 0x100) {
-        if (offset == 0)
-            return s->enabled;
+        if (offset == 0) {      /* GICD_CTLR */
+            if (s->security_extn && !attrs.secure) {
+                /* The NS bank of this register is just an alias of the
+                 * EnableGrp1 bit in the S bank version.
+                 */
+                return extract32(s->ctlr, 1, 1);
+            } else {
+                return s->ctlr;
+            }
+        }
         if (offset == 4)
-            return ((s->num_irq / 32) - 1) | ((NUM_CPU(s) - 1) << 5);
+            /* Interrupt Controller Type Register */
+            return ((s->num_irq / 32) - 1)
+                    | ((s->num_cpu - 1) << 5)
+                    | (s->security_extn << 10);
         if (offset < 0x08)
             return 0;
         if (offset >= 0x80) {
-            /* Interrupt Security , RAZ/WI */
-            return 0;
+            /* Interrupt Group Registers: these RAZ/WI if this is an NS
+             * access to a GIC with the security extensions, or if the GIC
+             * doesn't have groups at all.
+             */
+            res = 0;
+            if (!(s->security_extn && !attrs.secure) && gic_has_groups(s)) {
+                /* Every byte offset holds 8 group status bits */
+                irq = (offset - 0x080) * 8 + GIC_BASE_IRQ;
+                if (irq >= s->num_irq) {
+                    goto bad_reg;
+                }
+                for (i = 0; i < 8; i++) {
+                    if (GIC_TEST_GROUP(irq + i, cm)) {
+                        res |= (1 << i);
+                    }
+                }
+            }
+            return res;
         }
         goto bad_reg;
     } else if (offset < 0x200) {
@@ -317,6 +661,11 @@
             goto bad_reg;
         res = 0;
         for (i = 0; i < 8; i++) {
+            if (s->security_extn && !attrs.secure &&
+                !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                continue; /* Ignore Non-secure access of Group0 IRQ */
+            }
+
             if (GIC_TEST_ENABLED(irq + i, cm)) {
                 res |= (1 << i);
             }
@@ -333,6 +682,11 @@
         res = 0;
         mask = (irq < GIC_INTERNAL) ?  cm : ALL_CPU_MASK;
         for (i = 0; i < 8; i++) {
+            if (s->security_extn && !attrs.secure &&
+                !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                continue; /* Ignore Non-secure access of Group0 IRQ */
+            }
+
             if (gic_test_pending(s, irq + i, mask)) {
                 res |= (1 << i);
             }
@@ -345,6 +699,11 @@
         res = 0;
         mask = (irq < GIC_INTERNAL) ?  cm : ALL_CPU_MASK;
         for (i = 0; i < 8; i++) {
+            if (s->security_extn && !attrs.secure &&
+                !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                continue; /* Ignore Non-secure access of Group0 IRQ */
+            }
+
             if (GIC_TEST_ACTIVE(irq + i, mask)) {
                 res |= (1 << i);
             }
@@ -354,7 +713,7 @@
         irq = (offset - 0x400) + GIC_BASE_IRQ;
         if (irq >= s->num_irq)
             goto bad_reg;
-        res = GIC_GET_PRIORITY(irq, cpu);
+        res = gic_get_priority(s, cpu, irq, attrs);
     } else if (offset < 0xc00) {
         /* Interrupt CPU Target.  */
         if (s->num_cpu == 1 && s->revision != REV_11MPCORE) {
@@ -378,6 +737,11 @@
             goto bad_reg;
         res = 0;
         for (i = 0; i < 4; i++) {
+            if (s->security_extn && !attrs.secure &&
+                !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                continue; /* Ignore Non-secure access of Group0 IRQ */
+            }
+
             if (GIC_TEST_MODEL(irq + i))
                 res |= (1 << (i * 2));
             if (GIC_TEST_EDGE_TRIGGER(irq + i))
@@ -398,15 +762,37 @@
             /* GICD_SPENDSGIRn */
         }
 
-        res = s->sgi_pending[irq][cpu];
-    } else if (offset < 0xfe0) {
+        if (s->security_extn && !attrs.secure &&
+            !GIC_TEST_GROUP(irq, 1 << cpu)) {
+            res = 0; /* Ignore Non-secure access of Group0 IRQ */
+        } else {
+            res = s->sgi_pending[irq][cpu];
+        }
+    } else if (offset < 0xfd0) {
         goto bad_reg;
-    } else /* offset >= 0xfe0 */ {
+    } else if (offset < 0x1000) {
         if (offset & 3) {
             res = 0;
         } else {
-            res = gic_id[(offset - 0xfe0) >> 2];
+            switch (s->revision) {
+            case REV_11MPCORE:
+                res = gic_id_11mpcore[(offset - 0xfd0) >> 2];
+                break;
+            case 1:
+                res = gic_id_gicv1[(offset - 0xfd0) >> 2];
+                break;
+            case 2:
+                res = gic_id_gicv2[(offset - 0xfd0) >> 2];
+                break;
+            case REV_NVIC:
+                /* Shouldn't be able to get here */
+                abort();
+            default:
+                res = 0;
+            }
         }
+    } else {
+        g_assert_not_reached();
     }
     return res;
 bad_reg:
@@ -415,24 +801,30 @@
     return 0;
 }
 
-static uint32_t gic_dist_readw(void *opaque, hwaddr offset)
+static MemTxResult gic_dist_read(void *opaque, hwaddr offset, uint64_t *data,
+                                 unsigned size, MemTxAttrs attrs)
 {
-    uint32_t val;
-    val = gic_dist_readb(opaque, offset);
-    val |= gic_dist_readb(opaque, offset + 1) << 8;
-    return val;
-}
-
-static uint32_t gic_dist_readl(void *opaque, hwaddr offset)
-{
-    uint32_t val;
-    val = gic_dist_readw(opaque, offset);
-    val |= gic_dist_readw(opaque, offset + 2) << 16;
-    return val;
+    switch (size) {
+    case 1:
+        *data = gic_dist_readb(opaque, offset, attrs);
+        return MEMTX_OK;
+    case 2:
+        *data = gic_dist_readb(opaque, offset, attrs);
+        *data |= gic_dist_readb(opaque, offset + 1, attrs) << 8;
+        return MEMTX_OK;
+    case 4:
+        *data = gic_dist_readb(opaque, offset, attrs);
+        *data |= gic_dist_readb(opaque, offset + 1, attrs) << 8;
+        *data |= gic_dist_readb(opaque, offset + 2, attrs) << 16;
+        *data |= gic_dist_readb(opaque, offset + 3, attrs) << 24;
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
 }
 
 static void gic_dist_writeb(void *opaque, hwaddr offset,
-                            uint32_t value)
+                            uint32_t value, MemTxAttrs attrs)
 {
     GICState *s = (GICState *)opaque;
     int irq;
@@ -442,12 +834,41 @@
     cpu = gic_get_current_cpu(s);
     if (offset < 0x100) {
         if (offset == 0) {
-            s->enabled = (value & 1);
-            DPRINTF("Distribution %sabled\n", s->enabled ? "En" : "Dis");
+            if (s->security_extn && !attrs.secure) {
+                /* NS version is just an alias of the S version's bit 1 */
+                s->ctlr = deposit32(s->ctlr, 1, 1, value);
+            } else if (gic_has_groups(s)) {
+                s->ctlr = value & (GICD_CTLR_EN_GRP0 | GICD_CTLR_EN_GRP1);
+            } else {
+                s->ctlr = value & GICD_CTLR_EN_GRP0;
+            }
+            DPRINTF("Distributor: Group0 %sabled; Group 1 %sabled\n",
+                    s->ctlr & GICD_CTLR_EN_GRP0 ? "En" : "Dis",
+                    s->ctlr & GICD_CTLR_EN_GRP1 ? "En" : "Dis");
         } else if (offset < 4) {
             /* ignored.  */
         } else if (offset >= 0x80) {
-            /* Interrupt Security Registers, RAZ/WI */
+            /* Interrupt Group Registers: RAZ/WI for NS access to secure
+             * GIC, or for GICs without groups.
+             */
+            if (!(s->security_extn && !attrs.secure) && gic_has_groups(s)) {
+                /* Every byte offset holds 8 group status bits */
+                irq = (offset - 0x80) * 8 + GIC_BASE_IRQ;
+                if (irq >= s->num_irq) {
+                    goto bad_reg;
+                }
+                for (i = 0; i < 8; i++) {
+                    /* Group bits are banked for private interrupts */
+                    int cm = (irq < GIC_INTERNAL) ? (1 << cpu) : ALL_CPU_MASK;
+                    if (value & (1 << i)) {
+                        /* Group1 (Non-secure) */
+                        GIC_SET_GROUP(irq + i, cm);
+                    } else {
+                        /* Group0 (Secure) */
+                        GIC_CLEAR_GROUP(irq + i, cm);
+                    }
+                }
+            }
         } else {
             goto bad_reg;
         }
@@ -466,8 +887,14 @@
                     (irq < GIC_INTERNAL) ? (1 << cpu) : GIC_TARGET(irq + i);
                 int cm = (irq < GIC_INTERNAL) ? (1 << cpu) : ALL_CPU_MASK;
 
+                if (s->security_extn && !attrs.secure &&
+                    !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                    continue; /* Ignore Non-secure access of Group0 IRQ */
+                }
+
                 if (!GIC_TEST_ENABLED(irq + i, cm)) {
                     DPRINTF("Enabled IRQ %d\n", irq + i);
+                    trace_gic_enable_irq(irq + i);
                 }
                 GIC_SET_ENABLED(irq + i, cm);
                 /* If a raised level triggered IRQ enabled then mark
@@ -492,8 +919,14 @@
             if (value & (1 << i)) {
                 int cm = (irq < GIC_INTERNAL) ? (1 << cpu) : ALL_CPU_MASK;
 
+                if (s->security_extn && !attrs.secure &&
+                    !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                    continue; /* Ignore Non-secure access of Group0 IRQ */
+                }
+
                 if (GIC_TEST_ENABLED(irq + i, cm)) {
                     DPRINTF("Disabled IRQ %d\n", irq + i);
+                    trace_gic_disable_irq(irq + i);
                 }
                 GIC_CLEAR_ENABLED(irq + i, cm);
             }
@@ -509,6 +942,11 @@
 
         for (i = 0; i < 8; i++) {
             if (value & (1 << i)) {
+                if (s->security_extn && !attrs.secure &&
+                    !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                    continue; /* Ignore Non-secure access of Group0 IRQ */
+                }
+
                 GIC_SET_PENDING(irq + i, GIC_TARGET(irq + i));
             }
         }
@@ -522,6 +960,11 @@
         }
 
         for (i = 0; i < 8; i++) {
+            if (s->security_extn && !attrs.secure &&
+                !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                continue; /* Ignore Non-secure access of Group0 IRQ */
+            }
+
             /* ??? This currently clears the pending bit for all CPUs, even
                for per-CPU interrupts.  It's unclear whether this is the
                corect behavior.  */
@@ -537,7 +980,7 @@
         irq = (offset - 0x400) + GIC_BASE_IRQ;
         if (irq >= s->num_irq)
             goto bad_reg;
-        gic_set_priority(s, cpu, irq, value);
+        gic_set_priority(s, cpu, irq, value, attrs);
     } else if (offset < 0xc00) {
         /* Interrupt CPU Target. RAZ/WI on uniprocessor GICs, with the
          * annoying exception of the 11MPCore's GIC.
@@ -562,6 +1005,11 @@
         if (irq < GIC_NR_SGIS)
             value |= 0xaa;
         for (i = 0; i < 4; i++) {
+            if (s->security_extn && !attrs.secure &&
+                !GIC_TEST_GROUP(irq + i, 1 << cpu)) {
+                continue; /* Ignore Non-secure access of Group0 IRQ */
+            }
+
             if (s->revision == REV_11MPCORE || s->revision == REV_NVIC) {
                 if (value & (1 << (i * 2))) {
                     GIC_SET_MODEL(irq + i);
@@ -585,9 +1033,12 @@
         }
         irq = (offset - 0xf10);
 
-        s->sgi_pending[irq][cpu] &= ~value;
-        if (s->sgi_pending[irq][cpu] == 0) {
-            GIC_CLEAR_PENDING(irq, 1 << cpu);
+        if (!s->security_extn || attrs.secure ||
+            GIC_TEST_GROUP(irq, 1 << cpu)) {
+            s->sgi_pending[irq][cpu] &= ~value;
+            if (s->sgi_pending[irq][cpu] == 0) {
+                GIC_CLEAR_PENDING(irq, 1 << cpu);
+            }
         }
     } else if (offset < 0xf30) {
         /* GICD_SPENDSGIRn */
@@ -596,8 +1047,11 @@
         }
         irq = (offset - 0xf20);
 
-        GIC_SET_PENDING(irq, 1 << cpu);
-        s->sgi_pending[irq][cpu] |= value;
+        if (!s->security_extn || attrs.secure ||
+            GIC_TEST_GROUP(irq, 1 << cpu)) {
+            GIC_SET_PENDING(irq, 1 << cpu);
+            s->sgi_pending[irq][cpu] |= value;
+        }
     } else {
         goto bad_reg;
     }
@@ -609,14 +1063,14 @@
 }
 
 static void gic_dist_writew(void *opaque, hwaddr offset,
-                            uint32_t value)
+                            uint32_t value, MemTxAttrs attrs)
 {
-    gic_dist_writeb(opaque, offset, value & 0xff);
-    gic_dist_writeb(opaque, offset + 1, value >> 8);
+    gic_dist_writeb(opaque, offset, value & 0xff, attrs);
+    gic_dist_writeb(opaque, offset + 1, value >> 8, attrs);
 }
 
 static void gic_dist_writel(void *opaque, hwaddr offset,
-                            uint32_t value)
+                            uint32_t value, MemTxAttrs attrs)
 {
     GICState *s = (GICState *)opaque;
     if (offset == 0xf00) {
@@ -652,145 +1106,293 @@
         gic_update(s);
         return;
     }
-    gic_dist_writew(opaque, offset, value & 0xffff);
-    gic_dist_writew(opaque, offset + 2, value >> 16);
+    gic_dist_writew(opaque, offset, value & 0xffff, attrs);
+    gic_dist_writew(opaque, offset + 2, value >> 16, attrs);
 }
 
-static const MemoryRegionOps gic_dist_ops = {
-    .old_mmio = {
-        .read = { gic_dist_readb, gic_dist_readw, gic_dist_readl, },
-        .write = { gic_dist_writeb, gic_dist_writew, gic_dist_writel, },
-    },
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static uint32_t gic_cpu_read(GICState *s, int cpu, int offset)
+static MemTxResult gic_dist_write(void *opaque, hwaddr offset, uint64_t data,
+                                  unsigned size, MemTxAttrs attrs)
 {
-    switch (offset) {
-    case 0x00: /* Control */
-        return s->cpu_enabled[cpu];
-    case 0x04: /* Priority mask */
-        return s->priority_mask[cpu];
-    case 0x08: /* Binary Point */
-        return s->bpr[cpu];
-    case 0x0c: /* Acknowledge */
-        return gic_acknowledge_irq(s, cpu);
-    case 0x14: /* Running Priority */
-        return s->running_priority[cpu];
-    case 0x18: /* Highest Pending Interrupt */
-        return s->current_pending[cpu];
-    case 0x1c: /* Aliased Binary Point */
-        return s->abpr[cpu];
-    case 0xd0: case 0xd4: case 0xd8: case 0xdc:
-        return s->apr[(offset - 0xd0) / 4][cpu];
+    switch (size) {
+    case 1:
+        gic_dist_writeb(opaque, offset, data, attrs);
+        return MEMTX_OK;
+    case 2:
+        gic_dist_writew(opaque, offset, data, attrs);
+        return MEMTX_OK;
+    case 4:
+        gic_dist_writel(opaque, offset, data, attrs);
+        return MEMTX_OK;
     default:
-        qemu_log_mask(LOG_GUEST_ERROR,
-                      "gic_cpu_read: Bad offset %x\n", (int)offset);
-        return 0;
+        return MEMTX_ERROR;
     }
 }
 
-static void gic_cpu_write(GICState *s, int cpu, int offset, uint32_t value)
+static inline uint32_t gic_apr_ns_view(GICState *s, int cpu, int regno)
+{
+    /* Return the Nonsecure view of GICC_APR<regno>. This is the
+     * second half of GICC_NSAPR.
+     */
+    switch (GIC_MIN_BPR) {
+    case 0:
+        if (regno < 2) {
+            return s->nsapr[regno + 2][cpu];
+        }
+        break;
+    case 1:
+        if (regno == 0) {
+            return s->nsapr[regno + 1][cpu];
+        }
+        break;
+    case 2:
+        if (regno == 0) {
+            return extract32(s->nsapr[0][cpu], 16, 16);
+        }
+        break;
+    case 3:
+        if (regno == 0) {
+            return extract32(s->nsapr[0][cpu], 8, 8);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    return 0;
+}
+
+static inline void gic_apr_write_ns_view(GICState *s, int cpu, int regno,
+                                         uint32_t value)
+{
+    /* Write the Nonsecure view of GICC_APR<regno>. */
+    switch (GIC_MIN_BPR) {
+    case 0:
+        if (regno < 2) {
+            s->nsapr[regno + 2][cpu] = value;
+        }
+        break;
+    case 1:
+        if (regno == 0) {
+            s->nsapr[regno + 1][cpu] = value;
+        }
+        break;
+    case 2:
+        if (regno == 0) {
+            s->nsapr[0][cpu] = deposit32(s->nsapr[0][cpu], 16, 16, value);
+        }
+        break;
+    case 3:
+        if (regno == 0) {
+            s->nsapr[0][cpu] = deposit32(s->nsapr[0][cpu], 8, 8, value);
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static MemTxResult gic_cpu_read(GICState *s, int cpu, int offset,
+                                uint64_t *data, MemTxAttrs attrs)
 {
     switch (offset) {
     case 0x00: /* Control */
-        s->cpu_enabled[cpu] = (value & 1);
-        DPRINTF("CPU %d %sabled\n", cpu, s->cpu_enabled[cpu] ? "En" : "Dis");
+        *data = gic_get_cpu_control(s, cpu, attrs);
         break;
     case 0x04: /* Priority mask */
-        s->priority_mask[cpu] = (value & 0xff);
+        *data = gic_get_priority_mask(s, cpu, attrs);
         break;
     case 0x08: /* Binary Point */
-        s->bpr[cpu] = (value & 0x7);
+        if (s->security_extn && !attrs.secure) {
+            /* BPR is banked. Non-secure copy stored in ABPR. */
+            *data = s->abpr[cpu];
+        } else {
+            *data = s->bpr[cpu];
+        }
         break;
-    case 0x10: /* End Of Interrupt */
-        return gic_complete_irq(s, cpu, value & 0x3ff);
+    case 0x0c: /* Acknowledge */
+        *data = gic_acknowledge_irq(s, cpu, attrs);
+        break;
+    case 0x14: /* Running Priority */
+        *data = gic_get_running_priority(s, cpu, attrs);
+        break;
+    case 0x18: /* Highest Pending Interrupt */
+        *data = gic_get_current_pending_irq(s, cpu, attrs);
+        break;
     case 0x1c: /* Aliased Binary Point */
-        if (s->revision >= 2) {
-            s->abpr[cpu] = (value & 0x7);
+        /* GIC v2, no security: ABPR
+         * GIC v1, no security: not implemented (RAZ/WI)
+         * With security extensions, secure access: ABPR (alias of NS BPR)
+         * With security extensions, nonsecure access: RAZ/WI
+         */
+        if (!gic_has_groups(s) || (s->security_extn && !attrs.secure)) {
+            *data = 0;
+        } else {
+            *data = s->abpr[cpu];
         }
         break;
     case 0xd0: case 0xd4: case 0xd8: case 0xdc:
-        qemu_log_mask(LOG_UNIMP, "Writing APR not implemented\n");
+    {
+        int regno = (offset - 0xd0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            *data = 0;
+        } else if (s->security_extn && !attrs.secure) {
+            /* NS view of GICC_APR<n> is the top half of GIC_NSAPR<n> */
+            *data = gic_apr_ns_view(s, regno, cpu);
+        } else {
+            *data = s->apr[regno][cpu];
+        }
+        break;
+    }
+    case 0xe0: case 0xe4: case 0xe8: case 0xec:
+    {
+        int regno = (offset - 0xe0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2 || !gic_has_groups(s) ||
+            (s->security_extn && !attrs.secure)) {
+            *data = 0;
+        } else {
+            *data = s->nsapr[regno][cpu];
+        }
+        break;
+    }
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_cpu_read: Bad offset %x\n", (int)offset);
+        return MEMTX_ERROR;
+    }
+    return MEMTX_OK;
+}
+
+static MemTxResult gic_cpu_write(GICState *s, int cpu, int offset,
+                                 uint32_t value, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case 0x00: /* Control */
+        gic_set_cpu_control(s, cpu, value, attrs);
+        break;
+    case 0x04: /* Priority mask */
+        gic_set_priority_mask(s, cpu, value, attrs);
+        break;
+    case 0x08: /* Binary Point */
+        if (s->security_extn && !attrs.secure) {
+            s->abpr[cpu] = MAX(value & 0x7, GIC_MIN_ABPR);
+        } else {
+            s->bpr[cpu] = MAX(value & 0x7, GIC_MIN_BPR);
+        }
+        break;
+    case 0x10: /* End Of Interrupt */
+        gic_complete_irq(s, cpu, value & 0x3ff, attrs);
+        return MEMTX_OK;
+    case 0x1c: /* Aliased Binary Point */
+        if (!gic_has_groups(s) || (s->security_extn && !attrs.secure)) {
+            /* unimplemented, or NS access: RAZ/WI */
+            return MEMTX_OK;
+        } else {
+            s->abpr[cpu] = MAX(value & 0x7, GIC_MIN_ABPR);
+        }
+        break;
+    case 0xd0: case 0xd4: case 0xd8: case 0xdc:
+    {
+        int regno = (offset - 0xd0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            return MEMTX_OK;
+        }
+        if (s->security_extn && !attrs.secure) {
+            /* NS view of GICC_APR<n> is the top half of GIC_NSAPR<n> */
+            gic_apr_write_ns_view(s, regno, cpu, value);
+        } else {
+            s->apr[regno][cpu] = value;
+        }
+        break;
+    }
+    case 0xe0: case 0xe4: case 0xe8: case 0xec:
+    {
+        int regno = (offset - 0xe0) / 4;
+
+        if (regno >= GIC_NR_APRS || s->revision != 2) {
+            return MEMTX_OK;
+        }
+        if (!gic_has_groups(s) || (s->security_extn && !attrs.secure)) {
+            return MEMTX_OK;
+        }
+        s->nsapr[regno][cpu] = value;
+        break;
+    }
+    case 0x1000:
+        /* GICC_DIR */
+        gic_deactivate_irq(s, cpu, value & 0x3ff, attrs);
         break;
     default:
         qemu_log_mask(LOG_GUEST_ERROR,
                       "gic_cpu_write: Bad offset %x\n", (int)offset);
-        return;
+        return MEMTX_ERROR;
     }
     gic_update(s);
+    return MEMTX_OK;
 }
 
 /* Wrappers to read/write the GIC CPU interface for the current CPU */
-static uint64_t gic_thiscpu_read(void *opaque, hwaddr addr,
-                                 unsigned size)
+static MemTxResult gic_thiscpu_read(void *opaque, hwaddr addr, uint64_t *data,
+                                    unsigned size, MemTxAttrs attrs)
 {
     GICState *s = (GICState *)opaque;
-    return gic_cpu_read(s, gic_get_current_cpu(s), addr);
+    return gic_cpu_read(s, gic_get_current_cpu(s), addr, data, attrs);
 }
 
-static void gic_thiscpu_write(void *opaque, hwaddr addr,
-                              uint64_t value, unsigned size)
+static MemTxResult gic_thiscpu_write(void *opaque, hwaddr addr,
+                                     uint64_t value, unsigned size,
+                                     MemTxAttrs attrs)
 {
     GICState *s = (GICState *)opaque;
-    gic_cpu_write(s, gic_get_current_cpu(s), addr, value);
+    return gic_cpu_write(s, gic_get_current_cpu(s), addr, value, attrs);
 }
 
 /* Wrappers to read/write the GIC CPU interface for a specific CPU.
  * These just decode the opaque pointer into GICState* + cpu id.
  */
-static uint64_t gic_do_cpu_read(void *opaque, hwaddr addr,
-                                unsigned size)
+static MemTxResult gic_do_cpu_read(void *opaque, hwaddr addr, uint64_t *data,
+                                   unsigned size, MemTxAttrs attrs)
 {
     GICState **backref = (GICState **)opaque;
     GICState *s = *backref;
     int id = (backref - s->backref);
-    return gic_cpu_read(s, id, addr);
+    return gic_cpu_read(s, id, addr, data, attrs);
 }
 
-static void gic_do_cpu_write(void *opaque, hwaddr addr,
-                             uint64_t value, unsigned size)
+static MemTxResult gic_do_cpu_write(void *opaque, hwaddr addr,
+                                    uint64_t value, unsigned size,
+                                    MemTxAttrs attrs)
 {
     GICState **backref = (GICState **)opaque;
     GICState *s = *backref;
     int id = (backref - s->backref);
-    gic_cpu_write(s, id, addr, value);
+    return gic_cpu_write(s, id, addr, value, attrs);
 }
 
-static const MemoryRegionOps gic_thiscpu_ops = {
-    .read = gic_thiscpu_read,
-    .write = gic_thiscpu_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+static const MemoryRegionOps gic_ops[2] = {
+    {
+        .read_with_attrs = gic_dist_read,
+        .write_with_attrs = gic_dist_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    },
+    {
+        .read_with_attrs = gic_thiscpu_read,
+        .write_with_attrs = gic_thiscpu_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    }
 };
 
 static const MemoryRegionOps gic_cpu_ops = {
-    .read = gic_do_cpu_read,
-    .write = gic_do_cpu_write,
+    .read_with_attrs = gic_do_cpu_read,
+    .write_with_attrs = gic_do_cpu_write,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
+/* This function is used by nvic model */
 void gic_init_irqs_and_distributor(GICState *s)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
-    int i;
-
-    i = s->num_irq - GIC_INTERNAL;
-    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
-     * GPIO array layout is thus:
-     *  [0..N-1] SPIs
-     *  [N..N+31] PPIs for CPU 0
-     *  [N+32..N+63] PPIs for CPU 1
-     *   ...
-     */
-    if (s->revision != REV_NVIC) {
-        i += (GIC_INTERNAL * s->num_cpu);
-    }
-    qdev_init_gpio_in(DEVICE(s), gic_set_irq, i);
-    for (i = 0; i < NUM_CPU(s); i++) {
-        sysbus_init_irq(sbd, &s->parent_irq[i]);
-    }
-    memory_region_init_io(&s->iomem, OBJECT(s), &gic_dist_ops, s,
-                          "gic_dist", 0x1000);
+    gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 }
 
 static void arm_gic_realize(DeviceState *dev, Error **errp)
@@ -808,28 +1410,22 @@
         return;
     }
 
-    gic_init_irqs_and_distributor(s);
+    /* This creates distributor and main CPU interface (s->cpuiomem[0]) */
+    gic_init_irqs_and_mmio(s, gic_set_irq, gic_ops);
 
-    /* Memory regions for the CPU interfaces (NVIC doesn't have these):
-     * a region for "CPU interface for this core", then a region for
-     * "CPU interface for core 0", "for core 1", ...
+    /* Extra core-specific regions for the CPU interfaces. This is
+     * necessary for "franken-GIC" implementations, for example on
+     * Exynos 4.
      * NB that the memory region size of 0x100 applies for the 11MPCore
      * and also cores following the GIC v1 spec (ie A9).
      * GIC v2 defines a larger memory region (0x1000) so this will need
      * to be extended when we implement A15.
      */
-    memory_region_init_io(&s->cpuiomem[0], OBJECT(s), &gic_thiscpu_ops, s,
-                          "gic_cpu", 0x100);
-    for (i = 0; i < NUM_CPU(s); i++) {
+    for (i = 0; i < s->num_cpu; i++) {
         s->backref[i] = s;
         memory_region_init_io(&s->cpuiomem[i+1], OBJECT(s), &gic_cpu_ops,
                               &s->backref[i], "gic_cpu", 0x100);
-    }
-    /* Distributor */
-    sysbus_init_mmio(sbd, &s->iomem);
-    /* cpu interfaces (one for "current cpu" plus one per cpu) */
-    for (i = 0; i <= NUM_CPU(s); i++) {
-        sysbus_init_mmio(sbd, &s->cpuiomem[i]);
+        sysbus_init_mmio(sbd, &s->cpuiomem[i+1]);
     }
 }
 
diff --git a/hw/intc/arm_gic_common.c b/hw/intc/arm_gic_common.c
index 18b01ba..0a1f56a 100644
--- a/hw/intc/arm_gic_common.c
+++ b/hw/intc/arm_gic_common.c
@@ -18,7 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "gic_internal.h"
+#include "hw/arm/linux-boot-if.h"
 
 static void gic_pre_save(void *opaque)
 {
@@ -52,37 +55,78 @@
         VMSTATE_UINT8(level, gic_irq_state),
         VMSTATE_BOOL(model, gic_irq_state),
         VMSTATE_BOOL(edge_trigger, gic_irq_state),
+        VMSTATE_UINT8(group, gic_irq_state),
         VMSTATE_END_OF_LIST()
     }
 };
 
 static const VMStateDescription vmstate_gic = {
     .name = "arm_gic",
-    .version_id = 7,
-    .minimum_version_id = 7,
+    .version_id = 12,
+    .minimum_version_id = 12,
     .pre_save = gic_pre_save,
     .post_load = gic_post_load,
     .fields = (VMStateField[]) {
-        VMSTATE_BOOL(enabled, GICState),
-        VMSTATE_BOOL_ARRAY(cpu_enabled, GICState, GIC_NCPU),
+        VMSTATE_UINT32(ctlr, GICState),
+        VMSTATE_UINT32_ARRAY(cpu_ctlr, GICState, GIC_NCPU),
         VMSTATE_STRUCT_ARRAY(irq_state, GICState, GIC_MAXIRQ, 1,
                              vmstate_gic_irq_state, gic_irq_state),
         VMSTATE_UINT8_ARRAY(irq_target, GICState, GIC_MAXIRQ),
         VMSTATE_UINT8_2DARRAY(priority1, GICState, GIC_INTERNAL, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(priority2, GICState, GIC_MAXIRQ - GIC_INTERNAL),
-        VMSTATE_UINT16_2DARRAY(last_active, GICState, GIC_MAXIRQ, GIC_NCPU),
         VMSTATE_UINT8_2DARRAY(sgi_pending, GICState, GIC_NR_SGIS, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(priority_mask, GICState, GIC_NCPU),
-        VMSTATE_UINT16_ARRAY(running_irq, GICState, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(running_priority, GICState, GIC_NCPU),
         VMSTATE_UINT16_ARRAY(current_pending, GICState, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(bpr, GICState, GIC_NCPU),
         VMSTATE_UINT8_ARRAY(abpr, GICState, GIC_NCPU),
         VMSTATE_UINT32_2DARRAY(apr, GICState, GIC_NR_APRS, GIC_NCPU),
+        VMSTATE_UINT32_2DARRAY(nsapr, GICState, GIC_NR_APRS, GIC_NCPU),
         VMSTATE_END_OF_LIST()
     }
 };
 
+void gic_init_irqs_and_mmio(GICState *s, qemu_irq_handler handler,
+                            const MemoryRegionOps *ops)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
+    int i = s->num_irq - GIC_INTERNAL;
+
+    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
+     * GPIO array layout is thus:
+     *  [0..N-1] SPIs
+     *  [N..N+31] PPIs for CPU 0
+     *  [N+32..N+63] PPIs for CPU 1
+     *   ...
+     */
+    if (s->revision != REV_NVIC) {
+        i += (GIC_INTERNAL * s->num_cpu);
+    }
+    qdev_init_gpio_in(DEVICE(s), handler, i);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_irq[i]);
+    }
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->parent_fiq[i]);
+    }
+
+    /* Distributor */
+    memory_region_init_io(&s->iomem, OBJECT(s), ops, s, "gic_dist", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    if (s->revision != REV_NVIC) {
+        /* This is the main CPU interface "for this core". It is always
+         * present because it is required by both software emulation and KVM.
+         * NVIC is not handled here because its CPU interface is different,
+         * neither it can use KVM.
+         */
+        memory_region_init_io(&s->cpuiomem[0], OBJECT(s), ops ? &ops[1] : NULL,
+                              s, "gic_cpu", s->revision == 2 ? 0x2000 : 0x100);
+        sysbus_init_mmio(sbd, &s->cpuiomem[0]);
+    }
+}
+
 static void arm_gic_common_realize(DeviceState *dev, Error **errp)
 {
     GICState *s = ARM_GIC_COMMON(dev);
@@ -110,35 +154,94 @@
                    num_irq);
         return;
     }
+
+    if (s->security_extn &&
+        (s->revision == REV_11MPCORE || s->revision == REV_NVIC)) {
+        error_setg(errp, "this GIC revision does not implement "
+                   "the security extensions");
+        return;
+    }
 }
 
 static void arm_gic_common_reset(DeviceState *dev)
 {
     GICState *s = ARM_GIC_COMMON(dev);
-    int i;
+    int i, j;
+    int resetprio;
+
+    /* If we're resetting a TZ-aware GIC as if secure firmware
+     * had set it up ready to start a kernel in non-secure,
+     * we need to set interrupt priorities to a "zero for the
+     * NS view" value. This is particularly critical for the
+     * priority_mask[] values, because if they are zero then NS
+     * code cannot ever rewrite the priority to anything else.
+     */
+    if (s->security_extn && s->irq_reset_nonsecure) {
+        resetprio = 0x80;
+    } else {
+        resetprio = 0;
+    }
+
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
         if (s->revision == REV_11MPCORE) {
             s->priority_mask[i] = 0xf0;
         } else {
-            s->priority_mask[i] = 0;
+            s->priority_mask[i] = resetprio;
         }
         s->current_pending[i] = 1023;
-        s->running_irq[i] = 1023;
         s->running_priority[i] = 0x100;
-        s->cpu_enabled[i] = false;
+        s->cpu_ctlr[i] = 0;
+        s->bpr[i] = GIC_MIN_BPR;
+        s->abpr[i] = GIC_MIN_ABPR;
+        for (j = 0; j < GIC_INTERNAL; j++) {
+            s->priority1[j][i] = resetprio;
+        }
+        for (j = 0; j < GIC_NR_SGIS; j++) {
+            s->sgi_pending[j][i] = 0;
+        }
     }
     for (i = 0; i < GIC_NR_SGIS; i++) {
         GIC_SET_ENABLED(i, ALL_CPU_MASK);
         GIC_SET_EDGE_TRIGGER(i);
     }
-    if (s->num_cpu == 1) {
+
+    for (i = 0; i < ARRAY_SIZE(s->priority2); i++) {
+        s->priority2[i] = resetprio;
+    }
+
+    for (i = 0; i < GIC_MAXIRQ; i++) {
         /* For uniprocessor GICs all interrupts always target the sole CPU */
-        for (i = 0; i < GIC_MAXIRQ; i++) {
+        if (s->num_cpu == 1) {
             s->irq_target[i] = 1;
+        } else {
+            s->irq_target[i] = 0;
         }
     }
-    s->enabled = false;
+    if (s->security_extn && s->irq_reset_nonsecure) {
+        for (i = 0; i < GIC_MAXIRQ; i++) {
+            GIC_SET_GROUP(i, ALL_CPU_MASK);
+        }
+    }
+
+    s->ctlr = 0;
+}
+
+static void arm_gic_common_linux_init(ARMLinuxBootIf *obj,
+                                      bool secure_boot)
+{
+    GICState *s = ARM_GIC_COMMON(obj);
+
+    if (s->security_extn && !secure_boot) {
+        /* We're directly booting a kernel into NonSecure. If this GIC
+         * implements the security extensions then we must configure it
+         * to have all the interrupts be NonSecure (this is a job that
+         * is done by the Secure boot firmware in real hardware, and in
+         * this mode QEMU is acting as a minimalist firmware-and-bootloader
+         * equivalent).
+         */
+        s->irq_reset_nonsecure = true;
+    }
 }
 
 static Property arm_gic_common_properties[] = {
@@ -149,17 +252,21 @@
      * (Internally, 0xffffffff also indicates "not a GIC but an NVIC".)
      */
     DEFINE_PROP_UINT32("revision", GICState, revision, 1),
+    /* True if the GIC should implement the security extensions */
+    DEFINE_PROP_BOOL("has-security-extensions", GICState, security_extn, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
 static void arm_gic_common_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_CLASS(klass);
 
     dc->reset = arm_gic_common_reset;
     dc->realize = arm_gic_common_realize;
     dc->props = arm_gic_common_properties;
     dc->vmsd = &vmstate_gic;
+    albifc->arm_linux_init = arm_gic_common_linux_init;
 }
 
 static const TypeInfo arm_gic_common_type = {
@@ -169,6 +276,10 @@
     .class_size = sizeof(ARMGICCommonClass),
     .class_init = arm_gic_common_class_init,
     .abstract = true,
+    .interfaces = (InterfaceInfo []) {
+        { TYPE_ARM_LINUX_BOOT_IF },
+        { },
+    },
 };
 
 static void register_types(void)
diff --git a/hw/intc/arm_gic_kvm.c b/hw/intc/arm_gic_kvm.c
index 5038885..5593cdb 100644
--- a/hw/intc/arm_gic_kvm.c
+++ b/hw/intc/arm_gic_kvm.c
@@ -19,10 +19,16 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
+#include "migration/migration.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "gic_internal.h"
+#include "vgic_common.h"
 
 //#define DEBUG_GIC_KVM
 
@@ -52,7 +58,7 @@
     void (*parent_reset)(DeviceState *dev);
 } KVMARMGICClass;
 
-static void kvm_arm_gic_set_irq(void *opaque, int irq, int level)
+void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level)
 {
     /* Meaning of the 'irq' parameter:
      *  [0..N-1] : external interrupts
@@ -63,10 +69,9 @@
      * has separate fields in the irq number for type,
      * CPU number and interrupt number.
      */
-    GICState *s = (GICState *)opaque;
     int kvm_irq, irqtype, cpu;
 
-    if (irq < (s->num_irq - GIC_INTERNAL)) {
+    if (irq < (num_irq - GIC_INTERNAL)) {
         /* External interrupt. The kernel numbers these like the GIC
          * hardware, with external interrupt IDs starting after the
          * internal ones.
@@ -77,7 +82,7 @@
     } else {
         /* Internal interrupt: decode into (cpu, interrupt id) */
         irqtype = KVM_ARM_IRQ_TYPE_PPI;
-        irq -= (s->num_irq - GIC_INTERNAL);
+        irq -= (num_irq - GIC_INTERNAL);
         cpu = irq / GIC_INTERNAL;
         irq %= GIC_INTERNAL;
     }
@@ -87,54 +92,36 @@
     kvm_set_irq(kvm_state, kvm_irq, !!level);
 }
 
+static void kvm_arm_gicv2_set_irq(void *opaque, int irq, int level)
+{
+    GICState *s = (GICState *)opaque;
+
+    kvm_arm_gic_set_irq(s->num_irq, irq, level);
+}
+
 static bool kvm_arm_gic_can_save_restore(GICState *s)
 {
     return s->dev_fd >= 0;
 }
 
-static void kvm_gic_access(GICState *s, int group, int offset,
-                                   int cpu, uint32_t *val, bool write)
-{
-    struct kvm_device_attr attr;
-    int type;
-    int err;
-
-    cpu = cpu & 0xff;
-
-    attr.flags = 0;
-    attr.group = group;
-    attr.attr = (((uint64_t)cpu << KVM_DEV_ARM_VGIC_CPUID_SHIFT) &
-                 KVM_DEV_ARM_VGIC_CPUID_MASK) |
-                (((uint64_t)offset << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) &
-                 KVM_DEV_ARM_VGIC_OFFSET_MASK);
-    attr.addr = (uintptr_t)val;
-
-    if (write) {
-        type = KVM_SET_DEVICE_ATTR;
-    } else {
-        type = KVM_GET_DEVICE_ATTR;
-    }
-
-    err = kvm_device_ioctl(s->dev_fd, type, &attr);
-    if (err < 0) {
-        fprintf(stderr, "KVM_{SET/GET}_DEVICE_ATTR failed: %s\n",
-                strerror(-err));
-        abort();
-    }
-}
+#define KVM_VGIC_ATTR(offset, cpu) \
+    ((((uint64_t)(cpu) << KVM_DEV_ARM_VGIC_CPUID_SHIFT) & \
+      KVM_DEV_ARM_VGIC_CPUID_MASK) | \
+     (((uint64_t)(offset) << KVM_DEV_ARM_VGIC_OFFSET_SHIFT) & \
+      KVM_DEV_ARM_VGIC_OFFSET_MASK))
 
 static void kvm_gicd_access(GICState *s, int offset, int cpu,
                             uint32_t *val, bool write)
 {
-    kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
-                   offset, cpu, val, write);
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
+                      KVM_VGIC_ATTR(offset, cpu), val, write);
 }
 
 static void kvm_gicc_access(GICState *s, int offset, int cpu,
                             uint32_t *val, bool write)
 {
-    kvm_gic_access(s, KVM_DEV_ARM_VGIC_GRP_CPU_REGS,
-                   offset, cpu, val, write);
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CPU_REGS,
+                      KVM_VGIC_ATTR(offset, cpu), val, write);
 }
 
 #define for_each_irq_reg(_ctr, _max_irq, _field_width) \
@@ -161,6 +148,20 @@
     }
 }
 
+static void translate_group(GICState *s, int irq, int cpu,
+                            uint32_t *field, bool to_kernel)
+{
+    int cm = (irq < GIC_INTERNAL) ? (1 << cpu) : ALL_CPU_MASK;
+
+    if (to_kernel) {
+        *field = GIC_TEST_GROUP(irq, cm);
+    } else {
+        if (*field & 1) {
+            GIC_SET_GROUP(irq, cm);
+        }
+    }
+}
+
 static void translate_enabled(GICState *s, int irq, int cpu,
                               uint32_t *field, bool to_kernel)
 {
@@ -222,7 +223,7 @@
     if (to_kernel) {
         *field = GIC_GET_PRIORITY(irq, cpu) & 0xff;
     } else {
-        gic_set_priority(s, cpu, irq, *field & 0xff);
+        gic_set_priority(s, cpu, irq, *field & 0xff, MEMTXATTRS_UNSPECIFIED);
     }
 }
 
@@ -311,11 +312,6 @@
     int num_cpu;
     int num_irq;
 
-    if (!kvm_arm_gic_can_save_restore(s)) {
-            DPRINTF("Cannot put kernel gic state, no kernel interface");
-            return;
-    }
-
     /* Note: We do the restore in a slightly different order than the save
      * (where the order doesn't matter and is simply ordered according to the
      * register offset values */
@@ -324,8 +320,8 @@
      * Distributor State
      */
 
-    /* s->enabled -> GICD_CTLR */
-    reg = s->enabled;
+    /* s->ctlr -> GICD_CTLR */
+    reg = s->ctlr;
     kvm_gicd_access(s, 0x0, 0, &reg, true);
 
     /* Sanity checking on GICD_TYPER and s->num_irq, s->num_cpu */
@@ -350,11 +346,19 @@
     kvm_dist_put(s, 0x180, 1, s->num_irq, translate_clear);
     kvm_dist_put(s, 0x100, 1, s->num_irq, translate_enabled);
 
+    /* irq_state[n].group -> GICD_IGROUPRn */
+    kvm_dist_put(s, 0x80, 1, s->num_irq, translate_group);
+
     /* s->irq_target[irq] -> GICD_ITARGETSRn
      * (restore targets before pending to ensure the pending state is set on
      * the appropriate CPU interfaces in the kernel) */
     kvm_dist_put(s, 0x800, 8, s->num_irq, translate_targets);
 
+    /* irq_state[n].trigger -> GICD_ICFGRn
+     * (restore configuration registers before pending IRQs so we treat
+     * level/edge correctly) */
+    kvm_dist_put(s, 0xc00, 2, s->num_irq, translate_trigger);
+
     /* irq_state[n].pending + irq_state[n].level -> GICD_ISPENDRn */
     kvm_dist_put(s, 0x280, 1, s->num_irq, translate_clear);
     kvm_dist_put(s, 0x200, 1, s->num_irq, translate_pending);
@@ -363,8 +367,6 @@
     kvm_dist_put(s, 0x380, 1, s->num_irq, translate_clear);
     kvm_dist_put(s, 0x300, 1, s->num_irq, translate_active);
 
-    /* irq_state[n].trigger -> GICD_ICFRn */
-    kvm_dist_put(s, 0xc00, 2, s->num_irq, translate_trigger);
 
     /* s->priorityX[irq] -> ICD_IPRIORITYRn */
     kvm_dist_put(s, 0x400, 8, s->num_irq, translate_priority);
@@ -379,8 +381,8 @@
      */
 
     for (cpu = 0; cpu < s->num_cpu; cpu++) {
-        /* s->cpu_enabled[cpu] -> GICC_CTLR */
-        reg = s->cpu_enabled[cpu];
+        /* s->cpu_ctlr[cpu] -> GICC_CTLR */
+        reg = s->cpu_ctlr[cpu];
         kvm_gicc_access(s, 0x00, cpu, &reg, true);
 
         /* s->priority_mask[cpu] -> GICC_PMR */
@@ -409,18 +411,13 @@
     int i;
     int cpu;
 
-    if (!kvm_arm_gic_can_save_restore(s)) {
-            DPRINTF("Cannot get kernel gic state, no kernel interface");
-            return;
-    }
-
     /*****************************************************************
      * Distributor State
      */
 
-    /* GICD_CTLR -> s->enabled */
+    /* GICD_CTLR -> s->ctlr */
     kvm_gicd_access(s, 0x0, 0, &reg, false);
-    s->enabled = reg & 1;
+    s->ctlr = reg;
 
     /* Sanity checking on GICD_TYPER -> s->num_irq, s->num_cpu */
     kvm_gicd_access(s, 0x4, 0, &reg, false);
@@ -436,21 +433,14 @@
     /* GICD_IIDR -> ? */
     kvm_gicd_access(s, 0x8, 0, &reg, false);
 
-    /* Verify no GROUP 1 interrupts configured in the kernel */
-    for_each_irq_reg(i, s->num_irq, 1) {
-        kvm_gicd_access(s, 0x80 + (i * 4), 0, &reg, false);
-        if (reg != 0) {
-            fprintf(stderr, "Unsupported GICD_IGROUPRn value: %08x\n",
-                    reg);
-            abort();
-        }
-    }
-
     /* Clear all the IRQ settings */
     for (i = 0; i < s->num_irq; i++) {
         memset(&s->irq_state[i], 0, sizeof(s->irq_state[0]));
     }
 
+    /* GICD_IGROUPRn -> irq_state[n].group */
+    kvm_dist_get(s, 0x80, 1, s->num_irq, translate_group);
+
     /* GICD_ISENABLERn -> irq_state[n].enabled */
     kvm_dist_get(s, 0x100, 1, s->num_irq, translate_enabled);
 
@@ -478,9 +468,9 @@
      */
 
     for (cpu = 0; cpu < s->num_cpu; cpu++) {
-        /* GICC_CTLR -> s->cpu_enabled[cpu] */
+        /* GICC_CTLR -> s->cpu_ctlr[cpu] */
         kvm_gicc_access(s, 0x00, cpu, &reg, false);
-        s->cpu_enabled[cpu] = (reg & 1);
+        s->cpu_ctlr[cpu] = reg;
 
         /* GICC_PMR -> s->priority_mask[cpu] */
         kvm_gicc_access(s, 0x04, cpu, &reg, false);
@@ -508,14 +498,16 @@
     KVMARMGICClass *kgc = KVM_ARM_GIC_GET_CLASS(s);
 
     kgc->parent_reset(dev);
-    kvm_arm_gic_put(s);
+
+    if (kvm_arm_gic_can_save_restore(s)) {
+        kvm_arm_gic_put(s);
+    }
 }
 
 static void kvm_arm_gic_realize(DeviceState *dev, Error **errp)
 {
     int i;
     GICState *s = KVM_ARM_GIC(dev);
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
     KVMARMGICClass *kgc = KVM_ARM_GIC_GET_CLASS(s);
     Error *local_err = NULL;
     int ret;
@@ -526,21 +518,17 @@
         return;
     }
 
-    i = s->num_irq - GIC_INTERNAL;
-    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
-     * GPIO array layout is thus:
-     *  [0..N-1] SPIs
-     *  [N..N+31] PPIs for CPU 0
-     *  [N+32..N+63] PPIs for CPU 1
-     *   ...
-     */
-    i += (GIC_INTERNAL * s->num_cpu);
-    qdev_init_gpio_in(dev, kvm_arm_gic_set_irq, i);
-    /* We never use our outbound IRQ lines but provide them so that
-     * we maintain the same interface as the non-KVM GIC.
-     */
-    for (i = 0; i < s->num_cpu; i++) {
-        sysbus_init_irq(sbd, &s->parent_irq[i]);
+    if (s->security_extn) {
+        error_setg(errp, "the in-kernel VGIC does not implement the "
+                   "security extensions");
+        return;
+    }
+
+    gic_init_irqs_and_mmio(s, kvm_arm_gicv2_set_irq, NULL);
+
+    for (i = 0; i < s->num_irq - GIC_INTERNAL; i++) {
+        qemu_irq irq = qdev_get_gpio_in(dev, i);
+        kvm_irqchip_set_qemuirq_gsi(kvm_state, irq, i);
     }
 
     /* Try to create the device via the device control API */
@@ -548,15 +536,25 @@
     ret = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V2, false);
     if (ret >= 0) {
         s->dev_fd = ret;
+
+        /* Newstyle API is used, we may have attributes */
+        if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0)) {
+            uint32_t numirqs = s->num_irq;
+            kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS, 0,
+                              &numirqs, true);
+        }
+        /* Tell the kernel to complete VGIC initialization now */
+        if (kvm_device_check_attr(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                                  KVM_DEV_ARM_VGIC_CTRL_INIT)) {
+            kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                              KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
+        }
     } else if (ret != -ENODEV && ret != -ENOTSUP) {
         error_setg_errno(errp, -ret, "error creating in-kernel VGIC");
         return;
     }
 
     /* Distributor */
-    memory_region_init_reservation(&s->iomem, OBJECT(s),
-                                   "kvm-gic_dist", 0x1000);
-    sysbus_init_mmio(sbd, &s->iomem);
     kvm_arm_register_device(&s->iomem,
                             (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
                             | KVM_VGIC_V2_ADDR_TYPE_DIST,
@@ -567,15 +565,18 @@
      * provide the "interface for core #N" memory regions, because
      * cores with a VGIC don't have those.
      */
-    memory_region_init_reservation(&s->cpuiomem[0], OBJECT(s),
-                                   "kvm-gic_cpu", 0x1000);
-    sysbus_init_mmio(sbd, &s->cpuiomem[0]);
     kvm_arm_register_device(&s->cpuiomem[0],
                             (KVM_ARM_DEVICE_VGIC_V2 << KVM_ARM_DEVICE_ID_SHIFT)
                             | KVM_VGIC_V2_ADDR_TYPE_CPU,
                             KVM_DEV_ARM_VGIC_GRP_ADDR,
                             KVM_VGIC_V2_ADDR_TYPE_CPU,
                             s->dev_fd);
+
+    if (!kvm_arm_gic_can_save_restore(s)) {
+        error_setg(&s->migration_blocker, "This operating system kernel does "
+                                          "not support vGICv2 migration");
+        migrate_add_blocker(s->migration_blocker);
+    }
 }
 
 static void kvm_arm_gic_class_init(ObjectClass *klass, void *data)
diff --git a/hw/intc/arm_gicv2m.c b/hw/intc/arm_gicv2m.c
new file mode 100644
index 0000000..3922fbc
--- /dev/null
+++ b/hw/intc/arm_gicv2m.c
@@ -0,0 +1,196 @@
+/*
+ *  GICv2m extension for MSI/MSI-x support with a GICv2-based system
+ *
+ * Copyright (C) 2015 Linaro, All rights reserved.
+ *
+ * Author: Christoffer Dall <christoffer.dall@linaro.org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* This file implements an emulated GICv2m widget as described in the ARM
+ * Server Base System Architecture (SBSA) specification Version 2.2
+ * (ARM-DEN-0029 v2.2) pages 35-39 without any optional implementation defined
+ * identification registers and with a single non-secure MSI register frame.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/pci/msi.h"
+#include "sysemu/kvm.h"
+#include "qemu/log.h"
+
+#define TYPE_ARM_GICV2M "arm-gicv2m"
+#define ARM_GICV2M(obj) OBJECT_CHECK(ARMGICv2mState, (obj), TYPE_ARM_GICV2M)
+
+#define GICV2M_NUM_SPI_MAX 128
+
+#define V2M_MSI_TYPER           0x008
+#define V2M_MSI_SETSPI_NS       0x040
+#define V2M_MSI_IIDR            0xFCC
+#define V2M_IIDR0               0xFD0
+#define V2M_IIDR11              0xFFC
+
+#define PRODUCT_ID_QEMU         0x51 /* ASCII code Q */
+
+typedef struct ARMGICv2mState {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+    qemu_irq spi[GICV2M_NUM_SPI_MAX];
+
+    uint32_t base_spi;
+    uint32_t num_spi;
+} ARMGICv2mState;
+
+static void gicv2m_set_irq(void *opaque, int irq)
+{
+    ARMGICv2mState *s = (ARMGICv2mState *)opaque;
+
+    qemu_irq_pulse(s->spi[irq]);
+}
+
+static uint64_t gicv2m_read(void *opaque, hwaddr offset,
+                            unsigned size)
+{
+    ARMGICv2mState *s = (ARMGICv2mState *)opaque;
+    uint32_t val;
+
+    if (size != 4) {
+        qemu_log_mask(LOG_GUEST_ERROR, "gicv2m_read: bad size %u\n", size);
+        return 0;
+    }
+
+    switch (offset) {
+    case V2M_MSI_TYPER:
+        val = (s->base_spi + 32) << 16;
+        val |= s->num_spi;
+        return val;
+    case V2M_MSI_IIDR:
+        /* We don't have any valid implementor so we leave that field as zero
+         * and we return 0 in the arch revision as per the spec.
+         */
+        return (PRODUCT_ID_QEMU << 20);
+    case V2M_IIDR0 ... V2M_IIDR11:
+        /* We do not implement any optional identification registers and the
+         * mandatory MSI_PIDR2 register reads as 0x0, so we capture all
+         * implementation defined registers here.
+         */
+        return 0;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gicv2m_read: Bad offset %x\n", (int)offset);
+        return 0;
+    }
+}
+
+static void gicv2m_write(void *opaque, hwaddr offset,
+                        uint64_t value, unsigned size)
+{
+    ARMGICv2mState *s = (ARMGICv2mState *)opaque;
+
+    if (size != 2 && size != 4) {
+        qemu_log_mask(LOG_GUEST_ERROR, "gicv2m_write: bad size %u\n", size);
+        return;
+    }
+
+    switch (offset) {
+    case V2M_MSI_SETSPI_NS: {
+        int spi;
+
+        spi = (value & 0x3ff) - (s->base_spi + 32);
+        if (spi >= 0 && spi < s->num_spi) {
+            gicv2m_set_irq(s, spi);
+        }
+        return;
+    }
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gicv2m_write: Bad offset %x\n", (int)offset);
+    }
+}
+
+static const MemoryRegionOps gicv2m_ops = {
+    .read = gicv2m_read,
+    .write = gicv2m_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void gicv2m_realize(DeviceState *dev, Error **errp)
+{
+    ARMGICv2mState *s = ARM_GICV2M(dev);
+    int i;
+
+    if (s->num_spi > GICV2M_NUM_SPI_MAX) {
+        error_setg(errp,
+                   "requested %u SPIs exceeds GICv2m frame maximum %d",
+                   s->num_spi, GICV2M_NUM_SPI_MAX);
+        return;
+    }
+
+    if (s->base_spi + 32 > 1020 - s->num_spi) {
+        error_setg(errp,
+                   "requested base SPI %u+%u exceeds max. number 1020",
+                   s->base_spi + 32, s->num_spi);
+        return;
+    }
+
+    for (i = 0; i < s->num_spi; i++) {
+        sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->spi[i]);
+    }
+
+    msi_nonbroken = true;
+    kvm_gsi_direct_mapping = true;
+    kvm_msi_via_irqfd_allowed = kvm_irqfds_enabled();
+}
+
+static void gicv2m_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    ARMGICv2mState *s = ARM_GICV2M(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &gicv2m_ops, s,
+                          "gicv2m", 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static Property gicv2m_properties[] = {
+    DEFINE_PROP_UINT32("base-spi", ARMGICv2mState, base_spi, 0),
+    DEFINE_PROP_UINT32("num-spi", ARMGICv2mState, num_spi, 64),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void gicv2m_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = gicv2m_properties;
+    dc->realize = gicv2m_realize;
+}
+
+static const TypeInfo gicv2m_info = {
+    .name          = TYPE_ARM_GICV2M,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(ARMGICv2mState),
+    .instance_init = gicv2m_init,
+    .class_init    = gicv2m_class_init,
+};
+
+static void gicv2m_register_types(void)
+{
+    type_register_static(&gicv2m_info);
+}
+
+type_init(gicv2m_register_types)
diff --git a/hw/intc/arm_gicv3.c b/hw/intc/arm_gicv3.c
new file mode 100644
index 0000000..8a6c647
--- /dev/null
+++ b/hw/intc/arm_gicv3.c
@@ -0,0 +1,400 @@
+/*
+ * ARM Generic Interrupt Controller v3
+ *
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2016 Linaro Limited
+ * Written by Shlomo Pongratz, Peter Maydell
+ *
+ * This code is licensed under the GPL, version 2 or (at your option)
+ * any later version.
+ */
+
+/* This file contains implementation code for an interrupt controller
+ * which implements the GICv3 architecture. Specifically this is where
+ * the device class itself and the functions for handling interrupts
+ * coming in and going out live.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/sysbus.h"
+#include "hw/intc/arm_gicv3.h"
+#include "gicv3_internal.h"
+
+static bool irqbetter(GICv3CPUState *cs, int irq, uint8_t prio)
+{
+    /* Return true if this IRQ at this priority should take
+     * precedence over the current recorded highest priority
+     * pending interrupt for this CPU. We also return true if
+     * the current recorded highest priority pending interrupt
+     * is the same as this one (a property which the calling code
+     * relies on).
+     */
+    if (prio < cs->hppi.prio) {
+        return true;
+    }
+    /* If multiple pending interrupts have the same priority then it is an
+     * IMPDEF choice which of them to signal to the CPU. We choose to
+     * signal the one with the lowest interrupt number.
+     */
+    if (prio == cs->hppi.prio && irq <= cs->hppi.irq) {
+        return true;
+    }
+    return false;
+}
+
+static uint32_t gicd_int_pending(GICv3State *s, int irq)
+{
+    /* Recalculate which distributor interrupts are actually pending
+     * in the group of 32 interrupts starting at irq (which should be a multiple
+     * of 32), and return a 32-bit integer which has a bit set for each
+     * interrupt that is eligible to be signaled to the CPU interface.
+     *
+     * An interrupt is pending if:
+     *  + the PENDING latch is set OR it is level triggered and the input is 1
+     *  + its ENABLE bit is set
+     *  + the GICD enable bit for its group is set
+     * Conveniently we can bulk-calculate this with bitwise operations.
+     */
+    uint32_t pend, grpmask;
+    uint32_t pending = *gic_bmp_ptr32(s->pending, irq);
+    uint32_t edge_trigger = *gic_bmp_ptr32(s->edge_trigger, irq);
+    uint32_t level = *gic_bmp_ptr32(s->level, irq);
+    uint32_t group = *gic_bmp_ptr32(s->group, irq);
+    uint32_t grpmod = *gic_bmp_ptr32(s->grpmod, irq);
+    uint32_t enable = *gic_bmp_ptr32(s->enabled, irq);
+
+    pend = pending | (~edge_trigger & level);
+    pend &= enable;
+
+    if (s->gicd_ctlr & GICD_CTLR_DS) {
+        grpmod = 0;
+    }
+
+    grpmask = 0;
+    if (s->gicd_ctlr & GICD_CTLR_EN_GRP1NS) {
+        grpmask |= group;
+    }
+    if (s->gicd_ctlr & GICD_CTLR_EN_GRP1S) {
+        grpmask |= (~group & grpmod);
+    }
+    if (s->gicd_ctlr & GICD_CTLR_EN_GRP0) {
+        grpmask |= (~group & ~grpmod);
+    }
+    pend &= grpmask;
+
+    return pend;
+}
+
+static uint32_t gicr_int_pending(GICv3CPUState *cs)
+{
+    /* Recalculate which redistributor interrupts are actually pending,
+     * and return a 32-bit integer which has a bit set for each interrupt
+     * that is eligible to be signaled to the CPU interface.
+     *
+     * An interrupt is pending if:
+     *  + the PENDING latch is set OR it is level triggered and the input is 1
+     *  + its ENABLE bit is set
+     *  + the GICD enable bit for its group is set
+     * Conveniently we can bulk-calculate this with bitwise operations.
+     */
+    uint32_t pend, grpmask, grpmod;
+
+    pend = cs->gicr_ipendr0 | (~cs->edge_trigger & cs->level);
+    pend &= cs->gicr_ienabler0;
+
+    if (cs->gic->gicd_ctlr & GICD_CTLR_DS) {
+        grpmod = 0;
+    } else {
+        grpmod = cs->gicr_igrpmodr0;
+    }
+
+    grpmask = 0;
+    if (cs->gic->gicd_ctlr & GICD_CTLR_EN_GRP1NS) {
+        grpmask |= cs->gicr_igroupr0;
+    }
+    if (cs->gic->gicd_ctlr & GICD_CTLR_EN_GRP1S) {
+        grpmask |= (~cs->gicr_igroupr0 & grpmod);
+    }
+    if (cs->gic->gicd_ctlr & GICD_CTLR_EN_GRP0) {
+        grpmask |= (~cs->gicr_igroupr0 & ~grpmod);
+    }
+    pend &= grpmask;
+
+    return pend;
+}
+
+/* Update the interrupt status after state in a redistributor
+ * or CPU interface has changed, but don't tell the CPU i/f.
+ */
+static void gicv3_redist_update_noirqset(GICv3CPUState *cs)
+{
+    /* Find the highest priority pending interrupt among the
+     * redistributor interrupts (SGIs and PPIs).
+     */
+    bool seenbetter = false;
+    uint8_t prio;
+    int i;
+    uint32_t pend;
+
+    /* Find out which redistributor interrupts are eligible to be
+     * signaled to the CPU interface.
+     */
+    pend = gicr_int_pending(cs);
+
+    if (pend) {
+        for (i = 0; i < GIC_INTERNAL; i++) {
+            if (!(pend & (1 << i))) {
+                continue;
+            }
+            prio = cs->gicr_ipriorityr[i];
+            if (irqbetter(cs, i, prio)) {
+                cs->hppi.irq = i;
+                cs->hppi.prio = prio;
+                seenbetter = true;
+            }
+        }
+    }
+
+    if (seenbetter) {
+        cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq);
+    }
+
+    /* If the best interrupt we just found would preempt whatever
+     * was the previous best interrupt before this update, then
+     * we know it's definitely the best one now.
+     * If we didn't find an interrupt that would preempt the previous
+     * best, and the previous best is outside our range (or there was no
+     * previous pending interrupt at all), then that is still valid, and
+     * we leave it as the best.
+     * Otherwise, we need to do a full update (because the previous best
+     * interrupt has reduced in priority and any other interrupt could
+     * now be the new best one).
+     */
+    if (!seenbetter && cs->hppi.prio != 0xff && cs->hppi.irq < GIC_INTERNAL) {
+        gicv3_full_update_noirqset(cs->gic);
+    }
+}
+
+/* Update the GIC status after state in a redistributor or
+ * CPU interface has changed, and inform the CPU i/f of
+ * its new highest priority pending interrupt.
+ */
+void gicv3_redist_update(GICv3CPUState *cs)
+{
+    gicv3_redist_update_noirqset(cs);
+    gicv3_cpuif_update(cs);
+}
+
+/* Update the GIC status after state in the distributor has
+ * changed affecting @len interrupts starting at @start,
+ * but don't tell the CPU i/f.
+ */
+static void gicv3_update_noirqset(GICv3State *s, int start, int len)
+{
+    int i;
+    uint8_t prio;
+    uint32_t pend = 0;
+
+    assert(start >= GIC_INTERNAL);
+    assert(len > 0);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        s->cpu[i].seenbetter = false;
+    }
+
+    /* Find the highest priority pending interrupt in this range. */
+    for (i = start; i < start + len; i++) {
+        GICv3CPUState *cs;
+
+        if (i == start || (i & 0x1f) == 0) {
+            /* Calculate the next 32 bits worth of pending status */
+            pend = gicd_int_pending(s, i & ~0x1f);
+        }
+
+        if (!(pend & (1 << (i & 0x1f)))) {
+            continue;
+        }
+        cs = s->gicd_irouter_target[i];
+        if (!cs) {
+            /* Interrupts targeting no implemented CPU should remain pending
+             * and not be forwarded to any CPU.
+             */
+            continue;
+        }
+        prio = s->gicd_ipriority[i];
+        if (irqbetter(cs, i, prio)) {
+            cs->hppi.irq = i;
+            cs->hppi.prio = prio;
+            cs->seenbetter = true;
+        }
+    }
+
+    /* If the best interrupt we just found would preempt whatever
+     * was the previous best interrupt before this update, then
+     * we know it's definitely the best one now.
+     * If we didn't find an interrupt that would preempt the previous
+     * best, and the previous best is outside our range (or there was
+     * no previous pending interrupt at all), then that
+     * is still valid, and we leave it as the best.
+     * Otherwise, we need to do a full update (because the previous best
+     * interrupt has reduced in priority and any other interrupt could
+     * now be the new best one).
+     */
+    for (i = 0; i < s->num_cpu; i++) {
+        GICv3CPUState *cs = &s->cpu[i];
+
+        if (cs->seenbetter) {
+            cs->hppi.grp = gicv3_irq_group(cs->gic, cs, cs->hppi.irq);
+        }
+
+        if (!cs->seenbetter && cs->hppi.prio != 0xff &&
+            cs->hppi.irq >= start && cs->hppi.irq < start + len) {
+            gicv3_full_update_noirqset(s);
+            break;
+        }
+    }
+}
+
+void gicv3_update(GICv3State *s, int start, int len)
+{
+    int i;
+
+    gicv3_update_noirqset(s, start, len);
+    for (i = 0; i < s->num_cpu; i++) {
+        gicv3_cpuif_update(&s->cpu[i]);
+    }
+}
+
+void gicv3_full_update_noirqset(GICv3State *s)
+{
+    /* Completely recalculate the GIC status from scratch, but
+     * don't update any outbound IRQ lines.
+     */
+    int i;
+
+    for (i = 0; i < s->num_cpu; i++) {
+        s->cpu[i].hppi.prio = 0xff;
+    }
+
+    /* Note that we can guarantee that these functions will not
+     * recursively call back into gicv3_full_update(), because
+     * at each point the "previous best" is always outside the
+     * range we ask them to update.
+     */
+    gicv3_update_noirqset(s, GIC_INTERNAL, s->num_irq - GIC_INTERNAL);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        gicv3_redist_update_noirqset(&s->cpu[i]);
+    }
+}
+
+void gicv3_full_update(GICv3State *s)
+{
+    /* Completely recalculate the GIC status from scratch, including
+     * updating outbound IRQ lines.
+     */
+    int i;
+
+    gicv3_full_update_noirqset(s);
+    for (i = 0; i < s->num_cpu; i++) {
+        gicv3_cpuif_update(&s->cpu[i]);
+    }
+}
+
+/* Process a change in an external IRQ input. */
+static void gicv3_set_irq(void *opaque, int irq, int level)
+{
+    /* Meaning of the 'irq' parameter:
+     *  [0..N-1] : external interrupts
+     *  [N..N+31] : PPI (internal) interrupts for CPU 0
+     *  [N+32..N+63] : PPI (internal interrupts for CPU 1
+     *  ...
+     */
+    GICv3State *s = opaque;
+
+    if (irq < (s->num_irq - GIC_INTERNAL)) {
+        /* external interrupt (SPI) */
+        gicv3_dist_set_irq(s, irq + GIC_INTERNAL, level);
+    } else {
+        /* per-cpu interrupt (PPI) */
+        int cpu;
+
+        irq -= (s->num_irq - GIC_INTERNAL);
+        cpu = irq / GIC_INTERNAL;
+        irq %= GIC_INTERNAL;
+        assert(cpu < s->num_cpu);
+        /* Raising SGIs via this function would be a bug in how the board
+         * model wires up interrupts.
+         */
+        assert(irq >= GIC_NR_SGIS);
+        gicv3_redist_set_irq(&s->cpu[cpu], irq, level);
+    }
+}
+
+static void arm_gicv3_post_load(GICv3State *s)
+{
+    /* Recalculate our cached idea of the current highest priority
+     * pending interrupt, but don't set IRQ or FIQ lines.
+     */
+    gicv3_full_update_noirqset(s);
+    /* Repopulate the cache of GICv3CPUState pointers for target CPUs */
+    gicv3_cache_all_target_cpustates(s);
+}
+
+static const MemoryRegionOps gic_ops[] = {
+    {
+        .read_with_attrs = gicv3_dist_read,
+        .write_with_attrs = gicv3_dist_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    },
+    {
+        .read_with_attrs = gicv3_redist_read,
+        .write_with_attrs = gicv3_redist_write,
+        .endianness = DEVICE_NATIVE_ENDIAN,
+    }
+};
+
+static void arm_gic_realize(DeviceState *dev, Error **errp)
+{
+    /* Device instance realize function for the GIC sysbus device */
+    GICv3State *s = ARM_GICV3(dev);
+    ARMGICv3Class *agc = ARM_GICV3_GET_CLASS(s);
+    Error *local_err = NULL;
+
+    agc->parent_realize(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    gicv3_init_irqs_and_mmio(s, gicv3_set_irq, gic_ops);
+
+    gicv3_init_cpuif(s);
+}
+
+static void arm_gicv3_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
+    ARMGICv3Class *agc = ARM_GICV3_CLASS(klass);
+
+    agcc->post_load = arm_gicv3_post_load;
+    agc->parent_realize = dc->realize;
+    dc->realize = arm_gic_realize;
+}
+
+static const TypeInfo arm_gicv3_info = {
+    .name = TYPE_ARM_GICV3,
+    .parent = TYPE_ARM_GICV3_COMMON,
+    .instance_size = sizeof(GICv3State),
+    .class_init = arm_gicv3_class_init,
+    .class_size = sizeof(ARMGICv3Class),
+};
+
+static void arm_gicv3_register_types(void)
+{
+    type_register_static(&arm_gicv3_info);
+}
+
+type_init(arm_gicv3_register_types)
diff --git a/hw/intc/arm_gicv3_common.c b/hw/intc/arm_gicv3_common.c
new file mode 100644
index 0000000..0f8c4b8
--- /dev/null
+++ b/hw/intc/arm_gicv3_common.c
@@ -0,0 +1,351 @@
+/*
+ * ARM GICv3 support - common bits of emulated and KVM kernel model
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Written by Peter Maydell
+ * Reworked for GICv3 by Shlomo Pongratz and Pavel Fedin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qom/cpu.h"
+#include "hw/intc/arm_gicv3_common.h"
+#include "gicv3_internal.h"
+#include "hw/arm/linux-boot-if.h"
+
+static void gicv3_pre_save(void *opaque)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    ARMGICv3CommonClass *c = ARM_GICV3_COMMON_GET_CLASS(s);
+
+    if (c->pre_save) {
+        c->pre_save(s);
+    }
+}
+
+static int gicv3_post_load(void *opaque, int version_id)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    ARMGICv3CommonClass *c = ARM_GICV3_COMMON_GET_CLASS(s);
+
+    if (c->post_load) {
+        c->post_load(s);
+    }
+    return 0;
+}
+
+static const VMStateDescription vmstate_gicv3_cpu = {
+    .name = "arm_gicv3_cpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(level, GICv3CPUState),
+        VMSTATE_UINT32(gicr_ctlr, GICv3CPUState),
+        VMSTATE_UINT32_ARRAY(gicr_statusr, GICv3CPUState, 2),
+        VMSTATE_UINT32(gicr_waker, GICv3CPUState),
+        VMSTATE_UINT64(gicr_propbaser, GICv3CPUState),
+        VMSTATE_UINT64(gicr_pendbaser, GICv3CPUState),
+        VMSTATE_UINT32(gicr_igroupr0, GICv3CPUState),
+        VMSTATE_UINT32(gicr_ienabler0, GICv3CPUState),
+        VMSTATE_UINT32(gicr_ipendr0, GICv3CPUState),
+        VMSTATE_UINT32(gicr_iactiver0, GICv3CPUState),
+        VMSTATE_UINT32(edge_trigger, GICv3CPUState),
+        VMSTATE_UINT32(gicr_igrpmodr0, GICv3CPUState),
+        VMSTATE_UINT32(gicr_nsacr, GICv3CPUState),
+        VMSTATE_UINT8_ARRAY(gicr_ipriorityr, GICv3CPUState, GIC_INTERNAL),
+        VMSTATE_UINT64_ARRAY(icc_ctlr_el1, GICv3CPUState, 2),
+        VMSTATE_UINT64(icc_pmr_el1, GICv3CPUState),
+        VMSTATE_UINT64_ARRAY(icc_bpr, GICv3CPUState, 3),
+        VMSTATE_UINT64_2DARRAY(icc_apr, GICv3CPUState, 3, 4),
+        VMSTATE_UINT64_ARRAY(icc_igrpen, GICv3CPUState, 3),
+        VMSTATE_UINT64(icc_ctlr_el3, GICv3CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_gicv3 = {
+    .name = "arm_gicv3",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_save = gicv3_pre_save,
+    .post_load = gicv3_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(gicd_ctlr, GICv3State),
+        VMSTATE_UINT32_ARRAY(gicd_statusr, GICv3State, 2),
+        VMSTATE_UINT32_ARRAY(group, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT32_ARRAY(grpmod, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT32_ARRAY(enabled, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT32_ARRAY(pending, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT32_ARRAY(active, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT32_ARRAY(level, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT32_ARRAY(edge_trigger, GICv3State, GICV3_BMP_SIZE),
+        VMSTATE_UINT8_ARRAY(gicd_ipriority, GICv3State, GICV3_MAXIRQ),
+        VMSTATE_UINT64_ARRAY(gicd_irouter, GICv3State, GICV3_MAXIRQ),
+        VMSTATE_UINT32_ARRAY(gicd_nsacr, GICv3State,
+                             DIV_ROUND_UP(GICV3_MAXIRQ, 16)),
+        VMSTATE_STRUCT_VARRAY_POINTER_UINT32(cpu, GICv3State, num_cpu,
+                                             vmstate_gicv3_cpu, GICv3CPUState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler,
+                              const MemoryRegionOps *ops)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(s);
+    int i;
+
+    /* For the GIC, also expose incoming GPIO lines for PPIs for each CPU.
+     * GPIO array layout is thus:
+     *  [0..N-1] spi
+     *  [N..N+31] PPIs for CPU 0
+     *  [N+32..N+63] PPIs for CPU 1
+     *   ...
+     */
+    i = s->num_irq - GIC_INTERNAL + GIC_INTERNAL * s->num_cpu;
+    qdev_init_gpio_in(DEVICE(s), handler, i);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->cpu[i].parent_irq);
+    }
+    for (i = 0; i < s->num_cpu; i++) {
+        sysbus_init_irq(sbd, &s->cpu[i].parent_fiq);
+    }
+
+    memory_region_init_io(&s->iomem_dist, OBJECT(s), ops, s,
+                          "gicv3_dist", 0x10000);
+    memory_region_init_io(&s->iomem_redist, OBJECT(s), ops ? &ops[1] : NULL, s,
+                          "gicv3_redist", 0x20000 * s->num_cpu);
+
+    sysbus_init_mmio(sbd, &s->iomem_dist);
+    sysbus_init_mmio(sbd, &s->iomem_redist);
+}
+
+static void arm_gicv3_common_realize(DeviceState *dev, Error **errp)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+    int i;
+
+    /* revision property is actually reserved and currently used only in order
+     * to keep the interface compatible with GICv2 code, avoiding extra
+     * conditions. However, in future it could be used, for example, if we
+     * implement GICv4.
+     */
+    if (s->revision != 3) {
+        error_setg(errp, "unsupported GIC revision %d", s->revision);
+        return;
+    }
+
+    if (s->num_irq > GICV3_MAXIRQ) {
+        error_setg(errp,
+                   "requested %u interrupt lines exceeds GIC maximum %d",
+                   s->num_irq, GICV3_MAXIRQ);
+        return;
+    }
+    if (s->num_irq < GIC_INTERNAL) {
+        error_setg(errp,
+                   "requested %u interrupt lines is below GIC minimum %d",
+                   s->num_irq, GIC_INTERNAL);
+        return;
+    }
+
+    /* ITLinesNumber is represented as (N / 32) - 1, so this is an
+     * implementation imposed restriction, not an architectural one,
+     * so we don't have to deal with bitfields where only some of the
+     * bits in a 32-bit word should be valid.
+     */
+    if (s->num_irq % 32) {
+        error_setg(errp,
+                   "%d interrupt lines unsupported: not divisible by 32",
+                   s->num_irq);
+        return;
+    }
+
+    s->cpu = g_new0(GICv3CPUState, s->num_cpu);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        CPUState *cpu = qemu_get_cpu(i);
+        uint64_t cpu_affid;
+        int last;
+
+        s->cpu[i].cpu = cpu;
+        s->cpu[i].gic = s;
+
+        /* Pre-construct the GICR_TYPER:
+         * For our implementation:
+         *  Top 32 bits are the affinity value of the associated CPU
+         *  CommonLPIAff == 01 (redistributors with same Aff3 share LPI table)
+         *  Processor_Number == CPU index starting from 0
+         *  DPGS == 0 (GICR_CTLR.DPG* not supported)
+         *  Last == 1 if this is the last redistributor in a series of
+         *            contiguous redistributor pages
+         *  DirectLPI == 0 (direct injection of LPIs not supported)
+         *  VLPIS == 0 (virtual LPIs not supported)
+         *  PLPIS == 0 (physical LPIs not supported)
+         */
+        cpu_affid = object_property_get_int(OBJECT(cpu), "mp-affinity", NULL);
+        last = (i == s->num_cpu - 1);
+
+        /* The CPU mp-affinity property is in MPIDR register format; squash
+         * the affinity bytes into 32 bits as the GICR_TYPER has them.
+         */
+        cpu_affid = (cpu_affid & 0xFF00000000ULL >> 8) | (cpu_affid & 0xFFFFFF);
+        s->cpu[i].gicr_typer = (cpu_affid << 32) |
+            (1 << 24) |
+            (i << 8) |
+            (last << 4);
+    }
+}
+
+static void arm_gicv3_common_reset(DeviceState *dev)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+    int i;
+
+    for (i = 0; i < s->num_cpu; i++) {
+        GICv3CPUState *cs = &s->cpu[i];
+
+        cs->level = 0;
+        cs->gicr_ctlr = 0;
+        cs->gicr_statusr[GICV3_S] = 0;
+        cs->gicr_statusr[GICV3_NS] = 0;
+        cs->gicr_waker = GICR_WAKER_ProcessorSleep | GICR_WAKER_ChildrenAsleep;
+        cs->gicr_propbaser = 0;
+        cs->gicr_pendbaser = 0;
+        /* If we're resetting a TZ-aware GIC as if secure firmware
+         * had set it up ready to start a kernel in non-secure, we
+         * need to set interrupts to group 1 so the kernel can use them.
+         * Otherwise they reset to group 0 like the hardware.
+         */
+        if (s->irq_reset_nonsecure) {
+            cs->gicr_igroupr0 = 0xffffffff;
+        } else {
+            cs->gicr_igroupr0 = 0;
+        }
+
+        cs->gicr_ienabler0 = 0;
+        cs->gicr_ipendr0 = 0;
+        cs->gicr_iactiver0 = 0;
+        cs->edge_trigger = 0xffff;
+        cs->gicr_igrpmodr0 = 0;
+        cs->gicr_nsacr = 0;
+        memset(cs->gicr_ipriorityr, 0, sizeof(cs->gicr_ipriorityr));
+
+        cs->hppi.prio = 0xff;
+
+        /* State in the CPU interface must *not* be reset here, because it
+         * is part of the CPU's reset domain, not the GIC device's.
+         */
+    }
+
+    /* For our implementation affinity routing is always enabled */
+    if (s->security_extn) {
+        s->gicd_ctlr = GICD_CTLR_ARE_S | GICD_CTLR_ARE_NS;
+    } else {
+        s->gicd_ctlr = GICD_CTLR_DS | GICD_CTLR_ARE;
+    }
+
+    s->gicd_statusr[GICV3_S] = 0;
+    s->gicd_statusr[GICV3_NS] = 0;
+
+    memset(s->group, 0, sizeof(s->group));
+    memset(s->grpmod, 0, sizeof(s->grpmod));
+    memset(s->enabled, 0, sizeof(s->enabled));
+    memset(s->pending, 0, sizeof(s->pending));
+    memset(s->active, 0, sizeof(s->active));
+    memset(s->level, 0, sizeof(s->level));
+    memset(s->edge_trigger, 0, sizeof(s->edge_trigger));
+    memset(s->gicd_ipriority, 0, sizeof(s->gicd_ipriority));
+    memset(s->gicd_irouter, 0, sizeof(s->gicd_irouter));
+    memset(s->gicd_nsacr, 0, sizeof(s->gicd_nsacr));
+    /* GICD_IROUTER are UNKNOWN at reset so in theory the guest must
+     * write these to get sane behaviour and we need not populate the
+     * pointer cache here; however having the cache be different for
+     * "happened to be 0 from reset" and "guest wrote 0" would be
+     * too confusing.
+     */
+    gicv3_cache_all_target_cpustates(s);
+
+    if (s->irq_reset_nonsecure) {
+        /* If we're resetting a TZ-aware GIC as if secure firmware
+         * had set it up ready to start a kernel in non-secure, we
+         * need to set interrupts to group 1 so the kernel can use them.
+         * Otherwise they reset to group 0 like the hardware.
+         */
+        for (i = GIC_INTERNAL; i < s->num_irq; i++) {
+            gicv3_gicd_group_set(s, i);
+        }
+    }
+}
+
+static void arm_gic_common_linux_init(ARMLinuxBootIf *obj,
+                                      bool secure_boot)
+{
+    GICv3State *s = ARM_GICV3_COMMON(obj);
+
+    if (s->security_extn && !secure_boot) {
+        /* We're directly booting a kernel into NonSecure. If this GIC
+         * implements the security extensions then we must configure it
+         * to have all the interrupts be NonSecure (this is a job that
+         * is done by the Secure boot firmware in real hardware, and in
+         * this mode QEMU is acting as a minimalist firmware-and-bootloader
+         * equivalent).
+         */
+        s->irq_reset_nonsecure = true;
+    }
+}
+
+static Property arm_gicv3_common_properties[] = {
+    DEFINE_PROP_UINT32("num-cpu", GICv3State, num_cpu, 1),
+    DEFINE_PROP_UINT32("num-irq", GICv3State, num_irq, 32),
+    DEFINE_PROP_UINT32("revision", GICv3State, revision, 3),
+    DEFINE_PROP_BOOL("has-security-extensions", GICv3State, security_extn, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void arm_gicv3_common_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMLinuxBootIfClass *albifc = ARM_LINUX_BOOT_IF_CLASS(klass);
+
+    dc->reset = arm_gicv3_common_reset;
+    dc->realize = arm_gicv3_common_realize;
+    dc->props = arm_gicv3_common_properties;
+    dc->vmsd = &vmstate_gicv3;
+    albifc->arm_linux_init = arm_gic_common_linux_init;
+}
+
+static const TypeInfo arm_gicv3_common_type = {
+    .name = TYPE_ARM_GICV3_COMMON,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(GICv3State),
+    .class_size = sizeof(ARMGICv3CommonClass),
+    .class_init = arm_gicv3_common_class_init,
+    .abstract = true,
+    .interfaces = (InterfaceInfo []) {
+        { TYPE_ARM_LINUX_BOOT_IF },
+        { },
+    },
+};
+
+static void register_types(void)
+{
+    type_register_static(&arm_gicv3_common_type);
+}
+
+type_init(register_types)
diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c
new file mode 100644
index 0000000..4633172
--- /dev/null
+++ b/hw/intc/arm_gicv3_cpuif.c
@@ -0,0 +1,1348 @@
+/*
+ * ARM Generic Interrupt Controller v3
+ *
+ * Copyright (c) 2016 Linaro Limited
+ * Written by Peter Maydell
+ *
+ * This code is licensed under the GPL, version 2 or (at your option)
+ * any later version.
+ */
+
+/* This file contains the code for the system register interface
+ * portions of the GICv3.
+ */
+
+#include "qemu/osdep.h"
+#include "trace.h"
+#include "gicv3_internal.h"
+#include "cpu.h"
+
+static GICv3CPUState *icc_cs_from_env(CPUARMState *env)
+{
+    /* Given the CPU, find the right GICv3CPUState struct.
+     * Since we registered the CPU interface with the EL change hook as
+     * the opaque pointer, we can just directly get from the CPU to it.
+     */
+    return arm_get_el_change_hook_opaque(arm_env_get_cpu(env));
+}
+
+static bool gicv3_use_ns_bank(CPUARMState *env)
+{
+    /* Return true if we should use the NonSecure bank for a banked GIC
+     * CPU interface register. Note that this differs from the
+     * access_secure_reg() function because GICv3 banked registers are
+     * banked even for AArch64, unlike the other CPU system registers.
+     */
+    return !arm_is_secure_below_el3(env);
+}
+
+static int icc_highest_active_prio(GICv3CPUState *cs)
+{
+    /* Calculate the current running priority based on the set bits
+     * in the Active Priority Registers.
+     */
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(cs->icc_apr[0]); i++) {
+        uint32_t apr = cs->icc_apr[GICV3_G0][i] |
+            cs->icc_apr[GICV3_G1][i] | cs->icc_apr[GICV3_G1NS][i];
+
+        if (!apr) {
+            continue;
+        }
+        return (i * 32 + ctz32(apr)) << (GIC_MIN_BPR + 1);
+    }
+    /* No current active interrupts: return idle priority */
+    return 0xff;
+}
+
+static uint32_t icc_gprio_mask(GICv3CPUState *cs, int group)
+{
+    /* Return a mask word which clears the subpriority bits from
+     * a priority value for an interrupt in the specified group.
+     * This depends on the BPR value:
+     *  a BPR of 0 means the group priority bits are [7:1];
+     *  a BPR of 1 means they are [7:2], and so on down to
+     *  a BPR of 7 meaning no group priority bits at all.
+     * Which BPR to use depends on the group of the interrupt and
+     * the current ICC_CTLR.CBPR settings.
+     */
+    if ((group == GICV3_G1 && cs->icc_ctlr_el1[GICV3_S] & ICC_CTLR_EL1_CBPR) ||
+        (group == GICV3_G1NS &&
+         cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_CBPR)) {
+        group = GICV3_G0;
+    }
+
+    return ~0U << ((cs->icc_bpr[group] & 7) + 1);
+}
+
+static bool icc_no_enabled_hppi(GICv3CPUState *cs)
+{
+    /* Return true if there is no pending interrupt, or the
+     * highest priority pending interrupt is in a group which has been
+     * disabled at the CPU interface by the ICC_IGRPEN* register enable bits.
+     */
+    return cs->hppi.prio == 0xff || (cs->icc_igrpen[cs->hppi.grp] == 0);
+}
+
+static bool icc_hppi_can_preempt(GICv3CPUState *cs)
+{
+    /* Return true if we have a pending interrupt of sufficient
+     * priority to preempt.
+     */
+    int rprio;
+    uint32_t mask;
+
+    if (icc_no_enabled_hppi(cs)) {
+        return false;
+    }
+
+    if (cs->hppi.prio >= cs->icc_pmr_el1) {
+        /* Priority mask masks this interrupt */
+        return false;
+    }
+
+    rprio = icc_highest_active_prio(cs);
+    if (rprio == 0xff) {
+        /* No currently running interrupt so we can preempt */
+        return true;
+    }
+
+    mask = icc_gprio_mask(cs, cs->hppi.grp);
+
+    /* We only preempt a running interrupt if the pending interrupt's
+     * group priority is sufficient (the subpriorities are not considered).
+     */
+    if ((cs->hppi.prio & mask) < (rprio & mask)) {
+        return true;
+    }
+
+    return false;
+}
+
+void gicv3_cpuif_update(GICv3CPUState *cs)
+{
+    /* Tell the CPU about its highest priority pending interrupt */
+    int irqlevel = 0;
+    int fiqlevel = 0;
+    ARMCPU *cpu = ARM_CPU(cs->cpu);
+    CPUARMState *env = &cpu->env;
+
+    trace_gicv3_cpuif_update(gicv3_redist_affid(cs), cs->hppi.irq,
+                             cs->hppi.grp, cs->hppi.prio);
+
+    if (cs->hppi.grp == GICV3_G1 && !arm_feature(env, ARM_FEATURE_EL3)) {
+        /* If a Security-enabled GIC sends a G1S interrupt to a
+         * Security-disabled CPU, we must treat it as if it were G0.
+         */
+        cs->hppi.grp = GICV3_G0;
+    }
+
+    if (icc_hppi_can_preempt(cs)) {
+        /* We have an interrupt: should we signal it as IRQ or FIQ?
+         * This is described in the GICv3 spec section 4.6.2.
+         */
+        bool isfiq;
+
+        switch (cs->hppi.grp) {
+        case GICV3_G0:
+            isfiq = true;
+            break;
+        case GICV3_G1:
+            isfiq = (!arm_is_secure(env) ||
+                     (arm_current_el(env) == 3 && arm_el_is_aa64(env, 3)));
+            break;
+        case GICV3_G1NS:
+            isfiq = arm_is_secure(env);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+
+        if (isfiq) {
+            fiqlevel = 1;
+        } else {
+            irqlevel = 1;
+        }
+    }
+
+    trace_gicv3_cpuif_set_irqs(gicv3_redist_affid(cs), fiqlevel, irqlevel);
+
+    qemu_set_irq(cs->parent_fiq, fiqlevel);
+    qemu_set_irq(cs->parent_irq, irqlevel);
+}
+
+static uint64_t icc_pmr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint32_t value = cs->icc_pmr_el1;
+
+    if (arm_feature(env, ARM_FEATURE_EL3) && !arm_is_secure(env) &&
+        (env->cp15.scr_el3 & SCR_FIQ)) {
+        /* NS access and Group 0 is inaccessible to NS: return the
+         * NS view of the current priority
+         */
+        if (value & 0x80) {
+            /* Secure priorities not visible to NS */
+            value = 0;
+        } else if (value != 0xff) {
+            value = (value << 1) & 0xff;
+        }
+    }
+
+    trace_gicv3_icc_pmr_read(gicv3_redist_affid(cs), value);
+
+    return value;
+}
+
+static void icc_pmr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+
+    trace_gicv3_icc_pmr_write(gicv3_redist_affid(cs), value);
+
+    value &= 0xff;
+
+    if (arm_feature(env, ARM_FEATURE_EL3) && !arm_is_secure(env) &&
+        (env->cp15.scr_el3 & SCR_FIQ)) {
+        /* NS access and Group 0 is inaccessible to NS: return the
+         * NS view of the current priority
+         */
+        if (!(cs->icc_pmr_el1 & 0x80)) {
+            /* Current PMR in the secure range, don't allow NS to change it */
+            return;
+        }
+        value = (value >> 1) & 0x80;
+    }
+    cs->icc_pmr_el1 = value;
+    gicv3_cpuif_update(cs);
+}
+
+static void icc_activate_irq(GICv3CPUState *cs, int irq)
+{
+    /* Move the interrupt from the Pending state to Active, and update
+     * the Active Priority Registers
+     */
+    uint32_t mask = icc_gprio_mask(cs, cs->hppi.grp);
+    int prio = cs->hppi.prio & mask;
+    int aprbit = prio >> 1;
+    int regno = aprbit / 32;
+    int regbit = aprbit % 32;
+
+    cs->icc_apr[cs->hppi.grp][regno] |= (1 << regbit);
+
+    if (irq < GIC_INTERNAL) {
+        cs->gicr_iactiver0 = deposit32(cs->gicr_iactiver0, irq, 1, 1);
+        cs->gicr_ipendr0 = deposit32(cs->gicr_ipendr0, irq, 1, 0);
+        gicv3_redist_update(cs);
+    } else {
+        gicv3_gicd_active_set(cs->gic, irq);
+        gicv3_gicd_pending_clear(cs->gic, irq);
+        gicv3_update(cs->gic, irq, 1);
+    }
+}
+
+static uint64_t icc_hppir0_value(GICv3CPUState *cs, CPUARMState *env)
+{
+    /* Return the highest priority pending interrupt register value
+     * for group 0.
+     */
+    bool irq_is_secure;
+
+    if (cs->hppi.prio == 0xff) {
+        return INTID_SPURIOUS;
+    }
+
+    /* Check whether we can return the interrupt or if we should return
+     * a special identifier, as per the CheckGroup0ForSpecialIdentifiers
+     * pseudocode. (We can simplify a little because for us ICC_SRE_EL1.RM
+     * is always zero.)
+     */
+    irq_is_secure = (!(cs->gic->gicd_ctlr & GICD_CTLR_DS) &&
+                     (cs->hppi.grp != GICV3_G1NS));
+
+    if (cs->hppi.grp != GICV3_G0 && !arm_is_el3_or_mon(env)) {
+        return INTID_SPURIOUS;
+    }
+    if (irq_is_secure && !arm_is_secure(env)) {
+        /* Secure interrupts not visible to Nonsecure */
+        return INTID_SPURIOUS;
+    }
+
+    if (cs->hppi.grp != GICV3_G0) {
+        /* Indicate to EL3 that there's a Group 1 interrupt for the other
+         * state pending.
+         */
+        return irq_is_secure ? INTID_SECURE : INTID_NONSECURE;
+    }
+
+    return cs->hppi.irq;
+}
+
+static uint64_t icc_hppir1_value(GICv3CPUState *cs, CPUARMState *env)
+{
+    /* Return the highest priority pending interrupt register value
+     * for group 1.
+     */
+    bool irq_is_secure;
+
+    if (cs->hppi.prio == 0xff) {
+        return INTID_SPURIOUS;
+    }
+
+    /* Check whether we can return the interrupt or if we should return
+     * a special identifier, as per the CheckGroup1ForSpecialIdentifiers
+     * pseudocode. (We can simplify a little because for us ICC_SRE_EL1.RM
+     * is always zero.)
+     */
+    irq_is_secure = (!(cs->gic->gicd_ctlr & GICD_CTLR_DS) &&
+                     (cs->hppi.grp != GICV3_G1NS));
+
+    if (cs->hppi.grp == GICV3_G0) {
+        /* Group 0 interrupts not visible via HPPIR1 */
+        return INTID_SPURIOUS;
+    }
+    if (irq_is_secure) {
+        if (!arm_is_secure(env)) {
+            /* Secure interrupts not visible in Non-secure */
+            return INTID_SPURIOUS;
+        }
+    } else if (!arm_is_el3_or_mon(env) && arm_is_secure(env)) {
+        /* Group 1 non-secure interrupts not visible in Secure EL1 */
+        return INTID_SPURIOUS;
+    }
+
+    return cs->hppi.irq;
+}
+
+static uint64_t icc_iar0_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t intid;
+
+    if (!icc_hppi_can_preempt(cs)) {
+        intid = INTID_SPURIOUS;
+    } else {
+        intid = icc_hppir0_value(cs, env);
+    }
+
+    if (!(intid >= INTID_SECURE && intid <= INTID_SPURIOUS)) {
+        icc_activate_irq(cs, intid);
+    }
+
+    trace_gicv3_icc_iar0_read(gicv3_redist_affid(cs), intid);
+    return intid;
+}
+
+static uint64_t icc_iar1_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t intid;
+
+    if (!icc_hppi_can_preempt(cs)) {
+        intid = INTID_SPURIOUS;
+    } else {
+        intid = icc_hppir1_value(cs, env);
+    }
+
+    if (!(intid >= INTID_SECURE && intid <= INTID_SPURIOUS)) {
+        icc_activate_irq(cs, intid);
+    }
+
+    trace_gicv3_icc_iar1_read(gicv3_redist_affid(cs), intid);
+    return intid;
+}
+
+static void icc_drop_prio(GICv3CPUState *cs, int grp)
+{
+    /* Drop the priority of the currently active interrupt in
+     * the specified group.
+     *
+     * Note that we can guarantee (because of the requirement to nest
+     * ICC_IAR reads [which activate an interrupt and raise priority]
+     * with ICC_EOIR writes [which drop the priority for the interrupt])
+     * that the interrupt we're being called for is the highest priority
+     * active interrupt, meaning that it has the lowest set bit in the
+     * APR registers.
+     *
+     * If the guest does not honour the ordering constraints then the
+     * behaviour of the GIC is UNPREDICTABLE, which for us means that
+     * the values of the APR registers might become incorrect and the
+     * running priority will be wrong, so interrupts that should preempt
+     * might not do so, and interrupts that should not preempt might do so.
+     */
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(cs->icc_apr[grp]); i++) {
+        uint64_t *papr = &cs->icc_apr[grp][i];
+
+        if (!*papr) {
+            continue;
+        }
+        /* Clear the lowest set bit */
+        *papr &= *papr - 1;
+        break;
+    }
+
+    /* running priority change means we need an update for this cpu i/f */
+    gicv3_cpuif_update(cs);
+}
+
+static bool icc_eoi_split(CPUARMState *env, GICv3CPUState *cs)
+{
+    /* Return true if we should split priority drop and interrupt
+     * deactivation, ie whether the relevant EOIMode bit is set.
+     */
+    if (arm_is_el3_or_mon(env)) {
+        return cs->icc_ctlr_el3 & ICC_CTLR_EL3_EOIMODE_EL3;
+    }
+    if (arm_is_secure_below_el3(env)) {
+        return cs->icc_ctlr_el1[GICV3_S] & ICC_CTLR_EL1_EOIMODE;
+    } else {
+        return cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_EOIMODE;
+    }
+}
+
+static int icc_highest_active_group(GICv3CPUState *cs)
+{
+    /* Return the group with the highest priority active interrupt.
+     * We can do this by just comparing the APRs to see which one
+     * has the lowest set bit.
+     * (If more than one group is active at the same priority then
+     * we're in UNPREDICTABLE territory.)
+     */
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(cs->icc_apr[0]); i++) {
+        int g0ctz = ctz32(cs->icc_apr[GICV3_G0][i]);
+        int g1ctz = ctz32(cs->icc_apr[GICV3_G1][i]);
+        int g1nsctz = ctz32(cs->icc_apr[GICV3_G1NS][i]);
+
+        if (g1nsctz < g0ctz && g1nsctz < g1ctz) {
+            return GICV3_G1NS;
+        }
+        if (g1ctz < g0ctz) {
+            return GICV3_G1;
+        }
+        if (g0ctz < 32) {
+            return GICV3_G0;
+        }
+    }
+    /* No set active bits? UNPREDICTABLE; return -1 so the caller
+     * ignores the spurious EOI attempt.
+     */
+    return -1;
+}
+
+static void icc_deactivate_irq(GICv3CPUState *cs, int irq)
+{
+    if (irq < GIC_INTERNAL) {
+        cs->gicr_iactiver0 = deposit32(cs->gicr_iactiver0, irq, 1, 0);
+        gicv3_redist_update(cs);
+    } else {
+        gicv3_gicd_active_clear(cs->gic, irq);
+        gicv3_update(cs->gic, irq, 1);
+    }
+}
+
+static void icc_eoir_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                           uint64_t value)
+{
+    /* End of Interrupt */
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int irq = value & 0xffffff;
+    int grp;
+
+    trace_gicv3_icc_eoir_write(gicv3_redist_affid(cs), value);
+
+    if (ri->crm == 8) {
+        /* EOIR0 */
+        grp = GICV3_G0;
+    } else {
+        /* EOIR1 */
+        if (arm_is_secure(env)) {
+            grp = GICV3_G1;
+        } else {
+            grp = GICV3_G1NS;
+        }
+    }
+
+    if (irq >= cs->gic->num_irq) {
+        /* This handles two cases:
+         * 1. If software writes the ID of a spurious interrupt [ie 1020-1023]
+         * to the GICC_EOIR, the GIC ignores that write.
+         * 2. If software writes the number of a non-existent interrupt
+         * this must be a subcase of "value written does not match the last
+         * valid interrupt value read from the Interrupt Acknowledge
+         * register" and so this is UNPREDICTABLE. We choose to ignore it.
+         */
+        return;
+    }
+
+    if (icc_highest_active_group(cs) != grp) {
+        return;
+    }
+
+    icc_drop_prio(cs, grp);
+
+    if (!icc_eoi_split(env, cs)) {
+        /* Priority drop and deactivate not split: deactivate irq now */
+        icc_deactivate_irq(cs, irq);
+    }
+}
+
+static uint64_t icc_hppir0_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t value = icc_hppir0_value(cs, env);
+
+    trace_gicv3_icc_hppir0_read(gicv3_redist_affid(cs), value);
+    return value;
+}
+
+static uint64_t icc_hppir1_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t value = icc_hppir1_value(cs, env);
+
+    trace_gicv3_icc_hppir1_read(gicv3_redist_affid(cs), value);
+    return value;
+}
+
+static uint64_t icc_bpr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int grp = (ri->crm == 8) ? GICV3_G0 : GICV3_G1;
+    bool satinc = false;
+    uint64_t bpr;
+
+    if (grp == GICV3_G1 && gicv3_use_ns_bank(env)) {
+        grp = GICV3_G1NS;
+    }
+
+    if (grp == GICV3_G1 && !arm_is_el3_or_mon(env) &&
+        (cs->icc_ctlr_el1[GICV3_S] & ICC_CTLR_EL1_CBPR)) {
+        /* CBPR_EL1S means secure EL1 or AArch32 EL3 !Mon BPR1 accesses
+         * modify BPR0
+         */
+        grp = GICV3_G0;
+    }
+
+    if (grp == GICV3_G1NS && arm_current_el(env) < 3 &&
+        (cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_CBPR)) {
+        /* reads return bpr0 + 1 sat to 7, writes ignored */
+        grp = GICV3_G0;
+        satinc = true;
+    }
+
+    bpr = cs->icc_bpr[grp];
+    if (satinc) {
+        bpr++;
+        bpr = MIN(bpr, 7);
+    }
+
+    trace_gicv3_icc_bpr_read(gicv3_redist_affid(cs), bpr);
+
+    return bpr;
+}
+
+static void icc_bpr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int grp = (ri->crm == 8) ? GICV3_G0 : GICV3_G1;
+
+    trace_gicv3_icc_pmr_write(gicv3_redist_affid(cs), value);
+
+    if (grp == GICV3_G1 && gicv3_use_ns_bank(env)) {
+        grp = GICV3_G1NS;
+    }
+
+    if (grp == GICV3_G1 && !arm_is_el3_or_mon(env) &&
+        (cs->icc_ctlr_el1[GICV3_S] & ICC_CTLR_EL1_CBPR)) {
+        /* CBPR_EL1S means secure EL1 or AArch32 EL3 !Mon BPR1 accesses
+         * modify BPR0
+         */
+        grp = GICV3_G0;
+    }
+
+    if (grp == GICV3_G1NS && arm_current_el(env) < 3 &&
+        (cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_CBPR)) {
+        /* reads return bpr0 + 1 sat to 7, writes ignored */
+        return;
+    }
+
+    cs->icc_bpr[grp] = value & 7;
+    gicv3_cpuif_update(cs);
+}
+
+static uint64_t icc_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t value;
+
+    int regno = ri->opc2 & 3;
+    int grp = ri->crm & 1 ? GICV3_G0 : GICV3_G1;
+
+    if (grp == GICV3_G1 && gicv3_use_ns_bank(env)) {
+        grp = GICV3_G1NS;
+    }
+
+    value = cs->icc_apr[grp][regno];
+
+    trace_gicv3_icc_ap_read(regno, gicv3_redist_affid(cs), value);
+    return value;
+}
+
+static void icc_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+
+    int regno = ri->opc2 & 3;
+    int grp = ri->crm & 1 ? GICV3_G0 : GICV3_G1;
+
+    trace_gicv3_icc_ap_write(regno, gicv3_redist_affid(cs), value);
+
+    if (grp == GICV3_G1 && gicv3_use_ns_bank(env)) {
+        grp = GICV3_G1NS;
+    }
+
+    /* It's not possible to claim that a Non-secure interrupt is active
+     * at a priority outside the Non-secure range (128..255), since this
+     * would otherwise allow malicious NS code to block delivery of S interrupts
+     * by writing a bad value to these registers.
+     */
+    if (grp == GICV3_G1NS && regno < 2 && arm_feature(env, ARM_FEATURE_EL3)) {
+        return;
+    }
+
+    cs->icc_apr[grp][regno] = value & 0xFFFFFFFFU;
+    gicv3_cpuif_update(cs);
+}
+
+static void icc_dir_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          uint64_t value)
+{
+    /* Deactivate interrupt */
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int irq = value & 0xffffff;
+    bool irq_is_secure, single_sec_state, irq_is_grp0;
+    bool route_fiq_to_el3, route_irq_to_el3, route_fiq_to_el2, route_irq_to_el2;
+
+    trace_gicv3_icc_dir_write(gicv3_redist_affid(cs), value);
+
+    if (irq >= cs->gic->num_irq) {
+        /* Also catches special interrupt numbers and LPIs */
+        return;
+    }
+
+    if (!icc_eoi_split(env, cs)) {
+        return;
+    }
+
+    int grp = gicv3_irq_group(cs->gic, cs, irq);
+
+    single_sec_state = cs->gic->gicd_ctlr & GICD_CTLR_DS;
+    irq_is_secure = !single_sec_state && (grp != GICV3_G1NS);
+    irq_is_grp0 = grp == GICV3_G0;
+
+    /* Check whether we're allowed to deactivate this interrupt based
+     * on its group and the current CPU state.
+     * These checks are laid out to correspond to the spec's pseudocode.
+     */
+    route_fiq_to_el3 = env->cp15.scr_el3 & SCR_FIQ;
+    route_irq_to_el3 = env->cp15.scr_el3 & SCR_IRQ;
+    /* No need to include !IsSecure in route_*_to_el2 as it's only
+     * tested in cases where we know !IsSecure is true.
+     */
+    route_fiq_to_el2 = env->cp15.hcr_el2 & HCR_FMO;
+    route_irq_to_el2 = env->cp15.hcr_el2 & HCR_FMO;
+
+    switch (arm_current_el(env)) {
+    case 3:
+        break;
+    case 2:
+        if (single_sec_state && irq_is_grp0 && !route_fiq_to_el3) {
+            break;
+        }
+        if (!irq_is_secure && !irq_is_grp0 && !route_irq_to_el3) {
+            break;
+        }
+        return;
+    case 1:
+        if (!arm_is_secure_below_el3(env)) {
+            if (single_sec_state && irq_is_grp0 &&
+                !route_fiq_to_el3 && !route_fiq_to_el2) {
+                break;
+            }
+            if (!irq_is_secure && !irq_is_grp0 &&
+                !route_irq_to_el3 && !route_irq_to_el2) {
+                break;
+            }
+        } else {
+            if (irq_is_grp0 && !route_fiq_to_el3) {
+                break;
+            }
+            if (!irq_is_grp0 &&
+                (!irq_is_secure || !single_sec_state) &&
+                !route_irq_to_el3) {
+                break;
+            }
+        }
+        return;
+    default:
+        g_assert_not_reached();
+    }
+
+    icc_deactivate_irq(cs, irq);
+}
+
+static uint64_t icc_rpr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int prio = icc_highest_active_prio(cs);
+
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        !arm_is_secure(env) && (env->cp15.scr_el3 & SCR_FIQ)) {
+        /* NS GIC access and Group 0 is inaccessible to NS */
+        if (prio & 0x80) {
+            /* NS mustn't see priorities in the Secure half of the range */
+            prio = 0;
+        } else if (prio != 0xff) {
+            /* Non-idle priority: show the Non-secure view of it */
+            prio = (prio << 1) & 0xff;
+        }
+    }
+
+    trace_gicv3_icc_rpr_read(gicv3_redist_affid(cs), prio);
+    return prio;
+}
+
+static void icc_generate_sgi(CPUARMState *env, GICv3CPUState *cs,
+                             uint64_t value, int grp, bool ns)
+{
+    GICv3State *s = cs->gic;
+
+    /* Extract Aff3/Aff2/Aff1 and shift into the bottom 24 bits */
+    uint64_t aff = extract64(value, 48, 8) << 16 |
+        extract64(value, 32, 8) << 8 |
+        extract64(value, 16, 8);
+    uint32_t targetlist = extract64(value, 0, 16);
+    uint32_t irq = extract64(value, 24, 4);
+    bool irm = extract64(value, 40, 1);
+    int i;
+
+    if (grp == GICV3_G1 && s->gicd_ctlr & GICD_CTLR_DS) {
+        /* If GICD_CTLR.DS == 1, the Distributor treats Secure Group 1
+         * interrupts as Group 0 interrupts and must send Secure Group 0
+         * interrupts to the target CPUs.
+         */
+        grp = GICV3_G0;
+    }
+
+    trace_gicv3_icc_generate_sgi(gicv3_redist_affid(cs), irq, irm,
+                                 aff, targetlist);
+
+    for (i = 0; i < s->num_cpu; i++) {
+        GICv3CPUState *ocs = &s->cpu[i];
+
+        if (irm) {
+            /* IRM == 1 : route to all CPUs except self */
+            if (cs == ocs) {
+                continue;
+            }
+        } else {
+            /* IRM == 0 : route to Aff3.Aff2.Aff1.n for all n in [0..15]
+             * where the corresponding bit is set in targetlist
+             */
+            int aff0;
+
+            if (ocs->gicr_typer >> 40 != aff) {
+                continue;
+            }
+            aff0 = extract64(ocs->gicr_typer, 32, 8);
+            if (aff0 > 15 || extract32(targetlist, aff0, 1) == 0) {
+                continue;
+            }
+        }
+
+        /* The redistributor will check against its own GICR_NSACR as needed */
+        gicv3_redist_send_sgi(ocs, grp, irq, ns);
+    }
+}
+
+static void icc_sgi0r_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                           uint64_t value)
+{
+    /* Generate Secure Group 0 SGI. */
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    bool ns = !arm_is_secure(env);
+
+    icc_generate_sgi(env, cs, value, GICV3_G0, ns);
+}
+
+static void icc_sgi1r_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                           uint64_t value)
+{
+    /* Generate Group 1 SGI for the current Security state */
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int grp;
+    bool ns = !arm_is_secure(env);
+
+    grp = ns ? GICV3_G1NS : GICV3_G1;
+    icc_generate_sgi(env, cs, value, grp, ns);
+}
+
+static void icc_asgi1r_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
+{
+    /* Generate Group 1 SGI for the Security state that is not
+     * the current state
+     */
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int grp;
+    bool ns = !arm_is_secure(env);
+
+    grp = ns ? GICV3_G1 : GICV3_G1NS;
+    icc_generate_sgi(env, cs, value, grp, ns);
+}
+
+static uint64_t icc_igrpen_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int grp = ri->opc2 & 1 ? GICV3_G1 : GICV3_G0;
+    uint64_t value;
+
+    if (grp == GICV3_G1 && gicv3_use_ns_bank(env)) {
+        grp = GICV3_G1NS;
+    }
+
+    value = cs->icc_igrpen[grp];
+    trace_gicv3_icc_igrpen_read(gicv3_redist_affid(cs), value);
+    return value;
+}
+
+static void icc_igrpen_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                             uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int grp = ri->opc2 & 1 ? GICV3_G1 : GICV3_G0;
+
+    trace_gicv3_icc_igrpen_write(gicv3_redist_affid(cs), value);
+
+    if (grp == GICV3_G1 && gicv3_use_ns_bank(env)) {
+        grp = GICV3_G1NS;
+    }
+
+    cs->icc_igrpen[grp] = value & ICC_IGRPEN_ENABLE;
+    gicv3_cpuif_update(cs);
+}
+
+static uint64_t icc_igrpen1_el3_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+
+    /* IGRPEN1_EL3 bits 0 and 1 are r/w aliases into IGRPEN1_EL1 NS and S */
+    return cs->icc_igrpen[GICV3_G1NS] | (cs->icc_igrpen[GICV3_G1] << 1);
+}
+
+static void icc_igrpen1_el3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+
+    trace_gicv3_icc_igrpen1_el3_write(gicv3_redist_affid(cs), value);
+
+    /* IGRPEN1_EL3 bits 0 and 1 are r/w aliases into IGRPEN1_EL1 NS and S */
+    cs->icc_igrpen[GICV3_G1NS] = extract32(value, 0, 1);
+    cs->icc_igrpen[GICV3_G1] = extract32(value, 1, 1);
+    gicv3_cpuif_update(cs);
+}
+
+static uint64_t icc_ctlr_el1_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int bank = gicv3_use_ns_bank(env) ? GICV3_NS : GICV3_S;
+    uint64_t value;
+
+    value = cs->icc_ctlr_el1[bank];
+    trace_gicv3_icc_ctlr_read(gicv3_redist_affid(cs), value);
+    return value;
+}
+
+static void icc_ctlr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    int bank = gicv3_use_ns_bank(env) ? GICV3_NS : GICV3_S;
+    uint64_t mask;
+
+    trace_gicv3_icc_ctlr_write(gicv3_redist_affid(cs), value);
+
+    /* Only CBPR and EOIMODE can be RW;
+     * for us PMHE is RAZ/WI (we don't implement 1-of-N interrupts or
+     * the asseciated priority-based routing of them);
+     * if EL3 is implemented and GICD_CTLR.DS == 0, then PMHE and CBPR are RO.
+     */
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        ((cs->gic->gicd_ctlr & GICD_CTLR_DS) == 0)) {
+        mask = ICC_CTLR_EL1_EOIMODE;
+    } else {
+        mask = ICC_CTLR_EL1_CBPR | ICC_CTLR_EL1_EOIMODE;
+    }
+
+    cs->icc_ctlr_el1[bank] &= ~mask;
+    cs->icc_ctlr_el1[bank] |= (value & mask);
+    gicv3_cpuif_update(cs);
+}
+
+
+static uint64_t icc_ctlr_el3_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t value;
+
+    value = cs->icc_ctlr_el3;
+    if (cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_EOIMODE) {
+        value |= ICC_CTLR_EL3_EOIMODE_EL1NS;
+    }
+    if (cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_CBPR) {
+        value |= ICC_CTLR_EL3_CBPR_EL1NS;
+    }
+    if (cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_EOIMODE) {
+        value |= ICC_CTLR_EL3_EOIMODE_EL1S;
+    }
+    if (cs->icc_ctlr_el1[GICV3_NS] & ICC_CTLR_EL1_CBPR) {
+        value |= ICC_CTLR_EL3_CBPR_EL1S;
+    }
+
+    trace_gicv3_icc_ctlr_el3_read(gicv3_redist_affid(cs), value);
+    return value;
+}
+
+static void icc_ctlr_el3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+    uint64_t mask;
+
+    trace_gicv3_icc_ctlr_el3_write(gicv3_redist_affid(cs), value);
+
+    /* *_EL1NS and *_EL1S bits are aliases into the ICC_CTLR_EL1 bits. */
+    cs->icc_ctlr_el1[GICV3_NS] &= (ICC_CTLR_EL1_CBPR | ICC_CTLR_EL1_EOIMODE);
+    if (value & ICC_CTLR_EL3_EOIMODE_EL1NS) {
+        cs->icc_ctlr_el1[GICV3_NS] |= ICC_CTLR_EL1_EOIMODE;
+    }
+    if (value & ICC_CTLR_EL3_CBPR_EL1NS) {
+        cs->icc_ctlr_el1[GICV3_NS] |= ICC_CTLR_EL1_CBPR;
+    }
+
+    cs->icc_ctlr_el1[GICV3_S] &= (ICC_CTLR_EL1_CBPR | ICC_CTLR_EL1_EOIMODE);
+    if (value & ICC_CTLR_EL3_EOIMODE_EL1S) {
+        cs->icc_ctlr_el1[GICV3_S] |= ICC_CTLR_EL1_EOIMODE;
+    }
+    if (value & ICC_CTLR_EL3_CBPR_EL1S) {
+        cs->icc_ctlr_el1[GICV3_S] |= ICC_CTLR_EL1_CBPR;
+    }
+
+    /* The only bit stored in icc_ctlr_el3 which is writeable is EOIMODE_EL3: */
+    mask = ICC_CTLR_EL3_EOIMODE_EL3;
+
+    cs->icc_ctlr_el3 &= ~mask;
+    cs->icc_ctlr_el3 |= (value & mask);
+    gicv3_cpuif_update(cs);
+}
+
+static CPAccessResult gicv3_irqfiq_access(CPUARMState *env,
+                                          const ARMCPRegInfo *ri, bool isread)
+{
+    CPAccessResult r = CP_ACCESS_OK;
+
+    if ((env->cp15.scr_el3 & (SCR_FIQ | SCR_IRQ)) == (SCR_FIQ | SCR_IRQ)) {
+        switch (arm_current_el(env)) {
+        case 1:
+            if (arm_is_secure_below_el3(env) ||
+                ((env->cp15.hcr_el2 & (HCR_IMO | HCR_FMO)) == 0)) {
+                r = CP_ACCESS_TRAP_EL3;
+            }
+            break;
+        case 2:
+            r = CP_ACCESS_TRAP_EL3;
+            break;
+        case 3:
+            if (!is_a64(env) && !arm_is_el3_or_mon(env)) {
+                r = CP_ACCESS_TRAP_EL3;
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    }
+
+    if (r == CP_ACCESS_TRAP_EL3 && !arm_el_is_aa64(env, 3)) {
+        r = CP_ACCESS_TRAP;
+    }
+    return r;
+}
+
+static CPAccessResult gicv3_fiq_access(CPUARMState *env,
+                                       const ARMCPRegInfo *ri, bool isread)
+{
+    CPAccessResult r = CP_ACCESS_OK;
+
+    if (env->cp15.scr_el3 & SCR_FIQ) {
+        switch (arm_current_el(env)) {
+        case 1:
+            if (arm_is_secure_below_el3(env) ||
+                ((env->cp15.hcr_el2 & HCR_FMO) == 0)) {
+                r = CP_ACCESS_TRAP_EL3;
+            }
+            break;
+        case 2:
+            r = CP_ACCESS_TRAP_EL3;
+            break;
+        case 3:
+            if (!is_a64(env) && !arm_is_el3_or_mon(env)) {
+                r = CP_ACCESS_TRAP_EL3;
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    }
+
+    if (r == CP_ACCESS_TRAP_EL3 && !arm_el_is_aa64(env, 3)) {
+        r = CP_ACCESS_TRAP;
+    }
+    return r;
+}
+
+static CPAccessResult gicv3_irq_access(CPUARMState *env,
+                                       const ARMCPRegInfo *ri, bool isread)
+{
+    CPAccessResult r = CP_ACCESS_OK;
+
+    if (env->cp15.scr_el3 & SCR_IRQ) {
+        switch (arm_current_el(env)) {
+        case 1:
+            if (arm_is_secure_below_el3(env) ||
+                ((env->cp15.hcr_el2 & HCR_IMO) == 0)) {
+                r = CP_ACCESS_TRAP_EL3;
+            }
+            break;
+        case 2:
+            r = CP_ACCESS_TRAP_EL3;
+            break;
+        case 3:
+            if (!is_a64(env) && !arm_is_el3_or_mon(env)) {
+                r = CP_ACCESS_TRAP_EL3;
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+    }
+
+    if (r == CP_ACCESS_TRAP_EL3 && !arm_el_is_aa64(env, 3)) {
+        r = CP_ACCESS_TRAP;
+    }
+    return r;
+}
+
+static void icc_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    GICv3CPUState *cs = icc_cs_from_env(env);
+
+    cs->icc_ctlr_el1[GICV3_S] = ICC_CTLR_EL1_A3V |
+        (1 << ICC_CTLR_EL1_IDBITS_SHIFT) |
+        (7 << ICC_CTLR_EL1_PRIBITS_SHIFT);
+    cs->icc_ctlr_el1[GICV3_NS] = ICC_CTLR_EL1_A3V |
+        (1 << ICC_CTLR_EL1_IDBITS_SHIFT) |
+        (7 << ICC_CTLR_EL1_PRIBITS_SHIFT);
+    cs->icc_pmr_el1 = 0;
+    cs->icc_bpr[GICV3_G0] = GIC_MIN_BPR;
+    cs->icc_bpr[GICV3_G1] = GIC_MIN_BPR;
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        cs->icc_bpr[GICV3_G1NS] = GIC_MIN_BPR_NS;
+    } else {
+        cs->icc_bpr[GICV3_G1NS] = GIC_MIN_BPR;
+    }
+    memset(cs->icc_apr, 0, sizeof(cs->icc_apr));
+    memset(cs->icc_igrpen, 0, sizeof(cs->icc_igrpen));
+    cs->icc_ctlr_el3 = ICC_CTLR_EL3_NDS | ICC_CTLR_EL3_A3V |
+        (1 << ICC_CTLR_EL3_IDBITS_SHIFT) |
+        (7 << ICC_CTLR_EL3_PRIBITS_SHIFT);
+}
+
+static const ARMCPRegInfo gicv3_cpuif_reginfo[] = {
+    { .name = "ICC_PMR_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 6, .opc2 = 0,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irqfiq_access,
+      .readfn = icc_pmr_read,
+      .writefn = icc_pmr_write,
+      /* We hang the whole cpu interface reset routine off here
+       * rather than parcelling it out into one little function
+       * per register
+       */
+      .resetfn = icc_reset,
+    },
+    { .name = "ICC_IAR0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 0,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_R, .accessfn = gicv3_fiq_access,
+      .readfn = icc_iar0_read,
+    },
+    { .name = "ICC_EOIR0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 1,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_fiq_access,
+      .writefn = icc_eoir_write,
+    },
+    { .name = "ICC_HPPIR0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 2,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_R, .accessfn = gicv3_fiq_access,
+      .readfn = icc_hppir0_read,
+    },
+    { .name = "ICC_BPR0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 3,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_fiq_access,
+      .fieldoffset = offsetof(GICv3CPUState, icc_bpr[GICV3_G0]),
+      .writefn = icc_bpr_write,
+    },
+    { .name = "ICC_AP0R0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 4,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_fiq_access,
+      .fieldoffset = offsetof(GICv3CPUState, icc_apr[GICV3_G0][0]),
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_AP0R1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 5,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_fiq_access,
+      .fieldoffset = offsetof(GICv3CPUState, icc_apr[GICV3_G0][1]),
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_AP0R2_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 6,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_fiq_access,
+      .fieldoffset = offsetof(GICv3CPUState, icc_apr[GICV3_G0][2]),
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_AP0R3_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 8, .opc2 = 7,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_fiq_access,
+      .fieldoffset = offsetof(GICv3CPUState, icc_apr[GICV3_G0][3]),
+      .writefn = icc_ap_write,
+    },
+    /* All the ICC_AP1R*_EL1 registers are banked */
+    { .name = "ICC_AP1R0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 9, .opc2 = 0,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irq_access,
+      .readfn = icc_ap_read,
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_AP1R1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 9, .opc2 = 1,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irq_access,
+      .readfn = icc_ap_read,
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_AP1R2_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 9, .opc2 = 2,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irq_access,
+      .readfn = icc_ap_read,
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_AP1R3_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 9, .opc2 = 3,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irq_access,
+      .readfn = icc_ap_read,
+      .writefn = icc_ap_write,
+    },
+    { .name = "ICC_DIR_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 11, .opc2 = 1,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_dir_write,
+    },
+    { .name = "ICC_RPR_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 11, .opc2 = 3,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_R, .accessfn = gicv3_irqfiq_access,
+      .readfn = icc_rpr_read,
+    },
+    { .name = "ICC_SGI1R_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 11, .opc2 = 5,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_sgi1r_write,
+    },
+    { .name = "ICC_SGI1R",
+      .cp = 15, .opc1 = 0, .crm = 12,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_sgi1r_write,
+    },
+    { .name = "ICC_ASGI1R_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 11, .opc2 = 6,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_asgi1r_write,
+    },
+    { .name = "ICC_ASGI1R",
+      .cp = 15, .opc1 = 1, .crm = 12,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_asgi1r_write,
+    },
+    { .name = "ICC_SGI0R_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 11, .opc2 = 7,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_sgi0r_write,
+    },
+    { .name = "ICC_SGI0R",
+      .cp = 15, .opc1 = 2, .crm = 12,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irqfiq_access,
+      .writefn = icc_sgi0r_write,
+    },
+    { .name = "ICC_IAR1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 0,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_R, .accessfn = gicv3_irq_access,
+      .readfn = icc_iar1_read,
+    },
+    { .name = "ICC_EOIR1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 1,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_W, .accessfn = gicv3_irq_access,
+      .writefn = icc_eoir_write,
+    },
+    { .name = "ICC_HPPIR1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 2,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_R, .accessfn = gicv3_irq_access,
+      .readfn = icc_hppir1_read,
+    },
+    /* This register is banked */
+    { .name = "ICC_BPR1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 3,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irq_access,
+      .readfn = icc_bpr_read,
+      .writefn = icc_bpr_write,
+    },
+    /* This register is banked */
+    { .name = "ICC_CTLR_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 4,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irqfiq_access,
+      .readfn = icc_ctlr_el1_read,
+      .writefn = icc_ctlr_el1_write,
+    },
+    { .name = "ICC_SRE_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 5,
+      .type = ARM_CP_NO_RAW | ARM_CP_CONST,
+      .access = PL1_RW,
+      /* We don't support IRQ/FIQ bypass and system registers are
+       * always enabled, so all our bits are RAZ/WI or RAO/WI.
+       * This register is banked but since it's constant we don't
+       * need to do anything special.
+       */
+      .resetvalue = 0x7,
+    },
+    { .name = "ICC_IGRPEN0_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 6,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_fiq_access,
+      .fieldoffset = offsetof(GICv3CPUState, icc_igrpen[GICV3_G0]),
+      .writefn = icc_igrpen_write,
+    },
+    /* This register is banked */
+    { .name = "ICC_IGRPEN1_EL1", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 12, .opc2 = 7,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL1_RW, .accessfn = gicv3_irq_access,
+      .readfn = icc_igrpen_read,
+      .writefn = icc_igrpen_write,
+    },
+    { .name = "ICC_SRE_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 9, .opc2 = 5,
+      .type = ARM_CP_NO_RAW | ARM_CP_CONST,
+      .access = PL2_RW,
+      /* We don't support IRQ/FIQ bypass and system registers are
+       * always enabled, so all our bits are RAZ/WI or RAO/WI.
+       */
+      .resetvalue = 0xf,
+    },
+    { .name = "ICC_CTLR_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 12, .opc2 = 4,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL3_RW,
+      .fieldoffset = offsetof(GICv3CPUState, icc_ctlr_el3),
+      .readfn = icc_ctlr_el3_read,
+      .writefn = icc_ctlr_el3_write,
+    },
+    { .name = "ICC_SRE_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 12, .opc2 = 5,
+      .type = ARM_CP_NO_RAW | ARM_CP_CONST,
+      .access = PL3_RW,
+      /* We don't support IRQ/FIQ bypass and system registers are
+       * always enabled, so all our bits are RAZ/WI or RAO/WI.
+       */
+      .resetvalue = 0xf,
+    },
+    { .name = "ICC_IGRPEN1_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 12, .opc2 = 7,
+      .type = ARM_CP_IO | ARM_CP_NO_RAW,
+      .access = PL3_RW,
+      .readfn = icc_igrpen1_el3_read,
+      .writefn = icc_igrpen1_el3_write,
+    },
+    REGINFO_SENTINEL
+};
+
+static void gicv3_cpuif_el_change_hook(ARMCPU *cpu, void *opaque)
+{
+    GICv3CPUState *cs = opaque;
+
+    gicv3_cpuif_update(cs);
+}
+
+void gicv3_init_cpuif(GICv3State *s)
+{
+    /* Called from the GICv3 realize function; register our system
+     * registers with the CPU
+     */
+    int i;
+
+    for (i = 0; i < s->num_cpu; i++) {
+        ARMCPU *cpu = ARM_CPU(qemu_get_cpu(i));
+        GICv3CPUState *cs = &s->cpu[i];
+
+        /* Note that we can't just use the GICv3CPUState as an opaque pointer
+         * in define_arm_cp_regs_with_opaque(), because when we're called back
+         * it might be with code translated by CPU 0 but run by CPU 1, in
+         * which case we'd get the wrong value.
+         * So instead we define the regs with no ri->opaque info, and
+         * get back to the GICv3CPUState from the ARMCPU by reading back
+         * the opaque pointer from the el_change_hook, which we're going
+         * to need to register anyway.
+         */
+        define_arm_cp_regs(cpu, gicv3_cpuif_reginfo);
+        arm_register_el_change_hook(cpu, gicv3_cpuif_el_change_hook, cs);
+    }
+}
diff --git a/hw/intc/arm_gicv3_dist.c b/hw/intc/arm_gicv3_dist.c
new file mode 100644
index 0000000..3ea3dd0
--- /dev/null
+++ b/hw/intc/arm_gicv3_dist.c
@@ -0,0 +1,880 @@
+/*
+ * ARM GICv3 emulation: Distributor
+ *
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2016 Linaro Limited.
+ * Written by Shlomo Pongratz, Peter Maydell
+ *
+ * This code is licensed under the GPL, version 2 or (at your option)
+ * any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "trace.h"
+#include "gicv3_internal.h"
+
+/* The GICD_NSACR registers contain a two bit field for each interrupt which
+ * allows the guest to give NonSecure code access to registers controlling
+ * Secure interrupts:
+ *  0b00: no access (NS accesses to bits for Secure interrupts will RAZ/WI)
+ *  0b01: NS r/w accesses permitted to ISPENDR, SETSPI_NSR, SGIR
+ *  0b10: as 0b01, and also r/w to ICPENDR, r/o to ISACTIVER/ICACTIVER,
+ *        and w/o to CLRSPI_NSR
+ *  0b11: as 0b10, and also r/w to IROUTER and ITARGETSR
+ *
+ * Given a (multiple-of-32) interrupt number, these mask functions return
+ * a mask word where each bit is 1 if the NSACR settings permit access
+ * to the interrupt. The mask returned can then be ORed with the GICD_GROUP
+ * word for this set of interrupts to give an overall mask.
+ */
+
+typedef uint32_t maskfn(GICv3State *s, int irq);
+
+static uint32_t mask_nsacr_ge1(GICv3State *s, int irq)
+{
+    /* Return a mask where each bit is set if the NSACR field is >= 1 */
+    uint64_t raw_nsacr = s->gicd_nsacr[irq / 16 + 1];
+
+    raw_nsacr = raw_nsacr << 32 | s->gicd_nsacr[irq / 16];
+    raw_nsacr = (raw_nsacr >> 1) | raw_nsacr;
+    return half_unshuffle64(raw_nsacr);
+}
+
+static uint32_t mask_nsacr_ge2(GICv3State *s, int irq)
+{
+    /* Return a mask where each bit is set if the NSACR field is >= 2 */
+    uint64_t raw_nsacr = s->gicd_nsacr[irq / 16 + 1];
+
+    raw_nsacr = raw_nsacr << 32 | s->gicd_nsacr[irq / 16];
+    raw_nsacr = raw_nsacr >> 1;
+    return half_unshuffle64(raw_nsacr);
+}
+
+/* We don't need a mask_nsacr_ge3() because IROUTER<n> isn't a bitmap register,
+ * but it would be implemented using:
+ *  raw_nsacr = (raw_nsacr >> 1) & raw_nsacr;
+ */
+
+static uint32_t mask_group_and_nsacr(GICv3State *s, MemTxAttrs attrs,
+                                     maskfn *maskfn, int irq)
+{
+    /* Return a 32-bit mask which should be applied for this set of 32
+     * interrupts; each bit is 1 if access is permitted by the
+     * combination of attrs.secure, GICD_GROUPR and GICD_NSACR.
+     */
+    uint32_t mask;
+
+    if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+        /* bits for Group 0 or Secure Group 1 interrupts are RAZ/WI
+         * unless the NSACR bits permit access.
+         */
+        mask = *gic_bmp_ptr32(s->group, irq);
+        if (maskfn) {
+            mask |= maskfn(s, irq);
+        }
+        return mask;
+    }
+    return 0xFFFFFFFFU;
+}
+
+static int gicd_ns_access(GICv3State *s, int irq)
+{
+    /* Return the 2 bit NS_access<x> field from GICD_NSACR<n> for the
+     * specified interrupt.
+     */
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return 0;
+    }
+    return extract32(s->gicd_nsacr[irq / 16], (irq % 16) * 2, 2);
+}
+
+static void gicd_write_set_bitmap_reg(GICv3State *s, MemTxAttrs attrs,
+                                      uint32_t *bmp,
+                                      maskfn *maskfn,
+                                      int offset, uint32_t val)
+{
+    /* Helper routine to implement writing to a "set-bitmap" register
+     * (GICD_ISENABLER, GICD_ISPENDR, etc).
+     * Semantics implemented here:
+     * RAZ/WI for SGIs, PPIs, unimplemented IRQs
+     * Bits corresponding to Group 0 or Secure Group 1 interrupts RAZ/WI.
+     * Writing 1 means "set bit in bitmap"; writing 0 is ignored.
+     * offset should be the offset in bytes of the register from the start
+     * of its group.
+     */
+    int irq = offset * 8;
+
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return;
+    }
+    val &= mask_group_and_nsacr(s, attrs, maskfn, irq);
+    *gic_bmp_ptr32(bmp, irq) |= val;
+    gicv3_update(s, irq, 32);
+}
+
+static void gicd_write_clear_bitmap_reg(GICv3State *s, MemTxAttrs attrs,
+                                        uint32_t *bmp,
+                                        maskfn *maskfn,
+                                        int offset, uint32_t val)
+{
+    /* Helper routine to implement writing to a "clear-bitmap" register
+     * (GICD_ICENABLER, GICD_ICPENDR, etc).
+     * Semantics implemented here:
+     * RAZ/WI for SGIs, PPIs, unimplemented IRQs
+     * Bits corresponding to Group 0 or Secure Group 1 interrupts RAZ/WI.
+     * Writing 1 means "clear bit in bitmap"; writing 0 is ignored.
+     * offset should be the offset in bytes of the register from the start
+     * of its group.
+     */
+    int irq = offset * 8;
+
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return;
+    }
+    val &= mask_group_and_nsacr(s, attrs, maskfn, irq);
+    *gic_bmp_ptr32(bmp, irq) &= ~val;
+    gicv3_update(s, irq, 32);
+}
+
+static uint32_t gicd_read_bitmap_reg(GICv3State *s, MemTxAttrs attrs,
+                                     uint32_t *bmp,
+                                     maskfn *maskfn,
+                                     int offset)
+{
+    /* Helper routine to implement reading a "set/clear-bitmap" register
+     * (GICD_ICENABLER, GICD_ISENABLER, GICD_ICPENDR, etc).
+     * Semantics implemented here:
+     * RAZ/WI for SGIs, PPIs, unimplemented IRQs
+     * Bits corresponding to Group 0 or Secure Group 1 interrupts RAZ/WI.
+     * offset should be the offset in bytes of the register from the start
+     * of its group.
+     */
+    int irq = offset * 8;
+    uint32_t val;
+
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return 0;
+    }
+    val = *gic_bmp_ptr32(bmp, irq);
+    if (bmp == s->pending) {
+        /* The PENDING register is a special case -- for level triggered
+         * interrupts, the PENDING state is the logical OR of the state of
+         * the PENDING latch with the input line level.
+         */
+        uint32_t edge = *gic_bmp_ptr32(s->edge_trigger, irq);
+        uint32_t level = *gic_bmp_ptr32(s->level, irq);
+        val |= (~edge & level);
+    }
+    val &= mask_group_and_nsacr(s, attrs, maskfn, irq);
+    return val;
+}
+
+static uint8_t gicd_read_ipriorityr(GICv3State *s, MemTxAttrs attrs, int irq)
+{
+    /* Read the value of GICD_IPRIORITYR<n> for the specified interrupt,
+     * honouring security state (these are RAZ/WI for Group 0 or Secure
+     * Group 1 interrupts).
+     */
+    uint32_t prio;
+
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return 0;
+    }
+
+    prio = s->gicd_ipriority[irq];
+
+    if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+        if (!gicv3_gicd_group_test(s, irq)) {
+            /* Fields for Group 0 or Secure Group 1 interrupts are RAZ/WI */
+            return 0;
+        }
+        /* NS view of the interrupt priority */
+        prio = (prio << 1) & 0xff;
+    }
+    return prio;
+}
+
+static void gicd_write_ipriorityr(GICv3State *s, MemTxAttrs attrs, int irq,
+                                  uint8_t value)
+{
+    /* Write the value of GICD_IPRIORITYR<n> for the specified interrupt,
+     * honouring security state (these are RAZ/WI for Group 0 or Secure
+     * Group 1 interrupts).
+     */
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return;
+    }
+
+    if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+        if (!gicv3_gicd_group_test(s, irq)) {
+            /* Fields for Group 0 or Secure Group 1 interrupts are RAZ/WI */
+            return;
+        }
+        /* NS view of the interrupt priority */
+        value = 0x80 | (value >> 1);
+    }
+    s->gicd_ipriority[irq] = value;
+}
+
+static uint64_t gicd_read_irouter(GICv3State *s, MemTxAttrs attrs, int irq)
+{
+    /* Read the value of GICD_IROUTER<n> for the specified interrupt,
+     * honouring security state.
+     */
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return 0;
+    }
+
+    if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+        /* RAZ/WI for NS accesses to secure interrupts */
+        if (!gicv3_gicd_group_test(s, irq)) {
+            if (gicd_ns_access(s, irq) != 3) {
+                return 0;
+            }
+        }
+    }
+
+    return s->gicd_irouter[irq];
+}
+
+static void gicd_write_irouter(GICv3State *s, MemTxAttrs attrs, int irq,
+                               uint64_t val)
+{
+    /* Write the value of GICD_IROUTER<n> for the specified interrupt,
+     * honouring security state.
+     */
+    if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+        return;
+    }
+
+    if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+        /* RAZ/WI for NS accesses to secure interrupts */
+        if (!gicv3_gicd_group_test(s, irq)) {
+            if (gicd_ns_access(s, irq) != 3) {
+                return;
+            }
+        }
+    }
+
+    s->gicd_irouter[irq] = val;
+    gicv3_cache_target_cpustate(s, irq);
+    gicv3_update(s, irq, 1);
+}
+
+static MemTxResult gicd_readb(GICv3State *s, hwaddr offset,
+                              uint64_t *data, MemTxAttrs attrs)
+{
+    /* Most GICv3 distributor registers do not support byte accesses. */
+    switch (offset) {
+    case GICD_CPENDSGIR ... GICD_CPENDSGIR + 0xf:
+    case GICD_SPENDSGIR ... GICD_SPENDSGIR + 0xf:
+    case GICD_ITARGETSR ... GICD_ITARGETSR + 0x3ff:
+        /* This GIC implementation always has affinity routing enabled,
+         * so these registers are all RAZ/WI.
+         */
+        return MEMTX_OK;
+    case GICD_IPRIORITYR ... GICD_IPRIORITYR + 0x3ff:
+        *data = gicd_read_ipriorityr(s, attrs, offset - GICD_IPRIORITYR);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicd_writeb(GICv3State *s, hwaddr offset,
+                               uint64_t value, MemTxAttrs attrs)
+{
+    /* Most GICv3 distributor registers do not support byte accesses. */
+    switch (offset) {
+    case GICD_CPENDSGIR ... GICD_CPENDSGIR + 0xf:
+    case GICD_SPENDSGIR ... GICD_SPENDSGIR + 0xf:
+    case GICD_ITARGETSR ... GICD_ITARGETSR + 0x3ff:
+        /* This GIC implementation always has affinity routing enabled,
+         * so these registers are all RAZ/WI.
+         */
+        return MEMTX_OK;
+    case GICD_IPRIORITYR ... GICD_IPRIORITYR + 0x3ff:
+    {
+        int irq = offset - GICD_IPRIORITYR;
+
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            return MEMTX_OK;
+        }
+        gicd_write_ipriorityr(s, attrs, irq, value);
+        gicv3_update(s, irq, 1);
+        return MEMTX_OK;
+    }
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicd_readw(GICv3State *s, hwaddr offset,
+                              uint64_t *data, MemTxAttrs attrs)
+{
+    /* Only GICD_SETSPI_NSR, GICD_CLRSPI_NSR, GICD_SETSPI_SR and GICD_SETSPI_NSR
+     * support 16 bit accesses, and those registers are all part of the
+     * optional message-based SPI feature which this GIC does not currently
+     * implement (ie for us GICD_TYPER.MBIS == 0), so for us they are
+     * reserved.
+     */
+    return MEMTX_ERROR;
+}
+
+static MemTxResult gicd_writew(GICv3State *s, hwaddr offset,
+                               uint64_t value, MemTxAttrs attrs)
+{
+    /* Only GICD_SETSPI_NSR, GICD_CLRSPI_NSR, GICD_SETSPI_SR and GICD_SETSPI_NSR
+     * support 16 bit accesses, and those registers are all part of the
+     * optional message-based SPI feature which this GIC does not currently
+     * implement (ie for us GICD_TYPER.MBIS == 0), so for us they are
+     * reserved.
+     */
+    return MEMTX_ERROR;
+}
+
+static MemTxResult gicd_readl(GICv3State *s, hwaddr offset,
+                              uint64_t *data, MemTxAttrs attrs)
+{
+    /* Almost all GICv3 distributor registers are 32-bit.
+     * Note that WO registers must return an UNKNOWN value on reads,
+     * not an abort.
+     */
+
+    switch (offset) {
+    case GICD_CTLR:
+        if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+            /* The NS view of the GICD_CTLR sees only certain bits:
+             * + bit [31] (RWP) is an alias of the Secure bit [31]
+             * + bit [4] (ARE_NS) is an alias of Secure bit [5]
+             * + bit [1] (EnableGrp1A) is an alias of Secure bit [1] if
+             *   NS affinity routing is enabled, otherwise RES0
+             * + bit [0] (EnableGrp1) is an alias of Secure bit [1] if
+             *   NS affinity routing is not enabled, otherwise RES0
+             * Since for QEMU affinity routing is always enabled
+             * for both S and NS this means that bits [4] and [5] are
+             * both always 1, and we can simply make the NS view
+             * be bits 31, 4 and 1 of the S view.
+             */
+            *data = s->gicd_ctlr & (GICD_CTLR_ARE_S |
+                                    GICD_CTLR_EN_GRP1NS |
+                                    GICD_CTLR_RWP);
+        } else {
+            *data = s->gicd_ctlr;
+        }
+        return MEMTX_OK;
+    case GICD_TYPER:
+    {
+        /* For this implementation:
+         * No1N == 1 (1-of-N SPI interrupts not supported)
+         * A3V == 1 (non-zero values of Affinity level 3 supported)
+         * IDbits == 0xf (we support 16-bit interrupt identifiers)
+         * DVIS == 0 (Direct virtual LPI injection not supported)
+         * LPIS == 0 (LPIs not supported)
+         * MBIS == 0 (message-based SPIs not supported)
+         * SecurityExtn == 1 if security extns supported
+         * CPUNumber == 0 since for us ARE is always 1
+         * ITLinesNumber == (num external irqs / 32) - 1
+         */
+        int itlinesnumber = ((s->num_irq - GIC_INTERNAL) / 32) - 1;
+
+        *data = (1 << 25) | (1 << 24) | (s->security_extn << 10) |
+            (0xf << 19) | itlinesnumber;
+        return MEMTX_OK;
+    }
+    case GICD_IIDR:
+        /* We claim to be an ARM r0p0 with a zero ProductID.
+         * This is the same as an r0p0 GIC-500.
+         */
+        *data = gicv3_iidr();
+        return MEMTX_OK;
+    case GICD_STATUSR:
+        /* RAZ/WI for us (this is an optional register and our implementation
+         * does not track RO/WO/reserved violations to report them to the guest)
+         */
+        *data = 0;
+        return MEMTX_OK;
+    case GICD_IGROUPR ... GICD_IGROUPR + 0x7f:
+    {
+        int irq;
+
+        if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+            *data = 0;
+            return MEMTX_OK;
+        }
+        /* RAZ/WI for SGIs, PPIs, unimplemented irqs */
+        irq = (offset - GICD_IGROUPR) * 8;
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            *data = 0;
+            return MEMTX_OK;
+        }
+        *data = *gic_bmp_ptr32(s->group, irq);
+        return MEMTX_OK;
+    }
+    case GICD_ISENABLER ... GICD_ISENABLER + 0x7f:
+        *data = gicd_read_bitmap_reg(s, attrs, s->enabled, NULL,
+                                     offset - GICD_ISENABLER);
+        return MEMTX_OK;
+    case GICD_ICENABLER ... GICD_ICENABLER + 0x7f:
+        *data = gicd_read_bitmap_reg(s, attrs, s->enabled, NULL,
+                                     offset - GICD_ICENABLER);
+        return MEMTX_OK;
+    case GICD_ISPENDR ... GICD_ISPENDR + 0x7f:
+        *data = gicd_read_bitmap_reg(s, attrs, s->pending, mask_nsacr_ge1,
+                                     offset - GICD_ISPENDR);
+        return MEMTX_OK;
+    case GICD_ICPENDR ... GICD_ICPENDR + 0x7f:
+        *data = gicd_read_bitmap_reg(s, attrs, s->pending, mask_nsacr_ge2,
+                                     offset - GICD_ICPENDR);
+        return MEMTX_OK;
+    case GICD_ISACTIVER ... GICD_ISACTIVER + 0x7f:
+        *data = gicd_read_bitmap_reg(s, attrs, s->active, mask_nsacr_ge2,
+                                     offset - GICD_ISACTIVER);
+        return MEMTX_OK;
+    case GICD_ICACTIVER ... GICD_ICACTIVER + 0x7f:
+        *data = gicd_read_bitmap_reg(s, attrs, s->active, mask_nsacr_ge2,
+                                     offset - GICD_ICACTIVER);
+        return MEMTX_OK;
+    case GICD_IPRIORITYR ... GICD_IPRIORITYR + 0x3ff:
+    {
+        int i, irq = offset - GICD_IPRIORITYR;
+        uint32_t value = 0;
+
+        for (i = irq + 3; i >= irq; i--, value <<= 8) {
+            value |= gicd_read_ipriorityr(s, attrs, i);
+        }
+        *data = value;
+        return MEMTX_OK;
+    }
+    case GICD_ITARGETSR ... GICD_ITARGETSR + 0x3ff:
+        /* RAZ/WI since affinity routing is always enabled */
+        *data = 0;
+        return MEMTX_OK;
+    case GICD_ICFGR ... GICD_ICFGR + 0xff:
+    {
+        /* Here only the even bits are used; odd bits are RES0 */
+        int irq = (offset - GICD_ICFGR) * 4;
+        uint32_t value = 0;
+
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            *data = 0;
+            return MEMTX_OK;
+        }
+
+        /* Since our edge_trigger bitmap is one bit per irq, we only need
+         * half of the 32-bit word, which we can then spread out
+         * into the odd bits.
+         */
+        value = *gic_bmp_ptr32(s->edge_trigger, irq & ~0x1f);
+        value &= mask_group_and_nsacr(s, attrs, NULL, irq & ~0x1f);
+        value = extract32(value, (irq & 0x1f) ? 16 : 0, 16);
+        value = half_shuffle32(value) << 1;
+        *data = value;
+        return MEMTX_OK;
+    }
+    case GICD_IGRPMODR ... GICD_IGRPMODR + 0xff:
+    {
+        int irq;
+
+        if ((s->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            *data = 0;
+            return MEMTX_OK;
+        }
+        /* RAZ/WI for SGIs, PPIs, unimplemented irqs */
+        irq = (offset - GICD_IGRPMODR) * 8;
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            *data = 0;
+            return MEMTX_OK;
+        }
+        *data = *gic_bmp_ptr32(s->grpmod, irq);
+        return MEMTX_OK;
+    }
+    case GICD_NSACR ... GICD_NSACR + 0xff:
+    {
+        /* Two bits per interrupt */
+        int irq = (offset - GICD_NSACR) * 4;
+
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            *data = 0;
+            return MEMTX_OK;
+        }
+
+        if ((s->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            *data = 0;
+            return MEMTX_OK;
+        }
+
+        *data = s->gicd_nsacr[irq / 16];
+        return MEMTX_OK;
+    }
+    case GICD_CPENDSGIR ... GICD_CPENDSGIR + 0xf:
+    case GICD_SPENDSGIR ... GICD_SPENDSGIR + 0xf:
+        /* RAZ/WI since affinity routing is always enabled */
+        *data = 0;
+        return MEMTX_OK;
+    case GICD_IROUTER ... GICD_IROUTER + 0x1fdf:
+    {
+        uint64_t r;
+        int irq = (offset - GICD_IROUTER) / 8;
+
+        r = gicd_read_irouter(s, attrs, irq);
+        if (offset & 7) {
+            *data = r >> 32;
+        } else {
+            *data = (uint32_t)r;
+        }
+        return MEMTX_OK;
+    }
+    case GICD_IDREGS ... GICD_IDREGS + 0x1f:
+        /* ID registers */
+        *data = gicv3_idreg(offset - GICD_IDREGS);
+        return MEMTX_OK;
+    case GICD_SGIR:
+        /* WO registers, return unknown value */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest read from WO register at offset "
+                      TARGET_FMT_plx "\n", __func__, offset);
+        *data = 0;
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicd_writel(GICv3State *s, hwaddr offset,
+                               uint64_t value, MemTxAttrs attrs)
+{
+    /* Almost all GICv3 distributor registers are 32-bit. Note that
+     * RO registers must ignore writes, not abort.
+     */
+
+    switch (offset) {
+    case GICD_CTLR:
+    {
+        uint32_t mask;
+        /* GICv3 5.3.20 */
+        if (s->gicd_ctlr & GICD_CTLR_DS) {
+            /* With only one security state, E1NWF is RAZ/WI, DS is RAO/WI,
+             * ARE is RAO/WI (affinity routing always on), and only
+             * bits 0 and 1 (group enables) are writable.
+             */
+            mask = GICD_CTLR_EN_GRP0 | GICD_CTLR_EN_GRP1NS;
+        } else {
+            if (attrs.secure) {
+                /* for secure access:
+                 * ARE_NS and ARE_S are RAO/WI (affinity routing always on)
+                 * E1NWF is RAZ/WI (we don't support enable-1-of-n-wakeup)
+                 *
+                 * We can only modify bits[2:0] (the group enables).
+                 */
+                mask = GICD_CTLR_DS | GICD_CTLR_EN_GRP0 | GICD_CTLR_EN_GRP1_ALL;
+            } else {
+                /* For non secure access ARE_NS is RAO/WI and EnableGrp1
+                 * is RES0. The only writable bit is [1] (EnableGrp1A), which
+                 * is an alias of the Secure bit [1].
+                 */
+                mask = GICD_CTLR_EN_GRP1NS;
+            }
+        }
+        s->gicd_ctlr = (s->gicd_ctlr & ~mask) | (value & mask);
+        if (value & mask & GICD_CTLR_DS) {
+            /* We just set DS, so the ARE_NS and EnG1S bits are now RES0.
+             * Note that this is a one-way transition because if DS is set
+             * then it's not writeable, so it can only go back to 0 with a
+             * hardware reset.
+             */
+            s->gicd_ctlr &= ~(GICD_CTLR_EN_GRP1S | GICD_CTLR_ARE_NS);
+        }
+        gicv3_full_update(s);
+        return MEMTX_OK;
+    }
+    case GICD_STATUSR:
+        /* RAZ/WI for our implementation */
+        return MEMTX_OK;
+    case GICD_IGROUPR ... GICD_IGROUPR + 0x7f:
+    {
+        int irq;
+
+        if (!attrs.secure && !(s->gicd_ctlr & GICD_CTLR_DS)) {
+            return MEMTX_OK;
+        }
+        /* RAZ/WI for SGIs, PPIs, unimplemented irqs */
+        irq = (offset - GICD_IGROUPR) * 8;
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            return MEMTX_OK;
+        }
+        *gic_bmp_ptr32(s->group, irq) = value;
+        gicv3_update(s, irq, 32);
+        return MEMTX_OK;
+    }
+    case GICD_ISENABLER ... GICD_ISENABLER + 0x7f:
+        gicd_write_set_bitmap_reg(s, attrs, s->enabled, NULL,
+                                  offset - GICD_ISENABLER, value);
+        return MEMTX_OK;
+    case GICD_ICENABLER ... GICD_ICENABLER + 0x7f:
+        gicd_write_clear_bitmap_reg(s, attrs, s->enabled, NULL,
+                                    offset - GICD_ICENABLER, value);
+        return MEMTX_OK;
+    case GICD_ISPENDR ... GICD_ISPENDR + 0x7f:
+        gicd_write_set_bitmap_reg(s, attrs, s->pending, mask_nsacr_ge1,
+                                  offset - GICD_ISPENDR, value);
+        return MEMTX_OK;
+    case GICD_ICPENDR ... GICD_ICPENDR + 0x7f:
+        gicd_write_clear_bitmap_reg(s, attrs, s->pending, mask_nsacr_ge2,
+                                    offset - GICD_ICPENDR, value);
+        return MEMTX_OK;
+    case GICD_ISACTIVER ... GICD_ISACTIVER + 0x7f:
+        gicd_write_set_bitmap_reg(s, attrs, s->active, NULL,
+                                  offset - GICD_ISACTIVER, value);
+        return MEMTX_OK;
+    case GICD_ICACTIVER ... GICD_ICACTIVER + 0x7f:
+        gicd_write_clear_bitmap_reg(s, attrs, s->active, NULL,
+                                    offset - GICD_ICACTIVER, value);
+        return MEMTX_OK;
+    case GICD_IPRIORITYR ... GICD_IPRIORITYR + 0x3ff:
+    {
+        int i, irq = offset - GICD_IPRIORITYR;
+
+        if (irq < GIC_INTERNAL || irq + 3 >= s->num_irq) {
+            return MEMTX_OK;
+        }
+
+        for (i = irq; i < irq + 4; i++, value >>= 8) {
+            gicd_write_ipriorityr(s, attrs, i, value);
+        }
+        gicv3_update(s, irq, 4);
+        return MEMTX_OK;
+    }
+    case GICD_ITARGETSR ... GICD_ITARGETSR + 0x3ff:
+        /* RAZ/WI since affinity routing is always enabled */
+        return MEMTX_OK;
+    case GICD_ICFGR ... GICD_ICFGR + 0xff:
+    {
+        /* Here only the odd bits are used; even bits are RES0 */
+        int irq = (offset - GICD_ICFGR) * 4;
+        uint32_t mask, oldval;
+
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            return MEMTX_OK;
+        }
+
+        /* Since our edge_trigger bitmap is one bit per irq, our input
+         * 32-bits will compress down into 16 bits which we need
+         * to write into the bitmap.
+         */
+        value = half_unshuffle32(value >> 1);
+        mask = mask_group_and_nsacr(s, attrs, NULL, irq & ~0x1f);
+        if (irq & 0x1f) {
+            value <<= 16;
+            mask &= 0xffff0000U;
+        } else {
+            mask &= 0xffff;
+        }
+        oldval = *gic_bmp_ptr32(s->edge_trigger, (irq & ~0x1f));
+        value = (oldval & ~mask) | (value & mask);
+        *gic_bmp_ptr32(s->edge_trigger, irq & ~0x1f) = value;
+        return MEMTX_OK;
+    }
+    case GICD_IGRPMODR ... GICD_IGRPMODR + 0xff:
+    {
+        int irq;
+
+        if ((s->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            return MEMTX_OK;
+        }
+        /* RAZ/WI for SGIs, PPIs, unimplemented irqs */
+        irq = (offset - GICD_IGRPMODR) * 8;
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            return MEMTX_OK;
+        }
+        *gic_bmp_ptr32(s->grpmod, irq) = value;
+        gicv3_update(s, irq, 32);
+        return MEMTX_OK;
+    }
+    case GICD_NSACR ... GICD_NSACR + 0xff:
+    {
+        /* Two bits per interrupt */
+        int irq = (offset - GICD_NSACR) * 4;
+
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            return MEMTX_OK;
+        }
+
+        if ((s->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            return MEMTX_OK;
+        }
+
+        s->gicd_nsacr[irq / 16] = value;
+        /* No update required as this only affects access permission checks */
+        return MEMTX_OK;
+    }
+    case GICD_SGIR:
+        /* RES0 if affinity routing is enabled */
+        return MEMTX_OK;
+    case GICD_CPENDSGIR ... GICD_CPENDSGIR + 0xf:
+    case GICD_SPENDSGIR ... GICD_SPENDSGIR + 0xf:
+        /* RAZ/WI since affinity routing is always enabled */
+        return MEMTX_OK;
+    case GICD_IROUTER ... GICD_IROUTER + 0x1fdf:
+    {
+        uint64_t r;
+        int irq = (offset - GICD_IROUTER) / 8;
+
+        if (irq < GIC_INTERNAL || irq >= s->num_irq) {
+            return MEMTX_OK;
+        }
+
+        /* Write half of the 64-bit register */
+        r = gicd_read_irouter(s, attrs, irq);
+        r = deposit64(r, (offset & 7) ? 32 : 0, 32, value);
+        gicd_write_irouter(s, attrs, irq, r);
+        return MEMTX_OK;
+    }
+    case GICD_IDREGS ... GICD_IDREGS + 0x1f:
+    case GICD_TYPER:
+    case GICD_IIDR:
+        /* RO registers, ignore the write */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest write to RO register at offset "
+                      TARGET_FMT_plx "\n", __func__, offset);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicd_writell(GICv3State *s, hwaddr offset,
+                                uint64_t value, MemTxAttrs attrs)
+{
+    /* Our only 64-bit registers are GICD_IROUTER<n> */
+    int irq;
+
+    switch (offset) {
+    case GICD_IROUTER ... GICD_IROUTER + 0x1fdf:
+        irq = (offset - GICD_IROUTER) / 8;
+        gicd_write_irouter(s, attrs, irq, value);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicd_readll(GICv3State *s, hwaddr offset,
+                               uint64_t *data, MemTxAttrs attrs)
+{
+    /* Our only 64-bit registers are GICD_IROUTER<n> */
+    int irq;
+
+    switch (offset) {
+    case GICD_IROUTER ... GICD_IROUTER + 0x1fdf:
+        irq = (offset - GICD_IROUTER) / 8;
+        *data = gicd_read_irouter(s, attrs, irq);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+MemTxResult gicv3_dist_read(void *opaque, hwaddr offset, uint64_t *data,
+                            unsigned size, MemTxAttrs attrs)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    MemTxResult r;
+
+    switch (size) {
+    case 1:
+        r = gicd_readb(s, offset, data, attrs);
+        break;
+    case 2:
+        r = gicd_readw(s, offset, data, attrs);
+        break;
+    case 4:
+        r = gicd_readl(s, offset, data, attrs);
+        break;
+    case 8:
+        r = gicd_readll(s, offset, data, attrs);
+        break;
+    default:
+        r = MEMTX_ERROR;
+        break;
+    }
+
+    if (r == MEMTX_ERROR) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest read at offset " TARGET_FMT_plx
+                      "size %u\n", __func__, offset, size);
+        trace_gicv3_dist_badread(offset, size, attrs.secure);
+    } else {
+        trace_gicv3_dist_read(offset, *data, size, attrs.secure);
+    }
+    return r;
+}
+
+MemTxResult gicv3_dist_write(void *opaque, hwaddr offset, uint64_t data,
+                             unsigned size, MemTxAttrs attrs)
+{
+    GICv3State *s = (GICv3State *)opaque;
+    MemTxResult r;
+
+    switch (size) {
+    case 1:
+        r = gicd_writeb(s, offset, data, attrs);
+        break;
+    case 2:
+        r = gicd_writew(s, offset, data, attrs);
+        break;
+    case 4:
+        r = gicd_writel(s, offset, data, attrs);
+        break;
+    case 8:
+        r = gicd_writell(s, offset, data, attrs);
+        break;
+    default:
+        r = MEMTX_ERROR;
+        break;
+    }
+
+    if (r == MEMTX_ERROR) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest write at offset " TARGET_FMT_plx
+                      "size %u\n", __func__, offset, size);
+        trace_gicv3_dist_badwrite(offset, data, size, attrs.secure);
+    } else {
+        trace_gicv3_dist_write(offset, data, size, attrs.secure);
+    }
+    return r;
+}
+
+void gicv3_dist_set_irq(GICv3State *s, int irq, int level)
+{
+    /* Update distributor state for a change in an external SPI input line */
+    if (level == gicv3_gicd_level_test(s, irq)) {
+        return;
+    }
+
+    trace_gicv3_dist_set_irq(irq, level);
+
+    gicv3_gicd_level_replace(s, irq, level);
+
+    if (level) {
+        /* 0->1 edges latch the pending bit for edge-triggered interrupts */
+        if (gicv3_gicd_edge_trigger_test(s, irq)) {
+            gicv3_gicd_pending_set(s, irq);
+        }
+    }
+
+    gicv3_update(s, irq, 1);
+}
diff --git a/hw/intc/arm_gicv3_kvm.c b/hw/intc/arm_gicv3_kvm.c
new file mode 100644
index 0000000..711fde3
--- /dev/null
+++ b/hw/intc/arm_gicv3_kvm.c
@@ -0,0 +1,159 @@
+/*
+ * ARM Generic Interrupt Controller using KVM in-kernel support
+ *
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Written by Pavel Fedin
+ * Based on vGICv2 code by Peter Maydell
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/intc/arm_gicv3_common.h"
+#include "hw/sysbus.h"
+#include "sysemu/kvm.h"
+#include "kvm_arm.h"
+#include "vgic_common.h"
+#include "migration/migration.h"
+
+#ifdef DEBUG_GICV3_KVM
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "kvm_gicv3: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+#define TYPE_KVM_ARM_GICV3 "kvm-arm-gicv3"
+#define KVM_ARM_GICV3(obj) \
+     OBJECT_CHECK(GICv3State, (obj), TYPE_KVM_ARM_GICV3)
+#define KVM_ARM_GICV3_CLASS(klass) \
+     OBJECT_CLASS_CHECK(KVMARMGICv3Class, (klass), TYPE_KVM_ARM_GICV3)
+#define KVM_ARM_GICV3_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(KVMARMGICv3Class, (obj), TYPE_KVM_ARM_GICV3)
+
+typedef struct KVMARMGICv3Class {
+    ARMGICv3CommonClass parent_class;
+    DeviceRealize parent_realize;
+    void (*parent_reset)(DeviceState *dev);
+} KVMARMGICv3Class;
+
+static void kvm_arm_gicv3_set_irq(void *opaque, int irq, int level)
+{
+    GICv3State *s = (GICv3State *)opaque;
+
+    kvm_arm_gic_set_irq(s->num_irq, irq, level);
+}
+
+static void kvm_arm_gicv3_put(GICv3State *s)
+{
+    /* TODO */
+    DPRINTF("Cannot put kernel gic state, no kernel interface\n");
+}
+
+static void kvm_arm_gicv3_get(GICv3State *s)
+{
+    /* TODO */
+    DPRINTF("Cannot get kernel gic state, no kernel interface\n");
+}
+
+static void kvm_arm_gicv3_reset(DeviceState *dev)
+{
+    GICv3State *s = ARM_GICV3_COMMON(dev);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
+
+    DPRINTF("Reset\n");
+
+    kgc->parent_reset(dev);
+    kvm_arm_gicv3_put(s);
+}
+
+static void kvm_arm_gicv3_realize(DeviceState *dev, Error **errp)
+{
+    GICv3State *s = KVM_ARM_GICV3(dev);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_GET_CLASS(s);
+    Error *local_err = NULL;
+
+    DPRINTF("kvm_arm_gicv3_realize\n");
+
+    kgc->parent_realize(dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (s->security_extn) {
+        error_setg(errp, "the in-kernel VGICv3 does not implement the "
+                   "security extensions");
+        return;
+    }
+
+    gicv3_init_irqs_and_mmio(s, kvm_arm_gicv3_set_irq, NULL);
+
+    /* Try to create the device via the device control API */
+    s->dev_fd = kvm_create_device(kvm_state, KVM_DEV_TYPE_ARM_VGIC_V3, false);
+    if (s->dev_fd < 0) {
+        error_setg_errno(errp, -s->dev_fd, "error creating in-kernel VGIC");
+        return;
+    }
+
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_NR_IRQS,
+                      0, &s->num_irq, true);
+
+    /* Tell the kernel to complete VGIC initialization now */
+    kvm_device_access(s->dev_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
+                      KVM_DEV_ARM_VGIC_CTRL_INIT, NULL, true);
+
+    kvm_arm_register_device(&s->iomem_dist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
+                            KVM_VGIC_V3_ADDR_TYPE_DIST, s->dev_fd);
+    kvm_arm_register_device(&s->iomem_redist, -1, KVM_DEV_ARM_VGIC_GRP_ADDR,
+                            KVM_VGIC_V3_ADDR_TYPE_REDIST, s->dev_fd);
+
+    /* Block migration of a KVM GICv3 device: the API for saving and restoring
+     * the state in the kernel is not yet finalised in the kernel or
+     * implemented in QEMU.
+     */
+    error_setg(&s->migration_blocker, "vGICv3 migration is not implemented");
+    migrate_add_blocker(s->migration_blocker);
+}
+
+static void kvm_arm_gicv3_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ARMGICv3CommonClass *agcc = ARM_GICV3_COMMON_CLASS(klass);
+    KVMARMGICv3Class *kgc = KVM_ARM_GICV3_CLASS(klass);
+
+    agcc->pre_save = kvm_arm_gicv3_get;
+    agcc->post_load = kvm_arm_gicv3_put;
+    kgc->parent_realize = dc->realize;
+    kgc->parent_reset = dc->reset;
+    dc->realize = kvm_arm_gicv3_realize;
+    dc->reset = kvm_arm_gicv3_reset;
+}
+
+static const TypeInfo kvm_arm_gicv3_info = {
+    .name = TYPE_KVM_ARM_GICV3,
+    .parent = TYPE_ARM_GICV3_COMMON,
+    .instance_size = sizeof(GICv3State),
+    .class_init = kvm_arm_gicv3_class_init,
+    .class_size = sizeof(KVMARMGICv3Class),
+};
+
+static void kvm_arm_gicv3_register_types(void)
+{
+    type_register_static(&kvm_arm_gicv3_info);
+}
+
+type_init(kvm_arm_gicv3_register_types)
diff --git a/hw/intc/arm_gicv3_redist.c b/hw/intc/arm_gicv3_redist.c
new file mode 100644
index 0000000..77e5cfa
--- /dev/null
+++ b/hw/intc/arm_gicv3_redist.c
@@ -0,0 +1,567 @@
+/*
+ * ARM GICv3 emulation: Redistributor
+ *
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2016 Linaro Limited.
+ * Written by Shlomo Pongratz, Peter Maydell
+ *
+ * This code is licensed under the GPL, version 2 or (at your option)
+ * any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "trace.h"
+#include "gicv3_internal.h"
+
+static uint32_t mask_group(GICv3CPUState *cs, MemTxAttrs attrs)
+{
+    /* Return a 32-bit mask which should be applied for this set of 32
+     * interrupts; each bit is 1 if access is permitted by the
+     * combination of attrs.secure and GICR_GROUPR. (GICR_NSACR does
+     * not affect config register accesses, unlike GICD_NSACR.)
+     */
+    if (!attrs.secure && !(cs->gic->gicd_ctlr & GICD_CTLR_DS)) {
+        /* bits for Group 0 or Secure Group 1 interrupts are RAZ/WI */
+        return cs->gicr_igroupr0;
+    }
+    return 0xFFFFFFFFU;
+}
+
+static int gicr_ns_access(GICv3CPUState *cs, int irq)
+{
+    /* Return the 2 bit NSACR.NS_access field for this SGI */
+    assert(irq < 16);
+    return extract32(cs->gicr_nsacr, irq * 2, 2);
+}
+
+static void gicr_write_set_bitmap_reg(GICv3CPUState *cs, MemTxAttrs attrs,
+                                      uint32_t *reg, uint32_t val)
+{
+    /* Helper routine to implement writing to a "set-bitmap" register */
+    val &= mask_group(cs, attrs);
+    *reg |= val;
+    gicv3_redist_update(cs);
+}
+
+static void gicr_write_clear_bitmap_reg(GICv3CPUState *cs, MemTxAttrs attrs,
+                                        uint32_t *reg, uint32_t val)
+{
+    /* Helper routine to implement writing to a "clear-bitmap" register */
+    val &= mask_group(cs, attrs);
+    *reg &= ~val;
+    gicv3_redist_update(cs);
+}
+
+static uint32_t gicr_read_bitmap_reg(GICv3CPUState *cs, MemTxAttrs attrs,
+                                     uint32_t reg)
+{
+    reg &= mask_group(cs, attrs);
+    return reg;
+}
+
+static uint8_t gicr_read_ipriorityr(GICv3CPUState *cs, MemTxAttrs attrs,
+                                    int irq)
+{
+    /* Read the value of GICR_IPRIORITYR<n> for the specified interrupt,
+     * honouring security state (these are RAZ/WI for Group 0 or Secure
+     * Group 1 interrupts).
+     */
+    uint32_t prio;
+
+    prio = cs->gicr_ipriorityr[irq];
+
+    if (!attrs.secure && !(cs->gic->gicd_ctlr & GICD_CTLR_DS)) {
+        if (!(cs->gicr_igroupr0 & (1U << irq))) {
+            /* Fields for Group 0 or Secure Group 1 interrupts are RAZ/WI */
+            return 0;
+        }
+        /* NS view of the interrupt priority */
+        prio = (prio << 1) & 0xff;
+    }
+    return prio;
+}
+
+static void gicr_write_ipriorityr(GICv3CPUState *cs, MemTxAttrs attrs, int irq,
+                                  uint8_t value)
+{
+    /* Write the value of GICD_IPRIORITYR<n> for the specified interrupt,
+     * honouring security state (these are RAZ/WI for Group 0 or Secure
+     * Group 1 interrupts).
+     */
+    if (!attrs.secure && !(cs->gic->gicd_ctlr & GICD_CTLR_DS)) {
+        if (!(cs->gicr_igroupr0 & (1U << irq))) {
+            /* Fields for Group 0 or Secure Group 1 interrupts are RAZ/WI */
+            return;
+        }
+        /* NS view of the interrupt priority */
+        value = 0x80 | (value >> 1);
+    }
+    cs->gicr_ipriorityr[irq] = value;
+}
+
+static MemTxResult gicr_readb(GICv3CPUState *cs, hwaddr offset,
+                              uint64_t *data, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case GICR_IPRIORITYR ... GICR_IPRIORITYR + 0x1f:
+        *data = gicr_read_ipriorityr(cs, attrs, offset - GICR_IPRIORITYR);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicr_writeb(GICv3CPUState *cs, hwaddr offset,
+                               uint64_t value, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case GICR_IPRIORITYR ... GICR_IPRIORITYR + 0x1f:
+        gicr_write_ipriorityr(cs, attrs, offset - GICR_IPRIORITYR, value);
+        gicv3_redist_update(cs);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicr_readl(GICv3CPUState *cs, hwaddr offset,
+                              uint64_t *data, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case GICR_CTLR:
+        *data = cs->gicr_ctlr;
+        return MEMTX_OK;
+    case GICR_IIDR:
+        *data = gicv3_iidr();
+        return MEMTX_OK;
+    case GICR_TYPER:
+        *data = extract64(cs->gicr_typer, 0, 32);
+        return MEMTX_OK;
+    case GICR_TYPER + 4:
+        *data = extract64(cs->gicr_typer, 32, 32);
+        return MEMTX_OK;
+    case GICR_STATUSR:
+        /* RAZ/WI for us (this is an optional register and our implementation
+         * does not track RO/WO/reserved violations to report them to the guest)
+         */
+        *data = 0;
+        return MEMTX_OK;
+    case GICR_WAKER:
+        *data = cs->gicr_waker;
+        return MEMTX_OK;
+    case GICR_PROPBASER:
+        *data = extract64(cs->gicr_propbaser, 0, 32);
+        return MEMTX_OK;
+    case GICR_PROPBASER + 4:
+        *data = extract64(cs->gicr_propbaser, 32, 32);
+        return MEMTX_OK;
+    case GICR_PENDBASER:
+        *data = extract64(cs->gicr_pendbaser, 0, 32);
+        return MEMTX_OK;
+    case GICR_PENDBASER + 4:
+        *data = extract64(cs->gicr_pendbaser, 32, 32);
+        return MEMTX_OK;
+    case GICR_IGROUPR0:
+        if (!attrs.secure && !(cs->gic->gicd_ctlr & GICD_CTLR_DS)) {
+            *data = 0;
+            return MEMTX_OK;
+        }
+        *data = cs->gicr_igroupr0;
+        return MEMTX_OK;
+    case GICR_ISENABLER0:
+    case GICR_ICENABLER0:
+        *data = gicr_read_bitmap_reg(cs, attrs, cs->gicr_ienabler0);
+        return MEMTX_OK;
+    case GICR_ISPENDR0:
+    case GICR_ICPENDR0:
+    {
+        /* The pending register reads as the logical OR of the pending
+         * latch and the input line level for level-triggered interrupts.
+         */
+        uint32_t val = cs->gicr_ipendr0 | (~cs->edge_trigger & cs->level);
+        *data = gicr_read_bitmap_reg(cs, attrs, val);
+        return MEMTX_OK;
+    }
+    case GICR_ISACTIVER0:
+    case GICR_ICACTIVER0:
+        *data = gicr_read_bitmap_reg(cs, attrs, cs->gicr_iactiver0);
+        return MEMTX_OK;
+    case GICR_IPRIORITYR ... GICR_IPRIORITYR + 0x1f:
+    {
+        int i, irq = offset - GICR_IPRIORITYR;
+        uint32_t value = 0;
+
+        for (i = irq + 3; i >= irq; i--, value <<= 8) {
+            value |= gicr_read_ipriorityr(cs, attrs, i);
+        }
+        *data = value;
+        return MEMTX_OK;
+    }
+    case GICR_ICFGR0:
+    case GICR_ICFGR1:
+    {
+        /* Our edge_trigger bitmap is one bit per irq; take the correct
+         * half of it, and spread it out into the odd bits.
+         */
+        uint32_t value;
+
+        value = cs->edge_trigger & mask_group(cs, attrs);
+        value = extract32(value, (offset == GICR_ICFGR1) ? 16 : 0, 16);
+        value = half_shuffle32(value) << 1;
+        *data = value;
+        return MEMTX_OK;
+    }
+    case GICR_IGRPMODR0:
+        if ((cs->gic->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            *data = 0;
+            return MEMTX_OK;
+        }
+        *data = cs->gicr_igrpmodr0;
+        return MEMTX_OK;
+    case GICR_NSACR:
+        if ((cs->gic->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            *data = 0;
+            return MEMTX_OK;
+        }
+        *data = cs->gicr_nsacr;
+        return MEMTX_OK;
+    case GICR_IDREGS ... GICR_IDREGS + 0x1f:
+        *data = gicv3_idreg(offset - GICR_IDREGS);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicr_writel(GICv3CPUState *cs, hwaddr offset,
+                               uint64_t value, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case GICR_CTLR:
+        /* For our implementation, GICR_TYPER.DPGS is 0 and so all
+         * the DPG bits are RAZ/WI. We don't do anything asynchronously,
+         * so UWP and RWP are RAZ/WI. And GICR_TYPER.LPIS is 0 (we don't
+         * implement LPIs) so Enable_LPIs is RES0. So there are no writable
+         * bits for us.
+         */
+        return MEMTX_OK;
+    case GICR_STATUSR:
+        /* RAZ/WI for our implementation */
+        return MEMTX_OK;
+    case GICR_WAKER:
+        /* Only the ProcessorSleep bit is writeable. When the guest sets
+         * it it requests that we transition the channel between the
+         * redistributor and the cpu interface to quiescent, and that
+         * we set the ChildrenAsleep bit once the inteface has reached the
+         * quiescent state.
+         * Setting the ProcessorSleep to 0 reverses the quiescing, and
+         * ChildrenAsleep is cleared once the transition is complete.
+         * Since our interface is not asynchronous, we complete these
+         * transitions instantaneously, so we set ChildrenAsleep to the
+         * same value as ProcessorSleep here.
+         */
+        value &= GICR_WAKER_ProcessorSleep;
+        if (value & GICR_WAKER_ProcessorSleep) {
+            value |= GICR_WAKER_ChildrenAsleep;
+        }
+        cs->gicr_waker = value;
+        return MEMTX_OK;
+    case GICR_PROPBASER:
+        cs->gicr_propbaser = deposit64(cs->gicr_propbaser, 0, 32, value);
+        return MEMTX_OK;
+    case GICR_PROPBASER + 4:
+        cs->gicr_propbaser = deposit64(cs->gicr_propbaser, 32, 32, value);
+        return MEMTX_OK;
+    case GICR_PENDBASER:
+        cs->gicr_pendbaser = deposit64(cs->gicr_pendbaser, 0, 32, value);
+        return MEMTX_OK;
+    case GICR_PENDBASER + 4:
+        cs->gicr_pendbaser = deposit64(cs->gicr_pendbaser, 32, 32, value);
+        return MEMTX_OK;
+    case GICR_IGROUPR0:
+        if (!attrs.secure && !(cs->gic->gicd_ctlr & GICD_CTLR_DS)) {
+            return MEMTX_OK;
+        }
+        cs->gicr_igroupr0 = value;
+        gicv3_redist_update(cs);
+        return MEMTX_OK;
+    case GICR_ISENABLER0:
+        gicr_write_set_bitmap_reg(cs, attrs, &cs->gicr_ienabler0, value);
+        return MEMTX_OK;
+    case GICR_ICENABLER0:
+        gicr_write_clear_bitmap_reg(cs, attrs, &cs->gicr_ienabler0, value);
+        return MEMTX_OK;
+    case GICR_ISPENDR0:
+        gicr_write_set_bitmap_reg(cs, attrs, &cs->gicr_ipendr0, value);
+        return MEMTX_OK;
+    case GICR_ICPENDR0:
+        gicr_write_clear_bitmap_reg(cs, attrs, &cs->gicr_ipendr0, value);
+        return MEMTX_OK;
+    case GICR_ISACTIVER0:
+        gicr_write_set_bitmap_reg(cs, attrs, &cs->gicr_iactiver0, value);
+        return MEMTX_OK;
+    case GICR_ICACTIVER0:
+        gicr_write_clear_bitmap_reg(cs, attrs, &cs->gicr_iactiver0, value);
+        return MEMTX_OK;
+    case GICR_IPRIORITYR ... GICR_IPRIORITYR + 0x1f:
+    {
+        int i, irq = offset - GICR_IPRIORITYR;
+
+        for (i = irq; i < irq + 4; i++, value >>= 8) {
+            gicr_write_ipriorityr(cs, attrs, i, value);
+        }
+        gicv3_redist_update(cs);
+        return MEMTX_OK;
+    }
+    case GICR_ICFGR0:
+        /* Register is all RAZ/WI or RAO/WI bits */
+        return MEMTX_OK;
+    case GICR_ICFGR1:
+    {
+        uint32_t mask;
+
+        /* Since our edge_trigger bitmap is one bit per irq, our input
+         * 32-bits will compress down into 16 bits which we need
+         * to write into the bitmap.
+         */
+        value = half_unshuffle32(value >> 1) << 16;
+        mask = mask_group(cs, attrs) & 0xffff0000U;
+
+        cs->edge_trigger &= ~mask;
+        cs->edge_trigger |= (value & mask);
+
+        gicv3_redist_update(cs);
+        return MEMTX_OK;
+    }
+    case GICR_IGRPMODR0:
+        if ((cs->gic->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            return MEMTX_OK;
+        }
+        cs->gicr_igrpmodr0 = value;
+        gicv3_redist_update(cs);
+        return MEMTX_OK;
+    case GICR_NSACR:
+        if ((cs->gic->gicd_ctlr & GICD_CTLR_DS) || !attrs.secure) {
+            /* RAZ/WI if security disabled, or if
+             * security enabled and this is an NS access
+             */
+            return MEMTX_OK;
+        }
+        cs->gicr_nsacr = value;
+        /* no update required as this only affects access permission checks */
+        return MEMTX_OK;
+    case GICR_IIDR:
+    case GICR_TYPER:
+    case GICR_IDREGS ... GICR_IDREGS + 0x1f:
+        /* RO registers, ignore the write */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest write to RO register at offset "
+                      TARGET_FMT_plx "\n", __func__, offset);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicr_readll(GICv3CPUState *cs, hwaddr offset,
+                               uint64_t *data, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case GICR_TYPER:
+        *data = cs->gicr_typer;
+        return MEMTX_OK;
+    case GICR_PROPBASER:
+        *data = cs->gicr_propbaser;
+        return MEMTX_OK;
+    case GICR_PENDBASER:
+        *data = cs->gicr_pendbaser;
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+static MemTxResult gicr_writell(GICv3CPUState *cs, hwaddr offset,
+                                uint64_t value, MemTxAttrs attrs)
+{
+    switch (offset) {
+    case GICR_PROPBASER:
+        cs->gicr_propbaser = value;
+        return MEMTX_OK;
+    case GICR_PENDBASER:
+        cs->gicr_pendbaser = value;
+        return MEMTX_OK;
+    case GICR_TYPER:
+        /* RO register, ignore the write */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest write to RO register at offset "
+                      TARGET_FMT_plx "\n", __func__, offset);
+        return MEMTX_OK;
+    default:
+        return MEMTX_ERROR;
+    }
+}
+
+MemTxResult gicv3_redist_read(void *opaque, hwaddr offset, uint64_t *data,
+                              unsigned size, MemTxAttrs attrs)
+{
+    GICv3State *s = opaque;
+    GICv3CPUState *cs;
+    MemTxResult r;
+    int cpuidx;
+
+    assert((offset & (size - 1)) == 0);
+
+    /* This region covers all the redistributor pages; there are
+     * (for GICv3) two 64K pages per CPU. At the moment they are
+     * all contiguous (ie in this one region), though we might later
+     * want to allow splitting of redistributor pages into several
+     * blocks so we can support more CPUs.
+     */
+    cpuidx = offset / 0x20000;
+    offset %= 0x20000;
+    assert(cpuidx < s->num_cpu);
+
+    cs = &s->cpu[cpuidx];
+
+    switch (size) {
+    case 1:
+        r = gicr_readb(cs, offset, data, attrs);
+        break;
+    case 4:
+        r = gicr_readl(cs, offset, data, attrs);
+        break;
+    case 8:
+        r = gicr_readll(cs, offset, data, attrs);
+        break;
+    default:
+        r = MEMTX_ERROR;
+        break;
+    }
+
+    if (r == MEMTX_ERROR) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest read at offset " TARGET_FMT_plx
+                      "size %u\n", __func__, offset, size);
+        trace_gicv3_redist_badread(gicv3_redist_affid(cs), offset,
+                                   size, attrs.secure);
+    } else {
+        trace_gicv3_redist_read(gicv3_redist_affid(cs), offset, *data,
+                                size, attrs.secure);
+    }
+    return r;
+}
+
+MemTxResult gicv3_redist_write(void *opaque, hwaddr offset, uint64_t data,
+                               unsigned size, MemTxAttrs attrs)
+{
+    GICv3State *s = opaque;
+    GICv3CPUState *cs;
+    MemTxResult r;
+    int cpuidx;
+
+    assert((offset & (size - 1)) == 0);
+
+    /* This region covers all the redistributor pages; there are
+     * (for GICv3) two 64K pages per CPU. At the moment they are
+     * all contiguous (ie in this one region), though we might later
+     * want to allow splitting of redistributor pages into several
+     * blocks so we can support more CPUs.
+     */
+    cpuidx = offset / 0x20000;
+    offset %= 0x20000;
+    assert(cpuidx < s->num_cpu);
+
+    cs = &s->cpu[cpuidx];
+
+    switch (size) {
+    case 1:
+        r = gicr_writeb(cs, offset, data, attrs);
+        break;
+    case 4:
+        r = gicr_writel(cs, offset, data, attrs);
+        break;
+    case 8:
+        r = gicr_writell(cs, offset, data, attrs);
+        break;
+    default:
+        r = MEMTX_ERROR;
+        break;
+    }
+
+    if (r == MEMTX_ERROR) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: invalid guest write at offset " TARGET_FMT_plx
+                      "size %u\n", __func__, offset, size);
+        trace_gicv3_redist_badwrite(gicv3_redist_affid(cs), offset, data,
+                                    size, attrs.secure);
+    } else {
+        trace_gicv3_redist_write(gicv3_redist_affid(cs), offset, data,
+                                 size, attrs.secure);
+    }
+    return r;
+}
+
+void gicv3_redist_set_irq(GICv3CPUState *cs, int irq, int level)
+{
+    /* Update redistributor state for a change in an external PPI input line */
+    if (level == extract32(cs->level, irq, 1)) {
+        return;
+    }
+
+    trace_gicv3_redist_set_irq(gicv3_redist_affid(cs), irq, level);
+
+    cs->level = deposit32(cs->level, irq, 1, level);
+
+    if (level) {
+        /* 0->1 edges latch the pending bit for edge-triggered interrupts */
+        if (extract32(cs->edge_trigger, irq, 1)) {
+            cs->gicr_ipendr0 = deposit32(cs->gicr_ipendr0, irq, 1, 1);
+        }
+    }
+
+    gicv3_redist_update(cs);
+}
+
+void gicv3_redist_send_sgi(GICv3CPUState *cs, int grp, int irq, bool ns)
+{
+    /* Update redistributor state for a generated SGI */
+    int irqgrp = gicv3_irq_group(cs->gic, cs, irq);
+
+    /* If we are asked for a Secure Group 1 SGI and it's actually
+     * configured as Secure Group 0 this is OK (subject to the usual
+     * NSACR checks).
+     */
+    if (grp == GICV3_G1 && irqgrp == GICV3_G0) {
+        grp = GICV3_G0;
+    }
+
+    if (grp != irqgrp) {
+        return;
+    }
+
+    if (ns && !(cs->gic->gicd_ctlr & GICD_CTLR_DS)) {
+        /* If security is enabled we must test the NSACR bits */
+        int nsaccess = gicr_ns_access(cs, irq);
+
+        if ((irqgrp == GICV3_G0 && nsaccess < 1) ||
+            (irqgrp == GICV3_G1 && nsaccess < 2)) {
+            return;
+        }
+    }
+
+    /* OK, we can accept the SGI */
+    trace_gicv3_redist_send_sgi(gicv3_redist_affid(cs), irq);
+    cs->gicr_ipendr0 = deposit32(cs->gicr_ipendr0, irq, 1, 1);
+    gicv3_redist_update(cs);
+}
diff --git a/hw/intc/armv7m_nvic.c b/hw/intc/armv7m_nvic.c
index d0543d4..06d8db6 100644
--- a/hw/intc/armv7m_nvic.c
+++ b/hw/intc/armv7m_nvic.c
@@ -10,11 +10,16 @@
  * NVIC.  Much of that is also implemented here.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "hw/arm/arm.h"
 #include "exec/address-spaces.h"
 #include "gic_internal.h"
+#include "qemu/log.h"
 
 typedef struct {
     GICState gic;
@@ -28,6 +33,7 @@
     MemoryRegion gic_iomem_alias;
     MemoryRegion container;
     uint32_t num_irq;
+    qemu_irq sysresetreq;
 } nvic_state;
 
 #define TYPE_NVIC "armv7m_nvic"
@@ -77,6 +83,15 @@
 
 static void systick_reload(nvic_state *s, int reset)
 {
+    /* The Cortex-M3 Devices Generic User Guide says that "When the
+     * ENABLE bit is set to 1, the counter loads the RELOAD value from the
+     * SYST RVR register and then counts down". So, we need to check the
+     * ENABLE bit before reloading the value.
+     */
+    if ((s->systick.control & SYSTICK_ENABLE) == 0) {
+        return;
+    }
+
     if (reset)
         s->systick.tick = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     s->systick.tick += (s->systick.reload + 1) * systick_scale(s);
@@ -122,7 +137,7 @@
     nvic_state *s = (nvic_state *)opaque;
     uint32_t irq;
 
-    irq = gic_acknowledge_irq(&s->gic, 0);
+    irq = gic_acknowledge_irq(&s->gic, 0, MEMTXATTRS_UNSPECIFIED);
     if (irq == 1023)
         hw_error("Interrupt but no vector\n");
     if (irq >= 32)
@@ -135,7 +150,7 @@
     nvic_state *s = (nvic_state *)opaque;
     if (irq >= 16)
         irq += 16;
-    gic_complete_irq(&s->gic, 0, irq);
+    gic_complete_irq(&s->gic, 0, irq, MEMTXATTRS_UNSPECIFIED);
 }
 
 static uint32_t nvic_readl(nvic_state *s, uint32_t offset)
@@ -172,30 +187,29 @@
     case 0x1c: /* SysTick Calibration Value.  */
         return 10000;
     case 0xd00: /* CPUID Base.  */
-        cpu = ARM_CPU(current_cpu);
+        cpu = ARM_CPU(qemu_get_cpu(0));
         return cpu->midr;
     case 0xd04: /* Interrupt Control State.  */
         /* VECTACTIVE */
-        val = s->gic.running_irq[0];
+        cpu = ARM_CPU(qemu_get_cpu(0));
+        val = cpu->env.v7m.exception;
         if (val == 1023) {
             val = 0;
         } else if (val >= 32) {
             val -= 16;
         }
-        /* RETTOBASE */
-        if (s->gic.running_irq[0] == 1023
-                || s->gic.last_active[s->gic.running_irq[0]][0] == 1023) {
-            val |= (1 << 11);
-        }
         /* VECTPENDING */
         if (s->gic.current_pending[0] != 1023)
             val |= (s->gic.current_pending[0] << 12);
-        /* ISRPENDING */
+        /* ISRPENDING and RETTOBASE */
         for (irq = 32; irq < s->num_irq; irq++) {
             if (s->gic.irq_state[irq].pending) {
                 val |= (1 << 22);
                 break;
             }
+            if (irq != cpu->env.v7m.exception && s->gic.irq_state[irq].active) {
+                val |= (1 << 11);
+            }
         }
         /* PENDSTSET */
         if (s->gic.irq_state[ARMV7M_EXCP_SYSTICK].pending)
@@ -208,7 +222,7 @@
             val |= (1 << 31);
         return val;
     case 0xd08: /* Vector Table Offset.  */
-        cpu = ARM_CPU(current_cpu);
+        cpu = ARM_CPU(qemu_get_cpu(0));
         return cpu->env.v7m.vecbase;
     case 0xd0c: /* Application Interrupt/Reset Control.  */
         return 0xfa050000;
@@ -335,15 +349,18 @@
         }
         break;
     case 0xd08: /* Vector Table Offset.  */
-        cpu = ARM_CPU(current_cpu);
+        cpu = ARM_CPU(qemu_get_cpu(0));
         cpu->env.v7m.vecbase = value & 0xffffff80;
         break;
     case 0xd0c: /* Application Interrupt/Reset Control.  */
         if ((value >> 16) == 0x05fa) {
+            if (value & 4) {
+                qemu_irq_pulse(s->sysresetreq);
+            }
             if (value & 2) {
                 qemu_log_mask(LOG_UNIMP, "VECTCLRACTIVE unimplemented\n");
             }
-            if (value & 5) {
+            if (value & 1) {
                 qemu_log_mask(LOG_UNIMP, "AIRCR system reset unimplemented\n");
             }
             if (value & 0x700) {
@@ -450,7 +467,7 @@
         VMSTATE_UINT32(systick.control, nvic_state),
         VMSTATE_UINT32(systick.reload, nvic_state),
         VMSTATE_INT64(systick.tick, nvic_state),
-        VMSTATE_TIMER(systick.timer, nvic_state),
+        VMSTATE_TIMER_PTR(systick.timer, nvic_state),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -465,10 +482,10 @@
      * as enabled by default, and with a priority mask which allows
      * all interrupts through.
      */
-    s->gic.cpu_enabled[0] = true;
+    s->gic.cpu_ctlr[0] = GICC_CTLR_EN_GRP0;
     s->gic.priority_mask[0] = 0x100;
     /* The NVIC as a whole is always enabled. */
-    s->gic.enabled = true;
+    s->gic.ctlr = 1;
     systick_reset(s);
 }
 
@@ -527,11 +544,14 @@
      * value in the GICState struct.
      */
     GICState *s = ARM_GIC_COMMON(obj);
+    DeviceState *dev = DEVICE(obj);
+    nvic_state *nvic = NVIC(obj);
     /* The ARM v7m may have anything from 0 to 496 external interrupt
      * IRQ lines. We default to 64. Other boards may differ and should
      * set the num-irq property appropriately.
      */
     s->num_irq = 64;
+    qdev_init_gpio_out_named(dev, &nvic->sysresetreq, "SYSRESETREQ", 1);
 }
 
 static void armv7m_nvic_class_init(ObjectClass *klass, void *data)
diff --git a/hw/intc/aspeed_vic.c b/hw/intc/aspeed_vic.c
new file mode 100644
index 0000000..2370e74
--- /dev/null
+++ b/hw/intc/aspeed_vic.c
@@ -0,0 +1,339 @@
+/*
+ * ASPEED Interrupt Controller (New)
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2015, 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+/* The hardware exposes two register sets, a legacy set and a 'new' set. The
+ * model implements the 'new' register set, and logs warnings on accesses to
+ * the legacy IO space.
+ *
+ * The hardware uses 32bit registers to manage 51 IRQs, with low and high
+ * registers for each conceptual register. The device model's implementation
+ * uses 64bit data types to store both low and high register values (in the one
+ * member), but must cope with access offset values in multiples of 4 passed to
+ * the callbacks. As such the read() and write() implementations process the
+ * provided offset to understand whether the access is requesting the lower or
+ * upper 32 bits of the 64bit member.
+ *
+ * Additionally, the "Interrupt Enable", "Edge Status" and "Software Interrupt"
+ * fields have separate "enable"/"status" and "clear" registers, where set bits
+ * are written to one or the other to change state (avoiding a
+ * read-modify-write sequence).
+ */
+
+#include "qemu/osdep.h"
+#include "hw/intc/aspeed_vic.h"
+#include "qemu/bitops.h"
+#include "qemu/log.h"
+#include "trace.h"
+
+#define AVIC_NEW_BASE_OFFSET 0x80
+
+#define AVIC_L_MASK 0xFFFFFFFFU
+#define AVIC_H_MASK 0x0007FFFFU
+#define AVIC_EVENT_W_MASK (0x78000ULL << 32)
+
+static void aspeed_vic_update(AspeedVICState *s)
+{
+    uint64_t new = (s->raw & s->enable);
+    uint64_t flags;
+
+    flags = new & s->select;
+    trace_aspeed_vic_update_fiq(!!flags);
+    qemu_set_irq(s->fiq, !!flags);
+
+    flags = new & ~s->select;
+    trace_aspeed_vic_update_irq(!!flags);
+    qemu_set_irq(s->irq, !!flags);
+}
+
+static void aspeed_vic_set_irq(void *opaque, int irq, int level)
+{
+    uint64_t irq_mask;
+    bool raise;
+    AspeedVICState *s = (AspeedVICState *)opaque;
+
+    if (irq > ASPEED_VIC_NR_IRQS) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Invalid interrupt number: %d\n",
+                      __func__, irq);
+        return;
+    }
+
+    trace_aspeed_vic_set_irq(irq, level);
+
+    irq_mask = BIT(irq);
+    if (s->sense & irq_mask) {
+        /* level-triggered */
+        if (s->event & irq_mask) {
+            /* high-sensitive */
+            raise = level;
+        } else {
+            /* low-sensitive */
+            raise = !level;
+        }
+        s->raw = deposit64(s->raw, irq, 1, raise);
+    } else {
+        uint64_t old_level = s->level & irq_mask;
+
+        /* edge-triggered */
+        if (s->dual_edge & irq_mask) {
+            raise = (!!old_level) != (!!level);
+        } else {
+            if (s->event & irq_mask) {
+                /* rising-sensitive */
+                raise = !old_level && level;
+            } else {
+                /* falling-sensitive */
+                raise = old_level && !level;
+            }
+        }
+        if (raise) {
+            s->raw = deposit64(s->raw, irq, 1, raise);
+        }
+    }
+    s->level = deposit64(s->level, irq, 1, level);
+    aspeed_vic_update(s);
+}
+
+static uint64_t aspeed_vic_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint64_t val;
+    const bool high = !!(offset & 0x4);
+    hwaddr n_offset = (offset & ~0x4);
+    AspeedVICState *s = (AspeedVICState *)opaque;
+
+    if (offset < AVIC_NEW_BASE_OFFSET) {
+        qemu_log_mask(LOG_UNIMP, "%s: Ignoring read from legacy registers "
+                      "at 0x%" HWADDR_PRIx "[%u]\n", __func__, offset, size);
+        return 0;
+    }
+
+    n_offset -= AVIC_NEW_BASE_OFFSET;
+
+    switch (n_offset) {
+    case 0x0: /* IRQ Status */
+        val = s->raw & ~s->select & s->enable;
+        break;
+    case 0x08: /* FIQ Status */
+        val = s->raw & s->select & s->enable;
+        break;
+    case 0x10: /* Raw Interrupt Status */
+        val = s->raw;
+        break;
+    case 0x18: /* Interrupt Selection */
+        val = s->select;
+        break;
+    case 0x20: /* Interrupt Enable */
+        val = s->enable;
+        break;
+    case 0x30: /* Software Interrupt */
+        val = s->trigger;
+        break;
+    case 0x40: /* Interrupt Sensitivity */
+        val = s->sense;
+        break;
+    case 0x48: /* Interrupt Both Edge Trigger Control */
+        val = s->dual_edge;
+        break;
+    case 0x50: /* Interrupt Event */
+        val = s->event;
+        break;
+    case 0x60: /* Edge Triggered Interrupt Status */
+        val = s->raw & ~s->sense;
+        break;
+        /* Illegal */
+    case 0x28: /* Interrupt Enable Clear */
+    case 0x38: /* Software Interrupt Clear */
+    case 0x58: /* Edge Triggered Interrupt Clear */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Read of write-only register with offset 0x%"
+                      HWADDR_PRIx "\n", __func__, offset);
+        val = 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad register at offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        val = 0;
+        break;
+    }
+    if (high) {
+        val = extract64(val, 32, 19);
+    }
+    trace_aspeed_vic_read(offset, size, val);
+    return val;
+}
+
+static void aspeed_vic_write(void *opaque, hwaddr offset, uint64_t data,
+                             unsigned size)
+{
+    const bool high = !!(offset & 0x4);
+    hwaddr n_offset = (offset & ~0x4);
+    AspeedVICState *s = (AspeedVICState *)opaque;
+
+    if (offset < AVIC_NEW_BASE_OFFSET) {
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Ignoring write to legacy registers at 0x%"
+                      HWADDR_PRIx "[%u] <- 0x%" PRIx64 "\n", __func__, offset,
+                      size, data);
+        return;
+    }
+
+    n_offset -= AVIC_NEW_BASE_OFFSET;
+    trace_aspeed_vic_write(offset, size, data);
+
+    /* Given we have members using separate enable/clear registers, deposit64()
+     * isn't quite the tool for the job. Instead, relocate the incoming bits to
+     * the required bit offset based on the provided access address
+     */
+    if (high) {
+        data &= AVIC_H_MASK;
+        data <<= 32;
+    } else {
+        data &= AVIC_L_MASK;
+    }
+
+    switch (n_offset) {
+    case 0x18: /* Interrupt Selection */
+        /* Register has deposit64() semantics - overwrite requested 32 bits */
+        if (high) {
+            s->select &= AVIC_L_MASK;
+        } else {
+            s->select &= ((uint64_t) AVIC_H_MASK) << 32;
+        }
+        s->select |= data;
+        break;
+    case 0x20: /* Interrupt Enable */
+        s->enable |= data;
+        break;
+    case 0x28: /* Interrupt Enable Clear */
+        s->enable &= ~data;
+        break;
+    case 0x30: /* Software Interrupt */
+        qemu_log_mask(LOG_UNIMP, "%s: Software interrupts unavailable. "
+                      "IRQs requested: 0x%016" PRIx64 "\n", __func__, data);
+        break;
+    case 0x38: /* Software Interrupt Clear */
+        qemu_log_mask(LOG_UNIMP, "%s: Software interrupts unavailable. "
+                      "IRQs to be cleared: 0x%016" PRIx64 "\n", __func__, data);
+        break;
+    case 0x50: /* Interrupt Event */
+        /* Register has deposit64() semantics - overwrite the top four valid
+         * IRQ bits, as only the top four IRQs (GPIOs) can change their event
+         * type */
+        if (high) {
+            s->event &= ~AVIC_EVENT_W_MASK;
+            s->event |= (data & AVIC_EVENT_W_MASK);
+        } else {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "Ignoring invalid write to interrupt event register");
+        }
+        break;
+    case 0x58: /* Edge Triggered Interrupt Clear */
+        s->raw &= ~(data & ~s->sense);
+        break;
+    case 0x00: /* IRQ Status */
+    case 0x08: /* FIQ Status */
+    case 0x10: /* Raw Interrupt Status */
+    case 0x40: /* Interrupt Sensitivity */
+    case 0x48: /* Interrupt Both Edge Trigger Control */
+    case 0x60: /* Edge Triggered Interrupt Status */
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Write of read-only register with offset 0x%"
+                      HWADDR_PRIx "\n", __func__, offset);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad register at offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        break;
+    }
+    aspeed_vic_update(s);
+}
+
+static const MemoryRegionOps aspeed_vic_ops = {
+    .read = aspeed_vic_read,
+    .write = aspeed_vic_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .valid.unaligned = false,
+};
+
+static void aspeed_vic_reset(DeviceState *dev)
+{
+    AspeedVICState *s = ASPEED_VIC(dev);
+
+    s->level = 0;
+    s->raw = 0;
+    s->select = 0;
+    s->enable = 0;
+    s->trigger = 0;
+    s->sense = 0x1F07FFF8FFFFULL;
+    s->dual_edge = 0xF800070000ULL;
+    s->event = 0x5F07FFF8FFFFULL;
+}
+
+#define AVIC_IO_REGION_SIZE 0x20000
+
+static void aspeed_vic_realize(DeviceState *dev, Error **errp)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedVICState *s = ASPEED_VIC(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &aspeed_vic_ops, s,
+                          TYPE_ASPEED_VIC, AVIC_IO_REGION_SIZE);
+
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    qdev_init_gpio_in(dev, aspeed_vic_set_irq, ASPEED_VIC_NR_IRQS);
+    sysbus_init_irq(sbd, &s->irq);
+    sysbus_init_irq(sbd, &s->fiq);
+}
+
+static const VMStateDescription vmstate_aspeed_vic = {
+    .name = "aspeed.new-vic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(level, AspeedVICState),
+        VMSTATE_UINT64(raw, AspeedVICState),
+        VMSTATE_UINT64(select, AspeedVICState),
+        VMSTATE_UINT64(enable, AspeedVICState),
+        VMSTATE_UINT64(trigger, AspeedVICState),
+        VMSTATE_UINT64(sense, AspeedVICState),
+        VMSTATE_UINT64(dual_edge, AspeedVICState),
+        VMSTATE_UINT64(event, AspeedVICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void aspeed_vic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->realize = aspeed_vic_realize;
+    dc->reset = aspeed_vic_reset;
+    dc->desc = "ASPEED Interrupt Controller (New)";
+    dc->vmsd = &vmstate_aspeed_vic;
+}
+
+static const TypeInfo aspeed_vic_info = {
+    .name = TYPE_ASPEED_VIC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AspeedVICState),
+    .class_init = aspeed_vic_class_init,
+};
+
+static void aspeed_vic_register_types(void)
+{
+    type_register_static(&aspeed_vic_info);
+}
+
+type_init(aspeed_vic_register_types);
diff --git a/hw/intc/bcm2835_ic.c b/hw/intc/bcm2835_ic.c
new file mode 100644
index 0000000..00d2530
--- /dev/null
+++ b/hw/intc/bcm2835_ic.c
@@ -0,0 +1,238 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Refactoring for Pi2 Copyright (c) 2015, Microsoft. Written by Andrew Baumann.
+ * This code is licensed under the GNU GPLv2 and later.
+ * Heavily based on pl190.c, copyright terms below:
+ *
+ * Arm PrimeCell PL190 Vector Interrupt Controller
+ *
+ * Copyright (c) 2006 CodeSourcery.
+ * Written by Paul Brook
+ *
+ * This code is licensed under the GPL.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/intc/bcm2835_ic.h"
+#include "qemu/log.h"
+
+#define GPU_IRQS 64
+#define ARM_IRQS 8
+
+#define IRQ_PENDING_BASIC       0x00 /* IRQ basic pending */
+#define IRQ_PENDING_1           0x04 /* IRQ pending 1 */
+#define IRQ_PENDING_2           0x08 /* IRQ pending 2 */
+#define FIQ_CONTROL             0x0C /* FIQ register */
+#define IRQ_ENABLE_1            0x10 /* Interrupt enable register 1 */
+#define IRQ_ENABLE_2            0x14 /* Interrupt enable register 2 */
+#define IRQ_ENABLE_BASIC        0x18 /* Base interrupt enable register */
+#define IRQ_DISABLE_1           0x1C /* Interrupt disable register 1 */
+#define IRQ_DISABLE_2           0x20 /* Interrupt disable register 2 */
+#define IRQ_DISABLE_BASIC       0x24 /* Base interrupt disable register */
+
+/* Update interrupts.  */
+static void bcm2835_ic_update(BCM2835ICState *s)
+{
+    bool set = false;
+
+    if (s->fiq_enable) {
+        if (s->fiq_select >= GPU_IRQS) {
+            /* ARM IRQ */
+            set = extract32(s->arm_irq_level, s->fiq_select - GPU_IRQS, 1);
+        } else {
+            set = extract64(s->gpu_irq_level, s->fiq_select, 1);
+        }
+    }
+    qemu_set_irq(s->fiq, set);
+
+    set = (s->gpu_irq_level & s->gpu_irq_enable)
+        || (s->arm_irq_level & s->arm_irq_enable);
+    qemu_set_irq(s->irq, set);
+
+}
+
+static void bcm2835_ic_set_gpu_irq(void *opaque, int irq, int level)
+{
+    BCM2835ICState *s = opaque;
+
+    assert(irq >= 0 && irq < 64);
+    s->gpu_irq_level = deposit64(s->gpu_irq_level, irq, 1, level != 0);
+    bcm2835_ic_update(s);
+}
+
+static void bcm2835_ic_set_arm_irq(void *opaque, int irq, int level)
+{
+    BCM2835ICState *s = opaque;
+
+    assert(irq >= 0 && irq < 8);
+    s->arm_irq_level = deposit32(s->arm_irq_level, irq, 1, level != 0);
+    bcm2835_ic_update(s);
+}
+
+static const int irq_dups[] = { 7, 9, 10, 18, 19, 53, 54, 55, 56, 57, 62 };
+
+static uint64_t bcm2835_ic_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835ICState *s = opaque;
+    uint32_t res = 0;
+    uint64_t gpu_pending = s->gpu_irq_level & s->gpu_irq_enable;
+    int i;
+
+    switch (offset) {
+    case IRQ_PENDING_BASIC:
+        /* bits 0-7: ARM irqs */
+        res = s->arm_irq_level & s->arm_irq_enable;
+
+        /* bits 8 & 9: pending registers 1 & 2 */
+        res |= (((uint32_t)gpu_pending) != 0) << 8;
+        res |= ((gpu_pending >> 32) != 0) << 9;
+
+        /* bits 10-20: selected GPU IRQs */
+        for (i = 0; i < ARRAY_SIZE(irq_dups); i++) {
+            res |= extract64(gpu_pending, irq_dups[i], 1) << (i + 10);
+        }
+        break;
+    case IRQ_PENDING_1:
+        res = gpu_pending;
+        break;
+    case IRQ_PENDING_2:
+        res = gpu_pending >> 32;
+        break;
+    case FIQ_CONTROL:
+        res = (s->fiq_enable << 7) | s->fiq_select;
+        break;
+    case IRQ_ENABLE_1:
+        res = s->gpu_irq_enable;
+        break;
+    case IRQ_ENABLE_2:
+        res = s->gpu_irq_enable >> 32;
+        break;
+    case IRQ_ENABLE_BASIC:
+        res = s->arm_irq_enable;
+        break;
+    case IRQ_DISABLE_1:
+        res = ~s->gpu_irq_enable;
+        break;
+    case IRQ_DISABLE_2:
+        res = ~s->gpu_irq_enable >> 32;
+        break;
+    case IRQ_DISABLE_BASIC:
+        res = ~s->arm_irq_enable;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    return res;
+}
+
+static void bcm2835_ic_write(void *opaque, hwaddr offset, uint64_t val,
+                             unsigned size)
+{
+    BCM2835ICState *s = opaque;
+
+    switch (offset) {
+    case FIQ_CONTROL:
+        s->fiq_select = extract32(val, 0, 7);
+        s->fiq_enable = extract32(val, 7, 1);
+        break;
+    case IRQ_ENABLE_1:
+        s->gpu_irq_enable |= val;
+        break;
+    case IRQ_ENABLE_2:
+        s->gpu_irq_enable |= val << 32;
+        break;
+    case IRQ_ENABLE_BASIC:
+        s->arm_irq_enable |= val & 0xff;
+        break;
+    case IRQ_DISABLE_1:
+        s->gpu_irq_enable &= ~val;
+        break;
+    case IRQ_DISABLE_2:
+        s->gpu_irq_enable &= ~(val << 32);
+        break;
+    case IRQ_DISABLE_BASIC:
+        s->arm_irq_enable &= ~val & 0xff;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+    bcm2835_ic_update(s);
+}
+
+static const MemoryRegionOps bcm2835_ic_ops = {
+    .read = bcm2835_ic_read,
+    .write = bcm2835_ic_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static void bcm2835_ic_reset(DeviceState *d)
+{
+    BCM2835ICState *s = BCM2835_IC(d);
+
+    s->gpu_irq_enable = 0;
+    s->arm_irq_enable = 0;
+    s->fiq_enable = false;
+    s->fiq_select = 0;
+}
+
+static void bcm2835_ic_init(Object *obj)
+{
+    BCM2835ICState *s = BCM2835_IC(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2835_ic_ops, s, TYPE_BCM2835_IC,
+                          0x200);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+
+    qdev_init_gpio_in_named(DEVICE(s), bcm2835_ic_set_gpu_irq,
+                            BCM2835_IC_GPU_IRQ, GPU_IRQS);
+    qdev_init_gpio_in_named(DEVICE(s), bcm2835_ic_set_arm_irq,
+                            BCM2835_IC_ARM_IRQ, ARM_IRQS);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->irq);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->fiq);
+}
+
+static const VMStateDescription vmstate_bcm2835_ic = {
+    .name = TYPE_BCM2835_IC,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(gpu_irq_level, BCM2835ICState),
+        VMSTATE_UINT64(gpu_irq_enable, BCM2835ICState),
+        VMSTATE_UINT8(arm_irq_level, BCM2835ICState),
+        VMSTATE_UINT8(arm_irq_enable, BCM2835ICState),
+        VMSTATE_BOOL(fiq_enable, BCM2835ICState),
+        VMSTATE_UINT8(fiq_select, BCM2835ICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_ic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = bcm2835_ic_reset;
+    dc->vmsd = &vmstate_bcm2835_ic;
+}
+
+static TypeInfo bcm2835_ic_info = {
+    .name          = TYPE_BCM2835_IC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835ICState),
+    .class_init    = bcm2835_ic_class_init,
+    .instance_init = bcm2835_ic_init,
+};
+
+static void bcm2835_ic_register_types(void)
+{
+    type_register_static(&bcm2835_ic_info);
+}
+
+type_init(bcm2835_ic_register_types)
diff --git a/hw/intc/bcm2836_control.c b/hw/intc/bcm2836_control.c
new file mode 100644
index 0000000..cfa5bc7
--- /dev/null
+++ b/hw/intc/bcm2836_control.c
@@ -0,0 +1,305 @@
+/*
+ * Rasperry Pi 2 emulation ARM control logic module.
+ * Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * Based on bcm2835_ic.c (Raspberry Pi emulation) (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ *
+ * At present, only implements interrupt routing, and mailboxes (i.e.,
+ * not local timer, PMU interrupt, or AXI counters).
+ *
+ * Ref:
+ * https://www.raspberrypi.org/documentation/hardware/raspberrypi/bcm2836/QA7_rev3.4.pdf
+ */
+
+#include "qemu/osdep.h"
+#include "hw/intc/bcm2836_control.h"
+#include "qemu/log.h"
+
+#define REG_GPU_ROUTE           0x0c
+#define REG_TIMERCONTROL        0x40
+#define REG_MBOXCONTROL         0x50
+#define REG_IRQSRC              0x60
+#define REG_FIQSRC              0x70
+#define REG_MBOX0_WR            0x80
+#define REG_MBOX0_RDCLR         0xc0
+#define REG_LIMIT              0x100
+
+#define IRQ_BIT(cntrl, num) (((cntrl) & (1 << (num))) != 0)
+#define FIQ_BIT(cntrl, num) (((cntrl) & (1 << ((num) + 4))) != 0)
+
+#define IRQ_CNTPSIRQ    0
+#define IRQ_CNTPNSIRQ   1
+#define IRQ_CNTHPIRQ    2
+#define IRQ_CNTVIRQ     3
+#define IRQ_MAILBOX0    4
+#define IRQ_MAILBOX1    5
+#define IRQ_MAILBOX2    6
+#define IRQ_MAILBOX3    7
+#define IRQ_GPU         8
+#define IRQ_PMU         9
+#define IRQ_AXI         10
+#define IRQ_TIMER       11
+#define IRQ_MAX         IRQ_TIMER
+
+static void deliver_local(BCM2836ControlState *s, uint8_t core, uint8_t irq,
+                          uint32_t controlreg, uint8_t controlidx)
+{
+    if (FIQ_BIT(controlreg, controlidx)) {
+        /* deliver a FIQ */
+        s->fiqsrc[core] |= (uint32_t)1 << irq;
+    } else if (IRQ_BIT(controlreg, controlidx)) {
+        /* deliver an IRQ */
+        s->irqsrc[core] |= (uint32_t)1 << irq;
+    } else {
+        /* the interrupt is masked */
+    }
+}
+
+/* Update interrupts.  */
+static void bcm2836_control_update(BCM2836ControlState *s)
+{
+    int i, j;
+
+    /* reset pending IRQs/FIQs */
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        s->irqsrc[i] = s->fiqsrc[i] = 0;
+    }
+
+    /* apply routing logic, update status regs */
+    if (s->gpu_irq) {
+        assert(s->route_gpu_irq < BCM2836_NCORES);
+        s->irqsrc[s->route_gpu_irq] |= (uint32_t)1 << IRQ_GPU;
+    }
+
+    if (s->gpu_fiq) {
+        assert(s->route_gpu_fiq < BCM2836_NCORES);
+        s->fiqsrc[s->route_gpu_fiq] |= (uint32_t)1 << IRQ_GPU;
+    }
+
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        /* handle local timer interrupts for this core */
+        if (s->timerirqs[i]) {
+            assert(s->timerirqs[i] < (1 << (IRQ_CNTVIRQ + 1))); /* sane mask? */
+            for (j = 0; j <= IRQ_CNTVIRQ; j++) {
+                if ((s->timerirqs[i] & (1 << j)) != 0) {
+                    /* local interrupt j is set */
+                    deliver_local(s, i, j, s->timercontrol[i], j);
+                }
+            }
+        }
+
+        /* handle mailboxes for this core */
+        for (j = 0; j < BCM2836_MBPERCORE; j++) {
+            if (s->mailboxes[i * BCM2836_MBPERCORE + j] != 0) {
+                /* mailbox j is set */
+                deliver_local(s, i, j + IRQ_MAILBOX0, s->mailboxcontrol[i], j);
+            }
+        }
+    }
+
+    /* call set_irq appropriately for each output */
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        qemu_set_irq(s->irq[i], s->irqsrc[i] != 0);
+        qemu_set_irq(s->fiq[i], s->fiqsrc[i] != 0);
+    }
+}
+
+static void bcm2836_control_set_local_irq(void *opaque, int core, int local_irq,
+                                          int level)
+{
+    BCM2836ControlState *s = opaque;
+
+    assert(core >= 0 && core < BCM2836_NCORES);
+    assert(local_irq >= 0 && local_irq <= IRQ_CNTVIRQ);
+
+    s->timerirqs[core] = deposit32(s->timerirqs[core], local_irq, 1, !!level);
+
+    bcm2836_control_update(s);
+}
+
+/* XXX: the following wrapper functions are a kludgy workaround,
+ * needed because I can't seem to pass useful information in the "irq"
+ * parameter when using named interrupts. Feel free to clean this up!
+ */
+
+static void bcm2836_control_set_local_irq0(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 0, level);
+}
+
+static void bcm2836_control_set_local_irq1(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 1, level);
+}
+
+static void bcm2836_control_set_local_irq2(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 2, level);
+}
+
+static void bcm2836_control_set_local_irq3(void *opaque, int core, int level)
+{
+    bcm2836_control_set_local_irq(opaque, core, 3, level);
+}
+
+static void bcm2836_control_set_gpu_irq(void *opaque, int irq, int level)
+{
+    BCM2836ControlState *s = opaque;
+
+    s->gpu_irq = level;
+
+    bcm2836_control_update(s);
+}
+
+static void bcm2836_control_set_gpu_fiq(void *opaque, int irq, int level)
+{
+    BCM2836ControlState *s = opaque;
+
+    s->gpu_fiq = level;
+
+    bcm2836_control_update(s);
+}
+
+static uint64_t bcm2836_control_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2836ControlState *s = opaque;
+
+    if (offset == REG_GPU_ROUTE) {
+        assert(s->route_gpu_fiq < BCM2836_NCORES
+               && s->route_gpu_irq < BCM2836_NCORES);
+        return ((uint32_t)s->route_gpu_fiq << 2) | s->route_gpu_irq;
+    } else if (offset >= REG_TIMERCONTROL && offset < REG_MBOXCONTROL) {
+        return s->timercontrol[(offset - REG_TIMERCONTROL) >> 2];
+    } else if (offset >= REG_MBOXCONTROL && offset < REG_IRQSRC) {
+        return s->mailboxcontrol[(offset - REG_MBOXCONTROL) >> 2];
+    } else if (offset >= REG_IRQSRC && offset < REG_FIQSRC) {
+        return s->irqsrc[(offset - REG_IRQSRC) >> 2];
+    } else if (offset >= REG_FIQSRC && offset < REG_MBOX0_WR) {
+        return s->fiqsrc[(offset - REG_FIQSRC) >> 2];
+    } else if (offset >= REG_MBOX0_RDCLR && offset < REG_LIMIT) {
+        return s->mailboxes[(offset - REG_MBOX0_RDCLR) >> 2];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+}
+
+static void bcm2836_control_write(void *opaque, hwaddr offset,
+                                  uint64_t val, unsigned size)
+{
+    BCM2836ControlState *s = opaque;
+
+    if (offset == REG_GPU_ROUTE) {
+        s->route_gpu_irq = val & 0x3;
+        s->route_gpu_fiq = (val >> 2) & 0x3;
+    } else if (offset >= REG_TIMERCONTROL && offset < REG_MBOXCONTROL) {
+        s->timercontrol[(offset - REG_TIMERCONTROL) >> 2] = val & 0xff;
+    } else if (offset >= REG_MBOXCONTROL && offset < REG_IRQSRC) {
+        s->mailboxcontrol[(offset - REG_MBOXCONTROL) >> 2] = val & 0xff;
+    } else if (offset >= REG_MBOX0_WR && offset < REG_MBOX0_RDCLR) {
+        s->mailboxes[(offset - REG_MBOX0_WR) >> 2] |= val;
+    } else if (offset >= REG_MBOX0_RDCLR && offset < REG_LIMIT) {
+        s->mailboxes[(offset - REG_MBOX0_RDCLR) >> 2] &= ~val;
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+
+    bcm2836_control_update(s);
+}
+
+static const MemoryRegionOps bcm2836_control_ops = {
+    .read = bcm2836_control_read,
+    .write = bcm2836_control_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static void bcm2836_control_reset(DeviceState *d)
+{
+    BCM2836ControlState *s = BCM2836_CONTROL(d);
+    int i;
+
+    s->route_gpu_irq = s->route_gpu_fiq = 0;
+
+    for (i = 0; i < BCM2836_NCORES; i++) {
+        s->timercontrol[i] = 0;
+        s->mailboxcontrol[i] = 0;
+    }
+
+    for (i = 0; i < BCM2836_NCORES * BCM2836_MBPERCORE; i++) {
+        s->mailboxes[i] = 0;
+    }
+}
+
+static void bcm2836_control_init(Object *obj)
+{
+    BCM2836ControlState *s = BCM2836_CONTROL(obj);
+    DeviceState *dev = DEVICE(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2836_control_ops, s,
+                          TYPE_BCM2836_CONTROL, REG_LIMIT);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+
+    /* inputs from each CPU core */
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq0, "cntpsirq",
+                            BCM2836_NCORES);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq1, "cntpnsirq",
+                            BCM2836_NCORES);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq2, "cnthpirq",
+                            BCM2836_NCORES);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_local_irq3, "cntvirq",
+                            BCM2836_NCORES);
+
+    /* IRQ and FIQ inputs from upstream bcm2835 controller */
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_gpu_irq, "gpu-irq", 1);
+    qdev_init_gpio_in_named(dev, bcm2836_control_set_gpu_fiq, "gpu-fiq", 1);
+
+    /* outputs to CPU cores */
+    qdev_init_gpio_out_named(dev, s->irq, "irq", BCM2836_NCORES);
+    qdev_init_gpio_out_named(dev, s->fiq, "fiq", BCM2836_NCORES);
+}
+
+static const VMStateDescription vmstate_bcm2836_control = {
+    .name = TYPE_BCM2836_CONTROL,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(mailboxes, BCM2836ControlState,
+                             BCM2836_NCORES * BCM2836_MBPERCORE),
+        VMSTATE_UINT8(route_gpu_irq, BCM2836ControlState),
+        VMSTATE_UINT8(route_gpu_fiq, BCM2836ControlState),
+        VMSTATE_UINT32_ARRAY(timercontrol, BCM2836ControlState, BCM2836_NCORES),
+        VMSTATE_UINT32_ARRAY(mailboxcontrol, BCM2836ControlState,
+                             BCM2836_NCORES),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2836_control_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = bcm2836_control_reset;
+    dc->vmsd = &vmstate_bcm2836_control;
+}
+
+static TypeInfo bcm2836_control_info = {
+    .name          = TYPE_BCM2836_CONTROL,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2836ControlState),
+    .class_init    = bcm2836_control_class_init,
+    .instance_init = bcm2836_control_init,
+};
+
+static void bcm2836_control_register_types(void)
+{
+    type_register_static(&bcm2836_control_info);
+}
+
+type_init(bcm2836_control_register_types)
diff --git a/hw/intc/etraxfs_pic.c b/hw/intc/etraxfs_pic.c
index 636262b..64a6f4b 100644
--- a/hw/intc/etraxfs_pic.c
+++ b/hw/intc/etraxfs_pic.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 //#include "pc.h"
@@ -131,11 +132,13 @@
 }
 
 static void irq_handler(void *opaque, int irq, int level)
-{   
+{
     struct etrax_pic *fs = (void *)opaque;
 
-    if (irq >= 30)
-        return nmi_handler(opaque, irq, level);
+    if (irq >= 30) {
+        nmi_handler(opaque, irq, level);
+        return;
+    }
 
     irq -= 1;
     fs->regs[R_R_VECT] &= ~(1 << irq);
@@ -143,19 +146,19 @@
     pic_update(fs);
 }
 
-static int etraxfs_pic_init(SysBusDevice *sbd)
+static void etraxfs_pic_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    struct etrax_pic *s = ETRAX_FS_PIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    struct etrax_pic *s = ETRAX_FS_PIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     qdev_init_gpio_in(dev, irq_handler, 32);
     sysbus_init_irq(sbd, &s->parent_irq);
     sysbus_init_irq(sbd, &s->parent_nmi);
 
-    memory_region_init_io(&s->mmio, OBJECT(s), &pic_ops, s,
+    memory_region_init_io(&s->mmio, obj, &pic_ops, s,
                           "etraxfs-pic", R_MAX * 4);
     sysbus_init_mmio(sbd, &s->mmio);
-    return 0;
 }
 
 static Property etraxfs_pic_properties[] = {
@@ -166,9 +169,7 @@
 static void etraxfs_pic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = etraxfs_pic_init;
     dc->props = etraxfs_pic_properties;
     /*
      * Note: pointer property "interrupt_vector" may remain null, thus
@@ -180,6 +181,7 @@
     .name          = TYPE_ETRAX_FS_PIC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct etrax_pic),
+    .instance_init = etraxfs_pic_init,
     .class_init    = etraxfs_pic_class_init,
 };
 
diff --git a/hw/intc/exynos4210_combiner.c b/hw/intc/exynos4210_combiner.c
index a6b7028..f19a706 100644
--- a/hw/intc/exynos4210_combiner.c
+++ b/hw/intc/exynos4210_combiner.c
@@ -27,6 +27,7 @@
  * IRQs are passed to GIC through Combiner.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #include "hw/arm/exynos4210.h"
@@ -405,10 +406,11 @@
 /*
  * Internal Combiner initialization.
  */
-static int exynos4210_combiner_init(SysBusDevice *sbd)
+static void exynos4210_combiner_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    Exynos4210CombinerState *s = EXYNOS4210_COMBINER(dev);
+    DeviceState *dev = DEVICE(obj);
+    Exynos4210CombinerState *s = EXYNOS4210_COMBINER(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     unsigned int i;
 
     /* Allocate general purpose input signals and connect a handler to each of
@@ -420,11 +422,9 @@
         sysbus_init_irq(sbd, &s->output_irq[i]);
     }
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_combiner_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_combiner_ops, s,
                           "exynos4210-combiner", IIC_REGION_SIZE);
     sysbus_init_mmio(sbd, &s->iomem);
-
-    return 0;
 }
 
 static Property exynos4210_combiner_properties[] = {
@@ -435,9 +435,7 @@
 static void exynos4210_combiner_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_combiner_init;
     dc->reset = exynos4210_combiner_reset;
     dc->props = exynos4210_combiner_properties;
     dc->vmsd = &vmstate_exynos4210_combiner;
@@ -447,6 +445,7 @@
     .name          = TYPE_EXYNOS4210_COMBINER,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210CombinerState),
+    .instance_init = exynos4210_combiner_init,
     .class_init    = exynos4210_combiner_class_init,
 };
 
diff --git a/hw/intc/exynos4210_gic.c b/hw/intc/exynos4210_gic.c
index 0590d5d..fd7a8f3 100644
--- a/hw/intc/exynos4210_gic.c
+++ b/hw/intc/exynos4210_gic.c
@@ -20,6 +20,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu-common.h"
 #include "hw/irq.h"
@@ -213,9 +214,6 @@
     uint32_t grp, bit, irq_id, n;
 
     for (n = 0; n < EXYNOS4210_MAX_EXT_COMBINER_IN_IRQ; n++) {
-        s->board_irqs[n] = qemu_irq_split(s->int_combiner_irq[n],
-                s->ext_combiner_irq[n]);
-
         irq_id = 0;
         if (n == EXYNOS4210_COMBINER_GET_IRQ_NUM(1, 4) ||
                 n == EXYNOS4210_COMBINER_GET_IRQ_NUM(12, 4)) {
@@ -230,8 +228,10 @@
         if (irq_id) {
             s->board_irqs[n] = qemu_irq_split(s->int_combiner_irq[n],
                     s->ext_gic_irq[irq_id-32]);
+        } else {
+            s->board_irqs[n] = qemu_irq_split(s->int_combiner_irq[n],
+                    s->ext_combiner_irq[n]);
         }
-
     }
     for (; n < EXYNOS4210_MAX_INT_COMBINER_IN_IRQ; n++) {
         /* these IDs are passed to Internal Combiner and External GIC */
@@ -281,10 +281,11 @@
     qemu_set_irq(qdev_get_gpio_in(s->gic, irq), level);
 }
 
-static int exynos4210_gic_init(SysBusDevice *sbd)
+static void exynos4210_gic_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    Exynos4210GicState *s = EXYNOS4210_GIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    Exynos4210GicState *s = EXYNOS4210_GIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     uint32_t i;
     const char cpu_prefix[] = "exynos4210-gic-alias_cpu";
     const char dist_prefix[] = "exynos4210-gic-alias_dist";
@@ -305,15 +306,15 @@
     qdev_init_gpio_in(dev, exynos4210_gic_set_irq,
                       EXYNOS4210_GIC_NIRQ - 32);
 
-    memory_region_init(&s->cpu_container, OBJECT(s), "exynos4210-cpu-container",
+    memory_region_init(&s->cpu_container, obj, "exynos4210-cpu-container",
             EXYNOS4210_EXT_GIC_CPU_REGION_SIZE);
-    memory_region_init(&s->dist_container, OBJECT(s), "exynos4210-dist-container",
+    memory_region_init(&s->dist_container, obj, "exynos4210-dist-container",
             EXYNOS4210_EXT_GIC_DIST_REGION_SIZE);
 
     for (i = 0; i < s->num_cpu; i++) {
         /* Map CPU interface per SMP Core */
         sprintf(cpu_alias_name, "%s%x", cpu_prefix, i);
-        memory_region_init_alias(&s->cpu_alias[i], OBJECT(s),
+        memory_region_init_alias(&s->cpu_alias[i], obj,
                                  cpu_alias_name,
                                  sysbus_mmio_get_region(busdev, 1),
                                  0,
@@ -323,7 +324,7 @@
 
         /* Map Distributor per SMP Core */
         sprintf(dist_alias_name, "%s%x", dist_prefix, i);
-        memory_region_init_alias(&s->dist_alias[i], OBJECT(s),
+        memory_region_init_alias(&s->dist_alias[i], obj,
                                  dist_alias_name,
                                  sysbus_mmio_get_region(busdev, 0),
                                  0,
@@ -334,8 +335,6 @@
 
     sysbus_init_mmio(sbd, &s->cpu_container);
     sysbus_init_mmio(sbd, &s->dist_container);
-
-    return 0;
 }
 
 static Property exynos4210_gic_properties[] = {
@@ -346,9 +345,7 @@
 static void exynos4210_gic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_gic_init;
     dc->props = exynos4210_gic_properties;
 }
 
@@ -356,6 +353,7 @@
     .name          = TYPE_EXYNOS4210_GIC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210GicState),
+    .instance_init = exynos4210_gic_init,
     .class_init    = exynos4210_gic_class_init,
 };
 
@@ -430,9 +428,16 @@
 /*
  * IRQ Gate initialization.
  */
-static int exynos4210_irq_gate_init(SysBusDevice *sbd)
+static void exynos4210_irq_gate_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
+    Exynos4210IRQGateState *s = EXYNOS4210_IRQ_GATE(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+
+    sysbus_init_irq(sbd, &s->out);
+}
+
+static void exynos4210_irq_gate_realize(DeviceState *dev, Error **errp)
+{
     Exynos4210IRQGateState *s = EXYNOS4210_IRQ_GATE(dev);
 
     /* Allocate general purpose input signals and connect a handler to each of
@@ -440,27 +445,23 @@
     qdev_init_gpio_in(dev, exynos4210_irq_gate_handler, s->n_in);
 
     s->level = g_malloc0(s->n_in * sizeof(*s->level));
-
-    sysbus_init_irq(sbd, &s->out);
-
-    return 0;
 }
 
 static void exynos4210_irq_gate_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_irq_gate_init;
     dc->reset = exynos4210_irq_gate_reset;
     dc->vmsd = &vmstate_exynos4210_irq_gate;
     dc->props = exynos4210_irq_gate_properties;
+    dc->realize = exynos4210_irq_gate_realize;
 }
 
 static const TypeInfo exynos4210_irq_gate_info = {
     .name          = TYPE_EXYNOS4210_IRQ_GATE,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210IRQGateState),
+    .instance_init = exynos4210_irq_gate_init,
     .class_init    = exynos4210_irq_gate_class_init,
 };
 
diff --git a/hw/intc/gic_internal.h b/hw/intc/gic_internal.h
index e87ef36..3f31174 100644
--- a/hw/intc/gic_internal.h
+++ b/hw/intc/gic_internal.h
@@ -50,17 +50,40 @@
                                     s->priority1[irq][cpu] :            \
                                     s->priority2[(irq) - GIC_INTERNAL])
 #define GIC_TARGET(irq) s->irq_target[irq]
+#define GIC_CLEAR_GROUP(irq, cm) (s->irq_state[irq].group &= ~(cm))
+#define GIC_SET_GROUP(irq, cm) (s->irq_state[irq].group |= (cm))
+#define GIC_TEST_GROUP(irq, cm) ((s->irq_state[irq].group & (cm)) != 0)
+
+#define GICD_CTLR_EN_GRP0 (1U << 0)
+#define GICD_CTLR_EN_GRP1 (1U << 1)
+
+#define GICC_CTLR_EN_GRP0    (1U << 0)
+#define GICC_CTLR_EN_GRP1    (1U << 1)
+#define GICC_CTLR_ACK_CTL    (1U << 2)
+#define GICC_CTLR_FIQ_EN     (1U << 3)
+#define GICC_CTLR_CBPR       (1U << 4) /* GICv1: SBPR */
+#define GICC_CTLR_EOIMODE    (1U << 9)
+#define GICC_CTLR_EOIMODE_NS (1U << 10)
+
+/* Valid bits for GICC_CTLR for GICv1, v1 with security extensions,
+ * GICv2 and GICv2 with security extensions:
+ */
+#define GICC_CTLR_V1_MASK    0x1
+#define GICC_CTLR_V1_S_MASK  0x1f
+#define GICC_CTLR_V2_MASK    0x21f
+#define GICC_CTLR_V2_S_MASK  0x61f
 
 /* The special cases for the revision property: */
 #define REV_11MPCORE 0
 #define REV_NVIC 0xffffffff
 
 void gic_set_pending_private(GICState *s, int cpu, int irq);
-uint32_t gic_acknowledge_irq(GICState *s, int cpu);
-void gic_complete_irq(GICState *s, int cpu, int irq);
+uint32_t gic_acknowledge_irq(GICState *s, int cpu, MemTxAttrs attrs);
+void gic_complete_irq(GICState *s, int cpu, int irq, MemTxAttrs attrs);
 void gic_update(GICState *s);
 void gic_init_irqs_and_distributor(GICState *s);
-void gic_set_priority(GICState *s, int cpu, int irq, uint8_t val);
+void gic_set_priority(GICState *s, int cpu, int irq, uint8_t val,
+                      MemTxAttrs attrs);
 
 static inline bool gic_test_pending(GICState *s, int irq, int cm)
 {
@@ -77,4 +100,4 @@
     }
 }
 
-#endif /* !QEMU_ARM_GIC_INTERNAL_H */
+#endif /* QEMU_ARM_GIC_INTERNAL_H */
diff --git a/hw/intc/gicv3_internal.h b/hw/intc/gicv3_internal.h
new file mode 100644
index 0000000..8f3567e
--- /dev/null
+++ b/hw/intc/gicv3_internal.h
@@ -0,0 +1,331 @@
+/*
+ * ARM GICv3 support - internal interfaces
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Written by Peter Maydell
+ * Reworked for GICv3 by Shlomo Pongratz and Pavel Fedin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_ARM_GICV3_INTERNAL_H
+#define QEMU_ARM_GICV3_INTERNAL_H
+
+#include "hw/intc/arm_gicv3_common.h"
+
+/* Distributor registers, as offsets from the distributor base address */
+#define GICD_CTLR            0x0000
+#define GICD_TYPER           0x0004
+#define GICD_IIDR            0x0008
+#define GICD_STATUSR         0x0010
+#define GICD_SETSPI_NSR      0x0040
+#define GICD_CLRSPI_NSR      0x0048
+#define GICD_SETSPI_SR       0x0050
+#define GICD_CLRSPI_SR       0x0058
+#define GICD_SEIR            0x0068
+#define GICD_IGROUPR         0x0080
+#define GICD_ISENABLER       0x0100
+#define GICD_ICENABLER       0x0180
+#define GICD_ISPENDR         0x0200
+#define GICD_ICPENDR         0x0280
+#define GICD_ISACTIVER       0x0300
+#define GICD_ICACTIVER       0x0380
+#define GICD_IPRIORITYR      0x0400
+#define GICD_ITARGETSR       0x0800
+#define GICD_ICFGR           0x0C00
+#define GICD_IGRPMODR        0x0D00
+#define GICD_NSACR           0x0E00
+#define GICD_SGIR            0x0F00
+#define GICD_CPENDSGIR       0x0F10
+#define GICD_SPENDSGIR       0x0F20
+#define GICD_IROUTER         0x6000
+#define GICD_IDREGS          0xFFD0
+
+/* GICD_CTLR fields  */
+#define GICD_CTLR_EN_GRP0           (1U << 0)
+#define GICD_CTLR_EN_GRP1NS         (1U << 1) /* GICv3 5.3.20 */
+#define GICD_CTLR_EN_GRP1S          (1U << 2)
+#define GICD_CTLR_EN_GRP1_ALL       (GICD_CTLR_EN_GRP1NS | GICD_CTLR_EN_GRP1S)
+/* Bit 4 is ARE if the system doesn't support TrustZone, ARE_S otherwise */
+#define GICD_CTLR_ARE               (1U << 4)
+#define GICD_CTLR_ARE_S             (1U << 4)
+#define GICD_CTLR_ARE_NS            (1U << 5)
+#define GICD_CTLR_DS                (1U << 6)
+#define GICD_CTLR_E1NWF             (1U << 7)
+#define GICD_CTLR_RWP               (1U << 31)
+
+/*
+ * Redistributor frame offsets from RD_base
+ */
+#define GICR_SGI_OFFSET 0x10000
+
+/*
+ * Redistributor registers, offsets from RD_base
+ */
+#define GICR_CTLR             0x0000
+#define GICR_IIDR             0x0004
+#define GICR_TYPER            0x0008
+#define GICR_STATUSR          0x0010
+#define GICR_WAKER            0x0014
+#define GICR_SETLPIR          0x0040
+#define GICR_CLRLPIR          0x0048
+#define GICR_PROPBASER        0x0070
+#define GICR_PENDBASER        0x0078
+#define GICR_INVLPIR          0x00A0
+#define GICR_INVALLR          0x00B0
+#define GICR_SYNCR            0x00C0
+#define GICR_IDREGS           0xFFD0
+
+/* SGI and PPI Redistributor registers, offsets from RD_base */
+#define GICR_IGROUPR0         (GICR_SGI_OFFSET + 0x0080)
+#define GICR_ISENABLER0       (GICR_SGI_OFFSET + 0x0100)
+#define GICR_ICENABLER0       (GICR_SGI_OFFSET + 0x0180)
+#define GICR_ISPENDR0         (GICR_SGI_OFFSET + 0x0200)
+#define GICR_ICPENDR0         (GICR_SGI_OFFSET + 0x0280)
+#define GICR_ISACTIVER0       (GICR_SGI_OFFSET + 0x0300)
+#define GICR_ICACTIVER0       (GICR_SGI_OFFSET + 0x0380)
+#define GICR_IPRIORITYR       (GICR_SGI_OFFSET + 0x0400)
+#define GICR_ICFGR0           (GICR_SGI_OFFSET + 0x0C00)
+#define GICR_ICFGR1           (GICR_SGI_OFFSET + 0x0C04)
+#define GICR_IGRPMODR0        (GICR_SGI_OFFSET + 0x0D00)
+#define GICR_NSACR            (GICR_SGI_OFFSET + 0x0E00)
+
+#define GICR_CTLR_ENABLE_LPIS        (1U << 0)
+#define GICR_CTLR_RWP                (1U << 3)
+#define GICR_CTLR_DPG0               (1U << 24)
+#define GICR_CTLR_DPG1NS             (1U << 25)
+#define GICR_CTLR_DPG1S              (1U << 26)
+#define GICR_CTLR_UWP                (1U << 31)
+
+#define GICR_TYPER_PLPIS             (1U << 0)
+#define GICR_TYPER_VLPIS             (1U << 1)
+#define GICR_TYPER_DIRECTLPI         (1U << 3)
+#define GICR_TYPER_LAST              (1U << 4)
+#define GICR_TYPER_DPGS              (1U << 5)
+#define GICR_TYPER_PROCNUM           (0xFFFFU << 8)
+#define GICR_TYPER_COMMONLPIAFF      (0x3 << 24)
+#define GICR_TYPER_AFFINITYVALUE     (0xFFFFFFFFULL << 32)
+
+#define GICR_WAKER_ProcessorSleep    (1U << 1)
+#define GICR_WAKER_ChildrenAsleep    (1U << 2)
+
+#define GICR_PROPBASER_OUTER_CACHEABILITY_MASK (7ULL << 56)
+#define GICR_PROPBASER_ADDR_MASK               (0xfffffffffULL << 12)
+#define GICR_PROPBASER_SHAREABILITY_MASK       (3U << 10)
+#define GICR_PROPBASER_CACHEABILITY_MASK       (7U << 7)
+#define GICR_PROPBASER_IDBITS_MASK             (0x1f)
+
+#define GICR_PENDBASER_PTZ                     (1ULL << 62)
+#define GICR_PENDBASER_OUTER_CACHEABILITY_MASK (7ULL << 56)
+#define GICR_PENDBASER_ADDR_MASK               (0xffffffffULL << 16)
+#define GICR_PENDBASER_SHAREABILITY_MASK       (3U << 10)
+#define GICR_PENDBASER_CACHEABILITY_MASK       (7U << 7)
+
+#define ICC_CTLR_EL1_CBPR           (1U << 0)
+#define ICC_CTLR_EL1_EOIMODE        (1U << 1)
+#define ICC_CTLR_EL1_PMHE           (1U << 6)
+#define ICC_CTLR_EL1_PRIBITS_SHIFT 8
+#define ICC_CTLR_EL1_IDBITS_SHIFT 11
+#define ICC_CTLR_EL1_SEIS           (1U << 14)
+#define ICC_CTLR_EL1_A3V            (1U << 15)
+
+#define ICC_PMR_PRIORITY_MASK    0xff
+#define ICC_BPR_BINARYPOINT_MASK 0x07
+#define ICC_IGRPEN_ENABLE        0x01
+
+#define ICC_CTLR_EL3_CBPR_EL1S (1U << 0)
+#define ICC_CTLR_EL3_CBPR_EL1NS (1U << 1)
+#define ICC_CTLR_EL3_EOIMODE_EL3 (1U << 2)
+#define ICC_CTLR_EL3_EOIMODE_EL1S (1U << 3)
+#define ICC_CTLR_EL3_EOIMODE_EL1NS (1U << 4)
+#define ICC_CTLR_EL3_RM (1U << 5)
+#define ICC_CTLR_EL3_PMHE (1U << 6)
+#define ICC_CTLR_EL3_PRIBITS_SHIFT 8
+#define ICC_CTLR_EL3_IDBITS_SHIFT 11
+#define ICC_CTLR_EL3_SEIS (1U << 14)
+#define ICC_CTLR_EL3_A3V (1U << 15)
+#define ICC_CTLR_EL3_NDS (1U << 17)
+
+/* Special interrupt IDs */
+#define INTID_SECURE 1020
+#define INTID_NONSECURE 1021
+#define INTID_SPURIOUS 1023
+
+/* Functions internal to the emulated GICv3 */
+
+/**
+ * gicv3_redist_update:
+ * @cs: GICv3CPUState for this redistributor
+ *
+ * Recalculate the highest priority pending interrupt after a
+ * change to redistributor state, and inform the CPU accordingly.
+ */
+void gicv3_redist_update(GICv3CPUState *cs);
+
+/**
+ * gicv3_update:
+ * @s: GICv3State
+ * @start: first interrupt whose state changed
+ * @len: length of the range of interrupts whose state changed
+ *
+ * Recalculate the highest priority pending interrupts after a
+ * change to the distributor state affecting @len interrupts
+ * starting at @start, and inform the CPUs accordingly.
+ */
+void gicv3_update(GICv3State *s, int start, int len);
+
+/**
+ * gicv3_full_update_noirqset:
+ * @s: GICv3State
+ *
+ * Recalculate the cached information about highest priority
+ * pending interrupts, but don't inform the CPUs. This should be
+ * called after an incoming migration has loaded new state.
+ */
+void gicv3_full_update_noirqset(GICv3State *s);
+
+/**
+ * gicv3_full_update:
+ * @s: GICv3State
+ *
+ * Recalculate the highest priority pending interrupts after
+ * a change that could affect the status of all interrupts,
+ * and inform the CPUs accordingly.
+ */
+void gicv3_full_update(GICv3State *s);
+MemTxResult gicv3_dist_read(void *opaque, hwaddr offset, uint64_t *data,
+                            unsigned size, MemTxAttrs attrs);
+MemTxResult gicv3_dist_write(void *opaque, hwaddr addr, uint64_t data,
+                             unsigned size, MemTxAttrs attrs);
+MemTxResult gicv3_redist_read(void *opaque, hwaddr offset, uint64_t *data,
+                              unsigned size, MemTxAttrs attrs);
+MemTxResult gicv3_redist_write(void *opaque, hwaddr offset, uint64_t data,
+                               unsigned size, MemTxAttrs attrs);
+void gicv3_dist_set_irq(GICv3State *s, int irq, int level);
+void gicv3_redist_set_irq(GICv3CPUState *cs, int irq, int level);
+void gicv3_redist_send_sgi(GICv3CPUState *cs, int grp, int irq, bool ns);
+void gicv3_init_cpuif(GICv3State *s);
+
+/**
+ * gicv3_cpuif_update:
+ * @cs: GICv3CPUState for the CPU to update
+ *
+ * Recalculate whether to assert the IRQ or FIQ lines after a change
+ * to the current highest priority pending interrupt, the CPU's
+ * current running priority or the CPU's current exception level or
+ * security state.
+ */
+void gicv3_cpuif_update(GICv3CPUState *cs);
+
+static inline uint32_t gicv3_iidr(void)
+{
+    /* Return the Implementer Identification Register value
+     * for the emulated GICv3, as reported in GICD_IIDR and GICR_IIDR.
+     *
+     * We claim to be an ARM r0p0 with a zero ProductID.
+     * This is the same as an r0p0 GIC-500.
+     */
+    return 0x43b;
+}
+
+static inline uint32_t gicv3_idreg(int regoffset)
+{
+    /* Return the value of the CoreSight ID register at the specified
+     * offset from the first ID register (as found in the distributor
+     * and redistributor register banks).
+     * These values indicate an ARM implementation of a GICv3.
+     */
+    static const uint8_t gicd_ids[] = {
+        0x44, 0x00, 0x00, 0x00, 0x92, 0xB4, 0x3B, 0x00, 0x0D, 0xF0, 0x05, 0xB1
+    };
+    return gicd_ids[regoffset / 4];
+}
+
+/**
+ * gicv3_irq_group:
+ *
+ * Return the group which this interrupt is configured as (GICV3_G0,
+ * GICV3_G1 or GICV3_G1NS).
+ */
+static inline int gicv3_irq_group(GICv3State *s, GICv3CPUState *cs, int irq)
+{
+    bool grpbit, grpmodbit;
+
+    if (irq < GIC_INTERNAL) {
+        grpbit = extract32(cs->gicr_igroupr0, irq, 1);
+        grpmodbit = extract32(cs->gicr_igrpmodr0, irq, 1);
+    } else {
+        grpbit = gicv3_gicd_group_test(s, irq);
+        grpmodbit = gicv3_gicd_grpmod_test(s, irq);
+    }
+    if (grpbit) {
+        return GICV3_G1NS;
+    }
+    if (s->gicd_ctlr & GICD_CTLR_DS) {
+        return GICV3_G0;
+    }
+    return grpmodbit ? GICV3_G1 : GICV3_G0;
+}
+
+/**
+ * gicv3_redist_affid:
+ *
+ * Return the 32-bit affinity ID of the CPU connected to this redistributor
+ */
+static inline uint32_t gicv3_redist_affid(GICv3CPUState *cs)
+{
+    return cs->gicr_typer >> 32;
+}
+
+/**
+ * gicv3_cache_target_cpustate:
+ *
+ * Update the cached CPU state corresponding to the target for this interrupt
+ * (which is kept in s->gicd_irouter_target[]).
+ */
+static inline void gicv3_cache_target_cpustate(GICv3State *s, int irq)
+{
+    GICv3CPUState *cs = NULL;
+    int i;
+    uint32_t tgtaff = extract64(s->gicd_irouter[irq], 0, 24) |
+        extract64(s->gicd_irouter[irq], 32, 8) << 24;
+
+    for (i = 0; i < s->num_cpu; i++) {
+        if (s->cpu[i].gicr_typer >> 32 == tgtaff) {
+            cs = &s->cpu[i];
+            break;
+        }
+    }
+
+    s->gicd_irouter_target[irq] = cs;
+}
+
+/**
+ * gicv3_cache_all_target_cpustates:
+ *
+ * Populate the entire cache of CPU state pointers for interrupt targets
+ * (eg after inbound migration or CPU reset)
+ */
+static inline void gicv3_cache_all_target_cpustates(GICv3State *s)
+{
+    int irq;
+
+    for (irq = GIC_INTERNAL; irq < GICV3_MAXIRQ; irq++) {
+        gicv3_cache_target_cpustate(s, irq);
+    }
+}
+
+#endif /* QEMU_ARM_GICV3_INTERNAL_H */
diff --git a/hw/intc/goldfish_pic.c b/hw/intc/goldfish_pic.c
index ccbb094..d2f697f 100644
--- a/hw/intc/goldfish_pic.c
+++ b/hw/intc/goldfish_pic.c
@@ -10,6 +10,8 @@
 ** GNU General Public License for more details.
 */
 
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "exec/ram_addr.h"
 #include "migration/qemu-file.h"
 #include "sysemu/char.h"
diff --git a/hw/intc/grlib_irqmp.c b/hw/intc/grlib_irqmp.c
index d1813f7..ac7e63f 100644
--- a/hw/intc/grlib_irqmp.c
+++ b/hw/intc/grlib_irqmp.c
@@ -24,12 +24,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "cpu.h"
 
 #include "hw/sparc/grlib.h"
 
 #include "trace.h"
+#include "qapi/error.h"
 
 #define IRQMP_MAX_CPU 16
 #define IRQMP_REG_SIZE 256      /* Size of memory mapped registers */
@@ -322,23 +324,27 @@
     irqmp->state->parent = irqmp;
 }
 
-static int grlib_irqmp_init(SysBusDevice *dev)
+static void grlib_irqmp_init(Object *obj)
 {
-    IRQMP *irqmp = GRLIB_IRQMP(dev);
+    IRQMP *irqmp = GRLIB_IRQMP(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    /* Check parameters */
-    if (irqmp->set_pil_in == NULL) {
-        return -1;
-    }
-
-    memory_region_init_io(&irqmp->iomem, OBJECT(dev), &grlib_irqmp_ops, irqmp,
+    memory_region_init_io(&irqmp->iomem, obj, &grlib_irqmp_ops, irqmp,
                           "irqmp", IRQMP_REG_SIZE);
 
     irqmp->state = g_malloc0(sizeof *irqmp->state);
 
     sysbus_init_mmio(dev, &irqmp->iomem);
+}
 
-    return 0;
+static void grlib_irqmp_realize(DeviceState *dev, Error **errp)
+{
+    IRQMP *irqmp = GRLIB_IRQMP(dev);
+
+        /* Check parameters */
+    if (irqmp->set_pil_in == NULL) {
+        error_setg(errp, "set_pil_in cannot be NULL.");
+    }
 }
 
 static Property grlib_irqmp_properties[] = {
@@ -350,19 +356,19 @@
 static void grlib_irqmp_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = grlib_irqmp_init;
     dc->reset = grlib_irqmp_reset;
     dc->props = grlib_irqmp_properties;
     /* Reason: pointer properties "set_pil_in", "set_pil_in_opaque" */
     dc->cannot_instantiate_with_device_add_yet = true;
+    dc->realize = grlib_irqmp_realize;
 }
 
 static const TypeInfo grlib_irqmp_info = {
     .name          = TYPE_GRLIB_IRQMP,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IRQMP),
+    .instance_init = grlib_irqmp_init,
     .class_init    = grlib_irqmp_class_init,
 };
 
diff --git a/hw/intc/heathrow_pic.c b/hw/intc/heathrow_pic.c
index 9ff3119..171f5ed 100644
--- a/hw/intc/heathrow_pic.c
+++ b/hw/intc/heathrow_pic.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 
diff --git a/hw/intc/i8259.c b/hw/intc/i8259.c
index c51901b..c2607a5 100644
--- a/hw/intc/i8259.c
+++ b/hw/intc/i8259.c
@@ -21,11 +21,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
 #include "monitor/monitor.h"
 #include "qemu/timer.h"
+#include "qemu/log.h"
 #include "hw/isa/i8259_internal.h"
 
 /* debug PIC */
@@ -229,7 +231,7 @@
     printf("IRQ%d latency=%0.3fus\n",
            irq,
            (double)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                    irq_time[irq]) * 1000000.0 / get_ticks_per_sec());
+                    irq_time[irq]) * 1000000.0 / NANOSECONDS_PER_SECOND);
 #endif
     DPRINTF("pic_interrupt: irq=%d\n", irq);
     return intno;
@@ -429,7 +431,7 @@
     pc->parent_realize(dev, errp);
 }
 
-void pic_info(Monitor *mon, const QDict *qdict)
+void hmp_info_pic(Monitor *mon, const QDict *qdict)
 {
     int i;
     PICCommonState *s;
@@ -447,7 +449,7 @@
     }
 }
 
-void irq_info(Monitor *mon, const QDict *qdict)
+void hmp_info_irq(Monitor *mon, const QDict *qdict)
 {
 #ifndef DEBUG_IRQ_COUNT
     monitor_printf(mon, "irq statistic code not compiled.\n");
diff --git a/hw/intc/i8259_common.c b/hw/intc/i8259_common.c
index fbf26e5..3a850b0 100644
--- a/hw/intc/i8259_common.c
+++ b/hw/intc/i8259_common.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/i8259_internal.h"
 
diff --git a/hw/intc/imx_avic.c b/hw/intc/imx_avic.c
index e48f66c..813e587 100644
--- a/hw/intc/imx_avic.c
+++ b/hw/intc/imx_avic.c
@@ -7,6 +7,7 @@
  * Copyright (c) 2008 OKL
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under the GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
@@ -14,67 +15,24 @@
  * TODO: implement vectors.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "qemu/host-utils.h"
+#include "qemu/osdep.h"
+#include "hw/intc/imx_avic.h"
+#include "qemu/log.h"
 
-#define DEBUG_INT 1
-#undef DEBUG_INT /* comment out for debugging */
+#ifndef DEBUG_IMX_AVIC
+#define DEBUG_IMX_AVIC 0
+#endif
 
-#ifdef DEBUG_INT
 #define DPRINTF(fmt, args...) \
-do { printf("imx_avic: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-    do  { fprintf(stderr, "imx_avic: " fmt, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_AVIC_NUM_IRQS 64
-
-/* Interrupt Control Bits */
-#define ABFLAG (1<<25)
-#define ABFEN (1<<24)
-#define NIDIS (1<<22) /* Normal Interrupt disable */
-#define FIDIS (1<<21) /* Fast interrupt disable */
-#define NIAD  (1<<20) /* Normal Interrupt Arbiter Rise ARM level */
-#define FIAD  (1<<19) /* Fast Interrupt Arbiter Rise ARM level */
-#define NM    (1<<18) /* Normal interrupt mode */
-
-
-#define PRIO_PER_WORD (sizeof(uint32_t) * 8 / 4)
-#define PRIO_WORDS (IMX_AVIC_NUM_IRQS/PRIO_PER_WORD)
-
-#define TYPE_IMX_AVIC "imx_avic"
-#define IMX_AVIC(obj) \
-    OBJECT_CHECK(IMXAVICState, (obj), TYPE_IMX_AVIC)
-
-typedef struct IMXAVICState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    uint64_t pending;
-    uint64_t enabled;
-    uint64_t is_fiq;
-    uint32_t intcntl;
-    uint32_t intmask;
-    qemu_irq irq;
-    qemu_irq fiq;
-    uint32_t prio[PRIO_WORDS]; /* Priorities are 4-bits each */
-} IMXAVICState;
+    do { \
+        if (DEBUG_IMX_AVIC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_AVIC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static const VMStateDescription vmstate_imx_avic = {
-    .name = "imx-avic",
+    .name = TYPE_IMX_AVIC,
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
@@ -88,8 +46,6 @@
     },
 };
 
-
-
 static inline int imx_avic_prio(IMXAVICState *s, int irq)
 {
     uint32_t word = irq / PRIO_PER_WORD;
@@ -151,8 +107,8 @@
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
 
+    DPRINTF("read(offset = 0x%" HWADDR_PRIx ")\n", offset);
 
-    DPRINTF("read(offset = 0x%x)\n", offset >> 2);
     switch (offset >> 2) {
     case 0: /* INTCNTL */
         return s->intcntl;
@@ -249,7 +205,8 @@
         return 0x4;
 
     default:
-        IPRINTF("imx_avic_read: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_AVIC, __func__, offset);
         return 0;
     }
 }
@@ -261,13 +218,13 @@
 
     /* Vector Registers not yet supported */
     if (offset >= 0x100 && offset <= 0x2fc) {
-        IPRINTF("imx_avic_write to vector register %d ignored\n",
-                (unsigned int)((offset - 0x100) >> 2));
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: vector %d ignored\n",
+                      TYPE_IMX_AVIC, __func__, (int)((offset - 0x100) >> 2));
         return;
     }
 
-    DPRINTF("imx_avic_write(0x%x) = %x\n",
-            (unsigned int)offset>>2, (unsigned int)val);
+    DPRINTF("(0x%" HWADDR_PRIx ") = 0x%x\n", offset, (unsigned int)val);
+
     switch (offset >> 2) {
     case 0: /* Interrupt Control Register, INTCNTL */
         s->intcntl = val & (ABFEN | NIDIS | FIDIS | NIAD | FIAD | NM);
@@ -341,7 +298,8 @@
         return;
 
     default:
-        IPRINTF("imx_avic_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_AVIC, __func__, offset);
     }
     imx_avic_update(s);
 }
@@ -364,28 +322,26 @@
     memset(s->prio, 0, sizeof s->prio);
 }
 
-static int imx_avic_init(SysBusDevice *sbd)
+static void imx_avic_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    IMXAVICState *s = IMX_AVIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    IMXAVICState *s = IMX_AVIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &imx_avic_ops, s,
-                          "imx_avic", 0x1000);
+    memory_region_init_io(&s->iomem, obj, &imx_avic_ops, s,
+                          TYPE_IMX_AVIC, 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
 
     qdev_init_gpio_in(dev, imx_avic_set_irq, IMX_AVIC_NUM_IRQS);
     sysbus_init_irq(sbd, &s->irq);
     sysbus_init_irq(sbd, &s->fiq);
-
-    return 0;
 }
 
 
 static void imx_avic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-    k->init = imx_avic_init;
+
     dc->vmsd = &vmstate_imx_avic;
     dc->reset = imx_avic_reset;
     dc->desc = "i.MX Advanced Vector Interrupt Controller";
@@ -395,6 +351,7 @@
     .name = TYPE_IMX_AVIC,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXAVICState),
+    .instance_init = imx_avic_init,
     .class_init = imx_avic_class_init,
 };
 
diff --git a/hw/intc/ioapic.c b/hw/intc/ioapic.c
index b527932..31791b0 100644
--- a/hw/intc/ioapic.c
+++ b/hw/intc/ioapic.c
@@ -20,10 +20,19 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "monitor/monitor.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
+#include "hw/i386/apic.h"
 #include "hw/i386/ioapic.h"
 #include "hw/i386/ioapic_internal.h"
+#include "include/hw/pci/msi.h"
+#include "sysemu/kvm.h"
+#include "target-i386/cpu.h"
+#include "hw/i386/apic-msidef.h"
+#include "hw/i386/x86-iommu.h"
 
 //#define DEBUG_IOAPIC
 
@@ -34,44 +43,105 @@
 #define DPRINTF(fmt, ...)
 #endif
 
+#define APIC_DELIVERY_MODE_SHIFT 8
+#define APIC_POLARITY_SHIFT 14
+#define APIC_TRIG_MODE_SHIFT 15
+
 static IOAPICCommonState *ioapics[MAX_IOAPICS];
 
 /* global variable from ioapic_common.c */
 extern int ioapic_no;
 
+struct ioapic_entry_info {
+    /* fields parsed from IOAPIC entries */
+    uint8_t masked;
+    uint8_t trig_mode;
+    uint16_t dest_idx;
+    uint8_t dest_mode;
+    uint8_t delivery_mode;
+    uint8_t vector;
+
+    /* MSI message generated from above parsed fields */
+    uint32_t addr;
+    uint32_t data;
+};
+
+static void ioapic_entry_parse(uint64_t entry, struct ioapic_entry_info *info)
+{
+    memset(info, 0, sizeof(*info));
+    info->masked = (entry >> IOAPIC_LVT_MASKED_SHIFT) & 1;
+    info->trig_mode = (entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1;
+    /*
+     * By default, this would be dest_id[8] + reserved[8]. When IR
+     * is enabled, this would be interrupt_index[15] +
+     * interrupt_format[1]. This field never means anything, but
+     * only used to generate corresponding MSI.
+     */
+    info->dest_idx = (entry >> IOAPIC_LVT_DEST_IDX_SHIFT) & 0xffff;
+    info->dest_mode = (entry >> IOAPIC_LVT_DEST_MODE_SHIFT) & 1;
+    info->delivery_mode = (entry >> IOAPIC_LVT_DELIV_MODE_SHIFT) \
+        & IOAPIC_DM_MASK;
+    if (info->delivery_mode == IOAPIC_DM_EXTINT) {
+        info->vector = pic_read_irq(isa_pic);
+    } else {
+        info->vector = entry & IOAPIC_VECTOR_MASK;
+    }
+
+    info->addr = APIC_DEFAULT_ADDRESS | \
+        (info->dest_idx << MSI_ADDR_DEST_IDX_SHIFT) | \
+        (info->dest_mode << MSI_ADDR_DEST_MODE_SHIFT);
+    info->data = (info->vector << MSI_DATA_VECTOR_SHIFT) | \
+        (info->trig_mode << MSI_DATA_TRIGGER_SHIFT) | \
+        (info->delivery_mode << MSI_DATA_DELIVERY_MODE_SHIFT);
+}
+
 static void ioapic_service(IOAPICCommonState *s)
 {
+    AddressSpace *ioapic_as = PC_MACHINE(qdev_get_machine())->ioapic_as;
+    struct ioapic_entry_info info;
     uint8_t i;
-    uint8_t trig_mode;
-    uint8_t vector;
-    uint8_t delivery_mode;
     uint32_t mask;
     uint64_t entry;
-    uint8_t dest;
-    uint8_t dest_mode;
 
     for (i = 0; i < IOAPIC_NUM_PINS; i++) {
         mask = 1 << i;
         if (s->irr & mask) {
+            int coalesce = 0;
+
             entry = s->ioredtbl[i];
-            if (!(entry & IOAPIC_LVT_MASKED)) {
-                trig_mode = ((entry >> IOAPIC_LVT_TRIGGER_MODE_SHIFT) & 1);
-                dest = entry >> IOAPIC_LVT_DEST_SHIFT;
-                dest_mode = (entry >> IOAPIC_LVT_DEST_MODE_SHIFT) & 1;
-                delivery_mode =
-                    (entry >> IOAPIC_LVT_DELIV_MODE_SHIFT) & IOAPIC_DM_MASK;
-                if (trig_mode == IOAPIC_TRIGGER_EDGE) {
+            ioapic_entry_parse(entry, &info);
+            if (!info.masked) {
+                if (info.trig_mode == IOAPIC_TRIGGER_EDGE) {
                     s->irr &= ~mask;
                 } else {
+                    coalesce = s->ioredtbl[i] & IOAPIC_LVT_REMOTE_IRR;
                     s->ioredtbl[i] |= IOAPIC_LVT_REMOTE_IRR;
                 }
-                if (delivery_mode == IOAPIC_DM_EXTINT) {
-                    vector = pic_read_irq(isa_pic);
-                } else {
-                    vector = entry & IOAPIC_VECTOR_MASK;
+
+                if (coalesce) {
+                    /* We are level triggered interrupts, and the
+                     * guest should be still working on previous one,
+                     * so skip it. */
+                    continue;
                 }
-                apic_deliver_irq(dest, dest_mode, delivery_mode,
-                                 vector, trig_mode);
+
+#ifdef CONFIG_KVM
+                if (kvm_irqchip_is_split()) {
+                    if (info.trig_mode == IOAPIC_TRIGGER_EDGE) {
+                        kvm_set_irq(kvm_state, i, 1);
+                        kvm_set_irq(kvm_state, i, 0);
+                    } else {
+                        kvm_set_irq(kvm_state, i, 1);
+                    }
+                    continue;
+                }
+#endif
+
+                /* No matter whether IR is enabled, we translate
+                 * the IOAPIC message into a MSI one, and its
+                 * address space will decide whether we need a
+                 * translation. */
+                stl_le_phys(ioapic_as, info.addr, info.data);
             }
         }
     }
@@ -98,7 +168,9 @@
             /* level triggered */
             if (level) {
                 s->irr |= mask;
-                ioapic_service(s);
+                if (!(entry & IOAPIC_LVT_REMOTE_IRR)) {
+                    ioapic_service(s);
+                }
             } else {
                 s->irr &= ~mask;
             }
@@ -113,6 +185,35 @@
     }
 }
 
+static void ioapic_update_kvm_routes(IOAPICCommonState *s)
+{
+#ifdef CONFIG_KVM
+    int i;
+
+    if (kvm_irqchip_is_split()) {
+        for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+            MSIMessage msg;
+            struct ioapic_entry_info info;
+            ioapic_entry_parse(s->ioredtbl[i], &info);
+            msg.address = info.addr;
+            msg.data = info.data;
+            kvm_irqchip_update_msi_route(kvm_state, i, msg, NULL);
+        }
+        kvm_irqchip_commit_routes(kvm_state);
+    }
+#endif
+}
+
+#ifdef CONFIG_KVM
+static void ioapic_iec_notifier(void *private, bool global,
+                                uint32_t index, uint32_t mask)
+{
+    IOAPICCommonState *s = (IOAPICCommonState *)private;
+    /* For simplicity, we just update all the routes */
+    ioapic_update_kvm_routes(s);
+}
+#endif
+
 void ioapic_eoi_broadcast(int vector)
 {
     IOAPICCommonState *s;
@@ -137,6 +238,17 @@
     }
 }
 
+void ioapic_dump_state(Monitor *mon, const QDict *qdict)
+{
+    int i;
+
+    for (i = 0; i < MAX_IOAPICS; i++) {
+        if (ioapics[i] != 0) {
+            ioapic_print_redtbl(mon, ioapics[i]);
+        }
+    }
+}
+
 static uint64_t
 ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
 {
@@ -154,15 +266,13 @@
         }
         switch (s->ioregsel) {
         case IOAPIC_REG_ID:
+        case IOAPIC_REG_ARB:
             val = s->id << IOAPIC_ID_SHIFT;
             break;
         case IOAPIC_REG_VER:
-            val = IOAPIC_VERSION |
+            val = s->version |
                 ((IOAPIC_NUM_PINS - 1) << IOAPIC_VER_ENTRIES_SHIFT);
             break;
-        case IOAPIC_REG_ARB:
-            val = 0;
-            break;
         default:
             index = (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1;
             if (index >= 0 && index < IOAPIC_NUM_PINS) {
@@ -179,6 +289,34 @@
     return val;
 }
 
+/*
+ * This is to satisfy the hack in Linux kernel. One hack of it is to
+ * simulate clearing the Remote IRR bit of IOAPIC entry using the
+ * following:
+ *
+ * "For IO-APIC's with EOI register, we use that to do an explicit EOI.
+ * Otherwise, we simulate the EOI message manually by changing the trigger
+ * mode to edge and then back to level, with RTE being masked during
+ * this."
+ *
+ * (See linux kernel __eoi_ioapic_pin() comment in commit c0205701)
+ *
+ * This is based on the assumption that, Remote IRR bit will be
+ * cleared by IOAPIC hardware when configured as edge-triggered
+ * interrupts.
+ *
+ * Without this, level-triggered interrupts in IR mode might fail to
+ * work correctly.
+ */
+static inline void
+ioapic_fix_edge_remote_irr(uint64_t *entry)
+{
+    if (!(*entry & IOAPIC_LVT_TRIGGER_MODE)) {
+        /* Edge-triggered interrupts, make sure remote IRR is zero */
+        *entry &= ~((uint64_t)IOAPIC_LVT_REMOTE_IRR);
+    }
+}
+
 static void
 ioapic_mem_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned int size)
@@ -205,6 +343,7 @@
         default:
             index = (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1;
             if (index >= 0 && index < IOAPIC_NUM_PINS) {
+                uint64_t ro_bits = s->ioredtbl[index] & IOAPIC_RO_BITS;
                 if (s->ioregsel & 1) {
                     s->ioredtbl[index] &= 0xffffffff;
                     s->ioredtbl[index] |= (uint64_t)val << 32;
@@ -212,11 +351,24 @@
                     s->ioredtbl[index] &= ~0xffffffffULL;
                     s->ioredtbl[index] |= val;
                 }
+                /* restore RO bits */
+                s->ioredtbl[index] &= IOAPIC_RW_BITS;
+                s->ioredtbl[index] |= ro_bits;
+                ioapic_fix_edge_remote_irr(&s->ioredtbl[index]);
                 ioapic_service(s);
             }
         }
         break;
+    case IOAPIC_EOI:
+        /* Explicit EOI is only supported for IOAPIC version 0x20 */
+        if (size != 4 || s->version != 0x20) {
+            break;
+        }
+        ioapic_eoi_broadcast(val);
+        break;
     }
+
+    ioapic_update_kvm_routes(s);
 }
 
 static const MemoryRegionOps ioapic_io_ops = {
@@ -225,18 +377,49 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
+static void ioapic_machine_done_notify(Notifier *notifier, void *data)
+{
+#ifdef CONFIG_KVM
+    IOAPICCommonState *s = container_of(notifier, IOAPICCommonState,
+                                        machine_done);
+
+    if (kvm_irqchip_is_split()) {
+        X86IOMMUState *iommu = x86_iommu_get_default();
+        if (iommu) {
+            /* Register this IOAPIC with IOMMU IEC notifier, so that
+             * when there are IR invalidates, we can be notified to
+             * update kernel IR cache. */
+            x86_iommu_iec_register_notifier(iommu, ioapic_iec_notifier, s);
+        }
+    }
+#endif
+}
+
 static void ioapic_realize(DeviceState *dev, Error **errp)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
 
+    if (s->version != 0x11 && s->version != 0x20) {
+        error_report("IOAPIC only supports version 0x11 or 0x20 "
+                     "(default: 0x11).");
+        exit(1);
+    }
+
     memory_region_init_io(&s->io_memory, OBJECT(s), &ioapic_io_ops, s,
                           "ioapic", 0x1000);
 
     qdev_init_gpio_in(dev, ioapic_set_irq, IOAPIC_NUM_PINS);
 
     ioapics[ioapic_no] = s;
+    s->machine_done.notify = ioapic_machine_done_notify;
+    qemu_add_machine_init_done_notifier(&s->machine_done);
 }
 
+static Property ioapic_properties[] = {
+    DEFINE_PROP_UINT8("version", IOAPICCommonState, version, 0x11),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void ioapic_class_init(ObjectClass *klass, void *data)
 {
     IOAPICCommonClass *k = IOAPIC_COMMON_CLASS(klass);
@@ -244,6 +427,7 @@
 
     k->realize = ioapic_realize;
     dc->reset = ioapic_reset_common;
+    dc->props = ioapic_properties;
 }
 
 static const TypeInfo ioapic_info = {
diff --git a/hw/intc/ioapic_common.c b/hw/intc/ioapic_common.c
index 8b7d118..1b7ec5e 100644
--- a/hw/intc/ioapic_common.c
+++ b/hw/intc/ioapic_common.c
@@ -19,6 +19,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "monitor/monitor.h"
 #include "hw/i386/ioapic.h"
 #include "hw/i386/ioapic_internal.h"
 #include "hw/sysbus.h"
@@ -31,6 +34,60 @@
  */
 int ioapic_no;
 
+static void ioapic_irr_dump(Monitor *mon, const char *name, uint32_t bitmap)
+{
+    int i;
+
+    monitor_printf(mon, "%-10s ", name);
+    if (bitmap == 0) {
+        monitor_printf(mon, "(none)\n");
+        return;
+    }
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        if (bitmap & (1 << i)) {
+            monitor_printf(mon, "%-2u ", i);
+        }
+    }
+    monitor_printf(mon, "\n");
+}
+
+void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s)
+{
+    static const char *delm_str[] = {
+        "fixed", "lowest", "SMI", "...", "NMI", "INIT", "...", "extINT"};
+    uint32_t remote_irr = 0;
+    int i;
+
+    monitor_printf(mon, "ioapic id=0x%02x sel=0x%02x", s->id, s->ioregsel);
+    if (s->ioregsel) {
+        monitor_printf(mon, " (redir[%u])\n",
+                       (s->ioregsel - IOAPIC_REG_REDTBL_BASE) >> 1);
+    } else {
+        monitor_printf(mon, "\n");
+    }
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        uint64_t entry = s->ioredtbl[i];
+        uint32_t delm = (uint32_t)((entry & IOAPIC_LVT_DELIV_MODE) >>
+                                   IOAPIC_LVT_DELIV_MODE_SHIFT);
+        monitor_printf(mon, "pin %-2u 0x%016"PRIx64" dest=%"PRIx64
+                       " vec=%-3"PRIu64" %s %-5s %-6s %-6s %s\n",
+                       i, entry,
+                       (entry >> IOAPIC_LVT_DEST_SHIFT) &
+                            (entry & IOAPIC_LVT_DEST_MODE ? 0xff : 0xf),
+                       entry & IOAPIC_VECTOR_MASK,
+                       entry & IOAPIC_LVT_POLARITY ? "active-lo" : "active-hi",
+                       entry & IOAPIC_LVT_TRIGGER_MODE ? "level" : "edge",
+                       entry & IOAPIC_LVT_MASKED ? "masked" : "",
+                       delm_str[delm],
+                       entry & IOAPIC_LVT_DEST_MODE ? "logical" : "physical");
+
+        remote_irr |= entry & IOAPIC_LVT_TRIGGER_MODE ?
+                        (entry & IOAPIC_LVT_REMOTE_IRR ? (1 << i) : 0) : 0;
+    }
+    ioapic_irr_dump(mon, "IRR", s->irr);
+    ioapic_irr_dump(mon, "Remote IRR", remote_irr);
+}
+
 void ioapic_reset_common(DeviceState *dev)
 {
     IOAPICCommonState *s = IOAPIC_COMMON(dev);
diff --git a/hw/intc/lm32_pic.c b/hw/intc/lm32_pic.c
index 72fc9ef..3dad01c 100644
--- a/hw/intc/lm32_pic.c
+++ b/hw/intc/lm32_pic.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
@@ -43,7 +43,7 @@
 typedef struct LM32PicState LM32PicState;
 
 static LM32PicState *pic;
-void lm32_do_pic_info(Monitor *mon, const QDict *qdict)
+void lm32_hmp_info_pic(Monitor *mon, const QDict *qdict)
 {
     if (pic == NULL) {
         return;
@@ -53,7 +53,7 @@
             pic->im, pic->ip, pic->irq_state);
 }
 
-void lm32_irq_info(Monitor *mon, const QDict *qdict)
+void lm32_hmp_info_irq(Monitor *mon, const QDict *qdict)
 {
     int i;
     uint32_t count;
@@ -152,17 +152,16 @@
     }
 }
 
-static int lm32_pic_init(SysBusDevice *sbd)
+static void lm32_pic_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    LM32PicState *s = LM32_PIC(dev);
+    DeviceState *dev = DEVICE(obj);
+    LM32PicState *s = LM32_PIC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     qdev_init_gpio_in(dev, irq_handler, 32);
     sysbus_init_irq(sbd, &s->parent_irq);
 
     pic = s;
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_lm32_pic = {
@@ -181,9 +180,7 @@
 static void lm32_pic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = lm32_pic_init;
     dc->reset = pic_reset;
     dc->vmsd = &vmstate_lm32_pic;
 }
@@ -192,6 +189,7 @@
     .name          = TYPE_LM32_PIC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32PicState),
+    .instance_init = lm32_pic_init,
     .class_init    = lm32_pic_class_init,
 };
 
diff --git a/hw/intc/mips_gic.c b/hw/intc/mips_gic.c
new file mode 100644
index 0000000..6e25773
--- /dev/null
+++ b/hw/intc/mips_gic.c
@@ -0,0 +1,460 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
+ * Authors: Sanjay Lal <sanjayl@kymasys.com>
+ *
+ * Copyright (C) 2016 Imagination Technologies
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "exec/memory.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
+#include "kvm_mips.h"
+#include "hw/intc/mips_gic.h"
+
+static void mips_gic_set_vp_irq(MIPSGICState *gic, int vp, int pin, int level)
+{
+    int ored_level = level;
+    int i;
+
+    /* ORing pending registers sharing same pin */
+    if (!ored_level) {
+        for (i = 0; i < gic->num_irq; i++) {
+            if ((gic->irq_state[i].map_pin & GIC_MAP_MSK) == pin &&
+                    gic->irq_state[i].map_vp == vp &&
+                    gic->irq_state[i].enabled) {
+                ored_level |= gic->irq_state[i].pending;
+            }
+            if (ored_level) {
+                /* no need to iterate all interrupts */
+                break;
+            }
+        }
+        if (((gic->vps[vp].compare_map & GIC_MAP_MSK) == pin) &&
+                (gic->vps[vp].mask & GIC_VP_MASK_CMP_MSK)) {
+            /* ORing with local pending register (count/compare) */
+            ored_level |= (gic->vps[vp].pend & GIC_VP_MASK_CMP_MSK) >>
+                          GIC_VP_MASK_CMP_SHF;
+        }
+    }
+    if (kvm_enabled())  {
+        kvm_mips_set_ipi_interrupt(mips_env_get_cpu(gic->vps[vp].env),
+                                   pin + GIC_CPU_PIN_OFFSET,
+                                   ored_level);
+    } else {
+        qemu_set_irq(gic->vps[vp].env->irq[pin + GIC_CPU_PIN_OFFSET],
+                     ored_level);
+    }
+}
+
+static void gic_set_irq(void *opaque, int n_IRQ, int level)
+{
+    MIPSGICState *gic = (MIPSGICState *) opaque;
+    int vp = gic->irq_state[n_IRQ].map_vp;
+    int pin = gic->irq_state[n_IRQ].map_pin & GIC_MAP_MSK;
+
+    gic->irq_state[n_IRQ].pending = (uint8_t) level;
+    if (!gic->irq_state[n_IRQ].enabled) {
+        /* GIC interrupt source disabled */
+        return;
+    }
+    if (vp < 0 || vp >= gic->num_vps) {
+        return;
+    }
+    mips_gic_set_vp_irq(gic, vp, pin, level);
+}
+
+#define OFFSET_CHECK(c)                         \
+    do {                                        \
+        if (!(c)) {                             \
+            goto bad_offset;                    \
+        }                                       \
+    } while (0)
+
+/* GIC Read VP Local/Other Registers */
+static uint64_t gic_read_vp(MIPSGICState *gic, uint32_t vp_index, hwaddr addr,
+                            unsigned size)
+{
+    switch (addr) {
+    case GIC_VP_CTL_OFS:
+        return gic->vps[vp_index].ctl;
+    case GIC_VP_PEND_OFS:
+        mips_gictimer_get_sh_count(gic->gic_timer);
+        return gic->vps[vp_index].pend;
+    case GIC_VP_MASK_OFS:
+        return gic->vps[vp_index].mask;
+    case GIC_VP_COMPARE_MAP_OFS:
+        return gic->vps[vp_index].compare_map;
+    case GIC_VP_OTHER_ADDR_OFS:
+        return gic->vps[vp_index].other_addr;
+    case GIC_VP_IDENT_OFS:
+        return vp_index;
+    case GIC_VP_COMPARE_LO_OFS:
+        return mips_gictimer_get_vp_compare(gic->gic_timer, vp_index);
+    case GIC_VP_COMPARE_HI_OFS:
+        return 0;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Read %d bytes at GIC offset LOCAL/OTHER 0x%"
+                      PRIx64 "\n", size, addr);
+        break;
+    }
+    return 0;
+}
+
+static uint64_t gic_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSGICState *gic = (MIPSGICState *) opaque;
+    uint32_t vp_index = current_cpu->cpu_index;
+    uint64_t ret = 0;
+    int i, base, irq_src;
+    uint32_t other_index;
+
+    switch (addr) {
+    case GIC_SH_CONFIG_OFS:
+        ret = gic->sh_config | (mips_gictimer_get_countstop(gic->gic_timer) <<
+                               GIC_SH_CONFIG_COUNTSTOP_SHF);
+        break;
+    case GIC_SH_COUNTERLO_OFS:
+        ret = mips_gictimer_get_sh_count(gic->gic_timer);
+        break;
+    case GIC_SH_COUNTERHI_OFS:
+        ret = 0;
+        break;
+    case GIC_SH_PEND_OFS ... GIC_SH_PEND_LAST_OFS:
+        /* each bit represents pending status for an interrupt pin */
+        base = (addr - GIC_SH_PEND_OFS) * 8;
+        OFFSET_CHECK((base + size * 8) <= gic->num_irq);
+        for (i = 0; i < size * 8; i++) {
+            ret |= (uint64_t) (gic->irq_state[base + i].pending) << i;
+        }
+        break;
+    case GIC_SH_MASK_OFS ... GIC_SH_MASK_LAST_OFS:
+        /* each bit represents status for an interrupt pin */
+        base = (addr - GIC_SH_MASK_OFS) * 8;
+        OFFSET_CHECK((base + size * 8) <= gic->num_irq);
+        for (i = 0; i < size * 8; i++) {
+            ret |= (uint64_t) (gic->irq_state[base + i].enabled) << i;
+        }
+        break;
+    case GIC_SH_MAP0_PIN_OFS ... GIC_SH_MAP255_PIN_OFS:
+        /* 32 bits per a pin */
+        irq_src = (addr - GIC_SH_MAP0_PIN_OFS) / 4;
+        OFFSET_CHECK(irq_src < gic->num_irq);
+        ret = gic->irq_state[irq_src].map_pin;
+        break;
+    case GIC_SH_MAP0_VP_OFS ... GIC_SH_MAP255_VP_LAST_OFS:
+        /* up to 32 bytes per a pin */
+        irq_src = (addr - GIC_SH_MAP0_VP_OFS) / 32;
+        OFFSET_CHECK(irq_src < gic->num_irq);
+        if ((gic->irq_state[irq_src].map_vp) >= 0) {
+            ret = (uint64_t) 1 << (gic->irq_state[irq_src].map_vp);
+        } else {
+            ret = 0;
+        }
+        break;
+    /* VP-Local Register */
+    case VP_LOCAL_SECTION_OFS ... (VP_LOCAL_SECTION_OFS + GIC_VL_BRK_GROUP):
+        ret = gic_read_vp(gic, vp_index, addr - VP_LOCAL_SECTION_OFS, size);
+        break;
+    /* VP-Other Register */
+    case VP_OTHER_SECTION_OFS ... (VP_OTHER_SECTION_OFS + GIC_VL_BRK_GROUP):
+        other_index = gic->vps[vp_index].other_addr;
+        ret = gic_read_vp(gic, other_index, addr - VP_OTHER_SECTION_OFS, size);
+        break;
+    /* User-Mode Visible section */
+    case USM_VISIBLE_SECTION_OFS + GIC_USER_MODE_COUNTERLO:
+        ret = mips_gictimer_get_sh_count(gic->gic_timer);
+        break;
+    case USM_VISIBLE_SECTION_OFS + GIC_USER_MODE_COUNTERHI:
+        ret = 0;
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Read %d bytes at GIC offset 0x%" PRIx64 "\n",
+                      size, addr);
+        break;
+    }
+    return ret;
+bad_offset:
+    qemu_log_mask(LOG_GUEST_ERROR, "Wrong GIC offset at 0x%" PRIx64 "\n", addr);
+    return 0;
+}
+
+static void gic_timer_expire_cb(void *opaque, uint32_t vp_index)
+{
+    MIPSGICState *gic = opaque;
+
+    gic->vps[vp_index].pend |= (1 << GIC_LOCAL_INT_COMPARE);
+    if (gic->vps[vp_index].pend &
+            (gic->vps[vp_index].mask & GIC_VP_MASK_CMP_MSK)) {
+        if (gic->vps[vp_index].compare_map & GIC_MAP_TO_PIN_MSK) {
+            /* it is safe to set the irq high regardless of other GIC IRQs */
+            uint32_t pin = (gic->vps[vp_index].compare_map & GIC_MAP_MSK);
+            qemu_irq_raise(gic->vps[vp_index].env->irq
+                           [pin + GIC_CPU_PIN_OFFSET]);
+        }
+    }
+}
+
+static void gic_timer_store_vp_compare(MIPSGICState *gic, uint32_t vp_index,
+                                       uint64_t compare)
+{
+    gic->vps[vp_index].pend &= ~(1 << GIC_LOCAL_INT_COMPARE);
+    if (gic->vps[vp_index].compare_map & GIC_MAP_TO_PIN_MSK) {
+        uint32_t pin = (gic->vps[vp_index].compare_map & GIC_MAP_MSK);
+        mips_gic_set_vp_irq(gic, vp_index, pin, 0);
+    }
+    mips_gictimer_store_vp_compare(gic->gic_timer, vp_index, compare);
+}
+
+/* GIC Write VP Local/Other Registers */
+static void gic_write_vp(MIPSGICState *gic, uint32_t vp_index, hwaddr addr,
+                              uint64_t data, unsigned size)
+{
+    switch (addr) {
+    case GIC_VP_CTL_OFS:
+        /* EIC isn't supported */
+        break;
+    case GIC_VP_RMASK_OFS:
+        gic->vps[vp_index].mask &= ~(data & GIC_VP_SET_RESET_MSK) &
+                                   GIC_VP_SET_RESET_MSK;
+        break;
+    case GIC_VP_SMASK_OFS:
+        gic->vps[vp_index].mask |= data & GIC_VP_SET_RESET_MSK;
+        break;
+    case GIC_VP_COMPARE_MAP_OFS:
+        /* EIC isn't supported */
+        OFFSET_CHECK((data & GIC_MAP_MSK) <= GIC_CPU_INT_MAX);
+        gic->vps[vp_index].compare_map = data & GIC_MAP_TO_PIN_REG_MSK;
+        break;
+    case GIC_VP_OTHER_ADDR_OFS:
+        OFFSET_CHECK(data < gic->num_vps);
+        gic->vps[vp_index].other_addr = data;
+        break;
+    case GIC_VP_COMPARE_LO_OFS:
+        gic_timer_store_vp_compare(gic, vp_index, data);
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Write %d bytes at GIC offset LOCAL/OTHER "
+                      "0x%" PRIx64" 0x%08" PRIx64 "\n", size, addr, data);
+        break;
+    }
+    return;
+bad_offset:
+    qemu_log_mask(LOG_GUEST_ERROR, "Wrong GIC offset at 0x%" PRIx64 "\n", addr);
+    return;
+}
+
+static void gic_write(void *opaque, hwaddr addr, uint64_t data, unsigned size)
+{
+    int intr;
+    MIPSGICState *gic = (MIPSGICState *) opaque;
+    uint32_t vp_index = current_cpu->cpu_index;
+    int i, base, irq_src;
+    uint32_t other_index;
+
+    switch (addr) {
+    case GIC_SH_CONFIG_OFS:
+        {
+            uint32_t pre_cntstop = mips_gictimer_get_countstop(gic->gic_timer);
+            uint32_t new_cntstop = (data & GIC_SH_CONFIG_COUNTSTOP_MSK) >>
+                                   GIC_SH_CONFIG_COUNTSTOP_SHF;
+            if (pre_cntstop != new_cntstop) {
+                if (new_cntstop == 1) {
+                    mips_gictimer_stop_count(gic->gic_timer);
+                } else {
+                    mips_gictimer_start_count(gic->gic_timer);
+                }
+            }
+        }
+        break;
+    case GIC_SH_COUNTERLO_OFS:
+        if (mips_gictimer_get_countstop(gic->gic_timer)) {
+            mips_gictimer_store_sh_count(gic->gic_timer, data);
+        }
+        break;
+    case GIC_SH_RMASK_OFS ... GIC_SH_RMASK_LAST_OFS:
+        /* up to 64 bits per a pin */
+        base = (addr - GIC_SH_RMASK_OFS) * 8;
+        OFFSET_CHECK((base + size * 8) <= gic->num_irq);
+        for (i = 0; i < size * 8; i++) {
+            gic->irq_state[base + i].enabled &= !((data >> i) & 1);
+        }
+        break;
+    case GIC_SH_WEDGE_OFS:
+        /* Figure out which VP/HW Interrupt this maps to */
+        intr = data & ~GIC_SH_WEDGE_RW_MSK;
+        /* Mask/Enabled Checks */
+        OFFSET_CHECK(intr < gic->num_irq);
+        if (data & GIC_SH_WEDGE_RW_MSK) {
+            gic_set_irq(gic, intr, 1);
+        } else {
+            gic_set_irq(gic, intr, 0);
+        }
+        break;
+    case GIC_SH_SMASK_OFS ... GIC_SH_SMASK_LAST_OFS:
+        /* up to 64 bits per a pin */
+        base = (addr - GIC_SH_SMASK_OFS) * 8;
+        OFFSET_CHECK((base + size * 8) <= gic->num_irq);
+        for (i = 0; i < size * 8; i++) {
+            gic->irq_state[base + i].enabled |= (data >> i) & 1;
+        }
+        break;
+    case GIC_SH_MAP0_PIN_OFS ... GIC_SH_MAP255_PIN_OFS:
+        /* 32 bits per a pin */
+        irq_src = (addr - GIC_SH_MAP0_PIN_OFS) / 4;
+        OFFSET_CHECK(irq_src < gic->num_irq);
+        /* EIC isn't supported */
+        OFFSET_CHECK((data & GIC_MAP_MSK) <= GIC_CPU_INT_MAX);
+        gic->irq_state[irq_src].map_pin = data & GIC_MAP_TO_PIN_REG_MSK;
+        break;
+    case GIC_SH_MAP0_VP_OFS ... GIC_SH_MAP255_VP_LAST_OFS:
+        /* up to 32 bytes per a pin */
+        irq_src = (addr - GIC_SH_MAP0_VP_OFS) / 32;
+        OFFSET_CHECK(irq_src < gic->num_irq);
+        data = data ? ctz64(data) : -1;
+        OFFSET_CHECK(data < gic->num_vps);
+        gic->irq_state[irq_src].map_vp = data;
+        break;
+    case VP_LOCAL_SECTION_OFS ... (VP_LOCAL_SECTION_OFS + GIC_VL_BRK_GROUP):
+        gic_write_vp(gic, vp_index, addr - VP_LOCAL_SECTION_OFS, data, size);
+        break;
+    case VP_OTHER_SECTION_OFS ... (VP_OTHER_SECTION_OFS + GIC_VL_BRK_GROUP):
+        other_index = gic->vps[vp_index].other_addr;
+        gic_write_vp(gic, other_index, addr - VP_OTHER_SECTION_OFS, data, size);
+        break;
+    case USM_VISIBLE_SECTION_OFS + GIC_USER_MODE_COUNTERLO:
+    case USM_VISIBLE_SECTION_OFS + GIC_USER_MODE_COUNTERHI:
+        /* do nothing. Read-only section */
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Write %d bytes at GIC offset 0x%" PRIx64
+                      " 0x%08" PRIx64 "\n", size, addr, data);
+        break;
+    }
+    return;
+bad_offset:
+    qemu_log_mask(LOG_GUEST_ERROR, "Wrong GIC offset at 0x%" PRIx64 "\n", addr);
+}
+
+static void gic_reset(void *opaque)
+{
+    int i;
+    MIPSGICState *gic = (MIPSGICState *) opaque;
+    int numintrs = (gic->num_irq / 8) - 1;
+
+    gic->sh_config = /* COUNTSTOP = 0 it is accessible via MIPSGICTimer*/
+                     /* CounterHi not implemented */
+                     (0            << GIC_SH_CONFIG_COUNTBITS_SHF) |
+                     (numintrs     << GIC_SH_CONFIG_NUMINTRS_SHF)  |
+                     (gic->num_vps << GIC_SH_CONFIG_PVPS_SHF);
+    for (i = 0; i < gic->num_vps; i++) {
+        gic->vps[i].ctl         = 0x0;
+        gic->vps[i].pend        = 0x0;
+        /* PERFCNT, TIMER and WD not implemented */
+        gic->vps[i].mask        = 0x32;
+        gic->vps[i].compare_map = GIC_MAP_TO_PIN_MSK;
+        mips_gictimer_store_vp_compare(gic->gic_timer, i, 0xffffffff);
+        gic->vps[i].other_addr  = 0x0;
+    }
+    for (i = 0; i < gic->num_irq; i++) {
+        gic->irq_state[i].enabled = 0;
+        gic->irq_state[i].pending = 0;
+        gic->irq_state[i].map_pin = GIC_MAP_TO_PIN_MSK;
+        gic->irq_state[i].map_vp  = -1;
+    }
+    mips_gictimer_store_sh_count(gic->gic_timer, 0);
+    /* COUNTSTOP = 0 */
+    mips_gictimer_start_count(gic->gic_timer);
+}
+
+static const MemoryRegionOps gic_ops = {
+    .read = gic_read,
+    .write = gic_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .max_access_size = 8,
+    },
+};
+
+static void mips_gic_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSGICState *s = MIPS_GIC(obj);
+
+    memory_region_init_io(&s->mr, OBJECT(s), &gic_ops, s,
+                          "mips-gic", GIC_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->mr);
+    qemu_register_reset(gic_reset, s);
+}
+
+static void mips_gic_realize(DeviceState *dev, Error **errp)
+{
+    MIPSGICState *s = MIPS_GIC(dev);
+    CPUState *cs = first_cpu;
+    int i;
+
+    if (s->num_vps > GIC_MAX_VPS) {
+        error_setg(errp, "Exceeded maximum CPUs %d", s->num_vps);
+        return;
+    }
+    if ((s->num_irq > GIC_MAX_INTRS) || (s->num_irq % 8) || (s->num_irq <= 0)) {
+        error_setg(errp, "GIC supports up to %d external interrupts in "
+                   "multiples of 8 : %d", GIC_MAX_INTRS, s->num_irq);
+        return;
+    }
+    s->vps = g_new(MIPSGICVPState, s->num_vps);
+    s->irq_state = g_new(MIPSGICIRQState, s->num_irq);
+    /* Register the env for all VPs with the GIC */
+    for (i = 0; i < s->num_vps; i++) {
+        if (cs != NULL) {
+            s->vps[i].env = cs->env_ptr;
+            cs = CPU_NEXT(cs);
+        } else {
+            error_setg(errp,
+               "Unable to initialize GIC, CPUState for CPU#%d not valid.", i);
+            return;
+        }
+    }
+    s->gic_timer = mips_gictimer_init(s, s->num_vps, gic_timer_expire_cb);
+    qdev_init_gpio_in(dev, gic_set_irq, s->num_irq);
+    for (i = 0; i < s->num_irq; i++) {
+        s->irq_state[i].irq = qdev_get_gpio_in(dev, i);
+    }
+}
+
+static Property mips_gic_properties[] = {
+    DEFINE_PROP_INT32("num-vp", MIPSGICState, num_vps, 1),
+    DEFINE_PROP_INT32("num-irq", MIPSGICState, num_irq, 256),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_gic_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = mips_gic_properties;
+    dc->realize = mips_gic_realize;
+}
+
+static const TypeInfo mips_gic_info = {
+    .name          = TYPE_MIPS_GIC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSGICState),
+    .instance_init = mips_gic_init,
+    .class_init    = mips_gic_class_init,
+};
+
+static void mips_gic_register_types(void)
+{
+    type_register_static(&mips_gic_info);
+}
+
+type_init(mips_gic_register_types)
diff --git a/hw/intc/omap_intc.c b/hw/intc/omap_intc.c
index ad3931c..877be67 100644
--- a/hw/intc/omap_intc.c
+++ b/hw/intc/omap_intc.c
@@ -17,9 +17,12 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 #include "hw/sysbus.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
 
 /* Interrupt Handlers */
 struct omap_intr_handler_bank_s {
@@ -60,7 +63,7 @@
 
 static void omap_inth_sir_update(struct omap_intr_handler_s *s, int is_fiq)
 {
-    int i, j, sir_intr, p_intr, p, f;
+    int i, j, sir_intr, p_intr, p;
     uint32_t level;
     sir_intr = 0;
     p_intr = 255;
@@ -72,14 +75,15 @@
     for (j = 0; j < s->nbanks; ++j) {
         level = s->bank[j].irqs & ~s->bank[j].mask &
                 (is_fiq ? s->bank[j].fiq : ~s->bank[j].fiq);
-        for (f = ffs(level), i = f - 1, level >>= f - 1; f; i += f,
-                        level >>= f) {
+
+        while (level != 0) {
+            i = ctz32(level);
             p = s->bank[j].priority[i];
             if (p <= p_intr) {
                 p_intr = p;
                 sir_intr = 32 * j + i;
             }
-            f = ffs(level >> 1);
+            level &= level - 1;
         }
     }
     s->sir_intr[is_fiq] = sir_intr;
@@ -360,22 +364,28 @@
     qemu_set_irq(s->parent_intr[1], 0);
 }
 
-static int omap_intc_init(SysBusDevice *sbd)
+static void omap_intc_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    struct omap_intr_handler_s *s = OMAP_INTC(dev);
+    DeviceState *dev = DEVICE(obj);
+    struct omap_intr_handler_s *s = OMAP_INTC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    if (!s->iclk) {
-        hw_error("omap-intc: clk not connected\n");
-    }
     s->nbanks = 1;
     sysbus_init_irq(sbd, &s->parent_intr[0]);
     sysbus_init_irq(sbd, &s->parent_intr[1]);
     qdev_init_gpio_in(dev, omap_set_intr, s->nbanks * 32);
-    memory_region_init_io(&s->mmio, OBJECT(s), &omap_inth_mem_ops, s,
+    memory_region_init_io(&s->mmio, obj, &omap_inth_mem_ops, s,
                           "omap-intc", s->size);
     sysbus_init_mmio(sbd, &s->mmio);
-    return 0;
+}
+
+static void omap_intc_realize(DeviceState *dev, Error **errp)
+{
+    struct omap_intr_handler_s *s = OMAP_INTC(dev);
+
+    if (!s->iclk) {
+        error_setg(errp, "omap-intc: clk not connected");
+    }
 }
 
 static Property omap_intc_properties[] = {
@@ -387,18 +397,18 @@
 static void omap_intc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = omap_intc_init;
     dc->reset = omap_inth_reset;
     dc->props = omap_intc_properties;
     /* Reason: pointer property "clk" */
     dc->cannot_instantiate_with_device_add_yet = true;
+    dc->realize = omap_intc_realize;
 }
 
 static const TypeInfo omap_intc_info = {
     .name          = "omap-intc",
     .parent        = TYPE_OMAP_INTC,
+    .instance_init = omap_intc_init,
     .class_init    = omap_intc_class_init,
 };
 
@@ -601,26 +611,34 @@
     },
 };
 
-static int omap2_intc_init(SysBusDevice *sbd)
+static void omap2_intc_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    struct omap_intr_handler_s *s = OMAP_INTC(dev);
+    DeviceState *dev = DEVICE(obj);
+    struct omap_intr_handler_s *s = OMAP_INTC(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    if (!s->iclk) {
-        hw_error("omap2-intc: iclk not connected\n");
-    }
-    if (!s->fclk) {
-        hw_error("omap2-intc: fclk not connected\n");
-    }
     s->level_only = 1;
     s->nbanks = 3;
     sysbus_init_irq(sbd, &s->parent_intr[0]);
     sysbus_init_irq(sbd, &s->parent_intr[1]);
     qdev_init_gpio_in(dev, omap_set_intr_noedge, s->nbanks * 32);
-    memory_region_init_io(&s->mmio, OBJECT(s), &omap2_inth_mem_ops, s,
+    memory_region_init_io(&s->mmio, obj, &omap2_inth_mem_ops, s,
                           "omap2-intc", 0x1000);
     sysbus_init_mmio(sbd, &s->mmio);
-    return 0;
+}
+
+static void omap2_intc_realize(DeviceState *dev, Error **errp)
+{
+    struct omap_intr_handler_s *s = OMAP_INTC(dev);
+
+    if (!s->iclk) {
+        error_setg(errp, "omap2-intc: iclk not connected");
+        return;
+    }
+    if (!s->fclk) {
+        error_setg(errp, "omap2-intc: fclk not connected");
+        return;
+    }
 }
 
 static Property omap2_intc_properties[] = {
@@ -634,18 +652,18 @@
 static void omap2_intc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = omap2_intc_init;
     dc->reset = omap_inth_reset;
     dc->props = omap2_intc_properties;
     /* Reason: pointer property "iclk", "fclk" */
     dc->cannot_instantiate_with_device_add_yet = true;
+    dc->realize = omap2_intc_realize;
 }
 
 static const TypeInfo omap2_intc_info = {
     .name          = "omap2-intc",
     .parent        = TYPE_OMAP_INTC,
+    .instance_init = omap2_intc_init,
     .class_init    = omap2_intc_class_init,
 };
 
diff --git a/hw/intc/openpic.c b/hw/intc/openpic.c
index 7d1f3b9..4349e45 100644
--- a/hw/intc/openpic.c
+++ b/hw/intc/openpic.c
@@ -33,6 +33,7 @@
  * Serial interrupts, as implemented in Raven chipset are not supported yet.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -40,8 +41,10 @@
 #include "hw/ppc/ppc_e500.h"
 #include "hw/sysbus.h"
 #include "hw/pci/msi.h"
+#include "qapi/error.h"
 #include "qemu/bitops.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/log.h"
 
 //#define DEBUG_OPENPIC
 
@@ -200,11 +203,14 @@
     IRQ_TYPE_FSLSPECIAL,    /* FSL timer/IPI interrupt, edge, no polarity */
 } IRQType;
 
+/* Round up to the nearest 64 IRQs so that the queue length
+ * won't change when moving between 32 and 64 bit hosts.
+ */
+#define IRQQUEUE_SIZE_BITS ((OPENPIC_MAX_IRQ + 63) & ~63)
+
 typedef struct IRQQueue {
-    /* Round up to the nearest 64 IRQs so that the queue length
-     * won't change when moving between 32 and 64 bit hosts.
-     */
-    unsigned long queue[BITS_TO_LONGS((OPENPIC_MAX_IRQ + 63) & ~63)];
+    unsigned long *queue;
+    int32_t queue_size; /* Only used for VMSTATE_BITMAP */
     int next;
     int priority;
 } IRQQueue;
@@ -240,6 +246,15 @@
 #define IDR_EP      0x80000000  /* external pin */
 #define IDR_CI      0x40000000  /* critical interrupt */
 
+typedef struct OpenPICTimer {
+    uint32_t tccr;  /* Global timer current count register */
+    uint32_t tbcr;  /* Global timer base count register */
+} OpenPICTimer;
+
+typedef struct OpenPICMSI {
+    uint32_t msir;   /* Shared Message Signaled Interrupt Register */
+} OpenPICMSI;
+
 typedef struct IRQDest {
     int32_t ctpr; /* CPU current task priority */
     IRQQueue raised;
@@ -288,14 +303,9 @@
     IRQDest dst[MAX_CPU];
     uint32_t nb_cpus;
     /* Timer registers */
-    struct {
-        uint32_t tccr;  /* Global timer current count register */
-        uint32_t tbcr;  /* Global timer base count register */
-    } timers[OPENPIC_MAX_TMR];
+    OpenPICTimer timers[OPENPIC_MAX_TMR];
     /* Shared MSI registers */
-    struct {
-        uint32_t msir;   /* Shared Message Signaled Interrupt Register */
-    } msi[MAX_MSI];
+    OpenPICMSI msi[MAX_MSI];
     uint32_t max_irq;
     uint32_t irq_ipi0;
     uint32_t irq_tim0;
@@ -1013,7 +1023,7 @@
     DPRINTF("%s: cpu %d addr %#" HWADDR_PRIx " <= 0x%08x\n", __func__, idx,
             addr, val);
 
-    if (idx < 0) {
+    if (idx < 0 || idx >= opp->nb_cpus) {
         return;
     }
 
@@ -1152,7 +1162,7 @@
     DPRINTF("%s: cpu %d addr %#" HWADDR_PRIx "\n", __func__, idx, addr);
     retval = 0xFFFFFFFF;
 
-    if (idx < 0) {
+    if (idx < 0 || idx >= opp->nb_cpus) {
         return retval;
     }
 
@@ -1287,132 +1297,6 @@
     },
 };
 
-static void openpic_save_IRQ_queue(QEMUFile* f, IRQQueue *q)
-{
-    unsigned int i;
-
-    for (i = 0; i < ARRAY_SIZE(q->queue); i++) {
-        /* Always put the lower half of a 64-bit long first, in case we
-         * restore on a 32-bit host.  The least significant bits correspond
-         * to lower IRQ numbers in the bitmap.
-         */
-        qemu_put_be32(f, (uint32_t)q->queue[i]);
-#if LONG_MAX > 0x7FFFFFFF
-        qemu_put_be32(f, (uint32_t)(q->queue[i] >> 32));
-#endif
-    }
-
-    qemu_put_sbe32s(f, &q->next);
-    qemu_put_sbe32s(f, &q->priority);
-}
-
-static void openpic_save(QEMUFile* f, void *opaque)
-{
-    OpenPICState *opp = (OpenPICState *)opaque;
-    unsigned int i;
-
-    qemu_put_be32s(f, &opp->gcr);
-    qemu_put_be32s(f, &opp->vir);
-    qemu_put_be32s(f, &opp->pir);
-    qemu_put_be32s(f, &opp->spve);
-    qemu_put_be32s(f, &opp->tfrr);
-
-    qemu_put_be32s(f, &opp->nb_cpus);
-
-    for (i = 0; i < opp->nb_cpus; i++) {
-        qemu_put_sbe32s(f, &opp->dst[i].ctpr);
-        openpic_save_IRQ_queue(f, &opp->dst[i].raised);
-        openpic_save_IRQ_queue(f, &opp->dst[i].servicing);
-        qemu_put_buffer(f, (uint8_t *)&opp->dst[i].outputs_active,
-                        sizeof(opp->dst[i].outputs_active));
-    }
-
-    for (i = 0; i < OPENPIC_MAX_TMR; i++) {
-        qemu_put_be32s(f, &opp->timers[i].tccr);
-        qemu_put_be32s(f, &opp->timers[i].tbcr);
-    }
-
-    for (i = 0; i < opp->max_irq; i++) {
-        qemu_put_be32s(f, &opp->src[i].ivpr);
-        qemu_put_be32s(f, &opp->src[i].idr);
-        qemu_get_be32s(f, &opp->src[i].destmask);
-        qemu_put_sbe32s(f, &opp->src[i].last_cpu);
-        qemu_put_sbe32s(f, &opp->src[i].pending);
-    }
-}
-
-static void openpic_load_IRQ_queue(QEMUFile* f, IRQQueue *q)
-{
-    unsigned int i;
-
-    for (i = 0; i < ARRAY_SIZE(q->queue); i++) {
-        unsigned long val;
-
-        val = qemu_get_be32(f);
-#if LONG_MAX > 0x7FFFFFFF
-        val <<= 32;
-        val |= qemu_get_be32(f);
-#endif
-
-        q->queue[i] = val;
-    }
-
-    qemu_get_sbe32s(f, &q->next);
-    qemu_get_sbe32s(f, &q->priority);
-}
-
-static int openpic_load(QEMUFile* f, void *opaque, int version_id)
-{
-    OpenPICState *opp = (OpenPICState *)opaque;
-    unsigned int i, nb_cpus;
-
-    if (version_id != 1) {
-        return -EINVAL;
-    }
-
-    qemu_get_be32s(f, &opp->gcr);
-    qemu_get_be32s(f, &opp->vir);
-    qemu_get_be32s(f, &opp->pir);
-    qemu_get_be32s(f, &opp->spve);
-    qemu_get_be32s(f, &opp->tfrr);
-
-    qemu_get_be32s(f, &nb_cpus);
-    if (opp->nb_cpus != nb_cpus) {
-        return -EINVAL;
-    }
-    assert(nb_cpus > 0 && nb_cpus <= MAX_CPU);
-
-    for (i = 0; i < opp->nb_cpus; i++) {
-        qemu_get_sbe32s(f, &opp->dst[i].ctpr);
-        openpic_load_IRQ_queue(f, &opp->dst[i].raised);
-        openpic_load_IRQ_queue(f, &opp->dst[i].servicing);
-        qemu_get_buffer(f, (uint8_t *)&opp->dst[i].outputs_active,
-                        sizeof(opp->dst[i].outputs_active));
-    }
-
-    for (i = 0; i < OPENPIC_MAX_TMR; i++) {
-        qemu_get_be32s(f, &opp->timers[i].tccr);
-        qemu_get_be32s(f, &opp->timers[i].tbcr);
-    }
-
-    for (i = 0; i < opp->max_irq; i++) {
-        uint32_t val;
-
-        val = qemu_get_be32(f);
-        write_IRQreg_idr(opp, i, val);
-        val = qemu_get_be32(f);
-        write_IRQreg_ivpr(opp, i, val);
-
-        qemu_get_be32s(f, &opp->src[i].ivpr);
-        qemu_get_be32s(f, &opp->src[i].idr);
-        qemu_get_be32s(f, &opp->src[i].destmask);
-        qemu_get_sbe32s(f, &opp->src[i].last_cpu);
-        qemu_get_sbe32s(f, &opp->src[i].pending);
-    }
-
-    return 0;
-}
-
 static void openpic_reset(DeviceState *d)
 {
     OpenPICState *opp = OPENPIC(d);
@@ -1446,12 +1330,14 @@
         write_IRQreg_idr(opp, i, opp->idr_reset);
     }
     /* Initialise IRQ destinations */
-    for (i = 0; i < MAX_CPU; i++) {
+    for (i = 0; i < opp->nb_cpus; i++) {
         opp->dst[i].ctpr      = 15;
-        memset(&opp->dst[i].raised, 0, sizeof(IRQQueue));
         opp->dst[i].raised.next = -1;
-        memset(&opp->dst[i].servicing, 0, sizeof(IRQQueue));
+        opp->dst[i].raised.priority = 0;
+        bitmap_clear(opp->dst[i].raised.queue, 0, IRQQUEUE_SIZE_BITS);
         opp->dst[i].servicing.next = -1;
+        opp->dst[i].servicing.priority = 0;
+        bitmap_clear(opp->dst[i].servicing.queue, 0, IRQQUEUE_SIZE_BITS);
     }
     /* Initialise timers */
     for (i = 0; i < OPENPIC_MAX_TMR; i++) {
@@ -1491,7 +1377,7 @@
 
     opp->irq_msi = 224;
 
-    msi_supported = true;
+    msi_nonbroken = true;
     for (i = 0; i < opp->fsl->max_ext; i++) {
         opp->src[i].level = false;
     }
@@ -1525,6 +1411,110 @@
     }
 }
 
+static const VMStateDescription vmstate_openpic_irq_queue = {
+    .name = "openpic_irq_queue",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_BITMAP(queue, IRQQueue, 0, queue_size),
+        VMSTATE_INT32(next, IRQQueue),
+        VMSTATE_INT32(priority, IRQQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_openpic_irqdest = {
+    .name = "openpic_irqdest",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(ctpr, IRQDest),
+        VMSTATE_STRUCT(raised, IRQDest, 0, vmstate_openpic_irq_queue,
+                       IRQQueue),
+        VMSTATE_STRUCT(servicing, IRQDest, 0, vmstate_openpic_irq_queue,
+                       IRQQueue),
+        VMSTATE_UINT32_ARRAY(outputs_active, IRQDest, OPENPIC_OUTPUT_NB),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_openpic_irqsource = {
+    .name = "openpic_irqsource",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(ivpr, IRQSource),
+        VMSTATE_UINT32(idr, IRQSource),
+        VMSTATE_UINT32(destmask, IRQSource),
+        VMSTATE_INT32(last_cpu, IRQSource),
+        VMSTATE_INT32(pending, IRQSource),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_openpic_timer = {
+    .name = "openpic_timer",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(tccr, OpenPICTimer),
+        VMSTATE_UINT32(tbcr, OpenPICTimer),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_openpic_msi = {
+    .name = "openpic_msi",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(msir, OpenPICMSI),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static int openpic_post_load(void *opaque, int version_id)
+{
+    OpenPICState *opp = (OpenPICState *)opaque;
+    int i;
+
+    /* Update internal ivpr and idr variables */
+    for (i = 0; i < opp->max_irq; i++) {
+        write_IRQreg_idr(opp, i, opp->src[i].idr);
+        write_IRQreg_ivpr(opp, i, opp->src[i].ivpr);
+    }
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_openpic = {
+    .name = "openpic",
+    .version_id = 3,
+    .minimum_version_id = 3,
+    .post_load = openpic_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(gcr, OpenPICState),
+        VMSTATE_UINT32(vir, OpenPICState),
+        VMSTATE_UINT32(pir, OpenPICState),
+        VMSTATE_UINT32(spve, OpenPICState),
+        VMSTATE_UINT32(tfrr, OpenPICState),
+        VMSTATE_UINT32(max_irq, OpenPICState),
+        VMSTATE_STRUCT_VARRAY_UINT32(src, OpenPICState, max_irq, 0,
+                                     vmstate_openpic_irqsource, IRQSource),
+        VMSTATE_UINT32_EQUAL(nb_cpus, OpenPICState),
+        VMSTATE_STRUCT_VARRAY_UINT32(dst, OpenPICState, nb_cpus, 0,
+                                     vmstate_openpic_irqdest, IRQDest),
+        VMSTATE_STRUCT_ARRAY(timers, OpenPICState, OPENPIC_MAX_TMR, 0,
+                             vmstate_openpic_timer, OpenPICTimer),
+        VMSTATE_STRUCT_ARRAY(msi, OpenPICState, MAX_MSI, 0,
+                             vmstate_openpic_msi, OpenPICMSI),
+        VMSTATE_UINT32(irq_ipi0, OpenPICState),
+        VMSTATE_UINT32(irq_tim0, OpenPICState),
+        VMSTATE_UINT32(irq_msi, OpenPICState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void openpic_init(Object *obj)
 {
     OpenPICState *opp = OPENPIC(obj);
@@ -1569,9 +1559,9 @@
     };
 
     if (opp->nb_cpus > MAX_CPU) {
-        error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,
-                  TYPE_OPENPIC, "nb_cpus", (uint64_t)opp->nb_cpus,
-                  (uint64_t)0, (uint64_t)MAX_CPU);
+        error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,
+                   TYPE_OPENPIC, "nb_cpus", (uint64_t)opp->nb_cpus,
+                   (uint64_t)0, (uint64_t)MAX_CPU);
         return;
     }
 
@@ -1631,10 +1621,12 @@
         for (j = 0; j < OPENPIC_OUTPUT_NB; j++) {
             sysbus_init_irq(d, &opp->dst[i].irqs[j]);
         }
-    }
 
-    register_savevm(dev, "openpic", 0, 2,
-                    openpic_save, openpic_load, opp);
+        opp->dst[i].raised.queue_size = IRQQUEUE_SIZE_BITS;
+        opp->dst[i].raised.queue = bitmap_new(IRQQUEUE_SIZE_BITS);
+        opp->dst[i].servicing.queue_size = IRQQUEUE_SIZE_BITS;
+        opp->dst[i].servicing.queue = bitmap_new(IRQQUEUE_SIZE_BITS);
+    }
 
     sysbus_init_mmio(d, &opp->mem);
     qdev_init_gpio_in(dev, openpic_set_irq, opp->max_irq);
@@ -1653,6 +1645,8 @@
     dc->realize = openpic_realize;
     dc->props = openpic_properties;
     dc->reset = openpic_reset;
+    dc->vmsd = &vmstate_openpic;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo openpic_info = {
diff --git a/hw/intc/openpic_kvm.c b/hw/intc/openpic_kvm.c
index 3e2cd18..0518e01 100644
--- a/hw/intc/openpic_kvm.c
+++ b/hw/intc/openpic_kvm.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include <sys/ioctl.h>
 #include "exec/address-spaces.h"
 #include "hw/hw.h"
@@ -238,7 +242,7 @@
     memory_listener_register(&opp->mem_listener, &address_space_memory);
 
     /* indicate pic capabilities */
-    msi_supported = true;
+    msi_nonbroken = true;
     kvm_kernel_irqchip = true;
     kvm_async_interrupts_allowed = true;
 
@@ -248,7 +252,6 @@
         kvm_irqchip_add_irq_route(kvm_state, i, 0, i);
     }
 
-    kvm_irqfds_allowed = true;
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_routing_allowed = true;
 
@@ -276,6 +279,7 @@
     dc->realize = kvm_openpic_realize;
     dc->props = kvm_openpic_properties;
     dc->reset = kvm_openpic_reset;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo kvm_openpic_info = {
diff --git a/hw/intc/pl190.c b/hw/intc/pl190.c
index 2bf359a..55ea15d 100644
--- a/hw/intc/pl190.c
+++ b/hw/intc/pl190.c
@@ -7,7 +7,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qemu/log.h"
 
 /* The number of virtual priority levels.  16 user vectors plus the
    unvectored IRQ.  Chained interrupts would require an additional level
@@ -235,17 +237,17 @@
     pl190_update_vectors(s);
 }
 
-static int pl190_init(SysBusDevice *sbd)
+static void pl190_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    PL190State *s = PL190(dev);
+    DeviceState *dev = DEVICE(obj);
+    PL190State *s = PL190(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pl190_ops, s, "pl190", 0x1000);
+    memory_region_init_io(&s->iomem, obj, &pl190_ops, s, "pl190", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     qdev_init_gpio_in(dev, pl190_set_irq, 32);
     sysbus_init_irq(sbd, &s->irq);
     sysbus_init_irq(sbd, &s->fiq);
-    return 0;
 }
 
 static const VMStateDescription vmstate_pl190 = {
@@ -270,9 +272,7 @@
 static void pl190_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pl190_init;
     dc->reset = pl190_reset;
     dc->vmsd = &vmstate_pl190;
 }
@@ -281,6 +281,7 @@
     .name          = TYPE_PL190,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PL190State),
+    .instance_init = pl190_init,
     .class_init    = pl190_class_init,
 };
 
diff --git a/hw/intc/puv3_intc.c b/hw/intc/puv3_intc.c
index c2803d0..ef8488a 100644
--- a/hw/intc/puv3_intc.c
+++ b/hw/intc/puv3_intc.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #undef DEBUG_PUV3
diff --git a/hw/intc/realview_gic.c b/hw/intc/realview_gic.c
index 6c81296..50bbab6 100644
--- a/hw/intc/realview_gic.c
+++ b/hw/intc/realview_gic.c
@@ -7,6 +7,8 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/intc/realview_gic.h"
 
 static void realview_gic_set_irq(void *opaque, int irq, int level)
diff --git a/hw/intc/s390_flic.c b/hw/intc/s390_flic.c
index 03c5e89..6ab29ef 100644
--- a/hw/intc/s390_flic.c
+++ b/hw/intc/s390_flic.c
@@ -10,6 +10,7 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
 #include "migration/qemu-file.h"
@@ -30,7 +31,6 @@
 void s390_flic_init(void)
 {
     DeviceState *dev;
-    int r;
 
     dev = s390_flic_kvm_create();
     if (!dev) {
@@ -38,10 +38,7 @@
         object_property_add_child(qdev_get_machine(), TYPE_QEMU_S390_FLIC,
                                   OBJECT(dev), NULL);
     }
-    r = qdev_init(dev);
-    if (r) {
-        error_report("flic: couldn't create qdev");
-    }
+    qdev_init_nofail(dev);
 }
 
 static int qemu_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
@@ -70,6 +67,13 @@
 {
 }
 
+static int qemu_s390_clear_io_flic(S390FLICState *fs, uint16_t subchannel_id,
+                           uint16_t subchannel_nr)
+{
+    /* Fixme TCG */
+    return -ENOSYS;
+}
+
 static void qemu_s390_flic_class_init(ObjectClass *oc, void *data)
 {
     S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
@@ -78,6 +82,7 @@
     fsc->io_adapter_map = qemu_s390_io_adapter_map;
     fsc->add_adapter_routes = qemu_s390_add_adapter_routes;
     fsc->release_adapter_routes = qemu_s390_release_adapter_routes;
+    fsc->clear_io_irq = qemu_s390_clear_io_flic;
 }
 
 static const TypeInfo qemu_s390_flic_info = {
diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
index b471e7a..fef8080 100644
--- a/hw/intc/s390_flic_kvm.c
+++ b/hw/intc/s390_flic_kvm.c
@@ -10,6 +10,9 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include <sys/ioctl.h>
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
@@ -27,6 +30,7 @@
     S390FLICState parent_obj;
 
     uint32_t fd;
+    bool clear_io_supported;
 } KVMS390FLICState;
 
 DeviceState *s390_flic_kvm_create(void)
@@ -127,6 +131,24 @@
     return flic_enqueue_irqs(irq, sizeof(*irq), flic);
 }
 
+static int kvm_s390_clear_io_flic(S390FLICState *fs, uint16_t subchannel_id,
+                           uint16_t subchannel_nr)
+{
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    int rc;
+    uint32_t sid = subchannel_id << 16 | subchannel_nr;
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_CLEAR_IO_IRQ,
+        .addr = (uint64_t) &sid,
+        .attr = sizeof(sid),
+    };
+    if (unlikely(!flic->clear_io_supported)) {
+        return -ENOSYS;
+    }
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+    return rc ? -errno : 0;
+}
+
 /**
  * __get_all_irqs - store all pending irqs in buffer
  * @flic: pointer to flic device state
@@ -173,7 +195,7 @@
         .swap = swap,
     };
     KVMS390FLICState *flic = KVM_S390_FLIC(fs);
-    int r, ret;
+    int r;
     struct kvm_device_attr attr = {
         .group = KVM_DEV_FLIC_ADAPTER_REGISTER,
         .addr = (uint64_t)&adapter,
@@ -186,8 +208,7 @@
 
     r = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
 
-    ret = r ? -errno : 0;
-    return ret;
+    return r ? -errno : 0;
 }
 
 static int kvm_s390_io_adapter_map(S390FLICState *fs, uint32_t id,
@@ -228,6 +249,8 @@
         routes->gsi[i] = ret;
         routes->adapter.ind_offset++;
     }
+    kvm_irqchip_commit_routes(kvm_state);
+
     /* Restore passed-in structure to original state. */
     routes->adapter.ind_offset = ind_offset;
     return 0;
@@ -353,6 +376,7 @@
 {
     KVMS390FLICState *flic_state = KVM_S390_FLIC(dev);
     struct kvm_create_device cd = {0};
+    struct kvm_device_attr test_attr = {0};
     int ret;
 
     flic_state->fd = -1;
@@ -369,6 +393,11 @@
     }
     flic_state->fd = cd.fd;
 
+    /* Check clear_io_irq support */
+    test_attr.group = KVM_DEV_FLIC_CLEAR_IO_IRQ;
+    flic_state->clear_io_supported = !ioctl(flic_state->fd,
+                                            KVM_HAS_DEVICE_ATTR, test_attr);
+
     /* Register savevm handler for floating interrupts */
     register_savevm(NULL, "s390-flic", 0, 1, kvm_flic_save,
                     kvm_flic_load, (void *) flic_state);
@@ -415,6 +444,7 @@
     fsc->io_adapter_map = kvm_s390_io_adapter_map;
     fsc->add_adapter_routes = kvm_s390_add_adapter_routes;
     fsc->release_adapter_routes = kvm_s390_release_adapter_routes;
+    fsc->clear_io_irq = kvm_s390_clear_io_flic;
 }
 
 static const TypeInfo kvm_s390_flic_info = {
diff --git a/hw/intc/sh_intc.c b/hw/intc/sh_intc.c
index 55c76e4..6ce2a80 100644
--- a/hw/intc/sh_intc.c
+++ b/hw/intc/sh_intc.c
@@ -8,6 +8,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sh4/sh_intc.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
diff --git a/hw/intc/slavio_intctl.c b/hw/intc/slavio_intctl.c
index f22aba0..e82e893 100644
--- a/hw/intc/slavio_intctl.c
+++ b/hw/intc/slavio_intctl.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sparc/sun4m.h"
 #include "monitor/monitor.h"
 #include "hw/sysbus.h"
@@ -417,15 +418,16 @@
     slavio_check_interrupts(s, 0);
 }
 
-static int slavio_intctl_init1(SysBusDevice *sbd)
+static void slavio_intctl_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    SLAVIO_INTCTLState *s = SLAVIO_INTCTL(dev);
+    DeviceState *dev = DEVICE(obj);
+    SLAVIO_INTCTLState *s = SLAVIO_INTCTL(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
     unsigned int i, j;
     char slave_name[45];
 
     qdev_init_gpio_in(dev, slavio_set_irq_all, 32 + MAX_CPUS);
-    memory_region_init_io(&s->iomem, OBJECT(s), &slavio_intctlm_mem_ops, s,
+    memory_region_init_io(&s->iomem, obj, &slavio_intctlm_mem_ops, s,
                           "master-interrupt-controller", INTCTLM_SIZE);
     sysbus_init_mmio(sbd, &s->iomem);
 
@@ -442,16 +444,12 @@
         s->slaves[i].cpu = i;
         s->slaves[i].master = s;
     }
-
-    return 0;
 }
 
 static void slavio_intctl_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = slavio_intctl_init1;
     dc->reset = slavio_intctl_reset;
     dc->vmsd = &vmstate_intctl;
 }
@@ -460,6 +458,7 @@
     .name          = TYPE_SLAVIO_INTCTL,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SLAVIO_INTCTLState),
+    .instance_init = slavio_intctl_init,
     .class_init    = slavio_intctl_class_init,
 };
 
diff --git a/hw/intc/trace-events b/hw/intc/trace-events
new file mode 100644
index 0000000..f12192c
--- /dev/null
+++ b/hw/intc/trace-events
@@ -0,0 +1,123 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/intc/apic_common.c
+cpu_set_apic_base(uint64_t val) "%016"PRIx64
+cpu_get_apic_base(uint64_t val) "%016"PRIx64
+# coalescing
+apic_report_irq_delivered(int apic_irq_delivered) "coalescing %d"
+apic_reset_irq_delivered(int apic_irq_delivered) "old coalescing %d"
+apic_get_irq_delivered(int apic_irq_delivered) "returning coalescing %d"
+
+# hw/intc/apic.c
+apic_local_deliver(int vector, uint32_t lvt) "vector %d delivery mode %d"
+apic_deliver_irq(uint8_t dest, uint8_t dest_mode, uint8_t delivery_mode, uint8_t vector_num, uint8_t trigger_mode) "dest %d dest_mode %d delivery_mode %d vector %d trigger_mode %d"
+apic_mem_readl(uint64_t addr, uint32_t val)  "%"PRIx64" = %08x"
+apic_mem_writel(uint64_t addr, uint32_t val) "%"PRIx64" = %08x"
+
+# hw/intc/slavio_intctl.c
+slavio_intctl_mem_readl(uint32_t cpu, uint64_t addr, uint32_t ret) "read cpu %d reg 0x%"PRIx64" = %x"
+slavio_intctl_mem_writel(uint32_t cpu, uint64_t addr, uint32_t val) "write cpu %d reg 0x%"PRIx64" = %x"
+slavio_intctl_mem_writel_clear(uint32_t cpu, uint32_t val, uint32_t intreg_pending) "Cleared cpu %d irq mask %x, curmask %x"
+slavio_intctl_mem_writel_set(uint32_t cpu, uint32_t val, uint32_t intreg_pending) "Set cpu %d irq mask %x, curmask %x"
+slavio_intctlm_mem_readl(uint64_t addr, uint32_t ret) "read system reg 0x%"PRIx64" = %x"
+slavio_intctlm_mem_writel(uint64_t addr, uint32_t val) "write system reg 0x%"PRIx64" = %x"
+slavio_intctlm_mem_writel_enable(uint32_t val, uint32_t intregm_disabled) "Enabled master irq mask %x, curmask %x"
+slavio_intctlm_mem_writel_disable(uint32_t val, uint32_t intregm_disabled) "Disabled master irq mask %x, curmask %x"
+slavio_intctlm_mem_writel_target(uint32_t cpu) "Set master irq cpu %d"
+slavio_check_interrupts(uint32_t pending, uint32_t intregm_disabled) "pending %x disabled %x"
+slavio_set_irq(uint32_t target_cpu, int irq, uint32_t pil, int level) "Set cpu %d irq %d -> pil %d level %d"
+slavio_set_timer_irq_cpu(int cpu, int level) "Set cpu %d local timer level %d"
+
+# hw/intc/grlib_irqmp.c
+grlib_irqmp_check_irqs(uint32_t pend, uint32_t force, uint32_t mask, uint32_t lvl1, uint32_t lvl2) "pend:0x%04x force:0x%04x mask:0x%04x lvl1:0x%04x lvl0:0x%04x"
+grlib_irqmp_ack(int intno) "interrupt:%d"
+grlib_irqmp_set_irq(int irq) "Raise CPU IRQ %d"
+grlib_irqmp_readl_unknown(uint64_t addr) "addr 0x%"PRIx64
+grlib_irqmp_writel_unknown(uint64_t addr, uint32_t value) "addr 0x%"PRIx64" value 0x%x"
+
+# hw/intc/lm32_pic.c
+lm32_pic_raise_irq(void) "Raise CPU interrupt"
+lm32_pic_lower_irq(void) "Lower CPU interrupt"
+lm32_pic_interrupt(int irq, int level) "Set IRQ%d %d"
+lm32_pic_set_im(uint32_t im) "im 0x%08x"
+lm32_pic_set_ip(uint32_t ip) "ip 0x%08x"
+lm32_pic_get_im(uint32_t im) "im 0x%08x"
+lm32_pic_get_ip(uint32_t ip) "ip 0x%08x"
+
+# hw/intc/xics.c
+xics_icp_check_ipi(int server, uint8_t mfrr) "CPU %d can take IPI mfrr=%#x"
+xics_icp_accept(uint32_t old_xirr, uint32_t new_xirr) "icp_accept: XIRR %#"PRIx32"->%#"PRIx32
+xics_icp_eoi(int server, uint32_t xirr, uint32_t new_xirr) "icp_eoi: server %d given XIRR %#"PRIx32" new XIRR %#"PRIx32
+xics_icp_irq(int server, int nr, uint8_t priority) "cpu %d trying to deliver irq %#"PRIx32" priority %#x"
+xics_icp_raise(uint32_t xirr, uint8_t pending_priority) "raising IRQ new XIRR=%#x new pending priority=%#x"
+xics_set_irq_msi(int srcno, int nr) "set_irq_msi: srcno %d [irq %#x]"
+xics_masked_pending(void) "set_irq_msi: masked pending"
+xics_set_irq_lsi(int srcno, int nr) "set_irq_lsi: srcno %d [irq %#x]"
+xics_ics_write_xive(int nr, int srcno, int server, uint8_t priority) "ics_write_xive: irq %#x [src %d] server %#x prio %#x"
+xics_ics_reject(int nr, int srcno) "reject irq %#x [src %d]"
+xics_ics_eoi(int nr) "ics_eoi: irq %#x"
+xics_alloc(int src, int irq) "source#%d, irq %d"
+xics_alloc_block(int src, int first, int num, bool lsi, int align) "source#%d, first irq %d, %d irqs, lsi=%d, alignnum %d"
+xics_ics_free(int src, int irq, int num) "Source#%d, first irq %d, %d irqs"
+xics_ics_free_warn(int src, int irq) "Source#%d, irq %d is already free"
+
+# hw/intc/s390_flic_kvm.c
+flic_create_device(int err) "flic: create device failed %d"
+flic_no_device_api(int err) "flic: no Device Contral API support %d"
+flic_reset_failed(int err) "flic: reset failed %d"
+
+# hw/intc/aspeed_vic.c
+aspeed_vic_set_irq(int irq, int level) "Enabling IRQ %d: %d"
+aspeed_vic_update_fiq(int flags) "Raising FIQ: %d"
+aspeed_vic_update_irq(int flags) "Raising IRQ: %d"
+aspeed_vic_read(uint64_t offset, unsigned size, uint32_t value) "From 0x%" PRIx64 " of size %u: 0x%" PRIx32
+aspeed_vic_write(uint64_t offset, unsigned size, uint32_t data) "To 0x%" PRIx64 " of size %u: 0x%" PRIx32
+
+# hw/intc/arm_gic.c
+gic_enable_irq(int irq) "irq %d enabled"
+gic_disable_irq(int irq) "irq %d disabled"
+gic_set_irq(int irq, int level, int cpumask, int target) "irq %d level %d cpumask 0x%x target 0x%x"
+gic_update_bestirq(int cpu, int irq, int prio, int priority_mask, int running_priority) "cpu %d irq %d priority %d cpu priority mask %d cpu running priority %d"
+gic_update_set_irq(int cpu, const char *name, int level) "cpu[%d]: %s = %d"
+gic_acknowledge_irq(int cpu, int irq) "cpu %d acknowledged irq %d"
+
+# hw/intc/arm_gicv3_cpuif.c
+gicv3_icc_pmr_read(uint32_t cpu, uint64_t val) "GICv3 ICC_PMR read cpu %x value 0x%" PRIx64
+gicv3_icc_pmr_write(uint32_t cpu, uint64_t val) "GICv3 ICC_PMR write cpu %x value 0x%" PRIx64
+gicv3_icc_bpr_read(uint32_t cpu, uint64_t val) "GICv3 ICC_BPR read cpu %x value 0x%" PRIx64
+gicv3_icc_bpr_write(uint32_t cpu, uint64_t val) "GICv3 ICC_BPR write cpu %x value 0x%" PRIx64
+gicv3_icc_ap_read(int regno, uint32_t cpu, uint64_t val) "GICv3 ICC_AP%dR read cpu %x value 0x%" PRIx64
+gicv3_icc_ap_write(int regno, uint32_t cpu, uint64_t val) "GICv3 ICC_AP%dR write cpu %x value 0x%" PRIx64
+gicv3_icc_igrpen_read(uint32_t cpu, uint64_t val) "GICv3 ICC_IGRPEN read cpu %x value 0x%" PRIx64
+gicv3_icc_igrpen_write(uint32_t cpu, uint64_t val) "GICv3 ICC_IGRPEN write cpu %x value 0x%" PRIx64
+gicv3_icc_igrpen1_el3_read(uint32_t cpu, uint64_t val) "GICv3 ICC_IGRPEN1_EL3 read cpu %x value 0x%" PRIx64
+gicv3_icc_igrpen1_el3_write(uint32_t cpu, uint64_t val) "GICv3 ICC_IGRPEN1_EL3 write cpu %x value 0x%" PRIx64
+gicv3_icc_ctlr_read(uint32_t cpu, uint64_t val) "GICv3 ICC_CTLR read cpu %x value 0x%" PRIx64
+gicv3_icc_ctlr_write(uint32_t cpu, uint64_t val) "GICv3 ICC_CTLR write cpu %x value 0x%" PRIx64
+gicv3_icc_ctlr_el3_read(uint32_t cpu, uint64_t val) "GICv3 ICC_CTLR_EL3 read cpu %x value 0x%" PRIx64
+gicv3_icc_ctlr_el3_write(uint32_t cpu, uint64_t val) "GICv3 ICC_CTLR_EL3 write cpu %x value 0x%" PRIx64
+gicv3_cpuif_update(uint32_t cpuid, int irq, int grp, int prio) "GICv3 CPU i/f %x HPPI update: irq %d group %d prio %d"
+gicv3_cpuif_set_irqs(uint32_t cpuid, int fiqlevel, int irqlevel) "GICv3 CPU i/f %x HPPI update: setting FIQ %d IRQ %d"
+gicv3_icc_generate_sgi(uint32_t cpuid, int irq, int irm, uint32_t aff, uint32_t targetlist) "GICv3 CPU i/f %x generating SGI %d IRM %d target affinity 0x%xxx targetlist 0x%x"
+gicv3_icc_iar0_read(uint32_t cpu, uint64_t val) "GICv3 ICC_IAR0 read cpu %x value 0x%" PRIx64
+gicv3_icc_iar1_read(uint32_t cpu, uint64_t val) "GICv3 ICC_IAR1 read cpu %x value 0x%" PRIx64
+gicv3_icc_eoir_write(uint32_t cpu, uint64_t val) "GICv3 ICC_EOIR write cpu %x value 0x%" PRIx64
+gicv3_icc_hppir0_read(uint32_t cpu, uint64_t val) "GICv3 ICC_HPPIR0 read cpu %x value 0x%" PRIx64
+gicv3_icc_hppir1_read(uint32_t cpu, uint64_t val) "GICv3 ICC_HPPIR1 read cpu %x value 0x%" PRIx64
+gicv3_icc_dir_write(uint32_t cpu, uint64_t val) "GICv3 ICC_DIR write cpu %x value 0x%" PRIx64
+gicv3_icc_rpr_read(uint32_t cpu, uint64_t val) "GICv3 ICC_RPR read cpu %x value 0x%" PRIx64
+
+# hw/intc/arm_gicv3_dist.c
+gicv3_dist_read(uint64_t offset, uint64_t data, unsigned size, bool secure) "GICv3 distributor read: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u secure %d"
+gicv3_dist_badread(uint64_t offset, unsigned size, bool secure) "GICv3 distributor read: offset 0x%" PRIx64 " size %u secure %d: error"
+gicv3_dist_write(uint64_t offset, uint64_t data, unsigned size, bool secure) "GICv3 distributor write: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u secure %d"
+gicv3_dist_badwrite(uint64_t offset, uint64_t data, unsigned size, bool secure) "GICv3 distributor write: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u secure %d: error"
+gicv3_dist_set_irq(int irq, int level) "GICv3 distributor interrupt %d level changed to %d"
+
+# hw/intc/arm_gicv3_redist.c
+gicv3_redist_read(uint32_t cpu, uint64_t offset, uint64_t data, unsigned size, bool secure) "GICv3 redistributor %x read: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u secure %d"
+gicv3_redist_badread(uint32_t cpu, uint64_t offset, unsigned size, bool secure) "GICv3 redistributor %x read: offset 0x%" PRIx64 " size %u secure %d: error"
+gicv3_redist_write(uint32_t cpu, uint64_t offset, uint64_t data, unsigned size, bool secure) "GICv3 redistributor %x write: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u secure %d"
+gicv3_redist_badwrite(uint32_t cpu, uint64_t offset, uint64_t data, unsigned size, bool secure) "GICv3 redistributor %x write: offset 0x%" PRIx64 " data 0x%" PRIx64 " size %u secure %d: error"
+gicv3_redist_set_irq(uint32_t cpu, int irq, int level) "GICv3 redistributor %x interrupt %d level changed to %d"
+gicv3_redist_send_sgi(uint32_t cpu, int irq) "GICv3 redistributor %x pending SGI %d"
diff --git a/hw/intc/vgic_common.h b/hw/intc/vgic_common.h
new file mode 100644
index 0000000..80d919e
--- /dev/null
+++ b/hw/intc/vgic_common.h
@@ -0,0 +1,35 @@
+/*
+ * ARM KVM vGIC utility functions
+ *
+ * Copyright (c) 2015 Samsung Electronics
+ * Written by Pavel Fedin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_ARM_VGIC_COMMON_H
+#define QEMU_ARM_VGIC_COMMON_H
+
+/**
+ * kvm_arm_gic_set_irq - Send an IRQ to the in-kernel vGIC
+ * @num_irq: Total number of IRQs configured for the GIC instance
+ * @irq: qemu internal IRQ line number:
+ *  [0..N-1] : external interrupts
+ *  [N..N+31] : PPI (internal) interrupts for CPU 0
+ *  [N+32..N+63] : PPI (internal interrupts for CPU 1
+ * @level: level of the IRQ line.
+ */
+void kvm_arm_gic_set_irq(uint32_t num_irq, int irq, int level);
+
+#endif
diff --git a/hw/intc/xics.c b/hw/intc/xics.c
index 0fd2a84..cd48f42 100644
--- a/hw/intc/xics.c
+++ b/hw/intc/xics.c
@@ -25,15 +25,18 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "trace.h"
 #include "qemu/timer.h"
-#include "hw/ppc/spapr.h"
 #include "hw/ppc/xics.h"
 #include "qemu/error-report.h"
 #include "qapi/visitor.h"
 
-static int get_cpu_index_by_dt_id(int cpu_dt_id)
+int xics_get_cpu_index_by_dt_id(int cpu_dt_id)
 {
     PowerPCCPU *cpu = ppc_get_vcpu_by_dt_id(cpu_dt_id);
 
@@ -44,17 +47,31 @@
     return -1;
 }
 
-void xics_cpu_setup(XICSState *icp, PowerPCCPU *cpu)
+void xics_cpu_destroy(XICSState *xics, PowerPCCPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    ICPState *ss = &xics->ss[cs->cpu_index];
+
+    assert(cs->cpu_index < xics->nr_servers);
+    assert(cs == ss->cs);
+
+    ss->output = NULL;
+    ss->cs = NULL;
+}
+
+void xics_cpu_setup(XICSState *xics, PowerPCCPU *cpu)
 {
     CPUState *cs = CPU(cpu);
     CPUPPCState *env = &cpu->env;
-    ICPState *ss = &icp->ss[cs->cpu_index];
-    XICSStateClass *info = XICS_COMMON_GET_CLASS(icp);
+    ICPState *ss = &xics->ss[cs->cpu_index];
+    XICSStateClass *info = XICS_COMMON_GET_CLASS(xics);
 
-    assert(cs->cpu_index < icp->nr_servers);
+    assert(cs->cpu_index < xics->nr_servers);
+
+    ss->cs = cs;
 
     if (info->cpu_setup) {
-        info->cpu_setup(icp, cpu);
+        info->cpu_setup(xics, cpu);
     }
 
     switch (PPC_INPUT(env)) {
@@ -78,81 +95,81 @@
  */
 static void xics_common_reset(DeviceState *d)
 {
-    XICSState *icp = XICS_COMMON(d);
+    XICSState *xics = XICS_COMMON(d);
     int i;
 
-    for (i = 0; i < icp->nr_servers; i++) {
-        device_reset(DEVICE(&icp->ss[i]));
+    for (i = 0; i < xics->nr_servers; i++) {
+        device_reset(DEVICE(&xics->ss[i]));
     }
 
-    device_reset(DEVICE(icp->ics));
+    device_reset(DEVICE(xics->ics));
 }
 
-static void xics_prop_get_nr_irqs(Object *obj, Visitor *v,
-                                  void *opaque, const char *name, Error **errp)
+static void xics_prop_get_nr_irqs(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
-    XICSState *icp = XICS_COMMON(obj);
-    int64_t value = icp->nr_irqs;
+    XICSState *xics = XICS_COMMON(obj);
+    int64_t value = xics->nr_irqs;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void xics_prop_set_nr_irqs(Object *obj, Visitor *v,
-                                  void *opaque, const char *name, Error **errp)
+static void xics_prop_set_nr_irqs(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
-    XICSState *icp = XICS_COMMON(obj);
-    XICSStateClass *info = XICS_COMMON_GET_CLASS(icp);
+    XICSState *xics = XICS_COMMON(obj);
+    XICSStateClass *info = XICS_COMMON_GET_CLASS(xics);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
     }
-    if (icp->nr_irqs) {
+    if (xics->nr_irqs) {
         error_setg(errp, "Number of interrupts is already set to %u",
-                   icp->nr_irqs);
+                   xics->nr_irqs);
         return;
     }
 
     assert(info->set_nr_irqs);
-    assert(icp->ics);
-    info->set_nr_irqs(icp, value, errp);
+    assert(xics->ics);
+    info->set_nr_irqs(xics, value, errp);
 }
 
 static void xics_prop_get_nr_servers(Object *obj, Visitor *v,
-                                     void *opaque, const char *name,
+                                     const char *name, void *opaque,
                                      Error **errp)
 {
-    XICSState *icp = XICS_COMMON(obj);
-    int64_t value = icp->nr_servers;
+    XICSState *xics = XICS_COMMON(obj);
+    int64_t value = xics->nr_servers;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void xics_prop_set_nr_servers(Object *obj, Visitor *v,
-                                     void *opaque, const char *name,
+                                     const char *name, void *opaque,
                                      Error **errp)
 {
-    XICSState *icp = XICS_COMMON(obj);
-    XICSStateClass *info = XICS_COMMON_GET_CLASS(icp);
+    XICSState *xics = XICS_COMMON(obj);
+    XICSStateClass *info = XICS_COMMON_GET_CLASS(xics);
     Error *error = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &error);
+    visit_type_int(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
     }
-    if (icp->nr_servers) {
+    if (xics->nr_servers) {
         error_setg(errp, "Number of servers is already set to %u",
-                   icp->nr_servers);
+                   xics->nr_servers);
         return;
     }
 
     assert(info->set_nr_servers);
-    info->set_nr_servers(icp, value, errp);
+    info->set_nr_servers(xics, value, errp);
 }
 
 static void xics_common_initfn(Object *obj)
@@ -195,9 +212,9 @@
 static void ics_resend(ICSState *ics);
 static void ics_eoi(ICSState *ics, int nr);
 
-static void icp_check_ipi(XICSState *icp, int server)
+static void icp_check_ipi(XICSState *xics, int server)
 {
-    ICPState *ss = icp->ss + server;
+    ICPState *ss = xics->ss + server;
 
     if (XISR(ss) && (ss->pending_priority <= ss->mfrr)) {
         return;
@@ -206,7 +223,7 @@
     trace_xics_icp_check_ipi(server, ss->mfrr);
 
     if (XISR(ss)) {
-        ics_reject(icp->ics, XISR(ss));
+        ics_reject(xics->ics, XISR(ss));
     }
 
     ss->xirr = (ss->xirr & ~XISR_MASK) | XICS_IPI;
@@ -214,19 +231,19 @@
     qemu_irq_raise(ss->output);
 }
 
-static void icp_resend(XICSState *icp, int server)
+static void icp_resend(XICSState *xics, int server)
 {
-    ICPState *ss = icp->ss + server;
+    ICPState *ss = xics->ss + server;
 
     if (ss->mfrr < CPPR(ss)) {
-        icp_check_ipi(icp, server);
+        icp_check_ipi(xics, server);
     }
-    ics_resend(icp->ics);
+    ics_resend(xics->ics);
 }
 
-static void icp_set_cppr(XICSState *icp, int server, uint8_t cppr)
+void icp_set_cppr(XICSState *xics, int server, uint8_t cppr)
 {
-    ICPState *ss = icp->ss + server;
+    ICPState *ss = xics->ss + server;
     uint8_t old_cppr;
     uint32_t old_xisr;
 
@@ -239,26 +256,26 @@
             ss->xirr &= ~XISR_MASK; /* Clear XISR */
             ss->pending_priority = 0xff;
             qemu_irq_lower(ss->output);
-            ics_reject(icp->ics, old_xisr);
+            ics_reject(xics->ics, old_xisr);
         }
     } else {
         if (!XISR(ss)) {
-            icp_resend(icp, server);
+            icp_resend(xics, server);
         }
     }
 }
 
-static void icp_set_mfrr(XICSState *icp, int server, uint8_t mfrr)
+void icp_set_mfrr(XICSState *xics, int server, uint8_t mfrr)
 {
-    ICPState *ss = icp->ss + server;
+    ICPState *ss = xics->ss + server;
 
     ss->mfrr = mfrr;
     if (mfrr < CPPR(ss)) {
-        icp_check_ipi(icp, server);
+        icp_check_ipi(xics, server);
     }
 }
 
-static uint32_t icp_accept(ICPState *ss)
+uint32_t icp_accept(ICPState *ss)
 {
     uint32_t xirr = ss->xirr;
 
@@ -271,31 +288,39 @@
     return xirr;
 }
 
-static void icp_eoi(XICSState *icp, int server, uint32_t xirr)
+uint32_t icp_ipoll(ICPState *ss, uint32_t *mfrr)
 {
-    ICPState *ss = icp->ss + server;
+    if (mfrr) {
+        *mfrr = ss->mfrr;
+    }
+    return ss->xirr;
+}
+
+void icp_eoi(XICSState *xics, int server, uint32_t xirr)
+{
+    ICPState *ss = xics->ss + server;
 
     /* Send EOI -> ICS */
     ss->xirr = (ss->xirr & ~CPPR_MASK) | (xirr & CPPR_MASK);
     trace_xics_icp_eoi(server, xirr, ss->xirr);
-    ics_eoi(icp->ics, xirr & XISR_MASK);
+    ics_eoi(xics->ics, xirr & XISR_MASK);
     if (!XISR(ss)) {
-        icp_resend(icp, server);
+        icp_resend(xics, server);
     }
 }
 
-static void icp_irq(XICSState *icp, int server, int nr, uint8_t priority)
+static void icp_irq(XICSState *xics, int server, int nr, uint8_t priority)
 {
-    ICPState *ss = icp->ss + server;
+    ICPState *ss = xics->ss + server;
 
     trace_xics_icp_irq(server, nr, priority);
 
     if ((priority >= CPPR(ss))
         || (XISR(ss) && (ss->pending_priority <= priority))) {
-        ics_reject(icp->ics, nr);
+        ics_reject(xics->ics, nr);
     } else {
         if (XISR(ss)) {
-            ics_reject(icp->ics, XISR(ss));
+            ics_reject(xics->ics, XISR(ss));
         }
         ss->xirr = (ss->xirr & ~XISR_MASK) | (nr & XISR_MASK);
         ss->pending_priority = priority;
@@ -372,12 +397,6 @@
 /*
  * ICS: Source layer
  */
-static int ics_valid_irq(ICSState *ics, uint32_t nr)
-{
-    return (nr >= ics->offset)
-        && (nr < (ics->offset + ics->nr_irqs));
-}
-
 static void resend_msi(ICSState *ics, int srcno)
 {
     ICSIRQState *irq = ics->irqs + srcno;
@@ -386,7 +405,7 @@
     if (irq->status & XICS_STATUS_REJECTED) {
         irq->status &= ~XICS_STATUS_REJECTED;
         if (irq->priority != 0xff) {
-            icp_irq(ics->icp, irq->server, srcno + ics->offset,
+            icp_irq(ics->xics, irq->server, srcno + ics->offset,
                     irq->priority);
         }
     }
@@ -400,7 +419,7 @@
         && (irq->status & XICS_STATUS_ASSERTED)
         && !(irq->status & XICS_STATUS_SENT)) {
         irq->status |= XICS_STATUS_SENT;
-        icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
+        icp_irq(ics->xics, irq->server, srcno + ics->offset, irq->priority);
     }
 }
 
@@ -415,7 +434,7 @@
             irq->status |= XICS_STATUS_MASKED_PENDING;
             trace_xics_masked_pending();
         } else  {
-            icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
+            icp_irq(ics->xics, irq->server, srcno + ics->offset, irq->priority);
         }
     }
 }
@@ -454,7 +473,7 @@
     }
 
     irq->status &= ~XICS_STATUS_MASKED_PENDING;
-    icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
+    icp_irq(ics->xics, irq->server, srcno + ics->offset, irq->priority);
 }
 
 static void write_xive_lsi(ICSState *ics, int srcno)
@@ -462,8 +481,8 @@
     resend_lsi(ics, srcno);
 }
 
-static void ics_write_xive(ICSState *ics, int nr, int server,
-                           uint8_t priority, uint8_t saved_priority)
+void ics_write_xive(ICSState *ics, int nr, int server,
+                    uint8_t priority, uint8_t saved_priority)
 {
     int srcno = nr - ics->offset;
     ICSIRQState *irq = ics->irqs + srcno;
@@ -539,8 +558,8 @@
 {
     int i;
 
-    for (i = 0; i < ics->icp->nr_servers; i++) {
-        icp_resend(ics->icp, i);
+    for (i = 0; i < ics->xics->nr_servers; i++) {
+        icp_resend(ics->xics, i);
     }
 
     return 0;
@@ -640,14 +659,14 @@
 /*
  * Exported functions
  */
-static int xics_find_source(XICSState *icp, int irq)
+int xics_find_source(XICSState *xics, int irq)
 {
     int sources = 1;
     int src;
 
     /* FIXME: implement multiple sources */
     for (src = 0; src < sources; ++src) {
-        ICSState *ics = &icp->ics[src];
+        ICSState *ics = &xics->ics[src];
         if (ics_valid_irq(ics, irq)) {
             return src;
         }
@@ -656,19 +675,19 @@
     return -1;
 }
 
-qemu_irq xics_get_qirq(XICSState *icp, int irq)
+qemu_irq xics_get_qirq(XICSState *xics, int irq)
 {
-    int src = xics_find_source(icp, irq);
+    int src = xics_find_source(xics, irq);
 
     if (src >= 0) {
-        ICSState *ics = &icp->ics[src];
+        ICSState *ics = &xics->ics[src];
         return ics->qirqs[irq - ics->offset];
     }
 
     return NULL;
 }
 
-static void ics_set_irq_type(ICSState *ics, int srcno, bool lsi)
+void ics_set_irq_type(ICSState *ics, int srcno, bool lsi)
 {
     assert(!(ics->irqs[srcno].flags & XICS_FLAGS_IRQ_MASK));
 
@@ -676,407 +695,9 @@
         lsi ? XICS_FLAGS_IRQ_LSI : XICS_FLAGS_IRQ_MSI;
 }
 
-void xics_set_irq_type(XICSState *icp, int irq, bool lsi)
-{
-    int src = xics_find_source(icp, irq);
-    ICSState *ics;
-
-    assert(src >= 0);
-
-    ics = &icp->ics[src];
-    ics_set_irq_type(ics, irq - ics->offset, lsi);
-}
-
-#define ICS_IRQ_FREE(ics, srcno)   \
-    (!((ics)->irqs[(srcno)].flags & (XICS_FLAGS_IRQ_MASK)))
-
-static int ics_find_free_block(ICSState *ics, int num, int alignnum)
-{
-    int first, i;
-
-    for (first = 0; first < ics->nr_irqs; first += alignnum) {
-        if (num > (ics->nr_irqs - first)) {
-            return -1;
-        }
-        for (i = first; i < first + num; ++i) {
-            if (!ICS_IRQ_FREE(ics, i)) {
-                break;
-            }
-        }
-        if (i == (first + num)) {
-            return first;
-        }
-    }
-
-    return -1;
-}
-
-int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi)
-{
-    ICSState *ics = &icp->ics[src];
-    int irq;
-
-    if (irq_hint) {
-        assert(src == xics_find_source(icp, irq_hint));
-        if (!ICS_IRQ_FREE(ics, irq_hint - ics->offset)) {
-            trace_xics_alloc_failed_hint(src, irq_hint);
-            return -1;
-        }
-        irq = irq_hint;
-    } else {
-        irq = ics_find_free_block(ics, 1, 1);
-        if (irq < 0) {
-            trace_xics_alloc_failed_no_left(src);
-            return -1;
-        }
-        irq += ics->offset;
-    }
-
-    ics_set_irq_type(ics, irq - ics->offset, lsi);
-    trace_xics_alloc(src, irq);
-
-    return irq;
-}
-
-/*
- * Allocate block of consequtive IRQs, returns a number of the first.
- * If align==true, aligns the first IRQ number to num.
- */
-int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align)
-{
-    int i, first = -1;
-    ICSState *ics = &icp->ics[src];
-
-    assert(src == 0);
-    /*
-     * MSIMesage::data is used for storing VIRQ so
-     * it has to be aligned to num to support multiple
-     * MSI vectors. MSI-X is not affected by this.
-     * The hint is used for the first IRQ, the rest should
-     * be allocated continuously.
-     */
-    if (align) {
-        assert((num == 1) || (num == 2) || (num == 4) ||
-               (num == 8) || (num == 16) || (num == 32));
-        first = ics_find_free_block(ics, num, num);
-    } else {
-        first = ics_find_free_block(ics, num, 1);
-    }
-
-    if (first >= 0) {
-        for (i = first; i < first + num; ++i) {
-            ics_set_irq_type(ics, i, lsi);
-        }
-    }
-    first += ics->offset;
-
-    trace_xics_alloc_block(src, first, num, lsi, align);
-
-    return first;
-}
-
-static void ics_free(ICSState *ics, int srcno, int num)
-{
-    int i;
-
-    for (i = srcno; i < srcno + num; ++i) {
-        if (ICS_IRQ_FREE(ics, i)) {
-            trace_xics_ics_free_warn(ics - ics->icp->ics, i + ics->offset);
-        }
-        memset(&ics->irqs[i], 0, sizeof(ICSIRQState));
-    }
-}
-
-void xics_free(XICSState *icp, int irq, int num)
-{
-    int src = xics_find_source(icp, irq);
-
-    if (src >= 0) {
-        ICSState *ics = &icp->ics[src];
-
-        /* FIXME: implement multiple sources */
-        assert(src == 0);
-
-        trace_xics_ics_free(ics - icp->ics, irq, num);
-        ics_free(ics, irq - ics->offset, num);
-    }
-}
-
-/*
- * Guest interfaces
- */
-
-static target_ulong h_cppr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                           target_ulong opcode, target_ulong *args)
-{
-    CPUState *cs = CPU(cpu);
-    target_ulong cppr = args[0];
-
-    icp_set_cppr(spapr->icp, cs->cpu_index, cppr);
-    return H_SUCCESS;
-}
-
-static target_ulong h_ipi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                          target_ulong opcode, target_ulong *args)
-{
-    target_ulong server = get_cpu_index_by_dt_id(args[0]);
-    target_ulong mfrr = args[1];
-
-    if (server >= spapr->icp->nr_servers) {
-        return H_PARAMETER;
-    }
-
-    icp_set_mfrr(spapr->icp, server, mfrr);
-    return H_SUCCESS;
-}
-
-static target_ulong h_xirr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                           target_ulong opcode, target_ulong *args)
-{
-    CPUState *cs = CPU(cpu);
-    uint32_t xirr = icp_accept(spapr->icp->ss + cs->cpu_index);
-
-    args[0] = xirr;
-    return H_SUCCESS;
-}
-
-static target_ulong h_xirr_x(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                             target_ulong opcode, target_ulong *args)
-{
-    CPUState *cs = CPU(cpu);
-    ICPState *ss = &spapr->icp->ss[cs->cpu_index];
-    uint32_t xirr = icp_accept(ss);
-
-    args[0] = xirr;
-    args[1] = cpu_get_real_ticks();
-    return H_SUCCESS;
-}
-
-static target_ulong h_eoi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                          target_ulong opcode, target_ulong *args)
-{
-    CPUState *cs = CPU(cpu);
-    target_ulong xirr = args[0];
-
-    icp_eoi(spapr->icp, cs->cpu_index, xirr);
-    return H_SUCCESS;
-}
-
-static target_ulong h_ipoll(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                            target_ulong opcode, target_ulong *args)
-{
-    CPUState *cs = CPU(cpu);
-    ICPState *ss = &spapr->icp->ss[cs->cpu_index];
-
-    args[0] = ss->xirr;
-    args[1] = ss->mfrr;
-
-    return H_SUCCESS;
-}
-
-static void rtas_set_xive(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                          uint32_t token,
-                          uint32_t nargs, target_ulong args,
-                          uint32_t nret, target_ulong rets)
-{
-    ICSState *ics = spapr->icp->ics;
-    uint32_t nr, server, priority;
-
-    if ((nargs != 3) || (nret != 1)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    nr = rtas_ld(args, 0);
-    server = get_cpu_index_by_dt_id(rtas_ld(args, 1));
-    priority = rtas_ld(args, 2);
-
-    if (!ics_valid_irq(ics, nr) || (server >= ics->icp->nr_servers)
-        || (priority > 0xff)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    ics_write_xive(ics, nr, server, priority, priority);
-
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-}
-
-static void rtas_get_xive(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                          uint32_t token,
-                          uint32_t nargs, target_ulong args,
-                          uint32_t nret, target_ulong rets)
-{
-    ICSState *ics = spapr->icp->ics;
-    uint32_t nr;
-
-    if ((nargs != 1) || (nret != 3)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    nr = rtas_ld(args, 0);
-
-    if (!ics_valid_irq(ics, nr)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-    rtas_st(rets, 1, ics->irqs[nr - ics->offset].server);
-    rtas_st(rets, 2, ics->irqs[nr - ics->offset].priority);
-}
-
-static void rtas_int_off(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                         uint32_t token,
-                         uint32_t nargs, target_ulong args,
-                         uint32_t nret, target_ulong rets)
-{
-    ICSState *ics = spapr->icp->ics;
-    uint32_t nr;
-
-    if ((nargs != 1) || (nret != 1)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    nr = rtas_ld(args, 0);
-
-    if (!ics_valid_irq(ics, nr)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server, 0xff,
-                   ics->irqs[nr - ics->offset].priority);
-
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-}
-
-static void rtas_int_on(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                        uint32_t token,
-                        uint32_t nargs, target_ulong args,
-                        uint32_t nret, target_ulong rets)
-{
-    ICSState *ics = spapr->icp->ics;
-    uint32_t nr;
-
-    if ((nargs != 1) || (nret != 1)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    nr = rtas_ld(args, 0);
-
-    if (!ics_valid_irq(ics, nr)) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server,
-                   ics->irqs[nr - ics->offset].saved_priority,
-                   ics->irqs[nr - ics->offset].saved_priority);
-
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-}
-
-/*
- * XICS
- */
-
-static void xics_set_nr_irqs(XICSState *icp, uint32_t nr_irqs, Error **errp)
-{
-    icp->nr_irqs = icp->ics->nr_irqs = nr_irqs;
-}
-
-static void xics_set_nr_servers(XICSState *icp, uint32_t nr_servers,
-                                Error **errp)
-{
-    int i;
-
-    icp->nr_servers = nr_servers;
-
-    icp->ss = g_malloc0(icp->nr_servers*sizeof(ICPState));
-    for (i = 0; i < icp->nr_servers; i++) {
-        char buffer[32];
-        object_initialize(&icp->ss[i], sizeof(icp->ss[i]), TYPE_ICP);
-        snprintf(buffer, sizeof(buffer), "icp[%d]", i);
-        object_property_add_child(OBJECT(icp), buffer, OBJECT(&icp->ss[i]),
-                                  errp);
-    }
-}
-
-static void xics_realize(DeviceState *dev, Error **errp)
-{
-    XICSState *icp = XICS(dev);
-    Error *error = NULL;
-    int i;
-
-    if (!icp->nr_servers) {
-        error_setg(errp, "Number of servers needs to be greater 0");
-        return;
-    }
-
-    /* Registration of global state belongs into realize */
-    spapr_rtas_register(RTAS_IBM_SET_XIVE, "ibm,set-xive", rtas_set_xive);
-    spapr_rtas_register(RTAS_IBM_GET_XIVE, "ibm,get-xive", rtas_get_xive);
-    spapr_rtas_register(RTAS_IBM_INT_OFF, "ibm,int-off", rtas_int_off);
-    spapr_rtas_register(RTAS_IBM_INT_ON, "ibm,int-on", rtas_int_on);
-
-    spapr_register_hypercall(H_CPPR, h_cppr);
-    spapr_register_hypercall(H_IPI, h_ipi);
-    spapr_register_hypercall(H_XIRR, h_xirr);
-    spapr_register_hypercall(H_XIRR_X, h_xirr_x);
-    spapr_register_hypercall(H_EOI, h_eoi);
-    spapr_register_hypercall(H_IPOLL, h_ipoll);
-
-    object_property_set_bool(OBJECT(icp->ics), true, "realized", &error);
-    if (error) {
-        error_propagate(errp, error);
-        return;
-    }
-
-    for (i = 0; i < icp->nr_servers; i++) {
-        object_property_set_bool(OBJECT(&icp->ss[i]), true, "realized", &error);
-        if (error) {
-            error_propagate(errp, error);
-            return;
-        }
-    }
-}
-
-static void xics_initfn(Object *obj)
-{
-    XICSState *xics = XICS(obj);
-
-    xics->ics = ICS(object_new(TYPE_ICS));
-    object_property_add_child(obj, "ics", OBJECT(xics->ics), NULL);
-    xics->ics->icp = xics;
-}
-
-static void xics_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-    XICSStateClass *xsc = XICS_CLASS(oc);
-
-    dc->realize = xics_realize;
-    xsc->set_nr_irqs = xics_set_nr_irqs;
-    xsc->set_nr_servers = xics_set_nr_servers;
-}
-
-static const TypeInfo xics_info = {
-    .name          = TYPE_XICS,
-    .parent        = TYPE_XICS_COMMON,
-    .instance_size = sizeof(XICSState),
-    .class_size = sizeof(XICSStateClass),
-    .class_init    = xics_class_init,
-    .instance_init = xics_initfn,
-};
-
 static void xics_register_types(void)
 {
     type_register_static(&xics_common_info);
-    type_register_static(&xics_info);
     type_register_static(&ics_info);
     type_register_static(&icp_info);
 }
diff --git a/hw/intc/xics_kvm.c b/hw/intc/xics_kvm.c
index 20b19e9..edbd62f 100644
--- a/hw/intc/xics_kvm.c
+++ b/hw/intc/xics_kvm.c
@@ -25,8 +25,13 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "trace.h"
+#include "sysemu/kvm.h"
 #include "hw/ppc/spapr.h"
 #include "hw/ppc/xics.h"
 #include "kvm_ppc.h"
@@ -109,8 +114,10 @@
     icp->pending_priority = 0xff;
     icp->mfrr = 0xff;
 
-    /* Make all outputs are deasserted */
-    qemu_set_irq(icp->output, 0);
+    /* Make all outputs as deasserted only if the CPU thread is in use */
+    if (icp->output) {
+        qemu_set_irq(icp->output, 0);
+    }
 
     icp_set_kvm_state(icp, 1);
 }
@@ -138,7 +145,7 @@
  */
 static void ics_get_kvm_state(ICSState *ics)
 {
-    KVMXICSState *icpkvm = KVM_XICS(ics->icp);
+    KVMXICSState *xicskvm = XICS_SPAPR_KVM(ics->xics);
     uint64_t state;
     struct kvm_device_attr attr = {
         .flags = 0,
@@ -153,7 +160,7 @@
 
         attr.attr = i + ics->offset;
 
-        ret = ioctl(icpkvm->kernel_xics_fd, KVM_GET_DEVICE_ATTR, &attr);
+        ret = ioctl(xicskvm->kernel_xics_fd, KVM_GET_DEVICE_ATTR, &attr);
         if (ret != 0) {
             error_report("Unable to retrieve KVM interrupt controller state"
                     " for IRQ %d: %s", i + ics->offset, strerror(errno));
@@ -197,7 +204,7 @@
 
 static int ics_set_kvm_state(ICSState *ics, int version_id)
 {
-    KVMXICSState *icpkvm = KVM_XICS(ics->icp);
+    KVMXICSState *xicskvm = XICS_SPAPR_KVM(ics->xics);
     uint64_t state;
     struct kvm_device_attr attr = {
         .flags = 0,
@@ -231,7 +238,7 @@
             }
         }
 
-        ret = ioctl(icpkvm->kernel_xics_fd, KVM_SET_DEVICE_ATTR, &attr);
+        ret = ioctl(xicskvm->kernel_xics_fd, KVM_SET_DEVICE_ATTR, &attr);
         if (ret != 0) {
             error_report("Unable to restore KVM interrupt controller state"
                     " for IRQs %d: %s", i + ics->offset, strerror(errno));
@@ -317,58 +324,68 @@
 /*
  * XICS-KVM
  */
-static void xics_kvm_cpu_setup(XICSState *icp, PowerPCCPU *cpu)
+static void xics_kvm_cpu_setup(XICSState *xics, PowerPCCPU *cpu)
 {
     CPUState *cs;
     ICPState *ss;
-    KVMXICSState *icpkvm = KVM_XICS(icp);
+    KVMXICSState *xicskvm = XICS_SPAPR_KVM(xics);
 
     cs = CPU(cpu);
-    ss = &icp->ss[cs->cpu_index];
+    ss = &xics->ss[cs->cpu_index];
 
-    assert(cs->cpu_index < icp->nr_servers);
-    if (icpkvm->kernel_xics_fd == -1) {
+    assert(cs->cpu_index < xics->nr_servers);
+    if (xicskvm->kernel_xics_fd == -1) {
         abort();
     }
 
-    if (icpkvm->kernel_xics_fd != -1) {
+    /*
+     * If we are reusing a parked vCPU fd corresponding to the CPU
+     * which was hot-removed earlier we don't have to renable
+     * KVM_CAP_IRQ_XICS capability again.
+     */
+    if (ss->cap_irq_xics_enabled) {
+        return;
+    }
+
+    if (xicskvm->kernel_xics_fd != -1) {
         int ret;
 
-        ss->cs = cs;
-
         ret = kvm_vcpu_enable_cap(cs, KVM_CAP_IRQ_XICS, 0,
-                                  icpkvm->kernel_xics_fd, kvm_arch_vcpu_id(cs));
+                                  xicskvm->kernel_xics_fd,
+                                  kvm_arch_vcpu_id(cs));
         if (ret < 0) {
             error_report("Unable to connect CPU%ld to kernel XICS: %s",
                     kvm_arch_vcpu_id(cs), strerror(errno));
             exit(1);
         }
+        ss->cap_irq_xics_enabled = true;
     }
 }
 
-static void xics_kvm_set_nr_irqs(XICSState *icp, uint32_t nr_irqs, Error **errp)
+static void xics_kvm_set_nr_irqs(XICSState *xics, uint32_t nr_irqs,
+                                 Error **errp)
 {
-    icp->nr_irqs = icp->ics->nr_irqs = nr_irqs;
+    xics->nr_irqs = xics->ics->nr_irqs = nr_irqs;
 }
 
-static void xics_kvm_set_nr_servers(XICSState *icp, uint32_t nr_servers,
+static void xics_kvm_set_nr_servers(XICSState *xics, uint32_t nr_servers,
                                     Error **errp)
 {
     int i;
 
-    icp->nr_servers = nr_servers;
+    xics->nr_servers = nr_servers;
 
-    icp->ss = g_malloc0(icp->nr_servers*sizeof(ICPState));
-    for (i = 0; i < icp->nr_servers; i++) {
+    xics->ss = g_malloc0(xics->nr_servers * sizeof(ICPState));
+    for (i = 0; i < xics->nr_servers; i++) {
         char buffer[32];
-        object_initialize(&icp->ss[i], sizeof(icp->ss[i]), TYPE_KVM_ICP);
+        object_initialize(&xics->ss[i], sizeof(xics->ss[i]), TYPE_KVM_ICP);
         snprintf(buffer, sizeof(buffer), "icp[%d]", i);
-        object_property_add_child(OBJECT(icp), buffer, OBJECT(&icp->ss[i]),
+        object_property_add_child(OBJECT(xics), buffer, OBJECT(&xics->ss[i]),
                                   errp);
     }
 }
 
-static void rtas_dummy(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_dummy(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                        uint32_t token,
                        uint32_t nargs, target_ulong args,
                        uint32_t nret, target_ulong rets)
@@ -379,8 +396,8 @@
 
 static void xics_kvm_realize(DeviceState *dev, Error **errp)
 {
-    KVMXICSState *icpkvm = KVM_XICS(dev);
-    XICSState *icp = XICS_COMMON(dev);
+    KVMXICSState *xicskvm = XICS_SPAPR_KVM(dev);
+    XICSState *xics = XICS_COMMON(dev);
     int i, rc;
     Error *error = NULL;
     struct kvm_create_device xics_create_device = {
@@ -430,17 +447,18 @@
         goto fail;
     }
 
-    icpkvm->kernel_xics_fd = xics_create_device.fd;
+    xicskvm->kernel_xics_fd = xics_create_device.fd;
 
-    object_property_set_bool(OBJECT(icp->ics), true, "realized", &error);
+    object_property_set_bool(OBJECT(xics->ics), true, "realized", &error);
     if (error) {
         error_propagate(errp, error);
         goto fail;
     }
 
-    assert(icp->nr_servers);
-    for (i = 0; i < icp->nr_servers; i++) {
-        object_property_set_bool(OBJECT(&icp->ss[i]), true, "realized", &error);
+    assert(xics->nr_servers);
+    for (i = 0; i < xics->nr_servers; i++) {
+        object_property_set_bool(OBJECT(&xics->ss[i]), true, "realized",
+                                 &error);
         if (error) {
             error_propagate(errp, error);
             goto fail;
@@ -448,7 +466,6 @@
     }
 
     kvm_kernel_irqchip = true;
-    kvm_irqfds_allowed = true;
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_direct_mapping = true;
 
@@ -467,7 +484,7 @@
 
     xics->ics = ICS(object_new(TYPE_KVM_ICS));
     object_property_add_child(obj, "ics", OBJECT(xics->ics), NULL);
-    xics->ics->icp = xics;
+    xics->ics->xics = xics;
 }
 
 static void xics_kvm_class_init(ObjectClass *oc, void *data)
@@ -481,8 +498,8 @@
     xsc->set_nr_servers = xics_kvm_set_nr_servers;
 }
 
-static const TypeInfo xics_kvm_info = {
-    .name          = TYPE_KVM_XICS,
+static const TypeInfo xics_spapr_kvm_info = {
+    .name          = TYPE_XICS_SPAPR_KVM,
     .parent        = TYPE_XICS_COMMON,
     .instance_size = sizeof(KVMXICSState),
     .class_init    = xics_kvm_class_init,
@@ -491,7 +508,7 @@
 
 static void xics_kvm_register_types(void)
 {
-    type_register_static(&xics_kvm_info);
+    type_register_static(&xics_spapr_kvm_info);
     type_register_static(&ics_kvm_info);
     type_register_static(&icp_kvm_info);
 }
diff --git a/hw/intc/xics_spapr.c b/hw/intc/xics_spapr.c
new file mode 100644
index 0000000..618826d
--- /dev/null
+++ b/hw/intc/xics_spapr.c
@@ -0,0 +1,434 @@
+/*
+ * QEMU PowerPC pSeries Logical Partition (aka sPAPR) hardware System Emulator
+ *
+ * PAPR Virtualized Interrupt System, aka ICS/ICP aka xics
+ *
+ * Copyright (c) 2010,2011 David Gibson, IBM Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "hw/hw.h"
+#include "trace.h"
+#include "qemu/timer.h"
+#include "hw/ppc/spapr.h"
+#include "hw/ppc/xics.h"
+#include "qapi/visitor.h"
+#include "qapi/error.h"
+
+/*
+ * Guest interfaces
+ */
+
+static target_ulong h_cppr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                           target_ulong opcode, target_ulong *args)
+{
+    CPUState *cs = CPU(cpu);
+    target_ulong cppr = args[0];
+
+    icp_set_cppr(spapr->xics, cs->cpu_index, cppr);
+    return H_SUCCESS;
+}
+
+static target_ulong h_ipi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                          target_ulong opcode, target_ulong *args)
+{
+    target_ulong server = xics_get_cpu_index_by_dt_id(args[0]);
+    target_ulong mfrr = args[1];
+
+    if (server >= spapr->xics->nr_servers) {
+        return H_PARAMETER;
+    }
+
+    icp_set_mfrr(spapr->xics, server, mfrr);
+    return H_SUCCESS;
+}
+
+static target_ulong h_xirr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                           target_ulong opcode, target_ulong *args)
+{
+    CPUState *cs = CPU(cpu);
+    uint32_t xirr = icp_accept(spapr->xics->ss + cs->cpu_index);
+
+    args[0] = xirr;
+    return H_SUCCESS;
+}
+
+static target_ulong h_xirr_x(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                             target_ulong opcode, target_ulong *args)
+{
+    CPUState *cs = CPU(cpu);
+    ICPState *ss = &spapr->xics->ss[cs->cpu_index];
+    uint32_t xirr = icp_accept(ss);
+
+    args[0] = xirr;
+    args[1] = cpu_get_host_ticks();
+    return H_SUCCESS;
+}
+
+static target_ulong h_eoi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                          target_ulong opcode, target_ulong *args)
+{
+    CPUState *cs = CPU(cpu);
+    target_ulong xirr = args[0];
+
+    icp_eoi(spapr->xics, cs->cpu_index, xirr);
+    return H_SUCCESS;
+}
+
+static target_ulong h_ipoll(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                            target_ulong opcode, target_ulong *args)
+{
+    CPUState *cs = CPU(cpu);
+    uint32_t mfrr;
+    uint32_t xirr = icp_ipoll(spapr->xics->ss + cs->cpu_index, &mfrr);
+
+    args[0] = xirr;
+    args[1] = mfrr;
+
+    return H_SUCCESS;
+}
+
+static void rtas_set_xive(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                          uint32_t token,
+                          uint32_t nargs, target_ulong args,
+                          uint32_t nret, target_ulong rets)
+{
+    ICSState *ics = spapr->xics->ics;
+    uint32_t nr, server, priority;
+
+    if ((nargs != 3) || (nret != 1)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    nr = rtas_ld(args, 0);
+    server = xics_get_cpu_index_by_dt_id(rtas_ld(args, 1));
+    priority = rtas_ld(args, 2);
+
+    if (!ics_valid_irq(ics, nr) || (server >= ics->xics->nr_servers)
+        || (priority > 0xff)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    ics_write_xive(ics, nr, server, priority, priority);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+}
+
+static void rtas_get_xive(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                          uint32_t token,
+                          uint32_t nargs, target_ulong args,
+                          uint32_t nret, target_ulong rets)
+{
+    ICSState *ics = spapr->xics->ics;
+    uint32_t nr;
+
+    if ((nargs != 1) || (nret != 3)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    nr = rtas_ld(args, 0);
+
+    if (!ics_valid_irq(ics, nr)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, ics->irqs[nr - ics->offset].server);
+    rtas_st(rets, 2, ics->irqs[nr - ics->offset].priority);
+}
+
+static void rtas_int_off(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                         uint32_t token,
+                         uint32_t nargs, target_ulong args,
+                         uint32_t nret, target_ulong rets)
+{
+    ICSState *ics = spapr->xics->ics;
+    uint32_t nr;
+
+    if ((nargs != 1) || (nret != 1)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    nr = rtas_ld(args, 0);
+
+    if (!ics_valid_irq(ics, nr)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server, 0xff,
+                   ics->irqs[nr - ics->offset].priority);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+}
+
+static void rtas_int_on(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                        uint32_t token,
+                        uint32_t nargs, target_ulong args,
+                        uint32_t nret, target_ulong rets)
+{
+    ICSState *ics = spapr->xics->ics;
+    uint32_t nr;
+
+    if ((nargs != 1) || (nret != 1)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    nr = rtas_ld(args, 0);
+
+    if (!ics_valid_irq(ics, nr)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server,
+                   ics->irqs[nr - ics->offset].saved_priority,
+                   ics->irqs[nr - ics->offset].saved_priority);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+}
+
+static void xics_spapr_set_nr_irqs(XICSState *xics, uint32_t nr_irqs,
+                                   Error **errp)
+{
+    xics->nr_irqs = xics->ics->nr_irqs = nr_irqs;
+}
+
+static void xics_spapr_set_nr_servers(XICSState *xics, uint32_t nr_servers,
+                                      Error **errp)
+{
+    int i;
+
+    xics->nr_servers = nr_servers;
+
+    xics->ss = g_malloc0(xics->nr_servers * sizeof(ICPState));
+    for (i = 0; i < xics->nr_servers; i++) {
+        char buffer[32];
+        object_initialize(&xics->ss[i], sizeof(xics->ss[i]), TYPE_ICP);
+        snprintf(buffer, sizeof(buffer), "icp[%d]", i);
+        object_property_add_child(OBJECT(xics), buffer, OBJECT(&xics->ss[i]),
+                                  errp);
+    }
+}
+
+static void xics_spapr_realize(DeviceState *dev, Error **errp)
+{
+    XICSState *xics = XICS_SPAPR(dev);
+    Error *error = NULL;
+    int i;
+
+    if (!xics->nr_servers) {
+        error_setg(errp, "Number of servers needs to be greater 0");
+        return;
+    }
+
+    /* Registration of global state belongs into realize */
+    spapr_rtas_register(RTAS_IBM_SET_XIVE, "ibm,set-xive", rtas_set_xive);
+    spapr_rtas_register(RTAS_IBM_GET_XIVE, "ibm,get-xive", rtas_get_xive);
+    spapr_rtas_register(RTAS_IBM_INT_OFF, "ibm,int-off", rtas_int_off);
+    spapr_rtas_register(RTAS_IBM_INT_ON, "ibm,int-on", rtas_int_on);
+
+    spapr_register_hypercall(H_CPPR, h_cppr);
+    spapr_register_hypercall(H_IPI, h_ipi);
+    spapr_register_hypercall(H_XIRR, h_xirr);
+    spapr_register_hypercall(H_XIRR_X, h_xirr_x);
+    spapr_register_hypercall(H_EOI, h_eoi);
+    spapr_register_hypercall(H_IPOLL, h_ipoll);
+
+    object_property_set_bool(OBJECT(xics->ics), true, "realized", &error);
+    if (error) {
+        error_propagate(errp, error);
+        return;
+    }
+
+    for (i = 0; i < xics->nr_servers; i++) {
+        object_property_set_bool(OBJECT(&xics->ss[i]), true, "realized",
+                                 &error);
+        if (error) {
+            error_propagate(errp, error);
+            return;
+        }
+    }
+}
+
+static void xics_spapr_initfn(Object *obj)
+{
+    XICSState *xics = XICS_SPAPR(obj);
+
+    xics->ics = ICS(object_new(TYPE_ICS));
+    object_property_add_child(obj, "ics", OBJECT(xics->ics), NULL);
+    xics->ics->xics = xics;
+}
+
+static void xics_spapr_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    XICSStateClass *xsc = XICS_SPAPR_CLASS(oc);
+
+    dc->realize = xics_spapr_realize;
+    xsc->set_nr_irqs = xics_spapr_set_nr_irqs;
+    xsc->set_nr_servers = xics_spapr_set_nr_servers;
+}
+
+static const TypeInfo xics_spapr_info = {
+    .name          = TYPE_XICS_SPAPR,
+    .parent        = TYPE_XICS_COMMON,
+    .instance_size = sizeof(XICSState),
+    .class_size = sizeof(XICSStateClass),
+    .class_init    = xics_spapr_class_init,
+    .instance_init = xics_spapr_initfn,
+};
+
+#define ICS_IRQ_FREE(ics, srcno)   \
+    (!((ics)->irqs[(srcno)].flags & (XICS_FLAGS_IRQ_MASK)))
+
+static int ics_find_free_block(ICSState *ics, int num, int alignnum)
+{
+    int first, i;
+
+    for (first = 0; first < ics->nr_irqs; first += alignnum) {
+        if (num > (ics->nr_irqs - first)) {
+            return -1;
+        }
+        for (i = first; i < first + num; ++i) {
+            if (!ICS_IRQ_FREE(ics, i)) {
+                break;
+            }
+        }
+        if (i == (first + num)) {
+            return first;
+        }
+    }
+
+    return -1;
+}
+
+int xics_spapr_alloc(XICSState *xics, int src, int irq_hint, bool lsi,
+                     Error **errp)
+{
+    ICSState *ics = &xics->ics[src];
+    int irq;
+
+    if (irq_hint) {
+        assert(src == xics_find_source(xics, irq_hint));
+        if (!ICS_IRQ_FREE(ics, irq_hint - ics->offset)) {
+            error_setg(errp, "can't allocate IRQ %d: already in use", irq_hint);
+            return -1;
+        }
+        irq = irq_hint;
+    } else {
+        irq = ics_find_free_block(ics, 1, 1);
+        if (irq < 0) {
+            error_setg(errp, "can't allocate IRQ: no IRQ left");
+            return -1;
+        }
+        irq += ics->offset;
+    }
+
+    ics_set_irq_type(ics, irq - ics->offset, lsi);
+    trace_xics_alloc(src, irq);
+
+    return irq;
+}
+
+/*
+ * Allocate block of consecutive IRQs, and return the number of the first IRQ in
+ * the block. If align==true, aligns the first IRQ number to num.
+ */
+int xics_spapr_alloc_block(XICSState *xics, int src, int num, bool lsi,
+                           bool align, Error **errp)
+{
+    int i, first = -1;
+    ICSState *ics = &xics->ics[src];
+
+    assert(src == 0);
+    /*
+     * MSIMesage::data is used for storing VIRQ so
+     * it has to be aligned to num to support multiple
+     * MSI vectors. MSI-X is not affected by this.
+     * The hint is used for the first IRQ, the rest should
+     * be allocated continuously.
+     */
+    if (align) {
+        assert((num == 1) || (num == 2) || (num == 4) ||
+               (num == 8) || (num == 16) || (num == 32));
+        first = ics_find_free_block(ics, num, num);
+    } else {
+        first = ics_find_free_block(ics, num, 1);
+    }
+    if (first < 0) {
+        error_setg(errp, "can't find a free %d-IRQ block", num);
+        return -1;
+    }
+
+    if (first >= 0) {
+        for (i = first; i < first + num; ++i) {
+            ics_set_irq_type(ics, i, lsi);
+        }
+    }
+    first += ics->offset;
+
+    trace_xics_alloc_block(src, first, num, lsi, align);
+
+    return first;
+}
+
+static void ics_free(ICSState *ics, int srcno, int num)
+{
+    int i;
+
+    for (i = srcno; i < srcno + num; ++i) {
+        if (ICS_IRQ_FREE(ics, i)) {
+            trace_xics_ics_free_warn(ics - ics->xics->ics, i + ics->offset);
+        }
+        memset(&ics->irqs[i], 0, sizeof(ICSIRQState));
+    }
+}
+
+void xics_spapr_free(XICSState *xics, int irq, int num)
+{
+    int src = xics_find_source(xics, irq);
+
+    if (src >= 0) {
+        ICSState *ics = &xics->ics[src];
+
+        /* FIXME: implement multiple sources */
+        assert(src == 0);
+
+        trace_xics_ics_free(ics - xics->ics, irq, num);
+        ics_free(ics, irq - ics->offset, num);
+    }
+}
+
+static void xics_spapr_register_types(void)
+{
+    type_register_static(&xics_spapr_info);
+}
+
+type_init(xics_spapr_register_types)
diff --git a/hw/intc/xilinx_intc.c b/hw/intc/xilinx_intc.c
index 12804ab..9d8139b 100644
--- a/hw/intc/xilinx_intc.c
+++ b/hw/intc/xilinx_intc.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 
diff --git a/hw/ipack/ipack.c b/hw/ipack/ipack.c
index 59bfe286..6021e6d 100644
--- a/hw/ipack/ipack.c
+++ b/hw/ipack/ipack.c
@@ -2,12 +2,14 @@
  * QEMU IndustryPack emulation
  *
  * Copyright (C) 2012 Igalia, S.L.
- * Author: Alberto Garcia <agarcia@igalia.com>
+ * Author: Alberto Garcia <berto@igalia.com>
  *
  * This code is licensed under the GNU GPL v2 or (at your option) any
  * later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/ipack/ipack.h"
 
 IPackDevice *ipack_device_find(IPackBus *bus, int32_t slot)
diff --git a/hw/ipack/tpci200.c b/hw/ipack/tpci200.c
index b7031a0..4dfa6b3 100644
--- a/hw/ipack/tpci200.c
+++ b/hw/ipack/tpci200.c
@@ -2,16 +2,16 @@
  * QEMU TEWS TPCI200 IndustryPack carrier emulation
  *
  * Copyright (C) 2012 Igalia, S.L.
- * Author: Alberto Garcia <agarcia@igalia.com>
+ * Author: Alberto Garcia <berto@igalia.com>
  *
  * This code is licensed under the GNU GPL v2 or (at your option) any
  * later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/ipack/ipack.h"
 #include "hw/pci/pci.h"
 #include "qemu/bitops.h"
-#include <stdio.h>
 
 /* #define DEBUG_TPCI */
 
@@ -573,7 +573,7 @@
     }
 };
 
-static int tpci200_initfn(PCIDevice *pci_dev)
+static void tpci200_realize(PCIDevice *pci_dev, Error **errp)
 {
     TPCI200State *s = TPCI200(pci_dev);
     uint8_t *c = s->dev.config;
@@ -609,8 +609,6 @@
 
     ipack_bus_new_inplace(&s->bus, sizeof(s->bus), DEVICE(pci_dev), NULL,
                           N_MODULES, tpci200_set_irq);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_tpci200 = {
@@ -632,7 +630,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = tpci200_initfn;
+    k->realize = tpci200_realize;
     k->vendor_id = PCI_VENDOR_ID_TEWS;
     k->device_id = PCI_DEVICE_ID_TEWS_TPCI200;
     k->class_id = PCI_CLASS_BRIDGE_OTHER;
diff --git a/hw/ipmi/Makefile.objs b/hw/ipmi/Makefile.objs
new file mode 100644
index 0000000..a90318d
--- /dev/null
+++ b/hw/ipmi/Makefile.objs
@@ -0,0 +1,5 @@
+common-obj-$(CONFIG_IPMI) += ipmi.o
+common-obj-$(CONFIG_IPMI_LOCAL) += ipmi_bmc_sim.o
+common-obj-$(CONFIG_IPMI_LOCAL) += ipmi_bmc_extern.o
+common-obj-$(CONFIG_ISA_IPMI_KCS) += isa_ipmi_kcs.o
+common-obj-$(CONFIG_ISA_IPMI_BT) += isa_ipmi_bt.o
diff --git a/hw/ipmi/ipmi.c b/hw/ipmi/ipmi.c
new file mode 100644
index 0000000..f09f217
--- /dev/null
+++ b/hw/ipmi/ipmi.c
@@ -0,0 +1,131 @@
+/*
+ * QEMU IPMI emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/ipmi/ipmi.h"
+#include "sysemu/sysemu.h"
+#include "qmp-commands.h"
+#include "qom/object_interfaces.h"
+#include "qapi/visitor.h"
+
+static uint32_t ipmi_current_uuid = 1;
+
+uint32_t ipmi_next_uuid(void)
+{
+    return ipmi_current_uuid++;
+}
+
+static int ipmi_do_hw_op(IPMIInterface *s, enum ipmi_op op, int checkonly)
+{
+    switch (op) {
+    case IPMI_RESET_CHASSIS:
+        if (checkonly) {
+            return 0;
+        }
+        qemu_system_reset_request();
+        return 0;
+
+    case IPMI_POWEROFF_CHASSIS:
+        if (checkonly) {
+            return 0;
+        }
+        qemu_system_powerdown_request();
+        return 0;
+
+    case IPMI_SEND_NMI:
+        if (checkonly) {
+            return 0;
+        }
+        qmp_inject_nmi(NULL);
+        return 0;
+
+    case IPMI_POWERCYCLE_CHASSIS:
+    case IPMI_PULSE_DIAG_IRQ:
+    case IPMI_SHUTDOWN_VIA_ACPI_OVERTEMP:
+    case IPMI_POWERON_CHASSIS:
+    default:
+        return IPMI_CC_COMMAND_NOT_SUPPORTED;
+    }
+}
+
+static void ipmi_interface_class_init(ObjectClass *class, void *data)
+{
+    IPMIInterfaceClass *ik = IPMI_INTERFACE_CLASS(class);
+
+    ik->do_hw_op = ipmi_do_hw_op;
+}
+
+static TypeInfo ipmi_interface_type_info = {
+    .name = TYPE_IPMI_INTERFACE,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(IPMIInterfaceClass),
+    .class_init = ipmi_interface_class_init,
+};
+
+static void isa_ipmi_bmc_check(Object *obj, const char *name,
+                               Object *val, Error **errp)
+{
+    IPMIBmc *bmc = IPMI_BMC(val);
+
+    if (bmc->intf)
+        error_setg(errp, "BMC object is already in use");
+}
+
+void ipmi_bmc_find_and_link(Object *obj, Object **bmc)
+{
+    object_property_add_link(obj, "bmc", TYPE_IPMI_BMC, bmc,
+                             isa_ipmi_bmc_check,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+}
+
+static Property ipmi_bmc_properties[] = {
+    DEFINE_PROP_UINT8("slave_addr",  IPMIBmc, slave_addr, 0x20),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void bmc_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->props = ipmi_bmc_properties;
+}
+
+static TypeInfo ipmi_bmc_type_info = {
+    .name = TYPE_IPMI_BMC,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(IPMIBmc),
+    .abstract = true,
+    .class_size = sizeof(IPMIBmcClass),
+    .class_init = bmc_class_init,
+};
+
+static void ipmi_register_types(void)
+{
+    type_register_static(&ipmi_interface_type_info);
+    type_register_static(&ipmi_bmc_type_info);
+}
+
+type_init(ipmi_register_types)
diff --git a/hw/ipmi/ipmi_bmc_extern.c b/hw/ipmi/ipmi_bmc_extern.c
new file mode 100644
index 0000000..157879e
--- /dev/null
+++ b/hw/ipmi/ipmi_bmc_extern.c
@@ -0,0 +1,519 @@
+/*
+ * IPMI BMC external connection
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * This is designed to connect with OpenIPMI's lanserv serial interface
+ * using the "VM" connection type.  See that for details.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/timer.h"
+#include "sysemu/char.h"
+#include "sysemu/sysemu.h"
+#include "hw/ipmi/ipmi.h"
+
+#define VM_MSG_CHAR        0xA0 /* Marks end of message */
+#define VM_CMD_CHAR        0xA1 /* Marks end of a command */
+#define VM_ESCAPE_CHAR     0xAA /* Set bit 4 from the next byte to 0 */
+
+#define VM_PROTOCOL_VERSION        1
+#define VM_CMD_VERSION             0xff /* A version number byte follows */
+#define VM_CMD_NOATTN              0x00
+#define VM_CMD_ATTN                0x01
+#define VM_CMD_ATTN_IRQ            0x02
+#define VM_CMD_POWEROFF            0x03
+#define VM_CMD_RESET               0x04
+#define VM_CMD_ENABLE_IRQ          0x05 /* Enable/disable the messaging irq */
+#define VM_CMD_DISABLE_IRQ         0x06
+#define VM_CMD_SEND_NMI            0x07
+#define VM_CMD_CAPABILITIES        0x08
+#define   VM_CAPABILITIES_POWER    0x01
+#define   VM_CAPABILITIES_RESET    0x02
+#define   VM_CAPABILITIES_IRQ      0x04
+#define   VM_CAPABILITIES_NMI      0x08
+#define   VM_CAPABILITIES_ATTN     0x10
+#define VM_CMD_FORCEOFF            0x09
+
+#define TYPE_IPMI_BMC_EXTERN "ipmi-bmc-extern"
+#define IPMI_BMC_EXTERN(obj) OBJECT_CHECK(IPMIBmcExtern, (obj), \
+                                        TYPE_IPMI_BMC_EXTERN)
+typedef struct IPMIBmcExtern {
+    IPMIBmc parent;
+
+    CharDriverState *chr;
+
+    bool connected;
+
+    unsigned char inbuf[MAX_IPMI_MSG_SIZE + 2];
+    unsigned int inpos;
+    bool in_escape;
+    bool in_too_many;
+    bool waiting_rsp;
+    bool sending_cmd;
+
+    unsigned char outbuf[(MAX_IPMI_MSG_SIZE + 2) * 2 + 1];
+    unsigned int outpos;
+    unsigned int outlen;
+
+    struct QEMUTimer *extern_timer;
+
+    /* A reset event is pending to be sent upstream. */
+    bool send_reset;
+} IPMIBmcExtern;
+
+static int can_receive(void *opaque);
+static void receive(void *opaque, const uint8_t *buf, int size);
+static void chr_event(void *opaque, int event);
+
+static unsigned char
+ipmb_checksum(const unsigned char *data, int size, unsigned char start)
+{
+        unsigned char csum = start;
+
+        for (; size > 0; size--, data++) {
+                csum += *data;
+        }
+        return csum;
+}
+
+static void continue_send(IPMIBmcExtern *ibe)
+{
+    if (ibe->outlen == 0) {
+        goto check_reset;
+    }
+ send:
+    ibe->outpos += qemu_chr_fe_write(ibe->chr, ibe->outbuf + ibe->outpos,
+                                     ibe->outlen - ibe->outpos);
+    if (ibe->outpos < ibe->outlen) {
+        /* Not fully transmitted, try again in a 10ms */
+        timer_mod_ns(ibe->extern_timer,
+                     qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 10000000);
+    } else {
+        /* Sent */
+        ibe->outlen = 0;
+        ibe->outpos = 0;
+        if (!ibe->sending_cmd) {
+            ibe->waiting_rsp = true;
+        } else {
+            ibe->sending_cmd = false;
+        }
+    check_reset:
+        if (ibe->connected && ibe->send_reset) {
+            /* Send the reset */
+            ibe->outbuf[0] = VM_CMD_RESET;
+            ibe->outbuf[1] = VM_CMD_CHAR;
+            ibe->outlen = 2;
+            ibe->outpos = 0;
+            ibe->send_reset = false;
+            ibe->sending_cmd = true;
+            goto send;
+        }
+
+        if (ibe->waiting_rsp) {
+            /* Make sure we get a response within 4 seconds. */
+            timer_mod_ns(ibe->extern_timer,
+                         qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 4000000000ULL);
+        }
+    }
+    return;
+}
+
+static void extern_timeout(void *opaque)
+{
+    IPMIBmcExtern *ibe = opaque;
+    IPMIInterface *s = ibe->parent.intf;
+
+    if (ibe->connected) {
+        if (ibe->waiting_rsp && (ibe->outlen == 0)) {
+            IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+            /* The message response timed out, return an error. */
+            ibe->waiting_rsp = false;
+            ibe->inbuf[1] = ibe->outbuf[1] | 0x04;
+            ibe->inbuf[2] = ibe->outbuf[2];
+            ibe->inbuf[3] = IPMI_CC_TIMEOUT;
+            k->handle_rsp(s, ibe->outbuf[0], ibe->inbuf + 1, 3);
+        } else {
+            continue_send(ibe);
+        }
+    }
+}
+
+static void addchar(IPMIBmcExtern *ibe, unsigned char ch)
+{
+    switch (ch) {
+    case VM_MSG_CHAR:
+    case VM_CMD_CHAR:
+    case VM_ESCAPE_CHAR:
+        ibe->outbuf[ibe->outlen] = VM_ESCAPE_CHAR;
+        ibe->outlen++;
+        ch |= 0x10;
+        /* No break */
+
+    default:
+        ibe->outbuf[ibe->outlen] = ch;
+        ibe->outlen++;
+    }
+}
+
+static void ipmi_bmc_extern_handle_command(IPMIBmc *b,
+                                       uint8_t *cmd, unsigned int cmd_len,
+                                       unsigned int max_cmd_len,
+                                       uint8_t msg_id)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(b);
+    IPMIInterface *s = ibe->parent.intf;
+    uint8_t err = 0, csum;
+    unsigned int i;
+
+    if (ibe->outlen) {
+        /* We already have a command queued.  Shouldn't ever happen. */
+        fprintf(stderr, "IPMI KCS: Got command when not finished with the"
+                " previous command\n");
+        abort();
+    }
+
+    /* If it's too short or it was truncated, return an error. */
+    if (cmd_len < 2) {
+        err = IPMI_CC_REQUEST_DATA_LENGTH_INVALID;
+    } else if ((cmd_len > max_cmd_len) || (cmd_len > MAX_IPMI_MSG_SIZE)) {
+        err = IPMI_CC_REQUEST_DATA_TRUNCATED;
+    } else if (!ibe->connected) {
+        err = IPMI_CC_BMC_INIT_IN_PROGRESS;
+    }
+    if (err) {
+        IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+        unsigned char rsp[3];
+        rsp[0] = cmd[0] | 0x04;
+        rsp[1] = cmd[1];
+        rsp[2] = err;
+        ibe->waiting_rsp = false;
+        k->handle_rsp(s, msg_id, rsp, 3);
+        goto out;
+    }
+
+    addchar(ibe, msg_id);
+    for (i = 0; i < cmd_len; i++) {
+        addchar(ibe, cmd[i]);
+    }
+    csum = ipmb_checksum(&msg_id, 1, 0);
+    addchar(ibe, -ipmb_checksum(cmd, cmd_len, csum));
+
+    ibe->outbuf[ibe->outlen] = VM_MSG_CHAR;
+    ibe->outlen++;
+
+    /* Start the transmit */
+    continue_send(ibe);
+
+ out:
+    return;
+}
+
+static void handle_hw_op(IPMIBmcExtern *ibe, unsigned char hw_op)
+{
+    IPMIInterface *s = ibe->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    switch (hw_op) {
+    case VM_CMD_VERSION:
+        /* We only support one version at this time. */
+        break;
+
+    case VM_CMD_NOATTN:
+        k->set_atn(s, 0, 0);
+        break;
+
+    case VM_CMD_ATTN:
+        k->set_atn(s, 1, 0);
+        break;
+
+    case VM_CMD_ATTN_IRQ:
+        k->set_atn(s, 1, 1);
+        break;
+
+    case VM_CMD_POWEROFF:
+        k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 0);
+        break;
+
+    case VM_CMD_RESET:
+        k->do_hw_op(s, IPMI_RESET_CHASSIS, 0);
+        break;
+
+    case VM_CMD_ENABLE_IRQ:
+        k->set_irq_enable(s, 1);
+        break;
+
+    case VM_CMD_DISABLE_IRQ:
+        k->set_irq_enable(s, 0);
+        break;
+
+    case VM_CMD_SEND_NMI:
+        k->do_hw_op(s, IPMI_SEND_NMI, 0);
+        break;
+
+    case VM_CMD_FORCEOFF:
+        qemu_system_shutdown_request();
+        break;
+    }
+}
+
+static void handle_msg(IPMIBmcExtern *ibe)
+{
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(ibe->parent.intf);
+
+    if (ibe->in_escape) {
+        ipmi_debug("msg escape not ended\n");
+        return;
+    }
+    if (ibe->inpos < 5) {
+        ipmi_debug("msg too short\n");
+        return;
+    }
+    if (ibe->in_too_many) {
+        ibe->inbuf[3] = IPMI_CC_REQUEST_DATA_TRUNCATED;
+        ibe->inpos = 4;
+    } else if (ipmb_checksum(ibe->inbuf, ibe->inpos, 0) != 0) {
+        ipmi_debug("msg checksum failure\n");
+        return;
+    } else {
+        ibe->inpos--; /* Remove checkum */
+    }
+
+    timer_del(ibe->extern_timer);
+    ibe->waiting_rsp = false;
+    k->handle_rsp(ibe->parent.intf, ibe->inbuf[0], ibe->inbuf + 1, ibe->inpos - 1);
+}
+
+static int can_receive(void *opaque)
+{
+    return 1;
+}
+
+static void receive(void *opaque, const uint8_t *buf, int size)
+{
+    IPMIBmcExtern *ibe = opaque;
+    int i;
+    unsigned char hw_op;
+
+    for (i = 0; i < size; i++) {
+        unsigned char ch = buf[i];
+
+        switch (ch) {
+        case VM_MSG_CHAR:
+            handle_msg(ibe);
+            ibe->in_too_many = false;
+            ibe->inpos = 0;
+            break;
+
+        case VM_CMD_CHAR:
+            if (ibe->in_too_many) {
+                ipmi_debug("cmd in too many\n");
+                ibe->in_too_many = false;
+                ibe->inpos = 0;
+                break;
+            }
+            if (ibe->in_escape) {
+                ipmi_debug("cmd in escape\n");
+                ibe->in_too_many = false;
+                ibe->inpos = 0;
+                ibe->in_escape = false;
+                break;
+            }
+            ibe->in_too_many = false;
+            if (ibe->inpos < 1) {
+                break;
+            }
+            hw_op = ibe->inbuf[0];
+            ibe->inpos = 0;
+            goto out_hw_op;
+            break;
+
+        case VM_ESCAPE_CHAR:
+            ibe->in_escape = true;
+            break;
+
+        default:
+            if (ibe->in_escape) {
+                ch &= ~0x10;
+                ibe->in_escape = false;
+            }
+            if (ibe->in_too_many) {
+                break;
+            }
+            if (ibe->inpos >= sizeof(ibe->inbuf)) {
+                ibe->in_too_many = true;
+                break;
+            }
+            ibe->inbuf[ibe->inpos] = ch;
+            ibe->inpos++;
+            break;
+        }
+    }
+    return;
+
+ out_hw_op:
+    handle_hw_op(ibe, hw_op);
+}
+
+static void chr_event(void *opaque, int event)
+{
+    IPMIBmcExtern *ibe = opaque;
+    IPMIInterface *s = ibe->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    unsigned char v;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        ibe->connected = true;
+        ibe->outpos = 0;
+        ibe->outlen = 0;
+        addchar(ibe, VM_CMD_VERSION);
+        addchar(ibe, VM_PROTOCOL_VERSION);
+        ibe->outbuf[ibe->outlen] = VM_CMD_CHAR;
+        ibe->outlen++;
+        addchar(ibe, VM_CMD_CAPABILITIES);
+        v = VM_CAPABILITIES_IRQ | VM_CAPABILITIES_ATTN;
+        if (k->do_hw_op(ibe->parent.intf, IPMI_POWEROFF_CHASSIS, 1) == 0) {
+            v |= VM_CAPABILITIES_POWER;
+        }
+        if (k->do_hw_op(ibe->parent.intf, IPMI_RESET_CHASSIS, 1) == 0) {
+            v |= VM_CAPABILITIES_RESET;
+        }
+        if (k->do_hw_op(ibe->parent.intf, IPMI_SEND_NMI, 1) == 0) {
+            v |= VM_CAPABILITIES_NMI;
+        }
+        addchar(ibe, v);
+        ibe->outbuf[ibe->outlen] = VM_CMD_CHAR;
+        ibe->outlen++;
+        ibe->sending_cmd = false;
+        continue_send(ibe);
+        break;
+
+    case CHR_EVENT_CLOSED:
+        if (!ibe->connected) {
+            return;
+        }
+        ibe->connected = false;
+        if (ibe->waiting_rsp) {
+            ibe->waiting_rsp = false;
+            ibe->inbuf[1] = ibe->outbuf[1] | 0x04;
+            ibe->inbuf[2] = ibe->outbuf[2];
+            ibe->inbuf[3] = IPMI_CC_BMC_INIT_IN_PROGRESS;
+            k->handle_rsp(s, ibe->outbuf[0], ibe->inbuf + 1, 3);
+        }
+        break;
+    }
+}
+
+static void ipmi_bmc_extern_handle_reset(IPMIBmc *b)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(b);
+
+    ibe->send_reset = true;
+    continue_send(ibe);
+}
+
+static void ipmi_bmc_extern_realize(DeviceState *dev, Error **errp)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(dev);
+
+    if (!ibe->chr) {
+        error_setg(errp, "IPMI external bmc requires chardev attribute");
+        return;
+    }
+
+    qemu_chr_add_handlers(ibe->chr, can_receive, receive, chr_event, ibe);
+}
+
+static int ipmi_bmc_extern_post_migrate(void *opaque, int version_id)
+{
+    IPMIBmcExtern *ibe = opaque;
+
+    /*
+     * We don't directly restore waiting_rsp, Instead, we return an
+     * error on the interface if a response was being waited for.
+     */
+    if (ibe->waiting_rsp) {
+        IPMIInterface *ii = ibe->parent.intf;
+        IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+        ibe->waiting_rsp = false;
+        ibe->inbuf[1] = ibe->outbuf[1] | 0x04;
+        ibe->inbuf[2] = ibe->outbuf[2];
+        ibe->inbuf[3] = IPMI_CC_BMC_INIT_IN_PROGRESS;
+        iic->handle_rsp(ii, ibe->outbuf[0], ibe->inbuf + 1, 3);
+    }
+    return 0;
+}
+
+static const VMStateDescription vmstate_ipmi_bmc_extern = {
+    .name = TYPE_IPMI_BMC_EXTERN,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = ipmi_bmc_extern_post_migrate,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(send_reset, IPMIBmcExtern),
+        VMSTATE_BOOL(waiting_rsp, IPMIBmcExtern),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ipmi_bmc_extern_init(Object *obj)
+{
+    IPMIBmcExtern *ibe = IPMI_BMC_EXTERN(obj);
+
+    ibe->extern_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, extern_timeout, ibe);
+    vmstate_register(NULL, 0, &vmstate_ipmi_bmc_extern, ibe);
+}
+
+static Property ipmi_bmc_extern_properties[] = {
+    DEFINE_PROP_CHR("chardev", IPMIBmcExtern, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ipmi_bmc_extern_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIBmcClass *bk = IPMI_BMC_CLASS(oc);
+
+    bk->handle_command = ipmi_bmc_extern_handle_command;
+    bk->handle_reset = ipmi_bmc_extern_handle_reset;
+    dc->realize = ipmi_bmc_extern_realize;
+    dc->props = ipmi_bmc_extern_properties;
+}
+
+static const TypeInfo ipmi_bmc_extern_type = {
+    .name          = TYPE_IPMI_BMC_EXTERN,
+    .parent        = TYPE_IPMI_BMC,
+    .instance_size = sizeof(IPMIBmcExtern),
+    .instance_init = ipmi_bmc_extern_init,
+    .class_init    = ipmi_bmc_extern_class_init,
+ };
+
+static void ipmi_bmc_extern_register_types(void)
+{
+    type_register_static(&ipmi_bmc_extern_type);
+}
+
+type_init(ipmi_bmc_extern_register_types)
diff --git a/hw/ipmi/ipmi_bmc_sim.c b/hw/ipmi/ipmi_bmc_sim.c
new file mode 100644
index 0000000..dc9c14c
--- /dev/null
+++ b/hw/ipmi/ipmi_bmc_sim.c
@@ -0,0 +1,1810 @@
+/*
+ * IPMI BMC emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "hw/ipmi/ipmi.h"
+#include "qemu/error-report.h"
+
+#define IPMI_NETFN_CHASSIS            0x00
+
+#define IPMI_CMD_GET_CHASSIS_CAPABILITIES 0x00
+#define IPMI_CMD_GET_CHASSIS_STATUS       0x01
+#define IPMI_CMD_CHASSIS_CONTROL          0x02
+#define IPMI_CMD_GET_SYS_RESTART_CAUSE    0x09
+
+#define IPMI_NETFN_SENSOR_EVENT       0x04
+
+#define IPMI_CMD_SET_SENSOR_EVT_ENABLE    0x28
+#define IPMI_CMD_GET_SENSOR_EVT_ENABLE    0x29
+#define IPMI_CMD_REARM_SENSOR_EVTS        0x2a
+#define IPMI_CMD_GET_SENSOR_EVT_STATUS    0x2b
+#define IPMI_CMD_GET_SENSOR_READING       0x2d
+#define IPMI_CMD_SET_SENSOR_TYPE          0x2e
+#define IPMI_CMD_GET_SENSOR_TYPE          0x2f
+
+/* #define IPMI_NETFN_APP             0x06 In ipmi.h */
+
+#define IPMI_CMD_GET_DEVICE_ID            0x01
+#define IPMI_CMD_COLD_RESET               0x02
+#define IPMI_CMD_WARM_RESET               0x03
+#define IPMI_CMD_SET_ACPI_POWER_STATE     0x06
+#define IPMI_CMD_GET_ACPI_POWER_STATE     0x07
+#define IPMI_CMD_GET_DEVICE_GUID          0x08
+#define IPMI_CMD_RESET_WATCHDOG_TIMER     0x22
+#define IPMI_CMD_SET_WATCHDOG_TIMER       0x24
+#define IPMI_CMD_GET_WATCHDOG_TIMER       0x25
+#define IPMI_CMD_SET_BMC_GLOBAL_ENABLES   0x2e
+#define IPMI_CMD_GET_BMC_GLOBAL_ENABLES   0x2f
+#define IPMI_CMD_CLR_MSG_FLAGS            0x30
+#define IPMI_CMD_GET_MSG_FLAGS            0x31
+#define IPMI_CMD_GET_MSG                  0x33
+#define IPMI_CMD_SEND_MSG                 0x34
+#define IPMI_CMD_READ_EVT_MSG_BUF         0x35
+
+#define IPMI_NETFN_STORAGE            0x0a
+
+#define IPMI_CMD_GET_SDR_REP_INFO         0x20
+#define IPMI_CMD_GET_SDR_REP_ALLOC_INFO   0x21
+#define IPMI_CMD_RESERVE_SDR_REP          0x22
+#define IPMI_CMD_GET_SDR                  0x23
+#define IPMI_CMD_ADD_SDR                  0x24
+#define IPMI_CMD_PARTIAL_ADD_SDR          0x25
+#define IPMI_CMD_DELETE_SDR               0x26
+#define IPMI_CMD_CLEAR_SDR_REP            0x27
+#define IPMI_CMD_GET_SDR_REP_TIME         0x28
+#define IPMI_CMD_SET_SDR_REP_TIME         0x29
+#define IPMI_CMD_ENTER_SDR_REP_UPD_MODE   0x2A
+#define IPMI_CMD_EXIT_SDR_REP_UPD_MODE    0x2B
+#define IPMI_CMD_RUN_INIT_AGENT           0x2C
+#define IPMI_CMD_GET_SEL_INFO             0x40
+#define IPMI_CMD_GET_SEL_ALLOC_INFO       0x41
+#define IPMI_CMD_RESERVE_SEL              0x42
+#define IPMI_CMD_GET_SEL_ENTRY            0x43
+#define IPMI_CMD_ADD_SEL_ENTRY            0x44
+#define IPMI_CMD_PARTIAL_ADD_SEL_ENTRY    0x45
+#define IPMI_CMD_DELETE_SEL_ENTRY         0x46
+#define IPMI_CMD_CLEAR_SEL                0x47
+#define IPMI_CMD_GET_SEL_TIME             0x48
+#define IPMI_CMD_SET_SEL_TIME             0x49
+
+
+/* Same as a timespec struct. */
+struct ipmi_time {
+    long tv_sec;
+    long tv_nsec;
+};
+
+#define MAX_SEL_SIZE 128
+
+typedef struct IPMISel {
+    uint8_t sel[MAX_SEL_SIZE][16];
+    unsigned int next_free;
+    long time_offset;
+    uint16_t reservation;
+    uint8_t last_addition[4];
+    uint8_t last_clear[4];
+    uint8_t overflow;
+} IPMISel;
+
+#define MAX_SDR_SIZE 16384
+
+typedef struct IPMISdr {
+    uint8_t sdr[MAX_SDR_SIZE];
+    unsigned int next_free;
+    uint16_t next_rec_id;
+    uint16_t reservation;
+    uint8_t last_addition[4];
+    uint8_t last_clear[4];
+    uint8_t overflow;
+} IPMISdr;
+
+typedef struct IPMISensor {
+    uint8_t status;
+    uint8_t reading;
+    uint16_t states_suppt;
+    uint16_t assert_suppt;
+    uint16_t deassert_suppt;
+    uint16_t states;
+    uint16_t assert_states;
+    uint16_t deassert_states;
+    uint16_t assert_enable;
+    uint16_t deassert_enable;
+    uint8_t  sensor_type;
+    uint8_t  evt_reading_type_code;
+} IPMISensor;
+#define IPMI_SENSOR_GET_PRESENT(s)       ((s)->status & 0x01)
+#define IPMI_SENSOR_SET_PRESENT(s, v)    ((s)->status = (s->status & ~0x01) | \
+                                             !!(v))
+#define IPMI_SENSOR_GET_SCAN_ON(s)       ((s)->status & 0x40)
+#define IPMI_SENSOR_SET_SCAN_ON(s, v)    ((s)->status = (s->status & ~0x40) | \
+                                             ((!!(v)) << 6))
+#define IPMI_SENSOR_GET_EVENTS_ON(s)     ((s)->status & 0x80)
+#define IPMI_SENSOR_SET_EVENTS_ON(s, v)  ((s)->status = (s->status & ~0x80) | \
+                                             ((!!(v)) << 7))
+#define IPMI_SENSOR_GET_RET_STATUS(s)    ((s)->status & 0xc0)
+#define IPMI_SENSOR_SET_RET_STATUS(s, v) ((s)->status = (s->status & ~0xc0) | \
+                                             (v & 0xc0))
+#define IPMI_SENSOR_IS_DISCRETE(s) ((s)->evt_reading_type_code != 1)
+
+#define MAX_SENSORS 20
+#define IPMI_WATCHDOG_SENSOR 0
+
+typedef struct IPMIBmcSim IPMIBmcSim;
+typedef struct RspBuffer RspBuffer;
+
+#define MAX_NETFNS 64
+
+typedef struct IPMICmdHandler {
+    void (*cmd_handler)(IPMIBmcSim *s,
+                        uint8_t *cmd, unsigned int cmd_len,
+                        RspBuffer *rsp);
+    unsigned int cmd_len_min;
+} IPMICmdHandler;
+
+typedef struct IPMINetfn {
+    unsigned int cmd_nums;
+    const IPMICmdHandler *cmd_handlers;
+} IPMINetfn;
+
+typedef struct IPMIRcvBufEntry {
+    QTAILQ_ENTRY(IPMIRcvBufEntry) entry;
+    uint8_t len;
+    uint8_t buf[MAX_IPMI_MSG_SIZE];
+} IPMIRcvBufEntry;
+
+#define TYPE_IPMI_BMC_SIMULATOR "ipmi-bmc-sim"
+#define IPMI_BMC_SIMULATOR(obj) OBJECT_CHECK(IPMIBmcSim, (obj), \
+                                        TYPE_IPMI_BMC_SIMULATOR)
+struct IPMIBmcSim {
+    IPMIBmc parent;
+
+    QEMUTimer *timer;
+
+    uint8_t bmc_global_enables;
+    uint8_t msg_flags;
+
+    bool     watchdog_initialized;
+    uint8_t  watchdog_use;
+    uint8_t  watchdog_action;
+    uint8_t  watchdog_pretimeout; /* In seconds */
+    bool     watchdog_expired;
+    uint16_t watchdog_timeout; /* in 100's of milliseconds */
+
+    bool     watchdog_running;
+    bool     watchdog_preaction_ran;
+    int64_t  watchdog_expiry;
+
+    uint8_t device_id;
+    uint8_t ipmi_version;
+    uint8_t device_rev;
+    uint8_t fwrev1;
+    uint8_t fwrev2;
+    uint8_t mfg_id[3];
+    uint8_t product_id[2];
+
+    uint8_t restart_cause;
+
+    uint8_t acpi_power_state[2];
+    uint8_t uuid[16];
+
+    IPMISel sel;
+    IPMISdr sdr;
+    IPMISensor sensors[MAX_SENSORS];
+
+    /* Odd netfns are for responses, so we only need the even ones. */
+    const IPMINetfn *netfns[MAX_NETFNS / 2];
+
+    QemuMutex lock;
+    /* We allow one event in the buffer */
+    uint8_t evtbuf[16];
+
+    QTAILQ_HEAD(, IPMIRcvBufEntry) rcvbufs;
+};
+
+#define IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK        (1 << 3)
+#define IPMI_BMC_MSG_FLAG_EVT_BUF_FULL                 (1 << 1)
+#define IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE                (1 << 0)
+#define IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK_SET(s) \
+    (IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK & (s)->msg_flags)
+#define IPMI_BMC_MSG_FLAG_EVT_BUF_FULL_SET(s) \
+    (IPMI_BMC_MSG_FLAG_EVT_BUF_FULL & (s)->msg_flags)
+#define IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE_SET(s) \
+    (IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE & (s)->msg_flags)
+
+#define IPMI_BMC_RCV_MSG_QUEUE_INT_BIT    0
+#define IPMI_BMC_EVBUF_FULL_INT_BIT       1
+#define IPMI_BMC_EVENT_MSG_BUF_BIT        2
+#define IPMI_BMC_EVENT_LOG_BIT            3
+#define IPMI_BMC_MSG_INTS_ON(s) ((s)->bmc_global_enables & \
+                                 (1 << IPMI_BMC_RCV_MSG_QUEUE_INT_BIT))
+#define IPMI_BMC_EVBUF_FULL_INT_ENABLED(s) ((s)->bmc_global_enables & \
+                                        (1 << IPMI_BMC_EVBUF_FULL_INT_BIT))
+#define IPMI_BMC_EVENT_LOG_ENABLED(s) ((s)->bmc_global_enables & \
+                                       (1 << IPMI_BMC_EVENT_LOG_BIT))
+#define IPMI_BMC_EVENT_MSG_BUF_ENABLED(s) ((s)->bmc_global_enables & \
+                                           (1 << IPMI_BMC_EVENT_MSG_BUF_BIT))
+
+#define IPMI_BMC_WATCHDOG_USE_MASK 0xc7
+#define IPMI_BMC_WATCHDOG_ACTION_MASK 0x77
+#define IPMI_BMC_WATCHDOG_GET_USE(s) ((s)->watchdog_use & 0x7)
+#define IPMI_BMC_WATCHDOG_GET_DONT_LOG(s) (((s)->watchdog_use >> 7) & 0x1)
+#define IPMI_BMC_WATCHDOG_GET_DONT_STOP(s) (((s)->watchdog_use >> 6) & 0x1)
+#define IPMI_BMC_WATCHDOG_GET_PRE_ACTION(s) (((s)->watchdog_action >> 4) & 0x7)
+#define IPMI_BMC_WATCHDOG_PRE_NONE               0
+#define IPMI_BMC_WATCHDOG_PRE_SMI                1
+#define IPMI_BMC_WATCHDOG_PRE_NMI                2
+#define IPMI_BMC_WATCHDOG_PRE_MSG_INT            3
+#define IPMI_BMC_WATCHDOG_GET_ACTION(s) ((s)->watchdog_action & 0x7)
+#define IPMI_BMC_WATCHDOG_ACTION_NONE            0
+#define IPMI_BMC_WATCHDOG_ACTION_RESET           1
+#define IPMI_BMC_WATCHDOG_ACTION_POWER_DOWN      2
+#define IPMI_BMC_WATCHDOG_ACTION_POWER_CYCLE     3
+
+struct RspBuffer {
+    uint8_t buffer[MAX_IPMI_MSG_SIZE];
+    unsigned int len;
+};
+
+#define RSP_BUFFER_INITIALIZER { }
+
+static inline void rsp_buffer_set_error(RspBuffer *rsp, uint8_t byte)
+{
+    rsp->buffer[2] = byte;
+}
+
+/* Add a byte to the response. */
+static inline void rsp_buffer_push(RspBuffer *rsp, uint8_t byte)
+{
+    if (rsp->len >= sizeof(rsp->buffer)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        return;
+    }
+    rsp->buffer[rsp->len++] = byte;
+}
+
+static inline void rsp_buffer_pushmore(RspBuffer *rsp, uint8_t *bytes,
+                                       unsigned int n)
+{
+    if (rsp->len + n >= sizeof(rsp->buffer)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        return;
+    }
+
+    memcpy(&rsp->buffer[rsp->len], bytes, n);
+    rsp->len += n;
+}
+
+static void ipmi_sim_handle_timeout(IPMIBmcSim *ibs);
+
+static void ipmi_gettime(struct ipmi_time *time)
+{
+    int64_t stime;
+
+    stime = qemu_clock_get_ns(QEMU_CLOCK_HOST);
+    time->tv_sec = stime / 1000000000LL;
+    time->tv_nsec = stime % 1000000000LL;
+}
+
+static int64_t ipmi_getmonotime(void)
+{
+    return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+}
+
+static void ipmi_timeout(void *opaque)
+{
+    IPMIBmcSim *ibs = opaque;
+
+    ipmi_sim_handle_timeout(ibs);
+}
+
+static void set_timestamp(IPMIBmcSim *ibs, uint8_t *ts)
+{
+    unsigned int val;
+    struct ipmi_time now;
+
+    ipmi_gettime(&now);
+    val = now.tv_sec + ibs->sel.time_offset;
+    ts[0] = val & 0xff;
+    ts[1] = (val >> 8) & 0xff;
+    ts[2] = (val >> 16) & 0xff;
+    ts[3] = (val >> 24) & 0xff;
+}
+
+static void sdr_inc_reservation(IPMISdr *sdr)
+{
+    sdr->reservation++;
+    if (sdr->reservation == 0) {
+        sdr->reservation = 1;
+    }
+}
+
+static int sdr_add_entry(IPMIBmcSim *ibs,
+                         const struct ipmi_sdr_header *sdrh_entry,
+                         unsigned int len, uint16_t *recid)
+{
+    struct ipmi_sdr_header *sdrh =
+        (struct ipmi_sdr_header *) &ibs->sdr.sdr[ibs->sdr.next_free];
+
+    if ((len < IPMI_SDR_HEADER_SIZE) || (len > 255)) {
+        return 1;
+    }
+
+    if (ipmi_sdr_length(sdrh_entry) != len) {
+        return 1;
+    }
+
+    if (ibs->sdr.next_free + len > MAX_SDR_SIZE) {
+        ibs->sdr.overflow = 1;
+        return 1;
+    }
+
+    memcpy(sdrh, sdrh_entry, len);
+    sdrh->rec_id[0] = ibs->sdr.next_rec_id & 0xff;
+    sdrh->rec_id[1] = (ibs->sdr.next_rec_id >> 8) & 0xff;
+    sdrh->sdr_version = 0x51; /* Conform to IPMI 1.5 spec */
+
+    if (recid) {
+        *recid = ibs->sdr.next_rec_id;
+    }
+    ibs->sdr.next_rec_id++;
+    set_timestamp(ibs, ibs->sdr.last_addition);
+    ibs->sdr.next_free += len;
+    sdr_inc_reservation(&ibs->sdr);
+    return 0;
+}
+
+static int sdr_find_entry(IPMISdr *sdr, uint16_t recid,
+                          unsigned int *retpos, uint16_t *nextrec)
+{
+    unsigned int pos = *retpos;
+
+    while (pos < sdr->next_free) {
+        struct ipmi_sdr_header *sdrh =
+            (struct ipmi_sdr_header *) &sdr->sdr[pos];
+        uint16_t trec = ipmi_sdr_recid(sdrh);
+        unsigned int nextpos = pos + ipmi_sdr_length(sdrh);
+
+        if (trec == recid) {
+            if (nextrec) {
+                if (nextpos >= sdr->next_free) {
+                    *nextrec = 0xffff;
+                } else {
+                    *nextrec = (sdr->sdr[nextpos] |
+                                (sdr->sdr[nextpos + 1] << 8));
+                }
+            }
+            *retpos = pos;
+            return 0;
+        }
+        pos = nextpos;
+    }
+    return 1;
+}
+
+static void sel_inc_reservation(IPMISel *sel)
+{
+    sel->reservation++;
+    if (sel->reservation == 0) {
+        sel->reservation = 1;
+    }
+}
+
+/* Returns 1 if the SEL is full and can't hold the event. */
+static int sel_add_event(IPMIBmcSim *ibs, uint8_t *event)
+{
+    event[0] = 0xff;
+    event[1] = 0xff;
+    set_timestamp(ibs, event + 3);
+    if (ibs->sel.next_free == MAX_SEL_SIZE) {
+        ibs->sel.overflow = 1;
+        return 1;
+    }
+    event[0] = ibs->sel.next_free & 0xff;
+    event[1] = (ibs->sel.next_free >> 8) & 0xff;
+    memcpy(ibs->sel.last_addition, event + 3, 4);
+    memcpy(ibs->sel.sel[ibs->sel.next_free], event, 16);
+    ibs->sel.next_free++;
+    sel_inc_reservation(&ibs->sel);
+    return 0;
+}
+
+static int attn_set(IPMIBmcSim *ibs)
+{
+    return IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE_SET(ibs)
+        || IPMI_BMC_MSG_FLAG_EVT_BUF_FULL_SET(ibs)
+        || IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK_SET(ibs);
+}
+
+static int attn_irq_enabled(IPMIBmcSim *ibs)
+{
+    return (IPMI_BMC_MSG_INTS_ON(ibs) && IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE_SET(ibs))
+        || (IPMI_BMC_EVBUF_FULL_INT_ENABLED(ibs) &&
+            IPMI_BMC_MSG_FLAG_EVT_BUF_FULL_SET(ibs));
+}
+
+static void gen_event(IPMIBmcSim *ibs, unsigned int sens_num, uint8_t deassert,
+                      uint8_t evd1, uint8_t evd2, uint8_t evd3)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    uint8_t evt[16];
+    IPMISensor *sens = ibs->sensors + sens_num;
+
+    if (!IPMI_BMC_EVENT_MSG_BUF_ENABLED(ibs)) {
+        return;
+    }
+    if (!IPMI_SENSOR_GET_EVENTS_ON(sens)) {
+        return;
+    }
+
+    evt[2] = 0x2; /* System event record */
+    evt[7] = ibs->parent.slave_addr;
+    evt[8] = 0;
+    evt[9] = 0x04; /* Format version */
+    evt[10] = sens->sensor_type;
+    evt[11] = sens_num;
+    evt[12] = sens->evt_reading_type_code | (!!deassert << 7);
+    evt[13] = evd1;
+    evt[14] = evd2;
+    evt[15] = evd3;
+
+    if (IPMI_BMC_EVENT_LOG_ENABLED(ibs)) {
+        sel_add_event(ibs, evt);
+    }
+
+    if (ibs->msg_flags & IPMI_BMC_MSG_FLAG_EVT_BUF_FULL) {
+        return;
+    }
+
+    memcpy(ibs->evtbuf, evt, 16);
+    ibs->msg_flags |= IPMI_BMC_MSG_FLAG_EVT_BUF_FULL;
+    k->set_atn(s, 1, attn_irq_enabled(ibs));
+}
+
+static void sensor_set_discrete_bit(IPMIBmcSim *ibs, unsigned int sensor,
+                                    unsigned int bit, unsigned int val,
+                                    uint8_t evd1, uint8_t evd2, uint8_t evd3)
+{
+    IPMISensor *sens;
+    uint16_t mask;
+
+    if (sensor >= MAX_SENSORS) {
+        return;
+    }
+    if (bit >= 16) {
+        return;
+    }
+
+    mask = (1 << bit);
+    sens = ibs->sensors + sensor;
+    if (val) {
+        sens->states |= mask & sens->states_suppt;
+        if (sens->assert_states & mask) {
+            return; /* Already asserted */
+        }
+        sens->assert_states |= mask & sens->assert_suppt;
+        if (sens->assert_enable & mask & sens->assert_states) {
+            /* Send an event on assert */
+            gen_event(ibs, sensor, 0, evd1, evd2, evd3);
+        }
+    } else {
+        sens->states &= ~(mask & sens->states_suppt);
+        if (sens->deassert_states & mask) {
+            return; /* Already deasserted */
+        }
+        sens->deassert_states |= mask & sens->deassert_suppt;
+        if (sens->deassert_enable & mask & sens->deassert_states) {
+            /* Send an event on deassert */
+            gen_event(ibs, sensor, 1, evd1, evd2, evd3);
+        }
+    }
+}
+
+static void ipmi_init_sensors_from_sdrs(IPMIBmcSim *s)
+{
+    unsigned int i, pos;
+    IPMISensor *sens;
+
+    for (i = 0; i < MAX_SENSORS; i++) {
+        memset(s->sensors + i, 0, sizeof(*sens));
+    }
+
+    pos = 0;
+    for (i = 0; !sdr_find_entry(&s->sdr, i, &pos, NULL); i++) {
+        struct ipmi_sdr_compact *sdr =
+            (struct ipmi_sdr_compact *) &s->sdr.sdr[pos];
+        unsigned int len = sdr->header.rec_length;
+
+        if (len < 20) {
+            continue;
+        }
+        if (sdr->header.rec_type != IPMI_SDR_COMPACT_TYPE) {
+            continue; /* Not a sensor SDR we set from */
+        }
+
+        if (sdr->sensor_owner_number >= MAX_SENSORS) {
+            continue;
+        }
+        sens = s->sensors + sdr->sensor_owner_number;
+
+        IPMI_SENSOR_SET_PRESENT(sens, 1);
+        IPMI_SENSOR_SET_SCAN_ON(sens, (sdr->sensor_init >> 6) & 1);
+        IPMI_SENSOR_SET_EVENTS_ON(sens, (sdr->sensor_init >> 5) & 1);
+        sens->assert_suppt = sdr->assert_mask[0] | (sdr->assert_mask[1] << 8);
+        sens->deassert_suppt =
+            sdr->deassert_mask[0] | (sdr->deassert_mask[1] << 8);
+        sens->states_suppt =
+            sdr->discrete_mask[0] | (sdr->discrete_mask[1] << 8);
+        sens->sensor_type = sdr->sensor_type;
+        sens->evt_reading_type_code = sdr->reading_type & 0x7f;
+
+        /* Enable all the events that are supported. */
+        sens->assert_enable = sens->assert_suppt;
+        sens->deassert_enable = sens->deassert_suppt;
+    }
+}
+
+static int ipmi_register_netfn(IPMIBmcSim *s, unsigned int netfn,
+                               const IPMINetfn *netfnd)
+{
+    if ((netfn & 1) || (netfn >= MAX_NETFNS) || (s->netfns[netfn / 2])) {
+        return -1;
+    }
+    s->netfns[netfn / 2] = netfnd;
+    return 0;
+}
+
+static const IPMICmdHandler *ipmi_get_handler(IPMIBmcSim *ibs,
+                                              unsigned int netfn,
+                                              unsigned int cmd)
+{
+    const IPMICmdHandler *hdl;
+
+    if (netfn & 1 || netfn >= MAX_NETFNS || !ibs->netfns[netfn / 2]) {
+        return NULL;
+    }
+
+    if (cmd >= ibs->netfns[netfn / 2]->cmd_nums) {
+        return NULL;
+    }
+
+    hdl = &ibs->netfns[netfn / 2]->cmd_handlers[cmd];
+    if (!hdl->cmd_handler) {
+        return NULL;
+    }
+
+    return hdl;
+}
+
+static void next_timeout(IPMIBmcSim *ibs)
+{
+    int64_t next;
+    if (ibs->watchdog_running) {
+        next = ibs->watchdog_expiry;
+    } else {
+        /* Wait a minute */
+        next = ipmi_getmonotime() + 60 * 1000000000LL;
+    }
+    timer_mod_ns(ibs->timer, next);
+}
+
+static void ipmi_sim_handle_command(IPMIBmc *b,
+                                    uint8_t *cmd, unsigned int cmd_len,
+                                    unsigned int max_cmd_len,
+                                    uint8_t msg_id)
+{
+    IPMIBmcSim *ibs = IPMI_BMC_SIMULATOR(b);
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    const IPMICmdHandler *hdl;
+    RspBuffer rsp = RSP_BUFFER_INITIALIZER;
+
+    /* Set up the response, set the low bit of NETFN. */
+    /* Note that max_rsp_len must be at least 3 */
+    if (sizeof(rsp.buffer) < 3) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        goto out;
+    }
+
+    rsp_buffer_push(&rsp, cmd[0] | 0x04);
+    rsp_buffer_push(&rsp, cmd[1]);
+    rsp_buffer_push(&rsp, 0); /* Assume success */
+
+    /* If it's too short or it was truncated, return an error. */
+    if (cmd_len < 2) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_LENGTH_INVALID);
+        goto out;
+    }
+    if (cmd_len > max_cmd_len) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_TRUNCATED);
+        goto out;
+    }
+
+    if ((cmd[0] & 0x03) != 0) {
+        /* Only have stuff on LUN 0 */
+        rsp_buffer_set_error(&rsp, IPMI_CC_COMMAND_INVALID_FOR_LUN);
+        goto out;
+    }
+
+    hdl = ipmi_get_handler(ibs, cmd[0] >> 2, cmd[1]);
+    if (!hdl) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_INVALID_CMD);
+        goto out;
+    }
+
+    if (cmd_len < hdl->cmd_len_min) {
+        rsp_buffer_set_error(&rsp, IPMI_CC_REQUEST_DATA_LENGTH_INVALID);
+        goto out;
+    }
+
+    hdl->cmd_handler(ibs, cmd, cmd_len, &rsp);
+
+ out:
+    k->handle_rsp(s, msg_id, rsp.buffer, rsp.len);
+
+    next_timeout(ibs);
+}
+
+static void ipmi_sim_handle_timeout(IPMIBmcSim *ibs)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    if (!ibs->watchdog_running) {
+        goto out;
+    }
+
+    if (!ibs->watchdog_preaction_ran) {
+        switch (IPMI_BMC_WATCHDOG_GET_PRE_ACTION(ibs)) {
+        case IPMI_BMC_WATCHDOG_PRE_NMI:
+            ibs->msg_flags |= IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK;
+            k->do_hw_op(s, IPMI_SEND_NMI, 0);
+            sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 8, 1,
+                                    0xc8, (2 << 4) | 0xf, 0xff);
+            break;
+
+        case IPMI_BMC_WATCHDOG_PRE_MSG_INT:
+            ibs->msg_flags |= IPMI_BMC_MSG_FLAG_WATCHDOG_TIMEOUT_MASK;
+            k->set_atn(s, 1, attn_irq_enabled(ibs));
+            sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 8, 1,
+                                    0xc8, (3 << 4) | 0xf, 0xff);
+            break;
+
+        default:
+            goto do_full_expiry;
+        }
+
+        ibs->watchdog_preaction_ran = 1;
+        /* Issued the pretimeout, do the rest of the timeout now. */
+        ibs->watchdog_expiry = ipmi_getmonotime();
+        ibs->watchdog_expiry += ibs->watchdog_pretimeout * 1000000000LL;
+        goto out;
+    }
+
+ do_full_expiry:
+    ibs->watchdog_running = 0; /* Stop the watchdog on a timeout */
+    ibs->watchdog_expired |= (1 << IPMI_BMC_WATCHDOG_GET_USE(ibs));
+    switch (IPMI_BMC_WATCHDOG_GET_ACTION(ibs)) {
+    case IPMI_BMC_WATCHDOG_ACTION_NONE:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 0, 1,
+                                0xc0, ibs->watchdog_use & 0xf, 0xff);
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_RESET:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 1, 1,
+                                0xc1, ibs->watchdog_use & 0xf, 0xff);
+        k->do_hw_op(s, IPMI_RESET_CHASSIS, 0);
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_DOWN:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 2, 1,
+                                0xc2, ibs->watchdog_use & 0xf, 0xff);
+        k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 0);
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_CYCLE:
+        sensor_set_discrete_bit(ibs, IPMI_WATCHDOG_SENSOR, 2, 1,
+                                0xc3, ibs->watchdog_use & 0xf, 0xff);
+        k->do_hw_op(s, IPMI_POWERCYCLE_CHASSIS, 0);
+        break;
+    }
+
+ out:
+    next_timeout(ibs);
+}
+
+static void chassis_capabilities(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, 0);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+    rsp_buffer_push(rsp, ibs->parent.slave_addr);
+}
+
+static void chassis_status(IPMIBmcSim *ibs,
+                           uint8_t *cmd, unsigned int cmd_len,
+                           RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, 0x61); /* Unknown power restore, power is on */
+    rsp_buffer_push(rsp, 0);
+    rsp_buffer_push(rsp, 0);
+    rsp_buffer_push(rsp, 0);
+}
+
+static void chassis_control(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    switch (cmd[2] & 0xf) {
+    case 0: /* power down */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 0));
+        break;
+    case 1: /* power up */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWERON_CHASSIS, 0));
+        break;
+    case 2: /* power cycle */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWERCYCLE_CHASSIS, 0));
+        break;
+    case 3: /* hard reset */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_RESET_CHASSIS, 0));
+        break;
+    case 4: /* pulse diagnostic interrupt */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_PULSE_DIAG_IRQ, 0));
+        break;
+    case 5: /* soft shutdown via ACPI by overtemp emulation */
+        rsp_buffer_set_error(rsp, k->do_hw_op(s,
+                                          IPMI_SHUTDOWN_VIA_ACPI_OVERTEMP, 0));
+        break;
+    default:
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+}
+
+static void chassis_get_sys_restart_cause(IPMIBmcSim *ibs,
+                           uint8_t *cmd, unsigned int cmd_len,
+                           RspBuffer *rsp)
+
+{
+    rsp_buffer_push(rsp, ibs->restart_cause & 0xf); /* Restart Cause */
+    rsp_buffer_push(rsp, 0);  /* Channel 0 */
+}
+
+static void get_device_id(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->device_id);
+    rsp_buffer_push(rsp, ibs->device_rev & 0xf);
+    rsp_buffer_push(rsp, ibs->fwrev1 & 0x7f);
+    rsp_buffer_push(rsp, ibs->fwrev2);
+    rsp_buffer_push(rsp, ibs->ipmi_version);
+    rsp_buffer_push(rsp, 0x07); /* sensor, SDR, and SEL. */
+    rsp_buffer_push(rsp, ibs->mfg_id[0]);
+    rsp_buffer_push(rsp, ibs->mfg_id[1]);
+    rsp_buffer_push(rsp, ibs->mfg_id[2]);
+    rsp_buffer_push(rsp, ibs->product_id[0]);
+    rsp_buffer_push(rsp, ibs->product_id[1]);
+}
+
+static void set_global_enables(IPMIBmcSim *ibs, uint8_t val)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    bool irqs_on;
+
+    ibs->bmc_global_enables = val;
+
+    irqs_on = val & (IPMI_BMC_EVBUF_FULL_INT_BIT |
+                     IPMI_BMC_RCV_MSG_QUEUE_INT_BIT);
+
+    k->set_irq_enable(s, irqs_on);
+}
+
+static void cold_reset(IPMIBmcSim *ibs,
+                       uint8_t *cmd, unsigned int cmd_len,
+                       RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    /* Disable all interrupts */
+    set_global_enables(ibs, 1 << IPMI_BMC_EVENT_LOG_BIT);
+
+    if (k->reset) {
+        k->reset(s, true);
+    }
+}
+
+static void warm_reset(IPMIBmcSim *ibs,
+                       uint8_t *cmd, unsigned int cmd_len,
+                       RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    if (k->reset) {
+        k->reset(s, false);
+    }
+}
+static void set_acpi_power_state(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    ibs->acpi_power_state[0] = cmd[2];
+    ibs->acpi_power_state[1] = cmd[3];
+}
+
+static void get_acpi_power_state(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->acpi_power_state[0]);
+    rsp_buffer_push(rsp, ibs->acpi_power_state[1]);
+}
+
+static void get_device_guid(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    unsigned int i;
+
+    for (i = 0; i < 16; i++) {
+        rsp_buffer_push(rsp, ibs->uuid[i]);
+    }
+}
+
+static void set_bmc_global_enables(IPMIBmcSim *ibs,
+                                   uint8_t *cmd, unsigned int cmd_len,
+                                   RspBuffer *rsp)
+{
+    set_global_enables(ibs, cmd[2]);
+}
+
+static void get_bmc_global_enables(IPMIBmcSim *ibs,
+                                   uint8_t *cmd, unsigned int cmd_len,
+                                   RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->bmc_global_enables);
+}
+
+static void clr_msg_flags(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+    ibs->msg_flags &= ~cmd[2];
+    k->set_atn(s, attn_set(ibs), attn_irq_enabled(ibs));
+}
+
+static void get_msg_flags(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->msg_flags);
+}
+
+static void read_evt_msg_buf(IPMIBmcSim *ibs,
+                             uint8_t *cmd, unsigned int cmd_len,
+                             RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    unsigned int i;
+
+    if (!(ibs->msg_flags & IPMI_BMC_MSG_FLAG_EVT_BUF_FULL)) {
+        rsp_buffer_set_error(rsp, 0x80);
+        return;
+    }
+    for (i = 0; i < 16; i++) {
+        rsp_buffer_push(rsp, ibs->evtbuf[i]);
+    }
+    ibs->msg_flags &= ~IPMI_BMC_MSG_FLAG_EVT_BUF_FULL;
+    k->set_atn(s, attn_set(ibs), attn_irq_enabled(ibs));
+}
+
+static void get_msg(IPMIBmcSim *ibs,
+                    uint8_t *cmd, unsigned int cmd_len,
+                    RspBuffer *rsp)
+{
+    IPMIRcvBufEntry *msg;
+
+    qemu_mutex_lock(&ibs->lock);
+    if (QTAILQ_EMPTY(&ibs->rcvbufs)) {
+        rsp_buffer_set_error(rsp, 0x80); /* Queue empty */
+        goto out;
+    }
+    rsp_buffer_push(rsp, 0); /* Channel 0 */
+    msg = QTAILQ_FIRST(&ibs->rcvbufs);
+    rsp_buffer_pushmore(rsp, msg->buf, msg->len);
+    QTAILQ_REMOVE(&ibs->rcvbufs, msg, entry);
+    g_free(msg);
+
+    if (QTAILQ_EMPTY(&ibs->rcvbufs)) {
+        IPMIInterface *s = ibs->parent.intf;
+        IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+
+        ibs->msg_flags &= ~IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE;
+        k->set_atn(s, attn_set(ibs), attn_irq_enabled(ibs));
+    }
+
+out:
+    qemu_mutex_unlock(&ibs->lock);
+    return;
+}
+
+static unsigned char
+ipmb_checksum(unsigned char *data, int size, unsigned char csum)
+{
+    for (; size > 0; size--, data++) {
+            csum += *data;
+    }
+
+    return -csum;
+}
+
+static void send_msg(IPMIBmcSim *ibs,
+                     uint8_t *cmd, unsigned int cmd_len,
+                     RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    IPMIRcvBufEntry *msg;
+    uint8_t *buf;
+    uint8_t netfn, rqLun, rsLun, rqSeq;
+
+    if (cmd[2] != 0) {
+        /* We only handle channel 0 with no options */
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+
+    if (cmd_len < 10) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQUEST_DATA_LENGTH_INVALID);
+        return;
+    }
+
+    if (cmd[3] != 0x40) {
+        /* We only emulate a MC at address 0x40. */
+        rsp_buffer_set_error(rsp, 0x83); /* NAK on write */
+        return;
+    }
+
+    cmd += 3; /* Skip the header. */
+    cmd_len -= 3;
+
+    /*
+     * At this point we "send" the message successfully.  Any error will
+     * be returned in the response.
+     */
+    if (ipmb_checksum(cmd, cmd_len, 0) != 0 ||
+        cmd[3] != 0x20) { /* Improper response address */
+        return; /* No response */
+    }
+
+    netfn = cmd[1] >> 2;
+    rqLun = cmd[4] & 0x3;
+    rsLun = cmd[1] & 0x3;
+    rqSeq = cmd[4] >> 2;
+
+    if (rqLun != 2) {
+        /* We only support LUN 2 coming back to us. */
+        return;
+    }
+
+    msg = g_malloc(sizeof(*msg));
+    msg->buf[0] = ((netfn | 1) << 2) | rqLun; /* NetFN, and make a response */
+    msg->buf[1] = ipmb_checksum(msg->buf, 1, 0);
+    msg->buf[2] = cmd[0]; /* rsSA */
+    msg->buf[3] = (rqSeq << 2) | rsLun;
+    msg->buf[4] = cmd[5]; /* Cmd */
+    msg->buf[5] = 0; /* Completion Code */
+    msg->len = 6;
+
+    if ((cmd[1] >> 2) != IPMI_NETFN_APP || cmd[5] != IPMI_CMD_GET_DEVICE_ID) {
+        /* Not a command we handle. */
+        msg->buf[5] = IPMI_CC_INVALID_CMD;
+        goto end_msg;
+    }
+
+    buf = msg->buf + msg->len; /* After the CC */
+    buf[0] = 0;
+    buf[1] = 0;
+    buf[2] = 0;
+    buf[3] = 0;
+    buf[4] = 0x51;
+    buf[5] = 0;
+    buf[6] = 0;
+    buf[7] = 0;
+    buf[8] = 0;
+    buf[9] = 0;
+    buf[10] = 0;
+    msg->len += 11;
+
+ end_msg:
+    msg->buf[msg->len] = ipmb_checksum(msg->buf, msg->len, 0);
+    msg->len++;
+    qemu_mutex_lock(&ibs->lock);
+    QTAILQ_INSERT_TAIL(&ibs->rcvbufs, msg, entry);
+    ibs->msg_flags |= IPMI_BMC_MSG_FLAG_RCV_MSG_QUEUE;
+    k->set_atn(s, 1, attn_irq_enabled(ibs));
+    qemu_mutex_unlock(&ibs->lock);
+}
+
+static void do_watchdog_reset(IPMIBmcSim *ibs)
+{
+    if (IPMI_BMC_WATCHDOG_GET_ACTION(ibs) ==
+        IPMI_BMC_WATCHDOG_ACTION_NONE) {
+        ibs->watchdog_running = 0;
+        return;
+    }
+    ibs->watchdog_preaction_ran = 0;
+
+
+    /* Timeout is in tenths of a second, offset is in seconds */
+    ibs->watchdog_expiry = ipmi_getmonotime();
+    ibs->watchdog_expiry += ibs->watchdog_timeout * 100000000LL;
+    if (IPMI_BMC_WATCHDOG_GET_PRE_ACTION(ibs) != IPMI_BMC_WATCHDOG_PRE_NONE) {
+        ibs->watchdog_expiry -= ibs->watchdog_pretimeout * 1000000000LL;
+    }
+    ibs->watchdog_running = 1;
+}
+
+static void reset_watchdog_timer(IPMIBmcSim *ibs,
+                                 uint8_t *cmd, unsigned int cmd_len,
+                                 RspBuffer *rsp)
+{
+    if (!ibs->watchdog_initialized) {
+        rsp_buffer_set_error(rsp, 0x80);
+        return;
+    }
+    do_watchdog_reset(ibs);
+}
+
+static void set_watchdog_timer(IPMIBmcSim *ibs,
+                               uint8_t *cmd, unsigned int cmd_len,
+                               RspBuffer *rsp)
+{
+    IPMIInterface *s = ibs->parent.intf;
+    IPMIInterfaceClass *k = IPMI_INTERFACE_GET_CLASS(s);
+    unsigned int val;
+
+    val = cmd[2] & 0x7; /* Validate use */
+    if (val == 0 || val > 5) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    val = cmd[3] & 0x7; /* Validate action */
+    switch (val) {
+    case IPMI_BMC_WATCHDOG_ACTION_NONE:
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_RESET:
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_RESET_CHASSIS, 1));
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_DOWN:
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWEROFF_CHASSIS, 1));
+        break;
+
+    case IPMI_BMC_WATCHDOG_ACTION_POWER_CYCLE:
+        rsp_buffer_set_error(rsp, k->do_hw_op(s, IPMI_POWERCYCLE_CHASSIS, 1));
+        break;
+
+    default:
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+    }
+    if (rsp->buffer[2]) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+
+    val = (cmd[3] >> 4) & 0x7; /* Validate preaction */
+    switch (val) {
+    case IPMI_BMC_WATCHDOG_PRE_MSG_INT:
+    case IPMI_BMC_WATCHDOG_PRE_NONE:
+        break;
+
+    case IPMI_BMC_WATCHDOG_PRE_NMI:
+        if (!k->do_hw_op(s, IPMI_SEND_NMI, 1)) {
+            /* NMI not supported. */
+            rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+            return;
+        }
+        break;
+
+    default:
+        /* We don't support PRE_SMI */
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+
+    ibs->watchdog_initialized = 1;
+    ibs->watchdog_use = cmd[2] & IPMI_BMC_WATCHDOG_USE_MASK;
+    ibs->watchdog_action = cmd[3] & IPMI_BMC_WATCHDOG_ACTION_MASK;
+    ibs->watchdog_pretimeout = cmd[4];
+    ibs->watchdog_expired &= ~cmd[5];
+    ibs->watchdog_timeout = cmd[6] | (((uint16_t) cmd[7]) << 8);
+    if (ibs->watchdog_running & IPMI_BMC_WATCHDOG_GET_DONT_STOP(ibs)) {
+        do_watchdog_reset(ibs);
+    } else {
+        ibs->watchdog_running = 0;
+    }
+}
+
+static void get_watchdog_timer(IPMIBmcSim *ibs,
+                               uint8_t *cmd, unsigned int cmd_len,
+                               RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->watchdog_use);
+    rsp_buffer_push(rsp, ibs->watchdog_action);
+    rsp_buffer_push(rsp, ibs->watchdog_pretimeout);
+    rsp_buffer_push(rsp, ibs->watchdog_expired);
+    if (ibs->watchdog_running) {
+        long timeout;
+        timeout = ((ibs->watchdog_expiry - ipmi_getmonotime() + 50000000)
+                   / 100000000);
+        rsp_buffer_push(rsp, timeout & 0xff);
+        rsp_buffer_push(rsp, (timeout >> 8) & 0xff);
+    } else {
+        rsp_buffer_push(rsp, 0);
+        rsp_buffer_push(rsp, 0);
+    }
+}
+
+static void get_sdr_rep_info(IPMIBmcSim *ibs,
+                             uint8_t *cmd, unsigned int cmd_len,
+                             RspBuffer *rsp)
+{
+    unsigned int i;
+
+    rsp_buffer_push(rsp, 0x51); /* Conform to IPMI 1.5 spec */
+    rsp_buffer_push(rsp, ibs->sdr.next_rec_id & 0xff);
+    rsp_buffer_push(rsp, (ibs->sdr.next_rec_id >> 8) & 0xff);
+    rsp_buffer_push(rsp, (MAX_SDR_SIZE - ibs->sdr.next_free) & 0xff);
+    rsp_buffer_push(rsp, ((MAX_SDR_SIZE - ibs->sdr.next_free) >> 8) & 0xff);
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sdr.last_addition[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sdr.last_clear[i]);
+    }
+    /* Only modal support, reserve supported */
+    rsp_buffer_push(rsp, (ibs->sdr.overflow << 7) | 0x22);
+}
+
+static void reserve_sdr_rep(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->sdr.reservation & 0xff);
+    rsp_buffer_push(rsp, (ibs->sdr.reservation >> 8) & 0xff);
+}
+
+static void get_sdr(IPMIBmcSim *ibs,
+                    uint8_t *cmd, unsigned int cmd_len,
+                    RspBuffer *rsp)
+{
+    unsigned int pos;
+    uint16_t nextrec;
+    struct ipmi_sdr_header *sdrh;
+
+    if (cmd[6]) {
+        if ((cmd[2] | (cmd[3] << 8)) != ibs->sdr.reservation) {
+            rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+            return;
+        }
+    }
+
+    pos = 0;
+    if (sdr_find_entry(&ibs->sdr, cmd[4] | (cmd[5] << 8),
+                       &pos, &nextrec)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+
+    sdrh = (struct ipmi_sdr_header *) &ibs->sdr.sdr[pos];
+
+    if (cmd[6] > ipmi_sdr_length(sdrh)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_PARM_OUT_OF_RANGE);
+        return;
+    }
+
+    rsp_buffer_push(rsp, nextrec & 0xff);
+    rsp_buffer_push(rsp, (nextrec >> 8) & 0xff);
+
+    if (cmd[7] == 0xff) {
+        cmd[7] = ipmi_sdr_length(sdrh) - cmd[6];
+    }
+
+    if ((cmd[7] + rsp->len) > sizeof(rsp->buffer)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES);
+        return;
+    }
+
+    rsp_buffer_pushmore(rsp, ibs->sdr.sdr + pos + cmd[6], cmd[7]);
+}
+
+static void add_sdr(IPMIBmcSim *ibs,
+                    uint8_t *cmd, unsigned int cmd_len,
+                    RspBuffer *rsp)
+{
+    uint16_t recid;
+    struct ipmi_sdr_header *sdrh = (struct ipmi_sdr_header *) cmd + 2;
+
+    if (sdr_add_entry(ibs, sdrh, cmd_len - 2, &recid)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    rsp_buffer_push(rsp, recid & 0xff);
+    rsp_buffer_push(rsp, (recid >> 8) & 0xff);
+}
+
+static void clear_sdr_rep(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    if ((cmd[2] | (cmd[3] << 8)) != ibs->sdr.reservation) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+        return;
+    }
+
+    if (cmd[4] != 'C' || cmd[5] != 'L' || cmd[6] != 'R') {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    if (cmd[7] == 0xaa) {
+        ibs->sdr.next_free = 0;
+        ibs->sdr.overflow = 0;
+        set_timestamp(ibs, ibs->sdr.last_clear);
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+        sdr_inc_reservation(&ibs->sdr);
+    } else if (cmd[7] == 0) {
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+    } else {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+}
+
+static void get_sel_info(IPMIBmcSim *ibs,
+                         uint8_t *cmd, unsigned int cmd_len,
+                         RspBuffer *rsp)
+{
+    unsigned int i, val;
+
+    rsp_buffer_push(rsp, 0x51); /* Conform to IPMI 1.5 */
+    rsp_buffer_push(rsp, ibs->sel.next_free & 0xff);
+    rsp_buffer_push(rsp, (ibs->sel.next_free >> 8) & 0xff);
+    val = (MAX_SEL_SIZE - ibs->sel.next_free) * 16;
+    rsp_buffer_push(rsp, val & 0xff);
+    rsp_buffer_push(rsp, (val >> 8) & 0xff);
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sel.last_addition[i]);
+    }
+    for (i = 0; i < 4; i++) {
+        rsp_buffer_push(rsp, ibs->sel.last_clear[i]);
+    }
+    /* Only support Reserve SEL */
+    rsp_buffer_push(rsp, (ibs->sel.overflow << 7) | 0x02);
+}
+
+static void reserve_sel(IPMIBmcSim *ibs,
+                        uint8_t *cmd, unsigned int cmd_len,
+                        RspBuffer *rsp)
+{
+    rsp_buffer_push(rsp, ibs->sel.reservation & 0xff);
+    rsp_buffer_push(rsp, (ibs->sel.reservation >> 8) & 0xff);
+}
+
+static void get_sel_entry(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    unsigned int val;
+
+    if (cmd[6]) {
+        if ((cmd[2] | (cmd[3] << 8)) != ibs->sel.reservation) {
+            rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+            return;
+        }
+    }
+    if (ibs->sel.next_free == 0) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    if (cmd[6] > 15) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    if (cmd[7] == 0xff) {
+        cmd[7] = 16;
+    } else if ((cmd[7] + cmd[6]) > 16) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    } else {
+        cmd[7] += cmd[6];
+    }
+
+    val = cmd[4] | (cmd[5] << 8);
+    if (val == 0xffff) {
+        val = ibs->sel.next_free - 1;
+    } else if (val >= ibs->sel.next_free) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    if ((val + 1) == ibs->sel.next_free) {
+        rsp_buffer_push(rsp, 0xff);
+        rsp_buffer_push(rsp, 0xff);
+    } else {
+        rsp_buffer_push(rsp, (val + 1) & 0xff);
+        rsp_buffer_push(rsp, ((val + 1) >> 8) & 0xff);
+    }
+    for (; cmd[6] < cmd[7]; cmd[6]++) {
+        rsp_buffer_push(rsp, ibs->sel.sel[val][cmd[6]]);
+    }
+}
+
+static void add_sel_entry(IPMIBmcSim *ibs,
+                          uint8_t *cmd, unsigned int cmd_len,
+                          RspBuffer *rsp)
+{
+    if (sel_add_event(ibs, cmd + 2)) {
+        rsp_buffer_set_error(rsp, IPMI_CC_OUT_OF_SPACE);
+        return;
+    }
+    /* sel_add_event fills in the record number. */
+    rsp_buffer_push(rsp, cmd[2]);
+    rsp_buffer_push(rsp, cmd[3]);
+}
+
+static void clear_sel(IPMIBmcSim *ibs,
+                      uint8_t *cmd, unsigned int cmd_len,
+                      RspBuffer *rsp)
+{
+    if ((cmd[2] | (cmd[3] << 8)) != ibs->sel.reservation) {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_RESERVATION);
+        return;
+    }
+
+    if (cmd[4] != 'C' || cmd[5] != 'L' || cmd[6] != 'R') {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    if (cmd[7] == 0xaa) {
+        ibs->sel.next_free = 0;
+        ibs->sel.overflow = 0;
+        set_timestamp(ibs, ibs->sdr.last_clear);
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+        sel_inc_reservation(&ibs->sel);
+    } else if (cmd[7] == 0) {
+        rsp_buffer_push(rsp, 1); /* Erasure complete */
+    } else {
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+}
+
+static void get_sel_time(IPMIBmcSim *ibs,
+                         uint8_t *cmd, unsigned int cmd_len,
+                         RspBuffer *rsp)
+{
+    uint32_t val;
+    struct ipmi_time now;
+
+    ipmi_gettime(&now);
+    val = now.tv_sec + ibs->sel.time_offset;
+    rsp_buffer_push(rsp, val & 0xff);
+    rsp_buffer_push(rsp, (val >> 8) & 0xff);
+    rsp_buffer_push(rsp, (val >> 16) & 0xff);
+    rsp_buffer_push(rsp, (val >> 24) & 0xff);
+}
+
+static void set_sel_time(IPMIBmcSim *ibs,
+                         uint8_t *cmd, unsigned int cmd_len,
+                         RspBuffer *rsp)
+{
+    uint32_t val;
+    struct ipmi_time now;
+
+    val = cmd[2] | (cmd[3] << 8) | (cmd[4] << 16) | (cmd[5] << 24);
+    ipmi_gettime(&now);
+    ibs->sel.time_offset = now.tv_sec - ((long) val);
+}
+
+static void set_sensor_evt_enable(IPMIBmcSim *ibs,
+                                  uint8_t *cmd, unsigned int cmd_len,
+                                  RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    switch ((cmd[3] >> 4) & 0x3) {
+    case 0: /* Do not change */
+        break;
+    case 1: /* Enable bits */
+        if (cmd_len > 4) {
+            sens->assert_enable |= cmd[4];
+        }
+        if (cmd_len > 5) {
+            sens->assert_enable |= cmd[5] << 8;
+        }
+        if (cmd_len > 6) {
+            sens->deassert_enable |= cmd[6];
+        }
+        if (cmd_len > 7) {
+            sens->deassert_enable |= cmd[7] << 8;
+        }
+        break;
+    case 2: /* Disable bits */
+        if (cmd_len > 4) {
+            sens->assert_enable &= ~cmd[4];
+        }
+        if (cmd_len > 5) {
+            sens->assert_enable &= ~(cmd[5] << 8);
+        }
+        if (cmd_len > 6) {
+            sens->deassert_enable &= ~cmd[6];
+        }
+        if (cmd_len > 7) {
+            sens->deassert_enable &= ~(cmd[7] << 8);
+        }
+        break;
+    case 3:
+        rsp_buffer_set_error(rsp, IPMI_CC_INVALID_DATA_FIELD);
+        return;
+    }
+    IPMI_SENSOR_SET_RET_STATUS(sens, cmd[3]);
+}
+
+static void get_sensor_evt_enable(IPMIBmcSim *ibs,
+                                  uint8_t *cmd, unsigned int cmd_len,
+                                  RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+        !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, IPMI_SENSOR_GET_RET_STATUS(sens));
+    rsp_buffer_push(rsp, sens->assert_enable & 0xff);
+    rsp_buffer_push(rsp, (sens->assert_enable >> 8) & 0xff);
+    rsp_buffer_push(rsp, sens->deassert_enable & 0xff);
+    rsp_buffer_push(rsp, (sens->deassert_enable >> 8) & 0xff);
+}
+
+static void rearm_sensor_evts(IPMIBmcSim *ibs,
+                              uint8_t *cmd, unsigned int cmd_len,
+                              RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+        !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+
+    if ((cmd[3] & 0x80) == 0) {
+        /* Just clear everything */
+        sens->states = 0;
+        return;
+    }
+}
+
+static void get_sensor_evt_status(IPMIBmcSim *ibs,
+                                  uint8_t *cmd, unsigned int cmd_len,
+                                  RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+        !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, sens->reading);
+    rsp_buffer_push(rsp, IPMI_SENSOR_GET_RET_STATUS(sens));
+    rsp_buffer_push(rsp, sens->assert_states & 0xff);
+    rsp_buffer_push(rsp, (sens->assert_states >> 8) & 0xff);
+    rsp_buffer_push(rsp, sens->deassert_states & 0xff);
+    rsp_buffer_push(rsp, (sens->deassert_states >> 8) & 0xff);
+}
+
+static void get_sensor_reading(IPMIBmcSim *ibs,
+                               uint8_t *cmd, unsigned int cmd_len,
+                               RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, sens->reading);
+    rsp_buffer_push(rsp, IPMI_SENSOR_GET_RET_STATUS(sens));
+    rsp_buffer_push(rsp, sens->states & 0xff);
+    if (IPMI_SENSOR_IS_DISCRETE(sens)) {
+        rsp_buffer_push(rsp, (sens->states >> 8) & 0xff);
+    }
+}
+
+static void set_sensor_type(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    sens->sensor_type = cmd[3];
+    sens->evt_reading_type_code = cmd[4] & 0x7f;
+}
+
+static void get_sensor_type(IPMIBmcSim *ibs,
+                            uint8_t *cmd, unsigned int cmd_len,
+                            RspBuffer *rsp)
+{
+    IPMISensor *sens;
+
+
+    if ((cmd[2] >= MAX_SENSORS) ||
+            !IPMI_SENSOR_GET_PRESENT(ibs->sensors + cmd[2])) {
+        rsp_buffer_set_error(rsp, IPMI_CC_REQ_ENTRY_NOT_PRESENT);
+        return;
+    }
+    sens = ibs->sensors + cmd[2];
+    rsp_buffer_push(rsp, sens->sensor_type);
+    rsp_buffer_push(rsp, sens->evt_reading_type_code);
+}
+
+
+static const IPMICmdHandler chassis_cmds[] = {
+    [IPMI_CMD_GET_CHASSIS_CAPABILITIES] = { chassis_capabilities },
+    [IPMI_CMD_GET_CHASSIS_STATUS] = { chassis_status },
+    [IPMI_CMD_CHASSIS_CONTROL] = { chassis_control, 3 },
+    [IPMI_CMD_GET_SYS_RESTART_CAUSE] = { chassis_get_sys_restart_cause }
+};
+static const IPMINetfn chassis_netfn = {
+    .cmd_nums = ARRAY_SIZE(chassis_cmds),
+    .cmd_handlers = chassis_cmds
+};
+
+static const IPMICmdHandler sensor_event_cmds[] = {
+    [IPMI_CMD_SET_SENSOR_EVT_ENABLE] = { set_sensor_evt_enable, 4 },
+    [IPMI_CMD_GET_SENSOR_EVT_ENABLE] = { get_sensor_evt_enable, 3 },
+    [IPMI_CMD_REARM_SENSOR_EVTS] = { rearm_sensor_evts, 4 },
+    [IPMI_CMD_GET_SENSOR_EVT_STATUS] = { get_sensor_evt_status, 3 },
+    [IPMI_CMD_GET_SENSOR_READING] = { get_sensor_reading, 3 },
+    [IPMI_CMD_SET_SENSOR_TYPE] = { set_sensor_type, 5 },
+    [IPMI_CMD_GET_SENSOR_TYPE] = { get_sensor_type, 3 },
+};
+static const IPMINetfn sensor_event_netfn = {
+    .cmd_nums = ARRAY_SIZE(sensor_event_cmds),
+    .cmd_handlers = sensor_event_cmds
+};
+
+static const IPMICmdHandler app_cmds[] = {
+    [IPMI_CMD_GET_DEVICE_ID] = { get_device_id },
+    [IPMI_CMD_COLD_RESET] = { cold_reset },
+    [IPMI_CMD_WARM_RESET] = { warm_reset },
+    [IPMI_CMD_SET_ACPI_POWER_STATE] = { set_acpi_power_state, 4 },
+    [IPMI_CMD_GET_ACPI_POWER_STATE] = { get_acpi_power_state },
+    [IPMI_CMD_GET_DEVICE_GUID] = { get_device_guid },
+    [IPMI_CMD_SET_BMC_GLOBAL_ENABLES] = { set_bmc_global_enables, 3 },
+    [IPMI_CMD_GET_BMC_GLOBAL_ENABLES] = { get_bmc_global_enables },
+    [IPMI_CMD_CLR_MSG_FLAGS] = { clr_msg_flags, 3 },
+    [IPMI_CMD_GET_MSG_FLAGS] = { get_msg_flags },
+    [IPMI_CMD_GET_MSG] = { get_msg },
+    [IPMI_CMD_SEND_MSG] = { send_msg, 3 },
+    [IPMI_CMD_READ_EVT_MSG_BUF] = { read_evt_msg_buf },
+    [IPMI_CMD_RESET_WATCHDOG_TIMER] = { reset_watchdog_timer },
+    [IPMI_CMD_SET_WATCHDOG_TIMER] = { set_watchdog_timer, 8 },
+    [IPMI_CMD_GET_WATCHDOG_TIMER] = { get_watchdog_timer },
+};
+static const IPMINetfn app_netfn = {
+    .cmd_nums = ARRAY_SIZE(app_cmds),
+    .cmd_handlers = app_cmds
+};
+
+static const IPMICmdHandler storage_cmds[] = {
+    [IPMI_CMD_GET_SDR_REP_INFO] = { get_sdr_rep_info },
+    [IPMI_CMD_RESERVE_SDR_REP] = { reserve_sdr_rep },
+    [IPMI_CMD_GET_SDR] = { get_sdr, 8 },
+    [IPMI_CMD_ADD_SDR] = { add_sdr },
+    [IPMI_CMD_CLEAR_SDR_REP] = { clear_sdr_rep, 8 },
+    [IPMI_CMD_GET_SEL_INFO] = { get_sel_info },
+    [IPMI_CMD_RESERVE_SEL] = { reserve_sel },
+    [IPMI_CMD_GET_SEL_ENTRY] = { get_sel_entry, 8 },
+    [IPMI_CMD_ADD_SEL_ENTRY] = { add_sel_entry, 18 },
+    [IPMI_CMD_CLEAR_SEL] = { clear_sel, 8 },
+    [IPMI_CMD_GET_SEL_TIME] = { get_sel_time, 6 },
+    [IPMI_CMD_SET_SEL_TIME] = { set_sel_time },
+};
+
+static const IPMINetfn storage_netfn = {
+    .cmd_nums = ARRAY_SIZE(storage_cmds),
+    .cmd_handlers = storage_cmds
+};
+
+static void register_cmds(IPMIBmcSim *s)
+{
+    ipmi_register_netfn(s, IPMI_NETFN_CHASSIS, &chassis_netfn);
+    ipmi_register_netfn(s, IPMI_NETFN_SENSOR_EVENT, &sensor_event_netfn);
+    ipmi_register_netfn(s, IPMI_NETFN_APP, &app_netfn);
+    ipmi_register_netfn(s, IPMI_NETFN_STORAGE, &storage_netfn);
+}
+
+static uint8_t init_sdrs[] = {
+    /* Watchdog device */
+    0x00, 0x00, 0x51, 0x02,   35, 0x20, 0x00, 0x00,
+    0x23, 0x01, 0x63, 0x00, 0x23, 0x6f, 0x0f, 0x01,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc8,
+    'W',  'a',  't',  'c',  'h',  'd',  'o',  'g',
+};
+
+static void ipmi_sdr_init(IPMIBmcSim *ibs)
+{
+    unsigned int i;
+    int len;
+    size_t sdrs_size;
+    uint8_t *sdrs;
+
+    sdrs_size = sizeof(init_sdrs);
+    sdrs = init_sdrs;
+
+    for (i = 0; i < sdrs_size; i += len) {
+        struct ipmi_sdr_header *sdrh;
+
+        if (i + IPMI_SDR_HEADER_SIZE > sdrs_size) {
+            error_report("Problem with recid 0x%4.4x", i);
+            return;
+        }
+        sdrh = (struct ipmi_sdr_header *) &sdrs[i];
+        len = ipmi_sdr_length(sdrh);
+        if (i + len > sdrs_size) {
+            error_report("Problem with recid 0x%4.4x", i);
+            return;
+        }
+        sdr_add_entry(ibs, sdrh, len, NULL);
+    }
+}
+
+static const VMStateDescription vmstate_ipmi_sim = {
+    .name = TYPE_IPMI_BMC_SIMULATOR,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_UINT8(bmc_global_enables, IPMIBmcSim),
+        VMSTATE_UINT8(msg_flags, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_initialized, IPMIBmcSim),
+        VMSTATE_UINT8(watchdog_use, IPMIBmcSim),
+        VMSTATE_UINT8(watchdog_action, IPMIBmcSim),
+        VMSTATE_UINT8(watchdog_pretimeout, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_expired, IPMIBmcSim),
+        VMSTATE_UINT16(watchdog_timeout, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_running, IPMIBmcSim),
+        VMSTATE_BOOL(watchdog_preaction_ran, IPMIBmcSim),
+        VMSTATE_INT64(watchdog_expiry, IPMIBmcSim),
+        VMSTATE_UINT8_ARRAY(evtbuf, IPMIBmcSim, 16),
+        VMSTATE_UINT8(sensors[IPMI_WATCHDOG_SENSOR].status, IPMIBmcSim),
+        VMSTATE_UINT8(sensors[IPMI_WATCHDOG_SENSOR].reading, IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].states, IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].assert_states, IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].deassert_states,
+                       IPMIBmcSim),
+        VMSTATE_UINT16(sensors[IPMI_WATCHDOG_SENSOR].assert_enable, IPMIBmcSim),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ipmi_sim_realize(DeviceState *dev, Error **errp)
+{
+    IPMIBmc *b = IPMI_BMC(dev);
+    unsigned int i;
+    IPMIBmcSim *ibs = IPMI_BMC_SIMULATOR(b);
+
+    qemu_mutex_init(&ibs->lock);
+    QTAILQ_INIT(&ibs->rcvbufs);
+
+    ibs->bmc_global_enables = (1 << IPMI_BMC_EVENT_LOG_BIT);
+    ibs->device_id = 0x20;
+    ibs->ipmi_version = 0x02; /* IPMI 2.0 */
+    ibs->restart_cause = 0;
+    for (i = 0; i < 4; i++) {
+        ibs->sel.last_addition[i] = 0xff;
+        ibs->sel.last_clear[i] = 0xff;
+        ibs->sdr.last_addition[i] = 0xff;
+        ibs->sdr.last_clear[i] = 0xff;
+    }
+
+    ipmi_sdr_init(ibs);
+
+    ibs->acpi_power_state[0] = 0;
+    ibs->acpi_power_state[1] = 0;
+
+    if (qemu_uuid_set) {
+        memcpy(&ibs->uuid, qemu_uuid, 16);
+    } else {
+        memset(&ibs->uuid, 0, 16);
+    }
+
+    ipmi_init_sensors_from_sdrs(ibs);
+    register_cmds(ibs);
+
+    ibs->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, ipmi_timeout, ibs);
+
+    vmstate_register(NULL, 0, &vmstate_ipmi_sim, ibs);
+}
+
+static void ipmi_sim_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIBmcClass *bk = IPMI_BMC_CLASS(oc);
+
+    dc->realize = ipmi_sim_realize;
+    bk->handle_command = ipmi_sim_handle_command;
+}
+
+static const TypeInfo ipmi_sim_type = {
+    .name          = TYPE_IPMI_BMC_SIMULATOR,
+    .parent        = TYPE_IPMI_BMC,
+    .instance_size = sizeof(IPMIBmcSim),
+    .class_init    = ipmi_sim_class_init,
+};
+
+static void ipmi_sim_register_types(void)
+{
+    type_register_static(&ipmi_sim_type);
+}
+
+type_init(ipmi_sim_register_types)
diff --git a/hw/ipmi/isa_ipmi_bt.c b/hw/ipmi/isa_ipmi_bt.c
new file mode 100644
index 0000000..f036617
--- /dev/null
+++ b/hw/ipmi/isa_ipmi_bt.c
@@ -0,0 +1,537 @@
+/*
+ * QEMU ISA IPMI BT emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/ipmi/ipmi.h"
+#include "hw/isa/isa.h"
+#include "hw/i386/pc.h"
+
+/* Control register */
+#define IPMI_BT_CLR_WR_BIT         0
+#define IPMI_BT_CLR_RD_BIT         1
+#define IPMI_BT_H2B_ATN_BIT        2
+#define IPMI_BT_B2H_ATN_BIT        3
+#define IPMI_BT_SMS_ATN_BIT        4
+#define IPMI_BT_HBUSY_BIT          6
+#define IPMI_BT_BBUSY_BIT          7
+
+#define IPMI_BT_CLR_WR_MASK        (1 << IPMI_BT_CLR_WR_BIT)
+#define IPMI_BT_GET_CLR_WR(d)      (((d) >> IPMI_BT_CLR_WR_BIT) & 0x1)
+#define IPMI_BT_SET_CLR_WR(d, v)   (d) = (((d) & ~IPMI_BT_CLR_WR_MASK) | \
+                                       (((v & 1) << IPMI_BT_CLR_WR_BIT)))
+
+#define IPMI_BT_CLR_RD_MASK        (1 << IPMI_BT_CLR_RD_BIT)
+#define IPMI_BT_GET_CLR_RD(d)      (((d) >> IPMI_BT_CLR_RD_BIT) & 0x1)
+#define IPMI_BT_SET_CLR_RD(d, v)   (d) = (((d) & ~IPMI_BT_CLR_RD_MASK) | \
+                                       (((v & 1) << IPMI_BT_CLR_RD_BIT)))
+
+#define IPMI_BT_H2B_ATN_MASK       (1 << IPMI_BT_H2B_ATN_BIT)
+#define IPMI_BT_GET_H2B_ATN(d)     (((d) >> IPMI_BT_H2B_ATN_BIT) & 0x1)
+#define IPMI_BT_SET_H2B_ATN(d, v)  (d) = (((d) & ~IPMI_BT_H2B_ATN_MASK) | \
+                                        (((v & 1) << IPMI_BT_H2B_ATN_BIT)))
+
+#define IPMI_BT_B2H_ATN_MASK       (1 << IPMI_BT_B2H_ATN_BIT)
+#define IPMI_BT_GET_B2H_ATN(d)     (((d) >> IPMI_BT_B2H_ATN_BIT) & 0x1)
+#define IPMI_BT_SET_B2H_ATN(d, v)  (d) = (((d) & ~IPMI_BT_B2H_ATN_MASK) | \
+                                        (((v & 1) << IPMI_BT_B2H_ATN_BIT)))
+
+#define IPMI_BT_SMS_ATN_MASK       (1 << IPMI_BT_SMS_ATN_BIT)
+#define IPMI_BT_GET_SMS_ATN(d)     (((d) >> IPMI_BT_SMS_ATN_BIT) & 0x1)
+#define IPMI_BT_SET_SMS_ATN(d, v)  (d) = (((d) & ~IPMI_BT_SMS_ATN_MASK) | \
+                                        (((v & 1) << IPMI_BT_SMS_ATN_BIT)))
+
+#define IPMI_BT_HBUSY_MASK         (1 << IPMI_BT_HBUSY_BIT)
+#define IPMI_BT_GET_HBUSY(d)       (((d) >> IPMI_BT_HBUSY_BIT) & 0x1)
+#define IPMI_BT_SET_HBUSY(d, v)    (d) = (((d) & ~IPMI_BT_HBUSY_MASK) | \
+                                       (((v & 1) << IPMI_BT_HBUSY_BIT)))
+
+#define IPMI_BT_BBUSY_MASK         (1 << IPMI_BT_BBUSY_BIT)
+#define IPMI_BT_GET_BBUSY(d)       (((d) >> IPMI_BT_BBUSY_BIT) & 0x1)
+#define IPMI_BT_SET_BBUSY(d, v)    (d) = (((d) & ~IPMI_BT_BBUSY_MASK) | \
+                                       (((v & 1) << IPMI_BT_BBUSY_BIT)))
+
+
+/* Mask register */
+#define IPMI_BT_B2H_IRQ_EN_BIT     0
+#define IPMI_BT_B2H_IRQ_BIT        1
+
+#define IPMI_BT_B2H_IRQ_EN_MASK      (1 << IPMI_BT_B2H_IRQ_EN_BIT)
+#define IPMI_BT_GET_B2H_IRQ_EN(d)    (((d) >> IPMI_BT_B2H_IRQ_EN_BIT) & 0x1)
+#define IPMI_BT_SET_B2H_IRQ_EN(d, v) (d) = (((d) & ~IPMI_BT_B2H_IRQ_EN_MASK) | \
+                                        (((v & 1) << IPMI_BT_B2H_IRQ_EN_BIT)))
+
+#define IPMI_BT_B2H_IRQ_MASK         (1 << IPMI_BT_B2H_IRQ_BIT)
+#define IPMI_BT_GET_B2H_IRQ(d)       (((d) >> IPMI_BT_B2H_IRQ_BIT) & 0x1)
+#define IPMI_BT_SET_B2H_IRQ(d, v)    (d) = (((d) & ~IPMI_BT_B2H_IRQ_MASK) | \
+                                        (((v & 1) << IPMI_BT_B2H_IRQ_BIT)))
+
+typedef struct IPMIBT {
+    IPMIBmc *bmc;
+
+    bool do_wake;
+
+    qemu_irq irq;
+
+    uint32_t io_base;
+    unsigned long io_length;
+    MemoryRegion io;
+
+    bool obf_irq_set;
+    bool atn_irq_set;
+    bool use_irq;
+    bool irqs_enabled;
+
+    uint8_t outmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t outpos;
+    uint32_t outlen;
+
+    uint8_t inmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t inlen;
+
+    uint8_t control_reg;
+    uint8_t mask_reg;
+
+    /*
+     * This is a response number that we send with the command to make
+     * sure that the response matches the command.
+     */
+    uint8_t waiting_rsp;
+    uint8_t waiting_seq;
+} IPMIBT;
+
+#define IPMI_CMD_GET_BT_INTF_CAP        0x36
+
+static void ipmi_bt_handle_event(IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (ib->inlen < 4) {
+        goto out;
+    }
+    /* Note that overruns are handled by handle_command */
+    if (ib->inmsg[0] != (ib->inlen - 1)) {
+        /* Length mismatch, just ignore. */
+        IPMI_BT_SET_BBUSY(ib->control_reg, 1);
+        ib->inlen = 0;
+        goto out;
+    }
+    if ((ib->inmsg[1] == (IPMI_NETFN_APP << 2)) &&
+                        (ib->inmsg[3] == IPMI_CMD_GET_BT_INTF_CAP)) {
+        /* We handle this one ourselves. */
+        ib->outmsg[0] = 9;
+        ib->outmsg[1] = ib->inmsg[1] | 0x04;
+        ib->outmsg[2] = ib->inmsg[2];
+        ib->outmsg[3] = ib->inmsg[3];
+        ib->outmsg[4] = 0;
+        ib->outmsg[5] = 1; /* Only support 1 outstanding request. */
+        if (sizeof(ib->inmsg) > 0xff) { /* Input buffer size */
+            ib->outmsg[6] = 0xff;
+        } else {
+            ib->outmsg[6] = (unsigned char) sizeof(ib->inmsg);
+        }
+        if (sizeof(ib->outmsg) > 0xff) { /* Output buffer size */
+            ib->outmsg[7] = 0xff;
+        } else {
+            ib->outmsg[7] = (unsigned char) sizeof(ib->outmsg);
+        }
+        ib->outmsg[8] = 10; /* Max request to response time */
+        ib->outmsg[9] = 0; /* Don't recommend retries */
+        ib->outlen = 10;
+        IPMI_BT_SET_BBUSY(ib->control_reg, 0);
+        IPMI_BT_SET_B2H_ATN(ib->control_reg, 1);
+        if (ib->use_irq && ib->irqs_enabled &&
+                !IPMI_BT_GET_B2H_IRQ(ib->mask_reg) &&
+                IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+            qemu_irq_raise(ib->irq);
+        }
+        goto out;
+    }
+    ib->waiting_seq = ib->inmsg[2];
+    ib->inmsg[2] = ib->inmsg[1];
+    {
+        IPMIBmcClass *bk = IPMI_BMC_GET_CLASS(ib->bmc);
+        bk->handle_command(ib->bmc, ib->inmsg + 2, ib->inlen - 2,
+                           sizeof(ib->inmsg), ib->waiting_rsp);
+    }
+ out:
+    return;
+}
+
+static void ipmi_bt_handle_rsp(IPMIInterface *ii, uint8_t msg_id,
+                                unsigned char *rsp, unsigned int rsp_len)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (ib->waiting_rsp == msg_id) {
+        ib->waiting_rsp++;
+        if (rsp_len > (sizeof(ib->outmsg) - 2)) {
+            ib->outmsg[0] = 4;
+            ib->outmsg[1] = rsp[0];
+            ib->outmsg[2] = ib->waiting_seq;
+            ib->outmsg[3] = rsp[1];
+            ib->outmsg[4] = IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES;
+            ib->outlen = 5;
+        } else {
+            ib->outmsg[0] = rsp_len + 1;
+            ib->outmsg[1] = rsp[0];
+            ib->outmsg[2] = ib->waiting_seq;
+            memcpy(ib->outmsg + 3, rsp + 1, rsp_len - 1);
+            ib->outlen = rsp_len + 2;
+        }
+        IPMI_BT_SET_BBUSY(ib->control_reg, 0);
+        IPMI_BT_SET_B2H_ATN(ib->control_reg, 1);
+        if (ib->use_irq && ib->irqs_enabled &&
+                !IPMI_BT_GET_B2H_IRQ(ib->mask_reg) &&
+                IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+            qemu_irq_raise(ib->irq);
+        }
+    }
+}
+
+
+static uint64_t ipmi_bt_ioport_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+    uint32_t ret = 0xff;
+
+    switch (addr & 3) {
+    case 0:
+        ret = ib->control_reg;
+        break;
+    case 1:
+        if (ib->outpos < ib->outlen) {
+            ret = ib->outmsg[ib->outpos];
+            ib->outpos++;
+            if (ib->outpos == ib->outlen) {
+                ib->outpos = 0;
+                ib->outlen = 0;
+            }
+        } else {
+            ret = 0xff;
+        }
+        break;
+    case 2:
+        ret = ib->mask_reg;
+        break;
+    }
+    return ret;
+}
+
+static void ipmi_bt_signal(IPMIBT *ib, IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    ib->do_wake = 1;
+    while (ib->do_wake) {
+        ib->do_wake = 0;
+        iic->handle_if_event(ii);
+    }
+}
+
+static void ipmi_bt_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                 unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    switch (addr & 3) {
+    case 0:
+        if (IPMI_BT_GET_CLR_WR(val)) {
+            ib->inlen = 0;
+        }
+        if (IPMI_BT_GET_CLR_RD(val)) {
+            ib->outpos = 0;
+        }
+        if (IPMI_BT_GET_B2H_ATN(val)) {
+            IPMI_BT_SET_B2H_ATN(ib->control_reg, 0);
+        }
+        if (IPMI_BT_GET_SMS_ATN(val)) {
+            IPMI_BT_SET_SMS_ATN(ib->control_reg, 0);
+        }
+        if (IPMI_BT_GET_HBUSY(val)) {
+            /* Toggle */
+            IPMI_BT_SET_HBUSY(ib->control_reg,
+                              !IPMI_BT_GET_HBUSY(ib->control_reg));
+        }
+        if (IPMI_BT_GET_H2B_ATN(val)) {
+            IPMI_BT_SET_BBUSY(ib->control_reg, 1);
+            ipmi_bt_signal(ib, ii);
+        }
+        break;
+
+    case 1:
+        if (ib->inlen < sizeof(ib->inmsg)) {
+            ib->inmsg[ib->inlen] = val;
+        }
+        ib->inlen++;
+        break;
+
+    case 2:
+        if (IPMI_BT_GET_B2H_IRQ_EN(val) !=
+                        IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            if (IPMI_BT_GET_B2H_IRQ_EN(val)) {
+                if (IPMI_BT_GET_B2H_ATN(ib->control_reg) ||
+                        IPMI_BT_GET_SMS_ATN(ib->control_reg)) {
+                    IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+                    qemu_irq_raise(ib->irq);
+                }
+                IPMI_BT_SET_B2H_IRQ_EN(ib->mask_reg, 1);
+            } else {
+                if (IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+                    IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+                    qemu_irq_lower(ib->irq);
+                }
+                IPMI_BT_SET_B2H_IRQ_EN(ib->mask_reg, 0);
+            }
+        }
+        if (IPMI_BT_GET_B2H_IRQ(val) && IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+            qemu_irq_lower(ib->irq);
+        }
+        break;
+    }
+}
+
+static const MemoryRegionOps ipmi_bt_io_ops = {
+    .read = ipmi_bt_ioport_read,
+    .write = ipmi_bt_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ipmi_bt_set_atn(IPMIInterface *ii, int val, int irq)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (!!val == IPMI_BT_GET_SMS_ATN(ib->control_reg)) {
+        return;
+    }
+
+    IPMI_BT_SET_SMS_ATN(ib->control_reg, val);
+    if (val) {
+        if (irq && ib->use_irq && ib->irqs_enabled &&
+                !IPMI_BT_GET_B2H_ATN(ib->control_reg) &&
+                IPMI_BT_GET_B2H_IRQ_EN(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 1);
+            qemu_irq_raise(ib->irq);
+        }
+    } else {
+        if (!IPMI_BT_GET_B2H_ATN(ib->control_reg) &&
+                IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+            qemu_irq_lower(ib->irq);
+        }
+    }
+}
+
+static void ipmi_bt_handle_reset(IPMIInterface *ii, bool is_cold)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    if (is_cold) {
+        /* Disable the BT interrupt on reset */
+        if (IPMI_BT_GET_B2H_IRQ(ib->mask_reg)) {
+            IPMI_BT_SET_B2H_IRQ(ib->mask_reg, 0);
+            qemu_irq_lower(ib->irq);
+        }
+        IPMI_BT_SET_B2H_IRQ_EN(ib->mask_reg, 0);
+    }
+}
+
+static void ipmi_bt_set_irq_enable(IPMIInterface *ii, int val)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    ib->irqs_enabled = val;
+}
+
+static void ipmi_bt_init(IPMIInterface *ii, Error **errp)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIBT *ib = iic->get_backend_data(ii);
+
+    ib->io_length = 3;
+
+    memory_region_init_io(&ib->io, NULL, &ipmi_bt_io_ops, ii, "ipmi-bt", 3);
+}
+
+
+#define TYPE_ISA_IPMI_BT "isa-ipmi-bt"
+#define ISA_IPMI_BT(obj) OBJECT_CHECK(ISAIPMIBTDevice, (obj), \
+                                       TYPE_ISA_IPMI_BT)
+
+typedef struct ISAIPMIBTDevice {
+    ISADevice dev;
+    int32_t isairq;
+    IPMIBT bt;
+    uint32_t uuid;
+} ISAIPMIBTDevice;
+
+static void ipmi_bt_get_fwinfo(struct IPMIInterface *ii, IPMIFwInfo *info)
+{
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(ii);
+
+    info->interface_name = "bt";
+    info->interface_type = IPMI_SMBIOS_BT;
+    info->ipmi_spec_major_revision = 2;
+    info->ipmi_spec_minor_revision = 0;
+    info->base_address = iib->bt.io_base;
+    info->register_length = iib->bt.io_length;
+    info->register_spacing = 1;
+    info->memspace = IPMI_MEMSPACE_IO;
+    info->irq_type = IPMI_LEVEL_IRQ;
+    info->interrupt_number = iib->isairq;
+    info->i2c_slave_address = iib->bt.bmc->slave_addr;
+    info->uuid = iib->uuid;
+}
+
+static void ipmi_bt_class_init(IPMIInterfaceClass *iic)
+{
+    iic->init = ipmi_bt_init;
+    iic->set_atn = ipmi_bt_set_atn;
+    iic->handle_rsp = ipmi_bt_handle_rsp;
+    iic->handle_if_event = ipmi_bt_handle_event;
+    iic->set_irq_enable = ipmi_bt_set_irq_enable;
+    iic->reset = ipmi_bt_handle_reset;
+    iic->get_fwinfo = ipmi_bt_get_fwinfo;
+}
+
+static void isa_ipmi_bt_realize(DeviceState *dev, Error **errp)
+{
+    ISADevice *isadev = ISA_DEVICE(dev);
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(dev);
+    IPMIInterface *ii = IPMI_INTERFACE(dev);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    if (!iib->bt.bmc) {
+        error_setg(errp, "IPMI device requires a bmc attribute to be set");
+        return;
+    }
+
+    iib->uuid = ipmi_next_uuid();
+
+    iib->bt.bmc->intf = ii;
+
+    iic->init(ii, errp);
+    if (*errp)
+        return;
+
+    if (iib->isairq > 0) {
+        isa_init_irq(isadev, &iib->bt.irq, iib->isairq);
+        iib->bt.use_irq = 1;
+    }
+
+    qdev_set_legacy_instance_id(dev, iib->bt.io_base, iib->bt.io_length);
+
+    isa_register_ioport(isadev, &iib->bt.io, iib->bt.io_base);
+}
+
+static const VMStateDescription vmstate_ISAIPMIBTDevice = {
+    .name = TYPE_IPMI_INTERFACE,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(bt.obf_irq_set, ISAIPMIBTDevice),
+        VMSTATE_BOOL(bt.atn_irq_set, ISAIPMIBTDevice),
+        VMSTATE_BOOL(bt.use_irq, ISAIPMIBTDevice),
+        VMSTATE_BOOL(bt.irqs_enabled, ISAIPMIBTDevice),
+        VMSTATE_UINT32(bt.outpos, ISAIPMIBTDevice),
+        VMSTATE_VBUFFER_UINT32(bt.outmsg, ISAIPMIBTDevice, 1, NULL, 0,
+                               bt.outlen),
+        VMSTATE_VBUFFER_UINT32(bt.inmsg, ISAIPMIBTDevice, 1, NULL, 0,
+                               bt.inlen),
+        VMSTATE_UINT8(bt.control_reg, ISAIPMIBTDevice),
+        VMSTATE_UINT8(bt.mask_reg, ISAIPMIBTDevice),
+        VMSTATE_UINT8(bt.waiting_rsp, ISAIPMIBTDevice),
+        VMSTATE_UINT8(bt.waiting_seq, ISAIPMIBTDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void isa_ipmi_bt_init(Object *obj)
+{
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(obj);
+
+    ipmi_bmc_find_and_link(obj, (Object **) &iib->bt.bmc);
+
+    vmstate_register(NULL, 0, &vmstate_ISAIPMIBTDevice, iib);
+}
+
+static void *isa_ipmi_bt_get_backend_data(IPMIInterface *ii)
+{
+    ISAIPMIBTDevice *iib = ISA_IPMI_BT(ii);
+
+    return &iib->bt;
+}
+
+static Property ipmi_isa_properties[] = {
+    DEFINE_PROP_UINT32("ioport", ISAIPMIBTDevice, bt.io_base,  0xe4),
+    DEFINE_PROP_INT32("irq",   ISAIPMIBTDevice, isairq,  5),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void isa_ipmi_bt_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_CLASS(oc);
+
+    dc->realize = isa_ipmi_bt_realize;
+    dc->props = ipmi_isa_properties;
+
+    iic->get_backend_data = isa_ipmi_bt_get_backend_data;
+    ipmi_bt_class_init(iic);
+}
+
+static const TypeInfo isa_ipmi_bt_info = {
+    .name          = TYPE_ISA_IPMI_BT,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISAIPMIBTDevice),
+    .instance_init = isa_ipmi_bt_init,
+    .class_init    = isa_ipmi_bt_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_IPMI_INTERFACE },
+        { }
+    }
+};
+
+static void ipmi_register_types(void)
+{
+    type_register_static(&isa_ipmi_bt_info);
+}
+
+type_init(ipmi_register_types)
diff --git a/hw/ipmi/isa_ipmi_kcs.c b/hw/ipmi/isa_ipmi_kcs.c
new file mode 100644
index 0000000..9a38f8a
--- /dev/null
+++ b/hw/ipmi/isa_ipmi_kcs.c
@@ -0,0 +1,501 @@
+/*
+ * QEMU ISA IPMI KCS emulation
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hw.h"
+#include "hw/ipmi/ipmi.h"
+#include "hw/isa/isa.h"
+#include "hw/i386/pc.h"
+
+#define IPMI_KCS_OBF_BIT        0
+#define IPMI_KCS_IBF_BIT        1
+#define IPMI_KCS_SMS_ATN_BIT    2
+#define IPMI_KCS_CD_BIT         3
+
+#define IPMI_KCS_OBF_MASK          (1 << IPMI_KCS_OBF_BIT)
+#define IPMI_KCS_GET_OBF(d)        (((d) >> IPMI_KCS_OBF_BIT) & 0x1)
+#define IPMI_KCS_SET_OBF(d, v)     (d) = (((d) & ~IPMI_KCS_OBF_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_OBF_BIT))
+#define IPMI_KCS_IBF_MASK          (1 << IPMI_KCS_IBF_BIT)
+#define IPMI_KCS_GET_IBF(d)        (((d) >> IPMI_KCS_IBF_BIT) & 0x1)
+#define IPMI_KCS_SET_IBF(d, v)     (d) = (((d) & ~IPMI_KCS_IBF_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_IBF_BIT))
+#define IPMI_KCS_SMS_ATN_MASK      (1 << IPMI_KCS_SMS_ATN_BIT)
+#define IPMI_KCS_GET_SMS_ATN(d)    (((d) >> IPMI_KCS_SMS_ATN_BIT) & 0x1)
+#define IPMI_KCS_SET_SMS_ATN(d, v) (d) = (((d) & ~IPMI_KCS_SMS_ATN_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_SMS_ATN_BIT))
+#define IPMI_KCS_CD_MASK           (1 << IPMI_KCS_CD_BIT)
+#define IPMI_KCS_GET_CD(d)         (((d) >> IPMI_KCS_CD_BIT) & 0x1)
+#define IPMI_KCS_SET_CD(d, v)      (d) = (((d) & ~IPMI_KCS_CD_MASK) | \
+                                       (((v) & 1) << IPMI_KCS_CD_BIT))
+
+#define IPMI_KCS_IDLE_STATE        0
+#define IPMI_KCS_READ_STATE        1
+#define IPMI_KCS_WRITE_STATE       2
+#define IPMI_KCS_ERROR_STATE       3
+
+#define IPMI_KCS_GET_STATE(d)    (((d) >> 6) & 0x3)
+#define IPMI_KCS_SET_STATE(d, v) ((d) = ((d) & ~0xc0) | (((v) & 0x3) << 6))
+
+#define IPMI_KCS_ABORT_STATUS_CMD       0x60
+#define IPMI_KCS_WRITE_START_CMD        0x61
+#define IPMI_KCS_WRITE_END_CMD          0x62
+#define IPMI_KCS_READ_CMD               0x68
+
+#define IPMI_KCS_STATUS_NO_ERR          0x00
+#define IPMI_KCS_STATUS_ABORTED_ERR     0x01
+#define IPMI_KCS_STATUS_BAD_CC_ERR      0x02
+#define IPMI_KCS_STATUS_LENGTH_ERR      0x06
+
+typedef struct IPMIKCS {
+    IPMIBmc *bmc;
+
+    bool do_wake;
+
+    qemu_irq irq;
+
+    uint32_t io_base;
+    unsigned long io_length;
+    MemoryRegion io;
+
+    bool obf_irq_set;
+    bool atn_irq_set;
+    bool use_irq;
+    bool irqs_enabled;
+
+    uint8_t outmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t outpos;
+    uint32_t outlen;
+
+    uint8_t inmsg[MAX_IPMI_MSG_SIZE];
+    uint32_t inlen;
+    bool write_end;
+
+    uint8_t status_reg;
+    uint8_t data_out_reg;
+
+    int16_t data_in_reg; /* -1 means not written */
+    int16_t cmd_reg;
+
+    /*
+     * This is a response number that we send with the command to make
+     * sure that the response matches the command.
+     */
+    uint8_t waiting_rsp;
+} IPMIKCS;
+
+#define SET_OBF() \
+    do {                                                                      \
+        IPMI_KCS_SET_OBF(ik->status_reg, 1);                                  \
+        if (ik->use_irq && ik->irqs_enabled && !ik->obf_irq_set) {            \
+            ik->obf_irq_set = 1;                                              \
+            if (!ik->atn_irq_set) {                                           \
+                qemu_irq_raise(ik->irq);                                      \
+            }                                                                 \
+        }                                                                     \
+    } while (0)
+
+static void ipmi_kcs_signal(IPMIKCS *ik, IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    ik->do_wake = 1;
+    while (ik->do_wake) {
+        ik->do_wake = 0;
+        iic->handle_if_event(ii);
+    }
+}
+
+static void ipmi_kcs_handle_event(IPMIInterface *ii)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    if (ik->cmd_reg == IPMI_KCS_ABORT_STATUS_CMD) {
+        if (IPMI_KCS_GET_STATE(ik->status_reg) != IPMI_KCS_ERROR_STATE) {
+            ik->waiting_rsp++; /* Invalidate the message */
+            ik->outmsg[0] = IPMI_KCS_STATUS_ABORTED_ERR;
+            ik->outlen = 1;
+            ik->outpos = 0;
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_ERROR_STATE);
+            SET_OBF();
+        }
+        goto out;
+    }
+
+    switch (IPMI_KCS_GET_STATE(ik->status_reg)) {
+    case IPMI_KCS_IDLE_STATE:
+        if (ik->cmd_reg == IPMI_KCS_WRITE_START_CMD) {
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_WRITE_STATE);
+            ik->cmd_reg = -1;
+            ik->write_end = 0;
+            ik->inlen = 0;
+            SET_OBF();
+        }
+        break;
+
+    case IPMI_KCS_READ_STATE:
+    handle_read:
+        if (ik->outpos >= ik->outlen) {
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_IDLE_STATE);
+            SET_OBF();
+        } else if (ik->data_in_reg == IPMI_KCS_READ_CMD) {
+            ik->data_out_reg = ik->outmsg[ik->outpos];
+            ik->outpos++;
+            SET_OBF();
+        } else {
+            ik->outmsg[0] = IPMI_KCS_STATUS_BAD_CC_ERR;
+            ik->outlen = 1;
+            ik->outpos = 0;
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_ERROR_STATE);
+            SET_OBF();
+            goto out;
+        }
+        break;
+
+    case IPMI_KCS_WRITE_STATE:
+        if (ik->data_in_reg != -1) {
+            /*
+             * Don't worry about input overrun here, that will be
+             * handled in the BMC.
+             */
+            if (ik->inlen < sizeof(ik->inmsg)) {
+                ik->inmsg[ik->inlen] = ik->data_in_reg;
+            }
+            ik->inlen++;
+        }
+        if (ik->write_end) {
+            IPMIBmcClass *bk = IPMI_BMC_GET_CLASS(ik->bmc);
+            ik->outlen = 0;
+            ik->write_end = 0;
+            ik->outpos = 0;
+            bk->handle_command(ik->bmc, ik->inmsg, ik->inlen, sizeof(ik->inmsg),
+                               ik->waiting_rsp);
+            goto out_noibf;
+        } else if (ik->cmd_reg == IPMI_KCS_WRITE_END_CMD) {
+            ik->cmd_reg = -1;
+            ik->write_end = 1;
+        }
+        SET_OBF();
+        break;
+
+    case IPMI_KCS_ERROR_STATE:
+        if (ik->data_in_reg != -1) {
+            IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_READ_STATE);
+            ik->data_in_reg = IPMI_KCS_READ_CMD;
+            goto handle_read;
+        }
+        break;
+    }
+
+    if (ik->cmd_reg != -1) {
+        /* Got an invalid command */
+        ik->outmsg[0] = IPMI_KCS_STATUS_BAD_CC_ERR;
+        ik->outlen = 1;
+        ik->outpos = 0;
+        IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_ERROR_STATE);
+    }
+
+ out:
+    ik->cmd_reg = -1;
+    ik->data_in_reg = -1;
+    IPMI_KCS_SET_IBF(ik->status_reg, 0);
+ out_noibf:
+    return;
+}
+
+static void ipmi_kcs_handle_rsp(IPMIInterface *ii, uint8_t msg_id,
+                                unsigned char *rsp, unsigned int rsp_len)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    if (ik->waiting_rsp == msg_id) {
+        ik->waiting_rsp++;
+        if (rsp_len > sizeof(ik->outmsg)) {
+            ik->outmsg[0] = rsp[0];
+            ik->outmsg[1] = rsp[1];
+            ik->outmsg[2] = IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES;
+            ik->outlen = 3;
+        } else {
+            memcpy(ik->outmsg, rsp, rsp_len);
+            ik->outlen = rsp_len;
+        }
+        IPMI_KCS_SET_STATE(ik->status_reg, IPMI_KCS_READ_STATE);
+        ik->data_in_reg = IPMI_KCS_READ_CMD;
+        ipmi_kcs_signal(ik, ii);
+    }
+}
+
+
+static uint64_t ipmi_kcs_ioport_read(void *opaque, hwaddr addr, unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+    uint32_t ret;
+
+    switch (addr & 1) {
+    case 0:
+        ret = ik->data_out_reg;
+        IPMI_KCS_SET_OBF(ik->status_reg, 0);
+        if (ik->obf_irq_set) {
+            ik->obf_irq_set = 0;
+            if (!ik->atn_irq_set) {
+                qemu_irq_lower(ik->irq);
+            }
+        }
+        break;
+    case 1:
+        ret = ik->status_reg;
+        if (ik->atn_irq_set) {
+            ik->atn_irq_set = 0;
+            if (!ik->obf_irq_set) {
+                qemu_irq_lower(ik->irq);
+            }
+        }
+        break;
+    }
+    return ret;
+}
+
+static void ipmi_kcs_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                  unsigned size)
+{
+    IPMIInterface *ii = opaque;
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    if (IPMI_KCS_GET_IBF(ik->status_reg)) {
+        return;
+    }
+
+    switch (addr & 1) {
+    case 0:
+        ik->data_in_reg = val;
+        break;
+
+    case 1:
+        ik->cmd_reg = val;
+        break;
+    }
+    IPMI_KCS_SET_IBF(ik->status_reg, 1);
+    ipmi_kcs_signal(ik, ii);
+}
+
+const MemoryRegionOps ipmi_kcs_io_ops = {
+    .read = ipmi_kcs_ioport_read,
+    .write = ipmi_kcs_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void ipmi_kcs_set_atn(IPMIInterface *ii, int val, int irq)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    IPMI_KCS_SET_SMS_ATN(ik->status_reg, val);
+    if (val) {
+        if (irq && !ik->atn_irq_set && ik->use_irq && ik->irqs_enabled) {
+            ik->atn_irq_set = 1;
+            if (!ik->obf_irq_set) {
+                qemu_irq_raise(ik->irq);
+            }
+        }
+    } else {
+        if (ik->atn_irq_set) {
+            ik->atn_irq_set = 0;
+            if (!ik->obf_irq_set) {
+                qemu_irq_lower(ik->irq);
+            }
+        }
+    }
+}
+
+static void ipmi_kcs_set_irq_enable(IPMIInterface *ii, int val)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    ik->irqs_enabled = val;
+}
+
+static void ipmi_kcs_init(IPMIInterface *ii, Error **errp)
+{
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+    IPMIKCS *ik = iic->get_backend_data(ii);
+
+    ik->io_length = 2;
+    memory_region_init_io(&ik->io, NULL, &ipmi_kcs_io_ops, ii, "ipmi-kcs", 2);
+}
+
+#define TYPE_ISA_IPMI_KCS "isa-ipmi-kcs"
+#define ISA_IPMI_KCS(obj) OBJECT_CHECK(ISAIPMIKCSDevice, (obj), \
+                                       TYPE_ISA_IPMI_KCS)
+
+typedef struct ISAIPMIKCSDevice {
+    ISADevice dev;
+    int32_t isairq;
+    IPMIKCS kcs;
+    uint32_t uuid;
+} ISAIPMIKCSDevice;
+
+static void ipmi_kcs_get_fwinfo(IPMIInterface *ii, IPMIFwInfo *info)
+{
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(ii);
+
+    info->interface_name = "kcs";
+    info->interface_type = IPMI_SMBIOS_KCS;
+    info->ipmi_spec_major_revision = 2;
+    info->ipmi_spec_minor_revision = 0;
+    info->base_address = iik->kcs.io_base;
+    info->i2c_slave_address = iik->kcs.bmc->slave_addr;
+    info->register_length = iik->kcs.io_length;
+    info->register_spacing = 1;
+    info->memspace = IPMI_MEMSPACE_IO;
+    info->irq_type = IPMI_LEVEL_IRQ;
+    info->interrupt_number = iik->isairq;
+    info->uuid = iik->uuid;
+}
+
+static void ipmi_kcs_class_init(IPMIInterfaceClass *iic)
+{
+    iic->init = ipmi_kcs_init;
+    iic->set_atn = ipmi_kcs_set_atn;
+    iic->handle_rsp = ipmi_kcs_handle_rsp;
+    iic->handle_if_event = ipmi_kcs_handle_event;
+    iic->set_irq_enable = ipmi_kcs_set_irq_enable;
+    iic->get_fwinfo = ipmi_kcs_get_fwinfo;
+}
+
+static void ipmi_isa_realize(DeviceState *dev, Error **errp)
+{
+    ISADevice *isadev = ISA_DEVICE(dev);
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(dev);
+    IPMIInterface *ii = IPMI_INTERFACE(dev);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_GET_CLASS(ii);
+
+    if (!iik->kcs.bmc) {
+        error_setg(errp, "IPMI device requires a bmc attribute to be set");
+        return;
+    }
+
+    iik->uuid = ipmi_next_uuid();
+
+    iik->kcs.bmc->intf = ii;
+
+    iic->init(ii, errp);
+    if (*errp)
+        return;
+
+    if (iik->isairq > 0) {
+        isa_init_irq(isadev, &iik->kcs.irq, iik->isairq);
+        iik->kcs.use_irq = 1;
+    }
+
+    qdev_set_legacy_instance_id(dev, iik->kcs.io_base, iik->kcs.io_length);
+
+    isa_register_ioport(isadev, &iik->kcs.io, iik->kcs.io_base);
+}
+
+const VMStateDescription vmstate_ISAIPMIKCSDevice = {
+    .name = TYPE_IPMI_INTERFACE,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(kcs.obf_irq_set, ISAIPMIKCSDevice),
+        VMSTATE_BOOL(kcs.atn_irq_set, ISAIPMIKCSDevice),
+        VMSTATE_BOOL(kcs.use_irq, ISAIPMIKCSDevice),
+        VMSTATE_BOOL(kcs.irqs_enabled, ISAIPMIKCSDevice),
+        VMSTATE_UINT32(kcs.outpos, ISAIPMIKCSDevice),
+        VMSTATE_VBUFFER_UINT32(kcs.outmsg, ISAIPMIKCSDevice, 1, NULL, 0,
+                               kcs.outlen),
+        VMSTATE_VBUFFER_UINT32(kcs.inmsg, ISAIPMIKCSDevice, 1, NULL, 0,
+                               kcs.inlen),
+        VMSTATE_BOOL(kcs.write_end, ISAIPMIKCSDevice),
+        VMSTATE_UINT8(kcs.status_reg, ISAIPMIKCSDevice),
+        VMSTATE_UINT8(kcs.data_out_reg, ISAIPMIKCSDevice),
+        VMSTATE_INT16(kcs.data_in_reg, ISAIPMIKCSDevice),
+        VMSTATE_INT16(kcs.cmd_reg, ISAIPMIKCSDevice),
+        VMSTATE_UINT8(kcs.waiting_rsp, ISAIPMIKCSDevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void isa_ipmi_kcs_init(Object *obj)
+{
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(obj);
+
+    ipmi_bmc_find_and_link(obj, (Object **) &iik->kcs.bmc);
+
+    vmstate_register(NULL, 0, &vmstate_ISAIPMIKCSDevice, iik);
+}
+
+static void *isa_ipmi_kcs_get_backend_data(IPMIInterface *ii)
+{
+    ISAIPMIKCSDevice *iik = ISA_IPMI_KCS(ii);
+
+    return &iik->kcs;
+}
+
+static Property ipmi_isa_properties[] = {
+    DEFINE_PROP_UINT32("ioport", ISAIPMIKCSDevice, kcs.io_base,  0xca2),
+    DEFINE_PROP_INT32("irq",   ISAIPMIKCSDevice, isairq,  5),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void isa_ipmi_kcs_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    IPMIInterfaceClass *iic = IPMI_INTERFACE_CLASS(oc);
+
+    dc->realize = ipmi_isa_realize;
+    dc->props = ipmi_isa_properties;
+
+    iic->get_backend_data = isa_ipmi_kcs_get_backend_data;
+    ipmi_kcs_class_init(iic);
+}
+
+static const TypeInfo isa_ipmi_kcs_info = {
+    .name          = TYPE_ISA_IPMI_KCS,
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISAIPMIKCSDevice),
+    .instance_init = isa_ipmi_kcs_init,
+    .class_init    = isa_ipmi_kcs_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_IPMI_INTERFACE },
+        { }
+    }
+};
+
+static void ipmi_register_types(void)
+{
+    type_register_static(&isa_ipmi_kcs_info);
+}
+
+type_init(ipmi_register_types)
diff --git a/hw/isa/apm.c b/hw/isa/apm.c
index 26ab170..e232b0d 100644
--- a/hw/isa/apm.c
+++ b/hw/isa/apm.c
@@ -20,6 +20,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/isa/apm.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
diff --git a/hw/isa/i82378.c b/hw/isa/i82378.c
index a7d9aa6..4d29a99 100644
--- a/hw/isa/i82378.c
+++ b/hw/isa/i82378.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/pc.h"
 #include "hw/timer/i8254.h"
@@ -58,14 +59,13 @@
     qemu_set_irq(s->i8259[irq], level);
 }
 
-static int i82378_initfn(PCIDevice *pci)
+static void i82378_realize(PCIDevice *pci, Error **errp)
 {
     DeviceState *dev = DEVICE(pci);
     I82378State *s = I82378(dev);
     uint8_t *pci_conf;
     ISABus *isabus;
     ISADevice *isa;
-    qemu_irq *out0_irq;
 
     pci_conf = pci->config;
     pci_set_word(pci_conf + PCI_COMMAND,
@@ -75,7 +75,11 @@
 
     pci_config_set_interrupt_pin(pci_conf, 1); /* interrupt pin 0 */
 
-    isabus = isa_bus_new(dev, pci_address_space_io(pci));
+    isabus = isa_bus_new(dev, get_system_memory(),
+                         pci_address_space_io(pci), errp);
+    if (!isabus) {
+        return;
+    }
 
     /* This device has:
        2 82C59 (irq)
@@ -87,11 +91,9 @@
        All devices accept byte access only, except timer
      */
 
-    /* Workaround the fact that i8259 is not qdev'ified... */
-    out0_irq = qemu_allocate_irqs(i82378_request_out0_irq, s, 1);
-
     /* 2 82C59 (irq) */
-    s->i8259 = i8259_init(isabus, *out0_irq);
+    s->i8259 = i8259_init(isabus,
+                          qemu_allocate_irq(i82378_request_out0_irq, s, 0));
     isa_bus_irqs(isabus, s->i8259);
 
     /* 1 82C54 (pit) */
@@ -102,12 +104,9 @@
 
     /* 2 82C37 (dma) */
     isa = isa_create_simple(isabus, "i82374");
-    qdev_connect_gpio_out(DEVICE(isa), 0, s->out[1]);
 
     /* timer */
     isa_create_simple(isabus, "mc146818rtc");
-
-    return 0;
 }
 
 static void i82378_init(Object *obj)
@@ -115,7 +114,7 @@
     DeviceState *dev = DEVICE(obj);
     I82378State *s = I82378(obj);
 
-    qdev_init_gpio_out(dev, s->out, 2);
+    qdev_init_gpio_out(dev, s->out, 1);
     qdev_init_gpio_in(dev, i82378_request_pic_irq, 16);
 }
 
@@ -124,7 +123,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = i82378_initfn;
+    k->realize = i82378_realize;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82378;
     k->revision = 0x03;
diff --git a/hw/isa/isa-bus.c b/hw/isa/isa-bus.c
index 3117d04..ce74db2 100644
--- a/hw/isa/isa-bus.c
+++ b/hw/isa/isa-bus.c
@@ -16,15 +16,16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "monitor/monitor.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "hw/isa/isa.h"
-#include "exec/address-spaces.h"
+#include "hw/i386/pc.h"
 
 static ISABus *isabus;
-hwaddr isa_mem_base = 0;
 
 static void isabus_dev_print(Monitor *mon, DeviceState *dev, int indent);
 static char *isabus_get_fw_dev_path(DeviceState *dev);
@@ -37,6 +38,12 @@
     k->get_fw_dev_path = isabus_get_fw_dev_path;
 }
 
+static const TypeInfo isa_dma_info = {
+    .name = TYPE_ISADMA,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(IsaDmaClass),
+};
+
 static const TypeInfo isa_bus_info = {
     .name = TYPE_ISA_BUS,
     .parent = TYPE_BUS,
@@ -44,10 +51,11 @@
     .class_init = isa_bus_class_init,
 };
 
-ISABus *isa_bus_new(DeviceState *dev, MemoryRegion *address_space_io)
+ISABus *isa_bus_new(DeviceState *dev, MemoryRegion* address_space,
+                    MemoryRegion *address_space_io, Error **errp)
 {
     if (isabus) {
-        hw_error("Can't create a second ISA bus.");
+        error_setg(errp, "Can't create a second ISA bus");
         return NULL;
     }
     if (!dev) {
@@ -56,15 +64,13 @@
     }
 
     isabus = ISA_BUS(qbus_create(TYPE_ISA_BUS, dev, NULL));
+    isabus->address_space = address_space;
     isabus->address_space_io = address_space_io;
     return isabus;
 }
 
 void isa_bus_irqs(ISABus *bus, qemu_irq *irqs)
 {
-    if (!bus) {
-        hw_error("Can't set isa irqs with no isa bus present.");
-    }
     bus->irqs = irqs;
 }
 
@@ -91,6 +97,27 @@
     dev->nirqs++;
 }
 
+void isa_connect_gpio_out(ISADevice *isadev, int gpioirq, int isairq)
+{
+    qemu_irq irq;
+    isa_init_irq(isadev, &irq, isairq);
+    qdev_connect_gpio_out(DEVICE(isadev), gpioirq, irq);
+}
+
+void isa_bus_dma(ISABus *bus, IsaDma *dma8, IsaDma *dma16)
+{
+    assert(bus && dma8 && dma16);
+    assert(!bus->dma[0] && !bus->dma[1]);
+    bus->dma[0] = dma8;
+    bus->dma[1] = dma16;
+}
+
+IsaDma *isa_get_dma(ISABus *bus, int nchan)
+{
+    assert(bus);
+    return bus->dma[nchan > 3 ? 1 : 0];
+}
+
 static inline void isa_init_ioport(ISADevice *dev, uint16_t ioport)
 {
     if (dev && (dev->ioport_id == 0 || ioport < dev->ioport_id)) {
@@ -136,10 +163,6 @@
 {
     DeviceState *dev;
 
-    if (!bus) {
-        hw_error("Tried to create isa device %s with no isa bus present.",
-                 name);
-    }
     dev = qdev_create(BUS(bus), name);
     return ISA_DEVICE(dev);
 }
@@ -148,10 +171,6 @@
 {
     DeviceState *dev;
 
-    if (!bus) {
-        hw_error("Tried to create isa device %s with no isa bus present.",
-                 name);
-    }
     dev = qdev_try_create(BUS(bus), name);
     return ISA_DEVICE(dev);
 }
@@ -178,6 +197,9 @@
     case VGA_VMWARE:
         fprintf(stderr, "%s: vmware_vga: no PCI bus\n", __func__);
         return NULL;
+    case VGA_VIRTIO:
+        fprintf(stderr, "%s: virtio-vga: no PCI bus\n", __func__);
+        return NULL;
     case VGA_NONE:
     default:
         return NULL;
@@ -229,6 +251,7 @@
 
 static void isabus_register_types(void)
 {
+    type_register_static(&isa_dma_info);
     type_register_static(&isa_bus_info);
     type_register_static(&isabus_bridge_info);
     type_register_static(&isa_device_type_info);
@@ -250,7 +273,11 @@
 
 MemoryRegion *isa_address_space(ISADevice *dev)
 {
-    return get_system_memory();
+    if (dev) {
+        return isa_bus_from_device(dev)->address_space;
+    }
+
+    return isabus->address_space;
 }
 
 MemoryRegion *isa_address_space_io(ISADevice *dev)
@@ -263,3 +290,28 @@
 }
 
 type_init(isabus_register_types)
+
+static void parallel_init(ISABus *bus, int index, CharDriverState *chr)
+{
+    DeviceState *dev;
+    ISADevice *isadev;
+
+    isadev = isa_create(bus, "isa-parallel");
+    dev = DEVICE(isadev);
+    qdev_prop_set_uint32(dev, "index", index);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+}
+
+void parallel_hds_isa_init(ISABus *bus, int n)
+{
+    int i;
+
+    assert(n <= MAX_PARALLEL_PORTS);
+
+    for (i = 0; i < n; i++) {
+        if (parallel_hds[i]) {
+            parallel_init(bus, i, parallel_hds[i]);
+        }
+    }
+}
diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c
index 530b074..10d1ee8 100644
--- a/hw/isa/lpc_ich9.c
+++ b/hw/isa/lpc_ich9.c
@@ -27,7 +27,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "qapi/visitor.h"
 #include "qemu/range.h"
@@ -45,8 +47,7 @@
 #include "hw/pci/pci_bus.h"
 #include "exec/address-spaces.h"
 #include "sysemu/sysemu.h"
-
-static int ich9_lpc_sci_irq(ICH9LPCState *lpc);
+#include "qom/cpu.h"
 
 /*****************************************************************************/
 /* ICH9 LPC PCI to ISA bridge */
@@ -94,8 +95,8 @@
 
     /*
      * D30: DMI2PCI bridge
-     * It is arbitrarily decided how INTx lines of PCI devicesbehind the bridge
-     * are connected to pirq lines. Our choice is PIRQ[E-H].
+     * It is arbitrarily decided how INTx lines of PCI devices behind
+     * the bridge are connected to pirq lines. Our choice is PIRQ[E-H].
      * INT[A-D] are connected to PIRQ[E-H]
      */
     for (pci_intx = 0; pci_intx < PCI_NUM_PINS; pci_intx++) {
@@ -138,6 +139,7 @@
     pci_set_long(c + ICH9_CC_D27IR, ICH9_CC_DIR_DEFAULT);
     pci_set_long(c + ICH9_CC_D26IR, ICH9_CC_DIR_DEFAULT);
     pci_set_long(c + ICH9_CC_D25IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_GCS, ICH9_CC_GCS_DEFAULT);
 
     ich9_cc_update(lpc);
 }
@@ -200,41 +202,28 @@
     abort();
 }
 
-/* pic_irq: i8254 irq 0-15 */
-static void ich9_lpc_update_pic(ICH9LPCState *lpc, int pic_irq)
+/* gsi: i8259+ioapic irq 0-15, otherwise assert */
+static void ich9_lpc_update_pic(ICH9LPCState *lpc, int gsi)
 {
     int i, pic_level;
 
+    assert(gsi < ICH9_LPC_PIC_NUM_PINS);
+
     /* The pic level is the logical OR of all the PCI irqs mapped to it */
     pic_level = 0;
     for (i = 0; i < ICH9_LPC_NB_PIRQS; i++) {
         int tmp_irq;
         int tmp_dis;
         ich9_lpc_pic_irq(lpc, i, &tmp_irq, &tmp_dis);
-        if (!tmp_dis && pic_irq == tmp_irq) {
+        if (!tmp_dis && tmp_irq == gsi) {
             pic_level |= pci_bus_get_irq_level(lpc->d.bus, i);
         }
     }
-    if (pic_irq == ich9_lpc_sci_irq(lpc)) {
+    if (gsi == lpc->sci_gsi) {
         pic_level |= lpc->sci_level;
     }
 
-    qemu_set_irq(lpc->pic[pic_irq], pic_level);
-}
-
-/* pirq: pirq[A-H] 0-7*/
-static void ich9_lpc_update_by_pirq(ICH9LPCState *lpc, int pirq)
-{
-    int pic_irq;
-    int pic_dis;
-
-    ich9_lpc_pic_irq(lpc, pirq, &pic_irq, &pic_dis);
-    assert(pic_irq < ICH9_LPC_PIC_NUM_PINS);
-    if (pic_dis) {
-        return;
-    }
-
-    ich9_lpc_update_pic(lpc, pic_irq);
+    qemu_set_irq(lpc->gsi[gsi], pic_level);
 }
 
 /* APIC mode: GSIx: PIRQ[A-H] -> GSI 16, ... no pirq shares same APIC pins. */
@@ -248,29 +237,32 @@
     return gsi - ICH9_LPC_PIC_NUM_PINS;
 }
 
+/* gsi: ioapic irq 16-23, otherwise assert */
 static void ich9_lpc_update_apic(ICH9LPCState *lpc, int gsi)
 {
     int level = 0;
 
-    if (gsi >= ICH9_LPC_PIC_NUM_PINS) {
-        level |= pci_bus_get_irq_level(lpc->d.bus, ich9_gsi_to_pirq(gsi));
-    }
-    if (gsi == ich9_lpc_sci_irq(lpc)) {
+    assert(gsi >= ICH9_LPC_PIC_NUM_PINS);
+
+    level |= pci_bus_get_irq_level(lpc->d.bus, ich9_gsi_to_pirq(gsi));
+    if (gsi == lpc->sci_gsi) {
         level |= lpc->sci_level;
     }
 
-    qemu_set_irq(lpc->ioapic[gsi], level);
+    qemu_set_irq(lpc->gsi[gsi], level);
 }
 
 void ich9_lpc_set_irq(void *opaque, int pirq, int level)
 {
     ICH9LPCState *lpc = opaque;
+    int pic_irq, pic_dis;
 
     assert(0 <= pirq);
     assert(pirq < ICH9_LPC_NB_PIRQS);
 
     ich9_lpc_update_apic(lpc, ich9_pirq_to_gsi(pirq));
-    ich9_lpc_update_by_pirq(lpc, pirq);
+    ich9_lpc_pic_irq(lpc, pirq, &pic_irq, &pic_dis);
+    ich9_lpc_update_pic(lpc, pic_irq);
 }
 
 /* return the pirq number (PIRQ[A-H]:0-7) corresponding to
@@ -313,6 +305,16 @@
     return route;
 }
 
+void ich9_generate_smi(void)
+{
+    cpu_interrupt(first_cpu, CPU_INTERRUPT_SMI);
+}
+
+void ich9_generate_nmi(void)
+{
+    cpu_interrupt(first_cpu, CPU_INTERRUPT_NMI);
+}
+
 static int ich9_lpc_sci_irq(ICH9LPCState *lpc)
 {
     switch (lpc->d.config[ICH9_LPC_ACPI_CTRL] &
@@ -346,25 +348,25 @@
     }
     lpc->sci_level = level;
 
-    irq = ich9_lpc_sci_irq(lpc);
+    irq = lpc->sci_gsi;
     if (irq < 0) {
         return;
     }
 
-    ich9_lpc_update_apic(lpc, irq);
-    if (irq < ICH9_LPC_PIC_NUM_PINS) {
+    if (irq >= ICH9_LPC_PIC_NUM_PINS) {
+        ich9_lpc_update_apic(lpc, irq);
+    } else {
         ich9_lpc_update_pic(lpc, irq);
     }
 }
 
-void ich9_lpc_pm_init(PCIDevice *lpc_pci)
+void ich9_lpc_pm_init(PCIDevice *lpc_pci, bool smm_enabled)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(lpc_pci);
-    qemu_irq *sci_irq;
+    qemu_irq sci_irq;
 
-    sci_irq = qemu_allocate_irqs(ich9_set_sci, lpc, 1);
-    ich9_pm_init(lpc_pci, &lpc->pm, sci_irq[0]);
-
+    sci_irq = qemu_allocate_irq(ich9_set_sci, lpc, 0);
+    ich9_pm_init(lpc_pci, &lpc->pm, smm_enabled, sci_irq);
     ich9_lpc_reset(&lpc->d.qdev);
 }
 
@@ -378,35 +380,68 @@
     acpi_pm1_cnt_update(&lpc->pm.acpi_regs,
                         val == ICH9_APM_ACPI_ENABLE,
                         val == ICH9_APM_ACPI_DISABLE);
+    if (val == ICH9_APM_ACPI_ENABLE || val == ICH9_APM_ACPI_DISABLE) {
+        return;
+    }
 
     /* SMI_EN = PMBASE + 30. SMI control and enable register */
     if (lpc->pm.smi_en & ICH9_PMIO_SMI_EN_APMC_EN) {
-        cpu_interrupt(first_cpu, CPU_INTERRUPT_SMI);
+        cpu_interrupt(current_cpu, CPU_INTERRUPT_SMI);
     }
 }
 
 /* config:PMBASE */
 static void
-ich9_lpc_pmbase_update(ICH9LPCState *lpc)
+ich9_lpc_pmbase_sci_update(ICH9LPCState *lpc)
 {
     uint32_t pm_io_base = pci_get_long(lpc->d.config + ICH9_LPC_PMBASE);
-    pm_io_base &= ICH9_LPC_PMBASE_BASE_ADDRESS_MASK;
+    uint8_t acpi_cntl = pci_get_long(lpc->d.config + ICH9_LPC_ACPI_CTRL);
+    uint8_t new_gsi;
+
+    if (acpi_cntl & ICH9_LPC_ACPI_CTRL_ACPI_EN) {
+        pm_io_base &= ICH9_LPC_PMBASE_BASE_ADDRESS_MASK;
+    } else {
+        pm_io_base = 0;
+    }
 
     ich9_pm_iospace_update(&lpc->pm, pm_io_base);
+
+    new_gsi = ich9_lpc_sci_irq(lpc);
+    if (lpc->sci_level && new_gsi != lpc->sci_gsi) {
+        qemu_set_irq(lpc->pm.irq, 0);
+        lpc->sci_gsi = new_gsi;
+        qemu_set_irq(lpc->pm.irq, 1);
+    }
+    lpc->sci_gsi = new_gsi;
 }
 
-/* config:RBCA */
-static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rbca_old)
+/* config:RCBA */
+static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rcba_old)
 {
-    uint32_t rbca = pci_get_long(lpc->d.config + ICH9_LPC_RCBA);
+    uint32_t rcba = pci_get_long(lpc->d.config + ICH9_LPC_RCBA);
 
-    if (rbca_old & ICH9_LPC_RCBA_EN) {
-            memory_region_del_subregion(get_system_memory(), &lpc->rbca_mem);
+    if (rcba_old & ICH9_LPC_RCBA_EN) {
+        memory_region_del_subregion(get_system_memory(), &lpc->rcrb_mem);
     }
-    if (rbca & ICH9_LPC_RCBA_EN) {
-            memory_region_add_subregion_overlap(get_system_memory(),
-                                                rbca & ICH9_LPC_RCBA_BA_MASK,
-                                                &lpc->rbca_mem, 1);
+    if (rcba & ICH9_LPC_RCBA_EN) {
+        memory_region_add_subregion_overlap(get_system_memory(),
+                                            rcba & ICH9_LPC_RCBA_BA_MASK,
+                                            &lpc->rcrb_mem, 1);
+    }
+}
+
+/* config:GEN_PMCON* */
+static void
+ich9_lpc_pmcon_update(ICH9LPCState *lpc)
+{
+    uint16_t gen_pmcon_1 = pci_get_word(lpc->d.config + ICH9_LPC_GEN_PMCON_1);
+    uint16_t wmask;
+
+    if (gen_pmcon_1 & ICH9_LPC_GEN_PMCON_1_SMI_LOCK) {
+        wmask = pci_get_word(lpc->d.wmask + ICH9_LPC_GEN_PMCON_1);
+        wmask &= ~ICH9_LPC_GEN_PMCON_1_SMI_LOCK;
+        pci_set_word(lpc->d.wmask + ICH9_LPC_GEN_PMCON_1, wmask);
+        lpc->pm.smi_en_wmask &= ~1;
     }
 }
 
@@ -414,8 +449,9 @@
 {
     ICH9LPCState *lpc = opaque;
 
-    ich9_lpc_pmbase_update(lpc);
-    ich9_lpc_rcba_update(lpc, 0 /* disabled ICH9_LPC_RBCA_EN */);
+    ich9_lpc_pmbase_sci_update(lpc);
+    ich9_lpc_rcba_update(lpc, 0 /* disabled ICH9_LPC_RCBA_EN */);
+    ich9_lpc_pmcon_update(lpc);
     return 0;
 }
 
@@ -423,14 +459,15 @@
                                   uint32_t addr, uint32_t val, int len)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
-    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+    uint32_t rcba_old = pci_get_long(d->config + ICH9_LPC_RCBA);
 
     pci_default_write_config(d, addr, val, len);
-    if (ranges_overlap(addr, len, ICH9_LPC_PMBASE, 4)) {
-        ich9_lpc_pmbase_update(lpc);
+    if (ranges_overlap(addr, len, ICH9_LPC_PMBASE, 4) ||
+        ranges_overlap(addr, len, ICH9_LPC_ACPI_CTRL, 1)) {
+        ich9_lpc_pmbase_sci_update(lpc);
     }
     if (ranges_overlap(addr, len, ICH9_LPC_RCBA, 4)) {
-        ich9_lpc_rcba_update(lpc, rbca_old);
+        ich9_lpc_rcba_update(lpc, rcba_old);
     }
     if (ranges_overlap(addr, len, ICH9_LPC_PIRQA_ROUT, 4)) {
         pci_bus_fire_intx_routing_notifier(lpc->d.bus);
@@ -438,13 +475,16 @@
     if (ranges_overlap(addr, len, ICH9_LPC_PIRQE_ROUT, 4)) {
         pci_bus_fire_intx_routing_notifier(lpc->d.bus);
     }
+    if (ranges_overlap(addr, len, ICH9_LPC_GEN_PMCON_1, 8)) {
+        ich9_lpc_pmcon_update(lpc);
+    }
 }
 
 static void ich9_lpc_reset(DeviceState *qdev)
 {
     PCIDevice *d = PCI_DEVICE(qdev);
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
-    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+    uint32_t rcba_old = pci_get_long(d->config + ICH9_LPC_RCBA);
     int i;
 
     for (i = 0; i < 4; i++) {
@@ -462,14 +502,15 @@
 
     ich9_cc_reset(lpc);
 
-    ich9_lpc_pmbase_update(lpc);
-    ich9_lpc_rcba_update(lpc, rbca_old);
+    ich9_lpc_pmbase_sci_update(lpc);
+    ich9_lpc_rcba_update(lpc, rcba_old);
 
     lpc->sci_level = 0;
     lpc->rst_cnt = 0;
 }
 
-static const MemoryRegionOps rbca_mmio_ops = {
+/* root complex register block is mapped into memory space */
+static const MemoryRegionOps rcrb_mmio_ops = {
     .read = ich9_cc_read,
     .write = ich9_cc_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
@@ -494,7 +535,7 @@
         /* lpt */
         pci_conf[0x82] |= 0x04;
     }
-    if (memory_region_present(io_as, 0x3f0)) {
+    if (memory_region_present(io_as, 0x3f2)) {
         /* floppy */
         pci_conf[0x82] |= 0x08;
     }
@@ -537,14 +578,13 @@
     return o;
 }
 
-static void ich9_lpc_get_sci_int(Object *obj, Visitor *v,
-                                 void *opaque, const char *name,
-                                 Error **errp)
+static void ich9_lpc_get_sci_int(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(obj);
-    uint32_t value = ich9_lpc_sci_irq(lpc);
+    uint32_t value = lpc->sci_gsi;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void ich9_lpc_add_properties(ICH9LPCState *lpc)
@@ -570,18 +610,26 @@
     ich9_lpc_add_properties(lpc);
 }
 
-static int ich9_lpc_init(PCIDevice *d)
+static void ich9_lpc_realize(PCIDevice *d, Error **errp)
 {
     ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    DeviceState *dev = DEVICE(d);
     ISABus *isa_bus;
 
-    isa_bus = isa_bus_new(&d->qdev, get_system_io());
+    isa_bus = isa_bus_new(DEVICE(d), get_system_memory(), get_system_io(),
+                          errp);
+    if (!isa_bus) {
+        return;
+    }
 
     pci_set_long(d->wmask + ICH9_LPC_PMBASE,
                  ICH9_LPC_PMBASE_BASE_ADDRESS_MASK);
+    pci_set_byte(d->wmask + ICH9_LPC_PMBASE,
+                 ICH9_LPC_ACPI_CTRL_ACPI_EN |
+                 ICH9_LPC_ACPI_CTRL_SCI_IRQ_SEL_MASK);
 
-    memory_region_init_io(&lpc->rbca_mem, OBJECT(d), &rbca_mmio_ops, lpc,
-                            "lpc-rbca-mmio", ICH9_CC_SIZE);
+    memory_region_init_io(&lpc->rcrb_mem, OBJECT(d), &rcrb_mmio_ops, lpc,
+                          "lpc-rcrb-mmio", ICH9_CC_SIZE);
 
     lpc->isa_bus = isa_bus;
 
@@ -596,22 +644,10 @@
     memory_region_add_subregion_overlap(pci_address_space_io(d),
                                         ICH9_RST_CNT_IOPORT, &lpc->rst_cnt_mem,
                                         1);
-    return 0;
-}
 
-static void ich9_device_plug_cb(HotplugHandler *hotplug_dev,
-                                DeviceState *dev, Error **errp)
-{
-    ICH9LPCState *lpc = ICH9_LPC_DEVICE(hotplug_dev);
+    qdev_init_gpio_out_named(dev, lpc->gsi, ICH9_GPIO_GSI, GSI_NUM_PINS);
 
-    ich9_pm_device_plug_cb(&lpc->pm, dev, errp);
-}
-
-static void ich9_device_unplug_request_cb(HotplugHandler *hotplug_dev,
-                                          DeviceState *dev, Error **errp)
-{
-    error_setg(errp, "acpi: device unplug request for not supported device"
-               " type: %s", object_get_typename(OBJECT(dev)));
+    isa_bus_irqs(isa_bus, lpc->gsi);
 }
 
 static bool ich9_rst_cnt_needed(void *opaque)
@@ -625,6 +661,7 @@
     .name = "ICH9LPC/rst_cnt",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = ich9_rst_cnt_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(rst_cnt, ICH9LPCState),
         VMSTATE_END_OF_LIST()
@@ -644,15 +681,24 @@
         VMSTATE_UINT32(sci_level, ICH9LPCState),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_ich9_rst_cnt,
-            .needed = ich9_rst_cnt_needed
-        },
-        { 0 }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_ich9_rst_cnt,
+        NULL
     }
 };
 
+static Property ich9_lpc_properties[] = {
+    DEFINE_PROP_BOOL("noreboot", ICH9LPCState, pin_strap.spkr_hi, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ich9_send_gpe(AcpiDeviceIf *adev, AcpiEventStatusBits ev)
+{
+    ICH9LPCState *s = ICH9_LPC_DEVICE(adev);
+
+    acpi_send_gpe_event(&s->pm.acpi_regs, s->pm.irq, ev);
+}
+
 static void ich9_lpc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -662,8 +708,9 @@
 
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
     dc->reset = ich9_lpc_reset;
-    k->init = ich9_lpc_init;
+    k->realize = ich9_lpc_realize;
     dc->vmsd = &vmstate_ich9_lpc;
+    dc->props = ich9_lpc_properties;
     k->config_write = ich9_lpc_config_write;
     dc->desc = "ICH9 LPC bridge";
     k->vendor_id = PCI_VENDOR_ID_INTEL;
@@ -675,9 +722,12 @@
      * pc_q35_init()
      */
     dc->cannot_instantiate_with_device_add_yet = true;
-    hc->plug = ich9_device_plug_cb;
-    hc->unplug_request = ich9_device_unplug_request_cb;
+    hc->plug = ich9_pm_device_plug_cb;
+    hc->unplug_request = ich9_pm_device_unplug_request_cb;
+    hc->unplug = ich9_pm_device_unplug_cb;
     adevc->ospm_status = ich9_pm_ospm_status;
+    adevc->send_event = ich9_send_gpe;
+    adevc->madt_cpu = pc_madt_cpu_entry;
 }
 
 static const TypeInfo ich9_lpc_info = {
diff --git a/hw/isa/pc87312.c b/hw/isa/pc87312.c
index 40a1106..c3ebf3e 100644
--- a/hw/isa/pc87312.c
+++ b/hw/isa/pc87312.c
@@ -23,7 +23,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/isa/pc87312.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
@@ -278,6 +280,7 @@
     pc87312_hard_reset(s);
 
     if (is_parallel_enabled(s)) {
+        /* FIXME use a qdev chardev prop instead of parallel_hds[] */
         chr = parallel_hds[0];
         if (chr == NULL) {
             chr = qemu_chr_new("par0", "null", NULL);
@@ -296,6 +299,7 @@
 
     for (i = 0; i < 2; i++) {
         if (is_uart_enabled(s, i)) {
+            /* FIXME use a qdev chardev prop instead of serial_hds[] */
             chr = serial_hds[i];
             if (chr == NULL) {
                 snprintf(name, sizeof(name), "ser%d", i);
@@ -319,15 +323,17 @@
         d = DEVICE(isa);
         qdev_prop_set_uint32(d, "iobase", get_fdc_iobase(s));
         qdev_prop_set_uint32(d, "irq", 6);
+        /* FIXME use a qdev drive property instead of drive_get() */
         drive = drive_get(IF_FLOPPY, 0, 0);
         if (drive != NULL) {
-            qdev_prop_set_drive_nofail(d, "driveA",
-                                       blk_by_legacy_dinfo(drive));
+            qdev_prop_set_drive(d, "driveA", blk_by_legacy_dinfo(drive),
+                                &error_fatal);
         }
+        /* FIXME use a qdev drive property instead of drive_get() */
         drive = drive_get(IF_FLOPPY, 0, 1);
         if (drive != NULL) {
-            qdev_prop_set_drive_nofail(d, "driveB",
-                                       blk_by_legacy_dinfo(drive));
+            qdev_prop_set_drive(d, "driveB", blk_by_legacy_dinfo(drive),
+                                &error_fatal);
         }
         qdev_init_nofail(d);
         s->fdc.dev = isa;
diff --git a/hw/isa/piix4.c b/hw/isa/piix4.c
index 1aa17d7..5500fcc 100644
--- a/hw/isa/piix4.c
+++ b/hw/isa/piix4.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
@@ -34,6 +35,10 @@
     PCIDevice dev;
 } PIIX4State;
 
+#define TYPE_PIIX4_PCI_DEVICE "PIIX4"
+#define PIIX4_PCI_DEVICE(obj) \
+    OBJECT_CHECK(PIIX4State, (obj), TYPE_PIIX4_PCI_DEVICE)
+
 static void piix4_reset(void *opaque)
 {
     PIIX4State *d = opaque;
@@ -82,14 +87,16 @@
     }
 };
 
-static int piix4_initfn(PCIDevice *dev)
+static void piix4_realize(PCIDevice *dev, Error **errp)
 {
-    PIIX4State *d = DO_UPCAST(PIIX4State, dev, dev);
+    PIIX4State *d = PIIX4_PCI_DEVICE(dev);
 
-    isa_bus_new(&d->dev.qdev, pci_address_space_io(dev));
+    if (!isa_bus_new(DEVICE(d), pci_address_space(dev),
+                     pci_address_space_io(dev), errp)) {
+        return;
+    }
     piix4_dev = &d->dev;
     qemu_register_reset(piix4_reset, d);
-    return 0;
 }
 
 int piix4_init(PCIBus *bus, ISABus **isa_bus, int devfn)
@@ -106,7 +113,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = piix4_initfn;
+    k->realize = piix4_realize;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82371AB_0;
     k->class_id = PCI_CLASS_BRIDGE_ISA;
@@ -121,7 +128,7 @@
 }
 
 static const TypeInfo piix4_info = {
-    .name          = "PIIX4",
+    .name          = TYPE_PIIX4_PCI_DEVICE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PIIX4State),
     .class_init    = piix4_class_init,
diff --git a/hw/isa/trace-events b/hw/isa/trace-events
new file mode 100644
index 0000000..9faca41
--- /dev/null
+++ b/hw/isa/trace-events
@@ -0,0 +1,9 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/isa/pc87312.c
+pc87312_io_read(uint32_t addr, uint32_t val) "read addr=%x val=%x"
+pc87312_io_write(uint32_t addr, uint32_t val) "write addr=%x val=%x"
+pc87312_info_floppy(uint32_t base) "base 0x%x"
+pc87312_info_ide(uint32_t base) "base 0x%x"
+pc87312_info_parallel(uint32_t base, uint32_t irq) "base 0x%x, irq %u"
+pc87312_info_serial(int n, uint32_t base, uint32_t irq) "id=%d, base 0x%x, irq %u"
diff --git a/hw/isa/vt82c686.c b/hw/isa/vt82c686.c
index e0c235c..41d5254 100644
--- a/hw/isa/vt82c686.c
+++ b/hw/isa/vt82c686.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/vt82c686.h"
@@ -36,7 +37,7 @@
 
 typedef struct SuperIOConfig
 {
-    uint8_t config[0xff];
+    uint8_t config[0x100];
     uint8_t index;
     uint8_t data;
 } SuperIOConfig;
@@ -47,16 +48,20 @@
     SuperIOConfig superio_conf;
 } VT82C686BState;
 
+#define TYPE_VT82C686B_DEVICE "VT82C686B"
+#define VT82C686B_DEVICE(obj) \
+    OBJECT_CHECK(VT82C686BState, (obj), TYPE_VT82C686B_DEVICE)
+
 static void superio_ioport_writeb(void *opaque, hwaddr addr, uint64_t data,
                                   unsigned size)
 {
-    int can_write;
     SuperIOConfig *superio_conf = opaque;
 
     DPRINTF("superio_ioport_writeb  address 0x%x  val 0x%x\n", addr, data);
     if (addr == 0x3f0) {
         superio_conf->index = data & 0xff;
     } else {
+        bool can_write = true;
         /* 0x3f1 */
         switch (superio_conf->index) {
         case 0x00 ... 0xdf:
@@ -68,30 +73,27 @@
         case 0xf7:
         case 0xf9 ... 0xfb:
         case 0xfd ... 0xff:
-            can_write = 0;
+            can_write = false;
+            break;
+        case 0xe7:
+            if ((data & 0xff) != 0xfe) {
+                DPRINTF("change uart 1 base. unsupported yet\n");
+                can_write = false;
+            }
+            break;
+        case 0xe8:
+            if ((data & 0xff) != 0xbe) {
+                DPRINTF("change uart 2 base. unsupported yet\n");
+                can_write = false;
+            }
             break;
         default:
-            can_write = 1;
+            break;
 
-            if (can_write) {
-                switch (superio_conf->index) {
-                case 0xe7:
-                    if ((data & 0xff) != 0xfe) {
-                        DPRINTF("chage uart 1 base. unsupported yet\n");
-                    }
-                    break;
-                case 0xe8:
-                    if ((data & 0xff) != 0xbe) {
-                        DPRINTF("chage uart 2 base. unsupported yet\n");
-                    }
-                    break;
-
-                default:
-                    superio_conf->config[superio_conf->index] = data & 0xff;
-                }
-            }
         }
-        superio_conf->config[superio_conf->index] = data & 0xff;
+        if (can_write) {
+            superio_conf->config[superio_conf->index] = data & 0xff;
+        }
     }
 }
 
@@ -117,7 +119,7 @@
 {
     PCIDevice *d = opaque;
     uint8_t *pci_conf = d->config;
-    VT82C686BState *vt82c = DO_UPCAST(VT82C686BState, dev, d);
+    VT82C686BState *vt82c = VT82C686B_DEVICE(d);
 
     pci_set_long(pci_conf + PCI_CAPABILITY_LIST, 0x000000c0);
     pci_set_word(pci_conf + PCI_COMMAND, PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
@@ -145,7 +147,7 @@
 static void vt82c686b_write_config(PCIDevice * d, uint32_t address,
                                    uint32_t val, int len)
 {
-    VT82C686BState *vt686 = DO_UPCAST(VT82C686BState, dev, d);
+    VT82C686BState *vt686 = VT82C686B_DEVICE(d);
 
     DPRINTF("vt82c686b_write_config  address 0x%x  val 0x%x len 0x%x\n",
            address, val, len);
@@ -175,6 +177,18 @@
     PCIDevice dev;
 } VT686MC97State;
 
+#define TYPE_VT82C686B_PM_DEVICE "VT82C686B_PM"
+#define VT82C686B_PM_DEVICE(obj) \
+    OBJECT_CHECK(VT686PMState, (obj), TYPE_VT82C686B_PM_DEVICE)
+
+#define TYPE_VT82C686B_MC97_DEVICE "VT82C686B_MC97"
+#define VT82C686B_MC97_DEVICE(obj) \
+    OBJECT_CHECK(VT686MC97State, (obj), TYPE_VT82C686B_MC97_DEVICE)
+
+#define TYPE_VT82C686B_AC97_DEVICE "VT82C686B_AC97"
+#define VT82C686B_AC97_DEVICE(obj) \
+    OBJECT_CHECK(VT686AC97State, (obj), TYPE_VT82C686B_AC97_DEVICE)
+
 static void pm_update_sci(VT686PMState *s)
 {
     int sci_level, pmsts;
@@ -237,7 +251,7 @@
         VMSTATE_UINT16(ar.pm1.evt.en, VT686PMState),
         VMSTATE_UINT16(ar.pm1.cnt.cnt, VT686PMState),
         VMSTATE_STRUCT(apm, VT686PMState, 0, vmstate_apm, APMState),
-        VMSTATE_TIMER(ar.tmr.timer, VT686PMState),
+        VMSTATE_TIMER_PTR(ar.tmr.timer, VT686PMState),
         VMSTATE_INT64(ar.tmr.overflow_time, VT686PMState),
         VMSTATE_END_OF_LIST()
     }
@@ -248,9 +262,9 @@
  * just register a PCI device now, functionalities will be implemented later.
  */
 
-static int vt82c686b_ac97_initfn(PCIDevice *dev)
+static void vt82c686b_ac97_realize(PCIDevice *dev, Error **errp)
 {
-    VT686AC97State *s = DO_UPCAST(VT686AC97State, dev, dev);
+    VT686AC97State *s = VT82C686B_AC97_DEVICE(dev);
     uint8_t *pci_conf = s->dev.config;
 
     pci_set_word(pci_conf + PCI_COMMAND, PCI_COMMAND_INVALIDATE |
@@ -258,15 +272,13 @@
     pci_set_word(pci_conf + PCI_STATUS, PCI_STATUS_CAP_LIST |
                  PCI_STATUS_DEVSEL_MEDIUM);
     pci_set_long(pci_conf + PCI_INTERRUPT_PIN, 0x03);
-
-    return 0;
 }
 
 void vt82c686b_ac97_init(PCIBus *bus, int devfn)
 {
     PCIDevice *dev;
 
-    dev = pci_create(bus, devfn, "VT82C686B_AC97");
+    dev = pci_create(bus, devfn, TYPE_VT82C686B_AC97_DEVICE);
     qdev_init_nofail(&dev->qdev);
 }
 
@@ -275,7 +287,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = vt82c686b_ac97_initfn;
+    k->realize = vt82c686b_ac97_realize;
     k->vendor_id = PCI_VENDOR_ID_VIA;
     k->device_id = PCI_DEVICE_ID_VIA_AC97;
     k->revision = 0x50;
@@ -285,30 +297,28 @@
 }
 
 static const TypeInfo via_ac97_info = {
-    .name          = "VT82C686B_AC97",
+    .name          = TYPE_VT82C686B_AC97_DEVICE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686AC97State),
     .class_init    = via_ac97_class_init,
 };
 
-static int vt82c686b_mc97_initfn(PCIDevice *dev)
+static void vt82c686b_mc97_realize(PCIDevice *dev, Error **errp)
 {
-    VT686MC97State *s = DO_UPCAST(VT686MC97State, dev, dev);
+    VT686MC97State *s = VT82C686B_MC97_DEVICE(dev);
     uint8_t *pci_conf = s->dev.config;
 
     pci_set_word(pci_conf + PCI_COMMAND, PCI_COMMAND_INVALIDATE |
                  PCI_COMMAND_VGA_PALETTE);
     pci_set_word(pci_conf + PCI_STATUS, PCI_STATUS_DEVSEL_MEDIUM);
     pci_set_long(pci_conf + PCI_INTERRUPT_PIN, 0x03);
-
-    return 0;
 }
 
 void vt82c686b_mc97_init(PCIBus *bus, int devfn)
 {
     PCIDevice *dev;
 
-    dev = pci_create(bus, devfn, "VT82C686B_MC97");
+    dev = pci_create(bus, devfn, TYPE_VT82C686B_MC97_DEVICE);
     qdev_init_nofail(&dev->qdev);
 }
 
@@ -317,7 +327,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = vt82c686b_mc97_initfn;
+    k->realize = vt82c686b_mc97_realize;
     k->vendor_id = PCI_VENDOR_ID_VIA;
     k->device_id = PCI_DEVICE_ID_VIA_MC97;
     k->class_id = PCI_CLASS_COMMUNICATION_OTHER;
@@ -327,16 +337,16 @@
 }
 
 static const TypeInfo via_mc97_info = {
-    .name          = "VT82C686B_MC97",
+    .name          = TYPE_VT82C686B_MC97_DEVICE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686MC97State),
     .class_init    = via_mc97_class_init,
 };
 
 /* vt82c686 pm init */
-static int vt82c686b_pm_initfn(PCIDevice *dev)
+static void vt82c686b_pm_realize(PCIDevice *dev, Error **errp)
 {
-    VT686PMState *s = DO_UPCAST(VT686PMState, dev, dev);
+    VT686PMState *s = VT82C686B_PM_DEVICE(dev);
     uint8_t *pci_conf;
 
     pci_conf = s->dev.config;
@@ -363,9 +373,7 @@
 
     acpi_pm_tmr_init(&s->ar, pm_tmr_timer, &s->io);
     acpi_pm1_evt_init(&s->ar, pm_tmr_timer, &s->io);
-    acpi_pm1_cnt_init(&s->ar, &s->io, 2);
-
-    return 0;
+    acpi_pm1_cnt_init(&s->ar, &s->io, false, false, 2);
 }
 
 I2CBus *vt82c686b_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
@@ -374,10 +382,10 @@
     PCIDevice *dev;
     VT686PMState *s;
 
-    dev = pci_create(bus, devfn, "VT82C686B_PM");
+    dev = pci_create(bus, devfn, TYPE_VT82C686B_PM_DEVICE);
     qdev_prop_set_uint32(&dev->qdev, "smb_io_base", smb_io_base);
 
-    s = DO_UPCAST(VT686PMState, dev, dev);
+    s = VT82C686B_PM_DEVICE(dev);
 
     qdev_init_nofail(&dev->qdev);
 
@@ -394,7 +402,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = vt82c686b_pm_initfn;
+    k->realize = vt82c686b_pm_realize;
     k->config_write = pm_write_config;
     k->vendor_id = PCI_VENDOR_ID_VIA;
     k->device_id = PCI_DEVICE_ID_VIA_ACPI;
@@ -407,7 +415,7 @@
 }
 
 static const TypeInfo via_pm_info = {
-    .name          = "VT82C686B_PM",
+    .name          = TYPE_VT82C686B_PM_DEVICE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT686PMState),
     .class_init    = via_pm_class_init,
@@ -424,15 +432,19 @@
 };
 
 /* init the PCI-to-ISA bridge */
-static int vt82c686b_initfn(PCIDevice *d)
+static void vt82c686b_realize(PCIDevice *d, Error **errp)
 {
-    VT82C686BState *vt82c = DO_UPCAST(VT82C686BState, dev, d);
+    VT82C686BState *vt82c = VT82C686B_DEVICE(d);
     uint8_t *pci_conf;
     ISABus *isa_bus;
     uint8_t *wmask;
     int i;
 
-    isa_bus = isa_bus_new(&d->qdev, pci_address_space_io(d));
+    isa_bus = isa_bus_new(DEVICE(d), get_system_memory(),
+                          pci_address_space_io(d), errp);
+    if (!isa_bus) {
+        return;
+    }
 
     pci_conf = d->config;
     pci_config_set_prog_interface(pci_conf, 0x0);
@@ -453,15 +465,14 @@
                                 &vt82c->superio);
 
     qemu_register_reset(vt82c686b_reset, d);
-
-    return 0;
 }
 
 ISABus *vt82c686b_init(PCIBus *bus, int devfn)
 {
     PCIDevice *d;
 
-    d = pci_create_simple_multifunction(bus, devfn, true, "VT82C686B");
+    d = pci_create_simple_multifunction(bus, devfn, true,
+                                        TYPE_VT82C686B_DEVICE);
 
     return ISA_BUS(qdev_get_child_bus(DEVICE(d), "isa.0"));
 }
@@ -471,7 +482,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = vt82c686b_initfn;
+    k->realize = vt82c686b_realize;
     k->config_write = vt82c686b_write_config;
     k->vendor_id = PCI_VENDOR_ID_VIA;
     k->device_id = PCI_DEVICE_ID_VIA_ISA_BRIDGE;
@@ -487,7 +498,7 @@
 }
 
 static const TypeInfo via_info = {
-    .name          = "VT82C686B",
+    .name          = TYPE_VT82C686B_DEVICE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(VT82C686BState),
     .class_init    = via_class_init,
diff --git a/hw/lm32/lm32.h b/hw/lm32/lm32.h
index 18aa6fd..db9eb29 100644
--- a/hw/lm32/lm32.h
+++ b/hw/lm32/lm32.h
@@ -1,5 +1,5 @@
 #ifndef HW_LM32_H
-#define HW_LM32_H 1
+#define HW_LM32_H
 
 #include "hw/char/lm32_juart.h"
 
@@ -16,14 +16,31 @@
     return dev;
 }
 
-static inline DeviceState *lm32_juart_init(void)
+static inline DeviceState *lm32_juart_init(CharDriverState *chr)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, TYPE_LM32_JUART);
+    qdev_prop_set_chr(dev, "chardev", chr);
     qdev_init_nofail(dev);
 
     return dev;
 }
 
+static inline DeviceState *lm32_uart_create(hwaddr addr,
+                                            qemu_irq irq,
+                                            CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+
+    dev = qdev_create(NULL, "lm32-uart");
+    s = SYS_BUS_DEVICE(dev);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(s, 0, addr);
+    sysbus_connect_irq(s, 0, irq);
+    return dev;
+}
+
 #endif
diff --git a/hw/lm32/lm32_boards.c b/hw/lm32/lm32_boards.c
index af0abdb..8f0c307 100644
--- a/hw/lm32/lm32_boards.c
+++ b/hw/lm32/lm32_boards.c
@@ -17,6 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
@@ -28,6 +31,7 @@
 #include "lm32_hwsetup.h"
 #include "lm32.h"
 #include "exec/address-spaces.h"
+#include "sysemu/sysemu.h"
 
 typedef struct {
     LM32CPU *cpu;
@@ -78,7 +82,7 @@
     DriveInfo *dinfo;
     MemoryRegion *address_space_mem =  get_system_memory();
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
-    qemu_irq *cpu_irq, irq[32];
+    qemu_irq irq[32];
     ResetInfo *reset_info;
     int i;
 
@@ -111,9 +115,8 @@
 
     reset_info->flash_base = flash_base;
 
-    memory_region_init_ram(phys_ram, NULL, "lm32_evr.sdram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(phys_ram);
+    memory_region_allocate_system_memory(phys_ram, NULL, "lm32_evr.sdram",
+                                         ram_size);
     memory_region_add_subregion(address_space_mem, ram_base, phys_ram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
@@ -124,18 +127,17 @@
                           1, 2, 0x01, 0x7e, 0x43, 0x00, 0x555, 0x2aa, 1);
 
     /* create irq lines */
-    cpu_irq = qemu_allocate_irqs(cpu_irq_handler, cpu, 1);
-    env->pic_state = lm32_pic_init(*cpu_irq);
+    env->pic_state = lm32_pic_init(qemu_allocate_irq(cpu_irq_handler, cpu, 0));
     for (i = 0; i < 32; i++) {
         irq[i] = qdev_get_gpio_in(env->pic_state, i);
     }
 
-    sysbus_create_simple("lm32-uart", uart0_base, irq[uart0_irq]);
+    lm32_uart_create(uart0_base, irq[uart0_irq], serial_hds[0]);
     sysbus_create_simple("lm32-timer", timer0_base, irq[timer0_irq]);
     sysbus_create_simple("lm32-timer", timer1_base, irq[timer1_irq]);
 
     /* make sure juart isn't the first chardev */
-    env->juart_state = lm32_juart_init();
+    env->juart_state = lm32_juart_init(serial_hds[1]);
 
     reset_info->bootstrap_pc = flash_base;
 
@@ -144,7 +146,7 @@
         int kernel_size;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -174,7 +176,7 @@
     DriveInfo *dinfo;
     MemoryRegion *address_space_mem =  get_system_memory();
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
-    qemu_irq *cpu_irq, irq[32];
+    qemu_irq irq[32];
     HWSetup *hw;
     ResetInfo *reset_info;
     int i;
@@ -214,9 +216,8 @@
 
     reset_info->flash_base = flash_base;
 
-    memory_region_init_ram(phys_ram, NULL, "lm32_uclinux.sdram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(phys_ram);
+    memory_region_allocate_system_memory(phys_ram, NULL,
+                                         "lm32_uclinux.sdram", ram_size);
     memory_region_add_subregion(address_space_mem, ram_base, phys_ram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
@@ -227,19 +228,18 @@
                           1, 2, 0x01, 0x7e, 0x43, 0x00, 0x555, 0x2aa, 1);
 
     /* create irq lines */
-    cpu_irq = qemu_allocate_irqs(cpu_irq_handler, env, 1);
-    env->pic_state = lm32_pic_init(*cpu_irq);
+    env->pic_state = lm32_pic_init(qemu_allocate_irq(cpu_irq_handler, env, 0));
     for (i = 0; i < 32; i++) {
         irq[i] = qdev_get_gpio_in(env->pic_state, i);
     }
 
-    sysbus_create_simple("lm32-uart", uart0_base, irq[uart0_irq]);
+    lm32_uart_create(uart0_base, irq[uart0_irq], serial_hds[0]);
     sysbus_create_simple("lm32-timer", timer0_base, irq[timer0_irq]);
     sysbus_create_simple("lm32-timer", timer1_base, irq[timer1_irq]);
     sysbus_create_simple("lm32-timer", timer2_base, irq[timer2_irq]);
 
     /* make sure juart isn't the first chardev */
-    env->juart_state = lm32_juart_init();
+    env->juart_state = lm32_juart_init(serial_hds[1]);
 
     reset_info->bootstrap_pc = flash_base;
 
@@ -248,7 +248,7 @@
         int kernel_size;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -296,24 +296,40 @@
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static QEMUMachine lm32_evr_machine = {
-    .name = "lm32-evr",
-    .desc = "LatticeMico32 EVR32 eval system",
-    .init = lm32_evr_init,
-    .is_default = 1,
+static void lm32_evr_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "LatticeMico32 EVR32 eval system";
+    mc->init = lm32_evr_init;
+    mc->is_default = 1;
+}
+
+static const TypeInfo lm32_evr_type = {
+    .name = MACHINE_TYPE_NAME("lm32-evr"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm32_evr_class_init,
 };
 
-static QEMUMachine lm32_uclinux_machine = {
-    .name = "lm32-uclinux",
-    .desc = "lm32 platform for uClinux and u-boot by Theobroma Systems",
-    .init = lm32_uclinux_init,
-    .is_default = 0,
+static void lm32_uclinux_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lm32 platform for uClinux and u-boot by Theobroma Systems";
+    mc->init = lm32_uclinux_init;
+    mc->is_default = 0;
+}
+
+static const TypeInfo lm32_uclinux_type = {
+    .name = MACHINE_TYPE_NAME("lm32-uclinux"),
+    .parent = TYPE_MACHINE,
+    .class_init = lm32_uclinux_class_init,
 };
 
 static void lm32_machine_init(void)
 {
-    qemu_register_machine(&lm32_uclinux_machine);
-    qemu_register_machine(&lm32_evr_machine);
+    type_register_static(&lm32_evr_type);
+    type_register_static(&lm32_uclinux_type);
 }
 
-machine_init(lm32_machine_init);
+type_init(lm32_machine_init)
diff --git a/hw/lm32/lm32_hwsetup.h b/hw/lm32/lm32_hwsetup.h
index 9fd5e69..b71e6ea 100644
--- a/hw/lm32/lm32_hwsetup.h
+++ b/hw/lm32/lm32_hwsetup.h
@@ -26,6 +26,7 @@
 #define QEMU_HW_LM32_HWSETUP_H
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "hw/loader.h"
 
 typedef struct {
@@ -73,7 +74,8 @@
 static inline void hwsetup_create_rom(HWSetup *hw,
         hwaddr base)
 {
-    rom_add_blob("hwsetup", hw->data, TARGET_PAGE_SIZE, base, NULL, NULL, NULL);
+    rom_add_blob("hwsetup", hw->data, TARGET_PAGE_SIZE,
+                 TARGET_PAGE_SIZE, base, NULL, NULL, NULL);
 }
 
 static inline void hwsetup_add_u8(HWSetup *hw, uint8_t u)
diff --git a/hw/lm32/milkymist-hw.h b/hw/lm32/milkymist-hw.h
index 5317ce6..4418b44 100644
--- a/hw/lm32/milkymist-hw.h
+++ b/hw/lm32/milkymist-hw.h
@@ -1,15 +1,17 @@
-#ifndef QEMU_HW_MILKYMIST_H
-#define QEMU_HW_MILKYMIST_H
+#ifndef QEMU_HW_MILKYMIST_HW_H
+#define QEMU_HW_MILKYMIST_HW_H
 
 #include "hw/qdev.h"
 #include "net/net.h"
 
 static inline DeviceState *milkymist_uart_create(hwaddr base,
-        qemu_irq irq)
+                                                 qemu_irq irq,
+                                                 CharDriverState *chr)
 {
     DeviceState *dev;
 
     dev = qdev_create(NULL, "milkymist-uart");
+    qdev_prop_set_chr(dev, "chardev", chr);
     qdev_init_nofail(dev);
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq);
@@ -86,9 +88,10 @@
     return dev;
 }
 
-#ifdef CONFIG_GLX
+#ifdef CONFIG_OPENGL
 #include <X11/Xlib.h>
-#include <GL/glx.h>
+#include <epoxy/gl.h>
+#include <epoxy/glx.h>
 static const int glx_fbconfig_attr[] = {
     GLX_GREEN_SIZE, 5,
     GLX_GREEN_SIZE, 6,
@@ -100,17 +103,13 @@
 static inline DeviceState *milkymist_tmu2_create(hwaddr base,
         qemu_irq irq)
 {
-#ifdef CONFIG_GLX
+#ifdef CONFIG_OPENGL
     DeviceState *dev;
     Display *d;
     GLXFBConfig *configs;
     int nelements;
     int ver_major, ver_minor;
 
-    if (display_type == DT_NOGRAPHIC) {
-        return NULL;
-    }
-
     /* check that GLX will work */
     d = XOpenDisplay(NULL);
     if (d == NULL) {
@@ -204,4 +203,4 @@
     return dev;
 }
 
-#endif /* QEMU_HW_MILKYMIST_H */
+#endif /* QEMU_HW_MILKYMIST_HW_H */
diff --git a/hw/lm32/milkymist.c b/hw/lm32/milkymist.c
index 256c102..5cae0f1 100644
--- a/hw/lm32/milkymist.c
+++ b/hw/lm32/milkymist.c
@@ -17,6 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
@@ -30,6 +33,7 @@
 #include "milkymist-hw.h"
 #include "lm32.h"
 #include "exec/address-spaces.h"
+#include "qemu/cutils.h"
 
 #define BIOS_FILENAME    "mmone-bios.bin"
 #define BIOS_OFFSET      0x00860000
@@ -86,7 +90,7 @@
     DriveInfo *dinfo;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *phys_sdram = g_new(MemoryRegion, 1);
-    qemu_irq irq[32], *cpu_irq;
+    qemu_irq irq[32];
     int i;
     char *bios_filename;
     ResetInfo *reset_info;
@@ -118,9 +122,8 @@
 
     cpu_lm32_set_phys_msb_ignore(env, 1);
 
-    memory_region_init_ram(phys_sdram, NULL, "milkymist.sdram", sdram_size,
-                           &error_abort);
-    vmstate_register_ram_global(phys_sdram);
+    memory_region_allocate_system_memory(phys_sdram, NULL, "milkymist.sdram",
+                                         sdram_size);
     memory_region_add_subregion(address_space_mem, sdram_base, phys_sdram);
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
@@ -131,8 +134,7 @@
                           2, 0x00, 0x89, 0x00, 0x1d, 1);
 
     /* create irq lines */
-    cpu_irq = qemu_allocate_irqs(cpu_irq_handler, cpu, 1);
-    env->pic_state = lm32_pic_init(*cpu_irq);
+    env->pic_state = lm32_pic_init(qemu_allocate_irq(cpu_irq_handler, cpu, 0));
     for (i = 0; i < 32; i++) {
         irq[i] = qdev_get_gpio_in(env->pic_state, i);
     }
@@ -155,8 +157,9 @@
                 bios_name);
         exit(1);
     }
+    g_free(bios_filename);
 
-    milkymist_uart_create(0x60000000, irq[0]);
+    milkymist_uart_create(0x60000000, irq[0], serial_hds[0]);
     milkymist_sysctl_create(0x60001000, irq[1], irq[2], irq[3],
             80000000, 0x10014d31, 0x0000041f, 0x00000001);
     milkymist_hpdmc_create(0x60002000);
@@ -164,20 +167,22 @@
     milkymist_memcard_create(0x60004000);
     milkymist_ac97_create(0x60005000, irq[4], irq[5], irq[6], irq[7]);
     milkymist_pfpu_create(0x60006000, irq[8]);
-    milkymist_tmu2_create(0x60007000, irq[9]);
+    if (machine->enable_graphics) {
+        milkymist_tmu2_create(0x60007000, irq[9]);
+    }
     milkymist_minimac2_create(0x60008000, 0x30000000, irq[10], irq[11]);
     milkymist_softusb_create(0x6000f000, irq[15],
             0x20000000, 0x1000, 0x20020000, 0x2000);
 
     /* make sure juart isn't the first chardev */
-    env->juart_state = lm32_juart_init();
+    env->juart_state = lm32_juart_init(serial_hds[1]);
 
     if (kernel_filename) {
         uint64_t entry;
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1, ELF_MACHINE, 0);
+                               1, EM_LATTICEMICO32, 0, 0);
         reset_info->bootstrap_pc = entry;
 
         if (kernel_size < 0) {
@@ -210,16 +215,11 @@
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static QEMUMachine milkymist_machine = {
-    .name = "milkymist",
-    .desc = "Milkymist One",
-    .init = milkymist_init,
-    .is_default = 0,
-};
-
-static void milkymist_machine_init(void)
+static void milkymist_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&milkymist_machine);
+    mc->desc = "Milkymist One";
+    mc->init = milkymist_init;
+    mc->is_default = 0;
 }
 
-machine_init(milkymist_machine_init);
+DEFINE_MACHINE("milkymist", milkymist_machine_init)
diff --git a/hw/m68k/an5206.c b/hw/m68k/an5206.c
index f1f1350..142bab9 100644
--- a/hw/m68k/an5206.c
+++ b/hw/m68k/an5206.c
@@ -6,12 +6,17 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "exec/address-spaces.h"
+#include "qemu/error-report.h"
 #include "sysemu/qtest.h"
 
 #define KERNEL_LOAD_ADDR 0x10000
@@ -39,7 +44,8 @@
     }
     cpu = cpu_m68k_init(cpu_model);
     if (!cpu) {
-        hw_error("Unable to find m68k CPU definition\n");
+        error_report("Unable to find m68k CPU definition");
+        exit(1);
     }
     env = &cpu->env;
 
@@ -50,12 +56,11 @@
     env->rambar0 = AN5206_RAMBAR_ADDR | 1;
 
     /* DRAM at address zero */
-    memory_region_init_ram(ram, NULL, "an5206.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "an5206.ram", ram_size);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_abort);
+    memory_region_init_ram(sram, NULL, "an5206.sram", 512, &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, AN5206_RAMBAR_ADDR, sram);
 
@@ -71,7 +76,7 @@
     }
 
     kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, 1, ELF_MACHINE, 0);
+                           NULL, NULL, 1, EM_68K, 0, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -90,15 +95,10 @@
     env->pc = entry;
 }
 
-static QEMUMachine an5206_machine = {
-    .name = "an5206",
-    .desc = "Arnewsh 5206",
-    .init = an5206_init,
-};
-
-static void an5206_machine_init(void)
+static void an5206_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&an5206_machine);
+    mc->desc = "Arnewsh 5206";
+    mc->init = an5206_init;
 }
 
-machine_init(an5206_machine_init);
+DEFINE_MACHINE("an5206", an5206_machine_init)
diff --git a/hw/m68k/dummy_m68k.c b/hw/m68k/dummy_m68k.c
index facd561..0b11d20 100644
--- a/hw/m68k/dummy_m68k.c
+++ b/hw/m68k/dummy_m68k.c
@@ -6,6 +6,9 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -21,6 +24,7 @@
     ram_addr_t ram_size = machine->ram_size;
     const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
+    M68kCPU *cpu;
     CPUM68KState *env;
     MemoryRegion *address_space_mem =  get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -30,24 +34,25 @@
 
     if (!cpu_model)
         cpu_model = "cfv4e";
-    env = cpu_init(cpu_model);
-    if (!env) {
+    cpu = cpu_m68k_init(cpu_model);
+    if (!cpu) {
         fprintf(stderr, "Unable to find m68k CPU definition\n");
         exit(1);
     }
+    env = &cpu->env;
 
     /* Initialize CPU registers.  */
     env->vbr = 0;
 
     /* RAM at address zero */
-    memory_region_init_ram(ram, NULL, "dummy_m68k.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "dummy_m68k.ram",
+                                         ram_size);
     memory_region_add_subregion(address_space_mem, 0, ram);
 
     /* Load kernel.  */
     if (kernel_filename) {
         kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               NULL, NULL, 1, ELF_MACHINE, 0);
+                               NULL, NULL, 1, EM_68K, 0, 0);
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -70,15 +75,10 @@
     env->pc = entry;
 }
 
-static QEMUMachine dummy_m68k_machine = {
-    .name = "dummy",
-    .desc = "Dummy board",
-    .init = dummy_m68k_init,
-};
-
-static void dummy_m68k_machine_init(void)
+static void dummy_m68k_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&dummy_m68k_machine);
+    mc->desc = "Dummy board";
+    mc->init = dummy_m68k_init;
 }
 
-machine_init(dummy_m68k_machine_init);
+DEFINE_MACHINE("dummy", dummy_m68k_machine_init)
diff --git a/hw/m68k/mcf5206.c b/hw/m68k/mcf5206.c
index 1727a46..e14896e 100644
--- a/hw/m68k/mcf5206.c
+++ b/hw/m68k/mcf5206.c
@@ -5,6 +5,9 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "qemu/timer.h"
diff --git a/hw/m68k/mcf5208.c b/hw/m68k/mcf5208.c
index a01a445..2415557 100644
--- a/hw/m68k/mcf5208.c
+++ b/hw/m68k/mcf5208.c
@@ -5,6 +5,10 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "qemu/timer.h"
@@ -218,12 +222,11 @@
     /* TODO: Configure BARs.  */
 
     /* DRAM at 0x40000000 */
-    memory_region_init_ram(ram, NULL, "mcf5208.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "mcf5208.ram", ram_size);
     memory_region_add_subregion(address_space_mem, 0x40000000, ram);
 
     /* Internal SRAM.  */
-    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_abort);
+    memory_region_init_ram(sram, NULL, "mcf5208.sram", 16384, &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(address_space_mem, 0x80000000, sram);
 
@@ -276,7 +279,7 @@
     }
 
     kernel_size = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                           NULL, NULL, 1, ELF_MACHINE, 0);
+                           NULL, NULL, 1, EM_68K, 0, 0);
     entry = elf_entry;
     if (kernel_size < 0) {
         kernel_size = load_uimage(kernel_filename, &entry, NULL, NULL,
@@ -295,16 +298,11 @@
     env->pc = entry;
 }
 
-static QEMUMachine mcf5208evb_machine = {
-    .name = "mcf5208evb",
-    .desc = "MCF5206EVB",
-    .init = mcf5208evb_init,
-    .is_default = 1,
-};
-
-static void mcf5208evb_machine_init(void)
+static void mcf5208evb_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mcf5208evb_machine);
+    mc->desc = "MCF5206EVB";
+    mc->init = mcf5208evb_init;
+    mc->is_default = 1;
 }
 
-machine_init(mcf5208evb_machine_init);
+DEFINE_MACHINE("mcf5208evb", mcf5208evb_machine_init)
diff --git a/hw/m68k/mcf_intc.c b/hw/m68k/mcf_intc.c
index 621423c..cf58132 100644
--- a/hw/m68k/mcf_intc.c
+++ b/hw/m68k/mcf_intc.c
@@ -5,6 +5,9 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/m68k/mcf.h"
 #include "exec/address-spaces.h"
@@ -102,6 +105,20 @@
     case 0x0c:
         s->imr = (s->imr & 0xffffffff00000000ull) | (uint32_t)val;
         break;
+    case 0x1c:
+        if (val & 0x40) {
+            s->imr = ~0ull;
+        } else {
+            s->imr |= (0x1ull << (val & 0x3f));
+        }
+        break;
+    case 0x1d:
+        if (val & 0x40) {
+            s->imr = 0ull;
+        } else {
+            s->imr &= ~(0x1ull << (val & 0x3f));
+        }
+        break;
     default:
         hw_error("mcf_intc_write: Bad write offset %d\n", offset);
         break;
diff --git a/hw/mem/Makefile.objs b/hw/mem/Makefile.objs
index b000fb4..f12f8b9 100644
--- a/hw/mem/Makefile.objs
+++ b/hw/mem/Makefile.objs
@@ -1 +1,2 @@
 common-obj-$(CONFIG_MEM_HOTPLUG) += pc-dimm.o
+common-obj-$(CONFIG_NVDIMM) += nvdimm.o
diff --git a/hw/mem/nvdimm.c b/hw/mem/nvdimm.c
new file mode 100644
index 0000000..7895805
--- /dev/null
+++ b/hw/mem/nvdimm.c
@@ -0,0 +1,180 @@
+/*
+ * Non-Volatile Dual In-line Memory Module Virtualization Implementation
+ *
+ * Copyright(C) 2015 Intel Corporation.
+ *
+ * Author:
+ *  Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * Currently, it only supports PMEM Virtualization.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+#include "hw/mem/nvdimm.h"
+
+static void nvdimm_get_label_size(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    NVDIMMDevice *nvdimm = NVDIMM(obj);
+    uint64_t value = nvdimm->label_size;
+
+    visit_type_size(v, name, &value, errp);
+}
+
+static void nvdimm_set_label_size(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    NVDIMMDevice *nvdimm = NVDIMM(obj);
+    Error *local_err = NULL;
+    uint64_t value;
+
+    if (memory_region_size(&nvdimm->nvdimm_mr)) {
+        error_setg(&local_err, "cannot change property value");
+        goto out;
+    }
+
+    visit_type_size(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (value < MIN_NAMESPACE_LABEL_SIZE) {
+        error_setg(&local_err, "Property '%s.%s' (0x%" PRIx64 ") is required"
+                   " at least 0x%lx", object_get_typename(obj),
+                   name, value, MIN_NAMESPACE_LABEL_SIZE);
+        goto out;
+    }
+
+    nvdimm->label_size = value;
+out:
+    error_propagate(errp, local_err);
+}
+
+static void nvdimm_init(Object *obj)
+{
+    object_property_add(obj, "label-size", "int",
+                        nvdimm_get_label_size, nvdimm_set_label_size, NULL,
+                        NULL, NULL);
+}
+
+static MemoryRegion *nvdimm_get_memory_region(PCDIMMDevice *dimm)
+{
+    NVDIMMDevice *nvdimm = NVDIMM(dimm);
+
+    return &nvdimm->nvdimm_mr;
+}
+
+static void nvdimm_realize(PCDIMMDevice *dimm, Error **errp)
+{
+    MemoryRegion *mr = host_memory_backend_get_memory(dimm->hostmem, errp);
+    NVDIMMDevice *nvdimm = NVDIMM(dimm);
+    uint64_t align, pmem_size, size = memory_region_size(mr);
+
+    align = memory_region_get_alignment(mr);
+
+    pmem_size = size - nvdimm->label_size;
+    nvdimm->label_data = memory_region_get_ram_ptr(mr) + pmem_size;
+    pmem_size = QEMU_ALIGN_DOWN(pmem_size, align);
+
+    if (size <= nvdimm->label_size || !pmem_size) {
+        HostMemoryBackend *hostmem = dimm->hostmem;
+        char *path = object_get_canonical_path_component(OBJECT(hostmem));
+
+        error_setg(errp, "the size of memdev %s (0x%" PRIx64 ") is too "
+                   "small to contain nvdimm label (0x%" PRIx64 ") and "
+                   "aligned PMEM (0x%" PRIx64 ")",
+                   path, memory_region_size(mr), nvdimm->label_size, align);
+        g_free(path);
+        return;
+    }
+
+    memory_region_init_alias(&nvdimm->nvdimm_mr, OBJECT(dimm),
+                             "nvdimm-memory", mr, 0, pmem_size);
+    nvdimm->nvdimm_mr.align = align;
+}
+
+/*
+ * the caller should check the input parameters before calling
+ * label read/write functions.
+ */
+static void nvdimm_validate_rw_label_data(NVDIMMDevice *nvdimm, uint64_t size,
+                                        uint64_t offset)
+{
+    assert((nvdimm->label_size >= size + offset) && (offset + size > offset));
+}
+
+static void nvdimm_read_label_data(NVDIMMDevice *nvdimm, void *buf,
+                                   uint64_t size, uint64_t offset)
+{
+    nvdimm_validate_rw_label_data(nvdimm, size, offset);
+
+    memcpy(buf, nvdimm->label_data + offset, size);
+}
+
+static void nvdimm_write_label_data(NVDIMMDevice *nvdimm, const void *buf,
+                                    uint64_t size, uint64_t offset)
+{
+    MemoryRegion *mr;
+    PCDIMMDevice *dimm = PC_DIMM(nvdimm);
+    uint64_t backend_offset;
+
+    nvdimm_validate_rw_label_data(nvdimm, size, offset);
+
+    memcpy(nvdimm->label_data + offset, buf, size);
+
+    mr = host_memory_backend_get_memory(dimm->hostmem, &error_abort);
+    backend_offset = memory_region_size(mr) - nvdimm->label_size + offset;
+    memory_region_set_dirty(mr, backend_offset, size);
+}
+
+static MemoryRegion *nvdimm_get_vmstate_memory_region(PCDIMMDevice *dimm)
+{
+    return host_memory_backend_get_memory(dimm->hostmem, &error_abort);
+}
+
+static void nvdimm_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    PCDIMMDeviceClass *ddc = PC_DIMM_CLASS(oc);
+    NVDIMMClass *nvc = NVDIMM_CLASS(oc);
+
+    /* nvdimm hotplug has not been supported yet. */
+    dc->hotpluggable = false;
+
+    ddc->realize = nvdimm_realize;
+    ddc->get_memory_region = nvdimm_get_memory_region;
+    ddc->get_vmstate_memory_region = nvdimm_get_vmstate_memory_region;
+
+    nvc->read_label_data = nvdimm_read_label_data;
+    nvc->write_label_data = nvdimm_write_label_data;
+}
+
+static TypeInfo nvdimm_info = {
+    .name          = TYPE_NVDIMM,
+    .parent        = TYPE_PC_DIMM,
+    .class_size    = sizeof(NVDIMMClass),
+    .class_init    = nvdimm_class_init,
+    .instance_size = sizeof(NVDIMMDevice),
+    .instance_init = nvdimm_init,
+};
+
+static void nvdimm_register_types(void)
+{
+    type_register_static(&nvdimm_info);
+}
+
+type_init(nvdimm_register_types)
diff --git a/hw/mem/pc-dimm.c b/hw/mem/pc-dimm.c
index d431834..9e8dab0 100644
--- a/hw/mem/pc-dimm.c
+++ b/hw/mem/pc-dimm.c
@@ -18,10 +18,145 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
+#include "qemu/osdep.h"
 #include "hw/mem/pc-dimm.h"
+#include "qapi/error.h"
 #include "qemu/config-file.h"
 #include "qapi/visitor.h"
 #include "qemu/range.h"
+#include "sysemu/numa.h"
+#include "sysemu/kvm.h"
+#include "trace.h"
+#include "hw/virtio/vhost.h"
+
+typedef struct pc_dimms_capacity {
+     uint64_t size;
+     Error    **errp;
+} pc_dimms_capacity;
+
+void pc_dimm_memory_plug(DeviceState *dev, MemoryHotplugState *hpms,
+                         MemoryRegion *mr, uint64_t align, Error **errp)
+{
+    int slot;
+    MachineState *machine = MACHINE(qdev_get_machine());
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+    MemoryRegion *vmstate_mr = ddc->get_vmstate_memory_region(dimm);
+    Error *local_err = NULL;
+    uint64_t existing_dimms_capacity = 0;
+    uint64_t addr;
+
+    addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    addr = pc_dimm_get_free_addr(hpms->base,
+                                 memory_region_size(&hpms->mr),
+                                 !addr ? NULL : &addr, align,
+                                 memory_region_size(mr), &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    existing_dimms_capacity = pc_existing_dimms_capacity(&local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    if (existing_dimms_capacity + memory_region_size(mr) >
+        machine->maxram_size - machine->ram_size) {
+        error_setg(&local_err, "not enough space, currently 0x%" PRIx64
+                   " in use of total hot pluggable 0x" RAM_ADDR_FMT,
+                   existing_dimms_capacity,
+                   machine->maxram_size - machine->ram_size);
+        goto out;
+    }
+
+    object_property_set_int(OBJECT(dev), addr, PC_DIMM_ADDR_PROP, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    trace_mhp_pc_dimm_assigned_address(addr);
+
+    slot = object_property_get_int(OBJECT(dev), PC_DIMM_SLOT_PROP, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    slot = pc_dimm_get_free_slot(slot == PC_DIMM_UNASSIGNED_SLOT ? NULL : &slot,
+                                 machine->ram_slots, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    object_property_set_int(OBJECT(dev), slot, PC_DIMM_SLOT_PROP, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    trace_mhp_pc_dimm_assigned_slot(slot);
+
+    if (kvm_enabled() && !kvm_has_free_slot(machine)) {
+        error_setg(&local_err, "hypervisor has no free memory slots left");
+        goto out;
+    }
+
+    if (!vhost_has_free_slot()) {
+        error_setg(&local_err, "a used vhost backend has no free"
+                               " memory slots left");
+        goto out;
+    }
+
+    memory_region_add_subregion(&hpms->mr, addr - hpms->base, mr);
+    vmstate_register_ram(vmstate_mr, dev);
+    numa_set_mem_node_id(addr, memory_region_size(mr), dimm->node);
+
+out:
+    error_propagate(errp, local_err);
+}
+
+void pc_dimm_memory_unplug(DeviceState *dev, MemoryHotplugState *hpms,
+                           MemoryRegion *mr)
+{
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+    MemoryRegion *vmstate_mr = ddc->get_vmstate_memory_region(dimm);
+
+    numa_unset_mem_node_id(dimm->addr, memory_region_size(mr), dimm->node);
+    memory_region_del_subregion(&hpms->mr, mr);
+    vmstate_unregister_ram(vmstate_mr, dev);
+}
+
+static int pc_existing_dimms_capacity_internal(Object *obj, void *opaque)
+{
+    pc_dimms_capacity *cap = opaque;
+    uint64_t *size = &cap->size;
+
+    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
+        DeviceState *dev = DEVICE(obj);
+
+        if (dev->realized) {
+            (*size) += object_property_get_int(obj, PC_DIMM_SIZE_PROP,
+                cap->errp);
+        }
+
+        if (cap->errp && *cap->errp) {
+            return 1;
+        }
+    }
+    object_child_foreach(obj, pc_existing_dimms_capacity_internal, opaque);
+    return 0;
+}
+
+uint64_t pc_existing_dimms_capacity(Error **errp)
+{
+    pc_dimms_capacity cap;
+
+    cap.size = 0;
+    cap.errp = errp;
+
+    pc_existing_dimms_capacity_internal(qdev_get_machine(), &cap);
+    return cap.size;
+}
 
 int qmp_pc_dimm_device_list(Object *obj, void *opaque)
 {
@@ -50,7 +185,7 @@
                                                NULL);
             di->memdev = object_get_canonical_path(OBJECT(dimm->hostmem));
 
-            info->dimm = di;
+            info->u.dimm.data = di;
             elem->value = info;
             elem->next = NULL;
             **prev = elem;
@@ -147,7 +282,6 @@
     uint64_t address_space_end = address_space_start + address_space_size;
 
     g_assert(QEMU_ALIGN_UP(address_space_start, align) == address_space_start);
-    g_assert(QEMU_ALIGN_UP(address_space_size, align) == address_space_size);
 
     if (!address_space_size) {
         error_setg(errp, "memory hotplug is not enabled, "
@@ -218,32 +352,34 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void pc_dimm_get_size(Object *obj, Visitor *v, void *opaque,
-                          const char *name, Error **errp)
+static void pc_dimm_get_size(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     int64_t value;
     MemoryRegion *mr;
     PCDIMMDevice *dimm = PC_DIMM(obj);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(obj);
 
-    mr = host_memory_backend_get_memory(dimm->hostmem, errp);
+    mr = ddc->get_memory_region(dimm);
     value = memory_region_size(mr);
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void pc_dimm_check_memdev_is_busy(Object *obj, const char *name,
                                       Object *val, Error **errp)
 {
-    MemoryRegion *mr;
+    Error *local_err = NULL;
 
-    mr = host_memory_backend_get_memory(MEMORY_BACKEND(val), errp);
-    if (memory_region_is_mapped(mr)) {
+    if (host_memory_backend_is_mapped(MEMORY_BACKEND(val))) {
         char *path = object_get_canonical_path_component(val);
-        error_setg(errp, "can't use already busy memdev: %s", path);
+        error_setg(&local_err, "can't use already busy memdev: %s", path);
         g_free(path);
     } else {
-        qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
+        qdev_prop_allow_set_link_before_realize(obj, name, val, &local_err);
     }
+
+    error_propagate(errp, local_err);
 }
 
 static void pc_dimm_init(Object *obj)
@@ -262,17 +398,32 @@
 static void pc_dimm_realize(DeviceState *dev, Error **errp)
 {
     PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
 
     if (!dimm->hostmem) {
         error_setg(errp, "'" PC_DIMM_MEMDEV_PROP "' property is not set");
         return;
     }
-    if ((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) {
+    if (((nb_numa_nodes > 0) && (dimm->node >= nb_numa_nodes)) ||
+        (!nb_numa_nodes && dimm->node)) {
         error_setg(errp, "'DIMM property " PC_DIMM_NODE_PROP " has value %"
                    PRIu32 "' which exceeds the number of numa nodes: %d",
-                   dimm->node, nb_numa_nodes);
+                   dimm->node, nb_numa_nodes ? nb_numa_nodes : 1);
         return;
     }
+
+    if (ddc->realize) {
+        ddc->realize(dimm, errp);
+    }
+
+    host_memory_backend_set_mapped(dimm->hostmem, true);
+}
+
+static void pc_dimm_unrealize(DeviceState *dev, Error **errp)
+{
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+
+    host_memory_backend_set_mapped(dimm->hostmem, false);
 }
 
 static MemoryRegion *pc_dimm_get_memory_region(PCDIMMDevice *dimm)
@@ -280,15 +431,23 @@
     return host_memory_backend_get_memory(dimm->hostmem, &error_abort);
 }
 
+static MemoryRegion *pc_dimm_get_vmstate_memory_region(PCDIMMDevice *dimm)
+{
+    return host_memory_backend_get_memory(dimm->hostmem, &error_abort);
+}
+
 static void pc_dimm_class_init(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
     PCDIMMDeviceClass *ddc = PC_DIMM_CLASS(oc);
 
     dc->realize = pc_dimm_realize;
+    dc->unrealize = pc_dimm_unrealize;
     dc->props = pc_dimm_properties;
+    dc->desc = "DIMM memory module";
 
     ddc->get_memory_region = pc_dimm_get_memory_region;
+    ddc->get_vmstate_memory_region = pc_dimm_get_vmstate_memory_region;
 }
 
 static TypeInfo pc_dimm_info = {
diff --git a/hw/microblaze/boot.c b/hw/microblaze/boot.c
index a2843cd..9eebb1a 100644
--- a/hw/microblaze/boot.c
+++ b/hw/microblaze/boot.c
@@ -24,6 +24,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
@@ -32,6 +35,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/cutils.h"
 
 #include "boot.h"
 
@@ -48,13 +52,14 @@
 static void main_cpu_reset(void *opaque)
 {
     MicroBlazeCPU *cpu = opaque;
+    CPUState *cs = CPU(cpu);
     CPUMBState *env = &cpu->env;
 
-    cpu_reset(CPU(cpu));
+    cpu_reset(cs);
     env->regs[5] = boot_info.cmdline;
     env->regs[6] = boot_info.initrd_start;
     env->regs[7] = boot_info.fdt;
-    env->sregs[SR_PC] = boot_info.bootstrap_pc;
+    cpu_set_pc(cs, boot_info.bootstrap_pc);
     if (boot_info.machine_cpu_reset) {
         boot_info.machine_cpu_reset(cpu);
     }
@@ -113,15 +118,15 @@
     const char *kernel_filename;
     const char *kernel_cmdline;
     const char *dtb_arg;
+    char *filename = NULL;
 
     machine_opts = qemu_get_machine_opts();
     kernel_filename = qemu_opt_get(machine_opts, "kernel");
     kernel_cmdline = qemu_opt_get(machine_opts, "append");
     dtb_arg = qemu_opt_get(machine_opts, "dtb");
-    if (dtb_arg) { /* Preference a -dtb argument */
-        dtb_filename = dtb_arg;
-    } else { /* default to pcbios dtb as passed by machine_init */
-        dtb_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, dtb_filename);
+    /* default to pcbios dtb as passed by machine_init */
+    if (!dtb_arg) {
+        filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, dtb_filename);
     }
 
     boot_info.machine_cpu_reset = machine_cpu_reset;
@@ -140,12 +145,12 @@
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
                                &entry, &low, &high,
-                               big_endian, ELF_MACHINE, 0);
+                               big_endian, EM_MICROBLAZE, 0, 0);
         base32 = entry;
         if (base32 == 0xc0000000) {
             kernel_size = load_elf(kernel_filename, translate_kernel_address,
                                    NULL, &entry, NULL, NULL,
-                                   big_endian, ELF_MACHINE, 0);
+                                   big_endian, EM_MICROBLAZE, 0, 0);
         }
         /* Always boot into physical ram.  */
         boot_info.bootstrap_pc = (uint32_t)entry;
@@ -185,7 +190,7 @@
                                                   ram_size - initrd_offset);
             }
             if (initrd_size < 0) {
-                error_report("qemu: could not load initrd '%s'\n",
+                error_report("qemu: could not load initrd '%s'",
                              initrd_filename);
                 exit(EXIT_FAILURE);
             }
@@ -203,7 +208,8 @@
                             boot_info.initrd_start,
                             boot_info.initrd_end,
                             kernel_cmdline,
-                            dtb_filename);
+                            /* Preference a -dtb argument */
+                            dtb_arg ? dtb_arg : filename);
     }
-
+    g_free(filename);
 }
diff --git a/hw/microblaze/boot.h b/hw/microblaze/boot.h
index 0eb7f8e..dd1090d 100644
--- a/hw/microblaze/boot.h
+++ b/hw/microblaze/boot.h
@@ -1,5 +1,5 @@
-#ifndef __MICROBLAZE_BOOT__
-#define __MICROBLAZE_BOOT__
+#ifndef MICROBLAZE_BOOT_H
+#define MICROBLAZE_BOOT_H
 
 #include "hw/hw.h"
 
@@ -9,4 +9,4 @@
                             const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *));
 
-#endif /* __MICROBLAZE_BOOT __ */
+#endif /* MICROBLAZE_BOOT_H */
diff --git a/hw/microblaze/petalogix_ml605_mmu.c b/hw/microblaze/petalogix_ml605_mmu.c
index 48c264b..4968bdb 100644
--- a/hw/microblaze/petalogix_ml605_mmu.c
+++ b/hw/microblaze/petalogix_ml605_mmu.c
@@ -25,6 +25,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "net/net.h"
@@ -35,7 +39,7 @@
 #include "sysemu/block-backend.h"
 #include "hw/char/serial.h"
 #include "exec/address-spaces.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 
 #include "boot.h"
 
@@ -64,20 +68,6 @@
 #define SPI_IRQ             4
 #define UART16550_IRQ       5
 
-static void machine_cpu_reset(MicroBlazeCPU *cpu)
-{
-    CPUMBState *env = &cpu->env;
-
-    env->pvr.regs[10] = 0x0e000000; /* virtex 6 */
-    /* setup pvr to match kernel setting */
-    env->pvr.regs[5] |= PVR5_DCACHE_WRITEBACK_MASK;
-    env->pvr.regs[0] |= PVR0_USE_FPU_MASK | PVR0_ENDI;
-    env->pvr.regs[0] = (env->pvr.regs[0] & ~PVR0_VERSION_MASK) | (0x14 << 8);
-    env->pvr.regs[2] ^= PVR2_USE_FPU2_MASK;
-    env->pvr.regs[4] = 0xc56b8000;
-    env->pvr.regs[5] = 0xc56be000;
-}
-
 static void
 petalogix_ml605_init(MachineState *machine)
 {
@@ -95,16 +85,24 @@
 
     /* init CPUs */
     cpu = MICROBLAZE_CPU(object_new(TYPE_MICROBLAZE_CPU));
+    object_property_set_str(OBJECT(cpu), "8.10.a", "version", &error_abort);
+    /* Use FPU but don't use floating point conversion and square
+     * root instructions
+     */
+    object_property_set_int(OBJECT(cpu), 1, "use-fpu", &error_abort);
+    object_property_set_bool(OBJECT(cpu), true, "dcache-writeback",
+                             &error_abort);
+    object_property_set_bool(OBJECT(cpu), true, "endianness", &error_abort);
     object_property_set_bool(OBJECT(cpu), true, "realized", &error_abort);
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL, "petalogix_ml605.lmb_bram",
-                           LMB_BRAM_SIZE, &error_abort);
+                           LMB_BRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(address_space_mem, 0x00000000, phys_lmb_bram);
 
     memory_region_init_ram(phys_ram, NULL, "petalogix_ml605.ram", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(address_space_mem, MEMORY_BASEADDR, phys_ram);
 
@@ -193,31 +191,38 @@
         spi = (SSIBus *)qdev_get_child_bus(dev, "spi");
 
         for (i = 0; i < NUM_SPI_FLASHES; i++) {
+            DriveInfo *dinfo = drive_get_next(IF_MTD);
             qemu_irq cs_line;
 
-            dev = ssi_create_slave(spi, "n25q128");
+            dev = ssi_create_slave_no_init(spi, "n25q128");
+            if (dinfo) {
+                qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                                    &error_fatal);
+            }
+            qdev_init_nofail(dev);
+
             cs_line = qdev_get_gpio_in_named(dev, SSI_GPIO_CS, 0);
             sysbus_connect_irq(busdev, i+1, cs_line);
         }
     }
 
+    /* setup PVR to match kernel settings */
+    cpu->env.pvr.regs[4] = 0xc56b8000;
+    cpu->env.pvr.regs[5] = 0xc56be000;
+    cpu->env.pvr.regs[10] = 0x0e000000; /* virtex 6 */
+
     microblaze_load_kernel(cpu, MEMORY_BASEADDR, ram_size,
                            machine->initrd_filename,
                            BINARY_DEVICE_TREE_FILE,
-                           machine_cpu_reset);
+                           NULL);
 
 }
 
-static QEMUMachine petalogix_ml605_machine = {
-    .name = "petalogix-ml605",
-    .desc = "PetaLogix linux refdesign for xilinx ml605 little endian",
-    .init = petalogix_ml605_init,
-    .is_default = 0,
-};
-
-static void petalogix_ml605_machine_init(void)
+static void petalogix_ml605_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&petalogix_ml605_machine);
+    mc->desc = "PetaLogix linux refdesign for xilinx ml605 little endian";
+    mc->init = petalogix_ml605_init;
+    mc->is_default = 0;
 }
 
-machine_init(petalogix_ml605_machine_init);
+DEFINE_MACHINE("petalogix-ml605", petalogix_ml605_machine_init)
diff --git a/hw/microblaze/petalogix_s3adsp1800_mmu.c b/hw/microblaze/petalogix_s3adsp1800_mmu.c
index 84f6e74..423bcd7 100644
--- a/hw/microblaze/petalogix_s3adsp1800_mmu.c
+++ b/hw/microblaze/petalogix_s3adsp1800_mmu.c
@@ -23,6 +23,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "net/net.h"
@@ -32,6 +36,7 @@
 #include "hw/boards.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
+#include "hw/char/xilinx_uartlite.h"
 
 #include "boot.h"
 
@@ -51,18 +56,10 @@
 #define ETHLITE_IRQ         1
 #define UARTLITE_IRQ        3
 
-static void machine_cpu_reset(MicroBlazeCPU *cpu)
-{
-    CPUMBState *env = &cpu->env;
-
-    env->pvr.regs[10] = 0x0c000000; /* spartan 3a dsp family.  */
-}
-
 static void
 petalogix_s3adsp1800_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
     DeviceState *dev;
     MicroBlazeCPU *cpu;
     DriveInfo *dinfo;
@@ -73,21 +70,19 @@
     qemu_irq irq[32];
     MemoryRegion *sysmem = get_system_memory();
 
-    /* init CPUs */
-    if (cpu_model == NULL) {
-        cpu_model = "microblaze";
-    }
-    cpu = cpu_mb_init(cpu_model);
+    cpu = MICROBLAZE_CPU(object_new(TYPE_MICROBLAZE_CPU));
+    object_property_set_str(OBJECT(cpu), "7.10.d", "version", &error_abort);
+    object_property_set_bool(OBJECT(cpu), true, "realized", &error_abort);
 
     /* Attach emulated BRAM through the LMB.  */
     memory_region_init_ram(phys_lmb_bram, NULL,
                            "petalogix_s3adsp1800.lmb_bram", LMB_BRAM_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(phys_lmb_bram);
     memory_region_add_subregion(sysmem, 0x00000000, phys_lmb_bram);
 
     memory_region_init_ram(phys_ram, NULL, "petalogix_s3adsp1800.ram",
-                           ram_size, &error_abort);
+                           ram_size, &error_fatal);
     vmstate_register_ram_global(phys_ram);
     memory_region_add_subregion(sysmem, ddr_base, phys_ram);
 
@@ -109,8 +104,8 @@
         irq[i] = qdev_get_gpio_in(dev, i);
     }
 
-    sysbus_create_simple("xlnx.xps-uartlite", UARTLITE_BASEADDR,
-                         irq[UARTLITE_IRQ]);
+    xilinx_uartlite_create(UARTLITE_BASEADDR, irq[UARTLITE_IRQ],
+                           serial_hds[0]);
 
     /* 2 timers at irq 2 @ 62 Mhz.  */
     dev = qdev_create(NULL, "xlnx.xps-timer");
@@ -132,19 +127,14 @@
     microblaze_load_kernel(cpu, ddr_base, ram_size,
                            machine->initrd_filename,
                            BINARY_DEVICE_TREE_FILE,
-                           machine_cpu_reset);
+                           NULL);
 }
 
-static QEMUMachine petalogix_s3adsp1800_machine = {
-    .name = "petalogix-s3adsp1800",
-    .desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800",
-    .init = petalogix_s3adsp1800_init,
-    .is_default = 1,
-};
-
-static void petalogix_s3adsp1800_machine_init(void)
+static void petalogix_s3adsp1800_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&petalogix_s3adsp1800_machine);
+    mc->desc = "PetaLogix linux refdesign for xilinx Spartan 3ADSP1800";
+    mc->init = petalogix_s3adsp1800_init;
+    mc->is_default = 1;
 }
 
-machine_init(petalogix_s3adsp1800_machine_init);
+DEFINE_MACHINE("petalogix-s3adsp1800", petalogix_s3adsp1800_machine_init)
diff --git a/hw/mips/Makefile.objs b/hw/mips/Makefile.objs
index a9974dd..56310d3 100644
--- a/hw/mips/Makefile.objs
+++ b/hw/mips/Makefile.objs
@@ -1,5 +1,7 @@
-obj-y += mips_r4k.o mips_jazz.o mips_malta.o mips_mipssim.o
+obj-y += mips_r4k.o mips_malta.o mips_mipssim.o
 obj-y += addr.o cputimer.o mips_int.o
+obj-$(CONFIG_GOLDFISH) += mips_ranchu.o
+obj-$(CONFIG_JAZZ) += mips_jazz.o
 obj-$(CONFIG_FULONG) += mips_fulong2e.o
-obj-$(CONFIG_ANDROID) += mips_ranchu.o
 obj-y += gt64xxx_pci.o
+obj-$(CONFIG_MIPS_CPS) += cps.o
diff --git a/hw/mips/addr.c b/hw/mips/addr.c
index ff3b952..e4e86b4 100644
--- a/hw/mips/addr.c
+++ b/hw/mips/addr.c
@@ -20,6 +20,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/cpudevs.h"
 
diff --git a/hw/mips/cps.c b/hw/mips/cps.c
new file mode 100644
index 0000000..4ef337d
--- /dev/null
+++ b/hw/mips/cps.c
@@ -0,0 +1,192 @@
+/*
+ * Coherent Processing System emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/mips/cps.h"
+#include "hw/mips/mips.h"
+#include "hw/mips/cpudevs.h"
+#include "sysemu/kvm.h"
+
+qemu_irq get_cps_irq(MIPSCPSState *s, int pin_number)
+{
+    assert(pin_number < s->num_irq);
+    return s->gic.irq_state[pin_number].irq;
+}
+
+static void mips_cps_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSCPSState *s = MIPS_CPS(obj);
+
+    /* Cover entire address space as there do not seem to be any
+     * constraints for the base address of CPC and GIC. */
+    memory_region_init(&s->container, obj, "mips-cps-container", UINT64_MAX);
+    sysbus_init_mmio(sbd, &s->container);
+}
+
+static void main_cpu_reset(void *opaque)
+{
+    MIPSCPU *cpu = opaque;
+    CPUState *cs = CPU(cpu);
+
+    cpu_reset(cs);
+
+    /* All VPs are halted on reset. Leave powering up to CPC. */
+    cs->halted = 1;
+}
+
+static bool cpu_mips_itu_supported(CPUMIPSState *env)
+{
+    bool is_mt = (env->CP0_Config5 & (1 << CP0C5_VP)) ||
+                 (env->CP0_Config3 & (1 << CP0C3_MT));
+
+    return is_mt && !kvm_enabled();
+}
+
+static void mips_cps_realize(DeviceState *dev, Error **errp)
+{
+    MIPSCPSState *s = MIPS_CPS(dev);
+    CPUMIPSState *env;
+    MIPSCPU *cpu;
+    int i;
+    Error *err = NULL;
+    target_ulong gcr_base;
+    bool itu_present = false;
+
+    for (i = 0; i < s->num_vp; i++) {
+        cpu = cpu_mips_init(s->cpu_model);
+        if (cpu == NULL) {
+            error_setg(errp, "%s: CPU initialization failed",  __func__);
+            return;
+        }
+
+        /* Init internal devices */
+        cpu_mips_irq_init_cpu(cpu);
+        cpu_mips_clock_init(cpu);
+
+        env = &cpu->env;
+        if (cpu_mips_itu_supported(env)) {
+            itu_present = true;
+            /* Attach ITC Tag to the VP */
+            env->itc_tag = mips_itu_get_tag_region(&s->itu);
+        }
+        qemu_register_reset(main_cpu_reset, cpu);
+    }
+
+    cpu = MIPS_CPU(first_cpu);
+    env = &cpu->env;
+
+    /* Inter-Thread Communication Unit */
+    if (itu_present) {
+        object_initialize(&s->itu, sizeof(s->itu), TYPE_MIPS_ITU);
+        qdev_set_parent_bus(DEVICE(&s->itu), sysbus_get_default());
+
+        object_property_set_int(OBJECT(&s->itu), 16, "num-fifo", &err);
+        object_property_set_int(OBJECT(&s->itu), 16, "num-semaphores", &err);
+        object_property_set_bool(OBJECT(&s->itu), true, "realized", &err);
+        if (err != NULL) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        memory_region_add_subregion(&s->container, 0,
+                           sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->itu), 0));
+    }
+
+    /* Cluster Power Controller */
+    object_initialize(&s->cpc, sizeof(s->cpc), TYPE_MIPS_CPC);
+    qdev_set_parent_bus(DEVICE(&s->cpc), sysbus_get_default());
+
+    object_property_set_int(OBJECT(&s->cpc), s->num_vp, "num-vp", &err);
+    object_property_set_int(OBJECT(&s->cpc), 1, "vp-start-running", &err);
+    object_property_set_bool(OBJECT(&s->cpc), true, "realized", &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->container, 0,
+                            sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->cpc), 0));
+
+    /* Global Interrupt Controller */
+    object_initialize(&s->gic, sizeof(s->gic), TYPE_MIPS_GIC);
+    qdev_set_parent_bus(DEVICE(&s->gic), sysbus_get_default());
+
+    object_property_set_int(OBJECT(&s->gic), s->num_vp, "num-vp", &err);
+    object_property_set_int(OBJECT(&s->gic), 128, "num-irq", &err);
+    object_property_set_bool(OBJECT(&s->gic), true, "realized", &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->container, 0,
+                            sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->gic), 0));
+
+    /* Global Configuration Registers */
+    gcr_base = env->CP0_CMGCRBase << 4;
+
+    object_initialize(&s->gcr, sizeof(s->gcr), TYPE_MIPS_GCR);
+    qdev_set_parent_bus(DEVICE(&s->gcr), sysbus_get_default());
+
+    object_property_set_int(OBJECT(&s->gcr), s->num_vp, "num-vp", &err);
+    object_property_set_int(OBJECT(&s->gcr), 0x800, "gcr-rev", &err);
+    object_property_set_int(OBJECT(&s->gcr), gcr_base, "gcr-base", &err);
+    object_property_set_link(OBJECT(&s->gcr), OBJECT(&s->gic.mr), "gic", &err);
+    object_property_set_link(OBJECT(&s->gcr), OBJECT(&s->cpc.mr), "cpc", &err);
+    object_property_set_bool(OBJECT(&s->gcr), true, "realized", &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    memory_region_add_subregion(&s->container, gcr_base,
+                            sysbus_mmio_get_region(SYS_BUS_DEVICE(&s->gcr), 0));
+}
+
+static Property mips_cps_properties[] = {
+    DEFINE_PROP_UINT32("num-vp", MIPSCPSState, num_vp, 1),
+    DEFINE_PROP_UINT32("num-irq", MIPSCPSState, num_irq, 256),
+    DEFINE_PROP_STRING("cpu-model", MIPSCPSState, cpu_model),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void mips_cps_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = mips_cps_realize;
+    dc->props = mips_cps_properties;
+}
+
+static const TypeInfo mips_cps_info = {
+    .name = TYPE_MIPS_CPS,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSCPSState),
+    .instance_init = mips_cps_init,
+    .class_init = mips_cps_class_init,
+};
+
+static void mips_cps_register_types(void)
+{
+    type_register_static(&mips_cps_info);
+}
+
+type_init(mips_cps_register_types)
diff --git a/hw/mips/cputimer.c b/hw/mips/cputimer.c
index 577c9ae..8a166b3 100644
--- a/hw/mips/cputimer.c
+++ b/hw/mips/cputimer.c
@@ -20,23 +20,32 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/cpudevs.h"
 #include "qemu/timer.h"
 #include "sysemu/kvm.h"
 
-#define TIMER_FREQ	100 * 1000 * 1000
+#define TIMER_PERIOD 10 /* 10 ns period for 100 Mhz frequency */
 
 /* XXX: do not use a global */
 uint32_t cpu_mips_get_random (CPUMIPSState *env)
 {
-    static uint32_t lfsr = 1;
+    static uint32_t seed = 1;
     static uint32_t prev_idx = 0;
     uint32_t idx;
+    uint32_t nb_rand_tlb = env->tlb->nb_tlb - env->CP0_Wired;
+
+    if (nb_rand_tlb == 1) {
+        return env->tlb->nb_tlb - 1;
+    }
+
     /* Don't return same value twice, so get another value */
     do {
-        lfsr = (lfsr >> 1) ^ (-(lfsr & 1u) & 0xd0000001u);
-        idx = lfsr % (env->tlb->nb_tlb - env->CP0_Wired) + env->CP0_Wired;
+        /* Use a simple algorithm of Linear Congruential Generator
+         * from ISO/IEC 9899 standard. */
+        seed = 1103515245 * seed + 12345;
+        idx = (seed >> 16) % nb_rand_tlb + env->CP0_Wired;
     } while (idx == prev_idx);
     prev_idx = idx;
     return idx;
@@ -49,9 +58,8 @@
     uint32_t wait;
 
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    wait = env->CP0_Compare - env->CP0_Count -
-	    (uint32_t)muldiv64(now, TIMER_FREQ, get_ticks_per_sec());
-    next = now + muldiv64(wait, get_ticks_per_sec(), TIMER_FREQ);
+    wait = env->CP0_Compare - env->CP0_Count - (uint32_t)(now / TIMER_PERIOD);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
     timer_mod(env->timer, next);
 }
 
@@ -79,8 +87,7 @@
             cpu_mips_timer_expire(env);
         }
 
-        return env->CP0_Count +
-            (uint32_t)muldiv64(now, TIMER_FREQ, get_ticks_per_sec());
+        return env->CP0_Count + (uint32_t)(now / TIMER_PERIOD);
     }
 }
 
@@ -95,9 +102,8 @@
         env->CP0_Count = count;
     else {
         /* Store new count register */
-        env->CP0_Count =
-            count - (uint32_t)muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                                       TIMER_FREQ, get_ticks_per_sec());
+        env->CP0_Count = count -
+               (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / TIMER_PERIOD);
         /* Update timer timer */
         cpu_mips_timer_update(env);
     }
@@ -121,8 +127,8 @@
 void cpu_mips_stop_count(CPUMIPSState *env)
 {
     /* Store the current value */
-    env->CP0_Count += (uint32_t)muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                                         TIMER_FREQ, get_ticks_per_sec());
+    env->CP0_Count += (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) /
+                                 TIMER_PERIOD);
 }
 
 static void mips_timer_cb (void *opaque)
@@ -145,8 +151,10 @@
     env->CP0_Count--;
 }
 
-void cpu_mips_clock_init (CPUMIPSState *env)
+void cpu_mips_clock_init (MIPSCPU *cpu)
 {
+    CPUMIPSState *env = &cpu->env;
+
     /*
      * If we're in KVM mode, don't create the periodic timer, that is handled in
      * kernel.
diff --git a/hw/mips/gt64xxx_pci.c b/hw/mips/gt64xxx_pci.c
index 1f2fe5f..4811843 100644
--- a/hw/mips/gt64xxx_pci.c
+++ b/hw/mips/gt64xxx_pci.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/pci/pci.h"
@@ -239,7 +240,11 @@
 
     uint32_t regs[GT_REGS];
     PCI_MAPPING_ENTRY(PCI0IO);
+    PCI_MAPPING_ENTRY(PCI0M0);
+    PCI_MAPPING_ENTRY(PCI0M1);
     PCI_MAPPING_ENTRY(ISD);
+    MemoryRegion pci0_mem;
+    AddressSpace pci0_mem_as;
 } GT64120State;
 
 /* Adjust range to avoid touching space which isn't mappable via PCI */
@@ -271,7 +276,8 @@
 
 static void gt64120_isd_mapping(GT64120State *s)
 {
-    hwaddr start = s->regs[GT_ISD] << 21;
+    /* Bits 14:0 of ISD map to bits 35:21 of the start address.  */
+    hwaddr start = ((hwaddr)s->regs[GT_ISD] << 21) & 0xFFFE00000ull;
     hwaddr length = 0x1000;
 
     if (s->ISD_length) {
@@ -290,25 +296,63 @@
 
 static void gt64120_pci_mapping(GT64120State *s)
 {
-    /* Update IO mapping */
-    if ((s->regs[GT_PCI0IOLD] & 0x7f) <= s->regs[GT_PCI0IOHD])
-    {
-      /* Unmap old IO address */
-      if (s->PCI0IO_length)
-      {
-          memory_region_del_subregion(get_system_memory(), &s->PCI0IO_mem);
-          object_unparent(OBJECT(&s->PCI0IO_mem));
-      }
-      /* Map new IO address */
-      s->PCI0IO_start = s->regs[GT_PCI0IOLD] << 21;
-      s->PCI0IO_length = ((s->regs[GT_PCI0IOHD] + 1) - (s->regs[GT_PCI0IOLD] & 0x7f)) << 21;
-      isa_mem_base = s->PCI0IO_start;
-      if (s->PCI0IO_length) {
-          memory_region_init_alias(&s->PCI0IO_mem, OBJECT(s), "isa_mmio",
-                                   get_system_io(), 0, s->PCI0IO_length);
-          memory_region_add_subregion(get_system_memory(), s->PCI0IO_start,
-                                      &s->PCI0IO_mem);
-      }
+    /* Update PCI0IO mapping */
+    if ((s->regs[GT_PCI0IOLD] & 0x7f) <= s->regs[GT_PCI0IOHD]) {
+        /* Unmap old IO address */
+        if (s->PCI0IO_length) {
+            memory_region_del_subregion(get_system_memory(), &s->PCI0IO_mem);
+            object_unparent(OBJECT(&s->PCI0IO_mem));
+        }
+        /* Map new IO address */
+        s->PCI0IO_start = s->regs[GT_PCI0IOLD] << 21;
+        s->PCI0IO_length = ((s->regs[GT_PCI0IOHD] + 1) -
+                            (s->regs[GT_PCI0IOLD] & 0x7f)) << 21;
+        if (s->PCI0IO_length) {
+            memory_region_init_alias(&s->PCI0IO_mem, OBJECT(s), "pci0-io",
+                                     get_system_io(), 0, s->PCI0IO_length);
+            memory_region_add_subregion(get_system_memory(), s->PCI0IO_start,
+                                        &s->PCI0IO_mem);
+        }
+    }
+
+    /* Update PCI0M0 mapping */
+    if ((s->regs[GT_PCI0M0LD] & 0x7f) <= s->regs[GT_PCI0M0HD]) {
+        /* Unmap old MEM address */
+        if (s->PCI0M0_length) {
+            memory_region_del_subregion(get_system_memory(), &s->PCI0M0_mem);
+            object_unparent(OBJECT(&s->PCI0M0_mem));
+        }
+        /* Map new mem address */
+        s->PCI0M0_start = s->regs[GT_PCI0M0LD] << 21;
+        s->PCI0M0_length = ((s->regs[GT_PCI0M0HD] + 1) -
+                            (s->regs[GT_PCI0M0LD] & 0x7f)) << 21;
+        if (s->PCI0M0_length) {
+            memory_region_init_alias(&s->PCI0M0_mem, OBJECT(s), "pci0-mem0",
+                                     &s->pci0_mem, s->PCI0M0_start,
+                                     s->PCI0M0_length);
+            memory_region_add_subregion(get_system_memory(), s->PCI0M0_start,
+                                        &s->PCI0M0_mem);
+        }
+    }
+
+    /* Update PCI0M1 mapping */
+    if ((s->regs[GT_PCI0M1LD] & 0x7f) <= s->regs[GT_PCI0M1HD]) {
+        /* Unmap old MEM address */
+        if (s->PCI0M1_length) {
+            memory_region_del_subregion(get_system_memory(), &s->PCI0M1_mem);
+            object_unparent(OBJECT(&s->PCI0M1_mem));
+        }
+        /* Map new mem address */
+        s->PCI0M1_start = s->regs[GT_PCI0M1LD] << 21;
+        s->PCI0M1_length = ((s->regs[GT_PCI0M1HD] + 1) -
+                            (s->regs[GT_PCI0M1LD] & 0x7f)) << 21;
+        if (s->PCI0M1_length) {
+            memory_region_init_alias(&s->PCI0M1_mem, OBJECT(s), "pci0-mem1",
+                                     &s->pci0_mem, s->PCI0M1_start,
+                                     s->PCI0M1_length);
+            memory_region_add_subregion(get_system_memory(), s->PCI0M1_start,
+                                        &s->PCI0M1_mem);
+        }
     }
 }
 
@@ -363,10 +407,12 @@
     case GT_PCI0M0LD:
         s->regs[GT_PCI0M0LD]    = val & 0x00007fff;
         s->regs[GT_PCI0M0REMAP] = val & 0x000007ff;
+        gt64120_pci_mapping(s);
         break;
     case GT_PCI0M1LD:
         s->regs[GT_PCI0M1LD]    = val & 0x00007fff;
         s->regs[GT_PCI0M1REMAP] = val & 0x000007ff;
+        gt64120_pci_mapping(s);
         break;
     case GT_PCI1IOLD:
         s->regs[GT_PCI1IOLD]    = val & 0x00007fff;
@@ -380,12 +426,12 @@
         s->regs[GT_PCI1M1LD]    = val & 0x00007fff;
         s->regs[GT_PCI1M1REMAP] = val & 0x000007ff;
         break;
+    case GT_PCI0M0HD:
+    case GT_PCI0M1HD:
     case GT_PCI0IOHD:
         s->regs[saddr] = val & 0x0000007f;
         gt64120_pci_mapping(s);
         break;
-    case GT_PCI0M0HD:
-    case GT_PCI0M1HD:
     case GT_PCI1IOHD:
     case GT_PCI1M0HD:
     case GT_PCI1M1HD:
@@ -1121,15 +1167,17 @@
     DeviceState *dev;
 
     dev = qdev_create(NULL, TYPE_GT64120_PCI_HOST_BRIDGE);
-    qdev_init_nofail(dev);
     d = GT64120_PCI_HOST_BRIDGE(dev);
     phb = PCI_HOST_BRIDGE(dev);
+    memory_region_init(&d->pci0_mem, OBJECT(dev), "pci0-mem", UINT32_MAX);
+    address_space_init(&d->pci0_mem_as, &d->pci0_mem, "pci0-mem");
     phb->bus = pci_register_bus(dev, "pci",
                                 gt64120_pci_set_irq, gt64120_pci_map_irq,
                                 pic,
-                                get_system_memory(),
+                                &d->pci0_mem,
                                 get_system_io(),
                                 PCI_DEVFN(18, 0), 4, TYPE_PCI_BUS);
+    qdev_init_nofail(dev);
     memory_region_init_io(&d->ISD_mem, OBJECT(dev), &isd_mem_ops, d, "isd-mem", 0x1000);
 
     pci_create_simple(phb->bus, PCI_DEVFN(0, 0), "gt64120_pci");
@@ -1142,16 +1190,11 @@
 
     s = GT64120_PCI_HOST_BRIDGE(dev);
 
-    /* FIXME: This value is computed from registers during reset, but some
-       devices (e.g. VGA card) need to know it when they are registered.
-       This also mean that changing the register to change the mapping
-       does not fully work. */
-    isa_mem_base = 0x10000000;
     qemu_register_reset(gt64120_reset, s);
     return 0;
 }
 
-static int gt64120_pci_init(PCIDevice *d)
+static void gt64120_pci_realize(PCIDevice *d, Error **errp)
 {
     /* FIXME: Malta specific hw assumptions ahead */
     pci_set_word(d->config + PCI_COMMAND, 0);
@@ -1165,8 +1208,6 @@
     pci_set_long(d->config + PCI_BASE_ADDRESS_4, 0x14000000);
     pci_set_long(d->config + PCI_BASE_ADDRESS_5, 0x14000001);
     pci_set_byte(d->config + 0x3d, 0x01);
-
-    return 0;
 }
 
 static void gt64120_pci_class_init(ObjectClass *klass, void *data)
@@ -1174,7 +1215,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = gt64120_pci_init;
+    k->realize = gt64120_pci_realize;
     k->vendor_id = PCI_VENDOR_ID_MARVELL;
     k->device_id = PCI_DEVICE_ID_MARVELL_GT6412X;
     k->revision = 0x10;
diff --git a/hw/mips/mips_fulong2e.c b/hw/mips/mips_fulong2e.c
index 6a9ebfa..889cdc7 100644
--- a/hw/mips/mips_fulong2e.c
+++ b/hw/mips/mips_fulong2e.c
@@ -18,6 +18,8 @@
  * http://www.loongsondeveloper.com/doc/Loongson2EUserGuide.pdf
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/char/serial.h"
@@ -116,7 +118,7 @@
 
     if (load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
                  (uint64_t *)&kernel_entry, (uint64_t *)&kernel_low,
-                 (uint64_t *)&kernel_high, 0, ELF_MACHINE, 1) < 0) {
+                 (uint64_t *)&kernel_high, 0, EM_MIPS, 1, 0) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 loaderparams.kernel_filename);
         exit(1);
@@ -168,6 +170,7 @@
     rom_add_blob_fixed("prom", prom_buf, prom_size,
                        cpu_mips_kseg0_to_phys(NULL, ENVP_ADDR));
 
+    g_free(prom_buf);
     return kernel_entry;
 }
 
@@ -250,15 +253,6 @@
     }
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static void mips_fulong2e_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
@@ -273,11 +267,9 @@
     long bios_size;
     int64_t kernel_entry;
     qemu_irq *i8259;
-    qemu_irq *cpu_exit_irq;
     PCIBus *pci_bus;
     ISABus *isa_bus;
     I2CBus *smbus;
-    int i;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     MIPSCPU *cpu;
     CPUMIPSState *env;
@@ -302,10 +294,9 @@
     bios_size = 1024 * 1024;
 
     /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "fulong2e.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "fulong2e.ram", ram_size);
     memory_region_init_ram(bios, NULL, "fulong2e.bios", bios_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -343,8 +334,8 @@
     }
 
     /* Init internal devices */
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
+    cpu_mips_irq_init_cpu(cpu);
+    cpu_mips_clock_init(cpu);
 
     /* North bridge, Bonito --> IP2 */
     pci_bus = bonito_init((qemu_irq *)&(env->irq[2]));
@@ -376,23 +367,15 @@
 
     /* init other devices */
     pit = pit_init(isa_bus, 0x40, 0, NULL);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
 
     /* Super I/O */
     isa_create_simple(isa_bus, "i8042");
 
     rtc_init(isa_bus, 2000, NULL);
 
-    for(i = 0; i < MAX_SERIAL_PORTS; i++) {
-        if (serial_hds[i]) {
-            serial_isa_init(isa_bus, i, serial_hds[i]);
-        }
-    }
-
-    if (parallel_hds[0]) {
-        parallel_init(isa_bus, 0, parallel_hds[0]);
-    }
+    serial_hds_isa_init(isa_bus, MAX_SERIAL_PORTS);
+    parallel_hds_isa_init(isa_bus, 1);
 
     /* Sound card */
     audio_init(pci_bus);
@@ -400,15 +383,10 @@
     network_init(pci_bus);
 }
 
-static QEMUMachine mips_fulong2e_machine = {
-    .name = "fulong2e",
-    .desc = "Fulong 2e mini pc",
-    .init = mips_fulong2e_init,
-};
-
-static void mips_fulong2e_machine_init(void)
+static void mips_fulong2e_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_fulong2e_machine);
+    mc->desc = "Fulong 2e mini pc";
+    mc->init = mips_fulong2e_init;
 }
 
-machine_init(mips_fulong2e_machine_init);
+DEFINE_MACHINE("fulong2e", mips_fulong2e_machine_init)
diff --git a/hw/mips/mips_int.c b/hw/mips/mips_int.c
index d740046..48192d2 100644
--- a/hw/mips/mips_int.c
+++ b/hw/mips/mips_int.c
@@ -20,6 +20,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/cpudevs.h"
 #include "cpu.h"
@@ -57,8 +58,9 @@
     }
 }
 
-void cpu_mips_irq_init_cpu(CPUMIPSState *env)
+void cpu_mips_irq_init_cpu(MIPSCPU *cpu)
 {
+    CPUMIPSState *env = &cpu->env;
     qemu_irq *qi;
     int i;
 
diff --git a/hw/mips/mips_jazz.c b/hw/mips/mips_jazz.c
index 3f33093..73f6c9f 100644
--- a/hw/mips/mips_jazz.c
+++ b/hw/mips/mips_jazz.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -44,6 +45,7 @@
 #include "exec/address-spaces.h"
 #include "sysemu/qtest.h"
 #include "qemu/error-report.h"
+#include "qemu/help_option.h"
 
 enum jazz_model_e
 {
@@ -60,13 +62,18 @@
 
 static uint64_t rtc_read(void *opaque, hwaddr addr, unsigned size)
 {
-    return cpu_inw(0x71);
+    uint8_t val;
+    address_space_read(&address_space_memory, 0x90000071,
+                       MEMTXATTRS_UNSPECIFIED, &val, 1);
+    return val;
 }
 
 static void rtc_write(void *opaque, hwaddr addr,
                       uint64_t val, unsigned size)
 {
-    cpu_outw(0x71, val & 0xff);
+    uint8_t buf = val & 0xff;
+    address_space_write(&address_space_memory, 0x90000071,
+                        MEMTXATTRS_UNSPECIFIED, &buf, 1);
 }
 
 static const MemoryRegionOps rtc_ops = {
@@ -99,15 +106,6 @@
 #define MAGNUM_BIOS_SIZE_MAX 0x7e000
 #define MAGNUM_BIOS_SIZE (BIOS_SIZE < MAGNUM_BIOS_SIZE_MAX ? BIOS_SIZE : MAGNUM_BIOS_SIZE_MAX)
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static CPUUnassignedAccess real_do_unassigned_access;
 static void mips_jazz_do_unassigned_access(CPUState *cpu, hwaddr addr,
                                            bool is_write, bool is_exec,
@@ -120,44 +118,38 @@
     (*real_do_unassigned_access)(cpu, addr, is_write, is_exec, opaque, size);
 }
 
-static void mips_jazz_init(MemoryRegion *address_space,
-                           MemoryRegion *address_space_io,
-                           ram_addr_t ram_size,
-                           const char *cpu_model,
+static void mips_jazz_init(MachineState *machine,
                            enum jazz_model_e jazz_model)
 {
+    MemoryRegion *address_space = get_system_memory();
+    const char *cpu_model = machine->cpu_model;
     char *filename;
     int bios_size, n;
     MIPSCPU *cpu;
     CPUClass *cc;
     CPUMIPSState *env;
-    qemu_irq *rc4030, *i8259;
+    qemu_irq *i8259;
     rc4030_dma *dmas;
-    void* rc4030_opaque;
-    MemoryRegion *isa = g_new(MemoryRegion, 1);
+    MemoryRegion *rc4030_dma_mr;
+    MemoryRegion *isa_mem = g_new(MemoryRegion, 1);
+    MemoryRegion *isa_io = g_new(MemoryRegion, 1);
     MemoryRegion *rtc = g_new(MemoryRegion, 1);
     MemoryRegion *i8042 = g_new(MemoryRegion, 1);
     MemoryRegion *dma_dummy = g_new(MemoryRegion, 1);
     NICInfo *nd;
-    DeviceState *dev;
+    DeviceState *dev, *rc4030;
     SysBusDevice *sysbus;
     ISABus *isa_bus;
     ISADevice *pit;
     DriveInfo *fds[MAX_FD];
     qemu_irq esp_reset, dma_enable;
-    qemu_irq *cpu_exit_irq;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *bios = g_new(MemoryRegion, 1);
     MemoryRegion *bios2 = g_new(MemoryRegion, 1);
 
     /* init CPUs */
     if (cpu_model == NULL) {
-#ifdef TARGET_MIPS64
         cpu_model = "R4000";
-#else
-        /* FIXME: All wrong, this maybe should be R3000 for the older JAZZs. */
-        cpu_model = "24Kf";
-#endif
     }
     cpu = cpu_mips_init(cpu_model);
     if (cpu == NULL) {
@@ -179,12 +171,12 @@
     cc->do_unassigned_access = mips_jazz_do_unassigned_access;
 
     /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "mips_jazz.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "mips_jazz.ram",
+                                         machine->ram_size);
     memory_region_add_subregion(address_space, 0, ram);
 
     memory_region_init_ram(bios, NULL, "mips_jazz.bios", MAGNUM_BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
     memory_region_init_alias(bios2, NULL, "mips_jazz.bios", bios,
@@ -209,30 +201,35 @@
     }
 
     /* Init CPU internal devices */
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
+    cpu_mips_irq_init_cpu(cpu);
+    cpu_mips_clock_init(cpu);
 
     /* Chipset */
-    rc4030_opaque = rc4030_init(env->irq[6], env->irq[3], &rc4030, &dmas,
-                                address_space);
+    rc4030 = rc4030_init(&dmas, &rc4030_dma_mr);
+    sysbus = SYS_BUS_DEVICE(rc4030);
+    sysbus_connect_irq(sysbus, 0, env->irq[6]);
+    sysbus_connect_irq(sysbus, 1, env->irq[3]);
+    memory_region_add_subregion(address_space, 0x80000000,
+                                sysbus_mmio_get_region(sysbus, 0));
+    memory_region_add_subregion(address_space, 0xf0000000,
+                                sysbus_mmio_get_region(sysbus, 1));
     memory_region_init_io(dma_dummy, NULL, &dma_dummy_ops, NULL, "dummy_dma", 0x1000);
     memory_region_add_subregion(address_space, 0x8000d000, dma_dummy);
 
+    /* ISA bus: IO space at 0x90000000, mem space at 0x91000000 */
+    memory_region_init(isa_io, NULL, "isa-io", 0x00010000);
+    memory_region_init(isa_mem, NULL, "isa-mem", 0x01000000);
+    memory_region_add_subregion(address_space, 0x90000000, isa_io);
+    memory_region_add_subregion(address_space, 0x91000000, isa_mem);
+    isa_bus = isa_bus_new(NULL, isa_mem, isa_io, &error_abort);
+
     /* ISA devices */
-    isa_bus = isa_bus_new(NULL, address_space_io);
     i8259 = i8259_init(isa_bus, env->irq[4]);
     isa_bus_irqs(isa_bus, i8259);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
     pit = pit_init(isa_bus, 0x40, 0, NULL);
     pcspk_init(isa_bus, pit);
 
-    /* ISA IO space at 0x90000000 */
-    memory_region_init_alias(isa, NULL, "isa_mmio",
-                             get_system_io(), 0, 0x01000000);
-    memory_region_add_subregion(address_space, 0x90000000, isa);
-    isa_mem_base = 0x11000000;
-
     /* Video card */
     switch (jazz_model) {
     case JAZZ_MAGNUM:
@@ -241,12 +238,12 @@
         sysbus = SYS_BUS_DEVICE(dev);
         sysbus_mmio_map(sysbus, 0, 0x60080000);
         sysbus_mmio_map(sysbus, 1, 0x40000000);
-        sysbus_connect_irq(sysbus, 0, rc4030[3]);
+        sysbus_connect_irq(sysbus, 0, qdev_get_gpio_in(rc4030, 3));
         {
             /* Simple ROM, so user doesn't have to provide one */
             MemoryRegion *rom_mr = g_new(MemoryRegion, 1);
             memory_region_init_ram(rom_mr, NULL, "g364fb.rom", 0x80000,
-                                   &error_abort);
+                                   &error_fatal);
             vmstate_register_ram_global(rom_mr);
             memory_region_set_readonly(rom_mr, true);
             uint8_t *rom = memory_region_get_ram_ptr(rom_mr);
@@ -267,8 +264,17 @@
         if (!nd->model)
             nd->model = g_strdup("dp83932");
         if (strcmp(nd->model, "dp83932") == 0) {
-            dp83932_init(nd, 0x80001000, 2, get_system_memory(), rc4030[4],
-                         rc4030_opaque, rc4030_dma_memory_rw);
+            qemu_check_nic_model(nd, "dp83932");
+
+            dev = qdev_create(NULL, "dp8393x");
+            qdev_set_nic_properties(dev, nd);
+            qdev_prop_set_uint8(dev, "it_shift", 2);
+            qdev_prop_set_ptr(dev, "dma_mr", rc4030_dma_mr);
+            qdev_init_nofail(dev);
+            sysbus = SYS_BUS_DEVICE(dev);
+            sysbus_mmio_map(sysbus, 0, 0x80001000);
+            sysbus_mmio_map(sysbus, 1, 0x8000b000);
+            sysbus_connect_irq(sysbus, 0, qdev_get_gpio_in(rc4030, 4));
             break;
         } else if (is_help_option(nd->model)) {
             fprintf(stderr, "qemu: Supported NICs: dp83932\n");
@@ -282,7 +288,7 @@
     /* SCSI adapter */
     esp_init(0x80002000, 0,
              rc4030_dma_read, rc4030_dma_write, dmas[0],
-             rc4030[5], &esp_reset, &dma_enable);
+             qdev_get_gpio_in(rc4030, 5), &esp_reset, &dma_enable);
 
     /* Floppy */
     if (drive_get_max_bus(IF_FLOPPY) >= MAX_FD) {
@@ -292,7 +298,8 @@
     for (n = 0; n < MAX_FD; n++) {
         fds[n] = drive_get(IF_FLOPPY, 0, n);
     }
-    fdctrl_init_sysbus(rc4030[1], 0, 0x80003000, fds);
+    /* FIXME: we should enable DMA with a custom IsaDma device */
+    fdctrl_init_sysbus(qdev_get_gpio_in(rc4030, 1), -1, 0x80003000, fds);
 
     /* Real time clock */
     rtc_init(isa_bus, 1980, NULL);
@@ -300,23 +307,26 @@
     memory_region_add_subregion(address_space, 0x80004000, rtc);
 
     /* Keyboard (i8042) */
-    i8042_mm_init(rc4030[6], rc4030[7], i8042, 0x1000, 0x1);
+    i8042_mm_init(qdev_get_gpio_in(rc4030, 6), qdev_get_gpio_in(rc4030, 7),
+                  i8042, 0x1000, 0x1);
     memory_region_add_subregion(address_space, 0x80005000, i8042);
 
     /* Serial ports */
     if (serial_hds[0]) {
-        serial_mm_init(address_space, 0x80006000, 0, rc4030[8], 8000000/16,
+        serial_mm_init(address_space, 0x80006000, 0,
+                       qdev_get_gpio_in(rc4030, 8), 8000000/16,
                        serial_hds[0], DEVICE_NATIVE_ENDIAN);
     }
     if (serial_hds[1]) {
-        serial_mm_init(address_space, 0x80007000, 0, rc4030[9], 8000000/16,
+        serial_mm_init(address_space, 0x80007000, 0,
+                       qdev_get_gpio_in(rc4030, 9), 8000000/16,
                        serial_hds[1], DEVICE_NATIVE_ENDIAN);
     }
 
     /* Parallel port */
     if (parallel_hds[0])
-        parallel_mm_init(address_space, 0x80008000, 0, rc4030[0],
-                         parallel_hds[0]);
+        parallel_mm_init(address_space, 0x80008000, 0,
+                         qdev_get_gpio_in(rc4030, 0), parallel_hds[0]);
 
     /* FIXME: missing Jazz sound at 0x8000c000, rc4030[2] */
 
@@ -333,39 +343,49 @@
 static
 void mips_magnum_init(MachineState *machine)
 {
-    ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
-        mips_jazz_init(get_system_memory(), get_system_io(),
-                       ram_size, cpu_model, JAZZ_MAGNUM);
+    mips_jazz_init(machine, JAZZ_MAGNUM);
 }
 
 static
 void mips_pica61_init(MachineState *machine)
 {
-    ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
-    mips_jazz_init(get_system_memory(), get_system_io(),
-                   ram_size, cpu_model, JAZZ_PICA61);
+    mips_jazz_init(machine, JAZZ_PICA61);
 }
 
-static QEMUMachine mips_magnum_machine = {
-    .name = "magnum",
-    .desc = "MIPS Magnum",
-    .init = mips_magnum_init,
-    .block_default_type = IF_SCSI,
+static void mips_magnum_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "MIPS Magnum";
+    mc->init = mips_magnum_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo mips_magnum_type = {
+    .name = MACHINE_TYPE_NAME("magnum"),
+    .parent = TYPE_MACHINE,
+    .class_init = mips_magnum_class_init,
 };
 
-static QEMUMachine mips_pica61_machine = {
-    .name = "pica61",
-    .desc = "Acer Pica 61",
-    .init = mips_pica61_init,
-    .block_default_type = IF_SCSI,
+static void mips_pica61_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Acer Pica 61";
+    mc->init = mips_pica61_init;
+    mc->block_default_type = IF_SCSI;
+}
+
+static const TypeInfo mips_pica61_type = {
+    .name = MACHINE_TYPE_NAME("pica61"),
+    .parent = TYPE_MACHINE,
+    .class_init = mips_pica61_class_init,
 };
 
 static void mips_jazz_machine_init(void)
 {
-    qemu_register_machine(&mips_magnum_machine);
-    qemu_register_machine(&mips_pica61_machine);
+    type_register_static(&mips_magnum_type);
+    type_register_static(&mips_pica61_type);
 }
 
-machine_init(mips_jazz_machine_init);
+type_init(mips_jazz_machine_init)
diff --git a/hw/mips/mips_malta.c b/hw/mips/mips_malta.c
index 5845158..e90857e 100644
--- a/hw/mips/mips_malta.c
+++ b/hw/mips/mips_malta.c
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/char/serial.h"
@@ -53,6 +56,8 @@
 #include "qemu/error-report.h"
 #include "hw/empty_slot.h"
 #include "sysemu/kvm.h"
+#include "exec/semihost.h"
+#include "hw/mips/cps.h"
 
 //#define DEBUG_BOARD_INIT
 
@@ -91,13 +96,14 @@
 typedef struct {
     SysBusDevice parent_obj;
 
+    MIPSCPSState *cps;
     qemu_irq *i8259;
 } MaltaState;
 
 static ISADevice *pit;
 
 static struct _loaderparams {
-    int ram_size;
+    int ram_size, ram_low_size;
     const char *kernel_filename;
     const char *kernel_cmdline;
     const char *initrd_filename;
@@ -604,8 +610,8 @@
      a3 - RAM size in bytes
 */
 
-static void write_bootloader (CPUMIPSState *env, uint8_t *base,
-                              int64_t run_addr, int64_t kernel_entry)
+static void write_bootloader(uint8_t *base, int64_t run_addr,
+                             int64_t kernel_entry)
 {
     uint32_t *p;
 
@@ -634,15 +640,21 @@
 
     /* Second part of the bootloader */
     p = (uint32_t *) (base + 0x580);
-    stl_p(p++, 0x24040002);                                      /* addiu a0, zero, 2 */
+
+    if (semihosting_get_argc()) {
+        /* Preserve a0 content as arguments have been passed */
+        stl_p(p++, 0x00000000);                         /* nop */
+    } else {
+        stl_p(p++, 0x24040002);                         /* addiu a0, zero, 2 */
+    }
     stl_p(p++, 0x3c1d0000 | (((ENVP_ADDR - 64) >> 16) & 0xffff)); /* lui sp, high(ENVP_ADDR) */
     stl_p(p++, 0x37bd0000 | ((ENVP_ADDR - 64) & 0xffff));        /* ori sp, sp, low(ENVP_ADDR) */
     stl_p(p++, 0x3c050000 | ((ENVP_ADDR >> 16) & 0xffff));       /* lui a1, high(ENVP_ADDR) */
     stl_p(p++, 0x34a50000 | (ENVP_ADDR & 0xffff));               /* ori a1, a1, low(ENVP_ADDR) */
     stl_p(p++, 0x3c060000 | (((ENVP_ADDR + 8) >> 16) & 0xffff)); /* lui a2, high(ENVP_ADDR + 8) */
     stl_p(p++, 0x34c60000 | ((ENVP_ADDR + 8) & 0xffff));         /* ori a2, a2, low(ENVP_ADDR + 8) */
-    stl_p(p++, 0x3c070000 | (loaderparams.ram_size >> 16));     /* lui a3, high(ram_size) */
-    stl_p(p++, 0x34e70000 | (loaderparams.ram_size & 0xffff));  /* ori a3, a3, low(ram_size) */
+    stl_p(p++, 0x3c070000 | (loaderparams.ram_low_size >> 16));     /* lui a3, high(ram_low_size) */
+    stl_p(p++, 0x34e70000 | (loaderparams.ram_low_size & 0xffff));  /* ori a3, a3, low(ram_low_size) */
 
     /* Load BAR registers as done by YAMON */
     stl_p(p++, 0x3c09b400);                                      /* lui t1, 0xb400 */
@@ -715,7 +727,7 @@
     stl_p(p++, 0x00000000);                                     /* nop */
     stl_p(p++, 0x0ff0021c);                                     /* jal 870 */
     stl_p(p++, 0x00000000);                                     /* nop */
-    stl_p(p++, 0x08000205);                                     /* j 814 */
+    stl_p(p++, 0x1000fff9);                                     /* b 814 */
     stl_p(p++, 0x00000000);                                     /* nop */
     stl_p(p++, 0x01a00009);                                     /* jalr t5 */
     stl_p(p++, 0x01602021);                                     /* move a0,t3 */
@@ -788,7 +800,7 @@
 
     if (load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
                  (uint64_t *)&kernel_entry, NULL, (uint64_t *)&kernel_high,
-                 big_endian, ELF_MACHINE, 1) < 0) {
+                 big_endian, EM_MIPS, 1, 0) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 loaderparams.kernel_filename);
         exit(1);
@@ -851,8 +863,10 @@
     }
 
     prom_set(prom_buf, prom_index++, "memsize");
-    prom_set(prom_buf, prom_index++, "%i",
-             MIN(loaderparams.ram_size, 256 << 20));
+    prom_set(prom_buf, prom_index++, "%u", loaderparams.ram_low_size);
+
+    prom_set(prom_buf, prom_index++, "ememsize");
+    prom_set(prom_buf, prom_index++, "%u", loaderparams.ram_size);
 
     prom_set(prom_buf, prom_index++, "modetty0");
     prom_set(prom_buf, prom_index++, "38400n8r");
@@ -861,6 +875,7 @@
     rom_add_blob_fixed("prom", prom_buf, prom_size,
                        cpu_mips_kseg0_to_phys(NULL, ENVP_ADDR));
 
+    g_free(prom_buf);
     return kernel_entry;
 }
 
@@ -884,23 +899,81 @@
        read only location. The kernel location and the arguments table
        location does not change. */
     if (loaderparams.kernel_filename) {
-        env->CP0_Status &= ~((1 << CP0St_BEV) | (1 << CP0St_ERL));
+        env->CP0_Status &= ~(1 << CP0St_ERL);
     }
 
     malta_mips_config(cpu);
 
     if (kvm_enabled()) {
         /* Start running from the bootloader we wrote to end of RAM */
-        env->active_tc.PC = 0x40000000 + loaderparams.ram_size;
+        env->active_tc.PC = 0x40000000 + loaderparams.ram_low_size;
     }
 }
 
-static void cpu_request_exit(void *opaque, int irq, int level)
+static void create_cpu_without_cps(const char *cpu_model,
+                                   qemu_irq *cbus_irq, qemu_irq *i8259_irq)
 {
-    CPUState *cpu = current_cpu;
+    CPUMIPSState *env;
+    MIPSCPU *cpu;
+    int i;
 
-    if (cpu && level) {
-        cpu_exit(cpu);
+    for (i = 0; i < smp_cpus; i++) {
+        cpu = cpu_mips_init(cpu_model);
+        if (cpu == NULL) {
+            fprintf(stderr, "Unable to find CPU definition\n");
+            exit(1);
+        }
+
+        /* Init internal devices */
+        cpu_mips_irq_init_cpu(cpu);
+        cpu_mips_clock_init(cpu);
+        qemu_register_reset(main_cpu_reset, cpu);
+    }
+
+    cpu = MIPS_CPU(first_cpu);
+    env = &cpu->env;
+    *i8259_irq = env->irq[2];
+    *cbus_irq = env->irq[4];
+}
+
+static void create_cps(MaltaState *s, const char *cpu_model,
+                       qemu_irq *cbus_irq, qemu_irq *i8259_irq)
+{
+    Error *err = NULL;
+    s->cps = g_new0(MIPSCPSState, 1);
+
+    object_initialize(s->cps, sizeof(MIPSCPSState), TYPE_MIPS_CPS);
+    qdev_set_parent_bus(DEVICE(s->cps), sysbus_get_default());
+
+    object_property_set_str(OBJECT(s->cps), cpu_model, "cpu-model", &err);
+    object_property_set_int(OBJECT(s->cps), smp_cpus, "num-vp", &err);
+    object_property_set_bool(OBJECT(s->cps), true, "realized", &err);
+    if (err != NULL) {
+        error_report("%s", error_get_pretty(err));
+        exit(1);
+    }
+
+    sysbus_mmio_map_overlap(SYS_BUS_DEVICE(s->cps), 0, 0, 1);
+
+    *i8259_irq = get_cps_irq(s->cps, 3);
+    *cbus_irq = NULL;
+}
+
+static void create_cpu(MaltaState *s, const char *cpu_model,
+                       qemu_irq *cbus_irq, qemu_irq *i8259_irq)
+{
+    if (cpu_model == NULL) {
+#ifdef TARGET_MIPS64
+        cpu_model = "20Kc";
+#else
+        cpu_model = "24Kf";
+#endif
+    }
+
+    if ((smp_cpus > 1) && cpu_supports_cps_smp(cpu_model)) {
+        create_cps(s, cpu_model, cbus_irq, i8259_irq);
+    } else {
+        create_cpu_without_cps(cpu_model, cbus_irq, i8259_irq);
     }
 }
 
@@ -909,7 +982,6 @@
 {
     ram_addr_t ram_size = machine->ram_size;
     ram_addr_t ram_low_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -926,10 +998,8 @@
     int64_t kernel_entry, bootloader_run_addr;
     PCIBus *pci_bus;
     ISABus *isa_bus;
-    MIPSCPU *cpu;
-    CPUMIPSState *env;
     qemu_irq *isa_irq;
-    qemu_irq *cpu_exit_irq;
+    qemu_irq cbus_irq, i8259_irq;
     int piix4_devfn;
     I2CBus *smbus;
     int i;
@@ -959,30 +1029,8 @@
         }
     }
 
-    /* init CPUs */
-    if (cpu_model == NULL) {
-#ifdef TARGET_MIPS64
-        cpu_model = "20Kc";
-#else
-        cpu_model = "24Kf";
-#endif
-    }
-
-    for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_mips_init(cpu_model);
-        if (cpu == NULL) {
-            fprintf(stderr, "Unable to find CPU definition\n");
-            exit(1);
-        }
-        env = &cpu->env;
-
-        /* Init internal devices */
-        cpu_mips_irq_init_cpu(env);
-        cpu_mips_clock_init(env);
-        qemu_register_reset(main_cpu_reset, cpu);
-    }
-    cpu = MIPS_CPU(first_cpu);
-    env = &cpu->env;
+    /* create CPU */
+    create_cpu(s, machine->cpu_model, &cbus_irq, &i8259_irq);
 
     /* allocate RAM */
     if (ram_size > (2048u << 20)) {
@@ -993,9 +1041,8 @@
     }
 
     /* register RAM at high address where it is undisturbed by IO */
-    memory_region_init_ram(ram_high, NULL, "mips_malta.ram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ram_high);
+    memory_region_allocate_system_memory(ram_high, NULL, "mips_malta.ram",
+                                         ram_size);
     memory_region_add_subregion(system_memory, 0x80000000, ram_high);
 
     /* alias for pre IO hole access */
@@ -1024,7 +1071,7 @@
 #endif
     /* FPGA */
     /* The CBUS UART is attached to the MIPS CPU INT2 pin, ie interrupt 4 */
-    malta_fpga_init(system_memory, FPGA_ADDRESS, env->irq[4], serial_hds[2]);
+    malta_fpga_init(system_memory, FPGA_ADDRESS, cbus_irq, serial_hds[2]);
 
     /* Load firmware in flash / BIOS. */
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
@@ -1054,17 +1101,18 @@
         }
 
         /* Write a small bootloader to the flash location. */
-        loaderparams.ram_size = ram_low_size;
+        loaderparams.ram_size = ram_size;
+        loaderparams.ram_low_size = ram_low_size;
         loaderparams.kernel_filename = kernel_filename;
         loaderparams.kernel_cmdline = kernel_cmdline;
         loaderparams.initrd_filename = initrd_filename;
         kernel_entry = load_kernel();
 
-        write_bootloader(env, memory_region_get_ram_ptr(bios),
+        write_bootloader(memory_region_get_ram_ptr(bios),
                          bootloader_run_addr, kernel_entry);
         if (kvm_enabled()) {
             /* Write the bootloader code @ the end of RAM, 1MB reserved */
-            write_bootloader(env, memory_region_get_ram_ptr(ram_low_preio) +
+            write_bootloader(memory_region_get_ram_ptr(ram_low_preio) +
                                     ram_low_size,
                              bootloader_run_addr, kernel_entry);
         }
@@ -1120,7 +1168,7 @@
      * regions are not executable.
      */
     memory_region_init_ram(bios_copy, NULL, "bios.1fc", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     if (!rom_copy(memory_region_get_ram_ptr(bios_copy),
                   FLASH_ADDRESS, BIOS_SIZE)) {
         memcpy(memory_region_get_ram_ptr(bios_copy),
@@ -1132,10 +1180,6 @@
     /* Board ID = 0x420 (Malta Board with CoreLV) */
     stl_p(memory_region_get_ram_ptr(bios_copy) + 0x10, 0x00000420);
 
-    /* Init internal devices */
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
-
     /*
      * We have a circular dependency problem: pci_bus depends on isa_irq,
      * isa_irq is provided by i8259, i8259 depends on ISA, ISA depends
@@ -1155,27 +1199,25 @@
 
     /* Interrupt controller */
     /* The 8259 is attached to the MIPS CPU INT0 pin, ie interrupt 2 */
-    s->i8259 = i8259_init(isa_bus, env->irq[2]);
+    s->i8259 = i8259_init(isa_bus, i8259_irq);
 
     isa_bus_irqs(isa_bus, s->i8259);
     pci_piix4_ide_init(pci_bus, hd, piix4_devfn + 1);
     pci_create_simple(pci_bus, piix4_devfn + 2, "piix4-usb-uhci");
     smbus = piix4_pm_init(pci_bus, piix4_devfn + 3, 0x1100,
-                          isa_get_irq(NULL, 9), NULL, 0, NULL, NULL);
+                          isa_get_irq(NULL, 9), NULL, 0, NULL);
     smbus_eeprom_init(smbus, 8, smbus_eeprom_buf, smbus_eeprom_size);
     g_free(smbus_eeprom_buf);
     pit = pit_init(isa_bus, 0x40, 0, NULL);
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
-    DMA_init(0, cpu_exit_irq);
+    DMA_init(isa_bus, 0);
 
     /* Super I/O */
     isa_create_simple(isa_bus, "i8042");
 
     rtc_init(isa_bus, 2000, NULL);
-    serial_isa_init(isa_bus, 0, serial_hds[0]);
-    serial_isa_init(isa_bus, 1, serial_hds[1]);
-    if (parallel_hds[0])
-        parallel_init(isa_bus, 0, parallel_hds[0]);
+    serial_hds_isa_init(isa_bus, 2);
+    parallel_hds_isa_init(isa_bus, 1);
+
     for(i = 0; i < MAX_FD; i++) {
         fd[i] = drive_get(IF_FLOPPY, 0, i);
     }
@@ -1207,23 +1249,19 @@
     .class_init    = mips_malta_class_init,
 };
 
-static QEMUMachine mips_malta_machine = {
-    .name = "malta",
-    .desc = "MIPS Malta Core LV",
-    .init = mips_malta_init,
-    .max_cpus = 16,
-    .is_default = 1,
-};
+static void mips_malta_machine_init(MachineClass *mc)
+{
+    mc->desc = "MIPS Malta Core LV";
+    mc->init = mips_malta_init;
+    mc->max_cpus = 16;
+    mc->is_default = 1;
+}
+
+DEFINE_MACHINE("malta", mips_malta_machine_init)
 
 static void mips_malta_register_types(void)
 {
     type_register_static(&mips_malta_device);
 }
 
-static void mips_malta_machine_init(void)
-{
-    qemu_register_machine(&mips_malta_machine);
-}
-
 type_init(mips_malta_register_types)
-machine_init(mips_malta_machine_init);
diff --git a/hw/mips/mips_mipssim.c b/hw/mips/mips_mipssim.c
index 5d44c3f..1b91195 100644
--- a/hw/mips/mips_mipssim.c
+++ b/hw/mips/mips_mipssim.c
@@ -24,6 +24,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -69,7 +73,7 @@
     kernel_size = load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys,
                            NULL, (uint64_t *)&entry, NULL,
                            (uint64_t *)&kernel_high, big_endian,
-                           ELF_MACHINE, 1);
+                           EM_MIPS, 1, 0);
     if (kernel_size >= 0) {
         if ((entry & ~0x7fffffffULL) == 0x80000000)
             entry = (int32_t)entry;
@@ -171,11 +175,10 @@
     qemu_register_reset(main_cpu_reset, reset_info);
 
     /* Allocate RAM. */
-    memory_region_init_ram(ram, NULL, "mips_mipssim.ram", ram_size,
-                           &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "mips_mipssim.ram",
+                                         ram_size);
     memory_region_init_ram(bios, NULL, "mips_mipssim.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
     memory_region_set_readonly(bios, true);
 
@@ -213,8 +216,8 @@
     }
 
     /* Init CPU internal devices. */
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
+    cpu_mips_irq_init_cpu(cpu);
+    cpu_mips_clock_init(cpu);
 
     /* Register 64 KB of ISA IO space at 0x1fd00000. */
     memory_region_init_alias(isa, NULL, "isa_mmio",
@@ -232,15 +235,10 @@
         mipsnet_init(0x4200, env->irq[2], &nd_table[0]);
 }
 
-static QEMUMachine mips_mipssim_machine = {
-    .name = "mipssim",
-    .desc = "MIPS MIPSsim platform",
-    .init = mips_mipssim_init,
-};
-
-static void mips_mipssim_machine_init(void)
+static void mips_mipssim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_mipssim_machine);
+    mc->desc = "MIPS MIPSsim platform";
+    mc->init = mips_mipssim_init;
 }
 
-machine_init(mips_mipssim_machine_init);
+DEFINE_MACHINE("mipssim", mips_mipssim_machine_init)
diff --git a/hw/mips/mips_r4k.c b/hw/mips/mips_r4k.c
index a7fe0ce..16a59c7 100644
--- a/hw/mips/mips_r4k.c
+++ b/hw/mips/mips_r4k.c
@@ -7,6 +7,10 @@
  * All peripherial devices are attached to this "bus" with
  * the standard PC ISA addresses.
 */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/mips/mips.h"
 #include "hw/mips/cpudevs.h"
@@ -87,7 +91,7 @@
     kernel_size = load_elf(loaderparams.kernel_filename, cpu_mips_kseg0_to_phys,
                            NULL, (uint64_t *)&entry, NULL,
                            (uint64_t *)&kernel_high, big_endian,
-                           ELF_MACHINE, 1);
+                           EM_MIPS, 1, 0);
     if (kernel_size >= 0) {
         if ((entry & ~0x7fffffffULL) == 0x80000000)
             entry = (int32_t)entry;
@@ -139,6 +143,7 @@
     rom_add_blob_fixed("params", params_buf, params_size,
                        (16 << 20) - 264);
 
+    g_free(params_buf);
     return entry;
 }
 
@@ -165,7 +170,8 @@
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *bios;
     MemoryRegion *iomem = g_new(MemoryRegion, 1);
-    MemoryRegion *isa = g_new(MemoryRegion, 1);
+    MemoryRegion *isa_io = g_new(MemoryRegion, 1);
+    MemoryRegion *isa_mem = g_new(MemoryRegion, 1);
     int bios_size;
     MIPSCPU *cpu;
     CPUMIPSState *env;
@@ -204,8 +210,7 @@
                 ((unsigned int)ram_size / (1 << 20)));
         exit(1);
     }
-    memory_region_init_ram(ram, NULL, "mips_r4k.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "mips_r4k.ram", ram_size);
 
     memory_region_add_subregion(address_space_mem, 0, ram);
 
@@ -232,7 +237,7 @@
     if ((bios_size > 0) && (bios_size <= BIOS_SIZE)) {
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "mips_r4k.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
         memory_region_set_readonly(bios, true);
         memory_region_add_subregion(get_system_memory(), 0x1fc00000, bios);
@@ -251,9 +256,7 @@
         fprintf(stderr, "qemu: Warning, could not load MIPS bios '%s'\n",
 		bios_name);
     }
-    if (filename) {
-        g_free(filename);
-    }
+    g_free(filename);
 
     if (kernel_filename) {
         loaderparams.ram_size = ram_size;
@@ -264,30 +267,26 @@
     }
 
     /* Init CPU internal devices */
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
+    cpu_mips_irq_init_cpu(cpu);
+    cpu_mips_clock_init(cpu);
+
+    /* ISA bus: IO space at 0x14000000, mem space at 0x10000000 */
+    memory_region_init_alias(isa_io, NULL, "isa-io",
+                             get_system_io(), 0, 0x00010000);
+    memory_region_init(isa_mem, NULL, "isa-mem", 0x01000000);
+    memory_region_add_subregion(get_system_memory(), 0x14000000, isa_io);
+    memory_region_add_subregion(get_system_memory(), 0x10000000, isa_mem);
+    isa_bus = isa_bus_new(NULL, isa_mem, get_system_io(), &error_abort);
 
     /* The PIC is attached to the MIPS CPU INT0 pin */
-    isa_bus = isa_bus_new(NULL, get_system_io());
     i8259 = i8259_init(isa_bus, env->irq[2]);
     isa_bus_irqs(isa_bus, i8259);
 
     rtc_init(isa_bus, 2000, NULL);
 
-    /* Register 64 KB of ISA IO space at 0x14000000 */
-    memory_region_init_alias(isa, NULL, "isa_mmio",
-                             get_system_io(), 0, 0x00010000);
-    memory_region_add_subregion(get_system_memory(), 0x14000000, isa);
-
-    isa_mem_base = 0x10000000;
-
     pit = pit_init(isa_bus, 0x40, 0, NULL);
 
-    for(i = 0; i < MAX_SERIAL_PORTS; i++) {
-        if (serial_hds[i]) {
-            serial_isa_init(isa_bus, i, serial_hds[i]);
-        }
-    }
+    serial_hds_isa_init(isa_bus, MAX_SERIAL_PORTS);
 
     isa_vga_init(isa_bus);
 
@@ -303,15 +302,10 @@
     isa_create_simple(isa_bus, "i8042");
 }
 
-static QEMUMachine mips_machine = {
-    .name = "mips",
-    .desc = "mips r4k platform",
-    .init = mips_r4k_init,
-};
-
-static void mips_machine_init(void)
+static void mips_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&mips_machine);
+    mc->desc = "mips r4k platform";
+    mc->init = mips_r4k_init;
 }
 
-machine_init(mips_machine_init);
+DEFINE_MACHINE("mips", mips_machine_init)
diff --git a/hw/mips/mips_ranchu.c b/hw/mips/mips_ranchu.c
index 69763a2..75cabd6 100644
--- a/hw/mips/mips_ranchu.c
+++ b/hw/mips/mips_ranchu.c
@@ -1,3 +1,5 @@
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/devices.h"
 #include "hw/mips/mips.h"
@@ -18,10 +20,7 @@
 #include "elf.h"
 #include "hw/intc/goldfish_pic.h"
 #include "hw/irq.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/android.h"
-#endif  // CONFIG_ANDROID
+#include "qapi/error.h"
 
 #define PHYS_TO_VIRT(x) ((x) | ~(target_ulong)0x7fffffff)
 
@@ -58,16 +57,17 @@
 #define MAX_RAM_SIZE_MB 4079UL
 
 enum {
-    RANCHU_GF_PIC,
-    RANCHU_GF_TTY,
-    RANCHU_GF_TIMER,
-    RANCHU_GF_RTC,
-    RANCHU_GF_BATTERY,
-    RANCHU_GF_FB,
-    RANCHU_GF_EVDEV,
-    RANCHU_ANDROID_PIPE,
-    RANCHU_GF_AUDIO,
-    RANCHU_GF_RESET,
+    RANCHU_GOLDFISH_PIC,
+    RANCHU_GOLDFISH_TTY,
+    RANCHU_GOLDFISH_TIMER,
+    RANCHU_GOLDFISH_RTC,
+    RANCHU_GOLDFISH_BATTERY,
+    RANCHU_GOLDFISH_FB,
+    RANCHU_GOLDFISH_EVDEV,
+    RANCHU_GOLDFISH_PIPE,
+    RANCHU_GOLDFISH_AUDIO,
+    RANCHU_GOLDFISH_SYNC,
+    RANCHU_GOLDFISH_RESET,
     RANCHU_MMIO,
 };
 
@@ -81,31 +81,33 @@
 } DevMapEntry;
 
 static DevMapEntry devmap[] = {
-    [RANCHU_GF_PIC] =       { GOLDFISH_IO_SPACE, 0x1000, 0,
+    [RANCHU_GOLDFISH_PIC] =       { GOLDFISH_IO_SPACE, 0x1000, 0,
         NULL, "goldfish_pic", "generic,goldfish-pic" },
     /* ttyGF0 base address remains hardcoded in the kernel.
      * Early printing (prom_putchar()) relies on finding device
      * mapped on this address. DT can not be used at that early stage
      * for acquiring the base address of the device in the kernel.
      */
-    [RANCHU_GF_TTY] =       { GOLDFISH_IO_SPACE + 0x02000, 0x1000, 2,
+    [RANCHU_GOLDFISH_TTY] =       { GOLDFISH_IO_SPACE + 0x02000, 0x1000, 2,
         "goldfish_tty", "goldfish_tty", "generic,goldfish-tty" },
     /* repeats for a total of MAX_GF_TTYS */
-    [RANCHU_GF_TIMER] =     { GOLDFISH_IO_SPACE + 0x05000, 0x1000, 5,
+    [RANCHU_GOLDFISH_TIMER] =     { GOLDFISH_IO_SPACE + 0x05000, 0x1000, 5,
         "goldfish_timer", "goldfish_timer", "generic,goldfish-timer" },
-    [RANCHU_GF_RTC] =       { GOLDFISH_IO_SPACE + 0x06000, 0x1000, 6,
+    [RANCHU_GOLDFISH_RTC] =       { GOLDFISH_IO_SPACE + 0x06000, 0x1000, 6,
         "goldfish_rtc", "goldfish_rtc", "generic,goldfish-rtc" },
-    [RANCHU_GF_BATTERY] =   { GOLDFISH_IO_SPACE + 0x07000, 0x1000, 7,
+    [RANCHU_GOLDFISH_BATTERY] =   { GOLDFISH_IO_SPACE + 0x07000, 0x1000, 7,
         "goldfish_battery", "goldfish_battery", "generic,goldfish-battery" },
-    [RANCHU_GF_FB] =        { GOLDFISH_IO_SPACE + 0x08000, 0x0100, 8,
+    [RANCHU_GOLDFISH_FB] =        { GOLDFISH_IO_SPACE + 0x08000, 0x0100, 8,
         "goldfish_fb", "goldfish_fb", "generic,goldfish-fb" },
-    [RANCHU_GF_EVDEV] =     { GOLDFISH_IO_SPACE + 0x09000, 0x1000, 9,
+    [RANCHU_GOLDFISH_EVDEV] =     { GOLDFISH_IO_SPACE + 0x09000, 0x1000, 9,
         "goldfish-events", "goldfish_events", "generic,goldfish-events-keypad" },
-    [RANCHU_ANDROID_PIPE] = { GOLDFISH_IO_SPACE + 0x0A000, 0x2000, 10,
-        "android_pipe", "android_pipe", "generic,android-pipe" },
-    [RANCHU_GF_AUDIO] =     { GOLDFISH_IO_SPACE + 0x0C000, 0x0100, 11,
+    [RANCHU_GOLDFISH_PIPE] = { GOLDFISH_IO_SPACE + 0x0A000, 0x2000, 10,
+        "goldfish_pipe", "goldfish_pipe", "generic,android-pipe" },
+    [RANCHU_GOLDFISH_AUDIO] =     { GOLDFISH_IO_SPACE + 0x0C000, 0x0100, 11,
         "goldfish_audio", "goldfish_audio", "generic,goldfish-audio" },
-    [RANCHU_GF_RESET] =     { GOLDFISH_IO_SPACE + 0x0D000, 0x0100, 12,
+    [RANCHU_GOLDFISH_SYNC] =     { GOLDFISH_IO_SPACE + 0x0D000, 0x0100, 11,
+        "goldfish_sync", "goldfish_sync", "generic,goldfish-audio" },
+    [RANCHU_GOLDFISH_RESET] =     { GOLDFISH_IO_SPACE + 0x0D000, 0x0100, 12,
         "goldfish_reset", "goldfish_reset", "generic,goldfish-reset" },
     [RANCHU_MMIO] =         { GOLDFISH_IO_SPACE + 0x10000, 0x0200, 16,
         "virtio-mmio", "virtio_mmio", "virtio,mmio" },
@@ -214,7 +216,7 @@
 
     if (load_elf(rp->bootinfo.kernel_filename, cpu_mips_kseg0_to_phys, NULL,
         (uint64_t *)&kernel_entry, (uint64_t *)&kernel_low,
-        (uint64_t *)&kernel_high, 0, ELF_MACHINE, 1) < 0) {
+        (uint64_t *)&kernel_high, 0, EM_MIPS, 1, 0) < 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
                 rp->bootinfo.kernel_filename);
         exit(1);
@@ -388,7 +390,7 @@
     g_free(nodename);
 }
 
-static void ranchu_init(MachineState *machine)
+static void mips_ranchu_init(MachineState *machine)
 {
     MIPSCPU *cpu;
     CPUMIPSState *env;
@@ -424,14 +426,6 @@
 
     env = &cpu->env;
 
-    register_savevm(NULL,
-                    "cpu",
-                    0,
-                    MIPS_CPU_SAVE_VERSION,
-                    cpu_save,
-                    cpu_load,
-                    env);
-
     qemu_register_reset(main_cpu_reset, rp);
 
     if (ram_size > (MAX_RAM_SIZE_MB << 20)) {
@@ -453,7 +447,7 @@
     memory_region_add_subregion(get_system_memory(), 0, ram_lo);
 
     memory_region_init_io(iomem, NULL, &goldfish_reset_io_ops, NULL, "goldfish-reset", 0x0100);
-    memory_region_add_subregion(get_system_memory(), devmap[RANCHU_GF_RESET].base, iomem);
+    memory_region_add_subregion(get_system_memory(), devmap[RANCHU_GOLDFISH_RESET].base, iomem);
 
     memory_region_init_ram(bios, NULL, "ranchu.bios", RANCHU_BIOS_SIZE,
                            &error_abort);
@@ -478,20 +472,20 @@
     rp->fdt_size = fdt_size;
     rp->cpu = cpu;
 
-    cpu_mips_irq_init_cpu(env);
-    cpu_mips_clock_init(env);
+    cpu_mips_irq_init_cpu(cpu);
+    cpu_mips_clock_init(cpu);
 
     /* Initialize Goldfish PIC */
-    s->gfpic = goldfish_pic = goldfish_interrupt_init(devmap[RANCHU_GF_PIC].base,
+    s->gfpic = goldfish_pic = goldfish_interrupt_init(devmap[RANCHU_GOLDFISH_PIC].base,
                                             env->irq[2], env->irq[3]);
     /* Alocate dt handle (label) for interrupt-parent */
-    devmap[RANCHU_GF_PIC].irq = qemu_fdt_alloc_phandle(fdt);
+    devmap[RANCHU_GOLDFISH_PIC].irq = qemu_fdt_alloc_phandle(fdt);
 
     qemu_fdt_setprop_string(fdt, "/", "model", "ranchu");
     qemu_fdt_setprop_string(fdt, "/", "compatible", "mti,goldfish");
     qemu_fdt_setprop_cell(fdt, "/", "#address-cells", 0x1);
     qemu_fdt_setprop_cell(fdt, "/", "#size-cells", 0x2);
-    qemu_fdt_setprop_cell(fdt, "/", "interrupt-parent", devmap[RANCHU_GF_PIC].irq);
+    qemu_fdt_setprop_cell(fdt, "/", "interrupt-parent", devmap[RANCHU_GOLDFISH_PIC].irq);
 
     /* Firmware node */
     qemu_fdt_add_subnode(fdt, "/firmware");
@@ -518,17 +512,17 @@
     }
 
     /* Create goldfish_pic controller node in dt */
-    create_device(fdt, &devmap[RANCHU_GF_PIC], NULL, 1, 0);
+    create_device(fdt, &devmap[RANCHU_GOLDFISH_PIC], NULL, 1, 0);
 
     /* Create 4 Goldfish TTYs */
-    create_device(fdt, &devmap[RANCHU_GF_TTY], goldfish_pic, MAX_GF_TTYS, 0);
+    create_device(fdt, &devmap[RANCHU_GOLDFISH_TTY], goldfish_pic, MAX_GF_TTYS, 0);
 
     /* Other Goldfish Platform devices */
-    for (i = RANCHU_GF_AUDIO; i >= RANCHU_GF_TIMER ; i--) {
+    for (i = RANCHU_GOLDFISH_AUDIO; i >= RANCHU_GOLDFISH_TIMER ; i--) {
         create_device(fdt, &devmap[i], goldfish_pic, 1, 0);
     }
 
-    create_pm_device(fdt, &devmap[RANCHU_GF_RESET]);
+    create_pm_device(fdt, &devmap[RANCHU_GOLDFISH_RESET]);
 
     /* Virtio MMIO devices */
     create_device(fdt, &devmap[RANCHU_MMIO], goldfish_pic, VIRTIO_TRANSPORTS, 1);
@@ -554,28 +548,25 @@
 }
 
 static const TypeInfo mips_ranchu_device = {
-    .name          = TYPE_MIPS_RANCHU,
-    .parent        = TYPE_SYS_BUS_DEVICE,
+    .name = TYPE_MIPS_RANCHU,
+    .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(RanchuState),
-    .class_init    = mips_ranchu_class_init,
+    .class_init = mips_ranchu_class_init,
 };
 
-static QEMUMachine ranchu_machine = {
-    .name = "ranchu",
-    .desc = "Ranchu Virtual Machine for Android Emulator",
-    .init = ranchu_init,
-    .max_cpus = 1,
-};
+static void mips_ranchu_machine_init(MachineClass *mc)
+{
+    mc->desc = "Android/MIPS ranchu";
+    mc->init = mips_ranchu_init;
+    mc->max_cpus = 16;
+    mc->is_default = 1;
+}
+
+DEFINE_MACHINE("ranchu", mips_ranchu_machine_init)
 
 static void mips_ranchu_register_types(void)
 {
     type_register_static(&mips_ranchu_device);
 }
 
-static void ranchu_machine_init(void)
-{
-    qemu_register_machine(&ranchu_machine);
-}
-
 type_init(mips_ranchu_register_types)
-machine_init(ranchu_machine_init);
diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs
index 97296e3..19c1e2b 100644
--- a/hw/misc/Makefile.objs
+++ b/hw/misc/Makefile.objs
@@ -15,27 +15,23 @@
 common-obj-$(CONFIG_ARM11SCU) += arm11scu.o
 
 obj-$(CONFIG_ANDROID_PIPE) += android_pipe.o
-obj-$(CONFIG_ANDROID_PIPE) += android_qemud.o
-obj-$(CONFIG_ANDROID_PIPE) += android_boot_properties.o
-
-obj-$(CONFIG_GOLDFISH) += goldfish_sync.o
 
 # PKUnity SoC devices
 common-obj-$(CONFIG_PUV3) += puv3_pm.o
 
 common-obj-$(CONFIG_MACIO) += macio/
 
-ifeq ($(CONFIG_PCI), y)
-obj-$(CONFIG_KVM) += ivshmem.o
-obj-$(CONFIG_LINUX) += vfio.o
-endif
+obj-$(CONFIG_IVSHMEM) += ivshmem.o
 
 obj-$(CONFIG_REALVIEW) += arm_sysctl.o
 obj-$(CONFIG_NSERIES) += cbus.o
 obj-$(CONFIG_ECCMEMCTL) += eccmemctl.o
 obj-$(CONFIG_EXYNOS4) += exynos4210_pmu.o
-obj-$(CONFIG_GOLDFISH) += goldfish_battery.o
 obj-$(CONFIG_IMX) += imx_ccm.o
+obj-$(CONFIG_IMX) += imx31_ccm.o
+obj-$(CONFIG_IMX) += imx25_ccm.o
+obj-$(CONFIG_IMX) += imx6_ccm.o
+obj-$(CONFIG_IMX) += imx6_src.o
 obj-$(CONFIG_MILKYMIST) += milkymist-hpdmc.o
 obj-$(CONFIG_MILKYMIST) += milkymist-pfpu.o
 obj-$(CONFIG_MAINSTONE) += mst_fpga.o
@@ -44,7 +40,21 @@
 obj-$(CONFIG_OMAP) += omap_l4.o
 obj-$(CONFIG_OMAP) += omap_sdrc.o
 obj-$(CONFIG_OMAP) += omap_tap.o
+obj-$(CONFIG_RASPI) += bcm2835_mbox.o
+obj-$(CONFIG_RASPI) += bcm2835_property.o
 obj-$(CONFIG_SLAVIO) += slavio_misc.o
 obj-$(CONFIG_ZYNQ) += zynq_slcr.o
+obj-$(CONFIG_ZYNQ) += zynq-xadc.o
+obj-$(CONFIG_STM32F2XX_SYSCFG) += stm32f2xx_syscfg.o
+obj-$(CONFIG_MIPS_CPS) += mips_cmgcr.o
+obj-$(CONFIG_MIPS_CPS) += mips_cpc.o
+obj-$(CONFIG_MIPS_ITU) += mips_itu.o
 
 obj-$(CONFIG_PVPANIC) += pvpanic.o
+obj-$(CONFIG_EDU) += edu.o
+obj-$(CONFIG_HYPERV_TESTDEV) += hyperv_testdev.o
+obj-$(CONFIG_AUX) += auxbus.o
+obj-$(CONFIG_ASPEED_SOC) += aspeed_scu.o
+obj-$(CONFIG_GOLDFISH) += goldfish_battery.o
+obj-$(CONFIG_GOLDFISH) += goldfish_pipe.o
+obj-$(CONFIG_GOLDFISH) += goldfish_sync.o
diff --git a/hw/misc/a9scu.c b/hw/misc/a9scu.c
index 4434945..3e8ad8c 100644
--- a/hw/misc/a9scu.c
+++ b/hw/misc/a9scu.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/misc/a9scu.h"
 
 static uint64_t a9_scu_read(void *opaque, hwaddr offset,
diff --git a/hw/misc/applesmc.c b/hw/misc/applesmc.c
index 6a56b07..77fab5b 100644
--- a/hw/misc/applesmc.c
+++ b/hw/misc/applesmc.c
@@ -30,6 +30,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "ui/console.h"
@@ -43,7 +44,6 @@
 /* command/status port used by Apple SMC */
 #define APPLESMC_CMD_PORT              0x4
 #define APPLESMC_NR_PORTS              32
-#define APPLESMC_MAX_DATA_LENGTH       32
 
 #define APPLESMC_READ_CMD              0x10
 #define APPLESMC_WRITE_CMD             0x11
@@ -249,8 +249,8 @@
 }
 
 static Property applesmc_isa_properties[] = {
-    DEFINE_PROP_UINT32("iobase", AppleSMCState, iobase,
-                      APPLESMC_DEFAULT_IOBASE),
+    DEFINE_PROP_UINT32(APPLESMC_PROP_IO_BASE, AppleSMCState, iobase,
+                       APPLESMC_DEFAULT_IOBASE),
     DEFINE_PROP_STRING("osk", AppleSMCState, osk),
     DEFINE_PROP_END_OF_LIST(),
 };
diff --git a/hw/misc/arm11scu.c b/hw/misc/arm11scu.c
index a791675..7042ce1 100644
--- a/hw/misc/arm11scu.c
+++ b/hw/misc/arm11scu.c
@@ -8,7 +8,9 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/misc/arm11scu.h"
+#include "qemu/log.h"
 
 static uint64_t mpcore_scu_read(void *opaque, hwaddr offset,
                                 unsigned size)
diff --git a/hw/misc/arm_integrator_debug.c b/hw/misc/arm_integrator_debug.c
index 99b720f..8a5f295 100644
--- a/hw/misc/arm_integrator_debug.c
+++ b/hw/misc/arm_integrator_debug.c
@@ -14,10 +14,12 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "exec/address-spaces.h"
 #include "hw/misc/arm_integrator_debug.h"
+#include "qemu/log.h"
 
 #define INTEGRATOR_DEBUG(obj) \
     OBJECT_CHECK(IntegratorDebugState, (obj), TYPE_INTEGRATOR_DEBUG)
@@ -79,7 +81,7 @@
     SysBusDevice *sd = SYS_BUS_DEVICE(obj);
     IntegratorDebugState *s = INTEGRATOR_DEBUG(obj);
 
-    memory_region_init_io(&s->iomem, NULL, &intdbg_control_ops,
+    memory_region_init_io(&s->iomem, obj, &intdbg_control_ops,
                           NULL, "dbg-leds", 0x1000000);
     sysbus_init_mmio(sd, &s->iomem);
 }
diff --git a/hw/misc/arm_l2x0.c b/hw/misc/arm_l2x0.c
index 9e220c9..66a0787 100644
--- a/hw/misc/arm_l2x0.c
+++ b/hw/misc/arm_l2x0.c
@@ -18,7 +18,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qemu/log.h"
 
 /* L2C-310 r3p2 */
 #define CACHE_ID 0x410000c8
@@ -157,14 +159,14 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
  };
 
-static int l2x0_priv_init(SysBusDevice *dev)
+static void l2x0_priv_init(Object *obj)
 {
-    L2x0State *s = ARM_L2X0(dev);
+    L2x0State *s = ARM_L2X0(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(dev), &l2x0_mem_ops, s,
+    memory_region_init_io(&s->iomem, obj, &l2x0_mem_ops, s,
                           "l2x0_cc", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
-    return 0;
 }
 
 static Property l2x0_properties[] = {
@@ -174,10 +176,8 @@
 
 static void l2x0_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = l2x0_priv_init;
     dc->vmsd = &vmstate_l2x0;
     dc->props = l2x0_properties;
     dc->reset = l2x0_priv_reset;
@@ -187,6 +187,7 @@
     .name = TYPE_ARM_L2X0,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(L2x0State),
+    .instance_init = l2x0_priv_init,
     .class_init = l2x0_class_init,
 };
 
diff --git a/hw/misc/arm_sysctl.c b/hw/misc/arm_sysctl.c
index 3fad6f8..8524008 100644
--- a/hw/misc/arm_sysctl.c
+++ b/hw/misc/arm_sysctl.c
@@ -7,12 +7,14 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
 #include "hw/sysbus.h"
 #include "hw/arm/primecell.h"
 #include "sysemu/sysemu.h"
+#include "qemu/log.h"
 
 #define LOCK_VALUE 0xa05f
 
@@ -170,7 +172,8 @@
     case 0x58: /* BOOTCS */
         return 0;
     case 0x5c: /* 24MHz */
-        return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24000000, get_ticks_per_sec());
+        return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 24000000,
+                        NANOSECONDS_PER_SECOND);
     case 0x60: /* MISC */
         return 0;
     case 0x84: /* PROCID0 */
diff --git a/hw/misc/aspeed_scu.c b/hw/misc/aspeed_scu.c
new file mode 100644
index 0000000..c7e2c82
--- /dev/null
+++ b/hw/misc/aspeed_scu.c
@@ -0,0 +1,282 @@
+/*
+ * ASPEED System Control Unit
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/aspeed_scu.h"
+#include "hw/qdev-properties.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+#include "qemu/bitops.h"
+#include "qemu/log.h"
+#include "trace.h"
+
+#define TO_REG(offset) ((offset) >> 2)
+
+#define PROT_KEY             TO_REG(0x00)
+#define SYS_RST_CTRL         TO_REG(0x04)
+#define CLK_SEL              TO_REG(0x08)
+#define CLK_STOP_CTRL        TO_REG(0x0C)
+#define FREQ_CNTR_CTRL       TO_REG(0x10)
+#define FREQ_CNTR_EVAL       TO_REG(0x14)
+#define IRQ_CTRL             TO_REG(0x18)
+#define D2PLL_PARAM          TO_REG(0x1C)
+#define MPLL_PARAM           TO_REG(0x20)
+#define HPLL_PARAM           TO_REG(0x24)
+#define FREQ_CNTR_RANGE      TO_REG(0x28)
+#define MISC_CTRL1           TO_REG(0x2C)
+#define PCI_CTRL1            TO_REG(0x30)
+#define PCI_CTRL2            TO_REG(0x34)
+#define PCI_CTRL3            TO_REG(0x38)
+#define SYS_RST_STATUS       TO_REG(0x3C)
+#define SOC_SCRATCH1         TO_REG(0x40)
+#define SOC_SCRATCH2         TO_REG(0x44)
+#define MAC_CLK_DELAY        TO_REG(0x48)
+#define MISC_CTRL2           TO_REG(0x4C)
+#define VGA_SCRATCH1         TO_REG(0x50)
+#define VGA_SCRATCH2         TO_REG(0x54)
+#define VGA_SCRATCH3         TO_REG(0x58)
+#define VGA_SCRATCH4         TO_REG(0x5C)
+#define VGA_SCRATCH5         TO_REG(0x60)
+#define VGA_SCRATCH6         TO_REG(0x64)
+#define VGA_SCRATCH7         TO_REG(0x68)
+#define VGA_SCRATCH8         TO_REG(0x6C)
+#define HW_STRAP1            TO_REG(0x70)
+#define RNG_CTRL             TO_REG(0x74)
+#define RNG_DATA             TO_REG(0x78)
+#define SILICON_REV          TO_REG(0x7C)
+#define PINMUX_CTRL1         TO_REG(0x80)
+#define PINMUX_CTRL2         TO_REG(0x84)
+#define PINMUX_CTRL3         TO_REG(0x88)
+#define PINMUX_CTRL4         TO_REG(0x8C)
+#define PINMUX_CTRL5         TO_REG(0x90)
+#define PINMUX_CTRL6         TO_REG(0x94)
+#define WDT_RST_CTRL         TO_REG(0x9C)
+#define PINMUX_CTRL7         TO_REG(0xA0)
+#define PINMUX_CTRL8         TO_REG(0xA4)
+#define PINMUX_CTRL9         TO_REG(0xA8)
+#define WAKEUP_EN            TO_REG(0xC0)
+#define WAKEUP_CTRL          TO_REG(0xC4)
+#define HW_STRAP2            TO_REG(0xD0)
+#define FREE_CNTR4           TO_REG(0xE0)
+#define FREE_CNTR4_EXT       TO_REG(0xE4)
+#define CPU2_CTRL            TO_REG(0x100)
+#define CPU2_BASE_SEG1       TO_REG(0x104)
+#define CPU2_BASE_SEG2       TO_REG(0x108)
+#define CPU2_BASE_SEG3       TO_REG(0x10C)
+#define CPU2_BASE_SEG4       TO_REG(0x110)
+#define CPU2_BASE_SEG5       TO_REG(0x114)
+#define CPU2_CACHE_CTRL      TO_REG(0x118)
+#define UART_HPLL_CLK        TO_REG(0x160)
+#define PCIE_CTRL            TO_REG(0x180)
+#define BMC_MMIO_CTRL        TO_REG(0x184)
+#define RELOC_DECODE_BASE1   TO_REG(0x188)
+#define RELOC_DECODE_BASE2   TO_REG(0x18C)
+#define MAILBOX_DECODE_BASE  TO_REG(0x190)
+#define SRAM_DECODE_BASE1    TO_REG(0x194)
+#define SRAM_DECODE_BASE2    TO_REG(0x198)
+#define BMC_REV              TO_REG(0x19C)
+#define BMC_DEV_ID           TO_REG(0x1A4)
+
+#define PROT_KEY_UNLOCK 0x1688A8A8
+#define SCU_IO_REGION_SIZE 0x20000
+
+static const uint32_t ast2400_a0_resets[ASPEED_SCU_NR_REGS] = {
+     [SYS_RST_CTRL]    = 0xFFCFFEDCU,
+     [CLK_SEL]         = 0xF3F40000U,
+     [CLK_STOP_CTRL]   = 0x19FC3E8BU,
+     [D2PLL_PARAM]     = 0x00026108U,
+     [MPLL_PARAM]      = 0x00030291U,
+     [HPLL_PARAM]      = 0x00000291U,
+     [MISC_CTRL1]      = 0x00000010U,
+     [PCI_CTRL1]       = 0x20001A03U,
+     [PCI_CTRL2]       = 0x20001A03U,
+     [PCI_CTRL3]       = 0x04000030U,
+     [SYS_RST_STATUS]  = 0x00000001U,
+     [SOC_SCRATCH1]    = 0x000000C0U, /* SoC completed DRAM init */
+     [MISC_CTRL2]      = 0x00000023U,
+     [RNG_CTRL]        = 0x0000000EU,
+     [PINMUX_CTRL2]    = 0x0000F000U,
+     [PINMUX_CTRL3]    = 0x01000000U,
+     [PINMUX_CTRL4]    = 0x000000FFU,
+     [PINMUX_CTRL5]    = 0x0000A000U,
+     [WDT_RST_CTRL]    = 0x003FFFF3U,
+     [PINMUX_CTRL8]    = 0xFFFF0000U,
+     [PINMUX_CTRL9]    = 0x000FFFFFU,
+     [FREE_CNTR4]      = 0x000000FFU,
+     [FREE_CNTR4_EXT]  = 0x000000FFU,
+     [CPU2_BASE_SEG1]  = 0x80000000U,
+     [CPU2_BASE_SEG4]  = 0x1E600000U,
+     [CPU2_BASE_SEG5]  = 0xC0000000U,
+     [UART_HPLL_CLK]   = 0x00001903U,
+     [PCIE_CTRL]       = 0x0000007BU,
+     [BMC_DEV_ID]      = 0x00002402U
+};
+
+static uint64_t aspeed_scu_read(void *opaque, hwaddr offset, unsigned size)
+{
+    AspeedSCUState *s = ASPEED_SCU(opaque);
+    int reg = TO_REG(offset);
+
+    if (reg >= ARRAY_SIZE(s->regs)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Out-of-bounds read at offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    switch (reg) {
+    case WAKEUP_EN:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Read of write-only offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        break;
+    }
+
+    return s->regs[reg];
+}
+
+static void aspeed_scu_write(void *opaque, hwaddr offset, uint64_t data,
+                             unsigned size)
+{
+    AspeedSCUState *s = ASPEED_SCU(opaque);
+    int reg = TO_REG(offset);
+
+    if (reg >= ARRAY_SIZE(s->regs)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Out-of-bounds write at offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        return;
+    }
+
+    if (reg > PROT_KEY && reg < CPU2_BASE_SEG1 &&
+            s->regs[PROT_KEY] != PROT_KEY_UNLOCK) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: SCU is locked!\n", __func__);
+        return;
+    }
+
+    trace_aspeed_scu_write(offset, size, data);
+
+    switch (reg) {
+    case FREQ_CNTR_EVAL:
+    case VGA_SCRATCH1 ... VGA_SCRATCH8:
+    case RNG_DATA:
+    case SILICON_REV:
+    case FREE_CNTR4:
+    case FREE_CNTR4_EXT:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Write to read-only offset 0x%" HWADDR_PRIx "\n",
+                      __func__, offset);
+        return;
+    }
+
+    s->regs[reg] = data;
+}
+
+static const MemoryRegionOps aspeed_scu_ops = {
+    .read = aspeed_scu_read,
+    .write = aspeed_scu_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .valid.unaligned = false,
+};
+
+static void aspeed_scu_reset(DeviceState *dev)
+{
+    AspeedSCUState *s = ASPEED_SCU(dev);
+    const uint32_t *reset;
+
+    switch (s->silicon_rev) {
+    case AST2400_A0_SILICON_REV:
+        reset = ast2400_a0_resets;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    memcpy(s->regs, reset, sizeof(s->regs));
+    s->regs[SILICON_REV] = s->silicon_rev;
+    s->regs[HW_STRAP1] = s->hw_strap1;
+    s->regs[HW_STRAP2] = s->hw_strap2;
+}
+
+static uint32_t aspeed_silicon_revs[] = { AST2400_A0_SILICON_REV, };
+
+bool is_supported_silicon_rev(uint32_t silicon_rev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(aspeed_silicon_revs); i++) {
+        if (silicon_rev == aspeed_silicon_revs[i]) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void aspeed_scu_realize(DeviceState *dev, Error **errp)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedSCUState *s = ASPEED_SCU(dev);
+
+    if (!is_supported_silicon_rev(s->silicon_rev)) {
+        error_setg(errp, "Unknown silicon revision: 0x%" PRIx32,
+                s->silicon_rev);
+        return;
+    }
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &aspeed_scu_ops, s,
+                          TYPE_ASPEED_SCU, SCU_IO_REGION_SIZE);
+
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static const VMStateDescription vmstate_aspeed_scu = {
+    .name = "aspeed.scu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, AspeedSCUState, ASPEED_SCU_NR_REGS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property aspeed_scu_properties[] = {
+    DEFINE_PROP_UINT32("silicon-rev", AspeedSCUState, silicon_rev, 0),
+    DEFINE_PROP_UINT32("hw-strap1", AspeedSCUState, hw_strap1, 0),
+    DEFINE_PROP_UINT32("hw-strap2", AspeedSCUState, hw_strap2, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void aspeed_scu_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->realize = aspeed_scu_realize;
+    dc->reset = aspeed_scu_reset;
+    dc->desc = "ASPEED System Control Unit";
+    dc->vmsd = &vmstate_aspeed_scu;
+    dc->props = aspeed_scu_properties;
+}
+
+static const TypeInfo aspeed_scu_info = {
+    .name = TYPE_ASPEED_SCU,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AspeedSCUState),
+    .class_init = aspeed_scu_class_init,
+};
+
+static void aspeed_scu_register_types(void)
+{
+    type_register_static(&aspeed_scu_info);
+}
+
+type_init(aspeed_scu_register_types);
diff --git a/hw/misc/auxbus.c b/hw/misc/auxbus.c
new file mode 100644
index 0000000..e4a7ba4
--- /dev/null
+++ b/hw/misc/auxbus.c
@@ -0,0 +1,292 @@
+/*
+ * auxbus.c
+ *
+ *  Copyright 2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option)any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * This is an implementation of the AUX bus for VESA Display Port v1.1a.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "hw/misc/auxbus.h"
+#include "hw/i2c/i2c.h"
+#include "monitor/monitor.h"
+
+#ifndef DEBUG_AUX
+#define DEBUG_AUX 0
+#endif
+
+#define DPRINTF(fmt, ...) do {                                                 \
+    if (DEBUG_AUX) {                                                           \
+        qemu_log("aux: " fmt , ## __VA_ARGS__);                                \
+    }                                                                          \
+} while (0);
+
+#define TYPE_AUXTOI2C "aux-to-i2c-bridge"
+#define AUXTOI2C(obj) OBJECT_CHECK(AUXTOI2CState, (obj), TYPE_AUXTOI2C)
+
+static void aux_slave_dev_print(Monitor *mon, DeviceState *dev, int indent);
+static inline I2CBus *aux_bridge_get_i2c_bus(AUXTOI2CState *bridge);
+
+/* aux-bus implementation (internal not public) */
+static void aux_bus_class_init(ObjectClass *klass, void *data)
+{
+    BusClass *k = BUS_CLASS(klass);
+
+    /* AUXSlave has an MMIO so we need to change the way we print information
+     * in monitor.
+     */
+    k->print_dev = aux_slave_dev_print;
+}
+
+AUXBus *aux_init_bus(DeviceState *parent, const char *name)
+{
+    AUXBus *bus;
+
+    bus = AUX_BUS(qbus_create(TYPE_AUX_BUS, parent, name));
+    bus->bridge = AUXTOI2C(qdev_create(BUS(bus), TYPE_AUXTOI2C));
+
+    /* Memory related. */
+    bus->aux_io = g_malloc(sizeof(*bus->aux_io));
+    memory_region_init(bus->aux_io, OBJECT(bus), "aux-io", (1 << 20));
+    address_space_init(&bus->aux_addr_space, bus->aux_io, "aux-io");
+    return bus;
+}
+
+static void aux_bus_map_device(AUXBus *bus, AUXSlave *dev, hwaddr addr)
+{
+    memory_region_add_subregion(bus->aux_io, addr, dev->mmio);
+}
+
+static bool aux_bus_is_bridge(AUXBus *bus, DeviceState *dev)
+{
+    return (dev == DEVICE(bus->bridge));
+}
+
+I2CBus *aux_get_i2c_bus(AUXBus *bus)
+{
+    return aux_bridge_get_i2c_bus(bus->bridge);
+}
+
+AUXReply aux_request(AUXBus *bus, AUXCommand cmd, uint32_t address,
+                      uint8_t len, uint8_t *data)
+{
+    AUXReply ret = AUX_NACK;
+    I2CBus *i2c_bus = aux_get_i2c_bus(bus);
+    size_t i;
+    bool is_write = false;
+
+    DPRINTF("request at address 0x%" PRIX32 ", command %u, len %u\n", address,
+            cmd, len);
+
+    switch (cmd) {
+    /*
+     * Forward the request on the AUX bus..
+     */
+    case WRITE_AUX:
+    case READ_AUX:
+        is_write = cmd == READ_AUX ? false : true;
+        for (i = 0; i < len; i++) {
+            if (!address_space_rw(&bus->aux_addr_space, address++,
+                                  MEMTXATTRS_UNSPECIFIED, data++, 1,
+                                  is_write)) {
+                ret = AUX_I2C_ACK;
+            } else {
+                ret = AUX_NACK;
+                break;
+            }
+        }
+        break;
+    /*
+     * Classic I2C transactions..
+     */
+    case READ_I2C:
+    case WRITE_I2C:
+        is_write = cmd == READ_I2C ? false : true;
+        if (i2c_bus_busy(i2c_bus)) {
+            i2c_end_transfer(i2c_bus);
+        }
+
+        if (i2c_start_transfer(i2c_bus, address, is_write)) {
+            ret = AUX_I2C_NACK;
+            break;
+        }
+
+        ret = AUX_I2C_ACK;
+        while (len > 0) {
+            if (i2c_send_recv(i2c_bus, data++, is_write) < 0) {
+                ret = AUX_I2C_NACK;
+                break;
+            }
+            len--;
+        }
+        i2c_end_transfer(i2c_bus);
+        break;
+    /*
+     * I2C MOT transactions.
+     *
+     * Here we send a start when:
+     *  - We didn't start transaction yet.
+     *  - We had a READ and we do a WRITE.
+     *  - We changed the address.
+     */
+    case WRITE_I2C_MOT:
+    case READ_I2C_MOT:
+        is_write = cmd == READ_I2C_MOT ? false : true;
+        ret = AUX_I2C_NACK;
+        if (!i2c_bus_busy(i2c_bus)) {
+            /*
+             * No transactions started..
+             */
+            if (i2c_start_transfer(i2c_bus, address, is_write)) {
+                break;
+            }
+        } else if ((address != bus->last_i2c_address) ||
+                   (bus->last_transaction != cmd)) {
+            /*
+             * Transaction started but we need to restart..
+             */
+            i2c_end_transfer(i2c_bus);
+            if (i2c_start_transfer(i2c_bus, address, is_write)) {
+                break;
+            }
+        }
+
+        bus->last_transaction = cmd;
+        bus->last_i2c_address = address;
+        while (len > 0) {
+            if (i2c_send_recv(i2c_bus, data++, is_write) < 0) {
+                i2c_end_transfer(i2c_bus);
+                break;
+            }
+            len--;
+        }
+        if (len == 0) {
+            ret = AUX_I2C_ACK;
+        }
+        break;
+    default:
+        DPRINTF("Not implemented!\n");
+        return AUX_NACK;
+    }
+
+    DPRINTF("reply: %u\n", ret);
+    return ret;
+}
+
+static const TypeInfo aux_bus_info = {
+    .name = TYPE_AUX_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(AUXBus),
+    .class_init = aux_bus_class_init
+};
+
+/* aux-i2c implementation (internal not public) */
+struct AUXTOI2CState {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    I2CBus *i2c_bus;
+};
+
+static void aux_bridge_init(Object *obj)
+{
+    AUXTOI2CState *s = AUXTOI2C(obj);
+
+    s->i2c_bus = i2c_init_bus(DEVICE(obj), "aux-i2c");
+}
+
+static inline I2CBus *aux_bridge_get_i2c_bus(AUXTOI2CState *bridge)
+{
+    return bridge->i2c_bus;
+}
+
+static const TypeInfo aux_to_i2c_type_info = {
+    .name = TYPE_AUXTOI2C,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(AUXTOI2CState),
+    .instance_init = aux_bridge_init
+};
+
+/* aux-slave implementation */
+static void aux_slave_dev_print(Monitor *mon, DeviceState *dev, int indent)
+{
+    AUXBus *bus = AUX_BUS(qdev_get_parent_bus(dev));
+    AUXSlave *s;
+
+    /* Don't print anything if the device is I2C "bridge". */
+    if (aux_bus_is_bridge(bus, dev)) {
+        return;
+    }
+
+    s = AUX_SLAVE(dev);
+
+    monitor_printf(mon, "%*smemory " TARGET_FMT_plx "/" TARGET_FMT_plx "\n",
+                   indent, "",
+                   object_property_get_int(OBJECT(s->mmio), "addr", NULL),
+                   memory_region_size(s->mmio));
+}
+
+DeviceState *aux_create_slave(AUXBus *bus, const char *type, uint32_t addr)
+{
+    DeviceState *dev;
+
+    dev = DEVICE(object_new(type));
+    assert(dev);
+    qdev_set_parent_bus(dev, &bus->qbus);
+    qdev_init_nofail(dev);
+    aux_bus_map_device(AUX_BUS(qdev_get_parent_bus(dev)), AUX_SLAVE(dev), addr);
+    return dev;
+}
+
+void aux_init_mmio(AUXSlave *aux_slave, MemoryRegion *mmio)
+{
+    assert(!aux_slave->mmio);
+    aux_slave->mmio = mmio;
+}
+
+static void aux_slave_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *k = DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_MISC, k->categories);
+    k->bus_type = TYPE_AUX_BUS;
+}
+
+static const TypeInfo aux_slave_type_info = {
+    .name = TYPE_AUX_SLAVE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(AUXSlave),
+    .abstract = true,
+    .class_init = aux_slave_class_init,
+};
+
+static void aux_register_types(void)
+{
+    type_register_static(&aux_bus_info);
+    type_register_static(&aux_slave_type_info);
+    type_register_static(&aux_to_i2c_type_info);
+}
+
+type_init(aux_register_types)
diff --git a/hw/misc/bcm2835_mbox.c b/hw/misc/bcm2835_mbox.c
new file mode 100644
index 0000000..e97cc81
--- /dev/null
+++ b/hw/misc/bcm2835_mbox.c
@@ -0,0 +1,336 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ *
+ * This file models the system mailboxes, which are used for
+ * communication with low-bandwidth GPU peripherals. Refs:
+ *   https://github.com/raspberrypi/firmware/wiki/Mailboxes
+ *   https://github.com/raspberrypi/firmware/wiki/Accessing-mailboxes
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/misc/bcm2835_mbox.h"
+#include "qemu/log.h"
+
+#define MAIL0_PEEK   0x90
+#define MAIL0_SENDER 0x94
+#define MAIL1_STATUS 0xb8
+
+/* Mailbox status register */
+#define MAIL0_STATUS 0x98
+#define ARM_MS_FULL       0x80000000
+#define ARM_MS_EMPTY      0x40000000
+#define ARM_MS_LEVEL      0x400000FF /* Max. value depends on mailbox depth */
+
+/* MAILBOX config/status register */
+#define MAIL0_CONFIG 0x9c
+/* ANY write to this register clears the error bits! */
+#define ARM_MC_IHAVEDATAIRQEN    0x00000001 /* mbox irq enable:  has data */
+#define ARM_MC_IHAVESPACEIRQEN   0x00000002 /* mbox irq enable:  has space */
+#define ARM_MC_OPPISEMPTYIRQEN   0x00000004 /* mbox irq enable: Opp is empty */
+#define ARM_MC_MAIL_CLEAR        0x00000008 /* mbox clear write 1, then  0 */
+#define ARM_MC_IHAVEDATAIRQPEND  0x00000010 /* mbox irq pending:  has space */
+#define ARM_MC_IHAVESPACEIRQPEND 0x00000020 /* mbox irq pending: Opp is empty */
+#define ARM_MC_OPPISEMPTYIRQPEND 0x00000040 /* mbox irq pending */
+/* Bit 7 is unused */
+#define ARM_MC_ERRNOOWN   0x00000100 /* error : none owner read from mailbox */
+#define ARM_MC_ERROVERFLW 0x00000200 /* error : write to fill mailbox */
+#define ARM_MC_ERRUNDRFLW 0x00000400 /* error : read from empty mailbox */
+
+static void mbox_update_status(BCM2835Mbox *mb)
+{
+    mb->status &= ~(ARM_MS_EMPTY | ARM_MS_FULL);
+    if (mb->count == 0) {
+        mb->status |= ARM_MS_EMPTY;
+    } else if (mb->count == MBOX_SIZE) {
+        mb->status |= ARM_MS_FULL;
+    }
+}
+
+static void mbox_reset(BCM2835Mbox *mb)
+{
+    int n;
+
+    mb->count = 0;
+    mb->config = 0;
+    for (n = 0; n < MBOX_SIZE; n++) {
+        mb->reg[n] = MBOX_INVALID_DATA;
+    }
+    mbox_update_status(mb);
+}
+
+static uint32_t mbox_pull(BCM2835Mbox *mb, int index)
+{
+    int n;
+    uint32_t val;
+
+    assert(mb->count > 0);
+    assert(index < mb->count);
+
+    val = mb->reg[index];
+    for (n = index + 1; n < mb->count; n++) {
+        mb->reg[n - 1] = mb->reg[n];
+    }
+    mb->count--;
+    mb->reg[mb->count] = MBOX_INVALID_DATA;
+
+    mbox_update_status(mb);
+
+    return val;
+}
+
+static void mbox_push(BCM2835Mbox *mb, uint32_t val)
+{
+    assert(mb->count < MBOX_SIZE);
+    mb->reg[mb->count++] = val;
+    mbox_update_status(mb);
+}
+
+static void bcm2835_mbox_update(BCM2835MboxState *s)
+{
+    uint32_t value;
+    bool set;
+    int n;
+
+    s->mbox_irq_disabled = true;
+
+    /* Get pending responses and put them in the vc->arm mbox,
+     * as long as it's not full
+     */
+    for (n = 0; n < MBOX_CHAN_COUNT; n++) {
+        while (s->available[n] && !(s->mbox[0].status & ARM_MS_FULL)) {
+            value = ldl_le_phys(&s->mbox_as, n << MBOX_AS_CHAN_SHIFT);
+            assert(value != MBOX_INVALID_DATA); /* Pending interrupt but no data */
+            mbox_push(&s->mbox[0], value);
+        }
+    }
+
+    /* TODO (?): Try to push pending requests from the arm->vc mbox */
+
+    /* Re-enable calls from the IRQ routine */
+    s->mbox_irq_disabled = false;
+
+    /* Update ARM IRQ status */
+    set = false;
+    s->mbox[0].config &= ~ARM_MC_IHAVEDATAIRQPEND;
+    if (!(s->mbox[0].status & ARM_MS_EMPTY)) {
+        s->mbox[0].config |= ARM_MC_IHAVEDATAIRQPEND;
+        if (s->mbox[0].config & ARM_MC_IHAVEDATAIRQEN) {
+            set = true;
+        }
+    }
+    qemu_set_irq(s->arm_irq, set);
+}
+
+static void bcm2835_mbox_set_irq(void *opaque, int irq, int level)
+{
+    BCM2835MboxState *s = opaque;
+
+    s->available[irq] = level;
+
+    /* avoid recursively calling bcm2835_mbox_update when the interrupt
+     * status changes due to the ldl_phys call within that function
+     */
+    if (!s->mbox_irq_disabled) {
+        bcm2835_mbox_update(s);
+    }
+}
+
+static uint64_t bcm2835_mbox_read(void *opaque, hwaddr offset, unsigned size)
+{
+    BCM2835MboxState *s = opaque;
+    uint32_t res = 0;
+
+    offset &= 0xff;
+
+    switch (offset) {
+    case 0x80 ... 0x8c: /* MAIL0_READ */
+        if (s->mbox[0].status & ARM_MS_EMPTY) {
+            res = MBOX_INVALID_DATA;
+        } else {
+            res = mbox_pull(&s->mbox[0], 0);
+        }
+        break;
+
+    case MAIL0_PEEK:
+        res = s->mbox[0].reg[0];
+        break;
+
+    case MAIL0_SENDER:
+        break;
+
+    case MAIL0_STATUS:
+        res = s->mbox[0].status;
+        break;
+
+    case MAIL0_CONFIG:
+        res = s->mbox[0].config;
+        break;
+
+    case MAIL1_STATUS:
+        res = s->mbox[1].status;
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    bcm2835_mbox_update(s);
+
+    return res;
+}
+
+static void bcm2835_mbox_write(void *opaque, hwaddr offset,
+                               uint64_t value, unsigned size)
+{
+    BCM2835MboxState *s = opaque;
+    hwaddr childaddr;
+    uint8_t ch;
+
+    offset &= 0xff;
+
+    switch (offset) {
+    case MAIL0_SENDER:
+        break;
+
+    case MAIL0_CONFIG:
+        s->mbox[0].config &= ~ARM_MC_IHAVEDATAIRQEN;
+        s->mbox[0].config |= value & ARM_MC_IHAVEDATAIRQEN;
+        break;
+
+    case 0xa0 ... 0xac: /* MAIL1_WRITE */
+        if (s->mbox[1].status & ARM_MS_FULL) {
+            /* Mailbox full */
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: mailbox full\n", __func__);
+        } else {
+            ch = value & 0xf;
+            if (ch < MBOX_CHAN_COUNT) {
+                childaddr = ch << MBOX_AS_CHAN_SHIFT;
+                if (ldl_le_phys(&s->mbox_as, childaddr + MBOX_AS_PENDING)) {
+                    /* Child busy, push delayed. Push it in the arm->vc mbox */
+                    mbox_push(&s->mbox[1], value);
+                } else {
+                    /* Push it directly to the child device */
+                    stl_le_phys(&s->mbox_as, childaddr, value);
+                }
+            } else {
+                /* Invalid channel number */
+                qemu_log_mask(LOG_GUEST_ERROR, "%s: invalid channel %u\n",
+                              __func__, ch);
+            }
+        }
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+
+    bcm2835_mbox_update(s);
+}
+
+static const MemoryRegionOps bcm2835_mbox_ops = {
+    .read = bcm2835_mbox_read,
+    .write = bcm2835_mbox_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+/* vmstate of a single mailbox */
+static const VMStateDescription vmstate_bcm2835_mbox_box = {
+    .name = TYPE_BCM2835_MBOX "_box",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(reg, BCM2835Mbox, MBOX_SIZE),
+        VMSTATE_UINT32(count, BCM2835Mbox),
+        VMSTATE_UINT32(status, BCM2835Mbox),
+        VMSTATE_UINT32(config, BCM2835Mbox),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* vmstate of the entire device */
+static const VMStateDescription vmstate_bcm2835_mbox = {
+    .name = TYPE_BCM2835_MBOX,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL_ARRAY(available, BCM2835MboxState, MBOX_CHAN_COUNT),
+        VMSTATE_STRUCT_ARRAY(mbox, BCM2835MboxState, 2, 1,
+                             vmstate_bcm2835_mbox_box, BCM2835Mbox),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_mbox_init(Object *obj)
+{
+    BCM2835MboxState *s = BCM2835_MBOX(obj);
+
+    memory_region_init_io(&s->iomem, obj, &bcm2835_mbox_ops, s,
+                          TYPE_BCM2835_MBOX, 0x400);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->arm_irq);
+    qdev_init_gpio_in(DEVICE(s), bcm2835_mbox_set_irq, MBOX_CHAN_COUNT);
+}
+
+static void bcm2835_mbox_reset(DeviceState *dev)
+{
+    BCM2835MboxState *s = BCM2835_MBOX(dev);
+    int n;
+
+    mbox_reset(&s->mbox[0]);
+    mbox_reset(&s->mbox[1]);
+    s->mbox_irq_disabled = false;
+    for (n = 0; n < MBOX_CHAN_COUNT; n++) {
+        s->available[n] = false;
+    }
+}
+
+static void bcm2835_mbox_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835MboxState *s = BCM2835_MBOX(dev);
+    Object *obj;
+    Error *err = NULL;
+
+    obj = object_property_get_link(OBJECT(dev), "mbox-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required mbox-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->mbox_mr = MEMORY_REGION(obj);
+    address_space_init(&s->mbox_as, s->mbox_mr, NULL);
+    bcm2835_mbox_reset(dev);
+}
+
+static void bcm2835_mbox_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = bcm2835_mbox_realize;
+    dc->reset = bcm2835_mbox_reset;
+    dc->vmsd = &vmstate_bcm2835_mbox;
+}
+
+static TypeInfo bcm2835_mbox_info = {
+    .name          = TYPE_BCM2835_MBOX,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835MboxState),
+    .class_init    = bcm2835_mbox_class_init,
+    .instance_init = bcm2835_mbox_init,
+};
+
+static void bcm2835_mbox_register_types(void)
+{
+    type_register_static(&bcm2835_mbox_info);
+}
+
+type_init(bcm2835_mbox_register_types)
diff --git a/hw/misc/bcm2835_property.c b/hw/misc/bcm2835_property.c
new file mode 100644
index 0000000..70eaafd
--- /dev/null
+++ b/hw/misc/bcm2835_property.c
@@ -0,0 +1,440 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/misc/bcm2835_property.h"
+#include "hw/misc/bcm2835_mbox_defs.h"
+#include "sysemu/dma.h"
+#include "qemu/log.h"
+
+/* https://github.com/raspberrypi/firmware/wiki/Mailbox-property-interface */
+
+static void bcm2835_property_mbox_push(BCM2835PropertyState *s, uint32_t value)
+{
+    uint32_t tag;
+    uint32_t bufsize;
+    uint32_t tot_len;
+    size_t resplen;
+    uint32_t tmp;
+    int n;
+    uint32_t offset, length, color;
+    uint32_t xres, yres, xoffset, yoffset, bpp, pixo, alpha;
+    uint32_t tmp_xres, tmp_yres, tmp_xoffset, tmp_yoffset;
+    uint32_t tmp_bpp, tmp_pixo, tmp_alpha;
+    uint32_t *newxres = NULL, *newyres = NULL, *newxoffset = NULL,
+        *newyoffset = NULL, *newbpp = NULL, *newpixo = NULL, *newalpha = NULL;
+
+    value &= ~0xf;
+
+    s->addr = value;
+
+    tot_len = ldl_le_phys(&s->dma_as, value);
+
+    /* @(addr + 4) : Buffer response code */
+    value = s->addr + 8;
+    while (value + 8 <= s->addr + tot_len) {
+        tag = ldl_le_phys(&s->dma_as, value);
+        bufsize = ldl_le_phys(&s->dma_as, value + 4);
+        /* @(value + 8) : Request/response indicator */
+        resplen = 0;
+        switch (tag) {
+        case 0x00000000: /* End tag */
+            break;
+        case 0x00000001: /* Get firmware revision */
+            stl_le_phys(&s->dma_as, value + 12, 346337);
+            resplen = 4;
+            break;
+        case 0x00010001: /* Get board model */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x get board model NYI\n", tag);
+            resplen = 4;
+            break;
+        case 0x00010002: /* Get board revision */
+            stl_le_phys(&s->dma_as, value + 12, s->board_rev);
+            resplen = 4;
+            break;
+        case 0x00010003: /* Get board MAC address */
+            resplen = sizeof(s->macaddr.a);
+            dma_memory_write(&s->dma_as, value + 12, s->macaddr.a, resplen);
+            break;
+        case 0x00010004: /* Get board serial */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x get board serial NYI\n", tag);
+            resplen = 8;
+            break;
+        case 0x00010005: /* Get ARM memory */
+            /* base */
+            stl_le_phys(&s->dma_as, value + 12, 0);
+            /* size */
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->vcram_base);
+            resplen = 8;
+            break;
+        case 0x00010006: /* Get VC memory */
+            /* base */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->vcram_base);
+            /* size */
+            stl_le_phys(&s->dma_as, value + 16, s->fbdev->vcram_size);
+            resplen = 8;
+            break;
+        case 0x00028001: /* Set power state */
+            /* Assume that whatever device they asked for exists,
+             * and we'll just claim we set it to the desired state
+             */
+            tmp = ldl_le_phys(&s->dma_as, value + 16);
+            stl_le_phys(&s->dma_as, value + 16, (tmp & 1));
+            resplen = 8;
+            break;
+
+        /* Clocks */
+
+        case 0x00030001: /* Get clock state */
+            stl_le_phys(&s->dma_as, value + 16, 0x1);
+            resplen = 8;
+            break;
+
+        case 0x00038001: /* Set clock state */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x set clock state NYI\n", tag);
+            resplen = 8;
+            break;
+
+        case 0x00030002: /* Get clock rate */
+        case 0x00030004: /* Get max clock rate */
+        case 0x00030007: /* Get min clock rate */
+            switch (ldl_le_phys(&s->dma_as, value + 12)) {
+            case 1: /* EMMC */
+                stl_le_phys(&s->dma_as, value + 16, 50000000);
+                break;
+            case 2: /* UART */
+                stl_le_phys(&s->dma_as, value + 16, 3000000);
+                break;
+            default:
+                stl_le_phys(&s->dma_as, value + 16, 700000000);
+                break;
+            }
+            resplen = 8;
+            break;
+
+        case 0x00038002: /* Set clock rate */
+        case 0x00038004: /* Set max clock rate */
+        case 0x00038007: /* Set min clock rate */
+            qemu_log_mask(LOG_UNIMP,
+                          "bcm2835_property: %x set clock rates NYI\n", tag);
+            resplen = 8;
+            break;
+
+        /* Temperature */
+
+        case 0x00030006: /* Get temperature */
+            stl_le_phys(&s->dma_as, value + 16, 25000);
+            resplen = 8;
+            break;
+
+        case 0x0003000A: /* Get max temperature */
+            stl_le_phys(&s->dma_as, value + 16, 99000);
+            resplen = 8;
+            break;
+
+        /* Frame buffer */
+
+        case 0x00040001: /* Allocate buffer */
+            stl_le_phys(&s->dma_as, value + 12, s->fbdev->base);
+            tmp_xres = newxres != NULL ? *newxres : s->fbdev->xres;
+            tmp_yres = newyres != NULL ? *newyres : s->fbdev->yres;
+            tmp_bpp = newbpp != NULL ? *newbpp : s->fbdev->bpp;
+            stl_le_phys(&s->dma_as, value + 16,
+                        tmp_xres * tmp_yres * tmp_bpp / 8);
+            resplen = 8;
+            break;
+        case 0x00048001: /* Release buffer */
+            resplen = 0;
+            break;
+        case 0x00040002: /* Blank screen */
+            resplen = 4;
+            break;
+        case 0x00040003: /* Get display width/height */
+        case 0x00040004:
+            tmp_xres = newxres != NULL ? *newxres : s->fbdev->xres;
+            tmp_yres = newyres != NULL ? *newyres : s->fbdev->yres;
+            stl_le_phys(&s->dma_as, value + 12, tmp_xres);
+            stl_le_phys(&s->dma_as, value + 16, tmp_yres);
+            resplen = 8;
+            break;
+        case 0x00044003: /* Test display width/height */
+        case 0x00044004:
+            resplen = 8;
+            break;
+        case 0x00048003: /* Set display width/height */
+        case 0x00048004:
+            xres = ldl_le_phys(&s->dma_as, value + 12);
+            newxres = &xres;
+            yres = ldl_le_phys(&s->dma_as, value + 16);
+            newyres = &yres;
+            resplen = 8;
+            break;
+        case 0x00040005: /* Get depth */
+            tmp_bpp = newbpp != NULL ? *newbpp : s->fbdev->bpp;
+            stl_le_phys(&s->dma_as, value + 12, tmp_bpp);
+            resplen = 4;
+            break;
+        case 0x00044005: /* Test depth */
+            resplen = 4;
+            break;
+        case 0x00048005: /* Set depth */
+            bpp = ldl_le_phys(&s->dma_as, value + 12);
+            newbpp = &bpp;
+            resplen = 4;
+            break;
+        case 0x00040006: /* Get pixel order */
+            tmp_pixo = newpixo != NULL ? *newpixo : s->fbdev->pixo;
+            stl_le_phys(&s->dma_as, value + 12, tmp_pixo);
+            resplen = 4;
+            break;
+        case 0x00044006: /* Test pixel order */
+            resplen = 4;
+            break;
+        case 0x00048006: /* Set pixel order */
+            pixo = ldl_le_phys(&s->dma_as, value + 12);
+            newpixo = &pixo;
+            resplen = 4;
+            break;
+        case 0x00040007: /* Get alpha */
+            tmp_alpha = newalpha != NULL ? *newalpha : s->fbdev->alpha;
+            stl_le_phys(&s->dma_as, value + 12, tmp_alpha);
+            resplen = 4;
+            break;
+        case 0x00044007: /* Test pixel alpha */
+            resplen = 4;
+            break;
+        case 0x00048007: /* Set alpha */
+            alpha = ldl_le_phys(&s->dma_as, value + 12);
+            newalpha = &alpha;
+            resplen = 4;
+            break;
+        case 0x00040008: /* Get pitch */
+            tmp_xres = newxres != NULL ? *newxres : s->fbdev->xres;
+            tmp_bpp = newbpp != NULL ? *newbpp : s->fbdev->bpp;
+            stl_le_phys(&s->dma_as, value + 12, tmp_xres * tmp_bpp / 8);
+            resplen = 4;
+            break;
+        case 0x00040009: /* Get virtual offset */
+            tmp_xoffset = newxoffset != NULL ? *newxoffset : s->fbdev->xoffset;
+            tmp_yoffset = newyoffset != NULL ? *newyoffset : s->fbdev->yoffset;
+            stl_le_phys(&s->dma_as, value + 12, tmp_xoffset);
+            stl_le_phys(&s->dma_as, value + 16, tmp_yoffset);
+            resplen = 8;
+            break;
+        case 0x00044009: /* Test virtual offset */
+            resplen = 8;
+            break;
+        case 0x00048009: /* Set virtual offset */
+            xoffset = ldl_le_phys(&s->dma_as, value + 12);
+            newxoffset = &xoffset;
+            yoffset = ldl_le_phys(&s->dma_as, value + 16);
+            newyoffset = &yoffset;
+            resplen = 8;
+            break;
+        case 0x0004000a: /* Get/Test/Set overscan */
+        case 0x0004400a:
+        case 0x0004800a:
+            stl_le_phys(&s->dma_as, value + 12, 0);
+            stl_le_phys(&s->dma_as, value + 16, 0);
+            stl_le_phys(&s->dma_as, value + 20, 0);
+            stl_le_phys(&s->dma_as, value + 24, 0);
+            resplen = 16;
+            break;
+        case 0x0004800b: /* Set palette */
+            offset = ldl_le_phys(&s->dma_as, value + 12);
+            length = ldl_le_phys(&s->dma_as, value + 16);
+            n = 0;
+            while (n < length - offset) {
+                color = ldl_le_phys(&s->dma_as, value + 20 + (n << 2));
+                stl_le_phys(&s->dma_as,
+                            s->fbdev->vcram_base + ((offset + n) << 2), color);
+                n++;
+            }
+            stl_le_phys(&s->dma_as, value + 12, 0);
+            resplen = 4;
+            break;
+
+        case 0x00060001: /* Get DMA channels */
+            /* channels 2-5 */
+            stl_le_phys(&s->dma_as, value + 12, 0x003C);
+            resplen = 4;
+            break;
+
+        case 0x00050001: /* Get command line */
+            resplen = 0;
+            break;
+
+        default:
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "bcm2835_property: unhandled tag %08x\n", tag);
+            break;
+        }
+
+        if (tag == 0) {
+            break;
+        }
+
+        stl_le_phys(&s->dma_as, value + 8, (1 << 31) | resplen);
+        value += bufsize + 12;
+    }
+
+    /* Reconfigure framebuffer if required */
+    if (newxres || newyres || newxoffset || newyoffset || newbpp || newpixo
+        || newalpha) {
+        bcm2835_fb_reconfigure(s->fbdev, newxres, newyres, newxoffset,
+                               newyoffset, newbpp, newpixo, newalpha);
+    }
+
+    /* Buffer response code */
+    stl_le_phys(&s->dma_as, s->addr + 4, (1 << 31));
+}
+
+static uint64_t bcm2835_property_read(void *opaque, hwaddr offset,
+                                      unsigned size)
+{
+    BCM2835PropertyState *s = opaque;
+    uint32_t res = 0;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        res = MBOX_CHAN_PROPERTY | s->addr;
+        s->pending = false;
+        qemu_set_irq(s->mbox_irq, 0);
+        break;
+
+    case MBOX_AS_PENDING:
+        res = s->pending;
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return 0;
+    }
+
+    return res;
+}
+
+static void bcm2835_property_write(void *opaque, hwaddr offset,
+                                   uint64_t value, unsigned size)
+{
+    BCM2835PropertyState *s = opaque;
+
+    switch (offset) {
+    case MBOX_AS_DATA:
+        /* bcm2835_mbox should check our pending status before pushing */
+        assert(!s->pending);
+        s->pending = true;
+        bcm2835_property_mbox_push(s, value);
+        qemu_set_irq(s->mbox_irq, 1);
+        break;
+
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %"HWADDR_PRIx"\n",
+                      __func__, offset);
+        return;
+    }
+}
+
+static const MemoryRegionOps bcm2835_property_ops = {
+    .read = bcm2835_property_read,
+    .write = bcm2835_property_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+};
+
+static const VMStateDescription vmstate_bcm2835_property = {
+    .name = TYPE_BCM2835_PROPERTY,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_MACADDR(macaddr, BCM2835PropertyState),
+        VMSTATE_UINT32(addr, BCM2835PropertyState),
+        VMSTATE_BOOL(pending, BCM2835PropertyState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void bcm2835_property_init(Object *obj)
+{
+    BCM2835PropertyState *s = BCM2835_PROPERTY(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &bcm2835_property_ops, s,
+                          TYPE_BCM2835_PROPERTY, 0x10);
+    sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(s), &s->mbox_irq);
+}
+
+static void bcm2835_property_reset(DeviceState *dev)
+{
+    BCM2835PropertyState *s = BCM2835_PROPERTY(dev);
+
+    s->pending = false;
+}
+
+static void bcm2835_property_realize(DeviceState *dev, Error **errp)
+{
+    BCM2835PropertyState *s = BCM2835_PROPERTY(dev);
+    Object *obj;
+    Error *err = NULL;
+
+    obj = object_property_get_link(OBJECT(dev), "fb", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required fb link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->fbdev = BCM2835_FB(obj);
+
+    obj = object_property_get_link(OBJECT(dev), "dma-mr", &err);
+    if (obj == NULL) {
+        error_setg(errp, "%s: required dma-mr link not found: %s",
+                   __func__, error_get_pretty(err));
+        return;
+    }
+
+    s->dma_mr = MEMORY_REGION(obj);
+    address_space_init(&s->dma_as, s->dma_mr, NULL);
+
+    /* TODO: connect to MAC address of USB NIC device, once we emulate it */
+    qemu_macaddr_default_if_unset(&s->macaddr);
+
+    bcm2835_property_reset(dev);
+}
+
+static Property bcm2835_property_props[] = {
+    DEFINE_PROP_UINT32("board-rev", BCM2835PropertyState, board_rev, 0),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void bcm2835_property_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = bcm2835_property_props;
+    dc->realize = bcm2835_property_realize;
+    dc->vmsd = &vmstate_bcm2835_property;
+}
+
+static TypeInfo bcm2835_property_info = {
+    .name          = TYPE_BCM2835_PROPERTY,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(BCM2835PropertyState),
+    .class_init    = bcm2835_property_class_init,
+    .instance_init = bcm2835_property_init,
+};
+
+static void bcm2835_property_register_types(void)
+{
+    type_register_static(&bcm2835_property_info);
+}
+
+type_init(bcm2835_property_register_types)
diff --git a/hw/misc/cbus.c b/hw/misc/cbus.c
index 495d507..0c207e3 100644
--- a/hw/misc/cbus.c
+++ b/hw/misc/cbus.c
@@ -20,7 +20,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "hw/hw.h"
 #include "hw/irq.h"
 #include "hw/devices.h"
 #include "sysemu/sysemu.h"
diff --git a/hw/misc/debugexit.c b/hw/misc/debugexit.c
index 69a1b00..84fa1a5 100644
--- a/hw/misc/debugexit.c
+++ b/hw/misc/debugexit.c
@@ -7,6 +7,7 @@
  * (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 
diff --git a/hw/misc/eccmemctl.c b/hw/misc/eccmemctl.c
index 8bad6f6..a0071f3 100644
--- a/hw/misc/eccmemctl.c
+++ b/hw/misc/eccmemctl.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 
diff --git a/hw/misc/edu.c b/hw/misc/edu.c
new file mode 100644
index 0000000..888ba49
--- /dev/null
+++ b/hw/misc/edu.c
@@ -0,0 +1,407 @@
+/*
+ * QEMU educational PCI device
+ *
+ * Copyright (c) 2012-2015 Jiri Slaby
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/pci/pci.h"
+#include "qemu/timer.h"
+#include "qemu/main-loop.h" /* iothread mutex */
+#include "qapi/visitor.h"
+
+#define EDU(obj)        OBJECT_CHECK(EduState, obj, "edu")
+
+#define FACT_IRQ        0x00000001
+#define DMA_IRQ         0x00000100
+
+#define DMA_START       0x40000
+#define DMA_SIZE        4096
+
+typedef struct {
+    PCIDevice pdev;
+    MemoryRegion mmio;
+
+    QemuThread thread;
+    QemuMutex thr_mutex;
+    QemuCond thr_cond;
+    bool stopping;
+
+    uint32_t addr4;
+    uint32_t fact;
+#define EDU_STATUS_COMPUTING    0x01
+#define EDU_STATUS_IRQFACT      0x80
+    uint32_t status;
+
+    uint32_t irq_status;
+
+#define EDU_DMA_RUN             0x1
+#define EDU_DMA_DIR(cmd)        (((cmd) & 0x2) >> 1)
+# define EDU_DMA_FROM_PCI       0
+# define EDU_DMA_TO_PCI         1
+#define EDU_DMA_IRQ             0x4
+    struct dma_state {
+        dma_addr_t src;
+        dma_addr_t dst;
+        dma_addr_t cnt;
+        dma_addr_t cmd;
+    } dma;
+    QEMUTimer dma_timer;
+    char dma_buf[DMA_SIZE];
+    uint64_t dma_mask;
+} EduState;
+
+static void edu_raise_irq(EduState *edu, uint32_t val)
+{
+    edu->irq_status |= val;
+    if (edu->irq_status) {
+        pci_set_irq(&edu->pdev, 1);
+    }
+}
+
+static void edu_lower_irq(EduState *edu, uint32_t val)
+{
+    edu->irq_status &= ~val;
+
+    if (!edu->irq_status) {
+        pci_set_irq(&edu->pdev, 0);
+    }
+}
+
+static bool within(uint32_t addr, uint32_t start, uint32_t end)
+{
+    return start <= addr && addr < end;
+}
+
+static void edu_check_range(uint32_t addr, uint32_t size1, uint32_t start,
+                uint32_t size2)
+{
+    uint32_t end1 = addr + size1;
+    uint32_t end2 = start + size2;
+
+    if (within(addr, start, end2) &&
+            end1 > addr && within(end1, start, end2)) {
+        return;
+    }
+
+    hw_error("EDU: DMA range 0x%.8x-0x%.8x out of bounds (0x%.8x-0x%.8x)!",
+            addr, end1 - 1, start, end2 - 1);
+}
+
+static dma_addr_t edu_clamp_addr(const EduState *edu, dma_addr_t addr)
+{
+    dma_addr_t res = addr & edu->dma_mask;
+
+    if (addr != res) {
+        printf("EDU: clamping DMA %#.16"PRIx64" to %#.16"PRIx64"!\n", addr, res);
+    }
+
+    return res;
+}
+
+static void edu_dma_timer(void *opaque)
+{
+    EduState *edu = opaque;
+    bool raise_irq = false;
+
+    if (!(edu->dma.cmd & EDU_DMA_RUN)) {
+        return;
+    }
+
+    if (EDU_DMA_DIR(edu->dma.cmd) == EDU_DMA_FROM_PCI) {
+        uint32_t dst = edu->dma.dst;
+        edu_check_range(dst, edu->dma.cnt, DMA_START, DMA_SIZE);
+        dst -= DMA_START;
+        pci_dma_read(&edu->pdev, edu_clamp_addr(edu, edu->dma.src),
+                edu->dma_buf + dst, edu->dma.cnt);
+    } else {
+        uint32_t src = edu->dma.src;
+        edu_check_range(src, edu->dma.cnt, DMA_START, DMA_SIZE);
+        src -= DMA_START;
+        pci_dma_write(&edu->pdev, edu_clamp_addr(edu, edu->dma.dst),
+                edu->dma_buf + src, edu->dma.cnt);
+    }
+
+    edu->dma.cmd &= ~EDU_DMA_RUN;
+    if (edu->dma.cmd & EDU_DMA_IRQ) {
+        raise_irq = true;
+    }
+
+    if (raise_irq) {
+        edu_raise_irq(edu, DMA_IRQ);
+    }
+}
+
+static void dma_rw(EduState *edu, bool write, dma_addr_t *val, dma_addr_t *dma,
+                bool timer)
+{
+    if (write && (edu->dma.cmd & EDU_DMA_RUN)) {
+        return;
+    }
+
+    if (write) {
+        *dma = *val;
+    } else {
+        *val = *dma;
+    }
+
+    if (timer) {
+        timer_mod(&edu->dma_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 100);
+    }
+}
+
+static uint64_t edu_mmio_read(void *opaque, hwaddr addr, unsigned size)
+{
+    EduState *edu = opaque;
+    uint64_t val = ~0ULL;
+
+    if (size != 4) {
+        return val;
+    }
+
+    switch (addr) {
+    case 0x00:
+        val = 0x010000edu;
+        break;
+    case 0x04:
+        val = edu->addr4;
+        break;
+    case 0x08:
+        qemu_mutex_lock(&edu->thr_mutex);
+        val = edu->fact;
+        qemu_mutex_unlock(&edu->thr_mutex);
+        break;
+    case 0x20:
+        val = atomic_read(&edu->status);
+        break;
+    case 0x24:
+        val = edu->irq_status;
+        break;
+    case 0x80:
+        dma_rw(edu, false, &val, &edu->dma.src, false);
+        break;
+    case 0x88:
+        dma_rw(edu, false, &val, &edu->dma.dst, false);
+        break;
+    case 0x90:
+        dma_rw(edu, false, &val, &edu->dma.cnt, false);
+        break;
+    case 0x98:
+        dma_rw(edu, false, &val, &edu->dma.cmd, false);
+        break;
+    }
+
+    return val;
+}
+
+static void edu_mmio_write(void *opaque, hwaddr addr, uint64_t val,
+                unsigned size)
+{
+    EduState *edu = opaque;
+
+    if (addr < 0x80 && size != 4) {
+        return;
+    }
+
+    if (addr >= 0x80 && size != 4 && size != 8) {
+        return;
+    }
+
+    switch (addr) {
+    case 0x04:
+        edu->addr4 = ~val;
+        break;
+    case 0x08:
+        if (atomic_read(&edu->status) & EDU_STATUS_COMPUTING) {
+            break;
+        }
+        /* EDU_STATUS_COMPUTING cannot go 0->1 concurrently, because it is only
+         * set in this function and it is under the iothread mutex.
+         */
+        qemu_mutex_lock(&edu->thr_mutex);
+        edu->fact = val;
+        atomic_or(&edu->status, EDU_STATUS_COMPUTING);
+        qemu_cond_signal(&edu->thr_cond);
+        qemu_mutex_unlock(&edu->thr_mutex);
+        break;
+    case 0x20:
+        if (val & EDU_STATUS_IRQFACT) {
+            atomic_or(&edu->status, EDU_STATUS_IRQFACT);
+        } else {
+            atomic_and(&edu->status, ~EDU_STATUS_IRQFACT);
+        }
+        break;
+    case 0x60:
+        edu_raise_irq(edu, val);
+        break;
+    case 0x64:
+        edu_lower_irq(edu, val);
+        break;
+    case 0x80:
+        dma_rw(edu, true, &val, &edu->dma.src, false);
+        break;
+    case 0x88:
+        dma_rw(edu, true, &val, &edu->dma.dst, false);
+        break;
+    case 0x90:
+        dma_rw(edu, true, &val, &edu->dma.cnt, false);
+        break;
+    case 0x98:
+        if (!(val & EDU_DMA_RUN)) {
+            break;
+        }
+        dma_rw(edu, true, &val, &edu->dma.cmd, true);
+        break;
+    }
+}
+
+static const MemoryRegionOps edu_mmio_ops = {
+    .read = edu_mmio_read,
+    .write = edu_mmio_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+/*
+ * We purposely use a thread, so that users are forced to wait for the status
+ * register.
+ */
+static void *edu_fact_thread(void *opaque)
+{
+    EduState *edu = opaque;
+
+    while (1) {
+        uint32_t val, ret = 1;
+
+        qemu_mutex_lock(&edu->thr_mutex);
+        while ((atomic_read(&edu->status) & EDU_STATUS_COMPUTING) == 0 &&
+                        !edu->stopping) {
+            qemu_cond_wait(&edu->thr_cond, &edu->thr_mutex);
+        }
+
+        if (edu->stopping) {
+            qemu_mutex_unlock(&edu->thr_mutex);
+            break;
+        }
+
+        val = edu->fact;
+        qemu_mutex_unlock(&edu->thr_mutex);
+
+        while (val > 0) {
+            ret *= val--;
+        }
+
+        /*
+         * We should sleep for a random period here, so that students are
+         * forced to check the status properly.
+         */
+
+        qemu_mutex_lock(&edu->thr_mutex);
+        edu->fact = ret;
+        qemu_mutex_unlock(&edu->thr_mutex);
+        atomic_and(&edu->status, ~EDU_STATUS_COMPUTING);
+
+        if (atomic_read(&edu->status) & EDU_STATUS_IRQFACT) {
+            qemu_mutex_lock_iothread();
+            edu_raise_irq(edu, FACT_IRQ);
+            qemu_mutex_unlock_iothread();
+        }
+    }
+
+    return NULL;
+}
+
+static void pci_edu_realize(PCIDevice *pdev, Error **errp)
+{
+    EduState *edu = DO_UPCAST(EduState, pdev, pdev);
+    uint8_t *pci_conf = pdev->config;
+
+    timer_init_ms(&edu->dma_timer, QEMU_CLOCK_VIRTUAL, edu_dma_timer, edu);
+
+    qemu_mutex_init(&edu->thr_mutex);
+    qemu_cond_init(&edu->thr_cond);
+    qemu_thread_create(&edu->thread, "edu", edu_fact_thread,
+                       edu, QEMU_THREAD_JOINABLE);
+
+    pci_config_set_interrupt_pin(pci_conf, 1);
+
+    memory_region_init_io(&edu->mmio, OBJECT(edu), &edu_mmio_ops, edu,
+                    "edu-mmio", 1 << 20);
+    pci_register_bar(pdev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &edu->mmio);
+}
+
+static void pci_edu_uninit(PCIDevice *pdev)
+{
+    EduState *edu = DO_UPCAST(EduState, pdev, pdev);
+
+    qemu_mutex_lock(&edu->thr_mutex);
+    edu->stopping = true;
+    qemu_mutex_unlock(&edu->thr_mutex);
+    qemu_cond_signal(&edu->thr_cond);
+    qemu_thread_join(&edu->thread);
+
+    qemu_cond_destroy(&edu->thr_cond);
+    qemu_mutex_destroy(&edu->thr_mutex);
+
+    timer_del(&edu->dma_timer);
+}
+
+static void edu_obj_uint64(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
+{
+    uint64_t *val = opaque;
+
+    visit_type_uint64(v, name, val, errp);
+}
+
+static void edu_instance_init(Object *obj)
+{
+    EduState *edu = EDU(obj);
+
+    edu->dma_mask = (1UL << 28) - 1;
+    object_property_add(obj, "dma_mask", "uint64", edu_obj_uint64,
+                    edu_obj_uint64, NULL, &edu->dma_mask, NULL);
+}
+
+static void edu_class_init(ObjectClass *class, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(class);
+
+    k->realize = pci_edu_realize;
+    k->exit = pci_edu_uninit;
+    k->vendor_id = PCI_VENDOR_ID_QEMU;
+    k->device_id = 0x11e8;
+    k->revision = 0x10;
+    k->class_id = PCI_CLASS_OTHERS;
+}
+
+static void pci_edu_register_types(void)
+{
+    static const TypeInfo edu_info = {
+        .name          = "edu",
+        .parent        = TYPE_PCI_DEVICE,
+        .instance_size = sizeof(EduState),
+        .instance_init = edu_instance_init,
+        .class_init    = edu_class_init,
+    };
+
+    type_register_static(&edu_info);
+}
+type_init(pci_edu_register_types)
diff --git a/hw/misc/exynos4210_pmu.c b/hw/misc/exynos4210_pmu.c
index 2b118c7..e30dbc7 100644
--- a/hw/misc/exynos4210_pmu.c
+++ b/hw/misc/exynos4210_pmu.c
@@ -24,6 +24,7 @@
  * uses PMU INFORM5 register as a holding pen.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 
 #ifndef DEBUG_PMU
@@ -456,15 +457,15 @@
     }
 }
 
-static int exynos4210_pmu_init(SysBusDevice *dev)
+static void exynos4210_pmu_init(Object *obj)
 {
-    Exynos4210PmuState *s = EXYNOS4210_PMU(dev);
+    Exynos4210PmuState *s = EXYNOS4210_PMU(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     /* memory mapping */
-    memory_region_init_io(&s->iomem, OBJECT(dev), &exynos4210_pmu_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_pmu_ops, s,
                           "exynos4210.pmu", EXYNOS4210_PMU_REGS_MEM_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-    return 0;
 }
 
 static const VMStateDescription exynos4210_pmu_vmstate = {
@@ -480,9 +481,7 @@
 static void exynos4210_pmu_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_pmu_init;
     dc->reset = exynos4210_pmu_reset;
     dc->vmsd = &exynos4210_pmu_vmstate;
 }
@@ -491,6 +490,7 @@
     .name          = TYPE_EXYNOS4210_PMU,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PmuState),
+    .instance_init = exynos4210_pmu_init,
     .class_init    = exynos4210_pmu_class_init,
 };
 
diff --git a/hw/misc/goldfish_battery.c b/hw/misc/goldfish_battery.c
index ad69bd3..29fd87e 100644
--- a/hw/misc/goldfish_battery.c
+++ b/hw/misc/goldfish_battery.c
@@ -9,6 +9,7 @@
 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ** GNU General Public License for more details.
 */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "qemu/error-report.h"
@@ -148,7 +149,7 @@
 static void monitor_print_callback(void* opaque, const char* buf, int size)
 {
     Monitor* mon = (Monitor*)opaque;
-    monitor_printf(mon, buf);
+    monitor_printf(mon, "%.*s", size, buf);
 }
 
 void goldfish_battery_display(Monitor *mon)
diff --git a/hw/misc/goldfish_pipe.c b/hw/misc/goldfish_pipe.c
index a1d1bd8..f598857 100755
--- a/hw/misc/goldfish_pipe.c
+++ b/hw/misc/goldfish_pipe.c
@@ -30,9 +30,13 @@
 */
 #include "hw/misc/goldfish_pipe.h"
 
-#include "android-qemu2-glue/utils/stream.h"
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
+
+#include "qemu-common.h"
+#include "qemu/log.h"
+#include "qemu/timer.h"
 #include "qemu/error-report.h"
 
 #include <assert.h>
@@ -46,14 +50,14 @@
 
 #if PIPE_DEBUG >= 1
 #define D(fmt, ...) \
-    do { fprintf(stdout, "android_pipe: " fmt "\n", ## __VA_ARGS__); } while (0)
+    do { fprintf(stdout, "goldfish_pipe: " fmt "\n", ## __VA_ARGS__); } while (0)
 #else
 #define D(fmt, ...)  do { /* nothing */ } while (0)
 #endif
 
 #if PIPE_DEBUG >= 2
 #define DD(fmt, ...) \
-    do { fprintf(stdout, "android_pipe: " fmt "\n", ## __VA_ARGS__); } while (0)
+    do { fprintf(stdout, "goldfish_pipe: " fmt "\n", ## __VA_ARGS__); } while (0)
 #else
 #define DD(fmt, ...)  do { /* nothing */ } while (0)
 #endif
@@ -127,9 +131,9 @@
     COMMAND_BUFFER_SIZE = 4096,
 };
 
-#define TYPE_ANDROID_PIPE "android_pipe"
-#define ANDROID_PIPE(obj) \
-    OBJECT_CHECK(AndroidPipeState, (obj), TYPE_ANDROID_PIPE)
+#define TYPE_GOLDFISH_PIPE "goldfish_pipe"
+#define GOLDFISH_PIPE(obj) \
+    OBJECT_CHECK(GoldfishPipeState, (obj), TYPE_GOLDFISH_PIPE)
 
 /* Update this version number if the device's interface changes. */
 enum {
@@ -234,8 +238,9 @@
     MemoryRegion iomem;
     qemu_irq irq;
 
-    PipeDevice* dev;
-} AndroidPipeState;
+    /* TODO: roll into shared state */
+    PipeDevice *dev;
+} GoldfishPipeState;
 
 typedef struct PipeCommand {
     int32_t cmd;
@@ -281,7 +286,7 @@
 } OpenCommandParams;
 
 typedef struct PipeDevice {
-    AndroidPipeState* ps;  // backlink to instance state
+    GoldfishPipeState* ps;  // backlink to instance state
     int device_version;    // host device verion
     int driver_version;    // guest's driver version
 
@@ -848,7 +853,7 @@
                            hwaddr offset,
                            uint64_t value,
                            unsigned size) {
-    AndroidPipeState* state = opaque;
+    GoldfishPipeState* state = opaque;
     PipeDevice* dev = state->dev;
 
     DR("%s: offset = 0x%" HWADDR_PRIx " value=%" PRIu64 "/0x%" PRIx64, __func__,
@@ -861,7 +866,7 @@
 }
 
 static uint64_t pipe_dev_read(void* opaque, hwaddr offset, unsigned size) {
-    AndroidPipeState* s = (AndroidPipeState*)opaque;
+    GoldfishPipeState* s = (GoldfishPipeState*)opaque;
     PipeDevice* dev = s->dev;
     if (offset == PIPE_REG_VERSION) {
         // PIPE_REG_VERSION is issued on probe, which means that
@@ -1123,7 +1128,7 @@
 };
 
 static void goldfish_pipe_save(QEMUFile* f, void* opaque) {
-    AndroidPipeState* s = opaque;
+    GoldfishPipeState* s = opaque;
     PipeDevice* dev = s->dev;
     dev->ops->save(f, dev);
 }
@@ -1453,7 +1458,7 @@
 }
 
 static int goldfish_pipe_load(QEMUFile* f, void* opaque, int version_id) {
-    AndroidPipeState* s = opaque;
+    GoldfishPipeState* s = opaque;
     PipeDevice* dev = s->dev;
     int res = goldfish_pipe_load_v2(f, dev);
     return res;
@@ -1464,7 +1469,7 @@
      * been loaded. Raising IRQ in the load handler causes
      * problems.
      */
-    PipeDevice* dev = ((AndroidPipeState*)opaque)->dev;
+    PipeDevice* dev = ((GoldfishPipeState*)opaque)->dev;
     if (dev->wanted_pipes_first) {
         qemu_set_irq(dev->ps->irq, 1);
     } else {
@@ -1474,7 +1479,7 @@
 
 static void goldfish_pipe_realize(DeviceState* dev, Error** errp) {
     SysBusDevice* sbdev = SYS_BUS_DEVICE(dev);
-    AndroidPipeState* s = ANDROID_PIPE(dev);
+    GoldfishPipeState* s = GOLDFISH_PIPE(dev);
 
     s->dev = (PipeDevice*)g_malloc0(sizeof(PipeDevice));
     s->dev->ps = s; /* HACK: backlink */
@@ -1500,9 +1505,8 @@
     sysbus_init_mmio(sbdev, &s->iomem);
     sysbus_init_irq(sbdev, &s->irq);
 
-    /* NOTE: "android_pipe" is the legacy name used in snapshots. */
     register_savevm_with_post_load(
-            dev, "android_pipe", 0, GOLDFISH_PIPE_SAVE_VERSION,
+            dev, "goldfish_pipe", 0, GOLDFISH_PIPE_SAVE_VERSION,
             goldfish_pipe_save, goldfish_pipe_load, goldfish_pipe_post_load, s);
 }
 
@@ -1539,14 +1543,15 @@
 static void goldfish_pipe_class_init(ObjectClass* klass, void* data) {
     DeviceClass* dc = DEVICE_CLASS(klass);
     dc->realize = goldfish_pipe_realize;
-    dc->desc = "android pipe";
+    dc->desc = "goldfish pipe";
 }
 
 static const TypeInfo goldfish_pipe_info = {
-        .name = TYPE_ANDROID_PIPE,
-        .parent = TYPE_SYS_BUS_DEVICE,
-        .instance_size = sizeof(AndroidPipeState),
-        .class_init = goldfish_pipe_class_init};
+    .name          = TYPE_GOLDFISH_PIPE,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(GoldfishPipeState),
+    .class_init    = goldfish_pipe_class_init
+};
 
 static void goldfish_pipe_register(void) {
     type_register_static(&goldfish_pipe_info);
diff --git a/hw/misc/goldfish_sync.c b/hw/misc/goldfish_sync.c
index 443086b..68e5ab9 100644
--- a/hw/misc/goldfish_sync.c
+++ b/hw/misc/goldfish_sync.c
@@ -10,25 +10,19 @@
 ** GNU General Public License for more details.
 */
 
-#include "hw/misc/goldfish_sync.h"
-#include "android/utils/debug.h"
-
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "trace.h"
+#include "hw/misc/goldfish_sync.h"
 
-#define DEBUG_LOG_ALL_DEVICE_OPS 0
+#include <assert.h>
 
-#if DEBUG_LOG_ALL_DEVICE_OPS
-#define DPRINT(...) do { \
-    if (!VERBOSE_CHECK(goldfishsync)) VERBOSE_ENABLE(goldfishsync); \
-    VERBOSE_TID_FUNCTION_DPRINT(goldfishsync, __VA_ARGS__); } while(0)
-#else
+// TODO(digit): Plug better logging for Android emulator.
 #define DPRINT(...)
-#endif
 
 // Goldfish sync commands=======================================================
 
@@ -62,18 +56,6 @@
     struct goldfish_sync_pending_cmd* next;
 };
 
-// goldfish_sync_ops contains all the host-side operations that we want
-// triggered along with certain goldfish sync device commands.
-// For example, trigger_wait will run a eglClientWaitSyncKHR
-// on the host in a particular sync thread.
-struct goldfish_sync_ops {
-    // Wait for the specified glsync object glsync_ptr
-    // in thread thread_ptr.
-    void (*trigger_wait)(uint64_t glsync_ptr,
-                         uint64_t thread_ptr,
-                         uint64_t timeline);
-};
-
 // Command numbers that can only be sent from the guest to the device.
 #define SYNC_GUEST_CMD_READY 0
 #define SYNC_GUEST_CMD_TRIGGER_HOST_WAIT 5
@@ -130,7 +112,7 @@
 // Command list operations======================================================
 
 static struct goldfish_sync_pending_cmd*
-goldfish_sync_new_cmd() {
+goldfish_sync_new_cmd(void) {
     struct goldfish_sync_pending_cmd* res;
     res = g_malloc0(sizeof(struct goldfish_sync_pending_cmd));
     return res;
diff --git a/hw/misc/hyperv_testdev.c b/hw/misc/hyperv_testdev.c
new file mode 100644
index 0000000..6cae9e9
--- /dev/null
+++ b/hw/misc/hyperv_testdev.c
@@ -0,0 +1,168 @@
+/*
+ * QEMU KVM Hyper-V test device to support Hyper-V kvm-unit-tests
+ *
+ * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * Authors:
+ *  Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include <linux/kvm.h>
+#include "hw/hw.h"
+#include "hw/qdev.h"
+#include "hw/isa/isa.h"
+#include "sysemu/kvm.h"
+#include "target-i386/hyperv.h"
+#include "kvm_i386.h"
+
+#define HV_TEST_DEV_MAX_SINT_ROUTES 64
+
+struct HypervTestDev {
+    ISADevice parent_obj;
+    MemoryRegion sint_control;
+    HvSintRoute *sint_route[HV_TEST_DEV_MAX_SINT_ROUTES];
+};
+typedef struct HypervTestDev HypervTestDev;
+
+#define TYPE_HYPERV_TEST_DEV "hyperv-testdev"
+#define HYPERV_TEST_DEV(obj) \
+        OBJECT_CHECK(HypervTestDev, (obj), TYPE_HYPERV_TEST_DEV)
+
+enum {
+    HV_TEST_DEV_SINT_ROUTE_CREATE = 1,
+    HV_TEST_DEV_SINT_ROUTE_DESTROY,
+    HV_TEST_DEV_SINT_ROUTE_SET_SINT
+};
+
+static int alloc_sint_route_index(HypervTestDev *dev)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dev->sint_route); i++) {
+        if (dev->sint_route[i] == NULL) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+static void free_sint_route_index(HypervTestDev *dev, int i)
+{
+    assert(i >= 0 && i < ARRAY_SIZE(dev->sint_route));
+    dev->sint_route[i] = NULL;
+}
+
+static int find_sint_route_index(HypervTestDev *dev, uint32_t vcpu_id,
+                                 uint32_t sint)
+{
+    HvSintRoute *sint_route;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dev->sint_route); i++) {
+        sint_route = dev->sint_route[i];
+        if (sint_route && sint_route->vcpu_id == vcpu_id &&
+            sint_route->sint == sint) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+static void hv_synic_test_dev_control(HypervTestDev *dev, uint32_t ctl,
+                                      uint32_t vcpu_id, uint32_t sint)
+{
+    int i;
+    HvSintRoute *sint_route;
+
+    switch (ctl) {
+    case HV_TEST_DEV_SINT_ROUTE_CREATE:
+        i = alloc_sint_route_index(dev);
+        assert(i >= 0);
+        sint_route = kvm_hv_sint_route_create(vcpu_id, sint, NULL);
+        assert(sint_route);
+        dev->sint_route[i] = sint_route;
+        break;
+    case HV_TEST_DEV_SINT_ROUTE_DESTROY:
+        i = find_sint_route_index(dev, vcpu_id, sint);
+        assert(i >= 0);
+        sint_route = dev->sint_route[i];
+        kvm_hv_sint_route_destroy(sint_route);
+        free_sint_route_index(dev, i);
+        break;
+    case HV_TEST_DEV_SINT_ROUTE_SET_SINT:
+        i = find_sint_route_index(dev, vcpu_id, sint);
+        assert(i >= 0);
+        sint_route = dev->sint_route[i];
+        kvm_hv_sint_route_set_sint(sint_route);
+        break;
+    default:
+        break;
+    }
+}
+
+static void hv_test_dev_control(void *opaque, hwaddr addr, uint64_t data,
+                                uint32_t len)
+{
+    HypervTestDev *dev = HYPERV_TEST_DEV(opaque);
+    uint8_t ctl;
+
+    ctl = (data >> 16ULL) & 0xFF;
+    switch (ctl) {
+    case HV_TEST_DEV_SINT_ROUTE_CREATE:
+    case HV_TEST_DEV_SINT_ROUTE_DESTROY:
+    case HV_TEST_DEV_SINT_ROUTE_SET_SINT: {
+        uint8_t sint = data & 0xFF;
+        uint8_t vcpu_id = (data >> 8ULL) & 0xFF;
+        hv_synic_test_dev_control(dev, ctl, vcpu_id, sint);
+        break;
+    }
+    default:
+        break;
+    }
+}
+
+static const MemoryRegionOps synic_test_sint_ops = {
+    .write = hv_test_dev_control,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void hv_test_dev_realizefn(DeviceState *d, Error **errp)
+{
+    ISADevice *isa = ISA_DEVICE(d);
+    HypervTestDev *dev = HYPERV_TEST_DEV(d);
+    MemoryRegion *io = isa_address_space_io(isa);
+
+    memset(dev->sint_route, 0, sizeof(dev->sint_route));
+    memory_region_init_io(&dev->sint_control, OBJECT(dev),
+                          &synic_test_sint_ops, dev,
+                          "hyperv-testdev-ctl", 4);
+    memory_region_add_subregion(io, 0x3000, &dev->sint_control);
+}
+
+static void hv_test_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->realize = hv_test_dev_realizefn;
+}
+
+static const TypeInfo hv_test_dev_info = {
+    .name           = TYPE_HYPERV_TEST_DEV,
+    .parent         = TYPE_ISA_DEVICE,
+    .instance_size  = sizeof(HypervTestDev),
+    .class_init     = hv_test_dev_class_init,
+};
+
+static void hv_test_dev_register_types(void)
+{
+    type_register_static(&hv_test_dev_info);
+}
+type_init(hv_test_dev_register_types);
diff --git a/hw/misc/imx25_ccm.c b/hw/misc/imx25_ccm.c
new file mode 100644
index 0000000..5cd8c0a
--- /dev/null
+++ b/hw/misc/imx25_ccm.c
@@ -0,0 +1,318 @@
+/*
+ * IMX25 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * To get the timer frequencies right, we need to emulate at least part of
+ * the CCM.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx25_ccm.h"
+#include "qemu/log.h"
+
+#ifndef DEBUG_IMX25_CCM
+#define DEBUG_IMX25_CCM 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX25_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX25_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx25_ccm_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case IMX25_CCM_MPCTL_REG:
+        return "mpctl";
+    case IMX25_CCM_UPCTL_REG:
+        return "upctl";
+    case IMX25_CCM_CCTL_REG:
+        return "cctl";
+    case IMX25_CCM_CGCR0_REG:
+        return "cgcr0";
+    case IMX25_CCM_CGCR1_REG:
+        return "cgcr1";
+    case IMX25_CCM_CGCR2_REG:
+        return "cgcr2";
+    case IMX25_CCM_PCDR0_REG:
+        return "pcdr0";
+    case IMX25_CCM_PCDR1_REG:
+        return "pcdr1";
+    case IMX25_CCM_PCDR2_REG:
+        return "pcdr2";
+    case IMX25_CCM_PCDR3_REG:
+        return "pcdr3";
+    case IMX25_CCM_RCSR_REG:
+        return "rcsr";
+    case IMX25_CCM_CRDR_REG:
+        return "crdr";
+    case IMX25_CCM_DCVR0_REG:
+        return "dcvr0";
+    case IMX25_CCM_DCVR1_REG:
+        return "dcvr1";
+    case IMX25_CCM_DCVR2_REG:
+        return "dcvr2";
+    case IMX25_CCM_DCVR3_REG:
+        return "dcvr3";
+    case IMX25_CCM_LTR0_REG:
+        return "ltr0";
+    case IMX25_CCM_LTR1_REG:
+        return "ltr1";
+    case IMX25_CCM_LTR2_REG:
+        return "ltr2";
+    case IMX25_CCM_LTR3_REG:
+        return "ltr3";
+    case IMX25_CCM_LTBR0_REG:
+        return "ltbr0";
+    case IMX25_CCM_LTBR1_REG:
+        return "ltbr1";
+    case IMX25_CCM_PMCR0_REG:
+        return "pmcr0";
+    case IMX25_CCM_PMCR1_REG:
+        return "pmcr1";
+    case IMX25_CCM_PMCR2_REG:
+        return "pmcr2";
+    case IMX25_CCM_MCR_REG:
+        return "mcr";
+    case IMX25_CCM_LPIMR0_REG:
+        return "lpimr0";
+    case IMX25_CCM_LPIMR1_REG:
+        return "lpimr1";
+    default:
+        sprintf(unknown, "[%d ?]", reg);
+        return unknown;
+    }
+}
+#define CKIH_FREQ 24000000 /* 24MHz crystal input */
+
+static const VMStateDescription vmstate_imx25_ccm = {
+    .name = TYPE_IMX25_CCM,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(reg, IMX25CCMState, IMX25_CCM_MAX_REG),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static uint32_t imx25_ccm_get_mpll_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    if (EXTRACT(s->reg[IMX25_CCM_CCTL_REG], MPLL_BYPASS)) {
+        freq = CKIH_FREQ;
+    } else {
+        freq = imx_ccm_calc_pll(s->reg[IMX25_CCM_MPCTL_REG], CKIH_FREQ);
+    }
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_mcu_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    freq = imx25_ccm_get_mpll_clk(dev);
+
+    if (EXTRACT(s->reg[IMX25_CCM_CCTL_REG], ARM_SRC)) {
+        freq = (freq * 3 / 4);
+    }
+
+    freq = freq / (1 + EXTRACT(s->reg[IMX25_CCM_CCTL_REG], ARM_CLK_DIV));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_ahb_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    freq = imx25_ccm_get_mcu_clk(dev)
+           / (1 + EXTRACT(s->reg[IMX25_CCM_CCTL_REG], AHB_CLK_DIV));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_ipg_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+
+    freq = imx25_ccm_get_ahb_clk(dev) / 2;
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx25_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
+{
+    uint32_t freq = 0;
+    DPRINTF("Clock = %d)\n", clock);
+
+    switch (clock) {
+    case CLK_NONE:
+        break;
+    case CLK_IPG:
+    case CLK_IPG_HIGH:
+        freq = imx25_ccm_get_ipg_clk(dev);
+        break;
+    case CLK_32k:
+        freq = CKIL_FREQ;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: unsupported clock %d\n",
+                      TYPE_IMX25_CCM, __func__, clock);
+        break;
+    }
+
+    DPRINTF("Clock = %d) = %d\n", clock, freq);
+
+    return freq;
+}
+
+static void imx25_ccm_reset(DeviceState *dev)
+{
+    IMX25CCMState *s = IMX25_CCM(dev);
+
+    DPRINTF("\n");
+
+    memset(s->reg, 0, IMX25_CCM_MAX_REG * sizeof(uint32_t));
+    s->reg[IMX25_CCM_MPCTL_REG] = 0x800b2c01;
+    s->reg[IMX25_CCM_UPCTL_REG] = 0x84042800;
+    /* 
+     * The value below gives:
+     * CPU = 133 MHz, AHB = 66,5 MHz, IPG = 33 MHz. 
+     */
+    s->reg[IMX25_CCM_CCTL_REG]  = 0xd0030000;
+    s->reg[IMX25_CCM_CGCR0_REG] = 0x028A0100;
+    s->reg[IMX25_CCM_CGCR1_REG] = 0x04008100;
+    s->reg[IMX25_CCM_CGCR2_REG] = 0x00000438;
+    s->reg[IMX25_CCM_PCDR0_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PCDR1_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PCDR2_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PCDR3_REG] = 0x01010101;
+    s->reg[IMX25_CCM_PMCR0_REG] = 0x00A00000;
+    s->reg[IMX25_CCM_PMCR1_REG] = 0x0000A030;
+    s->reg[IMX25_CCM_PMCR2_REG] = 0x0000A030;
+    s->reg[IMX25_CCM_MCR_REG]   = 0x43000000;
+
+    /*
+     * default boot will change the reset values to allow:
+     * CPU = 399 MHz, AHB = 133 MHz, IPG = 66,5 MHz. 
+     * For some reason, this doesn't work. With the value below, linux
+     * detects a 88 MHz IPG CLK instead of 66,5 MHz.
+    s->reg[IMX25_CCM_CCTL_REG]  = 0x20032000;
+     */
+}
+
+static uint64_t imx25_ccm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMX25CCMState *s = (IMX25CCMState *)opaque;
+
+    if (offset < 0x70) {
+        value = s->reg[offset >> 2];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX25_CCM, __func__, offset);
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx25_ccm_reg_name(offset >> 2),
+            value);
+
+    return value;
+}
+
+static void imx25_ccm_write(void *opaque, hwaddr offset, uint64_t value,
+                            unsigned size)
+{
+    IMX25CCMState *s = (IMX25CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx25_ccm_reg_name(offset >> 2),
+            (uint32_t)value);
+
+    if (offset < 0x70) {
+        /*
+         * We will do a better implementation later. In particular some bits
+         * cannot be written to.
+         */
+        s->reg[offset >> 2] = value;
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX25_CCM, __func__, offset);
+    }
+}
+
+static const struct MemoryRegionOps imx25_ccm_ops = {
+    .read = imx25_ccm_read,
+    .write = imx25_ccm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static void imx25_ccm_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IMX25CCMState *s = IMX25_CCM(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx25_ccm_ops, s,
+                          TYPE_IMX25_CCM, 0x1000);
+    sysbus_init_mmio(sd, &s->iomem);
+}
+
+static void imx25_ccm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IMXCCMClass *ccm = IMX_CCM_CLASS(klass);
+
+    dc->reset = imx25_ccm_reset;
+    dc->vmsd = &vmstate_imx25_ccm;
+    dc->desc = "i.MX25 Clock Control Module";
+
+    ccm->get_clock_frequency = imx25_ccm_get_clock_frequency;
+}
+
+static const TypeInfo imx25_ccm_info = {
+    .name          = TYPE_IMX25_CCM,
+    .parent        = TYPE_IMX_CCM,
+    .instance_size = sizeof(IMX25CCMState),
+    .instance_init = imx25_ccm_init,
+    .class_init    = imx25_ccm_class_init,
+};
+
+static void imx25_ccm_register_types(void)
+{
+    type_register_static(&imx25_ccm_info);
+}
+
+type_init(imx25_ccm_register_types)
diff --git a/hw/misc/imx31_ccm.c b/hw/misc/imx31_ccm.c
new file mode 100644
index 0000000..1c03e52
--- /dev/null
+++ b/hw/misc/imx31_ccm.c
@@ -0,0 +1,345 @@
+/*
+ * IMX31 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * To get the timer frequencies right, we need to emulate at least part of
+ * the i.MX31 CCM.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx31_ccm.h"
+#include "qemu/log.h"
+
+#define CKIH_FREQ 26000000 /* 26MHz crystal input */
+
+#ifndef DEBUG_IMX31_CCM
+#define DEBUG_IMX31_CCM 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX31_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX31_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx31_ccm_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case IMX31_CCM_CCMR_REG:
+        return "CCMR";
+    case IMX31_CCM_PDR0_REG:
+        return "PDR0";
+    case IMX31_CCM_PDR1_REG:
+        return "PDR1";
+    case IMX31_CCM_RCSR_REG:
+        return "RCSR";
+    case IMX31_CCM_MPCTL_REG:
+        return "MPCTL";
+    case IMX31_CCM_UPCTL_REG:
+        return "UPCTL";
+    case IMX31_CCM_SPCTL_REG:
+        return "SPCTL";
+    case IMX31_CCM_COSR_REG:
+        return "COSR";
+    case IMX31_CCM_CGR0_REG:
+        return "CGR0";
+    case IMX31_CCM_CGR1_REG:
+        return "CGR1";
+    case IMX31_CCM_CGR2_REG:
+        return "CGR2";
+    case IMX31_CCM_WIMR_REG:
+        return "WIMR";
+    case IMX31_CCM_LDC_REG:
+        return "LDC";
+    case IMX31_CCM_DCVR0_REG:
+        return "DCVR0";
+    case IMX31_CCM_DCVR1_REG:
+        return "DCVR1";
+    case IMX31_CCM_DCVR2_REG:
+        return "DCVR2";
+    case IMX31_CCM_DCVR3_REG:
+        return "DCVR3";
+    case IMX31_CCM_LTR0_REG:
+        return "LTR0";
+    case IMX31_CCM_LTR1_REG:
+        return "LTR1";
+    case IMX31_CCM_LTR2_REG:
+        return "LTR2";
+    case IMX31_CCM_LTR3_REG:
+        return "LTR3";
+    case IMX31_CCM_LTBR0_REG:
+        return "LTBR0";
+    case IMX31_CCM_LTBR1_REG:
+        return "LTBR1";
+    case IMX31_CCM_PMCR0_REG:
+        return "PMCR0";
+    case IMX31_CCM_PMCR1_REG:
+        return "PMCR1";
+    case IMX31_CCM_PDR2_REG:
+        return "PDR2";
+    default:
+        sprintf(unknown, "[%d ?]", reg);
+        return unknown;
+    }
+}
+
+static const VMStateDescription vmstate_imx31_ccm = {
+    .name = TYPE_IMX31_CCM,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(reg, IMX31CCMState, IMX31_CCM_MAX_REG),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static uint32_t imx31_ccm_get_pll_ref_clk(IMXCCMState *dev)
+{
+    uint32_t freq = 0;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    if ((s->reg[IMX31_CCM_CCMR_REG] & CCMR_PRCS) == 2) {
+        if (s->reg[IMX31_CCM_CCMR_REG] & CCMR_FPME) {
+            freq = CKIL_FREQ;
+            if (s->reg[IMX31_CCM_CCMR_REG] & CCMR_FPMF) {
+                freq *= 1024;
+            }
+        } 
+    } else {
+        freq = CKIH_FREQ;
+    }
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_mpll_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    freq = imx_ccm_calc_pll(s->reg[IMX31_CCM_MPCTL_REG],
+                            imx31_ccm_get_pll_ref_clk(dev));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_mcu_main_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    if ((s->reg[IMX31_CCM_CCMR_REG] & CCMR_MDS) ||
+        !(s->reg[IMX31_CCM_CCMR_REG] & CCMR_MPE)) {
+        freq = imx31_ccm_get_pll_ref_clk(dev);
+    } else {
+        freq = imx31_ccm_get_mpll_clk(dev);
+    }
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_hclk_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    freq = imx31_ccm_get_mcu_main_clk(dev)
+           / (1 + EXTRACT(s->reg[IMX31_CCM_PDR0_REG], MAX));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_ipg_clk(IMXCCMState *dev)
+{
+    uint32_t freq;
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    freq = imx31_ccm_get_hclk_clk(dev)
+           / (1 + EXTRACT(s->reg[IMX31_CCM_PDR0_REG], IPG));
+
+    DPRINTF("freq = %d\n", freq);
+
+    return freq;
+}
+
+static uint32_t imx31_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
+{
+    uint32_t freq = 0;
+
+    switch (clock) {
+    case CLK_NONE:
+        break;
+    case CLK_IPG:
+    case CLK_IPG_HIGH:
+        freq = imx31_ccm_get_ipg_clk(dev);
+        break;
+    case CLK_32k:
+        freq = CKIL_FREQ;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: unsupported clock %d\n",
+                      TYPE_IMX31_CCM, __func__, clock);
+        break;
+    }
+
+    DPRINTF("Clock = %d) = %d\n", clock, freq);
+
+    return freq;
+}
+
+static void imx31_ccm_reset(DeviceState *dev)
+{
+    IMX31CCMState *s = IMX31_CCM(dev);
+
+    DPRINTF("()\n");
+
+    memset(s->reg, 0, sizeof(uint32_t) * IMX31_CCM_MAX_REG);
+
+    s->reg[IMX31_CCM_CCMR_REG]   = 0x074b0b7d;
+    s->reg[IMX31_CCM_PDR0_REG]   = 0xff870b48;
+    s->reg[IMX31_CCM_PDR1_REG]   = 0x49fcfe7f;
+    s->reg[IMX31_CCM_RCSR_REG]   = 0x007f0000;
+    s->reg[IMX31_CCM_MPCTL_REG]  = 0x04001800;
+    s->reg[IMX31_CCM_UPCTL_REG]  = 0x04051c03;
+    s->reg[IMX31_CCM_SPCTL_REG]  = 0x04043001;
+    s->reg[IMX31_CCM_COSR_REG]   = 0x00000280;
+    s->reg[IMX31_CCM_CGR0_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_CGR1_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_CGR2_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_WIMR_REG]   = 0xffffffff;
+    s->reg[IMX31_CCM_LTR1_REG]   = 0x00004040;
+    s->reg[IMX31_CCM_PMCR0_REG]  = 0x80209828;
+    s->reg[IMX31_CCM_PMCR1_REG]  = 0x00aa0000;
+    s->reg[IMX31_CCM_PDR2_REG]   = 0x00000285;
+}
+
+static uint64_t imx31_ccm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMX31CCMState *s = (IMX31CCMState *)opaque;
+
+    if ((offset >> 2) < IMX31_CCM_MAX_REG) {
+        value = s->reg[offset >> 2];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX31_CCM, __func__, offset);
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx31_ccm_reg_name(offset >> 2),
+            value);
+
+    return (uint64_t)value;
+}
+
+static void imx31_ccm_write(void *opaque, hwaddr offset, uint64_t value,
+                            unsigned size)
+{
+    IMX31CCMState *s = (IMX31CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx31_ccm_reg_name(offset >> 2),
+            (uint32_t)value);
+
+    switch (offset >> 2) {
+    case IMX31_CCM_CCMR_REG:
+        s->reg[IMX31_CCM_CCMR_REG] = CCMR_FPMF | (value & 0x3b6fdfff);
+        break;
+    case IMX31_CCM_PDR0_REG:
+        s->reg[IMX31_CCM_PDR0_REG] = value & 0xff9f3fff;
+        break;
+    case IMX31_CCM_PDR1_REG:
+        s->reg[IMX31_CCM_PDR1_REG] = value;
+        break;
+    case IMX31_CCM_MPCTL_REG:
+        s->reg[IMX31_CCM_MPCTL_REG] = value & 0xbfff3fff;
+        break;
+    case IMX31_CCM_SPCTL_REG:
+        s->reg[IMX31_CCM_SPCTL_REG] = value & 0xbfff3fff;
+        break;
+    case IMX31_CCM_CGR0_REG:
+        s->reg[IMX31_CCM_CGR0_REG] = value;
+        break;
+    case IMX31_CCM_CGR1_REG:
+        s->reg[IMX31_CCM_CGR1_REG] = value;
+        break;
+    case IMX31_CCM_CGR2_REG:
+        s->reg[IMX31_CCM_CGR2_REG] = value;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX31_CCM, __func__, offset);
+        break;
+    }
+}
+
+static const struct MemoryRegionOps imx31_ccm_ops = {
+    .read = imx31_ccm_read,
+    .write = imx31_ccm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+
+};
+
+static void imx31_ccm_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IMX31CCMState *s = IMX31_CCM(obj);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx31_ccm_ops, s,
+                          TYPE_IMX31_CCM, 0x1000);
+    sysbus_init_mmio(sd, &s->iomem);
+}
+
+static void imx31_ccm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc  = DEVICE_CLASS(klass);
+    IMXCCMClass *ccm = IMX_CCM_CLASS(klass);
+
+    dc->reset = imx31_ccm_reset;
+    dc->vmsd  = &vmstate_imx31_ccm;
+    dc->desc  = "i.MX31 Clock Control Module";
+
+    ccm->get_clock_frequency = imx31_ccm_get_clock_frequency;
+}
+
+static const TypeInfo imx31_ccm_info = {
+    .name          = TYPE_IMX31_CCM,
+    .parent        = TYPE_IMX_CCM,
+    .instance_size = sizeof(IMX31CCMState),
+    .instance_init = imx31_ccm_init,
+    .class_init    = imx31_ccm_class_init,
+};
+
+static void imx31_ccm_register_types(void)
+{
+    type_register_static(&imx31_ccm_info);
+}
+
+type_init(imx31_ccm_register_types)
diff --git a/hw/misc/imx6_ccm.c b/hw/misc/imx6_ccm.c
new file mode 100644
index 0000000..17e15d4
--- /dev/null
+++ b/hw/misc/imx6_ccm.c
@@ -0,0 +1,781 @@
+/*
+ * IMX6 Clock Control Module
+ *
+ * Copyright (c) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * To get the timer frequencies right, we need to emulate at least part of
+ * the CCM.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx6_ccm.h"
+#include "qemu/log.h"
+
+#ifndef DEBUG_IMX6_CCM
+#define DEBUG_IMX6_CCM 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX6_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX6_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx6_ccm_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case CCM_CCR:
+        return "CCR";
+    case CCM_CCDR:
+        return "CCDR";
+    case CCM_CSR:
+        return "CSR";
+    case CCM_CCSR:
+        return "CCSR";
+    case CCM_CACRR:
+        return "CACRR";
+    case CCM_CBCDR:
+        return "CBCDR";
+    case CCM_CBCMR:
+        return "CBCMR";
+    case CCM_CSCMR1:
+        return "CSCMR1";
+    case CCM_CSCMR2:
+        return "CSCMR2";
+    case CCM_CSCDR1:
+        return "CSCDR1";
+    case CCM_CS1CDR:
+        return "CS1CDR";
+    case CCM_CS2CDR:
+        return "CS2CDR";
+    case CCM_CDCDR:
+        return "CDCDR";
+    case CCM_CHSCCDR:
+        return "CHSCCDR";
+    case CCM_CSCDR2:
+        return "CSCDR2";
+    case CCM_CSCDR3:
+        return "CSCDR3";
+    case CCM_CDHIPR:
+        return "CDHIPR";
+    case CCM_CTOR:
+        return "CTOR";
+    case CCM_CLPCR:
+        return "CLPCR";
+    case CCM_CISR:
+        return "CISR";
+    case CCM_CIMR:
+        return "CIMR";
+    case CCM_CCOSR:
+        return "CCOSR";
+    case CCM_CGPR:
+        return "CGPR";
+    case CCM_CCGR0:
+        return "CCGR0";
+    case CCM_CCGR1:
+        return "CCGR1";
+    case CCM_CCGR2:
+        return "CCGR2";
+    case CCM_CCGR3:
+        return "CCGR3";
+    case CCM_CCGR4:
+        return "CCGR4";
+    case CCM_CCGR5:
+        return "CCGR5";
+    case CCM_CCGR6:
+        return "CCGR6";
+    case CCM_CMEOR:
+        return "CMEOR";
+    default:
+        sprintf(unknown, "%d ?", reg);
+        return unknown;
+    }
+}
+
+static char const *imx6_analog_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case CCM_ANALOG_PLL_ARM:
+        return "PLL_ARM";
+    case CCM_ANALOG_PLL_ARM_SET:
+        return "PLL_ARM_SET";
+    case CCM_ANALOG_PLL_ARM_CLR:
+        return "PLL_ARM_CLR";
+    case CCM_ANALOG_PLL_ARM_TOG:
+        return "PLL_ARM_TOG";
+    case CCM_ANALOG_PLL_USB1:
+        return "PLL_USB1";
+    case CCM_ANALOG_PLL_USB1_SET:
+        return "PLL_USB1_SET";
+    case CCM_ANALOG_PLL_USB1_CLR:
+        return "PLL_USB1_CLR";
+    case CCM_ANALOG_PLL_USB1_TOG:
+        return "PLL_USB1_TOG";
+    case CCM_ANALOG_PLL_USB2:
+        return "PLL_USB2";
+    case CCM_ANALOG_PLL_USB2_SET:
+        return "PLL_USB2_SET";
+    case CCM_ANALOG_PLL_USB2_CLR:
+        return "PLL_USB2_CLR";
+    case CCM_ANALOG_PLL_USB2_TOG:
+        return "PLL_USB2_TOG";
+    case CCM_ANALOG_PLL_SYS:
+        return "PLL_SYS";
+    case CCM_ANALOG_PLL_SYS_SET:
+        return "PLL_SYS_SET";
+    case CCM_ANALOG_PLL_SYS_CLR:
+        return "PLL_SYS_CLR";
+    case CCM_ANALOG_PLL_SYS_TOG:
+        return "PLL_SYS_TOG";
+    case CCM_ANALOG_PLL_SYS_SS:
+        return "PLL_SYS_SS";
+    case CCM_ANALOG_PLL_SYS_NUM:
+        return "PLL_SYS_NUM";
+    case CCM_ANALOG_PLL_SYS_DENOM:
+        return "PLL_SYS_DENOM";
+    case CCM_ANALOG_PLL_AUDIO:
+        return "PLL_AUDIO";
+    case CCM_ANALOG_PLL_AUDIO_SET:
+        return "PLL_AUDIO_SET";
+    case CCM_ANALOG_PLL_AUDIO_CLR:
+        return "PLL_AUDIO_CLR";
+    case CCM_ANALOG_PLL_AUDIO_TOG:
+        return "PLL_AUDIO_TOG";
+    case CCM_ANALOG_PLL_AUDIO_NUM:
+        return "PLL_AUDIO_NUM";
+    case CCM_ANALOG_PLL_AUDIO_DENOM:
+        return "PLL_AUDIO_DENOM";
+    case CCM_ANALOG_PLL_VIDEO:
+        return "PLL_VIDEO";
+    case CCM_ANALOG_PLL_VIDEO_SET:
+        return "PLL_VIDEO_SET";
+    case CCM_ANALOG_PLL_VIDEO_CLR:
+        return "PLL_VIDEO_CLR";
+    case CCM_ANALOG_PLL_VIDEO_TOG:
+        return "PLL_VIDEO_TOG";
+    case CCM_ANALOG_PLL_VIDEO_NUM:
+        return "PLL_VIDEO_NUM";
+    case CCM_ANALOG_PLL_VIDEO_DENOM:
+        return "PLL_VIDEO_DENOM";
+    case CCM_ANALOG_PLL_MLB:
+        return "PLL_MLB";
+    case CCM_ANALOG_PLL_MLB_SET:
+        return "PLL_MLB_SET";
+    case CCM_ANALOG_PLL_MLB_CLR:
+        return "PLL_MLB_CLR";
+    case CCM_ANALOG_PLL_MLB_TOG:
+        return "PLL_MLB_TOG";
+    case CCM_ANALOG_PLL_ENET:
+        return "PLL_ENET";
+    case CCM_ANALOG_PLL_ENET_SET:
+        return "PLL_ENET_SET";
+    case CCM_ANALOG_PLL_ENET_CLR:
+        return "PLL_ENET_CLR";
+    case CCM_ANALOG_PLL_ENET_TOG:
+        return "PLL_ENET_TOG";
+    case CCM_ANALOG_PFD_480:
+        return "PFD_480";
+    case CCM_ANALOG_PFD_480_SET:
+        return "PFD_480_SET";
+    case CCM_ANALOG_PFD_480_CLR:
+        return "PFD_480_CLR";
+    case CCM_ANALOG_PFD_480_TOG:
+        return "PFD_480_TOG";
+    case CCM_ANALOG_PFD_528:
+        return "PFD_528";
+    case CCM_ANALOG_PFD_528_SET:
+        return "PFD_528_SET";
+    case CCM_ANALOG_PFD_528_CLR:
+        return "PFD_528_CLR";
+    case CCM_ANALOG_PFD_528_TOG:
+        return "PFD_528_TOG";
+    case CCM_ANALOG_MISC0:
+        return "MISC0";
+    case CCM_ANALOG_MISC0_SET:
+        return "MISC0_SET";
+    case CCM_ANALOG_MISC0_CLR:
+        return "MISC0_CLR";
+    case CCM_ANALOG_MISC0_TOG:
+        return "MISC0_TOG";
+    case CCM_ANALOG_MISC2:
+        return "MISC2";
+    case CCM_ANALOG_MISC2_SET:
+        return "MISC2_SET";
+    case CCM_ANALOG_MISC2_CLR:
+        return "MISC2_CLR";
+    case CCM_ANALOG_MISC2_TOG:
+        return "MISC2_TOG";
+    case PMU_REG_1P1:
+        return "PMU_REG_1P1";
+    case PMU_REG_3P0:
+        return "PMU_REG_3P0";
+    case PMU_REG_2P5:
+        return "PMU_REG_2P5";
+    case PMU_REG_CORE:
+        return "PMU_REG_CORE";
+    case PMU_MISC1:
+        return "PMU_MISC1";
+    case PMU_MISC1_SET:
+        return "PMU_MISC1_SET";
+    case PMU_MISC1_CLR:
+        return "PMU_MISC1_CLR";
+    case PMU_MISC1_TOG:
+        return "PMU_MISC1_TOG";
+    case USB_ANALOG_DIGPROG:
+        return "USB_ANALOG_DIGPROG";
+    default:
+        sprintf(unknown, "%d ?", reg);
+        return unknown;
+    }
+}
+
+#define CKIH_FREQ 24000000 /* 24MHz crystal input */
+
+static const VMStateDescription vmstate_imx6_ccm = {
+    .name = TYPE_IMX6_CCM,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(ccm, IMX6CCMState, CCM_MAX),
+        VMSTATE_UINT32_ARRAY(analog, IMX6CCMState, CCM_ANALOG_MAX),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static uint64_t imx6_analog_get_pll2_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 24000000;
+
+    if (EXTRACT(dev->analog[CCM_ANALOG_PLL_SYS], DIV_SELECT)) {
+        freq *= 22;
+    } else {
+        freq *= 20;
+    }
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_analog_get_pll2_pfd0_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_analog_get_pll2_clk(dev) * 18
+           / EXTRACT(dev->analog[CCM_ANALOG_PFD_528], PFD0_FRAC);
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_analog_get_pll2_pfd2_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_analog_get_pll2_clk(dev) * 18
+           / EXTRACT(dev->analog[CCM_ANALOG_PFD_528], PFD2_FRAC);
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_analog_get_periph_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    switch (EXTRACT(dev->ccm[CCM_CBCMR], PRE_PERIPH_CLK_SEL)) {
+    case 0:
+        freq = imx6_analog_get_pll2_clk(dev);
+        break;
+    case 1:
+        freq = imx6_analog_get_pll2_pfd2_clk(dev);
+        break;
+    case 2:
+        freq = imx6_analog_get_pll2_pfd0_clk(dev);
+        break;
+    case 3:
+        freq = imx6_analog_get_pll2_pfd2_clk(dev) / 2;
+        break;
+    default:
+        /* We should never get there */
+        g_assert_not_reached();
+        break;
+    }
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_ccm_get_ahb_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_analog_get_periph_clk(dev)
+           / (1 + EXTRACT(dev->ccm[CCM_CBCDR], AHB_PODF));
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_ccm_get_ipg_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_ccm_get_ahb_clk(dev)
+           / (1 + EXTRACT(dev->ccm[CCM_CBCDR], IPG_PODF));;
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint64_t imx6_ccm_get_per_clk(IMX6CCMState *dev)
+{
+    uint64_t freq = 0;
+
+    freq = imx6_ccm_get_ipg_clk(dev)
+           / (1 + EXTRACT(dev->ccm[CCM_CSCMR1], PERCLK_PODF));
+
+    DPRINTF("freq = %d\n", (uint32_t)freq);
+
+    return freq;
+}
+
+static uint32_t imx6_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
+{
+    uint32_t freq = 0;
+    IMX6CCMState *s = IMX6_CCM(dev);
+
+    switch (clock) {
+    case CLK_NONE:
+        break;
+    case CLK_IPG:
+        freq = imx6_ccm_get_ipg_clk(s);
+        break;
+    case CLK_IPG_HIGH:
+        freq = imx6_ccm_get_per_clk(s);
+        break;
+    case CLK_32k:
+        freq = CKIL_FREQ;
+        break;
+    case CLK_HIGH:
+        freq = 24000000;
+        break;
+    case CLK_HIGH_DIV:
+        freq = 24000000 / 8;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: unsupported clock %d\n",
+                      TYPE_IMX6_CCM, __func__, clock);
+        break;
+    }
+
+    DPRINTF("Clock = %d) = %d\n", clock, freq);
+
+    return freq;
+}
+
+static void imx6_ccm_reset(DeviceState *dev)
+{
+    IMX6CCMState *s = IMX6_CCM(dev);
+
+    DPRINTF("\n");
+
+    s->ccm[CCM_CCR] = 0x040116FF;
+    s->ccm[CCM_CCDR] = 0x00000000;
+    s->ccm[CCM_CSR] = 0x00000010;
+    s->ccm[CCM_CCSR] = 0x00000100;
+    s->ccm[CCM_CACRR] = 0x00000000;
+    s->ccm[CCM_CBCDR] = 0x00018D40;
+    s->ccm[CCM_CBCMR] = 0x00022324;
+    s->ccm[CCM_CSCMR1] = 0x00F00000;
+    s->ccm[CCM_CSCMR2] = 0x02B92F06;
+    s->ccm[CCM_CSCDR1] = 0x00490B00;
+    s->ccm[CCM_CS1CDR] = 0x0EC102C1;
+    s->ccm[CCM_CS2CDR] = 0x000736C1;
+    s->ccm[CCM_CDCDR] = 0x33F71F92;
+    s->ccm[CCM_CHSCCDR] = 0x0002A150;
+    s->ccm[CCM_CSCDR2] = 0x0002A150;
+    s->ccm[CCM_CSCDR3] = 0x00014841;
+    s->ccm[CCM_CDHIPR] = 0x00000000;
+    s->ccm[CCM_CTOR] = 0x00000000;
+    s->ccm[CCM_CLPCR] = 0x00000079;
+    s->ccm[CCM_CISR] = 0x00000000;
+    s->ccm[CCM_CIMR] = 0xFFFFFFFF;
+    s->ccm[CCM_CCOSR] = 0x000A0001;
+    s->ccm[CCM_CGPR] = 0x0000FE62;
+    s->ccm[CCM_CCGR0] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR1] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR2] = 0xFC3FFFFF;
+    s->ccm[CCM_CCGR3] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR4] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR5] = 0xFFFFFFFF;
+    s->ccm[CCM_CCGR6] = 0xFFFFFFFF;
+    s->ccm[CCM_CMEOR] = 0xFFFFFFFF;
+
+    s->analog[CCM_ANALOG_PLL_ARM] = 0x00013042;
+    s->analog[CCM_ANALOG_PLL_USB1] = 0x00012000;
+    s->analog[CCM_ANALOG_PLL_USB2] = 0x00012000;
+    s->analog[CCM_ANALOG_PLL_SYS] = 0x00013001;
+    s->analog[CCM_ANALOG_PLL_SYS_SS] = 0x00000000;
+    s->analog[CCM_ANALOG_PLL_SYS_NUM] = 0x00000000;
+    s->analog[CCM_ANALOG_PLL_SYS_DENOM] = 0x00000012;
+    s->analog[CCM_ANALOG_PLL_AUDIO] = 0x00011006;
+    s->analog[CCM_ANALOG_PLL_AUDIO_NUM] = 0x05F5E100;
+    s->analog[CCM_ANALOG_PLL_AUDIO_DENOM] = 0x2964619C;
+    s->analog[CCM_ANALOG_PLL_VIDEO] = 0x0001100C;
+    s->analog[CCM_ANALOG_PLL_VIDEO_NUM] = 0x05F5E100;
+    s->analog[CCM_ANALOG_PLL_VIDEO_DENOM] = 0x10A24447;
+    s->analog[CCM_ANALOG_PLL_MLB] = 0x00010000;
+    s->analog[CCM_ANALOG_PLL_ENET] = 0x00011001;
+    s->analog[CCM_ANALOG_PFD_480] = 0x1311100C;
+    s->analog[CCM_ANALOG_PFD_528] = 0x1018101B;
+
+    s->analog[PMU_REG_1P1] = 0x00001073;
+    s->analog[PMU_REG_3P0] = 0x00000F74;
+    s->analog[PMU_REG_2P5] = 0x00005071;
+    s->analog[PMU_REG_CORE] = 0x00402010;
+    s->analog[PMU_MISC0] = 0x04000000;
+    s->analog[PMU_MISC1] = 0x00000000;
+    s->analog[PMU_MISC2] = 0x00272727;
+
+    s->analog[USB_ANALOG_USB1_VBUS_DETECT] = 0x00000004;
+    s->analog[USB_ANALOG_USB1_CHRG_DETECT] = 0x00000000;
+    s->analog[USB_ANALOG_USB1_VBUS_DETECT_STAT] = 0x00000000;
+    s->analog[USB_ANALOG_USB1_CHRG_DETECT_STAT] = 0x00000000;
+    s->analog[USB_ANALOG_USB1_MISC] = 0x00000002;
+    s->analog[USB_ANALOG_USB2_VBUS_DETECT] = 0x00000004;
+    s->analog[USB_ANALOG_USB2_CHRG_DETECT] = 0x00000000;
+    s->analog[USB_ANALOG_USB2_MISC] = 0x00000002;
+    s->analog[USB_ANALOG_DIGPROG] = 0x00000000;
+
+    /* all PLLs need to be locked */
+    s->analog[CCM_ANALOG_PLL_ARM]   |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_USB1]  |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_USB2]  |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_SYS]   |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_AUDIO] |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_VIDEO] |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_MLB]   |= CCM_ANALOG_PLL_LOCK;
+    s->analog[CCM_ANALOG_PLL_ENET]  |= CCM_ANALOG_PLL_LOCK;
+}
+
+static uint64_t imx6_ccm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    value = s->ccm[index];
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx6_ccm_reg_name(index), value);
+
+    return (uint64_t)value;
+}
+
+static void imx6_ccm_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx6_ccm_reg_name(index),
+            (uint32_t)value);
+
+    /*
+     * We will do a better implementation later. In particular some bits
+     * cannot be written to.
+     */
+    s->ccm[index] = (uint32_t)value;
+}
+
+static uint64_t imx6_analog_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value;
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    switch (index) {
+    case CCM_ANALOG_PLL_ARM_SET:
+    case CCM_ANALOG_PLL_USB1_SET:
+    case CCM_ANALOG_PLL_USB2_SET:
+    case CCM_ANALOG_PLL_SYS_SET:
+    case CCM_ANALOG_PLL_AUDIO_SET:
+    case CCM_ANALOG_PLL_VIDEO_SET:
+    case CCM_ANALOG_PLL_MLB_SET:
+    case CCM_ANALOG_PLL_ENET_SET:
+    case CCM_ANALOG_PFD_480_SET:
+    case CCM_ANALOG_PFD_528_SET:
+    case CCM_ANALOG_MISC0_SET:
+    case PMU_MISC1_SET:
+    case CCM_ANALOG_MISC2_SET:
+    case USB_ANALOG_USB1_VBUS_DETECT_SET:
+    case USB_ANALOG_USB1_CHRG_DETECT_SET:
+    case USB_ANALOG_USB1_MISC_SET:
+    case USB_ANALOG_USB2_VBUS_DETECT_SET:
+    case USB_ANALOG_USB2_CHRG_DETECT_SET:
+    case USB_ANALOG_USB2_MISC_SET:
+        /*
+         * All REG_NAME_SET register access are in fact targeting the
+         * the REG_NAME register.
+         */
+        value = s->analog[index - 1];
+        break;
+    case CCM_ANALOG_PLL_ARM_CLR:
+    case CCM_ANALOG_PLL_USB1_CLR:
+    case CCM_ANALOG_PLL_USB2_CLR:
+    case CCM_ANALOG_PLL_SYS_CLR:
+    case CCM_ANALOG_PLL_AUDIO_CLR:
+    case CCM_ANALOG_PLL_VIDEO_CLR:
+    case CCM_ANALOG_PLL_MLB_CLR:
+    case CCM_ANALOG_PLL_ENET_CLR:
+    case CCM_ANALOG_PFD_480_CLR:
+    case CCM_ANALOG_PFD_528_CLR:
+    case CCM_ANALOG_MISC0_CLR:
+    case PMU_MISC1_CLR:
+    case CCM_ANALOG_MISC2_CLR:
+    case USB_ANALOG_USB1_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB1_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB1_MISC_CLR:
+    case USB_ANALOG_USB2_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB2_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB2_MISC_CLR:
+        /*
+         * All REG_NAME_CLR register access are in fact targeting the
+         * the REG_NAME register.
+         */
+        value = s->analog[index - 2];
+        break;
+    case CCM_ANALOG_PLL_ARM_TOG:
+    case CCM_ANALOG_PLL_USB1_TOG:
+    case CCM_ANALOG_PLL_USB2_TOG:
+    case CCM_ANALOG_PLL_SYS_TOG:
+    case CCM_ANALOG_PLL_AUDIO_TOG:
+    case CCM_ANALOG_PLL_VIDEO_TOG:
+    case CCM_ANALOG_PLL_MLB_TOG:
+    case CCM_ANALOG_PLL_ENET_TOG:
+    case CCM_ANALOG_PFD_480_TOG:
+    case CCM_ANALOG_PFD_528_TOG:
+    case CCM_ANALOG_MISC0_TOG:
+    case PMU_MISC1_TOG:
+    case CCM_ANALOG_MISC2_TOG:
+    case USB_ANALOG_USB1_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB1_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB1_MISC_TOG:
+    case USB_ANALOG_USB2_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB2_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB2_MISC_TOG:
+        /*
+         * All REG_NAME_TOG register access are in fact targeting the
+         * the REG_NAME register.
+         */
+        value = s->analog[index - 3];
+        break;
+    default:
+        value = s->analog[index];
+        break;
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx6_analog_reg_name(index), value);
+
+    return (uint64_t)value;
+}
+
+static void imx6_analog_write(void *opaque, hwaddr offset, uint64_t value,
+                              unsigned size)
+{
+    uint32_t index = offset >> 2;
+    IMX6CCMState *s = (IMX6CCMState *)opaque;
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx6_analog_reg_name(index),
+            (uint32_t)value);
+
+    switch (index) {
+    case CCM_ANALOG_PLL_ARM_SET:
+    case CCM_ANALOG_PLL_USB1_SET:
+    case CCM_ANALOG_PLL_USB2_SET:
+    case CCM_ANALOG_PLL_SYS_SET:
+    case CCM_ANALOG_PLL_AUDIO_SET:
+    case CCM_ANALOG_PLL_VIDEO_SET:
+    case CCM_ANALOG_PLL_MLB_SET:
+    case CCM_ANALOG_PLL_ENET_SET:
+    case CCM_ANALOG_PFD_480_SET:
+    case CCM_ANALOG_PFD_528_SET:
+    case CCM_ANALOG_MISC0_SET:
+    case PMU_MISC1_SET:
+    case CCM_ANALOG_MISC2_SET:
+    case USB_ANALOG_USB1_VBUS_DETECT_SET:
+    case USB_ANALOG_USB1_CHRG_DETECT_SET:
+    case USB_ANALOG_USB1_MISC_SET:
+    case USB_ANALOG_USB2_VBUS_DETECT_SET:
+    case USB_ANALOG_USB2_CHRG_DETECT_SET:
+    case USB_ANALOG_USB2_MISC_SET:
+        /*
+         * All REG_NAME_SET register access are in fact targeting the
+         * the REG_NAME register. So we change the value of the
+         * REG_NAME register, setting bits passed in the value.
+         */
+        s->analog[index - 1] |= value;
+        break;
+    case CCM_ANALOG_PLL_ARM_CLR:
+    case CCM_ANALOG_PLL_USB1_CLR:
+    case CCM_ANALOG_PLL_USB2_CLR:
+    case CCM_ANALOG_PLL_SYS_CLR:
+    case CCM_ANALOG_PLL_AUDIO_CLR:
+    case CCM_ANALOG_PLL_VIDEO_CLR:
+    case CCM_ANALOG_PLL_MLB_CLR:
+    case CCM_ANALOG_PLL_ENET_CLR:
+    case CCM_ANALOG_PFD_480_CLR:
+    case CCM_ANALOG_PFD_528_CLR:
+    case CCM_ANALOG_MISC0_CLR:
+    case PMU_MISC1_CLR:
+    case CCM_ANALOG_MISC2_CLR:
+    case USB_ANALOG_USB1_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB1_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB1_MISC_CLR:
+    case USB_ANALOG_USB2_VBUS_DETECT_CLR:
+    case USB_ANALOG_USB2_CHRG_DETECT_CLR:
+    case USB_ANALOG_USB2_MISC_CLR:
+        /*
+         * All REG_NAME_CLR register access are in fact targeting the
+         * the REG_NAME register. So we change the value of the
+         * REG_NAME register, unsetting bits passed in the value.
+         */
+        s->analog[index - 2] &= ~value;
+        break;
+    case CCM_ANALOG_PLL_ARM_TOG:
+    case CCM_ANALOG_PLL_USB1_TOG:
+    case CCM_ANALOG_PLL_USB2_TOG:
+    case CCM_ANALOG_PLL_SYS_TOG:
+    case CCM_ANALOG_PLL_AUDIO_TOG:
+    case CCM_ANALOG_PLL_VIDEO_TOG:
+    case CCM_ANALOG_PLL_MLB_TOG:
+    case CCM_ANALOG_PLL_ENET_TOG:
+    case CCM_ANALOG_PFD_480_TOG:
+    case CCM_ANALOG_PFD_528_TOG:
+    case CCM_ANALOG_MISC0_TOG:
+    case PMU_MISC1_TOG:
+    case CCM_ANALOG_MISC2_TOG:
+    case USB_ANALOG_USB1_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB1_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB1_MISC_TOG:
+    case USB_ANALOG_USB2_VBUS_DETECT_TOG:
+    case USB_ANALOG_USB2_CHRG_DETECT_TOG:
+    case USB_ANALOG_USB2_MISC_TOG:
+        /*
+         * All REG_NAME_TOG register access are in fact targeting the
+         * the REG_NAME register. So we change the value of the
+         * REG_NAME register, toggling bits passed in the value.
+         */
+        s->analog[index - 3] ^= value;
+        break;
+    default:
+        /*
+         * We will do a better implementation later. In particular some bits
+         * cannot be written to.
+         */
+        s->analog[index] = value;
+        break;
+    }
+}
+
+static const struct MemoryRegionOps imx6_ccm_ops = {
+    .read = imx6_ccm_read,
+    .write = imx6_ccm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static const struct MemoryRegionOps imx6_analog_ops = {
+    .read = imx6_analog_read,
+    .write = imx6_analog_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static void imx6_ccm_init(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+    SysBusDevice *sd = SYS_BUS_DEVICE(obj);
+    IMX6CCMState *s = IMX6_CCM(obj);
+
+    /* initialize a container for the all memory range */
+    memory_region_init(&s->container, OBJECT(dev), TYPE_IMX6_CCM, 0x5000);
+
+    /* We initialize an IO memory region for the CCM part */
+    memory_region_init_io(&s->ioccm, OBJECT(dev), &imx6_ccm_ops, s,
+                          TYPE_IMX6_CCM ".ccm", CCM_MAX * sizeof(uint32_t));
+
+    /* Add the CCM as a subregion at offset 0 */
+    memory_region_add_subregion(&s->container, 0, &s->ioccm);
+
+    /* We initialize an IO memory region for the ANALOG part */
+    memory_region_init_io(&s->ioanalog, OBJECT(dev), &imx6_analog_ops, s,
+                          TYPE_IMX6_CCM ".analog",
+                          CCM_ANALOG_MAX * sizeof(uint32_t));
+
+    /* Add the ANALOG as a subregion at offset 0x4000 */
+    memory_region_add_subregion(&s->container, 0x4000, &s->ioanalog);
+
+    sysbus_init_mmio(sd, &s->container);
+}
+
+static void imx6_ccm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    IMXCCMClass *ccm = IMX_CCM_CLASS(klass);
+
+    dc->reset = imx6_ccm_reset;
+    dc->vmsd = &vmstate_imx6_ccm;
+    dc->desc = "i.MX6 Clock Control Module";
+
+    ccm->get_clock_frequency = imx6_ccm_get_clock_frequency;
+}
+
+static const TypeInfo imx6_ccm_info = {
+    .name          = TYPE_IMX6_CCM,
+    .parent        = TYPE_IMX_CCM,
+    .instance_size = sizeof(IMX6CCMState),
+    .instance_init = imx6_ccm_init,
+    .class_init    = imx6_ccm_class_init,
+};
+
+static void imx6_ccm_register_types(void)
+{
+    type_register_static(&imx6_ccm_info);
+}
+
+type_init(imx6_ccm_register_types)
diff --git a/hw/misc/imx6_src.c b/hw/misc/imx6_src.c
new file mode 100644
index 0000000..8bb6829
--- /dev/null
+++ b/hw/misc/imx6_src.c
@@ -0,0 +1,265 @@
+/*
+ * IMX6 System Reset Controller
+ *
+ * Copyright (c) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/imx6_src.h"
+#include "sysemu/sysemu.h"
+#include "qemu/bitops.h"
+#include "qemu/log.h"
+#include "arm-powerctl.h"
+
+#ifndef DEBUG_IMX6_SRC
+#define DEBUG_IMX6_SRC 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX6_SRC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX6_SRC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx6_src_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case SRC_SCR:
+        return "SRC_SCR";
+    case SRC_SBMR1:
+        return "SRC_SBMR1";
+    case SRC_SRSR:
+        return "SRC_SRSR";
+    case SRC_SISR:
+        return "SRC_SISR";
+    case SRC_SIMR:
+        return "SRC_SIMR";
+    case SRC_SBMR2:
+        return "SRC_SBMR2";
+    case SRC_GPR1:
+        return "SRC_GPR1";
+    case SRC_GPR2:
+        return "SRC_GPR2";
+    case SRC_GPR3:
+        return "SRC_GPR3";
+    case SRC_GPR4:
+        return "SRC_GPR4";
+    case SRC_GPR5:
+        return "SRC_GPR5";
+    case SRC_GPR6:
+        return "SRC_GPR6";
+    case SRC_GPR7:
+        return "SRC_GPR7";
+    case SRC_GPR8:
+        return "SRC_GPR8";
+    case SRC_GPR9:
+        return "SRC_GPR9";
+    case SRC_GPR10:
+        return "SRC_GPR10";
+    default:
+        sprintf(unknown, "%d ?", reg);
+        return unknown;
+    }
+}
+
+static const VMStateDescription vmstate_imx6_src = {
+    .name = TYPE_IMX6_SRC,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, IMX6SRCState, SRC_MAX),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static void imx6_src_reset(DeviceState *dev)
+{
+    IMX6SRCState *s = IMX6_SRC(dev);
+
+    DPRINTF("\n");
+
+    memset(s->regs, 0, sizeof(s->regs));
+
+    /* Set reset values */
+    s->regs[SRC_SCR] = 0x521;
+    s->regs[SRC_SRSR] = 0x1;
+    s->regs[SRC_SIMR] = 0x1F;
+}
+
+static uint64_t imx6_src_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMX6SRCState *s = (IMX6SRCState *)opaque;
+    uint32_t index = offset >> 2;
+
+    if (index < SRC_MAX) {
+        value = s->regs[index];
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX6_SRC, __func__, offset);
+
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx6_src_reg_name(index), value);
+
+    return value;
+}
+
+static void imx6_src_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    IMX6SRCState *s = (IMX6SRCState *)opaque;
+    uint32_t index = offset >> 2;
+    unsigned long change_mask;
+    unsigned long current_value = value;
+
+    if (index >=  SRC_MAX) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX6_SRC, __func__, offset);
+        return;
+    }
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx6_src_reg_name(index),
+            (uint32_t)current_value);
+
+    change_mask = s->regs[index] ^ (uint32_t)current_value;
+
+    switch (index) {
+    case SRC_SCR:
+        /*
+         * On real hardware when the system reset controller starts a
+         * secondary CPU it runs through some boot ROM code which reads
+         * the SRC_GPRX registers controlling the start address and branches
+         * to it.
+         * Here we are taking a short cut and branching directly to the
+         * requested address (we don't want to run the boot ROM code inside
+         * QEMU)
+         */
+        if (EXTRACT(change_mask, CORE3_ENABLE)) {
+            if (EXTRACT(current_value, CORE3_ENABLE)) {
+                /* CORE 3 is brought up */
+                arm_set_cpu_on(3, s->regs[SRC_GPR7], s->regs[SRC_GPR8],
+                               3, false);
+            } else {
+                /* CORE 3 is shut down */
+                arm_set_cpu_off(3);
+            }
+            /* We clear the reset bits as the processor changed state */
+            clear_bit(CORE3_RST_SHIFT, &current_value);
+            clear_bit(CORE3_RST_SHIFT, &change_mask);
+        }
+        if (EXTRACT(change_mask, CORE2_ENABLE)) {
+            if (EXTRACT(current_value, CORE2_ENABLE)) {
+                /* CORE 2 is brought up */
+                arm_set_cpu_on(2, s->regs[SRC_GPR5], s->regs[SRC_GPR6],
+                               3, false);
+            } else {
+                /* CORE 3 is shut down */
+                arm_set_cpu_off(2);
+            }
+            /* We clear the reset bits as the processor changed state */
+            clear_bit(CORE2_RST_SHIFT, &current_value);
+            clear_bit(CORE2_RST_SHIFT, &change_mask);
+        }
+        if (EXTRACT(change_mask, CORE1_ENABLE)) {
+            if (EXTRACT(current_value, CORE1_ENABLE)) {
+                /* CORE 1 is brought up */
+                arm_set_cpu_on(1, s->regs[SRC_GPR3], s->regs[SRC_GPR4],
+                               3, false);
+            } else {
+                /* CORE 3 is shut down */
+                arm_set_cpu_off(1);
+            }
+            /* We clear the reset bits as the processor changed state */
+            clear_bit(CORE1_RST_SHIFT, &current_value);
+            clear_bit(CORE1_RST_SHIFT, &change_mask);
+        }
+        if (EXTRACT(change_mask, CORE0_RST)) {
+            arm_reset_cpu(0);
+            clear_bit(CORE0_RST_SHIFT, &current_value);
+        }
+        if (EXTRACT(change_mask, CORE1_RST)) {
+            arm_reset_cpu(1);
+            clear_bit(CORE1_RST_SHIFT, &current_value);
+        }
+        if (EXTRACT(change_mask, CORE2_RST)) {
+            arm_reset_cpu(2);
+            clear_bit(CORE2_RST_SHIFT, &current_value);
+        }
+        if (EXTRACT(change_mask, CORE3_RST)) {
+            arm_reset_cpu(3);
+            clear_bit(CORE3_RST_SHIFT, &current_value);
+        }
+        if (EXTRACT(change_mask, SW_IPU2_RST)) {
+            /* We pretend the IPU2 is reset */
+            clear_bit(SW_IPU2_RST_SHIFT, &current_value);
+        }
+        if (EXTRACT(change_mask, SW_IPU1_RST)) {
+            /* We pretend the IPU1 is reset */
+            clear_bit(SW_IPU1_RST_SHIFT, &current_value);
+        }
+        s->regs[index] = current_value;
+        break;
+    default:
+        s->regs[index] = current_value;
+        break;
+    }
+}
+
+static const struct MemoryRegionOps imx6_src_ops = {
+    .read = imx6_src_read,
+    .write = imx6_src_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static void imx6_src_realize(DeviceState *dev, Error **errp)
+{
+    IMX6SRCState *s = IMX6_SRC(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx6_src_ops, s,
+                          TYPE_IMX6_SRC, 0x1000);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+}
+
+static void imx6_src_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = imx6_src_realize;
+    dc->reset = imx6_src_reset;
+    dc->vmsd = &vmstate_imx6_src;
+    dc->desc = "i.MX6 System Reset Controller";
+}
+
+static const TypeInfo imx6_src_info = {
+    .name          = TYPE_IMX6_SRC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMX6SRCState),
+    .class_init    = imx6_src_class_init,
+};
+
+static void imx6_src_register_types(void)
+{
+    type_register_static(&imx6_src_info);
+}
+
+type_init(imx6_src_register_types)
diff --git a/hw/misc/imx_ccm.c b/hw/misc/imx_ccm.c
index 0920288..7f239a4 100644
--- a/hw/misc/imx_ccm.c
+++ b/hw/misc/imx_ccm.c
@@ -2,138 +2,52 @@
  * IMX31 Clock Control Module
  *
  * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
- * To get the timer frequencies right, we need to emulate at least part of
- * the CCM.
+ * This is an abstract base class used to get a common interface to
+ * retrieve the CCM frequencies from the various i.MX SOC.
  */
 
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "sysemu/sysemu.h"
-#include "hw/arm/imx.h"
+#include "qemu/osdep.h"
+#include "hw/misc/imx_ccm.h"
+#include "qemu/log.h"
 
-#define CKIH_FREQ 26000000 /* 26MHz crystal input */
-#define CKIL_FREQ    32768 /* nominal 32khz clock */
-
-
-//#define DEBUG_CCM 1
-#ifdef DEBUG_CCM
-#define DPRINTF(fmt, args...) \
-do { printf("imx_ccm: " fmt , ##args); } while (0)
-#else
-#define DPRINTF(fmt, args...) do {} while (0)
+#ifndef DEBUG_IMX_CCM
+#define DEBUG_IMX_CCM 0
 #endif
 
-static int imx_ccm_post_load(void *opaque, int version_id);
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_CCM) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_CCM, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
-#define TYPE_IMX_CCM "imx_ccm"
-#define IMX_CCM(obj) OBJECT_CHECK(IMXCCMState, (obj), TYPE_IMX_CCM)
 
-typedef struct IMXCCMState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-
-    uint32_t ccmr;
-    uint32_t pdr0;
-    uint32_t pdr1;
-    uint32_t mpctl;
-    uint32_t spctl;
-    uint32_t cgr[3];
-    uint32_t pmcr0;
-    uint32_t pmcr1;
-
-    /* Frequencies precalculated on register changes */
-    uint32_t pll_refclk_freq;
-    uint32_t mcu_clk_freq;
-    uint32_t hsp_clk_freq;
-    uint32_t ipg_clk_freq;
-} IMXCCMState;
-
-static const VMStateDescription vmstate_imx_ccm = {
-    .name = "imx-ccm",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT32(ccmr, IMXCCMState),
-        VMSTATE_UINT32(pdr0, IMXCCMState),
-        VMSTATE_UINT32(pdr1, IMXCCMState),
-        VMSTATE_UINT32(mpctl, IMXCCMState),
-        VMSTATE_UINT32(spctl, IMXCCMState),
-        VMSTATE_UINT32_ARRAY(cgr, IMXCCMState, 3),
-        VMSTATE_UINT32(pmcr0, IMXCCMState),
-        VMSTATE_UINT32(pmcr1, IMXCCMState),
-        VMSTATE_UINT32(pll_refclk_freq, IMXCCMState),
-        VMSTATE_END_OF_LIST()
-    },
-    .post_load = imx_ccm_post_load,
-};
-
-/* CCMR */
-#define CCMR_FPME (1<<0)
-#define CCMR_MPE  (1<<3)
-#define CCMR_MDS  (1<<7)
-#define CCMR_FPMF (1<<26)
-#define CCMR_PRCS (3<<1)
-
-/* PDR0 */
-#define PDR0_MCU_PODF_SHIFT (0)
-#define PDR0_MCU_PODF_MASK (0x7)
-#define PDR0_MAX_PODF_SHIFT (3)
-#define PDR0_MAX_PODF_MASK (0x7)
-#define PDR0_IPG_PODF_SHIFT (6)
-#define PDR0_IPG_PODF_MASK (0x3)
-#define PDR0_NFC_PODF_SHIFT (8)
-#define PDR0_NFC_PODF_MASK (0x7)
-#define PDR0_HSP_PODF_SHIFT (11)
-#define PDR0_HSP_PODF_MASK (0x7)
-#define PDR0_PER_PODF_SHIFT (16)
-#define PDR0_PER_PODF_MASK (0x1f)
-#define PDR0_CSI_PODF_SHIFT (23)
-#define PDR0_CSI_PODF_MASK (0x1ff)
-
-#define EXTRACT(value, name) (((value) >> PDR0_##name##_PODF_SHIFT) \
-                              & PDR0_##name##_PODF_MASK)
-#define INSERT(value, name) (((value) & PDR0_##name##_PODF_MASK) << \
-                             PDR0_##name##_PODF_SHIFT)
-/* PLL control registers */
-#define PD(v) (((v) >> 26) & 0xf)
-#define MFD(v) (((v) >> 16) & 0x3ff)
-#define MFI(v) (((v) >> 10) & 0xf);
-#define MFN(v) ((v) & 0x3ff)
-
-#define PLL_PD(x)               (((x) & 0xf) << 26)
-#define PLL_MFD(x)              (((x) & 0x3ff) << 16)
-#define PLL_MFI(x)              (((x) & 0xf) << 10)
-#define PLL_MFN(x)              (((x) & 0x3ff) << 0)
-
-uint32_t imx_clock_frequency(DeviceState *dev, IMXClk clock)
+uint32_t imx_ccm_get_clock_frequency(IMXCCMState *dev, IMXClk clock)
 {
-    IMXCCMState *s = IMX_CCM(dev);
+    uint32_t freq = 0;
+    IMXCCMClass *klass = IMX_GET_CLASS(dev);
 
-    switch (clock) {
-    case NOCLK:
-        return 0;
-    case MCU:
-        return s->mcu_clk_freq;
-    case HSP:
-        return s->hsp_clk_freq;
-    case IPG:
-        return s->ipg_clk_freq;
-    case CLK_32k:
-        return CKIL_FREQ;
+    if (klass->get_clock_frequency) {
+        freq = klass->get_clock_frequency(dev, clock);
     }
-    return 0;
+
+    DPRINTF("(clock = %d) = %d\n", clock, freq);
+
+    return freq;
 }
 
 /*
  * Calculate PLL output frequency
  */
-static uint32_t calc_pll(uint32_t pllreg, uint32_t base_freq)
+uint32_t imx_ccm_calc_pll(uint32_t pllreg, uint32_t base_freq)
 {
+    int32_t freq;
     int32_t mfn = MFN(pllreg);  /* Numerator */
     uint32_t mfi = MFI(pllreg); /* Integer part */
     uint32_t mfd = 1 + MFD(pllreg); /* Denominator */
@@ -142,180 +56,26 @@
     if (mfi < 5) {
         mfi = 5;
     }
+
     /* mfn is 10-bit signed twos-complement */
     mfn <<= 32 - 10;
     mfn >>= 32 - 10;
 
-    return ((2 * (base_freq >> 10) * (mfi * mfd + mfn)) /
+    freq = ((2 * (base_freq >> 10) * (mfi * mfd + mfn)) /
             (mfd * pd)) << 10;
-}
 
-static void update_clocks(IMXCCMState *s)
-{
-    /*
-     * If we ever emulate more clocks, this should switch to a data-driven
-     * approach
-     */
+    DPRINTF("(pllreg = 0x%08x, base_freq = %d) = %d\n", pllreg, base_freq,
+            freq);
 
-    if ((s->ccmr & CCMR_PRCS) == 2) {
-        s->pll_refclk_freq = CKIL_FREQ * 1024;
-    } else {
-        s->pll_refclk_freq = CKIH_FREQ;
-    }
-
-    /* ipg_clk_arm aka MCU clock */
-    if ((s->ccmr & CCMR_MDS) || !(s->ccmr & CCMR_MPE)) {
-        s->mcu_clk_freq = s->pll_refclk_freq;
-    } else {
-        s->mcu_clk_freq = calc_pll(s->mpctl, s->pll_refclk_freq);
-    }
-
-    /* High-speed clock */
-    s->hsp_clk_freq = s->mcu_clk_freq / (1 + EXTRACT(s->pdr0, HSP));
-    s->ipg_clk_freq = s->hsp_clk_freq / (1 + EXTRACT(s->pdr0, IPG));
-
-    DPRINTF("Clocks: mcu %uMHz, HSP %uMHz, IPG %uHz\n",
-            s->mcu_clk_freq / 1000000,
-            s->hsp_clk_freq / 1000000,
-            s->ipg_clk_freq);
-}
-
-static void imx_ccm_reset(DeviceState *dev)
-{
-    IMXCCMState *s = IMX_CCM(dev);
-
-    s->ccmr = 0x074b0b7b;
-    s->pdr0 = 0xff870b48;
-    s->pdr1 = 0x49fcfe7f;
-    s->mpctl = PLL_PD(1) | PLL_MFD(0) | PLL_MFI(6) | PLL_MFN(0);
-    s->cgr[0] = s->cgr[1] = s->cgr[2] = 0xffffffff;
-    s->spctl = PLL_PD(1) | PLL_MFD(4) | PLL_MFI(0xc) | PLL_MFN(1);
-    s->pmcr0 = 0x80209828;
-
-    update_clocks(s);
-}
-
-static uint64_t imx_ccm_read(void *opaque, hwaddr offset,
-                                unsigned size)
-{
-    IMXCCMState *s = (IMXCCMState *)opaque;
-
-    DPRINTF("read(offset=%x)", offset >> 2);
-    switch (offset >> 2) {
-    case 0: /* CCMR */
-        DPRINTF(" ccmr = 0x%x\n", s->ccmr);
-        return s->ccmr;
-    case 1:
-        DPRINTF(" pdr0 = 0x%x\n", s->pdr0);
-        return s->pdr0;
-    case 2:
-        DPRINTF(" pdr1 = 0x%x\n", s->pdr1);
-        return s->pdr1;
-    case 4:
-        DPRINTF(" mpctl = 0x%x\n", s->mpctl);
-        return s->mpctl;
-    case 6:
-        DPRINTF(" spctl = 0x%x\n", s->spctl);
-        return s->spctl;
-    case 8:
-        DPRINTF(" cgr0 = 0x%x\n", s->cgr[0]);
-        return s->cgr[0];
-    case 9:
-        DPRINTF(" cgr1 = 0x%x\n", s->cgr[1]);
-        return s->cgr[1];
-    case 10:
-        DPRINTF(" cgr2 = 0x%x\n", s->cgr[2]);
-        return s->cgr[2];
-    case 18: /* LTR1 */
-        return 0x00004040;
-    case 23:
-        DPRINTF(" pcmr0 = 0x%x\n", s->pmcr0);
-        return s->pmcr0;
-    }
-    DPRINTF(" return 0\n");
-    return 0;
-}
-
-static void imx_ccm_write(void *opaque, hwaddr offset,
-                          uint64_t value, unsigned size)
-{
-    IMXCCMState *s = (IMXCCMState *)opaque;
-
-    DPRINTF("write(offset=%x, value = %x)\n",
-            offset >> 2, (unsigned int)value);
-    switch (offset >> 2) {
-    case 0:
-        s->ccmr = CCMR_FPMF | (value & 0x3b6fdfff);
-        break;
-    case 1:
-        s->pdr0 = value & 0xff9f3fff;
-        break;
-    case 2:
-        s->pdr1 = value;
-        break;
-    case 4:
-        s->mpctl = value & 0xbfff3fff;
-        break;
-    case 6:
-        s->spctl = value & 0xbfff3fff;
-        break;
-    case 8:
-        s->cgr[0] = value;
-        return;
-    case 9:
-        s->cgr[1] = value;
-        return;
-    case 10:
-        s->cgr[2] = value;
-        return;
-
-    default:
-        return;
-    }
-    update_clocks(s);
-}
-
-static const struct MemoryRegionOps imx_ccm_ops = {
-    .read = imx_ccm_read,
-    .write = imx_ccm_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static int imx_ccm_init(SysBusDevice *dev)
-{
-    IMXCCMState *s = IMX_CCM(dev);
-
-    memory_region_init_io(&s->iomem, OBJECT(dev), &imx_ccm_ops, s,
-                          "imx_ccm", 0x1000);
-    sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
-}
-
-static int imx_ccm_post_load(void *opaque, int version_id)
-{
-    IMXCCMState *s = (IMXCCMState *)opaque;
-
-    update_clocks(s);
-    return 0;
-}
-
-static void imx_ccm_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
-
-    sbc->init = imx_ccm_init;
-    dc->reset = imx_ccm_reset;
-    dc->vmsd = &vmstate_imx_ccm;
-    dc->desc = "i.MX Clock Control Module";
+    return freq;
 }
 
 static const TypeInfo imx_ccm_info = {
-    .name = TYPE_IMX_CCM,
-    .parent = TYPE_SYS_BUS_DEVICE,
+    .name          = TYPE_IMX_CCM,
+    .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXCCMState),
-    .class_init = imx_ccm_class_init,
+    .class_size    = sizeof(IMXCCMClass),
+    .abstract      = true,
 };
 
 static void imx_ccm_register_types(void)
diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index 5d272c8..40a2ebc 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -16,39 +16,54 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "sysemu/kvm.h"
 #include "migration/migration.h"
-#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "qemu/event_notifier.h"
-#include "qemu/fifo8.h"
+#include "qom/object_interfaces.h"
 #include "sysemu/char.h"
+#include "sysemu/hostmem.h"
+#include "sysemu/qtest.h"
+#include "qapi/visitor.h"
 
-#include <sys/mman.h>
-#include <sys/types.h>
-#include <limits.h>
+#include "hw/misc/ivshmem.h"
 
 #define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
 #define PCI_DEVICE_ID_IVSHMEM   0x1110
 
+#define IVSHMEM_MAX_PEERS UINT16_MAX
 #define IVSHMEM_IOEVENTFD   0
 #define IVSHMEM_MSI     1
 
-#define IVSHMEM_PEER    0
-#define IVSHMEM_MASTER  1
-
 #define IVSHMEM_REG_BAR_SIZE 0x100
 
-//#define DEBUG_IVSHMEM
-#ifdef DEBUG_IVSHMEM
-#define IVSHMEM_DPRINTF(fmt, ...)        \
-    do {printf("IVSHMEM: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define IVSHMEM_DPRINTF(fmt, ...)
-#endif
+#define IVSHMEM_DEBUG 0
+#define IVSHMEM_DPRINTF(fmt, ...)                       \
+    do {                                                \
+        if (IVSHMEM_DEBUG) {                            \
+            printf("IVSHMEM: " fmt, ## __VA_ARGS__);    \
+        }                                               \
+    } while (0)
+
+#define TYPE_IVSHMEM_COMMON "ivshmem-common"
+#define IVSHMEM_COMMON(obj) \
+    OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_COMMON)
+
+#define TYPE_IVSHMEM_PLAIN "ivshmem-plain"
+#define IVSHMEM_PLAIN(obj) \
+    OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_PLAIN)
+
+#define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell"
+#define IVSHMEM_DOORBELL(obj) \
+    OBJECT_CHECK(IVShmemState, (obj), TYPE_IVSHMEM_DOORBELL)
 
 #define TYPE_IVSHMEM "ivshmem"
 #define IVSHMEM(obj) \
@@ -59,51 +74,50 @@
     EventNotifier *eventfds;
 } Peer;
 
-typedef struct EventfdEntry {
+typedef struct MSIVector {
     PCIDevice *pdev;
-    int vector;
-} EventfdEntry;
+    int virq;
+} MSIVector;
 
 typedef struct IVShmemState {
     /*< private >*/
     PCIDevice parent_obj;
     /*< public >*/
 
+    uint32_t features;
+
+    /* exactly one of these two may be set */
+    HostMemoryBackend *hostmem; /* with interrupts */
+    CharDriverState *server_chr; /* without interrupts */
+
+    /* registers */
     uint32_t intrmask;
     uint32_t intrstatus;
-    uint32_t doorbell;
-
-    CharDriverState **eventfd_chr;
-    CharDriverState *server_chr;
-    Fifo8 incoming_fifo;
-    MemoryRegion ivshmem_mmio;
-
-    /* We might need to register the BAR before we actually have the memory.
-     * So prepare a container MemoryRegion for the BAR immediately and
-     * add a subregion when we have the memory.
-     */
-    MemoryRegion bar;
-    MemoryRegion ivshmem;
-    uint64_t ivshmem_size; /* size of shared memory region */
-    uint32_t ivshmem_attr;
-    uint32_t ivshmem_64bit;
-    int shm_fd; /* shared memory file descriptor */
-
-    Peer *peers;
-    int nb_peers; /* how many guests we have space for */
-    int max_peer; /* maximum numbered peer */
-
     int vm_id;
-    uint32_t vectors;
-    uint32_t features;
-    EventfdEntry *eventfd_table;
 
+    /* BARs */
+    MemoryRegion ivshmem_mmio;  /* BAR 0 (registers) */
+    MemoryRegion *ivshmem_bar2; /* BAR 2 (shared memory) */
+    MemoryRegion server_bar2;   /* used with server_chr */
+
+    /* interrupt support */
+    Peer *peers;
+    int nb_peers;               /* space in @peers[] */
+    uint32_t vectors;
+    MSIVector *msi_vectors;
+    uint64_t msg_buf;           /* buffer for receiving server messages */
+    int msg_buffered_bytes;     /* #bytes in @msg_buf */
+
+    /* migration stuff */
+    OnOffAuto master;
     Error *migration_blocker;
 
-    char * shmobj;
-    char * sizearg;
-    char * role;
-    int role_val;   /* scalar to avoid multiple string comparisons */
+    /* legacy cruft */
+    char *role;
+    char *shmobj;
+    char *sizearg;
+    size_t legacy_size;
+    uint32_t not_legacy_32bit;
 } IVShmemState;
 
 /* registers for the Inter-VM shared memory device */
@@ -119,16 +133,34 @@
     return (ivs->features & (1 << feature));
 }
 
-static inline bool is_power_of_two(uint64_t x) {
-    return (x & (x - 1)) == 0;
+static inline bool ivshmem_is_master(IVShmemState *s)
+{
+    assert(s->master != ON_OFF_AUTO_AUTO);
+    return s->master == ON_OFF_AUTO_ON;
 }
 
-/* accessing registers - based on rtl8139 */
-static void ivshmem_update_irq(IVShmemState *s, int val)
+static void ivshmem_update_irq(IVShmemState *s)
 {
     PCIDevice *d = PCI_DEVICE(s);
-    int isr;
-    isr = (s->intrstatus & s->intrmask) & 0xffffffff;
+    uint32_t isr = s->intrstatus & s->intrmask;
+
+    /*
+     * Do nothing unless the device actually uses INTx.  Here's how
+     * the device variants signal interrupts, what they put in PCI
+     * config space:
+     * Device variant    Interrupt  Interrupt Pin  MSI-X cap.
+     * ivshmem-plain         none            0         no
+     * ivshmem-doorbell     MSI-X            1        yes(1)
+     * ivshmem,msi=off       INTx            1         no
+     * ivshmem,msi=on       MSI-X            1(2)     yes(1)
+     * (1) if guest enabled MSI-X
+     * (2) the device lies
+     * Leads to the condition for doing nothing:
+     */
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)
+        || !d->config[PCI_INTERRUPT_PIN]) {
+        return;
+    }
 
     /* don't print ISR resets */
     if (isr) {
@@ -136,7 +168,7 @@
                         isr ? 1 : 0, s->intrstatus, s->intrmask);
     }
 
-    pci_set_irq(d, (isr != 0));
+    pci_set_irq(d, isr != 0);
 }
 
 static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
@@ -144,8 +176,7 @@
     IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val);
 
     s->intrmask = val;
-
-    ivshmem_update_irq(s, val);
+    ivshmem_update_irq(s);
 }
 
 static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
@@ -153,7 +184,6 @@
     uint32_t ret = s->intrmask;
 
     IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret);
-
     return ret;
 }
 
@@ -162,8 +192,7 @@
     IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val);
 
     s->intrstatus = val;
-
-    ivshmem_update_irq(s, val);
+    ivshmem_update_irq(s);
 }
 
 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
@@ -172,9 +201,7 @@
 
     /* reading ISR clears all interrupts */
     s->intrstatus = 0;
-
-    ivshmem_update_irq(s, 0);
-
+    ivshmem_update_irq(s);
     return ret;
 }
 
@@ -201,7 +228,7 @@
 
         case DOORBELL:
             /* check that dest VM ID is reasonable */
-            if (dest > s->max_peer) {
+            if (dest >= s->nb_peers) {
                 IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest);
                 break;
             }
@@ -210,10 +237,13 @@
             if (vector < s->peers[dest].nb_eventfds) {
                 IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector);
                 event_notifier_set(&s->peers[dest].eventfds[vector]);
+            } else {
+                IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
+                                vector, dest);
             }
             break;
         default:
-            IVSHMEM_DPRINTF("Invalid VM Doorbell VM %d\n", dest);
+            IVSHMEM_DPRINTF("Unhandled write " TARGET_FMT_plx "\n", addr);
     }
 }
 
@@ -235,12 +265,7 @@
             break;
 
         case IVPOSITION:
-            /* return my VM ID if the memory is mapped */
-            if (s->shm_fd > 0) {
-                ret = s->vm_id;
-            } else {
-                ret = -1;
-            }
+            ret = s->vm_id;
             break;
 
         default:
@@ -261,105 +286,96 @@
     },
 };
 
-static void ivshmem_receive(void *opaque, const uint8_t *buf, int size)
+static void ivshmem_vector_notify(void *opaque)
 {
-    IVShmemState *s = opaque;
-
-    ivshmem_IntrStatus_write(s, *buf);
-
-    IVSHMEM_DPRINTF("ivshmem_receive 0x%02x\n", *buf);
-}
-
-static int ivshmem_can_receive(void * opaque)
-{
-    return 8;
-}
-
-static void ivshmem_event(void *opaque, int event)
-{
-    IVSHMEM_DPRINTF("ivshmem_event %d\n", event);
-}
-
-static void fake_irqfd(void *opaque, const uint8_t *buf, int size) {
-
-    EventfdEntry *entry = opaque;
+    MSIVector *entry = opaque;
     PCIDevice *pdev = entry->pdev;
+    IVShmemState *s = IVSHMEM_COMMON(pdev);
+    int vector = entry - s->msi_vectors;
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
 
-    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, entry->vector);
-    msix_notify(pdev, entry->vector);
+    if (!event_notifier_test_and_clear(n)) {
+        return;
+    }
+
+    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector);
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        if (msix_enabled(pdev)) {
+            msix_notify(pdev, vector);
+        }
+    } else {
+        ivshmem_IntrStatus_write(s, 1);
+    }
 }
 
-static CharDriverState* create_eventfd_chr_device(void * opaque, EventNotifier *n,
-                                                  int vector)
+static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
+                                 MSIMessage msg)
 {
-    /* create a event character device based on the passed eventfd */
-    IVShmemState *s = opaque;
-    CharDriverState * chr;
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    MSIVector *v = &s->msi_vectors[vector];
+    int ret;
+
+    IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
+
+    ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
+    if (ret < 0) {
+        return ret;
+    }
+    kvm_irqchip_commit_routes(kvm_state);
+
+    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
+}
+
+static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    int ret;
+
+    IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
+
+    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n,
+                                                s->msi_vectors[vector].virq);
+    if (ret != 0) {
+        error_report("remove_irqfd_notifier_gsi failed");
+    }
+}
+
+static void ivshmem_vector_poll(PCIDevice *dev,
+                                unsigned int vector_start,
+                                unsigned int vector_end)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    unsigned int vector;
+
+    IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end);
+
+    vector_end = MIN(vector_end, s->vectors);
+
+    for (vector = vector_start; vector < vector_end; vector++) {
+        EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector];
+
+        if (!msix_is_masked(dev, vector)) {
+            continue;
+        }
+
+        if (event_notifier_test_and_clear(notifier)) {
+            msix_set_pending(dev, vector);
+        }
+    }
+}
+
+static void watch_vector_notifier(IVShmemState *s, EventNotifier *n,
+                                 int vector)
+{
     int eventfd = event_notifier_get_fd(n);
 
-    chr = qemu_chr_open_eventfd(eventfd);
+    assert(!s->msi_vectors[vector].pdev);
+    s->msi_vectors[vector].pdev = PCI_DEVICE(s);
 
-    if (chr == NULL) {
-        error_report("creating eventfd for eventfd %d failed", eventfd);
-        exit(1);
-    }
-    qemu_chr_fe_claim_no_fail(chr);
-
-    /* if MSI is supported we need multiple interrupts */
-    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        s->eventfd_table[vector].pdev = PCI_DEVICE(s);
-        s->eventfd_table[vector].vector = vector;
-
-        qemu_chr_add_handlers(chr, ivshmem_can_receive, fake_irqfd,
-                      ivshmem_event, &s->eventfd_table[vector]);
-    } else {
-        qemu_chr_add_handlers(chr, ivshmem_can_receive, ivshmem_receive,
-                      ivshmem_event, s);
-    }
-
-    return chr;
-
-}
-
-static int check_shm_size(IVShmemState *s, int fd) {
-    /* check that the guest isn't going to try and map more memory than the
-     * the object has allocated return -1 to indicate error */
-
-    struct stat buf;
-
-    if (fstat(fd, &buf) < 0) {
-        error_report("exiting: fstat on fd %d failed: %s",
-                     fd, strerror(errno));
-        return -1;
-    }
-
-    if (s->ivshmem_size > buf.st_size) {
-        error_report("Requested memory size greater"
-                     " than shared object size (%" PRIu64 " > %" PRIu64")",
-                     s->ivshmem_size, (uint64_t)buf.st_size);
-        return -1;
-    } else {
-        return 0;
-    }
-}
-
-/* create the shared memory BAR when we are not using the server, so we can
- * create the BAR and map the memory immediately */
-static void create_shared_memory_BAR(IVShmemState *s, int fd) {
-
-    void * ptr;
-
-    s->shm_fd = fd;
-
-    ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
-
-    memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s), "ivshmem.bar2",
-                               s->ivshmem_size, ptr);
-    vmstate_register_ram(&s->ivshmem, DEVICE(s));
-    memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
-
-    /* region for shared memory */
-    pci_register_bar(PCI_DEVICE(s), 2, s->ivshmem_attr, &s->bar);
+    qemu_set_fd_handler(eventfd, ivshmem_vector_notify,
+                        NULL, &s->msi_vectors[vector]);
 }
 
 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
@@ -382,25 +398,22 @@
                               &s->peers[posn].eventfds[i]);
 }
 
-static void close_guest_eventfds(IVShmemState *s, int posn)
+static void close_peer_eventfds(IVShmemState *s, int posn)
 {
-    int i, guest_curr_max;
+    int i, n;
 
-    if (!ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
-        return;
-    }
-    if (posn < 0 || posn >= s->nb_peers) {
-        return;
+    assert(posn >= 0 && posn < s->nb_peers);
+    n = s->peers[posn].nb_eventfds;
+
+    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
+        memory_region_transaction_begin();
+        for (i = 0; i < n; i++) {
+            ivshmem_del_eventfd(s, posn, i);
+        }
+        memory_region_transaction_commit();
     }
 
-    guest_curr_max = s->peers[posn].nb_eventfds;
-
-    memory_region_transaction_begin();
-    for (i = 0; i < guest_curr_max; i++) {
-        ivshmem_del_eventfd(s, posn, i);
-    }
-    memory_region_transaction_commit();
-    for (i = 0; i < guest_curr_max; i++) {
+    for (i = 0; i < n; i++) {
         event_notifier_cleanup(&s->peers[posn].eventfds[i]);
     }
 
@@ -408,187 +421,319 @@
     s->peers[posn].nb_eventfds = 0;
 }
 
-/* this function increase the dynamic storage need to store data about other
- * guests */
-static int increase_dynamic_storage(IVShmemState *s, int new_min_size)
+static void resize_peers(IVShmemState *s, int nb_peers)
 {
+    int old_nb_peers = s->nb_peers;
+    int i;
 
-    int j, old_nb_alloc;
+    assert(nb_peers > old_nb_peers);
+    IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers);
 
-    /* check for integer overflow */
-    if (new_min_size >= INT_MAX / sizeof(Peer) - 1 || new_min_size <= 0) {
-        return -1;
+    s->peers = g_realloc(s->peers, nb_peers * sizeof(Peer));
+    s->nb_peers = nb_peers;
+
+    for (i = old_nb_peers; i < nb_peers; i++) {
+        s->peers[i].eventfds = g_new0(EventNotifier, s->vectors);
+        s->peers[i].nb_eventfds = 0;
+    }
+}
+
+static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector,
+                                     Error **errp)
+{
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int ret;
+
+    IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector);
+    assert(!s->msi_vectors[vector].pdev);
+
+    ret = kvm_irqchip_add_msi_route(kvm_state, vector, pdev);
+    if (ret < 0) {
+        error_setg(errp, "kvm_irqchip_add_msi_route failed");
+        return;
     }
 
-    old_nb_alloc = s->nb_peers;
+    s->msi_vectors[vector].virq = ret;
+    s->msi_vectors[vector].pdev = pdev;
+}
 
-    if (new_min_size >= s->nb_peers) {
-        /* +1 because #new_min_size is used as last array index */
-        s->nb_peers = new_min_size + 1;
+static void setup_interrupt(IVShmemState *s, int vector, Error **errp)
+{
+    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    bool with_irqfd = kvm_msi_via_irqfd_enabled() &&
+        ivshmem_has_feature(s, IVSHMEM_MSI);
+    PCIDevice *pdev = PCI_DEVICE(s);
+    Error *err = NULL;
+
+    IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector);
+
+    if (!with_irqfd) {
+        IVSHMEM_DPRINTF("with eventfd\n");
+        watch_vector_notifier(s, n, vector);
+    } else if (msix_enabled(pdev)) {
+        IVSHMEM_DPRINTF("with irqfd\n");
+        ivshmem_add_kvm_msi_virq(s, vector, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+
+        if (!msix_is_masked(pdev, vector)) {
+            kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL,
+                                               s->msi_vectors[vector].virq);
+            /* TODO handle error */
+        }
     } else {
-        return 0;
+        /* it will be delayed until msix is enabled, in write_config */
+        IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
+    }
+}
+
+static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)
+{
+    struct stat buf;
+    size_t size;
+    void *ptr;
+
+    if (s->ivshmem_bar2) {
+        error_setg(errp, "server sent unexpected shared memory message");
+        close(fd);
+        return;
     }
 
-    IVSHMEM_DPRINTF("bumping storage to %d guests\n", s->nb_peers);
-    s->peers = g_realloc(s->peers, s->nb_peers * sizeof(Peer));
-
-    /* zero out new pointers */
-    for (j = old_nb_alloc; j < s->nb_peers; j++) {
-        s->peers[j].eventfds = NULL;
-        s->peers[j].nb_eventfds = 0;
+    if (fstat(fd, &buf) < 0) {
+        error_setg_errno(errp, errno,
+            "can't determine size of shared memory sent by server");
+        close(fd);
+        return;
     }
 
-    return 0;
+    size = buf.st_size;
+
+    /* Legacy cruft */
+    if (s->legacy_size != SIZE_MAX) {
+        if (size < s->legacy_size) {
+            error_setg(errp, "server sent only %zd bytes of shared memory",
+                       (size_t)buf.st_size);
+            close(fd);
+            return;
+        }
+        size = s->legacy_size;
+    }
+
+    /* mmap the region and map into the BAR2 */
+    ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (ptr == MAP_FAILED) {
+        error_setg_errno(errp, errno, "Failed to mmap shared memory");
+        close(fd);
+        return;
+    }
+    memory_region_init_ram_ptr(&s->server_bar2, OBJECT(s),
+                               "ivshmem.bar2", size, ptr);
+    memory_region_set_fd(&s->server_bar2, fd);
+    s->ivshmem_bar2 = &s->server_bar2;
+}
+
+static void process_msg_disconnect(IVShmemState *s, uint16_t posn,
+                                   Error **errp)
+{
+    IVSHMEM_DPRINTF("posn %d has gone away\n", posn);
+    if (posn >= s->nb_peers || posn == s->vm_id) {
+        error_setg(errp, "invalid peer %d", posn);
+        return;
+    }
+    close_peer_eventfds(s, posn);
+}
+
+static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd,
+                                Error **errp)
+{
+    Peer *peer = &s->peers[posn];
+    int vector;
+
+    /*
+     * The N-th connect message for this peer comes with the file
+     * descriptor for vector N-1.  Count messages to find the vector.
+     */
+    if (peer->nb_eventfds >= s->vectors) {
+        error_setg(errp, "Too many eventfd received, device has %d vectors",
+                   s->vectors);
+        close(fd);
+        return;
+    }
+    vector = peer->nb_eventfds++;
+
+    IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd);
+    event_notifier_init_fd(&peer->eventfds[vector], fd);
+    fcntl_setfl(fd, O_NONBLOCK); /* msix/irqfd poll non block */
+
+    if (posn == s->vm_id) {
+        setup_interrupt(s, vector, errp);
+        /* TODO do we need to handle the error? */
+    }
+
+    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
+        ivshmem_add_eventfd(s, posn, vector);
+    }
+}
+
+static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp)
+{
+    IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
+
+    if (msg < -1 || msg > IVSHMEM_MAX_PEERS) {
+        error_setg(errp, "server sent invalid message %" PRId64, msg);
+        close(fd);
+        return;
+    }
+
+    if (msg == -1) {
+        process_msg_shmem(s, fd, errp);
+        return;
+    }
+
+    if (msg >= s->nb_peers) {
+        resize_peers(s, msg + 1);
+    }
+
+    if (fd >= 0) {
+        process_msg_connect(s, msg, fd, errp);
+    } else {
+        process_msg_disconnect(s, msg, errp);
+    }
+}
+
+static int ivshmem_can_receive(void *opaque)
+{
+    IVShmemState *s = opaque;
+
+    assert(s->msg_buffered_bytes < sizeof(s->msg_buf));
+    return sizeof(s->msg_buf) - s->msg_buffered_bytes;
 }
 
 static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
 {
     IVShmemState *s = opaque;
-    int incoming_fd, tmp_fd;
-    int guest_max_eventfd;
-    long incoming_posn;
+    Error *err = NULL;
+    int fd;
+    int64_t msg;
 
-    if (fifo8_is_empty(&s->incoming_fifo) && size == sizeof(incoming_posn)) {
-        memcpy(&incoming_posn, buf, size);
-    } else {
-        const uint8_t *p;
-        uint32_t num;
-
-        IVSHMEM_DPRINTF("short read of %d bytes\n", size);
-        num = MAX(size, sizeof(long) - fifo8_num_used(&s->incoming_fifo));
-        fifo8_push_all(&s->incoming_fifo, buf, num);
-        if (fifo8_num_used(&s->incoming_fifo) < sizeof(incoming_posn)) {
-            return;
-        }
-        size -= num;
-        buf += num;
-        p = fifo8_pop_buf(&s->incoming_fifo, sizeof(incoming_posn), &num);
-        g_assert(num == sizeof(incoming_posn));
-        memcpy(&incoming_posn, p, sizeof(incoming_posn));
-        if (size > 0) {
-            fifo8_push_all(&s->incoming_fifo, buf, size);
-        }
+    assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf));
+    memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size);
+    s->msg_buffered_bytes += size;
+    if (s->msg_buffered_bytes < sizeof(s->msg_buf)) {
+        return;
     }
+    msg = le64_to_cpu(s->msg_buf);
+    s->msg_buffered_bytes = 0;
 
-    if (incoming_posn < -1) {
-        IVSHMEM_DPRINTF("invalid incoming_posn %ld\n", incoming_posn);
+    fd = qemu_chr_fe_get_msgfd(s->server_chr);
+    IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
+
+    process_msg(s, msg, fd, &err);
+    if (err) {
+        error_report_err(err);
+    }
+}
+
+static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp)
+{
+    int64_t msg;
+    int n, ret;
+
+    n = 0;
+    do {
+        ret = qemu_chr_fe_read_all(s->server_chr, (uint8_t *)&msg + n,
+                                 sizeof(msg) - n);
+        if (ret < 0 && ret != -EINTR) {
+            error_setg_errno(errp, -ret, "read from server failed");
+            return INT64_MIN;
+        }
+        n += ret;
+    } while (n < sizeof(msg));
+
+    *pfd = qemu_chr_fe_get_msgfd(s->server_chr);
+    return msg;
+}
+
+static void ivshmem_recv_setup(IVShmemState *s, Error **errp)
+{
+    Error *err = NULL;
+    int64_t msg;
+    int fd;
+
+    msg = ivshmem_recv_msg(s, &fd, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    if (msg != IVSHMEM_PROTOCOL_VERSION) {
+        error_setg(errp, "server sent version %" PRId64 ", expecting %d",
+                   msg, IVSHMEM_PROTOCOL_VERSION);
+        return;
+    }
+    if (fd != -1) {
+        error_setg(errp, "server sent invalid version message");
         return;
     }
 
-    /* pick off s->server_chr->msgfd and store it, posn should accompany msg */
-    tmp_fd = qemu_chr_fe_get_msgfd(s->server_chr);
-    IVSHMEM_DPRINTF("posn is %ld, fd is %d\n", incoming_posn, tmp_fd);
+    /*
+     * ivshmem-server sends the remaining initial messages in a fixed
+     * order, but the device has always accepted them in any order.
+     * Stay as compatible as practical, just in case people use
+     * servers that behave differently.
+     */
 
-    /* make sure we have enough space for this guest */
-    if (incoming_posn >= s->nb_peers) {
-        if (increase_dynamic_storage(s, incoming_posn) < 0) {
-            error_report("increase_dynamic_storage() failed");
-            if (tmp_fd != -1) {
-                close(tmp_fd);
-            }
-            return;
-        }
-    }
-
-    if (tmp_fd == -1) {
-        /* if posn is positive and unseen before then this is our posn*/
-        if ((incoming_posn >= 0) &&
-                            (s->peers[incoming_posn].eventfds == NULL)) {
-            /* receive our posn */
-            s->vm_id = incoming_posn;
-            return;
-        } else {
-            /* otherwise an fd == -1 means an existing guest has gone away */
-            IVSHMEM_DPRINTF("posn %ld has gone away\n", incoming_posn);
-            close_guest_eventfds(s, incoming_posn);
-            return;
-        }
-    }
-
-    /* because of the implementation of get_msgfd, we need a dup */
-    incoming_fd = dup(tmp_fd);
-
-    if (incoming_fd == -1) {
-        error_report("could not allocate file descriptor %s", strerror(errno));
-        close(tmp_fd);
+    /*
+     * ivshmem_device_spec.txt has always required the ID message
+     * right here, and ivshmem-server has always complied.  However,
+     * older versions of the device accepted it out of order, but
+     * broke when an interrupt setup message arrived before it.
+     */
+    msg = ivshmem_recv_msg(s, &fd, &err);
+    if (err) {
+        error_propagate(errp, err);
         return;
     }
-
-    /* if the position is -1, then it's shared memory region fd */
-    if (incoming_posn == -1) {
-
-        void * map_ptr;
-
-        s->max_peer = 0;
-
-        if (check_shm_size(s, incoming_fd) == -1) {
-            exit(1);
-        }
-
-        /* mmap the region and map into the BAR2 */
-        map_ptr = mmap(0, s->ivshmem_size, PROT_READ|PROT_WRITE, MAP_SHARED,
-                                                            incoming_fd, 0);
-        memory_region_init_ram_ptr(&s->ivshmem, OBJECT(s),
-                                   "ivshmem.bar2", s->ivshmem_size, map_ptr);
-        vmstate_register_ram(&s->ivshmem, DEVICE(s));
-
-        IVSHMEM_DPRINTF("guest h/w addr = %p, size = %" PRIu64 "\n",
-                        map_ptr, s->ivshmem_size);
-
-        memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
-
-        /* only store the fd if it is successfully mapped */
-        s->shm_fd = incoming_fd;
-
+    if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) {
+        error_setg(errp, "server sent invalid ID message");
         return;
     }
+    s->vm_id = msg;
 
-    /* each guest has an array of eventfds, and we keep track of how many
-     * guests for each VM */
-    guest_max_eventfd = s->peers[incoming_posn].nb_eventfds;
+    /*
+     * Receive more messages until we got shared memory.
+     */
+    do {
+        msg = ivshmem_recv_msg(s, &fd, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+        process_msg(s, msg, fd, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    } while (msg != -1);
 
-    if (guest_max_eventfd == 0) {
-        /* one eventfd per MSI vector */
-        s->peers[incoming_posn].eventfds = g_new(EventNotifier, s->vectors);
-    }
-
-    /* this is an eventfd for a particular guest VM */
-    IVSHMEM_DPRINTF("eventfds[%ld][%d] = %d\n", incoming_posn,
-                    guest_max_eventfd, incoming_fd);
-    event_notifier_init_fd(&s->peers[incoming_posn].eventfds[guest_max_eventfd],
-                           incoming_fd);
-
-    /* increment count for particular guest */
-    s->peers[incoming_posn].nb_eventfds++;
-
-    /* keep track of the maximum VM ID */
-    if (incoming_posn > s->max_peer) {
-        s->max_peer = incoming_posn;
-    }
-
-    if (incoming_posn == s->vm_id) {
-        s->eventfd_chr[guest_max_eventfd] = create_eventfd_chr_device(s,
-                   &s->peers[s->vm_id].eventfds[guest_max_eventfd],
-                   guest_max_eventfd);
-    }
-
-    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
-        ivshmem_add_eventfd(s, incoming_posn, guest_max_eventfd);
-    }
+    /*
+     * This function must either map the shared memory or fail.  The
+     * loop above ensures that: it terminates normally only after it
+     * successfully processed the server's shared memory message.
+     * Assert that actually mapped the shared memory:
+     */
+    assert(s->ivshmem_bar2);
 }
 
 /* Select the MSI-X vectors used by device.
  * ivshmem maps events to vectors statically, so
  * we just enable all vectors on init and after reset. */
-static void ivshmem_use_msix(IVShmemState * s)
+static void ivshmem_msix_vector_use(IVShmemState *s)
 {
     PCIDevice *d = PCI_DEVICE(s);
     int i;
 
-    if (!msix_present(d)) {
-        return;
-    }
-
     for (i = 0; i < s->vectors; i++) {
         msix_vector_use(d, i);
     }
@@ -596,161 +741,117 @@
 
 static void ivshmem_reset(DeviceState *d)
 {
-    IVShmemState *s = IVSHMEM(d);
+    IVShmemState *s = IVSHMEM_COMMON(d);
 
     s->intrstatus = 0;
-    ivshmem_use_msix(s);
+    s->intrmask = 0;
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        ivshmem_msix_vector_use(s);
+    }
 }
 
-static uint64_t ivshmem_get_size(IVShmemState * s) {
-
-    uint64_t value;
-    char *ptr;
-
-    value = strtoull(s->sizearg, &ptr, 10);
-    switch (*ptr) {
-        case 0: case 'M': case 'm':
-            value <<= 20;
-            break;
-        case 'G': case 'g':
-            value <<= 30;
-            break;
-        default:
-            error_report("invalid ram size: %s", s->sizearg);
-            exit(1);
-    }
-
-    /* BARs must be a power of 2 */
-    if (!is_power_of_two(value)) {
-        error_report("size must be power of 2");
-        exit(1);
-    }
-
-    return value;
-}
-
-static void ivshmem_setup_msi(IVShmemState * s)
+static int ivshmem_setup_interrupts(IVShmemState *s)
 {
-    if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) {
-        IVSHMEM_DPRINTF("msix initialization failed\n");
-        exit(1);
-    }
+    /* allocate QEMU callback data for receiving interrupts */
+    s->msi_vectors = g_malloc0(s->vectors * sizeof(MSIVector));
 
-    IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1)) {
+            return -1;
+        }
 
-    /* allocate QEMU char devices for receiving interrupts */
-    s->eventfd_table = g_malloc0(s->vectors * sizeof(EventfdEntry));
-
-    ivshmem_use_msix(s);
-}
-
-static void ivshmem_save(QEMUFile* f, void *opaque)
-{
-    IVShmemState *proxy = opaque;
-    PCIDevice *pci_dev = PCI_DEVICE(proxy);
-
-    IVSHMEM_DPRINTF("ivshmem_save\n");
-    pci_device_save(pci_dev, f);
-
-    if (ivshmem_has_feature(proxy, IVSHMEM_MSI)) {
-        msix_save(pci_dev, f);
-    } else {
-        qemu_put_be32(f, proxy->intrstatus);
-        qemu_put_be32(f, proxy->intrmask);
-    }
-
-}
-
-static int ivshmem_load(QEMUFile* f, void *opaque, int version_id)
-{
-    IVSHMEM_DPRINTF("ivshmem_load\n");
-
-    IVShmemState *proxy = opaque;
-    PCIDevice *pci_dev = PCI_DEVICE(proxy);
-    int ret;
-
-    if (version_id > 0) {
-        return -EINVAL;
-    }
-
-    if (proxy->role_val == IVSHMEM_PEER) {
-        error_report("'peer' devices are not migratable");
-        return -EINVAL;
-    }
-
-    ret = pci_device_load(pci_dev, f);
-    if (ret) {
-        return ret;
-    }
-
-    if (ivshmem_has_feature(proxy, IVSHMEM_MSI)) {
-        msix_load(pci_dev, f);
-	ivshmem_use_msix(proxy);
-    } else {
-        proxy->intrstatus = qemu_get_be32(f);
-        proxy->intrmask = qemu_get_be32(f);
+        IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
+        ivshmem_msix_vector_use(s);
     }
 
     return 0;
 }
 
-static void ivshmem_write_config(PCIDevice *pci_dev, uint32_t address,
-				 uint32_t val, int len)
+static void ivshmem_enable_irqfd(IVShmemState *s)
 {
-    pci_default_write_config(pci_dev, address, val, len);
-    msix_write_config(pci_dev, address, val, len);
-}
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int i;
 
-static int pci_ivshmem_init(PCIDevice *dev)
-{
-    IVShmemState *s = IVSHMEM(dev);
-    uint8_t *pci_conf;
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        Error *err = NULL;
 
-    if (s->sizearg == NULL)
-        s->ivshmem_size = 4 << 20; /* 4 MB default */
-    else {
-        s->ivshmem_size = ivshmem_get_size(s);
+        ivshmem_add_kvm_msi_virq(s, i, &err);
+        if (err) {
+            error_report_err(err);
+            /* TODO do we need to handle the error? */
+        }
     }
 
-    fifo8_create(&s->incoming_fifo, sizeof(long));
+    if (msix_set_vector_notifiers(pdev,
+                                  ivshmem_vector_unmask,
+                                  ivshmem_vector_mask,
+                                  ivshmem_vector_poll)) {
+        error_report("ivshmem: msix_set_vector_notifiers failed");
+    }
+}
 
-    register_savevm(DEVICE(dev), "ivshmem", 0, 0, ivshmem_save, ivshmem_load,
-                                                                        dev);
+static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
+{
+    IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
+
+    if (s->msi_vectors[vector].pdev == NULL) {
+        return;
+    }
+
+    /* it was cleaned when masked in the frontend. */
+    kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
+
+    s->msi_vectors[vector].pdev = NULL;
+}
+
+static void ivshmem_disable_irqfd(IVShmemState *s)
+{
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int i;
+
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        ivshmem_remove_kvm_msi_virq(s, i);
+    }
+
+    msix_unset_vector_notifiers(pdev);
+}
+
+static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
+                                 uint32_t val, int len)
+{
+    IVShmemState *s = IVSHMEM_COMMON(pdev);
+    int is_enabled, was_enabled = msix_enabled(pdev);
+
+    pci_default_write_config(pdev, address, val, len);
+    is_enabled = msix_enabled(pdev);
+
+    if (kvm_msi_via_irqfd_enabled()) {
+        if (!was_enabled && is_enabled) {
+            ivshmem_enable_irqfd(s);
+        } else if (was_enabled && !is_enabled) {
+            ivshmem_disable_irqfd(s);
+        }
+    }
+}
+
+static void ivshmem_common_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    Error *err = NULL;
+    uint8_t *pci_conf;
+    uint8_t attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
+        PCI_BASE_ADDRESS_MEM_PREFETCH;
 
     /* IRQFD requires MSI */
     if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
         !ivshmem_has_feature(s, IVSHMEM_MSI)) {
-        error_report("ioeventfd/irqfd requires MSI");
-        exit(1);
-    }
-
-    /* check that role is reasonable */
-    if (s->role) {
-        if (strncmp(s->role, "peer", 5) == 0) {
-            s->role_val = IVSHMEM_PEER;
-        } else if (strncmp(s->role, "master", 7) == 0) {
-            s->role_val = IVSHMEM_MASTER;
-        } else {
-            error_report("'role' must be 'peer' or 'master'");
-            exit(1);
-        }
-    } else {
-        s->role_val = IVSHMEM_MASTER; /* default */
-    }
-
-    if (s->role_val == IVSHMEM_PEER) {
-        error_setg(&s->migration_blocker,
-                   "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
-        migrate_add_blocker(s->migration_blocker);
+        error_setg(errp, "ioeventfd/irqfd requires MSI");
+        return;
     }
 
     pci_conf = dev->config;
     pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
 
-    pci_config_set_interrupt_pin(pci_conf, 1);
-
-    s->shm_fd = 0;
-
     memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
                           "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
 
@@ -758,134 +859,467 @@
     pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
                      &s->ivshmem_mmio);
 
-    memory_region_init(&s->bar, OBJECT(s), "ivshmem-bar2-container", s->ivshmem_size);
-    s->ivshmem_attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
-        PCI_BASE_ADDRESS_MEM_PREFETCH;
-    if (s->ivshmem_64bit) {
-        s->ivshmem_attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+    if (!s->not_legacy_32bit) {
+        attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
     }
 
-    if ((s->server_chr != NULL) &&
-                        (strncmp(s->server_chr->filename, "unix:", 5) == 0)) {
-        /* if we get a UNIX socket as the parameter we will talk
-         * to the ivshmem server to receive the memory region */
+    if (s->hostmem != NULL) {
+        IVSHMEM_DPRINTF("using hostmem\n");
 
-        if (s->shmobj != NULL) {
-            error_report("WARNING: do not specify both 'chardev' "
-                         "and 'shm' with ivshmem");
-        }
+        s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem,
+                                                         &error_abort);
+    } else {
+        assert(s->server_chr);
 
         IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
                         s->server_chr->filename);
 
-        if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
-            ivshmem_setup_msi(s);
+        /* we allocate enough space for 16 peers and grow as needed */
+        resize_peers(s, 16);
+
+        /*
+         * Receive setup messages from server synchronously.
+         * Older versions did it asynchronously, but that creates a
+         * number of entertaining race conditions.
+         */
+        ivshmem_recv_setup(s, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
         }
 
-        /* we allocate enough space for 16 guests and grow as needed */
-        s->nb_peers = 16;
-        s->vm_id = -1;
-
-        /* allocate/initialize space for interrupt handling */
-        s->peers = g_malloc0(s->nb_peers * sizeof(Peer));
-
-        pci_register_bar(dev, 2, s->ivshmem_attr, &s->bar);
-
-        s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *));
-
-        qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive, ivshmem_read,
-                     ivshmem_event, s);
-    } else {
-        /* just map the file immediately, we're not using a server */
-        int fd;
-
-        if (s->shmobj == NULL) {
-            error_report("Must specify 'chardev' or 'shm' to ivshmem");
-            exit(1);
+        if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) {
+            error_setg(errp,
+                       "master must connect to the server before any peers");
+            return;
         }
 
-        IVSHMEM_DPRINTF("using shm_open (shm object = %s)\n", s->shmobj);
+        qemu_chr_add_handlers(s->server_chr, ivshmem_can_receive,
+                              ivshmem_read, NULL, s);
 
-        /* try opening with O_EXCL and if it succeeds zero the memory
-         * by truncating to 0 */
-        if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR|O_EXCL,
-                        S_IRWXU|S_IRWXG|S_IRWXO)) > 0) {
-           /* truncate file to length PCI device's memory */
-            if (ftruncate(fd, s->ivshmem_size) != 0) {
-                error_report("could not truncate shared file");
-            }
-
-        } else if ((fd = shm_open(s->shmobj, O_CREAT|O_RDWR,
-                        S_IRWXU|S_IRWXG|S_IRWXO)) < 0) {
-            error_report("could not open shared file");
-            exit(1);
-
+        if (ivshmem_setup_interrupts(s) < 0) {
+            error_setg(errp, "failed to initialize interrupts");
+            return;
         }
-
-        if (check_shm_size(s, fd) == -1) {
-            exit(1);
-        }
-
-        create_shared_memory_BAR(s, fd);
-
     }
 
-    dev->config_write = ivshmem_write_config;
+    vmstate_register_ram(s->ivshmem_bar2, DEVICE(s));
+    pci_register_bar(PCI_DEVICE(s), 2, attr, s->ivshmem_bar2);
 
-    return 0;
+    if (s->master == ON_OFF_AUTO_AUTO) {
+        s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
+    }
+
+    if (!ivshmem_is_master(s)) {
+        error_setg(&s->migration_blocker,
+                   "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
+        migrate_add_blocker(s->migration_blocker);
+    }
 }
 
-static void pci_ivshmem_uninit(PCIDevice *dev)
+static void ivshmem_exit(PCIDevice *dev)
 {
-    IVShmemState *s = IVSHMEM(dev);
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+    int i;
 
     if (s->migration_blocker) {
         migrate_del_blocker(s->migration_blocker);
         error_free(s->migration_blocker);
     }
 
-    memory_region_del_subregion(&s->bar, &s->ivshmem);
-    vmstate_unregister_ram(&s->ivshmem, DEVICE(dev));
-    unregister_savevm(DEVICE(dev), "ivshmem", s);
-    fifo8_destroy(&s->incoming_fifo);
+    if (memory_region_is_mapped(s->ivshmem_bar2)) {
+        if (!s->hostmem) {
+            void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2);
+            int fd;
+
+            if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) {
+                error_report("Failed to munmap shared memory %s",
+                             strerror(errno));
+            }
+
+            fd = memory_region_get_fd(s->ivshmem_bar2);
+            close(fd);
+        }
+
+        vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev));
+    }
+
+    if (s->peers) {
+        for (i = 0; i < s->nb_peers; i++) {
+            close_peer_eventfds(s, i);
+        }
+        g_free(s->peers);
+    }
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        msix_uninit_exclusive_bar(dev);
+    }
+
+    g_free(s->msi_vectors);
 }
 
+static int ivshmem_pre_load(void *opaque)
+{
+    IVShmemState *s = opaque;
+
+    if (!ivshmem_is_master(s)) {
+        error_report("'peer' devices are not migratable");
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int ivshmem_post_load(void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        ivshmem_msix_vector_use(s);
+    }
+    return 0;
+}
+
+static void ivshmem_common_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = ivshmem_common_realize;
+    k->exit = ivshmem_exit;
+    k->config_write = ivshmem_write_config;
+    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
+    k->device_id = PCI_DEVICE_ID_IVSHMEM;
+    k->class_id = PCI_CLASS_MEMORY_RAM;
+    k->revision = 1;
+    dc->reset = ivshmem_reset;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->desc = "Inter-VM shared memory";
+}
+
+static const TypeInfo ivshmem_common_info = {
+    .name          = TYPE_IVSHMEM_COMMON,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(IVShmemState),
+    .abstract      = true,
+    .class_init    = ivshmem_common_class_init,
+};
+
+static void ivshmem_check_memdev_is_busy(Object *obj, const char *name,
+                                         Object *val, Error **errp)
+{
+    if (host_memory_backend_is_mapped(MEMORY_BACKEND(val))) {
+        char *path = object_get_canonical_path_component(val);
+        error_setg(errp, "can't use already busy memdev: %s", path);
+        g_free(path);
+    } else {
+        qdev_prop_allow_set_link_before_realize(obj, name, val, errp);
+    }
+}
+
+static const VMStateDescription ivshmem_plain_vmsd = {
+    .name = TYPE_IVSHMEM_PLAIN,
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+        VMSTATE_UINT32(intrstatus, IVShmemState),
+        VMSTATE_UINT32(intrmask, IVShmemState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property ivshmem_plain_properties[] = {
+    DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ivshmem_plain_init(Object *obj)
+{
+    IVShmemState *s = IVSHMEM_PLAIN(obj);
+
+    object_property_add_link(obj, "memdev", TYPE_MEMORY_BACKEND,
+                             (Object **)&s->hostmem,
+                             ivshmem_check_memdev_is_busy,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+}
+
+static void ivshmem_plain_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+
+    if (!s->hostmem) {
+        error_setg(errp, "You must specify a 'memdev'");
+        return;
+    }
+
+    ivshmem_common_realize(dev, errp);
+    host_memory_backend_set_mapped(s->hostmem, true);
+}
+
+static void ivshmem_plain_exit(PCIDevice *pci_dev)
+{
+    IVShmemState *s = IVSHMEM_COMMON(pci_dev);
+
+    host_memory_backend_set_mapped(s->hostmem, false);
+}
+
+static void ivshmem_plain_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = ivshmem_plain_realize;
+    k->exit = ivshmem_plain_exit;
+    dc->props = ivshmem_plain_properties;
+    dc->vmsd = &ivshmem_plain_vmsd;
+}
+
+static const TypeInfo ivshmem_plain_info = {
+    .name          = TYPE_IVSHMEM_PLAIN,
+    .parent        = TYPE_IVSHMEM_COMMON,
+    .instance_size = sizeof(IVShmemState),
+    .instance_init = ivshmem_plain_init,
+    .class_init    = ivshmem_plain_class_init,
+};
+
+static const VMStateDescription ivshmem_doorbell_vmsd = {
+    .name = TYPE_IVSHMEM_DOORBELL,
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+        VMSTATE_MSIX(parent_obj, IVShmemState),
+        VMSTATE_UINT32(intrstatus, IVShmemState),
+        VMSTATE_UINT32(intrmask, IVShmemState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property ivshmem_doorbell_properties[] = {
+    DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
+    DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
+    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
+                    true),
+    DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void ivshmem_doorbell_init(Object *obj)
+{
+    IVShmemState *s = IVSHMEM_DOORBELL(obj);
+
+    s->features |= (1 << IVSHMEM_MSI);
+    s->legacy_size = SIZE_MAX;  /* whatever the server sends */
+}
+
+static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+
+    if (!s->server_chr) {
+        error_setg(errp, "You must specify a 'chardev'");
+        return;
+    }
+
+    ivshmem_common_realize(dev, errp);
+}
+
+static void ivshmem_doorbell_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = ivshmem_doorbell_realize;
+    dc->props = ivshmem_doorbell_properties;
+    dc->vmsd = &ivshmem_doorbell_vmsd;
+}
+
+static const TypeInfo ivshmem_doorbell_info = {
+    .name          = TYPE_IVSHMEM_DOORBELL,
+    .parent        = TYPE_IVSHMEM_COMMON,
+    .instance_size = sizeof(IVShmemState),
+    .instance_init = ivshmem_doorbell_init,
+    .class_init    = ivshmem_doorbell_class_init,
+};
+
+static int ivshmem_load_old(QEMUFile *f, void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
+    PCIDevice *pdev = PCI_DEVICE(s);
+    int ret;
+
+    IVSHMEM_DPRINTF("ivshmem_load_old\n");
+
+    if (version_id != 0) {
+        return -EINVAL;
+    }
+
+    ret = ivshmem_pre_load(s);
+    if (ret) {
+        return ret;
+    }
+
+    ret = pci_device_load(pdev, f);
+    if (ret) {
+        return ret;
+    }
+
+    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
+        msix_load(pdev, f);
+        ivshmem_msix_vector_use(s);
+    } else {
+        s->intrstatus = qemu_get_be32(f);
+        s->intrmask = qemu_get_be32(f);
+    }
+
+    return 0;
+}
+
+static bool test_msix(void *opaque, int version_id)
+{
+    IVShmemState *s = opaque;
+
+    return ivshmem_has_feature(s, IVSHMEM_MSI);
+}
+
+static bool test_no_msix(void *opaque, int version_id)
+{
+    return !test_msix(opaque, version_id);
+}
+
+static const VMStateDescription ivshmem_vmsd = {
+    .name = "ivshmem",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_load = ivshmem_pre_load,
+    .post_load = ivshmem_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
+
+        VMSTATE_MSIX_TEST(parent_obj, IVShmemState, test_msix),
+        VMSTATE_UINT32_TEST(intrstatus, IVShmemState, test_no_msix),
+        VMSTATE_UINT32_TEST(intrmask, IVShmemState, test_no_msix),
+
+        VMSTATE_END_OF_LIST()
+    },
+    .load_state_old = ivshmem_load_old,
+    .minimum_version_id_old = 0
+};
+
 static Property ivshmem_properties[] = {
     DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
     DEFINE_PROP_STRING("size", IVShmemState, sizearg),
     DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
-    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD, false),
+    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
+                    false),
     DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true),
     DEFINE_PROP_STRING("shm", IVShmemState, shmobj),
     DEFINE_PROP_STRING("role", IVShmemState, role),
-    DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1),
+    DEFINE_PROP_UINT32("use64", IVShmemState, not_legacy_32bit, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void desugar_shm(IVShmemState *s)
+{
+    Object *obj;
+    char *path;
+
+    obj = object_new("memory-backend-file");
+    path = g_strdup_printf("/dev/shm/%s", s->shmobj);
+    object_property_set_str(obj, path, "mem-path", &error_abort);
+    g_free(path);
+    object_property_set_int(obj, s->legacy_size, "size", &error_abort);
+    object_property_set_bool(obj, true, "share", &error_abort);
+    object_property_add_child(OBJECT(s), "internal-shm-backend", obj,
+                              &error_abort);
+    user_creatable_complete(obj, &error_abort);
+    s->hostmem = MEMORY_BACKEND(obj);
+}
+
+static void ivshmem_realize(PCIDevice *dev, Error **errp)
+{
+    IVShmemState *s = IVSHMEM_COMMON(dev);
+
+    if (!qtest_enabled()) {
+        error_report("ivshmem is deprecated, please use ivshmem-plain"
+                     " or ivshmem-doorbell instead");
+    }
+
+    if (!!s->server_chr + !!s->shmobj != 1) {
+        error_setg(errp, "You must specify either 'shm' or 'chardev'");
+        return;
+    }
+
+    if (s->sizearg == NULL) {
+        s->legacy_size = 4 << 20; /* 4 MB default */
+    } else {
+        char *end;
+        int64_t size = qemu_strtosz(s->sizearg, &end);
+        if (size < 0 || (size_t)size != size || *end != '\0'
+            || !is_power_of_2(size)) {
+            error_setg(errp, "Invalid size %s", s->sizearg);
+            return;
+        }
+        s->legacy_size = size;
+    }
+
+    /* check that role is reasonable */
+    if (s->role) {
+        if (strncmp(s->role, "peer", 5) == 0) {
+            s->master = ON_OFF_AUTO_OFF;
+        } else if (strncmp(s->role, "master", 7) == 0) {
+            s->master = ON_OFF_AUTO_ON;
+        } else {
+            error_setg(errp, "'role' must be 'peer' or 'master'");
+            return;
+        }
+    } else {
+        s->master = ON_OFF_AUTO_AUTO;
+    }
+
+    if (s->shmobj) {
+        desugar_shm(s);
+    }
+
+    /*
+     * Note: we don't use INTx with IVSHMEM_MSI at all, so this is a
+     * bald-faced lie then.  But it's a backwards compatible lie.
+     */
+    pci_config_set_interrupt_pin(dev->config, 1);
+
+    ivshmem_common_realize(dev, errp);
+}
+
 static void ivshmem_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ivshmem_init;
-    k->exit = pci_ivshmem_uninit;
-    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
-    k->device_id = PCI_DEVICE_ID_IVSHMEM;
-    k->class_id = PCI_CLASS_MEMORY_RAM;
-    dc->reset = ivshmem_reset;
+    k->realize = ivshmem_realize;
+    k->revision = 0;
+    dc->desc = "Inter-VM shared memory (legacy)";
     dc->props = ivshmem_properties;
-    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->vmsd = &ivshmem_vmsd;
 }
 
 static const TypeInfo ivshmem_info = {
     .name          = TYPE_IVSHMEM,
-    .parent        = TYPE_PCI_DEVICE,
+    .parent        = TYPE_IVSHMEM_COMMON,
     .instance_size = sizeof(IVShmemState),
     .class_init    = ivshmem_class_init,
 };
 
 static void ivshmem_register_types(void)
 {
+    type_register_static(&ivshmem_common_info);
+    type_register_static(&ivshmem_plain_info);
+    type_register_static(&ivshmem_doorbell_info);
     type_register_static(&ivshmem_info);
 }
 
diff --git a/hw/misc/macio/cuda.c b/hw/misc/macio/cuda.c
index b4273aa..05c02fb 100644
--- a/hw/misc/macio/cuda.c
+++ b/hw/misc/macio/cuda.c
@@ -22,11 +22,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/input/adb.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
+#include "qemu/log.h"
 
 /* XXX: implement all timer modes */
 
@@ -57,6 +60,8 @@
 #define IER_SET		0x80		/* set bits in IER */
 #define IER_CLR		0		/* clear bits in IER */
 #define SR_INT		0x04		/* Shift register full/empty */
+#define SR_DATA_INT	0x08
+#define SR_CLOCK_INT	0x10
 #define T1_INT          0x40            /* Timer 1 interrupt */
 #define T2_INT          0x20            /* Timer 2 interrupt */
 
@@ -103,11 +108,28 @@
 #define CUDA_COMBINED_FORMAT_IIC	0x25
 
 #define CUDA_TIMER_FREQ (4700000 / 6)
-#define CUDA_ADB_POLL_FREQ 50
 
 /* CUDA returns time_t's offset from Jan 1, 1904, not 1970 */
 #define RTC_OFFSET                      2082844800
 
+/* CUDA registers */
+#define CUDA_REG_B       0x00
+#define CUDA_REG_A       0x01
+#define CUDA_REG_DIRB    0x02
+#define CUDA_REG_DIRA    0x03
+#define CUDA_REG_T1CL    0x04
+#define CUDA_REG_T1CH    0x05
+#define CUDA_REG_T1LL    0x06
+#define CUDA_REG_T1LH    0x07
+#define CUDA_REG_T2CL    0x08
+#define CUDA_REG_T2CH    0x09
+#define CUDA_REG_SR      0x0a
+#define CUDA_REG_ACR     0x0b
+#define CUDA_REG_PCR     0x0c
+#define CUDA_REG_IFR     0x0d
+#define CUDA_REG_IER     0x0e
+#define CUDA_REG_ANH     0x0f
+
 static void cuda_update(CUDAState *s);
 static void cuda_receive_packet_from_host(CUDAState *s,
                                           const uint8_t *data, int len);
@@ -116,47 +138,48 @@
 
 static void cuda_update_irq(CUDAState *s)
 {
-    if (s->ifr & s->ier & (SR_INT | T1_INT)) {
+    if (s->ifr & s->ier & (SR_INT | T1_INT | T2_INT)) {
         qemu_irq_raise(s->irq);
     } else {
         qemu_irq_lower(s->irq);
     }
 }
 
-static uint64_t get_tb(uint64_t freq)
+static uint64_t get_tb(uint64_t time, uint64_t freq)
 {
-    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
-                    freq, get_ticks_per_sec());
+    return muldiv64(time, freq, NANOSECONDS_PER_SECOND);
 }
 
-static unsigned int get_counter(CUDATimer *s)
+static unsigned int get_counter(CUDATimer *ti)
 {
     int64_t d;
     unsigned int counter;
     uint64_t tb_diff;
+    uint64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     /* Reverse of the tb calculation algorithm that Mac OS X uses on bootup. */
-    tb_diff = get_tb(s->frequency) - s->load_time;
-    d = (tb_diff * 0xBF401675E5DULL) / (s->frequency << 24);
+    tb_diff = get_tb(current_time, ti->frequency) - ti->load_time;
+    d = (tb_diff * 0xBF401675E5DULL) / (ti->frequency << 24);
 
-    if (s->index == 0) {
+    if (ti->index == 0) {
         /* the timer goes down from latch to -1 (period of latch + 2) */
-        if (d <= (s->counter_value + 1)) {
-            counter = (s->counter_value - d) & 0xffff;
+        if (d <= (ti->counter_value + 1)) {
+            counter = (ti->counter_value - d) & 0xffff;
         } else {
-            counter = (d - (s->counter_value + 1)) % (s->latch + 2);
-            counter = (s->latch - counter) & 0xffff;
+            counter = (d - (ti->counter_value + 1)) % (ti->latch + 2);
+            counter = (ti->latch - counter) & 0xffff;
         }
     } else {
-        counter = (s->counter_value - d) & 0xffff;
+        counter = (ti->counter_value - d) & 0xffff;
     }
     return counter;
 }
 
 static void set_counter(CUDAState *s, CUDATimer *ti, unsigned int val)
 {
-    CUDA_DPRINTF("T%d.counter=%d\n", 1 + (ti->timer == NULL), val);
-    ti->load_time = get_tb(s->frequency);
+    CUDA_DPRINTF("T%d.counter=%d\n", 1 + ti->index, val);
+    ti->load_time = get_tb(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           s->frequency);
     ti->counter_value = val;
     cuda_timer_update(s, ti, ti->load_time);
 }
@@ -168,7 +191,7 @@
 
     /* current counter value */
     d = muldiv64(current_time - s->load_time,
-                 CUDA_TIMER_FREQ, get_ticks_per_sec());
+                 CUDA_TIMER_FREQ, NANOSECONDS_PER_SECOND);
     /* the timer goes down from latch to -1 (period of latch + 2) */
     if (d <= (s->counter_value + 1)) {
         counter = (s->counter_value - d) & 0xffff;
@@ -187,7 +210,7 @@
     }
     CUDA_DPRINTF("latch=%d counter=%" PRId64 " delta_next=%" PRId64 "\n",
                  s->latch, d, next_time - d);
-    next_time = muldiv64(next_time, get_ticks_per_sec(), CUDA_TIMER_FREQ) +
+    next_time = muldiv64(next_time, NANOSECONDS_PER_SECOND, CUDA_TIMER_FREQ) +
         s->load_time;
     if (next_time <= current_time)
         next_time = current_time + 1;
@@ -199,7 +222,7 @@
 {
     if (!ti->timer)
         return;
-    if ((s->acr & T1MODE) != T1MODE_CONT) {
+    if (ti->index == 0 && (s->acr & T1MODE) != T1MODE_CONT) {
         timer_del(ti->timer);
     } else {
         ti->next_irq_time = get_next_irq_time(ti, current_time);
@@ -217,6 +240,41 @@
     cuda_update_irq(s);
 }
 
+static void cuda_timer2(void *opaque)
+{
+    CUDAState *s = opaque;
+    CUDATimer *ti = &s->timers[1];
+
+    cuda_timer_update(s, ti, ti->next_irq_time);
+    s->ifr |= T2_INT;
+    cuda_update_irq(s);
+}
+
+static void cuda_set_sr_int(void *opaque)
+{
+    CUDAState *s = opaque;
+
+    CUDA_DPRINTF("CUDA: %s:%d\n", __func__, __LINE__);
+    s->ifr |= SR_INT;
+    cuda_update_irq(s);
+}
+
+static void cuda_delay_set_sr_int(CUDAState *s)
+{
+    int64_t expire;
+
+    if (s->dirb == 0xff) {
+        /* Not in Mac OS, fire the IRQ directly */
+        cuda_set_sr_int(s);
+        return;
+    }
+
+    CUDA_DPRINTF("CUDA: %s:%d\n", __func__, __LINE__);
+
+    expire = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + 300 * SCALE_US;
+    timer_mod(s->sr_delay_timer, expire);
+}
+
 static uint32_t cuda_readb(void *opaque, hwaddr addr)
 {
     CUDAState *s = opaque;
@@ -224,66 +282,68 @@
 
     addr = (addr >> 9) & 0xf;
     switch(addr) {
-    case 0:
+    case CUDA_REG_B:
         val = s->b;
         break;
-    case 1:
+    case CUDA_REG_A:
         val = s->a;
         break;
-    case 2:
+    case CUDA_REG_DIRB:
         val = s->dirb;
         break;
-    case 3:
+    case CUDA_REG_DIRA:
         val = s->dira;
         break;
-    case 4:
+    case CUDA_REG_T1CL:
         val = get_counter(&s->timers[0]) & 0xff;
         s->ifr &= ~T1_INT;
         cuda_update_irq(s);
         break;
-    case 5:
+    case CUDA_REG_T1CH:
         val = get_counter(&s->timers[0]) >> 8;
         cuda_update_irq(s);
         break;
-    case 6:
+    case CUDA_REG_T1LL:
         val = s->timers[0].latch & 0xff;
         break;
-    case 7:
+    case CUDA_REG_T1LH:
         /* XXX: check this */
         val = (s->timers[0].latch >> 8) & 0xff;
         break;
-    case 8:
+    case CUDA_REG_T2CL:
         val = get_counter(&s->timers[1]) & 0xff;
         s->ifr &= ~T2_INT;
-        break;
-    case 9:
-        val = get_counter(&s->timers[1]) >> 8;
-        break;
-    case 10:
-        val = s->sr;
-        s->ifr &= ~SR_INT;
         cuda_update_irq(s);
         break;
-    case 11:
+    case CUDA_REG_T2CH:
+        val = get_counter(&s->timers[1]) >> 8;
+        break;
+    case CUDA_REG_SR:
+        val = s->sr;
+        s->ifr &= ~(SR_INT | SR_CLOCK_INT | SR_DATA_INT);
+        cuda_update_irq(s);
+        break;
+    case CUDA_REG_ACR:
         val = s->acr;
         break;
-    case 12:
+    case CUDA_REG_PCR:
         val = s->pcr;
         break;
-    case 13:
+    case CUDA_REG_IFR:
         val = s->ifr;
-        if (s->ifr & s->ier)
+        if (s->ifr & s->ier) {
             val |= 0x80;
+        }
         break;
-    case 14:
+    case CUDA_REG_IER:
         val = s->ier | 0x80;
         break;
     default:
-    case 15:
+    case CUDA_REG_ANH:
         val = s->anh;
         break;
     }
-    if (addr != 13 || val != 0) {
+    if (addr != CUDA_REG_IFR || val != 0) {
         CUDA_DPRINTF("read: reg=0x%x val=%02x\n", (int)addr, val);
     }
 
@@ -298,61 +358,65 @@
     CUDA_DPRINTF("write: reg=0x%x val=%02x\n", (int)addr, val);
 
     switch(addr) {
-    case 0:
+    case CUDA_REG_B:
         s->b = val;
         cuda_update(s);
         break;
-    case 1:
+    case CUDA_REG_A:
         s->a = val;
         break;
-    case 2:
+    case CUDA_REG_DIRB:
         s->dirb = val;
         break;
-    case 3:
+    case CUDA_REG_DIRA:
         s->dira = val;
         break;
-    case 4:
+    case CUDA_REG_T1CL:
         s->timers[0].latch = (s->timers[0].latch & 0xff00) | val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 5:
+    case CUDA_REG_T1CH:
         s->timers[0].latch = (s->timers[0].latch & 0xff) | (val << 8);
         s->ifr &= ~T1_INT;
         set_counter(s, &s->timers[0], s->timers[0].latch);
         break;
-    case 6:
+    case CUDA_REG_T1LL:
         s->timers[0].latch = (s->timers[0].latch & 0xff00) | val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 7:
+    case CUDA_REG_T1LH:
         s->timers[0].latch = (s->timers[0].latch & 0xff) | (val << 8);
         s->ifr &= ~T1_INT;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         break;
-    case 8:
-        s->timers[1].latch = val;
-        set_counter(s, &s->timers[1], val);
+    case CUDA_REG_T2CL:
+        s->timers[1].latch = (s->timers[1].latch & 0xff00) | val;
         break;
-    case 9:
-        set_counter(s, &s->timers[1], (val << 8) | s->timers[1].latch);
+    case CUDA_REG_T2CH:
+        /* To ensure T2 generates an interrupt on zero crossing with the
+           common timer code, write the value directly from the latch to
+           the counter */
+        s->timers[1].latch = (s->timers[1].latch & 0xff) | (val << 8);
+        s->ifr &= ~T2_INT;
+        set_counter(s, &s->timers[1], s->timers[1].latch);
         break;
-    case 10:
+    case CUDA_REG_SR:
         s->sr = val;
         break;
-    case 11:
+    case CUDA_REG_ACR:
         s->acr = val;
         cuda_timer_update(s, &s->timers[0], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
         cuda_update(s);
         break;
-    case 12:
+    case CUDA_REG_PCR:
         s->pcr = val;
         break;
-    case 13:
+    case CUDA_REG_IFR:
         /* reset bits */
         s->ifr &= ~val;
         cuda_update_irq(s);
         break;
-    case 14:
+    case CUDA_REG_IER:
         if (val & IER_SET) {
             /* set bits */
             s->ier |= val & 0x7f;
@@ -363,7 +427,7 @@
         cuda_update_irq(s);
         break;
     default:
-    case 15:
+    case CUDA_REG_ANH:
         s->anh = val;
         break;
     }
@@ -384,8 +448,7 @@
                 if (s->data_out_index < sizeof(s->data_out)) {
                     CUDA_DPRINTF("send: %02x\n", s->sr);
                     s->data_out[s->data_out_index++] = s->sr;
-                    s->ifr |= SR_INT;
-                    cuda_update_irq(s);
+                    cuda_delay_set_sr_int(s);
                 }
             }
         } else {
@@ -398,8 +461,7 @@
                     if (s->data_in_index >= s->data_in_size) {
                         s->b = (s->b | TREQ);
                     }
-                    s->ifr |= SR_INT;
-                    cuda_update_irq(s);
+                    cuda_delay_set_sr_int(s);
                 }
             }
         }
@@ -411,15 +473,13 @@
                 s->b = (s->b | TREQ);
             else
                 s->b = (s->b & ~TREQ);
-            s->ifr |= SR_INT;
-            cuda_update_irq(s);
+            cuda_delay_set_sr_int(s);
         } else {
             if (!(s->last_b & TIP)) {
                 /* handle end of host to cuda transfer */
                 packet_received = (s->data_out_index > 0);
                 /* always an IRQ at the end of transfer */
-                s->ifr |= SR_INT;
-                cuda_update_irq(s);
+                cuda_delay_set_sr_int(s);
             }
             /* signal if there is data to read */
             if (s->data_in_index < s->data_in_size) {
@@ -456,8 +516,7 @@
     s->data_in_size = len;
     s->data_in_index = 0;
     cuda_update(s);
-    s->ifr |= SR_INT;
-    cuda_update_irq(s);
+    cuda_delay_set_sr_int(s);
 }
 
 static void cuda_adb_poll(void *opaque)
@@ -466,7 +525,7 @@
     uint8_t obuf[ADB_MAX_OUT_LEN + 2];
     int olen;
 
-    olen = adb_poll(&s->adb_bus, obuf + 2);
+    olen = adb_poll(&s->adb_bus, obuf + 2, s->adb_poll_mask);
     if (olen > 0) {
         obuf[0] = ADB_PACKET;
         obuf[1] = 0x40; /* polled data */
@@ -474,75 +533,213 @@
     }
     timer_mod(s->adb_poll_timer,
                    qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   (get_ticks_per_sec() / CUDA_ADB_POLL_FREQ));
+                   (NANOSECONDS_PER_SECOND / (1000 / s->autopoll_rate_ms)));
 }
 
+/* description of commands */
+typedef struct CudaCommand {
+    uint8_t command;
+    const char *name;
+    bool (*handler)(CUDAState *s,
+                    const uint8_t *in_args, int in_len,
+                    uint8_t *out_args, int *out_len);
+} CudaCommand;
+
+static bool cuda_cmd_autopoll(CUDAState *s,
+                              const uint8_t *in_data, int in_len,
+                              uint8_t *out_data, int *out_len)
+{
+    int autopoll;
+
+    if (in_len != 1) {
+        return false;
+    }
+
+    autopoll = (in_data[0] != 0);
+    if (autopoll != s->autopoll) {
+        s->autopoll = autopoll;
+        if (autopoll) {
+            timer_mod(s->adb_poll_timer,
+                      qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      (NANOSECONDS_PER_SECOND / (1000 / s->autopoll_rate_ms)));
+        } else {
+            timer_del(s->adb_poll_timer);
+        }
+    }
+    return true;
+}
+
+static bool cuda_cmd_set_autorate(CUDAState *s,
+                                  const uint8_t *in_data, int in_len,
+                                  uint8_t *out_data, int *out_len)
+{
+    if (in_len != 1) {
+        return false;
+    }
+
+    /* we don't want a period of 0 ms */
+    /* FIXME: check what real hardware does */
+    if (in_data[0] == 0) {
+        return false;
+    }
+
+    s->autopoll_rate_ms = in_data[0];
+    if (s->autopoll) {
+        timer_mod(s->adb_poll_timer,
+                  qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  (NANOSECONDS_PER_SECOND / (1000 / s->autopoll_rate_ms)));
+    }
+    return true;
+}
+
+static bool cuda_cmd_set_device_list(CUDAState *s,
+                                     const uint8_t *in_data, int in_len,
+                                     uint8_t *out_data, int *out_len)
+{
+    if (in_len != 2) {
+        return false;
+    }
+
+    s->adb_poll_mask = (((uint16_t)in_data[0]) << 8) | in_data[1];
+    return true;
+}
+
+static bool cuda_cmd_powerdown(CUDAState *s,
+                               const uint8_t *in_data, int in_len,
+                               uint8_t *out_data, int *out_len)
+{
+    if (in_len != 0) {
+        return false;
+    }
+
+    qemu_system_shutdown_request();
+    return true;
+}
+
+static bool cuda_cmd_reset_system(CUDAState *s,
+                                  const uint8_t *in_data, int in_len,
+                                  uint8_t *out_data, int *out_len)
+{
+    if (in_len != 0) {
+        return false;
+    }
+
+    qemu_system_reset_request();
+    return true;
+}
+
+static bool cuda_cmd_set_file_server_flag(CUDAState *s,
+                                          const uint8_t *in_data, int in_len,
+                                          uint8_t *out_data, int *out_len)
+{
+    if (in_len != 1) {
+        return false;
+    }
+
+    qemu_log_mask(LOG_UNIMP,
+                  "CUDA: unimplemented command FILE_SERVER_FLAG %d\n",
+                  in_data[0]);
+    return true;
+}
+
+static bool cuda_cmd_set_power_message(CUDAState *s,
+                                       const uint8_t *in_data, int in_len,
+                                       uint8_t *out_data, int *out_len)
+{
+    if (in_len != 1) {
+        return false;
+    }
+
+    qemu_log_mask(LOG_UNIMP,
+                  "CUDA: unimplemented command SET_POWER_MESSAGE %d\n",
+                  in_data[0]);
+    return true;
+}
+
+static bool cuda_cmd_get_time(CUDAState *s,
+                              const uint8_t *in_data, int in_len,
+                              uint8_t *out_data, int *out_len)
+{
+    uint32_t ti;
+
+    if (in_len != 0) {
+        return false;
+    }
+
+    ti = s->tick_offset + (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)
+                           / NANOSECONDS_PER_SECOND);
+    out_data[0] = ti >> 24;
+    out_data[1] = ti >> 16;
+    out_data[2] = ti >> 8;
+    out_data[3] = ti;
+    *out_len = 4;
+    return true;
+}
+
+static bool cuda_cmd_set_time(CUDAState *s,
+                              const uint8_t *in_data, int in_len,
+                              uint8_t *out_data, int *out_len)
+{
+    uint32_t ti;
+
+    if (in_len != 4) {
+        return false;
+    }
+
+    ti = (((uint32_t)in_data[0]) << 24) + (((uint32_t)in_data[1]) << 16)
+         + (((uint32_t)in_data[2]) << 8) + in_data[3];
+    s->tick_offset = ti - (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)
+                           / NANOSECONDS_PER_SECOND);
+    return true;
+}
+
+static const CudaCommand handlers[] = {
+    { CUDA_AUTOPOLL, "AUTOPOLL", cuda_cmd_autopoll },
+    { CUDA_SET_AUTO_RATE, "SET_AUTO_RATE",  cuda_cmd_set_autorate },
+    { CUDA_SET_DEVICE_LIST, "SET_DEVICE_LIST", cuda_cmd_set_device_list },
+    { CUDA_POWERDOWN, "POWERDOWN", cuda_cmd_powerdown },
+    { CUDA_RESET_SYSTEM, "RESET_SYSTEM", cuda_cmd_reset_system },
+    { CUDA_FILE_SERVER_FLAG, "FILE_SERVER_FLAG",
+      cuda_cmd_set_file_server_flag },
+    { CUDA_SET_POWER_MESSAGES, "SET_POWER_MESSAGES",
+      cuda_cmd_set_power_message },
+    { CUDA_GET_TIME, "GET_TIME", cuda_cmd_get_time },
+    { CUDA_SET_TIME, "SET_TIME", cuda_cmd_set_time },
+};
+
 static void cuda_receive_packet(CUDAState *s,
                                 const uint8_t *data, int len)
 {
-    uint8_t obuf[16];
-    int autopoll;
-    uint32_t ti;
+    uint8_t obuf[16] = { CUDA_PACKET, 0, data[0] };
+    int i, out_len = 0;
 
-    switch(data[0]) {
-    case CUDA_AUTOPOLL:
-        autopoll = (data[1] != 0);
-        if (autopoll != s->autopoll) {
-            s->autopoll = autopoll;
-            if (autopoll) {
-                timer_mod(s->adb_poll_timer,
-                               qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                               (get_ticks_per_sec() / CUDA_ADB_POLL_FREQ));
+    for (i = 0; i < ARRAY_SIZE(handlers); i++) {
+        const CudaCommand *desc = &handlers[i];
+        if (desc->command == data[0]) {
+            CUDA_DPRINTF("handling command %s\n", desc->name);
+            out_len = 0;
+            if (desc->handler(s, data + 1, len - 1, obuf + 3, &out_len)) {
+                cuda_send_packet_to_host(s, obuf, 3 + out_len);
             } else {
-                timer_del(s->adb_poll_timer);
+                qemu_log_mask(LOG_GUEST_ERROR,
+                              "CUDA: %s: wrong parameters %d\n",
+                              desc->name, len);
+                obuf[0] = ERROR_PACKET;
+                obuf[1] = 0x5; /* bad parameters */
+                obuf[2] = CUDA_PACKET;
+                obuf[3] = data[0];
+                cuda_send_packet_to_host(s, obuf, 4);
             }
+            return;
         }
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = data[1];
-        cuda_send_packet_to_host(s, obuf, 2);
-        break;
-    case CUDA_SET_TIME:
-        ti = (((uint32_t)data[1]) << 24) + (((uint32_t)data[2]) << 16) + (((uint32_t)data[3]) << 8) + data[4];
-        s->tick_offset = ti - (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / get_ticks_per_sec());
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        obuf[2] = 0;
-        cuda_send_packet_to_host(s, obuf, 3);
-        break;
-    case CUDA_GET_TIME:
-        ti = s->tick_offset + (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / get_ticks_per_sec());
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        obuf[2] = 0;
-        obuf[3] = ti >> 24;
-        obuf[4] = ti >> 16;
-        obuf[5] = ti >> 8;
-        obuf[6] = ti;
-        cuda_send_packet_to_host(s, obuf, 7);
-        break;
-    case CUDA_FILE_SERVER_FLAG:
-    case CUDA_SET_DEVICE_LIST:
-    case CUDA_SET_AUTO_RATE:
-    case CUDA_SET_POWER_MESSAGES:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
-        break;
-    case CUDA_POWERDOWN:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
-        qemu_system_shutdown_request();
-        break;
-    case CUDA_RESET_SYSTEM:
-        obuf[0] = CUDA_PACKET;
-        obuf[1] = 0;
-        cuda_send_packet_to_host(s, obuf, 2);
-        qemu_system_reset_request();
-        break;
-    default:
-        break;
     }
+
+    qemu_log_mask(LOG_GUEST_ERROR, "CUDA: unknown command 0x%02x\n", data[0]);
+    obuf[0] = ERROR_PACKET;
+    obuf[1] = 0x2; /* unknown command */
+    obuf[2] = CUDA_PACKET;
+    obuf[3] = data[0];
+    cuda_send_packet_to_host(s, obuf, 4);
 }
 
 static void cuda_receive_packet_from_host(CUDAState *s,
@@ -560,19 +757,21 @@
     switch(data[0]) {
     case ADB_PACKET:
         {
-            uint8_t obuf[ADB_MAX_OUT_LEN + 2];
+            uint8_t obuf[ADB_MAX_OUT_LEN + 3];
             int olen;
             olen = adb_request(&s->adb_bus, obuf + 2, data + 1, len - 1);
             if (olen > 0) {
                 obuf[0] = ADB_PACKET;
                 obuf[1] = 0x00;
+                cuda_send_packet_to_host(s, obuf, olen + 2);
             } else {
                 /* error */
                 obuf[0] = ADB_PACKET;
                 obuf[1] = -olen;
+                obuf[2] = data[1];
                 olen = 0;
+                cuda_send_packet_to_host(s, obuf, olen + 3);
             }
-            cuda_send_packet_to_host(s, obuf, olen + 2);
         }
         break;
     case CUDA_PACKET:
@@ -631,22 +830,24 @@
         VMSTATE_UINT16(counter_value, CUDATimer),
         VMSTATE_INT64(load_time, CUDATimer),
         VMSTATE_INT64(next_irq_time, CUDATimer),
-        VMSTATE_TIMER_TEST(timer, CUDATimer, cuda_timer_exist),
+        VMSTATE_TIMER_PTR_TEST(timer, CUDATimer, cuda_timer_exist),
         VMSTATE_END_OF_LIST()
     }
 };
 
 static const VMStateDescription vmstate_cuda = {
     .name = "cuda",
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 4,
+    .minimum_version_id = 4,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(a, CUDAState),
         VMSTATE_UINT8(b, CUDAState),
+        VMSTATE_UINT8(last_b, CUDAState),
         VMSTATE_UINT8(dira, CUDAState),
         VMSTATE_UINT8(dirb, CUDAState),
         VMSTATE_UINT8(sr, CUDAState),
         VMSTATE_UINT8(acr, CUDAState),
+        VMSTATE_UINT8(last_acr, CUDAState),
         VMSTATE_UINT8(pcr, CUDAState),
         VMSTATE_UINT8(ifr, CUDAState),
         VMSTATE_UINT8(ier, CUDAState),
@@ -655,11 +856,15 @@
         VMSTATE_INT32(data_in_index, CUDAState),
         VMSTATE_INT32(data_out_index, CUDAState),
         VMSTATE_UINT8(autopoll, CUDAState),
+        VMSTATE_UINT8(autopoll_rate_ms, CUDAState),
+        VMSTATE_UINT16(adb_poll_mask, CUDAState),
         VMSTATE_BUFFER(data_in, CUDAState),
         VMSTATE_BUFFER(data_out, CUDAState),
         VMSTATE_UINT32(tick_offset, CUDAState),
         VMSTATE_STRUCT_ARRAY(timers, CUDAState, 2, 1,
                              vmstate_cuda_timer, CUDATimer),
+        VMSTATE_TIMER_PTR(adb_poll_timer, CUDAState),
+        VMSTATE_TIMER_PTR(sr_delay_timer, CUDAState),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -670,7 +875,7 @@
 
     s->b = 0;
     s->a = 0;
-    s->dirb = 0;
+    s->dirb = 0xff;
     s->dira = 0;
     s->sr = 0;
     s->acr = 0;
@@ -687,8 +892,9 @@
     s->timers[0].latch = 0xffff;
     set_counter(s, &s->timers[0], 0xffff);
 
-    s->timers[1].latch = 0;
-    set_counter(s, &s->timers[1], 0xffff);
+    s->timers[1].latch = 0xffff;
+
+    s->sr_delay_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_set_sr_int, s);
 }
 
 static void cuda_realizefn(DeviceState *dev, Error **errp)
@@ -698,12 +904,15 @@
 
     s->timers[0].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer1, s);
     s->timers[0].frequency = s->frequency;
-    s->timers[1].frequency = s->frequency;
+    s->timers[1].timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_timer2, s);
+    s->timers[1].frequency = (SCALE_US * 6000) / 4700;
 
     qemu_get_timedate(&tm, 0);
     s->tick_offset = (uint32_t)mktimegm(&tm) + RTC_OFFSET;
 
     s->adb_poll_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cuda_adb_poll, s);
+    s->autopoll_rate_ms = 20;
+    s->adb_poll_mask = 0xffff;
 }
 
 static void cuda_initfn(Object *obj)
@@ -712,7 +921,7 @@
     CUDAState *s = CUDA(obj);
     int i;
 
-    memory_region_init_io(&s->mem, NULL, &cuda_ops, s, "cuda", 0x2000);
+    memory_region_init_io(&s->mem, obj, &cuda_ops, s, "cuda", 0x2000);
     sysbus_init_mmio(d, &s->mem);
     sysbus_init_irq(d, &s->irq);
 
@@ -737,6 +946,7 @@
     dc->reset = cuda_reset;
     dc->vmsd = &vmstate_cuda;
     dc->props = cuda_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo cuda_type_info = {
diff --git a/hw/misc/macio/mac_dbdma.c b/hw/misc/macio/mac_dbdma.c
index b25e851..15452b9 100644
--- a/hw/misc/macio/mac_dbdma.c
+++ b/hw/misc/macio/mac_dbdma.c
@@ -36,20 +36,31 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "hw/ppc/mac_dbdma.h"
 #include "qemu/main-loop.h"
+#include "qemu/log.h"
+#include "sysemu/dma.h"
 
 /* debug DBDMA */
-//#define DEBUG_DBDMA
+#define DEBUG_DBDMA 0
+#define DEBUG_DBDMA_CHANMASK ((1ull << DBDMA_CHANNELS) - 1)
 
-#ifdef DEBUG_DBDMA
-#define DBDMA_DPRINTF(fmt, ...)                                 \
-    do { printf("DBDMA: " fmt , ## __VA_ARGS__); } while (0)
-#else
-#define DBDMA_DPRINTF(fmt, ...)
-#endif
+#define DBDMA_DPRINTF(fmt, ...) do { \
+    if (DEBUG_DBDMA) { \
+        printf("DBDMA: " fmt , ## __VA_ARGS__); \
+    } \
+} while (0);
+
+#define DBDMA_DPRINTFCH(ch, fmt, ...) do { \
+    if (DEBUG_DBDMA) { \
+        if ((1ul << (ch)->channel) & DEBUG_DBDMA_CHANMASK) { \
+            printf("DBDMA[%02x]: " fmt , (ch)->channel, ## __VA_ARGS__); \
+        } \
+    } \
+} while (0);
 
 /*
  */
@@ -59,7 +70,7 @@
     return container_of(ch, DBDMAState, channels[ch->channel]);
 }
 
-#ifdef DEBUG_DBDMA
+#if DEBUG_DBDMA
 static void dump_dbdma_cmd(dbdma_cmd *cmd)
 {
     printf("dbdma_cmd %p\n", cmd);
@@ -77,26 +88,26 @@
 #endif
 static void dbdma_cmdptr_load(DBDMA_channel *ch)
 {
-    DBDMA_DPRINTF("dbdma_cmdptr_load 0x%08x\n",
-                  ch->regs[DBDMA_CMDPTR_LO]);
-    cpu_physical_memory_read(ch->regs[DBDMA_CMDPTR_LO],
-                             &ch->current, sizeof(dbdma_cmd));
+    DBDMA_DPRINTFCH(ch, "dbdma_cmdptr_load 0x%08x\n",
+                    ch->regs[DBDMA_CMDPTR_LO]);
+    dma_memory_read(&address_space_memory, ch->regs[DBDMA_CMDPTR_LO],
+                    &ch->current, sizeof(dbdma_cmd));
 }
 
 static void dbdma_cmdptr_save(DBDMA_channel *ch)
 {
-    DBDMA_DPRINTF("dbdma_cmdptr_save 0x%08x\n",
-                  ch->regs[DBDMA_CMDPTR_LO]);
-    DBDMA_DPRINTF("xfer_status 0x%08x res_count 0x%04x\n",
-                  le16_to_cpu(ch->current.xfer_status),
-                  le16_to_cpu(ch->current.res_count));
-    cpu_physical_memory_write(ch->regs[DBDMA_CMDPTR_LO],
-                              &ch->current, sizeof(dbdma_cmd));
+    DBDMA_DPRINTFCH(ch, "dbdma_cmdptr_save 0x%08x\n",
+                    ch->regs[DBDMA_CMDPTR_LO]);
+    DBDMA_DPRINTFCH(ch, "xfer_status 0x%08x res_count 0x%04x\n",
+                    le16_to_cpu(ch->current.xfer_status),
+                    le16_to_cpu(ch->current.res_count));
+    dma_memory_write(&address_space_memory, ch->regs[DBDMA_CMDPTR_LO],
+                     &ch->current, sizeof(dbdma_cmd));
 }
 
 static void kill_channel(DBDMA_channel *ch)
 {
-    DBDMA_DPRINTF("kill_channel\n");
+    DBDMA_DPRINTFCH(ch, "kill_channel\n");
 
     ch->regs[DBDMA_STATUS] |= DEAD;
     ch->regs[DBDMA_STATUS] &= ~ACTIVE;
@@ -112,7 +123,7 @@
     uint32_t status;
     int cond;
 
-    DBDMA_DPRINTF("%s\n", __func__);
+    DBDMA_DPRINTFCH(ch, "%s\n", __func__);
 
     intr = le16_to_cpu(current->command) & INTR_MASK;
 
@@ -121,7 +132,7 @@
         return;
     case INTR_ALWAYS: /* always interrupt */
         qemu_irq_raise(ch->irq);
-        DBDMA_DPRINTF("%s: raise\n", __func__);
+        DBDMA_DPRINTFCH(ch, "%s: raise\n", __func__);
         return;
     }
 
@@ -136,13 +147,13 @@
     case INTR_IFSET:  /* intr if condition bit is 1 */
         if (cond) {
             qemu_irq_raise(ch->irq);
-            DBDMA_DPRINTF("%s: raise\n", __func__);
+            DBDMA_DPRINTFCH(ch, "%s: raise\n", __func__);
         }
         return;
     case INTR_IFCLR:  /* intr if condition bit is 0 */
         if (!cond) {
             qemu_irq_raise(ch->irq);
-            DBDMA_DPRINTF("%s: raise\n", __func__);
+            DBDMA_DPRINTFCH(ch, "%s: raise\n", __func__);
         }
         return;
     }
@@ -156,7 +167,7 @@
     uint32_t status;
     int cond;
 
-    DBDMA_DPRINTF("conditional_wait\n");
+    DBDMA_DPRINTFCH(ch, "conditional_wait\n");
 
     wait = le16_to_cpu(current->command) & WAIT_MASK;
 
@@ -202,7 +213,7 @@
 {
     dbdma_cmd *current = &ch->current;
 
-    ch->regs[DBDMA_CMDPTR_LO] = current->cmd_dep;
+    ch->regs[DBDMA_CMDPTR_LO] = le32_to_cpu(current->cmd_dep);
     ch->regs[DBDMA_STATUS] |= BT;
     dbdma_cmdptr_load(ch);
 }
@@ -215,7 +226,7 @@
     uint32_t status;
     int cond;
 
-    DBDMA_DPRINTF("conditional_branch\n");
+    DBDMA_DPRINTFCH(ch, "conditional_branch\n");
 
     /* check if we must branch */
 
@@ -260,7 +271,7 @@
     DBDMA_channel *ch = io->channel;
     dbdma_cmd *current = &ch->current;
 
-    DBDMA_DPRINTF("%s\n", __func__);
+    DBDMA_DPRINTFCH(ch, "%s\n", __func__);
 
     if (conditional_wait(ch))
         goto wait;
@@ -286,13 +297,13 @@
 static void start_output(DBDMA_channel *ch, int key, uint32_t addr,
                         uint16_t req_count, int is_last)
 {
-    DBDMA_DPRINTF("start_output\n");
+    DBDMA_DPRINTFCH(ch, "start_output\n");
 
     /* KEY_REGS, KEY_DEVICE and KEY_STREAM
      * are not implemented in the mac-io chip
      */
 
-    DBDMA_DPRINTF("addr 0x%x key 0x%x\n", addr, key);
+    DBDMA_DPRINTFCH(ch, "addr 0x%x key 0x%x\n", addr, key);
     if (!addr || key > KEY_STREAM3) {
         kill_channel(ch);
         return;
@@ -312,13 +323,13 @@
 static void start_input(DBDMA_channel *ch, int key, uint32_t addr,
                        uint16_t req_count, int is_last)
 {
-    DBDMA_DPRINTF("start_input\n");
+    DBDMA_DPRINTFCH(ch, "start_input\n");
 
     /* KEY_REGS, KEY_DEVICE and KEY_STREAM
      * are not implemented in the mac-io chip
      */
 
-    DBDMA_DPRINTF("addr 0x%x key 0x%x\n", addr, key);
+    DBDMA_DPRINTFCH(ch, "addr 0x%x key 0x%x\n", addr, key);
     if (!addr || key > KEY_STREAM3) {
         kill_channel(ch);
         return;
@@ -339,9 +350,8 @@
                      uint16_t len)
 {
     dbdma_cmd *current = &ch->current;
-    uint32_t val;
 
-    DBDMA_DPRINTF("load_word\n");
+    DBDMA_DPRINTFCH(ch, "load_word %d bytes, addr=%08x\n", len, addr);
 
     /* only implements KEY_SYSTEM */
 
@@ -351,14 +361,7 @@
         return;
     }
 
-    cpu_physical_memory_read(addr, &val, len);
-
-    if (len == 2)
-        val = (val << 16) | (current->cmd_dep & 0x0000ffff);
-    else if (len == 1)
-        val = (val << 24) | (current->cmd_dep & 0x00ffffff);
-
-    current->cmd_dep = val;
+    dma_memory_read(&address_space_memory, addr, &current->cmd_dep, len);
 
     if (conditional_wait(ch))
         goto wait;
@@ -378,9 +381,9 @@
                       uint16_t len)
 {
     dbdma_cmd *current = &ch->current;
-    uint32_t val;
 
-    DBDMA_DPRINTF("store_word\n");
+    DBDMA_DPRINTFCH(ch, "store_word %d bytes, addr=%08x pa=%x\n",
+                    len, addr, le32_to_cpu(current->cmd_dep));
 
     /* only implements KEY_SYSTEM */
 
@@ -390,13 +393,7 @@
         return;
     }
 
-    val = current->cmd_dep;
-    if (len == 2)
-        val >>= 16;
-    else if (len == 1)
-        val >>= 24;
-
-    cpu_physical_memory_write(addr, &val, len);
+    dma_memory_write(&address_space_memory, addr, &current->cmd_dep, len);
 
     if (conditional_wait(ch))
         goto wait;
@@ -443,7 +440,7 @@
     uint16_t req_count;
     uint32_t phy_addr;
 
-    DBDMA_DPRINTF("channel_run\n");
+    DBDMA_DPRINTFCH(ch, "channel_run\n");
     dump_dbdma_cmd(current);
 
     /* clear WAKE flag at command fetch */
@@ -537,9 +534,9 @@
 {
     DBDMAState *s = opaque;
 
-    DBDMA_DPRINTF("DBDMA_run_bh\n");
-
+    DBDMA_DPRINTF("-> DBDMA_run_bh\n");
     DBDMA_run(s);
+    DBDMA_DPRINTF("<- DBDMA_run_bh\n");
 }
 
 void DBDMA_kick(DBDMAState *dbdma)
@@ -554,14 +551,15 @@
     DBDMAState *s = dbdma;
     DBDMA_channel *ch = &s->channels[nchan];
 
-    DBDMA_DPRINTF("DBDMA_register_channel 0x%x\n", nchan);
+    DBDMA_DPRINTFCH(ch, "DBDMA_register_channel 0x%x\n", nchan);
+
+    assert(rw);
+    assert(flush);
 
     ch->irq = irq;
-    ch->channel = nchan;
     ch->rw = rw;
     ch->flush = flush;
     ch->io.opaque = opaque;
-    ch->io.channel = ch;
 }
 
 static void
@@ -590,22 +588,20 @@
     if ((ch->regs[DBDMA_STATUS] & RUN) && !(status & RUN)) {
         /* RUN is cleared */
         status &= ~(ACTIVE|DEAD);
-        if ((status & FLUSH) && ch->flush) {
-            ch->flush(&ch->io);
-            status &= ~FLUSH;
-        }
     }
 
-    DBDMA_DPRINTF("    status 0x%08x\n", status);
+    if ((status & FLUSH) && ch->flush) {
+        ch->flush(&ch->io);
+        status &= ~FLUSH;
+    }
+
+    DBDMA_DPRINTFCH(ch, "    status 0x%08x\n", status);
 
     ch->regs[DBDMA_STATUS] = status;
 
     if (status & ACTIVE) {
         DBDMA_kick(dbdma_from_ch(ch));
     }
-    if ((status & FLUSH) && ch->flush) {
-        ch->flush(&ch->io);
-    }
 }
 
 static void dbdma_write(void *opaque, hwaddr addr,
@@ -616,10 +612,10 @@
     DBDMA_channel *ch = &s->channels[channel];
     int reg = (addr - (channel << DBDMA_CHANNEL_SHIFT)) >> 2;
 
-    DBDMA_DPRINTF("writel 0x" TARGET_FMT_plx " <= 0x%08"PRIx64"\n",
-                  addr, value);
-    DBDMA_DPRINTF("channel 0x%x reg 0x%x\n",
-                  (uint32_t)addr >> DBDMA_CHANNEL_SHIFT, reg);
+    DBDMA_DPRINTFCH(ch, "writel 0x" TARGET_FMT_plx " <= 0x%08"PRIx64"\n",
+                    addr, value);
+    DBDMA_DPRINTFCH(ch, "channel 0x%x reg 0x%x\n",
+                    (uint32_t)addr >> DBDMA_CHANNEL_SHIFT, reg);
 
     /* cmdptr cannot be modified if channel is ACTIVE */
 
@@ -670,9 +666,9 @@
 
     value = ch->regs[reg];
 
-    DBDMA_DPRINTF("readl 0x" TARGET_FMT_plx " => 0x%08x\n", addr, value);
-    DBDMA_DPRINTF("channel 0x%x reg 0x%x\n",
-                  (uint32_t)addr >> DBDMA_CHANNEL_SHIFT, reg);
+    DBDMA_DPRINTFCH(ch, "readl 0x" TARGET_FMT_plx " => 0x%08x\n", addr, value);
+    DBDMA_DPRINTFCH(ch, "channel 0x%x reg 0x%x\n",
+                    (uint32_t)addr >> DBDMA_CHANNEL_SHIFT, reg);
 
     switch(reg) {
     case DBDMA_CONTROL:
@@ -715,20 +711,52 @@
     },
 };
 
-static const VMStateDescription vmstate_dbdma_channel = {
-    .name = "dbdma_channel",
+static const VMStateDescription vmstate_dbdma_io = {
+    .name = "dbdma_io",
     .version_id = 0,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
+        VMSTATE_UINT64(addr, struct DBDMA_io),
+        VMSTATE_INT32(len, struct DBDMA_io),
+        VMSTATE_INT32(is_last, struct DBDMA_io),
+        VMSTATE_INT32(is_dma_out, struct DBDMA_io),
+        VMSTATE_BOOL(processing, struct DBDMA_io),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_dbdma_cmd = {
+    .name = "dbdma_cmd",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(req_count, dbdma_cmd),
+        VMSTATE_UINT16(command, dbdma_cmd),
+        VMSTATE_UINT32(phy_addr, dbdma_cmd),
+        VMSTATE_UINT32(cmd_dep, dbdma_cmd),
+        VMSTATE_UINT16(res_count, dbdma_cmd),
+        VMSTATE_UINT16(xfer_status, dbdma_cmd),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_dbdma_channel = {
+    .name = "dbdma_channel",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
         VMSTATE_UINT32_ARRAY(regs, struct DBDMA_channel, DBDMA_REGS),
+        VMSTATE_STRUCT(io, struct DBDMA_channel, 0, vmstate_dbdma_io, DBDMA_io),
+        VMSTATE_STRUCT(current, struct DBDMA_channel, 0, vmstate_dbdma_cmd,
+                       dbdma_cmd),
         VMSTATE_END_OF_LIST()
     }
 };
 
 static const VMStateDescription vmstate_dbdma = {
     .name = "dbdma",
-    .version_id = 2,
-    .minimum_version_id = 2,
+    .version_id = 3,
+    .minimum_version_id = 3,
     .fields = (VMStateField[]) {
         VMSTATE_STRUCT_ARRAY(channels, DBDMAState, DBDMA_CHANNELS, 1,
                              vmstate_dbdma_channel, DBDMA_channel),
@@ -745,6 +773,31 @@
         memset(s->channels[i].regs, 0, DBDMA_SIZE);
 }
 
+static void dbdma_unassigned_rw(DBDMA_io *io)
+{
+    DBDMA_channel *ch = io->channel;
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: use of unassigned channel %d\n",
+                  __func__, ch->channel);
+    ch->io.processing = false;
+}
+
+static void dbdma_unassigned_flush(DBDMA_io *io)
+{
+    DBDMA_channel *ch = io->channel;
+    dbdma_cmd *current = &ch->current;
+    uint16_t cmd;
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: use of unassigned channel %d\n",
+                  __func__, ch->channel);
+
+    cmd = le16_to_cpu(current->command) & COMMAND_MASK;
+    if (cmd == OUTPUT_MORE || cmd == OUTPUT_LAST ||
+        cmd == INPUT_MORE || cmd == INPUT_LAST) {
+        current->xfer_status = cpu_to_le16(ch->regs[DBDMA_STATUS] | FLUSH);
+        current->res_count = cpu_to_le16(io->len);
+        dbdma_cmdptr_save(ch);
+    }
+}
+
 void* DBDMA_init (MemoryRegion **dbdma_mem)
 {
     DBDMAState *s;
@@ -754,7 +807,13 @@
 
     for (i = 0; i < DBDMA_CHANNELS; i++) {
         DBDMA_io *io = &s->channels[i].io;
+        DBDMA_channel *ch = &s->channels[i];
         qemu_iovec_init(&io->iov, 1);
+
+        ch->rw = dbdma_unassigned_rw;
+        ch->flush = dbdma_unassigned_flush;
+        ch->channel = i;
+        ch->io.channel = ch;
     }
 
     memory_region_init_io(&s->mem, NULL, &dbdma_ops, s, "dbdma", 0x1000);
diff --git a/hw/misc/macio/macio.c b/hw/misc/macio/macio.c
index e0f1e88..be03926 100644
--- a/hw/misc/macio/macio.c
+++ b/hw/misc/macio/macio.c
@@ -22,6 +22,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -105,10 +107,10 @@
         0xF0, 0xE0,
     };
 
-    memory_region_init(escc_legacy, NULL, "escc-legacy", 256);
+    memory_region_init(escc_legacy, OBJECT(macio_state), "escc-legacy", 256);
     for (i = 0; i < ARRAY_SIZE(maps); i += 2) {
         MemoryRegion *port = g_new(MemoryRegion, 1);
-        memory_region_init_alias(port, NULL, "escc-legacy-port",
+        memory_region_init_alias(port, OBJECT(macio_state), "escc-legacy-port",
                                  macio_state->escc_mem, maps[i+1], 0x2);
         memory_region_add_subregion(escc_legacy, maps[i], port);
     }
@@ -126,17 +128,20 @@
     }
 }
 
-static int macio_common_initfn(PCIDevice *d)
+static void macio_common_realize(PCIDevice *d, Error **errp)
 {
     MacIOState *s = MACIO(d);
     SysBusDevice *sysbus_dev;
-    int ret;
+    Error *err = NULL;
+    MemoryRegion *dbdma_mem;
 
-    d->config[0x3d] = 0x01; // interrupt on pin 1
+    s->dbdma = DBDMA_init(&dbdma_mem);
+    memory_region_add_subregion(&s->bar, 0x08000, dbdma_mem);
 
-    ret = qdev_init(DEVICE(&s->cuda));
-    if (ret < 0) {
-        return ret;
+    object_property_set_bool(OBJECT(&s->cuda), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
     sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
     memory_region_add_subregion(&s->bar, 0x16000,
@@ -144,12 +149,11 @@
 
     macio_bar_setup(s);
     pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
-
-    return 0;
 }
 
-static int macio_initfn_ide(MacIOState *s, MACIOIDEState *ide, qemu_irq irq0,
-                            qemu_irq irq1, int dmaid)
+static void macio_realize_ide(MacIOState *s, MACIOIDEState *ide,
+                              qemu_irq irq0, qemu_irq irq1, int dmaid,
+                              Error **errp)
 {
     SysBusDevice *sysbus_dev;
 
@@ -157,27 +161,31 @@
     sysbus_connect_irq(sysbus_dev, 0, irq0);
     sysbus_connect_irq(sysbus_dev, 1, irq1);
     macio_ide_register_dma(ide, s->dbdma, dmaid);
-    return qdev_init(DEVICE(ide));
+    object_property_set_bool(OBJECT(ide), true, "realized", errp);
 }
 
-static int macio_oldworld_initfn(PCIDevice *d)
+static void macio_oldworld_realize(PCIDevice *d, Error **errp)
 {
     MacIOState *s = MACIO(d);
     OldWorldMacIOState *os = OLDWORLD_MACIO(d);
+    Error *err = NULL;
     SysBusDevice *sysbus_dev;
     int i;
     int cur_irq = 0;
-    int ret = macio_common_initfn(d);
-    if (ret < 0) {
-        return ret;
+
+    macio_common_realize(d, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
 
     sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
     sysbus_connect_irq(sysbus_dev, 0, os->irqs[cur_irq++]);
 
-    ret = qdev_init(DEVICE(&os->nvram));
-    if (ret < 0) {
-        return ret;
+    object_property_set_bool(OBJECT(&os->nvram), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
     sysbus_dev = SYS_BUS_DEVICE(&os->nvram);
     memory_region_add_subregion(&s->bar, 0x60000,
@@ -194,13 +202,12 @@
         qemu_irq irq0 = os->irqs[cur_irq++];
         qemu_irq irq1 = os->irqs[cur_irq++];
 
-        ret = macio_initfn_ide(s, &os->ide[i], irq0, irq1, 0x16 + (i * 4));
-        if (ret < 0) {
-            return ret;
+        macio_realize_ide(s, &os->ide[i], irq0, irq1, 0x16 + (i * 4), &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
         }
     }
-
-    return 0;
 }
 
 static void macio_init_ide(MacIOState *s, MACIOIDEState *ide, size_t ide_size,
@@ -247,7 +254,7 @@
     uint64_t systime = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     uint64_t kltime;
 
-    kltime = muldiv64(systime, 4194300, get_ticks_per_sec() * 4);
+    kltime = muldiv64(systime, 4194300, NANOSECONDS_PER_SECOND * 4);
     kltime = muldiv64(kltime, 18432000, 1048575);
 
     switch (addr) {
@@ -268,17 +275,20 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int macio_newworld_initfn(PCIDevice *d)
+static void macio_newworld_realize(PCIDevice *d, Error **errp)
 {
     MacIOState *s = MACIO(d);
     NewWorldMacIOState *ns = NEWWORLD_MACIO(d);
+    Error *err = NULL;
     SysBusDevice *sysbus_dev;
-    MemoryRegion *timer_memory = g_new(MemoryRegion, 1);
+    MemoryRegion *timer_memory = NULL;
     int i;
     int cur_irq = 0;
-    int ret = macio_common_initfn(d);
-    if (ret < 0) {
-        return ret;
+
+    macio_common_realize(d, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
 
     sysbus_dev = SYS_BUS_DEVICE(&s->cuda);
@@ -294,18 +304,18 @@
         qemu_irq irq0 = ns->irqs[cur_irq++];
         qemu_irq irq1 = ns->irqs[cur_irq++];
 
-        ret = macio_initfn_ide(s, &ns->ide[i], irq0, irq1, 0x16 + (i * 4));
-        if (ret < 0) {
-            return ret;
+        macio_realize_ide(s, &ns->ide[i], irq0, irq1, 0x16 + (i * 4), &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
         }
     }
 
     /* Timer */
+    timer_memory = g_new(MemoryRegion, 1);
     memory_region_init_io(timer_memory, OBJECT(s), &timer_ops, NULL, "timer",
                           0x1000);
     memory_region_add_subregion(&s->bar, 0x15000, timer_memory);
-
-    return 0;
 }
 
 static void macio_newworld_init(Object *obj)
@@ -324,32 +334,52 @@
 static void macio_instance_init(Object *obj)
 {
     MacIOState *s = MACIO(obj);
-    MemoryRegion *dbdma_mem;
 
-    memory_region_init(&s->bar, NULL, "macio", 0x80000);
+    memory_region_init(&s->bar, obj, "macio", 0x80000);
 
     object_initialize(&s->cuda, sizeof(s->cuda), TYPE_CUDA);
     qdev_set_parent_bus(DEVICE(&s->cuda), sysbus_get_default());
     object_property_add_child(obj, "cuda", OBJECT(&s->cuda), NULL);
-
-    s->dbdma = DBDMA_init(&dbdma_mem);
-    memory_region_add_subregion(&s->bar, 0x08000, dbdma_mem);
 }
 
+static const VMStateDescription vmstate_macio_oldworld = {
+    .name = "macio-oldworld",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj.parent, OldWorldMacIOState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void macio_oldworld_class_init(ObjectClass *oc, void *data)
 {
     PCIDeviceClass *pdc = PCI_DEVICE_CLASS(oc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
 
-    pdc->init = macio_oldworld_initfn;
+    pdc->realize = macio_oldworld_realize;
     pdc->device_id = PCI_DEVICE_ID_APPLE_343S1201;
+    dc->vmsd = &vmstate_macio_oldworld;
 }
 
+static const VMStateDescription vmstate_macio_newworld = {
+    .name = "macio-newworld",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj.parent, NewWorldMacIOState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void macio_newworld_class_init(ObjectClass *oc, void *data)
 {
     PCIDeviceClass *pdc = PCI_DEVICE_CLASS(oc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
 
-    pdc->init = macio_newworld_initfn;
+    pdc->realize = macio_newworld_realize;
     pdc->device_id = PCI_DEVICE_ID_APPLE_UNI_N_KEYL;
+    dc->vmsd = &vmstate_macio_newworld;
 }
 
 static Property macio_properties[] = {
@@ -365,6 +395,7 @@
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->class_id = PCI_CLASS_OTHERS << 8;
     dc->props = macio_properties;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo macio_oldworld_type_info = {
diff --git a/hw/misc/max111x.c b/hw/misc/max111x.c
index bef3651..2a277bd 100644
--- a/hw/misc/max111x.c
+++ b/hw/misc/max111x.c
@@ -10,7 +10,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 
 typedef struct {
     SSISlave parent_obj;
@@ -146,14 +147,14 @@
     return 0;
 }
 
-static int max1110_init(SSISlave *dev)
+static void max1110_realize(SSISlave *dev, Error **errp)
 {
-    return max111x_init(dev, 8);
+    max111x_init(dev, 8);
 }
 
-static int max1111_init(SSISlave *dev)
+static void max1111_realize(SSISlave *dev, Error **errp)
 {
-    return max111x_init(dev, 4);
+    max111x_init(dev, 4);
 }
 
 void max111x_set_input(DeviceState *dev, int line, uint8_t value)
@@ -182,7 +183,7 @@
 {
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = max1110_init;
+    k->realize = max1110_realize;
 }
 
 static const TypeInfo max1110_info = {
@@ -195,7 +196,7 @@
 {
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = max1111_init;
+    k->realize = max1111_realize;
 }
 
 static const TypeInfo max1111_info = {
diff --git a/hw/misc/milkymist-hpdmc.c b/hw/misc/milkymist-hpdmc.c
index f5f4c1b..e6140ee 100644
--- a/hw/misc/milkymist-hpdmc.c
+++ b/hw/misc/milkymist-hpdmc.c
@@ -18,9 +18,10 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/hpdmc.pdf
+ *   http://milkymist.walle.cc/socdoc/hpdmc.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
diff --git a/hw/misc/milkymist-pfpu.c b/hw/misc/milkymist-pfpu.c
index 609f33f..1da21a6 100644
--- a/hw/misc/milkymist-pfpu.c
+++ b/hw/misc/milkymist-pfpu.c
@@ -18,10 +18,11 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/pfpu.pdf
+ *   http://milkymist.walle.cc/socdoc/pfpu.pdf
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -362,7 +363,7 @@
             i = 0;
             while (pfpu_decode_insn(s)) {
                 /* decode at most MICROCODE_WORDS instructions */
-                if (i++ >= MICROCODE_WORDS) {
+                if (++i >= MICROCODE_WORDS) {
                     error_report("milkymist_pfpu: too many instructions "
                             "executed in microcode. No VECTOUT?");
                     break;
diff --git a/hw/misc/mips_cmgcr.c b/hw/misc/mips_cmgcr.c
new file mode 100644
index 0000000..b3ba166
--- /dev/null
+++ b/hw/misc/mips_cmgcr.c
@@ -0,0 +1,246 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
+ * Authors: Sanjay Lal <sanjayl@kymasys.com>
+ *
+ * Copyright (C) 2015 Imagination Technologies
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/log.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
+#include "hw/misc/mips_cmgcr.h"
+#include "hw/misc/mips_cpc.h"
+#include "hw/intc/mips_gic.h"
+
+static inline bool is_cpc_connected(MIPSGCRState *s)
+{
+    return s->cpc_mr != NULL;
+}
+
+static inline bool is_gic_connected(MIPSGCRState *s)
+{
+    return s->gic_mr != NULL;
+}
+
+static inline void update_cpc_base(MIPSGCRState *gcr, uint64_t val)
+{
+    if (is_cpc_connected(gcr)) {
+        gcr->cpc_base = val & GCR_CPC_BASE_MSK;
+        memory_region_transaction_begin();
+        memory_region_set_address(gcr->cpc_mr,
+                                  gcr->cpc_base & GCR_CPC_BASE_CPCBASE_MSK);
+        memory_region_set_enabled(gcr->cpc_mr,
+                                  gcr->cpc_base & GCR_CPC_BASE_CPCEN_MSK);
+        memory_region_transaction_commit();
+    }
+}
+
+static inline void update_gic_base(MIPSGCRState *gcr, uint64_t val)
+{
+    if (is_gic_connected(gcr)) {
+        gcr->gic_base = val & GCR_GIC_BASE_MSK;
+        memory_region_transaction_begin();
+        memory_region_set_address(gcr->gic_mr,
+                                  gcr->gic_base & GCR_GIC_BASE_GICBASE_MSK);
+        memory_region_set_enabled(gcr->gic_mr,
+                                  gcr->gic_base & GCR_GIC_BASE_GICEN_MSK);
+        memory_region_transaction_commit();
+    }
+}
+
+/* Read GCR registers */
+static uint64_t gcr_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSGCRState *gcr = (MIPSGCRState *) opaque;
+    MIPSGCRVPState *current_vps = &gcr->vps[current_cpu->cpu_index];
+    MIPSGCRVPState *other_vps = &gcr->vps[current_vps->other];
+
+    switch (addr) {
+    /* Global Control Block Register */
+    case GCR_CONFIG_OFS:
+        /* Set PCORES to 0 */
+        return 0;
+    case GCR_BASE_OFS:
+        return gcr->gcr_base;
+    case GCR_REV_OFS:
+        return gcr->gcr_rev;
+    case GCR_GIC_BASE_OFS:
+        return gcr->gic_base;
+    case GCR_CPC_BASE_OFS:
+        return gcr->cpc_base;
+    case GCR_GIC_STATUS_OFS:
+        return is_gic_connected(gcr);
+    case GCR_CPC_STATUS_OFS:
+        return is_cpc_connected(gcr);
+    case GCR_L2_CONFIG_OFS:
+        /* L2 BYPASS */
+        return GCR_L2_CONFIG_BYPASS_MSK;
+        /* Core-Local and Core-Other Control Blocks */
+    case MIPS_CLCB_OFS + GCR_CL_CONFIG_OFS:
+    case MIPS_COCB_OFS + GCR_CL_CONFIG_OFS:
+        /* Set PVP to # of VPs - 1 */
+        return gcr->num_vps - 1;
+    case MIPS_CLCB_OFS + GCR_CL_RESETBASE_OFS:
+        return current_vps->reset_base;
+    case MIPS_COCB_OFS + GCR_CL_RESETBASE_OFS:
+        return other_vps->reset_base;
+    case MIPS_CLCB_OFS + GCR_CL_OTHER_OFS:
+        return current_vps->other;
+    case MIPS_COCB_OFS + GCR_CL_OTHER_OFS:
+        return other_vps->other;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Read %d bytes at GCR offset 0x%" HWADDR_PRIx
+                      "\n", size, addr);
+        return 0;
+    }
+    return 0;
+}
+
+static inline target_ulong get_exception_base(MIPSGCRVPState *vps)
+{
+    /* TODO: BEV_BASE and SELECT_BEV */
+    return (int32_t)(vps->reset_base & GCR_CL_RESET_BASE_RESETBASE_MSK);
+}
+
+/* Write GCR registers */
+static void gcr_write(void *opaque, hwaddr addr, uint64_t data, unsigned size)
+{
+    MIPSGCRState *gcr = (MIPSGCRState *)opaque;
+    MIPSGCRVPState *current_vps = &gcr->vps[current_cpu->cpu_index];
+    MIPSGCRVPState *other_vps = &gcr->vps[current_vps->other];
+
+    switch (addr) {
+    case GCR_GIC_BASE_OFS:
+        update_gic_base(gcr, data);
+        break;
+    case GCR_CPC_BASE_OFS:
+        update_cpc_base(gcr, data);
+        break;
+    case MIPS_CLCB_OFS + GCR_CL_RESETBASE_OFS:
+        current_vps->reset_base = data & GCR_CL_RESET_BASE_MSK;
+        cpu_set_exception_base(current_cpu->cpu_index,
+                               get_exception_base(current_vps));
+        break;
+    case MIPS_COCB_OFS + GCR_CL_RESETBASE_OFS:
+        other_vps->reset_base = data & GCR_CL_RESET_BASE_MSK;
+        cpu_set_exception_base(current_vps->other,
+                               get_exception_base(other_vps));
+        break;
+    case MIPS_CLCB_OFS + GCR_CL_OTHER_OFS:
+        if ((data & GCR_CL_OTHER_MSK) < gcr->num_vps) {
+            current_vps->other = data & GCR_CL_OTHER_MSK;
+        }
+        break;
+    case MIPS_COCB_OFS + GCR_CL_OTHER_OFS:
+        if ((data & GCR_CL_OTHER_MSK) < gcr->num_vps) {
+            other_vps->other = data & GCR_CL_OTHER_MSK;
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "Write %d bytes at GCR offset 0x%" HWADDR_PRIx
+                      " 0x%" PRIx64 "\n", size, addr, data);
+        break;
+    }
+}
+
+static const MemoryRegionOps gcr_ops = {
+    .read = gcr_read,
+    .write = gcr_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .max_access_size = 8,
+    },
+};
+
+static void mips_gcr_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSGCRState *s = MIPS_GCR(obj);
+
+    object_property_add_link(obj, "gic", TYPE_MEMORY_REGION,
+                             (Object **)&s->gic_mr,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+
+    object_property_add_link(obj, "cpc", TYPE_MEMORY_REGION,
+                             (Object **)&s->cpc_mr,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+
+    memory_region_init_io(&s->iomem, OBJECT(s), &gcr_ops, s,
+                          "mips-gcr", GCR_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static void mips_gcr_reset(DeviceState *dev)
+{
+    MIPSGCRState *s = MIPS_GCR(dev);
+    int i;
+
+    update_gic_base(s, 0);
+    update_cpc_base(s, 0);
+
+    for (i = 0; i < s->num_vps; i++) {
+        s->vps[i].other = 0;
+        s->vps[i].reset_base = 0xBFC00000 & GCR_CL_RESET_BASE_MSK;
+        cpu_set_exception_base(i, get_exception_base(&s->vps[i]));
+    }
+}
+
+static const VMStateDescription vmstate_mips_gcr = {
+    .name = "mips-gcr",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(cpc_base, MIPSGCRState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property mips_gcr_properties[] = {
+    DEFINE_PROP_INT32("num-vp", MIPSGCRState, num_vps, 1),
+    DEFINE_PROP_INT32("gcr-rev", MIPSGCRState, gcr_rev, 0x800),
+    DEFINE_PROP_UINT64("gcr-base", MIPSGCRState, gcr_base, GCR_BASE_ADDR),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_gcr_realize(DeviceState *dev, Error **errp)
+{
+    MIPSGCRState *s = MIPS_GCR(dev);
+
+    /* Create local set of registers for each VP */
+    s->vps = g_new(MIPSGCRVPState, s->num_vps);
+}
+
+static void mips_gcr_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->props = mips_gcr_properties;
+    dc->vmsd = &vmstate_mips_gcr;
+    dc->reset = mips_gcr_reset;
+    dc->realize = mips_gcr_realize;
+}
+
+static const TypeInfo mips_gcr_info = {
+    .name          = TYPE_MIPS_GCR,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSGCRState),
+    .instance_init = mips_gcr_init,
+    .class_init    = mips_gcr_class_init,
+};
+
+static void mips_gcr_register_types(void)
+{
+    type_register_static(&mips_gcr_info);
+}
+
+type_init(mips_gcr_register_types)
diff --git a/hw/misc/mips_cpc.c b/hw/misc/mips_cpc.c
new file mode 100644
index 0000000..6d34574
--- /dev/null
+++ b/hw/misc/mips_cpc.c
@@ -0,0 +1,178 @@
+/*
+ * Cluster Power Controller emulation
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu/log.h"
+#include "hw/sysbus.h"
+
+#include "hw/misc/mips_cpc.h"
+
+static inline uint64_t cpc_vp_run_mask(MIPSCPCState *cpc)
+{
+    return (1ULL << cpc->num_vp) - 1;
+}
+
+static void cpc_run_vp(MIPSCPCState *cpc, uint64_t vp_run)
+{
+    CPUState *cs = first_cpu;
+
+    CPU_FOREACH(cs) {
+        uint64_t i = 1ULL << cs->cpu_index;
+        if (i & vp_run & ~cpc->vp_running) {
+            cpu_reset(cs);
+            cpc->vp_running |= i;
+        }
+    }
+}
+
+static void cpc_stop_vp(MIPSCPCState *cpc, uint64_t vp_stop)
+{
+    CPUState *cs = first_cpu;
+
+    CPU_FOREACH(cs) {
+        uint64_t i = 1ULL << cs->cpu_index;
+        if (i & vp_stop & cpc->vp_running) {
+            cpu_interrupt(cs, CPU_INTERRUPT_HALT);
+            cpc->vp_running &= ~i;
+        }
+    }
+}
+
+static void cpc_write(void *opaque, hwaddr offset, uint64_t data,
+                      unsigned size)
+{
+    MIPSCPCState *s = opaque;
+
+    switch (offset) {
+    case CPC_CL_BASE_OFS + CPC_VP_RUN_OFS:
+    case CPC_CO_BASE_OFS + CPC_VP_RUN_OFS:
+        cpc_run_vp(s, data & cpc_vp_run_mask(s));
+        break;
+    case CPC_CL_BASE_OFS + CPC_VP_STOP_OFS:
+    case CPC_CO_BASE_OFS + CPC_VP_STOP_OFS:
+        cpc_stop_vp(s, data & cpc_vp_run_mask(s));
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Bad offset 0x%x\n",  __func__, (int)offset);
+        break;
+    }
+
+    return;
+}
+
+static uint64_t cpc_read(void *opaque, hwaddr offset, unsigned size)
+{
+    MIPSCPCState *s = opaque;
+
+    switch (offset) {
+    case CPC_CL_BASE_OFS + CPC_VP_RUNNING_OFS:
+    case CPC_CO_BASE_OFS + CPC_VP_RUNNING_OFS:
+        return s->vp_running;
+    default:
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Bad offset 0x%x\n",  __func__, (int)offset);
+        return 0;
+    }
+}
+
+static const MemoryRegionOps cpc_ops = {
+    .read = cpc_read,
+    .write = cpc_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .impl = {
+        .max_access_size = 8,
+    },
+};
+
+static void mips_cpc_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSCPCState *s = MIPS_CPC(obj);
+
+    memory_region_init_io(&s->mr, OBJECT(s), &cpc_ops, s, "mips-cpc",
+                          CPC_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->mr);
+}
+
+static void mips_cpc_realize(DeviceState *dev, Error **errp)
+{
+    MIPSCPCState *s = MIPS_CPC(dev);
+
+    if (s->vp_start_running > cpc_vp_run_mask(s)) {
+        error_setg(errp,
+                   "incorrect vp_start_running 0x%" PRIx64 " for num_vp = %d",
+                   s->vp_running, s->num_vp);
+        return;
+    }
+}
+
+static void mips_cpc_reset(DeviceState *dev)
+{
+    MIPSCPCState *s = MIPS_CPC(dev);
+
+    /* Reflect the fact that all VPs are halted on reset */
+    s->vp_running = 0;
+
+    /* Put selected VPs into run state */
+    cpc_run_vp(s, s->vp_start_running);
+}
+
+static const VMStateDescription vmstate_mips_cpc = {
+    .name = "mips-cpc",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(vp_running, MIPSCPCState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static Property mips_cpc_properties[] = {
+    DEFINE_PROP_UINT32("num-vp", MIPSCPCState, num_vp, 0x1),
+    DEFINE_PROP_UINT64("vp-start-running", MIPSCPCState, vp_start_running, 0x1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_cpc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = mips_cpc_realize;
+    dc->reset = mips_cpc_reset;
+    dc->vmsd = &vmstate_mips_cpc;
+    dc->props = mips_cpc_properties;
+}
+
+static const TypeInfo mips_cpc_info = {
+    .name          = TYPE_MIPS_CPC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSCPCState),
+    .instance_init = mips_cpc_init,
+    .class_init    = mips_cpc_class_init,
+};
+
+static void mips_cpc_register_types(void)
+{
+    type_register_static(&mips_cpc_info);
+}
+
+type_init(mips_cpc_register_types)
diff --git a/hw/misc/mips_itu.c b/hw/misc/mips_itu.c
new file mode 100644
index 0000000..ef935b5
--- /dev/null
+++ b/hw/misc/mips_itu.c
@@ -0,0 +1,524 @@
+/*
+ * Inter-Thread Communication Unit emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu/log.h"
+#include "exec/exec-all.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
+#include "hw/misc/mips_itu.h"
+
+#define ITC_TAG_ADDRSPACE_SZ (ITC_ADDRESSMAP_NUM * 8)
+/* Initialize as 4kB area to fit all 32 cells with default 128B grain.
+   Storage may be resized by the software. */
+#define ITC_STORAGE_ADDRSPACE_SZ 0x1000
+
+#define ITC_FIFO_NUM_MAX 16
+#define ITC_SEMAPH_NUM_MAX 16
+#define ITC_AM1_NUMENTRIES_OFS 20
+
+#define ITC_CELL_PV_MAX_VAL 0xFFFF
+
+#define ITC_CELL_TAG_FIFO_DEPTH 28
+#define ITC_CELL_TAG_FIFO_PTR 18
+#define ITC_CELL_TAG_FIFO 17
+#define ITC_CELL_TAG_T 16
+#define ITC_CELL_TAG_F 1
+#define ITC_CELL_TAG_E 0
+
+#define ITC_AM0_BASE_ADDRESS_MASK 0xFFFFFC00ULL
+#define ITC_AM0_EN_MASK 0x1
+
+#define ITC_AM1_ADDR_MASK_MASK 0x1FC00
+#define ITC_AM1_ENTRY_GRAIN_MASK 0x7
+
+typedef enum ITCView {
+    ITCVIEW_BYPASS  = 0,
+    ITCVIEW_CONTROL = 1,
+    ITCVIEW_EF_SYNC = 2,
+    ITCVIEW_EF_TRY  = 3,
+    ITCVIEW_PV_SYNC = 4,
+    ITCVIEW_PV_TRY  = 5
+} ITCView;
+
+MemoryRegion *mips_itu_get_tag_region(MIPSITUState *itu)
+{
+    return &itu->tag_io;
+}
+
+static uint64_t itc_tag_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSITUState *tag = (MIPSITUState *)opaque;
+    uint64_t index = addr >> 3;
+
+    if (index >= ITC_ADDRESSMAP_NUM) {
+        qemu_log_mask(LOG_GUEST_ERROR, "Read 0x%" PRIx64 "\n", addr);
+        return 0;
+    }
+
+    return tag->ITCAddressMap[index];
+}
+
+static void itc_reconfigure(MIPSITUState *tag)
+{
+    uint64_t *am = &tag->ITCAddressMap[0];
+    MemoryRegion *mr = &tag->storage_io;
+    hwaddr address = am[0] & ITC_AM0_BASE_ADDRESS_MASK;
+    uint64_t size = (1 << 10) + (am[1] & ITC_AM1_ADDR_MASK_MASK);
+    bool is_enabled = (am[0] & ITC_AM0_EN_MASK) != 0;
+
+    memory_region_transaction_begin();
+    if (!(size & (size - 1))) {
+        memory_region_set_size(mr, size);
+    }
+    memory_region_set_address(mr, address);
+    memory_region_set_enabled(mr, is_enabled);
+    memory_region_transaction_commit();
+}
+
+static void itc_tag_write(void *opaque, hwaddr addr,
+                          uint64_t data, unsigned size)
+{
+    MIPSITUState *tag = (MIPSITUState *)opaque;
+    uint64_t *am = &tag->ITCAddressMap[0];
+    uint64_t am_old, mask;
+    uint64_t index = addr >> 3;
+
+    switch (index) {
+    case 0:
+        mask = ITC_AM0_BASE_ADDRESS_MASK | ITC_AM0_EN_MASK;
+        break;
+    case 1:
+        mask = ITC_AM1_ADDR_MASK_MASK | ITC_AM1_ENTRY_GRAIN_MASK;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "Bad write 0x%" PRIx64 "\n", addr);
+        return;
+    }
+
+    am_old = am[index];
+    am[index] = (data & mask) | (am_old & ~mask);
+    if (am_old != am[index]) {
+        itc_reconfigure(tag);
+    }
+}
+
+static const MemoryRegionOps itc_tag_ops = {
+    .read = itc_tag_read,
+    .write = itc_tag_write,
+    .impl = {
+        .max_access_size = 8,
+    },
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static inline uint32_t get_num_cells(MIPSITUState *s)
+{
+    return s->num_fifo + s->num_semaphores;
+}
+
+static inline ITCView get_itc_view(hwaddr addr)
+{
+    return (addr >> 3) & 0xf;
+}
+
+static inline int get_cell_stride_shift(const MIPSITUState *s)
+{
+    /* Minimum interval (for EntryGain = 0) is 128 B */
+    return 7 + (s->ITCAddressMap[1] & ITC_AM1_ENTRY_GRAIN_MASK);
+}
+
+static inline ITCStorageCell *get_cell(MIPSITUState *s,
+                                       hwaddr addr)
+{
+    uint32_t cell_idx = addr >> get_cell_stride_shift(s);
+    uint32_t num_cells = get_num_cells(s);
+
+    if (cell_idx >= num_cells) {
+        cell_idx = num_cells - 1;
+    }
+
+    return &s->cell[cell_idx];
+}
+
+static void wake_blocked_threads(ITCStorageCell *c)
+{
+    CPUState *cs;
+    CPU_FOREACH(cs) {
+        if (cs->halted && (c->blocked_threads & (1ULL << cs->cpu_index))) {
+            cpu_interrupt(cs, CPU_INTERRUPT_WAKE);
+        }
+    }
+    c->blocked_threads = 0;
+}
+
+static void QEMU_NORETURN block_thread_and_exit(ITCStorageCell *c)
+{
+    c->blocked_threads |= 1ULL << current_cpu->cpu_index;
+    cpu_restore_state(current_cpu, current_cpu->mem_io_pc);
+    current_cpu->halted = 1;
+    current_cpu->exception_index = EXCP_HLT;
+    cpu_loop_exit(current_cpu);
+}
+
+/* ITC Bypass View */
+
+static inline uint64_t view_bypass_read(ITCStorageCell *c)
+{
+    if (c->tag.FIFO) {
+        return c->data[c->fifo_out];
+    } else {
+        return c->data[0];
+    }
+}
+
+static inline void view_bypass_write(ITCStorageCell *c, uint64_t val)
+{
+    if (c->tag.FIFO && (c->tag.FIFOPtr > 0)) {
+        int idx = (c->fifo_out + c->tag.FIFOPtr - 1) % ITC_CELL_DEPTH;
+        c->data[idx] = val;
+    }
+
+    /* ignore a write to the semaphore cell */
+}
+
+/* ITC Control View */
+
+static inline uint64_t view_control_read(ITCStorageCell *c)
+{
+    return ((uint64_t)c->tag.FIFODepth << ITC_CELL_TAG_FIFO_DEPTH) |
+           (c->tag.FIFOPtr << ITC_CELL_TAG_FIFO_PTR) |
+           (c->tag.FIFO << ITC_CELL_TAG_FIFO) |
+           (c->tag.T << ITC_CELL_TAG_T) |
+           (c->tag.E << ITC_CELL_TAG_E) |
+           (c->tag.F << ITC_CELL_TAG_F);
+}
+
+static inline void view_control_write(ITCStorageCell *c, uint64_t val)
+{
+    c->tag.T = (val >> ITC_CELL_TAG_T) & 1;
+    c->tag.E = (val >> ITC_CELL_TAG_E) & 1;
+    c->tag.F = (val >> ITC_CELL_TAG_F) & 1;
+
+    if (c->tag.E) {
+        c->tag.FIFOPtr = 0;
+    }
+}
+
+/* ITC Empty/Full View */
+
+static uint64_t view_ef_common_read(ITCStorageCell *c, bool blocking)
+{
+    uint64_t ret = 0;
+
+    if (!c->tag.FIFO) {
+        return 0;
+    }
+
+    c->tag.F = 0;
+
+    if (blocking && c->tag.E) {
+        block_thread_and_exit(c);
+    }
+
+    if (c->blocked_threads) {
+        wake_blocked_threads(c);
+    }
+
+    if (c->tag.FIFOPtr > 0) {
+        ret = c->data[c->fifo_out];
+        c->fifo_out = (c->fifo_out + 1) % ITC_CELL_DEPTH;
+        c->tag.FIFOPtr--;
+    }
+
+    if (c->tag.FIFOPtr == 0) {
+        c->tag.E = 1;
+    }
+
+    return ret;
+}
+
+static uint64_t view_ef_sync_read(ITCStorageCell *c)
+{
+    return view_ef_common_read(c, true);
+}
+
+static uint64_t view_ef_try_read(ITCStorageCell *c)
+{
+    return view_ef_common_read(c, false);
+}
+
+static inline void view_ef_common_write(ITCStorageCell *c, uint64_t val,
+                                        bool blocking)
+{
+    if (!c->tag.FIFO) {
+        return;
+    }
+
+    c->tag.E = 0;
+
+    if (blocking && c->tag.F) {
+        block_thread_and_exit(c);
+    }
+
+    if (c->blocked_threads) {
+        wake_blocked_threads(c);
+    }
+
+    if (c->tag.FIFOPtr < ITC_CELL_DEPTH) {
+        int idx = (c->fifo_out + c->tag.FIFOPtr) % ITC_CELL_DEPTH;
+        c->data[idx] = val;
+        c->tag.FIFOPtr++;
+    }
+
+    if (c->tag.FIFOPtr == ITC_CELL_DEPTH) {
+        c->tag.F = 1;
+    }
+}
+
+static void view_ef_sync_write(ITCStorageCell *c, uint64_t val)
+{
+    view_ef_common_write(c, val, true);
+}
+
+static void view_ef_try_write(ITCStorageCell *c, uint64_t val)
+{
+    view_ef_common_write(c, val, false);
+}
+
+/* ITC P/V View */
+
+static uint64_t view_pv_common_read(ITCStorageCell *c, bool blocking)
+{
+    uint64_t ret = c->data[0];
+
+    if (c->tag.FIFO) {
+        return 0;
+    }
+
+    if (c->data[0] > 0) {
+        c->data[0]--;
+    } else if (blocking) {
+        block_thread_and_exit(c);
+    }
+
+    return ret;
+}
+
+static uint64_t view_pv_sync_read(ITCStorageCell *c)
+{
+    return view_pv_common_read(c, true);
+}
+
+static uint64_t view_pv_try_read(ITCStorageCell *c)
+{
+    return view_pv_common_read(c, false);
+}
+
+static inline void view_pv_common_write(ITCStorageCell *c)
+{
+    if (c->tag.FIFO) {
+        return;
+    }
+
+    if (c->data[0] < ITC_CELL_PV_MAX_VAL) {
+        c->data[0]++;
+    }
+
+    if (c->blocked_threads) {
+        wake_blocked_threads(c);
+    }
+}
+
+static void view_pv_sync_write(ITCStorageCell *c)
+{
+    view_pv_common_write(c);
+}
+
+static void view_pv_try_write(ITCStorageCell *c)
+{
+    view_pv_common_write(c);
+}
+
+static uint64_t itc_storage_read(void *opaque, hwaddr addr, unsigned size)
+{
+    MIPSITUState *s = (MIPSITUState *)opaque;
+    ITCStorageCell *cell = get_cell(s, addr);
+    ITCView view = get_itc_view(addr);
+    uint64_t ret = -1;
+
+    switch (view) {
+    case ITCVIEW_BYPASS:
+        ret = view_bypass_read(cell);
+        break;
+    case ITCVIEW_CONTROL:
+        ret = view_control_read(cell);
+        break;
+    case ITCVIEW_EF_SYNC:
+        ret = view_ef_sync_read(cell);
+        break;
+    case ITCVIEW_EF_TRY:
+        ret = view_ef_try_read(cell);
+        break;
+    case ITCVIEW_PV_SYNC:
+        ret = view_pv_sync_read(cell);
+        break;
+    case ITCVIEW_PV_TRY:
+        ret = view_pv_try_read(cell);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "itc_storage_read: Bad ITC View %d\n", (int)view);
+        break;
+    }
+
+    return ret;
+}
+
+static void itc_storage_write(void *opaque, hwaddr addr, uint64_t data,
+                              unsigned size)
+{
+    MIPSITUState *s = (MIPSITUState *)opaque;
+    ITCStorageCell *cell = get_cell(s, addr);
+    ITCView view = get_itc_view(addr);
+
+    switch (view) {
+    case ITCVIEW_BYPASS:
+        view_bypass_write(cell, data);
+        break;
+    case ITCVIEW_CONTROL:
+        view_control_write(cell, data);
+        break;
+    case ITCVIEW_EF_SYNC:
+        view_ef_sync_write(cell, data);
+        break;
+    case ITCVIEW_EF_TRY:
+        view_ef_try_write(cell, data);
+        break;
+    case ITCVIEW_PV_SYNC:
+        view_pv_sync_write(cell);
+        break;
+    case ITCVIEW_PV_TRY:
+        view_pv_try_write(cell);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "itc_storage_write: Bad ITC View %d\n", (int)view);
+        break;
+    }
+
+}
+
+static const MemoryRegionOps itc_storage_ops = {
+    .read = itc_storage_read,
+    .write = itc_storage_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void itc_reset_cells(MIPSITUState *s)
+{
+    int i;
+
+    memset(s->cell, 0, get_num_cells(s) * sizeof(s->cell[0]));
+
+    for (i = 0; i < s->num_fifo; i++) {
+        s->cell[i].tag.E = 1;
+        s->cell[i].tag.FIFO = 1;
+        s->cell[i].tag.FIFODepth = ITC_CELL_DEPTH_SHIFT;
+    }
+}
+
+static void mips_itu_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    MIPSITUState *s = MIPS_ITU(obj);
+
+    memory_region_init_io(&s->storage_io, OBJECT(s), &itc_storage_ops, s,
+                          "mips-itc-storage", ITC_STORAGE_ADDRSPACE_SZ);
+    sysbus_init_mmio(sbd, &s->storage_io);
+
+    memory_region_init_io(&s->tag_io, OBJECT(s), &itc_tag_ops, s,
+                          "mips-itc-tag", ITC_TAG_ADDRSPACE_SZ);
+}
+
+static void mips_itu_realize(DeviceState *dev, Error **errp)
+{
+    MIPSITUState *s = MIPS_ITU(dev);
+
+    if (s->num_fifo > ITC_FIFO_NUM_MAX) {
+        error_setg(errp, "Exceed maximum number of FIFO cells: %d",
+                   s->num_fifo);
+        return;
+    }
+    if (s->num_semaphores > ITC_SEMAPH_NUM_MAX) {
+        error_setg(errp, "Exceed maximum number of Semaphore cells: %d",
+                   s->num_semaphores);
+        return;
+    }
+
+    s->cell = g_new(ITCStorageCell, get_num_cells(s));
+}
+
+static void mips_itu_reset(DeviceState *dev)
+{
+    MIPSITUState *s = MIPS_ITU(dev);
+
+    s->ITCAddressMap[0] = 0;
+    s->ITCAddressMap[1] =
+        ((ITC_STORAGE_ADDRSPACE_SZ - 1) & ITC_AM1_ADDR_MASK_MASK) |
+        (get_num_cells(s) << ITC_AM1_NUMENTRIES_OFS);
+    itc_reconfigure(s);
+
+    itc_reset_cells(s);
+}
+
+static Property mips_itu_properties[] = {
+    DEFINE_PROP_INT32("num-fifo", MIPSITUState, num_fifo,
+                      ITC_FIFO_NUM_MAX),
+    DEFINE_PROP_INT32("num-semaphores", MIPSITUState, num_semaphores,
+                      ITC_SEMAPH_NUM_MAX),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mips_itu_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->props = mips_itu_properties;
+    dc->realize = mips_itu_realize;
+    dc->reset = mips_itu_reset;
+}
+
+static const TypeInfo mips_itu_info = {
+    .name          = TYPE_MIPS_ITU,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(MIPSITUState),
+    .instance_init = mips_itu_init,
+    .class_init    = mips_itu_class_init,
+};
+
+static void mips_itu_register_types(void)
+{
+    type_register_static(&mips_itu_info);
+}
+
+type_init(mips_itu_register_types)
diff --git a/hw/misc/mst_fpga.c b/hw/misc/mst_fpga.c
index d509079..a10f049 100644
--- a/hw/misc/mst_fpga.c
+++ b/hw/misc/mst_fpga.c
@@ -10,6 +10,7 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
@@ -199,10 +200,11 @@
 	return 0;
 }
 
-static int mst_fpga_init(SysBusDevice *sbd)
+static void mst_fpga_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    mst_irq_state *s = MAINSTONE_FPGA(dev);
+    DeviceState *dev = DEVICE(obj);
+    mst_irq_state *s = MAINSTONE_FPGA(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
     s->pcmcia0 = MST_PCMCIAx_READY | MST_PCMCIAx_nCD;
     s->pcmcia1 = MST_PCMCIAx_READY | MST_PCMCIAx_nCD;
@@ -212,10 +214,9 @@
     /* alloc the external 16 irqs */
     qdev_init_gpio_in(dev, mst_fpga_set_irq, MST_NUM_IRQS);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &mst_fpga_ops, s,
+    memory_region_init_io(&s->iomem, obj, &mst_fpga_ops, s,
                           "fpga", 0x00100000);
     sysbus_init_mmio(sbd, &s->iomem);
-    return 0;
 }
 
 static VMStateDescription vmstate_mst_fpga_regs = {
@@ -244,9 +245,7 @@
 static void mst_fpga_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = mst_fpga_init;
     dc->desc = "Mainstone II FPGA";
     dc->vmsd = &vmstate_mst_fpga_regs;
 }
@@ -255,6 +254,7 @@
     .name          = TYPE_MAINSTONE_FPGA,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(mst_irq_state),
+    .instance_init = mst_fpga_init,
     .class_init    = mst_fpga_class_init,
 };
 
diff --git a/hw/misc/omap_clk.c b/hw/misc/omap_clk.c
index 80a3c50..19151d0 100644
--- a/hw/misc/omap_clk.c
+++ b/hw/misc/omap_clk.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -1239,7 +1240,7 @@
     for (i = onchip_clks, count = 0; *i; i ++)
         if ((*i)->flags & flag)
             count ++;
-    mpu->clks = (struct clk *) g_malloc0(sizeof(struct clk) * (count + 1));
+    mpu->clks = g_new0(struct clk, count + 1);
     for (i = onchip_clks, j = mpu->clks; *i; i ++)
         if ((*i)->flags & flag) {
             memcpy(j, *i, sizeof(struct clk));
diff --git a/hw/misc/omap_gpmc.c b/hw/misc/omap_gpmc.c
index a0de52f..67d8e2f 100644
--- a/hw/misc/omap_gpmc.c
+++ b/hw/misc/omap_gpmc.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/block/flash.h"
 #include "hw/arm/omap.h"
@@ -624,7 +625,8 @@
     struct omap_gpmc_cs_file_s *f;
 
     if (size != 4 && gpmc_wordaccess_only(addr)) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -825,8 +827,7 @@
                                    qemu_irq irq, qemu_irq drq)
 {
     int cs;
-    struct omap_gpmc_s *s = (struct omap_gpmc_s *)
-            g_malloc0(sizeof(struct omap_gpmc_s));
+    struct omap_gpmc_s *s = g_new0(struct omap_gpmc_s, 1);
 
     memory_region_init_io(&s->iomem, NULL, &omap_gpmc_ops, s, "omap-gpmc", 0x1000);
     memory_region_add_subregion(get_system_memory(), base, &s->iomem);
diff --git a/hw/misc/omap_l4.c b/hw/misc/omap_l4.c
index f237250..88c533a 100644
--- a/hw/misc/omap_l4.c
+++ b/hw/misc/omap_l4.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -82,7 +83,8 @@
     struct omap_target_agent_s *s = (struct omap_target_agent_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
diff --git a/hw/misc/omap_sdrc.c b/hw/misc/omap_sdrc.c
index ed62caf..dff37ec 100644
--- a/hw/misc/omap_sdrc.c
+++ b/hw/misc/omap_sdrc.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -92,7 +93,8 @@
     struct omap_sdrc_s *s = (struct omap_sdrc_s *) opaque;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -156,8 +158,7 @@
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
                                    hwaddr base)
 {
-    struct omap_sdrc_s *s = (struct omap_sdrc_s *)
-            g_malloc0(sizeof(struct omap_sdrc_s));
+    struct omap_sdrc_s *s = g_new0(struct omap_sdrc_s, 1);
 
     omap_sdrc_reset(s);
 
diff --git a/hw/misc/omap_tap.c b/hw/misc/omap_tap.c
index 9d2b710..e6ea8ee 100644
--- a/hw/misc/omap_tap.c
+++ b/hw/misc/omap_tap.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -95,7 +96,8 @@
                            uint64_t value, unsigned size)
 {
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     OMAP_BAD_REG(addr);
diff --git a/hw/misc/pc-testdev.c b/hw/misc/pc-testdev.c
index 18e94e0..b81d820 100644
--- a/hw/misc/pc-testdev.c
+++ b/hw/misc/pc-testdev.c
@@ -35,10 +35,7 @@
  * git://git.kernel.org/pub/scm/virt/kvm/kvm-unit-tests.git
 */
 
-#include "config-host.h"
-#if defined(CONFIG_POSIX)
-#include <sys/mman.h>
-#endif
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/qdev.h"
 #include "hw/isa/isa.h"
diff --git a/hw/misc/pci-testdev.c b/hw/misc/pci-testdev.c
index 3cd4a95..abfc2df 100644
--- a/hw/misc/pci-testdev.c
+++ b/hw/misc/pci-testdev.c
@@ -17,10 +17,11 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "qemu/event_notifier.h"
-#include "qemu/osdep.h"
+#include "sysemu/kvm.h"
 
 typedef struct PCITestDevHdr {
     uint8_t test;
@@ -233,12 +234,13 @@
     },
 };
 
-static int pci_testdev_init(PCIDevice *pci_dev)
+static void pci_testdev_realize(PCIDevice *pci_dev, Error **errp)
 {
     PCITestDevState *d = PCI_TEST_DEV(pci_dev);
     uint8_t *pci_conf;
     char *name;
     int r, i;
+    bool fastmmio = kvm_ioeventfd_any_length_enabled();
 
     pci_conf = pci_dev->config;
 
@@ -261,8 +263,12 @@
         memcpy(test->hdr->name, name, strlen(name) + 1);
         g_free(name);
         test->hdr->offset = cpu_to_le32(IOTEST_SIZE(i) + i * IOTEST_ACCESS_WIDTH);
-        test->size = IOTEST_ACCESS_WIDTH;
         test->match_data = strcmp(IOTEST_TEST(i), "wildcard-eventfd");
+        if (fastmmio && IOTEST_IS_MEM(i) && !test->match_data) {
+            test->size = 0;
+        } else {
+            test->size = IOTEST_ACCESS_WIDTH;
+        }
         test->hdr->test = i;
         test->hdr->data = test->match_data ? IOTEST_DATAMATCH : IOTEST_NOMATCH;
         test->hdr->width = IOTEST_ACCESS_WIDTH;
@@ -276,8 +282,6 @@
         (void)r;
         test->hasnotifier = true;
     }
-
-    return 0;
 }
 
 static void
@@ -307,7 +311,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_testdev_init;
+    k->realize = pci_testdev_realize;
     k->exit = pci_testdev_uninit;
     k->vendor_id = PCI_VENDOR_ID_REDHAT;
     k->device_id = PCI_DEVICE_ID_REDHAT_TEST;
diff --git a/hw/misc/puv3_pm.c b/hw/misc/puv3_pm.c
index 37f2369..577ceba 100644
--- a/hw/misc/puv3_pm.c
+++ b/hw/misc/puv3_pm.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 
diff --git a/hw/misc/pvpanic.c b/hw/misc/pvpanic.c
index 994f8af..0ac1e6a 100644
--- a/hw/misc/pvpanic.c
+++ b/hw/misc/pvpanic.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qjson.h"
 #include "sysemu/sysemu.h"
@@ -41,8 +42,7 @@
     }
 
     if (event & PVPANIC_PANICKED) {
-        qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
-        vm_stop(RUN_STATE_GUEST_PANICKED);
+        qemu_system_guest_panicked();
         return;
     }
 }
diff --git a/hw/misc/sga.c b/hw/misc/sga.c
index 83d2fd9..03b006d 100644
--- a/hw/misc/sga.c
+++ b/hw/misc/sga.c
@@ -24,6 +24,7 @@
  * sgabios code originally available at code.google.com/p/sgabios
  *
  */
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/pc.h"
 #include "hw/loader.h"
diff --git a/hw/misc/slavio_misc.c b/hw/misc/slavio_misc.c
index 5098595..edd5de0 100644
--- a/hw/misc/slavio_misc.c
+++ b/hw/misc/slavio_misc.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -68,6 +69,7 @@
 } APCState;
 
 #define MISC_SIZE 1
+#define LED_SIZE 2
 #define SYSCTRL_SIZE 4
 
 #define AUX1_TC        0x02
@@ -452,13 +454,13 @@
     /* 16 bit registers */
     /* ss600mp diag LEDs */
     memory_region_init_io(&s->led_iomem, OBJECT(s), &slavio_led_mem_ops, s,
-                          "leds", MISC_SIZE);
+                          "leds", LED_SIZE);
     sysbus_init_mmio(sbd, &s->led_iomem);
 
     /* 32 bit registers */
     /* System control */
     memory_region_init_io(&s->sysctrl_iomem, OBJECT(s), &slavio_sysctrl_mem_ops, s,
-                          "system-control", MISC_SIZE);
+                          "system-control", SYSCTRL_SIZE);
     sysbus_init_mmio(sbd, &s->sysctrl_iomem);
 
     /* AUX 1 (Misc System Functions) */
diff --git a/hw/misc/stm32f2xx_syscfg.c b/hw/misc/stm32f2xx_syscfg.c
new file mode 100644
index 0000000..7c45833
--- /dev/null
+++ b/hw/misc/stm32f2xx_syscfg.c
@@ -0,0 +1,162 @@
+/*
+ * STM32F2XX SYSCFG
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/misc/stm32f2xx_syscfg.h"
+#include "qemu/log.h"
+
+#ifndef STM_SYSCFG_ERR_DEBUG
+#define STM_SYSCFG_ERR_DEBUG 0
+#endif
+
+#define DB_PRINT_L(lvl, fmt, args...) do { \
+    if (STM_SYSCFG_ERR_DEBUG >= lvl) { \
+        qemu_log("%s: " fmt, __func__, ## args); \
+    } \
+} while (0);
+
+#define DB_PRINT(fmt, args...) DB_PRINT_L(1, fmt, ## args)
+
+static void stm32f2xx_syscfg_reset(DeviceState *dev)
+{
+    STM32F2XXSyscfgState *s = STM32F2XX_SYSCFG(dev);
+
+    s->syscfg_memrmp = 0x00000000;
+    s->syscfg_pmc = 0x00000000;
+    s->syscfg_exticr1 = 0x00000000;
+    s->syscfg_exticr2 = 0x00000000;
+    s->syscfg_exticr3 = 0x00000000;
+    s->syscfg_exticr4 = 0x00000000;
+    s->syscfg_cmpcr = 0x00000000;
+}
+
+static uint64_t stm32f2xx_syscfg_read(void *opaque, hwaddr addr,
+                                     unsigned int size)
+{
+    STM32F2XXSyscfgState *s = opaque;
+
+    DB_PRINT("0x%"HWADDR_PRIx"\n", addr);
+
+    switch (addr) {
+    case SYSCFG_MEMRMP:
+        return s->syscfg_memrmp;
+    case SYSCFG_PMC:
+        return s->syscfg_pmc;
+    case SYSCFG_EXTICR1:
+        return s->syscfg_exticr1;
+    case SYSCFG_EXTICR2:
+        return s->syscfg_exticr2;
+    case SYSCFG_EXTICR3:
+        return s->syscfg_exticr3;
+    case SYSCFG_EXTICR4:
+        return s->syscfg_exticr4;
+    case SYSCFG_CMPCR:
+        return s->syscfg_cmpcr;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%"HWADDR_PRIx"\n", __func__, addr);
+        return 0;
+    }
+
+    return 0;
+}
+
+static void stm32f2xx_syscfg_write(void *opaque, hwaddr addr,
+                       uint64_t val64, unsigned int size)
+{
+    STM32F2XXSyscfgState *s = opaque;
+    uint32_t value = val64;
+
+    DB_PRINT("0x%x, 0x%"HWADDR_PRIx"\n", value, addr);
+
+    switch (addr) {
+    case SYSCFG_MEMRMP:
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Changeing the memory mapping isn't supported " \
+                      "in QEMU\n", __func__);
+        return;
+    case SYSCFG_PMC:
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: Changeing the memory mapping isn't supported " \
+                      "in QEMU\n", __func__);
+        return;
+    case SYSCFG_EXTICR1:
+        s->syscfg_exticr1 = (value & 0xFFFF);
+        return;
+    case SYSCFG_EXTICR2:
+        s->syscfg_exticr2 = (value & 0xFFFF);
+        return;
+    case SYSCFG_EXTICR3:
+        s->syscfg_exticr3 = (value & 0xFFFF);
+        return;
+    case SYSCFG_EXTICR4:
+        s->syscfg_exticr4 = (value & 0xFFFF);
+        return;
+    case SYSCFG_CMPCR:
+        s->syscfg_cmpcr = value;
+        return;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%"HWADDR_PRIx"\n", __func__, addr);
+    }
+}
+
+static const MemoryRegionOps stm32f2xx_syscfg_ops = {
+    .read = stm32f2xx_syscfg_read,
+    .write = stm32f2xx_syscfg_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void stm32f2xx_syscfg_init(Object *obj)
+{
+    STM32F2XXSyscfgState *s = STM32F2XX_SYSCFG(obj);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(obj), &s->irq);
+
+    memory_region_init_io(&s->mmio, obj, &stm32f2xx_syscfg_ops, s,
+                          TYPE_STM32F2XX_SYSCFG, 0x400);
+    sysbus_init_mmio(SYS_BUS_DEVICE(obj), &s->mmio);
+}
+
+static void stm32f2xx_syscfg_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = stm32f2xx_syscfg_reset;
+}
+
+static const TypeInfo stm32f2xx_syscfg_info = {
+    .name          = TYPE_STM32F2XX_SYSCFG,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(STM32F2XXSyscfgState),
+    .instance_init = stm32f2xx_syscfg_init,
+    .class_init    = stm32f2xx_syscfg_class_init,
+};
+
+static void stm32f2xx_syscfg_register_types(void)
+{
+    type_register_static(&stm32f2xx_syscfg_info);
+}
+
+type_init(stm32f2xx_syscfg_register_types)
diff --git a/hw/misc/tmp105.c b/hw/misc/tmp105.c
index f3fe8b8..f5c2472 100644
--- a/hw/misc/tmp105.c
+++ b/hw/misc/tmp105.c
@@ -18,9 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i2c/i2c.h"
 #include "tmp105.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 static void tmp105_interrupt_update(TMP105State *s)
@@ -52,26 +54,26 @@
     tmp105_interrupt_update(s);
 }
 
-static void tmp105_get_temperature(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void tmp105_get_temperature(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     TMP105State *s = TMP105(obj);
     int64_t value = s->temperature * 1000 / 256;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 /* Units are 0.001 centigrades relative to 0 C.  s->temperature is 8.8
  * fixed point, so units are 1/256 centigrades.  A simple ratio will do.
  */
-static void tmp105_set_temperature(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void tmp105_set_temperature(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     TMP105State *s = TMP105(obj);
     Error *local_err = NULL;
     int64_t temp;
 
-    visit_type_int(v, &temp, name, &local_err);
+    visit_type_int(v, name, &temp, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
diff --git a/hw/misc/trace-events b/hw/misc/trace-events
new file mode 100644
index 0000000..0cc556c
--- /dev/null
+++ b/hw/misc/trace-events
@@ -0,0 +1,55 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/misc/eccmemctl.c
+ecc_mem_writel_mer(uint32_t val) "Write memory enable %08x"
+ecc_mem_writel_mdr(uint32_t val) "Write memory delay %08x"
+ecc_mem_writel_mfsr(uint32_t val) "Write memory fault status %08x"
+ecc_mem_writel_vcr(uint32_t val) "Write slot configuration %08x"
+ecc_mem_writel_dr(uint32_t val) "Write diagnostic %08x"
+ecc_mem_writel_ecr0(uint32_t val) "Write event count 1 %08x"
+ecc_mem_writel_ecr1(uint32_t val) "Write event count 2 %08x"
+ecc_mem_readl_mer(uint32_t ret) "Read memory enable %08x"
+ecc_mem_readl_mdr(uint32_t ret) "Read memory delay %08x"
+ecc_mem_readl_mfsr(uint32_t ret) "Read memory fault status %08x"
+ecc_mem_readl_vcr(uint32_t ret) "Read slot configuration %08x"
+ecc_mem_readl_mfar0(uint32_t ret) "Read memory fault address 0 %08x"
+ecc_mem_readl_mfar1(uint32_t ret) "Read memory fault address 1 %08x"
+ecc_mem_readl_dr(uint32_t ret) "Read diagnostic %08x"
+ecc_mem_readl_ecr0(uint32_t ret) "Read event count 1 %08x"
+ecc_mem_readl_ecr1(uint32_t ret) "Read event count 2 %08x"
+ecc_diag_mem_writeb(uint64_t addr, uint32_t val) "Write diagnostic %"PRId64" = %02x"
+ecc_diag_mem_readb(uint64_t addr, uint32_t ret) "Read diagnostic %"PRId64"= %02x"
+
+# hw/misc/slavio_misc.c
+slavio_misc_update_irq_raise(void) "Raise IRQ"
+slavio_misc_update_irq_lower(void) "Lower IRQ"
+slavio_set_power_fail(int power_failing, uint8_t config) "Power fail: %d, config: %d"
+slavio_cfg_mem_writeb(uint32_t val) "Write config %02x"
+slavio_cfg_mem_readb(uint32_t ret) "Read config %02x"
+slavio_diag_mem_writeb(uint32_t val) "Write diag %02x"
+slavio_diag_mem_readb(uint32_t ret) "Read diag %02x"
+slavio_mdm_mem_writeb(uint32_t val) "Write modem control %02x"
+slavio_mdm_mem_readb(uint32_t ret) "Read modem control %02x"
+slavio_aux1_mem_writeb(uint32_t val) "Write aux1 %02x"
+slavio_aux1_mem_readb(uint32_t ret) "Read aux1 %02x"
+slavio_aux2_mem_writeb(uint32_t val) "Write aux2 %02x"
+slavio_aux2_mem_readb(uint32_t ret) "Read aux2 %02x"
+apc_mem_writeb(uint32_t val) "Write power management %02x"
+apc_mem_readb(uint32_t ret) "Read power management %02x"
+slavio_sysctrl_mem_writel(uint32_t val) "Write system control %08x"
+slavio_sysctrl_mem_readl(uint32_t ret) "Read system control %08x"
+slavio_led_mem_writew(uint32_t val) "Write diagnostic LED %04x"
+slavio_led_mem_readw(uint32_t ret) "Read diagnostic LED %04x"
+
+# hw/misc/milkymist-hpdmc.c
+milkymist_hpdmc_memory_read(uint32_t addr, uint32_t value) "addr=%08x value=%08x"
+milkymist_hpdmc_memory_write(uint32_t addr, uint32_t value) "addr=%08x value=%08x"
+
+# hw/misc/milkymist-pfpu.c
+milkymist_pfpu_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_pfpu_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_pfpu_vectout(uint32_t a, uint32_t b, uint32_t dma_ptr) "a %08x b %08x dma_ptr %08x"
+milkymist_pfpu_pulse_irq(void) "Pulse IRQ"
+
+# hw/misc/aspeed_scu.c
+aspeed_scu_write(uint64_t offset, unsigned size, uint32_t data) "To 0x%" PRIx64 " of size %u: 0x%" PRIx32
diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
deleted file mode 100644
index 7002db1..0000000
--- a/hw/misc/vfio.c
+++ /dev/null
@@ -1,4479 +0,0 @@
-/*
- * vfio based device assignment support
- *
- * Copyright Red Hat, Inc. 2012
- *
- * Authors:
- *  Alex Williamson <alex.williamson@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Based on qemu-kvm device-assignment:
- *  Adapted for KVM by Qumranet.
- *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
- *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
- *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
- *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
- *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
- */
-
-#include <dirent.h>
-#include <linux/vfio.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "config.h"
-#include "exec/address-spaces.h"
-#include "exec/memory.h"
-#include "hw/pci/msi.h"
-#include "hw/pci/msix.h"
-#include "hw/pci/pci.h"
-#include "qemu-common.h"
-#include "qemu/error-report.h"
-#include "qemu/event_notifier.h"
-#include "qemu/queue.h"
-#include "qemu/range.h"
-#include "sysemu/kvm.h"
-#include "sysemu/sysemu.h"
-#include "hw/misc/vfio.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/utils/file_io.h"
-#endif
-
-/* #define DEBUG_VFIO */
-#ifdef DEBUG_VFIO
-#define DPRINTF(fmt, ...) \
-    do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-/* Extra debugging, trap acceleration paths for more logging */
-#define VFIO_ALLOW_MMAP 1
-#define VFIO_ALLOW_KVM_INTX 1
-#define VFIO_ALLOW_KVM_MSI 1
-#define VFIO_ALLOW_KVM_MSIX 1
-
-struct VFIODevice;
-
-typedef struct VFIOQuirk {
-    MemoryRegion mem;
-    struct VFIODevice *vdev;
-    QLIST_ENTRY(VFIOQuirk) next;
-    struct {
-        uint32_t base_offset:TARGET_PAGE_BITS;
-        uint32_t address_offset:TARGET_PAGE_BITS;
-        uint32_t address_size:3;
-        uint32_t bar:3;
-
-        uint32_t address_match;
-        uint32_t address_mask;
-
-        uint32_t address_val:TARGET_PAGE_BITS;
-        uint32_t data_offset:TARGET_PAGE_BITS;
-        uint32_t data_size:3;
-
-        uint8_t flags;
-        uint8_t read_flags;
-        uint8_t write_flags;
-    } data;
-} VFIOQuirk;
-
-typedef struct VFIOBAR {
-    off_t fd_offset; /* offset of BAR within device fd */
-    int fd; /* device fd, allows us to pass VFIOBAR as opaque data */
-    MemoryRegion mem; /* slow, read/write access */
-    MemoryRegion mmap_mem; /* direct mapped access */
-    void *mmap;
-    size_t size;
-    uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
-    uint8_t nr; /* cache the BAR number for debug */
-    bool ioport;
-    bool mem64;
-    QLIST_HEAD(, VFIOQuirk) quirks;
-} VFIOBAR;
-
-typedef struct VFIOVGARegion {
-    MemoryRegion mem;
-    off_t offset;
-    int nr;
-    QLIST_HEAD(, VFIOQuirk) quirks;
-} VFIOVGARegion;
-
-typedef struct VFIOVGA {
-    off_t fd_offset;
-    int fd;
-    VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
-} VFIOVGA;
-
-typedef struct VFIOINTx {
-    bool pending; /* interrupt pending */
-    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
-    uint8_t pin; /* which pin to pull for qemu_set_irq */
-    EventNotifier interrupt; /* eventfd triggered on interrupt */
-    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
-    PCIINTxRoute route; /* routing info for QEMU bypass */
-    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
-    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
-} VFIOINTx;
-
-typedef struct VFIOMSIVector {
-    /*
-     * Two interrupt paths are configured per vector.  The first, is only used
-     * for interrupts injected via QEMU.  This is typically the non-accel path,
-     * but may also be used when we want QEMU to handle masking and pending
-     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
-     * but requires masking at the device.  virq is used to track the MSI route
-     * through KVM, thus kvm_interrupt is only available when virq is set to a
-     * valid (>= 0) value.
-     */
-    EventNotifier interrupt;
-    EventNotifier kvm_interrupt;
-    struct VFIODevice *vdev; /* back pointer to device */
-    int virq;
-    bool use;
-} VFIOMSIVector;
-
-enum {
-    VFIO_INT_NONE = 0,
-    VFIO_INT_INTx = 1,
-    VFIO_INT_MSI  = 2,
-    VFIO_INT_MSIX = 3,
-};
-
-typedef struct VFIOAddressSpace {
-    AddressSpace *as;
-    QLIST_HEAD(, VFIOContainer) containers;
-    QLIST_ENTRY(VFIOAddressSpace) list;
-} VFIOAddressSpace;
-
-static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces =
-    QLIST_HEAD_INITIALIZER(vfio_address_spaces);
-
-struct VFIOGroup;
-
-typedef struct VFIOType1 {
-    MemoryListener listener;
-    int error;
-    bool initialized;
-} VFIOType1;
-
-typedef struct VFIOContainer {
-    VFIOAddressSpace *space;
-    int fd; /* /dev/vfio/vfio, empowered by the attached groups */
-    struct {
-        /* enable abstraction to support various iommu backends */
-        union {
-            VFIOType1 type1;
-        };
-        void (*release)(struct VFIOContainer *);
-    } iommu_data;
-    QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
-    QLIST_HEAD(, VFIOGroup) group_list;
-    QLIST_ENTRY(VFIOContainer) next;
-} VFIOContainer;
-
-typedef struct VFIOGuestIOMMU {
-    VFIOContainer *container;
-    MemoryRegion *iommu;
-    Notifier n;
-    QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
-} VFIOGuestIOMMU;
-
-/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
-typedef struct VFIOMSIXInfo {
-    uint8_t table_bar;
-    uint8_t pba_bar;
-    uint16_t entries;
-    uint32_t table_offset;
-    uint32_t pba_offset;
-    MemoryRegion mmap_mem;
-    void *mmap;
-} VFIOMSIXInfo;
-
-typedef struct VFIODevice {
-    PCIDevice pdev;
-    int fd;
-    VFIOINTx intx;
-    unsigned int config_size;
-    uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
-    off_t config_offset; /* Offset of config space region within device fd */
-    unsigned int rom_size;
-    off_t rom_offset; /* Offset of ROM region within device fd */
-    void *rom;
-    int msi_cap_size;
-    VFIOMSIVector *msi_vectors;
-    VFIOMSIXInfo *msix;
-    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
-    int interrupt; /* Current interrupt type */
-    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
-    VFIOVGA vga; /* 0xa0000, 0x3b0, 0x3c0 */
-    PCIHostDeviceAddress host;
-    QLIST_ENTRY(VFIODevice) next;
-    struct VFIOGroup *group;
-    EventNotifier err_notifier;
-    uint32_t features;
-#define VFIO_FEATURE_ENABLE_VGA_BIT 0
-#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
-    int32_t bootindex;
-    uint8_t pm_cap;
-    bool reset_works;
-    bool has_vga;
-    bool pci_aer;
-    bool has_flr;
-    bool has_pm_reset;
-    bool needs_reset;
-    bool rom_read_failed;
-} VFIODevice;
-
-typedef struct VFIOGroup {
-    int fd;
-    int groupid;
-    VFIOContainer *container;
-    QLIST_HEAD(, VFIODevice) device_list;
-    QLIST_ENTRY(VFIOGroup) next;
-    QLIST_ENTRY(VFIOGroup) container_next;
-} VFIOGroup;
-
-typedef struct VFIORomBlacklistEntry {
-    uint16_t vendor_id;
-    uint16_t device_id;
-} VFIORomBlacklistEntry;
-
-/*
- * List of device ids/vendor ids for which to disable
- * option rom loading. This avoids the guest hangs during rom
- * execution as noticed with the BCM 57810 card for lack of a
- * more better way to handle such issues.
- * The  user can still override by specifying a romfile or
- * rombar=1.
- * Please see https://bugs.launchpad.net/qemu/+bug/1284874
- * for an analysis of the 57810 card hang. When adding
- * a new vendor id/device id combination below, please also add
- * your card/environment details and information that could
- * help in debugging to the bug tracking this issue
- */
-static const VFIORomBlacklistEntry romblacklist[] = {
-    /* Broadcom BCM 57810 */
-    { 0x14e4, 0x168e }
-};
-
-#define MSIX_CAP_LENGTH 12
-
-static QLIST_HEAD(, VFIOGroup)
-    group_list = QLIST_HEAD_INITIALIZER(group_list);
-
-#ifdef CONFIG_KVM
-/*
- * We have a single VFIO pseudo device per KVM VM.  Once created it lives
- * for the life of the VM.  Closing the file descriptor only drops our
- * reference to it and the device's reference to kvm.  Therefore once
- * initialized, this file descriptor is only released on QEMU exit and
- * we'll re-use it should another vfio device be attached before then.
- */
-static int vfio_kvm_device_fd = -1;
-#endif
-
-static void vfio_disable_interrupts(VFIODevice *vdev);
-static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
-static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
-                                  uint32_t val, int len);
-static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
-
-/*
- * Common VFIO interrupt disable
- */
-static void vfio_disable_irqindex(VFIODevice *vdev, int index)
-{
-    struct vfio_irq_set irq_set = {
-        .argsz = sizeof(irq_set),
-        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
-        .index = index,
-        .start = 0,
-        .count = 0,
-    };
-
-    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
-}
-
-/*
- * INTx
- */
-static void vfio_unmask_intx(VFIODevice *vdev)
-{
-    struct vfio_irq_set irq_set = {
-        .argsz = sizeof(irq_set),
-        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
-        .index = VFIO_PCI_INTX_IRQ_INDEX,
-        .start = 0,
-        .count = 1,
-    };
-
-    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
-}
-
-#ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
-static void vfio_mask_intx(VFIODevice *vdev)
-{
-    struct vfio_irq_set irq_set = {
-        .argsz = sizeof(irq_set),
-        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
-        .index = VFIO_PCI_INTX_IRQ_INDEX,
-        .start = 0,
-        .count = 1,
-    };
-
-    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
-}
-#endif
-
-/*
- * Disabling BAR mmaping can be slow, but toggling it around INTx can
- * also be a huge overhead.  We try to get the best of both worlds by
- * waiting until an interrupt to disable mmaps (subsequent transitions
- * to the same state are effectively no overhead).  If the interrupt has
- * been serviced and the time gap is long enough, we re-enable mmaps for
- * performance.  This works well for things like graphics cards, which
- * may not use their interrupt at all and are penalized to an unusable
- * level by read/write BAR traps.  Other devices, like NICs, have more
- * regular interrupts and see much better latency by staying in non-mmap
- * mode.  We therefore set the default mmap_timeout such that a ping
- * is just enough to keep the mmap disabled.  Users can experiment with
- * other options with the x-intx-mmap-timeout-ms parameter (a value of
- * zero disables the timer).
- */
-static void vfio_intx_mmap_enable(void *opaque)
-{
-    VFIODevice *vdev = opaque;
-
-    if (vdev->intx.pending) {
-        timer_mod(vdev->intx.mmap_timer,
-                       qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
-        return;
-    }
-
-    vfio_mmap_set_enabled(vdev, true);
-}
-
-static void vfio_intx_interrupt(void *opaque)
-{
-    VFIODevice *vdev = opaque;
-
-    if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
-        return;
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function,
-            'A' + vdev->intx.pin);
-
-    vdev->intx.pending = true;
-    pci_irq_assert(&vdev->pdev);
-    vfio_mmap_set_enabled(vdev, false);
-    if (vdev->intx.mmap_timeout) {
-        timer_mod(vdev->intx.mmap_timer,
-                       qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
-    }
-}
-
-static void vfio_eoi(VFIODevice *vdev)
-{
-    if (!vdev->intx.pending) {
-        return;
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-    vdev->intx.pending = false;
-    pci_irq_deassert(&vdev->pdev);
-    vfio_unmask_intx(vdev);
-}
-
-static void vfio_enable_intx_kvm(VFIODevice *vdev)
-{
-#ifdef CONFIG_KVM
-    struct kvm_irqfd irqfd = {
-        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
-        .gsi = vdev->intx.route.irq,
-        .flags = KVM_IRQFD_FLAG_RESAMPLE,
-    };
-    struct vfio_irq_set *irq_set;
-    int ret, argsz;
-    int32_t *pfd;
-
-    if (!VFIO_ALLOW_KVM_INTX || !kvm_irqfds_enabled() ||
-        vdev->intx.route.mode != PCI_INTX_ENABLED ||
-        !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
-        return;
-    }
-
-    /* Get to a known interrupt state */
-    qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
-    vfio_mask_intx(vdev);
-    vdev->intx.pending = false;
-    pci_irq_deassert(&vdev->pdev);
-
-    /* Get an eventfd for resample/unmask */
-    if (event_notifier_init(&vdev->intx.unmask, 0)) {
-        error_report("vfio: Error: event_notifier_init failed eoi");
-        goto fail;
-    }
-
-    /* KVM triggers it, VFIO listens for it */
-    irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
-
-    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
-        error_report("vfio: Error: Failed to setup resample irqfd: %m");
-        goto fail_irqfd;
-    }
-
-    argsz = sizeof(*irq_set) + sizeof(*pfd);
-
-    irq_set = g_malloc0(argsz);
-    irq_set->argsz = argsz;
-    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
-    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
-    irq_set->start = 0;
-    irq_set->count = 1;
-    pfd = (int32_t *)&irq_set->data;
-
-    *pfd = irqfd.resamplefd;
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-    g_free(irq_set);
-    if (ret) {
-        error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
-        goto fail_vfio;
-    }
-
-    /* Let'em rip */
-    vfio_unmask_intx(vdev);
-
-    vdev->intx.kvm_accel = true;
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
-            __func__, vdev->host.domain, vdev->host.bus,
-            vdev->host.slot, vdev->host.function);
-
-    return;
-
-fail_vfio:
-    irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
-    kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
-fail_irqfd:
-    event_notifier_cleanup(&vdev->intx.unmask);
-fail:
-    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
-    vfio_unmask_intx(vdev);
-#endif
-}
-
-static void vfio_disable_intx_kvm(VFIODevice *vdev)
-{
-#ifdef CONFIG_KVM
-    struct kvm_irqfd irqfd = {
-        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
-        .gsi = vdev->intx.route.irq,
-        .flags = KVM_IRQFD_FLAG_DEASSIGN,
-    };
-
-    if (!vdev->intx.kvm_accel) {
-        return;
-    }
-
-    /*
-     * Get to a known state, hardware masked, QEMU ready to accept new
-     * interrupts, QEMU IRQ de-asserted.
-     */
-    vfio_mask_intx(vdev);
-    vdev->intx.pending = false;
-    pci_irq_deassert(&vdev->pdev);
-
-    /* Tell KVM to stop listening for an INTx irqfd */
-    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
-        error_report("vfio: Error: Failed to disable INTx irqfd: %m");
-    }
-
-    /* We only need to close the eventfd for VFIO to cleanup the kernel side */
-    event_notifier_cleanup(&vdev->intx.unmask);
-
-    /* QEMU starts listening for interrupt events. */
-    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
-
-    vdev->intx.kvm_accel = false;
-
-    /* If we've missed an event, let it re-fire through QEMU */
-    vfio_unmask_intx(vdev);
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
-            __func__, vdev->host.domain, vdev->host.bus,
-            vdev->host.slot, vdev->host.function);
-#endif
-}
-
-static void vfio_update_irq(PCIDevice *pdev)
-{
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    PCIINTxRoute route;
-
-    if (vdev->interrupt != VFIO_INT_INTx) {
-        return;
-    }
-
-    route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
-
-    if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
-        return; /* Nothing changed */
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, vdev->intx.route.irq, route.irq);
-
-    vfio_disable_intx_kvm(vdev);
-
-    vdev->intx.route = route;
-
-    if (route.mode != PCI_INTX_ENABLED) {
-        return;
-    }
-
-    vfio_enable_intx_kvm(vdev);
-
-    /* Re-enable the interrupt in cased we missed an EOI */
-    vfio_eoi(vdev);
-}
-
-static int vfio_enable_intx(VFIODevice *vdev)
-{
-    uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
-    int ret, argsz;
-    struct vfio_irq_set *irq_set;
-    int32_t *pfd;
-
-    if (!pin) {
-        return 0;
-    }
-
-    vfio_disable_interrupts(vdev);
-
-    vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
-    pci_config_set_interrupt_pin(vdev->pdev.config, pin);
-
-#ifdef CONFIG_KVM
-    /*
-     * Only conditional to avoid generating error messages on platforms
-     * where we won't actually use the result anyway.
-     */
-    if (kvm_irqfds_enabled() &&
-        kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
-        vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
-                                                        vdev->intx.pin);
-    }
-#endif
-
-    ret = event_notifier_init(&vdev->intx.interrupt, 0);
-    if (ret) {
-        error_report("vfio: Error: event_notifier_init failed");
-        return ret;
-    }
-
-    argsz = sizeof(*irq_set) + sizeof(*pfd);
-
-    irq_set = g_malloc0(argsz);
-    irq_set->argsz = argsz;
-    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
-    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
-    irq_set->start = 0;
-    irq_set->count = 1;
-    pfd = (int32_t *)&irq_set->data;
-
-    *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
-    qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-    g_free(irq_set);
-    if (ret) {
-        error_report("vfio: Error: Failed to setup INTx fd: %m");
-        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
-        event_notifier_cleanup(&vdev->intx.interrupt);
-        return -errno;
-    }
-
-    vfio_enable_intx_kvm(vdev);
-
-    vdev->interrupt = VFIO_INT_INTx;
-
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-    return 0;
-}
-
-static void vfio_disable_intx(VFIODevice *vdev)
-{
-    int fd;
-
-    timer_del(vdev->intx.mmap_timer);
-    vfio_disable_intx_kvm(vdev);
-    vfio_disable_irqindex(vdev, VFIO_PCI_INTX_IRQ_INDEX);
-    vdev->intx.pending = false;
-    pci_irq_deassert(&vdev->pdev);
-    vfio_mmap_set_enabled(vdev, true);
-
-    fd = event_notifier_get_fd(&vdev->intx.interrupt);
-    qemu_set_fd_handler(fd, NULL, NULL, vdev);
-    event_notifier_cleanup(&vdev->intx.interrupt);
-
-    vdev->interrupt = VFIO_INT_NONE;
-
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-}
-
-/*
- * MSI/X
- */
-static void vfio_msi_interrupt(void *opaque)
-{
-    VFIOMSIVector *vector = opaque;
-    VFIODevice *vdev = vector->vdev;
-    int nr = vector - vdev->msi_vectors;
-
-    if (!event_notifier_test_and_clear(&vector->interrupt)) {
-        return;
-    }
-
-#ifdef DEBUG_VFIO
-    MSIMessage msg;
-
-    if (vdev->interrupt == VFIO_INT_MSIX) {
-        msg = msix_get_message(&vdev->pdev, nr);
-    } else if (vdev->interrupt == VFIO_INT_MSI) {
-        msg = msi_get_message(&vdev->pdev, nr);
-    } else {
-        abort();
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d 0x%"PRIx64"/0x%x\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, nr, msg.address, msg.data);
-#endif
-
-    if (vdev->interrupt == VFIO_INT_MSIX) {
-        msix_notify(&vdev->pdev, nr);
-    } else if (vdev->interrupt == VFIO_INT_MSI) {
-        msi_notify(&vdev->pdev, nr);
-    } else {
-        error_report("vfio: MSI interrupt receieved, but not enabled?");
-    }
-}
-
-static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
-{
-    struct vfio_irq_set *irq_set;
-    int ret = 0, i, argsz;
-    int32_t *fds;
-
-    argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
-
-    irq_set = g_malloc0(argsz);
-    irq_set->argsz = argsz;
-    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
-    irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
-    irq_set->start = 0;
-    irq_set->count = vdev->nr_vectors;
-    fds = (int32_t *)&irq_set->data;
-
-    for (i = 0; i < vdev->nr_vectors; i++) {
-        int fd = -1;
-
-        /*
-         * MSI vs MSI-X - The guest has direct access to MSI mask and pending
-         * bits, therefore we always use the KVM signaling path when setup.
-         * MSI-X mask and pending bits are emulated, so we want to use the
-         * KVM signaling path only when configured and unmasked.
-         */
-        if (vdev->msi_vectors[i].use) {
-            if (vdev->msi_vectors[i].virq < 0 ||
-                (msix && msix_is_masked(&vdev->pdev, i))) {
-                fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
-            } else {
-                fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
-            }
-        }
-
-        fds[i] = fd;
-    }
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-
-    g_free(irq_set);
-
-    return ret;
-}
-
-static void vfio_add_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage *msg,
-                                  bool msix)
-{
-    int virq;
-
-    if ((msix && !VFIO_ALLOW_KVM_MSIX) ||
-        (!msix && !VFIO_ALLOW_KVM_MSI) || !msg) {
-        return;
-    }
-
-    if (event_notifier_init(&vector->kvm_interrupt, 0)) {
-        return;
-    }
-
-    virq = kvm_irqchip_add_msi_route(kvm_state, *msg);
-    if (virq < 0) {
-        event_notifier_cleanup(&vector->kvm_interrupt);
-        return;
-    }
-
-    if (kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->kvm_interrupt,
-                                       NULL, virq) < 0) {
-        kvm_irqchip_release_virq(kvm_state, virq);
-        event_notifier_cleanup(&vector->kvm_interrupt);
-        return;
-    }
-
-    vector->virq = virq;
-}
-
-static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
-{
-    kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->kvm_interrupt,
-                                      vector->virq);
-    kvm_irqchip_release_virq(kvm_state, vector->virq);
-    vector->virq = -1;
-    event_notifier_cleanup(&vector->kvm_interrupt);
-}
-
-static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg)
-{
-    kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg);
-}
-
-static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
-                                   MSIMessage *msg, IOHandler *handler)
-{
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    VFIOMSIVector *vector;
-    int ret;
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, nr);
-
-    vector = &vdev->msi_vectors[nr];
-
-    if (!vector->use) {
-        vector->vdev = vdev;
-        vector->virq = -1;
-        if (event_notifier_init(&vector->interrupt, 0)) {
-            error_report("vfio: Error: event_notifier_init failed");
-        }
-        vector->use = true;
-        msix_vector_use(pdev, nr);
-    }
-
-    qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
-                        handler, NULL, vector);
-
-    /*
-     * Attempt to enable route through KVM irqchip,
-     * default to userspace handling if unavailable.
-     */
-    if (vector->virq >= 0) {
-        if (!msg) {
-            vfio_remove_kvm_msi_virq(vector);
-        } else {
-            vfio_update_kvm_msi_virq(vector, *msg);
-        }
-    } else {
-        vfio_add_kvm_msi_virq(vector, msg, true);
-    }
-
-    /*
-     * We don't want to have the host allocate all possible MSI vectors
-     * for a device if they're not in use, so we shutdown and incrementally
-     * increase them as needed.
-     */
-    if (vdev->nr_vectors < nr + 1) {
-        vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
-        vdev->nr_vectors = nr + 1;
-        ret = vfio_enable_vectors(vdev, true);
-        if (ret) {
-            error_report("vfio: failed to enable vectors, %d", ret);
-        }
-    } else {
-        int argsz;
-        struct vfio_irq_set *irq_set;
-        int32_t *pfd;
-
-        argsz = sizeof(*irq_set) + sizeof(*pfd);
-
-        irq_set = g_malloc0(argsz);
-        irq_set->argsz = argsz;
-        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
-                         VFIO_IRQ_SET_ACTION_TRIGGER;
-        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
-        irq_set->start = nr;
-        irq_set->count = 1;
-        pfd = (int32_t *)&irq_set->data;
-
-        if (vector->virq >= 0) {
-            *pfd = event_notifier_get_fd(&vector->kvm_interrupt);
-        } else {
-            *pfd = event_notifier_get_fd(&vector->interrupt);
-        }
-
-        ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-        g_free(irq_set);
-        if (ret) {
-            error_report("vfio: failed to modify vector, %d", ret);
-        }
-    }
-
-    return 0;
-}
-
-static int vfio_msix_vector_use(PCIDevice *pdev,
-                                unsigned int nr, MSIMessage msg)
-{
-    return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
-}
-
-static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
-{
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    VFIOMSIVector *vector = &vdev->msi_vectors[nr];
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, nr);
-
-    /*
-     * There are still old guests that mask and unmask vectors on every
-     * interrupt.  If we're using QEMU bypass with a KVM irqfd, leave all of
-     * the KVM setup in place, simply switch VFIO to use the non-bypass
-     * eventfd.  We'll then fire the interrupt through QEMU and the MSI-X
-     * core will mask the interrupt and set pending bits, allowing it to
-     * be re-asserted on unmask.  Nothing to do if already using QEMU mode.
-     */
-    if (vector->virq >= 0) {
-        int argsz;
-        struct vfio_irq_set *irq_set;
-        int32_t *pfd;
-
-        argsz = sizeof(*irq_set) + sizeof(*pfd);
-
-        irq_set = g_malloc0(argsz);
-        irq_set->argsz = argsz;
-        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
-                         VFIO_IRQ_SET_ACTION_TRIGGER;
-        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
-        irq_set->start = nr;
-        irq_set->count = 1;
-        pfd = (int32_t *)&irq_set->data;
-
-        *pfd = event_notifier_get_fd(&vector->interrupt);
-
-        ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-
-        g_free(irq_set);
-    }
-}
-
-static void vfio_enable_msix(VFIODevice *vdev)
-{
-    vfio_disable_interrupts(vdev);
-
-    vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
-
-    vdev->interrupt = VFIO_INT_MSIX;
-
-    /*
-     * Some communication channels between VF & PF or PF & fw rely on the
-     * physical state of the device and expect that enabling MSI-X from the
-     * guest enables the same on the host.  When our guest is Linux, the
-     * guest driver call to pci_enable_msix() sets the enabling bit in the
-     * MSI-X capability, but leaves the vector table masked.  We therefore
-     * can't rely on a vector_use callback (from request_irq() in the guest)
-     * to switch the physical device into MSI-X mode because that may come a
-     * long time after pci_enable_msix().  This code enables vector 0 with
-     * triggering to userspace, then immediately release the vector, leaving
-     * the physical device with no vectors enabled, but MSI-X enabled, just
-     * like the guest view.
-     */
-    vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
-    vfio_msix_vector_release(&vdev->pdev, 0);
-
-    if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
-                                  vfio_msix_vector_release, NULL)) {
-        error_report("vfio: msix_set_vector_notifiers failed");
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-}
-
-static void vfio_enable_msi(VFIODevice *vdev)
-{
-    int ret, i;
-
-    vfio_disable_interrupts(vdev);
-
-    vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
-retry:
-    vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
-
-    for (i = 0; i < vdev->nr_vectors; i++) {
-        VFIOMSIVector *vector = &vdev->msi_vectors[i];
-        MSIMessage msg = msi_get_message(&vdev->pdev, i);
-
-        vector->vdev = vdev;
-        vector->virq = -1;
-        vector->use = true;
-
-        if (event_notifier_init(&vector->interrupt, 0)) {
-            error_report("vfio: Error: event_notifier_init failed");
-        }
-
-        qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
-                            vfio_msi_interrupt, NULL, vector);
-
-        /*
-         * Attempt to enable route through KVM irqchip,
-         * default to userspace handling if unavailable.
-         */
-        vfio_add_kvm_msi_virq(vector, &msg, false);
-    }
-
-    /* Set interrupt type prior to possible interrupts */
-    vdev->interrupt = VFIO_INT_MSI;
-
-    ret = vfio_enable_vectors(vdev, false);
-    if (ret) {
-        if (ret < 0) {
-            error_report("vfio: Error: Failed to setup MSI fds: %m");
-        } else if (ret != vdev->nr_vectors) {
-            error_report("vfio: Error: Failed to enable %d "
-                         "MSI vectors, retry with %d", vdev->nr_vectors, ret);
-        }
-
-        for (i = 0; i < vdev->nr_vectors; i++) {
-            VFIOMSIVector *vector = &vdev->msi_vectors[i];
-            if (vector->virq >= 0) {
-                vfio_remove_kvm_msi_virq(vector);
-            }
-            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
-                                NULL, NULL, NULL);
-            event_notifier_cleanup(&vector->interrupt);
-        }
-
-        g_free(vdev->msi_vectors);
-
-        if (ret > 0 && ret != vdev->nr_vectors) {
-            vdev->nr_vectors = ret;
-            goto retry;
-        }
-        vdev->nr_vectors = 0;
-
-        /*
-         * Failing to setup MSI doesn't really fall within any specification.
-         * Let's try leaving interrupts disabled and hope the guest figures
-         * out to fall back to INTx for this device.
-         */
-        error_report("vfio: Error: Failed to enable MSI");
-        vdev->interrupt = VFIO_INT_NONE;
-
-        return;
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, vdev->nr_vectors);
-}
-
-static void vfio_disable_msi_common(VFIODevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < vdev->nr_vectors; i++) {
-        VFIOMSIVector *vector = &vdev->msi_vectors[i];
-        if (vdev->msi_vectors[i].use) {
-            if (vector->virq >= 0) {
-                vfio_remove_kvm_msi_virq(vector);
-            }
-            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
-                                NULL, NULL, NULL);
-            event_notifier_cleanup(&vector->interrupt);
-        }
-    }
-
-    g_free(vdev->msi_vectors);
-    vdev->msi_vectors = NULL;
-    vdev->nr_vectors = 0;
-    vdev->interrupt = VFIO_INT_NONE;
-
-    vfio_enable_intx(vdev);
-}
-
-static void vfio_disable_msix(VFIODevice *vdev)
-{
-    int i;
-
-    msix_unset_vector_notifiers(&vdev->pdev);
-
-    /*
-     * MSI-X will only release vectors if MSI-X is still enabled on the
-     * device, check through the rest and release it ourselves if necessary.
-     */
-    for (i = 0; i < vdev->nr_vectors; i++) {
-        if (vdev->msi_vectors[i].use) {
-            vfio_msix_vector_release(&vdev->pdev, i);
-            msix_vector_unuse(&vdev->pdev, i);
-        }
-    }
-
-    if (vdev->nr_vectors) {
-        vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
-    }
-
-    vfio_disable_msi_common(vdev);
-
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-}
-
-static void vfio_disable_msi(VFIODevice *vdev)
-{
-    vfio_disable_irqindex(vdev, VFIO_PCI_MSI_IRQ_INDEX);
-    vfio_disable_msi_common(vdev);
-
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-}
-
-static void vfio_update_msi(VFIODevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < vdev->nr_vectors; i++) {
-        VFIOMSIVector *vector = &vdev->msi_vectors[i];
-        MSIMessage msg;
-
-        if (!vector->use || vector->virq < 0) {
-            continue;
-        }
-
-        msg = msi_get_message(&vdev->pdev, i);
-        vfio_update_kvm_msi_virq(vector, msg);
-    }
-}
-
-/*
- * IO Port/MMIO - Beware of the endians, VFIO is always little endian
- */
-static void vfio_bar_write(void *opaque, hwaddr addr,
-                           uint64_t data, unsigned size)
-{
-    VFIOBAR *bar = opaque;
-    union {
-        uint8_t byte;
-        uint16_t word;
-        uint32_t dword;
-        uint64_t qword;
-    } buf;
-
-    switch (size) {
-    case 1:
-        buf.byte = data;
-        break;
-    case 2:
-        buf.word = cpu_to_le16(data);
-        break;
-    case 4:
-        buf.dword = cpu_to_le32(data);
-        break;
-    default:
-        hw_error("vfio: unsupported write size, %d bytes", size);
-        break;
-    }
-
-    if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
-        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
-                     __func__, addr, data, size);
-    }
-
-#ifdef DEBUG_VFIO
-    {
-        VFIODevice *vdev = container_of(bar, VFIODevice, bars[bar->nr]);
-
-        DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"PRIx64
-                ", %d)\n", __func__, vdev->host.domain, vdev->host.bus,
-                vdev->host.slot, vdev->host.function, bar->nr, addr,
-                data, size);
-    }
-#endif
-
-    /*
-     * A read or write to a BAR always signals an INTx EOI.  This will
-     * do nothing if not pending (including not in INTx mode).  We assume
-     * that a BAR access is in response to an interrupt and that BAR
-     * accesses will service the interrupt.  Unfortunately, we don't know
-     * which access will service the interrupt, so we're potentially
-     * getting quite a few host interrupts per guest interrupt.
-     */
-    vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
-}
-
-static uint64_t vfio_bar_read(void *opaque,
-                              hwaddr addr, unsigned size)
-{
-    VFIOBAR *bar = opaque;
-    union {
-        uint8_t byte;
-        uint16_t word;
-        uint32_t dword;
-        uint64_t qword;
-    } buf;
-    uint64_t data = 0;
-
-    if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
-        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
-                     __func__, addr, size);
-        return (uint64_t)-1;
-    }
-
-    switch (size) {
-    case 1:
-        data = buf.byte;
-        break;
-    case 2:
-        data = le16_to_cpu(buf.word);
-        break;
-    case 4:
-        data = le32_to_cpu(buf.dword);
-        break;
-    default:
-        hw_error("vfio: unsupported read size, %d bytes", size);
-        break;
-    }
-
-#ifdef DEBUG_VFIO
-    {
-        VFIODevice *vdev = container_of(bar, VFIODevice, bars[bar->nr]);
-
-        DPRINTF("%s(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx
-                ", %d) = 0x%"PRIx64"\n", __func__, vdev->host.domain,
-                vdev->host.bus, vdev->host.slot, vdev->host.function,
-                bar->nr, addr, size, data);
-    }
-#endif
-
-    /* Same as write above */
-    vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
-
-    return data;
-}
-
-static const MemoryRegionOps vfio_bar_ops = {
-    .read = vfio_bar_read,
-    .write = vfio_bar_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_pci_load_rom(VFIODevice *vdev)
-{
-    struct vfio_region_info reg_info = {
-        .argsz = sizeof(reg_info),
-        .index = VFIO_PCI_ROM_REGION_INDEX
-    };
-    uint64_t size;
-    off_t off = 0;
-    size_t bytes;
-
-    if (ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info)) {
-        error_report("vfio: Error getting ROM info: %m");
-        return;
-    }
-
-    DPRINTF("Device %04x:%02x:%02x.%x ROM:\n", vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
-            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
-            (unsigned long)reg_info.flags);
-
-    vdev->rom_size = size = reg_info.size;
-    vdev->rom_offset = reg_info.offset;
-
-    if (!vdev->rom_size) {
-        vdev->rom_read_failed = true;
-        error_report("vfio-pci: Cannot read device rom at "
-                    "%04x:%02x:%02x.%x",
-                    vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                    vdev->host.function);
-        error_printf("Device option ROM contents are probably invalid "
-                    "(check dmesg).\nSkip option ROM probe with rombar=0, "
-                    "or load from file with romfile=\n");
-        return;
-    }
-
-    vdev->rom = g_malloc(size);
-    memset(vdev->rom, 0xff, size);
-
-    while (size) {
-        bytes = pread(vdev->fd, vdev->rom + off, size, vdev->rom_offset + off);
-        if (bytes == 0) {
-            break;
-        } else if (bytes > 0) {
-            off += bytes;
-            size -= bytes;
-        } else {
-            if (errno == EINTR || errno == EAGAIN) {
-                continue;
-            }
-            error_report("vfio: Error reading device ROM: %m");
-            break;
-        }
-    }
-}
-
-static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
-{
-    VFIODevice *vdev = opaque;
-    union {
-        uint8_t byte;
-        uint16_t word;
-        uint32_t dword;
-        uint64_t qword;
-    } val;
-    uint64_t data = 0;
-
-    /* Load the ROM lazily when the guest tries to read it */
-    if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
-        vfio_pci_load_rom(vdev);
-    }
-
-    memcpy(&val, vdev->rom + addr,
-           (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
-
-    switch (size) {
-    case 1:
-        data = val.byte;
-        break;
-    case 2:
-        data = le16_to_cpu(val.word);
-        break;
-    case 4:
-        data = le32_to_cpu(val.dword);
-        break;
-    default:
-        hw_error("vfio: unsupported read size, %d bytes\n", size);
-        break;
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x, 0x%"HWADDR_PRIx", 0x%x) = 0x%"PRIx64"\n",
-            __func__, vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, addr, size, data);
-
-    return data;
-}
-
-static void vfio_rom_write(void *opaque, hwaddr addr,
-                           uint64_t data, unsigned size)
-{
-}
-
-static const MemoryRegionOps vfio_rom_ops = {
-    .read = vfio_rom_read,
-    .write = vfio_rom_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static bool vfio_blacklist_opt_rom(VFIODevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint16_t vendor_id, device_id;
-    int count = 0;
-
-    vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
-
-    while (count < ARRAY_SIZE(romblacklist)) {
-        if (romblacklist[count].vendor_id == vendor_id &&
-            romblacklist[count].device_id == device_id) {
-                return true;
-        }
-        count++;
-    }
-
-    return false;
-}
-
-static void vfio_pci_size_rom(VFIODevice *vdev)
-{
-    uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
-    off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
-    DeviceState *dev = DEVICE(vdev);
-    char name[32];
-
-    if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
-        /* Since pci handles romfile, just print a message and return */
-        if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
-            error_printf("Warning : Device at %04x:%02x:%02x.%x "
-                         "is known to cause system instability issues during "
-                         "option rom execution. "
-                         "Proceeding anyway since user specified romfile\n",
-                         vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                         vdev->host.function);
-        }
-        return;
-    }
-
-    /*
-     * Use the same size ROM BAR as the physical device.  The contents
-     * will get filled in later when the guest tries to read it.
-     */
-    if (pread(vdev->fd, &orig, 4, offset) != 4 ||
-        pwrite(vdev->fd, &size, 4, offset) != 4 ||
-        pread(vdev->fd, &size, 4, offset) != 4 ||
-        pwrite(vdev->fd, &orig, 4, offset) != 4) {
-        error_report("%s(%04x:%02x:%02x.%x) failed: %m",
-                     __func__, vdev->host.domain, vdev->host.bus,
-                     vdev->host.slot, vdev->host.function);
-        return;
-    }
-
-    size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
-
-    if (!size) {
-        return;
-    }
-
-    if (vfio_blacklist_opt_rom(vdev)) {
-        if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
-            error_printf("Warning : Device at %04x:%02x:%02x.%x "
-                         "is known to cause system instability issues during "
-                         "option rom execution. "
-                         "Proceeding anyway since user specified non zero value for "
-                         "rombar\n",
-                         vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                         vdev->host.function);
-        } else {
-            error_printf("Warning : Rom loading for device at "
-                         "%04x:%02x:%02x.%x has been disabled due to "
-                         "system instability issues. "
-                         "Specify rombar=1 or romfile to force\n",
-                         vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                         vdev->host.function);
-            return;
-        }
-    }
-
-    DPRINTF("%04x:%02x:%02x.%x ROM size 0x%x\n", vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function, size);
-
-    snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function);
-
-    memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
-                          &vfio_rom_ops, vdev, name, size);
-
-    pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
-                     PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
-
-    vdev->pdev.has_rom = true;
-    vdev->rom_read_failed = false;
-}
-
-static void vfio_vga_write(void *opaque, hwaddr addr,
-                           uint64_t data, unsigned size)
-{
-    VFIOVGARegion *region = opaque;
-    VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
-    union {
-        uint8_t byte;
-        uint16_t word;
-        uint32_t dword;
-        uint64_t qword;
-    } buf;
-    off_t offset = vga->fd_offset + region->offset + addr;
-
-    switch (size) {
-    case 1:
-        buf.byte = data;
-        break;
-    case 2:
-        buf.word = cpu_to_le16(data);
-        break;
-    case 4:
-        buf.dword = cpu_to_le32(data);
-        break;
-    default:
-        hw_error("vfio: unsupported write size, %d bytes", size);
-        break;
-    }
-
-    if (pwrite(vga->fd, &buf, size, offset) != size) {
-        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
-                     __func__, region->offset + addr, data, size);
-    }
-
-    DPRINTF("%s(0x%"HWADDR_PRIx", 0x%"PRIx64", %d)\n",
-            __func__, region->offset + addr, data, size);
-}
-
-static uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
-{
-    VFIOVGARegion *region = opaque;
-    VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
-    union {
-        uint8_t byte;
-        uint16_t word;
-        uint32_t dword;
-        uint64_t qword;
-    } buf;
-    uint64_t data = 0;
-    off_t offset = vga->fd_offset + region->offset + addr;
-
-    if (pread(vga->fd, &buf, size, offset) != size) {
-        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
-                     __func__, region->offset + addr, size);
-        return (uint64_t)-1;
-    }
-
-    switch (size) {
-    case 1:
-        data = buf.byte;
-        break;
-    case 2:
-        data = le16_to_cpu(buf.word);
-        break;
-    case 4:
-        data = le32_to_cpu(buf.dword);
-        break;
-    default:
-        hw_error("vfio: unsupported read size, %d bytes", size);
-        break;
-    }
-
-    DPRINTF("%s(0x%"HWADDR_PRIx", %d) = 0x%"PRIx64"\n",
-            __func__, region->offset + addr, size, data);
-
-    return data;
-}
-
-static const MemoryRegionOps vfio_vga_ops = {
-    .read = vfio_vga_read,
-    .write = vfio_vga_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-/*
- * Device specific quirks
- */
-
-/* Is range1 fully contained within range2?  */
-static bool vfio_range_contained(uint64_t first1, uint64_t len1,
-                                 uint64_t first2, uint64_t len2) {
-    return (first1 >= first2 && first1 + len1 <= first2 + len2);
-}
-
-static bool vfio_flags_enabled(uint8_t flags, uint8_t mask)
-{
-    return (mask && (flags & mask) == mask);
-}
-
-static uint64_t vfio_generic_window_quirk_read(void *opaque,
-                                               hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    uint64_t data;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
-        ranges_overlap(addr, size,
-                       quirk->data.data_offset, quirk->data.data_size)) {
-        hwaddr offset = addr - quirk->data.data_offset;
-
-        if (!vfio_range_contained(addr, size, quirk->data.data_offset,
-                                  quirk->data.data_size)) {
-            hw_error("%s: window data read not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        data = vfio_pci_read_config(&vdev->pdev,
-                                    quirk->data.address_val + offset, size);
-
-        DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"
-                PRIx64"\n", memory_region_name(&quirk->mem), vdev->host.domain,
-                vdev->host.bus, vdev->host.slot, vdev->host.function,
-                quirk->data.bar, addr, size, data);
-    } else {
-        data = vfio_bar_read(&vdev->bars[quirk->data.bar],
-                             addr + quirk->data.base_offset, size);
-    }
-
-    return data;
-}
-
-static void vfio_generic_window_quirk_write(void *opaque, hwaddr addr,
-                                            uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-
-    if (ranges_overlap(addr, size,
-                       quirk->data.address_offset, quirk->data.address_size)) {
-
-        if (addr != quirk->data.address_offset) {
-            hw_error("%s: offset write into address window: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        if ((data & ~quirk->data.address_mask) == quirk->data.address_match) {
-            quirk->data.flags |= quirk->data.write_flags |
-                                 quirk->data.read_flags;
-            quirk->data.address_val = data & quirk->data.address_mask;
-        } else {
-            quirk->data.flags &= ~(quirk->data.write_flags |
-                                   quirk->data.read_flags);
-        }
-    }
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
-        ranges_overlap(addr, size,
-                       quirk->data.data_offset, quirk->data.data_size)) {
-        hwaddr offset = addr - quirk->data.data_offset;
-
-        if (!vfio_range_contained(addr, size, quirk->data.data_offset,
-                                  quirk->data.data_size)) {
-            hw_error("%s: window data write not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        vfio_pci_write_config(&vdev->pdev,
-                              quirk->data.address_val + offset, data, size);
-        DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"
-                PRIx64", %d)\n", memory_region_name(&quirk->mem),
-                vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                vdev->host.function, quirk->data.bar, addr, data, size);
-        return;
-    }
-
-    vfio_bar_write(&vdev->bars[quirk->data.bar],
-                   addr + quirk->data.base_offset, data, size);
-}
-
-static const MemoryRegionOps vfio_generic_window_quirk = {
-    .read = vfio_generic_window_quirk_read,
-    .write = vfio_generic_window_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static uint64_t vfio_generic_quirk_read(void *opaque,
-                                        hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-    hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
-    uint64_t data;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.read_flags) &&
-        ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
-        if (!vfio_range_contained(addr, size, offset,
-                                  quirk->data.address_mask + 1)) {
-            hw_error("%s: read not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        data = vfio_pci_read_config(&vdev->pdev, addr - offset, size);
-
-        DPRINTF("%s read(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"
-                PRIx64"\n", memory_region_name(&quirk->mem), vdev->host.domain,
-                vdev->host.bus, vdev->host.slot, vdev->host.function,
-                quirk->data.bar, addr + base, size, data);
-    } else {
-        data = vfio_bar_read(&vdev->bars[quirk->data.bar], addr + base, size);
-    }
-
-    return data;
-}
-
-static void vfio_generic_quirk_write(void *opaque, hwaddr addr,
-                                     uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-    hwaddr offset = quirk->data.address_match & ~TARGET_PAGE_MASK;
-
-    if (vfio_flags_enabled(quirk->data.flags, quirk->data.write_flags) &&
-        ranges_overlap(addr, size, offset, quirk->data.address_mask + 1)) {
-        if (!vfio_range_contained(addr, size, offset,
-                                  quirk->data.address_mask + 1)) {
-            hw_error("%s: write not fully contained: %s",
-                     __func__, memory_region_name(&quirk->mem));
-        }
-
-        vfio_pci_write_config(&vdev->pdev, addr - offset, data, size);
-
-        DPRINTF("%s write(%04x:%02x:%02x.%x:BAR%d+0x%"HWADDR_PRIx", 0x%"
-                PRIx64", %d)\n", memory_region_name(&quirk->mem),
-                vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                vdev->host.function, quirk->data.bar, addr + base, data, size);
-    } else {
-        vfio_bar_write(&vdev->bars[quirk->data.bar], addr + base, data, size);
-    }
-}
-
-static const MemoryRegionOps vfio_generic_quirk = {
-    .read = vfio_generic_quirk_read,
-    .write = vfio_generic_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-#define PCI_VENDOR_ID_ATI               0x1002
-
-/*
- * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
- * through VGA register 0x3c3.  On newer cards, the I/O port BAR is always
- * BAR4 (older cards like the X550 used BAR1, but we don't care to support
- * those).  Note that on bare metal, a read of 0x3c3 doesn't always return the
- * I/O port BAR address.  Originally this was coded to return the virtual BAR
- * address only if the physical register read returns the actual BAR address,
- * but users have reported greater success if we return the virtual address
- * unconditionally.
- */
-static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
-                                        hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    uint64_t data = vfio_pci_read_config(&vdev->pdev,
-                                         PCI_BASE_ADDRESS_0 + (4 * 4) + 1,
-                                         size);
-    DPRINTF("%s(0x3c3, 1) = 0x%"PRIx64"\n", __func__, data);
-
-    return data;
-}
-
-static const MemoryRegionOps vfio_ati_3c3_quirk = {
-    .read = vfio_ati_3c3_quirk_read,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_vga_probe_ati_3c3_quirk(VFIODevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    /*
-     * As long as the BAR is >= 256 bytes it will be aligned such that the
-     * lower byte is always zero.  Filter out anything else, if it exists.
-     */
-    if (!vdev->bars[4].ioport || vdev->bars[4].size < 256) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, quirk,
-                          "vfio-ati-3c3-quirk", 1);
-    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                                3 /* offset 3 bytes from 0x3c0 */, &quirk->mem);
-
-    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
-                      quirk, next);
-
-    DPRINTF("Enabled ATI/AMD quirk 0x3c3 BAR4for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-/*
- * Newer ATI/AMD devices, including HD5450 and HD7850, have a window to PCI
- * config space through MMIO BAR2 at offset 0x4000.  Nothing seems to access
- * the MMIO space directly, but a window to this space is provided through
- * I/O port BAR4.  Offset 0x0 is the address register and offset 0x4 is the
- * data register.  When the address is programmed to a range of 0x4000-0x4fff
- * PCI configuration space is available.  Experimentation seems to indicate
- * that only read-only access is provided, but we drop writes when the window
- * is enabled to config space nonetheless.
- */
-static void vfio_probe_ati_bar4_window_quirk(VFIODevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 4 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.address_size = 4;
-    quirk->data.data_offset = 4;
-    quirk->data.data_size = 4;
-    quirk->data.address_match = 0x4000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-    quirk->data.read_flags = quirk->data.write_flags = 1;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev),
-                          &vfio_generic_window_quirk, quirk,
-                          "vfio-ati-bar4-window-quirk", 8);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
-                          quirk->data.base_offset, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    DPRINTF("Enabled ATI/AMD BAR4 window quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-#define PCI_VENDOR_ID_REALTEK 0x10ec
-
-/*
- * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
- * offset 0x70 there is a dword data register, offset 0x74 is a dword address
- * register.  According to the Linux r8169 driver, the MSI-X table is addressed
- * when the "type" portion of the address register is set to 0x1.  This appears
- * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
- * "address latched" indicator.  Bits 12:15 are a mask field, which we can
- * ignore because the MSI-X table should always be accessed as a dword (full
- * mask).  Bits 0:11 is offset within the type.
- *
- * Example trace:
- *
- * Read from MSI-X table offset 0
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
- *
- * Write 0xfee00000 to MSI-X table offset 0
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
- * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
- * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
- */
-
-static uint64_t vfio_rtl8168_window_quirk_read(void *opaque,
-                                               hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-
-    switch (addr) {
-    case 4: /* address */
-        if (quirk->data.flags) {
-            DPRINTF("%s fake read(%04x:%02x:%02x.%d)\n",
-                    memory_region_name(&quirk->mem), vdev->host.domain,
-                    vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-            return quirk->data.address_match ^ 0x10000000U;
-        }
-        break;
-    case 0: /* data */
-        if (quirk->data.flags) {
-            uint64_t val;
-
-            DPRINTF("%s MSI-X table read(%04x:%02x:%02x.%d)\n",
-                    memory_region_name(&quirk->mem), vdev->host.domain,
-                    vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-            if (!(vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
-                return 0;
-            }
-
-            io_mem_read(&vdev->pdev.msix_table_mmio,
-                        (hwaddr)(quirk->data.address_match & 0xfff),
-                        &val, size);
-            return val;
-        }
-    }
-
-    DPRINTF("%s direct read(%04x:%02x:%02x.%d)\n",
-            memory_region_name(&quirk->mem), vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-    return vfio_bar_read(&vdev->bars[quirk->data.bar], addr + 0x70, size);
-}
-
-static void vfio_rtl8168_window_quirk_write(void *opaque, hwaddr addr,
-                                            uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-
-    switch (addr) {
-    case 4: /* address */
-        if ((data & 0x7fff0000) == 0x10000) {
-            if (data & 0x10000000U &&
-                vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
-
-                DPRINTF("%s MSI-X table write(%04x:%02x:%02x.%d)\n",
-                        memory_region_name(&quirk->mem), vdev->host.domain,
-                        vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-                io_mem_write(&vdev->pdev.msix_table_mmio,
-                             (hwaddr)(quirk->data.address_match & 0xfff),
-                             data, size);
-            }
-
-            quirk->data.flags = 1;
-            quirk->data.address_match = data;
-
-            return;
-        }
-        quirk->data.flags = 0;
-        break;
-    case 0: /* data */
-        quirk->data.address_mask = data;
-        break;
-    }
-
-    DPRINTF("%s direct write(%04x:%02x:%02x.%d)\n",
-            memory_region_name(&quirk->mem), vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-    vfio_bar_write(&vdev->bars[quirk->data.bar], addr + 0x70, data, size);
-}
-
-static const MemoryRegionOps vfio_rtl8168_window_quirk = {
-    .read = vfio_rtl8168_window_quirk_read,
-    .write = vfio_rtl8168_window_quirk_write,
-    .valid = {
-        .min_access_size = 4,
-        .max_access_size = 4,
-        .unaligned = false,
-    },
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_probe_rtl8168_bar2_window_quirk(VFIODevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_REALTEK ||
-        pci_get_word(pdev->config + PCI_DEVICE_ID) != 0x8168 || nr != 2) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_rtl8168_window_quirk,
-                          quirk, "vfio-rtl8168-window-quirk", 8);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
-                                        0x70, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    DPRINTF("Enabled RTL8168 BAR2 window quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-/*
- * Trap the BAR2 MMIO window to config space as well.
- */
-static void vfio_probe_ati_bar2_4000_quirk(VFIODevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    /* Only enable on newer devices where BAR2 is 64bit */
-    if (!vdev->has_vga || nr != 2 || !vdev->bars[2].mem64 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_ATI) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x4000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
-                          "vfio-ati-bar2-4000-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    DPRINTF("Enabled ATI/AMD BAR2 0x4000 quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-/*
- * Older ATI/AMD cards like the X550 have a similar window to that above.
- * I/O port BAR1 provides a window to a mirror of PCI config space located
- * in BAR2 at offset 0xf00.  We don't care to support such older cards, but
- * note it for future reference.
- */
-
-#define PCI_VENDOR_ID_NVIDIA                    0x10de
-
-/*
- * Nvidia has several different methods to get to config space, the
- * nouveu project has several of these documented here:
- * https://github.com/pathscale/envytools/tree/master/hwdocs
- *
- * The first quirk is actually not documented in envytools and is found
- * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]).  This is an
- * NV46 chipset.  The backdoor uses the legacy VGA I/O ports to access
- * the mirror of PCI config space found at BAR0 offset 0x1800.  The access
- * sequence first writes 0x338 to I/O port 0x3d4.  The target offset is
- * then written to 0x3d0.  Finally 0x538 is written for a read and 0x738
- * is written for a write to 0x3d4.  The BAR0 offset is then accessible
- * through 0x3d0.  This quirk doesn't seem to be necessary on newer cards
- * that use the I/O port BAR5 window but it doesn't hurt to leave it.
- */
-enum {
-    NV_3D0_NONE = 0,
-    NV_3D0_SELECT,
-    NV_3D0_WINDOW,
-    NV_3D0_READ,
-    NV_3D0_WRITE,
-};
-
-static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
-                                           hwaddr addr, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-    uint64_t data = vfio_vga_read(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                                  addr + quirk->data.base_offset, size);
-
-    if (quirk->data.flags == NV_3D0_READ && addr == quirk->data.data_offset) {
-        data = vfio_pci_read_config(pdev, quirk->data.address_val, size);
-        DPRINTF("%s(0x3d0, %d) = 0x%"PRIx64"\n", __func__, size, data);
-    }
-
-    quirk->data.flags = NV_3D0_NONE;
-
-    return data;
-}
-
-static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
-                                        uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-
-    switch (quirk->data.flags) {
-    case NV_3D0_NONE:
-        if (addr == quirk->data.address_offset && data == 0x338) {
-            quirk->data.flags = NV_3D0_SELECT;
-        }
-        break;
-    case NV_3D0_SELECT:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.data_offset &&
-            (data & ~quirk->data.address_mask) == quirk->data.address_match) {
-            quirk->data.flags = NV_3D0_WINDOW;
-            quirk->data.address_val = data & quirk->data.address_mask;
-        }
-        break;
-    case NV_3D0_WINDOW:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.address_offset) {
-            if (data == 0x538) {
-                quirk->data.flags = NV_3D0_READ;
-            } else if (data == 0x738) {
-                quirk->data.flags = NV_3D0_WRITE;
-            }
-        }
-        break;
-    case NV_3D0_WRITE:
-        quirk->data.flags = NV_3D0_NONE;
-        if (addr == quirk->data.data_offset) {
-            vfio_pci_write_config(pdev, quirk->data.address_val, data, size);
-            DPRINTF("%s(0x3d0, 0x%"PRIx64", %d)\n", __func__, data, size);
-            return;
-        }
-        break;
-    }
-
-    vfio_vga_write(&vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                   addr + quirk->data.base_offset, data, size);
-}
-
-static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
-    .read = vfio_nvidia_3d0_quirk_read,
-    .write = vfio_nvidia_3d0_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_vga_probe_nvidia_3d0_quirk(VFIODevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA ||
-        !vdev->bars[1].size) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.base_offset = 0x10;
-    quirk->data.address_offset = 4;
-    quirk->data.address_size = 2;
-    quirk->data.address_match = 0x1800;
-    quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
-    quirk->data.data_offset = 0;
-    quirk->data.data_size = 4;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_3d0_quirk,
-                          quirk, "vfio-nvidia-3d0-quirk", 6);
-    memory_region_add_subregion(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                                quirk->data.base_offset, &quirk->mem);
-
-    QLIST_INSERT_HEAD(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks,
-                      quirk, next);
-
-    DPRINTF("Enabled NVIDIA VGA 0x3d0 quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-/*
- * The second quirk is documented in envytools.  The I/O port BAR5 is just
- * a set of address/data ports to the MMIO BARs.  The BAR we care about is
- * again BAR0.  This backdoor is apparently a bit newer than the one above
- * so we need to not only trap 256 bytes @0x1800, but all of PCI config
- * space, including extended space is available at the 4k @0x88000.
- */
-enum {
-    NV_BAR5_ADDRESS = 0x1,
-    NV_BAR5_ENABLE = 0x2,
-    NV_BAR5_MASTER = 0x4,
-    NV_BAR5_VALID = 0x7,
-};
-
-static void vfio_nvidia_bar5_window_quirk_write(void *opaque, hwaddr addr,
-                                                uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-
-    switch (addr) {
-    case 0x0:
-        if (data & 0x1) {
-            quirk->data.flags |= NV_BAR5_MASTER;
-        } else {
-            quirk->data.flags &= ~NV_BAR5_MASTER;
-        }
-        break;
-    case 0x4:
-        if (data & 0x1) {
-            quirk->data.flags |= NV_BAR5_ENABLE;
-        } else {
-            quirk->data.flags &= ~NV_BAR5_ENABLE;
-        }
-        break;
-    case 0x8:
-        if (quirk->data.flags & NV_BAR5_MASTER) {
-            if ((data & ~0xfff) == 0x88000) {
-                quirk->data.flags |= NV_BAR5_ADDRESS;
-                quirk->data.address_val = data & 0xfff;
-            } else if ((data & ~0xff) == 0x1800) {
-                quirk->data.flags |= NV_BAR5_ADDRESS;
-                quirk->data.address_val = data & 0xff;
-            } else {
-                quirk->data.flags &= ~NV_BAR5_ADDRESS;
-            }
-        }
-        break;
-    }
-
-    vfio_generic_window_quirk_write(opaque, addr, data, size);
-}
-
-static const MemoryRegionOps vfio_nvidia_bar5_window_quirk = {
-    .read = vfio_generic_window_quirk_read,
-    .write = vfio_nvidia_bar5_window_quirk_write,
-    .valid.min_access_size = 4,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-static void vfio_probe_nvidia_bar5_window_quirk(VFIODevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 5 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.read_flags = quirk->data.write_flags = NV_BAR5_VALID;
-    quirk->data.address_offset = 0x8;
-    quirk->data.address_size = 0; /* actually 4, but avoids generic code */
-    quirk->data.data_offset = 0xc;
-    quirk->data.data_size = 4;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev),
-                          &vfio_nvidia_bar5_window_quirk, quirk,
-                          "vfio-nvidia-bar5-window-quirk", 16);
-    memory_region_add_subregion_overlap(&vdev->bars[nr].mem, 0, &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    DPRINTF("Enabled NVIDIA BAR5 window quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-static void vfio_nvidia_88000_quirk_write(void *opaque, hwaddr addr,
-                                          uint64_t data, unsigned size)
-{
-    VFIOQuirk *quirk = opaque;
-    VFIODevice *vdev = quirk->vdev;
-    PCIDevice *pdev = &vdev->pdev;
-    hwaddr base = quirk->data.address_match & TARGET_PAGE_MASK;
-
-    vfio_generic_quirk_write(opaque, addr, data, size);
-
-    /*
-     * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
-     * MSI capability ID register.  Both the ID and next register are
-     * read-only, so we allow writes covering either of those to real hw.
-     * NB - only fixed for the 0x88000 MMIO window.
-     */
-    if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
-        vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
-        vfio_bar_write(&vdev->bars[quirk->data.bar], addr + base, data, size);
-    }
-}
-
-static const MemoryRegionOps vfio_nvidia_88000_quirk = {
-    .read = vfio_generic_quirk_read,
-    .write = vfio_nvidia_88000_quirk_write,
-    .endianness = DEVICE_LITTLE_ENDIAN,
-};
-
-/*
- * Finally, BAR0 itself.  We want to redirect any accesses to either
- * 0x1800 or 0x88000 through the PCI config space access functions.
- *
- * NB - quirk at a page granularity or else they don't seem to work when
- *      BARs are mmap'd
- *
- * Here's offset 0x88000...
- */
-static void vfio_probe_nvidia_bar0_88000_quirk(VFIODevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-    uint16_t vendor, class;
-
-    vendor = pci_get_word(pdev->config + PCI_VENDOR_ID);
-    class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
-
-    if (nr != 0 || vendor != PCI_VENDOR_ID_NVIDIA ||
-        class != PCI_CLASS_DISPLAY_VGA) {
-        return;
-    }
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x88000;
-    quirk->data.address_mask = PCIE_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_nvidia_88000_quirk,
-                          quirk, "vfio-nvidia-bar0-88000-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    DPRINTF("Enabled NVIDIA BAR0 0x88000 quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-/*
- * And here's the same for BAR0 offset 0x1800...
- */
-static void vfio_probe_nvidia_bar0_1800_quirk(VFIODevice *vdev, int nr)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    VFIOQuirk *quirk;
-
-    if (!vdev->has_vga || nr != 0 ||
-        pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_NVIDIA) {
-        return;
-    }
-
-    /* Log the chipset ID */
-    DPRINTF("Nvidia NV%02x\n",
-            (unsigned int)(vfio_bar_read(&vdev->bars[0], 0, 4) >> 20) & 0xff);
-
-    quirk = g_malloc0(sizeof(*quirk));
-    quirk->vdev = vdev;
-    quirk->data.flags = quirk->data.read_flags = quirk->data.write_flags = 1;
-    quirk->data.address_match = 0x1800;
-    quirk->data.address_mask = PCI_CONFIG_SPACE_SIZE - 1;
-    quirk->data.bar = nr;
-
-    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_generic_quirk, quirk,
-                          "vfio-nvidia-bar0-1800-quirk",
-                          TARGET_PAGE_ALIGN(quirk->data.address_mask + 1));
-    memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
-                          quirk->data.address_match & TARGET_PAGE_MASK,
-                          &quirk->mem, 1);
-
-    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
-
-    DPRINTF("Enabled NVIDIA BAR0 0x1800 quirk for device %04x:%02x:%02x.%x\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-}
-
-/*
- * TODO - Some Nvidia devices provide config access to their companion HDA
- * device and even to their parent bridge via these config space mirrors.
- * Add quirks for those regions.
- */
-
-/*
- * Common quirk probe entry points.
- */
-static void vfio_vga_quirk_setup(VFIODevice *vdev)
-{
-    vfio_vga_probe_ati_3c3_quirk(vdev);
-    vfio_vga_probe_nvidia_3d0_quirk(vdev);
-}
-
-static void vfio_vga_quirk_teardown(VFIODevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(vdev->vga.region); i++) {
-        while (!QLIST_EMPTY(&vdev->vga.region[i].quirks)) {
-            VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga.region[i].quirks);
-            memory_region_del_subregion(&vdev->vga.region[i].mem, &quirk->mem);
-            object_unparent(OBJECT(&quirk->mem));
-            QLIST_REMOVE(quirk, next);
-            g_free(quirk);
-        }
-    }
-}
-
-static void vfio_bar_quirk_setup(VFIODevice *vdev, int nr)
-{
-    vfio_probe_ati_bar4_window_quirk(vdev, nr);
-    vfio_probe_ati_bar2_4000_quirk(vdev, nr);
-    vfio_probe_nvidia_bar5_window_quirk(vdev, nr);
-    vfio_probe_nvidia_bar0_88000_quirk(vdev, nr);
-    vfio_probe_nvidia_bar0_1800_quirk(vdev, nr);
-    vfio_probe_rtl8168_bar2_window_quirk(vdev, nr);
-}
-
-static void vfio_bar_quirk_teardown(VFIODevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-
-    while (!QLIST_EMPTY(&bar->quirks)) {
-        VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
-        memory_region_del_subregion(&bar->mem, &quirk->mem);
-        object_unparent(OBJECT(&quirk->mem));
-        QLIST_REMOVE(quirk, next);
-        g_free(quirk);
-    }
-}
-
-/*
- * PCI config space
- */
-static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
-{
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
-
-    memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
-    emu_bits = le32_to_cpu(emu_bits);
-
-    if (emu_bits) {
-        emu_val = pci_default_read_config(pdev, addr, len);
-    }
-
-    if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
-        ssize_t ret;
-
-        ret = pread(vdev->fd, &phys_val, len, vdev->config_offset + addr);
-        if (ret != len) {
-            error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m",
-                         __func__, vdev->host.domain, vdev->host.bus,
-                         vdev->host.slot, vdev->host.function, addr, len);
-            return -errno;
-        }
-        phys_val = le32_to_cpu(phys_val);
-    }
-
-    val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
-
-    DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, addr, len, val);
-
-    return val;
-}
-
-static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
-                                  uint32_t val, int len)
-{
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    uint32_t val_le = cpu_to_le32(val);
-
-    DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__,
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, addr, val, len);
-
-    /* Write everything to VFIO, let it filter out what we can't write */
-    if (pwrite(vdev->fd, &val_le, len, vdev->config_offset + addr) != len) {
-        error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m",
-                     __func__, vdev->host.domain, vdev->host.bus,
-                     vdev->host.slot, vdev->host.function, addr, val, len);
-    }
-
-    /* MSI/MSI-X Enabling/Disabling */
-    if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
-        ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
-        int is_enabled, was_enabled = msi_enabled(pdev);
-
-        pci_default_write_config(pdev, addr, val, len);
-
-        is_enabled = msi_enabled(pdev);
-
-        if (!was_enabled) {
-            if (is_enabled) {
-                vfio_enable_msi(vdev);
-            }
-        } else {
-            if (!is_enabled) {
-                vfio_disable_msi(vdev);
-            } else {
-                vfio_update_msi(vdev);
-            }
-        }
-    } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
-        ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
-        int is_enabled, was_enabled = msix_enabled(pdev);
-
-        pci_default_write_config(pdev, addr, val, len);
-
-        is_enabled = msix_enabled(pdev);
-
-        if (!was_enabled && is_enabled) {
-            vfio_enable_msix(vdev);
-        } else if (was_enabled && !is_enabled) {
-            vfio_disable_msix(vdev);
-        }
-    } else {
-        /* Write everything to QEMU to keep emulated bits correct */
-        pci_default_write_config(pdev, addr, val, len);
-    }
-}
-
-/*
- * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
- */
-static int vfio_dma_unmap(VFIOContainer *container,
-                          hwaddr iova, ram_addr_t size)
-{
-    struct vfio_iommu_type1_dma_unmap unmap = {
-        .argsz = sizeof(unmap),
-        .flags = 0,
-        .iova = iova,
-        .size = size,
-    };
-
-    if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
-        DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno);
-        return -errno;
-    }
-
-    return 0;
-}
-
-static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
-                        ram_addr_t size, void *vaddr, bool readonly)
-{
-    struct vfio_iommu_type1_dma_map map = {
-        .argsz = sizeof(map),
-        .flags = VFIO_DMA_MAP_FLAG_READ,
-        .vaddr = (__u64)(uintptr_t)vaddr,
-        .iova = iova,
-        .size = size,
-    };
-
-    if (!readonly) {
-        map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
-    }
-
-    /*
-     * Try the mapping, if it fails with EBUSY, unmap the region and try
-     * again.  This shouldn't be necessary, but we sometimes see it in
-     * the the VGA ROM space.
-     */
-    if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
-        (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
-         ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
-        return 0;
-    }
-
-    DPRINTF("VFIO_MAP_DMA: %d\n", -errno);
-    return -errno;
-}
-
-static bool vfio_listener_skipped_section(MemoryRegionSection *section)
-{
-    return (!memory_region_is_ram(section->mr) &&
-            !memory_region_is_iommu(section->mr)) ||
-           /*
-            * Sizing an enabled 64-bit BAR can cause spurious mappings to
-            * addresses in the upper part of the 64-bit address space.  These
-            * are never accessed by the CPU and beyond the address width of
-            * some IOMMU hardware.  TODO: VFIO should tell us the IOMMU width.
-            */
-           section->offset_within_address_space & (1ULL << 63);
-}
-
-static void vfio_iommu_map_notify(Notifier *n, void *data)
-{
-    VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
-    VFIOContainer *container = giommu->container;
-    IOMMUTLBEntry *iotlb = data;
-    MemoryRegion *mr;
-    hwaddr xlat;
-    hwaddr len = iotlb->addr_mask + 1;
-    void *vaddr;
-    int ret;
-
-    DPRINTF("iommu map @ %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
-            iotlb->iova, iotlb->iova + iotlb->addr_mask);
-
-    /*
-     * The IOMMU TLB entry we have just covers translation through
-     * this IOMMU to its immediate target.  We need to translate
-     * it the rest of the way through to memory.
-     */
-    mr = address_space_translate(&address_space_memory,
-                                 iotlb->translated_addr,
-                                 &xlat, &len, iotlb->perm & IOMMU_WO);
-    if (!memory_region_is_ram(mr)) {
-        DPRINTF("iommu map to non memory area %"HWADDR_PRIx"\n",
-                xlat);
-        return;
-    }
-    /*
-     * Translation truncates length to the IOMMU page size,
-     * check that it did not truncate too much.
-     */
-    if (len & iotlb->addr_mask) {
-        DPRINTF("iommu has granularity incompatible with target AS\n");
-        return;
-    }
-
-    if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
-        vaddr = memory_region_get_ram_ptr(mr) + xlat;
-
-        ret = vfio_dma_map(container, iotlb->iova,
-                           iotlb->addr_mask + 1, vaddr,
-                           !(iotlb->perm & IOMMU_WO) || mr->readonly);
-        if (ret) {
-            error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
-                         "0x%"HWADDR_PRIx", %p) = %d (%m)",
-                         container, iotlb->iova,
-                         iotlb->addr_mask + 1, vaddr, ret);
-        }
-    } else {
-        ret = vfio_dma_unmap(container, iotlb->iova, iotlb->addr_mask + 1);
-        if (ret) {
-            error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
-                         "0x%"HWADDR_PRIx") = %d (%m)",
-                         container, iotlb->iova,
-                         iotlb->addr_mask + 1, ret);
-        }
-    }
-}
-
-static void vfio_listener_region_add(MemoryListener *listener,
-                                     MemoryRegionSection *section)
-{
-    VFIOContainer *container = container_of(listener, VFIOContainer,
-                                            iommu_data.type1.listener);
-    hwaddr iova, end;
-    Int128 llend;
-    void *vaddr;
-    int ret;
-
-    if (vfio_listener_skipped_section(section)) {
-        DPRINTF("SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
-                section->offset_within_address_space,
-                section->offset_within_address_space +
-                int128_get64(int128_sub(section->size, int128_one())));
-        return;
-    }
-
-    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
-                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
-        error_report("%s received unaligned region", __func__);
-        return;
-    }
-
-    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
-    llend = int128_make64(section->offset_within_address_space);
-    llend = int128_add(llend, section->size);
-    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
-
-    if (int128_ge(int128_make64(iova), llend)) {
-        return;
-    }
-
-    memory_region_ref(section->mr);
-
-    if (memory_region_is_iommu(section->mr)) {
-        VFIOGuestIOMMU *giommu;
-
-        DPRINTF("region_add [iommu] %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
-                iova, int128_get64(int128_sub(llend, int128_one())));
-        /*
-         * FIXME: We should do some checking to see if the
-         * capabilities of the host VFIO IOMMU are adequate to model
-         * the guest IOMMU
-         *
-         * FIXME: For VFIO iommu types which have KVM acceleration to
-         * avoid bouncing all map/unmaps through qemu this way, this
-         * would be the right place to wire that up (tell the KVM
-         * device emulation the VFIO iommu handles to use).
-         */
-        /*
-         * This assumes that the guest IOMMU is empty of
-         * mappings at this point.
-         *
-         * One way of doing this is:
-         * 1. Avoid sharing IOMMUs between emulated devices or different
-         * IOMMU groups.
-         * 2. Implement VFIO_IOMMU_ENABLE in the host kernel to fail if
-         * there are some mappings in IOMMU.
-         *
-         * VFIO on SPAPR does that. Other IOMMU models may do that different,
-         * they must make sure there are no existing mappings or
-         * loop through existing mappings to map them into VFIO.
-         */
-        giommu = g_malloc0(sizeof(*giommu));
-        giommu->iommu = section->mr;
-        giommu->container = container;
-        giommu->n.notify = vfio_iommu_map_notify;
-        QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
-        memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
-
-        return;
-    }
-
-    /* Here we assume that memory_region_is_ram(section->mr)==true */
-
-    end = int128_get64(llend);
-    vaddr = memory_region_get_ram_ptr(section->mr) +
-            section->offset_within_region +
-            (iova - section->offset_within_address_space);
-
-    DPRINTF("region_add [ram] %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
-            iova, end - 1, vaddr);
-
-    ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
-    if (ret) {
-        error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
-                     "0x%"HWADDR_PRIx", %p) = %d (%m)",
-                     container, iova, end - iova, vaddr, ret);
-
-        /*
-         * On the initfn path, store the first error in the container so we
-         * can gracefully fail.  Runtime, there's not much we can do other
-         * than throw a hardware error.
-         */
-        if (!container->iommu_data.type1.initialized) {
-            if (!container->iommu_data.type1.error) {
-                container->iommu_data.type1.error = ret;
-            }
-        } else {
-            hw_error("vfio: DMA mapping failed, unable to continue");
-        }
-    }
-}
-
-static void vfio_listener_region_del(MemoryListener *listener,
-                                     MemoryRegionSection *section)
-{
-    VFIOContainer *container = container_of(listener, VFIOContainer,
-                                            iommu_data.type1.listener);
-    hwaddr iova, end;
-    int ret;
-
-    if (vfio_listener_skipped_section(section)) {
-        DPRINTF("SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
-                section->offset_within_address_space,
-                section->offset_within_address_space +
-                int128_get64(int128_sub(section->size, int128_one())));
-        return;
-    }
-
-    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
-                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
-        error_report("%s received unaligned region", __func__);
-        return;
-    }
-
-    if (memory_region_is_iommu(section->mr)) {
-        VFIOGuestIOMMU *giommu;
-
-        QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
-            if (giommu->iommu == section->mr) {
-                memory_region_unregister_iommu_notifier(&giommu->n);
-                QLIST_REMOVE(giommu, giommu_next);
-                g_free(giommu);
-                break;
-            }
-        }
-
-        /*
-         * FIXME: We assume the one big unmap below is adequate to
-         * remove any individual page mappings in the IOMMU which
-         * might have been copied into VFIO. This works for a page table
-         * based IOMMU where a big unmap flattens a large range of IO-PTEs.
-         * That may not be true for all IOMMU types.
-         */
-    }
-
-    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
-    end = (section->offset_within_address_space + int128_get64(section->size)) &
-          TARGET_PAGE_MASK;
-
-    if (iova >= end) {
-        return;
-    }
-
-    DPRINTF("region_del %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
-            iova, end - 1);
-
-    ret = vfio_dma_unmap(container, iova, end - iova);
-    memory_region_unref(section->mr);
-    if (ret) {
-        error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
-                     "0x%"HWADDR_PRIx") = %d (%m)",
-                     container, iova, end - iova, ret);
-    }
-}
-
-static MemoryListener vfio_memory_listener = {
-    .region_add = vfio_listener_region_add,
-    .region_del = vfio_listener_region_del,
-};
-
-static void vfio_listener_release(VFIOContainer *container)
-{
-    memory_listener_unregister(&container->iommu_data.type1.listener);
-}
-
-/*
- * Interrupt setup
- */
-static void vfio_disable_interrupts(VFIODevice *vdev)
-{
-    switch (vdev->interrupt) {
-    case VFIO_INT_INTx:
-        vfio_disable_intx(vdev);
-        break;
-    case VFIO_INT_MSI:
-        vfio_disable_msi(vdev);
-        break;
-    case VFIO_INT_MSIX:
-        vfio_disable_msix(vdev);
-        break;
-    }
-}
-
-static int vfio_setup_msi(VFIODevice *vdev, int pos)
-{
-    uint16_t ctrl;
-    bool msi_64bit, msi_maskbit;
-    int ret, entries;
-
-    if (pread(vdev->fd, &ctrl, sizeof(ctrl),
-              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
-        return -errno;
-    }
-    ctrl = le16_to_cpu(ctrl);
-
-    msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
-    msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
-    entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
-
-    DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function, pos);
-
-    ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
-    if (ret < 0) {
-        if (ret == -ENOTSUP) {
-            return 0;
-        }
-        error_report("vfio: msi_init failed");
-        return ret;
-    }
-    vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
-
-    return 0;
-}
-
-/*
- * We don't have any control over how pci_add_capability() inserts
- * capabilities into the chain.  In order to setup MSI-X we need a
- * MemoryRegion for the BAR.  In order to setup the BAR and not
- * attempt to mmap the MSI-X table area, which VFIO won't allow, we
- * need to first look for where the MSI-X table lives.  So we
- * unfortunately split MSI-X setup across two functions.
- */
-static int vfio_early_setup_msix(VFIODevice *vdev)
-{
-    uint8_t pos;
-    uint16_t ctrl;
-    uint32_t table, pba;
-
-    pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
-    if (!pos) {
-        return 0;
-    }
-
-    if (pread(vdev->fd, &ctrl, sizeof(ctrl),
-              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
-        return -errno;
-    }
-
-    if (pread(vdev->fd, &table, sizeof(table),
-              vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
-        return -errno;
-    }
-
-    if (pread(vdev->fd, &pba, sizeof(pba),
-              vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
-        return -errno;
-    }
-
-    ctrl = le16_to_cpu(ctrl);
-    table = le32_to_cpu(table);
-    pba = le32_to_cpu(pba);
-
-    vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
-    vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
-    vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
-
-    DPRINTF("%04x:%02x:%02x.%x "
-            "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function, pos, vdev->msix->table_bar,
-            vdev->msix->table_offset, vdev->msix->entries);
-
-    return 0;
-}
-
-static int vfio_setup_msix(VFIODevice *vdev, int pos)
-{
-    int ret;
-
-    ret = msix_init(&vdev->pdev, vdev->msix->entries,
-                    &vdev->bars[vdev->msix->table_bar].mem,
-                    vdev->msix->table_bar, vdev->msix->table_offset,
-                    &vdev->bars[vdev->msix->pba_bar].mem,
-                    vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
-    if (ret < 0) {
-        if (ret == -ENOTSUP) {
-            return 0;
-        }
-        error_report("vfio: msix_init failed");
-        return ret;
-    }
-
-    return 0;
-}
-
-static void vfio_teardown_msi(VFIODevice *vdev)
-{
-    msi_uninit(&vdev->pdev);
-
-    if (vdev->msix) {
-        msix_uninit(&vdev->pdev, &vdev->bars[vdev->msix->table_bar].mem,
-                    &vdev->bars[vdev->msix->pba_bar].mem);
-    }
-}
-
-/*
- * Resource setup
- */
-static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled)
-{
-    int i;
-
-    for (i = 0; i < PCI_ROM_SLOT; i++) {
-        VFIOBAR *bar = &vdev->bars[i];
-
-        if (!bar->size) {
-            continue;
-        }
-
-        memory_region_set_enabled(&bar->mmap_mem, enabled);
-        if (vdev->msix && vdev->msix->table_bar == i) {
-            memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
-        }
-    }
-}
-
-static void vfio_unmap_bar(VFIODevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-
-    if (!bar->size) {
-        return;
-    }
-
-    vfio_bar_quirk_teardown(vdev, nr);
-
-    memory_region_del_subregion(&bar->mem, &bar->mmap_mem);
-    munmap(bar->mmap, memory_region_size(&bar->mmap_mem));
-
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        memory_region_del_subregion(&bar->mem, &vdev->msix->mmap_mem);
-        munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
-    }
-}
-
-static int vfio_mmap_bar(VFIODevice *vdev, VFIOBAR *bar,
-                         MemoryRegion *mem, MemoryRegion *submem,
-                         void **map, size_t size, off_t offset,
-                         const char *name)
-{
-    int ret = 0;
-
-    if (VFIO_ALLOW_MMAP && size && bar->flags & VFIO_REGION_INFO_FLAG_MMAP) {
-        int prot = 0;
-
-        if (bar->flags & VFIO_REGION_INFO_FLAG_READ) {
-            prot |= PROT_READ;
-        }
-
-        if (bar->flags & VFIO_REGION_INFO_FLAG_WRITE) {
-            prot |= PROT_WRITE;
-        }
-
-        *map = mmap(NULL, size, prot, MAP_SHARED,
-                    bar->fd, bar->fd_offset + offset);
-        if (*map == MAP_FAILED) {
-            *map = NULL;
-            ret = -errno;
-            goto empty_region;
-        }
-
-        memory_region_init_ram_ptr(submem, OBJECT(vdev), name, size, *map);
-        memory_region_set_skip_dump(submem);
-    } else {
-empty_region:
-        /* Create a zero sized sub-region to make cleanup easy. */
-        memory_region_init(submem, OBJECT(vdev), name, 0);
-    }
-
-    memory_region_add_subregion(mem, offset, submem);
-
-    return ret;
-}
-
-static void vfio_map_bar(VFIODevice *vdev, int nr)
-{
-    VFIOBAR *bar = &vdev->bars[nr];
-    unsigned size = bar->size;
-    char name[64];
-    uint32_t pci_bar;
-    uint8_t type;
-    int ret;
-
-    /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
-    if (!size) {
-        return;
-    }
-
-    snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function, nr);
-
-    /* Determine what type of BAR this is for registration */
-    ret = pread(vdev->fd, &pci_bar, sizeof(pci_bar),
-                vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
-    if (ret != sizeof(pci_bar)) {
-        error_report("vfio: Failed to read BAR %d (%m)", nr);
-        return;
-    }
-
-    pci_bar = le32_to_cpu(pci_bar);
-    bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
-    bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
-    type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
-                                    ~PCI_BASE_ADDRESS_MEM_MASK);
-
-    /* A "slow" read/write mapping underlies all BARs */
-    memory_region_init_io(&bar->mem, OBJECT(vdev), &vfio_bar_ops,
-                          bar, name, size);
-    pci_register_bar(&vdev->pdev, nr, type, &bar->mem);
-
-    /*
-     * We can't mmap areas overlapping the MSIX vector table, so we
-     * potentially insert a direct-mapped subregion before and after it.
-     */
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        size = vdev->msix->table_offset & qemu_host_page_mask;
-    }
-
-    strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
-    if (vfio_mmap_bar(vdev, bar, &bar->mem,
-                      &bar->mmap_mem, &bar->mmap, size, 0, name)) {
-        error_report("%s unsupported. Performance may be slow", name);
-    }
-
-    if (vdev->msix && vdev->msix->table_bar == nr) {
-        unsigned start;
-
-        start = HOST_PAGE_ALIGN(vdev->msix->table_offset +
-                                (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
-
-        size = start < bar->size ? bar->size - start : 0;
-        strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1);
-        /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
-        if (vfio_mmap_bar(vdev, bar, &bar->mem, &vdev->msix->mmap_mem,
-                          &vdev->msix->mmap, size, start, name)) {
-            error_report("%s unsupported. Performance may be slow", name);
-        }
-    }
-
-    vfio_bar_quirk_setup(vdev, nr);
-}
-
-static void vfio_map_bars(VFIODevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < PCI_ROM_SLOT; i++) {
-        vfio_map_bar(vdev, i);
-    }
-
-    if (vdev->has_vga) {
-        memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
-                              OBJECT(vdev), &vfio_vga_ops,
-                              &vdev->vga.region[QEMU_PCI_VGA_MEM],
-                              "vfio-vga-mmio@0xa0000",
-                              QEMU_PCI_VGA_MEM_SIZE);
-        memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
-                              OBJECT(vdev), &vfio_vga_ops,
-                              &vdev->vga.region[QEMU_PCI_VGA_IO_LO],
-                              "vfio-vga-io@0x3b0",
-                              QEMU_PCI_VGA_IO_LO_SIZE);
-        memory_region_init_io(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem,
-                              OBJECT(vdev), &vfio_vga_ops,
-                              &vdev->vga.region[QEMU_PCI_VGA_IO_HI],
-                              "vfio-vga-io@0x3c0",
-                              QEMU_PCI_VGA_IO_HI_SIZE);
-
-        pci_register_vga(&vdev->pdev, &vdev->vga.region[QEMU_PCI_VGA_MEM].mem,
-                         &vdev->vga.region[QEMU_PCI_VGA_IO_LO].mem,
-                         &vdev->vga.region[QEMU_PCI_VGA_IO_HI].mem);
-        vfio_vga_quirk_setup(vdev);
-    }
-}
-
-static void vfio_unmap_bars(VFIODevice *vdev)
-{
-    int i;
-
-    for (i = 0; i < PCI_ROM_SLOT; i++) {
-        vfio_unmap_bar(vdev, i);
-    }
-
-    if (vdev->has_vga) {
-        vfio_vga_quirk_teardown(vdev);
-        pci_unregister_vga(&vdev->pdev);
-    }
-}
-
-/*
- * General setup
- */
-static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
-{
-    uint8_t tmp, next = 0xff;
-
-    for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
-         tmp = pdev->config[tmp + 1]) {
-        if (tmp > pos && tmp < next) {
-            next = tmp;
-        }
-    }
-
-    return next - pos;
-}
-
-static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
-{
-    pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
-}
-
-static void vfio_add_emulated_word(VFIODevice *vdev, int pos,
-                                   uint16_t val, uint16_t mask)
-{
-    vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
-    vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
-    vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
-}
-
-static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
-{
-    pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
-}
-
-static void vfio_add_emulated_long(VFIODevice *vdev, int pos,
-                                   uint32_t val, uint32_t mask)
-{
-    vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
-    vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
-    vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
-}
-
-static int vfio_setup_pcie_cap(VFIODevice *vdev, int pos, uint8_t size)
-{
-    uint16_t flags;
-    uint8_t type;
-
-    flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
-    type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
-
-    if (type != PCI_EXP_TYPE_ENDPOINT &&
-        type != PCI_EXP_TYPE_LEG_END &&
-        type != PCI_EXP_TYPE_RC_END) {
-
-        error_report("vfio: Assignment of PCIe type 0x%x "
-                     "devices is not currently supported", type);
-        return -EINVAL;
-    }
-
-    if (!pci_bus_is_express(vdev->pdev.bus)) {
-        /*
-         * Use express capability as-is on PCI bus.  It doesn't make much
-         * sense to even expose, but some drivers (ex. tg3) depend on it
-         * and guests don't seem to be particular about it.  We'll need
-         * to revist this or force express devices to express buses if we
-         * ever expose an IOMMU to the guest.
-         */
-    } else if (pci_bus_is_root(vdev->pdev.bus)) {
-        /*
-         * On a Root Complex bus Endpoints become Root Complex Integrated
-         * Endpoints, which changes the type and clears the LNK & LNK2 fields.
-         */
-        if (type == PCI_EXP_TYPE_ENDPOINT) {
-            vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
-                                   PCI_EXP_TYPE_RC_END << 4,
-                                   PCI_EXP_FLAGS_TYPE);
-
-            /* Link Capabilities, Status, and Control goes away */
-            if (size > PCI_EXP_LNKCTL) {
-                vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
-                vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
-                vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
-
-#ifndef PCI_EXP_LNKCAP2
-#define PCI_EXP_LNKCAP2 44
-#endif
-#ifndef PCI_EXP_LNKSTA2
-#define PCI_EXP_LNKSTA2 50
-#endif
-                /* Link 2 Capabilities, Status, and Control goes away */
-                if (size > PCI_EXP_LNKCAP2) {
-                    vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
-                    vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
-                    vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
-                }
-            }
-
-        } else if (type == PCI_EXP_TYPE_LEG_END) {
-            /*
-             * Legacy endpoints don't belong on the root complex.  Windows
-             * seems to be happier with devices if we skip the capability.
-             */
-            return 0;
-        }
-
-    } else {
-        /*
-         * Convert Root Complex Integrated Endpoints to regular endpoints.
-         * These devices don't support LNK/LNK2 capabilities, so make them up.
-         */
-        if (type == PCI_EXP_TYPE_RC_END) {
-            vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
-                                   PCI_EXP_TYPE_ENDPOINT << 4,
-                                   PCI_EXP_FLAGS_TYPE);
-            vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
-                                   PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0);
-            vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
-        }
-
-        /* Mark the Link Status bits as emulated to allow virtual negotiation */
-        vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA,
-                               pci_get_word(vdev->pdev.config + pos +
-                                            PCI_EXP_LNKSTA),
-                               PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS);
-    }
-
-    pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size);
-    if (pos >= 0) {
-        vdev->pdev.exp.exp_cap = pos;
-    }
-
-    return pos;
-}
-
-static void vfio_check_pcie_flr(VFIODevice *vdev, uint8_t pos)
-{
-    uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
-
-    if (cap & PCI_EXP_DEVCAP_FLR) {
-        DPRINTF("%04x:%02x:%02x.%x Supports FLR via PCIe cap\n",
-                vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                vdev->host.function);
-        vdev->has_flr = true;
-    }
-}
-
-static void vfio_check_pm_reset(VFIODevice *vdev, uint8_t pos)
-{
-    uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
-
-    if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
-        DPRINTF("%04x:%02x:%02x.%x Supports PM reset\n",
-                vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                vdev->host.function);
-        vdev->has_pm_reset = true;
-    }
-}
-
-static void vfio_check_af_flr(VFIODevice *vdev, uint8_t pos)
-{
-    uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
-
-    if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
-        DPRINTF("%04x:%02x:%02x.%x Supports FLR via AF cap\n",
-                vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                vdev->host.function);
-        vdev->has_flr = true;
-    }
-}
-
-static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint8_t cap_id, next, size;
-    int ret;
-
-    cap_id = pdev->config[pos];
-    next = pdev->config[pos + 1];
-
-    /*
-     * If it becomes important to configure capabilities to their actual
-     * size, use this as the default when it's something we don't recognize.
-     * Since QEMU doesn't actually handle many of the config accesses,
-     * exact size doesn't seem worthwhile.
-     */
-    size = vfio_std_cap_max_size(pdev, pos);
-
-    /*
-     * pci_add_capability always inserts the new capability at the head
-     * of the chain.  Therefore to end up with a chain that matches the
-     * physical device, we insert from the end by making this recursive.
-     * This is also why we pre-caclulate size above as cached config space
-     * will be changed as we unwind the stack.
-     */
-    if (next) {
-        ret = vfio_add_std_cap(vdev, next);
-        if (ret) {
-            return ret;
-        }
-    } else {
-        /* Begin the rebuild, use QEMU emulated list bits */
-        pdev->config[PCI_CAPABILITY_LIST] = 0;
-        vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
-        vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
-    }
-
-    /* Use emulated next pointer to allow dropping caps */
-    pci_set_byte(vdev->emulated_config_bits + pos + 1, 0xff);
-
-    switch (cap_id) {
-    case PCI_CAP_ID_MSI:
-        ret = vfio_setup_msi(vdev, pos);
-        break;
-    case PCI_CAP_ID_EXP:
-        vfio_check_pcie_flr(vdev, pos);
-        ret = vfio_setup_pcie_cap(vdev, pos, size);
-        break;
-    case PCI_CAP_ID_MSIX:
-        ret = vfio_setup_msix(vdev, pos);
-        break;
-    case PCI_CAP_ID_PM:
-        vfio_check_pm_reset(vdev, pos);
-        vdev->pm_cap = pos;
-        ret = pci_add_capability(pdev, cap_id, pos, size);
-        break;
-    case PCI_CAP_ID_AF:
-        vfio_check_af_flr(vdev, pos);
-        ret = pci_add_capability(pdev, cap_id, pos, size);
-        break;
-    default:
-        ret = pci_add_capability(pdev, cap_id, pos, size);
-        break;
-    }
-
-    if (ret < 0) {
-        error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
-                     "0x%x[0x%x]@0x%x: %d", vdev->host.domain,
-                     vdev->host.bus, vdev->host.slot, vdev->host.function,
-                     cap_id, size, pos, ret);
-        return ret;
-    }
-
-    return 0;
-}
-
-static int vfio_add_capabilities(VFIODevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-
-    if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
-        !pdev->config[PCI_CAPABILITY_LIST]) {
-        return 0; /* Nothing to add */
-    }
-
-    return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
-}
-
-static void vfio_pci_pre_reset(VFIODevice *vdev)
-{
-    PCIDevice *pdev = &vdev->pdev;
-    uint16_t cmd;
-
-    vfio_disable_interrupts(vdev);
-
-    /* Make sure the device is in D0 */
-    if (vdev->pm_cap) {
-        uint16_t pmcsr;
-        uint8_t state;
-
-        pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
-        state = pmcsr & PCI_PM_CTRL_STATE_MASK;
-        if (state) {
-            pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
-            vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
-            /* vfio handles the necessary delay here */
-            pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
-            state = pmcsr & PCI_PM_CTRL_STATE_MASK;
-            if (state) {
-                error_report("vfio: Unable to power on device, stuck in D%d",
-                             state);
-            }
-        }
-    }
-
-    /*
-     * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
-     * Also put INTx Disable in known state.
-     */
-    cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
-    cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
-             PCI_COMMAND_INTX_DISABLE);
-    vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
-}
-
-static void vfio_pci_post_reset(VFIODevice *vdev)
-{
-    vfio_enable_intx(vdev);
-}
-
-static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
-                                PCIHostDeviceAddress *host2)
-{
-    return (host1->domain == host2->domain && host1->bus == host2->bus &&
-            host1->slot == host2->slot && host1->function == host2->function);
-}
-
-static int vfio_pci_hot_reset(VFIODevice *vdev, bool single)
-{
-    VFIOGroup *group;
-    struct vfio_pci_hot_reset_info *info;
-    struct vfio_pci_dependent_device *devices;
-    struct vfio_pci_hot_reset *reset;
-    int32_t *fds;
-    int ret, i, count;
-    bool multi = false;
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) %s\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function,
-            single ? "one" : "multi");
-
-    vfio_pci_pre_reset(vdev);
-    vdev->needs_reset = false;
-
-    info = g_malloc0(sizeof(*info));
-    info->argsz = sizeof(*info);
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
-    if (ret && errno != ENOSPC) {
-        ret = -errno;
-        if (!vdev->has_pm_reset) {
-            error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
-                         "no available reset mechanism.", vdev->host.domain,
-                         vdev->host.bus, vdev->host.slot, vdev->host.function);
-        }
-        goto out_single;
-    }
-
-    count = info->count;
-    info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
-    info->argsz = sizeof(*info) + (count * sizeof(*devices));
-    devices = &info->devices[0];
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
-    if (ret) {
-        ret = -errno;
-        error_report("vfio: hot reset info failed: %m");
-        goto out_single;
-    }
-
-    DPRINTF("%04x:%02x:%02x.%x: hot reset dependent devices:\n",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-
-    /* Verify that we have all the groups required */
-    for (i = 0; i < info->count; i++) {
-        PCIHostDeviceAddress host;
-        VFIODevice *tmp;
-
-        host.domain = devices[i].segment;
-        host.bus = devices[i].bus;
-        host.slot = PCI_SLOT(devices[i].devfn);
-        host.function = PCI_FUNC(devices[i].devfn);
-
-        DPRINTF("\t%04x:%02x:%02x.%x group %d\n", host.domain,
-                host.bus, host.slot, host.function, devices[i].group_id);
-
-        if (vfio_pci_host_match(&host, &vdev->host)) {
-            continue;
-        }
-
-        QLIST_FOREACH(group, &group_list, next) {
-            if (group->groupid == devices[i].group_id) {
-                break;
-            }
-        }
-
-        if (!group) {
-            if (!vdev->has_pm_reset) {
-                error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
-                             "depends on group %d which is not owned.",
-                             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                             vdev->host.function, devices[i].group_id);
-            }
-            ret = -EPERM;
-            goto out;
-        }
-
-        /* Prep dependent devices for reset and clear our marker. */
-        QLIST_FOREACH(tmp, &group->device_list, next) {
-            if (vfio_pci_host_match(&host, &tmp->host)) {
-                if (single) {
-                    DPRINTF("vfio: found another in-use device "
-                            "%04x:%02x:%02x.%x\n", host.domain, host.bus,
-                            host.slot, host.function);
-                    ret = -EINVAL;
-                    goto out_single;
-                }
-                vfio_pci_pre_reset(tmp);
-                tmp->needs_reset = false;
-                multi = true;
-                break;
-            }
-        }
-    }
-
-    if (!single && !multi) {
-        DPRINTF("vfio: No other in-use devices for multi hot reset\n");
-        ret = -EINVAL;
-        goto out_single;
-    }
-
-    /* Determine how many group fds need to be passed */
-    count = 0;
-    QLIST_FOREACH(group, &group_list, next) {
-        for (i = 0; i < info->count; i++) {
-            if (group->groupid == devices[i].group_id) {
-                count++;
-                break;
-            }
-        }
-    }
-
-    reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
-    reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
-    fds = &reset->group_fds[0];
-
-    /* Fill in group fds */
-    QLIST_FOREACH(group, &group_list, next) {
-        for (i = 0; i < info->count; i++) {
-            if (group->groupid == devices[i].group_id) {
-                fds[reset->count++] = group->fd;
-                break;
-            }
-        }
-    }
-
-    /* Bus reset! */
-    ret = ioctl(vdev->fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
-    g_free(reset);
-
-    DPRINTF("%04x:%02x:%02x.%x hot reset: %s\n", vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function,
-            ret ? "%m" : "Success");
-
-out:
-    /* Re-enable INTx on affected devices */
-    for (i = 0; i < info->count; i++) {
-        PCIHostDeviceAddress host;
-        VFIODevice *tmp;
-
-        host.domain = devices[i].segment;
-        host.bus = devices[i].bus;
-        host.slot = PCI_SLOT(devices[i].devfn);
-        host.function = PCI_FUNC(devices[i].devfn);
-
-        if (vfio_pci_host_match(&host, &vdev->host)) {
-            continue;
-        }
-
-        QLIST_FOREACH(group, &group_list, next) {
-            if (group->groupid == devices[i].group_id) {
-                break;
-            }
-        }
-
-        if (!group) {
-            break;
-        }
-
-        QLIST_FOREACH(tmp, &group->device_list, next) {
-            if (vfio_pci_host_match(&host, &tmp->host)) {
-                vfio_pci_post_reset(tmp);
-                break;
-            }
-        }
-    }
-out_single:
-    vfio_pci_post_reset(vdev);
-    g_free(info);
-
-    return ret;
-}
-
-/*
- * We want to differentiate hot reset of mulitple in-use devices vs hot reset
- * of a single in-use device.  VFIO_DEVICE_RESET will already handle the case
- * of doing hot resets when there is only a single device per bus.  The in-use
- * here refers to how many VFIODevices are affected.  A hot reset that affects
- * multiple devices, but only a single in-use device, means that we can call
- * it from our bus ->reset() callback since the extent is effectively a single
- * device.  This allows us to make use of it in the hotplug path.  When there
- * are multiple in-use devices, we can only trigger the hot reset during a
- * system reset and thus from our reset handler.  We separate _one vs _multi
- * here so that we don't overlap and do a double reset on the system reset
- * path where both our reset handler and ->reset() callback are used.  Calling
- * _one() will only do a hot reset for the one in-use devices case, calling
- * _multi() will do nothing if a _one() would have been sufficient.
- */
-static int vfio_pci_hot_reset_one(VFIODevice *vdev)
-{
-    return vfio_pci_hot_reset(vdev, true);
-}
-
-static int vfio_pci_hot_reset_multi(VFIODevice *vdev)
-{
-    return vfio_pci_hot_reset(vdev, false);
-}
-
-static void vfio_pci_reset_handler(void *opaque)
-{
-    VFIOGroup *group;
-    VFIODevice *vdev;
-
-    QLIST_FOREACH(group, &group_list, next) {
-        QLIST_FOREACH(vdev, &group->device_list, next) {
-            if (!vdev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
-                vdev->needs_reset = true;
-            }
-        }
-    }
-
-    QLIST_FOREACH(group, &group_list, next) {
-        QLIST_FOREACH(vdev, &group->device_list, next) {
-            if (vdev->needs_reset) {
-                vfio_pci_hot_reset_multi(vdev);
-            }
-        }
-    }
-}
-
-static void vfio_kvm_device_add_group(VFIOGroup *group)
-{
-#ifdef CONFIG_KVM
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_VFIO_GROUP,
-        .attr = KVM_DEV_VFIO_GROUP_ADD,
-        .addr = (uint64_t)(unsigned long)&group->fd,
-    };
-
-    if (!kvm_enabled()) {
-        return;
-    }
-
-    if (vfio_kvm_device_fd < 0) {
-        struct kvm_create_device cd = {
-            .type = KVM_DEV_TYPE_VFIO,
-        };
-
-        if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
-            DPRINTF("KVM_CREATE_DEVICE: %m\n");
-            return;
-        }
-
-        vfio_kvm_device_fd = cd.fd;
-    }
-
-    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
-        error_report("Failed to add group %d to KVM VFIO device: %m",
-                     group->groupid);
-    }
-#endif
-}
-
-static void vfio_kvm_device_del_group(VFIOGroup *group)
-{
-#ifdef CONFIG_KVM
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_VFIO_GROUP,
-        .attr = KVM_DEV_VFIO_GROUP_DEL,
-        .addr = (uint64_t)(unsigned long)&group->fd,
-    };
-
-    if (vfio_kvm_device_fd < 0) {
-        return;
-    }
-
-    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
-        error_report("Failed to remove group %d from KVM VFIO device: %m",
-                     group->groupid);
-    }
-#endif
-}
-
-static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
-{
-    VFIOAddressSpace *space;
-
-    QLIST_FOREACH(space, &vfio_address_spaces, list) {
-        if (space->as == as) {
-            return space;
-        }
-    }
-
-    /* No suitable VFIOAddressSpace, create a new one */
-    space = g_malloc0(sizeof(*space));
-    space->as = as;
-    QLIST_INIT(&space->containers);
-
-    QLIST_INSERT_HEAD(&vfio_address_spaces, space, list);
-
-    return space;
-}
-
-static void vfio_put_address_space(VFIOAddressSpace *space)
-{
-    if (QLIST_EMPTY(&space->containers)) {
-        QLIST_REMOVE(space, list);
-        g_free(space);
-    }
-}
-
-static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
-{
-    VFIOContainer *container;
-    int ret, fd;
-    VFIOAddressSpace *space;
-
-    space = vfio_get_address_space(as);
-
-    QLIST_FOREACH(container, &space->containers, next) {
-        if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
-            group->container = container;
-            QLIST_INSERT_HEAD(&container->group_list, group, container_next);
-            return 0;
-        }
-    }
-
-    fd = qemu_open("/dev/vfio/vfio", O_RDWR);
-    if (fd < 0) {
-        error_report("vfio: failed to open /dev/vfio/vfio: %m");
-        ret = -errno;
-        goto put_space_exit;
-    }
-
-    ret = ioctl(fd, VFIO_GET_API_VERSION);
-    if (ret != VFIO_API_VERSION) {
-        error_report("vfio: supported vfio version: %d, "
-                     "reported version: %d", VFIO_API_VERSION, ret);
-        ret = -EINVAL;
-        goto close_fd_exit;
-    }
-
-    container = g_malloc0(sizeof(*container));
-    container->space = space;
-    container->fd = fd;
-
-    if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
-        ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
-        if (ret) {
-            error_report("vfio: failed to set group container: %m");
-            ret = -errno;
-            goto free_container_exit;
-        }
-
-        ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
-        if (ret) {
-            error_report("vfio: failed to set iommu for container: %m");
-            ret = -errno;
-            goto free_container_exit;
-        }
-
-        container->iommu_data.type1.listener = vfio_memory_listener;
-        container->iommu_data.release = vfio_listener_release;
-
-        memory_listener_register(&container->iommu_data.type1.listener,
-                                 &address_space_memory);
-
-        if (container->iommu_data.type1.error) {
-            ret = container->iommu_data.type1.error;
-            error_report("vfio: memory listener initialization failed for container");
-            goto listener_release_exit;
-        }
-
-        container->iommu_data.type1.initialized = true;
-
-    } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU)) {
-        ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
-        if (ret) {
-            error_report("vfio: failed to set group container: %m");
-            ret = -errno;
-            goto free_container_exit;
-        }
-
-        ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_SPAPR_TCE_IOMMU);
-        if (ret) {
-            error_report("vfio: failed to set iommu for container: %m");
-            ret = -errno;
-            goto free_container_exit;
-        }
-
-        /*
-         * The host kernel code implementing VFIO_IOMMU_DISABLE is called
-         * when container fd is closed so we do not call it explicitly
-         * in this file.
-         */
-        ret = ioctl(fd, VFIO_IOMMU_ENABLE);
-        if (ret) {
-            error_report("vfio: failed to enable container: %m");
-            ret = -errno;
-            goto free_container_exit;
-        }
-
-        container->iommu_data.type1.listener = vfio_memory_listener;
-        container->iommu_data.release = vfio_listener_release;
-
-        memory_listener_register(&container->iommu_data.type1.listener,
-                                 container->space->as);
-
-    } else {
-        error_report("vfio: No available IOMMU models");
-        ret = -EINVAL;
-        goto free_container_exit;
-    }
-
-    QLIST_INIT(&container->group_list);
-    QLIST_INSERT_HEAD(&space->containers, container, next);
-
-    group->container = container;
-    QLIST_INSERT_HEAD(&container->group_list, group, container_next);
-
-    return 0;
-
-listener_release_exit:
-    vfio_listener_release(container);
-
-free_container_exit:
-    g_free(container);
-
-close_fd_exit:
-    close(fd);
-
-put_space_exit:
-    vfio_put_address_space(space);
-
-    return ret;
-}
-
-static void vfio_disconnect_container(VFIOGroup *group)
-{
-    VFIOContainer *container = group->container;
-
-    if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
-        error_report("vfio: error disconnecting group %d from container",
-                     group->groupid);
-    }
-
-    QLIST_REMOVE(group, container_next);
-    group->container = NULL;
-
-    if (QLIST_EMPTY(&container->group_list)) {
-        VFIOAddressSpace *space = container->space;
-
-        if (container->iommu_data.release) {
-            container->iommu_data.release(container);
-        }
-        QLIST_REMOVE(container, next);
-        DPRINTF("vfio_disconnect_container: close container->fd\n");
-        close(container->fd);
-        g_free(container);
-
-        vfio_put_address_space(space);
-    }
-}
-
-static VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
-{
-    VFIOGroup *group;
-    char path[32];
-    struct vfio_group_status status = { .argsz = sizeof(status) };
-
-    QLIST_FOREACH(group, &group_list, next) {
-        if (group->groupid == groupid) {
-            /* Found it.  Now is it already in the right context? */
-            if (group->container->space->as == as) {
-                return group;
-            } else {
-                error_report("vfio: group %d used in multiple address spaces",
-                             group->groupid);
-                return NULL;
-            }
-        }
-    }
-
-    group = g_malloc0(sizeof(*group));
-
-    snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
-    group->fd = qemu_open(path, O_RDWR);
-    if (group->fd < 0) {
-        error_report("vfio: error opening %s: %m", path);
-        goto free_group_exit;
-    }
-
-    if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
-        error_report("vfio: error getting group status: %m");
-        goto close_fd_exit;
-    }
-
-    if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
-        error_report("vfio: error, group %d is not viable, please ensure "
-                     "all devices within the iommu_group are bound to their "
-                     "vfio bus driver.", groupid);
-        goto close_fd_exit;
-    }
-
-    group->groupid = groupid;
-    QLIST_INIT(&group->device_list);
-
-    if (vfio_connect_container(group, as)) {
-        error_report("vfio: failed to setup container for group %d", groupid);
-        goto close_fd_exit;
-    }
-
-    if (QLIST_EMPTY(&group_list)) {
-        qemu_register_reset(vfio_pci_reset_handler, NULL);
-    }
-
-    QLIST_INSERT_HEAD(&group_list, group, next);
-
-    vfio_kvm_device_add_group(group);
-
-    return group;
-
-close_fd_exit:
-    close(group->fd);
-
-free_group_exit:
-    g_free(group);
-
-    return NULL;
-}
-
-static void vfio_put_group(VFIOGroup *group)
-{
-    if (!QLIST_EMPTY(&group->device_list)) {
-        return;
-    }
-
-    vfio_kvm_device_del_group(group);
-    vfio_disconnect_container(group);
-    QLIST_REMOVE(group, next);
-    DPRINTF("vfio_put_group: close group->fd\n");
-    close(group->fd);
-    g_free(group);
-
-    if (QLIST_EMPTY(&group_list)) {
-        qemu_unregister_reset(vfio_pci_reset_handler, NULL);
-    }
-}
-
-static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
-{
-    struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
-    struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
-    struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
-    int ret, i;
-
-    ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
-    if (ret < 0) {
-        error_report("vfio: error getting device %s from group %d: %m",
-                     name, group->groupid);
-        error_printf("Verify all devices in group %d are bound to vfio-pci "
-                     "or pci-stub and not already in use\n", group->groupid);
-        return ret;
-    }
-
-    vdev->fd = ret;
-    vdev->group = group;
-    QLIST_INSERT_HEAD(&group->device_list, vdev, next);
-
-    /* Sanity check device */
-    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &dev_info);
-    if (ret) {
-        error_report("vfio: error getting device info: %m");
-        goto error;
-    }
-
-    DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name,
-            dev_info.flags, dev_info.num_regions, dev_info.num_irqs);
-
-    if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PCI)) {
-        error_report("vfio: Um, this isn't a PCI device");
-        goto error;
-    }
-
-    vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
-
-    if (dev_info.num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
-        error_report("vfio: unexpected number of io regions %u",
-                     dev_info.num_regions);
-        goto error;
-    }
-
-    if (dev_info.num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
-        error_report("vfio: unexpected number of irqs %u", dev_info.num_irqs);
-        goto error;
-    }
-
-    for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
-        reg_info.index = i;
-
-        ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
-        if (ret) {
-            error_report("vfio: Error getting region %d info: %m", i);
-            goto error;
-        }
-
-        DPRINTF("Device %s region %d:\n", name, i);
-        DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
-                (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
-                (unsigned long)reg_info.flags);
-
-        vdev->bars[i].flags = reg_info.flags;
-        vdev->bars[i].size = reg_info.size;
-        vdev->bars[i].fd_offset = reg_info.offset;
-        vdev->bars[i].fd = vdev->fd;
-        vdev->bars[i].nr = i;
-        QLIST_INIT(&vdev->bars[i].quirks);
-    }
-
-    reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
-    if (ret) {
-        error_report("vfio: Error getting config info: %m");
-        goto error;
-    }
-
-    DPRINTF("Device %s config:\n", name);
-    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
-            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
-            (unsigned long)reg_info.flags);
-
-    vdev->config_size = reg_info.size;
-    if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
-        vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
-    }
-    vdev->config_offset = reg_info.offset;
-
-    if ((vdev->features & VFIO_FEATURE_ENABLE_VGA) &&
-        dev_info.num_regions > VFIO_PCI_VGA_REGION_INDEX) {
-        struct vfio_region_info vga_info = {
-            .argsz = sizeof(vga_info),
-            .index = VFIO_PCI_VGA_REGION_INDEX,
-         };
-
-        ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &vga_info);
-        if (ret) {
-            error_report(
-                "vfio: Device does not support requested feature x-vga");
-            goto error;
-        }
-
-        if (!(vga_info.flags & VFIO_REGION_INFO_FLAG_READ) ||
-            !(vga_info.flags & VFIO_REGION_INFO_FLAG_WRITE) ||
-            vga_info.size < 0xbffff + 1) {
-            error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
-                         (unsigned long)vga_info.flags,
-                         (unsigned long)vga_info.size);
-            goto error;
-        }
-
-        vdev->vga.fd_offset = vga_info.offset;
-        vdev->vga.fd = vdev->fd;
-
-        vdev->vga.region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
-        vdev->vga.region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
-        QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_MEM].quirks);
-
-        vdev->vga.region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
-        vdev->vga.region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
-        QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_LO].quirks);
-
-        vdev->vga.region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
-        vdev->vga.region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
-        QLIST_INIT(&vdev->vga.region[QEMU_PCI_VGA_IO_HI].quirks);
-
-        vdev->has_vga = true;
-    }
-    irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
-    if (ret) {
-        /* This can fail for an old kernel or legacy PCI dev */
-        DPRINTF("VFIO_DEVICE_GET_IRQ_INFO failure: %m\n");
-        ret = 0;
-    } else if (irq_info.count == 1) {
-        vdev->pci_aer = true;
-    } else {
-        error_report("vfio: %04x:%02x:%02x.%x "
-                     "Could not enable error recovery for the device",
-                     vdev->host.domain, vdev->host.bus, vdev->host.slot,
-                     vdev->host.function);
-    }
-
-error:
-    if (ret) {
-        QLIST_REMOVE(vdev, next);
-        vdev->group = NULL;
-        close(vdev->fd);
-    }
-    return ret;
-}
-
-static void vfio_put_device(VFIODevice *vdev)
-{
-    QLIST_REMOVE(vdev, next);
-    vdev->group = NULL;
-    DPRINTF("vfio_put_device: close vdev->fd\n");
-    close(vdev->fd);
-    if (vdev->msix) {
-        g_free(vdev->msix);
-        vdev->msix = NULL;
-    }
-}
-
-static void vfio_err_notifier_handler(void *opaque)
-{
-    VFIODevice *vdev = opaque;
-
-    if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
-        return;
-    }
-
-    /*
-     * TBD. Retrieve the error details and decide what action
-     * needs to be taken. One of the actions could be to pass
-     * the error to the guest and have the guest driver recover
-     * from the error. This requires that PCIe capabilities be
-     * exposed to the guest. For now, we just terminate the
-     * guest to contain the error.
-     */
-
-    error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected.  "
-                 "Please collect any data possible and then kill the guest",
-                 __func__, vdev->host.domain, vdev->host.bus,
-                 vdev->host.slot, vdev->host.function);
-
-    vm_stop(RUN_STATE_INTERNAL_ERROR);
-}
-
-/*
- * Registers error notifier for devices supporting error recovery.
- * If we encounter a failure in this function, we report an error
- * and continue after disabling error recovery support for the
- * device.
- */
-static void vfio_register_err_notifier(VFIODevice *vdev)
-{
-    int ret;
-    int argsz;
-    struct vfio_irq_set *irq_set;
-    int32_t *pfd;
-
-    if (!vdev->pci_aer) {
-        return;
-    }
-
-    if (event_notifier_init(&vdev->err_notifier, 0)) {
-        error_report("vfio: Unable to init event notifier for error detection");
-        vdev->pci_aer = false;
-        return;
-    }
-
-    argsz = sizeof(*irq_set) + sizeof(*pfd);
-
-    irq_set = g_malloc0(argsz);
-    irq_set->argsz = argsz;
-    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
-                     VFIO_IRQ_SET_ACTION_TRIGGER;
-    irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
-    irq_set->start = 0;
-    irq_set->count = 1;
-    pfd = (int32_t *)&irq_set->data;
-
-    *pfd = event_notifier_get_fd(&vdev->err_notifier);
-    qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev);
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-    if (ret) {
-        error_report("vfio: Failed to set up error notification");
-        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
-        event_notifier_cleanup(&vdev->err_notifier);
-        vdev->pci_aer = false;
-    }
-    g_free(irq_set);
-}
-
-static void vfio_unregister_err_notifier(VFIODevice *vdev)
-{
-    int argsz;
-    struct vfio_irq_set *irq_set;
-    int32_t *pfd;
-    int ret;
-
-    if (!vdev->pci_aer) {
-        return;
-    }
-
-    argsz = sizeof(*irq_set) + sizeof(*pfd);
-
-    irq_set = g_malloc0(argsz);
-    irq_set->argsz = argsz;
-    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
-                     VFIO_IRQ_SET_ACTION_TRIGGER;
-    irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
-    irq_set->start = 0;
-    irq_set->count = 1;
-    pfd = (int32_t *)&irq_set->data;
-    *pfd = -1;
-
-    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
-    if (ret) {
-        error_report("vfio: Failed to de-assign error fd: %m");
-    }
-    g_free(irq_set);
-    qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
-                        NULL, NULL, vdev);
-    event_notifier_cleanup(&vdev->err_notifier);
-}
-
-static int vfio_initfn(PCIDevice *pdev)
-{
-    VFIODevice *pvdev, *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    VFIOGroup *group;
-    char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
-    ssize_t len;
-    struct stat st;
-    int groupid;
-    int ret;
-
-    /* Check that the host device exists */
-    snprintf(path, sizeof(path),
-             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
-             vdev->host.domain, vdev->host.bus, vdev->host.slot,
-             vdev->host.function);
-#ifdef CONFIG_ANDROID
-    if (android_stat(path, &st) < 0) {
-#else  // !CONFIG_ANDROID
-    if (stat(path, &st) < 0) {
-#endif  // CONFIG_ANDROID
-        error_report("vfio: error: no such host device: %s", path);
-        return -errno;
-    }
-
-    strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1);
-
-    len = readlink(path, iommu_group_path, sizeof(path));
-    if (len <= 0 || len >= sizeof(path)) {
-        error_report("vfio: error no iommu_group for device");
-        return len < 0 ? -errno : ENAMETOOLONG;
-    }
-
-    iommu_group_path[len] = 0;
-    group_name = basename(iommu_group_path);
-
-    if (sscanf(group_name, "%d", &groupid) != 1) {
-        error_report("vfio: error reading %s: %m", path);
-        return -errno;
-    }
-
-    DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function, groupid);
-
-    group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev));
-    if (!group) {
-        error_report("vfio: failed to get group %d", groupid);
-        return -ENOENT;
-    }
-
-    snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x",
-            vdev->host.domain, vdev->host.bus, vdev->host.slot,
-            vdev->host.function);
-
-    QLIST_FOREACH(pvdev, &group->device_list, next) {
-        if (pvdev->host.domain == vdev->host.domain &&
-            pvdev->host.bus == vdev->host.bus &&
-            pvdev->host.slot == vdev->host.slot &&
-            pvdev->host.function == vdev->host.function) {
-
-            error_report("vfio: error: device %s is already attached", path);
-            vfio_put_group(group);
-            return -EBUSY;
-        }
-    }
-
-    ret = vfio_get_device(group, path, vdev);
-    if (ret) {
-        error_report("vfio: failed to get device %s", path);
-        vfio_put_group(group);
-        return ret;
-    }
-
-    /* Get a copy of config space */
-    ret = pread(vdev->fd, vdev->pdev.config,
-                MIN(pci_config_size(&vdev->pdev), vdev->config_size),
-                vdev->config_offset);
-    if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
-        ret = ret < 0 ? -errno : -EFAULT;
-        error_report("vfio: Failed to read device config space");
-        goto out_put;
-    }
-
-    /* vfio emulates a lot for us, but some bits need extra love */
-    vdev->emulated_config_bits = g_malloc0(vdev->config_size);
-
-    /* QEMU can choose to expose the ROM or not */
-    memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
-
-    /* QEMU can change multi-function devices to single function, or reverse */
-    vdev->emulated_config_bits[PCI_HEADER_TYPE] =
-                                              PCI_HEADER_TYPE_MULTI_FUNCTION;
-
-    /* Restore or clear multifunction, this is always controlled by QEMU */
-    if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
-        vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
-    } else {
-        vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
-    }
-
-    /*
-     * Clear host resource mapping info.  If we choose not to register a
-     * BAR, such as might be the case with the option ROM, we can get
-     * confusing, unwritable, residual addresses from the host here.
-     */
-    memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
-    memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
-
-    vfio_pci_size_rom(vdev);
-
-    ret = vfio_early_setup_msix(vdev);
-    if (ret) {
-        goto out_put;
-    }
-
-    vfio_map_bars(vdev);
-
-    ret = vfio_add_capabilities(vdev);
-    if (ret) {
-        goto out_teardown;
-    }
-
-    /* QEMU emulates all of MSI & MSIX */
-    if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
-        memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
-               MSIX_CAP_LENGTH);
-    }
-
-    if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
-        memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
-               vdev->msi_cap_size);
-    }
-
-    if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
-        vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
-                                                  vfio_intx_mmap_enable, vdev);
-        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
-        ret = vfio_enable_intx(vdev);
-        if (ret) {
-            goto out_teardown;
-        }
-    }
-
-    vfio_register_err_notifier(vdev);
-
-    return 0;
-
-out_teardown:
-    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
-    vfio_teardown_msi(vdev);
-    vfio_unmap_bars(vdev);
-out_put:
-    g_free(vdev->emulated_config_bits);
-    vfio_put_device(vdev);
-    vfio_put_group(group);
-    return ret;
-}
-
-static void vfio_exitfn(PCIDevice *pdev)
-{
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-    VFIOGroup *group = vdev->group;
-
-    vfio_unregister_err_notifier(vdev);
-    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
-    vfio_disable_interrupts(vdev);
-    if (vdev->intx.mmap_timer) {
-        timer_free(vdev->intx.mmap_timer);
-    }
-    vfio_teardown_msi(vdev);
-    vfio_unmap_bars(vdev);
-    g_free(vdev->emulated_config_bits);
-    g_free(vdev->rom);
-    vfio_put_device(vdev);
-    vfio_put_group(group);
-}
-
-static void vfio_pci_reset(DeviceState *dev)
-{
-    PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
-
-    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-
-    vfio_pci_pre_reset(vdev);
-
-    if (vdev->reset_works && (vdev->has_flr || !vdev->has_pm_reset) &&
-        !ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
-        DPRINTF("%04x:%02x:%02x.%x FLR/VFIO_DEVICE_RESET\n", vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-        goto post_reset;
-    }
-
-    /* See if we can do our own bus reset */
-    if (!vfio_pci_hot_reset_one(vdev)) {
-        goto post_reset;
-    }
-
-    /* If nothing else works and the device supports PM reset, use it */
-    if (vdev->reset_works && vdev->has_pm_reset &&
-        !ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
-        DPRINTF("%04x:%02x:%02x.%x PCI PM Reset\n", vdev->host.domain,
-            vdev->host.bus, vdev->host.slot, vdev->host.function);
-        goto post_reset;
-    }
-
-post_reset:
-    vfio_pci_post_reset(vdev);
-}
-
-static void vfio_instance_init(Object *obj)
-{
-    PCIDevice *pci_dev = PCI_DEVICE(obj);
-    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, PCI_DEVICE(obj));
-
-    device_add_bootindex_property(obj, &vdev->bootindex,
-                                  "bootindex", NULL,
-                                  &pci_dev->qdev, NULL);
-}
-
-static Property vfio_pci_dev_properties[] = {
-    DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice, host),
-    DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice,
-                       intx.mmap_timeout, 1100),
-    DEFINE_PROP_BIT("x-vga", VFIODevice, features,
-                    VFIO_FEATURE_ENABLE_VGA_BIT, false),
-    /*
-     * TODO - support passed fds... is this necessary?
-     * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
-     * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
-     */
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static const VMStateDescription vfio_pci_vmstate = {
-    .name = "vfio-pci",
-    .unmigratable = 1,
-};
-
-static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
-
-    dc->reset = vfio_pci_reset;
-    dc->props = vfio_pci_dev_properties;
-    dc->vmsd = &vfio_pci_vmstate;
-    dc->desc = "VFIO-based PCI device assignment";
-    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-    pdc->init = vfio_initfn;
-    pdc->exit = vfio_exitfn;
-    pdc->config_read = vfio_pci_read_config;
-    pdc->config_write = vfio_pci_write_config;
-    pdc->is_express = 1; /* We might be */
-}
-
-static const TypeInfo vfio_pci_dev_info = {
-    .name = "vfio-pci",
-    .parent = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(VFIODevice),
-    .class_init = vfio_pci_dev_class_init,
-    .instance_init = vfio_instance_init,
-};
-
-static void register_vfio_pci_dev_type(void)
-{
-    type_register_static(&vfio_pci_dev_info);
-}
-
-type_init(register_vfio_pci_dev_type)
-
-static int vfio_container_do_ioctl(AddressSpace *as, int32_t groupid,
-                                   int req, void *param)
-{
-    VFIOGroup *group;
-    VFIOContainer *container;
-    int ret = -1;
-
-    group = vfio_get_group(groupid, as);
-    if (!group) {
-        error_report("vfio: group %d not registered", groupid);
-        return ret;
-    }
-
-    container = group->container;
-    if (group->container) {
-        ret = ioctl(container->fd, req, param);
-        if (ret < 0) {
-            error_report("vfio: failed to ioctl container: ret=%d, %s",
-                         ret, strerror(errno));
-        }
-    }
-
-    vfio_put_group(group);
-
-    return ret;
-}
-
-int vfio_container_ioctl(AddressSpace *as, int32_t groupid,
-                         int req, void *param)
-{
-    /* We allow only certain ioctls to the container */
-    switch (req) {
-    case VFIO_CHECK_EXTENSION:
-    case VFIO_IOMMU_SPAPR_TCE_GET_INFO:
-        break;
-    default:
-        /* Return an error on unknown requests */
-        error_report("vfio: unsupported ioctl %X", req);
-        return -1;
-    }
-
-    return vfio_container_do_ioctl(as, groupid, req, param);
-}
diff --git a/hw/misc/vmport.c b/hw/misc/vmport.c
index cd5716a..c763811 100644
--- a/hw/misc/vmport.c
+++ b/hw/misc/vmport.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/isa/isa.h"
 #include "hw/i386/pc.h"
@@ -35,7 +36,6 @@
 #define VMPORT_ENTRIES 0x2c
 #define VMPORT_MAGIC   0x564D5868
 
-#define TYPE_VMPORT "vmport"
 #define VMPORT(obj) OBJECT_CHECK(VMPortState, (obj), TYPE_VMPORT)
 
 typedef struct VMPortState
diff --git a/hw/misc/zynq-xadc.c b/hw/misc/zynq-xadc.c
new file mode 100644
index 0000000..1490610
--- /dev/null
+++ b/hw/misc/zynq-xadc.c
@@ -0,0 +1,304 @@
+/*
+ * ADC registers for Xilinx Zynq Platform
+ *
+ * Copyright (c) 2015 Guenter Roeck
+ * Based on hw/misc/zynq_slcr.c, written by Michal Simek
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/misc/zynq-xadc.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
+
+enum {
+    CFG                = 0x000 / 4,
+    INT_STS,
+    INT_MASK,
+    MSTS,
+    CMDFIFO,
+    RDFIFO,
+    MCTL,
+};
+
+#define CFG_ENABLE              BIT(31)
+#define CFG_CFIFOTH_SHIFT       20
+#define CFG_CFIFOTH_LENGTH      4
+#define CFG_DFIFOTH_SHIFT       16
+#define CFG_DFIFOTH_LENGTH      4
+#define CFG_WEDGE               BIT(13)
+#define CFG_REDGE               BIT(12)
+#define CFG_TCKRATE_SHIFT       8
+#define CFG_TCKRATE_LENGTH      2
+
+#define CFG_TCKRATE_DIV(x)      (0x1 << (x - 1))
+
+#define CFG_IGAP_SHIFT          0
+#define CFG_IGAP_LENGTH         5
+
+#define INT_CFIFO_LTH           BIT(9)
+#define INT_DFIFO_GTH           BIT(8)
+#define INT_OT                  BIT(7)
+#define INT_ALM_SHIFT           0
+#define INT_ALM_LENGTH          7
+#define INT_ALM_MASK            (((1 << INT_ALM_LENGTH) - 1) << INT_ALM_SHIFT)
+
+#define INT_ALL (INT_CFIFO_LTH | INT_DFIFO_GTH | INT_OT | INT_ALM_MASK)
+
+#define MSTS_CFIFO_LVL_SHIFT    16
+#define MSTS_CFIFO_LVL_LENGTH   4
+#define MSTS_DFIFO_LVL_SHIFT    12
+#define MSTS_DFIFO_LVL_LENGTH   4
+#define MSTS_CFIFOF             BIT(11)
+#define MSTS_CFIFOE             BIT(10)
+#define MSTS_DFIFOF             BIT(9)
+#define MSTS_DFIFOE             BIT(8)
+#define MSTS_OT                 BIT(7)
+#define MSTS_ALM_SHIFT          0
+#define MSTS_ALM_LENGTH         7
+
+#define MCTL_RESET              BIT(4)
+
+#define CMD_NOP                 0x00
+#define CMD_READ                0x01
+#define CMD_WRITE               0x02
+
+static void zynq_xadc_update_ints(ZynqXADCState *s)
+{
+
+    /* We are fast, commands are actioned instantly so the CFIFO is always
+     * empty (and below threshold).
+     */
+    s->regs[INT_STS] |= INT_CFIFO_LTH;
+
+    if (s->xadc_dfifo_entries >
+        extract32(s->regs[CFG], CFG_DFIFOTH_SHIFT, CFG_DFIFOTH_LENGTH)) {
+        s->regs[INT_STS] |= INT_DFIFO_GTH;
+    }
+
+    qemu_set_irq(s->qemu_irq, !!(s->regs[INT_STS] & ~s->regs[INT_MASK]));
+}
+
+static void zynq_xadc_reset(DeviceState *d)
+{
+    ZynqXADCState *s = ZYNQ_XADC(d);
+
+    s->regs[CFG] = 0x14 << CFG_IGAP_SHIFT |
+                   CFG_TCKRATE_DIV(4) << CFG_TCKRATE_SHIFT | CFG_REDGE;
+    s->regs[INT_STS] = INT_CFIFO_LTH;
+    s->regs[INT_MASK] = 0xffffffff;
+    s->regs[CMDFIFO] = 0;
+    s->regs[RDFIFO] = 0;
+    s->regs[MCTL] = MCTL_RESET;
+
+    memset(s->xadc_regs, 0, sizeof(s->xadc_regs));
+    memset(s->xadc_dfifo, 0, sizeof(s->xadc_dfifo));
+    s->xadc_dfifo_entries = 0;
+
+    zynq_xadc_update_ints(s);
+}
+
+static uint16_t xadc_pop_dfifo(ZynqXADCState *s)
+{
+    uint16_t rv = s->xadc_dfifo[0];
+    int i;
+
+    if (s->xadc_dfifo_entries > 0) {
+        s->xadc_dfifo_entries--;
+    }
+    for (i = 0; i < s->xadc_dfifo_entries; i++) {
+        s->xadc_dfifo[i] = s->xadc_dfifo[i + 1];
+    }
+    s->xadc_dfifo[s->xadc_dfifo_entries] = 0;
+    zynq_xadc_update_ints(s);
+    return rv;
+}
+
+static void xadc_push_dfifo(ZynqXADCState *s, uint16_t regval)
+{
+    if (s->xadc_dfifo_entries < ZYNQ_XADC_FIFO_DEPTH) {
+        s->xadc_dfifo[s->xadc_dfifo_entries++] = s->xadc_read_reg_previous;
+    }
+    s->xadc_read_reg_previous = regval;
+    zynq_xadc_update_ints(s);
+}
+
+static bool zynq_xadc_check_offset(hwaddr offset, bool rnw)
+{
+    switch (offset) {
+    case CFG:
+    case INT_MASK:
+    case INT_STS:
+    case MCTL:
+        return true;
+    case RDFIFO:
+    case MSTS:
+        return rnw;     /* read only */
+    case CMDFIFO:
+        return !rnw;    /* write only */
+    default:
+        return false;
+    }
+}
+
+static uint64_t zynq_xadc_read(void *opaque, hwaddr offset, unsigned size)
+{
+    ZynqXADCState *s = opaque;
+    int reg = offset / 4;
+    uint32_t rv = 0;
+
+    if (!zynq_xadc_check_offset(reg, true)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Invalid read access to "
+                      "addr %" HWADDR_PRIx "\n", offset);
+        return 0;
+    }
+
+    switch (reg) {
+    case CFG:
+    case INT_MASK:
+    case INT_STS:
+    case MCTL:
+        rv = s->regs[reg];
+        break;
+    case MSTS:
+        rv = MSTS_CFIFOE;
+        rv |= s->xadc_dfifo_entries << MSTS_DFIFO_LVL_SHIFT;
+        if (!s->xadc_dfifo_entries) {
+            rv |= MSTS_DFIFOE;
+        } else if (s->xadc_dfifo_entries == ZYNQ_XADC_FIFO_DEPTH) {
+            rv |= MSTS_DFIFOF;
+        }
+        break;
+    case RDFIFO:
+        rv = xadc_pop_dfifo(s);
+        break;
+    }
+    return rv;
+}
+
+static void zynq_xadc_write(void *opaque, hwaddr offset, uint64_t val,
+                            unsigned size)
+{
+    ZynqXADCState *s = (ZynqXADCState *)opaque;
+    int reg = offset / 4;
+    int xadc_reg;
+    int xadc_cmd;
+    int xadc_data;
+
+    if (!zynq_xadc_check_offset(reg, false)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Invalid write access "
+                      "to addr %" HWADDR_PRIx "\n", offset);
+        return;
+    }
+
+    switch (reg) {
+    case CFG:
+        s->regs[CFG] = val;
+        break;
+    case INT_STS:
+        s->regs[INT_STS] &= ~val;
+        break;
+    case INT_MASK:
+        s->regs[INT_MASK] = val & INT_ALL;
+        break;
+    case CMDFIFO:
+        xadc_cmd  = extract32(val, 26,  4);
+        xadc_reg  = extract32(val, 16, 10);
+        xadc_data = extract32(val,  0, 16);
+
+        if (s->regs[MCTL] & MCTL_RESET) {
+            qemu_log_mask(LOG_GUEST_ERROR, "zynq_xadc: Sending command "
+                          "while comm channel held in reset: %" PRIx32 "\n",
+                          (uint32_t) val);
+            break;
+        }
+
+        if (xadc_reg >= ZYNQ_XADC_NUM_ADC_REGS && xadc_cmd != CMD_NOP) {
+            qemu_log_mask(LOG_GUEST_ERROR, "read/write op to invalid xadc "
+                          "reg 0x%x\n", xadc_reg);
+            break;
+        }
+
+        switch (xadc_cmd) {
+        case CMD_READ:
+            xadc_push_dfifo(s, s->xadc_regs[xadc_reg]);
+            break;
+        case CMD_WRITE:
+            s->xadc_regs[xadc_reg] = xadc_data;
+            /* fallthrough */
+        case CMD_NOP:
+            xadc_push_dfifo(s, 0);
+            break;
+        }
+        break;
+    case MCTL:
+        s->regs[MCTL] = val & 0x00fffeff;
+        break;
+    }
+    zynq_xadc_update_ints(s);
+}
+
+static const MemoryRegionOps xadc_ops = {
+    .read = zynq_xadc_read,
+    .write = zynq_xadc_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void zynq_xadc_init(Object *obj)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    ZynqXADCState *s = ZYNQ_XADC(obj);
+
+    memory_region_init_io(&s->iomem, obj, &xadc_ops, s, "zynq-xadc",
+                          ZYNQ_XADC_MMIO_SIZE);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->qemu_irq);
+}
+
+static const VMStateDescription vmstate_zynq_xadc = {
+    .name = "zynq-xadc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, ZynqXADCState, ZYNQ_XADC_NUM_IO_REGS),
+        VMSTATE_UINT16_ARRAY(xadc_regs, ZynqXADCState,
+                             ZYNQ_XADC_NUM_ADC_REGS),
+        VMSTATE_UINT16_ARRAY(xadc_dfifo, ZynqXADCState,
+                             ZYNQ_XADC_FIFO_DEPTH),
+        VMSTATE_UINT16(xadc_read_reg_previous, ZynqXADCState),
+        VMSTATE_UINT16(xadc_dfifo_entries, ZynqXADCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void zynq_xadc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_zynq_xadc;
+    dc->reset = zynq_xadc_reset;
+}
+
+static const TypeInfo zynq_xadc_info = {
+    .class_init = zynq_xadc_class_init,
+    .name  = TYPE_ZYNQ_XADC,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(ZynqXADCState),
+    .instance_init = zynq_xadc_init,
+};
+
+static void zynq_xadc_register_types(void)
+{
+    type_register_static(&zynq_xadc_info);
+}
+
+type_init(zynq_xadc_register_types)
diff --git a/hw/misc/zynq_slcr.c b/hw/misc/zynq_slcr.c
index 964f253..7891219 100644
--- a/hw/misc/zynq_slcr.c
+++ b/hw/misc/zynq_slcr.c
@@ -14,10 +14,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
+#include "qemu/log.h"
 
 #ifndef ZYNQ_SLCR_ERR_DEBUG
 #define ZYNQ_SLCR_ERR_DEBUG 0
@@ -393,12 +395,12 @@
         return;
     }
 
-    if (!s->regs[LOCKSTA]) {
-        s->regs[offset / 4] = val;
-    } else {
-        DB_PRINT("SCLR registers are locked. Unlock them first\n");
+    if (s->regs[LOCKSTA]) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "SCLR registers are locked. Unlock them first\n");
         return;
     }
+    s->regs[offset] = val;
 
     switch (offset) {
     case PSS_RST_CTRL:
diff --git a/hw/moxie/moxiesim.c b/hw/moxie/moxiesim.c
index 80bcc5b..3069834 100644
--- a/hw/moxie/moxiesim.c
+++ b/hw/moxie/moxiesim.c
@@ -24,6 +24,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
@@ -34,6 +38,7 @@
 #include "hw/loader.h"
 #include "hw/char/serial.h"
 #include "exec/address-spaces.h"
+#include "elf.h"
 
 #define PHYS_MEM_BASE 0x80000000
 
@@ -52,8 +57,8 @@
     ram_addr_t initrd_offset;
 
     kernel_size = load_elf(loader_params->kernel_filename,  NULL, NULL,
-                           &entry, &kernel_low, &kernel_high, 1,
-                           ELF_MACHINE, 0);
+                           &entry, &kernel_low, &kernel_high, 1, EM_MOXIE,
+                           0, 0);
 
     if (kernel_size <= 0) {
         fprintf(stderr, "qemu: could not load kernel '%s'\n",
@@ -123,11 +128,11 @@
     qemu_register_reset(main_cpu_reset, cpu);
 
     /* Allocate RAM. */
-    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "moxiesim.ram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space_mem, ram_base, ram);
 
-    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_abort);
+    memory_region_init_ram(rom, NULL, "moxie.rom", 128*0x1000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0x1000, rom);
 
@@ -146,16 +151,11 @@
     }
 }
 
-static QEMUMachine moxiesim_machine = {
-    .name = "moxiesim",
-    .desc = "Moxie simulator platform",
-    .init = moxiesim_init,
-    .is_default = 1,
-};
-
-static void moxie_machine_init(void)
+static void moxiesim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&moxiesim_machine);
+    mc->desc = "Moxie simulator platform";
+    mc->init = moxiesim_init;
+    mc->is_default = 1;
 }
 
-machine_init(moxie_machine_init)
+DEFINE_MACHINE("moxiesim", moxiesim_machine_init)
diff --git a/hw/net/Makefile.objs b/hw/net/Makefile.objs
index ea93293..610ed3e 100644
--- a/hw/net/Makefile.objs
+++ b/hw/net/Makefile.objs
@@ -6,9 +6,11 @@
 common-obj-$(CONFIG_EEPRO100_PCI) += eepro100.o
 common-obj-$(CONFIG_PCNET_PCI) += pcnet-pci.o
 common-obj-$(CONFIG_PCNET_COMMON) += pcnet.o
-common-obj-$(CONFIG_E1000_PCI) += e1000.o
+common-obj-$(CONFIG_E1000_PCI) += e1000.o e1000x_common.o
+common-obj-$(CONFIG_E1000E_PCI) += net_tx_pkt.o net_rx_pkt.o
+common-obj-$(CONFIG_E1000E_PCI) += e1000e.o e1000e_core.o e1000x_common.o
 common-obj-$(CONFIG_RTL8139_PCI) += rtl8139.o
-common-obj-$(CONFIG_VMXNET3_PCI) += vmxnet_tx_pkt.o vmxnet_rx_pkt.o
+common-obj-$(CONFIG_VMXNET3_PCI) += net_tx_pkt.o net_rx_pkt.o
 common-obj-$(CONFIG_VMXNET3_PCI) += vmxnet3.o
 
 common-obj-$(CONFIG_SMC91C111) += smc91c111.o
@@ -19,6 +21,7 @@
 common-obj-$(CONFIG_MIPSNET) += mipsnet.o
 common-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
 common-obj-$(CONFIG_ALLWINNER_EMAC) += allwinner_emac.o
+common-obj-$(CONFIG_IMX_FEC) += imx_fec.o
 
 common-obj-$(CONFIG_CADENCE) += cadence_gem.o
 common-obj-$(CONFIG_STELLARIS_ENET) += stellaris_enet.o
@@ -35,3 +38,8 @@
 
 obj-$(CONFIG_ETSEC) += fsl_etsec/etsec.o fsl_etsec/registers.o \
 			fsl_etsec/rings.o fsl_etsec/miim.o
+
+common-obj-$(CONFIG_ROCKER) += rocker/rocker.o rocker/rocker_fp.o \
+                               rocker/rocker_desc.o rocker/rocker_world.o \
+                               rocker/rocker_of_dpa.o
+obj-$(call lnot,$(CONFIG_ROCKER)) += rocker/qmp-norocker.o
diff --git a/hw/net/allwinner_emac.c b/hw/net/allwinner_emac.c
index d780ba0..50e8361 100644
--- a/hw/net/allwinner_emac.c
+++ b/hw/net/allwinner_emac.c
@@ -16,10 +16,12 @@
  * GNU General Public License for more details.
  *
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "qemu/fifo8.h"
 #include "hw/net/allwinner_emac.h"
+#include "qemu/log.h"
 #include <zlib.h>
 
 static uint8_t padding[60];
@@ -218,13 +220,6 @@
     return size;
 }
 
-static void aw_emac_cleanup(NetClientState *nc)
-{
-    AwEmacState *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static void aw_emac_reset(DeviceState *dev)
 {
     AwEmacState *s = AW_EMAC(dev);
@@ -429,11 +424,10 @@
 };
 
 static NetClientInfo net_aw_emac_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = aw_emac_can_receive,
     .receive = aw_emac_receive,
-    .cleanup = aw_emac_cleanup,
     .link_status_changed = aw_emac_set_link,
 };
 
diff --git a/hw/net/cadence_gem.c b/hw/net/cadence_gem.c
index de26609..db1b301 100644
--- a/hw/net/cadence_gem.c
+++ b/hw/net/cadence_gem.c
@@ -22,10 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <zlib.h> /* For crc32 */
 
-#include "hw/sysbus.h"
-#include "net/net.h"
+#include "hw/net/cadence_gem.h"
 #include "net/checksum.h"
 
 #ifdef CADENCE_GEM_ERR_DEBUG
@@ -141,8 +141,6 @@
 #define GEM_DESCONF6      (0x00000294/4)
 #define GEM_DESCONF7      (0x00000298/4)
 
-#define GEM_MAXREG        (0x00000640/4) /* Last valid GEM address */
-
 /*****************************************/
 #define GEM_NWCTRL_TXSTART     0x00000200 /* Transmit Enable */
 #define GEM_NWCTRL_TXENA       0x00000008 /* Transmit Enable */
@@ -158,7 +156,7 @@
 #define GEM_NWCFG_BCAST_REJ    0x00000020 /* Reject broadcast packets */
 #define GEM_NWCFG_PROMISC      0x00000010 /* Accept all packets */
 
-#define GEM_DMACFG_RBUFSZ_M    0x007F0000 /* DMA RX Buffer Size mask */
+#define GEM_DMACFG_RBUFSZ_M    0x00FF0000 /* DMA RX Buffer Size mask */
 #define GEM_DMACFG_RBUFSZ_S    16         /* DMA RX Buffer Size shift */
 #define GEM_DMACFG_RBUFSZ_MUL  64         /* DMA RX Buffer Size multiplier */
 #define GEM_DMACFG_TXCSUM_OFFL 0x00000800 /* Transmit checksum offload */
@@ -276,6 +274,11 @@
     return (desc[1] & DESC_1_TX_LAST) ? 1 : 0;
 }
 
+static inline void tx_desc_set_last(unsigned *desc)
+{
+    desc[1] |= DESC_1_TX_LAST;
+}
+
 static inline unsigned tx_desc_get_length(unsigned *desc)
 {
     return desc[1] & DESC_1_LENGTH;
@@ -349,44 +352,6 @@
     desc[1] |= R_DESC_1_RX_SAR_MATCH;
 }
 
-#define TYPE_CADENCE_GEM "cadence_gem"
-#define GEM(obj) OBJECT_CHECK(GemState, (obj), TYPE_CADENCE_GEM)
-
-typedef struct GemState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    NICState *nic;
-    NICConf conf;
-    qemu_irq irq;
-
-    /* GEM registers backing store */
-    uint32_t regs[GEM_MAXREG];
-    /* Mask of register bits which are write only */
-    uint32_t regs_wo[GEM_MAXREG];
-    /* Mask of register bits which are read only */
-    uint32_t regs_ro[GEM_MAXREG];
-    /* Mask of register bits which are clear on read */
-    uint32_t regs_rtc[GEM_MAXREG];
-    /* Mask of register bits which are write 1 to clear */
-    uint32_t regs_w1c[GEM_MAXREG];
-
-    /* PHY registers backing store */
-    uint16_t phy_regs[32];
-
-    uint8_t phy_loop; /* Are we in phy loopback? */
-
-    /* The current DMA descriptor pointers */
-    uint32_t rx_desc_addr;
-    uint32_t tx_desc_addr;
-
-    uint8_t can_rx_state; /* Debug only */
-
-    unsigned rx_desc[2];
-
-    bool sar_active[4];
-} GemState;
-
 /* The broadcast MAC address: 0xFFFFFFFFFFFF */
 static const uint8_t broadcast_addr[] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
 
@@ -395,7 +360,7 @@
  * One time initialization.
  * Set masks to identify which register bits have magical clear properties
  */
-static void gem_init_register_masks(GemState *s)
+static void gem_init_register_masks(CadenceGEMState *s)
 {
     /* Mask of register bits which are read only */
     memset(&s->regs_ro[0], 0, sizeof(s->regs_ro));
@@ -430,7 +395,7 @@
  * phy_update_link:
  * Make the emulated PHY link state match the QEMU "interface" state.
  */
-static void phy_update_link(GemState *s)
+static void phy_update_link(CadenceGEMState *s)
 {
     DB_PRINT("down %d\n", qemu_get_queue(s->nic)->link_down);
 
@@ -450,7 +415,7 @@
 
 static int gem_can_receive(NetClientState *nc)
 {
-    GemState *s;
+    CadenceGEMState *s;
 
     s = qemu_get_nic_opaque(nc);
 
@@ -483,7 +448,7 @@
  * gem_update_int_status:
  * Raise or lower interrupt based on current status.
  */
-static void gem_update_int_status(GemState *s)
+static void gem_update_int_status(CadenceGEMState *s)
 {
     if (s->regs[GEM_ISR]) {
         DB_PRINT("asserting int. (0x%08x)\n", s->regs[GEM_ISR]);
@@ -495,7 +460,7 @@
  * gem_receive_updatestats:
  * Increment receive statistics.
  */
-static void gem_receive_updatestats(GemState *s, const uint8_t *packet,
+static void gem_receive_updatestats(CadenceGEMState *s, const uint8_t *packet,
                                     unsigned bytes)
 {
     uint64_t octets;
@@ -586,7 +551,7 @@
  * GEM_RM_PROMISCUOUS_ACCEPT, GEM_RX_BROADCAST_ACCEPT,
  * GEM_RX_MULTICAST_HASH_ACCEPT or GEM_RX_UNICAST_HASH_ACCEPT
  */
-static int gem_mac_address_filter(GemState *s, const uint8_t *packet)
+static int gem_mac_address_filter(CadenceGEMState *s, const uint8_t *packet)
 {
     uint8_t *gem_spaddr;
     int i;
@@ -636,7 +601,7 @@
     return GEM_RX_REJECT;
 }
 
-static void gem_get_rx_desc(GemState *s)
+static void gem_get_rx_desc(CadenceGEMState *s)
 {
     DB_PRINT("read descriptor 0x%x\n", (unsigned)s->rx_desc_addr);
     /* read current descriptor */
@@ -660,7 +625,7 @@
  */
 static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-    GemState *s;
+    CadenceGEMState *s;
     unsigned   rxbufsize, bytes_to_copy;
     unsigned   rxbuf_offset;
     uint8_t    rxbuf[2048];
@@ -704,6 +669,13 @@
                  GEM_DMACFG_RBUFSZ_S) * GEM_DMACFG_RBUFSZ_MUL;
     bytes_to_copy = size;
 
+    /* Hardware allows a zero value here but warns against it. To avoid QEMU
+     * indefinite loops we enforce a minimum value here
+     */
+    if (rxbufsize < GEM_DMACFG_RBUFSZ_MUL) {
+        rxbufsize = GEM_DMACFG_RBUFSZ_MUL;
+    }
+
     /* Pad to minimum length. Assume FCS field is stripped, logic
      * below will increment it to the real minimum of 64 when
      * not FCS stripping
@@ -718,6 +690,10 @@
     } else {
         unsigned crc_val;
 
+        if (size > sizeof(rxbuf) - sizeof(crc_val)) {
+            size = sizeof(rxbuf) - sizeof(crc_val);
+        }
+        bytes_to_copy = size;
         /* The application wants the FCS field, which QEMU does not provide.
          * We must try and calculate one.
          */
@@ -810,7 +786,7 @@
  * gem_transmit_updatestats:
  * Increment transmit statistics.
  */
-static void gem_transmit_updatestats(GemState *s, const uint8_t *packet,
+static void gem_transmit_updatestats(CadenceGEMState *s, const uint8_t *packet,
                                      unsigned bytes)
 {
     uint64_t octets;
@@ -856,7 +832,7 @@
  * gem_transmit:
  * Fish packets out of the descriptor ring and feed them to QEMU
  */
-static void gem_transmit(GemState *s)
+static void gem_transmit(CadenceGEMState *s)
 {
     unsigned    desc[2];
     hwaddr packet_desc_addr;
@@ -903,6 +879,14 @@
             break;
         }
 
+        if (tx_desc_get_length(desc) > sizeof(tx_packet) - (p - tx_packet)) {
+            DB_PRINT("TX descriptor @ 0x%x too large: size 0x%x space 0x%x\n",
+                     (unsigned)packet_desc_addr,
+                     (unsigned)tx_desc_get_length(desc),
+                     sizeof(tx_packet) - (p - tx_packet));
+            break;
+        }
+
         /* Gather this fragment of the packet from "dma memory" to our contig.
          * buffer.
          */
@@ -960,6 +944,7 @@
 
         /* read next descriptor */
         if (tx_desc_get_wrap(desc)) {
+            tx_desc_set_last(desc);
             packet_desc_addr = s->regs[GEM_TXQBASE];
         } else {
             packet_desc_addr += 8;
@@ -976,7 +961,7 @@
     }
 }
 
-static void gem_phy_reset(GemState *s)
+static void gem_phy_reset(CadenceGEMState *s)
 {
     memset(&s->phy_regs[0], 0, sizeof(s->phy_regs));
     s->phy_regs[PHY_REG_CONTROL] = 0x1140;
@@ -992,7 +977,7 @@
     s->phy_regs[PHY_REG_1000BTSTAT] = 0x7C00;
     s->phy_regs[PHY_REG_EXTSTAT] = 0x3000;
     s->phy_regs[PHY_REG_PHYSPCFC_CTL] = 0x0078;
-    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0xBC00;
+    s->phy_regs[PHY_REG_PHYSPCFC_ST] = 0x7C00;
     s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL] = 0x0C60;
     s->phy_regs[PHY_REG_LED] = 0x4100;
     s->phy_regs[PHY_REG_EXT_PHYSPCFC_CTL2] = 0x000A;
@@ -1004,7 +989,8 @@
 static void gem_reset(DeviceState *d)
 {
     int i;
-    GemState *s = GEM(d);
+    CadenceGEMState *s = CADENCE_GEM(d);
+    const uint8_t *a;
 
     DB_PRINT("\n");
 
@@ -1023,6 +1009,11 @@
     s->regs[GEM_DESCONF5] = 0x002f2145;
     s->regs[GEM_DESCONF6] = 0x00000200;
 
+    /* Set MAC address */
+    a = &s->conf.macaddr.a[0];
+    s->regs[GEM_SPADDR1LO] = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24);
+    s->regs[GEM_SPADDR1HI] = a[4] | (a[5] << 8);
+
     for (i = 0; i < 4; i++) {
         s->sar_active[i] = false;
     }
@@ -1032,13 +1023,13 @@
     gem_update_int_status(s);
 }
 
-static uint16_t gem_phy_read(GemState *s, unsigned reg_num)
+static uint16_t gem_phy_read(CadenceGEMState *s, unsigned reg_num)
 {
     DB_PRINT("reg: %d value: 0x%04x\n", reg_num, s->phy_regs[reg_num]);
     return s->phy_regs[reg_num];
 }
 
-static void gem_phy_write(GemState *s, unsigned reg_num, uint16_t val)
+static void gem_phy_write(CadenceGEMState *s, unsigned reg_num, uint16_t val)
 {
     DB_PRINT("reg: %d value: 0x%04x\n", reg_num, val);
 
@@ -1072,10 +1063,10 @@
  */
 static uint64_t gem_read(void *opaque, hwaddr offset, unsigned size)
 {
-    GemState *s;
+    CadenceGEMState *s;
     uint32_t retval;
 
-    s = (GemState *)opaque;
+    s = (CadenceGEMState *)opaque;
 
     offset >>= 2;
     retval = s->regs[offset];
@@ -1120,7 +1111,7 @@
 static void gem_write(void *opaque, hwaddr offset, uint64_t val,
         unsigned size)
 {
-    GemState *s = (GemState *)opaque;
+    CadenceGEMState *s = (CadenceGEMState *)opaque;
     uint32_t readonly;
 
     DB_PRINT("offset: 0x%04x write: 0x%08x ", (unsigned)offset, (unsigned)val);
@@ -1209,14 +1200,6 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void gem_cleanup(NetClientState *nc)
-{
-    GemState *s = qemu_get_nic_opaque(nc);
-
-    DB_PRINT("\n");
-    s->nic = NULL;
-}
-
 static void gem_set_link(NetClientState *nc)
 {
     DB_PRINT("\n");
@@ -1224,18 +1207,17 @@
 }
 
 static NetClientInfo net_gem_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = gem_can_receive,
     .receive = gem_receive,
-    .cleanup = gem_cleanup,
     .link_status_changed = gem_set_link,
 };
 
 static int gem_init(SysBusDevice *sbd)
 {
     DeviceState *dev = DEVICE(sbd);
-    GemState *s = GEM(dev);
+    CadenceGEMState *s = CADENCE_GEM(dev);
 
     DB_PRINT("\n");
 
@@ -1257,18 +1239,18 @@
     .version_id = 2,
     .minimum_version_id = 2,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT32_ARRAY(regs, GemState, GEM_MAXREG),
-        VMSTATE_UINT16_ARRAY(phy_regs, GemState, 32),
-        VMSTATE_UINT8(phy_loop, GemState),
-        VMSTATE_UINT32(rx_desc_addr, GemState),
-        VMSTATE_UINT32(tx_desc_addr, GemState),
-        VMSTATE_BOOL_ARRAY(sar_active, GemState, 4),
+        VMSTATE_UINT32_ARRAY(regs, CadenceGEMState, CADENCE_GEM_MAXREG),
+        VMSTATE_UINT16_ARRAY(phy_regs, CadenceGEMState, 32),
+        VMSTATE_UINT8(phy_loop, CadenceGEMState),
+        VMSTATE_UINT32(rx_desc_addr, CadenceGEMState),
+        VMSTATE_UINT32(tx_desc_addr, CadenceGEMState),
+        VMSTATE_BOOL_ARRAY(sar_active, CadenceGEMState, 4),
         VMSTATE_END_OF_LIST(),
     }
 };
 
 static Property gem_properties[] = {
-    DEFINE_NIC_PROPERTIES(GemState, conf),
+    DEFINE_NIC_PROPERTIES(CadenceGEMState, conf),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1286,7 +1268,7 @@
 static const TypeInfo gem_info = {
     .name  = TYPE_CADENCE_GEM,
     .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_size  = sizeof(GemState),
+    .instance_size  = sizeof(CadenceGEMState),
     .class_init = gem_class_init,
 };
 
diff --git a/hw/net/dp8393x.c b/hw/net/dp8393x.c
index 7eab7ad..17f0338 100644
--- a/hw/net/dp8393x.c
+++ b/hw/net/dp8393x.c
@@ -17,20 +17,17 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "hw/hw.h"
-#include "qemu/timer.h"
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+#include "hw/devices.h"
 #include "net/net.h"
-#include "hw/mips/mips.h"
+#include "qapi/error.h"
+#include "qemu/timer.h"
+#include <zlib.h>
 
 //#define DEBUG_SONIC
 
-/* Calculate CRCs properly on Rx packets */
-#define SONIC_CALCULATE_RXCRC
-
-#if defined(SONIC_CALCULATE_RXCRC)
-/* For crc32 */
-#include <zlib.h>
-#endif
+#define SONIC_PROM_SIZE 0x1000
 
 #ifdef DEBUG_SONIC
 #define DPRINTF(fmt, ...) \
@@ -145,9 +142,14 @@
 #define SONIC_ISR_PINT   0x0800
 #define SONIC_ISR_LCD    0x1000
 
+#define TYPE_DP8393X "dp8393x"
+#define DP8393X(obj) OBJECT_CHECK(dp8393xState, (obj), TYPE_DP8393X)
+
 typedef struct dp8393xState {
+    SysBusDevice parent_obj;
+
     /* Hardware */
-    int it_shift;
+    uint8_t it_shift;
     qemu_irq irq;
 #ifdef DEBUG_SONIC
     int irq_level;
@@ -156,8 +158,8 @@
     int64_t wt_last_update;
     NICConf conf;
     NICState *nic;
-    MemoryRegion *address_space;
     MemoryRegion mmio;
+    MemoryRegion prom;
 
     /* Registers */
     uint8_t cam[16][6];
@@ -168,8 +170,8 @@
     int loopback_packet;
 
     /* Memory access */
-    void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write);
-    void* mem_opaque;
+    void *dma_mr;
+    AddressSpace as;
 } dp8393xState;
 
 static void dp8393x_update_irq(dp8393xState *s)
@@ -190,7 +192,7 @@
     qemu_set_irq(s->irq, level);
 }
 
-static void do_load_cam(dp8393xState *s)
+static void dp8393x_do_load_cam(dp8393xState *s)
 {
     uint16_t data[8];
     int width, size;
@@ -201,9 +203,9 @@
 
     while (s->regs[SONIC_CDC] & 0x1f) {
         /* Fill current entry */
-        s->memory_rw(s->mem_opaque,
+        address_space_rw(&s->as,
             (s->regs[SONIC_URRA] << 16) | s->regs[SONIC_CDP],
-            (uint8_t *)data, size, 0);
+            MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
         s->cam[index][0] = data[1 * width] & 0xff;
         s->cam[index][1] = data[1 * width] >> 8;
         s->cam[index][2] = data[2 * width] & 0xff;
@@ -220,9 +222,9 @@
     }
 
     /* Read CAM enable */
-    s->memory_rw(s->mem_opaque,
+    address_space_rw(&s->as,
         (s->regs[SONIC_URRA] << 16) | s->regs[SONIC_CDP],
-        (uint8_t *)data, size, 0);
+        MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
     s->regs[SONIC_CE] = data[0 * width];
     DPRINTF("load cam done. cam enable mask 0x%04x\n", s->regs[SONIC_CE]);
 
@@ -232,7 +234,7 @@
     dp8393x_update_irq(s);
 }
 
-static void do_read_rra(dp8393xState *s)
+static void dp8393x_do_read_rra(dp8393xState *s)
 {
     uint16_t data[8];
     int width, size;
@@ -240,9 +242,9 @@
     /* Read memory */
     width = (s->regs[SONIC_DCR] & SONIC_DCR_DW) ? 2 : 1;
     size = sizeof(uint16_t) * 4 * width;
-    s->memory_rw(s->mem_opaque,
+    address_space_rw(&s->as,
         (s->regs[SONIC_URRA] << 16) | s->regs[SONIC_RRP],
-        (uint8_t *)data, size, 0);
+        MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
 
     /* Update SONIC registers */
     s->regs[SONIC_CRBA0] = data[0 * width];
@@ -272,7 +274,7 @@
     s->regs[SONIC_CR] &= ~SONIC_CR_RRRA;
 }
 
-static void do_software_reset(dp8393xState *s)
+static void dp8393x_do_software_reset(dp8393xState *s)
 {
     timer_del(s->watchdog);
 
@@ -280,7 +282,7 @@
     s->regs[SONIC_CR] |= SONIC_CR_RST | SONIC_CR_RXDIS;
 }
 
-static void set_next_tick(dp8393xState *s)
+static void dp8393x_set_next_tick(dp8393xState *s)
 {
     uint32_t ticks;
     int64_t delay;
@@ -292,11 +294,11 @@
 
     ticks = s->regs[SONIC_WT1] << 16 | s->regs[SONIC_WT0];
     s->wt_last_update = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    delay = get_ticks_per_sec() * ticks / 5000000;
+    delay = NANOSECONDS_PER_SECOND * ticks / 5000000;
     timer_mod(s->watchdog, s->wt_last_update + delay);
 }
 
-static void update_wt_regs(dp8393xState *s)
+static void dp8393x_update_wt_regs(dp8393xState *s)
 {
     int64_t elapsed;
     uint32_t val;
@@ -311,33 +313,38 @@
     val -= elapsed / 5000000;
     s->regs[SONIC_WT1] = (val >> 16) & 0xffff;
     s->regs[SONIC_WT0] = (val >> 0)  & 0xffff;
-    set_next_tick(s);
+    dp8393x_set_next_tick(s);
 
 }
 
-static void do_start_timer(dp8393xState *s)
+static void dp8393x_do_start_timer(dp8393xState *s)
 {
     s->regs[SONIC_CR] &= ~SONIC_CR_STP;
-    set_next_tick(s);
+    dp8393x_set_next_tick(s);
 }
 
-static void do_stop_timer(dp8393xState *s)
+static void dp8393x_do_stop_timer(dp8393xState *s)
 {
     s->regs[SONIC_CR] &= ~SONIC_CR_ST;
-    update_wt_regs(s);
+    dp8393x_update_wt_regs(s);
 }
 
-static void do_receiver_enable(dp8393xState *s)
+static int dp8393x_can_receive(NetClientState *nc);
+
+static void dp8393x_do_receiver_enable(dp8393xState *s)
 {
     s->regs[SONIC_CR] &= ~SONIC_CR_RXDIS;
+    if (dp8393x_can_receive(s->nic->ncs)) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
 }
 
-static void do_receiver_disable(dp8393xState *s)
+static void dp8393x_do_receiver_disable(dp8393xState *s)
 {
     s->regs[SONIC_CR] &= ~SONIC_CR_RXEN;
 }
 
-static void do_transmit_packets(dp8393xState *s)
+static void dp8393x_do_transmit_packets(dp8393xState *s)
 {
     NetClientState *nc = qemu_get_queue(s->nic);
     uint16_t data[12];
@@ -353,9 +360,9 @@
                 (s->regs[SONIC_UTDA] << 16) | s->regs[SONIC_CTDA]);
         size = sizeof(uint16_t) * 6 * width;
         s->regs[SONIC_TTDA] = s->regs[SONIC_CTDA];
-        s->memory_rw(s->mem_opaque,
+        address_space_rw(&s->as,
             ((s->regs[SONIC_UTDA] << 16) | s->regs[SONIC_TTDA]) + sizeof(uint16_t) * width,
-            (uint8_t *)data, size, 0);
+            MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
         tx_len = 0;
 
         /* Update registers */
@@ -379,18 +386,18 @@
             if (tx_len + len > sizeof(s->tx_buffer)) {
                 len = sizeof(s->tx_buffer) - tx_len;
             }
-            s->memory_rw(s->mem_opaque,
+            address_space_rw(&s->as,
                 (s->regs[SONIC_TSA1] << 16) | s->regs[SONIC_TSA0],
-                &s->tx_buffer[tx_len], len, 0);
+                MEMTXATTRS_UNSPECIFIED, &s->tx_buffer[tx_len], len, 0);
             tx_len += len;
 
             i++;
             if (i != s->regs[SONIC_TFC]) {
                 /* Read next fragment details */
                 size = sizeof(uint16_t) * 3 * width;
-                s->memory_rw(s->mem_opaque,
+                address_space_rw(&s->as,
                     ((s->regs[SONIC_UTDA] << 16) | s->regs[SONIC_TTDA]) + sizeof(uint16_t) * (4 + 3 * i) * width,
-                    (uint8_t *)data, size, 0);
+                    MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
                 s->regs[SONIC_TSA0] = data[0 * width];
                 s->regs[SONIC_TSA1] = data[1 * width];
                 s->regs[SONIC_TFS] = data[2 * width];
@@ -422,16 +429,16 @@
         /* Write status */
         data[0 * width] = s->regs[SONIC_TCR] & 0x0fff; /* status */
         size = sizeof(uint16_t) * width;
-        s->memory_rw(s->mem_opaque,
+        address_space_rw(&s->as,
             (s->regs[SONIC_UTDA] << 16) | s->regs[SONIC_TTDA],
-            (uint8_t *)data, size, 1);
+            MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 1);
 
         if (!(s->regs[SONIC_CR] & SONIC_CR_HTX)) {
             /* Read footer of packet */
             size = sizeof(uint16_t) * width;
-            s->memory_rw(s->mem_opaque,
+            address_space_rw(&s->as,
                 ((s->regs[SONIC_UTDA] << 16) | s->regs[SONIC_TTDA]) + sizeof(uint16_t) * (4 + 3 * s->regs[SONIC_TFC]) * width,
-                (uint8_t *)data, size, 0);
+                MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
             s->regs[SONIC_CTDA] = data[0 * width] & ~0x1;
             if (data[0 * width] & 0x1) {
                 /* EOL detected */
@@ -446,12 +453,12 @@
     dp8393x_update_irq(s);
 }
 
-static void do_halt_transmission(dp8393xState *s)
+static void dp8393x_do_halt_transmission(dp8393xState *s)
 {
     /* Nothing to do */
 }
 
-static void do_command(dp8393xState *s, uint16_t command)
+static void dp8393x_do_command(dp8393xState *s, uint16_t command)
 {
     if ((s->regs[SONIC_CR] & SONIC_CR_RST) && !(command & SONIC_CR_RST)) {
         s->regs[SONIC_CR] &= ~SONIC_CR_RST;
@@ -461,34 +468,36 @@
     s->regs[SONIC_CR] |= (command & SONIC_CR_MASK);
 
     if (command & SONIC_CR_HTX)
-        do_halt_transmission(s);
+        dp8393x_do_halt_transmission(s);
     if (command & SONIC_CR_TXP)
-        do_transmit_packets(s);
+        dp8393x_do_transmit_packets(s);
     if (command & SONIC_CR_RXDIS)
-        do_receiver_disable(s);
+        dp8393x_do_receiver_disable(s);
     if (command & SONIC_CR_RXEN)
-        do_receiver_enable(s);
+        dp8393x_do_receiver_enable(s);
     if (command & SONIC_CR_STP)
-        do_stop_timer(s);
+        dp8393x_do_stop_timer(s);
     if (command & SONIC_CR_ST)
-        do_start_timer(s);
+        dp8393x_do_start_timer(s);
     if (command & SONIC_CR_RST)
-        do_software_reset(s);
+        dp8393x_do_software_reset(s);
     if (command & SONIC_CR_RRRA)
-        do_read_rra(s);
+        dp8393x_do_read_rra(s);
     if (command & SONIC_CR_LCAM)
-        do_load_cam(s);
+        dp8393x_do_load_cam(s);
 }
 
-static uint16_t read_register(dp8393xState *s, int reg)
+static uint64_t dp8393x_read(void *opaque, hwaddr addr, unsigned int size)
 {
+    dp8393xState *s = opaque;
+    int reg = addr >> s->it_shift;
     uint16_t val = 0;
 
     switch (reg) {
         /* Update data before reading it */
         case SONIC_WT0:
         case SONIC_WT1:
-            update_wt_regs(s);
+            dp8393x_update_wt_regs(s);
             val = s->regs[reg];
             break;
         /* Accept read to some registers only when in reset mode */
@@ -510,14 +519,18 @@
     return val;
 }
 
-static void write_register(dp8393xState *s, int reg, uint16_t val)
+static void dp8393x_write(void *opaque, hwaddr addr, uint64_t data,
+                          unsigned int size)
 {
-    DPRINTF("write 0x%04x to reg %s\n", val, reg_names[reg]);
+    dp8393xState *s = opaque;
+    int reg = addr >> s->it_shift;
+
+    DPRINTF("write 0x%04x to reg %s\n", (uint16_t)data, reg_names[reg]);
 
     switch (reg) {
         /* Command register */
         case SONIC_CR:
-            do_command(s, val);
+            dp8393x_do_command(s, data);
             break;
         /* Prevent write to read-only registers */
         case SONIC_CAP2:
@@ -530,63 +543,74 @@
         /* Accept write to some registers only when in reset mode */
         case SONIC_DCR:
             if (s->regs[SONIC_CR] & SONIC_CR_RST) {
-                s->regs[reg] = val & 0xbfff;
+                s->regs[reg] = data & 0xbfff;
             } else {
                 DPRINTF("writing to DCR invalid\n");
             }
             break;
         case SONIC_DCR2:
             if (s->regs[SONIC_CR] & SONIC_CR_RST) {
-                s->regs[reg] = val & 0xf017;
+                s->regs[reg] = data & 0xf017;
             } else {
                 DPRINTF("writing to DCR2 invalid\n");
             }
             break;
         /* 12 lower bytes are Read Only */
         case SONIC_TCR:
-            s->regs[reg] = val & 0xf000;
+            s->regs[reg] = data & 0xf000;
             break;
         /* 9 lower bytes are Read Only */
         case SONIC_RCR:
-            s->regs[reg] = val & 0xffe0;
+            s->regs[reg] = data & 0xffe0;
             break;
         /* Ignore most significant bit */
         case SONIC_IMR:
-            s->regs[reg] = val & 0x7fff;
+            s->regs[reg] = data & 0x7fff;
             dp8393x_update_irq(s);
             break;
         /* Clear bits by writing 1 to them */
         case SONIC_ISR:
-            val &= s->regs[reg];
-            s->regs[reg] &= ~val;
-            if (val & SONIC_ISR_RBE) {
-                do_read_rra(s);
+            data &= s->regs[reg];
+            s->regs[reg] &= ~data;
+            if (data & SONIC_ISR_RBE) {
+                dp8393x_do_read_rra(s);
             }
             dp8393x_update_irq(s);
+            if (dp8393x_can_receive(s->nic->ncs)) {
+                qemu_flush_queued_packets(qemu_get_queue(s->nic));
+            }
             break;
         /* Ignore least significant bit */
         case SONIC_RSA:
         case SONIC_REA:
         case SONIC_RRP:
         case SONIC_RWP:
-            s->regs[reg] = val & 0xfffe;
+            s->regs[reg] = data & 0xfffe;
             break;
         /* Invert written value for some registers */
         case SONIC_CRCT:
         case SONIC_FAET:
         case SONIC_MPT:
-            s->regs[reg] = val ^ 0xffff;
+            s->regs[reg] = data ^ 0xffff;
             break;
         /* All other registers have no special contrainst */
         default:
-            s->regs[reg] = val;
+            s->regs[reg] = data;
     }
 
     if (reg == SONIC_WT0 || reg == SONIC_WT1) {
-        set_next_tick(s);
+        dp8393x_set_next_tick(s);
     }
 }
 
+static const MemoryRegionOps dp8393x_ops = {
+    .read = dp8393x_read,
+    .write = dp8393x_write,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
 static void dp8393x_watchdog(void *opaque)
 {
     dp8393xState *s = opaque;
@@ -597,84 +621,14 @@
 
     s->regs[SONIC_WT1] = 0xffff;
     s->regs[SONIC_WT0] = 0xffff;
-    set_next_tick(s);
+    dp8393x_set_next_tick(s);
 
     /* Signal underflow */
     s->regs[SONIC_ISR] |= SONIC_ISR_TC;
     dp8393x_update_irq(s);
 }
 
-static uint32_t dp8393x_readw(void *opaque, hwaddr addr)
-{
-    dp8393xState *s = opaque;
-    int reg;
-
-    if ((addr & ((1 << s->it_shift) - 1)) != 0) {
-        return 0;
-    }
-
-    reg = addr >> s->it_shift;
-    return read_register(s, reg);
-}
-
-static uint32_t dp8393x_readb(void *opaque, hwaddr addr)
-{
-    uint16_t v = dp8393x_readw(opaque, addr & ~0x1);
-    return (v >> (8 * (addr & 0x1))) & 0xff;
-}
-
-static uint32_t dp8393x_readl(void *opaque, hwaddr addr)
-{
-    uint32_t v;
-    v = dp8393x_readw(opaque, addr);
-    v |= dp8393x_readw(opaque, addr + 2) << 16;
-    return v;
-}
-
-static void dp8393x_writew(void *opaque, hwaddr addr, uint32_t val)
-{
-    dp8393xState *s = opaque;
-    int reg;
-
-    if ((addr & ((1 << s->it_shift) - 1)) != 0) {
-        return;
-    }
-
-    reg = addr >> s->it_shift;
-
-    write_register(s, reg, (uint16_t)val);
-}
-
-static void dp8393x_writeb(void *opaque, hwaddr addr, uint32_t val)
-{
-    uint16_t old_val = dp8393x_readw(opaque, addr & ~0x1);
-
-    switch (addr & 3) {
-    case 0:
-        val = val | (old_val & 0xff00);
-        break;
-    case 1:
-        val = (val << 8) | (old_val & 0x00ff);
-        break;
-    }
-    dp8393x_writew(opaque, addr & ~0x1, val);
-}
-
-static void dp8393x_writel(void *opaque, hwaddr addr, uint32_t val)
-{
-    dp8393x_writew(opaque, addr, val & 0xffff);
-    dp8393x_writew(opaque, addr + 2, (val >> 16) & 0xffff);
-}
-
-static const MemoryRegionOps dp8393x_ops = {
-    .old_mmio = {
-        .read = { dp8393x_readb, dp8393x_readw, dp8393x_readl, },
-        .write = { dp8393x_writeb, dp8393x_writew, dp8393x_writel, },
-    },
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static int nic_can_receive(NetClientState *nc)
+static int dp8393x_can_receive(NetClientState *nc)
 {
     dp8393xState *s = qemu_get_nic_opaque(nc);
 
@@ -685,16 +639,12 @@
     return 1;
 }
 
-static int receive_filter(dp8393xState *s, const uint8_t * buf, int size)
+static int dp8393x_receive_filter(dp8393xState *s, const uint8_t * buf,
+                                  int size)
 {
     static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
     int i;
 
-    /* Check for runt packet (remember that checksum is not there) */
-    if (size < 64 - 4) {
-        return (s->regs[SONIC_RCR] & SONIC_RCR_RNT) ? 0 : -1;
-    }
-
     /* Check promiscuous mode */
     if ((s->regs[SONIC_RCR] & SONIC_RCR_PRO) && (buf[0] & 1) == 0) {
         return 0;
@@ -723,7 +673,8 @@
     return -1;
 }
 
-static ssize_t nic_receive(NetClientState *nc, const uint8_t * buf, size_t size)
+static ssize_t dp8393x_receive(NetClientState *nc, const uint8_t * buf,
+                               size_t size)
 {
     dp8393xState *s = qemu_get_nic_opaque(nc);
     uint16_t data[10];
@@ -737,7 +688,7 @@
     s->regs[SONIC_RCR] &= ~(SONIC_RCR_PRX | SONIC_RCR_LBK | SONIC_RCR_FAER |
         SONIC_RCR_CRCR | SONIC_RCR_LPKT | SONIC_RCR_BC | SONIC_RCR_MC);
 
-    packet_type = receive_filter(s, buf, size);
+    packet_type = dp8393x_receive_filter(s, buf, size);
     if (packet_type < 0) {
         DPRINTF("packet not for netcard\n");
         return -1;
@@ -750,7 +701,8 @@
         /* Are we still in resource exhaustion? */
         size = sizeof(uint16_t) * 1 * width;
         address = ((s->regs[SONIC_URDA] << 16) | s->regs[SONIC_CRDA]) + sizeof(uint16_t) * 5 * width;
-        s->memory_rw(s->mem_opaque, address, (uint8_t*)data, size, 0);
+        address_space_rw(&s->as, address, MEMTXATTRS_UNSPECIFIED,
+                         (uint8_t *)data, size, 0);
         if (data[0 * width] & 0x1) {
             /* Still EOL ; stop reception */
             return -1;
@@ -764,18 +716,16 @@
     s->regs[SONIC_TRBA0] = s->regs[SONIC_CRBA0];
 
     /* Calculate the ethernet checksum */
-#ifdef SONIC_CALCULATE_RXCRC
     checksum = cpu_to_le32(crc32(0, buf, rx_len));
-#else
-    checksum = 0;
-#endif
 
     /* Put packet into RBA */
     DPRINTF("Receive packet at %08x\n", (s->regs[SONIC_CRBA1] << 16) | s->regs[SONIC_CRBA0]);
     address = (s->regs[SONIC_CRBA1] << 16) | s->regs[SONIC_CRBA0];
-    s->memory_rw(s->mem_opaque, address, (uint8_t*)buf, rx_len, 1);
+    address_space_rw(&s->as, address,
+        MEMTXATTRS_UNSPECIFIED, (uint8_t *)buf, rx_len, 1);
     address += rx_len;
-    s->memory_rw(s->mem_opaque, address, (uint8_t*)&checksum, 4, 1);
+    address_space_rw(&s->as, address,
+        MEMTXATTRS_UNSPECIFIED, (uint8_t *)&checksum, 4, 1);
     rx_len += 4;
     s->regs[SONIC_CRBA1] = address >> 16;
     s->regs[SONIC_CRBA0] = address & 0xffff;
@@ -803,29 +753,30 @@
     data[3 * width] = s->regs[SONIC_TRBA1]; /* pkt_ptr1 */
     data[4 * width] = s->regs[SONIC_RSC]; /* seq_no */
     size = sizeof(uint16_t) * 5 * width;
-    s->memory_rw(s->mem_opaque, (s->regs[SONIC_URDA] << 16) | s->regs[SONIC_CRDA], (uint8_t *)data, size, 1);
+    address_space_rw(&s->as, (s->regs[SONIC_URDA] << 16) | s->regs[SONIC_CRDA],
+        MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 1);
 
     /* Move to next descriptor */
     size = sizeof(uint16_t) * width;
-    s->memory_rw(s->mem_opaque,
+    address_space_rw(&s->as,
         ((s->regs[SONIC_URDA] << 16) | s->regs[SONIC_CRDA]) + sizeof(uint16_t) * 5 * width,
-        (uint8_t *)data, size, 0);
+        MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, size, 0);
     s->regs[SONIC_LLFA] = data[0 * width];
     if (s->regs[SONIC_LLFA] & 0x1) {
         /* EOL detected */
         s->regs[SONIC_ISR] |= SONIC_ISR_RDE;
     } else {
         data[0 * width] = 0; /* in_use */
-        s->memory_rw(s->mem_opaque,
+        address_space_rw(&s->as,
             ((s->regs[SONIC_URDA] << 16) | s->regs[SONIC_CRDA]) + sizeof(uint16_t) * 6 * width,
-            (uint8_t *)data, size, 1);
+            MEMTXATTRS_UNSPECIFIED, (uint8_t *)data, sizeof(uint16_t), 1);
         s->regs[SONIC_CRDA] = s->regs[SONIC_LLFA];
         s->regs[SONIC_ISR] |= SONIC_ISR_PKTRX;
         s->regs[SONIC_RSC] = (s->regs[SONIC_RSC] & 0xff00) | (((s->regs[SONIC_RSC] & 0x00ff) + 1) & 0x00ff);
 
         if (s->regs[SONIC_RCR] & SONIC_RCR_LPKT) {
             /* Read next RRA */
-            do_read_rra(s);
+            dp8393x_do_read_rra(s);
         }
     }
 
@@ -835,11 +786,12 @@
     return size;
 }
 
-static void nic_reset(void *opaque)
+static void dp8393x_reset(DeviceState *dev)
 {
-    dp8393xState *s = opaque;
+    dp8393xState *s = DP8393X(dev);
     timer_del(s->watchdog);
 
+    memset(s->regs, 0, sizeof(s->regs));
     s->regs[SONIC_CR] = SONIC_CR_RST | SONIC_CR_STP | SONIC_CR_RXDIS;
     s->regs[SONIC_DCR] &= ~(SONIC_DCR_EXBUS | SONIC_DCR_LBR);
     s->regs[SONIC_RCR] &= ~(SONIC_RCR_LB0 | SONIC_RCR_LB1 | SONIC_RCR_BRD | SONIC_RCR_RNT);
@@ -859,53 +811,102 @@
     dp8393x_update_irq(s);
 }
 
-static void nic_cleanup(NetClientState *nc)
-{
-    dp8393xState *s = qemu_get_nic_opaque(nc);
-
-    timer_del(s->watchdog);
-    timer_free(s->watchdog);
-
-    g_free(s);
-}
-
 static NetClientInfo net_dp83932_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = nic_can_receive,
-    .receive = nic_receive,
-    .cleanup = nic_cleanup,
+    .can_receive = dp8393x_can_receive,
+    .receive = dp8393x_receive,
 };
 
-void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
-                  MemoryRegion *address_space,
-                  qemu_irq irq, void* mem_opaque,
-                  void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write))
+static void dp8393x_instance_init(Object *obj)
 {
-    dp8393xState *s;
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    dp8393xState *s = DP8393X(obj);
 
-    qemu_check_nic_model(nd, "dp83932");
+    sysbus_init_mmio(sbd, &s->mmio);
+    sysbus_init_mmio(sbd, &s->prom);
+    sysbus_init_irq(sbd, &s->irq);
+}
 
-    s = g_malloc0(sizeof(dp8393xState));
+static void dp8393x_realize(DeviceState *dev, Error **errp)
+{
+    dp8393xState *s = DP8393X(dev);
+    int i, checksum;
+    uint8_t *prom;
+    Error *local_err = NULL;
 
-    s->address_space = address_space;
-    s->mem_opaque = mem_opaque;
-    s->memory_rw = memory_rw;
-    s->it_shift = it_shift;
-    s->irq = irq;
+    address_space_init(&s->as, s->dma_mr, "dp8393x");
+    memory_region_init_io(&s->mmio, OBJECT(dev), &dp8393x_ops, s,
+                          "dp8393x-regs", 0x40 << s->it_shift);
+
+    s->nic = qemu_new_nic(&net_dp83932_info, &s->conf,
+                          object_get_typename(OBJECT(dev)), dev->id, s);
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
+
     s->watchdog = timer_new_ns(QEMU_CLOCK_VIRTUAL, dp8393x_watchdog, s);
     s->regs[SONIC_SR] = 0x0004; /* only revision recognized by Linux */
 
-    s->conf.macaddr = nd->macaddr;
-    s->conf.peers.ncs[0] = nd->netdev;
-
-    s->nic = qemu_new_nic(&net_dp83932_info, &s->conf, nd->model, nd->name, s);
-
-    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
-    qemu_register_reset(nic_reset, s);
-    nic_reset(s);
-
-    memory_region_init_io(&s->mmio, NULL, &dp8393x_ops, s,
-                          "dp8393x", 0x40 << it_shift);
-    memory_region_add_subregion(address_space, base, &s->mmio);
+    memory_region_init_ram(&s->prom, OBJECT(dev),
+                           "dp8393x-prom", SONIC_PROM_SIZE, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    memory_region_set_readonly(&s->prom, true);
+    prom = memory_region_get_ram_ptr(&s->prom);
+    checksum = 0;
+    for (i = 0; i < 6; i++) {
+        prom[i] = s->conf.macaddr.a[i];
+        checksum += prom[i];
+        if (checksum > 0xff) {
+            checksum = (checksum + 1) & 0xff;
+        }
+    }
+    prom[7] = 0xff - checksum;
 }
+
+static const VMStateDescription vmstate_dp8393x = {
+    .name = "dp8393x",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField []) {
+        VMSTATE_BUFFER_UNSAFE(cam, dp8393xState, 0, 16 * 6),
+        VMSTATE_UINT16_ARRAY(regs, dp8393xState, 0x40),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property dp8393x_properties[] = {
+    DEFINE_NIC_PROPERTIES(dp8393xState, conf),
+    DEFINE_PROP_PTR("dma_mr", dp8393xState, dma_mr),
+    DEFINE_PROP_UINT8("it_shift", dp8393xState, it_shift, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void dp8393x_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
+    dc->realize = dp8393x_realize;
+    dc->reset = dp8393x_reset;
+    dc->vmsd = &vmstate_dp8393x;
+    dc->props = dp8393x_properties;
+    /* Reason: dma_mr property can't be set */
+    dc->cannot_instantiate_with_device_add_yet = true;
+}
+
+static const TypeInfo dp8393x_info = {
+    .name          = TYPE_DP8393X,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(dp8393xState),
+    .instance_init = dp8393x_instance_init,
+    .class_init    = dp8393x_class_init,
+};
+
+static void dp8393x_register_types(void)
+{
+    type_register_static(&dp8393x_info);
+}
+
+type_init(dp8393x_register_types)
diff --git a/hw/net/e1000.c b/hw/net/e1000.c
index e33a4da..9324949 100644
--- a/hw/net/e1000.c
+++ b/hw/net/e1000.c
@@ -25,6 +25,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -33,38 +34,36 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/dma.h"
 #include "qemu/iov.h"
+#include "qemu/range.h"
 
-#include "e1000_regs.h"
+#include "e1000x_common.h"
+
+static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
 
 #define E1000_DEBUG
 
 #ifdef E1000_DEBUG
 enum {
-    DEBUG_GENERAL,	DEBUG_IO,	DEBUG_MMIO,	DEBUG_INTERRUPT,
-    DEBUG_RX,		DEBUG_TX,	DEBUG_MDIC,	DEBUG_EEPROM,
-    DEBUG_UNKNOWN,	DEBUG_TXSUM,	DEBUG_TXERR,	DEBUG_RXERR,
+    DEBUG_GENERAL,      DEBUG_IO,       DEBUG_MMIO,     DEBUG_INTERRUPT,
+    DEBUG_RX,           DEBUG_TX,       DEBUG_MDIC,     DEBUG_EEPROM,
+    DEBUG_UNKNOWN,      DEBUG_TXSUM,    DEBUG_TXERR,    DEBUG_RXERR,
     DEBUG_RXFILTER,     DEBUG_PHY,      DEBUG_NOTYET,
 };
-#define DBGBIT(x)	(1<<DEBUG_##x)
+#define DBGBIT(x)    (1<<DEBUG_##x)
 static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 
-#define	DBGOUT(what, fmt, ...) do { \
+#define DBGOUT(what, fmt, ...) do { \
     if (debugflags & DBGBIT(what)) \
         fprintf(stderr, "e1000: " fmt, ## __VA_ARGS__); \
     } while (0)
 #else
-#define	DBGOUT(what, fmt, ...) do {} while (0)
+#define DBGOUT(what, fmt, ...) do {} while (0)
 #endif
 
 #define IOPORT_SIZE       0x40
 #define PNPMMIO_SIZE      0x20000
 #define MIN_BUF_SIZE      60 /* Min. octets in an ethernet frame sans FCS */
 
-/* this is the size past which hardware will drop packets when setting LPE=0 */
-#define MAXIMUM_ETHERNET_VLAN_SIZE 1522
-/* this is the size past which hardware will drop packets when setting LPE=1 */
-#define MAXIMUM_ETHERNET_LPE_SIZE 16384
-
 #define MAXIMUM_ETHERNET_HDR_LEN (14+4)
 
 /*
@@ -98,26 +97,13 @@
         unsigned char vlan[4];
         unsigned char data[0x10000];
         uint16_t size;
-        unsigned char sum_needed;
         unsigned char vlan_needed;
-        uint8_t ipcss;
-        uint8_t ipcso;
-        uint16_t ipcse;
-        uint8_t tucss;
-        uint8_t tucso;
-        uint16_t tucse;
-        uint8_t hdr_len;
-        uint16_t mss;
-        uint32_t paylen;
+        e1000x_txd_props props;
         uint16_t tso_frames;
-        char tse;
-        int8_t ip;
-        int8_t tcp;
-        char cptse;     // current packet tse bit
     } tx;
 
     struct {
-        uint32_t val_in;	// shifted in from guest driver
+        uint32_t val_in;    /* shifted in from guest driver */
         uint16_t bitnum_in;
         uint16_t bitnum_out;
         uint16_t reading;
@@ -134,11 +120,15 @@
 /* Compatibility flags for migration to/from qemu 1.3.0 and older */
 #define E1000_FLAG_AUTONEG_BIT 0
 #define E1000_FLAG_MIT_BIT 1
+#define E1000_FLAG_MAC_BIT 2
 #define E1000_FLAG_AUTONEG (1 << E1000_FLAG_AUTONEG_BIT)
 #define E1000_FLAG_MIT (1 << E1000_FLAG_MIT_BIT)
+#define E1000_FLAG_MAC (1 << E1000_FLAG_MAC_BIT)
     uint32_t compat_flags;
 } E1000State;
 
+#define chkflag(x)     (s->compat_flags & E1000_FLAG_##x)
+
 typedef struct E1000BaseClass {
     PCIDeviceClass parent_class;
     uint16_t phy_id2;
@@ -154,43 +144,28 @@
 #define E1000_DEVICE_GET_CLASS(obj) \
     OBJECT_GET_CLASS(E1000BaseClass, (obj), TYPE_E1000_BASE)
 
-#define	defreg(x)	x = (E1000_##x>>2)
-enum {
-    defreg(CTRL),	defreg(EECD),	defreg(EERD),	defreg(GPRC),
-    defreg(GPTC),	defreg(ICR),	defreg(ICS),	defreg(IMC),
-    defreg(IMS),	defreg(LEDCTL),	defreg(MANC),	defreg(MDIC),
-    defreg(MPC),	defreg(PBA),	defreg(RCTL),	defreg(RDBAH),
-    defreg(RDBAL),	defreg(RDH),	defreg(RDLEN),	defreg(RDT),
-    defreg(STATUS),	defreg(SWSM),	defreg(TCTL),	defreg(TDBAH),
-    defreg(TDBAL),	defreg(TDH),	defreg(TDLEN),	defreg(TDT),
-    defreg(TORH),	defreg(TORL),	defreg(TOTH),	defreg(TOTL),
-    defreg(TPR),	defreg(TPT),	defreg(TXDCTL),	defreg(WUFC),
-    defreg(RA),		defreg(MTA),	defreg(CRCERRS),defreg(VFTA),
-    defreg(VET),        defreg(RDTR),   defreg(RADV),   defreg(TADV),
-    defreg(ITR),
-};
-
-static void
-e1000_link_down(E1000State *s)
-{
-    s->mac_reg[STATUS] &= ~E1000_STATUS_LU;
-    s->phy_reg[PHY_STATUS] &= ~MII_SR_LINK_STATUS;
-    s->phy_reg[PHY_STATUS] &= ~MII_SR_AUTONEG_COMPLETE;
-    s->phy_reg[PHY_LP_ABILITY] &= ~MII_LPAR_LPACK;
-}
-
 static void
 e1000_link_up(E1000State *s)
 {
-    s->mac_reg[STATUS] |= E1000_STATUS_LU;
-    s->phy_reg[PHY_STATUS] |= MII_SR_LINK_STATUS;
+    e1000x_update_regs_on_link_up(s->mac_reg, s->phy_reg);
+
+    /* E1000_STATUS_LU is tested by e1000_can_receive() */
+    qemu_flush_queued_packets(qemu_get_queue(s->nic));
+}
+
+static void
+e1000_autoneg_done(E1000State *s)
+{
+    e1000x_update_regs_on_autoneg_done(s->mac_reg, s->phy_reg);
+
+    /* E1000_STATUS_LU is tested by e1000_can_receive() */
+    qemu_flush_queued_packets(qemu_get_queue(s->nic));
 }
 
 static bool
 have_autoneg(E1000State *s)
 {
-    return (s->compat_flags & E1000_FLAG_AUTONEG) &&
-           (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
+    return chkflag(AUTONEG) && (s->phy_reg[PHY_CTRL] & MII_CR_AUTO_NEG_EN);
 }
 
 static void
@@ -207,10 +182,7 @@
      * down.
      */
     if (have_autoneg(s) && (val & MII_CR_RESTART_AUTO_NEG)) {
-        e1000_link_down(s);
-        DBGOUT(PHY, "Start link auto negotiation\n");
-        timer_mod(s->autoneg_timer,
-                  qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
+        e1000x_restart_autoneg(s->mac_reg, s->phy_reg, s->autoneg_timer);
     }
 }
 
@@ -222,18 +194,18 @@
 
 enum { PHY_R = 1, PHY_W = 2, PHY_RW = PHY_R | PHY_W };
 static const char phy_regcap[0x20] = {
-    [PHY_STATUS] = PHY_R,	[M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW,
-    [PHY_ID1] = PHY_R,		[M88E1000_PHY_SPEC_CTRL] = PHY_RW,
-    [PHY_CTRL] = PHY_RW,	[PHY_1000T_CTRL] = PHY_RW,
-    [PHY_LP_ABILITY] = PHY_R,	[PHY_1000T_STATUS] = PHY_R,
-    [PHY_AUTONEG_ADV] = PHY_RW,	[M88E1000_RX_ERR_CNTR] = PHY_R,
-    [PHY_ID2] = PHY_R,		[M88E1000_PHY_SPEC_STATUS] = PHY_R,
+    [PHY_STATUS]      = PHY_R,     [M88E1000_EXT_PHY_SPEC_CTRL] = PHY_RW,
+    [PHY_ID1]         = PHY_R,     [M88E1000_PHY_SPEC_CTRL]     = PHY_RW,
+    [PHY_CTRL]        = PHY_RW,    [PHY_1000T_CTRL]             = PHY_RW,
+    [PHY_LP_ABILITY]  = PHY_R,     [PHY_1000T_STATUS]           = PHY_R,
+    [PHY_AUTONEG_ADV] = PHY_RW,    [M88E1000_RX_ERR_CNTR]       = PHY_R,
+    [PHY_ID2]         = PHY_R,     [M88E1000_PHY_SPEC_STATUS]   = PHY_R,
     [PHY_AUTONEG_EXP] = PHY_R,
 };
 
 /* PHY_ID2 documented in 8254x_GBe_SDM.pdf, pp. 250 */
 static const uint16_t phy_reg_init[] = {
-    [PHY_CTRL] =   MII_CR_SPEED_SELECT_MSB |
+    [PHY_CTRL]   = MII_CR_SPEED_SELECT_MSB |
                    MII_CR_FULL_DUPLEX |
                    MII_CR_AUTO_NEG_EN,
 
@@ -260,15 +232,15 @@
 };
 
 static const uint32_t mac_reg_init[] = {
-    [PBA] =     0x00100030,
-    [LEDCTL] =  0x602,
-    [CTRL] =    E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
+    [PBA]     = 0x00100030,
+    [LEDCTL]  = 0x602,
+    [CTRL]    = E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
                 E1000_CTRL_SPD_1000 | E1000_CTRL_SLU,
-    [STATUS] =  0x80000000 | E1000_STATUS_GIO_MASTER_ENABLE |
+    [STATUS]  = 0x80000000 | E1000_STATUS_GIO_MASTER_ENABLE |
                 E1000_STATUS_ASDV | E1000_STATUS_MTXCKOK |
                 E1000_STATUS_SPEED_1000 | E1000_STATUS_FD |
                 E1000_STATUS_LU,
-    [MANC] =    E1000_MANC_EN_MNG2HOST | E1000_MANC_RCV_TCO_EN |
+    [MANC]    = E1000_MANC_EN_MNG2HOST | E1000_MANC_RCV_TCO_EN |
                 E1000_MANC_ARP_EN | E1000_MANC_0298_EN |
                 E1000_MANC_RMCP_EN,
 };
@@ -315,7 +287,7 @@
         if (s->mit_timer_on) {
             return;
         }
-        if (s->compat_flags & E1000_FLAG_MIT) {
+        if (chkflag(MIT)) {
             /* Compute the next mitigation delay according to pending
              * interrupts and the current values of RADV (provided
              * RDTR!=0), TADV and ITR.
@@ -331,11 +303,17 @@
             }
             mit_update_delay(&mit_delay, s->mac_reg[ITR]);
 
-            if (mit_delay) {
-                s->mit_timer_on = 1;
-                timer_mod(s->mit_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                          mit_delay * 256);
-            }
+            /*
+             * According to e1000 SPEC, the Ethernet controller guarantees
+             * a maximum observable interrupt rate of 7813 interrupts/sec.
+             * Thus if mit_delay < 500 then the delay should be set to the
+             * minimum delay possible which is 500.
+             */
+            mit_delay = (mit_delay < 500) ? 500 : mit_delay;
+
+            s->mit_timer_on = 1;
+            timer_mod(s->mit_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                      mit_delay * 256);
             s->mit_ide = 0;
         }
     }
@@ -367,43 +345,16 @@
 {
     E1000State *s = opaque;
     if (!qemu_get_queue(s->nic)->link_down) {
-        e1000_link_up(s);
-        s->phy_reg[PHY_LP_ABILITY] |= MII_LPAR_LPACK;
-        s->phy_reg[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE;
-        DBGOUT(PHY, "Auto negotiation is completed\n");
+        e1000_autoneg_done(s);
         set_ics(s, 0, E1000_ICS_LSC); /* signal link status change to guest */
     }
 }
 
-static int
-rxbufsize(uint32_t v)
-{
-    v &= E1000_RCTL_BSEX | E1000_RCTL_SZ_16384 | E1000_RCTL_SZ_8192 |
-         E1000_RCTL_SZ_4096 | E1000_RCTL_SZ_2048 | E1000_RCTL_SZ_1024 |
-         E1000_RCTL_SZ_512 | E1000_RCTL_SZ_256;
-    switch (v) {
-    case E1000_RCTL_BSEX | E1000_RCTL_SZ_16384:
-        return 16384;
-    case E1000_RCTL_BSEX | E1000_RCTL_SZ_8192:
-        return 8192;
-    case E1000_RCTL_BSEX | E1000_RCTL_SZ_4096:
-        return 4096;
-    case E1000_RCTL_SZ_1024:
-        return 1024;
-    case E1000_RCTL_SZ_512:
-        return 512;
-    case E1000_RCTL_SZ_256:
-        return 256;
-    }
-    return 2048;
-}
-
 static void e1000_reset(void *opaque)
 {
     E1000State *d = opaque;
     E1000BaseClass *edc = E1000_DEVICE_GET_CLASS(d);
     uint8_t *macaddr = d->conf.macaddr.a;
-    int i;
 
     timer_del(d->autoneg_timer);
     timer_del(d->mit_timer);
@@ -419,17 +370,10 @@
     memset(&d->tx, 0, sizeof d->tx);
 
     if (qemu_get_queue(d->nic)->link_down) {
-        e1000_link_down(d);
+        e1000x_update_regs_on_link_down(d->mac_reg, d->phy_reg);
     }
 
-    /* Some guests expect pre-initialized RAH/RAL (AddrValid flag + MACaddr) */
-    d->mac_reg[RA] = 0;
-    d->mac_reg[RA + 1] = E1000_RAH_AV;
-    for (i = 0; i < 4; i++) {
-        d->mac_reg[RA] |= macaddr[i] << (8 * i);
-        d->mac_reg[RA + 1] |= (i < 2) ? macaddr[i + 4] << (8 * i) : 0;
-    }
-    qemu_format_nic_info_str(qemu_get_queue(d->nic), macaddr);
+    e1000x_reset_mac_addr(d->nic, d->mac_reg, macaddr);
 }
 
 static void
@@ -443,7 +387,7 @@
 set_rx_control(E1000State *s, int index, uint32_t val)
 {
     s->mac_reg[RCTL] = val;
-    s->rxbuf_size = rxbufsize(val);
+    s->rxbuf_size = e1000x_rxbufsize(val);
     s->rxbuf_min_shift = ((val / E1000_RCTL_RDMTS_QUAT) & 3) + 1;
     DBGOUT(RX, "RCTL: %d, mac_reg[RCTL] = 0x%x\n", s->mac_reg[RDT],
            s->mac_reg[RCTL]);
@@ -506,17 +450,19 @@
 
     s->eecd_state.old_eecd = val & (E1000_EECD_SK | E1000_EECD_CS |
             E1000_EECD_DI|E1000_EECD_FWE_MASK|E1000_EECD_REQ);
-    if (!(E1000_EECD_CS & val))			// CS inactive; nothing to do
-	return;
-    if (E1000_EECD_CS & (val ^ oldval)) {	// CS rise edge; reset state
-	s->eecd_state.val_in = 0;
-	s->eecd_state.bitnum_in = 0;
-	s->eecd_state.bitnum_out = 0;
-	s->eecd_state.reading = 0;
-    }
-    if (!(E1000_EECD_SK & (val ^ oldval)))	// no clock edge
+    if (!(E1000_EECD_CS & val)) {            /* CS inactive; nothing to do */
         return;
-    if (!(E1000_EECD_SK & val)) {		// falling edge
+    }
+    if (E1000_EECD_CS & (val ^ oldval)) {    /* CS rise edge; reset state */
+        s->eecd_state.val_in = 0;
+        s->eecd_state.bitnum_in = 0;
+        s->eecd_state.bitnum_out = 0;
+        s->eecd_state.reading = 0;
+    }
+    if (!(E1000_EECD_SK & (val ^ oldval))) {    /* no clock edge */
+        return;
+    }
+    if (!(E1000_EECD_SK & val)) {               /* falling edge */
         s->eecd_state.bitnum_out++;
         return;
     }
@@ -561,105 +507,97 @@
     }
 }
 
-static inline int
-vlan_enabled(E1000State *s)
+static inline void
+inc_tx_bcast_or_mcast_count(E1000State *s, const unsigned char *arr)
 {
-    return ((s->mac_reg[CTRL] & E1000_CTRL_VME) != 0);
-}
-
-static inline int
-vlan_rx_filter_enabled(E1000State *s)
-{
-    return ((s->mac_reg[RCTL] & E1000_RCTL_VFE) != 0);
-}
-
-static inline int
-is_vlan_packet(E1000State *s, const uint8_t *buf)
-{
-    return (be16_to_cpup((uint16_t *)(buf + 12)) ==
-                le16_to_cpup((uint16_t *)(s->mac_reg + VET)));
-}
-
-static inline int
-is_vlan_txd(uint32_t txd_lower)
-{
-    return ((txd_lower & E1000_TXD_CMD_VLE) != 0);
-}
-
-/* FCS aka Ethernet CRC-32. We don't get it from backends and can't
- * fill it in, just pad descriptor length by 4 bytes unless guest
- * told us to strip it off the packet. */
-static inline int
-fcs_len(E1000State *s)
-{
-    return (s->mac_reg[RCTL] & E1000_RCTL_SECRC) ? 0 : 4;
+    if (!memcmp(arr, bcast, sizeof bcast)) {
+        e1000x_inc_reg_if_not_full(s->mac_reg, BPTC);
+    } else if (arr[0] & 1) {
+        e1000x_inc_reg_if_not_full(s->mac_reg, MPTC);
+    }
 }
 
 static void
 e1000_send_packet(E1000State *s, const uint8_t *buf, int size)
 {
+    static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511,
+                                    PTC1023, PTC1522 };
+
     NetClientState *nc = qemu_get_queue(s->nic);
     if (s->phy_reg[PHY_CTRL] & MII_CR_LOOPBACK) {
         nc->info->receive(nc, buf, size);
     } else {
         qemu_send_packet(nc, buf, size);
     }
+    inc_tx_bcast_or_mcast_count(s, buf);
+    e1000x_increase_size_stats(s->mac_reg, PTCregs, size);
 }
 
 static void
 xmit_seg(E1000State *s)
 {
-    uint16_t len, *sp;
-    unsigned int frames = s->tx.tso_frames, css, sofar, n;
+    uint16_t len;
+    unsigned int frames = s->tx.tso_frames, css, sofar;
     struct e1000_tx *tp = &s->tx;
 
-    if (tp->tse && tp->cptse) {
-        css = tp->ipcss;
+    if (tp->props.tse && tp->props.cptse) {
+        css = tp->props.ipcss;
         DBGOUT(TXSUM, "frames %d size %d ipcss %d\n",
                frames, tp->size, css);
-        if (tp->ip) {		// IPv4
+        if (tp->props.ip) {    /* IPv4 */
             stw_be_p(tp->data+css+2, tp->size - css);
             stw_be_p(tp->data+css+4,
-                          be16_to_cpup((uint16_t *)(tp->data+css+4))+frames);
-        } else			// IPv6
+                     lduw_be_p(tp->data + css + 4) + frames);
+        } else {         /* IPv6 */
             stw_be_p(tp->data+css+4, tp->size - css);
-        css = tp->tucss;
+        }
+        css = tp->props.tucss;
         len = tp->size - css;
-        DBGOUT(TXSUM, "tcp %d tucss %d len %d\n", tp->tcp, css, len);
-        if (tp->tcp) {
-            sofar = frames * tp->mss;
+        DBGOUT(TXSUM, "tcp %d tucss %d len %d\n", tp->props.tcp, css, len);
+        if (tp->props.tcp) {
+            sofar = frames * tp->props.mss;
             stl_be_p(tp->data+css+4, ldl_be_p(tp->data+css+4)+sofar); /* seq */
-            if (tp->paylen - sofar > tp->mss)
-                tp->data[css + 13] &= ~9;		// PSH, FIN
-        } else	// UDP
+            if (tp->props.paylen - sofar > tp->props.mss) {
+                tp->data[css + 13] &= ~9;    /* PSH, FIN */
+            } else if (frames) {
+                e1000x_inc_reg_if_not_full(s->mac_reg, TSCTC);
+            }
+        } else    /* UDP */
             stw_be_p(tp->data+css+4, len);
-        if (tp->sum_needed & E1000_TXD_POPTS_TXSM) {
+        if (tp->props.sum_needed & E1000_TXD_POPTS_TXSM) {
             unsigned int phsum;
             // add pseudo-header length before checksum calculation
-            sp = (uint16_t *)(tp->data + tp->tucso);
-            phsum = be16_to_cpup(sp) + len;
+            void *sp = tp->data + tp->props.tucso;
+
+            phsum = lduw_be_p(sp) + len;
             phsum = (phsum >> 16) + (phsum & 0xffff);
             stw_be_p(sp, phsum);
         }
         tp->tso_frames++;
     }
 
-    if (tp->sum_needed & E1000_TXD_POPTS_TXSM)
-        putsum(tp->data, tp->size, tp->tucso, tp->tucss, tp->tucse);
-    if (tp->sum_needed & E1000_TXD_POPTS_IXSM)
-        putsum(tp->data, tp->size, tp->ipcso, tp->ipcss, tp->ipcse);
+    if (tp->props.sum_needed & E1000_TXD_POPTS_TXSM) {
+        putsum(tp->data, tp->size, tp->props.tucso,
+               tp->props.tucss, tp->props.tucse);
+    }
+    if (tp->props.sum_needed & E1000_TXD_POPTS_IXSM) {
+        putsum(tp->data, tp->size, tp->props.ipcso,
+               tp->props.ipcss, tp->props.ipcse);
+    }
     if (tp->vlan_needed) {
         memmove(tp->vlan, tp->data, 4);
         memmove(tp->data, tp->data + 4, 8);
         memcpy(tp->data + 8, tp->vlan_header, 4);
         e1000_send_packet(s, tp->vlan, tp->size + 4);
-    } else
+    } else {
         e1000_send_packet(s, tp->data, tp->size);
-    s->mac_reg[TPT]++;
-    s->mac_reg[GPTC]++;
-    n = s->mac_reg[TOTL];
-    if ((s->mac_reg[TOTL] += s->tx.size) < n)
-        s->mac_reg[TOTH]++;
+    }
+
+    e1000x_inc_reg_if_not_full(s->mac_reg, TPT);
+    e1000x_grow_8reg_if_not_full(s->mac_reg, TOTL, s->tx.size);
+    s->mac_reg[GPTC] = s->mac_reg[TPT];
+    s->mac_reg[GOTCL] = s->mac_reg[TOTL];
+    s->mac_reg[GOTCH] = s->mac_reg[TOTH];
 }
 
 static void
@@ -668,56 +606,45 @@
     PCIDevice *d = PCI_DEVICE(s);
     uint32_t txd_lower = le32_to_cpu(dp->lower.data);
     uint32_t dtype = txd_lower & (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D);
-    unsigned int split_size = txd_lower & 0xffff, bytes, sz, op;
+    unsigned int split_size = txd_lower & 0xffff, bytes, sz;
     unsigned int msh = 0xfffff;
     uint64_t addr;
     struct e1000_context_desc *xp = (struct e1000_context_desc *)dp;
     struct e1000_tx *tp = &s->tx;
 
     s->mit_ide |= (txd_lower & E1000_TXD_CMD_IDE);
-    if (dtype == E1000_TXD_CMD_DEXT) {	// context descriptor
-        op = le32_to_cpu(xp->cmd_and_length);
-        tp->ipcss = xp->lower_setup.ip_fields.ipcss;
-        tp->ipcso = xp->lower_setup.ip_fields.ipcso;
-        tp->ipcse = le16_to_cpu(xp->lower_setup.ip_fields.ipcse);
-        tp->tucss = xp->upper_setup.tcp_fields.tucss;
-        tp->tucso = xp->upper_setup.tcp_fields.tucso;
-        tp->tucse = le16_to_cpu(xp->upper_setup.tcp_fields.tucse);
-        tp->paylen = op & 0xfffff;
-        tp->hdr_len = xp->tcp_seg_setup.fields.hdr_len;
-        tp->mss = le16_to_cpu(xp->tcp_seg_setup.fields.mss);
-        tp->ip = (op & E1000_TXD_CMD_IP) ? 1 : 0;
-        tp->tcp = (op & E1000_TXD_CMD_TCP) ? 1 : 0;
-        tp->tse = (op & E1000_TXD_CMD_TSE) ? 1 : 0;
+    if (dtype == E1000_TXD_CMD_DEXT) {    /* context descriptor */
+        e1000x_read_tx_ctx_descr(xp, &tp->props);
         tp->tso_frames = 0;
-        if (tp->tucso == 0) {	// this is probably wrong
+        if (tp->props.tucso == 0) {    /* this is probably wrong */
             DBGOUT(TXSUM, "TCP/UDP: cso 0!\n");
-            tp->tucso = tp->tucss + (tp->tcp ? 16 : 6);
+            tp->props.tucso = tp->props.tucss + (tp->props.tcp ? 16 : 6);
         }
         return;
     } else if (dtype == (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D)) {
         // data descriptor
         if (tp->size == 0) {
-            tp->sum_needed = le32_to_cpu(dp->upper.data) >> 8;
+            tp->props.sum_needed = le32_to_cpu(dp->upper.data) >> 8;
         }
-        tp->cptse = ( txd_lower & E1000_TXD_CMD_TSE ) ? 1 : 0;
+        tp->props.cptse = (txd_lower & E1000_TXD_CMD_TSE) ? 1 : 0;
     } else {
         // legacy descriptor
-        tp->cptse = 0;
+        tp->props.cptse = 0;
     }
 
-    if (vlan_enabled(s) && is_vlan_txd(txd_lower) &&
-        (tp->cptse || txd_lower & E1000_TXD_CMD_EOP)) {
+    if (e1000x_vlan_enabled(s->mac_reg) &&
+        e1000x_is_vlan_txd(txd_lower) &&
+        (tp->props.cptse || txd_lower & E1000_TXD_CMD_EOP)) {
         tp->vlan_needed = 1;
         stw_be_p(tp->vlan_header,
-                      le16_to_cpup((uint16_t *)(s->mac_reg + VET)));
+                      le16_to_cpu(s->mac_reg[VET]));
         stw_be_p(tp->vlan_header + 2,
                       le16_to_cpu(dp->upper.fields.special));
     }
-        
+
     addr = le64_to_cpu(dp->buffer_addr);
-    if (tp->tse && tp->cptse) {
-        msh = tp->hdr_len + tp->mss;
+    if (tp->props.tse && tp->props.cptse) {
+        msh = tp->props.hdr_len + tp->props.mss;
         do {
             bytes = split_size;
             if (tp->size + bytes > msh)
@@ -726,18 +653,19 @@
             bytes = MIN(sizeof(tp->data) - tp->size, bytes);
             pci_dma_read(d, addr, tp->data + tp->size, bytes);
             sz = tp->size + bytes;
-            if (sz >= tp->hdr_len && tp->size < tp->hdr_len) {
-                memmove(tp->header, tp->data, tp->hdr_len);
+            if (sz >= tp->props.hdr_len && tp->size < tp->props.hdr_len) {
+                memmove(tp->header, tp->data, tp->props.hdr_len);
             }
             tp->size = sz;
             addr += bytes;
             if (sz == msh) {
                 xmit_seg(s);
-                memmove(tp->data, tp->header, tp->hdr_len);
-                tp->size = tp->hdr_len;
+                memmove(tp->data, tp->header, tp->props.hdr_len);
+                tp->size = tp->props.hdr_len;
             }
-        } while (split_size -= bytes);
-    } else if (!tp->tse && tp->cptse) {
+            split_size -= bytes;
+        } while (bytes && split_size);
+    } else if (!tp->props.tse && tp->props.cptse) {
         // context descriptor TSE is not set, while data descriptor TSE is set
         DBGOUT(TXERR, "TCP segmentation error\n");
     } else {
@@ -748,14 +676,14 @@
 
     if (!(txd_lower & E1000_TXD_CMD_EOP))
         return;
-    if (!(tp->tse && tp->cptse && tp->size < tp->hdr_len)) {
+    if (!(tp->props.tse && tp->props.cptse && tp->size < tp->props.hdr_len)) {
         xmit_seg(s);
     }
     tp->tso_frames = 0;
-    tp->sum_needed = 0;
+    tp->props.sum_needed = 0;
     tp->vlan_needed = 0;
     tp->size = 0;
-    tp->cptse = 0;
+    tp->props.cptse = 0;
 }
 
 static uint32_t
@@ -814,7 +742,8 @@
          * bogus values to TDT/TDLEN.
          * there's nothing too intelligent we could do about this.
          */
-        if (s->mac_reg[TDH] == tdh_start) {
+        if (s->mac_reg[TDH] == tdh_start ||
+            tdh_start >= s->mac_reg[TDLEN] / sizeof(desc)) {
             DBGOUT(TXERR, "TDH wraparound @%x, TDT %x, TDLEN %x\n",
                    tdh_start, s->mac_reg[TDT], s->mac_reg[TDLEN]);
             break;
@@ -826,54 +755,33 @@
 static int
 receive_filter(E1000State *s, const uint8_t *buf, int size)
 {
-    static const uint8_t bcast[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
-    static const int mta_shift[] = {4, 3, 2, 0};
-    uint32_t f, rctl = s->mac_reg[RCTL], ra[2], *rp;
+    uint32_t rctl = s->mac_reg[RCTL];
+    int isbcast = !memcmp(buf, bcast, sizeof bcast), ismcast = (buf[0] & 1);
 
-    if (is_vlan_packet(s, buf) && vlan_rx_filter_enabled(s)) {
-        uint16_t vid = be16_to_cpup((uint16_t *)(buf + 14));
-        uint32_t vfta = le32_to_cpup((uint32_t *)(s->mac_reg + VFTA) +
-                                     ((vid >> 5) & 0x7f));
+    if (e1000x_is_vlan_packet(buf, le16_to_cpu(s->mac_reg[VET])) &&
+        e1000x_vlan_rx_filter_enabled(s->mac_reg)) {
+        uint16_t vid = lduw_be_p(buf + 14);
+        uint32_t vfta = ldl_le_p((uint32_t*)(s->mac_reg + VFTA) +
+                                 ((vid >> 5) & 0x7f));
         if ((vfta & (1 << (vid & 0x1f))) == 0)
             return 0;
     }
 
-    if (rctl & E1000_RCTL_UPE)			// promiscuous
+    if (!isbcast && !ismcast && (rctl & E1000_RCTL_UPE)) { /* promiscuous ucast */
         return 1;
-
-    if ((buf[0] & 1) && (rctl & E1000_RCTL_MPE))	// promiscuous mcast
-        return 1;
-
-    if ((rctl & E1000_RCTL_BAM) && !memcmp(buf, bcast, sizeof bcast))
-        return 1;
-
-    for (rp = s->mac_reg + RA; rp < s->mac_reg + RA + 32; rp += 2) {
-        if (!(rp[1] & E1000_RAH_AV))
-            continue;
-        ra[0] = cpu_to_le32(rp[0]);
-        ra[1] = cpu_to_le32(rp[1]);
-        if (!memcmp(buf, (uint8_t *)ra, 6)) {
-            DBGOUT(RXFILTER,
-                   "unicast match[%d]: %02x:%02x:%02x:%02x:%02x:%02x\n",
-                   (int)(rp - s->mac_reg - RA)/2,
-                   buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
-            return 1;
-        }
     }
-    DBGOUT(RXFILTER, "unicast mismatch: %02x:%02x:%02x:%02x:%02x:%02x\n",
-           buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]);
 
-    f = mta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
-    f = (((buf[5] << 8) | buf[4]) >> f) & 0xfff;
-    if (s->mac_reg[MTA + (f >> 5)] & (1 << (f & 0x1f)))
+    if (ismcast && (rctl & E1000_RCTL_MPE)) {          /* promiscuous mcast */
+        e1000x_inc_reg_if_not_full(s->mac_reg, MPRC);
         return 1;
-    DBGOUT(RXFILTER,
-           "dropping, inexact filter mismatch: %02x:%02x:%02x:%02x:%02x:%02x MO %d MTA[%d] %x\n",
-           buf[0], buf[1], buf[2], buf[3], buf[4], buf[5],
-           (rctl >> E1000_RCTL_MO_SHIFT) & 3, f >> 5,
-           s->mac_reg[MTA + (f >> 5)]);
+    }
 
-    return 0;
+    if (isbcast && (rctl & E1000_RCTL_BAM)) {          /* broadcast enabled */
+        e1000x_inc_reg_if_not_full(s->mac_reg, BPRC);
+        return 1;
+    }
+
+    return e1000x_rx_group_filter(s->mac_reg, buf);
 }
 
 static void
@@ -883,13 +791,11 @@
     uint32_t old_status = s->mac_reg[STATUS];
 
     if (nc->link_down) {
-        e1000_link_down(s);
+        e1000x_update_regs_on_link_down(s->mac_reg, s->phy_reg);
     } else {
         if (have_autoneg(s) &&
             !(s->phy_reg[PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) {
-            /* emulate auto-negotiation if supported */
-            timer_mod(s->autoneg_timer,
-                      qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
+            e1000x_restart_autoneg(s->mac_reg, s->phy_reg, s->autoneg_timer);
         } else {
             e1000_link_up(s);
         }
@@ -922,8 +828,8 @@
 {
     E1000State *s = qemu_get_nic_opaque(nc);
 
-    return (s->mac_reg[STATUS] & E1000_STATUS_LU) &&
-        (s->mac_reg[RCTL] & E1000_RCTL_EN) && e1000_has_rxbufs(s, 1);
+    return e1000x_rx_ready(&s->parent_obj, s->mac_reg) &&
+        e1000_has_rxbufs(s, 1);
 }
 
 static uint64_t rx_desc_base(E1000State *s)
@@ -954,11 +860,7 @@
     size_t desc_size;
     size_t total_size;
 
-    if (!(s->mac_reg[STATUS] & E1000_STATUS_LU)) {
-        return -1;
-    }
-
-    if (!(s->mac_reg[RCTL] & E1000_RCTL_EN)) {
+    if (!e1000x_hw_rx_enabled(s->mac_reg)) {
         return -1;
     }
 
@@ -966,6 +868,7 @@
     if (size < sizeof(min_buf)) {
         iov_to_buf(iov, iovcnt, 0, min_buf, size);
         memset(&min_buf[size], 0, sizeof(min_buf) - size);
+        e1000x_inc_reg_if_not_full(s->mac_reg, RUC);
         min_iov.iov_base = filter_buf = min_buf;
         min_iov.iov_len = size = sizeof(min_buf);
         iovcnt = 1;
@@ -977,10 +880,7 @@
     }
 
     /* Discard oversized packets if !LPE and !SBP. */
-    if ((size > MAXIMUM_ETHERNET_LPE_SIZE ||
-        (size > MAXIMUM_ETHERNET_VLAN_SIZE
-        && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)))
-        && !(s->mac_reg[RCTL] & E1000_RCTL_SBP)) {
+    if (e1000x_is_oversized(s->mac_reg, size)) {
         return size;
     }
 
@@ -988,9 +888,9 @@
         return size;
     }
 
-    if (vlan_enabled(s) && is_vlan_packet(s, filter_buf)) {
-        vlan_special = cpu_to_le16(be16_to_cpup((uint16_t *)(filter_buf
-                                                                + 14)));
+    if (e1000x_vlan_enabled(s->mac_reg) &&
+        e1000x_is_vlan_packet(filter_buf, le16_to_cpu(s->mac_reg[VET]))) {
+        vlan_special = cpu_to_le16(lduw_be_p(filter_buf + 14));
         iov_ofs = 4;
         if (filter_buf == iov->iov_base) {
             memmove(filter_buf + 4, filter_buf, 12);
@@ -1007,7 +907,7 @@
 
     rdh_start = s->mac_reg[RDH];
     desc_offset = 0;
-    total_size = size + fcs_len(s);
+    total_size = size + e1000x_fcs_len(s->mac_reg);
     if (!e1000_has_rxbufs(s, total_size)) {
             set_ics(s, 0, E1000_ICS_RXO);
             return -1;
@@ -1058,7 +958,8 @@
         if (++s->mac_reg[RDH] * sizeof(desc) >= s->mac_reg[RDLEN])
             s->mac_reg[RDH] = 0;
         /* see comment in start_xmit; same here */
-        if (s->mac_reg[RDH] == rdh_start) {
+        if (s->mac_reg[RDH] == rdh_start ||
+            rdh_start >= s->mac_reg[RDLEN] / sizeof(desc)) {
             DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
                    rdh_start, s->mac_reg[RDT], s->mac_reg[RDLEN]);
             set_ics(s, 0, E1000_ICS_RXO);
@@ -1066,16 +967,7 @@
         }
     } while (desc_offset < total_size);
 
-    s->mac_reg[GPRC]++;
-    s->mac_reg[TPR]++;
-    /* TOR - Total Octets Received:
-     * This register includes bytes received in a packet from the <Destination
-     * Address> field through the <CRC> field, inclusively.
-     */
-    n = s->mac_reg[TORL] + size + /* Always include FCS length. */ 4;
-    if (n < s->mac_reg[TORL])
-        s->mac_reg[TORH]++;
-    s->mac_reg[TORL] = n;
+    e1000x_update_rx_total_stats(s->mac_reg, size, total_size);
 
     n = E1000_ICS_RXT0;
     if ((rdt = s->mac_reg[RDT]) < s->mac_reg[RDH])
@@ -1107,6 +999,30 @@
 }
 
 static uint32_t
+mac_low4_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0xf;
+}
+
+static uint32_t
+mac_low11_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0x7ff;
+}
+
+static uint32_t
+mac_low13_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0x1fff;
+}
+
+static uint32_t
+mac_low16_read(E1000State *s, int index)
+{
+    return s->mac_reg[index] & 0xffff;
+}
+
+static uint32_t
 mac_icr_read(E1000State *s, int index)
 {
     uint32_t ret = s->mac_reg[ICR];
@@ -1199,46 +1115,144 @@
     set_ics(s, 0, 0);
 }
 
-#define getreg(x)	[x] = mac_readreg
+#define getreg(x)    [x] = mac_readreg
 static uint32_t (*macreg_readops[])(E1000State *, int) = {
-    getreg(PBA),	getreg(RCTL),	getreg(TDH),	getreg(TXDCTL),
-    getreg(WUFC),	getreg(TDT),	getreg(CTRL),	getreg(LEDCTL),
-    getreg(MANC),	getreg(MDIC),	getreg(SWSM),	getreg(STATUS),
-    getreg(TORL),	getreg(TOTL),	getreg(IMS),	getreg(TCTL),
-    getreg(RDH),	getreg(RDT),	getreg(VET),	getreg(ICS),
-    getreg(TDBAL),	getreg(TDBAH),	getreg(RDBAH),	getreg(RDBAL),
-    getreg(TDLEN),      getreg(RDLEN),  getreg(RDTR),   getreg(RADV),
-    getreg(TADV),       getreg(ITR),
+    getreg(PBA),      getreg(RCTL),     getreg(TDH),      getreg(TXDCTL),
+    getreg(WUFC),     getreg(TDT),      getreg(CTRL),     getreg(LEDCTL),
+    getreg(MANC),     getreg(MDIC),     getreg(SWSM),     getreg(STATUS),
+    getreg(TORL),     getreg(TOTL),     getreg(IMS),      getreg(TCTL),
+    getreg(RDH),      getreg(RDT),      getreg(VET),      getreg(ICS),
+    getreg(TDBAL),    getreg(TDBAH),    getreg(RDBAH),    getreg(RDBAL),
+    getreg(TDLEN),    getreg(RDLEN),    getreg(RDTR),     getreg(RADV),
+    getreg(TADV),     getreg(ITR),      getreg(FCRUC),    getreg(IPAV),
+    getreg(WUC),      getreg(WUS),      getreg(SCC),      getreg(ECOL),
+    getreg(MCC),      getreg(LATECOL),  getreg(COLC),     getreg(DC),
+    getreg(TNCRS),    getreg(SEC),      getreg(CEXTERR),  getreg(RLEC),
+    getreg(XONRXC),   getreg(XONTXC),   getreg(XOFFRXC),  getreg(XOFFTXC),
+    getreg(RFC),      getreg(RJC),      getreg(RNBC),     getreg(TSCTFC),
+    getreg(MGTPRC),   getreg(MGTPDC),   getreg(MGTPTC),   getreg(GORCL),
+    getreg(GOTCL),
 
-    [TOTH] = mac_read_clr8,	[TORH] = mac_read_clr8,	[GPRC] = mac_read_clr4,
-    [GPTC] = mac_read_clr4,	[TPR] = mac_read_clr4,	[TPT] = mac_read_clr4,
-    [ICR] = mac_icr_read,	[EECD] = get_eecd,	[EERD] = flash_eerd_read,
-    [CRCERRS ... MPC] = &mac_readreg,
-    [RA ... RA+31] = &mac_readreg,
-    [MTA ... MTA+127] = &mac_readreg,
+    [TOTH]    = mac_read_clr8,      [TORH]    = mac_read_clr8,
+    [GOTCH]   = mac_read_clr8,      [GORCH]   = mac_read_clr8,
+    [PRC64]   = mac_read_clr4,      [PRC127]  = mac_read_clr4,
+    [PRC255]  = mac_read_clr4,      [PRC511]  = mac_read_clr4,
+    [PRC1023] = mac_read_clr4,      [PRC1522] = mac_read_clr4,
+    [PTC64]   = mac_read_clr4,      [PTC127]  = mac_read_clr4,
+    [PTC255]  = mac_read_clr4,      [PTC511]  = mac_read_clr4,
+    [PTC1023] = mac_read_clr4,      [PTC1522] = mac_read_clr4,
+    [GPRC]    = mac_read_clr4,      [GPTC]    = mac_read_clr4,
+    [TPT]     = mac_read_clr4,      [TPR]     = mac_read_clr4,
+    [RUC]     = mac_read_clr4,      [ROC]     = mac_read_clr4,
+    [BPRC]    = mac_read_clr4,      [MPRC]    = mac_read_clr4,
+    [TSCTC]   = mac_read_clr4,      [BPTC]    = mac_read_clr4,
+    [MPTC]    = mac_read_clr4,
+    [ICR]     = mac_icr_read,       [EECD]    = get_eecd,
+    [EERD]    = flash_eerd_read,
+    [RDFH]    = mac_low13_read,     [RDFT]    = mac_low13_read,
+    [RDFHS]   = mac_low13_read,     [RDFTS]   = mac_low13_read,
+    [RDFPC]   = mac_low13_read,
+    [TDFH]    = mac_low11_read,     [TDFT]    = mac_low11_read,
+    [TDFHS]   = mac_low13_read,     [TDFTS]   = mac_low13_read,
+    [TDFPC]   = mac_low13_read,
+    [AIT]     = mac_low16_read,
+
+    [CRCERRS ... MPC]   = &mac_readreg,
+    [IP6AT ... IP6AT+3] = &mac_readreg,    [IP4AT ... IP4AT+6] = &mac_readreg,
+    [FFLT ... FFLT+6]   = &mac_low11_read,
+    [RA ... RA+31]      = &mac_readreg,
+    [WUPM ... WUPM+31]  = &mac_readreg,
+    [MTA ... MTA+127]   = &mac_readreg,
     [VFTA ... VFTA+127] = &mac_readreg,
+    [FFMT ... FFMT+254] = &mac_low4_read,
+    [FFVT ... FFVT+254] = &mac_readreg,
+    [PBM ... PBM+16383] = &mac_readreg,
 };
 enum { NREADOPS = ARRAY_SIZE(macreg_readops) };
 
-#define putreg(x)	[x] = mac_writereg
+#define putreg(x)    [x] = mac_writereg
 static void (*macreg_writeops[])(E1000State *, int, uint32_t) = {
-    putreg(PBA),	putreg(EERD),	putreg(SWSM),	putreg(WUFC),
-    putreg(TDBAL),	putreg(TDBAH),	putreg(TXDCTL),	putreg(RDBAH),
-    putreg(RDBAL),	putreg(LEDCTL), putreg(VET),
-    [TDLEN] = set_dlen,	[RDLEN] = set_dlen,	[TCTL] = set_tctl,
-    [TDT] = set_tctl,	[MDIC] = set_mdic,	[ICS] = set_ics,
-    [TDH] = set_16bit,	[RDH] = set_16bit,	[RDT] = set_rdt,
-    [IMC] = set_imc,	[IMS] = set_ims,	[ICR] = set_icr,
-    [EECD] = set_eecd,	[RCTL] = set_rx_control, [CTRL] = set_ctrl,
-    [RDTR] = set_16bit, [RADV] = set_16bit,     [TADV] = set_16bit,
-    [ITR] = set_16bit,
-    [RA ... RA+31] = &mac_writereg,
-    [MTA ... MTA+127] = &mac_writereg,
+    putreg(PBA),      putreg(EERD),     putreg(SWSM),     putreg(WUFC),
+    putreg(TDBAL),    putreg(TDBAH),    putreg(TXDCTL),   putreg(RDBAH),
+    putreg(RDBAL),    putreg(LEDCTL),   putreg(VET),      putreg(FCRUC),
+    putreg(TDFH),     putreg(TDFT),     putreg(TDFHS),    putreg(TDFTS),
+    putreg(TDFPC),    putreg(RDFH),     putreg(RDFT),     putreg(RDFHS),
+    putreg(RDFTS),    putreg(RDFPC),    putreg(IPAV),     putreg(WUC),
+    putreg(WUS),      putreg(AIT),
+
+    [TDLEN]  = set_dlen,   [RDLEN]  = set_dlen,       [TCTL] = set_tctl,
+    [TDT]    = set_tctl,   [MDIC]   = set_mdic,       [ICS]  = set_ics,
+    [TDH]    = set_16bit,  [RDH]    = set_16bit,      [RDT]  = set_rdt,
+    [IMC]    = set_imc,    [IMS]    = set_ims,        [ICR]  = set_icr,
+    [EECD]   = set_eecd,   [RCTL]   = set_rx_control, [CTRL] = set_ctrl,
+    [RDTR]   = set_16bit,  [RADV]   = set_16bit,      [TADV] = set_16bit,
+    [ITR]    = set_16bit,
+
+    [IP6AT ... IP6AT+3] = &mac_writereg, [IP4AT ... IP4AT+6] = &mac_writereg,
+    [FFLT ... FFLT+6]   = &mac_writereg,
+    [RA ... RA+31]      = &mac_writereg,
+    [WUPM ... WUPM+31]  = &mac_writereg,
+    [MTA ... MTA+127]   = &mac_writereg,
     [VFTA ... VFTA+127] = &mac_writereg,
+    [FFMT ... FFMT+254] = &mac_writereg, [FFVT ... FFVT+254] = &mac_writereg,
+    [PBM ... PBM+16383] = &mac_writereg,
 };
 
 enum { NWRITEOPS = ARRAY_SIZE(macreg_writeops) };
 
+enum { MAC_ACCESS_PARTIAL = 1, MAC_ACCESS_FLAG_NEEDED = 2 };
+
+#define markflag(x)    ((E1000_FLAG_##x << 2) | MAC_ACCESS_FLAG_NEEDED)
+/* In the array below the meaning of the bits is: [f|f|f|f|f|f|n|p]
+ * f - flag bits (up to 6 possible flags)
+ * n - flag needed
+ * p - partially implenented */
+static const uint8_t mac_reg_access[0x8000] = {
+    [RDTR]    = markflag(MIT),    [TADV]    = markflag(MIT),
+    [RADV]    = markflag(MIT),    [ITR]     = markflag(MIT),
+
+    [IPAV]    = markflag(MAC),    [WUC]     = markflag(MAC),
+    [IP6AT]   = markflag(MAC),    [IP4AT]   = markflag(MAC),
+    [FFVT]    = markflag(MAC),    [WUPM]    = markflag(MAC),
+    [ECOL]    = markflag(MAC),    [MCC]     = markflag(MAC),
+    [DC]      = markflag(MAC),    [TNCRS]   = markflag(MAC),
+    [RLEC]    = markflag(MAC),    [XONRXC]  = markflag(MAC),
+    [XOFFTXC] = markflag(MAC),    [RFC]     = markflag(MAC),
+    [TSCTFC]  = markflag(MAC),    [MGTPRC]  = markflag(MAC),
+    [WUS]     = markflag(MAC),    [AIT]     = markflag(MAC),
+    [FFLT]    = markflag(MAC),    [FFMT]    = markflag(MAC),
+    [SCC]     = markflag(MAC),    [FCRUC]   = markflag(MAC),
+    [LATECOL] = markflag(MAC),    [COLC]    = markflag(MAC),
+    [SEC]     = markflag(MAC),    [CEXTERR] = markflag(MAC),
+    [XONTXC]  = markflag(MAC),    [XOFFRXC] = markflag(MAC),
+    [RJC]     = markflag(MAC),    [RNBC]    = markflag(MAC),
+    [MGTPDC]  = markflag(MAC),    [MGTPTC]  = markflag(MAC),
+    [RUC]     = markflag(MAC),    [ROC]     = markflag(MAC),
+    [GORCL]   = markflag(MAC),    [GORCH]   = markflag(MAC),
+    [GOTCL]   = markflag(MAC),    [GOTCH]   = markflag(MAC),
+    [BPRC]    = markflag(MAC),    [MPRC]    = markflag(MAC),
+    [TSCTC]   = markflag(MAC),    [PRC64]   = markflag(MAC),
+    [PRC127]  = markflag(MAC),    [PRC255]  = markflag(MAC),
+    [PRC511]  = markflag(MAC),    [PRC1023] = markflag(MAC),
+    [PRC1522] = markflag(MAC),    [PTC64]   = markflag(MAC),
+    [PTC127]  = markflag(MAC),    [PTC255]  = markflag(MAC),
+    [PTC511]  = markflag(MAC),    [PTC1023] = markflag(MAC),
+    [PTC1522] = markflag(MAC),    [MPTC]    = markflag(MAC),
+    [BPTC]    = markflag(MAC),
+
+    [TDFH]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFT]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFHS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFTS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [TDFPC] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFH]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFT]  = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFHS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFTS] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [RDFPC] = markflag(MAC) | MAC_ACCESS_PARTIAL,
+    [PBM]   = markflag(MAC) | MAC_ACCESS_PARTIAL,
+};
+
 static void
 e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned size)
@@ -1247,9 +1261,20 @@
     unsigned int index = (addr & 0x1ffff) >> 2;
 
     if (index < NWRITEOPS && macreg_writeops[index]) {
-        macreg_writeops[index](s, index, val);
+        if (!(mac_reg_access[index] & MAC_ACCESS_FLAG_NEEDED)
+            || (s->compat_flags & (mac_reg_access[index] >> 2))) {
+            if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+                DBGOUT(GENERAL, "Writing to register at offset: 0x%08x. "
+                       "It is not fully implemented.\n", index<<2);
+            }
+            macreg_writeops[index](s, index, val);
+        } else {    /* "flag needed" bit is set, but the flag is not active */
+            DBGOUT(MMIO, "MMIO write attempt to disabled reg. addr=0x%08x\n",
+                   index<<2);
+        }
     } else if (index < NREADOPS && macreg_readops[index]) {
-        DBGOUT(MMIO, "e1000_mmio_writel RO %x: 0x%04"PRIx64"\n", index<<2, val);
+        DBGOUT(MMIO, "e1000_mmio_writel RO %x: 0x%04"PRIx64"\n",
+               index<<2, val);
     } else {
         DBGOUT(UNKNOWN, "MMIO unknown write addr=0x%08x,val=0x%08"PRIx64"\n",
                index<<2, val);
@@ -1262,11 +1287,21 @@
     E1000State *s = opaque;
     unsigned int index = (addr & 0x1ffff) >> 2;
 
-    if (index < NREADOPS && macreg_readops[index])
-    {
-        return macreg_readops[index](s, index);
+    if (index < NREADOPS && macreg_readops[index]) {
+        if (!(mac_reg_access[index] & MAC_ACCESS_FLAG_NEEDED)
+            || (s->compat_flags & (mac_reg_access[index] >> 2))) {
+            if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+                DBGOUT(GENERAL, "Reading register at offset: 0x%08x. "
+                       "It is not fully implemented.\n", index<<2);
+            }
+            return macreg_readops[index](s, index);
+        } else {    /* "flag needed" bit is set, but the flag is not active */
+            DBGOUT(MMIO, "MMIO read attempt of disabled reg. addr=0x%08x\n",
+                   index<<2);
+        }
+    } else {
+        DBGOUT(UNKNOWN, "MMIO unknown read addr=0x%08x\n", index<<2);
     }
-    DBGOUT(UNKNOWN, "MMIO unknown read addr=0x%08x\n", index<<2);
     return 0;
 }
 
@@ -1333,7 +1368,7 @@
     E1000State *s = opaque;
     NetClientState *nc = qemu_get_queue(s->nic);
 
-    if (!(s->compat_flags & E1000_FLAG_MIT)) {
+    if (!chkflag(MIT)) {
         s->mac_reg[ITR] = s->mac_reg[RDTR] = s->mac_reg[RADV] =
             s->mac_reg[TADV] = 0;
         s->mit_irq_level = false;
@@ -1360,13 +1395,21 @@
 {
     E1000State *s = opaque;
 
-    return s->compat_flags & E1000_FLAG_MIT;
+    return chkflag(MIT);
+}
+
+static bool e1000_full_mac_needed(void *opaque)
+{
+    E1000State *s = opaque;
+
+    return chkflag(MAC);
 }
 
 static const VMStateDescription vmstate_e1000_mit_state = {
     .name = "e1000/mit_state",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = e1000_mit_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(mac_reg[RDTR], E1000State),
         VMSTATE_UINT32(mac_reg[RADV], E1000State),
@@ -1377,6 +1420,17 @@
     }
 };
 
+static const VMStateDescription vmstate_e1000_full_mac_state = {
+    .name = "e1000/full_mac_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = e1000_full_mac_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(mac_reg, E1000State, 0x8000),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_e1000 = {
     .name = "e1000",
     .version_id = 2,
@@ -1394,20 +1448,20 @@
         VMSTATE_UINT16(eecd_state.bitnum_out, E1000State),
         VMSTATE_UINT16(eecd_state.reading, E1000State),
         VMSTATE_UINT32(eecd_state.old_eecd, E1000State),
-        VMSTATE_UINT8(tx.ipcss, E1000State),
-        VMSTATE_UINT8(tx.ipcso, E1000State),
-        VMSTATE_UINT16(tx.ipcse, E1000State),
-        VMSTATE_UINT8(tx.tucss, E1000State),
-        VMSTATE_UINT8(tx.tucso, E1000State),
-        VMSTATE_UINT16(tx.tucse, E1000State),
-        VMSTATE_UINT32(tx.paylen, E1000State),
-        VMSTATE_UINT8(tx.hdr_len, E1000State),
-        VMSTATE_UINT16(tx.mss, E1000State),
+        VMSTATE_UINT8(tx.props.ipcss, E1000State),
+        VMSTATE_UINT8(tx.props.ipcso, E1000State),
+        VMSTATE_UINT16(tx.props.ipcse, E1000State),
+        VMSTATE_UINT8(tx.props.tucss, E1000State),
+        VMSTATE_UINT8(tx.props.tucso, E1000State),
+        VMSTATE_UINT16(tx.props.tucse, E1000State),
+        VMSTATE_UINT32(tx.props.paylen, E1000State),
+        VMSTATE_UINT8(tx.props.hdr_len, E1000State),
+        VMSTATE_UINT16(tx.props.mss, E1000State),
         VMSTATE_UINT16(tx.size, E1000State),
         VMSTATE_UINT16(tx.tso_frames, E1000State),
-        VMSTATE_UINT8(tx.sum_needed, E1000State),
-        VMSTATE_INT8(tx.ip, E1000State),
-        VMSTATE_INT8(tx.tcp, E1000State),
+        VMSTATE_UINT8(tx.props.sum_needed, E1000State),
+        VMSTATE_INT8(tx.props.ip, E1000State),
+        VMSTATE_INT8(tx.props.tcp, E1000State),
         VMSTATE_BUFFER(tx.header, E1000State),
         VMSTATE_BUFFER(tx.data, E1000State),
         VMSTATE_UINT16_ARRAY(eeprom_data, E1000State, 64),
@@ -1454,13 +1508,10 @@
         VMSTATE_UINT32_SUB_ARRAY(mac_reg, E1000State, VFTA, 128),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_e1000_mit_state,
-            .needed = e1000_mit_state_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_e1000_mit_state,
+        &vmstate_e1000_full_mac_state,
+        NULL
     }
 };
 
@@ -1500,14 +1551,6 @@
 }
 
 static void
-e1000_cleanup(NetClientState *nc)
-{
-    E1000State *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
-static void
 pci_e1000_uninit(PCIDevice *dev)
 {
     E1000State *d = E1000(dev);
@@ -1520,25 +1563,36 @@
 }
 
 static NetClientInfo net_e1000_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = e1000_can_receive,
     .receive = e1000_receive,
     .receive_iov = e1000_receive_iov,
-    .cleanup = e1000_cleanup,
     .link_status_changed = e1000_set_link_status,
 };
 
-static int pci_e1000_init(PCIDevice *pci_dev)
+static void e1000_write_config(PCIDevice *pci_dev, uint32_t address,
+                                uint32_t val, int len)
+{
+    E1000State *s = E1000(pci_dev);
+
+    pci_default_write_config(pci_dev, address, val, len);
+
+    if (range_covers_byte(address, len, PCI_COMMAND) &&
+        (pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+}
+
+static void pci_e1000_realize(PCIDevice *pci_dev, Error **errp)
 {
     DeviceState *dev = DEVICE(pci_dev);
     E1000State *d = E1000(pci_dev);
-    PCIDeviceClass *pdc = PCI_DEVICE_GET_CLASS(pci_dev);
     uint8_t *pci_conf;
-    uint16_t checksum = 0;
-    int i;
     uint8_t *macaddr;
 
+    pci_dev->config_write = e1000_write_config;
+
     pci_conf = pci_dev->config;
 
     /* TODO: RST# value should be 0, PCI spec 6.2.4 */
@@ -1552,17 +1606,14 @@
 
     pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_SPACE_IO, &d->io);
 
-    memmove(d->eeprom_data, e1000_eeprom_template,
-        sizeof e1000_eeprom_template);
     qemu_macaddr_default_if_unset(&d->conf.macaddr);
     macaddr = d->conf.macaddr.a;
-    for (i = 0; i < 3; i++)
-        d->eeprom_data[i] = (macaddr[2*i+1]<<8) | macaddr[2*i];
-    d->eeprom_data[11] = d->eeprom_data[13] = pdc->device_id;
-    for (i = 0; i < EEPROM_CHECKSUM_REG; i++)
-        checksum += d->eeprom_data[i];
-    checksum = (uint16_t) EEPROM_SUM - checksum;
-    d->eeprom_data[EEPROM_CHECKSUM_REG] = checksum;
+
+    e1000x_core_prepare_eeprom(d->eeprom_data,
+                               e1000_eeprom_template,
+                               sizeof(e1000_eeprom_template),
+                               PCI_DEVICE_GET_CLASS(pci_dev)->device_id,
+                               macaddr);
 
     d->nic = qemu_new_nic(&net_e1000_info, &d->conf,
                           object_get_typename(OBJECT(d)), dev->id, d);
@@ -1571,8 +1622,6 @@
 
     d->autoneg_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, e1000_autoneg_timer, d);
     d->mit_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000_mit_timer, d);
-
-    return 0;
 }
 
 static void qdev_e1000_reset(DeviceState *dev)
@@ -1587,6 +1636,8 @@
                     compat_flags, E1000_FLAG_AUTONEG_BIT, true),
     DEFINE_PROP_BIT("mitigation", E1000State,
                     compat_flags, E1000_FLAG_MIT_BIT, true),
+    DEFINE_PROP_BIT("extra_mac_registers", E1000State,
+                    compat_flags, E1000_FLAG_MAC_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1604,7 +1655,7 @@
     E1000BaseClass *e = E1000_DEVICE_CLASS(klass);
     const E1000Info *info = data;
 
-    k->init = pci_e1000_init;
+    k->realize = pci_e1000_realize;
     k->exit = pci_e1000_uninit;
     k->romfile = "efi-e1000.rom";
     k->vendor_id = PCI_VENDOR_ID_INTEL;
@@ -1638,7 +1689,7 @@
 
 static const E1000Info e1000_devices[] = {
     {
-        .name      = "e1000-82540em",
+        .name      = "e1000",
         .device_id = E1000_DEV_ID_82540EM,
         .revision  = 0x03,
         .phy_id2   = E1000_PHY_ID2_8254xx_DEFAULT,
@@ -1657,11 +1708,6 @@
     },
 };
 
-static const TypeInfo e1000_default_info = {
-    .name          = "e1000",
-    .parent        = "e1000-82540em",
-};
-
 static void e1000_register_types(void)
 {
     int i;
@@ -1679,7 +1725,6 @@
 
         type_register(&type_info);
     }
-    type_register_static(&e1000_default_info);
 }
 
 type_init(e1000_register_types)
diff --git a/hw/net/e1000_regs.h b/hw/net/e1000_regs.h
index 60b96aa..23eed50 100644
--- a/hw/net/e1000_regs.h
+++ b/hw/net/e1000_regs.h
@@ -29,9 +29,8 @@
  * Structures, enums, and macros for the MAC
  */
 
-#ifndef _E1000_HW_H_
-#define _E1000_HW_H_
-
+#ifndef HW_E1000_REGS_H
+#define HW_E1000_REGS_H
 
 /* PCI Device IDs */
 #define E1000_DEV_ID_82542               0x1000
@@ -85,6 +84,7 @@
 #define E1000_DEV_ID_82573E              0x108B
 #define E1000_DEV_ID_82573E_IAMT         0x108C
 #define E1000_DEV_ID_82573L              0x109A
+#define E1000_DEV_ID_82574L              0x10D3
 #define E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3 0x10B5
 #define E1000_DEV_ID_80003ES2LAN_COPPER_DPT     0x1096
 #define E1000_DEV_ID_80003ES2LAN_SERDES_DPT     0x1098
@@ -104,6 +104,7 @@
 #define E1000_PHY_ID2_82544x 0xC30
 #define E1000_PHY_ID2_8254xx_DEFAULT 0xC20 /* 82540x, 82545x, and 82546x */
 #define E1000_PHY_ID2_82573x 0xCC0
+#define E1000_PHY_ID2_82574x 0xCB1
 
 /* Register Set. (82543, 82544)
  *
@@ -135,8 +136,11 @@
 #define E1000_ITR      0x000C4  /* Interrupt Throttling Rate - RW */
 #define E1000_ICS      0x000C8  /* Interrupt Cause Set - WO */
 #define E1000_IMS      0x000D0  /* Interrupt Mask Set - RW */
+#define E1000_EIAC     0x000DC  /* Ext. Interrupt Auto Clear - RW */
 #define E1000_IMC      0x000D8  /* Interrupt Mask Clear - WO */
 #define E1000_IAM      0x000E0  /* Interrupt Acknowledge Auto Mask */
+#define E1000_IVAR     0x000E4  /* Interrupt Vector Allocation Register - RW */
+#define E1000_EITR     0x000E8  /* Extended Interrupt Throttling Rate - RW */
 #define E1000_RCTL     0x00100  /* RX Control - RW */
 #define E1000_RDTR1    0x02820  /* RX Delay Timer (1) - RW */
 #define E1000_RDBAL1   0x02900  /* RX Descriptor Base Address Low (1) - RW */
@@ -145,6 +149,7 @@
 #define E1000_RDH1     0x02910  /* RX Descriptor Head (1) - RW */
 #define E1000_RDT1     0x02918  /* RX Descriptor Tail (1) - RW */
 #define E1000_FCTTV    0x00170  /* Flow Control Transmit Timer Value - RW */
+#define E1000_FCRTV    0x05F40  /* Flow Control Refresh Timer Value - RW */
 #define E1000_TXCW     0x00178  /* TX Configuration Word - RW */
 #define E1000_RXCW     0x00180  /* RX Configuration Word - RO */
 #define E1000_TCTL     0x00400  /* TX Control - RW */
@@ -158,8 +163,13 @@
 #define E1000_PHY_CTRL     0x00F10  /* PHY Control Register in CSR */
 #define FEXTNVM_SW_CONFIG  0x0001
 #define E1000_PBA      0x01000  /* Packet Buffer Allocation - RW */
-#define E1000_PBS      0x01008  /* Packet Buffer Size */
+#define E1000_PBM      0x10000  /* Packet Buffer Memory - RW */
+#define E1000_PBS      0x01008  /* Packet Buffer Size - RW */
 #define E1000_EEMNGCTL 0x01010  /* MNG EEprom Control */
+#define E1000_EEMNGDATA    0x01014 /* MNG EEPROM Read/Write data */
+#define E1000_FLMNGCTL     0x01018 /* MNG Flash Control */
+#define E1000_FLMNGDATA    0x0101C /* MNG FLASH Read data */
+#define E1000_FLMNGCNT     0x01020 /* MNG FLASH Read Counter */
 #define E1000_FLASH_UPDATES 1000
 #define E1000_EEARBC   0x01024  /* EEPROM Auto Read Bus Control */
 #define E1000_FLASHT   0x01028  /* FLASH Timer Register */
@@ -168,9 +178,12 @@
 #define E1000_FLSWDATA 0x01034  /* FLASH data register */
 #define E1000_FLSWCNT  0x01038  /* FLASH Access Counter */
 #define E1000_FLOP     0x0103C  /* FLASH Opcode Register */
+#define E1000_FLOL     0x01050  /* FEEP Auto Load */
 #define E1000_ERT      0x02008  /* Early Rx Threshold - RW */
 #define E1000_FCRTL    0x02160  /* Flow Control Receive Threshold Low - RW */
+#define E1000_FCRTL_A  0x00168  /* Alias to FCRTL */
 #define E1000_FCRTH    0x02168  /* Flow Control Receive Threshold High - RW */
+#define E1000_FCRTH_A  0x00160  /* Alias to FCRTH */
 #define E1000_PSRCTL   0x02170  /* Packet Split Receive Control - RW */
 #define E1000_RDBAL    0x02800  /* RX Descriptor Base Address Low - RW */
 #define E1000_RDBAH    0x02804  /* RX Descriptor Base Address High - RW */
@@ -178,11 +191,17 @@
 #define E1000_RDH      0x02810  /* RX Descriptor Head - RW */
 #define E1000_RDT      0x02818  /* RX Descriptor Tail - RW */
 #define E1000_RDTR     0x02820  /* RX Delay Timer - RW */
+#define E1000_RDTR_A   0x00108  /* Alias to RDTR */
 #define E1000_RDBAL0   E1000_RDBAL /* RX Desc Base Address Low (0) - RW */
+#define E1000_RDBAL0_A 0x00110     /* Alias to RDBAL0 */
 #define E1000_RDBAH0   E1000_RDBAH /* RX Desc Base Address High (0) - RW */
+#define E1000_RDBAH0_A 0x00114     /* Alias to RDBAH0 */
 #define E1000_RDLEN0   E1000_RDLEN /* RX Desc Length (0) - RW */
+#define E1000_RDLEN0_A 0x00118     /* Alias to RDLEN0 */
 #define E1000_RDH0     E1000_RDH   /* RX Desc Head (0) - RW */
+#define E1000_RDH0_A   0x00120     /* Alias to RDH0 */
 #define E1000_RDT0     E1000_RDT   /* RX Desc Tail (0) - RW */
+#define E1000_RDT0_A   0x00128     /* Alias to RDT0 */
 #define E1000_RDTR0    E1000_RDTR  /* RX Delay Timer (0) - RW */
 #define E1000_RXDCTL   0x02828  /* RX Descriptor Control queue 0 - RW */
 #define E1000_RXDCTL1  0x02928  /* RX Descriptor Control queue 1 - RW */
@@ -191,17 +210,33 @@
 #define E1000_RAID     0x02C08  /* Receive Ack Interrupt Delay - RW */
 #define E1000_TXDMAC   0x03000  /* TX DMA Control - RW */
 #define E1000_KABGTXD  0x03004  /* AFE Band Gap Transmit Ref Data */
+#define E1000_POEMB    0x00F10  /* PHY OEM Bits Register - RW */
+#define E1000_RDFH     0x02410  /* Receive Data FIFO Head Register - RW */
+#define E1000_RDFH_A   0x08000  /* Alias to RDFH */
+#define E1000_RDFT     0x02418  /* Receive Data FIFO Tail Register - RW */
+#define E1000_RDFT_A   0x08008  /* Alias to RDFT */
+#define E1000_RDFHS    0x02420  /* Receive Data FIFO Head Saved Register - RW */
+#define E1000_RDFTS    0x02428  /* Receive Data FIFO Tail Saved Register - RW */
+#define E1000_RDFPC    0x02430  /* Receive Data FIFO Packet Count - RW */
 #define E1000_TDFH     0x03410  /* TX Data FIFO Head - RW */
+#define E1000_TDFH_A   0x08010  /* Alias to TDFH */
 #define E1000_TDFT     0x03418  /* TX Data FIFO Tail - RW */
+#define E1000_TDFT_A   0x08018  /* Alias to TDFT */
 #define E1000_TDFHS    0x03420  /* TX Data FIFO Head Saved - RW */
 #define E1000_TDFTS    0x03428  /* TX Data FIFO Tail Saved - RW */
 #define E1000_TDFPC    0x03430  /* TX Data FIFO Packet Count - RW */
 #define E1000_TDBAL    0x03800  /* TX Descriptor Base Address Low - RW */
+#define E1000_TDBAL_A  0x00420  /* Alias to TDBAL */
 #define E1000_TDBAH    0x03804  /* TX Descriptor Base Address High - RW */
+#define E1000_TDBAH_A  0x00424  /* Alias to TDBAH */
 #define E1000_TDLEN    0x03808  /* TX Descriptor Length - RW */
+#define E1000_TDLEN_A  0x00428  /* Alias to TDLEN */
 #define E1000_TDH      0x03810  /* TX Descriptor Head - RW */
+#define E1000_TDH_A    0x00430  /* Alias to TDH */
 #define E1000_TDT      0x03818  /* TX Descripotr Tail - RW */
+#define E1000_TDT_A    0x00438  /* Alias to TDT */
 #define E1000_TIDV     0x03820  /* TX Interrupt Delay Value - RW */
+#define E1000_TIDV_A   0x00440  /* Alias to TIDV */
 #define E1000_TXDCTL   0x03828  /* TX Descriptor Control - RW */
 #define E1000_TADV     0x0382C  /* TX Interrupt Absolute Delay Val - RW */
 #define E1000_TSPMT    0x03830  /* TCP Segmentation PAD & Min Threshold - RW */
@@ -282,9 +317,15 @@
 #define E1000_ICRXOC   0x04124  /* Interrupt Cause Receiver Overrun Count */
 #define E1000_RXCSUM   0x05000  /* RX Checksum Control - RW */
 #define E1000_RFCTL    0x05008  /* Receive Filter Control*/
+#define E1000_MAVTV0   0x05010  /* Management VLAN TAG Value 0 */
+#define E1000_MAVTV1   0x05014  /* Management VLAN TAG Value 1 */
+#define E1000_MAVTV2   0x05018  /* Management VLAN TAG Value 2 */
+#define E1000_MAVTV3   0x0501c  /* Management VLAN TAG Value 3 */
 #define E1000_MTA      0x05200  /* Multicast Table Array - RW Array */
 #define E1000_RA       0x05400  /* Receive Address - RW Array */
+#define E1000_RA_A     0x00040  /* Alias to RA */
 #define E1000_VFTA     0x05600  /* VLAN Filter Table Array - RW Array */
+#define E1000_VFTA_A   0x00600  /* Alias to VFTA */
 #define E1000_WUC      0x05800  /* Wakeup Control - RW */
 #define E1000_WUFC     0x05808  /* Wakeup Filter Control - RW */
 #define E1000_WUS      0x05810  /* Wakeup Status - RO */
@@ -294,27 +335,57 @@
 #define E1000_IP6AT    0x05880  /* IPv6 Address Table - RW Array */
 #define E1000_WUPL     0x05900  /* Wakeup Packet Length - RW */
 #define E1000_WUPM     0x05A00  /* Wakeup Packet Memory - RO A */
+#define E1000_MFUTP01  0x05828  /* Management Flex UDP/TCP Ports 0/1 - RW */
+#define E1000_MFUTP23  0x05830  /* Management Flex UDP/TCP Ports 2/3 - RW */
+#define E1000_MFVAL    0x05824  /* Manageability Filters Valid - RW */
+#define E1000_MDEF     0x05890  /* Manageability Decision Filters - RW Array */
 #define E1000_FFLT     0x05F00  /* Flexible Filter Length Table - RW Array */
 #define E1000_HOST_IF  0x08800  /* Host Interface */
 #define E1000_FFMT     0x09000  /* Flexible Filter Mask Table - RW Array */
+#define E1000_FTFT     0x09400  /* Flexible TCO Filter Table - RW Array */
 #define E1000_FFVT     0x09800  /* Flexible Filter Value Table - RW Array */
 
 #define E1000_KUMCTRLSTA 0x00034 /* MAC-PHY interface - RW */
-#define E1000_MDPHYA     0x0003C  /* PHY address - RW */
-#define E1000_MANC2H     0x05860  /* Management Control To Host - RW */
+#define E1000_MDPHYA     0x0003C /* PHY address - RW */
+#define E1000_MANC2H     0x05860 /* Management Control To Host - RW */
 #define E1000_SW_FW_SYNC 0x05B5C /* Software-Firmware Synchronization - RW */
 
 #define E1000_GCR       0x05B00 /* PCI-Ex Control */
+#define E1000_FUNCTAG   0x05B08 /* Function-Tag Register */
 #define E1000_GSCL_1    0x05B10 /* PCI-Ex Statistic Control #1 */
 #define E1000_GSCL_2    0x05B14 /* PCI-Ex Statistic Control #2 */
 #define E1000_GSCL_3    0x05B18 /* PCI-Ex Statistic Control #3 */
 #define E1000_GSCL_4    0x05B1C /* PCI-Ex Statistic Control #4 */
+#define E1000_GSCN_0    0x05B20 /* 3GIO Statistic Counter Register #0 */
+#define E1000_GSCN_1    0x05B24 /* 3GIO Statistic Counter Register #1 */
+#define E1000_GSCN_2    0x05B28 /* 3GIO Statistic Counter Register #2 */
+#define E1000_GSCN_3    0x05B2C /* 3GIO Statistic Counter Register #3 */
 #define E1000_FACTPS    0x05B30 /* Function Active and Power State to MNG */
 #define E1000_SWSM      0x05B50 /* SW Semaphore */
+#define E1000_GCR2      0x05B64 /* 3GIO Control Register 2 */
 #define E1000_FWSM      0x05B54 /* FW Semaphore */
+#define E1000_PBACLR    0x05B68 /* MSI-X PBA Clear */
 #define E1000_FFLT_DBG  0x05F04 /* Debug Register */
 #define E1000_HICR      0x08F00 /* Host Inteface Control */
 
+#define E1000_TSYNCRXCTL 0x0B620 /* Rx Time Sync Control register - RW */
+#define E1000_TSYNCTXCTL 0x0B614 /* Tx Time Sync Control register - RW */
+#define E1000_TIMINCA    0x0B608 /* Increment attributes register - RW */
+#define E1000_RXSTMPL    0x0B624 /* Rx timestamp Low - RO */
+#define E1000_RXSTMPH    0x0B628 /* Rx timestamp High - RO */
+#define E1000_TXSTMPL    0x0B618 /* Tx timestamp value Low - RO */
+#define E1000_TXSTMPH    0x0B61C /* Tx timestamp value High - RO */
+#define E1000_SYSTIML    0x0B600 /* System time register Low - RO */
+#define E1000_SYSTIMH    0x0B604 /* System time register High - RO */
+#define E1000_TIMINCA    0x0B608 /* Increment attributes register - RW */
+#define E1000_RXMTRL     0x0B634 /* Time sync Rx EtherType and Msg Type - RW */
+#define E1000_RXUDP      0x0B638 /* Time Sync Rx UDP Port - RW */
+#define E1000_RXSATRL    0x0B62C /* Rx timestamp attribute low - RO */
+#define E1000_RXSATRH    0x0B630 /* Rx timestamp attribute high - RO */
+#define E1000_TIMADJL    0x0B60C /* Time Adjustment Offset register Low - RW */
+#define E1000_TIMADJH    0x0B610 /* Time Adjustment Offset register High - RW */
+#define E1000_RXCFGL     0x0B634 /* RX Ethertype and Message Type - RW*/
+
 /* RSS registers */
 #define E1000_CPUVEC    0x02C10 /* CPU Vector Register - RW */
 #define E1000_MRQC      0x05818 /* Multiple Receive Control - RW */
@@ -323,6 +394,85 @@
 #define E1000_RSSIM     0x05864 /* RSS Interrupt Mask */
 #define E1000_RSSIR     0x05868 /* RSS Interrupt Request */
 
+#define E1000_MRQC_ENABLED(mrqc) (((mrqc) & (BIT(0) | BIT(1))) == BIT(0))
+
+#define E1000_RETA_IDX(hash)        ((hash) & (BIT(7) - 1))
+#define E1000_RETA_VAL(reta, hash)  (((uint8_t *)(reta))[E1000_RETA_IDX(hash)])
+#define E1000_RSS_QUEUE(reta, hash) ((E1000_RETA_VAL(reta, hash) & BIT(7)) >> 7)
+
+#define E1000_MRQC_EN_TCPIPV4(mrqc) ((mrqc) & BIT(16))
+#define E1000_MRQC_EN_IPV4(mrqc)    ((mrqc) & BIT(17))
+#define E1000_MRQC_EN_TCPIPV6(mrqc) ((mrqc) & BIT(18))
+#define E1000_MRQC_EN_IPV6EX(mrqc)  ((mrqc) & BIT(19))
+#define E1000_MRQC_EN_IPV6(mrqc)    ((mrqc) & BIT(20))
+
+#define E1000_MRQ_RSS_TYPE_NONE     (0)
+#define E1000_MRQ_RSS_TYPE_IPV4TCP  (1)
+#define E1000_MRQ_RSS_TYPE_IPV4     (2)
+#define E1000_MRQ_RSS_TYPE_IPV6TCP  (3)
+#define E1000_MRQ_RSS_TYPE_IPV6EX   (4)
+#define E1000_MRQ_RSS_TYPE_IPV6     (5)
+
+#define E1000_ICR_ASSERTED BIT(31)
+#define E1000_EIAC_MASK    0x01F00000
+
+/* [TR]DBAL and [TR]DLEN masks */
+#define E1000_XDBAL_MASK            (~(BIT(4) - 1))
+#define E1000_XDLEN_MASK            ((BIT(20) - 1) & (~(BIT(7) - 1)))
+
+/* IVAR register parsing helpers */
+#define E1000_IVAR_INT_ALLOC_VALID  (0x8)
+
+#define E1000_IVAR_RXQ0_SHIFT       (0)
+#define E1000_IVAR_RXQ1_SHIFT       (4)
+#define E1000_IVAR_TXQ0_SHIFT       (8)
+#define E1000_IVAR_TXQ1_SHIFT       (12)
+#define E1000_IVAR_OTHER_SHIFT      (16)
+
+#define E1000_IVAR_ENTRY_MASK       (0xF)
+#define E1000_IVAR_ENTRY_VALID_MASK E1000_IVAR_INT_ALLOC_VALID
+#define E1000_IVAR_ENTRY_VEC_MASK   (0x7)
+
+#define E1000_IVAR_RXQ0(x)          ((x) >> E1000_IVAR_RXQ0_SHIFT)
+#define E1000_IVAR_RXQ1(x)          ((x) >> E1000_IVAR_RXQ1_SHIFT)
+#define E1000_IVAR_TXQ0(x)          ((x) >> E1000_IVAR_TXQ0_SHIFT)
+#define E1000_IVAR_TXQ1(x)          ((x) >> E1000_IVAR_TXQ1_SHIFT)
+#define E1000_IVAR_OTHER(x)         ((x) >> E1000_IVAR_OTHER_SHIFT)
+
+#define E1000_IVAR_ENTRY_VALID(x)   ((x) & E1000_IVAR_ENTRY_VALID_MASK)
+#define E1000_IVAR_ENTRY_VEC(x)     ((x) & E1000_IVAR_ENTRY_VEC_MASK)
+
+#define E1000_IVAR_TX_INT_EVERY_WB  BIT(31)
+
+/* RFCTL register bits */
+#define E1000_RFCTL_ISCSI_DIS           0x00000001
+#define E1000_RFCTL_NFSW_DIS            0x00000040
+#define E1000_RFCTL_NFSR_DIS            0x00000080
+#define E1000_RFCTL_IPV6_DIS            0x00000400
+#define E1000_RFCTL_IPV6_XSUM_DIS       0x00000800
+#define E1000_RFCTL_ACK_DIS             0x00001000
+#define E1000_RFCTL_ACK_DATA_DIS        0x00002000
+#define E1000_RFCTL_IPFRSP_DIS          0x00004000
+#define E1000_RFCTL_EXTEN               0x00008000
+#define E1000_RFCTL_IPV6_EX_DIS         0x00010000
+#define E1000_RFCTL_NEW_IPV6_EXT_DIS    0x00020000
+
+/* PSRCTL parsing */
+#define E1000_PSRCTL_BSIZE0_MASK   0x0000007F
+#define E1000_PSRCTL_BSIZE1_MASK   0x00003F00
+#define E1000_PSRCTL_BSIZE2_MASK   0x003F0000
+#define E1000_PSRCTL_BSIZE3_MASK   0x3F000000
+
+#define E1000_PSRCTL_BSIZE0_SHIFT  0
+#define E1000_PSRCTL_BSIZE1_SHIFT  8
+#define E1000_PSRCTL_BSIZE2_SHIFT  16
+#define E1000_PSRCTL_BSIZE3_SHIFT  24
+
+#define E1000_PSRCTL_BUFFS_PER_DESC 4
+
+/* TARC* parsing */
+#define E1000_TARC_ENABLE BIT(10)
+
 /* PHY 1000 MII Register/Bit Definitions */
 /* PHY Registers defined by IEEE */
 #define PHY_CTRL         0x00 /* Control Register */
@@ -338,6 +488,40 @@
 #define PHY_1000T_STATUS 0x0A /* 1000Base-T Status Reg */
 #define PHY_EXT_STATUS   0x0F /* Extended Status Reg */
 
+/* 82574-specific registers */
+#define PHY_COPPER_CTRL1      0x10 /* Copper Specific Control Register 1 */
+#define PHY_COPPER_STAT1      0x11 /* Copper Specific Status Register 1 */
+#define PHY_COPPER_INT_ENABLE 0x12  /* Interrupt Enable Register */
+#define PHY_COPPER_STAT2      0x13 /* Copper Specific Status Register 2 */
+#define PHY_COPPER_CTRL3      0x14 /* Copper Specific Control Register 3 */
+#define PHY_COPPER_CTRL2      0x1A /* Copper Specific Control Register 2 */
+#define PHY_RX_ERR_CNTR       0x15  /* Receive Error Counter */
+#define PHY_PAGE              0x16 /* Page Address (Any page) */
+#define PHY_OEM_BITS          0x19 /* OEM Bits (Page 0) */
+#define PHY_BIAS_1            0x1d /* Bias Setting Register */
+#define PHY_BIAS_2            0x1e /* Bias Setting Register */
+
+/* 82574-specific registers - page 2 */
+#define PHY_MAC_CTRL1         0x10 /* MAC Specific Control Register 1 */
+#define PHY_MAC_INT_ENABLE    0x12 /* MAC Interrupt Enable Register */
+#define PHY_MAC_STAT          0x13 /* MAC Specific Status Register */
+#define PHY_MAC_CTRL2         0x15 /* MAC Specific Control Register 2 */
+
+/* 82574-specific registers - page 3 */
+#define PHY_LED_03_FUNC_CTRL1 0x10 /* LED[3:0] Function Control */
+#define PHY_LED_03_POL_CTRL   0x11 /* LED[3:0] Polarity Control */
+#define PHY_LED_TIMER_CTRL    0x12 /* LED Timer Control */
+#define PHY_LED_45_CTRL       0x13 /* LED[5:4] Function Control and Polarity */
+
+/* 82574-specific registers - page 5 */
+#define PHY_1000T_SKEW        0x14 /* 1000 BASE - T Pair Skew Register */
+#define PHY_1000T_SWAP        0x15 /* 1000 BASE - T Pair Swap and Polarity */
+
+/* 82574-specific registers - page 6 */
+#define PHY_CRC_COUNTERS      0x11 /* CRC Counters */
+
+#define PHY_PAGE_RW_MASK 0x7F /* R/W part of page address register */
+
 #define MAX_PHY_REG_ADDRESS        0x1F  /* 5 bit address bus (0-0x1F) */
 #define MAX_PHY_MULTI_PAGE_REG     0xF   /* Registers equal on all pages */
 
@@ -417,6 +601,18 @@
 #define E1000_ICR_DSW           0x00000020 /* FW changed the status of DISSW bit in the FWSM */
 #define E1000_ICR_PHYINT        0x00001000 /* LAN connected device generates an interrupt */
 #define E1000_ICR_EPRST         0x00100000 /* ME handware reset occurs */
+#define E1000_ICR_RXQ0          0x00100000 /* Rx Queue 0 Interrupt */
+#define E1000_ICR_RXQ1          0x00200000 /* Rx Queue 1 Interrupt */
+#define E1000_ICR_TXQ0          0x00400000 /* Tx Queue 0 Interrupt */
+#define E1000_ICR_TXQ1          0x00800000 /* Tx Queue 1 Interrupt */
+#define E1000_ICR_OTHER         0x01000000 /* Other Interrupts */
+
+#define E1000_ICR_OTHER_CAUSES (E1000_ICR_LSC  | \
+                                E1000_ICR_RXO  | \
+                                E1000_ICR_MDAC | \
+                                E1000_ICR_SRPD | \
+                                E1000_ICR_ACK  | \
+                                E1000_ICR_MNG)
 
 /* Interrupt Cause Set */
 #define E1000_ICS_TXDW      E1000_ICR_TXDW      /* Transmit desc written back */
@@ -465,6 +661,11 @@
 #define E1000_IMS_SRPD      E1000_ICR_SRPD
 #define E1000_IMS_ACK       E1000_ICR_ACK       /* Receive Ack frame */
 #define E1000_IMS_MNG       E1000_ICR_MNG       /* Manageability event */
+#define E1000_IMS_RXQ0      E1000_ICR_RXQ0
+#define E1000_IMS_RXQ1      E1000_ICR_RXQ1
+#define E1000_IMS_TXQ0      E1000_ICR_TXQ0
+#define E1000_IMS_TXQ1      E1000_ICR_TXQ1
+#define E1000_IMS_OTHER     E1000_ICR_OTHER
 #define E1000_IMS_DOCK      E1000_ICR_DOCK      /* Dock/Undock */
 #define E1000_IMS_RXD_FIFO_PAR0 E1000_ICR_RXD_FIFO_PAR0 /* queue 0 Rx descriptor FIFO parity error */
 #define E1000_IMS_TXD_FIFO_PAR0 E1000_ICR_TXD_FIFO_PAR0 /* queue 0 Tx descriptor FIFO parity error */
@@ -556,6 +757,15 @@
 #define E1000_EEPROM_RW_ADDR_SHIFT 8    /* Shift to the address bits */
 #define E1000_EEPROM_POLL_WRITE    1    /* Flag for polling for write complete */
 #define E1000_EEPROM_POLL_READ     0    /* Flag for polling for read complete */
+
+/* 82574 EERD/EEWR registers layout */
+#define E1000_EERW_START        BIT(0)
+#define E1000_EERW_DONE         BIT(1)
+#define E1000_EERW_ADDR_SHIFT   2
+#define E1000_EERW_ADDR_MASK    ((1L << 14) - 1)
+#define E1000_EERW_DATA_SHIFT   16
+#define E1000_EERW_DATA_MASK   ((1L << 16) - 1)
+
 /* Register Bit Masks */
 /* Device Control */
 #define E1000_CTRL_FD       0x00000001  /* Full duplex.0=half; 1=full */
@@ -578,7 +788,17 @@
 #define E1000_CTRL_D_UD_EN  0x00002000  /* Dock/Undock enable */
 #define E1000_CTRL_D_UD_POLARITY 0x00004000 /* Defined polarity of Dock/Undock indication in SDP[0] */
 #define E1000_CTRL_FORCE_PHY_RESET 0x00008000 /* Reset both PHY ports, through PHYRST_N pin */
+#define E1000_CTRL_SPD_SHIFT 8          /* Speed Select Shift */
+
+#define E1000_CTRL_EXT_ASDCHK  0x00001000 /* auto speed detection check */
+#define E1000_CTRL_EXT_EE_RST  0x00002000 /* EEPROM reset */
 #define E1000_CTRL_EXT_LINK_EN 0x00010000 /* enable link status from external LINK_0 and LINK_1 pins */
+#define E1000_CTRL_EXT_EIAME   0x01000000
+#define E1000_CTRL_EXT_IAME    0x08000000 /* Int ACK Auto-mask */
+#define E1000_CTRL_EXT_PBA_CLR 0x80000000 /* PBA Clear */
+#define E1000_CTRL_EXT_INT_TIMERS_CLEAR_ENA 0x20000000
+#define E1000_CTRL_EXT_SPD_BYPS  0x00008000 /* Speed Select Bypass */
+
 #define E1000_CTRL_SWDPIN0  0x00040000  /* SWDPIN 0 value */
 #define E1000_CTRL_SWDPIN1  0x00080000  /* SWDPIN 1 value */
 #define E1000_CTRL_SWDPIN2  0x00100000  /* SWDPIN 2 value */
@@ -587,6 +807,7 @@
 #define E1000_CTRL_SWDPIO1  0x00800000  /* SWDPIN 1 input or output */
 #define E1000_CTRL_SWDPIO2  0x01000000  /* SWDPIN 2 input or output */
 #define E1000_CTRL_SWDPIO3  0x02000000  /* SWDPIN 3 input or output */
+#define E1000_CTRL_ADVD3WUC 0x00100000  /* D3 WUC */
 #define E1000_CTRL_RST      0x04000000  /* Global reset */
 #define E1000_CTRL_RFCE     0x08000000  /* Receive Flow Control enable */
 #define E1000_CTRL_TFCE     0x10000000  /* Transmit flow control enable */
@@ -611,9 +832,13 @@
 #define E1000_STATUS_LAN_INIT_DONE 0x00000200   /* Lan Init Completion
                                                    by EEPROM/Flash */
 #define E1000_STATUS_ASDV       0x00000300      /* Auto speed detect value */
+#define E1000_STATUS_ASDV_10    0x00000000      /* ASDV 10Mb */
+#define E1000_STATUS_ASDV_100   0x00000100      /* ASDV 100Mb */
+#define E1000_STATUS_ASDV_1000  0x00000200      /* ASDV 1Gb */
 #define E1000_STATUS_DOCK_CI    0x00000800      /* Change in Dock/Undock state. Clear on write '0'. */
 #define E1000_STATUS_GIO_MASTER_ENABLE 0x00080000 /* Status of Master requests. */
 #define E1000_STATUS_MTXCKOK    0x00000400      /* MTX clock running OK */
+#define E1000_STATUS_PHYRA      0x00000400      /* PHY Reset Asserted */
 #define E1000_STATUS_PCI66      0x00000800      /* In 66Mhz slot */
 #define E1000_STATUS_BUS64      0x00001000      /* In 64 bit slot */
 #define E1000_STATUS_PCIX_MODE  0x00002000      /* PCI-X mode */
@@ -628,6 +853,8 @@
 #define E1000_STATUS_FUSE_9       0x08000000
 #define E1000_STATUS_SERDES0_DIS  0x10000000 /* SERDES disabled on port 0 */
 #define E1000_STATUS_SERDES1_DIS  0x20000000 /* SERDES disabled on port 1 */
+#define E1000_STATUS_SPEED_SHIFT  6
+#define E1000_STATUS_ASDV_SHIFT   8
 
 /* EEPROM/Flash Control */
 #define E1000_EECD_SK        0x00000001 /* EEPROM Clock */
@@ -658,6 +885,8 @@
 #define E1000_EECD_AUPDEN    0x00100000 /* Enable Autonomous FLASH update */
 #define E1000_EECD_SHADV     0x00200000 /* Shadow RAM Data Valid */
 #define E1000_EECD_SEC1VAL   0x00400000 /* Sector One Valid */
+
+
 #define E1000_EECD_SECVAL_SHIFT      22
 #define E1000_STM_OPCODE     0xDB00
 #define E1000_HICR_FW_RESET  0xC0
@@ -678,6 +907,18 @@
 #define E1000_MDIC_INT_EN    0x20000000
 #define E1000_MDIC_ERROR     0x40000000
 
+/* Rx Interrupt Delay Timer */
+#define E1000_RDTR_FPD       BIT(31)
+
+/* Tx Interrupt Delay Timer */
+#define E1000_TIDV_FPD       BIT(31)
+
+/* Delay increments in nanoseconds for delayed interrupts registers */
+#define E1000_INTR_DELAY_NS_RES (1024)
+
+/* Delay increments in nanoseconds for interrupt throttling registers */
+#define E1000_INTR_THROTTLING_NS_RES (256)
+
 /* EEPROM Commands - Microwire */
 #define EEPROM_READ_OPCODE_MICROWIRE  0x6  /* EEPROM read opcode */
 #define EEPROM_WRITE_OPCODE_MICROWIRE 0x5  /* EEPROM write opcode */
@@ -705,6 +946,21 @@
 #define E1000_EEPROM_CFG_DONE         0x00040000   /* MNG config cycle done */
 #define E1000_EEPROM_CFG_DONE_PORT_1  0x00080000   /* ...for second port */
 
+/* PCI Express Control */
+/* 3GIO Control Register - GCR (0x05B00; RW) */
+#define E1000_L0S_ADJUST              (1 << 9)
+#define E1000_L1_ENTRY_LATENCY_MSB    (1 << 23)
+#define E1000_L1_ENTRY_LATENCY_LSB    (1 << 25 | 1 << 26)
+
+#define E1000_L0S_ADJUST              (1 << 9)
+#define E1000_L1_ENTRY_LATENCY_MSB    (1 << 23)
+#define E1000_L1_ENTRY_LATENCY_LSB    (1 << 25 | 1 << 26)
+
+#define E1000_GCR_RO_BITS             (1 << 23 | 1 << 25 | 1 << 26)
+
+/* MSI-X PBA Clear register */
+#define E1000_PBACLR_VALID_MASK       (BIT(5) - 1)
+
 /* Transmit Descriptor */
 struct e1000_tx_desc {
     uint64_t buffer_addr;       /* Address of the descriptor's data buffer */
@@ -746,7 +1002,9 @@
 #define E1000_TXD_CMD_TCP    0x01000000 /* TCP packet */
 #define E1000_TXD_CMD_IP     0x02000000 /* IP packet */
 #define E1000_TXD_CMD_TSE    0x04000000 /* TCP Seg enable */
+#define E1000_TXD_CMD_SNAP   0x40000000 /* Update SNAP header */
 #define E1000_TXD_STAT_TC    0x00000004 /* Tx Underrun */
+#define E1000_TXD_EXTCMD_TSTAMP 0x00000010 /* IEEE1588 Timestamp packet */
 
 /* Transmit Control */
 #define E1000_TCTL_RST    0x00000001    /* software reset */
@@ -761,7 +1019,7 @@
 #define E1000_TCTL_NRTU   0x02000000    /* No Re-transmit on underrun */
 #define E1000_TCTL_MULR   0x10000000    /* Multiple request support */
 
-/* Receive Descriptor */
+/* Legacy Receive Descriptor */
 struct e1000_rx_desc {
     uint64_t buffer_addr; /* Address of the descriptor's data buffer */
     uint16_t length;     /* Length of data DMAed into data buffer */
@@ -771,6 +1029,78 @@
     uint16_t special;
 };
 
+/* Extended Receive Descriptor */
+union e1000_rx_desc_extended {
+    struct {
+        uint64_t buffer_addr;
+        uint64_t reserved;
+    } read;
+    struct {
+        struct {
+            uint32_t mrq;           /* Multiple Rx Queues */
+            union {
+                uint32_t rss;       /* RSS Hash */
+                struct {
+                    uint16_t ip_id; /* IP id */
+                    uint16_t csum;  /* Packet Checksum */
+                } csum_ip;
+            } hi_dword;
+        } lower;
+        struct {
+            uint32_t status_error;  /* ext status/error */
+            uint16_t length;
+            uint16_t vlan;          /* VLAN tag */
+        } upper;
+    } wb;                           /* writeback */
+};
+
+#define MAX_PS_BUFFERS 4
+
+/* Number of packet split data buffers (not including the header buffer) */
+#define PS_PAGE_BUFFERS    (MAX_PS_BUFFERS - 1)
+
+/* Receive Descriptor - Packet Split */
+union e1000_rx_desc_packet_split {
+    struct {
+        /* one buffer for protocol header(s), three data buffers */
+        uint64_t buffer_addr[MAX_PS_BUFFERS];
+    } read;
+    struct {
+        struct {
+            uint32_t mrq;          /* Multiple Rx Queues */
+            union {
+                uint32_t rss;          /* RSS Hash */
+                struct {
+                    uint16_t ip_id;    /* IP id */
+                    uint16_t csum;     /* Packet Checksum */
+                } csum_ip;
+            } hi_dword;
+        } lower;
+        struct {
+            uint32_t status_error;     /* ext status/error */
+            uint16_t length0;      /* length of buffer 0 */
+            uint16_t vlan;         /* VLAN tag */
+        } middle;
+        struct {
+            uint16_t header_status;
+            /* length of buffers 1-3 */
+            uint16_t length[PS_PAGE_BUFFERS];
+        } upper;
+        uint64_t reserved;
+    } wb; /* writeback */
+};
+
+/* Receive Checksum Control bits */
+#define E1000_RXCSUM_IPOFLD     0x100   /* IP Checksum Offload Enable */
+#define E1000_RXCSUM_TUOFLD     0x200   /* TCP/UDP Checksum Offload Enable */
+#define E1000_RXCSUM_PCSD       0x2000  /* Packet Checksum Disable */
+
+#define E1000_RING_DESC_LEN       (16)
+#define E1000_RING_DESC_LEN_SHIFT (4)
+
+#define E1000_MIN_RX_DESC_LEN   E1000_RING_DESC_LEN
+#define E1000_MAX_RX_DESC_LEN   (sizeof(union e1000_rx_desc_packet_split))
+
 /* Receive Descriptor bit definitions */
 #define E1000_RXD_STAT_DD       0x01    /* Descriptor Done */
 #define E1000_RXD_STAT_EOP      0x02    /* End of Packet */
@@ -796,6 +1126,15 @@
 #define E1000_RXD_SPC_CFI_MASK  0x1000  /* CFI is bit 12 */
 #define E1000_RXD_SPC_CFI_SHIFT 12
 
+/* RX packet types */
+#define E1000_RXD_PKT_MAC       (0)
+#define E1000_RXD_PKT_IP4       (1)
+#define E1000_RXD_PKT_IP4_XDP   (2)
+#define E1000_RXD_PKT_IP6       (5)
+#define E1000_RXD_PKT_IP6_XDP   (6)
+
+#define E1000_RXD_PKT_TYPE(t) ((t) << 16)
+
 #define E1000_RXDEXT_STATERR_CE    0x01000000
 #define E1000_RXDEXT_STATERR_SE    0x02000000
 #define E1000_RXDEXT_STATERR_SEQ   0x04000000
@@ -873,6 +1212,8 @@
 #define E1000_MANC_NEIGHBOR_EN   0x00004000 /* Enable Neighbor Discovery
                                              * Filtering */
 #define E1000_MANC_ARP_RES_EN    0x00008000 /* Enable ARP response Filtering */
+#define E1000_MANC_DIS_IP_CHK_ARP  0x10000000 /* Disable IP address chacking */
+                                              /*for ARP packets - in 82574 */
 #define E1000_MANC_TCO_RESET     0x00010000 /* TCO Reset Occurred */
 #define E1000_MANC_RCV_TCO_EN    0x00020000 /* Receive TCO Packets Enabled */
 #define E1000_MANC_REPORT_STATUS 0x00040000 /* Status Reporting Enabled */
@@ -896,7 +1237,14 @@
 #define E1000_MANC_SMB_DATA_OUT_SHIFT  28 /* SMBus Data Out Shift */
 #define E1000_MANC_SMB_CLK_OUT_SHIFT   29 /* SMBus Clock Out Shift */
 
+/* FACTPS Control */
+#define E1000_FACTPS_LAN0_ON     0x00000004 /* Lan 0 enable */
+
 /* For checksumming, the sum of all words in the EEPROM should equal 0xBABA. */
 #define EEPROM_SUM 0xBABA
 
-#endif /* _E1000_HW_H_ */
+/* I/O-Mapped Access to Internal Registers, Memories, and Flash */
+#define E1000_IOADDR 0x00
+#define E1000_IODATA 0x04
+
+#endif /* HW_E1000_REGS_H */
diff --git a/hw/net/e1000e.c b/hw/net/e1000e.c
new file mode 100644
index 0000000..bad43f4
--- /dev/null
+++ b/hw/net/e1000e.c
@@ -0,0 +1,711 @@
+/*
+* QEMU INTEL 82574 GbE NIC emulation
+*
+* Software developer's manuals:
+* http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf
+*
+* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com)
+* Developed by Daynix Computing LTD (http://www.daynix.com)
+*
+* Authors:
+* Dmitry Fleytman <dmitry@daynix.com>
+* Leonid Bloch <leonid@daynix.com>
+* Yan Vugenfirer <yan@daynix.com>
+*
+* Based on work done by:
+* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
+* Copyright (c) 2008 Qumranet
+* Based on work done by:
+* Copyright (c) 2007 Dan Aloni
+* Copyright (c) 2004 Antony T Curtis
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "qemu/osdep.h"
+#include "net/net.h"
+#include "net/tap.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+
+#include "hw/net/e1000_regs.h"
+
+#include "e1000x_common.h"
+#include "e1000e_core.h"
+
+#include "trace.h"
+
+#define TYPE_E1000E "e1000e"
+#define E1000E(obj) OBJECT_CHECK(E1000EState, (obj), TYPE_E1000E)
+
+typedef struct E1000EState {
+    PCIDevice parent_obj;
+    NICState *nic;
+    NICConf conf;
+
+    MemoryRegion mmio;
+    MemoryRegion flash;
+    MemoryRegion io;
+    MemoryRegion msix;
+
+    uint32_t ioaddr;
+
+    uint16_t subsys_ven;
+    uint16_t subsys;
+
+    uint16_t subsys_ven_used;
+    uint16_t subsys_used;
+
+    bool disable_vnet;
+
+    E1000ECore core;
+
+} E1000EState;
+
+#define E1000E_MMIO_IDX     0
+#define E1000E_FLASH_IDX    1
+#define E1000E_IO_IDX       2
+#define E1000E_MSIX_IDX     3
+
+#define E1000E_MMIO_SIZE    (128 * 1024)
+#define E1000E_FLASH_SIZE   (128 * 1024)
+#define E1000E_IO_SIZE      (32)
+#define E1000E_MSIX_SIZE    (16 * 1024)
+
+#define E1000E_MSIX_TABLE   (0x0000)
+#define E1000E_MSIX_PBA     (0x2000)
+
+static uint64_t
+e1000e_mmio_read(void *opaque, hwaddr addr, unsigned size)
+{
+    E1000EState *s = opaque;
+    return e1000e_core_read(&s->core, addr, size);
+}
+
+static void
+e1000e_mmio_write(void *opaque, hwaddr addr,
+                   uint64_t val, unsigned size)
+{
+    E1000EState *s = opaque;
+    e1000e_core_write(&s->core, addr, val, size);
+}
+
+static bool
+e1000e_io_get_reg_index(E1000EState *s, uint32_t *idx)
+{
+    if (s->ioaddr < 0x1FFFF) {
+        *idx = s->ioaddr;
+        return true;
+    }
+
+    if (s->ioaddr < 0x7FFFF) {
+        trace_e1000e_wrn_io_addr_undefined(s->ioaddr);
+        return false;
+    }
+
+    if (s->ioaddr < 0xFFFFF) {
+        trace_e1000e_wrn_io_addr_flash(s->ioaddr);
+        return false;
+    }
+
+    trace_e1000e_wrn_io_addr_unknown(s->ioaddr);
+    return false;
+}
+
+static uint64_t
+e1000e_io_read(void *opaque, hwaddr addr, unsigned size)
+{
+    E1000EState *s = opaque;
+    uint32_t idx = 0;
+    uint64_t val;
+
+    switch (addr) {
+    case E1000_IOADDR:
+        trace_e1000e_io_read_addr(s->ioaddr);
+        return s->ioaddr;
+    case E1000_IODATA:
+        if (e1000e_io_get_reg_index(s, &idx)) {
+            val = e1000e_core_read(&s->core, idx, sizeof(val));
+            trace_e1000e_io_read_data(idx, val);
+            return val;
+        }
+        return 0;
+    default:
+        trace_e1000e_wrn_io_read_unknown(addr);
+        return 0;
+    }
+}
+
+static void
+e1000e_io_write(void *opaque, hwaddr addr,
+                uint64_t val, unsigned size)
+{
+    E1000EState *s = opaque;
+    uint32_t idx = 0;
+
+    switch (addr) {
+    case E1000_IOADDR:
+        trace_e1000e_io_write_addr(val);
+        s->ioaddr = (uint32_t) val;
+        return;
+    case E1000_IODATA:
+        if (e1000e_io_get_reg_index(s, &idx)) {
+            trace_e1000e_io_write_data(idx, val);
+            e1000e_core_write(&s->core, idx, val, sizeof(val));
+        }
+        return;
+    default:
+        trace_e1000e_wrn_io_write_unknown(addr);
+        return;
+    }
+}
+
+static const MemoryRegionOps mmio_ops = {
+    .read = e1000e_mmio_read,
+    .write = e1000e_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static const MemoryRegionOps io_ops = {
+    .read = e1000e_io_read,
+    .write = e1000e_io_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+};
+
+static int
+e1000e_nc_can_receive(NetClientState *nc)
+{
+    E1000EState *s = qemu_get_nic_opaque(nc);
+    return e1000e_can_receive(&s->core);
+}
+
+static ssize_t
+e1000e_nc_receive_iov(NetClientState *nc, const struct iovec *iov, int iovcnt)
+{
+    E1000EState *s = qemu_get_nic_opaque(nc);
+    return e1000e_receive_iov(&s->core, iov, iovcnt);
+}
+
+static ssize_t
+e1000e_nc_receive(NetClientState *nc, const uint8_t *buf, size_t size)
+{
+    E1000EState *s = qemu_get_nic_opaque(nc);
+    return e1000e_receive(&s->core, buf, size);
+}
+
+static void
+e1000e_set_link_status(NetClientState *nc)
+{
+    E1000EState *s = qemu_get_nic_opaque(nc);
+    e1000e_core_set_link_status(&s->core);
+}
+
+static NetClientInfo net_e1000e_info = {
+    .type = NET_CLIENT_DRIVER_NIC,
+    .size = sizeof(NICState),
+    .can_receive = e1000e_nc_can_receive,
+    .receive = e1000e_nc_receive,
+    .receive_iov = e1000e_nc_receive_iov,
+    .link_status_changed = e1000e_set_link_status,
+};
+
+/*
+* EEPROM (NVM) contents documented in Table 36, section 6.1
+* and generally 6.1.2 Software accessed words.
+*/
+static const uint16_t e1000e_eeprom_template[64] = {
+  /*        Address        |    Compat.    | ImVer |   Compat.     */
+    0x0000, 0x0000, 0x0000, 0x0420, 0xf746, 0x2010, 0xffff, 0xffff,
+  /*      PBA      |ICtrl1 | SSID  | SVID  | DevID |-------|ICtrl2 */
+    0x0000, 0x0000, 0x026b, 0x0000, 0x8086, 0x0000, 0x0000, 0x8058,
+  /*    NVM words 1,2,3    |-------------------------------|PCI-EID*/
+    0x0000, 0x2001, 0x7e7c, 0xffff, 0x1000, 0x00c8, 0x0000, 0x2704,
+  /* PCIe Init. Conf 1,2,3 |PCICtrl|PHY|LD1|-------| RevID | LD0,2 */
+    0x6cc9, 0x3150, 0x070e, 0x460b, 0x2d84, 0x0100, 0xf000, 0x0706,
+  /* FLPAR |FLANADD|LAN-PWR|FlVndr |ICtrl3 |APTSMBA|APTRxEP|APTSMBC*/
+    0x6000, 0x0080, 0x0f04, 0x7fff, 0x4f01, 0xc600, 0x0000, 0x20ff,
+  /* APTIF | APTMC |APTuCP |LSWFWID|MSWFWID|NC-SIMC|NC-SIC | VPDP  */
+    0x0028, 0x0003, 0x0000, 0x0000, 0x0000, 0x0003, 0x0000, 0xffff,
+  /*                            SW Section                         */
+    0x0100, 0xc000, 0x121c, 0xc007, 0xffff, 0xffff, 0xffff, 0xffff,
+  /*                      SW Section                       |CHKSUM */
+    0xffff, 0xffff, 0xffff, 0xffff, 0x0000, 0x0120, 0xffff, 0x0000,
+};
+
+static void e1000e_core_realize(E1000EState *s)
+{
+    s->core.owner = &s->parent_obj;
+    s->core.owner_nic = s->nic;
+}
+
+static void
+e1000e_unuse_msix_vectors(E1000EState *s, int num_vectors)
+{
+    int i;
+    for (i = 0; i < num_vectors; i++) {
+        msix_vector_unuse(PCI_DEVICE(s), i);
+    }
+}
+
+static bool
+e1000e_use_msix_vectors(E1000EState *s, int num_vectors)
+{
+    int i;
+    for (i = 0; i < num_vectors; i++) {
+        int res = msix_vector_use(PCI_DEVICE(s), i);
+        if (res < 0) {
+            trace_e1000e_msix_use_vector_fail(i, res);
+            e1000e_unuse_msix_vectors(s, i);
+            return false;
+        }
+    }
+    return true;
+}
+
+static void
+e1000e_init_msix(E1000EState *s)
+{
+    PCIDevice *d = PCI_DEVICE(s);
+    int res = msix_init(PCI_DEVICE(s), E1000E_MSIX_VEC_NUM,
+                        &s->msix,
+                        E1000E_MSIX_IDX, E1000E_MSIX_TABLE,
+                        &s->msix,
+                        E1000E_MSIX_IDX, E1000E_MSIX_PBA,
+                        0xA0);
+
+    if (res < 0) {
+        trace_e1000e_msix_init_fail(res);
+    } else {
+        if (!e1000e_use_msix_vectors(s, E1000E_MSIX_VEC_NUM)) {
+            msix_uninit(d, &s->msix, &s->msix);
+        }
+    }
+}
+
+static void
+e1000e_cleanup_msix(E1000EState *s)
+{
+    if (msix_enabled(PCI_DEVICE(s))) {
+        e1000e_unuse_msix_vectors(s, E1000E_MSIX_VEC_NUM);
+        msix_uninit(PCI_DEVICE(s), &s->msix, &s->msix);
+    }
+}
+
+static void
+e1000e_init_net_peer(E1000EState *s, PCIDevice *pci_dev, uint8_t *macaddr)
+{
+    DeviceState *dev = DEVICE(pci_dev);
+    NetClientState *nc;
+    int i;
+
+    s->nic = qemu_new_nic(&net_e1000e_info, &s->conf,
+        object_get_typename(OBJECT(s)), dev->id, s);
+
+    s->core.max_queue_num = s->conf.peers.queues - 1;
+
+    trace_e1000e_mac_set_permanent(MAC_ARG(macaddr));
+    memcpy(s->core.permanent_mac, macaddr, sizeof(s->core.permanent_mac));
+
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), macaddr);
+
+    /* Setup virtio headers */
+    if (s->disable_vnet) {
+        s->core.has_vnet = false;
+        trace_e1000e_cfg_support_virtio(false);
+        return;
+    } else {
+        s->core.has_vnet = true;
+    }
+
+    for (i = 0; i < s->conf.peers.queues; i++) {
+        nc = qemu_get_subqueue(s->nic, i);
+        if (!nc->peer || !qemu_has_vnet_hdr(nc->peer)) {
+            s->core.has_vnet = false;
+            trace_e1000e_cfg_support_virtio(false);
+            return;
+        }
+    }
+
+    trace_e1000e_cfg_support_virtio(true);
+
+    for (i = 0; i < s->conf.peers.queues; i++) {
+        nc = qemu_get_subqueue(s->nic, i);
+        qemu_set_vnet_hdr_len(nc->peer, sizeof(struct virtio_net_hdr));
+        qemu_using_vnet_hdr(nc->peer, true);
+    }
+}
+
+static inline uint64_t
+e1000e_gen_dsn(uint8_t *mac)
+{
+    return (uint64_t)(mac[5])        |
+           (uint64_t)(mac[4])  << 8  |
+           (uint64_t)(mac[3])  << 16 |
+           (uint64_t)(0x00FF)  << 24 |
+           (uint64_t)(0x00FF)  << 32 |
+           (uint64_t)(mac[2])  << 40 |
+           (uint64_t)(mac[1])  << 48 |
+           (uint64_t)(mac[0])  << 56;
+}
+
+static int
+e1000e_add_pm_capability(PCIDevice *pdev, uint8_t offset, uint16_t pmc)
+{
+    int ret = pci_add_capability(pdev, PCI_CAP_ID_PM, offset, PCI_PM_SIZEOF);
+
+    if (ret >= 0) {
+        pci_set_word(pdev->config + offset + PCI_PM_PMC,
+                     PCI_PM_CAP_VER_1_1 |
+                     pmc);
+
+        pci_set_word(pdev->wmask + offset + PCI_PM_CTRL,
+                     PCI_PM_CTRL_STATE_MASK |
+                     PCI_PM_CTRL_PME_ENABLE |
+                     PCI_PM_CTRL_DATA_SEL_MASK);
+
+        pci_set_word(pdev->w1cmask + offset + PCI_PM_CTRL,
+                     PCI_PM_CTRL_PME_STATUS);
+    }
+
+    return ret;
+}
+
+static void e1000e_write_config(PCIDevice *pci_dev, uint32_t address,
+                                uint32_t val, int len)
+{
+    E1000EState *s = E1000E(pci_dev);
+
+    pci_default_write_config(pci_dev, address, val, len);
+
+    if (range_covers_byte(address, len, PCI_COMMAND) &&
+        (pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+}
+
+static void e1000e_pci_realize(PCIDevice *pci_dev, Error **errp)
+{
+    static const uint16_t e1000e_pmrb_offset = 0x0C8;
+    static const uint16_t e1000e_pcie_offset = 0x0E0;
+    static const uint16_t e1000e_aer_offset =  0x100;
+    static const uint16_t e1000e_dsn_offset =  0x140;
+    E1000EState *s = E1000E(pci_dev);
+    uint8_t *macaddr;
+    int ret;
+
+    trace_e1000e_cb_pci_realize();
+
+    pci_dev->config_write = e1000e_write_config;
+
+    pci_dev->config[PCI_CACHE_LINE_SIZE] = 0x10;
+    pci_dev->config[PCI_INTERRUPT_PIN] = 1;
+
+    pci_set_word(pci_dev->config + PCI_SUBSYSTEM_VENDOR_ID, s->subsys_ven);
+    pci_set_word(pci_dev->config + PCI_SUBSYSTEM_ID, s->subsys);
+
+    s->subsys_ven_used = s->subsys_ven;
+    s->subsys_used = s->subsys;
+
+    /* Define IO/MMIO regions */
+    memory_region_init_io(&s->mmio, OBJECT(s), &mmio_ops, s,
+                          "e1000e-mmio", E1000E_MMIO_SIZE);
+    pci_register_bar(pci_dev, E1000E_MMIO_IDX,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mmio);
+
+    /*
+     * We provide a dummy implementation for the flash BAR
+     * for drivers that may theoretically probe for its presence.
+     */
+    memory_region_init(&s->flash, OBJECT(s),
+                       "e1000e-flash", E1000E_FLASH_SIZE);
+    pci_register_bar(pci_dev, E1000E_FLASH_IDX,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &s->flash);
+
+    memory_region_init_io(&s->io, OBJECT(s), &io_ops, s,
+                          "e1000e-io", E1000E_IO_SIZE);
+    pci_register_bar(pci_dev, E1000E_IO_IDX,
+                     PCI_BASE_ADDRESS_SPACE_IO, &s->io);
+
+    memory_region_init(&s->msix, OBJECT(s), "e1000e-msix",
+                       E1000E_MSIX_SIZE);
+    pci_register_bar(pci_dev, E1000E_MSIX_IDX,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &s->msix);
+
+    /* Create networking backend */
+    qemu_macaddr_default_if_unset(&s->conf.macaddr);
+    macaddr = s->conf.macaddr.a;
+
+    e1000e_init_msix(s);
+
+    if (pcie_endpoint_cap_v1_init(pci_dev, e1000e_pcie_offset) < 0) {
+        hw_error("Failed to initialize PCIe capability");
+    }
+
+    ret = msi_init(PCI_DEVICE(s), 0xD0, 1, true, false, NULL);
+    if (ret) {
+        trace_e1000e_msi_init_fail(ret);
+    }
+
+    if (e1000e_add_pm_capability(pci_dev, e1000e_pmrb_offset,
+                                  PCI_PM_CAP_DSI) < 0) {
+        hw_error("Failed to initialize PM capability");
+    }
+
+    if (pcie_aer_init(pci_dev, e1000e_aer_offset, PCI_ERR_SIZEOF) < 0) {
+        hw_error("Failed to initialize AER capability");
+    }
+
+    pcie_dev_ser_num_init(pci_dev, e1000e_dsn_offset,
+                          e1000e_gen_dsn(macaddr));
+
+    e1000e_init_net_peer(s, pci_dev, macaddr);
+
+    /* Initialize core */
+    e1000e_core_realize(s);
+
+    e1000e_core_pci_realize(&s->core,
+                            e1000e_eeprom_template,
+                            sizeof(e1000e_eeprom_template),
+                            macaddr);
+}
+
+static void e1000e_pci_uninit(PCIDevice *pci_dev)
+{
+    E1000EState *s = E1000E(pci_dev);
+
+    trace_e1000e_cb_pci_uninit();
+
+    e1000e_core_pci_uninit(&s->core);
+
+    pcie_aer_exit(pci_dev);
+    pcie_cap_exit(pci_dev);
+
+    qemu_del_nic(s->nic);
+
+    e1000e_cleanup_msix(s);
+    msi_uninit(pci_dev);
+}
+
+static void e1000e_qdev_reset(DeviceState *dev)
+{
+    E1000EState *s = E1000E(dev);
+
+    trace_e1000e_cb_qdev_reset();
+
+    e1000e_core_reset(&s->core);
+}
+
+static void e1000e_pre_save(void *opaque)
+{
+    E1000EState *s = opaque;
+
+    trace_e1000e_cb_pre_save();
+
+    e1000e_core_pre_save(&s->core);
+}
+
+static int e1000e_post_load(void *opaque, int version_id)
+{
+    E1000EState *s = opaque;
+
+    trace_e1000e_cb_post_load();
+
+    if ((s->subsys != s->subsys_used) ||
+        (s->subsys_ven != s->subsys_ven_used)) {
+        fprintf(stderr,
+            "ERROR: Cannot migrate while device properties "
+            "(subsys/subsys_ven) differ");
+        return -1;
+    }
+
+    return e1000e_core_post_load(&s->core);
+}
+
+static const VMStateDescription e1000e_vmstate_tx = {
+    .name = "e1000e-tx",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(props.sum_needed, struct e1000e_tx),
+        VMSTATE_UINT8(props.ipcss, struct e1000e_tx),
+        VMSTATE_UINT8(props.ipcso, struct e1000e_tx),
+        VMSTATE_UINT16(props.ipcse, struct e1000e_tx),
+        VMSTATE_UINT8(props.tucss, struct e1000e_tx),
+        VMSTATE_UINT8(props.tucso, struct e1000e_tx),
+        VMSTATE_UINT16(props.tucse, struct e1000e_tx),
+        VMSTATE_UINT8(props.hdr_len, struct e1000e_tx),
+        VMSTATE_UINT16(props.mss, struct e1000e_tx),
+        VMSTATE_UINT32(props.paylen, struct e1000e_tx),
+        VMSTATE_INT8(props.ip, struct e1000e_tx),
+        VMSTATE_INT8(props.tcp, struct e1000e_tx),
+        VMSTATE_BOOL(props.tse, struct e1000e_tx),
+        VMSTATE_BOOL(props.cptse, struct e1000e_tx),
+        VMSTATE_BOOL(skip_cp, struct e1000e_tx),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription e1000e_vmstate_intr_timer = {
+    .name = "e1000e-intr-timer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_TIMER_PTR(timer, E1000IntrDelayTimer),
+        VMSTATE_BOOL(running, E1000IntrDelayTimer),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define VMSTATE_E1000E_INTR_DELAY_TIMER(_f, _s)                     \
+    VMSTATE_STRUCT(_f, _s, 0,                                       \
+                   e1000e_vmstate_intr_timer, E1000IntrDelayTimer)
+
+#define VMSTATE_E1000E_INTR_DELAY_TIMER_ARRAY(_f, _s, _num)         \
+    VMSTATE_STRUCT_ARRAY(_f, _s, _num, 0,                           \
+                         e1000e_vmstate_intr_timer, E1000IntrDelayTimer)
+
+static const VMStateDescription e1000e_vmstate = {
+    .name = "e1000e",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_save = e1000e_pre_save,
+    .post_load = e1000e_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(parent_obj, E1000EState),
+        VMSTATE_MSIX(parent_obj, E1000EState),
+
+        VMSTATE_UINT32(ioaddr, E1000EState),
+        VMSTATE_UINT32(core.rxbuf_min_shift, E1000EState),
+        VMSTATE_UINT8(core.rx_desc_len, E1000EState),
+        VMSTATE_UINT32_ARRAY(core.rxbuf_sizes, E1000EState,
+                             E1000_PSRCTL_BUFFS_PER_DESC),
+        VMSTATE_UINT32(core.rx_desc_buf_size, E1000EState),
+        VMSTATE_UINT16_ARRAY(core.eeprom, E1000EState, E1000E_EEPROM_SIZE),
+        VMSTATE_UINT16_2DARRAY(core.phy, E1000EState,
+                               E1000E_PHY_PAGES, E1000E_PHY_PAGE_SIZE),
+        VMSTATE_UINT32_ARRAY(core.mac, E1000EState, E1000E_MAC_SIZE),
+        VMSTATE_UINT8_ARRAY(core.permanent_mac, E1000EState, ETH_ALEN),
+
+        VMSTATE_UINT32(core.delayed_causes, E1000EState),
+
+        VMSTATE_UINT16(subsys, E1000EState),
+        VMSTATE_UINT16(subsys_ven, E1000EState),
+
+        VMSTATE_E1000E_INTR_DELAY_TIMER(core.rdtr, E1000EState),
+        VMSTATE_E1000E_INTR_DELAY_TIMER(core.radv, E1000EState),
+        VMSTATE_E1000E_INTR_DELAY_TIMER(core.raid, E1000EState),
+        VMSTATE_E1000E_INTR_DELAY_TIMER(core.tadv, E1000EState),
+        VMSTATE_E1000E_INTR_DELAY_TIMER(core.tidv, E1000EState),
+
+        VMSTATE_E1000E_INTR_DELAY_TIMER(core.itr, E1000EState),
+        VMSTATE_BOOL(core.itr_intr_pending, E1000EState),
+
+        VMSTATE_E1000E_INTR_DELAY_TIMER_ARRAY(core.eitr, E1000EState,
+                                              E1000E_MSIX_VEC_NUM),
+        VMSTATE_BOOL_ARRAY(core.eitr_intr_pending, E1000EState,
+                           E1000E_MSIX_VEC_NUM),
+
+        VMSTATE_UINT32(core.itr_guest_value, E1000EState),
+        VMSTATE_UINT32_ARRAY(core.eitr_guest_value, E1000EState,
+                             E1000E_MSIX_VEC_NUM),
+
+        VMSTATE_UINT16(core.vet, E1000EState),
+
+        VMSTATE_STRUCT_ARRAY(core.tx, E1000EState, E1000E_NUM_QUEUES, 0,
+                             e1000e_vmstate_tx, struct e1000e_tx),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static PropertyInfo e1000e_prop_disable_vnet,
+                    e1000e_prop_subsys_ven,
+                    e1000e_prop_subsys;
+
+static Property e1000e_properties[] = {
+    DEFINE_NIC_PROPERTIES(E1000EState, conf),
+    DEFINE_PROP_DEFAULT("disable_vnet_hdr", E1000EState, disable_vnet, false,
+                        e1000e_prop_disable_vnet, bool),
+    DEFINE_PROP_DEFAULT("subsys_ven", E1000EState, subsys_ven,
+                        PCI_VENDOR_ID_INTEL,
+                        e1000e_prop_subsys_ven, uint16_t),
+    DEFINE_PROP_DEFAULT("subsys", E1000EState, subsys, 0,
+                        e1000e_prop_subsys, uint16_t),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void e1000e_class_init(ObjectClass *class, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(class);
+    PCIDeviceClass *c = PCI_DEVICE_CLASS(class);
+
+    c->realize = e1000e_pci_realize;
+    c->exit = e1000e_pci_uninit;
+    c->vendor_id = PCI_VENDOR_ID_INTEL;
+    c->device_id = E1000_DEV_ID_82574L;
+    c->revision = 0;
+    c->romfile = "efi-e1000e.rom";
+    c->class_id = PCI_CLASS_NETWORK_ETHERNET;
+    c->is_express = 1;
+
+    dc->desc = "Intel 82574L GbE Controller";
+    dc->reset = e1000e_qdev_reset;
+    dc->vmsd = &e1000e_vmstate;
+    dc->props = e1000e_properties;
+
+    e1000e_prop_disable_vnet = qdev_prop_uint8;
+    e1000e_prop_disable_vnet.description = "Do not use virtio headers, "
+                                           "perform SW offloads emulation "
+                                           "instead";
+
+    e1000e_prop_subsys_ven = qdev_prop_uint16;
+    e1000e_prop_subsys_ven.description = "PCI device Subsystem Vendor ID";
+
+    e1000e_prop_subsys = qdev_prop_uint16;
+    e1000e_prop_subsys.description = "PCI device Subsystem ID";
+
+    set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
+}
+
+static void e1000e_instance_init(Object *obj)
+{
+    E1000EState *s = E1000E(obj);
+    device_add_bootindex_property(obj, &s->conf.bootindex,
+                                  "bootindex", "/ethernet-phy@0",
+                                  DEVICE(obj), NULL);
+}
+
+static const TypeInfo e1000e_info = {
+    .name = TYPE_E1000E,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(E1000EState),
+    .class_init = e1000e_class_init,
+    .instance_init = e1000e_instance_init,
+};
+
+static void e1000e_register_types(void)
+{
+    type_register_static(&e1000e_info);
+}
+
+type_init(e1000e_register_types)
diff --git a/hw/net/e1000e_core.c b/hw/net/e1000e_core.c
new file mode 100644
index 0000000..badb1fe
--- /dev/null
+++ b/hw/net/e1000e_core.c
@@ -0,0 +1,3483 @@
+/*
+* Core code for QEMU e1000e emulation
+*
+* Software developer's manuals:
+* http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf
+*
+* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com)
+* Developed by Daynix Computing LTD (http://www.daynix.com)
+*
+* Authors:
+* Dmitry Fleytman <dmitry@daynix.com>
+* Leonid Bloch <leonid@daynix.com>
+* Yan Vugenfirer <yan@daynix.com>
+*
+* Based on work done by:
+* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
+* Copyright (c) 2008 Qumranet
+* Based on work done by:
+* Copyright (c) 2007 Dan Aloni
+* Copyright (c) 2004 Antony T Curtis
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "qemu/osdep.h"
+#include "sysemu/sysemu.h"
+#include "net/net.h"
+#include "net/tap.h"
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+
+#include "net_tx_pkt.h"
+#include "net_rx_pkt.h"
+
+#include "e1000x_common.h"
+#include "e1000e_core.h"
+
+#include "trace.h"
+
+#define E1000E_MIN_XITR     (500) /* No more then 7813 interrupts per
+                                     second according to spec 10.2.4.2 */
+#define E1000E_MAX_TX_FRAGS (64)
+
+static void
+e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val);
+
+static inline void
+e1000e_process_ts_option(E1000ECore *core, struct e1000_tx_desc *dp)
+{
+    if (le32_to_cpu(dp->upper.data) & E1000_TXD_EXTCMD_TSTAMP) {
+        trace_e1000e_wrn_no_ts_support();
+    }
+}
+
+static inline void
+e1000e_process_snap_option(E1000ECore *core, uint32_t cmd_and_length)
+{
+    if (cmd_and_length & E1000_TXD_CMD_SNAP) {
+        trace_e1000e_wrn_no_snap_support();
+    }
+}
+
+static inline void
+e1000e_raise_legacy_irq(E1000ECore *core)
+{
+    trace_e1000e_irq_legacy_notify(true);
+    e1000x_inc_reg_if_not_full(core->mac, IAC);
+    pci_set_irq(core->owner, 1);
+}
+
+static inline void
+e1000e_lower_legacy_irq(E1000ECore *core)
+{
+    trace_e1000e_irq_legacy_notify(false);
+    pci_set_irq(core->owner, 0);
+}
+
+static inline void
+e1000e_intrmgr_rearm_timer(E1000IntrDelayTimer *timer)
+{
+    int64_t delay_ns = (int64_t) timer->core->mac[timer->delay_reg] *
+                                 timer->delay_resolution_ns;
+
+    trace_e1000e_irq_rearm_timer(timer->delay_reg << 2, delay_ns);
+
+    timer_mod(timer->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + delay_ns);
+
+    timer->running = true;
+}
+
+static void
+e1000e_intmgr_timer_resume(E1000IntrDelayTimer *timer)
+{
+    if (timer->running) {
+        e1000e_intrmgr_rearm_timer(timer);
+    }
+}
+
+static void
+e1000e_intmgr_timer_pause(E1000IntrDelayTimer *timer)
+{
+    if (timer->running) {
+        timer_del(timer->timer);
+    }
+}
+
+static inline void
+e1000e_intrmgr_stop_timer(E1000IntrDelayTimer *timer)
+{
+    if (timer->running) {
+        timer_del(timer->timer);
+        timer->running = false;
+    }
+}
+
+static inline void
+e1000e_intrmgr_fire_delayed_interrupts(E1000ECore *core)
+{
+    trace_e1000e_irq_fire_delayed_interrupts();
+    e1000e_set_interrupt_cause(core, 0);
+}
+
+static void
+e1000e_intrmgr_on_timer(void *opaque)
+{
+    E1000IntrDelayTimer *timer = opaque;
+
+    trace_e1000e_irq_throttling_timer(timer->delay_reg << 2);
+
+    timer->running = false;
+    e1000e_intrmgr_fire_delayed_interrupts(timer->core);
+}
+
+static void
+e1000e_intrmgr_on_throttling_timer(void *opaque)
+{
+    E1000IntrDelayTimer *timer = opaque;
+
+    assert(!msix_enabled(timer->core->owner));
+
+    timer->running = false;
+
+    if (!timer->core->itr_intr_pending) {
+        trace_e1000e_irq_throttling_no_pending_interrupts();
+        return;
+    }
+
+    if (msi_enabled(timer->core->owner)) {
+        trace_e1000e_irq_msi_notify_postponed();
+        e1000e_set_interrupt_cause(timer->core, 0);
+    } else {
+        trace_e1000e_irq_legacy_notify_postponed();
+        e1000e_set_interrupt_cause(timer->core, 0);
+    }
+}
+
+static void
+e1000e_intrmgr_on_msix_throttling_timer(void *opaque)
+{
+    E1000IntrDelayTimer *timer = opaque;
+    int idx = timer - &timer->core->eitr[0];
+
+    assert(msix_enabled(timer->core->owner));
+
+    timer->running = false;
+
+    if (!timer->core->eitr_intr_pending[idx]) {
+        trace_e1000e_irq_throttling_no_pending_vec(idx);
+        return;
+    }
+
+    trace_e1000e_irq_msix_notify_postponed_vec(idx);
+    msix_notify(timer->core->owner, idx);
+}
+
+static void
+e1000e_intrmgr_initialize_all_timers(E1000ECore *core, bool create)
+{
+    int i;
+
+    core->radv.delay_reg = RADV;
+    core->rdtr.delay_reg = RDTR;
+    core->raid.delay_reg = RAID;
+    core->tadv.delay_reg = TADV;
+    core->tidv.delay_reg = TIDV;
+
+    core->radv.delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
+    core->rdtr.delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
+    core->raid.delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
+    core->tadv.delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
+    core->tidv.delay_resolution_ns = E1000_INTR_DELAY_NS_RES;
+
+    core->radv.core = core;
+    core->rdtr.core = core;
+    core->raid.core = core;
+    core->tadv.core = core;
+    core->tidv.core = core;
+
+    core->itr.core = core;
+    core->itr.delay_reg = ITR;
+    core->itr.delay_resolution_ns = E1000_INTR_THROTTLING_NS_RES;
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        core->eitr[i].core = core;
+        core->eitr[i].delay_reg = EITR + i;
+        core->eitr[i].delay_resolution_ns = E1000_INTR_THROTTLING_NS_RES;
+    }
+
+    if (!create) {
+        return;
+    }
+
+    core->radv.timer =
+        timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000e_intrmgr_on_timer, &core->radv);
+    core->rdtr.timer =
+        timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000e_intrmgr_on_timer, &core->rdtr);
+    core->raid.timer =
+        timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000e_intrmgr_on_timer, &core->raid);
+
+    core->tadv.timer =
+        timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000e_intrmgr_on_timer, &core->tadv);
+    core->tidv.timer =
+        timer_new_ns(QEMU_CLOCK_VIRTUAL, e1000e_intrmgr_on_timer, &core->tidv);
+
+    core->itr.timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                   e1000e_intrmgr_on_throttling_timer,
+                                   &core->itr);
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        core->eitr[i].timer =
+            timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                         e1000e_intrmgr_on_msix_throttling_timer,
+                         &core->eitr[i]);
+    }
+}
+
+static inline void
+e1000e_intrmgr_stop_delay_timers(E1000ECore *core)
+{
+    e1000e_intrmgr_stop_timer(&core->radv);
+    e1000e_intrmgr_stop_timer(&core->rdtr);
+    e1000e_intrmgr_stop_timer(&core->raid);
+    e1000e_intrmgr_stop_timer(&core->tidv);
+    e1000e_intrmgr_stop_timer(&core->tadv);
+}
+
+static bool
+e1000e_intrmgr_delay_rx_causes(E1000ECore *core, uint32_t *causes)
+{
+    uint32_t delayable_causes;
+    uint32_t rdtr = core->mac[RDTR];
+    uint32_t radv = core->mac[RADV];
+    uint32_t raid = core->mac[RAID];
+
+    if (msix_enabled(core->owner)) {
+        return false;
+    }
+
+    delayable_causes = E1000_ICR_RXQ0 |
+                       E1000_ICR_RXQ1 |
+                       E1000_ICR_RXT0;
+
+    if (!(core->mac[RFCTL] & E1000_RFCTL_ACK_DIS)) {
+        delayable_causes |= E1000_ICR_ACK;
+    }
+
+    /* Clean up all causes that may be delayed */
+    core->delayed_causes |= *causes & delayable_causes;
+    *causes &= ~delayable_causes;
+
+    /* Check if delayed RX interrupts disabled by client
+       or if there are causes that cannot be delayed */
+    if ((rdtr == 0) || (*causes != 0)) {
+        return false;
+    }
+
+    /* Check if delayed RX ACK interrupts disabled by client
+       and there is an ACK packet received */
+    if ((raid == 0) && (core->delayed_causes & E1000_ICR_ACK)) {
+        return false;
+    }
+
+    /* All causes delayed */
+    e1000e_intrmgr_rearm_timer(&core->rdtr);
+
+    if (!core->radv.running && (radv != 0)) {
+        e1000e_intrmgr_rearm_timer(&core->radv);
+    }
+
+    if (!core->raid.running && (core->delayed_causes & E1000_ICR_ACK)) {
+        e1000e_intrmgr_rearm_timer(&core->raid);
+    }
+
+    return true;
+}
+
+static bool
+e1000e_intrmgr_delay_tx_causes(E1000ECore *core, uint32_t *causes)
+{
+    static const uint32_t delayable_causes = E1000_ICR_TXQ0 |
+                                             E1000_ICR_TXQ1 |
+                                             E1000_ICR_TXQE |
+                                             E1000_ICR_TXDW;
+
+    if (msix_enabled(core->owner)) {
+        return false;
+    }
+
+    /* Clean up all causes that may be delayed */
+    core->delayed_causes |= *causes & delayable_causes;
+    *causes &= ~delayable_causes;
+
+    /* If there are causes that cannot be delayed */
+    if (*causes != 0) {
+        return false;
+    }
+
+    /* All causes delayed */
+    e1000e_intrmgr_rearm_timer(&core->tidv);
+
+    if (!core->tadv.running && (core->mac[TADV] != 0)) {
+        e1000e_intrmgr_rearm_timer(&core->tadv);
+    }
+
+    return true;
+}
+
+static uint32_t
+e1000e_intmgr_collect_delayed_causes(E1000ECore *core)
+{
+    uint32_t res;
+
+    if (msix_enabled(core->owner)) {
+        assert(core->delayed_causes == 0);
+        return 0;
+    }
+
+    res = core->delayed_causes;
+    core->delayed_causes = 0;
+
+    e1000e_intrmgr_stop_delay_timers(core);
+
+    return res;
+}
+
+static void
+e1000e_intrmgr_fire_all_timers(E1000ECore *core)
+{
+    int i;
+    uint32_t val = e1000e_intmgr_collect_delayed_causes(core);
+
+    trace_e1000e_irq_adding_delayed_causes(val, core->mac[ICR]);
+    core->mac[ICR] |= val;
+
+    if (core->itr.running) {
+        timer_del(core->itr.timer);
+        e1000e_intrmgr_on_throttling_timer(&core->itr);
+    }
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        if (core->eitr[i].running) {
+            timer_del(core->eitr[i].timer);
+            e1000e_intrmgr_on_msix_throttling_timer(&core->eitr[i]);
+        }
+    }
+}
+
+static void
+e1000e_intrmgr_resume(E1000ECore *core)
+{
+    int i;
+
+    e1000e_intmgr_timer_resume(&core->radv);
+    e1000e_intmgr_timer_resume(&core->rdtr);
+    e1000e_intmgr_timer_resume(&core->raid);
+    e1000e_intmgr_timer_resume(&core->tidv);
+    e1000e_intmgr_timer_resume(&core->tadv);
+
+    e1000e_intmgr_timer_resume(&core->itr);
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        e1000e_intmgr_timer_resume(&core->eitr[i]);
+    }
+}
+
+static void
+e1000e_intrmgr_pause(E1000ECore *core)
+{
+    int i;
+
+    e1000e_intmgr_timer_pause(&core->radv);
+    e1000e_intmgr_timer_pause(&core->rdtr);
+    e1000e_intmgr_timer_pause(&core->raid);
+    e1000e_intmgr_timer_pause(&core->tidv);
+    e1000e_intmgr_timer_pause(&core->tadv);
+
+    e1000e_intmgr_timer_pause(&core->itr);
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        e1000e_intmgr_timer_pause(&core->eitr[i]);
+    }
+}
+
+static void
+e1000e_intrmgr_reset(E1000ECore *core)
+{
+    int i;
+
+    core->delayed_causes = 0;
+
+    e1000e_intrmgr_stop_delay_timers(core);
+
+    e1000e_intrmgr_stop_timer(&core->itr);
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        e1000e_intrmgr_stop_timer(&core->eitr[i]);
+    }
+}
+
+static void
+e1000e_intrmgr_pci_unint(E1000ECore *core)
+{
+    int i;
+
+    timer_del(core->radv.timer);
+    timer_free(core->radv.timer);
+    timer_del(core->rdtr.timer);
+    timer_free(core->rdtr.timer);
+    timer_del(core->raid.timer);
+    timer_free(core->raid.timer);
+
+    timer_del(core->tadv.timer);
+    timer_free(core->tadv.timer);
+    timer_del(core->tidv.timer);
+    timer_free(core->tidv.timer);
+
+    timer_del(core->itr.timer);
+    timer_free(core->itr.timer);
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        timer_del(core->eitr[i].timer);
+        timer_free(core->eitr[i].timer);
+    }
+}
+
+static void
+e1000e_intrmgr_pci_realize(E1000ECore *core)
+{
+    e1000e_intrmgr_initialize_all_timers(core, true);
+}
+
+static inline bool
+e1000e_rx_csum_enabled(E1000ECore *core)
+{
+    return (core->mac[RXCSUM] & E1000_RXCSUM_PCSD) ? false : true;
+}
+
+static inline bool
+e1000e_rx_use_legacy_descriptor(E1000ECore *core)
+{
+    return (core->mac[RFCTL] & E1000_RFCTL_EXTEN) ? false : true;
+}
+
+static inline bool
+e1000e_rx_use_ps_descriptor(E1000ECore *core)
+{
+    return !e1000e_rx_use_legacy_descriptor(core) &&
+           (core->mac[RCTL] & E1000_RCTL_DTYP_PS);
+}
+
+static inline bool
+e1000e_rss_enabled(E1000ECore *core)
+{
+    return E1000_MRQC_ENABLED(core->mac[MRQC]) &&
+           !e1000e_rx_csum_enabled(core) &&
+           !e1000e_rx_use_legacy_descriptor(core);
+}
+
+typedef struct E1000E_RSSInfo_st {
+    bool enabled;
+    uint32_t hash;
+    uint32_t queue;
+    uint32_t type;
+} E1000E_RSSInfo;
+
+static uint32_t
+e1000e_rss_get_hash_type(E1000ECore *core, struct NetRxPkt *pkt)
+{
+    bool isip4, isip6, isudp, istcp;
+
+    assert(e1000e_rss_enabled(core));
+
+    net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
+
+    if (isip4) {
+        bool fragment = net_rx_pkt_get_ip4_info(pkt)->fragment;
+
+        trace_e1000e_rx_rss_ip4(fragment, istcp, core->mac[MRQC],
+                                E1000_MRQC_EN_TCPIPV4(core->mac[MRQC]),
+                                E1000_MRQC_EN_IPV4(core->mac[MRQC]));
+
+        if (!fragment && istcp && E1000_MRQC_EN_TCPIPV4(core->mac[MRQC])) {
+            return E1000_MRQ_RSS_TYPE_IPV4TCP;
+        }
+
+        if (E1000_MRQC_EN_IPV4(core->mac[MRQC])) {
+            return E1000_MRQ_RSS_TYPE_IPV4;
+        }
+    } else if (isip6) {
+        eth_ip6_hdr_info *ip6info = net_rx_pkt_get_ip6_info(pkt);
+
+        bool ex_dis = core->mac[RFCTL] & E1000_RFCTL_IPV6_EX_DIS;
+        bool new_ex_dis = core->mac[RFCTL] & E1000_RFCTL_NEW_IPV6_EXT_DIS;
+
+        /*
+         * Following two traces must not be combined because resulting
+         * event will have 11 arguments totally and some trace backends
+         * (at least "ust") have limitation of maximum 10 arguments per
+         * event. Events with more arguments fail to compile for
+         * backends like these.
+         */
+        trace_e1000e_rx_rss_ip6_rfctl(core->mac[RFCTL]);
+        trace_e1000e_rx_rss_ip6(ex_dis, new_ex_dis, istcp,
+                                ip6info->has_ext_hdrs,
+                                ip6info->rss_ex_dst_valid,
+                                ip6info->rss_ex_src_valid,
+                                core->mac[MRQC],
+                                E1000_MRQC_EN_TCPIPV6(core->mac[MRQC]),
+                                E1000_MRQC_EN_IPV6EX(core->mac[MRQC]),
+                                E1000_MRQC_EN_IPV6(core->mac[MRQC]));
+
+        if ((!ex_dis || !ip6info->has_ext_hdrs) &&
+            (!new_ex_dis || !(ip6info->rss_ex_dst_valid ||
+                              ip6info->rss_ex_src_valid))) {
+
+            if (istcp && !ip6info->fragment &&
+                E1000_MRQC_EN_TCPIPV6(core->mac[MRQC])) {
+                return E1000_MRQ_RSS_TYPE_IPV6TCP;
+            }
+
+            if (E1000_MRQC_EN_IPV6EX(core->mac[MRQC])) {
+                return E1000_MRQ_RSS_TYPE_IPV6EX;
+            }
+
+        }
+
+        if (E1000_MRQC_EN_IPV6(core->mac[MRQC])) {
+            return E1000_MRQ_RSS_TYPE_IPV6;
+        }
+
+    }
+
+    return E1000_MRQ_RSS_TYPE_NONE;
+}
+
+static uint32_t
+e1000e_rss_calc_hash(E1000ECore *core,
+                     struct NetRxPkt *pkt,
+                     E1000E_RSSInfo *info)
+{
+    NetRxPktRssType type;
+
+    assert(e1000e_rss_enabled(core));
+
+    switch (info->type) {
+    case E1000_MRQ_RSS_TYPE_IPV4:
+        type = NetPktRssIpV4;
+        break;
+    case E1000_MRQ_RSS_TYPE_IPV4TCP:
+        type = NetPktRssIpV4Tcp;
+        break;
+    case E1000_MRQ_RSS_TYPE_IPV6TCP:
+        type = NetPktRssIpV6Tcp;
+        break;
+    case E1000_MRQ_RSS_TYPE_IPV6:
+        type = NetPktRssIpV6;
+        break;
+    case E1000_MRQ_RSS_TYPE_IPV6EX:
+        type = NetPktRssIpV6Ex;
+        break;
+    default:
+        assert(false);
+        return 0;
+    }
+
+    return net_rx_pkt_calc_rss_hash(pkt, type, (uint8_t *) &core->mac[RSSRK]);
+}
+
+static void
+e1000e_rss_parse_packet(E1000ECore *core,
+                        struct NetRxPkt *pkt,
+                        E1000E_RSSInfo *info)
+{
+    trace_e1000e_rx_rss_started();
+
+    if (!e1000e_rss_enabled(core)) {
+        info->enabled = false;
+        info->hash = 0;
+        info->queue = 0;
+        info->type = 0;
+        trace_e1000e_rx_rss_disabled();
+        return;
+    }
+
+    info->enabled = true;
+
+    info->type = e1000e_rss_get_hash_type(core, pkt);
+
+    trace_e1000e_rx_rss_type(info->type);
+
+    if (info->type == E1000_MRQ_RSS_TYPE_NONE) {
+        info->hash = 0;
+        info->queue = 0;
+        return;
+    }
+
+    info->hash = e1000e_rss_calc_hash(core, pkt, info);
+    info->queue = E1000_RSS_QUEUE(&core->mac[RETA], info->hash);
+}
+
+static void
+e1000e_setup_tx_offloads(E1000ECore *core, struct e1000e_tx *tx)
+{
+    if (tx->props.tse && tx->props.cptse) {
+        net_tx_pkt_build_vheader(tx->tx_pkt, true, true, tx->props.mss);
+        net_tx_pkt_update_ip_checksums(tx->tx_pkt);
+        e1000x_inc_reg_if_not_full(core->mac, TSCTC);
+        return;
+    }
+
+    if (tx->props.sum_needed & E1000_TXD_POPTS_TXSM) {
+        net_tx_pkt_build_vheader(tx->tx_pkt, false, true, 0);
+    }
+
+    if (tx->props.sum_needed & E1000_TXD_POPTS_IXSM) {
+        net_tx_pkt_update_ip_hdr_checksum(tx->tx_pkt);
+    }
+}
+
+static bool
+e1000e_tx_pkt_send(E1000ECore *core, struct e1000e_tx *tx, int queue_index)
+{
+    int target_queue = MIN(core->max_queue_num, queue_index);
+    NetClientState *queue = qemu_get_subqueue(core->owner_nic, target_queue);
+
+    e1000e_setup_tx_offloads(core, tx);
+
+    net_tx_pkt_dump(tx->tx_pkt);
+
+    if ((core->phy[0][PHY_CTRL] & MII_CR_LOOPBACK) ||
+        ((core->mac[RCTL] & E1000_RCTL_LBM_MAC) == E1000_RCTL_LBM_MAC)) {
+        return net_tx_pkt_send_loopback(tx->tx_pkt, queue);
+    } else {
+        return net_tx_pkt_send(tx->tx_pkt, queue);
+    }
+}
+
+static void
+e1000e_on_tx_done_update_stats(E1000ECore *core, struct NetTxPkt *tx_pkt)
+{
+    static const int PTCregs[6] = { PTC64, PTC127, PTC255, PTC511,
+                                    PTC1023, PTC1522 };
+
+    size_t tot_len = net_tx_pkt_get_total_len(tx_pkt);
+
+    e1000x_increase_size_stats(core->mac, PTCregs, tot_len);
+    e1000x_inc_reg_if_not_full(core->mac, TPT);
+    e1000x_grow_8reg_if_not_full(core->mac, TOTL, tot_len);
+
+    switch (net_tx_pkt_get_packet_type(tx_pkt)) {
+    case ETH_PKT_BCAST:
+        e1000x_inc_reg_if_not_full(core->mac, BPTC);
+        break;
+    case ETH_PKT_MCAST:
+        e1000x_inc_reg_if_not_full(core->mac, MPTC);
+        break;
+    case ETH_PKT_UCAST:
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    core->mac[GPTC] = core->mac[TPT];
+    core->mac[GOTCL] = core->mac[TOTL];
+    core->mac[GOTCH] = core->mac[TOTH];
+}
+
+static void
+e1000e_process_tx_desc(E1000ECore *core,
+                       struct e1000e_tx *tx,
+                       struct e1000_tx_desc *dp,
+                       int queue_index)
+{
+    uint32_t txd_lower = le32_to_cpu(dp->lower.data);
+    uint32_t dtype = txd_lower & (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D);
+    unsigned int split_size = txd_lower & 0xffff;
+    uint64_t addr;
+    struct e1000_context_desc *xp = (struct e1000_context_desc *)dp;
+    bool eop = txd_lower & E1000_TXD_CMD_EOP;
+
+    if (dtype == E1000_TXD_CMD_DEXT) { /* context descriptor */
+        e1000x_read_tx_ctx_descr(xp, &tx->props);
+        e1000e_process_snap_option(core, le32_to_cpu(xp->cmd_and_length));
+        return;
+    } else if (dtype == (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D)) {
+        /* data descriptor */
+        tx->props.sum_needed = le32_to_cpu(dp->upper.data) >> 8;
+        tx->props.cptse = (txd_lower & E1000_TXD_CMD_TSE) ? 1 : 0;
+        e1000e_process_ts_option(core, dp);
+    } else {
+        /* legacy descriptor */
+        e1000e_process_ts_option(core, dp);
+        tx->props.cptse = 0;
+    }
+
+    addr = le64_to_cpu(dp->buffer_addr);
+
+    if (!tx->skip_cp) {
+        if (!net_tx_pkt_add_raw_fragment(tx->tx_pkt, addr, split_size)) {
+            tx->skip_cp = true;
+        }
+    }
+
+    if (eop) {
+        if (!tx->skip_cp && net_tx_pkt_parse(tx->tx_pkt)) {
+            if (e1000x_vlan_enabled(core->mac) &&
+                e1000x_is_vlan_txd(txd_lower)) {
+                net_tx_pkt_setup_vlan_header_ex(tx->tx_pkt,
+                    le16_to_cpu(dp->upper.fields.special), core->vet);
+            }
+            if (e1000e_tx_pkt_send(core, tx, queue_index)) {
+                e1000e_on_tx_done_update_stats(core, tx->tx_pkt);
+            }
+        }
+
+        tx->skip_cp = false;
+        net_tx_pkt_reset(tx->tx_pkt);
+
+        tx->props.sum_needed = 0;
+        tx->props.cptse = 0;
+    }
+}
+
+static inline uint32_t
+e1000e_tx_wb_interrupt_cause(E1000ECore *core, int queue_idx)
+{
+    if (!msix_enabled(core->owner)) {
+        return E1000_ICR_TXDW;
+    }
+
+    return (queue_idx == 0) ? E1000_ICR_TXQ0 : E1000_ICR_TXQ1;
+}
+
+static inline uint32_t
+e1000e_rx_wb_interrupt_cause(E1000ECore *core, int queue_idx,
+                             bool min_threshold_hit)
+{
+    if (!msix_enabled(core->owner)) {
+        return E1000_ICS_RXT0 | (min_threshold_hit ? E1000_ICS_RXDMT0 : 0);
+    }
+
+    return (queue_idx == 0) ? E1000_ICR_RXQ0 : E1000_ICR_RXQ1;
+}
+
+static uint32_t
+e1000e_txdesc_writeback(E1000ECore *core, dma_addr_t base,
+                        struct e1000_tx_desc *dp, bool *ide, int queue_idx)
+{
+    uint32_t txd_upper, txd_lower = le32_to_cpu(dp->lower.data);
+
+    if (!(txd_lower & E1000_TXD_CMD_RS) &&
+        !(core->mac[IVAR] & E1000_IVAR_TX_INT_EVERY_WB)) {
+        return 0;
+    }
+
+    *ide = (txd_lower & E1000_TXD_CMD_IDE) ? true : false;
+
+    txd_upper = le32_to_cpu(dp->upper.data) | E1000_TXD_STAT_DD;
+
+    dp->upper.data = cpu_to_le32(txd_upper);
+    pci_dma_write(core->owner, base + ((char *)&dp->upper - (char *)dp),
+                  &dp->upper, sizeof(dp->upper));
+    return e1000e_tx_wb_interrupt_cause(core, queue_idx);
+}
+
+typedef struct E1000E_RingInfo_st {
+    int dbah;
+    int dbal;
+    int dlen;
+    int dh;
+    int dt;
+    int idx;
+} E1000E_RingInfo;
+
+static inline bool
+e1000e_ring_empty(E1000ECore *core, const E1000E_RingInfo *r)
+{
+    return core->mac[r->dh] == core->mac[r->dt];
+}
+
+static inline uint64_t
+e1000e_ring_base(E1000ECore *core, const E1000E_RingInfo *r)
+{
+    uint64_t bah = core->mac[r->dbah];
+    uint64_t bal = core->mac[r->dbal];
+
+    return (bah << 32) + bal;
+}
+
+static inline uint64_t
+e1000e_ring_head_descr(E1000ECore *core, const E1000E_RingInfo *r)
+{
+    return e1000e_ring_base(core, r) + E1000_RING_DESC_LEN * core->mac[r->dh];
+}
+
+static inline void
+e1000e_ring_advance(E1000ECore *core, const E1000E_RingInfo *r, uint32_t count)
+{
+    core->mac[r->dh] += count;
+
+    if (core->mac[r->dh] * E1000_RING_DESC_LEN >= core->mac[r->dlen]) {
+        core->mac[r->dh] = 0;
+    }
+}
+
+static inline uint32_t
+e1000e_ring_free_descr_num(E1000ECore *core, const E1000E_RingInfo *r)
+{
+    trace_e1000e_ring_free_space(r->idx, core->mac[r->dlen],
+                                 core->mac[r->dh],  core->mac[r->dt]);
+
+    if (core->mac[r->dh] <= core->mac[r->dt]) {
+        return core->mac[r->dt] - core->mac[r->dh];
+    }
+
+    if (core->mac[r->dh] > core->mac[r->dt]) {
+        return core->mac[r->dlen] / E1000_RING_DESC_LEN +
+               core->mac[r->dt] - core->mac[r->dh];
+    }
+
+    g_assert_not_reached();
+    return 0;
+}
+
+static inline bool
+e1000e_ring_enabled(E1000ECore *core, const E1000E_RingInfo *r)
+{
+    return core->mac[r->dlen] > 0;
+}
+
+static inline uint32_t
+e1000e_ring_len(E1000ECore *core, const E1000E_RingInfo *r)
+{
+    return core->mac[r->dlen];
+}
+
+typedef struct E1000E_TxRing_st {
+    const E1000E_RingInfo *i;
+    struct e1000e_tx *tx;
+} E1000E_TxRing;
+
+static inline int
+e1000e_mq_queue_idx(int base_reg_idx, int reg_idx)
+{
+    return (reg_idx - base_reg_idx) / (0x100 >> 2);
+}
+
+static inline void
+e1000e_tx_ring_init(E1000ECore *core, E1000E_TxRing *txr, int idx)
+{
+    static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = {
+        { TDBAH,  TDBAL,  TDLEN,  TDH,  TDT, 0 },
+        { TDBAH1, TDBAL1, TDLEN1, TDH1, TDT1, 1 }
+    };
+
+    assert(idx < ARRAY_SIZE(i));
+
+    txr->i     = &i[idx];
+    txr->tx    = &core->tx[idx];
+}
+
+typedef struct E1000E_RxRing_st {
+    const E1000E_RingInfo *i;
+} E1000E_RxRing;
+
+static inline void
+e1000e_rx_ring_init(E1000ECore *core, E1000E_RxRing *rxr, int idx)
+{
+    static const E1000E_RingInfo i[E1000E_NUM_QUEUES] = {
+        { RDBAH0, RDBAL0, RDLEN0, RDH0, RDT0, 0 },
+        { RDBAH1, RDBAL1, RDLEN1, RDH1, RDT1, 1 }
+    };
+
+    assert(idx < ARRAY_SIZE(i));
+
+    rxr->i      = &i[idx];
+}
+
+static void
+e1000e_start_xmit(E1000ECore *core, const E1000E_TxRing *txr)
+{
+    dma_addr_t base;
+    struct e1000_tx_desc desc;
+    bool ide = false;
+    const E1000E_RingInfo *txi = txr->i;
+    uint32_t cause = E1000_ICS_TXQE;
+
+    if (!(core->mac[TCTL] & E1000_TCTL_EN)) {
+        trace_e1000e_tx_disabled();
+        return;
+    }
+
+    while (!e1000e_ring_empty(core, txi)) {
+        base = e1000e_ring_head_descr(core, txi);
+
+        pci_dma_read(core->owner, base, &desc, sizeof(desc));
+
+        trace_e1000e_tx_descr((void *)(intptr_t)desc.buffer_addr,
+                              desc.lower.data, desc.upper.data);
+
+        e1000e_process_tx_desc(core, txr->tx, &desc, txi->idx);
+        cause |= e1000e_txdesc_writeback(core, base, &desc, &ide, txi->idx);
+
+        e1000e_ring_advance(core, txi, 1);
+    }
+
+    if (!ide || !e1000e_intrmgr_delay_tx_causes(core, &cause)) {
+        e1000e_set_interrupt_cause(core, cause);
+    }
+}
+
+static bool
+e1000e_has_rxbufs(E1000ECore *core, const E1000E_RingInfo *r,
+                  size_t total_size)
+{
+    uint32_t bufs = e1000e_ring_free_descr_num(core, r);
+
+    trace_e1000e_rx_has_buffers(r->idx, bufs, total_size,
+                                core->rx_desc_buf_size);
+
+    return total_size <= bufs / (core->rx_desc_len / E1000_MIN_RX_DESC_LEN) *
+                         core->rx_desc_buf_size;
+}
+
+static inline void
+e1000e_start_recv(E1000ECore *core)
+{
+    int i;
+
+    trace_e1000e_rx_start_recv();
+
+    for (i = 0; i <= core->max_queue_num; i++) {
+        qemu_flush_queued_packets(qemu_get_subqueue(core->owner_nic, i));
+    }
+}
+
+int
+e1000e_can_receive(E1000ECore *core)
+{
+    int i;
+
+    if (!e1000x_rx_ready(core->owner, core->mac)) {
+        return false;
+    }
+
+    for (i = 0; i < E1000E_NUM_QUEUES; i++) {
+        E1000E_RxRing rxr;
+
+        e1000e_rx_ring_init(core, &rxr, i);
+        if (e1000e_ring_enabled(core, rxr.i) &&
+            e1000e_has_rxbufs(core, rxr.i, 1)) {
+            trace_e1000e_rx_can_recv();
+            return true;
+        }
+    }
+
+    trace_e1000e_rx_can_recv_rings_full();
+    return false;
+}
+
+ssize_t
+e1000e_receive(E1000ECore *core, const uint8_t *buf, size_t size)
+{
+    const struct iovec iov = {
+        .iov_base = (uint8_t *)buf,
+        .iov_len = size
+    };
+
+    return e1000e_receive_iov(core, &iov, 1);
+}
+
+static inline bool
+e1000e_rx_l3_cso_enabled(E1000ECore *core)
+{
+    return !!(core->mac[RXCSUM] & E1000_RXCSUM_IPOFLD);
+}
+
+static inline bool
+e1000e_rx_l4_cso_enabled(E1000ECore *core)
+{
+    return !!(core->mac[RXCSUM] & E1000_RXCSUM_TUOFLD);
+}
+
+static bool
+e1000e_receive_filter(E1000ECore *core, const uint8_t *buf, int size)
+{
+    uint32_t rctl = core->mac[RCTL];
+
+    if (e1000x_is_vlan_packet(buf, core->vet) &&
+        e1000x_vlan_rx_filter_enabled(core->mac)) {
+        uint16_t vid = lduw_be_p(buf + 14);
+        uint32_t vfta = ldl_le_p((uint32_t *)(core->mac + VFTA) +
+                                 ((vid >> 5) & 0x7f));
+        if ((vfta & (1 << (vid & 0x1f))) == 0) {
+            trace_e1000e_rx_flt_vlan_mismatch(vid);
+            return false;
+        } else {
+            trace_e1000e_rx_flt_vlan_match(vid);
+        }
+    }
+
+    switch (net_rx_pkt_get_packet_type(core->rx_pkt)) {
+    case ETH_PKT_UCAST:
+        if (rctl & E1000_RCTL_UPE) {
+            return true; /* promiscuous ucast */
+        }
+        break;
+
+    case ETH_PKT_BCAST:
+        if (rctl & E1000_RCTL_BAM) {
+            return true; /* broadcast enabled */
+        }
+        break;
+
+    case ETH_PKT_MCAST:
+        if (rctl & E1000_RCTL_MPE) {
+            return true; /* promiscuous mcast */
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+    }
+
+    return e1000x_rx_group_filter(core->mac, buf);
+}
+
+static inline void
+e1000e_read_lgcy_rx_descr(E1000ECore *core, uint8_t *desc, hwaddr *buff_addr)
+{
+    struct e1000_rx_desc *d = (struct e1000_rx_desc *) desc;
+    *buff_addr = le64_to_cpu(d->buffer_addr);
+}
+
+static inline void
+e1000e_read_ext_rx_descr(E1000ECore *core, uint8_t *desc, hwaddr *buff_addr)
+{
+    union e1000_rx_desc_extended *d = (union e1000_rx_desc_extended *) desc;
+    *buff_addr = le64_to_cpu(d->read.buffer_addr);
+}
+
+static inline void
+e1000e_read_ps_rx_descr(E1000ECore *core, uint8_t *desc,
+                        hwaddr (*buff_addr)[MAX_PS_BUFFERS])
+{
+    int i;
+    union e1000_rx_desc_packet_split *d =
+        (union e1000_rx_desc_packet_split *) desc;
+
+    for (i = 0; i < MAX_PS_BUFFERS; i++) {
+        (*buff_addr)[i] = le64_to_cpu(d->read.buffer_addr[i]);
+    }
+
+    trace_e1000e_rx_desc_ps_read((*buff_addr)[0], (*buff_addr)[1],
+                                 (*buff_addr)[2], (*buff_addr)[3]);
+}
+
+static inline void
+e1000e_read_rx_descr(E1000ECore *core, uint8_t *desc,
+                     hwaddr (*buff_addr)[MAX_PS_BUFFERS])
+{
+    if (e1000e_rx_use_legacy_descriptor(core)) {
+        e1000e_read_lgcy_rx_descr(core, desc, &(*buff_addr)[0]);
+        (*buff_addr)[1] = (*buff_addr)[2] = (*buff_addr)[3] = 0;
+    } else {
+        if (core->mac[RCTL] & E1000_RCTL_DTYP_PS) {
+            e1000e_read_ps_rx_descr(core, desc, buff_addr);
+        } else {
+            e1000e_read_ext_rx_descr(core, desc, &(*buff_addr)[0]);
+            (*buff_addr)[1] = (*buff_addr)[2] = (*buff_addr)[3] = 0;
+        }
+    }
+}
+
+static void
+e1000e_verify_csum_in_sw(E1000ECore *core,
+                         struct NetRxPkt *pkt,
+                         uint32_t *status_flags,
+                         bool istcp, bool isudp)
+{
+    bool csum_valid;
+    uint32_t csum_error;
+
+    if (e1000e_rx_l3_cso_enabled(core)) {
+        if (!net_rx_pkt_validate_l3_csum(pkt, &csum_valid)) {
+            trace_e1000e_rx_metadata_l3_csum_validation_failed();
+        } else {
+            csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_IPE;
+            *status_flags |= E1000_RXD_STAT_IPCS | csum_error;
+        }
+    } else {
+        trace_e1000e_rx_metadata_l3_cso_disabled();
+    }
+
+    if (!e1000e_rx_l4_cso_enabled(core)) {
+        trace_e1000e_rx_metadata_l4_cso_disabled();
+        return;
+    }
+
+    if (!net_rx_pkt_validate_l4_csum(pkt, &csum_valid)) {
+        trace_e1000e_rx_metadata_l4_csum_validation_failed();
+        return;
+    }
+
+    csum_error = csum_valid ? 0 : E1000_RXDEXT_STATERR_TCPE;
+
+    if (istcp) {
+        *status_flags |= E1000_RXD_STAT_TCPCS |
+                         csum_error;
+    } else if (isudp) {
+        *status_flags |= E1000_RXD_STAT_TCPCS |
+                         E1000_RXD_STAT_UDPCS |
+                         csum_error;
+    }
+}
+
+static inline bool
+e1000e_is_tcp_ack(E1000ECore *core, struct NetRxPkt *rx_pkt)
+{
+    if (!net_rx_pkt_is_tcp_ack(rx_pkt)) {
+        return false;
+    }
+
+    if (core->mac[RFCTL] & E1000_RFCTL_ACK_DATA_DIS) {
+        return !net_rx_pkt_has_tcp_data(rx_pkt);
+    }
+
+    return true;
+}
+
+static void
+e1000e_build_rx_metadata(E1000ECore *core,
+                         struct NetRxPkt *pkt,
+                         bool is_eop,
+                         const E1000E_RSSInfo *rss_info,
+                         uint32_t *rss, uint32_t *mrq,
+                         uint32_t *status_flags,
+                         uint16_t *ip_id,
+                         uint16_t *vlan_tag)
+{
+    struct virtio_net_hdr *vhdr;
+    bool isip4, isip6, istcp, isudp;
+    uint32_t pkt_type;
+
+    *status_flags = E1000_RXD_STAT_DD;
+
+    /* No additional metadata needed for non-EOP descriptors */
+    if (!is_eop) {
+        goto func_exit;
+    }
+
+    *status_flags |= E1000_RXD_STAT_EOP;
+
+    net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
+    trace_e1000e_rx_metadata_protocols(isip4, isip6, isudp, istcp);
+
+    /* VLAN state */
+    if (net_rx_pkt_is_vlan_stripped(pkt)) {
+        *status_flags |= E1000_RXD_STAT_VP;
+        *vlan_tag = cpu_to_le16(net_rx_pkt_get_vlan_tag(pkt));
+        trace_e1000e_rx_metadata_vlan(*vlan_tag);
+    }
+
+    /* Packet parsing results */
+    if ((core->mac[RXCSUM] & E1000_RXCSUM_PCSD) != 0) {
+        if (rss_info->enabled) {
+            *rss = cpu_to_le32(rss_info->hash);
+            *mrq = cpu_to_le32(rss_info->type | (rss_info->queue << 8));
+            trace_e1000e_rx_metadata_rss(*rss, *mrq);
+        }
+    } else if (isip4) {
+            *status_flags |= E1000_RXD_STAT_IPIDV;
+            *ip_id = cpu_to_le16(net_rx_pkt_get_ip_id(pkt));
+            trace_e1000e_rx_metadata_ip_id(*ip_id);
+    }
+
+    if (istcp && e1000e_is_tcp_ack(core, pkt)) {
+        *status_flags |= E1000_RXD_STAT_ACK;
+        trace_e1000e_rx_metadata_ack();
+    }
+
+    if (isip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_DIS)) {
+        trace_e1000e_rx_metadata_ipv6_filtering_disabled();
+        pkt_type = E1000_RXD_PKT_MAC;
+    } else if (istcp || isudp) {
+        pkt_type = isip4 ? E1000_RXD_PKT_IP4_XDP : E1000_RXD_PKT_IP6_XDP;
+    } else if (isip4 || isip6) {
+        pkt_type = isip4 ? E1000_RXD_PKT_IP4 : E1000_RXD_PKT_IP6;
+    } else {
+        pkt_type = E1000_RXD_PKT_MAC;
+    }
+
+    *status_flags |= E1000_RXD_PKT_TYPE(pkt_type);
+    trace_e1000e_rx_metadata_pkt_type(pkt_type);
+
+    /* RX CSO information */
+    if (isip6 && (core->mac[RFCTL] & E1000_RFCTL_IPV6_XSUM_DIS)) {
+        trace_e1000e_rx_metadata_ipv6_sum_disabled();
+        goto func_exit;
+    }
+
+    if (!net_rx_pkt_has_virt_hdr(pkt)) {
+        trace_e1000e_rx_metadata_no_virthdr();
+        e1000e_verify_csum_in_sw(core, pkt, status_flags, istcp, isudp);
+        goto func_exit;
+    }
+
+    vhdr = net_rx_pkt_get_vhdr(pkt);
+
+    if (!(vhdr->flags & VIRTIO_NET_HDR_F_DATA_VALID) &&
+        !(vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
+        trace_e1000e_rx_metadata_virthdr_no_csum_info();
+        e1000e_verify_csum_in_sw(core, pkt, status_flags, istcp, isudp);
+        goto func_exit;
+    }
+
+    if (e1000e_rx_l3_cso_enabled(core)) {
+        *status_flags |= isip4 ? E1000_RXD_STAT_IPCS : 0;
+    } else {
+        trace_e1000e_rx_metadata_l3_cso_disabled();
+    }
+
+    if (e1000e_rx_l4_cso_enabled(core)) {
+        if (istcp) {
+            *status_flags |= E1000_RXD_STAT_TCPCS;
+        } else if (isudp) {
+            *status_flags |= E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS;
+        }
+    } else {
+        trace_e1000e_rx_metadata_l4_cso_disabled();
+    }
+
+    trace_e1000e_rx_metadata_status_flags(*status_flags);
+
+func_exit:
+    *status_flags = cpu_to_le32(*status_flags);
+}
+
+static inline void
+e1000e_write_lgcy_rx_descr(E1000ECore *core, uint8_t *desc,
+                           struct NetRxPkt *pkt,
+                           const E1000E_RSSInfo *rss_info,
+                           uint16_t length)
+{
+    uint32_t status_flags, rss, mrq;
+    uint16_t ip_id;
+
+    struct e1000_rx_desc *d = (struct e1000_rx_desc *) desc;
+
+    memset(d, 0, sizeof(*d));
+
+    assert(!rss_info->enabled);
+
+    d->length = cpu_to_le16(length);
+
+    e1000e_build_rx_metadata(core, pkt, pkt != NULL,
+                             rss_info,
+                             &rss, &mrq,
+                             &status_flags, &ip_id,
+                             &d->special);
+    d->errors = (uint8_t) (le32_to_cpu(status_flags) >> 24);
+    d->status = (uint8_t) le32_to_cpu(status_flags);
+}
+
+static inline void
+e1000e_write_ext_rx_descr(E1000ECore *core, uint8_t *desc,
+                          struct NetRxPkt *pkt,
+                          const E1000E_RSSInfo *rss_info,
+                          uint16_t length)
+{
+    union e1000_rx_desc_extended *d = (union e1000_rx_desc_extended *) desc;
+
+    memset(d, 0, sizeof(*d));
+
+    d->wb.upper.length = cpu_to_le16(length);
+
+    e1000e_build_rx_metadata(core, pkt, pkt != NULL,
+                             rss_info,
+                             &d->wb.lower.hi_dword.rss,
+                             &d->wb.lower.mrq,
+                             &d->wb.upper.status_error,
+                             &d->wb.lower.hi_dword.csum_ip.ip_id,
+                             &d->wb.upper.vlan);
+}
+
+static inline void
+e1000e_write_ps_rx_descr(E1000ECore *core, uint8_t *desc,
+                         struct NetRxPkt *pkt,
+                         const E1000E_RSSInfo *rss_info,
+                         size_t ps_hdr_len,
+                         uint16_t(*written)[MAX_PS_BUFFERS])
+{
+    int i;
+    union e1000_rx_desc_packet_split *d =
+        (union e1000_rx_desc_packet_split *) desc;
+
+    memset(d, 0, sizeof(*d));
+
+    d->wb.middle.length0 = cpu_to_le16((*written)[0]);
+
+    for (i = 0; i < PS_PAGE_BUFFERS; i++) {
+        d->wb.upper.length[i] = cpu_to_le16((*written)[i + 1]);
+    }
+
+    e1000e_build_rx_metadata(core, pkt, pkt != NULL,
+                             rss_info,
+                             &d->wb.lower.hi_dword.rss,
+                             &d->wb.lower.mrq,
+                             &d->wb.middle.status_error,
+                             &d->wb.lower.hi_dword.csum_ip.ip_id,
+                             &d->wb.middle.vlan);
+
+    d->wb.upper.header_status =
+        cpu_to_le16(ps_hdr_len | (ps_hdr_len ? E1000_RXDPS_HDRSTAT_HDRSP : 0));
+
+    trace_e1000e_rx_desc_ps_write((*written)[0], (*written)[1],
+                                  (*written)[2], (*written)[3]);
+}
+
+static inline void
+e1000e_write_rx_descr(E1000ECore *core, uint8_t *desc,
+struct NetRxPkt *pkt, const E1000E_RSSInfo *rss_info,
+    size_t ps_hdr_len, uint16_t(*written)[MAX_PS_BUFFERS])
+{
+    if (e1000e_rx_use_legacy_descriptor(core)) {
+        assert(ps_hdr_len == 0);
+        e1000e_write_lgcy_rx_descr(core, desc, pkt, rss_info, (*written)[0]);
+    } else {
+        if (core->mac[RCTL] & E1000_RCTL_DTYP_PS) {
+            e1000e_write_ps_rx_descr(core, desc, pkt, rss_info,
+                                      ps_hdr_len, written);
+        } else {
+            assert(ps_hdr_len == 0);
+            e1000e_write_ext_rx_descr(core, desc, pkt, rss_info,
+                                       (*written)[0]);
+        }
+    }
+}
+
+typedef struct e1000e_ba_state_st {
+    uint16_t written[MAX_PS_BUFFERS];
+    uint8_t cur_idx;
+} e1000e_ba_state;
+
+static inline void
+e1000e_write_hdr_to_rx_buffers(E1000ECore *core,
+                               hwaddr (*ba)[MAX_PS_BUFFERS],
+                               e1000e_ba_state *bastate,
+                               const char *data,
+                               dma_addr_t data_len)
+{
+    assert(data_len <= core->rxbuf_sizes[0] - bastate->written[0]);
+
+    pci_dma_write(core->owner, (*ba)[0] + bastate->written[0], data, data_len);
+    bastate->written[0] += data_len;
+
+    bastate->cur_idx = 1;
+}
+
+static void
+e1000e_write_to_rx_buffers(E1000ECore *core,
+                           hwaddr (*ba)[MAX_PS_BUFFERS],
+                           e1000e_ba_state *bastate,
+                           const char *data,
+                           dma_addr_t data_len)
+{
+    while (data_len > 0) {
+        uint32_t cur_buf_len = core->rxbuf_sizes[bastate->cur_idx];
+        uint32_t cur_buf_bytes_left = cur_buf_len -
+                                      bastate->written[bastate->cur_idx];
+        uint32_t bytes_to_write = MIN(data_len, cur_buf_bytes_left);
+
+        trace_e1000e_rx_desc_buff_write(bastate->cur_idx,
+                                        (*ba)[bastate->cur_idx],
+                                        bastate->written[bastate->cur_idx],
+                                        data,
+                                        bytes_to_write);
+
+        pci_dma_write(core->owner,
+            (*ba)[bastate->cur_idx] + bastate->written[bastate->cur_idx],
+            data, bytes_to_write);
+
+        bastate->written[bastate->cur_idx] += bytes_to_write;
+        data += bytes_to_write;
+        data_len -= bytes_to_write;
+
+        if (bastate->written[bastate->cur_idx] == cur_buf_len) {
+            bastate->cur_idx++;
+        }
+
+        assert(bastate->cur_idx < MAX_PS_BUFFERS);
+    }
+}
+
+static void
+e1000e_update_rx_stats(E1000ECore *core,
+                       size_t data_size,
+                       size_t data_fcs_size)
+{
+    e1000x_update_rx_total_stats(core->mac, data_size, data_fcs_size);
+
+    switch (net_rx_pkt_get_packet_type(core->rx_pkt)) {
+    case ETH_PKT_BCAST:
+        e1000x_inc_reg_if_not_full(core->mac, BPRC);
+        break;
+
+    case ETH_PKT_MCAST:
+        e1000x_inc_reg_if_not_full(core->mac, MPRC);
+        break;
+
+    default:
+        break;
+    }
+}
+
+static inline bool
+e1000e_rx_descr_threshold_hit(E1000ECore *core, const E1000E_RingInfo *rxi)
+{
+    return e1000e_ring_free_descr_num(core, rxi) ==
+           e1000e_ring_len(core, rxi) >> core->rxbuf_min_shift;
+}
+
+static bool
+e1000e_do_ps(E1000ECore *core, struct NetRxPkt *pkt, size_t *hdr_len)
+{
+    bool isip4, isip6, isudp, istcp;
+    bool fragment;
+
+    if (!e1000e_rx_use_ps_descriptor(core)) {
+        return false;
+    }
+
+    net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
+
+    if (isip4) {
+        fragment = net_rx_pkt_get_ip4_info(pkt)->fragment;
+    } else if (isip6) {
+        fragment = net_rx_pkt_get_ip6_info(pkt)->fragment;
+    } else {
+        return false;
+    }
+
+    if (fragment && (core->mac[RFCTL] & E1000_RFCTL_IPFRSP_DIS)) {
+        return false;
+    }
+
+    if (!fragment && (isudp || istcp)) {
+        *hdr_len = net_rx_pkt_get_l5_hdr_offset(pkt);
+    } else {
+        *hdr_len = net_rx_pkt_get_l4_hdr_offset(pkt);
+    }
+
+    if ((*hdr_len > core->rxbuf_sizes[0]) ||
+        (*hdr_len > net_rx_pkt_get_total_len(pkt))) {
+        return false;
+    }
+
+    return true;
+}
+
+static void
+e1000e_write_packet_to_guest(E1000ECore *core, struct NetRxPkt *pkt,
+                             const E1000E_RxRing *rxr,
+                             const E1000E_RSSInfo *rss_info)
+{
+    PCIDevice *d = core->owner;
+    dma_addr_t base;
+    uint8_t desc[E1000_MAX_RX_DESC_LEN];
+    size_t desc_size;
+    size_t desc_offset = 0;
+    size_t iov_ofs = 0;
+
+    struct iovec *iov = net_rx_pkt_get_iovec(pkt);
+    size_t size = net_rx_pkt_get_total_len(pkt);
+    size_t total_size = size + e1000x_fcs_len(core->mac);
+    const E1000E_RingInfo *rxi;
+    size_t ps_hdr_len = 0;
+    bool do_ps = e1000e_do_ps(core, pkt, &ps_hdr_len);
+
+    rxi = rxr->i;
+
+    do {
+        hwaddr ba[MAX_PS_BUFFERS];
+        e1000e_ba_state bastate = { { 0 } };
+        bool is_last = false;
+        bool is_first = true;
+
+        desc_size = total_size - desc_offset;
+
+        if (desc_size > core->rx_desc_buf_size) {
+            desc_size = core->rx_desc_buf_size;
+        }
+
+        base = e1000e_ring_head_descr(core, rxi);
+
+        pci_dma_read(d, base, &desc, core->rx_desc_len);
+
+        trace_e1000e_rx_descr(rxi->idx, base, core->rx_desc_len);
+
+        e1000e_read_rx_descr(core, desc, &ba);
+
+        if (ba[0]) {
+            if (desc_offset < size) {
+                static const uint32_t fcs_pad;
+                size_t iov_copy;
+                size_t copy_size = size - desc_offset;
+                if (copy_size > core->rx_desc_buf_size) {
+                    copy_size = core->rx_desc_buf_size;
+                }
+
+                /* For PS mode copy the packet header first */
+                if (do_ps) {
+                    if (is_first) {
+                        size_t ps_hdr_copied = 0;
+                        do {
+                            iov_copy = MIN(ps_hdr_len - ps_hdr_copied,
+                                           iov->iov_len - iov_ofs);
+
+                            e1000e_write_hdr_to_rx_buffers(core, &ba, &bastate,
+                                                      iov->iov_base, iov_copy);
+
+                            copy_size -= iov_copy;
+                            ps_hdr_copied += iov_copy;
+
+                            iov_ofs += iov_copy;
+                            if (iov_ofs == iov->iov_len) {
+                                iov++;
+                                iov_ofs = 0;
+                            }
+                        } while (ps_hdr_copied < ps_hdr_len);
+
+                        is_first = false;
+                    } else {
+                        /* Leave buffer 0 of each descriptor except first */
+                        /* empty as per spec 7.1.5.1                      */
+                        e1000e_write_hdr_to_rx_buffers(core, &ba, &bastate,
+                                                       NULL, 0);
+                    }
+                }
+
+                /* Copy packet payload */
+                while (copy_size) {
+                    iov_copy = MIN(copy_size, iov->iov_len - iov_ofs);
+
+                    e1000e_write_to_rx_buffers(core, &ba, &bastate,
+                                            iov->iov_base + iov_ofs, iov_copy);
+
+                    copy_size -= iov_copy;
+                    iov_ofs += iov_copy;
+                    if (iov_ofs == iov->iov_len) {
+                        iov++;
+                        iov_ofs = 0;
+                    }
+                }
+
+                if (desc_offset + desc_size >= total_size) {
+                    /* Simulate FCS checksum presence in the last descriptor */
+                    e1000e_write_to_rx_buffers(core, &ba, &bastate,
+                          (const char *) &fcs_pad, e1000x_fcs_len(core->mac));
+                }
+            }
+            desc_offset += desc_size;
+            if (desc_offset >= total_size) {
+                is_last = true;
+            }
+        } else { /* as per intel docs; skip descriptors with null buf addr */
+            trace_e1000e_rx_null_descriptor();
+        }
+
+        e1000e_write_rx_descr(core, desc, is_last ? core->rx_pkt : NULL,
+                           rss_info, do_ps ? ps_hdr_len : 0, &bastate.written);
+        pci_dma_write(d, base, &desc, core->rx_desc_len);
+
+        e1000e_ring_advance(core, rxi,
+                            core->rx_desc_len / E1000_MIN_RX_DESC_LEN);
+
+    } while (desc_offset < total_size);
+
+    e1000e_update_rx_stats(core, size, total_size);
+}
+
+static inline void
+e1000e_rx_fix_l4_csum(E1000ECore *core, struct NetRxPkt *pkt)
+{
+    if (net_rx_pkt_has_virt_hdr(pkt)) {
+        struct virtio_net_hdr *vhdr = net_rx_pkt_get_vhdr(pkt);
+
+        if (vhdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+            net_rx_pkt_fix_l4_csum(pkt);
+        }
+    }
+}
+
+ssize_t
+e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt)
+{
+    static const int maximum_ethernet_hdr_len = (14 + 4);
+    /* Min. octets in an ethernet frame sans FCS */
+    static const int min_buf_size = 60;
+
+    uint32_t n = 0;
+    uint8_t min_buf[min_buf_size];
+    struct iovec min_iov;
+    uint8_t *filter_buf;
+    size_t size, orig_size;
+    size_t iov_ofs = 0;
+    E1000E_RxRing rxr;
+    E1000E_RSSInfo rss_info;
+    size_t total_size;
+    ssize_t retval;
+    bool rdmts_hit;
+
+    trace_e1000e_rx_receive_iov(iovcnt);
+
+    if (!e1000x_hw_rx_enabled(core->mac)) {
+        return -1;
+    }
+
+    /* Pull virtio header in */
+    if (core->has_vnet) {
+        net_rx_pkt_set_vhdr_iovec(core->rx_pkt, iov, iovcnt);
+        iov_ofs = sizeof(struct virtio_net_hdr);
+    }
+
+    filter_buf = iov->iov_base + iov_ofs;
+    orig_size = iov_size(iov, iovcnt);
+    size = orig_size - iov_ofs;
+
+    /* Pad to minimum Ethernet frame length */
+    if (size < sizeof(min_buf)) {
+        iov_to_buf(iov, iovcnt, iov_ofs, min_buf, size);
+        memset(&min_buf[size], 0, sizeof(min_buf) - size);
+        e1000x_inc_reg_if_not_full(core->mac, RUC);
+        min_iov.iov_base = filter_buf = min_buf;
+        min_iov.iov_len = size = sizeof(min_buf);
+        iovcnt = 1;
+        iov = &min_iov;
+        iov_ofs = 0;
+    } else if (iov->iov_len < maximum_ethernet_hdr_len) {
+        /* This is very unlikely, but may happen. */
+        iov_to_buf(iov, iovcnt, iov_ofs, min_buf, maximum_ethernet_hdr_len);
+        filter_buf = min_buf;
+    }
+
+    /* Discard oversized packets if !LPE and !SBP. */
+    if (e1000x_is_oversized(core->mac, size)) {
+        return orig_size;
+    }
+
+    net_rx_pkt_set_packet_type(core->rx_pkt,
+        get_eth_packet_type(PKT_GET_ETH_HDR(filter_buf)));
+
+    if (!e1000e_receive_filter(core, filter_buf, size)) {
+        trace_e1000e_rx_flt_dropped();
+        return orig_size;
+    }
+
+    net_rx_pkt_attach_iovec_ex(core->rx_pkt, iov, iovcnt, iov_ofs,
+                               e1000x_vlan_enabled(core->mac), core->vet);
+
+    e1000e_rss_parse_packet(core, core->rx_pkt, &rss_info);
+    e1000e_rx_ring_init(core, &rxr, rss_info.queue);
+
+    trace_e1000e_rx_rss_dispatched_to_queue(rxr.i->idx);
+
+    total_size = net_rx_pkt_get_total_len(core->rx_pkt) +
+        e1000x_fcs_len(core->mac);
+
+    if (e1000e_has_rxbufs(core, rxr.i, total_size)) {
+        e1000e_rx_fix_l4_csum(core, core->rx_pkt);
+
+        e1000e_write_packet_to_guest(core, core->rx_pkt, &rxr, &rss_info);
+
+        retval = orig_size;
+
+        /* Perform small receive detection (RSRPD) */
+        if (total_size < core->mac[RSRPD]) {
+            n |= E1000_ICS_SRPD;
+        }
+
+        /* Perform ACK receive detection */
+        if (e1000e_is_tcp_ack(core, core->rx_pkt)) {
+            n |= E1000_ICS_ACK;
+        }
+
+        /* Check if receive descriptor minimum threshold hit */
+        rdmts_hit = e1000e_rx_descr_threshold_hit(core, rxr.i);
+        n |= e1000e_rx_wb_interrupt_cause(core, rxr.i->idx, rdmts_hit);
+
+        trace_e1000e_rx_written_to_guest(n);
+    } else {
+        n |= E1000_ICS_RXO;
+        retval = 0;
+
+        trace_e1000e_rx_not_written_to_guest(n);
+    }
+
+    if (!e1000e_intrmgr_delay_rx_causes(core, &n)) {
+        trace_e1000e_rx_interrupt_set(n);
+        e1000e_set_interrupt_cause(core, n);
+    } else {
+        trace_e1000e_rx_interrupt_delayed(n);
+    }
+
+    return retval;
+}
+
+static inline bool
+e1000e_have_autoneg(E1000ECore *core)
+{
+    return core->phy[0][PHY_CTRL] & MII_CR_AUTO_NEG_EN;
+}
+
+static void e1000e_update_flowctl_status(E1000ECore *core)
+{
+    if (e1000e_have_autoneg(core) &&
+        core->phy[0][PHY_STATUS] & MII_SR_AUTONEG_COMPLETE) {
+        trace_e1000e_link_autoneg_flowctl(true);
+        core->mac[CTRL] |= E1000_CTRL_TFCE | E1000_CTRL_RFCE;
+    } else {
+        trace_e1000e_link_autoneg_flowctl(false);
+    }
+}
+
+static inline void
+e1000e_link_down(E1000ECore *core)
+{
+    e1000x_update_regs_on_link_down(core->mac, core->phy[0]);
+    e1000e_update_flowctl_status(core);
+}
+
+static inline void
+e1000e_set_phy_ctrl(E1000ECore *core, int index, uint16_t val)
+{
+    /* bits 0-5 reserved; MII_CR_[RESTART_AUTO_NEG,RESET] are self clearing */
+    core->phy[0][PHY_CTRL] = val & ~(0x3f |
+                                     MII_CR_RESET |
+                                     MII_CR_RESTART_AUTO_NEG);
+
+    if ((val & MII_CR_RESTART_AUTO_NEG) &&
+        e1000e_have_autoneg(core)) {
+        e1000x_restart_autoneg(core->mac, core->phy[0], core->autoneg_timer);
+    }
+}
+
+static void
+e1000e_set_phy_oem_bits(E1000ECore *core, int index, uint16_t val)
+{
+    core->phy[0][PHY_OEM_BITS] = val & ~BIT(10);
+
+    if (val & BIT(10)) {
+        e1000x_restart_autoneg(core->mac, core->phy[0], core->autoneg_timer);
+    }
+}
+
+static void
+e1000e_set_phy_page(E1000ECore *core, int index, uint16_t val)
+{
+    core->phy[0][PHY_PAGE] = val & PHY_PAGE_RW_MASK;
+}
+
+void
+e1000e_core_set_link_status(E1000ECore *core)
+{
+    NetClientState *nc = qemu_get_queue(core->owner_nic);
+    uint32_t old_status = core->mac[STATUS];
+
+    trace_e1000e_link_status_changed(nc->link_down ? false : true);
+
+    if (nc->link_down) {
+        e1000x_update_regs_on_link_down(core->mac, core->phy[0]);
+    } else {
+        if (e1000e_have_autoneg(core) &&
+            !(core->phy[0][PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) {
+            e1000x_restart_autoneg(core->mac, core->phy[0],
+                                   core->autoneg_timer);
+        } else {
+            e1000x_update_regs_on_link_up(core->mac, core->phy[0]);
+        }
+    }
+
+    if (core->mac[STATUS] != old_status) {
+        e1000e_set_interrupt_cause(core, E1000_ICR_LSC);
+    }
+}
+
+static void
+e1000e_set_ctrl(E1000ECore *core, int index, uint32_t val)
+{
+    trace_e1000e_core_ctrl_write(index, val);
+
+    /* RST is self clearing */
+    core->mac[CTRL] = val & ~E1000_CTRL_RST;
+    core->mac[CTRL_DUP] = core->mac[CTRL];
+
+    trace_e1000e_link_set_params(
+        !!(val & E1000_CTRL_ASDE),
+        (val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT,
+        !!(val & E1000_CTRL_FRCSPD),
+        !!(val & E1000_CTRL_FRCDPX),
+        !!(val & E1000_CTRL_RFCE),
+        !!(val & E1000_CTRL_TFCE));
+
+    if (val & E1000_CTRL_RST) {
+        trace_e1000e_core_ctrl_sw_reset();
+        e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac);
+    }
+
+    if (val & E1000_CTRL_PHY_RST) {
+        trace_e1000e_core_ctrl_phy_reset();
+        core->mac[STATUS] |= E1000_STATUS_PHYRA;
+    }
+}
+
+static void
+e1000e_set_rfctl(E1000ECore *core, int index, uint32_t val)
+{
+    trace_e1000e_rx_set_rfctl(val);
+
+    if (!(val & E1000_RFCTL_ISCSI_DIS)) {
+        trace_e1000e_wrn_iscsi_filtering_not_supported();
+    }
+
+    if (!(val & E1000_RFCTL_NFSW_DIS)) {
+        trace_e1000e_wrn_nfsw_filtering_not_supported();
+    }
+
+    if (!(val & E1000_RFCTL_NFSR_DIS)) {
+        trace_e1000e_wrn_nfsr_filtering_not_supported();
+    }
+
+    core->mac[RFCTL] = val;
+}
+
+static void
+e1000e_calc_per_desc_buf_size(E1000ECore *core)
+{
+    int i;
+    core->rx_desc_buf_size = 0;
+
+    for (i = 0; i < ARRAY_SIZE(core->rxbuf_sizes); i++) {
+        core->rx_desc_buf_size += core->rxbuf_sizes[i];
+    }
+}
+
+static void
+e1000e_parse_rxbufsize(E1000ECore *core)
+{
+    uint32_t rctl = core->mac[RCTL];
+
+    memset(core->rxbuf_sizes, 0, sizeof(core->rxbuf_sizes));
+
+    if (rctl & E1000_RCTL_DTYP_MASK) {
+        uint32_t bsize;
+
+        bsize = core->mac[PSRCTL] & E1000_PSRCTL_BSIZE0_MASK;
+        core->rxbuf_sizes[0] = (bsize >> E1000_PSRCTL_BSIZE0_SHIFT) * 128;
+
+        bsize = core->mac[PSRCTL] & E1000_PSRCTL_BSIZE1_MASK;
+        core->rxbuf_sizes[1] = (bsize >> E1000_PSRCTL_BSIZE1_SHIFT) * 1024;
+
+        bsize = core->mac[PSRCTL] & E1000_PSRCTL_BSIZE2_MASK;
+        core->rxbuf_sizes[2] = (bsize >> E1000_PSRCTL_BSIZE2_SHIFT) * 1024;
+
+        bsize = core->mac[PSRCTL] & E1000_PSRCTL_BSIZE3_MASK;
+        core->rxbuf_sizes[3] = (bsize >> E1000_PSRCTL_BSIZE3_SHIFT) * 1024;
+    } else if (rctl & E1000_RCTL_FLXBUF_MASK) {
+        int flxbuf = rctl & E1000_RCTL_FLXBUF_MASK;
+        core->rxbuf_sizes[0] = (flxbuf >> E1000_RCTL_FLXBUF_SHIFT) * 1024;
+    } else {
+        core->rxbuf_sizes[0] = e1000x_rxbufsize(rctl);
+    }
+
+    trace_e1000e_rx_desc_buff_sizes(core->rxbuf_sizes[0], core->rxbuf_sizes[1],
+                                    core->rxbuf_sizes[2], core->rxbuf_sizes[3]);
+
+    e1000e_calc_per_desc_buf_size(core);
+}
+
+static void
+e1000e_calc_rxdesclen(E1000ECore *core)
+{
+    if (e1000e_rx_use_legacy_descriptor(core)) {
+        core->rx_desc_len = sizeof(struct e1000_rx_desc);
+    } else {
+        if (core->mac[RCTL] & E1000_RCTL_DTYP_PS) {
+            core->rx_desc_len = sizeof(union e1000_rx_desc_packet_split);
+        } else {
+            core->rx_desc_len = sizeof(union e1000_rx_desc_extended);
+        }
+    }
+    trace_e1000e_rx_desc_len(core->rx_desc_len);
+}
+
+static void
+e1000e_set_rx_control(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[RCTL] = val;
+    trace_e1000e_rx_set_rctl(core->mac[RCTL]);
+
+    if (val & E1000_RCTL_EN) {
+        e1000e_parse_rxbufsize(core);
+        e1000e_calc_rxdesclen(core);
+        core->rxbuf_min_shift = ((val / E1000_RCTL_RDMTS_QUAT) & 3) + 1 +
+                                E1000_RING_DESC_LEN_SHIFT;
+
+        e1000e_start_recv(core);
+    }
+}
+
+static
+void(*e1000e_phyreg_writeops[E1000E_PHY_PAGES][E1000E_PHY_PAGE_SIZE])
+(E1000ECore *, int, uint16_t) = {
+    [0] = {
+        [PHY_CTRL]     = e1000e_set_phy_ctrl,
+        [PHY_PAGE]     = e1000e_set_phy_page,
+        [PHY_OEM_BITS] = e1000e_set_phy_oem_bits
+    }
+};
+
+static inline void
+e1000e_clear_ims_bits(E1000ECore *core, uint32_t bits)
+{
+    trace_e1000e_irq_clear_ims(bits, core->mac[IMS], core->mac[IMS] & ~bits);
+    core->mac[IMS] &= ~bits;
+}
+
+static inline bool
+e1000e_postpone_interrupt(bool *interrupt_pending,
+                           E1000IntrDelayTimer *timer)
+{
+    if (timer->running) {
+        trace_e1000e_irq_postponed_by_xitr(timer->delay_reg << 2);
+
+        *interrupt_pending = true;
+        return true;
+    }
+
+    if (timer->core->mac[timer->delay_reg] != 0) {
+        e1000e_intrmgr_rearm_timer(timer);
+    }
+
+    return false;
+}
+
+static inline bool
+e1000e_itr_should_postpone(E1000ECore *core)
+{
+    return e1000e_postpone_interrupt(&core->itr_intr_pending, &core->itr);
+}
+
+static inline bool
+e1000e_eitr_should_postpone(E1000ECore *core, int idx)
+{
+    return e1000e_postpone_interrupt(&core->eitr_intr_pending[idx],
+                                     &core->eitr[idx]);
+}
+
+static void
+e1000e_msix_notify_one(E1000ECore *core, uint32_t cause, uint32_t int_cfg)
+{
+    uint32_t effective_eiac;
+
+    if (E1000_IVAR_ENTRY_VALID(int_cfg)) {
+        uint32_t vec = E1000_IVAR_ENTRY_VEC(int_cfg);
+        if (vec < E1000E_MSIX_VEC_NUM) {
+            if (!e1000e_eitr_should_postpone(core, vec)) {
+                trace_e1000e_irq_msix_notify_vec(vec);
+                msix_notify(core->owner, vec);
+            }
+        } else {
+            trace_e1000e_wrn_msix_vec_wrong(cause, int_cfg);
+        }
+    } else {
+        trace_e1000e_wrn_msix_invalid(cause, int_cfg);
+    }
+
+    if (core->mac[CTRL_EXT] & E1000_CTRL_EXT_EIAME) {
+        trace_e1000e_irq_ims_clear_eiame(core->mac[IAM], cause);
+        e1000e_clear_ims_bits(core, core->mac[IAM] & cause);
+    }
+
+    trace_e1000e_irq_icr_clear_eiac(core->mac[ICR], core->mac[EIAC]);
+
+    if (core->mac[EIAC] & E1000_ICR_OTHER) {
+        effective_eiac = (core->mac[EIAC] & E1000_EIAC_MASK) |
+                         E1000_ICR_OTHER_CAUSES;
+    } else {
+        effective_eiac = core->mac[EIAC] & E1000_EIAC_MASK;
+    }
+    core->mac[ICR] &= ~effective_eiac;
+}
+
+static void
+e1000e_msix_notify(E1000ECore *core, uint32_t causes)
+{
+    if (causes & E1000_ICR_RXQ0) {
+        e1000e_msix_notify_one(core, E1000_ICR_RXQ0,
+                               E1000_IVAR_RXQ0(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_RXQ1) {
+        e1000e_msix_notify_one(core, E1000_ICR_RXQ1,
+                               E1000_IVAR_RXQ1(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_TXQ0) {
+        e1000e_msix_notify_one(core, E1000_ICR_TXQ0,
+                               E1000_IVAR_TXQ0(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_TXQ1) {
+        e1000e_msix_notify_one(core, E1000_ICR_TXQ1,
+                               E1000_IVAR_TXQ1(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_OTHER) {
+        e1000e_msix_notify_one(core, E1000_ICR_OTHER,
+                               E1000_IVAR_OTHER(core->mac[IVAR]));
+    }
+}
+
+static void
+e1000e_msix_clear_one(E1000ECore *core, uint32_t cause, uint32_t int_cfg)
+{
+    if (E1000_IVAR_ENTRY_VALID(int_cfg)) {
+        uint32_t vec = E1000_IVAR_ENTRY_VEC(int_cfg);
+        if (vec < E1000E_MSIX_VEC_NUM) {
+            trace_e1000e_irq_msix_pending_clearing(cause, int_cfg, vec);
+            msix_clr_pending(core->owner, vec);
+        } else {
+            trace_e1000e_wrn_msix_vec_wrong(cause, int_cfg);
+        }
+    } else {
+        trace_e1000e_wrn_msix_invalid(cause, int_cfg);
+    }
+}
+
+static void
+e1000e_msix_clear(E1000ECore *core, uint32_t causes)
+{
+    if (causes & E1000_ICR_RXQ0) {
+        e1000e_msix_clear_one(core, E1000_ICR_RXQ0,
+                              E1000_IVAR_RXQ0(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_RXQ1) {
+        e1000e_msix_clear_one(core, E1000_ICR_RXQ1,
+                              E1000_IVAR_RXQ1(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_TXQ0) {
+        e1000e_msix_clear_one(core, E1000_ICR_TXQ0,
+                              E1000_IVAR_TXQ0(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_TXQ1) {
+        e1000e_msix_clear_one(core, E1000_ICR_TXQ1,
+                              E1000_IVAR_TXQ1(core->mac[IVAR]));
+    }
+
+    if (causes & E1000_ICR_OTHER) {
+        e1000e_msix_clear_one(core, E1000_ICR_OTHER,
+                              E1000_IVAR_OTHER(core->mac[IVAR]));
+    }
+}
+
+static inline void
+e1000e_fix_icr_asserted(E1000ECore *core)
+{
+    core->mac[ICR] &= ~E1000_ICR_ASSERTED;
+    if (core->mac[ICR]) {
+        core->mac[ICR] |= E1000_ICR_ASSERTED;
+    }
+
+    trace_e1000e_irq_fix_icr_asserted(core->mac[ICR]);
+}
+
+static void
+e1000e_send_msi(E1000ECore *core, bool msix)
+{
+    uint32_t causes = core->mac[ICR] & core->mac[IMS] & ~E1000_ICR_ASSERTED;
+
+    if (msix) {
+        e1000e_msix_notify(core, causes);
+    } else {
+        if (!e1000e_itr_should_postpone(core)) {
+            trace_e1000e_irq_msi_notify(causes);
+            msi_notify(core->owner, 0);
+        }
+    }
+}
+
+static void
+e1000e_update_interrupt_state(E1000ECore *core)
+{
+    bool interrupts_pending;
+    bool is_msix = msix_enabled(core->owner);
+
+    /* Set ICR[OTHER] for MSI-X */
+    if (is_msix) {
+        if (core->mac[ICR] & core->mac[IMS] & E1000_ICR_OTHER_CAUSES) {
+            core->mac[ICR] |= E1000_ICR_OTHER;
+            trace_e1000e_irq_add_msi_other(core->mac[ICR]);
+        }
+    }
+
+    e1000e_fix_icr_asserted(core);
+
+    /*
+     * Make sure ICR and ICS registers have the same value.
+     * The spec says that the ICS register is write-only.  However in practice,
+     * on real hardware ICS is readable, and for reads it has the same value as
+     * ICR (except that ICS does not have the clear on read behaviour of ICR).
+     *
+     * The VxWorks PRO/1000 driver uses this behaviour.
+     */
+    core->mac[ICS] = core->mac[ICR];
+
+    interrupts_pending = (core->mac[IMS] & core->mac[ICR]) ? true : false;
+
+    trace_e1000e_irq_pending_interrupts(core->mac[ICR] & core->mac[IMS],
+                                        core->mac[ICR], core->mac[IMS]);
+
+    if (is_msix || msi_enabled(core->owner)) {
+        if (interrupts_pending) {
+            e1000e_send_msi(core, is_msix);
+        }
+    } else {
+        if (interrupts_pending) {
+            if (!e1000e_itr_should_postpone(core)) {
+                e1000e_raise_legacy_irq(core);
+            }
+        } else {
+            e1000e_lower_legacy_irq(core);
+        }
+    }
+}
+
+static inline void
+e1000e_set_interrupt_cause(E1000ECore *core, uint32_t val)
+{
+    trace_e1000e_irq_set_cause_entry(val, core->mac[ICR]);
+
+    val |= e1000e_intmgr_collect_delayed_causes(core);
+    core->mac[ICR] |= val;
+
+    trace_e1000e_irq_set_cause_exit(val, core->mac[ICR]);
+
+    e1000e_update_interrupt_state(core);
+}
+
+static inline void
+e1000e_autoneg_timer(void *opaque)
+{
+    E1000ECore *core = opaque;
+    if (!qemu_get_queue(core->owner_nic)->link_down) {
+        e1000x_update_regs_on_autoneg_done(core->mac, core->phy[0]);
+        e1000e_update_flowctl_status(core);
+        /* signal link status change to the guest */
+        e1000e_set_interrupt_cause(core, E1000_ICR_LSC);
+    }
+}
+
+static inline uint16_t
+e1000e_get_reg_index_with_offset(const uint16_t *mac_reg_access, hwaddr addr)
+{
+    uint16_t index = (addr & 0x1ffff) >> 2;
+    return index + (mac_reg_access[index] & 0xfffe);
+}
+
+static const char e1000e_phy_regcap[E1000E_PHY_PAGES][0x20] = {
+    [0] = {
+        [PHY_CTRL]          = PHY_ANYPAGE | PHY_RW,
+        [PHY_STATUS]        = PHY_ANYPAGE | PHY_R,
+        [PHY_ID1]           = PHY_ANYPAGE | PHY_R,
+        [PHY_ID2]           = PHY_ANYPAGE | PHY_R,
+        [PHY_AUTONEG_ADV]   = PHY_ANYPAGE | PHY_RW,
+        [PHY_LP_ABILITY]    = PHY_ANYPAGE | PHY_R,
+        [PHY_AUTONEG_EXP]   = PHY_ANYPAGE | PHY_R,
+        [PHY_NEXT_PAGE_TX]  = PHY_ANYPAGE | PHY_RW,
+        [PHY_LP_NEXT_PAGE]  = PHY_ANYPAGE | PHY_R,
+        [PHY_1000T_CTRL]    = PHY_ANYPAGE | PHY_RW,
+        [PHY_1000T_STATUS]  = PHY_ANYPAGE | PHY_R,
+        [PHY_EXT_STATUS]    = PHY_ANYPAGE | PHY_R,
+        [PHY_PAGE]          = PHY_ANYPAGE | PHY_RW,
+
+        [PHY_COPPER_CTRL1]      = PHY_RW,
+        [PHY_COPPER_STAT1]      = PHY_R,
+        [PHY_COPPER_CTRL3]      = PHY_RW,
+        [PHY_RX_ERR_CNTR]       = PHY_R,
+        [PHY_OEM_BITS]          = PHY_RW,
+        [PHY_BIAS_1]            = PHY_RW,
+        [PHY_BIAS_2]            = PHY_RW,
+        [PHY_COPPER_INT_ENABLE] = PHY_RW,
+        [PHY_COPPER_STAT2]      = PHY_R,
+        [PHY_COPPER_CTRL2]      = PHY_RW
+    },
+    [2] = {
+        [PHY_MAC_CTRL1]         = PHY_RW,
+        [PHY_MAC_INT_ENABLE]    = PHY_RW,
+        [PHY_MAC_STAT]          = PHY_R,
+        [PHY_MAC_CTRL2]         = PHY_RW
+    },
+    [3] = {
+        [PHY_LED_03_FUNC_CTRL1] = PHY_RW,
+        [PHY_LED_03_POL_CTRL]   = PHY_RW,
+        [PHY_LED_TIMER_CTRL]    = PHY_RW,
+        [PHY_LED_45_CTRL]       = PHY_RW
+    },
+    [5] = {
+        [PHY_1000T_SKEW]        = PHY_R,
+        [PHY_1000T_SWAP]        = PHY_R
+    },
+    [6] = {
+        [PHY_CRC_COUNTERS]      = PHY_R
+    }
+};
+
+static bool
+e1000e_phy_reg_check_cap(E1000ECore *core, uint32_t addr,
+                         char cap, uint8_t *page)
+{
+    *page =
+        (e1000e_phy_regcap[0][addr] & PHY_ANYPAGE) ? 0
+                                                    : core->phy[0][PHY_PAGE];
+
+    if (*page >= E1000E_PHY_PAGES) {
+        return false;
+    }
+
+    return e1000e_phy_regcap[*page][addr] & cap;
+}
+
+static void
+e1000e_phy_reg_write(E1000ECore *core, uint8_t page,
+                     uint32_t addr, uint16_t data)
+{
+    assert(page < E1000E_PHY_PAGES);
+    assert(addr < E1000E_PHY_PAGE_SIZE);
+
+    if (e1000e_phyreg_writeops[page][addr]) {
+        e1000e_phyreg_writeops[page][addr](core, addr, data);
+    } else {
+        core->phy[page][addr] = data;
+    }
+}
+
+static void
+e1000e_set_mdic(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t data = val & E1000_MDIC_DATA_MASK;
+    uint32_t addr = ((val & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT);
+    uint8_t page;
+
+    if ((val & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT != 1) { /* phy # */
+        val = core->mac[MDIC] | E1000_MDIC_ERROR;
+    } else if (val & E1000_MDIC_OP_READ) {
+        if (!e1000e_phy_reg_check_cap(core, addr, PHY_R, &page)) {
+            trace_e1000e_core_mdic_read_unhandled(page, addr);
+            val |= E1000_MDIC_ERROR;
+        } else {
+            val = (val ^ data) | core->phy[page][addr];
+            trace_e1000e_core_mdic_read(page, addr, val);
+        }
+    } else if (val & E1000_MDIC_OP_WRITE) {
+        if (!e1000e_phy_reg_check_cap(core, addr, PHY_W, &page)) {
+            trace_e1000e_core_mdic_write_unhandled(page, addr);
+            val |= E1000_MDIC_ERROR;
+        } else {
+            trace_e1000e_core_mdic_write(page, addr, data);
+            e1000e_phy_reg_write(core, page, addr, data);
+        }
+    }
+    core->mac[MDIC] = val | E1000_MDIC_READY;
+
+    if (val & E1000_MDIC_INT_EN) {
+        e1000e_set_interrupt_cause(core, E1000_ICR_MDAC);
+    }
+}
+
+static void
+e1000e_set_rdt(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[index] = val & 0xffff;
+    trace_e1000e_rx_set_rdt(e1000e_mq_queue_idx(RDT0, index), val);
+    e1000e_start_recv(core);
+}
+
+static void
+e1000e_set_status(E1000ECore *core, int index, uint32_t val)
+{
+    if ((val & E1000_STATUS_PHYRA) == 0) {
+        core->mac[index] &= ~E1000_STATUS_PHYRA;
+    }
+}
+
+static void
+e1000e_set_ctrlext(E1000ECore *core, int index, uint32_t val)
+{
+    trace_e1000e_link_set_ext_params(!!(val & E1000_CTRL_EXT_ASDCHK),
+                                     !!(val & E1000_CTRL_EXT_SPD_BYPS));
+
+    /* Zero self-clearing bits */
+    val &= ~(E1000_CTRL_EXT_ASDCHK | E1000_CTRL_EXT_EE_RST);
+    core->mac[CTRL_EXT] = val;
+}
+
+static void
+e1000e_set_pbaclr(E1000ECore *core, int index, uint32_t val)
+{
+    int i;
+
+    core->mac[PBACLR] = val & E1000_PBACLR_VALID_MASK;
+
+    if (msix_enabled(core->owner)) {
+        return;
+    }
+
+    for (i = 0; i < E1000E_MSIX_VEC_NUM; i++) {
+        if (core->mac[PBACLR] & BIT(i)) {
+            msix_clr_pending(core->owner, i);
+        }
+    }
+}
+
+static void
+e1000e_set_fcrth(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[FCRTH] = val & 0xFFF8;
+}
+
+static void
+e1000e_set_fcrtl(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[FCRTL] = val & 0x8000FFF8;
+}
+
+static inline void
+e1000e_set_16bit(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[index] = val & 0xffff;
+}
+
+static void
+e1000e_set_12bit(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[index] = val & 0xfff;
+}
+
+static void
+e1000e_set_vet(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[VET] = val & 0xffff;
+    core->vet = le16_to_cpu(core->mac[VET]);
+    trace_e1000e_vlan_vet(core->vet);
+}
+
+static void
+e1000e_set_dlen(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[index] = val & E1000_XDLEN_MASK;
+}
+
+static void
+e1000e_set_dbal(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[index] = val & E1000_XDBAL_MASK;
+}
+
+static void
+e1000e_set_tctl(E1000ECore *core, int index, uint32_t val)
+{
+    E1000E_TxRing txr;
+    core->mac[index] = val;
+
+    if (core->mac[TARC0] & E1000_TARC_ENABLE) {
+        e1000e_tx_ring_init(core, &txr, 0);
+        e1000e_start_xmit(core, &txr);
+    }
+
+    if (core->mac[TARC1] & E1000_TARC_ENABLE) {
+        e1000e_tx_ring_init(core, &txr, 1);
+        e1000e_start_xmit(core, &txr);
+    }
+}
+
+static void
+e1000e_set_tdt(E1000ECore *core, int index, uint32_t val)
+{
+    E1000E_TxRing txr;
+    int qidx = e1000e_mq_queue_idx(TDT, index);
+    uint32_t tarc_reg = (qidx == 0) ? TARC0 : TARC1;
+
+    core->mac[index] = val & 0xffff;
+
+    if (core->mac[tarc_reg] & E1000_TARC_ENABLE) {
+        e1000e_tx_ring_init(core, &txr, qidx);
+        e1000e_start_xmit(core, &txr);
+    }
+}
+
+static void
+e1000e_set_ics(E1000ECore *core, int index, uint32_t val)
+{
+    trace_e1000e_irq_write_ics(val);
+    e1000e_set_interrupt_cause(core, val);
+}
+
+static void
+e1000e_set_icr(E1000ECore *core, int index, uint32_t val)
+{
+    if ((core->mac[ICR] & E1000_ICR_ASSERTED) &&
+        (core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) {
+        trace_e1000e_irq_icr_process_iame();
+        e1000e_clear_ims_bits(core, core->mac[IAM]);
+    }
+
+    trace_e1000e_irq_icr_write(val, core->mac[ICR], core->mac[ICR] & ~val);
+    core->mac[ICR] &= ~val;
+    e1000e_update_interrupt_state(core);
+}
+
+static void
+e1000e_set_imc(E1000ECore *core, int index, uint32_t val)
+{
+    trace_e1000e_irq_ims_clear_set_imc(val);
+    e1000e_clear_ims_bits(core, val);
+    e1000e_update_interrupt_state(core);
+}
+
+static void
+e1000e_set_ims(E1000ECore *core, int index, uint32_t val)
+{
+    static const uint32_t ims_ext_mask =
+        E1000_IMS_RXQ0 | E1000_IMS_RXQ1 |
+        E1000_IMS_TXQ0 | E1000_IMS_TXQ1 |
+        E1000_IMS_OTHER;
+
+    static const uint32_t ims_valid_mask =
+        E1000_IMS_TXDW      | E1000_IMS_TXQE    | E1000_IMS_LSC  |
+        E1000_IMS_RXDMT0    | E1000_IMS_RXO     | E1000_IMS_RXT0 |
+        E1000_IMS_MDAC      | E1000_IMS_TXD_LOW | E1000_IMS_SRPD |
+        E1000_IMS_ACK       | E1000_IMS_MNG     | E1000_IMS_RXQ0 |
+        E1000_IMS_RXQ1      | E1000_IMS_TXQ0    | E1000_IMS_TXQ1 |
+        E1000_IMS_OTHER;
+
+    uint32_t valid_val = val & ims_valid_mask;
+
+    trace_e1000e_irq_set_ims(val, core->mac[IMS], core->mac[IMS] | valid_val);
+    core->mac[IMS] |= valid_val;
+
+    if ((valid_val & ims_ext_mask) &&
+        (core->mac[CTRL_EXT] & E1000_CTRL_EXT_PBA_CLR) &&
+        msix_enabled(core->owner)) {
+        e1000e_msix_clear(core, valid_val);
+    }
+
+    if ((valid_val == ims_valid_mask) &&
+        (core->mac[CTRL_EXT] & E1000_CTRL_EXT_INT_TIMERS_CLEAR_ENA)) {
+        trace_e1000e_irq_fire_all_timers(val);
+        e1000e_intrmgr_fire_all_timers(core);
+    }
+
+    e1000e_update_interrupt_state(core);
+}
+
+static void
+e1000e_set_rdtr(E1000ECore *core, int index, uint32_t val)
+{
+    e1000e_set_16bit(core, index, val);
+
+    if ((val & E1000_RDTR_FPD) && (core->rdtr.running)) {
+        trace_e1000e_irq_rdtr_fpd_running();
+        e1000e_intrmgr_fire_delayed_interrupts(core);
+    } else {
+        trace_e1000e_irq_rdtr_fpd_not_running();
+    }
+}
+
+static void
+e1000e_set_tidv(E1000ECore *core, int index, uint32_t val)
+{
+    e1000e_set_16bit(core, index, val);
+
+    if ((val & E1000_TIDV_FPD) && (core->tidv.running)) {
+        trace_e1000e_irq_tidv_fpd_running();
+        e1000e_intrmgr_fire_delayed_interrupts(core);
+    } else {
+        trace_e1000e_irq_tidv_fpd_not_running();
+    }
+}
+
+static uint32_t
+e1000e_mac_readreg(E1000ECore *core, int index)
+{
+    return core->mac[index];
+}
+
+static uint32_t
+e1000e_mac_ics_read(E1000ECore *core, int index)
+{
+    trace_e1000e_irq_read_ics(core->mac[ICS]);
+    return core->mac[ICS];
+}
+
+static uint32_t
+e1000e_mac_ims_read(E1000ECore *core, int index)
+{
+    trace_e1000e_irq_read_ims(core->mac[IMS]);
+    return core->mac[IMS];
+}
+
+#define E1000E_LOW_BITS_READ_FUNC(num)                      \
+    static uint32_t                                         \
+    e1000e_mac_low##num##_read(E1000ECore *core, int index) \
+    {                                                       \
+        return core->mac[index] & (BIT(num) - 1);           \
+    }                                                       \
+
+#define E1000E_LOW_BITS_READ(num)                           \
+    e1000e_mac_low##num##_read
+
+E1000E_LOW_BITS_READ_FUNC(4);
+E1000E_LOW_BITS_READ_FUNC(6);
+E1000E_LOW_BITS_READ_FUNC(11);
+E1000E_LOW_BITS_READ_FUNC(13);
+E1000E_LOW_BITS_READ_FUNC(16);
+
+static uint32_t
+e1000e_mac_swsm_read(E1000ECore *core, int index)
+{
+    uint32_t val = core->mac[SWSM];
+    core->mac[SWSM] = val | 1;
+    return val;
+}
+
+static uint32_t
+e1000e_mac_itr_read(E1000ECore *core, int index)
+{
+    return core->itr_guest_value;
+}
+
+static uint32_t
+e1000e_mac_eitr_read(E1000ECore *core, int index)
+{
+    return core->eitr_guest_value[index - EITR];
+}
+
+static uint32_t
+e1000e_mac_icr_read(E1000ECore *core, int index)
+{
+    uint32_t ret = core->mac[ICR];
+    trace_e1000e_irq_icr_read_entry(ret);
+
+    if (core->mac[IMS] == 0) {
+        trace_e1000e_irq_icr_clear_zero_ims();
+        core->mac[ICR] = 0;
+    }
+
+    if ((core->mac[ICR] & E1000_ICR_ASSERTED) &&
+        (core->mac[CTRL_EXT] & E1000_CTRL_EXT_IAME)) {
+        trace_e1000e_irq_icr_clear_iame();
+        core->mac[ICR] = 0;
+        trace_e1000e_irq_icr_process_iame();
+        e1000e_clear_ims_bits(core, core->mac[IAM]);
+    }
+
+    trace_e1000e_irq_icr_read_exit(core->mac[ICR]);
+    e1000e_update_interrupt_state(core);
+    return ret;
+}
+
+static uint32_t
+e1000e_mac_read_clr4(E1000ECore *core, int index)
+{
+    uint32_t ret = core->mac[index];
+
+    core->mac[index] = 0;
+    return ret;
+}
+
+static uint32_t
+e1000e_mac_read_clr8(E1000ECore *core, int index)
+{
+    uint32_t ret = core->mac[index];
+
+    core->mac[index] = 0;
+    core->mac[index - 1] = 0;
+    return ret;
+}
+
+static uint32_t
+e1000e_get_ctrl(E1000ECore *core, int index)
+{
+    uint32_t val = core->mac[CTRL];
+
+    trace_e1000e_link_read_params(
+        !!(val & E1000_CTRL_ASDE),
+        (val & E1000_CTRL_SPD_SEL) >> E1000_CTRL_SPD_SHIFT,
+        !!(val & E1000_CTRL_FRCSPD),
+        !!(val & E1000_CTRL_FRCDPX),
+        !!(val & E1000_CTRL_RFCE),
+        !!(val & E1000_CTRL_TFCE));
+
+    return val;
+}
+
+static uint32_t
+e1000e_get_status(E1000ECore *core, int index)
+{
+    uint32_t res = core->mac[STATUS];
+
+    if (!(core->mac[CTRL] & E1000_CTRL_GIO_MASTER_DISABLE)) {
+        res |= E1000_STATUS_GIO_MASTER_ENABLE;
+    }
+
+    if (core->mac[CTRL] & E1000_CTRL_FRCDPX) {
+        res |= (core->mac[CTRL] & E1000_CTRL_FD) ? E1000_STATUS_FD : 0;
+    } else {
+        res |= E1000_STATUS_FD;
+    }
+
+    if ((core->mac[CTRL] & E1000_CTRL_FRCSPD) ||
+        (core->mac[CTRL_EXT] & E1000_CTRL_EXT_SPD_BYPS)) {
+        switch (core->mac[CTRL] & E1000_CTRL_SPD_SEL) {
+        case E1000_CTRL_SPD_10:
+            res |= E1000_STATUS_SPEED_10;
+            break;
+        case E1000_CTRL_SPD_100:
+            res |= E1000_STATUS_SPEED_100;
+            break;
+        case E1000_CTRL_SPD_1000:
+        default:
+            res |= E1000_STATUS_SPEED_1000;
+            break;
+        }
+    } else {
+        res |= E1000_STATUS_SPEED_1000;
+    }
+
+    trace_e1000e_link_status(
+        !!(res & E1000_STATUS_LU),
+        !!(res & E1000_STATUS_FD),
+        (res & E1000_STATUS_SPEED_MASK) >> E1000_STATUS_SPEED_SHIFT,
+        (res & E1000_STATUS_ASDV) >> E1000_STATUS_ASDV_SHIFT);
+
+    return res;
+}
+
+static uint32_t
+e1000e_get_tarc(E1000ECore *core, int index)
+{
+    return core->mac[index] & ((BIT(11) - 1) |
+                                BIT(27)      |
+                                BIT(28)      |
+                                BIT(29)      |
+                                BIT(30));
+}
+
+static void
+e1000e_mac_writereg(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[index] = val;
+}
+
+static void
+e1000e_mac_setmacaddr(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t macaddr[2];
+
+    core->mac[index] = val;
+
+    macaddr[0] = cpu_to_le32(core->mac[RA]);
+    macaddr[1] = cpu_to_le32(core->mac[RA + 1]);
+    qemu_format_nic_info_str(qemu_get_queue(core->owner_nic),
+        (uint8_t *) macaddr);
+
+    trace_e1000e_mac_set_sw(MAC_ARG(macaddr));
+}
+
+static void
+e1000e_set_eecd(E1000ECore *core, int index, uint32_t val)
+{
+    static const uint32_t ro_bits = E1000_EECD_PRES          |
+                                    E1000_EECD_AUTO_RD       |
+                                    E1000_EECD_SIZE_EX_MASK;
+
+    core->mac[EECD] = (core->mac[EECD] & ro_bits) | (val & ~ro_bits);
+}
+
+static void
+e1000e_set_eerd(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t addr = (val >> E1000_EERW_ADDR_SHIFT) & E1000_EERW_ADDR_MASK;
+    uint32_t flags = 0;
+    uint32_t data = 0;
+
+    if ((addr < E1000E_EEPROM_SIZE) && (val & E1000_EERW_START)) {
+        data = core->eeprom[addr];
+        flags = E1000_EERW_DONE;
+    }
+
+    core->mac[EERD] = flags                           |
+                      (addr << E1000_EERW_ADDR_SHIFT) |
+                      (data << E1000_EERW_DATA_SHIFT);
+}
+
+static void
+e1000e_set_eewr(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t addr = (val >> E1000_EERW_ADDR_SHIFT) & E1000_EERW_ADDR_MASK;
+    uint32_t data = (val >> E1000_EERW_DATA_SHIFT) & E1000_EERW_DATA_MASK;
+    uint32_t flags = 0;
+
+    if ((addr < E1000E_EEPROM_SIZE) && (val & E1000_EERW_START)) {
+        core->eeprom[addr] = data;
+        flags = E1000_EERW_DONE;
+    }
+
+    core->mac[EERD] = flags                           |
+                      (addr << E1000_EERW_ADDR_SHIFT) |
+                      (data << E1000_EERW_DATA_SHIFT);
+}
+
+static void
+e1000e_set_rxdctl(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[RXDCTL] = core->mac[RXDCTL1] = val;
+}
+
+static void
+e1000e_set_itr(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t interval = val & 0xffff;
+
+    trace_e1000e_irq_itr_set(val);
+
+    core->itr_guest_value = interval;
+    core->mac[index] = MAX(interval, E1000E_MIN_XITR);
+}
+
+static void
+e1000e_set_eitr(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t interval = val & 0xffff;
+    uint32_t eitr_num = index - EITR;
+
+    trace_e1000e_irq_eitr_set(eitr_num, val);
+
+    core->eitr_guest_value[eitr_num] = interval;
+    core->mac[index] = MAX(interval, E1000E_MIN_XITR);
+}
+
+static void
+e1000e_set_psrctl(E1000ECore *core, int index, uint32_t val)
+{
+    if ((val & E1000_PSRCTL_BSIZE0_MASK) == 0) {
+        hw_error("e1000e: PSRCTL.BSIZE0 cannot be zero");
+    }
+
+    if ((val & E1000_PSRCTL_BSIZE1_MASK) == 0) {
+        hw_error("e1000e: PSRCTL.BSIZE1 cannot be zero");
+    }
+
+    core->mac[PSRCTL] = val;
+}
+
+static void
+e1000e_update_rx_offloads(E1000ECore *core)
+{
+    int cso_state = e1000e_rx_l4_cso_enabled(core);
+
+    trace_e1000e_rx_set_cso(cso_state);
+
+    if (core->has_vnet) {
+        qemu_set_offload(qemu_get_queue(core->owner_nic)->peer,
+                         cso_state, 0, 0, 0, 0);
+    }
+}
+
+static void
+e1000e_set_rxcsum(E1000ECore *core, int index, uint32_t val)
+{
+    core->mac[RXCSUM] = val;
+    e1000e_update_rx_offloads(core);
+}
+
+static void
+e1000e_set_gcr(E1000ECore *core, int index, uint32_t val)
+{
+    uint32_t ro_bits = core->mac[GCR] & E1000_GCR_RO_BITS;
+    core->mac[GCR] = (val & ~E1000_GCR_RO_BITS) | ro_bits;
+}
+
+#define e1000e_getreg(x)    [x] = e1000e_mac_readreg
+static uint32_t (*e1000e_macreg_readops[])(E1000ECore *, int) = {
+    e1000e_getreg(PBA),
+    e1000e_getreg(WUFC),
+    e1000e_getreg(MANC),
+    e1000e_getreg(TOTL),
+    e1000e_getreg(RDT0),
+    e1000e_getreg(RDBAH0),
+    e1000e_getreg(TDBAL1),
+    e1000e_getreg(RDLEN0),
+    e1000e_getreg(RDH1),
+    e1000e_getreg(LATECOL),
+    e1000e_getreg(SEC),
+    e1000e_getreg(XONTXC),
+    e1000e_getreg(WUS),
+    e1000e_getreg(GORCL),
+    e1000e_getreg(MGTPRC),
+    e1000e_getreg(EERD),
+    e1000e_getreg(EIAC),
+    e1000e_getreg(PSRCTL),
+    e1000e_getreg(MANC2H),
+    e1000e_getreg(RXCSUM),
+    e1000e_getreg(GSCL_3),
+    e1000e_getreg(GSCN_2),
+    e1000e_getreg(RSRPD),
+    e1000e_getreg(RDBAL1),
+    e1000e_getreg(FCAH),
+    e1000e_getreg(FCRTH),
+    e1000e_getreg(FLOP),
+    e1000e_getreg(FLASHT),
+    e1000e_getreg(RXSTMPH),
+    e1000e_getreg(TXSTMPL),
+    e1000e_getreg(TIMADJL),
+    e1000e_getreg(TXDCTL),
+    e1000e_getreg(RDH0),
+    e1000e_getreg(TDT1),
+    e1000e_getreg(TNCRS),
+    e1000e_getreg(RJC),
+    e1000e_getreg(IAM),
+    e1000e_getreg(GSCL_2),
+    e1000e_getreg(RDBAH1),
+    e1000e_getreg(FLSWDATA),
+    e1000e_getreg(RXSATRH),
+    e1000e_getreg(TIPG),
+    e1000e_getreg(FLMNGCTL),
+    e1000e_getreg(FLMNGCNT),
+    e1000e_getreg(TSYNCTXCTL),
+    e1000e_getreg(EXTCNF_SIZE),
+    e1000e_getreg(EXTCNF_CTRL),
+    e1000e_getreg(EEMNGDATA),
+    e1000e_getreg(CTRL_EXT),
+    e1000e_getreg(SYSTIMH),
+    e1000e_getreg(EEMNGCTL),
+    e1000e_getreg(FLMNGDATA),
+    e1000e_getreg(TSYNCRXCTL),
+    e1000e_getreg(TDH),
+    e1000e_getreg(LEDCTL),
+    e1000e_getreg(STATUS),
+    e1000e_getreg(TCTL),
+    e1000e_getreg(TDBAL),
+    e1000e_getreg(TDLEN),
+    e1000e_getreg(TDH1),
+    e1000e_getreg(RADV),
+    e1000e_getreg(ECOL),
+    e1000e_getreg(DC),
+    e1000e_getreg(RLEC),
+    e1000e_getreg(XOFFTXC),
+    e1000e_getreg(RFC),
+    e1000e_getreg(RNBC),
+    e1000e_getreg(MGTPTC),
+    e1000e_getreg(TIMINCA),
+    e1000e_getreg(RXCFGL),
+    e1000e_getreg(MFUTP01),
+    e1000e_getreg(FACTPS),
+    e1000e_getreg(GSCL_1),
+    e1000e_getreg(GSCN_0),
+    e1000e_getreg(GCR2),
+    e1000e_getreg(RDT1),
+    e1000e_getreg(PBACLR),
+    e1000e_getreg(FCTTV),
+    e1000e_getreg(EEWR),
+    e1000e_getreg(FLSWCTL),
+    e1000e_getreg(RXDCTL1),
+    e1000e_getreg(RXSATRL),
+    e1000e_getreg(SYSTIML),
+    e1000e_getreg(RXUDP),
+    e1000e_getreg(TORL),
+    e1000e_getreg(TDLEN1),
+    e1000e_getreg(MCC),
+    e1000e_getreg(WUC),
+    e1000e_getreg(EECD),
+    e1000e_getreg(MFUTP23),
+    e1000e_getreg(RAID),
+    e1000e_getreg(FCRTV),
+    e1000e_getreg(TXDCTL1),
+    e1000e_getreg(RCTL),
+    e1000e_getreg(TDT),
+    e1000e_getreg(MDIC),
+    e1000e_getreg(FCRUC),
+    e1000e_getreg(VET),
+    e1000e_getreg(RDBAL0),
+    e1000e_getreg(TDBAH1),
+    e1000e_getreg(RDTR),
+    e1000e_getreg(SCC),
+    e1000e_getreg(COLC),
+    e1000e_getreg(CEXTERR),
+    e1000e_getreg(XOFFRXC),
+    e1000e_getreg(IPAV),
+    e1000e_getreg(GOTCL),
+    e1000e_getreg(MGTPDC),
+    e1000e_getreg(GCR),
+    e1000e_getreg(IVAR),
+    e1000e_getreg(POEMB),
+    e1000e_getreg(MFVAL),
+    e1000e_getreg(FUNCTAG),
+    e1000e_getreg(GSCL_4),
+    e1000e_getreg(GSCN_3),
+    e1000e_getreg(MRQC),
+    e1000e_getreg(RDLEN1),
+    e1000e_getreg(FCT),
+    e1000e_getreg(FLA),
+    e1000e_getreg(FLOL),
+    e1000e_getreg(RXDCTL),
+    e1000e_getreg(RXSTMPL),
+    e1000e_getreg(TXSTMPH),
+    e1000e_getreg(TIMADJH),
+    e1000e_getreg(FCRTL),
+    e1000e_getreg(TDBAH),
+    e1000e_getreg(TADV),
+    e1000e_getreg(XONRXC),
+    e1000e_getreg(TSCTFC),
+    e1000e_getreg(RFCTL),
+    e1000e_getreg(GSCN_1),
+    e1000e_getreg(FCAL),
+    e1000e_getreg(FLSWCNT),
+
+    [TOTH]    = e1000e_mac_read_clr8,
+    [GOTCH]   = e1000e_mac_read_clr8,
+    [PRC64]   = e1000e_mac_read_clr4,
+    [PRC255]  = e1000e_mac_read_clr4,
+    [PRC1023] = e1000e_mac_read_clr4,
+    [PTC64]   = e1000e_mac_read_clr4,
+    [PTC255]  = e1000e_mac_read_clr4,
+    [PTC1023] = e1000e_mac_read_clr4,
+    [GPRC]    = e1000e_mac_read_clr4,
+    [TPT]     = e1000e_mac_read_clr4,
+    [RUC]     = e1000e_mac_read_clr4,
+    [BPRC]    = e1000e_mac_read_clr4,
+    [MPTC]    = e1000e_mac_read_clr4,
+    [IAC]     = e1000e_mac_read_clr4,
+    [ICR]     = e1000e_mac_icr_read,
+    [RDFH]    = E1000E_LOW_BITS_READ(13),
+    [RDFHS]   = E1000E_LOW_BITS_READ(13),
+    [RDFPC]   = E1000E_LOW_BITS_READ(13),
+    [TDFH]    = E1000E_LOW_BITS_READ(13),
+    [TDFHS]   = E1000E_LOW_BITS_READ(13),
+    [STATUS]  = e1000e_get_status,
+    [TARC0]   = e1000e_get_tarc,
+    [PBS]     = E1000E_LOW_BITS_READ(6),
+    [ICS]     = e1000e_mac_ics_read,
+    [AIT]     = E1000E_LOW_BITS_READ(16),
+    [TORH]    = e1000e_mac_read_clr8,
+    [GORCH]   = e1000e_mac_read_clr8,
+    [PRC127]  = e1000e_mac_read_clr4,
+    [PRC511]  = e1000e_mac_read_clr4,
+    [PRC1522] = e1000e_mac_read_clr4,
+    [PTC127]  = e1000e_mac_read_clr4,
+    [PTC511]  = e1000e_mac_read_clr4,
+    [PTC1522] = e1000e_mac_read_clr4,
+    [GPTC]    = e1000e_mac_read_clr4,
+    [TPR]     = e1000e_mac_read_clr4,
+    [ROC]     = e1000e_mac_read_clr4,
+    [MPRC]    = e1000e_mac_read_clr4,
+    [BPTC]    = e1000e_mac_read_clr4,
+    [TSCTC]   = e1000e_mac_read_clr4,
+    [ITR]     = e1000e_mac_itr_read,
+    [RDFT]    = E1000E_LOW_BITS_READ(13),
+    [RDFTS]   = E1000E_LOW_BITS_READ(13),
+    [TDFPC]   = E1000E_LOW_BITS_READ(13),
+    [TDFT]    = E1000E_LOW_BITS_READ(13),
+    [TDFTS]   = E1000E_LOW_BITS_READ(13),
+    [CTRL]    = e1000e_get_ctrl,
+    [TARC1]   = e1000e_get_tarc,
+    [SWSM]    = e1000e_mac_swsm_read,
+    [IMS]     = e1000e_mac_ims_read,
+
+    [CRCERRS ... MPC]      = e1000e_mac_readreg,
+    [IP6AT ... IP6AT + 3]  = e1000e_mac_readreg,
+    [IP4AT ... IP4AT + 6]  = e1000e_mac_readreg,
+    [RA ... RA + 31]       = e1000e_mac_readreg,
+    [WUPM ... WUPM + 31]   = e1000e_mac_readreg,
+    [MTA ... MTA + 127]    = e1000e_mac_readreg,
+    [VFTA ... VFTA + 127]  = e1000e_mac_readreg,
+    [FFMT ... FFMT + 254]  = E1000E_LOW_BITS_READ(4),
+    [FFVT ... FFVT + 254]  = e1000e_mac_readreg,
+    [MDEF ... MDEF + 7]    = e1000e_mac_readreg,
+    [FFLT ... FFLT + 10]   = E1000E_LOW_BITS_READ(11),
+    [FTFT ... FTFT + 254]  = e1000e_mac_readreg,
+    [PBM ... PBM + 10239]  = e1000e_mac_readreg,
+    [RETA ... RETA + 31]   = e1000e_mac_readreg,
+    [RSSRK ... RSSRK + 31] = e1000e_mac_readreg,
+    [MAVTV0 ... MAVTV3]    = e1000e_mac_readreg,
+    [EITR...EITR + E1000E_MSIX_VEC_NUM - 1] = e1000e_mac_eitr_read
+};
+enum { E1000E_NREADOPS = ARRAY_SIZE(e1000e_macreg_readops) };
+
+#define e1000e_putreg(x)    [x] = e1000e_mac_writereg
+static void (*e1000e_macreg_writeops[])(E1000ECore *, int, uint32_t) = {
+    e1000e_putreg(PBA),
+    e1000e_putreg(SWSM),
+    e1000e_putreg(WUFC),
+    e1000e_putreg(RDBAH1),
+    e1000e_putreg(TDBAH),
+    e1000e_putreg(TXDCTL),
+    e1000e_putreg(RDBAH0),
+    e1000e_putreg(LEDCTL),
+    e1000e_putreg(FCAL),
+    e1000e_putreg(FCRUC),
+    e1000e_putreg(AIT),
+    e1000e_putreg(TDFH),
+    e1000e_putreg(TDFT),
+    e1000e_putreg(TDFHS),
+    e1000e_putreg(TDFTS),
+    e1000e_putreg(TDFPC),
+    e1000e_putreg(WUC),
+    e1000e_putreg(WUS),
+    e1000e_putreg(RDFH),
+    e1000e_putreg(RDFT),
+    e1000e_putreg(RDFHS),
+    e1000e_putreg(RDFTS),
+    e1000e_putreg(RDFPC),
+    e1000e_putreg(IPAV),
+    e1000e_putreg(TDBAH1),
+    e1000e_putreg(TIMINCA),
+    e1000e_putreg(IAM),
+    e1000e_putreg(EIAC),
+    e1000e_putreg(IVAR),
+    e1000e_putreg(TARC0),
+    e1000e_putreg(TARC1),
+    e1000e_putreg(FLSWDATA),
+    e1000e_putreg(POEMB),
+    e1000e_putreg(PBS),
+    e1000e_putreg(MFUTP01),
+    e1000e_putreg(MFUTP23),
+    e1000e_putreg(MANC),
+    e1000e_putreg(MANC2H),
+    e1000e_putreg(MFVAL),
+    e1000e_putreg(EXTCNF_CTRL),
+    e1000e_putreg(FACTPS),
+    e1000e_putreg(FUNCTAG),
+    e1000e_putreg(GSCL_1),
+    e1000e_putreg(GSCL_2),
+    e1000e_putreg(GSCL_3),
+    e1000e_putreg(GSCL_4),
+    e1000e_putreg(GSCN_0),
+    e1000e_putreg(GSCN_1),
+    e1000e_putreg(GSCN_2),
+    e1000e_putreg(GSCN_3),
+    e1000e_putreg(GCR2),
+    e1000e_putreg(MRQC),
+    e1000e_putreg(FLOP),
+    e1000e_putreg(FLOL),
+    e1000e_putreg(FLSWCTL),
+    e1000e_putreg(FLSWCNT),
+    e1000e_putreg(FLA),
+    e1000e_putreg(RXDCTL1),
+    e1000e_putreg(TXDCTL1),
+    e1000e_putreg(TIPG),
+    e1000e_putreg(RXSTMPH),
+    e1000e_putreg(RXSTMPL),
+    e1000e_putreg(RXSATRL),
+    e1000e_putreg(RXSATRH),
+    e1000e_putreg(TXSTMPL),
+    e1000e_putreg(TXSTMPH),
+    e1000e_putreg(SYSTIML),
+    e1000e_putreg(SYSTIMH),
+    e1000e_putreg(TIMADJL),
+    e1000e_putreg(TIMADJH),
+    e1000e_putreg(RXUDP),
+    e1000e_putreg(RXCFGL),
+    e1000e_putreg(TSYNCRXCTL),
+    e1000e_putreg(TSYNCTXCTL),
+    e1000e_putreg(FLSWDATA),
+    e1000e_putreg(EXTCNF_SIZE),
+    e1000e_putreg(EEMNGCTL),
+    e1000e_putreg(RA),
+
+    [TDH1]     = e1000e_set_16bit,
+    [TDT1]     = e1000e_set_tdt,
+    [TCTL]     = e1000e_set_tctl,
+    [TDT]      = e1000e_set_tdt,
+    [MDIC]     = e1000e_set_mdic,
+    [ICS]      = e1000e_set_ics,
+    [TDH]      = e1000e_set_16bit,
+    [RDH0]     = e1000e_set_16bit,
+    [RDT0]     = e1000e_set_rdt,
+    [IMC]      = e1000e_set_imc,
+    [IMS]      = e1000e_set_ims,
+    [ICR]      = e1000e_set_icr,
+    [EECD]     = e1000e_set_eecd,
+    [RCTL]     = e1000e_set_rx_control,
+    [CTRL]     = e1000e_set_ctrl,
+    [RDTR]     = e1000e_set_rdtr,
+    [RADV]     = e1000e_set_16bit,
+    [TADV]     = e1000e_set_16bit,
+    [ITR]      = e1000e_set_itr,
+    [EERD]     = e1000e_set_eerd,
+    [GCR]      = e1000e_set_gcr,
+    [PSRCTL]   = e1000e_set_psrctl,
+    [RXCSUM]   = e1000e_set_rxcsum,
+    [RAID]     = e1000e_set_16bit,
+    [RSRPD]    = e1000e_set_12bit,
+    [TIDV]     = e1000e_set_tidv,
+    [TDLEN1]   = e1000e_set_dlen,
+    [TDLEN]    = e1000e_set_dlen,
+    [RDLEN0]   = e1000e_set_dlen,
+    [RDLEN1]   = e1000e_set_dlen,
+    [TDBAL]    = e1000e_set_dbal,
+    [TDBAL1]   = e1000e_set_dbal,
+    [RDBAL0]   = e1000e_set_dbal,
+    [RDBAL1]   = e1000e_set_dbal,
+    [RDH1]     = e1000e_set_16bit,
+    [RDT1]     = e1000e_set_rdt,
+    [STATUS]   = e1000e_set_status,
+    [PBACLR]   = e1000e_set_pbaclr,
+    [CTRL_EXT] = e1000e_set_ctrlext,
+    [FCAH]     = e1000e_set_16bit,
+    [FCT]      = e1000e_set_16bit,
+    [FCTTV]    = e1000e_set_16bit,
+    [FCRTV]    = e1000e_set_16bit,
+    [FCRTH]    = e1000e_set_fcrth,
+    [FCRTL]    = e1000e_set_fcrtl,
+    [VET]      = e1000e_set_vet,
+    [RXDCTL]   = e1000e_set_rxdctl,
+    [FLASHT]   = e1000e_set_16bit,
+    [EEWR]     = e1000e_set_eewr,
+    [CTRL_DUP] = e1000e_set_ctrl,
+    [RFCTL]    = e1000e_set_rfctl,
+    [RA + 1]   = e1000e_mac_setmacaddr,
+
+    [IP6AT ... IP6AT + 3]    = e1000e_mac_writereg,
+    [IP4AT ... IP4AT + 6]    = e1000e_mac_writereg,
+    [RA + 2 ... RA + 31]     = e1000e_mac_writereg,
+    [WUPM ... WUPM + 31]     = e1000e_mac_writereg,
+    [MTA ... MTA + 127]      = e1000e_mac_writereg,
+    [VFTA ... VFTA + 127]    = e1000e_mac_writereg,
+    [FFMT ... FFMT + 254]    = e1000e_mac_writereg,
+    [FFVT ... FFVT + 254]    = e1000e_mac_writereg,
+    [PBM ... PBM + 10239]    = e1000e_mac_writereg,
+    [MDEF ... MDEF + 7]      = e1000e_mac_writereg,
+    [FFLT ... FFLT + 10]     = e1000e_mac_writereg,
+    [FTFT ... FTFT + 254]    = e1000e_mac_writereg,
+    [RETA ... RETA + 31]     = e1000e_mac_writereg,
+    [RSSRK ... RSSRK + 31]   = e1000e_mac_writereg,
+    [MAVTV0 ... MAVTV3]      = e1000e_mac_writereg,
+    [EITR...EITR + E1000E_MSIX_VEC_NUM - 1] = e1000e_set_eitr
+};
+enum { E1000E_NWRITEOPS = ARRAY_SIZE(e1000e_macreg_writeops) };
+
+enum { MAC_ACCESS_PARTIAL = 1 };
+
+/* The array below combines alias offsets of the index values for the
+ * MAC registers that have aliases, with the indication of not fully
+ * implemented registers (lowest bit). This combination is possible
+ * because all of the offsets are even. */
+static const uint16_t mac_reg_access[E1000E_MAC_SIZE] = {
+    /* Alias index offsets */
+    [FCRTL_A] = 0x07fe, [FCRTH_A] = 0x0802,
+    [RDH0_A]  = 0x09bc, [RDT0_A]  = 0x09bc, [RDTR_A] = 0x09c6,
+    [RDFH_A]  = 0xe904, [RDFT_A]  = 0xe904,
+    [TDH_A]   = 0x0cf8, [TDT_A]   = 0x0cf8, [TIDV_A] = 0x0cf8,
+    [TDFH_A]  = 0xed00, [TDFT_A]  = 0xed00,
+    [RA_A ... RA_A + 31]      = 0x14f0,
+    [VFTA_A ... VFTA_A + 127] = 0x1400,
+    [RDBAL0_A ... RDLEN0_A] = 0x09bc,
+    [TDBAL_A ... TDLEN_A]   = 0x0cf8,
+    /* Access options */
+    [RDFH]  = MAC_ACCESS_PARTIAL,    [RDFT]  = MAC_ACCESS_PARTIAL,
+    [RDFHS] = MAC_ACCESS_PARTIAL,    [RDFTS] = MAC_ACCESS_PARTIAL,
+    [RDFPC] = MAC_ACCESS_PARTIAL,
+    [TDFH]  = MAC_ACCESS_PARTIAL,    [TDFT]  = MAC_ACCESS_PARTIAL,
+    [TDFHS] = MAC_ACCESS_PARTIAL,    [TDFTS] = MAC_ACCESS_PARTIAL,
+    [TDFPC] = MAC_ACCESS_PARTIAL,    [EECD]  = MAC_ACCESS_PARTIAL,
+    [PBM]   = MAC_ACCESS_PARTIAL,    [FLA]   = MAC_ACCESS_PARTIAL,
+    [FCAL]  = MAC_ACCESS_PARTIAL,    [FCAH]  = MAC_ACCESS_PARTIAL,
+    [FCT]   = MAC_ACCESS_PARTIAL,    [FCTTV] = MAC_ACCESS_PARTIAL,
+    [FCRTV] = MAC_ACCESS_PARTIAL,    [FCRTL] = MAC_ACCESS_PARTIAL,
+    [FCRTH] = MAC_ACCESS_PARTIAL,    [TXDCTL] = MAC_ACCESS_PARTIAL,
+    [TXDCTL1] = MAC_ACCESS_PARTIAL,
+    [MAVTV0 ... MAVTV3] = MAC_ACCESS_PARTIAL
+};
+
+void
+e1000e_core_write(E1000ECore *core, hwaddr addr, uint64_t val, unsigned size)
+{
+    uint16_t index = e1000e_get_reg_index_with_offset(mac_reg_access, addr);
+
+    if (index < E1000E_NWRITEOPS && e1000e_macreg_writeops[index]) {
+        if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+            trace_e1000e_wrn_regs_write_trivial(index << 2);
+        }
+        trace_e1000e_core_write(index << 2, size, val);
+        e1000e_macreg_writeops[index](core, index, val);
+    } else if (index < E1000E_NREADOPS && e1000e_macreg_readops[index]) {
+        trace_e1000e_wrn_regs_write_ro(index << 2, size, val);
+    } else {
+        trace_e1000e_wrn_regs_write_unknown(index << 2, size, val);
+    }
+}
+
+uint64_t
+e1000e_core_read(E1000ECore *core, hwaddr addr, unsigned size)
+{
+    uint64_t val;
+    uint16_t index = e1000e_get_reg_index_with_offset(mac_reg_access, addr);
+
+    if (index < E1000E_NREADOPS && e1000e_macreg_readops[index]) {
+        if (mac_reg_access[index] & MAC_ACCESS_PARTIAL) {
+            trace_e1000e_wrn_regs_read_trivial(index << 2);
+        }
+        val = e1000e_macreg_readops[index](core, index);
+        trace_e1000e_core_read(index << 2, size, val);
+        return val;
+    } else {
+        trace_e1000e_wrn_regs_read_unknown(index << 2, size);
+    }
+    return 0;
+}
+
+static inline void
+e1000e_autoneg_pause(E1000ECore *core)
+{
+    timer_del(core->autoneg_timer);
+}
+
+static void
+e1000e_autoneg_resume(E1000ECore *core)
+{
+    if (e1000e_have_autoneg(core) &&
+        !(core->phy[0][PHY_STATUS] & MII_SR_AUTONEG_COMPLETE)) {
+        qemu_get_queue(core->owner_nic)->link_down = false;
+        timer_mod(core->autoneg_timer,
+                  qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
+    }
+}
+
+static void
+e1000e_vm_state_change(void *opaque, int running, RunState state)
+{
+    E1000ECore *core = opaque;
+
+    if (running) {
+        trace_e1000e_vm_state_running();
+        e1000e_intrmgr_resume(core);
+        e1000e_autoneg_resume(core);
+    } else {
+        trace_e1000e_vm_state_stopped();
+        e1000e_autoneg_pause(core);
+        e1000e_intrmgr_pause(core);
+    }
+}
+
+void
+e1000e_core_pci_realize(E1000ECore     *core,
+                        const uint16_t *eeprom_templ,
+                        uint32_t        eeprom_size,
+                        const uint8_t  *macaddr)
+{
+    int i;
+
+    core->autoneg_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                       e1000e_autoneg_timer, core);
+    e1000e_intrmgr_pci_realize(core);
+
+    core->vmstate =
+        qemu_add_vm_change_state_handler(e1000e_vm_state_change, core);
+
+    for (i = 0; i < E1000E_NUM_QUEUES; i++) {
+        net_tx_pkt_init(&core->tx[i].tx_pkt, core->owner,
+                        E1000E_MAX_TX_FRAGS, core->has_vnet);
+    }
+
+    net_rx_pkt_init(&core->rx_pkt, core->has_vnet);
+
+    e1000x_core_prepare_eeprom(core->eeprom,
+                               eeprom_templ,
+                               eeprom_size,
+                               PCI_DEVICE_GET_CLASS(core->owner)->device_id,
+                               macaddr);
+    e1000e_update_rx_offloads(core);
+}
+
+void
+e1000e_core_pci_uninit(E1000ECore *core)
+{
+    int i;
+
+    timer_del(core->autoneg_timer);
+    timer_free(core->autoneg_timer);
+
+    e1000e_intrmgr_pci_unint(core);
+
+    qemu_del_vm_change_state_handler(core->vmstate);
+
+    for (i = 0; i < E1000E_NUM_QUEUES; i++) {
+        net_tx_pkt_reset(core->tx[i].tx_pkt);
+        net_tx_pkt_uninit(core->tx[i].tx_pkt);
+    }
+
+    net_rx_pkt_uninit(core->rx_pkt);
+}
+
+static const uint16_t
+e1000e_phy_reg_init[E1000E_PHY_PAGES][E1000E_PHY_PAGE_SIZE] = {
+    [0] = {
+        [PHY_CTRL] =   MII_CR_SPEED_SELECT_MSB  |
+                       MII_CR_FULL_DUPLEX       |
+                       MII_CR_AUTO_NEG_EN,
+
+        [PHY_STATUS] = MII_SR_EXTENDED_CAPS     |
+                       MII_SR_LINK_STATUS       |
+                       MII_SR_AUTONEG_CAPS      |
+                       MII_SR_PREAMBLE_SUPPRESS |
+                       MII_SR_EXTENDED_STATUS   |
+                       MII_SR_10T_HD_CAPS       |
+                       MII_SR_10T_FD_CAPS       |
+                       MII_SR_100X_HD_CAPS      |
+                       MII_SR_100X_FD_CAPS,
+
+        [PHY_ID1]               = 0x141,
+        [PHY_ID2]               = E1000_PHY_ID2_82574x,
+        [PHY_AUTONEG_ADV]       = 0xde1,
+        [PHY_LP_ABILITY]        = 0x7e0,
+        [PHY_AUTONEG_EXP]       = BIT(2),
+        [PHY_NEXT_PAGE_TX]      = BIT(0) | BIT(13),
+        [PHY_1000T_CTRL]        = BIT(8) | BIT(9) | BIT(10) | BIT(11),
+        [PHY_1000T_STATUS]      = 0x3c00,
+        [PHY_EXT_STATUS]        = BIT(12) | BIT(13),
+
+        [PHY_COPPER_CTRL1]      = BIT(5) | BIT(6) | BIT(8) | BIT(9) |
+                                  BIT(12) | BIT(13),
+        [PHY_COPPER_STAT1]      = BIT(3) | BIT(10) | BIT(11) | BIT(13) | BIT(15)
+    },
+    [2] = {
+        [PHY_MAC_CTRL1]         = BIT(3) | BIT(7),
+        [PHY_MAC_CTRL2]         = BIT(1) | BIT(2) | BIT(6) | BIT(12)
+    },
+    [3] = {
+        [PHY_LED_TIMER_CTRL]    = BIT(0) | BIT(2) | BIT(14)
+    }
+};
+
+static const uint32_t e1000e_mac_reg_init[] = {
+    [PBA]           =     0x00140014,
+    [LEDCTL]        =  BIT(1) | BIT(8) | BIT(9) | BIT(15) | BIT(17) | BIT(18),
+    [EXTCNF_CTRL]   = BIT(3),
+    [EEMNGCTL]      = BIT(31),
+    [FLASHT]        = 0x2,
+    [FLSWCTL]       = BIT(30) | BIT(31),
+    [FLOL]          = BIT(0),
+    [RXDCTL]        = BIT(16),
+    [RXDCTL1]       = BIT(16),
+    [TIPG]          = 0x8 | (0x8 << 10) | (0x6 << 20),
+    [RXCFGL]        = 0x88F7,
+    [RXUDP]         = 0x319,
+    [CTRL]          = E1000_CTRL_FD | E1000_CTRL_SWDPIN2 | E1000_CTRL_SWDPIN0 |
+                      E1000_CTRL_SPD_1000 | E1000_CTRL_SLU |
+                      E1000_CTRL_ADVD3WUC,
+    [STATUS]        =  E1000_STATUS_ASDV_1000 | E1000_STATUS_LU,
+    [PSRCTL]        = (2 << E1000_PSRCTL_BSIZE0_SHIFT) |
+                      (4 << E1000_PSRCTL_BSIZE1_SHIFT) |
+                      (4 << E1000_PSRCTL_BSIZE2_SHIFT),
+    [TARC0]         = 0x3 | E1000_TARC_ENABLE,
+    [TARC1]         = 0x3 | E1000_TARC_ENABLE,
+    [EECD]          = E1000_EECD_AUTO_RD | E1000_EECD_PRES,
+    [EERD]          = E1000_EERW_DONE,
+    [EEWR]          = E1000_EERW_DONE,
+    [GCR]           = E1000_L0S_ADJUST |
+                      E1000_L1_ENTRY_LATENCY_MSB |
+                      E1000_L1_ENTRY_LATENCY_LSB,
+    [TDFH]          = 0x600,
+    [TDFT]          = 0x600,
+    [TDFHS]         = 0x600,
+    [TDFTS]         = 0x600,
+    [POEMB]         = 0x30D,
+    [PBS]           = 0x028,
+    [MANC]          = E1000_MANC_DIS_IP_CHK_ARP,
+    [FACTPS]        = E1000_FACTPS_LAN0_ON | 0x20000000,
+    [SWSM]          = 1,
+    [RXCSUM]        = E1000_RXCSUM_IPOFLD | E1000_RXCSUM_TUOFLD,
+    [ITR]           = E1000E_MIN_XITR,
+    [EITR...EITR + E1000E_MSIX_VEC_NUM - 1] = E1000E_MIN_XITR,
+};
+
+void
+e1000e_core_reset(E1000ECore *core)
+{
+    int i;
+
+    timer_del(core->autoneg_timer);
+
+    e1000e_intrmgr_reset(core);
+
+    memset(core->phy, 0, sizeof core->phy);
+    memmove(core->phy, e1000e_phy_reg_init, sizeof e1000e_phy_reg_init);
+    memset(core->mac, 0, sizeof core->mac);
+    memmove(core->mac, e1000e_mac_reg_init, sizeof e1000e_mac_reg_init);
+
+    core->rxbuf_min_shift = 1 + E1000_RING_DESC_LEN_SHIFT;
+
+    if (qemu_get_queue(core->owner_nic)->link_down) {
+        e1000e_link_down(core);
+    }
+
+    e1000x_reset_mac_addr(core->owner_nic, core->mac, core->permanent_mac);
+
+    for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
+        net_tx_pkt_reset(core->tx[i].tx_pkt);
+        memset(&core->tx[i].props, 0, sizeof(core->tx[i].props));
+        core->tx[i].skip_cp = false;
+    }
+}
+
+void e1000e_core_pre_save(E1000ECore *core)
+{
+    int i;
+    NetClientState *nc = qemu_get_queue(core->owner_nic);
+
+    /*
+    * If link is down and auto-negotiation is supported and ongoing,
+    * complete auto-negotiation immediately. This allows us to look
+    * at MII_SR_AUTONEG_COMPLETE to infer link status on load.
+    */
+    if (nc->link_down && e1000e_have_autoneg(core)) {
+        core->phy[0][PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE;
+        e1000e_update_flowctl_status(core);
+    }
+
+    for (i = 0; i < ARRAY_SIZE(core->tx); i++) {
+        if (net_tx_pkt_has_fragments(core->tx[i].tx_pkt)) {
+            core->tx[i].skip_cp = true;
+        }
+    }
+}
+
+int
+e1000e_core_post_load(E1000ECore *core)
+{
+    NetClientState *nc = qemu_get_queue(core->owner_nic);
+
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in core.mac[STATUS].
+     */
+    nc->link_down = (core->mac[STATUS] & E1000_STATUS_LU) == 0;
+
+    return 0;
+}
diff --git a/hw/net/e1000e_core.h b/hw/net/e1000e_core.h
new file mode 100644
index 0000000..5f413a9
--- /dev/null
+++ b/hw/net/e1000e_core.h
@@ -0,0 +1,146 @@
+/*
+* Core code for QEMU e1000e emulation
+*
+* Software developer's manuals:
+* http://www.intel.com/content/dam/doc/datasheet/82574l-gbe-controller-datasheet.pdf
+*
+* Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com)
+* Developed by Daynix Computing LTD (http://www.daynix.com)
+*
+* Authors:
+* Dmitry Fleytman <dmitry@daynix.com>
+* Leonid Bloch <leonid@daynix.com>
+* Yan Vugenfirer <yan@daynix.com>
+*
+* Based on work done by:
+* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
+* Copyright (c) 2008 Qumranet
+* Based on work done by:
+* Copyright (c) 2007 Dan Aloni
+* Copyright (c) 2004 Antony T Curtis
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define E1000E_PHY_PAGE_SIZE    (0x20)
+#define E1000E_PHY_PAGES        (0x07)
+#define E1000E_MAC_SIZE         (0x8000)
+#define E1000E_EEPROM_SIZE      (64)
+#define E1000E_MSIX_VEC_NUM     (5)
+#define E1000E_NUM_QUEUES       (2)
+
+typedef struct E1000Core E1000ECore;
+
+enum { PHY_R = BIT(0),
+       PHY_W = BIT(1),
+       PHY_RW = PHY_R | PHY_W,
+       PHY_ANYPAGE = BIT(2) };
+
+typedef struct E1000IntrDelayTimer_st {
+    QEMUTimer *timer;
+    bool running;
+    uint32_t delay_reg;
+    uint32_t delay_resolution_ns;
+    E1000ECore *core;
+} E1000IntrDelayTimer;
+
+struct E1000Core {
+    uint32_t mac[E1000E_MAC_SIZE];
+    uint16_t phy[E1000E_PHY_PAGES][E1000E_PHY_PAGE_SIZE];
+    uint16_t eeprom[E1000E_EEPROM_SIZE];
+
+    uint32_t rxbuf_sizes[E1000_PSRCTL_BUFFS_PER_DESC];
+    uint32_t rx_desc_buf_size;
+    uint32_t rxbuf_min_shift;
+    uint8_t rx_desc_len;
+
+    QEMUTimer *autoneg_timer;
+
+    struct e1000e_tx {
+        e1000x_txd_props props;
+
+        bool skip_cp;
+        struct NetTxPkt *tx_pkt;
+    } tx[E1000E_NUM_QUEUES];
+
+    struct NetRxPkt *rx_pkt;
+
+    bool has_vnet;
+    int max_queue_num;
+
+    /* Interrupt moderation management */
+    uint32_t delayed_causes;
+
+    E1000IntrDelayTimer radv;
+    E1000IntrDelayTimer rdtr;
+    E1000IntrDelayTimer raid;
+
+    E1000IntrDelayTimer tadv;
+    E1000IntrDelayTimer tidv;
+
+    E1000IntrDelayTimer itr;
+    bool itr_intr_pending;
+
+    E1000IntrDelayTimer eitr[E1000E_MSIX_VEC_NUM];
+    bool eitr_intr_pending[E1000E_MSIX_VEC_NUM];
+
+    VMChangeStateEntry *vmstate;
+
+    uint32_t itr_guest_value;
+    uint32_t eitr_guest_value[E1000E_MSIX_VEC_NUM];
+
+    uint16_t vet;
+
+    uint8_t permanent_mac[ETH_ALEN];
+
+    NICState *owner_nic;
+    PCIDevice *owner;
+    void (*owner_start_recv)(PCIDevice *d);
+};
+
+void
+e1000e_core_write(E1000ECore *core, hwaddr addr, uint64_t val, unsigned size);
+
+uint64_t
+e1000e_core_read(E1000ECore *core, hwaddr addr, unsigned size);
+
+void
+e1000e_core_pci_realize(E1000ECore      *regs,
+                       const uint16_t *eeprom_templ,
+                       uint32_t        eeprom_size,
+                       const uint8_t  *macaddr);
+
+void
+e1000e_core_reset(E1000ECore *core);
+
+void
+e1000e_core_pre_save(E1000ECore *core);
+
+int
+e1000e_core_post_load(E1000ECore *core);
+
+void
+e1000e_core_set_link_status(E1000ECore *core);
+
+void
+e1000e_core_pci_uninit(E1000ECore *core);
+
+int
+e1000e_can_receive(E1000ECore *core);
+
+ssize_t
+e1000e_receive(E1000ECore *core, const uint8_t *buf, size_t size);
+
+ssize_t
+e1000e_receive_iov(E1000ECore *core, const struct iovec *iov, int iovcnt);
diff --git a/hw/net/e1000x_common.c b/hw/net/e1000x_common.c
new file mode 100644
index 0000000..eb0e097
--- /dev/null
+++ b/hw/net/e1000x_common.c
@@ -0,0 +1,267 @@
+/*
+* QEMU e1000(e) emulation - shared code
+*
+* Copyright (c) 2008 Qumranet
+*
+* Based on work done by:
+* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
+* Copyright (c) 2007 Dan Aloni
+* Copyright (c) 2004 Antony T Curtis
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "net/net.h"
+
+#include "e1000x_common.h"
+
+#include "trace.h"
+
+bool e1000x_rx_ready(PCIDevice *d, uint32_t *mac)
+{
+    bool link_up = mac[STATUS] & E1000_STATUS_LU;
+    bool rx_enabled = mac[RCTL] & E1000_RCTL_EN;
+    bool pci_master = d->config[PCI_COMMAND] & PCI_COMMAND_MASTER;
+
+    if (!link_up || !rx_enabled || !pci_master) {
+        trace_e1000x_rx_can_recv_disabled(link_up, rx_enabled, pci_master);
+        return false;
+    }
+
+    return true;
+}
+
+bool e1000x_is_vlan_packet(const uint8_t *buf, uint16_t vet)
+{
+    uint16_t eth_proto = lduw_be_p(buf + 12);
+    bool res = (eth_proto == vet);
+
+    trace_e1000x_vlan_is_vlan_pkt(res, eth_proto, vet);
+
+    return res;
+}
+
+bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf)
+{
+    static const int mta_shift[] = { 4, 3, 2, 0 };
+    uint32_t f, ra[2], *rp, rctl = mac[RCTL];
+
+    for (rp = mac + RA; rp < mac + RA + 32; rp += 2) {
+        if (!(rp[1] & E1000_RAH_AV)) {
+            continue;
+        }
+        ra[0] = cpu_to_le32(rp[0]);
+        ra[1] = cpu_to_le32(rp[1]);
+        if (!memcmp(buf, (uint8_t *)ra, 6)) {
+            trace_e1000x_rx_flt_ucast_match((int)(rp - mac - RA) / 2,
+                                            MAC_ARG(buf));
+            return true;
+        }
+    }
+    trace_e1000x_rx_flt_ucast_mismatch(MAC_ARG(buf));
+
+    f = mta_shift[(rctl >> E1000_RCTL_MO_SHIFT) & 3];
+    f = (((buf[5] << 8) | buf[4]) >> f) & 0xfff;
+    if (mac[MTA + (f >> 5)] & (1 << (f & 0x1f))) {
+        e1000x_inc_reg_if_not_full(mac, MPRC);
+        return true;
+    }
+
+    trace_e1000x_rx_flt_inexact_mismatch(MAC_ARG(buf),
+                                         (rctl >> E1000_RCTL_MO_SHIFT) & 3,
+                                         f >> 5,
+                                         mac[MTA + (f >> 5)]);
+
+    return false;
+}
+
+bool e1000x_hw_rx_enabled(uint32_t *mac)
+{
+    if (!(mac[STATUS] & E1000_STATUS_LU)) {
+        trace_e1000x_rx_link_down(mac[STATUS]);
+        return false;
+    }
+
+    if (!(mac[RCTL] & E1000_RCTL_EN)) {
+        trace_e1000x_rx_disabled(mac[RCTL]);
+        return false;
+    }
+
+    return true;
+}
+
+bool e1000x_is_oversized(uint32_t *mac, size_t size)
+{
+    /* this is the size past which hardware will
+       drop packets when setting LPE=0 */
+    static const int maximum_ethernet_vlan_size = 1522;
+    /* this is the size past which hardware will
+       drop packets when setting LPE=1 */
+    static const int maximum_ethernet_lpe_size = 16384;
+
+    if ((size > maximum_ethernet_lpe_size ||
+        (size > maximum_ethernet_vlan_size
+            && !(mac[RCTL] & E1000_RCTL_LPE)))
+        && !(mac[RCTL] & E1000_RCTL_SBP)) {
+        e1000x_inc_reg_if_not_full(mac, ROC);
+        trace_e1000x_rx_oversized(size);
+        return true;
+    }
+
+    return false;
+}
+
+void e1000x_restart_autoneg(uint32_t *mac, uint16_t *phy, QEMUTimer *timer)
+{
+    e1000x_update_regs_on_link_down(mac, phy);
+    trace_e1000x_link_negotiation_start();
+    timer_mod(timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + 500);
+}
+
+void e1000x_reset_mac_addr(NICState *nic, uint32_t *mac_regs,
+                           uint8_t *mac_addr)
+{
+    int i;
+
+    mac_regs[RA] = 0;
+    mac_regs[RA + 1] = E1000_RAH_AV;
+    for (i = 0; i < 4; i++) {
+        mac_regs[RA] |= mac_addr[i] << (8 * i);
+        mac_regs[RA + 1] |=
+            (i < 2) ? mac_addr[i + 4] << (8 * i) : 0;
+    }
+
+    qemu_format_nic_info_str(qemu_get_queue(nic), mac_addr);
+    trace_e1000x_mac_indicate(MAC_ARG(mac_addr));
+}
+
+void e1000x_update_regs_on_autoneg_done(uint32_t *mac, uint16_t *phy)
+{
+    e1000x_update_regs_on_link_up(mac, phy);
+    phy[PHY_LP_ABILITY] |= MII_LPAR_LPACK;
+    phy[PHY_STATUS] |= MII_SR_AUTONEG_COMPLETE;
+    trace_e1000x_link_negotiation_done();
+}
+
+void
+e1000x_core_prepare_eeprom(uint16_t       *eeprom,
+                           const uint16_t *templ,
+                           uint32_t        templ_size,
+                           uint16_t        dev_id,
+                           const uint8_t  *macaddr)
+{
+    uint16_t checksum = 0;
+    int i;
+
+    memmove(eeprom, templ, templ_size);
+
+    for (i = 0; i < 3; i++) {
+        eeprom[i] = (macaddr[2 * i + 1] << 8) | macaddr[2 * i];
+    }
+
+    eeprom[11] = eeprom[13] = dev_id;
+
+    for (i = 0; i < EEPROM_CHECKSUM_REG; i++) {
+        checksum += eeprom[i];
+    }
+
+    checksum = (uint16_t) EEPROM_SUM - checksum;
+
+    eeprom[EEPROM_CHECKSUM_REG] = checksum;
+}
+
+uint32_t
+e1000x_rxbufsize(uint32_t rctl)
+{
+    rctl &= E1000_RCTL_BSEX | E1000_RCTL_SZ_16384 | E1000_RCTL_SZ_8192 |
+        E1000_RCTL_SZ_4096 | E1000_RCTL_SZ_2048 | E1000_RCTL_SZ_1024 |
+        E1000_RCTL_SZ_512 | E1000_RCTL_SZ_256;
+    switch (rctl) {
+    case E1000_RCTL_BSEX | E1000_RCTL_SZ_16384:
+        return 16384;
+    case E1000_RCTL_BSEX | E1000_RCTL_SZ_8192:
+        return 8192;
+    case E1000_RCTL_BSEX | E1000_RCTL_SZ_4096:
+        return 4096;
+    case E1000_RCTL_SZ_1024:
+        return 1024;
+    case E1000_RCTL_SZ_512:
+        return 512;
+    case E1000_RCTL_SZ_256:
+        return 256;
+    }
+    return 2048;
+}
+
+void
+e1000x_update_rx_total_stats(uint32_t *mac,
+                             size_t data_size,
+                             size_t data_fcs_size)
+{
+    static const int PRCregs[6] = { PRC64, PRC127, PRC255, PRC511,
+                                    PRC1023, PRC1522 };
+
+    e1000x_increase_size_stats(mac, PRCregs, data_fcs_size);
+    e1000x_inc_reg_if_not_full(mac, TPR);
+    mac[GPRC] = mac[TPR];
+    /* TOR - Total Octets Received:
+    * This register includes bytes received in a packet from the <Destination
+    * Address> field through the <CRC> field, inclusively.
+    * Always include FCS length (4) in size.
+    */
+    e1000x_grow_8reg_if_not_full(mac, TORL, data_size + 4);
+    mac[GORCL] = mac[TORL];
+    mac[GORCH] = mac[TORH];
+}
+
+void
+e1000x_increase_size_stats(uint32_t *mac, const int *size_regs, int size)
+{
+    if (size > 1023) {
+        e1000x_inc_reg_if_not_full(mac, size_regs[5]);
+    } else if (size > 511) {
+        e1000x_inc_reg_if_not_full(mac, size_regs[4]);
+    } else if (size > 255) {
+        e1000x_inc_reg_if_not_full(mac, size_regs[3]);
+    } else if (size > 127) {
+        e1000x_inc_reg_if_not_full(mac, size_regs[2]);
+    } else if (size > 64) {
+        e1000x_inc_reg_if_not_full(mac, size_regs[1]);
+    } else if (size == 64) {
+        e1000x_inc_reg_if_not_full(mac, size_regs[0]);
+    }
+}
+
+void
+e1000x_read_tx_ctx_descr(struct e1000_context_desc *d,
+                         e1000x_txd_props *props)
+{
+    uint32_t op = le32_to_cpu(d->cmd_and_length);
+
+    props->ipcss = d->lower_setup.ip_fields.ipcss;
+    props->ipcso = d->lower_setup.ip_fields.ipcso;
+    props->ipcse = le16_to_cpu(d->lower_setup.ip_fields.ipcse);
+    props->tucss = d->upper_setup.tcp_fields.tucss;
+    props->tucso = d->upper_setup.tcp_fields.tucso;
+    props->tucse = le16_to_cpu(d->upper_setup.tcp_fields.tucse);
+    props->paylen = op & 0xfffff;
+    props->hdr_len = d->tcp_seg_setup.fields.hdr_len;
+    props->mss = le16_to_cpu(d->tcp_seg_setup.fields.mss);
+    props->ip = (op & E1000_TXD_CMD_IP) ? 1 : 0;
+    props->tcp = (op & E1000_TXD_CMD_TCP) ? 1 : 0;
+    props->tse = (op & E1000_TXD_CMD_TSE) ? 1 : 0;
+}
diff --git a/hw/net/e1000x_common.h b/hw/net/e1000x_common.h
new file mode 100644
index 0000000..21bf28e
--- /dev/null
+++ b/hw/net/e1000x_common.h
@@ -0,0 +1,213 @@
+/*
+* QEMU e1000(e) emulation - shared code
+*
+* Copyright (c) 2008 Qumranet
+*
+* Based on work done by:
+* Nir Peleg, Tutis Systems Ltd. for Qumranet Inc.
+* Copyright (c) 2007 Dan Aloni
+* Copyright (c) 2004 Antony T Curtis
+*
+* This library is free software; you can redistribute it and/or
+* modify it under the terms of the GNU Lesser General Public
+* License as published by the Free Software Foundation; either
+* version 2 of the License, or (at your option) any later version.
+*
+* This library is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+* Lesser General Public License for more details.
+*
+* You should have received a copy of the GNU Lesser General Public
+* License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "e1000_regs.h"
+
+#define defreg(x)   x = (E1000_##x >> 2)
+enum {
+    defreg(CTRL),    defreg(EECD),    defreg(EERD),    defreg(GPRC),
+    defreg(GPTC),    defreg(ICR),     defreg(ICS),     defreg(IMC),
+    defreg(IMS),     defreg(LEDCTL),  defreg(MANC),    defreg(MDIC),
+    defreg(MPC),     defreg(PBA),     defreg(RCTL),    defreg(RDBAH0),
+    defreg(RDBAL0),  defreg(RDH0),    defreg(RDLEN0),  defreg(RDT0),
+    defreg(STATUS),  defreg(SWSM),    defreg(TCTL),    defreg(TDBAH),
+    defreg(TDBAL),   defreg(TDH),     defreg(TDLEN),   defreg(TDT),
+    defreg(TDLEN1),  defreg(TDBAL1),  defreg(TDBAH1),  defreg(TDH1),
+    defreg(TDT1),    defreg(TORH),    defreg(TORL),    defreg(TOTH),
+    defreg(TOTL),    defreg(TPR),     defreg(TPT),     defreg(TXDCTL),
+    defreg(WUFC),    defreg(RA),      defreg(MTA),     defreg(CRCERRS),
+    defreg(VFTA),    defreg(VET),     defreg(RDTR),    defreg(RADV),
+    defreg(TADV),    defreg(ITR),     defreg(SCC),     defreg(ECOL),
+    defreg(MCC),     defreg(LATECOL), defreg(COLC),    defreg(DC),
+    defreg(TNCRS),   defreg(SEC),     defreg(CEXTERR), defreg(RLEC),
+    defreg(XONRXC),  defreg(XONTXC),  defreg(XOFFRXC), defreg(XOFFTXC),
+    defreg(FCRUC),   defreg(AIT),     defreg(TDFH),    defreg(TDFT),
+    defreg(TDFHS),   defreg(TDFTS),   defreg(TDFPC),   defreg(WUC),
+    defreg(WUS),     defreg(POEMB),   defreg(PBS),     defreg(RDFH),
+    defreg(RDFT),    defreg(RDFHS),   defreg(RDFTS),   defreg(RDFPC),
+    defreg(PBM),     defreg(IPAV),    defreg(IP4AT),   defreg(IP6AT),
+    defreg(WUPM),    defreg(FFLT),    defreg(FFMT),    defreg(FFVT),
+    defreg(TARC0),   defreg(TARC1),   defreg(IAM),     defreg(EXTCNF_CTRL),
+    defreg(GCR),     defreg(TIMINCA), defreg(EIAC),    defreg(CTRL_EXT),
+    defreg(IVAR),    defreg(MFUTP01), defreg(MFUTP23), defreg(MANC2H),
+    defreg(MFVAL),   defreg(MDEF),    defreg(FACTPS),  defreg(FTFT),
+    defreg(RUC),     defreg(ROC),     defreg(RFC),     defreg(RJC),
+    defreg(PRC64),   defreg(PRC127),  defreg(PRC255),  defreg(PRC511),
+    defreg(PRC1023), defreg(PRC1522), defreg(PTC64),   defreg(PTC127),
+    defreg(PTC255),  defreg(PTC511),  defreg(PTC1023), defreg(PTC1522),
+    defreg(GORCL),   defreg(GORCH),   defreg(GOTCL),   defreg(GOTCH),
+    defreg(RNBC),    defreg(BPRC),    defreg(MPRC),    defreg(RFCTL),
+    defreg(PSRCTL),  defreg(MPTC),    defreg(BPTC),    defreg(TSCTFC),
+    defreg(IAC),     defreg(MGTPRC),  defreg(MGTPDC),  defreg(MGTPTC),
+    defreg(TSCTC),   defreg(RXCSUM),  defreg(FUNCTAG), defreg(GSCL_1),
+    defreg(GSCL_2),  defreg(GSCL_3),  defreg(GSCL_4),  defreg(GSCN_0),
+    defreg(GSCN_1),  defreg(GSCN_2),  defreg(GSCN_3),  defreg(GCR2),
+    defreg(RAID),    defreg(RSRPD),   defreg(TIDV),    defreg(EITR),
+    defreg(MRQC),    defreg(RETA),    defreg(RSSRK),   defreg(RDBAH1),
+    defreg(RDBAL1),  defreg(RDLEN1),  defreg(RDH1),    defreg(RDT1),
+    defreg(PBACLR),  defreg(FCAL),    defreg(FCAH),    defreg(FCT),
+    defreg(FCRTH),   defreg(FCRTL),   defreg(FCTTV),   defreg(FCRTV),
+    defreg(FLA),     defreg(EEWR),    defreg(FLOP),    defreg(FLOL),
+    defreg(FLSWCTL), defreg(FLSWCNT), defreg(RXDCTL),  defreg(RXDCTL1),
+    defreg(MAVTV0),  defreg(MAVTV1),  defreg(MAVTV2),  defreg(MAVTV3),
+    defreg(TXSTMPL), defreg(TXSTMPH), defreg(SYSTIML), defreg(SYSTIMH),
+    defreg(RXCFGL),  defreg(RXUDP),   defreg(TIMADJL), defreg(TIMADJH),
+    defreg(RXSTMPH), defreg(RXSTMPL), defreg(RXSATRL), defreg(RXSATRH),
+    defreg(FLASHT),  defreg(TIPG),    defreg(RDH),     defreg(RDT),
+    defreg(RDLEN),   defreg(RDBAH),   defreg(RDBAL),
+    defreg(TXDCTL1),
+    defreg(FLSWDATA),
+    defreg(CTRL_DUP),
+    defreg(EXTCNF_SIZE),
+    defreg(EEMNGCTL),
+    defreg(EEMNGDATA),
+    defreg(FLMNGCTL),
+    defreg(FLMNGDATA),
+    defreg(FLMNGCNT),
+    defreg(TSYNCRXCTL),
+    defreg(TSYNCTXCTL),
+
+    /* Aliases */
+    defreg(RDH0_A),  defreg(RDT0_A),  defreg(RDTR_A),  defreg(RDFH_A),
+    defreg(RDFT_A),  defreg(TDH_A),   defreg(TDT_A),   defreg(TIDV_A),
+    defreg(TDFH_A),  defreg(TDFT_A),  defreg(RA_A),    defreg(RDBAL0_A),
+    defreg(TDBAL_A), defreg(TDLEN_A), defreg(VFTA_A),  defreg(RDLEN0_A),
+    defreg(FCRTL_A), defreg(FCRTH_A)
+};
+
+static inline void
+e1000x_inc_reg_if_not_full(uint32_t *mac, int index)
+{
+    if (mac[index] != 0xffffffff) {
+        mac[index]++;
+    }
+}
+
+static inline void
+e1000x_grow_8reg_if_not_full(uint32_t *mac, int index, int size)
+{
+    uint64_t sum = mac[index] | (uint64_t)mac[index + 1] << 32;
+
+    if (sum + size < sum) {
+        sum = ~0ULL;
+    } else {
+        sum += size;
+    }
+    mac[index] = sum;
+    mac[index + 1] = sum >> 32;
+}
+
+static inline int
+e1000x_vlan_enabled(uint32_t *mac)
+{
+    return ((mac[CTRL] & E1000_CTRL_VME) != 0);
+}
+
+static inline int
+e1000x_is_vlan_txd(uint32_t txd_lower)
+{
+    return ((txd_lower & E1000_TXD_CMD_VLE) != 0);
+}
+
+static inline int
+e1000x_vlan_rx_filter_enabled(uint32_t *mac)
+{
+    return ((mac[RCTL] & E1000_RCTL_VFE) != 0);
+}
+
+static inline int
+e1000x_fcs_len(uint32_t *mac)
+{
+    /* FCS aka Ethernet CRC-32. We don't get it from backends and can't
+    * fill it in, just pad descriptor length by 4 bytes unless guest
+    * told us to strip it off the packet. */
+    return (mac[RCTL] & E1000_RCTL_SECRC) ? 0 : 4;
+}
+
+static inline void
+e1000x_update_regs_on_link_down(uint32_t *mac, uint16_t *phy)
+{
+    mac[STATUS] &= ~E1000_STATUS_LU;
+    phy[PHY_STATUS] &= ~MII_SR_LINK_STATUS;
+    phy[PHY_STATUS] &= ~MII_SR_AUTONEG_COMPLETE;
+    phy[PHY_LP_ABILITY] &= ~MII_LPAR_LPACK;
+}
+
+static inline void
+e1000x_update_regs_on_link_up(uint32_t *mac, uint16_t *phy)
+{
+    mac[STATUS] |= E1000_STATUS_LU;
+    phy[PHY_STATUS] |= MII_SR_LINK_STATUS;
+}
+
+void e1000x_update_rx_total_stats(uint32_t *mac,
+                                  size_t data_size,
+                                  size_t data_fcs_size);
+
+void e1000x_core_prepare_eeprom(uint16_t       *eeprom,
+                                const uint16_t *templ,
+                                uint32_t        templ_size,
+                                uint16_t        dev_id,
+                                const uint8_t  *macaddr);
+
+uint32_t e1000x_rxbufsize(uint32_t rctl);
+
+bool e1000x_rx_ready(PCIDevice *d, uint32_t *mac);
+
+bool e1000x_is_vlan_packet(const uint8_t *buf, uint16_t vet);
+
+bool e1000x_rx_group_filter(uint32_t *mac, const uint8_t *buf);
+
+bool e1000x_hw_rx_enabled(uint32_t *mac);
+
+bool e1000x_is_oversized(uint32_t *mac, size_t size);
+
+void e1000x_restart_autoneg(uint32_t *mac, uint16_t *phy, QEMUTimer *timer);
+
+void e1000x_reset_mac_addr(NICState *nic, uint32_t *mac_regs,
+                           uint8_t *mac_addr);
+
+void e1000x_update_regs_on_autoneg_done(uint32_t *mac, uint16_t *phy);
+
+void e1000x_increase_size_stats(uint32_t *mac, const int *size_regs, int size);
+
+typedef struct e1000x_txd_props {
+    unsigned char sum_needed;
+    uint8_t ipcss;
+    uint8_t ipcso;
+    uint16_t ipcse;
+    uint8_t tucss;
+    uint8_t tucso;
+    uint16_t tucse;
+    uint32_t paylen;
+    uint8_t hdr_len;
+    uint16_t mss;
+    int8_t ip;
+    int8_t tcp;
+    bool tse;
+    bool cptse;
+} e1000x_txd_props;
+
+void e1000x_read_tx_ctx_descr(struct e1000_context_desc *d,
+                              e1000x_txd_props *props);
diff --git a/hw/net/eepro100.c b/hw/net/eepro100.c
index 53e4c65..390c297 100644
--- a/hw/net/eepro100.c
+++ b/hw/net/eepro100.c
@@ -40,7 +40,7 @@
  *      * Wake-on-LAN is not implemented.
  */
 
-#include <stddef.h>             /* offsetof */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -352,14 +352,14 @@
 static uint16_t e100_read_reg2(EEPRO100State *s, E100RegisterOffset addr)
 {
     assert(!((uintptr_t)&s->mem[addr] & 1));
-    return le16_to_cpup((uint16_t *)&s->mem[addr]);
+    return lduw_le_p(&s->mem[addr]);
 }
 
 /* Read a 32 bit control/status (CSR) register. */
 static uint32_t e100_read_reg4(EEPRO100State *s, E100RegisterOffset addr)
 {
     assert(!((uintptr_t)&s->mem[addr] & 3));
-    return le32_to_cpup((uint32_t *)&s->mem[addr]);
+    return ldl_le_p(&s->mem[addr]);
 }
 
 /* Write a 16 bit control/status (CSR) register. */
@@ -367,7 +367,7 @@
                             uint16_t val)
 {
     assert(!((uintptr_t)&s->mem[addr] & 1));
-    cpu_to_le16w((uint16_t *)&s->mem[addr], val);
+    stw_le_p(&s->mem[addr], val);
 }
 
 /* Read a 32 bit control/status (CSR) register. */
@@ -375,7 +375,7 @@
                             uint32_t val)
 {
     assert(!((uintptr_t)&s->mem[addr] & 3));
-    cpu_to_le32w((uint32_t *)&s->mem[addr], val);
+    stl_le_p(&s->mem[addr], val);
 }
 
 #if defined(DEBUG_EEPRO100)
@@ -775,6 +775,11 @@
 #if 0
         uint16_t tx_buffer_el = lduw_le_pci_dma(&s->dev, tbd_address + 6);
 #endif
+        if (tx_buffer_size == 0) {
+            /* Prevent an endless loop. */
+            logout("loop in %s:%u\n", __FILE__, __LINE__);
+            break;
+        }
         tbd_address += 8;
         TRACE(RXTX, logout
             ("TBD (simplified mode): buffer address 0x%08x, size 0x%04x\n",
@@ -856,6 +861,10 @@
 
 static void action_command(EEPRO100State *s)
 {
+    /* The loop below won't stop if it gets special handcrafted data.
+       Therefore we limit the number of iterations. */
+    unsigned max_loop_count = 16;
+
     for (;;) {
         bool bit_el;
         bool bit_s;
@@ -871,6 +880,13 @@
 #if 0
         bool bit_sf = ((s->tx.command & COMMAND_SF) != 0);
 #endif
+
+        if (max_loop_count-- == 0) {
+            /* Prevent an endless loop. */
+            logout("loop in %s:%u\n", __FILE__, __LINE__);
+            break;
+        }
+
         s->cu_offset = s->tx.link;
         TRACE(OTHER,
               logout("val=(cu start), status=0x%04x, command=0x%04x, link=0x%08x\n",
@@ -1618,16 +1634,6 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int nic_can_receive(NetClientState *nc)
-{
-    EEPRO100State *s = qemu_get_nic_opaque(nc);
-    TRACE(RXTX, logout("%p\n", s));
-    return get_ru_state(s) == ru_ready;
-#if 0
-    return !eepro100_buffer_full(s);
-#endif
-}
-
 static ssize_t nic_receive(NetClientState *nc, const uint8_t * buf, size_t size)
 {
     /* TODO:
@@ -1833,13 +1839,6 @@
     }
 };
 
-static void nic_cleanup(NetClientState *nc)
-{
-    EEPRO100State *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static void pci_nic_uninit(PCIDevice *pci_dev)
 {
     EEPRO100State *s = DO_UPCAST(EEPRO100State, dev, pci_dev);
@@ -1850,14 +1849,12 @@
 }
 
 static NetClientInfo net_eepro100_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = nic_can_receive,
     .receive = nic_receive,
-    .cleanup = nic_cleanup,
 };
 
-static int e100_nic_init(PCIDevice *pci_dev)
+static void e100_nic_realize(PCIDevice *pci_dev, Error **errp)
 {
     EEPRO100State *s = DO_UPCAST(EEPRO100State, dev, pci_dev);
     E100PCIDeviceInfo *info = eepro100_get_class(s);
@@ -1901,8 +1898,6 @@
     memcpy(s->vmstate, &vmstate_eepro100, sizeof(vmstate_eepro100));
     s->vmstate->name = qemu_get_queue(s->nic)->model;
     vmstate_register(&pci_dev->qdev, -1, s->vmstate, s);
-
-    return 0;
 }
 
 static void eepro100_instance_init(Object *obj)
@@ -2092,7 +2087,7 @@
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->class_id = PCI_CLASS_NETWORK_ETHERNET;
     k->romfile = "pxe-eepro100.rom";
-    k->init = e100_nic_init;
+    k->realize = e100_nic_realize;
     k->exit = pci_nic_uninit;
     k->device_id = info->device_id;
     k->revision = info->revision;
diff --git a/hw/net/etraxfs_eth.c b/hw/net/etraxfs_eth.c
index 6a3c86d..efaa49f 100644
--- a/hw/net/etraxfs_eth.c
+++ b/hw/net/etraxfs_eth.c
@@ -22,10 +22,11 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/cris/etraxfs.h"
+#include "qemu/error-report.h"
 
 #define D(x)
 
@@ -520,11 +521,6 @@
     return match;
 }
 
-static int eth_can_receive(NetClientState *nc)
-{
-    return 1;
-}
-
 static ssize_t eth_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     unsigned char sa_bcast[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
@@ -581,24 +577,10 @@
     }
 };
 
-static void eth_cleanup(NetClientState *nc)
-{
-    ETRAXFSEthState *eth = qemu_get_nic_opaque(nc);
-
-    /* Disconnect the client.  */
-    eth->dma_out->client.push = NULL;
-    eth->dma_out->client.opaque = NULL;
-    eth->dma_in->client.opaque = NULL;
-    eth->dma_in->client.pull = NULL;
-        g_free(eth);
-}
-
 static NetClientInfo net_etraxfs_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = eth_can_receive,
     .receive = eth_receive,
-    .cleanup = eth_cleanup,
     .link_status_changed = eth_set_link,
 };
 
@@ -608,7 +590,8 @@
     ETRAXFSEthState *s = ETRAX_FS_ETH(dev);
 
     if (!s->dma_out || !s->dma_in) {
-        hw_error("Unconnected ETRAX-FS Ethernet MAC.\n");
+        error_report("Unconnected ETRAX-FS Ethernet MAC");
+        return -1;
     }
 
     s->dma_out->client.push = eth_tx_push;
diff --git a/hw/net/fsl_etsec/etsec.c b/hw/net/fsl_etsec/etsec.c
index d4b4429..b5c777f 100644
--- a/hw/net/fsl_etsec/etsec.c
+++ b/hw/net/fsl_etsec/etsec.c
@@ -26,12 +26,14 @@
  * This implementation doesn't include ring priority, TCP/IP Off-Load, QoS.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 #include "hw/ptimer.h"
 #include "etsec.h"
 #include "registers.h"
+#include "qemu/log.h"
 
 /* #define HEX_DUMP */
 /* #define DEBUG_REGISTER */
@@ -338,30 +340,26 @@
         MII_SR_100X_FD_CAPS     | MII_SR_100T4_CAPS;
 }
 
-static void etsec_cleanup(NetClientState *nc)
-{
-    /* qemu_log("eTSEC cleanup\n"); */
-}
-
-static int etsec_can_receive(NetClientState *nc)
-{
-    eTSEC *etsec = qemu_get_nic_opaque(nc);
-
-    return etsec->rx_buffer_len == 0;
-}
-
 static ssize_t etsec_receive(NetClientState *nc,
                              const uint8_t  *buf,
                              size_t          size)
 {
+    ssize_t ret;
     eTSEC *etsec = qemu_get_nic_opaque(nc);
 
 #if defined(HEX_DUMP)
     fprintf(stderr, "%s receive size:%d\n", etsec->nic->nc.name, size);
     qemu_hexdump(buf, stderr, "", size);
 #endif
-    etsec_rx_ring_write(etsec, buf, size);
-    return size;
+    /* Flush is unnecessary as are already in receiving path */
+    etsec->need_flush = false;
+    ret = etsec_rx_ring_write(etsec, buf, size);
+    if (ret == 0) {
+        /* The packet will be queued, let's flush it when buffer is available
+         * again. */
+        etsec->need_flush = true;
+    }
+    return ret;
 }
 
 
@@ -373,11 +371,9 @@
 }
 
 static NetClientInfo net_etsec_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = etsec_can_receive,
     .receive = etsec_receive,
-    .cleanup = etsec_cleanup,
     .link_status_changed = etsec_set_link_status,
 };
 
@@ -449,10 +445,7 @@
 
     dev = qdev_create(NULL, "eTSEC");
     qdev_set_nic_properties(dev, nd);
-
-    if (qdev_init(dev)) {
-        return NULL;
-    }
+    qdev_init_nofail(dev);
 
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, tx_irq);
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 1, rx_irq);
diff --git a/hw/net/fsl_etsec/etsec.h b/hw/net/fsl_etsec/etsec.h
index 78d2c57..30c828e 100644
--- a/hw/net/fsl_etsec/etsec.h
+++ b/hw/net/fsl_etsec/etsec.h
@@ -21,8 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef _ETSEC_H_
-#define _ETSEC_H_
+
+#ifndef ETSEC_H
+#define ETSEC_H
 
 #include "hw/qdev.h"
 #include "hw/sysbus.h"
@@ -144,6 +145,8 @@
     QEMUBH *bh;
     struct ptimer_state *ptimer;
 
+    /* Whether we should flush the rx queue when buffer becomes available. */
+    bool need_flush;
 } eTSEC;
 
 #define TYPE_ETSEC_COMMON "eTSEC"
@@ -162,7 +165,7 @@
 
 void etsec_walk_tx_ring(eTSEC *etsec, int ring_nbr);
 void etsec_walk_rx_ring(eTSEC *etsec, int ring_nbr);
-void etsec_rx_ring_write(eTSEC *etsec, const uint8_t *buf, size_t size);
+ssize_t etsec_rx_ring_write(eTSEC *etsec, const uint8_t *buf, size_t size);
 
 void etsec_write_miim(eTSEC          *etsec,
                       eTSEC_Register *reg,
@@ -171,4 +174,4 @@
 
 void etsec_miim_link_status(eTSEC *etsec, NetClientState *nc);
 
-#endif /* ! _ETSEC_H_ */
+#endif /* ETSEC_H */
diff --git a/hw/net/fsl_etsec/miim.c b/hw/net/fsl_etsec/miim.c
index 1931b74..6bba01c 100644
--- a/hw/net/fsl_etsec/miim.c
+++ b/hw/net/fsl_etsec/miim.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "etsec.h"
 #include "registers.h"
 
diff --git a/hw/net/fsl_etsec/registers.c b/hw/net/fsl_etsec/registers.c
index a7bbfa1..46ce7a8 100644
--- a/hw/net/fsl_etsec/registers.c
+++ b/hw/net/fsl_etsec/registers.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "registers.h"
 
 const eTSEC_Register_Definition eTSEC_registers_def[] = {
diff --git a/hw/net/fsl_etsec/registers.h b/hw/net/fsl_etsec/registers.h
index 7ad7686..c4ed2b9 100644
--- a/hw/net/fsl_etsec/registers.h
+++ b/hw/net/fsl_etsec/registers.h
@@ -21,10 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef _ETSEC_REGISTERS_H_
-#define _ETSEC_REGISTERS_H_
 
-#include <stdint.h>
+#ifndef ETSEC_REGISTERS_H
+#define ETSEC_REGISTERS_H
 
 enum eTSEC_Register_Access_Type {
     ACC_RW      = 1,            /* Read/Write */
@@ -317,4 +316,4 @@
 #define TMR_ETTS2_H  (0xEA8 / 4)
 #define TMR_ETTS2_L  (0xEAC / 4)
 
-#endif /* ! _ETSEC_REGISTERS_H_ */
+#endif /* ETSEC_REGISTERS_H */
diff --git a/hw/net/fsl_etsec/rings.c b/hw/net/fsl_etsec/rings.c
index d4a494f..79d2f14 100644
--- a/hw/net/fsl_etsec/rings.c
+++ b/hw/net/fsl_etsec/rings.c
@@ -21,8 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "net/checksum.h"
-
+#include "qemu/log.h"
 #include "etsec.h"
 #include "registers.h"
 
@@ -464,9 +465,7 @@
         etsec->rx_fcb_size = 0;
     }
 
-    if (etsec->rx_buffer != NULL) {
-        g_free(etsec->rx_buffer);
-    }
+    g_free(etsec->rx_buffer);
 
     /* Do not copy the frame for now */
     etsec->rx_buffer     = (uint8_t *)buf;
@@ -481,40 +480,42 @@
                etsec->rx_buffer_len, etsec->rx_padding);
 }
 
-void etsec_rx_ring_write(eTSEC *etsec, const uint8_t *buf, size_t size)
+ssize_t etsec_rx_ring_write(eTSEC *etsec, const uint8_t *buf, size_t size)
 {
     int ring_nbr = 0;           /* Always use ring0 (no filer) */
 
     if (etsec->rx_buffer_len != 0) {
         RING_DEBUG("%s: We can't receive now,"
                    " a buffer is already in the pipe\n", __func__);
-        return;
+        return 0;
     }
 
     if (etsec->regs[RSTAT].value & 1 << (23 - ring_nbr)) {
         RING_DEBUG("%s: The ring is halted\n", __func__);
-        return;
+        return -1;
     }
 
     if (etsec->regs[DMACTRL].value & DMACTRL_GRS) {
         RING_DEBUG("%s: Graceful receive stop\n", __func__);
-        return;
+        return -1;
     }
 
     if (!(etsec->regs[MACCFG1].value & MACCFG1_RX_EN)) {
         RING_DEBUG("%s: MAC Receive not enabled\n", __func__);
-        return;
+        return -1;
     }
 
     if ((etsec->regs[RCTRL].value & RCTRL_RSF) && (size < 60)) {
         /* CRC is not in the packet yet, so short frame is below 60 bytes */
         RING_DEBUG("%s: Drop short frame\n", __func__);
-        return;
+        return -1;
     }
 
     rx_init_frame(etsec, buf, size);
 
     etsec_walk_rx_ring(etsec, ring_nbr);
+
+    return size;
 }
 
 void etsec_walk_rx_ring(eTSEC *etsec, int ring_nbr)
@@ -644,6 +645,9 @@
     } else {
         etsec->rx_buffer_len = 0;
         etsec->rx_buffer     = NULL;
+        if (etsec->need_flush) {
+            qemu_flush_queued_packets(qemu_get_queue(etsec->nic));
+        }
     }
 
     RING_DEBUG("eTSEC End of ring_write: remaining_data:%zu\n", remaining_data);
diff --git a/hw/net/imx_fec.c b/hw/net/imx_fec.c
new file mode 100644
index 0000000..1c415ab
--- /dev/null
+++ b/hw/net/imx_fec.c
@@ -0,0 +1,1231 @@
+/*
+ * i.MX Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ * Based on Coldfire Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2007 CodeSourcery.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/net/imx_fec.h"
+#include "sysemu/dma.h"
+#include "qemu/log.h"
+#include "net/checksum.h"
+#include "net/eth.h"
+
+/* For crc32 */
+#include <zlib.h>
+
+#ifndef DEBUG_IMX_FEC
+#define DEBUG_IMX_FEC 0
+#endif
+
+#define FEC_PRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_FEC) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_FEC, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+#ifndef DEBUG_IMX_PHY
+#define DEBUG_IMX_PHY 0
+#endif
+
+#define PHY_PRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_PHY) { \
+            fprintf(stderr, "[%s.phy]%s: " fmt , TYPE_IMX_FEC, \
+                                                 __func__, ##args); \
+        } \
+    } while (0)
+
+static const char *imx_default_reg_name(IMXFECState *s, uint32_t index)
+{
+    static char tmp[20];
+    sprintf(tmp, "index %d", index);
+    return tmp;
+}
+
+static const char *imx_fec_reg_name(IMXFECState *s, uint32_t index)
+{
+    switch (index) {
+    case ENET_FRBR:
+        return "FRBR";
+    case ENET_FRSR:
+        return "FRSR";
+    case ENET_MIIGSK_CFGR:
+        return "MIIGSK_CFGR";
+    case ENET_MIIGSK_ENR:
+        return "MIIGSK_ENR";
+    default:
+        return imx_default_reg_name(s, index);
+    }
+}
+
+static const char *imx_enet_reg_name(IMXFECState *s, uint32_t index)
+{
+    switch (index) {
+    case ENET_RSFL:
+        return "RSFL";
+    case ENET_RSEM:
+        return "RSEM";
+    case ENET_RAEM:
+        return "RAEM";
+    case ENET_RAFL:
+        return "RAFL";
+    case ENET_TSEM:
+        return "TSEM";
+    case ENET_TAEM:
+        return "TAEM";
+    case ENET_TAFL:
+        return "TAFL";
+    case ENET_TIPG:
+        return "TIPG";
+    case ENET_FTRL:
+        return "FTRL";
+    case ENET_TACC:
+        return "TACC";
+    case ENET_RACC:
+        return "RACC";
+    case ENET_ATCR:
+        return "ATCR";
+    case ENET_ATVR:
+        return "ATVR";
+    case ENET_ATOFF:
+        return "ATOFF";
+    case ENET_ATPER:
+        return "ATPER";
+    case ENET_ATCOR:
+        return "ATCOR";
+    case ENET_ATINC:
+        return "ATINC";
+    case ENET_ATSTMP:
+        return "ATSTMP";
+    case ENET_TGSR:
+        return "TGSR";
+    case ENET_TCSR0:
+        return "TCSR0";
+    case ENET_TCCR0:
+        return "TCCR0";
+    case ENET_TCSR1:
+        return "TCSR1";
+    case ENET_TCCR1:
+        return "TCCR1";
+    case ENET_TCSR2:
+        return "TCSR2";
+    case ENET_TCCR2:
+        return "TCCR2";
+    case ENET_TCSR3:
+        return "TCSR3";
+    case ENET_TCCR3:
+        return "TCCR3";
+    default:
+        return imx_default_reg_name(s, index);
+    }
+}
+
+static const char *imx_eth_reg_name(IMXFECState *s, uint32_t index)
+{
+    switch (index) {
+    case ENET_EIR:
+        return "EIR";
+    case ENET_EIMR:
+        return "EIMR";
+    case ENET_RDAR:
+        return "RDAR";
+    case ENET_TDAR:
+        return "TDAR";
+    case ENET_ECR:
+        return "ECR";
+    case ENET_MMFR:
+        return "MMFR";
+    case ENET_MSCR:
+        return "MSCR";
+    case ENET_MIBC:
+        return "MIBC";
+    case ENET_RCR:
+        return "RCR";
+    case ENET_TCR:
+        return "TCR";
+    case ENET_PALR:
+        return "PALR";
+    case ENET_PAUR:
+        return "PAUR";
+    case ENET_OPD:
+        return "OPD";
+    case ENET_IAUR:
+        return "IAUR";
+    case ENET_IALR:
+        return "IALR";
+    case ENET_GAUR:
+        return "GAUR";
+    case ENET_GALR:
+        return "GALR";
+    case ENET_TFWR:
+        return "TFWR";
+    case ENET_RDSR:
+        return "RDSR";
+    case ENET_TDSR:
+        return "TDSR";
+    case ENET_MRBR:
+        return "MRBR";
+    default:
+        if (s->is_fec) {
+            return imx_fec_reg_name(s, index);
+        } else {
+            return imx_enet_reg_name(s, index);
+        }
+    }
+}
+
+static const VMStateDescription vmstate_imx_eth = {
+    .name = TYPE_IMX_FEC,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, IMXFECState, ENET_MAX),
+        VMSTATE_UINT32(rx_descriptor, IMXFECState),
+        VMSTATE_UINT32(tx_descriptor, IMXFECState),
+
+        VMSTATE_UINT32(phy_status, IMXFECState),
+        VMSTATE_UINT32(phy_control, IMXFECState),
+        VMSTATE_UINT32(phy_advertise, IMXFECState),
+        VMSTATE_UINT32(phy_int, IMXFECState),
+        VMSTATE_UINT32(phy_int_mask, IMXFECState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#define PHY_INT_ENERGYON            (1 << 7)
+#define PHY_INT_AUTONEG_COMPLETE    (1 << 6)
+#define PHY_INT_FAULT               (1 << 5)
+#define PHY_INT_DOWN                (1 << 4)
+#define PHY_INT_AUTONEG_LP          (1 << 3)
+#define PHY_INT_PARFAULT            (1 << 2)
+#define PHY_INT_AUTONEG_PAGE        (1 << 1)
+
+static void imx_eth_update(IMXFECState *s);
+
+/*
+ * The MII phy could raise a GPIO to the processor which in turn
+ * could be handled as an interrpt by the OS.
+ * For now we don't handle any GPIO/interrupt line, so the OS will
+ * have to poll for the PHY status.
+ */
+static void phy_update_irq(IMXFECState *s)
+{
+    imx_eth_update(s);
+}
+
+static void phy_update_link(IMXFECState *s)
+{
+    /* Autonegotiation status mirrors link status.  */
+    if (qemu_get_queue(s->nic)->link_down) {
+        PHY_PRINTF("link is down\n");
+        s->phy_status &= ~0x0024;
+        s->phy_int |= PHY_INT_DOWN;
+    } else {
+        PHY_PRINTF("link is up\n");
+        s->phy_status |= 0x0024;
+        s->phy_int |= PHY_INT_ENERGYON;
+        s->phy_int |= PHY_INT_AUTONEG_COMPLETE;
+    }
+    phy_update_irq(s);
+}
+
+static void imx_eth_set_link(NetClientState *nc)
+{
+    phy_update_link(IMX_FEC(qemu_get_nic_opaque(nc)));
+}
+
+static void phy_reset(IMXFECState *s)
+{
+    s->phy_status = 0x7809;
+    s->phy_control = 0x3000;
+    s->phy_advertise = 0x01e1;
+    s->phy_int_mask = 0;
+    s->phy_int = 0;
+    phy_update_link(s);
+}
+
+static uint32_t do_phy_read(IMXFECState *s, int reg)
+{
+    uint32_t val;
+
+    if (reg > 31) {
+        /* we only advertise one phy */
+        return 0;
+    }
+
+    switch (reg) {
+    case 0:     /* Basic Control */
+        val = s->phy_control;
+        break;
+    case 1:     /* Basic Status */
+        val = s->phy_status;
+        break;
+    case 2:     /* ID1 */
+        val = 0x0007;
+        break;
+    case 3:     /* ID2 */
+        val = 0xc0d1;
+        break;
+    case 4:     /* Auto-neg advertisement */
+        val = s->phy_advertise;
+        break;
+    case 5:     /* Auto-neg Link Partner Ability */
+        val = 0x0f71;
+        break;
+    case 6:     /* Auto-neg Expansion */
+        val = 1;
+        break;
+    case 29:    /* Interrupt source.  */
+        val = s->phy_int;
+        s->phy_int = 0;
+        phy_update_irq(s);
+        break;
+    case 30:    /* Interrupt mask */
+        val = s->phy_int_mask;
+        break;
+    case 17:
+    case 18:
+    case 27:
+    case 31:
+        qemu_log_mask(LOG_UNIMP, "[%s.phy]%s: reg %d not implemented\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        val = 0;
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s.phy]%s: Bad address at offset %d\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        val = 0;
+        break;
+    }
+
+    PHY_PRINTF("read 0x%04x @ %d\n", val, reg);
+
+    return val;
+}
+
+static void do_phy_write(IMXFECState *s, int reg, uint32_t val)
+{
+    PHY_PRINTF("write 0x%04x @ %d\n", val, reg);
+
+    if (reg > 31) {
+        /* we only advertise one phy */
+        return;
+    }
+
+    switch (reg) {
+    case 0:     /* Basic Control */
+        if (val & 0x8000) {
+            phy_reset(s);
+        } else {
+            s->phy_control = val & 0x7980;
+            /* Complete autonegotiation immediately.  */
+            if (val & 0x1000) {
+                s->phy_status |= 0x0020;
+            }
+        }
+        break;
+    case 4:     /* Auto-neg advertisement */
+        s->phy_advertise = (val & 0x2d7f) | 0x80;
+        break;
+    case 30:    /* Interrupt mask */
+        s->phy_int_mask = val & 0xff;
+        phy_update_irq(s);
+        break;
+    case 17:
+    case 18:
+    case 27:
+    case 31:
+        qemu_log_mask(LOG_UNIMP, "[%s.phy)%s: reg %d not implemented\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        break;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s.phy]%s: Bad address at offset %d\n",
+                      TYPE_IMX_FEC, __func__, reg);
+        break;
+    }
+}
+
+static void imx_fec_read_bd(IMXFECBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_read(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_fec_write_bd(IMXFECBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_write(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_enet_read_bd(IMXENETBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_read(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_enet_write_bd(IMXENETBufDesc *bd, dma_addr_t addr)
+{
+    dma_memory_write(&address_space_memory, addr, bd, sizeof(*bd));
+}
+
+static void imx_eth_update(IMXFECState *s)
+{
+    if (s->regs[ENET_EIR] & s->regs[ENET_EIMR] & ENET_INT_TS_TIMER) {
+        qemu_set_irq(s->irq[1], 1);
+    } else {
+        qemu_set_irq(s->irq[1], 0);
+    }
+
+    if (s->regs[ENET_EIR] & s->regs[ENET_EIMR] & ENET_INT_MAC) {
+        qemu_set_irq(s->irq[0], 1);
+    } else {
+        qemu_set_irq(s->irq[0], 0);
+    }
+}
+
+static void imx_fec_do_tx(IMXFECState *s)
+{
+    int frame_size = 0;
+    uint8_t frame[ENET_MAX_FRAME_SIZE];
+    uint8_t *ptr = frame;
+    uint32_t addr = s->tx_descriptor;
+
+    while (1) {
+        IMXFECBufDesc bd;
+        int len;
+
+        imx_fec_read_bd(&bd, addr);
+        FEC_PRINTF("tx_bd %x flags %04x len %d data %08x\n",
+                   addr, bd.flags, bd.length, bd.data);
+        if ((bd.flags & ENET_BD_R) == 0) {
+            /* Run out of descriptors to transmit.  */
+            FEC_PRINTF("tx_bd ran out of descriptors to transmit\n");
+            break;
+        }
+        len = bd.length;
+        if (frame_size + len > ENET_MAX_FRAME_SIZE) {
+            len = ENET_MAX_FRAME_SIZE - frame_size;
+            s->regs[ENET_EIR] |= ENET_INT_BABT;
+        }
+        dma_memory_read(&address_space_memory, bd.data, ptr, len);
+        ptr += len;
+        frame_size += len;
+        if (bd.flags & ENET_BD_L) {
+            /* Last buffer in frame.  */
+            qemu_send_packet(qemu_get_queue(s->nic), frame, len);
+            ptr = frame;
+            frame_size = 0;
+            s->regs[ENET_EIR] |= ENET_INT_TXF;
+        }
+        s->regs[ENET_EIR] |= ENET_INT_TXB;
+        bd.flags &= ~ENET_BD_R;
+        /* Write back the modified descriptor.  */
+        imx_fec_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & ENET_BD_W) != 0) {
+            addr = s->regs[ENET_TDSR];
+        } else {
+            addr += sizeof(bd);
+        }
+    }
+
+    s->tx_descriptor = addr;
+
+    imx_eth_update(s);
+}
+
+static void imx_enet_do_tx(IMXFECState *s)
+{
+    int frame_size = 0;
+    uint8_t frame[ENET_MAX_FRAME_SIZE];
+    uint8_t *ptr = frame;
+    uint32_t addr = s->tx_descriptor;
+
+    while (1) {
+        IMXENETBufDesc bd;
+        int len;
+
+        imx_enet_read_bd(&bd, addr);
+        FEC_PRINTF("tx_bd %x flags %04x len %d data %08x option %04x "
+                   "status %04x\n", addr, bd.flags, bd.length, bd.data,
+                   bd.option, bd.status);
+        if ((bd.flags & ENET_BD_R) == 0) {
+            /* Run out of descriptors to transmit.  */
+            break;
+        }
+        len = bd.length;
+        if (frame_size + len > ENET_MAX_FRAME_SIZE) {
+            len = ENET_MAX_FRAME_SIZE - frame_size;
+            s->regs[ENET_EIR] |= ENET_INT_BABT;
+        }
+        dma_memory_read(&address_space_memory, bd.data, ptr, len);
+        ptr += len;
+        frame_size += len;
+        if (bd.flags & ENET_BD_L) {
+            if (bd.option & ENET_BD_PINS) {
+                struct ip_header *ip_hd = PKT_GET_IP_HDR(frame);
+                if (IP_HEADER_VERSION(ip_hd) == 4) {
+                    net_checksum_calculate(frame, frame_size);
+                }
+            }
+            if (bd.option & ENET_BD_IINS) {
+                struct ip_header *ip_hd = PKT_GET_IP_HDR(frame);
+                /* We compute checksum only for IPv4 frames */
+                if (IP_HEADER_VERSION(ip_hd) == 4) {
+                    uint16_t csum;
+                    ip_hd->ip_sum = 0;
+                    csum = net_raw_checksum((uint8_t *)ip_hd, sizeof(*ip_hd));
+                    ip_hd->ip_sum = cpu_to_be16(csum);
+                }
+            }
+            /* Last buffer in frame.  */
+            qemu_send_packet(qemu_get_queue(s->nic), frame, len);
+            ptr = frame;
+            frame_size = 0;
+            if (bd.option & ENET_BD_TX_INT) {
+                s->regs[ENET_EIR] |= ENET_INT_TXF;
+            }
+        }
+        if (bd.option & ENET_BD_TX_INT) {
+            s->regs[ENET_EIR] |= ENET_INT_TXB;
+        }
+        bd.flags &= ~ENET_BD_R;
+        /* Write back the modified descriptor.  */
+        imx_enet_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & ENET_BD_W) != 0) {
+            addr = s->regs[ENET_TDSR];
+        } else {
+            addr += sizeof(bd);
+        }
+    }
+
+    s->tx_descriptor = addr;
+
+    imx_eth_update(s);
+}
+
+static void imx_eth_do_tx(IMXFECState *s)
+{
+    if (!s->is_fec && (s->regs[ENET_ECR] & ENET_ECR_EN1588)) {
+        imx_enet_do_tx(s);
+    } else {
+        imx_fec_do_tx(s);
+    }
+}
+
+static void imx_eth_enable_rx(IMXFECState *s)
+{
+    IMXFECBufDesc bd;
+    bool tmp;
+
+    imx_fec_read_bd(&bd, s->rx_descriptor);
+
+    tmp = ((bd.flags & ENET_BD_E) != 0);
+
+    if (!tmp) {
+        FEC_PRINTF("RX buffer full\n");
+    } else if (!s->regs[ENET_RDAR]) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+
+    s->regs[ENET_RDAR] = tmp ? ENET_RDAR_RDAR : 0;
+}
+
+static void imx_eth_reset(DeviceState *d)
+{
+    IMXFECState *s = IMX_FEC(d);
+
+    /* Reset the Device */
+    memset(s->regs, 0, sizeof(s->regs));
+    s->regs[ENET_ECR]   = 0xf0000000;
+    s->regs[ENET_MIBC]  = 0xc0000000;
+    s->regs[ENET_RCR]   = 0x05ee0001;
+    s->regs[ENET_OPD]   = 0x00010000;
+
+    s->regs[ENET_PALR]  = (s->conf.macaddr.a[0] << 24)
+                          | (s->conf.macaddr.a[1] << 16)
+                          | (s->conf.macaddr.a[2] << 8)
+                          | s->conf.macaddr.a[3];
+    s->regs[ENET_PAUR]  = (s->conf.macaddr.a[4] << 24)
+                          | (s->conf.macaddr.a[5] << 16)
+                          | 0x8808;
+
+    if (s->is_fec) {
+        s->regs[ENET_FRBR]  = 0x00000600;
+        s->regs[ENET_FRSR]  = 0x00000500;
+        s->regs[ENET_MIIGSK_ENR]  = 0x00000006;
+    } else {
+        s->regs[ENET_RAEM]  = 0x00000004;
+        s->regs[ENET_RAFL]  = 0x00000004;
+        s->regs[ENET_TAEM]  = 0x00000004;
+        s->regs[ENET_TAFL]  = 0x00000008;
+        s->regs[ENET_TIPG]  = 0x0000000c;
+        s->regs[ENET_FTRL]  = 0x000007ff;
+        s->regs[ENET_ATPER] = 0x3b9aca00;
+    }
+
+    s->rx_descriptor = 0;
+    s->tx_descriptor = 0;
+
+    /* We also reset the PHY */
+    phy_reset(s);
+}
+
+static uint32_t imx_default_read(IMXFECState *s, uint32_t index)
+{
+    qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                  PRIx32 "\n", TYPE_IMX_FEC, __func__, index * 4);
+    return 0;
+}
+
+static uint32_t imx_fec_read(IMXFECState *s, uint32_t index)
+{
+    switch (index) {
+    case ENET_FRBR:
+    case ENET_FRSR:
+    case ENET_MIIGSK_CFGR:
+    case ENET_MIIGSK_ENR:
+        return s->regs[index];
+    default:
+        return imx_default_read(s, index);
+    }
+}
+
+static uint32_t imx_enet_read(IMXFECState *s, uint32_t index)
+{
+    switch (index) {
+    case ENET_RSFL:
+    case ENET_RSEM:
+    case ENET_RAEM:
+    case ENET_RAFL:
+    case ENET_TSEM:
+    case ENET_TAEM:
+    case ENET_TAFL:
+    case ENET_TIPG:
+    case ENET_FTRL:
+    case ENET_TACC:
+    case ENET_RACC:
+    case ENET_ATCR:
+    case ENET_ATVR:
+    case ENET_ATOFF:
+    case ENET_ATPER:
+    case ENET_ATCOR:
+    case ENET_ATINC:
+    case ENET_ATSTMP:
+    case ENET_TGSR:
+    case ENET_TCSR0:
+    case ENET_TCCR0:
+    case ENET_TCSR1:
+    case ENET_TCCR1:
+    case ENET_TCSR2:
+    case ENET_TCCR2:
+    case ENET_TCSR3:
+    case ENET_TCCR3:
+        return s->regs[index];
+    default:
+        return imx_default_read(s, index);
+    }
+}
+
+static uint64_t imx_eth_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMXFECState *s = IMX_FEC(opaque);
+    uint32_t index = offset >> 2;
+
+    switch (index) {
+    case ENET_EIR:
+    case ENET_EIMR:
+    case ENET_RDAR:
+    case ENET_TDAR:
+    case ENET_ECR:
+    case ENET_MMFR:
+    case ENET_MSCR:
+    case ENET_MIBC:
+    case ENET_RCR:
+    case ENET_TCR:
+    case ENET_PALR:
+    case ENET_PAUR:
+    case ENET_OPD:
+    case ENET_IAUR:
+    case ENET_IALR:
+    case ENET_GAUR:
+    case ENET_GALR:
+    case ENET_TFWR:
+    case ENET_RDSR:
+    case ENET_TDSR:
+    case ENET_MRBR:
+        value = s->regs[index];
+        break;
+    default:
+        if (s->is_fec) {
+            value = imx_fec_read(s, index);
+        } else {
+            value = imx_enet_read(s, index);
+        }
+        break;
+    }
+
+    FEC_PRINTF("reg[%s] => 0x%" PRIx32 "\n", imx_eth_reg_name(s, index),
+                                              value);
+
+    return value;
+}
+
+static void imx_default_write(IMXFECState *s, uint32_t index, uint32_t value)
+{
+    qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad address at offset 0x%"
+                  PRIx32 "\n", TYPE_IMX_FEC, __func__, index * 4);
+    return;
+}
+
+static void imx_fec_write(IMXFECState *s, uint32_t index, uint32_t value)
+{
+    switch (index) {
+    case ENET_FRBR:
+        /* FRBR is read only */
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Register FRBR is read only\n",
+                      TYPE_IMX_FEC, __func__);
+        break;
+    case ENET_FRSR:
+        s->regs[index] = (value & 0x000003fc) | 0x00000400;
+        break;
+    case ENET_MIIGSK_CFGR:
+        s->regs[index] = value & 0x00000053;
+        break;
+    case ENET_MIIGSK_ENR:
+        s->regs[index] = (value & 0x00000002) ? 0x00000006 : 0;
+        break;
+    default:
+        imx_default_write(s, index, value);
+        break;
+    }
+}
+
+static void imx_enet_write(IMXFECState *s, uint32_t index, uint32_t value)
+{
+    switch (index) {
+    case ENET_RSFL:
+    case ENET_RSEM:
+    case ENET_RAEM:
+    case ENET_RAFL:
+    case ENET_TSEM:
+    case ENET_TAEM:
+    case ENET_TAFL:
+        s->regs[index] = value & 0x000001ff;
+        break;
+    case ENET_TIPG:
+        s->regs[index] = value & 0x0000001f;
+        break;
+    case ENET_FTRL:
+        s->regs[index] = value & 0x00003fff;
+        break;
+    case ENET_TACC:
+        s->regs[index] = value & 0x00000019;
+        break;
+    case ENET_RACC:
+        s->regs[index] = value & 0x000000C7;
+        break;
+    case ENET_ATCR:
+        s->regs[index] = value & 0x00002a9d;
+        break;
+    case ENET_ATVR:
+    case ENET_ATOFF:
+    case ENET_ATPER:
+        s->regs[index] = value;
+        break;
+    case ENET_ATSTMP:
+        /* ATSTMP is read only */
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Register ATSTMP is read only\n",
+                      TYPE_IMX_FEC, __func__);
+        break;
+    case ENET_ATCOR:
+        s->regs[index] = value & 0x7fffffff;
+        break;
+    case ENET_ATINC:
+        s->regs[index] = value & 0x00007f7f;
+        break;
+    case ENET_TGSR:
+        /* implement clear timer flag */
+        value = value & 0x0000000f;
+        break;
+    case ENET_TCSR0:
+    case ENET_TCSR1:
+    case ENET_TCSR2:
+    case ENET_TCSR3:
+        value = value & 0x000000fd;
+        break;
+    case ENET_TCCR0:
+    case ENET_TCCR1:
+    case ENET_TCCR2:
+    case ENET_TCCR3:
+        s->regs[index] = value;
+        break;
+    default:
+        imx_default_write(s, index, value);
+        break;
+    }
+}
+
+static void imx_eth_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    IMXFECState *s = IMX_FEC(opaque);
+    uint32_t index = offset >> 2;
+
+    FEC_PRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx_eth_reg_name(s, index),
+                (uint32_t)value);
+
+    switch (index) {
+    case ENET_EIR:
+        s->regs[index] &= ~value;
+        break;
+    case ENET_EIMR:
+        s->regs[index] = value;
+        break;
+    case ENET_RDAR:
+        if (s->regs[ENET_ECR] & ENET_ECR_ETHEREN) {
+            if (!s->regs[index]) {
+                s->regs[index] = ENET_RDAR_RDAR;
+                imx_eth_enable_rx(s);
+            }
+        } else {
+            s->regs[index] = 0;
+        }
+        break;
+    case ENET_TDAR:
+        if (s->regs[ENET_ECR] & ENET_ECR_ETHEREN) {
+            s->regs[index] = ENET_TDAR_TDAR;
+            imx_eth_do_tx(s);
+        }
+        s->regs[index] = 0;
+        break;
+    case ENET_ECR:
+        if (value & ENET_ECR_RESET) {
+            return imx_eth_reset(DEVICE(s));
+        }
+        s->regs[index] = value;
+        if ((s->regs[index] & ENET_ECR_ETHEREN) == 0) {
+            s->regs[ENET_RDAR] = 0;
+            s->rx_descriptor = s->regs[ENET_RDSR];
+            s->regs[ENET_TDAR] = 0;
+            s->tx_descriptor = s->regs[ENET_TDSR];
+        }
+        break;
+    case ENET_MMFR:
+        s->regs[index] = value;
+        if (extract32(value, 29, 1)) {
+            /* This is a read operation */
+            s->regs[ENET_MMFR] = deposit32(s->regs[ENET_MMFR], 0, 16,
+                                           do_phy_read(s,
+                                                       extract32(value,
+                                                                 18, 10)));
+        } else {
+            /* This a write operation */
+            do_phy_write(s, extract32(value, 18, 10), extract32(value, 0, 16));
+        }
+        /* raise the interrupt as the PHY operation is done */
+        s->regs[ENET_EIR] |= ENET_INT_MII;
+        break;
+    case ENET_MSCR:
+        s->regs[index] = value & 0xfe;
+        break;
+    case ENET_MIBC:
+        /* TODO: Implement MIB.  */
+        s->regs[index] = (value & 0x80000000) ? 0xc0000000 : 0;
+        break;
+    case ENET_RCR:
+        s->regs[index] = value & 0x07ff003f;
+        /* TODO: Implement LOOP mode.  */
+        break;
+    case ENET_TCR:
+        /* We transmit immediately, so raise GRA immediately.  */
+        s->regs[index] = value;
+        if (value & 1) {
+            s->regs[ENET_EIR] |= ENET_INT_GRA;
+        }
+        break;
+    case ENET_PALR:
+        s->regs[index] = value;
+        s->conf.macaddr.a[0] = value >> 24;
+        s->conf.macaddr.a[1] = value >> 16;
+        s->conf.macaddr.a[2] = value >> 8;
+        s->conf.macaddr.a[3] = value;
+        break;
+    case ENET_PAUR:
+        s->regs[index] = (value | 0x0000ffff) & 0xffff8808;
+        s->conf.macaddr.a[4] = value >> 24;
+        s->conf.macaddr.a[5] = value >> 16;
+        break;
+    case ENET_OPD:
+        s->regs[index] = (value & 0x0000ffff) | 0x00010000;
+        break;
+    case ENET_IAUR:
+    case ENET_IALR:
+    case ENET_GAUR:
+    case ENET_GALR:
+        /* TODO: implement MAC hash filtering.  */
+        break;
+    case ENET_TFWR:
+        if (s->is_fec) {
+            s->regs[index] = value & 0x3;
+        } else {
+            s->regs[index] = value & 0x13f;
+        }
+        break;
+    case ENET_RDSR:
+        if (s->is_fec) {
+            s->regs[index] = value & ~3;
+        } else {
+            s->regs[index] = value & ~7;
+        }
+        s->rx_descriptor = s->regs[index];
+        break;
+    case ENET_TDSR:
+        if (s->is_fec) {
+            s->regs[index] = value & ~3;
+        } else {
+            s->regs[index] = value & ~7;
+        }
+        s->tx_descriptor = s->regs[index];
+        break;
+    case ENET_MRBR:
+        s->regs[index] = value & 0x00003ff0;
+        break;
+    default:
+        if (s->is_fec) {
+            imx_fec_write(s, index, value);
+        } else {
+            imx_enet_write(s, index, value);
+        }
+        return;
+    }
+
+    imx_eth_update(s);
+}
+
+static int imx_eth_can_receive(NetClientState *nc)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    FEC_PRINTF("\n");
+
+    return s->regs[ENET_RDAR] ? 1 : 0;
+}
+
+static ssize_t imx_fec_receive(NetClientState *nc, const uint8_t *buf,
+                               size_t len)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+    IMXFECBufDesc bd;
+    uint32_t flags = 0;
+    uint32_t addr;
+    uint32_t crc;
+    uint32_t buf_addr;
+    uint8_t *crc_ptr;
+    unsigned int buf_len;
+    size_t size = len;
+
+    FEC_PRINTF("len %d\n", (int)size);
+
+    if (!s->regs[ENET_RDAR]) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Unexpected packet\n",
+                      TYPE_IMX_FEC, __func__);
+        return 0;
+    }
+
+    /* 4 bytes for the CRC.  */
+    size += 4;
+    crc = cpu_to_be32(crc32(~0, buf, size));
+    crc_ptr = (uint8_t *) &crc;
+
+    /* Huge frames are truncated.  */
+    if (size > ENET_MAX_FRAME_SIZE) {
+        size = ENET_MAX_FRAME_SIZE;
+        flags |= ENET_BD_TR | ENET_BD_LG;
+    }
+
+    /* Frames larger than the user limit just set error flags.  */
+    if (size > (s->regs[ENET_RCR] >> 16)) {
+        flags |= ENET_BD_LG;
+    }
+
+    addr = s->rx_descriptor;
+    while (size > 0) {
+        imx_fec_read_bd(&bd, addr);
+        if ((bd.flags & ENET_BD_E) == 0) {
+            /* No descriptors available.  Bail out.  */
+            /*
+             * FIXME: This is wrong. We should probably either
+             * save the remainder for when more RX buffers are
+             * available, or flag an error.
+             */
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Lost end of frame\n",
+                          TYPE_IMX_FEC, __func__);
+            break;
+        }
+        buf_len = (size <= s->regs[ENET_MRBR]) ? size : s->regs[ENET_MRBR];
+        bd.length = buf_len;
+        size -= buf_len;
+
+        FEC_PRINTF("rx_bd 0x%x length %d\n", addr, bd.length);
+
+        /* The last 4 bytes are the CRC.  */
+        if (size < 4) {
+            buf_len += size - 4;
+        }
+        buf_addr = bd.data;
+        dma_memory_write(&address_space_memory, buf_addr, buf, buf_len);
+        buf += buf_len;
+        if (size < 4) {
+            dma_memory_write(&address_space_memory, buf_addr + buf_len,
+                             crc_ptr, 4 - size);
+            crc_ptr += 4 - size;
+        }
+        bd.flags &= ~ENET_BD_E;
+        if (size == 0) {
+            /* Last buffer in frame.  */
+            bd.flags |= flags | ENET_BD_L;
+            FEC_PRINTF("rx frame flags %04x\n", bd.flags);
+            s->regs[ENET_EIR] |= ENET_INT_RXF;
+        } else {
+            s->regs[ENET_EIR] |= ENET_INT_RXB;
+        }
+        imx_fec_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & ENET_BD_W) != 0) {
+            addr = s->regs[ENET_RDSR];
+        } else {
+            addr += sizeof(bd);
+        }
+    }
+    s->rx_descriptor = addr;
+    imx_eth_enable_rx(s);
+    imx_eth_update(s);
+    return len;
+}
+
+static ssize_t imx_enet_receive(NetClientState *nc, const uint8_t *buf,
+                                size_t len)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+    IMXENETBufDesc bd;
+    uint32_t flags = 0;
+    uint32_t addr;
+    uint32_t crc;
+    uint32_t buf_addr;
+    uint8_t *crc_ptr;
+    unsigned int buf_len;
+    size_t size = len;
+
+    FEC_PRINTF("len %d\n", (int)size);
+
+    if (!s->regs[ENET_RDAR]) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Unexpected packet\n",
+                      TYPE_IMX_FEC, __func__);
+        return 0;
+    }
+
+    /* 4 bytes for the CRC.  */
+    size += 4;
+    crc = cpu_to_be32(crc32(~0, buf, size));
+    crc_ptr = (uint8_t *) &crc;
+
+    /* Huge frames are truncted.  */
+    if (size > ENET_MAX_FRAME_SIZE) {
+        size = ENET_MAX_FRAME_SIZE;
+        flags |= ENET_BD_TR | ENET_BD_LG;
+    }
+
+    /* Frames larger than the user limit just set error flags.  */
+    if (size > (s->regs[ENET_RCR] >> 16)) {
+        flags |= ENET_BD_LG;
+    }
+
+    addr = s->rx_descriptor;
+    while (size > 0) {
+        imx_enet_read_bd(&bd, addr);
+        if ((bd.flags & ENET_BD_E) == 0) {
+            /* No descriptors available.  Bail out.  */
+            /*
+             * FIXME: This is wrong. We should probably either
+             * save the remainder for when more RX buffers are
+             * available, or flag an error.
+             */
+            qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Lost end of frame\n",
+                          TYPE_IMX_FEC, __func__);
+            break;
+        }
+        buf_len = (size <= s->regs[ENET_MRBR]) ? size : s->regs[ENET_MRBR];
+        bd.length = buf_len;
+        size -= buf_len;
+
+        FEC_PRINTF("rx_bd 0x%x length %d\n", addr, bd.length);
+
+        /* The last 4 bytes are the CRC.  */
+        if (size < 4) {
+            buf_len += size - 4;
+        }
+        buf_addr = bd.data;
+        dma_memory_write(&address_space_memory, buf_addr, buf, buf_len);
+        buf += buf_len;
+        if (size < 4) {
+            dma_memory_write(&address_space_memory, buf_addr + buf_len,
+                             crc_ptr, 4 - size);
+            crc_ptr += 4 - size;
+        }
+        bd.flags &= ~ENET_BD_E;
+        if (size == 0) {
+            /* Last buffer in frame.  */
+            bd.flags |= flags | ENET_BD_L;
+            FEC_PRINTF("rx frame flags %04x\n", bd.flags);
+            if (bd.option & ENET_BD_RX_INT) {
+                s->regs[ENET_EIR] |= ENET_INT_RXF;
+            }
+        } else {
+            if (bd.option & ENET_BD_RX_INT) {
+                s->regs[ENET_EIR] |= ENET_INT_RXB;
+            }
+        }
+        imx_enet_write_bd(&bd, addr);
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & ENET_BD_W) != 0) {
+            addr = s->regs[ENET_RDSR];
+        } else {
+            addr += sizeof(bd);
+        }
+    }
+    s->rx_descriptor = addr;
+    imx_eth_enable_rx(s);
+    imx_eth_update(s);
+    return len;
+}
+
+static ssize_t imx_eth_receive(NetClientState *nc, const uint8_t *buf,
+                                size_t len)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    if (!s->is_fec && (s->regs[ENET_ECR] & ENET_ECR_EN1588)) {
+        return imx_enet_receive(nc, buf, len);
+    } else {
+        return imx_fec_receive(nc, buf, len);
+    }
+}
+
+static const MemoryRegionOps imx_eth_ops = {
+    .read                  = imx_eth_read,
+    .write                 = imx_eth_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness            = DEVICE_NATIVE_ENDIAN,
+};
+
+static void imx_eth_cleanup(NetClientState *nc)
+{
+    IMXFECState *s = IMX_FEC(qemu_get_nic_opaque(nc));
+
+    s->nic = NULL;
+}
+
+static NetClientInfo imx_eth_net_info = {
+    .type                = NET_CLIENT_DRIVER_NIC,
+    .size                = sizeof(NICState),
+    .can_receive         = imx_eth_can_receive,
+    .receive             = imx_eth_receive,
+    .cleanup             = imx_eth_cleanup,
+    .link_status_changed = imx_eth_set_link,
+};
+
+
+static void imx_eth_realize(DeviceState *dev, Error **errp)
+{
+    IMXFECState *s = IMX_FEC(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx_eth_ops, s,
+                          TYPE_IMX_FEC, 0x400);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq[0]);
+    sysbus_init_irq(sbd, &s->irq[1]);
+
+    qemu_macaddr_default_if_unset(&s->conf.macaddr);
+
+    s->conf.peers.ncs[0] = nd_table[0].netdev;
+
+    s->nic = qemu_new_nic(&imx_eth_net_info, &s->conf,
+                          object_get_typename(OBJECT(dev)),
+                          DEVICE(dev)->id, s);
+
+    qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
+}
+
+static Property imx_eth_properties[] = {
+    DEFINE_NIC_PROPERTIES(IMXFECState, conf),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void imx_eth_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd    = &vmstate_imx_eth;
+    dc->reset   = imx_eth_reset;
+    dc->props   = imx_eth_properties;
+    dc->realize = imx_eth_realize;
+    dc->desc    = "i.MX FEC/ENET Ethernet Controller";
+}
+
+static void imx_fec_init(Object *obj)
+{
+    IMXFECState *s = IMX_FEC(obj);
+
+    s->is_fec = true;
+}
+
+static void imx_enet_init(Object *obj)
+{
+    IMXFECState *s = IMX_FEC(obj);
+
+    s->is_fec = false;
+}
+
+static const TypeInfo imx_fec_info = {
+    .name          = TYPE_IMX_FEC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXFECState),
+    .instance_init = imx_fec_init,
+    .class_init    = imx_eth_class_init,
+};
+
+static const TypeInfo imx_enet_info = {
+    .name          = TYPE_IMX_ENET,
+    .parent        = TYPE_IMX_FEC,
+    .instance_init = imx_enet_init,
+};
+
+static void imx_eth_register_types(void)
+{
+    type_register_static(&imx_fec_info);
+    type_register_static(&imx_enet_info);
+}
+
+type_init(imx_eth_register_types)
diff --git a/hw/net/lan9118.c b/hw/net/lan9118.c
index e528290..4615d87 100644
--- a/hw/net/lan9118.c
+++ b/hw/net/lan9118.c
@@ -10,11 +10,13 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/devices.h"
 #include "sysemu/sysemu.h"
 #include "hw/ptimer.h"
+#include "qemu/log.h"
 /* For crc32 */
 #include <zlib.h>
 
@@ -56,6 +58,8 @@
 #define CSR_E2P_CMD     0xb0
 #define CSR_E2P_DATA    0xb4
 
+#define E2P_CMD_MAC_ADDR_LOADED 0x100
+
 /* IRQ_CFG */
 #define IRQ_INT         0x00001000
 #define IRQ_EN          0x00000100
@@ -352,14 +356,14 @@
 {
     int i;
     if (s->eeprom[0] != 0xa5) {
-        s->e2p_cmd &= ~0x10;
+        s->e2p_cmd &= ~E2P_CMD_MAC_ADDR_LOADED;
         DPRINTF("MACADDR load failed\n");
         return;
     }
     for (i = 0; i < 6; i++) {
         s->conf.macaddr.a[i] = s->eeprom[i + 1];
     }
-    s->e2p_cmd |= 0x10;
+    s->e2p_cmd |= E2P_CMD_MAC_ADDR_LOADED;
     DPRINTF("MACADDR loaded from eeprom\n");
     lan9118_mac_changed(s);
 }
@@ -461,11 +465,6 @@
     lan9118_reload_eeprom(s);
 }
 
-static int lan9118_can_receive(NetClientState *nc)
-{
-    return 1;
-}
-
 static void rx_fifo_push(lan9118_state *s, uint32_t val)
 {
     int fifo_pos;
@@ -907,7 +906,8 @@
          */
         break;
     default:
-        hw_error("lan9118: Unimplemented MAC register write: %d = 0x%x\n",
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "lan9118: Unimplemented MAC register write: %d = 0x%x\n",
                  s->mac_cmd & 0xf, val);
     }
 }
@@ -935,14 +935,16 @@
     case MAC_FLOW:
         return s->mac_flow;
     default:
-        hw_error("lan9118: Unimplemented MAC register read: %d\n",
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "lan9118: Unimplemented MAC register read: %d\n",
                  s->mac_cmd & 0xf);
+        return 0;
     }
 }
 
 static void lan9118_eeprom_cmd(lan9118_state *s, int cmd, int addr)
 {
-    s->e2p_cmd = (s->e2p_cmd & 0x10) | (cmd << 28) | addr;
+    s->e2p_cmd = (s->e2p_cmd & E2P_CMD_MAC_ADDR_LOADED) | (cmd << 28) | addr;
     switch (cmd) {
     case 0:
         s->e2p_data = s->eeprom[addr];
@@ -1133,7 +1135,8 @@
         break;
 
     default:
-        hw_error("lan9118_write: Bad reg 0x%x = %x\n", (int)offset, (int)val);
+        qemu_log_mask(LOG_GUEST_ERROR, "lan9118_write: Bad reg 0x%x = %x\n",
+                      (int)offset, (int)val);
         break;
     }
     lan9118_update(s);
@@ -1251,7 +1254,7 @@
     case CSR_E2P_DATA:
         return s->e2p_data;
     }
-    hw_error("lan9118_read: Bad reg 0x%x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR, "lan9118_read: Bad reg 0x%x\n", (int)offset);
     return 0;
 }
 
@@ -1309,19 +1312,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void lan9118_cleanup(NetClientState *nc)
-{
-    lan9118_state *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_lan9118_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = lan9118_can_receive,
     .receive = lan9118_receive,
-    .cleanup = lan9118_cleanup,
     .link_status_changed = lan9118_set_link,
 };
 
diff --git a/hw/net/lance.c b/hw/net/lance.c
index a1c49f1..573d724 100644
--- a/hw/net/lance.c
+++ b/hw/net/lance.c
@@ -35,6 +35,7 @@
  * http://www.ibiblio.org/pub/historic-linux/early-ports/Sparc/NCR/NCR92C990.txt
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "qemu/timer.h"
@@ -91,20 +92,11 @@
     },
 };
 
-static void lance_cleanup(NetClientState *nc)
-{
-    PCNetState *d = qemu_get_nic_opaque(nc);
-
-    pcnet_common_cleanup(d);
-}
-
 static NetClientInfo net_lance_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = pcnet_can_receive,
     .receive = pcnet_receive,
     .link_status_changed = pcnet_set_link_status,
-    .cleanup = lance_cleanup,
 };
 
 static const VMStateDescription vmstate_lance = {
@@ -134,7 +126,8 @@
 
     s->phys_mem_read = ledma_memory_read;
     s->phys_mem_write = ledma_memory_write;
-    return pcnet_common_init(dev, s, &net_lance_info);
+    pcnet_common_init(dev, s, &net_lance_info);
+    return 0;
 }
 
 static void lance_reset(DeviceState *dev)
diff --git a/hw/net/mcf_fec.c b/hw/net/mcf_fec.c
index 22cd7cf..0ee8ad9 100644
--- a/hw/net/mcf_fec.c
+++ b/hw/net/mcf_fec.c
@@ -5,9 +5,11 @@
  *
  * This code is licensed under the GPL
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "net/net.h"
 #include "hw/m68k/mcf.h"
+#include "hw/net/mii.h"
 /* For crc32 */
 #include <zlib.h>
 #include "exec/address-spaces.h"
@@ -195,12 +197,14 @@
 
 static void mcf_fec_enable_rx(mcf_fec_state *s)
 {
+    NetClientState *nc = qemu_get_queue(s->nic);
     mcf_fec_bd bd;
 
     mcf_fec_read_bd(&bd, s->rx_descriptor);
     s->rx_enabled = ((bd.flags & FEC_BD_E) != 0);
-    if (!s->rx_enabled)
-        DPRINTF("RX buffer full\n");
+    if (s->rx_enabled) {
+        qemu_flush_queued_packets(nc);
+    }
 }
 
 static void mcf_fec_reset(mcf_fec_state *s)
@@ -216,6 +220,51 @@
     s->rfsr = 0x500;
 }
 
+#define MMFR_WRITE_OP	(1 << 28)
+#define MMFR_READ_OP	(2 << 28)
+#define MMFR_PHYADDR(v)	(((v) >> 23) & 0x1f)
+#define MMFR_REGNUM(v)	(((v) >> 18) & 0x1f)
+
+static uint64_t mcf_fec_read_mdio(mcf_fec_state *s)
+{
+    uint64_t v;
+
+    if (s->mmfr & MMFR_WRITE_OP)
+        return s->mmfr;
+    if (MMFR_PHYADDR(s->mmfr) != 1)
+        return s->mmfr |= 0xffff;
+
+    switch (MMFR_REGNUM(s->mmfr)) {
+    case MII_BMCR:
+        v = MII_BMCR_SPEED | MII_BMCR_AUTOEN | MII_BMCR_FD;
+        break;
+    case MII_BMSR:
+        v = MII_BMSR_100TX_FD | MII_BMSR_100TX_HD | MII_BMSR_10T_FD |
+            MII_BMSR_10T_HD | MII_BMSR_MFPS | MII_BMSR_AN_COMP |
+            MII_BMSR_AUTONEG | MII_BMSR_LINK_ST;
+        break;
+    case MII_PHYID1:
+        v = DP83848_PHYID1;
+        break;
+    case MII_PHYID2:
+        v = DP83848_PHYID2;
+        break;
+    case MII_ANAR:
+        v = MII_ANAR_TXFD | MII_ANAR_TX | MII_ANAR_10FD |
+            MII_ANAR_10 | MII_ANAR_CSMACD;
+        break;
+    case MII_ANLPAR:
+        v = MII_ANLPAR_ACK | MII_ANLPAR_TXFD | MII_ANLPAR_TX |
+            MII_ANLPAR_10FD | MII_ANLPAR_10 | MII_ANLPAR_CSMACD;
+        break;
+    default:
+        v = 0xffff;
+        break;
+    }
+    s->mmfr = (s->mmfr & ~0xffff) | v;
+    return s->mmfr;
+}
+
 static uint64_t mcf_fec_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
@@ -226,7 +275,7 @@
     case 0x010: return s->rx_enabled ? (1 << 24) : 0; /* RDAR */
     case 0x014: return 0; /* TDAR */
     case 0x024: return s->ecr;
-    case 0x040: return s->mmfr;
+    case 0x040: return mcf_fec_read_mdio(s);
     case 0x044: return s->mscr;
     case 0x064: return 0; /* MIBC */
     case 0x084: return s->rcr;
@@ -287,8 +336,8 @@
         }
         break;
     case 0x040:
-        /* TODO: Implement MII.  */
         s->mmfr = value;
+        s->eir |= FEC_INT_MII;
         break;
     case 0x044:
         s->mscr = value & 0xfe;
@@ -351,10 +400,30 @@
     mcf_fec_update(s);
 }
 
-static int mcf_fec_can_receive(NetClientState *nc)
+static int mcf_fec_have_receive_space(mcf_fec_state *s, size_t want)
 {
-    mcf_fec_state *s = qemu_get_nic_opaque(nc);
-    return s->rx_enabled;
+    mcf_fec_bd bd;
+    uint32_t addr;
+
+    /* Walk descriptor list to determine if we have enough buffer */
+    addr = s->rx_descriptor;
+    while (want > 0) {
+        mcf_fec_read_bd(&bd, addr);
+        if ((bd.flags & FEC_BD_E) == 0) {
+            return 0;
+        }
+        if (want < s->emrbr) {
+            return 1;
+        }
+        want -= s->emrbr;
+        /* Advance to the next descriptor.  */
+        if ((bd.flags & FEC_BD_W) != 0) {
+            addr = s->erdsr;
+        } else {
+            addr += 8;
+        }
+    }
+    return 0;
 }
 
 static ssize_t mcf_fec_receive(NetClientState *nc, const uint8_t *buf, size_t size)
@@ -367,10 +436,11 @@
     uint32_t buf_addr;
     uint8_t *crc_ptr;
     unsigned int buf_len;
+    size_t retsize;
 
     DPRINTF("do_rx len %d\n", size);
     if (!s->rx_enabled) {
-        fprintf(stderr, "mcf_fec_receive: Unexpected packet\n");
+        return -1;
     }
     /* 4 bytes for the CRC.  */
     size += 4;
@@ -385,17 +455,14 @@
     if (size > (s->rcr >> 16)) {
         flags |= FEC_BD_LG;
     }
+    /* Check if we have enough space in current descriptors */
+    if (!mcf_fec_have_receive_space(s, size)) {
+        return 0;
+    }
     addr = s->rx_descriptor;
+    retsize = size;
     while (size > 0) {
         mcf_fec_read_bd(&bd, addr);
-        if ((bd.flags & FEC_BD_E) == 0) {
-            /* No descriptors available.  Bail out.  */
-            /* FIXME: This is wrong.  We should probably either save the
-               remainder for when more RX buffers are available, or
-               flag an error.  */
-            fprintf(stderr, "mcf_fec: Lost end of frame\n");
-            break;
-        }
         buf_len = (size <= s->emrbr) ? size: s->emrbr;
         bd.length = buf_len;
         size -= buf_len;
@@ -430,7 +497,7 @@
     s->rx_descriptor = addr;
     mcf_fec_enable_rx(s);
     mcf_fec_update(s);
-    return size;
+    return retsize;
 }
 
 static const MemoryRegionOps mcf_fec_ops = {
@@ -439,19 +506,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void mcf_fec_cleanup(NetClientState *nc)
-{
-    mcf_fec_state *s = qemu_get_nic_opaque(nc);
-
-    g_free(s);
-}
-
 static NetClientInfo net_mcf_fec_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = mcf_fec_can_receive,
     .receive = mcf_fec_receive,
-    .cleanup = mcf_fec_cleanup,
 };
 
 void mcf_fec_init(MemoryRegion *sysmem, NICInfo *nd,
diff --git a/hw/net/milkymist-minimac2.c b/hw/net/milkymist-minimac2.c
index c632672..c3a12e1 100644
--- a/hw/net/milkymist-minimac2.c
+++ b/hw/net/milkymist-minimac2.c
@@ -22,6 +22,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h" /* FIXME: why does this use TARGET_PAGE_ALIGN? */
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -303,8 +307,7 @@
         r_state = R_STATE1;
         rx_buf = s->rx1_buf;
     } else {
-        trace_milkymist_minimac2_drop_rx_frame(buf);
-        return size;
+        return 0;
     }
 
     /* assemble frame */
@@ -354,6 +357,18 @@
     return r;
 }
 
+static int minimac2_can_rx(MilkymistMinimac2State *s)
+{
+    if (s->regs[R_STATE0] == STATE_LOADED) {
+        return 1;
+    }
+    if (s->regs[R_STATE1] == STATE_LOADED) {
+        return 1;
+    }
+
+    return 0;
+}
+
 static void
 minimac2_write(void *opaque, hwaddr addr, uint64_t value,
                unsigned size)
@@ -387,6 +402,9 @@
     case R_STATE1:
         s->regs[addr] = value;
         update_rx_interrupt(s);
+        if (minimac2_can_rx(s)) {
+            qemu_flush_queued_packets(qemu_get_queue(s->nic));
+        }
         break;
     case R_SETUP:
     case R_COUNT0:
@@ -411,27 +429,6 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int minimac2_can_rx(NetClientState *nc)
-{
-    MilkymistMinimac2State *s = qemu_get_nic_opaque(nc);
-
-    if (s->regs[R_STATE0] == STATE_LOADED) {
-        return 1;
-    }
-    if (s->regs[R_STATE1] == STATE_LOADED) {
-        return 1;
-    }
-
-    return 0;
-}
-
-static void minimac2_cleanup(NetClientState *nc)
-{
-    MilkymistMinimac2State *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static void milkymist_minimac2_reset(DeviceState *d)
 {
     MilkymistMinimac2State *s = MILKYMIST_MINIMAC2(d);
@@ -450,11 +447,9 @@
 }
 
 static NetClientInfo net_milkymist_minimac2_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = minimac2_can_rx,
     .receive = minimac2_rx,
-    .cleanup = minimac2_cleanup,
 };
 
 static int milkymist_minimac2_init(SysBusDevice *sbd)
@@ -472,7 +467,7 @@
 
     /* register buffers memory */
     memory_region_init_ram(&s->buffers, OBJECT(dev), "milkymist-minimac2.buffers",
-                           buffers_size, &error_abort);
+                           buffers_size, &error_fatal);
     vmstate_register_ram_global(&s->buffers);
     s->rx0_buf = memory_region_get_ram_ptr(&s->buffers);
     s->rx1_buf = s->rx0_buf + MINIMAC2_BUFFER_SIZE;
diff --git a/hw/net/mipsnet.c b/hw/net/mipsnet.c
index b26c369..5a63df7 100644
--- a/hw/net/mipsnet.c
+++ b/hw/net/mipsnet.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "net/net.h"
 #include "trace.h"
@@ -80,8 +81,11 @@
 
     trace_mipsnet_receive(size);
     if (!mipsnet_can_receive(nc))
-        return -1;
+        return 0;
 
+    if (size >= sizeof(s->rx_buffer)) {
+        return 0;
+    }
     s->busy = 1;
 
     /* Just accept everything. */
@@ -134,6 +138,9 @@
         if (s->rx_count) {
             s->rx_count--;
             ret = s->rx_buffer[s->rx_read++];
+            if (mipsnet_can_receive(s->nic->ncs)) {
+                qemu_flush_queued_packets(qemu_get_queue(s->nic));
+            }
         }
         break;
     /* Reads as zero. */
@@ -170,13 +177,18 @@
         }
         s->busy = !!s->intctl;
         mipsnet_update_irq(s);
+        if (mipsnet_can_receive(s->nic->ncs)) {
+            qemu_flush_queued_packets(qemu_get_queue(s->nic));
+        }
         break;
     case MIPSNET_TX_DATA_BUFFER:
         s->tx_buffer[s->tx_written++] = val;
-        if (s->tx_written == s->tx_count) {
+        if ((s->tx_written >= MAX_ETH_FRAME_SIZE)
+            || (s->tx_written == s->tx_count)) {
             /* Send buffer. */
-            trace_mipsnet_send(s->tx_count);
-            qemu_send_packet(qemu_get_queue(s->nic), s->tx_buffer, s->tx_count);
+            trace_mipsnet_send(s->tx_written);
+            qemu_send_packet(qemu_get_queue(s->nic),
+                                s->tx_buffer, s->tx_written);
             s->tx_count = s->tx_written = 0;
             s->intctl |= MIPSNET_INTCTL_TXDONE;
             s->busy = 1;
@@ -211,19 +223,10 @@
     }
 };
 
-static void mipsnet_cleanup(NetClientState *nc)
-{
-    MIPSnetState *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_mipsnet_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = mipsnet_can_receive,
     .receive = mipsnet_receive,
-    .cleanup = mipsnet_cleanup,
 };
 
 static const MemoryRegionOps mipsnet_ioport_ops = {
diff --git a/hw/net/ne2000-isa.c b/hw/net/ne2000-isa.c
index 82e2ba1..f345533 100644
--- a/hw/net/ne2000-isa.c
+++ b/hw/net/ne2000-isa.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -28,6 +29,7 @@
 #include "net/net.h"
 #include "ne2000.h"
 #include "exec/address-spaces.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 #define TYPE_ISA_NE2000 "ne2k_isa"
@@ -41,19 +43,10 @@
     NE2000State ne2000;
 } ISANE2000State;
 
-static void isa_ne2000_cleanup(NetClientState *nc)
-{
-    NE2000State *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_ne2000_isa_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = ne2000_can_receive,
     .receive = ne2000_receive,
-    .cleanup = isa_ne2000_cleanup,
 };
 
 static const VMStateDescription vmstate_isa_ne2000 = {
@@ -102,24 +95,26 @@
     set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
 }
 
-static void isa_ne2000_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void isa_ne2000_get_bootindex(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     ISANE2000State *isa = ISA_NE2000(obj);
     NE2000State *s = &isa->ne2000;
 
-    visit_type_int32(v, &s->c.bootindex, name, errp);
+    visit_type_int32(v, name, &s->c.bootindex, errp);
 }
 
-static void isa_ne2000_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void isa_ne2000_set_bootindex(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     ISANE2000State *isa = ISA_NE2000(obj);
     NE2000State *s = &isa->ne2000;
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
@@ -132,9 +127,7 @@
     s->c.bootindex = boot_index;
 
 out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
 static void isa_ne2000_instance_init(Object *obj)
diff --git a/hw/net/ne2000.c b/hw/net/ne2000.c
index 3ab2d03..798d681 100644
--- a/hw/net/ne2000.c
+++ b/hw/net/ne2000.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -154,6 +155,10 @@
 {
     int avail, index, boundary;
 
+    if (s->stop <= s->start) {
+        return 1;
+    }
+
     index = s->curpag << 8;
     boundary = s->boundary << 8;
     if (index < boundary)
@@ -165,15 +170,6 @@
     return 0;
 }
 
-int ne2000_can_receive(NetClientState *nc)
-{
-    NE2000State *s = qemu_get_nic_opaque(nc);
-
-    if (s->cmd & E8390_STOP)
-        return 1;
-    return !ne2000_buffer_full(s);
-}
-
 #define MIN_BUF_SIZE 60
 
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
@@ -230,6 +226,9 @@
     }
 
     index = s->curpag << 8;
+    if (index >= NE2000_PMEM_END) {
+        index = s->start;
+    }
     /* 4 bytes for header */
     total_len = size + 4;
     /* address for next packet (4 bytes for CRC) */
@@ -253,7 +252,7 @@
         if (index <= s->stop)
             avail = s->stop - index;
         else
-            avail = 0;
+            break;
         len = size;
         if (len > avail)
             len = avail;
@@ -315,13 +314,19 @@
         offset = addr | (page << 4);
         switch(offset) {
         case EN0_STARTPG:
-            s->start = val << 8;
+            if (val << 8 <= NE2000_PMEM_END) {
+                s->start = val << 8;
+            }
             break;
         case EN0_STOPPG:
-            s->stop = val << 8;
+            if (val << 8 <= NE2000_PMEM_END) {
+                s->stop = val << 8;
+            }
             break;
         case EN0_BOUNDARY:
-            s->boundary = val;
+            if (val << 8 < NE2000_PMEM_END) {
+                s->boundary = val;
+            }
             break;
         case EN0_IMR:
             s->imr = val;
@@ -362,7 +367,9 @@
             s->phys[offset - EN1_PHYS] = val;
             break;
         case EN1_CURPAG:
-            s->curpag = val;
+            if (val << 8 < NE2000_PMEM_END) {
+                s->curpag = val;
+            }
             break;
         case EN1_MULT ... EN1_MULT + 7:
             s->mult[offset - EN1_MULT] = val;
@@ -465,8 +472,9 @@
                                      uint32_t val)
 {
     addr &= ~1; /* XXX: check exact behaviour if not even */
-    if (addr < 32 ||
-        (addr >= NE2000_PMEM_START && addr < NE2000_MEM_SIZE)) {
+    if (addr < 32
+        || (addr >= NE2000_PMEM_START
+            && addr + sizeof(uint32_t) <= NE2000_MEM_SIZE)) {
         stl_le_p(s->mem + addr, val);
     }
 }
@@ -495,8 +503,9 @@
 static inline uint32_t ne2000_mem_readl(NE2000State *s, uint32_t addr)
 {
     addr &= ~1; /* XXX: check exact behaviour if not even */
-    if (addr < 32 ||
-        (addr >= NE2000_PMEM_START && addr < NE2000_MEM_SIZE)) {
+    if (addr < 32
+        || (addr >= NE2000_PMEM_START
+            && addr + sizeof(uint32_t) <= NE2000_MEM_SIZE)) {
         return ldl_le_p(s->mem + addr);
     } else {
         return 0xffffffff;
@@ -702,22 +711,13 @@
     memory_region_init_io(&s->io, OBJECT(dev), &ne2000_ops, s, "ne2000", size);
 }
 
-static void ne2000_cleanup(NetClientState *nc)
-{
-    NE2000State *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_ne2000_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = ne2000_can_receive,
     .receive = ne2000_receive,
-    .cleanup = ne2000_cleanup,
 };
 
-static int pci_ne2000_init(PCIDevice *pci_dev)
+static void pci_ne2000_realize(PCIDevice *pci_dev, Error **errp)
 {
     PCINE2000State *d = DO_UPCAST(PCINE2000State, dev, pci_dev);
     NE2000State *s;
@@ -737,8 +737,6 @@
     s->nic = qemu_new_nic(&net_ne2000_info, &s->c,
                           object_get_typename(OBJECT(pci_dev)), pci_dev->qdev.id, s);
     qemu_format_nic_info_str(qemu_get_queue(s->nic), s->c.macaddr.a);
-
-    return 0;
 }
 
 static void pci_ne2000_exit(PCIDevice *pci_dev)
@@ -771,7 +769,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ne2000_init;
+    k->realize = pci_ne2000_realize;
     k->exit = pci_ne2000_exit;
     k->romfile = "efi-ne2k_pci.rom",
     k->vendor_id = PCI_VENDOR_ID_REALTEK;
diff --git a/hw/net/ne2000.h b/hw/net/ne2000.h
index e500306..d213dcc 100644
--- a/hw/net/ne2000.h
+++ b/hw/net/ne2000.h
@@ -1,5 +1,5 @@
 #ifndef HW_NE2000_H
-#define HW_NE2000_H 1
+#define HW_NE2000_H
 
 #define NE2000_PMEM_SIZE    (32*1024)
 #define NE2000_PMEM_START   (16*1024)
@@ -34,7 +34,6 @@
 void ne2000_setup_io(NE2000State *s, DeviceState *dev, unsigned size);
 extern const VMStateDescription vmstate_ne2000;
 void ne2000_reset(NE2000State *s);
-int ne2000_can_receive(NetClientState *nc);
 ssize_t ne2000_receive(NetClientState *nc, const uint8_t *buf, size_t size_);
 
 #endif
diff --git a/hw/net/net_rx_pkt.c b/hw/net/net_rx_pkt.c
new file mode 100644
index 0000000..1019b50
--- /dev/null
+++ b/hw/net/net_rx_pkt.c
@@ -0,0 +1,600 @@
+/*
+ * QEMU RX packets abstractions
+ *
+ * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Dmitry Fleytman <dmitry@daynix.com>
+ * Tamir Shomer <tamirs@daynix.com>
+ * Yan Vugenfirer <yan@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "trace.h"
+#include "net_rx_pkt.h"
+#include "net/checksum.h"
+#include "net/tap.h"
+
+struct NetRxPkt {
+    struct virtio_net_hdr virt_hdr;
+    uint8_t ehdr_buf[sizeof(struct eth_header)];
+    struct iovec *vec;
+    uint16_t vec_len_total;
+    uint16_t vec_len;
+    uint32_t tot_len;
+    uint16_t tci;
+    bool vlan_stripped;
+    bool has_virt_hdr;
+    eth_pkt_types_e packet_type;
+
+    /* Analysis results */
+    bool isip4;
+    bool isip6;
+    bool isudp;
+    bool istcp;
+
+    size_t l3hdr_off;
+    size_t l4hdr_off;
+    size_t l5hdr_off;
+
+    eth_ip6_hdr_info ip6hdr_info;
+    eth_ip4_hdr_info ip4hdr_info;
+    eth_l4_hdr_info  l4hdr_info;
+};
+
+void net_rx_pkt_init(struct NetRxPkt **pkt, bool has_virt_hdr)
+{
+    struct NetRxPkt *p = g_malloc0(sizeof *p);
+    p->has_virt_hdr = has_virt_hdr;
+    p->vec = NULL;
+    p->vec_len_total = 0;
+    *pkt = p;
+}
+
+void net_rx_pkt_uninit(struct NetRxPkt *pkt)
+{
+    if (pkt->vec_len_total != 0) {
+        g_free(pkt->vec);
+    }
+
+    g_free(pkt);
+}
+
+struct virtio_net_hdr *net_rx_pkt_get_vhdr(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+    return &pkt->virt_hdr;
+}
+
+static inline void
+net_rx_pkt_iovec_realloc(struct NetRxPkt *pkt,
+                            int new_iov_len)
+{
+    if (pkt->vec_len_total < new_iov_len) {
+        g_free(pkt->vec);
+        pkt->vec = g_malloc(sizeof(*pkt->vec) * new_iov_len);
+        pkt->vec_len_total = new_iov_len;
+    }
+}
+
+static void
+net_rx_pkt_pull_data(struct NetRxPkt *pkt,
+                        const struct iovec *iov, int iovcnt,
+                        size_t ploff)
+{
+    if (pkt->vlan_stripped) {
+        net_rx_pkt_iovec_realloc(pkt, iovcnt + 1);
+
+        pkt->vec[0].iov_base = pkt->ehdr_buf;
+        pkt->vec[0].iov_len = sizeof(pkt->ehdr_buf);
+
+        pkt->tot_len =
+            iov_size(iov, iovcnt) - ploff + sizeof(struct eth_header);
+
+        pkt->vec_len = iov_copy(pkt->vec + 1, pkt->vec_len_total - 1,
+                                iov, iovcnt, ploff, pkt->tot_len);
+    } else {
+        net_rx_pkt_iovec_realloc(pkt, iovcnt);
+
+        pkt->tot_len = iov_size(iov, iovcnt) - ploff;
+        pkt->vec_len = iov_copy(pkt->vec, pkt->vec_len_total,
+                                iov, iovcnt, ploff, pkt->tot_len);
+    }
+
+    eth_get_protocols(pkt->vec, pkt->vec_len, &pkt->isip4, &pkt->isip6,
+                      &pkt->isudp, &pkt->istcp,
+                      &pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off,
+                      &pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info);
+
+    trace_net_rx_pkt_parsed(pkt->isip4, pkt->isip6, pkt->isudp, pkt->istcp,
+                            pkt->l3hdr_off, pkt->l4hdr_off, pkt->l5hdr_off);
+}
+
+void net_rx_pkt_attach_iovec(struct NetRxPkt *pkt,
+                                const struct iovec *iov, int iovcnt,
+                                size_t iovoff, bool strip_vlan)
+{
+    uint16_t tci = 0;
+    uint16_t ploff = iovoff;
+    assert(pkt);
+    pkt->vlan_stripped = false;
+
+    if (strip_vlan) {
+        pkt->vlan_stripped = eth_strip_vlan(iov, iovcnt, iovoff, pkt->ehdr_buf,
+                                            &ploff, &tci);
+    }
+
+    pkt->tci = tci;
+
+    net_rx_pkt_pull_data(pkt, iov, iovcnt, ploff);
+}
+
+void net_rx_pkt_attach_iovec_ex(struct NetRxPkt *pkt,
+                                const struct iovec *iov, int iovcnt,
+                                size_t iovoff, bool strip_vlan,
+                                uint16_t vet)
+{
+    uint16_t tci = 0;
+    uint16_t ploff = iovoff;
+    assert(pkt);
+    pkt->vlan_stripped = false;
+
+    if (strip_vlan) {
+        pkt->vlan_stripped = eth_strip_vlan_ex(iov, iovcnt, iovoff, vet,
+                                               pkt->ehdr_buf,
+                                               &ploff, &tci);
+    }
+
+    pkt->tci = tci;
+
+    net_rx_pkt_pull_data(pkt, iov, iovcnt, ploff);
+}
+
+void net_rx_pkt_dump(struct NetRxPkt *pkt)
+{
+#ifdef NET_RX_PKT_DEBUG
+    NetRxPkt *pkt = (NetRxPkt *)pkt;
+    assert(pkt);
+
+    printf("RX PKT: tot_len: %d, vlan_stripped: %d, vlan_tag: %d\n",
+              pkt->tot_len, pkt->vlan_stripped, pkt->tci);
+#endif
+}
+
+void net_rx_pkt_set_packet_type(struct NetRxPkt *pkt,
+    eth_pkt_types_e packet_type)
+{
+    assert(pkt);
+
+    pkt->packet_type = packet_type;
+
+}
+
+eth_pkt_types_e net_rx_pkt_get_packet_type(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->packet_type;
+}
+
+size_t net_rx_pkt_get_total_len(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->tot_len;
+}
+
+void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data,
+                              size_t len)
+{
+    const struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = len
+    };
+
+    assert(pkt);
+
+    eth_get_protocols(&iov, 1, &pkt->isip4, &pkt->isip6,
+                      &pkt->isudp, &pkt->istcp,
+                      &pkt->l3hdr_off, &pkt->l4hdr_off, &pkt->l5hdr_off,
+                      &pkt->ip6hdr_info, &pkt->ip4hdr_info, &pkt->l4hdr_info);
+}
+
+void net_rx_pkt_get_protocols(struct NetRxPkt *pkt,
+                              bool *isip4, bool *isip6,
+                              bool *isudp, bool *istcp)
+{
+    assert(pkt);
+
+    *isip4 = pkt->isip4;
+    *isip6 = pkt->isip6;
+    *isudp = pkt->isudp;
+    *istcp = pkt->istcp;
+}
+
+size_t net_rx_pkt_get_l3_hdr_offset(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+    return pkt->l3hdr_off;
+}
+
+size_t net_rx_pkt_get_l4_hdr_offset(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+    return pkt->l4hdr_off;
+}
+
+size_t net_rx_pkt_get_l5_hdr_offset(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+    return pkt->l5hdr_off;
+}
+
+eth_ip6_hdr_info *net_rx_pkt_get_ip6_info(struct NetRxPkt *pkt)
+{
+    return &pkt->ip6hdr_info;
+}
+
+eth_ip4_hdr_info *net_rx_pkt_get_ip4_info(struct NetRxPkt *pkt)
+{
+    return &pkt->ip4hdr_info;
+}
+
+eth_l4_hdr_info *net_rx_pkt_get_l4_info(struct NetRxPkt *pkt)
+{
+    return &pkt->l4hdr_info;
+}
+
+static inline void
+_net_rx_rss_add_chunk(uint8_t *rss_input, size_t *bytes_written,
+                      void *ptr, size_t size)
+{
+    memcpy(&rss_input[*bytes_written], ptr, size);
+    trace_net_rx_pkt_rss_add_chunk(ptr, size, *bytes_written);
+    *bytes_written += size;
+}
+
+static inline void
+_net_rx_rss_prepare_ip4(uint8_t *rss_input,
+                        struct NetRxPkt *pkt,
+                        size_t *bytes_written)
+{
+    struct ip_header *ip4_hdr = &pkt->ip4hdr_info.ip4_hdr;
+
+    _net_rx_rss_add_chunk(rss_input, bytes_written,
+                          &ip4_hdr->ip_src, sizeof(uint32_t));
+
+    _net_rx_rss_add_chunk(rss_input, bytes_written,
+                          &ip4_hdr->ip_dst, sizeof(uint32_t));
+}
+
+static inline void
+_net_rx_rss_prepare_ip6(uint8_t *rss_input,
+                        struct NetRxPkt *pkt,
+                        bool ipv6ex, size_t *bytes_written)
+{
+    eth_ip6_hdr_info *ip6info = &pkt->ip6hdr_info;
+
+    _net_rx_rss_add_chunk(rss_input, bytes_written,
+           (ipv6ex && ip6info->rss_ex_src_valid) ? &ip6info->rss_ex_src
+                                                 : &ip6info->ip6_hdr.ip6_src,
+           sizeof(struct in6_address));
+
+    _net_rx_rss_add_chunk(rss_input, bytes_written,
+           (ipv6ex && ip6info->rss_ex_dst_valid) ? &ip6info->rss_ex_dst
+                                                 : &ip6info->ip6_hdr.ip6_dst,
+           sizeof(struct in6_address));
+}
+
+static inline void
+_net_rx_rss_prepare_tcp(uint8_t *rss_input,
+                        struct NetRxPkt *pkt,
+                        size_t *bytes_written)
+{
+    struct tcp_header *tcphdr = &pkt->l4hdr_info.hdr.tcp;
+
+    _net_rx_rss_add_chunk(rss_input, bytes_written,
+                          &tcphdr->th_sport, sizeof(uint16_t));
+
+    _net_rx_rss_add_chunk(rss_input, bytes_written,
+                          &tcphdr->th_dport, sizeof(uint16_t));
+}
+
+uint32_t
+net_rx_pkt_calc_rss_hash(struct NetRxPkt *pkt,
+                         NetRxPktRssType type,
+                         uint8_t *key)
+{
+    uint8_t rss_input[36];
+    size_t rss_length = 0;
+    uint32_t rss_hash = 0;
+    net_toeplitz_key key_data;
+
+    switch (type) {
+    case NetPktRssIpV4:
+        assert(pkt->isip4);
+        trace_net_rx_pkt_rss_ip4();
+        _net_rx_rss_prepare_ip4(&rss_input[0], pkt, &rss_length);
+        break;
+    case NetPktRssIpV4Tcp:
+        assert(pkt->isip4);
+        assert(pkt->istcp);
+        trace_net_rx_pkt_rss_ip4_tcp();
+        _net_rx_rss_prepare_ip4(&rss_input[0], pkt, &rss_length);
+        _net_rx_rss_prepare_tcp(&rss_input[0], pkt, &rss_length);
+        break;
+    case NetPktRssIpV6Tcp:
+        assert(pkt->isip6);
+        assert(pkt->istcp);
+        trace_net_rx_pkt_rss_ip6_tcp();
+        _net_rx_rss_prepare_ip6(&rss_input[0], pkt, true, &rss_length);
+        _net_rx_rss_prepare_tcp(&rss_input[0], pkt, &rss_length);
+        break;
+    case NetPktRssIpV6:
+        assert(pkt->isip6);
+        trace_net_rx_pkt_rss_ip6();
+        _net_rx_rss_prepare_ip6(&rss_input[0], pkt, false, &rss_length);
+        break;
+    case NetPktRssIpV6Ex:
+        assert(pkt->isip6);
+        trace_net_rx_pkt_rss_ip6_ex();
+        _net_rx_rss_prepare_ip6(&rss_input[0], pkt, true, &rss_length);
+        break;
+    default:
+        assert(false);
+        break;
+    }
+
+    net_toeplitz_key_init(&key_data, key);
+    net_toeplitz_add(&rss_hash, rss_input, rss_length, &key_data);
+
+    trace_net_rx_pkt_rss_hash(rss_length, rss_hash);
+
+    return rss_hash;
+}
+
+uint16_t net_rx_pkt_get_ip_id(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    if (pkt->isip4) {
+        return be16_to_cpu(pkt->ip4hdr_info.ip4_hdr.ip_id);
+    }
+
+    return 0;
+}
+
+bool net_rx_pkt_is_tcp_ack(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    if (pkt->istcp) {
+        return TCP_HEADER_FLAGS(&pkt->l4hdr_info.hdr.tcp) & TCP_FLAG_ACK;
+    }
+
+    return false;
+}
+
+bool net_rx_pkt_has_tcp_data(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    if (pkt->istcp) {
+        return pkt->l4hdr_info.has_tcp_data;
+    }
+
+    return false;
+}
+
+struct iovec *net_rx_pkt_get_iovec(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->vec;
+}
+
+uint16_t net_rx_pkt_get_iovec_len(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->vec_len;
+}
+
+void net_rx_pkt_set_vhdr(struct NetRxPkt *pkt,
+                            struct virtio_net_hdr *vhdr)
+{
+    assert(pkt);
+
+    memcpy(&pkt->virt_hdr, vhdr, sizeof pkt->virt_hdr);
+}
+
+void net_rx_pkt_set_vhdr_iovec(struct NetRxPkt *pkt,
+    const struct iovec *iov, int iovcnt)
+{
+    assert(pkt);
+
+    iov_to_buf(iov, iovcnt, 0, &pkt->virt_hdr, sizeof pkt->virt_hdr);
+}
+
+bool net_rx_pkt_is_vlan_stripped(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->vlan_stripped;
+}
+
+bool net_rx_pkt_has_virt_hdr(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->has_virt_hdr;
+}
+
+uint16_t net_rx_pkt_get_vlan_tag(struct NetRxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->tci;
+}
+
+bool net_rx_pkt_validate_l3_csum(struct NetRxPkt *pkt, bool *csum_valid)
+{
+    uint32_t cntr;
+    uint16_t csum;
+    uint32_t csl;
+
+    trace_net_rx_pkt_l3_csum_validate_entry();
+
+    if (!pkt->isip4) {
+        trace_net_rx_pkt_l3_csum_validate_not_ip4();
+        return false;
+    }
+
+    csl = pkt->l4hdr_off - pkt->l3hdr_off;
+
+    cntr = net_checksum_add_iov(pkt->vec, pkt->vec_len,
+                                pkt->l3hdr_off,
+                                csl, 0);
+
+    csum = net_checksum_finish(cntr);
+
+    *csum_valid = (csum == 0);
+
+    trace_net_rx_pkt_l3_csum_validate_csum(pkt->l3hdr_off, csl,
+                                           cntr, csum, *csum_valid);
+
+    return true;
+}
+
+static uint16_t
+_net_rx_pkt_calc_l4_csum(struct NetRxPkt *pkt)
+{
+    uint32_t cntr;
+    uint16_t csum;
+    uint16_t csl;
+    uint32_t cso;
+
+    trace_net_rx_pkt_l4_csum_calc_entry();
+
+    if (pkt->isip4) {
+        if (pkt->isudp) {
+            csl = be16_to_cpu(pkt->l4hdr_info.hdr.udp.uh_ulen);
+            trace_net_rx_pkt_l4_csum_calc_ip4_udp();
+        } else {
+            csl = be16_to_cpu(pkt->ip4hdr_info.ip4_hdr.ip_len) -
+                  IP_HDR_GET_LEN(&pkt->ip4hdr_info.ip4_hdr);
+            trace_net_rx_pkt_l4_csum_calc_ip4_tcp();
+        }
+
+        cntr = eth_calc_ip4_pseudo_hdr_csum(&pkt->ip4hdr_info.ip4_hdr,
+                                            csl, &cso);
+        trace_net_rx_pkt_l4_csum_calc_ph_csum(cntr, csl);
+    } else {
+        if (pkt->isudp) {
+            csl = be16_to_cpu(pkt->l4hdr_info.hdr.udp.uh_ulen);
+            trace_net_rx_pkt_l4_csum_calc_ip6_udp();
+        } else {
+            struct ip6_header *ip6hdr = &pkt->ip6hdr_info.ip6_hdr;
+            size_t full_ip6hdr_len = pkt->l4hdr_off - pkt->l3hdr_off;
+            size_t ip6opts_len = full_ip6hdr_len - sizeof(struct ip6_header);
+
+            csl = be16_to_cpu(ip6hdr->ip6_ctlun.ip6_un1.ip6_un1_plen) -
+                  ip6opts_len;
+            trace_net_rx_pkt_l4_csum_calc_ip6_tcp();
+        }
+
+        cntr = eth_calc_ip6_pseudo_hdr_csum(&pkt->ip6hdr_info.ip6_hdr, csl,
+                                            pkt->ip6hdr_info.l4proto, &cso);
+        trace_net_rx_pkt_l4_csum_calc_ph_csum(cntr, csl);
+    }
+
+    cntr += net_checksum_add_iov(pkt->vec, pkt->vec_len,
+                                 pkt->l4hdr_off, csl, cso);
+
+    csum = net_checksum_finish(cntr);
+
+    trace_net_rx_pkt_l4_csum_calc_csum(pkt->l4hdr_off, csl, cntr, csum);
+
+    return csum;
+}
+
+bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid)
+{
+    uint16_t csum;
+
+    trace_net_rx_pkt_l4_csum_validate_entry();
+
+    if (!pkt->istcp && !pkt->isudp) {
+        trace_net_rx_pkt_l4_csum_validate_not_xxp();
+        return false;
+    }
+
+    if (pkt->isudp && (pkt->l4hdr_info.hdr.udp.uh_sum == 0)) {
+        trace_net_rx_pkt_l4_csum_validate_udp_with_no_checksum();
+        return false;
+    }
+
+    if (pkt->isip4 && pkt->ip4hdr_info.fragment) {
+        trace_net_rx_pkt_l4_csum_validate_ip4_fragment();
+        return false;
+    }
+
+    csum = _net_rx_pkt_calc_l4_csum(pkt);
+
+    *csum_valid = ((csum == 0) || (csum == 0xFFFF));
+
+    trace_net_rx_pkt_l4_csum_validate_csum(*csum_valid);
+
+    return true;
+}
+
+bool net_rx_pkt_fix_l4_csum(struct NetRxPkt *pkt)
+{
+    uint16_t csum = 0;
+    uint32_t l4_cso;
+
+    trace_net_rx_pkt_l4_csum_fix_entry();
+
+    if (pkt->istcp) {
+        l4_cso = offsetof(struct tcp_header, th_sum);
+        trace_net_rx_pkt_l4_csum_fix_tcp(l4_cso);
+    } else if (pkt->isudp) {
+        if (pkt->l4hdr_info.hdr.udp.uh_sum == 0) {
+            trace_net_rx_pkt_l4_csum_fix_udp_with_no_checksum();
+            return false;
+        }
+        l4_cso = offsetof(struct udp_header, uh_sum);
+        trace_net_rx_pkt_l4_csum_fix_udp(l4_cso);
+    } else {
+        trace_net_rx_pkt_l4_csum_fix_not_xxp();
+        return false;
+    }
+
+    if (pkt->isip4 && pkt->ip4hdr_info.fragment) {
+            trace_net_rx_pkt_l4_csum_fix_ip4_fragment();
+            return false;
+    }
+
+    /* Set zero to checksum word */
+    iov_from_buf(pkt->vec, pkt->vec_len,
+                 pkt->l4hdr_off + l4_cso,
+                 &csum, sizeof(csum));
+
+    /* Calculate L4 checksum */
+    csum = cpu_to_be16(_net_rx_pkt_calc_l4_csum(pkt));
+
+    /* Set calculated checksum to checksum word */
+    iov_from_buf(pkt->vec, pkt->vec_len,
+                 pkt->l4hdr_off + l4_cso,
+                 &csum, sizeof(csum));
+
+    trace_net_rx_pkt_l4_csum_fix_csum(pkt->l4hdr_off + l4_cso, csum);
+
+    return true;
+}
diff --git a/hw/net/net_rx_pkt.h b/hw/net/net_rx_pkt.h
new file mode 100644
index 0000000..7adf0fa
--- /dev/null
+++ b/hw/net/net_rx_pkt.h
@@ -0,0 +1,363 @@
+/*
+ * QEMU RX packets abstraction
+ *
+ * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Dmitry Fleytman <dmitry@daynix.com>
+ * Tamir Shomer <tamirs@daynix.com>
+ * Yan Vugenfirer <yan@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef NET_RX_PKT_H
+#define NET_RX_PKT_H
+
+#include "net/eth.h"
+
+/* defines to enable packet dump functions */
+/*#define NET_RX_PKT_DEBUG*/
+
+struct NetRxPkt;
+
+/**
+ * Clean all rx packet resources
+ *
+ * @pkt:            packet
+ *
+ */
+void net_rx_pkt_uninit(struct NetRxPkt *pkt);
+
+/**
+ * Init function for rx packet functionality
+ *
+ * @pkt:            packet pointer
+ * @has_virt_hdr:   device uses virtio header
+ *
+ */
+void net_rx_pkt_init(struct NetRxPkt **pkt, bool has_virt_hdr);
+
+/**
+ * returns total length of data attached to rx context
+ *
+ * @pkt:            packet
+ *
+ * Return:  nothing
+ *
+ */
+size_t net_rx_pkt_get_total_len(struct NetRxPkt *pkt);
+
+/**
+ * parse and set packet analysis results
+ *
+ * @pkt:            packet
+ * @data:           pointer to the data buffer to be parsed
+ * @len:            data length
+ *
+ */
+void net_rx_pkt_set_protocols(struct NetRxPkt *pkt, const void *data,
+                              size_t len);
+
+/**
+ * fetches packet analysis results
+ *
+ * @pkt:            packet
+ * @isip4:          whether the packet given is IPv4
+ * @isip6:          whether the packet given is IPv6
+ * @isudp:          whether the packet given is UDP
+ * @istcp:          whether the packet given is TCP
+ *
+ */
+void net_rx_pkt_get_protocols(struct NetRxPkt *pkt,
+                                 bool *isip4, bool *isip6,
+                                 bool *isudp, bool *istcp);
+
+/**
+* fetches L3 header offset
+*
+* @pkt:            packet
+*
+*/
+size_t net_rx_pkt_get_l3_hdr_offset(struct NetRxPkt *pkt);
+
+/**
+* fetches L4 header offset
+*
+* @pkt:            packet
+*
+*/
+size_t net_rx_pkt_get_l4_hdr_offset(struct NetRxPkt *pkt);
+
+/**
+* fetches L5 header offset
+*
+* @pkt:            packet
+*
+*/
+size_t net_rx_pkt_get_l5_hdr_offset(struct NetRxPkt *pkt);
+
+/**
+ * fetches IP6 header analysis results
+ *
+ * Return:  pointer to analysis results structure which is stored in internal
+ *          packet area.
+ *
+ */
+eth_ip6_hdr_info *net_rx_pkt_get_ip6_info(struct NetRxPkt *pkt);
+
+/**
+ * fetches IP4 header analysis results
+ *
+ * Return:  pointer to analysis results structure which is stored in internal
+ *          packet area.
+ *
+ */
+eth_ip4_hdr_info *net_rx_pkt_get_ip4_info(struct NetRxPkt *pkt);
+
+/**
+ * fetches L4 header analysis results
+ *
+ * Return:  pointer to analysis results structure which is stored in internal
+ *          packet area.
+ *
+ */
+eth_l4_hdr_info *net_rx_pkt_get_l4_info(struct NetRxPkt *pkt);
+
+typedef enum {
+    NetPktRssIpV4,
+    NetPktRssIpV4Tcp,
+    NetPktRssIpV6Tcp,
+    NetPktRssIpV6,
+    NetPktRssIpV6Ex
+} NetRxPktRssType;
+
+/**
+* calculates RSS hash for packet
+*
+* @pkt:            packet
+* @type:           RSS hash type
+*
+* Return:  Toeplitz RSS hash.
+*
+*/
+uint32_t
+net_rx_pkt_calc_rss_hash(struct NetRxPkt *pkt,
+                         NetRxPktRssType type,
+                         uint8_t *key);
+
+/**
+* fetches IP identification for the packet
+*
+* @pkt:            packet
+*
+*/
+uint16_t net_rx_pkt_get_ip_id(struct NetRxPkt *pkt);
+
+/**
+* check if given packet is a TCP ACK packet
+*
+* @pkt:            packet
+*
+*/
+bool net_rx_pkt_is_tcp_ack(struct NetRxPkt *pkt);
+
+/**
+* check if given packet contains TCP data
+*
+* @pkt:            packet
+*
+*/
+bool net_rx_pkt_has_tcp_data(struct NetRxPkt *pkt);
+
+/**
+ * returns virtio header stored in rx context
+ *
+ * @pkt:            packet
+ * @ret:            virtio header
+ *
+ */
+struct virtio_net_hdr *net_rx_pkt_get_vhdr(struct NetRxPkt *pkt);
+
+/**
+ * returns packet type
+ *
+ * @pkt:            packet
+ * @ret:            packet type
+ *
+ */
+eth_pkt_types_e net_rx_pkt_get_packet_type(struct NetRxPkt *pkt);
+
+/**
+ * returns vlan tag
+ *
+ * @pkt:            packet
+ * @ret:            VLAN tag
+ *
+ */
+uint16_t net_rx_pkt_get_vlan_tag(struct NetRxPkt *pkt);
+
+/**
+ * tells whether vlan was stripped from the packet
+ *
+ * @pkt:            packet
+ * @ret:            VLAN stripped sign
+ *
+ */
+bool net_rx_pkt_is_vlan_stripped(struct NetRxPkt *pkt);
+
+/**
+ * notifies caller if the packet has virtio header
+ *
+ * @pkt:            packet
+ * @ret:            true if packet has virtio header, false otherwize
+ *
+ */
+bool net_rx_pkt_has_virt_hdr(struct NetRxPkt *pkt);
+
+/**
+* attach scatter-gather data to rx packet
+*
+* @pkt:            packet
+* @iov:            received data scatter-gather list
+* @iovcnt          number of elements in iov
+* @iovoff          data start offset in the iov
+* @strip_vlan:     should the module strip vlan from data
+*
+*/
+void net_rx_pkt_attach_iovec(struct NetRxPkt *pkt,
+                                const struct iovec *iov,
+                                int iovcnt, size_t iovoff,
+                                bool strip_vlan);
+
+/**
+* attach scatter-gather data to rx packet
+*
+* @pkt:            packet
+* @iov:            received data scatter-gather list
+* @iovcnt          number of elements in iov
+* @iovoff          data start offset in the iov
+* @strip_vlan:     should the module strip vlan from data
+* @vet:            VLAN tag Ethernet type
+*
+*/
+void net_rx_pkt_attach_iovec_ex(struct NetRxPkt *pkt,
+                                   const struct iovec *iov, int iovcnt,
+                                   size_t iovoff, bool strip_vlan,
+                                   uint16_t vet);
+
+/**
+ * attach data to rx packet
+ *
+ * @pkt:            packet
+ * @data:           pointer to the data buffer
+ * @len:            data length
+ * @strip_vlan:     should the module strip vlan from data
+ *
+ */
+static inline void
+net_rx_pkt_attach_data(struct NetRxPkt *pkt, const void *data,
+                          size_t len, bool strip_vlan)
+{
+    const struct iovec iov = {
+        .iov_base = (void *) data,
+        .iov_len = len
+    };
+
+    net_rx_pkt_attach_iovec(pkt, &iov, 1, 0, strip_vlan);
+}
+
+/**
+ * returns io vector that holds the attached data
+ *
+ * @pkt:            packet
+ * @ret:            pointer to IOVec
+ *
+ */
+struct iovec *net_rx_pkt_get_iovec(struct NetRxPkt *pkt);
+
+/**
+* returns io vector length that holds the attached data
+*
+* @pkt:            packet
+* @ret:            IOVec length
+*
+*/
+uint16_t net_rx_pkt_get_iovec_len(struct NetRxPkt *pkt);
+
+/**
+ * prints rx packet data if debug is enabled
+ *
+ * @pkt:            packet
+ *
+ */
+void net_rx_pkt_dump(struct NetRxPkt *pkt);
+
+/**
+ * copy passed vhdr data to packet context
+ *
+ * @pkt:            packet
+ * @vhdr:           VHDR buffer
+ *
+ */
+void net_rx_pkt_set_vhdr(struct NetRxPkt *pkt,
+    struct virtio_net_hdr *vhdr);
+
+/**
+* copy passed vhdr data to packet context
+*
+* @pkt:            packet
+* @iov:            VHDR iov
+* @iovcnt:         VHDR iov array size
+*
+*/
+void net_rx_pkt_set_vhdr_iovec(struct NetRxPkt *pkt,
+    const struct iovec *iov, int iovcnt);
+
+/**
+ * save packet type in packet context
+ *
+ * @pkt:            packet
+ * @packet_type:    the packet type
+ *
+ */
+void net_rx_pkt_set_packet_type(struct NetRxPkt *pkt,
+    eth_pkt_types_e packet_type);
+
+/**
+* validate TCP/UDP checksum of the packet
+*
+* @pkt:            packet
+* @csum_valid:     checksum validation result
+* @ret:            true if validation was performed, false in case packet is
+*                  not TCP/UDP or checksum validation is not possible
+*
+*/
+bool net_rx_pkt_validate_l4_csum(struct NetRxPkt *pkt, bool *csum_valid);
+
+/**
+* validate IPv4 checksum of the packet
+*
+* @pkt:            packet
+* @csum_valid:     checksum validation result
+* @ret:            true if validation was performed, false in case packet is
+*                  not TCP/UDP or checksum validation is not possible
+*
+*/
+bool net_rx_pkt_validate_l3_csum(struct NetRxPkt *pkt, bool *csum_valid);
+
+/**
+* fix IPv4 checksum of the packet
+*
+* @pkt:            packet
+* @ret:            true if checksum was fixed, false in case packet is
+*                  not TCP/UDP or checksum correction is not possible
+*
+*/
+bool net_rx_pkt_fix_l4_csum(struct NetRxPkt *pkt);
+
+#endif
diff --git a/hw/net/net_tx_pkt.c b/hw/net/net_tx_pkt.c
new file mode 100644
index 0000000..20b2549
--- /dev/null
+++ b/hw/net/net_tx_pkt.c
@@ -0,0 +1,637 @@
+/*
+ * QEMU TX packets abstractions
+ *
+ * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Dmitry Fleytman <dmitry@daynix.com>
+ * Tamir Shomer <tamirs@daynix.com>
+ * Yan Vugenfirer <yan@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "net_tx_pkt.h"
+#include "net/eth.h"
+#include "net/checksum.h"
+#include "net/tap.h"
+#include "net/net.h"
+#include "hw/pci/pci.h"
+
+enum {
+    NET_TX_PKT_VHDR_FRAG = 0,
+    NET_TX_PKT_L2HDR_FRAG,
+    NET_TX_PKT_L3HDR_FRAG,
+    NET_TX_PKT_PL_START_FRAG
+};
+
+/* TX packet private context */
+struct NetTxPkt {
+    PCIDevice *pci_dev;
+
+    struct virtio_net_hdr virt_hdr;
+    bool has_virt_hdr;
+
+    struct iovec *raw;
+    uint32_t raw_frags;
+    uint32_t max_raw_frags;
+
+    struct iovec *vec;
+
+    uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN];
+    uint8_t l3_hdr[ETH_MAX_IP_DGRAM_LEN];
+
+    uint32_t payload_len;
+
+    uint32_t payload_frags;
+    uint32_t max_payload_frags;
+
+    uint16_t hdr_len;
+    eth_pkt_types_e packet_type;
+    uint8_t l4proto;
+
+    bool is_loopback;
+};
+
+void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev,
+    uint32_t max_frags, bool has_virt_hdr)
+{
+    struct NetTxPkt *p = g_malloc0(sizeof *p);
+
+    p->pci_dev = pci_dev;
+
+    p->vec = g_new(struct iovec, max_frags + NET_TX_PKT_PL_START_FRAG);
+
+    p->raw = g_new(struct iovec, max_frags);
+
+    p->max_payload_frags = max_frags;
+    p->max_raw_frags = max_frags;
+    p->has_virt_hdr = has_virt_hdr;
+    p->vec[NET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr;
+    p->vec[NET_TX_PKT_VHDR_FRAG].iov_len =
+        p->has_virt_hdr ? sizeof p->virt_hdr : 0;
+    p->vec[NET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr;
+    p->vec[NET_TX_PKT_L3HDR_FRAG].iov_base = &p->l3_hdr;
+
+    *pkt = p;
+}
+
+void net_tx_pkt_uninit(struct NetTxPkt *pkt)
+{
+    if (pkt) {
+        g_free(pkt->vec);
+        g_free(pkt->raw);
+        g_free(pkt);
+    }
+}
+
+void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt)
+{
+    uint16_t csum;
+    assert(pkt);
+    struct ip_header *ip_hdr;
+    ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
+
+    ip_hdr->ip_len = cpu_to_be16(pkt->payload_len +
+        pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
+
+    ip_hdr->ip_sum = 0;
+    csum = net_raw_checksum((uint8_t *)ip_hdr,
+        pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len);
+    ip_hdr->ip_sum = cpu_to_be16(csum);
+}
+
+void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt)
+{
+    uint16_t csum;
+    uint32_t cntr, cso;
+    assert(pkt);
+    uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
+    void *ip_hdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
+
+    if (pkt->payload_len + pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len >
+        ETH_MAX_IP_DGRAM_LEN) {
+        return;
+    }
+
+    if (gso_type == VIRTIO_NET_HDR_GSO_TCPV4 ||
+        gso_type == VIRTIO_NET_HDR_GSO_UDP) {
+        /* Calculate IP header checksum */
+        net_tx_pkt_update_ip_hdr_checksum(pkt);
+
+        /* Calculate IP pseudo header checksum */
+        cntr = eth_calc_ip4_pseudo_hdr_csum(ip_hdr, pkt->payload_len, &cso);
+        csum = cpu_to_be16(~net_checksum_finish(cntr));
+    } else if (gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
+        /* Calculate IP pseudo header checksum */
+        cntr = eth_calc_ip6_pseudo_hdr_csum(ip_hdr, pkt->payload_len,
+                                            IP_PROTO_TCP, &cso);
+        csum = cpu_to_be16(~net_checksum_finish(cntr));
+    } else {
+        return;
+    }
+
+    iov_from_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
+                 pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
+}
+
+static void net_tx_pkt_calculate_hdr_len(struct NetTxPkt *pkt)
+{
+    pkt->hdr_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +
+        pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
+}
+
+static bool net_tx_pkt_parse_headers(struct NetTxPkt *pkt)
+{
+    struct iovec *l2_hdr, *l3_hdr;
+    size_t bytes_read;
+    size_t full_ip6hdr_len;
+    uint16_t l3_proto;
+
+    assert(pkt);
+
+    l2_hdr = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
+    l3_hdr = &pkt->vec[NET_TX_PKT_L3HDR_FRAG];
+
+    bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
+                            ETH_MAX_L2_HDR_LEN);
+    if (bytes_read < sizeof(struct eth_header)) {
+        l2_hdr->iov_len = 0;
+        return false;
+    }
+
+    l2_hdr->iov_len = sizeof(struct eth_header);
+    switch (be16_to_cpu(PKT_GET_ETH_HDR(l2_hdr->iov_base)->h_proto)) {
+    case ETH_P_VLAN:
+        l2_hdr->iov_len += sizeof(struct vlan_header);
+        break;
+    case ETH_P_DVLAN:
+        l2_hdr->iov_len += 2 * sizeof(struct vlan_header);
+        break;
+    }
+
+    if (bytes_read < l2_hdr->iov_len) {
+        l2_hdr->iov_len = 0;
+        l3_hdr->iov_len = 0;
+        pkt->packet_type = ETH_PKT_UCAST;
+        return false;
+    } else {
+        l2_hdr->iov_len = ETH_MAX_L2_HDR_LEN;
+        l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
+        pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base);
+    }
+
+    l3_proto = eth_get_l3_proto(l2_hdr, 1, l2_hdr->iov_len);
+
+    switch (l3_proto) {
+    case ETH_P_IP:
+        bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
+                                l3_hdr->iov_base, sizeof(struct ip_header));
+
+        if (bytes_read < sizeof(struct ip_header)) {
+            l3_hdr->iov_len = 0;
+            return false;
+        }
+
+        l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base);
+
+        if (l3_hdr->iov_len < sizeof(struct ip_header)) {
+            l3_hdr->iov_len = 0;
+            return false;
+        }
+
+        pkt->l4proto = ((struct ip_header *) l3_hdr->iov_base)->ip_p;
+
+        if (IP_HDR_GET_LEN(l3_hdr->iov_base) != sizeof(struct ip_header)) {
+            /* copy optional IPv4 header data if any*/
+            bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags,
+                                    l2_hdr->iov_len + sizeof(struct ip_header),
+                                    l3_hdr->iov_base + sizeof(struct ip_header),
+                                    l3_hdr->iov_len - sizeof(struct ip_header));
+            if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) {
+                l3_hdr->iov_len = 0;
+                return false;
+            }
+        }
+
+        break;
+
+    case ETH_P_IPV6:
+    {
+        eth_ip6_hdr_info hdrinfo;
+
+        if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
+                                &hdrinfo)) {
+            l3_hdr->iov_len = 0;
+            return false;
+        }
+
+        pkt->l4proto = hdrinfo.l4proto;
+        full_ip6hdr_len = hdrinfo.full_hdr_len;
+
+        if (full_ip6hdr_len > ETH_MAX_IP_DGRAM_LEN) {
+            l3_hdr->iov_len = 0;
+            return false;
+        }
+
+        bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
+                                l3_hdr->iov_base, full_ip6hdr_len);
+
+        if (bytes_read < full_ip6hdr_len) {
+            l3_hdr->iov_len = 0;
+            return false;
+        } else {
+            l3_hdr->iov_len = full_ip6hdr_len;
+        }
+        break;
+    }
+    default:
+        l3_hdr->iov_len = 0;
+        break;
+    }
+
+    net_tx_pkt_calculate_hdr_len(pkt);
+    return true;
+}
+
+static void net_tx_pkt_rebuild_payload(struct NetTxPkt *pkt)
+{
+    pkt->payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len;
+    pkt->payload_frags = iov_copy(&pkt->vec[NET_TX_PKT_PL_START_FRAG],
+                                pkt->max_payload_frags,
+                                pkt->raw, pkt->raw_frags,
+                                pkt->hdr_len, pkt->payload_len);
+}
+
+bool net_tx_pkt_parse(struct NetTxPkt *pkt)
+{
+    if (net_tx_pkt_parse_headers(pkt)) {
+        net_tx_pkt_rebuild_payload(pkt);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt)
+{
+    assert(pkt);
+    return &pkt->virt_hdr;
+}
+
+static uint8_t net_tx_pkt_get_gso_type(struct NetTxPkt *pkt,
+                                          bool tso_enable)
+{
+    uint8_t rc = VIRTIO_NET_HDR_GSO_NONE;
+    uint16_t l3_proto;
+
+    l3_proto = eth_get_l3_proto(&pkt->vec[NET_TX_PKT_L2HDR_FRAG], 1,
+        pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len);
+
+    if (!tso_enable) {
+        goto func_exit;
+    }
+
+    rc = eth_get_gso_type(l3_proto, pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base,
+                          pkt->l4proto);
+
+func_exit:
+    return rc;
+}
+
+void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable,
+    bool csum_enable, uint32_t gso_size)
+{
+    struct tcp_hdr l4hdr;
+    assert(pkt);
+
+    /* csum has to be enabled if tso is. */
+    assert(csum_enable || !tso_enable);
+
+    pkt->virt_hdr.gso_type = net_tx_pkt_get_gso_type(pkt, tso_enable);
+
+    switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
+    case VIRTIO_NET_HDR_GSO_NONE:
+        pkt->virt_hdr.hdr_len = 0;
+        pkt->virt_hdr.gso_size = 0;
+        break;
+
+    case VIRTIO_NET_HDR_GSO_UDP:
+        pkt->virt_hdr.gso_size = gso_size;
+        pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header);
+        break;
+
+    case VIRTIO_NET_HDR_GSO_TCPV4:
+    case VIRTIO_NET_HDR_GSO_TCPV6:
+        iov_to_buf(&pkt->vec[NET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
+                   0, &l4hdr, sizeof(l4hdr));
+        pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t);
+        pkt->virt_hdr.gso_size = gso_size;
+        break;
+
+    default:
+        g_assert_not_reached();
+    }
+
+    if (csum_enable) {
+        switch (pkt->l4proto) {
+        case IP_PROTO_TCP:
+            pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+            pkt->virt_hdr.csum_start = pkt->hdr_len;
+            pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum);
+            break;
+        case IP_PROTO_UDP:
+            pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
+            pkt->virt_hdr.csum_start = pkt->hdr_len;
+            pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum);
+            break;
+        default:
+            break;
+        }
+    }
+}
+
+void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
+    uint16_t vlan, uint16_t vlan_ethtype)
+{
+    bool is_new;
+    assert(pkt);
+
+    eth_setup_vlan_headers_ex(pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base,
+        vlan, vlan_ethtype, &is_new);
+
+    /* update l2hdrlen */
+    if (is_new) {
+        pkt->hdr_len += sizeof(struct vlan_header);
+        pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len +=
+            sizeof(struct vlan_header);
+    }
+}
+
+bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa,
+    size_t len)
+{
+    hwaddr mapped_len = 0;
+    struct iovec *ventry;
+    assert(pkt);
+    assert(pkt->max_raw_frags > pkt->raw_frags);
+
+    if (!len) {
+        return true;
+     }
+
+    ventry = &pkt->raw[pkt->raw_frags];
+    mapped_len = len;
+
+    ventry->iov_base = pci_dma_map(pkt->pci_dev, pa,
+                                   &mapped_len, DMA_DIRECTION_TO_DEVICE);
+
+    if ((ventry->iov_base != NULL) && (len == mapped_len)) {
+        ventry->iov_len = mapped_len;
+        pkt->raw_frags++;
+        return true;
+    } else {
+        return false;
+    }
+}
+
+bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt)
+{
+    return pkt->raw_frags > 0;
+}
+
+eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->packet_type;
+}
+
+size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt)
+{
+    assert(pkt);
+
+    return pkt->hdr_len + pkt->payload_len;
+}
+
+void net_tx_pkt_dump(struct NetTxPkt *pkt)
+{
+#ifdef NET_TX_PKT_DEBUG
+    assert(pkt);
+
+    printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, "
+        "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type,
+        pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len,
+        pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len);
+#endif
+}
+
+void net_tx_pkt_reset(struct NetTxPkt *pkt)
+{
+    int i;
+
+    /* no assert, as reset can be called before tx_pkt_init */
+    if (!pkt) {
+        return;
+    }
+
+    memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr));
+
+    assert(pkt->vec);
+
+    pkt->payload_len = 0;
+    pkt->payload_frags = 0;
+
+    assert(pkt->raw);
+    for (i = 0; i < pkt->raw_frags; i++) {
+        assert(pkt->raw[i].iov_base);
+        pci_dma_unmap(pkt->pci_dev, pkt->raw[i].iov_base, pkt->raw[i].iov_len,
+                      DMA_DIRECTION_TO_DEVICE, 0);
+    }
+    pkt->raw_frags = 0;
+
+    pkt->hdr_len = 0;
+    pkt->l4proto = 0;
+}
+
+static void net_tx_pkt_do_sw_csum(struct NetTxPkt *pkt)
+{
+    struct iovec *iov = &pkt->vec[NET_TX_PKT_L2HDR_FRAG];
+    uint32_t csum_cntr;
+    uint16_t csum = 0;
+    uint32_t cso;
+    /* num of iovec without vhdr */
+    uint32_t iov_len = pkt->payload_frags + NET_TX_PKT_PL_START_FRAG - 1;
+    uint16_t csl;
+    struct ip_header *iphdr;
+    size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset;
+
+    /* Put zero to checksum field */
+    iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
+
+    /* Calculate L4 TCP/UDP checksum */
+    csl = pkt->payload_len;
+
+    /* add pseudo header to csum */
+    iphdr = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
+    csum_cntr = eth_calc_ip4_pseudo_hdr_csum(iphdr, csl, &cso);
+
+    /* data checksum */
+    csum_cntr +=
+        net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl, cso);
+
+    /* Put the checksum obtained into the packet */
+    csum = cpu_to_be16(net_checksum_finish(csum_cntr));
+    iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
+}
+
+enum {
+    NET_TX_PKT_FRAGMENT_L2_HDR_POS = 0,
+    NET_TX_PKT_FRAGMENT_L3_HDR_POS,
+    NET_TX_PKT_FRAGMENT_HEADER_NUM
+};
+
+#define NET_MAX_FRAG_SG_LIST (64)
+
+static size_t net_tx_pkt_fetch_fragment(struct NetTxPkt *pkt,
+    int *src_idx, size_t *src_offset, struct iovec *dst, int *dst_idx)
+{
+    size_t fetched = 0;
+    struct iovec *src = pkt->vec;
+
+    *dst_idx = NET_TX_PKT_FRAGMENT_HEADER_NUM;
+
+    while (fetched < IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size)) {
+
+        /* no more place in fragment iov */
+        if (*dst_idx == NET_MAX_FRAG_SG_LIST) {
+            break;
+        }
+
+        /* no more data in iovec */
+        if (*src_idx == (pkt->payload_frags + NET_TX_PKT_PL_START_FRAG)) {
+            break;
+        }
+
+
+        dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset;
+        dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset,
+            IP_FRAG_ALIGN_SIZE(pkt->virt_hdr.gso_size) - fetched);
+
+        *src_offset += dst[*dst_idx].iov_len;
+        fetched += dst[*dst_idx].iov_len;
+
+        if (*src_offset == src[*src_idx].iov_len) {
+            *src_offset = 0;
+            (*src_idx)++;
+        }
+
+        (*dst_idx)++;
+    }
+
+    return fetched;
+}
+
+static inline void net_tx_pkt_sendv(struct NetTxPkt *pkt,
+    NetClientState *nc, const struct iovec *iov, int iov_cnt)
+{
+    if (pkt->is_loopback) {
+        nc->info->receive_iov(nc, iov, iov_cnt);
+    } else {
+        qemu_sendv_packet(nc, iov, iov_cnt);
+    }
+}
+
+static bool net_tx_pkt_do_sw_fragmentation(struct NetTxPkt *pkt,
+    NetClientState *nc)
+{
+    struct iovec fragment[NET_MAX_FRAG_SG_LIST];
+    size_t fragment_len = 0;
+    bool more_frags = false;
+
+    /* some pointers for shorter code */
+    void *l2_iov_base, *l3_iov_base;
+    size_t l2_iov_len, l3_iov_len;
+    int src_idx =  NET_TX_PKT_PL_START_FRAG, dst_idx;
+    size_t src_offset = 0;
+    size_t fragment_offset = 0;
+
+    l2_iov_base = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_base;
+    l2_iov_len = pkt->vec[NET_TX_PKT_L2HDR_FRAG].iov_len;
+    l3_iov_base = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_base;
+    l3_iov_len = pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len;
+
+    /* Copy headers */
+    fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_base = l2_iov_base;
+    fragment[NET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_len = l2_iov_len;
+    fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_base = l3_iov_base;
+    fragment[NET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_len = l3_iov_len;
+
+
+    /* Put as much data as possible and send */
+    do {
+        fragment_len = net_tx_pkt_fetch_fragment(pkt, &src_idx, &src_offset,
+            fragment, &dst_idx);
+
+        more_frags = (fragment_offset + fragment_len < pkt->payload_len);
+
+        eth_setup_ip4_fragmentation(l2_iov_base, l2_iov_len, l3_iov_base,
+            l3_iov_len, fragment_len, fragment_offset, more_frags);
+
+        eth_fix_ip4_checksum(l3_iov_base, l3_iov_len);
+
+        net_tx_pkt_sendv(pkt, nc, fragment, dst_idx);
+
+        fragment_offset += fragment_len;
+
+    } while (fragment_len && more_frags);
+
+    return true;
+}
+
+bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc)
+{
+    assert(pkt);
+
+    if (!pkt->has_virt_hdr &&
+        pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
+        net_tx_pkt_do_sw_csum(pkt);
+    }
+
+    /*
+     * Since underlying infrastructure does not support IP datagrams longer
+     * than 64K we should drop such packets and don't even try to send
+     */
+    if (VIRTIO_NET_HDR_GSO_NONE != pkt->virt_hdr.gso_type) {
+        if (pkt->payload_len >
+            ETH_MAX_IP_DGRAM_LEN -
+            pkt->vec[NET_TX_PKT_L3HDR_FRAG].iov_len) {
+            return false;
+        }
+    }
+
+    if (pkt->has_virt_hdr ||
+        pkt->virt_hdr.gso_type == VIRTIO_NET_HDR_GSO_NONE) {
+        net_tx_pkt_sendv(pkt, nc, pkt->vec,
+            pkt->payload_frags + NET_TX_PKT_PL_START_FRAG);
+        return true;
+    }
+
+    return net_tx_pkt_do_sw_fragmentation(pkt, nc);
+}
+
+bool net_tx_pkt_send_loopback(struct NetTxPkt *pkt, NetClientState *nc)
+{
+    bool res;
+
+    pkt->is_loopback = true;
+    res = net_tx_pkt_send(pkt, nc);
+    pkt->is_loopback = false;
+
+    return res;
+}
diff --git a/hw/net/net_tx_pkt.h b/hw/net/net_tx_pkt.h
new file mode 100644
index 0000000..212ecc6
--- /dev/null
+++ b/hw/net/net_tx_pkt.h
@@ -0,0 +1,190 @@
+/*
+ * QEMU TX packets abstraction
+ *
+ * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
+ *
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Dmitry Fleytman <dmitry@daynix.com>
+ * Tamir Shomer <tamirs@daynix.com>
+ * Yan Vugenfirer <yan@daynix.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef NET_TX_PKT_H
+#define NET_TX_PKT_H
+
+#include "net/eth.h"
+#include "exec/hwaddr.h"
+
+/* define to enable packet dump functions */
+/*#define NET_TX_PKT_DEBUG*/
+
+struct NetTxPkt;
+
+/**
+ * Init function for tx packet functionality
+ *
+ * @pkt:            packet pointer
+ * @pci_dev:        PCI device processing this packet
+ * @max_frags:      max tx ip fragments
+ * @has_virt_hdr:   device uses virtio header.
+ */
+void net_tx_pkt_init(struct NetTxPkt **pkt, PCIDevice *pci_dev,
+    uint32_t max_frags, bool has_virt_hdr);
+
+/**
+ * Clean all tx packet resources.
+ *
+ * @pkt:            packet.
+ */
+void net_tx_pkt_uninit(struct NetTxPkt *pkt);
+
+/**
+ * get virtio header
+ *
+ * @pkt:            packet
+ * @ret:            virtio header
+ */
+struct virtio_net_hdr *net_tx_pkt_get_vhdr(struct NetTxPkt *pkt);
+
+/**
+ * build virtio header (will be stored in module context)
+ *
+ * @pkt:            packet
+ * @tso_enable:     TSO enabled
+ * @csum_enable:    CSO enabled
+ * @gso_size:       MSS size for TSO
+ *
+ */
+void net_tx_pkt_build_vheader(struct NetTxPkt *pkt, bool tso_enable,
+    bool csum_enable, uint32_t gso_size);
+
+/**
+* updates vlan tag, and adds vlan header with custom ethernet type
+* in case it is missing.
+*
+* @pkt:            packet
+* @vlan:           VLAN tag
+* @vlan_ethtype:   VLAN header Ethernet type
+*
+*/
+void net_tx_pkt_setup_vlan_header_ex(struct NetTxPkt *pkt,
+    uint16_t vlan, uint16_t vlan_ethtype);
+
+/**
+* updates vlan tag, and adds vlan header in case it is missing
+*
+* @pkt:            packet
+* @vlan:           VLAN tag
+*
+*/
+static inline void
+net_tx_pkt_setup_vlan_header(struct NetTxPkt *pkt, uint16_t vlan)
+{
+    net_tx_pkt_setup_vlan_header_ex(pkt, vlan, ETH_P_VLAN);
+}
+
+/**
+ * populate data fragment into pkt context.
+ *
+ * @pkt:            packet
+ * @pa:             physical address of fragment
+ * @len:            length of fragment
+ *
+ */
+bool net_tx_pkt_add_raw_fragment(struct NetTxPkt *pkt, hwaddr pa,
+    size_t len);
+
+/**
+ * Fix ip header fields and calculate IP header and pseudo header checksums.
+ *
+ * @pkt:            packet
+ *
+ */
+void net_tx_pkt_update_ip_checksums(struct NetTxPkt *pkt);
+
+/**
+ * Calculate the IP header checksum.
+ *
+ * @pkt:            packet
+ *
+ */
+void net_tx_pkt_update_ip_hdr_checksum(struct NetTxPkt *pkt);
+
+/**
+ * get length of all populated data.
+ *
+ * @pkt:            packet
+ * @ret:            total data length
+ *
+ */
+size_t net_tx_pkt_get_total_len(struct NetTxPkt *pkt);
+
+/**
+ * get packet type
+ *
+ * @pkt:            packet
+ * @ret:            packet type
+ *
+ */
+eth_pkt_types_e net_tx_pkt_get_packet_type(struct NetTxPkt *pkt);
+
+/**
+ * prints packet data if debug is enabled
+ *
+ * @pkt:            packet
+ *
+ */
+void net_tx_pkt_dump(struct NetTxPkt *pkt);
+
+/**
+ * reset tx packet private context (needed to be called between packets)
+ *
+ * @pkt:            packet
+ *
+ */
+void net_tx_pkt_reset(struct NetTxPkt *pkt);
+
+/**
+ * Send packet to qemu. handles sw offloads if vhdr is not supported.
+ *
+ * @pkt:            packet
+ * @nc:             NetClientState
+ * @ret:            operation result
+ *
+ */
+bool net_tx_pkt_send(struct NetTxPkt *pkt, NetClientState *nc);
+
+/**
+* Redirect packet directly to receive path (emulate loopback phy).
+* Handles sw offloads if vhdr is not supported.
+*
+* @pkt:            packet
+* @nc:             NetClientState
+* @ret:            operation result
+*
+*/
+bool net_tx_pkt_send_loopback(struct NetTxPkt *pkt, NetClientState *nc);
+
+/**
+ * parse raw packet data and analyze offload requirements.
+ *
+ * @pkt:            packet
+ *
+ */
+bool net_tx_pkt_parse(struct NetTxPkt *pkt);
+
+/**
+* indicates if there are data fragments held by this packet object.
+*
+* @pkt:            packet
+*
+*/
+bool net_tx_pkt_has_fragments(struct NetTxPkt *pkt);
+
+#endif
diff --git a/hw/net/opencores_eth.c b/hw/net/opencores_eth.c
index 4a44304..268d6a7 100644
--- a/hw/net/opencores_eth.c
+++ b/hw/net/opencores_eth.c
@@ -31,7 +31,9 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "hw/net/mii.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "sysemu/sysemu.h"
@@ -54,12 +56,6 @@
 
 /* PHY MII registers */
 enum {
-    MII_BMCR,
-    MII_BMSR,
-    MII_PHYIDR1,
-    MII_PHYIDR2,
-    MII_ANAR,
-    MII_ANLPAR,
     MII_REG_MAX = 16,
 };
 
@@ -71,10 +67,11 @@
 static void mii_set_link(Mii *s, bool link_ok)
 {
     if (link_ok) {
-        s->regs[MII_BMSR] |= 0x4;
-        s->regs[MII_ANLPAR] |= 0x01e1;
+        s->regs[MII_BMSR] |= MII_BMSR_LINK_ST;
+        s->regs[MII_ANLPAR] |= MII_ANLPAR_TXFD | MII_ANLPAR_TX |
+            MII_ANLPAR_10FD | MII_ANLPAR_10 | MII_ANLPAR_CSMACD;
     } else {
-        s->regs[MII_BMSR] &= ~0x4;
+        s->regs[MII_BMSR] &= ~MII_BMSR_LINK_ST;
         s->regs[MII_ANLPAR] &= 0x01ff;
     }
     s->link_ok = link_ok;
@@ -83,11 +80,14 @@
 static void mii_reset(Mii *s)
 {
     memset(s->regs, 0, sizeof(s->regs));
-    s->regs[MII_BMCR] = 0x1000;
-    s->regs[MII_BMSR] = 0x7848; /* no ext regs */
-    s->regs[MII_PHYIDR1] = 0x2000;
-    s->regs[MII_PHYIDR2] = 0x5c90;
-    s->regs[MII_ANAR] = 0x01e1;
+    s->regs[MII_BMCR] = MII_BMCR_AUTOEN;
+    s->regs[MII_BMSR] = MII_BMSR_100TX_FD | MII_BMSR_100TX_HD |
+        MII_BMSR_10T_FD | MII_BMSR_10T_HD | MII_BMSR_MFPS |
+        MII_BMSR_AN_COMP | MII_BMSR_AUTONEG;
+    s->regs[MII_PHYID1] = 0x2000;
+    s->regs[MII_PHYID2] = 0x5c90;
+    s->regs[MII_ANAR] = MII_ANAR_TXFD | MII_ANAR_TX |
+        MII_ANAR_10FD | MII_ANAR_10 | MII_ANAR_CSMACD;
     mii_set_link(s, s->link_ok);
 }
 
@@ -97,7 +97,7 @@
 
 static void mii_write_bmcr(Mii *s, uint16_t v)
 {
-    if (v & 0x8000) {
+    if (v & MII_BMCR_RESET) {
         mii_reset(s);
     } else {
         s->regs[MII_BMCR] = v;
@@ -109,8 +109,8 @@
     static void (*reg_write[MII_REG_MAX])(Mii *s, uint16_t v) = {
         [MII_BMCR] = mii_write_bmcr,
         [MII_BMSR] = mii_ro,
-        [MII_PHYIDR1] = mii_ro,
-        [MII_PHYIDR2] = mii_ro,
+        [MII_PHYID1] = mii_ro,
+        [MII_PHYID2] = mii_ro,
     };
 
     if (idx < MII_REG_MAX) {
@@ -472,22 +472,18 @@
     return size;
 }
 
-static void open_eth_cleanup(NetClientState *nc)
-{
-}
-
 static NetClientInfo net_open_eth_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = open_eth_can_receive,
     .receive = open_eth_receive,
-    .cleanup = open_eth_cleanup,
     .link_status_changed = open_eth_set_link_status,
 };
 
 static void open_eth_start_xmit(OpenEthState *s, desc *tx)
 {
-    uint8_t buf[65536];
+    uint8_t *buf = NULL;
+    uint8_t buffer[0x600];
     unsigned len = GET_FIELD(tx->len_flags, TXD_LEN);
     unsigned tx_len = len;
 
@@ -502,6 +498,11 @@
 
     trace_open_eth_start_xmit(tx->buf_ptr, len, tx_len);
 
+    if (tx_len > sizeof(buffer)) {
+        buf = g_new(uint8_t, tx_len);
+    } else {
+        buf = buffer;
+    }
     if (len > tx_len) {
         len = tx_len;
     }
@@ -510,6 +511,9 @@
         memset(buf + len, 0, tx_len - len);
     }
     qemu_send_packet(qemu_get_queue(s->nic), buf, tx_len);
+    if (tx_len > sizeof(buffer)) {
+        g_free(buf);
+    }
 
     if (tx->len_flags & TXD_WR) {
         s->tx_desc = 0;
diff --git a/hw/net/pcnet-pci.c b/hw/net/pcnet-pci.c
index fb5f5d6..0acf8a4 100644
--- a/hw/net/pcnet-pci.c
+++ b/hw/net/pcnet-pci.c
@@ -27,12 +27,14 @@
  * AMD Publication# 19436  Rev:E  Amendment/0  Issue Date: June 2000
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
 #include "hw/loader.h"
 #include "qemu/timer.h"
 #include "sysemu/dma.h"
 #include "sysemu/sysemu.h"
+#include "trace.h"
 
 #include "pcnet.h"
 
@@ -61,9 +63,8 @@
 static void pcnet_aprom_writeb(void *opaque, uint32_t addr, uint32_t val)
 {
     PCNetState *s = opaque;
-#ifdef PCNET_DEBUG
-    printf("pcnet_aprom_writeb addr=0x%08x val=0x%02x\n", addr, val);
-#endif
+
+    trace_pcnet_aprom_writeb(opaque, addr, val);
     if (BCR_APROMWE(s)) {
         s->prom[addr & 15] = val;
     }
@@ -73,9 +74,8 @@
 {
     PCNetState *s = opaque;
     uint32_t val = s->prom[addr & 15];
-#ifdef PCNET_DEBUG
-    printf("pcnet_aprom_readb addr=0x%08x val=0x%02x\n", addr, val);
-#endif
+
+    trace_pcnet_aprom_readb(opaque, addr, val);
     return val;
 }
 
@@ -84,6 +84,7 @@
 {
     PCNetState *d = opaque;
 
+    trace_pcnet_ioport_read(opaque, addr, size);
     if (addr < 0x10) {
         if (!BCR_DWIO(d) && size == 1) {
             return pcnet_aprom_readb(d, addr);
@@ -111,6 +112,7 @@
 {
     PCNetState *d = opaque;
 
+    trace_pcnet_ioport_write(opaque, addr, data, size);
     if (addr < 0x10) {
         if (!BCR_DWIO(d) && size == 1) {
             pcnet_aprom_writeb(d, addr, data);
@@ -141,10 +143,8 @@
 static void pcnet_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
-#ifdef PCNET_DEBUG_IO
-    printf("pcnet_mmio_writeb addr=0x" TARGET_FMT_plx" val=0x%02x\n", addr,
-           val);
-#endif
+
+    trace_pcnet_mmio_writeb(opaque, addr, val);
     if (!(addr & 0x10))
         pcnet_aprom_writeb(d, addr & 0x0f, val);
 }
@@ -153,22 +153,18 @@
 {
     PCNetState *d = opaque;
     uint32_t val = -1;
+
     if (!(addr & 0x10))
         val = pcnet_aprom_readb(d, addr & 0x0f);
-#ifdef PCNET_DEBUG_IO
-    printf("pcnet_mmio_readb addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr,
-           val & 0xff);
-#endif
+    trace_pcnet_mmio_readb(opaque, addr, val);
     return val;
 }
 
 static void pcnet_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
-#ifdef PCNET_DEBUG_IO
-    printf("pcnet_mmio_writew addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr,
-           val);
-#endif
+
+    trace_pcnet_mmio_writew(opaque, addr, val);
     if (addr & 0x10)
         pcnet_ioport_writew(d, addr & 0x0f, val);
     else {
@@ -182,6 +178,7 @@
 {
     PCNetState *d = opaque;
     uint32_t val = -1;
+
     if (addr & 0x10)
         val = pcnet_ioport_readw(d, addr & 0x0f);
     else {
@@ -190,20 +187,15 @@
         val <<= 8;
         val |= pcnet_aprom_readb(d, addr);
     }
-#ifdef PCNET_DEBUG_IO
-    printf("pcnet_mmio_readw addr=0x" TARGET_FMT_plx" val = 0x%04x\n", addr,
-           val & 0xffff);
-#endif
+    trace_pcnet_mmio_readw(opaque, addr, val);
     return val;
 }
 
 static void pcnet_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
-#ifdef PCNET_DEBUG_IO
-    printf("pcnet_mmio_writel addr=0x" TARGET_FMT_plx" val=0x%08x\n", addr,
-           val);
-#endif
+
+    trace_pcnet_mmio_writel(opaque, addr, val);
     if (addr & 0x10)
         pcnet_ioport_writel(d, addr & 0x0f, val);
     else {
@@ -219,6 +211,7 @@
 {
     PCNetState *d = opaque;
     uint32_t val;
+
     if (addr & 0x10)
         val = pcnet_ioport_readl(d, addr & 0x0f);
     else {
@@ -231,10 +224,7 @@
         val <<= 8;
         val |= pcnet_aprom_readb(d, addr);
     }
-#ifdef PCNET_DEBUG_IO
-    printf("pcnet_mmio_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr,
-           val);
-#endif
+    trace_pcnet_mmio_readl(opaque, addr, val);
     return val;
 }
 
@@ -271,13 +261,6 @@
     pci_dma_read(dma_opaque, addr, buf, len);
 }
 
-static void pci_pcnet_cleanup(NetClientState *nc)
-{
-    PCNetState *d = qemu_get_nic_opaque(nc);
-
-    pcnet_common_cleanup(d);
-}
-
 static void pci_pcnet_uninit(PCIDevice *dev)
 {
     PCIPCNetState *d = PCI_PCNET(dev);
@@ -289,15 +272,13 @@
 }
 
 static NetClientInfo net_pci_pcnet_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = pcnet_can_receive,
     .receive = pcnet_receive,
     .link_status_changed = pcnet_set_link_status,
-    .cleanup = pci_pcnet_cleanup,
 };
 
-static int pci_pcnet_init(PCIDevice *pci_dev)
+static void pci_pcnet_realize(PCIDevice *pci_dev, Error **errp)
 {
     PCIPCNetState *d = PCI_PCNET(pci_dev);
     PCNetState *s = &d->state;
@@ -335,7 +316,7 @@
     s->phys_mem_write = pci_physical_memory_write;
     s->dma_opaque = pci_dev;
 
-    return pcnet_common_init(DEVICE(pci_dev), s, &net_pci_pcnet_info);
+    pcnet_common_init(DEVICE(pci_dev), s, &net_pci_pcnet_info);
 }
 
 static void pci_reset(DeviceState *dev)
@@ -365,7 +346,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_pcnet_init;
+    k->realize = pci_pcnet_realize;
     k->exit = pci_pcnet_uninit;
     k->romfile = "efi-pcnet.rom",
     k->vendor_id = PCI_VENDOR_ID_AMD;
diff --git a/hw/net/pcnet.c b/hw/net/pcnet.c
index f409b92..198a01f 100644
--- a/hw/net/pcnet.c
+++ b/hw/net/pcnet.c
@@ -35,11 +35,13 @@
  * http://www.ibiblio.org/pub/historic-linux/early-ports/Sparc/NCR/NCR92C990.txt
  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "net/net.h"
 #include "qemu/timer.h"
 #include "qemu/sockets.h"
 #include "sysemu/sysemu.h"
+#include "trace.h"
 
 #include "pcnet.h"
 
@@ -669,8 +671,7 @@
 static inline int64_t pcnet_get_next_poll_time(PCNetState *s, int64_t current_time)
 {
     int64_t next_time = current_time +
-        muldiv64(65536 - (CSR_SPND(s) ? 0 : CSR_POLL(s)),
-                 get_ticks_per_sec(), 33000000L);
+                        (65536 - (CSR_SPND(s) ? 0 : CSR_POLL(s))) * 30;
     if (next_time <= current_time)
         next_time = current_time + 1;
     return next_time;
@@ -685,9 +686,7 @@
 
 static void pcnet_s_reset(PCNetState *s)
 {
-#ifdef PCNET_DEBUG
-    printf("pcnet_s_reset\n");
-#endif
+    trace_pcnet_s_reset(s);
 
     s->rdra = 0;
     s->tdra = 0;
@@ -760,9 +759,7 @@
         s->csr[4] |= 0x0040;
         s->csr[0] |= 0x0080;
         isr = 1;
-#ifdef PCNET_DEBUG
-        printf("pcnet user int\n");
-#endif
+        trace_pcnet_user_int(s);
     }
 
 #if 1
@@ -777,9 +774,7 @@
     }
 
     if (isr != s->isr) {
-#ifdef PCNET_DEBUG
-        printf("pcnet: INTA=%d\n", isr);
-#endif
+        trace_pcnet_isr_change(s, isr, s->isr);
     }
     qemu_set_irq(s->irq, isr);
     s->isr = isr;
@@ -791,9 +786,7 @@
     uint16_t padr[3], ladrf[4], mode;
     uint32_t rdra, tdra;
 
-#ifdef PCNET_DEBUG
-    printf("pcnet_init init_addr=0x%08x\n", PHYSADDR(s,CSR_IADR(s)));
-#endif
+    trace_pcnet_init(s, PHYSADDR(s, CSR_IADR(s)));
 
     if (BCR_SSIZE32(s)) {
         struct pcnet_initblk32 initblk;
@@ -831,9 +824,7 @@
         tdra &= 0x00ffffff;
     }
 
-#if defined(PCNET_DEBUG)
-    printf("rlen=%d tlen=%d\n", rlen, tlen);
-#endif
+    trace_pcnet_rlen_tlen(s, rlen, tlen);
 
     CSR_RCVRL(s) = (rlen < 9) ? (1 << rlen) : 512;
     CSR_XMTRL(s) = (tlen < 9) ? (1 << tlen) : 512;
@@ -852,11 +843,8 @@
     CSR_RCVRC(s) = CSR_RCVRL(s);
     CSR_XMTRC(s) = CSR_XMTRL(s);
 
-#ifdef PCNET_DEBUG
-    printf("pcnet ss32=%d rdra=0x%08x[%d] tdra=0x%08x[%d]\n",
-        BCR_SSIZE32(s),
-        s->rdra, CSR_RCVRL(s), s->tdra, CSR_XMTRL(s));
-#endif
+    trace_pcnet_ss32_rdra_tdra(s, BCR_SSIZE32(s),
+                               s->rdra, CSR_RCVRL(s), s->tdra, CSR_XMTRL(s));
 
     s->csr[0] |= 0x0101;
     s->csr[0] &= ~0x0004;       /* clear STOP bit */
@@ -1007,15 +995,6 @@
     return !!(CSR_CXST(s) & 0x8000);
 }
 
-int pcnet_can_receive(NetClientState *nc)
-{
-    PCNetState *s = qemu_get_nic_opaque(nc);
-    if (CSR_STOP(s) || CSR_SPND(s))
-        return 0;
-
-    return sizeof(s->buffer)-16;
-}
-
 #define MIN_BUF_SIZE 60
 
 ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
@@ -1086,6 +1065,12 @@
             int pktcount = 0;
 
             if (!s->looptest) {
+                if (size > 4092) {
+#ifdef PCNET_DEBUG_RMD
+                    fprintf(stderr, "pcnet: truncates rx packet.\n");
+#endif
+                    size = 4092;
+                }
                 memcpy(src, buf, size);
                 /* no need to compute the CRC */
                 src[size] = 0;
@@ -1106,7 +1091,7 @@
                 uint32_t fcs = ~0;
                 uint8_t *p = src;
 
-                while (p != &src[size-4])
+                while (p != &src[size])
                     CRC(fcs, *p++);
                 crc_err = (*(uint32_t *)p != htonl(fcs));
             }
@@ -1253,6 +1238,16 @@
         }
 
         bcnt = 4096 - GET_FIELD(tmd.length, TMDL, BCNT);
+
+        /* if multi-tmd packet outsizes s->buffer then skip it silently.
+         * Note: this is not what real hw does.
+         * Last four bytes of s->buffer are used to store CRC FCS code.
+         */
+        if (s->xmit_pos + bcnt > sizeof(s->buffer) - 4) {
+            s->xmit_pos = -1;
+            goto txdone;
+        }
+
         s->phys_mem_read(s->dma_opaque, PHYSADDR(s, tmd.tbadr),
                          s->buffer + s->xmit_pos, bcnt, CSR_BSWP(s));
         s->xmit_pos += bcnt;
@@ -1719,17 +1714,12 @@
         VMSTATE_BUFFER(buffer, PCNetState),
         VMSTATE_UNUSED_TEST(is_version_2, 4),
         VMSTATE_INT32(tx_busy, PCNetState),
-        VMSTATE_TIMER(poll_timer, PCNetState),
+        VMSTATE_TIMER_PTR(poll_timer, PCNetState),
         VMSTATE_END_OF_LIST()
     }
 };
 
-void pcnet_common_cleanup(PCNetState *d)
-{
-    d->nic = NULL;
-}
-
-int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info)
+void pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info)
 {
     int i;
     uint16_t checksum;
@@ -1768,6 +1758,4 @@
     *(uint16_t *)&s->prom[12] = cpu_to_le16(checksum);
 
     s->lnkst = 0x40; /* initial link state: up */
-
-    return 0;
 }
diff --git a/hw/net/pcnet.h b/hw/net/pcnet.h
index f8e8a6f..40831a7 100644
--- a/hw/net/pcnet.h
+++ b/hw/net/pcnet.h
@@ -1,5 +1,5 @@
 #ifndef HW_PCNET_H
-#define HW_PCNET_H 1
+#define HW_PCNET_H
 
 #define PCNET_IOPORT_SIZE       0x20
 #define PCNET_PNPMMIO_SIZE      0x20
@@ -60,10 +60,8 @@
 void pcnet_ioport_writel(void *opaque, uint32_t addr, uint32_t val);
 uint32_t pcnet_ioport_readl(void *opaque, uint32_t addr);
 uint32_t pcnet_bcr_readw(PCNetState *s, uint32_t rap);
-int pcnet_can_receive(NetClientState *nc);
 ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_);
 void pcnet_set_link_status(NetClientState *nc);
-void pcnet_common_cleanup(PCNetState *d);
-int pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info);
+void pcnet_common_init(DeviceState *dev, PCNetState *s, NetClientInfo *info);
 extern const VMStateDescription vmstate_pcnet;
 #endif
diff --git a/hw/net/rocker/qmp-norocker.c b/hw/net/rocker/qmp-norocker.c
new file mode 100644
index 0000000..6acbcdb
--- /dev/null
+++ b/hw/net/rocker/qmp-norocker.c
@@ -0,0 +1,51 @@
+/*
+ * QMP Target options - Commands handled based on a target config
+ *                      versus a host config
+ *
+ * Copyright (c) 2015 David Ahern <dsahern@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qmp-commands.h"
+#include "qapi/qmp/qerror.h"
+
+RockerSwitch *qmp_query_rocker(const char *name, Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "rocker");
+    return NULL;
+};
+
+RockerPortList *qmp_query_rocker_ports(const char *name, Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "rocker");
+    return NULL;
+};
+
+RockerOfDpaFlowList *qmp_query_rocker_of_dpa_flows(const char *name,
+                                                   bool has_tbl_id,
+                                                   uint32_t tbl_id,
+                                                   Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "rocker");
+    return NULL;
+};
+
+RockerOfDpaGroupList *qmp_query_rocker_of_dpa_groups(const char *name,
+                                                     bool has_type,
+                                                     uint8_t type,
+                                                     Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "rocker");
+    return NULL;
+};
diff --git a/hw/net/rocker/rocker.c b/hw/net/rocker/rocker.c
new file mode 100644
index 0000000..30f2ce4
--- /dev/null
+++ b/hw/net/rocker/rocker.c
@@ -0,0 +1,1584 @@
+/*
+ * QEMU rocker switch emulation - PCI device
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/msix.h"
+#include "net/net.h"
+#include "net/eth.h"
+#include "qemu/iov.h"
+#include "qemu/bitops.h"
+#include "qmp-commands.h"
+
+#include "rocker.h"
+#include "rocker_hw.h"
+#include "rocker_fp.h"
+#include "rocker_desc.h"
+#include "rocker_tlv.h"
+#include "rocker_world.h"
+#include "rocker_of_dpa.h"
+
+struct rocker {
+    /* private */
+    PCIDevice parent_obj;
+    /* public */
+
+    MemoryRegion mmio;
+    MemoryRegion msix_bar;
+
+    /* switch configuration */
+    char *name;                  /* switch name */
+    char *world_name;            /* world name */
+    uint32_t fp_ports;           /* front-panel port count */
+    NICPeers *fp_ports_peers;
+    MACAddr fp_start_macaddr;    /* front-panel port 0 mac addr */
+    uint64_t switch_id;          /* switch id */
+
+    /* front-panel ports */
+    FpPort *fp_port[ROCKER_FP_PORTS_MAX];
+
+    /* register backings */
+    uint32_t test_reg;
+    uint64_t test_reg64;
+    dma_addr_t test_dma_addr;
+    uint32_t test_dma_size;
+    uint64_t lower32;            /* lower 32-bit val in 2-part 64-bit access */
+
+    /* desc rings */
+    DescRing **rings;
+
+    /* switch worlds */
+    World *worlds[ROCKER_WORLD_TYPE_MAX];
+    World *world_dflt;
+
+    QLIST_ENTRY(rocker) next;
+};
+
+#define ROCKER "rocker"
+
+#define to_rocker(obj) \
+    OBJECT_CHECK(Rocker, (obj), ROCKER)
+
+static QLIST_HEAD(, rocker) rockers;
+
+Rocker *rocker_find(const char *name)
+{
+    Rocker *r;
+
+    QLIST_FOREACH(r, &rockers, next)
+        if (strcmp(r->name, name) == 0) {
+            return r;
+        }
+
+    return NULL;
+}
+
+World *rocker_get_world(Rocker *r, enum rocker_world_type type)
+{
+    if (type < ROCKER_WORLD_TYPE_MAX) {
+        return r->worlds[type];
+    }
+    return NULL;
+}
+
+RockerSwitch *qmp_query_rocker(const char *name, Error **errp)
+{
+    RockerSwitch *rocker;
+    Rocker *r;
+
+    r = rocker_find(name);
+    if (!r) {
+        error_setg(errp, "rocker %s not found", name);
+        return NULL;
+    }
+
+    rocker = g_new0(RockerSwitch, 1);
+    rocker->name = g_strdup(r->name);
+    rocker->id = r->switch_id;
+    rocker->ports = r->fp_ports;
+
+    return rocker;
+}
+
+RockerPortList *qmp_query_rocker_ports(const char *name, Error **errp)
+{
+    RockerPortList *list = NULL;
+    Rocker *r;
+    int i;
+
+    r = rocker_find(name);
+    if (!r) {
+        error_setg(errp, "rocker %s not found", name);
+        return NULL;
+    }
+
+    for (i = r->fp_ports - 1; i >= 0; i--) {
+        RockerPortList *info = g_malloc0(sizeof(*info));
+        info->value = g_malloc0(sizeof(*info->value));
+        struct fp_port *port = r->fp_port[i];
+
+        fp_port_get_info(port, info);
+        info->next = list;
+        list = info;
+    }
+
+    return list;
+}
+
+uint32_t rocker_fp_ports(Rocker *r)
+{
+    return r->fp_ports;
+}
+
+static uint32_t rocker_get_pport_by_tx_ring(Rocker *r,
+                                            DescRing *ring)
+{
+    return (desc_ring_index(ring) - 2) / 2 + 1;
+}
+
+static int tx_consume(Rocker *r, DescInfo *info)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+    char *buf = desc_get_buf(info, true);
+    RockerTlv *tlv_frag;
+    RockerTlv *tlvs[ROCKER_TLV_TX_MAX + 1];
+    struct iovec iov[ROCKER_TX_FRAGS_MAX] = { { 0, }, };
+    uint32_t pport;
+    uint32_t port;
+    uint16_t tx_offload = ROCKER_TX_OFFLOAD_NONE;
+    uint16_t tx_l3_csum_off = 0;
+    uint16_t tx_tso_mss = 0;
+    uint16_t tx_tso_hdr_len = 0;
+    int iovcnt = 0;
+    int err = ROCKER_OK;
+    int rem;
+    int i;
+
+    if (!buf) {
+        return -ROCKER_ENXIO;
+    }
+
+    rocker_tlv_parse(tlvs, ROCKER_TLV_TX_MAX, buf, desc_tlv_size(info));
+
+    if (!tlvs[ROCKER_TLV_TX_FRAGS]) {
+        return -ROCKER_EINVAL;
+    }
+
+    pport = rocker_get_pport_by_tx_ring(r, desc_get_ring(info));
+    if (!fp_port_from_pport(pport, &port)) {
+        return -ROCKER_EINVAL;
+    }
+
+    if (tlvs[ROCKER_TLV_TX_OFFLOAD]) {
+        tx_offload = rocker_tlv_get_u8(tlvs[ROCKER_TLV_TX_OFFLOAD]);
+    }
+
+    switch (tx_offload) {
+    case ROCKER_TX_OFFLOAD_L3_CSUM:
+        if (!tlvs[ROCKER_TLV_TX_L3_CSUM_OFF]) {
+            return -ROCKER_EINVAL;
+        }
+        break;
+    case ROCKER_TX_OFFLOAD_TSO:
+        if (!tlvs[ROCKER_TLV_TX_TSO_MSS] ||
+            !tlvs[ROCKER_TLV_TX_TSO_HDR_LEN]) {
+            return -ROCKER_EINVAL;
+        }
+        break;
+    }
+
+    if (tlvs[ROCKER_TLV_TX_L3_CSUM_OFF]) {
+        tx_l3_csum_off = rocker_tlv_get_le16(tlvs[ROCKER_TLV_TX_L3_CSUM_OFF]);
+    }
+
+    if (tlvs[ROCKER_TLV_TX_TSO_MSS]) {
+        tx_tso_mss = rocker_tlv_get_le16(tlvs[ROCKER_TLV_TX_TSO_MSS]);
+    }
+
+    if (tlvs[ROCKER_TLV_TX_TSO_HDR_LEN]) {
+        tx_tso_hdr_len = rocker_tlv_get_le16(tlvs[ROCKER_TLV_TX_TSO_HDR_LEN]);
+    }
+
+    rocker_tlv_for_each_nested(tlv_frag, tlvs[ROCKER_TLV_TX_FRAGS], rem) {
+        hwaddr frag_addr;
+        uint16_t frag_len;
+
+        if (rocker_tlv_type(tlv_frag) != ROCKER_TLV_TX_FRAG) {
+            err = -ROCKER_EINVAL;
+            goto err_bad_attr;
+        }
+
+        rocker_tlv_parse_nested(tlvs, ROCKER_TLV_TX_FRAG_ATTR_MAX, tlv_frag);
+
+        if (!tlvs[ROCKER_TLV_TX_FRAG_ATTR_ADDR] ||
+            !tlvs[ROCKER_TLV_TX_FRAG_ATTR_LEN]) {
+            err = -ROCKER_EINVAL;
+            goto err_bad_attr;
+        }
+
+        frag_addr = rocker_tlv_get_le64(tlvs[ROCKER_TLV_TX_FRAG_ATTR_ADDR]);
+        frag_len = rocker_tlv_get_le16(tlvs[ROCKER_TLV_TX_FRAG_ATTR_LEN]);
+
+        if (iovcnt >= ROCKER_TX_FRAGS_MAX) {
+            goto err_too_many_frags;
+        }
+        iov[iovcnt].iov_len = frag_len;
+        iov[iovcnt].iov_base = g_malloc(frag_len);
+        if (!iov[iovcnt].iov_base) {
+            err = -ROCKER_ENOMEM;
+            goto err_no_mem;
+        }
+
+        if (pci_dma_read(dev, frag_addr, iov[iovcnt].iov_base,
+                     iov[iovcnt].iov_len)) {
+            err = -ROCKER_ENXIO;
+            goto err_bad_io;
+        }
+        iovcnt++;
+    }
+
+    if (iovcnt) {
+        /* XXX perform Tx offloads */
+        /* XXX   silence compiler for now */
+        tx_l3_csum_off += tx_tso_mss = tx_tso_hdr_len = 0;
+    }
+
+    err = fp_port_eg(r->fp_port[port], iov, iovcnt);
+
+err_too_many_frags:
+err_bad_io:
+err_no_mem:
+err_bad_attr:
+    for (i = 0; i < ROCKER_TX_FRAGS_MAX; i++) {
+        g_free(iov[i].iov_base);
+    }
+
+    return err;
+}
+
+static int cmd_get_port_settings(Rocker *r,
+                                 DescInfo *info, char *buf,
+                                 RockerTlv *cmd_info_tlv)
+{
+    RockerTlv *tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
+    RockerTlv *nest;
+    FpPort *fp_port;
+    uint32_t pport;
+    uint32_t port;
+    uint32_t speed;
+    uint8_t duplex;
+    uint8_t autoneg;
+    uint8_t learning;
+    char *phys_name;
+    MACAddr macaddr;
+    enum rocker_world_type mode;
+    size_t tlv_size;
+    int pos;
+    int err;
+
+    rocker_tlv_parse_nested(tlvs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
+                            cmd_info_tlv);
+
+    if (!tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_PPORT]) {
+        return -ROCKER_EINVAL;
+    }
+
+    pport = rocker_tlv_get_le32(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_PPORT]);
+    if (!fp_port_from_pport(pport, &port)) {
+        return -ROCKER_EINVAL;
+    }
+    fp_port = r->fp_port[port];
+
+    err = fp_port_get_settings(fp_port, &speed, &duplex, &autoneg);
+    if (err) {
+        return err;
+    }
+
+    fp_port_get_macaddr(fp_port, &macaddr);
+    mode = world_type(fp_port_get_world(fp_port));
+    learning = fp_port_get_learning(fp_port);
+    phys_name = fp_port_get_name(fp_port);
+
+    tlv_size = rocker_tlv_total_size(0) +                 /* nest */
+               rocker_tlv_total_size(sizeof(uint32_t)) +  /*   pport */
+               rocker_tlv_total_size(sizeof(uint32_t)) +  /*   speed */
+               rocker_tlv_total_size(sizeof(uint8_t)) +   /*   duplex */
+               rocker_tlv_total_size(sizeof(uint8_t)) +   /*   autoneg */
+               rocker_tlv_total_size(sizeof(macaddr.a)) + /*   macaddr */
+               rocker_tlv_total_size(sizeof(uint8_t)) +   /*   mode */
+               rocker_tlv_total_size(sizeof(uint8_t)) +   /*   learning */
+               rocker_tlv_total_size(strlen(phys_name));
+
+    if (tlv_size > desc_buf_size(info)) {
+        return -ROCKER_EMSGSIZE;
+    }
+
+    pos = 0;
+    nest = rocker_tlv_nest_start(buf, &pos, ROCKER_TLV_CMD_INFO);
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT, pport);
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_SPEED, speed);
+    rocker_tlv_put_u8(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX, duplex);
+    rocker_tlv_put_u8(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG, autoneg);
+    rocker_tlv_put(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,
+                   sizeof(macaddr.a), macaddr.a);
+    rocker_tlv_put_u8(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_MODE, mode);
+    rocker_tlv_put_u8(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,
+                      learning);
+    rocker_tlv_put(buf, &pos, ROCKER_TLV_CMD_PORT_SETTINGS_PHYS_NAME,
+                   strlen(phys_name), phys_name);
+    rocker_tlv_nest_end(buf, &pos, nest);
+
+    return desc_set_buf(info, tlv_size);
+}
+
+static int cmd_set_port_settings(Rocker *r,
+                                 RockerTlv *cmd_info_tlv)
+{
+    RockerTlv *tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MAX + 1];
+    FpPort *fp_port;
+    uint32_t pport;
+    uint32_t port;
+    uint32_t speed;
+    uint8_t duplex;
+    uint8_t autoneg;
+    uint8_t learning;
+    MACAddr macaddr;
+    enum rocker_world_type mode;
+    int err;
+
+    rocker_tlv_parse_nested(tlvs, ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
+                            cmd_info_tlv);
+
+    if (!tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_PPORT]) {
+        return -ROCKER_EINVAL;
+    }
+
+    pport = rocker_tlv_get_le32(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_PPORT]);
+    if (!fp_port_from_pport(pport, &port)) {
+        return -ROCKER_EINVAL;
+    }
+    fp_port = r->fp_port[port];
+
+    if (tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_SPEED] &&
+        tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX] &&
+        tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG]) {
+
+        speed = rocker_tlv_get_le32(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_SPEED]);
+        duplex = rocker_tlv_get_u8(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX]);
+        autoneg = rocker_tlv_get_u8(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG]);
+
+        err = fp_port_set_settings(fp_port, speed, duplex, autoneg);
+        if (err) {
+            return err;
+        }
+    }
+
+    if (tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR]) {
+        if (rocker_tlv_len(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR]) !=
+            sizeof(macaddr.a)) {
+            return -ROCKER_EINVAL;
+        }
+        memcpy(macaddr.a,
+               rocker_tlv_data(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR]),
+               sizeof(macaddr.a));
+        fp_port_set_macaddr(fp_port, &macaddr);
+    }
+
+    if (tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MODE]) {
+        mode = rocker_tlv_get_u8(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_MODE]);
+        if (mode >= ROCKER_WORLD_TYPE_MAX) {
+            return -ROCKER_EINVAL;
+        }
+        /* We don't support world change. */
+        if (!fp_port_check_world(fp_port, r->worlds[mode])) {
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING]) {
+        learning =
+            rocker_tlv_get_u8(tlvs[ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING]);
+        fp_port_set_learning(fp_port, learning);
+    }
+
+    return ROCKER_OK;
+}
+
+static int cmd_consume(Rocker *r, DescInfo *info)
+{
+    char *buf = desc_get_buf(info, false);
+    RockerTlv *tlvs[ROCKER_TLV_CMD_MAX + 1];
+    RockerTlv *info_tlv;
+    World *world;
+    uint16_t cmd;
+    int err;
+
+    if (!buf) {
+        return -ROCKER_ENXIO;
+    }
+
+    rocker_tlv_parse(tlvs, ROCKER_TLV_CMD_MAX, buf, desc_tlv_size(info));
+
+    if (!tlvs[ROCKER_TLV_CMD_TYPE] || !tlvs[ROCKER_TLV_CMD_INFO]) {
+        return -ROCKER_EINVAL;
+    }
+
+    cmd = rocker_tlv_get_le16(tlvs[ROCKER_TLV_CMD_TYPE]);
+    info_tlv = tlvs[ROCKER_TLV_CMD_INFO];
+
+    /* This might be reworked to something like this:
+     * Every world will have an array of command handlers from
+     * ROCKER_TLV_CMD_TYPE_UNSPEC to ROCKER_TLV_CMD_TYPE_MAX. There is
+     * up to each world to implement whatever command it want.
+     * It can reference "generic" commands as cmd_set_port_settings or
+     * cmd_get_port_settings
+     */
+
+    switch (cmd) {
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_ADD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_MOD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_DEL:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_GET_STATS:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_ADD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_DEL:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_GET_STATS:
+        world = r->worlds[ROCKER_WORLD_TYPE_OF_DPA];
+        err = world_do_cmd(world, info, buf, cmd, info_tlv);
+        break;
+    case ROCKER_TLV_CMD_TYPE_GET_PORT_SETTINGS:
+        err = cmd_get_port_settings(r, info, buf, info_tlv);
+        break;
+    case ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS:
+        err = cmd_set_port_settings(r, info_tlv);
+        break;
+    default:
+        err = -ROCKER_EINVAL;
+        break;
+    }
+
+    return err;
+}
+
+static void rocker_msix_irq(Rocker *r, unsigned vector)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+
+    DPRINTF("MSI-X notify request for vector %d\n", vector);
+    if (vector >= ROCKER_MSIX_VEC_COUNT(r->fp_ports)) {
+        DPRINTF("incorrect vector %d\n", vector);
+        return;
+    }
+    msix_notify(dev, vector);
+}
+
+int rocker_event_link_changed(Rocker *r, uint32_t pport, bool link_up)
+{
+    DescRing *ring = r->rings[ROCKER_RING_EVENT];
+    DescInfo *info = desc_ring_fetch_desc(ring);
+    RockerTlv *nest;
+    char *buf;
+    size_t tlv_size;
+    int pos;
+    int err;
+
+    if (!info) {
+        return -ROCKER_ENOBUFS;
+    }
+
+    tlv_size = rocker_tlv_total_size(sizeof(uint16_t)) +  /* event type */
+               rocker_tlv_total_size(0) +                 /* nest */
+               rocker_tlv_total_size(sizeof(uint32_t)) +  /*   pport */
+               rocker_tlv_total_size(sizeof(uint8_t));    /*   link up */
+
+    if (tlv_size > desc_buf_size(info)) {
+        err = -ROCKER_EMSGSIZE;
+        goto err_too_big;
+    }
+
+    buf = desc_get_buf(info, false);
+    if (!buf) {
+        err = -ROCKER_ENOMEM;
+        goto err_no_mem;
+    }
+
+    pos = 0;
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_EVENT_TYPE,
+                        ROCKER_TLV_EVENT_TYPE_LINK_CHANGED);
+    nest = rocker_tlv_nest_start(buf, &pos, ROCKER_TLV_EVENT_INFO);
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_EVENT_LINK_CHANGED_PPORT, pport);
+    rocker_tlv_put_u8(buf, &pos, ROCKER_TLV_EVENT_LINK_CHANGED_LINKUP,
+                      link_up ? 1 : 0);
+    rocker_tlv_nest_end(buf, &pos, nest);
+
+    err = desc_set_buf(info, tlv_size);
+
+err_too_big:
+err_no_mem:
+    if (desc_ring_post_desc(ring, err)) {
+        rocker_msix_irq(r, ROCKER_MSIX_VEC_EVENT);
+    }
+
+    return err;
+}
+
+int rocker_event_mac_vlan_seen(Rocker *r, uint32_t pport, uint8_t *addr,
+                               uint16_t vlan_id)
+{
+    DescRing *ring = r->rings[ROCKER_RING_EVENT];
+    DescInfo *info;
+    FpPort *fp_port;
+    uint32_t port;
+    RockerTlv *nest;
+    char *buf;
+    size_t tlv_size;
+    int pos;
+    int err;
+
+    if (!fp_port_from_pport(pport, &port)) {
+        return -ROCKER_EINVAL;
+    }
+    fp_port = r->fp_port[port];
+    if (!fp_port_get_learning(fp_port)) {
+        return ROCKER_OK;
+    }
+
+    info = desc_ring_fetch_desc(ring);
+    if (!info) {
+        return -ROCKER_ENOBUFS;
+    }
+
+    tlv_size = rocker_tlv_total_size(sizeof(uint16_t)) +  /* event type */
+               rocker_tlv_total_size(0) +                 /* nest */
+               rocker_tlv_total_size(sizeof(uint32_t)) +  /*   pport */
+               rocker_tlv_total_size(ETH_ALEN) +          /*   mac addr */
+               rocker_tlv_total_size(sizeof(uint16_t));   /*   vlan_id */
+
+    if (tlv_size > desc_buf_size(info)) {
+        err = -ROCKER_EMSGSIZE;
+        goto err_too_big;
+    }
+
+    buf = desc_get_buf(info, false);
+    if (!buf) {
+        err = -ROCKER_ENOMEM;
+        goto err_no_mem;
+    }
+
+    pos = 0;
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_EVENT_TYPE,
+                        ROCKER_TLV_EVENT_TYPE_MAC_VLAN_SEEN);
+    nest = rocker_tlv_nest_start(buf, &pos, ROCKER_TLV_EVENT_INFO);
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_EVENT_MAC_VLAN_PPORT, pport);
+    rocker_tlv_put(buf, &pos, ROCKER_TLV_EVENT_MAC_VLAN_MAC, ETH_ALEN, addr);
+    rocker_tlv_put_u16(buf, &pos, ROCKER_TLV_EVENT_MAC_VLAN_VLAN_ID, vlan_id);
+    rocker_tlv_nest_end(buf, &pos, nest);
+
+    err = desc_set_buf(info, tlv_size);
+
+err_too_big:
+err_no_mem:
+    if (desc_ring_post_desc(ring, err)) {
+        rocker_msix_irq(r, ROCKER_MSIX_VEC_EVENT);
+    }
+
+    return err;
+}
+
+static DescRing *rocker_get_rx_ring_by_pport(Rocker *r,
+                                                     uint32_t pport)
+{
+    return r->rings[(pport - 1) * 2 + 3];
+}
+
+int rx_produce(World *world, uint32_t pport,
+               const struct iovec *iov, int iovcnt, uint8_t copy_to_cpu)
+{
+    Rocker *r = world_rocker(world);
+    PCIDevice *dev = (PCIDevice *)r;
+    DescRing *ring = rocker_get_rx_ring_by_pport(r, pport);
+    DescInfo *info = desc_ring_fetch_desc(ring);
+    char *data;
+    size_t data_size = iov_size(iov, iovcnt);
+    char *buf;
+    uint16_t rx_flags = 0;
+    uint16_t rx_csum = 0;
+    size_t tlv_size;
+    RockerTlv *tlvs[ROCKER_TLV_RX_MAX + 1];
+    hwaddr frag_addr;
+    uint16_t frag_max_len;
+    int pos;
+    int err;
+
+    if (!info) {
+        return -ROCKER_ENOBUFS;
+    }
+
+    buf = desc_get_buf(info, false);
+    if (!buf) {
+        err = -ROCKER_ENXIO;
+        goto out;
+    }
+    rocker_tlv_parse(tlvs, ROCKER_TLV_RX_MAX, buf, desc_tlv_size(info));
+
+    if (!tlvs[ROCKER_TLV_RX_FRAG_ADDR] ||
+        !tlvs[ROCKER_TLV_RX_FRAG_MAX_LEN]) {
+        err = -ROCKER_EINVAL;
+        goto out;
+    }
+
+    frag_addr = rocker_tlv_get_le64(tlvs[ROCKER_TLV_RX_FRAG_ADDR]);
+    frag_max_len = rocker_tlv_get_le16(tlvs[ROCKER_TLV_RX_FRAG_MAX_LEN]);
+
+    if (data_size > frag_max_len) {
+        err = -ROCKER_EMSGSIZE;
+        goto out;
+    }
+
+    if (copy_to_cpu) {
+        rx_flags |= ROCKER_RX_FLAGS_FWD_OFFLOAD;
+    }
+
+    /* XXX calc rx flags/csum */
+
+    tlv_size = rocker_tlv_total_size(sizeof(uint16_t)) + /* flags */
+               rocker_tlv_total_size(sizeof(uint16_t)) + /* scum */
+               rocker_tlv_total_size(sizeof(uint64_t)) + /* frag addr */
+               rocker_tlv_total_size(sizeof(uint16_t)) + /* frag max len */
+               rocker_tlv_total_size(sizeof(uint16_t));  /* frag len */
+
+    if (tlv_size > desc_buf_size(info)) {
+        err = -ROCKER_EMSGSIZE;
+        goto out;
+    }
+
+    /* TODO:
+     * iov dma write can be optimized in similar way e1000 does it in
+     * e1000_receive_iov. But maybe if would make sense to introduce
+     * generic helper iov_dma_write.
+     */
+
+    data = g_malloc(data_size);
+    if (!data) {
+        err = -ROCKER_ENOMEM;
+        goto out;
+    }
+    iov_to_buf(iov, iovcnt, 0, data, data_size);
+    pci_dma_write(dev, frag_addr, data, data_size);
+    g_free(data);
+
+    pos = 0;
+    rocker_tlv_put_le16(buf, &pos, ROCKER_TLV_RX_FLAGS, rx_flags);
+    rocker_tlv_put_le16(buf, &pos, ROCKER_TLV_RX_CSUM, rx_csum);
+    rocker_tlv_put_le64(buf, &pos, ROCKER_TLV_RX_FRAG_ADDR, frag_addr);
+    rocker_tlv_put_le16(buf, &pos, ROCKER_TLV_RX_FRAG_MAX_LEN, frag_max_len);
+    rocker_tlv_put_le16(buf, &pos, ROCKER_TLV_RX_FRAG_LEN, data_size);
+
+    err = desc_set_buf(info, tlv_size);
+
+out:
+    if (desc_ring_post_desc(ring, err)) {
+        rocker_msix_irq(r, ROCKER_MSIX_VEC_RX(pport - 1));
+    }
+
+    return err;
+}
+
+int rocker_port_eg(Rocker *r, uint32_t pport,
+                   const struct iovec *iov, int iovcnt)
+{
+    FpPort *fp_port;
+    uint32_t port;
+
+    if (!fp_port_from_pport(pport, &port)) {
+        return -ROCKER_EINVAL;
+    }
+
+    fp_port = r->fp_port[port];
+
+    return fp_port_eg(fp_port, iov, iovcnt);
+}
+
+static void rocker_test_dma_ctrl(Rocker *r, uint32_t val)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+    char *buf;
+    int i;
+
+    buf = g_malloc(r->test_dma_size);
+
+    if (!buf) {
+        DPRINTF("test dma buffer alloc failed");
+        return;
+    }
+
+    switch (val) {
+    case ROCKER_TEST_DMA_CTRL_CLEAR:
+        memset(buf, 0, r->test_dma_size);
+        break;
+    case ROCKER_TEST_DMA_CTRL_FILL:
+        memset(buf, 0x96, r->test_dma_size);
+        break;
+    case ROCKER_TEST_DMA_CTRL_INVERT:
+        pci_dma_read(dev, r->test_dma_addr, buf, r->test_dma_size);
+        for (i = 0; i < r->test_dma_size; i++) {
+            buf[i] = ~buf[i];
+        }
+        break;
+    default:
+        DPRINTF("not test dma control val=0x%08x\n", val);
+        goto err_out;
+    }
+    pci_dma_write(dev, r->test_dma_addr, buf, r->test_dma_size);
+
+    rocker_msix_irq(r, ROCKER_MSIX_VEC_TEST);
+
+err_out:
+    g_free(buf);
+}
+
+static void rocker_reset(DeviceState *dev);
+
+static void rocker_control(Rocker *r, uint32_t val)
+{
+    if (val & ROCKER_CONTROL_RESET) {
+        rocker_reset(DEVICE(r));
+    }
+}
+
+static int rocker_pci_ring_count(Rocker *r)
+{
+    /* There are:
+     * - command ring
+     * - event ring
+     * - tx and rx ring per each port
+     */
+    return 2 + (2 * r->fp_ports);
+}
+
+static bool rocker_addr_is_desc_reg(Rocker *r, hwaddr addr)
+{
+    hwaddr start = ROCKER_DMA_DESC_BASE;
+    hwaddr end = start + (ROCKER_DMA_DESC_SIZE * rocker_pci_ring_count(r));
+
+    return addr >= start && addr < end;
+}
+
+static void rocker_port_phys_enable_write(Rocker *r, uint64_t new)
+{
+    int i;
+    bool old_enabled;
+    bool new_enabled;
+    FpPort *fp_port;
+
+    for (i = 0; i < r->fp_ports; i++) {
+        fp_port = r->fp_port[i];
+        old_enabled = fp_port_enabled(fp_port);
+        new_enabled = (new >> (i + 1)) & 0x1;
+        if (new_enabled == old_enabled) {
+            continue;
+        }
+        if (new_enabled) {
+            fp_port_enable(r->fp_port[i]);
+        } else {
+            fp_port_disable(r->fp_port[i]);
+        }
+    }
+}
+
+static void rocker_io_writel(void *opaque, hwaddr addr, uint32_t val)
+{
+    Rocker *r = opaque;
+
+    if (rocker_addr_is_desc_reg(r, addr)) {
+        unsigned index = ROCKER_RING_INDEX(addr);
+        unsigned offset = addr & ROCKER_DMA_DESC_MASK;
+
+        switch (offset) {
+        case ROCKER_DMA_DESC_ADDR_OFFSET:
+            r->lower32 = (uint64_t)val;
+            break;
+        case ROCKER_DMA_DESC_ADDR_OFFSET + 4:
+            desc_ring_set_base_addr(r->rings[index],
+                                    ((uint64_t)val) << 32 | r->lower32);
+            r->lower32 = 0;
+            break;
+        case ROCKER_DMA_DESC_SIZE_OFFSET:
+            desc_ring_set_size(r->rings[index], val);
+            break;
+        case ROCKER_DMA_DESC_HEAD_OFFSET:
+            if (desc_ring_set_head(r->rings[index], val)) {
+                rocker_msix_irq(r, desc_ring_get_msix_vector(r->rings[index]));
+            }
+            break;
+        case ROCKER_DMA_DESC_CTRL_OFFSET:
+            desc_ring_set_ctrl(r->rings[index], val);
+            break;
+        case ROCKER_DMA_DESC_CREDITS_OFFSET:
+            if (desc_ring_ret_credits(r->rings[index], val)) {
+                rocker_msix_irq(r, desc_ring_get_msix_vector(r->rings[index]));
+            }
+            break;
+        default:
+            DPRINTF("not implemented dma reg write(l) addr=0x" TARGET_FMT_plx
+                    " val=0x%08x (ring %d, addr=0x%02x)\n",
+                    addr, val, index, offset);
+            break;
+        }
+        return;
+    }
+
+    switch (addr) {
+    case ROCKER_TEST_REG:
+        r->test_reg = val;
+        break;
+    case ROCKER_TEST_REG64:
+    case ROCKER_TEST_DMA_ADDR:
+    case ROCKER_PORT_PHYS_ENABLE:
+        r->lower32 = (uint64_t)val;
+        break;
+    case ROCKER_TEST_REG64 + 4:
+        r->test_reg64 = ((uint64_t)val) << 32 | r->lower32;
+        r->lower32 = 0;
+        break;
+    case ROCKER_TEST_IRQ:
+        rocker_msix_irq(r, val);
+        break;
+    case ROCKER_TEST_DMA_SIZE:
+        r->test_dma_size = val;
+        break;
+    case ROCKER_TEST_DMA_ADDR + 4:
+        r->test_dma_addr = ((uint64_t)val) << 32 | r->lower32;
+        r->lower32 = 0;
+        break;
+    case ROCKER_TEST_DMA_CTRL:
+        rocker_test_dma_ctrl(r, val);
+        break;
+    case ROCKER_CONTROL:
+        rocker_control(r, val);
+        break;
+    case ROCKER_PORT_PHYS_ENABLE + 4:
+        rocker_port_phys_enable_write(r, ((uint64_t)val) << 32 | r->lower32);
+        r->lower32 = 0;
+        break;
+    default:
+        DPRINTF("not implemented write(l) addr=0x" TARGET_FMT_plx
+                " val=0x%08x\n", addr, val);
+        break;
+    }
+}
+
+static void rocker_io_writeq(void *opaque, hwaddr addr, uint64_t val)
+{
+    Rocker *r = opaque;
+
+    if (rocker_addr_is_desc_reg(r, addr)) {
+        unsigned index = ROCKER_RING_INDEX(addr);
+        unsigned offset = addr & ROCKER_DMA_DESC_MASK;
+
+        switch (offset) {
+        case ROCKER_DMA_DESC_ADDR_OFFSET:
+            desc_ring_set_base_addr(r->rings[index], val);
+            break;
+        default:
+            DPRINTF("not implemented dma reg write(q) addr=0x" TARGET_FMT_plx
+                    " val=0x" TARGET_FMT_plx " (ring %d, offset=0x%02x)\n",
+                    addr, val, index, offset);
+            break;
+        }
+        return;
+    }
+
+    switch (addr) {
+    case ROCKER_TEST_REG64:
+        r->test_reg64 = val;
+        break;
+    case ROCKER_TEST_DMA_ADDR:
+        r->test_dma_addr = val;
+        break;
+    case ROCKER_PORT_PHYS_ENABLE:
+        rocker_port_phys_enable_write(r, val);
+        break;
+    default:
+        DPRINTF("not implemented write(q) addr=0x" TARGET_FMT_plx
+                " val=0x" TARGET_FMT_plx "\n", addr, val);
+        break;
+    }
+}
+
+#ifdef DEBUG_ROCKER
+#define regname(reg) case (reg): return #reg
+static const char *rocker_reg_name(void *opaque, hwaddr addr)
+{
+    Rocker *r = opaque;
+
+    if (rocker_addr_is_desc_reg(r, addr)) {
+        unsigned index = ROCKER_RING_INDEX(addr);
+        unsigned offset = addr & ROCKER_DMA_DESC_MASK;
+        static char buf[100];
+        char ring_name[10];
+
+        switch (index) {
+        case 0:
+            sprintf(ring_name, "cmd");
+            break;
+        case 1:
+            sprintf(ring_name, "event");
+            break;
+        default:
+            sprintf(ring_name, "%s-%d", index % 2 ? "rx" : "tx",
+                    (index - 2) / 2);
+        }
+
+        switch (offset) {
+        case ROCKER_DMA_DESC_ADDR_OFFSET:
+            sprintf(buf, "Ring[%s] ADDR", ring_name);
+            return buf;
+        case ROCKER_DMA_DESC_ADDR_OFFSET+4:
+            sprintf(buf, "Ring[%s] ADDR+4", ring_name);
+            return buf;
+        case ROCKER_DMA_DESC_SIZE_OFFSET:
+            sprintf(buf, "Ring[%s] SIZE", ring_name);
+            return buf;
+        case ROCKER_DMA_DESC_HEAD_OFFSET:
+            sprintf(buf, "Ring[%s] HEAD", ring_name);
+            return buf;
+        case ROCKER_DMA_DESC_TAIL_OFFSET:
+            sprintf(buf, "Ring[%s] TAIL", ring_name);
+            return buf;
+        case ROCKER_DMA_DESC_CTRL_OFFSET:
+            sprintf(buf, "Ring[%s] CTRL", ring_name);
+            return buf;
+        case ROCKER_DMA_DESC_CREDITS_OFFSET:
+            sprintf(buf, "Ring[%s] CREDITS", ring_name);
+            return buf;
+        default:
+            sprintf(buf, "Ring[%s] ???", ring_name);
+            return buf;
+        }
+    } else {
+        switch (addr) {
+            regname(ROCKER_BOGUS_REG0);
+            regname(ROCKER_BOGUS_REG1);
+            regname(ROCKER_BOGUS_REG2);
+            regname(ROCKER_BOGUS_REG3);
+            regname(ROCKER_TEST_REG);
+            regname(ROCKER_TEST_REG64);
+            regname(ROCKER_TEST_REG64+4);
+            regname(ROCKER_TEST_IRQ);
+            regname(ROCKER_TEST_DMA_ADDR);
+            regname(ROCKER_TEST_DMA_ADDR+4);
+            regname(ROCKER_TEST_DMA_SIZE);
+            regname(ROCKER_TEST_DMA_CTRL);
+            regname(ROCKER_CONTROL);
+            regname(ROCKER_PORT_PHYS_COUNT);
+            regname(ROCKER_PORT_PHYS_LINK_STATUS);
+            regname(ROCKER_PORT_PHYS_LINK_STATUS+4);
+            regname(ROCKER_PORT_PHYS_ENABLE);
+            regname(ROCKER_PORT_PHYS_ENABLE+4);
+            regname(ROCKER_SWITCH_ID);
+            regname(ROCKER_SWITCH_ID+4);
+        }
+    }
+    return "???";
+}
+#else
+static const char *rocker_reg_name(void *opaque, hwaddr addr)
+{
+    return NULL;
+}
+#endif
+
+static void rocker_mmio_write(void *opaque, hwaddr addr, uint64_t val,
+                              unsigned size)
+{
+    DPRINTF("Write %s addr " TARGET_FMT_plx
+            ", size %u, val " TARGET_FMT_plx "\n",
+            rocker_reg_name(opaque, addr), addr, size, val);
+
+    switch (size) {
+    case 4:
+        rocker_io_writel(opaque, addr, val);
+        break;
+    case 8:
+        rocker_io_writeq(opaque, addr, val);
+        break;
+    }
+}
+
+static uint64_t rocker_port_phys_link_status(Rocker *r)
+{
+    int i;
+    uint64_t status = 0;
+
+    for (i = 0; i < r->fp_ports; i++) {
+        FpPort *port = r->fp_port[i];
+
+        if (fp_port_get_link_up(port)) {
+            status |= 1 << (i + 1);
+        }
+    }
+    return status;
+}
+
+static uint64_t rocker_port_phys_enable_read(Rocker *r)
+{
+    int i;
+    uint64_t ret = 0;
+
+    for (i = 0; i < r->fp_ports; i++) {
+        FpPort *port = r->fp_port[i];
+
+        if (fp_port_enabled(port)) {
+            ret |= 1 << (i + 1);
+        }
+    }
+    return ret;
+}
+
+static uint32_t rocker_io_readl(void *opaque, hwaddr addr)
+{
+    Rocker *r = opaque;
+    uint32_t ret;
+
+    if (rocker_addr_is_desc_reg(r, addr)) {
+        unsigned index = ROCKER_RING_INDEX(addr);
+        unsigned offset = addr & ROCKER_DMA_DESC_MASK;
+
+        switch (offset) {
+        case ROCKER_DMA_DESC_ADDR_OFFSET:
+            ret = (uint32_t)desc_ring_get_base_addr(r->rings[index]);
+            break;
+        case ROCKER_DMA_DESC_ADDR_OFFSET + 4:
+            ret = (uint32_t)(desc_ring_get_base_addr(r->rings[index]) >> 32);
+            break;
+        case ROCKER_DMA_DESC_SIZE_OFFSET:
+            ret = desc_ring_get_size(r->rings[index]);
+            break;
+        case ROCKER_DMA_DESC_HEAD_OFFSET:
+            ret = desc_ring_get_head(r->rings[index]);
+            break;
+        case ROCKER_DMA_DESC_TAIL_OFFSET:
+            ret = desc_ring_get_tail(r->rings[index]);
+            break;
+        case ROCKER_DMA_DESC_CREDITS_OFFSET:
+            ret = desc_ring_get_credits(r->rings[index]);
+            break;
+        default:
+            DPRINTF("not implemented dma reg read(l) addr=0x" TARGET_FMT_plx
+                    " (ring %d, addr=0x%02x)\n", addr, index, offset);
+            ret = 0;
+            break;
+        }
+        return ret;
+    }
+
+    switch (addr) {
+    case ROCKER_BOGUS_REG0:
+    case ROCKER_BOGUS_REG1:
+    case ROCKER_BOGUS_REG2:
+    case ROCKER_BOGUS_REG3:
+        ret = 0xDEADBABE;
+        break;
+    case ROCKER_TEST_REG:
+        ret = r->test_reg * 2;
+        break;
+    case ROCKER_TEST_REG64:
+        ret = (uint32_t)(r->test_reg64 * 2);
+        break;
+    case ROCKER_TEST_REG64 + 4:
+        ret = (uint32_t)((r->test_reg64 * 2) >> 32);
+        break;
+    case ROCKER_TEST_DMA_SIZE:
+        ret = r->test_dma_size;
+        break;
+    case ROCKER_TEST_DMA_ADDR:
+        ret = (uint32_t)r->test_dma_addr;
+        break;
+    case ROCKER_TEST_DMA_ADDR + 4:
+        ret = (uint32_t)(r->test_dma_addr >> 32);
+        break;
+    case ROCKER_PORT_PHYS_COUNT:
+        ret = r->fp_ports;
+        break;
+    case ROCKER_PORT_PHYS_LINK_STATUS:
+        ret = (uint32_t)rocker_port_phys_link_status(r);
+        break;
+    case ROCKER_PORT_PHYS_LINK_STATUS + 4:
+        ret = (uint32_t)(rocker_port_phys_link_status(r) >> 32);
+        break;
+    case ROCKER_PORT_PHYS_ENABLE:
+        ret = (uint32_t)rocker_port_phys_enable_read(r);
+        break;
+    case ROCKER_PORT_PHYS_ENABLE + 4:
+        ret = (uint32_t)(rocker_port_phys_enable_read(r) >> 32);
+        break;
+    case ROCKER_SWITCH_ID:
+        ret = (uint32_t)r->switch_id;
+        break;
+    case ROCKER_SWITCH_ID + 4:
+        ret = (uint32_t)(r->switch_id >> 32);
+        break;
+    default:
+        DPRINTF("not implemented read(l) addr=0x" TARGET_FMT_plx "\n", addr);
+        ret = 0;
+        break;
+    }
+    return ret;
+}
+
+static uint64_t rocker_io_readq(void *opaque, hwaddr addr)
+{
+    Rocker *r = opaque;
+    uint64_t ret;
+
+    if (rocker_addr_is_desc_reg(r, addr)) {
+        unsigned index = ROCKER_RING_INDEX(addr);
+        unsigned offset = addr & ROCKER_DMA_DESC_MASK;
+
+        switch (addr & ROCKER_DMA_DESC_MASK) {
+        case ROCKER_DMA_DESC_ADDR_OFFSET:
+            ret = desc_ring_get_base_addr(r->rings[index]);
+            break;
+        default:
+            DPRINTF("not implemented dma reg read(q) addr=0x" TARGET_FMT_plx
+                    " (ring %d, addr=0x%02x)\n", addr, index, offset);
+            ret = 0;
+            break;
+        }
+        return ret;
+    }
+
+    switch (addr) {
+    case ROCKER_BOGUS_REG0:
+    case ROCKER_BOGUS_REG2:
+        ret = 0xDEADBABEDEADBABEULL;
+        break;
+    case ROCKER_TEST_REG64:
+        ret = r->test_reg64 * 2;
+        break;
+    case ROCKER_TEST_DMA_ADDR:
+        ret = r->test_dma_addr;
+        break;
+    case ROCKER_PORT_PHYS_LINK_STATUS:
+        ret = rocker_port_phys_link_status(r);
+        break;
+    case ROCKER_PORT_PHYS_ENABLE:
+        ret = rocker_port_phys_enable_read(r);
+        break;
+    case ROCKER_SWITCH_ID:
+        ret = r->switch_id;
+        break;
+    default:
+        DPRINTF("not implemented read(q) addr=0x" TARGET_FMT_plx "\n", addr);
+        ret = 0;
+        break;
+    }
+    return ret;
+}
+
+static uint64_t rocker_mmio_read(void *opaque, hwaddr addr, unsigned size)
+{
+    DPRINTF("Read %s addr " TARGET_FMT_plx ", size %u\n",
+            rocker_reg_name(opaque, addr), addr, size);
+
+    switch (size) {
+    case 4:
+        return rocker_io_readl(opaque, addr);
+    case 8:
+        return rocker_io_readq(opaque, addr);
+    }
+
+    return -1;
+}
+
+static const MemoryRegionOps rocker_mmio_ops = {
+    .read = rocker_mmio_read,
+    .write = rocker_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+};
+
+static void rocker_msix_vectors_unuse(Rocker *r,
+                                      unsigned int num_vectors)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+    int i;
+
+    for (i = 0; i < num_vectors; i++) {
+        msix_vector_unuse(dev, i);
+    }
+}
+
+static int rocker_msix_vectors_use(Rocker *r,
+                                   unsigned int num_vectors)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+    int err;
+    int i;
+
+    for (i = 0; i < num_vectors; i++) {
+        err = msix_vector_use(dev, i);
+        if (err) {
+            goto rollback;
+        }
+    }
+    return 0;
+
+rollback:
+    rocker_msix_vectors_unuse(r, i);
+    return err;
+}
+
+static int rocker_msix_init(Rocker *r)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+    int err;
+
+    err = msix_init(dev, ROCKER_MSIX_VEC_COUNT(r->fp_ports),
+                    &r->msix_bar,
+                    ROCKER_PCI_MSIX_BAR_IDX, ROCKER_PCI_MSIX_TABLE_OFFSET,
+                    &r->msix_bar,
+                    ROCKER_PCI_MSIX_BAR_IDX, ROCKER_PCI_MSIX_PBA_OFFSET,
+                    0);
+    if (err) {
+        return err;
+    }
+
+    err = rocker_msix_vectors_use(r, ROCKER_MSIX_VEC_COUNT(r->fp_ports));
+    if (err) {
+        goto err_msix_vectors_use;
+    }
+
+    return 0;
+
+err_msix_vectors_use:
+    msix_uninit(dev, &r->msix_bar, &r->msix_bar);
+    return err;
+}
+
+static void rocker_msix_uninit(Rocker *r)
+{
+    PCIDevice *dev = PCI_DEVICE(r);
+
+    msix_uninit(dev, &r->msix_bar, &r->msix_bar);
+    rocker_msix_vectors_unuse(r, ROCKER_MSIX_VEC_COUNT(r->fp_ports));
+}
+
+static World *rocker_world_type_by_name(Rocker *r, const char *name)
+{
+    int i;
+
+    for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
+        if (strcmp(name, world_name(r->worlds[i])) == 0) {
+            return r->worlds[i];
+	}
+    }
+    return NULL;
+}
+
+static int pci_rocker_init(PCIDevice *dev)
+{
+    Rocker *r = to_rocker(dev);
+    const MACAddr zero = { .a = { 0, 0, 0, 0, 0, 0 } };
+    const MACAddr dflt = { .a = { 0x52, 0x54, 0x00, 0x12, 0x35, 0x01 } };
+    static int sw_index;
+    int i, err = 0;
+
+    /* allocate worlds */
+
+    r->worlds[ROCKER_WORLD_TYPE_OF_DPA] = of_dpa_world_alloc(r);
+
+    for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
+        if (!r->worlds[i]) {
+            err = -ENOMEM;
+            goto err_world_alloc;
+        }
+    }
+
+    if (!r->world_name) {
+        r->world_name = g_strdup(world_name(r->worlds[ROCKER_WORLD_TYPE_OF_DPA]));
+    }
+
+    r->world_dflt = rocker_world_type_by_name(r, r->world_name);
+    if (!r->world_dflt) {
+        fprintf(stderr,
+                "rocker: requested world \"%s\" does not exist\n",
+                r->world_name);
+        err = -EINVAL;
+        goto err_world_type_by_name;
+    }
+
+    /* set up memory-mapped region at BAR0 */
+
+    memory_region_init_io(&r->mmio, OBJECT(r), &rocker_mmio_ops, r,
+                          "rocker-mmio", ROCKER_PCI_BAR0_SIZE);
+    pci_register_bar(dev, ROCKER_PCI_BAR0_IDX,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &r->mmio);
+
+    /* set up memory-mapped region for MSI-X */
+
+    memory_region_init(&r->msix_bar, OBJECT(r), "rocker-msix-bar",
+                       ROCKER_PCI_MSIX_BAR_SIZE);
+    pci_register_bar(dev, ROCKER_PCI_MSIX_BAR_IDX,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &r->msix_bar);
+
+    /* MSI-X init */
+
+    err = rocker_msix_init(r);
+    if (err) {
+        goto err_msix_init;
+    }
+
+    /* validate switch properties */
+
+    if (!r->name) {
+        r->name = g_strdup(ROCKER);
+    }
+
+    if (rocker_find(r->name)) {
+        err = -EEXIST;
+        goto err_duplicate;
+    }
+
+    /* Rocker name is passed in port name requests to OS with the intention
+     * that the name is used in interface names. Limit the length of the
+     * rocker name to avoid naming problems in the OS. Also, adding the
+     * port number as p# and unganged breakout b#, where # is at most 2
+     * digits, so leave room for it too (-1 for string terminator, -3 for
+     * p# and -3 for b#)
+     */
+#define ROCKER_IFNAMSIZ 16
+#define MAX_ROCKER_NAME_LEN  (ROCKER_IFNAMSIZ - 1 - 3 - 3)
+    if (strlen(r->name) > MAX_ROCKER_NAME_LEN) {
+        fprintf(stderr,
+                "rocker: name too long; please shorten to at most %d chars\n",
+                MAX_ROCKER_NAME_LEN);
+        return -EINVAL;
+    }
+
+    if (memcmp(&r->fp_start_macaddr, &zero, sizeof(zero)) == 0) {
+        memcpy(&r->fp_start_macaddr, &dflt, sizeof(dflt));
+        r->fp_start_macaddr.a[4] += (sw_index++);
+    }
+
+    if (!r->switch_id) {
+        memcpy(&r->switch_id, &r->fp_start_macaddr,
+               sizeof(r->fp_start_macaddr));
+    }
+
+    if (r->fp_ports > ROCKER_FP_PORTS_MAX) {
+        r->fp_ports = ROCKER_FP_PORTS_MAX;
+    }
+
+    r->rings = g_new(DescRing *, rocker_pci_ring_count(r));
+    if (!r->rings) {
+        goto err_rings_alloc;
+    }
+
+    /* Rings are ordered like this:
+     * - command ring
+     * - event ring
+     * - port0 tx ring
+     * - port0 rx ring
+     * - port1 tx ring
+     * - port1 rx ring
+     * .....
+     */
+
+    err = -ENOMEM;
+    for (i = 0; i < rocker_pci_ring_count(r); i++) {
+        DescRing *ring = desc_ring_alloc(r, i);
+
+        if (!ring) {
+            goto err_ring_alloc;
+        }
+
+        if (i == ROCKER_RING_CMD) {
+            desc_ring_set_consume(ring, cmd_consume, ROCKER_MSIX_VEC_CMD);
+        } else if (i == ROCKER_RING_EVENT) {
+            desc_ring_set_consume(ring, NULL, ROCKER_MSIX_VEC_EVENT);
+        } else if (i % 2 == 0) {
+            desc_ring_set_consume(ring, tx_consume,
+                                  ROCKER_MSIX_VEC_TX((i - 2) / 2));
+        } else if (i % 2 == 1) {
+            desc_ring_set_consume(ring, NULL, ROCKER_MSIX_VEC_RX((i - 3) / 2));
+        }
+
+        r->rings[i] = ring;
+    }
+
+    for (i = 0; i < r->fp_ports; i++) {
+        FpPort *port =
+            fp_port_alloc(r, r->name, &r->fp_start_macaddr,
+                          i, &r->fp_ports_peers[i]);
+
+        if (!port) {
+            goto err_port_alloc;
+        }
+
+        r->fp_port[i] = port;
+        fp_port_set_world(port, r->world_dflt);
+    }
+
+    QLIST_INSERT_HEAD(&rockers, r, next);
+
+    return 0;
+
+err_port_alloc:
+    for (--i; i >= 0; i--) {
+        FpPort *port = r->fp_port[i];
+        fp_port_free(port);
+    }
+    i = rocker_pci_ring_count(r);
+err_ring_alloc:
+    for (--i; i >= 0; i--) {
+        desc_ring_free(r->rings[i]);
+    }
+    g_free(r->rings);
+err_rings_alloc:
+err_duplicate:
+    rocker_msix_uninit(r);
+err_msix_init:
+    object_unparent(OBJECT(&r->msix_bar));
+    object_unparent(OBJECT(&r->mmio));
+err_world_type_by_name:
+err_world_alloc:
+    for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
+        if (r->worlds[i]) {
+            world_free(r->worlds[i]);
+        }
+    }
+    return err;
+}
+
+static void pci_rocker_uninit(PCIDevice *dev)
+{
+    Rocker *r = to_rocker(dev);
+    int i;
+
+    QLIST_REMOVE(r, next);
+
+    for (i = 0; i < r->fp_ports; i++) {
+        FpPort *port = r->fp_port[i];
+
+        fp_port_free(port);
+        r->fp_port[i] = NULL;
+    }
+
+    for (i = 0; i < rocker_pci_ring_count(r); i++) {
+        if (r->rings[i]) {
+            desc_ring_free(r->rings[i]);
+        }
+    }
+    g_free(r->rings);
+
+    rocker_msix_uninit(r);
+    object_unparent(OBJECT(&r->msix_bar));
+    object_unparent(OBJECT(&r->mmio));
+
+    for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
+        if (r->worlds[i]) {
+            world_free(r->worlds[i]);
+        }
+    }
+    g_free(r->fp_ports_peers);
+}
+
+static void rocker_reset(DeviceState *dev)
+{
+    Rocker *r = to_rocker(dev);
+    int i;
+
+    for (i = 0; i < ROCKER_WORLD_TYPE_MAX; i++) {
+        if (r->worlds[i]) {
+            world_reset(r->worlds[i]);
+        }
+    }
+    for (i = 0; i < r->fp_ports; i++) {
+        fp_port_reset(r->fp_port[i]);
+        fp_port_set_world(r->fp_port[i], r->world_dflt);
+    }
+
+    r->test_reg = 0;
+    r->test_reg64 = 0;
+    r->test_dma_addr = 0;
+    r->test_dma_size = 0;
+
+    for (i = 0; i < rocker_pci_ring_count(r); i++) {
+        desc_ring_reset(r->rings[i]);
+    }
+
+    DPRINTF("Reset done\n");
+}
+
+static Property rocker_properties[] = {
+    DEFINE_PROP_STRING("name", Rocker, name),
+    DEFINE_PROP_STRING("world", Rocker, world_name),
+    DEFINE_PROP_MACADDR("fp_start_macaddr", Rocker,
+                        fp_start_macaddr),
+    DEFINE_PROP_UINT64("switch_id", Rocker,
+                       switch_id, 0),
+    DEFINE_PROP_ARRAY("ports", Rocker, fp_ports,
+                      fp_ports_peers, qdev_prop_netdev, NICPeers),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription rocker_vmsd = {
+    .name = ROCKER,
+    .unmigratable = 1,
+};
+
+static void rocker_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = pci_rocker_init;
+    k->exit = pci_rocker_uninit;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_ROCKER;
+    k->revision = ROCKER_PCI_REVISION;
+    k->class_id = PCI_CLASS_NETWORK_OTHER;
+    set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
+    dc->desc = "Rocker Switch";
+    dc->reset = rocker_reset;
+    dc->props = rocker_properties;
+    dc->vmsd = &rocker_vmsd;
+}
+
+static const TypeInfo rocker_info = {
+    .name          = ROCKER,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(Rocker),
+    .class_init    = rocker_class_init,
+};
+
+static void rocker_register_types(void)
+{
+    type_register_static(&rocker_info);
+}
+
+type_init(rocker_register_types)
diff --git a/hw/net/rocker/rocker.h b/hw/net/rocker/rocker.h
new file mode 100644
index 0000000..7ae0495
--- /dev/null
+++ b/hw/net/rocker/rocker.h
@@ -0,0 +1,84 @@
+/*
+ * QEMU rocker switch emulation
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
+ * Copyright (c) 2014 Neil Horman <nhorman@tuxdriver.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef ROCKER_H
+#define ROCKER_H
+
+#include "qemu/sockets.h"
+
+#if defined(DEBUG_ROCKER)
+#  define DPRINTF(fmt, ...) \
+    do {                                                           \
+        struct timeval tv;                                         \
+        char timestr[64];                                          \
+        time_t now;                                                \
+        gettimeofday(&tv, NULL);                                   \
+        now = tv.tv_sec;                                           \
+        strftime(timestr, sizeof(timestr), "%T", localtime(&now)); \
+        fprintf(stderr, "%s.%06ld ", timestr, tv.tv_usec);         \
+        fprintf(stderr, "ROCKER: " fmt, ## __VA_ARGS__);           \
+    } while (0)
+#else
+static inline GCC_FMT_ATTR(1, 2) int DPRINTF(const char *fmt, ...)
+{
+    return 0;
+}
+#endif
+
+#define __le16 uint16_t
+#define __le32 uint32_t
+#define __le64 uint64_t
+
+#define __be16 uint16_t
+#define __be32 uint32_t
+#define __be64 uint64_t
+
+static inline bool ipv4_addr_is_multicast(__be32 addr)
+{
+    return (addr & htonl(0xf0000000)) == htonl(0xe0000000);
+}
+
+typedef struct ipv6_addr {
+    union {
+        uint8_t addr8[16];
+        __be16 addr16[8];
+        __be32 addr32[4];
+    };
+} Ipv6Addr;
+
+static inline bool ipv6_addr_is_multicast(const Ipv6Addr *addr)
+{
+    return (addr->addr32[0] & htonl(0xFF000000)) == htonl(0xFF000000);
+}
+
+typedef struct rocker Rocker;
+typedef struct world World;
+typedef struct desc_info DescInfo;
+typedef struct desc_ring DescRing;
+
+Rocker *rocker_find(const char *name);
+uint32_t rocker_fp_ports(Rocker *r);
+int rocker_event_link_changed(Rocker *r, uint32_t pport, bool link_up);
+int rocker_event_mac_vlan_seen(Rocker *r, uint32_t pport, uint8_t *addr,
+                               uint16_t vlan_id);
+int rx_produce(World *world, uint32_t pport,
+               const struct iovec *iov, int iovcnt, uint8_t copy_to_cpu);
+int rocker_port_eg(Rocker *r, uint32_t pport,
+                   const struct iovec *iov, int iovcnt);
+
+#endif /* ROCKER_H */
diff --git a/hw/net/rocker/rocker_desc.c b/hw/net/rocker/rocker_desc.c
new file mode 100644
index 0000000..ac02797
--- /dev/null
+++ b/hw/net/rocker/rocker_desc.c
@@ -0,0 +1,374 @@
+/*
+ * QEMU rocker switch emulation - Descriptor ring support
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "net/net.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+
+#include "rocker.h"
+#include "rocker_hw.h"
+#include "rocker_desc.h"
+
+struct desc_ring {
+    hwaddr base_addr;
+    uint32_t size;
+    uint32_t head;
+    uint32_t tail;
+    uint32_t ctrl;
+    uint32_t credits;
+    Rocker *r;
+    DescInfo *info;
+    int index;
+    desc_ring_consume *consume;
+    unsigned msix_vector;
+};
+
+struct desc_info {
+    DescRing *ring;
+    RockerDesc desc;
+    char *buf;
+    size_t buf_size;
+};
+
+uint16_t desc_buf_size(DescInfo *info)
+{
+    return le16_to_cpu(info->desc.buf_size);
+}
+
+uint16_t desc_tlv_size(DescInfo *info)
+{
+    return le16_to_cpu(info->desc.tlv_size);
+}
+
+char *desc_get_buf(DescInfo *info, bool read_only)
+{
+    PCIDevice *dev = PCI_DEVICE(info->ring->r);
+    size_t size = read_only ? le16_to_cpu(info->desc.tlv_size) :
+                              le16_to_cpu(info->desc.buf_size);
+
+    if (size > info->buf_size) {
+        info->buf = g_realloc(info->buf, size);
+        info->buf_size = size;
+    }
+
+    if (!info->buf) {
+        return NULL;
+    }
+
+    if (pci_dma_read(dev, le64_to_cpu(info->desc.buf_addr), info->buf, size)) {
+        return NULL;
+    }
+
+    return info->buf;
+}
+
+int desc_set_buf(DescInfo *info, size_t tlv_size)
+{
+    PCIDevice *dev = PCI_DEVICE(info->ring->r);
+
+    if (tlv_size > info->buf_size) {
+        DPRINTF("ERROR: trying to write more to desc buf than it "
+                "can hold buf_size %zu tlv_size %zu\n",
+                info->buf_size, tlv_size);
+        return -ROCKER_EMSGSIZE;
+    }
+
+    info->desc.tlv_size = cpu_to_le16(tlv_size);
+    pci_dma_write(dev, le64_to_cpu(info->desc.buf_addr), info->buf, tlv_size);
+
+    return ROCKER_OK;
+}
+
+DescRing *desc_get_ring(DescInfo *info)
+{
+    return info->ring;
+}
+
+int desc_ring_index(DescRing *ring)
+{
+    return ring->index;
+}
+
+static bool desc_ring_empty(DescRing *ring)
+{
+    return ring->head == ring->tail;
+}
+
+bool desc_ring_set_base_addr(DescRing *ring, uint64_t base_addr)
+{
+    if (base_addr & 0x7) {
+        DPRINTF("ERROR: ring[%d] desc base addr (0x" TARGET_FMT_plx
+                ") not 8-byte aligned\n", ring->index, base_addr);
+        return false;
+    }
+
+    ring->base_addr = base_addr;
+
+    return true;
+}
+
+uint64_t desc_ring_get_base_addr(DescRing *ring)
+{
+    return ring->base_addr;
+}
+
+bool desc_ring_set_size(DescRing *ring, uint32_t size)
+{
+    int i;
+
+    if (size < 2 || size > 0x10000 || (size & (size - 1))) {
+        DPRINTF("ERROR: ring[%d] size (%d) not a power of 2 "
+                "or in range [2, 64K]\n", ring->index, size);
+        return false;
+    }
+
+    for (i = 0; i < ring->size; i++) {
+        g_free(ring->info[i].buf);
+    }
+
+    ring->size = size;
+    ring->head = ring->tail = 0;
+
+    ring->info = g_renew(DescInfo, ring->info, size);
+    if (!ring->info) {
+        return false;
+    }
+
+    memset(ring->info, 0, size * sizeof(DescInfo));
+
+    for (i = 0; i < size; i++) {
+        ring->info[i].ring = ring;
+    }
+
+    return true;
+}
+
+uint32_t desc_ring_get_size(DescRing *ring)
+{
+    return ring->size;
+}
+
+static DescInfo *desc_read(DescRing *ring, uint32_t index)
+{
+    PCIDevice *dev = PCI_DEVICE(ring->r);
+    DescInfo *info = &ring->info[index];
+    hwaddr addr = ring->base_addr + (sizeof(RockerDesc) * index);
+
+    pci_dma_read(dev, addr, &info->desc, sizeof(info->desc));
+
+    return info;
+}
+
+static void desc_write(DescRing *ring, uint32_t index)
+{
+    PCIDevice *dev = PCI_DEVICE(ring->r);
+    DescInfo *info = &ring->info[index];
+    hwaddr addr = ring->base_addr + (sizeof(RockerDesc) * index);
+
+    pci_dma_write(dev, addr, &info->desc, sizeof(info->desc));
+}
+
+static bool desc_ring_base_addr_check(DescRing *ring)
+{
+    if (!ring->base_addr) {
+        DPRINTF("ERROR: ring[%d] not-initialized desc base address!\n",
+                ring->index);
+        return false;
+    }
+    return true;
+}
+
+static DescInfo *__desc_ring_fetch_desc(DescRing *ring)
+{
+    return desc_read(ring, ring->tail);
+}
+
+DescInfo *desc_ring_fetch_desc(DescRing *ring)
+{
+    if (desc_ring_empty(ring) || !desc_ring_base_addr_check(ring)) {
+        return NULL;
+    }
+
+    return desc_read(ring, ring->tail);
+}
+
+static bool __desc_ring_post_desc(DescRing *ring, int err)
+{
+    uint16_t comp_err = 0x8000 | (uint16_t)-err;
+    DescInfo *info = &ring->info[ring->tail];
+
+    info->desc.comp_err = cpu_to_le16(comp_err);
+    desc_write(ring, ring->tail);
+    ring->tail = (ring->tail + 1) % ring->size;
+
+    /* return true if starting credit count */
+
+    return ring->credits++ == 0;
+}
+
+bool desc_ring_post_desc(DescRing *ring, int err)
+{
+    if (desc_ring_empty(ring)) {
+        DPRINTF("ERROR: ring[%d] trying to post desc to empty ring\n",
+                ring->index);
+        return false;
+    }
+
+    if (!desc_ring_base_addr_check(ring)) {
+        return false;
+    }
+
+    return __desc_ring_post_desc(ring, err);
+}
+
+static bool ring_pump(DescRing *ring)
+{
+    DescInfo *info;
+    bool primed = false;
+    int err;
+
+    /* If the ring has a consumer, call consumer for each
+     * desc starting at tail and stopping when tail reaches
+     * head (the empty ring condition).
+     */
+
+    if (ring->consume) {
+        while (ring->head != ring->tail) {
+            info = __desc_ring_fetch_desc(ring);
+            err = ring->consume(ring->r, info);
+            if (__desc_ring_post_desc(ring, err)) {
+                primed = true;
+            }
+        }
+    }
+
+    return primed;
+}
+
+bool desc_ring_set_head(DescRing *ring, uint32_t new)
+{
+    uint32_t tail = ring->tail;
+    uint32_t head = ring->head;
+
+    if (!desc_ring_base_addr_check(ring)) {
+        return false;
+    }
+
+    if (new >= ring->size) {
+        DPRINTF("ERROR: trying to set head (%d) past ring[%d] size (%d)\n",
+                new, ring->index, ring->size);
+        return false;
+    }
+
+    if (((head < tail) && ((new >= tail) || (new < head))) ||
+        ((head > tail) && ((new >= tail) && (new < head)))) {
+        DPRINTF("ERROR: trying to wrap ring[%d] "
+                "(head %d, tail %d, new head %d)\n",
+                ring->index, head, tail, new);
+        return false;
+    }
+
+    if (new == ring->head) {
+        DPRINTF("WARNING: setting head (%d) to current head position\n", new);
+    }
+
+    ring->head = new;
+
+    return ring_pump(ring);
+}
+
+uint32_t desc_ring_get_head(DescRing *ring)
+{
+    return ring->head;
+}
+
+uint32_t desc_ring_get_tail(DescRing *ring)
+{
+    return ring->tail;
+}
+
+void desc_ring_set_ctrl(DescRing *ring, uint32_t val)
+{
+    if (val & ROCKER_DMA_DESC_CTRL_RESET) {
+        DPRINTF("ring[%d] resetting\n", ring->index);
+        desc_ring_reset(ring);
+    }
+}
+
+bool desc_ring_ret_credits(DescRing *ring, uint32_t credits)
+{
+    if (credits > ring->credits) {
+        DPRINTF("ERROR: trying to return more credits (%d) "
+                "than are outstanding (%d)\n", credits, ring->credits);
+        ring->credits = 0;
+        return false;
+    }
+
+    ring->credits -= credits;
+
+    /* return true if credits are still outstanding */
+
+    return ring->credits > 0;
+}
+
+uint32_t desc_ring_get_credits(DescRing *ring)
+{
+    return ring->credits;
+}
+
+void desc_ring_set_consume(DescRing *ring, desc_ring_consume *consume,
+                           unsigned vector)
+{
+    ring->consume = consume;
+    ring->msix_vector = vector;
+}
+
+unsigned desc_ring_get_msix_vector(DescRing *ring)
+{
+    return ring->msix_vector;
+}
+
+DescRing *desc_ring_alloc(Rocker *r, int index)
+{
+    DescRing *ring;
+
+    ring = g_new0(DescRing, 1);
+    if (!ring) {
+        return NULL;
+    }
+
+    ring->r = r;
+    ring->index = index;
+
+    return ring;
+}
+
+void desc_ring_free(DescRing *ring)
+{
+    g_free(ring->info);
+    g_free(ring);
+}
+
+void desc_ring_reset(DescRing *ring)
+{
+    ring->base_addr = 0;
+    ring->size = 0;
+    ring->head = 0;
+    ring->tail = 0;
+    ring->ctrl = 0;
+    ring->credits = 0;
+}
diff --git a/hw/net/rocker/rocker_desc.h b/hw/net/rocker/rocker_desc.h
new file mode 100644
index 0000000..1dec335
--- /dev/null
+++ b/hw/net/rocker/rocker_desc.h
@@ -0,0 +1,52 @@
+/*
+ * QEMU rocker switch emulation - Descriptor ring support
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef ROCKER_DESC_H
+#define ROCKER_DESC_H
+
+#include "rocker_hw.h"
+
+typedef int (desc_ring_consume)(Rocker *r, DescInfo *info);
+
+uint16_t desc_buf_size(DescInfo *info);
+uint16_t desc_tlv_size(DescInfo *info);
+char *desc_get_buf(DescInfo *info, bool read_only);
+int desc_set_buf(DescInfo *info, size_t tlv_size);
+DescRing *desc_get_ring(DescInfo *info);
+
+int desc_ring_index(DescRing *ring);
+bool desc_ring_set_base_addr(DescRing *ring, uint64_t base_addr);
+uint64_t desc_ring_get_base_addr(DescRing *ring);
+bool desc_ring_set_size(DescRing *ring, uint32_t size);
+uint32_t desc_ring_get_size(DescRing *ring);
+bool desc_ring_set_head(DescRing *ring, uint32_t new);
+uint32_t desc_ring_get_head(DescRing *ring);
+uint32_t desc_ring_get_tail(DescRing *ring);
+void desc_ring_set_ctrl(DescRing *ring, uint32_t val);
+bool desc_ring_ret_credits(DescRing *ring, uint32_t credits);
+uint32_t desc_ring_get_credits(DescRing *ring);
+
+DescInfo *desc_ring_fetch_desc(DescRing *ring);
+bool desc_ring_post_desc(DescRing *ring, int status);
+
+void desc_ring_set_consume(DescRing *ring, desc_ring_consume *consume,
+                           unsigned vector);
+unsigned desc_ring_get_msix_vector(DescRing *ring);
+DescRing *desc_ring_alloc(Rocker *r, int index);
+void desc_ring_free(DescRing *ring);
+void desc_ring_reset(DescRing *ring);
+
+#endif
diff --git a/hw/net/rocker/rocker_fp.c b/hw/net/rocker/rocker_fp.c
new file mode 100644
index 0000000..1305ac3
--- /dev/null
+++ b/hw/net/rocker/rocker_fp.c
@@ -0,0 +1,269 @@
+/*
+ * QEMU rocker switch emulation - front-panel ports
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "net/clients.h"
+
+#include "rocker.h"
+#include "rocker_hw.h"
+#include "rocker_fp.h"
+#include "rocker_world.h"
+
+enum duplex {
+    DUPLEX_HALF = 0,
+    DUPLEX_FULL
+};
+
+struct fp_port {
+    Rocker *r;
+    World *world;
+    unsigned int index;
+    char *name;
+    uint32_t pport;
+    bool enabled;
+    uint32_t speed;
+    uint8_t duplex;
+    uint8_t autoneg;
+    uint8_t learning;
+    NICState *nic;
+    NICConf conf;
+};
+
+char *fp_port_get_name(FpPort *port)
+{
+    return port->name;
+}
+
+bool fp_port_get_link_up(FpPort *port)
+{
+    return !qemu_get_queue(port->nic)->link_down;
+}
+
+void fp_port_get_info(FpPort *port, RockerPortList *info)
+{
+    info->value->name = g_strdup(port->name);
+    info->value->enabled = port->enabled;
+    info->value->link_up = fp_port_get_link_up(port);
+    info->value->speed = port->speed;
+    info->value->duplex = port->duplex;
+    info->value->autoneg = port->autoneg;
+}
+
+void fp_port_get_macaddr(FpPort *port, MACAddr *macaddr)
+{
+    memcpy(macaddr->a, port->conf.macaddr.a, sizeof(macaddr->a));
+}
+
+void fp_port_set_macaddr(FpPort *port, MACAddr *macaddr)
+{
+/* XXX TODO implement and test setting mac addr
+ * XXX memcpy(port->conf.macaddr.a, macaddr.a, sizeof(port->conf.macaddr.a));
+ */
+}
+
+uint8_t fp_port_get_learning(FpPort *port)
+{
+    return port->learning;
+}
+
+void fp_port_set_learning(FpPort *port, uint8_t learning)
+{
+    port->learning = learning;
+}
+
+int fp_port_get_settings(FpPort *port, uint32_t *speed,
+                         uint8_t *duplex, uint8_t *autoneg)
+{
+    *speed = port->speed;
+    *duplex = port->duplex;
+    *autoneg = port->autoneg;
+
+    return ROCKER_OK;
+}
+
+int fp_port_set_settings(FpPort *port, uint32_t speed,
+                         uint8_t duplex, uint8_t autoneg)
+{
+    /* XXX validate inputs */
+
+    port->speed = speed;
+    port->duplex = duplex;
+    port->autoneg = autoneg;
+
+    return ROCKER_OK;
+}
+
+bool fp_port_from_pport(uint32_t pport, uint32_t *port)
+{
+    if (pport < 1 || pport > ROCKER_FP_PORTS_MAX) {
+        return false;
+    }
+    *port = pport - 1;
+    return true;
+}
+
+int fp_port_eg(FpPort *port, const struct iovec *iov, int iovcnt)
+{
+    NetClientState *nc = qemu_get_queue(port->nic);
+
+    if (port->enabled) {
+        qemu_sendv_packet(nc, iov, iovcnt);
+    }
+
+    return ROCKER_OK;
+}
+
+static ssize_t fp_port_receive_iov(NetClientState *nc, const struct iovec *iov,
+                                   int iovcnt)
+{
+    FpPort *port = qemu_get_nic_opaque(nc);
+
+    /* If the port is disabled, we want to drop this pkt
+     * now rather than queing it for later.  We don't want
+     * any stale pkts getting into the device when the port
+     * transitions to enabled.
+     */
+
+    if (!port->enabled) {
+        return -1;
+    }
+
+    return world_ingress(port->world, port->pport, iov, iovcnt);
+}
+
+static ssize_t fp_port_receive(NetClientState *nc, const uint8_t *buf,
+                               size_t size)
+{
+    const struct iovec iov = {
+        .iov_base = (uint8_t *)buf,
+        .iov_len = size
+    };
+
+    return fp_port_receive_iov(nc, &iov, 1);
+}
+
+static void fp_port_cleanup(NetClientState *nc)
+{
+}
+
+static void fp_port_set_link_status(NetClientState *nc)
+{
+    FpPort *port = qemu_get_nic_opaque(nc);
+
+    rocker_event_link_changed(port->r, port->pport, !nc->link_down);
+}
+
+static NetClientInfo fp_port_info = {
+    .type = NET_CLIENT_DRIVER_NIC,
+    .size = sizeof(NICState),
+    .receive = fp_port_receive,
+    .receive_iov = fp_port_receive_iov,
+    .cleanup = fp_port_cleanup,
+    .link_status_changed = fp_port_set_link_status,
+};
+
+World *fp_port_get_world(FpPort *port)
+{
+    return port->world;
+}
+
+void fp_port_set_world(FpPort *port, World *world)
+{
+    DPRINTF("port %d setting world \"%s\"\n", port->index, world_name(world));
+    port->world = world;
+}
+
+bool fp_port_check_world(FpPort *port, World *world)
+{
+    return port->world == world;
+}
+
+bool fp_port_enabled(FpPort *port)
+{
+    return port->enabled;
+}
+
+static void fp_port_set_link(FpPort *port, bool up)
+{
+    NetClientState *nc = qemu_get_queue(port->nic);
+
+    if (up == nc->link_down) {
+        nc->link_down = !up;
+        nc->info->link_status_changed(nc);
+    }
+}
+
+void fp_port_enable(FpPort *port)
+{
+    fp_port_set_link(port, true);
+    port->enabled = true;
+    DPRINTF("port %d enabled\n", port->index);
+}
+
+void fp_port_disable(FpPort *port)
+{
+    port->enabled = false;
+    fp_port_set_link(port, false);
+    DPRINTF("port %d disabled\n", port->index);
+}
+
+FpPort *fp_port_alloc(Rocker *r, char *sw_name,
+                      MACAddr *start_mac, unsigned int index,
+                      NICPeers *peers)
+{
+    FpPort *port = g_new0(FpPort, 1);
+
+    if (!port) {
+        return NULL;
+    }
+
+    port->r = r;
+    port->index = index;
+    port->pport = index + 1;
+
+    /* front-panel switch port names are 1-based */
+
+    port->name = g_strdup_printf("%sp%d", sw_name, port->pport);
+
+    memcpy(port->conf.macaddr.a, start_mac, sizeof(port->conf.macaddr.a));
+    port->conf.macaddr.a[5] += index;
+    port->conf.bootindex = -1;
+    port->conf.peers = *peers;
+
+    port->nic = qemu_new_nic(&fp_port_info, &port->conf,
+                             sw_name, NULL, port);
+    qemu_format_nic_info_str(qemu_get_queue(port->nic),
+                             port->conf.macaddr.a);
+
+    fp_port_reset(port);
+
+    return port;
+}
+
+void fp_port_free(FpPort *port)
+{
+    qemu_del_nic(port->nic);
+    g_free(port->name);
+    g_free(port);
+}
+
+void fp_port_reset(FpPort *port)
+{
+    fp_port_disable(port);
+    port->speed = 10000;   /* 10Gbps */
+    port->duplex = DUPLEX_FULL;
+    port->autoneg = 0;
+}
diff --git a/hw/net/rocker/rocker_fp.h b/hw/net/rocker/rocker_fp.h
new file mode 100644
index 0000000..dbe1dd3
--- /dev/null
+++ b/hw/net/rocker/rocker_fp.h
@@ -0,0 +1,54 @@
+/*
+ * QEMU rocker switch emulation - front-panel ports
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef ROCKER_FP_H
+#define ROCKER_FP_H
+
+#include "net/net.h"
+#include "qemu/iov.h"
+
+#define ROCKER_FP_PORTS_MAX 62
+
+typedef struct fp_port FpPort;
+
+int fp_port_eg(FpPort *port, const struct iovec *iov, int iovcnt);
+
+char *fp_port_get_name(FpPort *port);
+bool fp_port_get_link_up(FpPort *port);
+void fp_port_get_info(FpPort *port, RockerPortList *info);
+void fp_port_get_macaddr(FpPort *port, MACAddr *macaddr);
+void fp_port_set_macaddr(FpPort *port, MACAddr *macaddr);
+uint8_t fp_port_get_learning(FpPort *port);
+void fp_port_set_learning(FpPort *port, uint8_t learning);
+int fp_port_get_settings(FpPort *port, uint32_t *speed,
+                         uint8_t *duplex, uint8_t *autoneg);
+int fp_port_set_settings(FpPort *port, uint32_t speed,
+                         uint8_t duplex, uint8_t autoneg);
+bool fp_port_from_pport(uint32_t pport, uint32_t *port);
+World *fp_port_get_world(FpPort *port);
+void fp_port_set_world(FpPort *port, World *world);
+bool fp_port_check_world(FpPort *port, World *world);
+bool fp_port_enabled(FpPort *port);
+void fp_port_enable(FpPort *port);
+void fp_port_disable(FpPort *port);
+
+FpPort *fp_port_alloc(Rocker *r, char *sw_name,
+                      MACAddr *start_mac, unsigned int index,
+                      NICPeers *peers);
+void fp_port_free(FpPort *port);
+void fp_port_reset(FpPort *port);
+
+#endif /* ROCKER_FP_H */
diff --git a/hw/net/rocker/rocker_hw.h b/hw/net/rocker/rocker_hw.h
new file mode 100644
index 0000000..1786323
--- /dev/null
+++ b/hw/net/rocker/rocker_hw.h
@@ -0,0 +1,493 @@
+/*
+ * Rocker switch hardware register and descriptor definitions.
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
+ *
+ */
+
+#ifndef ROCKER_HW_H
+#define ROCKER_HW_H
+
+#define __le16 uint16_t
+#define __le32 uint32_t
+#define __le64 uint64_t
+
+/*
+ * Return codes
+ */
+
+enum {
+    ROCKER_OK = 0,
+    ROCKER_ENOENT = 2,
+    ROCKER_ENXIO = 6,
+    ROCKER_ENOMEM = 12,
+    ROCKER_EEXIST = 17,
+    ROCKER_EINVAL = 22,
+    ROCKER_EMSGSIZE = 90,
+    ROCKER_ENOTSUP = 95,
+    ROCKER_ENOBUFS = 105,
+};
+
+/*
+ * PCI configuration space
+ */
+
+#define ROCKER_PCI_REVISION             0x1
+#define ROCKER_PCI_BAR0_IDX             0
+#define ROCKER_PCI_BAR0_SIZE            0x2000
+#define ROCKER_PCI_MSIX_BAR_IDX         1
+#define ROCKER_PCI_MSIX_BAR_SIZE        0x2000
+#define ROCKER_PCI_MSIX_TABLE_OFFSET    0x0000
+#define ROCKER_PCI_MSIX_PBA_OFFSET      0x1000
+
+/*
+ * MSI-X vectors
+ */
+
+enum {
+    ROCKER_MSIX_VEC_CMD,
+    ROCKER_MSIX_VEC_EVENT,
+    ROCKER_MSIX_VEC_TEST,
+    ROCKER_MSIX_VEC_RESERVED0,
+    __ROCKER_MSIX_VEC_TX,
+    __ROCKER_MSIX_VEC_RX,
+#define ROCKER_MSIX_VEC_TX(port) \
+                (__ROCKER_MSIX_VEC_TX + ((port) * 2))
+#define ROCKER_MSIX_VEC_RX(port) \
+                (__ROCKER_MSIX_VEC_RX + ((port) * 2))
+#define ROCKER_MSIX_VEC_COUNT(portcnt) \
+                (ROCKER_MSIX_VEC_RX((portcnt) - 1) + 1)
+};
+
+/*
+ * Rocker bogus registers
+ */
+#define ROCKER_BOGUS_REG0               0x0000
+#define ROCKER_BOGUS_REG1               0x0004
+#define ROCKER_BOGUS_REG2               0x0008
+#define ROCKER_BOGUS_REG3               0x000c
+
+/*
+ * Rocker test registers
+ */
+#define ROCKER_TEST_REG                 0x0010
+#define ROCKER_TEST_REG64               0x0018  /* 8-byte */
+#define ROCKER_TEST_IRQ                 0x0020
+#define ROCKER_TEST_DMA_ADDR            0x0028  /* 8-byte */
+#define ROCKER_TEST_DMA_SIZE            0x0030
+#define ROCKER_TEST_DMA_CTRL            0x0034
+
+/*
+ * Rocker test register ctrl
+ */
+#define ROCKER_TEST_DMA_CTRL_CLEAR      (1 << 0)
+#define ROCKER_TEST_DMA_CTRL_FILL       (1 << 1)
+#define ROCKER_TEST_DMA_CTRL_INVERT     (1 << 2)
+
+/*
+ * Rocker DMA ring register offsets
+ */
+#define ROCKER_DMA_DESC_BASE            0x1000
+#define ROCKER_DMA_DESC_SIZE            32
+#define ROCKER_DMA_DESC_MASK            0x1F
+#define ROCKER_DMA_DESC_TOTAL_SIZE \
+    (ROCKER_DMA_DESC_SIZE * 64) /* 62 ports + event + cmd */
+#define ROCKER_DMA_DESC_ADDR_OFFSET     0x00     /* 8-byte */
+#define ROCKER_DMA_DESC_SIZE_OFFSET     0x08
+#define ROCKER_DMA_DESC_HEAD_OFFSET     0x0c
+#define ROCKER_DMA_DESC_TAIL_OFFSET     0x10
+#define ROCKER_DMA_DESC_CTRL_OFFSET     0x14
+#define ROCKER_DMA_DESC_CREDITS_OFFSET  0x18
+#define ROCKER_DMA_DESC_RSVD_OFFSET     0x1c
+
+/*
+ * Rocker dma ctrl register bits
+ */
+#define ROCKER_DMA_DESC_CTRL_RESET      (1 << 0)
+
+/*
+ * Rocker ring indices
+ */
+#define ROCKER_RING_CMD                 0
+#define ROCKER_RING_EVENT               1
+
+/*
+ * Helper macro to do convert a dma ring register
+ * to its index.  Based on the fact that the register
+ * group stride is 32 bytes.
+ */
+#define ROCKER_RING_INDEX(reg) ((reg >> 5) & 0x7F)
+
+/*
+ * Rocker DMA Descriptor
+ */
+
+typedef struct rocker_desc {
+    __le64 buf_addr;
+    uint64_t cookie;
+    __le16 buf_size;
+    __le16 tlv_size;
+    __le16 rsvd[5];   /* pad to 32 bytes */
+    __le16 comp_err;
+} __attribute__((packed, aligned(8))) RockerDesc;
+
+/*
+ * Rocker TLV type fields
+ */
+
+typedef struct rocker_tlv {
+    __le32 type;
+    __le16 len;
+    __le16 rsvd;
+} __attribute__((packed, aligned(8))) RockerTlv;
+
+/* cmd msg */
+enum {
+    ROCKER_TLV_CMD_UNSPEC,
+    ROCKER_TLV_CMD_TYPE,                /* u16 */
+    ROCKER_TLV_CMD_INFO,                /* nest */
+
+    __ROCKER_TLV_CMD_MAX,
+    ROCKER_TLV_CMD_MAX = __ROCKER_TLV_CMD_MAX - 1,
+};
+
+enum {
+    ROCKER_TLV_CMD_TYPE_UNSPEC,
+    ROCKER_TLV_CMD_TYPE_GET_PORT_SETTINGS,
+    ROCKER_TLV_CMD_TYPE_SET_PORT_SETTINGS,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_ADD,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_MOD,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_DEL,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_GET_STATS,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_ADD,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_DEL,
+    ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_GET_STATS,
+
+    __ROCKER_TLV_CMD_TYPE_MAX,
+    ROCKER_TLV_CMD_TYPE_MAX = __ROCKER_TLV_CMD_TYPE_MAX - 1,
+};
+
+/* cmd info nested for set/get port settings */
+enum {
+    ROCKER_TLV_CMD_PORT_SETTINGS_UNSPEC,
+    ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,         /* u32 */
+    ROCKER_TLV_CMD_PORT_SETTINGS_SPEED,         /* u32 */
+    ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX,        /* u8 */
+    ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG,       /* u8 */
+    ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,       /* binary */
+    ROCKER_TLV_CMD_PORT_SETTINGS_MODE,          /* u8 */
+    ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,      /* u8 */
+    ROCKER_TLV_CMD_PORT_SETTINGS_PHYS_NAME,     /* binary */
+
+    __ROCKER_TLV_CMD_PORT_SETTINGS_MAX,
+    ROCKER_TLV_CMD_PORT_SETTINGS_MAX = __ROCKER_TLV_CMD_PORT_SETTINGS_MAX - 1,
+};
+
+enum {
+    ROCKER_PORT_MODE_OF_DPA,
+};
+
+/* event msg */
+enum {
+    ROCKER_TLV_EVENT_UNSPEC,
+    ROCKER_TLV_EVENT_TYPE,              /* u16 */
+    ROCKER_TLV_EVENT_INFO,              /* nest */
+
+    __ROCKER_TLV_EVENT_MAX,
+    ROCKER_TLV_EVENT_MAX = __ROCKER_TLV_EVENT_MAX - 1,
+};
+
+enum {
+    ROCKER_TLV_EVENT_TYPE_UNSPEC,
+    ROCKER_TLV_EVENT_TYPE_LINK_CHANGED,
+    ROCKER_TLV_EVENT_TYPE_MAC_VLAN_SEEN,
+
+    __ROCKER_TLV_EVENT_TYPE_MAX,
+    ROCKER_TLV_EVENT_TYPE_MAX = __ROCKER_TLV_EVENT_TYPE_MAX - 1,
+};
+
+/* event info nested for link changed */
+enum {
+    ROCKER_TLV_EVENT_LINK_CHANGED_UNSPEC,
+    ROCKER_TLV_EVENT_LINK_CHANGED_PPORT,    /* u32 */
+    ROCKER_TLV_EVENT_LINK_CHANGED_LINKUP,   /* u8 */
+
+    __ROCKER_TLV_EVENT_LINK_CHANGED_MAX,
+    ROCKER_TLV_EVENT_LINK_CHANGED_MAX = __ROCKER_TLV_EVENT_LINK_CHANGED_MAX - 1,
+};
+
+/* event info nested for MAC/VLAN */
+enum {
+    ROCKER_TLV_EVENT_MAC_VLAN_UNSPEC,
+    ROCKER_TLV_EVENT_MAC_VLAN_PPORT,        /* u32 */
+    ROCKER_TLV_EVENT_MAC_VLAN_MAC,          /* binary */
+    ROCKER_TLV_EVENT_MAC_VLAN_VLAN_ID,      /* __be16 */
+
+    __ROCKER_TLV_EVENT_MAC_VLAN_MAX,
+    ROCKER_TLV_EVENT_MAC_VLAN_MAX = __ROCKER_TLV_EVENT_MAC_VLAN_MAX - 1,
+};
+
+/* Rx msg */
+enum {
+    ROCKER_TLV_RX_UNSPEC,
+    ROCKER_TLV_RX_FLAGS,                /* u16, see RX_FLAGS_ */
+    ROCKER_TLV_RX_CSUM,                 /* u16 */
+    ROCKER_TLV_RX_FRAG_ADDR,            /* u64 */
+    ROCKER_TLV_RX_FRAG_MAX_LEN,         /* u16 */
+    ROCKER_TLV_RX_FRAG_LEN,             /* u16 */
+
+    __ROCKER_TLV_RX_MAX,
+    ROCKER_TLV_RX_MAX = __ROCKER_TLV_RX_MAX - 1,
+};
+
+#define ROCKER_RX_FLAGS_IPV4                    (1 << 0)
+#define ROCKER_RX_FLAGS_IPV6                    (1 << 1)
+#define ROCKER_RX_FLAGS_CSUM_CALC               (1 << 2)
+#define ROCKER_RX_FLAGS_IPV4_CSUM_GOOD          (1 << 3)
+#define ROCKER_RX_FLAGS_IP_FRAG                 (1 << 4)
+#define ROCKER_RX_FLAGS_TCP                     (1 << 5)
+#define ROCKER_RX_FLAGS_UDP                     (1 << 6)
+#define ROCKER_RX_FLAGS_TCP_UDP_CSUM_GOOD       (1 << 7)
+#define ROCKER_RX_FLAGS_FWD_OFFLOAD             (1 << 8)
+
+/* Tx msg */
+enum {
+    ROCKER_TLV_TX_UNSPEC,
+    ROCKER_TLV_TX_OFFLOAD,              /* u8, see TX_OFFLOAD_ */
+    ROCKER_TLV_TX_L3_CSUM_OFF,          /* u16 */
+    ROCKER_TLV_TX_TSO_MSS,              /* u16 */
+    ROCKER_TLV_TX_TSO_HDR_LEN,          /* u16 */
+    ROCKER_TLV_TX_FRAGS,                /* array */
+
+    __ROCKER_TLV_TX_MAX,
+    ROCKER_TLV_TX_MAX = __ROCKER_TLV_TX_MAX - 1,
+};
+
+#define ROCKER_TX_OFFLOAD_NONE          0
+#define ROCKER_TX_OFFLOAD_IP_CSUM       1
+#define ROCKER_TX_OFFLOAD_TCP_UDP_CSUM  2
+#define ROCKER_TX_OFFLOAD_L3_CSUM       3
+#define ROCKER_TX_OFFLOAD_TSO           4
+
+#define ROCKER_TX_FRAGS_MAX             16
+
+enum {
+    ROCKER_TLV_TX_FRAG_UNSPEC,
+    ROCKER_TLV_TX_FRAG,                 /* nest */
+
+    __ROCKER_TLV_TX_FRAG_MAX,
+    ROCKER_TLV_TX_FRAG_MAX = __ROCKER_TLV_TX_FRAG_MAX - 1,
+};
+
+enum {
+    ROCKER_TLV_TX_FRAG_ATTR_UNSPEC,
+    ROCKER_TLV_TX_FRAG_ATTR_ADDR,       /* u64 */
+    ROCKER_TLV_TX_FRAG_ATTR_LEN,        /* u16 */
+
+    __ROCKER_TLV_TX_FRAG_ATTR_MAX,
+    ROCKER_TLV_TX_FRAG_ATTR_MAX = __ROCKER_TLV_TX_FRAG_ATTR_MAX - 1,
+};
+
+/*
+ * cmd info nested for OF-DPA msgs
+ */
+
+enum {
+    ROCKER_TLV_OF_DPA_UNSPEC,
+    ROCKER_TLV_OF_DPA_TABLE_ID,            /* u16 */
+    ROCKER_TLV_OF_DPA_PRIORITY,            /* u32 */
+    ROCKER_TLV_OF_DPA_HARDTIME,            /* u32 */
+    ROCKER_TLV_OF_DPA_IDLETIME,            /* u32 */
+    ROCKER_TLV_OF_DPA_COOKIE,              /* u64 */
+    ROCKER_TLV_OF_DPA_IN_PPORT,            /* u32 */
+    ROCKER_TLV_OF_DPA_IN_PPORT_MASK,       /* u32 */
+    ROCKER_TLV_OF_DPA_OUT_PPORT,           /* u32 */
+    ROCKER_TLV_OF_DPA_GOTO_TABLE_ID,       /* u16 */
+    ROCKER_TLV_OF_DPA_GROUP_ID,            /* u32 */
+    ROCKER_TLV_OF_DPA_GROUP_ID_LOWER,      /* u32 */
+    ROCKER_TLV_OF_DPA_GROUP_COUNT,         /* u16 */
+    ROCKER_TLV_OF_DPA_GROUP_IDS,           /* u32 array */
+    ROCKER_TLV_OF_DPA_VLAN_ID,             /* __be16 */
+    ROCKER_TLV_OF_DPA_VLAN_ID_MASK,        /* __be16 */
+    ROCKER_TLV_OF_DPA_VLAN_PCP,            /* __be16 */
+    ROCKER_TLV_OF_DPA_VLAN_PCP_MASK,       /* __be16 */
+    ROCKER_TLV_OF_DPA_VLAN_PCP_ACTION,     /* u8 */
+    ROCKER_TLV_OF_DPA_NEW_VLAN_ID,         /* __be16 */
+    ROCKER_TLV_OF_DPA_NEW_VLAN_PCP,        /* u8 */
+    ROCKER_TLV_OF_DPA_TUNNEL_ID,           /* u32 */
+    ROCKER_TLV_OF_DPA_TUNNEL_LPORT,        /* u32 */
+    ROCKER_TLV_OF_DPA_ETHERTYPE,           /* __be16 */
+    ROCKER_TLV_OF_DPA_DST_MAC,             /* binary */
+    ROCKER_TLV_OF_DPA_DST_MAC_MASK,        /* binary */
+    ROCKER_TLV_OF_DPA_SRC_MAC,             /* binary */
+    ROCKER_TLV_OF_DPA_SRC_MAC_MASK,        /* binary */
+    ROCKER_TLV_OF_DPA_IP_PROTO,            /* u8 */
+    ROCKER_TLV_OF_DPA_IP_PROTO_MASK,       /* u8 */
+    ROCKER_TLV_OF_DPA_IP_DSCP,             /* u8 */
+    ROCKER_TLV_OF_DPA_IP_DSCP_MASK,        /* u8 */
+    ROCKER_TLV_OF_DPA_IP_DSCP_ACTION,      /* u8 */
+    ROCKER_TLV_OF_DPA_NEW_IP_DSCP,         /* u8 */
+    ROCKER_TLV_OF_DPA_IP_ECN,              /* u8 */
+    ROCKER_TLV_OF_DPA_IP_ECN_MASK,         /* u8 */
+    ROCKER_TLV_OF_DPA_DST_IP,              /* __be32 */
+    ROCKER_TLV_OF_DPA_DST_IP_MASK,         /* __be32 */
+    ROCKER_TLV_OF_DPA_SRC_IP,              /* __be32 */
+    ROCKER_TLV_OF_DPA_SRC_IP_MASK,         /* __be32 */
+    ROCKER_TLV_OF_DPA_DST_IPV6,            /* binary */
+    ROCKER_TLV_OF_DPA_DST_IPV6_MASK,       /* binary */
+    ROCKER_TLV_OF_DPA_SRC_IPV6,            /* binary */
+    ROCKER_TLV_OF_DPA_SRC_IPV6_MASK,       /* binary */
+    ROCKER_TLV_OF_DPA_SRC_ARP_IP,          /* __be32 */
+    ROCKER_TLV_OF_DPA_SRC_ARP_IP_MASK,     /* __be32 */
+    ROCKER_TLV_OF_DPA_L4_DST_PORT,         /* __be16 */
+    ROCKER_TLV_OF_DPA_L4_DST_PORT_MASK,    /* __be16 */
+    ROCKER_TLV_OF_DPA_L4_SRC_PORT,         /* __be16 */
+    ROCKER_TLV_OF_DPA_L4_SRC_PORT_MASK,    /* __be16 */
+    ROCKER_TLV_OF_DPA_ICMP_TYPE,           /* u8 */
+    ROCKER_TLV_OF_DPA_ICMP_TYPE_MASK,      /* u8 */
+    ROCKER_TLV_OF_DPA_ICMP_CODE,           /* u8 */
+    ROCKER_TLV_OF_DPA_ICMP_CODE_MASK,      /* u8 */
+    ROCKER_TLV_OF_DPA_IPV6_LABEL,          /* __be32 */
+    ROCKER_TLV_OF_DPA_IPV6_LABEL_MASK,     /* __be32 */
+    ROCKER_TLV_OF_DPA_QUEUE_ID_ACTION,     /* u8 */
+    ROCKER_TLV_OF_DPA_NEW_QUEUE_ID,        /* u8 */
+    ROCKER_TLV_OF_DPA_CLEAR_ACTIONS,       /* u32 */
+    ROCKER_TLV_OF_DPA_POP_VLAN,            /* u8 */
+    ROCKER_TLV_OF_DPA_TTL_CHECK,           /* u8 */
+    ROCKER_TLV_OF_DPA_COPY_CPU_ACTION,     /* u8 */
+
+    __ROCKER_TLV_OF_DPA_MAX,
+    ROCKER_TLV_OF_DPA_MAX = __ROCKER_TLV_OF_DPA_MAX - 1,
+};
+
+/*
+ * OF-DPA table IDs
+ */
+
+enum rocker_of_dpa_table_id {
+    ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT = 0,
+    ROCKER_OF_DPA_TABLE_ID_VLAN = 10,
+    ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC = 20,
+    ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING = 30,
+    ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING = 40,
+    ROCKER_OF_DPA_TABLE_ID_BRIDGING = 50,
+    ROCKER_OF_DPA_TABLE_ID_ACL_POLICY = 60,
+};
+
+/*
+ * OF-DPA flow stats
+ */
+
+enum {
+    ROCKER_TLV_OF_DPA_FLOW_STAT_UNSPEC,
+    ROCKER_TLV_OF_DPA_FLOW_STAT_DURATION,    /* u32 */
+    ROCKER_TLV_OF_DPA_FLOW_STAT_RX_PKTS,     /* u64 */
+    ROCKER_TLV_OF_DPA_FLOW_STAT_TX_PKTS,     /* u64 */
+
+    __ROCKER_TLV_OF_DPA_FLOW_STAT_MAX,
+    ROCKER_TLV_OF_DPA_FLOW_STAT_MAX = __ROCKER_TLV_OF_DPA_FLOW_STAT_MAX - 1,
+};
+
+/*
+ * OF-DPA group types
+ */
+
+enum rocker_of_dpa_group_type {
+    ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE = 0,
+    ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE,
+    ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST,
+    ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST,
+    ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD,
+    ROCKER_OF_DPA_GROUP_TYPE_L3_INTERFACE,
+    ROCKER_OF_DPA_GROUP_TYPE_L3_MCAST,
+    ROCKER_OF_DPA_GROUP_TYPE_L3_ECMP,
+    ROCKER_OF_DPA_GROUP_TYPE_L2_OVERLAY,
+};
+
+/*
+ * OF-DPA group L2 overlay types
+ */
+
+enum rocker_of_dpa_overlay_type {
+    ROCKER_OF_DPA_OVERLAY_TYPE_FLOOD_UCAST = 0,
+    ROCKER_OF_DPA_OVERLAY_TYPE_FLOOD_MCAST,
+    ROCKER_OF_DPA_OVERLAY_TYPE_MCAST_UCAST,
+    ROCKER_OF_DPA_OVERLAY_TYPE_MCAST_MCAST,
+};
+
+/*
+ * OF-DPA group ID encoding
+ */
+
+#define ROCKER_GROUP_TYPE_SHIFT 28
+#define ROCKER_GROUP_TYPE_MASK 0xf0000000
+#define ROCKER_GROUP_VLAN_ID_SHIFT 16
+#define ROCKER_GROUP_VLAN_ID_MASK 0x0fff0000
+#define ROCKER_GROUP_PORT_SHIFT 0
+#define ROCKER_GROUP_PORT_MASK 0x0000ffff
+#define ROCKER_GROUP_TUNNEL_ID_SHIFT 12
+#define ROCKER_GROUP_TUNNEL_ID_MASK 0x0ffff000
+#define ROCKER_GROUP_SUBTYPE_SHIFT 10
+#define ROCKER_GROUP_SUBTYPE_MASK 0x00000c00
+#define ROCKER_GROUP_INDEX_SHIFT 0
+#define ROCKER_GROUP_INDEX_MASK 0x0000ffff
+#define ROCKER_GROUP_INDEX_LONG_SHIFT 0
+#define ROCKER_GROUP_INDEX_LONG_MASK 0x0fffffff
+
+#define ROCKER_GROUP_TYPE_GET(group_id) \
+    (((group_id) & ROCKER_GROUP_TYPE_MASK) >> ROCKER_GROUP_TYPE_SHIFT)
+#define ROCKER_GROUP_TYPE_SET(type) \
+    (((type) << ROCKER_GROUP_TYPE_SHIFT) & ROCKER_GROUP_TYPE_MASK)
+#define ROCKER_GROUP_VLAN_GET(group_id) \
+    (((group_id) & ROCKER_GROUP_VLAN_ID_MASK) >> ROCKER_GROUP_VLAN_ID_SHIFT)
+#define ROCKER_GROUP_VLAN_SET(vlan_id) \
+    (((vlan_id) << ROCKER_GROUP_VLAN_ID_SHIFT) & ROCKER_GROUP_VLAN_ID_MASK)
+#define ROCKER_GROUP_PORT_GET(group_id) \
+    (((group_id) & ROCKER_GROUP_PORT_MASK) >> ROCKER_GROUP_PORT_SHIFT)
+#define ROCKER_GROUP_PORT_SET(port) \
+    (((port) << ROCKER_GROUP_PORT_SHIFT) & ROCKER_GROUP_PORT_MASK)
+#define ROCKER_GROUP_INDEX_GET(group_id) \
+    (((group_id) & ROCKER_GROUP_INDEX_MASK) >> ROCKER_GROUP_INDEX_SHIFT)
+#define ROCKER_GROUP_INDEX_SET(index) \
+    (((index) << ROCKER_GROUP_INDEX_SHIFT) & ROCKER_GROUP_INDEX_MASK)
+#define ROCKER_GROUP_INDEX_LONG_GET(group_id) \
+    (((group_id) & ROCKER_GROUP_INDEX_LONG_MASK) >> \
+     ROCKER_GROUP_INDEX_LONG_SHIFT)
+#define ROCKER_GROUP_INDEX_LONG_SET(index) \
+    (((index) << ROCKER_GROUP_INDEX_LONG_SHIFT) & \
+     ROCKER_GROUP_INDEX_LONG_MASK)
+
+#define ROCKER_GROUP_NONE 0
+#define ROCKER_GROUP_L2_INTERFACE(vlan_id, port) \
+    (ROCKER_GROUP_TYPE_SET(ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE) |\
+     ROCKER_GROUP_VLAN_SET(ntohs(vlan_id)) | ROCKER_GROUP_PORT_SET(port))
+#define ROCKER_GROUP_L2_REWRITE(index) \
+    (ROCKER_GROUP_TYPE_SET(ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE) |\
+     ROCKER_GROUP_INDEX_LONG_SET(index))
+#define ROCKER_GROUP_L2_MCAST(vlan_id, index) \
+    (ROCKER_GROUP_TYPE_SET(ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST) |\
+     ROCKER_GROUP_VLAN_SET(ntohs(vlan_id)) | ROCKER_GROUP_INDEX_SET(index))
+#define ROCKER_GROUP_L2_FLOOD(vlan_id, index) \
+    (ROCKER_GROUP_TYPE_SET(ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD) |\
+     ROCKER_GROUP_VLAN_SET(ntohs(vlan_id)) | ROCKER_GROUP_INDEX_SET(index))
+#define ROCKER_GROUP_L3_UNICAST(index) \
+    (ROCKER_GROUP_TYPE_SET(ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST) |\
+     ROCKER_GROUP_INDEX_LONG_SET(index))
+
+/*
+ * Rocker general purpose registers
+ */
+#define ROCKER_CONTROL                  0x0300
+#define ROCKER_PORT_PHYS_COUNT          0x0304
+#define ROCKER_PORT_PHYS_LINK_STATUS    0x0310 /* 8-byte */
+#define ROCKER_PORT_PHYS_ENABLE         0x0318 /* 8-byte */
+#define ROCKER_SWITCH_ID                0x0320 /* 8-byte */
+
+/*
+ * Rocker control bits
+ */
+#define ROCKER_CONTROL_RESET            (1 << 0)
+
+#endif /* ROCKER_HW_H */
diff --git a/hw/net/rocker/rocker_of_dpa.c b/hw/net/rocker/rocker_of_dpa.c
new file mode 100644
index 0000000..9b1e0d2
--- /dev/null
+++ b/hw/net/rocker/rocker_of_dpa.c
@@ -0,0 +1,2626 @@
+/*
+ * QEMU rocker switch emulation - OF-DPA flow processing support
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "net/eth.h"
+#include "qemu/iov.h"
+#include "qemu/timer.h"
+#include "qmp-commands.h"
+
+#include "rocker.h"
+#include "rocker_hw.h"
+#include "rocker_fp.h"
+#include "rocker_tlv.h"
+#include "rocker_world.h"
+#include "rocker_desc.h"
+#include "rocker_of_dpa.h"
+
+static const MACAddr zero_mac = { .a = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } };
+static const MACAddr ff_mac =   { .a = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff } };
+
+typedef struct of_dpa {
+    World *world;
+    GHashTable *flow_tbl;
+    GHashTable *group_tbl;
+    unsigned int flow_tbl_max_size;
+    unsigned int group_tbl_max_size;
+} OfDpa;
+
+/* flow_key stolen mostly from OVS
+ *
+ * Note: fields that compare with network packet header fields
+ * are stored in network order (BE) to avoid per-packet field
+ * byte-swaps.
+ */
+
+typedef struct of_dpa_flow_key {
+    uint32_t in_pport;               /* ingress port */
+    uint32_t tunnel_id;              /* overlay tunnel id */
+    uint32_t tbl_id;                 /* table id */
+    struct {
+        __be16 vlan_id;              /* 0 if no VLAN */
+        MACAddr src;                 /* ethernet source address */
+        MACAddr dst;                 /* ethernet destination address */
+        __be16 type;                 /* ethernet frame type */
+    } eth;
+    struct {
+        uint8_t proto;               /* IP protocol or ARP opcode */
+        uint8_t tos;                 /* IP ToS */
+        uint8_t ttl;                 /* IP TTL/hop limit */
+        uint8_t frag;                /* one of FRAG_TYPE_* */
+    } ip;
+    union {
+        struct {
+            struct {
+                __be32 src;          /* IP source address */
+                __be32 dst;          /* IP destination address */
+            } addr;
+            union {
+                struct {
+                    __be16 src;      /* TCP/UDP/SCTP source port */
+                    __be16 dst;      /* TCP/UDP/SCTP destination port */
+                    __be16 flags;    /* TCP flags */
+                } tp;
+                struct {
+                    MACAddr sha;     /* ARP source hardware address */
+                    MACAddr tha;     /* ARP target hardware address */
+                } arp;
+            };
+        } ipv4;
+        struct {
+            struct {
+                Ipv6Addr src;       /* IPv6 source address */
+                Ipv6Addr dst;       /* IPv6 destination address */
+            } addr;
+            __be32 label;            /* IPv6 flow label */
+            struct {
+                __be16 src;          /* TCP/UDP/SCTP source port */
+                __be16 dst;          /* TCP/UDP/SCTP destination port */
+                __be16 flags;        /* TCP flags */
+            } tp;
+            struct {
+                Ipv6Addr target;    /* ND target address */
+                MACAddr sll;         /* ND source link layer address */
+                MACAddr tll;         /* ND target link layer address */
+            } nd;
+        } ipv6;
+    };
+    int width;                       /* how many uint64_t's in key? */
+} OfDpaFlowKey;
+
+/* Width of key which includes field 'f' in u64s, rounded up */
+#define FLOW_KEY_WIDTH(f) \
+    DIV_ROUND_UP(offsetof(OfDpaFlowKey, f) + sizeof(((OfDpaFlowKey *)0)->f), \
+    sizeof(uint64_t))
+
+typedef struct of_dpa_flow_action {
+    uint32_t goto_tbl;
+    struct {
+        uint32_t group_id;
+        uint32_t tun_log_lport;
+        __be16 vlan_id;
+    } write;
+    struct {
+        __be16 new_vlan_id;
+        uint32_t out_pport;
+        uint8_t copy_to_cpu;
+        __be16 vlan_id;
+    } apply;
+} OfDpaFlowAction;
+
+typedef struct of_dpa_flow {
+    uint32_t lpm;
+    uint32_t priority;
+    uint32_t hardtime;
+    uint32_t idletime;
+    uint64_t cookie;
+    OfDpaFlowKey key;
+    OfDpaFlowKey mask;
+    OfDpaFlowAction action;
+    struct {
+        uint64_t hits;
+        int64_t install_time;
+        int64_t refresh_time;
+        uint64_t rx_pkts;
+        uint64_t tx_pkts;
+    } stats;
+} OfDpaFlow;
+
+typedef struct of_dpa_flow_pkt_fields {
+    uint32_t tunnel_id;
+    struct eth_header *ethhdr;
+    __be16 *h_proto;
+    struct vlan_header *vlanhdr;
+    struct ip_header *ipv4hdr;
+    struct ip6_header *ipv6hdr;
+    Ipv6Addr *ipv6_src_addr;
+    Ipv6Addr *ipv6_dst_addr;
+} OfDpaFlowPktFields;
+
+typedef struct of_dpa_flow_context {
+    uint32_t in_pport;
+    uint32_t tunnel_id;
+    struct iovec *iov;
+    int iovcnt;
+    struct eth_header ethhdr_rewrite;
+    struct vlan_header vlanhdr_rewrite;
+    struct vlan_header vlanhdr;
+    OfDpa *of_dpa;
+    OfDpaFlowPktFields fields;
+    OfDpaFlowAction action_set;
+} OfDpaFlowContext;
+
+typedef struct of_dpa_flow_match {
+    OfDpaFlowKey value;
+    OfDpaFlow *best;
+} OfDpaFlowMatch;
+
+typedef struct of_dpa_group {
+    uint32_t id;
+    union {
+        struct {
+            uint32_t out_pport;
+            uint8_t pop_vlan;
+        } l2_interface;
+        struct {
+            uint32_t group_id;
+            MACAddr src_mac;
+            MACAddr dst_mac;
+            __be16 vlan_id;
+        } l2_rewrite;
+        struct {
+            uint16_t group_count;
+            uint32_t *group_ids;
+        } l2_flood;
+        struct {
+            uint32_t group_id;
+            MACAddr src_mac;
+            MACAddr dst_mac;
+            __be16 vlan_id;
+            uint8_t ttl_check;
+        } l3_unicast;
+    };
+} OfDpaGroup;
+
+static int of_dpa_mask2prefix(__be32 mask)
+{
+    int i;
+    int count = 32;
+
+    for (i = 0; i < 32; i++) {
+        if (!(ntohl(mask) & ((2 << i) - 1))) {
+            count--;
+        }
+    }
+
+    return count;
+}
+
+#if defined(DEBUG_ROCKER)
+static void of_dpa_flow_key_dump(OfDpaFlowKey *key, OfDpaFlowKey *mask)
+{
+    char buf[512], *b = buf, *mac;
+
+    b += sprintf(b, " tbl %2d", key->tbl_id);
+
+    if (key->in_pport || (mask && mask->in_pport)) {
+        b += sprintf(b, " in_pport %2d", key->in_pport);
+        if (mask && mask->in_pport != 0xffffffff) {
+            b += sprintf(b, "/0x%08x", key->in_pport);
+        }
+    }
+
+    if (key->tunnel_id || (mask && mask->tunnel_id)) {
+        b += sprintf(b, " tun %8d", key->tunnel_id);
+        if (mask && mask->tunnel_id != 0xffffffff) {
+            b += sprintf(b, "/0x%08x", key->tunnel_id);
+        }
+    }
+
+    if (key->eth.vlan_id || (mask && mask->eth.vlan_id)) {
+        b += sprintf(b, " vlan %4d", ntohs(key->eth.vlan_id));
+        if (mask && mask->eth.vlan_id != 0xffff) {
+            b += sprintf(b, "/0x%04x", ntohs(key->eth.vlan_id));
+        }
+    }
+
+    if (memcmp(key->eth.src.a, zero_mac.a, ETH_ALEN) ||
+        (mask && memcmp(mask->eth.src.a, zero_mac.a, ETH_ALEN))) {
+        mac = qemu_mac_strdup_printf(key->eth.src.a);
+        b += sprintf(b, " src %s", mac);
+        g_free(mac);
+        if (mask && memcmp(mask->eth.src.a, ff_mac.a, ETH_ALEN)) {
+            mac = qemu_mac_strdup_printf(mask->eth.src.a);
+            b += sprintf(b, "/%s", mac);
+            g_free(mac);
+        }
+    }
+
+    if (memcmp(key->eth.dst.a, zero_mac.a, ETH_ALEN) ||
+        (mask && memcmp(mask->eth.dst.a, zero_mac.a, ETH_ALEN))) {
+        mac = qemu_mac_strdup_printf(key->eth.dst.a);
+        b += sprintf(b, " dst %s", mac);
+        g_free(mac);
+        if (mask && memcmp(mask->eth.dst.a, ff_mac.a, ETH_ALEN)) {
+            mac = qemu_mac_strdup_printf(mask->eth.dst.a);
+            b += sprintf(b, "/%s", mac);
+            g_free(mac);
+        }
+    }
+
+    if (key->eth.type || (mask && mask->eth.type)) {
+        b += sprintf(b, " type 0x%04x", ntohs(key->eth.type));
+        if (mask && mask->eth.type != 0xffff) {
+            b += sprintf(b, "/0x%04x", ntohs(mask->eth.type));
+        }
+        switch (ntohs(key->eth.type)) {
+        case 0x0800:
+        case 0x86dd:
+            if (key->ip.proto || (mask && mask->ip.proto)) {
+                b += sprintf(b, " ip proto %2d", key->ip.proto);
+                if (mask && mask->ip.proto != 0xff) {
+                    b += sprintf(b, "/0x%02x", mask->ip.proto);
+                }
+            }
+            if (key->ip.tos || (mask && mask->ip.tos)) {
+                b += sprintf(b, " ip tos %2d", key->ip.tos);
+                if (mask && mask->ip.tos != 0xff) {
+                    b += sprintf(b, "/0x%02x", mask->ip.tos);
+                }
+            }
+            break;
+        }
+        switch (ntohs(key->eth.type)) {
+        case 0x0800:
+            if (key->ipv4.addr.dst || (mask && mask->ipv4.addr.dst)) {
+                b += sprintf(b, " dst %s",
+                    inet_ntoa(*(struct in_addr *)&key->ipv4.addr.dst));
+                if (mask) {
+                    b += sprintf(b, "/%d",
+                                 of_dpa_mask2prefix(mask->ipv4.addr.dst));
+                }
+            }
+            break;
+        }
+    }
+
+    DPRINTF("%s\n", buf);
+}
+#else
+#define of_dpa_flow_key_dump(k, m)
+#endif
+
+static void _of_dpa_flow_match(void *key, void *value, void *user_data)
+{
+    OfDpaFlow *flow = value;
+    OfDpaFlowMatch *match = user_data;
+    uint64_t *k = (uint64_t *)&flow->key;
+    uint64_t *m = (uint64_t *)&flow->mask;
+    uint64_t *v = (uint64_t *)&match->value;
+    int i;
+
+    if (flow->key.tbl_id == match->value.tbl_id) {
+        of_dpa_flow_key_dump(&flow->key, &flow->mask);
+    }
+
+    if (flow->key.width > match->value.width) {
+        return;
+    }
+
+    for (i = 0; i < flow->key.width; i++, k++, m++, v++) {
+        if ((~*k & *m & *v) | (*k & *m & ~*v)) {
+            return;
+        }
+    }
+
+    DPRINTF("match\n");
+
+    if (!match->best ||
+        flow->priority > match->best->priority ||
+        flow->lpm > match->best->lpm) {
+        match->best = flow;
+    }
+}
+
+static OfDpaFlow *of_dpa_flow_match(OfDpa *of_dpa, OfDpaFlowMatch *match)
+{
+    DPRINTF("\nnew search\n");
+    of_dpa_flow_key_dump(&match->value, NULL);
+
+    g_hash_table_foreach(of_dpa->flow_tbl, _of_dpa_flow_match, match);
+
+    return match->best;
+}
+
+static OfDpaFlow *of_dpa_flow_find(OfDpa *of_dpa, uint64_t cookie)
+{
+    return g_hash_table_lookup(of_dpa->flow_tbl, &cookie);
+}
+
+static int of_dpa_flow_add(OfDpa *of_dpa, OfDpaFlow *flow)
+{
+    g_hash_table_insert(of_dpa->flow_tbl, &flow->cookie, flow);
+
+    return ROCKER_OK;
+}
+
+static void of_dpa_flow_del(OfDpa *of_dpa, OfDpaFlow *flow)
+{
+    g_hash_table_remove(of_dpa->flow_tbl, &flow->cookie);
+}
+
+static OfDpaFlow *of_dpa_flow_alloc(uint64_t cookie)
+{
+    OfDpaFlow *flow;
+    int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) / 1000;
+
+    flow = g_new0(OfDpaFlow, 1);
+    if (!flow) {
+        return NULL;
+    }
+
+    flow->cookie = cookie;
+    flow->mask.tbl_id = 0xffffffff;
+
+    flow->stats.install_time = flow->stats.refresh_time = now;
+
+    return flow;
+}
+
+static void of_dpa_flow_pkt_hdr_reset(OfDpaFlowContext *fc)
+{
+    OfDpaFlowPktFields *fields = &fc->fields;
+
+    fc->iov[0].iov_base = fields->ethhdr;
+    fc->iov[0].iov_len = sizeof(struct eth_header);
+    fc->iov[1].iov_base = fields->vlanhdr;
+    fc->iov[1].iov_len = fields->vlanhdr ? sizeof(struct vlan_header) : 0;
+}
+
+static void of_dpa_flow_pkt_parse(OfDpaFlowContext *fc,
+                                  const struct iovec *iov, int iovcnt)
+{
+    OfDpaFlowPktFields *fields = &fc->fields;
+    size_t sofar = 0;
+    int i;
+
+    sofar += sizeof(struct eth_header);
+    if (iov->iov_len < sofar) {
+        DPRINTF("flow_pkt_parse underrun on eth_header\n");
+        return;
+    }
+
+    fields->ethhdr = iov->iov_base;
+    fields->h_proto = &fields->ethhdr->h_proto;
+
+    if (ntohs(*fields->h_proto) == ETH_P_VLAN) {
+        sofar += sizeof(struct vlan_header);
+        if (iov->iov_len < sofar) {
+            DPRINTF("flow_pkt_parse underrun on vlan_header\n");
+            return;
+        }
+        fields->vlanhdr = (struct vlan_header *)(fields->ethhdr + 1);
+        fields->h_proto = &fields->vlanhdr->h_proto;
+    }
+
+    switch (ntohs(*fields->h_proto)) {
+    case ETH_P_IP:
+        sofar += sizeof(struct ip_header);
+        if (iov->iov_len < sofar) {
+            DPRINTF("flow_pkt_parse underrun on ip_header\n");
+            return;
+        }
+        fields->ipv4hdr = (struct ip_header *)(fields->h_proto + 1);
+        break;
+    case ETH_P_IPV6:
+        sofar += sizeof(struct ip6_header);
+        if (iov->iov_len < sofar) {
+            DPRINTF("flow_pkt_parse underrun on ip6_header\n");
+            return;
+        }
+        fields->ipv6hdr = (struct ip6_header *)(fields->h_proto + 1);
+        break;
+    }
+
+    /* To facilitate (potential) VLAN tag insertion, Make a
+     * copy of the iov and insert two new vectors at the
+     * beginning for eth hdr and vlan hdr.  No data is copied,
+     * just the vectors.
+     */
+
+    of_dpa_flow_pkt_hdr_reset(fc);
+
+    fc->iov[2].iov_base = fields->h_proto + 1;
+    fc->iov[2].iov_len = iov->iov_len - fc->iov[0].iov_len - fc->iov[1].iov_len;
+
+    for (i = 1; i < iovcnt; i++) {
+        fc->iov[i+2] = iov[i];
+    }
+
+    fc->iovcnt = iovcnt + 2;
+}
+
+static void of_dpa_flow_pkt_insert_vlan(OfDpaFlowContext *fc, __be16 vlan_id)
+{
+    OfDpaFlowPktFields *fields = &fc->fields;
+    uint16_t h_proto = fields->ethhdr->h_proto;
+
+    if (fields->vlanhdr) {
+        DPRINTF("flow_pkt_insert_vlan packet already has vlan\n");
+        return;
+    }
+
+    fields->ethhdr->h_proto = htons(ETH_P_VLAN);
+    fields->vlanhdr = &fc->vlanhdr;
+    fields->vlanhdr->h_tci = vlan_id;
+    fields->vlanhdr->h_proto = h_proto;
+    fields->h_proto = &fields->vlanhdr->h_proto;
+
+    fc->iov[1].iov_base = fields->vlanhdr;
+    fc->iov[1].iov_len = sizeof(struct vlan_header);
+}
+
+static void of_dpa_flow_pkt_strip_vlan(OfDpaFlowContext *fc)
+{
+    OfDpaFlowPktFields *fields = &fc->fields;
+
+    if (!fields->vlanhdr) {
+        return;
+    }
+
+    fc->iov[0].iov_len -= sizeof(fields->ethhdr->h_proto);
+    fc->iov[1].iov_base = fields->h_proto;
+    fc->iov[1].iov_len = sizeof(fields->ethhdr->h_proto);
+}
+
+static void of_dpa_flow_pkt_hdr_rewrite(OfDpaFlowContext *fc,
+                                        uint8_t *src_mac, uint8_t *dst_mac,
+                                        __be16 vlan_id)
+{
+    OfDpaFlowPktFields *fields = &fc->fields;
+
+    if (src_mac || dst_mac) {
+        memcpy(&fc->ethhdr_rewrite, fields->ethhdr, sizeof(struct eth_header));
+        if (src_mac && memcmp(src_mac, zero_mac.a, ETH_ALEN)) {
+            memcpy(fc->ethhdr_rewrite.h_source, src_mac, ETH_ALEN);
+        }
+        if (dst_mac && memcmp(dst_mac, zero_mac.a, ETH_ALEN)) {
+            memcpy(fc->ethhdr_rewrite.h_dest, dst_mac, ETH_ALEN);
+        }
+        fc->iov[0].iov_base = &fc->ethhdr_rewrite;
+    }
+
+    if (vlan_id && fields->vlanhdr) {
+        fc->vlanhdr_rewrite = fc->vlanhdr;
+        fc->vlanhdr_rewrite.h_tci = vlan_id;
+        fc->iov[1].iov_base = &fc->vlanhdr_rewrite;
+    }
+}
+
+static void of_dpa_flow_ig_tbl(OfDpaFlowContext *fc, uint32_t tbl_id);
+
+static void of_dpa_ig_port_build_match(OfDpaFlowContext *fc,
+                                       OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT;
+    match->value.in_pport = fc->in_pport;
+    match->value.width = FLOW_KEY_WIDTH(tbl_id);
+}
+
+static void of_dpa_ig_port_miss(OfDpaFlowContext *fc)
+{
+    uint32_t port;
+
+    /* The default on miss is for packets from physical ports
+     * to go to the VLAN Flow Table. There is no default rule
+     * for packets from logical ports, which are dropped on miss.
+     */
+
+    if (fp_port_from_pport(fc->in_pport, &port)) {
+        of_dpa_flow_ig_tbl(fc, ROCKER_OF_DPA_TABLE_ID_VLAN);
+    }
+}
+
+static void of_dpa_vlan_build_match(OfDpaFlowContext *fc,
+                                    OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_VLAN;
+    match->value.in_pport = fc->in_pport;
+    if (fc->fields.vlanhdr) {
+        match->value.eth.vlan_id = fc->fields.vlanhdr->h_tci;
+    }
+    match->value.width = FLOW_KEY_WIDTH(eth.vlan_id);
+}
+
+static void of_dpa_vlan_insert(OfDpaFlowContext *fc,
+                               OfDpaFlow *flow)
+{
+    if (flow->action.apply.new_vlan_id) {
+        of_dpa_flow_pkt_insert_vlan(fc, flow->action.apply.new_vlan_id);
+    }
+}
+
+static void of_dpa_term_mac_build_match(OfDpaFlowContext *fc,
+                                        OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC;
+    match->value.in_pport = fc->in_pport;
+    match->value.eth.type = *fc->fields.h_proto;
+    match->value.eth.vlan_id = fc->fields.vlanhdr->h_tci;
+    memcpy(match->value.eth.dst.a, fc->fields.ethhdr->h_dest,
+           sizeof(match->value.eth.dst.a));
+    match->value.width = FLOW_KEY_WIDTH(eth.type);
+}
+
+static void of_dpa_term_mac_miss(OfDpaFlowContext *fc)
+{
+    of_dpa_flow_ig_tbl(fc, ROCKER_OF_DPA_TABLE_ID_BRIDGING);
+}
+
+static void of_dpa_apply_actions(OfDpaFlowContext *fc,
+                                 OfDpaFlow *flow)
+{
+    fc->action_set.apply.copy_to_cpu = flow->action.apply.copy_to_cpu;
+    fc->action_set.apply.vlan_id = flow->key.eth.vlan_id;
+}
+
+static void of_dpa_bridging_build_match(OfDpaFlowContext *fc,
+                                        OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_BRIDGING;
+    if (fc->fields.vlanhdr) {
+        match->value.eth.vlan_id = fc->fields.vlanhdr->h_tci;
+    } else if (fc->tunnel_id) {
+        match->value.tunnel_id = fc->tunnel_id;
+    }
+    memcpy(match->value.eth.dst.a, fc->fields.ethhdr->h_dest,
+           sizeof(match->value.eth.dst.a));
+    match->value.width = FLOW_KEY_WIDTH(eth.dst);
+}
+
+static void of_dpa_bridging_learn(OfDpaFlowContext *fc,
+                                  OfDpaFlow *dst_flow)
+{
+    OfDpaFlowMatch match = { { 0, }, };
+    OfDpaFlow *flow;
+    uint8_t *addr;
+    uint16_t vlan_id;
+    int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) / 1000;
+    int64_t refresh_delay = 1;
+
+    /* Do a lookup in bridge table by src_mac/vlan */
+
+    addr = fc->fields.ethhdr->h_source;
+    vlan_id = fc->fields.vlanhdr->h_tci;
+
+    match.value.tbl_id = ROCKER_OF_DPA_TABLE_ID_BRIDGING;
+    match.value.eth.vlan_id = vlan_id;
+    memcpy(match.value.eth.dst.a, addr, sizeof(match.value.eth.dst.a));
+    match.value.width = FLOW_KEY_WIDTH(eth.dst);
+
+    flow = of_dpa_flow_match(fc->of_dpa, &match);
+    if (flow) {
+        if (!memcmp(flow->mask.eth.dst.a, ff_mac.a,
+                    sizeof(flow->mask.eth.dst.a))) {
+            /* src_mac/vlan already learned; if in_port and out_port
+             * don't match, the end station has moved and the port
+             * needs updating */
+            /* XXX implement the in_port/out_port check */
+            if (now - flow->stats.refresh_time < refresh_delay) {
+                return;
+            }
+            flow->stats.refresh_time = now;
+        }
+    }
+
+    /* Let driver know about mac/vlan.  This may be a new mac/vlan
+     * or a refresh of existing mac/vlan that's been hit after the
+     * refresh_delay.
+     */
+
+    rocker_event_mac_vlan_seen(world_rocker(fc->of_dpa->world),
+                               fc->in_pport, addr, vlan_id);
+}
+
+static void of_dpa_bridging_miss(OfDpaFlowContext *fc)
+{
+    of_dpa_bridging_learn(fc, NULL);
+    of_dpa_flow_ig_tbl(fc, ROCKER_OF_DPA_TABLE_ID_ACL_POLICY);
+}
+
+static void of_dpa_bridging_action_write(OfDpaFlowContext *fc,
+                                         OfDpaFlow *flow)
+{
+    if (flow->action.write.group_id != ROCKER_GROUP_NONE) {
+        fc->action_set.write.group_id = flow->action.write.group_id;
+    }
+    fc->action_set.write.tun_log_lport = flow->action.write.tun_log_lport;
+}
+
+static void of_dpa_unicast_routing_build_match(OfDpaFlowContext *fc,
+                                               OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING;
+    match->value.eth.type = *fc->fields.h_proto;
+    if (fc->fields.ipv4hdr) {
+        match->value.ipv4.addr.dst = fc->fields.ipv4hdr->ip_dst;
+    }
+    if (fc->fields.ipv6_dst_addr) {
+        memcpy(&match->value.ipv6.addr.dst, fc->fields.ipv6_dst_addr,
+               sizeof(match->value.ipv6.addr.dst));
+    }
+    match->value.width = FLOW_KEY_WIDTH(ipv6.addr.dst);
+}
+
+static void of_dpa_unicast_routing_miss(OfDpaFlowContext *fc)
+{
+    of_dpa_flow_ig_tbl(fc, ROCKER_OF_DPA_TABLE_ID_ACL_POLICY);
+}
+
+static void of_dpa_unicast_routing_action_write(OfDpaFlowContext *fc,
+                                                OfDpaFlow *flow)
+{
+    if (flow->action.write.group_id != ROCKER_GROUP_NONE) {
+        fc->action_set.write.group_id = flow->action.write.group_id;
+    }
+}
+
+static void
+of_dpa_multicast_routing_build_match(OfDpaFlowContext *fc,
+                                     OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING;
+    match->value.eth.type = *fc->fields.h_proto;
+    match->value.eth.vlan_id = fc->fields.vlanhdr->h_tci;
+    if (fc->fields.ipv4hdr) {
+        match->value.ipv4.addr.src = fc->fields.ipv4hdr->ip_src;
+        match->value.ipv4.addr.dst = fc->fields.ipv4hdr->ip_dst;
+    }
+    if (fc->fields.ipv6_src_addr) {
+        memcpy(&match->value.ipv6.addr.src, fc->fields.ipv6_src_addr,
+               sizeof(match->value.ipv6.addr.src));
+    }
+    if (fc->fields.ipv6_dst_addr) {
+        memcpy(&match->value.ipv6.addr.dst, fc->fields.ipv6_dst_addr,
+               sizeof(match->value.ipv6.addr.dst));
+    }
+    match->value.width = FLOW_KEY_WIDTH(ipv6.addr.dst);
+}
+
+static void of_dpa_multicast_routing_miss(OfDpaFlowContext *fc)
+{
+    of_dpa_flow_ig_tbl(fc, ROCKER_OF_DPA_TABLE_ID_ACL_POLICY);
+}
+
+static void
+of_dpa_multicast_routing_action_write(OfDpaFlowContext *fc,
+                                      OfDpaFlow *flow)
+{
+    if (flow->action.write.group_id != ROCKER_GROUP_NONE) {
+        fc->action_set.write.group_id = flow->action.write.group_id;
+    }
+    fc->action_set.write.vlan_id = flow->action.write.vlan_id;
+}
+
+static void of_dpa_acl_build_match(OfDpaFlowContext *fc,
+                                   OfDpaFlowMatch *match)
+{
+    match->value.tbl_id = ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
+    match->value.in_pport = fc->in_pport;
+    memcpy(match->value.eth.src.a, fc->fields.ethhdr->h_source,
+           sizeof(match->value.eth.src.a));
+    memcpy(match->value.eth.dst.a, fc->fields.ethhdr->h_dest,
+           sizeof(match->value.eth.dst.a));
+    match->value.eth.type = *fc->fields.h_proto;
+    match->value.eth.vlan_id = fc->fields.vlanhdr->h_tci;
+    match->value.width = FLOW_KEY_WIDTH(eth.type);
+    if (fc->fields.ipv4hdr) {
+        match->value.ip.proto = fc->fields.ipv4hdr->ip_p;
+        match->value.ip.tos = fc->fields.ipv4hdr->ip_tos;
+        match->value.width = FLOW_KEY_WIDTH(ip.tos);
+    } else if (fc->fields.ipv6hdr) {
+        match->value.ip.proto =
+            fc->fields.ipv6hdr->ip6_ctlun.ip6_un1.ip6_un1_nxt;
+        match->value.ip.tos = 0; /* XXX what goes here? */
+        match->value.width = FLOW_KEY_WIDTH(ip.tos);
+    }
+}
+
+static void of_dpa_eg(OfDpaFlowContext *fc);
+static void of_dpa_acl_hit(OfDpaFlowContext *fc,
+                           OfDpaFlow *dst_flow)
+{
+    of_dpa_eg(fc);
+}
+
+static void of_dpa_acl_action_write(OfDpaFlowContext *fc,
+                                    OfDpaFlow *flow)
+{
+    if (flow->action.write.group_id != ROCKER_GROUP_NONE) {
+        fc->action_set.write.group_id = flow->action.write.group_id;
+    }
+}
+
+static void of_dpa_drop(OfDpaFlowContext *fc)
+{
+    /* drop packet */
+}
+
+static OfDpaGroup *of_dpa_group_find(OfDpa *of_dpa,
+                                              uint32_t group_id)
+{
+    return g_hash_table_lookup(of_dpa->group_tbl, &group_id);
+}
+
+static int of_dpa_group_add(OfDpa *of_dpa, OfDpaGroup *group)
+{
+    g_hash_table_insert(of_dpa->group_tbl, &group->id, group);
+
+    return 0;
+}
+
+#if 0
+static int of_dpa_group_mod(OfDpa *of_dpa, OfDpaGroup *group)
+{
+    OfDpaGroup *old_group = of_dpa_group_find(of_dpa, group->id);
+
+    if (!old_group) {
+        return -ENOENT;
+    }
+
+    /* XXX */
+
+    return 0;
+}
+#endif
+
+static int of_dpa_group_del(OfDpa *of_dpa, OfDpaGroup *group)
+{
+    g_hash_table_remove(of_dpa->group_tbl, &group->id);
+
+    return 0;
+}
+
+#if 0
+static int of_dpa_group_get_stats(OfDpa *of_dpa, uint32_t id)
+{
+    OfDpaGroup *group = of_dpa_group_find(of_dpa, id);
+
+    if (!group) {
+        return -ENOENT;
+    }
+
+    /* XXX get/return stats */
+
+    return 0;
+}
+#endif
+
+static OfDpaGroup *of_dpa_group_alloc(uint32_t id)
+{
+    OfDpaGroup *group = g_new0(OfDpaGroup, 1);
+
+    if (!group) {
+        return NULL;
+    }
+
+    group->id = id;
+
+    return group;
+}
+
+static void of_dpa_output_l2_interface(OfDpaFlowContext *fc,
+                                       OfDpaGroup *group)
+{
+    uint8_t copy_to_cpu = fc->action_set.apply.copy_to_cpu;
+
+    if (group->l2_interface.pop_vlan) {
+        of_dpa_flow_pkt_strip_vlan(fc);
+    }
+
+    /* Note: By default, and as per the OpenFlow 1.3.1
+     * specification, a packet cannot be forwarded back
+     * to the IN_PORT from which it came in. An action
+     * bucket that specifies the particular packet's
+     * egress port is not evaluated.
+     */
+
+    if (group->l2_interface.out_pport == 0) {
+        rx_produce(fc->of_dpa->world, fc->in_pport, fc->iov, fc->iovcnt,
+                   copy_to_cpu);
+    } else if (group->l2_interface.out_pport != fc->in_pport) {
+        rocker_port_eg(world_rocker(fc->of_dpa->world),
+                       group->l2_interface.out_pport,
+                       fc->iov, fc->iovcnt);
+    }
+}
+
+static void of_dpa_output_l2_rewrite(OfDpaFlowContext *fc,
+                                     OfDpaGroup *group)
+{
+    OfDpaGroup *l2_group =
+        of_dpa_group_find(fc->of_dpa, group->l2_rewrite.group_id);
+
+    if (!l2_group) {
+        return;
+    }
+
+    of_dpa_flow_pkt_hdr_rewrite(fc, group->l2_rewrite.src_mac.a,
+                         group->l2_rewrite.dst_mac.a,
+                         group->l2_rewrite.vlan_id);
+    of_dpa_output_l2_interface(fc, l2_group);
+}
+
+static void of_dpa_output_l2_flood(OfDpaFlowContext *fc,
+                                   OfDpaGroup *group)
+{
+    OfDpaGroup *l2_group;
+    int i;
+
+    for (i = 0; i < group->l2_flood.group_count; i++) {
+        of_dpa_flow_pkt_hdr_reset(fc);
+        l2_group = of_dpa_group_find(fc->of_dpa, group->l2_flood.group_ids[i]);
+        if (!l2_group) {
+            continue;
+        }
+        switch (ROCKER_GROUP_TYPE_GET(l2_group->id)) {
+        case ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE:
+            of_dpa_output_l2_interface(fc, l2_group);
+            break;
+        case ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE:
+            of_dpa_output_l2_rewrite(fc, l2_group);
+            break;
+        }
+    }
+}
+
+static void of_dpa_output_l3_unicast(OfDpaFlowContext *fc, OfDpaGroup *group)
+{
+    OfDpaGroup *l2_group =
+        of_dpa_group_find(fc->of_dpa, group->l3_unicast.group_id);
+
+    if (!l2_group) {
+        return;
+    }
+
+    of_dpa_flow_pkt_hdr_rewrite(fc, group->l3_unicast.src_mac.a,
+                                group->l3_unicast.dst_mac.a,
+                                group->l3_unicast.vlan_id);
+    /* XXX need ttl_check */
+    of_dpa_output_l2_interface(fc, l2_group);
+}
+
+static void of_dpa_eg(OfDpaFlowContext *fc)
+{
+    OfDpaFlowAction *set = &fc->action_set;
+    OfDpaGroup *group;
+    uint32_t group_id;
+
+    /* send a copy of pkt to CPU (controller)? */
+
+    if (set->apply.copy_to_cpu) {
+        group_id = ROCKER_GROUP_L2_INTERFACE(set->apply.vlan_id, 0);
+        group = of_dpa_group_find(fc->of_dpa, group_id);
+        if (group) {
+            of_dpa_output_l2_interface(fc, group);
+            of_dpa_flow_pkt_hdr_reset(fc);
+        }
+    }
+
+    /* process group write actions */
+
+    if (!set->write.group_id) {
+        return;
+    }
+
+    group = of_dpa_group_find(fc->of_dpa, set->write.group_id);
+    if (!group) {
+        return;
+    }
+
+    switch (ROCKER_GROUP_TYPE_GET(group->id)) {
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE:
+        of_dpa_output_l2_interface(fc, group);
+        break;
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE:
+        of_dpa_output_l2_rewrite(fc, group);
+        break;
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD:
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST:
+        of_dpa_output_l2_flood(fc, group);
+        break;
+    case ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST:
+        of_dpa_output_l3_unicast(fc, group);
+        break;
+    }
+}
+
+typedef struct of_dpa_flow_tbl_ops {
+    void (*build_match)(OfDpaFlowContext *fc, OfDpaFlowMatch *match);
+    void (*hit)(OfDpaFlowContext *fc, OfDpaFlow *flow);
+    void (*miss)(OfDpaFlowContext *fc);
+    void (*hit_no_goto)(OfDpaFlowContext *fc);
+    void (*action_apply)(OfDpaFlowContext *fc, OfDpaFlow *flow);
+    void (*action_write)(OfDpaFlowContext *fc, OfDpaFlow *flow);
+} OfDpaFlowTblOps;
+
+static OfDpaFlowTblOps of_dpa_tbl_ops[] = {
+    [ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT] = {
+        .build_match = of_dpa_ig_port_build_match,
+        .miss = of_dpa_ig_port_miss,
+        .hit_no_goto = of_dpa_drop,
+    },
+    [ROCKER_OF_DPA_TABLE_ID_VLAN] = {
+        .build_match = of_dpa_vlan_build_match,
+        .hit_no_goto = of_dpa_drop,
+        .action_apply = of_dpa_vlan_insert,
+    },
+    [ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC] = {
+        .build_match = of_dpa_term_mac_build_match,
+        .miss = of_dpa_term_mac_miss,
+        .hit_no_goto = of_dpa_drop,
+        .action_apply = of_dpa_apply_actions,
+    },
+    [ROCKER_OF_DPA_TABLE_ID_BRIDGING] = {
+        .build_match = of_dpa_bridging_build_match,
+        .hit = of_dpa_bridging_learn,
+        .miss = of_dpa_bridging_miss,
+        .hit_no_goto = of_dpa_drop,
+        .action_apply = of_dpa_apply_actions,
+        .action_write = of_dpa_bridging_action_write,
+    },
+    [ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING] = {
+        .build_match = of_dpa_unicast_routing_build_match,
+        .miss = of_dpa_unicast_routing_miss,
+        .hit_no_goto = of_dpa_drop,
+        .action_write = of_dpa_unicast_routing_action_write,
+    },
+    [ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING] = {
+        .build_match = of_dpa_multicast_routing_build_match,
+        .miss = of_dpa_multicast_routing_miss,
+        .hit_no_goto = of_dpa_drop,
+        .action_write = of_dpa_multicast_routing_action_write,
+    },
+    [ROCKER_OF_DPA_TABLE_ID_ACL_POLICY] = {
+        .build_match = of_dpa_acl_build_match,
+        .hit = of_dpa_acl_hit,
+        .miss = of_dpa_eg,
+        .action_apply = of_dpa_apply_actions,
+        .action_write = of_dpa_acl_action_write,
+    },
+};
+
+static void of_dpa_flow_ig_tbl(OfDpaFlowContext *fc, uint32_t tbl_id)
+{
+    OfDpaFlowTblOps *ops = &of_dpa_tbl_ops[tbl_id];
+    OfDpaFlowMatch match = { { 0, }, };
+    OfDpaFlow *flow;
+
+    if (ops->build_match) {
+        ops->build_match(fc, &match);
+    } else {
+        return;
+    }
+
+    flow = of_dpa_flow_match(fc->of_dpa, &match);
+    if (!flow) {
+        if (ops->miss) {
+            ops->miss(fc);
+        }
+        return;
+    }
+
+    flow->stats.hits++;
+
+    if (ops->action_apply) {
+        ops->action_apply(fc, flow);
+    }
+
+    if (ops->action_write) {
+        ops->action_write(fc, flow);
+    }
+
+    if (ops->hit) {
+        ops->hit(fc, flow);
+    }
+
+    if (flow->action.goto_tbl) {
+        of_dpa_flow_ig_tbl(fc, flow->action.goto_tbl);
+    } else if (ops->hit_no_goto) {
+        ops->hit_no_goto(fc);
+    }
+
+    /* drop packet */
+}
+
+static ssize_t of_dpa_ig(World *world, uint32_t pport,
+                         const struct iovec *iov, int iovcnt)
+{
+    struct iovec iov_copy[iovcnt + 2];
+    OfDpaFlowContext fc = {
+        .of_dpa = world_private(world),
+        .in_pport = pport,
+        .iov = iov_copy,
+        .iovcnt = iovcnt + 2,
+    };
+
+    of_dpa_flow_pkt_parse(&fc, iov, iovcnt);
+    of_dpa_flow_ig_tbl(&fc, ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT);
+
+    return iov_size(iov, iovcnt);
+}
+
+#define ROCKER_TUNNEL_LPORT 0x00010000
+
+static int of_dpa_cmd_add_ig_port(OfDpaFlow *flow, RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    bool overlay_tunnel;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]) {
+        return -ROCKER_EINVAL;
+    }
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT;
+    key->width = FLOW_KEY_WIDTH(tbl_id);
+
+    key->in_pport = rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT]);
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT_MASK]) {
+        mask->in_pport =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT_MASK]);
+    }
+
+    overlay_tunnel = !!(key->in_pport & ROCKER_TUNNEL_LPORT);
+
+    action->goto_tbl =
+        rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]);
+
+    if (!overlay_tunnel && action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_VLAN) {
+        return -ROCKER_EINVAL;
+    }
+
+    if (overlay_tunnel && action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_BRIDGING) {
+        return -ROCKER_EINVAL;
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_vlan(OfDpaFlow *flow, RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    uint32_t port;
+    bool untagged;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]) {
+        DPRINTF("Must give in_pport and vlan_id to install VLAN tbl entry\n");
+        return -ROCKER_EINVAL;
+    }
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_VLAN;
+    key->width = FLOW_KEY_WIDTH(eth.vlan_id);
+
+    key->in_pport = rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT]);
+    if (!fp_port_from_pport(key->in_pport, &port)) {
+        DPRINTF("in_pport (%d) not a front-panel port\n", key->in_pport);
+        return -ROCKER_EINVAL;
+    }
+    mask->in_pport = 0xffffffff;
+
+    key->eth.vlan_id = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID_MASK]) {
+        mask->eth.vlan_id =
+            rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID_MASK]);
+    }
+
+    if (key->eth.vlan_id) {
+        untagged = false; /* filtering */
+    } else {
+        untagged = true;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]) {
+        action->goto_tbl =
+            rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]);
+        if (action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC) {
+            DPRINTF("Goto tbl (%d) must be TERM_MAC\n", action->goto_tbl);
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (untagged) {
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_NEW_VLAN_ID]) {
+            DPRINTF("Must specify new vlan_id if untagged\n");
+            return -ROCKER_EINVAL;
+        }
+        action->apply.new_vlan_id =
+            rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_NEW_VLAN_ID]);
+        if (1 > ntohs(action->apply.new_vlan_id) ||
+            ntohs(action->apply.new_vlan_id) > 4095) {
+            DPRINTF("New vlan_id (%d) must be between 1 and 4095\n",
+                    ntohs(action->apply.new_vlan_id));
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_term_mac(OfDpaFlow *flow, RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    const MACAddr ipv4_mcast = { .a = { 0x01, 0x00, 0x5e, 0x00, 0x00, 0x00 } };
+    const MACAddr ipv4_mask =  { .a = { 0xff, 0xff, 0xff, 0x80, 0x00, 0x00 } };
+    const MACAddr ipv6_mcast = { .a = { 0x33, 0x33, 0x00, 0x00, 0x00, 0x00 } };
+    const MACAddr ipv6_mask =  { .a = { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 } };
+    uint32_t port;
+    bool unicast = false;
+    bool multicast = false;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT_MASK] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC_MASK] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID_MASK]) {
+        return -ROCKER_EINVAL;
+    }
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC;
+    key->width = FLOW_KEY_WIDTH(eth.type);
+
+    key->in_pport = rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT]);
+    if (!fp_port_from_pport(key->in_pport, &port)) {
+        return -ROCKER_EINVAL;
+    }
+    mask->in_pport =
+        rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT_MASK]);
+
+    key->eth.type = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE]);
+    if (key->eth.type != htons(0x0800) && key->eth.type != htons(0x86dd)) {
+        return -ROCKER_EINVAL;
+    }
+    mask->eth.type = htons(0xffff);
+
+    memcpy(key->eth.dst.a,
+           rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]),
+           sizeof(key->eth.dst.a));
+    memcpy(mask->eth.dst.a,
+           rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC_MASK]),
+           sizeof(mask->eth.dst.a));
+
+    if ((key->eth.dst.a[0] & 0x01) == 0x00) {
+        unicast = true;
+    }
+
+    /* only two wildcard rules are acceptable for IPv4 and IPv6 multicast */
+    if (memcmp(key->eth.dst.a, ipv4_mcast.a, sizeof(key->eth.dst.a)) == 0 &&
+        memcmp(mask->eth.dst.a, ipv4_mask.a, sizeof(mask->eth.dst.a)) == 0) {
+        multicast = true;
+    }
+    if (memcmp(key->eth.dst.a, ipv6_mcast.a, sizeof(key->eth.dst.a)) == 0 &&
+        memcmp(mask->eth.dst.a, ipv6_mask.a, sizeof(mask->eth.dst.a)) == 0) {
+        multicast = true;
+    }
+
+    if (!unicast && !multicast) {
+        return -ROCKER_EINVAL;
+    }
+
+    key->eth.vlan_id = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+    mask->eth.vlan_id =
+        rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID_MASK]);
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]) {
+        action->goto_tbl =
+            rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]);
+
+        if (action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING &&
+            action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING) {
+            return -ROCKER_EINVAL;
+        }
+
+        if (unicast &&
+            action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING) {
+            return -ROCKER_EINVAL;
+        }
+
+        if (multicast &&
+            action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING) {
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_COPY_CPU_ACTION]) {
+        action->apply.copy_to_cpu =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_COPY_CPU_ACTION]);
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_bridging(OfDpaFlow *flow, RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    bool unicast = false;
+    bool dst_mac = false;
+    bool dst_mac_mask = false;
+    enum {
+        BRIDGING_MODE_UNKNOWN,
+        BRIDGING_MODE_VLAN_UCAST,
+        BRIDGING_MODE_VLAN_MCAST,
+        BRIDGING_MODE_VLAN_DFLT,
+        BRIDGING_MODE_TUNNEL_UCAST,
+        BRIDGING_MODE_TUNNEL_MCAST,
+        BRIDGING_MODE_TUNNEL_DFLT,
+    } mode = BRIDGING_MODE_UNKNOWN;
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_BRIDGING;
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]) {
+        key->eth.vlan_id =
+            rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+        mask->eth.vlan_id = 0xffff;
+        key->width = FLOW_KEY_WIDTH(eth.vlan_id);
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_TUNNEL_ID]) {
+        key->tunnel_id =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_TUNNEL_ID]);
+        mask->tunnel_id = 0xffffffff;
+        key->width = FLOW_KEY_WIDTH(tunnel_id);
+    }
+
+    /* can't do VLAN bridging and tunnel bridging at same time */
+    if (key->eth.vlan_id && key->tunnel_id) {
+        DPRINTF("can't do VLAN bridging and tunnel bridging at same time\n");
+        return -ROCKER_EINVAL;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]) {
+        memcpy(key->eth.dst.a,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]),
+               sizeof(key->eth.dst.a));
+        key->width = FLOW_KEY_WIDTH(eth.dst);
+        dst_mac = true;
+        unicast = (key->eth.dst.a[0] & 0x01) == 0x00;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC_MASK]) {
+        memcpy(mask->eth.dst.a,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC_MASK]),
+               sizeof(mask->eth.dst.a));
+        key->width = FLOW_KEY_WIDTH(eth.dst);
+        dst_mac_mask = true;
+    } else if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]) {
+        memcpy(mask->eth.dst.a, ff_mac.a, sizeof(mask->eth.dst.a));
+    }
+
+    if (key->eth.vlan_id) {
+        if (dst_mac && !dst_mac_mask) {
+            mode = unicast ? BRIDGING_MODE_VLAN_UCAST :
+                             BRIDGING_MODE_VLAN_MCAST;
+        } else if ((dst_mac && dst_mac_mask) || !dst_mac) {
+            mode = BRIDGING_MODE_VLAN_DFLT;
+        }
+    } else if (key->tunnel_id) {
+        if (dst_mac && !dst_mac_mask) {
+            mode = unicast ? BRIDGING_MODE_TUNNEL_UCAST :
+                             BRIDGING_MODE_TUNNEL_MCAST;
+        } else if ((dst_mac && dst_mac_mask) || !dst_mac) {
+            mode = BRIDGING_MODE_TUNNEL_DFLT;
+        }
+    }
+
+    if (mode == BRIDGING_MODE_UNKNOWN) {
+        DPRINTF("Unknown bridging mode\n");
+        return -ROCKER_EINVAL;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]) {
+        action->goto_tbl =
+            rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]);
+        if (action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_ACL_POLICY) {
+            DPRINTF("Briding goto tbl must be ACL policy\n");
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]) {
+        action->write.group_id =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]);
+        switch (mode) {
+        case BRIDGING_MODE_VLAN_UCAST:
+            if (ROCKER_GROUP_TYPE_GET(action->write.group_id) !=
+                ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE) {
+                DPRINTF("Bridging mode vlan ucast needs L2 "
+                        "interface group (0x%08x)\n",
+                        action->write.group_id);
+                return -ROCKER_EINVAL;
+            }
+            break;
+        case BRIDGING_MODE_VLAN_MCAST:
+            if (ROCKER_GROUP_TYPE_GET(action->write.group_id) !=
+                ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST) {
+                DPRINTF("Bridging mode vlan mcast needs L2 "
+                        "mcast group (0x%08x)\n",
+                        action->write.group_id);
+                return -ROCKER_EINVAL;
+            }
+            break;
+        case BRIDGING_MODE_VLAN_DFLT:
+            if (ROCKER_GROUP_TYPE_GET(action->write.group_id) !=
+                ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD) {
+                DPRINTF("Bridging mode vlan dflt needs L2 "
+                        "flood group (0x%08x)\n",
+                        action->write.group_id);
+                return -ROCKER_EINVAL;
+            }
+            break;
+        case BRIDGING_MODE_TUNNEL_MCAST:
+            if (ROCKER_GROUP_TYPE_GET(action->write.group_id) !=
+                ROCKER_OF_DPA_GROUP_TYPE_L2_OVERLAY) {
+                DPRINTF("Bridging mode tunnel mcast needs L2 "
+                        "overlay group (0x%08x)\n",
+                        action->write.group_id);
+                return -ROCKER_EINVAL;
+            }
+            break;
+        case BRIDGING_MODE_TUNNEL_DFLT:
+            if (ROCKER_GROUP_TYPE_GET(action->write.group_id) !=
+                ROCKER_OF_DPA_GROUP_TYPE_L2_OVERLAY) {
+                DPRINTF("Bridging mode tunnel dflt needs L2 "
+                        "overlay group (0x%08x)\n",
+                        action->write.group_id);
+                return -ROCKER_EINVAL;
+            }
+            break;
+        default:
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_TUNNEL_LPORT]) {
+        action->write.tun_log_lport =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_TUNNEL_LPORT]);
+        if (mode != BRIDGING_MODE_TUNNEL_UCAST) {
+            DPRINTF("Have tunnel logical port but not "
+                    "in bridging tunnel mode\n");
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_COPY_CPU_ACTION]) {
+        action->apply.copy_to_cpu =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_COPY_CPU_ACTION]);
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_unicast_routing(OfDpaFlow *flow,
+                                          RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    enum {
+        UNICAST_ROUTING_MODE_UNKNOWN,
+        UNICAST_ROUTING_MODE_IPV4,
+        UNICAST_ROUTING_MODE_IPV6,
+    } mode = UNICAST_ROUTING_MODE_UNKNOWN;
+    uint8_t type;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE]) {
+        return -ROCKER_EINVAL;
+    }
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING;
+    key->width = FLOW_KEY_WIDTH(ipv6.addr.dst);
+
+    key->eth.type = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE]);
+    switch (ntohs(key->eth.type)) {
+    case 0x0800:
+        mode = UNICAST_ROUTING_MODE_IPV4;
+        break;
+    case 0x86dd:
+        mode = UNICAST_ROUTING_MODE_IPV6;
+        break;
+    default:
+        return -ROCKER_EINVAL;
+    }
+    mask->eth.type = htons(0xffff);
+
+    switch (mode) {
+    case UNICAST_ROUTING_MODE_IPV4:
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_DST_IP]) {
+            return -ROCKER_EINVAL;
+        }
+        key->ipv4.addr.dst =
+            rocker_tlv_get_u32(flow_tlvs[ROCKER_TLV_OF_DPA_DST_IP]);
+        if (ipv4_addr_is_multicast(key->ipv4.addr.dst)) {
+            return -ROCKER_EINVAL;
+        }
+        flow->lpm = of_dpa_mask2prefix(htonl(0xffffffff));
+        if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_IP_MASK]) {
+            mask->ipv4.addr.dst =
+                rocker_tlv_get_u32(flow_tlvs[ROCKER_TLV_OF_DPA_DST_IP_MASK]);
+            flow->lpm = of_dpa_mask2prefix(mask->ipv4.addr.dst);
+        }
+        break;
+    case UNICAST_ROUTING_MODE_IPV6:
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_DST_IPV6]) {
+            return -ROCKER_EINVAL;
+        }
+        memcpy(&key->ipv6.addr.dst,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_IPV6]),
+               sizeof(key->ipv6.addr.dst));
+        if (ipv6_addr_is_multicast(&key->ipv6.addr.dst)) {
+            return -ROCKER_EINVAL;
+        }
+        if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_IPV6_MASK]) {
+            memcpy(&mask->ipv6.addr.dst,
+                   rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_IPV6_MASK]),
+                   sizeof(mask->ipv6.addr.dst));
+        }
+        break;
+    default:
+        return -ROCKER_EINVAL;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]) {
+        action->goto_tbl =
+            rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]);
+        if (action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_ACL_POLICY) {
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]) {
+        action->write.group_id =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]);
+        type = ROCKER_GROUP_TYPE_GET(action->write.group_id);
+        if (type != ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE &&
+            type != ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST &&
+            type != ROCKER_OF_DPA_GROUP_TYPE_L3_ECMP) {
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_multicast_routing(OfDpaFlow *flow,
+                                            RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    enum {
+        MULTICAST_ROUTING_MODE_UNKNOWN,
+        MULTICAST_ROUTING_MODE_IPV4,
+        MULTICAST_ROUTING_MODE_IPV6,
+    } mode = MULTICAST_ROUTING_MODE_UNKNOWN;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]) {
+        return -ROCKER_EINVAL;
+    }
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING;
+    key->width = FLOW_KEY_WIDTH(ipv6.addr.dst);
+
+    key->eth.type = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE]);
+    switch (ntohs(key->eth.type)) {
+    case 0x0800:
+        mode = MULTICAST_ROUTING_MODE_IPV4;
+        break;
+    case 0x86dd:
+        mode = MULTICAST_ROUTING_MODE_IPV6;
+        break;
+    default:
+        return -ROCKER_EINVAL;
+    }
+
+    key->eth.vlan_id = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+
+    switch (mode) {
+    case MULTICAST_ROUTING_MODE_IPV4:
+
+        if (flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IP]) {
+            key->ipv4.addr.src =
+                rocker_tlv_get_u32(flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IP]);
+        }
+
+        if (flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IP_MASK]) {
+            mask->ipv4.addr.src =
+                rocker_tlv_get_u32(flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IP_MASK]);
+        }
+
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IP]) {
+            if (mask->ipv4.addr.src != 0) {
+                return -ROCKER_EINVAL;
+            }
+        }
+
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_DST_IP]) {
+            return -ROCKER_EINVAL;
+        }
+
+        key->ipv4.addr.dst =
+            rocker_tlv_get_u32(flow_tlvs[ROCKER_TLV_OF_DPA_DST_IP]);
+        if (!ipv4_addr_is_multicast(key->ipv4.addr.dst)) {
+            return -ROCKER_EINVAL;
+        }
+
+        break;
+
+    case MULTICAST_ROUTING_MODE_IPV6:
+
+        if (flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IPV6]) {
+            memcpy(&key->ipv6.addr.src,
+                   rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IPV6]),
+                   sizeof(key->ipv6.addr.src));
+        }
+
+        if (flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IPV6_MASK]) {
+            memcpy(&mask->ipv6.addr.src,
+                   rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IPV6_MASK]),
+                   sizeof(mask->ipv6.addr.src));
+        }
+
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_SRC_IPV6]) {
+            if (mask->ipv6.addr.src.addr32[0] != 0 &&
+                mask->ipv6.addr.src.addr32[1] != 0 &&
+                mask->ipv6.addr.src.addr32[2] != 0 &&
+                mask->ipv6.addr.src.addr32[3] != 0) {
+                return -ROCKER_EINVAL;
+            }
+        }
+
+        if (!flow_tlvs[ROCKER_TLV_OF_DPA_DST_IPV6]) {
+            return -ROCKER_EINVAL;
+        }
+
+        memcpy(&key->ipv6.addr.dst,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_IPV6]),
+               sizeof(key->ipv6.addr.dst));
+        if (!ipv6_addr_is_multicast(&key->ipv6.addr.dst)) {
+            return -ROCKER_EINVAL;
+        }
+
+        break;
+
+    default:
+        return -ROCKER_EINVAL;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]) {
+        action->goto_tbl =
+            rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_GOTO_TABLE_ID]);
+        if (action->goto_tbl != ROCKER_OF_DPA_TABLE_ID_ACL_POLICY) {
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]) {
+        action->write.group_id =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]);
+        if (ROCKER_GROUP_TYPE_GET(action->write.group_id) !=
+            ROCKER_OF_DPA_GROUP_TYPE_L3_MCAST) {
+            return -ROCKER_EINVAL;
+        }
+        action->write.vlan_id = key->eth.vlan_id;
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_acl_ip(OfDpaFlowKey *key, OfDpaFlowKey *mask,
+                                 RockerTlv **flow_tlvs)
+{
+    key->width = FLOW_KEY_WIDTH(ip.tos);
+
+    key->ip.proto = 0;
+    key->ip.tos = 0;
+    mask->ip.proto = 0;
+    mask->ip.tos = 0;
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IP_PROTO]) {
+        key->ip.proto =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_IP_PROTO]);
+    }
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IP_PROTO_MASK]) {
+        mask->ip.proto =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_IP_PROTO_MASK]);
+    }
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IP_DSCP]) {
+        key->ip.tos =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_IP_DSCP]);
+    }
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IP_DSCP_MASK]) {
+        mask->ip.tos =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_IP_DSCP_MASK]);
+    }
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IP_ECN]) {
+        key->ip.tos |=
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_IP_ECN]) << 6;
+    }
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IP_ECN_MASK]) {
+        mask->ip.tos |=
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_IP_ECN_MASK]) << 6;
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_acl(OfDpaFlow *flow, RockerTlv **flow_tlvs)
+{
+    OfDpaFlowKey *key = &flow->key;
+    OfDpaFlowKey *mask = &flow->mask;
+    OfDpaFlowAction *action = &flow->action;
+    enum {
+        ACL_MODE_UNKNOWN,
+        ACL_MODE_IPV4_VLAN,
+        ACL_MODE_IPV6_VLAN,
+        ACL_MODE_IPV4_TENANT,
+        ACL_MODE_IPV6_TENANT,
+        ACL_MODE_NON_IP_VLAN,
+        ACL_MODE_NON_IP_TENANT,
+        ACL_MODE_ANY_VLAN,
+        ACL_MODE_ANY_TENANT,
+    } mode = ACL_MODE_UNKNOWN;
+    int err = ROCKER_OK;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE]) {
+        return -ROCKER_EINVAL;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID] &&
+        flow_tlvs[ROCKER_TLV_OF_DPA_TUNNEL_ID]) {
+        return -ROCKER_EINVAL;
+    }
+
+    key->tbl_id = ROCKER_OF_DPA_TABLE_ID_ACL_POLICY;
+    key->width = FLOW_KEY_WIDTH(eth.type);
+
+    key->in_pport = rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT]);
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT_MASK]) {
+        mask->in_pport =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IN_PPORT_MASK]);
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC]) {
+        memcpy(key->eth.src.a,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC]),
+               sizeof(key->eth.src.a));
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC_MASK]) {
+        memcpy(mask->eth.src.a,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC_MASK]),
+               sizeof(mask->eth.src.a));
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]) {
+        memcpy(key->eth.dst.a,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]),
+               sizeof(key->eth.dst.a));
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC_MASK]) {
+        memcpy(mask->eth.dst.a,
+               rocker_tlv_data(flow_tlvs[ROCKER_TLV_OF_DPA_DST_MAC_MASK]),
+               sizeof(mask->eth.dst.a));
+    }
+
+    key->eth.type = rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_ETHERTYPE]);
+    if (key->eth.type) {
+        mask->eth.type = 0xffff;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]) {
+        key->eth.vlan_id =
+            rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID_MASK]) {
+        mask->eth.vlan_id =
+            rocker_tlv_get_u16(flow_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID_MASK]);
+    }
+
+    switch (ntohs(key->eth.type)) {
+    case 0x0000:
+        mode = (key->eth.vlan_id) ? ACL_MODE_ANY_VLAN : ACL_MODE_ANY_TENANT;
+        break;
+    case 0x0800:
+        mode = (key->eth.vlan_id) ? ACL_MODE_IPV4_VLAN : ACL_MODE_IPV4_TENANT;
+        break;
+    case 0x86dd:
+        mode = (key->eth.vlan_id) ? ACL_MODE_IPV6_VLAN : ACL_MODE_IPV6_TENANT;
+        break;
+    default:
+        mode = (key->eth.vlan_id) ? ACL_MODE_NON_IP_VLAN :
+                                    ACL_MODE_NON_IP_TENANT;
+        break;
+    }
+
+    /* XXX only supporting VLAN modes for now */
+    if (mode != ACL_MODE_IPV4_VLAN &&
+        mode != ACL_MODE_IPV6_VLAN &&
+        mode != ACL_MODE_NON_IP_VLAN &&
+        mode != ACL_MODE_ANY_VLAN) {
+        return -ROCKER_EINVAL;
+    }
+
+    switch (ntohs(key->eth.type)) {
+    case 0x0800:
+    case 0x86dd:
+        err = of_dpa_cmd_add_acl_ip(key, mask, flow_tlvs);
+        break;
+    }
+
+    if (err) {
+        return err;
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]) {
+        action->write.group_id =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]);
+    }
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_COPY_CPU_ACTION]) {
+        action->apply.copy_to_cpu =
+            rocker_tlv_get_u8(flow_tlvs[ROCKER_TLV_OF_DPA_COPY_CPU_ACTION]);
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_flow_add_mod(OfDpa *of_dpa, OfDpaFlow *flow,
+                                   RockerTlv **flow_tlvs)
+{
+    enum rocker_of_dpa_table_id tbl;
+    int err = ROCKER_OK;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_TABLE_ID] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_PRIORITY] ||
+        !flow_tlvs[ROCKER_TLV_OF_DPA_HARDTIME]) {
+        return -ROCKER_EINVAL;
+    }
+
+    tbl = rocker_tlv_get_le16(flow_tlvs[ROCKER_TLV_OF_DPA_TABLE_ID]);
+    flow->priority = rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_PRIORITY]);
+    flow->hardtime = rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_HARDTIME]);
+
+    if (flow_tlvs[ROCKER_TLV_OF_DPA_IDLETIME]) {
+        if (tbl == ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT ||
+            tbl == ROCKER_OF_DPA_TABLE_ID_VLAN ||
+            tbl == ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC) {
+            return -ROCKER_EINVAL;
+        }
+        flow->idletime =
+            rocker_tlv_get_le32(flow_tlvs[ROCKER_TLV_OF_DPA_IDLETIME]);
+    }
+
+    switch (tbl) {
+    case ROCKER_OF_DPA_TABLE_ID_INGRESS_PORT:
+        err = of_dpa_cmd_add_ig_port(flow, flow_tlvs);
+        break;
+    case ROCKER_OF_DPA_TABLE_ID_VLAN:
+        err = of_dpa_cmd_add_vlan(flow, flow_tlvs);
+        break;
+    case ROCKER_OF_DPA_TABLE_ID_TERMINATION_MAC:
+        err = of_dpa_cmd_add_term_mac(flow, flow_tlvs);
+        break;
+    case ROCKER_OF_DPA_TABLE_ID_BRIDGING:
+        err = of_dpa_cmd_add_bridging(flow, flow_tlvs);
+        break;
+    case ROCKER_OF_DPA_TABLE_ID_UNICAST_ROUTING:
+        err = of_dpa_cmd_add_unicast_routing(flow, flow_tlvs);
+        break;
+    case ROCKER_OF_DPA_TABLE_ID_MULTICAST_ROUTING:
+        err = of_dpa_cmd_add_multicast_routing(flow, flow_tlvs);
+        break;
+    case ROCKER_OF_DPA_TABLE_ID_ACL_POLICY:
+        err = of_dpa_cmd_add_acl(flow, flow_tlvs);
+        break;
+    }
+
+    return err;
+}
+
+static int of_dpa_cmd_flow_add(OfDpa *of_dpa, uint64_t cookie,
+                               RockerTlv **flow_tlvs)
+{
+    OfDpaFlow *flow = of_dpa_flow_find(of_dpa, cookie);
+    int err = ROCKER_OK;
+
+    if (flow) {
+        return -ROCKER_EEXIST;
+    }
+
+    flow = of_dpa_flow_alloc(cookie);
+    if (!flow) {
+        return -ROCKER_ENOMEM;
+    }
+
+    err = of_dpa_cmd_flow_add_mod(of_dpa, flow, flow_tlvs);
+    if (err) {
+        g_free(flow);
+        return err;
+    }
+
+    return of_dpa_flow_add(of_dpa, flow);
+}
+
+static int of_dpa_cmd_flow_mod(OfDpa *of_dpa, uint64_t cookie,
+                               RockerTlv **flow_tlvs)
+{
+    OfDpaFlow *flow = of_dpa_flow_find(of_dpa, cookie);
+
+    if (!flow) {
+        return -ROCKER_ENOENT;
+    }
+
+    return of_dpa_cmd_flow_add_mod(of_dpa, flow, flow_tlvs);
+}
+
+static int of_dpa_cmd_flow_del(OfDpa *of_dpa, uint64_t cookie)
+{
+    OfDpaFlow *flow = of_dpa_flow_find(of_dpa, cookie);
+
+    if (!flow) {
+        return -ROCKER_ENOENT;
+    }
+
+    of_dpa_flow_del(of_dpa, flow);
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_flow_get_stats(OfDpa *of_dpa, uint64_t cookie,
+                                     struct desc_info *info, char *buf)
+{
+    OfDpaFlow *flow = of_dpa_flow_find(of_dpa, cookie);
+    size_t tlv_size;
+    int64_t now = qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) / 1000;
+    int pos;
+
+    if (!flow) {
+        return -ROCKER_ENOENT;
+    }
+
+    tlv_size = rocker_tlv_total_size(sizeof(uint32_t)) +  /* duration */
+               rocker_tlv_total_size(sizeof(uint64_t)) +  /* rx_pkts */
+               rocker_tlv_total_size(sizeof(uint64_t));   /* tx_ptks */
+
+    if (tlv_size > desc_buf_size(info)) {
+        return -ROCKER_EMSGSIZE;
+    }
+
+    pos = 0;
+    rocker_tlv_put_le32(buf, &pos, ROCKER_TLV_OF_DPA_FLOW_STAT_DURATION,
+                        (int32_t)(now - flow->stats.install_time));
+    rocker_tlv_put_le64(buf, &pos, ROCKER_TLV_OF_DPA_FLOW_STAT_RX_PKTS,
+                        flow->stats.rx_pkts);
+    rocker_tlv_put_le64(buf, &pos, ROCKER_TLV_OF_DPA_FLOW_STAT_TX_PKTS,
+                        flow->stats.tx_pkts);
+
+    return desc_set_buf(info, tlv_size);
+}
+
+static int of_dpa_flow_cmd(OfDpa *of_dpa, struct desc_info *info,
+                           char *buf, uint16_t cmd,
+                           RockerTlv **flow_tlvs)
+{
+    uint64_t cookie;
+
+    if (!flow_tlvs[ROCKER_TLV_OF_DPA_COOKIE]) {
+        return -ROCKER_EINVAL;
+    }
+
+    cookie = rocker_tlv_get_le64(flow_tlvs[ROCKER_TLV_OF_DPA_COOKIE]);
+
+    switch (cmd) {
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_ADD:
+        return of_dpa_cmd_flow_add(of_dpa, cookie, flow_tlvs);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_MOD:
+        return of_dpa_cmd_flow_mod(of_dpa, cookie, flow_tlvs);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_DEL:
+        return of_dpa_cmd_flow_del(of_dpa, cookie);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_GET_STATS:
+        return of_dpa_cmd_flow_get_stats(of_dpa, cookie, info, buf);
+    }
+
+    return -ROCKER_ENOTSUP;
+}
+
+static int of_dpa_cmd_add_l2_interface(OfDpaGroup *group,
+                                       RockerTlv **group_tlvs)
+{
+    if (!group_tlvs[ROCKER_TLV_OF_DPA_OUT_PPORT] ||
+        !group_tlvs[ROCKER_TLV_OF_DPA_POP_VLAN]) {
+        return -ROCKER_EINVAL;
+    }
+
+    group->l2_interface.out_pport =
+        rocker_tlv_get_le32(group_tlvs[ROCKER_TLV_OF_DPA_OUT_PPORT]);
+    group->l2_interface.pop_vlan =
+        rocker_tlv_get_u8(group_tlvs[ROCKER_TLV_OF_DPA_POP_VLAN]);
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_l2_rewrite(OfDpa *of_dpa, OfDpaGroup *group,
+                                     RockerTlv **group_tlvs)
+{
+    OfDpaGroup *l2_interface_group;
+
+    if (!group_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID_LOWER]) {
+        return -ROCKER_EINVAL;
+    }
+
+    group->l2_rewrite.group_id =
+        rocker_tlv_get_le32(group_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID_LOWER]);
+
+    l2_interface_group = of_dpa_group_find(of_dpa, group->l2_rewrite.group_id);
+    if (!l2_interface_group ||
+        ROCKER_GROUP_TYPE_GET(l2_interface_group->id) !=
+                              ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE) {
+        DPRINTF("l2 rewrite group needs a valid l2 interface group\n");
+        return -ROCKER_EINVAL;
+    }
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC]) {
+        memcpy(group->l2_rewrite.src_mac.a,
+               rocker_tlv_data(group_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC]),
+               sizeof(group->l2_rewrite.src_mac.a));
+    }
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]) {
+        memcpy(group->l2_rewrite.dst_mac.a,
+               rocker_tlv_data(group_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]),
+               sizeof(group->l2_rewrite.dst_mac.a));
+    }
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]) {
+        group->l2_rewrite.vlan_id =
+            rocker_tlv_get_u16(group_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+        if (ROCKER_GROUP_VLAN_GET(l2_interface_group->id) !=
+            (ntohs(group->l2_rewrite.vlan_id) & VLAN_VID_MASK)) {
+            DPRINTF("Set VLAN ID must be same as L2 interface group\n");
+            return -ROCKER_EINVAL;
+        }
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_add_l2_flood(OfDpa *of_dpa, OfDpaGroup *group,
+                                   RockerTlv **group_tlvs)
+{
+    OfDpaGroup *l2_group;
+    RockerTlv **tlvs;
+    int err;
+    int i;
+
+    if (!group_tlvs[ROCKER_TLV_OF_DPA_GROUP_COUNT] ||
+        !group_tlvs[ROCKER_TLV_OF_DPA_GROUP_IDS]) {
+        return -ROCKER_EINVAL;
+    }
+
+    group->l2_flood.group_count =
+        rocker_tlv_get_le16(group_tlvs[ROCKER_TLV_OF_DPA_GROUP_COUNT]);
+
+    tlvs = g_new0(RockerTlv *, group->l2_flood.group_count + 1);
+    if (!tlvs) {
+        return -ROCKER_ENOMEM;
+    }
+
+    g_free(group->l2_flood.group_ids);
+    group->l2_flood.group_ids =
+        g_new0(uint32_t, group->l2_flood.group_count);
+    if (!group->l2_flood.group_ids) {
+        err = -ROCKER_ENOMEM;
+        goto err_out;
+    }
+
+    rocker_tlv_parse_nested(tlvs, group->l2_flood.group_count,
+                            group_tlvs[ROCKER_TLV_OF_DPA_GROUP_IDS]);
+
+    for (i = 0; i < group->l2_flood.group_count; i++) {
+        group->l2_flood.group_ids[i] = rocker_tlv_get_le32(tlvs[i + 1]);
+    }
+
+    /* All of the L2 interface groups referenced by the L2 flood
+     * must have same VLAN
+     */
+
+    for (i = 0; i < group->l2_flood.group_count; i++) {
+        l2_group = of_dpa_group_find(of_dpa, group->l2_flood.group_ids[i]);
+        if (!l2_group) {
+            continue;
+        }
+        if ((ROCKER_GROUP_TYPE_GET(l2_group->id) ==
+             ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE) &&
+            (ROCKER_GROUP_VLAN_GET(l2_group->id) !=
+             ROCKER_GROUP_VLAN_GET(group->id))) {
+            DPRINTF("l2 interface group 0x%08x VLAN doesn't match l2 "
+                    "flood group 0x%08x\n",
+                    group->l2_flood.group_ids[i], group->id);
+            err = -ROCKER_EINVAL;
+            goto err_out;
+        }
+    }
+
+    g_free(tlvs);
+    return ROCKER_OK;
+
+err_out:
+    group->l2_flood.group_count = 0;
+    g_free(group->l2_flood.group_ids);
+    g_free(tlvs);
+
+    return err;
+}
+
+static int of_dpa_cmd_add_l3_unicast(OfDpaGroup *group, RockerTlv **group_tlvs)
+{
+    if (!group_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID_LOWER]) {
+        return -ROCKER_EINVAL;
+    }
+
+    group->l3_unicast.group_id =
+        rocker_tlv_get_le32(group_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID_LOWER]);
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC]) {
+        memcpy(group->l3_unicast.src_mac.a,
+               rocker_tlv_data(group_tlvs[ROCKER_TLV_OF_DPA_SRC_MAC]),
+               sizeof(group->l3_unicast.src_mac.a));
+    }
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]) {
+        memcpy(group->l3_unicast.dst_mac.a,
+               rocker_tlv_data(group_tlvs[ROCKER_TLV_OF_DPA_DST_MAC]),
+               sizeof(group->l3_unicast.dst_mac.a));
+    }
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]) {
+        group->l3_unicast.vlan_id =
+            rocker_tlv_get_u16(group_tlvs[ROCKER_TLV_OF_DPA_VLAN_ID]);
+    }
+
+    if (group_tlvs[ROCKER_TLV_OF_DPA_TTL_CHECK]) {
+        group->l3_unicast.ttl_check =
+            rocker_tlv_get_u8(group_tlvs[ROCKER_TLV_OF_DPA_TTL_CHECK]);
+    }
+
+    return ROCKER_OK;
+}
+
+static int of_dpa_cmd_group_do(OfDpa *of_dpa, uint32_t group_id,
+                               OfDpaGroup *group, RockerTlv **group_tlvs)
+{
+    uint8_t type = ROCKER_GROUP_TYPE_GET(group_id);
+
+    switch (type) {
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE:
+        return of_dpa_cmd_add_l2_interface(group, group_tlvs);
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE:
+        return of_dpa_cmd_add_l2_rewrite(of_dpa, group, group_tlvs);
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD:
+    /* Treat L2 multicast group same as a L2 flood group */
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST:
+        return of_dpa_cmd_add_l2_flood(of_dpa, group, group_tlvs);
+    case ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST:
+        return of_dpa_cmd_add_l3_unicast(group, group_tlvs);
+    }
+
+    return -ROCKER_ENOTSUP;
+}
+
+static int of_dpa_cmd_group_add(OfDpa *of_dpa, uint32_t group_id,
+                                RockerTlv **group_tlvs)
+{
+    OfDpaGroup *group = of_dpa_group_find(of_dpa, group_id);
+    int err;
+
+    if (group) {
+        return -ROCKER_EEXIST;
+    }
+
+    group = of_dpa_group_alloc(group_id);
+    if (!group) {
+        return -ROCKER_ENOMEM;
+    }
+
+    err = of_dpa_cmd_group_do(of_dpa, group_id, group, group_tlvs);
+    if (err) {
+        goto err_cmd_add;
+    }
+
+    err = of_dpa_group_add(of_dpa, group);
+    if (err) {
+        goto err_cmd_add;
+    }
+
+    return ROCKER_OK;
+
+err_cmd_add:
+    g_free(group);
+    return err;
+}
+
+static int of_dpa_cmd_group_mod(OfDpa *of_dpa, uint32_t group_id,
+                                RockerTlv **group_tlvs)
+{
+    OfDpaGroup *group = of_dpa_group_find(of_dpa, group_id);
+
+    if (!group) {
+        return -ROCKER_ENOENT;
+    }
+
+    return of_dpa_cmd_group_do(of_dpa, group_id, group, group_tlvs);
+}
+
+static int of_dpa_cmd_group_del(OfDpa *of_dpa, uint32_t group_id)
+{
+    OfDpaGroup *group = of_dpa_group_find(of_dpa, group_id);
+
+    if (!group) {
+        return -ROCKER_ENOENT;
+    }
+
+    return of_dpa_group_del(of_dpa, group);
+}
+
+static int of_dpa_cmd_group_get_stats(OfDpa *of_dpa, uint32_t group_id,
+                                      struct desc_info *info, char *buf)
+{
+    return -ROCKER_ENOTSUP;
+}
+
+static int of_dpa_group_cmd(OfDpa *of_dpa, struct desc_info *info,
+                            char *buf, uint16_t cmd, RockerTlv **group_tlvs)
+{
+    uint32_t group_id;
+
+    if (!group_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]) {
+        return -ROCKER_EINVAL;
+    }
+
+    group_id = rocker_tlv_get_le32(group_tlvs[ROCKER_TLV_OF_DPA_GROUP_ID]);
+
+    switch (cmd) {
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_ADD:
+        return of_dpa_cmd_group_add(of_dpa, group_id, group_tlvs);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD:
+        return of_dpa_cmd_group_mod(of_dpa, group_id, group_tlvs);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_DEL:
+        return of_dpa_cmd_group_del(of_dpa, group_id);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_GET_STATS:
+        return of_dpa_cmd_group_get_stats(of_dpa, group_id, info, buf);
+    }
+
+    return -ROCKER_ENOTSUP;
+}
+
+static int of_dpa_cmd(World *world, struct desc_info *info,
+                      char *buf, uint16_t cmd, RockerTlv *cmd_info_tlv)
+{
+    OfDpa *of_dpa = world_private(world);
+    RockerTlv *tlvs[ROCKER_TLV_OF_DPA_MAX + 1];
+
+    rocker_tlv_parse_nested(tlvs, ROCKER_TLV_OF_DPA_MAX, cmd_info_tlv);
+
+    switch (cmd) {
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_ADD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_MOD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_DEL:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_FLOW_GET_STATS:
+        return of_dpa_flow_cmd(of_dpa, info, buf, cmd, tlvs);
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_ADD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_DEL:
+    case ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_GET_STATS:
+        return of_dpa_group_cmd(of_dpa, info, buf, cmd, tlvs);
+    }
+
+    return -ROCKER_ENOTSUP;
+}
+
+static gboolean rocker_int64_equal(gconstpointer v1, gconstpointer v2)
+{
+    return *((const uint64_t *)v1) == *((const uint64_t *)v2);
+}
+
+static guint rocker_int64_hash(gconstpointer v)
+{
+    return (guint)*(const uint64_t *)v;
+}
+
+static int of_dpa_init(World *world)
+{
+    OfDpa *of_dpa = world_private(world);
+
+    of_dpa->world = world;
+
+    of_dpa->flow_tbl = g_hash_table_new_full(rocker_int64_hash,
+                                             rocker_int64_equal,
+                                             NULL, g_free);
+    if (!of_dpa->flow_tbl) {
+        return -ENOMEM;
+    }
+
+    of_dpa->group_tbl = g_hash_table_new_full(g_int_hash, g_int_equal,
+                                              NULL, g_free);
+    if (!of_dpa->group_tbl) {
+        goto err_group_tbl;
+    }
+
+    /* XXX hardcode some artificial table max values */
+    of_dpa->flow_tbl_max_size = 100;
+    of_dpa->group_tbl_max_size = 100;
+
+    return 0;
+
+err_group_tbl:
+    g_hash_table_destroy(of_dpa->flow_tbl);
+    return -ENOMEM;
+}
+
+static void of_dpa_uninit(World *world)
+{
+    OfDpa *of_dpa = world_private(world);
+
+    g_hash_table_destroy(of_dpa->group_tbl);
+    g_hash_table_destroy(of_dpa->flow_tbl);
+}
+
+struct of_dpa_flow_fill_context {
+    RockerOfDpaFlowList *list;
+    uint32_t tbl_id;
+};
+
+static void of_dpa_flow_fill(void *cookie, void *value, void *user_data)
+{
+    struct of_dpa_flow *flow = value;
+    struct of_dpa_flow_key *key = &flow->key;
+    struct of_dpa_flow_key *mask = &flow->mask;
+    struct of_dpa_flow_fill_context *flow_context = user_data;
+    RockerOfDpaFlowList *new;
+    RockerOfDpaFlow *nflow;
+    RockerOfDpaFlowKey *nkey;
+    RockerOfDpaFlowMask *nmask;
+    RockerOfDpaFlowAction *naction;
+
+    if (flow_context->tbl_id != -1 &&
+        flow_context->tbl_id != key->tbl_id) {
+        return;
+    }
+
+    new = g_malloc0(sizeof(*new));
+    nflow = new->value = g_malloc0(sizeof(*nflow));
+    nkey = nflow->key = g_malloc0(sizeof(*nkey));
+    nmask = nflow->mask = g_malloc0(sizeof(*nmask));
+    naction = nflow->action = g_malloc0(sizeof(*naction));
+
+    nflow->cookie = flow->cookie;
+    nflow->hits = flow->stats.hits;
+    nkey->priority = flow->priority;
+    nkey->tbl_id = key->tbl_id;
+
+    if (key->in_pport || mask->in_pport) {
+        nkey->has_in_pport = true;
+        nkey->in_pport = key->in_pport;
+    }
+
+    if (nkey->has_in_pport && mask->in_pport != 0xffffffff) {
+        nmask->has_in_pport = true;
+        nmask->in_pport = mask->in_pport;
+    }
+
+    if (key->eth.vlan_id || mask->eth.vlan_id) {
+        nkey->has_vlan_id = true;
+        nkey->vlan_id = ntohs(key->eth.vlan_id);
+    }
+
+    if (nkey->has_vlan_id && mask->eth.vlan_id != 0xffff) {
+        nmask->has_vlan_id = true;
+        nmask->vlan_id = ntohs(mask->eth.vlan_id);
+    }
+
+    if (key->tunnel_id || mask->tunnel_id) {
+        nkey->has_tunnel_id = true;
+        nkey->tunnel_id = key->tunnel_id;
+    }
+
+    if (nkey->has_tunnel_id && mask->tunnel_id != 0xffffffff) {
+        nmask->has_tunnel_id = true;
+        nmask->tunnel_id = mask->tunnel_id;
+    }
+
+    if (memcmp(key->eth.src.a, zero_mac.a, ETH_ALEN) ||
+        memcmp(mask->eth.src.a, zero_mac.a, ETH_ALEN)) {
+        nkey->has_eth_src = true;
+        nkey->eth_src = qemu_mac_strdup_printf(key->eth.src.a);
+    }
+
+    if (nkey->has_eth_src && memcmp(mask->eth.src.a, ff_mac.a, ETH_ALEN)) {
+        nmask->has_eth_src = true;
+        nmask->eth_src = qemu_mac_strdup_printf(mask->eth.src.a);
+    }
+
+    if (memcmp(key->eth.dst.a, zero_mac.a, ETH_ALEN) ||
+        memcmp(mask->eth.dst.a, zero_mac.a, ETH_ALEN)) {
+        nkey->has_eth_dst = true;
+        nkey->eth_dst = qemu_mac_strdup_printf(key->eth.dst.a);
+    }
+
+    if (nkey->has_eth_dst && memcmp(mask->eth.dst.a, ff_mac.a, ETH_ALEN)) {
+        nmask->has_eth_dst = true;
+        nmask->eth_dst = qemu_mac_strdup_printf(mask->eth.dst.a);
+    }
+
+    if (key->eth.type) {
+
+        nkey->has_eth_type = true;
+        nkey->eth_type = ntohs(key->eth.type);
+
+        switch (ntohs(key->eth.type)) {
+        case 0x0800:
+        case 0x86dd:
+            if (key->ip.proto || mask->ip.proto) {
+                nkey->has_ip_proto = true;
+                nkey->ip_proto = key->ip.proto;
+            }
+            if (nkey->has_ip_proto && mask->ip.proto != 0xff) {
+                nmask->has_ip_proto = true;
+                nmask->ip_proto = mask->ip.proto;
+            }
+            if (key->ip.tos || mask->ip.tos) {
+                nkey->has_ip_tos = true;
+                nkey->ip_tos = key->ip.tos;
+            }
+            if (nkey->has_ip_tos && mask->ip.tos != 0xff) {
+                nmask->has_ip_tos = true;
+                nmask->ip_tos = mask->ip.tos;
+            }
+            break;
+        }
+
+        switch (ntohs(key->eth.type)) {
+        case 0x0800:
+            if (key->ipv4.addr.dst || mask->ipv4.addr.dst) {
+                char *dst = inet_ntoa(*(struct in_addr *)&key->ipv4.addr.dst);
+                int dst_len = of_dpa_mask2prefix(mask->ipv4.addr.dst);
+                nkey->has_ip_dst = true;
+                nkey->ip_dst = g_strdup_printf("%s/%d", dst, dst_len);
+            }
+            break;
+        }
+    }
+
+    if (flow->action.goto_tbl) {
+        naction->has_goto_tbl = true;
+        naction->goto_tbl = flow->action.goto_tbl;
+    }
+
+    if (flow->action.write.group_id) {
+        naction->has_group_id = true;
+        naction->group_id = flow->action.write.group_id;
+    }
+
+    if (flow->action.apply.new_vlan_id) {
+        naction->has_new_vlan_id = true;
+        naction->new_vlan_id = flow->action.apply.new_vlan_id;
+    }
+
+    new->next = flow_context->list;
+    flow_context->list = new;
+}
+
+RockerOfDpaFlowList *qmp_query_rocker_of_dpa_flows(const char *name,
+                                                   bool has_tbl_id,
+                                                   uint32_t tbl_id,
+                                                   Error **errp)
+{
+    struct rocker *r;
+    struct world *w;
+    struct of_dpa *of_dpa;
+    struct of_dpa_flow_fill_context fill_context = {
+        .list = NULL,
+        .tbl_id = tbl_id,
+    };
+
+    r = rocker_find(name);
+    if (!r) {
+        error_setg(errp, "rocker %s not found", name);
+        return NULL;
+    }
+
+    w = rocker_get_world(r, ROCKER_WORLD_TYPE_OF_DPA);
+    if (!w) {
+        error_setg(errp, "rocker %s doesn't have OF-DPA world", name);
+        return NULL;
+    }
+
+    of_dpa = world_private(w);
+
+    g_hash_table_foreach(of_dpa->flow_tbl, of_dpa_flow_fill, &fill_context);
+
+    return fill_context.list;
+}
+
+struct of_dpa_group_fill_context {
+    RockerOfDpaGroupList *list;
+    uint8_t type;
+};
+
+static void of_dpa_group_fill(void *key, void *value, void *user_data)
+{
+    struct of_dpa_group *group = value;
+    struct of_dpa_group_fill_context *flow_context = user_data;
+    RockerOfDpaGroupList *new;
+    RockerOfDpaGroup *ngroup;
+    struct uint32List *id;
+    int i;
+
+    if (flow_context->type != 9 &&
+        flow_context->type != ROCKER_GROUP_TYPE_GET(group->id)) {
+        return;
+    }
+
+    new = g_malloc0(sizeof(*new));
+    ngroup = new->value = g_malloc0(sizeof(*ngroup));
+
+    ngroup->id = group->id;
+
+    ngroup->type = ROCKER_GROUP_TYPE_GET(group->id);
+
+    switch (ngroup->type) {
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_INTERFACE:
+        ngroup->has_vlan_id = true;
+        ngroup->vlan_id = ROCKER_GROUP_VLAN_GET(group->id);
+        ngroup->has_pport = true;
+        ngroup->pport = ROCKER_GROUP_PORT_GET(group->id);
+        ngroup->has_out_pport = true;
+        ngroup->out_pport = group->l2_interface.out_pport;
+        ngroup->has_pop_vlan = true;
+        ngroup->pop_vlan = group->l2_interface.pop_vlan;
+        break;
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_REWRITE:
+        ngroup->has_index = true;
+        ngroup->index = ROCKER_GROUP_INDEX_LONG_GET(group->id);
+        ngroup->has_group_id = true;
+        ngroup->group_id = group->l2_rewrite.group_id;
+        if (group->l2_rewrite.vlan_id) {
+            ngroup->has_set_vlan_id = true;
+            ngroup->set_vlan_id = ntohs(group->l2_rewrite.vlan_id);
+        }
+        if (memcmp(group->l2_rewrite.src_mac.a, zero_mac.a, ETH_ALEN)) {
+            ngroup->has_set_eth_src = true;
+            ngroup->set_eth_src =
+                qemu_mac_strdup_printf(group->l2_rewrite.src_mac.a);
+        }
+        if (memcmp(group->l2_rewrite.dst_mac.a, zero_mac.a, ETH_ALEN)) {
+            ngroup->has_set_eth_dst = true;
+            ngroup->set_eth_dst =
+                qemu_mac_strdup_printf(group->l2_rewrite.dst_mac.a);
+        }
+        break;
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD:
+    case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST:
+        ngroup->has_vlan_id = true;
+        ngroup->vlan_id = ROCKER_GROUP_VLAN_GET(group->id);
+        ngroup->has_index = true;
+        ngroup->index = ROCKER_GROUP_INDEX_GET(group->id);
+        for (i = 0; i < group->l2_flood.group_count; i++) {
+            ngroup->has_group_ids = true;
+            id = g_malloc0(sizeof(*id));
+            id->value = group->l2_flood.group_ids[i];
+            id->next = ngroup->group_ids;
+            ngroup->group_ids = id;
+        }
+        break;
+    case ROCKER_OF_DPA_GROUP_TYPE_L3_UCAST:
+        ngroup->has_index = true;
+        ngroup->index = ROCKER_GROUP_INDEX_LONG_GET(group->id);
+        ngroup->has_group_id = true;
+        ngroup->group_id = group->l3_unicast.group_id;
+        if (group->l3_unicast.vlan_id) {
+            ngroup->has_set_vlan_id = true;
+            ngroup->set_vlan_id = ntohs(group->l3_unicast.vlan_id);
+        }
+        if (memcmp(group->l3_unicast.src_mac.a, zero_mac.a, ETH_ALEN)) {
+            ngroup->has_set_eth_src = true;
+            ngroup->set_eth_src =
+                qemu_mac_strdup_printf(group->l3_unicast.src_mac.a);
+        }
+        if (memcmp(group->l3_unicast.dst_mac.a, zero_mac.a, ETH_ALEN)) {
+            ngroup->has_set_eth_dst = true;
+            ngroup->set_eth_dst =
+                qemu_mac_strdup_printf(group->l3_unicast.dst_mac.a);
+        }
+        if (group->l3_unicast.ttl_check) {
+            ngroup->has_ttl_check = true;
+            ngroup->ttl_check = group->l3_unicast.ttl_check;
+        }
+        break;
+    }
+
+    new->next = flow_context->list;
+    flow_context->list = new;
+}
+
+RockerOfDpaGroupList *qmp_query_rocker_of_dpa_groups(const char *name,
+                                                     bool has_type,
+                                                     uint8_t type,
+                                                     Error **errp)
+{
+    struct rocker *r;
+    struct world *w;
+    struct of_dpa *of_dpa;
+    struct of_dpa_group_fill_context fill_context = {
+        .list = NULL,
+        .type = type,
+    };
+
+    r = rocker_find(name);
+    if (!r) {
+        error_setg(errp, "rocker %s not found", name);
+        return NULL;
+    }
+
+    w = rocker_get_world(r, ROCKER_WORLD_TYPE_OF_DPA);
+    if (!w) {
+        error_setg(errp, "rocker %s doesn't have OF-DPA world", name);
+        return NULL;
+    }
+
+    of_dpa = world_private(w);
+
+    g_hash_table_foreach(of_dpa->group_tbl, of_dpa_group_fill, &fill_context);
+
+    return fill_context.list;
+}
+
+static WorldOps of_dpa_ops = {
+    .name = "ofdpa",
+    .init = of_dpa_init,
+    .uninit = of_dpa_uninit,
+    .ig = of_dpa_ig,
+    .cmd = of_dpa_cmd,
+};
+
+World *of_dpa_world_alloc(Rocker *r)
+{
+    return world_alloc(r, sizeof(OfDpa), ROCKER_WORLD_TYPE_OF_DPA, &of_dpa_ops);
+}
diff --git a/hw/net/rocker/rocker_of_dpa.h b/hw/net/rocker/rocker_of_dpa.h
new file mode 100644
index 0000000..01c7a97
--- /dev/null
+++ b/hw/net/rocker/rocker_of_dpa.h
@@ -0,0 +1,22 @@
+/*
+ * QEMU rocker switch emulation - OF-DPA flow processing support
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef ROCKER_OF_DPA_H
+#define ROCKER_OF_DPA_H
+
+World *of_dpa_world_alloc(Rocker *r);
+
+#endif /* ROCKER_OF_DPA_H */
diff --git a/hw/net/rocker/rocker_tlv.h b/hw/net/rocker/rocker_tlv.h
new file mode 100644
index 0000000..dd28d08
--- /dev/null
+++ b/hw/net/rocker/rocker_tlv.h
@@ -0,0 +1,244 @@
+/*
+ * QEMU rocker switch emulation - TLV parsing and composing
+ *
+ * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef ROCKER_TLV_H
+#define ROCKER_TLV_H
+
+#define ROCKER_TLV_ALIGNTO 8U
+#define ROCKER_TLV_ALIGN(len) \
+    (((len) + ROCKER_TLV_ALIGNTO - 1) & ~(ROCKER_TLV_ALIGNTO - 1))
+#define ROCKER_TLV_HDRLEN ROCKER_TLV_ALIGN(sizeof(RockerTlv))
+
+/*
+ *  <------- ROCKER_TLV_HDRLEN -------> <--- ROCKER_TLV_ALIGN(payload) --->
+ * +-----------------------------+- - -+- - - - - - - - - - - - - - -+- - -+
+ * |             Header          | Pad |           Payload           | Pad |
+ * |          (RockerTlv)        | ing |                             | ing |
+ * +-----------------------------+- - -+- - - - - - - - - - - - - - -+- - -+
+ *  <--------------------------- tlv->len -------------------------->
+ */
+
+static inline RockerTlv *rocker_tlv_next(const RockerTlv *tlv, int *remaining)
+{
+    int totlen = ROCKER_TLV_ALIGN(le16_to_cpu(tlv->len));
+
+    *remaining -= totlen;
+    return (RockerTlv *) ((char *) tlv + totlen);
+}
+
+static inline int rocker_tlv_ok(const RockerTlv *tlv, int remaining)
+{
+    return remaining >= (int) ROCKER_TLV_HDRLEN &&
+           le16_to_cpu(tlv->len) >= ROCKER_TLV_HDRLEN &&
+           le16_to_cpu(tlv->len) <= remaining;
+}
+
+#define rocker_tlv_for_each(pos, head, len, rem) \
+    for (pos = head, rem = len; \
+         rocker_tlv_ok(pos, rem); \
+         pos = rocker_tlv_next(pos, &(rem)))
+
+#define rocker_tlv_for_each_nested(pos, tlv, rem) \
+        rocker_tlv_for_each(pos, rocker_tlv_data(tlv), rocker_tlv_len(tlv), rem)
+
+static inline int rocker_tlv_size(int payload)
+{
+    return ROCKER_TLV_HDRLEN + payload;
+}
+
+static inline int rocker_tlv_total_size(int payload)
+{
+    return ROCKER_TLV_ALIGN(rocker_tlv_size(payload));
+}
+
+static inline int rocker_tlv_padlen(int payload)
+{
+    return rocker_tlv_total_size(payload) - rocker_tlv_size(payload);
+}
+
+static inline int rocker_tlv_type(const RockerTlv *tlv)
+{
+    return le32_to_cpu(tlv->type);
+}
+
+static inline void *rocker_tlv_data(const RockerTlv *tlv)
+{
+    return (char *) tlv + ROCKER_TLV_HDRLEN;
+}
+
+static inline int rocker_tlv_len(const RockerTlv *tlv)
+{
+    return le16_to_cpu(tlv->len) - ROCKER_TLV_HDRLEN;
+}
+
+static inline uint8_t rocker_tlv_get_u8(const RockerTlv *tlv)
+{
+    return *(uint8_t *) rocker_tlv_data(tlv);
+}
+
+static inline uint16_t rocker_tlv_get_u16(const RockerTlv *tlv)
+{
+    return *(uint16_t *) rocker_tlv_data(tlv);
+}
+
+static inline uint32_t rocker_tlv_get_u32(const RockerTlv *tlv)
+{
+    return *(uint32_t *) rocker_tlv_data(tlv);
+}
+
+static inline uint64_t rocker_tlv_get_u64(const RockerTlv *tlv)
+{
+    return *(uint64_t *) rocker_tlv_data(tlv);
+}
+
+static inline uint16_t rocker_tlv_get_le16(const RockerTlv *tlv)
+{
+    return lduw_le_p(rocker_tlv_data(tlv));
+}
+
+static inline uint32_t rocker_tlv_get_le32(const RockerTlv *tlv)
+{
+    return ldl_le_p(rocker_tlv_data(tlv));
+}
+
+static inline uint64_t rocker_tlv_get_le64(const RockerTlv *tlv)
+{
+    return ldq_le_p(rocker_tlv_data(tlv));
+}
+
+static inline void rocker_tlv_parse(RockerTlv **tb, int maxtype,
+                                    const char *buf, int buf_len)
+{
+    const RockerTlv *tlv;
+    const RockerTlv *head = (const RockerTlv *) buf;
+    int rem;
+
+    memset(tb, 0, sizeof(RockerTlv *) * (maxtype + 1));
+
+    rocker_tlv_for_each(tlv, head, buf_len, rem) {
+        uint32_t type = rocker_tlv_type(tlv);
+
+        if (type > 0 && type <= maxtype) {
+            tb[type] = (RockerTlv *) tlv;
+        }
+    }
+}
+
+static inline void rocker_tlv_parse_nested(RockerTlv **tb, int maxtype,
+                                           const RockerTlv *tlv)
+{
+    rocker_tlv_parse(tb, maxtype, rocker_tlv_data(tlv), rocker_tlv_len(tlv));
+}
+
+static inline RockerTlv *rocker_tlv_start(char *buf, int buf_pos)
+{
+    return (RockerTlv *) (buf + buf_pos);
+}
+
+static inline void rocker_tlv_put_iov(char *buf, int *buf_pos,
+                                      int type, const struct iovec *iov,
+                                      const unsigned int iovcnt)
+{
+    size_t len = iov_size(iov, iovcnt);
+    int total_size = rocker_tlv_total_size(len);
+    RockerTlv *tlv;
+
+    tlv = rocker_tlv_start(buf, *buf_pos);
+    *buf_pos += total_size;
+    tlv->type = cpu_to_le32(type);
+    tlv->len = cpu_to_le16(rocker_tlv_size(len));
+    iov_to_buf(iov, iovcnt, 0, rocker_tlv_data(tlv), len);
+    memset((char *) tlv + le16_to_cpu(tlv->len), 0, rocker_tlv_padlen(len));
+}
+
+static inline void rocker_tlv_put(char *buf, int *buf_pos,
+                                  int type, int len, void *data)
+{
+    struct iovec iov = {
+        .iov_base = data,
+        .iov_len = len,
+    };
+
+    rocker_tlv_put_iov(buf, buf_pos, type, &iov, 1);
+}
+
+static inline void rocker_tlv_put_u8(char *buf, int *buf_pos,
+                                     int type, uint8_t value)
+{
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint8_t), &value);
+}
+
+static inline void rocker_tlv_put_u16(char *buf, int *buf_pos,
+                                      int type, uint16_t value)
+{
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint16_t), &value);
+}
+
+static inline void rocker_tlv_put_u32(char *buf, int *buf_pos,
+                                      int type, uint32_t value)
+{
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint32_t), &value);
+}
+
+static inline void rocker_tlv_put_u64(char *buf, int *buf_pos,
+                                      int type, uint64_t value)
+{
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint64_t), &value);
+}
+
+static inline void rocker_tlv_put_le16(char *buf, int *buf_pos,
+                                       int type, uint16_t value)
+{
+    value = cpu_to_le16(value);
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint16_t), &value);
+}
+
+static inline void rocker_tlv_put_le32(char *buf, int *buf_pos,
+                                       int type, uint32_t value)
+{
+    value = cpu_to_le32(value);
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint32_t), &value);
+}
+
+static inline void rocker_tlv_put_le64(char *buf, int *buf_pos,
+                                       int type, uint64_t value)
+{
+    value = cpu_to_le64(value);
+    rocker_tlv_put(buf, buf_pos, type, sizeof(uint64_t), &value);
+}
+
+static inline RockerTlv *rocker_tlv_nest_start(char *buf, int *buf_pos,
+                                               int type)
+{
+    RockerTlv *start = rocker_tlv_start(buf, *buf_pos);
+
+    rocker_tlv_put(buf, buf_pos, type, 0, NULL);
+    return start;
+}
+
+static inline void rocker_tlv_nest_end(char *buf, int *buf_pos,
+                                       RockerTlv *start)
+{
+    start->len = (char *) rocker_tlv_start(buf, *buf_pos) - (char *) start;
+}
+
+static inline void rocker_tlv_nest_cancel(char *buf, int *buf_pos,
+                                          RockerTlv *start)
+{
+    *buf_pos = (char *) start - buf;
+}
+
+#endif
diff --git a/hw/net/rocker/rocker_world.c b/hw/net/rocker/rocker_world.c
new file mode 100644
index 0000000..89777e9
--- /dev/null
+++ b/hw/net/rocker/rocker_world.c
@@ -0,0 +1,102 @@
+/*
+ * QEMU rocker switch emulation - switch worlds
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/iov.h"
+
+#include "rocker.h"
+#include "rocker_world.h"
+
+struct world {
+    Rocker *r;
+    enum rocker_world_type type;
+    WorldOps *ops;
+};
+
+ssize_t world_ingress(World *world, uint32_t pport,
+                      const struct iovec *iov, int iovcnt)
+{
+    if (world->ops->ig) {
+        return world->ops->ig(world, pport, iov, iovcnt);
+    }
+
+    return -1;
+}
+
+int world_do_cmd(World *world, DescInfo *info,
+                 char *buf, uint16_t cmd, RockerTlv *cmd_info_tlv)
+{
+    if (world->ops->cmd) {
+        return world->ops->cmd(world, info, buf, cmd, cmd_info_tlv);
+    }
+
+    return -ROCKER_ENOTSUP;
+}
+
+World *world_alloc(Rocker *r, size_t sizeof_private,
+                   enum rocker_world_type type, WorldOps *ops)
+{
+    World *w = g_malloc0(sizeof(World) + sizeof_private);
+
+    if (w) {
+        w->r = r;
+        w->type = type;
+        w->ops = ops;
+        if (w->ops->init) {
+            w->ops->init(w);
+        }
+    }
+
+    return w;
+}
+
+void world_free(World *world)
+{
+    if (world->ops->uninit) {
+        world->ops->uninit(world);
+    }
+    g_free(world);
+}
+
+void world_reset(World *world)
+{
+    if (world->ops->uninit) {
+        world->ops->uninit(world);
+    }
+    if (world->ops->init) {
+        world->ops->init(world);
+    }
+}
+
+void *world_private(World *world)
+{
+    return world + 1;
+}
+
+Rocker *world_rocker(World *world)
+{
+    return world->r;
+}
+
+enum rocker_world_type world_type(World *world)
+{
+    return world->type;
+}
+
+const char *world_name(World *world)
+{
+    return world->ops->name;
+}
diff --git a/hw/net/rocker/rocker_world.h b/hw/net/rocker/rocker_world.h
new file mode 100644
index 0000000..44f1fe3
--- /dev/null
+++ b/hw/net/rocker/rocker_world.h
@@ -0,0 +1,61 @@
+/*
+ * QEMU rocker switch emulation - switch worlds
+ *
+ * Copyright (c) 2014 Scott Feldman <sfeldma@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef ROCKER_WORLD_H
+#define ROCKER_WORLD_H
+
+#include "rocker_hw.h"
+
+enum rocker_world_type {
+    ROCKER_WORLD_TYPE_OF_DPA = ROCKER_PORT_MODE_OF_DPA,
+    ROCKER_WORLD_TYPE_MAX,
+};
+
+typedef int (world_init)(World *world);
+typedef void (world_uninit)(World *world);
+typedef ssize_t (world_ig)(World *world, uint32_t pport,
+                           const struct iovec *iov, int iovcnt);
+typedef int (world_cmd)(World *world, DescInfo *info,
+                        char *buf, uint16_t cmd,
+                        RockerTlv *cmd_info_tlv);
+
+typedef struct world_ops {
+    const char *name;
+    world_init *init;
+    world_uninit *uninit;
+    world_ig *ig;
+    world_cmd *cmd;
+} WorldOps;
+
+ssize_t world_ingress(World *world, uint32_t pport,
+                      const struct iovec *iov, int iovcnt);
+int world_do_cmd(World *world, DescInfo *info,
+                 char *buf, uint16_t cmd, RockerTlv *cmd_info_tlv);
+
+World *world_alloc(Rocker *r, size_t sizeof_private,
+                   enum rocker_world_type type, WorldOps *ops);
+void world_free(World *world);
+void world_reset(World *world);
+
+void *world_private(World *world);
+Rocker *world_rocker(World *world);
+
+enum rocker_world_type world_type(World *world);
+const char *world_name(World *world);
+
+World *rocker_get_world(Rocker *r, enum rocker_world_type type);
+
+#endif /* ROCKER_WORLD_H */
diff --git a/hw/net/rtl8139.c b/hw/net/rtl8139.c
index 5f0197c..3345bc6 100644
--- a/hw/net/rtl8139.c
+++ b/hw/net/rtl8139.c
@@ -43,12 +43,13 @@
  *                                  Added rx/tx buffer reset when enabling rx/tx operation
  *
  *  2010-Feb-04  Frediano Ziglio:   Rewrote timer support using QEMU timer only
- *                                  when strictly needed (required for for
+ *                                  when strictly needed (required for
  *                                  Darwin)
  *  2011-Mar-22  Benjamin Poirier:  Implemented VLAN offloading
  */
 
 /* For crc32 */
+#include "qemu/osdep.h"
 #include <zlib.h>
 
 #include "hw/hw.h"
@@ -56,6 +57,7 @@
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
 #include "net/net.h"
+#include "net/eth.h"
 #include "hw/loader.h"
 #include "sysemu/sysemu.h"
 #include "qemu/iov.h"
@@ -63,7 +65,7 @@
 /* debug RTL8139 card */
 //#define DEBUG_RTL8139 1
 
-#define PCI_FREQUENCY 33000000L
+#define PCI_PERIOD 30    /* 30 ns period = 33.333333 Mhz frequency */
 
 #define SET_MASKED(input, mask, curr) \
     ( ( (input) & ~(mask) ) | ( (curr) & (mask) ) )
@@ -72,11 +74,7 @@
 #define MOD2(input, size) \
     ( ( input ) & ( size - 1 )  )
 
-#define ETHER_ADDR_LEN 6
 #define ETHER_TYPE_LEN 2
-#define ETH_HLEN (ETHER_ADDR_LEN * 2 + ETHER_TYPE_LEN)
-#define ETH_P_IP    0x0800      /* Internet Protocol packet */
-#define ETH_P_8021Q 0x8100      /* 802.1Q VLAN Extended Header  */
 #define ETH_MTU     1500
 
 #define VLAN_TCI_LEN 2
@@ -508,7 +506,6 @@
 
     /* PCI interrupt timer */
     QEMUTimer *timer;
-    int64_t TimerExpire;
 
     MemoryRegion bar_io;
     MemoryRegion bar_mem;
@@ -520,7 +517,7 @@
 /* Writes tally counters to memory via DMA */
 static void RTL8139TallyCounters_dma_write(RTL8139State *s, dma_addr_t tc_addr);
 
-static void rtl8139_set_next_tctr_time(RTL8139State *s, int64_t current_time);
+static void rtl8139_set_next_tctr_time(RTL8139State *s);
 
 static void prom9346_decode_command(EEprom9346 *eeprom, uint8_t command)
 {
@@ -1016,9 +1013,9 @@
         uint32_t rx_space = rxdw0 & CP_RX_BUFFER_SIZE_MASK;
 
         /* write VLAN info to descriptor variables. */
-        if (s->CpCmd & CPlusRxVLAN && be16_to_cpup((uint16_t *)
-                &buf[ETHER_ADDR_LEN * 2]) == ETH_P_8021Q) {
-            dot1q_buf = &buf[ETHER_ADDR_LEN * 2];
+        if (s->CpCmd & CPlusRxVLAN &&
+            lduw_be_p(&buf[ETH_ALEN * 2]) == ETH_P_VLAN) {
+            dot1q_buf = &buf[ETH_ALEN * 2];
             size -= VLAN_HLEN;
             /* if too small buffer, use the tailroom added duing expansion */
             if (size < MIN_BUF_SIZE) {
@@ -1027,11 +1024,10 @@
 
             rxdw1 &= ~CP_RX_VLAN_TAG_MASK;
             /* BE + ~le_to_cpu()~ + cpu_to_le() = BE */
-            rxdw1 |= CP_RX_TAVA | le16_to_cpup((uint16_t *)
-                &dot1q_buf[ETHER_TYPE_LEN]);
+            rxdw1 |= CP_RX_TAVA | lduw_le_p(&dot1q_buf[ETHER_TYPE_LEN]);
 
             DPRINTF("C+ Rx mode : extracted vlan tag with tci: ""%u\n",
-                be16_to_cpup((uint16_t *)&dot1q_buf[ETHER_TYPE_LEN]));
+                lduw_be_p(&dot1q_buf[ETHER_TYPE_LEN]));
         } else {
             /* reset VLAN tag flag */
             rxdw1 &= ~CP_RX_TAVA;
@@ -1059,10 +1055,10 @@
 
         /* receive/copy to target memory */
         if (dot1q_buf) {
-            pci_dma_write(d, rx_addr, buf, 2 * ETHER_ADDR_LEN);
-            pci_dma_write(d, rx_addr + 2 * ETHER_ADDR_LEN,
-                          buf + 2 * ETHER_ADDR_LEN + VLAN_HLEN,
-                          size - 2 * ETHER_ADDR_LEN);
+            pci_dma_write(d, rx_addr, buf, 2 * ETH_ALEN);
+            pci_dma_write(d, rx_addr + 2 * ETH_ALEN,
+                          buf + 2 * ETH_ALEN + VLAN_HLEN,
+                          size - 2 * ETH_ALEN);
         } else {
             pci_dma_write(d, rx_addr, buf, size);
         }
@@ -1149,7 +1145,9 @@
 
         /* if receiver buffer is empty then avail == 0 */
 
-        if (avail != 0 && size + 8 >= avail)
+#define RX_ALIGN(x) (((x) + 3) & ~0x3)
+
+        if (avail != 0 && RX_ALIGN(size + 8) >= avail)
         {
             DPRINTF("rx overflow: rx buffer length %d head 0x%04x "
                 "read 0x%04x === available 0x%04x need 0x%04x\n",
@@ -1158,7 +1156,7 @@
             s->IntrStatus |= RxOverflow;
             ++s->RxMissed;
             rtl8139_update_irq(s);
-            return size_;
+            return 0;
         }
 
         packet_header |= RxStatusOK;
@@ -1177,7 +1175,7 @@
         rtl8139_write_buffer(s, (uint8_t *)&val, 4);
 
         /* correct buffer write pointer */
-        s->RxBufAddr = MOD2((s->RxBufAddr + 3) & ~0x3, s->RxBufferSize);
+        s->RxBufAddr = MOD2(RX_ALIGN(s->RxBufAddr), s->RxBufferSize);
 
         /* now we can signal we have received something */
 
@@ -1282,6 +1280,7 @@
     s->TCTR = 0;
     s->TimerInt = 0;
     s->TCTR_base = 0;
+    rtl8139_set_next_tctr_time(s);
 
     /* reset tally counters */
     RTL8139TallyCounters_clear(&s->tally_counters);
@@ -1352,29 +1351,6 @@
     pci_dma_write(d, tc_addr + 62,    (uint8_t *)&val16, 2);
 }
 
-/* Loads values of tally counters from VM state file */
-
-static const VMStateDescription vmstate_tally_counters = {
-    .name = "tally_counters",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT64(TxOk, RTL8139TallyCounters),
-        VMSTATE_UINT64(RxOk, RTL8139TallyCounters),
-        VMSTATE_UINT64(TxERR, RTL8139TallyCounters),
-        VMSTATE_UINT32(RxERR, RTL8139TallyCounters),
-        VMSTATE_UINT16(MissPkt, RTL8139TallyCounters),
-        VMSTATE_UINT16(FAE, RTL8139TallyCounters),
-        VMSTATE_UINT32(Tx1Col, RTL8139TallyCounters),
-        VMSTATE_UINT32(TxMCol, RTL8139TallyCounters),
-        VMSTATE_UINT64(RxOkPhy, RTL8139TallyCounters),
-        VMSTATE_UINT64(RxOkBrd, RTL8139TallyCounters),
-        VMSTATE_UINT16(TxAbt, RTL8139TallyCounters),
-        VMSTATE_UINT16(TxUndrn, RTL8139TallyCounters),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
 static void rtl8139_ChipCmd_write(RTL8139State *s, uint32_t val)
 {
     DeviceState *d = DEVICE(s);
@@ -1783,12 +1759,12 @@
         return;
     }
 
-    if (dot1q_buf && size >= ETHER_ADDR_LEN * 2) {
+    if (dot1q_buf && size >= ETH_ALEN * 2) {
         iov = (struct iovec[3]) {
-            { .iov_base = buf, .iov_len = ETHER_ADDR_LEN * 2 },
+            { .iov_base = buf, .iov_len = ETH_ALEN * 2 },
             { .iov_base = (void *) dot1q_buf, .iov_len = VLAN_HLEN },
-            { .iov_base = buf + ETHER_ADDR_LEN * 2,
-                .iov_len = size - ETHER_ADDR_LEN * 2 },
+            { .iov_base = buf + ETH_ALEN * 2,
+                .iov_len = size - ETH_ALEN * 2 },
         };
 
         memcpy(vlan_iov, iov, sizeof(vlan_iov));
@@ -1867,65 +1843,8 @@
     return 1;
 }
 
-/* structures and macros for task offloading */
-typedef struct ip_header
-{
-    uint8_t  ip_ver_len;    /* version and header length */
-    uint8_t  ip_tos;        /* type of service */
-    uint16_t ip_len;        /* total length */
-    uint16_t ip_id;         /* identification */
-    uint16_t ip_off;        /* fragment offset field */
-    uint8_t  ip_ttl;        /* time to live */
-    uint8_t  ip_p;          /* protocol */
-    uint16_t ip_sum;        /* checksum */
-    uint32_t ip_src,ip_dst; /* source and dest address */
-} ip_header;
-
-#define IP_HEADER_VERSION_4 4
-#define IP_HEADER_VERSION(ip) ((ip->ip_ver_len >> 4)&0xf)
-#define IP_HEADER_LENGTH(ip) (((ip->ip_ver_len)&0xf) << 2)
-
-typedef struct tcp_header
-{
-    uint16_t th_sport;		/* source port */
-    uint16_t th_dport;		/* destination port */
-    uint32_t th_seq;			/* sequence number */
-    uint32_t th_ack;			/* acknowledgement number */
-    uint16_t th_offset_flags; /* data offset, reserved 6 bits, TCP protocol flags */
-    uint16_t th_win;			/* window */
-    uint16_t th_sum;			/* checksum */
-    uint16_t th_urp;			/* urgent pointer */
-} tcp_header;
-
-typedef struct udp_header
-{
-    uint16_t uh_sport; /* source port */
-    uint16_t uh_dport; /* destination port */
-    uint16_t uh_ulen;  /* udp length */
-    uint16_t uh_sum;   /* udp checksum */
-} udp_header;
-
-typedef struct ip_pseudo_header
-{
-    uint32_t ip_src;
-    uint32_t ip_dst;
-    uint8_t  zeros;
-    uint8_t  ip_proto;
-    uint16_t ip_payload;
-} ip_pseudo_header;
-
-#define IP_PROTO_TCP 6
-#define IP_PROTO_UDP 17
-
-#define TCP_HEADER_DATA_OFFSET(tcp) (((be16_to_cpu(tcp->th_offset_flags) >> 12)&0xf) << 2)
-#define TCP_FLAGS_ONLY(flags) ((flags)&0x3f)
-#define TCP_HEADER_FLAGS(tcp) TCP_FLAGS_ONLY(be16_to_cpu(tcp->th_offset_flags))
-
 #define TCP_HEADER_CLEAR_FLAGS(tcp, off) ((tcp)->th_offset_flags &= cpu_to_be16(~TCP_FLAGS_ONLY(off)))
 
-#define TCP_FLAG_FIN  0x01
-#define TCP_FLAG_PUSH 0x08
-
 /* produces ones' complement sum of data */
 static uint16_t ones_complement_sum(uint8_t *data, size_t len)
 {
@@ -2075,20 +1994,6 @@
                 "length to %d\n", txsize);
     }
 
-    if (!s->cplus_txbuffer)
-    {
-        /* out of memory */
-
-        DPRINTF("+++ C+ mode transmiter failed to reallocate %d bytes\n",
-            s->cplus_txbuffer_len);
-
-        /* update tally counter */
-        ++s->tally_counters.TxERR;
-        ++s->tally_counters.TxAbt;
-
-        return 0;
-    }
-
     /* append more data to the packet */
 
     DPRINTF("+++ C+ mode transmit reading %d bytes from host memory at "
@@ -2112,7 +2017,7 @@
     }
 
     /* transfer ownership to target */
-    txdw0 &= ~CP_RX_OWN;
+    txdw0 &= ~CP_TX_OWN;
 
     /* reset error indicator bits */
     txdw0 &= ~CP_TX_STATUS_UNF;
@@ -2148,7 +2053,7 @@
                 bswap16(txdw1 & CP_TX_VLAN_TAG_MASK));
 
             dot1q_buffer = (uint16_t *) dot1q_buffer_space;
-            dot1q_buffer[0] = cpu_to_be16(ETH_P_8021Q);
+            dot1q_buffer[0] = cpu_to_be16(ETH_P_VLAN);
             /* BE + le_to_cpu() + ~cpu_to_le()~ = BE */
             dot1q_buffer[1] = cpu_to_le16(txdw1 & CP_TX_VLAN_TAG_MASK);
         } else {
@@ -2164,8 +2069,13 @@
         {
             DPRINTF("+++ C+ mode offloaded task checksum\n");
 
+            /* Large enough for Ethernet and IP headers? */
+            if (saved_size < ETH_HLEN + sizeof(struct ip_header)) {
+                goto skip_offload;
+            }
+
             /* ip packet header */
-            ip_header *ip = NULL;
+            struct ip_header *ip = NULL;
             int hlen = 0;
             uint8_t  ip_protocol = 0;
             uint16_t ip_data_len = 0;
@@ -2174,223 +2084,240 @@
             size_t   eth_payload_len  = 0;
 
             int proto = be16_to_cpu(*(uint16_t *)(saved_buffer + 12));
-            if (proto == ETH_P_IP)
+            if (proto != ETH_P_IP)
             {
-                DPRINTF("+++ C+ mode has IP packet\n");
-
-                /* not aligned */
-                eth_payload_data = saved_buffer + ETH_HLEN;
-                eth_payload_len  = saved_size   - ETH_HLEN;
-
-                ip = (ip_header*)eth_payload_data;
-
-                if (IP_HEADER_VERSION(ip) != IP_HEADER_VERSION_4) {
-                    DPRINTF("+++ C+ mode packet has bad IP version %d "
-                        "expected %d\n", IP_HEADER_VERSION(ip),
-                        IP_HEADER_VERSION_4);
-                    ip = NULL;
-                } else {
-                    hlen = IP_HEADER_LENGTH(ip);
-                    ip_protocol = ip->ip_p;
-                    ip_data_len = be16_to_cpu(ip->ip_len) - hlen;
-                }
+                goto skip_offload;
             }
 
-            if (ip)
+            DPRINTF("+++ C+ mode has IP packet\n");
+
+            /* Note on memory alignment: eth_payload_data is 16-bit aligned
+             * since saved_buffer is allocated with g_malloc() and ETH_HLEN is
+             * even.  32-bit accesses must use ldl/stl wrappers to avoid
+             * unaligned accesses.
+             */
+            eth_payload_data = saved_buffer + ETH_HLEN;
+            eth_payload_len  = saved_size   - ETH_HLEN;
+
+            ip = (struct ip_header*)eth_payload_data;
+
+            if (IP_HEADER_VERSION(ip) != IP_HEADER_VERSION_4) {
+                DPRINTF("+++ C+ mode packet has bad IP version %d "
+                    "expected %d\n", IP_HEADER_VERSION(ip),
+                    IP_HEADER_VERSION_4);
+                goto skip_offload;
+            }
+
+            hlen = IP_HDR_GET_LEN(ip);
+            if (hlen < sizeof(struct ip_header) || hlen > eth_payload_len) {
+                goto skip_offload;
+            }
+
+            ip_protocol = ip->ip_p;
+
+            ip_data_len = be16_to_cpu(ip->ip_len);
+            if (ip_data_len < hlen || ip_data_len > eth_payload_len) {
+                goto skip_offload;
+            }
+            ip_data_len -= hlen;
+
+            if (txdw0 & CP_TX_IPCS)
             {
-                if (txdw0 & CP_TX_IPCS)
-                {
-                    DPRINTF("+++ C+ mode need IP checksum\n");
+                DPRINTF("+++ C+ mode need IP checksum\n");
 
-                    if (hlen<sizeof(ip_header) || hlen>eth_payload_len) {/* min header length */
-                        /* bad packet header len */
-                        /* or packet too short */
-                    }
-                    else
-                    {
-                        ip->ip_sum = 0;
-                        ip->ip_sum = ip_checksum(ip, hlen);
-                        DPRINTF("+++ C+ mode IP header len=%d checksum=%04x\n",
-                            hlen, ip->ip_sum);
-                    }
+                ip->ip_sum = 0;
+                ip->ip_sum = ip_checksum(ip, hlen);
+                DPRINTF("+++ C+ mode IP header len=%d checksum=%04x\n",
+                    hlen, ip->ip_sum);
+            }
+
+            if ((txdw0 & CP_TX_LGSEN) && ip_protocol == IP_PROTO_TCP)
+            {
+                /* Large enough for the TCP header? */
+                if (ip_data_len < sizeof(tcp_header)) {
+                    goto skip_offload;
                 }
 
-                if ((txdw0 & CP_TX_LGSEN) && ip_protocol == IP_PROTO_TCP)
+                int large_send_mss = (txdw0 >> 16) & CP_TC_LGSEN_MSS_MASK;
+
+                DPRINTF("+++ C+ mode offloaded task TSO MTU=%d IP data %d "
+                    "frame data %d specified MSS=%d\n", ETH_MTU,
+                    ip_data_len, saved_size - ETH_HLEN, large_send_mss);
+
+                int tcp_send_offset = 0;
+                int send_count = 0;
+
+                /* maximum IP header length is 60 bytes */
+                uint8_t saved_ip_header[60];
+
+                /* save IP header template; data area is used in tcp checksum calculation */
+                memcpy(saved_ip_header, eth_payload_data, hlen);
+
+                /* a placeholder for checksum calculation routine in tcp case */
+                uint8_t *data_to_checksum     = eth_payload_data + hlen - 12;
+                //                    size_t   data_to_checksum_len = eth_payload_len  - hlen + 12;
+
+                /* pointer to TCP header */
+                tcp_header *p_tcp_hdr = (tcp_header*)(eth_payload_data + hlen);
+
+                int tcp_hlen = TCP_HEADER_DATA_OFFSET(p_tcp_hdr);
+
+                /* Invalid TCP data offset? */
+                if (tcp_hlen < sizeof(tcp_header) || tcp_hlen > ip_data_len) {
+                    goto skip_offload;
+                }
+
+                /* ETH_MTU = ip header len + tcp header len + payload */
+                int tcp_data_len = ip_data_len - tcp_hlen;
+                int tcp_chunk_size = ETH_MTU - hlen - tcp_hlen;
+
+                DPRINTF("+++ C+ mode TSO IP data len %d TCP hlen %d TCP "
+                    "data len %d TCP chunk size %d\n", ip_data_len,
+                    tcp_hlen, tcp_data_len, tcp_chunk_size);
+
+                /* note the cycle below overwrites IP header data,
+                   but restores it from saved_ip_header before sending packet */
+
+                int is_last_frame = 0;
+
+                for (tcp_send_offset = 0; tcp_send_offset < tcp_data_len; tcp_send_offset += tcp_chunk_size)
                 {
-                    int large_send_mss = (txdw0 >> 16) & CP_TC_LGSEN_MSS_MASK;
+                    uint16_t chunk_size = tcp_chunk_size;
 
-                    DPRINTF("+++ C+ mode offloaded task TSO MTU=%d IP data %d "
-                        "frame data %d specified MSS=%d\n", ETH_MTU,
-                        ip_data_len, saved_size - ETH_HLEN, large_send_mss);
-
-                    int tcp_send_offset = 0;
-                    int send_count = 0;
-
-                    /* maximum IP header length is 60 bytes */
-                    uint8_t saved_ip_header[60];
-
-                    /* save IP header template; data area is used in tcp checksum calculation */
-                    memcpy(saved_ip_header, eth_payload_data, hlen);
-
-                    /* a placeholder for checksum calculation routine in tcp case */
-                    uint8_t *data_to_checksum     = eth_payload_data + hlen - 12;
-                    //                    size_t   data_to_checksum_len = eth_payload_len  - hlen + 12;
-
-                    /* pointer to TCP header */
-                    tcp_header *p_tcp_hdr = (tcp_header*)(eth_payload_data + hlen);
-
-                    int tcp_hlen = TCP_HEADER_DATA_OFFSET(p_tcp_hdr);
-
-                    /* ETH_MTU = ip header len + tcp header len + payload */
-                    int tcp_data_len = ip_data_len - tcp_hlen;
-                    int tcp_chunk_size = ETH_MTU - hlen - tcp_hlen;
-
-                    DPRINTF("+++ C+ mode TSO IP data len %d TCP hlen %d TCP "
-                        "data len %d TCP chunk size %d\n", ip_data_len,
-                        tcp_hlen, tcp_data_len, tcp_chunk_size);
-
-                    /* note the cycle below overwrites IP header data,
-                       but restores it from saved_ip_header before sending packet */
-
-                    int is_last_frame = 0;
-
-                    for (tcp_send_offset = 0; tcp_send_offset < tcp_data_len; tcp_send_offset += tcp_chunk_size)
+                    /* check if this is the last frame */
+                    if (tcp_send_offset + tcp_chunk_size >= tcp_data_len)
                     {
-                        uint16_t chunk_size = tcp_chunk_size;
-
-                        /* check if this is the last frame */
-                        if (tcp_send_offset + tcp_chunk_size >= tcp_data_len)
-                        {
-                            is_last_frame = 1;
-                            chunk_size = tcp_data_len - tcp_send_offset;
-                        }
-
-                        DPRINTF("+++ C+ mode TSO TCP seqno %08x\n",
-                            be32_to_cpu(p_tcp_hdr->th_seq));
-
-                        /* add 4 TCP pseudoheader fields */
-                        /* copy IP source and destination fields */
-                        memcpy(data_to_checksum, saved_ip_header + 12, 8);
-
-                        DPRINTF("+++ C+ mode TSO calculating TCP checksum for "
-                            "packet with %d bytes data\n", tcp_hlen +
-                            chunk_size);
-
-                        if (tcp_send_offset)
-                        {
-                            memcpy((uint8_t*)p_tcp_hdr + tcp_hlen, (uint8_t*)p_tcp_hdr + tcp_hlen + tcp_send_offset, chunk_size);
-                        }
-
-                        /* keep PUSH and FIN flags only for the last frame */
-                        if (!is_last_frame)
-                        {
-                            TCP_HEADER_CLEAR_FLAGS(p_tcp_hdr, TCP_FLAG_PUSH|TCP_FLAG_FIN);
-                        }
-
-                        /* recalculate TCP checksum */
-                        ip_pseudo_header *p_tcpip_hdr = (ip_pseudo_header *)data_to_checksum;
-                        p_tcpip_hdr->zeros      = 0;
-                        p_tcpip_hdr->ip_proto   = IP_PROTO_TCP;
-                        p_tcpip_hdr->ip_payload = cpu_to_be16(tcp_hlen + chunk_size);
-
-                        p_tcp_hdr->th_sum = 0;
-
-                        int tcp_checksum = ip_checksum(data_to_checksum, tcp_hlen + chunk_size + 12);
-                        DPRINTF("+++ C+ mode TSO TCP checksum %04x\n",
-                            tcp_checksum);
-
-                        p_tcp_hdr->th_sum = tcp_checksum;
-
-                        /* restore IP header */
-                        memcpy(eth_payload_data, saved_ip_header, hlen);
-
-                        /* set IP data length and recalculate IP checksum */
-                        ip->ip_len = cpu_to_be16(hlen + tcp_hlen + chunk_size);
-
-                        /* increment IP id for subsequent frames */
-                        ip->ip_id = cpu_to_be16(tcp_send_offset/tcp_chunk_size + be16_to_cpu(ip->ip_id));
-
-                        ip->ip_sum = 0;
-                        ip->ip_sum = ip_checksum(eth_payload_data, hlen);
-                        DPRINTF("+++ C+ mode TSO IP header len=%d "
-                            "checksum=%04x\n", hlen, ip->ip_sum);
-
-                        int tso_send_size = ETH_HLEN + hlen + tcp_hlen + chunk_size;
-                        DPRINTF("+++ C+ mode TSO transferring packet size "
-                            "%d\n", tso_send_size);
-                        rtl8139_transfer_frame(s, saved_buffer, tso_send_size,
-                            0, (uint8_t *) dot1q_buffer);
-
-                        /* add transferred count to TCP sequence number */
-                        p_tcp_hdr->th_seq = cpu_to_be32(chunk_size + be32_to_cpu(p_tcp_hdr->th_seq));
-                        ++send_count;
+                        is_last_frame = 1;
+                        chunk_size = tcp_data_len - tcp_send_offset;
                     }
 
-                    /* Stop sending this frame */
-                    saved_size = 0;
-                }
-                else if (txdw0 & (CP_TX_TCPCS|CP_TX_UDPCS))
-                {
-                    DPRINTF("+++ C+ mode need TCP or UDP checksum\n");
-
-                    /* maximum IP header length is 60 bytes */
-                    uint8_t saved_ip_header[60];
-                    memcpy(saved_ip_header, eth_payload_data, hlen);
-
-                    uint8_t *data_to_checksum     = eth_payload_data + hlen - 12;
-                    //                    size_t   data_to_checksum_len = eth_payload_len  - hlen + 12;
+                    DPRINTF("+++ C+ mode TSO TCP seqno %08x\n",
+                            ldl_be_p(&p_tcp_hdr->th_seq));
 
                     /* add 4 TCP pseudoheader fields */
                     /* copy IP source and destination fields */
                     memcpy(data_to_checksum, saved_ip_header + 12, 8);
 
-                    if ((txdw0 & CP_TX_TCPCS) && ip_protocol == IP_PROTO_TCP)
+                    DPRINTF("+++ C+ mode TSO calculating TCP checksum for "
+                        "packet with %d bytes data\n", tcp_hlen +
+                        chunk_size);
+
+                    if (tcp_send_offset)
                     {
-                        DPRINTF("+++ C+ mode calculating TCP checksum for "
-                            "packet with %d bytes data\n", ip_data_len);
-
-                        ip_pseudo_header *p_tcpip_hdr = (ip_pseudo_header *)data_to_checksum;
-                        p_tcpip_hdr->zeros      = 0;
-                        p_tcpip_hdr->ip_proto   = IP_PROTO_TCP;
-                        p_tcpip_hdr->ip_payload = cpu_to_be16(ip_data_len);
-
-                        tcp_header* p_tcp_hdr = (tcp_header *) (data_to_checksum+12);
-
-                        p_tcp_hdr->th_sum = 0;
-
-                        int tcp_checksum = ip_checksum(data_to_checksum, ip_data_len + 12);
-                        DPRINTF("+++ C+ mode TCP checksum %04x\n",
-                            tcp_checksum);
-
-                        p_tcp_hdr->th_sum = tcp_checksum;
+                        memcpy((uint8_t*)p_tcp_hdr + tcp_hlen, (uint8_t*)p_tcp_hdr + tcp_hlen + tcp_send_offset, chunk_size);
                     }
-                    else if ((txdw0 & CP_TX_UDPCS) && ip_protocol == IP_PROTO_UDP)
+
+                    /* keep PUSH and FIN flags only for the last frame */
+                    if (!is_last_frame)
                     {
-                        DPRINTF("+++ C+ mode calculating UDP checksum for "
-                            "packet with %d bytes data\n", ip_data_len);
-
-                        ip_pseudo_header *p_udpip_hdr = (ip_pseudo_header *)data_to_checksum;
-                        p_udpip_hdr->zeros      = 0;
-                        p_udpip_hdr->ip_proto   = IP_PROTO_UDP;
-                        p_udpip_hdr->ip_payload = cpu_to_be16(ip_data_len);
-
-                        udp_header *p_udp_hdr = (udp_header *) (data_to_checksum+12);
-
-                        p_udp_hdr->uh_sum = 0;
-
-                        int udp_checksum = ip_checksum(data_to_checksum, ip_data_len + 12);
-                        DPRINTF("+++ C+ mode UDP checksum %04x\n",
-                            udp_checksum);
-
-                        p_udp_hdr->uh_sum = udp_checksum;
+                        TCP_HEADER_CLEAR_FLAGS(p_tcp_hdr, TH_PUSH | TH_FIN);
                     }
 
+                    /* recalculate TCP checksum */
+                    ip_pseudo_header *p_tcpip_hdr = (ip_pseudo_header *)data_to_checksum;
+                    p_tcpip_hdr->zeros      = 0;
+                    p_tcpip_hdr->ip_proto   = IP_PROTO_TCP;
+                    p_tcpip_hdr->ip_payload = cpu_to_be16(tcp_hlen + chunk_size);
+
+                    p_tcp_hdr->th_sum = 0;
+
+                    int tcp_checksum = ip_checksum(data_to_checksum, tcp_hlen + chunk_size + 12);
+                    DPRINTF("+++ C+ mode TSO TCP checksum %04x\n",
+                        tcp_checksum);
+
+                    p_tcp_hdr->th_sum = tcp_checksum;
+
                     /* restore IP header */
                     memcpy(eth_payload_data, saved_ip_header, hlen);
+
+                    /* set IP data length and recalculate IP checksum */
+                    ip->ip_len = cpu_to_be16(hlen + tcp_hlen + chunk_size);
+
+                    /* increment IP id for subsequent frames */
+                    ip->ip_id = cpu_to_be16(tcp_send_offset/tcp_chunk_size + be16_to_cpu(ip->ip_id));
+
+                    ip->ip_sum = 0;
+                    ip->ip_sum = ip_checksum(eth_payload_data, hlen);
+                    DPRINTF("+++ C+ mode TSO IP header len=%d "
+                        "checksum=%04x\n", hlen, ip->ip_sum);
+
+                    int tso_send_size = ETH_HLEN + hlen + tcp_hlen + chunk_size;
+                    DPRINTF("+++ C+ mode TSO transferring packet size "
+                        "%d\n", tso_send_size);
+                    rtl8139_transfer_frame(s, saved_buffer, tso_send_size,
+                        0, (uint8_t *) dot1q_buffer);
+
+                    /* add transferred count to TCP sequence number */
+                    stl_be_p(&p_tcp_hdr->th_seq,
+                             chunk_size + ldl_be_p(&p_tcp_hdr->th_seq));
+                    ++send_count;
                 }
+
+                /* Stop sending this frame */
+                saved_size = 0;
+            }
+            else if (txdw0 & (CP_TX_TCPCS|CP_TX_UDPCS))
+            {
+                DPRINTF("+++ C+ mode need TCP or UDP checksum\n");
+
+                /* maximum IP header length is 60 bytes */
+                uint8_t saved_ip_header[60];
+                memcpy(saved_ip_header, eth_payload_data, hlen);
+
+                uint8_t *data_to_checksum     = eth_payload_data + hlen - 12;
+                //                    size_t   data_to_checksum_len = eth_payload_len  - hlen + 12;
+
+                /* add 4 TCP pseudoheader fields */
+                /* copy IP source and destination fields */
+                memcpy(data_to_checksum, saved_ip_header + 12, 8);
+
+                if ((txdw0 & CP_TX_TCPCS) && ip_protocol == IP_PROTO_TCP)
+                {
+                    DPRINTF("+++ C+ mode calculating TCP checksum for "
+                        "packet with %d bytes data\n", ip_data_len);
+
+                    ip_pseudo_header *p_tcpip_hdr = (ip_pseudo_header *)data_to_checksum;
+                    p_tcpip_hdr->zeros      = 0;
+                    p_tcpip_hdr->ip_proto   = IP_PROTO_TCP;
+                    p_tcpip_hdr->ip_payload = cpu_to_be16(ip_data_len);
+
+                    tcp_header* p_tcp_hdr = (tcp_header *) (data_to_checksum+12);
+
+                    p_tcp_hdr->th_sum = 0;
+
+                    int tcp_checksum = ip_checksum(data_to_checksum, ip_data_len + 12);
+                    DPRINTF("+++ C+ mode TCP checksum %04x\n",
+                        tcp_checksum);
+
+                    p_tcp_hdr->th_sum = tcp_checksum;
+                }
+                else if ((txdw0 & CP_TX_UDPCS) && ip_protocol == IP_PROTO_UDP)
+                {
+                    DPRINTF("+++ C+ mode calculating UDP checksum for "
+                        "packet with %d bytes data\n", ip_data_len);
+
+                    ip_pseudo_header *p_udpip_hdr = (ip_pseudo_header *)data_to_checksum;
+                    p_udpip_hdr->zeros      = 0;
+                    p_udpip_hdr->ip_proto   = IP_PROTO_UDP;
+                    p_udpip_hdr->ip_payload = cpu_to_be16(ip_data_len);
+
+                    udp_header *p_udp_hdr = (udp_header *) (data_to_checksum+12);
+
+                    p_udp_hdr->uh_sum = 0;
+
+                    int udp_checksum = ip_checksum(data_to_checksum, ip_data_len + 12);
+                    DPRINTF("+++ C+ mode UDP checksum %04x\n",
+                        udp_checksum);
+
+                    p_udp_hdr->uh_sum = udp_checksum;
+                }
+
+                /* restore IP header */
+                memcpy(eth_payload_data, saved_ip_header, hlen);
             }
         }
 
+skip_offload:
         /* update tally counter */
         ++s->tally_counters.TxOk;
 
@@ -2652,7 +2579,6 @@
 
     s->IntrMask = val;
 
-    rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
     rtl8139_update_irq(s);
 
 }
@@ -2687,13 +2613,7 @@
     rtl8139_update_irq(s);
 
     s->IntrStatus = newStatus;
-    /*
-     * Computing if we miss an interrupt here is not that correct but
-     * considered that we should have had already an interrupt
-     * and probably emulated is slower is better to assume this resetting was
-     * done before testing on previous rtl8139_update_irq lead to IRQ losing
-     */
-    rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+    rtl8139_set_next_tctr_time(s);
     rtl8139_update_irq(s);
 
 #endif
@@ -2701,8 +2621,6 @@
 
 static uint32_t rtl8139_IntrStatus_read(RTL8139State *s)
 {
-    rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
-
     uint32_t ret = s->IntrStatus;
 
     DPRINTF("IntrStatus read(w) val=0x%04x\n", ret);
@@ -2885,43 +2803,31 @@
     }
 }
 
-static void rtl8139_set_next_tctr_time(RTL8139State *s, int64_t current_time)
+static void rtl8139_set_next_tctr_time(RTL8139State *s)
 {
-    int64_t pci_time, next_time;
-    uint32_t low_pci;
+    const uint64_t ns_per_period = (uint64_t)PCI_PERIOD << 32;
 
     DPRINTF("entered rtl8139_set_next_tctr_time\n");
 
-    if (s->TimerExpire && current_time >= s->TimerExpire) {
-        s->IntrStatus |= PCSTimeout;
-        rtl8139_update_irq(s);
-    }
-
-    /* Set QEMU timer only if needed that is
-     * - TimerInt <> 0 (we have a timer)
-     * - mask = 1 (we want an interrupt timer)
-     * - irq = 0  (irq is not already active)
-     * If any of above change we need to compute timer again
-     * Also we must check if timer is passed without QEMU timer
+    /* This function is called at least once per period, so it is a good
+     * place to update the timer base.
+     *
+     * After one iteration of this loop the value in the Timer register does
+     * not change, but the device model is counting up by 2^32 ticks (approx.
+     * 130 seconds).
      */
-    s->TimerExpire = 0;
+    while (s->TCTR_base + ns_per_period <= qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)) {
+        s->TCTR_base += ns_per_period;
+    }
+
     if (!s->TimerInt) {
-        return;
-    }
-
-    pci_time = muldiv64(current_time - s->TCTR_base, PCI_FREQUENCY,
-                                get_ticks_per_sec());
-    low_pci = pci_time & 0xffffffff;
-    pci_time = pci_time - low_pci + s->TimerInt;
-    if (low_pci >= s->TimerInt) {
-        pci_time += 0x100000000LL;
-    }
-    next_time = s->TCTR_base + muldiv64(pci_time, get_ticks_per_sec(),
-                                                PCI_FREQUENCY);
-    s->TimerExpire = next_time;
-
-    if ((s->IntrMask & PCSTimeout) != 0 && (s->IntrStatus & PCSTimeout) == 0) {
-        timer_mod(s->timer, next_time);
+        timer_del(s->timer);
+    } else {
+        uint64_t delta = (uint64_t)s->TimerInt * PCI_PERIOD;
+        if (s->TCTR_base + delta <= qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)) {
+            delta += ns_per_period;
+        }
+        timer_mod(s->timer, s->TCTR_base + delta);
     }
 }
 
@@ -2969,14 +2875,14 @@
         case Timer:
             DPRINTF("TCTR Timer reset on write\n");
             s->TCTR_base = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-            rtl8139_set_next_tctr_time(s, s->TCTR_base);
+            rtl8139_set_next_tctr_time(s);
             break;
 
         case FlashReg:
             DPRINTF("FlashReg TimerInt write val=0x%08x\n", val);
             if (s->TimerInt != val) {
                 s->TimerInt = val;
-                rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+                rtl8139_set_next_tctr_time(s);
             }
             break;
 
@@ -3191,8 +3097,8 @@
             break;
 
         case Timer:
-            ret = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->TCTR_base,
-                           PCI_FREQUENCY, get_ticks_per_sec());
+            ret = (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->TCTR_base) /
+                  PCI_PERIOD;
             DPRINTF("TCTR Timer read val=0x%08x\n", ret);
             break;
 
@@ -3253,7 +3159,7 @@
 static int rtl8139_post_load(void *opaque, int version_id)
 {
     RTL8139State* s = opaque;
-    rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+    rtl8139_set_next_tctr_time(s);
     if (version_id < 4) {
         s->cplus_enabled = s->CpCmd != 0;
     }
@@ -3274,6 +3180,7 @@
     .name = "rtl8139/hotplug_ready",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = rtl8139_hotplug_ready_needed,
     .fields = (VMStateField[]) {
         VMSTATE_END_OF_LIST()
     }
@@ -3284,16 +3191,14 @@
     RTL8139State* s = opaque;
     int64_t current_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
-    /* set IntrStatus correctly */
-    rtl8139_set_next_tctr_time(s, current_time);
-    s->TCTR = muldiv64(current_time - s->TCTR_base, PCI_FREQUENCY,
-                       get_ticks_per_sec());
+    /* for migration to older versions */
+    s->TCTR = (current_time - s->TCTR_base) / PCI_PERIOD;
     s->rtl8139_mmio_io_addr_dummy = 0;
 }
 
 static const VMStateDescription vmstate_rtl8139 = {
     .name = "rtl8139",
-    .version_id = 4,
+    .version_id = 5,
     .minimum_version_id = 3,
     .post_load = rtl8139_post_load,
     .pre_save  = rtl8139_pre_save,
@@ -3364,19 +3269,26 @@
         VMSTATE_UINT32(TimerInt, RTL8139State),
         VMSTATE_INT64(TCTR_base, RTL8139State),
 
-        VMSTATE_STRUCT(tally_counters, RTL8139State, 0,
-                       vmstate_tally_counters, RTL8139TallyCounters),
+        VMSTATE_UINT64(tally_counters.TxOk, RTL8139State),
+        VMSTATE_UINT64(tally_counters.RxOk, RTL8139State),
+        VMSTATE_UINT64(tally_counters.TxERR, RTL8139State),
+        VMSTATE_UINT32(tally_counters.RxERR, RTL8139State),
+        VMSTATE_UINT16(tally_counters.MissPkt, RTL8139State),
+        VMSTATE_UINT16(tally_counters.FAE, RTL8139State),
+        VMSTATE_UINT32(tally_counters.Tx1Col, RTL8139State),
+        VMSTATE_UINT32(tally_counters.TxMCol, RTL8139State),
+        VMSTATE_UINT64(tally_counters.RxOkPhy, RTL8139State),
+        VMSTATE_UINT64(tally_counters.RxOkBrd, RTL8139State),
+        VMSTATE_UINT32_V(tally_counters.RxOkMul, RTL8139State, 5),
+        VMSTATE_UINT16(tally_counters.TxAbt, RTL8139State),
+        VMSTATE_UINT16(tally_counters.TxUndrn, RTL8139State),
 
         VMSTATE_UINT32_V(cplus_enabled, RTL8139State, 4),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_rtl8139_hotplug_ready,
-            .needed = rtl8139_hotplug_ready_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_rtl8139_hotplug_ready,
+        NULL
     }
 };
 
@@ -3452,24 +3364,15 @@
 
     s->IntrStatus |= PCSTimeout;
     rtl8139_update_irq(s);
-    rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
-}
-
-static void rtl8139_cleanup(NetClientState *nc)
-{
-    RTL8139State *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
+    rtl8139_set_next_tctr_time(s);
 }
 
 static void pci_rtl8139_uninit(PCIDevice *dev)
 {
     RTL8139State *s = RTL8139(dev);
 
-    if (s->cplus_txbuffer) {
-        g_free(s->cplus_txbuffer);
-        s->cplus_txbuffer = NULL;
-    }
+    g_free(s->cplus_txbuffer);
+    s->cplus_txbuffer = NULL;
     timer_del(s->timer);
     timer_free(s->timer);
     qemu_del_nic(s->nic);
@@ -3490,15 +3393,14 @@
 }
 
 static NetClientInfo net_rtl8139_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = rtl8139_can_receive,
     .receive = rtl8139_receive,
-    .cleanup = rtl8139_cleanup,
     .link_status_changed = rtl8139_set_link_status,
 };
 
-static int pci_rtl8139_init(PCIDevice *dev)
+static void pci_rtl8139_realize(PCIDevice *dev, Error **errp)
 {
     RTL8139State *s = RTL8139(dev);
     DeviceState *d = DEVICE(dev);
@@ -3538,11 +3440,7 @@
     s->cplus_txbuffer_len = 0;
     s->cplus_txbuffer_offset = 0;
 
-    s->TimerExpire = 0;
     s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, rtl8139_timer, s);
-    rtl8139_set_next_tctr_time(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
-
-    return 0;
 }
 
 static void rtl8139_instance_init(Object *obj)
@@ -3564,7 +3462,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_rtl8139_init;
+    k->realize = pci_rtl8139_realize;
     k->exit = pci_rtl8139_uninit;
     k->romfile = "efi-rtl8139.rom";
     k->vendor_id = PCI_VENDOR_ID_REALTEK;
diff --git a/hw/net/smc91c111.c b/hw/net/smc91c111.c
index d1dca8f..3b16dcf 100644
--- a/hw/net/smc91c111.c
+++ b/hw/net/smc91c111.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include "hw/devices.h"
@@ -124,6 +125,25 @@
     qemu_set_irq(s->irq, level);
 }
 
+static int smc91c111_can_receive(smc91c111_state *s)
+{
+    if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST)) {
+        return 1;
+    }
+    if (s->allocated == (1 << NUM_PACKETS) - 1 ||
+        s->rx_fifo_len == NUM_PACKETS) {
+        return 0;
+    }
+    return 1;
+}
+
+static inline void smc91c111_flush_queued_packets(smc91c111_state *s)
+{
+    if (smc91c111_can_receive(s)) {
+        qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    }
+}
+
 /* Try to allocate a packet.  Returns 0x80 on failure.  */
 static int smc91c111_allocate_packet(smc91c111_state *s)
 {
@@ -164,6 +184,7 @@
     } else {
         s->int_level &= ~INT_RCV;
     }
+    smc91c111_flush_queued_packets(s);
     smc91c111_update(s);
 }
 
@@ -185,7 +206,7 @@
     s->allocated &= ~(1 << packet);
     if (s->tx_alloc == 0x80)
         smc91c111_tx_alloc(s);
-    qemu_flush_queued_packets(qemu_get_queue(s->nic));
+    smc91c111_flush_queued_packets(s);
 }
 
 /* Flush the TX FIFO.  */
@@ -311,6 +332,7 @@
             if (s->rcr & RCR_SOFT_RST) {
                 smc91c111_reset(DEVICE(s));
             }
+            smc91c111_flush_queued_packets(s);
             return;
         case 10: case 11: /* RPCR */
             /* Ignored */
@@ -636,15 +658,11 @@
     return val;
 }
 
-static int smc91c111_can_receive(NetClientState *nc)
+static int smc91c111_can_receive_nc(NetClientState *nc)
 {
     smc91c111_state *s = qemu_get_nic_opaque(nc);
 
-    if ((s->rcr & RCR_RXEN) == 0 || (s->rcr & RCR_SOFT_RST))
-        return 1;
-    if (s->allocated == (1 << NUM_PACKETS) - 1)
-        return 0;
-    return 1;
+    return smc91c111_can_receive(s);
 }
 
 static ssize_t smc91c111_receive(NetClientState *nc, const uint8_t *buf, size_t size)
@@ -736,19 +754,11 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void smc91c111_cleanup(NetClientState *nc)
-{
-    smc91c111_state *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_smc91c111_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = smc91c111_can_receive,
+    .can_receive = smc91c111_can_receive_nc,
     .receive = smc91c111_receive,
-    .cleanup = smc91c111_cleanup,
 };
 
 static int smc91c111_init1(SysBusDevice *sbd)
diff --git a/hw/net/spapr_llan.c b/hw/net/spapr_llan.c
index 2c8b038..b273eda 100644
--- a/hw/net/spapr_llan.c
+++ b/hw/net/spapr_llan.c
@@ -24,7 +24,11 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
+#include "qemu/log.h"
 #include "net/net.h"
 #include "hw/qdev.h"
 #include "hw/ppc/spapr.h"
@@ -44,6 +48,10 @@
 #define DPRINTF(fmt...)
 #endif
 
+/* Compatibility flags for migration */
+#define SPAPRVLAN_FLAG_RX_BUF_POOLS_BIT  0
+#define SPAPRVLAN_FLAG_RX_BUF_POOLS      (1 << SPAPRVLAN_FLAG_RX_BUF_POOLS_BIT)
+
 /*
  * Virtual LAN device
  */
@@ -85,14 +93,26 @@
 #define VIO_SPAPR_VLAN_DEVICE(obj) \
      OBJECT_CHECK(VIOsPAPRVLANDevice, (obj), TYPE_VIO_SPAPR_VLAN_DEVICE)
 
+#define RX_POOL_MAX_BDS 4096
+#define RX_MAX_POOLS 5
+
+typedef struct {
+    int32_t bufsize;
+    int32_t count;
+    vlan_bd_t bds[RX_POOL_MAX_BDS];
+} RxBufPool;
+
 typedef struct VIOsPAPRVLANDevice {
     VIOsPAPRDevice sdev;
     NICConf nicconf;
     NICState *nic;
     bool isopen;
-    target_ulong buf_list;
+    hwaddr buf_list;
     uint32_t add_buf_ptr, use_buf_ptr, rx_bufs;
-    target_ulong rxq_ptr;
+    hwaddr rxq_ptr;
+    QEMUTimer *rxp_timer;
+    uint32_t compat_flags;             /* Compatability flags for migration */
+    RxBufPool *rx_pool[RX_MAX_POOLS];  /* Receive buffer descriptor pools */
 } VIOsPAPRVLANDevice;
 
 static int spapr_vlan_can_receive(NetClientState *nc)
@@ -102,6 +122,88 @@
     return (dev->isopen && dev->rx_bufs > 0);
 }
 
+/**
+ * The last 8 bytes of the receive buffer list page (that has been
+ * supplied by the guest with the H_REGISTER_LOGICAL_LAN call) contain
+ * a counter for frames that have been dropped because there was no
+ * suitable receive buffer available. This function is used to increase
+ * this counter by one.
+ */
+static void spapr_vlan_record_dropped_rx_frame(VIOsPAPRVLANDevice *dev)
+{
+    uint64_t cnt;
+
+    cnt = vio_ldq(&dev->sdev, dev->buf_list + 4096 - 8);
+    vio_stq(&dev->sdev, dev->buf_list + 4096 - 8, cnt + 1);
+}
+
+/**
+ * Get buffer descriptor from one of our receive buffer pools
+ */
+static vlan_bd_t spapr_vlan_get_rx_bd_from_pool(VIOsPAPRVLANDevice *dev,
+                                                size_t size)
+{
+    vlan_bd_t bd;
+    int pool;
+
+    for (pool = 0; pool < RX_MAX_POOLS; pool++) {
+        if (dev->rx_pool[pool]->count > 0 &&
+            dev->rx_pool[pool]->bufsize >= size + 8) {
+            break;
+        }
+    }
+    if (pool == RX_MAX_POOLS) {
+        /* Failed to find a suitable buffer */
+        return 0;
+    }
+
+    DPRINTF("Found buffer: pool=%d count=%d rxbufs=%d\n", pool,
+            dev->rx_pool[pool]->count, dev->rx_bufs);
+
+    /* Remove the buffer from the pool */
+    dev->rx_pool[pool]->count--;
+    bd = dev->rx_pool[pool]->bds[dev->rx_pool[pool]->count];
+    dev->rx_pool[pool]->bds[dev->rx_pool[pool]->count] = 0;
+
+    return bd;
+}
+
+/**
+ * Get buffer descriptor from the receive buffer list page that has been
+ * supplied by the guest with the H_REGISTER_LOGICAL_LAN call
+ */
+static vlan_bd_t spapr_vlan_get_rx_bd_from_page(VIOsPAPRVLANDevice *dev,
+                                                size_t size)
+{
+    int buf_ptr = dev->use_buf_ptr;
+    vlan_bd_t bd;
+
+    do {
+        buf_ptr += 8;
+        if (buf_ptr >= VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF) {
+            buf_ptr = VLAN_RX_BDS_OFF;
+        }
+
+        bd = vio_ldq(&dev->sdev, dev->buf_list + buf_ptr);
+        DPRINTF("use_buf_ptr=%d bd=0x%016llx\n",
+                buf_ptr, (unsigned long long)bd);
+    } while ((!(bd & VLAN_BD_VALID) || VLAN_BD_LEN(bd) < size + 8)
+             && buf_ptr != dev->use_buf_ptr);
+
+    if (!(bd & VLAN_BD_VALID) || VLAN_BD_LEN(bd) < size + 8) {
+        /* Failed to find a suitable buffer */
+        return 0;
+    }
+
+    /* Remove the buffer from the pool */
+    dev->use_buf_ptr = buf_ptr;
+    vio_stq(&dev->sdev, dev->buf_list + dev->use_buf_ptr, 0);
+
+    DPRINTF("Found buffer: ptr=%d rxbufs=%d\n", dev->use_buf_ptr, dev->rx_bufs);
+
+    return bd;
+}
+
 static ssize_t spapr_vlan_receive(NetClientState *nc, const uint8_t *buf,
                                   size_t size)
 {
@@ -109,7 +211,6 @@
     VIOsPAPRDevice *sdev = VIO_SPAPR_DEVICE(dev);
     vlan_bd_t rxq_bd = vio_ldq(sdev, dev->buf_list + VLAN_RXQ_BD_OFF);
     vlan_bd_t bd;
-    int buf_ptr = dev->use_buf_ptr;
     uint64_t handle;
     uint8_t control;
 
@@ -121,32 +222,21 @@
     }
 
     if (!dev->rx_bufs) {
-        return -1;
+        spapr_vlan_record_dropped_rx_frame(dev);
+        return 0;
     }
 
-    do {
-        buf_ptr += 8;
-        if (buf_ptr >= (VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF)) {
-            buf_ptr = VLAN_RX_BDS_OFF;
-        }
-
-        bd = vio_ldq(sdev, dev->buf_list + buf_ptr);
-        DPRINTF("use_buf_ptr=%d bd=0x%016llx\n",
-                buf_ptr, (unsigned long long)bd);
-    } while ((!(bd & VLAN_BD_VALID) || (VLAN_BD_LEN(bd) < (size + 8)))
-             && (buf_ptr != dev->use_buf_ptr));
-
-    if (!(bd & VLAN_BD_VALID) || (VLAN_BD_LEN(bd) < (size + 8))) {
-        /* Failed to find a suitable buffer */
-        return -1;
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        bd = spapr_vlan_get_rx_bd_from_pool(dev, size);
+    } else {
+        bd = spapr_vlan_get_rx_bd_from_page(dev, size);
+    }
+    if (!bd) {
+        spapr_vlan_record_dropped_rx_frame(dev);
+        return 0;
     }
 
-    /* Remove the buffer from the pool */
     dev->rx_bufs--;
-    dev->use_buf_ptr = buf_ptr;
-    vio_stq(sdev, dev->buf_list + dev->use_buf_ptr, 0);
-
-    DPRINTF("Found buffer: ptr=%d num=%d\n", dev->use_buf_ptr, dev->rx_bufs);
 
     /* Transfer the packet data */
     if (spapr_vio_dma_write(sdev, VLAN_BD_ADDR(bd) + 8, buf, size) < 0) {
@@ -187,31 +277,48 @@
     return size;
 }
 
-static void spapr_vlan_cleanup(NetClientState *nc)
-{
-    VIOsPAPRVLANDevice *dev = qemu_get_nic_opaque(nc);
-
-    dev->nic = NULL;
-}
-
 static NetClientInfo net_spapr_vlan_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = spapr_vlan_can_receive,
     .receive = spapr_vlan_receive,
-    .cleanup = spapr_vlan_cleanup,
 };
 
+static void spapr_vlan_flush_rx_queue(void *opaque)
+{
+    VIOsPAPRVLANDevice *dev = opaque;
+
+    qemu_flush_queued_packets(qemu_get_queue(dev->nic));
+}
+
+static void spapr_vlan_reset_rx_pool(RxBufPool *rxp)
+{
+    /*
+     * Use INT_MAX as bufsize so that unused buffers are moved to the end
+     * of the list during the qsort in spapr_vlan_add_rxbuf_to_pool() later.
+     */
+    rxp->bufsize = INT_MAX;
+    rxp->count = 0;
+    memset(rxp->bds, 0, sizeof(rxp->bds));
+}
+
 static void spapr_vlan_reset(VIOsPAPRDevice *sdev)
 {
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(sdev);
+    int i;
 
     dev->buf_list = 0;
     dev->rx_bufs = 0;
     dev->isopen = 0;
+
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        for (i = 0; i < RX_MAX_POOLS; i++) {
+            spapr_vlan_reset_rx_pool(dev->rx_pool[i]);
+        }
+    }
 }
 
-static int spapr_vlan_init(VIOsPAPRDevice *sdev)
+static void spapr_vlan_realize(VIOsPAPRDevice *sdev, Error **errp)
 {
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(sdev);
 
@@ -221,16 +328,43 @@
                             object_get_typename(OBJECT(sdev)), sdev->qdev.id, dev);
     qemu_format_nic_info_str(qemu_get_queue(dev->nic), dev->nicconf.macaddr.a);
 
-    return 0;
+    dev->rxp_timer = timer_new_us(QEMU_CLOCK_VIRTUAL, spapr_vlan_flush_rx_queue,
+                                  dev);
 }
 
 static void spapr_vlan_instance_init(Object *obj)
 {
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(obj);
+    int i;
 
     device_add_bootindex_property(obj, &dev->nicconf.bootindex,
                                   "bootindex", "",
                                   DEVICE(dev), NULL);
+
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        for (i = 0; i < RX_MAX_POOLS; i++) {
+            dev->rx_pool[i] = g_new(RxBufPool, 1);
+            spapr_vlan_reset_rx_pool(dev->rx_pool[i]);
+        }
+    }
+}
+
+static void spapr_vlan_instance_finalize(Object *obj)
+{
+    VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(obj);
+    int i;
+
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        for (i = 0; i < RX_MAX_POOLS; i++) {
+            g_free(dev->rx_pool[i]);
+            dev->rx_pool[i] = NULL;
+        }
+    }
+
+    if (dev->rxp_timer) {
+        timer_del(dev->rxp_timer);
+        timer_free(dev->rxp_timer);
+    }
 }
 
 void spapr_vlan_create(VIOsPAPRBus *bus, NICInfo *nd)
@@ -294,7 +428,7 @@
 }
 
 static target_ulong h_register_logical_lan(PowerPCCPU *cpu,
-                                           sPAPREnvironment *spapr,
+                                           sPAPRMachineState *spapr,
                                            target_ulong opcode,
                                            target_ulong *args)
 {
@@ -359,7 +493,8 @@
 }
 
 
-static target_ulong h_free_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_free_logical_lan(PowerPCCPU *cpu,
+                                       sPAPRMachineState *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -380,8 +515,115 @@
     return H_SUCCESS;
 }
 
+/**
+ * Used for qsort, this function compares two RxBufPools by size.
+ */
+static int rx_pool_size_compare(const void *p1, const void *p2)
+{
+    const RxBufPool *pool1 = *(RxBufPool **)p1;
+    const RxBufPool *pool2 = *(RxBufPool **)p2;
+
+    if (pool1->bufsize < pool2->bufsize) {
+        return -1;
+    }
+    return pool1->bufsize > pool2->bufsize;
+}
+
+/**
+ * Search for a matching buffer pool with exact matching size,
+ * or return -1 if no matching pool has been found.
+ */
+static int spapr_vlan_get_rx_pool_id(VIOsPAPRVLANDevice *dev, int size)
+{
+    int pool;
+
+    for (pool = 0; pool < RX_MAX_POOLS; pool++) {
+        if (dev->rx_pool[pool]->bufsize == size) {
+            return pool;
+        }
+    }
+
+    return -1;
+}
+
+/**
+ * Enqueuing receive buffer by adding it to one of our receive buffer pools
+ */
+static target_long spapr_vlan_add_rxbuf_to_pool(VIOsPAPRVLANDevice *dev,
+                                                target_ulong buf)
+{
+    int size = VLAN_BD_LEN(buf);
+    int pool;
+
+    pool = spapr_vlan_get_rx_pool_id(dev, size);
+    if (pool < 0) {
+        /*
+         * No matching pool found? Try to use a new one. If the guest used all
+         * pools before, but changed the size of one pool inbetween, we might
+         * need to recycle that pool here (if it's empty already). Thus scan
+         * all buffer pools now, starting with the last (likely empty) one.
+         */
+        for (pool = RX_MAX_POOLS - 1; pool >= 0 ; pool--) {
+            if (dev->rx_pool[pool]->count == 0) {
+                dev->rx_pool[pool]->bufsize = size;
+                /*
+                 * Sort pools by size so that spapr_vlan_receive()
+                 * can later find the smallest buffer pool easily.
+                 */
+                qsort(dev->rx_pool, RX_MAX_POOLS, sizeof(dev->rx_pool[0]),
+                      rx_pool_size_compare);
+                pool = spapr_vlan_get_rx_pool_id(dev, size);
+                DPRINTF("created RX pool %d for size %lld\n", pool,
+                        VLAN_BD_LEN(buf));
+                break;
+            }
+        }
+    }
+    /* Still no usable pool? Give up */
+    if (pool < 0 || dev->rx_pool[pool]->count >= RX_POOL_MAX_BDS) {
+        return H_RESOURCE;
+    }
+
+    DPRINTF("h_add_llan_buf():  Add buf using pool %i (size %lli, count=%i)\n",
+            pool, VLAN_BD_LEN(buf), dev->rx_pool[pool]->count);
+
+    dev->rx_pool[pool]->bds[dev->rx_pool[pool]->count++] = buf;
+
+    return 0;
+}
+
+/**
+ * This is the old way of enqueuing receive buffers: Add it to the rx queue
+ * page that has been supplied by the guest (which is quite limited in size).
+ */
+static target_long spapr_vlan_add_rxbuf_to_page(VIOsPAPRVLANDevice *dev,
+                                                target_ulong buf)
+{
+    vlan_bd_t bd;
+
+    if (dev->rx_bufs >= VLAN_MAX_BUFS) {
+        return H_RESOURCE;
+    }
+
+    do {
+        dev->add_buf_ptr += 8;
+        if (dev->add_buf_ptr >= VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF) {
+            dev->add_buf_ptr = VLAN_RX_BDS_OFF;
+        }
+
+        bd = vio_ldq(&dev->sdev, dev->buf_list + dev->add_buf_ptr);
+    } while (bd & VLAN_BD_VALID);
+
+    vio_stq(&dev->sdev, dev->buf_list + dev->add_buf_ptr, buf);
+
+    DPRINTF("h_add_llan_buf():  Added buf  ptr=%d  rx_bufs=%d bd=0x%016llx\n",
+            dev->add_buf_ptr, dev->rx_bufs, (unsigned long long)buf);
+
+    return 0;
+}
+
 static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
-                                             sPAPREnvironment *spapr,
+                                             sPAPRMachineState *spapr,
                                              target_ulong opcode,
                                              target_ulong *args)
 {
@@ -389,7 +631,7 @@
     target_ulong buf = args[1];
     VIOsPAPRDevice *sdev = spapr_vio_find_by_reg(spapr->vio_bus, reg);
     VIOsPAPRVLANDevice *dev = VIO_SPAPR_VLAN_DEVICE(sdev);
-    vlan_bd_t bd;
+    target_long ret;
 
     DPRINTF("H_ADD_LOGICAL_LAN_BUFFER(0x" TARGET_FMT_lx
             ", 0x" TARGET_FMT_lx ")\n", reg, buf);
@@ -405,33 +647,34 @@
         return H_PARAMETER;
     }
 
-    if (!dev->isopen || dev->rx_bufs >= VLAN_MAX_BUFS) {
+    if (!dev->isopen) {
         return H_RESOURCE;
     }
 
-    do {
-        dev->add_buf_ptr += 8;
-        if (dev->add_buf_ptr >= (VLAN_RX_BDS_LEN + VLAN_RX_BDS_OFF)) {
-            dev->add_buf_ptr = VLAN_RX_BDS_OFF;
-        }
-
-        bd = vio_ldq(sdev, dev->buf_list + dev->add_buf_ptr);
-    } while (bd & VLAN_BD_VALID);
-
-    vio_stq(sdev, dev->buf_list + dev->add_buf_ptr, buf);
+    if (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) {
+        ret = spapr_vlan_add_rxbuf_to_pool(dev, buf);
+    } else {
+        ret = spapr_vlan_add_rxbuf_to_page(dev, buf);
+    }
+    if (ret) {
+        return ret;
+    }
 
     dev->rx_bufs++;
 
-    qemu_flush_queued_packets(qemu_get_queue(dev->nic));
-
-    DPRINTF("h_add_logical_lan_buffer():  Added buf  ptr=%d  rx_bufs=%d"
-            " bd=0x%016llx\n", dev->add_buf_ptr, dev->rx_bufs,
-            (unsigned long long)buf);
+    /*
+     * Give guest some more time to add additional RX buffers before we
+     * flush the receive queue, so that e.g. fragmented IP packets can
+     * be passed to the guest in one go later (instead of passing single
+     * fragments if there is only one receive buffer available).
+     */
+    timer_mod(dev->rxp_timer, qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + 500);
 
     return H_SUCCESS;
 }
 
-static target_ulong h_send_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_send_logical_lan(PowerPCCPU *cpu,
+                                       sPAPRMachineState *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -500,7 +743,7 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_multicast_ctrl(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_multicast_ctrl(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                      target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -516,9 +759,44 @@
 static Property spapr_vlan_properties[] = {
     DEFINE_SPAPR_PROPERTIES(VIOsPAPRVLANDevice, sdev),
     DEFINE_NIC_PROPERTIES(VIOsPAPRVLANDevice, nicconf),
+    DEFINE_PROP_BIT("use-rx-buffer-pools", VIOsPAPRVLANDevice,
+                    compat_flags, SPAPRVLAN_FLAG_RX_BUF_POOLS_BIT, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static bool spapr_vlan_rx_buffer_pools_needed(void *opaque)
+{
+    VIOsPAPRVLANDevice *dev = opaque;
+
+    return (dev->compat_flags & SPAPRVLAN_FLAG_RX_BUF_POOLS) != 0;
+}
+
+static const VMStateDescription vmstate_rx_buffer_pool = {
+    .name = "spapr_llan/rx_buffer_pool",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = spapr_vlan_rx_buffer_pools_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(bufsize, RxBufPool),
+        VMSTATE_INT32(count, RxBufPool),
+        VMSTATE_UINT64_ARRAY(bds, RxBufPool, RX_POOL_MAX_BDS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_rx_pools = {
+    .name = "spapr_llan/rx_pools",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = spapr_vlan_rx_buffer_pools_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_ARRAY_OF_POINTER_TO_STRUCT(rx_pool, VIOsPAPRVLANDevice,
+                                           RX_MAX_POOLS, 1,
+                                           vmstate_rx_buffer_pool, RxBufPool),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_spapr_llan = {
     .name = "spapr_llan",
     .version_id = 1,
@@ -527,14 +805,18 @@
         VMSTATE_SPAPR_VIO(sdev, VIOsPAPRVLANDevice),
         /* LLAN state */
         VMSTATE_BOOL(isopen, VIOsPAPRVLANDevice),
-        VMSTATE_UINTTL(buf_list, VIOsPAPRVLANDevice),
+        VMSTATE_UINT64(buf_list, VIOsPAPRVLANDevice),
         VMSTATE_UINT32(add_buf_ptr, VIOsPAPRVLANDevice),
         VMSTATE_UINT32(use_buf_ptr, VIOsPAPRVLANDevice),
         VMSTATE_UINT32(rx_bufs, VIOsPAPRVLANDevice),
-        VMSTATE_UINTTL(rxq_ptr, VIOsPAPRVLANDevice),
+        VMSTATE_UINT64(rxq_ptr, VIOsPAPRVLANDevice),
 
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_rx_pools,
+        NULL
+    }
 };
 
 static void spapr_vlan_class_init(ObjectClass *klass, void *data)
@@ -542,7 +824,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VIOsPAPRDeviceClass *k = VIO_SPAPR_DEVICE_CLASS(klass);
 
-    k->init = spapr_vlan_init;
+    k->realize = spapr_vlan_realize;
     k->reset = spapr_vlan_reset;
     k->devnode = spapr_vlan_devnode;
     k->dt_name = "l-lan";
@@ -561,6 +843,7 @@
     .instance_size = sizeof(VIOsPAPRVLANDevice),
     .class_init    = spapr_vlan_class_init,
     .instance_init = spapr_vlan_instance_init,
+    .instance_finalize = spapr_vlan_instance_finalize,
 };
 
 static void spapr_vlan_register_types(void)
diff --git a/hw/net/stellaris_enet.c b/hw/net/stellaris_enet.c
index c07e513..957730e 100644
--- a/hw/net/stellaris_enet.c
+++ b/hw/net/stellaris_enet.c
@@ -6,6 +6,7 @@
  *
  * This code is licensed under the GPL.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "net/net.h"
 #include <zlib.h>
@@ -228,16 +229,25 @@
     if ((s->rctl & SE_RCTL_RXEN) == 0)
         return -1;
     if (s->np >= 31) {
-        DPRINTF("Packet dropped\n");
-        return -1;
+        return 0;
     }
 
     DPRINTF("Received packet len=%zu\n", size);
     n = s->next_packet + s->np;
     if (n >= 31)
         n -= 31;
-    s->np++;
 
+    if (size >= sizeof(s->rx[n].data) - 6) {
+        /* If the packet won't fit into the
+         * emulated 2K RAM, this is reported
+         * as a FIFO overrun error.
+         */
+        s->ris |= SE_INT_FOV;
+        stellaris_enet_update(s);
+        return -1;
+    }
+
+    s->np++;
     s->rx[n].len = size + 6;
     p = s->rx[n].data;
     *(p++) = (size + 6);
@@ -260,13 +270,8 @@
     return size;
 }
 
-static int stellaris_enet_can_receive(NetClientState *nc)
+static int stellaris_enet_can_receive(stellaris_enet_state *s)
 {
-    stellaris_enet_state *s = qemu_get_nic_opaque(nc);
-
-    if ((s->rctl & SE_RCTL_RXEN) == 0)
-        return 1;
-
     return (s->np < 31);
 }
 
@@ -307,6 +312,9 @@
                 s->next_packet = 0;
             s->np--;
             DPRINTF("RX done np=%d\n", s->np);
+            if (!s->np && stellaris_enet_can_receive(s)) {
+                qemu_flush_queued_packets(qemu_get_queue(s->nic));
+            }
         }
         return val;
     }
@@ -451,19 +459,10 @@
     s->tx_fifo_len = 0;
 }
 
-static void stellaris_enet_cleanup(NetClientState *nc)
-{
-    stellaris_enet_state *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_stellaris_enet_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = stellaris_enet_can_receive,
     .receive = stellaris_enet_receive,
-    .cleanup = stellaris_enet_cleanup,
 };
 
 static int stellaris_enet_init(SysBusDevice *sbd)
diff --git a/hw/net/trace-events b/hw/net/trace-events
new file mode 100644
index 0000000..8d38d77
--- /dev/null
+++ b/hw/net/trace-events
@@ -0,0 +1,272 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/net/lance.c
+lance_mem_readw(uint64_t addr, uint32_t ret) "addr=%"PRIx64"val=0x%04x"
+lance_mem_writew(uint64_t addr, uint32_t val) "addr=%"PRIx64"val=0x%04x"
+
+# hw/net/milkymist-minimac2.c
+milkymist_minimac2_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_minimac2_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_minimac2_mdio_write(uint8_t phy_addr, uint8_t addr, uint16_t value) "phy_addr %02x addr %02x value %04x"
+milkymist_minimac2_mdio_read(uint8_t phy_addr, uint8_t addr, uint16_t value) "phy_addr %02x addr %02x value %04x"
+milkymist_minimac2_tx_frame(uint32_t length) "length %u"
+milkymist_minimac2_rx_frame(const void *buf, uint32_t length) "buf %p length %u"
+milkymist_minimac2_rx_transfer(const void *buf, uint32_t length) "buf %p length %d"
+milkymist_minimac2_raise_irq_rx(void) "Raise IRQ RX"
+milkymist_minimac2_lower_irq_rx(void) "Lower IRQ RX"
+milkymist_minimac2_pulse_irq_tx(void) "Pulse IRQ TX"
+
+# hw/net/mipsnet.c
+mipsnet_send(uint32_t size) "sending len=%u"
+mipsnet_receive(uint32_t size) "receiving len=%u"
+mipsnet_read(uint64_t addr, uint32_t val) "read addr=0x%" PRIx64 " val=0x%x"
+mipsnet_write(uint64_t addr, uint64_t val) "write addr=0x%" PRIx64 " val=0x%" PRIx64
+mipsnet_irq(uint32_t isr, uint32_t intctl) "set irq to %d (%02x)"
+
+# hw/net/opencores_eth.c
+open_eth_mii_write(unsigned idx, uint16_t v) "MII[%02x] <- %04x"
+open_eth_mii_read(unsigned idx, uint16_t v) "MII[%02x] -> %04x"
+open_eth_update_irq(uint32_t v) "IRQ <- %x"
+open_eth_receive(unsigned len) "RX: len: %u"
+open_eth_receive_mcast(unsigned idx, uint32_t h0, uint32_t h1) "MCAST: idx = %u, hash: %08x:%08x"
+open_eth_receive_reject(void) "RX: rejected"
+open_eth_receive_desc(uint32_t addr, uint32_t len_flags) "RX: %08x, len_flags: %08x"
+open_eth_start_xmit(uint32_t addr, unsigned len, unsigned tx_len) "TX: %08x, len: %u, tx_len: %u"
+open_eth_reg_read(uint32_t addr, uint32_t v) "MAC[%02x] -> %08x"
+open_eth_reg_write(uint32_t addr, uint32_t v) "MAC[%02x] <- %08x"
+open_eth_desc_read(uint32_t addr, uint32_t v) "DESC[%04x] -> %08x"
+open_eth_desc_write(uint32_t addr, uint32_t v) "DESC[%04x] <- %08x"
+
+# hw/net/pcnet.c
+pcnet_s_reset(void *s) "s=%p"
+pcnet_user_int(void *s) "s=%p"
+pcnet_isr_change(void *s, uint32_t isr, uint32_t isr_old) "s=%p INTA=%d<=%d"
+pcnet_init(void *s, uint64_t init_addr) "s=%p init_addr=%#"PRIx64
+pcnet_rlen_tlen(void *s, uint32_t rlen, uint32_t tlen) "s=%p rlen=%d tlen=%d"
+pcnet_ss32_rdra_tdra(void *s, uint32_t ss32, uint32_t rdra, uint32_t rcvrl, uint32_t tdra, uint32_t xmtrl) "s=%p ss32=%d rdra=0x%08x[%d] tdra=0x%08x[%d]"
+
+# hw/net/pcnet-pci.c
+pcnet_aprom_writeb(void *opaque, uint32_t addr, uint32_t val) "opaque=%p addr=0x%08x val=0x%02x"
+pcnet_aprom_readb(void *opaque, uint32_t addr, uint32_t val) "opaque=%p addr=0x%08x val=0x%02x"
+pcnet_ioport_read(void *opaque, uint64_t addr, unsigned size) "opaque=%p addr=%#"PRIx64" size=%d"
+pcnet_ioport_write(void *opaque, uint64_t addr, uint64_t data, unsigned size) "opaque=%p addr=%#"PRIx64" data=%#"PRIx64" size=%d"
+pcnet_mmio_writeb(void *opaque, uint64_t addr, uint32_t val) "opaque=%p addr=%#"PRIx64" val=0x%x"
+pcnet_mmio_writew(void *opaque, uint64_t addr, uint32_t val) "opaque=%p addr=%#"PRIx64" val=0x%x"
+pcnet_mmio_writel(void *opaque, uint64_t addr, uint32_t val) "opaque=%p addr=%#"PRIx64" val=0x%x"
+pcnet_mmio_readb(void *opaque, uint64_t addr, uint32_t val) "opaque=%p addr=%#"PRIx64" val=0x%x"
+pcnet_mmio_readw(void *opaque, uint64_t addr, uint32_t val) "opaque=%p addr=%#"PRIx64" val=0x%x"
+pcnet_mmio_readl(void *opaque, uint64_t addr, uint32_t val) "opaque=%p addr=%#"PRIx64" val=0x%x"
+
+# hw/net/net_rx_pkt.c
+net_rx_pkt_parsed(bool ip4, bool ip6, bool udp, bool tcp, size_t l3o, size_t l4o, size_t l5o) "RX packet parsed: ip4: %d, ip6: %d, udp: %d, tcp: %d, l3 offset: %zu, l4 offset: %zu, l5 offset: %zu"
+net_rx_pkt_l4_csum_validate_entry(void) "Starting L4 checksum validation"
+net_rx_pkt_l4_csum_validate_not_xxp(void) "Not a TCP/UDP packet"
+net_rx_pkt_l4_csum_validate_udp_with_no_checksum(void) "UDP packet without checksum"
+net_rx_pkt_l4_csum_validate_ip4_fragment(void) "IP4 fragment"
+net_rx_pkt_l4_csum_validate_ip4_udp(void) "IP4/UDP packet"
+net_rx_pkt_l4_csum_validate_ip4_tcp(void) "IP4/TCP packet"
+net_rx_pkt_l4_csum_validate_ip6_udp(void) "IP6/UDP packet"
+net_rx_pkt_l4_csum_validate_ip6_tcp(void) "IP6/TCP packet"
+net_rx_pkt_l4_csum_validate_csum(bool csum_valid) "Checksum valid: %d"
+
+net_rx_pkt_l4_csum_calc_entry(void) "Starting L4 checksum calculation"
+net_rx_pkt_l4_csum_calc_ip4_udp(void) "IP4/UDP packet"
+net_rx_pkt_l4_csum_calc_ip4_tcp(void) "IP4/TCP packet"
+net_rx_pkt_l4_csum_calc_ip6_udp(void) "IP6/UDP packet"
+net_rx_pkt_l4_csum_calc_ip6_tcp(void) "IP6/TCP packet"
+net_rx_pkt_l4_csum_calc_ph_csum(uint32_t cntr, uint16_t csl) "Pseudo-header: checksum counter %u, length %u"
+net_rx_pkt_l4_csum_calc_csum(size_t l4hdr_off, uint16_t csl, uint32_t cntr, uint16_t csum) "L4 Checksum: L4 header offset: %zu, length: %u, counter: 0x%X, final checksum: 0x%X"
+
+net_rx_pkt_l4_csum_fix_entry(void) "Starting L4 checksum correction"
+net_rx_pkt_l4_csum_fix_tcp(uint32_t l4_cso) "TCP packet, L4 cso: %u"
+net_rx_pkt_l4_csum_fix_udp(uint32_t l4_cso) "UDP packet, L4 cso: %u"
+net_rx_pkt_l4_csum_fix_not_xxp(void) "Not an IP4 packet"
+net_rx_pkt_l4_csum_fix_ip4_fragment(void) "IP4 fragment"
+net_rx_pkt_l4_csum_fix_udp_with_no_checksum(void) "UDP packet without checksum"
+net_rx_pkt_l4_csum_fix_csum(uint32_t cso, uint16_t csum) "L4 Checksum: Offset: %u, value 0x%X"
+
+net_rx_pkt_l3_csum_validate_entry(void) "Starting L3 checksum validation"
+net_rx_pkt_l3_csum_validate_not_ip4(void) "Not an IP4 packet"
+net_rx_pkt_l3_csum_validate_csum(size_t l3hdr_off, uint32_t csl, uint32_t cntr, uint16_t csum, bool csum_valid) "L3 Checksum: L3 header offset: %zu, length: %u, counter: 0x%X, final checksum: 0x%X, valid: %d"
+
+net_rx_pkt_rss_ip4(void) "Calculating IPv4 RSS  hash"
+net_rx_pkt_rss_ip4_tcp(void) "Calculating IPv4/TCP RSS  hash"
+net_rx_pkt_rss_ip6_tcp(void) "Calculating IPv6/TCP RSS  hash"
+net_rx_pkt_rss_ip6(void) "Calculating IPv6 RSS  hash"
+net_rx_pkt_rss_ip6_ex(void) "Calculating IPv6/EX RSS  hash"
+net_rx_pkt_rss_hash(size_t rss_length, uint32_t rss_hash) "RSS hash for %zu bytes: 0x%X"
+net_rx_pkt_rss_add_chunk(void* ptr, size_t size, size_t input_offset) "Add RSS chunk %p, %zu bytes, RSS input offset %zu bytes"
+
+# hw/net/e1000x_common.c
+e1000x_rx_can_recv_disabled(bool link_up, bool rx_enabled, bool pci_master) "link_up: %d, rx_enabled %d, pci_master %d"
+e1000x_vlan_is_vlan_pkt(bool is_vlan_pkt, uint16_t eth_proto, uint16_t vet) "Is VLAN packet: %d, ETH proto: 0x%X, VET: 0x%X"
+e1000x_rx_flt_ucast_match(uint32_t idx, uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "unicast match[%d]: %02x:%02x:%02x:%02x:%02x:%02x"
+e1000x_rx_flt_ucast_mismatch(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "unicast mismatch: %02x:%02x:%02x:%02x:%02x:%02x"
+e1000x_rx_flt_inexact_mismatch(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5, uint32_t mo, uint32_t mta, uint32_t mta_val) "inexact mismatch: %02x:%02x:%02x:%02x:%02x:%02x MO %d MTA[%d] %x"
+e1000x_rx_link_down(uint32_t status_reg) "Received packet dropped because the link is down STATUS = %u"
+e1000x_rx_disabled(uint32_t rctl_reg) "Received packet dropped because receive is disabled RCTL = %u"
+e1000x_rx_oversized(size_t size) "Received packet dropped because it was oversized (%zu bytes)"
+e1000x_mac_indicate(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "Indicating MAC to guest: %02x:%02x:%02x:%02x:%02x:%02x"
+e1000x_link_negotiation_start(void) "Start link auto negotiation"
+e1000x_link_negotiation_done(void) "Auto negotiation is completed"
+
+# hw/net/e1000e_core.c
+e1000e_core_write(uint64_t index, uint32_t size, uint64_t val) "Write to register 0x%"PRIx64", %d byte(s), value: 0x%"PRIx64
+e1000e_core_read(uint64_t index, uint32_t size, uint64_t val) "Read from register 0x%"PRIx64", %d byte(s), value: 0x%"PRIx64
+e1000e_core_mdic_read(uint8_t page, uint32_t addr, uint32_t data) "MDIC READ: PHY[%u][%u] = 0x%x"
+e1000e_core_mdic_read_unhandled(uint8_t page, uint32_t addr) "MDIC READ: PHY[%u][%u] UNHANDLED"
+e1000e_core_mdic_write(uint8_t page, uint32_t addr, uint32_t data) "MDIC WRITE: PHY[%u][%u] = 0x%x"
+e1000e_core_mdic_write_unhandled(uint8_t page, uint32_t addr) "MDIC WRITE: PHY[%u][%u] UNHANDLED"
+e1000e_core_eeeprom_write(uint16_t bit_in, uint16_t bit_out, uint16_t reading) "eeprom bitnum in %d out %d, reading %d"
+e1000e_core_ctrl_write(uint64_t index, uint32_t val) "Write CTRL register 0x%"PRIx64", value: 0x%X"
+e1000e_core_ctrl_sw_reset(void) "Doing SW reset"
+e1000e_core_ctrl_phy_reset(void) "Doing PHY reset"
+
+e1000e_link_autoneg_flowctl(bool enabled) "Auto-negotiated flow control state is %d"
+e1000e_link_set_params(bool autodetect, uint32_t speed, bool force_spd, bool force_dplx, bool rx_fctl, bool tx_fctl) "Set link params: Autodetect: %d, Speed: %d, Force speed: %d, Force duplex: %d, RX flow control %d, TX flow control %d"
+e1000e_link_read_params(bool autodetect, uint32_t speed, bool force_spd, bool force_dplx, bool rx_fctl, bool tx_fctl) "Get link params: Autodetect: %d, Speed: %d, Force speed: %d, Force duplex: %d, RX flow control %d, TX flow control %d"
+e1000e_link_set_ext_params(bool asd_check, bool speed_select_bypass) "Set extended link params: ASD check: %d, Speed select bypass: %d"
+e1000e_link_status(bool link_up, bool full_dplx, uint32_t speed, uint32_t asdv) "Link up: %d, Duplex: %d, Speed: %d, ASDV: %d"
+e1000e_link_status_changed(bool status) "New link status: %d"
+
+e1000e_wrn_regs_write_ro(uint64_t index, uint32_t size, uint64_t val) "WARNING: Write to RO register 0x%"PRIx64", %d byte(s), value: 0x%"PRIx64
+e1000e_wrn_regs_write_unknown(uint64_t index, uint32_t size, uint64_t val) "WARNING: Write to unknown register 0x%"PRIx64", %d byte(s), value: 0x%"PRIx64
+e1000e_wrn_regs_read_unknown(uint64_t index, uint32_t size) "WARNING: Read from unknown register 0x%"PRIx64", %d byte(s)"
+e1000e_wrn_regs_read_trivial(uint32_t index) "WARNING: Reading register at offset: 0x%05x. It is not fully implemented."
+e1000e_wrn_regs_write_trivial(uint32_t index) "WARNING: Writing to register at offset: 0x%05x. It is not fully implemented."
+e1000e_wrn_no_ts_support(void) "WARNING: Guest requested TX timestamping which is not supported"
+e1000e_wrn_no_snap_support(void) "WARNING: Guest requested TX SNAP header update which is not supported"
+e1000e_wrn_iscsi_filtering_not_supported(void) "WARNING: Guest requested iSCSI filtering  which is not supported"
+e1000e_wrn_nfsw_filtering_not_supported(void) "WARNING: Guest requested NFS write filtering  which is not supported"
+e1000e_wrn_nfsr_filtering_not_supported(void) "WARNING: Guest requested NFS read filtering  which is not supported"
+
+e1000e_tx_disabled(void) "TX Disabled"
+e1000e_tx_descr(void *addr, uint32_t lower, uint32_t upper) "%p : %x %x"
+
+e1000e_ring_free_space(int ridx, uint32_t rdlen, uint32_t rdh, uint32_t rdt) "ring #%d: LEN: %u, DH: %u, DT: %u"
+
+e1000e_rx_can_recv_rings_full(void) "Cannot receive: all rings are full"
+e1000e_rx_can_recv(void) "Can receive"
+e1000e_rx_has_buffers(int ridx, uint32_t free_desc, size_t total_size, uint32_t desc_buf_size) "ring #%d: free descr: %u, packet size %zu, descr buffer size %u"
+e1000e_rx_null_descriptor(void) "Null RX descriptor!!"
+e1000e_rx_flt_vlan_mismatch(uint16_t vid) "VID mismatch: 0x%X"
+e1000e_rx_flt_vlan_match(uint16_t vid) "VID match: 0x%X"
+e1000e_rx_desc_ps_read(uint64_t a0, uint64_t a1, uint64_t a2, uint64_t a3) "buffers: [0x%"PRIx64", 0x%"PRIx64", 0x%"PRIx64", 0x%"PRIx64"]"
+e1000e_rx_desc_ps_write(uint16_t a0, uint16_t a1, uint16_t a2, uint16_t a3) "bytes written: [%u, %u, %u, %u]"
+e1000e_rx_desc_buff_sizes(uint32_t b0, uint32_t b1, uint32_t b2, uint32_t b3) "buffer sizes: [%u, %u, %u, %u]"
+e1000e_rx_desc_len(uint8_t rx_desc_len) "RX descriptor length: %u"
+e1000e_rx_desc_buff_write(uint8_t idx, uint64_t addr, uint16_t offset, const void* source, uint32_t len) "buffer #%u, addr: 0x%"PRIx64", offset: %u, from: %p, length: %u"
+e1000e_rx_descr(int ridx, uint64_t base, uint8_t len) "Next RX descriptor: ring #%d, PA: 0x%"PRIx64", length: %u"
+e1000e_rx_set_rctl(uint32_t rctl) "RCTL = 0x%x"
+e1000e_rx_receive_iov(int iovcnt) "Received vector of %d fragments"
+e1000e_rx_packet_size(size_t full, size_t vhdr, size_t data) "Received packet of %zu bytes total, %zu virt header, %zu data"
+e1000e_rx_flt_dropped(void) "Received packet dropped by RX filter"
+e1000e_rx_written_to_guest(uint32_t causes) "Received packet written to guest (ICR causes %u)"
+e1000e_rx_not_written_to_guest(uint32_t causes) "Received packet NOT written to guest (ICR causes %u)"
+e1000e_rx_interrupt_set(uint32_t causes) "Receive interrupt set (ICR causes %u)"
+e1000e_rx_interrupt_delayed(uint32_t causes) "Receive interrupt delayed (ICR causes %u)"
+e1000e_rx_set_cso(int cso_state) "RX CSO state set to %d"
+e1000e_rx_set_rdt(int queue_idx, uint32_t val) "Setting RDT[%d] = %u"
+e1000e_rx_set_rfctl(uint32_t val) "Setting RFCTL = 0x%X"
+e1000e_rx_start_recv(void)
+
+e1000e_rx_rss_started(void) "Starting RSS processing"
+e1000e_rx_rss_disabled(void) "RSS is disabled"
+e1000e_rx_rss_type(uint32_t type) "RSS type is %u"
+e1000e_rx_rss_ip4(bool isfragment, bool istcp, uint32_t mrqc, bool tcpipv4_enabled, bool ipv4_enabled) "RSS IPv4: fragment %d, tcp %d, mrqc 0x%X, tcpipv4 enabled %d, ipv4 enabled %d"
+e1000e_rx_rss_ip6_rfctl(uint32_t rfctl) "RSS IPv6: rfctl 0x%X"
+e1000e_rx_rss_ip6(bool ex_dis, bool new_ex_dis, bool istcp, bool has_ext_headers, bool ex_dst_valid, bool ex_src_valid, uint32_t mrqc, bool tcpipv6_enabled, bool ipv6ex_enabled, bool ipv6_enabled) "RSS IPv6: ex_dis: %d, new_ex_dis: %d, tcp %d, has_ext_headers %d, ex_dst_valid %d, ex_src_valid %d, mrqc 0x%X, tcpipv6 enabled %d, ipv6ex enabled %d, ipv6 enabled %d"
+e1000e_rx_rss_dispatched_to_queue(int queue_idx) "Packet being dispatched to queue %d"
+
+e1000e_rx_metadata_protocols(bool isip4, bool isip6, bool isudp, bool istcp) "protocols: ip4: %d, ip6: %d, udp: %d, tcp: %d"
+e1000e_rx_metadata_vlan(uint16_t vlan_tag) "VLAN tag is 0x%X"
+e1000e_rx_metadata_rss(uint32_t rss, uint32_t mrq) "RSS data: rss: 0x%X, mrq: 0x%X"
+e1000e_rx_metadata_ip_id(uint16_t ip_id) "the IPv4 ID is 0x%X"
+e1000e_rx_metadata_ack(void) "the packet is TCP ACK"
+e1000e_rx_metadata_pkt_type(uint32_t pkt_type) "the packet type is %u"
+e1000e_rx_metadata_no_virthdr(void) "the packet has no virt-header"
+e1000e_rx_metadata_virthdr_no_csum_info(void) "virt-header does not contain checksum info"
+e1000e_rx_metadata_l3_cso_disabled(void) "IP4 CSO is disabled"
+e1000e_rx_metadata_l4_cso_disabled(void) "TCP/UDP CSO is disabled"
+e1000e_rx_metadata_l3_csum_validation_failed(void) "Cannot validate L3 checksum"
+e1000e_rx_metadata_l4_csum_validation_failed(void) "Cannot validate L4 checksum"
+e1000e_rx_metadata_status_flags(uint32_t status_flags) "status_flags is 0x%X"
+e1000e_rx_metadata_ipv6_sum_disabled(void) "IPv6 RX checksummimg disabled by RFCTL"
+e1000e_rx_metadata_ipv6_filtering_disabled(void) "IPv6 RX filtering disabled by RFCTL"
+
+e1000e_vlan_vet(uint16_t vet) "Setting VLAN ethernet type 0x%X"
+
+e1000e_irq_set_cause(uint32_t cause) "IRQ cause set 0x%x"
+e1000e_irq_msi_notify(uint32_t cause) "MSI notify 0x%x"
+e1000e_irq_throttling_no_pending_interrupts(void) "No pending interrupts to notify"
+e1000e_irq_msi_notify_postponed(void) "Sending MSI postponed by ITR"
+e1000e_irq_legacy_notify_postponed(void) "Raising legacy IRQ postponed by ITR"
+e1000e_irq_throttling_no_pending_vec(int idx) "No pending interrupts for vector %d"
+e1000e_irq_msix_notify_postponed_vec(int idx) "Sending MSI-X postponed by EITR[%d]"
+e1000e_irq_msix_notify(uint32_t cause) "MSI-X notify 0x%x"
+e1000e_irq_legacy_notify(bool level) "IRQ line state: %d"
+e1000e_irq_msix_notify_vec(uint32_t vector) "MSI-X notify vector 0x%x"
+e1000e_irq_postponed_by_xitr(uint32_t reg) "Interrupt postponed by [E]ITR register 0x%x"
+e1000e_irq_clear_ims(uint32_t bits, uint32_t old_ims, uint32_t new_ims) "Clearing IMS bits 0x%x: 0x%x --> 0x%x"
+e1000e_irq_set_ims(uint32_t bits, uint32_t old_ims, uint32_t new_ims) "Setting IMS bits 0x%x: 0x%x --> 0x%x"
+e1000e_irq_fix_icr_asserted(uint32_t new_val) "ICR_ASSERTED bit fixed: 0x%x"
+e1000e_irq_add_msi_other(uint32_t new_val) "ICR_OTHER bit added: 0x%x"
+e1000e_irq_pending_interrupts(uint32_t pending, uint32_t icr, uint32_t ims) "ICR PENDING: 0x%x (ICR: 0x%x, IMS: 0x%x)"
+e1000e_irq_set_cause_entry(uint32_t val, uint32_t icr) "Going to set IRQ cause 0x%x, ICR: 0x%x"
+e1000e_irq_set_cause_exit(uint32_t val, uint32_t icr) "Set IRQ cause 0x%x, ICR: 0x%x"
+e1000e_irq_icr_write(uint32_t bits, uint32_t old_icr, uint32_t new_icr) "Clearing ICR bits 0x%x: 0x%x --> 0x%x"
+e1000e_irq_write_ics(uint32_t val) "Adding ICR bits 0x%x"
+e1000e_irq_icr_process_iame(void) "Clearing IMS bits due to IAME"
+e1000e_irq_read_ics(uint32_t ics) "Current ICS: 0x%x"
+e1000e_irq_read_ims(uint32_t ims) "Current IMS: 0x%x"
+e1000e_irq_icr_read_entry(uint32_t icr) "Starting ICR read. Current ICR: 0x%x"
+e1000e_irq_icr_read_exit(uint32_t icr) "Ending ICR read. Current ICR: 0x%x"
+e1000e_irq_icr_clear_zero_ims(void) "Clearing ICR on read due to zero IMS"
+e1000e_irq_icr_clear_iame(void) "Clearing ICR on read due to IAME"
+e1000e_irq_ims_clear_eiame(uint32_t iam, uint32_t cause) "Clearing IMS due to EIAME, IAM: 0x%X, cause: 0x%X"
+e1000e_irq_icr_clear_eiac(uint32_t icr, uint32_t eiac) "Clearing ICR bits due to EIAC, ICR: 0x%X, EIAC: 0x%X"
+e1000e_irq_ims_clear_set_imc(uint32_t val) "Clearing IMS bits due to IMC write 0x%x"
+e1000e_irq_fire_delayed_interrupts(void) "Firing delayed interrupts"
+e1000e_irq_rearm_timer(uint32_t reg, int64_t delay_ns) "Mitigation timer armed for register 0x%X, delay %"PRId64" ns"
+e1000e_irq_throttling_timer(uint32_t reg) "Mitigation timer shot for register 0x%X"
+e1000e_irq_rdtr_fpd_running(void) "FPD written while RDTR was running"
+e1000e_irq_rdtr_fpd_not_running(void) "FPD written while RDTR was not running"
+e1000e_irq_tidv_fpd_running(void) "FPD written while TIDV was running"
+e1000e_irq_tidv_fpd_not_running(void) "FPD written while TIDV was not running"
+e1000e_irq_eitr_set(uint32_t eitr_num, uint32_t val) "EITR[%u] = %u"
+e1000e_irq_itr_set(uint32_t val) "ITR = %u"
+e1000e_irq_fire_all_timers(uint32_t val) "Firing all delay/throttling timers on all interrupts enable (0x%X written to IMS)"
+e1000e_irq_adding_delayed_causes(uint32_t val, uint32_t icr) "Merging delayed causes 0x%X to ICR 0x%X"
+e1000e_irq_msix_pending_clearing(uint32_t cause, uint32_t int_cfg, uint32_t vec) "Clearing MSI-X pending bit for cause 0x%x, IVAR config 0x%x, vector %u"
+
+e1000e_wrn_msix_vec_wrong(uint32_t cause, uint32_t cfg) "Invalid configuration for cause 0x%x: 0x%x"
+e1000e_wrn_msix_invalid(uint32_t cause, uint32_t cfg) "Invalid entry for cause 0x%x: 0x%x"
+
+e1000e_mac_set_permanent(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "Set permanent MAC: %02x:%02x:%02x:%02x:%02x:%02x"
+e1000e_mac_set_sw(uint8_t b0, uint8_t b1, uint8_t b2, uint8_t b3, uint8_t b4, uint8_t b5) "Set SW MAC: %02x:%02x:%02x:%02x:%02x:%02x"
+
+# hw/net/e1000e.c
+e1000e_cb_pci_realize(void) "E1000E PCI realize entry"
+e1000e_cb_pci_uninit(void) "E1000E PCI unit entry"
+e1000e_cb_qdev_reset(void) "E1000E qdev reset entry"
+e1000e_cb_pre_save(void) "E1000E pre save entry"
+e1000e_cb_post_load(void) "E1000E post load entry"
+
+e1000e_io_write_addr(uint64_t addr) "IOADDR write 0x%"PRIx64
+e1000e_io_write_data(uint64_t addr, uint64_t val) "IODATA write 0x%"PRIx64", value: 0x%"PRIx64
+e1000e_io_read_addr(uint64_t addr) "IOADDR read 0x%"PRIx64
+e1000e_io_read_data(uint64_t addr, uint64_t val) "IODATA read 0x%"PRIx64", value: 0x%"PRIx64
+e1000e_wrn_io_write_unknown(uint64_t addr) "IO write unknown address 0x%"PRIx64
+e1000e_wrn_io_read_unknown(uint64_t addr) "IO read unknown address 0x%"PRIx64
+e1000e_wrn_io_addr_undefined(uint64_t addr) "IO undefined register 0x%"PRIx64
+e1000e_wrn_io_addr_flash(uint64_t addr) "IO flash access (0x%"PRIx64") not implemented"
+e1000e_wrn_io_addr_unknown(uint64_t addr) "IO unknown register 0x%"PRIx64
+
+e1000e_msi_init_fail(int32_t res) "Failed to initialize MSI, error %d"
+e1000e_msix_init_fail(int32_t res) "Failed to initialize MSI-X, error %d"
+e1000e_msix_use_vector_fail(uint32_t vec, int32_t res) "Failed to use MSI-X vector %d, error %d"
+
+e1000e_cfg_support_virtio(bool support) "Virtio header supported: %d"
+
+e1000e_vm_state_running(void) "VM state is running"
+e1000e_vm_state_stopped(void) "VM state is stopped"
diff --git a/hw/net/vhost_net.c b/hw/net/vhost_net.c
index 4e3a061..f2d49ad 100644
--- a/hw/net/vhost_net.c
+++ b/hw/net/vhost_net.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "net/net.h"
 #include "net/tap.h"
 #include "net/vhost-user.h"
@@ -21,21 +22,18 @@
 #include "net/vhost_net.h"
 #include "qemu/error-report.h"
 
-#include "config.h"
 
 #ifdef CONFIG_VHOST_NET
 #include <linux/vhost.h>
 #include <sys/socket.h>
 #include <linux/kvm.h>
-#include <fcntl.h>
 #include <netpacket/packet.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <netinet/in.h>
 
-#include <stdio.h>
 
-#include "hw/virtio/virtio_ring.h"
+#include "standard-headers/linux/virtio_ring.h"
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/virtio-bus.h"
 
@@ -52,16 +50,18 @@
     VIRTIO_RING_F_INDIRECT_DESC,
     VIRTIO_RING_F_EVENT_IDX,
     VIRTIO_NET_F_MRG_RXBUF,
+    VIRTIO_F_VERSION_1,
     VHOST_INVALID_FEATURE_BIT
 };
 
 /* Features supported by others. */
-const int user_feature_bits[] = {
+static const int user_feature_bits[] = {
     VIRTIO_F_NOTIFY_ON_EMPTY,
     VIRTIO_RING_F_INDIRECT_DESC,
     VIRTIO_RING_F_EVENT_IDX,
 
     VIRTIO_F_ANY_LAYOUT,
+    VIRTIO_F_VERSION_1,
     VIRTIO_NET_F_CSUM,
     VIRTIO_NET_F_GUEST_CSUM,
     VIRTIO_NET_F_GSO,
@@ -74,13 +74,9 @@
     VIRTIO_NET_F_HOST_ECN,
     VIRTIO_NET_F_HOST_UFO,
     VIRTIO_NET_F_MRG_RXBUF,
-    VIRTIO_NET_F_STATUS,
-    VIRTIO_NET_F_CTRL_VQ,
-    VIRTIO_NET_F_CTRL_RX,
-    VIRTIO_NET_F_CTRL_VLAN,
-    VIRTIO_NET_F_CTRL_RX_EXTRA,
-    VIRTIO_NET_F_CTRL_MAC_ADDR,
-    VIRTIO_NET_F_CTRL_GUEST_OFFLOADS,
+
+    /* This bit implies RARP isn't sent by QEMU out of band */
+    VIRTIO_NET_F_GUEST_ANNOUNCE,
 
     VIRTIO_NET_F_MQ,
 
@@ -92,10 +88,10 @@
     const int *feature_bits = 0;
 
     switch (net->nc->info->type) {
-    case NET_CLIENT_OPTIONS_KIND_TAP:
+    case NET_CLIENT_DRIVER_TAP:
         feature_bits = kernel_feature_bits;
         break;
-    case NET_CLIENT_OPTIONS_KIND_VHOST_USER:
+    case NET_CLIENT_DRIVER_VHOST_USER:
         feature_bits = user_feature_bits;
         break;
     default:
@@ -107,22 +103,32 @@
     return feature_bits;
 }
 
-unsigned vhost_net_get_features(struct vhost_net *net, unsigned features)
+uint64_t vhost_net_get_features(struct vhost_net *net, uint64_t features)
 {
     return vhost_get_features(&net->dev, vhost_net_get_feature_bits(net),
             features);
 }
 
-void vhost_net_ack_features(struct vhost_net *net, unsigned features)
+void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
 {
     net->dev.acked_features = net->dev.backend_features;
     vhost_ack_features(&net->dev, vhost_net_get_feature_bits(net), features);
 }
 
+uint64_t vhost_net_get_max_queues(VHostNetState *net)
+{
+    return net->dev.max_queues;
+}
+
+uint64_t vhost_net_get_acked_features(VHostNetState *net)
+{
+    return net->dev.acked_features;
+}
+
 static int vhost_net_get_fd(NetClientState *backend)
 {
     switch (backend->info->type) {
-    case NET_CLIENT_OPTIONS_KIND_TAP:
+    case NET_CLIENT_DRIVER_TAP:
         return tap_get_fd(backend);
     default:
         fprintf(stderr, "vhost-net requires tap backend\n");
@@ -134,12 +140,18 @@
 {
     int r;
     bool backend_kernel = options->backend_type == VHOST_BACKEND_TYPE_KERNEL;
-    struct vhost_net *net = g_malloc(sizeof *net);
+    struct vhost_net *net = g_new0(struct vhost_net, 1);
+    uint64_t features = 0;
 
     if (!options->net_backend) {
         fprintf(stderr, "vhost-net requires net backend to be setup\n");
         goto fail;
     }
+    net->nc = options->net_backend;
+
+    net->dev.max_queues = 1;
+    net->dev.nvqs = 2;
+    net->dev.vqs = net->vqs;
 
     if (backend_kernel) {
         r = vhost_net_get_fd(options->net_backend);
@@ -147,48 +159,57 @@
             goto fail;
         }
         net->dev.backend_features = qemu_has_vnet_hdr(options->net_backend)
-            ? 0 : (1 << VHOST_NET_F_VIRTIO_NET_HDR);
+            ? 0 : (1ULL << VHOST_NET_F_VIRTIO_NET_HDR);
         net->backend = r;
+        net->dev.protocol_features = 0;
     } else {
         net->dev.backend_features = 0;
+        net->dev.protocol_features = 0;
         net->backend = -1;
-    }
-    net->nc = options->net_backend;
 
-    net->dev.nvqs = 2;
-    net->dev.vqs = net->vqs;
+        /* vhost-user needs vq_index to initiate a specific queue pair */
+        net->dev.vq_index = net->nc->queue_index * net->dev.nvqs;
+    }
 
     r = vhost_dev_init(&net->dev, options->opaque,
-                       options->backend_type, options->force);
+                       options->backend_type, options->busyloop_timeout);
     if (r < 0) {
         goto fail;
     }
     if (backend_kernel) {
         if (!qemu_has_vnet_hdr_len(options->net_backend,
                                sizeof(struct virtio_net_hdr_mrg_rxbuf))) {
-            net->dev.features &= ~(1 << VIRTIO_NET_F_MRG_RXBUF);
+            net->dev.features &= ~(1ULL << VIRTIO_NET_F_MRG_RXBUF);
         }
         if (~net->dev.features & net->dev.backend_features) {
             fprintf(stderr, "vhost lacks feature mask %" PRIu64
                    " for backend\n",
                    (uint64_t)(~net->dev.features & net->dev.backend_features));
-            vhost_dev_cleanup(&net->dev);
             goto fail;
         }
     }
+
     /* Set sane init value. Override when guest acks. */
-    vhost_net_ack_features(net, 0);
+    if (net->nc->info->type == NET_CLIENT_DRIVER_VHOST_USER) {
+        features = vhost_user_get_acked_features(net->nc);
+        if (~net->dev.features & features) {
+            fprintf(stderr, "vhost lacks feature mask %" PRIu64
+                    " for backend\n",
+                    (uint64_t)(~net->dev.features & features));
+            goto fail;
+        }
+    }
+
+    vhost_net_ack_features(net, features);
+
     return net;
+
 fail:
+    vhost_dev_cleanup(&net->dev);
     g_free(net);
     return NULL;
 }
 
-bool vhost_net_query(VHostNetState *net, VirtIODevice *dev)
-{
-    return vhost_dev_query(&net->dev, dev);
-}
-
 static void vhost_net_set_vq_index(struct vhost_net *net, int vq_index)
 {
     net->dev.vq_index = vq_index;
@@ -217,13 +238,11 @@
         net->nc->info->poll(net->nc, false);
     }
 
-    if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
         qemu_set_fd_handler(net->backend, NULL, NULL, NULL);
         file.fd = net->backend;
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
-            const VhostOps *vhost_ops = net->dev.vhost_ops;
-            r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                      &file);
+            r = vhost_net_set_backend(&net->dev, &file);
             if (r < 0) {
                 r = -errno;
                 goto fail;
@@ -233,11 +252,9 @@
     return 0;
 fail:
     file.fd = -1;
-    if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
         while (file.index-- > 0) {
-            const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                          &file);
+            int r = vhost_net_set_backend(&net->dev, &file);
             assert(r >= 0);
         }
     }
@@ -256,11 +273,9 @@
 {
     struct vhost_vring_file file = { .fd = -1 };
 
-    if (net->nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (net->nc->info->type == NET_CLIENT_DRIVER_TAP) {
         for (file.index = 0; file.index < net->dev.nvqs; ++file.index) {
-            const VhostOps *vhost_ops = net->dev.vhost_ops;
-            int r = vhost_ops->vhost_call(&net->dev, VHOST_NET_SET_BACKEND,
-                                          &file);
+            int r = vhost_net_set_backend(&net->dev, &file);
             assert(r >= 0);
         }
     }
@@ -271,19 +286,6 @@
     vhost_dev_disable_notifiers(&net->dev, dev);
 }
 
-static bool vhost_net_device_endian_ok(VirtIODevice *vdev)
-{
-#ifdef TARGET_IS_BIENDIAN
-#ifdef HOST_WORDS_BIGENDIAN
-    return virtio_is_big_endian(vdev);
-#else
-    return !virtio_is_big_endian(vdev);
-#endif
-#else
-    return true;
-#endif
-}
-
 int vhost_net_start(VirtIODevice *dev, NetClientState *ncs,
                     int total_queues)
 {
@@ -292,21 +294,25 @@
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
     int r, e, i;
 
-    if (!vhost_net_device_endian_ok(dev)) {
-        error_report("vhost-net does not support cross-endian");
-        r = -ENOSYS;
-        goto err;
-    }
-
     if (!k->set_guest_notifiers) {
         error_report("binding does not support guest notifiers");
-        r = -ENOSYS;
-        goto err;
+        return -ENOSYS;
     }
 
     for (i = 0; i < total_queues; i++) {
-        vhost_net_set_vq_index(get_vhost_net(ncs[i].peer), i * 2);
-    }
+        struct vhost_net *net;
+
+        net = get_vhost_net(ncs[i].peer);
+        vhost_net_set_vq_index(net, i * 2);
+
+        /* Suppress the masking guest notifiers on vhost user
+         * because vhost user doesn't interrupt masking/unmasking
+         * properly.
+         */
+        if (net->nc->info->type == NET_CLIENT_DRIVER_VHOST_USER) {
+            dev->use_guest_notifier_mask = false;
+        }
+     }
 
     r = k->set_guest_notifiers(qbus->parent, total_queues * 2, true);
     if (r < 0) {
@@ -320,6 +326,15 @@
         if (r < 0) {
             goto err_start;
         }
+
+        if (ncs[i].peer->vring_enable) {
+            /* restore vring enable state */
+            r = vhost_set_vring_enable(ncs[i].peer, ncs[i].peer->vring_enable);
+
+            if (r < 0) {
+                goto err_start;
+            }
+        }
     }
 
     return 0;
@@ -360,7 +375,16 @@
 void vhost_net_cleanup(struct vhost_net *net)
 {
     vhost_dev_cleanup(&net->dev);
-    g_free(net);
+}
+
+int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
+{
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    assert(vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+    assert(vhost_ops->vhost_migration_done);
+
+    return vhost_ops->vhost_migration_done(&net->dev, mac_addr);
 }
 
 bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
@@ -383,11 +407,12 @@
     }
 
     switch (nc->info->type) {
-    case NET_CLIENT_OPTIONS_KIND_TAP:
+    case NET_CLIENT_DRIVER_TAP:
         vhost_net = tap_get_vhost_net(nc);
         break;
-    case NET_CLIENT_OPTIONS_KIND_VHOST_USER:
+    case NET_CLIENT_DRIVER_VHOST_USER:
         vhost_net = vhost_user_get_vhost_net(nc);
+        assert(vhost_net);
         break;
     default:
         break;
@@ -395,18 +420,33 @@
 
     return vhost_net;
 }
+
+int vhost_set_vring_enable(NetClientState *nc, int enable)
+{
+    VHostNetState *net = get_vhost_net(nc);
+    const VhostOps *vhost_ops = net->dev.vhost_ops;
+
+    nc->vring_enable = enable;
+
+    if (vhost_ops && vhost_ops->vhost_set_vring_enable) {
+        return vhost_ops->vhost_set_vring_enable(&net->dev, enable);
+    }
+
+    return 0;
+}
+
 #else
+uint64_t vhost_net_get_max_queues(VHostNetState *net)
+{
+    return 1;
+}
+
 struct vhost_net *vhost_net_init(VhostNetOptions *options)
 {
     error_report("vhost-net support is not compiled in");
     return NULL;
 }
 
-bool vhost_net_query(VHostNetState *net, VirtIODevice *dev)
-{
-    return false;
-}
-
 int vhost_net_start(VirtIODevice *dev,
                     NetClientState *ncs,
                     int total_queues)
@@ -423,14 +463,20 @@
 {
 }
 
-unsigned vhost_net_get_features(struct vhost_net *net, unsigned features)
+uint64_t vhost_net_get_features(struct vhost_net *net, uint64_t features)
 {
     return features;
 }
-void vhost_net_ack_features(struct vhost_net *net, unsigned features)
+
+void vhost_net_ack_features(struct vhost_net *net, uint64_t features)
 {
 }
 
+uint64_t vhost_net_get_acked_features(VHostNetState *net)
+{
+    return 0;
+}
+
 bool vhost_net_virtqueue_pending(VHostNetState *net, int idx)
 {
     return false;
@@ -441,8 +487,18 @@
 {
 }
 
+int vhost_net_notify_migration_done(struct vhost_net *net, char* mac_addr)
+{
+    return -1;
+}
+
 VHostNetState *get_vhost_net(NetClientState *nc)
 {
     return 0;
 }
+
+int vhost_set_vring_enable(NetClientState *nc, int enable)
+{
+    return 0;
+}
 #endif
diff --git a/hw/net/virtio-net.c b/hw/net/virtio-net.c
index 239ac5e..b8cf936 100644
--- a/hw/net/virtio-net.c
+++ b/hw/net/virtio-net.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu/iov.h"
 #include "hw/virtio/virtio.h"
 #include "net/net.h"
@@ -86,7 +87,8 @@
 
     memcpy(&netcfg, config, n->config_size);
 
-    if (!(vdev->guest_features >> VIRTIO_NET_F_CTRL_MAC_ADDR & 1) &&
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR) &&
+        !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1) &&
         memcmp(netcfg.mac, n->mac, ETH_ALEN)) {
         memcpy(n->mac, netcfg.mac, ETH_ALEN);
         qemu_format_nic_info_str(qemu_get_queue(n->nic), n->mac);
@@ -120,14 +122,17 @@
         return;
     }
 
-    if (!!n->vhost_started ==
-        (virtio_net_started(n, status) && !nc->peer->link_down)) {
+    if ((virtio_net_started(n, status) && !nc->peer->link_down) ==
+        !!n->vhost_started) {
         return;
     }
     if (!n->vhost_started) {
         int r, i;
 
-        if (!vhost_net_query(get_vhost_net(nc->peer), vdev)) {
+        if (n->needs_vnet_hdr_swap) {
+            error_report("backend does not support %s vnet headers; "
+                         "falling back on userspace virtio",
+                         virtio_is_big_endian(vdev) ? "BE" : "LE");
             return;
         }
 
@@ -155,6 +160,59 @@
     }
 }
 
+static int virtio_net_set_vnet_endian_one(VirtIODevice *vdev,
+                                          NetClientState *peer,
+                                          bool enable)
+{
+    if (virtio_is_big_endian(vdev)) {
+        return qemu_set_vnet_be(peer, enable);
+    } else {
+        return qemu_set_vnet_le(peer, enable);
+    }
+}
+
+static bool virtio_net_set_vnet_endian(VirtIODevice *vdev, NetClientState *ncs,
+                                       int queues, bool enable)
+{
+    int i;
+
+    for (i = 0; i < queues; i++) {
+        if (virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, enable) < 0 &&
+            enable) {
+            while (--i >= 0) {
+                virtio_net_set_vnet_endian_one(vdev, ncs[i].peer, false);
+            }
+
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void virtio_net_vnet_endian_status(VirtIONet *n, uint8_t status)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(n);
+    int queues = n->multiqueue ? n->max_queues : 1;
+
+    if (virtio_net_started(n, status)) {
+        /* Before using the device, we tell the network backend about the
+         * endianness to use when parsing vnet headers. If the backend
+         * can't do it, we fallback onto fixing the headers in the core
+         * virtio-net code.
+         */
+        n->needs_vnet_hdr_swap = virtio_net_set_vnet_endian(vdev, n->nic->ncs,
+                                                            queues, true);
+    } else if (virtio_net_started(n, vdev->status)) {
+        /* After using the device, we need to reset the network backend to
+         * the default (guest native endianness), otherwise the guest may
+         * lose network connectivity if it is rebooted into a different
+         * endianness.
+         */
+        virtio_net_set_vnet_endian(vdev, n->nic->ncs, queues, false);
+    }
+}
+
 static void virtio_net_set_status(struct VirtIODevice *vdev, uint8_t status)
 {
     VirtIONet *n = VIRTIO_NET(vdev);
@@ -162,9 +220,12 @@
     int i;
     uint8_t queue_status;
 
+    virtio_net_vnet_endian_status(n, status);
     virtio_net_vhost_status(n, status);
 
     for (i = 0; i < n->max_queues; i++) {
+        NetClientState *ncs = qemu_get_subqueue(n->nic, i);
+        bool queue_started;
         q = &n->vqs[i];
 
         if ((!n->multiqueue && i != 0) || i >= n->curr_queues) {
@@ -172,12 +233,18 @@
         } else {
             queue_status = status;
         }
+        queue_started =
+            virtio_net_started(n, queue_status) && !n->vhost_started;
+
+        if (queue_started) {
+            qemu_flush_queued_packets(ncs);
+        }
 
         if (!q->tx_waiting) {
             continue;
         }
 
-        if (virtio_net_started(n, queue_status) && !n->vhost_started) {
+        if (queue_started) {
             if (q->tx_timer) {
                 timer_mod(q->tx_timer,
                                qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + n->tx_timeout);
@@ -226,12 +293,6 @@
     }
 }
 
-static char *mac_strdup_printf(const uint8_t *mac)
-{
-    return g_strdup_printf("%.2x:%.2x:%.2x:%.2x:%.2x:%.2x", mac[0],
-                            mac[1], mac[2], mac[3], mac[4], mac[5]);
-}
-
 static intList *get_vlan_table(VirtIONet *n)
 {
     intList *list, *entry;
@@ -284,12 +345,12 @@
     info->multicast_overflow = n->mac_table.multi_overflow;
     info->unicast_overflow = n->mac_table.uni_overflow;
 
-    info->main_mac = mac_strdup_printf(n->mac);
+    info->main_mac = qemu_mac_strdup_printf(n->mac);
 
     str_list = NULL;
     for (i = 0; i < n->mac_table.first_multi; i++) {
         entry = g_malloc0(sizeof(*entry));
-        entry->value = mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN);
+        entry->value = qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN);
         entry->next = str_list;
         str_list = entry;
     }
@@ -298,14 +359,14 @@
     str_list = NULL;
     for (i = n->mac_table.first_multi; i < n->mac_table.in_use; i++) {
         entry = g_malloc0(sizeof(*entry));
-        entry->value = mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN);
+        entry->value = qemu_mac_strdup_printf(n->mac_table.macs + i * ETH_ALEN);
         entry->next = str_list;
         str_list = entry;
     }
     info->multicast_table = str_list;
     info->vlan_table = get_vlan_table(n);
 
-    if (!((1 << VIRTIO_NET_F_CTRL_VLAN) & vdev->guest_features)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VLAN)) {
         info->vlan = RX_STATE_ALL;
     } else if (!info->vlan_table) {
         info->vlan = RX_STATE_NONE;
@@ -372,15 +433,21 @@
     return n->has_ufo;
 }
 
-static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs)
+static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs,
+                                       int version_1)
 {
     int i;
     NetClientState *nc;
 
     n->mergeable_rx_bufs = mergeable_rx_bufs;
 
-    n->guest_hdr_len = n->mergeable_rx_bufs ?
-        sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr);
+    if (version_1) {
+        n->guest_hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
+    } else {
+        n->guest_hdr_len = n->mergeable_rx_bufs ?
+            sizeof(struct virtio_net_hdr_mrg_rxbuf) :
+            sizeof(struct virtio_net_hdr);
+    }
 
     for (i = 0; i < n->max_queues; i++) {
         nc = qemu_get_subqueue(n->nic, i);
@@ -401,7 +468,11 @@
         return 0;
     }
 
-    if (nc->peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) {
+        vhost_set_vring_enable(nc->peer, 1);
+    }
+
+    if (nc->peer->info->type != NET_CLIENT_DRIVER_TAP) {
         return 0;
     }
 
@@ -416,7 +487,11 @@
         return 0;
     }
 
-    if (nc->peer->info->type !=  NET_CLIENT_OPTIONS_KIND_TAP) {
+    if (nc->peer->info->type == NET_CLIENT_DRIVER_VHOST_USER) {
+        vhost_set_vring_enable(nc->peer, 0);
+    }
+
+    if (nc->peer->info->type !=  NET_CLIENT_DRIVER_TAP) {
         return 0;
     }
 
@@ -443,28 +518,32 @@
 
 static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue);
 
-static uint32_t virtio_net_get_features(VirtIODevice *vdev, uint32_t features)
+static uint64_t virtio_net_get_features(VirtIODevice *vdev, uint64_t features,
+                                        Error **errp)
 {
     VirtIONet *n = VIRTIO_NET(vdev);
     NetClientState *nc = qemu_get_queue(n->nic);
 
-    features |= (1 << VIRTIO_NET_F_MAC);
+    /* Firstly sync all virtio-net possible supported features */
+    features |= n->host_features;
+
+    virtio_add_feature(&features, VIRTIO_NET_F_MAC);
 
     if (!peer_has_vnet_hdr(n)) {
-        features &= ~(0x1 << VIRTIO_NET_F_CSUM);
-        features &= ~(0x1 << VIRTIO_NET_F_HOST_TSO4);
-        features &= ~(0x1 << VIRTIO_NET_F_HOST_TSO6);
-        features &= ~(0x1 << VIRTIO_NET_F_HOST_ECN);
+        virtio_clear_feature(&features, VIRTIO_NET_F_CSUM);
+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO4);
+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_TSO6);
+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_ECN);
 
-        features &= ~(0x1 << VIRTIO_NET_F_GUEST_CSUM);
-        features &= ~(0x1 << VIRTIO_NET_F_GUEST_TSO4);
-        features &= ~(0x1 << VIRTIO_NET_F_GUEST_TSO6);
-        features &= ~(0x1 << VIRTIO_NET_F_GUEST_ECN);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_CSUM);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO4);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_TSO6);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_ECN);
     }
 
     if (!peer_has_vnet_hdr(n) || !peer_has_ufo(n)) {
-        features &= ~(0x1 << VIRTIO_NET_F_GUEST_UFO);
-        features &= ~(0x1 << VIRTIO_NET_F_HOST_UFO);
+        virtio_clear_feature(&features, VIRTIO_NET_F_GUEST_UFO);
+        virtio_clear_feature(&features, VIRTIO_NET_F_HOST_UFO);
     }
 
     if (!get_vhost_net(nc->peer)) {
@@ -473,17 +552,17 @@
     return vhost_net_get_features(get_vhost_net(nc->peer), features);
 }
 
-static uint32_t virtio_net_bad_features(VirtIODevice *vdev)
+static uint64_t virtio_net_bad_features(VirtIODevice *vdev)
 {
-    uint32_t features = 0;
+    uint64_t features = 0;
 
     /* Linux kernel 2.6.25.  It understood MAC (as everyone must),
      * but also these: */
-    features |= (1 << VIRTIO_NET_F_MAC);
-    features |= (1 << VIRTIO_NET_F_CSUM);
-    features |= (1 << VIRTIO_NET_F_HOST_TSO4);
-    features |= (1 << VIRTIO_NET_F_HOST_TSO6);
-    features |= (1 << VIRTIO_NET_F_HOST_ECN);
+    virtio_add_feature(&features, VIRTIO_NET_F_MAC);
+    virtio_add_feature(&features, VIRTIO_NET_F_CSUM);
+    virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO4);
+    virtio_add_feature(&features, VIRTIO_NET_F_HOST_TSO6);
+    virtio_add_feature(&features, VIRTIO_NET_F_HOST_ECN);
 
     return features;
 }
@@ -516,14 +595,19 @@
     return virtio_net_guest_offloads_by_features(vdev->guest_features);
 }
 
-static void virtio_net_set_features(VirtIODevice *vdev, uint32_t features)
+static void virtio_net_set_features(VirtIODevice *vdev, uint64_t features)
 {
     VirtIONet *n = VIRTIO_NET(vdev);
     int i;
 
-    virtio_net_set_multiqueue(n, !!(features & (1 << VIRTIO_NET_F_MQ)));
+    virtio_net_set_multiqueue(n,
+                              virtio_has_feature(features, VIRTIO_NET_F_MQ));
 
-    virtio_net_set_mrg_rx_bufs(n, !!(features & (1 << VIRTIO_NET_F_MRG_RXBUF)));
+    virtio_net_set_mrg_rx_bufs(n,
+                               virtio_has_feature(features,
+                                                  VIRTIO_NET_F_MRG_RXBUF),
+                               virtio_has_feature(features,
+                                                  VIRTIO_F_VERSION_1));
 
     if (n->has_vnet_hdr) {
         n->curr_guest_offloads =
@@ -540,7 +624,7 @@
         vhost_net_ack_features(get_vhost_net(nc->peer), features);
     }
 
-    if ((1 << VIRTIO_NET_F_CTRL_VLAN) & features) {
+    if (virtio_has_feature(features, VIRTIO_NET_F_CTRL_VLAN)) {
         memset(n->vlans, 0, MAX_VLAN >> 3);
     } else {
         memset(n->vlans, 0xff, MAX_VLAN >> 3);
@@ -587,7 +671,7 @@
     uint64_t offloads;
     size_t s;
 
-    if (!((1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) & vdev->guest_features)) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
         return VIRTIO_NET_ERR;
     }
 
@@ -798,20 +882,24 @@
     VirtIONet *n = VIRTIO_NET(vdev);
     struct virtio_net_ctrl_hdr ctrl;
     virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     size_t s;
     struct iovec *iov, *iov2;
     unsigned int iov_cnt;
 
-    while (virtqueue_pop(vq, &elem)) {
-        if (iov_size(elem.in_sg, elem.in_num) < sizeof(status) ||
-            iov_size(elem.out_sg, elem.out_num) < sizeof(ctrl)) {
+    for (;;) {
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+        if (iov_size(elem->in_sg, elem->in_num) < sizeof(status) ||
+            iov_size(elem->out_sg, elem->out_num) < sizeof(ctrl)) {
             error_report("virtio-net ctrl missing headers");
             exit(1);
         }
 
-        iov_cnt = elem.out_num;
-        iov2 = iov = g_memdup(elem.out_sg, sizeof(struct iovec) * elem.out_num);
+        iov_cnt = elem->out_num;
+        iov2 = iov = g_memdup(elem->out_sg, sizeof(struct iovec) * elem->out_num);
         s = iov_to_buf(iov, iov_cnt, 0, &ctrl, sizeof(ctrl));
         iov_discard_front(&iov, &iov_cnt, sizeof(ctrl));
         if (s != sizeof(ctrl)) {
@@ -830,12 +918,13 @@
             status = virtio_net_handle_offloads(n, ctrl.cmd, iov, iov_cnt);
         }
 
-        s = iov_from_buf(elem.in_sg, elem.in_num, 0, &status, sizeof(status));
+        s = iov_from_buf(elem->in_sg, elem->in_num, 0, &status, sizeof(status));
         assert(s == sizeof(status));
 
-        virtqueue_push(vq, &elem, sizeof(status));
+        virtqueue_push(vq, elem, sizeof(status));
         virtio_notify(vdev, vq);
         g_free(iov2);
+        g_free(elem);
     }
 }
 
@@ -937,7 +1026,10 @@
         void *wbuf = (void *)buf;
         work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len,
                                     size - n->host_hdr_len);
-        virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf);
+
+        if (n->needs_vnet_hdr_swap) {
+            virtio_net_hdr_swap(VIRTIO_DEVICE(n), wbuf);
+        }
         iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr));
     } else {
         struct virtio_net_hdr hdr = {
@@ -961,7 +1053,7 @@
     ptr += n->host_hdr_len;
 
     if (!memcmp(&ptr[12], vlan, sizeof(vlan))) {
-        int vid = be16_to_cpup((uint16_t *)(ptr + 14)) & 0xfff;
+        int vid = lduw_be_p(ptr + 14) & 0xfff;
         if (!(n->vlans[vid >> 5] & (1U << (vid & 0x1f))))
             return 0;
     }
@@ -1024,38 +1116,42 @@
     offset = i = 0;
 
     while (offset < size) {
-        VirtQueueElement elem;
+        VirtQueueElement *elem;
         int len, total;
-        const struct iovec *sg = elem.in_sg;
+        const struct iovec *sg;
 
         total = 0;
 
-        if (virtqueue_pop(q->rx_vq, &elem) == 0) {
+        elem = virtqueue_pop(q->rx_vq, sizeof(VirtQueueElement));
+        if (!elem) {
             if (i == 0)
                 return -1;
             error_report("virtio-net unexpected empty queue: "
-                    "i %zd mergeable %d offset %zd, size %zd, "
-                    "guest hdr len %zd, host hdr len %zd guest features 0x%x",
-                    i, n->mergeable_rx_bufs, offset, size,
-                    n->guest_hdr_len, n->host_hdr_len, vdev->guest_features);
+                         "i %zd mergeable %d offset %zd, size %zd, "
+                         "guest hdr len %zd, host hdr len %zd "
+                         "guest features 0x%" PRIx64,
+                         i, n->mergeable_rx_bufs, offset, size,
+                         n->guest_hdr_len, n->host_hdr_len,
+                         vdev->guest_features);
             exit(1);
         }
 
-        if (elem.in_num < 1) {
+        if (elem->in_num < 1) {
             error_report("virtio-net receive queue contains no in buffers");
             exit(1);
         }
 
+        sg = elem->in_sg;
         if (i == 0) {
             assert(offset == 0);
             if (n->mergeable_rx_bufs) {
                 mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg),
-                                    sg, elem.in_num,
+                                    sg, elem->in_num,
                                     offsetof(typeof(mhdr), num_buffers),
                                     sizeof(mhdr.num_buffers));
             }
 
-            receive_header(n, sg, elem.in_num, buf, size);
+            receive_header(n, sg, elem->in_num, buf, size);
             offset = n->host_hdr_len;
             total += n->guest_hdr_len;
             guest_offset = n->guest_hdr_len;
@@ -1064,7 +1160,7 @@
         }
 
         /* copy in packet.  ugh */
-        len = iov_from_buf(sg, elem.in_num, guest_offset,
+        len = iov_from_buf(sg, elem->in_num, guest_offset,
                            buf + offset, size - offset);
         total += len;
         offset += len;
@@ -1072,18 +1168,14 @@
          * must have consumed the complete packet.
          * Otherwise, drop it. */
         if (!n->mergeable_rx_bufs && offset < size) {
-#if 0
-            error_report("virtio-net truncated non-mergeable packet: "
-                         "i %zd mergeable %d offset %zd, size %zd, "
-                         "guest hdr len %zd, host hdr len %zd",
-                         i, n->mergeable_rx_bufs,
-                         offset, size, n->guest_hdr_len, n->host_hdr_len);
-#endif
+            virtqueue_discard(q->rx_vq, elem, total);
+            g_free(elem);
             return size;
         }
 
         /* signal other side */
-        virtqueue_fill(q->rx_vq, &elem, total, i++);
+        virtqueue_fill(q->rx_vq, elem, total, i++);
+        g_free(elem);
     }
 
     if (mhdr_cnt) {
@@ -1107,10 +1199,11 @@
     VirtIONetQueue *q = virtio_net_get_subqueue(nc);
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
 
-    virtqueue_push(q->tx_vq, &q->async_tx.elem, 0);
+    virtqueue_push(q->tx_vq, q->async_tx.elem, 0);
     virtio_notify(vdev, q->tx_vq);
 
-    q->async_tx.elem.out_num = q->async_tx.len = 0;
+    g_free(q->async_tx.elem);
+    q->async_tx.elem = NULL;
 
     virtio_queue_set_notification(q->tx_vq, 1);
     virtio_net_flush_tx(q);
@@ -1121,37 +1214,56 @@
 {
     VirtIONet *n = q->n;
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     int32_t num_packets = 0;
     int queue_index = vq2q(virtio_get_queue_index(q->tx_vq));
     if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
         return num_packets;
     }
 
-    if (q->async_tx.elem.out_num) {
+    if (q->async_tx.elem) {
         virtio_queue_set_notification(q->tx_vq, 0);
         return num_packets;
     }
 
-    while (virtqueue_pop(q->tx_vq, &elem)) {
-        ssize_t ret, len;
-        unsigned int out_num = elem.out_num;
-        struct iovec *out_sg = &elem.out_sg[0];
-        struct iovec sg[VIRTQUEUE_MAX_SIZE];
+    for (;;) {
+        ssize_t ret;
+        unsigned int out_num;
+        struct iovec sg[VIRTQUEUE_MAX_SIZE], sg2[VIRTQUEUE_MAX_SIZE + 1], *out_sg;
+        struct virtio_net_hdr_mrg_rxbuf mhdr;
 
+        elem = virtqueue_pop(q->tx_vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            break;
+        }
+
+        out_num = elem->out_num;
+        out_sg = elem->out_sg;
         if (out_num < 1) {
             error_report("virtio-net header not in first element");
             exit(1);
         }
 
         if (n->has_vnet_hdr) {
-            if (out_sg[0].iov_len < n->guest_hdr_len) {
+            if (iov_to_buf(out_sg, out_num, 0, &mhdr, n->guest_hdr_len) <
+                n->guest_hdr_len) {
                 error_report("virtio-net header incorrect");
                 exit(1);
             }
-            virtio_net_hdr_swap(vdev, (void *) out_sg[0].iov_base);
+            if (n->needs_vnet_hdr_swap) {
+                virtio_net_hdr_swap(vdev, (void *) &mhdr);
+                sg2[0].iov_base = &mhdr;
+                sg2[0].iov_len = n->guest_hdr_len;
+                out_num = iov_copy(&sg2[1], ARRAY_SIZE(sg2) - 1,
+                                   out_sg, out_num,
+                                   n->guest_hdr_len, -1);
+                if (out_num == VIRTQUEUE_MAX_SIZE) {
+                    goto drop;
+		}
+                out_num += 1;
+                out_sg = sg2;
+	    }
         }
-
         /*
          * If host wants to see the guest header as is, we can
          * pass it on unchanged. Otherwise, copy just the parts
@@ -1169,21 +1281,18 @@
             out_sg = sg;
         }
 
-        len = n->guest_hdr_len;
-
         ret = qemu_sendv_packet_async(qemu_get_subqueue(n->nic, queue_index),
                                       out_sg, out_num, virtio_net_tx_complete);
         if (ret == 0) {
             virtio_queue_set_notification(q->tx_vq, 0);
             q->async_tx.elem = elem;
-            q->async_tx.len  = len;
             return -EBUSY;
         }
 
-        len += ret;
-
-        virtqueue_push(q->tx_vq, &elem, 0);
+drop:
+        virtqueue_push(q->tx_vq, elem, 0);
         virtio_notify(vdev, q->tx_vq);
+        g_free(elem);
 
         if (++num_packets >= n->tx_burst) {
             break;
@@ -1301,45 +1410,91 @@
     }
 }
 
-static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue)
+static void virtio_net_add_queue(VirtIONet *n, int index)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
-    int i, max = multiqueue ? n->max_queues : 1;
+
+    n->vqs[index].rx_vq = virtio_add_queue(vdev, 256, virtio_net_handle_rx);
+    if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) {
+        n->vqs[index].tx_vq =
+            virtio_add_queue(vdev, 256, virtio_net_handle_tx_timer);
+        n->vqs[index].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                              virtio_net_tx_timer,
+                                              &n->vqs[index]);
+    } else {
+        n->vqs[index].tx_vq =
+            virtio_add_queue(vdev, 256, virtio_net_handle_tx_bh);
+        n->vqs[index].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[index]);
+    }
+
+    n->vqs[index].tx_waiting = 0;
+    n->vqs[index].n = n;
+}
+
+static void virtio_net_del_queue(VirtIONet *n, int index)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(n);
+    VirtIONetQueue *q = &n->vqs[index];
+    NetClientState *nc = qemu_get_subqueue(n->nic, index);
+
+    qemu_purge_queued_packets(nc);
+
+    virtio_del_queue(vdev, index * 2);
+    if (q->tx_timer) {
+        timer_del(q->tx_timer);
+        timer_free(q->tx_timer);
+    } else {
+        qemu_bh_delete(q->tx_bh);
+    }
+    virtio_del_queue(vdev, index * 2 + 1);
+}
+
+static void virtio_net_change_num_queues(VirtIONet *n, int new_max_queues)
+{
+    VirtIODevice *vdev = VIRTIO_DEVICE(n);
+    int old_num_queues = virtio_get_num_queues(vdev);
+    int new_num_queues = new_max_queues * 2 + 1;
+    int i;
+
+    assert(old_num_queues >= 3);
+    assert(old_num_queues % 2 == 1);
+
+    if (old_num_queues == new_num_queues) {
+        return;
+    }
+
+    /*
+     * We always need to remove and add ctrl vq if
+     * old_num_queues != new_num_queues. Remove ctrl_vq first,
+     * and then we only enter one of the following too loops.
+     */
+    virtio_del_queue(vdev, old_num_queues - 1);
+
+    for (i = new_num_queues - 1; i < old_num_queues - 1; i += 2) {
+        /* new_num_queues < old_num_queues */
+        virtio_net_del_queue(n, i / 2);
+    }
+
+    for (i = old_num_queues - 1; i < new_num_queues - 1; i += 2) {
+        /* new_num_queues > old_num_queues */
+        virtio_net_add_queue(n, i / 2);
+    }
+
+    /* add ctrl_vq last */
+    n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
+}
+
+static void virtio_net_set_multiqueue(VirtIONet *n, int multiqueue)
+{
+    int max = multiqueue ? n->max_queues : 1;
 
     n->multiqueue = multiqueue;
-
-    for (i = 2; i <= n->max_queues * 2 + 1; i++) {
-        virtio_del_queue(vdev, i);
-    }
-
-    for (i = 1; i < max; i++) {
-        n->vqs[i].rx_vq = virtio_add_queue(vdev, 256, virtio_net_handle_rx);
-        if (n->vqs[i].tx_timer) {
-            n->vqs[i].tx_vq =
-                virtio_add_queue(vdev, 256, virtio_net_handle_tx_timer);
-            n->vqs[i].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                                                   virtio_net_tx_timer,
-                                                   &n->vqs[i]);
-        } else {
-            n->vqs[i].tx_vq =
-                virtio_add_queue(vdev, 256, virtio_net_handle_tx_bh);
-            n->vqs[i].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[i]);
-        }
-
-        n->vqs[i].tx_waiting = 0;
-        n->vqs[i].n = n;
-    }
-
-    /* Note: Minux Guests (version 3.2.1) use ctrl vq but don't ack
-     * VIRTIO_NET_F_CTRL_VQ. Create ctrl vq unconditionally to avoid
-     * breaking them.
-     */
-    n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
+    virtio_net_change_num_queues(n, max);
 
     virtio_net_set_queues(n);
 }
 
-static void virtio_net_save(QEMUFile *f, void *opaque)
+static void virtio_net_save(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIONet *n = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
@@ -1380,20 +1535,17 @@
         }
     }
 
-    if ((1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) & vdev->guest_features) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
         qemu_put_be64(f, n->curr_guest_offloads);
     }
 }
 
-static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
+static int virtio_net_load(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIONet *n = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(n);
 
-    if (version_id < 2 || version_id > VIRTIO_NET_VM_VERSION)
-        return -EINVAL;
-
-    return virtio_load(vdev, f, version_id);
+    return virtio_load(vdev, f, VIRTIO_NET_VM_VERSION);
 }
 
 static int virtio_net_load_device(VirtIODevice *vdev, QEMUFile *f,
@@ -1405,70 +1557,53 @@
     qemu_get_buffer(f, n->mac, ETH_ALEN);
     n->vqs[0].tx_waiting = qemu_get_be32(f);
 
-    virtio_net_set_mrg_rx_bufs(n, qemu_get_be32(f));
+    virtio_net_set_mrg_rx_bufs(n, qemu_get_be32(f),
+                               virtio_vdev_has_feature(vdev,
+                                                       VIRTIO_F_VERSION_1));
 
-    if (version_id >= 3)
-        n->status = qemu_get_be16(f);
+    n->status = qemu_get_be16(f);
 
-    if (version_id >= 4) {
-        if (version_id < 8) {
-            n->promisc = qemu_get_be32(f);
-            n->allmulti = qemu_get_be32(f);
-        } else {
-            n->promisc = qemu_get_byte(f);
-            n->allmulti = qemu_get_byte(f);
+    n->promisc = qemu_get_byte(f);
+    n->allmulti = qemu_get_byte(f);
+
+    n->mac_table.in_use = qemu_get_be32(f);
+    /* MAC_TABLE_ENTRIES may be different from the saved image */
+    if (n->mac_table.in_use <= MAC_TABLE_ENTRIES) {
+        qemu_get_buffer(f, n->mac_table.macs,
+                        n->mac_table.in_use * ETH_ALEN);
+    } else {
+        int64_t i;
+
+        /* Overflow detected - can happen if source has a larger MAC table.
+         * We simply set overflow flag so there's no need to maintain the
+         * table of addresses, discard them all.
+         * Note: 64 bit math to avoid integer overflow.
+         */
+        for (i = 0; i < (int64_t)n->mac_table.in_use * ETH_ALEN; ++i) {
+            qemu_get_byte(f);
         }
-    }
-
-    if (version_id >= 5) {
-        n->mac_table.in_use = qemu_get_be32(f);
-        /* MAC_TABLE_ENTRIES may be different from the saved image */
-        if (n->mac_table.in_use <= MAC_TABLE_ENTRIES) {
-            qemu_get_buffer(f, n->mac_table.macs,
-                            n->mac_table.in_use * ETH_ALEN);
-        } else {
-            int64_t i;
-
-            /* Overflow detected - can happen if source has a larger MAC table.
-             * We simply set overflow flag so there's no need to maintain the
-             * table of addresses, discard them all.
-             * Note: 64 bit math to avoid integer overflow.
-             */
-            for (i = 0; i < (int64_t)n->mac_table.in_use * ETH_ALEN; ++i) {
-                qemu_get_byte(f);
-            }
-            n->mac_table.multi_overflow = n->mac_table.uni_overflow = 1;
-            n->mac_table.in_use = 0;
-        }
+        n->mac_table.multi_overflow = n->mac_table.uni_overflow = 1;
+        n->mac_table.in_use = 0;
     }
  
-    if (version_id >= 6)
-        qemu_get_buffer(f, (uint8_t *)n->vlans, MAX_VLAN >> 3);
+    qemu_get_buffer(f, (uint8_t *)n->vlans, MAX_VLAN >> 3);
 
-    if (version_id >= 7) {
-        if (qemu_get_be32(f) && !peer_has_vnet_hdr(n)) {
-            error_report("virtio-net: saved image requires vnet_hdr=on");
-            return -1;
-        }
+    if (qemu_get_be32(f) && !peer_has_vnet_hdr(n)) {
+        error_report("virtio-net: saved image requires vnet_hdr=on");
+        return -1;
     }
 
-    if (version_id >= 9) {
-        n->mac_table.multi_overflow = qemu_get_byte(f);
-        n->mac_table.uni_overflow = qemu_get_byte(f);
-    }
+    n->mac_table.multi_overflow = qemu_get_byte(f);
+    n->mac_table.uni_overflow = qemu_get_byte(f);
 
-    if (version_id >= 10) {
-        n->alluni = qemu_get_byte(f);
-        n->nomulti = qemu_get_byte(f);
-        n->nouni = qemu_get_byte(f);
-        n->nobcast = qemu_get_byte(f);
-    }
+    n->alluni = qemu_get_byte(f);
+    n->nomulti = qemu_get_byte(f);
+    n->nouni = qemu_get_byte(f);
+    n->nobcast = qemu_get_byte(f);
 
-    if (version_id >= 11) {
-        if (qemu_get_byte(f) && !peer_has_ufo(n)) {
-            error_report("virtio-net: saved image requires TUN_F_UFO support");
-            return -1;
-        }
+    if (qemu_get_byte(f) && !peer_has_ufo(n)) {
+        error_report("virtio-net: saved image requires TUN_F_UFO support");
+        return -1;
     }
 
     if (n->max_queues > 1) {
@@ -1488,7 +1623,7 @@
         }
     }
 
-    if ((1 << VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) & vdev->guest_features) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) {
         n->curr_guest_offloads = qemu_get_be64(f);
     } else {
         n->curr_guest_offloads = virtio_net_supported_guest_offloads(n);
@@ -1515,8 +1650,8 @@
         qemu_get_subqueue(n->nic, i)->link_down = link_down;
     }
 
-    if (vdev->guest_features & (0x1 << VIRTIO_NET_F_GUEST_ANNOUNCE) &&
-        vdev->guest_features & (0x1 << VIRTIO_NET_F_CTRL_VQ)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE) &&
+        virtio_vdev_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) {
         n->announce_counter = SELF_ANNOUNCE_ROUNDS;
         timer_mod(n->announce_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL));
     }
@@ -1524,19 +1659,11 @@
     return 0;
 }
 
-static void virtio_net_cleanup(NetClientState *nc)
-{
-    VirtIONet *n = qemu_get_nic_opaque(nc);
-
-    n->nic = NULL;
-}
-
 static NetClientInfo net_virtio_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = virtio_net_can_receive,
     .receive = virtio_net_receive,
-    .cleanup = virtio_net_cleanup,
     .link_status_changed = virtio_net_set_link_status,
     .query_rx_filter = virtio_net_query_rxfilter,
 };
@@ -1559,10 +1686,10 @@
                              vdev, idx, mask);
 }
 
-void virtio_net_set_config_size(VirtIONet *n, uint32_t host_features)
+static void virtio_net_set_config_size(VirtIONet *n, uint64_t host_features)
 {
     int i, config_size = 0;
-    host_features |= (1 << VIRTIO_NET_F_MAC);
+    virtio_add_feature(&host_features, VIRTIO_NET_F_MAC);
     for (i = 0; feature_sizes[i].flags != 0; i++) {
         if (host_features & feature_sizes[i].flags) {
             config_size = MAX(feature_sizes[i].end, config_size);
@@ -1592,13 +1719,19 @@
     NetClientState *nc;
     int i;
 
+    virtio_net_set_config_size(n, n->host_features);
     virtio_init(vdev, "virtio-net", VIRTIO_ID_NET, n->config_size);
 
     n->max_queues = MAX(n->nic_conf.peers.queues, 1);
+    if (n->max_queues * 2 + 1 > VIRTIO_QUEUE_MAX) {
+        error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
+                   "must be a positive integer less than %d.",
+                   n->max_queues, (VIRTIO_QUEUE_MAX - 1) / 2);
+        virtio_cleanup(vdev);
+        return;
+    }
     n->vqs = g_malloc0(sizeof(VirtIONetQueue) * n->max_queues);
-    n->vqs[0].rx_vq = virtio_add_queue(vdev, 256, virtio_net_handle_rx);
     n->curr_queues = 1;
-    n->vqs[0].n = n;
     n->tx_timeout = n->net_conf.txtimer;
 
     if (n->net_conf.tx && strcmp(n->net_conf.tx, "timer")
@@ -1609,16 +1742,10 @@
         error_report("Defaulting to \"bh\"");
     }
 
-    if (n->net_conf.tx && !strcmp(n->net_conf.tx, "timer")) {
-        n->vqs[0].tx_vq = virtio_add_queue(vdev, 256,
-                                           virtio_net_handle_tx_timer);
-        n->vqs[0].tx_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, virtio_net_tx_timer,
-                                               &n->vqs[0]);
-    } else {
-        n->vqs[0].tx_vq = virtio_add_queue(vdev, 256,
-                                           virtio_net_handle_tx_bh);
-        n->vqs[0].tx_bh = qemu_bh_new(virtio_net_tx_bh, &n->vqs[0]);
+    for (i = 0; i < n->max_queues; i++) {
+        virtio_net_add_queue(n, i);
     }
+
     n->ctrl_vq = virtio_add_queue(vdev, 64, virtio_net_handle_ctrl);
     qemu_macaddr_default_if_unset(&n->nic_conf.macaddr);
     memcpy(&n->mac[0], &n->nic_conf.macaddr, sizeof(n->mac));
@@ -1651,7 +1778,7 @@
 
     n->vqs[0].tx_waiting = 0;
     n->tx_burst = n->net_conf.txburst;
-    virtio_net_set_mrg_rx_bufs(n, 0);
+    virtio_net_set_mrg_rx_bufs(n, 0, 0);
     n->promisc = 1; /* for compatibility */
 
     n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN);
@@ -1662,21 +1789,17 @@
     nc->rxfilter_notify_enabled = 1;
 
     n->qdev = dev;
-    register_savevm(dev, "virtio-net", -1, VIRTIO_NET_VM_VERSION,
-                    virtio_net_save, virtio_net_load, n);
 }
 
 static void virtio_net_device_unrealize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIONet *n = VIRTIO_NET(dev);
-    int i;
+    int i, max_queues;
 
     /* This will stop vhost backend if appropriate. */
     virtio_net_set_status(vdev, 0);
 
-    unregister_savevm(dev, "virtio-net", n);
-
     g_free(n->netclient_name);
     n->netclient_name = NULL;
     g_free(n->netclient_type);
@@ -1685,18 +1808,9 @@
     g_free(n->mac_table.macs);
     g_free(n->vlans);
 
-    for (i = 0; i < n->max_queues; i++) {
-        VirtIONetQueue *q = &n->vqs[i];
-        NetClientState *nc = qemu_get_subqueue(n->nic, i);
-
-        qemu_purge_queued_packets(nc);
-
-        if (q->tx_timer) {
-            timer_del(q->tx_timer);
-            timer_free(q->tx_timer);
-        } else if (q->tx_bh) {
-            qemu_bh_delete(q->tx_bh);
-        }
+    max_queues = n->multiqueue ? n->max_queues : 1;
+    for (i = 0; i < max_queues; i++) {
+        virtio_net_del_queue(n, i);
     }
 
     timer_del(n->announce_timer);
@@ -1720,10 +1834,52 @@
                                   DEVICE(n), NULL);
 }
 
+VMSTATE_VIRTIO_DEVICE(net, VIRTIO_NET_VM_VERSION, virtio_net_load,
+                      virtio_net_save);
+
 static Property virtio_net_properties[] = {
+    DEFINE_PROP_BIT("csum", VirtIONet, host_features, VIRTIO_NET_F_CSUM, true),
+    DEFINE_PROP_BIT("guest_csum", VirtIONet, host_features,
+                    VIRTIO_NET_F_GUEST_CSUM, true),
+    DEFINE_PROP_BIT("gso", VirtIONet, host_features, VIRTIO_NET_F_GSO, true),
+    DEFINE_PROP_BIT("guest_tso4", VirtIONet, host_features,
+                    VIRTIO_NET_F_GUEST_TSO4, true),
+    DEFINE_PROP_BIT("guest_tso6", VirtIONet, host_features,
+                    VIRTIO_NET_F_GUEST_TSO6, true),
+    DEFINE_PROP_BIT("guest_ecn", VirtIONet, host_features,
+                    VIRTIO_NET_F_GUEST_ECN, true),
+    DEFINE_PROP_BIT("guest_ufo", VirtIONet, host_features,
+                    VIRTIO_NET_F_GUEST_UFO, true),
+    DEFINE_PROP_BIT("guest_announce", VirtIONet, host_features,
+                    VIRTIO_NET_F_GUEST_ANNOUNCE, true),
+    DEFINE_PROP_BIT("host_tso4", VirtIONet, host_features,
+                    VIRTIO_NET_F_HOST_TSO4, true),
+    DEFINE_PROP_BIT("host_tso6", VirtIONet, host_features,
+                    VIRTIO_NET_F_HOST_TSO6, true),
+    DEFINE_PROP_BIT("host_ecn", VirtIONet, host_features,
+                    VIRTIO_NET_F_HOST_ECN, true),
+    DEFINE_PROP_BIT("host_ufo", VirtIONet, host_features,
+                    VIRTIO_NET_F_HOST_UFO, true),
+    DEFINE_PROP_BIT("mrg_rxbuf", VirtIONet, host_features,
+                    VIRTIO_NET_F_MRG_RXBUF, true),
+    DEFINE_PROP_BIT("status", VirtIONet, host_features,
+                    VIRTIO_NET_F_STATUS, true),
+    DEFINE_PROP_BIT("ctrl_vq", VirtIONet, host_features,
+                    VIRTIO_NET_F_CTRL_VQ, true),
+    DEFINE_PROP_BIT("ctrl_rx", VirtIONet, host_features,
+                    VIRTIO_NET_F_CTRL_RX, true),
+    DEFINE_PROP_BIT("ctrl_vlan", VirtIONet, host_features,
+                    VIRTIO_NET_F_CTRL_VLAN, true),
+    DEFINE_PROP_BIT("ctrl_rx_extra", VirtIONet, host_features,
+                    VIRTIO_NET_F_CTRL_RX_EXTRA, true),
+    DEFINE_PROP_BIT("ctrl_mac_addr", VirtIONet, host_features,
+                    VIRTIO_NET_F_CTRL_MAC_ADDR, true),
+    DEFINE_PROP_BIT("ctrl_guest_offloads", VirtIONet, host_features,
+                    VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true),
+    DEFINE_PROP_BIT("mq", VirtIONet, host_features, VIRTIO_NET_F_MQ, false),
     DEFINE_NIC_PROPERTIES(VirtIONet, nic_conf),
     DEFINE_PROP_UINT32("x-txtimer", VirtIONet, net_conf.txtimer,
-                                               TX_TIMER_INTERVAL),
+                       TX_TIMER_INTERVAL),
     DEFINE_PROP_INT32("x-txburst", VirtIONet, net_conf.txburst, TX_BURST),
     DEFINE_PROP_STRING("tx", VirtIONet, net_conf.tx),
     DEFINE_PROP_END_OF_LIST(),
@@ -1735,6 +1891,7 @@
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 
     dc->props = virtio_net_properties;
+    dc->vmsd = &vmstate_virtio_net;
     set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
     vdc->realize = virtio_net_device_realize;
     vdc->unrealize = virtio_net_device_unrealize;
diff --git a/hw/net/vmware_utils.h b/hw/net/vmware_utils.h
index 1099df6..5500601 100644
--- a/hw/net/vmware_utils.h
+++ b/hw/net/vmware_utils.h
@@ -18,10 +18,7 @@
 #define VMWARE_UTILS_H
 
 #include "qemu/range.h"
-
-#ifndef VMW_SHPRN
-#define VMW_SHPRN(fmt, ...) do {} while (0)
-#endif
+#include "vmxnet_debug.h"
 
 /*
  * Shared memory access functions with byte swap support
@@ -29,97 +26,104 @@
  *
  */
 static inline void
-vmw_shmem_read(hwaddr addr, void *buf, int len)
+vmw_shmem_read(PCIDevice *d, hwaddr addr, void *buf, int len)
 {
     VMW_SHPRN("SHMEM r: %" PRIx64 ", len: %d to %p", addr, len, buf);
-    cpu_physical_memory_read(addr, buf, len);
+    pci_dma_read(d, addr, buf, len);
 }
 
 static inline void
-vmw_shmem_write(hwaddr addr, void *buf, int len)
+vmw_shmem_write(PCIDevice *d, hwaddr addr, void *buf, int len)
 {
     VMW_SHPRN("SHMEM w: %" PRIx64 ", len: %d to %p", addr, len, buf);
-    cpu_physical_memory_write(addr, buf, len);
+    pci_dma_write(d, addr, buf, len);
 }
 
 static inline void
-vmw_shmem_rw(hwaddr addr, void *buf, int len, int is_write)
+vmw_shmem_rw(PCIDevice *d, hwaddr addr, void *buf, int len, int is_write)
 {
     VMW_SHPRN("SHMEM r/w: %" PRIx64 ", len: %d (to %p), is write: %d",
               addr, len, buf, is_write);
 
-    cpu_physical_memory_rw(addr, buf, len, is_write);
+    if (is_write)
+        pci_dma_write(d, addr, buf, len);
+    else
+        pci_dma_read(d, addr, buf, len);
 }
 
 static inline void
-vmw_shmem_set(hwaddr addr, uint8 val, int len)
+vmw_shmem_set(PCIDevice *d, hwaddr addr, uint8_t val, int len)
 {
     int i;
     VMW_SHPRN("SHMEM set: %" PRIx64 ", len: %d (value 0x%X)", addr, len, val);
 
     for (i = 0; i < len; i++) {
-        cpu_physical_memory_write(addr + i, &val, 1);
+        pci_dma_write(d, addr + i, &val, 1);
     }
 }
 
 static inline uint32_t
-vmw_shmem_ld8(hwaddr addr)
+vmw_shmem_ld8(PCIDevice *d, hwaddr addr)
 {
-    uint8_t res = ldub_phys(&address_space_memory, addr);
+    uint8_t res;
+    pci_dma_read(d, addr, &res, 1);
     VMW_SHPRN("SHMEM load8: %" PRIx64 " (value 0x%X)", addr, res);
     return res;
 }
 
 static inline void
-vmw_shmem_st8(hwaddr addr, uint8_t value)
+vmw_shmem_st8(PCIDevice *d, hwaddr addr, uint8_t value)
 {
     VMW_SHPRN("SHMEM store8: %" PRIx64 " (value 0x%X)", addr, value);
-    stb_phys(&address_space_memory, addr, value);
+    pci_dma_write(d, addr, &value, 1);
 }
 
 static inline uint32_t
-vmw_shmem_ld16(hwaddr addr)
+vmw_shmem_ld16(PCIDevice *d, hwaddr addr)
 {
-    uint16_t res = lduw_le_phys(&address_space_memory, addr);
+    uint16_t res;
+    pci_dma_read(d, addr, &res, 2);
     VMW_SHPRN("SHMEM load16: %" PRIx64 " (value 0x%X)", addr, res);
     return res;
 }
 
 static inline void
-vmw_shmem_st16(hwaddr addr, uint16_t value)
+vmw_shmem_st16(PCIDevice *d, hwaddr addr, uint16_t value)
 {
     VMW_SHPRN("SHMEM store16: %" PRIx64 " (value 0x%X)", addr, value);
-    stw_le_phys(&address_space_memory, addr, value);
+    pci_dma_write(d, addr, &value, 2);
 }
 
 static inline uint32_t
-vmw_shmem_ld32(hwaddr addr)
+vmw_shmem_ld32(PCIDevice *d, hwaddr addr)
 {
-    uint32_t res = ldl_le_phys(&address_space_memory, addr);
+    uint32_t res;
+    pci_dma_read(d, addr, &res, 4);
     VMW_SHPRN("SHMEM load32: %" PRIx64 " (value 0x%X)", addr, res);
     return res;
 }
 
 static inline void
-vmw_shmem_st32(hwaddr addr, uint32_t value)
+vmw_shmem_st32(PCIDevice *d, hwaddr addr, uint32_t value)
 {
     VMW_SHPRN("SHMEM store32: %" PRIx64 " (value 0x%X)", addr, value);
-    stl_le_phys(&address_space_memory, addr, value);
+    pci_dma_write(d, addr, &value, 4);
 }
 
 static inline uint64_t
-vmw_shmem_ld64(hwaddr addr)
+vmw_shmem_ld64(PCIDevice *d, hwaddr addr)
 {
-    uint64_t res = ldq_le_phys(&address_space_memory, addr);
+    uint64_t res;
+    pci_dma_read(d, addr, &res, 8);
     VMW_SHPRN("SHMEM load64: %" PRIx64 " (value %" PRIx64 ")", addr, res);
     return res;
 }
 
 static inline void
-vmw_shmem_st64(hwaddr addr, uint64_t value)
+vmw_shmem_st64(PCIDevice *d, hwaddr addr, uint64_t value)
 {
     VMW_SHPRN("SHMEM store64: %" PRIx64 " (value %" PRIx64 ")", addr, value);
-    stq_le_phys(&address_space_memory, addr, value);
+    pci_dma_write(d, addr, &value, 8);
 }
 
 /* Macros for simplification of operations on array-style registers */
diff --git a/hw/net/vmxnet3.c b/hw/net/vmxnet3.c
index 31f266c..d67d83c 100644
--- a/hw/net/vmxnet3.c
+++ b/hw/net/vmxnet3.c
@@ -15,6 +15,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "net/net.h"
@@ -29,19 +30,35 @@
 #include "vmxnet3.h"
 #include "vmxnet_debug.h"
 #include "vmware_utils.h"
-#include "vmxnet_tx_pkt.h"
-#include "vmxnet_rx_pkt.h"
+#include "net_tx_pkt.h"
+#include "net_rx_pkt.h"
 
 #define PCI_DEVICE_ID_VMWARE_VMXNET3_REVISION 0x1
 #define VMXNET3_MSIX_BAR_SIZE 0x2000
 #define MIN_BUF_SIZE 60
 
+/* Compatibility flags for migration */
+#define VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS_BIT 0
+#define VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS \
+    (1 << VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS_BIT)
+#define VMXNET3_COMPAT_FLAG_DISABLE_PCIE_BIT 1
+#define VMXNET3_COMPAT_FLAG_DISABLE_PCIE \
+    (1 << VMXNET3_COMPAT_FLAG_DISABLE_PCIE_BIT)
+
+#define VMXNET3_EXP_EP_OFFSET (0x48)
+#define VMXNET3_MSI_OFFSET(s) \
+    ((s)->compat_flags & VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS ? 0x50 : 0x84)
+#define VMXNET3_MSIX_OFFSET(s) \
+    ((s)->compat_flags & VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS ? 0 : 0x9c)
+#define VMXNET3_DSN_OFFSET     (0x100)
+
 #define VMXNET3_BAR0_IDX      (0)
 #define VMXNET3_BAR1_IDX      (1)
 #define VMXNET3_MSIX_BAR_IDX  (2)
 
 #define VMXNET3_OFF_MSIX_TABLE (0x000)
-#define VMXNET3_OFF_MSIX_PBA   (0x800)
+#define VMXNET3_OFF_MSIX_PBA(s) \
+    ((s)->compat_flags & VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS ? 0x800 : 0x1000)
 
 /* Link speed in Mbps should be shifted by 16 */
 #define VMXNET3_LINK_SPEED      (1000 << 16)
@@ -50,67 +67,77 @@
 #define VMXNET3_LINK_STATUS_UP  0x1
 
 /* Least significant bit should be set for revision and version */
-#define VMXNET3_DEVICE_VERSION    0x1
+#define VMXNET3_UPT_REVISION      0x1
 #define VMXNET3_DEVICE_REVISION   0x1
 
 /* Number of interrupt vectors for non-MSIx modes */
 #define VMXNET3_MAX_NMSIX_INTRS   (1)
 
 /* Macros for rings descriptors access */
-#define VMXNET3_READ_TX_QUEUE_DESCR8(dpa, field) \
-    (vmw_shmem_ld8(dpa + offsetof(struct Vmxnet3_TxQueueDesc, field)))
+#define VMXNET3_READ_TX_QUEUE_DESCR8(_d, dpa, field) \
+    (vmw_shmem_ld8(_d, dpa + offsetof(struct Vmxnet3_TxQueueDesc, field)))
 
-#define VMXNET3_WRITE_TX_QUEUE_DESCR8(dpa, field, value) \
-    (vmw_shmem_st8(dpa + offsetof(struct Vmxnet3_TxQueueDesc, field, value)))
+#define VMXNET3_WRITE_TX_QUEUE_DESCR8(_d, dpa, field, value) \
+    (vmw_shmem_st8(_d, dpa + offsetof(struct Vmxnet3_TxQueueDesc, field, value)))
 
-#define VMXNET3_READ_TX_QUEUE_DESCR32(dpa, field) \
-    (vmw_shmem_ld32(dpa + offsetof(struct Vmxnet3_TxQueueDesc, field)))
+#define VMXNET3_READ_TX_QUEUE_DESCR32(_d, dpa, field) \
+    (vmw_shmem_ld32(_d, dpa + offsetof(struct Vmxnet3_TxQueueDesc, field)))
 
-#define VMXNET3_WRITE_TX_QUEUE_DESCR32(dpa, field, value) \
-    (vmw_shmem_st32(dpa + offsetof(struct Vmxnet3_TxQueueDesc, field), value))
+#define VMXNET3_WRITE_TX_QUEUE_DESCR32(_d, dpa, field, value) \
+    (vmw_shmem_st32(_d, dpa + offsetof(struct Vmxnet3_TxQueueDesc, field), value))
 
-#define VMXNET3_READ_TX_QUEUE_DESCR64(dpa, field) \
-    (vmw_shmem_ld64(dpa + offsetof(struct Vmxnet3_TxQueueDesc, field)))
+#define VMXNET3_READ_TX_QUEUE_DESCR64(_d, dpa, field) \
+    (vmw_shmem_ld64(_d, dpa + offsetof(struct Vmxnet3_TxQueueDesc, field)))
 
-#define VMXNET3_WRITE_TX_QUEUE_DESCR64(dpa, field, value) \
-    (vmw_shmem_st64(dpa + offsetof(struct Vmxnet3_TxQueueDesc, field), value))
+#define VMXNET3_WRITE_TX_QUEUE_DESCR64(_d, dpa, field, value) \
+    (vmw_shmem_st64(_d, dpa + offsetof(struct Vmxnet3_TxQueueDesc, field), value))
 
-#define VMXNET3_READ_RX_QUEUE_DESCR64(dpa, field) \
-    (vmw_shmem_ld64(dpa + offsetof(struct Vmxnet3_RxQueueDesc, field)))
+#define VMXNET3_READ_RX_QUEUE_DESCR64(_d, dpa, field) \
+    (vmw_shmem_ld64(_d, dpa + offsetof(struct Vmxnet3_RxQueueDesc, field)))
 
-#define VMXNET3_READ_RX_QUEUE_DESCR32(dpa, field) \
-    (vmw_shmem_ld32(dpa + offsetof(struct Vmxnet3_RxQueueDesc, field)))
+#define VMXNET3_READ_RX_QUEUE_DESCR32(_d, dpa, field) \
+    (vmw_shmem_ld32(_d, dpa + offsetof(struct Vmxnet3_RxQueueDesc, field)))
 
-#define VMXNET3_WRITE_RX_QUEUE_DESCR64(dpa, field, value) \
-    (vmw_shmem_st64(dpa + offsetof(struct Vmxnet3_RxQueueDesc, field), value))
+#define VMXNET3_WRITE_RX_QUEUE_DESCR64(_d, dpa, field, value) \
+    (vmw_shmem_st64(_d, dpa + offsetof(struct Vmxnet3_RxQueueDesc, field), value))
 
-#define VMXNET3_WRITE_RX_QUEUE_DESCR8(dpa, field, value) \
-    (vmw_shmem_st8(dpa + offsetof(struct Vmxnet3_RxQueueDesc, field), value))
+#define VMXNET3_WRITE_RX_QUEUE_DESCR8(_d, dpa, field, value) \
+    (vmw_shmem_st8(_d, dpa + offsetof(struct Vmxnet3_RxQueueDesc, field), value))
 
 /* Macros for guest driver shared area access */
-#define VMXNET3_READ_DRV_SHARED64(shpa, field) \
-    (vmw_shmem_ld64(shpa + offsetof(struct Vmxnet3_DriverShared, field)))
+#define VMXNET3_READ_DRV_SHARED64(_d, shpa, field) \
+    (vmw_shmem_ld64(_d, shpa + offsetof(struct Vmxnet3_DriverShared, field)))
 
-#define VMXNET3_READ_DRV_SHARED32(shpa, field) \
-    (vmw_shmem_ld32(shpa + offsetof(struct Vmxnet3_DriverShared, field)))
+#define VMXNET3_READ_DRV_SHARED32(_d, shpa, field) \
+    (vmw_shmem_ld32(_d, shpa + offsetof(struct Vmxnet3_DriverShared, field)))
 
-#define VMXNET3_WRITE_DRV_SHARED32(shpa, field, val) \
-    (vmw_shmem_st32(shpa + offsetof(struct Vmxnet3_DriverShared, field), val))
+#define VMXNET3_WRITE_DRV_SHARED32(_d, shpa, field, val) \
+    (vmw_shmem_st32(_d, shpa + offsetof(struct Vmxnet3_DriverShared, field), val))
 
-#define VMXNET3_READ_DRV_SHARED16(shpa, field) \
-    (vmw_shmem_ld16(shpa + offsetof(struct Vmxnet3_DriverShared, field)))
+#define VMXNET3_READ_DRV_SHARED16(_d, shpa, field) \
+    (vmw_shmem_ld16(_d, shpa + offsetof(struct Vmxnet3_DriverShared, field)))
 
-#define VMXNET3_READ_DRV_SHARED8(shpa, field) \
-    (vmw_shmem_ld8(shpa + offsetof(struct Vmxnet3_DriverShared, field)))
+#define VMXNET3_READ_DRV_SHARED8(_d, shpa, field) \
+    (vmw_shmem_ld8(_d, shpa + offsetof(struct Vmxnet3_DriverShared, field)))
 
-#define VMXNET3_READ_DRV_SHARED(shpa, field, b, l) \
-    (vmw_shmem_read(shpa + offsetof(struct Vmxnet3_DriverShared, field), b, l))
+#define VMXNET3_READ_DRV_SHARED(_d, shpa, field, b, l) \
+    (vmw_shmem_read(_d, shpa + offsetof(struct Vmxnet3_DriverShared, field), b, l))
 
 #define VMXNET_FLAG_IS_SET(field, flag) (((field) & (flag)) == (flag))
 
+typedef struct VMXNET3Class {
+    PCIDeviceClass parent_class;
+    DeviceRealize parent_dc_realize;
+} VMXNET3Class;
+
 #define TYPE_VMXNET3 "vmxnet3"
 #define VMXNET3(obj) OBJECT_CHECK(VMXNET3State, (obj), TYPE_VMXNET3)
 
+#define VMXNET3_DEVICE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(VMXNET3Class, (klass), TYPE_VMXNET3)
+#define VMXNET3_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(VMXNET3Class, (obj), TYPE_VMXNET3)
+
 /* Cyclic ring abstraction */
 typedef struct {
     hwaddr pa;
@@ -120,7 +147,8 @@
     uint8_t gen;
 } Vmxnet3Ring;
 
-static inline void vmxnet3_ring_init(Vmxnet3Ring *ring,
+static inline void vmxnet3_ring_init(PCIDevice *d,
+				     Vmxnet3Ring *ring,
                                      hwaddr pa,
                                      size_t size,
                                      size_t cell_size,
@@ -133,12 +161,12 @@
     ring->next = 0;
 
     if (zero_region) {
-        vmw_shmem_set(pa, 0, size * cell_size);
+        vmw_shmem_set(d, pa, 0, size * cell_size);
     }
 }
 
 #define VMXNET3_RING_DUMP(macro, ring_name, ridx, r)                         \
-    macro("%s#%d: base %" PRIx64 " size %lu cell_size %lu gen %d next %lu",  \
+    macro("%s#%d: base %" PRIx64 " size %zu cell_size %zu gen %d next %zu",  \
           (ring_name), (ridx),                                               \
           (r)->pa, (r)->size, (r)->cell_size, (r)->gen, (r)->next)
 
@@ -163,14 +191,16 @@
     return ring->pa + ring->next * ring->cell_size;
 }
 
-static inline void vmxnet3_ring_read_curr_cell(Vmxnet3Ring *ring, void *buff)
+static inline void vmxnet3_ring_read_curr_cell(PCIDevice *d, Vmxnet3Ring *ring,
+					       void *buff)
 {
-    vmw_shmem_read(vmxnet3_ring_curr_cell_pa(ring), buff, ring->cell_size);
+    vmw_shmem_read(d, vmxnet3_ring_curr_cell_pa(ring), buff, ring->cell_size);
 }
 
-static inline void vmxnet3_ring_write_curr_cell(Vmxnet3Ring *ring, void *buff)
+static inline void vmxnet3_ring_write_curr_cell(PCIDevice *d, Vmxnet3Ring *ring,
+						void *buff)
 {
-    vmw_shmem_write(vmxnet3_ring_curr_cell_pa(ring), buff, ring->cell_size);
+    vmw_shmem_write(d, vmxnet3_ring_curr_cell_pa(ring), buff, ring->cell_size);
 }
 
 static inline size_t vmxnet3_ring_curr_cell_idx(Vmxnet3Ring *ring)
@@ -253,8 +283,6 @@
 
         /* Whether MSI-X support was installed successfully */
         bool msix_used;
-        /* Whether MSI support was installed successfully */
-        bool msi_used;
         hwaddr drv_shmem;
         hwaddr temp_shared_guest_driver_memory;
 
@@ -287,13 +315,13 @@
         bool peer_has_vhdr;
 
         /* TX packets to QEMU interface */
-        struct VmxnetTxPkt *tx_pkt;
+        struct NetTxPkt *tx_pkt;
         uint32_t offload_mode;
         uint32_t cso_or_gso_size;
         uint16_t tci;
         bool needs_vlan;
 
-        struct VmxnetRxPkt *rx_pkt;
+        struct NetRxPkt *rx_pkt;
 
         bool tx_sop;
         bool skip_current_tx_pkt;
@@ -313,12 +341,15 @@
         MACAddr *mcast_list;
         uint32_t mcast_list_len;
         uint32_t mcast_list_buff_size; /* needed for live migration. */
+
+        /* Compatibility flags for migration */
+        uint32_t compat_flags;
 } VMXNET3State;
 
 /* Interrupt management */
 
 /*
- *This function returns sign whether interrupt line is in asserted state
+ * This function returns sign whether interrupt line is in asserted state
  * This depends on the type of interrupt used. For INTX interrupt line will
  * be asserted until explicit deassertion, for MSI(X) interrupt line will
  * be deasserted automatically due to notification semantics of the MSI(X)
@@ -333,7 +364,7 @@
         msix_notify(d, int_idx);
         return false;
     }
-    if (s->msi_used && msi_enabled(d)) {
+    if (msi_enabled(d)) {
         VMW_IRPRN("Sending MSI notification for vector %u", int_idx);
         msi_notify(d, int_idx);
         return false;
@@ -357,7 +388,7 @@
      * This function should never be called for MSI(X) interrupts
      * because deassertion never required for message interrupts
      */
-    assert(!s->msi_used || !msi_enabled(d));
+    assert(!msi_enabled(d));
 
     VMW_IRPRN("Deasserting line for interrupt %u", lidx);
     pci_irq_deassert(d);
@@ -394,7 +425,7 @@
         goto do_automask;
     }
 
-    if (s->msi_used && msi_enabled(d) && s->auto_int_masking) {
+    if (msi_enabled(d) && s->auto_int_masking) {
         goto do_automask;
     }
 
@@ -426,9 +457,9 @@
     vmxnet3_update_interrupt_line_state(s, lidx);
 }
 
-static bool vmxnet3_verify_driver_magic(hwaddr dshmem)
+static bool vmxnet3_verify_driver_magic(PCIDevice *d, hwaddr dshmem)
 {
-    return (VMXNET3_READ_DRV_SHARED32(dshmem, magic) == VMXNET3_REV1_MAGIC);
+    return (VMXNET3_READ_DRV_SHARED32(d, dshmem, magic) == VMXNET3_REV1_MAGIC);
 }
 
 #define VMXNET3_GET_BYTE(x, byte_num) (((x) >> (byte_num)*8) & 0xFF)
@@ -444,7 +475,7 @@
     s->conf.macaddr.a[4] = VMXNET3_GET_BYTE(h, 0);
     s->conf.macaddr.a[5] = VMXNET3_GET_BYTE(h, 1);
 
-    VMW_CFPRN("Variable MAC: " VMXNET_MF, VMXNET_MA(s->conf.macaddr.a));
+    VMW_CFPRN("Variable MAC: " MAC_FMT, MAC_ARG(s->conf.macaddr.a));
 
     qemu_format_nic_info_str(qemu_get_queue(s->nic), s->conf.macaddr.a);
 }
@@ -493,16 +524,17 @@
     vmxnet3_ring_dec(&s->rxq_descr[qidx].comp_ring);
 }
 
-static void vmxnet3_complete_packet(VMXNET3State *s, int qidx, uint32 tx_ridx)
+static void vmxnet3_complete_packet(VMXNET3State *s, int qidx, uint32_t tx_ridx)
 {
     struct Vmxnet3_TxCompDesc txcq_descr;
+    PCIDevice *d = PCI_DEVICE(s);
 
     VMXNET3_RING_DUMP(VMW_RIPRN, "TXC", qidx, &s->txq_descr[qidx].comp_ring);
 
     txcq_descr.txdIdx = tx_ridx;
     txcq_descr.gen = vmxnet3_ring_curr_gen(&s->txq_descr[qidx].comp_ring);
 
-    vmxnet3_ring_write_curr_cell(&s->txq_descr[qidx].comp_ring, &txcq_descr);
+    vmxnet3_ring_write_curr_cell(d, &s->txq_descr[qidx].comp_ring, &txcq_descr);
 
     /* Flush changes in TX descriptor before changing the counter value */
     smp_wmb();
@@ -516,18 +548,18 @@
 {
     switch (s->offload_mode) {
     case VMXNET3_OM_NONE:
-        vmxnet_tx_pkt_build_vheader(s->tx_pkt, false, false, 0);
+        net_tx_pkt_build_vheader(s->tx_pkt, false, false, 0);
         break;
 
     case VMXNET3_OM_CSUM:
-        vmxnet_tx_pkt_build_vheader(s->tx_pkt, false, true, 0);
+        net_tx_pkt_build_vheader(s->tx_pkt, false, true, 0);
         VMW_PKPRN("L4 CSO requested\n");
         break;
 
     case VMXNET3_OM_TSO:
-        vmxnet_tx_pkt_build_vheader(s->tx_pkt, true, true,
+        net_tx_pkt_build_vheader(s->tx_pkt, true, true,
             s->cso_or_gso_size);
-        vmxnet_tx_pkt_update_ip_checksums(s->tx_pkt);
+        net_tx_pkt_update_ip_checksums(s->tx_pkt);
         VMW_PKPRN("GSO offload requested.");
         break;
 
@@ -560,12 +592,12 @@
 vmxnet3_on_tx_done_update_stats(VMXNET3State *s, int qidx,
     Vmxnet3PktStatus status)
 {
-    size_t tot_len = vmxnet_tx_pkt_get_total_len(s->tx_pkt);
+    size_t tot_len = net_tx_pkt_get_total_len(s->tx_pkt);
     struct UPT1_TxStats *stats = &s->txq_descr[qidx].txq_stats;
 
     switch (status) {
     case VMXNET3_PKT_STATUS_OK:
-        switch (vmxnet_tx_pkt_get_packet_type(s->tx_pkt)) {
+        switch (net_tx_pkt_get_packet_type(s->tx_pkt)) {
         case ETH_PKT_BCAST:
             stats->bcastPktsTxOK++;
             stats->bcastBytesTxOK += tot_len;
@@ -613,7 +645,7 @@
                                 Vmxnet3PktStatus status)
 {
     struct UPT1_RxStats *stats = &s->rxq_descr[qidx].rxq_stats;
-    size_t tot_len = vmxnet_rx_pkt_get_total_len(s->rx_pkt);
+    size_t tot_len = net_rx_pkt_get_total_len(s->rx_pkt);
 
     switch (status) {
     case VMXNET3_PKT_STATUS_OUT_OF_BUF:
@@ -624,7 +656,7 @@
         stats->pktsRxError++;
         break;
     case VMXNET3_PKT_STATUS_OK:
-        switch (vmxnet_rx_pkt_get_packet_type(s->rx_pkt)) {
+        switch (net_rx_pkt_get_packet_type(s->rx_pkt)) {
         case ETH_PKT_BCAST:
             stats->bcastPktsRxOK++;
             stats->bcastBytesRxOK += tot_len;
@@ -658,13 +690,14 @@
                           uint32_t *descr_idx)
 {
     Vmxnet3Ring *ring = &s->txq_descr[qidx].tx_ring;
+    PCIDevice *d = PCI_DEVICE(s);
 
-    vmxnet3_ring_read_curr_cell(ring, txd);
+    vmxnet3_ring_read_curr_cell(d, ring, txd);
     if (txd->gen == vmxnet3_ring_curr_gen(ring)) {
         /* Only read after generation field verification */
         smp_rmb();
         /* Re-read to be sure we got the latest version */
-        vmxnet3_ring_read_curr_cell(ring, txd);
+        vmxnet3_ring_read_curr_cell(d, ring, txd);
         VMXNET3_RING_DUMP(VMW_RIPRN, "TX", qidx, ring);
         *descr_idx = vmxnet3_ring_curr_cell_idx(ring);
         vmxnet3_inc_tx_consumption_counter(s, qidx);
@@ -685,10 +718,10 @@
     }
 
     /* debug prints */
-    vmxnet3_dump_virt_hdr(vmxnet_tx_pkt_get_vhdr(s->tx_pkt));
-    vmxnet_tx_pkt_dump(s->tx_pkt);
+    vmxnet3_dump_virt_hdr(net_tx_pkt_get_vhdr(s->tx_pkt));
+    net_tx_pkt_dump(s->tx_pkt);
 
-    if (!vmxnet_tx_pkt_send(s->tx_pkt, qemu_get_queue(s->nic))) {
+    if (!net_tx_pkt_send(s->tx_pkt, qemu_get_queue(s->nic))) {
         status = VMXNET3_PKT_STATUS_DISCARD;
         goto func_exit;
     }
@@ -716,7 +749,7 @@
             data_len = (txd.len > 0) ? txd.len : VMXNET3_MAX_TX_BUF_SIZE;
             data_pa = le64_to_cpu(txd.addr);
 
-            if (!vmxnet_tx_pkt_add_raw_fragment(s->tx_pkt,
+            if (!net_tx_pkt_add_raw_fragment(s->tx_pkt,
                                                 data_pa,
                                                 data_len)) {
                 s->skip_current_tx_pkt = true;
@@ -729,11 +762,9 @@
         }
 
         if (txd.eop) {
-            if (!s->skip_current_tx_pkt) {
-                vmxnet_tx_pkt_parse(s->tx_pkt);
-
+            if (!s->skip_current_tx_pkt && net_tx_pkt_parse(s->tx_pkt)) {
                 if (s->needs_vlan) {
-                    vmxnet_tx_pkt_setup_vlan_header(s->tx_pkt, s->tci);
+                    net_tx_pkt_setup_vlan_header(s->tx_pkt, s->tci);
                 }
 
                 vmxnet3_send_packet(s, qidx);
@@ -745,7 +776,7 @@
             vmxnet3_complete_packet(s, qidx, txd_idx);
             s->tx_sop = true;
             s->skip_current_tx_pkt = false;
-            vmxnet_tx_pkt_reset(s->tx_pkt);
+            net_tx_pkt_reset(s->tx_pkt);
         }
     }
 }
@@ -754,9 +785,11 @@
 vmxnet3_read_next_rx_descr(VMXNET3State *s, int qidx, int ridx,
                            struct Vmxnet3_RxDesc *dbuf, uint32_t *didx)
 {
+    PCIDevice *d = PCI_DEVICE(s);
+
     Vmxnet3Ring *ring = &s->rxq_descr[qidx].rx_ring[ridx];
     *didx = vmxnet3_ring_curr_cell_idx(ring);
-    vmxnet3_ring_read_curr_cell(ring, dbuf);
+    vmxnet3_ring_read_curr_cell(d, ring, dbuf);
 }
 
 static inline uint8_t
@@ -774,7 +807,8 @@
     hwaddr daddr =
         vmxnet3_ring_curr_cell_pa(&s->rxq_descr[qidx].comp_ring);
 
-    cpu_physical_memory_read(daddr, &rxcd, sizeof(struct Vmxnet3_RxCompDesc));
+    pci_dma_read(PCI_DEVICE(s),
+                 daddr, &rxcd, sizeof(struct Vmxnet3_RxCompDesc));
     ring_gen = vmxnet3_ring_curr_gen(&s->rxq_descr[qidx].comp_ring);
 
     if (rxcd.gen != ring_gen) {
@@ -885,7 +919,64 @@
     }
 }
 
-static void vmxnet3_rx_update_descr(struct VmxnetRxPkt *pkt,
+/* In case packet was csum offloaded (either NEEDS_CSUM or DATA_VALID),
+ * the implementation always passes an RxCompDesc with a "Checksum
+ * calculated and found correct" to the OS (cnc=0 and tuc=1, see
+ * vmxnet3_rx_update_descr). This emulates the observed ESXi behavior.
+ *
+ * Therefore, if packet has the NEEDS_CSUM set, we must calculate
+ * and place a fully computed checksum into the tcp/udp header.
+ * Otherwise, the OS driver will receive a checksum-correct indication
+ * (CHECKSUM_UNNECESSARY), but with the actual tcp/udp checksum field
+ * having just the pseudo header csum value.
+ *
+ * While this is not a problem if packet is destined for local delivery,
+ * in the case the host OS performs forwarding, it will forward an
+ * incorrectly checksummed packet.
+ */
+static void vmxnet3_rx_need_csum_calculate(struct NetRxPkt *pkt,
+                                           const void *pkt_data,
+                                           size_t pkt_len)
+{
+    struct virtio_net_hdr *vhdr;
+    bool isip4, isip6, istcp, isudp;
+    uint8_t *data;
+    int len;
+
+    if (!net_rx_pkt_has_virt_hdr(pkt)) {
+        return;
+    }
+
+    vhdr = net_rx_pkt_get_vhdr(pkt);
+    if (!VMXNET_FLAG_IS_SET(vhdr->flags, VIRTIO_NET_HDR_F_NEEDS_CSUM)) {
+        return;
+    }
+
+    net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
+    if (!(isip4 || isip6) || !(istcp || isudp)) {
+        return;
+    }
+
+    vmxnet3_dump_virt_hdr(vhdr);
+
+    /* Validate packet len: csum_start + scum_offset + length of csum field */
+    if (pkt_len < (vhdr->csum_start + vhdr->csum_offset + 2)) {
+        VMW_PKPRN("packet len:%zu < csum_start(%d) + csum_offset(%d) + 2, "
+                  "cannot calculate checksum",
+                  pkt_len, vhdr->csum_start, vhdr->csum_offset);
+        return;
+    }
+
+    data = (uint8_t *)pkt_data + vhdr->csum_start;
+    len = pkt_len - vhdr->csum_start;
+    /* Put the checksum obtained into the packet */
+    stw_be_p(data + vhdr->csum_offset, net_raw_checksum(data, len));
+
+    vhdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
+    vhdr->flags |= VIRTIO_NET_HDR_F_DATA_VALID;
+}
+
+static void vmxnet3_rx_update_descr(struct NetRxPkt *pkt,
     struct Vmxnet3_RxCompDesc *rxcd)
 {
     int csum_ok, is_gso;
@@ -893,16 +984,16 @@
     struct virtio_net_hdr *vhdr;
     uint8_t offload_type;
 
-    if (vmxnet_rx_pkt_is_vlan_stripped(pkt)) {
+    if (net_rx_pkt_is_vlan_stripped(pkt)) {
         rxcd->ts = 1;
-        rxcd->tci = vmxnet_rx_pkt_get_vlan_tag(pkt);
+        rxcd->tci = net_rx_pkt_get_vlan_tag(pkt);
     }
 
-    if (!vmxnet_rx_pkt_has_virt_hdr(pkt)) {
+    if (!net_rx_pkt_has_virt_hdr(pkt)) {
         goto nocsum;
     }
 
-    vhdr = vmxnet_rx_pkt_get_vhdr(pkt);
+    vhdr = net_rx_pkt_get_vhdr(pkt);
     /*
      * Checksum is valid when lower level tell so or when lower level
      * requires checksum offload telling that packet produced/bridged
@@ -919,7 +1010,7 @@
         goto nocsum;
     }
 
-    vmxnet_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
+    net_rx_pkt_get_protocols(pkt, &isip4, &isip6, &isudp, &istcp);
     if ((!istcp && !isudp) || (!isip4 && !isip6)) {
         goto nocsum;
     }
@@ -938,10 +1029,11 @@
 }
 
 static void
-vmxnet3_physical_memory_writev(const struct iovec *iov,
-                               size_t start_iov_off,
-                               hwaddr target_addr,
-                               size_t bytes_to_copy)
+vmxnet3_pci_dma_writev(PCIDevice *pci_dev,
+                       const struct iovec *iov,
+                       size_t start_iov_off,
+                       hwaddr target_addr,
+                       size_t bytes_to_copy)
 {
     size_t curr_off = 0;
     size_t copied = 0;
@@ -951,9 +1043,9 @@
             size_t chunk_len =
                 MIN((curr_off + iov->iov_len) - start_iov_off, bytes_to_copy);
 
-            cpu_physical_memory_write(target_addr + copied,
-                                      iov->iov_base + start_iov_off - curr_off,
-                                      chunk_len);
+            pci_dma_write(pci_dev, target_addr + copied,
+                          iov->iov_base + start_iov_off - curr_off,
+                          chunk_len);
 
             copied += chunk_len;
             start_iov_off += chunk_len;
@@ -970,6 +1062,7 @@
 vmxnet3_indicate_packet(VMXNET3State *s)
 {
     struct Vmxnet3_RxDesc rxd;
+    PCIDevice *d = PCI_DEVICE(s);
     bool is_head = true;
     uint32_t rxd_idx;
     uint32_t rx_ridx = 0;
@@ -978,13 +1071,13 @@
     uint32_t new_rxcd_gen = VMXNET3_INIT_GEN;
     hwaddr new_rxcd_pa = 0;
     hwaddr ready_rxcd_pa = 0;
-    struct iovec *data = vmxnet_rx_pkt_get_iovec(s->rx_pkt);
+    struct iovec *data = net_rx_pkt_get_iovec(s->rx_pkt);
     size_t bytes_copied = 0;
-    size_t bytes_left = vmxnet_rx_pkt_get_total_len(s->rx_pkt);
+    size_t bytes_left = net_rx_pkt_get_total_len(s->rx_pkt);
     uint16_t num_frags = 0;
     size_t chunk_size;
 
-    vmxnet_rx_pkt_dump(s->rx_pkt);
+    net_rx_pkt_dump(s->rx_pkt);
 
     while (bytes_left > 0) {
 
@@ -1003,15 +1096,15 @@
         }
 
         chunk_size = MIN(bytes_left, rxd.len);
-        vmxnet3_physical_memory_writev(data, bytes_copied,
-                                       le64_to_cpu(rxd.addr), chunk_size);
+        vmxnet3_pci_dma_writev(d, data, bytes_copied,
+                               le64_to_cpu(rxd.addr), chunk_size);
         bytes_copied += chunk_size;
         bytes_left -= chunk_size;
 
         vmxnet3_dump_rx_descr(&rxd);
 
         if (ready_rxcd_pa != 0) {
-            cpu_physical_memory_write(ready_rxcd_pa, &rxcd, sizeof(rxcd));
+            pci_dma_write(d, ready_rxcd_pa, &rxcd, sizeof(rxcd));
         }
 
         memset(&rxcd, 0, sizeof(struct Vmxnet3_RxCompDesc));
@@ -1042,7 +1135,8 @@
     if (ready_rxcd_pa != 0) {
         rxcd.eop = 1;
         rxcd.err = (bytes_left != 0);
-        cpu_physical_memory_write(ready_rxcd_pa, &rxcd, sizeof(rxcd));
+
+        pci_dma_write(d, ready_rxcd_pa, &rxcd, sizeof(rxcd));
 
         /* Flush RX descriptor changes */
         smp_wmb();
@@ -1073,6 +1167,10 @@
 {
     VMXNET3State *s = opaque;
 
+    if (!s->device_active) {
+        return;
+    }
+
     if (VMW_IS_MULTIREG_ADDR(addr, VMXNET3_REG_TXPROD,
                         VMXNET3_DEVICE_MAX_TX_QUEUES, VMXNET3_REG_ALIGN)) {
         int tx_queue_idx =
@@ -1108,9 +1206,13 @@
 static uint64_t
 vmxnet3_io_bar0_read(void *opaque, hwaddr addr, unsigned size)
 {
+    VMXNET3State *s = opaque;
+
     if (VMW_IS_MULTIREG_ADDR(addr, VMXNET3_REG_IMR,
                         VMXNET3_MAX_INTRS, VMXNET3_REG_ALIGN)) {
-        g_assert_not_reached();
+        int l = VMW_MULTIREG_IDX_BY_ADDR(addr, VMXNET3_REG_IMR,
+                                         VMXNET3_REG_ALIGN);
+        return s->interrupt_states[l].is_masked;
     }
 
     VMW_CBPRN("BAR0 unknown read [%" PRIx64 "], size %d", addr, size);
@@ -1130,13 +1232,18 @@
 static void vmxnet3_reset_mac(VMXNET3State *s)
 {
     memcpy(&s->conf.macaddr.a, &s->perm_mac.a, sizeof(s->perm_mac.a));
-    VMW_CFPRN("MAC address set to: " VMXNET_MF, VMXNET_MA(s->conf.macaddr.a));
+    VMW_CFPRN("MAC address set to: " MAC_FMT, MAC_ARG(s->conf.macaddr.a));
 }
 
 static void vmxnet3_deactivate_device(VMXNET3State *s)
 {
-    VMW_CBPRN("Deactivating vmxnet3...");
-    s->device_active = false;
+    if (s->device_active) {
+        VMW_CBPRN("Deactivating vmxnet3...");
+        net_tx_pkt_reset(s->tx_pkt);
+        net_tx_pkt_uninit(s->tx_pkt);
+        net_rx_pkt_uninit(s->rx_pkt);
+        s->device_active = false;
+    }
 }
 
 static void vmxnet3_reset(VMXNET3State *s)
@@ -1145,7 +1252,6 @@
 
     vmxnet3_deactivate_device(s);
     vmxnet3_reset_interrupt_states(s);
-    vmxnet_tx_pkt_reset(s->tx_pkt);
     s->drv_shmem = 0;
     s->tx_sop = true;
     s->skip_current_tx_pkt = false;
@@ -1153,7 +1259,9 @@
 
 static void vmxnet3_update_rx_mode(VMXNET3State *s)
 {
-    s->rx_mode = VMXNET3_READ_DRV_SHARED32(s->drv_shmem,
+    PCIDevice *d = PCI_DEVICE(s);
+
+    s->rx_mode = VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem,
                                            devRead.rxFilterConf.rxMode);
     VMW_CFPRN("RX mode: 0x%08X", s->rx_mode);
 }
@@ -1161,9 +1269,10 @@
 static void vmxnet3_update_vlan_filters(VMXNET3State *s)
 {
     int i;
+    PCIDevice *d = PCI_DEVICE(s);
 
     /* Copy configuration from shared memory */
-    VMXNET3_READ_DRV_SHARED(s->drv_shmem,
+    VMXNET3_READ_DRV_SHARED(d, s->drv_shmem,
                             devRead.rxFilterConf.vfTable,
                             s->vlan_table,
                             sizeof(s->vlan_table));
@@ -1184,8 +1293,10 @@
 
 static void vmxnet3_update_mcast_filters(VMXNET3State *s)
 {
+    PCIDevice *d = PCI_DEVICE(s);
+
     uint16_t list_bytes =
-        VMXNET3_READ_DRV_SHARED16(s->drv_shmem,
+        VMXNET3_READ_DRV_SHARED16(d, s->drv_shmem,
                                   devRead.rxFilterConf.mfTableLen);
 
     s->mcast_list_len = list_bytes / sizeof(s->mcast_list[0]);
@@ -1202,13 +1313,14 @@
     } else {
         int i;
         hwaddr mcast_list_pa =
-            VMXNET3_READ_DRV_SHARED64(s->drv_shmem,
+            VMXNET3_READ_DRV_SHARED64(d, s->drv_shmem,
                                       devRead.rxFilterConf.mfTablePA);
 
-        cpu_physical_memory_read(mcast_list_pa, s->mcast_list, list_bytes);
+        pci_dma_read(d, mcast_list_pa, s->mcast_list, list_bytes);
+
         VMW_CFPRN("Current multicast list len is %d:", s->mcast_list_len);
         for (i = 0; i < s->mcast_list_len; i++) {
-            VMW_CFPRN("\t" VMXNET_MF, VMXNET_MA(s->mcast_list[i].a));
+            VMW_CFPRN("\t" MAC_FMT, MAC_ARG(s->mcast_list[i].a));
         }
     }
 }
@@ -1230,24 +1342,32 @@
 static void vmxnet3_fill_stats(VMXNET3State *s)
 {
     int i;
+    PCIDevice *d = PCI_DEVICE(s);
+
+    if (!s->device_active)
+        return;
+
     for (i = 0; i < s->txq_num; i++) {
-        cpu_physical_memory_write(s->txq_descr[i].tx_stats_pa,
-                                  &s->txq_descr[i].txq_stats,
-                                  sizeof(s->txq_descr[i].txq_stats));
+        pci_dma_write(d,
+                      s->txq_descr[i].tx_stats_pa,
+                      &s->txq_descr[i].txq_stats,
+                      sizeof(s->txq_descr[i].txq_stats));
     }
 
     for (i = 0; i < s->rxq_num; i++) {
-        cpu_physical_memory_write(s->rxq_descr[i].rx_stats_pa,
-                                  &s->rxq_descr[i].rxq_stats,
-                                  sizeof(s->rxq_descr[i].rxq_stats));
+        pci_dma_write(d,
+                      s->rxq_descr[i].rx_stats_pa,
+                      &s->rxq_descr[i].rxq_stats,
+                      sizeof(s->rxq_descr[i].rxq_stats));
     }
 }
 
 static void vmxnet3_adjust_by_guest_type(VMXNET3State *s)
 {
     struct Vmxnet3_GOSInfo gos;
+    PCIDevice *d = PCI_DEVICE(s);
 
-    VMXNET3_READ_DRV_SHARED(s->drv_shmem, devRead.misc.driverInfo.gos,
+    VMXNET3_READ_DRV_SHARED(d, s->drv_shmem, devRead.misc.driverInfo.gos,
                             &gos, sizeof(gos));
     s->rx_packets_compound =
         (gos.gosType == VMXNET3_GOS_TYPE_WIN) ? false : true;
@@ -1267,13 +1387,14 @@
 static void vmxnet3_update_pm_state(VMXNET3State *s)
 {
     struct Vmxnet3_VariableLenConfDesc pm_descr;
+    PCIDevice *d = PCI_DEVICE(s);
 
     pm_descr.confLen =
-        VMXNET3_READ_DRV_SHARED32(s->drv_shmem, devRead.pmConfDesc.confLen);
+        VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem, devRead.pmConfDesc.confLen);
     pm_descr.confVer =
-        VMXNET3_READ_DRV_SHARED32(s->drv_shmem, devRead.pmConfDesc.confVer);
+        VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem, devRead.pmConfDesc.confVer);
     pm_descr.confPA =
-        VMXNET3_READ_DRV_SHARED64(s->drv_shmem, devRead.pmConfDesc.confPA);
+        VMXNET3_READ_DRV_SHARED64(d, s->drv_shmem, devRead.pmConfDesc.confPA);
 
     vmxnet3_dump_conf_descr("PM State", &pm_descr);
 }
@@ -1282,8 +1403,9 @@
 {
     uint32_t guest_features;
     int rxcso_supported;
+    PCIDevice *d = PCI_DEVICE(s);
 
-    guest_features = VMXNET3_READ_DRV_SHARED32(s->drv_shmem,
+    guest_features = VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem,
                                                devRead.misc.uptFeatures);
 
     rxcso_supported = VMXNET_FLAG_IS_SET(guest_features, UPT1_F_RXCSUM);
@@ -1305,8 +1427,8 @@
 
 static bool vmxnet3_verify_intx(VMXNET3State *s, int intx)
 {
-    return s->msix_used || s->msi_used || (intx ==
-           (pci_get_byte(s->parent_obj.config + PCI_INTERRUPT_PIN) - 1));
+    return s->msix_used || msi_enabled(PCI_DEVICE(s))
+        || intx == pci_get_byte(s->parent_obj.config + PCI_INTERRUPT_PIN) - 1;
 }
 
 static void vmxnet3_validate_interrupt_idx(bool is_msix, int idx)
@@ -1358,26 +1480,33 @@
 {
     int i;
     static const uint32_t VMXNET3_DEF_TX_THRESHOLD = 1;
+    PCIDevice *d = PCI_DEVICE(s);
     hwaddr qdescr_table_pa;
     uint64_t pa;
     uint32_t size;
 
     /* Verify configuration consistency */
-    if (!vmxnet3_verify_driver_magic(s->drv_shmem)) {
+    if (!vmxnet3_verify_driver_magic(d, s->drv_shmem)) {
         VMW_ERPRN("Device configuration received from driver is invalid");
         return;
     }
 
+    /* Verify if device is active */
+    if (s->device_active) {
+        VMW_CFPRN("Vmxnet3 device is active");
+        return;
+    }
+
     vmxnet3_adjust_by_guest_type(s);
     vmxnet3_update_features(s);
     vmxnet3_update_pm_state(s);
     vmxnet3_setup_rx_filtering(s);
     /* Cache fields from shared memory */
-    s->mtu = VMXNET3_READ_DRV_SHARED32(s->drv_shmem, devRead.misc.mtu);
+    s->mtu = VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem, devRead.misc.mtu);
     VMW_CFPRN("MTU is %u", s->mtu);
 
     s->max_rx_frags =
-        VMXNET3_READ_DRV_SHARED16(s->drv_shmem, devRead.misc.maxNumRxSG);
+        VMXNET3_READ_DRV_SHARED16(d, s->drv_shmem, devRead.misc.maxNumRxSG);
 
     if (s->max_rx_frags == 0) {
         s->max_rx_frags = 1;
@@ -1386,25 +1515,25 @@
     VMW_CFPRN("Max RX fragments is %u", s->max_rx_frags);
 
     s->event_int_idx =
-        VMXNET3_READ_DRV_SHARED8(s->drv_shmem, devRead.intrConf.eventIntrIdx);
+        VMXNET3_READ_DRV_SHARED8(d, s->drv_shmem, devRead.intrConf.eventIntrIdx);
     assert(vmxnet3_verify_intx(s, s->event_int_idx));
     (void)vmxnet3_verify_intx;
     VMW_CFPRN("Events interrupt line is %u", s->event_int_idx);
 
     s->auto_int_masking =
-        VMXNET3_READ_DRV_SHARED8(s->drv_shmem, devRead.intrConf.autoMask);
+        VMXNET3_READ_DRV_SHARED8(d, s->drv_shmem, devRead.intrConf.autoMask);
     VMW_CFPRN("Automatic interrupt masking is %d", (int)s->auto_int_masking);
 
     s->txq_num =
-        VMXNET3_READ_DRV_SHARED8(s->drv_shmem, devRead.misc.numTxQueues);
+        VMXNET3_READ_DRV_SHARED8(d, s->drv_shmem, devRead.misc.numTxQueues);
     s->rxq_num =
-        VMXNET3_READ_DRV_SHARED8(s->drv_shmem, devRead.misc.numRxQueues);
+        VMXNET3_READ_DRV_SHARED8(d, s->drv_shmem, devRead.misc.numRxQueues);
 
     VMW_CFPRN("Number of TX/RX queues %u/%u", s->txq_num, s->rxq_num);
     vmxnet3_validate_queues(s);
 
     qdescr_table_pa =
-        VMXNET3_READ_DRV_SHARED64(s->drv_shmem, devRead.misc.queueDescPA);
+        VMXNET3_READ_DRV_SHARED64(d, s->drv_shmem, devRead.misc.queueDescPA);
     VMW_CFPRN("TX queues descriptors table is at 0x%" PRIx64, qdescr_table_pa);
 
     /*
@@ -1420,25 +1549,25 @@
 
         /* Read interrupt number for this TX queue */
         s->txq_descr[i].intr_idx =
-            VMXNET3_READ_TX_QUEUE_DESCR8(qdescr_pa, conf.intrIdx);
+            VMXNET3_READ_TX_QUEUE_DESCR8(d, qdescr_pa, conf.intrIdx);
         assert(vmxnet3_verify_intx(s, s->txq_descr[i].intr_idx));
 
         VMW_CFPRN("TX Queue %d interrupt: %d", i, s->txq_descr[i].intr_idx);
 
         /* Read rings memory locations for TX queues */
-        pa = VMXNET3_READ_TX_QUEUE_DESCR64(qdescr_pa, conf.txRingBasePA);
-        size = VMXNET3_READ_TX_QUEUE_DESCR32(qdescr_pa, conf.txRingSize);
+        pa = VMXNET3_READ_TX_QUEUE_DESCR64(d, qdescr_pa, conf.txRingBasePA);
+        size = VMXNET3_READ_TX_QUEUE_DESCR32(d, qdescr_pa, conf.txRingSize);
 
-        vmxnet3_ring_init(&s->txq_descr[i].tx_ring, pa, size,
+        vmxnet3_ring_init(d, &s->txq_descr[i].tx_ring, pa, size,
                           sizeof(struct Vmxnet3_TxDesc), false);
         VMXNET3_RING_DUMP(VMW_CFPRN, "TX", i, &s->txq_descr[i].tx_ring);
 
         s->max_tx_frags += size;
 
         /* TXC ring */
-        pa = VMXNET3_READ_TX_QUEUE_DESCR64(qdescr_pa, conf.compRingBasePA);
-        size = VMXNET3_READ_TX_QUEUE_DESCR32(qdescr_pa, conf.compRingSize);
-        vmxnet3_ring_init(&s->txq_descr[i].comp_ring, pa, size,
+        pa = VMXNET3_READ_TX_QUEUE_DESCR64(d, qdescr_pa, conf.compRingBasePA);
+        size = VMXNET3_READ_TX_QUEUE_DESCR32(d, qdescr_pa, conf.compRingSize);
+        vmxnet3_ring_init(d, &s->txq_descr[i].comp_ring, pa, size,
                           sizeof(struct Vmxnet3_TxCompDesc), true);
         VMXNET3_RING_DUMP(VMW_CFPRN, "TXC", i, &s->txq_descr[i].comp_ring);
 
@@ -1449,15 +1578,16 @@
                sizeof(s->txq_descr[i].txq_stats));
 
         /* Fill device-managed parameters for queues */
-        VMXNET3_WRITE_TX_QUEUE_DESCR32(qdescr_pa,
+        VMXNET3_WRITE_TX_QUEUE_DESCR32(d, qdescr_pa,
                                        ctrl.txThreshold,
                                        VMXNET3_DEF_TX_THRESHOLD);
     }
 
     /* Preallocate TX packet wrapper */
     VMW_CFPRN("Max TX fragments is %u", s->max_tx_frags);
-    vmxnet_tx_pkt_init(&s->tx_pkt, s->max_tx_frags, s->peer_has_vhdr);
-    vmxnet_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr);
+    net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s),
+                    s->max_tx_frags, s->peer_has_vhdr);
+    net_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr);
 
     /* Read rings memory locations for RX queues */
     for (i = 0; i < s->rxq_num; i++) {
@@ -1468,7 +1598,7 @@
 
         /* Read interrupt number for this RX queue */
         s->rxq_descr[i].intr_idx =
-            VMXNET3_READ_TX_QUEUE_DESCR8(qd_pa, conf.intrIdx);
+            VMXNET3_READ_TX_QUEUE_DESCR8(d, qd_pa, conf.intrIdx);
         assert(vmxnet3_verify_intx(s, s->rxq_descr[i].intr_idx));
 
         VMW_CFPRN("RX Queue %d interrupt: %d", i, s->rxq_descr[i].intr_idx);
@@ -1476,18 +1606,18 @@
         /* Read rings memory locations */
         for (j = 0; j < VMXNET3_RX_RINGS_PER_QUEUE; j++) {
             /* RX rings */
-            pa = VMXNET3_READ_RX_QUEUE_DESCR64(qd_pa, conf.rxRingBasePA[j]);
-            size = VMXNET3_READ_RX_QUEUE_DESCR32(qd_pa, conf.rxRingSize[j]);
-            vmxnet3_ring_init(&s->rxq_descr[i].rx_ring[j], pa, size,
+            pa = VMXNET3_READ_RX_QUEUE_DESCR64(d, qd_pa, conf.rxRingBasePA[j]);
+            size = VMXNET3_READ_RX_QUEUE_DESCR32(d, qd_pa, conf.rxRingSize[j]);
+            vmxnet3_ring_init(d, &s->rxq_descr[i].rx_ring[j], pa, size,
                               sizeof(struct Vmxnet3_RxDesc), false);
             VMW_CFPRN("RX queue %d:%d: Base: %" PRIx64 ", Size: %d",
                       i, j, pa, size);
         }
 
         /* RXC ring */
-        pa = VMXNET3_READ_RX_QUEUE_DESCR64(qd_pa, conf.compRingBasePA);
-        size = VMXNET3_READ_RX_QUEUE_DESCR32(qd_pa, conf.compRingSize);
-        vmxnet3_ring_init(&s->rxq_descr[i].comp_ring, pa, size,
+        pa = VMXNET3_READ_RX_QUEUE_DESCR64(d, qd_pa, conf.compRingBasePA);
+        size = VMXNET3_READ_RX_QUEUE_DESCR32(d, qd_pa, conf.compRingSize);
+        vmxnet3_ring_init(d, &s->rxq_descr[i].comp_ring, pa, size,
                           sizeof(struct Vmxnet3_RxCompDesc), true);
         VMW_CFPRN("RXC queue %d: Base: %" PRIx64 ", Size: %d", i, pa, size);
 
@@ -1565,7 +1695,7 @@
         break;
 
     case VMXNET3_CMD_QUIESCE_DEV:
-        VMW_CBPRN("Set: VMXNET3_CMD_QUIESCE_DEV - pause the device");
+        VMW_CBPRN("Set: VMXNET3_CMD_QUIESCE_DEV - deactivate the device");
         vmxnet3_deactivate_device(s);
         break;
 
@@ -1573,6 +1703,23 @@
         VMW_CBPRN("Set: VMXNET3_CMD_GET_CONF_INTR - interrupt configuration");
         break;
 
+    case VMXNET3_CMD_GET_ADAPTIVE_RING_INFO:
+        VMW_CBPRN("Set: VMXNET3_CMD_GET_ADAPTIVE_RING_INFO - "
+                  "adaptive ring info flags");
+        break;
+
+    case VMXNET3_CMD_GET_DID_LO:
+        VMW_CBPRN("Set: Get lower part of device ID");
+        break;
+
+    case VMXNET3_CMD_GET_DID_HI:
+        VMW_CBPRN("Set: Get upper part of device ID");
+        break;
+
+    case VMXNET3_CMD_GET_DEV_EXTRA_INFO:
+        VMW_CBPRN("Set: Get device extra info");
+        break;
+
     default:
         VMW_CBPRN("Received unknown command: %" PRIx64, cmd);
         break;
@@ -1585,13 +1732,14 @@
 
     switch (s->last_command) {
     case VMXNET3_CMD_ACTIVATE_DEV:
-        ret = (s->device_active) ? 0 : -1;
+        ret = (s->device_active) ? 0 : 1;
         VMW_CFPRN("Device active: %" PRIx64, ret);
         break;
 
     case VMXNET3_CMD_RESET_DEV:
     case VMXNET3_CMD_QUIESCE_DEV:
     case VMXNET3_CMD_GET_QUEUE_STATUS:
+    case VMXNET3_CMD_GET_DEV_EXTRA_INFO:
         ret = 0;
         break;
 
@@ -1612,9 +1760,21 @@
         ret = vmxnet3_get_interrupt_config(s);
         break;
 
+    case VMXNET3_CMD_GET_ADAPTIVE_RING_INFO:
+        ret = VMXNET3_DISABLE_ADAPTIVE_RING;
+        break;
+
+    case VMXNET3_CMD_GET_DID_LO:
+        ret = PCI_DEVICE_ID_VMWARE_VMXNET3;
+        break;
+
+    case VMXNET3_CMD_GET_DID_HI:
+        ret = VMXNET3_DEVICE_REVISION;
+        break;
+
     default:
         VMW_WRPRN("Received request for unknown command: %x", s->last_command);
-        ret = -1;
+        ret = 0;
         break;
     }
 
@@ -1624,19 +1784,21 @@
 static void vmxnet3_set_events(VMXNET3State *s, uint32_t val)
 {
     uint32_t events;
+    PCIDevice *d = PCI_DEVICE(s);
 
     VMW_CBPRN("Setting events: 0x%x", val);
-    events = VMXNET3_READ_DRV_SHARED32(s->drv_shmem, ecr) | val;
-    VMXNET3_WRITE_DRV_SHARED32(s->drv_shmem, ecr, events);
+    events = VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem, ecr) | val;
+    VMXNET3_WRITE_DRV_SHARED32(d, s->drv_shmem, ecr, events);
 }
 
 static void vmxnet3_ack_events(VMXNET3State *s, uint32_t val)
 {
+    PCIDevice *d = PCI_DEVICE(s);
     uint32_t events;
 
     VMW_CBPRN("Clearing events: 0x%x", val);
-    events = VMXNET3_READ_DRV_SHARED32(s->drv_shmem, ecr) & ~val;
-    VMXNET3_WRITE_DRV_SHARED32(s->drv_shmem, ecr, events);
+    events = VMXNET3_READ_DRV_SHARED32(d, s->drv_shmem, ecr) & ~val;
+    VMXNET3_WRITE_DRV_SHARED32(d, s->drv_shmem, ecr, events);
 }
 
 static void
@@ -1670,7 +1832,7 @@
          * shared address only after we get the high part
          */
         if (val == 0) {
-            s->device_active = false;
+            vmxnet3_deactivate_device(s);
         }
         s->temp_shared_guest_driver_memory = val;
         s->drv_shmem = 0;
@@ -1745,7 +1907,7 @@
         /* UPT Version Report Selection */
         case VMXNET3_REG_UVRS:
             VMW_CBPRN("Read BAR1 [VMXNET3_REG_UVRS], size %d", size);
-            ret = VMXNET3_DEVICE_VERSION;
+            ret = VMXNET3_UPT_REVISION;
             break;
 
         /* Command */
@@ -1833,7 +1995,7 @@
         return false;
     }
 
-    switch (vmxnet_rx_pkt_get_packet_type(s->rx_pkt)) {
+    switch (net_rx_pkt_get_packet_type(s->rx_pkt)) {
     case ETH_PKT_UCAST:
         if (!VMXNET_FLAG_IS_SET(s->rx_mode, VMXNET3_RXM_UCAST)) {
             return false;
@@ -1880,6 +2042,12 @@
         return -1;
     }
 
+    if (s->peer_has_vhdr) {
+        net_rx_pkt_set_vhdr(s->rx_pkt, (struct virtio_net_hdr *)buf);
+        buf += sizeof(struct virtio_net_hdr);
+        size -= sizeof(struct virtio_net_hdr);
+    }
+
     /* Pad to minimum Ethernet frame length */
     if (size < sizeof(min_buf)) {
         memcpy(min_buf, buf, size);
@@ -1888,20 +2056,16 @@
         size = sizeof(min_buf);
     }
 
-    if (s->peer_has_vhdr) {
-        vmxnet_rx_pkt_set_vhdr(s->rx_pkt, (struct virtio_net_hdr *)buf);
-        buf += sizeof(struct virtio_net_hdr);
-        size -= sizeof(struct virtio_net_hdr);
-    }
-
-    vmxnet_rx_pkt_set_packet_type(s->rx_pkt,
+    net_rx_pkt_set_packet_type(s->rx_pkt,
         get_eth_packet_type(PKT_GET_ETH_HDR(buf)));
 
     if (vmxnet3_rx_filter_may_indicate(s, buf, size)) {
-        vmxnet_rx_pkt_attach_data(s->rx_pkt, buf, size, s->rx_vlan_stripping);
+        net_rx_pkt_set_protocols(s->rx_pkt, buf, size);
+        vmxnet3_rx_need_csum_calculate(s->rx_pkt, buf, size);
+        net_rx_pkt_attach_data(s->rx_pkt, buf, size, s->rx_vlan_stripping);
         bytes_indicated = vmxnet3_indicate_packet(s) ? size : -1;
         if (bytes_indicated < size) {
-            VMW_PKPRN("RX: %lu of %lu bytes indicated", bytes_indicated, size);
+            VMW_PKPRN("RX: %zu of %zu bytes indicated", bytes_indicated, size);
         }
     } else {
         VMW_PKPRN("Packet dropped by RX filter");
@@ -1913,12 +2077,6 @@
     return bytes_indicated;
 }
 
-static void vmxnet3_cleanup(NetClientState *nc)
-{
-    VMXNET3State *s = qemu_get_nic_opaque(nc);
-    s->nic = NULL;
-}
-
 static void vmxnet3_set_link_status(NetClientState *nc)
 {
     VMXNET3State *s = qemu_get_nic_opaque(nc);
@@ -1934,11 +2092,9 @@
 }
 
 static NetClientInfo net_vmxnet3_info = {
-        .type = NET_CLIENT_OPTIONS_KIND_NIC,
+        .type = NET_CLIENT_DRIVER_NIC,
         .size = sizeof(NICState),
-        .can_receive = vmxnet3_can_receive,
         .receive = vmxnet3_receive,
-        .cleanup = vmxnet3_cleanup,
         .link_status_changed = vmxnet3_set_link_status,
 };
 
@@ -1950,16 +2106,13 @@
         return true;
     }
 
-    VMW_WRPRN("Peer has no virtio extension. Task offloads will be emulated.");
     return false;
 }
 
 static void vmxnet3_net_uninit(VMXNET3State *s)
 {
     g_free(s->mcast_list);
-    vmxnet_tx_pkt_reset(s->tx_pkt);
-    vmxnet_tx_pkt_uninit(s->tx_pkt);
-    vmxnet_rx_pkt_uninit(s->rx_pkt);
+    vmxnet3_deactivate_device(s);
     qemu_del_nic(s->nic);
 }
 
@@ -2037,8 +2190,8 @@
                         &s->msix_bar,
                         VMXNET3_MSIX_BAR_IDX, VMXNET3_OFF_MSIX_TABLE,
                         &s->msix_bar,
-                        VMXNET3_MSIX_BAR_IDX, VMXNET3_OFF_MSIX_PBA,
-                        0);
+                        VMXNET3_MSIX_BAR_IDX, VMXNET3_OFF_MSIX_PBA(s),
+                        VMXNET3_MSIX_OFFSET(s));
 
     if (0 > res) {
         VMW_WRPRN("Failed to initialize MSI-X, error %d", res);
@@ -2066,36 +2219,12 @@
     }
 }
 
-#define VMXNET3_MSI_OFFSET        (0x50)
-#define VMXNET3_USE_64BIT         (true)
-#define VMXNET3_PER_VECTOR_MASK   (false)
-
-static bool
-vmxnet3_init_msi(VMXNET3State *s)
-{
-    PCIDevice *d = PCI_DEVICE(s);
-    int res;
-
-    res = msi_init(d, VMXNET3_MSI_OFFSET, VMXNET3_MAX_NMSIX_INTRS,
-                   VMXNET3_USE_64BIT, VMXNET3_PER_VECTOR_MASK);
-    if (0 > res) {
-        VMW_WRPRN("Failed to initialize MSI, error %d", res);
-        s->msi_used = false;
-    } else {
-        s->msi_used = true;
-    }
-
-    return s->msi_used;
-}
-
 static void
 vmxnet3_cleanup_msi(VMXNET3State *s)
 {
     PCIDevice *d = PCI_DEVICE(s);
 
-    if (s->msi_used) {
-        msi_uninit(d);
-    }
+    msi_uninit(d);
 }
 
 static void
@@ -2133,10 +2262,31 @@
     },
 };
 
-static int vmxnet3_pci_init(PCIDevice *pci_dev)
+static uint64_t vmxnet3_device_serial_num(VMXNET3State *s)
+{
+    uint64_t dsn_payload;
+    uint8_t *dsnp = (uint8_t *)&dsn_payload;
+
+    dsnp[0] = 0xfe;
+    dsnp[1] = s->conf.macaddr.a[3];
+    dsnp[2] = s->conf.macaddr.a[4];
+    dsnp[3] = s->conf.macaddr.a[5];
+    dsnp[4] = s->conf.macaddr.a[0];
+    dsnp[5] = s->conf.macaddr.a[1];
+    dsnp[6] = s->conf.macaddr.a[2];
+    dsnp[7] = 0xff;
+    return dsn_payload;
+}
+
+
+#define VMXNET3_USE_64BIT         (true)
+#define VMXNET3_PER_VECTOR_MASK   (false)
+
+static void vmxnet3_pci_realize(PCIDevice *pci_dev, Error **errp)
 {
     DeviceState *dev = DEVICE(pci_dev);
     VMXNET3State *s = VMXNET3(pci_dev);
+    int ret;
 
     VMW_CBPRN("Starting init...");
 
@@ -2160,20 +2310,29 @@
     /* Interrupt pin A */
     pci_dev->config[PCI_INTERRUPT_PIN] = 0x01;
 
+    ret = msi_init(pci_dev, VMXNET3_MSI_OFFSET(s), VMXNET3_MAX_NMSIX_INTRS,
+                   VMXNET3_USE_64BIT, VMXNET3_PER_VECTOR_MASK, NULL);
+    /* Any error other than -ENOTSUP(board's MSI support is broken)
+     * is a programming error. Fall back to INTx silently on -ENOTSUP */
+    assert(!ret || ret == -ENOTSUP);
+
     if (!vmxnet3_init_msix(s)) {
         VMW_WRPRN("Failed to initialize MSI-X, configuration is inconsistent.");
     }
 
-    if (!vmxnet3_init_msi(s)) {
-        VMW_WRPRN("Failed to initialize MSI, configuration is inconsistent.");
-    }
-
     vmxnet3_net_init(s);
 
+    if (pci_is_express(pci_dev)) {
+        if (pci_bus_is_express(pci_dev->bus)) {
+            pcie_endpoint_cap_init(pci_dev, VMXNET3_EXP_EP_OFFSET);
+        }
+
+        pcie_dev_ser_num_init(pci_dev, VMXNET3_DSN_OFFSET,
+                              vmxnet3_device_serial_num(s));
+    }
+
     register_savevm(dev, "vmxnet3-msix", -1, 1,
                     vmxnet3_msix_save, vmxnet3_msix_load, s);
-
-    return 0;
 }
 
 static void vmxnet3_instance_init(Object *obj)
@@ -2236,6 +2395,7 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .pre_load = vmxnet3_mcast_list_pre_load,
+    .needed = vmxnet3_mc_list_needed,
     .fields = (VMStateField[]) {
         VMSTATE_VBUFFER_UINT32(mcast_list, VMXNET3State, 0, NULL, 0,
             mcast_list_buff_size),
@@ -2390,8 +2550,9 @@
     VMXNET3State *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
 
-    vmxnet_tx_pkt_init(&s->tx_pkt, s->max_tx_frags, s->peer_has_vhdr);
-    vmxnet_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr);
+    net_tx_pkt_init(&s->tx_pkt, PCI_DEVICE(s),
+                    s->max_tx_frags, s->peer_has_vhdr);
+    net_rx_pkt_init(&s->rx_pkt, s->peer_has_vhdr);
 
     if (s->msix_used) {
         if  (!vmxnet3_use_msix_vectors(s, VMXNET3_MAX_INTRS)) {
@@ -2440,6 +2601,29 @@
     .put = vmxnet3_put_int_state
 };
 
+static bool vmxnet3_vmstate_need_pcie_device(void *opaque)
+{
+    VMXNET3State *s = VMXNET3(opaque);
+
+    return !(s->compat_flags & VMXNET3_COMPAT_FLAG_DISABLE_PCIE);
+}
+
+static bool vmxnet3_vmstate_test_pci_device(void *opaque, int version_id)
+{
+    return !vmxnet3_vmstate_need_pcie_device(opaque);
+}
+
+static const VMStateDescription vmstate_vmxnet3_pcie_device = {
+    .name = "vmxnet3/pcie",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vmxnet3_vmstate_need_pcie_device,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(parent_obj, VMXNET3State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_vmxnet3 = {
     .name = "vmxnet3",
     .version_id = 1,
@@ -2447,7 +2631,9 @@
     .pre_save = vmxnet3_pre_save,
     .post_load = vmxnet3_post_load,
     .fields = (VMStateField[]) {
-            VMSTATE_PCI_DEVICE(parent_obj, VMXNET3State),
+            VMSTATE_STRUCT_TEST(parent_obj, VMXNET3State,
+                                vmxnet3_vmstate_test_pci_device, 0,
+                                vmstate_pci_device, PCIDevice),
             VMSTATE_BOOL(rx_packets_compound, VMXNET3State),
             VMSTATE_BOOL(rx_vlan_stripping, VMXNET3State),
             VMSTATE_BOOL(lro_supported, VMXNET3State),
@@ -2480,44 +2666,52 @@
 
             VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmxstate_vmxnet3_mcast_list,
-            .needed = vmxnet3_mc_list_needed
-        },
-        {
-            /* empty element. */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmxstate_vmxnet3_mcast_list,
+        &vmstate_vmxnet3_pcie_device,
+        NULL
     }
 };
 
-static void
-vmxnet3_write_config(PCIDevice *pci_dev, uint32_t addr, uint32_t val, int len)
-{
-    pci_default_write_config(pci_dev, addr, val, len);
-    msix_write_config(pci_dev, addr, val, len);
-    msi_write_config(pci_dev, addr, val, len);
-}
-
 static Property vmxnet3_properties[] = {
     DEFINE_NIC_PROPERTIES(VMXNET3State, conf),
+    DEFINE_PROP_BIT("x-old-msi-offsets", VMXNET3State, compat_flags,
+                    VMXNET3_COMPAT_FLAG_OLD_MSI_OFFSETS_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", VMXNET3State, compat_flags,
+                    VMXNET3_COMPAT_FLAG_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void vmxnet3_realize(DeviceState *qdev, Error **errp)
+{
+    VMXNET3Class *vc = VMXNET3_DEVICE_GET_CLASS(qdev);
+    PCIDevice *pci_dev = PCI_DEVICE(qdev);
+    VMXNET3State *s = VMXNET3(qdev);
+
+    if (!(s->compat_flags & VMXNET3_COMPAT_FLAG_DISABLE_PCIE)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    vc->parent_dc_realize(qdev, errp);
+}
+
 static void vmxnet3_class_init(ObjectClass *class, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(class);
     PCIDeviceClass *c = PCI_DEVICE_CLASS(class);
+    VMXNET3Class *vc = VMXNET3_DEVICE_CLASS(class);
 
-    c->init = vmxnet3_pci_init;
+    c->realize = vmxnet3_pci_realize;
     c->exit = vmxnet3_pci_uninit;
     c->vendor_id = PCI_VENDOR_ID_VMWARE;
     c->device_id = PCI_DEVICE_ID_VMWARE_VMXNET3;
     c->revision = PCI_DEVICE_ID_VMWARE_VMXNET3_REVISION;
+    c->romfile = "efi-vmxnet3.rom";
     c->class_id = PCI_CLASS_NETWORK_ETHERNET;
     c->subsystem_vendor_id = PCI_VENDOR_ID_VMWARE;
     c->subsystem_id = PCI_DEVICE_ID_VMWARE_VMXNET3;
-    c->config_write = vmxnet3_write_config,
+    vc->parent_dc_realize = dc->realize;
+    dc->realize = vmxnet3_realize;
     dc->desc = "VMWare Paravirtualized Ethernet v3";
     dc->reset = vmxnet3_qdev_reset;
     dc->vmsd = &vmstate_vmxnet3;
@@ -2528,6 +2722,7 @@
 static const TypeInfo vmxnet3_info = {
     .name          = TYPE_VMXNET3,
     .parent        = TYPE_PCI_DEVICE,
+    .class_size    = sizeof(VMXNET3Class),
     .instance_size = sizeof(VMXNET3State),
     .class_init    = vmxnet3_class_init,
     .instance_init = vmxnet3_instance_init,
diff --git a/hw/net/vmxnet3.h b/hw/net/vmxnet3.h
index f987d71..f9352c4 100644
--- a/hw/net/vmxnet3.h
+++ b/hw/net/vmxnet3.h
@@ -15,8 +15,8 @@
  *
  */
 
-#ifndef _QEMU_VMXNET3_H
-#define _QEMU_VMXNET3_H
+#ifndef QEMU_VMXNET3_H
+#define QEMU_VMXNET3_H
 
 #define VMXNET3_DEVICE_MAX_TX_QUEUES 8
 #define VMXNET3_DEVICE_MAX_RX_QUEUES 8   /* Keep this value as a power of 2 */
@@ -198,9 +198,13 @@
     VMXNET3_CMD_GET_DID_LO,                               /* 0xF00D0005 */
     VMXNET3_CMD_GET_DID_HI,                               /* 0xF00D0006 */
     VMXNET3_CMD_GET_DEV_EXTRA_INFO,                       /* 0xF00D0007 */
-    VMXNET3_CMD_GET_CONF_INTR                             /* 0xF00D0008 */
+    VMXNET3_CMD_GET_CONF_INTR,                            /* 0xF00D0008 */
+    VMXNET3_CMD_GET_ADAPTIVE_RING_INFO                    /* 0xF00D0009 */
 };
 
+/* Adaptive Ring Info Flags */
+#define VMXNET3_DISABLE_ADAPTIVE_RING 1
+
 /*
  *    Little Endian layout of bitfields -
  *    Byte 0 :    7.....len.....0
diff --git a/hw/net/vmxnet_debug.h b/hw/net/vmxnet_debug.h
index 96dae0f..cb50aa9 100644
--- a/hw/net/vmxnet_debug.h
+++ b/hw/net/vmxnet_debug.h
@@ -15,101 +15,131 @@
  *
  */
 
-#ifndef _QEMU_VMXNET_DEBUG_H
-#define _QEMU_VMXNET_DEBUG_H
+#ifndef QEMU_VMXNET_DEBUG_H
+#define QEMU_VMXNET_DEBUG_H
 
 #define VMXNET_DEVICE_NAME "vmxnet3"
 
-/* #define VMXNET_DEBUG_CB */
 #define VMXNET_DEBUG_WARNINGS
 #define VMXNET_DEBUG_ERRORS
-/* #define VMXNET_DEBUG_INTERRUPTS */
-/* #define VMXNET_DEBUG_CONFIG */
-/* #define VMXNET_DEBUG_RINGS */
-/* #define VMXNET_DEBUG_PACKETS */
-/* #define VMXNET_DEBUG_SHMEM_ACCESS */
 
-#ifdef VMXNET_DEBUG_SHMEM_ACCESS
-#define VMW_SHPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][SH][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
-#else
-#define VMW_SHPRN(fmt, ...) do {} while (0)
-#endif
+#undef VMXNET_DEBUG_CB
+#undef VMXNET_DEBUG_INTERRUPTS
+#undef VMXNET_DEBUG_CONFIG
+#undef VMXNET_DEBUG_RINGS
+#undef VMXNET_DEBUG_PACKETS
+#undef VMXNET_DEBUG_SHMEM_ACCESS
 
 #ifdef VMXNET_DEBUG_CB
-#define VMW_CBPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][CB][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
+#  define VMXNET_DEBUG_CB_ENABLED 1
 #else
-#define VMW_CBPRN(fmt, ...) do {} while (0)
-#endif
-
-#ifdef VMXNET_DEBUG_PACKETS
-#define VMW_PKPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][PK][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
-#else
-#define VMW_PKPRN(fmt, ...) do {} while (0)
+#  define VMXNET_DEBUG_CB_ENABLED 0
 #endif
 
 #ifdef VMXNET_DEBUG_WARNINGS
-#define VMW_WRPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][WR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
+#  define VMXNET_DEBUG_WARNINGS_ENABLED 1
 #else
-#define VMW_WRPRN(fmt, ...) do {} while (0)
+#  define VMXNET_DEBUG_WARNINGS_ENABLED 0
 #endif
 
 #ifdef VMXNET_DEBUG_ERRORS
-#define VMW_ERPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][ER][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
+#  define VMXNET_DEBUG_ERRORS_ENABLED 1
 #else
-#define VMW_ERPRN(fmt, ...) do {} while (0)
-#endif
-
-#ifdef VMXNET_DEBUG_INTERRUPTS
-#define VMW_IRPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][IR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
-#else
-#define VMW_IRPRN(fmt, ...) do {} while (0)
+#  define VMXNET_DEBUG_ERRORS_ENABLED 0
 #endif
 
 #ifdef VMXNET_DEBUG_CONFIG
-#define VMW_CFPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][CF][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
+#  define VMXNET_DEBUG_CONFIG_ENABLED 1
 #else
-#define VMW_CFPRN(fmt, ...) do {} while (0)
+#  define VMXNET_DEBUG_CONFIG_ENABLED 0
 #endif
 
 #ifdef VMXNET_DEBUG_RINGS
-#define VMW_RIPRN(fmt, ...)                                                   \
-    do {                                                                      \
-        printf("[%s][RI][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,       \
-            ## __VA_ARGS__);                                                  \
-    } while (0)
+#  define VMXNET_DEBUG_RINGS_ENABLED 1
 #else
-#define VMW_RIPRN(fmt, ...) do {} while (0)
+#  define VMXNET_DEBUG_RINGS_ENABLED 0
 #endif
 
-#define VMXNET_MF       "%02X:%02X:%02X:%02X:%02X:%02X"
-#define VMXNET_MA(a)    (a)[0], (a)[1], (a)[2], (a)[3], (a)[4], (a)[5]
+#ifdef VMXNET_DEBUG_PACKETS
+#  define VMXNET_DEBUG_PACKETS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_PACKETS_ENABLED 0
+#endif
 
-#endif /* _QEMU_VMXNET3_DEBUG_H  */
+#ifdef VMXNET_DEBUG_INTERRUPTS
+#  define VMXNET_DEBUG_INTERRUPTS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_INTERRUPTS_ENABLED 0
+#endif
+
+#ifdef VMXNET_DEBUG_SHMEM_ACCESS
+#  define VMXNET_DEBUG_SHMEM_ACCESS_ENABLED 1
+#else
+#  define VMXNET_DEBUG_SHMEM_ACCESS_ENABLED 0
+#endif
+
+#define VMW_SHPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_SHMEM_ACCESS_ENABLED) {                              \
+            printf("[%s][SH][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+       }                                                                      \
+    } while (0)
+
+#define VMW_CBPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_CB_ENABLED) {                                        \
+            printf("[%s][CB][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#define VMW_PKPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_PACKETS_ENABLED) {                                   \
+            printf("[%s][PK][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#define VMW_WRPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_WARNINGS_ENABLED) {                                  \
+            printf("[%s][WR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#define VMW_ERPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_ERRORS_ENABLED) {                                    \
+            printf("[%s][ER][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#define VMW_IRPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_INTERRUPTS_ENABLED) {                                \
+            printf("[%s][IR][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#define VMW_CFPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_CONFIG_ENABLED) {                                    \
+            printf("[%s][CF][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#define VMW_RIPRN(fmt, ...)                                                   \
+    do {                                                                      \
+        if (VMXNET_DEBUG_RINGS_ENABLED) {                                     \
+            printf("[%s][RI][%s]: " fmt "\n", VMXNET_DEVICE_NAME, __func__,   \
+                ## __VA_ARGS__);                                              \
+        }                                                                     \
+    } while (0)
+
+#endif /* QEMU_VMXNET_DEBUG_H */
diff --git a/hw/net/vmxnet_rx_pkt.c b/hw/net/vmxnet_rx_pkt.c
deleted file mode 100644
index a40e346..0000000
--- a/hw/net/vmxnet_rx_pkt.c
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * QEMU VMWARE VMXNET* paravirtual NICs - RX packets abstractions
- *
- * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
- *
- * Developed by Daynix Computing LTD (http://www.daynix.com)
- *
- * Authors:
- * Dmitry Fleytman <dmitry@daynix.com>
- * Tamir Shomer <tamirs@daynix.com>
- * Yan Vugenfirer <yan@daynix.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#include "vmxnet_rx_pkt.h"
-#include "net/eth.h"
-#include "qemu-common.h"
-#include "qemu/iov.h"
-#include "net/checksum.h"
-#include "net/tap.h"
-
-/*
- * RX packet may contain up to 2 fragments - rebuilt eth header
- * in case of VLAN tag stripping
- * and payload received from QEMU - in any case
- */
-#define VMXNET_MAX_RX_PACKET_FRAGMENTS (2)
-
-struct VmxnetRxPkt {
-    struct virtio_net_hdr virt_hdr;
-    uint8_t ehdr_buf[ETH_MAX_L2_HDR_LEN];
-    struct iovec vec[VMXNET_MAX_RX_PACKET_FRAGMENTS];
-    uint16_t vec_len;
-    uint32_t tot_len;
-    uint16_t tci;
-    bool vlan_stripped;
-    bool has_virt_hdr;
-    eth_pkt_types_e packet_type;
-
-    /* Analysis results */
-    bool isip4;
-    bool isip6;
-    bool isudp;
-    bool istcp;
-};
-
-void vmxnet_rx_pkt_init(struct VmxnetRxPkt **pkt, bool has_virt_hdr)
-{
-    struct VmxnetRxPkt *p = g_malloc0(sizeof *p);
-    p->has_virt_hdr = has_virt_hdr;
-    *pkt = p;
-}
-
-void vmxnet_rx_pkt_uninit(struct VmxnetRxPkt *pkt)
-{
-    g_free(pkt);
-}
-
-struct virtio_net_hdr *vmxnet_rx_pkt_get_vhdr(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-    return &pkt->virt_hdr;
-}
-
-void vmxnet_rx_pkt_attach_data(struct VmxnetRxPkt *pkt, const void *data,
-                               size_t len, bool strip_vlan)
-{
-    uint16_t tci = 0;
-    uint16_t ploff;
-    assert(pkt);
-    pkt->vlan_stripped = false;
-
-    if (strip_vlan) {
-        pkt->vlan_stripped = eth_strip_vlan(data, pkt->ehdr_buf, &ploff, &tci);
-    }
-
-    if (pkt->vlan_stripped) {
-        pkt->vec[0].iov_base = pkt->ehdr_buf;
-        pkt->vec[0].iov_len = ploff - sizeof(struct vlan_header);
-        pkt->vec[1].iov_base = (uint8_t *) data + ploff;
-        pkt->vec[1].iov_len = len - ploff;
-        pkt->vec_len = 2;
-        pkt->tot_len = len - ploff + sizeof(struct eth_header);
-    } else {
-        pkt->vec[0].iov_base = (void *)data;
-        pkt->vec[0].iov_len = len;
-        pkt->vec_len = 1;
-        pkt->tot_len = len;
-    }
-
-    pkt->tci = tci;
-
-    eth_get_protocols(data, len, &pkt->isip4, &pkt->isip6,
-        &pkt->isudp, &pkt->istcp);
-}
-
-void vmxnet_rx_pkt_dump(struct VmxnetRxPkt *pkt)
-{
-#ifdef VMXNET_RX_PKT_DEBUG
-    VmxnetRxPkt *pkt = (VmxnetRxPkt *)pkt;
-    assert(pkt);
-
-    printf("RX PKT: tot_len: %d, vlan_stripped: %d, vlan_tag: %d\n",
-              pkt->tot_len, pkt->vlan_stripped, pkt->tci);
-#endif
-}
-
-void vmxnet_rx_pkt_set_packet_type(struct VmxnetRxPkt *pkt,
-    eth_pkt_types_e packet_type)
-{
-    assert(pkt);
-
-    pkt->packet_type = packet_type;
-
-}
-
-eth_pkt_types_e vmxnet_rx_pkt_get_packet_type(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->packet_type;
-}
-
-size_t vmxnet_rx_pkt_get_total_len(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->tot_len;
-}
-
-void vmxnet_rx_pkt_get_protocols(struct VmxnetRxPkt *pkt,
-                                 bool *isip4, bool *isip6,
-                                 bool *isudp, bool *istcp)
-{
-    assert(pkt);
-
-    *isip4 = pkt->isip4;
-    *isip6 = pkt->isip6;
-    *isudp = pkt->isudp;
-    *istcp = pkt->istcp;
-}
-
-struct iovec *vmxnet_rx_pkt_get_iovec(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->vec;
-}
-
-void vmxnet_rx_pkt_set_vhdr(struct VmxnetRxPkt *pkt,
-                            struct virtio_net_hdr *vhdr)
-{
-    assert(pkt);
-
-    memcpy(&pkt->virt_hdr, vhdr, sizeof pkt->virt_hdr);
-}
-
-bool vmxnet_rx_pkt_is_vlan_stripped(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->vlan_stripped;
-}
-
-bool vmxnet_rx_pkt_has_virt_hdr(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->has_virt_hdr;
-}
-
-uint16_t vmxnet_rx_pkt_get_num_frags(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->vec_len;
-}
-
-uint16_t vmxnet_rx_pkt_get_vlan_tag(struct VmxnetRxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->tci;
-}
diff --git a/hw/net/vmxnet_rx_pkt.h b/hw/net/vmxnet_rx_pkt.h
deleted file mode 100644
index 6b2c60e..0000000
--- a/hw/net/vmxnet_rx_pkt.h
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * QEMU VMWARE VMXNET* paravirtual NICs - RX packets abstraction
- *
- * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
- *
- * Developed by Daynix Computing LTD (http://www.daynix.com)
- *
- * Authors:
- * Dmitry Fleytman <dmitry@daynix.com>
- * Tamir Shomer <tamirs@daynix.com>
- * Yan Vugenfirer <yan@daynix.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#ifndef VMXNET_RX_PKT_H
-#define VMXNET_RX_PKT_H
-
-#include "stdint.h"
-#include "stdbool.h"
-#include "net/eth.h"
-
-/* defines to enable packet dump functions */
-/*#define VMXNET_RX_PKT_DEBUG*/
-
-struct VmxnetRxPkt;
-
-/**
- * Clean all rx packet resources
- *
- * @pkt:            packet
- *
- */
-void vmxnet_rx_pkt_uninit(struct VmxnetRxPkt *pkt);
-
-/**
- * Init function for rx packet functionality
- *
- * @pkt:            packet pointer
- * @has_virt_hdr:   device uses virtio header
- *
- */
-void vmxnet_rx_pkt_init(struct VmxnetRxPkt **pkt, bool has_virt_hdr);
-
-/**
- * returns total length of data attached to rx context
- *
- * @pkt:            packet
- *
- * Return:  nothing
- *
- */
-size_t vmxnet_rx_pkt_get_total_len(struct VmxnetRxPkt *pkt);
-
-/**
- * fetches packet analysis results
- *
- * @pkt:            packet
- * @isip4:          whether the packet given is IPv4
- * @isip6:          whether the packet given is IPv6
- * @isudp:          whether the packet given is UDP
- * @istcp:          whether the packet given is TCP
- *
- */
-void vmxnet_rx_pkt_get_protocols(struct VmxnetRxPkt *pkt,
-                                 bool *isip4, bool *isip6,
-                                 bool *isudp, bool *istcp);
-
-/**
- * returns virtio header stored in rx context
- *
- * @pkt:            packet
- * @ret:            virtio header
- *
- */
-struct virtio_net_hdr *vmxnet_rx_pkt_get_vhdr(struct VmxnetRxPkt *pkt);
-
-/**
- * returns packet type
- *
- * @pkt:            packet
- * @ret:            packet type
- *
- */
-eth_pkt_types_e vmxnet_rx_pkt_get_packet_type(struct VmxnetRxPkt *pkt);
-
-/**
- * returns vlan tag
- *
- * @pkt:            packet
- * @ret:            VLAN tag
- *
- */
-uint16_t vmxnet_rx_pkt_get_vlan_tag(struct VmxnetRxPkt *pkt);
-
-/**
- * tells whether vlan was stripped from the packet
- *
- * @pkt:            packet
- * @ret:            VLAN stripped sign
- *
- */
-bool vmxnet_rx_pkt_is_vlan_stripped(struct VmxnetRxPkt *pkt);
-
-/**
- * notifies caller if the packet has virtio header
- *
- * @pkt:            packet
- * @ret:            true if packet has virtio header, false otherwize
- *
- */
-bool vmxnet_rx_pkt_has_virt_hdr(struct VmxnetRxPkt *pkt);
-
-/**
- * returns number of frags attached to the packet
- *
- * @pkt:            packet
- * @ret:            number of frags
- *
- */
-uint16_t vmxnet_rx_pkt_get_num_frags(struct VmxnetRxPkt *pkt);
-
-/**
- * attach data to rx packet
- *
- * @pkt:            packet
- * @data:           pointer to the data buffer
- * @len:            data length
- * @strip_vlan:     should the module strip vlan from data
- *
- */
-void vmxnet_rx_pkt_attach_data(struct VmxnetRxPkt *pkt, const void *data,
-    size_t len, bool strip_vlan);
-
-/**
- * returns io vector that holds the attached data
- *
- * @pkt:            packet
- * @ret:            pointer to IOVec
- *
- */
-struct iovec *vmxnet_rx_pkt_get_iovec(struct VmxnetRxPkt *pkt);
-
-/**
- * prints rx packet data if debug is enabled
- *
- * @pkt:            packet
- *
- */
-void vmxnet_rx_pkt_dump(struct VmxnetRxPkt *pkt);
-
-/**
- * copy passed vhdr data to packet context
- *
- * @pkt:            packet
- * @vhdr:           VHDR buffer
- *
- */
-void vmxnet_rx_pkt_set_vhdr(struct VmxnetRxPkt *pkt,
-    struct virtio_net_hdr *vhdr);
-
-/**
- * save packet type in packet context
- *
- * @pkt:            packet
- * @packet_type:    the packet type
- *
- */
-void vmxnet_rx_pkt_set_packet_type(struct VmxnetRxPkt *pkt,
-    eth_pkt_types_e packet_type);
-
-#endif
diff --git a/hw/net/vmxnet_tx_pkt.c b/hw/net/vmxnet_tx_pkt.c
deleted file mode 100644
index f7344c4..0000000
--- a/hw/net/vmxnet_tx_pkt.c
+++ /dev/null
@@ -1,567 +0,0 @@
-/*
- * QEMU VMWARE VMXNET* paravirtual NICs - TX packets abstractions
- *
- * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
- *
- * Developed by Daynix Computing LTD (http://www.daynix.com)
- *
- * Authors:
- * Dmitry Fleytman <dmitry@daynix.com>
- * Tamir Shomer <tamirs@daynix.com>
- * Yan Vugenfirer <yan@daynix.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#include "hw/hw.h"
-#include "vmxnet_tx_pkt.h"
-#include "net/eth.h"
-#include "qemu-common.h"
-#include "qemu/iov.h"
-#include "net/checksum.h"
-#include "net/tap.h"
-#include "net/net.h"
-
-enum {
-    VMXNET_TX_PKT_VHDR_FRAG = 0,
-    VMXNET_TX_PKT_L2HDR_FRAG,
-    VMXNET_TX_PKT_L3HDR_FRAG,
-    VMXNET_TX_PKT_PL_START_FRAG
-};
-
-/* TX packet private context */
-struct VmxnetTxPkt {
-    struct virtio_net_hdr virt_hdr;
-    bool has_virt_hdr;
-
-    struct iovec *raw;
-    uint32_t raw_frags;
-    uint32_t max_raw_frags;
-
-    struct iovec *vec;
-
-    uint8_t l2_hdr[ETH_MAX_L2_HDR_LEN];
-
-    uint32_t payload_len;
-
-    uint32_t payload_frags;
-    uint32_t max_payload_frags;
-
-    uint16_t hdr_len;
-    eth_pkt_types_e packet_type;
-    uint8_t l4proto;
-};
-
-void vmxnet_tx_pkt_init(struct VmxnetTxPkt **pkt, uint32_t max_frags,
-    bool has_virt_hdr)
-{
-    struct VmxnetTxPkt *p = g_malloc0(sizeof *p);
-
-    p->vec = g_malloc((sizeof *p->vec) *
-        (max_frags + VMXNET_TX_PKT_PL_START_FRAG));
-
-    p->raw = g_malloc((sizeof *p->raw) * max_frags);
-
-    p->max_payload_frags = max_frags;
-    p->max_raw_frags = max_frags;
-    p->has_virt_hdr = has_virt_hdr;
-    p->vec[VMXNET_TX_PKT_VHDR_FRAG].iov_base = &p->virt_hdr;
-    p->vec[VMXNET_TX_PKT_VHDR_FRAG].iov_len =
-        p->has_virt_hdr ? sizeof p->virt_hdr : 0;
-    p->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_base = &p->l2_hdr;
-    p->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base = NULL;
-    p->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len = 0;
-
-    *pkt = p;
-}
-
-void vmxnet_tx_pkt_uninit(struct VmxnetTxPkt *pkt)
-{
-    if (pkt) {
-        g_free(pkt->vec);
-        g_free(pkt->raw);
-        g_free(pkt);
-    }
-}
-
-void vmxnet_tx_pkt_update_ip_checksums(struct VmxnetTxPkt *pkt)
-{
-    uint16_t csum;
-    uint32_t ph_raw_csum;
-    assert(pkt);
-    uint8_t gso_type = pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN;
-    struct ip_header *ip_hdr;
-
-    if (VIRTIO_NET_HDR_GSO_TCPV4 != gso_type &&
-        VIRTIO_NET_HDR_GSO_UDP != gso_type) {
-        return;
-    }
-
-    ip_hdr = pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base;
-
-    if (pkt->payload_len + pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len >
-        ETH_MAX_IP_DGRAM_LEN) {
-        return;
-    }
-
-    ip_hdr->ip_len = cpu_to_be16(pkt->payload_len +
-        pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len);
-
-    /* Calculate IP header checksum                    */
-    ip_hdr->ip_sum = 0;
-    csum = net_raw_checksum((uint8_t *)ip_hdr,
-        pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len);
-    ip_hdr->ip_sum = cpu_to_be16(csum);
-
-    /* Calculate IP pseudo header checksum             */
-    ph_raw_csum = eth_calc_pseudo_hdr_csum(ip_hdr, pkt->payload_len);
-    csum = cpu_to_be16(~net_checksum_finish(ph_raw_csum));
-    iov_from_buf(&pkt->vec[VMXNET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
-                 pkt->virt_hdr.csum_offset, &csum, sizeof(csum));
-}
-
-static void vmxnet_tx_pkt_calculate_hdr_len(struct VmxnetTxPkt *pkt)
-{
-    pkt->hdr_len = pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_len +
-        pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len;
-}
-
-static bool vmxnet_tx_pkt_parse_headers(struct VmxnetTxPkt *pkt)
-{
-    struct iovec *l2_hdr, *l3_hdr;
-    size_t bytes_read;
-    size_t full_ip6hdr_len;
-    uint16_t l3_proto;
-
-    assert(pkt);
-
-    l2_hdr = &pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG];
-    l3_hdr = &pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG];
-
-    bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, 0, l2_hdr->iov_base,
-                            ETH_MAX_L2_HDR_LEN);
-    if (bytes_read < ETH_MAX_L2_HDR_LEN) {
-        l2_hdr->iov_len = 0;
-        return false;
-    } else {
-        l2_hdr->iov_len = eth_get_l2_hdr_length(l2_hdr->iov_base);
-    }
-
-    l3_proto = eth_get_l3_proto(l2_hdr->iov_base, l2_hdr->iov_len);
-
-    switch (l3_proto) {
-    case ETH_P_IP:
-        l3_hdr->iov_base = g_malloc(ETH_MAX_IP4_HDR_LEN);
-
-        bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
-                                l3_hdr->iov_base, sizeof(struct ip_header));
-
-        if (bytes_read < sizeof(struct ip_header)) {
-            l3_hdr->iov_len = 0;
-            return false;
-        }
-
-        l3_hdr->iov_len = IP_HDR_GET_LEN(l3_hdr->iov_base);
-        pkt->l4proto = ((struct ip_header *) l3_hdr->iov_base)->ip_p;
-
-        /* copy optional IPv4 header data */
-        bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags,
-                                l2_hdr->iov_len + sizeof(struct ip_header),
-                                l3_hdr->iov_base + sizeof(struct ip_header),
-                                l3_hdr->iov_len - sizeof(struct ip_header));
-        if (bytes_read < l3_hdr->iov_len - sizeof(struct ip_header)) {
-            l3_hdr->iov_len = 0;
-            return false;
-        }
-        break;
-
-    case ETH_P_IPV6:
-        if (!eth_parse_ipv6_hdr(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
-                               &pkt->l4proto, &full_ip6hdr_len)) {
-            l3_hdr->iov_len = 0;
-            return false;
-        }
-
-        l3_hdr->iov_base = g_malloc(full_ip6hdr_len);
-
-        bytes_read = iov_to_buf(pkt->raw, pkt->raw_frags, l2_hdr->iov_len,
-                                l3_hdr->iov_base, full_ip6hdr_len);
-
-        if (bytes_read < full_ip6hdr_len) {
-            l3_hdr->iov_len = 0;
-            return false;
-        } else {
-            l3_hdr->iov_len = full_ip6hdr_len;
-        }
-        break;
-
-    default:
-        l3_hdr->iov_len = 0;
-        break;
-    }
-
-    vmxnet_tx_pkt_calculate_hdr_len(pkt);
-    pkt->packet_type = get_eth_packet_type(l2_hdr->iov_base);
-    return true;
-}
-
-static bool vmxnet_tx_pkt_rebuild_payload(struct VmxnetTxPkt *pkt)
-{
-    size_t payload_len = iov_size(pkt->raw, pkt->raw_frags) - pkt->hdr_len;
-
-    pkt->payload_frags = iov_copy(&pkt->vec[VMXNET_TX_PKT_PL_START_FRAG],
-                                pkt->max_payload_frags,
-                                pkt->raw, pkt->raw_frags,
-                                pkt->hdr_len, payload_len);
-
-    if (pkt->payload_frags != (uint32_t) -1) {
-        pkt->payload_len = payload_len;
-        return true;
-    } else {
-        return false;
-    }
-}
-
-bool vmxnet_tx_pkt_parse(struct VmxnetTxPkt *pkt)
-{
-    return vmxnet_tx_pkt_parse_headers(pkt) &&
-           vmxnet_tx_pkt_rebuild_payload(pkt);
-}
-
-struct virtio_net_hdr *vmxnet_tx_pkt_get_vhdr(struct VmxnetTxPkt *pkt)
-{
-    assert(pkt);
-    return &pkt->virt_hdr;
-}
-
-static uint8_t vmxnet_tx_pkt_get_gso_type(struct VmxnetTxPkt *pkt,
-                                          bool tso_enable)
-{
-    uint8_t rc = VIRTIO_NET_HDR_GSO_NONE;
-    uint16_t l3_proto;
-
-    l3_proto = eth_get_l3_proto(pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_base,
-        pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_len);
-
-    if (!tso_enable) {
-        goto func_exit;
-    }
-
-    rc = eth_get_gso_type(l3_proto, pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base,
-                          pkt->l4proto);
-
-func_exit:
-    return rc;
-}
-
-void vmxnet_tx_pkt_build_vheader(struct VmxnetTxPkt *pkt, bool tso_enable,
-    bool csum_enable, uint32_t gso_size)
-{
-    struct tcp_hdr l4hdr;
-    assert(pkt);
-
-    /* csum has to be enabled if tso is. */
-    assert(csum_enable || !tso_enable);
-
-    pkt->virt_hdr.gso_type = vmxnet_tx_pkt_get_gso_type(pkt, tso_enable);
-
-    switch (pkt->virt_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
-    case VIRTIO_NET_HDR_GSO_NONE:
-        pkt->virt_hdr.hdr_len = 0;
-        pkt->virt_hdr.gso_size = 0;
-        break;
-
-    case VIRTIO_NET_HDR_GSO_UDP:
-        pkt->virt_hdr.gso_size = IP_FRAG_ALIGN_SIZE(gso_size);
-        pkt->virt_hdr.hdr_len = pkt->hdr_len + sizeof(struct udp_header);
-        break;
-
-    case VIRTIO_NET_HDR_GSO_TCPV4:
-    case VIRTIO_NET_HDR_GSO_TCPV6:
-        iov_to_buf(&pkt->vec[VMXNET_TX_PKT_PL_START_FRAG], pkt->payload_frags,
-                   0, &l4hdr, sizeof(l4hdr));
-        pkt->virt_hdr.hdr_len = pkt->hdr_len + l4hdr.th_off * sizeof(uint32_t);
-        pkt->virt_hdr.gso_size = IP_FRAG_ALIGN_SIZE(gso_size);
-        break;
-
-    default:
-        g_assert_not_reached();
-    }
-
-    if (csum_enable) {
-        switch (pkt->l4proto) {
-        case IP_PROTO_TCP:
-            pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-            pkt->virt_hdr.csum_start = pkt->hdr_len;
-            pkt->virt_hdr.csum_offset = offsetof(struct tcp_hdr, th_sum);
-            break;
-        case IP_PROTO_UDP:
-            pkt->virt_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
-            pkt->virt_hdr.csum_start = pkt->hdr_len;
-            pkt->virt_hdr.csum_offset = offsetof(struct udp_hdr, uh_sum);
-            break;
-        default:
-            break;
-        }
-    }
-}
-
-void vmxnet_tx_pkt_setup_vlan_header(struct VmxnetTxPkt *pkt, uint16_t vlan)
-{
-    bool is_new;
-    assert(pkt);
-
-    eth_setup_vlan_headers(pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_base,
-        vlan, &is_new);
-
-    /* update l2hdrlen */
-    if (is_new) {
-        pkt->hdr_len += sizeof(struct vlan_header);
-        pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_len +=
-            sizeof(struct vlan_header);
-    }
-}
-
-bool vmxnet_tx_pkt_add_raw_fragment(struct VmxnetTxPkt *pkt, hwaddr pa,
-    size_t len)
-{
-    hwaddr mapped_len = 0;
-    struct iovec *ventry;
-    assert(pkt);
-    assert(pkt->max_raw_frags > pkt->raw_frags);
-
-    if (!len) {
-        return true;
-     }
-
-    ventry = &pkt->raw[pkt->raw_frags];
-    mapped_len = len;
-
-    ventry->iov_base = cpu_physical_memory_map(pa, &mapped_len, false);
-    ventry->iov_len = mapped_len;
-    pkt->raw_frags += !!ventry->iov_base;
-
-    if ((ventry->iov_base == NULL) || (len != mapped_len)) {
-        return false;
-    }
-
-    return true;
-}
-
-eth_pkt_types_e vmxnet_tx_pkt_get_packet_type(struct VmxnetTxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->packet_type;
-}
-
-size_t vmxnet_tx_pkt_get_total_len(struct VmxnetTxPkt *pkt)
-{
-    assert(pkt);
-
-    return pkt->hdr_len + pkt->payload_len;
-}
-
-void vmxnet_tx_pkt_dump(struct VmxnetTxPkt *pkt)
-{
-#ifdef VMXNET_TX_PKT_DEBUG
-    assert(pkt);
-
-    printf("TX PKT: hdr_len: %d, pkt_type: 0x%X, l2hdr_len: %lu, "
-        "l3hdr_len: %lu, payload_len: %u\n", pkt->hdr_len, pkt->packet_type,
-        pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_len,
-        pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len, pkt->payload_len);
-#endif
-}
-
-void vmxnet_tx_pkt_reset(struct VmxnetTxPkt *pkt)
-{
-    int i;
-
-    /* no assert, as reset can be called before tx_pkt_init */
-    if (!pkt) {
-        return;
-    }
-
-    memset(&pkt->virt_hdr, 0, sizeof(pkt->virt_hdr));
-
-    g_free(pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base);
-    pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base = NULL;
-
-    assert(pkt->vec);
-    for (i = VMXNET_TX_PKT_L2HDR_FRAG;
-         i < pkt->payload_frags + VMXNET_TX_PKT_PL_START_FRAG; i++) {
-        pkt->vec[i].iov_len = 0;
-    }
-    pkt->payload_len = 0;
-    pkt->payload_frags = 0;
-
-    assert(pkt->raw);
-    for (i = 0; i < pkt->raw_frags; i++) {
-        assert(pkt->raw[i].iov_base);
-        cpu_physical_memory_unmap(pkt->raw[i].iov_base, pkt->raw[i].iov_len,
-                                  false, pkt->raw[i].iov_len);
-        pkt->raw[i].iov_len = 0;
-    }
-    pkt->raw_frags = 0;
-
-    pkt->hdr_len = 0;
-    pkt->packet_type = 0;
-    pkt->l4proto = 0;
-}
-
-static void vmxnet_tx_pkt_do_sw_csum(struct VmxnetTxPkt *pkt)
-{
-    struct iovec *iov = &pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG];
-    uint32_t csum_cntr;
-    uint16_t csum = 0;
-    /* num of iovec without vhdr */
-    uint32_t iov_len = pkt->payload_frags + VMXNET_TX_PKT_PL_START_FRAG - 1;
-    uint16_t csl;
-    struct ip_header *iphdr;
-    size_t csum_offset = pkt->virt_hdr.csum_start + pkt->virt_hdr.csum_offset;
-
-    /* Put zero to checksum field */
-    iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
-
-    /* Calculate L4 TCP/UDP checksum */
-    csl = pkt->payload_len;
-
-    /* data checksum */
-    csum_cntr =
-        net_checksum_add_iov(iov, iov_len, pkt->virt_hdr.csum_start, csl);
-    /* add pseudo header to csum */
-    iphdr = pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base;
-    csum_cntr += eth_calc_pseudo_hdr_csum(iphdr, csl);
-
-    /* Put the checksum obtained into the packet */
-    csum = cpu_to_be16(net_checksum_finish(csum_cntr));
-    iov_from_buf(iov, iov_len, csum_offset, &csum, sizeof csum);
-}
-
-enum {
-    VMXNET_TX_PKT_FRAGMENT_L2_HDR_POS = 0,
-    VMXNET_TX_PKT_FRAGMENT_L3_HDR_POS,
-    VMXNET_TX_PKT_FRAGMENT_HEADER_NUM
-};
-
-#define VMXNET_MAX_FRAG_SG_LIST (64)
-
-static size_t vmxnet_tx_pkt_fetch_fragment(struct VmxnetTxPkt *pkt,
-    int *src_idx, size_t *src_offset, struct iovec *dst, int *dst_idx)
-{
-    size_t fetched = 0;
-    struct iovec *src = pkt->vec;
-
-    *dst_idx = VMXNET_TX_PKT_FRAGMENT_HEADER_NUM;
-
-    while (fetched < pkt->virt_hdr.gso_size) {
-
-        /* no more place in fragment iov */
-        if (*dst_idx == VMXNET_MAX_FRAG_SG_LIST) {
-            break;
-        }
-
-        /* no more data in iovec */
-        if (*src_idx == (pkt->payload_frags + VMXNET_TX_PKT_PL_START_FRAG)) {
-            break;
-        }
-
-
-        dst[*dst_idx].iov_base = src[*src_idx].iov_base + *src_offset;
-        dst[*dst_idx].iov_len = MIN(src[*src_idx].iov_len - *src_offset,
-            pkt->virt_hdr.gso_size - fetched);
-
-        *src_offset += dst[*dst_idx].iov_len;
-        fetched += dst[*dst_idx].iov_len;
-
-        if (*src_offset == src[*src_idx].iov_len) {
-            *src_offset = 0;
-            (*src_idx)++;
-        }
-
-        (*dst_idx)++;
-    }
-
-    return fetched;
-}
-
-static bool vmxnet_tx_pkt_do_sw_fragmentation(struct VmxnetTxPkt *pkt,
-    NetClientState *nc)
-{
-    struct iovec fragment[VMXNET_MAX_FRAG_SG_LIST];
-    size_t fragment_len = 0;
-    bool more_frags = false;
-
-    /* some pointers for shorter code */
-    void *l2_iov_base, *l3_iov_base;
-    size_t l2_iov_len, l3_iov_len;
-    int src_idx =  VMXNET_TX_PKT_PL_START_FRAG, dst_idx;
-    size_t src_offset = 0;
-    size_t fragment_offset = 0;
-
-    l2_iov_base = pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_base;
-    l2_iov_len = pkt->vec[VMXNET_TX_PKT_L2HDR_FRAG].iov_len;
-    l3_iov_base = pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_base;
-    l3_iov_len = pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len;
-
-    /* Copy headers */
-    fragment[VMXNET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_base = l2_iov_base;
-    fragment[VMXNET_TX_PKT_FRAGMENT_L2_HDR_POS].iov_len = l2_iov_len;
-    fragment[VMXNET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_base = l3_iov_base;
-    fragment[VMXNET_TX_PKT_FRAGMENT_L3_HDR_POS].iov_len = l3_iov_len;
-
-
-    /* Put as much data as possible and send */
-    do {
-        fragment_len = vmxnet_tx_pkt_fetch_fragment(pkt, &src_idx, &src_offset,
-            fragment, &dst_idx);
-
-        more_frags = (fragment_offset + fragment_len < pkt->payload_len);
-
-        eth_setup_ip4_fragmentation(l2_iov_base, l2_iov_len, l3_iov_base,
-            l3_iov_len, fragment_len, fragment_offset, more_frags);
-
-        eth_fix_ip4_checksum(l3_iov_base, l3_iov_len);
-
-        qemu_sendv_packet(nc, fragment, dst_idx);
-
-        fragment_offset += fragment_len;
-
-    } while (more_frags);
-
-    return true;
-}
-
-bool vmxnet_tx_pkt_send(struct VmxnetTxPkt *pkt, NetClientState *nc)
-{
-    assert(pkt);
-
-    if (!pkt->has_virt_hdr &&
-        pkt->virt_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
-        vmxnet_tx_pkt_do_sw_csum(pkt);
-    }
-
-    /*
-     * Since underlying infrastructure does not support IP datagrams longer
-     * than 64K we should drop such packets and don't even try to send
-     */
-    if (VIRTIO_NET_HDR_GSO_NONE != pkt->virt_hdr.gso_type) {
-        if (pkt->payload_len >
-            ETH_MAX_IP_DGRAM_LEN -
-            pkt->vec[VMXNET_TX_PKT_L3HDR_FRAG].iov_len) {
-            return false;
-        }
-    }
-
-    if (pkt->has_virt_hdr ||
-        pkt->virt_hdr.gso_type == VIRTIO_NET_HDR_GSO_NONE) {
-        qemu_sendv_packet(nc, pkt->vec,
-            pkt->payload_frags + VMXNET_TX_PKT_PL_START_FRAG);
-        return true;
-    }
-
-    return vmxnet_tx_pkt_do_sw_fragmentation(pkt, nc);
-}
diff --git a/hw/net/vmxnet_tx_pkt.h b/hw/net/vmxnet_tx_pkt.h
deleted file mode 100644
index 57121a6..0000000
--- a/hw/net/vmxnet_tx_pkt.h
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * QEMU VMWARE VMXNET* paravirtual NICs - TX packets abstraction
- *
- * Copyright (c) 2012 Ravello Systems LTD (http://ravellosystems.com)
- *
- * Developed by Daynix Computing LTD (http://www.daynix.com)
- *
- * Authors:
- * Dmitry Fleytman <dmitry@daynix.com>
- * Tamir Shomer <tamirs@daynix.com>
- * Yan Vugenfirer <yan@daynix.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#ifndef VMXNET_TX_PKT_H
-#define VMXNET_TX_PKT_H
-
-#include "stdint.h"
-#include "stdbool.h"
-#include "net/eth.h"
-#include "exec/hwaddr.h"
-
-/* define to enable packet dump functions */
-/*#define VMXNET_TX_PKT_DEBUG*/
-
-struct VmxnetTxPkt;
-
-/**
- * Init function for tx packet functionality
- *
- * @pkt:            packet pointer
- * @max_frags:      max tx ip fragments
- * @has_virt_hdr:   device uses virtio header.
- */
-void vmxnet_tx_pkt_init(struct VmxnetTxPkt **pkt, uint32_t max_frags,
-    bool has_virt_hdr);
-
-/**
- * Clean all tx packet resources.
- *
- * @pkt:            packet.
- */
-void vmxnet_tx_pkt_uninit(struct VmxnetTxPkt *pkt);
-
-/**
- * get virtio header
- *
- * @pkt:            packet
- * @ret:            virtio header
- */
-struct virtio_net_hdr *vmxnet_tx_pkt_get_vhdr(struct VmxnetTxPkt *pkt);
-
-/**
- * build virtio header (will be stored in module context)
- *
- * @pkt:            packet
- * @tso_enable:     TSO enabled
- * @csum_enable:    CSO enabled
- * @gso_size:       MSS size for TSO
- *
- */
-void vmxnet_tx_pkt_build_vheader(struct VmxnetTxPkt *pkt, bool tso_enable,
-    bool csum_enable, uint32_t gso_size);
-
-/**
- * updates vlan tag, and adds vlan header in case it is missing
- *
- * @pkt:            packet
- * @vlan:           VLAN tag
- *
- */
-void vmxnet_tx_pkt_setup_vlan_header(struct VmxnetTxPkt *pkt, uint16_t vlan);
-
-/**
- * populate data fragment into pkt context.
- *
- * @pkt:            packet
- * @pa:             physical address of fragment
- * @len:            length of fragment
- *
- */
-bool vmxnet_tx_pkt_add_raw_fragment(struct VmxnetTxPkt *pkt, hwaddr pa,
-    size_t len);
-
-/**
- * fix ip header fields and calculate checksums needed.
- *
- * @pkt:            packet
- *
- */
-void vmxnet_tx_pkt_update_ip_checksums(struct VmxnetTxPkt *pkt);
-
-/**
- * get length of all populated data.
- *
- * @pkt:            packet
- * @ret:            total data length
- *
- */
-size_t vmxnet_tx_pkt_get_total_len(struct VmxnetTxPkt *pkt);
-
-/**
- * get packet type
- *
- * @pkt:            packet
- * @ret:            packet type
- *
- */
-eth_pkt_types_e vmxnet_tx_pkt_get_packet_type(struct VmxnetTxPkt *pkt);
-
-/**
- * prints packet data if debug is enabled
- *
- * @pkt:            packet
- *
- */
-void vmxnet_tx_pkt_dump(struct VmxnetTxPkt *pkt);
-
-/**
- * reset tx packet private context (needed to be called between packets)
- *
- * @pkt:            packet
- *
- */
-void vmxnet_tx_pkt_reset(struct VmxnetTxPkt *pkt);
-
-/**
- * Send packet to qemu. handles sw offloads if vhdr is not supported.
- *
- * @pkt:            packet
- * @nc:             NetClientState
- * @ret:            operation result
- *
- */
-bool vmxnet_tx_pkt_send(struct VmxnetTxPkt *pkt, NetClientState *nc);
-
-/**
- * parse raw packet data and analyze offload requirements.
- *
- * @pkt:            packet
- *
- */
-bool vmxnet_tx_pkt_parse(struct VmxnetTxPkt *pkt);
-
-#endif
diff --git a/hw/net/xen_nic.c b/hw/net/xen_nic.c
index 63918ae..6856b52 100644
--- a/hw/net/xen_nic.c
+++ b/hw/net/xen_nic.c
@@ -19,20 +19,9 @@
  *  GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <signal.h>
-#include <inttypes.h>
-#include <fcntl.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/socket.h>
 #include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
 #include <sys/wait.h>
 
 #include "hw/hw.h"
@@ -169,7 +158,7 @@
                           (txreq.flags & NETTXF_more_data)      ? " more_data"      : "",
                           (txreq.flags & NETTXF_extra_info)     ? " extra_info"     : "");
 
-            page = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+            page = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                            netdev->xendev.dom,
                                            txreq.gref, PROT_READ);
             if (page == NULL) {
@@ -191,7 +180,7 @@
                 qemu_send_packet(qemu_get_queue(netdev->nic),
                                  page + txreq.offset, txreq.size);
             }
-            xc_gnttab_munmap(netdev->xendev.gnttabdev, page, 1);
+            xengnttab_unmap(netdev->xendev.gnttabdev, page, 1);
             net_tx_response(netdev, &txreq, NETIF_RSP_OKAY);
         }
         if (!netdev->tx_work) {
@@ -234,27 +223,6 @@
 
 #define NET_IP_ALIGN 2
 
-static int net_rx_ok(NetClientState *nc)
-{
-    struct XenNetDev *netdev = qemu_get_nic_opaque(nc);
-    RING_IDX rc, rp;
-
-    if (netdev->xendev.be_state != XenbusStateConnected) {
-        return 0;
-    }
-
-    rc = netdev->rx_ring.req_cons;
-    rp = netdev->rx_ring.sring->req_prod;
-    xen_rmb();
-
-    if (rc == rp || RING_REQUEST_CONS_OVERFLOW(&netdev->rx_ring, rc)) {
-        xen_be_printf(&netdev->xendev, 2, "%s: no rx buffers (%d/%d)\n",
-                      __FUNCTION__, rc, rp);
-        return 0;
-    }
-    return 1;
-}
-
 static ssize_t net_rx_packet(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     struct XenNetDev *netdev = qemu_get_nic_opaque(nc);
@@ -271,8 +239,7 @@
     xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
 
     if (rc == rp || RING_REQUEST_CONS_OVERFLOW(&netdev->rx_ring, rc)) {
-        xen_be_printf(&netdev->xendev, 2, "no buffer, drop packet\n");
-        return -1;
+        return 0;
     }
     if (size > XC_PAGE_SIZE - NET_IP_ALIGN) {
         xen_be_printf(&netdev->xendev, 0, "packet too big (%lu > %ld)",
@@ -283,7 +250,7 @@
     memcpy(&rxreq, RING_GET_REQUEST(&netdev->rx_ring, rc), sizeof(rxreq));
     netdev->rx_ring.req_cons = ++rc;
 
-    page = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+    page = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                    netdev->xendev.dom,
                                    rxreq.gref, PROT_WRITE);
     if (page == NULL) {
@@ -293,7 +260,7 @@
         return -1;
     }
     memcpy(page + NET_IP_ALIGN, buf, size);
-    xc_gnttab_munmap(netdev->xendev.gnttabdev, page, 1);
+    xengnttab_unmap(netdev->xendev.gnttabdev, page, 1);
     net_rx_response(netdev, &rxreq, NETIF_RSP_OKAY, NET_IP_ALIGN, size, 0);
 
     return size;
@@ -302,9 +269,8 @@
 /* ------------------------------------------------------------- */
 
 static NetClientInfo net_xen_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = net_rx_ok,
     .receive = net_rx_packet,
 };
 
@@ -366,15 +332,20 @@
         return -1;
     }
 
-    netdev->txs = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+    netdev->txs = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                           netdev->xendev.dom,
                                           netdev->tx_ring_ref,
                                           PROT_READ | PROT_WRITE);
-    netdev->rxs = xc_gnttab_map_grant_ref(netdev->xendev.gnttabdev,
+    if (!netdev->txs) {
+        return -1;
+    }
+    netdev->rxs = xengnttab_map_grant_ref(netdev->xendev.gnttabdev,
                                           netdev->xendev.dom,
                                           netdev->rx_ring_ref,
                                           PROT_READ | PROT_WRITE);
-    if (!netdev->txs || !netdev->rxs) {
+    if (!netdev->rxs) {
+        xengnttab_unmap(netdev->xendev.gnttabdev, netdev->txs, 1);
+        netdev->txs = NULL;
         return -1;
     }
     BACK_RING_INIT(&netdev->tx_ring, netdev->txs, XC_PAGE_SIZE);
@@ -398,17 +369,13 @@
     xen_be_unbind_evtchn(&netdev->xendev);
 
     if (netdev->txs) {
-        xc_gnttab_munmap(netdev->xendev.gnttabdev, netdev->txs, 1);
+        xengnttab_unmap(netdev->xendev.gnttabdev, netdev->txs, 1);
         netdev->txs = NULL;
     }
     if (netdev->rxs) {
-        xc_gnttab_munmap(netdev->xendev.gnttabdev, netdev->rxs, 1);
+        xengnttab_unmap(netdev->xendev.gnttabdev, netdev->rxs, 1);
         netdev->rxs = NULL;
     }
-    if (netdev->nic) {
-        qemu_del_nic(netdev->nic);
-        netdev->nic = NULL;
-    }
 }
 
 static void net_event(struct XenDevice *xendev)
@@ -422,7 +389,12 @@
 {
     struct XenNetDev *netdev = container_of(xendev, struct XenNetDev, xendev);
 
+    if (netdev->nic) {
+        qemu_del_nic(netdev->nic);
+        netdev->nic = NULL;
+    }
     g_free(netdev->mac);
+    netdev->mac = NULL;
     return 0;
 }
 
diff --git a/hw/net/xgmac.c b/hw/net/xgmac.c
index aeffcb5..46b1aa1 100644
--- a/hw/net/xgmac.c
+++ b/hw/net/xgmac.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/char.h"
 #include "qemu/log.h"
@@ -312,10 +313,8 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int eth_can_rx(NetClientState *nc)
+static int eth_can_rx(XgmacState *s)
 {
-    XgmacState *s = qemu_get_nic_opaque(nc);
-
     /* RX enabled?  */
     return s->regs[DMA_CONTROL] & DMA_CONTROL_SR;
 }
@@ -329,6 +328,9 @@
     struct desc bd;
     ssize_t ret;
 
+    if (!eth_can_rx(s)) {
+        return -1;
+    }
     unicast = ~buf[0] & 0x1;
     broadcast = memcmp(buf, sa_bcast, 6) == 0;
     multicast = !unicast && !broadcast;
@@ -368,19 +370,10 @@
     return ret;
 }
 
-static void eth_cleanup(NetClientState *nc)
-{
-    XgmacState *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_xgmac_enet_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = eth_can_rx,
     .receive = eth_rx,
-    .cleanup = eth_cleanup,
 };
 
 static int xgmac_enet_init(SysBusDevice *sbd)
diff --git a/hw/net/xilinx_axienet.c b/hw/net/xilinx_axienet.c
index cd952d2..b670184 100644
--- a/hw/net/xilinx_axienet.c
+++ b/hw/net/xilinx_axienet.c
@@ -22,11 +22,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/log.h"
 #include "net/net.h"
 #include "net/checksum.h"
-#include "qapi/qmp/qerror.h"
 
 #include "hw/stream.h"
 
@@ -402,6 +403,9 @@
 
     uint8_t rxapp[CONTROL_PAYLOAD_SIZE];
     uint32_t rxappsize;
+
+    /* Whether axienet_eth_rx_notify should flush incoming queue. */
+    bool need_flush;
 };
 
 static void axienet_rx_reset(XilinxAXIEnet *s)
@@ -659,10 +663,8 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int eth_can_rx(NetClientState *nc)
+static int eth_can_rx(XilinxAXIEnet *s)
 {
-    XilinxAXIEnet *s = qemu_get_nic_opaque(nc);
-
     /* RX enabled?  */
     return !s->rxsize && !axienet_rx_resetting(s) && axienet_rx_enabled(s);
 }
@@ -702,6 +704,10 @@
         s->rxpos += ret;
         if (!s->rxsize) {
             s->regs[R_IS] |= IS_RX_COMPLETE;
+            if (s->need_flush) {
+                s->need_flush = false;
+                qemu_flush_queued_packets(qemu_get_queue(s->nic));
+            }
         }
     }
     enet_update_irq(s);
@@ -722,6 +728,11 @@
 
     DENET(qemu_log("%s: %zd bytes\n", __func__, size));
 
+    if (!eth_can_rx(s)) {
+        s->need_flush = true;
+        return 0;
+    }
+
     unicast = ~buf[0] & 0x1;
     broadcast = memcmp(buf, sa_bcast, 6) == 0;
     multicast = !unicast && !broadcast;
@@ -857,14 +868,6 @@
     return size;
 }
 
-static void eth_cleanup(NetClientState *nc)
-{
-    /* FIXME.  */
-    XilinxAXIEnet *s = qemu_get_nic_opaque(nc);
-    g_free(s->rxmem);
-    g_free(s);
-}
-
 static size_t
 xilinx_axienet_control_stream_push(StreamSlave *obj, uint8_t *buf, size_t len)
 {
@@ -932,11 +935,9 @@
 }
 
 static NetClientInfo net_xilinx_enet_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = eth_can_rx,
     .receive = eth_rx,
-    .cleanup = eth_cleanup,
 };
 
 static void xilinx_enet_realize(DeviceState *dev, Error **errp)
diff --git a/hw/net/xilinx_ethlite.c b/hw/net/xilinx_ethlite.c
index 1b177b3..35de353 100644
--- a/hw/net/xilinx_ethlite.c
+++ b/hw/net/xilinx_ethlite.c
@@ -22,6 +22,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h" /* FIXME should not use tswap* */
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "net/net.h"
@@ -146,6 +149,7 @@
             if (!(value & CTRL_S)) {
                 qemu_flush_queued_packets(qemu_get_queue(s->nic));
             }
+            /* fall through */
         case R_TX_LEN0:
         case R_TX_LEN1:
         case R_TX_GIE0:
@@ -193,6 +197,10 @@
     }
 
     D(qemu_log("%s %zd rxbase=%x\n", __func__, size, rxbase));
+    if (size > (R_MAX - R_RX_BUF0 - rxbase) * 4) {
+        D(qemu_log("ethlite packet is too big, size=%x\n", size));
+        return -1;
+    }
     memcpy(&s->regs[rxbase + R_RX_BUF0], buf, size);
 
     s->regs[rxbase + R_RX_CTRL0] |= CTRL_S;
@@ -212,19 +220,11 @@
     s->rxbuf = 0;
 }
 
-static void eth_cleanup(NetClientState *nc)
-{
-    struct xlx_ethlite *s = qemu_get_nic_opaque(nc);
-
-    s->nic = NULL;
-}
-
 static NetClientInfo net_xilinx_ethlite_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
     .can_receive = eth_can_rx,
     .receive = eth_rx,
-    .cleanup = eth_cleanup,
 };
 
 static void xilinx_ethlite_realize(DeviceState *dev, Error **errp)
diff --git a/hw/nvram/ds1225y.c b/hw/nvram/ds1225y.c
index 332598b..57d5ab2 100644
--- a/hw/nvram/ds1225y.c
+++ b/hw/nvram/ds1225y.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "trace.h"
 
diff --git a/hw/nvram/eeprom93xx.c b/hw/nvram/eeprom93xx.c
index 0af4d67..2c16fc2 100644
--- a/hw/nvram/eeprom93xx.c
+++ b/hw/nvram/eeprom93xx.c
@@ -35,6 +35,7 @@
  * - No emulation of EEPROM timings.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/nvram/eeprom93xx.h"
 
diff --git a/hw/nvram/fw_cfg.c b/hw/nvram/fw_cfg.c
index a7122ee..6a68e59 100644
--- a/hw/nvram/fw_cfg.c
+++ b/hw/nvram/fw_cfg.c
@@ -21,27 +21,47 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/dma.h"
+#include "hw/boards.h"
 #include "hw/isa/isa.h"
 #include "hw/nvram/fw_cfg.h"
 #include "hw/sysbus.h"
+#include "hw/boards.h"
 #include "trace.h"
 #include "qemu/error-report.h"
 #include "qemu/config-file.h"
+#include "qemu/cutils.h"
 
-#define FW_CFG_SIZE 2
-#define FW_CFG_DATA_SIZE 1
-#define TYPE_FW_CFG "fw_cfg"
 #define FW_CFG_NAME "fw_cfg"
 #define FW_CFG_PATH "/machine/" FW_CFG_NAME
-#define FW_CFG(obj) OBJECT_CHECK(FWCfgState, (obj), TYPE_FW_CFG)
+
+#define TYPE_FW_CFG     "fw_cfg"
+#define TYPE_FW_CFG_IO  "fw_cfg_io"
+#define TYPE_FW_CFG_MEM "fw_cfg_mem"
+
+#define FW_CFG(obj)     OBJECT_CHECK(FWCfgState,    (obj), TYPE_FW_CFG)
+#define FW_CFG_IO(obj)  OBJECT_CHECK(FWCfgIoState,  (obj), TYPE_FW_CFG_IO)
+#define FW_CFG_MEM(obj) OBJECT_CHECK(FWCfgMemState, (obj), TYPE_FW_CFG_MEM)
+
+/* FW_CFG_VERSION bits */
+#define FW_CFG_VERSION      0x01
+#define FW_CFG_VERSION_DMA  0x02
+
+/* FW_CFG_DMA_CONTROL bits */
+#define FW_CFG_DMA_CTL_ERROR   0x01
+#define FW_CFG_DMA_CTL_READ    0x02
+#define FW_CFG_DMA_CTL_SKIP    0x04
+#define FW_CFG_DMA_CTL_SELECT  0x08
+
+#define FW_CFG_DMA_SIGNATURE 0x51454d5520434647ULL /* "QEMU CFG" */
 
 typedef struct FWCfgEntry {
     uint32_t len;
     uint8_t *data;
     void *callback_opaque;
-    FWCfgCallback callback;
     FWCfgReadCallback read_callback;
 } FWCfgEntry;
 
@@ -50,13 +70,38 @@
     SysBusDevice parent_obj;
     /*< public >*/
 
-    MemoryRegion ctl_iomem, data_iomem, comb_iomem;
-    uint32_t ctl_iobase, data_iobase;
     FWCfgEntry entries[2][FW_CFG_MAX_ENTRY];
+    int entry_order[FW_CFG_MAX_ENTRY];
     FWCfgFiles *files;
     uint16_t cur_entry;
     uint32_t cur_offset;
     Notifier machine_ready;
+
+    int fw_cfg_order_override;
+
+    bool dma_enabled;
+    dma_addr_t dma_addr;
+    AddressSpace *dma_as;
+    MemoryRegion dma_iomem;
+};
+
+struct FWCfgIoState {
+    /*< private >*/
+    FWCfgState parent_obj;
+    /*< public >*/
+
+    MemoryRegion comb_iomem;
+    uint32_t iobase, dma_iobase;
+};
+
+struct FWCfgMemState {
+    /*< private >*/
+    FWCfgState parent_obj;
+    /*< public >*/
+
+    MemoryRegion ctl_iomem, data_iomem;
+    uint32_t data_width;
+    MemoryRegionOps wide_data_ops;
 };
 
 #define JPG_FILE 0
@@ -166,9 +211,7 @@
             g_free(filename);
             return;
         }
-        if (boot_splash_filedata != NULL) {
-            g_free(boot_splash_filedata);
-        }
+        g_free(boot_splash_filedata);
         boot_splash_filedata = (uint8_t *)file_data;
         boot_splash_filedata_size = file_size;
 
@@ -210,24 +253,13 @@
 
 static void fw_cfg_write(FWCfgState *s, uint8_t value)
 {
-    int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
-    FWCfgEntry *e = &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
-
-    trace_fw_cfg_write(s, value);
-
-    if (s->cur_entry & FW_CFG_WRITE_CHANNEL && e->callback &&
-        s->cur_offset < e->len) {
-        e->data[s->cur_offset++] = value;
-        if (s->cur_offset == e->len) {
-            e->callback(e->callback_opaque, e->data);
-            s->cur_offset = 0;
-        }
-    }
+    /* nothing, write support removed in QEMU v2.4+ */
 }
 
 static int fw_cfg_select(FWCfgState *s, uint16_t key)
 {
-    int ret;
+    int arch, ret;
+    FWCfgEntry *e;
 
     s->cur_offset = 0;
     if ((key & FW_CFG_ENTRY_MASK) >= FW_CFG_MAX_ENTRY) {
@@ -236,41 +268,183 @@
     } else {
         s->cur_entry = key;
         ret = 1;
+        /* entry successfully selected, now run callback if present */
+        arch = !!(key & FW_CFG_ARCH_LOCAL);
+        e = &s->entries[arch][key & FW_CFG_ENTRY_MASK];
+        if (e->read_callback) {
+            e->read_callback(e->callback_opaque);
+        }
     }
 
     trace_fw_cfg_select(s, key, ret);
     return ret;
 }
 
-static uint8_t fw_cfg_read(FWCfgState *s)
+static uint64_t fw_cfg_data_read(void *opaque, hwaddr addr, unsigned size)
 {
+    FWCfgState *s = opaque;
     int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
-    FWCfgEntry *e = &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
-    uint8_t ret;
+    FWCfgEntry *e = (s->cur_entry == FW_CFG_INVALID) ? NULL :
+                    &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
+    uint64_t value = 0;
 
-    if (s->cur_entry == FW_CFG_INVALID || !e->data || s->cur_offset >= e->len)
-        ret = 0;
-    else {
-        if (e->read_callback) {
-            e->read_callback(e->callback_opaque, s->cur_offset);
-        }
-        ret = e->data[s->cur_offset++];
+    assert(size > 0 && size <= sizeof(value));
+    if (s->cur_entry != FW_CFG_INVALID && e->data && s->cur_offset < e->len) {
+        /* The least significant 'size' bytes of the return value are
+         * expected to contain a string preserving portion of the item
+         * data, padded with zeros on the right in case we run out early.
+         * In technical terms, we're composing the host-endian representation
+         * of the big endian interpretation of the fw_cfg string.
+         */
+        do {
+            value = (value << 8) | e->data[s->cur_offset++];
+        } while (--size && s->cur_offset < e->len);
+        /* If size is still not zero, we *did* run out early, so continue
+         * left-shifting, to add the appropriate number of padding zeros
+         * on the right.
+         */
+        value <<= 8 * size;
     }
 
-    trace_fw_cfg_read(s, ret);
-    return ret;
-}
-
-static uint64_t fw_cfg_data_mem_read(void *opaque, hwaddr addr,
-                                     unsigned size)
-{
-    return fw_cfg_read(opaque);
+    trace_fw_cfg_read(s, value);
+    return value;
 }
 
 static void fw_cfg_data_mem_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
-    fw_cfg_write(opaque, (uint8_t)value);
+    FWCfgState *s = opaque;
+    unsigned i = size;
+
+    do {
+        fw_cfg_write(s, value >> (8 * --i));
+    } while (i);
+}
+
+static void fw_cfg_dma_transfer(FWCfgState *s)
+{
+    dma_addr_t len;
+    FWCfgDmaAccess dma;
+    int arch;
+    FWCfgEntry *e;
+    int read;
+    dma_addr_t dma_addr;
+
+    /* Reset the address before the next access */
+    dma_addr = s->dma_addr;
+    s->dma_addr = 0;
+
+    if (dma_memory_read(s->dma_as, dma_addr, &dma, sizeof(dma))) {
+        stl_be_dma(s->dma_as, dma_addr + offsetof(FWCfgDmaAccess, control),
+                   FW_CFG_DMA_CTL_ERROR);
+        return;
+    }
+
+    dma.address = be64_to_cpu(dma.address);
+    dma.length = be32_to_cpu(dma.length);
+    dma.control = be32_to_cpu(dma.control);
+
+    if (dma.control & FW_CFG_DMA_CTL_SELECT) {
+        fw_cfg_select(s, dma.control >> 16);
+    }
+
+    arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
+    e = (s->cur_entry == FW_CFG_INVALID) ? NULL :
+        &s->entries[arch][s->cur_entry & FW_CFG_ENTRY_MASK];
+
+    if (dma.control & FW_CFG_DMA_CTL_READ) {
+        read = 1;
+    } else if (dma.control & FW_CFG_DMA_CTL_SKIP) {
+        read = 0;
+    } else {
+        dma.length = 0;
+    }
+
+    dma.control = 0;
+
+    while (dma.length > 0 && !(dma.control & FW_CFG_DMA_CTL_ERROR)) {
+        if (s->cur_entry == FW_CFG_INVALID || !e->data ||
+                                s->cur_offset >= e->len) {
+            len = dma.length;
+
+            /* If the access is not a read access, it will be a skip access,
+             * tested before.
+             */
+            if (read) {
+                if (dma_memory_set(s->dma_as, dma.address, 0, len)) {
+                    dma.control |= FW_CFG_DMA_CTL_ERROR;
+                }
+            }
+
+        } else {
+            if (dma.length <= (e->len - s->cur_offset)) {
+                len = dma.length;
+            } else {
+                len = (e->len - s->cur_offset);
+            }
+
+            /* If the access is not a read access, it will be a skip access,
+             * tested before.
+             */
+            if (read) {
+                if (dma_memory_write(s->dma_as, dma.address,
+                                    &e->data[s->cur_offset], len)) {
+                    dma.control |= FW_CFG_DMA_CTL_ERROR;
+                }
+            }
+
+            s->cur_offset += len;
+        }
+
+        dma.address += len;
+        dma.length  -= len;
+
+    }
+
+    stl_be_dma(s->dma_as, dma_addr + offsetof(FWCfgDmaAccess, control),
+                dma.control);
+
+    trace_fw_cfg_read(s, 0);
+}
+
+static uint64_t fw_cfg_dma_mem_read(void *opaque, hwaddr addr,
+                                    unsigned size)
+{
+    /* Return a signature value (and handle various read sizes) */
+    return extract64(FW_CFG_DMA_SIGNATURE, (8 - addr - size) * 8, size * 8);
+}
+
+static void fw_cfg_dma_mem_write(void *opaque, hwaddr addr,
+                                 uint64_t value, unsigned size)
+{
+    FWCfgState *s = opaque;
+
+    if (size == 4) {
+        if (addr == 0) {
+            /* FWCfgDmaAccess high address */
+            s->dma_addr = value << 32;
+        } else if (addr == 4) {
+            /* FWCfgDmaAccess low address */
+            s->dma_addr |= value;
+            fw_cfg_dma_transfer(s);
+        }
+    } else if (size == 8 && addr == 0) {
+        s->dma_addr = value;
+        fw_cfg_dma_transfer(s);
+    }
+}
+
+static bool fw_cfg_dma_mem_valid(void *opaque, hwaddr addr,
+                                  unsigned size, bool is_write)
+{
+    return !is_write || ((size == 4 && (addr == 0 || addr == 4)) ||
+                         (size == 8 && addr == 0));
+}
+
+static bool fw_cfg_data_mem_valid(void *opaque, hwaddr addr,
+                                  unsigned size, bool is_write)
+{
+    return addr == 0;
 }
 
 static void fw_cfg_ctl_mem_write(void *opaque, hwaddr addr,
@@ -285,12 +459,6 @@
     return is_write && size == 2;
 }
 
-static uint64_t fw_cfg_comb_read(void *opaque, hwaddr addr,
-                                 unsigned size)
-{
-    return fw_cfg_read(opaque);
-}
-
 static void fw_cfg_comb_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
@@ -312,32 +480,43 @@
 
 static const MemoryRegionOps fw_cfg_ctl_mem_ops = {
     .write = fw_cfg_ctl_mem_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_BIG_ENDIAN,
     .valid.accepts = fw_cfg_ctl_mem_valid,
 };
 
 static const MemoryRegionOps fw_cfg_data_mem_ops = {
-    .read = fw_cfg_data_mem_read,
+    .read = fw_cfg_data_read,
     .write = fw_cfg_data_mem_write,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_BIG_ENDIAN,
     .valid = {
         .min_access_size = 1,
         .max_access_size = 1,
+        .accepts = fw_cfg_data_mem_valid,
     },
 };
 
 static const MemoryRegionOps fw_cfg_comb_mem_ops = {
-    .read = fw_cfg_comb_read,
+    .read = fw_cfg_data_read,
     .write = fw_cfg_comb_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .valid.accepts = fw_cfg_comb_valid,
 };
 
+static const MemoryRegionOps fw_cfg_dma_mem_ops = {
+    .read = fw_cfg_dma_mem_read,
+    .write = fw_cfg_dma_mem_write,
+    .endianness = DEVICE_BIG_ENDIAN,
+    .valid.accepts = fw_cfg_dma_mem_valid,
+    .valid.max_access_size = 8,
+    .impl.max_access_size = 8,
+};
+
 static void fw_cfg_reset(DeviceState *d)
 {
     FWCfgState *s = FW_CFG(d);
 
-    fw_cfg_select(s, 0);
+    /* we never register a read callback for FW_CFG_SIGNATURE */
+    fw_cfg_select(s, FW_CFG_SIGNATURE);
 }
 
 /* Save restore 32 bit int as uint16_t
@@ -373,6 +552,22 @@
     return version_id == 1;
 }
 
+bool fw_cfg_dma_enabled(void *opaque)
+{
+    FWCfgState *s = opaque;
+
+    return s->dma_enabled;
+}
+
+static const VMStateDescription vmstate_fw_cfg_dma = {
+    .name = "fw_cfg/dma",
+    .needed = fw_cfg_dma_enabled,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(dma_addr, FWCfgState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static const VMStateDescription vmstate_fw_cfg = {
     .name = "fw_cfg",
     .version_id = 2,
@@ -382,6 +577,10 @@
         VMSTATE_UINT16_HACK(cur_offset, FWCfgState, is_version_1),
         VMSTATE_UINT32_V(cur_offset, FWCfgState, 2),
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fw_cfg_dma,
+        NULL,
     }
 };
 
@@ -395,6 +594,7 @@
     key &= FW_CFG_ENTRY_MASK;
 
     assert(key < FW_CFG_MAX_ENTRY && len < UINT32_MAX);
+    assert(s->entries[arch][key].data == NULL); /* avoid key conflict */
 
     s->entries[arch][key].data = data;
     s->entries[arch][key].len = (uint32_t)len;
@@ -417,7 +617,6 @@
     s->entries[arch][key].data = data;
     s->entries[arch][key].len = len;
     s->entries[arch][key].callback_opaque = NULL;
-    s->entries[arch][key].callback = NULL;
 
     return ptr;
 }
@@ -431,7 +630,7 @@
 {
     size_t sz = strlen(value) + 1;
 
-    return fw_cfg_add_bytes(s, key, g_memdup(value, sz), sz);
+    fw_cfg_add_bytes(s, key, g_memdup(value, sz), sz);
 }
 
 void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value)
@@ -443,6 +642,16 @@
     fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
+void fw_cfg_modify_i16(FWCfgState *s, uint16_t key, uint16_t value)
+{
+    uint16_t *copy, *old;
+
+    copy = g_malloc(sizeof(value));
+    *copy = cpu_to_le16(value);
+    old = fw_cfg_modify_bytes_read(s, key, copy, sizeof(value));
+    g_free(old);
+}
+
 void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value)
 {
     uint32_t *copy;
@@ -461,29 +670,92 @@
     fw_cfg_add_bytes(s, key, copy, sizeof(value));
 }
 
-void fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
-                         void *callback_opaque, void *data, size_t len)
+void fw_cfg_set_order_override(FWCfgState *s, int order)
 {
-    int arch = !!(key & FW_CFG_ARCH_LOCAL);
+    assert(s->fw_cfg_order_override == 0);
+    s->fw_cfg_order_override = order;
+}
 
-    assert(key & FW_CFG_WRITE_CHANNEL);
+void fw_cfg_reset_order_override(FWCfgState *s)
+{
+    assert(s->fw_cfg_order_override != 0);
+    s->fw_cfg_order_override = 0;
+}
 
-    key &= FW_CFG_ENTRY_MASK;
+/*
+ * This is the legacy order list.  For legacy systems, files are in
+ * the fw_cfg in the order defined below, by the "order" value.  Note
+ * that some entries (VGA ROMs, NIC option ROMS, etc.) go into a
+ * specific area, but there may be more than one and they occur in the
+ * order that the user specifies them on the command line.  Those are
+ * handled in a special manner, using the order override above.
+ *
+ * For non-legacy, the files are sorted by filename to avoid this kind
+ * of complexity in the future.
+ *
+ * This is only for x86, other arches don't implement versioning so
+ * they won't set legacy mode.
+ */
+static struct {
+    const char *name;
+    int order;
+} fw_cfg_order[] = {
+    { "etc/boot-menu-wait", 10 },
+    { "bootsplash.jpg", 11 },
+    { "bootsplash.bmp", 12 },
+    { "etc/boot-fail-wait", 15 },
+    { "etc/smbios/smbios-tables", 20 },
+    { "etc/smbios/smbios-anchor", 30 },
+    { "etc/e820", 40 },
+    { "etc/reserved-memory-end", 50 },
+    { "genroms/kvmvapic.bin", 55 },
+    { "genroms/linuxboot.bin", 60 },
+    { }, /* VGA ROMs from pc_vga_init come here, 70. */
+    { }, /* NIC option ROMs from pc_nic_init come here, 80. */
+    { "etc/system-states", 90 },
+    { }, /* User ROMs come here, 100. */
+    { }, /* Device FW comes here, 110. */
+    { "etc/extra-pci-roots", 120 },
+    { "etc/acpi/tables", 130 },
+    { "etc/table-loader", 140 },
+    { "etc/tpm/log", 150 },
+    { "etc/acpi/rsdp", 160 },
+    { "bootorder", 170 },
 
-    assert(key < FW_CFG_MAX_ENTRY && len <= UINT32_MAX);
+#define FW_CFG_ORDER_OVERRIDE_LAST 200
+};
 
-    s->entries[arch][key].data = data;
-    s->entries[arch][key].len = (uint32_t)len;
-    s->entries[arch][key].callback_opaque = callback_opaque;
-    s->entries[arch][key].callback = callback;
+static int get_fw_cfg_order(FWCfgState *s, const char *name)
+{
+    int i;
+
+    if (s->fw_cfg_order_override > 0) {
+        return s->fw_cfg_order_override;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(fw_cfg_order); i++) {
+        if (fw_cfg_order[i].name == NULL) {
+            continue;
+        }
+
+        if (strcmp(name, fw_cfg_order[i].name) == 0) {
+            return fw_cfg_order[i].order;
+        }
+    }
+
+    /* Stick unknown stuff at the end. */
+    error_report("warning: Unknown firmware file in legacy mode: %s", name);
+    return FW_CFG_ORDER_OVERRIDE_LAST;
 }
 
 void fw_cfg_add_file_callback(FWCfgState *s,  const char *filename,
                               FWCfgReadCallback callback, void *callback_opaque,
                               void *data, size_t len)
 {
-    int i, index;
+    int i, index, count;
     size_t dsize;
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+    int order = 0;
 
     if (!s->files) {
         dsize = sizeof(uint32_t) + sizeof(FWCfgFile) * FW_CFG_FILE_SLOTS;
@@ -491,26 +763,63 @@
         fw_cfg_add_bytes(s, FW_CFG_FILE_DIR, s->files, dsize);
     }
 
-    index = be32_to_cpu(s->files->count);
-    assert(index < FW_CFG_FILE_SLOTS);
+    count = be32_to_cpu(s->files->count);
+    assert(count < FW_CFG_FILE_SLOTS);
+
+    /* Find the insertion point. */
+    if (mc->legacy_fw_cfg_order) {
+        /*
+         * Sort by order. For files with the same order, we keep them
+         * in the sequence in which they were added.
+         */
+        order = get_fw_cfg_order(s, filename);
+        for (index = count;
+             index > 0 && order < s->entry_order[index - 1];
+             index--);
+    } else {
+        /* Sort by file name. */
+        for (index = count;
+             index > 0 && strcmp(filename, s->files->f[index - 1].name) < 0;
+             index--);
+    }
+
+    /*
+     * Move all the entries from the index point and after down one
+     * to create a slot for the new entry.  Because calculations are
+     * being done with the index, make it so that "i" is the current
+     * index and "i - 1" is the one being copied from, thus the
+     * unusual start and end in the for statement.
+     */
+    for (i = count + 1; i > index; i--) {
+        s->files->f[i] = s->files->f[i - 1];
+        s->files->f[i].select = cpu_to_be16(FW_CFG_FILE_FIRST + i);
+        s->entries[0][FW_CFG_FILE_FIRST + i] =
+            s->entries[0][FW_CFG_FILE_FIRST + i - 1];
+        s->entry_order[i] = s->entry_order[i - 1];
+    }
+
+    memset(&s->files->f[index], 0, sizeof(FWCfgFile));
+    memset(&s->entries[0][FW_CFG_FILE_FIRST + index], 0, sizeof(FWCfgEntry));
+
+    pstrcpy(s->files->f[index].name, sizeof(s->files->f[index].name), filename);
+    for (i = 0; i <= count; i++) {
+        if (i != index &&
+            strcmp(s->files->f[index].name, s->files->f[i].name) == 0) {
+            error_report("duplicate fw_cfg file name: %s",
+                         s->files->f[index].name);
+            exit(1);
+        }
+    }
 
     fw_cfg_add_bytes_read_callback(s, FW_CFG_FILE_FIRST + index,
                                    callback, callback_opaque, data, len);
 
-    pstrcpy(s->files->f[index].name, sizeof(s->files->f[index].name),
-            filename);
-    for (i = 0; i < index; i++) {
-        if (strcmp(s->files->f[index].name, s->files->f[i].name) == 0) {
-            trace_fw_cfg_add_file_dupe(s, s->files->f[index].name);
-            return;
-        }
-    }
-
     s->files->f[index].size   = cpu_to_be32(len);
     s->files->f[index].select = cpu_to_be16(FW_CFG_FILE_FIRST + index);
+    s->entry_order[index] = order;
     trace_fw_cfg_add_file(s, index, s->files->f[index].name, len);
 
-    s->files->count = cpu_to_be32(index+1);
+    s->files->count = cpu_to_be32(count+1);
 }
 
 void fw_cfg_add_file(FWCfgState *s,  const char *filename,
@@ -560,35 +869,22 @@
     qemu_register_reset(fw_cfg_machine_reset, s);
 }
 
-FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
-                        hwaddr ctl_addr, hwaddr data_addr)
+
+
+static void fw_cfg_init1(DeviceState *dev)
 {
-    DeviceState *dev;
-    SysBusDevice *d;
-    FWCfgState *s;
-
-    dev = qdev_create(NULL, TYPE_FW_CFG);
-    qdev_prop_set_uint32(dev, "ctl_iobase", ctl_port);
-    qdev_prop_set_uint32(dev, "data_iobase", data_port);
-    d = SYS_BUS_DEVICE(dev);
-
-    s = FW_CFG(dev);
+    FWCfgState *s = FW_CFG(dev);
+    MachineState *machine = MACHINE(qdev_get_machine());
 
     assert(!object_resolve_path(FW_CFG_PATH, NULL));
 
-    object_property_add_child(qdev_get_machine(), FW_CFG_NAME, OBJECT(s), NULL);
+    object_property_add_child(OBJECT(machine), FW_CFG_NAME, OBJECT(s), NULL);
 
     qdev_init_nofail(dev);
 
-    if (ctl_addr) {
-        sysbus_mmio_map(d, 0, ctl_addr);
-    }
-    if (data_addr) {
-        sysbus_mmio_map(d, 1, data_addr);
-    }
     fw_cfg_add_bytes(s, FW_CFG_SIGNATURE, (char *)"QEMU", 4);
     fw_cfg_add_bytes(s, FW_CFG_UUID, qemu_uuid, 16);
-    fw_cfg_add_i16(s, FW_CFG_NOGRAPHIC, (uint16_t)(display_type == DT_NOGRAPHIC));
+    fw_cfg_add_i16(s, FW_CFG_NOGRAPHIC, (uint16_t)!machine->enable_graphics);
     fw_cfg_add_i16(s, FW_CFG_NB_CPUS, (uint16_t)smp_cpus);
     fw_cfg_add_i16(s, FW_CFG_BOOT_MENU, (uint16_t)boot_menu);
     fw_cfg_bootsplash(s);
@@ -596,49 +892,87 @@
 
     s->machine_ready.notify = fw_cfg_machine_ready;
     qemu_add_machine_init_done_notifier(&s->machine_ready);
+}
+
+FWCfgState *fw_cfg_init_io_dma(uint32_t iobase, uint32_t dma_iobase,
+                                AddressSpace *dma_as)
+{
+    DeviceState *dev;
+    FWCfgState *s;
+    uint32_t version = FW_CFG_VERSION;
+    bool dma_requested = dma_iobase && dma_as;
+
+    dev = qdev_create(NULL, TYPE_FW_CFG_IO);
+    qdev_prop_set_uint32(dev, "iobase", iobase);
+    qdev_prop_set_uint32(dev, "dma_iobase", dma_iobase);
+    if (!dma_requested) {
+        qdev_prop_set_bit(dev, "dma_enabled", false);
+    }
+
+    fw_cfg_init1(dev);
+    s = FW_CFG(dev);
+
+    if (s->dma_enabled) {
+        /* 64 bits for the address field */
+        s->dma_as = dma_as;
+        s->dma_addr = 0;
+
+        version |= FW_CFG_VERSION_DMA;
+    }
+
+    fw_cfg_add_i32(s, FW_CFG_ID, version);
 
     return s;
 }
 
-static void fw_cfg_initfn(Object *obj)
+FWCfgState *fw_cfg_init_io(uint32_t iobase)
 {
-    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
-    FWCfgState *s = FW_CFG(obj);
-
-    memory_region_init_io(&s->ctl_iomem, OBJECT(s), &fw_cfg_ctl_mem_ops, s,
-                          "fwcfg.ctl", FW_CFG_SIZE);
-    sysbus_init_mmio(sbd, &s->ctl_iomem);
-    memory_region_init_io(&s->data_iomem, OBJECT(s), &fw_cfg_data_mem_ops, s,
-                          "fwcfg.data", FW_CFG_DATA_SIZE);
-    sysbus_init_mmio(sbd, &s->data_iomem);
-    /* In case ctl and data overlap: */
-    memory_region_init_io(&s->comb_iomem, OBJECT(s), &fw_cfg_comb_mem_ops, s,
-                          "fwcfg", FW_CFG_SIZE);
+    return fw_cfg_init_io_dma(iobase, 0, NULL);
 }
 
-static void fw_cfg_realize(DeviceState *dev, Error **errp)
+FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr,
+                                 hwaddr data_addr, uint32_t data_width,
+                                 hwaddr dma_addr, AddressSpace *dma_as)
 {
-    FWCfgState *s = FW_CFG(dev);
-    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    DeviceState *dev;
+    SysBusDevice *sbd;
+    FWCfgState *s;
+    uint32_t version = FW_CFG_VERSION;
+    bool dma_requested = dma_addr && dma_as;
 
-
-    if (s->ctl_iobase + 1 == s->data_iobase) {
-        sysbus_add_io(sbd, s->ctl_iobase, &s->comb_iomem);
-    } else {
-        if (s->ctl_iobase) {
-            sysbus_add_io(sbd, s->ctl_iobase, &s->ctl_iomem);
-        }
-        if (s->data_iobase) {
-            sysbus_add_io(sbd, s->data_iobase, &s->data_iomem);
-        }
+    dev = qdev_create(NULL, TYPE_FW_CFG_MEM);
+    qdev_prop_set_uint32(dev, "data_width", data_width);
+    if (!dma_requested) {
+        qdev_prop_set_bit(dev, "dma_enabled", false);
     }
+
+    fw_cfg_init1(dev);
+
+    sbd = SYS_BUS_DEVICE(dev);
+    sysbus_mmio_map(sbd, 0, ctl_addr);
+    sysbus_mmio_map(sbd, 1, data_addr);
+
+    s = FW_CFG(dev);
+
+    if (s->dma_enabled) {
+        s->dma_as = dma_as;
+        s->dma_addr = 0;
+        sysbus_mmio_map(sbd, 2, dma_addr);
+        version |= FW_CFG_VERSION_DMA;
+    }
+
+    fw_cfg_add_i32(s, FW_CFG_ID, version);
+
+    return s;
 }
 
-static Property fw_cfg_properties[] = {
-    DEFINE_PROP_UINT32("ctl_iobase", FWCfgState, ctl_iobase, -1),
-    DEFINE_PROP_UINT32("data_iobase", FWCfgState, data_iobase, -1),
-    DEFINE_PROP_END_OF_LIST(),
-};
+FWCfgState *fw_cfg_init_mem(hwaddr ctl_addr, hwaddr data_addr)
+{
+    return fw_cfg_init_mem_wide(ctl_addr, data_addr,
+                                fw_cfg_data_mem_ops.valid.max_access_size,
+                                0, NULL);
+}
+
 
 FWCfgState *fw_cfg_find(void)
 {
@@ -649,23 +983,125 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    dc->realize = fw_cfg_realize;
     dc->reset = fw_cfg_reset;
     dc->vmsd = &vmstate_fw_cfg;
-    dc->props = fw_cfg_properties;
 }
 
 static const TypeInfo fw_cfg_info = {
     .name          = TYPE_FW_CFG,
     .parent        = TYPE_SYS_BUS_DEVICE,
+    .abstract      = true,
     .instance_size = sizeof(FWCfgState),
-    .instance_init = fw_cfg_initfn,
     .class_init    = fw_cfg_class_init,
 };
 
+
+static Property fw_cfg_io_properties[] = {
+    DEFINE_PROP_UINT32("iobase", FWCfgIoState, iobase, -1),
+    DEFINE_PROP_UINT32("dma_iobase", FWCfgIoState, dma_iobase, -1),
+    DEFINE_PROP_BOOL("dma_enabled", FWCfgIoState, parent_obj.dma_enabled,
+                     true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void fw_cfg_io_realize(DeviceState *dev, Error **errp)
+{
+    FWCfgIoState *s = FW_CFG_IO(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    /* when using port i/o, the 8-bit data register ALWAYS overlaps
+     * with half of the 16-bit control register. Hence, the total size
+     * of the i/o region used is FW_CFG_CTL_SIZE */
+    memory_region_init_io(&s->comb_iomem, OBJECT(s), &fw_cfg_comb_mem_ops,
+                          FW_CFG(s), "fwcfg", FW_CFG_CTL_SIZE);
+    sysbus_add_io(sbd, s->iobase, &s->comb_iomem);
+
+    if (FW_CFG(s)->dma_enabled) {
+        memory_region_init_io(&FW_CFG(s)->dma_iomem, OBJECT(s),
+                              &fw_cfg_dma_mem_ops, FW_CFG(s), "fwcfg.dma",
+                              sizeof(dma_addr_t));
+        sysbus_add_io(sbd, s->dma_iobase, &FW_CFG(s)->dma_iomem);
+    }
+}
+
+static void fw_cfg_io_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = fw_cfg_io_realize;
+    dc->props = fw_cfg_io_properties;
+}
+
+static const TypeInfo fw_cfg_io_info = {
+    .name          = TYPE_FW_CFG_IO,
+    .parent        = TYPE_FW_CFG,
+    .instance_size = sizeof(FWCfgIoState),
+    .class_init    = fw_cfg_io_class_init,
+};
+
+
+static Property fw_cfg_mem_properties[] = {
+    DEFINE_PROP_UINT32("data_width", FWCfgMemState, data_width, -1),
+    DEFINE_PROP_BOOL("dma_enabled", FWCfgMemState, parent_obj.dma_enabled,
+                     true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void fw_cfg_mem_realize(DeviceState *dev, Error **errp)
+{
+    FWCfgMemState *s = FW_CFG_MEM(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    const MemoryRegionOps *data_ops = &fw_cfg_data_mem_ops;
+
+    memory_region_init_io(&s->ctl_iomem, OBJECT(s), &fw_cfg_ctl_mem_ops,
+                          FW_CFG(s), "fwcfg.ctl", FW_CFG_CTL_SIZE);
+    sysbus_init_mmio(sbd, &s->ctl_iomem);
+
+    if (s->data_width > data_ops->valid.max_access_size) {
+        /* memberwise copy because the "old_mmio" member is const */
+        s->wide_data_ops.read       = data_ops->read;
+        s->wide_data_ops.write      = data_ops->write;
+        s->wide_data_ops.endianness = data_ops->endianness;
+        s->wide_data_ops.valid      = data_ops->valid;
+        s->wide_data_ops.impl       = data_ops->impl;
+
+        s->wide_data_ops.valid.max_access_size = s->data_width;
+        s->wide_data_ops.impl.max_access_size  = s->data_width;
+        data_ops = &s->wide_data_ops;
+    }
+    memory_region_init_io(&s->data_iomem, OBJECT(s), data_ops, FW_CFG(s),
+                          "fwcfg.data", data_ops->valid.max_access_size);
+    sysbus_init_mmio(sbd, &s->data_iomem);
+
+    if (FW_CFG(s)->dma_enabled) {
+        memory_region_init_io(&FW_CFG(s)->dma_iomem, OBJECT(s),
+                              &fw_cfg_dma_mem_ops, FW_CFG(s), "fwcfg.dma",
+                              sizeof(dma_addr_t));
+        sysbus_init_mmio(sbd, &FW_CFG(s)->dma_iomem);
+    }
+}
+
+static void fw_cfg_mem_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = fw_cfg_mem_realize;
+    dc->props = fw_cfg_mem_properties;
+}
+
+static const TypeInfo fw_cfg_mem_info = {
+    .name          = TYPE_FW_CFG_MEM,
+    .parent        = TYPE_FW_CFG,
+    .instance_size = sizeof(FWCfgMemState),
+    .class_init    = fw_cfg_mem_class_init,
+};
+
+
 static void fw_cfg_register_types(void)
 {
     type_register_static(&fw_cfg_info);
+    type_register_static(&fw_cfg_io_info);
+    type_register_static(&fw_cfg_mem_info);
 }
 
 type_init(fw_cfg_register_types)
diff --git a/hw/nvram/mac_nvram.c b/hw/nvram/mac_nvram.c
index d35f8a3..24f6121 100644
--- a/hw/nvram/mac_nvram.c
+++ b/hw/nvram/mac_nvram.c
@@ -22,10 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/nvram/openbios_firmware_abi.h"
 #include "sysemu/sysemu.h"
 #include "hw/ppc/mac.h"
+#include "qemu/cutils.h"
 #include <zlib.h>
 
 /* debug NVR */
@@ -48,7 +50,8 @@
 
     addr = (addr >> s->it_shift) & (s->size - 1);
     s->data[addr] = value;
-    NVR_DPRINTF("writeb addr %04" PHYS_PRIx " val %" PRIx64 "\n", addr, value);
+    NVR_DPRINTF("writeb addr %04" HWADDR_PRIx " val %" PRIx64 "\n",
+                addr, value);
 }
 
 static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
@@ -59,7 +62,8 @@
 
     addr = (addr >> s->it_shift) & (s->size - 1);
     value = s->data[addr];
-    NVR_DPRINTF("readb addr %04x val %x\n", (int)addr, value);
+    NVR_DPRINTF("readb addr %04" HWADDR_PRIx " val %" PRIx32 "\n",
+                addr, value);
 
     return value;
 }
@@ -123,6 +127,7 @@
     dc->reset = macio_nvram_reset;
     dc->vmsd = &vmstate_macio_nvram;
     dc->props = macio_nvram_properties;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo macio_nvram_type_info = {
diff --git a/hw/nvram/spapr_nvram.c b/hw/nvram/spapr_nvram.c
index 35dc6d5..4de5f70 100644
--- a/hw/nvram/spapr_nvram.c
+++ b/hw/nvram/spapr_nvram.c
@@ -22,6 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include <libfdt.h>
 
 #include "sysemu/block-backend.h"
@@ -35,6 +39,7 @@
     uint32_t size;
     uint8_t *buf;
     BlockBackend *blk;
+    VMChangeStateEntry *vmstate;
 } sPAPRNVRAM;
 
 #define TYPE_VIO_SPAPR_NVRAM "spapr-nvram"
@@ -45,7 +50,7 @@
 #define DEFAULT_NVRAM_SIZE 65536
 #define MAX_NVRAM_SIZE 1048576
 
-static void rtas_nvram_fetch(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_nvram_fetch(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                              uint32_t token, uint32_t nargs,
                              target_ulong args,
                              uint32_t nret, target_ulong rets)
@@ -86,7 +91,7 @@
     rtas_st(rets, 1, len);
 }
 
-static void rtas_nvram_store(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_nvram_store(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                              uint32_t token, uint32_t nargs,
                              target_ulong args,
                              uint32_t nret, target_ulong rets)
@@ -120,7 +125,7 @@
 
     alen = len;
     if (nvram->blk) {
-        alen = blk_pwrite(nvram->blk, offset, membuf, len);
+        alen = blk_pwrite(nvram->blk, offset, membuf, len, 0);
     }
 
     assert(nvram->buf);
@@ -132,7 +137,7 @@
     rtas_st(rets, 1, (alen < 0) ? 0 : alen);
 }
 
-static int spapr_nvram_init(VIOsPAPRDevice *dev)
+static void spapr_nvram_realize(VIOsPAPRDevice *dev, Error **errp)
 {
     sPAPRNVRAM *nvram = VIO_SPAPR_NVRAM(dev);
 
@@ -145,23 +150,22 @@
     nvram->buf = g_malloc0(nvram->size);
 
     if ((nvram->size < MIN_NVRAM_SIZE) || (nvram->size > MAX_NVRAM_SIZE)) {
-        fprintf(stderr, "spapr-nvram must be between %d and %d bytes in size\n",
-                MIN_NVRAM_SIZE, MAX_NVRAM_SIZE);
-        return -1;
+        error_setg(errp, "spapr-nvram must be between %d and %d bytes in size",
+                   MIN_NVRAM_SIZE, MAX_NVRAM_SIZE);
+        return;
     }
 
     if (nvram->blk) {
         int alen = blk_pread(nvram->blk, 0, nvram->buf, nvram->size);
 
         if (alen != nvram->size) {
-            return -1;
+            error_setg(errp, "can't read spapr-nvram contents");
+            return;
         }
     }
 
     spapr_rtas_register(RTAS_NVRAM_FETCH, "nvram-fetch", rtas_nvram_fetch);
     spapr_rtas_register(RTAS_NVRAM_STORE, "nvram-store", rtas_nvram_store);
-
-    return 0;
 }
 
 static int spapr_nvram_devnode(VIOsPAPRDevice *dev, void *fdt, int node_off)
@@ -182,19 +186,25 @@
     return 0;
 }
 
+static void postload_update_cb(void *opaque, int running, RunState state)
+{
+    sPAPRNVRAM *nvram = opaque;
+
+    /* This is called after bdrv_invalidate_cache_all.  */
+
+    qemu_del_vm_change_state_handler(nvram->vmstate);
+    nvram->vmstate = NULL;
+
+    blk_pwrite(nvram->blk, 0, nvram->buf, nvram->size, 0);
+}
+
 static int spapr_nvram_post_load(void *opaque, int version_id)
 {
     sPAPRNVRAM *nvram = VIO_SPAPR_NVRAM(opaque);
 
     if (nvram->blk) {
-        int alen = blk_pwrite(nvram->blk, 0, nvram->buf, nvram->size);
-
-        if (alen < 0) {
-            return alen;
-        }
-        if (alen != nvram->size) {
-            return -1;
-        }
+        nvram->vmstate = qemu_add_vm_change_state_handler(postload_update_cb,
+                                                          nvram);
     }
 
     return 0;
@@ -224,7 +234,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VIOsPAPRDeviceClass *k = VIO_SPAPR_DEVICE_CLASS(klass);
 
-    k->init = spapr_nvram_init;
+    k->realize = spapr_nvram_realize;
     k->devnode = spapr_nvram_devnode;
     k->dt_name = "nvram";
     k->dt_type = "nvram";
diff --git a/hw/nvram/trace-events b/hw/nvram/trace-events
new file mode 100644
index 0000000..1f1e05a
--- /dev/null
+++ b/hw/nvram/trace-events
@@ -0,0 +1,10 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/nvram/ds1225y.c
+nvram_read(uint32_t addr, uint32_t ret) "read addr %d: 0x%02x"
+nvram_write(uint32_t addr, uint32_t old, uint32_t val) "write addr %d: 0x%02x -> 0x%02x"
+
+# hw/nvram/fw_cfg.c
+fw_cfg_select(void *s, uint16_t key, int ret) "%p key %d = %d"
+fw_cfg_read(void *s, uint64_t ret) "%p = %"PRIx64
+fw_cfg_add_file(void *s, int index, char *name, size_t len) "%p #%d: %s (%zd bytes)"
diff --git a/hw/openrisc/cputimer.c b/hw/openrisc/cputimer.c
index 9c54945..a98c799 100644
--- a/hw/openrisc/cputimer.c
+++ b/hw/openrisc/cputimer.c
@@ -18,11 +18,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 
-#define TIMER_FREQ    (20 * 1000 * 1000)    /* 20MHz */
+#define TIMER_PERIOD 50 /* 50 ns period for 20 MHz timer */
 
 /* The time when TTCR changes */
 static uint64_t last_clk;
@@ -36,8 +37,7 @@
         return;
     }
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    cpu->env.ttcr += (uint32_t)muldiv64(now - last_clk, TIMER_FREQ,
-                                        get_ticks_per_sec());
+    cpu->env.ttcr += (uint32_t)((now - last_clk) / TIMER_PERIOD);
     last_clk = now;
 }
 
@@ -59,7 +59,7 @@
     } else {
         wait = (cpu->env.ttmr & TTMR_TP) - (cpu->env.ttcr & TTMR_TP);
     }
-    next = now + muldiv64(wait, get_ticks_per_sec(), TIMER_FREQ);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
     timer_mod(cpu->env.timer, next);
 }
 
diff --git a/hw/openrisc/openrisc_sim.c b/hw/openrisc/openrisc_sim.c
index 1da0657..6d06d5b 100644
--- a/hw/openrisc/openrisc_sim.c
+++ b/hw/openrisc/openrisc_sim.c
@@ -18,6 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "elf.h"
@@ -68,7 +72,8 @@
 
     if (kernel_filename && !qtest_enabled()) {
         kernel_size = load_elf(kernel_filename, NULL, NULL,
-                               &elf_entry, NULL, NULL, 1, ELF_MACHINE, 1);
+                               &elf_entry, NULL, NULL, 1, EM_OPENRISC,
+                               1, 0);
         entry = elf_entry;
         if (kernel_size < 0) {
             kernel_size = load_uimage(kernel_filename,
@@ -114,7 +119,7 @@
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "openrisc.ram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
@@ -132,17 +137,12 @@
     cpu_openrisc_load_kernel(ram_size, kernel_filename, cpu);
 }
 
-static QEMUMachine openrisc_sim_machine = {
-    .name = "or32-sim",
-    .desc = "or32 simulation",
-    .init = openrisc_sim_init,
-    .max_cpus = 1,
-    .is_default = 1,
-};
-
-static void openrisc_sim_machine_init(void)
+static void openrisc_sim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&openrisc_sim_machine);
+    mc->desc = "or32 simulation";
+    mc->init = openrisc_sim_init;
+    mc->max_cpus = 1;
+    mc->is_default = 1;
 }
 
-machine_init(openrisc_sim_machine_init);
+DEFINE_MACHINE("or32-sim", openrisc_sim_machine_init)
diff --git a/hw/openrisc/pic_cpu.c b/hw/openrisc/pic_cpu.c
index 2af1d60..569b443 100644
--- a/hw/openrisc/pic_cpu.c
+++ b/hw/openrisc/pic_cpu.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "cpu.h"
 
diff --git a/hw/pci-bridge/Makefile.objs b/hw/pci-bridge/Makefile.objs
index 968b369..f2adfe3 100644
--- a/hw/pci-bridge/Makefile.objs
+++ b/hw/pci-bridge/Makefile.objs
@@ -1,5 +1,7 @@
 common-obj-y += pci_bridge_dev.o
-common-obj-y += ioh3420.o xio3130_upstream.o xio3130_downstream.o
-common-obj-y += i82801b11.o
+common-obj-y += pci_expander_bridge.o
+common-obj-$(CONFIG_XIO3130) += xio3130_upstream.o xio3130_downstream.o
+common-obj-$(CONFIG_IOH3420) += ioh3420.o
+common-obj-$(CONFIG_I82801B11) += i82801b11.o
 # NewWorld PowerMac
 common-obj-$(CONFIG_DEC_PCI) += dec.o
diff --git a/hw/pci-bridge/dec.c b/hw/pci-bridge/dec.c
index a6ca940..840c961 100644
--- a/hw/pci-bridge/dec.c
+++ b/hw/pci-bridge/dec.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "dec.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
@@ -51,9 +52,9 @@
     return irq_num;
 }
 
-static int dec_pci_bridge_initfn(PCIDevice *pci_dev)
+static void dec_pci_bridge_realize(PCIDevice *pci_dev, Error **errp)
 {
-    return pci_bridge_initfn(pci_dev, TYPE_PCI_BUS);
+    pci_bridge_initfn(pci_dev, TYPE_PCI_BUS);
 }
 
 static void dec_21154_pci_bridge_class_init(ObjectClass *klass, void *data)
@@ -61,7 +62,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = dec_pci_bridge_initfn;
+    k->realize = dec_pci_bridge_realize;
     k->exit = pci_bridge_exitfn;
     k->vendor_id = PCI_VENDOR_ID_DEC;
     k->device_id = PCI_DEVICE_ID_DEC_21154;
@@ -107,10 +108,9 @@
     return 0;
 }
 
-static int dec_21154_pci_host_init(PCIDevice *d)
+static void dec_21154_pci_host_realize(PCIDevice *d, Error **errp)
 {
     /* PCI2PCI bridge same values as PearPC - check this */
-    return 0;
 }
 
 static void dec_21154_pci_host_class_init(ObjectClass *klass, void *data)
@@ -118,7 +118,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = dec_21154_pci_host_init;
+    k->realize = dec_21154_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_DEC;
     k->device_id = PCI_DEVICE_ID_DEC_21154;
     k->revision = 0x02;
diff --git a/hw/pci-bridge/dec.h b/hw/pci-bridge/dec.h
index 17dc0c2..ae17ca7 100644
--- a/hw/pci-bridge/dec.h
+++ b/hw/pci-bridge/dec.h
@@ -1,5 +1,5 @@
-#ifndef DEC_PCI_H
-#define DEC_PCI_H
+#ifndef HW_PCI_BRIDGE_DEC_H
+#define HW_PCI_BRIDGE_DEC_H
 
 #include "qemu-common.h"
 
diff --git a/hw/pci-bridge/i82801b11.c b/hw/pci-bridge/i82801b11.c
index 14cd7fd..2404e7e 100644
--- a/hw/pci-bridge/i82801b11.c
+++ b/hw/pci-bridge/i82801b11.c
@@ -41,6 +41,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/ich9.h"
 
@@ -61,10 +62,7 @@
 {
     int rc;
 
-    rc = pci_bridge_initfn(d, TYPE_PCI_BUS);
-    if (rc < 0) {
-        return rc;
-    }
+    pci_bridge_initfn(d, TYPE_PCI_BUS);
 
     rc = pci_bridge_ssvid_init(d, I82801ba_SSVID_OFFSET,
                                I82801ba_SSVID_SVID, I82801ba_SSVID_SSID);
@@ -80,6 +78,14 @@
     return rc;
 }
 
+static const VMStateDescription i82801b11_bridge_dev_vmstate = {
+    .name = "i82801b11_bridge",
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, PCIBridge),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void i82801b11_bridge_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
@@ -91,6 +97,7 @@
     k->revision = ICH9_D2P_A2_REVISION;
     k->init = i82801b11_bridge_initfn;
     k->config_write = pci_bridge_write_config;
+    dc->vmsd = &i82801b11_bridge_dev_vmstate;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
@@ -101,27 +108,6 @@
     .class_init    = i82801b11_bridge_class_init,
 };
 
-PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus)
-{
-    PCIDevice *d;
-    PCIBridge *br;
-    char buf[16];
-    DeviceState *qdev;
-
-    d = pci_create_multifunction(bus, devfn, true, "i82801b11-bridge");
-    if (!d) {
-        return NULL;
-    }
-    br = PCI_BRIDGE(d);
-    qdev = DEVICE(d);
-
-    snprintf(buf, sizeof(buf), "pci.%d", sec_bus);
-    pci_bridge_map_irq(br, buf, pci_swizzle_map_irq_fn);
-    qdev_init_nofail(qdev);
-
-    return pci_bridge_get_sec_bus(br);
-}
-
 static void d2pbr_register(void)
 {
     type_register_static(&i82801b11_bridge_info);
diff --git a/hw/pci-bridge/ioh3420.c b/hw/pci-bridge/ioh3420.c
index cce2fdd..c8b5ac4 100644
--- a/hw/pci-bridge/ioh3420.c
+++ b/hw/pci-bridge/ioh3420.c
@@ -20,10 +20,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
 #include "ioh3420.h"
+#include "qapi/error.h"
 
 #define PCI_DEVICE_ID_IOH_EPORT         0x3420  /* D0:F0 express mode */
 #define PCI_DEVICE_ID_IOH_REV           0x2
@@ -96,12 +98,10 @@
     PCIEPort *p = PCIE_PORT(d);
     PCIESlot *s = PCIE_SLOT(d);
     int rc;
+    Error *err = NULL;
 
-    rc = pci_bridge_initfn(d, TYPE_PCIE_BUS);
-    if (rc < 0) {
-        return rc;
-    }
-
+    pci_config_set_interrupt_pin(d->config, 1);
+    pci_bridge_initfn(d, TYPE_PCIE_BUS);
     pcie_port_init_reg(d);
 
     rc = pci_bridge_ssvid_init(d, IOH_EP_SSVID_OFFSET,
@@ -109,12 +109,16 @@
     if (rc < 0) {
         goto err_bridge;
     }
+
     rc = msi_init(d, IOH_EP_MSI_OFFSET, IOH_EP_MSI_NR_VECTOR,
                   IOH_EP_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_64BIT,
-                  IOH_EP_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT);
+                  IOH_EP_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT, &err);
     if (rc < 0) {
+        assert(rc == -ENOTSUP);
+        error_report_err(err);
         goto err_bridge;
     }
+
     rc = pcie_cap_init(d, IOH_EP_EXP_OFFSET, PCI_EXP_TYPE_ROOT_PORT, p->port);
     if (rc < 0) {
         goto err_msi;
@@ -123,18 +127,21 @@
     pcie_cap_arifwd_init(d);
     pcie_cap_deverr_init(d);
     pcie_cap_slot_init(d, s->slot);
+    pcie_cap_root_init(d);
+
     pcie_chassis_create(s->chassis);
     rc = pcie_chassis_add_slot(s);
     if (rc < 0) {
         goto err_pcie_cap;
     }
-    pcie_cap_root_init(d);
-    rc = pcie_aer_init(d, IOH_EP_AER_OFFSET);
+
+    rc = pcie_aer_init(d, IOH_EP_AER_OFFSET, PCI_ERR_SIZEOF);
     if (rc < 0) {
         goto err;
     }
     pcie_aer_root_init(d);
     ioh3420_aer_vector_update(d);
+
     return 0;
 
 err:
@@ -210,12 +217,3 @@
 }
 
 type_init(ioh3420_register_types)
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- *  tab-width: 8
- *  indent-tab-mode: nil
- * End:
- */
diff --git a/hw/pci-bridge/pci_bridge_dev.c b/hw/pci-bridge/pci_bridge_dev.c
index 252ea5e..5dbd933 100644
--- a/hw/pci-bridge/pci_bridge_dev.c
+++ b/hw/pci-bridge/pci_bridge_dev.c
@@ -19,6 +19,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
@@ -28,7 +30,8 @@
 #include "hw/pci/pci_bus.h"
 #include "hw/hotplug.h"
 
-#define TYPE_PCI_BRIDGE_DEV "pci-bridge"
+#define TYPE_PCI_BRIDGE_DEV      "pci-bridge"
+#define TYPE_PCI_BRIDGE_SEAT_DEV "pci-bridge-seat"
 #define PCI_BRIDGE_DEV(obj) \
     OBJECT_CHECK(PCIBridgeDev, (obj), TYPE_PCI_BRIDGE_DEV)
 
@@ -39,8 +42,10 @@
 
     MemoryRegion bar;
     uint8_t chassis_nr;
-#define PCI_BRIDGE_DEV_F_MSI_REQ 0
+#define PCI_BRIDGE_DEV_F_SHPC_REQ 0
     uint32_t flags;
+
+    OnOffAuto msi;
 };
 typedef struct PCIBridgeDev PCIBridgeDev;
 
@@ -49,40 +54,64 @@
     PCIBridge *br = PCI_BRIDGE(dev);
     PCIBridgeDev *bridge_dev = PCI_BRIDGE_DEV(dev);
     int err;
+    Error *local_err = NULL;
 
-    err = pci_bridge_initfn(dev, TYPE_PCI_BUS);
-    if (err) {
-        goto bridge_error;
+    pci_bridge_initfn(dev, TYPE_PCI_BUS);
+
+    if (bridge_dev->flags & (1 << PCI_BRIDGE_DEV_F_SHPC_REQ)) {
+        dev->config[PCI_INTERRUPT_PIN] = 0x1;
+        memory_region_init(&bridge_dev->bar, OBJECT(dev), "shpc-bar",
+                           shpc_bar_size(dev));
+        err = shpc_init(dev, &br->sec_bus, &bridge_dev->bar, 0);
+        if (err) {
+            goto shpc_error;
+        }
+    } else {
+        /* MSI is not applicable without SHPC */
+        bridge_dev->msi = ON_OFF_AUTO_OFF;
     }
-    dev->config[PCI_INTERRUPT_PIN] = 0x1;
-    memory_region_init(&bridge_dev->bar, OBJECT(dev), "shpc-bar", shpc_bar_size(dev));
-    err = shpc_init(dev, &br->sec_bus, &bridge_dev->bar, 0);
-    if (err) {
-        goto shpc_error;
-    }
+
     err = slotid_cap_init(dev, 0, bridge_dev->chassis_nr, 0);
     if (err) {
         goto slotid_error;
     }
-    if ((bridge_dev->flags & (1 << PCI_BRIDGE_DEV_F_MSI_REQ)) &&
-        msi_supported) {
-        err = msi_init(dev, 0, 1, true, true);
-        if (err < 0) {
+
+    if (bridge_dev->msi != ON_OFF_AUTO_OFF) {
+        /* it means SHPC exists, because MSI is needed by SHPC */
+
+        err = msi_init(dev, 0, 1, true, true, &local_err);
+        /* Any error other than -ENOTSUP(board's MSI support is broken)
+         * is a programming error */
+        assert(!err || err == -ENOTSUP);
+        if (err && bridge_dev->msi == ON_OFF_AUTO_ON) {
+            /* Can't satisfy user's explicit msi=on request, fail */
+            error_append_hint(&local_err, "You have to use msi=auto (default) "
+                    "or msi=off with this machine type.\n");
+            error_report_err(local_err);
             goto msi_error;
         }
+        assert(!local_err || bridge_dev->msi == ON_OFF_AUTO_AUTO);
+        /* With msi=auto, we fall back to MSI off silently */
+        error_free(local_err);
     }
-    /* TODO: spec recommends using 64 bit prefetcheable BAR.
-     * Check whether that works well. */
-    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
-		     PCI_BASE_ADDRESS_MEM_TYPE_64, &bridge_dev->bar);
+
+    if (shpc_present(dev)) {
+        /* TODO: spec recommends using 64 bit prefetcheable BAR.
+         * Check whether that works well. */
+        pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY |
+                         PCI_BASE_ADDRESS_MEM_TYPE_64, &bridge_dev->bar);
+    }
     return 0;
+
 msi_error:
     slotid_cap_cleanup(dev);
 slotid_error:
-    shpc_cleanup(dev, &bridge_dev->bar);
+    if (shpc_present(dev)) {
+        shpc_cleanup(dev, &bridge_dev->bar);
+    }
 shpc_error:
     pci_bridge_exitfn(dev);
-bridge_error:
+
     return err;
 }
 
@@ -93,10 +122,18 @@
         msi_uninit(dev);
     }
     slotid_cap_cleanup(dev);
-    shpc_cleanup(dev, &bridge_dev->bar);
+    if (shpc_present(dev)) {
+        shpc_cleanup(dev, &bridge_dev->bar);
+    }
     pci_bridge_exitfn(dev);
 }
 
+static void pci_bridge_dev_instance_finalize(Object *obj)
+{
+    /* this function is idempotent and handles (PCIDevice.shpc == NULL) */
+    shpc_free(PCI_DEVICE(obj));
+}
+
 static void pci_bridge_dev_write_config(PCIDevice *d,
                                         uint32_t address, uint32_t val, int len)
 {
@@ -104,7 +141,9 @@
     if (msi_present(d)) {
         msi_write_config(d, address, val, len);
     }
-    shpc_cap_write_config(d, address, val, len);
+    if (shpc_present(d)) {
+        shpc_cap_write_config(d, address, val, len);
+    }
 }
 
 static void qdev_pci_bridge_dev_reset(DeviceState *qdev)
@@ -112,25 +151,65 @@
     PCIDevice *dev = PCI_DEVICE(qdev);
 
     pci_bridge_reset(qdev);
-    shpc_reset(dev);
+    if (shpc_present(dev)) {
+        shpc_reset(dev);
+    }
 }
 
 static Property pci_bridge_dev_properties[] = {
                     /* Note: 0 is not a legal chassis number. */
-    DEFINE_PROP_UINT8("chassis_nr", PCIBridgeDev, chassis_nr, 0),
-    DEFINE_PROP_BIT("msi", PCIBridgeDev, flags, PCI_BRIDGE_DEV_F_MSI_REQ, true),
+    DEFINE_PROP_UINT8(PCI_BRIDGE_DEV_PROP_CHASSIS_NR, PCIBridgeDev, chassis_nr,
+                      0),
+    DEFINE_PROP_ON_OFF_AUTO(PCI_BRIDGE_DEV_PROP_MSI, PCIBridgeDev, msi,
+                            ON_OFF_AUTO_AUTO),
+    DEFINE_PROP_BIT(PCI_BRIDGE_DEV_PROP_SHPC, PCIBridgeDev, flags,
+                    PCI_BRIDGE_DEV_F_SHPC_REQ, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static bool pci_device_shpc_present(void *opaque, int version_id)
+{
+    PCIDevice *dev = opaque;
+
+    return shpc_present(dev);
+}
+
 static const VMStateDescription pci_bridge_dev_vmstate = {
     .name = "pci_bridge",
     .fields = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(parent_obj, PCIBridge),
-        SHPC_VMSTATE(shpc, PCIDevice),
+        SHPC_VMSTATE(shpc, PCIDevice, pci_device_shpc_present),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static void pci_bridge_dev_hotplug_cb(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    PCIDevice *pci_hotplug_dev = PCI_DEVICE(hotplug_dev);
+
+    if (!shpc_present(pci_hotplug_dev)) {
+        error_setg(errp, "standard hotplug controller has been disabled for "
+                   "this %s", TYPE_PCI_BRIDGE_DEV);
+        return;
+    }
+    shpc_device_hotplug_cb(hotplug_dev, dev, errp);
+}
+
+static void pci_bridge_dev_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                                 DeviceState *dev,
+                                                 Error **errp)
+{
+    PCIDevice *pci_hotplug_dev = PCI_DEVICE(hotplug_dev);
+
+    if (!shpc_present(pci_hotplug_dev)) {
+        error_setg(errp, "standard hotplug controller has been disabled for "
+                   "this %s", TYPE_PCI_BRIDGE_DEV);
+        return;
+    }
+    shpc_device_hot_unplug_request_cb(hotplug_dev, dev, errp);
+}
+
 static void pci_bridge_dev_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -149,24 +228,47 @@
     dc->props = pci_bridge_dev_properties;
     dc->vmsd = &pci_bridge_dev_vmstate;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-    hc->plug = shpc_device_hotplug_cb;
-    hc->unplug_request = shpc_device_hot_unplug_request_cb;
+    hc->plug = pci_bridge_dev_hotplug_cb;
+    hc->unplug_request = pci_bridge_dev_hot_unplug_request_cb;
 }
 
 static const TypeInfo pci_bridge_dev_info = {
-    .name          = TYPE_PCI_BRIDGE_DEV,
-    .parent        = TYPE_PCI_BRIDGE,
-    .instance_size = sizeof(PCIBridgeDev),
-    .class_init = pci_bridge_dev_class_init,
+    .name              = TYPE_PCI_BRIDGE_DEV,
+    .parent            = TYPE_PCI_BRIDGE,
+    .instance_size     = sizeof(PCIBridgeDev),
+    .class_init        = pci_bridge_dev_class_init,
+    .instance_finalize = pci_bridge_dev_instance_finalize,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_HOTPLUG_HANDLER },
         { }
     }
 };
 
+/*
+ * Multiseat bridge.  Same as the standard pci bridge, only with a
+ * different pci id, so we can match it easily in the guest for
+ * automagic multiseat configuration.  See docs/multiseat.txt for more.
+ */
+static void pci_bridge_dev_seat_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->device_id = PCI_DEVICE_ID_REDHAT_BRIDGE_SEAT;
+    dc->desc = "Standard PCI Bridge (multiseat)";
+}
+
+static const TypeInfo pci_bridge_dev_seat_info = {
+    .name              = TYPE_PCI_BRIDGE_SEAT_DEV,
+    .parent            = TYPE_PCI_BRIDGE_DEV,
+    .instance_size     = sizeof(PCIBridgeDev),
+    .class_init        = pci_bridge_dev_seat_class_init,
+};
+
 static void pci_bridge_dev_register(void)
 {
     type_register_static(&pci_bridge_dev_info);
+    type_register_static(&pci_bridge_dev_seat_info);
 }
 
 type_init(pci_bridge_dev_register);
diff --git a/hw/pci-bridge/pci_expander_bridge.c b/hw/pci-bridge/pci_expander_bridge.c
new file mode 100644
index 0000000..1cc598f
--- /dev/null
+++ b/hw/pci-bridge/pci_expander_bridge.c
@@ -0,0 +1,365 @@
+/*
+ * PCI Expander Bridge Device Emulation
+ *
+ * Copyright (C) 2015 Red Hat Inc
+ *
+ * Authors:
+ *   Marcel Apfelbaum <marcel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/pci_host.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/pci_bridge.h"
+#include "hw/i386/pc.h"
+#include "qemu/range.h"
+#include "qemu/error-report.h"
+#include "sysemu/numa.h"
+
+#define TYPE_PXB_BUS "pxb-bus"
+#define PXB_BUS(obj) OBJECT_CHECK(PXBBus, (obj), TYPE_PXB_BUS)
+
+#define TYPE_PXB_PCIE_BUS "pxb-pcie-bus"
+#define PXB_PCIE_BUS(obj) OBJECT_CHECK(PXBBus, (obj), TYPE_PXB_PCIE_BUS)
+
+typedef struct PXBBus {
+    /*< private >*/
+    PCIBus parent_obj;
+    /*< public >*/
+
+    char bus_path[8];
+} PXBBus;
+
+#define TYPE_PXB_DEVICE "pxb"
+#define PXB_DEV(obj) OBJECT_CHECK(PXBDev, (obj), TYPE_PXB_DEVICE)
+
+#define TYPE_PXB_PCIE_DEVICE "pxb-pcie"
+#define PXB_PCIE_DEV(obj) OBJECT_CHECK(PXBDev, (obj), TYPE_PXB_PCIE_DEVICE)
+
+typedef struct PXBDev {
+    /*< private >*/
+    PCIDevice parent_obj;
+    /*< public >*/
+
+    uint8_t bus_nr;
+    uint16_t numa_node;
+} PXBDev;
+
+static PXBDev *convert_to_pxb(PCIDevice *dev)
+{
+    return pci_bus_is_express(dev->bus) ? PXB_PCIE_DEV(dev) : PXB_DEV(dev);
+}
+
+static GList *pxb_dev_list;
+
+#define TYPE_PXB_HOST "pxb-host"
+
+static int pxb_bus_num(PCIBus *bus)
+{
+    PXBDev *pxb = convert_to_pxb(bus->parent_dev);
+
+    return pxb->bus_nr;
+}
+
+static bool pxb_is_root(PCIBus *bus)
+{
+    return true; /* by definition */
+}
+
+static uint16_t pxb_bus_numa_node(PCIBus *bus)
+{
+    PXBDev *pxb = convert_to_pxb(bus->parent_dev);
+
+    return pxb->numa_node;
+}
+
+static void pxb_bus_class_init(ObjectClass *class, void *data)
+{
+    PCIBusClass *pbc = PCI_BUS_CLASS(class);
+
+    pbc->bus_num = pxb_bus_num;
+    pbc->is_root = pxb_is_root;
+    pbc->numa_node = pxb_bus_numa_node;
+}
+
+static const TypeInfo pxb_bus_info = {
+    .name          = TYPE_PXB_BUS,
+    .parent        = TYPE_PCI_BUS,
+    .instance_size = sizeof(PXBBus),
+    .class_init    = pxb_bus_class_init,
+};
+
+static const TypeInfo pxb_pcie_bus_info = {
+    .name          = TYPE_PXB_PCIE_BUS,
+    .parent        = TYPE_PCIE_BUS,
+    .instance_size = sizeof(PXBBus),
+    .class_init    = pxb_bus_class_init,
+};
+
+static const char *pxb_host_root_bus_path(PCIHostState *host_bridge,
+                                          PCIBus *rootbus)
+{
+    PXBBus *bus = pci_bus_is_express(rootbus) ?
+                  PXB_PCIE_BUS(rootbus) : PXB_BUS(rootbus);
+
+    snprintf(bus->bus_path, 8, "0000:%02x", pxb_bus_num(rootbus));
+    return bus->bus_path;
+}
+
+static char *pxb_host_ofw_unit_address(const SysBusDevice *dev)
+{
+    const PCIHostState *pxb_host;
+    const PCIBus *pxb_bus;
+    const PXBDev *pxb_dev;
+    int position;
+    const DeviceState *pxb_dev_base;
+    const PCIHostState *main_host;
+    const SysBusDevice *main_host_sbd;
+
+    pxb_host = PCI_HOST_BRIDGE(dev);
+    pxb_bus = pxb_host->bus;
+    pxb_dev = convert_to_pxb(pxb_bus->parent_dev);
+    position = g_list_index(pxb_dev_list, pxb_dev);
+    assert(position >= 0);
+
+    pxb_dev_base = DEVICE(pxb_dev);
+    main_host = PCI_HOST_BRIDGE(pxb_dev_base->parent_bus->parent);
+    main_host_sbd = SYS_BUS_DEVICE(main_host);
+
+    if (main_host_sbd->num_mmio > 0) {
+        return g_strdup_printf(TARGET_FMT_plx ",%x",
+                               main_host_sbd->mmio[0].addr, position + 1);
+    }
+    if (main_host_sbd->num_pio > 0) {
+        return g_strdup_printf("i%04x,%x",
+                               main_host_sbd->pio[0], position + 1);
+    }
+    return NULL;
+}
+
+static void pxb_host_class_init(ObjectClass *class, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(class);
+    SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(class);
+    PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class);
+
+    dc->fw_name = "pci";
+    /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */
+    dc->cannot_instantiate_with_device_add_yet = true;
+    sbc->explicit_ofw_unit_address = pxb_host_ofw_unit_address;
+    hc->root_bus_path = pxb_host_root_bus_path;
+}
+
+static const TypeInfo pxb_host_info = {
+    .name          = TYPE_PXB_HOST,
+    .parent        = TYPE_PCI_HOST_BRIDGE,
+    .class_init    = pxb_host_class_init,
+};
+
+/*
+ * Registers the PXB bus as a child of pci host root bus.
+ */
+static void pxb_register_bus(PCIDevice *dev, PCIBus *pxb_bus, Error **errp)
+{
+    PCIBus *bus = dev->bus;
+    int pxb_bus_num = pci_bus_num(pxb_bus);
+
+    if (bus->parent_dev) {
+        error_setg(errp, "PXB devices can be attached only to root bus");
+        return;
+    }
+
+    QLIST_FOREACH(bus, &bus->child, sibling) {
+        if (pci_bus_num(bus) == pxb_bus_num) {
+            error_setg(errp, "Bus %d is already in use", pxb_bus_num);
+            return;
+        }
+    }
+    QLIST_INSERT_HEAD(&dev->bus->child, pxb_bus, sibling);
+}
+
+static int pxb_map_irq_fn(PCIDevice *pci_dev, int pin)
+{
+    PCIDevice *pxb = pci_dev->bus->parent_dev;
+
+    /*
+     * The bios does not index the pxb slot number when
+     * it computes the IRQ because it resides on bus 0
+     * and not on the current bus.
+     * However QEMU routes the irq through bus 0 and adds
+     * the pxb slot to the IRQ computation of the PXB
+     * device.
+     *
+     * Synchronize between bios and QEMU by canceling
+     * pxb's effect.
+     */
+    return pin - PCI_SLOT(pxb->devfn);
+}
+
+static gint pxb_compare(gconstpointer a, gconstpointer b)
+{
+    const PXBDev *pxb_a = a, *pxb_b = b;
+
+    return pxb_a->bus_nr < pxb_b->bus_nr ? -1 :
+           pxb_a->bus_nr > pxb_b->bus_nr ?  1 :
+           0;
+}
+
+static void pxb_dev_realize_common(PCIDevice *dev, bool pcie, Error **errp)
+{
+    PXBDev *pxb = convert_to_pxb(dev);
+    DeviceState *ds, *bds = NULL;
+    PCIBus *bus;
+    const char *dev_name = NULL;
+    Error *local_err = NULL;
+
+    if (pxb->numa_node != NUMA_NODE_UNASSIGNED &&
+        pxb->numa_node >= nb_numa_nodes) {
+        error_setg(errp, "Illegal numa node %d", pxb->numa_node);
+        return;
+    }
+
+    if (dev->qdev.id && *dev->qdev.id) {
+        dev_name = dev->qdev.id;
+    }
+
+    ds = qdev_create(NULL, TYPE_PXB_HOST);
+    if (pcie) {
+        bus = pci_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_PCIE_BUS);
+    } else {
+        bus = pci_bus_new(ds, "pxb-internal", NULL, NULL, 0, TYPE_PXB_BUS);
+        bds = qdev_create(BUS(bus), "pci-bridge");
+        bds->id = dev_name;
+        qdev_prop_set_uint8(bds, PCI_BRIDGE_DEV_PROP_CHASSIS_NR, pxb->bus_nr);
+        qdev_prop_set_bit(bds, PCI_BRIDGE_DEV_PROP_SHPC, false);
+    }
+
+    bus->parent_dev = dev;
+    bus->address_space_mem = dev->bus->address_space_mem;
+    bus->address_space_io = dev->bus->address_space_io;
+    bus->map_irq = pxb_map_irq_fn;
+
+    PCI_HOST_BRIDGE(ds)->bus = bus;
+
+    pxb_register_bus(dev, bus, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto err_register_bus;
+    }
+
+    qdev_init_nofail(ds);
+    if (bds) {
+        qdev_init_nofail(bds);
+    }
+
+    pci_word_test_and_set_mask(dev->config + PCI_STATUS,
+                               PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK);
+    pci_config_set_class(dev->config, PCI_CLASS_BRIDGE_HOST);
+
+    pxb_dev_list = g_list_insert_sorted(pxb_dev_list, pxb, pxb_compare);
+    return;
+
+err_register_bus:
+    object_unref(OBJECT(bds));
+    object_unparent(OBJECT(bus));
+    object_unref(OBJECT(ds));
+}
+
+static void pxb_dev_realize(PCIDevice *dev, Error **errp)
+{
+    if (pci_bus_is_express(dev->bus)) {
+        error_setg(errp, "pxb devices cannot reside on a PCIe bus");
+        return;
+    }
+
+    pxb_dev_realize_common(dev, false, errp);
+}
+
+static void pxb_dev_exitfn(PCIDevice *pci_dev)
+{
+    PXBDev *pxb = convert_to_pxb(pci_dev);
+
+    pxb_dev_list = g_list_remove(pxb_dev_list, pxb);
+}
+
+static Property pxb_dev_properties[] = {
+    /* Note: 0 is not a legal PXB bus number. */
+    DEFINE_PROP_UINT8("bus_nr", PXBDev, bus_nr, 0),
+    DEFINE_PROP_UINT16("numa_node", PXBDev, numa_node, NUMA_NODE_UNASSIGNED),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void pxb_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = pxb_dev_realize;
+    k->exit = pxb_dev_exitfn;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_PXB;
+    k->class_id = PCI_CLASS_BRIDGE_HOST;
+
+    dc->desc = "PCI Expander Bridge";
+    dc->props = pxb_dev_properties;
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo pxb_dev_info = {
+    .name          = TYPE_PXB_DEVICE,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PXBDev),
+    .class_init    = pxb_dev_class_init,
+};
+
+static void pxb_pcie_dev_realize(PCIDevice *dev, Error **errp)
+{
+    if (!pci_bus_is_express(dev->bus)) {
+        error_setg(errp, "pxb-pcie devices cannot reside on a PCI bus");
+        return;
+    }
+
+    pxb_dev_realize_common(dev, true, errp);
+}
+
+static void pxb_pcie_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = pxb_pcie_dev_realize;
+    k->exit = pxb_dev_exitfn;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_PXB_PCIE;
+    k->class_id = PCI_CLASS_BRIDGE_HOST;
+
+    dc->desc = "PCI Express Expander Bridge";
+    dc->props = pxb_dev_properties;
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+}
+
+static const TypeInfo pxb_pcie_dev_info = {
+    .name          = TYPE_PXB_PCIE_DEVICE,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PXBDev),
+    .class_init    = pxb_pcie_dev_class_init,
+};
+
+static void pxb_register_types(void)
+{
+    type_register_static(&pxb_bus_info);
+    type_register_static(&pxb_pcie_bus_info);
+    type_register_static(&pxb_host_info);
+    type_register_static(&pxb_dev_info);
+    type_register_static(&pxb_pcie_dev_info);
+}
+
+type_init(pxb_register_types)
diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
index b3a6479..cef6e13 100644
--- a/hw/pci-bridge/xio3130_downstream.c
+++ b/hw/pci-bridge/xio3130_downstream.c
@@ -19,10 +19,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
 #include "xio3130_downstream.h"
+#include "qapi/error.h"
 
 #define PCI_DEVICE_ID_TI_XIO3130D       0x8233  /* downstream port */
 #define XIO3130_REVISION                0x1
@@ -59,25 +61,26 @@
     PCIEPort *p = PCIE_PORT(d);
     PCIESlot *s = PCIE_SLOT(d);
     int rc;
+    Error *err = NULL;
 
-    rc = pci_bridge_initfn(d, TYPE_PCIE_BUS);
-    if (rc < 0) {
-        return rc;
-    }
-
+    pci_bridge_initfn(d, TYPE_PCIE_BUS);
     pcie_port_init_reg(d);
 
     rc = msi_init(d, XIO3130_MSI_OFFSET, XIO3130_MSI_NR_VECTOR,
                   XIO3130_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_64BIT,
-                  XIO3130_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT);
+                  XIO3130_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT, &err);
     if (rc < 0) {
+        assert(rc == -ENOTSUP);
+        error_report_err(err);
         goto err_bridge;
     }
+
     rc = pci_bridge_ssvid_init(d, XIO3130_SSVID_OFFSET,
                                XIO3130_SSVID_SVID, XIO3130_SSVID_SSID);
     if (rc < 0) {
         goto err_bridge;
     }
+
     rc = pcie_cap_init(d, XIO3130_EXP_OFFSET, PCI_EXP_TYPE_DOWNSTREAM,
                        p->port);
     if (rc < 0) {
@@ -86,13 +89,15 @@
     pcie_cap_flr_init(d);
     pcie_cap_deverr_init(d);
     pcie_cap_slot_init(d, s->slot);
+    pcie_cap_arifwd_init(d);
+
     pcie_chassis_create(s->chassis);
     rc = pcie_chassis_add_slot(s);
     if (rc < 0) {
         goto err_pcie_cap;
     }
-    pcie_cap_arifwd_init(d);
-    rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
+
+    rc = pcie_aer_init(d, XIO3130_AER_OFFSET, PCI_ERR_SIZEOF);
     if (rc < 0) {
         goto err;
     }
@@ -198,12 +203,3 @@
 }
 
 type_init(xio3130_downstream_register_types)
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- *  tab-width: 8
- *  indent-tab-mode: nil
- * End:
- */
diff --git a/hw/pci-bridge/xio3130_upstream.c b/hw/pci-bridge/xio3130_upstream.c
index eada582..4ad0440 100644
--- a/hw/pci-bridge/xio3130_upstream.c
+++ b/hw/pci-bridge/xio3130_upstream.c
@@ -19,10 +19,12 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/pcie.h"
 #include "xio3130_upstream.h"
+#include "qapi/error.h"
 
 #define PCI_DEVICE_ID_TI_XIO3130U       0x8232  /* upstream port */
 #define XIO3130_REVISION                0x2
@@ -55,25 +57,26 @@
 {
     PCIEPort *p = PCIE_PORT(d);
     int rc;
+    Error *err = NULL;
 
-    rc = pci_bridge_initfn(d, TYPE_PCIE_BUS);
-    if (rc < 0) {
-        return rc;
-    }
-
+    pci_bridge_initfn(d, TYPE_PCIE_BUS);
     pcie_port_init_reg(d);
 
     rc = msi_init(d, XIO3130_MSI_OFFSET, XIO3130_MSI_NR_VECTOR,
                   XIO3130_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_64BIT,
-                  XIO3130_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT);
+                  XIO3130_MSI_SUPPORTED_FLAGS & PCI_MSI_FLAGS_MASKBIT, &err);
     if (rc < 0) {
+        assert(rc == -ENOTSUP);
+        error_report_err(err);
         goto err_bridge;
     }
+
     rc = pci_bridge_ssvid_init(d, XIO3130_SSVID_OFFSET,
                                XIO3130_SSVID_SVID, XIO3130_SSVID_SSID);
     if (rc < 0) {
         goto err_bridge;
     }
+
     rc = pcie_cap_init(d, XIO3130_EXP_OFFSET, PCI_EXP_TYPE_UPSTREAM,
                        p->port);
     if (rc < 0) {
@@ -81,7 +84,8 @@
     }
     pcie_cap_flr_init(d);
     pcie_cap_deverr_init(d);
-    rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
+
+    rc = pcie_aer_init(d, XIO3130_AER_OFFSET, PCI_ERR_SIZEOF);
     if (rc < 0) {
         goto err;
     }
@@ -170,13 +174,3 @@
 }
 
 type_init(xio3130_upstream_register_types)
-
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- *  tab-width: 8
- *  indent-tab-mode: nil
- * End:
- */
diff --git a/hw/pci-bridge/xio3130_upstream.h b/hw/pci-bridge/xio3130_upstream.h
index 08c1d5f..d0ab757 100644
--- a/hw/pci-bridge/xio3130_upstream.h
+++ b/hw/pci-bridge/xio3130_upstream.h
@@ -7,4 +7,4 @@
                                 const char *bus_name, pci_map_irq_fn map_irq,
                                 uint8_t port);
 
-#endif /* QEMU_XIO3130_H */
+#endif /* QEMU_XIO3130_UPSTREAM_H */
diff --git a/hw/pci-host/Makefile.objs b/hw/pci-host/Makefile.objs
index bb65f9c..45f1f0e 100644
--- a/hw/pci-host/Makefile.objs
+++ b/hw/pci-host/Makefile.objs
@@ -15,3 +15,4 @@
 common-obj-$(CONFIG_FULONG) += bonito.o
 common-obj-$(CONFIG_PCI_PIIX) += piix.o
 common-obj-$(CONFIG_PCI_Q35) += q35.o
+common-obj-$(CONFIG_PCI_GENERIC) += gpex.o
diff --git a/hw/pci-host/apb.c b/hw/pci-host/apb.c
index f573875..653e711 100644
--- a/hw/pci-host/apb.c
+++ b/hw/pci-host/apb.c
@@ -27,6 +27,7 @@
    Ultrasparc PCI host is called the PCI Bus Module (PBM).  The APB is
    the secondary PCI bridge.  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
@@ -35,6 +36,7 @@
 #include "hw/pci-host/apb.h"
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
+#include "qemu/log.h"
 
 /* debug APB */
 //#define DEBUG_APB
@@ -205,6 +207,7 @@
     return &is->iommu_as;
 }
 
+/* Called from RCU critical section */
 static IOMMUTLBEntry pbm_translate_iommu(MemoryRegion *iommu, hwaddr addr,
                                          bool is_write)
 {
@@ -288,7 +291,8 @@
         }
     }
 
-    tte = ldq_be_phys(&address_space_memory, baseaddr + offset);
+    tte = address_space_ldq_be(&address_space_memory, baseaddr + offset,
+                               MEMTXATTRS_UNSPECIFIED, NULL);
 
     if (!(tte & IOMMU_TTE_DATA_V)) {
         /* Invalid mapping */
@@ -630,14 +634,9 @@
     }
 }
 
-static int apb_pci_bridge_initfn(PCIDevice *dev)
+static void apb_pci_bridge_realize(PCIDevice *dev, Error **errp)
 {
-    int rc;
-
-    rc = pci_bridge_initfn(dev, TYPE_PCI_BUS);
-    if (rc < 0) {
-        return rc;
-    }
+    pci_bridge_initfn(dev, TYPE_PCI_BUS);
 
     /*
      * command register:
@@ -653,7 +652,6 @@
     pci_set_word(dev->config + PCI_STATUS,
                  PCI_STATUS_FAST_BACK | PCI_STATUS_66MHZ |
                  PCI_STATUS_DEVSEL_MEDIUM);
-    return 0;
 }
 
 PCIBus *pci_apb_init(hwaddr special_base,
@@ -671,6 +669,13 @@
 
     /* Ultrasparc PBM main bus */
     dev = qdev_create(NULL, TYPE_APB);
+    d = APB_DEVICE(dev);
+    phb = PCI_HOST_BRIDGE(dev);
+    phb->bus = pci_register_bus(DEVICE(phb), "pci",
+                                pci_apb_set_irq, pci_pbm_map_irq, d,
+                                &d->pci_mmio,
+                                get_system_io(),
+                                0, 32, TYPE_PCI_BUS);
     qdev_init_nofail(dev);
     s = SYS_BUS_DEVICE(dev);
     /* apb_config */
@@ -679,18 +684,10 @@
     sysbus_mmio_map(s, 1, special_base + 0x1000000ULL);
     /* pci_ioport */
     sysbus_mmio_map(s, 2, special_base + 0x2000000ULL);
-    d = APB_DEVICE(dev);
 
     memory_region_init(&d->pci_mmio, OBJECT(s), "pci-mmio", 0x100000000ULL);
     memory_region_add_subregion(get_system_memory(), mem_base, &d->pci_mmio);
 
-    phb = PCI_HOST_BRIDGE(dev);
-    phb->bus = pci_register_bus(DEVICE(phb), "pci",
-                                pci_apb_set_irq, pci_pbm_map_irq, d,
-                                &d->pci_mmio,
-                                get_system_io(),
-                                0, 32, TYPE_PCI_BUS);
-
     *pbm_irqs = d->pbm_irqs;
     d->ivec_irqs = ivec_irqs;
 
@@ -791,14 +788,13 @@
     return 0;
 }
 
-static int pbm_pci_host_init(PCIDevice *d)
+static void pbm_pci_host_realize(PCIDevice *d, Error **errp)
 {
     pci_set_word(d->config + PCI_COMMAND,
                  PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
     pci_set_word(d->config + PCI_STATUS,
                  PCI_STATUS_FAST_BACK | PCI_STATUS_66MHZ |
                  PCI_STATUS_DEVSEL_MEDIUM);
-    return 0;
 }
 
 static void pbm_pci_host_class_init(ObjectClass *klass, void *data)
@@ -806,7 +802,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = pbm_pci_host_init;
+    k->realize = pbm_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_SUN;
     k->device_id = PCI_DEVICE_ID_SUN_SABRE;
     k->class_id = PCI_CLASS_BRIDGE_HOST;
@@ -846,7 +842,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = apb_pci_bridge_initfn;
+    k->realize = apb_pci_bridge_realize;
     k->exit = pci_bridge_exitfn;
     k->vendor_id = PCI_VENDOR_ID_SUN;
     k->device_id = PCI_DEVICE_ID_SUN_SIMBA;
diff --git a/hw/pci-host/bonito.c b/hw/pci-host/bonito.c
index 56292ad..1999ece 100644
--- a/hw/pci-host/bonito.c
+++ b/hw/pci-host/bonito.c
@@ -37,7 +37,7 @@
  * north bridge address to pci address.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
@@ -180,8 +180,6 @@
 #define PCI_ADDR(busno,devno,funno,regno)  \
     ((((busno)<<16)&0xff0000) + (((devno)<<11)&0xf800) + (((funno)<<8)&0x700) + (regno))
 
-#define TYPE_BONITO_PCI_HOST_BRIDGE "Bonito-pcihost"
-
 typedef struct BonitoState BonitoState;
 
 typedef struct PCIBonitoState
@@ -215,16 +213,19 @@
 
 } PCIBonitoState;
 
+struct BonitoState {
+    PCIHostState parent_obj;
+    qemu_irq *pic;
+    PCIBonitoState *pci_dev;
+};
+
+#define TYPE_BONITO_PCI_HOST_BRIDGE "Bonito-pcihost"
 #define BONITO_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(BonitoState, (obj), TYPE_BONITO_PCI_HOST_BRIDGE)
 
-struct BonitoState {
-    PCIHostState parent_obj;
-
-    qemu_irq *pic;
-
-    PCIBonitoState *pci_dev;
-};
+#define TYPE_PCI_BONITO "Bonito"
+#define PCI_BONITO(obj) \
+    OBJECT_CHECK(PCIBonitoState, (obj), TYPE_PCI_BONITO)
 
 static void bonito_writel(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
@@ -233,7 +234,7 @@
     uint32_t saddr;
     int reset = 0;
 
-    saddr = (addr - BONITO_REGBASE) >> 2;
+    saddr = addr >> 2;
 
     DPRINTF("bonito_writel "TARGET_FMT_plx" val %x saddr %x\n", addr, val, saddr);
     switch (saddr) {
@@ -295,7 +296,7 @@
     PCIBonitoState *s = opaque;
     uint32_t saddr;
 
-    saddr = (addr - BONITO_REGBASE) >> 2;
+    saddr = addr >> 2;
 
     DPRINTF("bonito_readl "TARGET_FMT_plx"\n", addr);
     switch (saddr) {
@@ -355,6 +356,10 @@
     uint32_t val;
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->bonldma)) {
+        return 0;
+    }
+
     val = ((uint32_t *)(&s->bonldma))[addr/sizeof(uint32_t)];
 
     return val;
@@ -365,6 +370,10 @@
 {
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->bonldma)) {
+        return;
+    }
+
     ((uint32_t *)(&s->bonldma))[addr/sizeof(uint32_t)] = val & 0xffffffff;
 }
 
@@ -384,6 +393,10 @@
     uint32_t val;
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->boncop)) {
+        return 0;
+    }
+
     val = ((uint32_t *)(&s->boncop))[addr/sizeof(uint32_t)];
 
     return val;
@@ -394,6 +407,10 @@
 {
     PCIBonitoState *s = opaque;
 
+    if (addr >= sizeof(s->boncop)) {
+        return;
+    }
+
     ((uint32_t *)(&s->boncop))[addr/sizeof(uint32_t)] = val & 0xffffffff;
 }
 
@@ -427,7 +444,7 @@
     cfgaddr |= (s->regs[BONITO_PCIMAP_CFG] & 0xffff) << 16;
 
     idsel = (cfgaddr & BONITO_PCICONF_IDSEL_MASK) >> BONITO_PCICONF_IDSEL_OFFSET;
-    devno = ffs(idsel) - 1;
+    devno = ctz32(idsel);
     funno = (cfgaddr & BONITO_PCICONF_FUN_MASK) >> BONITO_PCICONF_FUN_OFFSET;
     regno = (cfgaddr & BONITO_PCICONF_REG_MASK) >> BONITO_PCICONF_REG_OFFSET;
 
@@ -705,9 +722,9 @@
     return 0;
 }
 
-static int bonito_initfn(PCIDevice *dev)
+static void bonito_realize(PCIDevice *dev, Error **errp)
 {
-    PCIBonitoState *s = DO_UPCAST(PCIBonitoState, dev, dev);
+    PCIBonitoState *s = PCI_BONITO(dev);
     SysBusDevice *sysbus = SYS_BUS_DEVICE(s->pcihost);
     PCIHostState *phb = PCI_HOST_BRIDGE(s->pcihost);
 
@@ -766,8 +783,6 @@
     pci_set_byte(dev->config + PCI_MAX_LAT, 0x00);
 
     qemu_register_reset(bonito_reset, s);
-
-    return 0;
 }
 
 PCIBus *bonito_init(qemu_irq *pic)
@@ -785,8 +800,8 @@
     qdev_init_nofail(dev);
 
     /* set the pcihost pointer before bonito_initfn is called */
-    d = pci_create(phb->bus, PCI_DEVFN(0, 0), "Bonito");
-    s = DO_UPCAST(PCIBonitoState, dev, d);
+    d = pci_create(phb->bus, PCI_DEVFN(0, 0), TYPE_PCI_BONITO);
+    s = PCI_BONITO(d);
     s->pcihost = pcihost;
     pcihost->pci_dev = s;
     qdev_init_nofail(DEVICE(d));
@@ -799,7 +814,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = bonito_initfn;
+    k->realize = bonito_realize;
     k->vendor_id = 0xdf53;
     k->device_id = 0x00d5;
     k->revision = 0x01;
@@ -814,7 +829,7 @@
 }
 
 static const TypeInfo bonito_info = {
-    .name          = "Bonito",
+    .name          = TYPE_PCI_BONITO,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(PCIBonitoState),
     .class_init    = bonito_class_init,
diff --git a/hw/pci-host/gpex.c b/hw/pci-host/gpex.c
new file mode 100644
index 0000000..66055ee
--- /dev/null
+++ b/hw/pci-host/gpex.c
@@ -0,0 +1,155 @@
+/*
+ * QEMU Generic PCI Express Bridge Emulation
+ *
+ * Copyright (C) 2015 Alexander Graf <agraf@suse.de>
+ *
+ * Code loosely based on q35.c.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * Check out these documents for more information on the device:
+ *
+ * http://www.kernel.org/doc/Documentation/devicetree/bindings/pci/host-generic-pci.txt
+ * http://www.firmware.org/1275/practice/imap/imap0_9d.pdf
+ */
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci-host/gpex.h"
+
+/****************************************************************************
+ * GPEX host
+ */
+
+static void gpex_set_irq(void *opaque, int irq_num, int level)
+{
+    GPEXHost *s = opaque;
+
+    qemu_set_irq(s->irq[irq_num], level);
+}
+
+static void gpex_host_realize(DeviceState *dev, Error **errp)
+{
+    PCIHostState *pci = PCI_HOST_BRIDGE(dev);
+    GPEXHost *s = GPEX_HOST(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    PCIExpressHost *pex = PCIE_HOST_BRIDGE(dev);
+    int i;
+
+    pcie_host_mmcfg_init(pex, PCIE_MMCFG_SIZE_MAX);
+    memory_region_init(&s->io_mmio, OBJECT(s), "gpex_mmio", UINT64_MAX);
+    memory_region_init(&s->io_ioport, OBJECT(s), "gpex_ioport", 64 * 1024);
+
+    sysbus_init_mmio(sbd, &pex->mmio);
+    sysbus_init_mmio(sbd, &s->io_mmio);
+    sysbus_init_mmio(sbd, &s->io_ioport);
+    for (i = 0; i < GPEX_NUM_IRQS; i++) {
+        sysbus_init_irq(sbd, &s->irq[i]);
+    }
+
+    pci->bus = pci_register_bus(dev, "pcie.0", gpex_set_irq,
+                                pci_swizzle_map_irq_fn, s, &s->io_mmio,
+                                &s->io_ioport, 0, 4, TYPE_PCIE_BUS);
+
+    qdev_set_parent_bus(DEVICE(&s->gpex_root), BUS(pci->bus));
+    qdev_init_nofail(DEVICE(&s->gpex_root));
+}
+
+static const char *gpex_host_root_bus_path(PCIHostState *host_bridge,
+                                          PCIBus *rootbus)
+{
+    return "0000:00";
+}
+
+static void gpex_host_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(klass);
+
+    hc->root_bus_path = gpex_host_root_bus_path;
+    dc->realize = gpex_host_realize;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+    dc->fw_name = "pci";
+}
+
+static void gpex_host_initfn(Object *obj)
+{
+    GPEXHost *s = GPEX_HOST(obj);
+    GPEXRootState *root = &s->gpex_root;
+
+    object_initialize(root, sizeof(*root), TYPE_GPEX_ROOT_DEVICE);
+    object_property_add_child(obj, "gpex_root", OBJECT(root), NULL);
+    qdev_prop_set_uint32(DEVICE(root), "addr", PCI_DEVFN(0, 0));
+    qdev_prop_set_bit(DEVICE(root), "multifunction", false);
+}
+
+static const TypeInfo gpex_host_info = {
+    .name       = TYPE_GPEX_HOST,
+    .parent     = TYPE_PCIE_HOST_BRIDGE,
+    .instance_size = sizeof(GPEXHost),
+    .instance_init = gpex_host_initfn,
+    .class_init = gpex_host_class_init,
+};
+
+/****************************************************************************
+ * GPEX Root D0:F0
+ */
+
+static const VMStateDescription vmstate_gpex_root = {
+    .name = "gpex_root",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(parent_obj, GPEXRootState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void gpex_root_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+    dc->desc = "QEMU generic PCIe host bridge";
+    dc->vmsd = &vmstate_gpex_root;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_PCIE_HOST;
+    k->revision = 0;
+    k->class_id = PCI_CLASS_BRIDGE_HOST;
+    /*
+     * PCI-facing part of the host bridge, not usable without the
+     * host-facing part, which can't be device_add'ed, yet.
+     */
+    dc->cannot_instantiate_with_device_add_yet = true;
+}
+
+static const TypeInfo gpex_root_info = {
+    .name = TYPE_GPEX_ROOT_DEVICE,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(GPEXRootState),
+    .class_init = gpex_root_class_init,
+};
+
+static void gpex_register(void)
+{
+    type_register_static(&gpex_root_info);
+    type_register_static(&gpex_host_info);
+}
+
+type_init(gpex_register)
diff --git a/hw/pci-host/grackle.c b/hw/pci-host/grackle.c
index 6c7cfdb..2c8acda 100644
--- a/hw/pci-host/grackle.c
+++ b/hw/pci-host/grackle.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_host.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -71,7 +72,6 @@
     GrackleState *d;
 
     dev = qdev_create(NULL, TYPE_GRACKLE_PCI_HOST_BRIDGE);
-    qdev_init_nofail(dev);
     s = SYS_BUS_DEVICE(dev);
     phb = PCI_HOST_BRIDGE(dev);
     d = GRACKLE_PCI_HOST_BRIDGE(dev);
@@ -91,6 +91,7 @@
                                 0, 4, TYPE_PCI_BUS);
 
     pci_create_simple(phb->bus, 0, "grackle");
+    qdev_init_nofail(dev);
 
     sysbus_mmio_map(s, 0, base);
     sysbus_mmio_map(s, 1, base + 0x00200000);
@@ -114,10 +115,9 @@
     return 0;
 }
 
-static int grackle_pci_host_init(PCIDevice *d)
+static void grackle_pci_host_realize(PCIDevice *d, Error **errp)
 {
     d->config[0x09] = 0x01;
-    return 0;
 }
 
 static void grackle_pci_class_init(ObjectClass *klass, void *data)
@@ -125,7 +125,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init      = grackle_pci_host_init;
+    k->realize   = grackle_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_MOTOROLA;
     k->device_id = PCI_DEVICE_ID_MOTOROLA_MPC106;
     k->revision  = 0x00;
@@ -147,8 +147,10 @@
 static void pci_grackle_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     k->init = pci_grackle_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo grackle_pci_host_info = {
diff --git a/hw/pci-host/pam.c b/hw/pci-host/pam.c
index 8272de3..e361ecb 100644
--- a/hw/pci-host/pam.c
+++ b/hw/pci-host/pam.c
@@ -27,30 +27,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "sysemu/sysemu.h"
 #include "hw/pci-host/pam.h"
 
-void smram_update(MemoryRegion *smram_region, uint8_t smram,
-                  uint8_t smm_enabled)
-{
-    bool smram_enabled;
-
-    smram_enabled = ((smm_enabled && (smram & SMRAM_G_SMRAME)) ||
-                        (smram & SMRAM_D_OPEN));
-    memory_region_set_enabled(smram_region, !smram_enabled);
-}
-
-void smram_set_smm(uint8_t *host_smm_enabled, int smm, uint8_t smram,
-                   MemoryRegion *smram_region)
-{
-    uint8_t smm_enabled = (smm != 0);
-    if (*host_smm_enabled != smm_enabled) {
-        *host_smm_enabled = smm_enabled;
-        smram_update(smram_region, smram, *host_smm_enabled);
-    }
-}
-
 void init_pam(DeviceState *dev, MemoryRegion *ram_memory,
               MemoryRegion *system_memory, MemoryRegion *pci_address_space,
               PAMMemoryRegion *mem, uint32_t start, uint32_t size)
diff --git a/hw/pci-host/piix.c b/hw/pci-host/piix.c
index 1530038..f9218aa 100644
--- a/hw/pci-host/piix.c
+++ b/hw/pci-host/piix.c
@@ -22,31 +22,33 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
 #include "hw/isa/isa.h"
 #include "hw/sysbus.h"
+#include "qapi/error.h"
 #include "qemu/range.h"
 #include "hw/xen/xen.h"
 #include "hw/pci-host/pam.h"
 #include "sysemu/sysemu.h"
 #include "hw/i386/ioapic.h"
 #include "qapi/visitor.h"
+#include "qemu/error-report.h"
 
 /*
  * I440FX chipset data sheet.
  * http://download.intel.com/design/chipsets/datashts/29054901.pdf
  */
 
-#define TYPE_I440FX_PCI_HOST_BRIDGE "i440FX-pcihost"
 #define I440FX_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(I440FXState, (obj), TYPE_I440FX_PCI_HOST_BRIDGE)
 
 typedef struct I440FXState {
     PCIHostState parent_obj;
-    PcPciInfo pci_info;
+    Range pci_hole;
     uint64_t pci_hole64_size;
     uint32_t short_root_bus;
 } I440FXState;
@@ -91,7 +93,10 @@
     MemoryRegion rcr_mem;
 } PIIX3State;
 
-#define TYPE_I440FX_PCI_DEVICE "i440FX"
+#define TYPE_PIIX3_PCI_DEVICE "pci-piix3"
+#define PIIX3_PCI_DEVICE(obj) \
+    OBJECT_CHECK(PIIX3State, (obj), TYPE_PIIX3_PCI_DEVICE)
+
 #define I440FX_PCI_DEVICE(obj) \
     OBJECT_CHECK(PCII440FXState, (obj), TYPE_I440FX_PCI_DEVICE)
 
@@ -105,7 +110,7 @@
     MemoryRegion *ram_memory;
     PAMMemoryRegion pam_regions[13];
     MemoryRegion smram_region;
-    uint8_t smm_enabled;
+    MemoryRegion smram, low_smram;
 };
 
 
@@ -113,6 +118,11 @@
 #define I440FX_PAM_SIZE 7
 #define I440FX_SMRAM    0x72
 
+/* Older coreboot versions (4.0 and older) read a config register that doesn't
+ * exist in real hardware, to get the RAM size from QEMU.
+ */
+#define I440FX_COREBOOT_RAM_SIZE 0x57
+
 static void piix3_set_irq(void *opaque, int pirq, int level);
 static PCIINTxRoute piix3_route_intx_pin_to_irq(void *opaque, int pci_intx);
 static void piix3_write_config_xen(PCIDevice *dev,
@@ -138,18 +148,10 @@
         pam_update(&d->pam_regions[i], i,
                    pd->config[I440FX_PAM + ((i + 1) / 2)]);
     }
-    smram_update(&d->smram_region, pd->config[I440FX_SMRAM], d->smm_enabled);
-    memory_region_transaction_commit();
-}
-
-static void i440fx_set_smm(int val, void *arg)
-{
-    PCII440FXState *d = arg;
-    PCIDevice *pd = PCI_DEVICE(d);
-
-    memory_region_transaction_begin();
-    smram_set_smm(&d->smm_enabled, val, pd->config[I440FX_SMRAM],
-                  &d->smram_region);
+    memory_region_set_enabled(&d->smram_region,
+                              !(pd->config[I440FX_SMRAM] & SMRAM_D_OPEN));
+    memory_region_set_enabled(&d->smram,
+                              pd->config[I440FX_SMRAM] & SMRAM_G_SMRAME);
     memory_region_transaction_commit();
 }
 
@@ -172,12 +174,13 @@
     PCII440FXState *d = opaque;
     PCIDevice *pd = PCI_DEVICE(d);
     int ret, i;
+    uint8_t smm_enabled;
 
     ret = pci_device_load(pd, f);
     if (ret < 0)
         return ret;
     i440fx_update_memory_mappings(d);
-    qemu_get_8s(f, &d->smm_enabled);
+    qemu_get_8s(f, &smm_enabled);
 
     if (version_id == 2) {
         for (i = 0; i < PIIX_NUM_PIRQS; i++) {
@@ -205,59 +208,71 @@
     .post_load = i440fx_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(parent_obj, PCII440FXState),
-        VMSTATE_UINT8(smm_enabled, PCII440FXState),
+        /* Used to be smm_enabled, which was basically always zero because
+         * SeaBIOS hardly uses SMM.  SMRAM is now handled by CPU code.
+         */
+        VMSTATE_UNUSED(1),
         VMSTATE_END_OF_LIST()
     }
 };
 
 static void i440fx_pcihost_get_pci_hole_start(Object *obj, Visitor *v,
-                                              void *opaque, const char *name,
+                                              const char *name, void *opaque,
                                               Error **errp)
 {
     I440FXState *s = I440FX_PCI_HOST_BRIDGE(obj);
-    uint32_t value = s->pci_info.w32.begin;
+    uint64_t val64;
+    uint32_t value;
 
-    visit_type_uint32(v, &value, name, errp);
+    val64 = range_is_empty(&s->pci_hole) ? 0 : range_lob(&s->pci_hole);
+    value = val64;
+    assert(value == val64);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void i440fx_pcihost_get_pci_hole_end(Object *obj, Visitor *v,
-                                            void *opaque, const char *name,
+                                            const char *name, void *opaque,
                                             Error **errp)
 {
     I440FXState *s = I440FX_PCI_HOST_BRIDGE(obj);
-    uint32_t value = s->pci_info.w32.end;
+    uint64_t val64;
+    uint32_t value;
 
-    visit_type_uint32(v, &value, name, errp);
+    val64 = range_is_empty(&s->pci_hole) ? 0 : range_upb(&s->pci_hole) + 1;
+    value = val64;
+    assert(value == val64);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void i440fx_pcihost_get_pci_hole64_start(Object *obj, Visitor *v,
-                                                void *opaque, const char *name,
-                                                Error **errp)
+                                                const char *name,
+                                                void *opaque, Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
     Range w64;
+    uint64_t value;
 
     pci_bus_get_w64_range(h->bus, &w64);
-
-    visit_type_uint64(v, &w64.begin, name, errp);
+    value = range_is_empty(&w64) ? 0 : range_lob(&w64);
+    visit_type_uint64(v, name, &value, errp);
 }
 
 static void i440fx_pcihost_get_pci_hole64_end(Object *obj, Visitor *v,
-                                              void *opaque, const char *name,
+                                              const char *name, void *opaque,
                                               Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
     Range w64;
+    uint64_t value;
 
     pci_bus_get_w64_range(h->bus, &w64);
-
-    visit_type_uint64(v, &w64.end, name, errp);
+    value = range_is_empty(&w64) ? 0 : range_upb(&w64) + 1;
+    visit_type_uint64(v, name, &value, errp);
 }
 
 static void i440fx_pcihost_initfn(Object *obj)
 {
     PCIHostState *s = PCI_HOST_BRIDGE(obj);
-    I440FXState *d = I440FX_PCI_HOST_BRIDGE(obj);
 
     memory_region_init_io(&s->conf_mem, obj, &pci_host_conf_le_ops, s,
                           "pci-conf-idx", 4);
@@ -279,8 +294,6 @@
     object_property_add(obj, PCI_HOST_PROP_PCI_HOLE64_END, "int",
                         i440fx_pcihost_get_pci_hole64_end,
                         NULL, NULL, NULL, NULL);
-
-    d->pci_info.w32.end = IO_APIC_DEFAULT_ADDRESS;
 }
 
 static void i440fx_pcihost_realize(DeviceState *dev, Error **errp)
@@ -295,17 +308,17 @@
     sysbus_init_ioports(sbd, 0xcfc, 4);
 }
 
-static int i440fx_initfn(PCIDevice *dev)
+static void i440fx_realize(PCIDevice *dev, Error **errp)
 {
-    PCII440FXState *d = I440FX_PCI_DEVICE(dev);
-
     dev->config[I440FX_SMRAM] = 0x02;
 
-    cpu_smm_register(&i440fx_set_smm, d);
-    return 0;
+    if (object_property_get_bool(qdev_get_machine(), "iommu", NULL)) {
+        error_report("warning: i440fx doesn't support emulated iommu");
+    }
 }
 
-PCIBus *i440fx_init(PCII440FXState **pi440fx_state,
+PCIBus *i440fx_init(const char *host_type, const char *pci_type,
+                    PCII440FXState **pi440fx_state,
                     int *piix3_devfn,
                     ISABus **isa_bus, qemu_irq *pic,
                     MemoryRegion *address_space_mem,
@@ -325,7 +338,7 @@
     unsigned i;
     I440FXState *i440fx;
 
-    dev = qdev_create(NULL, TYPE_I440FX_PCI_HOST_BRIDGE);
+    dev = qdev_create(NULL, host_type);
     s = PCI_HOST_BRIDGE(dev);
     b = pci_bus_new(dev, NULL, pci_address_space,
                     address_space_io, 0, TYPE_PCI_BUS);
@@ -333,7 +346,7 @@
     object_property_add_child(qdev_get_machine(), "i440fx", OBJECT(dev), NULL);
     qdev_init_nofail(dev);
 
-    d = pci_create_simple(b, 0, TYPE_I440FX_PCI_DEVICE);
+    d = pci_create_simple(b, 0, pci_type);
     *pi440fx_state = I440FX_PCI_DEVICE(d);
     f = *pi440fx_state;
     f->system_memory = address_space_mem;
@@ -341,17 +354,30 @@
     f->ram_memory = ram_memory;
 
     i440fx = I440FX_PCI_HOST_BRIDGE(dev);
-    i440fx->pci_info.w32.begin = below_4g_mem_size;
+    range_set_bounds(&i440fx->pci_hole, below_4g_mem_size,
+                     IO_APIC_DEFAULT_ADDRESS - 1);
 
     /* setup pci memory mapping */
     pc_pci_as_mapping_init(OBJECT(f), f->system_memory,
                            f->pci_address_space);
 
+    /* if *disabled* show SMRAM to all CPUs */
     memory_region_init_alias(&f->smram_region, OBJECT(d), "smram-region",
                              f->pci_address_space, 0xa0000, 0x20000);
     memory_region_add_subregion_overlap(f->system_memory, 0xa0000,
                                         &f->smram_region, 1);
-    memory_region_set_enabled(&f->smram_region, false);
+    memory_region_set_enabled(&f->smram_region, true);
+
+    /* smram, as seen by SMM CPUs */
+    memory_region_init(&f->smram, OBJECT(d), "smram", 1ull << 32);
+    memory_region_set_enabled(&f->smram, true);
+    memory_region_init_alias(&f->low_smram, OBJECT(d), "smram-low",
+                             f->ram_memory, 0xa0000, 0x20000);
+    memory_region_set_enabled(&f->low_smram, true);
+    memory_region_add_subregion(&f->smram, 0xa0000, &f->low_smram);
+    object_property_add_const_link(qdev_get_machine(), "smram",
+                                   OBJECT(&f->smram), &error_abort);
+
     init_pam(dev, f->ram_memory, f->system_memory, f->pci_address_space,
              &f->pam_regions[0], PAM_BIOS_BASE, PAM_BIOS_SIZE);
     for (i = 0; i < 12; ++i) {
@@ -365,13 +391,15 @@
      * connected to the IOAPIC directly.
      * These additional routes can be discovered through ACPI. */
     if (xen_enabled()) {
-        piix3 = DO_UPCAST(PIIX3State, dev,
-                pci_create_simple_multifunction(b, -1, true, "PIIX3-xen"));
+        PCIDevice *pci_dev = pci_create_simple_multifunction(b,
+                             -1, true, "PIIX3-xen");
+        piix3 = PIIX3_PCI_DEVICE(pci_dev);
         pci_bus_irqs(b, xen_piix3_set_irq, xen_pci_slot_get_pirq,
                 piix3, XEN_PIIX_NUM_PIRQS);
     } else {
-        piix3 = DO_UPCAST(PIIX3State, dev,
-                pci_create_simple_multifunction(b, -1, true, "PIIX3"));
+        PCIDevice *pci_dev = pci_create_simple_multifunction(b,
+                             -1, true, "PIIX3");
+        piix3 = PIIX3_PCI_DEVICE(pci_dev);
         pci_bus_irqs(b, piix3_set_irq, pci_slot_get_pirq, piix3,
                 PIIX_NUM_PIRQS);
         pci_bus_set_route_irq_fn(b, piix3_route_intx_pin_to_irq);
@@ -385,7 +413,7 @@
     if (ram_size > 255) {
         ram_size = 255;
     }
-    d->config[0x57] = ram_size;
+    d->config[I440FX_COREBOOT_RAM_SIZE] = ram_size;
 
     i440fx_update_memory_mappings(f);
 
@@ -477,7 +505,7 @@
 {
     pci_default_write_config(dev, address, val, len);
     if (ranges_overlap(address, len, PIIX_PIRQC, 4)) {
-        PIIX3State *piix3 = DO_UPCAST(PIIX3State, dev, dev);
+        PIIX3State *piix3 = PIIX3_PCI_DEVICE(dev);
         int pic_irq;
 
         pci_bus_fire_intx_routing_notifier(piix3->dev.bus);
@@ -579,6 +607,7 @@
     .name = "PIIX3/rcr",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = piix3_rcr_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(rcr, PIIX3State),
         VMSTATE_END_OF_LIST()
@@ -597,12 +626,9 @@
                               PIIX_NUM_PIRQS, 3),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_piix3_rcr,
-            .needed = piix3_rcr_needed,
-        },
-        { 0 }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_piix3_rcr,
+        NULL
     }
 };
 
@@ -631,11 +657,14 @@
     .endianness = DEVICE_LITTLE_ENDIAN
 };
 
-static int piix3_initfn(PCIDevice *dev)
+static void piix3_realize(PCIDevice *dev, Error **errp)
 {
-    PIIX3State *d = DO_UPCAST(PIIX3State, dev, dev);
+    PIIX3State *d = PIIX3_PCI_DEVICE(dev);
 
-    isa_bus_new(DEVICE(d), pci_address_space_io(dev));
+    if (!isa_bus_new(DEVICE(d), get_system_memory(),
+                     pci_address_space_io(dev), errp)) {
+        return;
+    }
 
     memory_region_init_io(&d->rcr_mem, OBJECT(dev), &rcr_ops, d,
                           "piix3-reset-control", 1);
@@ -643,10 +672,9 @@
                                         &d->rcr_mem, 1);
 
     qemu_register_reset(piix3_reset, d);
-    return 0;
 }
 
-static void piix3_class_init(ObjectClass *klass, void *data)
+static void pci_piix3_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
@@ -654,8 +682,7 @@
     dc->desc        = "ISA bridge";
     dc->vmsd        = &vmstate_piix3;
     dc->hotpluggable   = false;
-    k->init         = piix3_initfn;
-    k->config_write = piix3_write_config;
+    k->realize      = piix3_realize;
     k->vendor_id    = PCI_VENDOR_ID_INTEL;
     /* 82371SB PIIX3 PCI-to-ISA bridge (Step A1) */
     k->device_id    = PCI_DEVICE_ID_INTEL_82371SB_0;
@@ -667,38 +694,37 @@
     dc->cannot_instantiate_with_device_add_yet = true;
 }
 
+static const TypeInfo piix3_pci_type_info = {
+    .name = TYPE_PIIX3_PCI_DEVICE,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PIIX3State),
+    .abstract = true,
+    .class_init = pci_piix3_class_init,
+};
+
+static void piix3_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->config_write = piix3_write_config;
+}
+
 static const TypeInfo piix3_info = {
     .name          = "PIIX3",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PIIX3State),
+    .parent        = TYPE_PIIX3_PCI_DEVICE,
     .class_init    = piix3_class_init,
 };
 
 static void piix3_xen_class_init(ObjectClass *klass, void *data)
 {
-    DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    dc->desc        = "ISA bridge";
-    dc->vmsd        = &vmstate_piix3;
-    dc->hotpluggable   = false;
-    k->init         = piix3_initfn;
     k->config_write = piix3_write_config_xen;
-    k->vendor_id    = PCI_VENDOR_ID_INTEL;
-    /* 82371SB PIIX3 PCI-to-ISA bridge (Step A1) */
-    k->device_id    = PCI_DEVICE_ID_INTEL_82371SB_0;
-    k->class_id     = PCI_CLASS_BRIDGE_ISA;
-    /*
-     * Reason: part of PIIX3 southbridge, needs to be wired up by
-     * pc_piix.c's pc_init1()
-     */
-    dc->cannot_instantiate_with_device_add_yet = true;
 };
 
 static const TypeInfo piix3_xen_info = {
     .name          = "PIIX3-xen",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(PIIX3State),
+    .parent        = TYPE_PIIX3_PCI_DEVICE,
     .class_init    = piix3_xen_class_init,
 };
 
@@ -707,7 +733,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = i440fx_initfn;
+    k->realize = i440fx_realize;
     k->config_write = i440fx_write_config;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_82441;
@@ -730,6 +756,95 @@
     .class_init    = i440fx_class_init,
 };
 
+/* IGD Passthrough Host Bridge. */
+typedef struct {
+    uint8_t offset;
+    uint8_t len;
+} IGDHostInfo;
+
+/* Here we just expose minimal host bridge offset subset. */
+static const IGDHostInfo igd_host_bridge_infos[] = {
+    {0x08, 2},  /* revision id */
+    {0x2c, 2},  /* sybsystem vendor id */
+    {0x2e, 2},  /* sybsystem id */
+    {0x50, 2},  /* SNB: processor graphics control register */
+    {0x52, 2},  /* processor graphics control register */
+    {0xa4, 4},  /* SNB: graphics base of stolen memory */
+    {0xa8, 4},  /* SNB: base of GTT stolen memory */
+};
+
+static int host_pci_config_read(int pos, int len, uint32_t *val)
+{
+    char path[PATH_MAX];
+    int config_fd;
+    ssize_t size = sizeof(path);
+    /* Access real host bridge. */
+    int rc = snprintf(path, size, "/sys/bus/pci/devices/%04x:%02x:%02x.%d/%s",
+                      0, 0, 0, 0, "config");
+    int ret = 0;
+
+    if (rc >= size || rc < 0) {
+        return -ENODEV;
+    }
+
+    config_fd = open(path, O_RDWR);
+    if (config_fd < 0) {
+        return -ENODEV;
+    }
+
+    if (lseek(config_fd, pos, SEEK_SET) != pos) {
+        ret = -errno;
+        goto out;
+    }
+
+    do {
+        rc = read(config_fd, (uint8_t *)val, len);
+    } while (rc < 0 && (errno == EINTR || errno == EAGAIN));
+    if (rc != len) {
+        ret = -errno;
+    }
+
+out:
+    close(config_fd);
+    return ret;
+}
+
+static int igd_pt_i440fx_initfn(struct PCIDevice *pci_dev)
+{
+    uint32_t val = 0;
+    int rc, i, num;
+    int pos, len;
+
+    num = ARRAY_SIZE(igd_host_bridge_infos);
+    for (i = 0; i < num; i++) {
+        pos = igd_host_bridge_infos[i].offset;
+        len = igd_host_bridge_infos[i].len;
+        rc = host_pci_config_read(pos, len, &val);
+        if (rc) {
+            return -ENODEV;
+        }
+        pci_default_write_config(pci_dev, pos, val, len);
+    }
+
+    return 0;
+}
+
+static void igd_passthrough_i440fx_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = igd_pt_i440fx_initfn;
+    dc->desc = "IGD Passthrough Host bridge";
+}
+
+static const TypeInfo igd_passthrough_i440fx_info = {
+    .name          = TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE,
+    .parent        = TYPE_I440FX_PCI_DEVICE,
+    .instance_size = sizeof(PCII440FXState),
+    .class_init    = igd_passthrough_i440fx_class_init,
+};
+
 static const char *i440fx_pcihost_root_bus_path(PCIHostState *host_bridge,
                                                 PCIBus *rootbus)
 {
@@ -758,6 +873,8 @@
     dc->realize = i440fx_pcihost_realize;
     dc->fw_name = "pci";
     dc->props = i440fx_props;
+    /* Reason: needs to be wired up by pc_init1 */
+    dc->cannot_instantiate_with_device_add_yet = true;
 }
 
 static const TypeInfo i440fx_pcihost_info = {
@@ -771,6 +888,8 @@
 static void i440fx_register_types(void)
 {
     type_register_static(&i440fx_info);
+    type_register_static(&igd_passthrough_i440fx_info);
+    type_register_static(&piix3_pci_type_info);
     type_register_static(&piix3_info);
     type_register_static(&piix3_xen_info);
     type_register_static(&i440fx_pcihost_info);
diff --git a/hw/pci-host/ppce500.c b/hw/pci-host/ppce500.c
index 1b4c0f0..e502bc0 100644
--- a/hw/pci-host/ppce500.c
+++ b/hw/pci-host/ppce500.c
@@ -14,6 +14,7 @@
  * (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/e500-ccsr.h"
 #include "hw/pci/pci.h"
@@ -62,11 +63,19 @@
 #define PPCE500_PCI_NR_POBS     5
 #define PPCE500_PCI_NR_PIBS     3
 
+#define PIWAR_EN                0x80000000      /* Enable */
+#define PIWAR_PF                0x20000000      /* prefetch */
+#define PIWAR_TGI_LOCAL         0x00f00000      /* target - local memory */
+#define PIWAR_READ_SNOOP        0x00050000
+#define PIWAR_WRITE_SNOOP       0x00005000
+#define PIWAR_SZ_MASK           0x0000003f
+
 struct  pci_outbound {
     uint32_t potar;
     uint32_t potear;
     uint32_t powbar;
     uint32_t powar;
+    MemoryRegion mem;
 };
 
 struct pci_inbound {
@@ -74,6 +83,7 @@
     uint32_t piwbar;
     uint32_t piwbear;
     uint32_t piwar;
+    MemoryRegion mem;
 };
 
 #define TYPE_PPC_E500_PCI_HOST_BRIDGE "e500-pcihost"
@@ -91,10 +101,13 @@
     uint32_t irq_num[PCI_NUM_PINS];
     uint32_t first_slot;
     uint32_t first_pin_irq;
+    AddressSpace bm_as;
+    MemoryRegion bm;
     /* mmio maps */
     MemoryRegion container;
     MemoryRegion iomem;
     MemoryRegion pio;
+    MemoryRegion busmem;
 };
 
 #define TYPE_PPC_E500_PCI_BRIDGE "e500-host-bridge"
@@ -128,7 +141,7 @@
     case PPCE500_PCI_OW3:
     case PPCE500_PCI_OW4:
         idx = (addr >> 5) & 0x7;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_POTAR:
             value = pci->pob[idx].potar;
             break;
@@ -150,7 +163,7 @@
     case PPCE500_PCI_IW2:
     case PPCE500_PCI_IW1:
         idx = ((addr >> 5) & 0x3) - 1;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_PITAR:
             value = pci->pib[idx].pitar;
             break;
@@ -181,6 +194,71 @@
     return value;
 }
 
+/* DMA mapping */
+static void e500_update_piw(PPCE500PCIState *pci, int idx)
+{
+    uint64_t tar = ((uint64_t)pci->pib[idx].pitar) << 12;
+    uint64_t wbar = ((uint64_t)pci->pib[idx].piwbar) << 12;
+    uint64_t war = pci->pib[idx].piwar;
+    uint64_t size = 2ULL << (war & PIWAR_SZ_MASK);
+    MemoryRegion *address_space_mem = get_system_memory();
+    MemoryRegion *mem = &pci->pib[idx].mem;
+    MemoryRegion *bm = &pci->bm;
+    char *name;
+
+    if (memory_region_is_mapped(mem)) {
+        /* Before we modify anything, unmap and destroy the region */
+        memory_region_del_subregion(bm, mem);
+        object_unparent(OBJECT(mem));
+    }
+
+    if (!(war & PIWAR_EN)) {
+        /* Not enabled, nothing to do */
+        return;
+    }
+
+    name = g_strdup_printf("PCI Inbound Window %d", idx);
+    memory_region_init_alias(mem, OBJECT(pci), name, address_space_mem, tar,
+                             size);
+    memory_region_add_subregion_overlap(bm, wbar, mem, -1);
+    g_free(name);
+
+    pci_debug("%s: Added window of size=%#lx from PCI=%#lx to CPU=%#lx\n",
+              __func__, size, wbar, tar);
+}
+
+/* BAR mapping */
+static void e500_update_pow(PPCE500PCIState *pci, int idx)
+{
+    uint64_t tar = ((uint64_t)pci->pob[idx].potar) << 12;
+    uint64_t wbar = ((uint64_t)pci->pob[idx].powbar) << 12;
+    uint64_t war = pci->pob[idx].powar;
+    uint64_t size = 2ULL << (war & PIWAR_SZ_MASK);
+    MemoryRegion *mem = &pci->pob[idx].mem;
+    MemoryRegion *address_space_mem = get_system_memory();
+    char *name;
+
+    if (memory_region_is_mapped(mem)) {
+        /* Before we modify anything, unmap and destroy the region */
+        memory_region_del_subregion(address_space_mem, mem);
+        object_unparent(OBJECT(mem));
+    }
+
+    if (!(war & PIWAR_EN)) {
+        /* Not enabled, nothing to do */
+        return;
+    }
+
+    name = g_strdup_printf("PCI Outbound Window %d", idx);
+    memory_region_init_alias(mem, OBJECT(pci), name, &pci->busmem, tar,
+                             size);
+    memory_region_add_subregion(address_space_mem, wbar, mem);
+    g_free(name);
+
+    pci_debug("%s: Added window of size=%#lx from CPU=%#lx to PCI=%#lx\n",
+              __func__, size, wbar, tar);
+}
+
 static void pci_reg_write4(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
@@ -199,18 +277,22 @@
     case PPCE500_PCI_OW3:
     case PPCE500_PCI_OW4:
         idx = (addr >> 5) & 0x7;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_POTAR:
             pci->pob[idx].potar = value;
+            e500_update_pow(pci, idx);
             break;
         case PCI_POTEAR:
             pci->pob[idx].potear = value;
+            e500_update_pow(pci, idx);
             break;
         case PCI_POWBAR:
             pci->pob[idx].powbar = value;
+            e500_update_pow(pci, idx);
             break;
         case PCI_POWAR:
             pci->pob[idx].powar = value;
+            e500_update_pow(pci, idx);
             break;
         default:
             break;
@@ -221,18 +303,22 @@
     case PPCE500_PCI_IW2:
     case PPCE500_PCI_IW1:
         idx = ((addr >> 5) & 0x3) - 1;
-        switch (addr & 0xC) {
+        switch (addr & 0x1F) {
         case PCI_PITAR:
             pci->pib[idx].pitar = value;
+            e500_update_piw(pci, idx);
             break;
         case PCI_PIWBAR:
             pci->pib[idx].piwbar = value;
+            e500_update_piw(pci, idx);
             break;
         case PCI_PIWBEAR:
             pci->pib[idx].piwbear = value;
+            e500_update_piw(pci, idx);
             break;
         case PCI_PIWAR:
             pci->pib[idx].piwar = value;
+            e500_update_piw(pci, idx);
             break;
         default:
             break;
@@ -331,7 +417,7 @@
 
 #include "exec/address-spaces.h"
 
-static int e500_pcihost_bridge_initfn(PCIDevice *d)
+static void e500_pcihost_bridge_realize(PCIDevice *d, Error **errp)
 {
     PPCE500PCIBridgeState *b = PPC_E500_PCI_BRIDGE(d);
     PPCE500CCSRState *ccsr = CCSR(container_get(qdev_get_machine(),
@@ -345,8 +431,14 @@
     memory_region_init_alias(&b->bar0, OBJECT(ccsr), "e500-pci-bar0", &ccsr->ccsr_space,
                              0, int128_get64(ccsr->ccsr_space.size));
     pci_register_bar(d, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &b->bar0);
+}
 
-    return 0;
+static AddressSpace *e500_pcihost_set_iommu(PCIBus *bus, void *opaque,
+                                            int devfn)
+{
+    PPCE500PCIState *s = opaque;
+
+    return &s->bm_as;
 }
 
 static int e500_pcihost_initfn(SysBusDevice *dev)
@@ -355,7 +447,6 @@
     PPCE500PCIState *s;
     PCIBus *b;
     int i;
-    MemoryRegion *address_space_mem = get_system_memory();
 
     h = PCI_HOST_BRIDGE(dev);
     s = PPC_E500_PCI_HOST_BRIDGE(dev);
@@ -369,12 +460,22 @@
     }
 
     memory_region_init(&s->pio, OBJECT(s), "pci-pio", PCIE500_PCI_IOLEN);
+    memory_region_init(&s->busmem, OBJECT(s), "pci bus memory", UINT64_MAX);
+
+    /* PIO lives at the bottom of our bus space */
+    memory_region_add_subregion_overlap(&s->busmem, 0, &s->pio, -2);
 
     b = pci_register_bus(DEVICE(dev), NULL, mpc85xx_pci_set_irq,
-                         mpc85xx_pci_map_irq, s, address_space_mem,
-                         &s->pio, PCI_DEVFN(s->first_slot, 0), 4, TYPE_PCI_BUS);
+                         mpc85xx_pci_map_irq, s, &s->busmem, &s->pio,
+                         PCI_DEVFN(s->first_slot, 0), 4, TYPE_PCI_BUS);
     h->bus = b;
 
+    /* Set up PCI view of memory */
+    memory_region_init(&s->bm, OBJECT(s), "bm-e500", UINT64_MAX);
+    memory_region_add_subregion(&s->bm, 0x0, &s->busmem);
+    address_space_init(&s->bm_as, &s->bm, "pci-bm");
+    pci_setup_iommu(b, e500_pcihost_set_iommu, s);
+
     pci_create_simple(b, 0, "e500-host-bridge");
 
     memory_region_init(&s->container, OBJECT(h), "pci-container", PCIE500_ALL_SIZE);
@@ -388,7 +489,6 @@
     memory_region_add_subregion(&s->container, PCIE500_CFGDATA, &h->data_mem);
     memory_region_add_subregion(&s->container, PCIE500_REG_BASE, &s->iomem);
     sysbus_init_mmio(dev, &s->container);
-    sysbus_init_mmio(dev, &s->pio);
     pci_bus_set_route_irq_fn(b, e500_route_intx_pin_to_irq);
 
     return 0;
@@ -399,7 +499,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = e500_pcihost_bridge_initfn;
+    k->realize = e500_pcihost_bridge_realize;
     k->vendor_id = PCI_VENDOR_ID_FREESCALE;
     k->device_id = PCI_DEVICE_ID_MPC8533E;
     k->class_id = PCI_CLASS_PROCESSOR_POWERPC;
diff --git a/hw/pci-host/prep.c b/hw/pci-host/prep.c
index 1de3681..5580293 100644
--- a/hw/pci-host/prep.c
+++ b/hw/pci-host/prep.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bus.h"
@@ -140,7 +142,8 @@
     uint8_t buf[4];
 
     addr = raven_io_address(s, addr);
-    address_space_read(&s->pci_io_as, addr + 0x80000000, buf, size);
+    address_space_read(&s->pci_io_as, addr + 0x80000000,
+                       MEMTXATTRS_UNSPECIFIED, buf, size);
 
     if (size == 1) {
         return buf[0];
@@ -171,7 +174,8 @@
         g_assert_not_reached();
     }
 
-    address_space_write(&s->pci_io_as, addr + 0x80000000, buf, size);
+    address_space_write(&s->pci_io_as, addr + 0x80000000,
+                        MEMTXATTRS_UNSPECIFIED, buf, size);
 }
 
 static const MemoryRegionOps raven_io_ops = {
@@ -243,6 +247,7 @@
     memory_region_add_subregion(address_space_mem, 0xbffffff0, &s->pci_intack);
 
     /* TODO Remove once realize propagates to child devices. */
+    object_property_set_bool(OBJECT(&s->pci_bus), true, "realized", errp);
     object_property_set_bool(OBJECT(&s->pci_dev), true, "realized", errp);
 }
 
@@ -289,7 +294,7 @@
     qdev_prop_set_bit(pci_dev, "multifunction", false);
 }
 
-static int raven_init(PCIDevice *d)
+static void raven_realize(PCIDevice *d, Error **errp)
 {
     RavenPCIState *s = RAVEN_PCI_DEVICE(d);
     char *filename;
@@ -300,7 +305,7 @@
     d->config[0x34] = 0x00; // capabilities_pointer
 
     memory_region_init_ram(&s->bios, OBJECT(s), "bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     memory_region_set_readonly(&s->bios, true);
     memory_region_add_subregion(get_system_memory(), (uint32_t)(-BIOS_SIZE),
                                 &s->bios);
@@ -310,7 +315,7 @@
         if (filename) {
             if (s->elf_machine != EM_NONE) {
                 bios_size = load_elf(filename, NULL, NULL, NULL,
-                                     NULL, NULL, 1, s->elf_machine, 0);
+                                     NULL, NULL, 1, s->elf_machine, 0, 0);
             }
             if (bios_size < 0) {
                 bios_size = get_image_size(filename);
@@ -324,14 +329,11 @@
             }
         }
         if (bios_size < 0 || bios_size > BIOS_SIZE) {
+            /* FIXME should error_setg() */
             hw_error("qemu: could not load bios image '%s'\n", s->bios_name);
         }
-        if (filename) {
-            g_free(filename);
-        }
+        g_free(filename);
     }
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_raven = {
@@ -349,7 +351,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = raven_init;
+    k->realize = raven_realize;
     k->vendor_id = PCI_VENDOR_ID_MOTOROLA;
     k->device_id = PCI_DEVICE_ID_MOTOROLA_RAVEN;
     k->revision = 0x00;
@@ -357,8 +359,9 @@
     dc->desc = "PReP Host Bridge - Motorola Raven";
     dc->vmsd = &vmstate_raven;
     /*
-     * PCI-facing part of the host bridge, not usable without the
-     * host-facing part, which can't be device_add'ed, yet.
+     * Reason: PCI-facing part of the host bridge, not usable without
+     * the host-facing part, which can't be device_add'ed, yet.
+     * Reason: realize() method uses hw_error().
      */
     dc->cannot_instantiate_with_device_add_yet = true;
 }
diff --git a/hw/pci-host/q35.c b/hw/pci-host/q35.c
index b20bad8..344f77b 100644
--- a/hw/pci-host/q35.c
+++ b/hw/pci-host/q35.c
@@ -27,8 +27,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci-host/q35.h"
+#include "qapi/error.h"
 #include "qapi/visitor.h"
 
 /****************************************************************************
@@ -50,6 +52,7 @@
     pci->bus = pci_bus_new(DEVICE(s), "pcie.0",
                            s->mch.pci_address_space, s->mch.address_space_io,
                            0, TYPE_PCIE_BUS);
+    PC_MACHINE(qdev_get_machine())->bus = pci->bus;
     qdev_set_parent_bus(DEVICE(&s->mch), BUS(pci->bus));
     qdev_init_nofail(DEVICE(&s->mch));
 }
@@ -67,57 +70,68 @@
 }
 
 static void q35_host_get_pci_hole_start(Object *obj, Visitor *v,
-                                        void *opaque, const char *name,
+                                        const char *name, void *opaque,
                                         Error **errp)
 {
     Q35PCIHost *s = Q35_HOST_DEVICE(obj);
-    uint32_t value = s->mch.pci_info.w32.begin;
+    uint64_t val64;
+    uint32_t value;
 
-    visit_type_uint32(v, &value, name, errp);
+    val64 = range_is_empty(&s->mch.pci_hole)
+        ? 0 : range_lob(&s->mch.pci_hole);
+    value = val64;
+    assert(value == val64);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void q35_host_get_pci_hole_end(Object *obj, Visitor *v,
-                                      void *opaque, const char *name,
+                                      const char *name, void *opaque,
                                       Error **errp)
 {
     Q35PCIHost *s = Q35_HOST_DEVICE(obj);
-    uint32_t value = s->mch.pci_info.w32.end;
+    uint64_t val64;
+    uint32_t value;
 
-    visit_type_uint32(v, &value, name, errp);
+    val64 = range_is_empty(&s->mch.pci_hole)
+        ? 0 : range_upb(&s->mch.pci_hole) + 1;
+    value = val64;
+    assert(value == val64);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static void q35_host_get_pci_hole64_start(Object *obj, Visitor *v,
-                                          void *opaque, const char *name,
+                                          const char *name, void *opaque,
                                           Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
     Range w64;
+    uint64_t value;
 
     pci_bus_get_w64_range(h->bus, &w64);
-
-    visit_type_uint64(v, &w64.begin, name, errp);
+    value = range_is_empty(&w64) ? 0 : range_lob(&w64);
+    visit_type_uint64(v, name, &value, errp);
 }
 
 static void q35_host_get_pci_hole64_end(Object *obj, Visitor *v,
-                                        void *opaque, const char *name,
+                                        const char *name, void *opaque,
                                         Error **errp)
 {
     PCIHostState *h = PCI_HOST_BRIDGE(obj);
     Range w64;
+    uint64_t value;
 
     pci_bus_get_w64_range(h->bus, &w64);
-
-    visit_type_uint64(v, &w64.end, name, errp);
+    value = range_is_empty(&w64) ? 0 : range_upb(&w64) + 1;
+    visit_type_uint64(v, name, &value, errp);
 }
 
-static void q35_host_get_mmcfg_size(Object *obj, Visitor *v,
-                                    void *opaque, const char *name,
-                                    Error **errp)
+static void q35_host_get_mmcfg_size(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     PCIExpressHost *e = PCIE_HOST_BRIDGE(obj);
     uint32_t value = e->size;
 
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
 static Property mch_props[] = {
@@ -126,6 +140,10 @@
     DEFINE_PROP_SIZE(PCI_HOST_PROP_PCI_HOLE64_SIZE, Q35PCIHost,
                      mch.pci_hole64_size, DEFAULT_PCI_HOLE64_SIZE),
     DEFINE_PROP_UINT32("short_root_bus", Q35PCIHost, mch.short_root_bus, 0),
+    DEFINE_PROP_SIZE(PCI_HOST_BELOW_4G_MEM_SIZE, Q35PCIHost,
+                     mch.below_4g_mem_size, 0),
+    DEFINE_PROP_SIZE(PCI_HOST_ABOVE_4G_MEM_SIZE, Q35PCIHost,
+                     mch.above_4g_mem_size, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -137,6 +155,8 @@
     hc->root_bus_path = q35_host_root_bus_path;
     dc->realize = q35_host_realize;
     dc->props = mch_props;
+    /* Reason: needs to be wired up by pc_q35_init */
+    dc->cannot_instantiate_with_device_add_yet = true;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
     dc->fw_name = "pci";
 }
@@ -176,14 +196,30 @@
                         q35_host_get_mmcfg_size,
                         NULL, NULL, NULL, NULL);
 
+    object_property_add_link(obj, MCH_HOST_PROP_RAM_MEM, TYPE_MEMORY_REGION,
+                             (Object **) &s->mch.ram_memory,
+                             qdev_prop_allow_set_link_before_realize, 0, NULL);
+
+    object_property_add_link(obj, MCH_HOST_PROP_PCI_MEM, TYPE_MEMORY_REGION,
+                             (Object **) &s->mch.pci_address_space,
+                             qdev_prop_allow_set_link_before_realize, 0, NULL);
+
+    object_property_add_link(obj, MCH_HOST_PROP_SYSTEM_MEM, TYPE_MEMORY_REGION,
+                             (Object **) &s->mch.system_memory,
+                             qdev_prop_allow_set_link_before_realize, 0, NULL);
+
+    object_property_add_link(obj, MCH_HOST_PROP_IO_MEM, TYPE_MEMORY_REGION,
+                             (Object **) &s->mch.address_space_io,
+                             qdev_prop_allow_set_link_before_realize, 0, NULL);
+
     /* Leave enough space for the biggest MCFG BAR */
     /* TODO: this matches current bios behaviour, but
      * it's not a power of two, which means an MTRR
      * can't cover it exactly.
      */
-    s->mch.pci_info.w32.begin = MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT +
-        MCH_HOST_BRIDGE_PCIEXBAR_MAX;
-    s->mch.pci_info.w32.end = IO_APIC_DEFAULT_ADDRESS;
+    range_set_bounds(&s->mch.pci_hole,
+            MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT + MCH_HOST_BRIDGE_PCIEXBAR_MAX,
+            IO_APIC_DEFAULT_ADDRESS - 1);
 }
 
 static const TypeInfo q35_host_info = {
@@ -198,6 +234,28 @@
  * MCH D0:F0
  */
 
+static uint64_t tseg_blackhole_read(void *ptr, hwaddr reg, unsigned size)
+{
+    return 0xffffffff;
+}
+
+static void tseg_blackhole_write(void *opaque, hwaddr addr, uint64_t val,
+                                 unsigned width)
+{
+    /* nothing */
+}
+
+static const MemoryRegionOps tseg_blackhole_ops = {
+    .read = tseg_blackhole_read,
+    .write = tseg_blackhole_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 4,
+    .impl.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
 /* PCIe MMCFG */
 static void mch_update_pciexbar(MCHPCIState *mch)
 {
@@ -229,10 +287,7 @@
         break;
     case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_RVD:
     default:
-        enable = 0;
-        length = 0;
         abort();
-        break;
     }
     addr = pciexbar & addr_mask;
     pcie_host_mmcfg_update(pehb, enable, addr, length);
@@ -242,9 +297,13 @@
      * which means an MTRR can't cover it exactly.
      */
     if (enable) {
-        mch->pci_info.w32.begin = addr + length;
+        range_set_bounds(&mch->pci_hole,
+                         addr + length,
+                         IO_APIC_DEFAULT_ADDRESS - 1);
     } else {
-        mch->pci_info.w32.begin = MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT;
+        range_set_bounds(&mch->pci_hole,
+                         MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT,
+                         IO_APIC_DEFAULT_ADDRESS - 1);
     }
 }
 
@@ -266,21 +325,70 @@
 static void mch_update_smram(MCHPCIState *mch)
 {
     PCIDevice *pd = PCI_DEVICE(mch);
+    bool h_smrame = (pd->config[MCH_HOST_BRIDGE_ESMRAMC] & MCH_HOST_BRIDGE_ESMRAMC_H_SMRAME);
+    uint32_t tseg_size;
+
+    /* implement SMRAM.D_LCK */
+    if (pd->config[MCH_HOST_BRIDGE_SMRAM] & MCH_HOST_BRIDGE_SMRAM_D_LCK) {
+        pd->config[MCH_HOST_BRIDGE_SMRAM] &= ~MCH_HOST_BRIDGE_SMRAM_D_OPEN;
+        pd->wmask[MCH_HOST_BRIDGE_SMRAM] = MCH_HOST_BRIDGE_SMRAM_WMASK_LCK;
+        pd->wmask[MCH_HOST_BRIDGE_ESMRAMC] = MCH_HOST_BRIDGE_ESMRAMC_WMASK_LCK;
+    }
 
     memory_region_transaction_begin();
-    smram_update(&mch->smram_region, pd->config[MCH_HOST_BRIDGE_SMRAM],
-                    mch->smm_enabled);
-    memory_region_transaction_commit();
-}
 
-static void mch_set_smm(int smm, void *arg)
-{
-    MCHPCIState *mch = arg;
-    PCIDevice *pd = PCI_DEVICE(mch);
+    if (pd->config[MCH_HOST_BRIDGE_SMRAM] & SMRAM_D_OPEN) {
+        /* Hide (!) low SMRAM if H_SMRAME = 1 */
+        memory_region_set_enabled(&mch->smram_region, h_smrame);
+        /* Show high SMRAM if H_SMRAME = 1 */
+        memory_region_set_enabled(&mch->open_high_smram, h_smrame);
+    } else {
+        /* Hide high SMRAM and low SMRAM */
+        memory_region_set_enabled(&mch->smram_region, true);
+        memory_region_set_enabled(&mch->open_high_smram, false);
+    }
 
-    memory_region_transaction_begin();
-    smram_set_smm(&mch->smm_enabled, smm, pd->config[MCH_HOST_BRIDGE_SMRAM],
-                    &mch->smram_region);
+    if (pd->config[MCH_HOST_BRIDGE_SMRAM] & SMRAM_G_SMRAME) {
+        memory_region_set_enabled(&mch->low_smram, !h_smrame);
+        memory_region_set_enabled(&mch->high_smram, h_smrame);
+    } else {
+        memory_region_set_enabled(&mch->low_smram, false);
+        memory_region_set_enabled(&mch->high_smram, false);
+    }
+
+    if (pd->config[MCH_HOST_BRIDGE_ESMRAMC] & MCH_HOST_BRIDGE_ESMRAMC_T_EN) {
+        switch (pd->config[MCH_HOST_BRIDGE_ESMRAMC] &
+                MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_MASK) {
+        case MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_1MB:
+            tseg_size = 1024 * 1024;
+            break;
+        case MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_2MB:
+            tseg_size = 1024 * 1024 * 2;
+            break;
+        case MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_8MB:
+            tseg_size = 1024 * 1024 * 8;
+            break;
+        default:
+            tseg_size = 0;
+            break;
+        }
+    } else {
+        tseg_size = 0;
+    }
+    memory_region_del_subregion(mch->system_memory, &mch->tseg_blackhole);
+    memory_region_set_enabled(&mch->tseg_blackhole, tseg_size);
+    memory_region_set_size(&mch->tseg_blackhole, tseg_size);
+    memory_region_add_subregion_overlap(mch->system_memory,
+                                        mch->below_4g_mem_size - tseg_size,
+                                        &mch->tseg_blackhole, 1);
+
+    memory_region_set_enabled(&mch->tseg_window, tseg_size);
+    memory_region_set_size(&mch->tseg_window, tseg_size);
+    memory_region_set_address(&mch->tseg_window,
+                              mch->below_4g_mem_size - tseg_size);
+    memory_region_set_alias_offset(&mch->tseg_window,
+                                   mch->below_4g_mem_size - tseg_size);
+
     memory_region_transaction_commit();
 }
 
@@ -289,7 +397,6 @@
 {
     MCHPCIState *mch = MCH_PCI_DEVICE(d);
 
-    /* XXX: implement SMRAM.D_LOCK */
     pci_default_write_config(d, address, val, len);
 
     if (ranges_overlap(address, len, MCH_HOST_BRIDGE_PAM0,
@@ -329,7 +436,10 @@
     .post_load = mch_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(parent_obj, MCHPCIState),
-        VMSTATE_UINT8(smm_enabled, MCHPCIState),
+        /* Used to be smm_enabled, which was basically always zero because
+         * SeaBIOS hardly uses SMM.  SMRAM is now handled by CPU code.
+         */
+        VMSTATE_UNUSED(1),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -343,54 +453,14 @@
                  MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT);
 
     d->config[MCH_HOST_BRIDGE_SMRAM] = MCH_HOST_BRIDGE_SMRAM_DEFAULT;
+    d->config[MCH_HOST_BRIDGE_ESMRAMC] = MCH_HOST_BRIDGE_ESMRAMC_DEFAULT;
+    d->wmask[MCH_HOST_BRIDGE_SMRAM] = MCH_HOST_BRIDGE_SMRAM_WMASK;
+    d->wmask[MCH_HOST_BRIDGE_ESMRAMC] = MCH_HOST_BRIDGE_ESMRAMC_WMASK;
 
     mch_update(mch);
 }
 
-static AddressSpace *q35_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
-{
-    IntelIOMMUState *s = opaque;
-    VTDAddressSpace **pvtd_as;
-    int bus_num = pci_bus_num(bus);
-
-    assert(0 <= bus_num && bus_num <= VTD_PCI_BUS_MAX);
-    assert(0 <= devfn && devfn <= VTD_PCI_DEVFN_MAX);
-
-    pvtd_as = s->address_spaces[bus_num];
-    if (!pvtd_as) {
-        /* No corresponding free() */
-        pvtd_as = g_malloc0(sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX);
-        s->address_spaces[bus_num] = pvtd_as;
-    }
-    if (!pvtd_as[devfn]) {
-        pvtd_as[devfn] = g_malloc0(sizeof(VTDAddressSpace));
-
-        pvtd_as[devfn]->bus_num = (uint8_t)bus_num;
-        pvtd_as[devfn]->devfn = (uint8_t)devfn;
-        pvtd_as[devfn]->iommu_state = s;
-        pvtd_as[devfn]->context_cache_entry.context_cache_gen = 0;
-        memory_region_init_iommu(&pvtd_as[devfn]->iommu, OBJECT(s),
-                                 &s->iommu_ops, "intel_iommu", UINT64_MAX);
-        address_space_init(&pvtd_as[devfn]->as,
-                           &pvtd_as[devfn]->iommu, "intel_iommu");
-    }
-    return &pvtd_as[devfn]->as;
-}
-
-static void mch_init_dmar(MCHPCIState *mch)
-{
-    PCIBus *pci_bus = PCI_BUS(qdev_get_parent_bus(DEVICE(mch)));
-
-    mch->iommu = INTEL_IOMMU_DEVICE(qdev_create(NULL, TYPE_INTEL_IOMMU_DEVICE));
-    object_property_add_child(OBJECT(mch), "intel-iommu",
-                              OBJECT(mch->iommu), NULL);
-    qdev_init_nofail(DEVICE(mch->iommu));
-    sysbus_mmio_map(SYS_BUS_DEVICE(mch->iommu), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
-
-    pci_setup_iommu(pci_bus, q35_host_dma_iommu, mch->iommu);
-}
-
-static int mch_init(PCIDevice *d)
+static void mch_realize(PCIDevice *d, Error **errp)
 {
     int i;
     MCHPCIState *mch = MCH_PCI_DEVICE(d);
@@ -399,13 +469,47 @@
     pc_pci_as_mapping_init(OBJECT(mch), mch->system_memory,
                            mch->pci_address_space);
 
-    /* smram */
-    cpu_smm_register(&mch_set_smm, mch);
+    /* if *disabled* show SMRAM to all CPUs */
     memory_region_init_alias(&mch->smram_region, OBJECT(mch), "smram-region",
                              mch->pci_address_space, 0xa0000, 0x20000);
     memory_region_add_subregion_overlap(mch->system_memory, 0xa0000,
                                         &mch->smram_region, 1);
-    memory_region_set_enabled(&mch->smram_region, false);
+    memory_region_set_enabled(&mch->smram_region, true);
+
+    memory_region_init_alias(&mch->open_high_smram, OBJECT(mch), "smram-open-high",
+                             mch->ram_memory, 0xa0000, 0x20000);
+    memory_region_add_subregion_overlap(mch->system_memory, 0xfeda0000,
+                                        &mch->open_high_smram, 1);
+    memory_region_set_enabled(&mch->open_high_smram, false);
+
+    /* smram, as seen by SMM CPUs */
+    memory_region_init(&mch->smram, OBJECT(mch), "smram", 1ull << 32);
+    memory_region_set_enabled(&mch->smram, true);
+    memory_region_init_alias(&mch->low_smram, OBJECT(mch), "smram-low",
+                             mch->ram_memory, 0xa0000, 0x20000);
+    memory_region_set_enabled(&mch->low_smram, true);
+    memory_region_add_subregion(&mch->smram, 0xa0000, &mch->low_smram);
+    memory_region_init_alias(&mch->high_smram, OBJECT(mch), "smram-high",
+                             mch->ram_memory, 0xa0000, 0x20000);
+    memory_region_set_enabled(&mch->high_smram, true);
+    memory_region_add_subregion(&mch->smram, 0xfeda0000, &mch->high_smram);
+
+    memory_region_init_io(&mch->tseg_blackhole, OBJECT(mch),
+                          &tseg_blackhole_ops, NULL,
+                          "tseg-blackhole", 0);
+    memory_region_set_enabled(&mch->tseg_blackhole, false);
+    memory_region_add_subregion_overlap(mch->system_memory,
+                                        mch->below_4g_mem_size,
+                                        &mch->tseg_blackhole, 1);
+
+    memory_region_init_alias(&mch->tseg_window, OBJECT(mch), "tseg-window",
+                             mch->ram_memory, mch->below_4g_mem_size, 0);
+    memory_region_set_enabled(&mch->tseg_window, false);
+    memory_region_add_subregion(&mch->smram, mch->below_4g_mem_size,
+                                &mch->tseg_window);
+    object_property_add_const_link(qdev_get_machine(), "smram",
+                                   OBJECT(&mch->smram), &error_abort);
+
     init_pam(DEVICE(mch), mch->ram_memory, mch->system_memory,
              mch->pci_address_space, &mch->pam_regions[0],
              PAM_BIOS_BASE, PAM_BIOS_SIZE);
@@ -414,11 +518,6 @@
                  mch->pci_address_space, &mch->pam_regions[i+1],
                  PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE, PAM_EXPAN_SIZE);
     }
-    /* Intel IOMMU (VT-d) */
-    if (qemu_opt_get_bool(qemu_get_machine_opts(), "iommu", false)) {
-        mch_init_dmar(mch);
-    }
-    return 0;
 }
 
 uint64_t mch_mcfg_base(void)
@@ -436,7 +535,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = mch_init;
+    k->realize = mch_realize;
     k->config_write = mch_write_config;
     dc->reset = mch_reset;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
diff --git a/hw/pci-host/uninorth.c b/hw/pci-host/uninorth.c
index 21f805f..7aac4d6 100644
--- a/hw/pci-host/uninorth.c
+++ b/hw/pci-host/uninorth.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/mac.h"
 #include "hw/pci/pci.h"
@@ -61,12 +62,9 @@
 
 static int pci_unin_map_irq(PCIDevice *pci_dev, int irq_num)
 {
-    int retval;
     int devfn = pci_dev->devfn & 0x00FFFFFF;
 
-    retval = (((devfn >> 11) & 0x1F) + irq_num) & 3;
-
-    return retval;
+    return (((devfn >> 11) & 0x1F) + irq_num) & 3;
 }
 
 static void pci_unin_set_irq(void *opaque, int irq_num, int level)
@@ -92,7 +90,10 @@
         uint32_t slot, func;
 
         /* Grab CFA0 style values */
-        slot = ffs(reg & 0xfffff800) - 1;
+        slot = ctz32(reg & 0xfffff800);
+        if (slot == 32) {
+            slot = -1; /* XXX: should this be 0? */
+        }
         func = (reg >> 8) & 7;
 
         /* ... and then convert them to x86 format */
@@ -116,7 +117,7 @@
 {
     UNINState *s = opaque;
     PCIHostState *phb = PCI_HOST_BRIDGE(s);
-    UNIN_DPRINTF("write addr %" TARGET_FMT_plx " len %d val %"PRIx64"\n",
+    UNIN_DPRINTF("write addr " TARGET_FMT_plx " len %d val %"PRIx64"\n",
                  addr, len, val);
     pci_data_write(phb->bus,
                    unin_get_config_reg(phb->config_reg, addr),
@@ -133,7 +134,7 @@
     val = pci_data_read(phb->bus,
                         unin_get_config_reg(phb->config_reg, addr),
                         len);
-    UNIN_DPRINTF("read addr %" TARGET_FMT_plx " len %d val %x\n",
+    UNIN_DPRINTF("read addr " TARGET_FMT_plx " len %d val %x\n",
                  addr, len, val);
     return val;
 }
@@ -315,37 +316,42 @@
     return h->bus;
 }
 
-static int unin_main_pci_host_init(PCIDevice *d)
+static void unin_main_pci_host_realize(PCIDevice *d, Error **errp)
 {
     d->config[0x0C] = 0x08; // cache_line_size
     d->config[0x0D] = 0x10; // latency_timer
     d->config[0x34] = 0x00; // capabilities_pointer
-    return 0;
 }
 
-static int unin_agp_pci_host_init(PCIDevice *d)
+static void unin_agp_pci_host_realize(PCIDevice *d, Error **errp)
 {
     d->config[0x0C] = 0x08; // cache_line_size
     d->config[0x0D] = 0x10; // latency_timer
     //    d->config[0x34] = 0x80; // capabilities_pointer
-    return 0;
+    /*
+     * Set kMacRISCPCIAddressSelect (0x48) register to indicate PCI
+     * memory space with base 0x80000000, size 0x10000000 for Apple's
+     * AppleMacRiscPCI driver
+     */
+    d->config[0x48] = 0x0;
+    d->config[0x49] = 0x0;
+    d->config[0x4a] = 0x0;
+    d->config[0x4b] = 0x1;
 }
 
-static int u3_agp_pci_host_init(PCIDevice *d)
+static void u3_agp_pci_host_realize(PCIDevice *d, Error **errp)
 {
     /* cache line size */
     d->config[0x0C] = 0x08;
     /* latency timer */
     d->config[0x0D] = 0x10;
-    return 0;
 }
 
-static int unin_internal_pci_host_init(PCIDevice *d)
+static void unin_internal_pci_host_realize(PCIDevice *d, Error **errp)
 {
     d->config[0x0C] = 0x08; // cache_line_size
     d->config[0x0D] = 0x10; // latency_timer
     d->config[0x34] = 0x00; // capabilities_pointer
-    return 0;
 }
 
 static void unin_main_pci_host_class_init(ObjectClass *klass, void *data)
@@ -353,7 +359,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init      = unin_main_pci_host_init;
+    k->realize   = unin_main_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_UNI_N_PCI;
     k->revision  = 0x00;
@@ -377,7 +383,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init      = u3_agp_pci_host_init;
+    k->realize   = u3_agp_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_U3_AGP;
     k->revision  = 0x00;
@@ -401,7 +407,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init      = unin_agp_pci_host_init;
+    k->realize   = unin_agp_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_UNI_N_AGP;
     k->revision  = 0x00;
@@ -425,7 +431,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init      = unin_internal_pci_host_init;
+    k->realize   = unin_internal_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_UNI_N_I_PCI;
     k->revision  = 0x00;
@@ -447,8 +453,10 @@
 static void pci_unin_main_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_main_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_main_info = {
@@ -461,8 +469,10 @@
 static void pci_u3_agp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_u3_agp_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_u3_agp_info = {
@@ -475,8 +485,10 @@
 static void pci_unin_agp_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_agp_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_agp_info = {
@@ -489,8 +501,10 @@
 static void pci_unin_internal_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
 
     sbc->init = pci_unin_internal_init_device;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 }
 
 static const TypeInfo pci_unin_internal_info = {
diff --git a/hw/pci-host/versatile.c b/hw/pci-host/versatile.c
index 71ff0de..467cbb9 100644
--- a/hw/pci-host/versatile.c
+++ b/hw/pci-host/versatile.c
@@ -7,11 +7,13 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci/pci_host.h"
 #include "exec/address-spaces.h"
+#include "qemu/log.h"
 
 /* Old and buggy versions of QEMU used the wrong mapping from
  * PCI IRQs to system interrupt lines. Unfortunately the Linux
@@ -453,15 +455,15 @@
     }
 
     /* TODO Remove once realize propagates to child devices. */
+    object_property_set_bool(OBJECT(&s->pci_bus), true, "realized", errp);
     object_property_set_bool(OBJECT(&s->pci_dev), true, "realized", errp);
 }
 
-static int versatile_pci_host_init(PCIDevice *d)
+static void versatile_pci_host_realize(PCIDevice *d, Error **errp)
 {
     pci_set_word(d->config + PCI_STATUS,
                  PCI_STATUS_66MHZ | PCI_STATUS_DEVSEL_MEDIUM);
     pci_set_byte(d->config + PCI_LATENCY_TIMER, 0x10);
-    return 0;
 }
 
 static void versatile_pci_host_class_init(ObjectClass *klass, void *data)
@@ -469,7 +471,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = versatile_pci_host_init;
+    k->realize = versatile_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_XILINX;
     k->device_id = PCI_DEVICE_ID_XILINX_XC2VP30;
     k->class_id = PCI_CLASS_PROCESSOR_CO;
@@ -501,6 +503,8 @@
     dc->reset = pci_vpb_reset;
     dc->vmsd = &pci_vpb_vmstate;
     dc->props = pci_vpb_properties;
+    /* Reason: object_unref() hangs */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo pci_vpb_info = {
@@ -522,10 +526,19 @@
     s->mem_win_size[2] = 0x08000000;
 }
 
+static void pci_realview_class_init(ObjectClass *class, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(class);
+
+    /* Reason: object_unref() hangs */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
 static const TypeInfo pci_realview_info = {
     .name          = "realview_pci",
     .parent        = TYPE_VERSATILE_PCI,
     .instance_init = pci_realview_init,
+    .class_init    = pci_realview_class_init,
 };
 
 static void versatile_pci_register_types(void)
diff --git a/hw/pci/Makefile.objs b/hw/pci/Makefile.objs
index 80f8aa6..9f905e6 100644
--- a/hw/pci/Makefile.objs
+++ b/hw/pci/Makefile.objs
@@ -7,5 +7,3 @@
 
 common-obj-$(call lnot,$(CONFIG_PCI)) += pci-stub.o
 common-obj-$(CONFIG_ALL) += pci-stub.o
-
-common-obj-$(CONFIG_PCI_HOTPLUG_OLD) += pci-hotplug-old.o
diff --git a/hw/pci/msi.c b/hw/pci/msi.c
index 330284a..c328a6c 100644
--- a/hw/pci/msi.c
+++ b/hw/pci/msi.c
@@ -18,12 +18,11 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/msi.h"
+#include "hw/xen/xen.h"
 #include "qemu/range.h"
-
-/* Eventually those constants should go to Linux pci_regs.h */
-#define PCI_MSI_PENDING_32      0x10
-#define PCI_MSI_PENDING_64      0x14
+#include "qapi/error.h"
 
 /* PCI_MSI_ADDRESS_LO */
 #define PCI_MSI_ADDRESS_LO_MASK         (~0x3)
@@ -36,8 +35,21 @@
 
 #define PCI_MSI_VECTORS_MAX     32
 
-/* Flag for interrupt controller to declare MSI/MSI-X support */
-bool msi_supported;
+/*
+ * Flag for interrupt controllers to declare broken MSI/MSI-X support.
+ * values: false - broken; true - non-broken.
+ *
+ * Setting this flag to false will remove MSI/MSI-X capability from all devices.
+ *
+ * It is preferable for controllers to set this to true (non-broken) even if
+ * they do not actually support MSI/MSI-X: guests normally probe the controller
+ * type and do not attempt to enable MSI/MSI-X with interrupt controllers not
+ * supporting such, so removing the capability is not required, and
+ * it seems cleaner to have a given device look the same for all boards.
+ *
+ * TODO: some existing controllers violate the above rule. Identify and fix them.
+ */
+bool msi_nonbroken;
 
 /* If we get rid of cap allocator, we won't need this. */
 static inline uint8_t msi_cap_sizeof(uint16_t flags)
@@ -72,7 +84,7 @@
 static inline unsigned int msi_nr_vectors(uint16_t flags)
 {
     return 1U <<
-        ((flags & PCI_MSI_FLAGS_QSIZE) >> (ffs(PCI_MSI_FLAGS_QSIZE) - 1));
+        ((flags & PCI_MSI_FLAGS_QSIZE) >> ctz32(PCI_MSI_FLAGS_QSIZE));
 }
 
 static inline uint8_t msi_flags_off(const PCIDevice* dev)
@@ -154,15 +166,33 @@
          PCI_MSI_FLAGS_ENABLE);
 }
 
+/*
+ * Make PCI device @dev MSI-capable.
+ * Non-zero @offset puts capability MSI at that offset in PCI config
+ * space.
+ * @nr_vectors is the number of MSI vectors (1, 2, 4, 8, 16 or 32).
+ * If @msi64bit, make the device capable of sending a 64-bit message
+ * address.
+ * If @msi_per_vector_mask, make the device support per-vector masking.
+ * @errp is for returning errors.
+ * Return 0 on success; set @errp and return -errno on error.
+ *
+ * -ENOTSUP means lacking msi support for a msi-capable platform.
+ * -EINVAL means capability overlap, happens when @offset is non-zero,
+ *  also means a programming error, except device assignment, which can check
+ *  if a real HW is broken.
+ */
 int msi_init(struct PCIDevice *dev, uint8_t offset,
-             unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask)
+             unsigned int nr_vectors, bool msi64bit,
+             bool msi_per_vector_mask, Error **errp)
 {
     unsigned int vectors_order;
     uint16_t flags;
     uint8_t cap_size;
     int config_offset;
 
-    if (!msi_supported) {
+    if (!msi_nonbroken) {
+        error_setg(errp, "MSI is not supported by interrupt controller");
         return -ENOTSUP;
     }
 
@@ -175,9 +205,9 @@
     assert(nr_vectors > 0);
     assert(nr_vectors <= PCI_MSI_VECTORS_MAX);
     /* the nr of MSI vectors is up to 32 */
-    vectors_order = ffs(nr_vectors) - 1;
+    vectors_order = ctz32(nr_vectors);
 
-    flags = vectors_order << (ffs(PCI_MSI_FLAGS_QMASK) - 1);
+    flags = vectors_order << ctz32(PCI_MSI_FLAGS_QMASK);
     if (msi64bit) {
         flags |= PCI_MSI_FLAGS_64BIT;
     }
@@ -186,7 +216,8 @@
     }
 
     cap_size = msi_cap_sizeof(flags);
-    config_offset = pci_add_capability(dev, PCI_CAP_ID_MSI, offset, cap_size);
+    config_offset = pci_add_capability2(dev, PCI_CAP_ID_MSI, offset,
+                                        cap_size, errp);
     if (config_offset < 0) {
         return config_offset;
     }
@@ -209,7 +240,8 @@
         pci_set_long(dev->wmask + msi_mask_off(dev, msi64bit),
                      0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors));
     }
-    return config_offset;
+
+    return 0;
 }
 
 void msi_uninit(struct PCIDevice *dev)
@@ -257,13 +289,19 @@
 static bool msi_is_masked(const PCIDevice *dev, unsigned int vector)
 {
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
-    uint32_t mask;
+    uint32_t mask, data;
+    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
     assert(vector < PCI_MSI_VECTORS_MAX);
 
     if (!(flags & PCI_MSI_FLAGS_MASKBIT)) {
         return false;
     }
 
+    data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
+    if (xen_is_pirq_msi(data)) {
+        return false;
+    }
+
     mask = pci_get_long(dev->config +
                         msi_mask_off(dev, flags & PCI_MSI_FLAGS_64BIT));
     return mask & (1U << vector);
@@ -292,7 +330,16 @@
                    "notify vector 0x%x"
                    " address: 0x%"PRIx64" data: 0x%"PRIx32"\n",
                    vector, msg.address, msg.data);
-    stl_le_phys(&dev->bus_master_as, msg.address, msg.data);
+    msi_send_message(dev, msg);
+}
+
+void msi_send_message(PCIDevice *dev, MSIMessage msg)
+{
+    MemTxAttrs attrs = {};
+
+    attrs.requester_id = pci_requester_id(dev);
+    address_space_stl_le(&dev->bus_master_as, msg.address, msg.data,
+                         attrs, NULL);
 }
 
 /* Normally called by pci_default_write_config(). */
@@ -355,12 +402,12 @@
      * just don't crash the host
      */
     log_num_vecs =
-        (flags & PCI_MSI_FLAGS_QSIZE) >> (ffs(PCI_MSI_FLAGS_QSIZE) - 1);
+        (flags & PCI_MSI_FLAGS_QSIZE) >> ctz32(PCI_MSI_FLAGS_QSIZE);
     log_max_vecs =
-        (flags & PCI_MSI_FLAGS_QMASK) >> (ffs(PCI_MSI_FLAGS_QMASK) - 1);
+        (flags & PCI_MSI_FLAGS_QMASK) >> ctz32(PCI_MSI_FLAGS_QMASK);
     if (log_num_vecs > log_max_vecs) {
         flags &= ~PCI_MSI_FLAGS_QSIZE;
-        flags |= log_max_vecs << (ffs(PCI_MSI_FLAGS_QSIZE) - 1);
+        flags |= log_max_vecs << ctz32(PCI_MSI_FLAGS_QSIZE);
         pci_set_word(dev->config + msi_flags_off(dev), flags);
     }
 
diff --git a/hw/pci/msix.c b/hw/pci/msix.c
index 128f707..ab457b0 100644
--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -14,10 +14,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "hw/pci/pci.h"
+#include "hw/xen/xen.h"
 #include "qemu/range.h"
 
 #define MSIX_CAP_LENGTH 12
@@ -70,15 +72,22 @@
     *msix_pending_byte(dev, vector) |= msix_pending_mask(vector);
 }
 
-static void msix_clr_pending(PCIDevice *dev, int vector)
+void msix_clr_pending(PCIDevice *dev, int vector)
 {
     *msix_pending_byte(dev, vector) &= ~msix_pending_mask(vector);
 }
 
 static bool msix_vector_masked(PCIDevice *dev, unsigned int vector, bool fmask)
 {
-    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE + PCI_MSIX_ENTRY_VECTOR_CTRL;
-    return fmask || dev->msix_table[offset] & PCI_MSIX_ENTRY_CTRL_MASKBIT;
+    unsigned offset = vector * PCI_MSIX_ENTRY_SIZE;
+    uint8_t *data = &dev->msix_table[offset + PCI_MSIX_ENTRY_DATA];
+    /* MSIs on Xen can be remapped into pirqs. In those cases, masking
+     * and unmasking go through the PV evtchn path. */
+    if (xen_enabled() && xen_is_pirq_msi(pci_get_long(data))) {
+        return false;
+    }
+    return fmask || dev->msix_table[offset + PCI_MSIX_ENTRY_VECTOR_CTRL] &
+        PCI_MSIX_ENTRY_CTRL_MASKBIT;
 }
 
 bool msix_is_masked(PCIDevice *dev, unsigned int vector)
@@ -201,8 +210,14 @@
     return pci_get_long(dev->msix_pba + addr);
 }
 
+static void msix_pba_mmio_write(void *opaque, hwaddr addr,
+                                uint64_t val, unsigned size)
+{
+}
+
 static const MemoryRegionOps msix_pba_mmio_ops = {
     .read = msix_pba_mmio_read,
+    .write = msix_pba_mmio_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .valid = {
         .min_access_size = 4,
@@ -235,7 +250,7 @@
     uint8_t *config;
 
     /* Nothing to do if MSI is not supported by interrupt controller */
-    if (!msi_supported) {
+    if (!msi_nonbroken) {
         return -ENOTSUP;
     }
 
@@ -296,29 +311,35 @@
 {
     int ret;
     char *name;
+    uint32_t bar_size = 4096;
+    uint32_t bar_pba_offset = bar_size / 2;
+    uint32_t bar_pba_size = (nentries / 8 + 1) * 8;
 
     /*
      * Migration compatibility dictates that this remains a 4k
      * BAR with the vector table in the lower half and PBA in
-     * the upper half.  Do not use these elsewhere!
+     * the upper half for nentries which is lower or equal to 128.
+     * No need to care about using more than 65 entries for legacy
+     * machine types who has at most 64 queues.
      */
-#define MSIX_EXCLUSIVE_BAR_SIZE 4096
-#define MSIX_EXCLUSIVE_BAR_TABLE_OFFSET 0
-#define MSIX_EXCLUSIVE_BAR_PBA_OFFSET (MSIX_EXCLUSIVE_BAR_SIZE / 2)
-#define MSIX_EXCLUSIVE_CAP_OFFSET 0
-
-    if (nentries * PCI_MSIX_ENTRY_SIZE > MSIX_EXCLUSIVE_BAR_PBA_OFFSET) {
-        return -EINVAL;
+    if (nentries * PCI_MSIX_ENTRY_SIZE > bar_pba_offset) {
+        bar_pba_offset = nentries * PCI_MSIX_ENTRY_SIZE;
     }
 
+    if (bar_pba_offset + bar_pba_size > 4096) {
+        bar_size = bar_pba_offset + bar_pba_size;
+    }
+
+    bar_size = pow2ceil(bar_size);
+
     name = g_strdup_printf("%s-msix", dev->name);
-    memory_region_init(&dev->msix_exclusive_bar, OBJECT(dev), name, MSIX_EXCLUSIVE_BAR_SIZE);
+    memory_region_init(&dev->msix_exclusive_bar, OBJECT(dev), name, bar_size);
     g_free(name);
 
     ret = msix_init(dev, nentries, &dev->msix_exclusive_bar, bar_nr,
-                    MSIX_EXCLUSIVE_BAR_TABLE_OFFSET, &dev->msix_exclusive_bar,
-                    bar_nr, MSIX_EXCLUSIVE_BAR_PBA_OFFSET,
-                    MSIX_EXCLUSIVE_CAP_OFFSET);
+                    0, &dev->msix_exclusive_bar,
+                    bar_nr, bar_pba_offset,
+                    0);
     if (ret) {
         return ret;
     }
@@ -436,7 +457,7 @@
 
     msg = msix_get_message(dev, vector);
 
-    stl_le_phys(&dev->bus_master_as, msg.address, msg.data);
+    msi_send_message(dev, msg);
 }
 
 void msix_reset(PCIDevice *dev)
diff --git a/hw/pci/pci-hotplug-old.c b/hw/pci/pci-hotplug-old.c
deleted file mode 100644
index 0c09c72..0000000
--- a/hw/pci/pci-hotplug-old.c
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * Deprecated PCI hotplug interface support
- * This covers the old pci_add / pci_del command, whereas the more general
- * device_add / device_del commands are now preferred.
- *
- * Copyright (c) 2004 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "hw/hw.h"
-#include "hw/boards.h"
-#include "hw/pci/pci.h"
-#include "net/net.h"
-#include "hw/i386/pc.h"
-#include "monitor/monitor.h"
-#include "hw/scsi/scsi.h"
-#include "hw/virtio/virtio-blk.h"
-#include "qemu/config-file.h"
-#include "sysemu/block-backend.h"
-#include "qapi/error.h"
-
-static int pci_read_devaddr(Monitor *mon, const char *addr,
-                            int *busp, unsigned *slotp)
-{
-    int dom;
-
-    /* strip legacy tag */
-    if (!strncmp(addr, "pci_addr=", 9)) {
-        addr += 9;
-    }
-    if (pci_parse_devaddr(addr, &dom, busp, slotp, NULL)) {
-        monitor_printf(mon, "Invalid pci address\n");
-        return -1;
-    }
-    if (dom != 0) {
-        monitor_printf(mon, "Multiple PCI domains not supported, use device_add\n");
-        return -1;
-    }
-    return 0;
-}
-
-static PCIDevice *qemu_pci_hot_add_nic(Monitor *mon,
-                                       const char *devaddr,
-                                       const char *opts_str)
-{
-    Error *local_err = NULL;
-    QemuOpts *opts;
-    PCIBus *root = pci_find_primary_bus();
-    PCIBus *bus;
-    int ret, devfn;
-
-    if (!root) {
-        monitor_printf(mon, "no primary PCI bus (if there are multiple"
-                       " PCI roots, you must use device_add instead)");
-        return NULL;
-    }
-
-    bus = pci_get_bus_devfn(&devfn, root, devaddr);
-    if (!bus) {
-        monitor_printf(mon, "Invalid PCI device address %s\n", devaddr);
-        return NULL;
-    }
-    if (!qbus_is_hotpluggable(BUS(bus))) {
-        monitor_printf(mon, "PCI bus doesn't support hotplug\n");
-        return NULL;
-    }
-
-    opts = qemu_opts_parse(qemu_find_opts("net"), opts_str ? opts_str : "", 0);
-    if (!opts) {
-        return NULL;
-    }
-
-    qemu_opt_set(opts, "type", "nic");
-
-    ret = net_client_init(opts, 0, &local_err);
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return NULL;
-    }
-    if (nd_table[ret].devaddr) {
-        monitor_printf(mon, "Parameter addr not supported\n");
-        return NULL;
-    }
-    return pci_nic_init(&nd_table[ret], root, "rtl8139", devaddr);
-}
-
-static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
-                        DriveInfo *dinfo, int printinfo)
-{
-    SCSIBus *scsibus;
-    SCSIDevice *scsidev;
-    Error *local_err = NULL;
-
-    scsibus = (SCSIBus *)
-        object_dynamic_cast(OBJECT(QLIST_FIRST(&adapter->child_bus)),
-                            TYPE_SCSI_BUS);
-    if (!scsibus) {
-	error_report("Device is not a SCSI adapter");
-	return -1;
-    }
-
-    /*
-     * drive_init() tries to find a default for dinfo->unit.  Doesn't
-     * work at all for hotplug though as we assign the device to a
-     * specific bus instead of the first bus with spare scsi ids.
-     *
-     * Ditch the calculated value and reload from option string (if
-     * specified).
-     */
-    dinfo->unit = qemu_opt_get_number(dinfo->opts, "unit", -1);
-    dinfo->bus = scsibus->busnr;
-    scsidev = scsi_bus_legacy_add_drive(scsibus,
-                                        blk_by_legacy_dinfo(dinfo),
-                                        dinfo->unit, false, -1, NULL,
-                                        &local_err);
-    if (!scsidev) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
-        return -1;
-    }
-    dinfo->unit = scsidev->id;
-
-    if (printinfo)
-        monitor_printf(mon, "OK bus %d, unit %d\n",
-                       scsibus->busnr, scsidev->id);
-    return 0;
-}
-
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo)
-{
-    int pci_bus;
-    unsigned slot;
-    PCIBus *root = pci_find_primary_bus();
-    PCIDevice *dev;
-    const char *pci_addr = qdict_get_str(qdict, "pci_addr");
-
-    switch (dinfo->type) {
-    case IF_SCSI:
-        if (!root) {
-            monitor_printf(mon, "no primary PCI bus (if there are multiple"
-                           " PCI roots, you must use device_add instead)");
-            goto err;
-        }
-        if (pci_read_devaddr(mon, pci_addr, &pci_bus, &slot)) {
-            goto err;
-        }
-        dev = pci_find_device(root, pci_bus, PCI_DEVFN(slot, 0));
-        if (!dev) {
-            monitor_printf(mon, "no pci device with address %s\n", pci_addr);
-            goto err;
-        }
-        if (scsi_hot_add(mon, &dev->qdev, dinfo, 1) != 0) {
-            goto err;
-        }
-        break;
-    default:
-        monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
-        goto err;
-    }
-
-    return 0;
-err:
-    return -1;
-}
-
-static PCIDevice *qemu_pci_hot_add_storage(Monitor *mon,
-                                           const char *devaddr,
-                                           const char *opts)
-{
-    PCIDevice *dev;
-    DriveInfo *dinfo = NULL;
-    int type = -1;
-    char buf[128];
-    PCIBus *root = pci_find_primary_bus();
-    PCIBus *bus;
-    int devfn;
-
-    if (get_param_value(buf, sizeof(buf), "if", opts)) {
-        if (!strcmp(buf, "scsi"))
-            type = IF_SCSI;
-        else if (!strcmp(buf, "virtio")) {
-            type = IF_VIRTIO;
-        } else {
-            monitor_printf(mon, "type %s not a hotpluggable PCI device.\n", buf);
-            return NULL;
-        }
-    } else {
-        monitor_printf(mon, "no if= specified\n");
-        return NULL;
-    }
-
-    if (get_param_value(buf, sizeof(buf), "file", opts)) {
-        dinfo = add_init_drive(opts);
-        if (!dinfo)
-            return NULL;
-        if (dinfo->devaddr) {
-            monitor_printf(mon, "Parameter addr not supported\n");
-            return NULL;
-        }
-    } else {
-        dinfo = NULL;
-    }
-
-    if (!root) {
-        monitor_printf(mon, "no primary PCI bus (if there are multiple"
-                       " PCI roots, you must use device_add instead)");
-        return NULL;
-    }
-    bus = pci_get_bus_devfn(&devfn, root, devaddr);
-    if (!bus) {
-        monitor_printf(mon, "Invalid PCI device address %s\n", devaddr);
-        return NULL;
-    }
-    if (!qbus_is_hotpluggable(BUS(bus))) {
-        monitor_printf(mon, "PCI bus doesn't support hotplug\n");
-        return NULL;
-    }
-
-    switch (type) {
-    case IF_SCSI:
-        dev = pci_create(bus, devfn, "lsi53c895a");
-        if (qdev_init(&dev->qdev) < 0)
-            dev = NULL;
-        if (dev && dinfo) {
-            if (scsi_hot_add(mon, &dev->qdev, dinfo, 0) != 0) {
-                qdev_unplug(&dev->qdev, NULL);
-                dev = NULL;
-            }
-        }
-        break;
-    case IF_VIRTIO:
-        if (!dinfo) {
-            monitor_printf(mon, "virtio requires a backing file/device.\n");
-            return NULL;
-        }
-        dev = pci_create(bus, devfn, "virtio-blk-pci");
-        if (qdev_prop_set_drive(&dev->qdev, "drive",
-                                blk_by_legacy_dinfo(dinfo)) < 0) {
-            object_unparent(OBJECT(dev));
-            dev = NULL;
-            break;
-        }
-        if (qdev_init(&dev->qdev) < 0)
-            dev = NULL;
-        break;
-    default:
-        dev = NULL;
-    }
-    return dev;
-}
-
-void pci_device_hot_add(Monitor *mon, const QDict *qdict)
-{
-    PCIDevice *dev = NULL;
-    const char *pci_addr = qdict_get_str(qdict, "pci_addr");
-    const char *type = qdict_get_str(qdict, "type");
-    const char *opts = qdict_get_try_str(qdict, "opts");
-
-    /* strip legacy tag */
-    if (!strncmp(pci_addr, "pci_addr=", 9)) {
-        pci_addr += 9;
-    }
-
-    if (!opts) {
-        opts = "";
-    }
-
-    if (!strcmp(pci_addr, "auto"))
-        pci_addr = NULL;
-
-    if (strcmp(type, "nic") == 0) {
-        dev = qemu_pci_hot_add_nic(mon, pci_addr, opts);
-    } else if (strcmp(type, "storage") == 0) {
-        dev = qemu_pci_hot_add_storage(mon, pci_addr, opts);
-    } else {
-        monitor_printf(mon, "invalid type: %s\n", type);
-    }
-
-    if (dev) {
-        monitor_printf(mon, "OK root bus %s, bus %d, slot %d, function %d\n",
-                       pci_root_bus_path(dev),
-                       pci_bus_num(dev->bus), PCI_SLOT(dev->devfn),
-                       PCI_FUNC(dev->devfn));
-    } else
-        monitor_printf(mon, "failed to add %s\n", opts);
-}
-
-static int pci_device_hot_remove(Monitor *mon, const char *pci_addr)
-{
-    PCIBus *root = pci_find_primary_bus();
-    PCIDevice *d;
-    int bus;
-    unsigned slot;
-    Error *local_err = NULL;
-
-    if (!root) {
-        monitor_printf(mon, "no primary PCI bus (if there are multiple"
-                       " PCI roots, you must use device_del instead)");
-        return -1;
-    }
-
-    if (pci_read_devaddr(mon, pci_addr, &bus, &slot)) {
-        return -1;
-    }
-
-    d = pci_find_device(root, bus, PCI_DEVFN(slot, 0));
-    if (!d) {
-        monitor_printf(mon, "slot %d empty\n", slot);
-        return -1;
-    }
-
-    qdev_unplug(&d->qdev, &local_err);
-    if (local_err) {
-        monitor_printf(mon, "%s\n", error_get_pretty(local_err));
-        error_free(local_err);
-        return -1;
-    }
-
-    return 0;
-}
-
-void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict)
-{
-    pci_device_hot_remove(mon, qdict_get_str(qdict, "pci_addr"));
-}
diff --git a/hw/pci/pci-stub.c b/hw/pci/pci-stub.c
index 1dda89b..36d2c43 100644
--- a/hw/pci/pci-stub.c
+++ b/hw/pci/pci-stub.c
@@ -18,30 +18,20 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
+#include "qapi/qmp/qerror.h"
 #include "hw/pci/pci.h"
 #include "qmp-commands.h"
 
 PciInfoList *qmp_query_pci(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return NULL;
 }
 
-static void pci_error_message(Monitor *mon)
+void hmp_pcie_aer_inject_error(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "PCI devices not supported\n");
 }
-
-int do_pcie_aer_inject_error(Monitor *mon,
-                             const QDict *qdict, QObject **ret_data)
-{
-    pci_error_message(mon);
-    return -ENOSYS;
-}
-
-void pcie_aer_inject_error_print(Monitor *mon, const QObject *data)
-{
-    pci_error_message(mon);
-}
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index be1de2e..32d9df4 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bridge.h"
@@ -30,12 +31,16 @@
 #include "net/net.h"
 #include "sysemu/sysemu.h"
 #include "hw/loader.h"
+#include "qemu/error-report.h"
 #include "qemu/range.h"
 #include "qmp-commands.h"
+#include "trace.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "exec/address-spaces.h"
 #include "hw/hotplug.h"
+#include "hw/boards.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_PCI
 #ifdef DEBUG_PCI
@@ -57,6 +62,8 @@
                     QEMU_PCI_CAP_MULTIFUNCTION_BITNR, false),
     DEFINE_PROP_BIT("command_serr_enable", PCIDevice, cap_present,
                     QEMU_PCI_CAP_SERR_BITNR, true),
+    DEFINE_PROP_BIT("x-pcie-lnksta-dllla", PCIDevice, cap_present,
+                    QEMU_PCIE_LNKSTA_DLLLA_BITNR, true),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -73,10 +80,37 @@
     }
 };
 
+static void pci_init_bus_master(PCIDevice *pci_dev)
+{
+    AddressSpace *dma_as = pci_device_iommu_address_space(pci_dev);
+
+    memory_region_init_alias(&pci_dev->bus_master_enable_region,
+                             OBJECT(pci_dev), "bus master",
+                             dma_as->root, 0, memory_region_size(dma_as->root));
+    memory_region_set_enabled(&pci_dev->bus_master_enable_region, false);
+    address_space_init(&pci_dev->bus_master_as,
+                       &pci_dev->bus_master_enable_region, pci_dev->name);
+}
+
+static void pcibus_machine_done(Notifier *notifier, void *data)
+{
+    PCIBus *bus = container_of(notifier, PCIBus, machine_done);
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(bus->devices); ++i) {
+        if (bus->devices[i]) {
+            pci_init_bus_master(bus->devices[i]);
+        }
+    }
+}
+
 static void pci_bus_realize(BusState *qbus, Error **errp)
 {
     PCIBus *bus = PCI_BUS(qbus);
 
+    bus->machine_done.notify = pcibus_machine_done;
+    qemu_add_machine_init_done_notifier(&bus->machine_done);
+
     vmstate_register(NULL, -1, &vmstate_pcibus, bus);
 }
 
@@ -84,12 +118,33 @@
 {
     PCIBus *bus = PCI_BUS(qbus);
 
+    qemu_remove_machine_init_done_notifier(&bus->machine_done);
+
     vmstate_unregister(NULL, &vmstate_pcibus, bus);
 }
 
+static bool pcibus_is_root(PCIBus *bus)
+{
+    return !bus->parent_dev;
+}
+
+static int pcibus_num(PCIBus *bus)
+{
+    if (pcibus_is_root(bus)) {
+        return 0; /* pci host bridge */
+    }
+    return bus->parent_dev->config[PCI_SECONDARY_BUS];
+}
+
+static uint16_t pcibus_numa_node(PCIBus *bus)
+{
+    return NUMA_NODE_UNASSIGNED;
+}
+
 static void pci_bus_class_init(ObjectClass *klass, void *data)
 {
     BusClass *k = BUS_CLASS(klass);
+    PCIBusClass *pbc = PCI_BUS_CLASS(klass);
 
     k->print_dev = pcibus_dev_print;
     k->get_dev_path = pcibus_get_dev_path;
@@ -97,12 +152,17 @@
     k->realize = pci_bus_realize;
     k->unrealize = pci_bus_unrealize;
     k->reset = pcibus_reset;
+
+    pbc->is_root = pcibus_is_root;
+    pbc->bus_num = pcibus_num;
+    pbc->numa_node = pcibus_numa_node;
 }
 
 static const TypeInfo pci_bus_info = {
     .name = TYPE_PCI_BUS,
     .parent = TYPE_BUS,
     .instance_size = sizeof(PCIBus),
+    .class_size = sizeof(PCIBusClass),
     .class_init = pci_bus_class_init,
 };
 
@@ -114,7 +174,7 @@
 static PCIBus *pci_find_bus_nr(PCIBus *bus, int bus_num);
 static void pci_update_mappings(PCIDevice *d);
 static void pci_irq_handler(void *opaque, int irq_num, int level);
-static int pci_add_option_rom(PCIDevice *pdev, bool is_default_rom);
+static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom, Error **);
 static void pci_del_option_rom(PCIDevice *pdev);
 
 static uint16_t pci_default_sub_vendor_id = PCI_SUBVENDOR_ID_REDHAT_QUMRANET;
@@ -122,7 +182,7 @@
 
 static QLIST_HEAD(, PCIHostState) pci_host_bridges;
 
-static int pci_bar(PCIDevice *d, int reg)
+int pci_bar(PCIDevice *d, int reg)
 {
     uint8_t type;
 
@@ -250,9 +310,9 @@
     }
 }
 
-static void pci_host_bus_register(PCIBus *bus, DeviceState *parent)
+static void pci_host_bus_register(DeviceState *host)
 {
-    PCIHostState *host_bridge = PCI_HOST_BRIDGE(parent);
+    PCIHostState *host_bridge = PCI_HOST_BRIDGE(host);
 
     QLIST_INSERT_HEAD(&pci_host_bridges, host_bridge, next);
 }
@@ -277,7 +337,10 @@
 {
     PCIBus *bus = d->bus;
 
-    while ((d = bus->parent_dev) != NULL) {
+    while (!pci_bus_is_root(bus)) {
+        d = bus->parent_dev;
+        assert(d != NULL);
+
         bus = d->bus;
     }
 
@@ -290,7 +353,6 @@
     PCIHostState *host_bridge = PCI_HOST_BRIDGE(rootbus->qbus.parent);
     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_GET_CLASS(host_bridge);
 
-    assert(!rootbus->parent_dev);
     assert(host_bridge->bus == rootbus);
 
     if (hc->root_bus_path) {
@@ -301,7 +363,6 @@
 }
 
 static void pci_bus_init(PCIBus *bus, DeviceState *parent,
-                         const char *name,
                          MemoryRegion *address_space_mem,
                          MemoryRegion *address_space_io,
                          uint8_t devfn_min)
@@ -314,7 +375,7 @@
     /* host bridge */
     QLIST_INIT(&bus->child);
 
-    pci_host_bus_register(bus, parent);
+    pci_host_bus_register(parent);
 }
 
 bool pci_bus_is_express(PCIBus *bus)
@@ -324,7 +385,7 @@
 
 bool pci_bus_is_root(PCIBus *bus)
 {
-    return !bus->parent_dev;
+    return PCI_BUS_GET_CLASS(bus)->is_root(bus);
 }
 
 void pci_bus_new_inplace(PCIBus *bus, size_t bus_size, DeviceState *parent,
@@ -334,8 +395,7 @@
                          uint8_t devfn_min, const char *typename)
 {
     qbus_create_inplace(bus, bus_size, typename, parent, name);
-    pci_bus_init(bus, parent, name, address_space_mem,
-                 address_space_io, devfn_min);
+    pci_bus_init(bus, parent, address_space_mem, address_space_io, devfn_min);
 }
 
 PCIBus *pci_bus_new(DeviceState *parent, const char *name,
@@ -346,8 +406,7 @@
     PCIBus *bus;
 
     bus = PCI_BUS(qbus_create(typename, parent, name));
-    pci_bus_init(bus, parent, name, address_space_mem,
-                 address_space_io, devfn_min);
+    pci_bus_init(bus, parent, address_space_mem, address_space_io, devfn_min);
     return bus;
 }
 
@@ -378,9 +437,12 @@
 
 int pci_bus_num(PCIBus *s)
 {
-    if (pci_bus_is_root(s))
-        return 0;       /* pci host bridge */
-    return s->parent_dev->config[PCI_SECONDARY_BUS];
+    return PCI_BUS_GET_CLASS(s)->bus_num(s);
+}
+
+int pci_bus_numa_node(PCIBus *bus)
+{
+    return PCI_BUS_GET_CLASS(bus)->numa_node(bus);
 }
 
 static int get_pci_config_device(QEMUFile *f, void *pv, size_t size)
@@ -397,6 +459,10 @@
     for (i = 0; i < size; ++i) {
         if ((config[i] ^ s->config[i]) &
             s->cmask[i] & ~s->wmask[i] & ~s->w1cmask[i]) {
+            error_report("%s: Bad config data: i=0x%x read: %x device: %x "
+                         "cmask: %x wmask: %x w1cmask:%x", __func__,
+                         i, config[i], s->config[i],
+                         s->cmask[i], s->wmask[i], s->w1cmask[i]);
             g_free(config);
             return -EINVAL;
         }
@@ -512,7 +578,7 @@
      * This makes us compatible with old devices
      * which never set or clear this bit. */
     s->config[PCI_STATUS] &= ~PCI_STATUS_INTERRUPT;
-    vmstate_save_state(f, pci_get_vmstate(s), s);
+    vmstate_save_state(f, pci_get_vmstate(s), s, NULL);
     /* Restore the interrupt status bit. */
     pci_update_irq_status(s);
 }
@@ -538,8 +604,8 @@
  * Parse [[<domain>:]<bus>:]<slot>, return -1 on error if funcp == NULL
  *       [[<domain>:]<bus>:]<slot>.<func>, return -1 on error
  */
-int pci_parse_devaddr(const char *addr, int *domp, int *busp,
-                      unsigned int *slotp, unsigned int *funcp)
+static int pci_parse_devaddr(const char *addr, int *domp, int *busp,
+                             unsigned int *slotp, unsigned int *funcp)
 {
     const char *p;
     char *e;
@@ -597,7 +663,8 @@
     return 0;
 }
 
-PCIBus *pci_get_bus_devfn(int *devfnp, PCIBus *root, const char *devaddr)
+static PCIBus *pci_get_bus_devfn(int *devfnp, PCIBus *root,
+                                 const char *devaddr)
 {
     int dom, bus;
     unsigned slot;
@@ -725,7 +792,7 @@
                                PCI_PREF_RANGE_TYPE_MASK);
 }
 
-static int pci_init_multifunction(PCIBus *bus, PCIDevice *dev)
+static void pci_init_multifunction(PCIBus *bus, PCIDevice *dev, Error **errp)
 {
     uint8_t slot = PCI_SLOT(dev->devfn);
     uint8_t func;
@@ -751,26 +818,25 @@
         PCIDevice *f0 = bus->devices[PCI_DEVFN(slot, 0)];
         if (f0 && !(f0->cap_present & QEMU_PCI_CAP_MULTIFUNCTION)) {
             /* function 0 should set multifunction bit */
-            error_report("PCI: single function device can't be populated "
-                         "in function %x.%x", slot, PCI_FUNC(dev->devfn));
-            return -1;
+            error_setg(errp, "PCI: single function device can't be populated "
+                       "in function %x.%x", slot, PCI_FUNC(dev->devfn));
+            return;
         }
-        return 0;
+        return;
     }
 
     if (dev->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
-        return 0;
+        return;
     }
     /* function 0 indicates single function, so function > 0 must be NULL */
     for (func = 1; func < PCI_FUNC_MAX; ++func) {
         if (bus->devices[PCI_DEVFN(slot, func)]) {
-            error_report("PCI: %x.0 indicates single function, "
-                         "but %x.%x is already populated.",
-                         slot, slot, func);
-            return -1;
+            error_setg(errp, "PCI: %x.0 indicates single function, "
+                       "but %x.%x is already populated.",
+                       slot, slot, func);
+            return;
         }
     }
-    return 0;
 }
 
 static void pci_config_alloc(PCIDevice *pci_dev)
@@ -801,14 +867,100 @@
     address_space_destroy(&pci_dev->bus_master_as);
 }
 
+/* Extract PCIReqIDCache into BDF format */
+static uint16_t pci_req_id_cache_extract(PCIReqIDCache *cache)
+{
+    uint8_t bus_n;
+    uint16_t result;
+
+    switch (cache->type) {
+    case PCI_REQ_ID_BDF:
+        result = pci_get_bdf(cache->dev);
+        break;
+    case PCI_REQ_ID_SECONDARY_BUS:
+        bus_n = pci_bus_num(cache->dev->bus);
+        result = PCI_BUILD_BDF(bus_n, 0);
+        break;
+    default:
+        error_printf("Invalid PCI requester ID cache type: %d\n",
+                     cache->type);
+        exit(1);
+        break;
+    }
+
+    return result;
+}
+
+/* Parse bridges up to the root complex and return requester ID
+ * cache for specific device.  For full PCIe topology, the cache
+ * result would be exactly the same as getting BDF of the device.
+ * However, several tricks are required when system mixed up with
+ * legacy PCI devices and PCIe-to-PCI bridges.
+ *
+ * Here we cache the proxy device (and type) not requester ID since
+ * bus number might change from time to time.
+ */
+static PCIReqIDCache pci_req_id_cache_get(PCIDevice *dev)
+{
+    PCIDevice *parent;
+    PCIReqIDCache cache = {
+        .dev = dev,
+        .type = PCI_REQ_ID_BDF,
+    };
+
+    while (!pci_bus_is_root(dev->bus)) {
+        /* We are under PCI/PCIe bridges */
+        parent = dev->bus->parent_dev;
+        if (pci_is_express(parent)) {
+            if (pcie_cap_get_type(parent) == PCI_EXP_TYPE_PCI_BRIDGE) {
+                /* When we pass through PCIe-to-PCI/PCIX bridges, we
+                 * override the requester ID using secondary bus
+                 * number of parent bridge with zeroed devfn
+                 * (pcie-to-pci bridge spec chap 2.3). */
+                cache.type = PCI_REQ_ID_SECONDARY_BUS;
+                cache.dev = dev;
+            }
+        } else {
+            /* Legacy PCI, override requester ID with the bridge's
+             * BDF upstream.  When the root complex connects to
+             * legacy PCI devices (including buses), it can only
+             * obtain requester ID info from directly attached
+             * devices.  If devices are attached under bridges, only
+             * the requester ID of the bridge that is directly
+             * attached to the root complex can be recognized. */
+            cache.type = PCI_REQ_ID_BDF;
+            cache.dev = parent;
+        }
+        dev = parent;
+    }
+
+    return cache;
+}
+
+uint16_t pci_requester_id(PCIDevice *dev)
+{
+    return pci_req_id_cache_extract(&dev->requester_id_cache);
+}
+
 /* -1 for devfn means auto assign */
 static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
-                                         const char *name, int devfn)
+                                         const char *name, int devfn,
+                                         Error **errp)
 {
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(pci_dev);
     PCIConfigReadFunc *config_read = pc->config_read;
     PCIConfigWriteFunc *config_write = pc->config_write;
-    AddressSpace *dma_as;
+    Error *local_err = NULL;
+    DeviceState *dev = DEVICE(pci_dev);
+
+    pci_dev->bus = bus;
+    /* Only pci bridges can be attached to extra PCI root buses */
+    if (pci_bus_is_root(bus) && bus->parent_dev && !pc->is_bridge) {
+        error_setg(errp,
+                   "PCI: Only PCI/PCIe bridges can be plugged into %s",
+                    bus->parent_dev->name);
+        return NULL;
+    }
 
     if (devfn < 0) {
         for(devfn = bus->devfn_min ; devfn < ARRAY_SIZE(bus->devices);
@@ -816,26 +968,33 @@
             if (!bus->devices[devfn])
                 goto found;
         }
-        error_report("PCI: no slot/function available for %s, all in use", name);
+        error_setg(errp, "PCI: no slot/function available for %s, all in use",
+                   name);
         return NULL;
     found: ;
     } else if (bus->devices[devfn]) {
-        error_report("PCI: slot %d function %d not available for %s, in use by %s",
-                     PCI_SLOT(devfn), PCI_FUNC(devfn), name, bus->devices[devfn]->name);
+        error_setg(errp, "PCI: slot %d function %d not available for %s,"
+                   " in use by %s",
+                   PCI_SLOT(devfn), PCI_FUNC(devfn), name,
+                   bus->devices[devfn]->name);
         return NULL;
+    } else if (dev->hotplugged &&
+               pci_get_function_0(pci_dev)) {
+        error_setg(errp, "PCI: slot %d function 0 already ocuppied by %s,"
+                   " new func %s cannot be exposed to guest.",
+                   PCI_SLOT(devfn),
+                   bus->devices[PCI_DEVFN(PCI_SLOT(devfn), 0)]->name,
+                   name);
+
+       return NULL;
     }
 
-    pci_dev->bus = bus;
     pci_dev->devfn = devfn;
-    dma_as = pci_device_iommu_address_space(pci_dev);
+    pci_dev->requester_id_cache = pci_req_id_cache_get(pci_dev);
 
-    memory_region_init_alias(&pci_dev->bus_master_enable_region,
-                             OBJECT(pci_dev), "bus master",
-                             dma_as->root, 0, memory_region_size(dma_as->root));
-    memory_region_set_enabled(&pci_dev->bus_master_enable_region, false);
-    address_space_init(&pci_dev->bus_master_as, &pci_dev->bus_master_enable_region,
-                       name);
-
+    if (qdev_hotplug) {
+        pci_init_bus_master(pci_dev);
+    }
     pstrcpy(pci_dev->name, sizeof(pci_dev->name), name);
     pci_dev->irq_state = 0;
     pci_config_alloc(pci_dev);
@@ -865,7 +1024,9 @@
     if (pc->is_bridge) {
         pci_init_mask_bridge(pci_dev);
     }
-    if (pci_init_multifunction(bus, pci_dev)) {
+    pci_init_multifunction(bus, pci_dev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         do_pci_unregister_device(pci_dev);
         return NULL;
     }
@@ -896,7 +1057,7 @@
     pci_unregister_vga(pci_dev);
 }
 
-static int pci_unregister_device(DeviceState *dev)
+static void pci_qdev_unrealize(DeviceState *dev, Error **errp)
 {
     PCIDevice *pci_dev = PCI_DEVICE(dev);
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(pci_dev);
@@ -909,14 +1070,13 @@
     }
 
     do_pci_unregister_device(pci_dev);
-    return 0;
 }
 
 void pci_register_bar(PCIDevice *pci_dev, int region_num,
                       uint8_t type, MemoryRegion *memory)
 {
     PCIIORegion *r;
-    uint32_t addr;
+    uint32_t addr; /* offset in pci config space */
     uint64_t wmask;
     pcibus_t size = memory_region_size(memory);
 
@@ -932,15 +1092,20 @@
     r->addr = PCI_BAR_UNMAPPED;
     r->size = size;
     r->type = type;
-    r->memory = NULL;
+    r->memory = memory;
+    r->address_space = type & PCI_BASE_ADDRESS_SPACE_IO
+                        ? pci_dev->bus->address_space_io
+                        : pci_dev->bus->address_space_mem;
 
     wmask = ~(size - 1);
-    addr = pci_bar(pci_dev, region_num);
     if (region_num == PCI_ROM_SLOT) {
         /* ROM enable bit is writable */
         wmask |= PCI_ROM_ADDRESS_ENABLE;
     }
+
+    addr = pci_bar(pci_dev, region_num);
     pci_set_long(pci_dev->config + addr, type);
+
     if (!(r->type & PCI_BASE_ADDRESS_SPACE_IO) &&
         r->type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
         pci_set_quad(pci_dev->wmask + addr, wmask);
@@ -949,11 +1114,6 @@
         pci_set_long(pci_dev->wmask + addr, wmask & 0xffffffff);
         pci_set_long(pci_dev->cmask + addr, 0xffffffff);
     }
-    pci_dev->io_regions[region_num].memory = memory;
-    pci_dev->io_regions[region_num].address_space
-        = type & PCI_BASE_ADDRESS_SPACE_IO
-        ? pci_dev->bus->address_space_io
-        : pci_dev->bus->address_space_mem;
 }
 
 static void pci_update_vga(PCIDevice *pci_dev)
@@ -1024,6 +1184,10 @@
     pcibus_t new_addr, last_addr;
     int bar = pci_bar(d, reg);
     uint16_t cmd = pci_get_word(d->config + PCI_COMMAND);
+    Object *machine = qdev_get_machine();
+    ObjectClass *oc = object_get_class(machine);
+    MachineClass *mc = MACHINE_CLASS(oc);
+    bool allow_0_address = mc->pci_allow_0_address;
 
     if (type & PCI_BASE_ADDRESS_SPACE_IO) {
         if (!(cmd & PCI_COMMAND_IO)) {
@@ -1034,7 +1198,8 @@
         /* Check if 32 bit BAR wraps around explicitly.
          * TODO: make priorities correct and remove this work around.
          */
-        if (last_addr <= new_addr || new_addr == 0 || last_addr >= UINT32_MAX) {
+        if (last_addr <= new_addr || last_addr >= UINT32_MAX ||
+            (!allow_0_address && new_addr == 0)) {
             return PCI_BAR_UNMAPPED;
         }
         return new_addr;
@@ -1058,8 +1223,8 @@
     /* XXX: as we cannot support really dynamic
        mappings, we handle specific values as invalid
        mappings. */
-    if (last_addr <= new_addr || new_addr == 0 ||
-        last_addr == PCI_BAR_UNMAPPED) {
+    if (last_addr <= new_addr || last_addr == PCI_BAR_UNMAPPED ||
+        (!allow_0_address && new_addr == 0)) {
         return PCI_BAR_UNMAPPED;
     }
 
@@ -1106,10 +1271,18 @@
 
         /* now do the real mapping */
         if (r->addr != PCI_BAR_UNMAPPED) {
+            trace_pci_update_mappings_del(d, pci_bus_num(d->bus),
+                                          PCI_SLOT(d->devfn),
+                                          PCI_FUNC(d->devfn),
+                                          i, r->addr, r->size);
             memory_region_del_subregion(r->address_space, r->memory);
         }
         r->addr = new_addr;
         if (r->addr != PCI_BAR_UNMAPPED) {
+            trace_pci_update_mappings_add(d, pci_bus_num(d->bus),
+                                          PCI_SLOT(d->devfn),
+                                          PCI_FUNC(d->devfn),
+                                          i, r->addr, r->size);
             memory_region_add_subregion_overlap(r->address_space,
                                                 r->addr, r->memory, 1);
         }
@@ -1441,24 +1614,26 @@
                                            int bus_num)
 {
     PciBridgeInfo *info;
+    PciMemoryRange *range;
 
-    info = g_malloc0(sizeof(*info));
+    info = g_new0(PciBridgeInfo, 1);
 
-    info->bus.number = dev->config[PCI_PRIMARY_BUS];
-    info->bus.secondary = dev->config[PCI_SECONDARY_BUS];
-    info->bus.subordinate = dev->config[PCI_SUBORDINATE_BUS];
+    info->bus = g_new0(PciBusInfo, 1);
+    info->bus->number = dev->config[PCI_PRIMARY_BUS];
+    info->bus->secondary = dev->config[PCI_SECONDARY_BUS];
+    info->bus->subordinate = dev->config[PCI_SUBORDINATE_BUS];
 
-    info->bus.io_range = g_malloc0(sizeof(*info->bus.io_range));
-    info->bus.io_range->base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_SPACE_IO);
-    info->bus.io_range->limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_SPACE_IO);
+    range = info->bus->io_range = g_new0(PciMemoryRange, 1);
+    range->base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_SPACE_IO);
+    range->limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_SPACE_IO);
 
-    info->bus.memory_range = g_malloc0(sizeof(*info->bus.memory_range));
-    info->bus.memory_range->base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_SPACE_MEMORY);
-    info->bus.memory_range->limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_SPACE_MEMORY);
+    range = info->bus->memory_range = g_new0(PciMemoryRange, 1);
+    range->base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_SPACE_MEMORY);
+    range->limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_SPACE_MEMORY);
 
-    info->bus.prefetchable_range = g_malloc0(sizeof(*info->bus.prefetchable_range));
-    info->bus.prefetchable_range->base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_MEM_PREFETCH);
-    info->bus.prefetchable_range->limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_MEM_PREFETCH);
+    range = info->bus->prefetchable_range = g_new0(PciMemoryRange, 1);
+    range->base = pci_bridge_get_base(dev, PCI_BASE_ADDRESS_MEM_PREFETCH);
+    range->limit = pci_bridge_get_limit(dev, PCI_BASE_ADDRESS_MEM_PREFETCH);
 
     if (dev->config[PCI_SECONDARY_BUS] != 0) {
         PCIBus *child_bus = pci_find_bus_nr(bus, dev->config[PCI_SECONDARY_BUS]);
@@ -1479,21 +1654,23 @@
     uint8_t type;
     int class;
 
-    info = g_malloc0(sizeof(*info));
+    info = g_new0(PciDeviceInfo, 1);
     info->bus = bus_num;
     info->slot = PCI_SLOT(dev->devfn);
     info->function = PCI_FUNC(dev->devfn);
 
+    info->class_info = g_new0(PciDeviceClass, 1);
     class = pci_get_word(dev->config + PCI_CLASS_DEVICE);
-    info->class_info.q_class = class;
+    info->class_info->q_class = class;
     desc = get_class_desc(class);
     if (desc->desc) {
-        info->class_info.has_desc = true;
-        info->class_info.desc = g_strdup(desc->desc);
+        info->class_info->has_desc = true;
+        info->class_info->desc = g_strdup(desc->desc);
     }
 
-    info->id.vendor = pci_get_word(dev->config + PCI_VENDOR_ID);
-    info->id.device = pci_get_word(dev->config + PCI_DEVICE_ID);
+    info->id = g_new0(PciDeviceId, 1);
+    info->id->vendor = pci_get_word(dev->config + PCI_VENDOR_ID);
+    info->id->device = pci_get_word(dev->config + PCI_DEVICE_ID);
     info->regions = qmp_query_pci_regions(dev);
     info->qdev_id = g_strdup(dev->qdev.id ? dev->qdev.id : "");
 
@@ -1557,7 +1734,8 @@
 
     QLIST_FOREACH(host_bridge, &pci_host_bridges, next) {
         info = g_malloc0(sizeof(*info));
-        info->value = qmp_query_pci_bus(host_bridge->bus, 0);
+        info->value = qmp_query_pci_bus(host_bridge->bus,
+                                        pci_bus_num(host_bridge->bus));
 
         /* XXX: waiting for the qapi to support GSList */
         if (!cur_item) {
@@ -1596,50 +1774,46 @@
 };
 
 /* Initialize a PCI NIC.  */
-/* FIXME callers should check for failure, but don't */
-PCIDevice *pci_nic_init(NICInfo *nd, PCIBus *rootbus,
-                        const char *default_model,
-                        const char *default_devaddr)
+PCIDevice *pci_nic_init_nofail(NICInfo *nd, PCIBus *rootbus,
+                               const char *default_model,
+                               const char *default_devaddr)
 {
     const char *devaddr = nd->devaddr ? nd->devaddr : default_devaddr;
+    Error *err = NULL;
     PCIBus *bus;
-    int devfn;
     PCIDevice *pci_dev;
     DeviceState *dev;
+    int devfn;
     int i;
 
+    if (qemu_show_nic_models(nd->model, pci_nic_models)) {
+        exit(0);
+    }
+
     i = qemu_find_nic_model(nd, pci_nic_models, default_model);
-    if (i < 0)
-        return NULL;
+    if (i < 0) {
+        exit(1);
+    }
 
     bus = pci_get_bus_devfn(&devfn, rootbus, devaddr);
     if (!bus) {
         error_report("Invalid PCI device address %s for device %s",
                      devaddr, pci_nic_names[i]);
-        return NULL;
+        exit(1);
     }
 
     pci_dev = pci_create(bus, devfn, pci_nic_names[i]);
     dev = &pci_dev->qdev;
     qdev_set_nic_properties(dev, nd);
-    if (qdev_init(dev) < 0)
-        return NULL;
-    return pci_dev;
-}
 
-PCIDevice *pci_nic_init_nofail(NICInfo *nd, PCIBus *rootbus,
-                               const char *default_model,
-                               const char *default_devaddr)
-{
-    PCIDevice *res;
-
-    if (qemu_show_nic_models(nd->model, pci_nic_models))
-        exit(0);
-
-    res = pci_nic_init(nd, rootbus, default_model, default_devaddr);
-    if (!res)
+    object_property_set_bool(OBJECT(dev), true, "realized", &err);
+    if (err) {
+        error_report_err(err);
+        object_unparent(OBJECT(dev));
         exit(1);
-    return res;
+    }
+
+    return pci_dev;
 }
 
 PCIDevice *pci_vga_init(PCIBus *bus)
@@ -1653,6 +1827,8 @@
         return pci_create_simple(bus, -1, "VGA");
     case VGA_VMWARE:
         return pci_create_simple(bus, -1, "vmware-svga");
+    case VGA_VIRTIO:
+        return pci_create_simple(bus, -1, "virtio-vga");
     case VGA_NONE:
     default: /* Other non-PCI types. Checking for unsupported types is already
                 done in vl.c. */
@@ -1666,10 +1842,28 @@
 {
     return !(pci_get_word(dev->config + PCI_BRIDGE_CONTROL) &
              PCI_BRIDGE_CTL_BUS_RESET) /* Don't walk the bus if it's reset. */ &&
-        dev->config[PCI_SECONDARY_BUS] < bus_num &&
+        dev->config[PCI_SECONDARY_BUS] <= bus_num &&
         bus_num <= dev->config[PCI_SUBORDINATE_BUS];
 }
 
+/* Whether a given bus number is in a range of a root bus */
+static bool pci_root_bus_in_range(PCIBus *bus, int bus_num)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(bus->devices); ++i) {
+        PCIDevice *dev = bus->devices[i];
+
+        if (dev && PCI_DEVICE_GET_CLASS(dev)->is_bridge) {
+            if (pci_secondary_bus_in_range(dev, bus_num)) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
 static PCIBus *pci_find_bus_nr(PCIBus *bus, int bus_num)
 {
     PCIBus *sec;
@@ -1691,12 +1885,18 @@
     /* try child bus */
     for (; bus; bus = sec) {
         QLIST_FOREACH(sec, &bus->child, sibling) {
-            assert(!pci_bus_is_root(sec));
-            if (sec->parent_dev->config[PCI_SECONDARY_BUS] == bus_num) {
+            if (pci_bus_num(sec) == bus_num) {
                 return sec;
             }
-            if (pci_secondary_bus_in_range(sec->parent_dev, bus_num)) {
-                break;
+            /* PXB buses assumed to be children of bus 0 */
+            if (pci_bus_is_root(sec)) {
+                if (pci_root_bus_in_range(sec, bus_num)) {
+                    break;
+                }
+            } else {
+                if (pci_secondary_bus_in_range(sec->parent_dev, bus_num)) {
+                    break;
+                }
             }
         }
     }
@@ -1742,12 +1942,12 @@
     return bus->devices[devfn];
 }
 
-static int pci_qdev_init(DeviceState *qdev)
+static void pci_qdev_realize(DeviceState *qdev, Error **errp)
 {
     PCIDevice *pci_dev = (PCIDevice *)qdev;
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(pci_dev);
+    Error *local_err = NULL;
     PCIBus *bus;
-    int rc;
     bool is_default_rom;
 
     /* initialize cap_present for pci_is_express() and pci_config_size() */
@@ -1758,15 +1958,16 @@
     bus = PCI_BUS(qdev_get_parent_bus(qdev));
     pci_dev = do_pci_register_device(pci_dev, bus,
                                      object_get_typename(OBJECT(qdev)),
-                                     pci_dev->devfn);
+                                     pci_dev->devfn, errp);
     if (pci_dev == NULL)
-        return -1;
+        return;
 
-    if (pc->init) {
-        rc = pc->init(pci_dev);
-        if (rc != 0) {
+    if (pc->realize) {
+        pc->realize(pci_dev, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
             do_pci_unregister_device(pci_dev);
-            return rc;
+            return;
         }
     }
 
@@ -1777,13 +1978,24 @@
         is_default_rom = true;
     }
 
-    rc = pci_add_option_rom(pci_dev, is_default_rom);
-    if (rc != 0) {
-        pci_unregister_device(DEVICE(pci_dev));
-        return rc;
+    pci_add_option_rom(pci_dev, is_default_rom, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        pci_qdev_unrealize(DEVICE(pci_dev), NULL);
+        return;
     }
+}
 
-    return 0;
+static void pci_default_realize(PCIDevice *dev, Error **errp)
+{
+    PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
+
+    if (pc->init) {
+        if (pc->init(dev) < 0) {
+            error_setg(errp, "Device initialization failed");
+            return;
+        }
+    }
 }
 
 PCIDevice *pci_create_multifunction(PCIBus *bus, int devfn, bool multifunction,
@@ -1923,7 +2135,8 @@
 }
 
 /* Add an option rom for the device */
-static int pci_add_option_rom(PCIDevice *pdev, bool is_default_rom)
+static void pci_add_option_rom(PCIDevice *pdev, bool is_default_rom,
+                               Error **errp)
 {
     int size;
     char *path;
@@ -1932,9 +2145,9 @@
     const VMStateDescription *vmsd;
 
     if (!pdev->romfile)
-        return 0;
+        return;
     if (strlen(pdev->romfile) == 0)
-        return 0;
+        return;
 
     if (!pdev->rom_bar) {
         /*
@@ -1948,7 +2161,9 @@
          * if the rom bar is disabled.
          */
         if (DEVICE(pdev)->hotplugged) {
-            return -1;
+            error_setg(errp, "Hot-plugged device without ROM bar"
+                       " can't have an option ROM");
+            return;
         }
 
         if (class == 0x0300) {
@@ -1956,7 +2171,7 @@
         } else {
             rom_add_option(pdev->romfile, -1);
         }
-        return 0;
+        return;
     }
 
     path = qemu_find_file(QEMU_FILE_TYPE_BIOS, pdev->romfile);
@@ -1966,19 +2181,15 @@
 
     size = get_image_size(path);
     if (size < 0) {
-        error_report("%s: failed to find romfile \"%s\"",
-                     __func__, pdev->romfile);
+        error_setg(errp, "failed to find romfile \"%s\"", pdev->romfile);
         g_free(path);
-        return -1;
+        return;
     } else if (size == 0) {
-        error_report("%s: ignoring empty romfile \"%s\"",
-                     __func__, pdev->romfile);
+        error_setg(errp, "romfile \"%s\" is empty", pdev->romfile);
         g_free(path);
-        return -1;
+        return;
     }
-    if (size & (size - 1)) {
-        size = 1 << qemu_fls(size);
-    }
+    size = pow2ceil(size);
 
     vmsd = qdev_get_vmsd(DEVICE(pdev));
 
@@ -1988,7 +2199,7 @@
         snprintf(name, sizeof(name), "%s.rom", object_get_typename(OBJECT(pdev)));
     }
     pdev->has_rom = true;
-    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_abort);
+    memory_region_init_ram(&pdev->rom, OBJECT(pdev), name, size, &error_fatal);
     vmstate_register_ram(&pdev->rom, &pdev->qdev);
     ptr = memory_region_get_ram_ptr(&pdev->rom);
     load_image(path, ptr);
@@ -2000,8 +2211,6 @@
     }
 
     pci_register_bar(pdev, PCI_ROM_SLOT, 0, &pdev->rom);
-
-    return 0;
 }
 
 static void pci_del_option_rom(PCIDevice *pdev)
@@ -2014,12 +2223,10 @@
 }
 
 /*
- * if !offset
- * Reserve space and add capability to the linked list in pci config space
- *
  * if offset = 0,
  * Find and reserve space and add capability to the linked list
- * in pci config space */
+ * in pci config space
+ */
 int pci_add_capability(PCIDevice *pdev, uint8_t cap_id,
                        uint8_t offset, uint8_t size)
 {
@@ -2029,8 +2236,7 @@
     ret = pci_add_capability2(pdev, cap_id, offset, size, &local_err);
     if (local_err) {
         assert(ret < 0);
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
     } else {
         /* success implies a positive offset in config space */
         assert(ret > 0);
@@ -2047,10 +2253,8 @@
 
     if (!offset) {
         offset = pci_find_space(pdev, size);
-        if (!offset) {
-            error_setg(errp, "out of PCI config space");
-            return -ENOSPC;
-        }
+        /* out of PCI config space is programming error */
+        assert(offset);
     } else {
         /* Verify that capabilities don't overlap.  Note: device assignment
          * depends on this check to verify that the device is not broken.
@@ -2284,26 +2488,26 @@
 static void pci_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *k = DEVICE_CLASS(klass);
-    k->init = pci_qdev_init;
-    k->exit = pci_unregister_device;
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+
+    k->realize = pci_qdev_realize;
+    k->unrealize = pci_qdev_unrealize;
     k->bus_type = TYPE_PCI_BUS;
     k->props = pci_props;
+    pc->realize = pci_default_realize;
 }
 
 AddressSpace *pci_device_iommu_address_space(PCIDevice *dev)
 {
     PCIBus *bus = PCI_BUS(dev->bus);
+    PCIBus *iommu_bus = bus;
 
-    if (bus->iommu_fn) {
-        return bus->iommu_fn(bus, bus->iommu_opaque, dev->devfn);
+    while(iommu_bus && !iommu_bus->iommu_fn && iommu_bus->parent_dev) {
+        iommu_bus = PCI_BUS(iommu_bus->parent_dev->bus);
     }
-
-    if (bus->parent_dev) {
-        /** We are ignoring the bus master DMA bit of the bridge
-         *  as it would complicate things such as VFIO for no good reason */
-        return pci_device_iommu_address_space(bus->parent_dev);
+    if (iommu_bus && iommu_bus->iommu_fn) {
+        return iommu_bus->iommu_fn(bus, iommu_bus->iommu_opaque, dev->devfn);
     }
-
     return &address_space_memory;
 }
 
@@ -2332,13 +2536,13 @@
 
         if (limit >= base) {
             Range pref_range;
-            pref_range.begin = base;
-            pref_range.end = limit + 1;
+            range_set_bounds(&pref_range, base, limit);
             range_extend(range, &pref_range);
         }
     }
     for (i = 0; i < PCI_NUM_REGIONS; ++i) {
         PCIIORegion *r = &dev->io_regions[i];
+        pcibus_t lob, upb;
         Range region_range;
 
         if (!r->size ||
@@ -2346,16 +2550,17 @@
             !(r->type & PCI_BASE_ADDRESS_MEM_TYPE_64)) {
             continue;
         }
-        region_range.begin = pci_bar_address(dev, i, r->type, r->size);
-        region_range.end = region_range.begin + r->size;
 
-        if (region_range.begin == PCI_BAR_UNMAPPED) {
+        lob = pci_bar_address(dev, i, r->type, r->size);
+        upb = lob + r->size - 1;
+        if (lob == PCI_BAR_UNMAPPED) {
             continue;
         }
 
-        region_range.begin = MAX(region_range.begin, 0x1ULL << 32);
+        lob = MAX(lob, 0x1ULL << 32);
 
-        if (region_range.end - 1 >= region_range.begin) {
+        if (upb >= lob) {
+            range_set_bounds(&region_range, lob, upb);
             range_extend(range, &region_range);
         }
     }
@@ -2363,10 +2568,52 @@
 
 void pci_bus_get_w64_range(PCIBus *bus, Range *range)
 {
-    range->begin = range->end = 0;
+    range_make_empty(range);
     pci_for_each_device_under_bus(bus, pci_dev_get_w64, range);
 }
 
+static bool pcie_has_upstream_port(PCIDevice *dev)
+{
+    PCIDevice *parent_dev = pci_bridge_get_device(dev->bus);
+
+    /* Device associated with an upstream port.
+     * As there are several types of these, it's easier to check the
+     * parent device: upstream ports are always connected to
+     * root or downstream ports.
+     */
+    return parent_dev &&
+        pci_is_express(parent_dev) &&
+        parent_dev->exp.exp_cap &&
+        (pcie_cap_get_type(parent_dev) == PCI_EXP_TYPE_ROOT_PORT ||
+         pcie_cap_get_type(parent_dev) == PCI_EXP_TYPE_DOWNSTREAM);
+}
+
+PCIDevice *pci_get_function_0(PCIDevice *pci_dev)
+{
+    if(pcie_has_upstream_port(pci_dev)) {
+        /* With an upstream PCIe port, we only support 1 device at slot 0 */
+        return pci_dev->bus->devices[0];
+    } else {
+        /* Other bus types might support multiple devices at slots 0-31 */
+        return pci_dev->bus->devices[PCI_DEVFN(PCI_SLOT(pci_dev->devfn), 0)];
+    }
+}
+
+MSIMessage pci_get_msi_message(PCIDevice *dev, int vector)
+{
+    MSIMessage msg;
+    if (msix_enabled(dev)) {
+        msg = msix_get_message(dev, vector);
+    } else if (msi_enabled(dev)) {
+        msg = msi_get_message(dev, vector);
+    } else {
+        /* Should never happen */
+        error_report("%s: unknown interrupt type", __func__);
+        abort();
+    }
+    return msg;
+}
+
 static const TypeInfo pci_device_type_info = {
     .name = TYPE_PCI_DEVICE,
     .parent = TYPE_DEVICE,
diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c
index 40c97b1..5118ef4 100644
--- a/hw/pci/pci_bridge.c
+++ b/hw/pci/pci_bridge.c
@@ -29,6 +29,7 @@
  *                    VA Linux Systems Japan K.K.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pci_bus.h"
 #include "qemu/range.h"
@@ -115,7 +116,7 @@
     return base;
 }
 
-/* accessor funciton to get bridge filtering limit */
+/* accessor function to get bridge filtering limit */
 pcibus_t pci_bridge_get_limit(const PCIDevice *bridge, uint8_t type)
 {
     pcibus_t limit;
@@ -332,7 +333,7 @@
 }
 
 /* default qdev initialization function for PCI-to-PCI bridge */
-int pci_bridge_initfn(PCIDevice *dev, const char *typename)
+void pci_bridge_initfn(PCIDevice *dev, const char *typename)
 {
     PCIBus *parent = dev->bus;
     PCIBridge *br = PCI_BRIDGE(dev);
@@ -378,7 +379,6 @@
     br->windows = pci_bridge_region_init(br);
     QLIST_INIT(&sec_bus->child);
     QLIST_INSERT_HEAD(&parent->child, sec_bus, sibling);
-    return 0;
 }
 
 /* default qdev clean up function for PCI-to-PCI bridge */
diff --git a/hw/pci/pci_host.c b/hw/pci/pci_host.c
index 3e26f92..5eaa935 100644
--- a/hw/pci/pci_host.c
+++ b/hw/pci/pci_host.c
@@ -18,8 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
+#include "hw/pci/pci_bus.h"
 #include "trace.h"
 
 /* debug PCI */
@@ -52,6 +54,13 @@
                                   uint32_t limit, uint32_t val, uint32_t len)
 {
     assert(len <= 4);
+    /* non-zero functions are only exposed when function 0 is present,
+     * allowing direct removal of unexposed functions.
+     */
+    if (pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) {
+        return;
+    }
+
     trace_pci_cfg_write(pci_dev->name, PCI_SLOT(pci_dev->devfn),
                         PCI_FUNC(pci_dev->devfn), addr, val);
     pci_dev->config_write(pci_dev, addr, val, MIN(len, limit - addr));
@@ -63,6 +72,13 @@
     uint32_t ret;
 
     assert(len <= 4);
+    /* non-zero functions are only exposed when function 0 is present,
+     * allowing direct removal of unexposed functions.
+     */
+    if (pci_dev->qdev.hotplugged && !pci_get_function_0(pci_dev)) {
+        return ~0x0;
+    }
+
     ret = pci_dev->config_read(pci_dev, addr, MIN(len, limit - addr));
     trace_pci_cfg_read(pci_dev->name, PCI_SLOT(pci_dev->devfn),
                        PCI_FUNC(pci_dev->devfn), addr, ret);
diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
index d762529..2c29af1 100644
--- a/hw/pci/pcie.c
+++ b/hw/pci/pcie.c
@@ -18,6 +18,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pcie.h"
@@ -26,7 +28,6 @@
 #include "hw/pci/pci_bus.h"
 #include "hw/pci/pcie_regs.h"
 #include "qemu/range.h"
-#include "qapi/qmp/qerror.h"
 
 //#define DEBUG_PCIE
 #ifdef DEBUG_PCIE
@@ -42,26 +43,18 @@
 /***************************************************************************
  * pci express capability helper functions
  */
-int pcie_cap_init(PCIDevice *dev, uint8_t offset, uint8_t type, uint8_t port)
+
+static void
+pcie_cap_v1_fill(PCIDevice *dev, uint8_t port, uint8_t type, uint8_t version)
 {
-    int pos;
-    uint8_t *exp_cap;
-
-    assert(pci_is_express(dev));
-
-    pos = pci_add_capability(dev, PCI_CAP_ID_EXP, offset,
-                                 PCI_EXP_VER2_SIZEOF);
-    if (pos < 0) {
-        return pos;
-    }
-    dev->exp.exp_cap = pos;
-    exp_cap = dev->config + pos;
+    uint8_t *exp_cap = dev->config + dev->exp.exp_cap;
+    uint8_t *cmask = dev->cmask + dev->exp.exp_cap;
 
     /* capability register
-       interrupt message number defaults to 0 */
+    interrupt message number defaults to 0 */
     pci_set_word(exp_cap + PCI_EXP_FLAGS,
                  ((type << PCI_EXP_FLAGS_TYPE_SHIFT) & PCI_EXP_FLAGS_TYPE) |
-                 PCI_EXP_FLAGS_VER2);
+                 version);
 
     /* device capability register
      * table 7-12:
@@ -81,14 +74,65 @@
     pci_set_word(exp_cap + PCI_EXP_LNKSTA,
                  PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25);
 
+    if (dev->cap_present & QEMU_PCIE_LNKSTA_DLLLA) {
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_LNKSTA,
+                                   PCI_EXP_LNKSTA_DLLLA);
+    }
+
+    /* We changed link status bits over time, and changing them across
+     * migrations is generally fine as hardware changes them too.
+     * Let's not bother checking.
+     */
+    pci_set_word(cmask + PCI_EXP_LNKSTA, 0);
+}
+
+int pcie_cap_init(PCIDevice *dev, uint8_t offset, uint8_t type, uint8_t port)
+{
+    /* PCIe cap v2 init */
+    int pos;
+    uint8_t *exp_cap;
+
+    assert(pci_is_express(dev));
+
+    pos = pci_add_capability(dev, PCI_CAP_ID_EXP, offset, PCI_EXP_VER2_SIZEOF);
+    if (pos < 0) {
+        return pos;
+    }
+    dev->exp.exp_cap = pos;
+    exp_cap = dev->config + pos;
+
+    /* Filling values common with v1 */
+    pcie_cap_v1_fill(dev, port, type, PCI_EXP_FLAGS_VER2);
+
+    /* Filling v2 specific values */
     pci_set_long(exp_cap + PCI_EXP_DEVCAP2,
                  PCI_EXP_DEVCAP2_EFF | PCI_EXP_DEVCAP2_EETLPP);
 
-    pci_set_word(dev->wmask + pos, PCI_EXP_DEVCTL2_EETLPPB);
+    pci_set_word(dev->wmask + pos + PCI_EXP_DEVCTL2, PCI_EXP_DEVCTL2_EETLPPB);
     return pos;
 }
 
-int pcie_endpoint_cap_init(PCIDevice *dev, uint8_t offset)
+int pcie_cap_v1_init(PCIDevice *dev, uint8_t offset, uint8_t type,
+                     uint8_t port)
+{
+    /* PCIe cap v1 init */
+    int pos;
+
+    assert(pci_is_express(dev));
+
+    pos = pci_add_capability(dev, PCI_CAP_ID_EXP, offset, PCI_EXP_VER1_SIZEOF);
+    if (pos < 0) {
+        return pos;
+    }
+    dev->exp.exp_cap = pos;
+
+    pcie_cap_v1_fill(dev, port, type, PCI_EXP_FLAGS_VER1);
+
+    return pos;
+}
+
+static int
+pcie_endpoint_cap_common_init(PCIDevice *dev, uint8_t offset, uint8_t cap_size)
 {
     uint8_t type = PCI_EXP_TYPE_ENDPOINT;
 
@@ -101,7 +145,19 @@
         type = PCI_EXP_TYPE_RC_END;
     }
 
-    return pcie_cap_init(dev, offset, type, 0);
+    return (cap_size == PCI_EXP_VER1_SIZEOF)
+        ? pcie_cap_v1_init(dev, offset, type, 0)
+        : pcie_cap_init(dev, offset, type, 0);
+}
+
+int pcie_endpoint_cap_init(PCIDevice *dev, uint8_t offset)
+{
+    return pcie_endpoint_cap_common_init(dev, offset, PCI_EXP_VER2_SIZEOF);
+}
+
+int pcie_endpoint_cap_v1_init(PCIDevice *dev, uint8_t offset)
+{
+    return pcie_endpoint_cap_common_init(dev, offset, PCI_EXP_VER1_SIZEOF);
 }
 
 void pcie_cap_exit(PCIDevice *dev)
@@ -109,6 +165,11 @@
     pci_del_capability(dev, PCI_CAP_ID_EXP, PCI_EXP_VER2_SIZEOF);
 }
 
+void pcie_cap_v1_exit(PCIDevice *dev)
+{
+    pci_del_capability(dev, PCI_CAP_ID_EXP, PCI_EXP_VER1_SIZEOF);
+}
+
 uint8_t pcie_cap_get_type(const PCIDevice *dev)
 {
     uint32_t pos = dev->exp.exp_cap;
@@ -250,26 +311,43 @@
         return;
     }
 
-    /* TODO: multifunction hot-plug.
-     * Right now, only a device of function = 0 is allowed to be
-     * hot plugged/unplugged.
+    /* To enable multifunction hot-plug, we just ensure the function
+     * 0 added last. When function 0 is added, we set the sltsta and
+     * inform OS via event notification.
      */
-    assert(PCI_FUNC(pci_dev->devfn) == 0);
-    (void)pci_dev;
+    if (pci_get_function_0(pci_dev)) {
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
+                                   PCI_EXP_SLTSTA_PDS);
+        pcie_cap_slot_event(PCI_DEVICE(hotplug_dev),
+                            PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
+    }
+}
 
-    pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
-                               PCI_EXP_SLTSTA_PDS);
-    pcie_cap_slot_event(PCI_DEVICE(hotplug_dev),
-                        PCI_EXP_HP_EV_PDC | PCI_EXP_HP_EV_ABP);
+static void pcie_unplug_device(PCIBus *bus, PCIDevice *dev, void *opaque)
+{
+    object_unparent(OBJECT(dev));
 }
 
 void pcie_cap_slot_hot_unplug_request_cb(HotplugHandler *hotplug_dev,
                                          DeviceState *dev, Error **errp)
 {
     uint8_t *exp_cap;
+    PCIDevice *pci_dev = PCI_DEVICE(dev);
+    PCIBus *bus = pci_dev->bus;
 
     pcie_cap_slot_hotplug_common(PCI_DEVICE(hotplug_dev), dev, &exp_cap, errp);
 
+    /* In case user cancel the operation of multi-function hot-add,
+     * remove the function that is unexposed to guest individually,
+     * without interaction with guest.
+     */
+    if (pci_dev->devfn &&
+        !bus->devices[0]) {
+        pcie_unplug_device(bus, pci_dev, NULL);
+
+        return;
+    }
+
     pcie_cap_slot_push_attention_button(PCI_DEVICE(hotplug_dev));
 }
 
@@ -381,11 +459,6 @@
     hotplug_event_update_event_status(dev);
 }
 
-static void pcie_unplug_device(PCIBus *bus, PCIDevice *dev, void *opaque)
-{
-    object_unparent(OBJECT(dev));
-}
-
 void pcie_cap_slot_write_config(PCIDevice *dev,
                                 uint32_t addr, uint32_t val, int len)
 {
@@ -416,13 +489,13 @@
      */
     if ((sltsta & PCI_EXP_SLTSTA_PDS) && (val & PCI_EXP_SLTCTL_PCC) &&
         ((val & PCI_EXP_SLTCTL_PIC_OFF) == PCI_EXP_SLTCTL_PIC_OFF)) {
-            PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
-            pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
-                                pcie_unplug_device, NULL);
+        PCIBus *sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(dev));
+        pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
+                            pcie_unplug_device, NULL);
 
-            pci_word_test_and_clear_mask(exp_cap + PCI_EXP_SLTSTA,
-                                         PCI_EXP_SLTSTA_PDS);
-            pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
+        pci_word_test_and_clear_mask(exp_cap + PCI_EXP_SLTSTA,
+                                     PCI_EXP_SLTSTA_PDS);
+        pci_word_test_and_set_mask(exp_cap + PCI_EXP_SLTSTA,
                                        PCI_EXP_SLTSTA_PDC);
     }
 
@@ -597,7 +670,7 @@
 
     assert(offset >= PCI_CONFIG_SPACE_SIZE);
     assert(offset < offset + size);
-    assert(offset + size < PCIE_CONFIG_SPACE_SIZE);
+    assert(offset + size <= PCIE_CONFIG_SPACE_SIZE);
     assert(size >= 8);
     assert(pci_is_express(dev));
 
@@ -635,3 +708,13 @@
                         offset, PCI_ARI_SIZEOF);
     pci_set_long(dev->config + offset + PCI_ARI_CAP, (nextfn & 0xff) << 8);
 }
+
+void pcie_dev_ser_num_init(PCIDevice *dev, uint16_t offset, uint64_t ser_num)
+{
+    static const int pci_dsn_ver = 1;
+    static const int pci_dsn_cap = 4;
+
+    pcie_add_capability(dev, PCI_EXT_CAP_ID_DSN, pci_dsn_ver, offset,
+                        PCI_EXT_CAP_DSN_SIZEOF);
+    pci_set_quad(dev->config + offset + pci_dsn_cap, ser_num);
+}
diff --git a/hw/pci/pcie_aer.c b/hw/pci/pcie_aer.c
index aeeee49..6f87e3c 100644
--- a/hw/pci/pcie_aer.c
+++ b/hw/pci/pcie_aer.c
@@ -18,8 +18,10 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "qapi/qmp/types.h"
+#include "qapi/qmp/qjson.h"
 #include "monitor/monitor.h"
 #include "hw/pci/pci_bridge.h"
 #include "hw/pci/pcie.h"
@@ -94,12 +96,12 @@
     aer_log->log_num = 0;
 }
 
-int pcie_aer_init(PCIDevice *dev, uint16_t offset)
+int pcie_aer_init(PCIDevice *dev, uint16_t offset, uint16_t size)
 {
     PCIExpressDevice *exp;
 
     pcie_add_capability(dev, PCI_EXT_CAP_ID_ERR, PCI_ERR_VER,
-                        offset, PCI_ERR_SIZEOF);
+                        offset, size);
     exp = &dev->exp;
     exp->aer_cap = offset;
 
@@ -123,7 +125,7 @@
                  PCI_ERR_UNC_SUPPORTED);
 
     pci_long_test_and_set_mask(dev->w1cmask + offset + PCI_ERR_COR_STATUS,
-                               PCI_ERR_COR_STATUS);
+                               PCI_ERR_COR_SUPPORTED);
 
     pci_set_long(dev->config + offset + PCI_ERR_COR_MASK,
                  PCI_ERR_COR_MASK_DEFAULT);
@@ -370,7 +372,7 @@
  *
  * Walk up the bus tree from the device, propagate the error message.
  */
-static void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg)
+void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg)
 {
     uint8_t type;
 
@@ -410,7 +412,7 @@
 static void pcie_aer_update_log(PCIDevice *dev, const PCIEAERErr *err)
 {
     uint8_t *aer_cap = dev->config + dev->exp.aer_cap;
-    uint8_t first_bit = ffs(err->status) - 1;
+    uint8_t first_bit = ctz32(err->status);
     uint32_t errcap = pci_get_long(aer_cap + PCI_ERR_CAP);
     int i;
 
@@ -433,7 +435,7 @@
     }
 
     if ((err->flags & PCIE_AER_ERR_TLP_PREFIX_PRESENT) &&
-        (pci_get_long(dev->config + dev->exp.exp_cap + PCI_EXP_DEVCTL2) &
+        (pci_get_long(dev->config + dev->exp.exp_cap + PCI_EXP_DEVCAP2) &
          PCI_EXP_DEVCAP2_EETLPP)) {
         for (i = 0; i < ARRAY_SIZE(err->prefix); ++i) {
             /* 7.10.12 tlp prefix log register */
@@ -618,12 +620,12 @@
  * non-Function specific error must be recorded in all functions.
  * It is the responsibility of the caller of this function.
  * It is also caller's responsibility to determine which function should
- * report the rerror.
+ * report the error.
  *
  * 6.2.4 Error Logging
- * 6.2.5 Sqeunce of Device Error Signaling and Logging Operations
- * table 6-2: Flowchard Showing Sequence of Device Error Signaling and Logging
- *            Operations
+ * 6.2.5 Sequence of Device Error Signaling and Logging Operations
+ * Figure 6-2: Flowchart Showing Sequence of Device Error Signaling and Logging
+ *             Operations
  */
 int pcie_aer_inject_error(PCIDevice *dev, const PCIEAERErr *err)
 {
@@ -816,21 +818,6 @@
     }
 };
 
-void pcie_aer_inject_error_print(Monitor *mon, const QObject *data)
-{
-    QDict *qdict;
-    int devfn;
-    assert(qobject_type(data) == QTYPE_QDICT);
-    qdict = qobject_to_qdict(data);
-
-    devfn = (int)qdict_get_int(qdict, "devfn");
-    monitor_printf(mon, "OK id: %s root bus: %s, bus: %x devfn: %x.%x\n",
-                   qdict_get_str(qdict, "id"),
-                   qdict_get_str(qdict, "root_bus"),
-                   (int) qdict_get_int(qdict, "bus"),
-                   PCI_SLOT(devfn), PCI_FUNC(devfn));
-}
-
 typedef struct PCIEAERErrorName {
     const char *name;
     uint32_t val;
@@ -843,10 +830,6 @@
  */
 static const struct PCIEAERErrorName pcie_aer_error_list[] = {
     {
-        .name = "TRAIN",
-        .val = PCI_ERR_UNC_TRAIN,
-        .correctable = false,
-    }, {
         .name = "DLP",
         .val = PCI_ERR_UNC_DLP,
         .correctable = false,
@@ -963,8 +946,8 @@
     return -EINVAL;
 }
 
-int do_pcie_aer_inject_error(Monitor *mon,
-                             const QDict *qdict, QObject **ret_data)
+static int do_pcie_aer_inject_error(Monitor *mon,
+                                    const QDict *qdict, QObject **ret_data)
 {
     const char *id = qdict_get_str(qdict, "id");
     const char *error_name;
@@ -991,7 +974,7 @@
     if (pcie_aer_parse_error_string(error_name, &error_status, &correctable)) {
         char *e = NULL;
         error_status = strtoul(error_name, &e, 0);
-        correctable = qdict_get_try_bool(qdict, "correctable", 0);
+        correctable = qdict_get_try_bool(qdict, "correctable", false);
         if (!e || *e != '\0') {
             monitor_printf(mon, "invalid error status value. \"%s\"",
                            error_name);
@@ -999,13 +982,13 @@
         }
     }
     err.status = error_status;
-    err.source_id = (pci_bus_num(dev->bus) << 8) | dev->devfn;
+    err.source_id = pci_requester_id(dev);
 
     err.flags = 0;
     if (correctable) {
         err.flags |= PCIE_AER_ERR_IS_CORRECTABLE;
     }
-    if (qdict_get_try_bool(qdict, "advisory_non_fatal", 0)) {
+    if (qdict_get_try_bool(qdict, "advisory_non_fatal", false)) {
         err.flags |= PCIE_AER_ERR_MAYBE_ADVISORY;
     }
     if (qdict_haskey(qdict, "header0")) {
@@ -1036,3 +1019,23 @@
 
     return 0;
 }
+
+void hmp_pcie_aer_inject_error(Monitor *mon, const QDict *qdict)
+{
+    QObject *data;
+    int devfn;
+
+    if (do_pcie_aer_inject_error(mon, qdict, &data) < 0) {
+        return;
+    }
+
+    assert(qobject_type(data) == QTYPE_QDICT);
+    qdict = qobject_to_qdict(data);
+
+    devfn = (int)qdict_get_int(qdict, "devfn");
+    monitor_printf(mon, "OK id: %s root bus: %s, bus: %x devfn: %x.%x\n",
+                   qdict_get_str(qdict, "id"),
+                   qdict_get_str(qdict, "root_bus"),
+                   (int) qdict_get_int(qdict, "bus"),
+                   PCI_SLOT(devfn), PCI_FUNC(devfn));
+}
diff --git a/hw/pci/pcie_host.c b/hw/pci/pcie_host.c
index 3db038f..dcebf57 100644
--- a/hw/pci/pcie_host.c
+++ b/hw/pci/pcie_host.c
@@ -19,6 +19,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pcie_host.h"
@@ -88,6 +89,8 @@
     PCIExpressHost *e = PCIE_HOST_BRIDGE(obj);
 
     e->base_addr = PCIE_BASE_ADDR_UNMAPPED;
+    memory_region_init_io(&e->mmio, OBJECT(e), &pcie_mmcfg_ops, e, "pcie-mmcfg-mmio",
+                          PCIE_MMCFG_SIZE_MAX);
 }
 
 void pcie_host_mmcfg_unmap(PCIExpressHost *e)
@@ -98,15 +101,19 @@
     }
 }
 
-void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr,
-                         uint32_t size)
+void pcie_host_mmcfg_init(PCIExpressHost *e, uint32_t size)
 {
     assert(!(size & (size - 1)));       /* power of 2 */
     assert(size >= PCIE_MMCFG_SIZE_MIN);
     assert(size <= PCIE_MMCFG_SIZE_MAX);
     e->size = size;
-    memory_region_init_io(&e->mmio, OBJECT(e), &pcie_mmcfg_ops, e,
-                          "pcie-mmcfg", e->size);
+    memory_region_set_size(&e->mmio, e->size);
+}
+
+void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr,
+                         uint32_t size)
+{
+    pcie_host_mmcfg_init(e, size);
     e->base_addr = addr;
     memory_region_add_subregion(get_system_memory(), e->base_addr, &e->mmio);
 }
@@ -116,10 +123,12 @@
                             hwaddr addr,
                             uint32_t size)
 {
+    memory_region_transaction_begin();
     pcie_host_mmcfg_unmap(e);
     if (enable) {
         pcie_host_mmcfg_map(e, addr, size);
     }
+    memory_region_transaction_commit();
 }
 
 static const TypeInfo pcie_host_type_info = {
diff --git a/hw/pci/pcie_port.c b/hw/pci/pcie_port.c
index 40ca8d5..6432b9a 100644
--- a/hw/pci/pcie_port.c
+++ b/hw/pci/pcie_port.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pcie_port.h"
 #include "hw/hotplug.h"
 
diff --git a/hw/pci/shpc.c b/hw/pci/shpc.c
index 27c496e..3dcd472 100644
--- a/hw/pci/shpc.c
+++ b/hw/pci/shpc.c
@@ -1,13 +1,12 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
-#include <strings.h>
-#include <stdint.h>
 #include "qemu/range.h"
 #include "qemu/error-report.h"
 #include "hw/pci/shpc.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_bus.h"
 #include "hw/pci/msi.h"
-#include "qapi/qmp/qerror.h"
 
 /* TODO: model power only and disabled slot states. */
 /* TODO: handle SERR and wakeups */
@@ -61,7 +60,7 @@
 /* Same slot state masks are used for command and status registers */
 #define SHPC_SLOT_STATE_MASK     0x03
 #define SHPC_SLOT_STATE_SHIFT \
-    (ffs(SHPC_SLOT_STATE_MASK) - 1)
+    ctz32(SHPC_SLOT_STATE_MASK)
 
 #define SHPC_STATE_NO       0x0
 #define SHPC_STATE_PWRONLY  0x1
@@ -70,10 +69,10 @@
 
 #define SHPC_SLOT_PWR_LED_MASK   0xC
 #define SHPC_SLOT_PWR_LED_SHIFT \
-    (ffs(SHPC_SLOT_PWR_LED_MASK) - 1)
+    ctz32(SHPC_SLOT_PWR_LED_MASK)
 #define SHPC_SLOT_ATTN_LED_MASK  0x30
 #define SHPC_SLOT_ATTN_LED_SHIFT \
-    (ffs(SHPC_SLOT_ATTN_LED_MASK) - 1)
+    ctz32(SHPC_SLOT_ATTN_LED_MASK)
 
 #define SHPC_LED_NO     0x0
 #define SHPC_LED_ON     0x1
@@ -136,7 +135,7 @@
 static uint16_t shpc_get_status(SHPCDevice *shpc, int slot, uint16_t msk)
 {
     uint8_t *status = shpc->config + SHPC_SLOT_STATUS(slot);
-    return (pci_get_word(status) & msk) >> (ffs(msk) - 1);
+    return (pci_get_word(status) & msk) >> ctz32(msk);
 }
 
 static void shpc_set_status(SHPCDevice *shpc,
@@ -144,7 +143,7 @@
 {
     uint8_t *status = shpc->config + SHPC_SLOT_STATUS(slot);
     pci_word_test_and_clear_mask(status, msk);
-    pci_word_test_and_set_mask(status, value << (ffs(msk) - 1));
+    pci_word_test_and_set_mask(status, value << ctz32(msk));
 }
 
 static void shpc_interrupt_update(PCIDevice *d)
@@ -159,7 +158,7 @@
     for (slot = 0; slot < shpc->nslots; ++slot) {
         uint8_t event = shpc->config[SHPC_SLOT_EVENT_LATCH(slot)];
         uint8_t disable = shpc->config[SHPC_SLOT_EVENT_SERR_INT_DIS(d, slot)];
-        uint32_t mask = 1 << SHPC_IDX_TO_LOGICAL(slot);
+        uint32_t mask = 1U << SHPC_IDX_TO_LOGICAL(slot);
         if (event & ~disable) {
             int_locator |= mask;
         }
@@ -663,13 +662,22 @@
     SHPCDevice *shpc = d->shpc;
     d->cap_present &= ~QEMU_PCI_CAP_SHPC;
     memory_region_del_subregion(bar, &shpc->mmio);
-    object_unparent(OBJECT(&shpc->mmio));
     /* TODO: cleanup config space changes? */
+}
+
+void shpc_free(PCIDevice *d)
+{
+    SHPCDevice *shpc = d->shpc;
+    if (!shpc) {
+        return;
+    }
+    object_unparent(OBJECT(&shpc->mmio));
     g_free(shpc->config);
     g_free(shpc->cmask);
     g_free(shpc->wmask);
     g_free(shpc->w1cmask);
     g_free(shpc);
+    d->shpc = NULL;
 }
 
 void shpc_cap_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int l)
diff --git a/hw/pci/slotid_cap.c b/hw/pci/slotid_cap.c
index 62f7bae..aec1e91 100644
--- a/hw/pci/slotid_cap.c
+++ b/hw/pci/slotid_cap.c
@@ -1,9 +1,10 @@
+#include "qemu/osdep.h"
 #include "hw/pci/slotid_cap.h"
 #include "hw/pci/pci.h"
 #include "qemu/error-report.h"
 
 #define SLOTID_CAP_LENGTH 4
-#define SLOTID_NSLOTS_SHIFT (ffs(PCI_SID_ESR_NSLOTS) - 1)
+#define SLOTID_NSLOTS_SHIFT ctz32(PCI_SID_ESR_NSLOTS)
 
 int slotid_cap_init(PCIDevice *d, int nslots,
                     uint8_t chassis,
diff --git a/hw/pci/trace-events b/hw/pci/trace-events
new file mode 100644
index 0000000..2b9cf24
--- /dev/null
+++ b/hw/pci/trace-events
@@ -0,0 +1,9 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/pci/pci.c
+pci_update_mappings_del(void *d, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
+pci_update_mappings_add(void *d, uint32_t bus, uint32_t slot, uint32_t func, int bar, uint64_t addr, uint64_t size) "d=%p %02x:%02x.%x %d,%#"PRIx64"+%#"PRIx64
+
+# hw/pci/pci_host.c
+pci_cfg_read(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x -> 0x%x"
+pci_cfg_write(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x <- 0x%x"
diff --git a/hw/pcmcia/pcmcia.c b/hw/pcmcia/pcmcia.c
index 78efe5a..1956721 100644
--- a/hw/pcmcia/pcmcia.c
+++ b/hw/pcmcia/pcmcia.c
@@ -4,6 +4,7 @@
  * Copyright 2013 SUSE LINUX Products GmbH
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/pcmcia.h"
diff --git a/hw/pcmcia/pxa2xx.c b/hw/pcmcia/pxa2xx.c
index a7e1877..20c9c75 100644
--- a/hw/pcmcia/pxa2xx.c
+++ b/hw/pcmcia/pxa2xx.c
@@ -10,6 +10,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "hw/pcmcia.h"
@@ -163,7 +164,7 @@
     sysbus_init_mmio(sbd, &s->container_mem);
 
     /* Socket I/O Memory Space */
-    memory_region_init_io(&s->iomem, NULL, &pxa2xx_pcmcia_io_ops, s,
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_pcmcia_io_ops, s,
                           "pxa2xx-pcmcia-io", 0x04000000);
     memory_region_add_subregion(&s->container_mem, 0x00000000,
                                 &s->iomem);
@@ -171,13 +172,13 @@
     /* Then next 64 MB is reserved */
 
     /* Socket Attribute Memory Space */
-    memory_region_init_io(&s->attr_iomem, NULL, &pxa2xx_pcmcia_attr_ops, s,
+    memory_region_init_io(&s->attr_iomem, obj, &pxa2xx_pcmcia_attr_ops, s,
                           "pxa2xx-pcmcia-attribute", 0x04000000);
     memory_region_add_subregion(&s->container_mem, 0x08000000,
                                 &s->attr_iomem);
 
     /* Socket Common Memory Space */
-    memory_region_init_io(&s->common_iomem, NULL, &pxa2xx_pcmcia_common_ops, s,
+    memory_region_init_io(&s->common_iomem, obj, &pxa2xx_pcmcia_common_ops, s,
                           "pxa2xx-pcmcia-common", 0x04000000);
     memory_region_add_subregion(&s->container_mem, 0x0c000000,
                                 &s->common_iomem);
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index 19d9920..91a3420 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -3,10 +3,12 @@
 # IBM pSeries (sPAPR)
 obj-$(CONFIG_PSERIES) += spapr.o spapr_vio.o spapr_events.o
 obj-$(CONFIG_PSERIES) += spapr_hcall.o spapr_iommu.o spapr_rtas.o
-obj-$(CONFIG_PSERIES) += spapr_pci.o
+obj-$(CONFIG_PSERIES) += spapr_pci.o spapr_rtc.o spapr_drc.o spapr_rng.o
+obj-$(CONFIG_PSERIES) += spapr_cpu_core.o
 ifeq ($(CONFIG_PCI)$(CONFIG_PSERIES)$(CONFIG_LINUX), yyy)
 obj-y += spapr_pci_vfio.o
 endif
+obj-$(CONFIG_PSERIES) += spapr_rtas_ddw.o
 # PowerPC 4xx boards
 obj-y += ppc405_boards.o ppc4xx_devs.o ppc405_uc.o ppc440_bamboo.o
 obj-y += ppc4xx_pci.o
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index 2832fc0..0cd534d 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -14,7 +14,8 @@
  * (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "e500.h"
 #include "e500-ccsr.h"
@@ -51,21 +52,16 @@
 #define RAM_SIZES_ALIGN            (64UL << 20)
 
 /* TODO: parameterize */
-#define MPC8544_CCSRBAR_BASE       0xE0000000ULL
 #define MPC8544_CCSRBAR_SIZE       0x00100000ULL
 #define MPC8544_MPIC_REGS_OFFSET   0x40000ULL
 #define MPC8544_MSI_REGS_OFFSET   0x41600ULL
 #define MPC8544_SERIAL0_REGS_OFFSET 0x4500ULL
 #define MPC8544_SERIAL1_REGS_OFFSET 0x4600ULL
 #define MPC8544_PCI_REGS_OFFSET    0x8000ULL
-#define MPC8544_PCI_REGS_BASE      (MPC8544_CCSRBAR_BASE + \
-                                    MPC8544_PCI_REGS_OFFSET)
 #define MPC8544_PCI_REGS_SIZE      0x1000ULL
-#define MPC8544_PCI_IO             0xE1000000ULL
 #define MPC8544_UTIL_OFFSET        0xe0000ULL
-#define MPC8544_SPIN_BASE          0xEF000000ULL
 #define MPC8XXX_GPIO_OFFSET        0x000FF000ULL
-#define MPC8XXX_GPIO_IRQ           43
+#define MPC8XXX_GPIO_IRQ           47
 
 struct boot_info
 {
@@ -293,12 +289,12 @@
     int len;
     uint32_t pci_ranges[14] =
         {
-            0x2000000, 0x0, 0xc0000000,
-            0x0, 0xc0000000,
+            0x2000000, 0x0, params->pci_mmio_bus_base,
+            params->pci_mmio_base >> 32, params->pci_mmio_base,
             0x0, 0x20000000,
 
             0x1000000, 0x0, 0x0,
-            0x0, 0xe1000000,
+            params->pci_pio_base >> 32, params->pci_pio_base,
             0x0, 0x10000,
         };
     QemuOpts *machine_opts = qemu_get_machine_opts();
@@ -313,6 +309,7 @@
         }
 
         fdt = load_device_tree(filename, &fdt_size);
+        g_free(filename);
         if (!fdt) {
             goto out;
         }
@@ -389,7 +386,7 @@
         CPUState *cpu;
         PowerPCCPU *pcpu;
         char cpu_name[128];
-        uint64_t cpu_release_addr = MPC8544_SPIN_BASE + (i * 0x20);
+        uint64_t cpu_release_addr = params->spin_base + (i * 0x20);
 
         cpu = qemu_get_cpu(i);
         if (cpu == NULL) {
@@ -426,7 +423,7 @@
 
     qemu_fdt_add_subnode(fdt, "/aliases");
     /* XXX These should go into their respective devices' code */
-    snprintf(soc, sizeof(soc), "/soc@%llx", MPC8544_CCSRBAR_BASE);
+    snprintf(soc, sizeof(soc), "/soc@%"PRIx64, params->ccsrbar_base);
     qemu_fdt_add_subnode(fdt, soc);
     qemu_fdt_setprop_string(fdt, soc, "device_type", "soc");
     qemu_fdt_setprop(fdt, soc, "compatible", compatible_sb,
@@ -434,7 +431,7 @@
     qemu_fdt_setprop_cell(fdt, soc, "#address-cells", 1);
     qemu_fdt_setprop_cell(fdt, soc, "#size-cells", 1);
     qemu_fdt_setprop_cells(fdt, soc, "ranges", 0x0,
-                           MPC8544_CCSRBAR_BASE >> 32, MPC8544_CCSRBAR_BASE,
+                           params->ccsrbar_base >> 32, params->ccsrbar_base,
                            MPC8544_CCSRBAR_SIZE);
     /* XXX should contain a reasonable value */
     qemu_fdt_setprop_cell(fdt, soc, "bus-frequency", 0);
@@ -493,7 +490,8 @@
     qemu_fdt_setprop_cell(fdt, msi, "phandle", msi_ph);
     qemu_fdt_setprop_cell(fdt, msi, "linux,phandle", msi_ph);
 
-    snprintf(pci, sizeof(pci), "/pci@%llx", MPC8544_PCI_REGS_BASE);
+    snprintf(pci, sizeof(pci), "/pci@%llx",
+             params->ccsrbar_base + MPC8544_PCI_REGS_OFFSET);
     qemu_fdt_add_subnode(fdt, pci);
     qemu_fdt_setprop_cell(fdt, pci, "cell-index", 0);
     qemu_fdt_setprop_string(fdt, pci, "compatible", "fsl,mpc8540-pci");
@@ -512,8 +510,10 @@
     }
     qemu_fdt_setprop_cell(fdt, pci, "fsl,msi", msi_ph);
     qemu_fdt_setprop(fdt, pci, "ranges", pci_ranges, sizeof(pci_ranges));
-    qemu_fdt_setprop_cells(fdt, pci, "reg", MPC8544_PCI_REGS_BASE >> 32,
-                           MPC8544_PCI_REGS_BASE, 0, 0x1000);
+    qemu_fdt_setprop_cells(fdt, pci, "reg",
+                           (params->ccsrbar_base + MPC8544_PCI_REGS_OFFSET) >> 32,
+                           (params->ccsrbar_base + MPC8544_PCI_REGS_OFFSET),
+                           0, 0x1000);
     qemu_fdt_setprop_cell(fdt, pci, "clock-frequency", 66666666);
     qemu_fdt_setprop_cell(fdt, pci, "#interrupt-cells", 1);
     qemu_fdt_setprop_cell(fdt, pci, "#size-cells", 2);
@@ -601,7 +601,7 @@
 }
 
 /* Create -kernel TLB entries for BookE.  */
-static inline hwaddr booke206_page_size_to_tlb(uint64_t size)
+hwaddr booke206_page_size_to_tlb(uint64_t size)
 {
     return 63 - clz64(size >> 10);
 }
@@ -708,17 +708,19 @@
 }
 
 static DeviceState *ppce500_init_mpic_kvm(PPCE500Params *params,
-                                          qemu_irq **irqs)
+                                          qemu_irq **irqs, Error **errp)
 {
+    Error *err = NULL;
     DeviceState *dev;
     CPUState *cs;
-    int r;
 
     dev = qdev_create(NULL, TYPE_KVM_OPENPIC);
     qdev_prop_set_uint32(dev, "model", params->mpic_version);
 
-    r = qdev_init(dev);
-    if (r) {
+    object_property_set_bool(OBJECT(dev), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        object_unparent(OBJECT(dev));
         return NULL;
     }
 
@@ -733,8 +735,8 @@
     return dev;
 }
 
-static qemu_irq *ppce500_init_mpic(PPCE500Params *params, MemoryRegion *ccsr,
-                                   qemu_irq **irqs)
+static qemu_irq *ppce500_init_mpic(MachineState *machine, PPCE500Params *params,
+                                   MemoryRegion *ccsr, qemu_irq **irqs)
 {
     qemu_irq *mpic;
     DeviceState *dev = NULL;
@@ -744,20 +746,15 @@
     mpic = g_new0(qemu_irq, 256);
 
     if (kvm_enabled()) {
-        QemuOpts *machine_opts = qemu_get_machine_opts();
-        bool irqchip_allowed = qemu_opt_get_bool(machine_opts,
-                                                "kernel_irqchip", true);
-        bool irqchip_required = qemu_opt_get_bool(machine_opts,
-                                                  "kernel_irqchip", false);
+        Error *err = NULL;
 
-        if (irqchip_allowed) {
-            dev = ppce500_init_mpic_kvm(params, irqs);
+        if (machine_kernel_irqchip_allowed(machine)) {
+            dev = ppce500_init_mpic_kvm(params, irqs, &err);
         }
-
-        if (irqchip_required && !dev) {
-            fprintf(stderr, "%s: irqchip requested but unavailable\n",
-                    __func__);
-            abort();
+        if (machine_kernel_irqchip_required(machine) && !dev) {
+            error_reportf_err(err,
+                              "kernel_irqchip requested but unavailable: ");
+            exit(1);
         }
     }
 
@@ -841,7 +838,7 @@
         irqs[i][OPENPIC_OUTPUT_INT] = input[PPCE500_INPUT_INT];
         irqs[i][OPENPIC_OUTPUT_CINT] = input[PPCE500_INPUT_CINT];
         env->spr_cb[SPR_BOOKE_PIR].default_value = cs->cpu_index = i;
-        env->mpic_iack = MPC8544_CCSRBAR_BASE +
+        env->mpic_iack = params->ccsrbar_base +
                          MPC8544_MPIC_REGS_OFFSET + 0xa0;
 
         ppc_booke_timers_init(cpu, 400000000, PPC_TIMER_E500);
@@ -875,10 +872,10 @@
     qdev_init_nofail(dev);
     ccsr = CCSR(dev);
     ccsr_addr_space = &ccsr->ccsr_space;
-    memory_region_add_subregion(address_space_mem, MPC8544_CCSRBAR_BASE,
+    memory_region_add_subregion(address_space_mem, params->ccsrbar_base,
                                 ccsr_addr_space);
 
-    mpic = ppce500_init_mpic(params, ccsr_addr_space, irqs);
+    mpic = ppce500_init_mpic(machine, params, ccsr_addr_space, irqs);
 
     /* Serial */
     if (serial_hds[0]) {
@@ -917,8 +914,6 @@
     if (!pci_bus)
         printf("couldn't create PCI controller!\n");
 
-    sysbus_mmio_map(SYS_BUS_DEVICE(dev), 1, MPC8544_PCI_IO);
-
     if (pci_bus) {
         /* Register network interfaces. */
         for (i = 0; i < nb_nics; i++) {
@@ -927,7 +922,7 @@
     }
 
     /* Register spinning region */
-    sysbus_create_simple("e500-spin", MPC8544_SPIN_BASE, NULL);
+    sysbus_create_simple("e500-spin", params->spin_base, NULL);
 
     if (cur_base < (32 * 1024 * 1024)) {
         /* u-boot occupies memory up to 32MB, so load blobs above */
@@ -1023,7 +1018,7 @@
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
 
     bios_size = load_elf(filename, NULL, NULL, &bios_entry, &loadaddr, NULL,
-                         1, ELF_MACHINE, 0);
+                         1, PPC_ELF_MACHINE, 0, 0);
     if (bios_size < 0) {
         /*
          * Hrm. No ELF image? Try a uImage, maybe someone is giving us an
@@ -1036,6 +1031,7 @@
             exit(1);
         }
     }
+    g_free(filename);
 
     /* Reserve space for dtb */
     dt_base = (loadaddr + bios_size + DTC_LOAD_PAD) & ~DTC_PAD_MASK;
@@ -1053,10 +1049,6 @@
     boot_info->entry = bios_entry;
     boot_info->dt_base = dt_base;
     boot_info->dt_size = dt_size;
-
-    if (kvm_enabled()) {
-        kvmppc_init();
-    }
 }
 
 static int e500_ccsr_initfn(SysBusDevice *dev)
diff --git a/hw/ppc/e500.h b/hw/ppc/e500.h
index 9f61ab2..70ba1d8 100644
--- a/hw/ppc/e500.h
+++ b/hw/ppc/e500.h
@@ -17,8 +17,15 @@
     hwaddr platform_bus_size;
     int platform_bus_first_irq;
     int platform_bus_num_irqs;
+    hwaddr ccsrbar_base;
+    hwaddr pci_pio_base;
+    hwaddr pci_mmio_base;
+    hwaddr pci_mmio_bus_base;
+    hwaddr spin_base;
 } PPCE500Params;
 
 void ppce500_init(MachineState *machine, PPCE500Params *params);
 
+hwaddr booke206_page_size_to_tlb(uint64_t size);
+
 #endif
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index d50ae00..94b4545 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -9,11 +9,12 @@
  * (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "e500.h"
 #include "hw/boards.h"
 #include "sysemu/device_tree.h"
+#include "sysemu/kvm.h"
 #include "hw/pci/pci.h"
 #include "hw/ppc/openpic.h"
 #include "kvm_ppc.h"
@@ -41,6 +42,11 @@
         .platform_bus_size = (128ULL * 1024 * 1024),
         .platform_bus_first_irq = 5,
         .platform_bus_num_irqs = 10,
+        .ccsrbar_base = 0xFE0000000ULL,
+        .pci_pio_base = 0xFE1000000ULL,
+        .pci_mmio_base = 0xC00000000ULL,
+        .pci_mmio_bus_base = 0xE0000000ULL,
+        .spin_base = 0xFEF000000ULL,
     };
 
     /* Older KVM versions don't support EPR which breaks guests when we announce
@@ -52,17 +58,12 @@
     ppce500_init(machine, &params);
 }
 
-static QEMUMachine e500plat_machine = {
-    .name = "ppce500",
-    .desc = "generic paravirt e500 platform",
-    .init = e500plat_init,
-    .max_cpus = 32,
-    .has_dynamic_sysbus = true,
-};
-
-static void e500plat_machine_init(void)
+static void e500plat_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&e500plat_machine);
+    mc->desc = "generic paravirt e500 platform";
+    mc->init = e500plat_init;
+    mc->max_cpus = 32;
+    mc->has_dynamic_sysbus = true;
 }
 
-machine_init(e500plat_machine_init);
+DEFINE_MACHINE("ppce500", e500plat_machine_init)
diff --git a/hw/ppc/mac.h b/hw/ppc/mac.h
index 8bdba30..20cbddb 100644
--- a/hw/ppc/mac.h
+++ b/hw/ppc/mac.h
@@ -22,8 +22,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#if !defined(__PPC_MAC_H__)
-#define __PPC_MAC_H__
+
+#ifndef PPC_MAC_H
+#define PPC_MAC_H
 
 #include "exec/memory.h"
 #include "hw/sysbus.h"
@@ -103,11 +104,16 @@
     uint8_t last_b;
     uint8_t last_acr;
 
+    /* MacOS 9 is racy and requires a delay upon setting the SR_INT bit */
+    QEMUTimer *sr_delay_timer;
+
     int data_in_size;
     int data_in_index;
     int data_out_index;
 
     qemu_irq irq;
+    uint16_t adb_poll_mask;
+    uint8_t autopoll_rate_ms;
     uint8_t autopoll;
     uint8_t data_in[128];
     uint8_t data_out[16];
@@ -131,7 +137,6 @@
 
     MemoryRegion mem;
     IDEBus bus;
-    BlockAIOCB *aiocb;
     IDEDMA dma;
     void *dbdma;
     bool dma_active;
@@ -180,4 +185,4 @@
 } MacIONVRAMState;
 
 void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
-#endif /* !defined(__PPC_MAC_H__) */
+#endif /* PPC_MAC_H */
diff --git a/hw/ppc/mac_newworld.c b/hw/ppc/mac_newworld.c
index 89aee71..7d25106 100644
--- a/hw/ppc/mac_newworld.c
+++ b/hw/ppc/mac_newworld.c
@@ -46,6 +46,8 @@
  * 0001:05:0c.0 IDE interface [0101]: Broadcom K2 SATA [1166:0240]
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/mac.h"
@@ -62,12 +64,14 @@
 #include "hw/ide.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "hw/usb.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
 #include "hw/sysbus.h"
+#include "qemu/cutils.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -116,10 +120,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int fw_cfg_boot_set(void *opaque, const char *boot_device)
+static void fw_cfg_boot_set(void *opaque, const char *boot_device,
+                            Error **errp)
 {
-    fw_cfg_add_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
-    return 0;
+    fw_cfg_modify_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
 }
 
 static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
@@ -145,7 +149,6 @@
 static void ppc_core99_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -182,14 +185,15 @@
     linux_boot = (kernel_filename != NULL);
 
     /* init CPUs */
-    if (cpu_model == NULL)
+    if (machine->cpu_model == NULL) {
 #ifdef TARGET_PPC64
-        cpu_model = "970fx";
+        machine->cpu_model = "970fx";
 #else
-        cpu_model = "G4";
+        machine->cpu_model = "G4";
 #endif
+    }
     for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_ppc_init(cpu_model);
+        cpu = cpu_ppc_init(machine->cpu_model);
         if (cpu == NULL) {
             fprintf(stderr, "Unable to find PowerPC CPU definition\n");
             exit(1);
@@ -207,7 +211,7 @@
 
     /* allocate and load BIOS */
     memory_region_init_ram(bios, NULL, "ppc_core99.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -219,14 +223,14 @@
     /* Load OpenBIOS (ELF) */
     if (filename) {
         bios_size = load_elf(filename, NULL, NULL, NULL,
-                             NULL, NULL, 1, ELF_MACHINE, 0);
+                             NULL, NULL, 1, PPC_ELF_MACHINE, 0, 0);
 
         g_free(filename);
     } else {
         bios_size = -1;
     }
     if (bios_size < 0 || bios_size > BIOS_SIZE) {
-        hw_error("qemu: could not load PowerPC bios '%s'\n", bios_name);
+        error_report("could not load PowerPC bios '%s'", bios_name);
         exit(1);
     }
 
@@ -242,7 +246,8 @@
         kernel_base = KERNEL_LOAD_ADDR;
 
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, kernel_base,
                                     ram_size - kernel_base, bswap_needed,
@@ -252,7 +257,7 @@
                                               kernel_base,
                                               ram_size - kernel_base);
         if (kernel_size < 0) {
-            hw_error("qemu: could not load kernel '%s'\n", kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
         /* load initrd */
@@ -261,8 +266,8 @@
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               ram_size - initrd_base);
             if (initrd_size < 0) {
-                hw_error("qemu: could not load initial ram disk '%s'\n",
-                         initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
             cmdline_base = round_page(initrd_base + initrd_size);
@@ -344,7 +349,7 @@
             break;
 #endif /* defined(TARGET_PPC64) */
         default:
-            hw_error("Bus model not supported on mac99 machine\n");
+            error_report("Bus model not supported on mac99 machine");
             exit(1);
         }
     }
@@ -375,6 +380,9 @@
         pci_bus = pci_pmac_init(pic, get_system_memory(), get_system_io());
         machine_arch = ARCH_MAC99;
     }
+    object_property_set_bool(OBJECT(pci_bus), true, "realized", &error_abort);
+
+    machine->usb |= defaults_enabled() && !machine->usb_disabled;
 
     /* Timebase Frequency */
     if (kvm_enabled()) {
@@ -417,13 +425,16 @@
     dev = qdev_create(adb_bus, TYPE_ADB_MOUSE);
     qdev_init_nofail(dev);
 
-    if (usb_enabled(machine_arch == ARCH_MAC99_U3)) {
+    if (machine->usb) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
+
         /* U3 needs to use USB for input because Linux doesn't support via-cuda
         on PPC64 */
         if (machine_arch == ARCH_MAC99_U3) {
-            usbdevice_create("keyboard");
-            usbdevice_create("mouse");
+            USBBus *usb_bus = usb_bus_find(-1);
+
+            usb_create_simple(usb_bus, "usb-kbd");
+            usb_create_simple(usb_bus, "usb-mouse");
         }
     }
 
@@ -454,9 +465,8 @@
     pmac_format_nvram_partition(nvr, 0x2000);
     /* No PCI init: the BIOS will do it */
 
-    fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg = fw_cfg_init_mem(CFG_ADDR, CFG_ADDR + 2);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
-    fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, machine_arch);
     fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_ADDR, kernel_base);
@@ -501,18 +511,26 @@
     return 2;
 }
 
-static QEMUMachine core99_machine = {
-    .name = "mac99",
-    .desc = "Mac99 based PowerMAC",
-    .init = ppc_core99_init,
-    .max_cpus = MAX_CPUS,
-    .default_boot_order = "cd",
-    .kvm_type = core99_kvm_type,
-};
-
-static void core99_machine_init(void)
+static void core99_machine_class_init(ObjectClass *oc, void *data)
 {
-    qemu_register_machine(&core99_machine);
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Mac99 based PowerMAC";
+    mc->init = ppc_core99_init;
+    mc->max_cpus = MAX_CPUS;
+    mc->default_boot_order = "cd";
+    mc->kvm_type = core99_kvm_type;
 }
 
-machine_init(core99_machine_init);
+static const TypeInfo core99_machine_info = {
+    .name          = MACHINE_TYPE_NAME("mac99"),
+    .parent        = TYPE_MACHINE,
+    .class_init    = core99_machine_class_init,
+};
+
+static void mac_machine_register_types(void)
+{
+    type_register_static(&core99_machine_info);
+}
+
+type_init(mac_machine_register_types)
diff --git a/hw/ppc/mac_oldworld.c b/hw/ppc/mac_oldworld.c
index 32c21a4..4479487 100644
--- a/hw/ppc/mac_oldworld.c
+++ b/hw/ppc/mac_oldworld.c
@@ -23,6 +23,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "mac.h"
@@ -38,10 +40,12 @@
 #include "hw/ide.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
+#include "qemu/cutils.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -49,13 +53,12 @@
 #define CLOCKFREQ 266000000UL
 #define BUSFREQ 66000000UL
 
-static int fw_cfg_boot_set(void *opaque, const char *boot_device)
+static void fw_cfg_boot_set(void *opaque, const char *boot_device,
+                            Error **errp)
 {
-    fw_cfg_add_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
-    return 0;
+    fw_cfg_modify_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
 }
 
-
 static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
 {
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
@@ -76,7 +79,6 @@
 static void ppc_heathrow_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -108,10 +110,10 @@
     linux_boot = (kernel_filename != NULL);
 
     /* init CPUs */
-    if (cpu_model == NULL)
-        cpu_model = "G3";
+    if (machine->cpu_model == NULL)
+        machine->cpu_model = "G3";
     for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_ppc_init(cpu_model);
+        cpu = cpu_ppc_init(machine->cpu_model);
         if (cpu == NULL) {
             fprintf(stderr, "Unable to find PowerPC CPU definition\n");
             exit(1);
@@ -137,7 +139,7 @@
 
     /* allocate and load BIOS */
     memory_region_init_ram(bios, NULL, "ppc_heathrow.bios", BIOS_SIZE,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(bios);
 
     if (bios_name == NULL)
@@ -149,13 +151,13 @@
     /* Load OpenBIOS (ELF) */
     if (filename) {
         bios_size = load_elf(filename, 0, NULL, NULL, NULL, NULL,
-                             1, ELF_MACHINE, 0);
+                             1, PPC_ELF_MACHINE, 0, 0);
         g_free(filename);
     } else {
         bios_size = -1;
     }
     if (bios_size < 0 || bios_size > BIOS_SIZE) {
-        hw_error("qemu: could not load PowerPC bios '%s'\n", bios_name);
+        error_report("could not load PowerPC bios '%s'", bios_name);
         exit(1);
     }
 
@@ -170,7 +172,8 @@
 #endif
         kernel_base = KERNEL_LOAD_ADDR;
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, kernel_base,
                                     ram_size - kernel_base, bswap_needed,
@@ -180,8 +183,7 @@
                                               kernel_base,
                                               ram_size - kernel_base);
         if (kernel_size < 0) {
-            hw_error("qemu: could not load kernel '%s'\n",
-                      kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
         /* load initrd */
@@ -190,8 +192,8 @@
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               ram_size - initrd_base);
             if (initrd_size < 0) {
-                hw_error("qemu: could not load initial ram disk '%s'\n",
-                         initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
             cmdline_base = round_page(initrd_base + initrd_size);
@@ -248,7 +250,8 @@
                 ((qemu_irq *)env->irq_inputs)[PPC6xx_INPUT_INT];
             break;
         default:
-            hw_error("Bus model not supported on OldWorld Mac machine\n");
+            error_report("Bus model not supported on OldWorld Mac machine");
+            exit(1);
         }
     }
 
@@ -261,7 +264,8 @@
 
     /* init basic PC hardware */
     if (PPC_INPUT(env) != PPC_FLAGS_INPUT_6xx) {
-        hw_error("Only 6xx bus is supported on heathrow machine\n");
+        error_report("Only 6xx bus is supported on heathrow machine");
+        exit(1);
     }
     pic = heathrow_pic_init(&pic_mem, 1, heathrow_irqs);
     pci_bus = pci_grackle_init(0xfec00000, pic,
@@ -305,7 +309,7 @@
     dev = qdev_create(adb_bus, TYPE_ADB_MOUSE);
     qdev_init_nofail(dev);
 
-    if (usb_enabled(false)) {
+    if (machine_usb(machine)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
 
@@ -314,9 +318,8 @@
 
     /* No PCI init: the BIOS will do it */
 
-    fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg = fw_cfg_init_mem(CFG_ADDR, CFG_ADDR + 2);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
-    fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, ARCH_HEATHROW);
     fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_ADDR, kernel_base);
@@ -360,21 +363,17 @@
     return 2;
 }
 
-static QEMUMachine heathrow_machine = {
-    .name = "g3beige",
-    .desc = "Heathrow based PowerMAC",
-    .init = ppc_heathrow_init,
-    .max_cpus = MAX_CPUS,
-#ifndef TARGET_PPC64
-    .is_default = 1,
-#endif
-    .default_boot_order = "cd", /* TOFIX "cad" when Mac floppy is implemented */
-    .kvm_type = heathrow_kvm_type,
-};
-
-static void heathrow_machine_init(void)
+static void heathrow_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&heathrow_machine);
+    mc->desc = "Heathrow based PowerMAC";
+    mc->init = ppc_heathrow_init;
+    mc->max_cpus = MAX_CPUS;
+#ifndef TARGET_PPC64
+    mc->is_default = 1;
+#endif
+    /* TOFIX "cad" when Mac floppy is implemented */
+    mc->default_boot_order = "cd";
+    mc->kvm_type = heathrow_kvm_type;
 }
 
-machine_init(heathrow_machine_init);
+DEFINE_MACHINE("g3beige", heathrow_machine_init)
diff --git a/hw/ppc/mpc8544_guts.c b/hw/ppc/mpc8544_guts.c
index a10abe9..ba69178 100644
--- a/hw/ppc/mpc8544_guts.c
+++ b/hw/ppc/mpc8544_guts.c
@@ -17,6 +17,9 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
diff --git a/hw/ppc/mpc8544ds.c b/hw/ppc/mpc8544ds.c
index b99f74a..27b8289 100644
--- a/hw/ppc/mpc8544ds.c
+++ b/hw/ppc/mpc8544ds.c
@@ -9,12 +9,13 @@
  * (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "e500.h"
 #include "hw/boards.h"
 #include "sysemu/device_tree.h"
 #include "hw/ppc/openpic.h"
+#include "qemu/error-report.h"
 
 static void mpc8544ds_fixup_devtree(PPCE500Params *params, void *fdt)
 {
@@ -33,22 +34,27 @@
         .pci_nr_slots = 2,
         .fixup_devtree = mpc8544ds_fixup_devtree,
         .mpic_version = OPENPIC_MODEL_FSL_MPIC_20,
+        .ccsrbar_base = 0xE0000000ULL,
+        .pci_mmio_base = 0xC0000000ULL,
+        .pci_mmio_bus_base = 0xC0000000ULL,
+        .pci_pio_base = 0xE1000000ULL,
+        .spin_base = 0xEF000000ULL,
     };
 
+    if (machine->ram_size > 0xc0000000) {
+        error_report("The MPC8544DS board only supports up to 3GB of RAM");
+        exit(1);
+    }
+
     ppce500_init(machine, &params);
 }
 
 
-static QEMUMachine ppce500_machine = {
-    .name = "mpc8544ds",
-    .desc = "mpc8544ds",
-    .init = mpc8544ds_init,
-    .max_cpus = 15,
-};
-
-static void ppce500_machine_init(void)
+static void ppce500_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ppce500_machine);
+    mc->desc = "mpc8544ds";
+    mc->init = mpc8544ds_init;
+    mc->max_cpus = 15;
 }
 
-machine_init(ppce500_machine_init);
+DEFINE_MACHINE("mpc8544ds", ppce500_machine_init)
diff --git a/hw/ppc/ppc.c b/hw/ppc/ppc.c
index bec82cd..8945869 100644
--- a/hw/ppc/ppc.c
+++ b/hw/ppc/ppc.c
@@ -21,6 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/ppc_e500.h"
@@ -30,6 +33,7 @@
 #include "hw/timer/m48t59.h"
 #include "qemu/log.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "hw/loader.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
@@ -51,8 +55,6 @@
 #  define LOG_TB(...) do { } while (0)
 #endif
 
-#define NSEC_PER_SEC    1000000000LL
-
 static void cpu_ppc_tb_stop (CPUPPCState *env);
 static void cpu_ppc_tb_start (CPUPPCState *env);
 
@@ -163,9 +165,9 @@
     }
 }
 
-void ppc6xx_irq_init(CPUPPCState *env)
+void ppc6xx_irq_init(PowerPCCPU *cpu)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+    CPUPPCState *env = &cpu->env;
 
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppc6xx_set_irq, cpu,
                                                   PPC6xx_INPUT_NB);
@@ -250,9 +252,9 @@
     }
 }
 
-void ppc970_irq_init(CPUPPCState *env)
+void ppc970_irq_init(PowerPCCPU *cpu)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+    CPUPPCState *env = &cpu->env;
 
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppc970_set_irq, cpu,
                                                   PPC970_INPUT_NB);
@@ -286,9 +288,9 @@
     }
 }
 
-void ppcPOWER7_irq_init(CPUPPCState *env)
+void ppcPOWER7_irq_init(PowerPCCPU *cpu)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+    CPUPPCState *env = &cpu->env;
 
     env->irq_inputs = (void **)qemu_allocate_irqs(&power7_set_irq, cpu,
                                                   POWER7_INPUT_NB);
@@ -371,9 +373,9 @@
     }
 }
 
-void ppc40x_irq_init(CPUPPCState *env)
+void ppc40x_irq_init(PowerPCCPU *cpu)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+    CPUPPCState *env = &cpu->env;
 
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppc40x_set_irq,
                                                   cpu, PPC40x_INPUT_NB);
@@ -435,9 +437,9 @@
     }
 }
 
-void ppce500_irq_init(CPUPPCState *env)
+void ppce500_irq_init(PowerPCCPU *cpu)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+    CPUPPCState *env = &cpu->env;
 
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppce500_set_irq,
                                                   cpu, PPCE500_INPUT_NB);
@@ -464,7 +466,7 @@
 uint64_t cpu_ppc_get_tb(ppc_tb_t *tb_env, uint64_t vmclk, int64_t tb_offset)
 {
     /* TB time in tb periods */
-    return muldiv64(vmclk, tb_env->tb_freq, get_ticks_per_sec()) + tb_offset;
+    return muldiv64(vmclk, tb_env->tb_freq, NANOSECONDS_PER_SECOND) + tb_offset;
 }
 
 uint64_t cpu_ppc_load_tbl (CPUPPCState *env)
@@ -505,7 +507,9 @@
 static inline void cpu_ppc_store_tb(ppc_tb_t *tb_env, uint64_t vmclk,
                                     int64_t *tb_offsetp, uint64_t value)
 {
-    *tb_offsetp = value - muldiv64(vmclk, tb_env->tb_freq, get_ticks_per_sec());
+    *tb_offsetp = value -
+        muldiv64(vmclk, tb_env->tb_freq, NANOSECONDS_PER_SECOND);
+
     LOG_TB("%s: tb %016" PRIx64 " offset %08" PRIx64 "\n",
                 __func__, value, *tb_offsetp);
 }
@@ -639,11 +643,11 @@
 
     diff = next - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     if (diff >= 0) {
-        decr = muldiv64(diff, tb_env->decr_freq, get_ticks_per_sec());
+        decr = muldiv64(diff, tb_env->decr_freq, NANOSECONDS_PER_SECOND);
     } else if (tb_env->flags & PPC_TIMER_BOOKE) {
         decr = 0;
     }  else {
-        decr = -muldiv64(-diff, tb_env->decr_freq, get_ticks_per_sec());
+        decr = -muldiv64(-diff, tb_env->decr_freq, NANOSECONDS_PER_SECOND);
     }
     LOG_TB("%s: %08" PRIx32 "\n", __func__, decr);
 
@@ -675,7 +679,8 @@
 
     diff = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - tb_env->purr_start;
 
-    return tb_env->purr_load + muldiv64(diff, tb_env->tb_freq, get_ticks_per_sec());
+    return tb_env->purr_load +
+        muldiv64(diff, tb_env->tb_freq, NANOSECONDS_PER_SECOND);
 }
 
 /* When decrementer expires,
@@ -695,9 +700,18 @@
 
 static inline void cpu_ppc_hdecr_excp(PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
+
     /* Raise it */
-    LOG_TB("raise decrementer exception\n");
-    ppc_set_irq(cpu, PPC_INTERRUPT_HDECR, 1);
+    LOG_TB("raise hv decrementer exception\n");
+
+    /* The architecture specifies that we don't deliver HDEC
+     * interrupts in a PM state. Not only they don't cause a
+     * wakeup but they also get effectively discarded.
+     */
+    if (!env->in_pm_state) {
+        ppc_set_irq(cpu, PPC_INTERRUPT_HDECR, 1);
+    }
 }
 
 static inline void cpu_ppc_hdecr_lower(PowerPCCPU *cpu)
@@ -751,7 +765,7 @@
 
     /* Calculate the next timer event */
     now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    next = now + muldiv64(value, get_ticks_per_sec(), tb_env->decr_freq);
+    next = now + muldiv64(value, NANOSECONDS_PER_SECOND, tb_env->decr_freq);
     *nextp = next;
 
     /* Adjust timer */
@@ -836,7 +850,7 @@
 static void timebase_pre_save(void *opaque)
 {
     PPCTimebase *tb = opaque;
-    uint64_t ticks = cpu_get_real_ticks();
+    uint64_t ticks = cpu_get_host_ticks();
     PowerPCCPU *first_ppc_cpu = POWERPC_CPU(first_cpu);
 
     if (!first_ppc_cpu->env.tb_env) {
@@ -844,7 +858,7 @@
         return;
     }
 
-    tb->time_of_the_day_ns = get_clock_realtime();
+    tb->time_of_the_day_ns = qemu_clock_get_ns(QEMU_CLOCK_HOST);
     /*
      * tb_offset is only expected to be changed by migration so
      * there is no need to update it from KVM here
@@ -873,13 +887,14 @@
      * We try to adjust timebase by downtime if host clocks are not
      * too much out of sync (1 second for now).
      */
-    host_ns = get_clock_realtime();
+    host_ns = qemu_clock_get_ns(QEMU_CLOCK_HOST);
     ns_diff = MAX(0, host_ns - tb_remote->time_of_the_day_ns);
-    migration_duration_ns = MIN(NSEC_PER_SEC, ns_diff);
-    migration_duration_tb = muldiv64(migration_duration_ns, freq, NSEC_PER_SEC);
+    migration_duration_ns = MIN(NANOSECONDS_PER_SECOND, ns_diff);
+    migration_duration_tb = muldiv64(freq, migration_duration_ns,
+                                     NANOSECONDS_PER_SECOND);
     guest_tb = tb_remote->guest_timebase + MIN(0, migration_duration_tb);
 
-    tb_off_adj = guest_tb - cpu_get_real_ticks();
+    tb_off_adj = guest_tb - cpu_get_host_ticks();
 
     tb_off = first_ppc_cpu->env.tb_env->tb_offset;
     trace_ppc_tb_adjust(tb_off, tb_off_adj, tb_off_adj - tb_off,
@@ -923,9 +938,7 @@
     }
     /* Create new timer */
     tb_env->decr_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, &cpu_ppc_decr_cb, cpu);
-    if (0) {
-        /* XXX: find a suitable condition to enable the hypervisor decrementer
-         */
+    if (env->has_hv_mode) {
         tb_env->hdecr_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, &cpu_ppc_hdecr_cb,
                                                 cpu);
     } else {
@@ -1011,7 +1024,7 @@
         /* Cannot occur, but makes gcc happy */
         return;
     }
-    next = now + muldiv64(next, get_ticks_per_sec(), tb_env->tb_freq);
+    next = now + muldiv64(next, NANOSECONDS_PER_SECOND, tb_env->tb_freq);
     if (next == now)
         next++;
     timer_mod(ppc40x_timer->fit_timer, next);
@@ -1042,7 +1055,7 @@
                     __func__, ppc40x_timer->pit_reload);
         now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
         next = now + muldiv64(ppc40x_timer->pit_reload,
-                              get_ticks_per_sec(), tb_env->decr_freq);
+                              NANOSECONDS_PER_SECOND, tb_env->decr_freq);
         if (is_excp)
             next += tb_env->decr_next - now;
         if (next == now)
@@ -1107,7 +1120,7 @@
         /* Cannot occur, but makes gcc happy */
         return;
     }
-    next = now + muldiv64(next, get_ticks_per_sec(), tb_env->decr_freq);
+    next = now + muldiv64(next, NANOSECONDS_PER_SECOND, tb_env->decr_freq);
     if (next == now)
         next++;
     LOG_TB("%s: TCR " TARGET_FMT_lx " TSR " TARGET_FMT_lx "\n", __func__,
@@ -1318,167 +1331,6 @@
     }
 }
 
-/*****************************************************************************/
-/* NVRAM helpers */
-static inline uint32_t nvram_read (nvram_t *nvram, uint32_t addr)
-{
-    return (*nvram->read_fn)(nvram->opaque, addr);
-}
-
-static inline void nvram_write (nvram_t *nvram, uint32_t addr, uint32_t val)
-{
-    (*nvram->write_fn)(nvram->opaque, addr, val);
-}
-
-static void NVRAM_set_byte(nvram_t *nvram, uint32_t addr, uint8_t value)
-{
-    nvram_write(nvram, addr, value);
-}
-
-static uint8_t NVRAM_get_byte(nvram_t *nvram, uint32_t addr)
-{
-    return nvram_read(nvram, addr);
-}
-
-static void NVRAM_set_word(nvram_t *nvram, uint32_t addr, uint16_t value)
-{
-    nvram_write(nvram, addr, value >> 8);
-    nvram_write(nvram, addr + 1, value & 0xFF);
-}
-
-static uint16_t NVRAM_get_word(nvram_t *nvram, uint32_t addr)
-{
-    uint16_t tmp;
-
-    tmp = nvram_read(nvram, addr) << 8;
-    tmp |= nvram_read(nvram, addr + 1);
-
-    return tmp;
-}
-
-static void NVRAM_set_lword(nvram_t *nvram, uint32_t addr, uint32_t value)
-{
-    nvram_write(nvram, addr, value >> 24);
-    nvram_write(nvram, addr + 1, (value >> 16) & 0xFF);
-    nvram_write(nvram, addr + 2, (value >> 8) & 0xFF);
-    nvram_write(nvram, addr + 3, value & 0xFF);
-}
-
-uint32_t NVRAM_get_lword (nvram_t *nvram, uint32_t addr)
-{
-    uint32_t tmp;
-
-    tmp = nvram_read(nvram, addr) << 24;
-    tmp |= nvram_read(nvram, addr + 1) << 16;
-    tmp |= nvram_read(nvram, addr + 2) << 8;
-    tmp |= nvram_read(nvram, addr + 3);
-
-    return tmp;
-}
-
-static void NVRAM_set_string(nvram_t *nvram, uint32_t addr, const char *str,
-                             uint32_t max)
-{
-    int i;
-
-    for (i = 0; i < max && str[i] != '\0'; i++) {
-        nvram_write(nvram, addr + i, str[i]);
-    }
-    nvram_write(nvram, addr + i, str[i]);
-    nvram_write(nvram, addr + max - 1, '\0');
-}
-
-int NVRAM_get_string (nvram_t *nvram, uint8_t *dst, uint16_t addr, int max)
-{
-    int i;
-
-    memset(dst, 0, max);
-    for (i = 0; i < max; i++) {
-        dst[i] = NVRAM_get_byte(nvram, addr + i);
-        if (dst[i] == '\0')
-            break;
-    }
-
-    return i;
-}
-
-static uint16_t NVRAM_crc_update (uint16_t prev, uint16_t value)
-{
-    uint16_t tmp;
-    uint16_t pd, pd1, pd2;
-
-    tmp = prev >> 8;
-    pd = prev ^ value;
-    pd1 = pd & 0x000F;
-    pd2 = ((pd >> 4) & 0x000F) ^ pd1;
-    tmp ^= (pd1 << 3) | (pd1 << 8);
-    tmp ^= pd2 | (pd2 << 7) | (pd2 << 12);
-
-    return tmp;
-}
-
-static uint16_t NVRAM_compute_crc (nvram_t *nvram, uint32_t start, uint32_t count)
-{
-    uint32_t i;
-    uint16_t crc = 0xFFFF;
-    int odd;
-
-    odd = count & 1;
-    count &= ~1;
-    for (i = 0; i != count; i++) {
-        crc = NVRAM_crc_update(crc, NVRAM_get_word(nvram, start + i));
-    }
-    if (odd) {
-        crc = NVRAM_crc_update(crc, NVRAM_get_byte(nvram, start + i) << 8);
-    }
-
-    return crc;
-}
-
-#define CMDLINE_ADDR 0x017ff000
-
-int PPC_NVRAM_set_params (nvram_t *nvram, uint16_t NVRAM_size,
-                          const char *arch,
-                          uint32_t RAM_size, int boot_device,
-                          uint32_t kernel_image, uint32_t kernel_size,
-                          const char *cmdline,
-                          uint32_t initrd_image, uint32_t initrd_size,
-                          uint32_t NVRAM_image,
-                          int width, int height, int depth)
-{
-    uint16_t crc;
-
-    /* Set parameters for Open Hack'Ware BIOS */
-    NVRAM_set_string(nvram, 0x00, "QEMU_BIOS", 16);
-    NVRAM_set_lword(nvram,  0x10, 0x00000002); /* structure v2 */
-    NVRAM_set_word(nvram,   0x14, NVRAM_size);
-    NVRAM_set_string(nvram, 0x20, arch, 16);
-    NVRAM_set_lword(nvram,  0x30, RAM_size);
-    NVRAM_set_byte(nvram,   0x34, boot_device);
-    NVRAM_set_lword(nvram,  0x38, kernel_image);
-    NVRAM_set_lword(nvram,  0x3C, kernel_size);
-    if (cmdline) {
-        /* XXX: put the cmdline in NVRAM too ? */
-        pstrcpy_targphys("cmdline", CMDLINE_ADDR, RAM_size - CMDLINE_ADDR, cmdline);
-        NVRAM_set_lword(nvram,  0x40, CMDLINE_ADDR);
-        NVRAM_set_lword(nvram,  0x44, strlen(cmdline));
-    } else {
-        NVRAM_set_lword(nvram,  0x40, 0);
-        NVRAM_set_lword(nvram,  0x44, 0);
-    }
-    NVRAM_set_lword(nvram,  0x48, initrd_image);
-    NVRAM_set_lword(nvram,  0x4C, initrd_size);
-    NVRAM_set_lword(nvram,  0x50, NVRAM_image);
-
-    NVRAM_set_word(nvram,   0x54, width);
-    NVRAM_set_word(nvram,   0x56, height);
-    NVRAM_set_word(nvram,   0x58, depth);
-    crc = NVRAM_compute_crc(nvram, 0x00, 0xF8);
-    NVRAM_set_word(nvram,   0xFC, crc);
-
-    return 0;
-}
-
 /* CPU device-tree ID helpers */
 int ppc_get_vcpu_dt_id(PowerPCCPU *cpu)
 {
@@ -1499,3 +1351,28 @@
 
     return NULL;
 }
+
+void ppc_cpu_parse_features(const char *cpu_model)
+{
+    CPUClass *cc;
+    ObjectClass *oc;
+    const char *typename;
+    gchar **model_pieces;
+
+    model_pieces = g_strsplit(cpu_model, ",", 2);
+    if (!model_pieces[0]) {
+        error_report("Invalid/empty CPU model name");
+        exit(1);
+    }
+
+    oc = cpu_class_by_name(TYPE_POWERPC_CPU, model_pieces[0]);
+    if (oc == NULL) {
+        error_report("Unable to find CPU definition: %s", model_pieces[0]);
+        exit(1);
+    }
+
+    typename = object_class_get_name(oc);
+    cc = CPU_CLASS(oc);
+    cc->parse_features(typename, model_pieces[1], &error_fatal);
+    g_strfreev(model_pieces);
+}
diff --git a/hw/ppc/ppc405.h b/hw/ppc/ppc405.h
index 1c5f04f..c67febc 100644
--- a/hw/ppc/ppc405.h
+++ b/hw/ppc/ppc405.h
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#if !defined(PPC_405_H)
-#define PPC_405_H
+#ifndef PPC405_H
+#define PPC405_H
 
 #include "hw/ppc/ppc4xx.h"
 
@@ -78,4 +78,4 @@
                            uint32_t sysclk, qemu_irq **picp,
                            ram_addr_t *offsetp);
 
-#endif /* !defined(PPC_405_H) */
+#endif /* PPC405_H */
diff --git a/hw/ppc/ppc405_boards.c b/hw/ppc/ppc405_boards.c
index 1dcea77..4b2f07a 100644
--- a/hw/ppc/ppc405_boards.c
+++ b/hw/ppc/ppc405_boards.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "ppc405.h"
@@ -215,7 +219,8 @@
                         33333333, &pic, kernel_filename == NULL ? 0 : 1);
     /* allocate SRAM */
     sram_size = 512 * 1024;
-    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size, &error_abort);
+    memory_region_init_ram(sram, NULL, "ef405ep.sram", sram_size,
+                           &error_fatal);
     vmstate_register_ram_global(sram);
     memory_region_add_subregion(sysmem, 0xFFF00000, sram);
     /* allocate and load BIOS */
@@ -250,7 +255,7 @@
 #endif
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "ef405ep.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
 
         if (bios_name == NULL)
@@ -283,7 +288,7 @@
 #ifdef DEBUG_BOARD_INIT
     printf("%s: register NVRAM\n", __func__);
 #endif
-    m48t59_init(NULL, 0xF0000000, 0, 8192, 8);
+    m48t59_init(NULL, 0xF0000000, 0, 8192, 1968, 8);
     /* Load kernel */
     linux_boot = (kernel_filename != NULL);
     if (linux_boot) {
@@ -368,10 +373,18 @@
 #endif
 }
 
-static QEMUMachine ref405ep_machine = {
-    .name = "ref405ep",
-    .desc = "ref405ep",
-    .init = ref405ep_init,
+static void ref405ep_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ref405ep";
+    mc->init = ref405ep_init;
+}
+
+static const TypeInfo ref405ep_type = {
+    .name = MACHINE_TYPE_NAME("ref405ep"),
+    .parent = TYPE_MACHINE,
+    .class_init = ref405ep_class_init,
 };
 
 /*****************************************************************************/
@@ -399,7 +412,7 @@
     uint8_t reg1;
 };
 
-static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
+static uint64_t taihu_cpld_read(void *opaque, hwaddr addr, unsigned size)
 {
     taihu_cpld_t *cpld;
     uint32_t ret;
@@ -420,8 +433,8 @@
     return ret;
 }
 
-static void taihu_cpld_writeb (void *opaque,
-                               hwaddr addr, uint32_t value)
+static void taihu_cpld_write(void *opaque, hwaddr addr,
+                             uint64_t value, unsigned size)
 {
     taihu_cpld_t *cpld;
 
@@ -438,48 +451,12 @@
     }
 }
 
-static uint32_t taihu_cpld_readw (void *opaque, hwaddr addr)
-{
-    uint32_t ret;
-
-    ret = taihu_cpld_readb(opaque, addr) << 8;
-    ret |= taihu_cpld_readb(opaque, addr + 1);
-
-    return ret;
-}
-
-static void taihu_cpld_writew (void *opaque,
-                               hwaddr addr, uint32_t value)
-{
-    taihu_cpld_writeb(opaque, addr, (value >> 8) & 0xFF);
-    taihu_cpld_writeb(opaque, addr + 1, value & 0xFF);
-}
-
-static uint32_t taihu_cpld_readl (void *opaque, hwaddr addr)
-{
-    uint32_t ret;
-
-    ret = taihu_cpld_readb(opaque, addr) << 24;
-    ret |= taihu_cpld_readb(opaque, addr + 1) << 16;
-    ret |= taihu_cpld_readb(opaque, addr + 2) << 8;
-    ret |= taihu_cpld_readb(opaque, addr + 3);
-
-    return ret;
-}
-
-static void taihu_cpld_writel (void *opaque,
-                               hwaddr addr, uint32_t value)
-{
-    taihu_cpld_writel(opaque, addr, (value >> 24) & 0xFF);
-    taihu_cpld_writel(opaque, addr + 1, (value >> 16) & 0xFF);
-    taihu_cpld_writel(opaque, addr + 2, (value >> 8) & 0xFF);
-    taihu_cpld_writeb(opaque, addr + 3, value & 0xFF);
-}
-
 static const MemoryRegionOps taihu_cpld_ops = {
-    .old_mmio = {
-        .read = { taihu_cpld_readb, taihu_cpld_readw, taihu_cpld_readl, },
-        .write = { taihu_cpld_writeb, taihu_cpld_writew, taihu_cpld_writel, },
+    .read = taihu_cpld_read,
+    .write = taihu_cpld_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
     },
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
@@ -579,7 +556,7 @@
             bios_name = BIOS_FILENAME;
         bios = g_new(MemoryRegion, 1);
         memory_region_init_ram(bios, NULL, "taihu_405ep.bios", BIOS_SIZE,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(bios);
         filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (filename) {
@@ -664,16 +641,24 @@
 #endif
 }
 
-static QEMUMachine taihu_machine = {
-    .name = "taihu",
-    .desc = "taihu",
-    .init = taihu_405ep_init,
+static void taihu_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "taihu";
+    mc->init = taihu_405ep_init;
+}
+
+static const TypeInfo taihu_type = {
+    .name = MACHINE_TYPE_NAME("taihu"),
+    .parent = TYPE_MACHINE,
+    .class_init = taihu_class_init,
 };
 
 static void ppc405_machine_init(void)
 {
-    qemu_register_machine(&ref405ep_machine);
-    qemu_register_machine(&taihu_machine);
+    type_register_static(&ref405ep_type);
+    type_register_static(&taihu_type);
 }
 
-machine_init(ppc405_machine_init);
+type_init(ppc405_machine_init)
diff --git a/hw/ppc/ppc405_uc.c b/hw/ppc/ppc405_uc.c
index c77434a..d6d3fc2 100644
--- a/hw/ppc/ppc405_uc.c
+++ b/hw/ppc/ppc405_uc.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/boards.h"
@@ -975,7 +979,7 @@
     ocm = g_malloc0(sizeof(ppc405_ocm_t));
     /* XXX: Size is 4096 or 0x04000000 */
     memory_region_init_ram(&ocm->isarc_ram, NULL, "ppc405.ocm", 4096,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&ocm->isarc_ram);
     memory_region_init_alias(&ocm->dsarc_ram, NULL, "ppc405.dsarc", &ocm->isarc_ram,
                              0, 4096);
@@ -1352,7 +1356,7 @@
     case 0x00:
         /* Time base counter */
         ret = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + gpt->tb_offset,
-                       gpt->tb_freq, get_ticks_per_sec());
+                       gpt->tb_freq, NANOSECONDS_PER_SECOND);
         break;
     case 0x10:
         /* Output enable */
@@ -1407,7 +1411,7 @@
     switch (addr) {
     case 0x00:
         /* Time base counter */
-        gpt->tb_offset = muldiv64(value, get_ticks_per_sec(), gpt->tb_freq)
+        gpt->tb_offset = muldiv64(value, NANOSECONDS_PER_SECOND, gpt->tb_freq)
             - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
         ppc4xx_gpt_compute_timer(gpt);
         break;
diff --git a/hw/ppc/ppc440_bamboo.c b/hw/ppc/ppc440_bamboo.c
index 778970a..5c535b1 100644
--- a/hw/ppc/ppc440_bamboo.c
+++ b/hw/ppc/ppc440_bamboo.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "net/net.h"
 #include "hw/hw.h"
@@ -159,7 +159,6 @@
 static void bamboo_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -184,10 +183,10 @@
     int i;
 
     /* Setup CPU. */
-    if (cpu_model == NULL) {
-        cpu_model = "440EP";
+    if (machine->cpu_model == NULL) {
+        machine->cpu_model = "440EP";
     }
-    cpu = cpu_ppc_init(cpu_model);
+    cpu = cpu_ppc_init(machine->cpu_model);
     if (cpu == NULL) {
         fprintf(stderr, "Unable to initialize CPU!\n");
         exit(1);
@@ -257,7 +256,8 @@
                               NULL, NULL);
         if (success < 0) {
             success = load_elf(kernel_filename, NULL, NULL, &elf_entry,
-                               &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               &elf_lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
             entry = elf_entry;
             loadaddr = elf_lowaddr;
         }
@@ -289,20 +289,12 @@
             exit(1);
         }
     }
-
-    if (kvm_enabled())
-        kvmppc_init();
 }
 
-static QEMUMachine bamboo_machine = {
-    .name = "bamboo",
-    .desc = "bamboo",
-    .init = bamboo_init,
-};
-
-static void bamboo_machine_init(void)
+static void bamboo_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&bamboo_machine);
+    mc->desc = "bamboo";
+    mc->init = bamboo_init;
 }
 
-machine_init(bamboo_machine_init);
+DEFINE_MACHINE("bamboo", bamboo_machine_init)
diff --git a/hw/ppc/ppc4xx_devs.c b/hw/ppc/ppc4xx_devs.c
index 2f38ff7..e7f413e 100644
--- a/hw/ppc/ppc4xx_devs.c
+++ b/hw/ppc/ppc4xx_devs.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/ppc4xx.h"
diff --git a/hw/ppc/ppc4xx_pci.c b/hw/ppc/ppc4xx_pci.c
index 0bb3cdb..683218e 100644
--- a/hw/ppc/ppc4xx_pci.c
+++ b/hw/ppc/ppc4xx_pci.c
@@ -19,6 +19,7 @@
 /* This file implements emulation of the 32-bit PCI controller found in some
  * 4xx SoCs, such as the 440EP. */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "hw/ppc/ppc4xx.h"
diff --git a/hw/ppc/ppc_booke.c b/hw/ppc/ppc_booke.c
index 8b94da6..ab8d026 100644
--- a/hw/ppc/ppc_booke.c
+++ b/hw/ppc/ppc_booke.c
@@ -21,6 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/ppc/ppc.h"
 #include "qemu/timer.h"
@@ -162,7 +165,7 @@
         ticks += delta_tick;
     }
 
-    *next = now + muldiv64(ticks, get_ticks_per_sec(), tb_env->tb_freq);
+    *next = now + muldiv64(ticks, NANOSECONDS_PER_SECOND, tb_env->tb_freq);
     if ((*next < now) || (*next > INT64_MAX)) {
         /* Overflow, so assume the biggest number the qemu timer supports. */
         *next = INT64_MAX;
diff --git a/hw/ppc/ppce500_spin.c b/hw/ppc/ppce500_spin.c
index d49f2b8..22c584e 100644
--- a/hw/ppc/ppce500_spin.c
+++ b/hw/ppc/ppce500_spin.c
@@ -27,10 +27,12 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "sysemu/kvm.h"
+#include "e500.h"
 
 #define MAX_CPUS 32
 
@@ -71,12 +73,6 @@
     }
 }
 
-/* Create -kernel TLB entries for BookE, linearly spanning 256MB.  */
-static inline hwaddr booke206_page_size_to_tlb(uint64_t size)
-{
-    return (ffs(size >> 10) - 1) >> 1;
-}
-
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
                                      hwaddr pa,
@@ -103,7 +99,7 @@
     hwaddr map_start;
 
     cpu_synchronize_state(cpu);
-    stl_p(&curspin->pir, env->spr[SPR_PIR]);
+    stl_p(&curspin->pir, env->spr[SPR_BOOKE_PIR]);
     env->nip = ldq_p(&curspin->addr) & (map_size - 1);
     env->gpr[3] = ldq_p(&curspin->r3);
     env->gpr[4] = 0;
diff --git a/hw/ppc/prep.c b/hw/ppc/prep.c
index dd8433d..054af1e 100644
--- a/hw/ppc/prep.c
+++ b/hw/ppc/prep.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/timer/m48t59.h"
 #include "hw/i386/pc.h"
@@ -33,6 +35,7 @@
 #include "hw/pci/pci_host.h"
 #include "hw/ppc/ppc.h"
 #include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "qemu/log.h"
 #include "hw/ide.h"
 #include "hw/loader.h"
@@ -42,10 +45,9 @@
 #include "sysemu/arch_init.h"
 #include "sysemu/qtest.h"
 #include "exec/address-spaces.h"
+#include "trace.h"
 #include "elf.h"
-
-//#define HARD_DEBUG_PPC_IO
-//#define DEBUG_PPC_IO
+#include "qemu/cutils.h"
 
 /* SMP is not enabled, for now */
 #define MAX_CPUS 1
@@ -57,26 +59,6 @@
 #define KERNEL_LOAD_ADDR 0x01000000
 #define INITRD_LOAD_ADDR 0x01800000
 
-#if defined (HARD_DEBUG_PPC_IO) && !defined (DEBUG_PPC_IO)
-#define DEBUG_PPC_IO
-#endif
-
-#if defined (HARD_DEBUG_PPC_IO)
-#define PPC_IO_DPRINTF(fmt, ...)                         \
-do {                                                     \
-    if (qemu_loglevel_mask(CPU_LOG_IOPORT)) {            \
-        qemu_log("%s: " fmt, __func__ , ## __VA_ARGS__); \
-    } else {                                             \
-        printf("%s : " fmt, __func__ , ## __VA_ARGS__);  \
-    }                                                    \
-} while (0)
-#elif defined (DEBUG_PPC_IO)
-#define PPC_IO_DPRINTF(fmt, ...) \
-qemu_log_mask(CPU_LOG_IOPORT, fmt, ## __VA_ARGS__)
-#else
-#define PPC_IO_DPRINTF(fmt, ...) do { } while (0)
-#endif
-
 /* Constants for devices init */
 static const int ide_iobase[2] = { 0x1f0, 0x170 };
 static const int ide_iobase2[2] = { 0x3f6, 0x376 };
@@ -181,7 +163,7 @@
 /* Fake super-io ports for PREP platform (Intel 82378ZB) */
 typedef struct sysctrl_t {
     qemu_irq reset_irq;
-    M48t59State *nvram;
+    Nvram *nvram;
     uint8_t state;
     uint8_t syscontrol;
     int contiguous_map;
@@ -199,8 +181,7 @@
 {
     sysctrl_t *sysctrl = opaque;
 
-    PPC_IO_DPRINTF("0x%08" PRIx32 " => 0x%02" PRIx32 "\n",
-                   addr - PPC_IO_BASE, val);
+    trace_prep_io_800_writeb(addr - PPC_IO_BASE, val);
     switch (addr) {
     case 0x0092:
         /* Special port 92 */
@@ -235,13 +216,17 @@
         break;
     case 0x0810:
         /* Password protect 1 register */
-        if (sysctrl->nvram != NULL)
-            m48t59_toggle_lock(sysctrl->nvram, 1);
+        if (sysctrl->nvram != NULL) {
+            NvramClass *k = NVRAM_GET_CLASS(sysctrl->nvram);
+            (k->toggle_lock)(sysctrl->nvram, 1);
+        }
         break;
     case 0x0812:
         /* Password protect 2 register */
-        if (sysctrl->nvram != NULL)
-            m48t59_toggle_lock(sysctrl->nvram, 2);
+        if (sysctrl->nvram != NULL) {
+            NvramClass *k = NVRAM_GET_CLASS(sysctrl->nvram);
+            (k->toggle_lock)(sysctrl->nvram, 2);
+        }
         break;
     case 0x0814:
         /* L2 invalidate register */
@@ -323,8 +308,7 @@
         printf("ERROR: unaffected IO port: %04" PRIx32 " read\n", addr);
         break;
     }
-    PPC_IO_DPRINTF("0x%08" PRIx32 " <= 0x%02" PRIx32 "\n",
-                   addr - PPC_IO_BASE, retval);
+    trace_prep_io_800_readb(addr - PPC_IO_BASE, retval);
 
     return retval;
 }
@@ -332,15 +316,6 @@
 
 #define NVRAM_SIZE        0x2000
 
-static void cpu_request_exit(void *opaque, int irq, int level)
-{
-    CPUState *cpu = current_cpu;
-
-    if (cpu && level) {
-        cpu_exit(cpu);
-    }
-}
-
 static void ppc_prep_reset(void *opaque)
 {
     PowerPCCPU *cpu = opaque;
@@ -360,11 +335,148 @@
 
 static PortioList prep_port_list;
 
+/*****************************************************************************/
+/* NVRAM helpers */
+static inline uint32_t nvram_read(Nvram *nvram, uint32_t addr)
+{
+    NvramClass *k = NVRAM_GET_CLASS(sysctrl->nvram);
+    return (k->read)(nvram, addr);
+}
+
+static inline void nvram_write(Nvram *nvram, uint32_t addr, uint32_t val)
+{
+    NvramClass *k = NVRAM_GET_CLASS(sysctrl->nvram);
+    (k->write)(nvram, addr, val);
+}
+
+static void NVRAM_set_byte(Nvram *nvram, uint32_t addr, uint8_t value)
+{
+    nvram_write(nvram, addr, value);
+}
+
+static uint8_t NVRAM_get_byte(Nvram *nvram, uint32_t addr)
+{
+    return nvram_read(nvram, addr);
+}
+
+static void NVRAM_set_word(Nvram *nvram, uint32_t addr, uint16_t value)
+{
+    nvram_write(nvram, addr, value >> 8);
+    nvram_write(nvram, addr + 1, value & 0xFF);
+}
+
+static uint16_t NVRAM_get_word(Nvram *nvram, uint32_t addr)
+{
+    uint16_t tmp;
+
+    tmp = nvram_read(nvram, addr) << 8;
+    tmp |= nvram_read(nvram, addr + 1);
+
+    return tmp;
+}
+
+static void NVRAM_set_lword(Nvram *nvram, uint32_t addr, uint32_t value)
+{
+    nvram_write(nvram, addr, value >> 24);
+    nvram_write(nvram, addr + 1, (value >> 16) & 0xFF);
+    nvram_write(nvram, addr + 2, (value >> 8) & 0xFF);
+    nvram_write(nvram, addr + 3, value & 0xFF);
+}
+
+static void NVRAM_set_string(Nvram *nvram, uint32_t addr, const char *str,
+                             uint32_t max)
+{
+    int i;
+
+    for (i = 0; i < max && str[i] != '\0'; i++) {
+        nvram_write(nvram, addr + i, str[i]);
+    }
+    nvram_write(nvram, addr + i, str[i]);
+    nvram_write(nvram, addr + max - 1, '\0');
+}
+
+static uint16_t NVRAM_crc_update (uint16_t prev, uint16_t value)
+{
+    uint16_t tmp;
+    uint16_t pd, pd1, pd2;
+
+    tmp = prev >> 8;
+    pd = prev ^ value;
+    pd1 = pd & 0x000F;
+    pd2 = ((pd >> 4) & 0x000F) ^ pd1;
+    tmp ^= (pd1 << 3) | (pd1 << 8);
+    tmp ^= pd2 | (pd2 << 7) | (pd2 << 12);
+
+    return tmp;
+}
+
+static uint16_t NVRAM_compute_crc (Nvram *nvram, uint32_t start, uint32_t count)
+{
+    uint32_t i;
+    uint16_t crc = 0xFFFF;
+    int odd;
+
+    odd = count & 1;
+    count &= ~1;
+    for (i = 0; i != count; i++) {
+        crc = NVRAM_crc_update(crc, NVRAM_get_word(nvram, start + i));
+    }
+    if (odd) {
+        crc = NVRAM_crc_update(crc, NVRAM_get_byte(nvram, start + i) << 8);
+    }
+
+    return crc;
+}
+
+#define CMDLINE_ADDR 0x017ff000
+
+static int PPC_NVRAM_set_params (Nvram *nvram, uint16_t NVRAM_size,
+                          const char *arch,
+                          uint32_t RAM_size, int boot_device,
+                          uint32_t kernel_image, uint32_t kernel_size,
+                          const char *cmdline,
+                          uint32_t initrd_image, uint32_t initrd_size,
+                          uint32_t NVRAM_image,
+                          int width, int height, int depth)
+{
+    uint16_t crc;
+
+    /* Set parameters for Open Hack'Ware BIOS */
+    NVRAM_set_string(nvram, 0x00, "QEMU_BIOS", 16);
+    NVRAM_set_lword(nvram,  0x10, 0x00000002); /* structure v2 */
+    NVRAM_set_word(nvram,   0x14, NVRAM_size);
+    NVRAM_set_string(nvram, 0x20, arch, 16);
+    NVRAM_set_lword(nvram,  0x30, RAM_size);
+    NVRAM_set_byte(nvram,   0x34, boot_device);
+    NVRAM_set_lword(nvram,  0x38, kernel_image);
+    NVRAM_set_lword(nvram,  0x3C, kernel_size);
+    if (cmdline) {
+        /* XXX: put the cmdline in NVRAM too ? */
+        pstrcpy_targphys("cmdline", CMDLINE_ADDR, RAM_size - CMDLINE_ADDR,
+                         cmdline);
+        NVRAM_set_lword(nvram,  0x40, CMDLINE_ADDR);
+        NVRAM_set_lword(nvram,  0x44, strlen(cmdline));
+    } else {
+        NVRAM_set_lword(nvram,  0x40, 0);
+        NVRAM_set_lword(nvram,  0x44, 0);
+    }
+    NVRAM_set_lword(nvram,  0x48, initrd_image);
+    NVRAM_set_lword(nvram,  0x4C, initrd_size);
+    NVRAM_set_lword(nvram,  0x50, NVRAM_image);
+
+    NVRAM_set_word(nvram,   0x54, width);
+    NVRAM_set_word(nvram,   0x56, height);
+    NVRAM_set_word(nvram,   0x58, depth);
+    crc = NVRAM_compute_crc(nvram, 0x00, 0xF8);
+    NVRAM_set_word(nvram,   0xFC, crc);
+
+    return 0;
+}
+
 /* PowerPC PREP hardware initialisation */
 static void ppc_prep_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
@@ -372,8 +484,7 @@
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
-    nvram_t nvram;
-    M48t59State *m48t59;
+    Nvram *m48t59;
 #if 0
     MemoryRegion *xcsr = g_new(MemoryRegion, 1);
 #endif
@@ -387,7 +498,6 @@
     PCIDevice *pci;
     ISABus *isa_bus;
     ISADevice *isa;
-    qemu_irq *cpu_exit_irq;
     int ppc_boot_device;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
 
@@ -396,10 +506,10 @@
     linux_boot = (kernel_filename != NULL);
 
     /* init CPUs */
-    if (cpu_model == NULL)
-        cpu_model = "602";
+    if (machine->cpu_model == NULL)
+        machine->cpu_model = "602";
     for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_ppc_init(cpu_model);
+        cpu = cpu_ppc_init(machine->cpu_model);
         if (cpu == NULL) {
             fprintf(stderr, "Unable to find PowerPC CPU definition\n");
             exit(1);
@@ -426,7 +536,7 @@
         kernel_size = load_image_targphys(kernel_filename, kernel_base,
                                           ram_size - kernel_base);
         if (kernel_size < 0) {
-            hw_error("qemu: could not load kernel '%s'\n", kernel_filename);
+            error_report("could not load kernel '%s'", kernel_filename);
             exit(1);
         }
         /* load initrd */
@@ -435,8 +545,9 @@
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               ram_size - initrd_base);
             if (initrd_size < 0) {
-                hw_error("qemu: could not load initial ram disk '%s'\n",
-                          initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
+                exit(1);
             }
         } else {
             initrd_base = 0;
@@ -463,7 +574,8 @@
     }
 
     if (PPC_INPUT(env) != PPC_FLAGS_INPUT_6xx) {
-        hw_error("Only 6xx bus is supported on PREP machine\n");
+        error_report("Only 6xx bus is supported on PREP machine");
+        exit(1);
     }
 
     dev = qdev_create(NULL, "raven-pcihost");
@@ -471,7 +583,7 @@
         bios_name = BIOS_FILENAME;
     }
     qdev_prop_set_string(dev, "bios-name", bios_name);
-    qdev_prop_set_uint32(dev, "elf-machine", ELF_MACHINE);
+    qdev_prop_set_uint32(dev, "elf-machine", PPC_ELF_MACHINE);
     pcihost = PCI_HOST_BRIDGE(dev);
     object_property_add_child(qdev_get_machine(), "raven", OBJECT(dev), NULL);
     qdev_init_nofail(dev);
@@ -484,11 +596,9 @@
 
     /* PCI -> ISA bridge */
     pci = pci_create_simple(pci_bus, PCI_DEVFN(1, 0), "i82378");
-    cpu_exit_irq = qemu_allocate_irqs(cpu_request_exit, NULL, 1);
     cpu = POWERPC_CPU(first_cpu);
     qdev_connect_gpio_out(&pci->qdev, 0,
                           cpu->env.irq_inputs[PPC6xx_INPUT_INT]);
-    qdev_connect_gpio_out(&pci->qdev, 1, *cpu_exit_irq);
     sysbus_connect_irq(&pcihost->busdev, 0, qdev_get_gpio_in(&pci->qdev, 9));
     sysbus_connect_irq(&pcihost->busdev, 1, qdev_get_gpio_in(&pci->qdev, 11));
     sysbus_connect_irq(&pcihost->busdev, 2, qdev_get_gpio_in(&pci->qdev, 9));
@@ -539,20 +649,18 @@
     memory_region_add_subregion(sysmem, 0xFEFF0000, xcsr);
 #endif
 
-    if (usb_enabled(false)) {
+    if (machine_usb(machine)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
 
-    m48t59 = m48t59_init_isa(isa_bus, 0x0074, NVRAM_SIZE, 59);
+    m48t59 = m48t59_init_isa(isa_bus, 0x0074, NVRAM_SIZE, 2000, 59);
     if (m48t59 == NULL)
         return;
     sysctrl->nvram = m48t59;
 
     /* Initialise NVRAM */
-    nvram.opaque = m48t59;
-    nvram.read_fn = &m48t59_read;
-    nvram.write_fn = &m48t59_write;
-    PPC_NVRAM_set_params(&nvram, NVRAM_SIZE, "PREP", ram_size, ppc_boot_device,
+    PPC_NVRAM_set_params(m48t59, NVRAM_SIZE, "PREP", ram_size,
+                         ppc_boot_device,
                          kernel_base, kernel_size,
                          kernel_cmdline,
                          initrd_base, initrd_size,
@@ -561,17 +669,12 @@
                          graphic_width, graphic_height, graphic_depth);
 }
 
-static QEMUMachine prep_machine = {
-    .name = "prep",
-    .desc = "PowerPC PREP platform",
-    .init = ppc_prep_init,
-    .max_cpus = MAX_CPUS,
-    .default_boot_order = "cad",
-};
-
-static void prep_machine_init(void)
+static void prep_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&prep_machine);
+    mc->desc = "PowerPC PREP platform";
+    mc->init = ppc_prep_init;
+    mc->max_cpus = MAX_CPUS;
+    mc->default_boot_order = "cad";
 }
 
-machine_init(prep_machine_init);
+DEFINE_MACHINE("prep", prep_machine_init)
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index 30de25d..30d6800 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -24,15 +24,22 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/numa.h"
 #include "hw/hw.h"
+#include "qemu/log.h"
 #include "hw/fw-path-provider.h"
 #include "elf.h"
 #include "net/net.h"
+#include "sysemu/device_tree.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
+#include "sysemu/device_tree.h"
 #include "kvm_ppc.h"
+#include "migration/migration.h"
 #include "mmu-hash64.h"
 #include "qom/cpu.h"
 
@@ -58,6 +65,9 @@
 #include "hw/nmi.h"
 
 #include "hw/compat.h"
+#include "qemu/cutils.h"
+#include "hw/ppc/spapr_cpu_core.h"
+#include "qmp-commands.h"
 
 #include <libfdt.h>
 
@@ -71,7 +81,7 @@
  *
  * We load our kernel at 4M, leaving space for SLOF initial image
  */
-#define FDT_MAX_SIZE            0x40000
+#define FDT_MAX_SIZE            0x100000
 #define RTAS_MAX_SIZE           0x10000
 #define RTAS_MAX_ADDR           0x80000000 /* RTAS must stay below that */
 #define FW_MAX_SIZE             0x400000
@@ -81,78 +91,53 @@
 
 #define MIN_RMA_SLOF            128UL
 
-#define TIMEBASE_FREQ           512000000ULL
-
-#define MAX_CPUS                255
-
 #define PHANDLE_XICP            0x00001111
 
 #define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
 
-typedef struct sPAPRMachineState sPAPRMachineState;
-
-#define TYPE_SPAPR_MACHINE      "spapr-machine"
-#define SPAPR_MACHINE(obj) \
-    OBJECT_CHECK(sPAPRMachineState, (obj), TYPE_SPAPR_MACHINE)
-
-/**
- * sPAPRMachineState:
- */
-struct sPAPRMachineState {
-    /*< private >*/
-    MachineState parent_obj;
-
-    /*< public >*/
-    char *kvm_type;
-};
-
-sPAPREnvironment *spapr;
-
 static XICSState *try_create_xics(const char *type, int nr_servers,
-                                  int nr_irqs)
+                                  int nr_irqs, Error **errp)
 {
+    Error *err = NULL;
     DeviceState *dev;
 
     dev = qdev_create(NULL, type);
     qdev_prop_set_uint32(dev, "nr_servers", nr_servers);
     qdev_prop_set_uint32(dev, "nr_irqs", nr_irqs);
-    if (qdev_init(dev) < 0) {
+    object_property_set_bool(OBJECT(dev), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        object_unparent(OBJECT(dev));
         return NULL;
     }
-
     return XICS_COMMON(dev);
 }
 
-static XICSState *xics_system_init(int nr_servers, int nr_irqs)
+static XICSState *xics_system_init(MachineState *machine,
+                                   int nr_servers, int nr_irqs, Error **errp)
 {
-    XICSState *icp = NULL;
+    XICSState *xics = NULL;
 
     if (kvm_enabled()) {
-        QemuOpts *machine_opts = qemu_get_machine_opts();
-        bool irqchip_allowed = qemu_opt_get_bool(machine_opts,
-                                                "kernel_irqchip", true);
-        bool irqchip_required = qemu_opt_get_bool(machine_opts,
-                                                  "kernel_irqchip", false);
-        if (irqchip_allowed) {
-            icp = try_create_xics(TYPE_KVM_XICS, nr_servers, nr_irqs);
-        }
+        Error *err = NULL;
 
-        if (irqchip_required && !icp) {
-            perror("Failed to create in-kernel XICS\n");
-            abort();
+        if (machine_kernel_irqchip_allowed(machine)) {
+            xics = try_create_xics(TYPE_XICS_SPAPR_KVM, nr_servers, nr_irqs,
+                                   &err);
+        }
+        if (machine_kernel_irqchip_required(machine) && !xics) {
+            error_reportf_err(err,
+                              "kernel_irqchip requested but unavailable: ");
+        } else {
+            error_free(err);
         }
     }
 
-    if (!icp) {
-        icp = try_create_xics(TYPE_XICS, nr_servers, nr_irqs);
+    if (!xics) {
+        xics = try_create_xics(TYPE_XICS_SPAPR, nr_servers, nr_irqs, errp);
     }
 
-    if (!icp) {
-        perror("Failed to create XICS\n");
-        abort();
-    }
-
-    return icp;
+    return xics;
 }
 
 static int spapr_fixup_cpu_smt_dt(void *fdt, int offset, PowerPCCPU *cpu,
@@ -188,7 +173,28 @@
     return ret;
 }
 
-static int spapr_fixup_cpu_dt(void *fdt, sPAPREnvironment *spapr)
+static int spapr_fixup_cpu_numa_dt(void *fdt, int offset, CPUState *cs)
+{
+    int ret = 0;
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    int index = ppc_get_vcpu_dt_id(cpu);
+    uint32_t associativity[] = {cpu_to_be32(0x5),
+                                cpu_to_be32(0x0),
+                                cpu_to_be32(0x0),
+                                cpu_to_be32(0x0),
+                                cpu_to_be32(cs->numa_node),
+                                cpu_to_be32(index)};
+
+    /* Advertise NUMA via ibm,associativity */
+    if (nb_numa_nodes > 1) {
+        ret = fdt_setprop(fdt, offset, "ibm,associativity", associativity,
+                          sizeof(associativity));
+    }
+
+    return ret;
+}
+
+static int spapr_fixup_cpu_dt(void *fdt, sPAPRMachineState *spapr)
 {
     int ret = 0, offset, cpus_offset;
     CPUState *cs;
@@ -200,12 +206,6 @@
         PowerPCCPU *cpu = POWERPC_CPU(cs);
         DeviceClass *dc = DEVICE_GET_CLASS(cs);
         int index = ppc_get_vcpu_dt_id(cpu);
-        uint32_t associativity[] = {cpu_to_be32(0x5),
-                                    cpu_to_be32(0x0),
-                                    cpu_to_be32(0x0),
-                                    cpu_to_be32(0x0),
-                                    cpu_to_be32(cs->numa_node),
-                                    cpu_to_be32(index)};
 
         if ((index % smt) != 0) {
             continue;
@@ -229,20 +229,17 @@
             }
         }
 
-        if (nb_numa_nodes > 1) {
-            ret = fdt_setprop(fdt, offset, "ibm,associativity", associativity,
-                              sizeof(associativity));
-            if (ret < 0) {
-                return ret;
-            }
-        }
-
         ret = fdt_setprop(fdt, offset, "ibm,pft-size",
                           pft_size_prop, sizeof(pft_size_prop));
         if (ret < 0) {
             return ret;
         }
 
+        ret = spapr_fixup_cpu_numa_dt(fdt, offset, cs);
+        if (ret < 0) {
+            return ret;
+        }
+
         ret = spapr_fixup_cpu_smt_dt(fdt, offset, cpu,
                                      ppc_get_compat_smt_threads(cpu));
         if (ret < 0) {
@@ -288,15 +285,18 @@
 
 static hwaddr spapr_node0_size(void)
 {
+    MachineState *machine = MACHINE(qdev_get_machine());
+
     if (nb_numa_nodes) {
         int i;
         for (i = 0; i < nb_numa_nodes; ++i) {
             if (numa_info[i].node_mem) {
-                return MIN(pow2floor(numa_info[i].node_mem), ram_size);
+                return MIN(pow2floor(numa_info[i].node_mem),
+                           machine->ram_size);
             }
         }
     }
-    return ram_size;
+    return machine->ram_size;
 }
 
 #define _FDT(exp) \
@@ -318,23 +318,17 @@
                                    hwaddr initrd_size,
                                    hwaddr kernel_size,
                                    bool little_endian,
-                                   const char *boot_device,
                                    const char *kernel_cmdline,
                                    uint32_t epow_irq)
 {
     void *fdt;
-    CPUState *cs;
     uint32_t start_prop = cpu_to_be32(initrd_base);
     uint32_t end_prop = cpu_to_be32(initrd_base + initrd_size);
     GString *hypertas = g_string_sized_new(256);
     GString *qemu_hypertas = g_string_sized_new(256);
     uint32_t refpoints[] = {cpu_to_be32(0x4), cpu_to_be32(0x4)};
-    uint32_t interrupt_server_ranges_prop[] = {0, cpu_to_be32(smp_cpus)};
-    int smt = kvmppc_smt_threads();
+    uint32_t interrupt_server_ranges_prop[] = {0, cpu_to_be32(max_cpus)};
     unsigned char vec5[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x80};
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("smp-opts"), NULL);
-    unsigned sockets = opts ? qemu_opt_get_number(opts, "sockets", 0) : 0;
-    uint32_t cpus_per_socket = sockets ? (smp_cpus / sockets) : 1;
     char *buf;
 
     add_str(hypertas, "hcall-pft");
@@ -346,6 +340,9 @@
     add_str(hypertas, "hcall-splpar");
     add_str(hypertas, "hcall-bulk");
     add_str(hypertas, "hcall-set-mode");
+    add_str(hypertas, "hcall-sprg0");
+    add_str(hypertas, "hcall-copy");
+    add_str(hypertas, "hcall-debug");
     add_str(qemu_hypertas, "hcall-memop1");
 
     fdt = g_malloc0(FDT_MAX_SIZE);
@@ -386,8 +383,16 @@
                           qemu_uuid[14], qemu_uuid[15]);
 
     _FDT((fdt_property_string(fdt, "vm,uuid", buf)));
+    if (qemu_uuid_set) {
+        _FDT((fdt_property_string(fdt, "system-id", buf)));
+    }
     g_free(buf);
 
+    if (qemu_get_vm_name()) {
+        _FDT((fdt_property_string(fdt, "ibm,partition-name",
+                                  qemu_get_vm_name())));
+    }
+
     _FDT((fdt_property_cell(fdt, "#address-cells", 0x2)));
     _FDT((fdt_property_cell(fdt, "#size-cells", 0x2)));
 
@@ -411,9 +416,6 @@
             _FDT((fdt_property(fdt, "qemu,boot-kernel-le", NULL, 0)));
         }
     }
-    if (boot_device) {
-        _FDT((fdt_property_string(fdt, "qemu,boot-device", boot_device)));
-    }
     if (boot_menu) {
         _FDT((fdt_property_cell(fdt, "qemu,boot-menu", boot_menu)));
     }
@@ -423,107 +425,6 @@
 
     _FDT((fdt_end_node(fdt)));
 
-    /* cpus */
-    _FDT((fdt_begin_node(fdt, "cpus")));
-
-    _FDT((fdt_property_cell(fdt, "#address-cells", 0x1)));
-    _FDT((fdt_property_cell(fdt, "#size-cells", 0x0)));
-
-    CPU_FOREACH(cs) {
-        PowerPCCPU *cpu = POWERPC_CPU(cs);
-        CPUPPCState *env = &cpu->env;
-        DeviceClass *dc = DEVICE_GET_CLASS(cs);
-        PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cs);
-        int index = ppc_get_vcpu_dt_id(cpu);
-        char *nodename;
-        uint32_t segs[] = {cpu_to_be32(28), cpu_to_be32(40),
-                           0xffffffff, 0xffffffff};
-        uint32_t tbfreq = kvm_enabled() ? kvmppc_get_tbfreq() : TIMEBASE_FREQ;
-        uint32_t cpufreq = kvm_enabled() ? kvmppc_get_clockfreq() : 1000000000;
-        uint32_t page_sizes_prop[64];
-        size_t page_sizes_prop_size;
-
-        if ((index % smt) != 0) {
-            continue;
-        }
-
-        nodename = g_strdup_printf("%s@%x", dc->fw_name, index);
-
-        _FDT((fdt_begin_node(fdt, nodename)));
-
-        g_free(nodename);
-
-        _FDT((fdt_property_cell(fdt, "reg", index)));
-        _FDT((fdt_property_string(fdt, "device_type", "cpu")));
-
-        _FDT((fdt_property_cell(fdt, "cpu-version", env->spr[SPR_PVR])));
-        _FDT((fdt_property_cell(fdt, "d-cache-block-size",
-                                env->dcache_line_size)));
-        _FDT((fdt_property_cell(fdt, "d-cache-line-size",
-                                env->dcache_line_size)));
-        _FDT((fdt_property_cell(fdt, "i-cache-block-size",
-                                env->icache_line_size)));
-        _FDT((fdt_property_cell(fdt, "i-cache-line-size",
-                                env->icache_line_size)));
-
-        if (pcc->l1_dcache_size) {
-            _FDT((fdt_property_cell(fdt, "d-cache-size", pcc->l1_dcache_size)));
-        } else {
-            fprintf(stderr, "Warning: Unknown L1 dcache size for cpu\n");
-        }
-        if (pcc->l1_icache_size) {
-            _FDT((fdt_property_cell(fdt, "i-cache-size", pcc->l1_icache_size)));
-        } else {
-            fprintf(stderr, "Warning: Unknown L1 icache size for cpu\n");
-        }
-
-        _FDT((fdt_property_cell(fdt, "timebase-frequency", tbfreq)));
-        _FDT((fdt_property_cell(fdt, "clock-frequency", cpufreq)));
-        _FDT((fdt_property_cell(fdt, "ibm,slb-size", env->slb_nr)));
-        _FDT((fdt_property_string(fdt, "status", "okay")));
-        _FDT((fdt_property(fdt, "64-bit", NULL, 0)));
-
-        if (env->spr_cb[SPR_PURR].oea_read) {
-            _FDT((fdt_property(fdt, "ibm,purr", NULL, 0)));
-        }
-
-        if (env->mmu_model & POWERPC_MMU_1TSEG) {
-            _FDT((fdt_property(fdt, "ibm,processor-segment-sizes",
-                               segs, sizeof(segs))));
-        }
-
-        /* Advertise VMX/VSX (vector extensions) if available
-         *   0 / no property == no vector extensions
-         *   1               == VMX / Altivec available
-         *   2               == VSX available */
-        if (env->insns_flags & PPC_ALTIVEC) {
-            uint32_t vmx = (env->insns_flags2 & PPC2_VSX) ? 2 : 1;
-
-            _FDT((fdt_property_cell(fdt, "ibm,vmx", vmx)));
-        }
-
-        /* Advertise DFP (Decimal Floating Point) if available
-         *   0 / no property == no DFP
-         *   1               == DFP available */
-        if (env->insns_flags2 & PPC2_DFP) {
-            _FDT((fdt_property_cell(fdt, "ibm,dfp", 1)));
-        }
-
-        page_sizes_prop_size = create_page_sizes_prop(env, page_sizes_prop,
-                                                      sizeof(page_sizes_prop));
-        if (page_sizes_prop_size) {
-            _FDT((fdt_property(fdt, "ibm,segment-page-sizes",
-                               page_sizes_prop, page_sizes_prop_size)));
-        }
-
-        _FDT((fdt_property_cell(fdt, "ibm,chip-id",
-                                cs->cpu_index / cpus_per_socket)));
-
-        _FDT((fdt_end_node(fdt)));
-    }
-
-    _FDT((fdt_end_node(fdt)));
-
     /* RTAS */
     _FDT((fdt_begin_node(fdt, "rtas")));
 
@@ -541,6 +442,12 @@
         refpoints, sizeof(refpoints))));
 
     _FDT((fdt_property_cell(fdt, "rtas-error-log-max", RTAS_ERROR_LOG_MAX)));
+    _FDT((fdt_property_cell(fdt, "rtas-event-scan-rate",
+                            RTAS_EVENT_SCAN_RATE)));
+
+    if (msi_nonbroken) {
+        _FDT((fdt_property(fdt, "ibm,change-msix-capable", NULL, 0)));
+    }
 
     /*
      * According to PAPR, rtas ibm,os-term does not guarantee a return
@@ -596,10 +503,11 @@
              * Older KVM versions with older guest kernels were broken with the
              * magic page, don't allow the guest to map it.
              */
-            kvmppc_get_hypercall(first_cpu->env_ptr, hypercall,
-                                 sizeof(hypercall));
-            _FDT((fdt_property(fdt, "hcall-instructions", hypercall,
-                              sizeof(hypercall))));
+            if (!kvmppc_get_hypercall(first_cpu->env_ptr, hypercall,
+                                      sizeof(hypercall))) {
+                _FDT((fdt_property(fdt, "hcall-instructions", hypercall,
+                                   sizeof(hypercall))));
+            }
         }
         _FDT((fdt_end_node(fdt)));
     }
@@ -610,43 +518,7 @@
     return fdt;
 }
 
-int spapr_h_cas_compose_response(target_ulong addr, target_ulong size)
-{
-    void *fdt, *fdt_skel;
-    sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
-
-    size -= sizeof(hdr);
-
-    /* Create sceleton */
-    fdt_skel = g_malloc0(size);
-    _FDT((fdt_create(fdt_skel, size)));
-    _FDT((fdt_begin_node(fdt_skel, "")));
-    _FDT((fdt_end_node(fdt_skel)));
-    _FDT((fdt_finish(fdt_skel)));
-    fdt = g_malloc0(size);
-    _FDT((fdt_open_into(fdt_skel, fdt, size)));
-    g_free(fdt_skel);
-
-    /* Fix skeleton up */
-    _FDT((spapr_fixup_cpu_dt(fdt, spapr)));
-
-    /* Pack resulting tree */
-    _FDT((fdt_pack(fdt)));
-
-    if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
-        trace_spapr_cas_failed(size);
-        return -1;
-    }
-
-    cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
-    cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
-    trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
-    g_free(fdt);
-
-    return 0;
-}
-
-static void spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
+static int spapr_populate_memory_node(void *fdt, int nodeid, hwaddr start,
                                        hwaddr size)
 {
     uint32_t associativity[] = {
@@ -669,10 +541,12 @@
                       sizeof(mem_reg_property))));
     _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
                       sizeof(associativity))));
+    return off;
 }
 
-static int spapr_populate_memory(sPAPREnvironment *spapr, void *fdt)
+static int spapr_populate_memory(sPAPRMachineState *spapr, void *fdt)
 {
+    MachineState *machine = MACHINE(spapr);
     hwaddr mem_start, node_size;
     int i, nb_nodes = nb_numa_nodes;
     NodeInfo *nodes = numa_info;
@@ -681,7 +555,7 @@
     /* No NUMA nodes, assume there is just one node with whole RAM */
     if (!nb_numa_nodes) {
         nb_nodes = 1;
-        ramnode.node_mem = ram_size;
+        ramnode.node_mem = machine->ram_size;
         nodes = &ramnode;
     }
 
@@ -689,12 +563,12 @@
         if (!nodes[i].node_mem) {
             continue;
         }
-        if (mem_start >= ram_size) {
+        if (mem_start >= machine->ram_size) {
             node_size = 0;
         } else {
             node_size = nodes[i].node_mem;
-            if (node_size > ram_size - mem_start) {
-                node_size = ram_size - mem_start;
+            if (node_size > machine->ram_size - mem_start) {
+                node_size = machine->ram_size - mem_start;
             }
         }
         if (!mem_start) {
@@ -720,11 +594,353 @@
     return 0;
 }
 
-static void spapr_finalize_fdt(sPAPREnvironment *spapr,
+static void spapr_populate_cpu_dt(CPUState *cs, void *fdt, int offset,
+                                  sPAPRMachineState *spapr)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cs);
+    int index = ppc_get_vcpu_dt_id(cpu);
+    uint32_t segs[] = {cpu_to_be32(28), cpu_to_be32(40),
+                       0xffffffff, 0xffffffff};
+    uint32_t tbfreq = kvm_enabled() ? kvmppc_get_tbfreq()
+        : SPAPR_TIMEBASE_FREQ;
+    uint32_t cpufreq = kvm_enabled() ? kvmppc_get_clockfreq() : 1000000000;
+    uint32_t page_sizes_prop[64];
+    size_t page_sizes_prop_size;
+    uint32_t vcpus_per_socket = smp_threads * smp_cores;
+    uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    int drc_index;
+
+    drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index);
+    if (drc) {
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+        drc_index = drck->get_index(drc);
+        _FDT((fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index)));
+    }
+
+    /* Note: we keep CI large pages off for now because a 64K capable guest
+     * provisioned with large pages might otherwise try to map a qemu
+     * framebuffer (or other kind of memory mapped PCI BAR) using 64K pages
+     * even if that qemu runs on a 4k host.
+     *
+     * We can later add this bit back when we are confident this is not
+     * an issue (!HV KVM or 64K host)
+     */
+    uint8_t pa_features_206[] = { 6, 0,
+        0xf6, 0x1f, 0xc7, 0x00, 0x80, 0xc0 };
+    uint8_t pa_features_207[] = { 24, 0,
+        0xf6, 0x1f, 0xc7, 0xc0, 0x80, 0xf0,
+        0x80, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x80, 0x00,
+        0x80, 0x00, 0x80, 0x00, 0x80, 0x00 };
+    uint8_t *pa_features;
+    size_t pa_size;
+
+    _FDT((fdt_setprop_cell(fdt, offset, "reg", index)));
+    _FDT((fdt_setprop_string(fdt, offset, "device_type", "cpu")));
+
+    _FDT((fdt_setprop_cell(fdt, offset, "cpu-version", env->spr[SPR_PVR])));
+    _FDT((fdt_setprop_cell(fdt, offset, "d-cache-block-size",
+                           env->dcache_line_size)));
+    _FDT((fdt_setprop_cell(fdt, offset, "d-cache-line-size",
+                           env->dcache_line_size)));
+    _FDT((fdt_setprop_cell(fdt, offset, "i-cache-block-size",
+                           env->icache_line_size)));
+    _FDT((fdt_setprop_cell(fdt, offset, "i-cache-line-size",
+                           env->icache_line_size)));
+
+    if (pcc->l1_dcache_size) {
+        _FDT((fdt_setprop_cell(fdt, offset, "d-cache-size",
+                               pcc->l1_dcache_size)));
+    } else {
+        fprintf(stderr, "Warning: Unknown L1 dcache size for cpu\n");
+    }
+    if (pcc->l1_icache_size) {
+        _FDT((fdt_setprop_cell(fdt, offset, "i-cache-size",
+                               pcc->l1_icache_size)));
+    } else {
+        fprintf(stderr, "Warning: Unknown L1 icache size for cpu\n");
+    }
+
+    _FDT((fdt_setprop_cell(fdt, offset, "timebase-frequency", tbfreq)));
+    _FDT((fdt_setprop_cell(fdt, offset, "clock-frequency", cpufreq)));
+    _FDT((fdt_setprop_cell(fdt, offset, "slb-size", env->slb_nr)));
+    _FDT((fdt_setprop_cell(fdt, offset, "ibm,slb-size", env->slb_nr)));
+    _FDT((fdt_setprop_string(fdt, offset, "status", "okay")));
+    _FDT((fdt_setprop(fdt, offset, "64-bit", NULL, 0)));
+
+    if (env->spr_cb[SPR_PURR].oea_read) {
+        _FDT((fdt_setprop(fdt, offset, "ibm,purr", NULL, 0)));
+    }
+
+    if (env->mmu_model & POWERPC_MMU_1TSEG) {
+        _FDT((fdt_setprop(fdt, offset, "ibm,processor-segment-sizes",
+                          segs, sizeof(segs))));
+    }
+
+    /* Advertise VMX/VSX (vector extensions) if available
+     *   0 / no property == no vector extensions
+     *   1               == VMX / Altivec available
+     *   2               == VSX available */
+    if (env->insns_flags & PPC_ALTIVEC) {
+        uint32_t vmx = (env->insns_flags2 & PPC2_VSX) ? 2 : 1;
+
+        _FDT((fdt_setprop_cell(fdt, offset, "ibm,vmx", vmx)));
+    }
+
+    /* Advertise DFP (Decimal Floating Point) if available
+     *   0 / no property == no DFP
+     *   1               == DFP available */
+    if (env->insns_flags2 & PPC2_DFP) {
+        _FDT((fdt_setprop_cell(fdt, offset, "ibm,dfp", 1)));
+    }
+
+    page_sizes_prop_size = create_page_sizes_prop(env, page_sizes_prop,
+                                                  sizeof(page_sizes_prop));
+    if (page_sizes_prop_size) {
+        _FDT((fdt_setprop(fdt, offset, "ibm,segment-page-sizes",
+                          page_sizes_prop, page_sizes_prop_size)));
+    }
+
+    /* Do the ibm,pa-features property, adjust it for ci-large-pages */
+    if (env->mmu_model == POWERPC_MMU_2_06) {
+        pa_features = pa_features_206;
+        pa_size = sizeof(pa_features_206);
+    } else /* env->mmu_model == POWERPC_MMU_2_07 */ {
+        pa_features = pa_features_207;
+        pa_size = sizeof(pa_features_207);
+    }
+    if (env->ci_large_pages) {
+        pa_features[3] |= 0x20;
+    }
+    _FDT((fdt_setprop(fdt, offset, "ibm,pa-features", pa_features, pa_size)));
+
+    _FDT((fdt_setprop_cell(fdt, offset, "ibm,chip-id",
+                           cs->cpu_index / vcpus_per_socket)));
+
+    _FDT((fdt_setprop(fdt, offset, "ibm,pft-size",
+                      pft_size_prop, sizeof(pft_size_prop))));
+
+    _FDT(spapr_fixup_cpu_numa_dt(fdt, offset, cs));
+
+    _FDT(spapr_fixup_cpu_smt_dt(fdt, offset, cpu,
+                                ppc_get_compat_smt_threads(cpu)));
+}
+
+static void spapr_populate_cpus_dt_node(void *fdt, sPAPRMachineState *spapr)
+{
+    CPUState *cs;
+    int cpus_offset;
+    char *nodename;
+    int smt = kvmppc_smt_threads();
+
+    cpus_offset = fdt_add_subnode(fdt, 0, "cpus");
+    _FDT(cpus_offset);
+    _FDT((fdt_setprop_cell(fdt, cpus_offset, "#address-cells", 0x1)));
+    _FDT((fdt_setprop_cell(fdt, cpus_offset, "#size-cells", 0x0)));
+
+    /*
+     * We walk the CPUs in reverse order to ensure that CPU DT nodes
+     * created by fdt_add_subnode() end up in the right order in FDT
+     * for the guest kernel the enumerate the CPUs correctly.
+     */
+    CPU_FOREACH_REVERSE(cs) {
+        PowerPCCPU *cpu = POWERPC_CPU(cs);
+        int index = ppc_get_vcpu_dt_id(cpu);
+        DeviceClass *dc = DEVICE_GET_CLASS(cs);
+        int offset;
+
+        if ((index % smt) != 0) {
+            continue;
+        }
+
+        nodename = g_strdup_printf("%s@%x", dc->fw_name, index);
+        offset = fdt_add_subnode(fdt, cpus_offset, nodename);
+        g_free(nodename);
+        _FDT(offset);
+        spapr_populate_cpu_dt(cs, fdt, offset, spapr);
+    }
+
+}
+
+/*
+ * Adds ibm,dynamic-reconfiguration-memory node.
+ * Refer to docs/specs/ppc-spapr-hotplug.txt for the documentation
+ * of this device tree node.
+ */
+static int spapr_populate_drconf_memory(sPAPRMachineState *spapr, void *fdt)
+{
+    MachineState *machine = MACHINE(spapr);
+    int ret, i, offset;
+    uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
+    uint32_t prop_lmb_size[] = {0, cpu_to_be32(lmb_size)};
+    uint32_t hotplug_lmb_start = spapr->hotplug_memory.base / lmb_size;
+    uint32_t nr_lmbs = (spapr->hotplug_memory.base +
+                       memory_region_size(&spapr->hotplug_memory.mr)) /
+                       lmb_size;
+    uint32_t *int_buf, *cur_index, buf_len;
+    int nr_nodes = nb_numa_nodes ? nb_numa_nodes : 1;
+
+    /*
+     * Don't create the node if there is no hotpluggable memory
+     */
+    if (machine->ram_size == machine->maxram_size) {
+        return 0;
+    }
+
+    /*
+     * Allocate enough buffer size to fit in ibm,dynamic-memory
+     * or ibm,associativity-lookup-arrays
+     */
+    buf_len = MAX(nr_lmbs * SPAPR_DR_LMB_LIST_ENTRY_SIZE + 1, nr_nodes * 4 + 2)
+              * sizeof(uint32_t);
+    cur_index = int_buf = g_malloc0(buf_len);
+
+    offset = fdt_add_subnode(fdt, 0, "ibm,dynamic-reconfiguration-memory");
+
+    ret = fdt_setprop(fdt, offset, "ibm,lmb-size", prop_lmb_size,
+                    sizeof(prop_lmb_size));
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = fdt_setprop_cell(fdt, offset, "ibm,memory-flags-mask", 0xff);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = fdt_setprop_cell(fdt, offset, "ibm,memory-preservation-time", 0x0);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* ibm,dynamic-memory */
+    int_buf[0] = cpu_to_be32(nr_lmbs);
+    cur_index++;
+    for (i = 0; i < nr_lmbs; i++) {
+        uint64_t addr = i * lmb_size;
+        uint32_t *dynamic_memory = cur_index;
+
+        if (i >= hotplug_lmb_start) {
+            sPAPRDRConnector *drc;
+            sPAPRDRConnectorClass *drck;
+
+            drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_LMB, i);
+            g_assert(drc);
+            drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+            dynamic_memory[0] = cpu_to_be32(addr >> 32);
+            dynamic_memory[1] = cpu_to_be32(addr & 0xffffffff);
+            dynamic_memory[2] = cpu_to_be32(drck->get_index(drc));
+            dynamic_memory[3] = cpu_to_be32(0); /* reserved */
+            dynamic_memory[4] = cpu_to_be32(numa_get_node(addr, NULL));
+            if (memory_region_present(get_system_memory(), addr)) {
+                dynamic_memory[5] = cpu_to_be32(SPAPR_LMB_FLAGS_ASSIGNED);
+            } else {
+                dynamic_memory[5] = cpu_to_be32(0);
+            }
+        } else {
+            /*
+             * LMB information for RMA, boot time RAM and gap b/n RAM and
+             * hotplug memory region -- all these are marked as reserved
+             * and as having no valid DRC.
+             */
+            dynamic_memory[0] = cpu_to_be32(addr >> 32);
+            dynamic_memory[1] = cpu_to_be32(addr & 0xffffffff);
+            dynamic_memory[2] = cpu_to_be32(0);
+            dynamic_memory[3] = cpu_to_be32(0); /* reserved */
+            dynamic_memory[4] = cpu_to_be32(-1);
+            dynamic_memory[5] = cpu_to_be32(SPAPR_LMB_FLAGS_RESERVED |
+                                            SPAPR_LMB_FLAGS_DRC_INVALID);
+        }
+
+        cur_index += SPAPR_DR_LMB_LIST_ENTRY_SIZE;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,dynamic-memory", int_buf, buf_len);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /* ibm,associativity-lookup-arrays */
+    cur_index = int_buf;
+    int_buf[0] = cpu_to_be32(nr_nodes);
+    int_buf[1] = cpu_to_be32(4); /* Number of entries per associativity list */
+    cur_index += 2;
+    for (i = 0; i < nr_nodes; i++) {
+        uint32_t associativity[] = {
+            cpu_to_be32(0x0),
+            cpu_to_be32(0x0),
+            cpu_to_be32(0x0),
+            cpu_to_be32(i)
+        };
+        memcpy(cur_index, associativity, sizeof(associativity));
+        cur_index += 4;
+    }
+    ret = fdt_setprop(fdt, offset, "ibm,associativity-lookup-arrays", int_buf,
+            (cur_index - int_buf) * sizeof(uint32_t));
+out:
+    g_free(int_buf);
+    return ret;
+}
+
+int spapr_h_cas_compose_response(sPAPRMachineState *spapr,
+                                 target_ulong addr, target_ulong size,
+                                 bool cpu_update, bool memory_update)
+{
+    void *fdt, *fdt_skel;
+    sPAPRDeviceTreeUpdateHeader hdr = { .version_id = 1 };
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
+    size -= sizeof(hdr);
+
+    /* Create sceleton */
+    fdt_skel = g_malloc0(size);
+    _FDT((fdt_create(fdt_skel, size)));
+    _FDT((fdt_begin_node(fdt_skel, "")));
+    _FDT((fdt_end_node(fdt_skel)));
+    _FDT((fdt_finish(fdt_skel)));
+    fdt = g_malloc0(size);
+    _FDT((fdt_open_into(fdt_skel, fdt, size)));
+    g_free(fdt_skel);
+
+    /* Fixup cpu nodes */
+    if (cpu_update) {
+        _FDT((spapr_fixup_cpu_dt(fdt, spapr)));
+    }
+
+    /* Generate ibm,dynamic-reconfiguration-memory node if required */
+    if (memory_update && smc->dr_lmb_enabled) {
+        _FDT((spapr_populate_drconf_memory(spapr, fdt)));
+    }
+
+    /* Pack resulting tree */
+    _FDT((fdt_pack(fdt)));
+
+    if (fdt_totalsize(fdt) + sizeof(hdr) > size) {
+        trace_spapr_cas_failed(size);
+        return -1;
+    }
+
+    cpu_physical_memory_write(addr, &hdr, sizeof(hdr));
+    cpu_physical_memory_write(addr + sizeof(hdr), fdt, fdt_totalsize(fdt));
+    trace_spapr_cas_continue(fdt_totalsize(fdt) + sizeof(hdr));
+    g_free(fdt);
+
+    return 0;
+}
+
+static void spapr_finalize_fdt(sPAPRMachineState *spapr,
                                hwaddr fdt_addr,
                                hwaddr rtas_addr,
                                hwaddr rtas_size)
 {
+    MachineState *machine = MACHINE(qdev_get_machine());
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
+    const char *boot_device = machine->boot_order;
     int ret, i;
     size_t cb = 0;
     char *bootlist;
@@ -748,13 +964,20 @@
         exit(1);
     }
 
-    QLIST_FOREACH(phb, &spapr->phbs, list) {
-        ret = spapr_populate_pci_dt(phb, PHANDLE_XICP, fdt);
+    if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL)) {
+        ret = spapr_rng_populate_dt(fdt);
+        if (ret < 0) {
+            fprintf(stderr, "could not set up rng device in the fdt\n");
+            exit(1);
+        }
     }
 
-    if (ret < 0) {
-        fprintf(stderr, "couldn't setup PCI devices in fdt\n");
-        exit(1);
+    QLIST_FOREACH(phb, &spapr->phbs, list) {
+        ret = spapr_populate_pci_dt(phb, PHANDLE_XICP, fdt);
+        if (ret < 0) {
+            error_report("couldn't setup PCI devices in fdt");
+            exit(1);
+        }
     }
 
     /* RTAS */
@@ -763,11 +986,8 @@
         fprintf(stderr, "Couldn't set up RTAS device tree properties\n");
     }
 
-    /* Advertise NUMA via ibm,associativity */
-    ret = spapr_fixup_cpu_dt(fdt, spapr);
-    if (ret < 0) {
-        fprintf(stderr, "Couldn't finalize CPU device tree properties\n");
-    }
+    /* cpus */
+    spapr_populate_cpus_dt_node(fdt, spapr);
 
     bootlist = get_boot_devices_list(&cb, true);
     if (cb && bootlist) {
@@ -784,18 +1004,42 @@
         ret = fdt_setprop_string(fdt, offset, "qemu,boot-list", bootlist);
     }
 
+    if (boot_device && strlen(boot_device)) {
+        int offset = fdt_path_offset(fdt, "/chosen");
+
+        if (offset < 0) {
+            exit(1);
+        }
+        fdt_setprop_string(fdt, offset, "qemu,boot-device", boot_device);
+    }
+
     if (!spapr->has_graphics) {
         spapr_populate_chosen_stdout(fdt, spapr->vio_bus);
     }
 
+    if (smc->dr_lmb_enabled) {
+        _FDT(spapr_drc_populate_dt(fdt, 0, NULL, SPAPR_DR_CONNECTOR_TYPE_LMB));
+    }
+
+    if (mc->query_hotpluggable_cpus) {
+        int offset = fdt_path_offset(fdt, "/cpus");
+        ret = spapr_drc_populate_dt(fdt, offset, NULL,
+                                    SPAPR_DR_CONNECTOR_TYPE_CPU);
+        if (ret < 0) {
+            error_report("Couldn't set up CPU DR device tree properties");
+            exit(1);
+        }
+    }
+
     _FDT((fdt_pack(fdt)));
 
     if (fdt_totalsize(fdt) > FDT_MAX_SIZE) {
-        hw_error("FDT too big ! 0x%x bytes (max is 0x%x)\n",
-                 fdt_totalsize(fdt), FDT_MAX_SIZE);
+        error_report("FDT too big ! 0x%x bytes (max is 0x%x)",
+                     fdt_totalsize(fdt), FDT_MAX_SIZE);
         exit(1);
     }
 
+    qemu_fdt_dumpdtb(fdt, fdt_totalsize(fdt));
     cpu_physical_memory_write(fdt_addr, fdt, fdt_totalsize(fdt));
 
     g_free(bootlist);
@@ -819,34 +1063,98 @@
     }
 }
 
-static void spapr_reset_htab(sPAPREnvironment *spapr)
+#define HPTE(_table, _i)   (void *)(((uint64_t *)(_table)) + ((_i) * 2))
+#define HPTE_VALID(_hpte)  (tswap64(*((uint64_t *)(_hpte))) & HPTE64_V_VALID)
+#define HPTE_DIRTY(_hpte)  (tswap64(*((uint64_t *)(_hpte))) & HPTE64_V_HPTE_DIRTY)
+#define CLEAN_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) &= tswap64(~HPTE64_V_HPTE_DIRTY))
+#define DIRTY_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) |= tswap64(HPTE64_V_HPTE_DIRTY))
+
+/*
+ * Get the fd to access the kernel htab, re-opening it if necessary
+ */
+static int get_htab_fd(sPAPRMachineState *spapr)
 {
-    long shift;
-
-    /* allocate hash page table.  For now we always make this 16mb,
-     * later we should probably make it scale to the size of guest
-     * RAM */
-
-    shift = kvmppc_reset_htab(spapr->htab_shift);
-
-    if (shift > 0) {
-        /* Kernel handles htab, we don't need to allocate one */
-        spapr->htab_shift = shift;
-        kvmppc_kern_htab = true;
-    } else {
-        if (!spapr->htab) {
-            /* Allocate an htab if we don't yet have one */
-            spapr->htab = qemu_memalign(HTAB_SIZE(spapr), HTAB_SIZE(spapr));
-        }
-
-        /* And clear it */
-        memset(spapr->htab, 0, HTAB_SIZE(spapr));
+    if (spapr->htab_fd >= 0) {
+        return spapr->htab_fd;
     }
 
-    /* Update the RMA size if necessary */
-    if (spapr->vrma_adjust) {
-        spapr->rma_size = kvmppc_rma_size(spapr_node0_size(),
-                                          spapr->htab_shift);
+    spapr->htab_fd = kvmppc_get_htab_fd(false);
+    if (spapr->htab_fd < 0) {
+        error_report("Unable to open fd for reading hash table from KVM: %s",
+                     strerror(errno));
+    }
+
+    return spapr->htab_fd;
+}
+
+static void close_htab_fd(sPAPRMachineState *spapr)
+{
+    if (spapr->htab_fd >= 0) {
+        close(spapr->htab_fd);
+    }
+    spapr->htab_fd = -1;
+}
+
+static int spapr_hpt_shift_for_ramsize(uint64_t ramsize)
+{
+    int shift;
+
+    /* We aim for a hash table of size 1/128 the size of RAM (rounded
+     * up).  The PAPR recommendation is actually 1/64 of RAM size, but
+     * that's much more than is needed for Linux guests */
+    shift = ctz64(pow2ceil(ramsize)) - 7;
+    shift = MAX(shift, 18); /* Minimum architected size */
+    shift = MIN(shift, 46); /* Maximum architected size */
+    return shift;
+}
+
+static void spapr_reallocate_hpt(sPAPRMachineState *spapr, int shift,
+                                 Error **errp)
+{
+    long rc;
+
+    /* Clean up any HPT info from a previous boot */
+    g_free(spapr->htab);
+    spapr->htab = NULL;
+    spapr->htab_shift = 0;
+    close_htab_fd(spapr);
+
+    rc = kvmppc_reset_htab(shift);
+    if (rc < 0) {
+        /* kernel-side HPT needed, but couldn't allocate one */
+        error_setg_errno(errp, errno,
+                         "Failed to allocate KVM HPT of order %d (try smaller maxmem?)",
+                         shift);
+        /* This is almost certainly fatal, but if the caller really
+         * wants to carry on with shift == 0, it's welcome to try */
+    } else if (rc > 0) {
+        /* kernel-side HPT allocated */
+        if (rc != shift) {
+            error_setg(errp,
+                       "Requested order %d HPT, but kernel allocated order %ld (try smaller maxmem?)",
+                       shift, rc);
+        }
+
+        spapr->htab_shift = shift;
+        spapr->htab = NULL;
+    } else {
+        /* kernel-side HPT not needed, allocate in userspace instead */
+        size_t size = 1ULL << shift;
+        int i;
+
+        spapr->htab = qemu_memalign(size, size);
+        if (!spapr->htab) {
+            error_setg_errno(errp, errno,
+                             "Could not allocate HPT of order %d", shift);
+            return;
+        }
+
+        memset(spapr->htab, 0, size);
+        spapr->htab_shift = shift;
+
+        for (i = 0; i < size / HASH_PTE_SIZE_64; i++) {
+            DIRTY_HPTE(HPTE(spapr->htab, i));
+        }
     }
 }
 
@@ -869,14 +1177,24 @@
 
 static void ppc_spapr_reset(void)
 {
+    MachineState *machine = MACHINE(qdev_get_machine());
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
     PowerPCCPU *first_ppc_cpu;
     uint32_t rtas_limit;
 
     /* Check for unknown sysbus devices */
     foreach_dynamic_sysbus_device(find_unknown_sysbus_device, NULL);
 
-    /* Reset the hash table & recalc the RMA */
-    spapr_reset_htab(spapr);
+    /* Allocate and/or reset the hash page table */
+    spapr_reallocate_hpt(spapr,
+                         spapr_hpt_shift_for_ramsize(machine->maxram_size),
+                         &error_fatal);
+
+    /* Update the RMA size if necessary */
+    if (spapr->vrma_adjust) {
+        spapr->rma_size = kvmppc_rma_size(spapr_node0_size(),
+                                          spapr->htab_shift);
+    }
 
     qemu_devices_reset();
 
@@ -902,52 +1220,18 @@
     first_ppc_cpu->env.gpr[3] = spapr->fdt_addr;
     first_ppc_cpu->env.gpr[5] = 0;
     first_cpu->halted = 0;
-    first_ppc_cpu->env.nip = spapr->entry_point;
+    first_ppc_cpu->env.nip = SPAPR_ENTRY_POINT;
 
 }
 
-static void spapr_cpu_reset(void *opaque)
-{
-    PowerPCCPU *cpu = opaque;
-    CPUState *cs = CPU(cpu);
-    CPUPPCState *env = &cpu->env;
-
-    cpu_reset(cs);
-
-    /* All CPUs start halted.  CPU0 is unhalted from the machine level
-     * reset code and the rest are explicitly started up by the guest
-     * using an RTAS call */
-    cs->halted = 1;
-
-    env->spr[SPR_HIOR] = 0;
-
-    env->external_htab = (uint8_t *)spapr->htab;
-    if (kvm_enabled() && !env->external_htab) {
-        /*
-         * HV KVM, set external_htab to 1 so our ppc_hash64_load_hpte*
-         * functions do the right thing.
-         */
-        env->external_htab = (void *)1;
-    }
-    env->htab_base = -1;
-    /*
-     * htab_mask is the mask used to normalize hash value to PTEG index.
-     * htab_shift is log2 of hash table size.
-     * We have 8 hpte per group, and each hpte is 16 bytes.
-     * ie have 128 bytes per hpte entry.
-     */
-    env->htab_mask = (1ULL << ((spapr)->htab_shift - 7)) - 1;
-    env->spr[SPR_SDR1] = (target_ulong)(uintptr_t)spapr->htab |
-        (spapr->htab_shift - 18);
-}
-
-static void spapr_create_nvram(sPAPREnvironment *spapr)
+static void spapr_create_nvram(sPAPRMachineState *spapr)
 {
     DeviceState *dev = qdev_create(&spapr->vio_bus->bus, "spapr-nvram");
     DriveInfo *dinfo = drive_get(IF_PFLASH, 0, 0);
 
     if (dinfo) {
-        qdev_prop_set_drive_nofail(dev, "drive", blk_by_legacy_dinfo(dinfo));
+        qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                            &error_fatal);
     }
 
     qdev_init_nofail(dev);
@@ -955,8 +1239,19 @@
     spapr->nvram = (struct sPAPRNVRAM *)dev;
 }
 
+static void spapr_rtc_create(sPAPRMachineState *spapr)
+{
+    DeviceState *dev = qdev_create(NULL, TYPE_SPAPR_RTC);
+
+    qdev_init_nofail(dev);
+    spapr->rtc = dev;
+
+    object_property_add_alias(qdev_get_machine(), "rtc-time",
+                              OBJECT(spapr->rtc), "date", NULL);
+}
+
 /* Returns whether we want to use VGA or not */
-static int spapr_vga_init(PCIBus *pci_bus)
+static bool spapr_vga_init(PCIBus *pci_bus, Error **errp)
 {
     switch (vga_interface_type) {
     case VGA_NONE:
@@ -964,36 +1259,56 @@
     case VGA_DEVICE:
         return true;
     case VGA_STD:
+    case VGA_VIRTIO:
         return pci_vga_init(pci_bus) != NULL;
     default:
-        fprintf(stderr, "This vga model is not supported,"
-                "currently it only supports -vga std\n");
-        exit(0);
+        error_setg(errp,
+                   "Unsupported VGA mode, only -vga std or -vga virtio is supported");
+        return false;
     }
 }
 
+static int spapr_post_load(void *opaque, int version_id)
+{
+    sPAPRMachineState *spapr = (sPAPRMachineState *)opaque;
+    int err = 0;
+
+    /* In earlier versions, there was no separate qdev for the PAPR
+     * RTC, so the RTC offset was stored directly in sPAPREnvironment.
+     * So when migrating from those versions, poke the incoming offset
+     * value into the RTC device */
+    if (version_id < 3) {
+        err = spapr_rtc_import_offset(spapr->rtc, spapr->rtc_offset);
+    }
+
+    return err;
+}
+
+static bool version_before_3(void *opaque, int version_id)
+{
+    return version_id < 3;
+}
+
 static const VMStateDescription vmstate_spapr = {
     .name = "spapr",
-    .version_id = 2,
+    .version_id = 3,
     .minimum_version_id = 1,
+    .post_load = spapr_post_load,
     .fields = (VMStateField[]) {
-        VMSTATE_UNUSED(4), /* used to be @next_irq */
+        /* used to be @next_irq */
+        VMSTATE_UNUSED_BUFFER(version_before_3, 0, 4),
 
         /* RTC offset */
-        VMSTATE_UINT64(rtc_offset, sPAPREnvironment),
-        VMSTATE_PPC_TIMEBASE_V(tb, sPAPREnvironment, 2),
+        VMSTATE_UINT64_TEST(rtc_offset, sPAPRMachineState, version_before_3),
+
+        VMSTATE_PPC_TIMEBASE_V(tb, sPAPRMachineState, 2),
         VMSTATE_END_OF_LIST()
     },
 };
 
-#define HPTE(_table, _i)   (void *)(((uint64_t *)(_table)) + ((_i) * 2))
-#define HPTE_VALID(_hpte)  (tswap64(*((uint64_t *)(_hpte))) & HPTE64_V_VALID)
-#define HPTE_DIRTY(_hpte)  (tswap64(*((uint64_t *)(_hpte))) & HPTE64_V_HPTE_DIRTY)
-#define CLEAN_HPTE(_hpte)  ((*(uint64_t *)(_hpte)) &= tswap64(~HPTE64_V_HPTE_DIRTY))
-
 static int htab_save_setup(QEMUFile *f, void *opaque)
 {
-    sPAPREnvironment *spapr = opaque;
+    sPAPRMachineState *spapr = opaque;
 
     /* "Iteration" header */
     qemu_put_be32(f, spapr->htab_shift);
@@ -1003,22 +1318,16 @@
         spapr->htab_first_pass = true;
     } else {
         assert(kvm_enabled());
-
-        spapr->htab_fd = kvmppc_get_htab_fd(false);
-        if (spapr->htab_fd < 0) {
-            fprintf(stderr, "Unable to open fd for reading hash table from KVM: %s\n",
-                    strerror(errno));
-            return -1;
-        }
     }
 
 
     return 0;
 }
 
-static void htab_save_first_pass(QEMUFile *f, sPAPREnvironment *spapr,
+static void htab_save_first_pass(QEMUFile *f, sPAPRMachineState *spapr,
                                  int64_t max_ns)
 {
+    bool has_timeout = max_ns != -1;
     int htabslots = HTAB_SIZE(spapr) / HASH_PTE_SIZE_64;
     int index = spapr->htab_save_index;
     int64_t starttime = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
@@ -1037,7 +1346,7 @@
 
         /* Consume valid HPTEs */
         chunkstart = index;
-        while ((index < htabslots)
+        while ((index < htabslots) && (index - chunkstart < USHRT_MAX)
                && HPTE_VALID(HPTE(spapr->htab, index))) {
             index++;
             CLEAN_HPTE(HPTE(spapr->htab, index));
@@ -1052,7 +1361,8 @@
             qemu_put_buffer(f, HPTE(spapr->htab, chunkstart),
                             HASH_PTE_SIZE_64 * n_valid);
 
-            if ((qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - starttime) > max_ns) {
+            if (has_timeout &&
+                (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - starttime) > max_ns) {
                 break;
             }
         }
@@ -1066,7 +1376,7 @@
     spapr->htab_save_index = index;
 }
 
-static int htab_save_later_pass(QEMUFile *f, sPAPREnvironment *spapr,
+static int htab_save_later_pass(QEMUFile *f, sPAPRMachineState *spapr,
                                 int64_t max_ns)
 {
     bool final = max_ns < 0;
@@ -1089,7 +1399,7 @@
 
         chunkstart = index;
         /* Consume valid dirty HPTEs */
-        while ((index < htabslots)
+        while ((index < htabslots) && (index - chunkstart < USHRT_MAX)
                && HPTE_DIRTY(HPTE(spapr->htab, index))
                && HPTE_VALID(HPTE(spapr->htab, index))) {
             CLEAN_HPTE(HPTE(spapr->htab, index));
@@ -1099,7 +1409,7 @@
 
         invalidstart = index;
         /* Consume invalid dirty HPTEs */
-        while ((index < htabslots)
+        while ((index < htabslots) && (index - invalidstart < USHRT_MAX)
                && HPTE_DIRTY(HPTE(spapr->htab, index))
                && !HPTE_VALID(HPTE(spapr->htab, index))) {
             CLEAN_HPTE(HPTE(spapr->htab, index));
@@ -1148,7 +1458,8 @@
 
 static int htab_save_iterate(QEMUFile *f, void *opaque)
 {
-    sPAPREnvironment *spapr = opaque;
+    sPAPRMachineState *spapr = opaque;
+    int fd;
     int rc = 0;
 
     /* Iteration header */
@@ -1157,8 +1468,12 @@
     if (!spapr->htab) {
         assert(kvm_enabled());
 
-        rc = kvmppc_save_htab(f, spapr->htab_fd,
-                              MAX_KVM_BUF_SIZE, MAX_ITERATION_NS);
+        fd = get_htab_fd(spapr);
+        if (fd < 0) {
+            return fd;
+        }
+
+        rc = kvmppc_save_htab(f, fd, MAX_KVM_BUF_SIZE, MAX_ITERATION_NS);
         if (rc < 0) {
             return rc;
         }
@@ -1178,7 +1493,8 @@
 
 static int htab_save_complete(QEMUFile *f, void *opaque)
 {
-    sPAPREnvironment *spapr = opaque;
+    sPAPRMachineState *spapr = opaque;
+    int fd;
 
     /* Iteration header */
     qemu_put_be32(f, 0);
@@ -1188,13 +1504,19 @@
 
         assert(kvm_enabled());
 
-        rc = kvmppc_save_htab(f, spapr->htab_fd, MAX_KVM_BUF_SIZE, -1);
+        fd = get_htab_fd(spapr);
+        if (fd < 0) {
+            return fd;
+        }
+
+        rc = kvmppc_save_htab(f, fd, MAX_KVM_BUF_SIZE, -1);
         if (rc < 0) {
             return rc;
         }
-        close(spapr->htab_fd);
-        spapr->htab_fd = -1;
     } else {
+        if (spapr->htab_first_pass) {
+            htab_save_first_pass(f, spapr, -1);
+        }
         htab_save_later_pass(f, spapr, -1);
     }
 
@@ -1208,20 +1530,24 @@
 
 static int htab_load(QEMUFile *f, void *opaque, int version_id)
 {
-    sPAPREnvironment *spapr = opaque;
+    sPAPRMachineState *spapr = opaque;
     uint32_t section_hdr;
     int fd = -1;
 
     if (version_id < 1 || version_id > 1) {
-        fprintf(stderr, "htab_load() bad version\n");
+        error_report("htab_load() bad version");
         return -EINVAL;
     }
 
     section_hdr = qemu_get_be32(f);
 
     if (section_hdr) {
-        /* First section, just the hash shift */
-        if (spapr->htab_shift != section_hdr) {
+        Error *local_err = NULL;
+
+        /* First section gives the htab size */
+        spapr_reallocate_hpt(spapr, section_hdr, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
             return -EINVAL;
         }
         return 0;
@@ -1232,8 +1558,8 @@
 
         fd = kvmppc_get_htab_fd(true);
         if (fd < 0) {
-            fprintf(stderr, "Unable to open fd to restore KVM hash table: %s\n",
-                    strerror(errno));
+            error_report("Unable to open fd to restore KVM hash table: %s",
+                         strerror(errno));
         }
     }
 
@@ -1253,9 +1579,9 @@
         if ((index + n_valid + n_invalid) >
             (HTAB_SIZE(spapr) / HASH_PTE_SIZE_64)) {
             /* Bad index in stream */
-            fprintf(stderr, "htab_load() bad index %d (%hd+%hd entries) "
-                    "in htab stream (htab_shift=%d)\n", index, n_valid, n_invalid,
-                    spapr->htab_shift);
+            error_report(
+                "htab_load() bad index %d (%hd+%hd entries) in htab stream (htab_shift=%d)",
+                index, n_valid, n_invalid, spapr->htab_shift);
             return -EINVAL;
         }
 
@@ -1288,24 +1614,110 @@
     return 0;
 }
 
+static void htab_cleanup(void *opaque)
+{
+    sPAPRMachineState *spapr = opaque;
+
+    close_htab_fd(spapr);
+}
+
 static SaveVMHandlers savevm_htab_handlers = {
     .save_live_setup = htab_save_setup,
     .save_live_iterate = htab_save_iterate,
-    .save_live_complete = htab_save_complete,
+    .save_live_complete_precopy = htab_save_complete,
+    .cleanup = htab_cleanup,
     .load_state = htab_load,
 };
 
+static void spapr_boot_set(void *opaque, const char *boot_device,
+                           Error **errp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    machine->boot_order = g_strdup(boot_device);
+}
+
+/*
+ * Reset routine for LMB DR devices.
+ *
+ * Unlike PCI DR devices, LMB DR devices explicitly register this reset
+ * routine. Reset for PCI DR devices will be handled by PHB reset routine
+ * when it walks all its children devices. LMB devices reset occurs
+ * as part of spapr_ppc_reset().
+ */
+static void spapr_drc_reset(void *opaque)
+{
+    sPAPRDRConnector *drc = opaque;
+    DeviceState *d = DEVICE(drc);
+
+    if (d) {
+        device_reset(d);
+    }
+}
+
+static void spapr_create_lmb_dr_connectors(sPAPRMachineState *spapr)
+{
+    MachineState *machine = MACHINE(spapr);
+    uint64_t lmb_size = SPAPR_MEMORY_BLOCK_SIZE;
+    uint32_t nr_lmbs = (machine->maxram_size - machine->ram_size)/lmb_size;
+    int i;
+
+    for (i = 0; i < nr_lmbs; i++) {
+        sPAPRDRConnector *drc;
+        uint64_t addr;
+
+        addr = i * lmb_size + spapr->hotplug_memory.base;
+        drc = spapr_dr_connector_new(OBJECT(spapr), SPAPR_DR_CONNECTOR_TYPE_LMB,
+                                     addr/lmb_size);
+        qemu_register_reset(spapr_drc_reset, drc);
+    }
+}
+
+/*
+ * If RAM size, maxmem size and individual node mem sizes aren't aligned
+ * to SPAPR_MEMORY_BLOCK_SIZE(256MB), then refuse to start the guest
+ * since we can't support such unaligned sizes with DRCONF_MEMORY.
+ */
+static void spapr_validate_node_memory(MachineState *machine, Error **errp)
+{
+    int i;
+
+    if (machine->ram_size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(errp, "Memory size 0x" RAM_ADDR_FMT
+                   " is not aligned to %llu MiB",
+                   machine->ram_size,
+                   SPAPR_MEMORY_BLOCK_SIZE / M_BYTE);
+        return;
+    }
+
+    if (machine->maxram_size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(errp, "Maximum memory size 0x" RAM_ADDR_FMT
+                   " is not aligned to %llu MiB",
+                   machine->ram_size,
+                   SPAPR_MEMORY_BLOCK_SIZE / M_BYTE);
+        return;
+    }
+
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (numa_info[i].node_mem % SPAPR_MEMORY_BLOCK_SIZE) {
+            error_setg(errp,
+                       "Node %d memory size 0x%" PRIx64
+                       " is not aligned to %llu MiB",
+                       i, numa_info[i].node_mem,
+                       SPAPR_MEMORY_BLOCK_SIZE / M_BYTE);
+            return;
+        }
+    }
+}
+
 /* pSeries LPAR / sPAPR hardware init */
 static void ppc_spapr_init(MachineState *machine)
 {
-    ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    MachineClass *mc = MACHINE_GET_CLASS(machine);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(machine);
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     const char *initrd_filename = machine->initrd_filename;
-    const char *boot_device = machine->boot_order;
-    PowerPCCPU *cpu;
-    CPUPPCState *env;
     PCIHostState *phb;
     int i;
     MemoryRegion *sysmem = get_system_memory();
@@ -1319,10 +1731,25 @@
     long load_limit, fw_size;
     bool kernel_le = false;
     char *filename;
+    int smt = kvmppc_smt_threads();
+    int spapr_cores = smp_cpus / smp_threads;
+    int spapr_max_cores = max_cpus / smp_threads;
 
-    msi_supported = true;
+    if (mc->query_hotpluggable_cpus) {
+        if (smp_cpus % smp_threads) {
+            error_report("smp_cpus (%u) must be multiple of threads (%u)",
+                         smp_cpus, smp_threads);
+            exit(1);
+        }
+        if (max_cpus % smp_threads) {
+            error_report("max_cpus (%u) must be multiple of threads (%u)",
+                         max_cpus, smp_threads);
+            exit(1);
+        }
+    }
 
-    spapr = g_malloc0(sizeof(*spapr));
+    msi_nonbroken = true;
+
     QLIST_INIT(&spapr->phbs);
 
     cpu_ppc_hypercall = emulate_spapr_hypercall;
@@ -1331,7 +1758,7 @@
     rma_alloc_size = kvmppc_alloc_rma(&rma);
 
     if (rma_alloc_size == -1) {
-        hw_error("qemu: Unable to create RMA\n");
+        error_report("Unable to create RMA");
         exit(1);
     }
 
@@ -1353,72 +1780,88 @@
             spapr->vrma_adjust = 1;
             spapr->rma_size = MIN(spapr->rma_size, 0x10000000);
         }
+
+        /* Actually we don't support unbounded RMA anymore since we
+         * added proper emulation of HV mode. The max we can get is
+         * 16G which also happens to be what we configure for PAPR
+         * mode so make sure we don't do anything bigger than that
+         */
+        spapr->rma_size = MIN(spapr->rma_size, 0x400000000ull);
     }
 
     if (spapr->rma_size > node0_size) {
-        fprintf(stderr, "Error: Numa node 0 has to span the RMA (%#08"HWADDR_PRIx")\n",
-                spapr->rma_size);
+        error_report("Numa node 0 has to span the RMA (%#08"HWADDR_PRIx")",
+                     spapr->rma_size);
         exit(1);
     }
 
     /* Setup a load limit for the ramdisk leaving room for SLOF and FDT */
     load_limit = MIN(spapr->rma_size, RTAS_MAX_ADDR) - FW_OVERHEAD;
 
-    /* We aim for a hash table of size 1/128 the size of RAM.  The
-     * normal rule of thumb is 1/64 the size of RAM, but that's much
-     * more than needed for the Linux guests we support. */
-    spapr->htab_shift = 18; /* Minimum architected size */
-    while (spapr->htab_shift <= 46) {
-        if ((1ULL << (spapr->htab_shift + 7)) >= ram_size) {
-            break;
-        }
-        spapr->htab_shift++;
-    }
-
     /* Set up Interrupt Controller before we create the VCPUs */
-    spapr->icp = xics_system_init(smp_cpus * kvmppc_smt_threads() / smp_threads,
-                                  XICS_IRQS);
+    spapr->xics = xics_system_init(machine,
+                                   DIV_ROUND_UP(max_cpus * smt, smp_threads),
+                                   XICS_IRQS_SPAPR, &error_fatal);
+
+    if (smc->dr_lmb_enabled) {
+        spapr_validate_node_memory(machine, &error_fatal);
+    }
 
     /* init CPUs */
-    if (cpu_model == NULL) {
-        cpu_model = kvm_enabled() ? "host" : "POWER7";
+    if (machine->cpu_model == NULL) {
+        machine->cpu_model = kvm_enabled() ? "host" : "POWER7";
     }
-    for (i = 0; i < smp_cpus; i++) {
-        cpu = cpu_ppc_init(cpu_model);
-        if (cpu == NULL) {
-            fprintf(stderr, "Unable to find PowerPC CPU definition\n");
+
+    ppc_cpu_parse_features(machine->cpu_model);
+
+    if (mc->query_hotpluggable_cpus) {
+        char *type = spapr_get_cpu_core_type(machine->cpu_model);
+
+        if (type == NULL) {
+            error_report("Unable to find sPAPR CPU Core definition");
             exit(1);
         }
-        env = &cpu->env;
 
-        /* Set time-base frequency to 512 MHz */
-        cpu_ppc_tb_init(env, TIMEBASE_FREQ);
+        spapr->cores = g_new0(Object *, spapr_max_cores);
+        for (i = 0; i < spapr_max_cores; i++) {
+            int core_id = i * smp_threads;
+            sPAPRDRConnector *drc =
+                spapr_dr_connector_new(OBJECT(spapr),
+                                       SPAPR_DR_CONNECTOR_TYPE_CPU,
+                                       (core_id / smp_threads) * smt);
 
-        /* PAPR always has exception vectors in RAM not ROM. To ensure this,
-         * MSR[IP] should never be set.
-         */
-        env->msr_mask &= ~(1 << 6);
+            qemu_register_reset(spapr_drc_reset, drc);
 
-        /* Tell KVM that we're in PAPR mode */
-        if (kvm_enabled()) {
-            kvmppc_set_papr(cpu);
-        }
-
-        if (cpu->max_compat) {
-            if (ppc_set_compat(cpu, cpu->max_compat) < 0) {
-                exit(1);
+            if (i < spapr_cores) {
+                Object *core  = object_new(type);
+                object_property_set_int(core, smp_threads, "nr-threads",
+                                        &error_fatal);
+                object_property_set_int(core, core_id, CPU_CORE_PROP_CORE_ID,
+                                        &error_fatal);
+                object_property_set_bool(core, true, "realized", &error_fatal);
             }
         }
+        g_free(type);
+    } else {
+        for (i = 0; i < smp_cpus; i++) {
+            PowerPCCPU *cpu = cpu_ppc_init(machine->cpu_model);
+            if (cpu == NULL) {
+                error_report("Unable to find PowerPC CPU definition");
+                exit(1);
+            }
+            spapr_cpu_init(spapr, cpu, &error_fatal);
+       }
+    }
 
-        xics_cpu_setup(spapr->icp, cpu);
-
-        qemu_register_reset(spapr_cpu_reset, cpu);
+    if (kvm_enabled()) {
+        /* Enable H_LOGICAL_CI_* so SLOF can talk to in-kernel devices */
+        kvmppc_enable_logical_ci_hcalls();
+        kvmppc_enable_set_mode_hcall();
     }
 
     /* allocate RAM */
-    spapr->ram_limit = ram_size;
     memory_region_allocate_system_memory(ram, NULL, "ppc_spapr.ram",
-                                         spapr->ram_limit);
+                                         machine->ram_size);
     memory_region_add_subregion(sysmem, 0, ram);
 
     if (rma_alloc_size && rma) {
@@ -1429,16 +1872,57 @@
         memory_region_add_subregion(sysmem, 0, rma_region);
     }
 
+    /* initialize hotplug memory address space */
+    if (machine->ram_size < machine->maxram_size) {
+        ram_addr_t hotplug_mem_size = machine->maxram_size - machine->ram_size;
+        /*
+         * Limit the number of hotpluggable memory slots to half the number
+         * slots that KVM supports, leaving the other half for PCI and other
+         * devices. However ensure that number of slots doesn't drop below 32.
+         */
+        int max_memslots = kvm_enabled() ? kvm_get_max_memslots() / 2 :
+                           SPAPR_MAX_RAM_SLOTS;
+
+        if (max_memslots < SPAPR_MAX_RAM_SLOTS) {
+            max_memslots = SPAPR_MAX_RAM_SLOTS;
+        }
+        if (machine->ram_slots > max_memslots) {
+            error_report("Specified number of memory slots %"
+                         PRIu64" exceeds max supported %d",
+                         machine->ram_slots, max_memslots);
+            exit(1);
+        }
+
+        spapr->hotplug_memory.base = ROUND_UP(machine->ram_size,
+                                              SPAPR_HOTPLUG_MEM_ALIGN);
+        memory_region_init(&spapr->hotplug_memory.mr, OBJECT(spapr),
+                           "hotplug-memory", hotplug_mem_size);
+        memory_region_add_subregion(sysmem, spapr->hotplug_memory.base,
+                                    &spapr->hotplug_memory.mr);
+    }
+
+    if (smc->dr_lmb_enabled) {
+        spapr_create_lmb_dr_connectors(spapr);
+    }
+
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
+    if (!filename) {
+        error_report("Could not find LPAR rtas '%s'", "spapr-rtas.bin");
+        exit(1);
+    }
     spapr->rtas_size = get_image_size(filename);
+    if (spapr->rtas_size < 0) {
+        error_report("Could not get size of LPAR rtas '%s'", filename);
+        exit(1);
+    }
     spapr->rtas_blob = g_malloc(spapr->rtas_size);
     if (load_image_size(filename, spapr->rtas_blob, spapr->rtas_size) < 0) {
-        hw_error("qemu: could not load LPAR rtas '%s'\n", filename);
+        error_report("Could not load LPAR rtas '%s'", filename);
         exit(1);
     }
     if (spapr->rtas_size > RTAS_MAX_SIZE) {
-        hw_error("RTAS too big ! 0x%zx bytes (max is 0x%x)\n",
-                 spapr->rtas_size, RTAS_MAX_SIZE);
+        error_report("RTAS too big ! 0x%zx bytes (max is 0x%x)",
+                     (size_t)spapr->rtas_size, RTAS_MAX_SIZE);
         exit(1);
     }
     g_free(filename);
@@ -1446,6 +1930,9 @@
     /* Set up EPOW events infrastructure */
     spapr_events_init(spapr);
 
+    /* Set up the RTC RTAS interfaces */
+    spapr_rtc_create(spapr);
+
     /* Set up VIO bus */
     spapr->vio_bus = spapr_vio_bus_init();
 
@@ -1482,21 +1969,30 @@
     }
 
     /* Graphics */
-    if (spapr_vga_init(phb->bus)) {
+    if (spapr_vga_init(phb->bus, &error_fatal)) {
         spapr->has_graphics = true;
+        machine->usb |= defaults_enabled() && !machine->usb_disabled;
     }
 
-    if (usb_enabled(spapr->has_graphics)) {
-        pci_create_simple(phb->bus, -1, "pci-ohci");
+    if (machine->usb) {
+        if (smc->use_ohci_by_default) {
+            pci_create_simple(phb->bus, -1, "pci-ohci");
+        } else {
+            pci_create_simple(phb->bus, -1, "nec-usb-xhci");
+        }
+
         if (spapr->has_graphics) {
-            usbdevice_create("keyboard");
-            usbdevice_create("mouse");
+            USBBus *usb_bus = usb_bus_find(-1);
+
+            usb_create_simple(usb_bus, "usb-kbd");
+            usb_create_simple(usb_bus, "usb-mouse");
         }
     }
 
     if (spapr->rma_size < (MIN_RMA_SLOF << 20)) {
-        fprintf(stderr, "qemu: pSeries SLOF firmware requires >= "
-                "%ldM guest RMA (Real Mode Area memory)\n", MIN_RMA_SLOF);
+        error_report(
+            "pSeries SLOF firmware requires >= %ldM guest RMA (Real Mode Area memory)",
+            MIN_RMA_SLOF);
         exit(1);
     }
 
@@ -1504,16 +2000,18 @@
         uint64_t lowaddr = 0;
 
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, &lowaddr, NULL, 1, ELF_MACHINE, 0);
+                               NULL, &lowaddr, NULL, 1, PPC_ELF_MACHINE,
+                               0, 0);
         if (kernel_size == ELF_LOAD_WRONG_ENDIAN) {
             kernel_size = load_elf(kernel_filename,
                                    translate_kernel_address, NULL,
-                                   NULL, &lowaddr, NULL, 0, ELF_MACHINE, 0);
+                                   NULL, &lowaddr, NULL, 0, PPC_ELF_MACHINE,
+                                   0, 0);
             kernel_le = kernel_size > 0;
         }
         if (kernel_size < 0) {
-            fprintf(stderr, "qemu: error loading %s: %s\n",
-                    kernel_filename, load_elf_strerror(kernel_size));
+            error_report("error loading %s: %s",
+                         kernel_filename, load_elf_strerror(kernel_size));
             exit(1);
         }
 
@@ -1526,8 +2024,8 @@
             initrd_size = load_image_targphys(initrd_filename, initrd_base,
                                               load_limit - initrd_base);
             if (initrd_size < 0) {
-                fprintf(stderr, "qemu: could not load initial ram disk '%s'\n",
-                        initrd_filename);
+                error_report("could not load initial ram disk '%s'",
+                             initrd_filename);
                 exit(1);
             }
         } else {
@@ -1540,15 +2038,20 @@
         bios_name = FW_FILE_NAME;
     }
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
+    if (!filename) {
+        error_report("Could not find LPAR firmware '%s'", bios_name);
+        exit(1);
+    }
     fw_size = load_image_targphys(filename, 0, FW_MAX_SIZE);
-    if (fw_size < 0) {
-        hw_error("qemu: could not load LPAR rtas '%s'\n", filename);
+    if (fw_size <= 0) {
+        error_report("Could not load LPAR firmware '%s'", filename);
         exit(1);
     }
     g_free(filename);
 
-    spapr->entry_point = 0x100;
-
+    /* FIXME: Should register things through the MachineState's qdev
+     * interface, this is a legacy from the sPAPREnvironment structure
+     * which predated MachineState but had a similar function */
     vmstate_register(NULL, 0, &vmstate_spapr, spapr);
     register_savevm_live(NULL, "spapr/htab", -1, 1,
                          &savevm_htab_handlers, spapr);
@@ -1556,9 +2059,15 @@
     /* Prepare the device tree */
     spapr->fdt_skel = spapr_create_fdt_skel(initrd_base, initrd_size,
                                             kernel_size, kernel_le,
-                                            boot_device, kernel_cmdline,
-                                            spapr->epow_irq);
+                                            kernel_cmdline,
+                                            spapr->check_exception_irq);
     assert(spapr->fdt_skel != NULL);
+
+    /* used by RTAS */
+    QTAILQ_INIT(&spapr->ccs_list);
+    qemu_register_reset(spapr_ccs_reset_hook, spapr);
+
+    qemu_register_boot_set(spapr_boot_set, spapr);
 }
 
 static int spapr_kvm_type(const char *vm_type)
@@ -1580,7 +2089,7 @@
 }
 
 /*
- * Implementation of an interface to adjust firmware patch
+ * Implementation of an interface to adjust firmware path
  * for the bootindex property handling.
  */
 static char *spapr_get_fw_dev_path(FWPathProvider *p, BusState *bus,
@@ -1638,23 +2147,36 @@
 
 static char *spapr_get_kvm_type(Object *obj, Error **errp)
 {
-    sPAPRMachineState *sm = SPAPR_MACHINE(obj);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
 
-    return g_strdup(sm->kvm_type);
+    return g_strdup(spapr->kvm_type);
 }
 
 static void spapr_set_kvm_type(Object *obj, const char *value, Error **errp)
 {
-    sPAPRMachineState *sm = SPAPR_MACHINE(obj);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
 
-    g_free(sm->kvm_type);
-    sm->kvm_type = g_strdup(value);
+    g_free(spapr->kvm_type);
+    spapr->kvm_type = g_strdup(value);
 }
 
 static void spapr_machine_initfn(Object *obj)
 {
+    sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
+
+    spapr->htab_fd = -1;
     object_property_add_str(obj, "kvm-type",
                             spapr_get_kvm_type, spapr_set_kvm_type, NULL);
+    object_property_set_description(obj, "kvm-type",
+                                    "Specifies the KVM virtualization mode (HV, PR)",
+                                    NULL);
+}
+
+static void spapr_machine_finalizefn(Object *obj)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(obj);
+
+    g_free(spapr->kvm_type);
 }
 
 static void ppc_cpu_do_nmi_on_cpu(void *arg)
@@ -1674,21 +2196,246 @@
     }
 }
 
+static void spapr_add_lmbs(DeviceState *dev, uint64_t addr, uint64_t size,
+                           uint32_t node, Error **errp)
+{
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    uint32_t nr_lmbs = size/SPAPR_MEMORY_BLOCK_SIZE;
+    int i, fdt_offset, fdt_size;
+    void *fdt;
+
+    for (i = 0; i < nr_lmbs; i++) {
+        drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_LMB,
+                addr/SPAPR_MEMORY_BLOCK_SIZE);
+        g_assert(drc);
+
+        fdt = create_device_tree(&fdt_size);
+        fdt_offset = spapr_populate_memory_node(fdt, node, addr,
+                                                SPAPR_MEMORY_BLOCK_SIZE);
+
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+        drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, errp);
+        addr += SPAPR_MEMORY_BLOCK_SIZE;
+    }
+    /* send hotplug notification to the
+     * guest only in case of hotplugged memory
+     */
+    if (dev->hotplugged) {
+       spapr_hotplug_req_add_by_count(SPAPR_DR_CONNECTOR_TYPE_LMB, nr_lmbs);
+    }
+}
+
+static void spapr_memory_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                              uint32_t node, Error **errp)
+{
+    Error *local_err = NULL;
+    sPAPRMachineState *ms = SPAPR_MACHINE(hotplug_dev);
+    PCDIMMDevice *dimm = PC_DIMM(dev);
+    PCDIMMDeviceClass *ddc = PC_DIMM_GET_CLASS(dimm);
+    MemoryRegion *mr = ddc->get_memory_region(dimm);
+    uint64_t align = memory_region_get_alignment(mr);
+    uint64_t size = memory_region_size(mr);
+    uint64_t addr;
+
+    if (size % SPAPR_MEMORY_BLOCK_SIZE) {
+        error_setg(&local_err, "Hotplugged memory size must be a multiple of "
+                      "%lld MB", SPAPR_MEMORY_BLOCK_SIZE/M_BYTE);
+        goto out;
+    }
+
+    pc_dimm_memory_plug(dev, &ms->hotplug_memory, mr, align, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    addr = object_property_get_int(OBJECT(dimm), PC_DIMM_ADDR_PROP, &local_err);
+    if (local_err) {
+        pc_dimm_memory_unplug(dev, &ms->hotplug_memory, mr);
+        goto out;
+    }
+
+    spapr_add_lmbs(dev, addr, size, node, &error_abort);
+
+out:
+    error_propagate(errp, local_err);
+}
+
+void *spapr_populate_hotplug_cpu_dt(CPUState *cs, int *fdt_offset,
+                                    sPAPRMachineState *spapr)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    DeviceClass *dc = DEVICE_GET_CLASS(cs);
+    int id = ppc_get_vcpu_dt_id(cpu);
+    void *fdt;
+    int offset, fdt_size;
+    char *nodename;
+
+    fdt = create_device_tree(&fdt_size);
+    nodename = g_strdup_printf("%s@%x", dc->fw_name, id);
+    offset = fdt_add_subnode(fdt, 0, nodename);
+
+    spapr_populate_cpu_dt(cs, fdt, offset, spapr);
+    g_free(nodename);
+
+    *fdt_offset = offset;
+    return fdt;
+}
+
+static void spapr_machine_device_plug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_GET_CLASS(qdev_get_machine());
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        int node;
+
+        if (!smc->dr_lmb_enabled) {
+            error_setg(errp, "Memory hotplug not supported for this machine");
+            return;
+        }
+        node = object_property_get_int(OBJECT(dev), PC_DIMM_NODE_PROP, errp);
+        if (*errp) {
+            return;
+        }
+        if (node < 0 || node >= MAX_NODES) {
+            error_setg(errp, "Invaild node %d", node);
+            return;
+        }
+
+        /*
+         * Currently PowerPC kernel doesn't allow hot-adding memory to
+         * memory-less node, but instead will silently add the memory
+         * to the first node that has some memory. This causes two
+         * unexpected behaviours for the user.
+         *
+         * - Memory gets hotplugged to a different node than what the user
+         *   specified.
+         * - Since pc-dimm subsystem in QEMU still thinks that memory belongs
+         *   to memory-less node, a reboot will set things accordingly
+         *   and the previously hotplugged memory now ends in the right node.
+         *   This appears as if some memory moved from one node to another.
+         *
+         * So until kernel starts supporting memory hotplug to memory-less
+         * nodes, just prevent such attempts upfront in QEMU.
+         */
+        if (nb_numa_nodes && !numa_info[node].node_mem) {
+            error_setg(errp, "Can't hotplug memory to memory-less node %d",
+                       node);
+            return;
+        }
+
+        spapr_memory_plug(hotplug_dev, dev, node, errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        spapr_core_plug(hotplug_dev, dev, errp);
+    }
+}
+
+static void spapr_machine_device_unplug(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp)
+{
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM)) {
+        error_setg(errp, "Memory hot unplug not supported by sPAPR");
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        if (!mc->query_hotpluggable_cpus) {
+            error_setg(errp, "CPU hot unplug not supported on this machine");
+            return;
+        }
+        spapr_core_unplug(hotplug_dev, dev, errp);
+    }
+}
+
+static void spapr_machine_device_pre_plug(HotplugHandler *hotplug_dev,
+                                          DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        spapr_core_pre_plug(hotplug_dev, dev, errp);
+    }
+}
+
+static HotplugHandler *spapr_get_hotpug_handler(MachineState *machine,
+                                             DeviceState *dev)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PC_DIMM) ||
+        object_dynamic_cast(OBJECT(dev), TYPE_SPAPR_CPU_CORE)) {
+        return HOTPLUG_HANDLER(machine);
+    }
+    return NULL;
+}
+
+static unsigned spapr_cpu_index_to_socket_id(unsigned cpu_index)
+{
+    /* Allocate to NUMA nodes on a "socket" basis (not that concept of
+     * socket means much for the paravirtualized PAPR platform) */
+    return cpu_index / smp_threads / smp_cores;
+}
+
+static HotpluggableCPUList *spapr_query_hotpluggable_cpus(MachineState *machine)
+{
+    int i;
+    HotpluggableCPUList *head = NULL;
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    int spapr_max_cores = max_cpus / smp_threads;
+
+    for (i = 0; i < spapr_max_cores; i++) {
+        HotpluggableCPUList *list_item = g_new0(typeof(*list_item), 1);
+        HotpluggableCPU *cpu_item = g_new0(typeof(*cpu_item), 1);
+        CpuInstanceProperties *cpu_props = g_new0(typeof(*cpu_props), 1);
+
+        cpu_item->type = spapr_get_cpu_core_type(machine->cpu_model);
+        cpu_item->vcpus_count = smp_threads;
+        cpu_props->has_core_id = true;
+        cpu_props->core_id = i * smp_threads;
+        /* TODO: add 'has_node/node' here to describe
+           to which node core belongs */
+
+        cpu_item->props = cpu_props;
+        if (spapr->cores[i]) {
+            cpu_item->has_qom_path = true;
+            cpu_item->qom_path = object_get_canonical_path(spapr->cores[i]);
+        }
+        list_item->value = cpu_item;
+        list_item->next = head;
+        head = list_item;
+    }
+    return head;
+}
+
 static void spapr_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(oc);
     FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(oc);
     NMIClass *nc = NMI_CLASS(oc);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
 
+    mc->desc = "pSeries Logical Partition (PAPR compliant)";
+
+    /*
+     * We set up the default / latest behaviour here.  The class_init
+     * functions for the specific versioned machine types can override
+     * these details for backwards compatibility
+     */
     mc->init = ppc_spapr_init;
     mc->reset = ppc_spapr_reset;
     mc->block_default_type = IF_SCSI;
-    mc->max_cpus = MAX_CPUS;
+    mc->max_cpus = MAX_CPUMASK_BITS;
     mc->no_parallel = 1;
-    mc->default_boot_order = NULL;
+    mc->default_boot_order = "";
+    mc->default_ram_size = 512 * M_BYTE;
     mc->kvm_type = spapr_kvm_type;
     mc->has_dynamic_sysbus = true;
+    mc->pci_allow_0_address = true;
+    mc->get_hotplug_handler = spapr_get_hotpug_handler;
+    hc->pre_plug = spapr_machine_device_pre_plug;
+    hc->plug = spapr_machine_device_plug;
+    hc->unplug = spapr_machine_device_unplug;
+    mc->cpu_index_to_socket_id = spapr_cpu_index_to_socket_id;
 
+    smc->dr_lmb_enabled = true;
+    mc->query_hotpluggable_cpus = spapr_query_hotpluggable_cpus;
     fwc->get_dev_path = spapr_get_fw_dev_path;
     nc->nmi_monitor_handler = spapr_nmi;
 }
@@ -1699,54 +2446,203 @@
     .abstract      = true,
     .instance_size = sizeof(sPAPRMachineState),
     .instance_init = spapr_machine_initfn,
+    .instance_finalize = spapr_machine_finalizefn,
+    .class_size    = sizeof(sPAPRMachineClass),
     .class_init    = spapr_machine_class_init,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_FW_PATH_PROVIDER },
         { TYPE_NMI },
+        { TYPE_HOTPLUG_HANDLER },
         { }
     },
 };
 
-static void spapr_machine_2_1_class_init(ObjectClass *oc, void *data)
-{
-    MachineClass *mc = MACHINE_CLASS(oc);
-    static GlobalProperty compat_props[] = {
-        HW_COMPAT_2_1,
-        { /* end of list */ }
-    };
+#define DEFINE_SPAPR_MACHINE(suffix, verstr, latest)                 \
+    static void spapr_machine_##suffix##_class_init(ObjectClass *oc, \
+                                                    void *data)      \
+    {                                                                \
+        MachineClass *mc = MACHINE_CLASS(oc);                        \
+        spapr_machine_##suffix##_class_options(mc);                  \
+        if (latest) {                                                \
+            mc->alias = "pseries";                                   \
+            mc->is_default = 1;                                      \
+        }                                                            \
+    }                                                                \
+    static void spapr_machine_##suffix##_instance_init(Object *obj)  \
+    {                                                                \
+        MachineState *machine = MACHINE(obj);                        \
+        spapr_machine_##suffix##_instance_options(machine);          \
+    }                                                                \
+    static const TypeInfo spapr_machine_##suffix##_info = {          \
+        .name = MACHINE_TYPE_NAME("pseries-" verstr),                \
+        .parent = TYPE_SPAPR_MACHINE,                                \
+        .class_init = spapr_machine_##suffix##_class_init,           \
+        .instance_init = spapr_machine_##suffix##_instance_init,     \
+    };                                                               \
+    static void spapr_machine_register_##suffix(void)                \
+    {                                                                \
+        type_register(&spapr_machine_##suffix##_info);               \
+    }                                                                \
+    type_init(spapr_machine_register_##suffix)
 
-    mc->name = "pseries-2.1";
-    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.1";
-    mc->compat_props = compat_props;
+/*
+ * pseries-2.7
+ */
+static void spapr_machine_2_7_instance_options(MachineState *machine)
+{
 }
 
-static const TypeInfo spapr_machine_2_1_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.1",
-    .parent        = TYPE_SPAPR_MACHINE,
-    .class_init    = spapr_machine_2_1_class_init,
-};
-
-static void spapr_machine_2_2_class_init(ObjectClass *oc, void *data)
+static void spapr_machine_2_7_class_options(MachineClass *mc)
 {
-    MachineClass *mc = MACHINE_CLASS(oc);
-
-    mc->name = "pseries-2.2";
-    mc->desc = "pSeries Logical Partition (PAPR compliant) v2.2";
-    mc->alias = "pseries";
-    mc->is_default = 1;
+    /* Defaults for the latest behaviour inherited from the base class */
 }
 
-static const TypeInfo spapr_machine_2_2_info = {
-    .name          = TYPE_SPAPR_MACHINE "2.2",
-    .parent        = TYPE_SPAPR_MACHINE,
-    .class_init    = spapr_machine_2_2_class_init,
-};
+DEFINE_SPAPR_MACHINE(2_7, "2.7", true);
+
+/*
+ * pseries-2.6
+ */
+#define SPAPR_COMPAT_2_6 \
+    HW_COMPAT_2_6 \
+    { \
+        .driver   = TYPE_SPAPR_PCI_HOST_BRIDGE,\
+        .property = "ddw",\
+        .value    = stringify(off),\
+    },
+
+static void spapr_machine_2_6_instance_options(MachineState *machine)
+{
+}
+
+static void spapr_machine_2_6_class_options(MachineClass *mc)
+{
+    spapr_machine_2_7_class_options(mc);
+    mc->query_hotpluggable_cpus = NULL;
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_6);
+}
+
+DEFINE_SPAPR_MACHINE(2_6, "2.6", false);
+
+/*
+ * pseries-2.5
+ */
+#define SPAPR_COMPAT_2_5 \
+    HW_COMPAT_2_5 \
+    { \
+        .driver   = "spapr-vlan", \
+        .property = "use-rx-buffer-pools", \
+        .value    = "off", \
+    },
+
+static void spapr_machine_2_5_instance_options(MachineState *machine)
+{
+}
+
+static void spapr_machine_2_5_class_options(MachineClass *mc)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
+    spapr_machine_2_6_class_options(mc);
+    smc->use_ohci_by_default = true;
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_5);
+}
+
+DEFINE_SPAPR_MACHINE(2_5, "2.5", false);
+
+/*
+ * pseries-2.4
+ */
+#define SPAPR_COMPAT_2_4 \
+        HW_COMPAT_2_4
+
+static void spapr_machine_2_4_instance_options(MachineState *machine)
+{
+    spapr_machine_2_5_instance_options(machine);
+}
+
+static void spapr_machine_2_4_class_options(MachineClass *mc)
+{
+    sPAPRMachineClass *smc = SPAPR_MACHINE_CLASS(mc);
+
+    spapr_machine_2_5_class_options(mc);
+    smc->dr_lmb_enabled = false;
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_4);
+}
+
+DEFINE_SPAPR_MACHINE(2_4, "2.4", false);
+
+/*
+ * pseries-2.3
+ */
+#define SPAPR_COMPAT_2_3 \
+        HW_COMPAT_2_3 \
+        {\
+            .driver   = "spapr-pci-host-bridge",\
+            .property = "dynamic-reconfiguration",\
+            .value    = "off",\
+        },
+
+static void spapr_machine_2_3_instance_options(MachineState *machine)
+{
+    spapr_machine_2_4_instance_options(machine);
+    savevm_skip_section_footers();
+    global_state_set_optional();
+    savevm_skip_configuration();
+}
+
+static void spapr_machine_2_3_class_options(MachineClass *mc)
+{
+    spapr_machine_2_4_class_options(mc);
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_3);
+}
+DEFINE_SPAPR_MACHINE(2_3, "2.3", false);
+
+/*
+ * pseries-2.2
+ */
+
+#define SPAPR_COMPAT_2_2 \
+        HW_COMPAT_2_2 \
+        {\
+            .driver   = TYPE_SPAPR_PCI_HOST_BRIDGE,\
+            .property = "mem_win_size",\
+            .value    = "0x20000000",\
+        },
+
+static void spapr_machine_2_2_instance_options(MachineState *machine)
+{
+    spapr_machine_2_3_instance_options(machine);
+    machine->suppress_vmdesc = true;
+}
+
+static void spapr_machine_2_2_class_options(MachineClass *mc)
+{
+    spapr_machine_2_3_class_options(mc);
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_2);
+}
+DEFINE_SPAPR_MACHINE(2_2, "2.2", false);
+
+/*
+ * pseries-2.1
+ */
+#define SPAPR_COMPAT_2_1 \
+        HW_COMPAT_2_1
+
+static void spapr_machine_2_1_instance_options(MachineState *machine)
+{
+    spapr_machine_2_2_instance_options(machine);
+}
+
+static void spapr_machine_2_1_class_options(MachineClass *mc)
+{
+    spapr_machine_2_2_class_options(mc);
+    SET_MACHINE_COMPAT(mc, SPAPR_COMPAT_2_1);
+}
+DEFINE_SPAPR_MACHINE(2_1, "2.1", false);
 
 static void spapr_machine_register_types(void)
 {
     type_register_static(&spapr_machine_info);
-    type_register_static(&spapr_machine_2_1_info);
-    type_register_static(&spapr_machine_2_2_info);
 }
 
 type_init(spapr_machine_register_types)
diff --git a/hw/ppc/spapr_cpu_core.c b/hw/ppc/spapr_cpu_core.c
new file mode 100644
index 0000000..bcb483d
--- /dev/null
+++ b/hw/ppc/spapr_cpu_core.c
@@ -0,0 +1,432 @@
+/*
+ * sPAPR CPU core device, acts as container of CPU thread devices.
+ *
+ * Copyright (C) 2016 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "hw/cpu/core.h"
+#include "hw/ppc/spapr_cpu_core.h"
+#include "target-ppc/cpu.h"
+#include "hw/ppc/spapr.h"
+#include "hw/boards.h"
+#include "qapi/error.h"
+#include "sysemu/cpus.h"
+#include "target-ppc/kvm_ppc.h"
+#include "hw/ppc/ppc.h"
+#include "target-ppc/mmu-hash64.h"
+#include "sysemu/numa.h"
+
+static void spapr_cpu_reset(void *opaque)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    PowerPCCPU *cpu = opaque;
+    CPUState *cs = CPU(cpu);
+    CPUPPCState *env = &cpu->env;
+
+    cpu_reset(cs);
+
+    /* All CPUs start halted.  CPU0 is unhalted from the machine level
+     * reset code and the rest are explicitly started up by the guest
+     * using an RTAS call */
+    cs->halted = 1;
+
+    env->spr[SPR_HIOR] = 0;
+
+    ppc_hash64_set_external_hpt(cpu, spapr->htab, spapr->htab_shift,
+                                &error_fatal);
+}
+
+static void spapr_cpu_destroy(PowerPCCPU *cpu)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+
+    xics_cpu_destroy(spapr->xics, cpu);
+    qemu_unregister_reset(spapr_cpu_reset, cpu);
+}
+
+void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp)
+{
+    CPUPPCState *env = &cpu->env;
+    CPUState *cs = CPU(cpu);
+    int i;
+
+    /* Set time-base frequency to 512 MHz */
+    cpu_ppc_tb_init(env, SPAPR_TIMEBASE_FREQ);
+
+    /* Enable PAPR mode in TCG or KVM */
+    cpu_ppc_set_papr(cpu);
+
+    if (cpu->max_compat) {
+        Error *local_err = NULL;
+
+        ppc_set_compat(cpu, cpu->max_compat, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+    /* Set NUMA node for the added CPUs  */
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (test_bit(cs->cpu_index, numa_info[i].node_cpu)) {
+            cs->numa_node = i;
+            break;
+        }
+    }
+
+    xics_cpu_setup(spapr->xics, cpu);
+
+    qemu_register_reset(spapr_cpu_reset, cpu);
+    spapr_cpu_reset(cpu);
+}
+
+/*
+ * Return the sPAPR CPU core type for @model which essentially is the CPU
+ * model specified with -cpu cmdline option.
+ */
+char *spapr_get_cpu_core_type(const char *model)
+{
+    char *core_type;
+    gchar **model_pieces = g_strsplit(model, ",", 2);
+
+    core_type = g_strdup_printf("%s-%s", model_pieces[0], TYPE_SPAPR_CPU_CORE);
+    g_strfreev(model_pieces);
+
+    /* Check whether it exists or whether we have to look up an alias name */
+    if (!object_class_by_name(core_type)) {
+        const char *realmodel;
+
+        g_free(core_type);
+        realmodel = ppc_cpu_lookup_alias(model);
+        if (realmodel) {
+            return spapr_get_cpu_core_type(realmodel);
+        }
+        return NULL;
+    }
+
+    return core_type;
+}
+
+static void spapr_core_release(DeviceState *dev, void *opaque)
+{
+    sPAPRCPUCore *sc = SPAPR_CPU_CORE(OBJECT(dev));
+    const char *typename = object_class_get_name(sc->cpu_class);
+    size_t size = object_type_get_instance_size(typename);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    CPUCore *cc = CPU_CORE(dev);
+    int i;
+
+    for (i = 0; i < cc->nr_threads; i++) {
+        void *obj = sc->threads + i * size;
+        DeviceState *dev = DEVICE(obj);
+        CPUState *cs = CPU(dev);
+        PowerPCCPU *cpu = POWERPC_CPU(cs);
+
+        spapr_cpu_destroy(cpu);
+        cpu_remove_sync(cs);
+        object_unparent(obj);
+    }
+
+    spapr->cores[cc->core_id / smp_threads] = NULL;
+
+    g_free(sc->threads);
+    object_unparent(OBJECT(dev));
+}
+
+void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                       Error **errp)
+{
+    CPUCore *cc = CPU_CORE(dev);
+    int smt = kvmppc_smt_threads();
+    int index = cc->core_id / smp_threads;
+    sPAPRDRConnector *drc =
+        spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index * smt);
+    sPAPRDRConnectorClass *drck;
+    Error *local_err = NULL;
+
+    if (index == 0) {
+        error_setg(errp, "Boot CPU core may not be unplugged");
+        return;
+    }
+
+    g_assert(drc);
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->detach(drc, dev, spapr_core_release, NULL, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    spapr_hotplug_req_remove_by_index(drc);
+}
+
+void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                     Error **errp)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(OBJECT(dev));
+    CPUCore *cc = CPU_CORE(dev);
+    CPUState *cs = CPU(core->threads);
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    Error *local_err = NULL;
+    void *fdt = NULL;
+    int fdt_offset = 0;
+    int index = cc->core_id / smp_threads;
+    int smt = kvmppc_smt_threads();
+
+    drc = spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_CPU, index * smt);
+    spapr->cores[index] = OBJECT(dev);
+
+    g_assert(drc);
+
+    /*
+     * Setup CPU DT entries only for hotplugged CPUs. For boot time or
+     * coldplugged CPUs DT entries are setup in spapr_finalize_fdt().
+     */
+    if (dev->hotplugged) {
+        fdt = spapr_populate_hotplug_cpu_dt(cs, &fdt_offset, spapr);
+    }
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->attach(drc, dev, fdt, fdt_offset, !dev->hotplugged, &local_err);
+    if (local_err) {
+        g_free(fdt);
+        spapr->cores[index] = NULL;
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (dev->hotplugged) {
+        /*
+         * Send hotplug notification interrupt to the guest only in case
+         * of hotplugged CPUs.
+         */
+        spapr_hotplug_req_add_by_index(drc);
+    } else {
+        /*
+         * Set the right DRC states for cold plugged CPU.
+         */
+        drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_USABLE);
+        drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_UNISOLATED);
+    }
+}
+
+void spapr_core_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                         Error **errp)
+{
+    MachineState *machine = MACHINE(OBJECT(hotplug_dev));
+    MachineClass *mc = MACHINE_GET_CLASS(hotplug_dev);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(OBJECT(hotplug_dev));
+    int spapr_max_cores = max_cpus / smp_threads;
+    int index;
+    Error *local_err = NULL;
+    CPUCore *cc = CPU_CORE(dev);
+    char *base_core_type = spapr_get_cpu_core_type(machine->cpu_model);
+    const char *type = object_get_typename(OBJECT(dev));
+
+    if (!mc->query_hotpluggable_cpus) {
+        error_setg(&local_err, "CPU hotplug not supported for this machine");
+        goto out;
+    }
+
+    if (strcmp(base_core_type, type)) {
+        error_setg(&local_err, "CPU core type should be %s", base_core_type);
+        goto out;
+    }
+
+    if (cc->nr_threads != smp_threads) {
+        error_setg(&local_err, "threads must be %d", smp_threads);
+        goto out;
+    }
+
+    if (cc->core_id % smp_threads) {
+        error_setg(&local_err, "invalid core id %d", cc->core_id);
+        goto out;
+    }
+
+    index = cc->core_id / smp_threads;
+    if (index < 0 || index >= spapr_max_cores) {
+        error_setg(&local_err, "core id %d out of range", cc->core_id);
+        goto out;
+    }
+
+    if (spapr->cores[index]) {
+        error_setg(&local_err, "core %d already populated", cc->core_id);
+        goto out;
+    }
+
+out:
+    g_free(base_core_type);
+    error_propagate(errp, local_err);
+}
+
+static void spapr_cpu_core_realize_child(Object *child, Error **errp)
+{
+    Error *local_err = NULL;
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    CPUState *cs = CPU(child);
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+
+    object_property_set_bool(child, true, "realized", &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    spapr_cpu_init(spapr, cpu, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+}
+
+static void spapr_cpu_core_realize(DeviceState *dev, Error **errp)
+{
+    sPAPRCPUCore *sc = SPAPR_CPU_CORE(OBJECT(dev));
+    CPUCore *cc = CPU_CORE(OBJECT(dev));
+    const char *typename = object_class_get_name(sc->cpu_class);
+    size_t size = object_type_get_instance_size(typename);
+    Error *local_err = NULL;
+    void *obj;
+    int i, j;
+
+    sc->threads = g_malloc0(size * cc->nr_threads);
+    for (i = 0; i < cc->nr_threads; i++) {
+        char id[32];
+        CPUState *cs;
+
+        obj = sc->threads + i * size;
+
+        object_initialize(obj, size, typename);
+        cs = CPU(obj);
+        cs->cpu_index = cc->core_id + i;
+        snprintf(id, sizeof(id), "thread[%d]", i);
+        object_property_add_child(OBJECT(sc), id, obj, &local_err);
+        if (local_err) {
+            goto err;
+        }
+        object_unref(obj);
+    }
+
+    for (j = 0; j < cc->nr_threads; j++) {
+        obj = sc->threads + j * size;
+
+        spapr_cpu_core_realize_child(obj, &local_err);
+        if (local_err) {
+            goto err;
+        }
+    }
+    return;
+
+err:
+    while (--i >= 0) {
+        obj = sc->threads + i * size;
+        object_unparent(obj);
+    }
+    g_free(sc->threads);
+    error_propagate(errp, local_err);
+}
+
+static void spapr_cpu_core_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    dc->realize = spapr_cpu_core_realize;
+}
+
+/*
+ * instance_init routines from different flavours of sPAPR CPU cores.
+ */
+#define SPAPR_CPU_CORE_INITFN(_type, _fname) \
+static void glue(glue(spapr_cpu_core_, _fname), _initfn(Object *obj)) \
+{ \
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(obj); \
+    char *name = g_strdup_printf("%s-" TYPE_POWERPC_CPU, stringify(_type)); \
+    ObjectClass *oc = object_class_by_name(name); \
+    g_assert(oc); \
+    g_free((void *)name); \
+    core->cpu_class = oc; \
+}
+
+SPAPR_CPU_CORE_INITFN(970mp_v1.0, 970MP_v10);
+SPAPR_CPU_CORE_INITFN(970mp_v1.1, 970MP_v11);
+SPAPR_CPU_CORE_INITFN(970_v2.2, 970);
+SPAPR_CPU_CORE_INITFN(POWER5+_v2.1, POWER5plus);
+SPAPR_CPU_CORE_INITFN(POWER7_v2.3, POWER7);
+SPAPR_CPU_CORE_INITFN(POWER7+_v2.1, POWER7plus);
+SPAPR_CPU_CORE_INITFN(POWER8_v2.0, POWER8);
+SPAPR_CPU_CORE_INITFN(POWER8E_v2.1, POWER8E);
+SPAPR_CPU_CORE_INITFN(POWER8NVL_v1.0, POWER8NVL);
+
+typedef struct SPAPRCoreInfo {
+    const char *name;
+    void (*initfn)(Object *obj);
+} SPAPRCoreInfo;
+
+static const SPAPRCoreInfo spapr_cores[] = {
+    /* 970 */
+    { .name = "970_v2.2", .initfn = spapr_cpu_core_970_initfn },
+
+    /* 970MP variants */
+    { .name = "970MP_v1.0", .initfn = spapr_cpu_core_970MP_v10_initfn },
+    { .name = "970mp_v1.0", .initfn = spapr_cpu_core_970MP_v10_initfn },
+    { .name = "970MP_v1.1", .initfn = spapr_cpu_core_970MP_v11_initfn },
+    { .name = "970mp_v1.1", .initfn = spapr_cpu_core_970MP_v11_initfn },
+
+    /* POWER5+ */
+    { .name = "POWER5+_v2.1", .initfn = spapr_cpu_core_POWER5plus_initfn },
+
+    /* POWER7 */
+    { .name = "POWER7_v2.3", .initfn = spapr_cpu_core_POWER7_initfn },
+
+    /* POWER7+ */
+    { .name = "POWER7+_v2.1", .initfn = spapr_cpu_core_POWER7plus_initfn },
+
+    /* POWER8 */
+    { .name = "POWER8_v2.0", .initfn = spapr_cpu_core_POWER8_initfn },
+
+    /* POWER8E */
+    { .name = "POWER8E_v2.1", .initfn = spapr_cpu_core_POWER8E_initfn },
+
+    /* POWER8NVL */
+    { .name = "POWER8NVL_v1.0", .initfn = spapr_cpu_core_POWER8NVL_initfn },
+
+    { .name = NULL }
+};
+
+static void spapr_cpu_core_register(const SPAPRCoreInfo *info)
+{
+    TypeInfo type_info = {
+        .parent = TYPE_SPAPR_CPU_CORE,
+        .instance_size = sizeof(sPAPRCPUCore),
+        .instance_init = info->initfn,
+    };
+
+    type_info.name = g_strdup_printf("%s-" TYPE_SPAPR_CPU_CORE, info->name);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
+}
+
+static const TypeInfo spapr_cpu_core_type_info = {
+    .name = TYPE_SPAPR_CPU_CORE,
+    .parent = TYPE_CPU_CORE,
+    .abstract = true,
+    .instance_size = sizeof(sPAPRCPUCore),
+    .class_init = spapr_cpu_core_class_init,
+};
+
+static void spapr_cpu_core_register_types(void)
+{
+    const SPAPRCoreInfo *info = spapr_cores;
+
+    type_register_static(&spapr_cpu_core_type_info);
+    while (info->name) {
+        spapr_cpu_core_register(info);
+        info++;
+    }
+}
+
+type_init(spapr_cpu_core_register_types)
diff --git a/hw/ppc/spapr_drc.c b/hw/ppc/spapr_drc.c
new file mode 100644
index 0000000..26a0679
--- /dev/null
+++ b/hw/ppc/spapr_drc.c
@@ -0,0 +1,852 @@
+/*
+ * QEMU SPAPR Dynamic Reconfiguration Connector Implementation
+ *
+ * Copyright IBM Corp. 2014
+ *
+ * Authors:
+ *  Michael Roth      <mdroth@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu/cutils.h"
+#include "hw/ppc/spapr_drc.h"
+#include "qom/object.h"
+#include "hw/qdev.h"
+#include "qapi/visitor.h"
+#include "qemu/error-report.h"
+#include "hw/ppc/spapr.h" /* for RTAS return codes */
+
+/* #define DEBUG_SPAPR_DRC */
+
+#ifdef DEBUG_SPAPR_DRC
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
+#define DPRINTFN(fmt, ...) \
+    do { DPRINTF(fmt, ## __VA_ARGS__); fprintf(stderr, "\n"); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#define DPRINTFN(fmt, ...) \
+    do { } while (0)
+#endif
+
+#define DRC_CONTAINER_PATH "/dr-connector"
+#define DRC_INDEX_TYPE_SHIFT 28
+#define DRC_INDEX_ID_MASK ((1ULL << DRC_INDEX_TYPE_SHIFT) - 1)
+
+static sPAPRDRConnectorTypeShift get_type_shift(sPAPRDRConnectorType type)
+{
+    uint32_t shift = 0;
+
+    /* make sure this isn't SPAPR_DR_CONNECTOR_TYPE_ANY, or some
+     * other wonky value.
+     */
+    g_assert(is_power_of_2(type));
+
+    while (type != (1 << shift)) {
+        shift++;
+    }
+    return shift;
+}
+
+static uint32_t get_index(sPAPRDRConnector *drc)
+{
+    /* no set format for a drc index: it only needs to be globally
+     * unique. this is how we encode the DRC type on bare-metal
+     * however, so might as well do that here
+     */
+    return (get_type_shift(drc->type) << DRC_INDEX_TYPE_SHIFT) |
+            (drc->id & DRC_INDEX_ID_MASK);
+}
+
+static uint32_t set_isolation_state(sPAPRDRConnector *drc,
+                                    sPAPRDRIsolationState state)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+    DPRINTFN("drc: %x, set_isolation_state: %x", get_index(drc), state);
+
+    if (state == SPAPR_DR_ISOLATION_STATE_UNISOLATED) {
+        /* cannot unisolate a non-existant resource, and, or resources
+         * which are in an 'UNUSABLE' allocation state. (PAPR 2.7, 13.5.3.5)
+         */
+        if (!drc->dev ||
+            drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
+            return RTAS_OUT_NO_SUCH_INDICATOR;
+        }
+    }
+
+    drc->isolation_state = state;
+
+    if (drc->isolation_state == SPAPR_DR_ISOLATION_STATE_ISOLATED) {
+        /* if we're awaiting release, but still in an unconfigured state,
+         * it's likely the guest is still in the process of configuring
+         * the device and is transitioning the devices to an ISOLATED
+         * state as a part of that process. so we only complete the
+         * removal when this transition happens for a device in a
+         * configured state, as suggested by the state diagram from
+         * PAPR+ 2.7, 13.4
+         */
+        if (drc->awaiting_release) {
+            if (drc->configured) {
+                DPRINTFN("finalizing device removal");
+                drck->detach(drc, DEVICE(drc->dev), drc->detach_cb,
+                             drc->detach_cb_opaque, NULL);
+            } else {
+                DPRINTFN("deferring device removal on unconfigured device\n");
+            }
+        }
+        drc->configured = false;
+    }
+
+    return RTAS_OUT_SUCCESS;
+}
+
+static uint32_t set_indicator_state(sPAPRDRConnector *drc,
+                                    sPAPRDRIndicatorState state)
+{
+    DPRINTFN("drc: %x, set_indicator_state: %x", get_index(drc), state);
+    drc->indicator_state = state;
+    return RTAS_OUT_SUCCESS;
+}
+
+static uint32_t set_allocation_state(sPAPRDRConnector *drc,
+                                     sPAPRDRAllocationState state)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+    DPRINTFN("drc: %x, set_allocation_state: %x", get_index(drc), state);
+
+    if (state == SPAPR_DR_ALLOCATION_STATE_USABLE) {
+        /* if there's no resource/device associated with the DRC, there's
+         * no way for us to put it in an allocation state consistent with
+         * being 'USABLE'. PAPR 2.7, 13.5.3.4 documents that this should
+         * result in an RTAS return code of -3 / "no such indicator"
+         */
+        if (!drc->dev) {
+            return RTAS_OUT_NO_SUCH_INDICATOR;
+        }
+    }
+
+    if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI) {
+        drc->allocation_state = state;
+        if (drc->awaiting_release &&
+            drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
+            DPRINTFN("finalizing device removal");
+            drck->detach(drc, DEVICE(drc->dev), drc->detach_cb,
+                         drc->detach_cb_opaque, NULL);
+        } else if (drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_USABLE) {
+            drc->awaiting_allocation = false;
+        }
+    }
+    return RTAS_OUT_SUCCESS;
+}
+
+static uint32_t get_type(sPAPRDRConnector *drc)
+{
+    return drc->type;
+}
+
+static const char *get_name(sPAPRDRConnector *drc)
+{
+    return drc->name;
+}
+
+static const void *get_fdt(sPAPRDRConnector *drc, int *fdt_start_offset)
+{
+    if (fdt_start_offset) {
+        *fdt_start_offset = drc->fdt_start_offset;
+    }
+    return drc->fdt;
+}
+
+static void set_configured(sPAPRDRConnector *drc)
+{
+    DPRINTFN("drc: %x, set_configured", get_index(drc));
+
+    if (drc->isolation_state != SPAPR_DR_ISOLATION_STATE_UNISOLATED) {
+        /* guest should be not configuring an isolated device */
+        DPRINTFN("drc: %x, set_configured: skipping isolated device",
+                 get_index(drc));
+        return;
+    }
+    drc->configured = true;
+}
+
+/* has the guest been notified of device attachment? */
+static void set_signalled(sPAPRDRConnector *drc)
+{
+    drc->signalled = true;
+}
+
+/*
+ * dr-entity-sense sensor value
+ * returned via get-sensor-state RTAS calls
+ * as expected by state diagram in PAPR+ 2.7, 13.4
+ * based on the current allocation/indicator/power states
+ * for the DR connector.
+ */
+static uint32_t entity_sense(sPAPRDRConnector *drc, sPAPRDREntitySense *state)
+{
+    if (drc->dev) {
+        if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI &&
+            drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
+            /* for logical DR, we return a state of UNUSABLE
+             * iff the allocation state UNUSABLE.
+             * Otherwise, report the state as USABLE/PRESENT,
+             * as we would for PCI.
+             */
+            *state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
+        } else {
+            /* this assumes all PCI devices are assigned to
+             * a 'live insertion' power domain, where QEMU
+             * manages power state automatically as opposed
+             * to the guest. present, non-PCI resources are
+             * unaffected by power state.
+             */
+            *state = SPAPR_DR_ENTITY_SENSE_PRESENT;
+        }
+    } else {
+        if (drc->type == SPAPR_DR_CONNECTOR_TYPE_PCI) {
+            /* PCI devices, and only PCI devices, use EMPTY
+             * in cases where we'd otherwise use UNUSABLE
+             */
+            *state = SPAPR_DR_ENTITY_SENSE_EMPTY;
+        } else {
+            *state = SPAPR_DR_ENTITY_SENSE_UNUSABLE;
+        }
+    }
+
+    DPRINTFN("drc: %x, entity_sense: %x", get_index(drc), state);
+    return RTAS_OUT_SUCCESS;
+}
+
+static void prop_get_index(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    uint32_t value = (uint32_t)drck->get_index(drc);
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void prop_get_type(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    uint32_t value = (uint32_t)drck->get_type(drc);
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static char *prop_get_name(Object *obj, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    return g_strdup(drck->get_name(drc));
+}
+
+static void prop_get_entity_sense(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    uint32_t value;
+
+    drck->entity_sense(drc, &value);
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void prop_get_fdt(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+    Error *err = NULL;
+    int fdt_offset_next, fdt_offset, fdt_depth;
+    void *fdt;
+
+    if (!drc->fdt) {
+        visit_type_null(v, NULL, errp);
+        return;
+    }
+
+    fdt = drc->fdt;
+    fdt_offset = drc->fdt_start_offset;
+    fdt_depth = 0;
+
+    do {
+        const char *name = NULL;
+        const struct fdt_property *prop = NULL;
+        int prop_len = 0, name_len = 0;
+        uint32_t tag;
+
+        tag = fdt_next_tag(fdt, fdt_offset, &fdt_offset_next);
+        switch (tag) {
+        case FDT_BEGIN_NODE:
+            fdt_depth++;
+            name = fdt_get_name(fdt, fdt_offset, &name_len);
+            visit_start_struct(v, name, NULL, 0, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
+            break;
+        case FDT_END_NODE:
+            /* shouldn't ever see an FDT_END_NODE before FDT_BEGIN_NODE */
+            g_assert(fdt_depth > 0);
+            visit_check_struct(v, &err);
+            visit_end_struct(v, NULL);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
+            fdt_depth--;
+            break;
+        case FDT_PROP: {
+            int i;
+            prop = fdt_get_property_by_offset(fdt, fdt_offset, &prop_len);
+            name = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
+            visit_start_list(v, name, NULL, 0, &err);
+            if (err) {
+                error_propagate(errp, err);
+                return;
+            }
+            for (i = 0; i < prop_len; i++) {
+                visit_type_uint8(v, NULL, (uint8_t *)&prop->data[i], &err);
+                if (err) {
+                    error_propagate(errp, err);
+                    return;
+                }
+            }
+            visit_end_list(v, NULL);
+            break;
+        }
+        default:
+            error_setg(&error_abort, "device FDT in unexpected state: %d", tag);
+        }
+        fdt_offset = fdt_offset_next;
+    } while (fdt_depth != 0);
+}
+
+static void attach(sPAPRDRConnector *drc, DeviceState *d, void *fdt,
+                   int fdt_start_offset, bool coldplug, Error **errp)
+{
+    DPRINTFN("drc: %x, attach", get_index(drc));
+
+    if (drc->isolation_state != SPAPR_DR_ISOLATION_STATE_ISOLATED) {
+        error_setg(errp, "an attached device is still awaiting release");
+        return;
+    }
+    if (drc->type == SPAPR_DR_CONNECTOR_TYPE_PCI) {
+        g_assert(drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_USABLE);
+    }
+    g_assert(fdt || coldplug);
+
+    /* NOTE: setting initial isolation state to UNISOLATED means we can't
+     * detach unless guest has a userspace/kernel that moves this state
+     * back to ISOLATED in response to an unplug event, or this is done
+     * manually by the admin prior. if we force things while the guest
+     * may be accessing the device, we can easily crash the guest, so we
+     * we defer completion of removal in such cases to the reset() hook.
+     */
+    if (drc->type == SPAPR_DR_CONNECTOR_TYPE_PCI) {
+        drc->isolation_state = SPAPR_DR_ISOLATION_STATE_UNISOLATED;
+    }
+    drc->indicator_state = SPAPR_DR_INDICATOR_STATE_ACTIVE;
+
+    drc->dev = d;
+    drc->fdt = fdt;
+    drc->fdt_start_offset = fdt_start_offset;
+    drc->configured = coldplug;
+    /* 'logical' DR resources such as memory/cpus are in some cases treated
+     * as a pool of resources from which the guest is free to choose from
+     * based on only a count. for resources that can be assigned in this
+     * fashion, we must assume the resource is signalled immediately
+     * since a single hotplug request might make an arbitrary number of
+     * such attached resources available to the guest, as opposed to
+     * 'physical' DR resources such as PCI where each device/resource is
+     * signalled individually.
+     */
+    drc->signalled = (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI)
+                     ? true : coldplug;
+
+    if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI) {
+        drc->awaiting_allocation = true;
+    }
+
+    object_property_add_link(OBJECT(drc), "device",
+                             object_get_typename(OBJECT(drc->dev)),
+                             (Object **)(&drc->dev),
+                             NULL, 0, NULL);
+}
+
+static void detach(sPAPRDRConnector *drc, DeviceState *d,
+                   spapr_drc_detach_cb *detach_cb,
+                   void *detach_cb_opaque, Error **errp)
+{
+    DPRINTFN("drc: %x, detach", get_index(drc));
+
+    drc->detach_cb = detach_cb;
+    drc->detach_cb_opaque = detach_cb_opaque;
+
+    /* if we've signalled device presence to the guest, or if the guest
+     * has gone ahead and configured the device (via manually-executed
+     * device add via drmgr in guest, namely), we need to wait
+     * for the guest to quiesce the device before completing detach.
+     * Otherwise, we can assume the guest hasn't seen it and complete the
+     * detach immediately. Note that there is a small race window
+     * just before, or during, configuration, which is this context
+     * refers mainly to fetching the device tree via RTAS.
+     * During this window the device access will be arbitrated by
+     * associated DRC, which will simply fail the RTAS calls as invalid.
+     * This is recoverable within guest and current implementations of
+     * drmgr should be able to cope.
+     */
+    if (!drc->signalled && !drc->configured) {
+        /* if the guest hasn't seen the device we can't rely on it to
+         * set it back to an isolated state via RTAS, so do it here manually
+         */
+        drc->isolation_state = SPAPR_DR_ISOLATION_STATE_ISOLATED;
+    }
+
+    if (drc->isolation_state != SPAPR_DR_ISOLATION_STATE_ISOLATED) {
+        DPRINTFN("awaiting transition to isolated state before removal");
+        drc->awaiting_release = true;
+        return;
+    }
+
+    if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI &&
+        drc->allocation_state != SPAPR_DR_ALLOCATION_STATE_UNUSABLE) {
+        DPRINTFN("awaiting transition to unusable state before removal");
+        drc->awaiting_release = true;
+        return;
+    }
+
+    if (drc->awaiting_allocation) {
+        drc->awaiting_release = true;
+        DPRINTFN("awaiting allocation to complete before removal");
+        return;
+    }
+
+    drc->indicator_state = SPAPR_DR_INDICATOR_STATE_INACTIVE;
+
+    if (drc->detach_cb) {
+        drc->detach_cb(drc->dev, drc->detach_cb_opaque);
+    }
+
+    drc->awaiting_release = false;
+    g_free(drc->fdt);
+    drc->fdt = NULL;
+    drc->fdt_start_offset = 0;
+    object_property_del(OBJECT(drc), "device", NULL);
+    drc->dev = NULL;
+    drc->detach_cb = NULL;
+    drc->detach_cb_opaque = NULL;
+}
+
+static bool release_pending(sPAPRDRConnector *drc)
+{
+    return drc->awaiting_release;
+}
+
+static void reset(DeviceState *d)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(d);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDREntitySense state;
+
+    DPRINTFN("drc reset: %x", drck->get_index(drc));
+    /* immediately upon reset we can safely assume DRCs whose devices
+     * are pending removal can be safely removed, and that they will
+     * subsequently be left in an ISOLATED state. move the DRC to this
+     * state in these cases (which will in turn complete any pending
+     * device removals)
+     */
+    if (drc->awaiting_release) {
+        drck->set_isolation_state(drc, SPAPR_DR_ISOLATION_STATE_ISOLATED);
+        /* generally this should also finalize the removal, but if the device
+         * hasn't yet been configured we normally defer removal under the
+         * assumption that this transition is taking place as part of device
+         * configuration. so check if we're still waiting after this, and
+         * force removal if we are
+         */
+        if (drc->awaiting_release) {
+            drck->detach(drc, DEVICE(drc->dev), drc->detach_cb,
+                         drc->detach_cb_opaque, NULL);
+        }
+
+        /* non-PCI devices may be awaiting a transition to UNUSABLE */
+        if (drc->type != SPAPR_DR_CONNECTOR_TYPE_PCI &&
+            drc->awaiting_release) {
+            drck->set_allocation_state(drc, SPAPR_DR_ALLOCATION_STATE_UNUSABLE);
+        }
+    }
+
+    drck->entity_sense(drc, &state);
+    if (state == SPAPR_DR_ENTITY_SENSE_PRESENT) {
+        drck->set_signalled(drc);
+    }
+}
+
+static void realize(DeviceState *d, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(d);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    Object *root_container;
+    char link_name[256];
+    gchar *child_name;
+    Error *err = NULL;
+
+    DPRINTFN("drc realize: %x", drck->get_index(drc));
+    /* NOTE: we do this as part of realize/unrealize due to the fact
+     * that the guest will communicate with the DRC via RTAS calls
+     * referencing the global DRC index. By unlinking the DRC
+     * from DRC_CONTAINER_PATH/<drc_index> we effectively make it
+     * inaccessible by the guest, since lookups rely on this path
+     * existing in the composition tree
+     */
+    root_container = container_get(object_get_root(), DRC_CONTAINER_PATH);
+    snprintf(link_name, sizeof(link_name), "%x", drck->get_index(drc));
+    child_name = object_get_canonical_path_component(OBJECT(drc));
+    DPRINTFN("drc child name: %s", child_name);
+    object_property_add_alias(root_container, link_name,
+                              drc->owner, child_name, &err);
+    if (err) {
+        error_report_err(err);
+        object_unref(OBJECT(drc));
+    }
+    g_free(child_name);
+    DPRINTFN("drc realize complete");
+}
+
+static void unrealize(DeviceState *d, Error **errp)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(d);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    Object *root_container;
+    char name[256];
+    Error *err = NULL;
+
+    DPRINTFN("drc unrealize: %x", drck->get_index(drc));
+    root_container = container_get(object_get_root(), DRC_CONTAINER_PATH);
+    snprintf(name, sizeof(name), "%x", drck->get_index(drc));
+    object_property_del(root_container, name, &err);
+    if (err) {
+        error_report_err(err);
+        object_unref(OBJECT(drc));
+    }
+}
+
+sPAPRDRConnector *spapr_dr_connector_new(Object *owner,
+                                         sPAPRDRConnectorType type,
+                                         uint32_t id)
+{
+    sPAPRDRConnector *drc =
+        SPAPR_DR_CONNECTOR(object_new(TYPE_SPAPR_DR_CONNECTOR));
+    char *prop_name;
+
+    g_assert(type);
+
+    drc->type = type;
+    drc->id = id;
+    drc->owner = owner;
+    prop_name = g_strdup_printf("dr-connector[%"PRIu32"]", get_index(drc));
+    object_property_add_child(owner, prop_name, OBJECT(drc), NULL);
+    object_property_set_bool(OBJECT(drc), true, "realized", NULL);
+    g_free(prop_name);
+
+    /* human-readable name for a DRC to encode into the DT
+     * description. this is mainly only used within a guest in place
+     * of the unique DRC index.
+     *
+     * in the case of VIO/PCI devices, it corresponds to a
+     * "location code" that maps a logical device/function (DRC index)
+     * to a physical (or virtual in the case of VIO) location in the
+     * system by chaining together the "location label" for each
+     * encapsulating component.
+     *
+     * since this is more to do with diagnosing physical hardware
+     * issues than guest compatibility, we choose location codes/DRC
+     * names that adhere to the documented format, but avoid encoding
+     * the entire topology information into the label/code, instead
+     * just using the location codes based on the labels for the
+     * endpoints (VIO/PCI adaptor connectors), which is basically
+     * just "C" followed by an integer ID.
+     *
+     * DRC names as documented by PAPR+ v2.7, 13.5.2.4
+     * location codes as documented by PAPR+ v2.7, 12.3.1.5
+     */
+    switch (drc->type) {
+    case SPAPR_DR_CONNECTOR_TYPE_CPU:
+        drc->name = g_strdup_printf("CPU %d", id);
+        break;
+    case SPAPR_DR_CONNECTOR_TYPE_PHB:
+        drc->name = g_strdup_printf("PHB %d", id);
+        break;
+    case SPAPR_DR_CONNECTOR_TYPE_VIO:
+    case SPAPR_DR_CONNECTOR_TYPE_PCI:
+        drc->name = g_strdup_printf("C%d", id);
+        break;
+    case SPAPR_DR_CONNECTOR_TYPE_LMB:
+        drc->name = g_strdup_printf("LMB %d", id);
+        break;
+    default:
+        g_assert(false);
+    }
+
+    /* PCI slot always start in a USABLE state, and stay there */
+    if (drc->type == SPAPR_DR_CONNECTOR_TYPE_PCI) {
+        drc->allocation_state = SPAPR_DR_ALLOCATION_STATE_USABLE;
+    }
+
+    return drc;
+}
+
+static void spapr_dr_connector_instance_init(Object *obj)
+{
+    sPAPRDRConnector *drc = SPAPR_DR_CONNECTOR(obj);
+
+    object_property_add_uint32_ptr(obj, "isolation-state",
+                                   &drc->isolation_state, NULL);
+    object_property_add_uint32_ptr(obj, "indicator-state",
+                                   &drc->indicator_state, NULL);
+    object_property_add_uint32_ptr(obj, "allocation-state",
+                                   &drc->allocation_state, NULL);
+    object_property_add_uint32_ptr(obj, "id", &drc->id, NULL);
+    object_property_add(obj, "index", "uint32", prop_get_index,
+                        NULL, NULL, NULL, NULL);
+    object_property_add(obj, "connector_type", "uint32", prop_get_type,
+                        NULL, NULL, NULL, NULL);
+    object_property_add_str(obj, "name", prop_get_name, NULL, NULL);
+    object_property_add(obj, "entity-sense", "uint32", prop_get_entity_sense,
+                        NULL, NULL, NULL, NULL);
+    object_property_add(obj, "fdt", "struct", prop_get_fdt,
+                        NULL, NULL, NULL, NULL);
+}
+
+static void spapr_dr_connector_class_init(ObjectClass *k, void *data)
+{
+    DeviceClass *dk = DEVICE_CLASS(k);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_CLASS(k);
+
+    dk->reset = reset;
+    dk->realize = realize;
+    dk->unrealize = unrealize;
+    drck->set_isolation_state = set_isolation_state;
+    drck->set_indicator_state = set_indicator_state;
+    drck->set_allocation_state = set_allocation_state;
+    drck->get_index = get_index;
+    drck->get_type = get_type;
+    drck->get_name = get_name;
+    drck->get_fdt = get_fdt;
+    drck->set_configured = set_configured;
+    drck->entity_sense = entity_sense;
+    drck->attach = attach;
+    drck->detach = detach;
+    drck->release_pending = release_pending;
+    drck->set_signalled = set_signalled;
+    /*
+     * Reason: it crashes FIXME find and document the real reason
+     */
+    dk->cannot_instantiate_with_device_add_yet = true;
+}
+
+static const TypeInfo spapr_dr_connector_info = {
+    .name          = TYPE_SPAPR_DR_CONNECTOR,
+    .parent        = TYPE_DEVICE,
+    .instance_size = sizeof(sPAPRDRConnector),
+    .instance_init = spapr_dr_connector_instance_init,
+    .class_size    = sizeof(sPAPRDRConnectorClass),
+    .class_init    = spapr_dr_connector_class_init,
+};
+
+static void spapr_drc_register_types(void)
+{
+    type_register_static(&spapr_dr_connector_info);
+}
+
+type_init(spapr_drc_register_types)
+
+/* helper functions for external users */
+
+sPAPRDRConnector *spapr_dr_connector_by_index(uint32_t index)
+{
+    Object *obj;
+    char name[256];
+
+    snprintf(name, sizeof(name), "%s/%x", DRC_CONTAINER_PATH, index);
+    obj = object_resolve_path(name, NULL);
+
+    return !obj ? NULL : SPAPR_DR_CONNECTOR(obj);
+}
+
+sPAPRDRConnector *spapr_dr_connector_by_id(sPAPRDRConnectorType type,
+                                           uint32_t id)
+{
+    return spapr_dr_connector_by_index(
+            (get_type_shift(type) << DRC_INDEX_TYPE_SHIFT) |
+            (id & DRC_INDEX_ID_MASK));
+}
+
+/* generate a string the describes the DRC to encode into the
+ * device tree.
+ *
+ * as documented by PAPR+ v2.7, 13.5.2.6 and C.6.1
+ */
+static const char *spapr_drc_get_type_str(sPAPRDRConnectorType type)
+{
+    switch (type) {
+    case SPAPR_DR_CONNECTOR_TYPE_CPU:
+        return "CPU";
+    case SPAPR_DR_CONNECTOR_TYPE_PHB:
+        return "PHB";
+    case SPAPR_DR_CONNECTOR_TYPE_VIO:
+        return "SLOT";
+    case SPAPR_DR_CONNECTOR_TYPE_PCI:
+        return "28";
+    case SPAPR_DR_CONNECTOR_TYPE_LMB:
+        return "MEM";
+    default:
+        g_assert(false);
+    }
+
+    return NULL;
+}
+
+/**
+ * spapr_drc_populate_dt
+ *
+ * @fdt: libfdt device tree
+ * @path: path in the DT to generate properties
+ * @owner: parent Object/DeviceState for which to generate DRC
+ *         descriptions for
+ * @drc_type_mask: mask of sPAPRDRConnectorType values corresponding
+ *   to the types of DRCs to generate entries for
+ *
+ * generate OF properties to describe DRC topology/indices to guests
+ *
+ * as documented in PAPR+ v2.1, 13.5.2
+ */
+int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
+                          uint32_t drc_type_mask)
+{
+    Object *root_container;
+    ObjectProperty *prop;
+    ObjectPropertyIterator iter;
+    uint32_t drc_count = 0;
+    GArray *drc_indexes, *drc_power_domains;
+    GString *drc_names, *drc_types;
+    int ret;
+
+    /* the first entry of each properties is a 32-bit integer encoding
+     * the number of elements in the array. we won't know this until
+     * we complete the iteration through all the matching DRCs, but
+     * reserve the space now and set the offsets accordingly so we
+     * can fill them in later.
+     */
+    drc_indexes = g_array_new(false, true, sizeof(uint32_t));
+    drc_indexes = g_array_set_size(drc_indexes, 1);
+    drc_power_domains = g_array_new(false, true, sizeof(uint32_t));
+    drc_power_domains = g_array_set_size(drc_power_domains, 1);
+    drc_names = g_string_set_size(g_string_new(NULL), sizeof(uint32_t));
+    drc_types = g_string_set_size(g_string_new(NULL), sizeof(uint32_t));
+
+    /* aliases for all DRConnector objects will be rooted in QOM
+     * composition tree at DRC_CONTAINER_PATH
+     */
+    root_container = container_get(object_get_root(), DRC_CONTAINER_PATH);
+
+    object_property_iter_init(&iter, root_container);
+    while ((prop = object_property_iter_next(&iter))) {
+        Object *obj;
+        sPAPRDRConnector *drc;
+        sPAPRDRConnectorClass *drck;
+        uint32_t drc_index, drc_power_domain;
+
+        if (!strstart(prop->type, "link<", NULL)) {
+            continue;
+        }
+
+        obj = object_property_get_link(root_container, prop->name, NULL);
+        drc = SPAPR_DR_CONNECTOR(obj);
+        drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+        if (owner && (drc->owner != owner)) {
+            continue;
+        }
+
+        if ((drc->type & drc_type_mask) == 0) {
+            continue;
+        }
+
+        drc_count++;
+
+        /* ibm,drc-indexes */
+        drc_index = cpu_to_be32(drck->get_index(drc));
+        g_array_append_val(drc_indexes, drc_index);
+
+        /* ibm,drc-power-domains */
+        drc_power_domain = cpu_to_be32(-1);
+        g_array_append_val(drc_power_domains, drc_power_domain);
+
+        /* ibm,drc-names */
+        drc_names = g_string_append(drc_names, drck->get_name(drc));
+        drc_names = g_string_insert_len(drc_names, -1, "\0", 1);
+
+        /* ibm,drc-types */
+        drc_types = g_string_append(drc_types,
+                                    spapr_drc_get_type_str(drc->type));
+        drc_types = g_string_insert_len(drc_types, -1, "\0", 1);
+    }
+
+    /* now write the drc count into the space we reserved at the
+     * beginning of the arrays previously
+     */
+    *(uint32_t *)drc_indexes->data = cpu_to_be32(drc_count);
+    *(uint32_t *)drc_power_domains->data = cpu_to_be32(drc_count);
+    *(uint32_t *)drc_names->str = cpu_to_be32(drc_count);
+    *(uint32_t *)drc_types->str = cpu_to_be32(drc_count);
+
+    ret = fdt_setprop(fdt, fdt_offset, "ibm,drc-indexes",
+                      drc_indexes->data,
+                      drc_indexes->len * sizeof(uint32_t));
+    if (ret) {
+        fprintf(stderr, "Couldn't create ibm,drc-indexes property\n");
+        goto out;
+    }
+
+    ret = fdt_setprop(fdt, fdt_offset, "ibm,drc-power-domains",
+                      drc_power_domains->data,
+                      drc_power_domains->len * sizeof(uint32_t));
+    if (ret) {
+        fprintf(stderr, "Couldn't finalize ibm,drc-power-domains property\n");
+        goto out;
+    }
+
+    ret = fdt_setprop(fdt, fdt_offset, "ibm,drc-names",
+                      drc_names->str, drc_names->len);
+    if (ret) {
+        fprintf(stderr, "Couldn't finalize ibm,drc-names property\n");
+        goto out;
+    }
+
+    ret = fdt_setprop(fdt, fdt_offset, "ibm,drc-types",
+                      drc_types->str, drc_types->len);
+    if (ret) {
+        fprintf(stderr, "Couldn't finalize ibm,drc-types property\n");
+        goto out;
+    }
+
+out:
+    g_array_free(drc_indexes, true);
+    g_array_free(drc_power_domains, true);
+    g_string_free(drc_names, true);
+    g_string_free(drc_types, true);
+
+    return ret;
+}
diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c
index 1b6157d..b0668b3 100644
--- a/hw/ppc/spapr_events.c
+++ b/hw/ppc/spapr_events.c
@@ -24,6 +24,8 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
@@ -32,7 +34,11 @@
 
 #include "hw/ppc/spapr.h"
 #include "hw/ppc/spapr_vio.h"
-
+#include "hw/pci/pci.h"
+#include "hw/pci-host/spapr.h"
+#include "hw/ppc/spapr_drc.h"
+#include "qemu/help_option.h"
+#include "qemu/bcd.h"
 #include <libfdt.h>
 
 struct rtas_error_log {
@@ -77,6 +83,7 @@
 #define   RTAS_LOG_TYPE_ECC_UNCORR              0x00000009
 #define   RTAS_LOG_TYPE_ECC_CORR                0x0000000a
 #define   RTAS_LOG_TYPE_EPOW                    0x00000040
+#define   RTAS_LOG_TYPE_HOTPLUG                 0x000000e5
     uint32_t extended_length;
 } QEMU_PACKED;
 
@@ -166,6 +173,38 @@
     struct rtas_event_log_v6_epow epow;
 } QEMU_PACKED;
 
+struct rtas_event_log_v6_hp {
+#define RTAS_LOG_V6_SECTION_ID_HOTPLUG              0x4850 /* HP */
+    struct rtas_event_log_v6_section_header hdr;
+    uint8_t hotplug_type;
+#define RTAS_LOG_V6_HP_TYPE_CPU                          1
+#define RTAS_LOG_V6_HP_TYPE_MEMORY                       2
+#define RTAS_LOG_V6_HP_TYPE_SLOT                         3
+#define RTAS_LOG_V6_HP_TYPE_PHB                          4
+#define RTAS_LOG_V6_HP_TYPE_PCI                          5
+    uint8_t hotplug_action;
+#define RTAS_LOG_V6_HP_ACTION_ADD                        1
+#define RTAS_LOG_V6_HP_ACTION_REMOVE                     2
+    uint8_t hotplug_identifier;
+#define RTAS_LOG_V6_HP_ID_DRC_NAME                       1
+#define RTAS_LOG_V6_HP_ID_DRC_INDEX                      2
+#define RTAS_LOG_V6_HP_ID_DRC_COUNT                      3
+    uint8_t reserved;
+    union {
+        uint32_t index;
+        uint32_t count;
+        char name[1];
+    } drc;
+} QEMU_PACKED;
+
+struct hp_log_full {
+    struct rtas_error_log hdr;
+    struct rtas_event_log_v6 v6hdr;
+    struct rtas_event_log_v6_maina maina;
+    struct rtas_event_log_v6_mainb mainb;
+    struct rtas_event_log_v6_hp hp;
+} QEMU_PACKED;
+
 #define EVENT_MASK_INTERNAL_ERRORS           0x80000000
 #define EVENT_MASK_EPOW                      0x40000000
 #define EVENT_MASK_HOTPLUG                   0x10000000
@@ -181,72 +220,114 @@
         }                                                          \
     } while (0)
 
-void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq)
+void spapr_events_fdt_skel(void *fdt, uint32_t check_exception_irq)
 {
-    uint32_t epow_irq_ranges[] = {cpu_to_be32(epow_irq), cpu_to_be32(1)};
-    uint32_t epow_interrupts[] = {cpu_to_be32(epow_irq), 0};
+    uint32_t irq_ranges[] = {cpu_to_be32(check_exception_irq), cpu_to_be32(1)};
+    uint32_t interrupts[] = {cpu_to_be32(check_exception_irq), 0};
 
     _FDT((fdt_begin_node(fdt, "event-sources")));
 
     _FDT((fdt_property(fdt, "interrupt-controller", NULL, 0)));
     _FDT((fdt_property_cell(fdt, "#interrupt-cells", 2)));
     _FDT((fdt_property(fdt, "interrupt-ranges",
-                       epow_irq_ranges, sizeof(epow_irq_ranges))));
+                       irq_ranges, sizeof(irq_ranges))));
 
     _FDT((fdt_begin_node(fdt, "epow-events")));
-    _FDT((fdt_property(fdt, "interrupts",
-                       epow_interrupts, sizeof(epow_interrupts))));
+    _FDT((fdt_property(fdt, "interrupts", interrupts, sizeof(interrupts))));
     _FDT((fdt_end_node(fdt)));
 
     _FDT((fdt_end_node(fdt)));
 }
 
-static struct epow_log_full *pending_epow;
+static void rtas_event_log_queue(int log_type, void *data, bool exception)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    sPAPREventLogEntry *entry = g_new(sPAPREventLogEntry, 1);
+
+    g_assert(data);
+    entry->log_type = log_type;
+    entry->exception = exception;
+    entry->data = data;
+    QTAILQ_INSERT_TAIL(&spapr->pending_events, entry, next);
+}
+
+static sPAPREventLogEntry *rtas_event_log_dequeue(uint32_t event_mask,
+                                                  bool exception)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    sPAPREventLogEntry *entry = NULL;
+
+    /* we only queue EPOW events atm. */
+    if ((event_mask & EVENT_MASK_EPOW) == 0) {
+        return NULL;
+    }
+
+    QTAILQ_FOREACH(entry, &spapr->pending_events, next) {
+        if (entry->exception != exception) {
+            continue;
+        }
+
+        /* EPOW and hotplug events are surfaced in the same manner */
+        if (entry->log_type == RTAS_LOG_TYPE_EPOW ||
+            entry->log_type == RTAS_LOG_TYPE_HOTPLUG) {
+            break;
+        }
+    }
+
+    if (entry) {
+        QTAILQ_REMOVE(&spapr->pending_events, entry, next);
+    }
+
+    return entry;
+}
+
+static bool rtas_event_log_contains(uint32_t event_mask, bool exception)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    sPAPREventLogEntry *entry = NULL;
+
+    /* we only queue EPOW events atm. */
+    if ((event_mask & EVENT_MASK_EPOW) == 0) {
+        return false;
+    }
+
+    QTAILQ_FOREACH(entry, &spapr->pending_events, next) {
+        if (entry->exception != exception) {
+            continue;
+        }
+
+        /* EPOW and hotplug events are surfaced in the same manner */
+        if (entry->log_type == RTAS_LOG_TYPE_EPOW ||
+            entry->log_type == RTAS_LOG_TYPE_HOTPLUG) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 static uint32_t next_plid;
 
-static void spapr_powerdown_req(Notifier *n, void *opaque)
+static void spapr_init_v6hdr(struct rtas_event_log_v6 *v6hdr)
 {
-    sPAPREnvironment *spapr = container_of(n, sPAPREnvironment, epow_notifier);
-    struct rtas_error_log *hdr;
-    struct rtas_event_log_v6 *v6hdr;
-    struct rtas_event_log_v6_maina *maina;
-    struct rtas_event_log_v6_mainb *mainb;
-    struct rtas_event_log_v6_epow *epow;
-    struct tm tm;
-    int year;
-
-    if (pending_epow) {
-        /* For now, we just throw away earlier events if two come
-         * along before any are consumed.  This is sufficient for our
-         * powerdown messages, but we'll need more if we do more
-         * general error/event logging */
-        g_free(pending_epow);
-    }
-    pending_epow = g_malloc0(sizeof(*pending_epow));
-    hdr = &pending_epow->hdr;
-    v6hdr = &pending_epow->v6hdr;
-    maina = &pending_epow->maina;
-    mainb = &pending_epow->mainb;
-    epow = &pending_epow->epow;
-
-    hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
-                               | RTAS_LOG_SEVERITY_EVENT
-                               | RTAS_LOG_DISPOSITION_NOT_RECOVERED
-                               | RTAS_LOG_OPTIONAL_PART_PRESENT
-                               | RTAS_LOG_TYPE_EPOW);
-    hdr->extended_length = cpu_to_be32(sizeof(*pending_epow)
-                                       - sizeof(pending_epow->hdr));
-
     v6hdr->b0 = RTAS_LOG_V6_B0_VALID | RTAS_LOG_V6_B0_NEW_LOG
         | RTAS_LOG_V6_B0_BIGENDIAN;
     v6hdr->b2 = RTAS_LOG_V6_B2_POWERPC_FORMAT
         | RTAS_LOG_V6_B2_LOG_FORMAT_PLATFORM_EVENT;
     v6hdr->company = cpu_to_be32(RTAS_LOG_V6_COMPANY_IBM);
+}
+
+static void spapr_init_maina(struct rtas_event_log_v6_maina *maina,
+                             int section_count)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    struct tm tm;
+    int year;
 
     maina->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINA);
     maina->hdr.section_length = cpu_to_be16(sizeof(*maina));
     /* FIXME: section version, subtype and creator id? */
-    qemu_get_timedate(&tm, spapr->rtc_offset);
+    spapr_rtc_read(spapr->rtc, &tm, NULL);
     year = tm.tm_year + 1900;
     maina->creation_date = cpu_to_be32((to_bcd(year / 100) << 24)
                                        | (to_bcd(year % 100) << 16)
@@ -256,8 +337,37 @@
                                        | (to_bcd(tm.tm_min) << 16)
                                        | (to_bcd(tm.tm_sec) << 8));
     maina->creator_id = 'H'; /* Hypervisor */
-    maina->section_count = 3; /* Main-A, Main-B and EPOW */
+    maina->section_count = section_count;
     maina->plid = next_plid++;
+}
+
+static void spapr_powerdown_req(Notifier *n, void *opaque)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    struct rtas_error_log *hdr;
+    struct rtas_event_log_v6 *v6hdr;
+    struct rtas_event_log_v6_maina *maina;
+    struct rtas_event_log_v6_mainb *mainb;
+    struct rtas_event_log_v6_epow *epow;
+    struct epow_log_full *new_epow;
+
+    new_epow = g_malloc0(sizeof(*new_epow));
+    hdr = &new_epow->hdr;
+    v6hdr = &new_epow->v6hdr;
+    maina = &new_epow->maina;
+    mainb = &new_epow->mainb;
+    epow = &new_epow->epow;
+
+    hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
+                               | RTAS_LOG_SEVERITY_EVENT
+                               | RTAS_LOG_DISPOSITION_NOT_RECOVERED
+                               | RTAS_LOG_OPTIONAL_PART_PRESENT
+                               | RTAS_LOG_TYPE_EPOW);
+    hdr->extended_length = cpu_to_be32(sizeof(*new_epow)
+                                       - sizeof(new_epow->hdr));
+
+    spapr_init_v6hdr(v6hdr);
+    spapr_init_maina(maina, 3 /* Main-A, Main-B and EPOW */);
 
     mainb->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINB);
     mainb->hdr.section_length = cpu_to_be16(sizeof(*mainb));
@@ -274,16 +384,136 @@
     epow->event_modifier = RTAS_LOG_V6_EPOW_MODIFIER_NORMAL;
     epow->extended_modifier = RTAS_LOG_V6_EPOW_XMODIFIER_PARTITION_SPECIFIC;
 
-    qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->epow_irq));
+    rtas_event_log_queue(RTAS_LOG_TYPE_EPOW, new_epow, true);
+
+    qemu_irq_pulse(xics_get_qirq(spapr->xics, spapr->check_exception_irq));
 }
 
-static void check_exception(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void spapr_hotplug_set_signalled(uint32_t drc_index)
+{
+    sPAPRDRConnector *drc = spapr_dr_connector_by_index(drc_index);
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    drck->set_signalled(drc);
+}
+
+static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
+                                    sPAPRDRConnectorType drc_type,
+                                    uint32_t drc)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    struct hp_log_full *new_hp;
+    struct rtas_error_log *hdr;
+    struct rtas_event_log_v6 *v6hdr;
+    struct rtas_event_log_v6_maina *maina;
+    struct rtas_event_log_v6_mainb *mainb;
+    struct rtas_event_log_v6_hp *hp;
+
+    new_hp = g_malloc0(sizeof(struct hp_log_full));
+    hdr = &new_hp->hdr;
+    v6hdr = &new_hp->v6hdr;
+    maina = &new_hp->maina;
+    mainb = &new_hp->mainb;
+    hp = &new_hp->hp;
+
+    hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
+                               | RTAS_LOG_SEVERITY_EVENT
+                               | RTAS_LOG_DISPOSITION_NOT_RECOVERED
+                               | RTAS_LOG_OPTIONAL_PART_PRESENT
+                               | RTAS_LOG_INITIATOR_HOTPLUG
+                               | RTAS_LOG_TYPE_HOTPLUG);
+    hdr->extended_length = cpu_to_be32(sizeof(*new_hp)
+                                       - sizeof(new_hp->hdr));
+
+    spapr_init_v6hdr(v6hdr);
+    spapr_init_maina(maina, 3 /* Main-A, Main-B, HP */);
+
+    mainb->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINB);
+    mainb->hdr.section_length = cpu_to_be16(sizeof(*mainb));
+    mainb->subsystem_id = 0x80; /* External environment */
+    mainb->event_severity = 0x00; /* Informational / non-error */
+    mainb->event_subtype = 0x00; /* Normal shutdown */
+
+    hp->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_HOTPLUG);
+    hp->hdr.section_length = cpu_to_be16(sizeof(*hp));
+    hp->hdr.section_version = 1; /* includes extended modifier */
+    hp->hotplug_action = hp_action;
+    hp->hotplug_identifier = hp_id;
+
+    switch (drc_type) {
+    case SPAPR_DR_CONNECTOR_TYPE_PCI:
+        hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_PCI;
+        if (hp->hotplug_action == RTAS_LOG_V6_HP_ACTION_ADD) {
+            spapr_hotplug_set_signalled(drc);
+        }
+        break;
+    case SPAPR_DR_CONNECTOR_TYPE_LMB:
+        hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_MEMORY;
+        break;
+    case SPAPR_DR_CONNECTOR_TYPE_CPU:
+        hp->hotplug_type = RTAS_LOG_V6_HP_TYPE_CPU;
+        break;
+    default:
+        /* we shouldn't be signaling hotplug events for resources
+         * that don't support them
+         */
+        g_assert(false);
+        return;
+    }
+
+    if (hp_id == RTAS_LOG_V6_HP_ID_DRC_COUNT) {
+        hp->drc.count = cpu_to_be32(drc);
+    } else if (hp_id == RTAS_LOG_V6_HP_ID_DRC_INDEX) {
+        hp->drc.index = cpu_to_be32(drc);
+    }
+
+    rtas_event_log_queue(RTAS_LOG_TYPE_HOTPLUG, new_hp, true);
+
+    qemu_irq_pulse(xics_get_qirq(spapr->xics, spapr->check_exception_irq));
+}
+
+void spapr_hotplug_req_add_by_index(sPAPRDRConnector *drc)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDRConnectorType drc_type = drck->get_type(drc);
+    uint32_t index = drck->get_index(drc);
+
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_INDEX,
+                            RTAS_LOG_V6_HP_ACTION_ADD, drc_type, index);
+}
+
+void spapr_hotplug_req_remove_by_index(sPAPRDRConnector *drc)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    sPAPRDRConnectorType drc_type = drck->get_type(drc);
+    uint32_t index = drck->get_index(drc);
+
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_INDEX,
+                            RTAS_LOG_V6_HP_ACTION_REMOVE, drc_type, index);
+}
+
+void spapr_hotplug_req_add_by_count(sPAPRDRConnectorType drc_type,
+                                       uint32_t count)
+{
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_COUNT,
+                            RTAS_LOG_V6_HP_ACTION_ADD, drc_type, count);
+}
+
+void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
+                                          uint32_t count)
+{
+    spapr_hotplug_req_event(RTAS_LOG_V6_HP_ID_DRC_COUNT,
+                            RTAS_LOG_V6_HP_ACTION_REMOVE, drc_type, count);
+}
+
+static void check_exception(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                             uint32_t token, uint32_t nargs,
                             target_ulong args,
                             uint32_t nret, target_ulong rets)
 {
-    uint32_t mask, buf, len;
+    uint32_t mask, buf, len, event_len;
     uint64_t xinfo;
+    sPAPREventLogEntry *event;
+    struct rtas_error_log *hdr;
 
     if ((nargs < 6) || (nargs > 7) || nret != 1) {
         rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
@@ -298,25 +528,86 @@
         xinfo |= (uint64_t)rtas_ld(args, 6) << 32;
     }
 
-    if ((mask & EVENT_MASK_EPOW) && pending_epow) {
-        if (sizeof(*pending_epow) < len) {
-            len = sizeof(*pending_epow);
-        }
-
-        cpu_physical_memory_write(buf, pending_epow, len);
-        g_free(pending_epow);
-        pending_epow = NULL;
-        rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-    } else {
-        rtas_st(rets, 0, RTAS_OUT_NO_ERRORS_FOUND);
+    event = rtas_event_log_dequeue(mask, true);
+    if (!event) {
+        goto out_no_events;
     }
+
+    hdr = event->data;
+    event_len = be32_to_cpu(hdr->extended_length) + sizeof(*hdr);
+
+    if (event_len < len) {
+        len = event_len;
+    }
+
+    cpu_physical_memory_write(buf, event->data, len);
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    g_free(event->data);
+    g_free(event);
+
+    /* according to PAPR+, the IRQ must be left asserted, or re-asserted, if
+     * there are still pending events to be fetched via check-exception. We
+     * do the latter here, since our code relies on edge-triggered
+     * interrupts.
+     */
+    if (rtas_event_log_contains(mask, true)) {
+        qemu_irq_pulse(xics_get_qirq(spapr->xics, spapr->check_exception_irq));
+    }
+
+    return;
+
+out_no_events:
+    rtas_st(rets, 0, RTAS_OUT_NO_ERRORS_FOUND);
 }
 
-void spapr_events_init(sPAPREnvironment *spapr)
+static void event_scan(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                       uint32_t token, uint32_t nargs,
+                       target_ulong args,
+                       uint32_t nret, target_ulong rets)
 {
-    spapr->epow_irq = xics_alloc(spapr->icp, 0, 0, false);
+    uint32_t mask, buf, len, event_len;
+    sPAPREventLogEntry *event;
+    struct rtas_error_log *hdr;
+
+    if (nargs != 4 || nret != 1) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    mask = rtas_ld(args, 0);
+    buf = rtas_ld(args, 2);
+    len = rtas_ld(args, 3);
+
+    event = rtas_event_log_dequeue(mask, false);
+    if (!event) {
+        goto out_no_events;
+    }
+
+    hdr = event->data;
+    event_len = be32_to_cpu(hdr->extended_length) + sizeof(*hdr);
+
+    if (event_len < len) {
+        len = event_len;
+    }
+
+    cpu_physical_memory_write(buf, event->data, len);
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    g_free(event->data);
+    g_free(event);
+    return;
+
+out_no_events:
+    rtas_st(rets, 0, RTAS_OUT_NO_ERRORS_FOUND);
+}
+
+void spapr_events_init(sPAPRMachineState *spapr)
+{
+    QTAILQ_INIT(&spapr->pending_events);
+    spapr->check_exception_irq = xics_spapr_alloc(spapr->xics, 0, 0, false,
+                                            &error_fatal);
     spapr->epow_notifier.notify = spapr_powerdown_req;
     qemu_register_powerdown_notifier(&spapr->epow_notifier);
     spapr_rtas_register(RTAS_CHECK_EXCEPTION, "check-exception",
                         check_exception);
+    spapr_rtas_register(RTAS_EVENT_SCAN, "event-scan", event_scan);
 }
diff --git a/hw/ppc/spapr_hcall.c b/hw/ppc/spapr_hcall.c
index 8651447..73af112 100644
--- a/hw/ppc/spapr_hcall.c
+++ b/hw/ppc/spapr_hcall.c
@@ -1,10 +1,15 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
+#include "qemu/log.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "helper_regs.h"
 #include "hw/ppc/spapr.h"
 #include "mmu-hash64.h"
 #include "cpu-models.h"
 #include "trace.h"
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 
 struct SPRSyncState {
@@ -37,40 +42,10 @@
     run_on_cpu(cs, do_spr_sync, &s);
 }
 
-static target_ulong compute_tlbie_rb(target_ulong v, target_ulong r,
-                                     target_ulong pte_index)
+static bool has_spr(PowerPCCPU *cpu, int spr)
 {
-    target_ulong rb, va_low;
-
-    rb = (v & ~0x7fULL) << 16; /* AVA field */
-    va_low = pte_index >> 3;
-    if (v & HPTE64_V_SECONDARY) {
-        va_low = ~va_low;
-    }
-    /* xor vsid from AVA */
-    if (!(v & HPTE64_V_1TB_SEG)) {
-        va_low ^= v >> 12;
-    } else {
-        va_low ^= v >> 24;
-    }
-    va_low &= 0x7ff;
-    if (v & HPTE64_V_LARGE) {
-        rb |= 1;                         /* L field */
-#if 0 /* Disable that P7 specific bit for now */
-        if (r & 0xff000) {
-            /* non-16MB large page, must be 64k */
-            /* (masks depend on page size) */
-            rb |= 0x1000;                /* page encoding in LP field */
-            rb |= (va_low & 0x7f) << 16; /* 7b of VA in AVA/LP field */
-            rb |= (va_low & 0xfe);       /* AVAL field */
-        }
-#endif
-    } else {
-        /* 4kB page */
-        rb |= (va_low & 0x7ff) << 12;   /* remaining 11b of AVA */
-    }
-    rb |= (v >> 54) & 0x300;            /* B field */
-    return rb;
+    /* We can test whether the SPR is defined by checking for a valid name */
+    return cpu->env.spr_cb[spr].name != NULL;
 }
 
 static inline bool valid_pte_index(CPUPPCState *env, target_ulong pte_index)
@@ -84,7 +59,23 @@
     return true;
 }
 
-static target_ulong h_enter(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static bool is_ram_address(sPAPRMachineState *spapr, hwaddr addr)
+{
+    MachineState *machine = MACHINE(spapr);
+    MemoryHotplugState *hpms = &spapr->hotplug_memory;
+
+    if (addr < machine->ram_size) {
+        return true;
+    }
+    if ((addr >= hpms->base)
+        && ((addr - hpms->base) < memory_region_size(&hpms->mr))) {
+        return true;
+    }
+
+    return false;
+}
+
+static target_ulong h_enter(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                             target_ulong opcode, target_ulong *args)
 {
     CPUPPCState *env = &cpu->env;
@@ -92,43 +83,34 @@
     target_ulong pte_index = args[1];
     target_ulong pteh = args[2];
     target_ulong ptel = args[3];
-    target_ulong page_shift = 12;
+    unsigned apshift;
     target_ulong raddr;
     target_ulong index;
     uint64_t token;
 
-    /* only handle 4k and 16M pages for now */
-    if (pteh & HPTE64_V_LARGE) {
-#if 0 /* We don't support 64k pages yet */
-        if ((ptel & 0xf000) == 0x1000) {
-            /* 64k page */
-        } else
-#endif
-        if ((ptel & 0xff000) == 0) {
-            /* 16M page */
-            page_shift = 24;
-            /* lowest AVA bit must be 0 for 16M pages */
-            if (pteh & 0x80) {
-                return H_PARAMETER;
-            }
-        } else {
-            return H_PARAMETER;
-        }
+    apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
+    if (!apshift) {
+        /* Bad page size encoding */
+        return H_PARAMETER;
     }
 
-    raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << page_shift) - 1);
+    raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
 
-    if (raddr < spapr->ram_limit) {
+    if (is_ram_address(spapr, raddr)) {
         /* Regular RAM - should have WIMG=0010 */
         if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
             return H_PARAMETER;
         }
     } else {
+        target_ulong wimg_flags;
         /* Looks like an IO address */
         /* FIXME: What WIMG combinations could be sensible for IO?
          * For now we allow WIMG=010x, but are there others? */
         /* FIXME: Should we check against registered IO addresses? */
-        if ((ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M)) != HPTE64_R_I) {
+        wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
+
+        if (wimg_flags != HPTE64_R_I &&
+            wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
             return H_PARAMETER;
         }
     }
@@ -144,24 +126,24 @@
         pte_index &= ~7ULL;
         token = ppc_hash64_start_access(cpu, pte_index);
         for (; index < 8; index++) {
-            if ((ppc_hash64_load_hpte0(env, token, index) & HPTE64_V_VALID) == 0) {
+            if (!(ppc_hash64_load_hpte0(cpu, token, index) & HPTE64_V_VALID)) {
                 break;
             }
         }
-        ppc_hash64_stop_access(token);
+        ppc_hash64_stop_access(cpu, token);
         if (index == 8) {
             return H_PTEG_FULL;
         }
     } else {
         token = ppc_hash64_start_access(cpu, pte_index);
-        if (ppc_hash64_load_hpte0(env, token, 0) & HPTE64_V_VALID) {
-            ppc_hash64_stop_access(token);
+        if (ppc_hash64_load_hpte0(cpu, token, 0) & HPTE64_V_VALID) {
+            ppc_hash64_stop_access(cpu, token);
             return H_PTEG_FULL;
         }
-        ppc_hash64_stop_access(token);
+        ppc_hash64_stop_access(cpu, token);
     }
 
-    ppc_hash64_store_hpte(env, pte_index + index,
+    ppc_hash64_store_hpte(cpu, pte_index + index,
                           pteh | HPTE64_V_HPTE_DIRTY, ptel);
 
     args[0] = pte_index + index;
@@ -175,22 +157,23 @@
     REMOVE_HW = 3,
 } RemoveResult;
 
-static RemoveResult remove_hpte(CPUPPCState *env, target_ulong ptex,
+static RemoveResult remove_hpte(PowerPCCPU *cpu, target_ulong ptex,
                                 target_ulong avpn,
                                 target_ulong flags,
                                 target_ulong *vp, target_ulong *rp)
 {
+    CPUPPCState *env = &cpu->env;
     uint64_t token;
-    target_ulong v, r, rb;
+    target_ulong v, r;
 
     if (!valid_pte_index(env, ptex)) {
         return REMOVE_PARM;
     }
 
-    token = ppc_hash64_start_access(ppc_env_get_cpu(env), ptex);
-    v = ppc_hash64_load_hpte0(env, token, 0);
-    r = ppc_hash64_load_hpte1(env, token, 0);
-    ppc_hash64_stop_access(token);
+    token = ppc_hash64_start_access(cpu, ptex);
+    v = ppc_hash64_load_hpte0(cpu, token, 0);
+    r = ppc_hash64_load_hpte1(cpu, token, 0);
+    ppc_hash64_stop_access(cpu, token);
 
     if ((v & HPTE64_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
@@ -199,13 +182,12 @@
     }
     *vp = v;
     *rp = r;
-    ppc_hash64_store_hpte(env, ptex, HPTE64_V_HPTE_DIRTY, 0);
-    rb = compute_tlbie_rb(v, r, ptex);
-    ppc_tlb_invalidate_one(env, rb);
+    ppc_hash64_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
+    ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
     return REMOVE_SUCCESS;
 }
 
-static target_ulong h_remove(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                              target_ulong opcode, target_ulong *args)
 {
     CPUPPCState *env = &cpu->env;
@@ -214,11 +196,12 @@
     target_ulong avpn = args[2];
     RemoveResult ret;
 
-    ret = remove_hpte(env, pte_index, avpn, flags,
+    ret = remove_hpte(cpu, pte_index, avpn, flags,
                       &args[0], &args[1]);
 
     switch (ret) {
     case REMOVE_SUCCESS:
+        check_tlb_flush(env);
         return H_SUCCESS;
 
     case REMOVE_NOT_FOUND:
@@ -252,11 +235,12 @@
 
 #define H_BULK_REMOVE_MAX_BATCH        4
 
-static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                   target_ulong opcode, target_ulong *args)
 {
     CPUPPCState *env = &cpu->env;
     int i;
+    target_ulong rc = H_SUCCESS;
 
     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
         target_ulong *tsh = &args[i*2];
@@ -277,7 +261,7 @@
             return H_PARAMETER;
         }
 
-        ret = remove_hpte(env, *tsh & H_BULK_REMOVE_PTEX, tsl,
+        ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
                           (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
                           &v, &r);
 
@@ -289,17 +273,21 @@
             break;
 
         case REMOVE_PARM:
-            return H_PARAMETER;
+            rc = H_PARAMETER;
+            goto exit;
 
         case REMOVE_HW:
-            return H_HARDWARE;
+            rc = H_HARDWARE;
+            goto exit;
         }
     }
+ exit:
+    check_tlb_flush(env);
 
-    return H_SUCCESS;
+    return rc;
 }
 
-static target_ulong h_protect(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_protect(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     CPUPPCState *env = &cpu->env;
@@ -307,16 +295,16 @@
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
     uint64_t token;
-    target_ulong v, r, rb;
+    target_ulong v, r;
 
     if (!valid_pte_index(env, pte_index)) {
         return H_PARAMETER;
     }
 
     token = ppc_hash64_start_access(cpu, pte_index);
-    v = ppc_hash64_load_hpte0(env, token, 0);
-    r = ppc_hash64_load_hpte1(env, token, 0);
-    ppc_hash64_stop_access(token);
+    v = ppc_hash64_load_hpte0(cpu, token, 0);
+    r = ppc_hash64_load_hpte1(cpu, token, 0);
+    ppc_hash64_stop_access(cpu, token);
 
     if ((v & HPTE64_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
@@ -328,16 +316,15 @@
     r |= (flags << 55) & HPTE64_R_PP0;
     r |= (flags << 48) & HPTE64_R_KEY_HI;
     r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
-    rb = compute_tlbie_rb(v, r, pte_index);
-    ppc_hash64_store_hpte(env, pte_index,
+    ppc_hash64_store_hpte(cpu, pte_index,
                           (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
-    ppc_tlb_invalidate_one(env, rb);
+    ppc_hash64_tlb_flush_hpte(cpu, pte_index, v, r);
     /* Don't need a memory barrier, due to qemu's global lock */
-    ppc_hash64_store_hpte(env, pte_index, v | HPTE64_V_HPTE_DIRTY, r);
+    ppc_hash64_store_hpte(cpu, pte_index, v | HPTE64_V_HPTE_DIRTY, r);
     return H_SUCCESS;
 }
 
-static target_ulong h_read(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_read(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                            target_ulong opcode, target_ulong *args)
 {
     CPUPPCState *env = &cpu->env;
@@ -367,11 +354,111 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_set_dabr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_set_sprg0(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                target_ulong opcode, target_ulong *args)
+{
+    cpu_synchronize_state(CPU(cpu));
+    cpu->env.spr[SPR_SPRG0] = args[0];
+
+    return H_SUCCESS;
+}
+
+static target_ulong h_set_dabr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                target_ulong opcode, target_ulong *args)
 {
-    /* FIXME: actually implement this */
-    return H_HARDWARE;
+    if (!has_spr(cpu, SPR_DABR)) {
+        return H_HARDWARE;              /* DABR register not available */
+    }
+    cpu_synchronize_state(CPU(cpu));
+
+    if (has_spr(cpu, SPR_DABRX)) {
+        cpu->env.spr[SPR_DABRX] = 0x3;  /* Use Problem and Privileged state */
+    } else if (!(args[0] & 0x4)) {      /* Breakpoint Translation set? */
+        return H_RESERVED_DABR;
+    }
+
+    cpu->env.spr[SPR_DABR] = args[0];
+    return H_SUCCESS;
+}
+
+static target_ulong h_set_xdabr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                target_ulong opcode, target_ulong *args)
+{
+    target_ulong dabrx = args[1];
+
+    if (!has_spr(cpu, SPR_DABR) || !has_spr(cpu, SPR_DABRX)) {
+        return H_HARDWARE;
+    }
+
+    if ((dabrx & ~0xfULL) != 0 || (dabrx & H_DABRX_HYPERVISOR) != 0
+        || (dabrx & (H_DABRX_KERNEL | H_DABRX_USER)) == 0) {
+        return H_PARAMETER;
+    }
+
+    cpu_synchronize_state(CPU(cpu));
+    cpu->env.spr[SPR_DABRX] = dabrx;
+    cpu->env.spr[SPR_DABR] = args[0];
+
+    return H_SUCCESS;
+}
+
+static target_ulong h_page_init(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                target_ulong opcode, target_ulong *args)
+{
+    target_ulong flags = args[0];
+    hwaddr dst = args[1];
+    hwaddr src = args[2];
+    hwaddr len = TARGET_PAGE_SIZE;
+    uint8_t *pdst, *psrc;
+    target_long ret = H_SUCCESS;
+
+    if (flags & ~(H_ICACHE_SYNCHRONIZE | H_ICACHE_INVALIDATE
+                  | H_COPY_PAGE | H_ZERO_PAGE)) {
+        qemu_log_mask(LOG_UNIMP, "h_page_init: Bad flags (" TARGET_FMT_lx "\n",
+                      flags);
+        return H_PARAMETER;
+    }
+
+    /* Map-in destination */
+    if (!is_ram_address(spapr, dst) || (dst & ~TARGET_PAGE_MASK) != 0) {
+        return H_PARAMETER;
+    }
+    pdst = cpu_physical_memory_map(dst, &len, 1);
+    if (!pdst || len != TARGET_PAGE_SIZE) {
+        return H_PARAMETER;
+    }
+
+    if (flags & H_COPY_PAGE) {
+        /* Map-in source, copy to destination, and unmap source again */
+        if (!is_ram_address(spapr, src) || (src & ~TARGET_PAGE_MASK) != 0) {
+            ret = H_PARAMETER;
+            goto unmap_out;
+        }
+        psrc = cpu_physical_memory_map(src, &len, 0);
+        if (!psrc || len != TARGET_PAGE_SIZE) {
+            ret = H_PARAMETER;
+            goto unmap_out;
+        }
+        memcpy(pdst, psrc, len);
+        cpu_physical_memory_unmap(psrc, len, 0, len);
+    } else if (flags & H_ZERO_PAGE) {
+        memset(pdst, 0, len);          /* Just clear the destination page */
+    }
+
+    if (kvm_enabled() && (flags & H_ICACHE_SYNCHRONIZE) != 0) {
+        kvmppc_dcbst_range(cpu, pdst, len);
+    }
+    if (flags & (H_ICACHE_SYNCHRONIZE | H_ICACHE_INVALIDATE)) {
+        if (kvm_enabled()) {
+            kvmppc_icbi_range(cpu, pdst, len);
+        } else {
+            tb_flush(CPU(cpu));
+        }
+    }
+
+unmap_out:
+    cpu_physical_memory_unmap(pdst, TARGET_PAGE_SIZE, 1, len);
+    return ret;
 }
 
 #define FLAGS_REGISTER_VPA         0x0000200000000000ULL
@@ -506,7 +593,7 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_register_vpa(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_register_vpa(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     target_ulong flags = args[0];
@@ -551,7 +638,7 @@
     return ret;
 }
 
-static target_ulong h_cede(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_cede(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                            target_ulong opcode, target_ulong *args)
 {
     CPUPPCState *env = &cpu->env;
@@ -567,7 +654,7 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_rtas(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_rtas(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                            target_ulong opcode, target_ulong *args)
 {
     target_ulong rtas_r3 = args[0];
@@ -579,7 +666,7 @@
                            nret, rtas_r3 + 12 + 4*nargs);
 }
 
-static target_ulong h_logical_load(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_logical_load(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     CPUState *cs = CPU(cpu);
@@ -603,7 +690,7 @@
     return H_PARAMETER;
 }
 
-static target_ulong h_logical_store(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_logical_store(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     CPUState *cs = CPU(cpu);
@@ -629,7 +716,7 @@
     return H_PARAMETER;
 }
 
-static target_ulong h_logical_memop(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_logical_memop(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     CPUState *cs = CPU(cpu);
@@ -698,14 +785,14 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_logical_icbi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_logical_icbi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     /* Nothing to do on emulation, KVM will trap this in the kernel */
     return H_SUCCESS;
 }
 
-static target_ulong h_logical_dcbf(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_logical_dcbf(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     /* Nothing to do on emulation, KVM will trap this in the kernel */
@@ -731,12 +818,14 @@
         CPU_FOREACH(cs) {
             set_spr(cs, SPR_LPCR, 0, LPCR_ILE);
         }
+        spapr_pci_switch_vga(true);
         return H_SUCCESS;
 
     case H_SET_MODE_ENDIAN_LITTLE:
         CPU_FOREACH(cs) {
             set_spr(cs, SPR_LPCR, LPCR_ILE, LPCR_ILE);
         }
+        spapr_pci_switch_vga(false);
         return H_SUCCESS;
     }
 
@@ -750,7 +839,6 @@
 {
     CPUState *cs;
     PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
-    target_ulong prefix;
 
     if (!(pcc->insns_flags2 & PPC2_ISA207S)) {
         return H_P2;
@@ -762,31 +850,18 @@
         return H_P4;
     }
 
-    switch (mflags) {
-    case H_SET_MODE_ADDR_TRANS_NONE:
-        prefix = 0;
-        break;
-    case H_SET_MODE_ADDR_TRANS_0001_8000:
-        prefix = 0x18000;
-        break;
-    case H_SET_MODE_ADDR_TRANS_C000_0000_0000_4000:
-        prefix = 0xC000000000004000ULL;
-        break;
-    default:
+    if (mflags == AIL_RESERVED) {
         return H_UNSUPPORTED_FLAG;
     }
 
     CPU_FOREACH(cs) {
-        CPUPPCState *env = &POWERPC_CPU(cpu)->env;
-
         set_spr(cs, SPR_LPCR, mflags << LPCR_AIL_SHIFT, LPCR_AIL);
-        env->excp_prefix = prefix;
     }
 
     return H_SUCCESS;
 }
 
-static target_ulong h_set_mode(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_set_mode(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong resource = args[1];
@@ -805,10 +880,36 @@
     return ret;
 }
 
+/*
+ * Return the offset to the requested option vector @vector in the
+ * option vector table @table.
+ */
+static target_ulong cas_get_option_vector(int vector, target_ulong table)
+{
+    int i;
+    char nr_vectors, nr_entries;
+
+    if (!table) {
+        return 0;
+    }
+
+    nr_vectors = (ldl_phys(&address_space_memory, table) >> 24) + 1;
+    if (!vector || vector > nr_vectors) {
+        return 0;
+    }
+    table++; /* skip nr option vectors */
+
+    for (i = 0; i < vector - 1; i++) {
+        nr_entries = ldl_phys(&address_space_memory, table) >> 24;
+        table += nr_entries + 2;
+    }
+    return table;
+}
+
 typedef struct {
     PowerPCCPU *cpu;
     uint32_t cpu_version;
-    int ret;
+    Error *err;
 } SetCompatState;
 
 static void do_set_compat(void *arg)
@@ -816,7 +917,7 @@
     SetCompatState *s = arg;
 
     cpu_synchronize_state(CPU(s->cpu));
-    s->ret = ppc_set_compat(s->cpu, s->cpu_version);
+    ppc_set_compat(s->cpu, s->cpu_version, &s->err);
 }
 
 #define get_compat_level(cpuver) ( \
@@ -825,27 +926,66 @@
     ((cpuver) == CPU_POWERPC_LOGICAL_2_06_PLUS) ? 2061 : \
     ((cpuver) == CPU_POWERPC_LOGICAL_2_07) ? 2070 : 0)
 
+static void cas_handle_compat_cpu(PowerPCCPUClass *pcc, uint32_t pvr,
+                                  unsigned max_lvl, unsigned *compat_lvl,
+                                  unsigned *cpu_version)
+{
+    unsigned lvl = get_compat_level(pvr);
+    bool is205, is206, is207;
+
+    if (!lvl) {
+        return;
+    }
+
+    /* If it is a logical PVR, try to determine the highest level */
+    is205 = (pcc->pcr_supported & PCR_COMPAT_2_05) &&
+            (lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_05));
+    is206 = (pcc->pcr_supported & PCR_COMPAT_2_06) &&
+            ((lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_06)) ||
+             (lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_06_PLUS)));
+    is207 = (pcc->pcr_supported & PCR_COMPAT_2_07) &&
+            (lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_07));
+
+    if (is205 || is206 || is207) {
+        if (!max_lvl) {
+            /* User did not set the level, choose the highest */
+            if (*compat_lvl <= lvl) {
+                *compat_lvl = lvl;
+                *cpu_version = pvr;
+            }
+        } else if (max_lvl >= lvl) {
+            /* User chose the level, don't set higher than this */
+            *compat_lvl = lvl;
+            *cpu_version = pvr;
+        }
+    }
+}
+
+#define OV5_DRCONF_MEMORY 0x20
+
 static target_ulong h_client_architecture_support(PowerPCCPU *cpu_,
-                                                  sPAPREnvironment *spapr,
+                                                  sPAPRMachineState *spapr,
                                                   target_ulong opcode,
                                                   target_ulong *args)
 {
-    target_ulong list = args[0];
-    PowerPCCPUClass *pcc_ = POWERPC_CPU_GET_CLASS(cpu_);
+    target_ulong list = ppc64_phys_to_real(args[0]);
+    target_ulong ov_table, ov5;
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu_);
     CPUState *cs;
-    bool cpu_match = false;
+    bool cpu_match = false, cpu_update = true, memory_update = false;
     unsigned old_cpu_version = cpu_->cpu_version;
     unsigned compat_lvl = 0, cpu_version = 0;
     unsigned max_lvl = get_compat_level(cpu_->max_compat);
     int counter;
+    char ov5_byte2;
 
     /* Parse PVR list */
     for (counter = 0; counter < 512; ++counter) {
         uint32_t pvr, pvr_mask;
 
-        pvr_mask = rtas_ld(list, 0);
+        pvr_mask = ldl_be_phys(&address_space_memory, list);
         list += 4;
-        pvr = rtas_ld(list, 0);
+        pvr = ldl_be_phys(&address_space_memory, list);
         list += 4;
 
         trace_spapr_cas_pvr_try(pvr);
@@ -857,29 +997,7 @@
             cpu_match = true;
             cpu_version = cpu_->cpu_version;
         } else if (!cpu_match) {
-            /* If it is a logical PVR, try to determine the highest level */
-            unsigned lvl = get_compat_level(pvr);
-            if (lvl) {
-                bool is205 = (pcc_->pcr_mask & PCR_COMPAT_2_05) &&
-                     (lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_05));
-                bool is206 = (pcc_->pcr_mask & PCR_COMPAT_2_06) &&
-                    ((lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_06)) ||
-                    (lvl == get_compat_level(CPU_POWERPC_LOGICAL_2_06_PLUS)));
-
-                if (is205 || is206) {
-                    if (!max_lvl) {
-                        /* User did not set the level, choose the highest */
-                        if (compat_lvl <= lvl) {
-                            compat_lvl = lvl;
-                            cpu_version = pvr;
-                        }
-                    } else if (max_lvl >= lvl) {
-                        /* User chose the level, don't set higher than this */
-                        compat_lvl = lvl;
-                        cpu_version = pvr;
-                    }
-                }
-            }
+            cas_handle_compat_cpu(pcc, pvr, max_lvl, &compat_lvl, &cpu_version);
         }
         /* Terminator record */
         if (~pvr_mask & pvr) {
@@ -887,11 +1005,9 @@
         }
     }
 
-    /* For the future use: here @list points to the first capability */
-
     /* Parsing finished */
     trace_spapr_cas_pvr(cpu_->cpu_version, cpu_match,
-                        cpu_version, pcc_->pcr_mask);
+                        cpu_version, pcc->pcr_mask);
 
     /* Update CPUs */
     if (old_cpu_version != cpu_version) {
@@ -899,27 +1015,38 @@
             SetCompatState s = {
                 .cpu = POWERPC_CPU(cs),
                 .cpu_version = cpu_version,
-                .ret = 0
+                .err = NULL,
             };
 
             run_on_cpu(cs, do_set_compat, &s);
 
-            if (s.ret < 0) {
-                fprintf(stderr, "Unable to set compatibility mode\n");
+            if (s.err) {
+                error_report_err(s.err);
                 return H_HARDWARE;
             }
         }
     }
 
     if (!cpu_version) {
+        cpu_update = false;
+    }
+
+    /* For the future use: here @ov_table points to the first option vector */
+    ov_table = list;
+
+    ov5 = cas_get_option_vector(5, ov_table);
+    if (!ov5) {
         return H_SUCCESS;
     }
 
-    if (!list) {
-        return H_SUCCESS;
+    /* @list now points to OV 5 */
+    ov5_byte2 = ldub_phys(&address_space_memory, ov5 + 2);
+    if (ov5_byte2 & OV5_DRCONF_MEMORY) {
+        memory_update = true;
     }
 
-    if (spapr_h_cas_compose_response(args[1], args[2])) {
+    if (spapr_h_cas_compose_response(spapr, args[1], args[2],
+                                     cpu_update, memory_update)) {
         qemu_system_reset_request();
     }
 
@@ -950,6 +1077,8 @@
 target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args)
 {
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+
     if ((opcode <= MAX_HCALL_OPCODE)
         && ((opcode & 0x3) == 0)) {
         spapr_hcall_fn fn = papr_hypercall_table[opcode / 4];
@@ -966,7 +1095,8 @@
         }
     }
 
-    hcall_dprintf("Unimplemented hcall 0x" TARGET_FMT_lx "\n", opcode);
+    qemu_log_mask(LOG_UNIMP, "Unimplemented SPAPR hcall 0x" TARGET_FMT_lx "\n",
+                  opcode);
     return H_FUNCTION;
 }
 
@@ -981,13 +1111,17 @@
     /* hcall-bulk */
     spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
 
-    /* hcall-dabr */
-    spapr_register_hypercall(H_SET_DABR, h_set_dabr);
-
     /* hcall-splpar */
     spapr_register_hypercall(H_REGISTER_VPA, h_register_vpa);
     spapr_register_hypercall(H_CEDE, h_cede);
 
+    /* processor register resource access h-calls */
+    spapr_register_hypercall(H_SET_SPRG0, h_set_sprg0);
+    spapr_register_hypercall(H_SET_DABR, h_set_dabr);
+    spapr_register_hypercall(H_SET_XDABR, h_set_xdabr);
+    spapr_register_hypercall(H_PAGE_INIT, h_page_init);
+    spapr_register_hypercall(H_SET_MODE, h_set_mode);
+
     /* "debugger" hcalls (also used by SLOF). Note: We do -not- differenciate
      * here between the "CI" and the "CACHE" variants, they will use whatever
      * mapping attributes qemu is using. When using KVM, the kernel will
@@ -1004,8 +1138,6 @@
     /* qemu/KVM-PPC specific hcalls */
     spapr_register_hypercall(KVMPPC_H_RTAS, h_rtas);
 
-    spapr_register_hypercall(H_SET_MODE, h_set_mode);
-
     /* ibm,client-architecture-support support */
     spapr_register_hypercall(KVMPPC_H_CAS, h_client_architecture_support);
 }
diff --git a/hw/ppc/spapr_iommu.c b/hw/ppc/spapr_iommu.c
index 6c91d8e..6bc4d4d 100644
--- a/hw/ppc/spapr_iommu.c
+++ b/hw/ppc/spapr_iommu.c
@@ -16,7 +16,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
 #include "hw/hw.h"
+#include "qemu/log.h"
 #include "sysemu/kvm.h"
 #include "hw/qdev.h"
 #include "kvm_ppc.h"
@@ -25,6 +28,7 @@
 #include "trace.h"
 
 #include "hw/ppc/spapr.h"
+#include "hw/ppc/spapr_vio.h"
 
 #include <libfdt.h>
 
@@ -40,7 +44,7 @@
 
 static QLIST_HEAD(spapr_tce_tables, sPAPRTCETable) spapr_tce_tables;
 
-static sPAPRTCETable *spapr_tce_find_by_liobn(uint32_t liobn)
+sPAPRTCETable *spapr_tce_find_by_liobn(target_ulong liobn)
 {
     sPAPRTCETable *tcet;
 
@@ -51,7 +55,7 @@
     }
 
     QLIST_FOREACH(tcet, &spapr_tce_tables, list) {
-        if (tcet->liobn == liobn) {
+        if (tcet->liobn == (uint32_t)liobn) {
             return tcet;
         }
     }
@@ -59,6 +63,52 @@
     return NULL;
 }
 
+static IOMMUAccessFlags spapr_tce_iommu_access_flags(uint64_t tce)
+{
+    switch (tce & SPAPR_TCE_RW) {
+    case SPAPR_TCE_FAULT:
+        return IOMMU_NONE;
+    case SPAPR_TCE_RO:
+        return IOMMU_RO;
+    case SPAPR_TCE_WO:
+        return IOMMU_WO;
+    default: /* SPAPR_TCE_RW */
+        return IOMMU_RW;
+    }
+}
+
+static uint64_t *spapr_tce_alloc_table(uint32_t liobn,
+                                       uint32_t page_shift,
+                                       uint32_t nb_table,
+                                       int *fd,
+                                       bool need_vfio)
+{
+    uint64_t *table = NULL;
+    uint64_t window_size = (uint64_t)nb_table << page_shift;
+
+    if (kvm_enabled() && !(window_size >> 32)) {
+        table = kvmppc_create_spapr_tce(liobn, window_size, fd, need_vfio);
+    }
+
+    if (!table) {
+        *fd = -1;
+        table = g_malloc0(nb_table * sizeof(uint64_t));
+    }
+
+    trace_spapr_iommu_new_table(liobn, table, *fd);
+
+    return table;
+}
+
+static void spapr_tce_free_table(uint64_t *table, int fd, uint32_t nb_table)
+{
+    if (!kvm_enabled() ||
+        (kvmppc_remove_spapr_tce(table, fd, nb_table) != 0)) {
+        g_free(table);
+    }
+}
+
+/* Called from RCU critical section */
 static IOMMUTLBEntry spapr_tce_translate_iommu(MemoryRegion *iommu, hwaddr addr,
                                                bool is_write)
 {
@@ -72,9 +122,7 @@
         .perm = IOMMU_NONE,
     };
 
-    if (tcet->bypass) {
-        ret.perm = IOMMU_RW;
-    } else if ((addr >> tcet->page_shift) < tcet->nb_table) {
+    if ((addr >> tcet->page_shift) < tcet->nb_table) {
         /* Check if we are in bound */
         hwaddr page_mask = IOMMU_PAGE_MASK(tcet->page_shift);
 
@@ -82,7 +130,7 @@
         ret.iova = addr & page_mask;
         ret.translated_addr = tce & page_mask;
         ret.addr_mask = ~page_mask;
-        ret.perm = tce & IOMMU_RW;
+        ret.perm = spapr_tce_iommu_access_flags(tce);
     }
     trace_spapr_iommu_xlate(tcet->liobn, addr, ret.iova, ret.perm,
                             ret.addr_mask);
@@ -90,48 +138,131 @@
     return ret;
 }
 
-static const VMStateDescription vmstate_spapr_tce_table = {
-    .name = "spapr_iommu",
-    .version_id = 2,
-    .minimum_version_id = 2,
-    .fields      = (VMStateField []) {
-        /* Sanity check */
-        VMSTATE_UINT32_EQUAL(liobn, sPAPRTCETable),
-        VMSTATE_UINT32_EQUAL(nb_table, sPAPRTCETable),
+static void spapr_tce_table_pre_save(void *opaque)
+{
+    sPAPRTCETable *tcet = SPAPR_TCE_TABLE(opaque);
 
-        /* IOMMU state */
-        VMSTATE_BOOL(bypass, sPAPRTCETable),
-        VMSTATE_VARRAY_UINT32(table, sPAPRTCETable, nb_table, 0, vmstate_info_uint64, uint64_t),
+    tcet->mig_table = tcet->table;
+    tcet->mig_nb_table = tcet->nb_table;
 
+    trace_spapr_iommu_pre_save(tcet->liobn, tcet->mig_nb_table,
+                               tcet->bus_offset, tcet->page_shift);
+}
+
+static uint64_t spapr_tce_get_min_page_size(MemoryRegion *iommu)
+{
+    sPAPRTCETable *tcet = container_of(iommu, sPAPRTCETable, iommu);
+
+    return 1ULL << tcet->page_shift;
+}
+
+static void spapr_tce_notify_started(MemoryRegion *iommu)
+{
+    spapr_tce_set_need_vfio(container_of(iommu, sPAPRTCETable, iommu), true);
+}
+
+static void spapr_tce_notify_stopped(MemoryRegion *iommu)
+{
+    spapr_tce_set_need_vfio(container_of(iommu, sPAPRTCETable, iommu), false);
+}
+
+static int spapr_tce_table_post_load(void *opaque, int version_id)
+{
+    sPAPRTCETable *tcet = SPAPR_TCE_TABLE(opaque);
+    uint32_t old_nb_table = tcet->nb_table;
+    uint64_t old_bus_offset = tcet->bus_offset;
+    uint32_t old_page_shift = tcet->page_shift;
+
+    if (tcet->vdev) {
+        spapr_vio_set_bypass(tcet->vdev, tcet->bypass);
+    }
+
+    if (tcet->mig_nb_table != tcet->nb_table) {
+        spapr_tce_table_disable(tcet);
+    }
+
+    if (tcet->mig_nb_table) {
+        if (!tcet->nb_table) {
+            spapr_tce_table_enable(tcet, old_page_shift, old_bus_offset,
+                                   tcet->mig_nb_table);
+        }
+
+        memcpy(tcet->table, tcet->mig_table,
+               tcet->nb_table * sizeof(tcet->table[0]));
+
+        free(tcet->mig_table);
+        tcet->mig_table = NULL;
+    }
+
+    trace_spapr_iommu_post_load(tcet->liobn, old_nb_table, tcet->nb_table,
+                                tcet->bus_offset, tcet->page_shift);
+
+    return 0;
+}
+
+static bool spapr_tce_table_ex_needed(void *opaque)
+{
+    sPAPRTCETable *tcet = opaque;
+
+    return tcet->bus_offset || tcet->page_shift != 0xC;
+}
+
+static const VMStateDescription vmstate_spapr_tce_table_ex = {
+    .name = "spapr_iommu_ex",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = spapr_tce_table_ex_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(bus_offset, sPAPRTCETable),
+        VMSTATE_UINT32(page_shift, sPAPRTCETable),
         VMSTATE_END_OF_LIST()
     },
 };
 
+static const VMStateDescription vmstate_spapr_tce_table = {
+    .name = "spapr_iommu",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .pre_save = spapr_tce_table_pre_save,
+    .post_load = spapr_tce_table_post_load,
+    .fields      = (VMStateField []) {
+        /* Sanity check */
+        VMSTATE_UINT32_EQUAL(liobn, sPAPRTCETable),
+
+        /* IOMMU state */
+        VMSTATE_UINT32(mig_nb_table, sPAPRTCETable),
+        VMSTATE_BOOL(bypass, sPAPRTCETable),
+        VMSTATE_VARRAY_UINT32_ALLOC(mig_table, sPAPRTCETable, mig_nb_table, 0,
+                                    vmstate_info_uint64, uint64_t),
+
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_spapr_tce_table_ex,
+        NULL
+    }
+};
+
 static MemoryRegionIOMMUOps spapr_iommu_ops = {
     .translate = spapr_tce_translate_iommu,
+    .get_min_page_size = spapr_tce_get_min_page_size,
+    .notify_started = spapr_tce_notify_started,
+    .notify_stopped = spapr_tce_notify_stopped,
 };
 
 static int spapr_tce_table_realize(DeviceState *dev)
 {
     sPAPRTCETable *tcet = SPAPR_TCE_TABLE(dev);
+    Object *tcetobj = OBJECT(tcet);
+    char tmp[32];
 
-    if (kvm_enabled()) {
-        tcet->table = kvmppc_create_spapr_tce(tcet->liobn,
-                                              tcet->nb_table <<
-                                              tcet->page_shift,
-                                              &tcet->fd,
-                                              tcet->vfio_accel);
-    }
+    tcet->fd = -1;
+    tcet->need_vfio = false;
+    snprintf(tmp, sizeof(tmp), "tce-root-%x", tcet->liobn);
+    memory_region_init(&tcet->root, tcetobj, tmp, UINT64_MAX);
 
-    if (!tcet->table) {
-        size_t table_size = tcet->nb_table * sizeof(uint64_t);
-        tcet->table = g_malloc0(table_size);
-    }
-
-    trace_spapr_iommu_new_table(tcet->liobn, tcet, tcet->table, tcet->fd);
-
-    memory_region_init_iommu(&tcet->iommu, OBJECT(dev), &spapr_iommu_ops,
-                             "iommu-spapr", ram_size);
+    snprintf(tmp, sizeof(tmp), "tce-iommu-%x", tcet->liobn);
+    memory_region_init_iommu(&tcet->iommu, tcetobj, &spapr_iommu_ops, tmp, 0);
 
     QLIST_INSERT_HEAD(&spapr_tce_tables, tcet, list);
 
@@ -141,13 +272,42 @@
     return 0;
 }
 
-sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
-                                   uint64_t bus_offset,
-                                   uint32_t page_shift,
-                                   uint32_t nb_table,
-                                   bool vfio_accel)
+void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool need_vfio)
+{
+    size_t table_size = tcet->nb_table * sizeof(uint64_t);
+    void *newtable;
+
+    if (need_vfio == tcet->need_vfio) {
+        /* Nothing to do */
+        return;
+    }
+
+    if (!need_vfio) {
+        /* FIXME: We don't support transition back to KVM accelerated
+         * TCEs yet */
+        return;
+    }
+
+    tcet->need_vfio = true;
+
+    if (tcet->fd < 0) {
+        /* Table is already in userspace, nothing to be do */
+        return;
+    }
+
+    newtable = g_malloc(table_size);
+    memcpy(newtable, tcet->table, table_size);
+
+    kvmppc_remove_spapr_tce(tcet->table, tcet->fd, tcet->nb_table);
+
+    tcet->fd = -1;
+    tcet->table = newtable;
+}
+
+sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn)
 {
     sPAPRTCETable *tcet;
+    char tmp[32];
 
     if (spapr_tce_find_by_liobn(liobn)) {
         fprintf(stderr, "Attempted to create TCE table with duplicate"
@@ -155,45 +315,69 @@
         return NULL;
     }
 
-    if (!nb_table) {
-        return NULL;
-    }
-
     tcet = SPAPR_TCE_TABLE(object_new(TYPE_SPAPR_TCE_TABLE));
     tcet->liobn = liobn;
-    tcet->bus_offset = bus_offset;
-    tcet->page_shift = page_shift;
-    tcet->nb_table = nb_table;
-    tcet->vfio_accel = vfio_accel;
 
-    object_property_add_child(OBJECT(owner), "tce-table", OBJECT(tcet), NULL);
+    snprintf(tmp, sizeof(tmp), "tce-table-%x", liobn);
+    object_property_add_child(OBJECT(owner), tmp, OBJECT(tcet), NULL);
 
     object_property_set_bool(OBJECT(tcet), true, "realized", NULL);
 
     return tcet;
 }
 
-static void spapr_tce_table_finalize(Object *obj)
+void spapr_tce_table_enable(sPAPRTCETable *tcet,
+                            uint32_t page_shift, uint64_t bus_offset,
+                            uint32_t nb_table)
 {
-    sPAPRTCETable *tcet = SPAPR_TCE_TABLE(obj);
+    if (tcet->nb_table) {
+        error_report("Warning: trying to enable already enabled TCE table");
+        return;
+    }
+
+    tcet->bus_offset = bus_offset;
+    tcet->page_shift = page_shift;
+    tcet->nb_table = nb_table;
+    tcet->table = spapr_tce_alloc_table(tcet->liobn,
+                                        tcet->page_shift,
+                                        tcet->nb_table,
+                                        &tcet->fd,
+                                        tcet->need_vfio);
+
+    memory_region_set_size(&tcet->iommu,
+                           (uint64_t)tcet->nb_table << tcet->page_shift);
+    memory_region_add_subregion(&tcet->root, tcet->bus_offset, &tcet->iommu);
+}
+
+void spapr_tce_table_disable(sPAPRTCETable *tcet)
+{
+    if (!tcet->nb_table) {
+        return;
+    }
+
+    memory_region_del_subregion(&tcet->root, &tcet->iommu);
+    memory_region_set_size(&tcet->iommu, 0);
+
+    spapr_tce_free_table(tcet->table, tcet->fd, tcet->nb_table);
+    tcet->fd = -1;
+    tcet->table = NULL;
+    tcet->bus_offset = 0;
+    tcet->page_shift = 0;
+    tcet->nb_table = 0;
+}
+
+static void spapr_tce_table_unrealize(DeviceState *dev, Error **errp)
+{
+    sPAPRTCETable *tcet = SPAPR_TCE_TABLE(dev);
 
     QLIST_REMOVE(tcet, list);
 
-    if (!kvm_enabled() ||
-        (kvmppc_remove_spapr_tce(tcet->table, tcet->fd,
-                                 tcet->nb_table) != 0)) {
-        g_free(tcet->table);
-    }
+    spapr_tce_table_disable(tcet);
 }
 
 MemoryRegion *spapr_tce_get_iommu(sPAPRTCETable *tcet)
 {
-    return &tcet->iommu;
-}
-
-void spapr_tce_set_bypass(sPAPRTCETable *tcet, bool bypass)
-{
-    tcet->bypass = bypass;
+    return &tcet->root;
 }
 
 static void spapr_tce_reset(DeviceState *dev)
@@ -201,8 +385,9 @@
     sPAPRTCETable *tcet = SPAPR_TCE_TABLE(dev);
     size_t table_size = tcet->nb_table * sizeof(uint64_t);
 
-    tcet->bypass = false;
-    memset(tcet->table, 0, table_size);
+    if (tcet->nb_table) {
+        memset(tcet->table, 0, table_size);
+    }
 }
 
 static target_ulong put_tce_emu(sPAPRTCETable *tcet, target_ulong ioba,
@@ -221,17 +406,17 @@
     tcet->table[index] = tce;
 
     entry.target_as = &address_space_memory,
-    entry.iova = ioba & page_mask;
+    entry.iova = (ioba - tcet->bus_offset) & page_mask;
     entry.translated_addr = tce & page_mask;
     entry.addr_mask = ~page_mask;
-    entry.perm = tce & IOMMU_RW;
+    entry.perm = spapr_tce_iommu_access_flags(tce);
     memory_region_notify_iommu(&tcet->iommu, entry);
 
     return H_SUCCESS;
 }
 
 static target_ulong h_put_tce_indirect(PowerPCCPU *cpu,
-                                       sPAPREnvironment *spapr,
+                                       sPAPRMachineState *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     int i;
@@ -240,7 +425,7 @@
     target_ulong ioba1 = ioba;
     target_ulong tce_list = args[2];
     target_ulong npages = args[3];
-    target_ulong ret = H_PARAMETER;
+    target_ulong ret = H_PARAMETER, tce = 0;
     sPAPRTCETable *tcet = spapr_tce_find_by_liobn(liobn);
     CPUState *cs = CPU(cpu);
     hwaddr page_mask, page_size;
@@ -258,9 +443,7 @@
     ioba &= page_mask;
 
     for (i = 0; i < npages; ++i, ioba += page_size) {
-        target_ulong off = (tce_list & ~SPAPR_TCE_RW) +
-                                i * sizeof(target_ulong);
-        target_ulong tce = ldq_phys(cs->as, off);
+        tce = ldq_be_phys(cs->as, tce_list + i * sizeof(target_ulong));
 
         ret = put_tce_emu(tcet, ioba, tce);
         if (ret) {
@@ -270,15 +453,15 @@
 
     /* Trace last successful or the first problematic entry */
     i = i ? (i - 1) : 0;
-    trace_spapr_iommu_indirect(liobn, ioba1, tce_list, i,
-                               ldq_phys(cs->as,
-                               tce_list + i * sizeof(target_ulong)),
-                               ret);
-
+    if (SPAPR_IS_PCI_LIOBN(liobn)) {
+        trace_spapr_iommu_pci_indirect(liobn, ioba1, tce_list, i, tce, ret);
+    } else {
+        trace_spapr_iommu_indirect(liobn, ioba1, tce_list, i, tce, ret);
+    }
     return ret;
 }
 
-static target_ulong h_stuff_tce(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_stuff_tce(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     int i;
@@ -308,12 +491,16 @@
             break;
         }
     }
-    trace_spapr_iommu_stuff(liobn, ioba, tce_value, npages, ret);
+    if (SPAPR_IS_PCI_LIOBN(liobn)) {
+        trace_spapr_iommu_pci_stuff(liobn, ioba, tce_value, npages, ret);
+    } else {
+        trace_spapr_iommu_stuff(liobn, ioba, tce_value, npages, ret);
+    }
 
     return ret;
 }
 
-static target_ulong h_put_tce(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_put_tce(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong liobn = args[0];
@@ -329,7 +516,11 @@
 
         ret = put_tce_emu(tcet, ioba, tce);
     }
-    trace_spapr_iommu_put(liobn, ioba, tce, ret);
+    if (SPAPR_IS_PCI_LIOBN(liobn)) {
+        trace_spapr_iommu_pci_put(liobn, ioba, tce, ret);
+    } else {
+        trace_spapr_iommu_put(liobn, ioba, tce, ret);
+    }
 
     return ret;
 }
@@ -350,7 +541,7 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_get_tce(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_get_tce(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong liobn = args[0];
@@ -369,7 +560,11 @@
             args[0] = tce;
         }
     }
-    trace_spapr_iommu_get(liobn, ioba, ret, tce);
+    if (SPAPR_IS_PCI_LIOBN(liobn)) {
+        trace_spapr_iommu_pci_get(liobn, ioba, ret, tce);
+    } else {
+        trace_spapr_iommu_get(liobn, ioba, ret, tce);
+    }
 
     return ret;
 }
@@ -420,6 +615,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     dc->init = spapr_tce_table_realize;
     dc->reset = spapr_tce_reset;
+    dc->unrealize = spapr_tce_table_unrealize;
 
     QLIST_INIT(&spapr_tce_tables);
 
@@ -435,7 +631,6 @@
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(sPAPRTCETable),
     .class_init = spapr_tce_table_class_init,
-    .instance_finalize = spapr_tce_table_finalize,
 };
 
 static void register_types(void)
diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c
index 21b95b3..949c44f 100644
--- a/hw/ppc/spapr_pci.c
+++ b/hw/ppc/spapr_pci.c
@@ -22,7 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
+#include "hw/sysbus.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
@@ -30,11 +35,20 @@
 #include "hw/ppc/spapr.h"
 #include "hw/pci-host/spapr.h"
 #include "exec/address-spaces.h"
+#include "exec/ram_addr.h"
 #include <libfdt.h>
 #include "trace.h"
 #include "qemu/error-report.h"
+#include "qapi/qmp/qerror.h"
 
+#include "hw/pci/pci_bridge.h"
 #include "hw/pci/pci_bus.h"
+#include "hw/ppc/spapr_drc.h"
+#include "sysemu/device_tree.h"
+#include "sysemu/kvm.h"
+#include "sysemu/hostmem.h"
+
+#include "hw/vfio/vfio.h"
 
 /* Copied from the kernel arch/powerpc/platforms/pseries/msi.c */
 #define RTAS_QUERY_FN           0
@@ -47,7 +61,17 @@
 #define RTAS_TYPE_MSI           1
 #define RTAS_TYPE_MSIX          2
 
-static sPAPRPHBState *find_phb(sPAPREnvironment *spapr, uint64_t buid)
+#define FDT_NAME_MAX          128
+
+#define _FDT(exp) \
+    do { \
+        int ret = (exp);                                           \
+        if (ret < 0) {                                             \
+            return ret;                                            \
+        }                                                          \
+    } while (0)
+
+sPAPRPHBState *spapr_pci_find_phb(sPAPRMachineState *spapr, uint64_t buid)
 {
     sPAPRPHBState *sphb;
 
@@ -61,10 +85,10 @@
     return NULL;
 }
 
-static PCIDevice *find_dev(sPAPREnvironment *spapr, uint64_t buid,
-                           uint32_t config_addr)
+PCIDevice *spapr_pci_find_dev(sPAPRMachineState *spapr, uint64_t buid,
+                              uint32_t config_addr)
 {
-    sPAPRPHBState *sphb = find_phb(spapr, buid);
+    sPAPRPHBState *sphb = spapr_pci_find_phb(spapr, buid);
     PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
     int bus_num = (config_addr >> 16) & 0xFF;
     int devfn = (config_addr >> 8) & 0xFF;
@@ -82,7 +106,7 @@
     return ((arg >> 20) & 0xf00) | (arg & 0xff);
 }
 
-static void finish_read_pci_config(sPAPREnvironment *spapr, uint64_t buid,
+static void finish_read_pci_config(sPAPRMachineState *spapr, uint64_t buid,
                                    uint32_t addr, uint32_t size,
                                    target_ulong rets)
 {
@@ -95,7 +119,7 @@
         return;
     }
 
-    pci_dev = find_dev(spapr, buid, addr);
+    pci_dev = spapr_pci_find_dev(spapr, buid, addr);
     addr = rtas_pci_cfgaddr(addr);
 
     if (!pci_dev || (addr % size) || (addr >= pci_config_size(pci_dev))) {
@@ -112,7 +136,7 @@
     rtas_st(rets, 1, val);
 }
 
-static void rtas_ibm_read_pci_config(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_ibm_read_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                      uint32_t token, uint32_t nargs,
                                      target_ulong args,
                                      uint32_t nret, target_ulong rets)
@@ -125,14 +149,14 @@
         return;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     size = rtas_ld(args, 3);
     addr = rtas_ld(args, 0);
 
     finish_read_pci_config(spapr, buid, addr, size, rets);
 }
 
-static void rtas_read_pci_config(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_read_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                  uint32_t token, uint32_t nargs,
                                  target_ulong args,
                                  uint32_t nret, target_ulong rets)
@@ -150,7 +174,7 @@
     finish_read_pci_config(spapr, 0, addr, size, rets);
 }
 
-static void finish_write_pci_config(sPAPREnvironment *spapr, uint64_t buid,
+static void finish_write_pci_config(sPAPRMachineState *spapr, uint64_t buid,
                                     uint32_t addr, uint32_t size,
                                     uint32_t val, target_ulong rets)
 {
@@ -162,7 +186,7 @@
         return;
     }
 
-    pci_dev = find_dev(spapr, buid, addr);
+    pci_dev = spapr_pci_find_dev(spapr, buid, addr);
     addr = rtas_pci_cfgaddr(addr);
 
     if (!pci_dev || (addr % size) || (addr >= pci_config_size(pci_dev))) {
@@ -178,7 +202,7 @@
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
 }
 
-static void rtas_ibm_write_pci_config(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_ibm_write_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                       uint32_t token, uint32_t nargs,
                                       target_ulong args,
                                       uint32_t nret, target_ulong rets)
@@ -191,7 +215,7 @@
         return;
     }
 
-    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    buid = rtas_ldq(args, 1);
     val = rtas_ld(args, 4);
     size = rtas_ld(args, 3);
     addr = rtas_ld(args, 0);
@@ -199,7 +223,7 @@
     finish_write_pci_config(spapr, buid, addr, size, val, rets);
 }
 
-static void rtas_write_pci_config(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_write_pci_config(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                   uint32_t token, uint32_t nargs,
                                   target_ulong args,
                                   uint32_t nret, target_ulong rets)
@@ -248,22 +272,23 @@
     }
 }
 
-static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_ibm_change_msi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                 uint32_t token, uint32_t nargs,
                                 target_ulong args, uint32_t nret,
                                 target_ulong rets)
 {
     uint32_t config_addr = rtas_ld(args, 0);
-    uint64_t buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    uint64_t buid = rtas_ldq(args, 1);
     unsigned int func = rtas_ld(args, 3);
     unsigned int req_num = rtas_ld(args, 4); /* 0 == remove all */
     unsigned int seq_num = rtas_ld(args, 5);
     unsigned int ret_intr_type;
-    unsigned int irq, max_irqs = 0, num = 0;
+    unsigned int irq, max_irqs = 0;
     sPAPRPHBState *phb = NULL;
     PCIDevice *pdev = NULL;
     spapr_pci_msi *msi;
     int *config_addr_key;
+    Error *err = NULL;
 
     switch (func) {
     case RTAS_CHANGE_MSI_FN:
@@ -280,30 +305,31 @@
     }
 
     /* Fins sPAPRPHBState */
-    phb = find_phb(spapr, buid);
+    phb = spapr_pci_find_phb(spapr, buid);
     if (phb) {
-        pdev = find_dev(spapr, buid, config_addr);
+        pdev = spapr_pci_find_dev(spapr, buid, config_addr);
     }
     if (!phb || !pdev) {
         rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
         return;
     }
 
+    msi = (spapr_pci_msi *) g_hash_table_lookup(phb->msi, &config_addr);
+
     /* Releasing MSIs */
     if (!req_num) {
-        msi = (spapr_pci_msi *) g_hash_table_lookup(phb->msi, &config_addr);
         if (!msi) {
             trace_spapr_pci_msi("Releasing wrong config", config_addr);
             rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
             return;
         }
 
-        xics_free(spapr->icp, msi->first_irq, msi->num);
+        xics_spapr_free(spapr->xics, msi->first_irq, msi->num);
         if (msi_present(pdev)) {
-            spapr_msi_setmsg(pdev, 0, false, 0, num);
+            spapr_msi_setmsg(pdev, 0, false, 0, 0);
         }
         if (msix_present(pdev)) {
-            spapr_msi_setmsg(pdev, 0, true, 0, num);
+            spapr_msi_setmsg(pdev, 0, true, 0, 0);
         }
         g_hash_table_remove(phb->msi, &config_addr);
 
@@ -336,14 +362,21 @@
     }
 
     /* Allocate MSIs */
-    irq = xics_alloc_block(spapr->icp, 0, req_num, false,
-                           ret_intr_type == RTAS_TYPE_MSI);
-    if (!irq) {
-        error_report("Cannot allocate MSIs for device %x", config_addr);
+    irq = xics_spapr_alloc_block(spapr->xics, 0, req_num, false,
+                           ret_intr_type == RTAS_TYPE_MSI, &err);
+    if (err) {
+        error_reportf_err(err, "Can't allocate MSIs for device %x: ",
+                          config_addr);
         rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
         return;
     }
 
+    /* Release previous MSIs */
+    if (msi) {
+        xics_spapr_free(spapr->xics, msi->first_irq, msi->num);
+        g_hash_table_remove(phb->msi, &config_addr);
+    }
+
     /* Setup MSI/MSIX vectors in the device (via cfgspace or MSIX BAR) */
     spapr_msi_setmsg(pdev, SPAPR_PCI_MSI_WINDOW, ret_intr_type == RTAS_TYPE_MSIX,
                      irq, req_num);
@@ -360,13 +393,15 @@
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
     rtas_st(rets, 1, req_num);
     rtas_st(rets, 2, ++seq_num);
-    rtas_st(rets, 3, ret_intr_type);
+    if (nret > 3) {
+        rtas_st(rets, 3, ret_intr_type);
+    }
 
     trace_spapr_pci_rtas_ibm_change_msi(config_addr, func, req_num, irq);
 }
 
 static void rtas_ibm_query_interrupt_source_number(PowerPCCPU *cpu,
-                                                   sPAPREnvironment *spapr,
+                                                   sPAPRMachineState *spapr,
                                                    uint32_t token,
                                                    uint32_t nargs,
                                                    target_ulong args,
@@ -374,16 +409,16 @@
                                                    target_ulong rets)
 {
     uint32_t config_addr = rtas_ld(args, 0);
-    uint64_t buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    uint64_t buid = rtas_ldq(args, 1);
     unsigned int intr_src_num = -1, ioa_intr_num = rtas_ld(args, 3);
     sPAPRPHBState *phb = NULL;
     PCIDevice *pdev = NULL;
     spapr_pci_msi *msi;
 
     /* Find sPAPRPHBState */
-    phb = find_phb(spapr, buid);
+    phb = spapr_pci_find_phb(spapr, buid);
     if (phb) {
-        pdev = find_dev(spapr, buid, config_addr);
+        pdev = spapr_pci_find_dev(spapr, buid, config_addr);
     }
     if (!phb || !pdev) {
         rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
@@ -406,6 +441,246 @@
     rtas_st(rets, 2, 1);/* 0 == level; 1 == edge */
 }
 
+static void rtas_ibm_set_eeh_option(PowerPCCPU *cpu,
+                                    sPAPRMachineState *spapr,
+                                    uint32_t token, uint32_t nargs,
+                                    target_ulong args, uint32_t nret,
+                                    target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    uint32_t addr, option;
+    uint64_t buid;
+    int ret;
+
+    if ((nargs != 4) || (nret != 1)) {
+        goto param_error_exit;
+    }
+
+    buid = rtas_ldq(args, 1);
+    addr = rtas_ld(args, 0);
+    option = rtas_ld(args, 3);
+
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb) {
+        goto param_error_exit;
+    }
+
+    if (!spapr_phb_eeh_available(sphb)) {
+        goto param_error_exit;
+    }
+
+    ret = spapr_phb_vfio_eeh_set_option(sphb, addr, option);
+    rtas_st(rets, 0, ret);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_get_config_addr_info2(PowerPCCPU *cpu,
+                                           sPAPRMachineState *spapr,
+                                           uint32_t token, uint32_t nargs,
+                                           target_ulong args, uint32_t nret,
+                                           target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    PCIDevice *pdev;
+    uint32_t addr, option;
+    uint64_t buid;
+
+    if ((nargs != 4) || (nret != 2)) {
+        goto param_error_exit;
+    }
+
+    buid = rtas_ldq(args, 1);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb) {
+        goto param_error_exit;
+    }
+
+    if (!spapr_phb_eeh_available(sphb)) {
+        goto param_error_exit;
+    }
+
+    /*
+     * We always have PE address of form "00BB0001". "BB"
+     * represents the bus number of PE's primary bus.
+     */
+    option = rtas_ld(args, 3);
+    switch (option) {
+    case RTAS_GET_PE_ADDR:
+        addr = rtas_ld(args, 0);
+        pdev = spapr_pci_find_dev(spapr, buid, addr);
+        if (!pdev) {
+            goto param_error_exit;
+        }
+
+        rtas_st(rets, 1, (pci_bus_num(pdev->bus) << 16) + 1);
+        break;
+    case RTAS_GET_PE_MODE:
+        rtas_st(rets, 1, RTAS_PE_MODE_SHARED);
+        break;
+    default:
+        goto param_error_exit;
+    }
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_read_slot_reset_state2(PowerPCCPU *cpu,
+                                            sPAPRMachineState *spapr,
+                                            uint32_t token, uint32_t nargs,
+                                            target_ulong args, uint32_t nret,
+                                            target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    uint64_t buid;
+    int state, ret;
+
+    if ((nargs != 3) || (nret != 4 && nret != 5)) {
+        goto param_error_exit;
+    }
+
+    buid = rtas_ldq(args, 1);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb) {
+        goto param_error_exit;
+    }
+
+    if (!spapr_phb_eeh_available(sphb)) {
+        goto param_error_exit;
+    }
+
+    ret = spapr_phb_vfio_eeh_get_state(sphb, &state);
+    rtas_st(rets, 0, ret);
+    if (ret != RTAS_OUT_SUCCESS) {
+        return;
+    }
+
+    rtas_st(rets, 1, state);
+    rtas_st(rets, 2, RTAS_EEH_SUPPORT);
+    rtas_st(rets, 3, RTAS_EEH_PE_UNAVAIL_INFO);
+    if (nret >= 5) {
+        rtas_st(rets, 4, RTAS_EEH_PE_RECOVER_INFO);
+    }
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_set_slot_reset(PowerPCCPU *cpu,
+                                    sPAPRMachineState *spapr,
+                                    uint32_t token, uint32_t nargs,
+                                    target_ulong args, uint32_t nret,
+                                    target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    uint32_t option;
+    uint64_t buid;
+    int ret;
+
+    if ((nargs != 4) || (nret != 1)) {
+        goto param_error_exit;
+    }
+
+    buid = rtas_ldq(args, 1);
+    option = rtas_ld(args, 3);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb) {
+        goto param_error_exit;
+    }
+
+    if (!spapr_phb_eeh_available(sphb)) {
+        goto param_error_exit;
+    }
+
+    ret = spapr_phb_vfio_eeh_reset(sphb, option);
+    rtas_st(rets, 0, ret);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_configure_pe(PowerPCCPU *cpu,
+                                  sPAPRMachineState *spapr,
+                                  uint32_t token, uint32_t nargs,
+                                  target_ulong args, uint32_t nret,
+                                  target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    uint64_t buid;
+    int ret;
+
+    if ((nargs != 3) || (nret != 1)) {
+        goto param_error_exit;
+    }
+
+    buid = rtas_ldq(args, 1);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb) {
+        goto param_error_exit;
+    }
+
+    if (!spapr_phb_eeh_available(sphb)) {
+        goto param_error_exit;
+    }
+
+    ret = spapr_phb_vfio_eeh_configure(sphb);
+    rtas_st(rets, 0, ret);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+/* To support it later */
+static void rtas_ibm_slot_error_detail(PowerPCCPU *cpu,
+                                       sPAPRMachineState *spapr,
+                                       uint32_t token, uint32_t nargs,
+                                       target_ulong args, uint32_t nret,
+                                       target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    int option;
+    uint64_t buid;
+
+    if ((nargs != 8) || (nret != 1)) {
+        goto param_error_exit;
+    }
+
+    buid = rtas_ldq(args, 1);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb) {
+        goto param_error_exit;
+    }
+
+    if (!spapr_phb_eeh_available(sphb)) {
+        goto param_error_exit;
+    }
+
+    option = rtas_ld(args, 7);
+    switch (option) {
+    case RTAS_SLOT_TEMP_ERR_LOG:
+    case RTAS_SLOT_PERM_ERR_LOG:
+        break;
+    default:
+        goto param_error_exit;
+    }
+
+    /* We don't have error log yet */
+    rtas_st(rets, 0, RTAS_OUT_NO_ERRORS_FOUND);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
 static int pci_spapr_swizzle(int slot, int pin)
 {
     return (slot + pin) % PCI_NUM_PINS;
@@ -455,11 +730,12 @@
 static void spapr_msi_write(void *opaque, hwaddr addr,
                             uint64_t data, unsigned size)
 {
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
     uint32_t irq = data;
 
     trace_spapr_pci_msi_write(addr, data, irq);
 
-    qemu_irq_pulse(xics_get_qirq(spapr->icp, irq));
+    qemu_irq_pulse(xics_get_qirq(spapr->xics, irq));
 }
 
 static const MemoryRegionOps spapr_msi_ops = {
@@ -479,30 +755,582 @@
     return &phb->iommu_as;
 }
 
+static char *spapr_phb_vfio_get_loc_code(sPAPRPHBState *sphb,  PCIDevice *pdev)
+{
+    char *path = NULL, *buf = NULL, *host = NULL;
+
+    /* Get the PCI VFIO host id */
+    host = object_property_get_str(OBJECT(pdev), "host", NULL);
+    if (!host) {
+        goto err_out;
+    }
+
+    /* Construct the path of the file that will give us the DT location */
+    path = g_strdup_printf("/sys/bus/pci/devices/%s/devspec", host);
+    g_free(host);
+    if (!path || !g_file_get_contents(path, &buf, NULL, NULL)) {
+        goto err_out;
+    }
+    g_free(path);
+
+    /* Construct and read from host device tree the loc-code */
+    path = g_strdup_printf("/proc/device-tree%s/ibm,loc-code", buf);
+    g_free(buf);
+    if (!path || !g_file_get_contents(path, &buf, NULL, NULL)) {
+        goto err_out;
+    }
+    return buf;
+
+err_out:
+    g_free(path);
+    return NULL;
+}
+
+static char *spapr_phb_get_loc_code(sPAPRPHBState *sphb, PCIDevice *pdev)
+{
+    char *buf;
+    const char *devtype = "qemu";
+    uint32_t busnr = pci_bus_num(PCI_BUS(qdev_get_parent_bus(DEVICE(pdev))));
+
+    if (object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+        buf = spapr_phb_vfio_get_loc_code(sphb, pdev);
+        if (buf) {
+            return buf;
+        }
+        devtype = "vfio";
+    }
+    /*
+     * For emulated devices and VFIO-failure case, make up
+     * the loc-code.
+     */
+    buf = g_strdup_printf("%s_%s:%04x:%02x:%02x.%x",
+                          devtype, pdev->name, sphb->index, busnr,
+                          PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
+    return buf;
+}
+
+/* Macros to operate with address in OF binding to PCI */
+#define b_x(x, p, l)    (((x) & ((1<<(l))-1)) << (p))
+#define b_n(x)          b_x((x), 31, 1) /* 0 if relocatable */
+#define b_p(x)          b_x((x), 30, 1) /* 1 if prefetchable */
+#define b_t(x)          b_x((x), 29, 1) /* 1 if the address is aliased */
+#define b_ss(x)         b_x((x), 24, 2) /* the space code */
+#define b_bbbbbbbb(x)   b_x((x), 16, 8) /* bus number */
+#define b_ddddd(x)      b_x((x), 11, 5) /* device number */
+#define b_fff(x)        b_x((x), 8, 3)  /* function number */
+#define b_rrrrrrrr(x)   b_x((x), 0, 8)  /* register number */
+
+/* for 'reg'/'assigned-addresses' OF properties */
+#define RESOURCE_CELLS_SIZE 2
+#define RESOURCE_CELLS_ADDRESS 3
+
+typedef struct ResourceFields {
+    uint32_t phys_hi;
+    uint32_t phys_mid;
+    uint32_t phys_lo;
+    uint32_t size_hi;
+    uint32_t size_lo;
+} QEMU_PACKED ResourceFields;
+
+typedef struct ResourceProps {
+    ResourceFields reg[8];
+    ResourceFields assigned[7];
+    uint32_t reg_len;
+    uint32_t assigned_len;
+} ResourceProps;
+
+/* fill in the 'reg'/'assigned-resources' OF properties for
+ * a PCI device. 'reg' describes resource requirements for a
+ * device's IO/MEM regions, 'assigned-addresses' describes the
+ * actual resource assignments.
+ *
+ * the properties are arrays of ('phys-addr', 'size') pairs describing
+ * the addressable regions of the PCI device, where 'phys-addr' is a
+ * RESOURCE_CELLS_ADDRESS-tuple of 32-bit integers corresponding to
+ * (phys.hi, phys.mid, phys.lo), and 'size' is a
+ * RESOURCE_CELLS_SIZE-tuple corresponding to (size.hi, size.lo).
+ *
+ * phys.hi = 0xYYXXXXZZ, where:
+ *   0xYY = npt000ss
+ *          |||   |
+ *          |||   +-- space code
+ *          |||               |
+ *          |||               +  00 if configuration space
+ *          |||               +  01 if IO region,
+ *          |||               +  10 if 32-bit MEM region
+ *          |||               +  11 if 64-bit MEM region
+ *          |||
+ *          ||+------ for non-relocatable IO: 1 if aliased
+ *          ||        for relocatable IO: 1 if below 64KB
+ *          ||        for MEM: 1 if below 1MB
+ *          |+------- 1 if region is prefetchable
+ *          +-------- 1 if region is non-relocatable
+ *   0xXXXX = bbbbbbbb dddddfff, encoding bus, slot, and function
+ *            bits respectively
+ *   0xZZ = rrrrrrrr, the register number of the BAR corresponding
+ *          to the region
+ *
+ * phys.mid and phys.lo correspond respectively to the hi/lo portions
+ * of the actual address of the region.
+ *
+ * how the phys-addr/size values are used differ slightly between
+ * 'reg' and 'assigned-addresses' properties. namely, 'reg' has
+ * an additional description for the config space region of the
+ * device, and in the case of QEMU has n=0 and phys.mid=phys.lo=0
+ * to describe the region as relocatable, with an address-mapping
+ * that corresponds directly to the PHB's address space for the
+ * resource. 'assigned-addresses' always has n=1 set with an absolute
+ * address assigned for the resource. in general, 'assigned-addresses'
+ * won't be populated, since addresses for PCI devices are generally
+ * unmapped initially and left to the guest to assign.
+ *
+ * note also that addresses defined in these properties are, at least
+ * for PAPR guests, relative to the PHBs IO/MEM windows, and
+ * correspond directly to the addresses in the BARs.
+ *
+ * in accordance with PCI Bus Binding to Open Firmware,
+ * IEEE Std 1275-1994, section 4.1.1, as implemented by PAPR+ v2.7,
+ * Appendix C.
+ */
+static void populate_resource_props(PCIDevice *d, ResourceProps *rp)
+{
+    int bus_num = pci_bus_num(PCI_BUS(qdev_get_parent_bus(DEVICE(d))));
+    uint32_t dev_id = (b_bbbbbbbb(bus_num) |
+                       b_ddddd(PCI_SLOT(d->devfn)) |
+                       b_fff(PCI_FUNC(d->devfn)));
+    ResourceFields *reg, *assigned;
+    int i, reg_idx = 0, assigned_idx = 0;
+
+    /* config space region */
+    reg = &rp->reg[reg_idx++];
+    reg->phys_hi = cpu_to_be32(dev_id);
+    reg->phys_mid = 0;
+    reg->phys_lo = 0;
+    reg->size_hi = 0;
+    reg->size_lo = 0;
+
+    for (i = 0; i < PCI_NUM_REGIONS; i++) {
+        if (!d->io_regions[i].size) {
+            continue;
+        }
+
+        reg = &rp->reg[reg_idx++];
+
+        reg->phys_hi = cpu_to_be32(dev_id | b_rrrrrrrr(pci_bar(d, i)));
+        if (d->io_regions[i].type & PCI_BASE_ADDRESS_SPACE_IO) {
+            reg->phys_hi |= cpu_to_be32(b_ss(1));
+        } else if (d->io_regions[i].type & PCI_BASE_ADDRESS_MEM_TYPE_64) {
+            reg->phys_hi |= cpu_to_be32(b_ss(3));
+        } else {
+            reg->phys_hi |= cpu_to_be32(b_ss(2));
+        }
+        reg->phys_mid = 0;
+        reg->phys_lo = 0;
+        reg->size_hi = cpu_to_be32(d->io_regions[i].size >> 32);
+        reg->size_lo = cpu_to_be32(d->io_regions[i].size);
+
+        if (d->io_regions[i].addr == PCI_BAR_UNMAPPED) {
+            continue;
+        }
+
+        assigned = &rp->assigned[assigned_idx++];
+        assigned->phys_hi = cpu_to_be32(reg->phys_hi | b_n(1));
+        assigned->phys_mid = cpu_to_be32(d->io_regions[i].addr >> 32);
+        assigned->phys_lo = cpu_to_be32(d->io_regions[i].addr);
+        assigned->size_hi = reg->size_hi;
+        assigned->size_lo = reg->size_lo;
+    }
+
+    rp->reg_len = reg_idx * sizeof(ResourceFields);
+    rp->assigned_len = assigned_idx * sizeof(ResourceFields);
+}
+
+static uint32_t spapr_phb_get_pci_drc_index(sPAPRPHBState *phb,
+                                            PCIDevice *pdev);
+
+static int spapr_populate_pci_child_dt(PCIDevice *dev, void *fdt, int offset,
+                                       sPAPRPHBState *sphb)
+{
+    ResourceProps rp;
+    bool is_bridge = false;
+    int pci_status, err;
+    char *buf = NULL;
+    uint32_t drc_index = spapr_phb_get_pci_drc_index(sphb, dev);
+    uint32_t max_msi, max_msix;
+
+    if (pci_default_read_config(dev, PCI_HEADER_TYPE, 1) ==
+        PCI_HEADER_TYPE_BRIDGE) {
+        is_bridge = true;
+    }
+
+    /* in accordance with PAPR+ v2.7 13.6.3, Table 181 */
+    _FDT(fdt_setprop_cell(fdt, offset, "vendor-id",
+                          pci_default_read_config(dev, PCI_VENDOR_ID, 2)));
+    _FDT(fdt_setprop_cell(fdt, offset, "device-id",
+                          pci_default_read_config(dev, PCI_DEVICE_ID, 2)));
+    _FDT(fdt_setprop_cell(fdt, offset, "revision-id",
+                          pci_default_read_config(dev, PCI_REVISION_ID, 1)));
+    _FDT(fdt_setprop_cell(fdt, offset, "class-code",
+                          pci_default_read_config(dev, PCI_CLASS_PROG, 3)));
+    if (pci_default_read_config(dev, PCI_INTERRUPT_PIN, 1)) {
+        _FDT(fdt_setprop_cell(fdt, offset, "interrupts",
+                 pci_default_read_config(dev, PCI_INTERRUPT_PIN, 1)));
+    }
+
+    if (!is_bridge) {
+        _FDT(fdt_setprop_cell(fdt, offset, "min-grant",
+            pci_default_read_config(dev, PCI_MIN_GNT, 1)));
+        _FDT(fdt_setprop_cell(fdt, offset, "max-latency",
+            pci_default_read_config(dev, PCI_MAX_LAT, 1)));
+    }
+
+    if (pci_default_read_config(dev, PCI_SUBSYSTEM_ID, 2)) {
+        _FDT(fdt_setprop_cell(fdt, offset, "subsystem-id",
+                 pci_default_read_config(dev, PCI_SUBSYSTEM_ID, 2)));
+    }
+
+    if (pci_default_read_config(dev, PCI_SUBSYSTEM_VENDOR_ID, 2)) {
+        _FDT(fdt_setprop_cell(fdt, offset, "subsystem-vendor-id",
+                 pci_default_read_config(dev, PCI_SUBSYSTEM_VENDOR_ID, 2)));
+    }
+
+    _FDT(fdt_setprop_cell(fdt, offset, "cache-line-size",
+        pci_default_read_config(dev, PCI_CACHE_LINE_SIZE, 1)));
+
+    /* the following fdt cells are masked off the pci status register */
+    pci_status = pci_default_read_config(dev, PCI_STATUS, 2);
+    _FDT(fdt_setprop_cell(fdt, offset, "devsel-speed",
+                          PCI_STATUS_DEVSEL_MASK & pci_status));
+
+    if (pci_status & PCI_STATUS_FAST_BACK) {
+        _FDT(fdt_setprop(fdt, offset, "fast-back-to-back", NULL, 0));
+    }
+    if (pci_status & PCI_STATUS_66MHZ) {
+        _FDT(fdt_setprop(fdt, offset, "66mhz-capable", NULL, 0));
+    }
+    if (pci_status & PCI_STATUS_UDF) {
+        _FDT(fdt_setprop(fdt, offset, "udf-supported", NULL, 0));
+    }
+
+    /* NOTE: this is normally generated by firmware via path/unit name,
+     * but in our case we must set it manually since it does not get
+     * processed by OF beforehand
+     */
+    _FDT(fdt_setprop_string(fdt, offset, "name", "pci"));
+    buf = spapr_phb_get_loc_code(sphb, dev);
+    if (!buf) {
+        error_report("Failed setting the ibm,loc-code");
+        return -1;
+    }
+
+    err = fdt_setprop_string(fdt, offset, "ibm,loc-code", buf);
+    g_free(buf);
+    if (err < 0) {
+        return err;
+    }
+
+    if (drc_index) {
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,my-drc-index", drc_index));
+    }
+
+    _FDT(fdt_setprop_cell(fdt, offset, "#address-cells",
+                          RESOURCE_CELLS_ADDRESS));
+    _FDT(fdt_setprop_cell(fdt, offset, "#size-cells",
+                          RESOURCE_CELLS_SIZE));
+
+    max_msi = msi_nr_vectors_allocated(dev);
+    if (max_msi) {
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,req#msi", max_msi));
+    }
+    max_msix = dev->msix_entries_nr;
+    if (max_msix) {
+        _FDT(fdt_setprop_cell(fdt, offset, "ibm,req#msi-x", max_msix));
+    }
+
+    populate_resource_props(dev, &rp);
+    _FDT(fdt_setprop(fdt, offset, "reg", (uint8_t *)rp.reg, rp.reg_len));
+    _FDT(fdt_setprop(fdt, offset, "assigned-addresses",
+                     (uint8_t *)rp.assigned, rp.assigned_len));
+
+    return 0;
+}
+
+/* create OF node for pci device and required OF DT properties */
+static int spapr_create_pci_child_dt(sPAPRPHBState *phb, PCIDevice *dev,
+                                     void *fdt, int node_offset)
+{
+    int offset, ret;
+    int slot = PCI_SLOT(dev->devfn);
+    int func = PCI_FUNC(dev->devfn);
+    char nodename[FDT_NAME_MAX];
+
+    if (func != 0) {
+        snprintf(nodename, FDT_NAME_MAX, "pci@%x,%x", slot, func);
+    } else {
+        snprintf(nodename, FDT_NAME_MAX, "pci@%x", slot);
+    }
+    offset = fdt_add_subnode(fdt, node_offset, nodename);
+    ret = spapr_populate_pci_child_dt(dev, fdt, offset, phb);
+
+    g_assert(!ret);
+    if (ret) {
+        return 0;
+    }
+    return offset;
+}
+
+static void spapr_phb_add_pci_device(sPAPRDRConnector *drc,
+                                     sPAPRPHBState *phb,
+                                     PCIDevice *pdev,
+                                     Error **errp)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    DeviceState *dev = DEVICE(pdev);
+    void *fdt = NULL;
+    int fdt_start_offset = 0, fdt_size;
+
+    fdt = create_device_tree(&fdt_size);
+    fdt_start_offset = spapr_create_pci_child_dt(phb, pdev, fdt, 0);
+    if (!fdt_start_offset) {
+        error_setg(errp, "Failed to create pci child device tree node");
+        goto out;
+    }
+
+    drck->attach(drc, DEVICE(pdev),
+                 fdt, fdt_start_offset, !dev->hotplugged, errp);
+out:
+    if (*errp) {
+        g_free(fdt);
+    }
+}
+
+static void spapr_phb_remove_pci_device_cb(DeviceState *dev, void *opaque)
+{
+    /* some version guests do not wait for completion of a device
+     * cleanup (generally done asynchronously by the kernel) before
+     * signaling to QEMU that the device is safe, but instead sleep
+     * for some 'safe' period of time. unfortunately on a busy host
+     * this sleep isn't guaranteed to be long enough, resulting in
+     * bad things like IRQ lines being left asserted during final
+     * device removal. to deal with this we call reset just prior
+     * to finalizing the device, which will put the device back into
+     * an 'idle' state, as the device cleanup code expects.
+     */
+    pci_device_reset(PCI_DEVICE(dev));
+    object_unparent(OBJECT(dev));
+}
+
+static void spapr_phb_remove_pci_device(sPAPRDRConnector *drc,
+                                        sPAPRPHBState *phb,
+                                        PCIDevice *pdev,
+                                        Error **errp)
+{
+    sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+    drck->detach(drc, DEVICE(pdev), spapr_phb_remove_pci_device_cb, phb, errp);
+}
+
+static sPAPRDRConnector *spapr_phb_get_pci_func_drc(sPAPRPHBState *phb,
+                                                    uint32_t busnr,
+                                                    int32_t devfn)
+{
+    return spapr_dr_connector_by_id(SPAPR_DR_CONNECTOR_TYPE_PCI,
+                                    (phb->index << 16) |
+                                    (busnr << 8) |
+                                    devfn);
+}
+
+static sPAPRDRConnector *spapr_phb_get_pci_drc(sPAPRPHBState *phb,
+                                               PCIDevice *pdev)
+{
+    uint32_t busnr = pci_bus_num(PCI_BUS(qdev_get_parent_bus(DEVICE(pdev))));
+    return spapr_phb_get_pci_func_drc(phb, busnr, pdev->devfn);
+}
+
+static uint32_t spapr_phb_get_pci_drc_index(sPAPRPHBState *phb,
+                                            PCIDevice *pdev)
+{
+    sPAPRDRConnector *drc = spapr_phb_get_pci_drc(phb, pdev);
+    sPAPRDRConnectorClass *drck;
+
+    if (!drc) {
+        return 0;
+    }
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    return drck->get_index(drc);
+}
+
+static void spapr_phb_hot_plug_child(HotplugHandler *plug_handler,
+                                     DeviceState *plugged_dev, Error **errp)
+{
+    sPAPRPHBState *phb = SPAPR_PCI_HOST_BRIDGE(DEVICE(plug_handler));
+    PCIDevice *pdev = PCI_DEVICE(plugged_dev);
+    sPAPRDRConnector *drc = spapr_phb_get_pci_drc(phb, pdev);
+    Error *local_err = NULL;
+    PCIBus *bus = PCI_BUS(qdev_get_parent_bus(DEVICE(pdev)));
+    uint32_t slotnr = PCI_SLOT(pdev->devfn);
+
+    /* if DR is disabled we don't need to do anything in the case of
+     * hotplug or coldplug callbacks
+     */
+    if (!phb->dr_enabled) {
+        /* if this is a hotplug operation initiated by the user
+         * we need to let them know it's not enabled
+         */
+        if (plugged_dev->hotplugged) {
+            error_setg(errp, QERR_BUS_NO_HOTPLUG,
+                       object_get_typename(OBJECT(phb)));
+        }
+        return;
+    }
+
+    g_assert(drc);
+
+    /* Following the QEMU convention used for PCIe multifunction
+     * hotplug, we do not allow functions to be hotplugged to a
+     * slot that already has function 0 present
+     */
+    if (plugged_dev->hotplugged && bus->devices[PCI_DEVFN(slotnr, 0)] &&
+        PCI_FUNC(pdev->devfn) != 0) {
+        error_setg(errp, "PCI: slot %d function 0 already ocuppied by %s,"
+                   " additional functions can no longer be exposed to guest.",
+                   slotnr, bus->devices[PCI_DEVFN(slotnr, 0)]->name);
+        return;
+    }
+
+    spapr_phb_add_pci_device(drc, phb, pdev, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* If this is function 0, signal hotplug for all the device functions.
+     * Otherwise defer sending the hotplug event.
+     */
+    if (plugged_dev->hotplugged && PCI_FUNC(pdev->devfn) == 0) {
+        int i;
+
+        for (i = 0; i < 8; i++) {
+            sPAPRDRConnector *func_drc;
+            sPAPRDRConnectorClass *func_drck;
+            sPAPRDREntitySense state;
+
+            func_drc = spapr_phb_get_pci_func_drc(phb, pci_bus_num(bus),
+                                                  PCI_DEVFN(slotnr, i));
+            func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc);
+            func_drck->entity_sense(func_drc, &state);
+
+            if (state == SPAPR_DR_ENTITY_SENSE_PRESENT) {
+                spapr_hotplug_req_add_by_index(func_drc);
+            }
+        }
+    }
+}
+
+static void spapr_phb_hot_unplug_child(HotplugHandler *plug_handler,
+                                       DeviceState *plugged_dev, Error **errp)
+{
+    sPAPRPHBState *phb = SPAPR_PCI_HOST_BRIDGE(DEVICE(plug_handler));
+    PCIDevice *pdev = PCI_DEVICE(plugged_dev);
+    sPAPRDRConnectorClass *drck;
+    sPAPRDRConnector *drc = spapr_phb_get_pci_drc(phb, pdev);
+    Error *local_err = NULL;
+
+    if (!phb->dr_enabled) {
+        error_setg(errp, QERR_BUS_NO_HOTPLUG,
+                   object_get_typename(OBJECT(phb)));
+        return;
+    }
+
+    g_assert(drc);
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    if (!drck->release_pending(drc)) {
+        PCIBus *bus = PCI_BUS(qdev_get_parent_bus(DEVICE(pdev)));
+        uint32_t slotnr = PCI_SLOT(pdev->devfn);
+        sPAPRDRConnector *func_drc;
+        sPAPRDRConnectorClass *func_drck;
+        sPAPRDREntitySense state;
+        int i;
+
+        /* ensure any other present functions are pending unplug */
+        if (PCI_FUNC(pdev->devfn) == 0) {
+            for (i = 1; i < 8; i++) {
+                func_drc = spapr_phb_get_pci_func_drc(phb, pci_bus_num(bus),
+                                                      PCI_DEVFN(slotnr, i));
+                func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc);
+                func_drck->entity_sense(func_drc, &state);
+                if (state == SPAPR_DR_ENTITY_SENSE_PRESENT
+                    && !func_drck->release_pending(func_drc)) {
+                    error_setg(errp,
+                               "PCI: slot %d, function %d still present. "
+                               "Must unplug all non-0 functions first.",
+                               slotnr, i);
+                    return;
+                }
+            }
+        }
+
+        spapr_phb_remove_pci_device(drc, phb, pdev, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+
+        /* if this isn't func 0, defer unplug event. otherwise signal removal
+         * for all present functions
+         */
+        if (PCI_FUNC(pdev->devfn) == 0) {
+            for (i = 7; i >= 0; i--) {
+                func_drc = spapr_phb_get_pci_func_drc(phb, pci_bus_num(bus),
+                                                      PCI_DEVFN(slotnr, i));
+                func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc);
+                func_drck->entity_sense(func_drc, &state);
+                if (state == SPAPR_DR_ENTITY_SENSE_PRESENT) {
+                    spapr_hotplug_req_remove_by_index(func_drc);
+                }
+            }
+        }
+    }
+}
+
 static void spapr_phb_realize(DeviceState *dev, Error **errp)
 {
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
     SysBusDevice *s = SYS_BUS_DEVICE(dev);
     sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(s);
     PCIHostState *phb = PCI_HOST_BRIDGE(s);
-    sPAPRPHBClass *info = SPAPR_PCI_HOST_BRIDGE_GET_CLASS(s);
     char *namebuf;
     int i;
     PCIBus *bus;
     uint64_t msi_window_size = 4096;
+    sPAPRTCETable *tcet;
+    const unsigned windows_supported =
+        sphb->ddw_enabled ? SPAPR_PCI_DMA_MAX_WINDOWS : 1;
 
-    if (sphb->index != -1) {
+    if (sphb->index != (uint32_t)-1) {
         hwaddr windows_base;
 
-        if ((sphb->buid != -1) || (sphb->dma_liobn != -1)
-            || (sphb->mem_win_addr != -1)
-            || (sphb->io_win_addr != -1)) {
+        if ((sphb->buid != (uint64_t)-1) || (sphb->dma_liobn[0] != (uint32_t)-1)
+            || (sphb->dma_liobn[1] != (uint32_t)-1 && windows_supported == 2)
+            || (sphb->mem_win_addr != (hwaddr)-1)
+            || (sphb->io_win_addr != (hwaddr)-1)) {
             error_setg(errp, "Either \"index\" or other parameters must"
                        " be specified for PAPR PHB, not both");
             return;
         }
 
+        if (sphb->index > SPAPR_PCI_MAX_INDEX) {
+            error_setg(errp, "\"index\" for PAPR PHB is too large (max %u)",
+                       SPAPR_PCI_MAX_INDEX);
+            return;
+        }
+
         sphb->buid = SPAPR_PCI_BASE_BUID + sphb->index;
-        sphb->dma_liobn = SPAPR_PCI_BASE_LIOBN + sphb->index;
+        for (i = 0; i < windows_supported; ++i) {
+            sphb->dma_liobn[i] = SPAPR_PCI_LIOBN(sphb->index, i);
+        }
 
         windows_base = SPAPR_PCI_WINDOW_BASE
             + sphb->index * SPAPR_PCI_WINDOW_SPACING;
@@ -510,27 +1338,28 @@
         sphb->io_win_addr = windows_base + SPAPR_PCI_IO_WIN_OFF;
     }
 
-    if (sphb->buid == -1) {
+    if (sphb->buid == (uint64_t)-1) {
         error_setg(errp, "BUID not specified for PHB");
         return;
     }
 
-    if (sphb->dma_liobn == -1) {
-        error_setg(errp, "LIOBN not specified for PHB");
+    if ((sphb->dma_liobn[0] == (uint32_t)-1) ||
+        ((sphb->dma_liobn[1] == (uint32_t)-1) && (windows_supported > 1))) {
+        error_setg(errp, "LIOBN(s) not specified for PHB");
         return;
     }
 
-    if (sphb->mem_win_addr == -1) {
+    if (sphb->mem_win_addr == (hwaddr)-1) {
         error_setg(errp, "Memory window address not specified for PHB");
         return;
     }
 
-    if (sphb->io_win_addr == -1) {
+    if (sphb->io_win_addr == (hwaddr)-1) {
         error_setg(errp, "IO window address not specified for PHB");
         return;
     }
 
-    if (find_phb(spapr, sphb->buid)) {
+    if (spapr_pci_find_phb(spapr, sphb->buid)) {
         error_setg(errp, "PCI host bridges must have unique BUIDs");
         return;
     }
@@ -566,6 +1395,7 @@
                            &sphb->memspace, &sphb->iospace,
                            PCI_DEVFN(0, 0), PCI_NUM_PINS, TYPE_PCI_BUS);
     phb->bus = bus;
+    qbus_set_hotplug_handler(BUS(phb->bus), DEVICE(sphb), NULL);
 
     /*
      * Initialize PHB address space.
@@ -612,45 +1442,43 @@
     /* Initialize the LSI table */
     for (i = 0; i < PCI_NUM_PINS; i++) {
         uint32_t irq;
+        Error *local_err = NULL;
 
-        irq = xics_alloc_block(spapr->icp, 0, 1, true, false);
-        if (!irq) {
-            error_setg(errp, "spapr_allocate_lsi failed");
+        irq = xics_spapr_alloc_block(spapr->xics, 0, 1, true, false,
+                                     &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            error_prepend(errp, "can't allocate LSIs: ");
             return;
         }
 
         sphb->lsi_table[i].irq = irq;
     }
 
-    if (!info->finish_realize) {
-        error_setg(errp, "finish_realize not defined");
-        return;
+    /* allocate connectors for child PCI devices */
+    if (sphb->dr_enabled) {
+        for (i = 0; i < PCI_SLOT_MAX * 8; i++) {
+            spapr_dr_connector_new(OBJECT(phb),
+                                   SPAPR_DR_CONNECTOR_TYPE_PCI,
+                                   (sphb->index << 16) | i);
+        }
     }
 
-    info->finish_realize(sphb, errp);
+    /* DMA setup */
+    for (i = 0; i < windows_supported; ++i) {
+        tcet = spapr_tce_new_table(DEVICE(sphb), sphb->dma_liobn[i]);
+        if (!tcet) {
+            error_setg(errp, "Creating window#%d failed for %s",
+                       i, sphb->dtbusname);
+            return;
+        }
+        memory_region_add_subregion_overlap(&sphb->iommu_root, 0,
+                                            spapr_tce_get_iommu(tcet), 0);
+    }
 
     sphb->msi = g_hash_table_new_full(g_int_hash, g_int_equal, g_free, g_free);
 }
 
-static void spapr_phb_finish_realize(sPAPRPHBState *sphb, Error **errp)
-{
-    sPAPRTCETable *tcet;
-
-    tcet = spapr_tce_new_table(DEVICE(sphb), sphb->dma_liobn,
-                               0,
-                               SPAPR_TCE_PAGE_SHIFT,
-                               0x40000000 >> SPAPR_TCE_PAGE_SHIFT, false);
-    if (!tcet) {
-        error_setg(errp, "Unable to create TCE table for %s",
-                   sphb->dtbusname);
-        return ;
-    }
-
-    /* Register default 32bit DMA window */
-    memory_region_add_subregion(&sphb->iommu_root, 0,
-                                spapr_tce_get_iommu(tcet));
-}
-
 static int spapr_phb_children_reset(Object *child, void *opaque)
 {
     DeviceState *dev = (DeviceState *) object_dynamic_cast(child, TYPE_DEVICE);
@@ -662,22 +1490,60 @@
     return 0;
 }
 
+void spapr_phb_dma_reset(sPAPRPHBState *sphb)
+{
+    int i;
+    sPAPRTCETable *tcet;
+
+    for (i = 0; i < SPAPR_PCI_DMA_MAX_WINDOWS; ++i) {
+        tcet = spapr_tce_find_by_liobn(sphb->dma_liobn[i]);
+
+        if (tcet && tcet->nb_table) {
+            spapr_tce_table_disable(tcet);
+        }
+    }
+
+    /* Register default 32bit DMA window */
+    tcet = spapr_tce_find_by_liobn(sphb->dma_liobn[0]);
+    spapr_tce_table_enable(tcet, SPAPR_TCE_PAGE_SHIFT, sphb->dma_win_addr,
+                           sphb->dma_win_size >> SPAPR_TCE_PAGE_SHIFT);
+}
+
 static void spapr_phb_reset(DeviceState *qdev)
 {
+    sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(qdev);
+
+    spapr_phb_dma_reset(sphb);
+
     /* Reset the IOMMU state */
     object_child_foreach(OBJECT(qdev), spapr_phb_children_reset, NULL);
+
+    if (spapr_phb_eeh_available(SPAPR_PCI_HOST_BRIDGE(qdev))) {
+        spapr_phb_vfio_reset(qdev);
+    }
 }
 
 static Property spapr_phb_properties[] = {
-    DEFINE_PROP_INT32("index", sPAPRPHBState, index, -1),
+    DEFINE_PROP_UINT32("index", sPAPRPHBState, index, -1),
     DEFINE_PROP_UINT64("buid", sPAPRPHBState, buid, -1),
-    DEFINE_PROP_UINT32("liobn", sPAPRPHBState, dma_liobn, -1),
+    DEFINE_PROP_UINT32("liobn", sPAPRPHBState, dma_liobn[0], -1),
+    DEFINE_PROP_UINT32("liobn64", sPAPRPHBState, dma_liobn[1], -1),
     DEFINE_PROP_UINT64("mem_win_addr", sPAPRPHBState, mem_win_addr, -1),
     DEFINE_PROP_UINT64("mem_win_size", sPAPRPHBState, mem_win_size,
                        SPAPR_PCI_MMIO_WIN_SIZE),
     DEFINE_PROP_UINT64("io_win_addr", sPAPRPHBState, io_win_addr, -1),
     DEFINE_PROP_UINT64("io_win_size", sPAPRPHBState, io_win_size,
                        SPAPR_PCI_IO_WIN_SIZE),
+    DEFINE_PROP_BOOL("dynamic-reconfiguration", sPAPRPHBState, dr_enabled,
+                     true),
+    /* Default DMA window is 0..1GB */
+    DEFINE_PROP_UINT64("dma_win_addr", sPAPRPHBState, dma_win_addr, 0),
+    DEFINE_PROP_UINT64("dma_win_size", sPAPRPHBState, dma_win_size, 0x40000000),
+    DEFINE_PROP_UINT64("dma64_win_addr", sPAPRPHBState, dma64_win_addr,
+                       0x800000000000000ULL),
+    DEFINE_PROP_BOOL("ddw", sPAPRPHBState, ddw_enabled, true),
+    DEFINE_PROP_UINT64("pgsz", sPAPRPHBState, page_size_mask,
+                       (1ULL << 12) | (1ULL << 16)),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -704,34 +1570,26 @@
     },
 };
 
-static void spapr_pci_fill_msi_devs(gpointer key, gpointer value,
-                                    gpointer opaque)
-{
-    sPAPRPHBState *sphb = opaque;
-
-    sphb->msi_devs[sphb->msi_devs_num].key = *(uint32_t *)key;
-    sphb->msi_devs[sphb->msi_devs_num].value = *(spapr_pci_msi *)value;
-    sphb->msi_devs_num++;
-}
-
 static void spapr_pci_pre_save(void *opaque)
 {
     sPAPRPHBState *sphb = opaque;
-    int msi_devs_num;
+    GHashTableIter iter;
+    gpointer key, value;
+    int i;
 
-    if (sphb->msi_devs) {
-        g_free(sphb->msi_devs);
-        sphb->msi_devs = NULL;
-    }
-    sphb->msi_devs_num = 0;
-    msi_devs_num = g_hash_table_size(sphb->msi);
-    if (!msi_devs_num) {
+    g_free(sphb->msi_devs);
+    sphb->msi_devs = NULL;
+    sphb->msi_devs_num = g_hash_table_size(sphb->msi);
+    if (!sphb->msi_devs_num) {
         return;
     }
-    sphb->msi_devs = g_malloc(msi_devs_num * sizeof(spapr_pci_msi_mig));
+    sphb->msi_devs = g_malloc(sphb->msi_devs_num * sizeof(spapr_pci_msi_mig));
 
-    g_hash_table_foreach(sphb->msi, spapr_pci_fill_msi_devs, sphb);
-    assert(sphb->msi_devs_num == msi_devs_num);
+    g_hash_table_iter_init(&iter, sphb->msi);
+    for (i = 0; g_hash_table_iter_next(&iter, &key, &value); ++i) {
+        sphb->msi_devs[i].key = *(uint32_t *) key;
+        sphb->msi_devs[i].value = *(spapr_pci_msi *) value;
+    }
 }
 
 static int spapr_pci_post_load(void *opaque, int version_id)
@@ -747,10 +1605,8 @@
                          sizeof(sphb->msi_devs[i].value));
         g_hash_table_insert(sphb->msi, key, value);
     }
-    if (sphb->msi_devs) {
-        g_free(sphb->msi_devs);
-        sphb->msi_devs = NULL;
-    }
+    g_free(sphb->msi_devs);
+    sphb->msi_devs = NULL;
     sphb->msi_devs_num = 0;
 
     return 0;
@@ -764,7 +1620,7 @@
     .post_load = spapr_pci_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64_EQUAL(buid, sPAPRPHBState),
-        VMSTATE_UINT32_EQUAL(dma_liobn, sPAPRPHBState),
+        VMSTATE_UINT32_EQUAL(dma_liobn[0], sPAPRPHBState),
         VMSTATE_UINT64_EQUAL(mem_win_addr, sPAPRPHBState),
         VMSTATE_UINT64_EQUAL(mem_win_size, sPAPRPHBState),
         VMSTATE_UINT64_EQUAL(io_win_addr, sPAPRPHBState),
@@ -790,7 +1646,7 @@
 {
     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
-    sPAPRPHBClass *spc = SPAPR_PCI_HOST_BRIDGE_CLASS(klass);
+    HotplugHandlerClass *hp = HOTPLUG_HANDLER_CLASS(klass);
 
     hc->root_bus_path = spapr_phb_root_bus_path;
     dc->realize = spapr_phb_realize;
@@ -798,8 +1654,8 @@
     dc->reset = spapr_phb_reset;
     dc->vmsd = &vmstate_spapr_pci;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
-    dc->cannot_instantiate_with_device_add_yet = false;
-    spc->finish_realize = spapr_phb_finish_realize;
+    hp->plug = spapr_phb_hot_plug_child;
+    hp->unplug = spapr_phb_hot_unplug_child;
 }
 
 static const TypeInfo spapr_phb_info = {
@@ -807,10 +1663,13 @@
     .parent        = TYPE_PCI_HOST_BRIDGE,
     .instance_size = sizeof(sPAPRPHBState),
     .class_init    = spapr_phb_class_init,
-    .class_size    = sizeof(sPAPRPHBClass),
+    .interfaces    = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
 };
 
-PCIHostState *spapr_create_phb(sPAPREnvironment *spapr, int index)
+PCIHostState *spapr_create_phb(sPAPRMachineState *spapr, int index)
 {
     DeviceState *dev;
 
@@ -821,47 +1680,95 @@
     return PCI_HOST_BRIDGE(dev);
 }
 
-/* Macros to operate with address in OF binding to PCI */
-#define b_x(x, p, l)    (((x) & ((1<<(l))-1)) << (p))
-#define b_n(x)          b_x((x), 31, 1) /* 0 if relocatable */
-#define b_p(x)          b_x((x), 30, 1) /* 1 if prefetchable */
-#define b_t(x)          b_x((x), 29, 1) /* 1 if the address is aliased */
-#define b_ss(x)         b_x((x), 24, 2) /* the space code */
-#define b_bbbbbbbb(x)   b_x((x), 16, 8) /* bus number */
-#define b_ddddd(x)      b_x((x), 11, 5) /* device number */
-#define b_fff(x)        b_x((x), 8, 3)  /* function number */
-#define b_rrrrrrrr(x)   b_x((x), 0, 8)  /* register number */
-
-typedef struct sPAPRTCEDT {
+typedef struct sPAPRFDT {
     void *fdt;
     int node_off;
-} sPAPRTCEDT;
+    sPAPRPHBState *sphb;
+} sPAPRFDT;
 
-static int spapr_phb_children_dt(Object *child, void *opaque)
+static void spapr_populate_pci_devices_dt(PCIBus *bus, PCIDevice *pdev,
+                                          void *opaque)
 {
-    sPAPRTCEDT *p = opaque;
-    sPAPRTCETable *tcet;
+    PCIBus *sec_bus;
+    sPAPRFDT *p = opaque;
+    int offset;
+    sPAPRFDT s_fdt;
 
-    tcet = (sPAPRTCETable *) object_dynamic_cast(child, TYPE_SPAPR_TCE_TABLE);
-    if (!tcet) {
-        return 0;
+    offset = spapr_create_pci_child_dt(p->sphb, pdev, p->fdt, p->node_off);
+    if (!offset) {
+        error_report("Failed to create pci child device tree node");
+        return;
     }
 
-    spapr_dma_dt(p->fdt, p->node_off, "ibm,dma-window",
-                 tcet->liobn, tcet->bus_offset,
-                 tcet->nb_table << tcet->page_shift);
-    /* Stop after the first window */
+    if ((pci_default_read_config(pdev, PCI_HEADER_TYPE, 1) !=
+         PCI_HEADER_TYPE_BRIDGE)) {
+        return;
+    }
 
-    return 1;
+    sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev));
+    if (!sec_bus) {
+        return;
+    }
+
+    s_fdt.fdt = p->fdt;
+    s_fdt.node_off = offset;
+    s_fdt.sphb = p->sphb;
+    pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
+                        spapr_populate_pci_devices_dt,
+                        &s_fdt);
+}
+
+static void spapr_phb_pci_enumerate_bridge(PCIBus *bus, PCIDevice *pdev,
+                                           void *opaque)
+{
+    unsigned int *bus_no = opaque;
+    unsigned int primary = *bus_no;
+    unsigned int subordinate = 0xff;
+    PCIBus *sec_bus = NULL;
+
+    if ((pci_default_read_config(pdev, PCI_HEADER_TYPE, 1) !=
+         PCI_HEADER_TYPE_BRIDGE)) {
+        return;
+    }
+
+    (*bus_no)++;
+    pci_default_write_config(pdev, PCI_PRIMARY_BUS, primary, 1);
+    pci_default_write_config(pdev, PCI_SECONDARY_BUS, *bus_no, 1);
+    pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, *bus_no, 1);
+
+    sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev));
+    if (!sec_bus) {
+        return;
+    }
+
+    pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, subordinate, 1);
+    pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
+                        spapr_phb_pci_enumerate_bridge, bus_no);
+    pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, *bus_no, 1);
+}
+
+static void spapr_phb_pci_enumerate(sPAPRPHBState *phb)
+{
+    PCIBus *bus = PCI_HOST_BRIDGE(phb)->bus;
+    unsigned int bus_no = 0;
+
+    pci_for_each_device(bus, pci_bus_num(bus),
+                        spapr_phb_pci_enumerate_bridge,
+                        &bus_no);
+
 }
 
 int spapr_populate_pci_dt(sPAPRPHBState *phb,
                           uint32_t xics_phandle,
                           void *fdt)
 {
-    int bus_off, i, j;
-    char nodename[256];
+    int bus_off, i, j, ret;
+    char nodename[FDT_NAME_MAX];
     uint32_t bus_range[] = { cpu_to_be32(0), cpu_to_be32(0xff) };
+    const uint64_t mmiosize = memory_region_size(&phb->memwindow);
+    const uint64_t w32max = (1ULL << 32) - SPAPR_PCI_MEM_WIN_BUS_OFFSET;
+    const uint64_t w32size = MIN(w32max, mmiosize);
+    const uint64_t w64size = (mmiosize > w32size) ? (mmiosize - w32size) : 0;
     struct {
         uint32_t hi;
         uint64_t child;
@@ -876,29 +1783,39 @@
         {
             cpu_to_be32(b_ss(2)), cpu_to_be64(SPAPR_PCI_MEM_WIN_BUS_OFFSET),
             cpu_to_be64(phb->mem_win_addr),
-            cpu_to_be64(memory_region_size(&phb->memwindow)),
+            cpu_to_be64(w32size),
+        },
+        {
+            cpu_to_be32(b_ss(3)), cpu_to_be64(1ULL << 32),
+            cpu_to_be64(phb->mem_win_addr + w32size),
+            cpu_to_be64(w64size)
         },
     };
+    const unsigned sizeof_ranges = (w64size ? 3 : 2) * sizeof(ranges[0]);
     uint64_t bus_reg[] = { cpu_to_be64(phb->buid), 0 };
     uint32_t interrupt_map_mask[] = {
         cpu_to_be32(b_ddddd(-1)|b_fff(0)), 0x0, 0x0, cpu_to_be32(-1)};
     uint32_t interrupt_map[PCI_SLOT_MAX * PCI_NUM_PINS][7];
+    uint32_t ddw_applicable[] = {
+        cpu_to_be32(RTAS_IBM_QUERY_PE_DMA_WINDOW),
+        cpu_to_be32(RTAS_IBM_CREATE_PE_DMA_WINDOW),
+        cpu_to_be32(RTAS_IBM_REMOVE_PE_DMA_WINDOW)
+    };
+    uint32_t ddw_extensions[] = {
+        cpu_to_be32(1),
+        cpu_to_be32(RTAS_IBM_RESET_PE_DMA_WINDOW)
+    };
+    sPAPRTCETable *tcet;
+    PCIBus *bus = PCI_HOST_BRIDGE(phb)->bus;
+    sPAPRFDT s_fdt;
 
     /* Start populating the FDT */
-    sprintf(nodename, "pci@%" PRIx64, phb->buid);
+    snprintf(nodename, FDT_NAME_MAX, "pci@%" PRIx64, phb->buid);
     bus_off = fdt_add_subnode(fdt, 0, nodename);
     if (bus_off < 0) {
         return bus_off;
     }
 
-#define _FDT(exp) \
-    do { \
-        int ret = (exp);                                           \
-        if (ret < 0) {                                             \
-            return ret;                                            \
-        }                                                          \
-    } while (0)
-
     /* Write PHB properties */
     _FDT(fdt_setprop_string(fdt, bus_off, "device_type", "pci"));
     _FDT(fdt_setprop_string(fdt, bus_off, "compatible", "IBM,Logical_PHB"));
@@ -907,10 +1824,18 @@
     _FDT(fdt_setprop_cell(fdt, bus_off, "#interrupt-cells", 0x1));
     _FDT(fdt_setprop(fdt, bus_off, "used-by-rtas", NULL, 0));
     _FDT(fdt_setprop(fdt, bus_off, "bus-range", &bus_range, sizeof(bus_range)));
-    _FDT(fdt_setprop(fdt, bus_off, "ranges", &ranges, sizeof(ranges)));
+    _FDT(fdt_setprop(fdt, bus_off, "ranges", &ranges, sizeof_ranges));
     _FDT(fdt_setprop(fdt, bus_off, "reg", &bus_reg, sizeof(bus_reg)));
     _FDT(fdt_setprop_cell(fdt, bus_off, "ibm,pci-config-space-type", 0x1));
-    _FDT(fdt_setprop_cell(fdt, bus_off, "ibm,pe-total-#msi", XICS_IRQS));
+    _FDT(fdt_setprop_cell(fdt, bus_off, "ibm,pe-total-#msi", XICS_IRQS_SPAPR));
+
+    /* Dynamic DMA window */
+    if (phb->ddw_enabled) {
+        _FDT(fdt_setprop(fdt, bus_off, "ibm,ddw-applicable", &ddw_applicable,
+                         sizeof(ddw_applicable)));
+        _FDT(fdt_setprop(fdt, bus_off, "ibm,ddw-extensions",
+                         &ddw_extensions, sizeof(ddw_extensions)));
+    }
 
     /* Build the interrupt-map, this must matches what is done
      * in pci_spapr_map_irq
@@ -935,8 +1860,31 @@
     _FDT(fdt_setprop(fdt, bus_off, "interrupt-map", &interrupt_map,
                      sizeof(interrupt_map)));
 
-    object_child_foreach(OBJECT(phb), spapr_phb_children_dt,
-                         &((sPAPRTCEDT){ .fdt = fdt, .node_off = bus_off }));
+    tcet = spapr_tce_find_by_liobn(phb->dma_liobn[0]);
+    if (!tcet) {
+        return -1;
+    }
+    spapr_dma_dt(fdt, bus_off, "ibm,dma-window",
+                 tcet->liobn, tcet->bus_offset,
+                 tcet->nb_table << tcet->page_shift);
+
+    /* Walk the bridges and program the bus numbers*/
+    spapr_phb_pci_enumerate(phb);
+    _FDT(fdt_setprop_cell(fdt, bus_off, "qemu,phb-enumerated", 0x1));
+
+    /* Populate tree nodes with PCI devices attached */
+    s_fdt.fdt = fdt;
+    s_fdt.node_off = bus_off;
+    s_fdt.sphb = phb;
+    pci_for_each_device(bus, pci_bus_num(bus),
+                        spapr_populate_pci_devices_dt,
+                        &s_fdt);
+
+    ret = spapr_drc_populate_dt(fdt, bus_off, OBJECT(phb),
+                                SPAPR_DR_CONNECTOR_TYPE_PCI);
+    if (ret) {
+        return ret;
+    }
 
     return 0;
 }
@@ -951,13 +1899,32 @@
                         rtas_ibm_read_pci_config);
     spapr_rtas_register(RTAS_IBM_WRITE_PCI_CONFIG, "ibm,write-pci-config",
                         rtas_ibm_write_pci_config);
-    if (msi_supported) {
+    if (msi_nonbroken) {
         spapr_rtas_register(RTAS_IBM_QUERY_INTERRUPT_SOURCE_NUMBER,
                             "ibm,query-interrupt-source-number",
                             rtas_ibm_query_interrupt_source_number);
         spapr_rtas_register(RTAS_IBM_CHANGE_MSI, "ibm,change-msi",
                             rtas_ibm_change_msi);
     }
+
+    spapr_rtas_register(RTAS_IBM_SET_EEH_OPTION,
+                        "ibm,set-eeh-option",
+                        rtas_ibm_set_eeh_option);
+    spapr_rtas_register(RTAS_IBM_GET_CONFIG_ADDR_INFO2,
+                        "ibm,get-config-addr-info2",
+                        rtas_ibm_get_config_addr_info2);
+    spapr_rtas_register(RTAS_IBM_READ_SLOT_RESET_STATE2,
+                        "ibm,read-slot-reset-state2",
+                        rtas_ibm_read_slot_reset_state2);
+    spapr_rtas_register(RTAS_IBM_SET_SLOT_RESET,
+                        "ibm,set-slot-reset",
+                        rtas_ibm_set_slot_reset);
+    spapr_rtas_register(RTAS_IBM_CONFIGURE_PE,
+                        "ibm,configure-pe",
+                        rtas_ibm_configure_pe);
+    spapr_rtas_register(RTAS_IBM_SLOT_ERROR_DETAIL,
+                        "ibm,slot-error-detail",
+                        rtas_ibm_slot_error_detail);
 }
 
 static void spapr_pci_register_types(void)
@@ -966,3 +1933,32 @@
 }
 
 type_init(spapr_pci_register_types)
+
+static int spapr_switch_one_vga(DeviceState *dev, void *opaque)
+{
+    bool be = *(bool *)opaque;
+
+    if (object_dynamic_cast(OBJECT(dev), "VGA")
+        || object_dynamic_cast(OBJECT(dev), "secondary-vga")) {
+        object_property_set_bool(OBJECT(dev), be, "big-endian-framebuffer",
+                                 &error_abort);
+    }
+    return 0;
+}
+
+void spapr_pci_switch_vga(bool big_endian)
+{
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+    sPAPRPHBState *sphb;
+
+    /*
+     * For backward compatibility with existing guests, we switch
+     * the endianness of the VGA controller when changing the guest
+     * interrupt mode
+     */
+    QLIST_FOREACH(sphb, &spapr->phbs, list) {
+        BusState *bus = &PCI_HOST_BRIDGE(sphb)->bus->qbus;
+        qbus_walk_children(bus, spapr_switch_one_vga, NULL, NULL, NULL,
+                           &big_endian);
+    }
+}
diff --git a/hw/ppc/spapr_pci_vfio.c b/hw/ppc/spapr_pci_vfio.c
index d3bddf2..8448e0b 100644
--- a/hw/ppc/spapr_pci_vfio.c
+++ b/hw/ppc/spapr_pci_vfio.c
@@ -17,81 +17,221 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include <linux/vfio.h>
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/ppc/spapr.h"
 #include "hw/pci-host/spapr.h"
-#include "linux/vfio.h"
-#include "hw/misc/vfio.h"
+#include "hw/pci/msix.h"
+#include "hw/vfio/vfio.h"
+#include "qemu/error-report.h"
+#include "sysemu/qtest.h"
+
+#define TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE "spapr-pci-vfio-host-bridge"
+
+#define SPAPR_PCI_VFIO_HOST_BRIDGE(obj) \
+    OBJECT_CHECK(sPAPRPHBVFIOState, (obj), TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE)
+
+typedef struct sPAPRPHBVFIOState sPAPRPHBVFIOState;
+
+struct sPAPRPHBVFIOState {
+    sPAPRPHBState phb;
+
+    int32_t iommugroupid;
+};
 
 static Property spapr_phb_vfio_properties[] = {
     DEFINE_PROP_INT32("iommu", sPAPRPHBVFIOState, iommugroupid, -1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void spapr_phb_vfio_finish_realize(sPAPRPHBState *sphb, Error **errp)
+static void spapr_phb_vfio_instance_init(Object *obj)
 {
-    sPAPRPHBVFIOState *svphb = SPAPR_PCI_VFIO_HOST_BRIDGE(sphb);
-    struct vfio_iommu_spapr_tce_info info = { .argsz = sizeof(info) };
-    int ret;
-    sPAPRTCETable *tcet;
-    uint32_t liobn = svphb->phb.dma_liobn;
-
-    if (svphb->iommugroupid == -1) {
-        error_setg(errp, "Wrong IOMMU group ID %d", svphb->iommugroupid);
-        return;
+    if (!qtest_enabled()) {
+        error_report("spapr-pci-vfio-host-bridge is deprecated");
     }
-
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_CHECK_EXTENSION,
-                               (void *) VFIO_SPAPR_TCE_IOMMU);
-    if (ret != 1) {
-        error_setg_errno(errp, -ret,
-                         "spapr-vfio: SPAPR extension is not supported");
-        return;
-    }
-
-    ret = vfio_container_ioctl(&svphb->phb.iommu_as, svphb->iommugroupid,
-                               VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
-    if (ret) {
-        error_setg_errno(errp, -ret,
-                         "spapr-vfio: get info from container failed");
-        return;
-    }
-
-    tcet = spapr_tce_new_table(DEVICE(sphb), liobn, info.dma32_window_start,
-                               SPAPR_TCE_PAGE_SHIFT,
-                               info.dma32_window_size >> SPAPR_TCE_PAGE_SHIFT,
-                               true);
-    if (!tcet) {
-        error_setg(errp, "spapr-vfio: failed to create VFIO TCE table");
-        return;
-    }
-
-    /* Register default 32bit DMA window */
-    memory_region_add_subregion(&sphb->iommu_root, tcet->bus_offset,
-                                spapr_tce_get_iommu(tcet));
 }
 
-static void spapr_phb_vfio_reset(DeviceState *qdev)
+bool spapr_phb_eeh_available(sPAPRPHBState *sphb)
 {
-    /* Do nothing */
+    return vfio_eeh_as_ok(&sphb->iommu_as);
+}
+
+static void spapr_phb_vfio_eeh_reenable(sPAPRPHBState *sphb)
+{
+    vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_ENABLE);
+}
+
+void spapr_phb_vfio_reset(DeviceState *qdev)
+{
+    /*
+     * The PE might be in frozen state. To reenable the EEH
+     * functionality on it will clean the frozen state, which
+     * ensures that the contained PCI devices will work properly
+     * after reboot.
+     */
+    spapr_phb_vfio_eeh_reenable(SPAPR_PCI_HOST_BRIDGE(qdev));
+}
+
+int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
+                                  unsigned int addr, int option)
+{
+    uint32_t op;
+    int ret;
+
+    switch (option) {
+    case RTAS_EEH_DISABLE:
+        op = VFIO_EEH_PE_DISABLE;
+        break;
+    case RTAS_EEH_ENABLE: {
+        PCIHostState *phb;
+        PCIDevice *pdev;
+
+        /*
+         * The EEH functionality is enabled on basis of PCI device,
+         * instead of PE. We need check the validity of the PCI
+         * device address.
+         */
+        phb = PCI_HOST_BRIDGE(sphb);
+        pdev = pci_find_device(phb->bus,
+                               (addr >> 16) & 0xFF, (addr >> 8) & 0xFF);
+        if (!pdev || !object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+            return RTAS_OUT_PARAM_ERROR;
+        }
+
+        op = VFIO_EEH_PE_ENABLE;
+        break;
+    }
+    case RTAS_EEH_THAW_IO:
+        op = VFIO_EEH_PE_UNFREEZE_IO;
+        break;
+    case RTAS_EEH_THAW_DMA:
+        op = VFIO_EEH_PE_UNFREEZE_DMA;
+        break;
+    default:
+        return RTAS_OUT_PARAM_ERROR;
+    }
+
+    ret = vfio_eeh_as_op(&sphb->iommu_as, op);
+    if (ret < 0) {
+        return RTAS_OUT_HW_ERROR;
+    }
+
+    return RTAS_OUT_SUCCESS;
+}
+
+int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state)
+{
+    int ret;
+
+    ret = vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_GET_STATE);
+    if (ret < 0) {
+        return RTAS_OUT_PARAM_ERROR;
+    }
+
+    *state = ret;
+    return RTAS_OUT_SUCCESS;
+}
+
+static void spapr_phb_vfio_eeh_clear_dev_msix(PCIBus *bus,
+                                              PCIDevice *pdev,
+                                              void *opaque)
+{
+    /* Check if the device is VFIO PCI device */
+    if (!object_dynamic_cast(OBJECT(pdev), "vfio-pci")) {
+        return;
+    }
+
+    /*
+     * The MSIx table will be cleaned out by reset. We need
+     * disable it so that it can be reenabled properly. Also,
+     * the cached MSIx table should be cleared as it's not
+     * reflecting the contents in hardware.
+     */
+    if (msix_enabled(pdev)) {
+        uint16_t flags;
+
+        flags = pci_host_config_read_common(pdev,
+                                            pdev->msix_cap + PCI_MSIX_FLAGS,
+                                            pci_config_size(pdev), 2);
+        flags &= ~PCI_MSIX_FLAGS_ENABLE;
+        pci_host_config_write_common(pdev,
+                                     pdev->msix_cap + PCI_MSIX_FLAGS,
+                                     pci_config_size(pdev), flags, 2);
+    }
+
+    msix_reset(pdev);
+}
+
+static void spapr_phb_vfio_eeh_clear_bus_msix(PCIBus *bus, void *opaque)
+{
+       pci_for_each_device(bus, pci_bus_num(bus),
+                           spapr_phb_vfio_eeh_clear_dev_msix, NULL);
+}
+
+static void spapr_phb_vfio_eeh_pre_reset(sPAPRPHBState *sphb)
+{
+       PCIHostState *phb = PCI_HOST_BRIDGE(sphb);
+
+       pci_for_each_bus(phb->bus, spapr_phb_vfio_eeh_clear_bus_msix, NULL);
+}
+
+int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
+{
+    uint32_t op;
+    int ret;
+
+    switch (option) {
+    case RTAS_SLOT_RESET_DEACTIVATE:
+        op = VFIO_EEH_PE_RESET_DEACTIVATE;
+        break;
+    case RTAS_SLOT_RESET_HOT:
+        spapr_phb_vfio_eeh_pre_reset(sphb);
+        op = VFIO_EEH_PE_RESET_HOT;
+        break;
+    case RTAS_SLOT_RESET_FUNDAMENTAL:
+        spapr_phb_vfio_eeh_pre_reset(sphb);
+        op = VFIO_EEH_PE_RESET_FUNDAMENTAL;
+        break;
+    default:
+        return RTAS_OUT_PARAM_ERROR;
+    }
+
+    ret = vfio_eeh_as_op(&sphb->iommu_as, op);
+    if (ret < 0) {
+        return RTAS_OUT_HW_ERROR;
+    }
+
+    return RTAS_OUT_SUCCESS;
+}
+
+int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
+{
+    int ret;
+
+    ret = vfio_eeh_as_op(&sphb->iommu_as, VFIO_EEH_PE_CONFIGURE);
+    if (ret < 0) {
+        return RTAS_OUT_PARAM_ERROR;
+    }
+
+    return RTAS_OUT_SUCCESS;
 }
 
 static void spapr_phb_vfio_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    sPAPRPHBClass *spc = SPAPR_PCI_HOST_BRIDGE_CLASS(klass);
 
     dc->props = spapr_phb_vfio_properties;
-    dc->reset = spapr_phb_vfio_reset;
-    spc->finish_realize = spapr_phb_vfio_finish_realize;
 }
 
 static const TypeInfo spapr_phb_vfio_info = {
     .name          = TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE,
     .parent        = TYPE_SPAPR_PCI_HOST_BRIDGE,
     .instance_size = sizeof(sPAPRPHBVFIOState),
+    .instance_init = spapr_phb_vfio_instance_init,
     .class_init    = spapr_phb_vfio_class_init,
-    .class_size    = sizeof(sPAPRPHBClass),
 };
 
 static void spapr_pci_vfio_register_types(void)
diff --git a/hw/ppc/spapr_rng.c b/hw/ppc/spapr_rng.c
new file mode 100644
index 0000000..80515eb
--- /dev/null
+++ b/hw/ppc/spapr_rng.c
@@ -0,0 +1,191 @@
+/*
+ * QEMU sPAPR random number generator "device" for H_RANDOM hypercall
+ *
+ * Copyright 2015 Thomas Huth, Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/device_tree.h"
+#include "sysemu/rng.h"
+#include "hw/ppc/spapr.h"
+#include "kvm_ppc.h"
+
+#define SPAPR_RNG(obj) \
+    OBJECT_CHECK(sPAPRRngState, (obj), TYPE_SPAPR_RNG)
+
+struct sPAPRRngState {
+    /*< private >*/
+    DeviceState ds;
+    RngBackend *backend;
+    bool use_kvm;
+};
+typedef struct sPAPRRngState sPAPRRngState;
+
+struct HRandomData {
+    QemuSemaphore sem;
+    union {
+        uint64_t v64;
+        uint8_t v8[8];
+    } val;
+    int received;
+};
+typedef struct HRandomData HRandomData;
+
+/* Callback function for the RngBackend */
+static void random_recv(void *dest, const void *src, size_t size)
+{
+    HRandomData *hrdp = dest;
+
+    if (src && size > 0) {
+        assert(size + hrdp->received <= sizeof(hrdp->val.v8));
+        memcpy(&hrdp->val.v8[hrdp->received], src, size);
+        hrdp->received += size;
+    }
+
+    qemu_sem_post(&hrdp->sem);
+}
+
+/* Handler for the H_RANDOM hypercall */
+static target_ulong h_random(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                             target_ulong opcode, target_ulong *args)
+{
+    sPAPRRngState *rngstate;
+    HRandomData hrdata;
+
+    rngstate = SPAPR_RNG(object_resolve_path_type("", TYPE_SPAPR_RNG, NULL));
+
+    if (!rngstate || !rngstate->backend) {
+        return H_HARDWARE;
+    }
+
+    qemu_sem_init(&hrdata.sem, 0);
+    hrdata.val.v64 = 0;
+    hrdata.received = 0;
+
+    while (hrdata.received < 8) {
+        rng_backend_request_entropy(rngstate->backend, 8 - hrdata.received,
+                                    random_recv, &hrdata);
+        qemu_mutex_unlock_iothread();
+        qemu_sem_wait(&hrdata.sem);
+        qemu_mutex_lock_iothread();
+    }
+
+    qemu_sem_destroy(&hrdata.sem);
+    args[0] = hrdata.val.v64;
+
+    return H_SUCCESS;
+}
+
+static void spapr_rng_instance_init(Object *obj)
+{
+    sPAPRRngState *rngstate = SPAPR_RNG(obj);
+
+    if (object_resolve_path_type("", TYPE_SPAPR_RNG, NULL) != NULL) {
+        error_report("spapr-rng can not be instantiated twice!");
+        return;
+    }
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&rngstate->backend,
+                             object_property_allow_set_link,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL);
+    object_property_set_description(obj, "rng",
+                                    "ID of the random number generator backend",
+                                    NULL);
+}
+
+static void spapr_rng_realize(DeviceState *dev, Error **errp)
+{
+
+    sPAPRRngState *rngstate = SPAPR_RNG(dev);
+
+    if (rngstate->use_kvm) {
+        if (kvmppc_enable_hwrng() == 0) {
+            return;
+        }
+        /*
+         * If user specified both, use-kvm and a backend, we fall back to
+         * the backend now. If not, provide an appropriate error message.
+         */
+        if (!rngstate->backend) {
+            error_setg(errp, "Could not initialize in-kernel H_RANDOM call!");
+            return;
+        }
+    }
+
+    if (rngstate->backend) {
+        spapr_register_hypercall(H_RANDOM, h_random);
+    } else {
+        error_setg(errp, "spapr-rng needs an RNG backend!");
+    }
+}
+
+int spapr_rng_populate_dt(void *fdt)
+{
+    int node;
+    int ret;
+
+    node = qemu_fdt_add_subnode(fdt, "/ibm,platform-facilities");
+    if (node <= 0) {
+        return -1;
+    }
+    ret = fdt_setprop_string(fdt, node, "device_type",
+                             "ibm,platform-facilities");
+    ret |= fdt_setprop_cell(fdt, node, "#address-cells", 0x1);
+    ret |= fdt_setprop_cell(fdt, node, "#size-cells", 0x0);
+
+    node = fdt_add_subnode(fdt, node, "ibm,random-v1");
+    if (node <= 0) {
+        return -1;
+    }
+    ret |= fdt_setprop_string(fdt, node, "compatible", "ibm,random");
+
+    return ret ? -1 : 0;
+}
+
+static Property spapr_rng_properties[] = {
+    DEFINE_PROP_BOOL("use-kvm", sPAPRRngState, use_kvm, false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void spapr_rng_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = spapr_rng_realize;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->props = spapr_rng_properties;
+    dc->hotpluggable = false;
+}
+
+static const TypeInfo spapr_rng_info = {
+    .name          = TYPE_SPAPR_RNG,
+    .parent        = TYPE_DEVICE,
+    .instance_size = sizeof(sPAPRRngState),
+    .instance_init = spapr_rng_instance_init,
+    .class_init    = spapr_rng_class_init,
+};
+
+static void spapr_rng_register_type(void)
+{
+    type_register_static(&spapr_rng_info);
+}
+type_init(spapr_rng_register_type)
diff --git a/hw/ppc/spapr_rtas.c b/hw/ppc/spapr_rtas.c
index 2ec2a8e..dc058e5 100644
--- a/hw/ppc/spapr_rtas.c
+++ b/hw/ppc/spapr_rtas.c
@@ -24,19 +24,75 @@
  * THE SOFTWARE.
  *
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "qemu/log.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/char.h"
 #include "hw/qdev.h"
 #include "sysemu/device_tree.h"
+#include "sysemu/cpus.h"
+#include "sysemu/kvm.h"
 
 #include "hw/ppc/spapr.h"
 #include "hw/ppc/spapr_vio.h"
+#include "hw/ppc/ppc.h"
 #include "qapi-event.h"
+#include "hw/boards.h"
 
 #include <libfdt.h>
+#include "hw/ppc/spapr_drc.h"
+#include "qemu/cutils.h"
 
-static void rtas_display_character(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+/* #define DEBUG_SPAPR */
+
+#ifdef DEBUG_SPAPR
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+static sPAPRConfigureConnectorState *spapr_ccs_find(sPAPRMachineState *spapr,
+                                                    uint32_t drc_index)
+{
+    sPAPRConfigureConnectorState *ccs = NULL;
+
+    QTAILQ_FOREACH(ccs, &spapr->ccs_list, next) {
+        if (ccs->drc_index == drc_index) {
+            break;
+        }
+    }
+
+    return ccs;
+}
+
+static void spapr_ccs_add(sPAPRMachineState *spapr,
+                          sPAPRConfigureConnectorState *ccs)
+{
+    g_assert(!spapr_ccs_find(spapr, ccs->drc_index));
+    QTAILQ_INSERT_HEAD(&spapr->ccs_list, ccs, next);
+}
+
+static void spapr_ccs_remove(sPAPRMachineState *spapr,
+                             sPAPRConfigureConnectorState *ccs)
+{
+    QTAILQ_REMOVE(&spapr->ccs_list, ccs, next);
+    g_free(ccs);
+}
+
+void spapr_ccs_reset_hook(void *opaque)
+{
+    sPAPRMachineState *spapr = opaque;
+    sPAPRConfigureConnectorState *ccs, *ccs_tmp;
+
+    QTAILQ_FOREACH_SAFE(ccs, &spapr->ccs_list, next, ccs_tmp) {
+        spapr_ccs_remove(spapr, ccs);
+    }
+}
+
+static void rtas_display_character(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                    uint32_t token, uint32_t nargs,
                                    target_ulong args,
                                    uint32_t nret, target_ulong rets)
@@ -52,52 +108,7 @@
     }
 }
 
-static void rtas_get_time_of_day(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                                 uint32_t token, uint32_t nargs,
-                                 target_ulong args,
-                                 uint32_t nret, target_ulong rets)
-{
-    struct tm tm;
-
-    if (nret != 8) {
-        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
-        return;
-    }
-
-    qemu_get_timedate(&tm, spapr->rtc_offset);
-
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-    rtas_st(rets, 1, tm.tm_year + 1900);
-    rtas_st(rets, 2, tm.tm_mon + 1);
-    rtas_st(rets, 3, tm.tm_mday);
-    rtas_st(rets, 4, tm.tm_hour);
-    rtas_st(rets, 5, tm.tm_min);
-    rtas_st(rets, 6, tm.tm_sec);
-    rtas_st(rets, 7, 0); /* we don't do nanoseconds */
-}
-
-static void rtas_set_time_of_day(PowerPCCPU *cpu, sPAPREnvironment *spapr,
-                                 uint32_t token, uint32_t nargs,
-                                 target_ulong args,
-                                 uint32_t nret, target_ulong rets)
-{
-    struct tm tm;
-
-    tm.tm_year = rtas_ld(args, 0) - 1900;
-    tm.tm_mon = rtas_ld(args, 1) - 1;
-    tm.tm_mday = rtas_ld(args, 2);
-    tm.tm_hour = rtas_ld(args, 3);
-    tm.tm_min = rtas_ld(args, 4);
-    tm.tm_sec = rtas_ld(args, 5);
-
-    /* Just generate a monitor event for the change */
-    qapi_event_send_rtc_change(qemu_timedate_diff(&tm), &error_abort);
-    spapr->rtc_offset = qemu_timedate_diff(&tm);
-
-    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
-}
-
-static void rtas_power_off(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_power_off(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                            uint32_t token, uint32_t nargs, target_ulong args,
                            uint32_t nret, target_ulong rets)
 {
@@ -106,10 +117,11 @@
         return;
     }
     qemu_system_shutdown_request();
+    cpu_stop_current();
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
 }
 
-static void rtas_system_reboot(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_system_reboot(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                uint32_t token, uint32_t nargs,
                                target_ulong args,
                                uint32_t nret, target_ulong rets)
@@ -123,7 +135,7 @@
 }
 
 static void rtas_query_cpu_stopped_state(PowerPCCPU *cpu_,
-                                         sPAPREnvironment *spapr,
+                                         sPAPRMachineState *spapr,
                                          uint32_t token, uint32_t nargs,
                                          target_ulong args,
                                          uint32_t nret, target_ulong rets)
@@ -153,7 +165,28 @@
     rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
 }
 
-static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPREnvironment *spapr,
+/*
+ * Set the timebase offset of the CPU to that of first CPU.
+ * This helps hotplugged CPU to have the correct timebase offset.
+ */
+static void spapr_cpu_update_tb_offset(PowerPCCPU *cpu)
+{
+    PowerPCCPU *fcpu = POWERPC_CPU(first_cpu);
+
+    cpu->env.tb_env->tb_offset = fcpu->env.tb_env->tb_offset;
+}
+
+static void spapr_cpu_set_endianness(PowerPCCPU *cpu)
+{
+    PowerPCCPU *fcpu = POWERPC_CPU(first_cpu);
+    PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(fcpu);
+
+    if (!pcc->interrupts_big_endian(fcpu)) {
+        cpu->env.spr[SPR_LPCR] |= LPCR_ILE;
+    }
+}
+
+static void rtas_start_cpu(PowerPCCPU *cpu_, sPAPRMachineState *spapr,
                            uint32_t token, uint32_t nargs,
                            target_ulong args,
                            uint32_t nret, target_ulong rets)
@@ -189,6 +222,8 @@
         env->nip = start;
         env->gpr[3] = r3;
         cs->halted = 0;
+        spapr_cpu_set_endianness(cpu);
+        spapr_cpu_update_tb_offset(cpu);
 
         qemu_cpu_kick(cs);
 
@@ -200,7 +235,7 @@
     rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
 }
 
-static void rtas_stop_self(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_stop_self(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                            uint32_t token, uint32_t nargs,
                            target_ulong args,
                            uint32_t nret, target_ulong rets)
@@ -209,7 +244,7 @@
     CPUPPCState *env = &cpu->env;
 
     cs->halted = 1;
-    cpu_exit(cs);
+    qemu_cpu_kick(cs);
     /*
      * While stopping a CPU, the guest calls H_CPPR which
      * effectively disables interrupts on XICS level.
@@ -222,8 +257,21 @@
     env->msr = 0;
 }
 
+static inline int sysparm_st(target_ulong addr, target_ulong len,
+                             const void *val, uint16_t vallen)
+{
+    hwaddr phys = ppc64_phys_to_real(addr);
+
+    if (len < 2) {
+        return RTAS_OUT_SYSPARM_PARAM_ERROR;
+    }
+    stw_be_phys(&address_space_memory, phys, vallen);
+    cpu_physical_memory_write(phys + 2, val, MIN(len - 2, vallen));
+    return RTAS_OUT_SUCCESS;
+}
+
 static void rtas_ibm_get_system_parameter(PowerPCCPU *cpu,
-                                          sPAPREnvironment *spapr,
+                                          sPAPRMachineState *spapr,
                                           uint32_t token, uint32_t nargs,
                                           target_ulong args,
                                           uint32_t nret, target_ulong rets)
@@ -231,24 +279,30 @@
     target_ulong parameter = rtas_ld(args, 0);
     target_ulong buffer = rtas_ld(args, 1);
     target_ulong length = rtas_ld(args, 2);
-    target_ulong ret = RTAS_OUT_SUCCESS;
+    target_ulong ret;
 
     switch (parameter) {
     case RTAS_SYSPARM_SPLPAR_CHARACTERISTICS: {
-        char *param_val = g_strdup_printf("MaxEntCap=%d,MaxPlatProcs=%d",
-                                          max_cpus, smp_cpus);
-        rtas_st_buffer(buffer, length, (uint8_t *)param_val, strlen(param_val));
+        char *param_val = g_strdup_printf("MaxEntCap=%d,"
+                                          "DesMem=%llu,"
+                                          "DesProcs=%d,"
+                                          "MaxPlatProcs=%d",
+                                          max_cpus,
+                                          current_machine->ram_size / M_BYTE,
+                                          smp_cpus,
+                                          max_cpus);
+        ret = sysparm_st(buffer, length, param_val, strlen(param_val) + 1);
         g_free(param_val);
         break;
     }
     case RTAS_SYSPARM_DIAGNOSTICS_RUN_MODE: {
         uint8_t param_val = DIAGNOSTICS_RUN_MODE_DISABLED;
 
-        rtas_st_buffer(buffer, length, &param_val, sizeof(param_val));
+        ret = sysparm_st(buffer, length, &param_val, sizeof(param_val));
         break;
     }
     case RTAS_SYSPARM_UUID:
-        rtas_st_buffer(buffer, length, qemu_uuid, (qemu_uuid_set ? 16 : 0));
+        ret = sysparm_st(buffer, length, qemu_uuid, (qemu_uuid_set ? 16 : 0));
         break;
     default:
         ret = RTAS_OUT_NOT_SUPPORTED;
@@ -258,7 +312,7 @@
 }
 
 static void rtas_ibm_set_system_parameter(PowerPCCPU *cpu,
-                                          sPAPREnvironment *spapr,
+                                          sPAPRMachineState *spapr,
                                           uint32_t token, uint32_t nargs,
                                           target_ulong args,
                                           uint32_t nret, target_ulong rets)
@@ -278,7 +332,7 @@
 }
 
 static void rtas_ibm_os_term(PowerPCCPU *cpu,
-                            sPAPREnvironment *spapr,
+                            sPAPRMachineState *spapr,
                             uint32_t token, uint32_t nargs,
                             target_ulong args,
                             uint32_t nret, target_ulong rets)
@@ -290,12 +344,327 @@
     rtas_st(rets, 0, ret);
 }
 
+static void rtas_set_power_level(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                 uint32_t token, uint32_t nargs,
+                                 target_ulong args, uint32_t nret,
+                                 target_ulong rets)
+{
+    int32_t power_domain;
+
+    if (nargs != 2 || nret != 2) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    /* we currently only use a single, "live insert" powerdomain for
+     * hotplugged/dlpar'd resources, so the power is always live/full (100)
+     */
+    power_domain = rtas_ld(args, 0);
+    if (power_domain != -1) {
+        rtas_st(rets, 0, RTAS_OUT_NOT_SUPPORTED);
+        return;
+    }
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, 100);
+}
+
+static void rtas_get_power_level(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                  uint32_t token, uint32_t nargs,
+                                  target_ulong args, uint32_t nret,
+                                  target_ulong rets)
+{
+    int32_t power_domain;
+
+    if (nargs != 1 || nret != 2) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    /* we currently only use a single, "live insert" powerdomain for
+     * hotplugged/dlpar'd resources, so the power is always live/full (100)
+     */
+    power_domain = rtas_ld(args, 0);
+    if (power_domain != -1) {
+        rtas_st(rets, 0, RTAS_OUT_NOT_SUPPORTED);
+        return;
+    }
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, 100);
+}
+
+static bool sensor_type_is_dr(uint32_t sensor_type)
+{
+    switch (sensor_type) {
+    case RTAS_SENSOR_TYPE_ISOLATION_STATE:
+    case RTAS_SENSOR_TYPE_DR:
+    case RTAS_SENSOR_TYPE_ALLOCATION_STATE:
+        return true;
+    }
+
+    return false;
+}
+
+static void rtas_set_indicator(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                               uint32_t token, uint32_t nargs,
+                               target_ulong args, uint32_t nret,
+                               target_ulong rets)
+{
+    uint32_t sensor_type;
+    uint32_t sensor_index;
+    uint32_t sensor_state;
+    uint32_t ret = RTAS_OUT_SUCCESS;
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+
+    if (nargs != 3 || nret != 1) {
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
+    }
+
+    sensor_type = rtas_ld(args, 0);
+    sensor_index = rtas_ld(args, 1);
+    sensor_state = rtas_ld(args, 2);
+
+    if (!sensor_type_is_dr(sensor_type)) {
+        goto out_unimplemented;
+    }
+
+    /* if this is a DR sensor we can assume sensor_index == drc_index */
+    drc = spapr_dr_connector_by_index(sensor_index);
+    if (!drc) {
+        DPRINTF("rtas_set_indicator: invalid sensor/DRC index: %xh\n",
+                sensor_index);
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
+    }
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+
+    switch (sensor_type) {
+    case RTAS_SENSOR_TYPE_ISOLATION_STATE:
+        /* if the guest is configuring a device attached to this
+         * DRC, we should reset the configuration state at this
+         * point since it may no longer be reliable (guest released
+         * device and needs to start over, or unplug occurred so
+         * the FDT is no longer valid)
+         */
+        if (sensor_state == SPAPR_DR_ISOLATION_STATE_ISOLATED) {
+            sPAPRConfigureConnectorState *ccs = spapr_ccs_find(spapr,
+                                                               sensor_index);
+            if (ccs) {
+                spapr_ccs_remove(spapr, ccs);
+            }
+        }
+        ret = drck->set_isolation_state(drc, sensor_state);
+        break;
+    case RTAS_SENSOR_TYPE_DR:
+        ret = drck->set_indicator_state(drc, sensor_state);
+        break;
+    case RTAS_SENSOR_TYPE_ALLOCATION_STATE:
+        ret = drck->set_allocation_state(drc, sensor_state);
+        break;
+    default:
+        goto out_unimplemented;
+    }
+
+out:
+    rtas_st(rets, 0, ret);
+    return;
+
+out_unimplemented:
+    /* currently only DR-related sensors are implemented */
+    DPRINTF("rtas_set_indicator: sensor/indicator not implemented: %d\n",
+            sensor_type);
+    rtas_st(rets, 0, RTAS_OUT_NOT_SUPPORTED);
+}
+
+static void rtas_get_sensor_state(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                  uint32_t token, uint32_t nargs,
+                                  target_ulong args, uint32_t nret,
+                                  target_ulong rets)
+{
+    uint32_t sensor_type;
+    uint32_t sensor_index;
+    uint32_t sensor_state = 0;
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    uint32_t ret = RTAS_OUT_SUCCESS;
+
+    if (nargs != 2 || nret != 2) {
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
+    }
+
+    sensor_type = rtas_ld(args, 0);
+    sensor_index = rtas_ld(args, 1);
+
+    if (sensor_type != RTAS_SENSOR_TYPE_ENTITY_SENSE) {
+        /* currently only DR-related sensors are implemented */
+        DPRINTF("rtas_get_sensor_state: sensor/indicator not implemented: %d\n",
+                sensor_type);
+        ret = RTAS_OUT_NOT_SUPPORTED;
+        goto out;
+    }
+
+    drc = spapr_dr_connector_by_index(sensor_index);
+    if (!drc) {
+        DPRINTF("rtas_get_sensor_state: invalid sensor/DRC index: %xh\n",
+                sensor_index);
+        ret = RTAS_OUT_PARAM_ERROR;
+        goto out;
+    }
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    ret = drck->entity_sense(drc, &sensor_state);
+
+out:
+    rtas_st(rets, 0, ret);
+    rtas_st(rets, 1, sensor_state);
+}
+
+/* configure-connector work area offsets, int32_t units for field
+ * indexes, bytes for field offset/len values.
+ *
+ * as documented by PAPR+ v2.7, 13.5.3.5
+ */
+#define CC_IDX_NODE_NAME_OFFSET 2
+#define CC_IDX_PROP_NAME_OFFSET 2
+#define CC_IDX_PROP_LEN 3
+#define CC_IDX_PROP_DATA_OFFSET 4
+#define CC_VAL_DATA_OFFSET ((CC_IDX_PROP_DATA_OFFSET + 1) * 4)
+#define CC_WA_LEN 4096
+
+static void configure_connector_st(target_ulong addr, target_ulong offset,
+                                   const void *buf, size_t len)
+{
+    cpu_physical_memory_write(ppc64_phys_to_real(addr + offset),
+                              buf, MIN(len, CC_WA_LEN - offset));
+}
+
+static void rtas_ibm_configure_connector(PowerPCCPU *cpu,
+                                         sPAPRMachineState *spapr,
+                                         uint32_t token, uint32_t nargs,
+                                         target_ulong args, uint32_t nret,
+                                         target_ulong rets)
+{
+    uint64_t wa_addr;
+    uint64_t wa_offset;
+    uint32_t drc_index;
+    sPAPRDRConnector *drc;
+    sPAPRDRConnectorClass *drck;
+    sPAPRConfigureConnectorState *ccs;
+    sPAPRDRCCResponse resp = SPAPR_DR_CC_RESPONSE_CONTINUE;
+    int rc;
+    const void *fdt;
+
+    if (nargs != 2 || nret != 1) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    wa_addr = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 0);
+
+    drc_index = rtas_ld(wa_addr, 0);
+    drc = spapr_dr_connector_by_index(drc_index);
+    if (!drc) {
+        DPRINTF("rtas_ibm_configure_connector: invalid DRC index: %xh\n",
+                drc_index);
+        rc = RTAS_OUT_PARAM_ERROR;
+        goto out;
+    }
+
+    drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc);
+    fdt = drck->get_fdt(drc, NULL);
+    if (!fdt) {
+        DPRINTF("rtas_ibm_configure_connector: Missing FDT for DRC index: %xh\n",
+                drc_index);
+        rc = SPAPR_DR_CC_RESPONSE_NOT_CONFIGURABLE;
+        goto out;
+    }
+
+    ccs = spapr_ccs_find(spapr, drc_index);
+    if (!ccs) {
+        ccs = g_new0(sPAPRConfigureConnectorState, 1);
+        (void)drck->get_fdt(drc, &ccs->fdt_offset);
+        ccs->drc_index = drc_index;
+        spapr_ccs_add(spapr, ccs);
+    }
+
+    do {
+        uint32_t tag;
+        const char *name;
+        const struct fdt_property *prop;
+        int fdt_offset_next, prop_len;
+
+        tag = fdt_next_tag(fdt, ccs->fdt_offset, &fdt_offset_next);
+
+        switch (tag) {
+        case FDT_BEGIN_NODE:
+            ccs->fdt_depth++;
+            name = fdt_get_name(fdt, ccs->fdt_offset, NULL);
+
+            /* provide the name of the next OF node */
+            wa_offset = CC_VAL_DATA_OFFSET;
+            rtas_st(wa_addr, CC_IDX_NODE_NAME_OFFSET, wa_offset);
+            configure_connector_st(wa_addr, wa_offset, name, strlen(name) + 1);
+            resp = SPAPR_DR_CC_RESPONSE_NEXT_CHILD;
+            break;
+        case FDT_END_NODE:
+            ccs->fdt_depth--;
+            if (ccs->fdt_depth == 0) {
+                /* done sending the device tree, don't need to track
+                 * the state anymore
+                 */
+                drck->set_configured(drc);
+                spapr_ccs_remove(spapr, ccs);
+                ccs = NULL;
+                resp = SPAPR_DR_CC_RESPONSE_SUCCESS;
+            } else {
+                resp = SPAPR_DR_CC_RESPONSE_PREV_PARENT;
+            }
+            break;
+        case FDT_PROP:
+            prop = fdt_get_property_by_offset(fdt, ccs->fdt_offset,
+                                              &prop_len);
+            name = fdt_string(fdt, fdt32_to_cpu(prop->nameoff));
+
+            /* provide the name of the next OF property */
+            wa_offset = CC_VAL_DATA_OFFSET;
+            rtas_st(wa_addr, CC_IDX_PROP_NAME_OFFSET, wa_offset);
+            configure_connector_st(wa_addr, wa_offset, name, strlen(name) + 1);
+
+            /* provide the length and value of the OF property. data gets
+             * placed immediately after NULL terminator of the OF property's
+             * name string
+             */
+            wa_offset += strlen(name) + 1,
+            rtas_st(wa_addr, CC_IDX_PROP_LEN, prop_len);
+            rtas_st(wa_addr, CC_IDX_PROP_DATA_OFFSET, wa_offset);
+            configure_connector_st(wa_addr, wa_offset, prop->data, prop_len);
+            resp = SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY;
+            break;
+        case FDT_END:
+            resp = SPAPR_DR_CC_RESPONSE_ERROR;
+        default:
+            /* keep seeking for an actionable tag */
+            break;
+        }
+        if (ccs) {
+            ccs->fdt_offset = fdt_offset_next;
+        }
+    } while (resp == SPAPR_DR_CC_RESPONSE_CONTINUE);
+
+    rc = resp;
+out:
+    rtas_st(rets, 0, rc);
+}
+
 static struct rtas_call {
     const char *name;
     spapr_rtas_fn fn;
 } rtas_table[RTAS_TOKEN_MAX - RTAS_TOKEN_BASE];
 
-target_ulong spapr_rtas_call(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+target_ulong spapr_rtas_call(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                              uint32_t token, uint32_t nargs, target_ulong args,
                              uint32_t nret, target_ulong rets)
 {
@@ -324,17 +693,11 @@
 
 void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn)
 {
-    if (!((token >= RTAS_TOKEN_BASE) && (token < RTAS_TOKEN_MAX))) {
-        fprintf(stderr, "RTAS invalid token 0x%x\n", token);
-        exit(1);
-    }
+    assert((token >= RTAS_TOKEN_BASE) && (token < RTAS_TOKEN_MAX));
 
     token -= RTAS_TOKEN_BASE;
-    if (rtas_table[token].name) {
-        fprintf(stderr, "RTAS call \"%s\" is registered already as 0x%x\n",
-                rtas_table[token].name, token);
-        exit(1);
-    }
+
+    assert(!rtas_table[token].name);
 
     rtas_table[token].name = name;
     rtas_table[token].fn = fn;
@@ -345,6 +708,11 @@
 {
     int ret;
     int i;
+    uint32_t lrdr_capacity[5];
+    MachineState *machine = MACHINE(qdev_get_machine());
+    sPAPRMachineState *spapr = SPAPR_MACHINE(machine);
+    uint64_t max_hotplug_addr = spapr->hotplug_memory.base +
+                                memory_region_size(&spapr->hotplug_memory.mr);
 
     ret = fdt_add_mem_rsv(fdt, rtas_addr, rtas_size);
     if (ret < 0) {
@@ -393,6 +761,19 @@
         }
 
     }
+
+    lrdr_capacity[0] = cpu_to_be32(max_hotplug_addr >> 32);
+    lrdr_capacity[1] = cpu_to_be32(max_hotplug_addr & 0xffffffff);
+    lrdr_capacity[2] = 0;
+    lrdr_capacity[3] = cpu_to_be32(SPAPR_MEMORY_BLOCK_SIZE);
+    lrdr_capacity[4] = cpu_to_be32(max_cpus/smp_threads);
+    ret = qemu_fdt_setprop(fdt, "/rtas", "ibm,lrdr-capacity", lrdr_capacity,
+                     sizeof(lrdr_capacity));
+    if (ret < 0) {
+        fprintf(stderr, "Couldn't add ibm,lrdr-capacity rtas property\n");
+        return ret;
+    }
+
     return 0;
 }
 
@@ -400,10 +781,6 @@
 {
     spapr_rtas_register(RTAS_DISPLAY_CHARACTER, "display-character",
                         rtas_display_character);
-    spapr_rtas_register(RTAS_GET_TIME_OF_DAY, "get-time-of-day",
-                        rtas_get_time_of_day);
-    spapr_rtas_register(RTAS_SET_TIME_OF_DAY, "set-time-of-day",
-                        rtas_set_time_of_day);
     spapr_rtas_register(RTAS_POWER_OFF, "power-off", rtas_power_off);
     spapr_rtas_register(RTAS_SYSTEM_REBOOT, "system-reboot",
                         rtas_system_reboot);
@@ -419,6 +796,16 @@
                         rtas_ibm_set_system_parameter);
     spapr_rtas_register(RTAS_IBM_OS_TERM, "ibm,os-term",
                         rtas_ibm_os_term);
+    spapr_rtas_register(RTAS_SET_POWER_LEVEL, "set-power-level",
+                        rtas_set_power_level);
+    spapr_rtas_register(RTAS_GET_POWER_LEVEL, "get-power-level",
+                        rtas_get_power_level);
+    spapr_rtas_register(RTAS_SET_INDICATOR, "set-indicator",
+                        rtas_set_indicator);
+    spapr_rtas_register(RTAS_GET_SENSOR_STATE, "get-sensor-state",
+                        rtas_get_sensor_state);
+    spapr_rtas_register(RTAS_IBM_CONFIGURE_CONNECTOR, "ibm,configure-connector",
+                        rtas_ibm_configure_connector);
 }
 
 type_init(core_rtas_register_types)
diff --git a/hw/ppc/spapr_rtas_ddw.c b/hw/ppc/spapr_rtas_ddw.c
new file mode 100644
index 0000000..177dcff
--- /dev/null
+++ b/hw/ppc/spapr_rtas_ddw.c
@@ -0,0 +1,295 @@
+/*
+ * QEMU sPAPR Dynamic DMA windows support
+ *
+ * Copyright (c) 2015 Alexey Kardashevskiy, IBM Corporation.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/error-report.h"
+#include "hw/ppc/spapr.h"
+#include "hw/pci-host/spapr.h"
+#include "trace.h"
+
+static int spapr_phb_get_active_win_num_cb(Object *child, void *opaque)
+{
+    sPAPRTCETable *tcet;
+
+    tcet = (sPAPRTCETable *) object_dynamic_cast(child, TYPE_SPAPR_TCE_TABLE);
+    if (tcet && tcet->nb_table) {
+        ++*(unsigned *)opaque;
+    }
+    return 0;
+}
+
+static unsigned spapr_phb_get_active_win_num(sPAPRPHBState *sphb)
+{
+    unsigned ret = 0;
+
+    object_child_foreach(OBJECT(sphb), spapr_phb_get_active_win_num_cb, &ret);
+
+    return ret;
+}
+
+static int spapr_phb_get_free_liobn_cb(Object *child, void *opaque)
+{
+    sPAPRTCETable *tcet;
+
+    tcet = (sPAPRTCETable *) object_dynamic_cast(child, TYPE_SPAPR_TCE_TABLE);
+    if (tcet && !tcet->nb_table) {
+        *(uint32_t *)opaque = tcet->liobn;
+        return 1;
+    }
+    return 0;
+}
+
+static unsigned spapr_phb_get_free_liobn(sPAPRPHBState *sphb)
+{
+    uint32_t liobn = 0;
+
+    object_child_foreach(OBJECT(sphb), spapr_phb_get_free_liobn_cb, &liobn);
+
+    return liobn;
+}
+
+static uint32_t spapr_page_mask_to_query_mask(uint64_t page_mask)
+{
+    int i;
+    uint32_t mask = 0;
+    const struct { int shift; uint32_t mask; } masks[] = {
+        { 12, RTAS_DDW_PGSIZE_4K },
+        { 16, RTAS_DDW_PGSIZE_64K },
+        { 24, RTAS_DDW_PGSIZE_16M },
+        { 25, RTAS_DDW_PGSIZE_32M },
+        { 26, RTAS_DDW_PGSIZE_64M },
+        { 27, RTAS_DDW_PGSIZE_128M },
+        { 28, RTAS_DDW_PGSIZE_256M },
+        { 34, RTAS_DDW_PGSIZE_16G },
+    };
+
+    for (i = 0; i < ARRAY_SIZE(masks); ++i) {
+        if (page_mask & (1ULL << masks[i].shift)) {
+            mask |= masks[i].mask;
+        }
+    }
+
+    return mask;
+}
+
+static void rtas_ibm_query_pe_dma_window(PowerPCCPU *cpu,
+                                         sPAPRMachineState *spapr,
+                                         uint32_t token, uint32_t nargs,
+                                         target_ulong args,
+                                         uint32_t nret, target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    uint64_t buid, max_window_size;
+    uint32_t avail, addr, pgmask = 0;
+    MachineState *machine = MACHINE(spapr);
+
+    if ((nargs != 3) || (nret != 5)) {
+        goto param_error_exit;
+    }
+
+    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    addr = rtas_ld(args, 0);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb || !sphb->ddw_enabled) {
+        goto param_error_exit;
+    }
+
+    /* Translate page mask to LoPAPR format */
+    pgmask = spapr_page_mask_to_query_mask(sphb->page_size_mask);
+
+    /*
+     * This is "Largest contiguous block of TCEs allocated specifically
+     * for (that is, are reserved for) this PE".
+     * Return the maximum number as maximum supported RAM size was in 4K pages.
+     */
+    if (machine->ram_size == machine->maxram_size) {
+        max_window_size = machine->ram_size;
+    } else {
+        MemoryHotplugState *hpms = &spapr->hotplug_memory;
+
+        max_window_size = hpms->base + memory_region_size(&hpms->mr);
+    }
+
+    avail = SPAPR_PCI_DMA_MAX_WINDOWS - spapr_phb_get_active_win_num(sphb);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, avail);
+    rtas_st(rets, 2, max_window_size >> SPAPR_TCE_PAGE_SHIFT);
+    rtas_st(rets, 3, pgmask);
+    rtas_st(rets, 4, 0); /* DMA migration mask, not supported */
+
+    trace_spapr_iommu_ddw_query(buid, addr, avail, max_window_size, pgmask);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_create_pe_dma_window(PowerPCCPU *cpu,
+                                          sPAPRMachineState *spapr,
+                                          uint32_t token, uint32_t nargs,
+                                          target_ulong args,
+                                          uint32_t nret, target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    sPAPRTCETable *tcet = NULL;
+    uint32_t addr, page_shift, window_shift, liobn;
+    uint64_t buid, win_addr;
+    int windows;
+
+    if ((nargs != 5) || (nret != 4)) {
+        goto param_error_exit;
+    }
+
+    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    addr = rtas_ld(args, 0);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb || !sphb->ddw_enabled) {
+        goto param_error_exit;
+    }
+
+    page_shift = rtas_ld(args, 3);
+    window_shift = rtas_ld(args, 4);
+    liobn = spapr_phb_get_free_liobn(sphb);
+    windows = spapr_phb_get_active_win_num(sphb);
+
+    if (!(sphb->page_size_mask & (1ULL << page_shift)) ||
+        (window_shift < page_shift)) {
+        goto param_error_exit;
+    }
+
+    if (!liobn || !sphb->ddw_enabled || windows == SPAPR_PCI_DMA_MAX_WINDOWS) {
+        goto hw_error_exit;
+    }
+
+    tcet = spapr_tce_find_by_liobn(liobn);
+    if (!tcet) {
+        goto hw_error_exit;
+    }
+
+    win_addr = (windows == 0) ? sphb->dma_win_addr : sphb->dma64_win_addr;
+    spapr_tce_table_enable(tcet, page_shift, win_addr,
+                           1ULL << (window_shift - page_shift));
+    if (!tcet->nb_table) {
+        goto hw_error_exit;
+    }
+
+    trace_spapr_iommu_ddw_create(buid, addr, 1ULL << page_shift,
+                                 1ULL << window_shift, tcet->bus_offset, liobn);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, liobn);
+    rtas_st(rets, 2, tcet->bus_offset >> 32);
+    rtas_st(rets, 3, tcet->bus_offset & ((uint32_t) -1));
+
+    return;
+
+hw_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_remove_pe_dma_window(PowerPCCPU *cpu,
+                                          sPAPRMachineState *spapr,
+                                          uint32_t token, uint32_t nargs,
+                                          target_ulong args,
+                                          uint32_t nret, target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    sPAPRTCETable *tcet;
+    uint32_t liobn;
+
+    if ((nargs != 1) || (nret != 1)) {
+        goto param_error_exit;
+    }
+
+    liobn = rtas_ld(args, 0);
+    tcet = spapr_tce_find_by_liobn(liobn);
+    if (!tcet) {
+        goto param_error_exit;
+    }
+
+    sphb = SPAPR_PCI_HOST_BRIDGE(OBJECT(tcet)->parent);
+    if (!sphb || !sphb->ddw_enabled || !tcet->nb_table) {
+        goto param_error_exit;
+    }
+
+    spapr_tce_table_disable(tcet);
+    trace_spapr_iommu_ddw_remove(liobn);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void rtas_ibm_reset_pe_dma_window(PowerPCCPU *cpu,
+                                         sPAPRMachineState *spapr,
+                                         uint32_t token, uint32_t nargs,
+                                         target_ulong args,
+                                         uint32_t nret, target_ulong rets)
+{
+    sPAPRPHBState *sphb;
+    uint64_t buid;
+    uint32_t addr;
+
+    if ((nargs != 3) || (nret != 1)) {
+        goto param_error_exit;
+    }
+
+    buid = ((uint64_t)rtas_ld(args, 1) << 32) | rtas_ld(args, 2);
+    addr = rtas_ld(args, 0);
+    sphb = spapr_pci_find_phb(spapr, buid);
+    if (!sphb || !sphb->ddw_enabled) {
+        goto param_error_exit;
+    }
+
+    spapr_phb_dma_reset(sphb);
+    trace_spapr_iommu_ddw_reset(buid, addr);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+
+    return;
+
+param_error_exit:
+    rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+}
+
+static void spapr_rtas_ddw_init(void)
+{
+    spapr_rtas_register(RTAS_IBM_QUERY_PE_DMA_WINDOW,
+                        "ibm,query-pe-dma-window",
+                        rtas_ibm_query_pe_dma_window);
+    spapr_rtas_register(RTAS_IBM_CREATE_PE_DMA_WINDOW,
+                        "ibm,create-pe-dma-window",
+                        rtas_ibm_create_pe_dma_window);
+    spapr_rtas_register(RTAS_IBM_REMOVE_PE_DMA_WINDOW,
+                        "ibm,remove-pe-dma-window",
+                        rtas_ibm_remove_pe_dma_window);
+    spapr_rtas_register(RTAS_IBM_RESET_PE_DMA_WINDOW,
+                        "ibm,reset-pe-dma-window",
+                        rtas_ibm_reset_pe_dma_window);
+}
+
+type_init(spapr_rtas_ddw_init)
diff --git a/hw/ppc/spapr_rtc.c b/hw/ppc/spapr_rtc.c
new file mode 100644
index 0000000..3a17ac4
--- /dev/null
+++ b/hw/ppc/spapr_rtc.c
@@ -0,0 +1,212 @@
+/*
+ * QEMU PowerPC pSeries Logical Partition (aka sPAPR) hardware System Emulator
+ *
+ * RTAS Real Time Clock
+ *
+ * Copyright (c) 2010-2011 David Gibson, IBM Corporation.
+ * Copyright 2014 David Gibson, Red Hat.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+#include "hw/ppc/spapr.h"
+#include "qapi-event.h"
+#include "qemu/cutils.h"
+
+#define SPAPR_RTC(obj) \
+    OBJECT_CHECK(sPAPRRTCState, (obj), TYPE_SPAPR_RTC)
+
+typedef struct sPAPRRTCState sPAPRRTCState;
+struct sPAPRRTCState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    int64_t ns_offset;
+};
+
+void spapr_rtc_read(DeviceState *dev, struct tm *tm, uint32_t *ns)
+{
+    sPAPRRTCState *rtc = SPAPR_RTC(dev);
+    int64_t host_ns = qemu_clock_get_ns(rtc_clock);
+    int64_t guest_ns;
+    time_t guest_s;
+
+    assert(rtc);
+
+    guest_ns = host_ns + rtc->ns_offset;
+    guest_s = guest_ns / NANOSECONDS_PER_SECOND;
+
+    if (tm) {
+        gmtime_r(&guest_s, tm);
+    }
+    if (ns) {
+        *ns = guest_ns;
+    }
+}
+
+int spapr_rtc_import_offset(DeviceState *dev, int64_t legacy_offset)
+{
+    sPAPRRTCState *rtc;
+
+    if (!dev) {
+        return -ENODEV;
+    }
+
+    rtc = SPAPR_RTC(dev);
+
+    rtc->ns_offset = legacy_offset * NANOSECONDS_PER_SECOND;
+
+    return 0;
+}
+
+static void rtas_get_time_of_day(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                 uint32_t token, uint32_t nargs,
+                                 target_ulong args,
+                                 uint32_t nret, target_ulong rets)
+{
+    struct tm tm;
+    uint32_t ns;
+
+    if ((nargs != 0) || (nret != 8)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    if (!spapr->rtc) {
+        rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
+        return;
+    }
+
+    spapr_rtc_read(spapr->rtc, &tm, &ns);
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+    rtas_st(rets, 1, tm.tm_year + 1900);
+    rtas_st(rets, 2, tm.tm_mon + 1);
+    rtas_st(rets, 3, tm.tm_mday);
+    rtas_st(rets, 4, tm.tm_hour);
+    rtas_st(rets, 5, tm.tm_min);
+    rtas_st(rets, 6, tm.tm_sec);
+    rtas_st(rets, 7, ns);
+}
+
+static void rtas_set_time_of_day(PowerPCCPU *cpu, sPAPRMachineState *spapr,
+                                 uint32_t token, uint32_t nargs,
+                                 target_ulong args,
+                                 uint32_t nret, target_ulong rets)
+{
+    sPAPRRTCState *rtc;
+    struct tm tm;
+    time_t new_s;
+    int64_t host_ns;
+
+    if ((nargs != 7) || (nret != 1)) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    if (!spapr->rtc) {
+        rtas_st(rets, 0, RTAS_OUT_HW_ERROR);
+        return;
+    }
+
+    tm.tm_year = rtas_ld(args, 0) - 1900;
+    tm.tm_mon = rtas_ld(args, 1) - 1;
+    tm.tm_mday = rtas_ld(args, 2);
+    tm.tm_hour = rtas_ld(args, 3);
+    tm.tm_min = rtas_ld(args, 4);
+    tm.tm_sec = rtas_ld(args, 5);
+
+    new_s = mktimegm(&tm);
+    if (new_s == -1) {
+        rtas_st(rets, 0, RTAS_OUT_PARAM_ERROR);
+        return;
+    }
+
+    /* Generate a monitor event for the change */
+    qapi_event_send_rtc_change(qemu_timedate_diff(&tm), &error_abort);
+
+    rtc = SPAPR_RTC(spapr->rtc);
+
+    host_ns = qemu_clock_get_ns(rtc_clock);
+
+    rtc->ns_offset = (new_s * NANOSECONDS_PER_SECOND) - host_ns;
+
+    rtas_st(rets, 0, RTAS_OUT_SUCCESS);
+}
+
+static void spapr_rtc_qom_date(Object *obj, struct tm *current_tm, Error **errp)
+{
+    spapr_rtc_read(DEVICE(obj), current_tm, NULL);
+}
+
+static void spapr_rtc_realize(DeviceState *dev, Error **errp)
+{
+    sPAPRRTCState *rtc = SPAPR_RTC(dev);
+    struct tm tm;
+    time_t host_s;
+    int64_t rtc_ns;
+
+    /* Initialize the RTAS RTC from host time */
+
+    qemu_get_timedate(&tm, 0);
+    host_s = mktimegm(&tm);
+    rtc_ns = qemu_clock_get_ns(rtc_clock);
+    rtc->ns_offset = host_s * NANOSECONDS_PER_SECOND - rtc_ns;
+
+    object_property_add_tm(OBJECT(rtc), "date", spapr_rtc_qom_date, NULL);
+}
+
+static const VMStateDescription vmstate_spapr_rtc = {
+    .name = "spapr/rtc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT64(ns_offset, sPAPRRTCState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static void spapr_rtc_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->realize = spapr_rtc_realize;
+    dc->vmsd = &vmstate_spapr_rtc;
+
+    spapr_rtas_register(RTAS_GET_TIME_OF_DAY, "get-time-of-day",
+                        rtas_get_time_of_day);
+    spapr_rtas_register(RTAS_SET_TIME_OF_DAY, "set-time-of-day",
+                        rtas_set_time_of_day);
+}
+
+static const TypeInfo spapr_rtc_info = {
+    .name          = TYPE_SPAPR_RTC,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(sPAPRRTCState),
+    .class_init    = spapr_rtc_class_init,
+};
+
+static void spapr_rtc_register_types(void)
+{
+    type_register_static(&spapr_rtc_info);
+}
+type_init(spapr_rtc_register_types)
diff --git a/hw/ppc/spapr_vio.c b/hw/ppc/spapr_vio.c
index dc9e46a..f93244d 100644
--- a/hw/ppc/spapr_vio.c
+++ b/hw/ppc/spapr_vio.c
@@ -19,10 +19,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
+#include "qemu/log.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
-#include "monitor/monitor.h"
 #include "hw/loader.h"
 #include "elf.h"
 #include "hw/sysbus.h"
@@ -55,12 +57,9 @@
 {
     VIOsPAPRDevice *dev = VIO_SPAPR_DEVICE(qdev);
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-    char *name;
 
     /* Device tree style name device@reg */
-    name = g_strdup_printf("%s@%x", pc->dt_name, dev->reg);
-
-    return name;
+    return g_strdup_printf("%s@%x", pc->dt_name, dev->reg);
 }
 
 static void spapr_vio_bus_class_init(ObjectClass *klass, void *data)
@@ -161,7 +160,7 @@
 /*
  * CRQ handling
  */
-static target_ulong h_reg_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_reg_crq(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -219,7 +218,7 @@
     return H_SUCCESS;
 }
 
-static target_ulong h_free_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_free_crq(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -233,7 +232,7 @@
     return free_crq(dev);
 }
 
-static target_ulong h_send_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_send_crq(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -256,7 +255,7 @@
     return H_HARDWARE;
 }
 
-static target_ulong h_enable_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_enable_crq(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                  target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -322,7 +321,19 @@
     free_crq(dev);
 }
 
-static void rtas_set_tce_bypass(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+void spapr_vio_set_bypass(VIOsPAPRDevice *dev, bool bypass)
+{
+    if (!dev->tcet) {
+        return;
+    }
+
+    memory_region_set_enabled(&dev->mrbypass, bypass);
+    memory_region_set_enabled(spapr_tce_get_iommu(dev->tcet), !bypass);
+
+    dev->tcet->bypass = bypass;
+}
+
+static void rtas_set_tce_bypass(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                 uint32_t token,
                                 uint32_t nargs, target_ulong args,
                                 uint32_t nret, target_ulong rets)
@@ -348,12 +359,12 @@
         return;
     }
 
-    spapr_tce_set_bypass(dev->tcet, !!enable);
+    spapr_vio_set_bypass(dev, !!enable);
 
     rtas_st(rets, 0, RTAS_OUT_SUCCESS);
 }
 
-static void rtas_quiesce(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static void rtas_quiesce(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                          uint32_t token,
                          uint32_t nargs, target_ulong args,
                          uint32_t nret, target_ulong rets)
@@ -377,7 +388,7 @@
 
 static VIOsPAPRDevice *reg_conflict(VIOsPAPRDevice *dev)
 {
-    VIOsPAPRBus *bus = DO_UPCAST(VIOsPAPRBus, bus, dev->qdev.parent_bus);
+    VIOsPAPRBus *bus = SPAPR_VIO_BUS(dev->qdev.parent_bus);
     BusChild *kid;
     VIOsPAPRDevice *other;
 
@@ -407,16 +418,19 @@
 
     dev->signal_state = 0;
 
+    spapr_vio_set_bypass(dev, false);
     if (pc->reset) {
         pc->reset(dev);
     }
 }
 
-static int spapr_vio_busdev_init(DeviceState *qdev)
+static void spapr_vio_busdev_realize(DeviceState *qdev, Error **errp)
 {
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
     VIOsPAPRDevice *dev = (VIOsPAPRDevice *)qdev;
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
     char *id;
+    Error *local_err = NULL;
 
     if (dev->reg != -1) {
         /*
@@ -428,15 +442,15 @@
         VIOsPAPRDevice *other = reg_conflict(dev);
 
         if (other) {
-            fprintf(stderr, "vio: %s and %s devices conflict at address %#x\n",
-                    object_get_typename(OBJECT(qdev)),
-                    object_get_typename(OBJECT(&other->qdev)),
-                    dev->reg);
-            return -1;
+            error_setg(errp, "%s and %s devices conflict at address %#x",
+                       object_get_typename(OBJECT(qdev)),
+                       object_get_typename(OBJECT(&other->qdev)),
+                       dev->reg);
+            return;
         }
     } else {
         /* Need to assign an address */
-        VIOsPAPRBus *bus = DO_UPCAST(VIOsPAPRBus, bus, dev->qdev.parent_bus);
+        VIOsPAPRBus *bus = SPAPR_VIO_BUS(dev->qdev.parent_bus);
 
         do {
             dev->reg = bus->next_reg++;
@@ -449,25 +463,35 @@
         dev->qdev.id = id;
     }
 
-    dev->irq = xics_alloc(spapr->icp, 0, dev->irq, false);
-    if (!dev->irq) {
-        return -1;
+    dev->irq = xics_spapr_alloc(spapr->xics, 0, dev->irq, false, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
     }
 
     if (pc->rtce_window_size) {
-        uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
-        dev->tcet = spapr_tce_new_table(qdev, liobn,
-                                        0,
-                                        SPAPR_TCE_PAGE_SHIFT,
-                                        pc->rtce_window_size >>
-                                        SPAPR_TCE_PAGE_SHIFT, false);
-        address_space_init(&dev->as, spapr_tce_get_iommu(dev->tcet), qdev->id);
+        uint32_t liobn = SPAPR_VIO_LIOBN(dev->reg);
+
+        memory_region_init(&dev->mrroot, OBJECT(dev), "iommu-spapr-root",
+                           ram_size);
+        memory_region_init_alias(&dev->mrbypass, OBJECT(dev),
+                                 "iommu-spapr-bypass", get_system_memory(),
+                                 0, ram_size);
+        memory_region_add_subregion_overlap(&dev->mrroot, 0, &dev->mrbypass, 1);
+        address_space_init(&dev->as, &dev->mrroot, qdev->id);
+
+        dev->tcet = spapr_tce_new_table(qdev, liobn);
+        spapr_tce_table_enable(dev->tcet, SPAPR_TCE_PAGE_SHIFT, 0,
+                               pc->rtce_window_size >> SPAPR_TCE_PAGE_SHIFT);
+        dev->tcet->vdev = dev;
+        memory_region_add_subregion_overlap(&dev->mrroot, 0,
+                                            spapr_tce_get_iommu(dev->tcet), 2);
     }
 
-    return pc->init(dev);
+    pc->realize(dev, errp);
 }
 
-static target_ulong h_vio_signal(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+static target_ulong h_vio_signal(PowerPCCPU *cpu, sPAPRMachineState *spapr,
                                  target_ulong opcode,
                                  target_ulong *args)
 {
@@ -498,13 +522,12 @@
     DeviceState *dev;
 
     /* Create bridge device */
-    dev = qdev_create(NULL, "spapr-vio-bridge");
+    dev = qdev_create(NULL, TYPE_SPAPR_VIO_BRIDGE);
     qdev_init_nofail(dev);
 
     /* Create bus on bridge device */
-
     qbus = qbus_create(TYPE_SPAPR_VIO_BUS, dev, "spapr-vio");
-    bus = DO_UPCAST(VIOsPAPRBus, bus, qbus);
+    bus = SPAPR_VIO_BUS(qbus);
     bus->next_reg = 0x71000000;
 
     /* hcall-vio */
@@ -542,9 +565,8 @@
 }
 
 static const TypeInfo spapr_vio_bridge_info = {
-    .name          = "spapr-vio-bridge",
+    .name          = TYPE_SPAPR_VIO_BRIDGE,
     .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(SysBusDevice),
     .class_init    = spapr_vio_bridge_class_init,
 };
 
@@ -558,7 +580,7 @@
         VMSTATE_UINT32_EQUAL(irq, VIOsPAPRDevice),
 
         /* General VIO device state */
-        VMSTATE_UINTTL(signal_state, VIOsPAPRDevice),
+        VMSTATE_UINT64(signal_state, VIOsPAPRDevice),
         VMSTATE_UINT64(crq.qladdr, VIOsPAPRDevice),
         VMSTATE_UINT32(crq.qsize, VIOsPAPRDevice),
         VMSTATE_UINT32(crq.qnext, VIOsPAPRDevice),
@@ -570,7 +592,7 @@
 static void vio_spapr_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *k = DEVICE_CLASS(klass);
-    k->init = spapr_vio_busdev_init;
+    k->realize = spapr_vio_busdev_realize;
     k->reset = spapr_vio_busdev_reset;
     k->bus_type = TYPE_SPAPR_VIO_BUS;
     k->props = spapr_vio_props;
@@ -648,7 +670,7 @@
 
     ret = 0;
 out:
-    free(qdevs);
+    g_free(qdevs);
 
     return ret;
 }
diff --git a/hw/ppc/trace-events b/hw/ppc/trace-events
new file mode 100644
index 0000000..dfeab93
--- /dev/null
+++ b/hw/ppc/trace-events
@@ -0,0 +1,43 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/ppc/spapr_pci.c
+spapr_pci_msi(const char *msg, uint32_t ca) "%s (cfg=%x)"
+spapr_pci_msi_setup(const char *name, unsigned vector, uint64_t addr) "dev\"%s\" vector %u, addr=%"PRIx64
+spapr_pci_rtas_ibm_change_msi(unsigned cfg, unsigned func, unsigned req, unsigned first) "cfgaddr %x func %u, requested %u, first irq %u"
+spapr_pci_rtas_ibm_query_interrupt_source_number(unsigned ioa, unsigned intr) "queries for #%u, IRQ%u"
+spapr_pci_msi_write(uint64_t addr, uint64_t data, uint32_t dt_irq) "@%"PRIx64"<=%"PRIx64" IRQ %u"
+spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
+spapr_pci_msi_retry(unsigned config_addr, unsigned req_num, unsigned max_irqs) "Guest device at %x asked %u, have only %u"
+
+# hw/ppc/spapr.c
+spapr_cas_failed(unsigned long n) "DT diff buffer is too small: %ld bytes"
+spapr_cas_continue(unsigned long n) "Copy changes to the guest: %ld bytes"
+
+# hw/ppc/spapr_hcall.c
+spapr_cas_pvr_try(uint32_t pvr) "%x"
+spapr_cas_pvr(uint32_t cur_pvr, bool cpu_match, uint32_t new_pvr, uint64_t pcr) "current=%x, cpu_match=%u, new=%x, compat flags=%"PRIx64
+
+# hw/ppc/spapr_iommu.c
+spapr_iommu_put(uint64_t liobn, uint64_t ioba, uint64_t tce, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tce=0x%"PRIx64" ret=%"PRId64
+spapr_iommu_get(uint64_t liobn, uint64_t ioba, uint64_t ret, uint64_t tce) "liobn=%"PRIx64" ioba=0x%"PRIx64" ret=%"PRId64" tce=0x%"PRIx64
+spapr_iommu_indirect(uint64_t liobn, uint64_t ioba, uint64_t tce, uint64_t iobaN, uint64_t tceN, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tcelist=0x%"PRIx64" iobaN=0x%"PRIx64" tceN=0x%"PRIx64" ret=%"PRId64
+spapr_iommu_stuff(uint64_t liobn, uint64_t ioba, uint64_t tce_value, uint64_t npages, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tcevalue=0x%"PRIx64" npages=%"PRId64" ret=%"PRId64
+spapr_iommu_pci_put(uint64_t liobn, uint64_t ioba, uint64_t tce, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tce=0x%"PRIx64" ret=%"PRId64
+spapr_iommu_pci_get(uint64_t liobn, uint64_t ioba, uint64_t ret, uint64_t tce) "liobn=%"PRIx64" ioba=0x%"PRIx64" ret=%"PRId64" tce=0x%"PRIx64
+spapr_iommu_pci_indirect(uint64_t liobn, uint64_t ioba, uint64_t tce, uint64_t iobaN, uint64_t tceN, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tcelist=0x%"PRIx64" iobaN=0x%"PRIx64" tceN=0x%"PRIx64" ret=%"PRId64
+spapr_iommu_pci_stuff(uint64_t liobn, uint64_t ioba, uint64_t tce_value, uint64_t npages, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tcevalue=0x%"PRIx64" npages=%"PRId64" ret=%"PRId64
+spapr_iommu_xlate(uint64_t liobn, uint64_t ioba, uint64_t tce, unsigned perm, unsigned pgsize) "liobn=%"PRIx64" 0x%"PRIx64" -> 0x%"PRIx64" perm=%u mask=%x"
+spapr_iommu_new_table(uint64_t liobn, void *table, int fd) "liobn=%"PRIx64" table=%p fd=%d"
+spapr_iommu_pre_save(uint64_t liobn, uint32_t nb, uint64_t offs, uint32_t ps) "liobn=%"PRIx64" %"PRIx32" bus_offset=%"PRIx64" ps=%"PRIu32
+spapr_iommu_post_load(uint64_t liobn, uint32_t pre_nb, uint32_t post_nb, uint64_t offs, uint32_t ps) "liobn=%"PRIx64" %"PRIx32" => %"PRIx32" bus_offset=%"PRIx64" ps=%"PRIu32
+spapr_iommu_ddw_query(uint64_t buid, uint32_t cfgaddr, unsigned wa, uint64_t win_size, uint32_t pgmask) "buid=%"PRIx64" addr=%"PRIx32", %u windows available, max window size=%"PRIx64", mask=%"PRIx32
+spapr_iommu_ddw_create(uint64_t buid, uint32_t cfgaddr, uint64_t pg_size, uint64_t req_size, uint64_t start, uint32_t liobn) "buid=%"PRIx64" addr=%"PRIx32", page size=0x%"PRIx64", requested=0x%"PRIx64", start addr=%"PRIx64", liobn=%"PRIx32
+spapr_iommu_ddw_remove(uint32_t liobn) "liobn=%"PRIx32
+spapr_iommu_ddw_reset(uint64_t buid, uint32_t cfgaddr) "buid=%"PRIx64" addr=%"PRIx32
+
+# hw/ppc/ppc.c
+ppc_tb_adjust(uint64_t offs1, uint64_t offs2, int64_t diff, int64_t seconds) "adjusted from 0x%"PRIx64" to 0x%"PRIx64", diff %"PRId64" (%"PRId64"s)"
+
+# hw/ppc/prep.c
+prep_io_800_writeb(uint32_t addr, uint32_t val) "0x%08" PRIx32 " => 0x%02" PRIx32
+prep_io_800_readb(uint32_t addr, uint32_t retval) "0x%08" PRIx32 " <= 0x%02" PRIx32
diff --git a/hw/ppc/virtex_ml507.c b/hw/ppc/virtex_ml507.c
index 6ebd5be..b97d966 100644
--- a/hw/ppc/virtex_ml507.c
+++ b/hw/ppc/virtex_ml507.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/char/serial.h"
@@ -32,6 +34,7 @@
 #include "sysemu/device_tree.h"
 #include "hw/loader.h"
 #include "elf.h"
+#include "qemu/error-report.h"
 #include "qemu/log.h"
 #include "exec/address-spaces.h"
 
@@ -40,7 +43,6 @@
 #include "ppc405.h"
 
 #include "sysemu/block-backend.h"
-#include "qapi/qmp/qerror.h"
 
 #define EPAPR_MAGIC    (0x45504150)
 #define FLASH_SIZE     (16 * 1024 * 1024)
@@ -197,7 +199,6 @@
 static void virtex_init(MachineState *machine)
 {
     ram_addr_t ram_size = machine->ram_size;
-    const char *cpu_model = machine->cpu_model;
     const char *kernel_filename = machine->kernel_filename;
     const char *kernel_cmdline = machine->kernel_cmdline;
     hwaddr initrd_base = 0;
@@ -214,11 +215,11 @@
     int i;
 
     /* init CPUs */
-    if (cpu_model == NULL) {
-        cpu_model = "440-Xilinx";
+    if (machine->cpu_model == NULL) {
+        machine->cpu_model = "440-Xilinx";
     }
 
-    cpu = ppc440_init_xilinx(&ram_size, 1, cpu_model, 400000000);
+    cpu = ppc440_init_xilinx(&ram_size, 1, machine->cpu_model, 400000000);
     env = &cpu->env;
     qemu_register_reset(main_cpu_reset, cpu);
 
@@ -258,7 +259,8 @@
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
-                               &entry, &low, &high, 1, ELF_MACHINE, 0);
+                               &entry, &low, &high, 1, PPC_ELF_MACHINE,
+                               0, 0);
         boot_info.bootstrap_pc = entry & 0x00ffffff;
 
         if (kernel_size < 0) {
@@ -298,15 +300,10 @@
     env->load_info = &boot_info;
 }
 
-static QEMUMachine virtex_machine = {
-    .name = "virtex-ml507",
-    .desc = "Xilinx Virtex ML507 reference design",
-    .init = virtex_init,
-};
-
-static void virtex_machine_init(void)
+static void virtex_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&virtex_machine);
+    mc->desc = "Xilinx Virtex ML507 reference design";
+    mc->init = virtex_init;
 }
 
-machine_init(virtex_machine_init);
+DEFINE_MACHINE("virtex-ml507", virtex_machine_init)
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index 1ba6c3a..41ac4ec 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -1,4 +1,4 @@
-obj-y = s390-virtio-bus.o s390-virtio.o
+obj-y += s390-virtio.o
 obj-y += s390-virtio-hcall.o
 obj-y += sclp.o
 obj-y += event-facility.o
@@ -8,3 +8,8 @@
 obj-y += css.o
 obj-y += s390-virtio-ccw.o
 obj-y += virtio-ccw.o
+obj-y += css-bridge.o
+obj-y += ccw-device.o
+obj-y += s390-pci-bus.o s390-pci-inst.o
+obj-y += s390-skeys.o
+obj-$(CONFIG_KVM) += s390-skeys-kvm.o
diff --git a/hw/s390x/ccw-device.c b/hw/s390x/ccw-device.c
new file mode 100644
index 0000000..28ea204
--- /dev/null
+++ b/hw/s390x/ccw-device.c
@@ -0,0 +1,27 @@
+/*
+ * Common device infrastructure for devices in the virtual css
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Jing Liu <liujbjl@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "ccw-device.h"
+
+static const TypeInfo ccw_device_info = {
+    .name = TYPE_CCW_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(CcwDevice),
+    .class_size = sizeof(CCWDeviceClass),
+    .abstract = true,
+};
+
+static void ccw_device_register(void)
+{
+    type_register_static(&ccw_device_info);
+}
+
+type_init(ccw_device_register)
diff --git a/hw/s390x/ccw-device.h b/hw/s390x/ccw-device.h
new file mode 100644
index 0000000..59ba01b
--- /dev/null
+++ b/hw/s390x/ccw-device.h
@@ -0,0 +1,43 @@
+/*
+ * Common device infrastructure for devices in the virtual css
+ *
+ * Copyright 2016 IBM Corp.
+ * Author(s): Jing Liu <liujbjl@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390X_CCW_DEVICE_H
+#define HW_S390X_CCW_DEVICE_H
+#include "qom/object.h"
+#include "hw/qdev-core.h"
+#include "hw/s390x/css.h"
+
+typedef struct CcwDevice {
+    DeviceState parent_obj;
+    SubchDev *sch;
+    /* <cssid>.<ssid>.<device number> */
+    CssDevId bus_id;
+} CcwDevice;
+
+typedef struct CCWDeviceClass {
+    DeviceClass parent_class;
+    void (*unplug)(HotplugHandler *, DeviceState *, Error **);
+} CCWDeviceClass;
+
+static inline CcwDevice *to_ccw_dev_fast(DeviceState *d)
+{
+    return container_of(d, CcwDevice, parent_obj);
+}
+
+#define TYPE_CCW_DEVICE "ccw-device"
+
+#define CCW_DEVICE(obj) OBJECT_CHECK(CcwDevice, (obj), TYPE_CCW_DEVICE)
+#define CCW_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(CCWDeviceClass, (obj), TYPE_CCW_DEVICE)
+#define CCW_DEVICE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(CCWDeviceClass, (klass), TYPE_CCW_DEVICE)
+
+#endif
diff --git a/hw/s390x/css-bridge.c b/hw/s390x/css-bridge.c
new file mode 100644
index 0000000..9a7f7ee
--- /dev/null
+++ b/hw/s390x/css-bridge.c
@@ -0,0 +1,148 @@
+/*
+ * css bridge implementation
+ *
+ * Copyright 2012,2016 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *            Pierre Morel <pmorel@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "hw/hotplug.h"
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+#include "hw/s390x/css.h"
+#include "ccw-device.h"
+#include "hw/s390x/css-bridge.h"
+
+/*
+ * Invoke device-specific unplug handler, disable the subchannel
+ * (including sending a channel report to the guest) and remove the
+ * device from the virtual css bus.
+ */
+static void ccw_device_unplug(HotplugHandler *hotplug_dev,
+                              DeviceState *dev, Error **errp)
+{
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
+    CCWDeviceClass *k = CCW_DEVICE_GET_CLASS(ccw_dev);
+    SubchDev *sch = ccw_dev->sch;
+    Error *err = NULL;
+
+    if (k->unplug) {
+        k->unplug(hotplug_dev, dev, &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
+        }
+    }
+
+    /*
+     * We should arrive here only for device_del, since we don't support
+     * direct hot(un)plug of channels.
+     */
+    assert(sch != NULL);
+    /* Subchannel is now disabled and no longer valid. */
+    sch->curr_status.pmcw.flags &= ~(PMCW_FLAGS_MASK_ENA |
+                                     PMCW_FLAGS_MASK_DNV);
+
+    css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid, 1, 0);
+
+    object_unparent(OBJECT(dev));
+}
+
+static void virtual_css_bus_reset(BusState *qbus)
+{
+    /* This should actually be modelled via the generic css */
+    css_reset();
+}
+
+static char *virtual_css_bus_get_dev_path(DeviceState *dev)
+{
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
+    SubchDev *sch = ccw_dev->sch;
+    VirtualCssBridge *bridge =
+        VIRTUAL_CSS_BRIDGE(qdev_get_parent_bus(dev)->parent);
+
+    /*
+     * We can't provide a dev path for backward compatibility on
+     * older machines, as it is visible in the migration stream.
+     */
+    return bridge->css_dev_path ?
+        g_strdup_printf("/%02x.%1x.%04x", sch->cssid, sch->ssid, sch->devno) :
+        NULL;
+}
+
+static void virtual_css_bus_class_init(ObjectClass *klass, void *data)
+{
+    BusClass *k = BUS_CLASS(klass);
+
+    k->reset = virtual_css_bus_reset;
+    k->get_dev_path = virtual_css_bus_get_dev_path;
+}
+
+static const TypeInfo virtual_css_bus_info = {
+    .name = TYPE_VIRTUAL_CSS_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(VirtualCssBus),
+    .class_init = virtual_css_bus_class_init,
+};
+
+VirtualCssBus *virtual_css_bus_init(void)
+{
+    VirtualCssBus *cbus;
+    BusState *bus;
+    DeviceState *dev;
+
+    /* Create bridge device */
+    dev = qdev_create(NULL, TYPE_VIRTUAL_CSS_BRIDGE);
+    qdev_init_nofail(dev);
+
+    /* Create bus on bridge device */
+    bus = qbus_create(TYPE_VIRTUAL_CSS_BUS, dev, "virtual-css");
+    cbus = VIRTUAL_CSS_BUS(bus);
+
+    /* Enable hotplugging */
+    qbus_set_hotplug_handler(bus, dev, &error_abort);
+
+    return cbus;
+ }
+
+/***************** Virtual-css Bus Bridge Device ********************/
+
+static Property virtual_css_bridge_properties[] = {
+    DEFINE_PROP_BOOL("css_dev_path", VirtualCssBridge, css_dev_path,
+                     true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtual_css_bridge_class_init(ObjectClass *klass, void *data)
+{
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    hc->unplug = ccw_device_unplug;
+    set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
+    dc->props = virtual_css_bridge_properties;
+}
+
+static const TypeInfo virtual_css_bridge_info = {
+    .name          = TYPE_VIRTUAL_CSS_BRIDGE,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(VirtualCssBridge),
+    .class_init    = virtual_css_bridge_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
+};
+
+static void virtual_css_register(void)
+{
+    type_register_static(&virtual_css_bridge_info);
+    type_register_static(&virtual_css_bus_info);
+}
+
+type_init(virtual_css_register)
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index b67c039..bb8e4be 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -9,12 +9,15 @@
  * directory.
  */
 
-#include <hw/qdev.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+#include "hw/qdev.h"
 #include "qemu/bitops.h"
 #include "exec/address-spaces.h"
 #include "cpu.h"
-#include "ioinst.h"
-#include "css.h"
+#include "hw/s390x/ioinst.h"
+#include "hw/s390x/css.h"
 #include "trace.h"
 #include "hw/s390x/s390_flic.h"
 
@@ -49,6 +52,7 @@
 
 typedef struct ChannelSubSys {
     QTAILQ_HEAD(, CrwContainer) pending_crws;
+    bool sei_pending;
     bool do_crw_mchk;
     bool crws_lost;
     uint8_t max_cssid;
@@ -58,9 +62,81 @@
     CssImage *css[MAX_CSSID + 1];
     uint8_t default_cssid;
     QTAILQ_HEAD(, IoAdapter) io_adapters;
+    QTAILQ_HEAD(, IndAddr) indicator_addresses;
 } ChannelSubSys;
 
-static ChannelSubSys *channel_subsys;
+static ChannelSubSys channel_subsys = {
+    .pending_crws = QTAILQ_HEAD_INITIALIZER(channel_subsys.pending_crws),
+    .do_crw_mchk = true,
+    .sei_pending = false,
+    .do_crw_mchk = true,
+    .crws_lost = false,
+    .chnmon_active = false,
+    .io_adapters = QTAILQ_HEAD_INITIALIZER(channel_subsys.io_adapters),
+    .indicator_addresses =
+        QTAILQ_HEAD_INITIALIZER(channel_subsys.indicator_addresses),
+};
+
+IndAddr *get_indicator(hwaddr ind_addr, int len)
+{
+    IndAddr *indicator;
+
+    QTAILQ_FOREACH(indicator, &channel_subsys.indicator_addresses, sibling) {
+        if (indicator->addr == ind_addr) {
+            indicator->refcnt++;
+            return indicator;
+        }
+    }
+    indicator = g_new0(IndAddr, 1);
+    indicator->addr = ind_addr;
+    indicator->len = len;
+    indicator->refcnt = 1;
+    QTAILQ_INSERT_TAIL(&channel_subsys.indicator_addresses,
+                       indicator, sibling);
+    return indicator;
+}
+
+static int s390_io_adapter_map(AdapterInfo *adapter, uint64_t map_addr,
+                               bool do_map)
+{
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+
+    return fsc->io_adapter_map(fs, adapter->adapter_id, map_addr, do_map);
+}
+
+void release_indicator(AdapterInfo *adapter, IndAddr *indicator)
+{
+    assert(indicator->refcnt > 0);
+    indicator->refcnt--;
+    if (indicator->refcnt > 0) {
+        return;
+    }
+    QTAILQ_REMOVE(&channel_subsys.indicator_addresses, indicator, sibling);
+    if (indicator->map) {
+        s390_io_adapter_map(adapter, indicator->map, false);
+    }
+    g_free(indicator);
+}
+
+int map_indicator(AdapterInfo *adapter, IndAddr *indicator)
+{
+    int ret;
+
+    if (indicator->map) {
+        return 0; /* already mapped is not an error */
+    }
+    indicator->map = indicator->addr;
+    ret = s390_io_adapter_map(adapter, indicator->map, true);
+    if ((ret != 0) && (ret != -ENOSYS)) {
+        goto out_err;
+    }
+    return 0;
+
+out_err:
+    indicator->map = 0;
+    return ret;
+}
 
 int css_create_css_image(uint8_t cssid, bool default_image)
 {
@@ -68,12 +144,12 @@
     if (cssid > MAX_CSSID) {
         return -EINVAL;
     }
-    if (channel_subsys->css[cssid]) {
+    if (channel_subsys.css[cssid]) {
         return -EBUSY;
     }
-    channel_subsys->css[cssid] = g_malloc0(sizeof(CssImage));
+    channel_subsys.css[cssid] = g_malloc0(sizeof(CssImage));
     if (default_image) {
-        channel_subsys->default_cssid = cssid;
+        channel_subsys.default_cssid = cssid;
     }
     return 0;
 }
@@ -88,7 +164,7 @@
     S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
 
     *id = 0;
-    QTAILQ_FOREACH(adapter, &channel_subsys->io_adapters, sibling) {
+    QTAILQ_FOREACH(adapter, &channel_subsys.io_adapters, sibling) {
         if ((adapter->type == type) && (adapter->isc == isc)) {
             *id = adapter->id;
             found = true;
@@ -108,7 +184,7 @@
         adapter->id = *id;
         adapter->isc = isc;
         adapter->type = type;
-        QTAILQ_INSERT_TAIL(&channel_subsys->io_adapters, adapter, sibling);
+        QTAILQ_INSERT_TAIL(&channel_subsys.io_adapters, adapter, sibling);
     } else {
         g_free(adapter);
         fprintf(stderr, "Unexpected error %d when registering adapter %d\n",
@@ -118,12 +194,46 @@
     return ret;
 }
 
+static void css_clear_io_interrupt(uint16_t subchannel_id,
+                                   uint16_t subchannel_nr)
+{
+    Error *err = NULL;
+    static bool no_clear_irq;
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+    int r;
+
+    if (unlikely(no_clear_irq)) {
+        return;
+    }
+    r = fsc->clear_io_irq(fs, subchannel_id, subchannel_nr);
+    switch (r) {
+    case 0:
+        break;
+    case -ENOSYS:
+        no_clear_irq = true;
+        /*
+        * Ignore unavailability, as the user can't do anything
+        * about it anyway.
+        */
+        break;
+    default:
+        error_setg_errno(&err, -r, "unexpected error condition");
+        error_propagate(&error_abort, err);
+    }
+}
+
+static inline uint16_t css_do_build_subchannel_id(uint8_t cssid, uint8_t ssid)
+{
+    if (channel_subsys.max_cssid > 0) {
+        return (cssid << 8) | (1 << 3) | (ssid << 1) | 1;
+    }
+    return (ssid << 1) | 1;
+}
+
 uint16_t css_build_subchannel_id(SubchDev *sch)
 {
-    if (channel_subsys->max_cssid > 0) {
-        return (sch->cssid << 8) | (1 << 3) | (sch->ssid << 1) | 1;
-    }
-    return (sch->ssid << 1) | 1;
+    return css_do_build_subchannel_id(sch->cssid, sch->ssid);
 }
 
 static void css_inject_io_interrupt(SubchDev *sch)
@@ -261,11 +371,15 @@
         ret.flags = tmp0.flags;
         ret.count = be16_to_cpu(tmp0.count);
         ret.cda = be16_to_cpu(tmp0.cda1) | (tmp0.cda0 << 16);
+        if ((ret.cmd_code & 0x0f) == CCW_CMD_TIC) {
+            ret.cmd_code &= 0x0f;
+        }
     }
     return ret;
 }
 
-static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr)
+static int css_interpret_ccw(SubchDev *sch, hwaddr ccw_addr,
+                             bool suspend_allowed)
 {
     int ret;
     bool check_len;
@@ -287,9 +401,13 @@
         ((ccw.cmd_code & 0xf0) != 0)) {
         return -EINVAL;
     }
+    if (!sch->ccw_fmt_1 && (ccw.count == 0) &&
+        (ccw.cmd_code != CCW_CMD_TIC)) {
+        return -EINVAL;
+    }
 
     if (ccw.flags & CCW_FLAG_SUSPEND) {
-        return -EINPROGRESS;
+        return suspend_allowed ? -EINPROGRESS : -EINVAL;
     }
 
     check_len = !((ccw.flags & CCW_FLAG_SLI) && !(ccw.flags & CCW_FLAG_DC));
@@ -387,11 +505,15 @@
     SCSW *s = &sch->curr_status.scsw;
     int path;
     int ret;
+    bool suspend_allowed;
 
     /* Path management: In our simple css, we always choose the only path. */
     path = 0x80;
 
     if (!(s->ctrl & SCSW_ACTL_SUSP)) {
+        /* Start Function triggered via ssch, i.e. we have an ORB */
+        s->cstat = 0;
+        s->dstat = 0;
         /* Look at the orb and try to execute the channel program. */
         assert(orb != NULL); /* resume does not pass an orb */
         p->intparm = orb->intparm;
@@ -403,13 +525,19 @@
             return;
         }
         sch->ccw_fmt_1 = !!(orb->ctrl0 & ORB_CTRL0_MASK_FMT);
+        s->flags |= (sch->ccw_fmt_1) ? SCSW_FLAGS_MASK_FMT : 0;
         sch->ccw_no_data_cnt = 0;
+        suspend_allowed = !!(orb->ctrl0 & ORB_CTRL0_MASK_SPND);
     } else {
+        /* Start Function resumed via rsch, i.e. we don't have an
+         * ORB */
         s->ctrl &= ~(SCSW_ACTL_SUSP | SCSW_ACTL_RESUME_PEND);
+        /* The channel program had been suspended before. */
+        suspend_allowed = true;
     }
     sch->last_cmd_valid = false;
     do {
-        ret = css_interpret_ccw(sch, sch->channel_prog);
+        ret = css_interpret_ccw(sch, sch->channel_prog, suspend_allowed);
         switch (ret) {
         case -EAGAIN:
             /* ccw chain, continue processing */
@@ -485,6 +613,7 @@
     } else if (s->ctrl & SCSW_FCTL_HALT_FUNC) {
         sch_handle_halt_func(sch);
     } else if (s->ctrl & SCSW_FCTL_START_FUNC) {
+        /* Triggered by both ssch and rsch. */
         sch_handle_start_func(sch, orb);
     } else {
         /* Cannot happen. */
@@ -584,10 +713,11 @@
     }
 }
 
-int css_do_msch(SubchDev *sch, SCHIB *orig_schib)
+int css_do_msch(SubchDev *sch, const SCHIB *orig_schib)
 {
     SCSW *s = &sch->curr_status.scsw;
     PMCW *p = &sch->curr_status.pmcw;
+    uint16_t oldflags;
     int ret;
     SCHIB schib;
 
@@ -610,6 +740,7 @@
     copy_schib_from_guest(&schib, orig_schib);
     /* Only update the program-modifiable fields. */
     p->intparm = schib.pmcw.intparm;
+    oldflags = p->flags;
     p->flags &= ~(PMCW_FLAGS_MASK_ISC | PMCW_FLAGS_MASK_ENA |
                   PMCW_FLAGS_MASK_LM | PMCW_FLAGS_MASK_MME |
                   PMCW_FLAGS_MASK_MP);
@@ -625,6 +756,12 @@
             (PMCW_CHARS_MASK_MBFC | PMCW_CHARS_MASK_CSENSE);
     sch->curr_status.mba = schib.mba;
 
+    /* Has the channel been disabled? */
+    if (sch->disable_cb && (oldflags & PMCW_FLAGS_MASK_ENA) != 0
+        && (p->flags & PMCW_FLAGS_MASK_ENA) == 0) {
+        sch->disable_cb(sch);
+    }
+
     ret = 0;
 
 out:
@@ -684,7 +821,7 @@
 
     /* Trigger the clear function. */
     s->ctrl &= ~(SCSW_CTRL_MASK_FCTL | SCSW_CTRL_MASK_ACTL);
-    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_FCTL_CLEAR_FUNC;
+    s->ctrl |= SCSW_FCTL_CLEAR_FUNC | SCSW_ACTL_CLEAR_PEND;
 
     do_subchannel_work(sch, NULL);
     ret = 0;
@@ -745,20 +882,27 @@
         /* Format 1, per-subchannel area. */
         uint32_t count;
 
-        count = ldl_phys(&address_space_memory, sch->curr_status.mba);
+        count = address_space_ldl(&address_space_memory,
+                                  sch->curr_status.mba,
+                                  MEMTXATTRS_UNSPECIFIED,
+                                  NULL);
         count++;
-        stl_phys(&address_space_memory, sch->curr_status.mba, count);
+        address_space_stl(&address_space_memory, sch->curr_status.mba, count,
+                          MEMTXATTRS_UNSPECIFIED, NULL);
     } else {
         /* Format 0, global area. */
         uint32_t offset;
         uint16_t count;
 
         offset = sch->curr_status.pmcw.mbi << 5;
-        count = lduw_phys(&address_space_memory,
-                          channel_subsys->chnmon_area + offset);
+        count = address_space_lduw(&address_space_memory,
+                                   channel_subsys.chnmon_area + offset,
+                                   MEMTXATTRS_UNSPECIFIED,
+                                   NULL);
         count++;
-        stw_phys(&address_space_memory,
-                 channel_subsys->chnmon_area + offset, count);
+        address_space_stw(&address_space_memory,
+                          channel_subsys.chnmon_area + offset, count,
+                          MEMTXATTRS_UNSPECIFIED, NULL);
     }
 }
 
@@ -786,7 +930,7 @@
     }
 
     /* If monitoring is active, update counter. */
-    if (channel_subsys->chnmon_active) {
+    if (channel_subsys.chnmon_active) {
         css_update_chnmon(sch);
     }
     sch->channel_prog = orb->cpa;
@@ -801,7 +945,8 @@
     return ret;
 }
 
-static void copy_irb_to_guest(IRB *dest, const IRB *src, PMCW *pmcw)
+static void copy_irb_to_guest(IRB *dest, const IRB *src, PMCW *pmcw,
+                              int *irb_len)
 {
     int i;
     uint16_t stctl = src->scsw.ctrl & SCSW_CTRL_MASK_STCTL;
@@ -815,6 +960,8 @@
     for (i = 0; i < ARRAY_SIZE(dest->ecw); i++) {
         dest->ecw[i] = cpu_to_be32(src->ecw[i]);
     }
+    *irb_len = sizeof(*dest) - sizeof(dest->emw);
+
     /* extended measurements enabled? */
     if ((src->scsw.flags & SCSW_FLAGS_MASK_ESWF) ||
         !(pmcw->flags & PMCW_FLAGS_MASK_TF) ||
@@ -832,26 +979,21 @@
             dest->emw[i] = cpu_to_be32(src->emw[i]);
         }
     }
+    *irb_len = sizeof(*dest);
 }
 
-int css_do_tsch(SubchDev *sch, IRB *target_irb)
+int css_do_tsch_get_irb(SubchDev *sch, IRB *target_irb, int *irb_len)
 {
     SCSW *s = &sch->curr_status.scsw;
     PMCW *p = &sch->curr_status.pmcw;
     uint16_t stctl;
-    uint16_t fctl;
-    uint16_t actl;
     IRB irb;
-    int ret;
 
     if (!(p->flags & (PMCW_FLAGS_MASK_DNV | PMCW_FLAGS_MASK_ENA))) {
-        ret = 3;
-        goto out;
+        return 3;
     }
 
     stctl = s->ctrl & SCSW_CTRL_MASK_STCTL;
-    fctl = s->ctrl & SCSW_CTRL_MASK_FCTL;
-    actl = s->ctrl & SCSW_CTRL_MASK_ACTL;
 
     /* Prepare the irb for the guest. */
     memset(&irb, 0, sizeof(IRB));
@@ -870,13 +1012,34 @@
         /* If a unit check is pending, copy sense data. */
         if ((s->dstat & SCSW_DSTAT_UNIT_CHECK) &&
             (p->chars & PMCW_CHARS_MASK_CSENSE)) {
+            int i;
+
             irb.scsw.flags |= SCSW_FLAGS_MASK_ESWF | SCSW_FLAGS_MASK_ECTL;
+            /* Attention: sense_data is already BE! */
             memcpy(irb.ecw, sch->sense_data, sizeof(sch->sense_data));
+            for (i = 0; i < ARRAY_SIZE(irb.ecw); i++) {
+                irb.ecw[i] = be32_to_cpu(irb.ecw[i]);
+            }
             irb.esw[1] = 0x01000000 | (sizeof(sch->sense_data) << 8);
         }
     }
     /* Store the irb to the guest. */
-    copy_irb_to_guest(target_irb, &irb, p);
+    copy_irb_to_guest(target_irb, &irb, p, irb_len);
+
+    return ((stctl & SCSW_STCTL_STATUS_PEND) == 0);
+}
+
+void css_do_tsch_update_subch(SubchDev *sch)
+{
+    SCSW *s = &sch->curr_status.scsw;
+    PMCW *p = &sch->curr_status.pmcw;
+    uint16_t stctl;
+    uint16_t fctl;
+    uint16_t actl;
+
+    stctl = s->ctrl & SCSW_CTRL_MASK_STCTL;
+    fctl = s->ctrl & SCSW_CTRL_MASK_FCTL;
+    actl = s->ctrl & SCSW_CTRL_MASK_ACTL;
 
     /* Clear conditions on subchannel, if applicable. */
     if (stctl & SCSW_STCTL_STATUS_PEND) {
@@ -913,11 +1076,6 @@
             memset(sch->sense_data, 0 , sizeof(sch->sense_data));
         }
     }
-
-    ret = ((stctl & SCSW_STCTL_STATUS_PEND) == 0);
-
-out:
-    return ret;
 }
 
 static void copy_crw_to_guest(CRW *dest, const CRW *src)
@@ -931,22 +1089,42 @@
     CrwContainer *crw_cont;
     int ret;
 
-    crw_cont = QTAILQ_FIRST(&channel_subsys->pending_crws);
+    crw_cont = QTAILQ_FIRST(&channel_subsys.pending_crws);
     if (crw_cont) {
-        QTAILQ_REMOVE(&channel_subsys->pending_crws, crw_cont, sibling);
+        QTAILQ_REMOVE(&channel_subsys.pending_crws, crw_cont, sibling);
         copy_crw_to_guest(crw, &crw_cont->crw);
         g_free(crw_cont);
         ret = 0;
     } else {
         /* List was empty, turn crw machine checks on again. */
         memset(crw, 0, sizeof(*crw));
-        channel_subsys->do_crw_mchk = true;
+        channel_subsys.do_crw_mchk = true;
         ret = 1;
     }
 
     return ret;
 }
 
+static void copy_crw_from_guest(CRW *dest, const CRW *src)
+{
+    dest->flags = be16_to_cpu(src->flags);
+    dest->rsid = be16_to_cpu(src->rsid);
+}
+
+void css_undo_stcrw(CRW *crw)
+{
+    CrwContainer *crw_cont;
+
+    crw_cont = g_try_malloc0(sizeof(CrwContainer));
+    if (!crw_cont) {
+        channel_subsys.crws_lost = true;
+        return;
+    }
+    copy_crw_from_guest(&crw_cont->crw, crw);
+
+    QTAILQ_INSERT_HEAD(&channel_subsys.pending_crws, crw_cont, sibling);
+}
+
 int css_do_tpi(IOIntCode *int_code, int lowcore)
 {
     /* No pending interrupts for !KVM. */
@@ -962,9 +1140,9 @@
     CssImage *css;
 
     if (!m && !cssid) {
-        css = channel_subsys->css[channel_subsys->default_cssid];
+        css = channel_subsys.css[channel_subsys.default_cssid];
     } else {
-        css = channel_subsys->css[cssid];
+        css = channel_subsys.css[cssid];
     }
     if (!css) {
         return 0;
@@ -999,15 +1177,15 @@
 {
     /* dct is currently ignored (not really meaningful for our devices) */
     /* TODO: Don't ignore mbk. */
-    if (update && !channel_subsys->chnmon_active) {
+    if (update && !channel_subsys.chnmon_active) {
         /* Enable measuring. */
-        channel_subsys->chnmon_area = mbo;
-        channel_subsys->chnmon_active = true;
+        channel_subsys.chnmon_area = mbo;
+        channel_subsys.chnmon_active = true;
     }
-    if (!update && channel_subsys->chnmon_active) {
+    if (!update && channel_subsys.chnmon_active) {
         /* Disable measuring. */
-        channel_subsys->chnmon_area = 0;
-        channel_subsys->chnmon_active = false;
+        channel_subsys.chnmon_area = 0;
+        channel_subsys.chnmon_active = false;
     }
 }
 
@@ -1035,7 +1213,7 @@
     }
 
     /* If monitoring is active, update counter. */
-    if (channel_subsys->chnmon_active) {
+    if (channel_subsys.chnmon_active) {
         css_update_chnmon(sch);
     }
 
@@ -1051,23 +1229,23 @@
 {
     uint8_t real_cssid;
 
-    if (cssid > channel_subsys->max_cssid) {
+    if (cssid > channel_subsys.max_cssid) {
         return -EINVAL;
     }
-    if (channel_subsys->max_cssid == 0) {
-        real_cssid = channel_subsys->default_cssid;
+    if (channel_subsys.max_cssid == 0) {
+        real_cssid = channel_subsys.default_cssid;
     } else {
         real_cssid = cssid;
     }
-    if (!channel_subsys->css[real_cssid]) {
+    if (!channel_subsys.css[real_cssid]) {
         return -EINVAL;
     }
 
-    if (!channel_subsys->css[real_cssid]->chpids[chpid].in_use) {
+    if (!channel_subsys.css[real_cssid]->chpids[chpid].in_use) {
         return -ENODEV;
     }
 
-    if (!channel_subsys->css[real_cssid]->chpids[chpid].is_virtual) {
+    if (!channel_subsys.css[real_cssid]->chpids[chpid].is_virtual) {
         fprintf(stderr,
                 "rchp unsupported for non-virtual chpid %x.%02x!\n",
                 real_cssid, chpid);
@@ -1076,8 +1254,8 @@
 
     /* We don't really use a channel path, so we're done here. */
     css_queue_crw(CRW_RSC_CHP, CRW_ERC_INIT,
-                  channel_subsys->max_cssid > 0 ? 1 : 0, chpid);
-    if (channel_subsys->max_cssid > 0) {
+                  channel_subsys.max_cssid > 0 ? 1 : 0, chpid);
+    if (channel_subsys.max_cssid > 0) {
         css_queue_crw(CRW_RSC_CHP, CRW_ERC_INIT, 0, real_cssid << 8);
     }
     return 0;
@@ -1088,13 +1266,13 @@
     SubchSet *set;
     uint8_t real_cssid;
 
-    real_cssid = (!m && (cssid == 0)) ? channel_subsys->default_cssid : cssid;
+    real_cssid = (!m && (cssid == 0)) ? channel_subsys.default_cssid : cssid;
     if (real_cssid > MAX_CSSID || ssid > MAX_SSID ||
-        !channel_subsys->css[real_cssid] ||
-        !channel_subsys->css[real_cssid]->sch_set[ssid]) {
+        !channel_subsys.css[real_cssid] ||
+        !channel_subsys.css[real_cssid]->sch_set[ssid]) {
         return true;
     }
-    set = channel_subsys->css[real_cssid]->sch_set[ssid];
+    set = channel_subsys.css[real_cssid]->sch_set[ssid];
     return schid > find_last_bit(set->schids_used,
                                  (MAX_SCHID + 1) / sizeof(unsigned long));
 }
@@ -1107,7 +1285,7 @@
     if (cssid > MAX_CSSID) {
         return -EINVAL;
     }
-    css = channel_subsys->css[cssid];
+    css = channel_subsys.css[cssid];
     if (!css) {
         return -EINVAL;
     }
@@ -1128,7 +1306,7 @@
     PMCW *p = &sch->curr_status.pmcw;
     SCSW *s = &sch->curr_status.scsw;
     int i;
-    CssImage *css = channel_subsys->css[sch->cssid];
+    CssImage *css = channel_subsys.css[sch->cssid];
 
     assert(css != NULL);
     memset(p, 0, sizeof(PMCW));
@@ -1154,27 +1332,137 @@
 {
     uint8_t real_cssid;
 
-    real_cssid = (!m && (cssid == 0)) ? channel_subsys->default_cssid : cssid;
+    real_cssid = (!m && (cssid == 0)) ? channel_subsys.default_cssid : cssid;
 
-    if (!channel_subsys->css[real_cssid]) {
+    if (!channel_subsys.css[real_cssid]) {
         return NULL;
     }
 
-    if (!channel_subsys->css[real_cssid]->sch_set[ssid]) {
+    if (!channel_subsys.css[real_cssid]->sch_set[ssid]) {
         return NULL;
     }
 
-    return channel_subsys->css[real_cssid]->sch_set[ssid]->sch[schid];
+    return channel_subsys.css[real_cssid]->sch_set[ssid]->sch[schid];
+}
+
+/**
+ * Return free device number in subchannel set.
+ *
+ * Return index of the first free device number in the subchannel set
+ * identified by @p cssid and @p ssid, beginning the search at @p
+ * start and wrapping around at MAX_DEVNO. Return a value exceeding
+ * MAX_SCHID if there are no free device numbers in the subchannel
+ * set.
+ */
+static uint32_t css_find_free_devno(uint8_t cssid, uint8_t ssid,
+                                    uint16_t start)
+{
+    uint32_t round;
+
+    for (round = 0; round <= MAX_DEVNO; round++) {
+        uint16_t devno = (start + round) % MAX_DEVNO;
+
+        if (!css_devno_used(cssid, ssid, devno)) {
+            return devno;
+        }
+    }
+    return MAX_DEVNO + 1;
+}
+
+/**
+ * Return first free subchannel (id) in subchannel set.
+ *
+ * Return index of the first free subchannel in the subchannel set
+ * identified by @p cssid and @p ssid, if there is any. Return a value
+ * exceeding MAX_SCHID if there are no free subchannels in the
+ * subchannel set.
+ */
+static uint32_t css_find_free_subch(uint8_t cssid, uint8_t ssid)
+{
+    uint32_t schid;
+
+    for (schid = 0; schid <= MAX_SCHID; schid++) {
+        if (!css_find_subch(1, cssid, ssid, schid)) {
+            return schid;
+        }
+    }
+    return MAX_SCHID + 1;
+}
+
+/**
+ * Return first free subchannel (id) in subchannel set for a device number
+ *
+ * Verify the device number @p devno is not used yet in the subchannel
+ * set identified by @p cssid and @p ssid. Set @p schid to the index
+ * of the first free subchannel in the subchannel set, if there is
+ * any. Return true if everything succeeded and false otherwise.
+ */
+static bool css_find_free_subch_for_devno(uint8_t cssid, uint8_t ssid,
+                                          uint16_t devno, uint16_t *schid,
+                                          Error **errp)
+{
+    uint32_t free_schid;
+
+    assert(schid);
+    if (css_devno_used(cssid, ssid, devno)) {
+        error_setg(errp, "Device %x.%x.%04x already exists",
+                   cssid, ssid, devno);
+        return false;
+    }
+    free_schid = css_find_free_subch(cssid, ssid);
+    if (free_schid > MAX_SCHID) {
+        error_setg(errp, "No free subchannel found for %x.%x.%04x",
+                   cssid, ssid, devno);
+        return false;
+    }
+    *schid = free_schid;
+    return true;
+}
+
+/**
+ * Return first free subchannel (id) and device number
+ *
+ * Locate the first free subchannel and first free device number in
+ * any of the subchannel sets of the channel subsystem identified by
+ * @p cssid. Return false if no free subchannel / device number could
+ * be found. Otherwise set @p ssid, @p devno and @p schid to identify
+ * the available subchannel and device number and return true.
+ *
+ * May modify @p ssid, @p devno and / or @p schid even if no free
+ * subchannel / device number could be found.
+ */
+static bool css_find_free_subch_and_devno(uint8_t cssid, uint8_t *ssid,
+                                          uint16_t *devno, uint16_t *schid,
+                                          Error **errp)
+{
+    uint32_t free_schid, free_devno;
+
+    assert(ssid && devno && schid);
+    for (*ssid = 0; *ssid <= MAX_SSID; (*ssid)++) {
+        free_schid = css_find_free_subch(cssid, *ssid);
+        if (free_schid > MAX_SCHID) {
+            continue;
+        }
+        free_devno = css_find_free_devno(cssid, *ssid, free_schid);
+        if (free_devno > MAX_DEVNO) {
+            continue;
+        }
+        *schid = free_schid;
+        *devno = free_devno;
+        return true;
+    }
+    error_setg(errp, "Virtual channel subsystem is full!");
+    return false;
 }
 
 bool css_subch_visible(SubchDev *sch)
 {
-    if (sch->ssid > channel_subsys->max_ssid) {
+    if (sch->ssid > channel_subsys.max_ssid) {
         return false;
     }
 
-    if (sch->cssid != channel_subsys->default_cssid) {
-        return (channel_subsys->max_cssid > 0);
+    if (sch->cssid != channel_subsys.default_cssid) {
+        return (channel_subsys.max_cssid > 0);
     }
 
     return true;
@@ -1182,20 +1470,20 @@
 
 bool css_present(uint8_t cssid)
 {
-    return (channel_subsys->css[cssid] != NULL);
+    return (channel_subsys.css[cssid] != NULL);
 }
 
 bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno)
 {
-    if (!channel_subsys->css[cssid]) {
+    if (!channel_subsys.css[cssid]) {
         return false;
     }
-    if (!channel_subsys->css[cssid]->sch_set[ssid]) {
+    if (!channel_subsys.css[cssid]->sch_set[ssid]) {
         return false;
     }
 
     return !!test_bit(devno,
-                      channel_subsys->css[cssid]->sch_set[ssid]->devnos_used);
+                      channel_subsys.css[cssid]->sch_set[ssid]->devnos_used);
 }
 
 void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
@@ -1206,13 +1494,13 @@
 
     trace_css_assign_subch(sch ? "assign" : "deassign", cssid, ssid, schid,
                            devno);
-    if (!channel_subsys->css[cssid]) {
+    if (!channel_subsys.css[cssid]) {
         fprintf(stderr,
                 "Suspicious call to %s (%x.%x.%04x) for non-existing css!\n",
                 __func__, cssid, ssid, schid);
         return;
     }
-    css = channel_subsys->css[cssid];
+    css = channel_subsys.css[cssid];
 
     if (!css->sch_set[ssid]) {
         css->sch_set[ssid] = g_malloc0(sizeof(SubchSet));
@@ -1237,7 +1525,7 @@
     /* TODO: Maybe use a static crw pool? */
     crw_cont = g_try_malloc0(sizeof(CrwContainer));
     if (!crw_cont) {
-        channel_subsys->crws_lost = true;
+        channel_subsys.crws_lost = true;
         return;
     }
     crw_cont->crw.flags = (rsc << 8) | erc;
@@ -1245,15 +1533,15 @@
         crw_cont->crw.flags |= CRW_FLAGS_MASK_C;
     }
     crw_cont->crw.rsid = rsid;
-    if (channel_subsys->crws_lost) {
+    if (channel_subsys.crws_lost) {
         crw_cont->crw.flags |= CRW_FLAGS_MASK_R;
-        channel_subsys->crws_lost = false;
+        channel_subsys.crws_lost = false;
     }
 
-    QTAILQ_INSERT_TAIL(&channel_subsys->pending_crws, crw_cont, sibling);
+    QTAILQ_INSERT_TAIL(&channel_subsys.pending_crws, crw_cont, sibling);
 
-    if (channel_subsys->do_crw_mchk) {
-        channel_subsys->do_crw_mchk = false;
+    if (channel_subsys.do_crw_mchk) {
+        channel_subsys.do_crw_mchk = false;
         /* Inject crw pending machine check. */
         s390_crw_mchk();
     }
@@ -1268,9 +1556,9 @@
     if (add && !hotplugged) {
         return;
     }
-    if (channel_subsys->max_cssid == 0) {
+    if (channel_subsys.max_cssid == 0) {
         /* Default cssid shows up as 0. */
-        guest_cssid = (cssid == channel_subsys->default_cssid) ? 0 : cssid;
+        guest_cssid = (cssid == channel_subsys.default_cssid) ? 0 : cssid;
     } else {
         /* Show real cssid to the guest. */
         guest_cssid = cssid;
@@ -1279,19 +1567,21 @@
      * Only notify for higher subchannel sets/channel subsystems if the
      * guest has enabled it.
      */
-    if ((ssid > channel_subsys->max_ssid) ||
-        (guest_cssid > channel_subsys->max_cssid) ||
-        ((channel_subsys->max_cssid == 0) &&
-         (cssid != channel_subsys->default_cssid))) {
+    if ((ssid > channel_subsys.max_ssid) ||
+        (guest_cssid > channel_subsys.max_cssid) ||
+        ((channel_subsys.max_cssid == 0) &&
+         (cssid != channel_subsys.default_cssid))) {
         return;
     }
-    chain_crw = (channel_subsys->max_ssid > 0) ||
-            (channel_subsys->max_cssid > 0);
+    chain_crw = (channel_subsys.max_ssid > 0) ||
+            (channel_subsys.max_cssid > 0);
     css_queue_crw(CRW_RSC_SUBCH, CRW_ERC_IPI, chain_crw ? 1 : 0, schid);
     if (chain_crw) {
         css_queue_crw(CRW_RSC_SUBCH, CRW_ERC_IPI, 0,
                       (guest_cssid << 8) | (ssid << 4));
     }
+    /* RW_ERC_IPI --> clear pending interrupts */
+    css_clear_io_interrupt(css_do_build_subchannel_id(cssid, ssid), schid);
 }
 
 void css_generate_chp_crws(uint8_t cssid, uint8_t chpid)
@@ -1299,17 +1589,30 @@
     /* TODO */
 }
 
+void css_generate_css_crws(uint8_t cssid)
+{
+    if (!channel_subsys.sei_pending) {
+        css_queue_crw(CRW_RSC_CSS, 0, 0, cssid);
+    }
+    channel_subsys.sei_pending = true;
+}
+
+void css_clear_sei_pending(void)
+{
+    channel_subsys.sei_pending = false;
+}
+
 int css_enable_mcsse(void)
 {
     trace_css_enable_facility("mcsse");
-    channel_subsys->max_cssid = MAX_CSSID;
+    channel_subsys.max_cssid = MAX_CSSID;
     return 0;
 }
 
 int css_enable_mss(void)
 {
     trace_css_enable_facility("mss");
-    channel_subsys->max_ssid = MAX_SSID;
+    channel_subsys.max_ssid = MAX_SSID;
     return 0;
 }
 
@@ -1367,7 +1670,6 @@
     }
     qemu_put_byte(f, s->ccw_fmt_1);
     qemu_put_byte(f, s->ccw_no_data_cnt);
-    return;
 }
 
 int subch_device_load(SubchDev *s, QEMUFile *f)
@@ -1424,25 +1726,32 @@
     }
     s->ccw_fmt_1 = qemu_get_byte(f);
     s->ccw_no_data_cnt = qemu_get_byte(f);
+    /*
+     * Hack alert. We don't migrate the channel subsystem status (no
+     * device!), but we need to find out if the guest enabled mss/mcss-e.
+     * If the subchannel is enabled, it certainly was able to access it,
+     * so adjust the max_ssid/max_cssid values for relevant ssid/cssid
+     * values. This is not watertight, but better than nothing.
+     */
+    if (s->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ENA) {
+        if (s->ssid) {
+            channel_subsys.max_ssid = MAX_SSID;
+        }
+        if (s->cssid != channel_subsys.default_cssid) {
+            channel_subsys.max_cssid = MAX_CSSID;
+        }
+    }
     return 0;
 }
 
-
-static void css_init(void)
-{
-    channel_subsys = g_malloc0(sizeof(*channel_subsys));
-    QTAILQ_INIT(&channel_subsys->pending_crws);
-    channel_subsys->do_crw_mchk = true;
-    channel_subsys->crws_lost = false;
-    channel_subsys->chnmon_active = false;
-    QTAILQ_INIT(&channel_subsys->io_adapters);
-}
-machine_init(css_init);
-
 void css_reset_sch(SubchDev *sch)
 {
     PMCW *p = &sch->curr_status.pmcw;
 
+    if ((p->flags & PMCW_FLAGS_MASK_ENA) != 0 && sch->disable_cb) {
+        sch->disable_cb(sch);
+    }
+
     p->intparm = 0;
     p->flags &= ~(PMCW_FLAGS_MASK_ISC | PMCW_FLAGS_MASK_ENA |
                   PMCW_FLAGS_MASK_LM | PMCW_FLAGS_MASK_MME |
@@ -1472,18 +1781,132 @@
     CrwContainer *crw_cont;
 
     /* Clean up monitoring. */
-    channel_subsys->chnmon_active = false;
-    channel_subsys->chnmon_area = 0;
+    channel_subsys.chnmon_active = false;
+    channel_subsys.chnmon_area = 0;
 
     /* Clear pending CRWs. */
-    while ((crw_cont = QTAILQ_FIRST(&channel_subsys->pending_crws))) {
-        QTAILQ_REMOVE(&channel_subsys->pending_crws, crw_cont, sibling);
+    while ((crw_cont = QTAILQ_FIRST(&channel_subsys.pending_crws))) {
+        QTAILQ_REMOVE(&channel_subsys.pending_crws, crw_cont, sibling);
         g_free(crw_cont);
     }
-    channel_subsys->do_crw_mchk = true;
-    channel_subsys->crws_lost = false;
+    channel_subsys.sei_pending = false;
+    channel_subsys.do_crw_mchk = true;
+    channel_subsys.crws_lost = false;
 
     /* Reset maximum ids. */
-    channel_subsys->max_cssid = 0;
-    channel_subsys->max_ssid = 0;
+    channel_subsys.max_cssid = 0;
+    channel_subsys.max_ssid = 0;
+}
+
+static void get_css_devid(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    CssDevId *dev_id = qdev_get_prop_ptr(dev, prop);
+    char buffer[] = "xx.x.xxxx";
+    char *p = buffer;
+    int r;
+
+    if (dev_id->valid) {
+
+        r = snprintf(buffer, sizeof(buffer), "%02x.%1x.%04x", dev_id->cssid,
+                     dev_id->ssid, dev_id->devid);
+        assert(r == sizeof(buffer) - 1);
+
+        /* drop leading zero */
+        if (dev_id->cssid <= 0xf) {
+            p++;
+        }
+    } else {
+        snprintf(buffer, sizeof(buffer), "<unset>");
+    }
+
+    visit_type_str(v, name, &p, errp);
+}
+
+/*
+ * parse <cssid>.<ssid>.<devid> and assert valid range for cssid/ssid
+ */
+static void set_css_devid(Object *obj, Visitor *v, const char *name,
+                          void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    Property *prop = opaque;
+    CssDevId *dev_id = qdev_get_prop_ptr(dev, prop);
+    Error *local_err = NULL;
+    char *str;
+    int num, n1, n2;
+    unsigned int cssid, ssid, devid;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_str(v, name, &str, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    num = sscanf(str, "%2x.%1x%n.%4x%n", &cssid, &ssid, &n1, &devid, &n2);
+    if (num != 3 || (n2 - n1) != 5 || strlen(str) != n2) {
+        error_set_from_qdev_prop_error(errp, EINVAL, dev, prop, str);
+        goto out;
+    }
+    if ((cssid > MAX_CSSID) || (ssid > MAX_SSID)) {
+        error_setg(errp, "Invalid cssid or ssid: cssid %x, ssid %x",
+                   cssid, ssid);
+        goto out;
+    }
+
+    dev_id->cssid = cssid;
+    dev_id->ssid = ssid;
+    dev_id->devid = devid;
+    dev_id->valid = true;
+
+out:
+    g_free(str);
+}
+
+PropertyInfo css_devid_propinfo = {
+    .name = "str",
+    .description = "Identifier of an I/O device in the channel "
+                   "subsystem, example: fe.1.23ab",
+    .get = get_css_devid,
+    .set = set_css_devid,
+};
+
+SubchDev *css_create_virtual_sch(CssDevId bus_id, Error **errp)
+{
+    uint16_t schid = 0;
+    SubchDev *sch;
+
+    if (bus_id.valid) {
+        /* Enforce use of virtual cssid. */
+        if (bus_id.cssid != VIRTUAL_CSSID) {
+            error_setg(errp, "cssid %hhx not valid for virtual devices",
+                       bus_id.cssid);
+            return NULL;
+        }
+        if (!css_find_free_subch_for_devno(bus_id.cssid, bus_id.ssid,
+                                           bus_id.devid, &schid, errp)) {
+            return NULL;
+        }
+    } else {
+        bus_id.cssid = VIRTUAL_CSSID;
+        if (!css_find_free_subch_and_devno(bus_id.cssid, &bus_id.ssid,
+                                           &bus_id.devid, &schid, errp)) {
+            return NULL;
+        }
+    }
+
+    sch = g_malloc0(sizeof(*sch));
+    sch->cssid = bus_id.cssid;
+    sch->ssid = bus_id.ssid;
+    sch->devno = bus_id.devid;
+    sch->schid = schid;
+    css_subch_assign(sch->cssid, sch->ssid, schid, sch->devno, sch);
+    return sch;
 }
diff --git a/hw/s390x/css.h b/hw/s390x/css.h
deleted file mode 100644
index 33104ac..0000000
--- a/hw/s390x/css.h
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Channel subsystem structures and definitions.
- *
- * Copyright 2012 IBM Corp.
- * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or (at
- * your option) any later version. See the COPYING file in the top-level
- * directory.
- */
-
-#ifndef CSS_H
-#define CSS_H
-
-#include "ioinst.h"
-
-/* Channel subsystem constants. */
-#define MAX_SCHID 65535
-#define MAX_SSID 3
-#define MAX_CSSID 254 /* 255 is reserved */
-#define MAX_CHPID 255
-
-#define MAX_CIWS 62
-
-typedef struct CIW {
-    uint8_t type;
-    uint8_t command;
-    uint16_t count;
-} QEMU_PACKED CIW;
-
-typedef struct SenseId {
-    /* common part */
-    uint8_t reserved;        /* always 0x'FF' */
-    uint16_t cu_type;        /* control unit type */
-    uint8_t cu_model;        /* control unit model */
-    uint16_t dev_type;       /* device type */
-    uint8_t dev_model;       /* device model */
-    uint8_t unused;          /* padding byte */
-    /* extended part */
-    CIW ciw[MAX_CIWS];       /* variable # of CIWs */
-} QEMU_PACKED SenseId;
-
-/* Channel measurements, from linux/drivers/s390/cio/cmf.c. */
-typedef struct CMB {
-    uint16_t ssch_rsch_count;
-    uint16_t sample_count;
-    uint32_t device_connect_time;
-    uint32_t function_pending_time;
-    uint32_t device_disconnect_time;
-    uint32_t control_unit_queuing_time;
-    uint32_t device_active_only_time;
-    uint32_t reserved[2];
-} QEMU_PACKED CMB;
-
-typedef struct CMBE {
-    uint32_t ssch_rsch_count;
-    uint32_t sample_count;
-    uint32_t device_connect_time;
-    uint32_t function_pending_time;
-    uint32_t device_disconnect_time;
-    uint32_t control_unit_queuing_time;
-    uint32_t device_active_only_time;
-    uint32_t device_busy_time;
-    uint32_t initial_command_response_time;
-    uint32_t reserved[7];
-} QEMU_PACKED CMBE;
-
-struct SubchDev {
-    /* channel-subsystem related things: */
-    uint8_t cssid;
-    uint8_t ssid;
-    uint16_t schid;
-    uint16_t devno;
-    SCHIB curr_status;
-    uint8_t sense_data[32];
-    hwaddr channel_prog;
-    CCW1 last_cmd;
-    bool last_cmd_valid;
-    bool ccw_fmt_1;
-    bool thinint_active;
-    uint8_t ccw_no_data_cnt;
-    /* transport-provided data: */
-    int (*ccw_cb) (SubchDev *, CCW1);
-    SenseId id;
-    void *driver_data;
-};
-
-typedef SubchDev *(*css_subch_cb_func)(uint8_t m, uint8_t cssid, uint8_t ssid,
-                                       uint16_t schid);
-void subch_device_save(SubchDev *s, QEMUFile *f);
-int subch_device_load(SubchDev *s, QEMUFile *f);
-int css_create_css_image(uint8_t cssid, bool default_image);
-bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno);
-void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
-                      uint16_t devno, SubchDev *sch);
-void css_sch_build_virtual_schib(SubchDev *sch, uint8_t chpid, uint8_t type);
-uint16_t css_build_subchannel_id(SubchDev *sch);
-void css_reset(void);
-void css_reset_sch(SubchDev *sch);
-void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid);
-void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
-                           int hotplugged, int add);
-void css_generate_chp_crws(uint8_t cssid, uint8_t chpid);
-void css_adapter_interrupt(uint8_t isc);
-
-#define CSS_IO_ADAPTER_VIRTIO 1
-int css_register_io_adapter(uint8_t type, uint8_t isc, bool swap,
-                            bool maskable, uint32_t *id);
-#endif
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
index 78da718..34b2faf 100644
--- a/hw/s390x/event-facility.c
+++ b/hw/s390x/event-facility.c
@@ -15,7 +15,8 @@
  *
  */
 
-#include "monitor/monitor.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 
 #include "hw/s390x/sclp.h"
@@ -32,8 +33,6 @@
     unsigned int receive_mask;
 };
 
-static SCLPEvent cpu_hotplug;
-
 /* return true if any child has event pending set */
 static bool event_pending(SCLPEventFacility *ef)
 {
@@ -241,12 +240,13 @@
         sclp_active_selection_mask = sclp_cp_receive_mask;
         break;
     case SCLP_SELECTIVE_READ:
-        if (!(sclp_cp_receive_mask & be32_to_cpu(red->mask))) {
+        sclp_active_selection_mask = be32_to_cpu(red->mask);
+        if (!sclp_cp_receive_mask ||
+            (sclp_active_selection_mask & ~sclp_cp_receive_mask)) {
             sccb->h.response_code =
                     cpu_to_be16(SCLP_RC_INVALID_SELECTION_MASK);
             goto out;
         }
-        sclp_active_selection_mask = be32_to_cpu(red->mask);
         break;
     default:
         sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
@@ -287,8 +287,26 @@
 
 #define TYPE_SCLP_EVENTS_BUS "s390-sclp-events-bus"
 
+static void sclp_events_bus_realize(BusState *bus, Error **errp)
+{
+    BusChild *kid;
+
+    /* TODO: recursive realization has to be done in common code */
+    QTAILQ_FOREACH(kid, &bus->children, sibling) {
+        DeviceState *dev = kid->child;
+
+        object_property_set_bool(OBJECT(dev), true, "realized", errp);
+        if (*errp) {
+            return;
+        }
+    }
+}
+
 static void sclp_events_bus_class_init(ObjectClass *klass, void *data)
 {
+    BusClass *bc = BUS_CLASS(klass);
+
+    bc->realize = sclp_events_bus_realize;
 }
 
 static const TypeInfo sclp_events_bus_info = {
@@ -325,26 +343,26 @@
      }
 };
 
-static int init_event_facility(SCLPEventFacility *event_facility)
+static void init_event_facility(Object *obj)
 {
-    DeviceState *sdev = DEVICE(event_facility);
-    DeviceState *quiesce;
+    SCLPEventFacility *event_facility = EVENT_FACILITY(obj);
+    DeviceState *sdev = DEVICE(obj);
+    Object *new;
 
     /* Spawn a new bus for SCLP events */
     qbus_create_inplace(&event_facility->sbus, sizeof(event_facility->sbus),
                         TYPE_SCLP_EVENTS_BUS, sdev, NULL);
 
-    quiesce = qdev_create(&event_facility->sbus.qbus, "sclpquiesce");
-    if (!quiesce) {
-        return -1;
-    }
-    qdev_init_nofail(quiesce);
+    new = object_new(TYPE_SCLP_QUIESCE);
+    object_property_add_child(obj, TYPE_SCLP_QUIESCE, new, NULL);
+    object_unref(new);
+    qdev_set_parent_bus(DEVICE(new), &event_facility->sbus.qbus);
 
-    object_initialize(&cpu_hotplug, sizeof(cpu_hotplug), TYPE_SCLP_CPU_HOTPLUG);
-    qdev_set_parent_bus(DEVICE(&cpu_hotplug), BUS(&event_facility->sbus));
-    object_property_set_bool(OBJECT(&cpu_hotplug), true, "realized", NULL);
-
-    return 0;
+    new = object_new(TYPE_SCLP_CPU_HOTPLUG);
+    object_property_add_child(obj, TYPE_SCLP_CPU_HOTPLUG, new, NULL);
+    object_unref(new);
+    qdev_set_parent_bus(DEVICE(new), &event_facility->sbus.qbus);
+    /* the facility will automatically realize the devices via the bus */
 }
 
 static void reset_event_facility(DeviceState *dev)
@@ -362,7 +380,7 @@
 
     dc->reset = reset_event_facility;
     dc->vmsd = &vmstate_event_facility;
-    k->init = init_event_facility;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     k->command_handler = command_handler;
     k->event_pending = event_pending;
 }
@@ -370,6 +388,7 @@
 static const TypeInfo sclp_event_facility_info = {
     .name          = TYPE_SCLP_EVENT_FACILITY,
     .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_init = init_event_facility,
     .instance_size = sizeof(SCLPEventFacility),
     .class_init    = init_event_facility_class,
     .class_size    = sizeof(SCLPEventFacilityClass),
diff --git a/hw/s390x/ipl.c b/hw/s390x/ipl.c
index 3b77c9a..2e2664f 100644
--- a/hw/s390x/ipl.c
+++ b/hw/s390x/ipl.c
@@ -11,13 +11,15 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "cpu.h"
 #include "elf.h"
 #include "hw/loader.h"
-#include "hw/sysbus.h"
 #include "hw/s390x/virtio-ccw.h"
 #include "hw/s390x/css.h"
+#include "ipl.h"
 
 #define KERN_IMAGE_START                0x010000UL
 #define KERN_PARM_AREA                  0x010480UL
@@ -28,82 +30,130 @@
 #define ZIPL_IMAGE_START                0x009000UL
 #define IPL_PSW_MASK                    (PSW_MASK_32 | PSW_MASK_64)
 
-#define TYPE_S390_IPL "s390-ipl"
-#define S390_IPL(obj) \
-    OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
-#if 0
-#define S390_IPL_CLASS(klass) \
-    OBJECT_CLASS_CHECK(S390IPLState, (klass), TYPE_S390_IPL)
-#define S390_IPL_GET_CLASS(obj) \
-    OBJECT_GET_CLASS(S390IPLState, (obj), TYPE_S390_IPL)
-#endif
+static bool iplb_extended_needed(void *opaque)
+{
+    S390IPLState *ipl = S390_IPL(object_resolve_path(TYPE_S390_IPL, NULL));
 
-typedef struct S390IPLClass {
-    /*< private >*/
-    SysBusDeviceClass parent_class;
-    /*< public >*/
+    return ipl->iplbext_migration;
+}
 
-    void (*parent_reset) (SysBusDevice *dev);
-} S390IPLClass;
+static const VMStateDescription vmstate_iplb_extended = {
+    .name = "ipl/iplb_extended",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .needed = iplb_extended_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8_ARRAY(reserved_ext, IplParameterBlock, 4096 - 200),
+        VMSTATE_END_OF_LIST()
+    }
+};
 
-typedef struct S390IPLState {
-    /*< private >*/
-    SysBusDevice parent_obj;
-    uint64_t start_addr;
+static const VMStateDescription vmstate_iplb = {
+    .name = "ipl/iplb",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8_ARRAY(reserved1, IplParameterBlock, 110),
+        VMSTATE_UINT16(devno, IplParameterBlock),
+        VMSTATE_UINT8_ARRAY(reserved2, IplParameterBlock, 88),
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_iplb_extended,
+        NULL
+    }
+};
 
-    /*< public >*/
-    char *kernel;
-    char *initrd;
-    char *cmdline;
-    char *firmware;
-} S390IPLState;
+static const VMStateDescription vmstate_ipl = {
+    .name = "ipl",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(compat_start_addr, S390IPLState),
+        VMSTATE_UINT64(compat_bios_start_addr, S390IPLState),
+        VMSTATE_STRUCT(iplb, S390IPLState, 0, vmstate_iplb, IplParameterBlock),
+        VMSTATE_BOOL(iplb_valid, S390IPLState),
+        VMSTATE_UINT8(cssid, S390IPLState),
+        VMSTATE_UINT8(ssid, S390IPLState),
+        VMSTATE_UINT16(devno, S390IPLState),
+        VMSTATE_END_OF_LIST()
+     }
+};
 
+static S390IPLState *get_ipl_device(void)
+{
+    return S390_IPL(object_resolve_path_type("", TYPE_S390_IPL, NULL));
+}
 
-static int s390_ipl_init(SysBusDevice *dev)
+static uint64_t bios_translate_addr(void *opaque, uint64_t srcaddr)
+{
+    uint64_t dstaddr = *(uint64_t *) opaque;
+    /*
+     * Assuming that our s390-ccw.img was linked for starting at address 0,
+     * we can simply add the destination address for the final location
+     */
+    return srcaddr + dstaddr;
+}
+
+static void s390_ipl_realize(DeviceState *dev, Error **errp)
 {
     S390IPLState *ipl = S390_IPL(dev);
+    uint64_t pentry = KERN_IMAGE_START;
     int kernel_size;
+    Error *err = NULL;
 
-    if (!ipl->kernel) {
-        int bios_size;
-        char *bios_filename;
+    int bios_size;
+    char *bios_filename;
 
-        /* Load zipl bootloader */
+    /*
+     * Always load the bios if it was enforced,
+     * even if an external kernel has been defined.
+     */
+    if (!ipl->kernel || ipl->enforce_bios) {
+        uint64_t fwbase = (MIN(ram_size, 0x80000000U) - 0x200000) & ~0xffffUL;
+
         if (bios_name == NULL) {
             bios_name = ipl->firmware;
         }
 
         bios_filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
         if (bios_filename == NULL) {
-            hw_error("could not find stage1 bootloader\n");
+            error_setg(&err, "could not find stage1 bootloader");
+            goto error;
         }
 
-        bios_size = load_elf(bios_filename, NULL, NULL, &ipl->start_addr, NULL,
-                             NULL, 1, ELF_MACHINE, 0);
-        if (bios_size < 0) {
+        bios_size = load_elf(bios_filename, bios_translate_addr, &fwbase,
+                             &ipl->bios_start_addr, NULL, NULL, 1,
+                             EM_S390, 0, 0);
+        if (bios_size > 0) {
+            /* Adjust ELF start address to final location */
+            ipl->bios_start_addr += fwbase;
+        } else {
+            /* Try to load non-ELF file */
             bios_size = load_image_targphys(bios_filename, ZIPL_IMAGE_START,
                                             4096);
-            ipl->start_addr = ZIPL_IMAGE_START;
-            if (bios_size > 4096) {
-                hw_error("stage1 bootloader is > 4k\n");
-            }
+            ipl->bios_start_addr = ZIPL_IMAGE_START;
         }
         g_free(bios_filename);
 
         if (bios_size == -1) {
-            hw_error("could not load bootloader '%s'\n", bios_name);
+            error_setg(&err, "could not load bootloader '%s'", bios_name);
+            goto error;
         }
-        return 0;
-    } else {
-        uint64_t pentry = KERN_IMAGE_START;
+
+        /* default boot target is the bios */
+        ipl->start_addr = ipl->bios_start_addr;
+    }
+
+    if (ipl->kernel) {
         kernel_size = load_elf(ipl->kernel, NULL, NULL, &pentry, NULL,
-                               NULL, 1, ELF_MACHINE, 0);
+                               NULL, 1, EM_S390, 0, 0);
         if (kernel_size < 0) {
             kernel_size = load_image_targphys(ipl->kernel, 0, ram_size);
         }
         if (kernel_size < 0) {
-            fprintf(stderr, "could not load kernel '%s'\n", ipl->kernel);
-            return -1;
+            error_setg(&err, "could not load kernel '%s'", ipl->kernel);
+            goto error;
         }
         /*
          * Is it a Linux kernel (starting at 0x10000)? If yes, we fill in the
@@ -118,28 +168,40 @@
         } else {
             ipl->start_addr = pentry;
         }
-    }
-    if (ipl->initrd) {
-        ram_addr_t initrd_offset;
-        int initrd_size;
 
-        initrd_offset = INITRD_START;
-        while (kernel_size + 0x100000 > initrd_offset) {
-            initrd_offset += 0x100000;
+        if (ipl->initrd) {
+            ram_addr_t initrd_offset;
+            int initrd_size;
+
+            initrd_offset = INITRD_START;
+            while (kernel_size + 0x100000 > initrd_offset) {
+                initrd_offset += 0x100000;
+            }
+            initrd_size = load_image_targphys(ipl->initrd, initrd_offset,
+                                              ram_size - initrd_offset);
+            if (initrd_size == -1) {
+                error_setg(&err, "could not load initrd '%s'", ipl->initrd);
+                goto error;
+            }
+
+            /*
+             * we have to overwrite values in the kernel image,
+             * which are "rom"
+             */
+            stq_p(rom_ptr(INITRD_PARM_START), initrd_offset);
+            stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
         }
-        initrd_size = load_image_targphys(ipl->initrd, initrd_offset,
-                                          ram_size - initrd_offset);
-        if (initrd_size == -1) {
-            fprintf(stderr, "qemu: could not load initrd '%s'\n", ipl->initrd);
-            exit(1);
-        }
-
-        /* we have to overwrite values in the kernel image, which are "rom" */
-        stq_p(rom_ptr(INITRD_PARM_START), initrd_offset);
-        stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
     }
-
-    return 0;
+    /*
+     * Don't ever use the migrated values, they could come from a different
+     * BIOS and therefore don't work. But still migrate the values, so
+     * QEMUs relying on it don't break.
+     */
+    ipl->compat_start_addr = ipl->start_addr;
+    ipl->compat_bios_start_addr = ipl->bios_start_addr;
+    qemu_register_reset(qdev_reset_all_fn, dev);
+error:
+    error_propagate(errp, err);
 }
 
 static Property s390_ipl_properties[] = {
@@ -147,52 +209,122 @@
     DEFINE_PROP_STRING("initrd", S390IPLState, initrd),
     DEFINE_PROP_STRING("cmdline", S390IPLState, cmdline),
     DEFINE_PROP_STRING("firmware", S390IPLState, firmware),
+    DEFINE_PROP_BOOL("enforce_bios", S390IPLState, enforce_bios, false),
+    DEFINE_PROP_BOOL("iplbext_migration", S390IPLState, iplbext_migration,
+                     true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static bool s390_gen_initial_iplb(S390IPLState *ipl)
+{
+    DeviceState *dev_st;
+
+    dev_st = get_boot_device(0);
+    if (dev_st) {
+        VirtioCcwDevice *virtio_ccw_dev = (VirtioCcwDevice *)
+            object_dynamic_cast(OBJECT(qdev_get_parent_bus(dev_st)->parent),
+                TYPE_VIRTIO_CCW_DEVICE);
+        SCSIDevice *sd = (SCSIDevice *) object_dynamic_cast(OBJECT(dev_st),
+                                                            TYPE_SCSI_DEVICE);
+        if (virtio_ccw_dev) {
+            CcwDevice *ccw_dev = CCW_DEVICE(virtio_ccw_dev);
+
+            ipl->iplb.len = cpu_to_be32(S390_IPLB_MIN_CCW_LEN);
+            ipl->iplb.blk0_len =
+                cpu_to_be32(S390_IPLB_MIN_CCW_LEN - S390_IPLB_HEADER_LEN);
+            ipl->iplb.pbt = S390_IPL_TYPE_CCW;
+            ipl->iplb.ccw.devno = cpu_to_be16(ccw_dev->sch->devno);
+            ipl->iplb.ccw.ssid = ccw_dev->sch->ssid & 3;
+            return true;
+        } else if (sd) {
+            SCSIBus *bus = scsi_bus_from_device(sd);
+            VirtIOSCSI *vdev = container_of(bus, VirtIOSCSI, bus);
+            VirtIOSCSICcw *scsi_ccw = container_of(vdev, VirtIOSCSICcw, vdev);
+            CcwDevice *ccw_dev = CCW_DEVICE(scsi_ccw);
+
+            ipl->iplb.len = cpu_to_be32(S390_IPLB_MIN_QEMU_SCSI_LEN);
+            ipl->iplb.blk0_len =
+                cpu_to_be32(S390_IPLB_MIN_QEMU_SCSI_LEN - S390_IPLB_HEADER_LEN);
+            ipl->iplb.pbt = S390_IPL_TYPE_QEMU_SCSI;
+            ipl->iplb.scsi.lun = cpu_to_be32(sd->lun);
+            ipl->iplb.scsi.target = cpu_to_be16(sd->id);
+            ipl->iplb.scsi.channel = cpu_to_be16(sd->channel);
+            ipl->iplb.scsi.devno = cpu_to_be16(ccw_dev->sch->devno);
+            ipl->iplb.scsi.ssid = ccw_dev->sch->ssid & 3;
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void s390_ipl_update_diag308(IplParameterBlock *iplb)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    ipl->iplb = *iplb;
+    ipl->iplb_valid = true;
+}
+
+IplParameterBlock *s390_ipl_get_iplb(void)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    if (!ipl->iplb_valid) {
+        return NULL;
+    }
+    return &ipl->iplb;
+}
+
+void s390_reipl_request(void)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    ipl->reipl_requested = true;
+    qemu_system_reset_request();
+}
+
+void s390_ipl_prepare_cpu(S390CPU *cpu)
+{
+    S390IPLState *ipl = get_ipl_device();
+
+    cpu->env.psw.addr = ipl->start_addr;
+    cpu->env.psw.mask = IPL_PSW_MASK;
+
+    if (!ipl->kernel || ipl->iplb_valid) {
+        cpu->env.psw.addr = ipl->bios_start_addr;
+        if (!ipl->iplb_valid) {
+            ipl->iplb_valid = s390_gen_initial_iplb(ipl);
+        }
+    }
+}
+
 static void s390_ipl_reset(DeviceState *dev)
 {
     S390IPLState *ipl = S390_IPL(dev);
-    S390CPU *cpu = S390_CPU(qemu_get_cpu(0));
-    CPUS390XState *env = &cpu->env;
 
-    env->psw.addr = ipl->start_addr;
-    env->psw.mask = IPL_PSW_MASK;
-
-    if (!ipl->kernel) {
-        /* Tell firmware, if there is a preferred boot device */
-        env->regs[7] = -1;
-        DeviceState *dev_st = get_boot_device(0);
-        if (dev_st) {
-            VirtioCcwDevice *ccw_dev = (VirtioCcwDevice *) object_dynamic_cast(
-                OBJECT(qdev_get_parent_bus(dev_st)->parent),
-                TYPE_VIRTIO_CCW_DEVICE);
-
-            if (ccw_dev) {
-                env->regs[7] = ccw_dev->sch->cssid << 24 |
-                               ccw_dev->sch->ssid << 16 |
-                               ccw_dev->sch->devno;
-            }
-        }
+    if (!ipl->reipl_requested) {
+        ipl->iplb_valid = false;
+        memset(&ipl->iplb, 0, sizeof(IplParameterBlock));
     }
-
-    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
+    ipl->reipl_requested = false;
 }
 
 static void s390_ipl_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = s390_ipl_init;
+    dc->realize = s390_ipl_realize;
     dc->props = s390_ipl_properties;
     dc->reset = s390_ipl_reset;
+    dc->vmsd = &vmstate_ipl;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo s390_ipl_info = {
     .class_init = s390_ipl_class_init,
-    .parent = TYPE_SYS_BUS_DEVICE,
-    .name  = "s390-ipl",
+    .parent = TYPE_DEVICE,
+    .name  = TYPE_S390_IPL,
     .instance_size  = sizeof(S390IPLState),
 };
 
diff --git a/hw/s390x/ipl.h b/hw/s390x/ipl.h
new file mode 100644
index 0000000..c891095
--- /dev/null
+++ b/hw/s390x/ipl.h
@@ -0,0 +1,142 @@
+/*
+ * s390 IPL device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Zhang Fan <bjfanzh@cn.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_IPL_H
+#define HW_S390_IPL_H
+
+#include "hw/qdev.h"
+#include "cpu.h"
+
+struct IplBlockCcw {
+    uint8_t  reserved0[85];
+    uint8_t  ssid;
+    uint16_t devno;
+    uint8_t  vm_flags;
+    uint8_t  reserved3[3];
+    uint32_t vm_parm_len;
+    uint8_t  nss_name[8];
+    uint8_t  vm_parm[64];
+    uint8_t  reserved4[8];
+} QEMU_PACKED;
+typedef struct IplBlockCcw IplBlockCcw;
+
+struct IplBlockFcp {
+    uint8_t  reserved1[305 - 1];
+    uint8_t  opt;
+    uint8_t  reserved2[3];
+    uint16_t reserved3;
+    uint16_t devno;
+    uint8_t  reserved4[4];
+    uint64_t wwpn;
+    uint64_t lun;
+    uint32_t bootprog;
+    uint8_t  reserved5[12];
+    uint64_t br_lba;
+    uint32_t scp_data_len;
+    uint8_t  reserved6[260];
+    uint8_t  scp_data[];
+} QEMU_PACKED;
+typedef struct IplBlockFcp IplBlockFcp;
+
+struct IplBlockQemuScsi {
+    uint32_t lun;
+    uint16_t target;
+    uint16_t channel;
+    uint8_t  reserved0[77];
+    uint8_t  ssid;
+    uint16_t devno;
+} QEMU_PACKED;
+typedef struct IplBlockQemuScsi IplBlockQemuScsi;
+
+union IplParameterBlock {
+    struct {
+        uint32_t len;
+        uint8_t  reserved0[3];
+        uint8_t  version;
+        uint32_t blk0_len;
+        uint8_t  pbt;
+        uint8_t  flags;
+        uint16_t reserved01;
+        uint8_t  loadparm[8];
+        union {
+            IplBlockCcw ccw;
+            IplBlockFcp fcp;
+            IplBlockQemuScsi scsi;
+        };
+    } QEMU_PACKED;
+    struct {
+        uint8_t  reserved1[110];
+        uint16_t devno;
+        uint8_t  reserved2[88];
+        uint8_t  reserved_ext[4096 - 200];
+    } QEMU_PACKED;
+} QEMU_PACKED;
+typedef union IplParameterBlock IplParameterBlock;
+
+void s390_ipl_update_diag308(IplParameterBlock *iplb);
+void s390_ipl_prepare_cpu(S390CPU *cpu);
+IplParameterBlock *s390_ipl_get_iplb(void);
+void s390_reipl_request(void);
+
+#define TYPE_S390_IPL "s390-ipl"
+#define S390_IPL(obj) OBJECT_CHECK(S390IPLState, (obj), TYPE_S390_IPL)
+
+struct S390IPLState {
+    /*< private >*/
+    DeviceState parent_obj;
+    uint64_t start_addr;
+    uint64_t compat_start_addr;
+    uint64_t bios_start_addr;
+    uint64_t compat_bios_start_addr;
+    bool enforce_bios;
+    IplParameterBlock iplb;
+    bool iplb_valid;
+    bool reipl_requested;
+
+    /*< public >*/
+    char *kernel;
+    char *initrd;
+    char *cmdline;
+    char *firmware;
+    uint8_t cssid;
+    uint8_t ssid;
+    uint16_t devno;
+    bool iplbext_migration;
+};
+typedef struct S390IPLState S390IPLState;
+
+#define S390_IPL_TYPE_FCP 0x00
+#define S390_IPL_TYPE_CCW 0x02
+#define S390_IPL_TYPE_QEMU_SCSI 0xff
+
+#define S390_IPLB_HEADER_LEN 8
+#define S390_IPLB_MIN_CCW_LEN 200
+#define S390_IPLB_MIN_FCP_LEN 384
+#define S390_IPLB_MIN_QEMU_SCSI_LEN 200
+
+static inline bool iplb_valid_len(IplParameterBlock *iplb)
+{
+    return be32_to_cpu(iplb->len) <= sizeof(IplParameterBlock);
+}
+
+static inline bool iplb_valid_ccw(IplParameterBlock *iplb)
+{
+    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_CCW_LEN &&
+           iplb->pbt == S390_IPL_TYPE_CCW;
+}
+
+static inline bool iplb_valid_fcp(IplParameterBlock *iplb)
+{
+    return be32_to_cpu(iplb->len) >= S390_IPLB_MIN_FCP_LEN &&
+           iplb->pbt == S390_IPL_TYPE_FCP;
+}
+
+#endif
diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
new file mode 100644
index 0000000..9c1c04e
--- /dev/null
+++ b/hw/s390x/s390-pci-bus.c
@@ -0,0 +1,961 @@
+/*
+ * s390 PCI BUS
+ *
+ * Copyright 2014 IBM Corp.
+ * Author(s): Frank Blaschka <frank.blaschka@de.ibm.com>
+ *            Hong Bo Li <lihbbj@cn.ibm.com>
+ *            Yi Min Zhao <zyimin@cn.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qapi/visitor.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "s390-pci-bus.h"
+#include "s390-pci-inst.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/pci/msi.h"
+#include "qemu/error-report.h"
+
+/* #define DEBUG_S390PCI_BUS */
+#ifdef DEBUG_S390PCI_BUS
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "S390pci-bus: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+static S390pciState *s390_get_phb(void)
+{
+    static S390pciState *phb;
+
+    if (!phb) {
+        phb = S390_PCI_HOST_BRIDGE(
+            object_resolve_path(TYPE_S390_PCI_HOST_BRIDGE, NULL));
+        assert(phb != NULL);
+    }
+
+    return phb;
+}
+
+int chsc_sei_nt2_get_event(void *res)
+{
+    ChscSeiNt2Res *nt2_res = (ChscSeiNt2Res *)res;
+    PciCcdfAvail *accdf;
+    PciCcdfErr *eccdf;
+    int rc = 1;
+    SeiContainer *sei_cont;
+    S390pciState *s = s390_get_phb();
+
+    sei_cont = QTAILQ_FIRST(&s->pending_sei);
+    if (sei_cont) {
+        QTAILQ_REMOVE(&s->pending_sei, sei_cont, link);
+        nt2_res->nt = 2;
+        nt2_res->cc = sei_cont->cc;
+        nt2_res->length = cpu_to_be16(sizeof(ChscSeiNt2Res));
+        switch (sei_cont->cc) {
+        case 1: /* error event */
+            eccdf = (PciCcdfErr *)nt2_res->ccdf;
+            eccdf->fid = cpu_to_be32(sei_cont->fid);
+            eccdf->fh = cpu_to_be32(sei_cont->fh);
+            eccdf->e = cpu_to_be32(sei_cont->e);
+            eccdf->faddr = cpu_to_be64(sei_cont->faddr);
+            eccdf->pec = cpu_to_be16(sei_cont->pec);
+            break;
+        case 2: /* availability event */
+            accdf = (PciCcdfAvail *)nt2_res->ccdf;
+            accdf->fid = cpu_to_be32(sei_cont->fid);
+            accdf->fh = cpu_to_be32(sei_cont->fh);
+            accdf->pec = cpu_to_be16(sei_cont->pec);
+            break;
+        default:
+            abort();
+        }
+        g_free(sei_cont);
+        rc = 0;
+    }
+
+    return rc;
+}
+
+int chsc_sei_nt2_have_event(void)
+{
+    S390pciState *s = s390_get_phb();
+
+    return !QTAILQ_EMPTY(&s->pending_sei);
+}
+
+S390PCIBusDevice *s390_pci_find_next_avail_dev(S390PCIBusDevice *pbdev)
+{
+    int idx = 0;
+    S390PCIBusDevice *dev = NULL;
+    S390pciState *s = s390_get_phb();
+
+    if (pbdev) {
+        idx = (pbdev->fh & FH_MASK_INDEX) + 1;
+    }
+
+    for (; idx < PCI_SLOT_MAX; idx++) {
+        dev = s->pbdev[idx];
+        if (dev && dev->state != ZPCI_FS_RESERVED) {
+            return dev;
+        }
+    }
+
+    return NULL;
+}
+
+S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid)
+{
+    S390PCIBusDevice *pbdev;
+    int i;
+    S390pciState *s = s390_get_phb();
+
+    for (i = 0; i < PCI_SLOT_MAX; i++) {
+        pbdev = s->pbdev[i];
+        if (pbdev && pbdev->fid == fid) {
+            return pbdev;
+        }
+    }
+
+    return NULL;
+}
+
+void s390_pci_sclp_configure(SCCB *sccb)
+{
+    PciCfgSccb *psccb = (PciCfgSccb *)sccb;
+    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_fid(be32_to_cpu(psccb->aid));
+    uint16_t rc;
+
+    if (be16_to_cpu(sccb->h.length) < 16) {
+        rc = SCLP_RC_INSUFFICIENT_SCCB_LENGTH;
+        goto out;
+    }
+
+    if (!pbdev) {
+        DPRINTF("sclp config no dev found\n");
+        rc = SCLP_RC_ADAPTER_ID_NOT_RECOGNIZED;
+        goto out;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+        rc = SCLP_RC_ADAPTER_IN_RESERVED_STATE;
+        break;
+    case ZPCI_FS_STANDBY:
+        pbdev->state = ZPCI_FS_DISABLED;
+        rc = SCLP_RC_NORMAL_COMPLETION;
+        break;
+    default:
+        rc = SCLP_RC_NO_ACTION_REQUIRED;
+    }
+out:
+    psccb->header.response_code = cpu_to_be16(rc);
+}
+
+void s390_pci_sclp_deconfigure(SCCB *sccb)
+{
+    PciCfgSccb *psccb = (PciCfgSccb *)sccb;
+    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_fid(be32_to_cpu(psccb->aid));
+    uint16_t rc;
+
+    if (be16_to_cpu(sccb->h.length) < 16) {
+        rc = SCLP_RC_INSUFFICIENT_SCCB_LENGTH;
+        goto out;
+    }
+
+    if (!pbdev) {
+        DPRINTF("sclp deconfig no dev found\n");
+        rc = SCLP_RC_ADAPTER_ID_NOT_RECOGNIZED;
+        goto out;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+        rc = SCLP_RC_ADAPTER_IN_RESERVED_STATE;
+        break;
+    case ZPCI_FS_STANDBY:
+        rc = SCLP_RC_NO_ACTION_REQUIRED;
+        break;
+    default:
+        if (pbdev->summary_ind) {
+            pci_dereg_irqs(pbdev);
+        }
+        if (pbdev->iommu_enabled) {
+            pci_dereg_ioat(pbdev);
+        }
+        pbdev->state = ZPCI_FS_STANDBY;
+        rc = SCLP_RC_NORMAL_COMPLETION;
+
+        if (pbdev->release_timer) {
+            qdev_unplug(DEVICE(pbdev->pdev), NULL);
+        }
+    }
+out:
+    psccb->header.response_code = cpu_to_be16(rc);
+}
+
+static S390PCIBusDevice *s390_pci_find_dev_by_uid(uint16_t uid)
+{
+    int i;
+    S390PCIBusDevice *pbdev;
+    S390pciState *s = s390_get_phb();
+
+    for (i = 0; i < PCI_SLOT_MAX; i++) {
+        pbdev = s->pbdev[i];
+        if (!pbdev) {
+            continue;
+        }
+
+        if (pbdev->uid == uid) {
+            return pbdev;
+        }
+    }
+
+    return NULL;
+}
+
+static S390PCIBusDevice *s390_pci_find_dev_by_target(const char *target)
+{
+    int i;
+    S390PCIBusDevice *pbdev;
+    S390pciState *s = s390_get_phb();
+
+    if (!target) {
+        return NULL;
+    }
+
+    for (i = 0; i < PCI_SLOT_MAX; i++) {
+        pbdev = s->pbdev[i];
+        if (!pbdev) {
+            continue;
+        }
+
+        if (!strcmp(pbdev->target, target)) {
+            return pbdev;
+        }
+    }
+
+    return NULL;
+}
+
+S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
+{
+    S390pciState *s = s390_get_phb();
+
+    return s->pbdev[idx & FH_MASK_INDEX];
+}
+
+S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh)
+{
+    S390pciState *s = s390_get_phb();
+    S390PCIBusDevice *pbdev;
+
+    pbdev = s->pbdev[fh & FH_MASK_INDEX];
+    if (pbdev && pbdev->fh == fh) {
+        return pbdev;
+    }
+
+    return NULL;
+}
+
+static void s390_pci_generate_event(uint8_t cc, uint16_t pec, uint32_t fh,
+                                    uint32_t fid, uint64_t faddr, uint32_t e)
+{
+    SeiContainer *sei_cont;
+    S390pciState *s = s390_get_phb();
+
+    sei_cont = g_malloc0(sizeof(SeiContainer));
+    sei_cont->fh = fh;
+    sei_cont->fid = fid;
+    sei_cont->cc = cc;
+    sei_cont->pec = pec;
+    sei_cont->faddr = faddr;
+    sei_cont->e = e;
+
+    QTAILQ_INSERT_TAIL(&s->pending_sei, sei_cont, link);
+    css_generate_css_crws(0);
+}
+
+static void s390_pci_generate_plug_event(uint16_t pec, uint32_t fh,
+                                         uint32_t fid)
+{
+    s390_pci_generate_event(2, pec, fh, fid, 0, 0);
+}
+
+void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid,
+                                   uint64_t faddr, uint32_t e)
+{
+    s390_pci_generate_event(1, pec, fh, fid, faddr, e);
+}
+
+static void s390_pci_set_irq(void *opaque, int irq, int level)
+{
+    /* nothing to do */
+}
+
+static int s390_pci_map_irq(PCIDevice *pci_dev, int irq_num)
+{
+    /* nothing to do */
+    return 0;
+}
+
+static uint64_t s390_pci_get_table_origin(uint64_t iota)
+{
+    return iota & ~ZPCI_IOTA_RTTO_FLAG;
+}
+
+static unsigned int calc_rtx(dma_addr_t ptr)
+{
+    return ((unsigned long) ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK;
+}
+
+static unsigned int calc_sx(dma_addr_t ptr)
+{
+    return ((unsigned long) ptr >> ZPCI_ST_SHIFT) & ZPCI_INDEX_MASK;
+}
+
+static unsigned int calc_px(dma_addr_t ptr)
+{
+    return ((unsigned long) ptr >> PAGE_SHIFT) & ZPCI_PT_MASK;
+}
+
+static uint64_t get_rt_sto(uint64_t entry)
+{
+    return ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_RTX)
+                ? (entry & ZPCI_RTE_ADDR_MASK)
+                : 0;
+}
+
+static uint64_t get_st_pto(uint64_t entry)
+{
+    return ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_SX)
+            ? (entry & ZPCI_STE_ADDR_MASK)
+            : 0;
+}
+
+static uint64_t s390_guest_io_table_walk(uint64_t guest_iota,
+                                  uint64_t guest_dma_address)
+{
+    uint64_t sto_a, pto_a, px_a;
+    uint64_t sto, pto, pte;
+    uint32_t rtx, sx, px;
+
+    rtx = calc_rtx(guest_dma_address);
+    sx = calc_sx(guest_dma_address);
+    px = calc_px(guest_dma_address);
+
+    sto_a = guest_iota + rtx * sizeof(uint64_t);
+    sto = address_space_ldq(&address_space_memory, sto_a,
+                            MEMTXATTRS_UNSPECIFIED, NULL);
+    sto = get_rt_sto(sto);
+    if (!sto) {
+        pte = 0;
+        goto out;
+    }
+
+    pto_a = sto + sx * sizeof(uint64_t);
+    pto = address_space_ldq(&address_space_memory, pto_a,
+                            MEMTXATTRS_UNSPECIFIED, NULL);
+    pto = get_st_pto(pto);
+    if (!pto) {
+        pte = 0;
+        goto out;
+    }
+
+    px_a = pto + px * sizeof(uint64_t);
+    pte = address_space_ldq(&address_space_memory, px_a,
+                            MEMTXATTRS_UNSPECIFIED, NULL);
+
+out:
+    return pte;
+}
+
+static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
+                                          bool is_write)
+{
+    uint64_t pte;
+    uint32_t flags;
+    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, iommu_mr);
+    S390pciState *s;
+    IOMMUTLBEntry ret = {
+        .target_as = &address_space_memory,
+        .iova = 0,
+        .translated_addr = 0,
+        .addr_mask = ~(hwaddr)0,
+        .perm = IOMMU_NONE,
+    };
+
+    switch (pbdev->state) {
+    case ZPCI_FS_ENABLED:
+    case ZPCI_FS_BLOCKED:
+        if (!pbdev->iommu_enabled) {
+            return ret;
+        }
+        break;
+    default:
+        return ret;
+    }
+
+    DPRINTF("iommu trans addr 0x%" PRIx64 "\n", addr);
+
+    s = S390_PCI_HOST_BRIDGE(pci_device_root_bus(pbdev->pdev)->qbus.parent);
+    /* s390 does not have an APIC mapped to main storage so we use
+     * a separate AddressSpace only for msix notifications
+     */
+    if (addr == ZPCI_MSI_ADDR) {
+        ret.target_as = &s->msix_notify_as;
+        ret.iova = addr;
+        ret.translated_addr = addr;
+        ret.addr_mask = 0xfff;
+        ret.perm = IOMMU_RW;
+        return ret;
+    }
+
+    if (addr < pbdev->pba || addr > pbdev->pal) {
+        return ret;
+    }
+
+    pte = s390_guest_io_table_walk(s390_pci_get_table_origin(pbdev->g_iota),
+                                   addr);
+    if (!pte) {
+        return ret;
+    }
+
+    flags = pte & ZPCI_PTE_FLAG_MASK;
+    ret.iova = addr;
+    ret.translated_addr = pte & ZPCI_PTE_ADDR_MASK;
+    ret.addr_mask = 0xfff;
+
+    if (flags & ZPCI_PTE_INVALID) {
+        ret.perm = IOMMU_NONE;
+    } else {
+        ret.perm = IOMMU_RW;
+    }
+
+    return ret;
+}
+
+static const MemoryRegionIOMMUOps s390_iommu_ops = {
+    .translate = s390_translate_iommu,
+};
+
+static AddressSpace *s390_pci_dma_iommu(PCIBus *bus, void *opaque, int devfn)
+{
+    S390pciState *s = opaque;
+
+    return &s->iommu[PCI_SLOT(devfn)]->as;
+}
+
+static uint8_t set_ind_atomic(uint64_t ind_loc, uint8_t to_be_set)
+{
+    uint8_t ind_old, ind_new;
+    hwaddr len = 1;
+    uint8_t *ind_addr;
+
+    ind_addr = cpu_physical_memory_map(ind_loc, &len, 1);
+    if (!ind_addr) {
+        s390_pci_generate_error_event(ERR_EVENT_AIRERR, 0, 0, 0, 0);
+        return -1;
+    }
+    do {
+        ind_old = *ind_addr;
+        ind_new = ind_old | to_be_set;
+    } while (atomic_cmpxchg(ind_addr, ind_old, ind_new) != ind_old);
+    cpu_physical_memory_unmap(ind_addr, len, 1, len);
+
+    return ind_old;
+}
+
+static void s390_msi_ctrl_write(void *opaque, hwaddr addr, uint64_t data,
+                                unsigned int size)
+{
+    S390PCIBusDevice *pbdev;
+    uint32_t io_int_word;
+    uint32_t idx = data >> ZPCI_MSI_VEC_BITS;
+    uint32_t vec = data & ZPCI_MSI_VEC_MASK;
+    uint64_t ind_bit;
+    uint32_t sum_bit;
+    uint32_t e = 0;
+
+    DPRINTF("write_msix data 0x%" PRIx64 " idx %d vec 0x%x\n", data, idx, vec);
+
+    pbdev = s390_pci_find_dev_by_idx(idx);
+    if (!pbdev) {
+        e |= (vec << ERR_EVENT_MVN_OFFSET);
+        s390_pci_generate_error_event(ERR_EVENT_NOMSI, idx, 0, addr, e);
+        return;
+    }
+
+    if (pbdev->state != ZPCI_FS_ENABLED) {
+        return;
+    }
+
+    ind_bit = pbdev->routes.adapter.ind_offset;
+    sum_bit = pbdev->routes.adapter.summary_offset;
+
+    set_ind_atomic(pbdev->routes.adapter.ind_addr + (ind_bit + vec) / 8,
+                   0x80 >> ((ind_bit + vec) % 8));
+    if (!set_ind_atomic(pbdev->routes.adapter.summary_addr + sum_bit / 8,
+                                       0x80 >> (sum_bit % 8))) {
+        io_int_word = (pbdev->isc << 27) | IO_INT_WORD_AI;
+        s390_io_interrupt(0, 0, 0, io_int_word);
+    }
+}
+
+static uint64_t s390_msi_ctrl_read(void *opaque, hwaddr addr, unsigned size)
+{
+    return 0xffffffff;
+}
+
+static const MemoryRegionOps s390_msi_ctrl_ops = {
+    .write = s390_msi_ctrl_write,
+    .read = s390_msi_ctrl_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
+{
+    memory_region_init_iommu(&pbdev->iommu_mr, OBJECT(&pbdev->iommu->mr),
+                             &s390_iommu_ops, "iommu-s390", pbdev->pal + 1);
+    memory_region_add_subregion(&pbdev->iommu->mr, 0, &pbdev->iommu_mr);
+    pbdev->iommu_enabled = true;
+}
+
+void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
+{
+    memory_region_del_subregion(&pbdev->iommu->mr, &pbdev->iommu_mr);
+    object_unparent(OBJECT(&pbdev->iommu_mr));
+    pbdev->iommu_enabled = false;
+}
+
+static void s390_pcihost_init_as(S390pciState *s)
+{
+    int i;
+    S390PCIIOMMU *iommu;
+
+    for (i = 0; i < PCI_SLOT_MAX; i++) {
+        iommu = g_malloc0(sizeof(S390PCIIOMMU));
+        memory_region_init(&iommu->mr, OBJECT(s),
+                           "iommu-root-s390", UINT64_MAX);
+        address_space_init(&iommu->as, &iommu->mr, "iommu-pci");
+
+        s->iommu[i] = iommu;
+    }
+
+    memory_region_init_io(&s->msix_notify_mr, OBJECT(s),
+                          &s390_msi_ctrl_ops, s, "msix-s390", UINT64_MAX);
+    address_space_init(&s->msix_notify_as, &s->msix_notify_mr, "msix-pci");
+}
+
+static int s390_pcihost_init(SysBusDevice *dev)
+{
+    PCIBus *b;
+    BusState *bus;
+    PCIHostState *phb = PCI_HOST_BRIDGE(dev);
+    S390pciState *s = S390_PCI_HOST_BRIDGE(dev);
+
+    DPRINTF("host_init\n");
+
+    b = pci_register_bus(DEVICE(dev), NULL,
+                         s390_pci_set_irq, s390_pci_map_irq, NULL,
+                         get_system_memory(), get_system_io(), 0, 64,
+                         TYPE_PCI_BUS);
+    s390_pcihost_init_as(s);
+    pci_setup_iommu(b, s390_pci_dma_iommu, s);
+
+    bus = BUS(b);
+    qbus_set_hotplug_handler(bus, DEVICE(dev), NULL);
+    phb->bus = b;
+
+    s->bus = S390_PCI_BUS(qbus_create(TYPE_S390_PCI_BUS, DEVICE(s), NULL));
+    qbus_set_hotplug_handler(BUS(s->bus), DEVICE(s), NULL);
+
+    QTAILQ_INIT(&s->pending_sei);
+    return 0;
+}
+
+static int s390_pcihost_setup_msix(S390PCIBusDevice *pbdev)
+{
+    uint8_t pos;
+    uint16_t ctrl;
+    uint32_t table, pba;
+
+    pos = pci_find_capability(pbdev->pdev, PCI_CAP_ID_MSIX);
+    if (!pos) {
+        pbdev->msix.available = false;
+        return 0;
+    }
+
+    ctrl = pci_host_config_read_common(pbdev->pdev, pos + PCI_MSIX_FLAGS,
+             pci_config_size(pbdev->pdev), sizeof(ctrl));
+    table = pci_host_config_read_common(pbdev->pdev, pos + PCI_MSIX_TABLE,
+             pci_config_size(pbdev->pdev), sizeof(table));
+    pba = pci_host_config_read_common(pbdev->pdev, pos + PCI_MSIX_PBA,
+             pci_config_size(pbdev->pdev), sizeof(pba));
+
+    pbdev->msix.table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
+    pbdev->msix.table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
+    pbdev->msix.pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
+    pbdev->msix.pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
+    pbdev->msix.entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+    pbdev->msix.available = true;
+    return 0;
+}
+
+static S390PCIBusDevice *s390_pci_device_new(const char *target)
+{
+    DeviceState *dev = NULL;
+    S390pciState *s = s390_get_phb();
+
+    dev = qdev_try_create(BUS(s->bus), TYPE_S390_PCI_DEVICE);
+    if (!dev) {
+        return NULL;
+    }
+
+    qdev_prop_set_string(dev, "target", target);
+    qdev_init_nofail(dev);
+
+    return S390_PCI_DEVICE(dev);
+}
+
+static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
+                                  DeviceState *dev, Error **errp)
+{
+    PCIDevice *pdev = NULL;
+    S390PCIBusDevice *pbdev = NULL;
+    S390pciState *s = s390_get_phb();
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+        pdev = PCI_DEVICE(dev);
+
+        if (!dev->id) {
+            /* In the case the PCI device does not define an id */
+            /* we generate one based on the PCI address         */
+            dev->id = g_strdup_printf("auto_%02x:%02x.%01x",
+                                      pci_bus_num(pdev->bus),
+                                      PCI_SLOT(pdev->devfn),
+                                      PCI_FUNC(pdev->devfn));
+        }
+
+        pbdev = s390_pci_find_dev_by_target(dev->id);
+        if (!pbdev) {
+            pbdev = s390_pci_device_new(dev->id);
+            if (!pbdev) {
+                error_setg(errp, "create zpci device failed");
+            }
+        }
+
+        if (object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
+            pbdev->fh |= FH_SHM_VFIO;
+        } else {
+            pbdev->fh |= FH_SHM_EMUL;
+        }
+
+        pbdev->pdev = pdev;
+        pbdev->iommu = s->iommu[PCI_SLOT(pdev->devfn)];
+        pbdev->state = ZPCI_FS_STANDBY;
+        s390_pcihost_setup_msix(pbdev);
+
+        if (dev->hotplugged) {
+            s390_pci_generate_plug_event(HP_EVENT_RESERVED_TO_STANDBY,
+                                         pbdev->fh, pbdev->fid);
+        }
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_S390_PCI_DEVICE)) {
+        int idx;
+
+        pbdev = S390_PCI_DEVICE(dev);
+        for (idx = 0; idx < PCI_SLOT_MAX; idx++) {
+            if (!s->pbdev[idx]) {
+                s->pbdev[idx] = pbdev;
+                pbdev->fh = idx;
+                return;
+            }
+        }
+
+        error_setg(errp, "no slot for plugging zpci device");
+    }
+}
+
+static void s390_pcihost_timer_cb(void *opaque)
+{
+    S390PCIBusDevice *pbdev = opaque;
+
+    if (pbdev->summary_ind) {
+        pci_dereg_irqs(pbdev);
+    }
+    if (pbdev->iommu_enabled) {
+        pci_dereg_ioat(pbdev);
+    }
+
+    pbdev->state = ZPCI_FS_STANDBY;
+    s390_pci_generate_plug_event(HP_EVENT_CONFIGURED_TO_STBRES,
+                                 pbdev->fh, pbdev->fid);
+    qdev_unplug(DEVICE(pbdev), NULL);
+}
+
+static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
+                                    DeviceState *dev, Error **errp)
+{
+    int i;
+    PCIDevice *pci_dev = NULL;
+    S390PCIBusDevice *pbdev = NULL;
+    S390pciState *s = s390_get_phb();
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+        pci_dev = PCI_DEVICE(dev);
+
+        for (i = 0 ; i < PCI_SLOT_MAX; i++) {
+            if (s->pbdev[i] && s->pbdev[i]->pdev == pci_dev) {
+                pbdev = s->pbdev[i];
+                break;
+            }
+        }
+
+        if (!pbdev) {
+            object_unparent(OBJECT(pci_dev));
+            return;
+        }
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_S390_PCI_DEVICE)) {
+        pbdev = S390_PCI_DEVICE(dev);
+        pci_dev = pbdev->pdev;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+        goto out;
+    case ZPCI_FS_STANDBY:
+        break;
+    default:
+        s390_pci_generate_plug_event(HP_EVENT_DECONFIGURE_REQUEST,
+                                     pbdev->fh, pbdev->fid);
+        pbdev->release_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                            s390_pcihost_timer_cb,
+                                            pbdev);
+        timer_mod(pbdev->release_timer,
+                  qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + HOT_UNPLUG_TIMEOUT);
+        return;
+    }
+
+    if (pbdev->release_timer && timer_pending(pbdev->release_timer)) {
+        timer_del(pbdev->release_timer);
+        timer_free(pbdev->release_timer);
+        pbdev->release_timer = NULL;
+    }
+
+    s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
+                                 pbdev->fh, pbdev->fid);
+    object_unparent(OBJECT(pci_dev));
+    pbdev->pdev = NULL;
+    pbdev->state = ZPCI_FS_RESERVED;
+out:
+    pbdev->fid = 0;
+    s->pbdev[pbdev->fh & FH_MASK_INDEX] = NULL;
+    object_unparent(OBJECT(pbdev));
+}
+
+static void s390_pcihost_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
+
+    dc->cannot_instantiate_with_device_add_yet = true;
+    k->init = s390_pcihost_init;
+    hc->plug = s390_pcihost_hot_plug;
+    hc->unplug = s390_pcihost_hot_unplug;
+    msi_nonbroken = true;
+}
+
+static const TypeInfo s390_pcihost_info = {
+    .name          = TYPE_S390_PCI_HOST_BRIDGE,
+    .parent        = TYPE_PCI_HOST_BRIDGE,
+    .instance_size = sizeof(S390pciState),
+    .class_init    = s390_pcihost_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_HOTPLUG_HANDLER },
+        { }
+    }
+};
+
+static const TypeInfo s390_pcibus_info = {
+    .name = TYPE_S390_PCI_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(S390PCIBus),
+};
+
+static uint16_t s390_pci_generate_uid(void)
+{
+    uint16_t uid = 0;
+
+    do {
+        uid++;
+        if (!s390_pci_find_dev_by_uid(uid)) {
+            return uid;
+        }
+    } while (uid < ZPCI_MAX_UID);
+
+    return UID_UNDEFINED;
+}
+
+static uint32_t s390_pci_generate_fid(Error **errp)
+{
+    uint32_t fid = 0;
+
+    while (fid <= ZPCI_MAX_FID) {
+        if (!s390_pci_find_dev_by_fid(fid)) {
+            return fid;
+        }
+
+        if (fid == ZPCI_MAX_FID) {
+            break;
+        }
+
+        fid++;
+    }
+
+    error_setg(errp, "no free fid could be found");
+    return 0;
+}
+
+static void s390_pci_device_realize(DeviceState *dev, Error **errp)
+{
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+
+    if (!zpci->target) {
+        error_setg(errp, "target must be defined");
+        return;
+    }
+
+    if (s390_pci_find_dev_by_target(zpci->target)) {
+        error_setg(errp, "target %s already has an associated zpci device",
+                   zpci->target);
+        return;
+    }
+
+    if (zpci->uid == UID_UNDEFINED) {
+        zpci->uid = s390_pci_generate_uid();
+        if (!zpci->uid) {
+            error_setg(errp, "no free uid could be found");
+            return;
+        }
+    } else if (s390_pci_find_dev_by_uid(zpci->uid)) {
+        error_setg(errp, "uid %u already in use", zpci->uid);
+        return;
+    }
+
+    if (!zpci->fid_defined) {
+        Error *local_error = NULL;
+
+        zpci->fid = s390_pci_generate_fid(&local_error);
+        if (local_error) {
+            error_propagate(errp, local_error);
+            return;
+        }
+    } else if (s390_pci_find_dev_by_fid(zpci->fid)) {
+        error_setg(errp, "fid %u already in use", zpci->fid);
+        return;
+    }
+
+    zpci->state = ZPCI_FS_RESERVED;
+}
+
+static void s390_pci_device_reset(DeviceState *dev)
+{
+    S390PCIBusDevice *pbdev = S390_PCI_DEVICE(dev);
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+        return;
+    case ZPCI_FS_STANDBY:
+        break;
+    default:
+        pbdev->fh &= ~FH_MASK_ENABLE;
+        pbdev->state = ZPCI_FS_DISABLED;
+        break;
+    }
+
+    if (pbdev->summary_ind) {
+        pci_dereg_irqs(pbdev);
+    }
+    if (pbdev->iommu_enabled) {
+        pci_dereg_ioat(pbdev);
+    }
+
+    pbdev->fmb_addr = 0;
+}
+
+static void s390_pci_get_fid(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
+{
+    Property *prop = opaque;
+    uint32_t *ptr = qdev_get_prop_ptr(DEVICE(obj), prop);
+
+    visit_type_uint32(v, name, ptr, errp);
+}
+
+static void s390_pci_set_fid(Object *obj, Visitor *v, const char *name,
+                         void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    S390PCIBusDevice *zpci = S390_PCI_DEVICE(obj);
+    Property *prop = opaque;
+    uint32_t *ptr = qdev_get_prop_ptr(dev, prop);
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
+    }
+
+    visit_type_uint32(v, name, ptr, errp);
+    zpci->fid_defined = true;
+}
+
+static PropertyInfo s390_pci_fid_propinfo = {
+    .name = "zpci_fid",
+    .get = s390_pci_get_fid,
+    .set = s390_pci_set_fid,
+};
+
+#define DEFINE_PROP_S390_PCI_FID(_n, _s, _f) \
+    DEFINE_PROP(_n, _s, _f, s390_pci_fid_propinfo, uint32_t)
+
+static Property s390_pci_device_properties[] = {
+    DEFINE_PROP_UINT16("uid", S390PCIBusDevice, uid, UID_UNDEFINED),
+    DEFINE_PROP_S390_PCI_FID("fid", S390PCIBusDevice, fid),
+    DEFINE_PROP_STRING("target", S390PCIBusDevice, target),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void s390_pci_device_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->desc = "zpci device";
+    dc->reset = s390_pci_device_reset;
+    dc->bus_type = TYPE_S390_PCI_BUS;
+    dc->realize = s390_pci_device_realize;
+    dc->props = s390_pci_device_properties;
+}
+
+static const TypeInfo s390_pci_device_info = {
+    .name = TYPE_S390_PCI_DEVICE,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(S390PCIBusDevice),
+    .class_init = s390_pci_device_class_init,
+};
+
+static void s390_pci_register_types(void)
+{
+    type_register_static(&s390_pcihost_info);
+    type_register_static(&s390_pcibus_info);
+    type_register_static(&s390_pci_device_info);
+}
+
+type_init(s390_pci_register_types)
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
new file mode 100644
index 0000000..4f564e0
--- /dev/null
+++ b/hw/s390x/s390-pci-bus.h
@@ -0,0 +1,318 @@
+/*
+ * s390 PCI BUS definitions
+ *
+ * Copyright 2014 IBM Corp.
+ * Author(s): Frank Blaschka <frank.blaschka@de.ibm.com>
+ *            Hong Bo Li <lihbbj@cn.ibm.com>
+ *            Yi Min Zhao <zyimin@cn.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_PCI_BUS_H
+#define HW_S390_PCI_BUS_H
+
+#include "hw/pci/pci.h"
+#include "hw/pci/pci_host.h"
+#include "hw/s390x/sclp.h"
+#include "hw/s390x/s390_flic.h"
+#include "hw/s390x/css.h"
+
+#define TYPE_S390_PCI_HOST_BRIDGE "s390-pcihost"
+#define TYPE_S390_PCI_BUS "s390-pcibus"
+#define TYPE_S390_PCI_DEVICE "zpci"
+#define FH_MASK_ENABLE   0x80000000
+#define FH_MASK_INSTANCE 0x7f000000
+#define FH_MASK_SHM      0x00ff0000
+#define FH_MASK_INDEX    0x0000001f
+#define FH_SHM_VFIO      0x00010000
+#define FH_SHM_EMUL      0x00020000
+#define S390_PCIPT_ADAPTER 2
+#define ZPCI_MAX_FID 0xffffffff
+#define ZPCI_MAX_UID 0xffff
+#define UID_UNDEFINED 0
+#define UID_CHECKING_ENABLED 0x01
+#define HOT_UNPLUG_TIMEOUT (NANOSECONDS_PER_SECOND * 60 * 5)
+
+#define S390_PCI_HOST_BRIDGE(obj) \
+    OBJECT_CHECK(S390pciState, (obj), TYPE_S390_PCI_HOST_BRIDGE)
+#define S390_PCI_BUS(obj) \
+    OBJECT_CHECK(S390PCIBus, (obj), TYPE_S390_PCI_BUS)
+#define S390_PCI_DEVICE(obj) \
+    OBJECT_CHECK(S390PCIBusDevice, (obj), TYPE_S390_PCI_DEVICE)
+
+#define HP_EVENT_TO_CONFIGURED        0x0301
+#define HP_EVENT_RESERVED_TO_STANDBY  0x0302
+#define HP_EVENT_DECONFIGURE_REQUEST  0x0303
+#define HP_EVENT_CONFIGURED_TO_STBRES 0x0304
+#define HP_EVENT_STANDBY_TO_RESERVED  0x0308
+
+#define ERR_EVENT_INVALAS 0x1
+#define ERR_EVENT_OORANGE 0x2
+#define ERR_EVENT_INVALTF 0x3
+#define ERR_EVENT_TPROTE  0x4
+#define ERR_EVENT_APROTE  0x5
+#define ERR_EVENT_KEYE    0x6
+#define ERR_EVENT_INVALTE 0x7
+#define ERR_EVENT_INVALTL 0x8
+#define ERR_EVENT_TT      0x9
+#define ERR_EVENT_INVALMS 0xa
+#define ERR_EVENT_SERR    0xb
+#define ERR_EVENT_NOMSI   0x10
+#define ERR_EVENT_INVALBV 0x11
+#define ERR_EVENT_AIBV    0x12
+#define ERR_EVENT_AIRERR  0x13
+#define ERR_EVENT_FMBA    0x2a
+#define ERR_EVENT_FMBUP   0x2b
+#define ERR_EVENT_FMBPRO  0x2c
+#define ERR_EVENT_CCONF   0x30
+#define ERR_EVENT_SERVAC  0x3a
+#define ERR_EVENT_PERMERR 0x3b
+
+#define ERR_EVENT_Q_BIT 0x2
+#define ERR_EVENT_MVN_OFFSET 16
+
+#define ZPCI_MSI_VEC_BITS 11
+#define ZPCI_MSI_VEC_MASK 0x7ff
+
+#define ZPCI_MSI_ADDR  0xfe00000000000000ULL
+#define ZPCI_SDMA_ADDR 0x100000000ULL
+#define ZPCI_EDMA_ADDR 0x1ffffffffffffffULL
+
+#define PAGE_SHIFT      12
+#define PAGE_MASK       (~(PAGE_SIZE-1))
+#define PAGE_DEFAULT_ACC        0
+#define PAGE_DEFAULT_KEY        (PAGE_DEFAULT_ACC << 4)
+
+/* I/O Translation Anchor (IOTA) */
+enum ZpciIoatDtype {
+    ZPCI_IOTA_STO = 0,
+    ZPCI_IOTA_RTTO = 1,
+    ZPCI_IOTA_RSTO = 2,
+    ZPCI_IOTA_RFTO = 3,
+    ZPCI_IOTA_PFAA = 4,
+    ZPCI_IOTA_IOPFAA = 5,
+    ZPCI_IOTA_IOPTO = 7
+};
+
+#define ZPCI_IOTA_IOT_ENABLED           0x800ULL
+#define ZPCI_IOTA_DT_ST                 (ZPCI_IOTA_STO  << 2)
+#define ZPCI_IOTA_DT_RT                 (ZPCI_IOTA_RTTO << 2)
+#define ZPCI_IOTA_DT_RS                 (ZPCI_IOTA_RSTO << 2)
+#define ZPCI_IOTA_DT_RF                 (ZPCI_IOTA_RFTO << 2)
+#define ZPCI_IOTA_DT_PF                 (ZPCI_IOTA_PFAA << 2)
+#define ZPCI_IOTA_FS_4K                 0
+#define ZPCI_IOTA_FS_1M                 1
+#define ZPCI_IOTA_FS_2G                 2
+#define ZPCI_KEY                        (PAGE_DEFAULT_KEY << 5)
+
+#define ZPCI_IOTA_STO_FLAG  (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_ST)
+#define ZPCI_IOTA_RTTO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_RT)
+#define ZPCI_IOTA_RSTO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_RS)
+#define ZPCI_IOTA_RFTO_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY | ZPCI_IOTA_DT_RF)
+#define ZPCI_IOTA_RFAA_FLAG (ZPCI_IOTA_IOT_ENABLED | ZPCI_KEY |\
+                             ZPCI_IOTA_DT_PF | ZPCI_IOTA_FS_2G)
+
+/* I/O Region and segment tables */
+#define ZPCI_INDEX_MASK         0x7ffULL
+
+#define ZPCI_TABLE_TYPE_MASK    0xc
+#define ZPCI_TABLE_TYPE_RFX     0xc
+#define ZPCI_TABLE_TYPE_RSX     0x8
+#define ZPCI_TABLE_TYPE_RTX     0x4
+#define ZPCI_TABLE_TYPE_SX      0x0
+
+#define ZPCI_TABLE_LEN_RFX      0x3
+#define ZPCI_TABLE_LEN_RSX      0x3
+#define ZPCI_TABLE_LEN_RTX      0x3
+
+#define ZPCI_TABLE_OFFSET_MASK  0xc0
+#define ZPCI_TABLE_SIZE         0x4000
+#define ZPCI_TABLE_ALIGN        ZPCI_TABLE_SIZE
+#define ZPCI_TABLE_ENTRY_SIZE   (sizeof(unsigned long))
+#define ZPCI_TABLE_ENTRIES      (ZPCI_TABLE_SIZE / ZPCI_TABLE_ENTRY_SIZE)
+
+#define ZPCI_TABLE_BITS         11
+#define ZPCI_PT_BITS            8
+#define ZPCI_ST_SHIFT           (ZPCI_PT_BITS + PAGE_SHIFT)
+#define ZPCI_RT_SHIFT           (ZPCI_ST_SHIFT + ZPCI_TABLE_BITS)
+
+#define ZPCI_RTE_FLAG_MASK      0x3fffULL
+#define ZPCI_RTE_ADDR_MASK      (~ZPCI_RTE_FLAG_MASK)
+#define ZPCI_STE_FLAG_MASK      0x7ffULL
+#define ZPCI_STE_ADDR_MASK      (~ZPCI_STE_FLAG_MASK)
+
+/* I/O Page tables */
+#define ZPCI_PTE_VALID_MASK             0x400
+#define ZPCI_PTE_INVALID                0x400
+#define ZPCI_PTE_VALID                  0x000
+#define ZPCI_PT_SIZE                    0x800
+#define ZPCI_PT_ALIGN                   ZPCI_PT_SIZE
+#define ZPCI_PT_ENTRIES                 (ZPCI_PT_SIZE / ZPCI_TABLE_ENTRY_SIZE)
+#define ZPCI_PT_MASK                    (ZPCI_PT_ENTRIES - 1)
+
+#define ZPCI_PTE_FLAG_MASK              0xfffULL
+#define ZPCI_PTE_ADDR_MASK              (~ZPCI_PTE_FLAG_MASK)
+
+/* Shared bits */
+#define ZPCI_TABLE_VALID                0x00
+#define ZPCI_TABLE_INVALID              0x20
+#define ZPCI_TABLE_PROTECTED            0x200
+#define ZPCI_TABLE_UNPROTECTED          0x000
+
+#define ZPCI_TABLE_VALID_MASK           0x20
+#define ZPCI_TABLE_PROT_MASK            0x200
+
+/* PCI Function States
+ *
+ * reserved: default; device has just been plugged or is in progress of being
+ *           unplugged
+ * standby: device is present but not configured; transition from any
+ *          configured state/to this state via sclp configure/deconfigure
+ *
+ * The following states make up the "configured" meta-state:
+ * disabled: device is configured but not enabled; transition between this
+ *           state and enabled via clp enable/disable
+ * enbaled: device is ready for use; transition to disabled via clp disable;
+ *          may enter an error state
+ * blocked: ignore all DMA and interrupts; transition back to enabled or from
+ *          error state via mpcifc
+ * error: an error occured; transition back to enabled via mpcifc
+ * permanent error: an unrecoverable error occured; transition to standby via
+ *                  sclp deconfigure
+ */
+typedef enum {
+    ZPCI_FS_RESERVED,
+    ZPCI_FS_STANDBY,
+    ZPCI_FS_DISABLED,
+    ZPCI_FS_ENABLED,
+    ZPCI_FS_BLOCKED,
+    ZPCI_FS_ERROR,
+    ZPCI_FS_PERMANENT_ERROR,
+} ZpciState;
+
+typedef struct SeiContainer {
+    QTAILQ_ENTRY(SeiContainer) link;
+    uint32_t fid;
+    uint32_t fh;
+    uint8_t cc;
+    uint16_t pec;
+    uint64_t faddr;
+    uint32_t e;
+} SeiContainer;
+
+typedef struct PciCcdfErr {
+    uint32_t reserved1;
+    uint32_t fh;
+    uint32_t fid;
+    uint32_t e;
+    uint64_t faddr;
+    uint32_t reserved3;
+    uint16_t reserved4;
+    uint16_t pec;
+} QEMU_PACKED PciCcdfErr;
+
+typedef struct PciCcdfAvail {
+    uint32_t reserved1;
+    uint32_t fh;
+    uint32_t fid;
+    uint32_t reserved2;
+    uint32_t reserved3;
+    uint32_t reserved4;
+    uint32_t reserved5;
+    uint16_t reserved6;
+    uint16_t pec;
+} QEMU_PACKED PciCcdfAvail;
+
+typedef struct ChscSeiNt2Res {
+    uint16_t length;
+    uint16_t code;
+    uint16_t reserved1;
+    uint8_t reserved2;
+    uint8_t nt;
+    uint8_t flags;
+    uint8_t reserved3;
+    uint8_t reserved4;
+    uint8_t cc;
+    uint32_t reserved5[13];
+    uint8_t ccdf[4016];
+} QEMU_PACKED ChscSeiNt2Res;
+
+typedef struct PciCfgSccb {
+    SCCBHeader header;
+    uint8_t atype;
+    uint8_t reserved1;
+    uint16_t reserved2;
+    uint32_t aid;
+} QEMU_PACKED PciCfgSccb;
+
+typedef struct S390MsixInfo {
+    bool available;
+    uint8_t table_bar;
+    uint8_t pba_bar;
+    uint16_t entries;
+    uint32_t table_offset;
+    uint32_t pba_offset;
+} S390MsixInfo;
+
+typedef struct S390PCIIOMMU {
+    AddressSpace as;
+    MemoryRegion mr;
+} S390PCIIOMMU;
+
+typedef struct S390PCIBusDevice {
+    DeviceState qdev;
+    PCIDevice *pdev;
+    ZpciState state;
+    bool iommu_enabled;
+    char *target;
+    uint16_t uid;
+    uint32_t fh;
+    uint32_t fid;
+    bool fid_defined;
+    uint64_t g_iota;
+    uint64_t pba;
+    uint64_t pal;
+    uint64_t fmb_addr;
+    uint8_t isc;
+    uint16_t noi;
+    uint8_t sum;
+    S390MsixInfo msix;
+    AdapterRoutes routes;
+    S390PCIIOMMU *iommu;
+    MemoryRegion iommu_mr;
+    IndAddr *summary_ind;
+    IndAddr *indicator;
+    QEMUTimer *release_timer;
+} S390PCIBusDevice;
+
+typedef struct S390PCIBus {
+    BusState qbus;
+} S390PCIBus;
+
+typedef struct S390pciState {
+    PCIHostState parent_obj;
+    S390PCIBus *bus;
+    S390PCIBusDevice *pbdev[PCI_SLOT_MAX];
+    S390PCIIOMMU *iommu[PCI_SLOT_MAX];
+    AddressSpace msix_notify_as;
+    MemoryRegion msix_notify_mr;
+    QTAILQ_HEAD(, SeiContainer) pending_sei;
+} S390pciState;
+
+int chsc_sei_nt2_get_event(void *res);
+int chsc_sei_nt2_have_event(void);
+void s390_pci_sclp_configure(SCCB *sccb);
+void s390_pci_sclp_deconfigure(SCCB *sccb);
+void s390_pci_iommu_enable(S390PCIBusDevice *pbdev);
+void s390_pci_iommu_disable(S390PCIBusDevice *pbdev);
+void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid,
+                                   uint64_t faddr, uint32_t e);
+S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
+S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
+S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);
+S390PCIBusDevice *s390_pci_find_next_avail_dev(S390PCIBusDevice *pbdev);
+
+#endif
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
new file mode 100644
index 0000000..f069b11
--- /dev/null
+++ b/hw/s390x/s390-pci-inst.c
@@ -0,0 +1,1033 @@
+/*
+ * s390 PCI instructions
+ *
+ * Copyright 2014 IBM Corp.
+ * Author(s): Frank Blaschka <frank.blaschka@de.ibm.com>
+ *            Hong Bo Li <lihbbj@cn.ibm.com>
+ *            Yi Min Zhao <zyimin@cn.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "s390-pci-inst.h"
+#include "s390-pci-bus.h"
+#include "exec/memory-internal.h"
+#include "qemu/error-report.h"
+
+/* #define DEBUG_S390PCI_INST */
+#ifdef DEBUG_S390PCI_INST
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "s390pci-inst: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+static void s390_set_status_code(CPUS390XState *env,
+                                 uint8_t r, uint64_t status_code)
+{
+    env->regs[r] &= ~0xff000000ULL;
+    env->regs[r] |= (status_code & 0xff) << 24;
+}
+
+static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
+{
+    S390PCIBusDevice *pbdev = NULL;
+    uint32_t res_code, initial_l2, g_l2;
+    int rc, i;
+    uint64_t resume_token;
+
+    rc = 0;
+    if (lduw_p(&rrb->request.hdr.len) != 32) {
+        res_code = CLP_RC_LEN;
+        rc = -EINVAL;
+        goto out;
+    }
+
+    if ((ldl_p(&rrb->request.fmt) & CLP_MASK_FMT) != 0) {
+        res_code = CLP_RC_FMT;
+        rc = -EINVAL;
+        goto out;
+    }
+
+    if ((ldl_p(&rrb->request.fmt) & ~CLP_MASK_FMT) != 0 ||
+        ldq_p(&rrb->request.reserved1) != 0) {
+        res_code = CLP_RC_RESNOT0;
+        rc = -EINVAL;
+        goto out;
+    }
+
+    resume_token = ldq_p(&rrb->request.resume_token);
+
+    if (resume_token) {
+        pbdev = s390_pci_find_dev_by_idx(resume_token);
+        if (!pbdev) {
+            res_code = CLP_RC_LISTPCI_BADRT;
+            rc = -EINVAL;
+            goto out;
+        }
+    } else {
+        pbdev = s390_pci_find_next_avail_dev(NULL);
+    }
+
+    if (lduw_p(&rrb->response.hdr.len) < 48) {
+        res_code = CLP_RC_8K;
+        rc = -EINVAL;
+        goto out;
+    }
+
+    initial_l2 = lduw_p(&rrb->response.hdr.len);
+    if ((initial_l2 - LIST_PCI_HDR_LEN) % sizeof(ClpFhListEntry)
+        != 0) {
+        res_code = CLP_RC_LEN;
+        rc = -EINVAL;
+        *cc = 3;
+        goto out;
+    }
+
+    stl_p(&rrb->response.fmt, 0);
+    stq_p(&rrb->response.reserved1, 0);
+    stl_p(&rrb->response.mdd, FH_MASK_SHM);
+    stw_p(&rrb->response.max_fn, PCI_MAX_FUNCTIONS);
+    rrb->response.flags = UID_CHECKING_ENABLED;
+    rrb->response.entry_size = sizeof(ClpFhListEntry);
+
+    i = 0;
+    g_l2 = LIST_PCI_HDR_LEN;
+    while (g_l2 < initial_l2 && pbdev) {
+        stw_p(&rrb->response.fh_list[i].device_id,
+            pci_get_word(pbdev->pdev->config + PCI_DEVICE_ID));
+        stw_p(&rrb->response.fh_list[i].vendor_id,
+            pci_get_word(pbdev->pdev->config + PCI_VENDOR_ID));
+        /* Ignore RESERVED devices. */
+        stl_p(&rrb->response.fh_list[i].config,
+            pbdev->state == ZPCI_FS_STANDBY ? 0 : 1 << 31);
+        stl_p(&rrb->response.fh_list[i].fid, pbdev->fid);
+        stl_p(&rrb->response.fh_list[i].fh, pbdev->fh);
+
+        g_l2 += sizeof(ClpFhListEntry);
+        /* Add endian check for DPRINTF? */
+        DPRINTF("g_l2 %d vendor id 0x%x device id 0x%x fid 0x%x fh 0x%x\n",
+                g_l2,
+                lduw_p(&rrb->response.fh_list[i].vendor_id),
+                lduw_p(&rrb->response.fh_list[i].device_id),
+                ldl_p(&rrb->response.fh_list[i].fid),
+                ldl_p(&rrb->response.fh_list[i].fh));
+        pbdev = s390_pci_find_next_avail_dev(pbdev);
+        i++;
+    }
+
+    if (!pbdev) {
+        resume_token = 0;
+    } else {
+        resume_token = pbdev->fh & FH_MASK_INDEX;
+    }
+    stq_p(&rrb->response.resume_token, resume_token);
+    stw_p(&rrb->response.hdr.len, g_l2);
+    stw_p(&rrb->response.hdr.rsp, CLP_RC_OK);
+out:
+    if (rc) {
+        DPRINTF("list pci failed rc 0x%x\n", rc);
+        stw_p(&rrb->response.hdr.rsp, res_code);
+    }
+    return rc;
+}
+
+int clp_service_call(S390CPU *cpu, uint8_t r2)
+{
+    ClpReqHdr *reqh;
+    ClpRspHdr *resh;
+    S390PCIBusDevice *pbdev;
+    uint32_t req_len;
+    uint32_t res_len;
+    uint8_t buffer[4096 * 2];
+    uint8_t cc = 0;
+    CPUS390XState *env = &cpu->env;
+    int i;
+
+    cpu_synchronize_state(CPU(cpu));
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 4);
+        return 0;
+    }
+
+    if (s390_cpu_virt_mem_read(cpu, env->regs[r2], r2, buffer, sizeof(*reqh))) {
+        return 0;
+    }
+    reqh = (ClpReqHdr *)buffer;
+    req_len = lduw_p(&reqh->len);
+    if (req_len < 16 || req_len > 8184 || (req_len % 8 != 0)) {
+        program_interrupt(env, PGM_OPERAND, 4);
+        return 0;
+    }
+
+    if (s390_cpu_virt_mem_read(cpu, env->regs[r2], r2, buffer,
+                               req_len + sizeof(*resh))) {
+        return 0;
+    }
+    resh = (ClpRspHdr *)(buffer + req_len);
+    res_len = lduw_p(&resh->len);
+    if (res_len < 8 || res_len > 8176 || (res_len % 8 != 0)) {
+        program_interrupt(env, PGM_OPERAND, 4);
+        return 0;
+    }
+    if ((req_len + res_len) > 8192) {
+        program_interrupt(env, PGM_OPERAND, 4);
+        return 0;
+    }
+
+    if (s390_cpu_virt_mem_read(cpu, env->regs[r2], r2, buffer,
+                               req_len + res_len)) {
+        return 0;
+    }
+
+    if (req_len != 32) {
+        stw_p(&resh->rsp, CLP_RC_LEN);
+        goto out;
+    }
+
+    switch (lduw_p(&reqh->cmd)) {
+    case CLP_LIST_PCI: {
+        ClpReqRspListPci *rrb = (ClpReqRspListPci *)buffer;
+        list_pci(rrb, &cc);
+        break;
+    }
+    case CLP_SET_PCI_FN: {
+        ClpReqSetPci *reqsetpci = (ClpReqSetPci *)reqh;
+        ClpRspSetPci *ressetpci = (ClpRspSetPci *)resh;
+
+        pbdev = s390_pci_find_dev_by_fh(ldl_p(&reqsetpci->fh));
+        if (!pbdev) {
+                stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FH);
+                goto out;
+        }
+
+        switch (reqsetpci->oc) {
+        case CLP_SET_ENABLE_PCI_FN:
+            switch (reqsetpci->ndas) {
+            case 0:
+                stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_DMAAS);
+                goto out;
+            case 1:
+                break;
+            default:
+                stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_RES);
+                goto out;
+            }
+
+            if (pbdev->fh & FH_MASK_ENABLE) {
+                stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FHOP);
+                goto out;
+            }
+
+            pbdev->fh |= FH_MASK_ENABLE;
+            pbdev->state = ZPCI_FS_ENABLED;
+            stl_p(&ressetpci->fh, pbdev->fh);
+            stw_p(&ressetpci->hdr.rsp, CLP_RC_OK);
+            break;
+        case CLP_SET_DISABLE_PCI_FN:
+            if (!(pbdev->fh & FH_MASK_ENABLE)) {
+                stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FHOP);
+                goto out;
+            }
+            device_reset(DEVICE(pbdev));
+            pbdev->fh &= ~FH_MASK_ENABLE;
+            pbdev->state = ZPCI_FS_DISABLED;
+            stl_p(&ressetpci->fh, pbdev->fh);
+            stw_p(&ressetpci->hdr.rsp, CLP_RC_OK);
+            break;
+        default:
+            DPRINTF("unknown set pci command\n");
+            stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FHOP);
+            break;
+        }
+        break;
+    }
+    case CLP_QUERY_PCI_FN: {
+        ClpReqQueryPci *reqquery = (ClpReqQueryPci *)reqh;
+        ClpRspQueryPci *resquery = (ClpRspQueryPci *)resh;
+
+        pbdev = s390_pci_find_dev_by_fh(ldl_p(&reqquery->fh));
+        if (!pbdev) {
+            DPRINTF("query pci no pci dev\n");
+            stw_p(&resquery->hdr.rsp, CLP_RC_SETPCIFN_FH);
+            goto out;
+        }
+
+        for (i = 0; i < PCI_BAR_COUNT; i++) {
+            uint32_t data = pci_get_long(pbdev->pdev->config +
+                PCI_BASE_ADDRESS_0 + (i * 4));
+
+            stl_p(&resquery->bar[i], data);
+            resquery->bar_size[i] = pbdev->pdev->io_regions[i].size ?
+                                    ctz64(pbdev->pdev->io_regions[i].size) : 0;
+            DPRINTF("bar %d addr 0x%x size 0x%" PRIx64 "barsize 0x%x\n", i,
+                    ldl_p(&resquery->bar[i]),
+                    pbdev->pdev->io_regions[i].size,
+                    resquery->bar_size[i]);
+        }
+
+        stq_p(&resquery->sdma, ZPCI_SDMA_ADDR);
+        stq_p(&resquery->edma, ZPCI_EDMA_ADDR);
+        stl_p(&resquery->fid, pbdev->fid);
+        stw_p(&resquery->pchid, 0);
+        stw_p(&resquery->ug, 1);
+        stl_p(&resquery->uid, pbdev->uid);
+        stw_p(&resquery->hdr.rsp, CLP_RC_OK);
+        break;
+    }
+    case CLP_QUERY_PCI_FNGRP: {
+        ClpRspQueryPciGrp *resgrp = (ClpRspQueryPciGrp *)resh;
+        resgrp->fr = 1;
+        stq_p(&resgrp->dasm, 0);
+        stq_p(&resgrp->msia, ZPCI_MSI_ADDR);
+        stw_p(&resgrp->mui, 0);
+        stw_p(&resgrp->i, 128);
+        resgrp->version = 0;
+
+        stw_p(&resgrp->hdr.rsp, CLP_RC_OK);
+        break;
+    }
+    default:
+        DPRINTF("unknown clp command\n");
+        stw_p(&resh->rsp, CLP_RC_CMD);
+        break;
+    }
+
+out:
+    if (s390_cpu_virt_mem_write(cpu, env->regs[r2], r2, buffer,
+                                req_len + res_len)) {
+        return 0;
+    }
+    setcc(cpu, cc);
+    return 0;
+}
+
+int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
+{
+    CPUS390XState *env = &cpu->env;
+    S390PCIBusDevice *pbdev;
+    uint64_t offset;
+    uint64_t data;
+    uint8_t len;
+    uint32_t fh;
+    uint8_t pcias;
+
+    cpu_synchronize_state(CPU(cpu));
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 4);
+        return 0;
+    }
+
+    if (r2 & 0x1) {
+        program_interrupt(env, PGM_SPECIFICATION, 4);
+        return 0;
+    }
+
+    fh = env->regs[r2] >> 32;
+    pcias = (env->regs[r2] >> 16) & 0xf;
+    len = env->regs[r2] & 0xf;
+    offset = env->regs[r2 + 1];
+
+    pbdev = s390_pci_find_dev_by_fh(fh);
+    if (!pbdev) {
+        DPRINTF("pcilg no pci dev\n");
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+    case ZPCI_FS_STANDBY:
+    case ZPCI_FS_DISABLED:
+    case ZPCI_FS_PERMANENT_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    case ZPCI_FS_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r2, ZPCI_PCI_ST_BLOCKED);
+        return 0;
+    default:
+        break;
+    }
+
+    if (pcias < 6) {
+        if ((8 - (offset & 0x7)) < len) {
+            program_interrupt(env, PGM_OPERAND, 4);
+            return 0;
+        }
+        MemoryRegion *mr = pbdev->pdev->io_regions[pcias].memory;
+        memory_region_dispatch_read(mr, offset, &data, len,
+                                    MEMTXATTRS_UNSPECIFIED);
+    } else if (pcias == 15) {
+        if ((4 - (offset & 0x3)) < len) {
+            program_interrupt(env, PGM_OPERAND, 4);
+            return 0;
+        }
+        data =  pci_host_config_read_common(
+                   pbdev->pdev, offset, pci_config_size(pbdev->pdev), len);
+
+        switch (len) {
+        case 1:
+            break;
+        case 2:
+            data = bswap16(data);
+            break;
+        case 4:
+            data = bswap32(data);
+            break;
+        case 8:
+            data = bswap64(data);
+            break;
+        default:
+            program_interrupt(env, PGM_OPERAND, 4);
+            return 0;
+        }
+    } else {
+        DPRINTF("invalid space\n");
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r2, ZPCI_PCI_ST_INVAL_AS);
+        return 0;
+    }
+
+    env->regs[r1] = data;
+    setcc(cpu, ZPCI_PCI_LS_OK);
+    return 0;
+}
+
+static void update_msix_table_msg_data(S390PCIBusDevice *pbdev, uint64_t offset,
+                                       uint64_t *data, uint8_t len)
+{
+    uint32_t val;
+    uint8_t *msg_data;
+
+    if (offset % PCI_MSIX_ENTRY_SIZE != 8) {
+        return;
+    }
+
+    if (len != 4) {
+        DPRINTF("access msix table msg data but len is %d\n", len);
+        return;
+    }
+
+    msg_data = (uint8_t *)data - offset % PCI_MSIX_ENTRY_SIZE +
+               PCI_MSIX_ENTRY_VECTOR_CTRL;
+    val = pci_get_long(msg_data) |
+        ((pbdev->fh & FH_MASK_INDEX) << ZPCI_MSI_VEC_BITS);
+    pci_set_long(msg_data, val);
+    DPRINTF("update msix msg_data to 0x%" PRIx64 "\n", *data);
+}
+
+static int trap_msix(S390PCIBusDevice *pbdev, uint64_t offset, uint8_t pcias)
+{
+    if (pbdev->msix.available && pbdev->msix.table_bar == pcias &&
+        offset >= pbdev->msix.table_offset &&
+        offset <= pbdev->msix.table_offset +
+                  (pbdev->msix.entries - 1) * PCI_MSIX_ENTRY_SIZE) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
+{
+    CPUS390XState *env = &cpu->env;
+    uint64_t offset, data;
+    S390PCIBusDevice *pbdev;
+    uint8_t len;
+    uint32_t fh;
+    uint8_t pcias;
+
+    cpu_synchronize_state(CPU(cpu));
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 4);
+        return 0;
+    }
+
+    if (r2 & 0x1) {
+        program_interrupt(env, PGM_SPECIFICATION, 4);
+        return 0;
+    }
+
+    fh = env->regs[r2] >> 32;
+    pcias = (env->regs[r2] >> 16) & 0xf;
+    len = env->regs[r2] & 0xf;
+    offset = env->regs[r2 + 1];
+
+    pbdev = s390_pci_find_dev_by_fh(fh);
+    if (!pbdev) {
+        DPRINTF("pcistg no pci dev\n");
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+    case ZPCI_FS_STANDBY:
+    case ZPCI_FS_DISABLED:
+    case ZPCI_FS_PERMANENT_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    case ZPCI_FS_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r2, ZPCI_PCI_ST_BLOCKED);
+        return 0;
+    default:
+        break;
+    }
+
+    data = env->regs[r1];
+    if (pcias < 6) {
+        if ((8 - (offset & 0x7)) < len) {
+            program_interrupt(env, PGM_OPERAND, 4);
+            return 0;
+        }
+        MemoryRegion *mr;
+        if (trap_msix(pbdev, offset, pcias)) {
+            offset = offset - pbdev->msix.table_offset;
+            mr = &pbdev->pdev->msix_table_mmio;
+            update_msix_table_msg_data(pbdev, offset, &data, len);
+        } else {
+            mr = pbdev->pdev->io_regions[pcias].memory;
+        }
+
+        memory_region_dispatch_write(mr, offset, data, len,
+                                     MEMTXATTRS_UNSPECIFIED);
+    } else if (pcias == 15) {
+        if ((4 - (offset & 0x3)) < len) {
+            program_interrupt(env, PGM_OPERAND, 4);
+            return 0;
+        }
+        switch (len) {
+        case 1:
+            break;
+        case 2:
+            data = bswap16(data);
+            break;
+        case 4:
+            data = bswap32(data);
+            break;
+        case 8:
+            data = bswap64(data);
+            break;
+        default:
+            program_interrupt(env, PGM_OPERAND, 4);
+            return 0;
+        }
+
+        pci_host_config_write_common(pbdev->pdev, offset,
+                                     pci_config_size(pbdev->pdev),
+                                     data, len);
+    } else {
+        DPRINTF("pcistg invalid space\n");
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r2, ZPCI_PCI_ST_INVAL_AS);
+        return 0;
+    }
+
+    setcc(cpu, ZPCI_PCI_LS_OK);
+    return 0;
+}
+
+int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
+{
+    CPUS390XState *env = &cpu->env;
+    uint32_t fh;
+    S390PCIBusDevice *pbdev;
+    hwaddr start, end;
+    IOMMUTLBEntry entry;
+    MemoryRegion *mr;
+
+    cpu_synchronize_state(CPU(cpu));
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 4);
+        goto out;
+    }
+
+    if (r2 & 0x1) {
+        program_interrupt(env, PGM_SPECIFICATION, 4);
+        goto out;
+    }
+
+    fh = env->regs[r1] >> 32;
+    start = env->regs[r2];
+    end = start + env->regs[r2 + 1];
+
+    pbdev = s390_pci_find_dev_by_fh(fh);
+    if (!pbdev) {
+        DPRINTF("rpcit no pci dev\n");
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        goto out;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+    case ZPCI_FS_STANDBY:
+    case ZPCI_FS_DISABLED:
+    case ZPCI_FS_PERMANENT_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    case ZPCI_FS_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_MOD_ST_ERROR_RECOVER);
+        return 0;
+    default:
+        break;
+    }
+
+    if (!pbdev->g_iota) {
+        pbdev->state = ZPCI_FS_ERROR;
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_PCI_ST_INSUF_RES);
+        s390_pci_generate_error_event(ERR_EVENT_INVALAS, pbdev->fh, pbdev->fid,
+                                      start, 0);
+        goto out;
+    }
+
+    if (end < pbdev->pba || start > pbdev->pal) {
+        pbdev->state = ZPCI_FS_ERROR;
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_PCI_ST_INSUF_RES);
+        s390_pci_generate_error_event(ERR_EVENT_OORANGE, pbdev->fh, pbdev->fid,
+                                      start, 0);
+        goto out;
+    }
+
+    mr = &pbdev->iommu_mr;
+    while (start < end) {
+        entry = mr->iommu_ops->translate(mr, start, 0);
+
+        if (!entry.translated_addr) {
+            pbdev->state = ZPCI_FS_ERROR;
+            setcc(cpu, ZPCI_PCI_LS_ERR);
+            s390_set_status_code(env, r1, ZPCI_PCI_ST_INSUF_RES);
+            s390_pci_generate_error_event(ERR_EVENT_SERR, pbdev->fh, pbdev->fid,
+                                          start, ERR_EVENT_Q_BIT);
+            goto out;
+        }
+
+        memory_region_notify_iommu(mr, entry);
+        start += entry.addr_mask + 1;
+    }
+
+    setcc(cpu, ZPCI_PCI_LS_OK);
+out:
+    return 0;
+}
+
+int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
+                        uint8_t ar)
+{
+    CPUS390XState *env = &cpu->env;
+    S390PCIBusDevice *pbdev;
+    MemoryRegion *mr;
+    int i;
+    uint32_t fh;
+    uint8_t pcias;
+    uint8_t len;
+    uint8_t buffer[128];
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 6);
+        return 0;
+    }
+
+    fh = env->regs[r1] >> 32;
+    pcias = (env->regs[r1] >> 16) & 0xf;
+    len = env->regs[r1] & 0xff;
+
+    if (pcias > 5) {
+        DPRINTF("pcistb invalid space\n");
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_PCI_ST_INVAL_AS);
+        return 0;
+    }
+
+    switch (len) {
+    case 16:
+    case 32:
+    case 64:
+    case 128:
+        break;
+    default:
+        program_interrupt(env, PGM_SPECIFICATION, 6);
+        return 0;
+    }
+
+    pbdev = s390_pci_find_dev_by_fh(fh);
+    if (!pbdev) {
+        DPRINTF("pcistb no pci dev fh 0x%x\n", fh);
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+    case ZPCI_FS_STANDBY:
+    case ZPCI_FS_DISABLED:
+    case ZPCI_FS_PERMANENT_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    case ZPCI_FS_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_PCI_ST_BLOCKED);
+        return 0;
+    default:
+        break;
+    }
+
+    mr = pbdev->pdev->io_regions[pcias].memory;
+    if (!memory_region_access_valid(mr, env->regs[r3], len, true)) {
+        program_interrupt(env, PGM_ADDRESSING, 6);
+        return 0;
+    }
+
+    if (s390_cpu_virt_mem_read(cpu, gaddr, ar, buffer, len)) {
+        return 0;
+    }
+
+    for (i = 0; i < len / 8; i++) {
+        memory_region_dispatch_write(mr, env->regs[r3] + i * 8,
+                                     ldq_p(buffer + i * 8), 8,
+                                     MEMTXATTRS_UNSPECIFIED);
+    }
+
+    setcc(cpu, ZPCI_PCI_LS_OK);
+    return 0;
+}
+
+static int reg_irqs(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
+{
+    int ret, len;
+
+    ret = css_register_io_adapter(S390_PCIPT_ADAPTER,
+                                  FIB_DATA_ISC(ldl_p(&fib.data)), true, false,
+                                  &pbdev->routes.adapter.adapter_id);
+    assert(ret == 0);
+
+    pbdev->summary_ind = get_indicator(ldq_p(&fib.aisb), sizeof(uint64_t));
+    len = BITS_TO_LONGS(FIB_DATA_NOI(ldl_p(&fib.data))) * sizeof(unsigned long);
+    pbdev->indicator = get_indicator(ldq_p(&fib.aibv), len);
+
+    ret = map_indicator(&pbdev->routes.adapter, pbdev->summary_ind);
+    if (ret) {
+        goto out;
+    }
+
+    ret = map_indicator(&pbdev->routes.adapter, pbdev->indicator);
+    if (ret) {
+        goto out;
+    }
+
+    pbdev->routes.adapter.summary_addr = ldq_p(&fib.aisb);
+    pbdev->routes.adapter.summary_offset = FIB_DATA_AISBO(ldl_p(&fib.data));
+    pbdev->routes.adapter.ind_addr = ldq_p(&fib.aibv);
+    pbdev->routes.adapter.ind_offset = FIB_DATA_AIBVO(ldl_p(&fib.data));
+    pbdev->isc = FIB_DATA_ISC(ldl_p(&fib.data));
+    pbdev->noi = FIB_DATA_NOI(ldl_p(&fib.data));
+    pbdev->sum = FIB_DATA_SUM(ldl_p(&fib.data));
+
+    DPRINTF("reg_irqs adapter id %d\n", pbdev->routes.adapter.adapter_id);
+    return 0;
+out:
+    release_indicator(&pbdev->routes.adapter, pbdev->summary_ind);
+    release_indicator(&pbdev->routes.adapter, pbdev->indicator);
+    pbdev->summary_ind = NULL;
+    pbdev->indicator = NULL;
+    return ret;
+}
+
+int pci_dereg_irqs(S390PCIBusDevice *pbdev)
+{
+    release_indicator(&pbdev->routes.adapter, pbdev->summary_ind);
+    release_indicator(&pbdev->routes.adapter, pbdev->indicator);
+
+    pbdev->summary_ind = NULL;
+    pbdev->indicator = NULL;
+    pbdev->routes.adapter.summary_addr = 0;
+    pbdev->routes.adapter.summary_offset = 0;
+    pbdev->routes.adapter.ind_addr = 0;
+    pbdev->routes.adapter.ind_offset = 0;
+    pbdev->isc = 0;
+    pbdev->noi = 0;
+    pbdev->sum = 0;
+
+    DPRINTF("dereg_irqs adapter id %d\n", pbdev->routes.adapter.adapter_id);
+    return 0;
+}
+
+static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
+{
+    uint64_t pba = ldq_p(&fib.pba);
+    uint64_t pal = ldq_p(&fib.pal);
+    uint64_t g_iota = ldq_p(&fib.iota);
+    uint8_t dt = (g_iota >> 2) & 0x7;
+    uint8_t t = (g_iota >> 11) & 0x1;
+
+    if (pba > pal || pba < ZPCI_SDMA_ADDR || pal > ZPCI_EDMA_ADDR) {
+        program_interrupt(env, PGM_OPERAND, 6);
+        return -EINVAL;
+    }
+
+    /* currently we only support designation type 1 with translation */
+    if (!(dt == ZPCI_IOTA_RTTO && t)) {
+        error_report("unsupported ioat dt %d t %d", dt, t);
+        program_interrupt(env, PGM_OPERAND, 6);
+        return -EINVAL;
+    }
+
+    pbdev->pba = pba;
+    pbdev->pal = pal;
+    pbdev->g_iota = g_iota;
+
+    s390_pci_iommu_enable(pbdev);
+
+    return 0;
+}
+
+void pci_dereg_ioat(S390PCIBusDevice *pbdev)
+{
+    s390_pci_iommu_disable(pbdev);
+    pbdev->pba = 0;
+    pbdev->pal = 0;
+    pbdev->g_iota = 0;
+}
+
+int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
+{
+    CPUS390XState *env = &cpu->env;
+    uint8_t oc, dmaas;
+    uint32_t fh;
+    ZpciFib fib;
+    S390PCIBusDevice *pbdev;
+    uint64_t cc = ZPCI_PCI_LS_OK;
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 6);
+        return 0;
+    }
+
+    oc = env->regs[r1] & 0xff;
+    dmaas = (env->regs[r1] >> 16) & 0xff;
+    fh = env->regs[r1] >> 32;
+
+    if (fiba & 0x7) {
+        program_interrupt(env, PGM_SPECIFICATION, 6);
+        return 0;
+    }
+
+    pbdev = s390_pci_find_dev_by_fh(fh);
+    if (!pbdev) {
+        DPRINTF("mpcifc no pci dev fh 0x%x\n", fh);
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    }
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+    case ZPCI_FS_STANDBY:
+    case ZPCI_FS_DISABLED:
+    case ZPCI_FS_PERMANENT_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    default:
+        break;
+    }
+
+    if (s390_cpu_virt_mem_read(cpu, fiba, ar, (uint8_t *)&fib, sizeof(fib))) {
+        return 0;
+    }
+
+    if (fib.fmt != 0) {
+        program_interrupt(env, PGM_OPERAND, 6);
+        return 0;
+    }
+
+    switch (oc) {
+    case ZPCI_MOD_FC_REG_INT:
+        if (pbdev->summary_ind) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        } else if (reg_irqs(env, pbdev, fib)) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_RES_NOT_AVAIL);
+        }
+        break;
+    case ZPCI_MOD_FC_DEREG_INT:
+        if (!pbdev->summary_ind) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        } else {
+            pci_dereg_irqs(pbdev);
+        }
+        break;
+    case ZPCI_MOD_FC_REG_IOAT:
+        if (dmaas != 0) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
+        } else if (pbdev->iommu_enabled) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        } else if (reg_ioat(env, pbdev, fib)) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES);
+        }
+        break;
+    case ZPCI_MOD_FC_DEREG_IOAT:
+        if (dmaas != 0) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
+        } else if (!pbdev->iommu_enabled) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        } else {
+            pci_dereg_ioat(pbdev);
+        }
+        break;
+    case ZPCI_MOD_FC_REREG_IOAT:
+        if (dmaas != 0) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
+        } else if (!pbdev->iommu_enabled) {
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        } else {
+            pci_dereg_ioat(pbdev);
+            if (reg_ioat(env, pbdev, fib)) {
+                cc = ZPCI_PCI_LS_ERR;
+                s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES);
+            }
+        }
+        break;
+    case ZPCI_MOD_FC_RESET_ERROR:
+        switch (pbdev->state) {
+        case ZPCI_FS_BLOCKED:
+        case ZPCI_FS_ERROR:
+            pbdev->state = ZPCI_FS_ENABLED;
+            break;
+        default:
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        }
+        break;
+    case ZPCI_MOD_FC_RESET_BLOCK:
+        switch (pbdev->state) {
+        case ZPCI_FS_ERROR:
+            pbdev->state = ZPCI_FS_BLOCKED;
+            break;
+        default:
+            cc = ZPCI_PCI_LS_ERR;
+            s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
+        }
+        break;
+    case ZPCI_MOD_FC_SET_MEASURE:
+        pbdev->fmb_addr = ldq_p(&fib.fmb_addr);
+        break;
+    default:
+        program_interrupt(&cpu->env, PGM_OPERAND, 6);
+        cc = ZPCI_PCI_LS_ERR;
+    }
+
+    setcc(cpu, cc);
+    return 0;
+}
+
+int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
+{
+    CPUS390XState *env = &cpu->env;
+    uint8_t dmaas;
+    uint32_t fh;
+    ZpciFib fib;
+    S390PCIBusDevice *pbdev;
+    uint32_t data;
+    uint64_t cc = ZPCI_PCI_LS_OK;
+
+    if (env->psw.mask & PSW_MASK_PSTATE) {
+        program_interrupt(env, PGM_PRIVILEGED, 6);
+        return 0;
+    }
+
+    fh = env->regs[r1] >> 32;
+    dmaas = (env->regs[r1] >> 16) & 0xff;
+
+    if (dmaas) {
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_STPCIFC_ST_INVAL_DMAAS);
+        return 0;
+    }
+
+    if (fiba & 0x7) {
+        program_interrupt(env, PGM_SPECIFICATION, 6);
+        return 0;
+    }
+
+    pbdev = s390_pci_find_dev_by_idx(fh & FH_MASK_INDEX);
+    if (!pbdev) {
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    }
+
+    memset(&fib, 0, sizeof(fib));
+
+    switch (pbdev->state) {
+    case ZPCI_FS_RESERVED:
+    case ZPCI_FS_STANDBY:
+        setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+        return 0;
+    case ZPCI_FS_DISABLED:
+        if (fh & FH_MASK_ENABLE) {
+            setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
+            return 0;
+        }
+        goto out;
+    /* BLOCKED bit is set to one coincident with the setting of ERROR bit.
+     * FH Enabled bit is set to one in states of ENABLED, BLOCKED or ERROR. */
+    case ZPCI_FS_ERROR:
+        fib.fc |= 0x20;
+    case ZPCI_FS_BLOCKED:
+        fib.fc |= 0x40;
+    case ZPCI_FS_ENABLED:
+        fib.fc |= 0x80;
+        if (pbdev->iommu_enabled) {
+            fib.fc |= 0x10;
+        }
+        if (!(fh & FH_MASK_ENABLE)) {
+            env->regs[r1] |= 1ULL << 63;
+        }
+        break;
+    case ZPCI_FS_PERMANENT_ERROR:
+        setcc(cpu, ZPCI_PCI_LS_ERR);
+        s390_set_status_code(env, r1, ZPCI_STPCIFC_ST_PERM_ERROR);
+        return 0;
+    }
+
+    stq_p(&fib.pba, pbdev->pba);
+    stq_p(&fib.pal, pbdev->pal);
+    stq_p(&fib.iota, pbdev->g_iota);
+    stq_p(&fib.aibv, pbdev->routes.adapter.ind_addr);
+    stq_p(&fib.aisb, pbdev->routes.adapter.summary_addr);
+    stq_p(&fib.fmb_addr, pbdev->fmb_addr);
+
+    data = ((uint32_t)pbdev->isc << 28) | ((uint32_t)pbdev->noi << 16) |
+           ((uint32_t)pbdev->routes.adapter.ind_offset << 8) |
+           ((uint32_t)pbdev->sum << 7) | pbdev->routes.adapter.summary_offset;
+    stl_p(&fib.data, data);
+
+out:
+    if (s390_cpu_virt_mem_write(cpu, fiba, ar, (uint8_t *)&fib, sizeof(fib))) {
+        return 0;
+    }
+
+    setcc(cpu, cc);
+    return 0;
+}
diff --git a/hw/s390x/s390-pci-inst.h b/hw/s390x/s390-pci-inst.h
new file mode 100644
index 0000000..23f4bfa
--- /dev/null
+++ b/hw/s390x/s390-pci-inst.h
@@ -0,0 +1,305 @@
+/*
+ * s390 PCI instruction definitions
+ *
+ * Copyright 2014 IBM Corp.
+ * Author(s): Frank Blaschka <frank.blaschka@de.ibm.com>
+ *            Hong Bo Li <lihbbj@cn.ibm.com>
+ *            Yi Min Zhao <zyimin@cn.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_PCI_INST_H
+#define HW_S390_PCI_INST_H
+
+#include "s390-pci-bus.h"
+#include "sysemu/dma.h"
+
+/* CLP common request & response block size */
+#define CLP_BLK_SIZE 4096
+#define PCI_BAR_COUNT 6
+#define PCI_MAX_FUNCTIONS 4096
+
+typedef struct ClpReqHdr {
+    uint16_t len;
+    uint16_t cmd;
+} QEMU_PACKED ClpReqHdr;
+
+typedef struct ClpRspHdr {
+    uint16_t len;
+    uint16_t rsp;
+} QEMU_PACKED ClpRspHdr;
+
+/* CLP Response Codes */
+#define CLP_RC_OK         0x0010  /* Command request successfully */
+#define CLP_RC_CMD        0x0020  /* Command code not recognized */
+#define CLP_RC_PERM       0x0030  /* Command not authorized */
+#define CLP_RC_FMT        0x0040  /* Invalid command request format */
+#define CLP_RC_LEN        0x0050  /* Invalid command request length */
+#define CLP_RC_8K         0x0060  /* Command requires 8K LPCB */
+#define CLP_RC_RESNOT0    0x0070  /* Reserved field not zero */
+#define CLP_RC_NODATA     0x0080  /* No data available */
+#define CLP_RC_FC_UNKNOWN 0x0100  /* Function code not recognized */
+
+/*
+ * Call Logical Processor - Command Codes
+ */
+#define CLP_LIST_PCI            0x0002
+#define CLP_QUERY_PCI_FN        0x0003
+#define CLP_QUERY_PCI_FNGRP     0x0004
+#define CLP_SET_PCI_FN          0x0005
+
+/* PCI function handle list entry */
+typedef struct ClpFhListEntry {
+    uint16_t device_id;
+    uint16_t vendor_id;
+#define CLP_FHLIST_MASK_CONFIG 0x80000000
+    uint32_t config;
+    uint32_t fid;
+    uint32_t fh;
+} QEMU_PACKED ClpFhListEntry;
+
+#define CLP_RC_SETPCIFN_FH      0x0101 /* Invalid PCI fn handle */
+#define CLP_RC_SETPCIFN_FHOP    0x0102 /* Fn handle not valid for op */
+#define CLP_RC_SETPCIFN_DMAAS   0x0103 /* Invalid DMA addr space */
+#define CLP_RC_SETPCIFN_RES     0x0104 /* Insufficient resources */
+#define CLP_RC_SETPCIFN_ALRDY   0x0105 /* Fn already in requested state */
+#define CLP_RC_SETPCIFN_ERR     0x0106 /* Fn in permanent error state */
+#define CLP_RC_SETPCIFN_RECPND  0x0107 /* Error recovery pending */
+#define CLP_RC_SETPCIFN_BUSY    0x0108 /* Fn busy */
+#define CLP_RC_LISTPCI_BADRT    0x010a /* Resume token not recognized */
+#define CLP_RC_QUERYPCIFG_PFGID 0x010b /* Unrecognized PFGID */
+
+/* request or response block header length */
+#define LIST_PCI_HDR_LEN 32
+
+/* Number of function handles fitting in response block */
+#define CLP_FH_LIST_NR_ENTRIES \
+    ((CLP_BLK_SIZE - 2 * LIST_PCI_HDR_LEN) \
+        / sizeof(ClpFhListEntry))
+
+#define CLP_SET_ENABLE_PCI_FN  0 /* Yes, 0 enables it */
+#define CLP_SET_DISABLE_PCI_FN 1 /* Yes, 1 disables it */
+
+#define CLP_UTIL_STR_LEN 64
+
+#define CLP_MASK_FMT 0xf0000000
+
+/* List PCI functions request */
+typedef struct ClpReqListPci {
+    ClpReqHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+    uint64_t resume_token;
+    uint64_t reserved2;
+} QEMU_PACKED ClpReqListPci;
+
+/* List PCI functions response */
+typedef struct ClpRspListPci {
+    ClpRspHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+    uint64_t resume_token;
+    uint32_t mdd;
+    uint16_t max_fn;
+    uint8_t flags;
+    uint8_t entry_size;
+    ClpFhListEntry fh_list[CLP_FH_LIST_NR_ENTRIES];
+} QEMU_PACKED ClpRspListPci;
+
+/* Query PCI function request */
+typedef struct ClpReqQueryPci {
+    ClpReqHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+    uint32_t fh; /* function handle */
+    uint32_t reserved2;
+    uint64_t reserved3;
+} QEMU_PACKED ClpReqQueryPci;
+
+/* Query PCI function response */
+typedef struct ClpRspQueryPci {
+    ClpRspHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+    uint16_t vfn; /* virtual fn number */
+#define CLP_RSP_QPCI_MASK_UTIL  0x100
+#define CLP_RSP_QPCI_MASK_PFGID 0xff
+    uint16_t ug;
+    uint32_t fid; /* pci function id */
+    uint8_t bar_size[PCI_BAR_COUNT];
+    uint16_t pchid;
+    uint32_t bar[PCI_BAR_COUNT];
+    uint64_t reserved2;
+    uint64_t sdma; /* start dma as */
+    uint64_t edma; /* end dma as */
+    uint32_t reserved3[11];
+    uint32_t uid;
+    uint8_t util_str[CLP_UTIL_STR_LEN]; /* utility string */
+} QEMU_PACKED ClpRspQueryPci;
+
+/* Query PCI function group request */
+typedef struct ClpReqQueryPciGrp {
+    ClpReqHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+#define CLP_REQ_QPCIG_MASK_PFGID 0xff
+    uint32_t g;
+    uint32_t reserved2;
+    uint64_t reserved3;
+} QEMU_PACKED ClpReqQueryPciGrp;
+
+/* Query PCI function group response */
+typedef struct ClpRspQueryPciGrp {
+    ClpRspHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+#define CLP_RSP_QPCIG_MASK_NOI 0xfff
+    uint16_t i;
+    uint8_t version;
+#define CLP_RSP_QPCIG_MASK_FRAME   0x2
+#define CLP_RSP_QPCIG_MASK_REFRESH 0x1
+    uint8_t fr;
+    uint16_t reserved2;
+    uint16_t mui;
+    uint64_t reserved3;
+    uint64_t dasm; /* dma address space mask */
+    uint64_t msia; /* MSI address */
+    uint64_t reserved4;
+    uint64_t reserved5;
+} QEMU_PACKED ClpRspQueryPciGrp;
+
+/* Set PCI function request */
+typedef struct ClpReqSetPci {
+    ClpReqHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+    uint32_t fh; /* function handle */
+    uint16_t reserved2;
+    uint8_t oc; /* operation controls */
+    uint8_t ndas; /* number of dma spaces */
+    uint64_t reserved3;
+} QEMU_PACKED ClpReqSetPci;
+
+/* Set PCI function response */
+typedef struct ClpRspSetPci {
+    ClpRspHdr hdr;
+    uint32_t fmt;
+    uint64_t reserved1;
+    uint32_t fh; /* function handle */
+    uint32_t reserved3;
+    uint64_t reserved4;
+} QEMU_PACKED ClpRspSetPci;
+
+typedef struct ClpReqRspListPci {
+    ClpReqListPci request;
+    ClpRspListPci response;
+} QEMU_PACKED ClpReqRspListPci;
+
+typedef struct ClpReqRspSetPci {
+    ClpReqSetPci request;
+    ClpRspSetPci response;
+} QEMU_PACKED ClpReqRspSetPci;
+
+typedef struct ClpReqRspQueryPci {
+    ClpReqQueryPci request;
+    ClpRspQueryPci response;
+} QEMU_PACKED ClpReqRspQueryPci;
+
+typedef struct ClpReqRspQueryPciGrp {
+    ClpReqQueryPciGrp request;
+    ClpRspQueryPciGrp response;
+} QEMU_PACKED ClpReqRspQueryPciGrp;
+
+/* Load/Store status codes */
+#define ZPCI_PCI_ST_FUNC_NOT_ENABLED        4
+#define ZPCI_PCI_ST_FUNC_IN_ERR             8
+#define ZPCI_PCI_ST_BLOCKED                 12
+#define ZPCI_PCI_ST_INSUF_RES               16
+#define ZPCI_PCI_ST_INVAL_AS                20
+#define ZPCI_PCI_ST_FUNC_ALREADY_ENABLED    24
+#define ZPCI_PCI_ST_DMA_AS_NOT_ENABLED      28
+#define ZPCI_PCI_ST_2ND_OP_IN_INV_AS        36
+#define ZPCI_PCI_ST_FUNC_NOT_AVAIL          40
+#define ZPCI_PCI_ST_ALREADY_IN_RQ_STATE     44
+
+/* Load/Store return codes */
+#define ZPCI_PCI_LS_OK              0
+#define ZPCI_PCI_LS_ERR             1
+#define ZPCI_PCI_LS_BUSY            2
+#define ZPCI_PCI_LS_INVAL_HANDLE    3
+
+/* Modify PCI status codes */
+#define ZPCI_MOD_ST_RES_NOT_AVAIL 4
+#define ZPCI_MOD_ST_INSUF_RES     16
+#define ZPCI_MOD_ST_SEQUENCE      24
+#define ZPCI_MOD_ST_DMAAS_INVAL   28
+#define ZPCI_MOD_ST_FRAME_INVAL   32
+#define ZPCI_MOD_ST_ERROR_RECOVER 40
+
+/* Modify PCI Function Controls */
+#define ZPCI_MOD_FC_REG_INT     2
+#define ZPCI_MOD_FC_DEREG_INT   3
+#define ZPCI_MOD_FC_REG_IOAT    4
+#define ZPCI_MOD_FC_DEREG_IOAT  5
+#define ZPCI_MOD_FC_REREG_IOAT  6
+#define ZPCI_MOD_FC_RESET_ERROR 7
+#define ZPCI_MOD_FC_RESET_BLOCK 9
+#define ZPCI_MOD_FC_SET_MEASURE 10
+
+/* Store PCI Function Controls status codes */
+#define ZPCI_STPCIFC_ST_PERM_ERROR    8
+#define ZPCI_STPCIFC_ST_INVAL_DMAAS   28
+#define ZPCI_STPCIFC_ST_ERROR_RECOVER 40
+
+/* FIB function controls */
+#define ZPCI_FIB_FC_ENABLED     0x80
+#define ZPCI_FIB_FC_ERROR       0x40
+#define ZPCI_FIB_FC_LS_BLOCKED  0x20
+#define ZPCI_FIB_FC_DMAAS_REG   0x10
+
+/* FIB function controls */
+#define ZPCI_FIB_FC_ENABLED     0x80
+#define ZPCI_FIB_FC_ERROR       0x40
+#define ZPCI_FIB_FC_LS_BLOCKED  0x20
+#define ZPCI_FIB_FC_DMAAS_REG   0x10
+
+/* Function Information Block */
+typedef struct ZpciFib {
+    uint8_t fmt;   /* format */
+    uint8_t reserved1[7];
+    uint8_t fc;                  /* function controls */
+    uint8_t reserved2;
+    uint16_t reserved3;
+    uint32_t reserved4;
+    uint64_t pba;                /* PCI base address */
+    uint64_t pal;                /* PCI address limit */
+    uint64_t iota;               /* I/O Translation Anchor */
+#define FIB_DATA_ISC(x)    (((x) >> 28) & 0x7)
+#define FIB_DATA_NOI(x)    (((x) >> 16) & 0xfff)
+#define FIB_DATA_AIBVO(x) (((x) >> 8) & 0x3f)
+#define FIB_DATA_SUM(x)    (((x) >> 7) & 0x1)
+#define FIB_DATA_AISBO(x)  ((x) & 0x3f)
+    uint32_t data;
+    uint32_t reserved5;
+    uint64_t aibv;               /* Adapter int bit vector address */
+    uint64_t aisb;               /* Adapter int summary bit address */
+    uint64_t fmb_addr;           /* Function measurement address and key */
+    uint32_t reserved6;
+    uint32_t gd;
+} QEMU_PACKED ZpciFib;
+
+int pci_dereg_irqs(S390PCIBusDevice *pbdev);
+void pci_dereg_ioat(S390PCIBusDevice *pbdev);
+int clp_service_call(S390CPU *cpu, uint8_t r2);
+int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2);
+int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2);
+int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2);
+int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
+                        uint8_t ar);
+int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar);
+int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar);
+
+#endif
diff --git a/hw/s390x/s390-skeys-kvm.c b/hw/s390x/s390-skeys-kvm.c
new file mode 100644
index 0000000..131da56
--- /dev/null
+++ b/hw/s390x/s390-skeys-kvm.c
@@ -0,0 +1,76 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/s390x/storage-keys.h"
+#include "sysemu/kvm.h"
+#include "qemu/error-report.h"
+
+static int kvm_s390_skeys_enabled(S390SKeysState *ss)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint8_t single_key;
+    int r;
+
+    r = skeyclass->get_skeys(ss, 0, 1, &single_key);
+    if (r != 0 && r != KVM_S390_GET_SKEYS_NONE) {
+        error_report("S390_GET_KEYS error %d", r);
+    }
+    return (r == 0);
+}
+
+static int kvm_s390_skeys_get(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    struct kvm_s390_skeys args = {
+        .start_gfn = start_gfn,
+        .count = count,
+        .skeydata_addr = (__u64)keys
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_GET_SKEYS, &args);
+}
+
+static int kvm_s390_skeys_set(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    struct kvm_s390_skeys args = {
+        .start_gfn = start_gfn,
+        .count = count,
+        .skeydata_addr = (__u64)keys
+    };
+
+    return kvm_vm_ioctl(kvm_state, KVM_S390_SET_SKEYS, &args);
+}
+
+static void kvm_s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_CLASS(oc);
+
+    skeyclass->skeys_enabled = kvm_s390_skeys_enabled;
+    skeyclass->get_skeys = kvm_s390_skeys_get;
+    skeyclass->set_skeys = kvm_s390_skeys_set;
+}
+
+static const TypeInfo kvm_s390_skeys_info = {
+    .name          = TYPE_KVM_S390_SKEYS,
+    .parent        = TYPE_S390_SKEYS,
+    .instance_size = sizeof(S390SKeysState),
+    .class_init    = kvm_s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+};
+
+static void kvm_s390_skeys_register_types(void)
+{
+    type_register_static(&kvm_s390_skeys_info);
+}
+
+type_init(kvm_s390_skeys_register_types)
diff --git a/hw/s390x/s390-skeys.c b/hw/s390x/s390-skeys.c
new file mode 100644
index 0000000..e2d4e1a
--- /dev/null
+++ b/hw/s390x/s390-skeys.c
@@ -0,0 +1,418 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/boards.h"
+#include "qmp-commands.h"
+#include "migration/qemu-file.h"
+#include "hw/s390x/storage-keys.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+
+#define S390_SKEYS_BUFFER_SIZE 131072  /* Room for 128k storage keys */
+#define S390_SKEYS_SAVE_FLAG_EOS 0x01
+#define S390_SKEYS_SAVE_FLAG_SKEYS 0x02
+#define S390_SKEYS_SAVE_FLAG_ERROR 0x04
+
+S390SKeysState *s390_get_skeys_device(void)
+{
+    S390SKeysState *ss;
+
+    ss = S390_SKEYS(object_resolve_path_type("", TYPE_S390_SKEYS, NULL));
+    assert(ss);
+    return ss;
+}
+
+void s390_skeys_init(void)
+{
+    Object *obj;
+
+    if (kvm_enabled()) {
+        obj = object_new(TYPE_KVM_S390_SKEYS);
+    } else {
+        obj = object_new(TYPE_QEMU_S390_SKEYS);
+    }
+    object_property_add_child(qdev_get_machine(), TYPE_S390_SKEYS,
+                              obj, NULL);
+    object_unref(obj);
+
+    qdev_init_nofail(DEVICE(obj));
+}
+
+static void write_keys(FILE *f, uint8_t *keys, uint64_t startgfn,
+                       uint64_t count, Error **errp)
+{
+    uint64_t curpage = startgfn;
+    uint64_t maxpage = curpage + count - 1;
+
+    for (; curpage <= maxpage; curpage++) {
+        uint8_t acc = (*keys & 0xF0) >> 4;
+        int fp =  (*keys & 0x08);
+        int ref = (*keys & 0x04);
+        int ch = (*keys & 0x02);
+        int res = (*keys & 0x01);
+
+        fprintf(f, "page=%03" PRIx64 ": key(%d) => ACC=%X, FP=%d, REF=%d,"
+                " ch=%d, reserved=%d\n",
+                curpage, *keys, acc, fp, ref, ch, res);
+        keys++;
+    }
+}
+
+void hmp_info_skeys(Monitor *mon, const QDict *qdict)
+{
+    S390SKeysState *ss = s390_get_skeys_device();
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint8_t key;
+    int r;
+
+    /* Quick check to see if guest is using storage keys*/
+    if (!skeyclass->skeys_enabled(ss)) {
+        monitor_printf(mon, "Error: This guest is not using storage keys\n");
+        return;
+    }
+
+    r = skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
+    if (r < 0) {
+        monitor_printf(mon, "Error: %s\n", strerror(-r));
+        return;
+    }
+
+    monitor_printf(mon, "  key: 0x%X\n", key);
+}
+
+void hmp_dump_skeys(Monitor *mon, const QDict *qdict)
+{
+    const char *filename = qdict_get_str(qdict, "filename");
+    Error *err = NULL;
+
+    qmp_dump_skeys(filename, &err);
+    if (err) {
+        error_report_err(err);
+    }
+}
+
+void qmp_dump_skeys(const char *filename, Error **errp)
+{
+    S390SKeysState *ss = s390_get_skeys_device();
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    const uint64_t total_count = ram_size / TARGET_PAGE_SIZE;
+    uint64_t handled_count = 0, cur_count;
+    Error *lerr = NULL;
+    vaddr cur_gfn = 0;
+    uint8_t *buf;
+    int ret;
+    int fd;
+    FILE *f;
+
+    /* Quick check to see if guest is using storage keys*/
+    if (!skeyclass->skeys_enabled(ss)) {
+        error_setg(errp, "This guest is not using storage keys - "
+                         "nothing to dump");
+        return;
+    }
+
+    fd = qemu_open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0600);
+    if (fd < 0) {
+        error_setg_file_open(errp, errno, filename);
+        return;
+    }
+    f = fdopen(fd, "wb");
+    if (!f) {
+        close(fd);
+        error_setg_file_open(errp, errno, filename);
+        return;
+    }
+
+    buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+    if (!buf) {
+        error_setg(errp, "Could not allocate memory");
+        goto out;
+    }
+
+    /* we'll only dump initial memory for now */
+    while (handled_count < total_count) {
+        /* Calculate how many keys to ask for & handle overflow case */
+        cur_count = MIN(total_count - handled_count, S390_SKEYS_BUFFER_SIZE);
+
+        ret = skeyclass->get_skeys(ss, cur_gfn, cur_count, buf);
+        if (ret < 0) {
+            error_setg(errp, "get_keys error %d", ret);
+            goto out_free;
+        }
+
+        /* write keys to stream */
+        write_keys(f, buf, cur_gfn, cur_count, &lerr);
+        if (lerr) {
+            goto out_free;
+        }
+
+        cur_gfn += cur_count;
+        handled_count += cur_count;
+    }
+
+out_free:
+    error_propagate(errp, lerr);
+    g_free(buf);
+out:
+    fclose(f);
+}
+
+static void qemu_s390_skeys_init(Object *obj)
+{
+    QEMUS390SKeysState *skeys = QEMU_S390_SKEYS(obj);
+    MachineState *machine = MACHINE(qdev_get_machine());
+
+    skeys->key_count = machine->maxram_size / TARGET_PAGE_SIZE;
+    skeys->keydata = g_malloc0(skeys->key_count);
+}
+
+static int qemu_s390_skeys_enabled(S390SKeysState *ss)
+{
+    return 1;
+}
+
+/*
+ * TODO: for memory hotplug support qemu_s390_skeys_set and qemu_s390_skeys_get
+ * will have to make sure that the given gfn belongs to a memory region and not
+ * a memory hole.
+ */
+static int qemu_s390_skeys_set(S390SKeysState *ss, uint64_t start_gfn,
+                              uint64_t count, uint8_t *keys)
+{
+    QEMUS390SKeysState *skeydev = QEMU_S390_SKEYS(ss);
+    int i;
+
+    /* Check for uint64 overflow and access beyond end of key data */
+    if (start_gfn + count > skeydev->key_count || start_gfn + count < count) {
+        error_report("Error: Setting storage keys for page beyond the end "
+                     "of memory: gfn=%" PRIx64 " count=%" PRId64,
+                     start_gfn, count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < count; i++) {
+        skeydev->keydata[start_gfn + i] = keys[i];
+    }
+    return 0;
+}
+
+static int qemu_s390_skeys_get(S390SKeysState *ss, uint64_t start_gfn,
+                               uint64_t count, uint8_t *keys)
+{
+    QEMUS390SKeysState *skeydev = QEMU_S390_SKEYS(ss);
+    int i;
+
+    /* Check for uint64 overflow and access beyond end of key data */
+    if (start_gfn + count > skeydev->key_count || start_gfn + count < count) {
+        error_report("Error: Getting storage keys for page beyond the end "
+                     "of memory: gfn=%" PRIx64 " count=%" PRId64,
+                     start_gfn, count);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < count; i++) {
+        keys[i] = skeydev->keydata[start_gfn + i];
+    }
+    return 0;
+}
+
+static void qemu_s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    S390SKeysClass *skeyclass = S390_SKEYS_CLASS(oc);
+
+    skeyclass->skeys_enabled = qemu_s390_skeys_enabled;
+    skeyclass->get_skeys = qemu_s390_skeys_get;
+    skeyclass->set_skeys = qemu_s390_skeys_set;
+}
+
+static const TypeInfo qemu_s390_skeys_info = {
+    .name          = TYPE_QEMU_S390_SKEYS,
+    .parent        = TYPE_S390_SKEYS,
+    .instance_init = qemu_s390_skeys_init,
+    .instance_size = sizeof(QEMUS390SKeysState),
+    .class_init    = qemu_s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+};
+
+static void s390_storage_keys_save(QEMUFile *f, void *opaque)
+{
+    S390SKeysState *ss = S390_SKEYS(opaque);
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    uint64_t pages_left = ram_size / TARGET_PAGE_SIZE;
+    uint64_t read_count, eos = S390_SKEYS_SAVE_FLAG_EOS;
+    vaddr cur_gfn = 0;
+    int error = 0;
+    uint8_t *buf;
+
+    if (!skeyclass->skeys_enabled(ss)) {
+        goto end_stream;
+    }
+
+    buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+    if (!buf) {
+        error_report("storage key save could not allocate memory");
+        goto end_stream;
+    }
+
+    /* We only support initial memory. Standby memory is not handled yet. */
+    qemu_put_be64(f, (cur_gfn * TARGET_PAGE_SIZE) | S390_SKEYS_SAVE_FLAG_SKEYS);
+    qemu_put_be64(f, pages_left);
+
+    while (pages_left) {
+        read_count = MIN(pages_left, S390_SKEYS_BUFFER_SIZE);
+
+        if (!error) {
+            error = skeyclass->get_skeys(ss, cur_gfn, read_count, buf);
+            if (error) {
+                /*
+                 * If error: we want to fill the stream with valid data instead
+                 * of stopping early so we pad the stream with 0x00 values and
+                 * use S390_SKEYS_SAVE_FLAG_ERROR to indicate failure to the
+                 * reading side.
+                 */
+                error_report("S390_GET_KEYS error %d", error);
+                memset(buf, 0, S390_SKEYS_BUFFER_SIZE);
+                eos = S390_SKEYS_SAVE_FLAG_ERROR;
+            }
+        }
+
+        qemu_put_buffer(f, buf, read_count);
+        cur_gfn += read_count;
+        pages_left -= read_count;
+    }
+
+    g_free(buf);
+end_stream:
+    qemu_put_be64(f, eos);
+}
+
+static int s390_storage_keys_load(QEMUFile *f, void *opaque, int version_id)
+{
+    S390SKeysState *ss = S390_SKEYS(opaque);
+    S390SKeysClass *skeyclass = S390_SKEYS_GET_CLASS(ss);
+    int ret = 0;
+
+    while (!ret) {
+        ram_addr_t addr;
+        int flags;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        switch (flags) {
+        case S390_SKEYS_SAVE_FLAG_SKEYS: {
+            const uint64_t total_count = qemu_get_be64(f);
+            uint64_t handled_count = 0, cur_count;
+            uint64_t cur_gfn = addr / TARGET_PAGE_SIZE;
+            uint8_t *buf = g_try_malloc(S390_SKEYS_BUFFER_SIZE);
+
+            if (!buf) {
+                error_report("storage key load could not allocate memory");
+                ret = -ENOMEM;
+                break;
+            }
+
+            while (handled_count < total_count) {
+                cur_count = MIN(total_count - handled_count,
+                                S390_SKEYS_BUFFER_SIZE);
+                qemu_get_buffer(f, buf, cur_count);
+
+                ret = skeyclass->set_skeys(ss, cur_gfn, cur_count, buf);
+                if (ret < 0) {
+                    error_report("S390_SET_KEYS error %d", ret);
+                    break;
+                }
+                handled_count += cur_count;
+                cur_gfn += cur_count;
+            }
+            g_free(buf);
+            break;
+        }
+        case S390_SKEYS_SAVE_FLAG_ERROR: {
+            error_report("Storage key data is incomplete");
+            ret = -EINVAL;
+            break;
+        }
+        case S390_SKEYS_SAVE_FLAG_EOS:
+            /* normal exit */
+            return 0;
+        default:
+            error_report("Unexpected storage key flag data: %#x", flags);
+            ret = -EINVAL;
+        }
+    }
+
+    return ret;
+}
+
+static inline bool s390_skeys_get_migration_enabled(Object *obj, Error **errp)
+{
+    S390SKeysState *ss = S390_SKEYS(obj);
+
+    return ss->migration_enabled;
+}
+
+static inline void s390_skeys_set_migration_enabled(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390SKeysState *ss = S390_SKEYS(obj);
+
+    /* Prevent double registration of savevm handler */
+    if (ss->migration_enabled == value) {
+        return;
+    }
+
+    ss->migration_enabled = value;
+
+    if (ss->migration_enabled) {
+        register_savevm(NULL, TYPE_S390_SKEYS, 0, 1, s390_storage_keys_save,
+                        s390_storage_keys_load, ss);
+    } else {
+        unregister_savevm(DEVICE(ss), TYPE_S390_SKEYS, ss);
+    }
+}
+
+static void s390_skeys_instance_init(Object *obj)
+{
+    object_property_add_bool(obj, "migration-enabled",
+                             s390_skeys_get_migration_enabled,
+                             s390_skeys_set_migration_enabled, NULL);
+    object_property_set_bool(obj, true, "migration-enabled", NULL);
+}
+
+static void s390_skeys_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
+static const TypeInfo s390_skeys_info = {
+    .name          = TYPE_S390_SKEYS,
+    .parent        = TYPE_DEVICE,
+    .instance_init = s390_skeys_instance_init,
+    .instance_size = sizeof(S390SKeysState),
+    .class_init    = s390_skeys_class_init,
+    .class_size    = sizeof(S390SKeysClass),
+    .abstract = true,
+};
+
+static void qemu_s390_skeys_register_types(void)
+{
+    type_register_static(&s390_skeys_info);
+    type_register_static(&qemu_s390_skeys_info);
+}
+
+type_init(qemu_s390_skeys_register_types)
diff --git a/hw/s390x/s390-virtio-bus.c b/hw/s390x/s390-virtio-bus.c
deleted file mode 100644
index 39dc201..0000000
--- a/hw/s390x/s390-virtio-bus.c
+++ /dev/null
@@ -1,735 +0,0 @@
-/*
- * QEMU S390 virtio target
- *
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "hw/hw.h"
-#include "sysemu/block-backend.h"
-#include "sysemu/sysemu.h"
-#include "hw/boards.h"
-#include "monitor/monitor.h"
-#include "hw/loader.h"
-#include "elf.h"
-#include "hw/virtio/virtio.h"
-#include "hw/virtio/virtio-rng.h"
-#include "hw/virtio/virtio-serial.h"
-#include "hw/virtio/virtio-net.h"
-#include "hw/virtio/vhost-scsi.h"
-#include "hw/sysbus.h"
-#include "sysemu/kvm.h"
-
-#include "hw/s390x/s390-virtio-bus.h"
-#include "hw/virtio/virtio-bus.h"
-
-/* #define DEBUG_S390 */
-
-#ifdef DEBUG_S390
-#define DPRINTF(fmt, ...) \
-    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-static void virtio_s390_bus_new(VirtioBusState *bus, size_t bus_size,
-                                VirtIOS390Device *dev);
-
-static const TypeInfo s390_virtio_bus_info = {
-    .name = TYPE_S390_VIRTIO_BUS,
-    .parent = TYPE_BUS,
-    .instance_size = sizeof(VirtIOS390Bus),
-};
-
-static ram_addr_t s390_virtio_device_num_vq(VirtIOS390Device *dev);
-
-/* length of VirtIO device pages */
-const hwaddr virtio_size = S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
-
-static void s390_virtio_bus_reset(void *opaque)
-{
-    VirtIOS390Bus *bus = opaque;
-    bus->next_ring = bus->dev_page + TARGET_PAGE_SIZE;
-}
-
-void s390_virtio_reset_idx(VirtIOS390Device *dev)
-{
-    int i;
-    hwaddr idx_addr;
-    uint8_t num_vq;
-
-    num_vq = s390_virtio_device_num_vq(dev);
-    for (i = 0; i < num_vq; i++) {
-        idx_addr = virtio_queue_get_avail_addr(dev->vdev, i) +
-            VIRTIO_VRING_AVAIL_IDX_OFFS;
-        stw_phys(&address_space_memory, idx_addr, 0);
-        idx_addr = virtio_queue_get_used_addr(dev->vdev, i) +
-            VIRTIO_VRING_USED_IDX_OFFS;
-        stw_phys(&address_space_memory, idx_addr, 0);
-    }
-}
-
-VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size)
-{
-    VirtIOS390Bus *bus;
-    BusState *_bus;
-    DeviceState *dev;
-
-    /* Create bridge device */
-    dev = qdev_create(NULL, "s390-virtio-bridge");
-    qdev_init_nofail(dev);
-
-    /* Create bus on bridge device */
-
-    _bus = qbus_create(TYPE_S390_VIRTIO_BUS, dev, "s390-virtio");
-    bus = DO_UPCAST(VirtIOS390Bus, bus, _bus);
-
-    bus->dev_page = *ram_size;
-    bus->dev_offs = bus->dev_page;
-    bus->next_ring = bus->dev_page + TARGET_PAGE_SIZE;
-
-    /* Enable hotplugging */
-    qbus_set_hotplug_handler(_bus, dev, &error_abort);
-
-    /* Allocate RAM for VirtIO device pages (descriptors, queues, rings) */
-    *ram_size += S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
-
-    qemu_register_reset(s390_virtio_bus_reset, bus);
-    return bus;
-}
-
-static int s390_virtio_device_init(VirtIOS390Device *dev, VirtIODevice *vdev)
-{
-    VirtIOS390Bus *bus;
-    int dev_len;
-
-    bus = DO_UPCAST(VirtIOS390Bus, bus, dev->qdev.parent_bus);
-    dev->vdev = vdev;
-    dev->dev_offs = bus->dev_offs;
-    dev->feat_len = sizeof(uint32_t); /* always keep 32 bits features */
-
-    dev_len = VIRTIO_DEV_OFFS_CONFIG;
-    dev_len += s390_virtio_device_num_vq(dev) * VIRTIO_VQCONFIG_LEN;
-    dev_len += dev->feat_len * 2;
-    dev_len += virtio_bus_get_vdev_config_len(&dev->bus);
-
-    bus->dev_offs += dev_len;
-
-    dev->host_features = virtio_bus_get_vdev_features(&dev->bus,
-                                                      dev->host_features);
-    s390_virtio_device_sync(dev);
-    s390_virtio_reset_idx(dev);
-    if (dev->qdev.hotplugged) {
-        s390_virtio_irq(VIRTIO_PARAM_DEV_ADD, dev->dev_offs);
-    }
-
-    return 0;
-}
-
-static int s390_virtio_net_init(VirtIOS390Device *s390_dev)
-{
-    DeviceState *qdev = DEVICE(s390_dev);
-    VirtIONetS390 *dev = VIRTIO_NET_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-
-    virtio_net_set_config_size(&dev->vdev, s390_dev->host_features);
-    virtio_net_set_netclient_name(&dev->vdev, qdev->id,
-                                  object_get_typename(OBJECT(qdev)));
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_net_instance_init(Object *obj)
-{
-    VirtIONetS390 *dev = VIRTIO_NET_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_NET);
-    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
-                              "bootindex", &error_abort);
-}
-
-static int s390_virtio_blk_init(VirtIOS390Device *s390_dev)
-{
-    VirtIOBlkS390 *dev = VIRTIO_BLK_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_blk_instance_init(Object *obj)
-{
-    VirtIOBlkS390 *dev = VIRTIO_BLK_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_BLK);
-    object_property_add_alias(obj, "iothread", OBJECT(&dev->vdev),"iothread",
-                              &error_abort);
-    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
-                              "bootindex", &error_abort);
-}
-
-static int s390_virtio_serial_init(VirtIOS390Device *s390_dev)
-{
-    VirtIOSerialS390 *dev = VIRTIO_SERIAL_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    DeviceState *qdev = DEVICE(s390_dev);
-    VirtIOS390Bus *bus;
-    int r;
-    char *bus_name;
-
-    bus = DO_UPCAST(VirtIOS390Bus, bus, qdev->parent_bus);
-
-    /*
-     * For command line compatibility, this sets the virtio-serial-device bus
-     * name as before.
-     */
-    if (qdev->id) {
-        bus_name = g_strdup_printf("%s.0", qdev->id);
-        virtio_device_set_child_bus_name(VIRTIO_DEVICE(vdev), bus_name);
-        g_free(bus_name);
-    }
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    r = s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-    if (!r) {
-        bus->console = s390_dev;
-    }
-
-    return r;
-}
-
-static void s390_virtio_serial_instance_init(Object *obj)
-{
-    VirtIOSerialS390 *dev = VIRTIO_SERIAL_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_SERIAL);
-}
-
-static int s390_virtio_scsi_init(VirtIOS390Device *s390_dev)
-{
-    VirtIOSCSIS390 *dev = VIRTIO_SCSI_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-    DeviceState *qdev = DEVICE(s390_dev);
-    char *bus_name;
-
-    /*
-     * For command line compatibility, this sets the virtio-scsi-device bus
-     * name as before.
-     */
-    if (qdev->id) {
-        bus_name = g_strdup_printf("%s.0", qdev->id);
-        virtio_device_set_child_bus_name(VIRTIO_DEVICE(vdev), bus_name);
-        g_free(bus_name);
-    }
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_scsi_instance_init(Object *obj)
-{
-    VirtIOSCSIS390 *dev = VIRTIO_SCSI_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_SCSI);
-}
-
-#ifdef CONFIG_VHOST_SCSI
-static int s390_vhost_scsi_init(VirtIOS390Device *s390_dev)
-{
-    VHostSCSIS390 *dev = VHOST_SCSI_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_vhost_scsi_instance_init(Object *obj)
-{
-    VHostSCSIS390 *dev = VHOST_SCSI_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VHOST_SCSI);
-}
-#endif
-
-
-static int s390_virtio_rng_init(VirtIOS390Device *s390_dev)
-{
-    VirtIORNGS390 *dev = VIRTIO_RNG_S390(s390_dev);
-    DeviceState *vdev = DEVICE(&dev->vdev);
-
-    qdev_set_parent_bus(vdev, BUS(&s390_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    object_property_set_link(OBJECT(dev),
-                             OBJECT(dev->vdev.conf.rng), "rng",
-                             NULL);
-
-    return s390_virtio_device_init(s390_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void s390_virtio_rng_instance_init(Object *obj)
-{
-    VirtIORNGS390 *dev = VIRTIO_RNG_S390(obj);
-
-    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
-                                TYPE_VIRTIO_RNG);
-    object_property_add_alias(obj, "rng", OBJECT(&dev->vdev),
-                              "rng", &error_abort);
-}
-
-static uint64_t s390_virtio_device_vq_token(VirtIOS390Device *dev, int vq)
-{
-    ram_addr_t token_off;
-
-    token_off = (dev->dev_offs + VIRTIO_DEV_OFFS_CONFIG) +
-                (vq * VIRTIO_VQCONFIG_LEN) +
-                VIRTIO_VQCONFIG_OFFS_TOKEN;
-
-    return ldq_be_phys(&address_space_memory, token_off);
-}
-
-static ram_addr_t s390_virtio_device_num_vq(VirtIOS390Device *dev)
-{
-    VirtIODevice *vdev = dev->vdev;
-    int num_vq;
-
-    for (num_vq = 0; num_vq < VIRTIO_PCI_QUEUE_MAX; num_vq++) {
-        if (!virtio_queue_get_num(vdev, num_vq)) {
-            break;
-        }
-    }
-
-    return num_vq;
-}
-
-static ram_addr_t s390_virtio_next_ring(VirtIOS390Bus *bus)
-{
-    ram_addr_t r = bus->next_ring;
-
-    bus->next_ring += VIRTIO_RING_LEN;
-    return r;
-}
-
-void s390_virtio_device_sync(VirtIOS390Device *dev)
-{
-    VirtIOS390Bus *bus = DO_UPCAST(VirtIOS390Bus, bus, dev->qdev.parent_bus);
-    ram_addr_t cur_offs;
-    uint8_t num_vq;
-    int i;
-
-    virtio_reset(dev->vdev);
-
-    /* Sync dev space */
-    stb_phys(&address_space_memory,
-             dev->dev_offs + VIRTIO_DEV_OFFS_TYPE, dev->vdev->device_id);
-
-    stb_phys(&address_space_memory,
-             dev->dev_offs + VIRTIO_DEV_OFFS_NUM_VQ,
-             s390_virtio_device_num_vq(dev));
-    stb_phys(&address_space_memory,
-             dev->dev_offs + VIRTIO_DEV_OFFS_FEATURE_LEN, dev->feat_len);
-
-    stb_phys(&address_space_memory,
-             dev->dev_offs + VIRTIO_DEV_OFFS_CONFIG_LEN, dev->vdev->config_len);
-
-    num_vq = s390_virtio_device_num_vq(dev);
-    stb_phys(&address_space_memory,
-             dev->dev_offs + VIRTIO_DEV_OFFS_NUM_VQ, num_vq);
-
-    /* Sync virtqueues */
-    for (i = 0; i < num_vq; i++) {
-        ram_addr_t vq = (dev->dev_offs + VIRTIO_DEV_OFFS_CONFIG) +
-                        (i * VIRTIO_VQCONFIG_LEN);
-        ram_addr_t vring;
-
-        vring = s390_virtio_next_ring(bus);
-        virtio_queue_set_addr(dev->vdev, i, vring);
-        virtio_queue_set_vector(dev->vdev, i, i);
-        stq_be_phys(&address_space_memory,
-                    vq + VIRTIO_VQCONFIG_OFFS_ADDRESS, vring);
-        stw_be_phys(&address_space_memory,
-                    vq + VIRTIO_VQCONFIG_OFFS_NUM,
-                    virtio_queue_get_num(dev->vdev, i));
-    }
-
-    cur_offs = dev->dev_offs;
-    cur_offs += VIRTIO_DEV_OFFS_CONFIG;
-    cur_offs += num_vq * VIRTIO_VQCONFIG_LEN;
-
-    /* Sync feature bitmap */
-    stl_le_phys(&address_space_memory, cur_offs, dev->host_features);
-
-    dev->feat_offs = cur_offs + dev->feat_len;
-    cur_offs += dev->feat_len * 2;
-
-    /* Sync config space */
-    virtio_bus_get_vdev_config(&dev->bus, dev->vdev->config);
-
-    cpu_physical_memory_write(cur_offs,
-                              dev->vdev->config, dev->vdev->config_len);
-    cur_offs += dev->vdev->config_len;
-}
-
-void s390_virtio_device_update_status(VirtIOS390Device *dev)
-{
-    VirtIODevice *vdev = dev->vdev;
-    uint32_t features;
-
-    virtio_set_status(vdev, ldub_phys(&address_space_memory,
-                                      dev->dev_offs + VIRTIO_DEV_OFFS_STATUS));
-
-    /* Update guest supported feature bitmap */
-
-    features = bswap32(ldl_be_phys(&address_space_memory, dev->feat_offs));
-    virtio_set_features(vdev, features);
-}
-
-VirtIOS390Device *s390_virtio_bus_console(VirtIOS390Bus *bus)
-{
-    return bus->console;
-}
-
-/* Find a device by vring address */
-VirtIOS390Device *s390_virtio_bus_find_vring(VirtIOS390Bus *bus,
-                                             ram_addr_t mem,
-                                             int *vq_num)
-{
-    BusChild *kid;
-    int i;
-
-    QTAILQ_FOREACH(kid, &bus->bus.children, sibling) {
-        VirtIOS390Device *dev = (VirtIOS390Device *)kid->child;
-
-        for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
-            if (!virtio_queue_get_addr(dev->vdev, i))
-                break;
-            if (virtio_queue_get_addr(dev->vdev, i) == mem) {
-                if (vq_num) {
-                    *vq_num = i;
-                }
-                return dev;
-            }
-        }
-    }
-
-    return NULL;
-}
-
-/* Find a device by device descriptor location */
-VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem)
-{
-    BusChild *kid;
-
-    QTAILQ_FOREACH(kid, &bus->bus.children, sibling) {
-        VirtIOS390Device *dev = (VirtIOS390Device *)kid->child;
-        if (dev->dev_offs == mem) {
-            return dev;
-        }
-    }
-
-    return NULL;
-}
-
-/* DeviceState to VirtIOS390Device. Note: used on datapath,
- * be careful and test performance if you change this.
- */
-static inline VirtIOS390Device *to_virtio_s390_device_fast(DeviceState *d)
-{
-    return container_of(d, VirtIOS390Device, qdev);
-}
-
-/* DeviceState to VirtIOS390Device. TODO: use QOM. */
-static inline VirtIOS390Device *to_virtio_s390_device(DeviceState *d)
-{
-    return container_of(d, VirtIOS390Device, qdev);
-}
-
-static void virtio_s390_notify(DeviceState *d, uint16_t vector)
-{
-    VirtIOS390Device *dev = to_virtio_s390_device_fast(d);
-    uint64_t token = s390_virtio_device_vq_token(dev, vector);
-
-    s390_virtio_irq(0, token);
-}
-
-static unsigned virtio_s390_get_features(DeviceState *d)
-{
-    VirtIOS390Device *dev = to_virtio_s390_device(d);
-    return dev->host_features;
-}
-
-/**************** S390 Virtio Bus Device Descriptions *******************/
-
-static Property s390_virtio_net_properties[] = {
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_VIRTIO_NET_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void s390_virtio_net_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_net_init;
-    dc->props = s390_virtio_net_properties;
-}
-
-static const TypeInfo s390_virtio_net = {
-    .name          = TYPE_VIRTIO_NET_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIONetS390),
-    .instance_init = s390_virtio_net_instance_init,
-    .class_init    = s390_virtio_net_class_init,
-};
-
-static void s390_virtio_blk_class_init(ObjectClass *klass, void *data)
-{
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_blk_init;
-}
-
-static const TypeInfo s390_virtio_blk = {
-    .name          = "virtio-blk-s390",
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIOBlkS390),
-    .instance_init = s390_virtio_blk_instance_init,
-    .class_init    = s390_virtio_blk_class_init,
-};
-
-static Property s390_virtio_serial_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void s390_virtio_serial_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_serial_init;
-    dc->props = s390_virtio_serial_properties;
-}
-
-static const TypeInfo s390_virtio_serial = {
-    .name          = TYPE_VIRTIO_SERIAL_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIOSerialS390),
-    .instance_init = s390_virtio_serial_instance_init,
-    .class_init    = s390_virtio_serial_class_init,
-};
-
-static Property s390_virtio_rng_properties[] = {
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void s390_virtio_rng_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_rng_init;
-    dc->props = s390_virtio_rng_properties;
-}
-
-static const TypeInfo s390_virtio_rng = {
-    .name          = TYPE_VIRTIO_RNG_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIORNGS390),
-    .instance_init = s390_virtio_rng_instance_init,
-    .class_init    = s390_virtio_rng_class_init,
-};
-
-static int s390_virtio_busdev_init(DeviceState *dev)
-{
-    VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
-    VirtIOS390DeviceClass *_info = VIRTIO_S390_DEVICE_GET_CLASS(dev);
-
-    virtio_s390_bus_new(&_dev->bus, sizeof(_dev->bus), _dev);
-
-    return _info->init(_dev);
-}
-
-static void s390_virtio_busdev_reset(DeviceState *dev)
-{
-    VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
-
-    virtio_reset(_dev->vdev);
-}
-
-static void virtio_s390_device_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-
-    dc->init = s390_virtio_busdev_init;
-    dc->bus_type = TYPE_S390_VIRTIO_BUS;
-    dc->reset = s390_virtio_busdev_reset;
-}
-
-static const TypeInfo virtio_s390_device_info = {
-    .name = TYPE_VIRTIO_S390_DEVICE,
-    .parent = TYPE_DEVICE,
-    .instance_size = sizeof(VirtIOS390Device),
-    .class_init = virtio_s390_device_class_init,
-    .class_size = sizeof(VirtIOS390DeviceClass),
-    .abstract = true,
-};
-
-static Property s390_virtio_scsi_properties[] = {
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_VIRTIO_SCSI_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void s390_virtio_scsi_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_scsi_init;
-    dc->props = s390_virtio_scsi_properties;
-}
-
-static const TypeInfo s390_virtio_scsi = {
-    .name          = TYPE_VIRTIO_SCSI_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VirtIOSCSIS390),
-    .instance_init = s390_virtio_scsi_instance_init,
-    .class_init    = s390_virtio_scsi_class_init,
-};
-
-#ifdef CONFIG_VHOST_SCSI
-static Property s390_vhost_scsi_properties[] = {
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOS390Device, host_features),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void s390_vhost_scsi_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
-
-    k->init = s390_vhost_scsi_init;
-    dc->props = s390_vhost_scsi_properties;
-}
-
-static const TypeInfo s390_vhost_scsi = {
-    .name          = TYPE_VHOST_SCSI_S390,
-    .parent        = TYPE_VIRTIO_S390_DEVICE,
-    .instance_size = sizeof(VHostSCSIS390),
-    .instance_init = s390_vhost_scsi_instance_init,
-    .class_init    = s390_vhost_scsi_class_init,
-};
-#endif
-
-/***************** S390 Virtio Bus Bridge Device *******************/
-/* Only required to have the virtio bus as child in the system bus */
-
-static int s390_virtio_bridge_init(SysBusDevice *dev)
-{
-    /* nothing */
-    return 0;
-}
-
-static void s390_virtio_bridge_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-
-    k->init = s390_virtio_bridge_init;
-}
-
-static const TypeInfo s390_virtio_bridge_info = {
-    .name          = "s390-virtio-bridge",
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(SysBusDevice),
-    .class_init    = s390_virtio_bridge_class_init,
-    .interfaces = (InterfaceInfo[]) {
-        { TYPE_HOTPLUG_HANDLER },
-        { }
-    }
-};
-
-/* virtio-s390-bus */
-
-static void virtio_s390_bus_new(VirtioBusState *bus, size_t bus_size,
-                                VirtIOS390Device *dev)
-{
-    DeviceState *qdev = DEVICE(dev);
-    char virtio_bus_name[] = "virtio-bus";
-
-    qbus_create_inplace(bus, bus_size, TYPE_VIRTIO_S390_BUS,
-                        qdev, virtio_bus_name);
-}
-
-static void virtio_s390_bus_class_init(ObjectClass *klass, void *data)
-{
-    VirtioBusClass *k = VIRTIO_BUS_CLASS(klass);
-    BusClass *bus_class = BUS_CLASS(klass);
-    bus_class->max_dev = 1;
-    k->notify = virtio_s390_notify;
-    k->get_features = virtio_s390_get_features;
-}
-
-static const TypeInfo virtio_s390_bus_info = {
-    .name          = TYPE_VIRTIO_S390_BUS,
-    .parent        = TYPE_VIRTIO_BUS,
-    .instance_size = sizeof(VirtioS390BusState),
-    .class_init    = virtio_s390_bus_class_init,
-};
-
-static void s390_virtio_register_types(void)
-{
-    type_register_static(&virtio_s390_bus_info);
-    type_register_static(&s390_virtio_bus_info);
-    type_register_static(&virtio_s390_device_info);
-    type_register_static(&s390_virtio_serial);
-    type_register_static(&s390_virtio_blk);
-    type_register_static(&s390_virtio_net);
-    type_register_static(&s390_virtio_scsi);
-#ifdef CONFIG_VHOST_SCSI
-    type_register_static(&s390_vhost_scsi);
-#endif
-    type_register_static(&s390_virtio_rng);
-    type_register_static(&s390_virtio_bridge_info);
-}
-
-type_init(s390_virtio_register_types)
diff --git a/hw/s390x/s390-virtio-bus.h b/hw/s390x/s390-virtio-bus.h
deleted file mode 100644
index ffd0df7..0000000
--- a/hw/s390x/s390-virtio-bus.h
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * QEMU S390x VirtIO BUS definitions
- *
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef HW_S390_VIRTIO_BUS_H
-#define HW_S390_VIRTIO_BUS_H 1
-
-#include "hw/virtio/virtio-blk.h"
-#include "hw/virtio/virtio-net.h"
-#include "hw/virtio/virtio-rng.h"
-#include "hw/virtio/virtio-serial.h"
-#include "hw/virtio/virtio-scsi.h"
-#include "hw/virtio/virtio-bus.h"
-#ifdef CONFIG_VHOST_SCSI
-#include "hw/virtio/vhost-scsi.h"
-#endif
-
-#define VIRTIO_DEV_OFFS_TYPE		0	/* 8 bits */
-#define VIRTIO_DEV_OFFS_NUM_VQ		1	/* 8 bits */
-#define VIRTIO_DEV_OFFS_FEATURE_LEN	2	/* 8 bits */
-#define VIRTIO_DEV_OFFS_CONFIG_LEN	3	/* 8 bits */
-#define VIRTIO_DEV_OFFS_STATUS		4	/* 8 bits */
-#define VIRTIO_DEV_OFFS_CONFIG		5	/* dynamic */
-
-#define VIRTIO_VQCONFIG_OFFS_TOKEN	0	/* 64 bits */
-#define VIRTIO_VQCONFIG_OFFS_ADDRESS	8	/* 64 bits */
-#define VIRTIO_VQCONFIG_OFFS_NUM	16	/* 16 bits */
-#define VIRTIO_VQCONFIG_LEN		24
-
-#define VIRTIO_RING_LEN			(TARGET_PAGE_SIZE * 3)
-#define VIRTIO_VRING_AVAIL_IDX_OFFS 2
-#define VIRTIO_VRING_USED_IDX_OFFS 2
-#define S390_DEVICE_PAGES		512
-
-#define VIRTIO_PARAM_MASK               0xff
-#define VIRTIO_PARAM_VRING_INTERRUPT    0x0
-#define VIRTIO_PARAM_CONFIG_CHANGED     0x1
-#define VIRTIO_PARAM_DEV_ADD            0x2
-
-#define TYPE_VIRTIO_S390_DEVICE "virtio-s390-device"
-#define VIRTIO_S390_DEVICE(obj) \
-     OBJECT_CHECK(VirtIOS390Device, (obj), TYPE_VIRTIO_S390_DEVICE)
-#define VIRTIO_S390_DEVICE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(VirtIOS390DeviceClass, (klass), TYPE_VIRTIO_S390_DEVICE)
-#define VIRTIO_S390_DEVICE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(VirtIOS390DeviceClass, (obj), TYPE_VIRTIO_S390_DEVICE)
-
-#define TYPE_S390_VIRTIO_BUS "s390-virtio-bus"
-#define S390_VIRTIO_BUS(obj) \
-     OBJECT_CHECK(VirtIOS390Bus, (obj), TYPE_S390_VIRTIO_BUS)
-
-/* virtio-s390-bus */
-
-typedef struct VirtioBusState VirtioS390BusState;
-typedef struct VirtioBusClass VirtioS390BusClass;
-
-#define TYPE_VIRTIO_S390_BUS "virtio-s390-bus"
-#define VIRTIO_S390_BUS(obj) \
-        OBJECT_CHECK(VirtioS390BusState, (obj), TYPE_VIRTIO_S390_BUS)
-#define VIRTIO_S390_BUS_GET_CLASS(obj) \
-        OBJECT_GET_CLASS(VirtioS390BusClass, obj, TYPE_VIRTIO_S390_BUS)
-#define VIRTIO_S390_BUS_CLASS(klass) \
-        OBJECT_CLASS_CHECK(VirtioS390BusClass, klass, TYPE_VIRTIO_S390_BUS)
-
-
-typedef struct VirtIOS390Device VirtIOS390Device;
-
-typedef struct VirtIOS390DeviceClass {
-    DeviceClass qdev;
-    int (*init)(VirtIOS390Device *dev);
-} VirtIOS390DeviceClass;
-
-struct VirtIOS390Device {
-    DeviceState qdev;
-    ram_addr_t dev_offs;
-    ram_addr_t feat_offs;
-    uint8_t feat_len;
-    VirtIODevice *vdev;
-    uint32_t host_features;
-    VirtioBusState bus;
-};
-
-typedef struct VirtIOS390Bus {
-    BusState bus;
-
-    VirtIOS390Device *console;
-    ram_addr_t dev_page;
-    ram_addr_t dev_offs;
-    ram_addr_t next_ring;
-} VirtIOS390Bus;
-
-
-void s390_virtio_device_update_status(VirtIOS390Device *dev);
-
-VirtIOS390Device *s390_virtio_bus_console(VirtIOS390Bus *bus);
-VirtIOS390Bus *s390_virtio_bus_init(ram_addr_t *ram_size);
-
-VirtIOS390Device *s390_virtio_bus_find_vring(VirtIOS390Bus *bus,
-                                             ram_addr_t mem, int *vq_num);
-VirtIOS390Device *s390_virtio_bus_find_mem(VirtIOS390Bus *bus, ram_addr_t mem);
-void s390_virtio_device_sync(VirtIOS390Device *dev);
-void s390_virtio_reset_idx(VirtIOS390Device *dev);
-
-/* virtio-blk-s390 */
-
-#define TYPE_VIRTIO_BLK_S390 "virtio-blk-s390"
-#define VIRTIO_BLK_S390(obj) \
-        OBJECT_CHECK(VirtIOBlkS390, (obj), TYPE_VIRTIO_BLK_S390)
-
-typedef struct VirtIOBlkS390 {
-    VirtIOS390Device parent_obj;
-    VirtIOBlock vdev;
-} VirtIOBlkS390;
-
-/* virtio-scsi-s390 */
-
-#define TYPE_VIRTIO_SCSI_S390 "virtio-scsi-s390"
-#define VIRTIO_SCSI_S390(obj) \
-        OBJECT_CHECK(VirtIOSCSIS390, (obj), TYPE_VIRTIO_SCSI_S390)
-
-typedef struct VirtIOSCSIS390 {
-    VirtIOS390Device parent_obj;
-    VirtIOSCSI vdev;
-} VirtIOSCSIS390;
-
-/* virtio-serial-s390 */
-
-#define TYPE_VIRTIO_SERIAL_S390 "virtio-serial-s390"
-#define VIRTIO_SERIAL_S390(obj) \
-        OBJECT_CHECK(VirtIOSerialS390, (obj), TYPE_VIRTIO_SERIAL_S390)
-
-typedef struct VirtIOSerialS390 {
-    VirtIOS390Device parent_obj;
-    VirtIOSerial vdev;
-} VirtIOSerialS390;
-
-/* virtio-net-s390 */
-
-#define TYPE_VIRTIO_NET_S390 "virtio-net-s390"
-#define VIRTIO_NET_S390(obj) \
-        OBJECT_CHECK(VirtIONetS390, (obj), TYPE_VIRTIO_NET_S390)
-
-typedef struct VirtIONetS390 {
-    VirtIOS390Device parent_obj;
-    VirtIONet vdev;
-} VirtIONetS390;
-
-/* vhost-scsi-s390 */
-
-#ifdef CONFIG_VHOST_SCSI
-#define TYPE_VHOST_SCSI_S390 "vhost-scsi-s390"
-#define VHOST_SCSI_S390(obj) \
-        OBJECT_CHECK(VHostSCSIS390, (obj), TYPE_VHOST_SCSI_S390)
-
-typedef struct VHostSCSIS390 {
-    VirtIOS390Device parent_obj;
-    VHostSCSI vdev;
-} VHostSCSIS390;
-#endif
-
-/* virtio-rng-s390 */
-
-#define TYPE_VIRTIO_RNG_S390 "virtio-rng-s390"
-#define VIRTIO_RNG_S390(obj) \
-        OBJECT_CHECK(VirtIORNGS390, (obj), TYPE_VIRTIO_RNG_S390)
-
-typedef struct VirtIORNGS390 {
-    VirtIOS390Device parent_obj;
-    VirtIORNG vdev;
-} VirtIORNGS390;
-
-#endif
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index bc4dc2a..91d9cef 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -9,34 +9,43 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/boards.h"
 #include "exec/address-spaces.h"
 #include "s390-virtio.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
-#include "ioinst.h"
-#include "css.h"
+#include "hw/s390x/ioinst.h"
+#include "hw/s390x/css.h"
 #include "virtio-ccw.h"
 #include "qemu/config-file.h"
+#include "s390-pci-bus.h"
+#include "hw/s390x/storage-keys.h"
+#include "hw/compat.h"
+#include "ipl.h"
+#include "hw/s390x/s390-virtio-ccw.h"
+#include "hw/s390x/css-bridge.h"
 
-#define TYPE_S390_CCW_MACHINE               "s390-ccw-machine"
+static const char *const reset_dev_types[] = {
+    TYPE_VIRTUAL_CSS_BRIDGE,
+    "s390-sclp-event-facility",
+    "s390-flic",
+    "diag288",
+};
 
-void io_subsystem_reset(void)
+void subsystem_reset(void)
 {
-    DeviceState *css, *sclp, *flic;
+    DeviceState *dev;
+    int i;
 
-    css = DEVICE(object_resolve_path_type("", "virtual-css-bridge", NULL));
-    if (css) {
-        qdev_reset_all(css);
-    }
-    sclp = DEVICE(object_resolve_path_type("",
-                  "s390-sclp-event-facility", NULL));
-    if (sclp) {
-        qdev_reset_all(sclp);
-    }
-    flic = DEVICE(object_resolve_path_type("", "s390-flic", NULL));
-    if (flic) {
-        qdev_reset_all(flic);
+    for (i = 0; i < ARRAY_SIZE(reset_dev_types); i++) {
+        dev = DEVICE(object_resolve_path_type("", reset_dev_types[i], NULL));
+        if (dev) {
+            qdev_reset_all(dev);
+        }
     }
 }
 
@@ -54,7 +63,7 @@
     if (!sch || !css_subch_visible(sch)) {
         return -EINVAL;
     }
-    if (queue >= VIRTIO_PCI_QUEUE_MAX) {
+    if (queue >= VIRTIO_CCW_QUEUE_MAX) {
         return -EINVAL;
     }
     virtio_queue_notify(virtio_ccw_get_vdev(sch), queue);
@@ -82,75 +91,44 @@
                                    virtio_ccw_hcall_early_printk);
 }
 
-static void ccw_init(MachineState *machine)
+void s390_memory_init(ram_addr_t mem_size)
 {
-    ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
-    uint8_t *storage_keys;
+
+    /* allocate RAM for core */
+    memory_region_allocate_system_memory(ram, NULL, "s390.ram", mem_size);
+    memory_region_add_subregion(sysmem, 0, ram);
+
+    /* Initialize storage key device */
+    s390_skeys_init();
+}
+
+static void ccw_init(MachineState *machine)
+{
     int ret;
     VirtualCssBus *css_bus;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
-    ram_addr_t pad_size = 0;
-    ram_addr_t maxmem = qemu_opt_get_size(opts, "maxmem", my_ram_size);
-    ram_addr_t standby_mem_size = maxmem - my_ram_size;
+    DeviceState *dev;
 
-    /* The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
-     * The variable 'mhd->increment_size' is an exponent of 2 that can be
-     * used to calculate the size (in bytes) of an increment. */
-    mhd->increment_size = 20;
-    while ((my_ram_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
-        mhd->increment_size++;
-    }
-    while ((standby_mem_size >> mhd->increment_size) > MAX_STORAGE_INCREMENTS) {
-        mhd->increment_size++;
-    }
-
-    /* The core and standby memory areas need to be aligned with
-     * the increment size.  In effect, this can cause the
-     * user-specified memory size to be rounded down to align
-     * with the nearest increment boundary. */
-    standby_mem_size = standby_mem_size >> mhd->increment_size
-                                        << mhd->increment_size;
-    my_ram_size = my_ram_size >> mhd->increment_size
-                              << mhd->increment_size;
-
-    /* let's propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
+    s390_sclp_init();
+    s390_memory_init(machine->ram_size);
 
     /* get a BUS */
     css_bus = virtual_css_bus_init();
-    s390_sclp_init();
     s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
-                      machine->initrd_filename, "s390-ccw.img");
+                      machine->initrd_filename, "s390-ccw.img", true);
     s390_flic_init();
 
+    dev = qdev_create(NULL, TYPE_S390_PCI_HOST_BRIDGE);
+    object_property_add_child(qdev_get_machine(), TYPE_S390_PCI_HOST_BRIDGE,
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+
     /* register hypercalls */
     virtio_ccw_register_hcalls();
 
-    /* allocate RAM for core */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
-
-    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
-       calculate the pad size necessary to force this boundary. */
-    if (standby_mem_size) {
-        if (my_ram_size % MEM_SECTION_SIZE) {
-            pad_size = MEM_SECTION_SIZE - my_ram_size % MEM_SECTION_SIZE;
-        }
-        my_ram_size += standby_mem_size + pad_size;
-        mhd->pad_size = pad_size;
-        mhd->standby_mem_size = standby_mem_size;
-    }
-
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
     /* init CPUs */
-    s390_init_cpus(machine->cpu_model, storage_keys);
+    s390_init_cpus(machine);
 
     if (kvm_enabled()) {
         kvm_s390_enable_css_support(s390_cpu_addr2state(0));
@@ -164,38 +142,284 @@
 
     /* Create VirtIO network adapters */
     s390_create_virtio_net(BUS(css_bus), "virtio-net-ccw");
+
+    /* Register savevm handler for guest TOD clock */
+    register_savevm(NULL, "todclock", 0, 1,
+                    gtod_save, gtod_load, kvm_state);
+}
+
+static void s390_cpu_plug(HotplugHandler *hotplug_dev,
+                        DeviceState *dev, Error **errp)
+{
+    gchar *name;
+    S390CPU *cpu = S390_CPU(dev);
+    CPUState *cs = CPU(dev);
+
+    name = g_strdup_printf("cpu[%i]", cpu->env.cpu_num);
+    object_property_set_link(OBJECT(hotplug_dev), OBJECT(cs), name,
+                             errp);
+    g_free(name);
+}
+
+static void s390_machine_device_plug(HotplugHandler *hotplug_dev,
+                                     DeviceState *dev, Error **errp)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        s390_cpu_plug(hotplug_dev, dev, errp);
+    }
+}
+
+static HotplugHandler *s390_get_hotplug_handler(MachineState *machine,
+                                                DeviceState *dev)
+{
+    if (object_dynamic_cast(OBJECT(dev), TYPE_CPU)) {
+        return HOTPLUG_HANDLER(machine);
+    }
+    return NULL;
+}
+
+static void s390_hot_add_cpu(const int64_t id, Error **errp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+
+    s390x_new_cpu(machine->cpu_model, id, errp);
 }
 
 static void ccw_machine_class_init(ObjectClass *oc, void *data)
 {
     MachineClass *mc = MACHINE_CLASS(oc);
     NMIClass *nc = NMI_CLASS(oc);
+    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(oc);
+    S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
 
-    mc->name = "s390-ccw-virtio";
-    mc->alias = "s390-ccw";
-    mc->desc = "VirtIO-ccw based S390 machine";
+    s390mc->ri_allowed = true;
     mc->init = ccw_init;
+    mc->reset = s390_machine_reset;
+    mc->hot_add_cpu = s390_hot_add_cpu;
     mc->block_default_type = IF_VIRTIO;
     mc->no_cdrom = 1;
     mc->no_floppy = 1;
     mc->no_serial = 1;
     mc->no_parallel = 1;
     mc->no_sdcard = 1;
-    mc->use_sclp = 1,
-    mc->max_cpus = 255;
+    mc->use_sclp = 1;
+    mc->max_cpus = 248;
+    mc->get_hotplug_handler = s390_get_hotplug_handler;
+    hc->plug = s390_machine_device_plug;
     nc->nmi_monitor_handler = s390_nmi;
 }
 
+static inline bool machine_get_aes_key_wrap(Object *obj, Error **errp)
+{
+    S390CcwMachineState *ms = S390_CCW_MACHINE(obj);
+
+    return ms->aes_key_wrap;
+}
+
+static inline void machine_set_aes_key_wrap(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390CcwMachineState *ms = S390_CCW_MACHINE(obj);
+
+    ms->aes_key_wrap = value;
+}
+
+static inline bool machine_get_dea_key_wrap(Object *obj, Error **errp)
+{
+    S390CcwMachineState *ms = S390_CCW_MACHINE(obj);
+
+    return ms->dea_key_wrap;
+}
+
+static inline void machine_set_dea_key_wrap(Object *obj, bool value,
+                                            Error **errp)
+{
+    S390CcwMachineState *ms = S390_CCW_MACHINE(obj);
+
+    ms->dea_key_wrap = value;
+}
+
+bool ri_allowed(void)
+{
+    if (kvm_enabled()) {
+        MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+        if (object_class_dynamic_cast(OBJECT_CLASS(mc),
+                                      TYPE_S390_CCW_MACHINE)) {
+            S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
+
+            return s390mc->ri_allowed;
+        }
+    }
+    return 0;
+}
+
+static inline void s390_machine_initfn(Object *obj)
+{
+    object_property_add_bool(obj, "aes-key-wrap",
+                             machine_get_aes_key_wrap,
+                             machine_set_aes_key_wrap, NULL);
+    object_property_set_description(obj, "aes-key-wrap",
+            "enable/disable AES key wrapping using the CPACF wrapping key",
+            NULL);
+    object_property_set_bool(obj, true, "aes-key-wrap", NULL);
+
+    object_property_add_bool(obj, "dea-key-wrap",
+                             machine_get_dea_key_wrap,
+                             machine_set_dea_key_wrap, NULL);
+    object_property_set_description(obj, "dea-key-wrap",
+            "enable/disable DEA key wrapping using the CPACF wrapping key",
+            NULL);
+    object_property_set_bool(obj, true, "dea-key-wrap", NULL);
+}
+
 static const TypeInfo ccw_machine_info = {
     .name          = TYPE_S390_CCW_MACHINE,
     .parent        = TYPE_MACHINE,
+    .abstract      = true,
+    .instance_size = sizeof(S390CcwMachineState),
+    .instance_init = s390_machine_initfn,
+    .class_size = sizeof(S390CcwMachineClass),
     .class_init    = ccw_machine_class_init,
     .interfaces = (InterfaceInfo[]) {
         { TYPE_NMI },
+        { TYPE_HOTPLUG_HANDLER},
         { }
     },
 };
 
+#define DEFINE_CCW_MACHINE(suffix, verstr, latest)                            \
+    static void ccw_machine_##suffix##_class_init(ObjectClass *oc,            \
+                                                  void *data)                 \
+    {                                                                         \
+        MachineClass *mc = MACHINE_CLASS(oc);                                 \
+        ccw_machine_##suffix##_class_options(mc);                             \
+        mc->desc = "VirtIO-ccw based S390 machine v" verstr;                  \
+        if (latest) {                                                         \
+            mc->alias = "s390-ccw-virtio";                                    \
+            mc->is_default = 1;                                               \
+        }                                                                     \
+    }                                                                         \
+    static void ccw_machine_##suffix##_instance_init(Object *obj)             \
+    {                                                                         \
+        MachineState *machine = MACHINE(obj);                                 \
+        ccw_machine_##suffix##_instance_options(machine);                     \
+    }                                                                         \
+    static const TypeInfo ccw_machine_##suffix##_info = {                     \
+        .name = MACHINE_TYPE_NAME("s390-ccw-virtio-" verstr),                 \
+        .parent = TYPE_S390_CCW_MACHINE,                                      \
+        .class_init = ccw_machine_##suffix##_class_init,                      \
+        .instance_init = ccw_machine_##suffix##_instance_init,                \
+    };                                                                        \
+    static void ccw_machine_register_##suffix(void)                           \
+    {                                                                         \
+        type_register_static(&ccw_machine_##suffix##_info);                   \
+    }                                                                         \
+    type_init(ccw_machine_register_##suffix)
+
+#define CCW_COMPAT_2_6 \
+        HW_COMPAT_2_6 \
+        {\
+            .driver   = TYPE_S390_IPL,\
+            .property = "iplbext_migration",\
+            .value    = "off",\
+        }, {\
+            .driver   = TYPE_VIRTUAL_CSS_BRIDGE,\
+            .property = "css_dev_path",\
+            .value    = "off",\
+        },
+
+#define CCW_COMPAT_2_5 \
+        CCW_COMPAT_2_6 \
+        HW_COMPAT_2_5
+
+#define CCW_COMPAT_2_4 \
+        HW_COMPAT_2_4 \
+        {\
+            .driver   = TYPE_S390_SKEYS,\
+            .property = "migration-enabled",\
+            .value    = "off",\
+        },{\
+            .driver   = "virtio-blk-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-balloon-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-serial-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-9p-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-rng-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-net-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "virtio-scsi-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },{\
+            .driver   = "vhost-scsi-ccw",\
+            .property = "max_revision",\
+            .value    = "0",\
+        },
+
+static void ccw_machine_2_7_instance_options(MachineState *machine)
+{
+}
+
+static void ccw_machine_2_7_class_options(MachineClass *mc)
+{
+}
+DEFINE_CCW_MACHINE(2_7, "2.7", true);
+
+static void ccw_machine_2_6_instance_options(MachineState *machine)
+{
+    ccw_machine_2_7_instance_options(machine);
+}
+
+static void ccw_machine_2_6_class_options(MachineClass *mc)
+{
+    S390CcwMachineClass *s390mc = S390_MACHINE_CLASS(mc);
+
+    s390mc->ri_allowed = false;
+    ccw_machine_2_7_class_options(mc);
+    SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_6);
+}
+DEFINE_CCW_MACHINE(2_6, "2.6", false);
+
+static void ccw_machine_2_5_instance_options(MachineState *machine)
+{
+    ccw_machine_2_6_instance_options(machine);
+}
+
+static void ccw_machine_2_5_class_options(MachineClass *mc)
+{
+    ccw_machine_2_6_class_options(mc);
+    SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_5);
+}
+DEFINE_CCW_MACHINE(2_5, "2.5", false);
+
+static void ccw_machine_2_4_instance_options(MachineState *machine)
+{
+    ccw_machine_2_5_instance_options(machine);
+}
+
+static void ccw_machine_2_4_class_options(MachineClass *mc)
+{
+    ccw_machine_2_5_class_options(mc);
+    SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_4);
+}
+DEFINE_CCW_MACHINE(2_4, "2.4", false);
+
 static void ccw_machine_register_types(void)
 {
     type_register_static(&ccw_machine_info);
diff --git a/hw/s390x/s390-virtio-hcall.c b/hw/s390x/s390-virtio-hcall.c
index c7bdc20..23d67d6 100644
--- a/hw/s390x/s390-virtio-hcall.c
+++ b/hw/s390x/s390-virtio-hcall.c
@@ -9,6 +9,7 @@
  * directory.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "hw/s390x/s390-virtio.h"
 
diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
index c215cd8..544c616 100644
--- a/hw/s390x/s390-virtio.c
+++ b/hw/s390x/s390-virtio.c
@@ -21,23 +21,28 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
 #include "net/net.h"
 #include "hw/boards.h"
-#include "monitor/monitor.h"
 #include "hw/loader.h"
 #include "hw/virtio/virtio.h"
-#include "hw/sysbus.h"
 #include "sysemu/kvm.h"
 #include "exec/address-spaces.h"
+#include "sysemu/qtest.h"
 
-#include "hw/s390x/s390-virtio-bus.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/s390_flic.h"
 #include "hw/s390x/s390-virtio.h"
+#include "hw/s390x/storage-keys.h"
+#include "hw/s390x/ipl.h"
+#include "cpu.h"
 
 //#define DEBUG_S390
 
@@ -50,89 +55,31 @@
 #endif
 
 #define MAX_BLK_DEVS                    10
-#define ZIPL_FILENAME                   "s390-zipl.rom"
-#define TYPE_S390_MACHINE               "s390-machine"
 
-static VirtIOS390Bus *s390_bus;
-static S390CPU **ipi_states;
+#define S390_TOD_CLOCK_VALUE_MISSING    0x00
+#define S390_TOD_CLOCK_VALUE_PRESENT    0x01
+
+static S390CPU **cpu_states;
 
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr)
 {
-    if (cpu_addr >= smp_cpus) {
+    if (cpu_addr >= max_cpus) {
         return NULL;
     }
 
-    return ipi_states[cpu_addr];
-}
-
-static int s390_virtio_hcall_notify(const uint64_t *args)
-{
-    uint64_t mem = args[0];
-    int r = 0, i;
-
-    if (mem > ram_size) {
-        VirtIOS390Device *dev = s390_virtio_bus_find_vring(s390_bus, mem, &i);
-        if (dev) {
-            virtio_queue_notify(dev->vdev, i);
-        } else {
-            r = -EINVAL;
-        }
-    } else {
-        /* Early printk */
-    }
-    return r;
-}
-
-static int s390_virtio_hcall_reset(const uint64_t *args)
-{
-    uint64_t mem = args[0];
-    VirtIOS390Device *dev;
-
-    dev = s390_virtio_bus_find_mem(s390_bus, mem);
-    if (dev == NULL) {
-        return -EINVAL;
-    }
-    virtio_reset(dev->vdev);
-    stb_phys(&address_space_memory, dev->dev_offs + VIRTIO_DEV_OFFS_STATUS, 0);
-    s390_virtio_device_sync(dev);
-    s390_virtio_reset_idx(dev);
-
-    return 0;
-}
-
-static int s390_virtio_hcall_set_status(const uint64_t *args)
-{
-    uint64_t mem = args[0];
-    int r = 0;
-    VirtIOS390Device *dev;
-
-    dev = s390_virtio_bus_find_mem(s390_bus, mem);
-    if (dev) {
-        s390_virtio_device_update_status(dev);
-    } else {
-        r = -EINVAL;
-    }
-    return r;
-}
-
-static void s390_virtio_register_hcalls(void)
-{
-    s390_register_virtio_hypercall(KVM_S390_VIRTIO_NOTIFY,
-                                   s390_virtio_hcall_notify);
-    s390_register_virtio_hypercall(KVM_S390_VIRTIO_RESET,
-                                   s390_virtio_hcall_reset);
-    s390_register_virtio_hypercall(KVM_S390_VIRTIO_SET_STATUS,
-                                   s390_virtio_hcall_set_status);
+    /* Fast lookup via CPU ID */
+    return cpu_states[cpu_addr];
 }
 
 void s390_init_ipl_dev(const char *kernel_filename,
                        const char *kernel_cmdline,
                        const char *initrd_filename,
-                       const char *firmware)
+                       const char *firmware,
+                       bool enforce_bios)
 {
-    DeviceState *dev;
+    Object *new = object_new(TYPE_S390_IPL);
+    DeviceState *dev = DEVICE(new);
 
-    dev  = qdev_create(NULL, "s390-ipl");
     if (kernel_filename) {
         qdev_prop_set_string(dev, "kernel", kernel_filename);
     }
@@ -141,30 +88,36 @@
     }
     qdev_prop_set_string(dev, "cmdline", kernel_cmdline);
     qdev_prop_set_string(dev, "firmware", firmware);
+    qdev_prop_set_bit(dev, "enforce_bios", enforce_bios);
+    object_property_add_child(qdev_get_machine(), TYPE_S390_IPL,
+                              new, NULL);
+    object_unref(new);
     qdev_init_nofail(dev);
 }
 
-void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys)
+void s390_init_cpus(MachineState *machine)
 {
     int i;
+    gchar *name;
 
-    if (cpu_model == NULL) {
-        cpu_model = "host";
+    if (machine->cpu_model == NULL) {
+        machine->cpu_model = "host";
     }
 
-    ipi_states = g_malloc(sizeof(S390CPU *) * smp_cpus);
+    cpu_states = g_new0(S390CPU *, max_cpus);
+
+    for (i = 0; i < max_cpus; i++) {
+        name = g_strdup_printf("cpu[%i]", i);
+        object_property_add_link(OBJECT(machine), name, TYPE_S390_CPU,
+                                 (Object **) &cpu_states[i],
+                                 object_property_allow_set_link,
+                                 OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                                 &error_abort);
+        g_free(name);
+    }
 
     for (i = 0; i < smp_cpus; i++) {
-        S390CPU *cpu;
-        CPUState *cs;
-
-        cpu = cpu_s390x_init(cpu_model);
-        cs = CPU(cpu);
-
-        ipi_states[i] = cpu;
-        cs->halted = 1;
-        cs->exception_index = EXCP_HLT;
-        cpu->env.storage_keys = storage_keys;
+        s390x_new_cpu(machine->cpu_model, i, &error_fatal);
     }
 }
 
@@ -181,10 +134,7 @@
             nd->model = g_strdup("virtio");
         }
 
-        if (strcmp(nd->model, "virtio")) {
-            fprintf(stderr, "S390 only supports VirtIO nics\n");
-            exit(1);
-        }
+        qemu_check_nic_model(nd, "virtio");
 
         dev = qdev_create(bus, name);
         qdev_set_nic_properties(dev, nd);
@@ -192,63 +142,49 @@
     }
 }
 
-/* PC hardware initialisation */
-static void s390_init(MachineState *machine)
+void gtod_save(QEMUFile *f, void *opaque)
 {
-    ram_addr_t my_ram_size = machine->ram_size;
-    MemoryRegion *sysmem = get_system_memory();
-    MemoryRegion *ram = g_new(MemoryRegion, 1);
-    int increment_size = 20;
-    uint8_t *storage_keys;
-    void *virtio_region;
-    hwaddr virtio_region_len;
-    hwaddr virtio_region_start;
+    uint64_t tod_low;
+    uint8_t tod_high;
+    int r;
 
-    /*
-     * The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or
-     * fewer.
-     */
-    while ((my_ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
-        increment_size++;
+    r = s390_get_clock(&tod_high, &tod_low);
+    if (r) {
+        fprintf(stderr, "WARNING: Unable to get guest clock for migration. "
+                        "Error code %d. Guest clock will not be migrated "
+                        "which could cause the guest to hang.\n", r);
+        qemu_put_byte(f, S390_TOD_CLOCK_VALUE_MISSING);
+        return;
     }
-    my_ram_size = my_ram_size >> increment_size << increment_size;
 
-    /* let's propagate the changed ram size into the global variable. */
-    ram_size = my_ram_size;
+    qemu_put_byte(f, S390_TOD_CLOCK_VALUE_PRESENT);
+    qemu_put_byte(f, tod_high);
+    qemu_put_be64(f, tod_low);
+}
 
-    /* get a BUS */
-    s390_bus = s390_virtio_bus_init(&my_ram_size);
-    s390_sclp_init();
-    s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
-                      machine->initrd_filename, ZIPL_FILENAME);
-    s390_flic_init();
+int gtod_load(QEMUFile *f, void *opaque, int version_id)
+{
+    uint64_t tod_low;
+    uint8_t tod_high;
+    int r;
 
-    /* register hypercalls */
-    s390_virtio_register_hcalls();
+    if (qemu_get_byte(f) == S390_TOD_CLOCK_VALUE_MISSING) {
+        fprintf(stderr, "WARNING: Guest clock was not migrated. This could "
+                        "cause the guest to hang.\n");
+        return 0;
+    }
 
-    /* allocate RAM */
-    memory_region_init_ram(ram, NULL, "s390.ram", my_ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
-    memory_region_add_subregion(sysmem, 0, ram);
+    tod_high = qemu_get_byte(f);
+    tod_low = qemu_get_be64(f);
 
-    /* clear virtio region */
-    virtio_region_len = my_ram_size - ram_size;
-    virtio_region_start = ram_size;
-    virtio_region = cpu_physical_memory_map(virtio_region_start,
-                                            &virtio_region_len, true);
-    memset(virtio_region, 0, virtio_region_len);
-    cpu_physical_memory_unmap(virtio_region, virtio_region_len, 1,
-                              virtio_region_len);
+    r = s390_set_clock(&tod_high, &tod_low);
+    if (r) {
+        fprintf(stderr, "WARNING: Unable to set guest clock value. "
+                        "s390_get_clock returned error %d. This could cause "
+                        "the guest to hang.\n", r);
+    }
 
-    /* allocate storage keys */
-    storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
-
-    /* init CPUs */
-    s390_init_cpus(machine->cpu_model, storage_keys);
-
-    /* Create VirtIO network adapters */
-    s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
+    return 0;
 }
 
 void s390_nmi(NMIState *n, int cpu_index, Error **errp)
@@ -256,44 +192,19 @@
     CPUState *cs = qemu_get_cpu(cpu_index);
 
     if (s390_cpu_restart(S390_CPU(cs))) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
     }
 }
 
-static void s390_machine_class_init(ObjectClass *oc, void *data)
+void s390_machine_reset(void)
 {
-    MachineClass *mc = MACHINE_CLASS(oc);
-    NMIClass *nc = NMI_CLASS(oc);
+    S390CPU *ipl_cpu = S390_CPU(qemu_get_cpu(0));
 
-    mc->name = "s390-virtio";
-    mc->alias = "s390";
-    mc->desc = "VirtIO based S390 machine";
-    mc->init = s390_init;
-    mc->block_default_type = IF_VIRTIO;
-    mc->max_cpus = 255;
-    mc->no_serial = 1;
-    mc->no_parallel = 1;
-    mc->use_virtcon = 1;
-    mc->no_floppy = 1;
-    mc->no_cdrom = 1;
-    mc->no_sdcard = 1;
-    mc->is_default = 1;
-    nc->nmi_monitor_handler = s390_nmi;
+    qemu_devices_reset();
+    s390_cmma_reset();
+    s390_crypto_reset();
+
+    /* all cpus are stopped - configure and start the ipl cpu only */
+    s390_ipl_prepare_cpu(ipl_cpu);
+    s390_cpu_set_state(CPU_STATE_OPERATING, ipl_cpu);
 }
-
-static const TypeInfo s390_machine_info = {
-    .name          = TYPE_S390_MACHINE,
-    .parent        = TYPE_MACHINE,
-    .class_init    = s390_machine_class_init,
-    .interfaces = (InterfaceInfo[]) {
-        { TYPE_NMI },
-        { }
-    },
-};
-
-static void s390_machine_register_types(void)
-{
-    type_register_static(&s390_machine_info);
-}
-
-type_init(s390_machine_register_types)
diff --git a/hw/s390x/s390-virtio.h b/hw/s390x/s390-virtio.h
index 33847ae..f588b80 100644
--- a/hw/s390x/s390-virtio.h
+++ b/hw/s390x/s390-virtio.h
@@ -10,23 +10,23 @@
  */
 
 #ifndef HW_S390_VIRTIO_H
-#define HW_S390_VIRTIO_H 1
+#define HW_S390_VIRTIO_H
 
 #include "hw/nmi.h"
-
-#define KVM_S390_VIRTIO_NOTIFY          0
-#define KVM_S390_VIRTIO_RESET           1
-#define KVM_S390_VIRTIO_SET_STATUS      2
-#define KVM_S390_VIRTIO_CCW_NOTIFY      3
+#include "standard-headers/asm-s390/kvm_virtio.h"
+#include "standard-headers/asm-s390/virtio-ccw.h"
 
 typedef int (*s390_virtio_fn)(const uint64_t *args);
 void s390_register_virtio_hypercall(uint64_t code, s390_virtio_fn fn);
 
-void s390_init_cpus(const char *cpu_model, uint8_t *storage_keys);
+void s390_init_cpus(MachineState *machine);
 void s390_init_ipl_dev(const char *kernel_filename,
                        const char *kernel_cmdline,
                        const char *initrd_filename,
-                       const char *firmware);
+                       const char *firmware,
+                       bool enforce_bios);
 void s390_create_virtio_net(BusState *bus, const char *name);
 void s390_nmi(NMIState *n, int cpu_index, Error **errp);
+void s390_machine_reset(void);
+void s390_memory_init(ram_addr_t mem_size);
 #endif
diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
index a759da7..fca37f5 100644
--- a/hw/s390x/sclp.c
+++ b/hw/s390x/sclp.c
@@ -12,41 +12,34 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
 #include "exec/memory.h"
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
-#include "qemu/config-file.h"
+#include "hw/boards.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
+#include "hw/s390x/s390-pci-bus.h"
 
-static inline SCLPEventFacility *get_event_facility(void)
+static inline SCLPDevice *get_sclp_device(void)
 {
-    ObjectProperty *op = object_property_find(qdev_get_machine(),
-                                              TYPE_SCLP_EVENT_FACILITY,
-                                              NULL);
-    assert(op);
-    return op->opaque;
+    return SCLP(object_resolve_path_type("", TYPE_SCLP, NULL));
 }
 
 /* Provide information about the configuration, CPUs and storage */
-static void read_SCP_info(SCCB *sccb)
+static void read_SCP_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadInfo *read_info = (ReadInfo *) sccb;
+    MachineState *machine = MACHINE(qdev_get_machine());
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
     CPUState *cpu;
     int cpu_count = 0;
     int i = 0;
-    int increment_size = 20;
     int rnsize, rnmax;
-    QemuOpts *opts = qemu_opts_find(qemu_find_opts("memory"), NULL);
-    int slots = qemu_opt_get_number(opts, "slots", 0);
-    int max_avail_slots = s390_get_memslot_count(kvm_state);
-
-    if (slots > max_avail_slots) {
-        slots = max_avail_slots;
-    }
+    int slots = MIN(machine->ram_slots, s390_get_memslot_count(kvm_state));
 
     CPU_FOREACH(cpu) {
         cpu_count++;
@@ -62,25 +55,11 @@
         read_info->entries[i].type = 0;
     }
 
-    read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO);
-
-    /*
-     * The storage increment size is a multiple of 1M and is a power of 2.
-     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
-     */
-    while ((ram_size >> increment_size) > MAX_STORAGE_INCREMENTS) {
-        increment_size++;
-    }
-    rnmax = ram_size >> increment_size;
+    read_info->facilities = cpu_to_be64(SCLP_HAS_CPU_INFO |
+                                        SCLP_HAS_PCI_RECONFIG);
 
     /* Memory Hotplug is only supported for the ccw machine type */
     if (mhd) {
-        while ((mhd->standby_mem_size >> increment_size) >
-               MAX_STORAGE_INCREMENTS) {
-            increment_size++;
-        }
-        assert(increment_size == mhd->increment_size);
-
         mhd->standby_subregion_size = MEM_SECTION_SIZE;
         /* Deduct the memory slot already used for core */
         if (slots > 0) {
@@ -106,13 +85,11 @@
         }
         mhd->padded_ram_size = ram_size + mhd->pad_size;
         mhd->rzm = 1 << mhd->increment_size;
-        rnmax = ((ram_size + mhd->standby_mem_size + mhd->pad_size)
-             >> mhd->increment_size);
 
         read_info->facilities |= cpu_to_be64(SCLP_FC_ASSIGN_ATTACH_READ_STOR);
     }
 
-    rnsize = 1 << (increment_size - 20);
+    rnsize = 1 << (sclp->increment_size - 20);
     if (rnsize <= 128) {
         read_info->rnsize = rnsize;
     } else {
@@ -120,6 +97,7 @@
         read_info->rnsize2 = cpu_to_be32(rnsize);
     }
 
+    rnmax = machine->maxram_size >> sclp->increment_size;
     if (rnmax < 0x10000) {
         read_info->rnmax = cpu_to_be16(rnmax);
     } else {
@@ -130,14 +108,17 @@
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void read_storage_element0_info(SCCB *sccb)
+static void read_storage_element0_info(SCLPDevice *sclp, SCCB *sccb)
 {
     int i, assigned;
     int subincrement_id = SCLP_STARTING_SUBINCREMENT_ID;
     ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if ((ram_size >> mhd->increment_size) >= 0x10000) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
@@ -156,12 +137,15 @@
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void read_storage_element1_info(SCCB *sccb)
+static void read_storage_element1_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadStorageElementInfo *storage_info = (ReadStorageElementInfo *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if ((mhd->standby_mem_size >> mhd->increment_size) >= 0x10000) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_SCCB_BOUNDARY_VIOLATION);
@@ -177,13 +161,17 @@
     sccb->h.response_code = cpu_to_be16(SCLP_RC_STANDBY_READ_COMPLETION);
 }
 
-static void attach_storage_element(SCCB *sccb, uint16_t element)
+static void attach_storage_element(SCLPDevice *sclp, SCCB *sccb,
+                                   uint16_t element)
 {
     int i, assigned, subincrement_id;
     AttachStorageElement *attach_info = (AttachStorageElement *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
 
-    assert(mhd);
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
 
     if (element != 1) {
         sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
@@ -201,20 +189,26 @@
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
 }
 
-static void assign_storage(SCCB *sccb)
+static void assign_storage(SCLPDevice *sclp, SCCB *sccb)
 {
     MemoryRegion *mr = NULL;
     uint64_t this_subregion_size;
     AssignStorage *assign_info = (AssignStorage *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
-    assert(mhd);
-    ram_addr_t assign_addr = (assign_info->rn - 1) * mhd->rzm;
+    ram_addr_t assign_addr;
     MemoryRegion *sysmem = get_system_memory();
 
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+    assign_addr = (assign_info->rn - 1) * mhd->rzm;
+
     if ((assign_addr % MEM_SECTION_SIZE == 0) &&
         (assign_addr >= mhd->padded_ram_size)) {
         /* Re-use existing memory region if found */
         mr = memory_region_find(sysmem, assign_addr, 1).mr;
+        memory_region_unref(mr);
         if (!mr) {
 
             MemoryRegion *standby_ram = g_new(MemoryRegion, 1);
@@ -239,7 +233,13 @@
                 this_subregion_size = mhd->standby_subregion_size;
             }
 
-            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size, &error_abort);
+            memory_region_init_ram(standby_ram, NULL, id, this_subregion_size,
+                                   &error_fatal);
+            /* This is a hack to make memory hotunplug work again. Once we have
+             * subdevices, we have to unparent them when unassigning memory,
+             * instead of doing it via the ref count of the MemoryRegion. */
+            object_ref(OBJECT(standby_ram));
+            object_unparent(OBJECT(standby_ram));
             vmstate_register_ram_global(standby_ram);
             memory_region_add_subregion(sysmem, offset, standby_ram);
         }
@@ -250,15 +250,20 @@
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
 }
 
-static void unassign_storage(SCCB *sccb)
+static void unassign_storage(SCLPDevice *sclp, SCCB *sccb)
 {
     MemoryRegion *mr = NULL;
     AssignStorage *assign_info = (AssignStorage *) sccb;
     sclpMemoryHotplugDev *mhd = get_sclp_memory_hotplug_dev();
-    assert(mhd);
-    ram_addr_t unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+    ram_addr_t unassign_addr;
     MemoryRegion *sysmem = get_system_memory();
 
+    if (!mhd) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        return;
+    }
+    unassign_addr = (assign_info->rn - 1) * mhd->rzm;
+
     /* if the addr is a multiple of 256 MB */
     if ((unassign_addr % MEM_SECTION_SIZE == 0) &&
         (unassign_addr >= mhd->padded_ram_size)) {
@@ -267,6 +272,7 @@
 
         /* find the specified memory region and destroy it */
         mr = memory_region_find(sysmem, unassign_addr, 1).mr;
+        memory_region_unref(mr);
         if (mr) {
             int i;
             int is_removable = 1;
@@ -285,8 +291,7 @@
             }
             if (is_removable) {
                 memory_region_del_subregion(sysmem, mr);
-                object_unparent(OBJECT(mr));
-                g_free(mr);
+                object_unref(OBJECT(mr));
             }
         }
     }
@@ -294,7 +299,7 @@
 }
 
 /* Provide information about the CPU */
-static void sclp_read_cpu_info(SCCB *sccb)
+static void sclp_read_cpu_info(SCLPDevice *sclp, SCCB *sccb)
 {
     ReadCpuInfo *cpu_info = (ReadCpuInfo *) sccb;
     CPUState *cpu;
@@ -321,34 +326,41 @@
     sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
 }
 
-static void sclp_execute(SCCB *sccb, uint32_t code)
+static void sclp_execute(SCLPDevice *sclp, SCCB *sccb, uint32_t code)
 {
-    SCLPEventFacility *ef = get_event_facility();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+    SCLPEventFacility *ef = sclp->event_facility;
     SCLPEventFacilityClass *efc = EVENT_FACILITY_GET_CLASS(ef);
 
     switch (code & SCLP_CMD_CODE_MASK) {
     case SCLP_CMDW_READ_SCP_INFO:
     case SCLP_CMDW_READ_SCP_INFO_FORCED:
-        read_SCP_info(sccb);
+        sclp_c->read_SCP_info(sclp, sccb);
         break;
     case SCLP_CMDW_READ_CPU_INFO:
-        sclp_read_cpu_info(sccb);
+        sclp_c->read_cpu_info(sclp, sccb);
         break;
     case SCLP_READ_STORAGE_ELEMENT_INFO:
         if (code & 0xff00) {
-            read_storage_element1_info(sccb);
+            sclp_c->read_storage_element1_info(sclp, sccb);
         } else {
-            read_storage_element0_info(sccb);
+            sclp_c->read_storage_element0_info(sclp, sccb);
         }
         break;
     case SCLP_ATTACH_STORAGE_ELEMENT:
-        attach_storage_element(sccb, (code & 0xff00) >> 8);
+        sclp_c->attach_storage_element(sclp, sccb, (code & 0xff00) >> 8);
         break;
     case SCLP_ASSIGN_STORAGE:
-        assign_storage(sccb);
+        sclp_c->assign_storage(sclp, sccb);
         break;
     case SCLP_UNASSIGN_STORAGE:
-        unassign_storage(sccb);
+        sclp_c->unassign_storage(sclp, sccb);
+        break;
+    case SCLP_CMDW_CONFIGURE_PCI:
+        s390_pci_sclp_configure(sccb);
+        break;
+    case SCLP_CMDW_DECONFIGURE_PCI:
+        s390_pci_sclp_deconfigure(sccb);
         break;
     default:
         efc->command_handler(ef, sccb, code);
@@ -358,6 +370,8 @@
 
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code)
 {
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
     int r = 0;
     SCCB work_sccb;
 
@@ -392,20 +406,20 @@
         goto out;
     }
 
-    sclp_execute((SCCB *)&work_sccb, code);
+    sclp_c->execute(sclp, (SCCB *)&work_sccb, code);
 
     cpu_physical_memory_write(sccb, &work_sccb,
                               be16_to_cpu(work_sccb.h.length));
 
-    sclp_service_interrupt(sccb);
+    sclp_c->service_interrupt(sclp, sccb);
 
 out:
     return r;
 }
 
-void sclp_service_interrupt(uint32_t sccb)
+static void service_interrupt(SCLPDevice *sclp, uint32_t sccb)
 {
-    SCLPEventFacility *ef = get_event_facility();
+    SCLPEventFacility *ef = sclp->event_facility;
     SCLPEventFacilityClass *efc = EVENT_FACILITY_GET_CLASS(ef);
 
     uint32_t param = sccb & ~3;
@@ -420,17 +434,149 @@
     s390_sclp_extint(param);
 }
 
+void sclp_service_interrupt(uint32_t sccb)
+{
+    SCLPDevice *sclp = get_sclp_device();
+    SCLPDeviceClass *sclp_c = SCLP_GET_CLASS(sclp);
+
+    sclp_c->service_interrupt(sclp, sccb);
+}
+
 /* qemu object creation and initialization functions */
 
 void s390_sclp_init(void)
 {
-    DeviceState *dev  = qdev_create(NULL, TYPE_SCLP_EVENT_FACILITY);
+    Object *new = object_new(TYPE_SCLP);
 
-    object_property_add_child(qdev_get_machine(), TYPE_SCLP_EVENT_FACILITY,
-                              OBJECT(dev), NULL);
-    qdev_init_nofail(dev);
+    object_property_add_child(qdev_get_machine(), TYPE_SCLP, new,
+                              NULL);
+    object_unref(OBJECT(new));
+    qdev_init_nofail(DEVICE(new));
 }
 
+static void sclp_realize(DeviceState *dev, Error **errp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    SCLPDevice *sclp = SCLP(dev);
+    Error *err = NULL;
+    uint64_t hw_limit;
+    int ret;
+
+    object_property_set_bool(OBJECT(sclp->event_facility), true, "realized",
+                             &err);
+    if (err) {
+        goto out;
+    }
+    /*
+     * qdev_device_add searches the sysbus for TYPE_SCLP_EVENTS_BUS. As long
+     * as we can't find a fitting bus via the qom tree, we have to add the
+     * event facility to the sysbus, so e.g. a sclp console can be created.
+     */
+    qdev_set_parent_bus(DEVICE(sclp->event_facility), sysbus_get_default());
+
+    ret = s390_set_memory_limit(machine->maxram_size, &hw_limit);
+    if (ret == -E2BIG) {
+        error_setg(&err, "qemu: host supports a maximum of %" PRIu64 " GB",
+                   hw_limit >> 30);
+    } else if (ret) {
+        error_setg(&err, "qemu: setting the guest size failed");
+    }
+
+out:
+    error_propagate(errp, err);
+}
+
+static void sclp_memory_init(SCLPDevice *sclp)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    ram_addr_t initial_mem = machine->ram_size;
+    ram_addr_t max_mem = machine->maxram_size;
+    ram_addr_t standby_mem = max_mem - initial_mem;
+    ram_addr_t pad_mem = 0;
+    int increment_size = 20;
+
+    /* The storage increment size is a multiple of 1M and is a power of 2.
+     * The number of storage increments must be MAX_STORAGE_INCREMENTS or fewer.
+     * The variable 'increment_size' is an exponent of 2 that can be
+     * used to calculate the size (in bytes) of an increment. */
+    while ((initial_mem >> increment_size) > MAX_STORAGE_INCREMENTS) {
+        increment_size++;
+    }
+    if (machine->ram_slots) {
+        while ((standby_mem >> increment_size) > MAX_STORAGE_INCREMENTS) {
+            increment_size++;
+        }
+    }
+    sclp->increment_size = increment_size;
+
+    /* The core and standby memory areas need to be aligned with
+     * the increment size.  In effect, this can cause the
+     * user-specified memory size to be rounded down to align
+     * with the nearest increment boundary. */
+    initial_mem = initial_mem >> increment_size << increment_size;
+    standby_mem = standby_mem >> increment_size << increment_size;
+
+    /* If the size of ram is not on a MEM_SECTION_SIZE boundary,
+       calculate the pad size necessary to force this boundary. */
+    if (machine->ram_slots && standby_mem) {
+        sclpMemoryHotplugDev *mhd = init_sclp_memory_hotplug_dev();
+
+        if (initial_mem % MEM_SECTION_SIZE) {
+            pad_mem = MEM_SECTION_SIZE - initial_mem % MEM_SECTION_SIZE;
+        }
+        mhd->increment_size = increment_size;
+        mhd->pad_size = pad_mem;
+        mhd->standby_mem_size = standby_mem;
+    }
+    machine->ram_size = initial_mem;
+    machine->maxram_size = initial_mem + pad_mem + standby_mem;
+    /* let's propagate the changed ram size into the global variable. */
+    ram_size = initial_mem;
+}
+
+static void sclp_init(Object *obj)
+{
+    SCLPDevice *sclp = SCLP(obj);
+    Object *new;
+
+    new = object_new(TYPE_SCLP_EVENT_FACILITY);
+    object_property_add_child(obj, TYPE_SCLP_EVENT_FACILITY, new, NULL);
+    object_unref(new);
+    sclp->event_facility = EVENT_FACILITY(new);
+
+    sclp_memory_init(sclp);
+}
+
+static void sclp_class_init(ObjectClass *oc, void *data)
+{
+    SCLPDeviceClass *sc = SCLP_CLASS(oc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
+
+    dc->desc = "SCLP (Service-Call Logical Processor)";
+    dc->realize = sclp_realize;
+    dc->hotpluggable = false;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+
+    sc->read_SCP_info = read_SCP_info;
+    sc->read_storage_element0_info = read_storage_element0_info;
+    sc->read_storage_element1_info = read_storage_element1_info;
+    sc->attach_storage_element = attach_storage_element;
+    sc->assign_storage = assign_storage;
+    sc->unassign_storage = unassign_storage;
+    sc->read_cpu_info = sclp_read_cpu_info;
+    sc->execute = sclp_execute;
+    sc->service_interrupt = service_interrupt;
+}
+
+static TypeInfo sclp_info = {
+    .name = TYPE_SCLP,
+    .parent = TYPE_DEVICE,
+    .instance_init = sclp_init,
+    .instance_size = sizeof(SCLPDevice),
+    .class_init = sclp_class_init,
+    .class_size = sizeof(SCLPDeviceClass),
+};
+
 sclpMemoryHotplugDev *init_sclp_memory_hotplug_dev(void)
 {
     DeviceState *dev;
@@ -449,14 +595,24 @@
                                    TYPE_SCLP_MEMORY_HOTPLUG_DEV, NULL));
 }
 
+static void sclp_memory_hotplug_dev_class_init(ObjectClass *klass,
+                                               void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
 static TypeInfo sclp_memory_hotplug_dev_info = {
     .name = TYPE_SCLP_MEMORY_HOTPLUG_DEV,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(sclpMemoryHotplugDev),
+    .class_init = sclp_memory_hotplug_dev_class_init,
 };
 
 static void register_types(void)
 {
     type_register_static(&sclp_memory_hotplug_dev_info);
+    type_register_static(&sclp_info);
 }
 type_init(register_types);
diff --git a/hw/s390x/sclpcpu.c b/hw/s390x/sclpcpu.c
index 3600fe2..b1f3ef8 100644
--- a/hw/s390x/sclpcpu.c
+++ b/hw/s390x/sclpcpu.c
@@ -12,6 +12,7 @@
  * option) any later version.  See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
@@ -25,13 +26,16 @@
     uint8_t event_qualifier;
 } QEMU_PACKED ConfigMgtData;
 
-static qemu_irq *irq_cpu_hotplug; /* Only used in this file */
-
 #define EVENT_QUAL_CPU_CHANGE  1
 
 void raise_irq_cpu_hotplug(void)
 {
-    qemu_irq_raise(*irq_cpu_hotplug);
+    Object *obj = object_resolve_path_type("", TYPE_SCLP_CPU_HOTPLUG, NULL);
+
+    SCLP_EVENT(obj)->event_pending = true;
+
+    /* Trigger SCLP read operation */
+    sclp_service_interrupt(0);
 }
 
 static unsigned int send_mask(void)
@@ -70,34 +74,19 @@
     return 1;
 }
 
-static void trigger_signal(void *opaque, int n, int level)
-{
-    SCLPEvent *event = opaque;
-    event->event_pending = true;
-
-    /* Trigger SCLP read operation */
-    sclp_service_interrupt(0);
-}
-
-static int irq_cpu_hotplug_init(SCLPEvent *event)
-{
-    irq_cpu_hotplug = qemu_allocate_irqs(trigger_signal, event, 1);
-    return 0;
-}
-
 static void cpu_class_init(ObjectClass *oc, void *data)
 {
     SCLPEventClass *k = SCLP_EVENT_CLASS(oc);
+    DeviceClass *dc = DEVICE_CLASS(oc);
 
-    k->init = irq_cpu_hotplug_init;
     k->get_send_mask = send_mask;
     k->get_receive_mask = receive_mask;
     k->read_event_data = read_event_data;
-    k->write_event_data = NULL;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo sclp_cpu_info = {
-    .name          = "sclp-cpu-hotplug",
+    .name          = TYPE_SCLP_CPU_HOTPLUG,
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
     .class_init    = cpu_class_init,
diff --git a/hw/s390x/sclpquiesce.c b/hw/s390x/sclpquiesce.c
index 1a399bd..762cb18 100644
--- a/hw/s390x/sclpquiesce.c
+++ b/hw/s390x/sclpquiesce.c
@@ -11,7 +11,8 @@
  * option) any later version.  See the COPYING file in the top-level directory.
  *
  */
-#include <hw/qdev.h>
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/sclp.h"
 #include "hw/s390x/event-facility.h"
@@ -66,7 +67,7 @@
 }
 
 static const VMStateDescription vmstate_sclpquiesce = {
-    .name = "sclpquiesce",
+    .name = TYPE_SCLP_QUIESCE,
     .version_id = 0,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
@@ -116,6 +117,7 @@
 
     dc->reset = quiesce_reset;
     dc->vmsd = &vmstate_sclpquiesce;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     k->init = quiesce_init;
 
     k->get_send_mask = send_mask;
@@ -126,7 +128,7 @@
 }
 
 static const TypeInfo sclp_quiesce_info = {
-    .name          = "sclpquiesce",
+    .name          = TYPE_SCLP_QUIESCE,
     .parent        = TYPE_SCLP_EVENT,
     .instance_size = sizeof(SCLPEvent),
     .class_init    = quiesce_class_init,
diff --git a/hw/s390x/trace-events b/hw/s390x/trace-events
new file mode 100644
index 0000000..84ea964
--- /dev/null
+++ b/hw/s390x/trace-events
@@ -0,0 +1,15 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/s390x/css.c
+css_enable_facility(const char *facility) "CSS: enable %s"
+css_crw(uint8_t rsc, uint8_t erc, uint16_t rsid, const char *chained) "CSS: queueing crw: rsc=%x, erc=%x, rsid=%x %s"
+css_chpid_add(uint8_t cssid, uint8_t chpid, uint8_t type) "CSS: add chpid %x.%02x (type %02x)"
+css_new_image(uint8_t cssid, const char *default_cssid) "CSS: add css image %02x %s"
+css_assign_subch(const char *do_assign, uint8_t cssid, uint8_t ssid, uint16_t schid, uint16_t devno) "CSS: %s %x.%x.%04x (devno %04x)"
+css_io_interrupt(int cssid, int ssid, int schid, uint32_t intparm, uint8_t isc, const char *conditional) "CSS: I/O interrupt on sch %x.%x.%04x (intparm %08x, isc %x) %s"
+css_adapter_interrupt(uint8_t isc) "CSS: adapter I/O interrupt (isc %x)"
+
+# hw/s390x/virtio-ccw.c
+virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code) "VIRTIO-CCW: %x.%x.%04x: interpret command %x"
+virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *devno_mode) "VIRTIO-CCW: add subchannel %x.%x.%04x, devno %04x (%s)"
+virtio_ccw_set_ind(uint64_t ind_loc, uint8_t ind_old, uint8_t ind_new) "VIRTIO-CCW: indicator at %" PRIu64 ": %x->%x"
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index ea236c9..a554a24 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -1,121 +1,43 @@
 /*
  * virtio ccw target implementation
  *
- * Copyright 2012,2014 IBM Corp.
+ * Copyright 2012,2015 IBM Corp.
  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *            Pierre Morel <pmorel@linux.vnet.ibm.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or (at
  * your option) any later version. See the COPYING file in the top-level
  * directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/kvm.h"
 #include "net/net.h"
-#include "monitor/monitor.h"
 #include "hw/virtio/virtio.h"
 #include "hw/virtio/virtio-serial.h"
 #include "hw/virtio/virtio-net.h"
 #include "hw/sysbus.h"
 #include "qemu/bitops.h"
+#include "qemu/error-report.h"
+#include "hw/virtio/virtio-access.h"
 #include "hw/virtio/virtio-bus.h"
 #include "hw/s390x/adapter.h"
 #include "hw/s390x/s390_flic.h"
 
-#include "ioinst.h"
-#include "css.h"
+#include "hw/s390x/ioinst.h"
+#include "hw/s390x/css.h"
 #include "virtio-ccw.h"
 #include "trace.h"
-
-static QTAILQ_HEAD(, IndAddr) indicator_addresses =
-    QTAILQ_HEAD_INITIALIZER(indicator_addresses);
-
-static IndAddr *get_indicator(hwaddr ind_addr, int len)
-{
-    IndAddr *indicator;
-
-    QTAILQ_FOREACH(indicator, &indicator_addresses, sibling) {
-        if (indicator->addr == ind_addr) {
-            indicator->refcnt++;
-            return indicator;
-        }
-    }
-    indicator = g_new0(IndAddr, 1);
-    indicator->addr = ind_addr;
-    indicator->len = len;
-    indicator->refcnt = 1;
-    QTAILQ_INSERT_TAIL(&indicator_addresses, indicator, sibling);
-    return indicator;
-}
-
-static int s390_io_adapter_map(AdapterInfo *adapter, uint64_t map_addr,
-                               bool do_map)
-{
-    S390FLICState *fs = s390_get_flic();
-    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
-
-    return fsc->io_adapter_map(fs, adapter->adapter_id, map_addr, do_map);
-}
-
-static void release_indicator(AdapterInfo *adapter, IndAddr *indicator)
-{
-    assert(indicator->refcnt > 0);
-    indicator->refcnt--;
-    if (indicator->refcnt > 0) {
-        return;
-    }
-    QTAILQ_REMOVE(&indicator_addresses, indicator, sibling);
-    if (indicator->map) {
-        s390_io_adapter_map(adapter, indicator->map, false);
-    }
-    g_free(indicator);
-}
-
-static int map_indicator(AdapterInfo *adapter, IndAddr *indicator)
-{
-    int ret;
-
-    if (indicator->map) {
-        return 0; /* already mapped is not an error */
-    }
-    indicator->map = indicator->addr;
-    ret = s390_io_adapter_map(adapter, indicator->map, true);
-    if ((ret != 0) && (ret != -ENOSYS)) {
-        goto out_err;
-    }
-    return 0;
-
-out_err:
-    indicator->map = 0;
-    return ret;
-}
+#include "hw/s390x/css-bridge.h"
 
 static void virtio_ccw_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtioCcwDevice *dev);
 
-static void virtual_css_bus_reset(BusState *qbus)
-{
-    /* This should actually be modelled via the generic css */
-    css_reset();
-}
-
-
-static void virtual_css_bus_class_init(ObjectClass *klass, void *data)
-{
-    BusClass *k = BUS_CLASS(klass);
-
-    k->reset = virtual_css_bus_reset;
-}
-
-static const TypeInfo virtual_css_bus_info = {
-    .name = TYPE_VIRTUAL_CSS_BUS,
-    .parent = TYPE_BUS,
-    .instance_size = sizeof(VirtualCssBus),
-    .class_init = virtual_css_bus_class_init,
-};
-
 VirtIODevice *virtio_ccw_get_vdev(SubchDev *sch)
 {
     VirtIODevice *vdev = NULL;
@@ -127,120 +49,76 @@
     return vdev;
 }
 
-static int virtio_ccw_set_guest2host_notifier(VirtioCcwDevice *dev, int n,
-                                              bool assign, bool set_handler)
-{
-    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
-    VirtQueue *vq = virtio_get_queue(vdev, n);
-    EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
-    int r = 0;
-    SubchDev *sch = dev->sch;
-    uint32_t sch_id = (css_build_subchannel_id(sch) << 16) | sch->schid;
-
-    if (assign) {
-        r = event_notifier_init(notifier, 1);
-        if (r < 0) {
-            error_report("%s: unable to init event notifier: %d", __func__, r);
-            return r;
-        }
-        virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
-        r = s390_assign_subch_ioeventfd(notifier, sch_id, n, assign);
-        if (r < 0) {
-            error_report("%s: unable to assign ioeventfd: %d", __func__, r);
-            virtio_queue_set_host_notifier_fd_handler(vq, false, false);
-            event_notifier_cleanup(notifier);
-            return r;
-        }
-    } else {
-        virtio_queue_set_host_notifier_fd_handler(vq, false, false);
-        s390_assign_subch_ioeventfd(notifier, sch_id, n, assign);
-        event_notifier_cleanup(notifier);
-    }
-    return r;
-}
-
 static void virtio_ccw_start_ioeventfd(VirtioCcwDevice *dev)
 {
-    VirtIODevice *vdev;
-    int n, r;
-
-    if (!(dev->flags & VIRTIO_CCW_FLAG_USE_IOEVENTFD) ||
-        dev->ioeventfd_disabled ||
-        dev->ioeventfd_started) {
-        return;
-    }
-    vdev = virtio_bus_get_device(&dev->bus);
-    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
-        if (!virtio_queue_get_num(vdev, n)) {
-            continue;
-        }
-        r = virtio_ccw_set_guest2host_notifier(dev, n, true, true);
-        if (r < 0) {
-            goto assign_error;
-        }
-    }
-    dev->ioeventfd_started = true;
-    return;
-
-  assign_error:
-    while (--n >= 0) {
-        if (!virtio_queue_get_num(vdev, n)) {
-            continue;
-        }
-        r = virtio_ccw_set_guest2host_notifier(dev, n, false, false);
-        assert(r >= 0);
-    }
-    dev->ioeventfd_started = false;
-    /* Disable ioeventfd for this device. */
-    dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
-    error_report("%s: failed. Fallback to userspace (slower).", __func__);
+    virtio_bus_start_ioeventfd(&dev->bus);
 }
 
 static void virtio_ccw_stop_ioeventfd(VirtioCcwDevice *dev)
 {
-    VirtIODevice *vdev;
-    int n, r;
-
-    if (!dev->ioeventfd_started) {
-        return;
-    }
-    vdev = virtio_bus_get_device(&dev->bus);
-    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
-        if (!virtio_queue_get_num(vdev, n)) {
-            continue;
-        }
-        r = virtio_ccw_set_guest2host_notifier(dev, n, false, false);
-        assert(r >= 0);
-    }
-    dev->ioeventfd_started = false;
+    virtio_bus_stop_ioeventfd(&dev->bus);
 }
 
-VirtualCssBus *virtual_css_bus_init(void)
+static bool virtio_ccw_ioeventfd_started(DeviceState *d)
 {
-    VirtualCssBus *cbus;
-    BusState *bus;
-    DeviceState *dev;
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
 
-    /* Create bridge device */
-    dev = qdev_create(NULL, "virtual-css-bridge");
-    qdev_init_nofail(dev);
+    return dev->ioeventfd_started;
+}
 
-    /* Create bus on bridge device */
-    bus = qbus_create(TYPE_VIRTUAL_CSS_BUS, dev, "virtual-css");
-    cbus = VIRTUAL_CSS_BUS(bus);
+static void virtio_ccw_ioeventfd_set_started(DeviceState *d, bool started,
+                                             bool err)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
 
-    /* Enable hotplugging */
-    qbus_set_hotplug_handler(bus, dev, &error_abort);
+    dev->ioeventfd_started = started;
+    if (err) {
+        /* Disable ioeventfd for this device. */
+        dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
+    }
+}
 
-    return cbus;
+static bool virtio_ccw_ioeventfd_disabled(DeviceState *d)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+    return dev->ioeventfd_disabled ||
+        !(dev->flags & VIRTIO_CCW_FLAG_USE_IOEVENTFD);
+}
+
+static void virtio_ccw_ioeventfd_set_disabled(DeviceState *d, bool disabled)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+    dev->ioeventfd_disabled = disabled;
+}
+
+static int virtio_ccw_ioeventfd_assign(DeviceState *d, EventNotifier *notifier,
+                                       int n, bool assign)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
+    SubchDev *sch = ccw_dev->sch;
+    uint32_t sch_id = (css_build_subchannel_id(sch) << 16) | sch->schid;
+
+    return s390_assign_subch_ioeventfd(notifier, sch_id, n, assign);
 }
 
 /* Communication blocks used by several channel commands. */
-typedef struct VqInfoBlock {
+typedef struct VqInfoBlockLegacy {
     uint64_t queue;
     uint32_t align;
     uint16_t index;
     uint16_t num;
+} QEMU_PACKED VqInfoBlockLegacy;
+
+typedef struct VqInfoBlock {
+    uint64_t desc;
+    uint32_t res0;
+    uint16_t index;
+    uint16_t num;
+    uint64_t avail;
+    uint64_t used;
 } QEMU_PACKED VqInfoBlock;
 
 typedef struct VqConfigBlock {
@@ -260,18 +138,27 @@
     uint8_t isc;
 } QEMU_PACKED VirtioThinintInfo;
 
+typedef struct VirtioRevInfo {
+    uint16_t revision;
+    uint16_t length;
+    uint8_t data[0];
+} QEMU_PACKED VirtioRevInfo;
+
 /* Specify where the virtqueues for the subchannel are in guest memory. */
-static int virtio_ccw_set_vqs(SubchDev *sch, uint64_t addr, uint32_t align,
-                              uint16_t index, uint16_t num)
+static int virtio_ccw_set_vqs(SubchDev *sch, VqInfoBlock *info,
+                              VqInfoBlockLegacy *linfo)
 {
     VirtIODevice *vdev = virtio_ccw_get_vdev(sch);
+    uint16_t index = info ? info->index : linfo->index;
+    uint16_t num = info ? info->num : linfo->num;
+    uint64_t desc = info ? info->desc : linfo->queue;
 
-    if (index > VIRTIO_PCI_QUEUE_MAX) {
+    if (index >= VIRTIO_CCW_QUEUE_MAX) {
         return -EINVAL;
     }
 
     /* Current code in virtio.c relies on 4K alignment. */
-    if (addr && (align != 4096)) {
+    if (linfo && desc && (linfo->align != 4096)) {
         return -EINVAL;
     }
 
@@ -279,26 +166,127 @@
         return -EINVAL;
     }
 
-    virtio_queue_set_addr(vdev, index, addr);
-    if (!addr) {
-        virtio_queue_set_vector(vdev, index, 0);
+    if (info) {
+        virtio_queue_set_rings(vdev, index, desc, info->avail, info->used);
     } else {
-        /* Fail if we don't have a big enough queue. */
-        /* TODO: Add interface to handle vring.num changing */
-        if (virtio_queue_get_num(vdev, index) > num) {
+        virtio_queue_set_addr(vdev, index, desc);
+    }
+    if (!desc) {
+        virtio_queue_set_vector(vdev, index, VIRTIO_NO_VECTOR);
+    } else {
+        if (info) {
+            /* virtio-1 allows changing the ring size. */
+            if (virtio_queue_get_num(vdev, index) < num) {
+                /* Fail if we exceed the maximum number. */
+                return -EINVAL;
+            }
+            virtio_queue_set_num(vdev, index, num);
+        } else if (virtio_queue_get_num(vdev, index) > num) {
+            /* Fail if we don't have a big enough queue. */
             return -EINVAL;
         }
+        /* We ignore possible increased num for legacy for compatibility. */
         virtio_queue_set_vector(vdev, index, index);
     }
     /* tell notify handler in case of config change */
-    vdev->config_vector = VIRTIO_PCI_QUEUE_MAX;
+    vdev->config_vector = VIRTIO_CCW_QUEUE_MAX;
     return 0;
 }
 
+static void virtio_ccw_reset_virtio(VirtioCcwDevice *dev, VirtIODevice *vdev)
+{
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
+
+    virtio_ccw_stop_ioeventfd(dev);
+    virtio_reset(vdev);
+    if (dev->indicators) {
+        release_indicator(&dev->routes.adapter, dev->indicators);
+        dev->indicators = NULL;
+    }
+    if (dev->indicators2) {
+        release_indicator(&dev->routes.adapter, dev->indicators2);
+        dev->indicators2 = NULL;
+    }
+    if (dev->summary_indicator) {
+        release_indicator(&dev->routes.adapter, dev->summary_indicator);
+        dev->summary_indicator = NULL;
+    }
+    ccw_dev->sch->thinint_active = false;
+}
+
+static int virtio_ccw_handle_set_vq(SubchDev *sch, CCW1 ccw, bool check_len,
+                                    bool is_legacy)
+{
+    int ret;
+    VqInfoBlock info;
+    VqInfoBlockLegacy linfo;
+    size_t info_len = is_legacy ? sizeof(linfo) : sizeof(info);
+
+    if (check_len) {
+        if (ccw.count != info_len) {
+            return -EINVAL;
+        }
+    } else if (ccw.count < info_len) {
+        /* Can't execute command. */
+        return -EINVAL;
+    }
+    if (!ccw.cda) {
+        return -EFAULT;
+    }
+    if (is_legacy) {
+        linfo.queue = address_space_ldq_be(&address_space_memory, ccw.cda,
+                                           MEMTXATTRS_UNSPECIFIED, NULL);
+        linfo.align = address_space_ldl_be(&address_space_memory,
+                                           ccw.cda + sizeof(linfo.queue),
+                                           MEMTXATTRS_UNSPECIFIED,
+                                           NULL);
+        linfo.index = address_space_lduw_be(&address_space_memory,
+                                            ccw.cda + sizeof(linfo.queue)
+                                            + sizeof(linfo.align),
+                                            MEMTXATTRS_UNSPECIFIED,
+                                            NULL);
+        linfo.num = address_space_lduw_be(&address_space_memory,
+                                          ccw.cda + sizeof(linfo.queue)
+                                          + sizeof(linfo.align)
+                                          + sizeof(linfo.index),
+                                          MEMTXATTRS_UNSPECIFIED,
+                                          NULL);
+        ret = virtio_ccw_set_vqs(sch, NULL, &linfo);
+    } else {
+        info.desc = address_space_ldq_be(&address_space_memory, ccw.cda,
+                                           MEMTXATTRS_UNSPECIFIED, NULL);
+        info.index = address_space_lduw_be(&address_space_memory,
+                                           ccw.cda + sizeof(info.desc)
+                                           + sizeof(info.res0),
+                                           MEMTXATTRS_UNSPECIFIED, NULL);
+        info.num = address_space_lduw_be(&address_space_memory,
+                                         ccw.cda + sizeof(info.desc)
+                                         + sizeof(info.res0)
+                                         + sizeof(info.index),
+                                         MEMTXATTRS_UNSPECIFIED, NULL);
+        info.avail = address_space_ldq_be(&address_space_memory,
+                                          ccw.cda + sizeof(info.desc)
+                                          + sizeof(info.res0)
+                                          + sizeof(info.index)
+                                          + sizeof(info.num),
+                                          MEMTXATTRS_UNSPECIFIED, NULL);
+        info.used = address_space_ldq_be(&address_space_memory,
+                                         ccw.cda + sizeof(info.desc)
+                                         + sizeof(info.res0)
+                                         + sizeof(info.index)
+                                         + sizeof(info.num)
+                                         + sizeof(info.avail),
+                                         MEMTXATTRS_UNSPECIFIED, NULL);
+        ret = virtio_ccw_set_vqs(sch, &info, NULL);
+    }
+    sch->curr_status.scsw.count = 0;
+    return ret;
+}
+
 static int virtio_ccw_cb(SubchDev *sch, CCW1 ccw)
 {
     int ret;
-    VqInfoBlock info;
+    VirtioRevInfo revinfo;
     uint8_t status;
     VirtioFeatDesc features;
     void *config;
@@ -322,37 +310,10 @@
     /* Look at the command. */
     switch (ccw.cmd_code) {
     case CCW_CMD_SET_VQ:
-        if (check_len) {
-            if (ccw.count != sizeof(info)) {
-                ret = -EINVAL;
-                break;
-            }
-        } else if (ccw.count < sizeof(info)) {
-            /* Can't execute command. */
-            ret = -EINVAL;
-            break;
-        }
-        if (!ccw.cda) {
-            ret = -EFAULT;
-        } else {
-            info.queue = ldq_phys(&address_space_memory, ccw.cda);
-            info.align = ldl_phys(&address_space_memory,
-                                  ccw.cda + sizeof(info.queue));
-            info.index = lduw_phys(&address_space_memory,
-                                   ccw.cda + sizeof(info.queue)
-                                   + sizeof(info.align));
-            info.num = lduw_phys(&address_space_memory,
-                                 ccw.cda + sizeof(info.queue)
-                                 + sizeof(info.align)
-                                 + sizeof(info.index));
-            ret = virtio_ccw_set_vqs(sch, info.queue, info.align, info.index,
-                                     info.num);
-            sch->curr_status.scsw.count = 0;
-        }
+        ret = virtio_ccw_handle_set_vq(sch, ccw, check_len, dev->revision < 1);
         break;
     case CCW_CMD_VDEV_RESET:
-        virtio_ccw_stop_ioeventfd(dev);
-        virtio_reset(vdev);
+        virtio_ccw_reset_virtio(dev, vdev);
         ret = 0;
         break;
     case CCW_CMD_READ_FEAT:
@@ -369,15 +330,32 @@
         if (!ccw.cda) {
             ret = -EFAULT;
         } else {
-            features.index = ldub_phys(&address_space_memory,
-                                       ccw.cda + sizeof(features.features));
-            if (features.index < ARRAY_SIZE(dev->host_features)) {
-                features.features = dev->host_features[features.index];
+            features.index = address_space_ldub(&address_space_memory,
+                                                ccw.cda
+                                                + sizeof(features.features),
+                                                MEMTXATTRS_UNSPECIFIED,
+                                                NULL);
+            if (features.index == 0) {
+                if (dev->revision >= 1) {
+                    /* Don't offer legacy features for modern devices. */
+                    features.features = (uint32_t)
+                        (vdev->host_features & ~VIRTIO_LEGACY_FEATURES);
+                } else {
+                    features.features = (uint32_t)vdev->host_features;
+                }
+            } else if ((features.index == 1) && (dev->revision >= 1)) {
+                /*
+                 * Only offer feature bits beyond 31 if the guest has
+                 * negotiated at least revision 1.
+                 */
+                features.features = (uint32_t)(vdev->host_features >> 32);
             } else {
                 /* Return zeroes if the guest supports more feature bits. */
                 features.features = 0;
             }
-            stl_le_phys(&address_space_memory, ccw.cda, features.features);
+            address_space_stl_le(&address_space_memory, ccw.cda,
+                                 features.features, MEMTXATTRS_UNSPECIFIED,
+                                 NULL);
             sch->curr_status.scsw.count = ccw.count - sizeof(features);
             ret = 0;
         }
@@ -396,12 +374,28 @@
         if (!ccw.cda) {
             ret = -EFAULT;
         } else {
-            features.index = ldub_phys(&address_space_memory,
-                                       ccw.cda + sizeof(features.features));
-            features.features = ldl_le_phys(&address_space_memory, ccw.cda);
-            if (features.index < ARRAY_SIZE(dev->host_features)) {
-                virtio_bus_set_vdev_features(&dev->bus, features.features);
-                vdev->guest_features = features.features;
+            features.index = address_space_ldub(&address_space_memory,
+                                                ccw.cda
+                                                + sizeof(features.features),
+                                                MEMTXATTRS_UNSPECIFIED,
+                                                NULL);
+            features.features = address_space_ldl_le(&address_space_memory,
+                                                     ccw.cda,
+                                                     MEMTXATTRS_UNSPECIFIED,
+                                                     NULL);
+            if (features.index == 0) {
+                virtio_set_features(vdev,
+                                    (vdev->guest_features & 0xffffffff00000000ULL) |
+                                    features.features);
+            } else if ((features.index == 1) && (dev->revision >= 1)) {
+                /*
+                 * If the guest did not negotiate at least revision 1,
+                 * we did not offer it any feature bits beyond 31. Such a
+                 * guest passing us any bit here is therefore buggy.
+                 */
+                virtio_set_features(vdev,
+                                    (vdev->guest_features & 0x00000000ffffffffULL) |
+                                    ((uint64_t)features.features << 32));
             } else {
                 /*
                  * If the guest supports more feature bits, assert that it
@@ -475,19 +469,24 @@
         if (!ccw.cda) {
             ret = -EFAULT;
         } else {
-            status = ldub_phys(&address_space_memory, ccw.cda);
+            status = address_space_ldub(&address_space_memory, ccw.cda,
+                                        MEMTXATTRS_UNSPECIFIED, NULL);
             if (!(status & VIRTIO_CONFIG_S_DRIVER_OK)) {
                 virtio_ccw_stop_ioeventfd(dev);
             }
-            virtio_set_status(vdev, status);
-            if (vdev->status == 0) {
-                virtio_reset(vdev);
+            if (virtio_set_status(vdev, status) == 0) {
+                if (vdev->status == 0) {
+                    virtio_ccw_reset_virtio(dev, vdev);
+                }
+                if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
+                    virtio_ccw_start_ioeventfd(dev);
+                }
+                sch->curr_status.scsw.count = ccw.count - sizeof(status);
+                ret = 0;
+            } else {
+                /* Trigger a command reject. */
+                ret = -ENOSYS;
             }
-            if (status & VIRTIO_CONFIG_S_DRIVER_OK) {
-                virtio_ccw_start_ioeventfd(dev);
-            }
-            sch->curr_status.scsw.count = ccw.count - sizeof(status);
-            ret = 0;
         }
         break;
     case CCW_CMD_SET_IND:
@@ -509,7 +508,8 @@
         if (!ccw.cda) {
             ret = -EFAULT;
         } else {
-            indicators = ldq_phys(&address_space_memory, ccw.cda);
+            indicators = address_space_ldq_be(&address_space_memory, ccw.cda,
+                                              MEMTXATTRS_UNSPECIFIED, NULL);
             dev->indicators = get_indicator(indicators, sizeof(uint64_t));
             sch->curr_status.scsw.count = ccw.count - sizeof(indicators);
             ret = 0;
@@ -529,7 +529,8 @@
         if (!ccw.cda) {
             ret = -EFAULT;
         } else {
-            indicators = ldq_phys(&address_space_memory, ccw.cda);
+            indicators = address_space_ldq_be(&address_space_memory, ccw.cda,
+                                              MEMTXATTRS_UNSPECIFIED, NULL);
             dev->indicators2 = get_indicator(indicators, sizeof(uint64_t));
             sch->curr_status.scsw.count = ccw.count - sizeof(indicators);
             ret = 0;
@@ -549,11 +550,21 @@
         if (!ccw.cda) {
             ret = -EFAULT;
         } else {
-            vq_config.index = lduw_phys(&address_space_memory, ccw.cda);
+            vq_config.index = address_space_lduw_be(&address_space_memory,
+                                                    ccw.cda,
+                                                    MEMTXATTRS_UNSPECIFIED,
+                                                    NULL);
+            if (vq_config.index >= VIRTIO_CCW_QUEUE_MAX) {
+                ret = -EINVAL;
+                break;
+            }
             vq_config.num_max = virtio_queue_get_num(vdev,
                                                      vq_config.index);
-            stw_phys(&address_space_memory,
-                     ccw.cda + sizeof(vq_config.index), vq_config.num_max);
+            address_space_stw_be(&address_space_memory,
+                                 ccw.cda + sizeof(vq_config.index),
+                                 vq_config.num_max,
+                                 MEMTXATTRS_UNSPECIFIED,
+                                 NULL);
             sch->curr_status.scsw.count = ccw.count - sizeof(vq_config);
             ret = 0;
         }
@@ -581,13 +592,17 @@
             if (!thinint) {
                 ret = -EFAULT;
             } else {
+                uint64_t ind_bit = ldq_be_p(&thinint->ind_bit);
+
                 len = hw_len;
                 dev->summary_indicator =
-                    get_indicator(thinint->summary_indicator, sizeof(uint8_t));
-                dev->indicators = get_indicator(thinint->device_indicator,
-                                                thinint->ind_bit / 8 + 1);
+                    get_indicator(ldq_be_p(&thinint->summary_indicator),
+                                  sizeof(uint8_t));
+                dev->indicators =
+                    get_indicator(ldq_be_p(&thinint->device_indicator),
+                                  ind_bit / 8 + 1);
                 dev->thinint_isc = thinint->isc;
-                dev->routes.adapter.ind_offset = thinint->ind_bit;
+                dev->routes.adapter.ind_offset = ind_bit;
                 dev->routes.adapter.summary_offset = 7;
                 cpu_physical_memory_unmap(thinint, hw_len, 0, hw_len);
                 ret = css_register_io_adapter(CSS_IO_ADAPTER_VIRTIO,
@@ -601,6 +616,40 @@
             }
         }
         break;
+    case CCW_CMD_SET_VIRTIO_REV:
+        len = sizeof(revinfo);
+        if (ccw.count < len) {
+            ret = -EINVAL;
+            break;
+        }
+        if (!ccw.cda) {
+            ret = -EFAULT;
+            break;
+        }
+        revinfo.revision =
+            address_space_lduw_be(&address_space_memory, ccw.cda,
+                                  MEMTXATTRS_UNSPECIFIED, NULL);
+        revinfo.length =
+            address_space_lduw_be(&address_space_memory,
+                                  ccw.cda + sizeof(revinfo.revision),
+                                  MEMTXATTRS_UNSPECIFIED, NULL);
+        if (ccw.count < len + revinfo.length ||
+            (check_len && ccw.count > len + revinfo.length)) {
+            ret = -EINVAL;
+            break;
+        }
+        /*
+         * Once we start to support revisions with additional data, we'll
+         * need to fetch it here. Nothing to do for now, though.
+         */
+        if (dev->revision >= 0 ||
+            revinfo.revision > virtio_ccw_rev_max(dev)) {
+            ret = -ENOSYS;
+            break;
+        }
+        ret = 0;
+        dev->revision = revinfo.revision;
+        break;
     default:
         ret = -ENOSYS;
         break;
@@ -608,158 +657,53 @@
     return ret;
 }
 
-static int virtio_ccw_device_init(VirtioCcwDevice *dev, VirtIODevice *vdev)
+static void virtio_sch_disable_cb(SubchDev *sch)
 {
-    unsigned int cssid = 0;
-    unsigned int ssid = 0;
-    unsigned int schid;
-    unsigned int devno;
-    bool have_devno = false;
-    bool found = false;
-    SubchDev *sch;
-    int ret;
-    int num;
-    DeviceState *parent = DEVICE(dev);
+    VirtioCcwDevice *dev = sch->driver_data;
 
-    sch = g_malloc0(sizeof(SubchDev));
+    dev->revision = -1;
+}
+
+static void virtio_ccw_device_realize(VirtioCcwDevice *dev, Error **errp)
+{
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_GET_CLASS(dev);
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
+    SubchDev *sch = css_create_virtual_sch(ccw_dev->bus_id, errp);
+    Error *err = NULL;
+
+    if (!sch) {
+        return;
+    }
 
     sch->driver_data = dev;
-    dev->sch = sch;
-
-    dev->indicators = NULL;
-
-    /* Initialize subchannel structure. */
-    sch->channel_prog = 0x0;
-    sch->last_cmd_valid = false;
-    sch->thinint_active = false;
-    /*
-     * Use a device number if provided. Otherwise, fall back to subchannel
-     * number.
-     */
-    if (dev->bus_id) {
-        num = sscanf(dev->bus_id, "%x.%x.%04x", &cssid, &ssid, &devno);
-        if (num == 3) {
-            if ((cssid > MAX_CSSID) || (ssid > MAX_SSID)) {
-                ret = -EINVAL;
-                error_report("Invalid cssid or ssid: cssid %x, ssid %x",
-                             cssid, ssid);
-                goto out_err;
-            }
-            /* Enforce use of virtual cssid. */
-            if (cssid != VIRTUAL_CSSID) {
-                ret = -EINVAL;
-                error_report("cssid %x not valid for virtio devices", cssid);
-                goto out_err;
-            }
-            if (css_devno_used(cssid, ssid, devno)) {
-                ret = -EEXIST;
-                error_report("Device %x.%x.%04x already exists", cssid, ssid,
-                             devno);
-                goto out_err;
-            }
-            sch->cssid = cssid;
-            sch->ssid = ssid;
-            sch->devno = devno;
-            have_devno = true;
-        } else {
-            ret = -EINVAL;
-            error_report("Malformed devno parameter '%s'", dev->bus_id);
-            goto out_err;
-        }
-    }
-
-    /* Find the next free id. */
-    if (have_devno) {
-        for (schid = 0; schid <= MAX_SCHID; schid++) {
-            if (!css_find_subch(1, cssid, ssid, schid)) {
-                sch->schid = schid;
-                css_subch_assign(cssid, ssid, schid, devno, sch);
-                found = true;
-                break;
-            }
-        }
-        if (!found) {
-            ret = -ENODEV;
-            error_report("No free subchannel found for %x.%x.%04x", cssid, ssid,
-                         devno);
-            goto out_err;
-        }
-        trace_virtio_ccw_new_device(cssid, ssid, schid, devno,
-                                    "user-configured");
-    } else {
-        cssid = VIRTUAL_CSSID;
-        for (ssid = 0; ssid <= MAX_SSID; ssid++) {
-            for (schid = 0; schid <= MAX_SCHID; schid++) {
-                if (!css_find_subch(1, cssid, ssid, schid)) {
-                    sch->cssid = cssid;
-                    sch->ssid = ssid;
-                    sch->schid = schid;
-                    devno = schid;
-                    /*
-                     * If the devno is already taken, look further in this
-                     * subchannel set.
-                     */
-                    while (css_devno_used(cssid, ssid, devno)) {
-                        if (devno == MAX_SCHID) {
-                            devno = 0;
-                        } else if (devno == schid - 1) {
-                            ret = -ENODEV;
-                            error_report("No free devno found");
-                            goto out_err;
-                        } else {
-                            devno++;
-                        }
-                    }
-                    sch->devno = devno;
-                    css_subch_assign(cssid, ssid, schid, devno, sch);
-                    found = true;
-                    break;
-                }
-            }
-            if (found) {
-                break;
-            }
-        }
-        if (!found) {
-            ret = -ENODEV;
-            error_report("Virtual channel subsystem is full!");
-            goto out_err;
-        }
-        trace_virtio_ccw_new_device(cssid, ssid, schid, devno,
-                                    "auto-configured");
-    }
-
-    /* Build initial schib. */
-    css_sch_build_virtual_schib(sch, 0, VIRTIO_CCW_CHPID_TYPE);
-
     sch->ccw_cb = virtio_ccw_cb;
-
-    /* Build senseid data. */
-    memset(&sch->id, 0, sizeof(SenseId));
+    sch->disable_cb = virtio_sch_disable_cb;
     sch->id.reserved = 0xff;
     sch->id.cu_type = VIRTIO_CCW_CU_TYPE;
-    sch->id.cu_model = vdev->device_id;
+    ccw_dev->sch = sch;
+    dev->indicators = NULL;
+    dev->revision = -1;
+    css_sch_build_virtual_schib(sch, 0, VIRTIO_CCW_CHPID_TYPE);
 
-    /* Only the first 32 feature bits are used. */
-    dev->host_features[0] = virtio_bus_get_vdev_features(&dev->bus,
-                                                         dev->host_features[0]);
+    trace_virtio_ccw_new_device(
+        sch->cssid, sch->ssid, sch->schid, sch->devno,
+        ccw_dev->bus_id.valid ? "user-configured" : "auto-configured");
 
-    dev->host_features[0] |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
-    dev->host_features[0] |= 0x1 << VIRTIO_F_BAD_FEATURE;
-
-    css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid,
-                          parent->hotplugged, 1);
-    return 0;
-
-out_err:
-    dev->sch = NULL;
-    g_free(sch);
-    return ret;
+    if (k->realize) {
+        k->realize(dev, &err);
+    }
+    if (err) {
+        error_propagate(errp, err);
+        css_subch_assign(sch->cssid, sch->ssid, sch->schid, sch->devno, NULL);
+        ccw_dev->sch = NULL;
+        g_free(sch);
+    }
 }
 
 static int virtio_ccw_exit(VirtioCcwDevice *dev)
 {
-    SubchDev *sch = dev->sch;
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
+    SubchDev *sch = ccw_dev->sch;
 
     if (sch) {
         css_subch_assign(sch->cssid, sch->ssid, sch->schid, sch->devno, NULL);
@@ -772,21 +716,16 @@
     return 0;
 }
 
-static int virtio_ccw_net_init(VirtioCcwDevice *ccw_dev)
+static void virtio_ccw_net_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     DeviceState *qdev = DEVICE(ccw_dev);
     VirtIONetCcw *dev = VIRTIO_NET_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
 
-    virtio_net_set_config_size(&dev->vdev, ccw_dev->host_features[0]);
     virtio_net_set_netclient_name(&dev->vdev, qdev->id,
                                   object_get_typename(OBJECT(qdev)));
     qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_ccw_net_instance_init(Object *obj)
@@ -799,16 +738,13 @@
                               "bootindex", &error_abort);
 }
 
-static int virtio_ccw_blk_init(VirtioCcwDevice *ccw_dev)
+static void virtio_ccw_blk_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     VirtIOBlkCcw *dev = VIRTIO_BLK_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
-    qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
 
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
+    qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_ccw_blk_instance_init(Object *obj)
@@ -823,7 +759,7 @@
                               "bootindex", &error_abort);
 }
 
-static int virtio_ccw_serial_init(VirtioCcwDevice *ccw_dev)
+static void virtio_ccw_serial_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     VirtioSerialCcw *dev = VIRTIO_SERIAL_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
@@ -841,11 +777,7 @@
     }
 
     qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 
@@ -857,61 +789,29 @@
                                 TYPE_VIRTIO_SERIAL);
 }
 
-static int virtio_ccw_balloon_init(VirtioCcwDevice *ccw_dev)
+static void virtio_ccw_balloon_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     VirtIOBalloonCcw *dev = VIRTIO_BALLOON_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
 
     qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
-}
-
-static void balloon_ccw_stats_get_all(Object *obj, struct Visitor *v,
-                                      void *opaque, const char *name,
-                                      Error **errp)
-{
-    VirtIOBalloonCcw *dev = opaque;
-    object_property_get(OBJECT(&dev->vdev), v, "guest-stats", errp);
-}
-
-static void balloon_ccw_stats_get_poll_interval(Object *obj, struct Visitor *v,
-                                                void *opaque, const char *name,
-                                                Error **errp)
-{
-    VirtIOBalloonCcw *dev = opaque;
-    object_property_get(OBJECT(&dev->vdev), v, "guest-stats-polling-interval",
-                        errp);
-}
-
-static void balloon_ccw_stats_set_poll_interval(Object *obj, struct Visitor *v,
-                                                void *opaque, const char *name,
-                                                Error **errp)
-{
-    VirtIOBalloonCcw *dev = opaque;
-    object_property_set(OBJECT(&dev->vdev), v, "guest-stats-polling-interval",
-                        errp);
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_ccw_balloon_instance_init(Object *obj)
 {
     VirtIOBalloonCcw *dev = VIRTIO_BALLOON_CCW(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BALLOON);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_unref(OBJECT(&dev->vdev));
-    object_property_add(obj, "guest-stats", "guest statistics",
-                        balloon_ccw_stats_get_all, NULL, NULL, dev, NULL);
 
-    object_property_add(obj, "guest-stats-polling-interval", "int",
-                        balloon_ccw_stats_get_poll_interval,
-                        balloon_ccw_stats_set_poll_interval,
-                        NULL, dev, NULL);
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_BALLOON);
+    object_property_add_alias(obj, "guest-stats", OBJECT(&dev->vdev),
+                              "guest-stats", &error_abort);
+    object_property_add_alias(obj, "guest-stats-polling-interval",
+                              OBJECT(&dev->vdev),
+                              "guest-stats-polling-interval", &error_abort);
 }
 
-static int virtio_ccw_scsi_init(VirtioCcwDevice *ccw_dev)
+static void virtio_ccw_scsi_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     VirtIOSCSICcw *dev = VIRTIO_SCSI_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
@@ -929,11 +829,7 @@
     }
 
     qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_ccw_scsi_instance_init(Object *obj)
@@ -947,17 +843,13 @@
 }
 
 #ifdef CONFIG_VHOST_SCSI
-static int vhost_ccw_scsi_init(VirtioCcwDevice *ccw_dev)
+static void vhost_ccw_scsi_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     VHostSCSICcw *dev = VHOST_SCSI_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
 
     qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void vhost_ccw_scsi_instance_init(Object *obj)
@@ -969,21 +861,22 @@
 }
 #endif
 
-static int virtio_ccw_rng_init(VirtioCcwDevice *ccw_dev)
+static void virtio_ccw_rng_realize(VirtioCcwDevice *ccw_dev, Error **errp)
 {
     VirtIORNGCcw *dev = VIRTIO_RNG_CCW(ccw_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
+    Error *err = NULL;
 
     qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
+    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
 
     object_property_set_link(OBJECT(dev),
                              OBJECT(dev->vdev.conf.rng), "rng",
                              NULL);
-
-    return virtio_ccw_device_init(ccw_dev, VIRTIO_DEVICE(vdev));
 }
 
 /* DeviceState to VirtioCcwDevice. Note: used on datapath,
@@ -991,7 +884,9 @@
  */
 static inline VirtioCcwDevice *to_virtio_ccw_dev_fast(DeviceState *d)
 {
-    return container_of(d, VirtioCcwDevice, parent_obj);
+    CcwDevice *ccw_dev = to_ccw_dev_fast(d);
+
+    return container_of(ccw_dev, VirtioCcwDevice, parent_obj);
 }
 
 static uint8_t virtio_set_ind_atomic(SubchDev *sch, uint64_t ind_loc,
@@ -1011,6 +906,7 @@
         ind_old = *ind_addr;
         ind_new = ind_old | to_be_set;
     } while (atomic_cmpxchg(ind_addr, ind_old, ind_new) != ind_old);
+    trace_virtio_ccw_set_ind(ind_loc, ind_old, ind_new);
     cpu_physical_memory_unmap(ind_addr, len, 1, len);
 
     return ind_old;
@@ -1019,14 +915,16 @@
 static void virtio_ccw_notify(DeviceState *d, uint16_t vector)
 {
     VirtioCcwDevice *dev = to_virtio_ccw_dev_fast(d);
-    SubchDev *sch = dev->sch;
+    CcwDevice *ccw_dev = to_ccw_dev_fast(d);
+    SubchDev *sch = ccw_dev->sch;
     uint64_t indicators;
 
-    if (vector >= 128) {
+    /* queue indicators + secondary indicators */
+    if (vector >= VIRTIO_CCW_QUEUE_MAX + 64) {
         return;
     }
 
-    if (vector < VIRTIO_PCI_QUEUE_MAX) {
+    if (vector < VIRTIO_CCW_QUEUE_MAX) {
         if (!dev->indicators) {
             return;
         }
@@ -1047,9 +945,13 @@
                 css_adapter_interrupt(dev->thinint_isc);
             }
         } else {
-            indicators = ldq_phys(&address_space_memory, dev->indicators->addr);
+            indicators = address_space_ldq(&address_space_memory,
+                                           dev->indicators->addr,
+                                           MEMTXATTRS_UNSPECIFIED,
+                                           NULL);
             indicators |= 1ULL << vector;
-            stq_phys(&address_space_memory, dev->indicators->addr, indicators);
+            address_space_stq(&address_space_memory, dev->indicators->addr,
+                              indicators, MEMTXATTRS_UNSPECIFIED, NULL);
             css_conditional_io_interrupt(sch);
         }
     } else {
@@ -1057,41 +959,25 @@
             return;
         }
         vector = 0;
-        indicators = ldq_phys(&address_space_memory, dev->indicators2->addr);
+        indicators = address_space_ldq(&address_space_memory,
+                                       dev->indicators2->addr,
+                                       MEMTXATTRS_UNSPECIFIED,
+                                       NULL);
         indicators |= 1ULL << vector;
-        stq_phys(&address_space_memory, dev->indicators2->addr, indicators);
+        address_space_stq(&address_space_memory, dev->indicators2->addr,
+                          indicators, MEMTXATTRS_UNSPECIFIED, NULL);
         css_conditional_io_interrupt(sch);
     }
 }
 
-static unsigned virtio_ccw_get_features(DeviceState *d)
-{
-    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
-
-    /* Only the first 32 feature bits are used. */
-    return dev->host_features[0];
-}
-
 static void virtio_ccw_reset(DeviceState *d)
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
     VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    CcwDevice *ccw_dev = CCW_DEVICE(d);
 
-    virtio_ccw_stop_ioeventfd(dev);
-    virtio_reset(vdev);
-    css_reset_sch(dev->sch);
-    if (dev->indicators) {
-        release_indicator(&dev->routes.adapter, dev->indicators);
-        dev->indicators = NULL;
-    }
-    if (dev->indicators2) {
-        release_indicator(&dev->routes.adapter, dev->indicators2);
-        dev->indicators2 = NULL;
-    }
-    if (dev->summary_indicator) {
-        release_indicator(&dev->routes.adapter, dev->summary_indicator);
-        dev->summary_indicator = NULL;
-    }
+    virtio_ccw_reset_virtio(dev, vdev);
+    css_reset_sch(ccw_dev->sch);
 }
 
 static void virtio_ccw_vmstate_change(DeviceState *d, bool running)
@@ -1107,29 +993,17 @@
 
 static bool virtio_ccw_query_guest_notifiers(DeviceState *d)
 {
-    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    CcwDevice *dev = CCW_DEVICE(d);
 
     return !!(dev->sch->curr_status.pmcw.flags & PMCW_FLAGS_MASK_ENA);
 }
 
-static int virtio_ccw_set_host_notifier(DeviceState *d, int n, bool assign)
-{
-    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
-
-    /* Stop using the generic ioeventfd, we are doing eventfd handling
-     * ourselves below */
-    dev->ioeventfd_disabled = assign;
-    if (assign) {
-        virtio_ccw_stop_ioeventfd(dev);
-    }
-    return virtio_ccw_set_guest2host_notifier(dev, n, assign, false);
-}
-
 static int virtio_ccw_get_mappings(VirtioCcwDevice *dev)
 {
     int r;
+    CcwDevice *ccw_dev = CCW_DEVICE(dev);
 
-    if (!dev->sch->thinint_active) {
+    if (!ccw_dev->sch->thinint_active) {
         return -EINVAL;
     }
 
@@ -1182,8 +1056,8 @@
     VirtQueue *vq = virtio_get_queue(vdev, n);
     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
 
-    return kvm_irqchip_add_irqfd_notifier(kvm_state, notifier, NULL,
-                                          dev->routes.gsi[n]);
+    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, notifier, NULL,
+                                              dev->routes.gsi[n]);
 }
 
 static void virtio_ccw_remove_irqfd(VirtioCcwDevice *dev, int n)
@@ -1193,8 +1067,8 @@
     EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
     int ret;
 
-    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, notifier,
-                                            dev->routes.gsi[n]);
+    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, notifier,
+                                                dev->routes.gsi[n]);
     assert(ret == 0);
 }
 
@@ -1251,7 +1125,8 @@
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
     VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
-    bool with_irqfd = dev->sch->thinint_active && kvm_irqfds_enabled();
+    CcwDevice *ccw_dev = CCW_DEVICE(d);
+    bool with_irqfd = ccw_dev->sch->thinint_active && kvm_irqfds_enabled();
     int r, n;
 
     if (with_irqfd && assigned) {
@@ -1310,7 +1185,9 @@
 static void virtio_ccw_save_config(DeviceState *d, QEMUFile *f)
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
-    SubchDev *s = dev->sch;
+    CcwDevice *ccw_dev = CCW_DEVICE(d);
+    SubchDev *s = ccw_dev->sch;
+    VirtIODevice *vdev = virtio_ccw_get_vdev(s);
 
     subch_device_save(s, f);
     if (dev->indicators != NULL) {
@@ -1334,14 +1211,18 @@
         qemu_put_be32(f, 0);
         qemu_put_be64(f, 0UL);
     }
+    qemu_put_be16(f, vdev->config_vector);
     qemu_put_be64(f, dev->routes.adapter.ind_offset);
     qemu_put_byte(f, dev->thinint_isc);
+    qemu_put_be32(f, dev->revision);
 }
 
 static int virtio_ccw_load_config(DeviceState *d, QEMUFile *f)
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
-    SubchDev *s = dev->sch;
+    CcwDevice *ccw_dev = CCW_DEVICE(d);
+    SubchDev *s = ccw_dev->sch;
+    VirtIODevice *vdev = virtio_ccw_get_vdev(s);
     int len;
 
     s->driver_data = dev;
@@ -1367,8 +1248,10 @@
         qemu_get_be64(f);
         dev->summary_indicator = NULL;
     }
+    qemu_get_be16s(f, &vdev->config_vector);
     dev->routes.adapter.ind_offset = qemu_get_be64(f);
     dev->thinint_isc = qemu_get_byte(f);
+    dev->revision = qemu_get_be32(f);
     if (s->thinint_active) {
         return css_register_io_adapter(CSS_IO_ADAPTER_VIRTIO,
                                        dev->thinint_isc, true, false,
@@ -1378,13 +1261,61 @@
     return 0;
 }
 
+/* This is called by virtio-bus just after the device is plugged. */
+static void virtio_ccw_device_plugged(DeviceState *d, Error **errp)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    CcwDevice *ccw_dev = CCW_DEVICE(d);
+    SubchDev *sch = ccw_dev->sch;
+    int n = virtio_get_num_queues(vdev);
+
+    if (virtio_get_num_queues(vdev) > VIRTIO_CCW_QUEUE_MAX) {
+        error_setg(errp, "The number of virtqueues %d "
+                   "exceeds ccw limit %d", n,
+                   VIRTIO_CCW_QUEUE_MAX);
+        return;
+    }
+
+    if (!kvm_eventfds_enabled()) {
+        dev->flags &= ~VIRTIO_CCW_FLAG_USE_IOEVENTFD;
+    }
+
+    sch->id.cu_model = virtio_bus_get_vdev_id(&dev->bus);
+
+    if (dev->max_rev >= 1) {
+        virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
+    }
+
+    css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid,
+                          d->hotplugged, 1);
+}
+
+static void virtio_ccw_post_plugged(DeviceState *d, Error **errp)
+{
+   VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+   VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+
+   if (!virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        /* A backend didn't support modern virtio. */
+       dev->max_rev = 0;
+   }
+}
+
+static void virtio_ccw_device_unplugged(DeviceState *d)
+{
+    VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
+
+    virtio_ccw_stop_ioeventfd(dev);
+}
 /**************** Virtio-ccw Bus Device Descriptions *******************/
 
 static Property virtio_ccw_net_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
-    DEFINE_VIRTIO_NET_FEATURES(VirtioCcwDevice, host_features[0]),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1393,10 +1324,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = virtio_ccw_net_init;
+    k->realize = virtio_ccw_net_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = virtio_ccw_net_properties;
+    set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
 }
 
 static const TypeInfo virtio_ccw_net = {
@@ -1408,9 +1340,11 @@
 };
 
 static Property virtio_ccw_blk_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1419,10 +1353,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = virtio_ccw_blk_init;
+    k->realize = virtio_ccw_blk_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = virtio_ccw_blk_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo virtio_ccw_blk = {
@@ -1434,9 +1369,11 @@
 };
 
 static Property virtio_ccw_serial_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1445,10 +1382,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = virtio_ccw_serial_init;
+    k->realize = virtio_ccw_serial_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = virtio_ccw_serial_properties;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo virtio_ccw_serial = {
@@ -1460,9 +1398,11 @@
 };
 
 static Property virtio_ccw_balloon_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1471,10 +1411,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = virtio_ccw_balloon_init;
+    k->realize = virtio_ccw_balloon_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = virtio_ccw_balloon_properties;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo virtio_ccw_balloon = {
@@ -1486,10 +1427,11 @@
 };
 
 static Property virtio_ccw_scsi_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
-    DEFINE_VIRTIO_SCSI_FEATURES(VirtioCcwDevice, host_features[0]),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1498,10 +1440,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = virtio_ccw_scsi_init;
+    k->realize = virtio_ccw_scsi_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = virtio_ccw_scsi_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo virtio_ccw_scsi = {
@@ -1514,7 +1457,9 @@
 
 #ifdef CONFIG_VHOST_SCSI
 static Property vhost_ccw_scsi_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1523,10 +1468,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = vhost_ccw_scsi_init;
+    k->realize = vhost_ccw_scsi_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = vhost_ccw_scsi_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
 }
 
 static const TypeInfo vhost_ccw_scsi = {
@@ -1549,9 +1495,11 @@
 }
 
 static Property virtio_ccw_rng_properties[] = {
-    DEFINE_PROP_STRING("devno", VirtioCcwDevice, bus_id),
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
     DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
                     VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -1560,10 +1508,11 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
 
-    k->init = virtio_ccw_rng_init;
+    k->realize = virtio_ccw_rng_realize;
     k->exit = virtio_ccw_exit;
     dc->reset = virtio_ccw_reset;
     dc->props = virtio_ccw_rng_properties;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
 }
 
 static const TypeInfo virtio_ccw_rng = {
@@ -1574,14 +1523,12 @@
     .class_init    = virtio_ccw_rng_class_init,
 };
 
-static int virtio_ccw_busdev_init(DeviceState *dev)
+static void virtio_ccw_busdev_realize(DeviceState *dev, Error **errp)
 {
     VirtioCcwDevice *_dev = (VirtioCcwDevice *)dev;
-    VirtIOCCWDeviceClass *_info = VIRTIO_CCW_DEVICE_GET_CLASS(dev);
 
     virtio_ccw_bus_new(&_dev->bus, sizeof(_dev->bus), _dev);
-
-    return _info->init(_dev);
+    virtio_ccw_device_realize(_dev, errp);
 }
 
 static int virtio_ccw_busdev_exit(DeviceState *dev)
@@ -1595,78 +1542,31 @@
 static void virtio_ccw_busdev_unplug(HotplugHandler *hotplug_dev,
                                      DeviceState *dev, Error **errp)
 {
-    VirtioCcwDevice *_dev = (VirtioCcwDevice *)dev;
-    SubchDev *sch = _dev->sch;
+    VirtioCcwDevice *_dev = to_virtio_ccw_dev_fast(dev);
 
     virtio_ccw_stop_ioeventfd(_dev);
-
-    /*
-     * We should arrive here only for device_del, since we don't support
-     * direct hot(un)plug of channels, but only through virtio.
-     */
-    assert(sch != NULL);
-    /* Subchannel is now disabled and no longer valid. */
-    sch->curr_status.pmcw.flags &= ~(PMCW_FLAGS_MASK_ENA |
-                                     PMCW_FLAGS_MASK_DNV);
-
-    css_generate_sch_crws(sch->cssid, sch->ssid, sch->schid, 1, 0);
-
-    object_unparent(OBJECT(dev));
 }
 
-static Property virtio_ccw_properties[] = {
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtioCcwDevice, host_features[0]),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void virtio_ccw_device_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    CCWDeviceClass *k = CCW_DEVICE_CLASS(dc);
 
-    dc->props = virtio_ccw_properties;
-    dc->init = virtio_ccw_busdev_init;
+    k->unplug = virtio_ccw_busdev_unplug;
+    dc->realize = virtio_ccw_busdev_realize;
     dc->exit = virtio_ccw_busdev_exit;
     dc->bus_type = TYPE_VIRTUAL_CSS_BUS;
 }
 
 static const TypeInfo virtio_ccw_device_info = {
     .name = TYPE_VIRTIO_CCW_DEVICE,
-    .parent = TYPE_DEVICE,
+    .parent = TYPE_CCW_DEVICE,
     .instance_size = sizeof(VirtioCcwDevice),
     .class_init = virtio_ccw_device_class_init,
     .class_size = sizeof(VirtIOCCWDeviceClass),
     .abstract = true,
 };
 
-/***************** Virtual-css Bus Bridge Device ********************/
-/* Only required to have the virtio bus as child in the system bus */
-
-static int virtual_css_bridge_init(SysBusDevice *dev)
-{
-    /* nothing */
-    return 0;
-}
-
-static void virtual_css_bridge_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-    HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
-
-    k->init = virtual_css_bridge_init;
-    hc->unplug = virtio_ccw_busdev_unplug;
-}
-
-static const TypeInfo virtual_css_bridge_info = {
-    .name          = "virtual-css-bridge",
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(SysBusDevice),
-    .class_init    = virtual_css_bridge_class_init,
-    .interfaces = (InterfaceInfo[]) {
-        { TYPE_HOTPLUG_HANDLER },
-        { }
-    }
-};
-
 /* virtio-ccw-bus */
 
 static void virtio_ccw_bus_new(VirtioBusState *bus, size_t bus_size,
@@ -1686,15 +1586,21 @@
 
     bus_class->max_dev = 1;
     k->notify = virtio_ccw_notify;
-    k->get_features = virtio_ccw_get_features;
     k->vmstate_change = virtio_ccw_vmstate_change;
     k->query_guest_notifiers = virtio_ccw_query_guest_notifiers;
-    k->set_host_notifier = virtio_ccw_set_host_notifier;
     k->set_guest_notifiers = virtio_ccw_set_guest_notifiers;
     k->save_queue = virtio_ccw_save_queue;
     k->load_queue = virtio_ccw_load_queue;
     k->save_config = virtio_ccw_save_config;
     k->load_config = virtio_ccw_load_config;
+    k->device_plugged = virtio_ccw_device_plugged;
+    k->post_plugged = virtio_ccw_post_plugged;
+    k->device_unplugged = virtio_ccw_device_unplugged;
+    k->ioeventfd_started = virtio_ccw_ioeventfd_started;
+    k->ioeventfd_set_started = virtio_ccw_ioeventfd_set_started;
+    k->ioeventfd_disabled = virtio_ccw_ioeventfd_disabled;
+    k->ioeventfd_set_disabled = virtio_ccw_ioeventfd_set_disabled;
+    k->ioeventfd_assign = virtio_ccw_ioeventfd_assign;
 }
 
 static const TypeInfo virtio_ccw_bus_info = {
@@ -1704,10 +1610,57 @@
     .class_init = virtio_ccw_bus_class_init,
 };
 
+#ifdef CONFIG_VIRTFS
+static Property virtio_ccw_9p_properties[] = {
+    DEFINE_PROP_CSS_DEV_ID("devno", VirtioCcwDevice, parent_obj.bus_id),
+    DEFINE_PROP_BIT("ioeventfd", VirtioCcwDevice, flags,
+            VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT, true),
+    DEFINE_PROP_UINT32("max_revision", VirtioCcwDevice, max_rev,
+                       VIRTIO_CCW_MAX_REV),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_ccw_9p_realize(VirtioCcwDevice *ccw_dev, Error **errp)
+{
+    V9fsCCWState *dev = VIRTIO_9P_CCW(ccw_dev);
+    DeviceState *vdev = DEVICE(&dev->vdev);
+
+    qdev_set_parent_bus(vdev, BUS(&ccw_dev->bus));
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
+}
+
+static void virtio_ccw_9p_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtIOCCWDeviceClass *k = VIRTIO_CCW_DEVICE_CLASS(klass);
+
+    k->exit = virtio_ccw_exit;
+    k->realize = virtio_ccw_9p_realize;
+    dc->reset = virtio_ccw_reset;
+    dc->props = virtio_ccw_9p_properties;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
+}
+
+static void virtio_ccw_9p_instance_init(Object *obj)
+{
+    V9fsCCWState *dev = VIRTIO_9P_CCW(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_9P);
+}
+
+static const TypeInfo virtio_ccw_9p_info = {
+    .name          = TYPE_VIRTIO_9P_CCW,
+    .parent        = TYPE_VIRTIO_CCW_DEVICE,
+    .instance_size = sizeof(V9fsCCWState),
+    .instance_init = virtio_ccw_9p_instance_init,
+    .class_init    = virtio_ccw_9p_class_init,
+};
+#endif
+
 static void virtio_ccw_register(void)
 {
     type_register_static(&virtio_ccw_bus_info);
-    type_register_static(&virtual_css_bus_info);
     type_register_static(&virtio_ccw_device_info);
     type_register_static(&virtio_ccw_serial);
     type_register_static(&virtio_ccw_blk);
@@ -1718,7 +1671,9 @@
     type_register_static(&vhost_ccw_scsi);
 #endif
     type_register_static(&virtio_ccw_rng);
-    type_register_static(&virtual_css_bridge_info);
+#ifdef CONFIG_VIRTFS
+    type_register_static(&virtio_ccw_9p_info);
+#endif
 }
 
 type_init(virtio_ccw_register)
diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
index 5a1f16e..1c6bc86 100644
--- a/hw/s390x/virtio-ccw.h
+++ b/hw/s390x/virtio-ccw.h
@@ -1,8 +1,9 @@
 /*
  * virtio ccw target definitions
  *
- * Copyright 2012 IBM Corp.
+ * Copyright 2012,2015 IBM Corp.
  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *            Pierre Morel <pmorel@linux.vnet.ibm.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or (at
  * your option) any later version. See the COPYING file in the top-level
@@ -12,19 +13,21 @@
 #ifndef HW_S390X_VIRTIO_CCW_H
 #define HW_S390X_VIRTIO_CCW_H
 
-#include <hw/virtio/virtio-blk.h>
-#include <hw/virtio/virtio-net.h>
-#include <hw/virtio/virtio-serial.h>
-#include <hw/virtio/virtio-scsi.h>
+#include "hw/virtio/virtio-blk.h"
+#include "hw/virtio/virtio-net.h"
+#include "hw/virtio/virtio-serial.h"
+#include "hw/virtio/virtio-scsi.h"
 #ifdef CONFIG_VHOST_SCSI
-#include <hw/virtio/vhost-scsi.h>
+#include "hw/virtio/vhost-scsi.h"
 #endif
-#include <hw/virtio/virtio-balloon.h>
-#include <hw/virtio/virtio-rng.h>
-#include <hw/virtio/virtio-bus.h>
-#include <hw/s390x/s390_flic.h>
+#include "hw/virtio/virtio-balloon.h"
+#include "hw/virtio/virtio-rng.h"
+#include "hw/virtio/virtio-bus.h"
 
-#define VIRTUAL_CSSID 0xfe
+#include "hw/s390x/s390_flic.h"
+#include "hw/s390x/css.h"
+#include "ccw-device.h"
+#include "hw/s390x/css-bridge.h"
 
 #define VIRTIO_CCW_CU_TYPE 0x3832
 #define VIRTIO_CCW_CHPID_TYPE 0x32
@@ -40,6 +43,7 @@
 #define CCW_CMD_SET_CONF_IND 0x53
 #define CCW_CMD_READ_VQ_CONF 0x32
 #define CCW_CMD_SET_IND_ADAPTER 0x73
+#define CCW_CMD_SET_VIRTIO_REV 0x83
 
 #define TYPE_VIRTIO_CCW_DEVICE "virtio-ccw-device"
 #define VIRTIO_CCW_DEVICE(obj) \
@@ -63,32 +67,20 @@
 typedef struct VirtioCcwDevice VirtioCcwDevice;
 
 typedef struct VirtIOCCWDeviceClass {
-    DeviceClass parent_class;
-    int (*init)(VirtioCcwDevice *dev);
+    CCWDeviceClass parent_class;
+    void (*realize)(VirtioCcwDevice *dev, Error **errp);
     int (*exit)(VirtioCcwDevice *dev);
 } VirtIOCCWDeviceClass;
 
-/* Change here if we want to support more feature bits. */
-#define VIRTIO_CCW_FEATURE_SIZE 1
-
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
 #define VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT 1
 #define VIRTIO_CCW_FLAG_USE_IOEVENTFD   (1 << VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT)
 
-typedef struct IndAddr {
-    hwaddr addr;
-    uint64_t map;
-    unsigned long refcnt;
-    int len;
-    QTAILQ_ENTRY(IndAddr) sibling;
-} IndAddr;
-
 struct VirtioCcwDevice {
-    DeviceState parent_obj;
-    SubchDev *sch;
-    char *bus_id;
-    uint32_t host_features[VIRTIO_CCW_FEATURE_SIZE];
+    CcwDevice parent_obj;
+    int revision;
+    uint32_t max_rev;
     VirtioBusState bus;
     bool ioeventfd_started;
     bool ioeventfd_disabled;
@@ -102,14 +94,12 @@
     uint64_t ind_bit;
 };
 
-/* virtual css bus type */
-typedef struct VirtualCssBus {
-    BusState parent_obj;
-} VirtualCssBus;
-
-#define TYPE_VIRTUAL_CSS_BUS "virtual-css-bus"
-#define VIRTUAL_CSS_BUS(obj) \
-     OBJECT_CHECK(VirtualCssBus, (obj), TYPE_VIRTUAL_CSS_BUS)
+/* The maximum virtio revision we support. */
+#define VIRTIO_CCW_MAX_REV 1
+static inline int virtio_ccw_rev_max(VirtioCcwDevice *dev)
+{
+    return dev->max_rev;
+}
 
 /* virtio-scsi-ccw */
 
@@ -190,7 +180,21 @@
     VirtIORNG vdev;
 } VirtIORNGCcw;
 
-VirtualCssBus *virtual_css_bus_init(void);
 void virtio_ccw_device_update_status(SubchDev *sch);
 VirtIODevice *virtio_ccw_get_vdev(SubchDev *sch);
+
+#ifdef CONFIG_VIRTFS
+#include "hw/9pfs/virtio-9p.h"
+
+#define TYPE_VIRTIO_9P_CCW "virtio-9p-ccw"
+#define VIRTIO_9P_CCW(obj) \
+    OBJECT_CHECK(V9fsCCWState, (obj), TYPE_VIRTIO_9P_CCW)
+
+typedef struct V9fsCCWState {
+    VirtioCcwDevice parent_obj;
+    V9fsVirtioState vdev;
+} V9fsCCWState;
+
+#endif /* CONFIG_VIRTFS */
+
 #endif
diff --git a/hw/scsi/Makefile.objs b/hw/scsi/Makefile.objs
index 40c79d3..5a2248b 100644
--- a/hw/scsi/Makefile.objs
+++ b/hw/scsi/Makefile.objs
@@ -1,6 +1,7 @@
 common-obj-y += scsi-disk.o
 common-obj-y += scsi-generic.o scsi-bus.o
 common-obj-$(CONFIG_LSI_SCSI_PCI) += lsi53c895a.o
+common-obj-$(CONFIG_MPTSAS_SCSI_PCI) += mptsas.o mptconfig.o mptendian.o
 common-obj-$(CONFIG_MEGASAS_SCSI_PCI) += megasas.o
 common-obj-$(CONFIG_VMW_PVSCSI_SCSI_PCI) += vmw_pvscsi.o
 common-obj-$(CONFIG_ESP) += esp.o
diff --git a/hw/scsi/esp-pci.c b/hw/scsi/esp-pci.c
index 00b7297..595f88b 100644
--- a/hw/scsi/esp-pci.c
+++ b/hw/scsi/esp-pci.c
@@ -23,10 +23,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/pci/pci.h"
 #include "hw/nvram/eeprom93xx.h"
 #include "hw/scsi/esp.h"
 #include "trace.h"
+#include "qapi/error.h"
 #include "qemu/log.h"
 
 #define TYPE_AM53C974_DEVICE "am53c974"
@@ -342,13 +344,12 @@
     .cancel = esp_request_cancelled,
 };
 
-static int esp_pci_scsi_init(PCIDevice *dev)
+static void esp_pci_scsi_realize(PCIDevice *dev, Error **errp)
 {
     PCIESPState *pci = PCI_ESP(dev);
     DeviceState *d = DEVICE(dev);
     ESPState *s = &pci->esp;
     uint8_t *pci_conf;
-    Error *err = NULL;
 
     pci_conf = dev->config;
 
@@ -367,13 +368,8 @@
 
     scsi_bus_new(&s->bus, sizeof(s->bus), d, &esp_pci_scsi_info, NULL);
     if (!d->hotplugged) {
-        scsi_bus_legacy_handle_cmdline(&s->bus, &err);
-        if (err != NULL) {
-            error_free(err);
-            return -1;
-        }
+        scsi_bus_legacy_handle_cmdline(&s->bus, errp);
     }
-    return 0;
 }
 
 static void esp_pci_scsi_uninit(PCIDevice *d)
@@ -388,7 +384,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = esp_pci_scsi_init;
+    k->realize = esp_pci_scsi_realize;
     k->exit = esp_pci_scsi_uninit;
     k->vendor_id = PCI_VENDOR_ID_AMD;
     k->device_id = PCI_DEVICE_ID_AMD_SCSI;
@@ -466,17 +462,19 @@
     }
 }
 
-static int dc390_scsi_init(PCIDevice *dev)
+static void dc390_scsi_realize(PCIDevice *dev, Error **errp)
 {
     DC390State *pci = DC390(dev);
+    Error *err = NULL;
     uint8_t *contents;
     uint16_t chksum = 0;
-    int i, ret;
+    int i;
 
     /* init base class */
-    ret = esp_pci_scsi_init(dev);
-    if (ret < 0) {
-        return ret;
+    esp_pci_scsi_realize(dev, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
 
     /* EEPROM */
@@ -503,8 +501,6 @@
     chksum = 0x1234 - chksum;
     contents[EE_CHKSUM1] = chksum & 0xff;
     contents[EE_CHKSUM2] = chksum >> 8;
-
-    return 0;
 }
 
 static void dc390_class_init(ObjectClass *klass, void *data)
@@ -512,7 +508,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = dc390_scsi_init;
+    k->realize = dc390_scsi_realize;
     k->config_read = dc390_read_config;
     k->config_write = dc390_write_config;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
diff --git a/hw/scsi/esp.c b/hw/scsi/esp.c
index 272d13d..1f2f2d3 100644
--- a/hw/scsi/esp.c
+++ b/hw/scsi/esp.c
@@ -23,9 +23,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/scsi/esp.h"
 #include "trace.h"
+#include "qapi/error.h"
 #include "qemu/log.h"
 
 /*
@@ -80,7 +82,7 @@
     }
 }
 
-static uint32_t get_cmd(ESPState *s, uint8_t *buf)
+static uint32_t get_cmd(ESPState *s, uint8_t *buf, uint8_t buflen)
 {
     uint32_t dmalen;
     int target;
@@ -90,9 +92,15 @@
         dmalen = s->rregs[ESP_TCLO];
         dmalen |= s->rregs[ESP_TCMID] << 8;
         dmalen |= s->rregs[ESP_TCHI] << 16;
+        if (dmalen > buflen) {
+            return 0;
+        }
         s->dma_memory_read(s->dma_opaque, buf, dmalen);
     } else {
         dmalen = s->ti_size;
+        if (dmalen > TI_BUFSZ) {
+            return 0;
+        }
         memcpy(buf, s->ti_buf, dmalen);
         buf[0] = buf[2] >> 5;
     }
@@ -164,7 +172,7 @@
         s->dma_cb = handle_satn;
         return;
     }
-    len = get_cmd(s, buf);
+    len = get_cmd(s, buf, sizeof(buf));
     if (len)
         do_cmd(s, buf);
 }
@@ -178,7 +186,7 @@
         s->dma_cb = handle_s_without_atn;
         return;
     }
-    len = get_cmd(s, buf);
+    len = get_cmd(s, buf, sizeof(buf));
     if (len) {
         do_busid_cmd(s, buf, 0);
     }
@@ -190,7 +198,7 @@
         s->dma_cb = handle_satn_stop;
         return;
     }
-    s->cmdlen = get_cmd(s, s->cmdbuf);
+    s->cmdlen = get_cmd(s, s->cmdbuf, sizeof(s->cmdbuf));
     if (s->cmdlen) {
         trace_esp_handle_satn_stop(s->cmdlen);
         s->do_cmd = 1;
@@ -214,7 +222,7 @@
     } else {
         s->ti_size = 2;
         s->ti_rptr = 0;
-        s->ti_wptr = 0;
+        s->ti_wptr = 2;
         s->rregs[ESP_RFLAGS] = 2;
     }
     esp_raise_irq(s);
@@ -237,15 +245,12 @@
     uint32_t len;
     int to_device;
 
-    to_device = (s->ti_size < 0);
     len = s->dma_left;
     if (s->do_cmd) {
         trace_esp_do_dma(s->cmdlen, len);
+        assert (s->cmdlen <= sizeof(s->cmdbuf) &&
+                len <= sizeof(s->cmdbuf) - s->cmdlen);
         s->dma_memory_read(s->dma_opaque, &s->cmdbuf[s->cmdlen], len);
-        s->ti_size = 0;
-        s->cmdlen = 0;
-        s->do_cmd = 0;
-        do_cmd(s, s->cmdbuf);
         return;
     }
     if (s->async_len == 0) {
@@ -255,6 +260,7 @@
     if (len > s->async_len) {
         len = s->async_len;
     }
+    to_device = (s->ti_size < 0);
     if (to_device) {
         s->dma_memory_read(s->dma_opaque, s->async_buf, len);
     } else {
@@ -310,6 +316,7 @@
 {
     ESPState *s = req->hba_private;
 
+    assert(!s->do_cmd);
     trace_esp_transfer_data(s->dma_left, s->ti_size);
     s->async_len = len;
     s->async_buf = scsi_req_get_buf(req);
@@ -340,7 +347,7 @@
     s->dma_counter = dmalen;
 
     if (s->do_cmd)
-        minlen = (dmalen < 32) ? dmalen : 32;
+        minlen = (dmalen < ESP_CMDBUF_SZ) ? dmalen : ESP_CMDBUF_SZ;
     else if (s->ti_size < 0)
         minlen = (dmalen < -s->ti_size) ? dmalen : -s->ti_size;
     else
@@ -350,13 +357,13 @@
         s->dma_left = minlen;
         s->rregs[ESP_RSTAT] &= ~STAT_TC;
         esp_do_dma(s);
-    } else if (s->do_cmd) {
+    }
+    if (s->do_cmd) {
         trace_esp_handle_ti_cmd(s->cmdlen);
         s->ti_size = 0;
         s->cmdlen = 0;
         s->do_cmd = 0;
         do_cmd(s, s->cmdbuf);
-        return;
     }
 }
 
@@ -395,19 +402,17 @@
     trace_esp_mem_readb(saddr, s->rregs[saddr]);
     switch (saddr) {
     case ESP_FIFO:
-        if (s->ti_size > 0) {
+        if ((s->rregs[ESP_RSTAT] & STAT_PIO_MASK) == 0) {
+            /* Data out.  */
+            qemu_log_mask(LOG_UNIMP, "esp: PIO data read not implemented\n");
+            s->rregs[ESP_FIFO] = 0;
+            esp_raise_irq(s);
+        } else if (s->ti_rptr < s->ti_wptr) {
             s->ti_size--;
-            if ((s->rregs[ESP_RSTAT] & STAT_PIO_MASK) == 0) {
-                /* Data out.  */
-                qemu_log_mask(LOG_UNIMP,
-                              "esp: PIO data read not implemented\n");
-                s->rregs[ESP_FIFO] = 0;
-            } else {
-                s->rregs[ESP_FIFO] = s->ti_buf[s->ti_rptr++];
-            }
+            s->rregs[ESP_FIFO] = s->ti_buf[s->ti_rptr++];
             esp_raise_irq(s);
         }
-        if (s->ti_size == 0) {
+        if (s->ti_rptr == s->ti_wptr) {
             s->ti_rptr = 0;
             s->ti_wptr = 0;
         }
@@ -446,8 +451,12 @@
         break;
     case ESP_FIFO:
         if (s->do_cmd) {
-            s->cmdbuf[s->cmdlen++] = val & 0xff;
-        } else if (s->ti_size == TI_BUFSZ - 1) {
+            if (s->cmdlen < ESP_CMDBUF_SZ) {
+                s->cmdbuf[s->cmdlen++] = val & 0xff;
+            } else {
+                trace_esp_error_fifo_overrun();
+            }
+        } else if (s->ti_wptr == TI_BUFSZ - 1) {
             trace_esp_error_fifo_overrun();
         } else {
             s->ti_size++;
@@ -565,7 +574,7 @@
 
 const VMStateDescription vmstate_esp = {
     .name ="esp",
-    .version_id = 3,
+    .version_id = 4,
     .minimum_version_id = 3,
     .fields = (VMStateField[]) {
         VMSTATE_BUFFER(rregs, ESPState),
@@ -576,7 +585,8 @@
         VMSTATE_BUFFER(ti_buf, ESPState),
         VMSTATE_UINT32(status, ESPState),
         VMSTATE_UINT32(dma, ESPState),
-        VMSTATE_BUFFER(cmdbuf, ESPState),
+        VMSTATE_PARTIAL_BUFFER(cmdbuf, ESPState, 16),
+        VMSTATE_BUFFER_START_MIDDLE_V(cmdbuf, ESPState, 16, 4),
         VMSTATE_UINT32(cmdlen, ESPState),
         VMSTATE_UINT32(do_cmd, ESPState),
         VMSTATE_UINT32(dma_left, ESPState),
diff --git a/hw/scsi/lsi53c895a.c b/hw/scsi/lsi53c895a.c
index c0852ff..dbaf4d4 100644
--- a/hw/scsi/lsi53c895a.c
+++ b/hw/scsi/lsi53c895a.c
@@ -13,13 +13,12 @@
  * as well-behaved operating systems will not try to use them.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/scsi/scsi.h"
 #include "sysemu/dma.h"
-#include "qemu/error-report.h"
 
 //#define DEBUG_LSI
 //#define DEBUG_LSI_REG
@@ -277,6 +276,7 @@
     uint32_t csbc;
     uint32_t scratch[18]; /* SCRATCHA-SCRATCHR */
     uint8_t sbr;
+    uint32_t adder;
 
     /* Script ram is stored as 32-bit words in host byteorder.  */
     uint32_t script_ram[2048];
@@ -782,7 +782,7 @@
     }
 
     assert(s->current == NULL);
-    s->current = g_malloc0(sizeof(lsi_request));
+    s->current = g_new0(lsi_request, 1);
     s->current->tag = s->select_tag;
     s->current->req = scsi_req_new(dev, s->current->tag, s->current_lun, buf,
                                    s->current);
@@ -1390,6 +1390,7 @@
                 switch ((insn >> 27) & 7) {
                 case 0: /* Jump */
                     DPRINTF("Jump to 0x%08x\n", addr);
+                    s->adder = addr;
                     s->dsp = addr;
                     break;
                 case 1: /* Call */
@@ -1514,6 +1515,8 @@
         return 0x7f;
     case 0x08: /* Revision ID */
         return 0x00;
+    case 0x09: /* SOCL */
+        return s->socl;
     case 0xa: /* SSID */
         return s->ssid;
     case 0xb: /* SBCL */
@@ -1578,6 +1581,8 @@
         return s->sbr;
     case 0x3b: /* DCNTL */
         return s->dcntl;
+    /* ADDER Output (Debug of relative jump address) */
+    CASE_GET_REG32(adder, 0x3c)
     case 0x40: /* SIEN0 */
         return s->sien0;
     case 0x41: /* SIEN1 */
@@ -2084,12 +2089,11 @@
     .cancel = lsi_request_cancelled
 };
 
-static int lsi_scsi_init(PCIDevice *dev)
+static void lsi_scsi_realize(PCIDevice *dev, Error **errp)
 {
     LSIState *s = LSI53C895A(dev);
     DeviceState *d = DEVICE(dev);
     uint8_t *pci_conf;
-    Error *err = NULL;
 
     pci_conf = dev->config;
 
@@ -2112,13 +2116,8 @@
 
     scsi_bus_new(&s->bus, sizeof(s->bus), d, &lsi_scsi_info, NULL);
     if (!d->hotplugged) {
-        scsi_bus_legacy_handle_cmdline(&s->bus, &err);
-        if (err != NULL) {
-            error_free(err);
-            return -1;
-        }
+        scsi_bus_legacy_handle_cmdline(&s->bus, errp);
     }
-    return 0;
 }
 
 static void lsi_class_init(ObjectClass *klass, void *data)
@@ -2126,7 +2125,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = lsi_scsi_init;
+    k->realize = lsi_scsi_realize;
     k->vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
     k->device_id = PCI_DEVICE_ID_LSI_53C895A;
     k->class_id = PCI_CLASS_STORAGE_SCSI;
diff --git a/hw/scsi/megasas.c b/hw/scsi/megasas.c
index 604252a..e968302 100644
--- a/hw/scsi/megasas.c
+++ b/hw/scsi/megasas.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "sysemu/dma.h"
@@ -28,7 +29,7 @@
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
 #include "trace.h"
-
+#include "qapi/error.h"
 #include "mfi.h"
 
 #define MEGASAS_VERSION_GEN1 "1.70"
@@ -47,11 +48,7 @@
 
 #define MEGASAS_FLAG_USE_JBOD      0
 #define MEGASAS_MASK_USE_JBOD      (1 << MEGASAS_FLAG_USE_JBOD)
-#define MEGASAS_FLAG_USE_MSI       1
-#define MEGASAS_MASK_USE_MSI       (1 << MEGASAS_FLAG_USE_MSI)
-#define MEGASAS_FLAG_USE_MSIX      2
-#define MEGASAS_MASK_USE_MSIX      (1 << MEGASAS_FLAG_USE_MSIX)
-#define MEGASAS_FLAG_USE_QUEUE64   3
+#define MEGASAS_FLAG_USE_QUEUE64   1
 #define MEGASAS_MASK_USE_QUEUE64   (1 << MEGASAS_FLAG_USE_QUEUE64)
 
 static const char *mfi_frame_desc[] = {
@@ -95,6 +92,8 @@
     int busy;
     int diag;
     int adp_reset;
+    OnOffAuto msi;
+    OnOffAuto msix;
 
     MegasasCmd *event_cmd;
     int event_locale;
@@ -156,14 +155,9 @@
     return s->flags & MEGASAS_MASK_USE_QUEUE64;
 }
 
-static bool megasas_use_msi(MegasasState *s)
-{
-    return s->flags & MEGASAS_MASK_USE_MSI;
-}
-
 static bool megasas_use_msix(MegasasState *s)
 {
-    return s->flags & MEGASAS_MASK_USE_MSIX;
+    return s->msix != ON_OFF_AUTO_OFF;
 }
 
 static bool megasas_is_jbod(MegasasState *s)
@@ -171,26 +165,29 @@
     return s->flags & MEGASAS_MASK_USE_JBOD;
 }
 
-static void megasas_frame_set_cmd_status(unsigned long frame, uint8_t v)
+static void megasas_frame_set_cmd_status(MegasasState *s,
+                                         unsigned long frame, uint8_t v)
 {
-    stb_phys(&address_space_memory,
-             frame + offsetof(struct mfi_frame_header, cmd_status), v);
+    PCIDevice *pci = &s->parent_obj;
+    stb_pci_dma(pci, frame + offsetof(struct mfi_frame_header, cmd_status), v);
 }
 
-static void megasas_frame_set_scsi_status(unsigned long frame, uint8_t v)
+static void megasas_frame_set_scsi_status(MegasasState *s,
+                                          unsigned long frame, uint8_t v)
 {
-    stb_phys(&address_space_memory,
-             frame + offsetof(struct mfi_frame_header, scsi_status), v);
+    PCIDevice *pci = &s->parent_obj;
+    stb_pci_dma(pci, frame + offsetof(struct mfi_frame_header, scsi_status), v);
 }
 
 /*
  * Context is considered opaque, but the HBA firmware is running
  * in little endian mode. So convert it to little endian, too.
  */
-static uint64_t megasas_frame_get_context(unsigned long frame)
+static uint64_t megasas_frame_get_context(MegasasState *s,
+                                          unsigned long frame)
 {
-    return ldq_le_phys(&address_space_memory,
-                       frame + offsetof(struct mfi_frame_header, context));
+    PCIDevice *pci = &s->parent_obj;
+    return ldq_le_pci_dma(pci, frame + offsetof(struct mfi_frame_header, context));
 }
 
 static bool megasas_frame_is_ieee_sgl(MegasasCmd *cmd)
@@ -406,17 +403,14 @@
 static uint64_t megasas_fw_time(void)
 {
     struct tm curtime;
-    uint64_t bcd_time;
 
     qemu_get_timedate(&curtime, 0);
-    bcd_time = ((uint64_t)curtime.tm_sec & 0xff) << 48 |
+    return ((uint64_t)curtime.tm_sec & 0xff) << 48 |
         ((uint64_t)curtime.tm_min & 0xff)  << 40 |
         ((uint64_t)curtime.tm_hour & 0xff) << 32 |
         ((uint64_t)curtime.tm_mday & 0xff) << 24 |
         ((uint64_t)curtime.tm_mon & 0xff)  << 16 |
         ((uint64_t)(curtime.tm_year + 1900) & 0xffff);
-
-    return bcd_time;
 }
 
 /*
@@ -428,7 +422,7 @@
 static uint64_t megasas_get_sata_addr(uint16_t id)
 {
     uint64_t addr = (0x1221ULL << 48);
-    return addr & (id << 24);
+    return addr | ((uint64_t)id << 24);
 }
 
 /*
@@ -523,8 +517,7 @@
     s->busy++;
 
     if (s->consumer_pa) {
-        s->reply_queue_tail = ldl_le_phys(&address_space_memory,
-                                          s->consumer_pa);
+        s->reply_queue_tail = ldl_le_pci_dma(pcid, s->consumer_pa);
     }
     trace_megasas_qf_enqueue(cmd->index, cmd->count, cmd->context,
                              s->reply_queue_head, s->reply_queue_tail, s->busy);
@@ -547,29 +540,24 @@
          */
         if (megasas_use_queue64(s)) {
             queue_offset = s->reply_queue_head * sizeof(uint64_t);
-            stq_le_phys(&address_space_memory,
-                        s->reply_queue_pa + queue_offset, context);
+            stq_le_pci_dma(pci_dev, s->reply_queue_pa + queue_offset, context);
         } else {
             queue_offset = s->reply_queue_head * sizeof(uint32_t);
-            stl_le_phys(&address_space_memory,
-                        s->reply_queue_pa + queue_offset, context);
+            stl_le_pci_dma(pci_dev, s->reply_queue_pa + queue_offset, context);
         }
-        s->reply_queue_tail = ldl_le_phys(&address_space_memory,
-                                          s->consumer_pa);
+        s->reply_queue_tail = ldl_le_pci_dma(pci_dev, s->consumer_pa);
         trace_megasas_qf_complete(context, s->reply_queue_head,
                                   s->reply_queue_tail, s->busy);
     }
 
     if (megasas_intr_enabled(s)) {
         /* Update reply queue pointer */
-        s->reply_queue_tail = ldl_le_phys(&address_space_memory,
-                                          s->consumer_pa);
+        s->reply_queue_tail = ldl_le_pci_dma(pci_dev, s->consumer_pa);
         tail = s->reply_queue_head;
         s->reply_queue_head = megasas_next_index(s, tail, s->fw_cmds);
         trace_megasas_qf_update(s->reply_queue_head, s->reply_queue_tail,
                                 s->busy);
-        stl_le_phys(&address_space_memory,
-                    s->producer_pa, s->reply_queue_head);
+        stl_le_pci_dma(pci_dev, s->producer_pa, s->reply_queue_head);
         /* Notify HBA */
         if (msix_enabled(pci_dev)) {
             trace_megasas_msix_raise(0);
@@ -651,8 +639,10 @@
     pa_lo = le32_to_cpu(initq->pi_addr_lo);
     pa_hi = le32_to_cpu(initq->pi_addr_hi);
     s->producer_pa = ((uint64_t) pa_hi << 32) | pa_lo;
-    s->reply_queue_head = ldl_le_phys(&address_space_memory, s->producer_pa);
-    s->reply_queue_tail = ldl_le_phys(&address_space_memory, s->consumer_pa);
+    s->reply_queue_head = ldl_le_pci_dma(pcid, s->producer_pa);
+    s->reply_queue_head %= MEGASAS_MAX_FRAMES;
+    s->reply_queue_tail = ldl_le_pci_dma(pcid, s->consumer_pa);
+    s->reply_queue_tail %= MEGASAS_MAX_FRAMES;
     flags = le32_to_cpu(initq->flags);
     if (flags & MFI_QUEUE_FLAG_CONTEXT64) {
         s->flags |= MEGASAS_MASK_USE_QUEUE64;
@@ -721,7 +711,7 @@
     BusChild *kid;
     int num_pd_disks = 0;
 
-    memset(&info, 0x0, cmd->iov_size);
+    memset(&info, 0x0, dcmd_size);
     if (cmd->iov_size < dcmd_size) {
         trace_megasas_dcmd_invalid_xfer_len(cmd->index, cmd->iov_size,
                                             dcmd_size);
@@ -747,7 +737,7 @@
     info.device.type = MFI_INFO_DEV_SAS3G;
     info.device.port_count = 8;
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
         uint16_t pd_id;
 
         if (num_pd_disks < 8) {
@@ -760,7 +750,7 @@
 
     memcpy(info.product_name, base_class->product_name, 24);
     snprintf(info.serial_number, 32, "%s", s->hba_serial);
-    snprintf(info.package_version, 0x60, "%s-QEMU", QEMU_VERSION);
+    snprintf(info.package_version, 0x60, "%s-QEMU", qemu_hw_version());
     memcpy(info.image_component[0].name, "APP", 3);
     snprintf(info.image_component[0].version, 10, "%s-QEMU",
              base_class->product_version);
@@ -773,6 +763,7 @@
 
         ptr = memory_region_get_ram_ptr(&pci_dev->rom);
         memcpy(biosver, ptr + 0x41, 31);
+        biosver[31] = 0;
         memcpy(info.image_component[1].name, "BIOS", 4);
         memcpy(info.image_component[1].version, biosver,
                strlen((const char *)biosver));
@@ -807,7 +798,7 @@
                                MFI_INFO_LDOPS_READ_POLICY);
     info.max_strips_per_io = cpu_to_le16(s->fw_sge);
     info.stripe_sz_ops.min = 3;
-    info.stripe_sz_ops.max = ffs(MEGASAS_MAX_SECTORS + 1) - 1;
+    info.stripe_sz_ops.max = ctz32(MEGASAS_MAX_SECTORS + 1);
     info.properties.pred_fail_poll_interval = cpu_to_le16(300);
     info.properties.intr_throttle_cnt = cpu_to_le16(16);
     info.properties.intr_throttle_timeout = cpu_to_le16(50);
@@ -963,7 +954,7 @@
         max_pd_disks = MFI_MAX_SYS_PDS;
     }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
         uint16_t pd_id;
 
         if (num_pd_disks >= max_pd_disks)
@@ -1018,8 +1009,7 @@
     size_t len, resid;
 
     if (!cmd->iov_buf) {
-        cmd->iov_buf = g_malloc(dcmd_size);
-        memset(cmd->iov_buf, 0, dcmd_size);
+        cmd->iov_buf = g_malloc0(dcmd_size);
         info = cmd->iov_buf;
         info->inquiry_data[0] = 0x7f; /* Force PQual 0x3, PType 0x1f */
         info->vpd_page83[0] = 0x7f;
@@ -1140,7 +1130,7 @@
         max_ld_disks = MFI_MAX_LD;
     }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
 
         if (num_ld_disks >= max_ld_disks) {
             break;
@@ -1191,7 +1181,7 @@
         max_ld_disks = MFI_MAX_LD;
     }
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
 
         if (num_ld_disks >= max_ld_disks) {
             break;
@@ -1221,8 +1211,7 @@
     uint64_t ld_size;
 
     if (!cmd->iov_buf) {
-        cmd->iov_buf = g_malloc(dcmd_size);
-        memset(cmd->iov_buf, 0x0, dcmd_size);
+        cmd->iov_buf = g_malloc0(dcmd_size);
         info = cmd->iov_buf;
         megasas_setup_inquiry(cdb, 0x83, sizeof(info->vpd_page83));
         req = scsi_req_new(sdev, cmd->index, lun, cdb, cmd);
@@ -1297,7 +1286,7 @@
 
 static int megasas_dcmd_cfg_read(MegasasState *s, MegasasCmd *cmd)
 {
-    uint8_t data[4096];
+    uint8_t data[4096] = { 0 };
     struct mfi_config_data *info;
     int num_pd_disks = 0, array_offset, ld_offset;
     BusChild *kid;
@@ -1332,7 +1321,7 @@
     ld_offset = array_offset + sizeof(struct mfi_array) * num_pd_disks;
 
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-        SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+        SCSIDevice *sdev = SCSI_DEVICE(kid->child);
         uint16_t sdev_id = ((sdev->id & 0xFF) << 8) | (sdev->lun & 0xFF);
         struct mfi_array *array;
         struct mfi_ld_config *ld;
@@ -1450,7 +1439,7 @@
                                             dcmd_size);
         return MFI_STAT_INVALID_PARAMETER;
     }
-    dma_buf_write((uint8_t *)&info, cmd->iov_size, &cmd->qsg);
+    dma_buf_write((uint8_t *)&info, dcmd_size, &cmd->qsg);
     trace_megasas_dcmd_unsupported(cmd->index, cmd->iov_size);
     return MFI_STAT_OK;
 }
@@ -1953,14 +1942,14 @@
      * Always read 64bit context, top bits will be
      * masked out if required in megasas_enqueue_frame()
      */
-    frame_context = megasas_frame_get_context(frame_addr);
+    frame_context = megasas_frame_get_context(s, frame_addr);
 
     cmd = megasas_enqueue_frame(s, frame_addr, frame_context, frame_count);
     if (!cmd) {
         /* reply queue full */
         trace_megasas_frame_busy(frame_addr);
-        megasas_frame_set_scsi_status(frame_addr, BUSY);
-        megasas_frame_set_cmd_status(frame_addr, MFI_STAT_SCSI_DONE_WITH_ERROR);
+        megasas_frame_set_scsi_status(s, frame_addr, BUSY);
+        megasas_frame_set_cmd_status(s, frame_addr, MFI_STAT_SCSI_DONE_WITH_ERROR);
         megasas_complete_frame(s, frame_context);
         s->event_count++;
         return;
@@ -1992,11 +1981,7 @@
         break;
     }
     if (frame_status != MFI_STAT_INVALID_STATUS) {
-        if (cmd->frame) {
-            cmd->frame->header.cmd_status = frame_status;
-        } else {
-            megasas_frame_set_cmd_status(frame_addr, frame_status);
-        }
+        cmd->frame->header.cmd_status = frame_status;
         megasas_unmap_frame(s, cmd);
         megasas_complete_frame(s, cmd->context);
     }
@@ -2207,8 +2192,15 @@
     return 0;
 }
 
+static void megasas_queue_write(void *opaque, hwaddr addr,
+                               uint64_t val, unsigned size)
+{
+    return;
+}
+
 static const MemoryRegionOps megasas_queue_ops = {
     .read = megasas_queue_read,
+    .write = megasas_queue_write,
     .endianness = DEVICE_LITTLE_ENDIAN,
     .impl = {
         .min_access_size = 8,
@@ -2235,7 +2227,7 @@
          * after the initial reset.
          */
         QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-            SCSIDevice *sdev = DO_UPCAST(SCSIDevice, qdev, kid->child);
+            SCSIDevice *sdev = SCSI_DEVICE(kid->child);
 
             sdev->unit_attention = SENSE_CODE(NO_SENSE);
             scsi_device_unit_attention_reported(sdev);
@@ -2306,9 +2298,7 @@
     if (megasas_use_msix(s)) {
         msix_uninit(d, &s->mmio_io, &s->mmio_io);
     }
-    if (megasas_use_msi(s)) {
-        msi_uninit(d);
-    }
+    msi_uninit(d);
 }
 
 static const struct SCSIBusInfo megasas_scsi_info = {
@@ -2322,7 +2312,7 @@
     .cancel = megasas_command_cancel,
 };
 
-static int megasas_scsi_init(PCIDevice *dev)
+static void megasas_scsi_realize(PCIDevice *dev, Error **errp)
 {
     DeviceState *d = DEVICE(dev);
     MegasasState *s = MEGASAS(dev);
@@ -2330,6 +2320,7 @@
     uint8_t *pci_conf;
     int i, bar_type;
     Error *err = NULL;
+    int ret;
 
     pci_conf = dev->config;
 
@@ -2338,6 +2329,24 @@
     /* Interrupt pin 1 */
     pci_conf[PCI_INTERRUPT_PIN] = 0x01;
 
+    if (s->msi != ON_OFF_AUTO_OFF) {
+        ret = msi_init(dev, 0x50, 1, true, false, &err);
+        /* Any error other than -ENOTSUP(board's MSI support is broken)
+         * is a programming error */
+        assert(!ret || ret == -ENOTSUP);
+        if (ret && s->msi == ON_OFF_AUTO_ON) {
+            /* Can't satisfy user's explicit msi=on request, fail */
+            error_append_hint(&err, "You have to use msi=auto (default) or "
+                    "msi=off with this machine type.\n");
+            error_propagate(errp, err);
+            return;
+        } else if (ret) {
+            /* With msi=auto, we fall back to MSI off silently */
+            s->msi = ON_OFF_AUTO_OFF;
+            error_free(err);
+        }
+    }
+
     memory_region_init_io(&s->mmio_io, OBJECT(s), &megasas_mmio_ops, s,
                           "megasas-mmio", 0x4000);
     memory_region_init_io(&s->port_io, OBJECT(s), &megasas_port_ops, s,
@@ -2345,14 +2354,10 @@
     memory_region_init_io(&s->queue_io, OBJECT(s), &megasas_queue_ops, s,
                           "megasas-queue", 0x40000);
 
-    if (megasas_use_msi(s) &&
-        msi_init(dev, 0x50, 1, true, false)) {
-        s->flags &= ~MEGASAS_MASK_USE_MSI;
-    }
     if (megasas_use_msix(s) &&
         msix_init(dev, 15, &s->mmio_io, b->mmio_bar, 0x2000,
                   &s->mmio_io, b->mmio_bar, 0x3800, 0x68)) {
-        s->flags &= ~MEGASAS_MASK_USE_MSIX;
+        s->msix = ON_OFF_AUTO_OFF;
     }
     if (pci_is_express(dev)) {
         pcie_endpoint_cap_init(dev, 0xa0);
@@ -2409,20 +2414,8 @@
     scsi_bus_new(&s->bus, sizeof(s->bus), DEVICE(dev),
                  &megasas_scsi_info, NULL);
     if (!d->hotplugged) {
-        scsi_bus_legacy_handle_cmdline(&s->bus, &err);
-        if (err != NULL) {
-            error_free(err);
-            return -1;
-        }
+        scsi_bus_legacy_handle_cmdline(&s->bus, errp);
     }
-    return 0;
-}
-
-static void
-megasas_write_config(PCIDevice *pci, uint32_t addr, uint32_t val, int len)
-{
-    pci_default_write_config(pci, addr, val, len);
-    msi_write_config(pci, addr, val, len);
 }
 
 static Property megasas_properties_gen1[] = {
@@ -2432,10 +2425,8 @@
                        MEGASAS_DEFAULT_FRAMES),
     DEFINE_PROP_STRING("hba_serial", MegasasState, hba_serial),
     DEFINE_PROP_UINT64("sas_address", MegasasState, sas_addr, 0),
-    DEFINE_PROP_BIT("use_msi", MegasasState, flags,
-                    MEGASAS_FLAG_USE_MSI, false),
-    DEFINE_PROP_BIT("use_msix", MegasasState, flags,
-                    MEGASAS_FLAG_USE_MSIX, false),
+    DEFINE_PROP_ON_OFF_AUTO("msi", MegasasState, msi, ON_OFF_AUTO_AUTO),
+    DEFINE_PROP_ON_OFF_AUTO("msix", MegasasState, msix, ON_OFF_AUTO_AUTO),
     DEFINE_PROP_BIT("use_jbod", MegasasState, flags,
                     MEGASAS_FLAG_USE_JBOD, false),
     DEFINE_PROP_END_OF_LIST(),
@@ -2448,10 +2439,8 @@
                        MEGASAS_GEN2_DEFAULT_FRAMES),
     DEFINE_PROP_STRING("hba_serial", MegasasState, hba_serial),
     DEFINE_PROP_UINT64("sas_address", MegasasState, sas_addr, 0),
-    DEFINE_PROP_BIT("use_msi", MegasasState, flags,
-                    MEGASAS_FLAG_USE_MSI, true),
-    DEFINE_PROP_BIT("use_msix", MegasasState, flags,
-                    MEGASAS_FLAG_USE_MSIX, true),
+    DEFINE_PROP_ON_OFF_AUTO("msi", MegasasState, msi, ON_OFF_AUTO_AUTO),
+    DEFINE_PROP_ON_OFF_AUTO("msix", MegasasState, msix, ON_OFF_AUTO_AUTO),
     DEFINE_PROP_BIT("use_jbod", MegasasState, flags,
                     MEGASAS_FLAG_USE_JBOD, false),
     DEFINE_PROP_END_OF_LIST(),
@@ -2509,7 +2498,7 @@
     MegasasBaseClass *e = MEGASAS_DEVICE_CLASS(oc);
     const MegasasInfo *info = data;
 
-    pc->init = megasas_scsi_init;
+    pc->realize = megasas_scsi_realize;
     pc->exit = megasas_scsi_uninit;
     pc->vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
     pc->device_id = info->device_id;
@@ -2527,7 +2516,6 @@
     dc->vmsd = info->vmsd;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->desc = info->desc;
-    pc->config_write = megasas_write_config;
 }
 
 static const TypeInfo megasas_info = {
diff --git a/hw/scsi/mfi.h b/hw/scsi/mfi.h
index 29d4177..e67a5c0 100644
--- a/hw/scsi/mfi.h
+++ b/hw/scsi/mfi.h
@@ -30,8 +30,8 @@
  * SUCH DAMAGE.
  */
 
-#ifndef MFI_REG_H
-#define MFI_REG_H
+#ifndef SCSI_MFI_H
+#define SCSI_MFI_H
 
 /*
  * MegaRAID SAS MFI firmware definitions
@@ -1269,4 +1269,4 @@
 #define MFI_SCSI_MAX_CMDS       8
 #define MFI_SCSI_MAX_CDB_LEN   16
 
-#endif /* MFI_REG_H */
+#endif /* SCSI_MFI_H */
diff --git a/hw/scsi/mpi.h b/hw/scsi/mpi.h
new file mode 100644
index 0000000..0568e19
--- /dev/null
+++ b/hw/scsi/mpi.h
@@ -0,0 +1,1153 @@
+/*-
+ * Based on FreeBSD sys/dev/mpt/mpilib headers.
+ *
+ * Copyright (c) 2000-2010, LSI Logic Corporation and its contributors.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    substantially similar to the "NO WARRANTY" disclaimer below
+ *    ("Disclaimer") and any redistribution must be conditioned upon including
+ *    a substantially similar Disclaimer requirement for further binary
+ *    redistribution.
+ * 3. Neither the name of the LSI Logic Corporation nor the names of its
+ *    contributors may be used to endorse or promote products derived from
+ *    this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF THE COPYRIGHT
+ * OWNER OR CONTRIBUTOR IS ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#ifndef MPI_H
+#define MPI_H
+
+enum {
+    MPI_FUNCTION_SCSI_IO_REQUEST                = 0x00,
+    MPI_FUNCTION_SCSI_TASK_MGMT                 = 0x01,
+    MPI_FUNCTION_IOC_INIT                       = 0x02,
+    MPI_FUNCTION_IOC_FACTS                      = 0x03,
+    MPI_FUNCTION_CONFIG                         = 0x04,
+    MPI_FUNCTION_PORT_FACTS                     = 0x05,
+    MPI_FUNCTION_PORT_ENABLE                    = 0x06,
+    MPI_FUNCTION_EVENT_NOTIFICATION             = 0x07,
+    MPI_FUNCTION_EVENT_ACK                      = 0x08,
+    MPI_FUNCTION_FW_DOWNLOAD                    = 0x09,
+    MPI_FUNCTION_TARGET_CMD_BUFFER_POST         = 0x0A,
+    MPI_FUNCTION_TARGET_ASSIST                  = 0x0B,
+    MPI_FUNCTION_TARGET_STATUS_SEND             = 0x0C,
+    MPI_FUNCTION_TARGET_MODE_ABORT              = 0x0D,
+    MPI_FUNCTION_FC_LINK_SRVC_BUF_POST          = 0x0E,
+    MPI_FUNCTION_FC_LINK_SRVC_RSP               = 0x0F,
+    MPI_FUNCTION_FC_EX_LINK_SRVC_SEND           = 0x10,
+    MPI_FUNCTION_FC_ABORT                       = 0x11,
+    MPI_FUNCTION_FW_UPLOAD                      = 0x12,
+    MPI_FUNCTION_FC_COMMON_TRANSPORT_SEND       = 0x13,
+    MPI_FUNCTION_FC_PRIMITIVE_SEND              = 0x14,
+
+    MPI_FUNCTION_RAID_ACTION                    = 0x15,
+    MPI_FUNCTION_RAID_SCSI_IO_PASSTHROUGH       = 0x16,
+
+    MPI_FUNCTION_TOOLBOX                        = 0x17,
+
+    MPI_FUNCTION_SCSI_ENCLOSURE_PROCESSOR       = 0x18,
+
+    MPI_FUNCTION_MAILBOX                        = 0x19,
+
+    MPI_FUNCTION_SMP_PASSTHROUGH                = 0x1A,
+    MPI_FUNCTION_SAS_IO_UNIT_CONTROL            = 0x1B,
+    MPI_FUNCTION_SATA_PASSTHROUGH               = 0x1C,
+
+    MPI_FUNCTION_DIAG_BUFFER_POST               = 0x1D,
+    MPI_FUNCTION_DIAG_RELEASE                   = 0x1E,
+
+    MPI_FUNCTION_SCSI_IO_32                     = 0x1F,
+
+    MPI_FUNCTION_LAN_SEND                       = 0x20,
+    MPI_FUNCTION_LAN_RECEIVE                    = 0x21,
+    MPI_FUNCTION_LAN_RESET                      = 0x22,
+
+    MPI_FUNCTION_TARGET_ASSIST_EXTENDED         = 0x23,
+    MPI_FUNCTION_TARGET_CMD_BUF_BASE_POST       = 0x24,
+    MPI_FUNCTION_TARGET_CMD_BUF_LIST_POST       = 0x25,
+
+    MPI_FUNCTION_INBAND_BUFFER_POST             = 0x28,
+    MPI_FUNCTION_INBAND_SEND                    = 0x29,
+    MPI_FUNCTION_INBAND_RSP                     = 0x2A,
+    MPI_FUNCTION_INBAND_ABORT                   = 0x2B,
+
+    MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET         = 0x40,
+    MPI_FUNCTION_IO_UNIT_RESET                  = 0x41,
+    MPI_FUNCTION_HANDSHAKE                      = 0x42,
+    MPI_FUNCTION_REPLY_FRAME_REMOVAL            = 0x43,
+    MPI_FUNCTION_HOST_PAGEBUF_ACCESS_CONTROL    = 0x44,
+};
+
+/****************************************************************************/
+/*  Registers                                                               */
+/****************************************************************************/
+
+enum {
+    MPI_IOC_STATE_RESET                 = 0x00000000,
+    MPI_IOC_STATE_READY                 = 0x10000000,
+    MPI_IOC_STATE_OPERATIONAL           = 0x20000000,
+    MPI_IOC_STATE_FAULT                 = 0x40000000,
+
+    MPI_DOORBELL_OFFSET                 = 0x00000000,
+    MPI_DOORBELL_ACTIVE                 = 0x08000000, /* DoorbellUsed */
+    MPI_DOORBELL_WHO_INIT_MASK          = 0x07000000,
+    MPI_DOORBELL_WHO_INIT_SHIFT         = 24,
+    MPI_DOORBELL_FUNCTION_MASK          = 0xFF000000,
+    MPI_DOORBELL_FUNCTION_SHIFT         = 24,
+    MPI_DOORBELL_ADD_DWORDS_MASK        = 0x00FF0000,
+    MPI_DOORBELL_ADD_DWORDS_SHIFT       = 16,
+    MPI_DOORBELL_DATA_MASK              = 0x0000FFFF,
+    MPI_DOORBELL_FUNCTION_SPECIFIC_MASK = 0x0000FFFF,
+
+    MPI_DB_HPBAC_VALUE_MASK             = 0x0000F000,
+    MPI_DB_HPBAC_ENABLE_ACCESS          = 0x01,
+    MPI_DB_HPBAC_DISABLE_ACCESS         = 0x02,
+    MPI_DB_HPBAC_FREE_BUFFER            = 0x03,
+
+    MPI_WRITE_SEQUENCE_OFFSET           = 0x00000004,
+    MPI_WRSEQ_KEY_VALUE_MASK            = 0x0000000F,
+    MPI_WRSEQ_1ST_KEY_VALUE             = 0x04,
+    MPI_WRSEQ_2ND_KEY_VALUE             = 0x0B,
+    MPI_WRSEQ_3RD_KEY_VALUE             = 0x02,
+    MPI_WRSEQ_4TH_KEY_VALUE             = 0x07,
+    MPI_WRSEQ_5TH_KEY_VALUE             = 0x0D,
+
+    MPI_DIAGNOSTIC_OFFSET               = 0x00000008,
+    MPI_DIAG_CLEAR_FLASH_BAD_SIG        = 0x00000400,
+    MPI_DIAG_PREVENT_IOC_BOOT           = 0x00000200,
+    MPI_DIAG_DRWE                       = 0x00000080,
+    MPI_DIAG_FLASH_BAD_SIG              = 0x00000040,
+    MPI_DIAG_RESET_HISTORY              = 0x00000020,
+    MPI_DIAG_RW_ENABLE                  = 0x00000010,
+    MPI_DIAG_RESET_ADAPTER              = 0x00000004,
+    MPI_DIAG_DISABLE_ARM                = 0x00000002,
+    MPI_DIAG_MEM_ENABLE                 = 0x00000001,
+
+    MPI_TEST_BASE_ADDRESS_OFFSET        = 0x0000000C,
+
+    MPI_DIAG_RW_DATA_OFFSET             = 0x00000010,
+
+    MPI_DIAG_RW_ADDRESS_OFFSET          = 0x00000014,
+
+    MPI_HOST_INTERRUPT_STATUS_OFFSET    = 0x00000030,
+    MPI_HIS_IOP_DOORBELL_STATUS         = 0x80000000,
+    MPI_HIS_REPLY_MESSAGE_INTERRUPT     = 0x00000008,
+    MPI_HIS_DOORBELL_INTERRUPT          = 0x00000001,
+
+    MPI_HOST_INTERRUPT_MASK_OFFSET      = 0x00000034,
+    MPI_HIM_RIM                         = 0x00000008,
+    MPI_HIM_DIM                         = 0x00000001,
+
+    MPI_REQUEST_QUEUE_OFFSET            = 0x00000040,
+    MPI_REQUEST_POST_FIFO_OFFSET        = 0x00000040,
+
+    MPI_REPLY_QUEUE_OFFSET              = 0x00000044,
+    MPI_REPLY_POST_FIFO_OFFSET          = 0x00000044,
+    MPI_REPLY_FREE_FIFO_OFFSET          = 0x00000044,
+
+    MPI_HI_PRI_REQUEST_QUEUE_OFFSET     = 0x00000048,
+};
+
+#define MPI_ADDRESS_REPLY_A_BIT          0x80000000
+
+/****************************************************************************/
+/*  Scatter/gather elements                                                 */
+/****************************************************************************/
+
+typedef struct MPISGEntry {
+    uint32_t                FlagsLength;
+    union
+    {
+        uint32_t            Address32;
+        uint64_t            Address64;
+    } u;
+} QEMU_PACKED MPISGEntry;
+
+/* Flags field bit definitions */
+
+enum {
+    MPI_SGE_FLAGS_LAST_ELEMENT              = 0x80000000,
+    MPI_SGE_FLAGS_END_OF_BUFFER             = 0x40000000,
+    MPI_SGE_FLAGS_ELEMENT_TYPE_MASK         = 0x30000000,
+    MPI_SGE_FLAGS_LOCAL_ADDRESS             = 0x08000000,
+    MPI_SGE_FLAGS_DIRECTION                 = 0x04000000,
+    MPI_SGE_FLAGS_64_BIT_ADDRESSING         = 0x02000000,
+    MPI_SGE_FLAGS_END_OF_LIST               = 0x01000000,
+
+    MPI_SGE_LENGTH_MASK                     = 0x00FFFFFF,
+    MPI_SGE_CHAIN_LENGTH_MASK               = 0x0000FFFF,
+
+    MPI_SGE_FLAGS_TRANSACTION_ELEMENT       = 0x00000000,
+    MPI_SGE_FLAGS_SIMPLE_ELEMENT            = 0x10000000,
+    MPI_SGE_FLAGS_CHAIN_ELEMENT             = 0x30000000,
+
+    /* Direction */
+
+    MPI_SGE_FLAGS_IOC_TO_HOST               = 0x00000000,
+    MPI_SGE_FLAGS_HOST_TO_IOC               = 0x04000000,
+
+    MPI_SGE_CHAIN_OFFSET_MASK               = 0x00FF0000,
+};
+
+#define MPI_SGE_CHAIN_OFFSET_SHIFT 16
+
+/****************************************************************************/
+/* Standard message request header for all request messages                 */
+/****************************************************************************/
+
+typedef struct MPIRequestHeader {
+    uint8_t                 Reserved[2];      /* function specific */
+    uint8_t                 ChainOffset;
+    uint8_t                 Function;
+    uint8_t                 Reserved1[3];     /* function specific */
+    uint8_t                 MsgFlags;
+    uint32_t                MsgContext;
+} QEMU_PACKED MPIRequestHeader;
+
+
+typedef struct MPIDefaultReply {
+    uint8_t                 Reserved[2];      /* function specific */
+    uint8_t                 MsgLength;
+    uint8_t                 Function;
+    uint8_t                 Reserved1[3];     /* function specific */
+    uint8_t                 MsgFlags;
+    uint32_t                MsgContext;
+    uint8_t                 Reserved2[2];     /* function specific */
+    uint16_t                IOCStatus;
+    uint32_t                IOCLogInfo;
+} QEMU_PACKED MPIDefaultReply;
+
+/* MsgFlags definition for all replies */
+
+#define MPI_MSGFLAGS_CONTINUATION_REPLY         (0x80)
+
+enum {
+
+    /************************************************************************/
+    /*  Common IOCStatus values for all replies                             */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_SUCCESS                   = 0x0000,
+    MPI_IOCSTATUS_INVALID_FUNCTION          = 0x0001,
+    MPI_IOCSTATUS_BUSY                      = 0x0002,
+    MPI_IOCSTATUS_INVALID_SGL               = 0x0003,
+    MPI_IOCSTATUS_INTERNAL_ERROR            = 0x0004,
+    MPI_IOCSTATUS_RESERVED                  = 0x0005,
+    MPI_IOCSTATUS_INSUFFICIENT_RESOURCES    = 0x0006,
+    MPI_IOCSTATUS_INVALID_FIELD             = 0x0007,
+    MPI_IOCSTATUS_INVALID_STATE             = 0x0008,
+    MPI_IOCSTATUS_OP_STATE_NOT_SUPPORTED    = 0x0009,
+
+    /************************************************************************/
+    /*  Config IOCStatus values                                             */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_CONFIG_INVALID_ACTION     = 0x0020,
+    MPI_IOCSTATUS_CONFIG_INVALID_TYPE       = 0x0021,
+    MPI_IOCSTATUS_CONFIG_INVALID_PAGE       = 0x0022,
+    MPI_IOCSTATUS_CONFIG_INVALID_DATA       = 0x0023,
+    MPI_IOCSTATUS_CONFIG_NO_DEFAULTS        = 0x0024,
+    MPI_IOCSTATUS_CONFIG_CANT_COMMIT        = 0x0025,
+
+    /************************************************************************/
+    /*  SCSIIO Reply = SPI & FCP, initiator values                           */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_SCSI_RECOVERED_ERROR      = 0x0040,
+    MPI_IOCSTATUS_SCSI_INVALID_BUS          = 0x0041,
+    MPI_IOCSTATUS_SCSI_INVALID_TARGETID     = 0x0042,
+    MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE     = 0x0043,
+    MPI_IOCSTATUS_SCSI_DATA_OVERRUN         = 0x0044,
+    MPI_IOCSTATUS_SCSI_DATA_UNDERRUN        = 0x0045,
+    MPI_IOCSTATUS_SCSI_IO_DATA_ERROR        = 0x0046,
+    MPI_IOCSTATUS_SCSI_PROTOCOL_ERROR       = 0x0047,
+    MPI_IOCSTATUS_SCSI_TASK_TERMINATED      = 0x0048,
+    MPI_IOCSTATUS_SCSI_RESIDUAL_MISMATCH    = 0x0049,
+    MPI_IOCSTATUS_SCSI_TASK_MGMT_FAILED     = 0x004A,
+    MPI_IOCSTATUS_SCSI_IOC_TERMINATED       = 0x004B,
+    MPI_IOCSTATUS_SCSI_EXT_TERMINATED       = 0x004C,
+
+    /************************************************************************/
+    /*  For use by SCSI Initiator and SCSI Target end-to-end data protection*/
+    /************************************************************************/
+
+    MPI_IOCSTATUS_EEDP_GUARD_ERROR          = 0x004D,
+    MPI_IOCSTATUS_EEDP_REF_TAG_ERROR        = 0x004E,
+    MPI_IOCSTATUS_EEDP_APP_TAG_ERROR        = 0x004F,
+
+    /************************************************************************/
+    /*  SCSI Target values                                                  */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_TARGET_PRIORITY_IO         = 0x0060,
+    MPI_IOCSTATUS_TARGET_INVALID_PORT        = 0x0061,
+    MPI_IOCSTATUS_TARGET_INVALID_IO_INDEX    = 0x0062,
+    MPI_IOCSTATUS_TARGET_ABORTED             = 0x0063,
+    MPI_IOCSTATUS_TARGET_NO_CONN_RETRYABLE   = 0x0064,
+    MPI_IOCSTATUS_TARGET_NO_CONNECTION       = 0x0065,
+    MPI_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH = 0x006A,
+    MPI_IOCSTATUS_TARGET_STS_DATA_NOT_SENT   = 0x006B,
+    MPI_IOCSTATUS_TARGET_DATA_OFFSET_ERROR   = 0x006D,
+    MPI_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA = 0x006E,
+    MPI_IOCSTATUS_TARGET_IU_TOO_SHORT        = 0x006F,
+    MPI_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT     = 0x0070,
+    MPI_IOCSTATUS_TARGET_NAK_RECEIVED        = 0x0071,
+
+    /************************************************************************/
+    /*  Fibre Channel Direct Access values                                  */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_FC_ABORTED                = 0x0066,
+    MPI_IOCSTATUS_FC_RX_ID_INVALID          = 0x0067,
+    MPI_IOCSTATUS_FC_DID_INVALID            = 0x0068,
+    MPI_IOCSTATUS_FC_NODE_LOGGED_OUT        = 0x0069,
+    MPI_IOCSTATUS_FC_EXCHANGE_CANCELED      = 0x006C,
+
+    /************************************************************************/
+    /*  LAN values                                                          */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_LAN_DEVICE_NOT_FOUND      = 0x0080,
+    MPI_IOCSTATUS_LAN_DEVICE_FAILURE        = 0x0081,
+    MPI_IOCSTATUS_LAN_TRANSMIT_ERROR        = 0x0082,
+    MPI_IOCSTATUS_LAN_TRANSMIT_ABORTED      = 0x0083,
+    MPI_IOCSTATUS_LAN_RECEIVE_ERROR         = 0x0084,
+    MPI_IOCSTATUS_LAN_RECEIVE_ABORTED       = 0x0085,
+    MPI_IOCSTATUS_LAN_PARTIAL_PACKET        = 0x0086,
+    MPI_IOCSTATUS_LAN_CANCELED              = 0x0087,
+
+    /************************************************************************/
+    /*  Serial Attached SCSI values                                         */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_SAS_SMP_REQUEST_FAILED    = 0x0090,
+    MPI_IOCSTATUS_SAS_SMP_DATA_OVERRUN      = 0x0091,
+
+    /************************************************************************/
+    /*  Inband values                                                       */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_INBAND_ABORTED            = 0x0098,
+    MPI_IOCSTATUS_INBAND_NO_CONNECTION      = 0x0099,
+
+    /************************************************************************/
+    /*  Diagnostic Tools values                                             */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_DIAGNOSTIC_RELEASED       = 0x00A0,
+
+    /************************************************************************/
+    /*  IOCStatus flag to indicate that log info is available               */
+    /************************************************************************/
+
+    MPI_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE   = 0x8000,
+    MPI_IOCSTATUS_MASK                      = 0x7FFF,
+
+    /************************************************************************/
+    /*  LogInfo Types                                                       */
+    /************************************************************************/
+
+    MPI_IOCLOGINFO_TYPE_MASK                = 0xF0000000,
+    MPI_IOCLOGINFO_TYPE_SHIFT               = 28,
+    MPI_IOCLOGINFO_TYPE_NONE                = 0x0,
+    MPI_IOCLOGINFO_TYPE_SCSI                = 0x1,
+    MPI_IOCLOGINFO_TYPE_FC                  = 0x2,
+    MPI_IOCLOGINFO_TYPE_SAS                 = 0x3,
+    MPI_IOCLOGINFO_TYPE_ISCSI               = 0x4,
+    MPI_IOCLOGINFO_LOG_DATA_MASK            = 0x0FFFFFFF,
+};
+
+/****************************************************************************/
+/*  SCSI IO messages and associated structures                              */
+/****************************************************************************/
+
+typedef struct MPIMsgSCSIIORequest {
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 ChainOffset;        /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 CDBLength;          /* 04h */
+    uint8_t                 SenseBufferLength;  /* 05h */
+    uint8_t                 Reserved;           /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 LUN[8];             /* 0Ch */
+    uint32_t                Control;            /* 14h */
+    uint8_t                 CDB[16];            /* 18h */
+    uint32_t                DataLength;         /* 28h */
+    uint32_t                SenseBufferLowAddr; /* 2Ch */
+} QEMU_PACKED MPIMsgSCSIIORequest;
+
+/* SCSI IO MsgFlags bits */
+
+#define MPI_SCSIIO_MSGFLGS_SENSE_WIDTH              (0x01)
+#define MPI_SCSIIO_MSGFLGS_SENSE_WIDTH_32           (0x00)
+#define MPI_SCSIIO_MSGFLGS_SENSE_WIDTH_64           (0x01)
+
+#define MPI_SCSIIO_MSGFLGS_SENSE_LOCATION           (0x02)
+#define MPI_SCSIIO_MSGFLGS_SENSE_LOC_HOST           (0x00)
+#define MPI_SCSIIO_MSGFLGS_SENSE_LOC_IOC            (0x02)
+
+#define MPI_SCSIIO_MSGFLGS_CMD_DETERMINES_DATA_DIR  (0x04)
+
+/* SCSI IO LUN fields */
+
+#define MPI_SCSIIO_LUN_FIRST_LEVEL_ADDRESSING   (0x0000FFFF)
+#define MPI_SCSIIO_LUN_SECOND_LEVEL_ADDRESSING  (0xFFFF0000)
+#define MPI_SCSIIO_LUN_THIRD_LEVEL_ADDRESSING   (0x0000FFFF)
+#define MPI_SCSIIO_LUN_FOURTH_LEVEL_ADDRESSING  (0xFFFF0000)
+#define MPI_SCSIIO_LUN_LEVEL_1_WORD             (0xFF00)
+#define MPI_SCSIIO_LUN_LEVEL_1_DWORD            (0x0000FF00)
+
+/* SCSI IO Control bits */
+
+#define MPI_SCSIIO_CONTROL_DATADIRECTION_MASK   (0x03000000)
+#define MPI_SCSIIO_CONTROL_NODATATRANSFER       (0x00000000)
+#define MPI_SCSIIO_CONTROL_WRITE                (0x01000000)
+#define MPI_SCSIIO_CONTROL_READ                 (0x02000000)
+
+#define MPI_SCSIIO_CONTROL_ADDCDBLEN_MASK       (0x3C000000)
+#define MPI_SCSIIO_CONTROL_ADDCDBLEN_SHIFT      (26)
+
+#define MPI_SCSIIO_CONTROL_TASKATTRIBUTE_MASK   (0x00000700)
+#define MPI_SCSIIO_CONTROL_SIMPLEQ              (0x00000000)
+#define MPI_SCSIIO_CONTROL_HEADOFQ              (0x00000100)
+#define MPI_SCSIIO_CONTROL_ORDEREDQ             (0x00000200)
+#define MPI_SCSIIO_CONTROL_ACAQ                 (0x00000400)
+#define MPI_SCSIIO_CONTROL_UNTAGGED             (0x00000500)
+#define MPI_SCSIIO_CONTROL_NO_DISCONNECT        (0x00000700)
+
+#define MPI_SCSIIO_CONTROL_TASKMANAGE_MASK      (0x00FF0000)
+#define MPI_SCSIIO_CONTROL_OBSOLETE             (0x00800000)
+#define MPI_SCSIIO_CONTROL_CLEAR_ACA_RSV        (0x00400000)
+#define MPI_SCSIIO_CONTROL_TARGET_RESET         (0x00200000)
+#define MPI_SCSIIO_CONTROL_LUN_RESET_RSV        (0x00100000)
+#define MPI_SCSIIO_CONTROL_RESERVED             (0x00080000)
+#define MPI_SCSIIO_CONTROL_CLR_TASK_SET_RSV     (0x00040000)
+#define MPI_SCSIIO_CONTROL_ABORT_TASK_SET       (0x00020000)
+#define MPI_SCSIIO_CONTROL_RESERVED2            (0x00010000)
+
+/* SCSI IO reply structure */
+typedef struct MPIMsgSCSIIOReply
+{
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 MsgLength;          /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 CDBLength;          /* 04h */
+    uint8_t                 SenseBufferLength;  /* 05h */
+    uint8_t                 Reserved;           /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 SCSIStatus;         /* 0Ch */
+    uint8_t                 SCSIState;          /* 0Dh */
+    uint16_t                IOCStatus;          /* 0Eh */
+    uint32_t                IOCLogInfo;         /* 10h */
+    uint32_t                TransferCount;      /* 14h */
+    uint32_t                SenseCount;         /* 18h */
+    uint32_t                ResponseInfo;       /* 1Ch */
+    uint16_t                TaskTag;            /* 20h */
+    uint16_t                Reserved1;          /* 22h */
+} QEMU_PACKED MPIMsgSCSIIOReply;
+
+/* SCSI IO Reply SCSIStatus values (SAM-2 status codes) */
+
+#define MPI_SCSI_STATUS_SUCCESS                 (0x00)
+#define MPI_SCSI_STATUS_CHECK_CONDITION         (0x02)
+#define MPI_SCSI_STATUS_CONDITION_MET           (0x04)
+#define MPI_SCSI_STATUS_BUSY                    (0x08)
+#define MPI_SCSI_STATUS_INTERMEDIATE            (0x10)
+#define MPI_SCSI_STATUS_INTERMEDIATE_CONDMET    (0x14)
+#define MPI_SCSI_STATUS_RESERVATION_CONFLICT    (0x18)
+#define MPI_SCSI_STATUS_COMMAND_TERMINATED      (0x22)
+#define MPI_SCSI_STATUS_TASK_SET_FULL           (0x28)
+#define MPI_SCSI_STATUS_ACA_ACTIVE              (0x30)
+
+#define MPI_SCSI_STATUS_FCPEXT_DEVICE_LOGGED_OUT    (0x80)
+#define MPI_SCSI_STATUS_FCPEXT_NO_LINK              (0x81)
+#define MPI_SCSI_STATUS_FCPEXT_UNASSIGNED           (0x82)
+
+
+/* SCSI IO Reply SCSIState values */
+
+#define MPI_SCSI_STATE_AUTOSENSE_VALID          (0x01)
+#define MPI_SCSI_STATE_AUTOSENSE_FAILED         (0x02)
+#define MPI_SCSI_STATE_NO_SCSI_STATUS           (0x04)
+#define MPI_SCSI_STATE_TERMINATED               (0x08)
+#define MPI_SCSI_STATE_RESPONSE_INFO_VALID      (0x10)
+#define MPI_SCSI_STATE_QUEUE_TAG_REJECTED       (0x20)
+
+/* SCSI IO Reply ResponseInfo values */
+/* (FCP-1 RSP_CODE values and SPI-3 Packetized Failure codes) */
+
+#define MPI_SCSI_RSP_INFO_FUNCTION_COMPLETE     (0x00000000)
+#define MPI_SCSI_RSP_INFO_FCP_BURST_LEN_ERROR   (0x01000000)
+#define MPI_SCSI_RSP_INFO_CMND_FIELDS_INVALID   (0x02000000)
+#define MPI_SCSI_RSP_INFO_FCP_DATA_RO_ERROR     (0x03000000)
+#define MPI_SCSI_RSP_INFO_TASK_MGMT_UNSUPPORTED (0x04000000)
+#define MPI_SCSI_RSP_INFO_TASK_MGMT_FAILED      (0x05000000)
+#define MPI_SCSI_RSP_INFO_SPI_LQ_INVALID_TYPE   (0x06000000)
+
+#define MPI_SCSI_TASKTAG_UNKNOWN                (0xFFFF)
+
+
+/****************************************************************************/
+/*  SCSI Task Management messages                                           */
+/****************************************************************************/
+
+typedef struct MPIMsgSCSITaskMgmt {
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 ChainOffset;        /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 Reserved;           /* 04h */
+    uint8_t                 TaskType;           /* 05h */
+    uint8_t                 Reserved1;          /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 LUN[8];             /* 0Ch */
+    uint32_t                Reserved2[7];       /* 14h */
+    uint32_t                TaskMsgContext;     /* 30h */
+} QEMU_PACKED MPIMsgSCSITaskMgmt;
+
+enum {
+    /* TaskType values */
+
+    MPI_SCSITASKMGMT_TASKTYPE_ABORT_TASK            = 0x01,
+    MPI_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET         = 0x02,
+    MPI_SCSITASKMGMT_TASKTYPE_TARGET_RESET          = 0x03,
+    MPI_SCSITASKMGMT_TASKTYPE_RESET_BUS             = 0x04,
+    MPI_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET    = 0x05,
+    MPI_SCSITASKMGMT_TASKTYPE_CLEAR_TASK_SET        = 0x06,
+    MPI_SCSITASKMGMT_TASKTYPE_QUERY_TASK            = 0x07,
+    MPI_SCSITASKMGMT_TASKTYPE_CLR_ACA               = 0x08,
+
+    /* MsgFlags bits */
+
+    MPI_SCSITASKMGMT_MSGFLAGS_DO_NOT_SEND_TASK_IU   = 0x01,
+
+    MPI_SCSITASKMGMT_MSGFLAGS_TARGET_RESET_OPTION   = 0x00,
+    MPI_SCSITASKMGMT_MSGFLAGS_LIP_RESET_OPTION      = 0x02,
+    MPI_SCSITASKMGMT_MSGFLAGS_LIPRESET_RESET_OPTION = 0x04,
+
+    MPI_SCSITASKMGMT_MSGFLAGS_SOFT_RESET_OPTION     = 0x08,
+};
+
+/* SCSI Task Management Reply */
+typedef struct MPIMsgSCSITaskMgmtReply {
+    uint8_t                 TargetID;           /* 00h */
+    uint8_t                 Bus;                /* 01h */
+    uint8_t                 MsgLength;          /* 02h */
+    uint8_t                 Function;           /* 03h */
+    uint8_t                 ResponseCode;       /* 04h */
+    uint8_t                 TaskType;           /* 05h */
+    uint8_t                 Reserved1;          /* 06h */
+    uint8_t                 MsgFlags;           /* 07h */
+    uint32_t                MsgContext;         /* 08h */
+    uint8_t                 Reserved2[2];       /* 0Ch */
+    uint16_t                IOCStatus;          /* 0Eh */
+    uint32_t                IOCLogInfo;         /* 10h */
+    uint32_t                TerminationCount;   /* 14h */
+} QEMU_PACKED MPIMsgSCSITaskMgmtReply;
+
+/* ResponseCode values */
+enum {
+    MPI_SCSITASKMGMT_RSP_TM_COMPLETE                = 0x00,
+    MPI_SCSITASKMGMT_RSP_INVALID_FRAME              = 0x02,
+    MPI_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED           = 0x04,
+    MPI_SCSITASKMGMT_RSP_TM_FAILED                  = 0x05,
+    MPI_SCSITASKMGMT_RSP_TM_SUCCEEDED               = 0x08,
+    MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN             = 0x09,
+    MPI_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC           = 0x80,
+};
+
+/****************************************************************************/
+/*  IOCInit message                                                         */
+/****************************************************************************/
+
+typedef struct MPIMsgIOCInit {
+    uint8_t                 WhoInit;                    /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Flags;                      /* 04h */
+    uint8_t                 MaxDevices;                 /* 05h */
+    uint8_t                 MaxBuses;                   /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                ReplyFrameSize;             /* 0Ch */
+    uint8_t                 Reserved1[2];               /* 0Eh */
+    uint32_t                HostMfaHighAddr;            /* 10h */
+    uint32_t                SenseBufferHighAddr;        /* 14h */
+    uint32_t                ReplyFifoHostSignalingAddr; /* 18h */
+    MPISGEntry              HostPageBufferSGE;          /* 1Ch */
+    uint16_t                MsgVersion;                 /* 28h */
+    uint16_t                HeaderVersion;              /* 2Ah */
+} QEMU_PACKED MPIMsgIOCInit;
+
+enum {
+    /* WhoInit values */
+
+    MPI_WHOINIT_NO_ONE                              = 0x00,
+    MPI_WHOINIT_SYSTEM_BIOS                         = 0x01,
+    MPI_WHOINIT_ROM_BIOS                            = 0x02,
+    MPI_WHOINIT_PCI_PEER                            = 0x03,
+    MPI_WHOINIT_HOST_DRIVER                         = 0x04,
+    MPI_WHOINIT_MANUFACTURER                        = 0x05,
+
+    /* Flags values */
+
+    MPI_IOCINIT_FLAGS_HOST_PAGE_BUFFER_PERSISTENT   = 0x04,
+    MPI_IOCINIT_FLAGS_REPLY_FIFO_HOST_SIGNAL        = 0x02,
+    MPI_IOCINIT_FLAGS_DISCARD_FW_IMAGE              = 0x01,
+
+    /* MsgVersion */
+
+    MPI_IOCINIT_MSGVERSION_MAJOR_MASK               = 0xFF00,
+    MPI_IOCINIT_MSGVERSION_MAJOR_SHIFT              = 8,
+    MPI_IOCINIT_MSGVERSION_MINOR_MASK               = 0x00FF,
+    MPI_IOCINIT_MSGVERSION_MINOR_SHIFT              = 0,
+
+    /* HeaderVersion */
+
+    MPI_IOCINIT_HEADERVERSION_UNIT_MASK             = 0xFF00,
+    MPI_IOCINIT_HEADERVERSION_UNIT_SHIFT            = 8,
+    MPI_IOCINIT_HEADERVERSION_DEV_MASK              = 0x00FF,
+    MPI_IOCINIT_HEADERVERSION_DEV_SHIFT             = 0,
+};
+
+typedef struct MPIMsgIOCInitReply {
+    uint8_t                 WhoInit;                    /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Flags;                      /* 04h */
+    uint8_t                 MaxDevices;                 /* 05h */
+    uint8_t                 MaxBuses;                   /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                Reserved2;                  /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+} QEMU_PACKED MPIMsgIOCInitReply;
+
+
+
+/****************************************************************************/
+/*  IOC Facts message                                                       */
+/****************************************************************************/
+
+typedef struct MPIMsgIOCFacts {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 ChainOffset;                /* 01h */
+    uint8_t                 Function;                   /* 02h */
+    uint8_t                 Reserved1[3];               /* 03h */
+    uint8_t                 MsgFlags;                   /* 04h */
+    uint32_t                MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgIOCFacts;
+
+/* IOC Facts Reply */
+typedef struct MPIMsgIOCFactsReply {
+    uint16_t                MsgVersion;                 /* 00h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint16_t                HeaderVersion;              /* 04h */
+    uint8_t                 IOCNumber;                  /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                IOCExceptions;              /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+    uint8_t                 MaxChainDepth;              /* 14h */
+    uint8_t                 WhoInit;                    /* 15h */
+    uint8_t                 BlockSize;                  /* 16h */
+    uint8_t                 Flags;                      /* 17h */
+    uint16_t                ReplyQueueDepth;            /* 18h */
+    uint16_t                RequestFrameSize;           /* 1Ah */
+    uint16_t                Reserved_0101_FWVersion;    /* 1Ch */ /* obsolete 16-bit FWVersion */
+    uint16_t                ProductID;                  /* 1Eh */
+    uint32_t                CurrentHostMfaHighAddr;     /* 20h */
+    uint16_t                GlobalCredits;              /* 24h */
+    uint8_t                 NumberOfPorts;              /* 26h */
+    uint8_t                 EventState;                 /* 27h */
+    uint32_t                CurrentSenseBufferHighAddr; /* 28h */
+    uint16_t                CurReplyFrameSize;          /* 2Ch */
+    uint8_t                 MaxDevices;                 /* 2Eh */
+    uint8_t                 MaxBuses;                   /* 2Fh */
+    uint32_t                FWImageSize;                /* 30h */
+    uint32_t                IOCCapabilities;            /* 34h */
+    uint8_t                 FWVersionDev;               /* 38h */
+    uint8_t                 FWVersionUnit;              /* 39h */
+    uint8_t                 FWVersionMinor;             /* 3ah */
+    uint8_t                 FWVersionMajor;             /* 3bh */
+    uint16_t                HighPriorityQueueDepth;     /* 3Ch */
+    uint16_t                Reserved2;                  /* 3Eh */
+    MPISGEntry              HostPageBufferSGE;          /* 40h */
+    uint32_t                ReplyFifoHostSignalingAddr; /* 4Ch */
+} QEMU_PACKED MPIMsgIOCFactsReply;
+
+enum {
+    MPI_IOCFACTS_MSGVERSION_MAJOR_MASK              = 0xFF00,
+    MPI_IOCFACTS_MSGVERSION_MAJOR_SHIFT             = 8,
+    MPI_IOCFACTS_MSGVERSION_MINOR_MASK              = 0x00FF,
+    MPI_IOCFACTS_MSGVERSION_MINOR_SHIFT             = 0,
+
+    MPI_IOCFACTS_HDRVERSION_UNIT_MASK               = 0xFF00,
+    MPI_IOCFACTS_HDRVERSION_UNIT_SHIFT              = 8,
+    MPI_IOCFACTS_HDRVERSION_DEV_MASK                = 0x00FF,
+    MPI_IOCFACTS_HDRVERSION_DEV_SHIFT               = 0,
+
+    MPI_IOCFACTS_EXCEPT_CONFIG_CHECKSUM_FAIL        = 0x0001,
+    MPI_IOCFACTS_EXCEPT_RAID_CONFIG_INVALID         = 0x0002,
+    MPI_IOCFACTS_EXCEPT_FW_CHECKSUM_FAIL            = 0x0004,
+    MPI_IOCFACTS_EXCEPT_PERSISTENT_TABLE_FULL       = 0x0008,
+    MPI_IOCFACTS_EXCEPT_METADATA_UNSUPPORTED        = 0x0010,
+
+    MPI_IOCFACTS_FLAGS_FW_DOWNLOAD_BOOT             = 0x01,
+    MPI_IOCFACTS_FLAGS_REPLY_FIFO_HOST_SIGNAL       = 0x02,
+    MPI_IOCFACTS_FLAGS_HOST_PAGE_BUFFER_PERSISTENT  = 0x04,
+
+    MPI_IOCFACTS_EVENTSTATE_DISABLED                = 0x00,
+    MPI_IOCFACTS_EVENTSTATE_ENABLED                 = 0x01,
+
+    MPI_IOCFACTS_CAPABILITY_HIGH_PRI_Q              = 0x00000001,
+    MPI_IOCFACTS_CAPABILITY_REPLY_HOST_SIGNAL       = 0x00000002,
+    MPI_IOCFACTS_CAPABILITY_QUEUE_FULL_HANDLING     = 0x00000004,
+    MPI_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER       = 0x00000008,
+    MPI_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER         = 0x00000010,
+    MPI_IOCFACTS_CAPABILITY_EXTENDED_BUFFER         = 0x00000020,
+    MPI_IOCFACTS_CAPABILITY_EEDP                    = 0x00000040,
+    MPI_IOCFACTS_CAPABILITY_BIDIRECTIONAL           = 0x00000080,
+    MPI_IOCFACTS_CAPABILITY_MULTICAST               = 0x00000100,
+    MPI_IOCFACTS_CAPABILITY_SCSIIO32                = 0x00000200,
+    MPI_IOCFACTS_CAPABILITY_NO_SCSIIO16             = 0x00000400,
+    MPI_IOCFACTS_CAPABILITY_TLR                     = 0x00000800,
+};
+
+/****************************************************************************/
+/*  Port Facts message and Reply                                            */
+/****************************************************************************/
+
+typedef struct MPIMsgPortFacts {
+     uint8_t                Reserved[2];                /* 00h */
+     uint8_t                ChainOffset;                /* 02h */
+     uint8_t                Function;                   /* 03h */
+     uint8_t                Reserved1[2];               /* 04h */
+     uint8_t                PortNumber;                 /* 06h */
+     uint8_t                MsgFlags;                   /* 07h */
+     uint32_t               MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgPortFacts;
+
+typedef struct MPIMsgPortFactsReply {
+     uint16_t               Reserved;                   /* 00h */
+     uint8_t                MsgLength;                  /* 02h */
+     uint8_t                Function;                   /* 03h */
+     uint16_t               Reserved1;                  /* 04h */
+     uint8_t                PortNumber;                 /* 06h */
+     uint8_t                MsgFlags;                   /* 07h */
+     uint32_t               MsgContext;                 /* 08h */
+     uint16_t               Reserved2;                  /* 0Ch */
+     uint16_t               IOCStatus;                  /* 0Eh */
+     uint32_t               IOCLogInfo;                 /* 10h */
+     uint8_t                Reserved3;                  /* 14h */
+     uint8_t                PortType;                   /* 15h */
+     uint16_t               MaxDevices;                 /* 16h */
+     uint16_t               PortSCSIID;                 /* 18h */
+     uint16_t               ProtocolFlags;              /* 1Ah */
+     uint16_t               MaxPostedCmdBuffers;        /* 1Ch */
+     uint16_t               MaxPersistentIDs;           /* 1Eh */
+     uint16_t               MaxLanBuckets;              /* 20h */
+     uint8_t                MaxInitiators;              /* 22h */
+     uint8_t                Reserved4;                  /* 23h */
+     uint32_t               Reserved5;                  /* 24h */
+} QEMU_PACKED MPIMsgPortFactsReply;
+
+
+enum {
+    /* PortTypes values */
+    MPI_PORTFACTS_PORTTYPE_INACTIVE         = 0x00,
+    MPI_PORTFACTS_PORTTYPE_SCSI             = 0x01,
+    MPI_PORTFACTS_PORTTYPE_FC               = 0x10,
+    MPI_PORTFACTS_PORTTYPE_ISCSI            = 0x20,
+    MPI_PORTFACTS_PORTTYPE_SAS              = 0x30,
+
+    /* ProtocolFlags values */
+    MPI_PORTFACTS_PROTOCOL_LOGBUSADDR       = 0x01,
+    MPI_PORTFACTS_PROTOCOL_LAN              = 0x02,
+    MPI_PORTFACTS_PROTOCOL_TARGET           = 0x04,
+    MPI_PORTFACTS_PROTOCOL_INITIATOR        = 0x08,
+};
+
+
+/****************************************************************************/
+/*  Port Enable Message                                                     */
+/****************************************************************************/
+
+typedef struct MPIMsgPortEnable {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[2];               /* 04h */
+    uint8_t                 PortNumber;                 /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgPortEnable;
+
+typedef struct MPIMsgPortEnableReply {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[2];               /* 04h */
+    uint8_t                 PortNumber;                 /* 05h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                Reserved2;                  /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+} QEMU_PACKED MPIMsgPortEnableReply;
+
+/****************************************************************************/
+/*  Event Notification messages                                             */
+/****************************************************************************/
+
+typedef struct MPIMsgEventNotify {
+    uint8_t                 Switch;                     /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[3];               /* 04h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+} QEMU_PACKED MPIMsgEventNotify;
+
+/* Event Notification Reply */
+
+typedef struct MPIMsgEventNotifyReply {
+     uint16_t               EventDataLength;            /* 00h */
+     uint8_t                MsgLength;                  /* 02h */
+     uint8_t                Function;                   /* 03h */
+     uint8_t                Reserved1[2];               /* 04h */
+     uint8_t                AckRequired;                /* 06h */
+     uint8_t                MsgFlags;                   /* 07h */
+     uint32_t               MsgContext;                 /* 08h */
+     uint8_t                Reserved2[2];               /* 0Ch */
+     uint16_t               IOCStatus;                  /* 0Eh */
+     uint32_t               IOCLogInfo;                 /* 10h */
+     uint32_t               Event;                      /* 14h */
+     uint32_t               EventContext;               /* 18h */
+     uint32_t               Data[1];                    /* 1Ch */
+} QEMU_PACKED MPIMsgEventNotifyReply;
+
+/* Event Acknowledge */
+
+typedef struct MPIMsgEventAck {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[3];               /* 04h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint32_t                Event;                      /* 0Ch */
+    uint32_t                EventContext;               /* 10h */
+} QEMU_PACKED MPIMsgEventAck;
+
+typedef struct MPIMsgEventAckReply {
+    uint8_t                 Reserved[2];                /* 00h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint8_t                 Reserved1[3];               /* 04h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint16_t                Reserved2;                  /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+} QEMU_PACKED MPIMsgEventAckReply;
+
+enum {
+    /* Switch */
+
+    MPI_EVENT_NOTIFICATION_SWITCH_OFF   = 0x00,
+    MPI_EVENT_NOTIFICATION_SWITCH_ON    = 0x01,
+
+    /* Event */
+
+    MPI_EVENT_NONE                          = 0x00000000,
+    MPI_EVENT_LOG_DATA                      = 0x00000001,
+    MPI_EVENT_STATE_CHANGE                  = 0x00000002,
+    MPI_EVENT_UNIT_ATTENTION                = 0x00000003,
+    MPI_EVENT_IOC_BUS_RESET                 = 0x00000004,
+    MPI_EVENT_EXT_BUS_RESET                 = 0x00000005,
+    MPI_EVENT_RESCAN                        = 0x00000006,
+    MPI_EVENT_LINK_STATUS_CHANGE            = 0x00000007,
+    MPI_EVENT_LOOP_STATE_CHANGE             = 0x00000008,
+    MPI_EVENT_LOGOUT                        = 0x00000009,
+    MPI_EVENT_EVENT_CHANGE                  = 0x0000000A,
+    MPI_EVENT_INTEGRATED_RAID               = 0x0000000B,
+    MPI_EVENT_SCSI_DEVICE_STATUS_CHANGE     = 0x0000000C,
+    MPI_EVENT_ON_BUS_TIMER_EXPIRED          = 0x0000000D,
+    MPI_EVENT_QUEUE_FULL                    = 0x0000000E,
+    MPI_EVENT_SAS_DEVICE_STATUS_CHANGE      = 0x0000000F,
+    MPI_EVENT_SAS_SES                       = 0x00000010,
+    MPI_EVENT_PERSISTENT_TABLE_FULL         = 0x00000011,
+    MPI_EVENT_SAS_PHY_LINK_STATUS           = 0x00000012,
+    MPI_EVENT_SAS_DISCOVERY_ERROR           = 0x00000013,
+    MPI_EVENT_IR_RESYNC_UPDATE              = 0x00000014,
+    MPI_EVENT_IR2                           = 0x00000015,
+    MPI_EVENT_SAS_DISCOVERY                 = 0x00000016,
+    MPI_EVENT_SAS_BROADCAST_PRIMITIVE       = 0x00000017,
+    MPI_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE = 0x00000018,
+    MPI_EVENT_SAS_INIT_TABLE_OVERFLOW       = 0x00000019,
+    MPI_EVENT_SAS_SMP_ERROR                 = 0x0000001A,
+    MPI_EVENT_SAS_EXPANDER_STATUS_CHANGE    = 0x0000001B,
+    MPI_EVENT_LOG_ENTRY_ADDED               = 0x00000021,
+
+    /* AckRequired field values */
+
+    MPI_EVENT_NOTIFICATION_ACK_NOT_REQUIRED = 0x00,
+    MPI_EVENT_NOTIFICATION_ACK_REQUIRED     = 0x01,
+};
+
+/****************************************************************************
+*   Config Request Message
+****************************************************************************/
+
+typedef struct MPIMsgConfig {
+    uint8_t                 Action;                     /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 ChainOffset;                /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint16_t                ExtPageLength;              /* 04h */
+    uint8_t                 ExtPageType;                /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint8_t                 Reserved2[8];               /* 0Ch */
+    uint8_t                 PageVersion;                /* 14h */
+    uint8_t                 PageLength;                 /* 15h */
+    uint8_t                 PageNumber;                 /* 16h */
+    uint8_t                 PageType;                   /* 17h */
+    uint32_t                PageAddress;                /* 18h */
+    MPISGEntry              PageBufferSGE;              /* 1Ch */
+} QEMU_PACKED MPIMsgConfig;
+
+/* Action field values */
+
+enum {
+    MPI_CONFIG_ACTION_PAGE_HEADER               = 0x00,
+    MPI_CONFIG_ACTION_PAGE_READ_CURRENT         = 0x01,
+    MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT        = 0x02,
+    MPI_CONFIG_ACTION_PAGE_DEFAULT              = 0x03,
+    MPI_CONFIG_ACTION_PAGE_WRITE_NVRAM          = 0x04,
+    MPI_CONFIG_ACTION_PAGE_READ_DEFAULT         = 0x05,
+    MPI_CONFIG_ACTION_PAGE_READ_NVRAM           = 0x06,
+};
+
+
+/* Config Reply Message */
+typedef struct MPIMsgConfigReply {
+    uint8_t                 Action;                     /* 00h */
+    uint8_t                 Reserved;                   /* 01h */
+    uint8_t                 MsgLength;                  /* 02h */
+    uint8_t                 Function;                   /* 03h */
+    uint16_t                ExtPageLength;              /* 04h */
+    uint8_t                 ExtPageType;                /* 06h */
+    uint8_t                 MsgFlags;                   /* 07h */
+    uint32_t                MsgContext;                 /* 08h */
+    uint8_t                 Reserved2[2];               /* 0Ch */
+    uint16_t                IOCStatus;                  /* 0Eh */
+    uint32_t                IOCLogInfo;                 /* 10h */
+    uint8_t                 PageVersion;                /* 14h */
+    uint8_t                 PageLength;                 /* 15h */
+    uint8_t                 PageNumber;                 /* 16h */
+    uint8_t                 PageType;                   /* 17h */
+} QEMU_PACKED MPIMsgConfigReply;
+
+enum {
+    /* PageAddress field values */
+    MPI_CONFIG_PAGEATTR_READ_ONLY               = 0x00,
+    MPI_CONFIG_PAGEATTR_CHANGEABLE              = 0x10,
+    MPI_CONFIG_PAGEATTR_PERSISTENT              = 0x20,
+    MPI_CONFIG_PAGEATTR_RO_PERSISTENT           = 0x30,
+    MPI_CONFIG_PAGEATTR_MASK                    = 0xF0,
+
+    MPI_CONFIG_PAGETYPE_IO_UNIT                 = 0x00,
+    MPI_CONFIG_PAGETYPE_IOC                     = 0x01,
+    MPI_CONFIG_PAGETYPE_BIOS                    = 0x02,
+    MPI_CONFIG_PAGETYPE_SCSI_PORT               = 0x03,
+    MPI_CONFIG_PAGETYPE_SCSI_DEVICE             = 0x04,
+    MPI_CONFIG_PAGETYPE_FC_PORT                 = 0x05,
+    MPI_CONFIG_PAGETYPE_FC_DEVICE               = 0x06,
+    MPI_CONFIG_PAGETYPE_LAN                     = 0x07,
+    MPI_CONFIG_PAGETYPE_RAID_VOLUME             = 0x08,
+    MPI_CONFIG_PAGETYPE_MANUFACTURING           = 0x09,
+    MPI_CONFIG_PAGETYPE_RAID_PHYSDISK           = 0x0A,
+    MPI_CONFIG_PAGETYPE_INBAND                  = 0x0B,
+    MPI_CONFIG_PAGETYPE_EXTENDED                = 0x0F,
+    MPI_CONFIG_PAGETYPE_MASK                    = 0x0F,
+
+    MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT          = 0x10,
+    MPI_CONFIG_EXTPAGETYPE_SAS_EXPANDER         = 0x11,
+    MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE           = 0x12,
+    MPI_CONFIG_EXTPAGETYPE_SAS_PHY              = 0x13,
+    MPI_CONFIG_EXTPAGETYPE_LOG                  = 0x14,
+    MPI_CONFIG_EXTPAGETYPE_ENCLOSURE            = 0x15,
+
+    MPI_SCSI_PORT_PGAD_PORT_MASK                = 0x000000FF,
+
+    MPI_SCSI_DEVICE_FORM_MASK                   = 0xF0000000,
+    MPI_SCSI_DEVICE_FORM_BUS_TID                = 0x00000000,
+    MPI_SCSI_DEVICE_TARGET_ID_MASK              = 0x000000FF,
+    MPI_SCSI_DEVICE_TARGET_ID_SHIFT             = 0,
+    MPI_SCSI_DEVICE_BUS_MASK                    = 0x0000FF00,
+    MPI_SCSI_DEVICE_BUS_SHIFT                   = 8,
+    MPI_SCSI_DEVICE_FORM_TARGET_MODE            = 0x10000000,
+    MPI_SCSI_DEVICE_TM_RESPOND_ID_MASK          = 0x000000FF,
+    MPI_SCSI_DEVICE_TM_RESPOND_ID_SHIFT         = 0,
+    MPI_SCSI_DEVICE_TM_BUS_MASK                 = 0x0000FF00,
+    MPI_SCSI_DEVICE_TM_BUS_SHIFT                = 8,
+    MPI_SCSI_DEVICE_TM_INIT_ID_MASK             = 0x00FF0000,
+    MPI_SCSI_DEVICE_TM_INIT_ID_SHIFT            = 16,
+
+    MPI_FC_PORT_PGAD_PORT_MASK                  = 0xF0000000,
+    MPI_FC_PORT_PGAD_PORT_SHIFT                 = 28,
+    MPI_FC_PORT_PGAD_FORM_MASK                  = 0x0F000000,
+    MPI_FC_PORT_PGAD_FORM_INDEX                 = 0x01000000,
+    MPI_FC_PORT_PGAD_INDEX_MASK                 = 0x0000FFFF,
+    MPI_FC_PORT_PGAD_INDEX_SHIFT                = 0,
+
+    MPI_FC_DEVICE_PGAD_PORT_MASK                = 0xF0000000,
+    MPI_FC_DEVICE_PGAD_PORT_SHIFT               = 28,
+    MPI_FC_DEVICE_PGAD_FORM_MASK                = 0x0F000000,
+    MPI_FC_DEVICE_PGAD_FORM_NEXT_DID            = 0x00000000,
+    MPI_FC_DEVICE_PGAD_ND_PORT_MASK             = 0xF0000000,
+    MPI_FC_DEVICE_PGAD_ND_PORT_SHIFT            = 28,
+    MPI_FC_DEVICE_PGAD_ND_DID_MASK              = 0x00FFFFFF,
+    MPI_FC_DEVICE_PGAD_ND_DID_SHIFT             = 0,
+    MPI_FC_DEVICE_PGAD_FORM_BUS_TID             = 0x01000000,
+    MPI_FC_DEVICE_PGAD_BT_BUS_MASK              = 0x0000FF00,
+    MPI_FC_DEVICE_PGAD_BT_BUS_SHIFT             = 8,
+    MPI_FC_DEVICE_PGAD_BT_TID_MASK              = 0x000000FF,
+    MPI_FC_DEVICE_PGAD_BT_TID_SHIFT             = 0,
+
+    MPI_PHYSDISK_PGAD_PHYSDISKNUM_MASK          = 0x000000FF,
+    MPI_PHYSDISK_PGAD_PHYSDISKNUM_SHIFT         = 0,
+
+    MPI_SAS_EXPAND_PGAD_FORM_MASK             = 0xF0000000,
+    MPI_SAS_EXPAND_PGAD_FORM_SHIFT            = 28,
+    MPI_SAS_EXPAND_PGAD_FORM_GET_NEXT_HANDLE  = 0x00000000,
+    MPI_SAS_EXPAND_PGAD_FORM_HANDLE_PHY_NUM   = 0x00000001,
+    MPI_SAS_EXPAND_PGAD_FORM_HANDLE           = 0x00000002,
+    MPI_SAS_EXPAND_PGAD_GNH_MASK_HANDLE       = 0x0000FFFF,
+    MPI_SAS_EXPAND_PGAD_GNH_SHIFT_HANDLE      = 0,
+    MPI_SAS_EXPAND_PGAD_HPN_MASK_PHY          = 0x00FF0000,
+    MPI_SAS_EXPAND_PGAD_HPN_SHIFT_PHY         = 16,
+    MPI_SAS_EXPAND_PGAD_HPN_MASK_HANDLE       = 0x0000FFFF,
+    MPI_SAS_EXPAND_PGAD_HPN_SHIFT_HANDLE      = 0,
+    MPI_SAS_EXPAND_PGAD_H_MASK_HANDLE         = 0x0000FFFF,
+    MPI_SAS_EXPAND_PGAD_H_SHIFT_HANDLE        = 0,
+
+    MPI_SAS_DEVICE_PGAD_FORM_MASK               = 0xF0000000,
+    MPI_SAS_DEVICE_PGAD_FORM_SHIFT              = 28,
+    MPI_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE    = 0x00000000,
+    MPI_SAS_DEVICE_PGAD_FORM_BUS_TARGET_ID      = 0x00000001,
+    MPI_SAS_DEVICE_PGAD_FORM_HANDLE             = 0x00000002,
+    MPI_SAS_DEVICE_PGAD_GNH_HANDLE_MASK         = 0x0000FFFF,
+    MPI_SAS_DEVICE_PGAD_GNH_HANDLE_SHIFT        = 0,
+    MPI_SAS_DEVICE_PGAD_BT_BUS_MASK             = 0x0000FF00,
+    MPI_SAS_DEVICE_PGAD_BT_BUS_SHIFT            = 8,
+    MPI_SAS_DEVICE_PGAD_BT_TID_MASK             = 0x000000FF,
+    MPI_SAS_DEVICE_PGAD_BT_TID_SHIFT            = 0,
+    MPI_SAS_DEVICE_PGAD_H_HANDLE_MASK           = 0x0000FFFF,
+    MPI_SAS_DEVICE_PGAD_H_HANDLE_SHIFT          = 0,
+
+    MPI_SAS_PHY_PGAD_FORM_MASK                  = 0xF0000000,
+    MPI_SAS_PHY_PGAD_FORM_SHIFT                 = 28,
+    MPI_SAS_PHY_PGAD_FORM_PHY_NUMBER            = 0x0,
+    MPI_SAS_PHY_PGAD_FORM_PHY_TBL_INDEX         = 0x1,
+    MPI_SAS_PHY_PGAD_PHY_NUMBER_MASK            = 0x000000FF,
+    MPI_SAS_PHY_PGAD_PHY_NUMBER_SHIFT           = 0,
+    MPI_SAS_PHY_PGAD_PHY_TBL_INDEX_MASK         = 0x0000FFFF,
+    MPI_SAS_PHY_PGAD_PHY_TBL_INDEX_SHIFT        = 0,
+
+    MPI_SAS_ENCLOS_PGAD_FORM_MASK               = 0xF0000000,
+    MPI_SAS_ENCLOS_PGAD_FORM_SHIFT              = 28,
+    MPI_SAS_ENCLOS_PGAD_FORM_GET_NEXT_HANDLE    = 0x00000000,
+    MPI_SAS_ENCLOS_PGAD_FORM_HANDLE             = 0x00000001,
+    MPI_SAS_ENCLOS_PGAD_GNH_HANDLE_MASK         = 0x0000FFFF,
+    MPI_SAS_ENCLOS_PGAD_GNH_HANDLE_SHIFT        = 0,
+    MPI_SAS_ENCLOS_PGAD_H_HANDLE_MASK           = 0x0000FFFF,
+    MPI_SAS_ENCLOS_PGAD_H_HANDLE_SHIFT          = 0,
+};
+
+/* Too many structs and definitions... see mptconfig.c for the few
+ * that are used.
+ */
+
+/****************************************************************************/
+/*  Firmware Upload message and associated structures                       */
+/****************************************************************************/
+
+enum {
+    /* defines for using the ProductId field */
+    MPI_FW_HEADER_PID_TYPE_MASK                     = 0xF000,
+    MPI_FW_HEADER_PID_TYPE_SCSI                     = 0x0000,
+    MPI_FW_HEADER_PID_TYPE_FC                       = 0x1000,
+    MPI_FW_HEADER_PID_TYPE_SAS                      = 0x2000,
+
+    MPI_FW_HEADER_PID_PROD_MASK                     = 0x0F00,
+    MPI_FW_HEADER_PID_PROD_INITIATOR_SCSI           = 0x0100,
+    MPI_FW_HEADER_PID_PROD_TARGET_INITIATOR_SCSI    = 0x0200,
+    MPI_FW_HEADER_PID_PROD_TARGET_SCSI              = 0x0300,
+    MPI_FW_HEADER_PID_PROD_IM_SCSI                  = 0x0400,
+    MPI_FW_HEADER_PID_PROD_IS_SCSI                  = 0x0500,
+    MPI_FW_HEADER_PID_PROD_CTX_SCSI                 = 0x0600,
+    MPI_FW_HEADER_PID_PROD_IR_SCSI                  = 0x0700,
+
+    MPI_FW_HEADER_PID_FAMILY_MASK                   = 0x00FF,
+
+    /* SCSI */
+    MPI_FW_HEADER_PID_FAMILY_1030A0_SCSI            = 0x0001,
+    MPI_FW_HEADER_PID_FAMILY_1030B0_SCSI            = 0x0002,
+    MPI_FW_HEADER_PID_FAMILY_1030B1_SCSI            = 0x0003,
+    MPI_FW_HEADER_PID_FAMILY_1030C0_SCSI            = 0x0004,
+    MPI_FW_HEADER_PID_FAMILY_1020A0_SCSI            = 0x0005,
+    MPI_FW_HEADER_PID_FAMILY_1020B0_SCSI            = 0x0006,
+    MPI_FW_HEADER_PID_FAMILY_1020B1_SCSI            = 0x0007,
+    MPI_FW_HEADER_PID_FAMILY_1020C0_SCSI            = 0x0008,
+    MPI_FW_HEADER_PID_FAMILY_1035A0_SCSI            = 0x0009,
+    MPI_FW_HEADER_PID_FAMILY_1035B0_SCSI            = 0x000A,
+    MPI_FW_HEADER_PID_FAMILY_1030TA0_SCSI           = 0x000B,
+    MPI_FW_HEADER_PID_FAMILY_1020TA0_SCSI           = 0x000C,
+
+    /* Fibre Channel */
+    MPI_FW_HEADER_PID_FAMILY_909_FC                 = 0x0000,
+    MPI_FW_HEADER_PID_FAMILY_919_FC                 = 0x0001, /* 919 and 929     */
+    MPI_FW_HEADER_PID_FAMILY_919X_FC                = 0x0002, /* 919X and 929X   */
+    MPI_FW_HEADER_PID_FAMILY_919XL_FC               = 0x0003, /* 919XL and 929XL */
+    MPI_FW_HEADER_PID_FAMILY_939X_FC                = 0x0004, /* 939X and 949X   */
+    MPI_FW_HEADER_PID_FAMILY_959_FC                 = 0x0005,
+    MPI_FW_HEADER_PID_FAMILY_949E_FC                = 0x0006,
+
+    /* SAS */
+    MPI_FW_HEADER_PID_FAMILY_1064_SAS               = 0x0001,
+    MPI_FW_HEADER_PID_FAMILY_1068_SAS               = 0x0002,
+    MPI_FW_HEADER_PID_FAMILY_1078_SAS               = 0x0003,
+    MPI_FW_HEADER_PID_FAMILY_106xE_SAS              = 0x0004, /* 1068E, 1066E, and 1064E */
+};
+
+#endif
diff --git a/hw/scsi/mptconfig.c b/hw/scsi/mptconfig.c
new file mode 100644
index 0000000..7071854
--- /dev/null
+++ b/hw/scsi/mptconfig.c
@@ -0,0 +1,905 @@
+/*
+ * QEMU LSI SAS1068 Host Bus Adapter emulation - configuration pages
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ */
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "hw/scsi/scsi.h"
+
+#include "mptsas.h"
+#include "mpi.h"
+#include "trace.h"
+
+/* Generic functions for marshaling and unmarshaling.  */
+
+#define repl1(x) x
+#define repl2(x) x x
+#define repl3(x) x x x
+#define repl4(x) x x x x
+#define repl5(x) x x x x x
+#define repl6(x) x x x x x x
+#define repl7(x) x x x x x x x
+#define repl8(x) x x x x x x x x
+
+#define repl(n, x) glue(repl, n)(x)
+
+typedef union PackValue {
+    uint64_t ll;
+    char *str;
+} PackValue;
+
+static size_t vfill(uint8_t *data, size_t size, const char *fmt, va_list ap)
+{
+    size_t ofs;
+    PackValue val;
+    const char *p;
+
+    ofs = 0;
+    p = fmt;
+    while (*p) {
+        memset(&val, 0, sizeof(val));
+        switch (*p) {
+        case '*':
+            p++;
+            break;
+        case 'b':
+        case 'w':
+        case 'l':
+            val.ll = va_arg(ap, int);
+            break;
+        case 'q':
+            val.ll = va_arg(ap, int64_t);
+            break;
+        case 's':
+            val.str = va_arg(ap, void *);
+            break;
+        }
+        switch (*p++) {
+        case 'b':
+            if (data) {
+                stb_p(data + ofs, val.ll);
+            }
+            ofs++;
+            break;
+        case 'w':
+            if (data) {
+                stw_le_p(data + ofs, val.ll);
+            }
+            ofs += 2;
+            break;
+        case 'l':
+            if (data) {
+                stl_le_p(data + ofs, val.ll);
+            }
+            ofs += 4;
+            break;
+        case 'q':
+            if (data) {
+                stq_le_p(data + ofs, val.ll);
+            }
+            ofs += 8;
+            break;
+        case 's':
+            {
+                int cnt = atoi(p);
+                if (data) {
+                    if (val.str) {
+                        strncpy((void *)data + ofs, val.str, cnt);
+                    } else {
+                        memset((void *)data + ofs, 0, cnt);
+                    }
+                }
+                ofs += cnt;
+                break;
+            }
+        }
+    }
+
+    return ofs;
+}
+
+static size_t vpack(uint8_t **p_data, const char *fmt, va_list ap1)
+{
+    size_t size = 0;
+    uint8_t *data = NULL;
+
+    if (p_data) {
+        va_list ap2;
+
+        va_copy(ap2, ap1);
+        size = vfill(NULL, 0, fmt, ap2);
+        *p_data = data = g_malloc(size);
+        va_end(ap2);
+    }
+    return vfill(data, size, fmt, ap1);
+}
+
+static size_t fill(uint8_t *data, size_t size, const char *fmt, ...)
+{
+    va_list ap;
+    size_t ret;
+
+    va_start(ap, fmt);
+    ret = vfill(data, size, fmt, ap);
+    va_end(ap);
+
+    return ret;
+}
+
+/* Functions to build the page header and fill in the length, always used
+ * through the macros.
+ */
+
+#define MPTSAS_CONFIG_PACK(number, type, version, fmt, ...)                  \
+    mptsas_config_pack(data, "b*bbb" fmt, version, number, type,             \
+                       ## __VA_ARGS__)
+
+static size_t mptsas_config_pack(uint8_t **data, const char *fmt, ...)
+{
+    va_list ap;
+    size_t ret;
+
+    va_start(ap, fmt);
+    ret = vpack(data, fmt, ap);
+    va_end(ap);
+
+    if (data) {
+        assert(ret < 256 && (ret % 4) == 0);
+        stb_p(*data + 1, ret / 4);
+    }
+    return ret;
+}
+
+#define MPTSAS_CONFIG_PACK_EXT(number, type, version, fmt, ...)              \
+    mptsas_config_pack_ext(data, "b*bbb*wb*b" fmt, version, number,          \
+                           MPI_CONFIG_PAGETYPE_EXTENDED, type, ## __VA_ARGS__)
+
+static size_t mptsas_config_pack_ext(uint8_t **data, const char *fmt, ...)
+{
+    va_list ap;
+    size_t ret;
+
+    va_start(ap, fmt);
+    ret = vpack(data, fmt, ap);
+    va_end(ap);
+
+    if (data) {
+        assert(ret < 65536 && (ret % 4) == 0);
+        stw_le_p(*data + 4, ret / 4);
+    }
+    return ret;
+}
+
+/* Manufacturing pages */
+
+static
+size_t mptsas_config_manufacturing_0(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(0, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "s16s8s16s16s16",
+                              "QEMU MPT Fusion",
+                              "2.5",
+                              "QEMU MPT Fusion",
+                              "QEMU",
+                              "0000111122223333");
+}
+
+static
+size_t mptsas_config_manufacturing_1(MPTSASState *s, uint8_t **data, int address)
+{
+    /* VPD - all zeros */
+    return MPTSAS_CONFIG_PACK(1, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "s256");
+}
+
+static
+size_t mptsas_config_manufacturing_2(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDeviceClass *pcic = PCI_DEVICE_GET_CLASS(s);
+    return MPTSAS_CONFIG_PACK(2, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "wb*b*l",
+                              pcic->device_id, pcic->revision);
+}
+
+static
+size_t mptsas_config_manufacturing_3(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDeviceClass *pcic = PCI_DEVICE_GET_CLASS(s);
+    return MPTSAS_CONFIG_PACK(3, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "wb*b*l",
+                              pcic->device_id, pcic->revision);
+}
+
+static
+size_t mptsas_config_manufacturing_4(MPTSASState *s, uint8_t **data, int address)
+{
+    /* All zeros */
+    return MPTSAS_CONFIG_PACK(4, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x05,
+                              "*l*b*b*b*b*b*b*w*s56*l*l*l*l*l*l"
+                              "*b*b*w*b*b*w*l*l");
+}
+
+static
+size_t mptsas_config_manufacturing_5(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(5, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x02,
+                              "q*b*b*w*l*l", s->sas_addr);
+}
+
+static
+size_t mptsas_config_manufacturing_6(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(6, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+static
+size_t mptsas_config_manufacturing_7(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(7, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l*l*l*s16*b*b*w", MPTSAS_NUM_PORTS);
+}
+
+static
+size_t mptsas_config_manufacturing_8(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(8, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+static
+size_t mptsas_config_manufacturing_9(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(9, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+static
+size_t mptsas_config_manufacturing_10(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(10, MPI_CONFIG_PAGETYPE_MANUFACTURING, 0x00,
+                              "*l");
+}
+
+/* I/O unit pages */
+
+static
+size_t mptsas_config_io_unit_0(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDevice *pci = PCI_DEVICE(s);
+    uint64_t unique_value = 0x53504D554D4551LL;  /* "QEMUMPTx" */
+
+    unique_value |= (uint64_t)pci->devfn << 56;
+    return MPTSAS_CONFIG_PACK(0, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x00,
+                              "q", unique_value);
+}
+
+static
+size_t mptsas_config_io_unit_1(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(1, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x02, "l",
+                              0x41 /* single function, RAID disabled */ );
+}
+
+static
+size_t mptsas_config_io_unit_2(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDevice *pci = PCI_DEVICE(s);
+    uint8_t devfn = pci->devfn;
+    return MPTSAS_CONFIG_PACK(2, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x02,
+                              "llbbw*b*b*w*b*b*w*b*b*w*l",
+                              0, 0x100, 0 /* pci bus? */, devfn, 0);
+}
+
+static
+size_t mptsas_config_io_unit_3(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(3, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x01,
+                              "*b*b*w*l");
+}
+
+static
+size_t mptsas_config_io_unit_4(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(4, MPI_CONFIG_PAGETYPE_IO_UNIT, 0x00, "*l*l*q");
+}
+
+/* I/O controller pages */
+
+static
+size_t mptsas_config_ioc_0(MPTSASState *s, uint8_t **data, int address)
+{
+    PCIDeviceClass *pcic = PCI_DEVICE_GET_CLASS(s);
+
+    return MPTSAS_CONFIG_PACK(0, MPI_CONFIG_PAGETYPE_IOC, 0x01,
+                              "*l*lwwb*b*b*blww",
+                              pcic->vendor_id, pcic->device_id, pcic->revision,
+                              pcic->subsystem_vendor_id,
+                              pcic->subsystem_id);
+}
+
+static
+size_t mptsas_config_ioc_1(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(1, MPI_CONFIG_PAGETYPE_IOC, 0x03,
+                              "*l*l*b*b*b*b");
+}
+
+static
+size_t mptsas_config_ioc_2(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(2, MPI_CONFIG_PAGETYPE_IOC, 0x04,
+                              "*l*b*b*b*b");
+}
+
+static
+size_t mptsas_config_ioc_3(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(3, MPI_CONFIG_PAGETYPE_IOC, 0x00,
+                              "*b*b*w");
+}
+
+static
+size_t mptsas_config_ioc_4(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(4, MPI_CONFIG_PAGETYPE_IOC, 0x00,
+                              "*b*b*w");
+}
+
+static
+size_t mptsas_config_ioc_5(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(5, MPI_CONFIG_PAGETYPE_IOC, 0x00,
+                              "*l*b*b*w");
+}
+
+static
+size_t mptsas_config_ioc_6(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK(6, MPI_CONFIG_PAGETYPE_IOC, 0x01,
+                              "*l*b*b*b*b*b*b*b*b*b*b*w*l*l*l*l*b*b*w"
+                              "*w*w*w*w*l*l*l");
+}
+
+/* SAS I/O unit pages (extended) */
+
+#define MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE 16
+
+#define MPI_SAS_IOUNIT0_RATE_FAILED_SPEED_NEGOTIATION 0x02
+#define MPI_SAS_IOUNIT0_RATE_1_5                      0x08
+#define MPI_SAS_IOUNIT0_RATE_3_0                      0x09
+
+#define MPI_SAS_DEVICE_INFO_NO_DEVICE                 0x00000000
+#define MPI_SAS_DEVICE_INFO_END_DEVICE                0x00000001
+#define MPI_SAS_DEVICE_INFO_SSP_TARGET                0x00000400
+
+#define MPI_SAS_DEVICE0_ASTATUS_NO_ERRORS             0x00
+
+#define MPI_SAS_DEVICE0_FLAGS_DEVICE_PRESENT          0x0001
+#define MPI_SAS_DEVICE0_FLAGS_DEVICE_MAPPED           0x0002
+#define MPI_SAS_DEVICE0_FLAGS_MAPPING_PERSISTENT      0x0004
+
+
+
+static SCSIDevice *mptsas_phy_get_device(MPTSASState *s, int i,
+                                         int *phy_handle, int *dev_handle)
+{
+    SCSIDevice *d = scsi_device_find(&s->bus, 0, i, 0);
+
+    if (phy_handle) {
+        *phy_handle = i + 1;
+    }
+    if (dev_handle) {
+        *dev_handle = d ? i + 1 + MPTSAS_NUM_PORTS : 0;
+    }
+    return d;
+}
+
+static
+size_t mptsas_config_sas_io_unit_0(MPTSASState *s, uint8_t **data, int address)
+{
+    size_t size = MPTSAS_CONFIG_PACK_EXT(0, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x04,
+                                         "*w*wb*b*w"
+                                         repl(MPTSAS_NUM_PORTS, "*s16"),
+                                         MPTSAS_NUM_PORTS);
+
+    if (data) {
+        size_t ofs = size - MPTSAS_NUM_PORTS * MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE;
+        int i;
+
+        for (i = 0; i < MPTSAS_NUM_PORTS; i++) {
+            int phy_handle, dev_handle;
+            SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+            fill(*data + ofs, MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE,
+                 "bbbblwwl", i, 0, 0,
+                 (dev
+                  ? MPI_SAS_IOUNIT0_RATE_3_0
+                  : MPI_SAS_IOUNIT0_RATE_FAILED_SPEED_NEGOTIATION),
+                 (dev
+                  ? MPI_SAS_DEVICE_INFO_END_DEVICE | MPI_SAS_DEVICE_INFO_SSP_TARGET
+                  : MPI_SAS_DEVICE_INFO_NO_DEVICE),
+                 dev_handle,
+                 dev_handle,
+                 0);
+            ofs += MPTSAS_CONFIG_SAS_IO_UNIT_0_SIZE;
+        }
+        assert(ofs == size);
+    }
+    return size;
+}
+
+#define MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE 12
+
+static
+size_t mptsas_config_sas_io_unit_1(MPTSASState *s, uint8_t **data, int address)
+{
+    size_t size = MPTSAS_CONFIG_PACK_EXT(1, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x07,
+                                         "*w*w*w*wb*b*b*b"
+                                         repl(MPTSAS_NUM_PORTS, "*s12"),
+                                         MPTSAS_NUM_PORTS);
+
+    if (data) {
+        size_t ofs = size - MPTSAS_NUM_PORTS * MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE;
+        int i;
+
+        for (i = 0; i < MPTSAS_NUM_PORTS; i++) {
+            SCSIDevice *dev = mptsas_phy_get_device(s, i, NULL, NULL);
+            fill(*data + ofs, MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE,
+                 "bbbblww", i, 0, 0,
+                 (MPI_SAS_IOUNIT0_RATE_3_0 << 4) | MPI_SAS_IOUNIT0_RATE_1_5,
+                 (dev
+                  ? MPI_SAS_DEVICE_INFO_END_DEVICE | MPI_SAS_DEVICE_INFO_SSP_TARGET
+                  : MPI_SAS_DEVICE_INFO_NO_DEVICE),
+                 0, 0);
+            ofs += MPTSAS_CONFIG_SAS_IO_UNIT_1_SIZE;
+        }
+        assert(ofs == size);
+    }
+    return size;
+}
+
+static
+size_t mptsas_config_sas_io_unit_2(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK_EXT(2, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x06,
+                                  "*b*b*w*w*w*b*b*w");
+}
+
+static
+size_t mptsas_config_sas_io_unit_3(MPTSASState *s, uint8_t **data, int address)
+{
+    return MPTSAS_CONFIG_PACK_EXT(3, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT, 0x06,
+                                  "*l*l*l*l*l*l*l*l*l");
+}
+
+/* SAS PHY pages (extended) */
+
+static int mptsas_phy_addr_get(MPTSASState *s, int address)
+{
+    int i;
+    if ((address >> MPI_SAS_PHY_PGAD_FORM_SHIFT) == 0) {
+        i = address & 255;
+    } else if ((address >> MPI_SAS_PHY_PGAD_FORM_SHIFT) == 1) {
+        i = address & 65535;
+    } else {
+        return -EINVAL;
+    }
+
+    if (i >= MPTSAS_NUM_PORTS) {
+        return -EINVAL;
+    }
+
+    return i;
+}
+
+static
+size_t mptsas_config_phy_0(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_phy_addr_get(s, address);
+    SCSIDevice *dev;
+
+    if (i < 0) {
+        trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 0);
+        return i;
+    }
+
+    dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+    trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 0);
+
+    return MPTSAS_CONFIG_PACK_EXT(0, MPI_CONFIG_EXTPAGETYPE_SAS_PHY, 0x01,
+                                  "w*wqwb*blbb*b*b*l",
+                                  dev_handle, s->sas_addr, dev_handle, i,
+                                  (dev
+                                   ? MPI_SAS_DEVICE_INFO_END_DEVICE /* | MPI_SAS_DEVICE_INFO_SSP_TARGET?? */
+                                   : MPI_SAS_DEVICE_INFO_NO_DEVICE),
+                                  (MPI_SAS_IOUNIT0_RATE_3_0 << 4) | MPI_SAS_IOUNIT0_RATE_1_5,
+                                  (MPI_SAS_IOUNIT0_RATE_3_0 << 4) | MPI_SAS_IOUNIT0_RATE_1_5);
+}
+
+static
+size_t mptsas_config_phy_1(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_phy_addr_get(s, address);
+
+    if (i < 0) {
+        trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 1);
+        return i;
+    }
+
+    (void) mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+    trace_mptsas_config_sas_phy(s, address, i, phy_handle, dev_handle, 1);
+
+    return MPTSAS_CONFIG_PACK_EXT(1, MPI_CONFIG_EXTPAGETYPE_SAS_PHY, 0x01,
+                                  "*l*l*l*l*l");
+}
+
+/* SAS device pages (extended) */
+
+static int mptsas_device_addr_get(MPTSASState *s, int address)
+{
+    uint32_t handle, i;
+    uint32_t form = address >> MPI_SAS_PHY_PGAD_FORM_SHIFT;
+    if (form == MPI_SAS_DEVICE_PGAD_FORM_GET_NEXT_HANDLE) {
+        handle = address & MPI_SAS_DEVICE_PGAD_GNH_HANDLE_MASK;
+        do {
+            if (handle == 65535) {
+                handle = MPTSAS_NUM_PORTS + 1;
+            } else {
+                ++handle;
+            }
+            i = handle - 1 - MPTSAS_NUM_PORTS;
+        } while (i < MPTSAS_NUM_PORTS && !scsi_device_find(&s->bus, 0, i, 0));
+
+    } else if (form == MPI_SAS_DEVICE_PGAD_FORM_BUS_TARGET_ID) {
+        if (address & MPI_SAS_DEVICE_PGAD_BT_BUS_MASK) {
+            return -EINVAL;
+        }
+        i = address & MPI_SAS_DEVICE_PGAD_BT_TID_MASK;
+
+    } else if (form == MPI_SAS_DEVICE_PGAD_FORM_HANDLE) {
+        handle = address & MPI_SAS_DEVICE_PGAD_H_HANDLE_MASK;
+        i = handle - 1 - MPTSAS_NUM_PORTS;
+
+    } else {
+        return -EINVAL;
+    }
+
+    if (i >= MPTSAS_NUM_PORTS) {
+        return -EINVAL;
+    }
+
+    return i;
+}
+
+static
+size_t mptsas_config_sas_device_0(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_device_addr_get(s, address);
+    SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+    trace_mptsas_config_sas_device(s, address, i, phy_handle, dev_handle, 0);
+    if (!dev) {
+        return -ENOENT;
+    }
+
+    return MPTSAS_CONFIG_PACK_EXT(0, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE, 0x05,
+                                  "*w*wqwbbwbblwb*b",
+                                  dev->wwn, phy_handle, i,
+                                  MPI_SAS_DEVICE0_ASTATUS_NO_ERRORS,
+                                  dev_handle, i, 0,
+                                  MPI_SAS_DEVICE_INFO_END_DEVICE | MPI_SAS_DEVICE_INFO_SSP_TARGET,
+                                  (MPI_SAS_DEVICE0_FLAGS_DEVICE_PRESENT |
+                                   MPI_SAS_DEVICE0_FLAGS_DEVICE_MAPPED |
+                                   MPI_SAS_DEVICE0_FLAGS_MAPPING_PERSISTENT), i);
+}
+
+static
+size_t mptsas_config_sas_device_1(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_device_addr_get(s, address);
+    SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+    trace_mptsas_config_sas_device(s, address, i, phy_handle, dev_handle, 1);
+    if (!dev) {
+        return -ENOENT;
+    }
+
+    return MPTSAS_CONFIG_PACK_EXT(1, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE, 0x00,
+                                  "*lq*lwbb*s20",
+                                  dev->wwn, dev_handle, i, 0);
+}
+
+static
+size_t mptsas_config_sas_device_2(MPTSASState *s, uint8_t **data, int address)
+{
+    int phy_handle = -1;
+    int dev_handle = -1;
+    int i = mptsas_device_addr_get(s, address);
+    SCSIDevice *dev = mptsas_phy_get_device(s, i, &phy_handle, &dev_handle);
+
+    trace_mptsas_config_sas_device(s, address, i, phy_handle, dev_handle, 2);
+    if (!dev) {
+        return -ENOENT;
+    }
+
+    return MPTSAS_CONFIG_PACK_EXT(2, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE, 0x01,
+                                  "ql", dev->wwn, 0);
+}
+
+typedef struct MPTSASConfigPage {
+    uint8_t number;
+    uint8_t type;
+    size_t (*mpt_config_build)(MPTSASState *s, uint8_t **data, int address);
+} MPTSASConfigPage;
+
+static const MPTSASConfigPage mptsas_config_pages[] = {
+    {
+        0, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_0,
+    }, {
+        1, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_1,
+    }, {
+        2, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_2,
+    }, {
+        3, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_3,
+    }, {
+        4, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_4,
+    }, {
+        5, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_5,
+    }, {
+        6, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_6,
+    }, {
+        7, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_7,
+    }, {
+        8, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_8,
+    }, {
+        9, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_9,
+    }, {
+        10, MPI_CONFIG_PAGETYPE_MANUFACTURING,
+        mptsas_config_manufacturing_10,
+    }, {
+        0, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_0,
+    }, {
+        1, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_1,
+    }, {
+        2, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_2,
+    }, {
+        3, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_3,
+    }, {
+        4, MPI_CONFIG_PAGETYPE_IO_UNIT,
+        mptsas_config_io_unit_4,
+    }, {
+        0, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_0,
+    }, {
+        1, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_1,
+    }, {
+        2, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_2,
+    }, {
+        3, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_3,
+    }, {
+        4, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_4,
+    }, {
+        5, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_5,
+    }, {
+        6, MPI_CONFIG_PAGETYPE_IOC,
+        mptsas_config_ioc_6,
+    }, {
+        0, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_0,
+    }, {
+        1, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_1,
+    }, {
+        2, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_2,
+    }, {
+        3, MPI_CONFIG_EXTPAGETYPE_SAS_IO_UNIT,
+        mptsas_config_sas_io_unit_3,
+    }, {
+        0, MPI_CONFIG_EXTPAGETYPE_SAS_PHY,
+        mptsas_config_phy_0,
+    }, {
+        1, MPI_CONFIG_EXTPAGETYPE_SAS_PHY,
+        mptsas_config_phy_1,
+    }, {
+        0, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE,
+        mptsas_config_sas_device_0,
+    }, {
+        1, MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE,
+        mptsas_config_sas_device_1,
+    }, {
+       2,  MPI_CONFIG_EXTPAGETYPE_SAS_DEVICE,
+        mptsas_config_sas_device_2,
+    }
+};
+
+static const MPTSASConfigPage *mptsas_find_config_page(int type, int number)
+{
+    const MPTSASConfigPage *page;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(mptsas_config_pages); i++) {
+        page = &mptsas_config_pages[i];
+        if (page->type == type && page->number == number) {
+            return page;
+        }
+    }
+
+    return NULL;
+}
+
+void mptsas_process_config(MPTSASState *s, MPIMsgConfig *req)
+{
+    PCIDevice *pci = PCI_DEVICE(s);
+
+    MPIMsgConfigReply reply;
+    const MPTSASConfigPage *page;
+    size_t length;
+    uint8_t type;
+    uint8_t *data = NULL;
+    uint32_t flags_and_length;
+    uint32_t dmalen;
+    uint64_t pa;
+
+    mptsas_fix_config_endianness(req);
+
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    /* Copy common bits from the request into the reply. */
+    memset(&reply, 0, sizeof(reply));
+    reply.Action      = req->Action;
+    reply.Function    = req->Function;
+    reply.MsgContext  = req->MsgContext;
+    reply.MsgLength   = sizeof(reply) / 4;
+    reply.PageType    = req->PageType;
+    reply.PageNumber  = req->PageNumber;
+    reply.PageLength  = req->PageLength;
+    reply.PageVersion = req->PageVersion;
+
+    type = req->PageType & MPI_CONFIG_PAGETYPE_MASK;
+    if (type == MPI_CONFIG_PAGETYPE_EXTENDED) {
+        type = req->ExtPageType;
+        if (type <= MPI_CONFIG_PAGETYPE_MASK) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_TYPE;
+            goto out;
+        }
+
+        reply.ExtPageType = req->ExtPageType;
+    }
+
+    page = mptsas_find_config_page(type, req->PageNumber);
+
+    switch(req->Action) {
+    case MPI_CONFIG_ACTION_PAGE_DEFAULT:
+    case MPI_CONFIG_ACTION_PAGE_HEADER:
+    case MPI_CONFIG_ACTION_PAGE_READ_NVRAM:
+    case MPI_CONFIG_ACTION_PAGE_READ_CURRENT:
+    case MPI_CONFIG_ACTION_PAGE_READ_DEFAULT:
+    case MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT:
+    case MPI_CONFIG_ACTION_PAGE_WRITE_NVRAM:
+        break;
+
+    default:
+        reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_ACTION;
+        goto out;
+    }
+
+    if (!page) {
+        page = mptsas_find_config_page(type, 1);
+        if (page) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+        } else {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_TYPE;
+        }
+        goto out;
+    }
+
+    if (req->Action == MPI_CONFIG_ACTION_PAGE_DEFAULT ||
+        req->Action == MPI_CONFIG_ACTION_PAGE_HEADER) {
+        length = page->mpt_config_build(s, NULL, req->PageAddress);
+        if ((ssize_t)length < 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+            goto out;
+        } else {
+            goto done;
+        }
+    }
+
+    if (req->Action == MPI_CONFIG_ACTION_PAGE_WRITE_CURRENT ||
+        req->Action == MPI_CONFIG_ACTION_PAGE_WRITE_NVRAM) {
+        length = page->mpt_config_build(s, NULL, req->PageAddress);
+        if ((ssize_t)length < 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+        } else {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_CANT_COMMIT;
+        }
+        goto out;
+    }
+
+    flags_and_length = req->PageBufferSGE.FlagsLength;
+    dmalen = flags_and_length & MPI_SGE_LENGTH_MASK;
+    if (dmalen == 0) {
+        length = page->mpt_config_build(s, NULL, req->PageAddress);
+        if ((ssize_t)length < 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+            goto out;
+        } else {
+            goto done;
+        }
+    }
+
+    if (flags_and_length & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+        pa = req->PageBufferSGE.u.Address64;
+    } else {
+        pa = req->PageBufferSGE.u.Address32;
+    }
+
+    /* Only read actions left.  */
+    length = page->mpt_config_build(s, &data, req->PageAddress);
+    if ((ssize_t)length < 0) {
+        reply.IOCStatus = MPI_IOCSTATUS_CONFIG_INVALID_PAGE;
+        goto out;
+    } else {
+        assert(data[2] == page->number);
+        pci_dma_write(pci, pa, data, MIN(length, dmalen));
+        goto done;
+    }
+
+    abort();
+
+done:
+    if (type > MPI_CONFIG_PAGETYPE_MASK) {
+        reply.ExtPageLength = length / 4;
+        reply.ExtPageType   = req->ExtPageType;
+    } else {
+        reply.PageLength    = length / 4;
+    }
+
+out:
+    mptsas_fix_config_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+    g_free(data);
+}
diff --git a/hw/scsi/mptendian.c b/hw/scsi/mptendian.c
new file mode 100644
index 0000000..b7fe2a2
--- /dev/null
+++ b/hw/scsi/mptendian.c
@@ -0,0 +1,204 @@
+/*
+ * QEMU LSI SAS1068 Host Bus Adapter emulation
+ * Endianness conversion for MPI data structures
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * Authors: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "sysemu/dma.h"
+#include "sysemu/block-backend.h"
+#include "hw/pci/msi.h"
+#include "qemu/iov.h"
+#include "hw/scsi/scsi.h"
+#include "block/scsi.h"
+#include "trace.h"
+
+#include "mptsas.h"
+#include "mpi.h"
+
+static void mptsas_fix_sgentry_endianness(MPISGEntry *sge)
+{
+    le32_to_cpus(&sge->FlagsLength);
+    if (sge->FlagsLength & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+       le64_to_cpus(&sge->u.Address64);
+    } else {
+       le32_to_cpus(&sge->u.Address32);
+    }
+}
+
+static void mptsas_fix_sgentry_endianness_reply(MPISGEntry *sge)
+{
+    if (sge->FlagsLength & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+       cpu_to_le64s(&sge->u.Address64);
+    } else {
+       cpu_to_le32s(&sge->u.Address32);
+    }
+    cpu_to_le32s(&sge->FlagsLength);
+}
+
+void mptsas_fix_scsi_io_endianness(MPIMsgSCSIIORequest *req)
+{
+    le32_to_cpus(&req->MsgContext);
+    le32_to_cpus(&req->Control);
+    le32_to_cpus(&req->DataLength);
+    le32_to_cpus(&req->SenseBufferLowAddr);
+}
+
+void mptsas_fix_scsi_io_reply_endianness(MPIMsgSCSIIOReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le32s(&reply->TransferCount);
+    cpu_to_le32s(&reply->SenseCount);
+    cpu_to_le32s(&reply->ResponseInfo);
+    cpu_to_le16s(&reply->TaskTag);
+}
+
+void mptsas_fix_scsi_task_mgmt_endianness(MPIMsgSCSITaskMgmt *req)
+{
+    le32_to_cpus(&req->MsgContext);
+    le32_to_cpus(&req->TaskMsgContext);
+}
+
+void mptsas_fix_scsi_task_mgmt_reply_endianness(MPIMsgSCSITaskMgmtReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le32s(&reply->TerminationCount);
+}
+
+void mptsas_fix_ioc_init_endianness(MPIMsgIOCInit *req)
+{
+    le32_to_cpus(&req->MsgContext);
+    le16_to_cpus(&req->ReplyFrameSize);
+    le32_to_cpus(&req->HostMfaHighAddr);
+    le32_to_cpus(&req->SenseBufferHighAddr);
+    le32_to_cpus(&req->ReplyFifoHostSignalingAddr);
+    mptsas_fix_sgentry_endianness(&req->HostPageBufferSGE);
+    le16_to_cpus(&req->MsgVersion);
+    le16_to_cpus(&req->HeaderVersion);
+}
+
+void mptsas_fix_ioc_init_reply_endianness(MPIMsgIOCInitReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+}
+
+void mptsas_fix_ioc_facts_endianness(MPIMsgIOCFacts *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_ioc_facts_reply_endianness(MPIMsgIOCFactsReply *reply)
+{
+    cpu_to_le16s(&reply->MsgVersion);
+    cpu_to_le16s(&reply->HeaderVersion);
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCExceptions);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le16s(&reply->ReplyQueueDepth);
+    cpu_to_le16s(&reply->RequestFrameSize);
+    cpu_to_le16s(&reply->ProductID);
+    cpu_to_le32s(&reply->CurrentHostMfaHighAddr);
+    cpu_to_le16s(&reply->GlobalCredits);
+    cpu_to_le32s(&reply->CurrentSenseBufferHighAddr);
+    cpu_to_le16s(&reply->CurReplyFrameSize);
+    cpu_to_le32s(&reply->FWImageSize);
+    cpu_to_le32s(&reply->IOCCapabilities);
+    cpu_to_le16s(&reply->HighPriorityQueueDepth);
+    mptsas_fix_sgentry_endianness_reply(&reply->HostPageBufferSGE);
+    cpu_to_le32s(&reply->ReplyFifoHostSignalingAddr);
+}
+
+void mptsas_fix_config_endianness(MPIMsgConfig *req)
+{
+    le16_to_cpus(&req->ExtPageLength);
+    le32_to_cpus(&req->MsgContext);
+    le32_to_cpus(&req->PageAddress);
+    mptsas_fix_sgentry_endianness(&req->PageBufferSGE);
+}
+
+void mptsas_fix_config_reply_endianness(MPIMsgConfigReply *reply)
+{
+    cpu_to_le16s(&reply->ExtPageLength);
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+}
+
+void mptsas_fix_port_facts_endianness(MPIMsgPortFacts *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_port_facts_reply_endianness(MPIMsgPortFactsReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le16s(&reply->MaxDevices);
+    cpu_to_le16s(&reply->PortSCSIID);
+    cpu_to_le16s(&reply->ProtocolFlags);
+    cpu_to_le16s(&reply->MaxPostedCmdBuffers);
+    cpu_to_le16s(&reply->MaxPersistentIDs);
+    cpu_to_le16s(&reply->MaxLanBuckets);
+}
+
+void mptsas_fix_port_enable_endianness(MPIMsgPortEnable *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_port_enable_reply_endianness(MPIMsgPortEnableReply *reply)
+{
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+}
+
+void mptsas_fix_event_notification_endianness(MPIMsgEventNotify *req)
+{
+    le32_to_cpus(&req->MsgContext);
+}
+
+void mptsas_fix_event_notification_reply_endianness(MPIMsgEventNotifyReply *reply)
+{
+    int length = reply->EventDataLength;
+    int i;
+
+    cpu_to_le16s(&reply->EventDataLength);
+    cpu_to_le32s(&reply->MsgContext);
+    cpu_to_le16s(&reply->IOCStatus);
+    cpu_to_le32s(&reply->IOCLogInfo);
+    cpu_to_le32s(&reply->Event);
+    cpu_to_le32s(&reply->EventContext);
+
+    /* Really depends on the event kind.  This will do for now.  */
+    for (i = 0; i < length; i++) {
+        cpu_to_le32s(&reply->Data[i]);
+    }
+}
+
diff --git a/hw/scsi/mptsas.c b/hw/scsi/mptsas.c
new file mode 100644
index 0000000..0e0a22f
--- /dev/null
+++ b/hw/scsi/mptsas.c
@@ -0,0 +1,1455 @@
+/*
+ * QEMU LSI SAS1068 Host Bus Adapter emulation
+ * Based on the QEMU Megaraid emulator
+ *
+ * Copyright (c) 2009-2012 Hannes Reinecke, SUSE Labs
+ * Copyright (c) 2012 Verizon, Inc.
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * Authors: Don Slutz, Paolo Bonzini
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/pci/pci.h"
+#include "sysemu/dma.h"
+#include "sysemu/block-backend.h"
+#include "hw/pci/msi.h"
+#include "qemu/iov.h"
+#include "hw/scsi/scsi.h"
+#include "block/scsi.h"
+#include "trace.h"
+#include "qapi/error.h"
+#include "mptsas.h"
+#include "mpi.h"
+
+#define NAA_LOCALLY_ASSIGNED_ID 0x3ULL
+#define IEEE_COMPANY_LOCALLY_ASSIGNED 0x525400
+
+#define TYPE_MPTSAS1068 "mptsas1068"
+
+#define MPT_SAS(obj) \
+    OBJECT_CHECK(MPTSASState, (obj), TYPE_MPTSAS1068)
+
+#define MPTSAS1068_PRODUCT_ID                  \
+    (MPI_FW_HEADER_PID_FAMILY_1068_SAS |       \
+     MPI_FW_HEADER_PID_PROD_INITIATOR_SCSI |   \
+     MPI_FW_HEADER_PID_TYPE_SAS)
+
+struct MPTSASRequest {
+    MPIMsgSCSIIORequest scsi_io;
+    SCSIRequest *sreq;
+    QEMUSGList qsg;
+    MPTSASState *dev;
+
+    QTAILQ_ENTRY(MPTSASRequest) next;
+};
+
+static void mptsas_update_interrupt(MPTSASState *s)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    uint32_t state = s->intr_status & ~(s->intr_mask | MPI_HIS_IOP_DOORBELL_STATUS);
+
+    if (msi_enabled(pci)) {
+        if (state) {
+            trace_mptsas_irq_msi(s);
+            msi_notify(pci, 0);
+        }
+    }
+
+    trace_mptsas_irq_intx(s, !!state);
+    pci_set_irq(pci, !!state);
+}
+
+static void mptsas_set_fault(MPTSASState *s, uint32_t code)
+{
+    if ((s->state & MPI_IOC_STATE_FAULT) == 0) {
+        s->state = MPI_IOC_STATE_FAULT | code;
+    }
+}
+
+#define MPTSAS_FIFO_INVALID(s, name)                     \
+    ((s)->name##_head > ARRAY_SIZE((s)->name) ||         \
+     (s)->name##_tail > ARRAY_SIZE((s)->name))
+
+#define MPTSAS_FIFO_EMPTY(s, name)                       \
+    ((s)->name##_head == (s)->name##_tail)
+
+#define MPTSAS_FIFO_FULL(s, name)                        \
+    ((s)->name##_head == ((s)->name##_tail + 1) % ARRAY_SIZE((s)->name))
+
+#define MPTSAS_FIFO_GET(s, name) ({                      \
+    uint32_t _val = (s)->name[(s)->name##_head++];       \
+    (s)->name##_head %= ARRAY_SIZE((s)->name);           \
+    _val;                                                \
+})
+
+#define MPTSAS_FIFO_PUT(s, name, val) do {       \
+    (s)->name[(s)->name##_tail++] = (val);       \
+    (s)->name##_tail %= ARRAY_SIZE((s)->name);   \
+} while(0)
+
+static void mptsas_post_reply(MPTSASState *s, MPIDefaultReply *reply)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    uint32_t addr_lo;
+
+    if (MPTSAS_FIFO_EMPTY(s, reply_free) || MPTSAS_FIFO_FULL(s, reply_post)) {
+        mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        return;
+    }
+
+    addr_lo = MPTSAS_FIFO_GET(s, reply_free);
+
+    pci_dma_write(pci, addr_lo | s->host_mfa_high_addr, reply,
+                  MIN(s->reply_frame_size, 4 * reply->MsgLength));
+
+    MPTSAS_FIFO_PUT(s, reply_post, MPI_ADDRESS_REPLY_A_BIT | (addr_lo >> 1));
+
+    s->intr_status |= MPI_HIS_REPLY_MESSAGE_INTERRUPT;
+    if (s->doorbell_state == DOORBELL_WRITE) {
+        s->doorbell_state = DOORBELL_NONE;
+        s->intr_status |= MPI_HIS_DOORBELL_INTERRUPT;
+    }
+    mptsas_update_interrupt(s);
+}
+
+void mptsas_reply(MPTSASState *s, MPIDefaultReply *reply)
+{
+    if (s->doorbell_state == DOORBELL_WRITE) {
+        /* The reply is sent out in 16 bit chunks, while the size
+         * in the reply is in 32 bit units.
+         */
+        s->doorbell_state = DOORBELL_READ;
+        s->doorbell_reply_idx = 0;
+        s->doorbell_reply_size = reply->MsgLength * 2;
+        memcpy(s->doorbell_reply, reply, s->doorbell_reply_size * 2);
+        s->intr_status |= MPI_HIS_DOORBELL_INTERRUPT;
+        mptsas_update_interrupt(s);
+    } else {
+        mptsas_post_reply(s, reply);
+    }
+}
+
+static void mptsas_turbo_reply(MPTSASState *s, uint32_t msgctx)
+{
+    if (MPTSAS_FIFO_FULL(s, reply_post)) {
+        mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        return;
+    }
+
+    /* The reply is just the message context ID (bit 31 = clear). */
+    MPTSAS_FIFO_PUT(s, reply_post, msgctx);
+
+    s->intr_status |= MPI_HIS_REPLY_MESSAGE_INTERRUPT;
+    mptsas_update_interrupt(s);
+}
+
+#define MPTSAS_MAX_REQUEST_SIZE 52
+
+static const int mpi_request_sizes[] = {
+    [MPI_FUNCTION_SCSI_IO_REQUEST]    = sizeof(MPIMsgSCSIIORequest),
+    [MPI_FUNCTION_SCSI_TASK_MGMT]     = sizeof(MPIMsgSCSITaskMgmt),
+    [MPI_FUNCTION_IOC_INIT]           = sizeof(MPIMsgIOCInit),
+    [MPI_FUNCTION_IOC_FACTS]          = sizeof(MPIMsgIOCFacts),
+    [MPI_FUNCTION_CONFIG]             = sizeof(MPIMsgConfig),
+    [MPI_FUNCTION_PORT_FACTS]         = sizeof(MPIMsgPortFacts),
+    [MPI_FUNCTION_PORT_ENABLE]        = sizeof(MPIMsgPortEnable),
+    [MPI_FUNCTION_EVENT_NOTIFICATION] = sizeof(MPIMsgEventNotify),
+};
+
+static dma_addr_t mptsas_ld_sg_base(MPTSASState *s, uint32_t flags_and_length,
+                                    dma_addr_t *sgaddr)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    dma_addr_t addr;
+
+    if (flags_and_length & MPI_SGE_FLAGS_64_BIT_ADDRESSING) {
+        addr = ldq_le_pci_dma(pci, *sgaddr + 4);
+        *sgaddr += 12;
+    } else {
+        addr = ldl_le_pci_dma(pci, *sgaddr + 4);
+        *sgaddr += 8;
+    }
+    return addr;
+}
+
+static int mptsas_build_sgl(MPTSASState *s, MPTSASRequest *req, hwaddr addr)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    hwaddr next_chain_addr;
+    uint32_t left;
+    hwaddr sgaddr;
+    uint32_t chain_offset;
+
+    chain_offset = req->scsi_io.ChainOffset;
+    next_chain_addr = addr + chain_offset * sizeof(uint32_t);
+    sgaddr = addr + sizeof(MPIMsgSCSIIORequest);
+    pci_dma_sglist_init(&req->qsg, pci, 4);
+    left = req->scsi_io.DataLength;
+
+    for(;;) {
+        dma_addr_t addr, len;
+        uint32_t flags_and_length;
+
+        flags_and_length = ldl_le_pci_dma(pci, sgaddr);
+        len = flags_and_length & MPI_SGE_LENGTH_MASK;
+        if ((flags_and_length & MPI_SGE_FLAGS_ELEMENT_TYPE_MASK)
+            != MPI_SGE_FLAGS_SIMPLE_ELEMENT ||
+            (!len &&
+             !(flags_and_length & MPI_SGE_FLAGS_END_OF_LIST) &&
+             !(flags_and_length & MPI_SGE_FLAGS_END_OF_BUFFER))) {
+            return MPI_IOCSTATUS_INVALID_SGL;
+        }
+
+        len = MIN(len, left);
+        if (!len) {
+            /* We reached the desired transfer length, ignore extra
+             * elements of the s/g list.
+             */
+            break;
+        }
+
+        addr = mptsas_ld_sg_base(s, flags_and_length, &sgaddr);
+        qemu_sglist_add(&req->qsg, addr, len);
+        left -= len;
+
+        if (flags_and_length & MPI_SGE_FLAGS_END_OF_LIST) {
+            break;
+        }
+
+        if (flags_and_length & MPI_SGE_FLAGS_LAST_ELEMENT) {
+            if (!chain_offset) {
+                break;
+            }
+
+            flags_and_length = ldl_le_pci_dma(pci, next_chain_addr);
+            if ((flags_and_length & MPI_SGE_FLAGS_ELEMENT_TYPE_MASK)
+                != MPI_SGE_FLAGS_CHAIN_ELEMENT) {
+                return MPI_IOCSTATUS_INVALID_SGL;
+            }
+
+            sgaddr = mptsas_ld_sg_base(s, flags_and_length, &next_chain_addr);
+            chain_offset =
+                (flags_and_length & MPI_SGE_CHAIN_OFFSET_MASK) >> MPI_SGE_CHAIN_OFFSET_SHIFT;
+            next_chain_addr = sgaddr + chain_offset * sizeof(uint32_t);
+        }
+    }
+    return 0;
+}
+
+static void mptsas_free_request(MPTSASRequest *req)
+{
+    MPTSASState *s = req->dev;
+
+    if (req->sreq != NULL) {
+        req->sreq->hba_private = NULL;
+        scsi_req_unref(req->sreq);
+        req->sreq = NULL;
+        QTAILQ_REMOVE(&s->pending, req, next);
+    }
+    qemu_sglist_destroy(&req->qsg);
+    g_free(req);
+}
+
+static int mptsas_scsi_device_find(MPTSASState *s, int bus, int target,
+                                   uint8_t *lun, SCSIDevice **sdev)
+{
+    if (bus != 0) {
+        return MPI_IOCSTATUS_SCSI_INVALID_BUS;
+    }
+
+    if (target >= s->max_devices) {
+        return MPI_IOCSTATUS_SCSI_INVALID_TARGETID;
+    }
+
+    *sdev = scsi_device_find(&s->bus, bus, target, lun[1]);
+    if (!*sdev) {
+        return MPI_IOCSTATUS_SCSI_DEVICE_NOT_THERE;
+    }
+
+    return 0;
+}
+
+static int mptsas_process_scsi_io_request(MPTSASState *s,
+                                          MPIMsgSCSIIORequest *scsi_io,
+                                          hwaddr addr)
+{
+    MPTSASRequest *req;
+    MPIMsgSCSIIOReply reply;
+    SCSIDevice *sdev;
+    int status;
+
+    mptsas_fix_scsi_io_endianness(scsi_io);
+
+    trace_mptsas_process_scsi_io_request(s, scsi_io->Bus, scsi_io->TargetID,
+                                         scsi_io->LUN[1], scsi_io->DataLength);
+
+    status = mptsas_scsi_device_find(s, scsi_io->Bus, scsi_io->TargetID,
+                                     scsi_io->LUN, &sdev);
+    if (status) {
+        goto bad;
+    }
+
+    req = g_new(MPTSASRequest, 1);
+    QTAILQ_INSERT_TAIL(&s->pending, req, next);
+    req->scsi_io = *scsi_io;
+    req->dev = s;
+
+    status = mptsas_build_sgl(s, req, addr);
+    if (status) {
+        goto free_bad;
+    }
+
+    if (req->qsg.size < scsi_io->DataLength) {
+        trace_mptsas_sgl_overflow(s, scsi_io->MsgContext, scsi_io->DataLength,
+                                  req->qsg.size);
+        status = MPI_IOCSTATUS_INVALID_SGL;
+        goto free_bad;
+    }
+
+    req->sreq = scsi_req_new(sdev, scsi_io->MsgContext,
+                            scsi_io->LUN[1], scsi_io->CDB, req);
+
+    if (req->sreq->cmd.xfer > scsi_io->DataLength) {
+        goto overrun;
+    }
+    switch (scsi_io->Control & MPI_SCSIIO_CONTROL_DATADIRECTION_MASK) {
+    case MPI_SCSIIO_CONTROL_NODATATRANSFER:
+        if (req->sreq->cmd.mode != SCSI_XFER_NONE) {
+            goto overrun;
+        }
+        break;
+
+    case MPI_SCSIIO_CONTROL_WRITE:
+        if (req->sreq->cmd.mode != SCSI_XFER_TO_DEV) {
+            goto overrun;
+        }
+        break;
+
+    case MPI_SCSIIO_CONTROL_READ:
+        if (req->sreq->cmd.mode != SCSI_XFER_FROM_DEV) {
+            goto overrun;
+        }
+        break;
+    }
+
+    if (scsi_req_enqueue(req->sreq)) {
+        scsi_req_continue(req->sreq);
+    }
+    return 0;
+
+overrun:
+    trace_mptsas_scsi_overflow(s, scsi_io->MsgContext, req->sreq->cmd.xfer,
+                               scsi_io->DataLength);
+    status = MPI_IOCSTATUS_SCSI_DATA_OVERRUN;
+free_bad:
+    mptsas_free_request(req);
+bad:
+    memset(&reply, 0, sizeof(reply));
+    reply.TargetID          = scsi_io->TargetID;
+    reply.Bus               = scsi_io->Bus;
+    reply.MsgLength         = sizeof(reply) / 4;
+    reply.Function          = scsi_io->Function;
+    reply.CDBLength         = scsi_io->CDBLength;
+    reply.SenseBufferLength = scsi_io->SenseBufferLength;
+    reply.MsgContext        = scsi_io->MsgContext;
+    reply.SCSIState         = MPI_SCSI_STATE_NO_SCSI_STATUS;
+    reply.IOCStatus         = status;
+
+    mptsas_fix_scsi_io_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+
+    return 0;
+}
+
+typedef struct {
+    Notifier                notifier;
+    MPTSASState             *s;
+    MPIMsgSCSITaskMgmtReply *reply;
+} MPTSASCancelNotifier;
+
+static void mptsas_cancel_notify(Notifier *notifier, void *data)
+{
+    MPTSASCancelNotifier *n = container_of(notifier,
+                                           MPTSASCancelNotifier,
+                                           notifier);
+
+    /* Abusing IOCLogInfo to store the expected number of requests... */
+    if (++n->reply->TerminationCount == n->reply->IOCLogInfo) {
+        n->reply->IOCLogInfo = 0;
+        mptsas_fix_scsi_task_mgmt_reply_endianness(n->reply);
+        mptsas_post_reply(n->s, (MPIDefaultReply *)n->reply);
+        g_free(n->reply);
+    }
+    g_free(n);
+}
+
+static void mptsas_process_scsi_task_mgmt(MPTSASState *s, MPIMsgSCSITaskMgmt *req)
+{
+    MPIMsgSCSITaskMgmtReply reply;
+    MPIMsgSCSITaskMgmtReply *reply_async;
+    int status, count;
+    SCSIDevice *sdev;
+    SCSIRequest *r, *next;
+    BusChild *kid;
+
+    mptsas_fix_scsi_task_mgmt_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.TargetID   = req->TargetID;
+    reply.Bus        = req->Bus;
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.Function   = req->Function;
+    reply.TaskType   = req->TaskType;
+    reply.MsgContext = req->MsgContext;
+
+    switch (req->TaskType) {
+    case MPI_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
+    case MPI_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
+        status = mptsas_scsi_device_find(s, req->Bus, req->TargetID,
+                                         req->LUN, &sdev);
+        if (status) {
+            reply.IOCStatus = status;
+            goto out;
+        }
+        if (sdev->lun != req->LUN[1]) {
+            reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN;
+            goto out;
+        }
+
+        QTAILQ_FOREACH_SAFE(r, &sdev->requests, next, next) {
+            MPTSASRequest *cmd_req = r->hba_private;
+            if (cmd_req && cmd_req->scsi_io.MsgContext == req->TaskMsgContext) {
+                break;
+            }
+        }
+        if (r) {
+            /*
+             * Assert that the request has not been completed yet, we
+             * check for it in the loop above.
+             */
+            assert(r->hba_private);
+            if (req->TaskType == MPI_SCSITASKMGMT_TASKTYPE_QUERY_TASK) {
+                /* "If the specified command is present in the task set, then
+                 * return a service response set to FUNCTION SUCCEEDED".
+                 */
+                reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_SUCCEEDED;
+            } else {
+                MPTSASCancelNotifier *notifier;
+
+                reply_async = g_memdup(&reply, sizeof(MPIMsgSCSITaskMgmtReply));
+                reply_async->IOCLogInfo = INT_MAX;
+
+                count = 1;
+                notifier = g_new(MPTSASCancelNotifier, 1);
+                notifier->s = s;
+                notifier->reply = reply_async;
+                notifier->notifier.notify = mptsas_cancel_notify;
+                scsi_req_cancel_async(r, &notifier->notifier);
+                goto reply_maybe_async;
+            }
+        }
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
+    case MPI_SCSITASKMGMT_TASKTYPE_CLEAR_TASK_SET:
+        status = mptsas_scsi_device_find(s, req->Bus, req->TargetID,
+                                         req->LUN, &sdev);
+        if (status) {
+            reply.IOCStatus = status;
+            goto out;
+        }
+        if (sdev->lun != req->LUN[1]) {
+            reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN;
+            goto out;
+        }
+
+        reply_async = g_memdup(&reply, sizeof(MPIMsgSCSITaskMgmtReply));
+        reply_async->IOCLogInfo = INT_MAX;
+
+        count = 0;
+        QTAILQ_FOREACH_SAFE(r, &sdev->requests, next, next) {
+            if (r->hba_private) {
+                MPTSASCancelNotifier *notifier;
+
+                count++;
+                notifier = g_new(MPTSASCancelNotifier, 1);
+                notifier->s = s;
+                notifier->reply = reply_async;
+                notifier->notifier.notify = mptsas_cancel_notify;
+                scsi_req_cancel_async(r, &notifier->notifier);
+            }
+        }
+
+reply_maybe_async:
+        if (reply_async->TerminationCount < count) {
+            reply_async->IOCLogInfo = count;
+            return;
+        }
+        g_free(reply_async);
+        reply.TerminationCount = count;
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
+        status = mptsas_scsi_device_find(s, req->Bus, req->TargetID,
+                                         req->LUN, &sdev);
+        if (status) {
+            reply.IOCStatus = status;
+            goto out;
+        }
+        if (sdev->lun != req->LUN[1]) {
+            reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_INVALID_LUN;
+            goto out;
+        }
+        qdev_reset_all(&sdev->qdev);
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
+        if (req->Bus != 0) {
+            reply.IOCStatus = MPI_IOCSTATUS_SCSI_INVALID_BUS;
+            goto out;
+        }
+        if (req->TargetID > s->max_devices) {
+            reply.IOCStatus = MPI_IOCSTATUS_SCSI_INVALID_TARGETID;
+            goto out;
+        }
+
+        QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
+            sdev = SCSI_DEVICE(kid->child);
+            if (sdev->channel == 0 && sdev->id == req->TargetID) {
+                qdev_reset_all(kid->child);
+            }
+        }
+        break;
+
+    case MPI_SCSITASKMGMT_TASKTYPE_RESET_BUS:
+        qbus_reset_all(&s->bus.qbus);
+        break;
+
+    default:
+        reply.ResponseCode = MPI_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED;
+        break;
+    }
+
+out:
+    mptsas_fix_scsi_task_mgmt_reply_endianness(&reply);
+    mptsas_post_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_ioc_init(MPTSASState *s, MPIMsgIOCInit *req)
+{
+    MPIMsgIOCInitReply reply;
+
+    mptsas_fix_ioc_init_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    s->who_init               = req->WhoInit;
+    s->reply_frame_size       = req->ReplyFrameSize;
+    s->max_buses              = req->MaxBuses;
+    s->max_devices            = req->MaxDevices ? req->MaxDevices : 256;
+    s->host_mfa_high_addr     = (hwaddr)req->HostMfaHighAddr << 32;
+    s->sense_buffer_high_addr = (hwaddr)req->SenseBufferHighAddr << 32;
+
+    if (s->state == MPI_IOC_STATE_READY) {
+        s->state = MPI_IOC_STATE_OPERATIONAL;
+    }
+
+    memset(&reply, 0, sizeof(reply));
+    reply.WhoInit    = s->who_init;
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.Function   = req->Function;
+    reply.MaxDevices = s->max_devices;
+    reply.MaxBuses   = s->max_buses;
+    reply.MsgContext = req->MsgContext;
+
+    mptsas_fix_ioc_init_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_ioc_facts(MPTSASState *s,
+                                     MPIMsgIOCFacts *req)
+{
+    MPIMsgIOCFactsReply reply;
+
+    mptsas_fix_ioc_facts_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.MsgVersion                 = 0x0105;
+    reply.MsgLength                  = sizeof(reply) / 4;
+    reply.Function                   = req->Function;
+    reply.MsgContext                 = req->MsgContext;
+    reply.MaxChainDepth              = MPTSAS_MAXIMUM_CHAIN_DEPTH;
+    reply.WhoInit                    = s->who_init;
+    reply.BlockSize                  = MPTSAS_MAX_REQUEST_SIZE / sizeof(uint32_t);
+    reply.ReplyQueueDepth            = ARRAY_SIZE(s->reply_post) - 1;
+    QEMU_BUILD_BUG_ON(ARRAY_SIZE(s->reply_post) != ARRAY_SIZE(s->reply_free));
+
+    reply.RequestFrameSize           = 128;
+    reply.ProductID                  = MPTSAS1068_PRODUCT_ID;
+    reply.CurrentHostMfaHighAddr     = s->host_mfa_high_addr >> 32;
+    reply.GlobalCredits              = ARRAY_SIZE(s->request_post) - 1;
+    reply.NumberOfPorts              = MPTSAS_NUM_PORTS;
+    reply.CurrentSenseBufferHighAddr = s->sense_buffer_high_addr >> 32;
+    reply.CurReplyFrameSize          = s->reply_frame_size;
+    reply.MaxDevices                 = s->max_devices;
+    reply.MaxBuses                   = s->max_buses;
+    reply.FWVersionDev               = 0;
+    reply.FWVersionUnit              = 0x92;
+    reply.FWVersionMinor             = 0x32;
+    reply.FWVersionMajor             = 0x1;
+
+    mptsas_fix_ioc_facts_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_port_facts(MPTSASState *s,
+                                     MPIMsgPortFacts *req)
+{
+    MPIMsgPortFactsReply reply;
+
+    mptsas_fix_port_facts_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.Function   = req->Function;
+    reply.PortNumber = req->PortNumber;
+    reply.MsgContext = req->MsgContext;
+
+    if (req->PortNumber < MPTSAS_NUM_PORTS) {
+        reply.PortType      = MPI_PORTFACTS_PORTTYPE_SAS;
+        reply.MaxDevices    = MPTSAS_NUM_PORTS;
+        reply.PortSCSIID    = MPTSAS_NUM_PORTS;
+        reply.ProtocolFlags = MPI_PORTFACTS_PROTOCOL_LOGBUSADDR | MPI_PORTFACTS_PROTOCOL_INITIATOR;
+    }
+
+    mptsas_fix_port_facts_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_port_enable(MPTSASState *s,
+                                       MPIMsgPortEnable *req)
+{
+    MPIMsgPortEnableReply reply;
+
+    mptsas_fix_port_enable_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    memset(&reply, 0, sizeof(reply));
+    reply.MsgLength  = sizeof(reply) / 4;
+    reply.PortNumber = req->PortNumber;
+    reply.Function   = req->Function;
+    reply.MsgContext = req->MsgContext;
+
+    mptsas_fix_port_enable_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_event_notification(MPTSASState *s,
+                                              MPIMsgEventNotify *req)
+{
+    MPIMsgEventNotifyReply reply;
+
+    mptsas_fix_event_notification_endianness(req);
+
+    QEMU_BUILD_BUG_ON(MPTSAS_MAX_REQUEST_SIZE < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_msg) < sizeof(*req));
+    QEMU_BUILD_BUG_ON(sizeof(s->doorbell_reply) < sizeof(reply));
+
+    /* Don't even bother storing whether event notification is enabled,
+     * since it is not accessible.
+     */
+
+    memset(&reply, 0, sizeof(reply));
+    reply.EventDataLength = sizeof(reply.Data) / 4;
+    reply.MsgLength       = sizeof(reply) / 4;
+    reply.Function        = req->Function;
+
+    /* This is set because events are sent through the reply FIFOs.  */
+    reply.MsgFlags        = MPI_MSGFLAGS_CONTINUATION_REPLY;
+
+    reply.MsgContext      = req->MsgContext;
+    reply.Event           = MPI_EVENT_EVENT_CHANGE;
+    reply.Data[0]         = !!req->Switch;
+
+    mptsas_fix_event_notification_reply_endianness(&reply);
+    mptsas_reply(s, (MPIDefaultReply *)&reply);
+}
+
+static void mptsas_process_message(MPTSASState *s, MPIRequestHeader *req)
+{
+    trace_mptsas_process_message(s, req->Function, req->MsgContext);
+    switch (req->Function) {
+    case MPI_FUNCTION_SCSI_TASK_MGMT:
+        mptsas_process_scsi_task_mgmt(s, (MPIMsgSCSITaskMgmt *)req);
+        break;
+
+    case MPI_FUNCTION_IOC_INIT:
+        mptsas_process_ioc_init(s, (MPIMsgIOCInit *)req);
+        break;
+
+    case MPI_FUNCTION_IOC_FACTS:
+        mptsas_process_ioc_facts(s, (MPIMsgIOCFacts *)req);
+        break;
+
+    case MPI_FUNCTION_PORT_FACTS:
+        mptsas_process_port_facts(s, (MPIMsgPortFacts *)req);
+        break;
+
+    case MPI_FUNCTION_PORT_ENABLE:
+        mptsas_process_port_enable(s, (MPIMsgPortEnable *)req);
+        break;
+
+    case MPI_FUNCTION_EVENT_NOTIFICATION:
+        mptsas_process_event_notification(s, (MPIMsgEventNotify *)req);
+        break;
+
+    case MPI_FUNCTION_CONFIG:
+        mptsas_process_config(s, (MPIMsgConfig *)req);
+        break;
+
+    default:
+        trace_mptsas_unhandled_cmd(s, req->Function, 0);
+        mptsas_set_fault(s, MPI_IOCSTATUS_INVALID_FUNCTION);
+        break;
+    }
+}
+
+static void mptsas_fetch_request(MPTSASState *s)
+{
+    PCIDevice *pci = (PCIDevice *) s;
+    char req[MPTSAS_MAX_REQUEST_SIZE];
+    MPIRequestHeader *hdr = (MPIRequestHeader *)req;
+    hwaddr addr;
+    int size;
+
+    /* Read the message header from the guest first. */
+    addr = s->host_mfa_high_addr | MPTSAS_FIFO_GET(s, request_post);
+    pci_dma_read(pci, addr, req, sizeof(hdr));
+
+    if (hdr->Function < ARRAY_SIZE(mpi_request_sizes) &&
+        mpi_request_sizes[hdr->Function]) {
+        /* Read the rest of the request based on the type.  Do not
+         * reread everything, as that could cause a TOC/TOU mismatch
+         * and leak data from the QEMU stack.
+         */
+        size = mpi_request_sizes[hdr->Function];
+        assert(size <= MPTSAS_MAX_REQUEST_SIZE);
+        pci_dma_read(pci, addr + sizeof(hdr), &req[sizeof(hdr)],
+                     size - sizeof(hdr));
+    }
+
+    if (hdr->Function == MPI_FUNCTION_SCSI_IO_REQUEST) {
+        /* SCSI I/O requests are separate from mptsas_process_message
+         * because they cannot be sent through the doorbell yet.
+         */
+        mptsas_process_scsi_io_request(s, (MPIMsgSCSIIORequest *)req, addr);
+    } else {
+        mptsas_process_message(s, (MPIRequestHeader *)req);
+    }
+}
+
+static void mptsas_fetch_requests(void *opaque)
+{
+    MPTSASState *s = opaque;
+
+    if (s->state != MPI_IOC_STATE_OPERATIONAL) {
+        mptsas_set_fault(s, MPI_IOCSTATUS_INVALID_STATE);
+        return;
+    }
+    while (!MPTSAS_FIFO_EMPTY(s, request_post)) {
+        mptsas_fetch_request(s);
+    }
+}
+
+static void mptsas_soft_reset(MPTSASState *s)
+{
+    uint32_t save_mask;
+
+    trace_mptsas_reset(s);
+
+    /* Temporarily disable interrupts */
+    save_mask = s->intr_mask;
+    s->intr_mask = MPI_HIM_DIM | MPI_HIM_RIM;
+    mptsas_update_interrupt(s);
+
+    qbus_reset_all(&s->bus.qbus);
+    s->intr_status = 0;
+    s->intr_mask = save_mask;
+
+    s->reply_free_tail = 0;
+    s->reply_free_head = 0;
+    s->reply_post_tail = 0;
+    s->reply_post_head = 0;
+    s->request_post_tail = 0;
+    s->request_post_head = 0;
+    qemu_bh_cancel(s->request_bh);
+
+    s->state = MPI_IOC_STATE_READY;
+}
+
+static uint32_t mptsas_doorbell_read(MPTSASState *s)
+{
+    uint32_t ret;
+
+    ret = (s->who_init << MPI_DOORBELL_WHO_INIT_SHIFT) & MPI_DOORBELL_WHO_INIT_MASK;
+    ret |= s->state;
+    switch (s->doorbell_state) {
+    case DOORBELL_NONE:
+        break;
+
+    case DOORBELL_WRITE:
+        ret |= MPI_DOORBELL_ACTIVE;
+        break;
+
+    case DOORBELL_READ:
+        /* Get rid of the IOC fault code.  */
+        ret &= ~MPI_DOORBELL_DATA_MASK;
+
+        assert(s->intr_status & MPI_HIS_DOORBELL_INTERRUPT);
+        assert(s->doorbell_reply_idx <= s->doorbell_reply_size);
+
+        ret |= MPI_DOORBELL_ACTIVE;
+        if (s->doorbell_reply_idx < s->doorbell_reply_size) {
+            /* For more information about this endian switch, see the
+             * commit message for commit 36b62ae ("fw_cfg: fix endianness in
+             * fw_cfg_data_mem_read() / _write()", 2015-01-16).
+             */
+            ret |= le16_to_cpu(s->doorbell_reply[s->doorbell_reply_idx++]);
+        }
+        break;
+
+    default:
+        abort();
+    }
+
+    return ret;
+}
+
+static void mptsas_doorbell_write(MPTSASState *s, uint32_t val)
+{
+    if (s->doorbell_state == DOORBELL_WRITE) {
+        if (s->doorbell_idx < s->doorbell_cnt) {
+            /* For more information about this endian switch, see the
+             * commit message for commit 36b62ae ("fw_cfg: fix endianness in
+             * fw_cfg_data_mem_read() / _write()", 2015-01-16).
+             */
+            s->doorbell_msg[s->doorbell_idx++] = cpu_to_le32(val);
+            if (s->doorbell_idx == s->doorbell_cnt) {
+                mptsas_process_message(s, (MPIRequestHeader *)s->doorbell_msg);
+            }
+        }
+        return;
+    }
+
+    switch ((val & MPI_DOORBELL_FUNCTION_MASK) >> MPI_DOORBELL_FUNCTION_SHIFT) {
+    case MPI_FUNCTION_IOC_MESSAGE_UNIT_RESET:
+        mptsas_soft_reset(s);
+        break;
+    case MPI_FUNCTION_IO_UNIT_RESET:
+        break;
+    case MPI_FUNCTION_HANDSHAKE:
+        s->doorbell_state = DOORBELL_WRITE;
+        s->doorbell_idx = 0;
+        s->doorbell_cnt = (val & MPI_DOORBELL_ADD_DWORDS_MASK)
+            >> MPI_DOORBELL_ADD_DWORDS_SHIFT;
+        s->intr_status |= MPI_HIS_DOORBELL_INTERRUPT;
+        mptsas_update_interrupt(s);
+        break;
+    default:
+        trace_mptsas_unhandled_doorbell_cmd(s, val);
+        break;
+    }
+}
+
+static void mptsas_write_sequence_write(MPTSASState *s, uint32_t val)
+{
+    /* If the diagnostic register is enabled, any write to this register
+     * will disable it.  Otherwise, the guest has to do a magic five-write
+     * sequence.
+     */
+    if (s->diagnostic & MPI_DIAG_DRWE) {
+        goto disable;
+    }
+
+    switch (s->diagnostic_idx) {
+    case 0:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_1ST_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 1:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_2ND_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 2:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_3RD_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 3:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_4TH_KEY_VALUE) {
+            goto disable;
+        }
+        break;
+    case 4:
+        if ((val & MPI_WRSEQ_KEY_VALUE_MASK) != MPI_WRSEQ_5TH_KEY_VALUE) {
+            goto disable;
+        }
+        /* Prepare Spaceball One for departure, and change the
+         * combination on my luggage!
+         */
+        s->diagnostic |= MPI_DIAG_DRWE;
+        break;
+    }
+    s->diagnostic_idx++;
+    return;
+
+disable:
+    s->diagnostic &= ~MPI_DIAG_DRWE;
+    s->diagnostic_idx = 0;
+}
+
+static int mptsas_hard_reset(MPTSASState *s)
+{
+    mptsas_soft_reset(s);
+
+    s->intr_mask = MPI_HIM_DIM | MPI_HIM_RIM;
+
+    s->host_mfa_high_addr = 0;
+    s->sense_buffer_high_addr = 0;
+    s->reply_frame_size = 0;
+    s->max_devices = MPTSAS_NUM_PORTS;
+    s->max_buses = 1;
+
+    return 0;
+}
+
+static void mptsas_interrupt_status_write(MPTSASState *s)
+{
+    switch (s->doorbell_state) {
+    case DOORBELL_NONE:
+    case DOORBELL_WRITE:
+        s->intr_status &= ~MPI_HIS_DOORBELL_INTERRUPT;
+        break;
+
+    case DOORBELL_READ:
+        /* The reply can be read continuously, so leave the interrupt up.  */
+        assert(s->intr_status & MPI_HIS_DOORBELL_INTERRUPT);
+        if (s->doorbell_reply_idx == s->doorbell_reply_size) {
+            s->doorbell_state = DOORBELL_NONE;
+        }
+        break;
+
+    default:
+        abort();
+    }
+    mptsas_update_interrupt(s);
+}
+
+static uint32_t mptsas_reply_post_read(MPTSASState *s)
+{
+    uint32_t ret;
+
+    if (!MPTSAS_FIFO_EMPTY(s, reply_post)) {
+        ret = MPTSAS_FIFO_GET(s, reply_post);
+    } else {
+        ret = -1;
+        s->intr_status &= ~MPI_HIS_REPLY_MESSAGE_INTERRUPT;
+        mptsas_update_interrupt(s);
+    }
+
+    return ret;
+}
+
+static uint64_t mptsas_mmio_read(void *opaque, hwaddr addr,
+                                  unsigned size)
+{
+    MPTSASState *s = opaque;
+    uint32_t ret = 0;
+
+    switch (addr & ~3) {
+    case MPI_DOORBELL_OFFSET:
+        ret = mptsas_doorbell_read(s);
+        break;
+
+    case MPI_DIAGNOSTIC_OFFSET:
+        ret = s->diagnostic;
+        break;
+
+    case MPI_HOST_INTERRUPT_STATUS_OFFSET:
+        ret = s->intr_status;
+        break;
+
+    case MPI_HOST_INTERRUPT_MASK_OFFSET:
+        ret = s->intr_mask;
+        break;
+
+    case MPI_REPLY_POST_FIFO_OFFSET:
+        ret = mptsas_reply_post_read(s);
+        break;
+
+    default:
+        trace_mptsas_mmio_unhandled_read(s, addr);
+        break;
+    }
+    trace_mptsas_mmio_read(s, addr, ret);
+    return ret;
+}
+
+static void mptsas_mmio_write(void *opaque, hwaddr addr,
+                               uint64_t val, unsigned size)
+{
+    MPTSASState *s = opaque;
+
+    trace_mptsas_mmio_write(s, addr, val);
+    switch (addr) {
+    case MPI_DOORBELL_OFFSET:
+        mptsas_doorbell_write(s, val);
+        break;
+
+    case MPI_WRITE_SEQUENCE_OFFSET:
+        mptsas_write_sequence_write(s, val);
+        break;
+
+    case MPI_DIAGNOSTIC_OFFSET:
+        if (val & MPI_DIAG_RESET_ADAPTER) {
+            mptsas_hard_reset(s);
+        }
+        break;
+
+    case MPI_HOST_INTERRUPT_STATUS_OFFSET:
+        mptsas_interrupt_status_write(s);
+        break;
+
+    case MPI_HOST_INTERRUPT_MASK_OFFSET:
+        s->intr_mask = val & (MPI_HIM_RIM | MPI_HIM_DIM);
+        mptsas_update_interrupt(s);
+        break;
+
+    case MPI_REQUEST_POST_FIFO_OFFSET:
+        if (MPTSAS_FIFO_FULL(s, request_post)) {
+            mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        } else {
+            MPTSAS_FIFO_PUT(s, request_post, val & ~0x03);
+            qemu_bh_schedule(s->request_bh);
+        }
+        break;
+
+    case MPI_REPLY_FREE_FIFO_OFFSET:
+        if (MPTSAS_FIFO_FULL(s, reply_free)) {
+            mptsas_set_fault(s, MPI_IOCSTATUS_INSUFFICIENT_RESOURCES);
+        } else {
+            MPTSAS_FIFO_PUT(s, reply_free, val);
+        }
+        break;
+
+    default:
+        trace_mptsas_mmio_unhandled_write(s, addr, val);
+        break;
+    }
+}
+
+static const MemoryRegionOps mptsas_mmio_ops = {
+    .read = mptsas_mmio_read,
+    .write = mptsas_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static const MemoryRegionOps mptsas_port_ops = {
+    .read = mptsas_mmio_read,
+    .write = mptsas_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static uint64_t mptsas_diag_read(void *opaque, hwaddr addr,
+                                   unsigned size)
+{
+    MPTSASState *s = opaque;
+    trace_mptsas_diag_read(s, addr, 0);
+    return 0;
+}
+
+static void mptsas_diag_write(void *opaque, hwaddr addr,
+                               uint64_t val, unsigned size)
+{
+    MPTSASState *s = opaque;
+    trace_mptsas_diag_write(s, addr, val);
+}
+
+static const MemoryRegionOps mptsas_diag_ops = {
+    .read = mptsas_diag_read,
+    .write = mptsas_diag_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    }
+};
+
+static QEMUSGList *mptsas_get_sg_list(SCSIRequest *sreq)
+{
+    MPTSASRequest *req = sreq->hba_private;
+
+    return &req->qsg;
+}
+
+static void mptsas_command_complete(SCSIRequest *sreq,
+        uint32_t status, size_t resid)
+{
+    MPTSASRequest *req = sreq->hba_private;
+    MPTSASState *s = req->dev;
+    uint8_t sense_buf[SCSI_SENSE_BUF_SIZE];
+    uint8_t sense_len;
+
+    hwaddr sense_buffer_addr = req->dev->sense_buffer_high_addr |
+            req->scsi_io.SenseBufferLowAddr;
+
+    trace_mptsas_command_complete(s, req->scsi_io.MsgContext, status, resid);
+
+    sense_len = scsi_req_get_sense(sreq, sense_buf, SCSI_SENSE_BUF_SIZE);
+    if (sense_len > 0) {
+        pci_dma_write(PCI_DEVICE(s), sense_buffer_addr, sense_buf,
+                      MIN(req->scsi_io.SenseBufferLength, sense_len));
+    }
+
+    if (sreq->status != GOOD || resid ||
+        req->dev->doorbell_state == DOORBELL_WRITE) {
+        MPIMsgSCSIIOReply reply;
+
+        memset(&reply, 0, sizeof(reply));
+        reply.TargetID          = req->scsi_io.TargetID;
+        reply.Bus               = req->scsi_io.Bus;
+        reply.MsgLength         = sizeof(reply) / 4;
+        reply.Function          = req->scsi_io.Function;
+        reply.CDBLength         = req->scsi_io.CDBLength;
+        reply.SenseBufferLength = req->scsi_io.SenseBufferLength;
+        reply.MsgFlags          = req->scsi_io.MsgFlags;
+        reply.MsgContext        = req->scsi_io.MsgContext;
+        reply.SCSIStatus        = sreq->status;
+        if (sreq->status == GOOD) {
+            reply.TransferCount = req->scsi_io.DataLength - resid;
+            if (resid) {
+                reply.IOCStatus     = MPI_IOCSTATUS_SCSI_DATA_UNDERRUN;
+            }
+        } else {
+            reply.SCSIState     = MPI_SCSI_STATE_AUTOSENSE_VALID;
+            reply.SenseCount    = sense_len;
+            reply.IOCStatus     = MPI_IOCSTATUS_SCSI_DATA_UNDERRUN;
+        }
+
+        mptsas_fix_scsi_io_reply_endianness(&reply);
+        mptsas_post_reply(req->dev, (MPIDefaultReply *)&reply);
+    } else {
+        mptsas_turbo_reply(req->dev, req->scsi_io.MsgContext);
+    }
+
+    mptsas_free_request(req);
+}
+
+static void mptsas_request_cancelled(SCSIRequest *sreq)
+{
+    MPTSASRequest *req = sreq->hba_private;
+    MPIMsgSCSIIOReply reply;
+
+    memset(&reply, 0, sizeof(reply));
+    reply.TargetID          = req->scsi_io.TargetID;
+    reply.Bus               = req->scsi_io.Bus;
+    reply.MsgLength         = sizeof(reply) / 4;
+    reply.Function          = req->scsi_io.Function;
+    reply.CDBLength         = req->scsi_io.CDBLength;
+    reply.SenseBufferLength = req->scsi_io.SenseBufferLength;
+    reply.MsgFlags          = req->scsi_io.MsgFlags;
+    reply.MsgContext        = req->scsi_io.MsgContext;
+    reply.SCSIState         = MPI_SCSI_STATE_NO_SCSI_STATUS;
+    reply.IOCStatus         = MPI_IOCSTATUS_SCSI_TASK_TERMINATED;
+
+    mptsas_fix_scsi_io_reply_endianness(&reply);
+    mptsas_post_reply(req->dev, (MPIDefaultReply *)&reply);
+    mptsas_free_request(req);
+}
+
+static void mptsas_save_request(QEMUFile *f, SCSIRequest *sreq)
+{
+    MPTSASRequest *req = sreq->hba_private;
+    int i;
+
+    qemu_put_buffer(f, (unsigned char *)&req->scsi_io, sizeof(req->scsi_io));
+    qemu_put_be32(f, req->qsg.nsg);
+    for (i = 0; i < req->qsg.nsg; i++) {
+        qemu_put_be64(f, req->qsg.sg[i].base);
+        qemu_put_be64(f, req->qsg.sg[i].len);
+    }
+}
+
+static void *mptsas_load_request(QEMUFile *f, SCSIRequest *sreq)
+{
+    SCSIBus *bus = sreq->bus;
+    MPTSASState *s = container_of(bus, MPTSASState, bus);
+    PCIDevice *pci = PCI_DEVICE(s);
+    MPTSASRequest *req;
+    int i, n;
+
+    req = g_new(MPTSASRequest, 1);
+    qemu_get_buffer(f, (unsigned char *)&req->scsi_io, sizeof(req->scsi_io));
+
+    n = qemu_get_be32(f);
+    /* TODO: add a way for SCSIBusInfo's load_request to fail,
+     * and fail migration instead of asserting here.
+     * When we do, we might be able to re-enable NDEBUG below.
+     */
+#ifdef NDEBUG
+#error building with NDEBUG is not supported
+#endif
+    assert(n >= 0);
+
+    pci_dma_sglist_init(&req->qsg, pci, n);
+    for (i = 0; i < n; i++) {
+        uint64_t base = qemu_get_be64(f);
+        uint64_t len = qemu_get_be64(f);
+        qemu_sglist_add(&req->qsg, base, len);
+    }
+
+    scsi_req_ref(sreq);
+    req->sreq = sreq;
+    req->dev = s;
+
+    return req;
+}
+
+static const struct SCSIBusInfo mptsas_scsi_info = {
+    .tcq = true,
+    .max_target = MPTSAS_NUM_PORTS,
+    .max_lun = 1,
+
+    .get_sg_list = mptsas_get_sg_list,
+    .complete = mptsas_command_complete,
+    .cancel = mptsas_request_cancelled,
+    .save_request = mptsas_save_request,
+    .load_request = mptsas_load_request,
+};
+
+static void mptsas_scsi_init(PCIDevice *dev, Error **errp)
+{
+    DeviceState *d = DEVICE(dev);
+    MPTSASState *s = MPT_SAS(dev);
+    Error *err = NULL;
+    int ret;
+
+    dev->config[PCI_LATENCY_TIMER] = 0;
+    dev->config[PCI_INTERRUPT_PIN] = 0x01;
+
+    if (s->msi != ON_OFF_AUTO_OFF) {
+        ret = msi_init(dev, 0, 1, true, false, &err);
+        /* Any error other than -ENOTSUP(board's MSI support is broken)
+         * is a programming error */
+        assert(!ret || ret == -ENOTSUP);
+        if (ret && s->msi == ON_OFF_AUTO_ON) {
+            /* Can't satisfy user's explicit msi=on request, fail */
+            error_append_hint(&err, "You have to use msi=auto (default) or "
+                    "msi=off with this machine type.\n");
+            error_propagate(errp, err);
+            return;
+        }
+        assert(!err || s->msi == ON_OFF_AUTO_AUTO);
+        /* With msi=auto, we fall back to MSI off silently */
+        error_free(err);
+
+        /* Only used for migration.  */
+        s->msi_in_use = (ret == 0);
+    }
+
+    memory_region_init_io(&s->mmio_io, OBJECT(s), &mptsas_mmio_ops, s,
+                          "mptsas-mmio", 0x4000);
+    memory_region_init_io(&s->port_io, OBJECT(s), &mptsas_port_ops, s,
+                          "mptsas-io", 256);
+    memory_region_init_io(&s->diag_io, OBJECT(s), &mptsas_diag_ops, s,
+                          "mptsas-diag", 0x10000);
+
+    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->port_io);
+    pci_register_bar(dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY |
+                                 PCI_BASE_ADDRESS_MEM_TYPE_32, &s->mmio_io);
+    pci_register_bar(dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY |
+                                 PCI_BASE_ADDRESS_MEM_TYPE_32, &s->diag_io);
+
+    if (!s->sas_addr) {
+        s->sas_addr = ((NAA_LOCALLY_ASSIGNED_ID << 24) |
+                       IEEE_COMPANY_LOCALLY_ASSIGNED) << 36;
+        s->sas_addr |= (pci_bus_num(dev->bus) << 16);
+        s->sas_addr |= (PCI_SLOT(dev->devfn) << 8);
+        s->sas_addr |= PCI_FUNC(dev->devfn);
+    }
+    s->max_devices = MPTSAS_NUM_PORTS;
+
+    s->request_bh = qemu_bh_new(mptsas_fetch_requests, s);
+
+    QTAILQ_INIT(&s->pending);
+
+    scsi_bus_new(&s->bus, sizeof(s->bus), &dev->qdev, &mptsas_scsi_info, NULL);
+    if (!d->hotplugged) {
+        scsi_bus_legacy_handle_cmdline(&s->bus, errp);
+    }
+}
+
+static void mptsas_scsi_uninit(PCIDevice *dev)
+{
+    MPTSASState *s = MPT_SAS(dev);
+
+    qemu_bh_delete(s->request_bh);
+    msi_uninit(dev);
+}
+
+static void mptsas_reset(DeviceState *dev)
+{
+    MPTSASState *s = MPT_SAS(dev);
+
+    mptsas_hard_reset(s);
+}
+
+static int mptsas_post_load(void *opaque, int version_id)
+{
+    MPTSASState *s = opaque;
+
+    if (s->doorbell_idx > s->doorbell_cnt ||
+        s->doorbell_cnt > ARRAY_SIZE(s->doorbell_msg) ||
+        s->doorbell_reply_idx > s->doorbell_reply_size ||
+        s->doorbell_reply_size > ARRAY_SIZE(s->doorbell_reply) ||
+        MPTSAS_FIFO_INVALID(s, request_post) ||
+        MPTSAS_FIFO_INVALID(s, reply_post) ||
+        MPTSAS_FIFO_INVALID(s, reply_free) ||
+        s->diagnostic_idx > 4) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_mptsas = {
+    .name = "mptsas",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .minimum_version_id_old = 0,
+    .post_load = mptsas_post_load,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, MPTSASState),
+        VMSTATE_BOOL(msi_in_use, MPTSASState),
+        VMSTATE_UINT32(state, MPTSASState),
+        VMSTATE_UINT8(who_init, MPTSASState),
+        VMSTATE_UINT8(doorbell_state, MPTSASState),
+        VMSTATE_UINT32_ARRAY(doorbell_msg, MPTSASState, 256),
+        VMSTATE_INT32(doorbell_idx, MPTSASState),
+        VMSTATE_INT32(doorbell_cnt, MPTSASState),
+
+        VMSTATE_UINT16_ARRAY(doorbell_reply, MPTSASState, 256),
+        VMSTATE_INT32(doorbell_reply_idx, MPTSASState),
+        VMSTATE_INT32(doorbell_reply_size, MPTSASState),
+
+        VMSTATE_UINT32(diagnostic, MPTSASState),
+        VMSTATE_UINT8(diagnostic_idx, MPTSASState),
+
+        VMSTATE_UINT32(intr_status, MPTSASState),
+        VMSTATE_UINT32(intr_mask, MPTSASState),
+
+        VMSTATE_UINT32_ARRAY(request_post, MPTSASState,
+                             MPTSAS_REQUEST_QUEUE_DEPTH + 1),
+        VMSTATE_UINT16(request_post_head, MPTSASState),
+        VMSTATE_UINT16(request_post_tail, MPTSASState),
+
+        VMSTATE_UINT32_ARRAY(reply_post, MPTSASState,
+                             MPTSAS_REPLY_QUEUE_DEPTH + 1),
+        VMSTATE_UINT16(reply_post_head, MPTSASState),
+        VMSTATE_UINT16(reply_post_tail, MPTSASState),
+
+        VMSTATE_UINT32_ARRAY(reply_free, MPTSASState,
+                             MPTSAS_REPLY_QUEUE_DEPTH + 1),
+        VMSTATE_UINT16(reply_free_head, MPTSASState),
+        VMSTATE_UINT16(reply_free_tail, MPTSASState),
+
+        VMSTATE_UINT16(max_buses, MPTSASState),
+        VMSTATE_UINT16(max_devices, MPTSASState),
+        VMSTATE_UINT16(reply_frame_size, MPTSASState),
+        VMSTATE_UINT64(host_mfa_high_addr, MPTSASState),
+        VMSTATE_UINT64(sense_buffer_high_addr, MPTSASState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property mptsas_properties[] = {
+    DEFINE_PROP_UINT64("sas_address", MPTSASState, sas_addr, 0),
+    /* TODO: test MSI support under Windows */
+    DEFINE_PROP_ON_OFF_AUTO("msi", MPTSASState, msi, ON_OFF_AUTO_AUTO),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void mptsas1068_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc);
+
+    pc->realize = mptsas_scsi_init;
+    pc->exit = mptsas_scsi_uninit;
+    pc->romfile = 0;
+    pc->vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
+    pc->device_id = PCI_DEVICE_ID_LSI_SAS1068;
+    pc->subsystem_vendor_id = PCI_VENDOR_ID_LSI_LOGIC;
+    pc->subsystem_id = 0x8000;
+    pc->class_id = PCI_CLASS_STORAGE_SCSI;
+    dc->props = mptsas_properties;
+    dc->reset = mptsas_reset;
+    dc->vmsd = &vmstate_mptsas;
+    dc->desc = "LSI SAS 1068";
+}
+
+static const TypeInfo mptsas_info = {
+    .name = TYPE_MPTSAS1068,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(MPTSASState),
+    .class_init = mptsas1068_class_init,
+};
+
+static void mptsas_register_types(void)
+{
+    type_register(&mptsas_info);
+}
+
+type_init(mptsas_register_types)
diff --git a/hw/scsi/mptsas.h b/hw/scsi/mptsas.h
new file mode 100644
index 0000000..0436a33
--- /dev/null
+++ b/hw/scsi/mptsas.h
@@ -0,0 +1,101 @@
+#ifndef MPTSAS_H
+#define MPTSAS_H
+
+#include "mpi.h"
+
+#define MPTSAS_NUM_PORTS 8
+#define MPTSAS_MAX_FRAMES 2048     /* Firmware limit at 65535 */
+
+#define MPTSAS_REQUEST_QUEUE_DEPTH 128
+#define MPTSAS_REPLY_QUEUE_DEPTH   128
+
+#define MPTSAS_MAXIMUM_CHAIN_DEPTH 0x22
+
+typedef struct MPTSASState MPTSASState;
+typedef struct MPTSASRequest MPTSASRequest;
+
+enum {
+    DOORBELL_NONE,
+    DOORBELL_WRITE,
+    DOORBELL_READ
+};
+
+struct MPTSASState {
+    PCIDevice dev;
+    MemoryRegion mmio_io;
+    MemoryRegion port_io;
+    MemoryRegion diag_io;
+    QEMUBH *request_bh;
+
+    /* properties */
+    OnOffAuto msi;
+    uint64_t sas_addr;
+
+    bool msi_in_use;
+
+    /* Doorbell register */
+    uint32_t state;
+    uint8_t who_init;
+    uint8_t doorbell_state;
+
+    /* Buffer for requests that are sent through the doorbell register.  */
+    uint32_t doorbell_msg[256];
+    int doorbell_idx;
+    int doorbell_cnt;
+
+    uint16_t doorbell_reply[256];
+    int doorbell_reply_idx;
+    int doorbell_reply_size;
+
+    /* Other registers */
+    uint8_t diagnostic_idx;
+    uint32_t diagnostic;
+    uint32_t intr_mask;
+    uint32_t intr_status;
+
+    /* Request queues */
+    uint32_t request_post[MPTSAS_REQUEST_QUEUE_DEPTH + 1];
+    uint16_t request_post_head;
+    uint16_t request_post_tail;
+
+    uint32_t reply_post[MPTSAS_REPLY_QUEUE_DEPTH + 1];
+    uint16_t reply_post_head;
+    uint16_t reply_post_tail;
+
+    uint32_t reply_free[MPTSAS_REPLY_QUEUE_DEPTH + 1];
+    uint16_t reply_free_head;
+    uint16_t reply_free_tail;
+
+    /* IOC Facts */
+    hwaddr host_mfa_high_addr;
+    hwaddr sense_buffer_high_addr;
+    uint16_t max_devices;
+    uint16_t max_buses;
+    uint16_t reply_frame_size;
+
+    SCSIBus bus;
+    QTAILQ_HEAD(, MPTSASRequest) pending;
+};
+
+void mptsas_fix_scsi_io_endianness(MPIMsgSCSIIORequest *req);
+void mptsas_fix_scsi_io_reply_endianness(MPIMsgSCSIIOReply *reply);
+void mptsas_fix_scsi_task_mgmt_endianness(MPIMsgSCSITaskMgmt *req);
+void mptsas_fix_scsi_task_mgmt_reply_endianness(MPIMsgSCSITaskMgmtReply *reply);
+void mptsas_fix_ioc_init_endianness(MPIMsgIOCInit *req);
+void mptsas_fix_ioc_init_reply_endianness(MPIMsgIOCInitReply *reply);
+void mptsas_fix_ioc_facts_endianness(MPIMsgIOCFacts *req);
+void mptsas_fix_ioc_facts_reply_endianness(MPIMsgIOCFactsReply *reply);
+void mptsas_fix_config_endianness(MPIMsgConfig *req);
+void mptsas_fix_config_reply_endianness(MPIMsgConfigReply *reply);
+void mptsas_fix_port_facts_endianness(MPIMsgPortFacts *req);
+void mptsas_fix_port_facts_reply_endianness(MPIMsgPortFactsReply *reply);
+void mptsas_fix_port_enable_endianness(MPIMsgPortEnable *req);
+void mptsas_fix_port_enable_reply_endianness(MPIMsgPortEnableReply *reply);
+void mptsas_fix_event_notification_endianness(MPIMsgEventNotify *req);
+void mptsas_fix_event_notification_reply_endianness(MPIMsgEventNotifyReply *reply);
+
+void mptsas_reply(MPTSASState *s, MPIDefaultReply *reply);
+
+void mptsas_process_config(MPTSASState *s, MPIMsgConfig *req);
+
+#endif /* MPTSAS_H */
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index 24f7b74..297216d 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -1,4 +1,6 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
@@ -7,6 +9,7 @@
 #include "sysemu/blockdev.h"
 #include "trace.h"
 #include "sysemu/dma.h"
+#include "qemu/cutils.h"
 
 static char *scsibus_get_dev_path(DeviceState *dev);
 static char *scsibus_get_fw_dev_path(DeviceState *dev);
@@ -136,7 +139,8 @@
         return;
     }
     if (!s->bh) {
-        s->bh = qemu_bh_new(scsi_dma_restart_bh, s);
+        AioContext *ctx = blk_get_aio_context(s->conf.blk);
+        s->bh = aio_bh_new(ctx, scsi_dma_restart_bh, s);
         qemu_bh_schedule(s->bh);
     }
 }
@@ -221,11 +225,16 @@
                                       const char *serial, Error **errp)
 {
     const char *driver;
+    char *name;
     DeviceState *dev;
     Error *err = NULL;
 
     driver = blk_is_sg(blk) ? "scsi-generic" : "scsi-disk";
     dev = qdev_create(&bus->qbus, driver);
+    name = g_strdup_printf("legacy[%d]", unit);
+    object_property_add_child(OBJECT(bus), name, OBJECT(dev), NULL);
+    g_free(name);
+
     qdev_prop_set_uint32(dev, "scsi-id", unit);
     if (bootindex >= 0) {
         object_property_set_int(OBJECT(dev), bootindex, "bootindex",
@@ -237,8 +246,9 @@
     if (serial && object_property_find(OBJECT(dev), "serial", NULL)) {
         qdev_prop_set_string(dev, "serial", serial);
     }
-    if (qdev_prop_set_drive(dev, "drive", blk) < 0) {
-        error_setg(errp, "Setting drive property failed");
+    qdev_prop_set_drive(dev, "drive", blk, &err);
+    if (err) {
+        error_propagate(errp, err);
         object_unparent(OBJECT(dev));
         return NULL;
     }
@@ -268,7 +278,6 @@
         scsi_bus_legacy_add_drive(bus, blk_by_legacy_dinfo(dinfo),
                                   unit, false, -1, NULL, &err);
         if (err != NULL) {
-            error_report("%s", error_get_pretty(err));
             error_propagate(errp, err);
             break;
         }
@@ -447,11 +456,19 @@
         r->buf[7] = 0x10 | (r->req.bus->info->tcq ? 0x02 : 0); /* Sync, TCQ.  */
         memcpy(&r->buf[8], "QEMU    ", 8);
         memcpy(&r->buf[16], "QEMU TARGET     ", 16);
-        pstrcpy((char *) &r->buf[32], 4, qemu_get_version());
+        pstrcpy((char *) &r->buf[32], 4, qemu_hw_version());
     }
     return true;
 }
 
+static size_t scsi_sense_len(SCSIRequest *req)
+{
+    if (req->dev->type == TYPE_SCANNER)
+        return SCSI_SENSE_LEN_SCANNER;
+    else
+        return SCSI_SENSE_LEN;
+}
+
 static int32_t scsi_target_send_command(SCSIRequest *req, uint8_t *buf)
 {
     SCSITargetReq *r = DO_UPCAST(SCSITargetReq, req, req);
@@ -468,7 +485,7 @@
         }
         break;
     case REQUEST_SENSE:
-        scsi_target_alloc_buf(&r->req, SCSI_SENSE_LEN);
+        scsi_target_alloc_buf(&r->req, scsi_sense_len(req));
         r->len = scsi_device_get_sense(r->req.dev, r->buf,
                                        MIN(req->cmd.xfer, r->buf_len),
                                        (req->cmd.buf[1] & 1) == 0);
@@ -552,7 +569,7 @@
     const int memset_off = offsetof(SCSIRequest, sense)
                            + sizeof(req->sense);
 
-    req = g_slice_alloc(reqops->size);
+    req = g_malloc(reqops->size);
     memset((uint8_t *)req + memset_off, 0, reqops->size - memset_off);
     req->refcount = 1;
     req->bus = bus;
@@ -982,7 +999,6 @@
         }
         /* fall through */
     case READ_10:
-    case RECOVER_BUFFERED_DATA:
     case READ_12:
     case READ_16:
         cmd->xfer *= dev->blocksize;
@@ -1124,6 +1140,29 @@
     return 0;
 }
 
+static int scsi_req_scanner_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+{
+    switch (buf[0]) {
+    /* Scanner commands */
+    case OBJECT_POSITION:
+        cmd->xfer = 0;
+        break;
+    case SCAN:
+        cmd->xfer = buf[4];
+        break;
+    case READ_10:
+    case SEND:
+    case GET_WINDOW:
+    case SET_WINDOW:
+        cmd->xfer = buf[8] | (buf[7] << 8) | (buf[6] << 16);
+        break;
+    default:
+        /* GET_DATA_BUFFER_STATUS xfer handled by scsi_req_xfer */
+        return scsi_req_xfer(cmd, dev, buf);
+    }
+
+    return 0;
+}
 
 static void scsi_cmd_xfer_mode(SCSICommand *cmd)
 {
@@ -1170,6 +1209,11 @@
     case SEND_DVD_STRUCTURE:
     case PERSISTENT_RESERVE_OUT:
     case MAINTENANCE_OUT:
+    case SET_WINDOW:
+    case SCAN:
+        /* SCAN conflicts with START_STOP.  START_STOP has cmd->xfer set to 0 for
+         * non-scanner devices, so we only get here for SCAN and not for START_STOP.
+         */
         cmd->mode = SCSI_XFER_TO_DEV;
         break;
     case ATA_PASSTHROUGH_12:
@@ -1234,10 +1278,15 @@
 int scsi_req_parse_cdb(SCSIDevice *dev, SCSICommand *cmd, uint8_t *buf)
 {
     int rc;
+    int len;
 
     cmd->lba = -1;
-    cmd->len = scsi_cdb_length(buf);
+    len = scsi_cdb_length(buf);
+    if (len < 0) {
+        return -1;
+    }
 
+    cmd->len = len;
     switch (dev->type) {
     case TYPE_TAPE:
         rc = scsi_req_stream_xfer(cmd, dev, buf);
@@ -1245,6 +1294,9 @@
     case TYPE_MEDIUM_CHANGER:
         rc = scsi_req_medium_changer_xfer(cmd, dev, buf);
         break;
+    case TYPE_SCANNER:
+        rc = scsi_req_scanner_length(cmd, dev, buf);
+        break;
     default:
         rc = scsi_req_xfer(cmd, dev, buf);
         break;
@@ -1611,7 +1663,7 @@
         }
         object_unref(OBJECT(req->dev));
         object_unref(OBJECT(qbus->parent));
-        g_slice_free1(req->ops->size, req);
+        g_free(req);
     }
 }
 
@@ -1749,13 +1801,21 @@
         notifier_list_add(&req->cancel_notifiers, notifier);
     }
     if (req->io_canceled) {
+        /* A blk_aio_cancel_async is pending; when it finishes,
+         * scsi_req_cancel_complete will be called and will
+         * call the notifier we just added.  Just wait for that.
+         */
+        assert(req->aiocb);
         return;
     }
+    /* Dropped in scsi_req_cancel_complete.  */
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->io_canceled = true;
     if (req->aiocb) {
         blk_aio_cancel_async(req->aiocb);
+    } else {
+        scsi_req_cancel_complete(req);
     }
 }
 
@@ -1765,11 +1825,15 @@
     if (!req->enqueued) {
         return;
     }
+    assert(!req->io_canceled);
+    /* Dropped in scsi_req_cancel_complete.  */
     scsi_req_ref(req);
     scsi_req_dequeue(req);
     req->io_canceled = true;
     if (req->aiocb) {
         blk_aio_cancel(req->aiocb);
+    } else {
+        scsi_req_cancel_complete(req);
     }
 }
 
@@ -1826,17 +1890,19 @@
 {
     SCSIRequest *req;
 
+    aio_context_acquire(blk_get_aio_context(sdev->conf.blk));
     while (!QTAILQ_EMPTY(&sdev->requests)) {
         req = QTAILQ_FIRST(&sdev->requests);
-        scsi_req_cancel(req);
+        scsi_req_cancel_async(req, NULL);
     }
-
+    blk_drain(sdev->conf.blk);
+    aio_context_release(blk_get_aio_context(sdev->conf.blk));
     scsi_device_set_ua(sdev, sense);
 }
 
 static char *scsibus_get_dev_path(DeviceState *dev)
 {
-    SCSIDevice *d = DO_UPCAST(SCSIDevice, qdev, dev);
+    SCSIDevice *d = SCSI_DEVICE(dev);
     DeviceState *hba = dev->parent_bus->parent;
     char *id;
     char *path;
@@ -1959,6 +2025,7 @@
     .name = "SCSIDevice/sense",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = scsi_sense_state_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8_SUB_ARRAY(sense, SCSIDevice,
                                 SCSI_SENSE_BUF_SIZE_OLD,
@@ -1989,13 +2056,9 @@
         },
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_scsi_sense_state,
-            .needed = scsi_sense_state_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_scsi_sense_state,
+        NULL
     }
 };
 
@@ -2012,7 +2075,7 @@
 static void scsi_dev_instance_init(Object *obj)
 {
     DeviceState *dev = DEVICE(obj);
-    SCSIDevice *s = DO_UPCAST(SCSIDevice, qdev, dev);
+    SCSIDevice *s = SCSI_DEVICE(dev);
 
     device_add_bootindex_property(obj, &s->conf.bootindex,
                                   "bootindex", NULL,
diff --git a/hw/scsi/scsi-disk.c b/hw/scsi/scsi-disk.c
index 506bd34..edae532 100644
--- a/hw/scsi/scsi-disk.c
+++ b/hw/scsi/scsi-disk.c
@@ -28,7 +28,8 @@
 #define DPRINTF(fmt, ...) do {} while(0)
 #endif
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
@@ -37,6 +38,7 @@
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "sysemu/dma.h"
+#include "qemu/cutils.h"
 
 #ifdef __linux
 #include <scsi/sg.h>
@@ -49,8 +51,23 @@
 
 #define DEFAULT_DISCARD_GRANULARITY 4096
 #define DEFAULT_MAX_UNMAP_SIZE      (1 << 30)   /* 1 GB */
+#define DEFAULT_MAX_IO_SIZE         INT_MAX     /* 2 GB - 1 block */
 
-typedef struct SCSIDiskState SCSIDiskState;
+#define TYPE_SCSI_DISK_BASE         "scsi-disk-base"
+
+#define SCSI_DISK_BASE(obj) \
+     OBJECT_CHECK(SCSIDiskState, (obj), TYPE_SCSI_DISK_BASE)
+#define SCSI_DISK_BASE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(SCSIDiskClass, (klass), TYPE_SCSI_DISK_BASE)
+#define SCSI_DISK_BASE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(SCSIDiskClass, (obj), TYPE_SCSI_DISK_BASE)
+
+typedef struct SCSIDiskClass {
+    SCSIDeviceClass parent_class;
+    DMAIOFunc       *dma_readv;
+    DMAIOFunc       *dma_writev;
+    bool            (*need_fua_emulation)(SCSICommand *cmd);
+} SCSIDiskClass;
 
 typedef struct SCSIDiskReq {
     SCSIRequest req;
@@ -59,26 +76,27 @@
     uint32_t sector_count;
     uint32_t buflen;
     bool started;
+    bool need_fua_emulation;
     struct iovec iov;
     QEMUIOVector qiov;
     BlockAcctCookie acct;
+    unsigned char *status;
 } SCSIDiskReq;
 
 #define SCSI_DISK_F_REMOVABLE             0
 #define SCSI_DISK_F_DPOFUA                1
 #define SCSI_DISK_F_NO_REMOVABLE_DEVOPS   2
 
-struct SCSIDiskState
+typedef struct SCSIDiskState
 {
     SCSIDevice qdev;
     uint32_t features;
     bool media_changed;
     bool media_event;
     bool eject_request;
-    uint64_t wwn;
-    uint64_t port_wwn;
     uint16_t port_index;
     uint64_t max_unmap_size;
+    uint64_t max_io_size;
     QEMUBH *bh;
     char *version;
     char *serial;
@@ -86,9 +104,9 @@
     char *product;
     bool tray_open;
     bool tray_locked;
-};
+} SCSIDiskState;
 
-static int scsi_handle_rw_error(SCSIDiskReq *r, int error);
+static int scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed);
 
 static void scsi_free_request(SCSIRequest *req)
 {
@@ -106,7 +124,7 @@
     scsi_req_complete(&r->req, CHECK_CONDITION);
 }
 
-static uint32_t scsi_init_iovec(SCSIDiskReq *r, size_t size)
+static void scsi_init_iovec(SCSIDiskReq *r, size_t size)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
@@ -116,7 +134,6 @@
     }
     r->iov.iov_len = MIN(r->sector_count * 512, r->buflen);
     qemu_iovec_init_external(&r->qiov, &r->iov, 1);
-    return r->qiov.size / 512;
 }
 
 static void scsi_disk_save_request(QEMUFile *f, SCSIRequest *req)
@@ -160,6 +177,29 @@
     qemu_iovec_init_external(&r->qiov, &r->iov, 1);
 }
 
+static bool scsi_disk_req_check_error(SCSIDiskReq *r, int ret, bool acct_failed)
+{
+    if (r->req.io_canceled) {
+        scsi_req_cancel_complete(&r->req);
+        return true;
+    }
+
+    if (ret < 0) {
+        return scsi_handle_rw_error(r, -ret, acct_failed);
+    }
+
+    if (r->status && *r->status) {
+        if (acct_failed) {
+            SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+            block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+        }
+        scsi_req_complete(&r->req, *r->status);
+        return true;
+    }
+
+    return false;
+}
+
 static void scsi_aio_complete(void *opaque, int ret)
 {
     SCSIDiskReq *r = (SCSIDiskReq *)opaque;
@@ -167,18 +207,11 @@
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    if (scsi_disk_req_check_error(r, ret, true)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
-
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     scsi_req_complete(&r->req, GOOD);
 
 done:
@@ -215,12 +248,10 @@
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
-        goto done;
-    }
+    assert(r->req.aiocb == NULL);
+    assert(!r->req.io_canceled);
 
-    if (scsi_is_cmd_fua(&r->req.cmd)) {
+    if (r->need_fua_emulation) {
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
                          BLOCK_ACCT_FLUSH);
         r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_aio_complete, r);
@@ -228,31 +259,16 @@
     }
 
     scsi_req_complete(&r->req, GOOD);
-
-done:
     scsi_req_unref(&r->req);
 }
 
-static void scsi_dma_complete_noio(void *opaque, int ret)
+static void scsi_dma_complete_noio(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
-
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    assert(r->req.aiocb == NULL);
+    if (scsi_disk_req_check_error(r, ret, false)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
-
     r->sector += r->sector_count;
     r->sector_count = 0;
     if (r->req.cmd.mode == SCSI_XFER_TO_DEV) {
@@ -269,10 +285,17 @@
 static void scsi_dma_complete(void *opaque, int ret)
 {
     SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
 
     assert(r->req.aiocb != NULL);
-    (void)r;
-    scsi_dma_complete_noio(opaque, ret);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_dma_complete_noio(r, ret);
 }
 
 static void scsi_read_complete(void * opaque, int ret)
@@ -283,18 +306,11 @@
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    if (scsi_disk_req_check_error(r, ret, true)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
-
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
     DPRINTF("Data ready tag=0x%x len=%zd\n", r->req.tag, r->qiov.size);
 
     n = r->qiov.size / 512;
@@ -307,47 +323,54 @@
 }
 
 /* Actually issue a read to the block device.  */
-static void scsi_do_read(void *opaque, int ret)
+static void scsi_do_read(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = opaque;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
-    uint32_t n;
+    SCSIDiskClass *sdc = (SCSIDiskClass *) object_get_class(OBJECT(s));
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    assert (r->req.aiocb == NULL);
+    if (scsi_disk_req_check_error(r, ret, false)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
-
     /* The request is used as the AIO opaque value, so add a ref.  */
     scsi_req_ref(&r->req);
 
     if (r->req.sg) {
         dma_acct_start(s->qdev.conf.blk, &r->acct, r->req.sg, BLOCK_ACCT_READ);
         r->req.resid -= r->req.sg->size;
-        r->req.aiocb = dma_blk_read(s->qdev.conf.blk, r->req.sg, r->sector,
-                                    scsi_dma_complete, r);
+        r->req.aiocb = dma_blk_io(blk_get_aio_context(s->qdev.conf.blk),
+                                  r->req.sg, r->sector << BDRV_SECTOR_BITS,
+                                  sdc->dma_readv, r, scsi_dma_complete, r,
+                                  DMA_DIRECTION_FROM_DEVICE);
     } else {
-        n = scsi_init_iovec(r, SCSI_DMA_BUF_SIZE);
+        scsi_init_iovec(r, SCSI_DMA_BUF_SIZE);
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
-                         n * BDRV_SECTOR_SIZE, BLOCK_ACCT_READ);
-        r->req.aiocb = blk_aio_readv(s->qdev.conf.blk, r->sector, &r->qiov, n,
-                                     scsi_read_complete, r);
+                         r->qiov.size, BLOCK_ACCT_READ);
+        r->req.aiocb = sdc->dma_readv(r->sector << BDRV_SECTOR_BITS, &r->qiov,
+                                      scsi_read_complete, r, r);
     }
 
 done:
     scsi_req_unref(&r->req);
 }
 
+static void scsi_do_read_cb(void *opaque, int ret)
+{
+    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+
+    assert (r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_do_read(opaque, ret);
+}
+
 /* Read more data from scsi device into buffer.  */
 static void scsi_read_data(SCSIRequest *req)
 {
@@ -380,10 +403,10 @@
 
     first = !r->started;
     r->started = true;
-    if (first && scsi_is_cmd_fua(&r->req.cmd)) {
+    if (first && r->need_fua_emulation) {
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct, 0,
                          BLOCK_ACCT_FLUSH);
-        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read, r);
+        r->req.aiocb = blk_aio_flush(s->qdev.conf.blk, scsi_do_read_cb, r);
     } else {
         scsi_do_read(r, 0);
     }
@@ -396,14 +419,17 @@
  * scsi_handle_rw_error always manages its reference counts, independent
  * of the return value.
  */
-static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
+static int scsi_handle_rw_error(SCSIDiskReq *r, int error, bool acct_failed)
 {
-    bool is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
+    bool is_read = (r->req.cmd.mode == SCSI_XFER_FROM_DEV);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     BlockErrorAction action = blk_get_error_action(s->qdev.conf.blk,
                                                    is_read, error);
 
     if (action == BLOCK_ERROR_ACTION_REPORT) {
+        if (acct_failed) {
+            block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+        }
         switch (error) {
         case ENOMEDIUM:
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
@@ -429,27 +455,15 @@
     return action != BLOCK_ERROR_ACTION_IGNORE;
 }
 
-static void scsi_write_complete(void * opaque, int ret)
+static void scsi_write_complete_noio(SCSIDiskReq *r, int ret)
 {
-    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
-    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint32_t n;
 
-    if (r->req.aiocb != NULL) {
-        r->req.aiocb = NULL;
-        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    }
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    assert (r->req.aiocb == NULL);
+    if (scsi_disk_req_check_error(r, ret, false)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
-
     n = r->qiov.size / 512;
     r->sector += n;
     r->sector_count -= n;
@@ -466,11 +480,27 @@
     scsi_req_unref(&r->req);
 }
 
+static void scsi_write_complete(void * opaque, int ret)
+{
+    SCSIDiskReq *r = (SCSIDiskReq *)opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+
+    assert (r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    if (ret < 0) {
+        block_acct_failed(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    } else {
+        block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
+    }
+    scsi_write_complete_noio(r, ret);
+}
+
 static void scsi_write_data(SCSIRequest *req)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
-    uint32_t n;
+    SCSIDiskClass *sdc = (SCSIDiskClass *) object_get_class(OBJECT(s));
 
     /* No data transfer may already be in progress */
     assert(r->req.aiocb == NULL);
@@ -479,18 +509,18 @@
     scsi_req_ref(&r->req);
     if (r->req.cmd.mode != SCSI_XFER_TO_DEV) {
         DPRINTF("Data transfer direction invalid\n");
-        scsi_write_complete(r, -EINVAL);
+        scsi_write_complete_noio(r, -EINVAL);
         return;
     }
 
     if (!r->req.sg && !r->qiov.size) {
         /* Called for the first time.  Ask the driver to send us more data.  */
         r->started = true;
-        scsi_write_complete(r, 0);
+        scsi_write_complete_noio(r, 0);
         return;
     }
     if (s->tray_open) {
-        scsi_write_complete(r, -ENOMEDIUM);
+        scsi_write_complete_noio(r, -ENOMEDIUM);
         return;
     }
 
@@ -499,7 +529,7 @@
         if (r->req.sg) {
             scsi_dma_complete_noio(r, 0);
         } else {
-            scsi_write_complete(r, 0);
+            scsi_write_complete_noio(r, 0);
         }
         return;
     }
@@ -507,14 +537,15 @@
     if (r->req.sg) {
         dma_acct_start(s->qdev.conf.blk, &r->acct, r->req.sg, BLOCK_ACCT_WRITE);
         r->req.resid -= r->req.sg->size;
-        r->req.aiocb = dma_blk_write(s->qdev.conf.blk, r->req.sg, r->sector,
-                                     scsi_dma_complete, r);
+        r->req.aiocb = dma_blk_io(blk_get_aio_context(s->qdev.conf.blk),
+                                  r->req.sg, r->sector << BDRV_SECTOR_BITS,
+                                  sdc->dma_writev, r, scsi_dma_complete, r,
+                                  DMA_DIRECTION_TO_DEVICE);
     } else {
-        n = r->qiov.size / 512;
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
-                         n * BDRV_SECTOR_SIZE, BLOCK_ACCT_WRITE);
-        r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, r->sector, &r->qiov, n,
-                                      scsi_write_complete, r);
+                         r->qiov.size, BLOCK_ACCT_WRITE);
+        r->req.aiocb = sdc->dma_writev(r->sector << BDRV_SECTOR_BITS, &r->qiov,
+                                       scsi_write_complete, r, r);
     }
 }
 
@@ -598,21 +629,21 @@
             memcpy(outbuf+buflen, str, id_len);
             buflen += id_len;
 
-            if (s->wwn) {
+            if (s->qdev.wwn) {
                 outbuf[buflen++] = 0x1; // Binary
                 outbuf[buflen++] = 0x3; // NAA
                 outbuf[buflen++] = 0;   // reserved
                 outbuf[buflen++] = 8;
-                stq_be_p(&outbuf[buflen], s->wwn);
+                stq_be_p(&outbuf[buflen], s->qdev.wwn);
                 buflen += 8;
             }
 
-            if (s->port_wwn) {
+            if (s->qdev.port_wwn) {
                 outbuf[buflen++] = 0x61; // SAS / Binary
                 outbuf[buflen++] = 0x93; // PIV / Target port / NAA
                 outbuf[buflen++] = 0;    // reserved
                 outbuf[buflen++] = 8;
-                stq_be_p(&outbuf[buflen], s->port_wwn);
+                stq_be_p(&outbuf[buflen], s->qdev.port_wwn);
                 buflen += 8;
             }
 
@@ -636,6 +667,8 @@
                     s->qdev.conf.opt_io_size / s->qdev.blocksize;
             unsigned int max_unmap_sectors =
                     s->max_unmap_size / s->qdev.blocksize;
+            unsigned int max_io_sectors =
+                    s->max_io_size / s->qdev.blocksize;
 
             if (s->qdev.type == TYPE_ROM) {
                 DPRINTF("Inquiry (EVPD[%02X] not supported for CDROM\n",
@@ -652,6 +685,12 @@
             outbuf[6] = (min_io_size >> 8) & 0xff;
             outbuf[7] = min_io_size & 0xff;
 
+            /* maximum transfer length */
+            outbuf[8] = (max_io_sectors >> 24) & 0xff;
+            outbuf[9] = (max_io_sectors >> 16) & 0xff;
+            outbuf[10] = (max_io_sectors >> 8) & 0xff;
+            outbuf[11] = max_io_sectors & 0xff;
+
             /* optimal transfer length */
             outbuf[12] = (opt_io_size >> 24) & 0xff;
             outbuf[13] = (opt_io_size >> 16) & 0xff;
@@ -675,6 +714,17 @@
             outbuf[29] = (unmap_sectors >> 16) & 0xff;
             outbuf[30] = (unmap_sectors >> 8) & 0xff;
             outbuf[31] = unmap_sectors & 0xff;
+
+            /* max write same size */
+            outbuf[36] = 0;
+            outbuf[37] = 0;
+            outbuf[38] = 0;
+            outbuf[39] = 0;
+
+            outbuf[40] = (max_io_sectors >> 24) & 0xff;
+            outbuf[41] = (max_io_sectors >> 16) & 0xff;
+            outbuf[42] = (max_io_sectors >> 8) & 0xff;
+            outbuf[43] = max_io_sectors & 0xff;
             break;
         }
         case 0xb2: /* thin provisioning */
@@ -745,6 +795,9 @@
     if (!blk_is_inserted(s->qdev.conf.blk)) {
         return false;
     }
+    if (s->tray_open) {
+        return false;
+    }
     blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     return nb_sectors > CD_MAX_SECTORS;
 }
@@ -758,6 +811,9 @@
     if (!blk_is_inserted(s->qdev.conf.blk)) {
         return false;
     }
+    if (s->tray_open) {
+        return false;
+    }
     blk_get_geometry(s->qdev.conf.blk, &nb_sectors);
     return nb_sectors <= CD_MAX_SECTORS;
 }
@@ -955,7 +1011,15 @@
     if (s->qdev.type != TYPE_ROM) {
         return -1;
     }
-    current = media_is_dvd(s) ? MMC_PROFILE_DVD_ROM : MMC_PROFILE_CD_ROM;
+
+    if (media_is_dvd(s)) {
+        current = MMC_PROFILE_DVD_ROM;
+    } else if (media_is_cd(s)) {
+        current = MMC_PROFILE_CD_ROM;
+    } else {
+        current = MMC_PROFILE_NONE;
+    }
+
     memset(outbuf, 0, 40);
     stl_be_p(&outbuf[0], 36); /* Bytes after the data length field */
     stw_be_p(&outbuf[6], current);
@@ -1524,26 +1588,20 @@
     int count;
 } UnmapCBData;
 
-static void scsi_unmap_complete(void *opaque, int ret)
+static void scsi_unmap_complete(void *opaque, int ret);
+
+static void scsi_unmap_complete_noio(UnmapCBData *data, int ret)
 {
-    UnmapCBData *data = opaque;
     SCSIDiskReq *r = data->r;
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint64_t sector_num;
     uint32_t nb_sectors;
 
-    r->req.aiocb = NULL;
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    assert(r->req.aiocb == NULL);
+    if (scsi_disk_req_check_error(r, ret, false)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
-
     if (data->count > 0) {
         sector_num = ldq_be_p(&data->inbuf[0]);
         nb_sectors = ldl_be_p(&data->inbuf[8]) & 0xffffffffULL;
@@ -1552,10 +1610,10 @@
             goto done;
         }
 
-        r->req.aiocb = blk_aio_discard(s->qdev.conf.blk,
-                                       sector_num * (s->qdev.blocksize / 512),
-                                       nb_sectors * (s->qdev.blocksize / 512),
-                                       scsi_unmap_complete, data);
+        r->req.aiocb = blk_aio_pdiscard(s->qdev.conf.blk,
+                                        sector_num * s->qdev.blocksize,
+                                        nb_sectors * s->qdev.blocksize,
+                                        scsi_unmap_complete, data);
         data->count--;
         data->inbuf += 16;
         return;
@@ -1568,6 +1626,17 @@
     g_free(data);
 }
 
+static void scsi_unmap_complete(void *opaque, int ret)
+{
+    UnmapCBData *data = opaque;
+    SCSIDiskReq *r = data->r;
+
+    assert(r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+
+    scsi_unmap_complete_noio(data, ret);
+}
+
 static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
@@ -1605,7 +1674,7 @@
 
     /* The matching unref is in scsi_unmap_complete, before data is freed.  */
     scsi_req_ref(&r->req);
-    scsi_unmap_complete(data, 0);
+    scsi_unmap_complete_noio(data, 0);
     return;
 
 invalid_param_len:
@@ -1632,17 +1701,11 @@
 
     assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
-    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
-    if (r->req.io_canceled) {
-        scsi_req_cancel_complete(&r->req);
+    if (scsi_disk_req_check_error(r, ret, true)) {
         goto done;
     }
 
-    if (ret < 0) {
-        if (scsi_handle_rw_error(r, -ret)) {
-            goto done;
-        }
-    }
+    block_acct_done(blk_get_stats(s->qdev.conf.blk), &r->acct);
 
     data->nb_sectors -= data->iov.iov_len / 512;
     data->sector += data->iov.iov_len / 512;
@@ -1650,9 +1713,13 @@
     if (data->iov.iov_len) {
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
                          data->iov.iov_len, BLOCK_ACCT_WRITE);
-        r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, data->sector,
-                                      &data->qiov, data->iov.iov_len / 512,
-                                      scsi_write_same_complete, data);
+        /* Reinitialize qiov, to handle unaligned WRITE SAME request
+         * where final qiov may need smaller size */
+        qemu_iovec_init_external(&data->qiov, &data->iov, 1);
+        r->req.aiocb = blk_aio_pwritev(s->qdev.conf.blk,
+                                       data->sector << BDRV_SECTOR_BITS,
+                                       &data->qiov, 0,
+                                       scsi_write_same_complete, data);
         return;
     }
 
@@ -1696,9 +1763,9 @@
         block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
                          nb_sectors * s->qdev.blocksize,
                         BLOCK_ACCT_WRITE);
-        r->req.aiocb = blk_aio_write_zeroes(s->qdev.conf.blk,
-                                r->req.cmd.lba * (s->qdev.blocksize / 512),
-                                nb_sectors * (s->qdev.blocksize / 512),
+        r->req.aiocb = blk_aio_pwrite_zeroes(s->qdev.conf.blk,
+                                r->req.cmd.lba * s->qdev.blocksize,
+                                nb_sectors * s->qdev.blocksize,
                                 flags, scsi_aio_complete, r);
         return;
     }
@@ -1719,9 +1786,10 @@
     scsi_req_ref(&r->req);
     block_acct_start(blk_get_stats(s->qdev.conf.blk), &r->acct,
                      data->iov.iov_len, BLOCK_ACCT_WRITE);
-    r->req.aiocb = blk_aio_writev(s->qdev.conf.blk, data->sector,
-                                  &data->qiov, data->iov.iov_len / 512,
-                                  scsi_write_same_complete, data);
+    r->req.aiocb = blk_aio_pwritev(s->qdev.conf.blk,
+                                   data->sector << BDRV_SECTOR_BITS,
+                                   &data->qiov, 0,
+                                   scsi_write_same_complete, data);
 }
 
 static void scsi_disk_emulate_write_data(SCSIRequest *req)
@@ -1993,13 +2061,13 @@
         }
         break;
     case MODE_SELECT:
-        DPRINTF("Mode Select(6) (len %lu)\n", (long)r->req.cmd.xfer);
+        DPRINTF("Mode Select(6) (len %lu)\n", (unsigned long)r->req.cmd.xfer);
         break;
     case MODE_SELECT_10:
-        DPRINTF("Mode Select(10) (len %lu)\n", (long)r->req.cmd.xfer);
+        DPRINTF("Mode Select(10) (len %lu)\n", (unsigned long)r->req.cmd.xfer);
         break;
     case UNMAP:
-        DPRINTF("Unmap (len %lu)\n", (long)r->req.cmd.xfer);
+        DPRINTF("Unmap (len %lu)\n", (unsigned long)r->req.cmd.xfer);
         break;
     case VERIFY_10:
     case VERIFY_12:
@@ -2013,7 +2081,7 @@
     case WRITE_SAME_16:
         DPRINTF("WRITE SAME %d (len %lu)\n",
                 req->cmd.buf[0] == WRITE_SAME_10 ? 10 : 16,
-                (long)r->req.cmd.xfer);
+                (unsigned long)r->req.cmd.xfer);
         break;
     default:
         DPRINTF("Unknown SCSI command (%2.2x=%s)\n", buf[0],
@@ -2053,6 +2121,7 @@
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, req->dev);
+    SCSIDiskClass *sdc = (SCSIDiskClass *) object_get_class(OBJECT(s));
     uint32_t len;
     uint8_t command;
 
@@ -2111,6 +2180,7 @@
         scsi_check_condition(r, SENSE_CODE(LBA_OUT_OF_RANGE));
         return 0;
     }
+    r->need_fua_emulation = sdc->need_fua_emulation(&r->req.cmd);
     if (r->sector_count == 0) {
         scsi_req_complete(&r->req, GOOD);
     }
@@ -2232,6 +2302,7 @@
     }
 
     blkconf_serial(&s->qdev.conf, &s->serial);
+    blkconf_blocksizes(&s->qdev.conf);
     if (dev->type == TYPE_DISK) {
         blkconf_geometry(&dev->conf, NULL, 65535, 255, 255, &err);
         if (err) {
@@ -2239,6 +2310,7 @@
             return;
         }
     }
+    blkconf_apply_backend_options(&dev->conf);
 
     if (s->qdev.conf.discard_granularity == -1) {
         s->qdev.conf.discard_granularity =
@@ -2246,7 +2318,7 @@
     }
 
     if (!s->version) {
-        s->version = g_strdup(qemu_get_version());
+        s->version = g_strdup(qemu_hw_version());
     }
     if (!s->vendor) {
         s->vendor = g_strdup("QEMU");
@@ -2271,6 +2343,12 @@
 static void scsi_hd_realize(SCSIDevice *dev, Error **errp)
 {
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, dev);
+    /* can happen for devices without drive. The error message for missing
+     * backend will be issued in scsi_realize
+     */
+    if (s->qdev.conf.blk) {
+        blkconf_blocksizes(&s->qdev.conf);
+    }
     s->qdev.blocksize = s->qdev.conf.logical_block_size;
     s->qdev.type = TYPE_DISK;
     if (!s->product) {
@@ -2483,18 +2561,148 @@
     s->features |= (1 << SCSI_DISK_F_NO_REMOVABLE_DEVOPS);
 
     scsi_realize(&s->qdev, errp);
+    scsi_generic_read_device_identification(&s->qdev);
+}
+
+typedef struct SCSIBlockReq {
+    SCSIDiskReq req;
+    sg_io_hdr_t io_header;
+
+    /* Selected bytes of the original CDB, copied into our own CDB.  */
+    uint8_t cmd, cdb1, group_number;
+
+    /* CDB passed to SG_IO.  */
+    uint8_t cdb[16];
+} SCSIBlockReq;
+
+static BlockAIOCB *scsi_block_do_sgio(SCSIBlockReq *req,
+                                      int64_t offset, QEMUIOVector *iov,
+                                      int direction,
+                                      BlockCompletionFunc *cb, void *opaque)
+{
+    sg_io_hdr_t *io_header = &req->io_header;
+    SCSIDiskReq *r = &req->req;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+    int nb_logical_blocks;
+    uint64_t lba;
+    BlockAIOCB *aiocb;
+
+    /* This is not supported yet.  It can only happen if the guest does
+     * reads and writes that are not aligned to one logical sectors
+     * _and_ cover multiple MemoryRegions.
+     */
+    assert(offset % s->qdev.blocksize == 0);
+    assert(iov->size % s->qdev.blocksize == 0);
+
+    io_header->interface_id = 'S';
+
+    /* The data transfer comes from the QEMUIOVector.  */
+    io_header->dxfer_direction = direction;
+    io_header->dxfer_len = iov->size;
+    io_header->dxferp = (void *)iov->iov;
+    io_header->iovec_count = iov->niov;
+    assert(io_header->iovec_count == iov->niov); /* no overflow! */
+
+    /* Build a new CDB with the LBA and length patched in, in case
+     * DMA helpers split the transfer in multiple segments.  Do not
+     * build a CDB smaller than what the guest wanted, and only build
+     * a larger one if strictly necessary.
+     */
+    io_header->cmdp = req->cdb;
+    lba = offset / s->qdev.blocksize;
+    nb_logical_blocks = io_header->dxfer_len / s->qdev.blocksize;
+
+    if ((req->cmd >> 5) == 0 && lba <= 0x1ffff) {
+        /* 6-byte CDB */
+        stl_be_p(&req->cdb[0], lba | (req->cmd << 24));
+        req->cdb[4] = nb_logical_blocks;
+        req->cdb[5] = 0;
+        io_header->cmd_len = 6;
+    } else if ((req->cmd >> 5) <= 1 && lba <= 0xffffffffULL) {
+        /* 10-byte CDB */
+        req->cdb[0] = (req->cmd & 0x1f) | 0x20;
+        req->cdb[1] = req->cdb1;
+        stl_be_p(&req->cdb[2], lba);
+        req->cdb[6] = req->group_number;
+        stw_be_p(&req->cdb[7], nb_logical_blocks);
+        req->cdb[9] = 0;
+        io_header->cmd_len = 10;
+    } else if ((req->cmd >> 5) != 4 && lba <= 0xffffffffULL) {
+        /* 12-byte CDB */
+        req->cdb[0] = (req->cmd & 0x1f) | 0xA0;
+        req->cdb[1] = req->cdb1;
+        stl_be_p(&req->cdb[2], lba);
+        stl_be_p(&req->cdb[6], nb_logical_blocks);
+        req->cdb[10] = req->group_number;
+        req->cdb[11] = 0;
+        io_header->cmd_len = 12;
+    } else {
+        /* 16-byte CDB */
+        req->cdb[0] = (req->cmd & 0x1f) | 0x80;
+        req->cdb[1] = req->cdb1;
+        stq_be_p(&req->cdb[2], lba);
+        stl_be_p(&req->cdb[10], nb_logical_blocks);
+        req->cdb[14] = req->group_number;
+        req->cdb[15] = 0;
+        io_header->cmd_len = 16;
+    }
+
+    /* The rest is as in scsi-generic.c.  */
+    io_header->mx_sb_len = sizeof(r->req.sense);
+    io_header->sbp = r->req.sense;
+    io_header->timeout = UINT_MAX;
+    io_header->usr_ptr = r;
+    io_header->flags |= SG_FLAG_DIRECT_IO;
+
+    aiocb = blk_aio_ioctl(s->qdev.conf.blk, SG_IO, io_header, cb, opaque);
+    assert(aiocb != NULL);
+    return aiocb;
+}
+
+static bool scsi_block_no_fua(SCSICommand *cmd)
+{
+    return false;
+}
+
+static BlockAIOCB *scsi_block_dma_readv(int64_t offset,
+                                        QEMUIOVector *iov,
+                                        BlockCompletionFunc *cb, void *cb_opaque,
+                                        void *opaque)
+{
+    SCSIBlockReq *r = opaque;
+    return scsi_block_do_sgio(r, offset, iov,
+                              SG_DXFER_FROM_DEV, cb, cb_opaque);
+}
+
+static BlockAIOCB *scsi_block_dma_writev(int64_t offset,
+                                         QEMUIOVector *iov,
+                                         BlockCompletionFunc *cb, void *cb_opaque,
+                                         void *opaque)
+{
+    SCSIBlockReq *r = opaque;
+    return scsi_block_do_sgio(r, offset, iov,
+                              SG_DXFER_TO_DEV, cb, cb_opaque);
 }
 
 static bool scsi_block_is_passthrough(SCSIDiskState *s, uint8_t *buf)
 {
     switch (buf[0]) {
+    case VERIFY_10:
+    case VERIFY_12:
+    case VERIFY_16:
+        /* Check if BYTCHK == 0x01 (data-out buffer contains data
+         * for the number of logical blocks specified in the length
+         * field).  For other modes, do not use scatter/gather operation.
+         */
+        if ((buf[1] & 6) != 2) {
+            return false;
+        }
+        break;
+
     case READ_6:
     case READ_10:
     case READ_12:
     case READ_16:
-    case VERIFY_10:
-    case VERIFY_12:
-    case VERIFY_16:
     case WRITE_6:
     case WRITE_10:
     case WRITE_12:
@@ -2502,21 +2710,8 @@
     case WRITE_VERIFY_10:
     case WRITE_VERIFY_12:
     case WRITE_VERIFY_16:
-        /* If we are not using O_DIRECT, we might read stale data from the
-         * host cache if writes were made using other commands than these
-         * ones (such as WRITE SAME or EXTENDED COPY, etc.).  So, without
-         * O_DIRECT everything must go through SG_IO.
-         */
-        if (!(blk_get_flags(s->qdev.conf.blk) & BDRV_O_NOCACHE)) {
-            break;
-        }
-
-        /* MMC writing cannot be done via pread/pwrite, because it sometimes
+        /* MMC writing cannot be done via DMA helpers, because it sometimes
          * involves writing beyond the maximum LBA or to negative LBA (lead-in).
-         * And once you do these writes, reading from the block device is
-         * unreliable, too.  It is even possible that reads deliver random data
-         * from the host page cache (this is probably a Linux bug).
-         *
          * We might use scsi_disk_dma_reqops as long as no writing commands are
          * seen, but performance usually isn't paramount on optical media.  So,
          * just make scsi-block operate the same as scsi-generic for them.
@@ -2534,6 +2729,55 @@
 }
 
 
+static int32_t scsi_block_dma_command(SCSIRequest *req, uint8_t *buf)
+{
+    SCSIBlockReq *r = (SCSIBlockReq *)req;
+    r->cmd = req->cmd.buf[0];
+    switch (r->cmd >> 5) {
+    case 0:
+        /* 6-byte CDB.  */
+        r->cdb1 = r->group_number = 0;
+        break;
+    case 1:
+        /* 10-byte CDB.  */
+        r->cdb1 = req->cmd.buf[1];
+        r->group_number = req->cmd.buf[6];
+        break;
+    case 4:
+        /* 12-byte CDB.  */
+        r->cdb1 = req->cmd.buf[1];
+        r->group_number = req->cmd.buf[10];
+        break;
+    case 5:
+        /* 16-byte CDB.  */
+        r->cdb1 = req->cmd.buf[1];
+        r->group_number = req->cmd.buf[14];
+        break;
+    default:
+        abort();
+    }
+
+    if (r->cdb1 & 0xe0) {
+        /* Protection information is not supported.  */
+        scsi_check_condition(&r->req, SENSE_CODE(INVALID_FIELD));
+        return 0;
+    }
+
+    r->req.status = &r->io_header.status;
+    return scsi_disk_dma_command(req, buf);
+}
+
+static const SCSIReqOps scsi_block_dma_reqops = {
+    .size         = sizeof(SCSIBlockReq),
+    .free_req     = scsi_free_request,
+    .send_command = scsi_block_dma_command,
+    .read_data    = scsi_read_data,
+    .write_data   = scsi_write_data,
+    .get_buf      = scsi_get_buf,
+    .load_request = scsi_disk_load_request,
+    .save_request = scsi_disk_save_request,
+};
+
 static SCSIRequest *scsi_block_new_request(SCSIDevice *d, uint32_t tag,
                                            uint32_t lun, uint8_t *buf,
                                            void *hba_private)
@@ -2544,7 +2788,7 @@
         return scsi_req_alloc(&scsi_generic_req_ops, &s->qdev, tag, lun,
                               hba_private);
     } else {
-        return scsi_req_alloc(&scsi_disk_dma_reqops, &s->qdev, tag, lun,
+        return scsi_req_alloc(&scsi_block_dma_reqops, &s->qdev, tag, lun,
                               hba_private);
     }
 }
@@ -2563,8 +2807,50 @@
 
 #endif
 
+static
+BlockAIOCB *scsi_dma_readv(int64_t offset, QEMUIOVector *iov,
+                           BlockCompletionFunc *cb, void *cb_opaque,
+                           void *opaque)
+{
+    SCSIDiskReq *r = opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+    return blk_aio_preadv(s->qdev.conf.blk, offset, iov, 0, cb, cb_opaque);
+}
+
+static
+BlockAIOCB *scsi_dma_writev(int64_t offset, QEMUIOVector *iov,
+                            BlockCompletionFunc *cb, void *cb_opaque,
+                            void *opaque)
+{
+    SCSIDiskReq *r = opaque;
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
+    return blk_aio_pwritev(s->qdev.conf.blk, offset, iov, 0, cb, cb_opaque);
+}
+
+static void scsi_disk_base_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SCSIDiskClass *sdc = SCSI_DISK_BASE_CLASS(klass);
+
+    dc->fw_name = "disk";
+    dc->reset = scsi_disk_reset;
+    sdc->dma_readv = scsi_dma_readv;
+    sdc->dma_writev = scsi_dma_writev;
+    sdc->need_fua_emulation = scsi_is_cmd_fua;
+}
+
+static const TypeInfo scsi_disk_base_info = {
+    .name          = TYPE_SCSI_DISK_BASE,
+    .parent        = TYPE_SCSI_DEVICE,
+    .class_init    = scsi_disk_base_class_initfn,
+    .instance_size = sizeof(SCSIDiskState),
+    .class_size    = sizeof(SCSIDiskClass),
+    .abstract      = true,
+};
+
 #define DEFINE_SCSI_DISK_PROPERTIES()                                \
     DEFINE_BLOCK_PROPERTIES(SCSIDiskState, qdev.conf),               \
+    DEFINE_BLOCK_ERROR_PROPERTIES(SCSIDiskState, qdev.conf),         \
     DEFINE_PROP_STRING("ver", SCSIDiskState, version),               \
     DEFINE_PROP_STRING("serial", SCSIDiskState, serial),             \
     DEFINE_PROP_STRING("vendor", SCSIDiskState, vendor),             \
@@ -2576,11 +2862,13 @@
                     SCSI_DISK_F_REMOVABLE, false),
     DEFINE_PROP_BIT("dpofua", SCSIDiskState, features,
                     SCSI_DISK_F_DPOFUA, false),
-    DEFINE_PROP_UINT64("wwn", SCSIDiskState, wwn, 0),
-    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, port_wwn, 0),
+    DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
+    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
     DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
     DEFINE_PROP_UINT64("max_unmap_size", SCSIDiskState, max_unmap_size,
                        DEFAULT_MAX_UNMAP_SIZE),
+    DEFINE_PROP_UINT64("max_io_size", SCSIDiskState, max_io_size,
+                       DEFAULT_MAX_IO_SIZE),
     DEFINE_BLOCK_CHS_PROPERTIES(SCSIDiskState, qdev.conf),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -2608,25 +2896,24 @@
     sc->realize      = scsi_hd_realize;
     sc->alloc_req    = scsi_new_request;
     sc->unit_attention_reported = scsi_disk_unit_attention_reported;
-    dc->fw_name = "disk";
     dc->desc = "virtual SCSI disk";
-    dc->reset = scsi_disk_reset;
     dc->props = scsi_hd_properties;
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
 static const TypeInfo scsi_hd_info = {
     .name          = "scsi-hd",
-    .parent        = TYPE_SCSI_DEVICE,
-    .instance_size = sizeof(SCSIDiskState),
+    .parent        = TYPE_SCSI_DISK_BASE,
     .class_init    = scsi_hd_class_initfn,
 };
 
 static Property scsi_cd_properties[] = {
     DEFINE_SCSI_DISK_PROPERTIES(),
-    DEFINE_PROP_UINT64("wwn", SCSIDiskState, wwn, 0),
-    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, port_wwn, 0),
+    DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
+    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
     DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
+    DEFINE_PROP_UINT64("max_io_size", SCSIDiskState, max_io_size,
+                       DEFAULT_MAX_IO_SIZE),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2638,17 +2925,14 @@
     sc->realize      = scsi_cd_realize;
     sc->alloc_req    = scsi_new_request;
     sc->unit_attention_reported = scsi_disk_unit_attention_reported;
-    dc->fw_name = "disk";
     dc->desc = "virtual SCSI CD-ROM";
-    dc->reset = scsi_disk_reset;
     dc->props = scsi_cd_properties;
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
 static const TypeInfo scsi_cd_info = {
     .name          = "scsi-cd",
-    .parent        = TYPE_SCSI_DEVICE,
-    .instance_size = sizeof(SCSIDiskState),
+    .parent        = TYPE_SCSI_DISK_BASE,
     .class_init    = scsi_cd_class_initfn,
 };
 
@@ -2662,21 +2946,22 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     SCSIDeviceClass *sc = SCSI_DEVICE_CLASS(klass);
+    SCSIDiskClass *sdc = SCSI_DISK_BASE_CLASS(klass);
 
     sc->realize      = scsi_block_realize;
     sc->alloc_req    = scsi_block_new_request;
     sc->parse_cdb    = scsi_block_parse_cdb;
-    dc->fw_name = "disk";
+    sdc->dma_readv   = scsi_block_dma_readv;
+    sdc->dma_writev  = scsi_block_dma_writev;
+    sdc->need_fua_emulation = scsi_block_no_fua;
     dc->desc = "SCSI block device passthrough";
-    dc->reset = scsi_disk_reset;
     dc->props = scsi_block_properties;
     dc->vmsd  = &vmstate_scsi_disk_state;
 }
 
 static const TypeInfo scsi_block_info = {
     .name          = "scsi-block",
-    .parent        = TYPE_SCSI_DEVICE,
-    .instance_size = sizeof(SCSIDiskState),
+    .parent        = TYPE_SCSI_DISK_BASE,
     .class_init    = scsi_block_class_initfn,
 };
 #endif
@@ -2687,11 +2972,13 @@
                     SCSI_DISK_F_REMOVABLE, false),
     DEFINE_PROP_BIT("dpofua", SCSIDiskState, features,
                     SCSI_DISK_F_DPOFUA, false),
-    DEFINE_PROP_UINT64("wwn", SCSIDiskState, wwn, 0),
-    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, port_wwn, 0),
+    DEFINE_PROP_UINT64("wwn", SCSIDiskState, qdev.wwn, 0),
+    DEFINE_PROP_UINT64("port_wwn", SCSIDiskState, qdev.port_wwn, 0),
     DEFINE_PROP_UINT16("port_index", SCSIDiskState, port_index, 0),
     DEFINE_PROP_UINT64("max_unmap_size", SCSIDiskState, max_unmap_size,
                        DEFAULT_MAX_UNMAP_SIZE),
+    DEFINE_PROP_UINT64("max_io_size", SCSIDiskState, max_io_size,
+                       DEFAULT_MAX_IO_SIZE),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2712,13 +2999,13 @@
 
 static const TypeInfo scsi_disk_info = {
     .name          = "scsi-disk",
-    .parent        = TYPE_SCSI_DEVICE,
-    .instance_size = sizeof(SCSIDiskState),
+    .parent        = TYPE_SCSI_DISK_BASE,
     .class_init    = scsi_disk_class_initfn,
 };
 
 static void scsi_disk_register_types(void)
 {
+    type_register_static(&scsi_disk_base_info);
     type_register_static(&scsi_hd_info);
     type_register_static(&scsi_cd_info);
 #ifdef __linux__
diff --git a/hw/scsi/scsi-generic.c b/hw/scsi/scsi-generic.c
index 6b9e4e1..7a588a7 100644
--- a/hw/scsi/scsi-generic.c
+++ b/hw/scsi/scsi-generic.c
@@ -11,6 +11,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/scsi/scsi.h"
@@ -31,10 +33,6 @@
 #define BADF(fmt, ...) \
 do { fprintf(stderr, "scsi-generic: " fmt , ## __VA_ARGS__); } while (0)
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
 #include <scsi/sg.h>
 #include "block/scsi.h"
 
@@ -88,12 +86,12 @@
 }
 
 /* Helper function for command completion.  */
-static void scsi_command_complete(void *opaque, int ret)
+static void scsi_command_complete_noio(SCSIGenericReq *r, int ret)
 {
     int status;
-    SCSIGenericReq *r = (SCSIGenericReq *)opaque;
 
-    r->req.aiocb = NULL;
+    assert(r->req.aiocb == NULL);
+
     if (r->req.io_canceled) {
         scsi_req_cancel_complete(&r->req);
         goto done;
@@ -142,6 +140,15 @@
     scsi_req_unref(&r->req);
 }
 
+static void scsi_command_complete(void *opaque, int ret)
+{
+    SCSIGenericReq *r = (SCSIGenericReq *)opaque;
+
+    assert(r->req.aiocb != NULL);
+    r->req.aiocb = NULL;
+    scsi_command_complete_noio(r, ret);
+}
+
 static int execute_command(BlockBackend *blk,
                            SCSIGenericReq *r, int direction,
                            BlockCompletionFunc *complete)
@@ -172,33 +179,64 @@
     SCSIDevice *s = r->req.dev;
     int len;
 
+    assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
+
     if (ret || r->req.io_canceled) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
         return;
     }
+
     len = r->io_header.dxfer_len - r->io_header.resid;
     DPRINTF("Data ready tag=0x%x len=%d\n", r->req.tag, len);
 
     r->len = -1;
     if (len == 0) {
-        scsi_command_complete(r, 0);
-    } else {
-        /* Snoop READ CAPACITY output to set the blocksize.  */
-        if (r->req.cmd.buf[0] == READ_CAPACITY_10 &&
-            (ldl_be_p(&r->buf[0]) != 0xffffffffU || s->max_lba == 0)) {
-            s->blocksize = ldl_be_p(&r->buf[4]);
-            s->max_lba = ldl_be_p(&r->buf[0]) & 0xffffffffULL;
-        } else if (r->req.cmd.buf[0] == SERVICE_ACTION_IN_16 &&
-                   (r->req.cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
-            s->blocksize = ldl_be_p(&r->buf[8]);
-            s->max_lba = ldq_be_p(&r->buf[0]);
-        }
-        blk_set_guest_block_size(s->conf.blk, s->blocksize);
-
-        scsi_req_data(&r->req, len);
-        scsi_req_unref(&r->req);
+        scsi_command_complete_noio(r, 0);
+        return;
     }
+
+    /* Snoop READ CAPACITY output to set the blocksize.  */
+    if (r->req.cmd.buf[0] == READ_CAPACITY_10 &&
+        (ldl_be_p(&r->buf[0]) != 0xffffffffU || s->max_lba == 0)) {
+        s->blocksize = ldl_be_p(&r->buf[4]);
+        s->max_lba = ldl_be_p(&r->buf[0]) & 0xffffffffULL;
+    } else if (r->req.cmd.buf[0] == SERVICE_ACTION_IN_16 &&
+               (r->req.cmd.buf[1] & 31) == SAI_READ_CAPACITY_16) {
+        s->blocksize = ldl_be_p(&r->buf[8]);
+        s->max_lba = ldq_be_p(&r->buf[0]);
+    }
+    blk_set_guest_block_size(s->conf.blk, s->blocksize);
+
+    /* Patch MODE SENSE device specific parameters if the BDS is opened
+     * readonly.
+     */
+    if ((s->type == TYPE_DISK || s->type == TYPE_TAPE) &&
+        blk_is_read_only(s->conf.blk) &&
+        (r->req.cmd.buf[0] == MODE_SENSE ||
+         r->req.cmd.buf[0] == MODE_SENSE_10) &&
+        (r->req.cmd.buf[1] & 0x8) == 0) {
+        if (r->req.cmd.buf[0] == MODE_SENSE) {
+            r->buf[2] |= 0x80;
+        } else  {
+            r->buf[3] |= 0x80;
+        }
+    }
+    if (s->type == TYPE_DISK &&
+        r->req.cmd.buf[0] == INQUIRY &&
+        r->req.cmd.buf[2] == 0xb0) {
+        uint32_t max_transfer =
+            blk_get_max_transfer(s->conf.blk) / s->blocksize;
+
+        assert(max_transfer);
+        stl_be_p(&r->buf[8], max_transfer);
+        /* Also take care of the opt xfer len. */
+        if (ldl_be_p(&r->buf[12]) > max_transfer) {
+            stl_be_p(&r->buf[12], max_transfer);
+        }
+    }
+    scsi_req_data(&r->req, len);
+    scsi_req_unref(&r->req);
 }
 
 /* Read more data from scsi device into buffer.  */
@@ -213,14 +251,14 @@
     /* The request is used as the AIO opaque value, so add a ref.  */
     scsi_req_ref(&r->req);
     if (r->len == -1) {
-        scsi_command_complete(r, 0);
+        scsi_command_complete_noio(r, 0);
         return;
     }
 
     ret = execute_command(s->conf.blk, r, SG_DXFER_FROM_DEV,
                           scsi_read_complete);
     if (ret < 0) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
     }
 }
 
@@ -230,9 +268,12 @@
     SCSIDevice *s = r->req.dev;
 
     DPRINTF("scsi_write_complete() ret = %d\n", ret);
+
+    assert(r->req.aiocb != NULL);
     r->req.aiocb = NULL;
+
     if (ret || r->req.io_canceled) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
         return;
     }
 
@@ -242,7 +283,7 @@
         DPRINTF("block size %d\n", s->blocksize);
     }
 
-    scsi_command_complete(r, ret);
+    scsi_command_complete_noio(r, ret);
 }
 
 /* Write data to a scsi device.  Returns nonzero on failure.
@@ -264,7 +305,7 @@
     scsi_req_ref(&r->req);
     ret = execute_command(s->conf.blk, r, SG_DXFER_TO_DEV, scsi_write_complete);
     if (ret < 0) {
-        scsi_command_complete(r, ret);
+        scsi_command_complete_noio(r, ret);
     }
 }
 
@@ -298,8 +339,7 @@
 #endif
 
     if (r->req.cmd.xfer == 0) {
-        if (r->buf != NULL)
-            g_free(r->buf);
+        g_free(r->buf);
         r->buflen = 0;
         r->buf = NULL;
         /* The request is used as the AIO opaque value, so add a ref.  */
@@ -307,15 +347,14 @@
         ret = execute_command(s->conf.blk, r, SG_DXFER_NONE,
                               scsi_command_complete);
         if (ret < 0) {
-            scsi_command_complete(r, ret);
+            scsi_command_complete_noio(r, ret);
             return 0;
         }
         return 0;
     }
 
     if (r->buflen != r->req.cmd.xfer) {
-        if (r->buf != NULL)
-            g_free(r->buf);
+        g_free(r->buf);
         r->buf = g_malloc(r->req.cmd.xfer);
         r->buflen = r->req.cmd.xfer;
     }
@@ -330,6 +369,96 @@
     }
 }
 
+static int read_naa_id(const uint8_t *p, uint64_t *p_wwn)
+{
+    int i;
+
+    if ((p[1] & 0xF) == 3) {
+        /* NAA designator type */
+        if (p[3] != 8) {
+            return -EINVAL;
+        }
+        *p_wwn = ldq_be_p(p + 4);
+        return 0;
+    }
+
+    if ((p[1] & 0xF) == 8) {
+        /* SCSI name string designator type */
+        if (p[3] < 20 || memcmp(&p[4], "naa.", 4)) {
+            return -EINVAL;
+        }
+        if (p[3] > 20 && p[24] != ',') {
+            return -EINVAL;
+        }
+        *p_wwn = 0;
+        for (i = 8; i < 24; i++) {
+            char c = toupper(p[i]);
+            c -= (c >= '0' && c <= '9' ? '0' : 'A' - 10);
+            *p_wwn = (*p_wwn << 4) | c;
+        }
+        return 0;
+    }
+
+    return -EINVAL;
+}
+
+void scsi_generic_read_device_identification(SCSIDevice *s)
+{
+    uint8_t cmd[6];
+    uint8_t buf[250];
+    uint8_t sensebuf[8];
+    sg_io_hdr_t io_header;
+    int ret;
+    int i, len;
+
+    memset(cmd, 0, sizeof(cmd));
+    memset(buf, 0, sizeof(buf));
+    cmd[0] = INQUIRY;
+    cmd[1] = 1;
+    cmd[2] = 0x83;
+    cmd[4] = sizeof(buf);
+
+    memset(&io_header, 0, sizeof(io_header));
+    io_header.interface_id = 'S';
+    io_header.dxfer_direction = SG_DXFER_FROM_DEV;
+    io_header.dxfer_len = sizeof(buf);
+    io_header.dxferp = buf;
+    io_header.cmdp = cmd;
+    io_header.cmd_len = sizeof(cmd);
+    io_header.mx_sb_len = sizeof(sensebuf);
+    io_header.sbp = sensebuf;
+    io_header.timeout = 6000; /* XXX */
+
+    ret = blk_ioctl(s->conf.blk, SG_IO, &io_header);
+    if (ret < 0 || io_header.driver_status || io_header.host_status) {
+        return;
+    }
+
+    len = MIN((buf[2] << 8) | buf[3], sizeof(buf) - 4);
+    for (i = 0; i + 3 <= len; ) {
+        const uint8_t *p = &buf[i + 4];
+        uint64_t wwn;
+
+        if (i + (p[3] + 4) > len) {
+            break;
+        }
+
+        if ((p[1] & 0x10) == 0) {
+            /* Associated with the logical unit */
+            if (read_naa_id(p, &wwn) == 0) {
+                s->wwn = wwn;
+            }
+        } else if ((p[1] & 0x10) == 0x10) {
+            /* Associated with the target port */
+            if (read_naa_id(p, &wwn) == 0) {
+                s->port_wwn = wwn;
+            }
+        }
+
+        i += p[3] + 4;
+    }
+}
+
 static int get_stream_blocksize(BlockBackend *blk)
 {
     uint8_t cmd[6];
@@ -433,6 +562,8 @@
     }
 
     DPRINTF("block size %d\n", s->blocksize);
+
+    scsi_generic_read_device_identification(s);
 }
 
 const SCSIReqOps scsi_generic_req_ops = {
@@ -449,10 +580,7 @@
 static SCSIRequest *scsi_new_request(SCSIDevice *d, uint32_t tag, uint32_t lun,
                                      uint8_t *buf, void *hba_private)
 {
-    SCSIRequest *req;
-
-    req = scsi_req_alloc(&scsi_generic_req_ops, d, tag, lun, hba_private);
-    return req;
+    return scsi_req_alloc(&scsi_generic_req_ops, d, tag, lun, hba_private);
 }
 
 static Property scsi_generic_properties[] = {
diff --git a/hw/scsi/spapr_vscsi.c b/hw/scsi/spapr_vscsi.c
index 20b20f0..8fbd50f 100644
--- a/hw/scsi/spapr_vscsi.c
+++ b/hw/scsi/spapr_vscsi.c
@@ -31,6 +31,9 @@
  *  - Add indirect descriptors support
  *  - Maybe do autosense (PAPR seems to mandate it, linux doesn't care)
  */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/scsi/scsi.h"
 #include "block/scsi.h"
@@ -630,7 +633,7 @@
     vscsi_req *req = sreq->hba_private;
     assert(req->active);
 
-    vmstate_save_state(f, &vmstate_spapr_vscsi_req, req);
+    vmstate_save_state(f, &vmstate_spapr_vscsi_req, req, NULL);
 
     DPRINTF("VSCSI: saving tag=%u, current desc#%d, offset=%x\n",
             req->qtag, req->cur_desc_num, req->cur_desc_offset);
@@ -695,7 +698,7 @@
     uint8_t resp_data[36];
     int rc, len, alen;
 
-    /* We dont do EVPD. Also check that page_code is 0 */
+    /* We don't do EVPD. Also check that page_code is 0 */
     if ((cdb[1] & 0x01) || cdb[2] != 0) {
         /* Send INVALID FIELD IN CDB */
         vscsi_makeup_sense(s, req, ILLEGAL_REQUEST, 0x24, 0);
@@ -750,7 +753,6 @@
     len = n+8;
 
     resp_data = g_malloc0(len);
-    memset(resp_data, 0, len);
     stl_be_p(resp_data, n);
     i = found_lun0 ? 8 : 16;
     QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
@@ -1212,24 +1214,17 @@
     }
 }
 
-static int spapr_vscsi_init(VIOsPAPRDevice *dev)
+static void spapr_vscsi_realize(VIOsPAPRDevice *dev, Error **errp)
 {
     VSCSIState *s = VIO_SPAPR_VSCSI_DEVICE(dev);
-    Error *err = NULL;
 
     dev->crq.SendFunc = vscsi_do_crq;
 
     scsi_bus_new(&s->bus, sizeof(s->bus), DEVICE(dev),
                  &vscsi_scsi_info, NULL);
     if (!dev->qdev.hotplugged) {
-        scsi_bus_legacy_handle_cmdline(&s->bus, &err);
-        if (err != NULL) {
-            error_free(err);
-            return -1;
-        }
+        scsi_bus_legacy_handle_cmdline(&s->bus, errp);
     }
-
-    return 0;
 }
 
 void spapr_vscsi_create(VIOsPAPRBus *bus)
@@ -1281,7 +1276,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VIOsPAPRDeviceClass *k = VIO_SPAPR_DEVICE_CLASS(klass);
 
-    k->init = spapr_vscsi_init;
+    k->realize = spapr_vscsi_realize;
     k->reset = spapr_vscsi_reset;
     k->devnode = spapr_vscsi_devnode;
     k->dt_name = "v-scsi";
diff --git a/hw/scsi/trace-events b/hw/scsi/trace-events
new file mode 100644
index 0000000..ed64858
--- /dev/null
+++ b/hw/scsi/trace-events
@@ -0,0 +1,204 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/scsi/scsi-bus.c
+scsi_req_alloc(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_req_cancel(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_req_data(int target, int lun, int tag, int len) "target %d lun %d tag %d len %d"
+scsi_req_data_canceled(int target, int lun, int tag, int len) "target %d lun %d tag %d len %d"
+scsi_req_dequeue(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_req_continue(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_req_continue_canceled(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_req_parsed(int target, int lun, int tag, int cmd, int mode, int xfer) "target %d lun %d tag %d command %d dir %d length %d"
+scsi_req_parsed_lba(int target, int lun, int tag, int cmd, uint64_t lba) "target %d lun %d tag %d command %d lba %"PRIu64
+scsi_req_parse_bad(int target, int lun, int tag, int cmd) "target %d lun %d tag %d command %d"
+scsi_req_build_sense(int target, int lun, int tag, int key, int asc, int ascq) "target %d lun %d tag %d key %#02x asc %#02x ascq %#02x"
+scsi_device_set_ua(int target, int lun, int key, int asc, int ascq) "target %d lun %d key %#02x asc %#02x ascq %#02x"
+scsi_report_luns(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_inquiry(int target, int lun, int tag, int cdb1, int cdb2) "target %d lun %d tag %d page %#02x/%#02x"
+scsi_test_unit_ready(int target, int lun, int tag) "target %d lun %d tag %d"
+scsi_request_sense(int target, int lun, int tag) "target %d lun %d tag %d"
+
+# hw/scsi/mptsas.c
+mptsas_command_complete(void *dev, uint32_t ctx, uint32_t status, uint32_t resid) "dev %p context 0x%08x status %x resid %d"
+mptsas_diag_read(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%08x"
+mptsas_diag_write(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%08x"
+mptsas_irq_intx(void *dev, int level) "dev %p level %d"
+mptsas_irq_msi(void *dev) "dev %p "
+mptsas_mmio_read(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%x"
+mptsas_mmio_unhandled_read(void *dev, uint32_t addr) "dev %p addr 0x%08x"
+mptsas_mmio_unhandled_write(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%x"
+mptsas_mmio_write(void *dev, uint32_t addr, uint32_t val) "dev %p addr 0x%08x value 0x%x"
+mptsas_process_message(void *dev, int msg, uint32_t ctx) "dev %p cmd %d context 0x%08x\n"
+mptsas_process_scsi_io_request(void *dev, int bus, int target, int lun, uint64_t len) "dev %p dev %d:%d:%d length %"PRIu64""
+mptsas_reset(void *dev) "dev %p "
+mptsas_scsi_overflow(void *dev, uint32_t ctx, uint64_t req, uint64_t found) "dev %p context 0x%08x: %"PRIu64"/%"PRIu64""
+mptsas_sgl_overflow(void *dev, uint32_t ctx, uint64_t req, uint64_t found) "dev %p context 0x%08x: %"PRIu64"/%"PRIu64""
+mptsas_unhandled_cmd(void *dev, uint32_t ctx, uint8_t msg_cmd) "dev %p context 0x%08x: Unhandled cmd %x"
+mptsas_unhandled_doorbell_cmd(void *dev, int cmd) "dev %p value 0x%08x"
+
+# hw/scsi/mptconfig.c
+mptsas_config_sas_device(void *dev, int address, int port, int phy_handle, int dev_handle, int page) "dev %p address %d (port %d, handles: phy %d dev %d) page %d"
+mptsas_config_sas_phy(void *dev, int address, int port, int phy_handle, int dev_handle, int page) "dev %p address %d (port %d, handles: phy %d dev %d) page %d"
+
+# hw/scsi/megasas.c
+megasas_init_firmware(uint64_t pa) "pa %" PRIx64 " "
+megasas_init_queue(uint64_t queue_pa, int queue_len, uint64_t head, uint64_t tail, uint32_t flags) "queue at %" PRIx64 " len %d head %" PRIx64 " tail %" PRIx64 " flags %x"
+megasas_initq_map_failed(int frame) "scmd %d: failed to map queue"
+megasas_initq_mapped(uint64_t pa) "queue already mapped at %" PRIx64
+megasas_initq_mismatch(int queue_len, int fw_cmds) "queue size %d max fw cmds %d"
+megasas_qf_mapped(unsigned int index) "skip mapped frame %x"
+megasas_qf_new(unsigned int index, uint64_t frame) "frame %x addr %" PRIx64
+megasas_qf_busy(unsigned long pa) "all frames busy for frame %lx"
+megasas_qf_enqueue(unsigned int index, unsigned int count, uint64_t context, unsigned int head, unsigned int tail, int busy) "frame %x count %d context %" PRIx64 " head %x tail %x busy %d"
+megasas_qf_update(unsigned int head, unsigned int tail, unsigned int busy) "head %x tail %x busy %d"
+megasas_qf_map_failed(int cmd, unsigned long frame) "scmd %d: frame %lu"
+megasas_qf_complete_noirq(uint64_t context) "context %" PRIx64 " "
+megasas_qf_complete(uint64_t context, unsigned int head, unsigned int tail, int busy) "context %" PRIx64 " head %x tail %x busy %d"
+megasas_frame_busy(uint64_t addr) "frame %" PRIx64 " busy"
+megasas_unhandled_frame_cmd(int cmd, uint8_t frame_cmd) "scmd %d: MFI cmd %x"
+megasas_handle_scsi(const char *frame, int bus, int dev, int lun, void *sdev, unsigned long size) "%s dev %x/%x/%x sdev %p xfer %lu"
+megasas_scsi_target_not_present(const char *frame, int bus, int dev, int lun) "%s dev %x/%x/%x"
+megasas_scsi_invalid_cdb_len(const char *frame, int bus, int dev, int lun, int len) "%s dev %x/%x/%x invalid cdb len %d"
+megasas_iov_read_overflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
+megasas_iov_write_overflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
+megasas_iov_read_underflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
+megasas_iov_write_underflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
+megasas_scsi_req_alloc_failed(const char *frame, int dev, int lun) "%s dev %x/%x"
+megasas_scsi_read_start(int cmd, int len) "scmd %d: transfer %d bytes of data"
+megasas_scsi_write_start(int cmd, int len) "scmd %d: transfer %d bytes of data"
+megasas_scsi_nodata(int cmd) "scmd %d: no data to be transferred"
+megasas_scsi_complete(int cmd, uint32_t status, int len, int xfer) "scmd %d: status %x, len %u/%u"
+megasas_command_complete(int cmd, uint32_t status, uint32_t resid) "scmd %d: status %x, residual %d"
+megasas_handle_io(int cmd, const char *frame, int dev, int lun, unsigned long lba, unsigned long count) "scmd %d: %s dev %x/%x lba %lx count %lu"
+megasas_io_target_not_present(int cmd, const char *frame, int dev, int lun) "scmd %d: %s dev 1/%x/%x LUN not present"
+megasas_io_read_start(int cmd, unsigned long lba, unsigned long count, unsigned long len) "scmd %d: start LBA %lx %lu blocks (%lu bytes)"
+megasas_io_write_start(int cmd, unsigned long lba, unsigned long count, unsigned long len) "scmd %d: start LBA %lx %lu blocks (%lu bytes)"
+megasas_io_complete(int cmd, uint32_t len) "scmd %d: %d bytes"
+megasas_iovec_sgl_overflow(int cmd, int index, int limit) "scmd %d: iovec count %d limit %d"
+megasas_iovec_sgl_underflow(int cmd, int index) "scmd %d: iovec count %d"
+megasas_iovec_sgl_invalid(int cmd, int index, uint64_t pa, uint32_t len) "scmd %d: element %d pa %" PRIx64 " len %u"
+megasas_iovec_overflow(int cmd, int len, int limit) "scmd %d: len %d limit %d"
+megasas_iovec_underflow(int cmd, int len, int limit) "scmd %d: len %d limit %d"
+megasas_handle_dcmd(int cmd, int opcode) "scmd %d: MFI DCMD opcode %x"
+megasas_finish_dcmd(int cmd, int size) "scmd %d: MFI DCMD wrote %d bytes"
+megasas_dcmd_req_alloc_failed(int cmd, const char *desc) "scmd %d: %s"
+megasas_dcmd_internal_submit(int cmd, const char *desc, int dev) "scmd %d: %s to dev %d"
+megasas_dcmd_internal_finish(int cmd, int opcode, int lun) "scmd %d: cmd %x lun %d"
+megasas_dcmd_internal_invalid(int cmd, int opcode) "scmd %d: DCMD %x"
+megasas_dcmd_unhandled(int cmd, int opcode, int len) "scmd %d: opcode %x, len %d"
+megasas_dcmd_zero_sge(int cmd) "scmd %d: zero DCMD sge count"
+megasas_dcmd_invalid_sge(int cmd, int count) "scmd %d: DCMD sge count %d"
+megasas_dcmd_invalid_xfer_len(int cmd, unsigned long size, unsigned long max) "scmd %d: xfer len %ld, max %ld"
+megasas_dcmd_enter(int cmd, const char *dcmd, int len) "scmd %d: DCMD %s len %d"
+megasas_dcmd_dummy(int cmd, unsigned long size) "scmd %d: xfer len %ld"
+megasas_dcmd_set_fw_time(int cmd, unsigned long time) "scmd %d: Set FW time %lx"
+megasas_dcmd_pd_get_list(int cmd, int num, int max, int offset) "scmd %d: DCMD PD get list: %d / %d PDs, size %d"
+megasas_dcmd_ld_get_list(int cmd, int num, int max) "scmd %d: DCMD LD get list: found %d / %d LDs"
+megasas_dcmd_ld_get_info(int cmd, int ld_id) "scmd %d: dev %d"
+megasas_dcmd_ld_list_query(int cmd, int flags) "scmd %d: query flags %x"
+megasas_dcmd_pd_get_info(int cmd, int pd_id) "scmd %d: dev %d"
+megasas_dcmd_pd_list_query(int cmd, int flags) "scmd %d: query flags %x"
+megasas_dcmd_reset_ld(int cmd, int target_id) "scmd %d: dev %d"
+megasas_dcmd_unsupported(int cmd, unsigned long size) "scmd %d: set properties len %ld"
+megasas_abort_frame(int cmd, int abort_cmd) "scmd %d: frame %x"
+megasas_abort_no_cmd(int cmd, uint64_t context) "scmd %d: no active command for frame context %" PRIx64
+megasas_abort_invalid_context(int cmd, uint64_t context, int abort_cmd) "scmd %d: invalid frame context %" PRIx64 " for abort frame %x"
+megasas_reset(int fw_state) "firmware state %x"
+megasas_init(int sges, int cmds, const char *mode) "Using %d sges, %d cmds, %s mode"
+megasas_msix_raise(int vector) "vector %d"
+megasas_msi_raise(int vector) "vector %d"
+megasas_irq_lower(void) "INTx"
+megasas_irq_raise(void) "INTx"
+megasas_intr_enabled(void) "Interrupts enabled"
+megasas_intr_disabled(void) "Interrupts disabled"
+megasas_msix_enabled(int vector) "vector %d"
+megasas_msi_enabled(int vector) "vector %d"
+megasas_mmio_readl(const char *reg, uint32_t val) "reg %s: 0x%x"
+megasas_mmio_invalid_readl(unsigned long addr) "addr 0x%lx"
+megasas_mmio_writel(const char *reg, uint32_t val) "reg %s: 0x%x"
+megasas_mmio_invalid_writel(uint32_t addr, uint32_t val) "addr 0x%x: 0x%x"
+
+# hw/scsi/vmw_pvscsi.c
+pvscsi_ring_init_data(uint32_t txr_len_log2, uint32_t rxr_len_log2) "TX/RX rings logarithms set to %d/%d"
+pvscsi_ring_init_msg(uint32_t len_log2) "MSG ring logarithm set to %d"
+pvscsi_ring_flush_cmp(uint64_t filled_cmp_ptr) "new production counter of completion ring is 0x%"PRIx64
+pvscsi_ring_flush_msg(uint64_t filled_cmp_ptr) "new production counter of message ring is 0x%"PRIx64
+pvscsi_update_irq_level(bool raise, uint64_t mask, uint64_t status) "interrupt level set to %d (MASK: 0x%"PRIx64", STATUS: 0x%"PRIx64")"
+pvscsi_update_irq_msi(void) "sending MSI notification"
+pvscsi_cmp_ring_put(unsigned long addr) "got completion descriptor 0x%lx"
+pvscsi_msg_ring_put(unsigned long addr) "got message descriptor 0x%lx"
+pvscsi_complete_request(uint64_t context, uint64_t len, uint8_t sense_key) "completion: ctx: 0x%"PRIx64", len: 0x%"PRIx64", sense key: %u"
+pvscsi_get_sg_list(int nsg, size_t size) "get SG list: depth: %u, size: %zu"
+pvscsi_get_next_sg_elem(uint32_t flags) "unknown flags in SG element (val: 0x%x)"
+pvscsi_command_complete_not_found(uint32_t tag) "can't find request for tag 0x%x"
+pvscsi_command_complete_data_run(void) "not all data required for command transferred"
+pvscsi_command_complete_sense_len(int len) "sense information length is %d bytes"
+pvscsi_convert_sglist(uint64_t context, unsigned long addr, uint32_t resid) "element: ctx: 0x%"PRIx64" addr: 0x%lx, len: %ul"
+pvscsi_process_req_descr(uint8_t cmd, uint64_t ctx) "SCSI cmd 0x%x, ctx: 0x%"PRIx64
+pvscsi_process_req_descr_unknown_device(void) "command directed to unknown device rejected"
+pvscsi_process_req_descr_invalid_dir(void) "command with invalid transfer direction rejected"
+pvscsi_process_io(unsigned long addr) "got descriptor 0x%lx"
+pvscsi_on_cmd_noimpl(const char* cmd) "unimplemented command %s ignored"
+pvscsi_on_cmd_reset_dev(uint32_t tgt, int lun, void* dev) "PVSCSI_CMD_RESET_DEVICE[target %u lun %d (dev 0x%p)]"
+pvscsi_on_cmd_arrived(const char* cmd) "command %s arrived"
+pvscsi_on_cmd_abort(uint64_t ctx, uint32_t tgt) "command PVSCSI_CMD_ABORT_CMD for ctx 0x%"PRIx64", target %u"
+pvscsi_on_cmd_unknown(uint64_t cmd_id) "unknown command %"PRIx64
+pvscsi_on_cmd_unknown_data(uint32_t data) "data for unknown command 0x:%x"
+pvscsi_io_write(const char* cmd, uint64_t val) "%s write: %"PRIx64
+pvscsi_io_write_unknown(unsigned long addr, unsigned sz, uint64_t val) "unknown write address: 0x%lx size: %u bytes value: 0x%"PRIx64
+pvscsi_io_read(const char* cmd, uint64_t status) "%s read: 0x%"PRIx64
+pvscsi_io_read_unknown(unsigned long addr, unsigned sz) "unknown read address: 0x%lx size: %u bytes"
+pvscsi_init_msi_fail(int res) "failed to initialize MSI, error %d"
+pvscsi_state(const char* state) "starting %s ..."
+pvscsi_tx_rings_ppn(const char* label, uint64_t ppn) "%s page: %"PRIx64
+pvscsi_tx_rings_num_pages(const char* label, uint32_t num) "Number of %s pages: %u"
+
+# hw/scsi/esp.c
+esp_error_fifo_overrun(void) "FIFO overrun"
+esp_error_unhandled_command(uint32_t val) "unhandled command (%2.2x)"
+esp_error_invalid_write(uint32_t val, uint32_t addr) "invalid write of 0x%02x at [0x%x]"
+esp_raise_irq(void) "Raise IRQ"
+esp_lower_irq(void) "Lower IRQ"
+esp_dma_enable(void) "Raise enable"
+esp_dma_disable(void) "Lower enable"
+esp_get_cmd(uint32_t dmalen, int target) "len %d target %d"
+esp_do_busid_cmd(uint8_t busid) "busid 0x%x"
+esp_handle_satn_stop(uint32_t cmdlen) "cmdlen %d"
+esp_write_response(uint32_t status) "Transfer status (status=%d)"
+esp_do_dma(uint32_t cmdlen, uint32_t len) "command len %d + %d"
+esp_command_complete(void) "SCSI Command complete"
+esp_command_complete_unexpected(void) "SCSI command completed unexpectedly"
+esp_command_complete_fail(void) "Command failed"
+esp_transfer_data(uint32_t dma_left, int32_t ti_size) "transfer %d/%d"
+esp_handle_ti(uint32_t minlen) "Transfer Information len %d"
+esp_handle_ti_cmd(uint32_t cmdlen) "command len %d"
+esp_mem_readb(uint32_t saddr, uint8_t reg) "reg[%d]: 0x%2.2x"
+esp_mem_writeb(uint32_t saddr, uint8_t reg, uint32_t val) "reg[%d]: 0x%2.2x -> 0x%2.2x"
+esp_mem_writeb_cmd_nop(uint32_t val) "NOP (%2.2x)"
+esp_mem_writeb_cmd_flush(uint32_t val) "Flush FIFO (%2.2x)"
+esp_mem_writeb_cmd_reset(uint32_t val) "Chip reset (%2.2x)"
+esp_mem_writeb_cmd_bus_reset(uint32_t val) "Bus reset (%2.2x)"
+esp_mem_writeb_cmd_iccs(uint32_t val) "Initiator Command Complete Sequence (%2.2x)"
+esp_mem_writeb_cmd_msgacc(uint32_t val) "Message Accepted (%2.2x)"
+esp_mem_writeb_cmd_pad(uint32_t val) "Transfer padding (%2.2x)"
+esp_mem_writeb_cmd_satn(uint32_t val) "Set ATN (%2.2x)"
+esp_mem_writeb_cmd_rstatn(uint32_t val) "Reset ATN (%2.2x)"
+esp_mem_writeb_cmd_sel(uint32_t val) "Select without ATN (%2.2x)"
+esp_mem_writeb_cmd_selatn(uint32_t val) "Select with ATN (%2.2x)"
+esp_mem_writeb_cmd_selatns(uint32_t val) "Select with ATN & stop (%2.2x)"
+esp_mem_writeb_cmd_ensel(uint32_t val) "Enable selection (%2.2x)"
+esp_mem_writeb_cmd_dissel(uint32_t val) "Disable selection (%2.2x)"
+
+# hw/scsi/esp-pci.c
+esp_pci_error_invalid_dma_direction(void) "invalid DMA transfer direction"
+esp_pci_error_invalid_read(uint32_t reg) "read access outside bounds (reg 0x%x)"
+esp_pci_error_invalid_write(uint32_t reg) "write access outside bounds (reg 0x%x)"
+esp_pci_error_invalid_write_dma(uint32_t val, uint32_t addr) "invalid write of 0x%02x at [0x%x]"
+esp_pci_dma_read(uint32_t saddr, uint32_t reg) "reg[%d]: 0x%8.8x"
+esp_pci_dma_write(uint32_t saddr, uint32_t reg, uint32_t val) "reg[%d]: 0x%8.8x -> 0x%8.8x"
+esp_pci_dma_idle(uint32_t val) "IDLE (%.8x)"
+esp_pci_dma_blast(uint32_t val) "BLAST (%.8x)"
+esp_pci_dma_abort(uint32_t val) "ABORT (%.8x)"
+esp_pci_dma_start(uint32_t val) "START (%.8x)"
+esp_pci_sbac_read(uint32_t reg) "sbac: 0x%8.8x"
+esp_pci_sbac_write(uint32_t reg, uint32_t val) "sbac: 0x%8.8x -> 0x%8.8x"
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index dcb2bc5..5b26946 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -14,8 +14,11 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
 #include <sys/ioctl.h>
-#include "config.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
 #include "qemu/queue.h"
 #include "monitor/monitor.h"
 #include "migration/migration.h"
@@ -24,6 +27,8 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
+#include "hw/fw-path-provider.h"
+#include "qemu/cutils.h"
 
 /* Features supported by host kernel. */
 static const int kernel_feature_bits[] = {
@@ -43,7 +48,7 @@
 
     memset(&backend, 0, sizeof(backend));
     pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->conf.wwpn);
-    ret = vhost_ops->vhost_call(&s->dev, VHOST_SCSI_SET_ENDPOINT, &backend);
+    ret = vhost_ops->vhost_scsi_set_endpoint(&s->dev, &backend);
     if (ret < 0) {
         return -errno;
     }
@@ -58,7 +63,7 @@
 
     memset(&backend, 0, sizeof(backend));
     pstrcpy(backend.vhost_wwpn, sizeof(backend.vhost_wwpn), vs->conf.wwpn);
-    vhost_ops->vhost_call(&s->dev, VHOST_SCSI_CLEAR_ENDPOINT, &backend);
+    vhost_ops->vhost_scsi_clear_endpoint(&s->dev, &backend);
 }
 
 static int vhost_scsi_start(VHostSCSI *s)
@@ -74,15 +79,14 @@
         return -ENOSYS;
     }
 
-    ret = vhost_ops->vhost_call(&s->dev,
-                                VHOST_SCSI_GET_ABI_VERSION, &abi_version);
+    ret = vhost_ops->vhost_scsi_get_abi_version(&s->dev, &abi_version);
     if (ret < 0) {
         return -errno;
     }
     if (abi_version > VHOST_SCSI_ABI_VERSION) {
         error_report("vhost-scsi: The running tcm_vhost kernel abi_version:"
                      " %d is greater than vhost_scsi userspace supports: %d, please"
-                     " upgrade your version of QEMU\n", abi_version,
+                     " upgrade your version of QEMU", abi_version,
                      VHOST_SCSI_ABI_VERSION);
         return -ENOSYS;
     }
@@ -116,7 +120,7 @@
      * enabling/disabling irqfd.
      */
     for (i = 0; i < s->dev.nvqs; i++) {
-        vhost_virtqueue_mask(&s->dev, vdev, i, false);
+        vhost_virtqueue_mask(&s->dev, vdev, s->dev.vq_index + i, false);
     }
 
     return ret;
@@ -140,7 +144,7 @@
     if (k->set_guest_notifiers) {
         ret = k->set_guest_notifiers(qbus->parent, s->dev.nvqs, false);
         if (ret < 0) {
-                error_report("vhost guest notifier cleanup failed: %d\n", ret);
+                error_report("vhost guest notifier cleanup failed: %d", ret);
         }
     }
     assert(ret >= 0);
@@ -150,8 +154,9 @@
     vhost_dev_disable_notifiers(&s->dev, vdev);
 }
 
-static uint32_t vhost_scsi_get_features(VirtIODevice *vdev,
-                                        uint32_t features)
+static uint64_t vhost_scsi_get_features(VirtIODevice *vdev,
+                                        uint64_t features,
+                                        Error **errp)
 {
     VHostSCSI *s = VHOST_SCSI(vdev);
 
@@ -185,7 +190,7 @@
 
         ret = vhost_scsi_start(s);
         if (ret < 0) {
-            error_report("virtio-scsi: unable to start vhost: %s\n",
+            error_report("virtio-scsi: unable to start vhost: %s",
                          strerror(-ret));
 
             /* There is no userspace virtio-scsi fallback so exit */
@@ -214,9 +219,9 @@
     }
 
     if (vs->conf.vhostfd) {
-        vhostfd = monitor_handle_fd_param(cur_mon, vs->conf.vhostfd);
+        vhostfd = monitor_fd_param(cur_mon, vs->conf.vhostfd, errp);
         if (vhostfd == -1) {
-            error_setg(errp, "vhost-scsi: unable to parse vhostfd");
+            error_prepend(errp, "vhost-scsi: unable to parse vhostfd: ");
             return;
         }
     } else {
@@ -243,13 +248,19 @@
     s->dev.backend_features = 0;
 
     ret = vhost_dev_init(&s->dev, (void *)(uintptr_t)vhostfd,
-                         VHOST_BACKEND_TYPE_KERNEL, true);
+                         VHOST_BACKEND_TYPE_KERNEL, 0);
     if (ret < 0) {
         error_setg(errp, "vhost-scsi: vhost initialization failed: %s",
                    strerror(-ret));
         return;
     }
 
+    /* At present, channel and lun both are 0 for bootable vhost-scsi disk */
+    s->channel = 0;
+    s->lun = 0;
+    /* Note: we can also get the minimum tpgt from kernel */
+    s->target = vs->conf.boot_tpgt;
+
     error_setg(&s->migration_blocker,
             "vhost-scsi does not support migration");
     migrate_add_blocker(s->migration_blocker);
@@ -266,13 +277,34 @@
     /* This will stop vhost backend. */
     vhost_scsi_set_status(vdev, 0);
 
+    vhost_dev_cleanup(&s->dev);
     g_free(s->dev.vqs);
 
     virtio_scsi_common_unrealize(dev, errp);
 }
 
+/*
+ * Implementation of an interface to adjust firmware path
+ * for the bootindex property handling.
+ */
+static char *vhost_scsi_get_fw_dev_path(FWPathProvider *p, BusState *bus,
+                                        DeviceState *dev)
+{
+    VHostSCSI *s = VHOST_SCSI(dev);
+    /* format: channel@channel/vhost-scsi@target,lun */
+    return g_strdup_printf("/channel@%x/%s@%x,%x", s->channel,
+                           qdev_fw_name(dev), s->target, s->lun);
+}
+
 static Property vhost_scsi_properties[] = {
-    DEFINE_VHOST_SCSI_PROPERTIES(VHostSCSI, parent_obj.conf),
+    DEFINE_PROP_STRING("vhostfd", VHostSCSI, parent_obj.conf.vhostfd),
+    DEFINE_PROP_STRING("wwpn", VHostSCSI, parent_obj.conf.wwpn),
+    DEFINE_PROP_UINT32("boot_tpgt", VHostSCSI, parent_obj.conf.boot_tpgt, 0),
+    DEFINE_PROP_UINT32("num_queues", VHostSCSI, parent_obj.conf.num_queues, 1),
+    DEFINE_PROP_UINT32("max_sectors", VHostSCSI, parent_obj.conf.max_sectors,
+                                                 0xFFFF),
+    DEFINE_PROP_UINT32("cmd_per_lun", VHostSCSI, parent_obj.conf.cmd_per_lun,
+                                                 128),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -280,6 +312,7 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
+    FWPathProviderClass *fwc = FW_PATH_PROVIDER_CLASS(klass);
 
     dc->props = vhost_scsi_properties;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
@@ -288,6 +321,15 @@
     vdc->get_features = vhost_scsi_get_features;
     vdc->set_config = vhost_scsi_set_config;
     vdc->set_status = vhost_scsi_set_status;
+    fwc->get_dev_path = vhost_scsi_get_fw_dev_path;
+}
+
+static void vhost_scsi_instance_init(Object *obj)
+{
+    VHostSCSI *dev = VHOST_SCSI(obj);
+
+    device_add_bootindex_property(obj, &dev->bootindex, "bootindex", NULL,
+                                  DEVICE(dev), NULL);
 }
 
 static const TypeInfo vhost_scsi_info = {
@@ -295,6 +337,11 @@
     .parent = TYPE_VIRTIO_SCSI_COMMON,
     .instance_size = sizeof(VHostSCSI),
     .class_init = vhost_scsi_class_init,
+    .instance_init = vhost_scsi_instance_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_FW_PATH_PROVIDER },
+        { }
+    },
 };
 
 static void virtio_register_types(void)
diff --git a/hw/scsi/virtio-scsi-dataplane.c b/hw/scsi/virtio-scsi-dataplane.c
index 03a1e8c..b173b94 100644
--- a/hw/scsi/virtio-scsi-dataplane.c
+++ b/hw/scsi/virtio-scsi-dataplane.c
@@ -11,14 +11,14 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
-#include <hw/scsi/scsi.h>
-#include <block/scsi.h>
-#include <hw/virtio/virtio-bus.h>
+#include "hw/scsi/scsi.h"
+#include "block/scsi.h"
+#include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
-#include "stdio.h"
 
 /* Context: QEMU global mutex held */
 void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread)
@@ -31,125 +31,63 @@
     s->ctx = iothread_get_aio_context(vs->conf.iothread);
 
     /* Don't try if transport does not support notifiers. */
-    if (!k->set_guest_notifiers || !k->set_host_notifier) {
+    if (!k->set_guest_notifiers || !k->ioeventfd_started) {
         fprintf(stderr, "virtio-scsi: Failed to set iothread "
                    "(transport does not support notifiers)");
         exit(1);
     }
 }
 
-static VirtIOSCSIVring *virtio_scsi_vring_init(VirtIOSCSI *s,
-                                               VirtQueue *vq,
-                                               EventNotifierHandler *handler,
-                                               int n)
+static void virtio_scsi_data_plane_handle_cmd(VirtIODevice *vdev,
+                                              VirtQueue *vq)
+{
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+
+    assert(s->ctx && s->dataplane_started);
+    virtio_scsi_handle_cmd_vq(s, vq);
+}
+
+static void virtio_scsi_data_plane_handle_ctrl(VirtIODevice *vdev,
+                                               VirtQueue *vq)
+{
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+
+    assert(s->ctx && s->dataplane_started);
+    virtio_scsi_handle_ctrl_vq(s, vq);
+}
+
+static void virtio_scsi_data_plane_handle_event(VirtIODevice *vdev,
+                                                VirtQueue *vq)
+{
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+
+    assert(s->ctx && s->dataplane_started);
+    virtio_scsi_handle_event_vq(s, vq);
+}
+
+static int virtio_scsi_vring_init(VirtIOSCSI *s, VirtQueue *vq, int n,
+                                  void (*fn)(VirtIODevice *vdev, VirtQueue *vq))
 {
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(s)));
-    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
-    VirtIOSCSIVring *r = g_slice_new(VirtIOSCSIVring);
     int rc;
 
     /* Set up virtqueue notify */
-    rc = k->set_host_notifier(qbus->parent, n, true);
+    rc = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), n, true);
     if (rc != 0) {
         fprintf(stderr, "virtio-scsi: Failed to set host notifier (%d)\n",
                 rc);
         s->dataplane_fenced = true;
-        return NULL;
+        return rc;
     }
-    r->host_notifier = *virtio_queue_get_host_notifier(vq);
-    r->guest_notifier = *virtio_queue_get_guest_notifier(vq);
-    aio_set_event_notifier(s->ctx, &r->host_notifier, handler);
 
-    r->parent = s;
-
-    if (!vring_setup(&r->vring, VIRTIO_DEVICE(s), n)) {
-        fprintf(stderr, "virtio-scsi: VRing setup failed\n");
-        goto fail_vring;
-    }
-    return r;
-
-fail_vring:
-    aio_set_event_notifier(s->ctx, &r->host_notifier, NULL);
-    k->set_host_notifier(qbus->parent, n, false);
-    g_slice_free(VirtIOSCSIVring, r);
-    return NULL;
+    virtio_queue_aio_set_host_notifier_handler(vq, s->ctx, fn);
+    return 0;
 }
 
-VirtIOSCSIReq *virtio_scsi_pop_req_vring(VirtIOSCSI *s,
-                                         VirtIOSCSIVring *vring)
+void virtio_scsi_dataplane_notify(VirtIODevice *vdev, VirtIOSCSIReq *req)
 {
-    VirtIOSCSIReq *req = virtio_scsi_init_req(s, NULL);
-    int r;
-
-    req->vring = vring;
-    r = vring_pop((VirtIODevice *)s, &vring->vring, &req->elem);
-    if (r < 0) {
-        virtio_scsi_free_req(req);
-        req = NULL;
-    }
-    return req;
-}
-
-void virtio_scsi_vring_push_notify(VirtIOSCSIReq *req)
-{
-    VirtIODevice *vdev = VIRTIO_DEVICE(req->vring->parent);
-
-    vring_push(&req->vring->vring, &req->elem,
-               req->qsgl.size + req->resp_iov.size);
-
-    if (vring_should_notify(vdev, &req->vring->vring)) {
-        event_notifier_set(&req->vring->guest_notifier);
-    }
-}
-
-static void virtio_scsi_iothread_handle_ctrl(EventNotifier *notifier)
-{
-    VirtIOSCSIVring *vring = container_of(notifier,
-                                          VirtIOSCSIVring, host_notifier);
-    VirtIOSCSI *s = VIRTIO_SCSI(vring->parent);
-    VirtIOSCSIReq *req;
-
-    event_notifier_test_and_clear(notifier);
-    while ((req = virtio_scsi_pop_req_vring(s, vring))) {
-        virtio_scsi_handle_ctrl_req(s, req);
-    }
-}
-
-static void virtio_scsi_iothread_handle_event(EventNotifier *notifier)
-{
-    VirtIOSCSIVring *vring = container_of(notifier,
-                                          VirtIOSCSIVring, host_notifier);
-    VirtIOSCSI *s = vring->parent;
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
-
-    event_notifier_test_and_clear(notifier);
-
-    if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) {
-        return;
-    }
-
-    if (s->events_dropped) {
-        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
-    }
-}
-
-static void virtio_scsi_iothread_handle_cmd(EventNotifier *notifier)
-{
-    VirtIOSCSIVring *vring = container_of(notifier,
-                                          VirtIOSCSIVring, host_notifier);
-    VirtIOSCSI *s = (VirtIOSCSI *)vring->parent;
-    VirtIOSCSIReq *req, *next;
-    QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
-
-    event_notifier_test_and_clear(notifier);
-    while ((req = virtio_scsi_pop_req_vring(s, vring))) {
-        if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
-            QTAILQ_INSERT_TAIL(&reqs, req, next);
-        }
-    }
-
-    QTAILQ_FOREACH_SAFE(req, &reqs, next, next) {
-        virtio_scsi_handle_cmd_req_submit(s, req);
+    if (virtio_should_notify(vdev, req->vq)) {
+        event_notifier_set(virtio_queue_get_guest_notifier(req->vq));
     }
 }
 
@@ -159,37 +97,10 @@
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
     int i;
 
-    if (s->ctrl_vring) {
-        aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
-    }
-    if (s->event_vring) {
-        aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
-    }
-    if (s->cmd_vrings) {
-        for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
-            aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
-        }
-    }
-}
-
-static void virtio_scsi_vring_teardown(VirtIOSCSI *s)
-{
-    VirtIODevice *vdev = VIRTIO_DEVICE(s);
-    VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
-    int i;
-
-    if (s->ctrl_vring) {
-        vring_teardown(&s->ctrl_vring->vring, vdev, 0);
-    }
-    if (s->event_vring) {
-        vring_teardown(&s->event_vring->vring, vdev, 1);
-    }
-    if (s->cmd_vrings) {
-        for (i = 0; i < vs->conf.num_queues && s->cmd_vrings[i]; i++) {
-            vring_teardown(&s->cmd_vrings[i]->vring, vdev, 2 + i);
-        }
-        free(s->cmd_vrings);
-        s->cmd_vrings = NULL;
+    virtio_queue_aio_set_host_notifier_handler(vs->ctrl_vq, s->ctx, NULL);
+    virtio_queue_aio_set_host_notifier_handler(vs->event_vq, s->ctx, NULL);
+    for (i = 0; i < vs->conf.num_queues; i++) {
+        virtio_queue_aio_set_host_notifier_handler(vs->cmd_vqs[i], s->ctx, NULL);
     }
 }
 
@@ -211,37 +122,29 @@
 
     s->dataplane_starting = true;
 
-    assert(!s->blocker);
-    error_setg(&s->blocker, "block device is in use by data plane");
     /* Set up guest notifier (irq) */
     rc = k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, true);
     if (rc != 0) {
         fprintf(stderr, "virtio-scsi: Failed to set guest notifiers (%d), "
                 "ensure -enable-kvm is set\n", rc);
-        s->dataplane_fenced = true;
         goto fail_guest_notifiers;
     }
 
     aio_context_acquire(s->ctx);
-    s->ctrl_vring = virtio_scsi_vring_init(s, vs->ctrl_vq,
-                                           virtio_scsi_iothread_handle_ctrl,
-                                           0);
-    if (!s->ctrl_vring) {
+    rc = virtio_scsi_vring_init(s, vs->ctrl_vq, 0,
+                                virtio_scsi_data_plane_handle_ctrl);
+    if (rc) {
         goto fail_vrings;
     }
-    s->event_vring = virtio_scsi_vring_init(s, vs->event_vq,
-                                            virtio_scsi_iothread_handle_event,
-                                            1);
-    if (!s->event_vring) {
+    rc = virtio_scsi_vring_init(s, vs->event_vq, 1,
+                                virtio_scsi_data_plane_handle_event);
+    if (rc) {
         goto fail_vrings;
     }
-    s->cmd_vrings = g_new(VirtIOSCSIVring *, vs->conf.num_queues);
     for (i = 0; i < vs->conf.num_queues; i++) {
-        s->cmd_vrings[i] =
-            virtio_scsi_vring_init(s, vs->cmd_vqs[i],
-                                   virtio_scsi_iothread_handle_cmd,
-                                   i + 2);
-        if (!s->cmd_vrings[i]) {
+        rc = virtio_scsi_vring_init(s, vs->cmd_vqs[i], i + 2,
+                                    virtio_scsi_data_plane_handle_cmd);
+        if (rc) {
             goto fail_vrings;
         }
     }
@@ -254,13 +157,14 @@
 fail_vrings:
     virtio_scsi_clear_aio(s);
     aio_context_release(s->ctx);
-    virtio_scsi_vring_teardown(s);
     for (i = 0; i < vs->conf.num_queues + 2; i++) {
-        k->set_host_notifier(qbus->parent, i, false);
+        virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
     }
     k->set_guest_notifiers(qbus->parent, vs->conf.num_queues + 2, false);
 fail_guest_notifiers:
+    s->dataplane_fenced = true;
     s->dataplane_starting = false;
+    s->dataplane_started = true;
 }
 
 /* Context: QEMU global mutex held */
@@ -271,38 +175,29 @@
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
     int i;
 
-    /* Better luck next time. */
-    if (s->dataplane_fenced) {
-        s->dataplane_fenced = false;
-        return;
-    }
     if (!s->dataplane_started || s->dataplane_stopping) {
         return;
     }
-    error_free(s->blocker);
-    s->blocker = NULL;
+
+    /* Better luck next time. */
+    if (s->dataplane_fenced) {
+        s->dataplane_fenced = false;
+        s->dataplane_started = false;
+        return;
+    }
     s->dataplane_stopping = true;
     assert(s->ctx == iothread_get_aio_context(vs->conf.iothread));
 
     aio_context_acquire(s->ctx);
 
-    aio_set_event_notifier(s->ctx, &s->ctrl_vring->host_notifier, NULL);
-    aio_set_event_notifier(s->ctx, &s->event_vring->host_notifier, NULL);
-    for (i = 0; i < vs->conf.num_queues; i++) {
-        aio_set_event_notifier(s->ctx, &s->cmd_vrings[i]->host_notifier, NULL);
-    }
+    virtio_scsi_clear_aio(s);
 
     blk_drain_all(); /* ensure there are no in-flight requests */
 
     aio_context_release(s->ctx);
 
-    /* Sync vring state back to virtqueue so that non-dataplane request
-     * processing can continue when we disable the host notifier below.
-     */
-    virtio_scsi_vring_teardown(s);
-
     for (i = 0; i < vs->conf.num_queues + 2; i++) {
-        k->set_host_notifier(qbus->parent, i, false);
+        virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), i, false);
     }
 
     /* Clean up guest notifier (irq) */
diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c
index 9da98f5..db64574 100644
--- a/hw/scsi/virtio-scsi.c
+++ b/hw/scsi/virtio-scsi.c
@@ -13,15 +13,17 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "standard-headers/linux/virtio_ids.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "sysemu/block-backend.h"
-#include <hw/scsi/scsi.h>
-#include <block/scsi.h>
-#include <hw/virtio/virtio-bus.h>
+#include "hw/scsi/scsi.h"
+#include "block/scsi.h"
+#include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
-#include "migration/migration.h"
 
 static inline int virtio_scsi_get_lun(uint8_t *lun)
 {
@@ -39,29 +41,23 @@
     return scsi_device_find(&s->bus, 0, lun[1], virtio_scsi_get_lun(lun));
 }
 
-VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq)
+void virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq, VirtIOSCSIReq *req)
 {
-    VirtIOSCSIReq *req;
-    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)s;
-    const size_t zero_skip = offsetof(VirtIOSCSIReq, elem)
-                             + sizeof(VirtQueueElement);
+    const size_t zero_skip =
+        offsetof(VirtIOSCSIReq, resp_iov) + sizeof(req->resp_iov);
 
-    req = g_slice_alloc(sizeof(*req) + vs->cdb_size);
     req->vq = vq;
     req->dev = s;
     qemu_sglist_init(&req->qsgl, DEVICE(s), 8, &address_space_memory);
     qemu_iovec_init(&req->resp_iov, 1);
     memset((uint8_t *)req + zero_skip, 0, sizeof(*req) - zero_skip);
-    return req;
 }
 
 void virtio_scsi_free_req(VirtIOSCSIReq *req)
 {
-    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)req->dev;
-
     qemu_iovec_destroy(&req->resp_iov);
     qemu_sglist_destroy(&req->qsgl);
-    g_slice_free1(sizeof(*req) + vs->cdb_size, req);
+    g_free(req);
 }
 
 static void virtio_scsi_complete_req(VirtIOSCSIReq *req)
@@ -71,11 +67,10 @@
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
     qemu_iovec_from_buf(&req->resp_iov, 0, &req->resp, req->resp_size);
-    if (req->vring) {
-        assert(req->vq == NULL);
-        virtio_scsi_vring_push_notify(req);
+    virtqueue_push(vq, &req->elem, req->qsgl.size + req->resp_iov.size);
+    if (s->dataplane_started && !s->dataplane_fenced) {
+        virtio_scsi_dataplane_notify(vdev, req);
     } else {
-        virtqueue_push(vq, &req->elem, req->qsgl.size + req->resp_iov.size);
         virtio_notify(vdev, vq);
     }
 
@@ -144,9 +139,13 @@
      *
      * TODO: always disable this workaround for virtio 1.0 devices.
      */
-    if ((vdev->guest_features & VIRTIO_F_ANY_LAYOUT) == 0) {
-        req_size = req->elem.out_sg[0].iov_len;
-        resp_size = req->elem.in_sg[0].iov_len;
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_ANY_LAYOUT)) {
+        if (req->elem.out_num) {
+            req_size = req->elem.out_sg[0].iov_len;
+        }
+        if (req->elem.in_num) {
+            resp_size = req->elem.in_sg[0].iov_len;
+        }
     }
 
     out_size = qemu_sgl_concat(req, req->elem.out_sg,
@@ -171,11 +170,14 @@
 
 static VirtIOSCSIReq *virtio_scsi_pop_req(VirtIOSCSI *s, VirtQueue *vq)
 {
-    VirtIOSCSIReq *req = virtio_scsi_init_req(s, vq);
-    if (!virtqueue_pop(vq, &req->elem)) {
-        virtio_scsi_free_req(req);
+    VirtIOSCSICommon *vs = (VirtIOSCSICommon *)s;
+    VirtIOSCSIReq *req;
+
+    req = virtqueue_pop(vq, sizeof(VirtIOSCSIReq) + vs->cdb_size);
+    if (!req) {
         return NULL;
     }
+    virtio_scsi_init_req(s, vq, req);
     return req;
 }
 
@@ -183,12 +185,12 @@
 {
     VirtIOSCSIReq *req = sreq->hba_private;
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(req->dev);
-    uint32_t n = virtio_queue_get_id(req->vq) - 2;
+    uint32_t n = virtio_get_queue_index(req->vq) - 2;
 
     assert(n < vs->conf.num_queues);
     (void)vs;
     qemu_put_be32s(f, &n);
-    qemu_put_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
+    qemu_put_virtqueue_element(f, &req->elem);
 }
 
 static void *virtio_scsi_load_request(QEMUFile *f, SCSIRequest *sreq)
@@ -201,17 +203,8 @@
 
     qemu_get_be32s(f, &n);
     assert(n < vs->conf.num_queues);
-    req = virtio_scsi_init_req(s, vs->cmd_vqs[n]);
-    qemu_get_buffer(f, (unsigned char *)&req->elem, sizeof(req->elem));
-    /* TODO: add a way for SCSIBusInfo's load_request to fail,
-     * and fail migration instead of aborting here.
-     */
-    if (!(req->elem.in_num <= ARRAY_SIZE(req->elem.in_sg))) {
-        abort();
-    }
-    if (!(req->elem.out_num <= ARRAY_SIZE(req->elem.out_sg))) {
-        abort();
-    }
+    req = qemu_get_virtqueue_element(f, sizeof(VirtIOSCSIReq) + vs->cdb_size);
+    virtio_scsi_init_req(s, vs->cmd_vqs[n], req);
 
     if (virtio_scsi_parse_req(req, sizeof(VirtIOSCSICmdReq) + vs->cdb_size,
                               sizeof(VirtIOSCSICmdResp) + vs->sense_size) < 0) {
@@ -243,7 +236,7 @@
     if (--n->tmf_req->remaining == 0) {
         virtio_scsi_complete_req(n->tmf_req);
     }
-    g_slice_free(VirtIOSCSICancelNotifier, n);
+    g_free(n);
 }
 
 /* Return 0 if the request is ready to be completed and return to guest;
@@ -257,10 +250,8 @@
     int target;
     int ret = 0;
 
-    if (s->dataplane_started && blk_get_aio_context(d->conf.blk) != s->ctx) {
-        aio_context_acquire(s->ctx);
-        blk_set_aio_context(d->conf.blk, s->ctx);
-        aio_context_release(s->ctx);
+    if (s->dataplane_started && d) {
+        assert(blk_get_aio_context(d->conf.blk) == s->ctx);
     }
     /* Here VIRTIO_SCSI_S_OK means "FUNCTION COMPLETE".  */
     req->resp.tmf.response = VIRTIO_SCSI_S_OK;
@@ -296,7 +287,7 @@
                 VirtIOSCSICancelNotifier *notifier;
 
                 req->remaining = 1;
-                notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                notifier = g_new(VirtIOSCSICancelNotifier, 1);
                 notifier->tmf_req = req;
                 notifier->notifier.notify = virtio_scsi_cancel_notify;
                 scsi_req_cancel_async(r, &notifier->notifier);
@@ -345,7 +336,7 @@
                     VirtIOSCSICancelNotifier *notifier;
 
                     req->remaining++;
-                    notifier = g_slice_new(VirtIOSCSICancelNotifier);
+                    notifier = g_new(VirtIOSCSICancelNotifier, 1);
                     notifier->notifier.notify = virtio_scsi_cancel_notify;
                     notifier->tmf_req = req;
                     scsi_req_cancel_async(r, &notifier->notifier);
@@ -361,7 +352,7 @@
         target = req->req.tmf.lun[1];
         s->resetting++;
         QTAILQ_FOREACH(kid, &s->bus.qbus.children, sibling) {
-             d = DO_UPCAST(SCSIDevice, qdev, kid->child);
+             d = SCSI_DEVICE(kid->child);
              if (d->channel == 0 && d->id == target) {
                 qdev_reset_all(&d->qdev);
              }
@@ -386,7 +377,7 @@
     return ret;
 }
 
-void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
+static void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     VirtIODevice *vdev = (VirtIODevice *)s;
     uint32_t type;
@@ -424,20 +415,28 @@
     }
 }
 
-static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_scsi_handle_ctrl_vq(VirtIOSCSI *s, VirtQueue *vq)
 {
-    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
     VirtIOSCSIReq *req;
 
-    if (s->ctx && !s->dataplane_disabled) {
-        virtio_scsi_dataplane_start(s);
-        return;
-    }
     while ((req = virtio_scsi_pop_req(s, vq))) {
         virtio_scsi_handle_ctrl_req(s, req);
     }
 }
 
+static void virtio_scsi_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+
+    if (s->ctx) {
+        virtio_scsi_dataplane_start(s);
+        if (!s->dataplane_fenced) {
+            return;
+        }
+    }
+    virtio_scsi_handle_ctrl_vq(s, vq);
+}
+
 static void virtio_scsi_complete_cmd_req(VirtIOSCSIReq *req)
 {
     /* Sense data is not in req->resp and is copied separately
@@ -480,7 +479,7 @@
     VirtIOSCSIReq *req = hba_private;
 
     if (cmd->len == 0) {
-        cmd->len = MIN(VIRTIO_SCSI_CDB_SIZE, SCSI_CMD_BUF_SIZE);
+        cmd->len = MIN(VIRTIO_SCSI_CDB_DEFAULT_SIZE, SCSI_CMD_BUF_SIZE);
         memcpy(cmd->buf, buf, cmd->len);
     }
 
@@ -520,7 +519,7 @@
     virtio_scsi_complete_cmd_req(req);
 }
 
-bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
+static bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     VirtIOSCSICommon *vs = &s->parent_obj;
     SCSIDevice *d;
@@ -543,14 +542,12 @@
         virtio_scsi_complete_cmd_req(req);
         return false;
     }
-    if (s->dataplane_started && blk_get_aio_context(d->conf.blk) != s->ctx) {
-        aio_context_acquire(s->ctx);
-        blk_set_aio_context(d->conf.blk, s->ctx);
-        aio_context_release(s->ctx);
+    if (s->dataplane_started) {
+        assert(blk_get_aio_context(d->conf.blk) == s->ctx);
     }
     req->sreq = scsi_req_new(d, req->req.cmd.tag,
                              virtio_scsi_get_lun(req->req.cmd.lun),
-                             req->req.cdb, req);
+                             req->req.cmd.cdb, req);
 
     if (req->sreq->cmd.mode != SCSI_XFER_NONE
         && (req->sreq->cmd.mode != req->mode ||
@@ -564,7 +561,7 @@
     return true;
 }
 
-void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
+static void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req)
 {
     SCSIRequest *sreq = req->sreq;
     if (scsi_req_enqueue(sreq)) {
@@ -574,17 +571,11 @@
     scsi_req_unref(sreq);
 }
 
-static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
+void virtio_scsi_handle_cmd_vq(VirtIOSCSI *s, VirtQueue *vq)
 {
-    /* use non-QOM casts in the data path */
-    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
     VirtIOSCSIReq *req, *next;
     QTAILQ_HEAD(, VirtIOSCSIReq) reqs = QTAILQ_HEAD_INITIALIZER(reqs);
 
-    if (s->ctx && !s->dataplane_disabled) {
-        virtio_scsi_dataplane_start(s);
-        return;
-    }
     while ((req = virtio_scsi_pop_req(s, vq))) {
         if (virtio_scsi_handle_cmd_req_prepare(s, req)) {
             QTAILQ_INSERT_TAIL(&reqs, req, next);
@@ -596,6 +587,20 @@
     }
 }
 
+static void virtio_scsi_handle_cmd(VirtIODevice *vdev, VirtQueue *vq)
+{
+    /* use non-QOM casts in the data path */
+    VirtIOSCSI *s = (VirtIOSCSI *)vdev;
+
+    if (s->ctx) {
+        virtio_scsi_dataplane_start(s);
+        if (!s->dataplane_fenced) {
+            return;
+        }
+    }
+    virtio_scsi_handle_cmd_vq(s, vq);
+}
+
 static void virtio_scsi_get_config(VirtIODevice *vdev,
                                    uint8_t *config)
 {
@@ -630,9 +635,14 @@
     vs->cdb_size = virtio_ldl_p(vdev, &scsiconf->cdb_size);
 }
 
-static uint32_t virtio_scsi_get_features(VirtIODevice *vdev,
-                                         uint32_t requested_features)
+static uint64_t virtio_scsi_get_features(VirtIODevice *vdev,
+                                         uint64_t requested_features,
+                                         Error **errp)
 {
+    VirtIOSCSI *s = VIRTIO_SCSI(vdev);
+
+    /* Firstly sync all virtio-scsi possible supported features */
+    requested_features |= s->host_features;
     return requested_features;
 }
 
@@ -648,30 +658,25 @@
     qbus_reset_all(&s->bus.qbus);
     s->resetting--;
 
-    vs->sense_size = VIRTIO_SCSI_SENSE_SIZE;
-    vs->cdb_size = VIRTIO_SCSI_CDB_SIZE;
+    vs->sense_size = VIRTIO_SCSI_SENSE_DEFAULT_SIZE;
+    vs->cdb_size = VIRTIO_SCSI_CDB_DEFAULT_SIZE;
     s->events_dropped = false;
 }
 
 /* The device does not have anything to save beyond the virtio data.
  * Request data is saved with callbacks from SCSI devices.
  */
-static void virtio_scsi_save(QEMUFile *f, void *opaque)
+static void virtio_scsi_save(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
     virtio_save(vdev, f);
 }
 
-static int virtio_scsi_load(QEMUFile *f, void *opaque, int version_id)
+static int virtio_scsi_load(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
-    int ret;
 
-    ret = virtio_load(vdev, f, version_id);
-    if (ret) {
-        return ret;
-    }
-    return 0;
+    return virtio_load(vdev, f, 1);
 }
 
 void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
@@ -691,11 +696,7 @@
         aio_context_acquire(s->ctx);
     }
 
-    if (s->dataplane_started) {
-        req = virtio_scsi_pop_req_vring(s, s->event_vring);
-    } else {
-        req = virtio_scsi_pop_req(s, vs->event_vq);
-    }
+    req = virtio_scsi_pop_req(s, vs->event_vq);
     if (!req) {
         s->events_dropped = true;
         goto out;
@@ -733,17 +734,24 @@
     }
 }
 
+void virtio_scsi_handle_event_vq(VirtIOSCSI *s, VirtQueue *vq)
+{
+    if (s->events_dropped) {
+        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
+    }
+}
+
 static void virtio_scsi_handle_event(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
 
-    if (s->ctx && !s->dataplane_disabled) {
+    if (s->ctx) {
         virtio_scsi_dataplane_start(s);
-        return;
+        if (!s->dataplane_fenced) {
+            return;
+        }
     }
-    if (s->events_dropped) {
-        virtio_scsi_push_event(s, NULL, VIRTIO_SCSI_T_NO_EVENT, 0);
-    }
+    virtio_scsi_handle_event_vq(s, vq);
 }
 
 static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense sense)
@@ -751,7 +759,7 @@
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
-    if (((vdev->guest_features >> VIRTIO_SCSI_F_CHANGE) & 1) &&
+    if (virtio_vdev_has_feature(vdev, VIRTIO_SCSI_F_CHANGE) &&
         dev->type != TYPE_ROM) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_PARAM_CHANGE,
                                sense.asc | (sense.ascq << 8));
@@ -765,14 +773,17 @@
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
     SCSIDevice *sd = SCSI_DEVICE(dev);
 
-    if (s->ctx && !s->dataplane_disabled) {
+    if (s->ctx && !s->dataplane_fenced) {
         if (blk_op_is_blocked(sd->conf.blk, BLOCK_OP_TYPE_DATAPLANE, errp)) {
             return;
         }
-        blk_op_block_all(sd->conf.blk, s->blocker);
+        aio_context_acquire(s->ctx);
+        blk_set_aio_context(sd->conf.blk, s->ctx);
+        aio_context_release(s->ctx);
+
     }
 
-    if ((vdev->guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
         virtio_scsi_push_event(s, sd,
                                VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_RESCAN);
@@ -786,15 +797,12 @@
     VirtIOSCSI *s = VIRTIO_SCSI(vdev);
     SCSIDevice *sd = SCSI_DEVICE(dev);
 
-    if ((vdev->guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_SCSI_F_HOTPLUG)) {
         virtio_scsi_push_event(s, sd,
                                VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_REMOVED);
     }
 
-    if (s->ctx) {
-        blk_op_unblock_all(sd->conf.blk, s->blocker);
-    }
     qdev_simple_device_unplug_cb(hotplug_dev, dev, errp);
 }
 
@@ -814,8 +822,9 @@
 };
 
 void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
-                                HandleOutput ctrl, HandleOutput evt,
-                                HandleOutput cmd)
+                                VirtIOHandleOutput ctrl,
+                                VirtIOHandleOutput evt,
+                                VirtIOHandleOutput cmd)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOSCSICommon *s = VIRTIO_SCSI_COMMON(dev);
@@ -825,24 +834,21 @@
                 sizeof(VirtIOSCSIConfig));
 
     if (s->conf.num_queues == 0 ||
-            s->conf.num_queues > VIRTIO_PCI_QUEUE_MAX - 2) {
+            s->conf.num_queues > VIRTIO_QUEUE_MAX - 2) {
         error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
                          "must be a positive integer less than %d.",
-                   s->conf.num_queues, VIRTIO_PCI_QUEUE_MAX - 2);
+                   s->conf.num_queues, VIRTIO_QUEUE_MAX - 2);
         virtio_cleanup(vdev);
         return;
     }
-    s->cmd_vqs = g_malloc0(s->conf.num_queues * sizeof(VirtQueue *));
-    s->sense_size = VIRTIO_SCSI_SENSE_SIZE;
-    s->cdb_size = VIRTIO_SCSI_CDB_SIZE;
+    s->cmd_vqs = g_new0(VirtQueue *, s->conf.num_queues);
+    s->sense_size = VIRTIO_SCSI_SENSE_DEFAULT_SIZE;
+    s->cdb_size = VIRTIO_SCSI_CDB_DEFAULT_SIZE;
 
-    s->ctrl_vq = virtio_add_queue(vdev, VIRTIO_SCSI_VQ_SIZE,
-                                  ctrl);
-    s->event_vq = virtio_add_queue(vdev, VIRTIO_SCSI_VQ_SIZE,
-                                   evt);
+    s->ctrl_vq = virtio_add_queue_aio(vdev, VIRTIO_SCSI_VQ_SIZE, ctrl);
+    s->event_vq = virtio_add_queue_aio(vdev, VIRTIO_SCSI_VQ_SIZE, evt);
     for (i = 0; i < s->conf.num_queues; i++) {
-        s->cmd_vqs[i] = virtio_add_queue(vdev, VIRTIO_SCSI_VQ_SIZE,
-                                         cmd);
+        s->cmd_vqs[i] = virtio_add_queue_aio(vdev, VIRTIO_SCSI_VQ_SIZE, cmd);
     }
 
     if (s->conf.iothread) {
@@ -850,36 +856,10 @@
     }
 }
 
-/* Disable dataplane thread during live migration since it does not
- * update the dirty memory bitmap yet.
- */
-static void virtio_scsi_migration_state_changed(Notifier *notifier, void *data)
-{
-    VirtIOSCSI *s = container_of(notifier, VirtIOSCSI,
-                                 migration_state_notifier);
-    MigrationState *mig = data;
-
-    if (migration_in_setup(mig)) {
-        if (!s->dataplane_started) {
-            return;
-        }
-        virtio_scsi_dataplane_stop(s);
-        s->dataplane_disabled = true;
-    } else if (migration_has_finished(mig) ||
-               migration_has_failed(mig)) {
-        if (s->dataplane_started) {
-            return;
-        }
-        blk_drain_all(); /* complete in-flight non-dataplane requests */
-        s->dataplane_disabled = false;
-    }
-}
-
 static void virtio_scsi_device_realize(DeviceState *dev, Error **errp)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
     VirtIOSCSI *s = VIRTIO_SCSI(dev);
-    static int virtio_scsi_id;
     Error *err = NULL;
 
     virtio_scsi_common_realize(dev, &err, virtio_scsi_handle_ctrl,
@@ -902,11 +882,6 @@
             return;
         }
     }
-
-    register_savevm(dev, "virtio-scsi", virtio_scsi_id++, 1,
-                    virtio_scsi_save, virtio_scsi_load, s);
-    s->migration_state_notifier.notify = virtio_scsi_migration_state_changed;
-    add_migration_state_change_notifier(&s->migration_state_notifier);
 }
 
 static void virtio_scsi_instance_init(Object *obj)
@@ -930,19 +905,24 @@
 
 static void virtio_scsi_device_unrealize(DeviceState *dev, Error **errp)
 {
-    VirtIOSCSI *s = VIRTIO_SCSI(dev);
-
-    unregister_savevm(dev, "virtio-scsi", s);
-    remove_migration_state_change_notifier(&s->migration_state_notifier);
-
     virtio_scsi_common_unrealize(dev, errp);
 }
 
 static Property virtio_scsi_properties[] = {
-    DEFINE_VIRTIO_SCSI_PROPERTIES(VirtIOSCSI, parent_obj.conf),
+    DEFINE_PROP_UINT32("num_queues", VirtIOSCSI, parent_obj.conf.num_queues, 1),
+    DEFINE_PROP_UINT32("max_sectors", VirtIOSCSI, parent_obj.conf.max_sectors,
+                                                  0xFFFF),
+    DEFINE_PROP_UINT32("cmd_per_lun", VirtIOSCSI, parent_obj.conf.cmd_per_lun,
+                                                  128),
+    DEFINE_PROP_BIT("hotplug", VirtIOSCSI, host_features,
+                                           VIRTIO_SCSI_F_HOTPLUG, true),
+    DEFINE_PROP_BIT("param_change", VirtIOSCSI, host_features,
+                                                VIRTIO_SCSI_F_CHANGE, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+VMSTATE_VIRTIO_DEVICE(scsi, 1, virtio_scsi_load, virtio_scsi_save);
+
 static void virtio_scsi_common_class_init(ObjectClass *klass, void *data)
 {
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
@@ -959,6 +939,7 @@
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     dc->props = virtio_scsi_properties;
+    dc->vmsd = &vmstate_virtio_scsi;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     vdc->realize = virtio_scsi_device_realize;
     vdc->unrealize = virtio_scsi_device_unrealize;
diff --git a/hw/scsi/vmw_pvscsi.c b/hw/scsi/vmw_pvscsi.c
index 7bd2ada..d6a91e1 100644
--- a/hw/scsi/vmw_pvscsi.c
+++ b/hw/scsi/vmw_pvscsi.c
@@ -25,14 +25,15 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/scsi/scsi.h"
-#include <block/scsi.h>
+#include "block/scsi.h"
 #include "hw/pci/msi.h"
 #include "vmw_pvscsi.h"
 #include "trace.h"
 
 
-#define PVSCSI_MSI_OFFSET        (0x50)
 #define PVSCSI_USE_64BIT         (true)
 #define PVSCSI_PER_VECTOR_MASK   (false)
 
@@ -42,16 +43,40 @@
 #define PVSCSI_MAX_CMD_DATA_WORDS \
     (sizeof(PVSCSICmdDescSetupRings)/sizeof(uint32_t))
 
-#define RS_GET_FIELD(rs_pa, field) \
-    (ldl_le_phys(&address_space_memory, \
-                 rs_pa + offsetof(struct PVSCSIRingsState, field)))
-#define RS_SET_FIELD(rs_pa, field, val) \
-    (stl_le_phys(&address_space_memory, \
-                 rs_pa + offsetof(struct PVSCSIRingsState, field), val))
+#define RS_GET_FIELD(m, field) \
+    (ldl_le_pci_dma(&container_of(m, PVSCSIState, rings)->parent_obj, \
+                 (m)->rs_pa + offsetof(struct PVSCSIRingsState, field)))
+#define RS_SET_FIELD(m, field, val) \
+    (stl_le_pci_dma(&container_of(m, PVSCSIState, rings)->parent_obj, \
+                 (m)->rs_pa + offsetof(struct PVSCSIRingsState, field), val))
+
+typedef struct PVSCSIClass {
+    PCIDeviceClass parent_class;
+    DeviceRealize parent_dc_realize;
+} PVSCSIClass;
 
 #define TYPE_PVSCSI "pvscsi"
 #define PVSCSI(obj) OBJECT_CHECK(PVSCSIState, (obj), TYPE_PVSCSI)
 
+#define PVSCSI_DEVICE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(PVSCSIClass, (klass), TYPE_PVSCSI)
+#define PVSCSI_DEVICE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(PVSCSIClass, (obj), TYPE_PVSCSI)
+
+/* Compatibility flags for migration */
+#define PVSCSI_COMPAT_OLD_PCI_CONFIGURATION_BIT 0
+#define PVSCSI_COMPAT_OLD_PCI_CONFIGURATION \
+    (1 << PVSCSI_COMPAT_OLD_PCI_CONFIGURATION_BIT)
+#define PVSCSI_COMPAT_DISABLE_PCIE_BIT 1
+#define PVSCSI_COMPAT_DISABLE_PCIE \
+    (1 << PVSCSI_COMPAT_DISABLE_PCIE_BIT)
+
+#define PVSCSI_USE_OLD_PCI_CONFIGURATION(s) \
+    ((s)->compat_flags & PVSCSI_COMPAT_OLD_PCI_CONFIGURATION)
+#define PVSCSI_MSI_OFFSET(s) \
+    (PVSCSI_USE_OLD_PCI_CONFIGURATION(s) ? 0x50 : 0x7c)
+#define PVSCSI_EXP_EP_OFFSET (0x40)
+
 typedef struct PVSCSIRingInfo {
     uint64_t            rs_pa;
     uint32_t            txr_len_mask;
@@ -96,10 +121,11 @@
     uint8_t msg_ring_info_valid;         /* Whether message ring initialized */
     uint8_t use_msg;                     /* Whether to use message ring      */
 
-    uint8_t msi_used;    /* Whether MSI support was installed successfully   */
-
+    uint8_t msi_used;                    /* For migration compatibility      */
     PVSCSIRingInfo rings;                /* Data transfer rings manager      */
     uint32_t resetting;                  /* Reset in progress                */
+
+    uint32_t compat_flags;
 } PVSCSIState;
 
 typedef struct PVSCSIRequest {
@@ -126,7 +152,7 @@
     return log;
 }
 
-static void
+static int
 pvscsi_ring_init_data(PVSCSIRingInfo *m, PVSCSICmdDescSetupRings *ri)
 {
     int i;
@@ -134,6 +160,10 @@
     uint32_t req_ring_size, cmp_ring_size;
     m->rs_pa = ri->ringsStatePPN << VMW_PAGE_SHIFT;
 
+    if ((ri->reqRingNumPages > PVSCSI_SETUP_RINGS_MAX_NUM_PAGES)
+        || (ri->cmpRingNumPages > PVSCSI_SETUP_RINGS_MAX_NUM_PAGES)) {
+        return -1;
+    }
     req_ring_size = ri->reqRingNumPages * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
     cmp_ring_size = ri->cmpRingNumPages * PVSCSI_MAX_NUM_CMP_ENTRIES_PER_PAGE;
     txr_len_log2 = pvscsi_log2(req_ring_size - 1);
@@ -153,27 +183,32 @@
         m->cmp_ring_pages_pa[i] = ri->cmpRingPPNs[i] << VMW_PAGE_SHIFT;
     }
 
-    RS_SET_FIELD(m->rs_pa, reqProdIdx, 0);
-    RS_SET_FIELD(m->rs_pa, reqConsIdx, 0);
-    RS_SET_FIELD(m->rs_pa, reqNumEntriesLog2, txr_len_log2);
+    RS_SET_FIELD(m, reqProdIdx, 0);
+    RS_SET_FIELD(m, reqConsIdx, 0);
+    RS_SET_FIELD(m, reqNumEntriesLog2, txr_len_log2);
 
-    RS_SET_FIELD(m->rs_pa, cmpProdIdx, 0);
-    RS_SET_FIELD(m->rs_pa, cmpConsIdx, 0);
-    RS_SET_FIELD(m->rs_pa, cmpNumEntriesLog2, rxr_len_log2);
+    RS_SET_FIELD(m, cmpProdIdx, 0);
+    RS_SET_FIELD(m, cmpConsIdx, 0);
+    RS_SET_FIELD(m, cmpNumEntriesLog2, rxr_len_log2);
 
     trace_pvscsi_ring_init_data(txr_len_log2, rxr_len_log2);
 
     /* Flush ring state page changes */
     smp_wmb();
+
+    return 0;
 }
 
-static void
+static int
 pvscsi_ring_init_msg(PVSCSIRingInfo *m, PVSCSICmdDescSetupMsgRing *ri)
 {
     int i;
     uint32_t len_log2;
     uint32_t ring_size;
 
+    if (ri->numPages > PVSCSI_SETUP_MSG_RING_MAX_NUM_PAGES) {
+        return -1;
+    }
     ring_size = ri->numPages * PVSCSI_MAX_NUM_MSG_ENTRIES_PER_PAGE;
     len_log2 = pvscsi_log2(ring_size - 1);
 
@@ -185,14 +220,16 @@
         m->msg_ring_pages_pa[i] = ri->ringPPNs[i] << VMW_PAGE_SHIFT;
     }
 
-    RS_SET_FIELD(m->rs_pa, msgProdIdx, 0);
-    RS_SET_FIELD(m->rs_pa, msgConsIdx, 0);
-    RS_SET_FIELD(m->rs_pa, msgNumEntriesLog2, len_log2);
+    RS_SET_FIELD(m, msgProdIdx, 0);
+    RS_SET_FIELD(m, msgConsIdx, 0);
+    RS_SET_FIELD(m, msgNumEntriesLog2, len_log2);
 
     trace_pvscsi_ring_init_msg(len_log2);
 
     /* Flush ring state page changes */
     smp_wmb();
+
+    return 0;
 }
 
 static void
@@ -213,7 +250,7 @@
 static hwaddr
 pvscsi_ring_pop_req_descr(PVSCSIRingInfo *mgr)
 {
-    uint32_t ready_ptr = RS_GET_FIELD(mgr->rs_pa, reqProdIdx);
+    uint32_t ready_ptr = RS_GET_FIELD(mgr, reqProdIdx);
 
     if (ready_ptr != mgr->consumed_ptr) {
         uint32_t next_ready_ptr =
@@ -233,7 +270,7 @@
 static void
 pvscsi_ring_flush_req(PVSCSIRingInfo *mgr)
 {
-    RS_SET_FIELD(mgr->rs_pa, reqConsIdx, mgr->consumed_ptr);
+    RS_SET_FIELD(mgr, reqConsIdx, mgr->consumed_ptr);
 }
 
 static hwaddr
@@ -278,14 +315,14 @@
 
     trace_pvscsi_ring_flush_cmp(mgr->filled_cmp_ptr);
 
-    RS_SET_FIELD(mgr->rs_pa, cmpProdIdx, mgr->filled_cmp_ptr);
+    RS_SET_FIELD(mgr, cmpProdIdx, mgr->filled_cmp_ptr);
 }
 
 static bool
 pvscsi_ring_msg_has_room(PVSCSIRingInfo *mgr)
 {
-    uint32_t prodIdx = RS_GET_FIELD(mgr->rs_pa, msgProdIdx);
-    uint32_t consIdx = RS_GET_FIELD(mgr->rs_pa, msgConsIdx);
+    uint32_t prodIdx = RS_GET_FIELD(mgr, msgProdIdx);
+    uint32_t consIdx = RS_GET_FIELD(mgr, msgConsIdx);
 
     return (prodIdx - consIdx) < (mgr->msg_len_mask + 1);
 }
@@ -298,7 +335,7 @@
 
     trace_pvscsi_ring_flush_msg(mgr->filled_msg_ptr);
 
-    RS_SET_FIELD(mgr->rs_pa, msgProdIdx, mgr->filled_msg_ptr);
+    RS_SET_FIELD(mgr, msgProdIdx, mgr->filled_msg_ptr);
 }
 
 static void
@@ -324,7 +361,7 @@
     trace_pvscsi_update_irq_level(should_raise, s->reg_interrupt_enabled,
                                   s->reg_interrupt_status);
 
-    if (s->msi_used && msi_enabled(d)) {
+    if (msi_enabled(d)) {
         if (should_raise) {
             trace_pvscsi_update_irq_msi();
             msi_notify(d, PVSCSI_VECTOR_COMPLETION);
@@ -743,7 +780,10 @@
     trace_pvscsi_on_cmd_arrived("PVSCSI_CMD_SETUP_RINGS");
 
     pvscsi_dbg_dump_tx_rings_config(rc);
-    pvscsi_ring_init_data(&s->rings, rc);
+    if (pvscsi_ring_init_data(&s->rings, rc) < 0) {
+        return PVSCSI_COMMAND_PROCESSING_FAILED;
+    }
+
     s->rings_info_valid = TRUE;
     return PVSCSI_COMMAND_PROCESSING_SUCCEEDED;
 }
@@ -823,7 +863,9 @@
     }
 
     if (s->rings_info_valid) {
-        pvscsi_ring_init_msg(&s->rings, rc);
+        if (pvscsi_ring_init_msg(&s->rings, rc) < 0) {
+            return PVSCSI_COMMAND_PROCESSING_FAILED;
+        }
         s->msg_ring_info_valid = TRUE;
     }
     return sizeof(PVSCSICmdDescSetupMsgRing) / sizeof(uint32_t);
@@ -1014,22 +1056,20 @@
 }
 
 
-static bool
+static void
 pvscsi_init_msi(PVSCSIState *s)
 {
     int res;
     PCIDevice *d = PCI_DEVICE(s);
 
-    res = msi_init(d, PVSCSI_MSI_OFFSET, PVSCSI_MSIX_NUM_VECTORS,
-                   PVSCSI_USE_64BIT, PVSCSI_PER_VECTOR_MASK);
+    res = msi_init(d, PVSCSI_MSI_OFFSET(s), PVSCSI_MSIX_NUM_VECTORS,
+                   PVSCSI_USE_64BIT, PVSCSI_PER_VECTOR_MASK, NULL);
     if (res < 0) {
         trace_pvscsi_init_msi_fail(res);
         s->msi_used = false;
     } else {
         s->msi_used = true;
     }
-
-    return s->msi_used;
 }
 
 static void
@@ -1037,9 +1077,7 @@
 {
     PCIDevice *d = PCI_DEVICE(s);
 
-    if (s->msi_used) {
-        msi_uninit(d);
-    }
+    msi_uninit(d);
 }
 
 static const MemoryRegionOps pvscsi_ops = {
@@ -1070,9 +1108,16 @@
 
     trace_pvscsi_state("init");
 
-    /* PCI subsystem ID */
-    pci_dev->config[PCI_SUBSYSTEM_ID] = 0x00;
-    pci_dev->config[PCI_SUBSYSTEM_ID + 1] = 0x10;
+    /* PCI subsystem ID, subsystem vendor ID, revision */
+    if (PVSCSI_USE_OLD_PCI_CONFIGURATION(s)) {
+        pci_set_word(pci_dev->config + PCI_SUBSYSTEM_ID, 0x1000);
+    } else {
+        pci_set_word(pci_dev->config + PCI_SUBSYSTEM_VENDOR_ID,
+                     PCI_VENDOR_ID_VMWARE);
+        pci_set_word(pci_dev->config + PCI_SUBSYSTEM_ID,
+                     PCI_DEVICE_ID_VMWARE_PVSCSI);
+        pci_config_set_revision(pci_dev->config, 0x2);
+    }
 
     /* PCI latency timer = 255 */
     pci_dev->config[PCI_LATENCY_TIMER] = 0xff;
@@ -1086,6 +1131,10 @@
 
     pvscsi_init_msi(s);
 
+    if (pci_is_express(pci_dev) && pci_bus_is_express(pci_dev->bus)) {
+        pcie_endpoint_cap_init(pci_dev, PVSCSI_EXP_EP_OFFSET);
+    }
+
     s->completion_worker = qemu_bh_new(pvscsi_process_completion_queue, s);
     if (!s->completion_worker) {
         pvscsi_cleanup_msi(s);
@@ -1141,6 +1190,27 @@
     return 0;
 }
 
+static bool pvscsi_vmstate_need_pcie_device(void *opaque)
+{
+    PVSCSIState *s = PVSCSI(opaque);
+
+    return !(s->compat_flags & PVSCSI_COMPAT_DISABLE_PCIE);
+}
+
+static bool pvscsi_vmstate_test_pci_device(void *opaque, int version_id)
+{
+    return !pvscsi_vmstate_need_pcie_device(opaque);
+}
+
+static const VMStateDescription vmstate_pvscsi_pcie_device = {
+    .name = "pvscsi/pcie",
+    .needed = pvscsi_vmstate_need_pcie_device,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(parent_obj, PVSCSIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_pvscsi = {
     .name = "pvscsi",
     .version_id = 0,
@@ -1148,7 +1218,9 @@
     .pre_save = pvscsi_pre_save,
     .post_load = pvscsi_post_load,
     .fields = (VMStateField[]) {
-        VMSTATE_PCI_DEVICE(parent_obj, PVSCSIState),
+        VMSTATE_STRUCT_TEST(parent_obj, PVSCSIState,
+                            pvscsi_vmstate_test_pci_device, 0,
+                            vmstate_pci_device, PCIDevice),
         VMSTATE_UINT8(msi_used, PVSCSIState),
         VMSTATE_UINT32(resetting, PVSCSIState),
         VMSTATE_UINT64(reg_interrupt_status, PVSCSIState),
@@ -1173,25 +1245,40 @@
         VMSTATE_UINT64(rings.filled_cmp_ptr, PVSCSIState),
 
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_pvscsi_pcie_device,
+        NULL
     }
 };
 
-static void
-pvscsi_write_config(PCIDevice *pci, uint32_t addr, uint32_t val, int len)
-{
-    pci_default_write_config(pci, addr, val, len);
-    msi_write_config(pci, addr, val, len);
-}
-
 static Property pvscsi_properties[] = {
     DEFINE_PROP_UINT8("use_msg", PVSCSIState, use_msg, 1),
+    DEFINE_PROP_BIT("x-old-pci-configuration", PVSCSIState, compat_flags,
+                    PVSCSI_COMPAT_OLD_PCI_CONFIGURATION_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", PVSCSIState, compat_flags,
+                    PVSCSI_COMPAT_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void pvscsi_realize(DeviceState *qdev, Error **errp)
+{
+    PVSCSIClass *pvs_c = PVSCSI_DEVICE_GET_CLASS(qdev);
+    PCIDevice *pci_dev = PCI_DEVICE(qdev);
+    PVSCSIState *s = PVSCSI(qdev);
+
+    if (!(s->compat_flags & PVSCSI_COMPAT_DISABLE_PCIE)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    pvs_c->parent_dc_realize(qdev, errp);
+}
+
 static void pvscsi_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    PVSCSIClass *pvs_k = PVSCSI_DEVICE_CLASS(klass);
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     k->init = pvscsi_init;
@@ -1200,11 +1287,12 @@
     k->device_id = PCI_DEVICE_ID_VMWARE_PVSCSI;
     k->class_id = PCI_CLASS_STORAGE_SCSI;
     k->subsystem_id = 0x1000;
+    pvs_k->parent_dc_realize = dc->realize;
+    dc->realize = pvscsi_realize;
     dc->reset = pvscsi_reset;
     dc->vmsd = &vmstate_pvscsi;
     dc->props = pvscsi_properties;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
-    k->config_write = pvscsi_write_config;
     hc->unplug = pvscsi_hot_unplug;
     hc->plug = pvscsi_hotplug;
 }
@@ -1212,6 +1300,7 @@
 static const TypeInfo pvscsi_info = {
     .name          = TYPE_PVSCSI,
     .parent        = TYPE_PCI_DEVICE,
+    .class_size    = sizeof(PVSCSIClass),
     .instance_size = sizeof(PVSCSIState),
     .class_init    = pvscsi_class_init,
     .interfaces = (InterfaceInfo[]) {
diff --git a/hw/sd/Makefile.objs b/hw/sd/Makefile.objs
index f1aed83..31c8330 100644
--- a/hw/sd/Makefile.objs
+++ b/hw/sd/Makefile.objs
@@ -1,6 +1,6 @@
 common-obj-$(CONFIG_PL181) += pl181.o
 common-obj-$(CONFIG_SSI_SD) += ssi-sd.o
-common-obj-$(CONFIG_SD) += sd.o
+common-obj-$(CONFIG_SD) += sd.o core.o
 common-obj-$(CONFIG_SDHCI) += sdhci.o
 
 obj-$(CONFIG_MILKYMIST) += milkymist-memcard.o
diff --git a/hw/sd/core.c b/hw/sd/core.c
new file mode 100644
index 0000000..14c2bdf
--- /dev/null
+++ b/hw/sd/core.c
@@ -0,0 +1,146 @@
+/*
+ * SD card bus interface code.
+ *
+ * Copyright (c) 2015 Linaro Limited
+ *
+ * Author:
+ *  Peter Maydell <peter.maydell@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/qdev-core.h"
+#include "sysemu/block-backend.h"
+#include "hw/sd/sd.h"
+
+static SDState *get_card(SDBus *sdbus)
+{
+    /* We only ever have one child on the bus so just return it */
+    BusChild *kid = QTAILQ_FIRST(&sdbus->qbus.children);
+
+    if (!kid) {
+        return NULL;
+    }
+    return SD_CARD(kid->child);
+}
+
+int sdbus_do_command(SDBus *sdbus, SDRequest *req, uint8_t *response)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->do_command(card, req, response);
+    }
+
+    return 0;
+}
+
+void sdbus_write_data(SDBus *sdbus, uint8_t value)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        sc->write_data(card, value);
+    }
+}
+
+uint8_t sdbus_read_data(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->read_data(card);
+    }
+
+    return 0;
+}
+
+bool sdbus_data_ready(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->data_ready(card);
+    }
+
+    return false;
+}
+
+bool sdbus_get_inserted(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->get_inserted(card);
+    }
+
+    return false;
+}
+
+bool sdbus_get_readonly(SDBus *sdbus)
+{
+    SDState *card = get_card(sdbus);
+
+    if (card) {
+        SDCardClass *sc = SD_CARD_GET_CLASS(card);
+
+        return sc->get_readonly(card);
+    }
+
+    return false;
+}
+
+void sdbus_set_inserted(SDBus *sdbus, bool inserted)
+{
+    SDBusClass *sbc = SD_BUS_GET_CLASS(sdbus);
+    BusState *qbus = BUS(sdbus);
+
+    if (sbc->set_inserted) {
+        sbc->set_inserted(qbus->parent, inserted);
+    }
+}
+
+void sdbus_set_readonly(SDBus *sdbus, bool readonly)
+{
+    SDBusClass *sbc = SD_BUS_GET_CLASS(sdbus);
+    BusState *qbus = BUS(sdbus);
+
+    if (sbc->set_readonly) {
+        sbc->set_readonly(qbus->parent, readonly);
+    }
+}
+
+static const TypeInfo sd_bus_info = {
+    .name = TYPE_SD_BUS,
+    .parent = TYPE_BUS,
+    .instance_size = sizeof(SDBus),
+    .class_size = sizeof(SDBusClass),
+};
+
+static void sd_bus_register_types(void)
+{
+    type_register_static(&sd_bus_info);
+}
+
+type_init(sd_bus_register_types)
diff --git a/hw/sd/milkymist-memcard.c b/hw/sd/milkymist-memcard.c
index 9661eaf..1f2f0ed 100644
--- a/hw/sd/milkymist-memcard.c
+++ b/hw/sd/milkymist-memcard.c
@@ -18,9 +18,10 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/memcard.pdf
+ *   http://milkymist.walle.cc/socdoc/memcard.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
@@ -28,7 +29,7 @@
 #include "qemu/error-report.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 enum {
     ENABLE_CMD_TX   = (1<<0),
@@ -255,6 +256,7 @@
     DriveInfo *dinfo;
     BlockBackend *blk;
 
+    /* FIXME use a qdev drive property instead of drive_get_next() */
     dinfo = drive_get_next(IF_SD);
     blk = dinfo ? blk_by_legacy_dinfo(dinfo) : NULL;
     s->card = sd_init(blk, false);
@@ -296,6 +298,8 @@
     k->init = milkymist_memcard_init;
     dc->reset = milkymist_memcard_reset;
     dc->vmsd = &vmstate_milkymist_memcard;
+    /* Reason: init() method uses drive_get_next() */
+    dc->cannot_instantiate_with_device_add_yet = true;
 }
 
 static const TypeInfo milkymist_memcard_info = {
diff --git a/hw/sd/omap_mmc.c b/hw/sd/omap_mmc.c
index 86c477d..e934cd3 100644
--- a/hw/sd/omap_mmc.c
+++ b/hw/sd/omap_mmc.c
@@ -16,9 +16,10 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 
 struct omap_mmc_s {
     qemu_irq irq;
@@ -406,7 +407,8 @@
     struct omap_mmc_s *s = (struct omap_mmc_s *) opaque;
 
     if (size != 2) {
-        return omap_badwidth_write16(opaque, offset, value);
+        omap_badwidth_write16(opaque, offset, value);
+        return;
     }
 
     switch (offset) {
@@ -577,8 +579,7 @@
                 BlockBackend *blk,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk)
 {
-    struct omap_mmc_s *s = (struct omap_mmc_s *)
-            g_malloc0(sizeof(struct omap_mmc_s));
+    struct omap_mmc_s *s = g_new0(struct omap_mmc_s, 1);
 
     s->irq = irq;
     s->dma = dma;
@@ -604,8 +605,7 @@
                 BlockBackend *blk, qemu_irq irq, qemu_irq dma[],
                 omap_clk fclk, omap_clk iclk)
 {
-    struct omap_mmc_s *s = (struct omap_mmc_s *)
-            g_malloc0(sizeof(struct omap_mmc_s));
+    struct omap_mmc_s *s = g_new0(struct omap_mmc_s, 1);
 
     s->irq = irq;
     s->dma = dma;
diff --git a/hw/sd/pl181.c b/hw/sd/pl181.c
index e704b6e..82c63a4 100644
--- a/hw/sd/pl181.c
+++ b/hw/sd/pl181.c
@@ -7,10 +7,13 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "hw/sysbus.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
+#include "qemu/log.h"
+#include "qapi/error.h"
 
 //#define DEBUG_PL181 1
 
@@ -46,7 +49,7 @@
     int32_t fifo_pos;
     int32_t fifo_len;
     /* The linux 2.6.21 driver is buggy, and misbehaves if new data arrives
-       while it is reading the FIFO.  We hack around this be defering
+       while it is reading the FIFO.  We hack around this by deferring
        subsequent transfers until after the driver polls the status word.
        http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=4446/1
      */
@@ -479,40 +482,48 @@
     sd_set_cb(s->card, s->cardstatus[0], s->cardstatus[1]);
 }
 
-static int pl181_init(SysBusDevice *sbd)
+static void pl181_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    PL181State *s = PL181(dev);
-    DriveInfo *dinfo;
+    DeviceState *dev = DEVICE(obj);
+    PL181State *s = PL181(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pl181_ops, s, "pl181", 0x1000);
+    memory_region_init_io(&s->iomem, obj, &pl181_ops, s, "pl181", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
     sysbus_init_irq(sbd, &s->irq[0]);
     sysbus_init_irq(sbd, &s->irq[1]);
     qdev_init_gpio_out(dev, s->cardstatus, 2);
+}
+
+static void pl181_realize(DeviceState *dev, Error **errp)
+{
+    PL181State *s = PL181(dev);
+    DriveInfo *dinfo;
+
+    /* FIXME use a qdev drive property instead of drive_get_next() */
     dinfo = drive_get_next(IF_SD);
     s->card = sd_init(dinfo ? blk_by_legacy_dinfo(dinfo) : NULL, false);
     if (s->card == NULL) {
-        return -1;
+        error_setg(errp, "sd_init failed");
     }
-
-    return 0;
 }
 
 static void pl181_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *k = DEVICE_CLASS(klass);
 
-    sdc->init = pl181_init;
     k->vmsd = &vmstate_pl181;
     k->reset = pl181_reset;
+    /* Reason: init() method uses drive_get_next() */
+    k->cannot_instantiate_with_device_add_yet = true;
+    k->realize = pl181_realize;
 }
 
 static const TypeInfo pl181_info = {
     .name          = TYPE_PL181,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PL181State),
+    .instance_init = pl181_init,
     .class_init    = pl181_class_init,
 };
 
diff --git a/hw/sd/pxa2xx_mmci.c b/hw/sd/pxa2xx_mmci.c
index ac3ab39..3deccf0 100644
--- a/hw/sd/pxa2xx_mmci.c
+++ b/hw/sd/pxa2xx_mmci.c
@@ -10,18 +10,34 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/hw.h"
+#include "hw/sysbus.h"
 #include "hw/arm/pxa.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
 #include "hw/qdev.h"
+#include "hw/qdev-properties.h"
+#include "qemu/error-report.h"
+
+#define TYPE_PXA2XX_MMCI "pxa2xx-mmci"
+#define PXA2XX_MMCI(obj) OBJECT_CHECK(PXA2xxMMCIState, (obj), TYPE_PXA2XX_MMCI)
+
+#define TYPE_PXA2XX_MMCI_BUS "pxa2xx-mmci-bus"
+#define PXA2XX_MMCI_BUS(obj) OBJECT_CHECK(SDBus, (obj), TYPE_PXA2XX_MMCI_BUS)
 
 struct PXA2xxMMCIState {
+    SysBusDevice parent_obj;
+
     MemoryRegion iomem;
     qemu_irq irq;
     qemu_irq rx_dma;
     qemu_irq tx_dma;
+    qemu_irq inserted;
+    qemu_irq readonly;
 
-    SDState *card;
+    BlockBackend *blk;
+    SDBus sdbus;
 
     uint32_t status;
     uint32_t clkrt;
@@ -29,26 +45,70 @@
     uint32_t cmdat;
     uint32_t resp_tout;
     uint32_t read_tout;
-    int blklen;
-    int numblk;
+    int32_t blklen;
+    int32_t numblk;
     uint32_t intmask;
     uint32_t intreq;
-    int cmd;
+    int32_t cmd;
     uint32_t arg;
 
-    int active;
-    int bytesleft;
+    int32_t active;
+    int32_t bytesleft;
     uint8_t tx_fifo[64];
-    int tx_start;
-    int tx_len;
+    uint32_t tx_start;
+    uint32_t tx_len;
     uint8_t rx_fifo[32];
-    int rx_start;
-    int rx_len;
+    uint32_t rx_start;
+    uint32_t rx_len;
     uint16_t resp_fifo[9];
-    int resp_len;
+    uint32_t resp_len;
 
-    int cmdreq;
-    int ac_width;
+    int32_t cmdreq;
+};
+
+static bool pxa2xx_mmci_vmstate_validate(void *opaque, int version_id)
+{
+    PXA2xxMMCIState *s = opaque;
+
+    return s->tx_start < ARRAY_SIZE(s->tx_fifo)
+        && s->rx_start < ARRAY_SIZE(s->rx_fifo)
+        && s->tx_len <= ARRAY_SIZE(s->tx_fifo)
+        && s->rx_len <= ARRAY_SIZE(s->rx_fifo)
+        && s->resp_len <= ARRAY_SIZE(s->resp_fifo);
+}
+
+
+static const VMStateDescription vmstate_pxa2xx_mmci = {
+    .name = "pxa2xx-mmci",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(status, PXA2xxMMCIState),
+        VMSTATE_UINT32(clkrt, PXA2xxMMCIState),
+        VMSTATE_UINT32(spi, PXA2xxMMCIState),
+        VMSTATE_UINT32(cmdat, PXA2xxMMCIState),
+        VMSTATE_UINT32(resp_tout, PXA2xxMMCIState),
+        VMSTATE_UINT32(read_tout, PXA2xxMMCIState),
+        VMSTATE_INT32(blklen, PXA2xxMMCIState),
+        VMSTATE_INT32(numblk, PXA2xxMMCIState),
+        VMSTATE_UINT32(intmask, PXA2xxMMCIState),
+        VMSTATE_UINT32(intreq, PXA2xxMMCIState),
+        VMSTATE_INT32(cmd, PXA2xxMMCIState),
+        VMSTATE_UINT32(arg, PXA2xxMMCIState),
+        VMSTATE_INT32(cmdreq, PXA2xxMMCIState),
+        VMSTATE_INT32(active, PXA2xxMMCIState),
+        VMSTATE_INT32(bytesleft, PXA2xxMMCIState),
+        VMSTATE_UINT32(tx_start, PXA2xxMMCIState),
+        VMSTATE_UINT32(tx_len, PXA2xxMMCIState),
+        VMSTATE_UINT32(rx_start, PXA2xxMMCIState),
+        VMSTATE_UINT32(rx_len, PXA2xxMMCIState),
+        VMSTATE_UINT32(resp_len, PXA2xxMMCIState),
+        VMSTATE_VALIDATE("fifo size incorrect", pxa2xx_mmci_vmstate_validate),
+        VMSTATE_UINT8_ARRAY(tx_fifo, PXA2xxMMCIState, 64),
+        VMSTATE_UINT8_ARRAY(rx_fifo, PXA2xxMMCIState, 32),
+        VMSTATE_UINT16_ARRAY(resp_fifo, PXA2xxMMCIState, 9),
+        VMSTATE_END_OF_LIST()
+    }
 };
 
 #define MMC_STRPCL	0x00	/* MMC Clock Start/Stop register */
@@ -122,7 +182,7 @@
 
     if (s->cmdat & CMDAT_WR_RD) {
         while (s->bytesleft && s->tx_len) {
-            sd_write_data(s->card, s->tx_fifo[s->tx_start ++]);
+            sdbus_write_data(&s->sdbus, s->tx_fifo[s->tx_start++]);
             s->tx_start &= 0x1f;
             s->tx_len --;
             s->bytesleft --;
@@ -132,7 +192,7 @@
     } else
         while (s->bytesleft && s->rx_len < 32) {
             s->rx_fifo[(s->rx_start + (s->rx_len ++)) & 0x1f] =
-                sd_read_data(s->card);
+                sdbus_read_data(&s->sdbus);
             s->bytesleft --;
             s->intreq |= INT_RXFIFO_REQ;
         }
@@ -166,7 +226,7 @@
     request.arg = s->arg;
     request.crc = 0;	/* FIXME */
 
-    rsplen = sd_do_command(s->card, &request, response);
+    rsplen = sdbus_do_command(&s->sdbus, &request, response);
     s->intreq |= INT_END_CMD;
 
     memset(s->resp_fifo, 0, sizeof(s->resp_fifo));
@@ -215,7 +275,7 @@
     pxa2xx_mmci_fifo_update(s);
 }
 
-static uint32_t pxa2xx_mmci_read(void *opaque, hwaddr offset)
+static uint64_t pxa2xx_mmci_read(void *opaque, hwaddr offset, unsigned size)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     uint32_t ret;
@@ -257,8 +317,8 @@
         return 0;
     case MMC_RXFIFO:
         ret = 0;
-        while (s->ac_width -- && s->rx_len) {
-            ret |= s->rx_fifo[s->rx_start ++] << (s->ac_width << 3);
+        while (size-- && s->rx_len) {
+            ret |= s->rx_fifo[s->rx_start++] << (size << 3);
             s->rx_start &= 0x1f;
             s->rx_len --;
         }
@@ -277,7 +337,7 @@
 }
 
 static void pxa2xx_mmci_write(void *opaque,
-                hwaddr offset, uint32_t value)
+                              hwaddr offset, uint64_t value, unsigned size)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
 
@@ -370,9 +430,9 @@
         break;
 
     case MMC_TXFIFO:
-        while (s->ac_width -- && s->tx_len < 0x20)
+        while (size-- && s->tx_len < 0x20)
             s->tx_fifo[(s->tx_start + (s->tx_len ++)) & 0x1f] =
-                    (value >> (s->ac_width << 3)) & 0xff;
+                    (value >> (size << 3)) & 0xff;
         s->intreq &= ~INT_TXFIFO_REQ;
         pxa2xx_mmci_fifo_update(s);
         break;
@@ -386,171 +446,153 @@
     }
 }
 
-static uint32_t pxa2xx_mmci_readb(void *opaque, hwaddr offset)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    s->ac_width = 1;
-    return pxa2xx_mmci_read(opaque, offset);
-}
-
-static uint32_t pxa2xx_mmci_readh(void *opaque, hwaddr offset)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    s->ac_width = 2;
-    return pxa2xx_mmci_read(opaque, offset);
-}
-
-static uint32_t pxa2xx_mmci_readw(void *opaque, hwaddr offset)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    s->ac_width = 4;
-    return pxa2xx_mmci_read(opaque, offset);
-}
-
-static void pxa2xx_mmci_writeb(void *opaque,
-                hwaddr offset, uint32_t value)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    s->ac_width = 1;
-    pxa2xx_mmci_write(opaque, offset, value);
-}
-
-static void pxa2xx_mmci_writeh(void *opaque,
-                hwaddr offset, uint32_t value)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    s->ac_width = 2;
-    pxa2xx_mmci_write(opaque, offset, value);
-}
-
-static void pxa2xx_mmci_writew(void *opaque,
-                hwaddr offset, uint32_t value)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    s->ac_width = 4;
-    pxa2xx_mmci_write(opaque, offset, value);
-}
-
 static const MemoryRegionOps pxa2xx_mmci_ops = {
-    .old_mmio = {
-        .read = { pxa2xx_mmci_readb,
-                  pxa2xx_mmci_readh,
-                  pxa2xx_mmci_readw, },
-        .write = { pxa2xx_mmci_writeb,
-                   pxa2xx_mmci_writeh,
-                   pxa2xx_mmci_writew, },
-    },
+    .read = pxa2xx_mmci_read,
+    .write = pxa2xx_mmci_write,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pxa2xx_mmci_save(QEMUFile *f, void *opaque)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    int i;
-
-    qemu_put_be32s(f, &s->status);
-    qemu_put_be32s(f, &s->clkrt);
-    qemu_put_be32s(f, &s->spi);
-    qemu_put_be32s(f, &s->cmdat);
-    qemu_put_be32s(f, &s->resp_tout);
-    qemu_put_be32s(f, &s->read_tout);
-    qemu_put_be32(f, s->blklen);
-    qemu_put_be32(f, s->numblk);
-    qemu_put_be32s(f, &s->intmask);
-    qemu_put_be32s(f, &s->intreq);
-    qemu_put_be32(f, s->cmd);
-    qemu_put_be32s(f, &s->arg);
-    qemu_put_be32(f, s->cmdreq);
-    qemu_put_be32(f, s->active);
-    qemu_put_be32(f, s->bytesleft);
-
-    qemu_put_byte(f, s->tx_len);
-    for (i = 0; i < s->tx_len; i ++)
-        qemu_put_byte(f, s->tx_fifo[(s->tx_start + i) & 63]);
-
-    qemu_put_byte(f, s->rx_len);
-    for (i = 0; i < s->rx_len; i ++)
-        qemu_put_byte(f, s->rx_fifo[(s->rx_start + i) & 31]);
-
-    qemu_put_byte(f, s->resp_len);
-    for (i = s->resp_len; i < 9; i ++)
-        qemu_put_be16s(f, &s->resp_fifo[i]);
-}
-
-static int pxa2xx_mmci_load(QEMUFile *f, void *opaque, int version_id)
-{
-    PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
-    int i;
-
-    qemu_get_be32s(f, &s->status);
-    qemu_get_be32s(f, &s->clkrt);
-    qemu_get_be32s(f, &s->spi);
-    qemu_get_be32s(f, &s->cmdat);
-    qemu_get_be32s(f, &s->resp_tout);
-    qemu_get_be32s(f, &s->read_tout);
-    s->blklen = qemu_get_be32(f);
-    s->numblk = qemu_get_be32(f);
-    qemu_get_be32s(f, &s->intmask);
-    qemu_get_be32s(f, &s->intreq);
-    s->cmd = qemu_get_be32(f);
-    qemu_get_be32s(f, &s->arg);
-    s->cmdreq = qemu_get_be32(f);
-    s->active = qemu_get_be32(f);
-    s->bytesleft = qemu_get_be32(f);
-
-    s->tx_len = qemu_get_byte(f);
-    s->tx_start = 0;
-    if (s->tx_len >= sizeof(s->tx_fifo) || s->tx_len < 0)
-        return -EINVAL;
-    for (i = 0; i < s->tx_len; i ++)
-        s->tx_fifo[i] = qemu_get_byte(f);
-
-    s->rx_len = qemu_get_byte(f);
-    s->rx_start = 0;
-    if (s->rx_len >= sizeof(s->rx_fifo) || s->rx_len < 0)
-        return -EINVAL;
-    for (i = 0; i < s->rx_len; i ++)
-        s->rx_fifo[i] = qemu_get_byte(f);
-
-    s->resp_len = qemu_get_byte(f);
-    if (s->resp_len > 9 || s->resp_len < 0)
-        return -EINVAL;
-    for (i = s->resp_len; i < 9; i ++)
-         qemu_get_be16s(f, &s->resp_fifo[i]);
-
-    return 0;
-}
-
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
                 hwaddr base,
                 BlockBackend *blk, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma)
 {
+    DeviceState *dev, *carddev;
+    SysBusDevice *sbd;
     PXA2xxMMCIState *s;
+    Error *err = NULL;
 
-    s = (PXA2xxMMCIState *) g_malloc0(sizeof(PXA2xxMMCIState));
-    s->irq = irq;
-    s->rx_dma = rx_dma;
-    s->tx_dma = tx_dma;
+    dev = qdev_create(NULL, TYPE_PXA2XX_MMCI);
+    s = PXA2XX_MMCI(dev);
+    sbd = SYS_BUS_DEVICE(dev);
+    sysbus_mmio_map(sbd, 0, base);
+    sysbus_connect_irq(sbd, 0, irq);
+    qdev_connect_gpio_out_named(dev, "rx-dma", 0, rx_dma);
+    qdev_connect_gpio_out_named(dev, "tx-dma", 0, tx_dma);
 
-    memory_region_init_io(&s->iomem, NULL, &pxa2xx_mmci_ops, s,
-                          "pxa2xx-mmci", 0x00100000);
-    memory_region_add_subregion(sysmem, base, &s->iomem);
-
-    /* Instantiate the actual storage */
-    s->card = sd_init(blk, false);
-    if (s->card == NULL) {
-        exit(1);
+    /* Create and plug in the sd card */
+    carddev = qdev_create(qdev_get_child_bus(dev, "sd-bus"), TYPE_SD_CARD);
+    qdev_prop_set_drive(carddev, "drive", blk, &err);
+    if (err) {
+        error_report("failed to init SD card: %s", error_get_pretty(err));
+        return NULL;
     }
-
-    register_savevm(NULL, "pxa2xx_mmci", 0, 0,
-                    pxa2xx_mmci_save, pxa2xx_mmci_load, s);
+    object_property_set_bool(OBJECT(carddev), true, "realized", &err);
+    if (err) {
+        error_report("failed to init SD card: %s", error_get_pretty(err));
+        return NULL;
+    }
 
     return s;
 }
 
-void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
-                qemu_irq coverswitch)
+static void pxa2xx_mmci_set_inserted(DeviceState *dev, bool inserted)
 {
-    sd_set_cb(s->card, readonly, coverswitch);
+    PXA2xxMMCIState *s = PXA2XX_MMCI(dev);
+
+    qemu_set_irq(s->inserted, inserted);
 }
+
+static void pxa2xx_mmci_set_readonly(DeviceState *dev, bool readonly)
+{
+    PXA2xxMMCIState *s = PXA2XX_MMCI(dev);
+
+    qemu_set_irq(s->readonly, readonly);
+}
+
+void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
+                          qemu_irq coverswitch)
+{
+    DeviceState *dev = DEVICE(s);
+
+    s->readonly = readonly;
+    s->inserted = coverswitch;
+
+    pxa2xx_mmci_set_inserted(dev, sdbus_get_inserted(&s->sdbus));
+    pxa2xx_mmci_set_readonly(dev, sdbus_get_readonly(&s->sdbus));
+}
+
+static void pxa2xx_mmci_reset(DeviceState *d)
+{
+    PXA2xxMMCIState *s = PXA2XX_MMCI(d);
+
+    s->status = 0;
+    s->clkrt = 0;
+    s->spi = 0;
+    s->cmdat = 0;
+    s->resp_tout = 0;
+    s->read_tout = 0;
+    s->blklen = 0;
+    s->numblk = 0;
+    s->intmask = 0;
+    s->intreq = 0;
+    s->cmd = 0;
+    s->arg = 0;
+    s->active = 0;
+    s->bytesleft = 0;
+    s->tx_start = 0;
+    s->tx_len = 0;
+    s->rx_start = 0;
+    s->rx_len = 0;
+    s->resp_len = 0;
+    s->cmdreq = 0;
+    memset(s->tx_fifo, 0, sizeof(s->tx_fifo));
+    memset(s->rx_fifo, 0, sizeof(s->rx_fifo));
+    memset(s->resp_fifo, 0, sizeof(s->resp_fifo));
+}
+
+static void pxa2xx_mmci_instance_init(Object *obj)
+{
+    PXA2xxMMCIState *s = PXA2XX_MMCI(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
+    DeviceState *dev = DEVICE(obj);
+
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_mmci_ops, s,
+                          "pxa2xx-mmci", 0x00100000);
+    sysbus_init_mmio(sbd, &s->iomem);
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_out_named(dev, &s->rx_dma, "rx-dma", 1);
+    qdev_init_gpio_out_named(dev, &s->tx_dma, "tx-dma", 1);
+
+    qbus_create_inplace(&s->sdbus, sizeof(s->sdbus),
+                        TYPE_PXA2XX_MMCI_BUS, DEVICE(obj), "sd-bus");
+}
+
+static void pxa2xx_mmci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->vmsd = &vmstate_pxa2xx_mmci;
+    dc->reset = pxa2xx_mmci_reset;
+}
+
+static void pxa2xx_mmci_bus_class_init(ObjectClass *klass, void *data)
+{
+    SDBusClass *sbc = SD_BUS_CLASS(klass);
+
+    sbc->set_inserted = pxa2xx_mmci_set_inserted;
+    sbc->set_readonly = pxa2xx_mmci_set_readonly;
+}
+
+static const TypeInfo pxa2xx_mmci_info = {
+    .name = TYPE_PXA2XX_MMCI,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(PXA2xxMMCIState),
+    .instance_init = pxa2xx_mmci_instance_init,
+    .class_init = pxa2xx_mmci_class_init,
+};
+
+static const TypeInfo pxa2xx_mmci_bus_info = {
+    .name = TYPE_PXA2XX_MMCI_BUS,
+    .parent = TYPE_SD_BUS,
+    .instance_size = sizeof(SDBus),
+    .class_init = pxa2xx_mmci_bus_class_init,
+};
+
+static void pxa2xx_mmci_register_types(void)
+{
+    type_register_static(&pxa2xx_mmci_info);
+    type_register_static(&pxa2xx_mmci_bus_info);
+}
+
+type_init(pxa2xx_mmci_register_types)
diff --git a/hw/sd/sd.c b/hw/sd/sd.c
index f955265..87c6dc1 100644
--- a/hw/sd/sd.c
+++ b/hw/sd/sd.c
@@ -29,10 +29,17 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
-#include "hw/sd.h"
+#include "hw/sd/sd.h"
+#include "qapi/error.h"
 #include "qemu/bitmap.h"
+#include "hw/qdev-properties.h"
+#include "qemu/error-report.h"
+#include "qemu/timer.h"
+#include "qemu/log.h"
 
 //#define DEBUG_SD 1
 
@@ -43,7 +50,9 @@
 #define DPRINTF(fmt, ...) do {} while(0)
 #endif
 
-#define ACMD41_ENQUIRY_MASK 0x00ffffff
+#define ACMD41_ENQUIRY_MASK     0x00ffffff
+#define OCR_POWER_UP            0x80000000
+#define OCR_POWER_DELAY_NS      500000 /* 0.5ms */
 
 typedef enum {
     sd_r0 = 0,    /* no response */
@@ -77,9 +86,12 @@
 };
 
 struct SDState {
+    DeviceState parent_obj;
+
     uint32_t mode;    /* current card mode, one of SDCardModes */
     int32_t state;    /* current card state, one of SDCardStates */
     uint32_t ocr;
+    QEMUTimer *ocr_power_timer;
     uint8_t scr[8];
     uint8_t cid[16];
     uint8_t csd[16];
@@ -92,6 +104,7 @@
     int32_t wpgrps_size;
     uint64_t size;
     uint32_t blk_len;
+    uint32_t multi_blk_cnt;
     uint32_t erase_start;
     uint32_t erase_end;
     uint8_t pwd[16];
@@ -111,7 +124,6 @@
     qemu_irq readonly_cb;
     qemu_irq inserted_cb;
     BlockBackend *blk;
-    uint8_t *buf;
 
     bool enable;
 };
@@ -193,8 +205,17 @@
 
 static void sd_set_ocr(SDState *sd)
 {
-    /* All voltages OK, card power-up OK, Standard Capacity SD Memory Card */
-    sd->ocr = 0x80ffff00;
+    /* All voltages OK, Standard Capacity SD Memory Card, not yet powered up */
+    sd->ocr = 0x00ffff00;
+}
+
+static void sd_ocr_powerup(void *opaque)
+{
+    SDState *sd = opaque;
+
+    /* Set powered up bit in OCR */
+    assert(!(sd->ocr & OCR_POWER_UP));
+    sd->ocr |= OCR_POWER_UP;
 }
 
 static void sd_set_scr(SDState *sd)
@@ -389,13 +410,14 @@
     return addr >> (HWBLOCK_SHIFT + SECTOR_SHIFT + WPGROUP_SHIFT);
 }
 
-static void sd_reset(SDState *sd, BlockBackend *blk)
+static void sd_reset(DeviceState *dev)
 {
+    SDState *sd = SD_CARD(dev);
     uint64_t size;
     uint64_t sect;
 
-    if (blk) {
-        blk_get_geometry(blk, &sect);
+    if (sd->blk) {
+        blk_get_geometry(sd->blk, &sect);
     } else {
         sect = 0;
     }
@@ -412,11 +434,8 @@
     sd_set_cardstatus(sd);
     sd_set_sdstatus(sd);
 
-    sd->blk = blk;
-
-    if (sd->wp_groups)
-        g_free(sd->wp_groups);
-    sd->wp_switch = blk ? blk_is_read_only(blk) : false;
+    g_free(sd->wp_groups);
+    sd->wp_switch = sd->blk ? blk_is_read_only(sd->blk) : false;
     sd->wpgrps_size = sect;
     sd->wp_groups = bitmap_new(sd->wpgrps_size);
     memset(sd->function_group, 0, sizeof(sd->function_group));
@@ -426,16 +445,44 @@
     sd->blk_len = 0x200;
     sd->pwd_len = 0;
     sd->expecting_acmd = false;
+    sd->multi_blk_cnt = 0;
+}
+
+static bool sd_get_inserted(SDState *sd)
+{
+    return sd->blk && blk_is_inserted(sd->blk);
+}
+
+static bool sd_get_readonly(SDState *sd)
+{
+    return sd->wp_switch;
 }
 
 static void sd_cardchange(void *opaque, bool load)
 {
     SDState *sd = opaque;
+    DeviceState *dev = DEVICE(sd);
+    SDBus *sdbus = SD_BUS(qdev_get_parent_bus(dev));
+    bool inserted = sd_get_inserted(sd);
+    bool readonly = sd_get_readonly(sd);
 
-    qemu_set_irq(sd->inserted_cb, blk_is_inserted(sd->blk));
-    if (blk_is_inserted(sd->blk)) {
-        sd_reset(sd, sd->blk);
-        qemu_set_irq(sd->readonly_cb, sd->wp_switch);
+    if (inserted) {
+        sd_reset(dev);
+    }
+
+    /* The IRQ notification is for legacy non-QOM SD controller devices;
+     * QOMified controllers use the SDBus APIs.
+     */
+    if (sdbus) {
+        sdbus_set_inserted(sdbus, inserted);
+        if (inserted) {
+            sdbus_set_readonly(sdbus, readonly);
+        }
+    } else {
+        qemu_set_irq(sd->inserted_cb, inserted);
+        if (inserted) {
+            qemu_set_irq(sd->readonly_cb, readonly);
+        }
     }
 }
 
@@ -443,10 +490,44 @@
     .change_media_cb = sd_cardchange,
 };
 
+static bool sd_ocr_vmstate_needed(void *opaque)
+{
+    SDState *sd = opaque;
+
+    /* Include the OCR state (and timer) if it is not yet powered up */
+    return !(sd->ocr & OCR_POWER_UP);
+}
+
+static const VMStateDescription sd_ocr_vmstate = {
+    .name = "sd-card/ocr-state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = sd_ocr_vmstate_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(ocr, SDState),
+        VMSTATE_TIMER_PTR(ocr_power_timer, SDState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static int sd_vmstate_pre_load(void *opaque)
+{
+    SDState *sd = opaque;
+
+    /* If the OCR state is not included (prior versions, or not
+     * needed), then the OCR must be set as powered up. If the OCR state
+     * is included, this will be replaced by the state restore.
+     */
+    sd_ocr_powerup(sd);
+
+    return 0;
+}
+
 static const VMStateDescription sd_vmstate = {
     .name = "sd-card",
     .version_id = 1,
     .minimum_version_id = 1,
+    .pre_load = sd_vmstate_pre_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(mode, SDState),
         VMSTATE_INT32(state, SDState),
@@ -458,6 +539,7 @@
         VMSTATE_UINT32(vhs, SDState),
         VMSTATE_BITMAP(wp_groups, SDState, 0, wpgrps_size),
         VMSTATE_UINT32(blk_len, SDState),
+        VMSTATE_UINT32(multi_blk_cnt, SDState),
         VMSTATE_UINT32(erase_start, SDState),
         VMSTATE_UINT32(erase_end, SDState),
         VMSTATE_UINT8_ARRAY(pwd, SDState, 16),
@@ -469,36 +551,38 @@
         VMSTATE_UINT64(data_start, SDState),
         VMSTATE_UINT32(data_offset, SDState),
         VMSTATE_UINT8_ARRAY(data, SDState, 512),
-        VMSTATE_BUFFER_POINTER_UNSAFE(buf, SDState, 1, 512),
+        VMSTATE_UNUSED_V(1, 512),
         VMSTATE_BOOL(enable, SDState),
         VMSTATE_END_OF_LIST()
-    }
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &sd_ocr_vmstate,
+        NULL
+    },
 };
 
-/* We do not model the chip select pin, so allow the board to select
-   whether card should be in SSI or MMC/SD mode.  It is also up to the
-   board to ensure that ssi transfers only occur when the chip select
-   is asserted.  */
+/* Legacy initialization function for use by non-qdevified callers */
 SDState *sd_init(BlockBackend *blk, bool is_spi)
 {
-    SDState *sd;
+    Object *obj;
+    DeviceState *dev;
+    Error *err = NULL;
 
-    if (blk && blk_is_read_only(blk)) {
-        fprintf(stderr, "sd_init: Cannot use read-only drive\n");
+    obj = object_new(TYPE_SD_CARD);
+    dev = DEVICE(obj);
+    qdev_prop_set_drive(dev, "drive", blk, &err);
+    if (err) {
+        error_report("sd_init failed: %s", error_get_pretty(err));
+        return NULL;
+    }
+    qdev_prop_set_bit(dev, "spi", is_spi);
+    object_property_set_bool(obj, true, "realized", &err);
+    if (err) {
+        error_report("sd_init failed: %s", error_get_pretty(err));
         return NULL;
     }
 
-    sd = (SDState *) g_malloc0(sizeof(SDState));
-    sd->buf = blk_blockalign(blk, 512);
-    sd->spi = is_spi;
-    sd->enable = true;
-    sd_reset(sd, blk);
-    if (sd->blk) {
-        blk_attach_dev_nofail(sd->blk, sd);
-        blk_set_dev_ops(sd->blk, &sd_block_ops, sd);
-    }
-    vmstate_register(NULL, -1, &sd_vmstate, sd);
-    return sd;
+    return SD_CARD(dev);
 }
 
 void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert)
@@ -667,8 +751,16 @@
     /* Not interpreting this as an app command */
     sd->card_status &= ~APP_CMD;
 
-    if (sd_cmd_type[req.cmd] == sd_ac || sd_cmd_type[req.cmd] == sd_adtc)
+    if (sd_cmd_type[req.cmd & 0x3F] == sd_ac
+        || sd_cmd_type[req.cmd & 0x3F] == sd_adtc) {
         rca = req.arg >> 16;
+    }
+
+    /* CMD23 (set block count) must be immediately followed by CMD18 or CMD25
+     * if not, its effects are cancelled */
+    if (sd->multi_blk_cnt != 0 && !(req.cmd == 18 || req.cmd == 25)) {
+        sd->multi_blk_cnt = 0;
+    }
 
     DPRINTF("CMD%d 0x%08x state %d\n", req.cmd, req.arg, sd->state);
     switch (req.cmd) {
@@ -680,7 +772,7 @@
 
         default:
             sd->state = sd_idle_state;
-            sd_reset(sd, sd->blk);
+            sd_reset(DEVICE(sd));
             return sd->spi ? sd_r1 : sd_r0;
         }
         break;
@@ -796,8 +888,9 @@
             sd->vhs = 0;
 
             /* No response if not exactly one VHS bit is set.  */
-            if (!(req.arg >> 8) || (req.arg >> ffs(req.arg & ~0xff)))
+            if (!(req.arg >> 8) || (req.arg >> (ctz32(req.arg & ~0xff) + 1))) {
                 return sd->spi ? sd_r7 : sd_r0;
+            }
 
             /* Accept.  */
             sd->vhs = req.arg;
@@ -964,6 +1057,17 @@
         }
         break;
 
+    case 23:    /* CMD23: SET_BLOCK_COUNT */
+        switch (sd->state) {
+        case sd_transfer_state:
+            sd->multi_blk_cnt = req.arg;
+            return sd_r1;
+
+        default:
+            break;
+        }
+        break;
+
     /* Block write commands (Class 4) */
     case 24:	/* CMD24:  WRITE_SINGLE_BLOCK */
         if (sd->spi)
@@ -1196,16 +1300,17 @@
 
     default:
     bad_cmd:
-        fprintf(stderr, "SD: Unknown CMD%i\n", req.cmd);
+        qemu_log_mask(LOG_GUEST_ERROR, "SD: Unknown CMD%i\n", req.cmd);
         return sd_illegal;
 
     unimplemented_cmd:
         /* Commands that are recognised but not yet implemented in SPI mode.  */
-        fprintf(stderr, "SD: CMD%i not implemented in SPI mode\n", req.cmd);
+        qemu_log_mask(LOG_UNIMP, "SD: CMD%i not implemented in SPI mode\n",
+                      req.cmd);
         return sd_illegal;
     }
 
-    fprintf(stderr, "SD: CMD%i in a wrong state\n", req.cmd);
+    qemu_log_mask(LOG_GUEST_ERROR, "SD: CMD%i in a wrong state\n", req.cmd);
     return sd_illegal;
 }
 
@@ -1273,9 +1378,28 @@
         }
         switch (sd->state) {
         case sd_idle_state:
+            /* If it's the first ACMD41 since reset, we need to decide
+             * whether to power up. If this is not an enquiry ACMD41,
+             * we immediately report power on and proceed below to the
+             * ready state, but if it is, we set a timer to model a
+             * delay for power up. This works around a bug in EDK2
+             * UEFI, which sends an initial enquiry ACMD41, but
+             * assumes that the card is in ready state as soon as it
+             * sees the power up bit set. */
+            if (!(sd->ocr & OCR_POWER_UP)) {
+                if ((req.arg & ACMD41_ENQUIRY_MASK) != 0) {
+                    timer_del(sd->ocr_power_timer);
+                    sd_ocr_powerup(sd);
+                } else if (!timer_pending(sd->ocr_power_timer)) {
+                    timer_mod_ns(sd->ocr_power_timer,
+                                 (qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)
+                                  + OCR_POWER_DELAY_NS));
+                }
+            }
+
             /* We accept any voltage.  10000 V is nothing.
              *
-             * We don't model init delay so just advance straight to ready state
+             * Once we're powered up, we advance straight to ready state
              * unless it's an enquiry ACMD41 (bits 23:0 == 0).
              */
             if (req.arg & ACMD41_ENQUIRY_MASK) {
@@ -1318,7 +1442,7 @@
         return sd_normal_command(sd, req);
     }
 
-    fprintf(stderr, "SD: ACMD%i in a wrong state\n", req.cmd);
+    qemu_log_mask(LOG_GUEST_ERROR, "SD: ACMD%i in a wrong state\n", req.cmd);
     return sd_illegal;
 }
 
@@ -1338,7 +1462,8 @@
     if (req->cmd == 16 || req->cmd == 55) {
         return 1;
     }
-    return sd_cmd_class[req->cmd] == 0 || sd_cmd_class[req->cmd] == 7;
+    return sd_cmd_class[req->cmd & 0x3F] == 0
+            || sd_cmd_class[req->cmd & 0x3F] == 7;
 }
 
 int sd_do_command(SDState *sd, SDRequest *req,
@@ -1361,7 +1486,7 @@
         if (!cmd_valid_while_locked(sd, req)) {
             sd->card_status |= ILLEGAL_COMMAND;
             sd->expecting_acmd = false;
-            fprintf(stderr, "SD: Card is locked\n");
+            qemu_log_mask(LOG_GUEST_ERROR, "SD: Card is locked\n");
             rtype = sd_illegal;
             goto send_response;
         }
@@ -1452,57 +1577,17 @@
 
 static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
 {
-    uint64_t end = addr + len;
-
     DPRINTF("sd_blk_read: addr = 0x%08llx, len = %d\n",
             (unsigned long long) addr, len);
-    if (!sd->blk || blk_read(sd->blk, addr >> 9, sd->buf, 1) < 0) {
+    if (!sd->blk || blk_pread(sd->blk, addr, sd->data, len) < 0) {
         fprintf(stderr, "sd_blk_read: read error on host side\n");
-        return;
     }
-
-    if (end > (addr & ~511) + 512) {
-        memcpy(sd->data, sd->buf + (addr & 511), 512 - (addr & 511));
-
-        if (blk_read(sd->blk, end >> 9, sd->buf, 1) < 0) {
-            fprintf(stderr, "sd_blk_read: read error on host side\n");
-            return;
-        }
-        memcpy(sd->data + 512 - (addr & 511), sd->buf, end & 511);
-    } else
-        memcpy(sd->data, sd->buf + (addr & 511), len);
 }
 
 static void sd_blk_write(SDState *sd, uint64_t addr, uint32_t len)
 {
-    uint64_t end = addr + len;
-
-    if ((addr & 511) || len < 512)
-        if (!sd->blk || blk_read(sd->blk, addr >> 9, sd->buf, 1) < 0) {
-            fprintf(stderr, "sd_blk_write: read error on host side\n");
-            return;
-        }
-
-    if (end > (addr & ~511) + 512) {
-        memcpy(sd->buf + (addr & 511), sd->data, 512 - (addr & 511));
-        if (blk_write(sd->blk, addr >> 9, sd->buf, 1) < 0) {
-            fprintf(stderr, "sd_blk_write: write error on host side\n");
-            return;
-        }
-
-        if (blk_read(sd->blk, end >> 9, sd->buf, 1) < 0) {
-            fprintf(stderr, "sd_blk_write: read error on host side\n");
-            return;
-        }
-        memcpy(sd->buf, sd->data + 512 - (addr & 511), end & 511);
-        if (blk_write(sd->blk, end >> 9, sd->buf, 1) < 0) {
-            fprintf(stderr, "sd_blk_write: write error on host side\n");
-        }
-    } else {
-        memcpy(sd->buf + (addr & 511), sd->data, len);
-        if (!sd->blk || blk_write(sd->blk, addr >> 9, sd->buf, 1) < 0) {
-            fprintf(stderr, "sd_blk_write: write error on host side\n");
-        }
+    if (!sd->blk || blk_pwrite(sd->blk, addr, sd->data, len, 0) < 0) {
+        fprintf(stderr, "sd_blk_write: write error on host side\n");
     }
 }
 
@@ -1519,7 +1604,8 @@
         return;
 
     if (sd->state != sd_receivingdata_state) {
-        fprintf(stderr, "sd_write_data: not in Receiving-Data state\n");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "sd_write_data: not in Receiving-Data state\n");
         return;
     }
 
@@ -1563,6 +1649,14 @@
             sd->csd[14] |= 0x40;
 
             /* Bzzzzzzztt .... Operation complete.  */
+            if (sd->multi_blk_cnt != 0) {
+                if (--sd->multi_blk_cnt == 0) {
+                    /* Stop! */
+                    sd->state = sd_transfer_state;
+                    break;
+                }
+            }
+
             sd->state = sd_receivingdata_state;
         }
         break;
@@ -1630,7 +1724,7 @@
         break;
 
     default:
-        fprintf(stderr, "sd_write_data: unknown command\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "sd_write_data: unknown command\n");
         break;
     }
 }
@@ -1645,7 +1739,8 @@
         return 0x00;
 
     if (sd->state != sd_sendingdata_state) {
-        fprintf(stderr, "sd_read_data: not in Sending-Data state\n");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "sd_read_data: not in Sending-Data state\n");
         return 0x00;
     }
 
@@ -1709,6 +1804,15 @@
         if (sd->data_offset >= io_len) {
             sd->data_start += io_len;
             sd->data_offset = 0;
+
+            if (sd->multi_blk_cnt != 0) {
+                if (--sd->multi_blk_cnt == 0) {
+                    /* Stop! */
+                    sd->state = sd_transfer_state;
+                    break;
+                }
+            }
+
             if (sd->data_start + io_len > sd->size) {
                 sd->card_status |= ADDRESS_ERROR;
                 break;
@@ -1747,7 +1851,7 @@
         break;
 
     default:
-        fprintf(stderr, "sd_read_data: unknown command\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "sd_read_data: unknown command\n");
         return 0x00;
     }
 
@@ -1763,3 +1867,71 @@
 {
     sd->enable = enable;
 }
+
+static void sd_instance_init(Object *obj)
+{
+    SDState *sd = SD_CARD(obj);
+
+    sd->enable = true;
+    sd->ocr_power_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sd_ocr_powerup, sd);
+}
+
+static void sd_realize(DeviceState *dev, Error **errp)
+{
+    SDState *sd = SD_CARD(dev);
+
+    if (sd->blk && blk_is_read_only(sd->blk)) {
+        error_setg(errp, "Cannot use read-only drive as SD card");
+        return;
+    }
+
+    if (sd->blk) {
+        blk_set_dev_ops(sd->blk, &sd_block_ops, sd);
+    }
+}
+
+static Property sd_properties[] = {
+    DEFINE_PROP_DRIVE("drive", SDState, blk),
+    /* We do not model the chip select pin, so allow the board to select
+     * whether card should be in SSI or MMC/SD mode.  It is also up to the
+     * board to ensure that ssi transfers only occur when the chip select
+     * is asserted.  */
+    DEFINE_PROP_BOOL("spi", SDState, spi, false),
+    DEFINE_PROP_END_OF_LIST()
+};
+
+static void sd_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SDCardClass *sc = SD_CARD_CLASS(klass);
+
+    dc->realize = sd_realize;
+    dc->props = sd_properties;
+    dc->vmsd = &sd_vmstate;
+    dc->reset = sd_reset;
+    dc->bus_type = TYPE_SD_BUS;
+
+    sc->do_command = sd_do_command;
+    sc->write_data = sd_write_data;
+    sc->read_data = sd_read_data;
+    sc->data_ready = sd_data_ready;
+    sc->enable = sd_enable;
+    sc->get_inserted = sd_get_inserted;
+    sc->get_readonly = sd_get_readonly;
+}
+
+static const TypeInfo sd_info = {
+    .name = TYPE_SD_CARD,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(SDState),
+    .class_size = sizeof(SDCardClass),
+    .class_init = sd_class_init,
+    .instance_init = sd_instance_init,
+};
+
+static void sd_register_types(void)
+{
+    type_register_static(&sd_info);
+}
+
+type_init(sd_register_types)
diff --git a/hw/sd/sdhci-internal.h b/hw/sd/sdhci-internal.h
new file mode 100644
index 0000000..161177c
--- /dev/null
+++ b/hw/sd/sdhci-internal.h
@@ -0,0 +1,232 @@
+/*
+ * SD Association Host Standard Specification v2.0 controller emulation
+ *
+ * Copyright (c) 2011 Samsung Electronics Co., Ltd.
+ * Mitsyanko Igor <i.mitsyanko@samsung.com>
+ * Peter A.G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ *
+ * Based on MMC controller for Samsung S5PC1xx-based board emulation
+ * by Alexey Merkulov and Vladimir Monakhov.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef SDHCI_INTERNAL_H
+#define SDHCI_INTERNAL_H
+
+#include "hw/sd/sdhci.h"
+
+/* R/W SDMA System Address register 0x0 */
+#define SDHC_SYSAD                     0x00
+
+/* R/W Host DMA Buffer Boundary and Transfer Block Size Register 0x0 */
+#define SDHC_BLKSIZE                   0x04
+
+/* R/W Blocks count for current transfer 0x0 */
+#define SDHC_BLKCNT                    0x06
+
+/* R/W Command Argument Register 0x0 */
+#define SDHC_ARGUMENT                  0x08
+
+/* R/W Transfer Mode Setting Register 0x0 */
+#define SDHC_TRNMOD                    0x0C
+#define SDHC_TRNS_DMA                  0x0001
+#define SDHC_TRNS_BLK_CNT_EN           0x0002
+#define SDHC_TRNS_ACMD12               0x0004
+#define SDHC_TRNS_READ                 0x0010
+#define SDHC_TRNS_MULTI                0x0020
+
+/* R/W Command Register 0x0 */
+#define SDHC_CMDREG                    0x0E
+#define SDHC_CMD_RSP_WITH_BUSY         (3 << 0)
+#define SDHC_CMD_DATA_PRESENT          (1 << 5)
+#define SDHC_CMD_SUSPEND               (1 << 6)
+#define SDHC_CMD_RESUME                (1 << 7)
+#define SDHC_CMD_ABORT                 ((1 << 6)|(1 << 7))
+#define SDHC_CMD_TYPE_MASK             ((1 << 6)|(1 << 7))
+#define SDHC_COMMAND_TYPE(x)           ((x) & SDHC_CMD_TYPE_MASK)
+
+/* ROC Response Register 0 0x0 */
+#define SDHC_RSPREG0                   0x10
+/* ROC Response Register 1 0x0 */
+#define SDHC_RSPREG1                   0x14
+/* ROC Response Register 2 0x0 */
+#define SDHC_RSPREG2                   0x18
+/* ROC Response Register 3 0x0 */
+#define SDHC_RSPREG3                   0x1C
+
+/* R/W Buffer Data Register 0x0 */
+#define SDHC_BDATA                     0x20
+
+/* R/ROC Present State Register 0x000A0000 */
+#define SDHC_PRNSTS                    0x24
+#define SDHC_CMD_INHIBIT               0x00000001
+#define SDHC_DATA_INHIBIT              0x00000002
+#define SDHC_DAT_LINE_ACTIVE           0x00000004
+#define SDHC_DOING_WRITE               0x00000100
+#define SDHC_DOING_READ                0x00000200
+#define SDHC_SPACE_AVAILABLE           0x00000400
+#define SDHC_DATA_AVAILABLE            0x00000800
+#define SDHC_CARD_PRESENT              0x00010000
+#define SDHC_CARD_DETECT               0x00040000
+#define SDHC_WRITE_PROTECT             0x00080000
+#define TRANSFERRING_DATA(x)           \
+    ((x) & (SDHC_DOING_READ | SDHC_DOING_WRITE))
+
+/* R/W Host control Register 0x0 */
+#define SDHC_HOSTCTL                   0x28
+#define SDHC_CTRL_DMA_CHECK_MASK       0x18
+#define SDHC_CTRL_SDMA                 0x00
+#define SDHC_CTRL_ADMA1_32             0x08
+#define SDHC_CTRL_ADMA2_32             0x10
+#define SDHC_CTRL_ADMA2_64             0x18
+#define SDHC_DMA_TYPE(x)               ((x) & SDHC_CTRL_DMA_CHECK_MASK)
+
+/* R/W Power Control Register 0x0 */
+#define SDHC_PWRCON                    0x29
+#define SDHC_POWER_ON                  (1 << 0)
+
+/* R/W Block Gap Control Register 0x0 */
+#define SDHC_BLKGAP                    0x2A
+#define SDHC_STOP_AT_GAP_REQ           0x01
+#define SDHC_CONTINUE_REQ              0x02
+
+/* R/W WakeUp Control Register 0x0 */
+#define SDHC_WAKCON                    0x2B
+#define SDHC_WKUP_ON_INS               (1 << 1)
+#define SDHC_WKUP_ON_RMV               (1 << 2)
+
+/* CLKCON */
+#define SDHC_CLKCON                    0x2C
+#define SDHC_CLOCK_INT_STABLE          0x0002
+#define SDHC_CLOCK_INT_EN              0x0001
+#define SDHC_CLOCK_SDCLK_EN            (1 << 2)
+#define SDHC_CLOCK_CHK_MASK            0x0007
+#define SDHC_CLOCK_IS_ON(x)            \
+    (((x) & SDHC_CLOCK_CHK_MASK) == SDHC_CLOCK_CHK_MASK)
+
+/* R/W Timeout Control Register 0x0 */
+#define SDHC_TIMEOUTCON                0x2E
+
+/* R/W Software Reset Register 0x0 */
+#define SDHC_SWRST                     0x2F
+#define SDHC_RESET_ALL                 0x01
+#define SDHC_RESET_CMD                 0x02
+#define SDHC_RESET_DATA                0x04
+
+/* ROC/RW1C Normal Interrupt Status Register 0x0 */
+#define SDHC_NORINTSTS                 0x30
+#define SDHC_NIS_ERR                   0x8000
+#define SDHC_NIS_CMDCMP                0x0001
+#define SDHC_NIS_TRSCMP                0x0002
+#define SDHC_NIS_BLKGAP                0x0004
+#define SDHC_NIS_DMA                   0x0008
+#define SDHC_NIS_WBUFRDY               0x0010
+#define SDHC_NIS_RBUFRDY               0x0020
+#define SDHC_NIS_INSERT                0x0040
+#define SDHC_NIS_REMOVE                0x0080
+#define SDHC_NIS_CARDINT               0x0100
+
+/* ROC/RW1C Error Interrupt Status Register 0x0 */
+#define SDHC_ERRINTSTS                 0x32
+#define SDHC_EIS_CMDTIMEOUT            0x0001
+#define SDHC_EIS_BLKGAP                0x0004
+#define SDHC_EIS_CMDIDX                0x0008
+#define SDHC_EIS_CMD12ERR              0x0100
+#define SDHC_EIS_ADMAERR               0x0200
+
+/* R/W Normal Interrupt Status Enable Register 0x0 */
+#define SDHC_NORINTSTSEN               0x34
+#define SDHC_NISEN_CMDCMP              0x0001
+#define SDHC_NISEN_TRSCMP              0x0002
+#define SDHC_NISEN_DMA                 0x0008
+#define SDHC_NISEN_WBUFRDY             0x0010
+#define SDHC_NISEN_RBUFRDY             0x0020
+#define SDHC_NISEN_INSERT              0x0040
+#define SDHC_NISEN_REMOVE              0x0080
+#define SDHC_NISEN_CARDINT             0x0100
+
+/* R/W Error Interrupt Status Enable Register 0x0 */
+#define SDHC_ERRINTSTSEN               0x36
+#define SDHC_EISEN_CMDTIMEOUT          0x0001
+#define SDHC_EISEN_BLKGAP              0x0004
+#define SDHC_EISEN_CMDIDX              0x0008
+#define SDHC_EISEN_ADMAERR             0x0200
+
+/* R/W Normal Interrupt Signal Enable Register 0x0 */
+#define SDHC_NORINTSIGEN               0x38
+#define SDHC_NORINTSIG_INSERT          (1 << 6)
+#define SDHC_NORINTSIG_REMOVE          (1 << 7)
+
+/* R/W Error Interrupt Signal Enable Register 0x0 */
+#define SDHC_ERRINTSIGEN               0x3A
+
+/* ROC Auto CMD12 error status register 0x0 */
+#define SDHC_ACMD12ERRSTS              0x3C
+
+/* HWInit Capabilities Register 0x05E80080 */
+#define SDHC_CAPAREG                   0x40
+#define SDHC_CAN_DO_DMA                0x00400000
+#define SDHC_CAN_DO_ADMA2              0x00080000
+#define SDHC_CAN_DO_ADMA1              0x00100000
+#define SDHC_64_BIT_BUS_SUPPORT        (1 << 28)
+#define SDHC_CAPAB_BLOCKSIZE(x)        (((x) >> 16) & 0x3)
+
+/* HWInit Maximum Current Capabilities Register 0x0 */
+#define SDHC_MAXCURR                   0x48
+
+/* W Force Event Auto CMD12 Error Interrupt Register 0x0000 */
+#define SDHC_FEAER                     0x50
+/* W Force Event Error Interrupt Register Error Interrupt 0x0000 */
+#define SDHC_FEERR                     0x52
+
+/* R/W ADMA Error Status Register 0x00 */
+#define SDHC_ADMAERR                   0x54
+#define SDHC_ADMAERR_LENGTH_MISMATCH   (1 << 2)
+#define SDHC_ADMAERR_STATE_ST_STOP     (0 << 0)
+#define SDHC_ADMAERR_STATE_ST_FDS      (1 << 0)
+#define SDHC_ADMAERR_STATE_ST_TFR      (3 << 0)
+#define SDHC_ADMAERR_STATE_MASK        (3 << 0)
+
+/* R/W ADMA System Address Register 0x00 */
+#define SDHC_ADMASYSADDR               0x58
+#define SDHC_ADMA_ATTR_SET_LEN         (1 << 4)
+#define SDHC_ADMA_ATTR_ACT_TRAN        (1 << 5)
+#define SDHC_ADMA_ATTR_ACT_LINK        (3 << 4)
+#define SDHC_ADMA_ATTR_INT             (1 << 2)
+#define SDHC_ADMA_ATTR_END             (1 << 1)
+#define SDHC_ADMA_ATTR_VALID           (1 << 0)
+#define SDHC_ADMA_ATTR_ACT_MASK        ((1 << 4)|(1 << 5))
+
+/* Slot interrupt status */
+#define SDHC_SLOT_INT_STATUS            0xFC
+
+/* HWInit Host Controller Version Register 0x0401 */
+#define SDHC_HCVER                      0xFE
+#define SD_HOST_SPECv2_VERS             0x2401
+
+#define SDHC_REGISTERS_MAP_SIZE         0x100
+#define SDHC_INSERTION_DELAY            (NANOSECONDS_PER_SECOND)
+#define SDHC_TRANSFER_DELAY             100
+#define SDHC_ADMA_DESCS_PER_DELAY       5
+#define SDHC_CMD_RESPONSE               (3 << 0)
+
+enum {
+    sdhc_not_stopped = 0, /* normal SDHC state */
+    sdhc_gap_read   = 1,  /* SDHC stopped at block gap during read operation */
+    sdhc_gap_write  = 2   /* SDHC stopped at block gap during write operation */
+};
+
+extern const VMStateDescription sdhci_vmstate;
+
+#endif
diff --git a/hw/sd/sdhci.c b/hw/sd/sdhci.c
index b380050..01fbf22 100644
--- a/hw/sd/sdhci.c
+++ b/hw/sd/sdhci.c
@@ -22,38 +22,42 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/dma.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
-
-#include "sdhci.h"
+#include "sdhci-internal.h"
+#include "qemu/log.h"
 
 /* host controller debug messages */
 #ifndef SDHC_DEBUG
 #define SDHC_DEBUG                        0
 #endif
 
-#if SDHC_DEBUG == 0
-    #define DPRINT_L1(fmt, args...)       do { } while (0)
-    #define DPRINT_L2(fmt, args...)       do { } while (0)
-    #define ERRPRINT(fmt, args...)        do { } while (0)
-#elif SDHC_DEBUG == 1
-    #define DPRINT_L1(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define DPRINT_L2(fmt, args...)       do { } while (0)
-    #define ERRPRINT(fmt, args...)        \
-        do {fprintf(stderr, "QEMU SDHC ERROR: "fmt, ## args); } while (0)
-#else
-    #define DPRINT_L1(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define DPRINT_L2(fmt, args...)       \
-        do {fprintf(stderr, "QEMU SDHC: "fmt, ## args); } while (0)
-    #define ERRPRINT(fmt, args...)        \
-        do {fprintf(stderr, "QEMU SDHC ERROR: "fmt, ## args); } while (0)
-#endif
+#define DPRINT_L1(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG) { \
+            fprintf(stderr, "QEMU SDHC: " fmt, ## args); \
+        } \
+    } while (0)
+#define DPRINT_L2(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG > 1) { \
+            fprintf(stderr, "QEMU SDHC: " fmt, ## args); \
+        } \
+    } while (0)
+#define ERRPRINT(fmt, args...) \
+    do { \
+        if (SDHC_DEBUG) { \
+            fprintf(stderr, "QEMU SDHC ERROR: " fmt, ## args); \
+        } \
+    } while (0)
+
+#define TYPE_SDHCI_BUS "sdhci-bus"
+#define SDHCI_BUS(obj) OBJECT_CHECK(SDBus, (obj), TYPE_SDHCI_BUS)
 
 /* Default SD/MMC host controller features information, which will be
  * presented in CAPABILITIES register of generic SD host controller at reset.
@@ -74,10 +78,10 @@
 #define SDHC_CAPAB_MAXBLOCKLENGTH 512ul
 /* Maximum clock frequency for SDclock in MHz
  * value in range 10-63 MHz, 0 - not defined */
-#define SDHC_CAPAB_BASECLKFREQ    0ul
+#define SDHC_CAPAB_BASECLKFREQ    52ul
 #define SDHC_CAPAB_TOUNIT         1ul  /* Timeout clock unit 0 - kHz, 1 - MHz */
 /* Timeout clock frequency 1-63, 0 - not defined */
-#define SDHC_CAPAB_TOCLKFREQ      0ul
+#define SDHC_CAPAB_TOCLKFREQ      52ul
 
 /* Now check all parameters and calculate CAPABILITIES REGISTER value */
 #if SDHC_CAPAB_64BITBUS > 1 || SDHC_CAPAB_18V > 1 || SDHC_CAPAB_30V > 1 ||     \
@@ -145,9 +149,9 @@
     }
 }
 
-static void sdhci_insert_eject_cb(void *opaque, int irq, int level)
+static void sdhci_set_inserted(DeviceState *dev, bool level)
 {
-    SDHCIState *s = (SDHCIState *)opaque;
+    SDHCIState *s = (SDHCIState *)dev;
     DPRINT_L1("Card state changed: %s!\n", level ? "insert" : "eject");
 
     if ((s->norintsts & SDHC_NIS_REMOVE) && level) {
@@ -172,9 +176,9 @@
     }
 }
 
-static void sdhci_card_readonly_cb(void *opaque, int irq, int level)
+static void sdhci_set_readonly(DeviceState *dev, bool level)
 {
-    SDHCIState *s = (SDHCIState *)opaque;
+    SDHCIState *s = (SDHCIState *)dev;
 
     if (level) {
         s->prnsts &= ~SDHC_WRITE_PROTECT;
@@ -186,6 +190,8 @@
 
 static void sdhci_reset(SDHCIState *s)
 {
+    DeviceState *dev = DEVICE(s);
+
     timer_del(s->insert_timer);
     timer_del(s->transfer_timer);
     /* Set all registers to 0. Capabilities registers are not cleared
@@ -193,18 +199,32 @@
      * initialization */
     memset(&s->sdmasysad, 0, (uintptr_t)&s->capareg - (uintptr_t)&s->sdmasysad);
 
-    sd_set_cb(s->card, s->ro_cb, s->eject_cb);
+    /* Reset other state based on current card insertion/readonly status */
+    sdhci_set_inserted(dev, sdbus_get_inserted(&s->sdbus));
+    sdhci_set_readonly(dev, sdbus_get_readonly(&s->sdbus));
+
     s->data_count = 0;
     s->stopped_state = sdhc_not_stopped;
+    s->pending_insert_state = false;
 }
 
-static void sdhci_do_data_transfer(void *opaque)
+static void sdhci_poweron_reset(DeviceState *dev)
 {
-    SDHCIState *s = (SDHCIState *)opaque;
+    /* QOM (ie power-on) reset. This is identical to reset
+     * commanded via device register apart from handling of the
+     * 'pending insert on powerup' quirk.
+     */
+    SDHCIState *s = (SDHCIState *)dev;
 
-    SDHCI_GET_CLASS(s)->data_transfer(s);
+    sdhci_reset(s);
+
+    if (s->pending_insert_quirk) {
+        s->pending_insert_state = true;
+    }
 }
 
+static void sdhci_data_transfer(void *opaque);
+
 static void sdhci_send_command(SDHCIState *s)
 {
     SDRequest request;
@@ -216,7 +236,7 @@
     request.cmd = s->cmdreg >> 8;
     request.arg = s->argument;
     DPRINT_L1("sending CMD%u ARG[0x%08x]\n", request.cmd, request.arg);
-    rlen = sd_do_command(s->card, &request, response);
+    rlen = sdbus_do_command(&s->sdbus, &request, response);
 
     if (s->cmdreg & SDHC_CMD_RESPONSE) {
         if (rlen == 4) {
@@ -248,9 +268,6 @@
             (s->cmdreg & SDHC_CMD_RESPONSE) == SDHC_CMD_RSP_WITH_BUSY) {
             s->norintsts |= SDHC_NIS_TRSCMP;
         }
-    } else if (rlen != 0 && (s->errintstsen & SDHC_EISEN_CMDIDX)) {
-        s->errintsts |= SDHC_EIS_CMDIDX;
-        s->norintsts |= SDHC_NIS_ERR;
     }
 
     if (s->norintstsen & SDHC_NISEN_CMDCMP) {
@@ -261,7 +278,7 @@
 
     if (s->blksize && (s->cmdreg & SDHC_CMD_DATA_PRESENT)) {
         s->data_count = 0;
-        sdhci_do_data_transfer(s);
+        sdhci_data_transfer(s);
     }
 }
 
@@ -275,7 +292,7 @@
         request.cmd = 0x0C;
         request.arg = 0;
         DPRINT_L1("Automatically issue CMD%d %08x\n", request.cmd, request.arg);
-        sd_do_command(s->card, &request, response);
+        sdbus_do_command(&s->sdbus, &request, response);
         /* Auto CMD12 response goes to the upper Response register */
         s->rspreg[3] = (response[0] << 24) | (response[1] << 16) |
                 (response[2] << 8) | response[3];
@@ -307,7 +324,7 @@
     }
 
     for (index = 0; index < (s->blksize & 0x0fff); index++) {
-        s->fifo_buffer[index] = sd_read_data(s->card);
+        s->fifo_buffer[index] = sdbus_read_data(&s->sdbus);
     }
 
     /* New data now available for READ through Buffer Port Register */
@@ -367,9 +384,9 @@
                  /* stop at gap request */
                 (s->stopped_state == sdhc_gap_read &&
                  !(s->prnsts & SDHC_DAT_LINE_ACTIVE))) {
-                SDHCI_GET_CLASS(s)->end_data_transfer(s);
+                sdhci_end_transfer(s);
             } else { /* if there are more data, read next block from card */
-                SDHCI_GET_CLASS(s)->read_block_from_card(s);
+                sdhci_read_block_from_card(s);
             }
             break;
         }
@@ -400,7 +417,7 @@
     }
 
     for (index = 0; index < (s->blksize & 0x0fff); index++) {
-        sd_write_data(s->card, s->fifo_buffer[index]);
+        sdbus_write_data(&s->sdbus, s->fifo_buffer[index]);
     }
 
     /* Next data can be written through BUFFER DATORT register */
@@ -410,7 +427,7 @@
     if ((s->trnmod & SDHC_TRNS_MULTI) == 0 ||
             ((s->trnmod & SDHC_TRNS_MULTI) &&
             (s->trnmod & SDHC_TRNS_BLK_CNT_EN) && (s->blkcnt == 0))) {
-        SDHCI_GET_CLASS(s)->end_data_transfer(s);
+        sdhci_end_transfer(s);
     } else if (s->norintstsen & SDHC_NISEN_WBUFRDY) {
         s->norintsts |= SDHC_NIS_WBUFRDY;
     }
@@ -422,7 +439,7 @@
         if (s->norintstsen & SDHC_EISEN_BLKGAP) {
             s->norintsts |= SDHC_EIS_BLKGAP;
         }
-        SDHCI_GET_CLASS(s)->end_data_transfer(s);
+        sdhci_end_transfer(s);
     }
 
     sdhci_update_irq(s);
@@ -450,7 +467,7 @@
             s->data_count = 0;
             s->prnsts &= ~SDHC_SPACE_AVAILABLE;
             if (s->prnsts & SDHC_DOING_WRITE) {
-                SDHCI_GET_CLASS(s)->write_block_to_card(s);
+                sdhci_write_block_to_card(s);
             }
         }
     }
@@ -482,7 +499,7 @@
         while (s->blkcnt) {
             if (s->data_count == 0) {
                 for (n = 0; n < block_size; n++) {
-                    s->fifo_buffer[n] = sd_read_data(s->card);
+                    s->fifo_buffer[n] = sdbus_read_data(&s->sdbus);
                 }
             }
             begin = s->data_count;
@@ -523,7 +540,7 @@
             s->sdmasysad += s->data_count - begin;
             if (s->data_count == block_size) {
                 for (n = 0; n < block_size; n++) {
-                    sd_write_data(s->card, s->fifo_buffer[n]);
+                    sdbus_write_data(&s->sdbus, s->fifo_buffer[n]);
                 }
                 s->data_count = 0;
                 if (s->trnmod & SDHC_TRNS_BLK_CNT_EN) {
@@ -537,7 +554,7 @@
     }
 
     if (s->blkcnt == 0) {
-        SDHCI_GET_CLASS(s)->end_data_transfer(s);
+        sdhci_end_transfer(s);
     } else {
         if (s->norintstsen & SDHC_NISEN_DMA) {
             s->norintsts |= SDHC_NIS_DMA;
@@ -555,7 +572,7 @@
 
     if (s->trnmod & SDHC_TRNS_READ) {
         for (n = 0; n < datacnt; n++) {
-            s->fifo_buffer[n] = sd_read_data(s->card);
+            s->fifo_buffer[n] = sdbus_read_data(&s->sdbus);
         }
         dma_memory_write(&address_space_memory, s->sdmasysad, s->fifo_buffer,
                          datacnt);
@@ -563,7 +580,7 @@
         dma_memory_read(&address_space_memory, s->sdmasysad, s->fifo_buffer,
                         datacnt);
         for (n = 0; n < datacnt; n++) {
-            sd_write_data(s->card, s->fifo_buffer[n]);
+            sdbus_write_data(&s->sdbus, s->fifo_buffer[n]);
         }
     }
 
@@ -571,7 +588,7 @@
         s->blkcnt--;
     }
 
-    SDHCI_GET_CLASS(s)->end_data_transfer(s);
+    sdhci_end_transfer(s);
 }
 
 typedef struct ADMADescr {
@@ -667,7 +684,7 @@
                 while (length) {
                     if (s->data_count == 0) {
                         for (n = 0; n < block_size; n++) {
-                            s->fifo_buffer[n] = sd_read_data(s->card);
+                            s->fifo_buffer[n] = sdbus_read_data(&s->sdbus);
                         }
                     }
                     begin = s->data_count;
@@ -708,7 +725,7 @@
                     dscr.addr += s->data_count - begin;
                     if (s->data_count == block_size) {
                         for (n = 0; n < block_size; n++) {
-                            sd_write_data(s->card, s->fifo_buffer[n]);
+                            sdbus_write_data(&s->sdbus, s->fifo_buffer[n]);
                         }
                         s->data_count = 0;
                         if (s->trnmod & SDHC_TRNS_BLK_CNT_EN) {
@@ -724,7 +741,8 @@
             break;
         case SDHC_ADMA_ATTR_ACT_LINK:   /* link to next descriptor table */
             s->admasysaddr = dscr.addr;
-            DPRINT_L1("ADMA link: admasysaddr=0x%lx\n", s->admasysaddr);
+            DPRINT_L1("ADMA link: admasysaddr=0x%" PRIx64 "\n",
+                      s->admasysaddr);
             break;
         default:
             s->admasysaddr += dscr.incr;
@@ -732,7 +750,8 @@
         }
 
         if (dscr.attr & SDHC_ADMA_ATTR_INT) {
-            DPRINT_L1("ADMA interrupt: admasysaddr=0x%lx\n", s->admasysaddr);
+            DPRINT_L1("ADMA interrupt: admasysaddr=0x%" PRIx64 "\n",
+                      s->admasysaddr);
             if (s->norintstsen & SDHC_NISEN_DMA) {
                 s->norintsts |= SDHC_NIS_DMA;
             }
@@ -758,7 +777,7 @@
 
                 sdhci_update_irq(s);
             }
-            SDHCI_GET_CLASS(s)->end_data_transfer(s);
+            sdhci_end_transfer(s);
             return;
         }
 
@@ -771,9 +790,9 @@
 
 /* Perform data transfer according to controller configuration */
 
-static void sdhci_data_transfer(SDHCIState *s)
+static void sdhci_data_transfer(void *opaque)
 {
-    SDHCIClass *k = SDHCI_GET_CLASS(s);
+    SDHCIState *s = (SDHCIState *)opaque;
 
     if (s->trnmod & SDHC_TRNS_DMA) {
         switch (SDHC_DMA_TYPE(s->hostctl)) {
@@ -784,9 +803,9 @@
             }
 
             if ((s->blkcnt == 1) || !(s->trnmod & SDHC_TRNS_MULTI)) {
-                k->do_sdma_single(s);
+                sdhci_sdma_transfer_single_block(s);
             } else {
-                k->do_sdma_multi(s);
+                sdhci_sdma_transfer_multi_blocks(s);
             }
 
             break;
@@ -796,7 +815,7 @@
                 break;
             }
 
-            k->do_adma(s);
+            sdhci_do_adma(s);
             break;
         case SDHC_CTRL_ADMA2_32:
             if (!(s->capareg & SDHC_CAN_DO_ADMA2)) {
@@ -804,7 +823,7 @@
                 break;
             }
 
-            k->do_adma(s);
+            sdhci_do_adma(s);
             break;
         case SDHC_CTRL_ADMA2_64:
             if (!(s->capareg & SDHC_CAN_DO_ADMA2) ||
@@ -813,28 +832,28 @@
                 break;
             }
 
-            k->do_adma(s);
+            sdhci_do_adma(s);
             break;
         default:
             ERRPRINT("Unsupported DMA type\n");
             break;
         }
     } else {
-        if ((s->trnmod & SDHC_TRNS_READ) && sd_data_ready(s->card)) {
+        if ((s->trnmod & SDHC_TRNS_READ) && sdbus_data_ready(&s->sdbus)) {
             s->prnsts |= SDHC_DOING_READ | SDHC_DATA_INHIBIT |
                     SDHC_DAT_LINE_ACTIVE;
-            SDHCI_GET_CLASS(s)->read_block_from_card(s);
+            sdhci_read_block_from_card(s);
         } else {
             s->prnsts |= SDHC_DOING_WRITE | SDHC_DAT_LINE_ACTIVE |
                     SDHC_SPACE_AVAILABLE | SDHC_DATA_INHIBIT;
-            SDHCI_GET_CLASS(s)->write_block_to_card(s);
+            sdhci_write_block_to_card(s);
         }
     }
 }
 
 static bool sdhci_can_issue_command(SDHCIState *s)
 {
-    if (!SDHC_CLOCK_IS_ON(s->clkcon) || !(s->pwrcon & SDHC_POWER_ON) ||
+    if (!SDHC_CLOCK_IS_ON(s->clkcon) ||
         (((s->prnsts & SDHC_DATA_INHIBIT) || s->stopped_state) &&
         ((s->cmdreg & SDHC_CMD_DATA_PRESENT) ||
         ((s->cmdreg & SDHC_CMD_RESPONSE) == SDHC_CMD_RSP_WITH_BUSY &&
@@ -858,8 +877,9 @@
     return true;
 }
 
-static uint32_t sdhci_read(SDHCIState *s, unsigned int offset, unsigned size)
+static uint64_t sdhci_read(void *opaque, hwaddr offset, unsigned size)
 {
+    SDHCIState *s = (SDHCIState *)opaque;
     uint32_t ret = 0;
 
     switch (offset & ~0x3) {
@@ -880,8 +900,8 @@
         break;
     case  SDHC_BDATA:
         if (sdhci_buff_access_is_sequential(s, offset - SDHC_BDATA)) {
-            ret = SDHCI_GET_CLASS(s)->bdata_read(s, size);
-            DPRINT_L2("read %ub: addr[0x%04x] -> %u(0x%x)\n", size, offset,
+            ret = sdhci_read_dataport(s, size);
+            DPRINT_L2("read %ub: addr[0x%04x] -> %u(0x%x)\n", size, (int)offset,
                       ret, ret);
             return ret;
         }
@@ -927,13 +947,13 @@
         ret = (SD_HOST_SPECv2_VERS << 16) | sdhci_slotint(s);
         break;
     default:
-        ERRPRINT("bad %ub read: addr[0x%04x]\n", size, offset);
+        ERRPRINT("bad %ub read: addr[0x%04x]\n", size, (int)offset);
         break;
     }
 
     ret >>= (offset & 0x3) * 8;
     ret &= (1ULL << (size * 8)) - 1;
-    DPRINT_L2("read %ub: addr[0x%04x] -> %u(0x%x)\n", size, offset, ret, ret);
+    DPRINT_L2("read %ub: addr[0x%04x] -> %u(0x%x)\n", size, (int)offset, ret, ret);
     return ret;
 }
 
@@ -948,10 +968,10 @@
             (s->blkgap & SDHC_STOP_AT_GAP_REQ) == 0) {
         if (s->stopped_state == sdhc_gap_read) {
             s->prnsts |= SDHC_DAT_LINE_ACTIVE | SDHC_DOING_READ;
-            SDHCI_GET_CLASS(s)->read_block_from_card(s);
+            sdhci_read_block_from_card(s);
         } else {
             s->prnsts |= SDHC_DAT_LINE_ACTIVE | SDHC_DOING_WRITE;
-            SDHCI_GET_CLASS(s)->write_block_to_card(s);
+            sdhci_write_block_to_card(s);
         }
         s->stopped_state = sdhc_not_stopped;
     } else if (!s->stopped_state && (value & SDHC_STOP_AT_GAP_REQ)) {
@@ -967,7 +987,7 @@
 {
     switch (value) {
     case SDHC_RESET_ALL:
-        DEVICE_GET_CLASS(s)->reset(DEVICE(s));
+        sdhci_reset(s);
         break;
     case SDHC_RESET_CMD:
         s->prnsts &= ~SDHC_CMD_INHIBIT;
@@ -987,10 +1007,12 @@
 }
 
 static void
-sdhci_write(SDHCIState *s, unsigned int offset, uint32_t value, unsigned size)
+sdhci_write(void *opaque, hwaddr offset, uint64_t val, unsigned size)
 {
+    SDHCIState *s = (SDHCIState *)opaque;
     unsigned shift =  8 * (offset & 0x3);
     uint32_t mask = ~(((1ULL << (size * 8)) - 1) << shift);
+    uint32_t value = val;
     value <<= shift;
 
     switch (offset & ~0x3) {
@@ -1000,7 +1022,7 @@
         /* Writing to last byte of sdmasysad might trigger transfer */
         if (!(mask & 0xFF000000) && TRANSFERRING_DATA(s->prnsts) && s->blkcnt &&
                 s->blksize && SDHC_DMA_TYPE(s->hostctl) == SDHC_CTRL_SDMA) {
-            SDHCI_GET_CLASS(s)->do_sdma_multi(s);
+            sdhci_sdma_transfer_multi_blocks(s);
         }
         break;
     case SDHC_BLKSIZE:
@@ -1008,6 +1030,16 @@
             MASKED_WRITE(s->blksize, mask, value);
             MASKED_WRITE(s->blkcnt, mask >> 16, value >> 16);
         }
+
+        /* Limit block size to the maximum buffer size */
+        if (extract32(s->blksize, 0, 12) > s->buf_maxsz) {
+            qemu_log_mask(LOG_GUEST_ERROR, "%s: Size 0x%x is larger than " \
+                          "the maximum buffer 0x%x", __func__, s->blksize,
+                          s->buf_maxsz);
+
+            s->blksize = deposit32(s->blksize, 0, 12, s->buf_maxsz);
+        }
+
         break;
     case SDHC_ARGUMENT:
         MASKED_WRITE(s->argument, mask, value);
@@ -1022,15 +1054,15 @@
         MASKED_WRITE(s->cmdreg, mask >> 16, value >> 16);
 
         /* Writing to the upper byte of CMDREG triggers SD command generation */
-        if ((mask & 0xFF000000) || !SDHCI_GET_CLASS(s)->can_issue_command(s)) {
+        if ((mask & 0xFF000000) || !sdhci_can_issue_command(s)) {
             break;
         }
 
-        SDHCI_GET_CLASS(s)->send_command(s);
+        sdhci_send_command(s);
         break;
     case  SDHC_BDATA:
         if (sdhci_buff_access_is_sequential(s, offset - SDHC_BDATA)) {
-            SDHCI_GET_CLASS(s)->bdata_write(s, value >> shift, size);
+            sdhci_write_dataport(s, value >> shift, size);
         }
         break;
     case SDHC_HOSTCTL:
@@ -1080,6 +1112,13 @@
         } else {
             s->norintsts &= ~SDHC_NIS_ERR;
         }
+        /* Quirk for Raspberry Pi: pending card insert interrupt
+         * appears when first enabled after power on */
+        if ((s->norintstsen & SDHC_NISEN_INSERT) && s->pending_insert_state) {
+            assert(s->pending_insert_quirk);
+            s->norintsts |= SDHC_NIS_INSERT;
+            s->pending_insert_state = false;
+        }
         sdhci_update_irq(s);
         break;
     case SDHC_NORINTSIGEN:
@@ -1111,32 +1150,16 @@
         break;
     default:
         ERRPRINT("bad %ub write offset: addr[0x%04x] <- %u(0x%x)\n",
-                size, offset, value >> shift, value >> shift);
+                 size, (int)offset, value >> shift, value >> shift);
         break;
     }
     DPRINT_L2("write %ub: addr[0x%04x] <- %u(0x%x)\n",
-            size, offset, value >> shift, value >> shift);
-}
-
-static uint64_t
-sdhci_readfn(void *opaque, hwaddr offset, unsigned size)
-{
-    SDHCIState *s = (SDHCIState *)opaque;
-
-    return SDHCI_GET_CLASS(s)->mem_read(s, offset, size);
-}
-
-static void
-sdhci_writefn(void *opaque, hwaddr off, uint64_t val, unsigned sz)
-{
-    SDHCIState *s = (SDHCIState *)opaque;
-
-    SDHCI_GET_CLASS(s)->mem_write(s, off, val, sz);
+              size, (int)offset, value >> shift, value >> shift);
 }
 
 static const MemoryRegionOps sdhci_mmio_ops = {
-    .read = sdhci_readfn,
-    .write = sdhci_writefn,
+    .read = sdhci_read,
+    .write = sdhci_write,
     .valid = {
         .min_access_size = 1,
         .max_access_size = 4,
@@ -1160,28 +1183,17 @@
     }
 }
 
-static void sdhci_initfn(Object *obj)
+static void sdhci_initfn(SDHCIState *s)
 {
-    SDHCIState *s = SDHCI(obj);
-    DriveInfo *di;
-
-    di = drive_get_next(IF_SD);
-    s->card = sd_init(di ? blk_by_legacy_dinfo(di) : NULL, false);
-    if (s->card == NULL) {
-        exit(1);
-    }
-    s->eject_cb = qemu_allocate_irq(sdhci_insert_eject_cb, s, 0);
-    s->ro_cb = qemu_allocate_irq(sdhci_card_readonly_cb, s, 0);
-    sd_set_cb(s->card, s->ro_cb, s->eject_cb);
+    qbus_create_inplace(&s->sdbus, sizeof(s->sdbus),
+                        TYPE_SDHCI_BUS, DEVICE(s), "sd-bus");
 
     s->insert_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sdhci_raise_insertion_irq, s);
-    s->transfer_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sdhci_do_data_transfer, s);
+    s->transfer_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sdhci_data_transfer, s);
 }
 
-static void sdhci_uninitfn(Object *obj)
+static void sdhci_uninitfn(SDHCIState *s)
 {
-    SDHCIState *s = SDHCI(obj);
-
     timer_del(s->insert_timer);
     timer_free(s->insert_timer);
     timer_del(s->transfer_timer);
@@ -1189,12 +1201,28 @@
     qemu_free_irq(s->eject_cb);
     qemu_free_irq(s->ro_cb);
 
-    if (s->fifo_buffer) {
-        g_free(s->fifo_buffer);
-        s->fifo_buffer = NULL;
-    }
+    g_free(s->fifo_buffer);
+    s->fifo_buffer = NULL;
 }
 
+static bool sdhci_pending_insert_vmstate_needed(void *opaque)
+{
+    SDHCIState *s = opaque;
+
+    return s->pending_insert_state;
+}
+
+static const VMStateDescription sdhci_pending_insert_vmstate = {
+    .name = "sdhci/pending-insert",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = sdhci_pending_insert_vmstate_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(pending_insert_state, SDHCIState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 const VMStateDescription sdhci_vmstate = {
     .name = "sdhci",
     .version_id = 1,
@@ -1226,24 +1254,93 @@
         VMSTATE_UINT64(admasysaddr, SDHCIState),
         VMSTATE_UINT8(stopped_state, SDHCIState),
         VMSTATE_VBUFFER_UINT32(fifo_buffer, SDHCIState, 1, NULL, 0, buf_maxsz),
-        VMSTATE_TIMER(insert_timer, SDHCIState),
-        VMSTATE_TIMER(transfer_timer, SDHCIState),
+        VMSTATE_TIMER_PTR(insert_timer, SDHCIState),
+        VMSTATE_TIMER_PTR(transfer_timer, SDHCIState),
         VMSTATE_END_OF_LIST()
-    }
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &sdhci_pending_insert_vmstate,
+        NULL
+    },
 };
 
 /* Capabilities registers provide information on supported features of this
  * specific host controller implementation */
-static Property sdhci_properties[] = {
+static Property sdhci_pci_properties[] = {
     DEFINE_PROP_UINT32("capareg", SDHCIState, capareg,
             SDHC_CAPAB_REG_DEFAULT),
     DEFINE_PROP_UINT32("maxcurr", SDHCIState, maxcurr, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void sdhci_realize(DeviceState *dev, Error ** errp)
+static void sdhci_pci_realize(PCIDevice *dev, Error **errp)
 {
-    SDHCIState *s = SDHCI(dev);
+    SDHCIState *s = PCI_SDHCI(dev);
+    dev->config[PCI_CLASS_PROG] = 0x01; /* Standard Host supported DMA */
+    dev->config[PCI_INTERRUPT_PIN] = 0x01; /* interrupt pin A */
+    sdhci_initfn(s);
+    s->buf_maxsz = sdhci_get_fifolen(s);
+    s->fifo_buffer = g_malloc0(s->buf_maxsz);
+    s->irq = pci_allocate_irq(dev);
+    memory_region_init_io(&s->iomem, OBJECT(s), &sdhci_mmio_ops, s, "sdhci",
+            SDHC_REGISTERS_MAP_SIZE);
+    pci_register_bar(dev, 0, 0, &s->iomem);
+}
+
+static void sdhci_pci_exit(PCIDevice *dev)
+{
+    SDHCIState *s = PCI_SDHCI(dev);
+    sdhci_uninitfn(s);
+}
+
+static void sdhci_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->realize = sdhci_pci_realize;
+    k->exit = sdhci_pci_exit;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT;
+    k->device_id = PCI_DEVICE_ID_REDHAT_SDHCI;
+    k->class_id = PCI_CLASS_SYSTEM_SDHCI;
+    set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
+    dc->vmsd = &sdhci_vmstate;
+    dc->props = sdhci_pci_properties;
+    dc->reset = sdhci_poweron_reset;
+}
+
+static const TypeInfo sdhci_pci_info = {
+    .name = TYPE_PCI_SDHCI,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(SDHCIState),
+    .class_init = sdhci_pci_class_init,
+};
+
+static Property sdhci_sysbus_properties[] = {
+    DEFINE_PROP_UINT32("capareg", SDHCIState, capareg,
+            SDHC_CAPAB_REG_DEFAULT),
+    DEFINE_PROP_UINT32("maxcurr", SDHCIState, maxcurr, 0),
+    DEFINE_PROP_BOOL("pending-insert-quirk", SDHCIState, pending_insert_quirk,
+                     false),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void sdhci_sysbus_init(Object *obj)
+{
+    SDHCIState *s = SYSBUS_SDHCI(obj);
+
+    sdhci_initfn(s);
+}
+
+static void sdhci_sysbus_finalize(Object *obj)
+{
+    SDHCIState *s = SYSBUS_SDHCI(obj);
+    sdhci_uninitfn(s);
+}
+
+static void sdhci_sysbus_realize(DeviceState *dev, Error ** errp)
+{
+    SDHCIState *s = SYSBUS_SDHCI(dev);
     SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
 
     s->buf_maxsz = sdhci_get_fifolen(s);
@@ -1254,51 +1351,45 @@
     sysbus_init_mmio(sbd, &s->iomem);
 }
 
-static void sdhci_generic_reset(DeviceState *ds)
-{
-    SDHCIState *s = SDHCI(ds);
-    SDHCI_GET_CLASS(s)->reset(s);
-}
-
-static void sdhci_class_init(ObjectClass *klass, void *data)
+static void sdhci_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SDHCIClass *k = SDHCI_CLASS(klass);
 
     dc->vmsd = &sdhci_vmstate;
-    dc->props = sdhci_properties;
-    dc->reset = sdhci_generic_reset;
-    dc->realize = sdhci_realize;
-
-    k->reset = sdhci_reset;
-    k->mem_read = sdhci_read;
-    k->mem_write = sdhci_write;
-    k->send_command = sdhci_send_command;
-    k->can_issue_command = sdhci_can_issue_command;
-    k->data_transfer = sdhci_data_transfer;
-    k->end_data_transfer = sdhci_end_transfer;
-    k->do_sdma_single = sdhci_sdma_transfer_single_block;
-    k->do_sdma_multi = sdhci_sdma_transfer_multi_blocks;
-    k->do_adma = sdhci_do_adma;
-    k->read_block_from_card = sdhci_read_block_from_card;
-    k->write_block_to_card = sdhci_write_block_to_card;
-    k->bdata_read = sdhci_read_dataport;
-    k->bdata_write = sdhci_write_dataport;
+    dc->props = sdhci_sysbus_properties;
+    dc->realize = sdhci_sysbus_realize;
+    dc->reset = sdhci_poweron_reset;
 }
 
-static const TypeInfo sdhci_type_info = {
-    .name = TYPE_SDHCI,
+static const TypeInfo sdhci_sysbus_info = {
+    .name = TYPE_SYSBUS_SDHCI,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SDHCIState),
-    .instance_init = sdhci_initfn,
-    .instance_finalize = sdhci_uninitfn,
-    .class_init = sdhci_class_init,
-    .class_size = sizeof(SDHCIClass)
+    .instance_init = sdhci_sysbus_init,
+    .instance_finalize = sdhci_sysbus_finalize,
+    .class_init = sdhci_sysbus_class_init,
+};
+
+static void sdhci_bus_class_init(ObjectClass *klass, void *data)
+{
+    SDBusClass *sbc = SD_BUS_CLASS(klass);
+
+    sbc->set_inserted = sdhci_set_inserted;
+    sbc->set_readonly = sdhci_set_readonly;
+}
+
+static const TypeInfo sdhci_bus_info = {
+    .name = TYPE_SDHCI_BUS,
+    .parent = TYPE_SD_BUS,
+    .instance_size = sizeof(SDBus),
+    .class_init = sdhci_bus_class_init,
 };
 
 static void sdhci_register_types(void)
 {
-    type_register_static(&sdhci_type_info);
+    type_register_static(&sdhci_pci_info);
+    type_register_static(&sdhci_sysbus_info);
+    type_register_static(&sdhci_bus_info);
 }
 
 type_init(sdhci_register_types)
diff --git a/hw/sd/sdhci.h b/hw/sd/sdhci.h
deleted file mode 100644
index a560c3c..0000000
--- a/hw/sd/sdhci.h
+++ /dev/null
@@ -1,312 +0,0 @@
-/*
- * SD Association Host Standard Specification v2.0 controller emulation
- *
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- * Mitsyanko Igor <i.mitsyanko@samsung.com>
- * Peter A.G. Crosthwaite <peter.crosthwaite@petalogix.com>
- *
- * Based on MMC controller for Samsung S5PC1xx-based board emulation
- * by Alexey Merkulov and Vladimir Monakhov.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; either version 2 of the License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- * See the GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef SDHCI_H
-#define SDHCI_H
-
-#include "qemu-common.h"
-#include "hw/sysbus.h"
-#include "hw/sd.h"
-
-/* R/W SDMA System Address register 0x0 */
-#define SDHC_SYSAD                     0x00
-
-/* R/W Host DMA Buffer Boundary and Transfer Block Size Register 0x0 */
-#define SDHC_BLKSIZE                   0x04
-
-/* R/W Blocks count for current transfer 0x0 */
-#define SDHC_BLKCNT                    0x06
-
-/* R/W Command Argument Register 0x0 */
-#define SDHC_ARGUMENT                  0x08
-
-/* R/W Transfer Mode Setting Register 0x0 */
-#define SDHC_TRNMOD                    0x0C
-#define SDHC_TRNS_DMA                  0x0001
-#define SDHC_TRNS_BLK_CNT_EN           0x0002
-#define SDHC_TRNS_ACMD12               0x0004
-#define SDHC_TRNS_READ                 0x0010
-#define SDHC_TRNS_MULTI                0x0020
-
-/* R/W Command Register 0x0 */
-#define SDHC_CMDREG                    0x0E
-#define SDHC_CMD_RSP_WITH_BUSY         (3 << 0)
-#define SDHC_CMD_DATA_PRESENT          (1 << 5)
-#define SDHC_CMD_SUSPEND               (1 << 6)
-#define SDHC_CMD_RESUME                (1 << 7)
-#define SDHC_CMD_ABORT                 ((1 << 6)|(1 << 7))
-#define SDHC_CMD_TYPE_MASK             ((1 << 6)|(1 << 7))
-#define SDHC_COMMAND_TYPE(x)           ((x) & SDHC_CMD_TYPE_MASK)
-
-/* ROC Response Register 0 0x0 */
-#define SDHC_RSPREG0                   0x10
-/* ROC Response Register 1 0x0 */
-#define SDHC_RSPREG1                   0x14
-/* ROC Response Register 2 0x0 */
-#define SDHC_RSPREG2                   0x18
-/* ROC Response Register 3 0x0 */
-#define SDHC_RSPREG3                   0x1C
-
-/* R/W Buffer Data Register 0x0 */
-#define SDHC_BDATA                     0x20
-
-/* R/ROC Present State Register 0x000A0000 */
-#define SDHC_PRNSTS                    0x24
-#define SDHC_CMD_INHIBIT               0x00000001
-#define SDHC_DATA_INHIBIT              0x00000002
-#define SDHC_DAT_LINE_ACTIVE           0x00000004
-#define SDHC_DOING_WRITE               0x00000100
-#define SDHC_DOING_READ                0x00000200
-#define SDHC_SPACE_AVAILABLE           0x00000400
-#define SDHC_DATA_AVAILABLE            0x00000800
-#define SDHC_CARD_PRESENT              0x00010000
-#define SDHC_CARD_DETECT               0x00040000
-#define SDHC_WRITE_PROTECT             0x00080000
-#define TRANSFERRING_DATA(x)           \
-    ((x) & (SDHC_DOING_READ | SDHC_DOING_WRITE))
-
-/* R/W Host control Register 0x0 */
-#define SDHC_HOSTCTL                   0x28
-#define SDHC_CTRL_DMA_CHECK_MASK       0x18
-#define SDHC_CTRL_SDMA                 0x00
-#define SDHC_CTRL_ADMA1_32             0x08
-#define SDHC_CTRL_ADMA2_32             0x10
-#define SDHC_CTRL_ADMA2_64             0x18
-#define SDHC_DMA_TYPE(x)               ((x) & SDHC_CTRL_DMA_CHECK_MASK)
-
-/* R/W Power Control Register 0x0 */
-#define SDHC_PWRCON                    0x29
-#define SDHC_POWER_ON                  (1 << 0)
-
-/* R/W Block Gap Control Register 0x0 */
-#define SDHC_BLKGAP                    0x2A
-#define SDHC_STOP_AT_GAP_REQ           0x01
-#define SDHC_CONTINUE_REQ              0x02
-
-/* R/W WakeUp Control Register 0x0 */
-#define SDHC_WAKCON                    0x2B
-#define SDHC_WKUP_ON_INS               (1 << 1)
-#define SDHC_WKUP_ON_RMV               (1 << 2)
-
-/* CLKCON */
-#define SDHC_CLKCON                    0x2C
-#define SDHC_CLOCK_INT_STABLE          0x0002
-#define SDHC_CLOCK_INT_EN              0x0001
-#define SDHC_CLOCK_SDCLK_EN            (1 << 2)
-#define SDHC_CLOCK_CHK_MASK            0x0007
-#define SDHC_CLOCK_IS_ON(x)            \
-    (((x) & SDHC_CLOCK_CHK_MASK) == SDHC_CLOCK_CHK_MASK)
-
-/* R/W Timeout Control Register 0x0 */
-#define SDHC_TIMEOUTCON                0x2E
-
-/* R/W Software Reset Register 0x0 */
-#define SDHC_SWRST                     0x2F
-#define SDHC_RESET_ALL                 0x01
-#define SDHC_RESET_CMD                 0x02
-#define SDHC_RESET_DATA                0x04
-
-/* ROC/RW1C Normal Interrupt Status Register 0x0 */
-#define SDHC_NORINTSTS                 0x30
-#define SDHC_NIS_ERR                   0x8000
-#define SDHC_NIS_CMDCMP                0x0001
-#define SDHC_NIS_TRSCMP                0x0002
-#define SDHC_NIS_BLKGAP                0x0004
-#define SDHC_NIS_DMA                   0x0008
-#define SDHC_NIS_WBUFRDY               0x0010
-#define SDHC_NIS_RBUFRDY               0x0020
-#define SDHC_NIS_INSERT                0x0040
-#define SDHC_NIS_REMOVE                0x0080
-#define SDHC_NIS_CARDINT               0x0100
-
-/* ROC/RW1C Error Interrupt Status Register 0x0 */
-#define SDHC_ERRINTSTS                 0x32
-#define SDHC_EIS_CMDTIMEOUT            0x0001
-#define SDHC_EIS_BLKGAP                0x0004
-#define SDHC_EIS_CMDIDX                0x0008
-#define SDHC_EIS_CMD12ERR              0x0100
-#define SDHC_EIS_ADMAERR               0x0200
-
-/* R/W Normal Interrupt Status Enable Register 0x0 */
-#define SDHC_NORINTSTSEN               0x34
-#define SDHC_NISEN_CMDCMP              0x0001
-#define SDHC_NISEN_TRSCMP              0x0002
-#define SDHC_NISEN_DMA                 0x0008
-#define SDHC_NISEN_WBUFRDY             0x0010
-#define SDHC_NISEN_RBUFRDY             0x0020
-#define SDHC_NISEN_INSERT              0x0040
-#define SDHC_NISEN_REMOVE              0x0080
-#define SDHC_NISEN_CARDINT             0x0100
-
-/* R/W Error Interrupt Status Enable Register 0x0 */
-#define SDHC_ERRINTSTSEN               0x36
-#define SDHC_EISEN_CMDTIMEOUT          0x0001
-#define SDHC_EISEN_BLKGAP              0x0004
-#define SDHC_EISEN_CMDIDX              0x0008
-#define SDHC_EISEN_ADMAERR             0x0200
-
-/* R/W Normal Interrupt Signal Enable Register 0x0 */
-#define SDHC_NORINTSIGEN               0x38
-#define SDHC_NORINTSIG_INSERT          (1 << 6)
-#define SDHC_NORINTSIG_REMOVE          (1 << 7)
-
-/* R/W Error Interrupt Signal Enable Register 0x0 */
-#define SDHC_ERRINTSIGEN               0x3A
-
-/* ROC Auto CMD12 error status register 0x0 */
-#define SDHC_ACMD12ERRSTS              0x3C
-
-/* HWInit Capabilities Register 0x05E80080 */
-#define SDHC_CAPAREG                   0x40
-#define SDHC_CAN_DO_DMA                0x00400000
-#define SDHC_CAN_DO_ADMA2              0x00080000
-#define SDHC_CAN_DO_ADMA1              0x00100000
-#define SDHC_64_BIT_BUS_SUPPORT        (1 << 28)
-#define SDHC_CAPAB_BLOCKSIZE(x)        (((x) >> 16) & 0x3)
-
-/* HWInit Maximum Current Capabilities Register 0x0 */
-#define SDHC_MAXCURR                   0x48
-
-/* W Force Event Auto CMD12 Error Interrupt Register 0x0000 */
-#define SDHC_FEAER                     0x50
-/* W Force Event Error Interrupt Register Error Interrupt 0x0000 */
-#define SDHC_FEERR                     0x52
-
-/* R/W ADMA Error Status Register 0x00 */
-#define SDHC_ADMAERR                   0x54
-#define SDHC_ADMAERR_LENGTH_MISMATCH   (1 << 2)
-#define SDHC_ADMAERR_STATE_ST_STOP     (0 << 0)
-#define SDHC_ADMAERR_STATE_ST_FDS      (1 << 0)
-#define SDHC_ADMAERR_STATE_ST_TFR      (3 << 0)
-#define SDHC_ADMAERR_STATE_MASK        (3 << 0)
-
-/* R/W ADMA System Address Register 0x00 */
-#define SDHC_ADMASYSADDR               0x58
-#define SDHC_ADMA_ATTR_SET_LEN         (1 << 4)
-#define SDHC_ADMA_ATTR_ACT_TRAN        (1 << 5)
-#define SDHC_ADMA_ATTR_ACT_LINK        (3 << 4)
-#define SDHC_ADMA_ATTR_INT             (1 << 2)
-#define SDHC_ADMA_ATTR_END             (1 << 1)
-#define SDHC_ADMA_ATTR_VALID           (1 << 0)
-#define SDHC_ADMA_ATTR_ACT_MASK        ((1 << 4)|(1 << 5))
-
-/* Slot interrupt status */
-#define SDHC_SLOT_INT_STATUS            0xFC
-
-/* HWInit Host Controller Version Register 0x0401 */
-#define SDHC_HCVER                      0xFE
-#define SD_HOST_SPECv2_VERS             0x2401
-
-#define SDHC_REGISTERS_MAP_SIZE         0x100
-#define SDHC_INSERTION_DELAY            (get_ticks_per_sec())
-#define SDHC_TRANSFER_DELAY             100
-#define SDHC_ADMA_DESCS_PER_DELAY       5
-#define SDHC_CMD_RESPONSE               (3 << 0)
-
-enum {
-    sdhc_not_stopped = 0, /* normal SDHC state */
-    sdhc_gap_read   = 1,  /* SDHC stopped at block gap during read operation */
-    sdhc_gap_write  = 2   /* SDHC stopped at block gap during write operation */
-};
-
-/* SD/MMC host controller state */
-typedef struct SDHCIState {
-    SysBusDevice busdev;
-    SDState *card;
-    MemoryRegion iomem;
-
-    QEMUTimer *insert_timer;       /* timer for 'changing' sd card. */
-    QEMUTimer *transfer_timer;
-    qemu_irq eject_cb;
-    qemu_irq ro_cb;
-    qemu_irq irq;
-
-    uint32_t sdmasysad;    /* SDMA System Address register */
-    uint16_t blksize;      /* Host DMA Buff Boundary and Transfer BlkSize Reg */
-    uint16_t blkcnt;       /* Blocks count for current transfer */
-    uint32_t argument;     /* Command Argument Register */
-    uint16_t trnmod;       /* Transfer Mode Setting Register */
-    uint16_t cmdreg;       /* Command Register */
-    uint32_t rspreg[4];    /* Response Registers 0-3 */
-    uint32_t prnsts;       /* Present State Register */
-    uint8_t  hostctl;      /* Host Control Register */
-    uint8_t  pwrcon;       /* Power control Register */
-    uint8_t  blkgap;       /* Block Gap Control Register */
-    uint8_t  wakcon;       /* WakeUp Control Register */
-    uint16_t clkcon;       /* Clock control Register */
-    uint8_t  timeoutcon;   /* Timeout Control Register */
-    uint8_t  admaerr;      /* ADMA Error Status Register */
-    uint16_t norintsts;    /* Normal Interrupt Status Register */
-    uint16_t errintsts;    /* Error Interrupt Status Register */
-    uint16_t norintstsen;  /* Normal Interrupt Status Enable Register */
-    uint16_t errintstsen;  /* Error Interrupt Status Enable Register */
-    uint16_t norintsigen;  /* Normal Interrupt Signal Enable Register */
-    uint16_t errintsigen;  /* Error Interrupt Signal Enable Register */
-    uint16_t acmd12errsts; /* Auto CMD12 error status register */
-    uint64_t admasysaddr;  /* ADMA System Address Register */
-
-    uint32_t capareg;      /* Capabilities Register */
-    uint32_t maxcurr;      /* Maximum Current Capabilities Register */
-    uint8_t  *fifo_buffer; /* SD host i/o FIFO buffer */
-    uint32_t buf_maxsz;
-    uint16_t data_count;   /* current element in FIFO buffer */
-    uint8_t  stopped_state;/* Current SDHC state */
-    /* Buffer Data Port Register - virtual access point to R and W buffers */
-    /* Software Reset Register - always reads as 0 */
-    /* Force Event Auto CMD12 Error Interrupt Reg - write only */
-    /* Force Event Error Interrupt Register- write only */
-    /* RO Host Controller Version Register always reads as 0x2401 */
-} SDHCIState;
-
-typedef struct SDHCIClass {
-    SysBusDeviceClass busdev_class;
-
-    void (*reset)(SDHCIState *s);
-    uint32_t (*mem_read)(SDHCIState *s, unsigned int offset, unsigned size);
-    void (*mem_write)(SDHCIState *s, unsigned int offset, uint32_t value,
-            unsigned size);
-    void (*send_command)(SDHCIState *s);
-    bool (*can_issue_command)(SDHCIState *s);
-    void (*data_transfer)(SDHCIState *s);
-    void (*end_data_transfer)(SDHCIState *s);
-    void (*do_sdma_single)(SDHCIState *s);
-    void (*do_sdma_multi)(SDHCIState *s);
-    void (*do_adma)(SDHCIState *s);
-    void (*read_block_from_card)(SDHCIState *s);
-    void (*write_block_to_card)(SDHCIState *s);
-    uint32_t (*bdata_read)(SDHCIState *s, unsigned size);
-    void (*bdata_write)(SDHCIState *s, uint32_t value, unsigned size);
-} SDHCIClass;
-
-extern const VMStateDescription sdhci_vmstate;
-
-#define TYPE_SDHCI            "generic-sdhci"
-#define SDHCI(obj)            \
-     OBJECT_CHECK(SDHCIState, (obj), TYPE_SDHCI)
-#define SDHCI_CLASS(klass)    \
-     OBJECT_CLASS_CHECK(SDHCIClass, (klass), TYPE_SDHCI)
-#define SDHCI_GET_CLASS(obj)  \
-     OBJECT_GET_CLASS(SDHCIClass, (obj), TYPE_SDHCI)
-
-#endif /* SDHCI_H */
diff --git a/hw/sd/ssi-sd.c b/hw/sd/ssi-sd.c
index a71fbca..3ff0886 100644
--- a/hw/sd/ssi-sd.c
+++ b/hw/sd/ssi-sd.c
@@ -10,10 +10,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
-#include "hw/ssi.h"
-#include "hw/sd.h"
+#include "hw/ssi/ssi.h"
+#include "hw/sd/sd.h"
+#include "qapi/error.h"
 
 //#define DEBUG_SSI_SD 1
 
@@ -248,27 +250,28 @@
     return 0;
 }
 
-static int ssi_sd_init(SSISlave *d)
+static void ssi_sd_realize(SSISlave *d, Error **errp)
 {
     DeviceState *dev = DEVICE(d);
     ssi_sd_state *s = FROM_SSI_SLAVE(ssi_sd_state, d);
     DriveInfo *dinfo;
 
     s->mode = SSI_SD_CMD;
+    /* FIXME use a qdev drive property instead of drive_get_next() */
     dinfo = drive_get_next(IF_SD);
     s->sd = sd_init(dinfo ? blk_by_legacy_dinfo(dinfo) : NULL, true);
     if (s->sd == NULL) {
-        return -1;
+        error_setg(errp, "Device initialization failed.");
+        return;
     }
     register_savevm(dev, "ssi_sd", -1, 1, ssi_sd_save, ssi_sd_load, s);
-    return 0;
 }
 
 static void ssi_sd_class_init(ObjectClass *klass, void *data)
 {
     SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
 
-    k->init = ssi_sd_init;
+    k->realize = ssi_sd_realize;
     k->transfer = ssi_sd_transfer;
     k->cs_polarity = SSI_CS_LOW;
 }
diff --git a/hw/sd/trace-events b/hw/sd/trace-events
new file mode 100644
index 0000000..b17e7ba
--- /dev/null
+++ b/hw/sd/trace-events
@@ -0,0 +1,5 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/sd/milkymist-memcard.c
+milkymist_memcard_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_memcard_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
diff --git a/hw/sh4/r2d.c b/hw/sh4/r2d.c
index 12f44d2..db373c7 100644
--- a/hw/sh4/r2d.c
+++ b/hw/sh4/r2d.c
@@ -23,6 +23,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
@@ -127,7 +131,7 @@
     update_irl(fpga);
 }
 
-static uint32_t r2d_fpga_read(void *opaque, hwaddr addr)
+static uint64_t r2d_fpga_read(void *opaque, hwaddr addr, unsigned int size)
 {
     r2d_fpga_t *s = opaque;
 
@@ -146,7 +150,7 @@
 }
 
 static void
-r2d_fpga_write(void *opaque, hwaddr addr, uint32_t value)
+r2d_fpga_write(void *opaque, hwaddr addr, uint64_t value, unsigned int size)
 {
     r2d_fpga_t *s = opaque;
 
@@ -170,10 +174,10 @@
 }
 
 static const MemoryRegionOps r2d_fpga_ops = {
-    .old_mmio = {
-        .read = { r2d_fpga_read, r2d_fpga_read, NULL, },
-        .write = { r2d_fpga_write, r2d_fpga_write, NULL, },
-    },
+    .read = r2d_fpga_read,
+    .write = r2d_fpga_write,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
@@ -255,7 +259,7 @@
     qemu_register_reset(main_cpu_reset, reset_info);
 
     /* Allocate memory space */
-    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_abort);
+    memory_region_init_ram(sdram, NULL, "r2d.sdram", SDRAM_SIZE, &error_fatal);
     vmstate_register_ram_global(sdram);
     memory_region_add_subregion(address_space_mem, SDRAM_BASE, sdram);
     /* Register peripherals */
@@ -301,7 +305,7 @@
                             "rtl8139", i==0 ? "2" : NULL);
 
     /* USB keyboard */
-    usbdevice_create("keyboard");
+    usb_create_simple(usb_bus_find(-1), "usb-kbd");
 
     /* Todo: register on board registers */
     memset(&boot_params, 0, sizeof(boot_params));
@@ -318,8 +322,10 @@
         }
 
         /* initialization which should be done by firmware */
-        stl_phys(&address_space_memory, SH7750_BCR1, 1<<3); /* cs3 SDRAM */
-        stw_phys(&address_space_memory, SH7750_BCR2, 3<<(3*2)); /* cs3 32bit */
+        address_space_stl(&address_space_memory, SH7750_BCR1, 1 << 3,
+                          MEMTXATTRS_UNSPECIFIED, NULL); /* cs3 SDRAM */
+        address_space_stw(&address_space_memory, SH7750_BCR2, 3 << (3 * 2),
+                          MEMTXATTRS_UNSPECIFIED, NULL); /* cs3 32bit */
         reset_info->vector = (SDRAM_BASE + LINUX_LOAD_OFFSET) | 0xa0000000; /* Start from P2 area */
     }
 
@@ -336,9 +342,9 @@
         }
 
         /* initialization which should be done by firmware */
-        boot_params.loader_type = 1;
-        boot_params.initrd_start = INITRD_LOAD_OFFSET;
-        boot_params.initrd_size = initrd_size;
+        boot_params.loader_type = tswap32(1);
+        boot_params.initrd_start = tswap32(INITRD_LOAD_OFFSET);
+        boot_params.initrd_size = tswap32(initrd_size);
     }
 
     if (kernel_cmdline) {
@@ -352,15 +358,10 @@
                        SDRAM_BASE + BOOT_PARAMS_OFFSET);
 }
 
-static QEMUMachine r2d_machine = {
-    .name = "r2d",
-    .desc = "r2d-plus board",
-    .init = r2d_init,
-};
-
-static void r2d_machine_init(void)
+static void r2d_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&r2d_machine);
+    mc->desc = "r2d-plus board";
+    mc->init = r2d_init;
 }
 
-machine_init(r2d_machine_init);
+DEFINE_MACHINE("r2d", r2d_machine_init)
diff --git a/hw/sh4/sh7750.c b/hw/sh4/sh7750.c
index 5dda5de..3132d55 100644
--- a/hw/sh4/sh7750.c
+++ b/hw/sh4/sh7750.c
@@ -22,7 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sysemu/sysemu.h"
@@ -30,6 +30,7 @@
 #include "sh7750_regnames.h"
 #include "hw/sh4/sh_intc.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/address-spaces.h"
 
 #define NB_DEVICES 4
diff --git a/hw/sh4/sh7750_regnames.c b/hw/sh4/sh7750_regnames.c
index 52ac1cc..34b4f99 100644
--- a/hw/sh4/sh7750_regnames.c
+++ b/hw/sh4/sh7750_regnames.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sh7750_regs.h"
diff --git a/hw/sh4/sh7750_regnames.h b/hw/sh4/sh7750_regnames.h
index 7463709..e3ba886 100644
--- a/hw/sh4/sh7750_regnames.h
+++ b/hw/sh4/sh7750_regnames.h
@@ -1,6 +1,6 @@
-#ifndef _SH7750_REGNAMES_H
-#define _SH7750_REGNAMES_H
+#ifndef SH7750_REGNAMES_H
+#define SH7750_REGNAMES_H
 
 const char *regname(uint32_t addr);
 
-#endif				/* _SH7750_REGNAMES_H */
+#endif /* SH7750_REGNAMES_H */
diff --git a/hw/sh4/sh7750_regs.h b/hw/sh4/sh7750_regs.h
index 534aa48..3e4554a 100644
--- a/hw/sh4/sh7750_regs.h
+++ b/hw/sh4/sh7750_regs.h
@@ -16,8 +16,8 @@
  * @(#) sh7750_regs.h,v 1.2.4.1 2003/09/04 18:46:00 joel Exp
  */
 
-#ifndef __SH7750_REGS_H__
-#define __SH7750_REGS_H__
+#ifndef SH7750_REGS_H
+#define SH7750_REGS_H
 
 /*
  * All register has 2 addresses: in 0xff000000 - 0xffffffff (P4 address)  and
diff --git a/hw/sh4/sh_pci.c b/hw/sh4/sh_pci.c
index a2f6d9e..1747628 100644
--- a/hw/sh4/sh_pci.c
+++ b/hw/sh4/sh_pci.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/sh4/sh.h"
 #include "hw/pci/pci.h"
@@ -54,7 +55,7 @@
 
     switch(addr) {
     case 0 ... 0xfc:
-        cpu_to_le32w((uint32_t*)(pcic->dev->config + addr), val);
+        stl_le_p(pcic->dev->config + addr, val);
         break;
     case 0x1c0:
         pcic->par = val;
@@ -84,7 +85,7 @@
 
     switch(addr) {
     case 0 ... 0xfc:
-        return le32_to_cpup((uint32_t*)(pcic->dev->config + addr));
+        return ldl_le_p(pcic->dev->config + addr);
     case 0x1c0:
         return pcic->par;
     case 0x1c4:
@@ -151,12 +152,11 @@
     return 0;
 }
 
-static int sh_pci_host_init(PCIDevice *d)
+static void sh_pci_host_realize(PCIDevice *d, Error **errp)
 {
     pci_set_word(d->config + PCI_COMMAND, PCI_COMMAND_WAIT);
     pci_set_word(d->config + PCI_STATUS, PCI_STATUS_CAP_LIST |
                  PCI_STATUS_FAST_BACK | PCI_STATUS_DEVSEL_MEDIUM);
-    return 0;
 }
 
 static void sh_pci_host_class_init(ObjectClass *klass, void *data)
@@ -164,7 +164,7 @@
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    k->init = sh_pci_host_init;
+    k->realize = sh_pci_host_realize;
     k->vendor_id = PCI_VENDOR_ID_HITACHI;
     k->device_id = PCI_DEVICE_ID_HITACHI_SH7751R;
     /*
diff --git a/hw/sh4/shix.c b/hw/sh4/shix.c
index f93f98e..ccc9e75 100644
--- a/hw/sh4/shix.c
+++ b/hw/sh4/shix.c
@@ -27,6 +27,10 @@
 
    More information in target-sh4/README.sh4
 */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "sysemu/sysemu.h"
@@ -59,16 +63,16 @@
     }
 
     /* Allocate memory space */
-    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_abort);
+    memory_region_init_ram(rom, NULL, "shix.rom", 0x4000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_set_readonly(rom, true);
     memory_region_add_subregion(sysmem, 0x00000000, rom);
     memory_region_init_ram(&sdram[0], NULL, "shix.sdram1", 0x01000000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&sdram[0]);
     memory_region_add_subregion(sysmem, 0x08000000, &sdram[0]);
     memory_region_init_ram(&sdram[1], NULL, "shix.sdram2", 0x01000000,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&sdram[1]);
     memory_region_add_subregion(sysmem, 0x0c000000, &sdram[1]);
 
@@ -87,16 +91,11 @@
     tc58128_init(s, "shix_linux_nand.bin", NULL);
 }
 
-static QEMUMachine shix_machine = {
-    .name = "shix",
-    .desc = "shix card",
-    .init = shix_init,
-    .is_default = 1,
-};
-
-static void shix_machine_init(void)
+static void shix_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&shix_machine);
+    mc->desc = "shix card";
+    mc->init = shix_init;
+    mc->is_default = 1;
 }
 
-machine_init(shix_machine_init);
+DEFINE_MACHINE("shix", shix_machine_init)
diff --git a/hw/smbios/Makefile.objs b/hw/smbios/Makefile.objs
new file mode 100644
index 0000000..c3d3753
--- /dev/null
+++ b/hw/smbios/Makefile.objs
@@ -0,0 +1,2 @@
+common-obj-$(CONFIG_SMBIOS) += smbios.o
+common-obj-$(call land,$(CONFIG_SMBIOS),$(CONFIG_IPMI)) += smbios_type_38.o
diff --git a/hw/smbios/smbios.c b/hw/smbios/smbios.c
new file mode 100644
index 0000000..74c7102
--- /dev/null
+++ b/hw/smbios/smbios.c
@@ -0,0 +1,1056 @@
+/*
+ * SMBIOS Support
+ *
+ * Copyright (C) 2009 Hewlett-Packard Development Company, L.P.
+ * Copyright (C) 2013 Red Hat, Inc.
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@hp.com>
+ *  Markus Armbruster <armbru@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/cpus.h"
+#include "hw/smbios/smbios.h"
+#include "hw/loader.h"
+#include "exec/cpu-common.h"
+#include "smbios_build.h"
+#include "hw/smbios/ipmi.h"
+
+/* legacy structures and constants for <= 2.0 machines */
+struct smbios_header {
+    uint16_t length;
+    uint8_t type;
+} QEMU_PACKED;
+
+struct smbios_field {
+    struct smbios_header header;
+    uint8_t type;
+    uint16_t offset;
+    uint8_t data[];
+} QEMU_PACKED;
+
+struct smbios_table {
+    struct smbios_header header;
+    uint8_t data[];
+} QEMU_PACKED;
+
+#define SMBIOS_FIELD_ENTRY 0
+#define SMBIOS_TABLE_ENTRY 1
+
+static uint8_t *smbios_entries;
+static size_t smbios_entries_len;
+static bool smbios_legacy = true;
+static bool smbios_uuid_encoded = true;
+/* end: legacy structures & constants for <= 2.0 machines */
+
+
+uint8_t *smbios_tables;
+size_t smbios_tables_len;
+unsigned smbios_table_max;
+unsigned smbios_table_cnt;
+static SmbiosEntryPointType smbios_ep_type = SMBIOS_ENTRY_POINT_21;
+
+static SmbiosEntryPoint ep;
+
+static int smbios_type4_count = 0;
+static bool smbios_immutable;
+static bool smbios_have_defaults;
+static uint32_t smbios_cpuid_version, smbios_cpuid_features, smbios_smp_sockets;
+
+static DECLARE_BITMAP(have_binfile_bitmap, SMBIOS_MAX_TYPE+1);
+static DECLARE_BITMAP(have_fields_bitmap, SMBIOS_MAX_TYPE+1);
+
+static struct {
+    const char *vendor, *version, *date;
+    bool have_major_minor, uefi;
+    uint8_t major, minor;
+} type0;
+
+static struct {
+    const char *manufacturer, *product, *version, *serial, *sku, *family;
+    /* uuid is in qemu_uuid[] */
+} type1;
+
+static struct {
+    const char *manufacturer, *product, *version, *serial, *asset, *location;
+} type2;
+
+static struct {
+    const char *manufacturer, *version, *serial, *asset, *sku;
+} type3;
+
+static struct {
+    const char *sock_pfx, *manufacturer, *version, *serial, *asset, *part;
+} type4;
+
+static struct {
+    const char *loc_pfx, *bank, *manufacturer, *serial, *asset, *part;
+    uint16_t speed;
+} type17;
+
+static QemuOptsList qemu_smbios_opts = {
+    .name = "smbios",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_smbios_opts.head),
+    .desc = {
+        /*
+         * no elements => accept any params
+         * validation will happen later
+         */
+        { /* end of list */ }
+    }
+};
+
+static const QemuOptDesc qemu_smbios_file_opts[] = {
+    {
+        .name = "file",
+        .type = QEMU_OPT_STRING,
+        .help = "binary file containing an SMBIOS element",
+    },
+    { /* end of list */ }
+};
+
+static const QemuOptDesc qemu_smbios_type0_opts[] = {
+    {
+        .name = "type",
+        .type = QEMU_OPT_NUMBER,
+        .help = "SMBIOS element type",
+    },{
+        .name = "vendor",
+        .type = QEMU_OPT_STRING,
+        .help = "vendor name",
+    },{
+        .name = "version",
+        .type = QEMU_OPT_STRING,
+        .help = "version number",
+    },{
+        .name = "date",
+        .type = QEMU_OPT_STRING,
+        .help = "release date",
+    },{
+        .name = "release",
+        .type = QEMU_OPT_STRING,
+        .help = "revision number",
+    },{
+        .name = "uefi",
+        .type = QEMU_OPT_BOOL,
+        .help = "uefi support",
+    },
+    { /* end of list */ }
+};
+
+static const QemuOptDesc qemu_smbios_type1_opts[] = {
+    {
+        .name = "type",
+        .type = QEMU_OPT_NUMBER,
+        .help = "SMBIOS element type",
+    },{
+        .name = "manufacturer",
+        .type = QEMU_OPT_STRING,
+        .help = "manufacturer name",
+    },{
+        .name = "product",
+        .type = QEMU_OPT_STRING,
+        .help = "product name",
+    },{
+        .name = "version",
+        .type = QEMU_OPT_STRING,
+        .help = "version number",
+    },{
+        .name = "serial",
+        .type = QEMU_OPT_STRING,
+        .help = "serial number",
+    },{
+        .name = "uuid",
+        .type = QEMU_OPT_STRING,
+        .help = "UUID",
+    },{
+        .name = "sku",
+        .type = QEMU_OPT_STRING,
+        .help = "SKU number",
+    },{
+        .name = "family",
+        .type = QEMU_OPT_STRING,
+        .help = "family name",
+    },
+    { /* end of list */ }
+};
+
+static const QemuOptDesc qemu_smbios_type2_opts[] = {
+    {
+        .name = "type",
+        .type = QEMU_OPT_NUMBER,
+        .help = "SMBIOS element type",
+    },{
+        .name = "manufacturer",
+        .type = QEMU_OPT_STRING,
+        .help = "manufacturer name",
+    },{
+        .name = "product",
+        .type = QEMU_OPT_STRING,
+        .help = "product name",
+    },{
+        .name = "version",
+        .type = QEMU_OPT_STRING,
+        .help = "version number",
+    },{
+        .name = "serial",
+        .type = QEMU_OPT_STRING,
+        .help = "serial number",
+    },{
+        .name = "asset",
+        .type = QEMU_OPT_STRING,
+        .help = "asset tag number",
+    },{
+        .name = "location",
+        .type = QEMU_OPT_STRING,
+        .help = "location in chassis",
+    },
+    { /* end of list */ }
+};
+
+static const QemuOptDesc qemu_smbios_type3_opts[] = {
+    {
+        .name = "type",
+        .type = QEMU_OPT_NUMBER,
+        .help = "SMBIOS element type",
+    },{
+        .name = "manufacturer",
+        .type = QEMU_OPT_STRING,
+        .help = "manufacturer name",
+    },{
+        .name = "version",
+        .type = QEMU_OPT_STRING,
+        .help = "version number",
+    },{
+        .name = "serial",
+        .type = QEMU_OPT_STRING,
+        .help = "serial number",
+    },{
+        .name = "asset",
+        .type = QEMU_OPT_STRING,
+        .help = "asset tag number",
+    },{
+        .name = "sku",
+        .type = QEMU_OPT_STRING,
+        .help = "SKU number",
+    },
+    { /* end of list */ }
+};
+
+static const QemuOptDesc qemu_smbios_type4_opts[] = {
+    {
+        .name = "type",
+        .type = QEMU_OPT_NUMBER,
+        .help = "SMBIOS element type",
+    },{
+        .name = "sock_pfx",
+        .type = QEMU_OPT_STRING,
+        .help = "socket designation string prefix",
+    },{
+        .name = "manufacturer",
+        .type = QEMU_OPT_STRING,
+        .help = "manufacturer name",
+    },{
+        .name = "version",
+        .type = QEMU_OPT_STRING,
+        .help = "version number",
+    },{
+        .name = "serial",
+        .type = QEMU_OPT_STRING,
+        .help = "serial number",
+    },{
+        .name = "asset",
+        .type = QEMU_OPT_STRING,
+        .help = "asset tag number",
+    },{
+        .name = "part",
+        .type = QEMU_OPT_STRING,
+        .help = "part number",
+    },
+    { /* end of list */ }
+};
+
+static const QemuOptDesc qemu_smbios_type17_opts[] = {
+    {
+        .name = "type",
+        .type = QEMU_OPT_NUMBER,
+        .help = "SMBIOS element type",
+    },{
+        .name = "loc_pfx",
+        .type = QEMU_OPT_STRING,
+        .help = "device locator string prefix",
+    },{
+        .name = "bank",
+        .type = QEMU_OPT_STRING,
+        .help = "bank locator string",
+    },{
+        .name = "manufacturer",
+        .type = QEMU_OPT_STRING,
+        .help = "manufacturer name",
+    },{
+        .name = "serial",
+        .type = QEMU_OPT_STRING,
+        .help = "serial number",
+    },{
+        .name = "asset",
+        .type = QEMU_OPT_STRING,
+        .help = "asset tag number",
+    },{
+        .name = "part",
+        .type = QEMU_OPT_STRING,
+        .help = "part number",
+    },{
+        .name = "speed",
+        .type = QEMU_OPT_NUMBER,
+        .help = "maximum capable speed",
+    },
+    { /* end of list */ }
+};
+
+static void smbios_register_config(void)
+{
+    qemu_add_opts(&qemu_smbios_opts);
+}
+
+opts_init(smbios_register_config);
+
+static void smbios_validate_table(void)
+{
+    uint32_t expect_t4_count = smbios_legacy ? smp_cpus : smbios_smp_sockets;
+
+    if (smbios_type4_count && smbios_type4_count != expect_t4_count) {
+        error_report("Expected %d SMBIOS Type 4 tables, got %d instead",
+                     expect_t4_count, smbios_type4_count);
+        exit(1);
+    }
+}
+
+
+/* legacy setup functions for <= 2.0 machines */
+static void smbios_add_field(int type, int offset, const void *data, size_t len)
+{
+    struct smbios_field *field;
+
+    if (!smbios_entries) {
+        smbios_entries_len = sizeof(uint16_t);
+        smbios_entries = g_malloc0(smbios_entries_len);
+    }
+    smbios_entries = g_realloc(smbios_entries, smbios_entries_len +
+                                                  sizeof(*field) + len);
+    field = (struct smbios_field *)(smbios_entries + smbios_entries_len);
+    field->header.type = SMBIOS_FIELD_ENTRY;
+    field->header.length = cpu_to_le16(sizeof(*field) + len);
+
+    field->type = type;
+    field->offset = cpu_to_le16(offset);
+    memcpy(field->data, data, len);
+
+    smbios_entries_len += sizeof(*field) + len;
+    (*(uint16_t *)smbios_entries) =
+            cpu_to_le16(le16_to_cpu(*(uint16_t *)smbios_entries) + 1);
+}
+
+static void smbios_maybe_add_str(int type, int offset, const char *data)
+{
+    if (data) {
+        smbios_add_field(type, offset, data, strlen(data) + 1);
+    }
+}
+
+static void smbios_build_type_0_fields(void)
+{
+    smbios_maybe_add_str(0, offsetof(struct smbios_type_0, vendor_str),
+                         type0.vendor);
+    smbios_maybe_add_str(0, offsetof(struct smbios_type_0, bios_version_str),
+                         type0.version);
+    smbios_maybe_add_str(0, offsetof(struct smbios_type_0,
+                                     bios_release_date_str),
+                         type0.date);
+    if (type0.have_major_minor) {
+        smbios_add_field(0, offsetof(struct smbios_type_0,
+                                     system_bios_major_release),
+                         &type0.major, 1);
+        smbios_add_field(0, offsetof(struct smbios_type_0,
+                                     system_bios_minor_release),
+                         &type0.minor, 1);
+    }
+}
+
+static void smbios_build_type_1_fields(void)
+{
+    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, manufacturer_str),
+                         type1.manufacturer);
+    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, product_name_str),
+                         type1.product);
+    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, version_str),
+                         type1.version);
+    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, serial_number_str),
+                         type1.serial);
+    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, sku_number_str),
+                         type1.sku);
+    smbios_maybe_add_str(1, offsetof(struct smbios_type_1, family_str),
+                         type1.family);
+    if (qemu_uuid_set) {
+        /* We don't encode the UUID in the "wire format" here because this
+         * function is for legacy mode and needs to keep the guest ABI, and
+         * because we don't know what's the SMBIOS version advertised by the
+         * BIOS.
+         */
+        smbios_add_field(1, offsetof(struct smbios_type_1, uuid),
+                         qemu_uuid, 16);
+    }
+}
+
+uint8_t *smbios_get_table_legacy(size_t *length)
+{
+    if (!smbios_legacy) {
+        *length = 0;
+        return NULL;
+    }
+
+    if (!smbios_immutable) {
+        smbios_build_type_0_fields();
+        smbios_build_type_1_fields();
+        smbios_validate_table();
+        smbios_immutable = true;
+    }
+    *length = smbios_entries_len;
+    return smbios_entries;
+}
+/* end: legacy setup functions for <= 2.0 machines */
+
+
+bool smbios_skip_table(uint8_t type, bool required_table)
+{
+    if (test_bit(type, have_binfile_bitmap)) {
+        return true; /* user provided their own binary blob(s) */
+    }
+    if (test_bit(type, have_fields_bitmap)) {
+        return false; /* user provided fields via command line */
+    }
+    if (smbios_have_defaults && required_table) {
+        return false; /* we're building tables, and this one's required */
+    }
+    return true;
+}
+
+static void smbios_build_type_0_table(void)
+{
+    SMBIOS_BUILD_TABLE_PRE(0, 0x000, false); /* optional, leave up to BIOS */
+
+    SMBIOS_TABLE_SET_STR(0, vendor_str, type0.vendor);
+    SMBIOS_TABLE_SET_STR(0, bios_version_str, type0.version);
+
+    t->bios_starting_address_segment = cpu_to_le16(0xE800); /* from SeaBIOS */
+
+    SMBIOS_TABLE_SET_STR(0, bios_release_date_str, type0.date);
+
+    t->bios_rom_size = 0; /* hardcoded in SeaBIOS with FIXME comment */
+
+    t->bios_characteristics = cpu_to_le64(0x08); /* Not supported */
+    t->bios_characteristics_extension_bytes[0] = 0;
+    t->bios_characteristics_extension_bytes[1] = 0x14; /* TCD/SVVP | VM */
+    if (type0.uefi) {
+        t->bios_characteristics_extension_bytes[1] |= 0x08; /* |= UEFI */
+    }
+
+    if (type0.have_major_minor) {
+        t->system_bios_major_release = type0.major;
+        t->system_bios_minor_release = type0.minor;
+    } else {
+        t->system_bios_major_release = 0;
+        t->system_bios_minor_release = 0;
+    }
+
+    /* hardcoded in SeaBIOS */
+    t->embedded_controller_major_release = 0xFF;
+    t->embedded_controller_minor_release = 0xFF;
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+/* Encode UUID from the big endian encoding described on RFC4122 to the wire
+ * format specified by SMBIOS version 2.6.
+ */
+static void smbios_encode_uuid(struct smbios_uuid *uuid, const uint8_t *buf)
+{
+    memcpy(uuid, buf, 16);
+    if (smbios_uuid_encoded) {
+        uuid->time_low = bswap32(uuid->time_low);
+        uuid->time_mid = bswap16(uuid->time_mid);
+        uuid->time_hi_and_version = bswap16(uuid->time_hi_and_version);
+    }
+}
+
+static void smbios_build_type_1_table(void)
+{
+    SMBIOS_BUILD_TABLE_PRE(1, 0x100, true); /* required */
+
+    SMBIOS_TABLE_SET_STR(1, manufacturer_str, type1.manufacturer);
+    SMBIOS_TABLE_SET_STR(1, product_name_str, type1.product);
+    SMBIOS_TABLE_SET_STR(1, version_str, type1.version);
+    SMBIOS_TABLE_SET_STR(1, serial_number_str, type1.serial);
+    if (qemu_uuid_set) {
+        smbios_encode_uuid(&t->uuid, qemu_uuid);
+    } else {
+        memset(&t->uuid, 0, 16);
+    }
+    t->wake_up_type = 0x06; /* power switch */
+    SMBIOS_TABLE_SET_STR(1, sku_number_str, type1.sku);
+    SMBIOS_TABLE_SET_STR(1, family_str, type1.family);
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_build_type_2_table(void)
+{
+    SMBIOS_BUILD_TABLE_PRE(2, 0x200, false); /* optional */
+
+    SMBIOS_TABLE_SET_STR(2, manufacturer_str, type2.manufacturer);
+    SMBIOS_TABLE_SET_STR(2, product_str, type2.product);
+    SMBIOS_TABLE_SET_STR(2, version_str, type2.version);
+    SMBIOS_TABLE_SET_STR(2, serial_number_str, type2.serial);
+    SMBIOS_TABLE_SET_STR(2, asset_tag_number_str, type2.asset);
+    t->feature_flags = 0x01; /* Motherboard */
+    SMBIOS_TABLE_SET_STR(2, location_str, type2.location);
+    t->chassis_handle = cpu_to_le16(0x300); /* Type 3 (System enclosure) */
+    t->board_type = 0x0A; /* Motherboard */
+    t->contained_element_count = 0;
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_build_type_3_table(void)
+{
+    SMBIOS_BUILD_TABLE_PRE(3, 0x300, true); /* required */
+
+    SMBIOS_TABLE_SET_STR(3, manufacturer_str, type3.manufacturer);
+    t->type = 0x01; /* Other */
+    SMBIOS_TABLE_SET_STR(3, version_str, type3.version);
+    SMBIOS_TABLE_SET_STR(3, serial_number_str, type3.serial);
+    SMBIOS_TABLE_SET_STR(3, asset_tag_number_str, type3.asset);
+    t->boot_up_state = 0x03; /* Safe */
+    t->power_supply_state = 0x03; /* Safe */
+    t->thermal_state = 0x03; /* Safe */
+    t->security_status = 0x02; /* Unknown */
+    t->oem_defined = cpu_to_le32(0);
+    t->height = 0;
+    t->number_of_power_cords = 0;
+    t->contained_element_count = 0;
+    SMBIOS_TABLE_SET_STR(3, sku_number_str, type3.sku);
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_build_type_4_table(unsigned instance)
+{
+    char sock_str[128];
+
+    SMBIOS_BUILD_TABLE_PRE(4, 0x400 + instance, true); /* required */
+
+    snprintf(sock_str, sizeof(sock_str), "%s%2x", type4.sock_pfx, instance);
+    SMBIOS_TABLE_SET_STR(4, socket_designation_str, sock_str);
+    t->processor_type = 0x03; /* CPU */
+    t->processor_family = 0x01; /* Other */
+    SMBIOS_TABLE_SET_STR(4, processor_manufacturer_str, type4.manufacturer);
+    t->processor_id[0] = cpu_to_le32(smbios_cpuid_version);
+    t->processor_id[1] = cpu_to_le32(smbios_cpuid_features);
+    SMBIOS_TABLE_SET_STR(4, processor_version_str, type4.version);
+    t->voltage = 0;
+    t->external_clock = cpu_to_le16(0); /* Unknown */
+    /* SVVP requires max_speed and current_speed to not be unknown. */
+    t->max_speed = cpu_to_le16(2000); /* 2000 MHz */
+    t->current_speed = cpu_to_le16(2000); /* 2000 MHz */
+    t->status = 0x41; /* Socket populated, CPU enabled */
+    t->processor_upgrade = 0x01; /* Other */
+    t->l1_cache_handle = cpu_to_le16(0xFFFF); /* N/A */
+    t->l2_cache_handle = cpu_to_le16(0xFFFF); /* N/A */
+    t->l3_cache_handle = cpu_to_le16(0xFFFF); /* N/A */
+    SMBIOS_TABLE_SET_STR(4, serial_number_str, type4.serial);
+    SMBIOS_TABLE_SET_STR(4, asset_tag_number_str, type4.asset);
+    SMBIOS_TABLE_SET_STR(4, part_number_str, type4.part);
+    t->core_count = t->core_enabled = smp_cores;
+    t->thread_count = smp_threads;
+    t->processor_characteristics = cpu_to_le16(0x02); /* Unknown */
+    t->processor_family2 = cpu_to_le16(0x01); /* Other */
+
+    SMBIOS_BUILD_TABLE_POST;
+    smbios_type4_count++;
+}
+
+#define ONE_KB ((ram_addr_t)1 << 10)
+#define ONE_MB ((ram_addr_t)1 << 20)
+#define ONE_GB ((ram_addr_t)1 << 30)
+
+#define MAX_T16_STD_SZ 0x80000000 /* 2T in Kilobytes */
+
+static void smbios_build_type_16_table(unsigned dimm_cnt)
+{
+    uint64_t size_kb;
+
+    SMBIOS_BUILD_TABLE_PRE(16, 0x1000, true); /* required */
+
+    t->location = 0x01; /* Other */
+    t->use = 0x03; /* System memory */
+    t->error_correction = 0x06; /* Multi-bit ECC (for Microsoft, per SeaBIOS) */
+    size_kb = QEMU_ALIGN_UP(ram_size, ONE_KB) / ONE_KB;
+    if (size_kb < MAX_T16_STD_SZ) {
+        t->maximum_capacity = cpu_to_le32(size_kb);
+        t->extended_maximum_capacity = cpu_to_le64(0);
+    } else {
+        t->maximum_capacity = cpu_to_le32(MAX_T16_STD_SZ);
+        t->extended_maximum_capacity = cpu_to_le64(ram_size);
+    }
+    t->memory_error_information_handle = cpu_to_le16(0xFFFE); /* Not provided */
+    t->number_of_memory_devices = cpu_to_le16(dimm_cnt);
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+#define MAX_T17_STD_SZ 0x7FFF /* (32G - 1M), in Megabytes */
+#define MAX_T17_EXT_SZ 0x80000000 /* 2P, in Megabytes */
+
+static void smbios_build_type_17_table(unsigned instance, uint64_t size)
+{
+    char loc_str[128];
+    uint64_t size_mb;
+
+    SMBIOS_BUILD_TABLE_PRE(17, 0x1100 + instance, true); /* required */
+
+    t->physical_memory_array_handle = cpu_to_le16(0x1000); /* Type 16 above */
+    t->memory_error_information_handle = cpu_to_le16(0xFFFE); /* Not provided */
+    t->total_width = cpu_to_le16(0xFFFF); /* Unknown */
+    t->data_width = cpu_to_le16(0xFFFF); /* Unknown */
+    size_mb = QEMU_ALIGN_UP(size, ONE_MB) / ONE_MB;
+    if (size_mb < MAX_T17_STD_SZ) {
+        t->size = cpu_to_le16(size_mb);
+        t->extended_size = cpu_to_le32(0);
+    } else {
+        assert(size_mb < MAX_T17_EXT_SZ);
+        t->size = cpu_to_le16(MAX_T17_STD_SZ);
+        t->extended_size = cpu_to_le32(size_mb);
+    }
+    t->form_factor = 0x09; /* DIMM */
+    t->device_set = 0; /* Not in a set */
+    snprintf(loc_str, sizeof(loc_str), "%s %d", type17.loc_pfx, instance);
+    SMBIOS_TABLE_SET_STR(17, device_locator_str, loc_str);
+    SMBIOS_TABLE_SET_STR(17, bank_locator_str, type17.bank);
+    t->memory_type = 0x07; /* RAM */
+    t->type_detail = cpu_to_le16(0x02); /* Other */
+    t->speed = cpu_to_le16(type17.speed);
+    SMBIOS_TABLE_SET_STR(17, manufacturer_str, type17.manufacturer);
+    SMBIOS_TABLE_SET_STR(17, serial_number_str, type17.serial);
+    SMBIOS_TABLE_SET_STR(17, asset_tag_number_str, type17.asset);
+    SMBIOS_TABLE_SET_STR(17, part_number_str, type17.part);
+    t->attributes = 0; /* Unknown */
+    t->configured_clock_speed = t->speed; /* reuse value for max speed */
+    t->minimum_voltage = cpu_to_le16(0); /* Unknown */
+    t->maximum_voltage = cpu_to_le16(0); /* Unknown */
+    t->configured_voltage = cpu_to_le16(0); /* Unknown */
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_build_type_19_table(unsigned instance,
+                                       uint64_t start, uint64_t size)
+{
+    uint64_t end, start_kb, end_kb;
+
+    SMBIOS_BUILD_TABLE_PRE(19, 0x1300 + instance, true); /* required */
+
+    end = start + size - 1;
+    assert(end > start);
+    start_kb = start / ONE_KB;
+    end_kb = end / ONE_KB;
+    if (start_kb < UINT32_MAX && end_kb < UINT32_MAX) {
+        t->starting_address = cpu_to_le32(start_kb);
+        t->ending_address = cpu_to_le32(end_kb);
+        t->extended_starting_address =
+            t->extended_ending_address = cpu_to_le64(0);
+    } else {
+        t->starting_address = t->ending_address = cpu_to_le32(UINT32_MAX);
+        t->extended_starting_address = cpu_to_le64(start);
+        t->extended_ending_address = cpu_to_le64(end);
+    }
+    t->memory_array_handle = cpu_to_le16(0x1000); /* Type 16 above */
+    t->partition_width = 1; /* One device per row */
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_build_type_32_table(void)
+{
+    SMBIOS_BUILD_TABLE_PRE(32, 0x2000, true); /* required */
+
+    memset(t->reserved, 0, 6);
+    t->boot_status = 0; /* No errors detected */
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_build_type_127_table(void)
+{
+    SMBIOS_BUILD_TABLE_PRE(127, 0x7F00, true); /* required */
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+void smbios_set_cpuid(uint32_t version, uint32_t features)
+{
+    smbios_cpuid_version = version;
+    smbios_cpuid_features = features;
+}
+
+#define SMBIOS_SET_DEFAULT(field, value)                                  \
+    if (!field) {                                                         \
+        field = value;                                                    \
+    }
+
+void smbios_set_defaults(const char *manufacturer, const char *product,
+                         const char *version, bool legacy_mode,
+                         bool uuid_encoded, SmbiosEntryPointType ep_type)
+{
+    smbios_have_defaults = true;
+    smbios_legacy = legacy_mode;
+    smbios_uuid_encoded = uuid_encoded;
+    smbios_ep_type = ep_type;
+
+    /* drop unwanted version of command-line file blob(s) */
+    if (smbios_legacy) {
+        g_free(smbios_tables);
+        /* in legacy mode, also complain if fields were given for types > 1 */
+        if (find_next_bit(have_fields_bitmap,
+                          SMBIOS_MAX_TYPE+1, 2) < SMBIOS_MAX_TYPE+1) {
+            error_report("can't process fields for smbios "
+                         "types > 1 on machine versions < 2.1!");
+            exit(1);
+        }
+    } else {
+        g_free(smbios_entries);
+    }
+
+    SMBIOS_SET_DEFAULT(type1.manufacturer, manufacturer);
+    SMBIOS_SET_DEFAULT(type1.product, product);
+    SMBIOS_SET_DEFAULT(type1.version, version);
+    SMBIOS_SET_DEFAULT(type2.manufacturer, manufacturer);
+    SMBIOS_SET_DEFAULT(type2.product, product);
+    SMBIOS_SET_DEFAULT(type2.version, version);
+    SMBIOS_SET_DEFAULT(type3.manufacturer, manufacturer);
+    SMBIOS_SET_DEFAULT(type3.version, version);
+    SMBIOS_SET_DEFAULT(type4.sock_pfx, "CPU");
+    SMBIOS_SET_DEFAULT(type4.manufacturer, manufacturer);
+    SMBIOS_SET_DEFAULT(type4.version, version);
+    SMBIOS_SET_DEFAULT(type17.loc_pfx, "DIMM");
+    SMBIOS_SET_DEFAULT(type17.manufacturer, manufacturer);
+}
+
+static void smbios_entry_point_setup(void)
+{
+    switch (smbios_ep_type) {
+    case SMBIOS_ENTRY_POINT_21:
+        memcpy(ep.ep21.anchor_string, "_SM_", 4);
+        memcpy(ep.ep21.intermediate_anchor_string, "_DMI_", 5);
+        ep.ep21.length = sizeof(struct smbios_21_entry_point);
+        ep.ep21.entry_point_revision = 0; /* formatted_area reserved */
+        memset(ep.ep21.formatted_area, 0, 5);
+
+        /* compliant with smbios spec v2.8 */
+        ep.ep21.smbios_major_version = 2;
+        ep.ep21.smbios_minor_version = 8;
+        ep.ep21.smbios_bcd_revision = 0x28;
+
+        /* set during table construction, but BIOS may override: */
+        ep.ep21.structure_table_length = cpu_to_le16(smbios_tables_len);
+        ep.ep21.max_structure_size = cpu_to_le16(smbios_table_max);
+        ep.ep21.number_of_structures = cpu_to_le16(smbios_table_cnt);
+
+        /* BIOS must recalculate */
+        ep.ep21.checksum = 0;
+        ep.ep21.intermediate_checksum = 0;
+        ep.ep21.structure_table_address = cpu_to_le32(0);
+
+        break;
+    case SMBIOS_ENTRY_POINT_30:
+        memcpy(ep.ep30.anchor_string, "_SM3_", 5);
+        ep.ep30.length = sizeof(struct smbios_30_entry_point);
+        ep.ep30.entry_point_revision = 1;
+        ep.ep30.reserved = 0;
+
+        /* compliant with smbios spec 3.0 */
+        ep.ep30.smbios_major_version = 3;
+        ep.ep30.smbios_minor_version = 0;
+        ep.ep30.smbios_doc_rev = 0;
+
+        /* set during table construct, but BIOS might override */
+        ep.ep30.structure_table_max_size = cpu_to_le32(smbios_tables_len);
+
+        /* BIOS must recalculate */
+        ep.ep30.checksum = 0;
+        ep.ep30.structure_table_address = cpu_to_le64(0);
+
+        break;
+    default:
+        abort();
+        break;
+    }
+}
+
+void smbios_get_tables(const struct smbios_phys_mem_area *mem_array,
+                       const unsigned int mem_array_size,
+                       uint8_t **tables, size_t *tables_len,
+                       uint8_t **anchor, size_t *anchor_len)
+{
+    unsigned i, dimm_cnt;
+
+    if (smbios_legacy) {
+        *tables = *anchor = NULL;
+        *tables_len = *anchor_len = 0;
+        return;
+    }
+
+    if (!smbios_immutable) {
+        smbios_build_type_0_table();
+        smbios_build_type_1_table();
+        smbios_build_type_2_table();
+        smbios_build_type_3_table();
+
+        smbios_smp_sockets = DIV_ROUND_UP(smp_cpus, smp_cores * smp_threads);
+        assert(smbios_smp_sockets >= 1);
+
+        for (i = 0; i < smbios_smp_sockets; i++) {
+            smbios_build_type_4_table(i);
+        }
+
+#define MAX_DIMM_SZ (16ll * ONE_GB)
+#define GET_DIMM_SZ ((i < dimm_cnt - 1) ? MAX_DIMM_SZ \
+                                        : ((ram_size - 1) % MAX_DIMM_SZ) + 1)
+
+        dimm_cnt = QEMU_ALIGN_UP(ram_size, MAX_DIMM_SZ) / MAX_DIMM_SZ;
+
+        smbios_build_type_16_table(dimm_cnt);
+
+        for (i = 0; i < dimm_cnt; i++) {
+            smbios_build_type_17_table(i, GET_DIMM_SZ);
+        }
+
+        for (i = 0; i < mem_array_size; i++) {
+            smbios_build_type_19_table(i, mem_array[i].address,
+                                       mem_array[i].length);
+        }
+
+        smbios_build_type_32_table();
+        smbios_build_type_38_table();
+        smbios_build_type_127_table();
+
+        smbios_validate_table();
+        smbios_entry_point_setup();
+        smbios_immutable = true;
+    }
+
+    /* return tables blob and entry point (anchor), and their sizes */
+    *tables = smbios_tables;
+    *tables_len = smbios_tables_len;
+    *anchor = (uint8_t *)&ep;
+
+    /* calculate length based on anchor string */
+    if (!strncmp((char *)&ep, "_SM_", 4)) {
+        *anchor_len = sizeof(struct smbios_21_entry_point);
+    } else if (!strncmp((char *)&ep, "_SM3_", 5)) {
+        *anchor_len = sizeof(struct smbios_30_entry_point);
+    } else {
+        abort();
+    }
+}
+
+static void save_opt(const char **dest, QemuOpts *opts, const char *name)
+{
+    const char *val = qemu_opt_get(opts, name);
+
+    if (val) {
+        *dest = val;
+    }
+}
+
+void smbios_entry_add(QemuOpts *opts)
+{
+    const char *val;
+
+    assert(!smbios_immutable);
+
+    val = qemu_opt_get(opts, "file");
+    if (val) {
+        struct smbios_structure_header *header;
+        int size;
+        struct smbios_table *table; /* legacy mode only */
+
+        qemu_opts_validate(opts, qemu_smbios_file_opts, &error_fatal);
+
+        size = get_image_size(val);
+        if (size == -1 || size < sizeof(struct smbios_structure_header)) {
+            error_report("Cannot read SMBIOS file %s", val);
+            exit(1);
+        }
+
+        /*
+         * NOTE: standard double '\0' terminator expected, per smbios spec.
+         * (except in legacy mode, where the second '\0' is implicit and
+         *  will be inserted by the BIOS).
+         */
+        smbios_tables = g_realloc(smbios_tables, smbios_tables_len + size);
+        header = (struct smbios_structure_header *)(smbios_tables +
+                                                    smbios_tables_len);
+
+        if (load_image(val, (uint8_t *)header) != size) {
+            error_report("Failed to load SMBIOS file %s", val);
+            exit(1);
+        }
+
+        if (test_bit(header->type, have_fields_bitmap)) {
+            error_report("can't load type %d struct, fields already specified!",
+                         header->type);
+            exit(1);
+        }
+        set_bit(header->type, have_binfile_bitmap);
+
+        if (header->type == 4) {
+            smbios_type4_count++;
+        }
+
+        smbios_tables_len += size;
+        if (size > smbios_table_max) {
+            smbios_table_max = size;
+        }
+        smbios_table_cnt++;
+
+        /* add a copy of the newly loaded blob to legacy smbios_entries */
+        /* NOTE: This code runs before smbios_set_defaults(), so we don't
+         *       yet know which mode (legacy vs. aggregate-table) will be
+         *       required. We therefore add the binary blob to both legacy
+         *       (smbios_entries) and aggregate (smbios_tables) tables, and
+         *       delete the one we don't need from smbios_set_defaults(),
+         *       once we know which machine version has been requested.
+         */
+        if (!smbios_entries) {
+            smbios_entries_len = sizeof(uint16_t);
+            smbios_entries = g_malloc0(smbios_entries_len);
+        }
+        smbios_entries = g_realloc(smbios_entries, smbios_entries_len +
+                                                   size + sizeof(*table));
+        table = (struct smbios_table *)(smbios_entries + smbios_entries_len);
+        table->header.type = SMBIOS_TABLE_ENTRY;
+        table->header.length = cpu_to_le16(sizeof(*table) + size);
+        memcpy(table->data, header, size);
+        smbios_entries_len += sizeof(*table) + size;
+        (*(uint16_t *)smbios_entries) =
+                cpu_to_le16(le16_to_cpu(*(uint16_t *)smbios_entries) + 1);
+        /* end: add a copy of the newly loaded blob to legacy smbios_entries */
+
+        return;
+    }
+
+    val = qemu_opt_get(opts, "type");
+    if (val) {
+        unsigned long type = strtoul(val, NULL, 0);
+
+        if (type > SMBIOS_MAX_TYPE) {
+            error_report("out of range!");
+            exit(1);
+        }
+
+        if (test_bit(type, have_binfile_bitmap)) {
+            error_report("can't add fields, binary file already loaded!");
+            exit(1);
+        }
+        set_bit(type, have_fields_bitmap);
+
+        switch (type) {
+        case 0:
+            qemu_opts_validate(opts, qemu_smbios_type0_opts, &error_fatal);
+            save_opt(&type0.vendor, opts, "vendor");
+            save_opt(&type0.version, opts, "version");
+            save_opt(&type0.date, opts, "date");
+            type0.uefi = qemu_opt_get_bool(opts, "uefi", false);
+
+            val = qemu_opt_get(opts, "release");
+            if (val) {
+                if (sscanf(val, "%hhu.%hhu", &type0.major, &type0.minor) != 2) {
+                    error_report("Invalid release");
+                    exit(1);
+                }
+                type0.have_major_minor = true;
+            }
+            return;
+        case 1:
+            qemu_opts_validate(opts, qemu_smbios_type1_opts, &error_fatal);
+            save_opt(&type1.manufacturer, opts, "manufacturer");
+            save_opt(&type1.product, opts, "product");
+            save_opt(&type1.version, opts, "version");
+            save_opt(&type1.serial, opts, "serial");
+            save_opt(&type1.sku, opts, "sku");
+            save_opt(&type1.family, opts, "family");
+
+            val = qemu_opt_get(opts, "uuid");
+            if (val) {
+                if (qemu_uuid_parse(val, qemu_uuid) != 0) {
+                    error_report("Invalid UUID");
+                    exit(1);
+                }
+                qemu_uuid_set = true;
+            }
+            return;
+        case 2:
+            qemu_opts_validate(opts, qemu_smbios_type2_opts, &error_fatal);
+            save_opt(&type2.manufacturer, opts, "manufacturer");
+            save_opt(&type2.product, opts, "product");
+            save_opt(&type2.version, opts, "version");
+            save_opt(&type2.serial, opts, "serial");
+            save_opt(&type2.asset, opts, "asset");
+            save_opt(&type2.location, opts, "location");
+            return;
+        case 3:
+            qemu_opts_validate(opts, qemu_smbios_type3_opts, &error_fatal);
+            save_opt(&type3.manufacturer, opts, "manufacturer");
+            save_opt(&type3.version, opts, "version");
+            save_opt(&type3.serial, opts, "serial");
+            save_opt(&type3.asset, opts, "asset");
+            save_opt(&type3.sku, opts, "sku");
+            return;
+        case 4:
+            qemu_opts_validate(opts, qemu_smbios_type4_opts, &error_fatal);
+            save_opt(&type4.sock_pfx, opts, "sock_pfx");
+            save_opt(&type4.manufacturer, opts, "manufacturer");
+            save_opt(&type4.version, opts, "version");
+            save_opt(&type4.serial, opts, "serial");
+            save_opt(&type4.asset, opts, "asset");
+            save_opt(&type4.part, opts, "part");
+            return;
+        case 17:
+            qemu_opts_validate(opts, qemu_smbios_type17_opts, &error_fatal);
+            save_opt(&type17.loc_pfx, opts, "loc_pfx");
+            save_opt(&type17.bank, opts, "bank");
+            save_opt(&type17.manufacturer, opts, "manufacturer");
+            save_opt(&type17.serial, opts, "serial");
+            save_opt(&type17.asset, opts, "asset");
+            save_opt(&type17.part, opts, "part");
+            type17.speed = qemu_opt_get_number(opts, "speed", 0);
+            return;
+        default:
+            error_report("Don't know how to build fields for SMBIOS type %ld",
+                         type);
+            exit(1);
+        }
+    }
+
+    error_report("Must specify type= or file=");
+    exit(1);
+}
diff --git a/hw/smbios/smbios_build.h b/hw/smbios/smbios_build.h
new file mode 100644
index 0000000..68b8b72
--- /dev/null
+++ b/hw/smbios/smbios_build.h
@@ -0,0 +1,87 @@
+/*
+ * SMBIOS Support
+ *
+ * Copyright (C) 2009 Hewlett-Packard Development Company, L.P.
+ * Copyright (C) 2013 Red Hat, Inc.
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@hp.com>
+ *  Markus Armbruster <armbru@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#ifndef QEMU_SMBIOS_BUILD_H
+#define QEMU_SMBIOS_BUILD_H
+
+bool smbios_skip_table(uint8_t type, bool required_table);
+
+extern uint8_t *smbios_tables;
+extern size_t smbios_tables_len;
+extern unsigned smbios_table_max;
+extern unsigned smbios_table_cnt;
+
+#define SMBIOS_BUILD_TABLE_PRE(tbl_type, tbl_handle, tbl_required)        \
+    struct smbios_type_##tbl_type *t;                                     \
+    size_t t_off; /* table offset into smbios_tables */                   \
+    int str_index = 0;                                                    \
+    do {                                                                  \
+        /* should we skip building this table ? */                        \
+        if (smbios_skip_table(tbl_type, tbl_required)) {                  \
+            return;                                                       \
+        }                                                                 \
+                                                                          \
+        /* use offset of table t within smbios_tables */                  \
+        /* (pointer must be updated after each realloc) */                \
+        t_off = smbios_tables_len;                                        \
+        smbios_tables_len += sizeof(*t);                                  \
+        smbios_tables = g_realloc(smbios_tables, smbios_tables_len);      \
+        t = (struct smbios_type_##tbl_type *)(smbios_tables + t_off);     \
+                                                                          \
+        t->header.type = tbl_type;                                        \
+        t->header.length = sizeof(*t);                                    \
+        t->header.handle = cpu_to_le16(tbl_handle);                       \
+    } while (0)
+
+#define SMBIOS_TABLE_SET_STR(tbl_type, field, value)                      \
+    do {                                                                  \
+        int len = (value != NULL) ? strlen(value) + 1 : 0;                \
+        if (len > 1) {                                                    \
+            smbios_tables = g_realloc(smbios_tables,                      \
+                                      smbios_tables_len + len);           \
+            memcpy(smbios_tables + smbios_tables_len, value, len);        \
+            smbios_tables_len += len;                                     \
+            /* update pointer post-realloc */                             \
+            t = (struct smbios_type_##tbl_type *)(smbios_tables + t_off); \
+            t->field = ++str_index;                                       \
+        } else {                                                          \
+            t->field = 0;                                                 \
+        }                                                                 \
+    } while (0)
+
+#define SMBIOS_BUILD_TABLE_POST                                           \
+    do {                                                                  \
+        size_t term_cnt, t_size;                                          \
+                                                                          \
+        /* add '\0' terminator (add two if no strings defined) */         \
+        term_cnt = (str_index == 0) ? 2 : 1;                              \
+        smbios_tables = g_realloc(smbios_tables,                          \
+                                  smbios_tables_len + term_cnt);          \
+        memset(smbios_tables + smbios_tables_len, 0, term_cnt);           \
+        smbios_tables_len += term_cnt;                                    \
+                                                                          \
+        /* update smbios max. element size */                             \
+        t_size = smbios_tables_len - t_off;                               \
+        if (t_size > smbios_table_max) {                                  \
+            smbios_table_max = t_size;                                    \
+        }                                                                 \
+                                                                          \
+        /* update smbios element count */                                 \
+        smbios_table_cnt++;                                               \
+    } while (0)
+
+#endif /* QEMU_SMBIOS_BUILD_H */
diff --git a/hw/smbios/smbios_type_38.c b/hw/smbios/smbios_type_38.c
new file mode 100644
index 0000000..56e8609
--- /dev/null
+++ b/hw/smbios/smbios_type_38.c
@@ -0,0 +1,117 @@
+/*
+ * IPMI SMBIOS firmware handling
+ *
+ * Copyright (c) 2015,2016 Corey Minyard, MontaVista Software, LLC
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/ipmi/ipmi.h"
+#include "hw/smbios/ipmi.h"
+#include "hw/smbios/smbios.h"
+#include "qemu/error-report.h"
+#include "smbios_build.h"
+
+/* SMBIOS type 38 - IPMI */
+struct smbios_type_38 {
+    struct smbios_structure_header header;
+    uint8_t interface_type;
+    uint8_t ipmi_spec_revision;
+    uint8_t i2c_slave_address;
+    uint8_t nv_storage_device_address;
+    uint64_t base_address;
+    uint8_t base_address_modifier;
+    uint8_t interrupt_number;
+} QEMU_PACKED;
+
+static void smbios_build_one_type_38(IPMIFwInfo *info)
+{
+    uint64_t baseaddr = info->base_address;
+    SMBIOS_BUILD_TABLE_PRE(38, 0x3000, true);
+
+    t->interface_type = info->interface_type;
+    t->ipmi_spec_revision = ((info->ipmi_spec_major_revision << 4)
+                             | info->ipmi_spec_minor_revision);
+    t->i2c_slave_address = info->i2c_slave_address;
+    t->nv_storage_device_address = 0;
+
+    assert(info->ipmi_spec_minor_revision <= 15);
+    assert(info->ipmi_spec_major_revision <= 15);
+
+    /* or 1 to set it to I/O space */
+    switch (info->memspace) {
+    case IPMI_MEMSPACE_IO:
+        baseaddr |= 1;
+        break;
+    case IPMI_MEMSPACE_MEM32:
+    case IPMI_MEMSPACE_MEM64:
+        break;
+    case IPMI_MEMSPACE_SMBUS:
+        baseaddr <<= 1;
+        break;
+    }
+
+    t->base_address = cpu_to_le64(baseaddr);
+
+    t->base_address_modifier = 0;
+    if (info->irq_type == IPMI_LEVEL_IRQ) {
+        t->base_address_modifier |= 1;
+    }
+    switch (info->register_spacing) {
+    case 1:
+        break;
+    case 4:
+        t->base_address_modifier |= 1 << 6;
+        break;
+    case 16:
+        t->base_address_modifier |= 2 << 6;
+        break;
+    default:
+        error_report("IPMI register spacing %d is not compatible with"
+                     " SMBIOS, ignoring this entry.", info->register_spacing);
+        return;
+    }
+    t->interrupt_number = info->interrupt_number;
+
+    SMBIOS_BUILD_TABLE_POST;
+}
+
+static void smbios_add_ipmi_devices(BusState *bus)
+{
+    BusChild *kid;
+
+    QTAILQ_FOREACH(kid, &bus->children,  sibling) {
+        DeviceState *dev = kid->child;
+        Object *obj = object_dynamic_cast(OBJECT(dev), TYPE_IPMI_INTERFACE);
+        BusState *childbus;
+
+        if (obj) {
+            IPMIInterface *ii;
+            IPMIInterfaceClass *iic;
+            IPMIFwInfo info;
+
+            ii = IPMI_INTERFACE(obj);
+            iic = IPMI_INTERFACE_GET_CLASS(obj);
+            memset(&info, 0, sizeof(info));
+            iic->get_fwinfo(ii, &info);
+            smbios_build_one_type_38(&info);
+            continue;
+        }
+
+        QLIST_FOREACH(childbus, &dev->child_bus, sibling) {
+            smbios_add_ipmi_devices(childbus);
+        }
+    }
+}
+
+void smbios_build_type_38_table(void)
+{
+    BusState *bus;
+
+    bus = sysbus_get_default();
+    if (bus) {
+        smbios_add_ipmi_devices(bus);
+    }
+}
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index 751392e..6e16478 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
@@ -151,13 +155,12 @@
         exit(1);
     }
 
-    memory_region_init_ram(ram, NULL, "leon3.ram", ram_size, &error_abort);
-    vmstate_register_ram_global(ram);
+    memory_region_allocate_system_memory(ram, NULL, "leon3.ram", ram_size);
     memory_region_add_subregion(address_space_mem, 0x40000000, ram);
 
     /* Allocate BIOS */
     prom_size = 8 * 1024 * 1024; /* 8Mb */
-    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_abort);
+    memory_region_init_ram(prom, NULL, "Leon3.bios", prom_size, &error_fatal);
     vmstate_register_ram_global(prom);
     memory_region_set_readonly(prom, true);
     memory_region_add_subregion(address_space_mem, 0x00000000, prom);
@@ -168,7 +171,11 @@
     }
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
 
-    bios_size = get_image_size(filename);
+    if (filename) {
+        bios_size = get_image_size(filename);
+    } else {
+        bios_size = -1;
+    }
 
     if (bios_size > prom_size) {
         fprintf(stderr, "qemu: could not load prom '%s': file too big\n",
@@ -186,6 +193,7 @@
         fprintf(stderr, "Can't read bios image %s\n", filename);
         exit(1);
     }
+    g_free(filename);
 
     /* Can directly load an application. */
     if (kernel_filename != NULL) {
@@ -193,7 +201,7 @@
         uint64_t entry;
 
         kernel_size = load_elf(kernel_filename, NULL, NULL, &entry, NULL, NULL,
-                               1 /* big endian */, ELF_MACHINE, 0);
+                               1 /* big endian */, EM_SPARC, 0, 0);
         if (kernel_size < 0) {
             fprintf(stderr, "qemu: could not load kernel '%s'\n",
                     kernel_filename);
@@ -216,15 +224,10 @@
     }
 }
 
-static QEMUMachine leon3_generic_machine = {
-    .name     = "leon3_generic",
-    .desc     = "Leon-3 generic",
-    .init     = leon3_generic_hw_init,
-};
-
-static void leon3_machine_init(void)
+static void leon3_generic_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&leon3_generic_machine);
+    mc->desc = "Leon-3 generic";
+    mc->init = leon3_generic_hw_init;
 }
 
-machine_init(leon3_machine_init);
+DEFINE_MACHINE("leon3_generic", leon3_generic_machine_init)
diff --git a/hw/sparc/sun4m.c b/hw/sparc/sun4m.c
index 8273199..478fda8 100644
--- a/hw/sparc/sun4m.c
+++ b/hw/sparc/sun4m.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/sysbus.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
@@ -42,6 +46,7 @@
 #include "elf.h"
 #include "sysemu/block-backend.h"
 #include "trace.h"
+#include "qemu/cutils.h"
 
 /*
  * Sun4m architecture was used in the following machines:
@@ -95,39 +100,17 @@
     uint8_t nvram_machine_id;
 };
 
-int DMA_get_channel_mode (int nchan)
-{
-    return 0;
-}
-int DMA_read_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-int DMA_write_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-void DMA_hold_DREQ (int nchan) {}
-void DMA_release_DREQ (int nchan) {}
-void DMA_schedule(int nchan) {}
-
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+void DMA_init(ISABus *bus, int high_page_enable)
 {
 }
 
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque)
+static void fw_cfg_boot_set(void *opaque, const char *boot_device,
+                            Error **errp)
 {
+    fw_cfg_modify_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
 }
 
-static int fw_cfg_boot_set(void *opaque, const char *boot_device)
-{
-    fw_cfg_add_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
-    return 0;
-}
-
-static void nvram_init(M48t59State *nvram, uint8_t *macaddr,
+static void nvram_init(Nvram *nvram, uint8_t *macaddr,
                        const char *cmdline, const char *boot_devices,
                        ram_addr_t RAM_size, uint32_t kernel_size,
                        int width, int height, int depth,
@@ -137,6 +120,7 @@
     uint32_t start, end;
     uint8_t image[0x1ff0];
     struct OpenBIOS_nvpart_v1 *part_header;
+    NvramClass *k = NVRAM_GET_CLASS(nvram);
 
     memset(image, '\0', sizeof(image));
 
@@ -170,19 +154,20 @@
     Sun_init_header((struct Sun_nvram *)&image[0x1fd8], macaddr,
                     nvram_machine_id);
 
-    for (i = 0; i < sizeof(image); i++)
-        m48t59_write(nvram, i, image[i]);
+    for (i = 0; i < sizeof(image); i++) {
+        (k->write)(nvram, i, image[i]);
+    }
 }
 
 static DeviceState *slavio_intctl;
 
-void sun4m_pic_info(Monitor *mon, const QDict *qdict)
+void sun4m_hmp_info_pic(Monitor *mon, const QDict *qdict)
 {
     if (slavio_intctl)
         slavio_pic_info(mon, slavio_intctl);
 }
 
-void sun4m_irq_info(Monitor *mon, const QDict *qdict)
+void sun4m_hmp_info_irq(Monitor *mon, const QDict *qdict)
 {
     if (slavio_intctl)
         slavio_irq_info(mon, slavio_intctl);
@@ -298,7 +283,7 @@
         bswap_needed = 0;
 #endif
         kernel_size = load_elf(kernel_filename, translate_kernel_address, NULL,
-                               NULL, NULL, NULL, 1, ELF_MACHINE, 0);
+                               NULL, NULL, NULL, 1, EM_SPARC, 0, 0);
         if (kernel_size < 0)
             kernel_size = load_aout(kernel_filename, KERNEL_LOAD_ADDR,
                                     RAM_size - KERNEL_LOAD_ADDR, bswap_needed,
@@ -639,7 +624,7 @@
     IDRegState *s = MACIO_ID_REGISTER(dev);
 
     memory_region_init_ram(&s->mem, OBJECT(s),
-                           "sun4m.idreg", sizeof(idreg_data), &error_abort);
+                           "sun4m.idreg", sizeof(idreg_data), &error_fatal);
     vmstate_register_ram_global(&s->mem);
     memory_region_set_readonly(&s->mem, true);
     sysbus_init_mmio(dev, &s->mem);
@@ -686,7 +671,7 @@
 {
     AFXState *s = TCX_AFX(dev);
 
-    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_abort);
+    memory_region_init_ram(&s->mem, OBJECT(s), "sun4m.afx", 4, &error_fatal);
     vmstate_register_ram_global(&s->mem);
     sysbus_init_mmio(dev, &s->mem);
     return 0;
@@ -742,7 +727,7 @@
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
     if (filename) {
         ret = load_elf(filename, translate_prom_address, &addr, NULL,
-                       NULL, NULL, 1, ELF_MACHINE, 0);
+                       NULL, NULL, 1, EM_SPARC, 0, 0);
         if (ret < 0 || ret > PROM_SIZE_MAX) {
             ret = load_image_targphys(filename, addr, PROM_SIZE_MAX);
         }
@@ -761,7 +746,7 @@
     PROMState *s = OPENPROM(dev);
 
     memory_region_init_ram(&s->prom, OBJECT(s), "sun4m.prom", PROM_SIZE_MAX,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -803,9 +788,8 @@
 {
     RamDevice *d = SUN4M_RAM(dev);
 
-    memory_region_init_ram(&d->ram, OBJECT(d), "sun4m.ram", d->size,
-                           &error_abort);
-    vmstate_register_ram_global(&d->ram);
+    memory_region_allocate_system_memory(&d->ram, OBJECT(d), "sun4m.ram",
+                                         d->size);
     sysbus_init_mmio(dev, &d->ram);
     return 0;
 }
@@ -896,7 +880,6 @@
         espdma_irq, ledma_irq;
     qemu_irq esp_reset, dma_enable;
     qemu_irq fdc_tc;
-    qemu_irq *cpu_halt;
     unsigned long kernel_size;
     DriveInfo *fd[MAX_FD];
     FWCfgState *fw_cfg;
@@ -1012,20 +995,19 @@
 
     lance_init(&nd_table[0], hwdef->le_base, ledma, ledma_irq);
 
-    nvram = m48t59_init(slavio_irq[0], hwdef->nvram_base, 0, 0x2000, 8);
+    nvram = m48t59_init(slavio_irq[0], hwdef->nvram_base, 0, 0x2000, 1968, 8);
 
     slavio_timer_init_all(hwdef->counter_base, slavio_irq[19], slavio_cpu_irq, smp_cpus);
 
     slavio_serial_ms_kbd_init(hwdef->ms_kb_base, slavio_irq[14],
-                              display_type == DT_NOGRAPHIC, ESCC_CLOCK, 1);
+                              !machine->enable_graphics, ESCC_CLOCK, 1);
     /* Slavio TTYA (base+4, Linux ttyS0) is the first QEMU serial device
        Slavio TTYB (base+0, Linux ttyS1) is the second QEMU serial device */
     escc_init(hwdef->serial_base, slavio_irq[15], slavio_irq[15],
               serial_hds[0], serial_hds[1], ESCC_CLOCK, 1);
 
-    cpu_halt = qemu_allocate_irqs(cpu_halt_signal, NULL, 1);
     if (hwdef->apc_base) {
-        apc_init(hwdef->apc_base, cpu_halt[0]);
+        apc_init(hwdef->apc_base, qemu_allocate_irq(cpu_halt_signal, NULL, 0));
     }
 
     if (hwdef->fd_base) {
@@ -1035,7 +1017,7 @@
         sun4m_fdctrl_init(slavio_irq[22], hwdef->fd_base, fd,
                           &fdc_tc);
     } else {
-        fdc_tc = *qemu_allocate_irqs(dummy_fdc_tc, NULL, 1);
+        fdc_tc = qemu_allocate_irq(dummy_fdc_tc, NULL, 0);
     }
 
     slavio_misc_init(hwdef->slavio_base, hwdef->aux1_base, hwdef->aux2_base,
@@ -1084,9 +1066,8 @@
         ecc_init(hwdef->ecc_base, slavio_irq[28],
                  hwdef->ecc_version);
 
-    fw_cfg = fw_cfg_init(0, 0, CFG_ADDR, CFG_ADDR + 2);
+    fw_cfg = fw_cfg_init_mem(CFG_ADDR, CFG_ADDR + 2);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
-    fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
     fw_cfg_add_i16(fw_cfg, FW_CFG_SUN4M_DEPTH, graphic_depth);
@@ -1422,80 +1403,152 @@
     sun4m_hw_init(&sun4m_hwdefs[8], machine);
 }
 
-static QEMUMachine ss5_machine = {
-    .name = "SS-5",
-    .desc = "Sun4m platform, SPARCstation 5",
-    .init = ss5_init,
-    .block_default_type = IF_SCSI,
-    .is_default = 1,
-    .default_boot_order = "c",
+static void ss5_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 5";
+    mc->init = ss5_init;
+    mc->block_default_type = IF_SCSI;
+    mc->is_default = 1;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss5_type = {
+    .name = MACHINE_TYPE_NAME("SS-5"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss5_class_init,
 };
 
-static QEMUMachine ss10_machine = {
-    .name = "SS-10",
-    .desc = "Sun4m platform, SPARCstation 10",
-    .init = ss10_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss10_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 10";
+    mc->init = ss10_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss10_type = {
+    .name = MACHINE_TYPE_NAME("SS-10"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss10_class_init,
 };
 
-static QEMUMachine ss600mp_machine = {
-    .name = "SS-600MP",
-    .desc = "Sun4m platform, SPARCserver 600MP",
-    .init = ss600mp_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss600mp_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCserver 600MP";
+    mc->init = ss600mp_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss600mp_type = {
+    .name = MACHINE_TYPE_NAME("SS-600MP"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss600mp_class_init,
 };
 
-static QEMUMachine ss20_machine = {
-    .name = "SS-20",
-    .desc = "Sun4m platform, SPARCstation 20",
-    .init = ss20_init,
-    .block_default_type = IF_SCSI,
-    .max_cpus = 4,
-    .default_boot_order = "c",
+static void ss20_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 20";
+    mc->init = ss20_init;
+    mc->block_default_type = IF_SCSI;
+    mc->max_cpus = 4;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss20_type = {
+    .name = MACHINE_TYPE_NAME("SS-20"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss20_class_init,
 };
 
-static QEMUMachine voyager_machine = {
-    .name = "Voyager",
-    .desc = "Sun4m platform, SPARCstation Voyager",
-    .init = vger_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void voyager_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation Voyager";
+    mc->init = vger_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo voyager_type = {
+    .name = MACHINE_TYPE_NAME("Voyager"),
+    .parent = TYPE_MACHINE,
+    .class_init = voyager_class_init,
 };
 
-static QEMUMachine ss_lx_machine = {
-    .name = "LX",
-    .desc = "Sun4m platform, SPARCstation LX",
-    .init = ss_lx_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void ss_lx_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation LX";
+    mc->init = ss_lx_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss_lx_type = {
+    .name = MACHINE_TYPE_NAME("LX"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss_lx_class_init,
 };
 
-static QEMUMachine ss4_machine = {
-    .name = "SS-4",
-    .desc = "Sun4m platform, SPARCstation 4",
-    .init = ss4_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void ss4_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCstation 4";
+    mc->init = ss4_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo ss4_type = {
+    .name = MACHINE_TYPE_NAME("SS-4"),
+    .parent = TYPE_MACHINE,
+    .class_init = ss4_class_init,
 };
 
-static QEMUMachine scls_machine = {
-    .name = "SPARCClassic",
-    .desc = "Sun4m platform, SPARCClassic",
-    .init = scls_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void scls_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCClassic";
+    mc->init = scls_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo scls_type = {
+    .name = MACHINE_TYPE_NAME("SPARCClassic"),
+    .parent = TYPE_MACHINE,
+    .class_init = scls_class_init,
 };
 
-static QEMUMachine sbook_machine = {
-    .name = "SPARCbook",
-    .desc = "Sun4m platform, SPARCbook",
-    .init = sbook_init,
-    .block_default_type = IF_SCSI,
-    .default_boot_order = "c",
+static void sbook_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4m platform, SPARCbook";
+    mc->init = sbook_init;
+    mc->block_default_type = IF_SCSI;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sbook_type = {
+    .name = MACHINE_TYPE_NAME("SPARCbook"),
+    .parent = TYPE_MACHINE,
+    .class_init = sbook_class_init,
 };
 
 static void sun4m_register_types(void)
@@ -1504,20 +1557,16 @@
     type_register_static(&afx_info);
     type_register_static(&prom_info);
     type_register_static(&ram_info);
-}
 
-static void sun4m_machine_init(void)
-{
-    qemu_register_machine(&ss5_machine);
-    qemu_register_machine(&ss10_machine);
-    qemu_register_machine(&ss600mp_machine);
-    qemu_register_machine(&ss20_machine);
-    qemu_register_machine(&voyager_machine);
-    qemu_register_machine(&ss_lx_machine);
-    qemu_register_machine(&ss4_machine);
-    qemu_register_machine(&scls_machine);
-    qemu_register_machine(&sbook_machine);
+    type_register_static(&ss5_type);
+    type_register_static(&ss10_type);
+    type_register_static(&ss600mp_type);
+    type_register_static(&ss20_type);
+    type_register_static(&voyager_type);
+    type_register_static(&ss_lx_type);
+    type_register_static(&ss4_type);
+    type_register_static(&scls_type);
+    type_register_static(&sbook_type);
 }
 
 type_init(sun4m_register_types)
-machine_init(sun4m_machine_init);
diff --git a/hw/sparc/trace-events b/hw/sparc/trace-events
new file mode 100644
index 0000000..30fb037
--- /dev/null
+++ b/hw/sparc/trace-events
@@ -0,0 +1,11 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/sparc/sun4m.c
+sun4m_cpu_interrupt(unsigned int level) "Set CPU IRQ %d"
+sun4m_cpu_reset_interrupt(unsigned int level) "Reset CPU IRQ %d"
+sun4m_cpu_set_irq_raise(int level) "Raise CPU IRQ %d"
+sun4m_cpu_set_irq_lower(int level) "Lower CPU IRQ %d"
+
+# hw/sparc/leon3.c
+leon3_set_irq(int intno) "Set CPU IRQ %d"
+leon3_reset_irq(int intno) "Reset CPU IRQ %d"
diff --git a/hw/sparc64/sun4u.c b/hw/sparc64/sun4u.c
index f42112c..3165e18 100644
--- a/hw/sparc64/sun4u.c
+++ b/hw/sparc64/sun4u.c
@@ -21,6 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/pci/pci.h"
 #include "hw/pci-host/apb.h"
@@ -40,6 +44,7 @@
 #include "elf.h"
 #include "sysemu/block-backend.h"
 #include "exec/address-spaces.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_IRQ
 //#define DEBUG_EBUS
@@ -98,39 +103,17 @@
     MemoryRegion bar1;
 } EbusState;
 
-int DMA_get_channel_mode (int nchan)
-{
-    return 0;
-}
-int DMA_read_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-int DMA_write_memory (int nchan, void *buf, int pos, int size)
-{
-    return 0;
-}
-void DMA_hold_DREQ (int nchan) {}
-void DMA_release_DREQ (int nchan) {}
-void DMA_schedule(int nchan) {}
-
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit)
+void DMA_init(ISABus *bus, int high_page_enable)
 {
 }
 
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque)
+static void fw_cfg_boot_set(void *opaque, const char *boot_device,
+                            Error **errp)
 {
+    fw_cfg_modify_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
 }
 
-static int fw_cfg_boot_set(void *opaque, const char *boot_device)
-{
-    fw_cfg_add_i16(opaque, FW_CFG_BOOT_DEVICE, boot_device[0]);
-    return 0;
-}
-
-static int sun4u_NVRAM_set_params(M48t59State *nvram, uint16_t NVRAM_size,
+static int sun4u_NVRAM_set_params(Nvram *nvram, uint16_t NVRAM_size,
                                   const char *arch, ram_addr_t RAM_size,
                                   const char *boot_devices,
                                   uint32_t kernel_image, uint32_t kernel_size,
@@ -144,6 +127,7 @@
     uint32_t start, end;
     uint8_t image[0x1ff0];
     struct OpenBIOS_nvpart_v1 *part_header;
+    NvramClass *k = NVRAM_GET_CLASS(nvram);
 
     memset(image, '\0', sizeof(image));
 
@@ -176,8 +160,9 @@
 
     Sun_init_header((struct Sun_nvram *)&image[0x1fd8], macaddr, 0x80);
 
-    for (i = 0; i < sizeof(image); i++)
-        m48t59_write(nvram, i, image[i]);
+    for (i = 0; i < sizeof(image); i++) {
+        (k->write)(nvram, i, image[i]);
+    }
 
     return 0;
 }
@@ -206,7 +191,7 @@
         bswap_needed = 0;
 #endif
         kernel_size = load_elf(kernel_filename, NULL, NULL, kernel_entry,
-                               kernel_addr, &kernel_top, 1, ELF_MACHINE, 0);
+                               kernel_addr, &kernel_top, 1, EM_SPARCV9, 0, 0);
         if (kernel_size < 0) {
             *kernel_addr = KERNEL_LOAD_ADDR;
             *kernel_entry = KERNEL_LOAD_ADDR;
@@ -356,37 +341,19 @@
     uint64_t prom_addr;
 } ResetData;
 
-void cpu_put_timer(QEMUFile *f, CPUTimer *s)
-{
-    qemu_put_be32s(f, &s->frequency);
-    qemu_put_be32s(f, &s->disabled);
-    qemu_put_be64s(f, &s->disabled_mask);
-    qemu_put_sbe64s(f, &s->clock_offset);
-
-    timer_put(f, s->qtimer);
-}
-
-void cpu_get_timer(QEMUFile *f, CPUTimer *s)
-{
-    qemu_get_be32s(f, &s->frequency);
-    qemu_get_be32s(f, &s->disabled);
-    qemu_get_be64s(f, &s->disabled_mask);
-    qemu_get_sbe64s(f, &s->clock_offset);
-
-    timer_get(f, s->qtimer);
-}
-
 static CPUTimer *cpu_timer_create(const char *name, SPARCCPU *cpu,
                                   QEMUBHFunc *cb, uint32_t frequency,
-                                  uint64_t disabled_mask)
+                                  uint64_t disabled_mask, uint64_t npt_mask)
 {
     CPUTimer *timer = g_malloc0(sizeof (CPUTimer));
 
     timer->name = name;
     timer->frequency = frequency;
     timer->disabled_mask = disabled_mask;
+    timer->npt_mask = npt_mask;
 
     timer->disabled = 1;
+    timer->npt = 1;
     timer->clock_offset = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
 
     timer->qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL, cb, cpu);
@@ -482,27 +449,27 @@
 
 static int64_t cpu_to_timer_ticks(int64_t cpu_ticks, uint32_t frequency)
 {
-    return muldiv64(cpu_ticks, get_ticks_per_sec(), frequency);
+    return muldiv64(cpu_ticks, NANOSECONDS_PER_SECOND, frequency);
 }
 
 static uint64_t timer_to_cpu_ticks(int64_t timer_ticks, uint32_t frequency)
 {
-    return muldiv64(timer_ticks, frequency, get_ticks_per_sec());
+    return muldiv64(timer_ticks, frequency, NANOSECONDS_PER_SECOND);
 }
 
 void cpu_tick_set_count(CPUTimer *timer, uint64_t count)
 {
-    uint64_t real_count = count & ~timer->disabled_mask;
-    uint64_t disabled_bit = count & timer->disabled_mask;
+    uint64_t real_count = count & ~timer->npt_mask;
+    uint64_t npt_bit = count & timer->npt_mask;
 
     int64_t vm_clock_offset = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
                     cpu_to_timer_ticks(real_count, timer->frequency);
 
-    TIMER_DPRINTF("%s set_count count=0x%016lx (%s) p=%p\n",
+    TIMER_DPRINTF("%s set_count count=0x%016lx (npt %s) p=%p\n",
                   timer->name, real_count,
-                  timer->disabled?"disabled":"enabled", timer);
+                  timer->npt ? "disabled" : "enabled", timer);
 
-    timer->disabled = disabled_bit ? 1 : 0;
+    timer->npt = npt_bit ? 1 : 0;
     timer->clock_offset = vm_clock_offset;
 }
 
@@ -512,12 +479,13 @@
                     qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - timer->clock_offset,
                     timer->frequency);
 
-    TIMER_DPRINTF("%s get_count count=0x%016lx (%s) p=%p\n",
+    TIMER_DPRINTF("%s get_count count=0x%016lx (npt %s) p=%p\n",
            timer->name, real_count,
-           timer->disabled?"disabled":"enabled", timer);
+           timer->npt ? "disabled" : "enabled", timer);
 
-    if (timer->disabled)
-        real_count |= timer->disabled_mask;
+    if (timer->npt) {
+        real_count |= timer->npt_mask;
+    }
 
     return real_count;
 }
@@ -591,12 +559,14 @@
     return isa_bus;
 }
 
-static int
-pci_ebus_init1(PCIDevice *pci_dev)
+static void pci_ebus_realize(PCIDevice *pci_dev, Error **errp)
 {
     EbusState *s = DO_UPCAST(EbusState, pci_dev, pci_dev);
 
-    isa_bus_new(&pci_dev->qdev, pci_address_space_io(pci_dev));
+    if (!isa_bus_new(DEVICE(pci_dev), get_system_memory(),
+                     pci_address_space_io(pci_dev), errp)) {
+        return;
+    }
 
     pci_dev->config[0x04] = 0x06; // command = bus master, pci mem
     pci_dev->config[0x05] = 0x00;
@@ -609,16 +579,15 @@
                              0, 0x1000000);
     pci_register_bar(pci_dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar0);
     memory_region_init_alias(&s->bar1, OBJECT(s), "bar1", get_system_io(),
-                             0, 0x1000);
+                             0, 0x4000);
     pci_register_bar(pci_dev, 1, PCI_BASE_ADDRESS_SPACE_IO, &s->bar1);
-    return 0;
 }
 
 static void ebus_class_init(ObjectClass *klass, void *data)
 {
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = pci_ebus_init1;
+    k->realize = pci_ebus_realize;
     k->vendor_id = PCI_VENDOR_ID_SUN;
     k->device_id = PCI_DEVICE_ID_SUN_EBUS;
     k->revision = 0x01;
@@ -668,7 +637,7 @@
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, bios_name);
     if (filename) {
         ret = load_elf(filename, translate_prom_address, &addr,
-                       NULL, NULL, NULL, 1, ELF_MACHINE, 0);
+                       NULL, NULL, NULL, 1, EM_SPARCV9, 0, 0);
         if (ret < 0 || ret > PROM_SIZE_MAX) {
             ret = load_image_targphys(filename, addr, PROM_SIZE_MAX);
         }
@@ -687,7 +656,7 @@
     PROMState *s = OPENPROM(dev);
 
     memory_region_init_ram(&s->prom, OBJECT(s), "sun4u.prom", PROM_SIZE_MAX,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&s->prom);
     memory_region_set_readonly(&s->prom, true);
     sysbus_init_mmio(dev, &s->prom);
@@ -731,7 +700,7 @@
     RamDevice *d = SUN4U_RAM(dev);
 
     memory_region_init_ram(&d->ram, OBJECT(d), "sun4u.ram", d->size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(&d->ram);
     sysbus_init_mmio(dev, &d->ram);
     return 0;
@@ -796,13 +765,16 @@
     env = &cpu->env;
 
     env->tick = cpu_timer_create("tick", cpu, tick_irq,
-                                  tick_frequency, TICK_NPT_MASK);
+                                  tick_frequency, TICK_INT_DIS,
+                                  TICK_NPT_MASK);
 
     env->stick = cpu_timer_create("stick", cpu, stick_irq,
-                                   stick_frequency, TICK_INT_DIS);
+                                   stick_frequency, TICK_INT_DIS,
+                                   TICK_NPT_MASK);
 
     env->hstick = cpu_timer_create("hstick", cpu, hstick_irq,
-                                    hstick_frequency, TICK_INT_DIS);
+                                    hstick_frequency, TICK_INT_DIS,
+                                    TICK_NPT_MASK);
 
     reset_info = g_malloc0(sizeof(ResetData));
     reset_info->cpu = cpu;
@@ -817,14 +789,16 @@
                         const struct hwdef *hwdef)
 {
     SPARCCPU *cpu;
-    M48t59State *nvram;
+    Nvram *nvram;
     unsigned int i;
     uint64_t initrd_addr, initrd_size, kernel_addr, kernel_size, kernel_entry;
     PCIBus *pci_bus, *pci_bus2, *pci_bus3;
     ISABus *isa_bus;
+    SysBusDevice *s;
     qemu_irq *ivec_irqs, *pbm_irqs;
     DriveInfo *hd[MAX_IDE_BUS * MAX_IDE_DEVS];
     DriveInfo *fd[MAX_FD];
+    DeviceState *dev;
     FWCfgState *fw_cfg;
 
     /* init CPUs */
@@ -849,17 +823,9 @@
                        NULL, 115200, serial_hds[i], DEVICE_BIG_ENDIAN);
         i++;
     }
-    for(; i < MAX_SERIAL_PORTS; i++) {
-        if (serial_hds[i]) {
-            serial_isa_init(isa_bus, i, serial_hds[i]);
-        }
-    }
 
-    for(i = 0; i < MAX_PARALLEL_PORTS; i++) {
-        if (parallel_hds[i]) {
-            parallel_init(isa_bus, i, parallel_hds[i]);
-        }
-    }
+    serial_hds_isa_init(isa_bus, MAX_SERIAL_PORTS);
+    parallel_hds_isa_init(isa_bus, MAX_PARALLEL_PORTS);
 
     for(i = 0; i < nb_nics; i++)
         pci_nic_init_nofail(&nd_table[i], pci_bus, "ne2k_pci", NULL);
@@ -869,12 +835,29 @@
     pci_cmd646_ide_init(pci_bus, hd, 1);
 
     isa_create_simple(isa_bus, "i8042");
+
+    /* Floppy */
     for(i = 0; i < MAX_FD; i++) {
         fd[i] = drive_get(IF_FLOPPY, 0, i);
     }
-    fdctrl_init_isa(isa_bus, fd);
-    nvram = m48t59_init_isa(isa_bus, 0x0074, NVRAM_SIZE, 59);
+    dev = DEVICE(isa_create(isa_bus, TYPE_ISA_FDC));
+    if (fd[0]) {
+        qdev_prop_set_drive(dev, "driveA", blk_by_legacy_dinfo(fd[0]),
+                            &error_abort);
+    }
+    if (fd[1]) {
+        qdev_prop_set_drive(dev, "driveB", blk_by_legacy_dinfo(fd[1]),
+                            &error_abort);
+    }
+    qdev_prop_set_uint32(dev, "dma", -1);
+    qdev_init_nofail(dev);
 
+    /* Map NVRAM into I/O (ebus) space */
+    nvram = m48t59_init(NULL, 0, 0, NVRAM_SIZE, 1968, 59);
+    s = SYS_BUS_DEVICE(nvram);
+    memory_region_add_subregion(get_system_io(), 0x2000,
+                                sysbus_mmio_get_region(s, 0));
+ 
     initrd_size = 0;
     initrd_addr = 0;
     kernel_size = sun4u_load_kernel(machine->kernel_filename,
@@ -892,9 +875,8 @@
                            graphic_width, graphic_height, graphic_depth,
                            (uint8_t *)&nd_table[0].macaddr);
 
-    fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
+    fw_cfg = fw_cfg_init_io(BIOS_CFG_IOPORT);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
-    fw_cfg_add_i32(fw_cfg, FW_CFG_ID, 1);
     fw_cfg_add_i64(fw_cfg, FW_CFG_RAM_SIZE, (uint64_t)ram_size);
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
     fw_cfg_add_i64(fw_cfg, FW_CFG_KERNEL_ADDR, kernel_entry);
@@ -965,29 +947,53 @@
     sun4uv_init(get_system_memory(), machine, &hwdefs[2]);
 }
 
-static QEMUMachine sun4u_machine = {
-    .name = "sun4u",
-    .desc = "Sun4u platform",
-    .init = sun4u_init,
-    .max_cpus = 1, // XXX for now
-    .is_default = 1,
-    .default_boot_order = "c",
+static void sun4u_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4u platform";
+    mc->init = sun4u_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->is_default = 1;
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sun4u_type = {
+    .name = MACHINE_TYPE_NAME("sun4u"),
+    .parent = TYPE_MACHINE,
+    .class_init = sun4u_class_init,
 };
 
-static QEMUMachine sun4v_machine = {
-    .name = "sun4v",
-    .desc = "Sun4v platform",
-    .init = sun4v_init,
-    .max_cpus = 1, // XXX for now
-    .default_boot_order = "c",
+static void sun4v_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4v platform";
+    mc->init = sun4v_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo sun4v_type = {
+    .name = MACHINE_TYPE_NAME("sun4v"),
+    .parent = TYPE_MACHINE,
+    .class_init = sun4v_class_init,
 };
 
-static QEMUMachine niagara_machine = {
-    .name = "Niagara",
-    .desc = "Sun4v platform, Niagara",
-    .init = niagara_init,
-    .max_cpus = 1, // XXX for now
-    .default_boot_order = "c",
+static void niagara_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "Sun4v platform, Niagara";
+    mc->init = niagara_init;
+    mc->max_cpus = 1; /* XXX for now */
+    mc->default_boot_order = "c";
+}
+
+static const TypeInfo niagara_type = {
+    .name = MACHINE_TYPE_NAME("Niagara"),
+    .parent = TYPE_MACHINE,
+    .class_init = niagara_class_init,
 };
 
 static void sun4u_register_types(void)
@@ -995,14 +1001,10 @@
     type_register_static(&ebus_info);
     type_register_static(&prom_info);
     type_register_static(&ram_info);
-}
 
-static void sun4u_machine_init(void)
-{
-    qemu_register_machine(&sun4u_machine);
-    qemu_register_machine(&sun4v_machine);
-    qemu_register_machine(&niagara_machine);
+    type_register_static(&sun4u_type);
+    type_register_static(&sun4v_type);
+    type_register_static(&niagara_type);
 }
 
 type_init(sun4u_register_types)
-machine_init(sun4u_machine_init);
diff --git a/hw/ssi/Makefile.objs b/hw/ssi/Makefile.objs
index 9555825..c79a8dc 100644
--- a/hw/ssi/Makefile.objs
+++ b/hw/ssi/Makefile.objs
@@ -2,5 +2,7 @@
 common-obj-$(CONFIG_SSI) += ssi.o
 common-obj-$(CONFIG_XILINX_SPI) += xilinx_spi.o
 common-obj-$(CONFIG_XILINX_SPIPS) += xilinx_spips.o
+common-obj-$(CONFIG_ASPEED_SOC) += aspeed_smc.o
 
 obj-$(CONFIG_OMAP) += omap_spi.o
+obj-$(CONFIG_IMX) += imx_spi.o
diff --git a/hw/ssi/aspeed_smc.c b/hw/ssi/aspeed_smc.c
new file mode 100644
index 0000000..d319e04
--- /dev/null
+++ b/hw/ssi/aspeed_smc.c
@@ -0,0 +1,469 @@
+/*
+ * ASPEED AST2400 SMC Controller (SPI Flash Only)
+ *
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
+#include "include/qemu/error-report.h"
+#include "exec/address-spaces.h"
+
+#include "hw/ssi/aspeed_smc.h"
+
+/* CE Type Setting Register */
+#define R_CONF            (0x00 / 4)
+#define   CONF_LEGACY_DISABLE  (1 << 31)
+#define   CONF_ENABLE_W4       20
+#define   CONF_ENABLE_W3       19
+#define   CONF_ENABLE_W2       18
+#define   CONF_ENABLE_W1       17
+#define   CONF_ENABLE_W0       16
+#define   CONF_FLASH_TYPE4     9
+#define   CONF_FLASH_TYPE3     7
+#define   CONF_FLASH_TYPE2     5
+#define   CONF_FLASH_TYPE1     3
+#define   CONF_FLASH_TYPE0     1
+
+/* CE Control Register */
+#define R_CE_CTRL            (0x04 / 4)
+#define   CTRL_EXTENDED4       4  /* 32 bit addressing for SPI */
+#define   CTRL_EXTENDED3       3  /* 32 bit addressing for SPI */
+#define   CTRL_EXTENDED2       2  /* 32 bit addressing for SPI */
+#define   CTRL_EXTENDED1       1  /* 32 bit addressing for SPI */
+#define   CTRL_EXTENDED0       0  /* 32 bit addressing for SPI */
+
+/* Interrupt Control and Status Register */
+#define R_INTR_CTRL       (0x08 / 4)
+#define   INTR_CTRL_DMA_STATUS            (1 << 11)
+#define   INTR_CTRL_CMD_ABORT_STATUS      (1 << 10)
+#define   INTR_CTRL_WRITE_PROTECT_STATUS  (1 << 9)
+#define   INTR_CTRL_DMA_EN                (1 << 3)
+#define   INTR_CTRL_CMD_ABORT_EN          (1 << 2)
+#define   INTR_CTRL_WRITE_PROTECT_EN      (1 << 1)
+
+/* CEx Control Register */
+#define R_CTRL0           (0x10 / 4)
+#define   CTRL_CMD_SHIFT           16
+#define   CTRL_CMD_MASK            0xff
+#define   CTRL_CE_STOP_ACTIVE      (1 << 2)
+#define   CTRL_CMD_MODE_MASK       0x3
+#define     CTRL_READMODE          0x0
+#define     CTRL_FREADMODE         0x1
+#define     CTRL_WRITEMODE         0x2
+#define     CTRL_USERMODE          0x3
+#define R_CTRL1           (0x14 / 4)
+#define R_CTRL2           (0x18 / 4)
+#define R_CTRL3           (0x1C / 4)
+#define R_CTRL4           (0x20 / 4)
+
+/* CEx Segment Address Register */
+#define R_SEG_ADDR0       (0x30 / 4)
+#define   SEG_SIZE_SHIFT       24   /* 8MB units */
+#define   SEG_SIZE_MASK        0x7f
+#define   SEG_START_SHIFT      16   /* address bit [A29-A23] */
+#define   SEG_START_MASK       0x7f
+#define R_SEG_ADDR1       (0x34 / 4)
+#define R_SEG_ADDR2       (0x38 / 4)
+#define R_SEG_ADDR3       (0x3C / 4)
+#define R_SEG_ADDR4       (0x40 / 4)
+
+/* Misc Control Register #1 */
+#define R_MISC_CTRL1      (0x50 / 4)
+
+/* Misc Control Register #2 */
+#define R_MISC_CTRL2      (0x54 / 4)
+
+/* DMA Control/Status Register */
+#define R_DMA_CTRL        (0x80 / 4)
+#define   DMA_CTRL_DELAY_MASK   0xf
+#define   DMA_CTRL_DELAY_SHIFT  8
+#define   DMA_CTRL_FREQ_MASK    0xf
+#define   DMA_CTRL_FREQ_SHIFT   4
+#define   DMA_CTRL_MODE         (1 << 3)
+#define   DMA_CTRL_CKSUM        (1 << 2)
+#define   DMA_CTRL_DIR          (1 << 1)
+#define   DMA_CTRL_EN           (1 << 0)
+
+/* DMA Flash Side Address */
+#define R_DMA_FLASH_ADDR  (0x84 / 4)
+
+/* DMA DRAM Side Address */
+#define R_DMA_DRAM_ADDR   (0x88 / 4)
+
+/* DMA Length Register */
+#define R_DMA_LEN         (0x8C / 4)
+
+/* Checksum Calculation Result */
+#define R_DMA_CHECKSUM    (0x90 / 4)
+
+/* Misc Control Register #2 */
+#define R_TIMINGS         (0x94 / 4)
+
+/* SPI controller registers and bits */
+#define R_SPI_CONF        (0x00 / 4)
+#define   SPI_CONF_ENABLE_W0   0
+#define R_SPI_CTRL0       (0x4 / 4)
+#define R_SPI_MISC_CTRL   (0x10 / 4)
+#define R_SPI_TIMINGS     (0x14 / 4)
+
+/*
+ * Default segments mapping addresses and size for each slave per
+ * controller. These can be changed when board is initialized with the
+ * Segment Address Registers but they don't seem do be used on the
+ * field.
+ */
+static const AspeedSegments aspeed_segments_legacy[] = {
+    { 0x10000000, 32 * 1024 * 1024 },
+};
+
+static const AspeedSegments aspeed_segments_fmc[] = {
+    { 0x20000000, 64 * 1024 * 1024 },
+    { 0x24000000, 32 * 1024 * 1024 },
+    { 0x26000000, 32 * 1024 * 1024 },
+    { 0x28000000, 32 * 1024 * 1024 },
+    { 0x2A000000, 32 * 1024 * 1024 }
+};
+
+static const AspeedSegments aspeed_segments_spi[] = {
+    { 0x30000000, 64 * 1024 * 1024 },
+};
+
+static const AspeedSMCController controllers[] = {
+    { "aspeed.smc.smc", R_CONF, R_CE_CTRL, R_CTRL0, R_TIMINGS,
+      CONF_ENABLE_W0, 5, aspeed_segments_legacy, 0x6000000 },
+    { "aspeed.smc.fmc", R_CONF, R_CE_CTRL, R_CTRL0, R_TIMINGS,
+      CONF_ENABLE_W0, 5, aspeed_segments_fmc, 0x10000000 },
+    { "aspeed.smc.spi", R_SPI_CONF, 0xff, R_SPI_CTRL0, R_SPI_TIMINGS,
+      SPI_CONF_ENABLE_W0, 1, aspeed_segments_spi, 0x10000000 },
+};
+
+static uint64_t aspeed_smc_flash_default_read(void *opaque, hwaddr addr,
+                                              unsigned size)
+{
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: To 0x%" HWADDR_PRIx " of size %u"
+                  PRIx64 "\n", __func__, addr, size);
+    return 0;
+}
+
+static void aspeed_smc_flash_default_write(void *opaque, hwaddr addr,
+                                           uint64_t data, unsigned size)
+{
+   qemu_log_mask(LOG_GUEST_ERROR, "%s: To 0x%" HWADDR_PRIx " of size %u: 0x%"
+                 PRIx64 "\n", __func__, addr, size, data);
+}
+
+static const MemoryRegionOps aspeed_smc_flash_default_ops = {
+    .read = aspeed_smc_flash_default_read,
+    .write = aspeed_smc_flash_default_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+};
+
+static inline int aspeed_smc_flash_mode(const AspeedSMCState *s, int cs)
+{
+    return s->regs[s->r_ctrl0 + cs] & CTRL_CMD_MODE_MASK;
+}
+
+static inline bool aspeed_smc_is_usermode(const AspeedSMCState *s, int cs)
+{
+    return aspeed_smc_flash_mode(s, cs) == CTRL_USERMODE;
+}
+
+static inline bool aspeed_smc_is_writable(const AspeedSMCState *s, int cs)
+{
+    return s->regs[s->r_conf] & (1 << (s->conf_enable_w0 + cs));
+}
+
+static uint64_t aspeed_smc_flash_read(void *opaque, hwaddr addr, unsigned size)
+{
+    AspeedSMCFlash *fl = opaque;
+    const AspeedSMCState *s = fl->controller;
+    uint64_t ret = 0;
+    int i;
+
+    if (aspeed_smc_is_usermode(s, fl->id)) {
+        for (i = 0; i < size; i++) {
+            ret |= ssi_transfer(s->spi, 0x0) << (8 * i);
+        }
+    } else {
+        qemu_log_mask(LOG_UNIMP, "%s: usermode not implemented\n",
+                      __func__);
+        ret = -1;
+    }
+
+    return ret;
+}
+
+static void aspeed_smc_flash_write(void *opaque, hwaddr addr, uint64_t data,
+                           unsigned size)
+{
+    AspeedSMCFlash *fl = opaque;
+    const AspeedSMCState *s = fl->controller;
+    int i;
+
+    if (!aspeed_smc_is_writable(s, fl->id)) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: flash is not writable at 0x%"
+                      HWADDR_PRIx "\n", __func__, addr);
+        return;
+    }
+
+    if (!aspeed_smc_is_usermode(s, fl->id)) {
+        qemu_log_mask(LOG_UNIMP, "%s: usermode not implemented\n",
+                      __func__);
+        return;
+    }
+
+    for (i = 0; i < size; i++) {
+        ssi_transfer(s->spi, (data >> (8 * i)) & 0xff);
+    }
+}
+
+static const MemoryRegionOps aspeed_smc_flash_ops = {
+    .read = aspeed_smc_flash_read,
+    .write = aspeed_smc_flash_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+};
+
+static bool aspeed_smc_is_ce_stop_active(const AspeedSMCState *s, int cs)
+{
+    return s->regs[s->r_ctrl0 + cs] & CTRL_CE_STOP_ACTIVE;
+}
+
+static void aspeed_smc_update_cs(const AspeedSMCState *s)
+{
+    int i;
+
+    for (i = 0; i < s->num_cs; ++i) {
+        qemu_set_irq(s->cs_lines[i], aspeed_smc_is_ce_stop_active(s, i));
+    }
+}
+
+static void aspeed_smc_reset(DeviceState *d)
+{
+    AspeedSMCState *s = ASPEED_SMC(d);
+    int i;
+
+    memset(s->regs, 0, sizeof s->regs);
+
+    /* Pretend DMA is done (u-boot initialization) */
+    s->regs[R_INTR_CTRL] = INTR_CTRL_DMA_STATUS;
+
+    /* Unselect all slaves */
+    for (i = 0; i < s->num_cs; ++i) {
+        s->regs[s->r_ctrl0 + i] |= CTRL_CE_STOP_ACTIVE;
+    }
+
+    aspeed_smc_update_cs(s);
+}
+
+static uint64_t aspeed_smc_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    AspeedSMCState *s = ASPEED_SMC(opaque);
+
+    addr >>= 2;
+
+    if (addr >= ARRAY_SIZE(s->regs)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Out-of-bounds read at 0x%" HWADDR_PRIx "\n",
+                      __func__, addr);
+        return 0;
+    }
+
+    if (addr == s->r_conf ||
+        addr == s->r_timings ||
+        addr == s->r_ce_ctrl ||
+        addr == R_INTR_CTRL ||
+        (addr >= s->r_ctrl0 && addr < s->r_ctrl0 + s->num_cs)) {
+        return s->regs[addr];
+    } else {
+        qemu_log_mask(LOG_UNIMP, "%s: not implemented: 0x%" HWADDR_PRIx "\n",
+                      __func__, addr);
+        return 0;
+    }
+}
+
+static void aspeed_smc_write(void *opaque, hwaddr addr, uint64_t data,
+                             unsigned int size)
+{
+    AspeedSMCState *s = ASPEED_SMC(opaque);
+    uint32_t value = data;
+
+    addr >>= 2;
+
+    if (addr >= ARRAY_SIZE(s->regs)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Out-of-bounds write at 0x%" HWADDR_PRIx "\n",
+                      __func__, addr);
+        return;
+    }
+
+    if (addr == s->r_conf ||
+        addr == s->r_timings ||
+        addr == s->r_ce_ctrl) {
+        s->regs[addr] = value;
+    } else if (addr >= s->r_ctrl0 && addr < s->r_ctrl0 + s->num_cs) {
+        s->regs[addr] = value;
+        aspeed_smc_update_cs(s);
+    } else {
+        qemu_log_mask(LOG_UNIMP, "%s: not implemented: 0x%" HWADDR_PRIx "\n",
+                      __func__, addr);
+        return;
+    }
+}
+
+static const MemoryRegionOps aspeed_smc_ops = {
+    .read = aspeed_smc_read,
+    .write = aspeed_smc_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid.unaligned = true,
+};
+
+static void aspeed_smc_realize(DeviceState *dev, Error **errp)
+{
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedSMCState *s = ASPEED_SMC(dev);
+    AspeedSMCClass *mc = ASPEED_SMC_GET_CLASS(s);
+    int i;
+    char name[32];
+    hwaddr offset = 0;
+
+    s->ctrl = mc->ctrl;
+
+    /* keep a copy under AspeedSMCState to speed up accesses */
+    s->r_conf = s->ctrl->r_conf;
+    s->r_ce_ctrl = s->ctrl->r_ce_ctrl;
+    s->r_ctrl0 = s->ctrl->r_ctrl0;
+    s->r_timings = s->ctrl->r_timings;
+    s->conf_enable_w0 = s->ctrl->conf_enable_w0;
+
+    /* Enforce some real HW limits */
+    if (s->num_cs > s->ctrl->max_slaves) {
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: num_cs cannot exceed: %d\n",
+                      __func__, s->ctrl->max_slaves);
+        s->num_cs = s->ctrl->max_slaves;
+    }
+
+    s->spi = ssi_create_bus(dev, "spi");
+
+    /* Setup cs_lines for slaves */
+    sysbus_init_irq(sbd, &s->irq);
+    s->cs_lines = g_new0(qemu_irq, s->num_cs);
+    ssi_auto_connect_slaves(dev, s->cs_lines, s->spi);
+
+    for (i = 0; i < s->num_cs; ++i) {
+        sysbus_init_irq(sbd, &s->cs_lines[i]);
+    }
+
+    aspeed_smc_reset(dev);
+
+    memory_region_init_io(&s->mmio, OBJECT(s), &aspeed_smc_ops, s,
+                          s->ctrl->name, ASPEED_SMC_R_MAX * 4);
+    sysbus_init_mmio(sbd, &s->mmio);
+
+    /*
+     * Memory region where flash modules are remapped
+     */
+    snprintf(name, sizeof(name), "%s.flash", s->ctrl->name);
+
+    memory_region_init_io(&s->mmio_flash, OBJECT(s),
+                          &aspeed_smc_flash_default_ops, s, name,
+                          s->ctrl->mapping_window_size);
+    sysbus_init_mmio(sbd, &s->mmio_flash);
+
+    s->flashes = g_new0(AspeedSMCFlash, s->num_cs);
+
+    for (i = 0; i < s->num_cs; ++i) {
+        AspeedSMCFlash *fl = &s->flashes[i];
+
+        snprintf(name, sizeof(name), "%s.%d", s->ctrl->name, i);
+
+        fl->id = i;
+        fl->controller = s;
+        fl->size = s->ctrl->segments[i].size;
+        memory_region_init_io(&fl->mmio, OBJECT(s), &aspeed_smc_flash_ops,
+                              fl, name, fl->size);
+        memory_region_add_subregion(&s->mmio_flash, offset, &fl->mmio);
+        offset += fl->size;
+    }
+}
+
+static const VMStateDescription vmstate_aspeed_smc = {
+    .name = "aspeed.smc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, AspeedSMCState, ASPEED_SMC_R_MAX),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property aspeed_smc_properties[] = {
+    DEFINE_PROP_UINT32("num-cs", AspeedSMCState, num_cs, 1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void aspeed_smc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    AspeedSMCClass *mc = ASPEED_SMC_CLASS(klass);
+
+    dc->realize = aspeed_smc_realize;
+    dc->reset = aspeed_smc_reset;
+    dc->props = aspeed_smc_properties;
+    dc->vmsd = &vmstate_aspeed_smc;
+    mc->ctrl = data;
+}
+
+static const TypeInfo aspeed_smc_info = {
+    .name           = TYPE_ASPEED_SMC,
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(AspeedSMCState),
+    .class_size     = sizeof(AspeedSMCClass),
+    .abstract       = true,
+};
+
+static void aspeed_smc_register_types(void)
+{
+    int i;
+
+    type_register_static(&aspeed_smc_info);
+    for (i = 0; i < ARRAY_SIZE(controllers); ++i) {
+        TypeInfo ti = {
+            .name       = controllers[i].name,
+            .parent     = TYPE_ASPEED_SMC,
+            .class_init = aspeed_smc_class_init,
+            .class_data = (void *)&controllers[i],
+        };
+        type_register(&ti);
+    }
+}
+
+type_init(aspeed_smc_register_types)
diff --git a/hw/ssi/imx_spi.c b/hw/ssi/imx_spi.c
new file mode 100644
index 0000000..4226199
--- /dev/null
+++ b/hw/ssi/imx_spi.c
@@ -0,0 +1,455 @@
+/*
+ * IMX SPI Controller
+ *
+ * Copyright (c) 2016 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/ssi/imx_spi.h"
+#include "sysemu/sysemu.h"
+#include "qemu/log.h"
+
+#ifndef DEBUG_IMX_SPI
+#define DEBUG_IMX_SPI 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_SPI) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_SPI, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
+
+static char const *imx_spi_reg_name(uint32_t reg)
+{
+    static char unknown[20];
+
+    switch (reg) {
+    case ECSPI_RXDATA:
+        return  "ECSPI_RXDATA";
+    case ECSPI_TXDATA:
+        return  "ECSPI_TXDATA";
+    case ECSPI_CONREG:
+        return  "ECSPI_CONREG";
+    case ECSPI_CONFIGREG:
+        return  "ECSPI_CONFIGREG";
+    case ECSPI_INTREG:
+        return  "ECSPI_INTREG";
+    case ECSPI_DMAREG:
+        return  "ECSPI_DMAREG";
+    case ECSPI_STATREG:
+        return  "ECSPI_STATREG";
+    case ECSPI_PERIODREG:
+        return  "ECSPI_PERIODREG";
+    case ECSPI_TESTREG:
+        return  "ECSPI_TESTREG";
+    case ECSPI_MSGDATA:
+        return  "ECSPI_MSGDATA";
+    default:
+        sprintf(unknown, "%d ?", reg);
+        return unknown;
+    }
+}
+
+static const VMStateDescription vmstate_imx_spi = {
+    .name = TYPE_IMX_SPI,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_FIFO32(tx_fifo, IMXSPIState),
+        VMSTATE_FIFO32(rx_fifo, IMXSPIState),
+        VMSTATE_INT16(burst_length, IMXSPIState),
+        VMSTATE_UINT32_ARRAY(regs, IMXSPIState, ECSPI_MAX),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static void imx_spi_txfifo_reset(IMXSPIState *s)
+{
+    fifo32_reset(&s->tx_fifo);
+    s->regs[ECSPI_STATREG] |= ECSPI_STATREG_TE;
+    s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_TF;
+}
+
+static void imx_spi_rxfifo_reset(IMXSPIState *s)
+{
+    fifo32_reset(&s->rx_fifo);
+    s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_RR;
+    s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_RF;
+    s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_RO;
+}
+
+static void imx_spi_update_irq(IMXSPIState *s)
+{
+    int level;
+
+    if (fifo32_is_empty(&s->rx_fifo)) {
+        s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_RR;
+    } else {
+        s->regs[ECSPI_STATREG] |= ECSPI_STATREG_RR;
+    }
+
+    if (fifo32_is_full(&s->rx_fifo)) {
+        s->regs[ECSPI_STATREG] |= ECSPI_STATREG_RF;
+    } else {
+        s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_RF;
+    }
+
+    if (fifo32_is_empty(&s->tx_fifo)) {
+        s->regs[ECSPI_STATREG] |= ECSPI_STATREG_TE;
+    } else {
+        s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_TE;
+    }
+
+    if (fifo32_is_full(&s->tx_fifo)) {
+        s->regs[ECSPI_STATREG] |= ECSPI_STATREG_TF;
+    } else {
+        s->regs[ECSPI_STATREG] &= ~ECSPI_STATREG_TF;
+    }
+
+    level = s->regs[ECSPI_STATREG] & s->regs[ECSPI_INTREG] ? 1 : 0;
+
+    qemu_set_irq(s->irq, level);
+
+    DPRINTF("IRQ level is %d\n", level);
+}
+
+static uint8_t imx_spi_selected_channel(IMXSPIState *s)
+{
+    return EXTRACT(s->regs[ECSPI_CONREG], ECSPI_CONREG_CHANNEL_SELECT);
+}
+
+static uint32_t imx_spi_burst_length(IMXSPIState *s)
+{
+    return EXTRACT(s->regs[ECSPI_CONREG], ECSPI_CONREG_BURST_LENGTH) + 1;
+}
+
+static bool imx_spi_is_enabled(IMXSPIState *s)
+{
+    return s->regs[ECSPI_CONREG] & ECSPI_CONREG_EN;
+}
+
+static bool imx_spi_channel_is_master(IMXSPIState *s)
+{
+    uint8_t mode = EXTRACT(s->regs[ECSPI_CONREG], ECSPI_CONREG_CHANNEL_MODE);
+
+    return (mode & (1 << imx_spi_selected_channel(s))) ? true : false;
+}
+
+static bool imx_spi_is_multiple_master_burst(IMXSPIState *s)
+{
+    uint8_t wave = EXTRACT(s->regs[ECSPI_CONFIGREG], ECSPI_CONFIGREG_SS_CTL);
+
+    return imx_spi_channel_is_master(s) &&
+           !(s->regs[ECSPI_CONREG] & ECSPI_CONREG_SMC) &&
+           ((wave & (1 << imx_spi_selected_channel(s))) ? true : false);
+}
+
+static void imx_spi_flush_txfifo(IMXSPIState *s)
+{
+    uint32_t tx;
+    uint32_t rx;
+
+    DPRINTF("Begin: TX Fifo Size = %d, RX Fifo Size = %d\n",
+            fifo32_num_used(&s->tx_fifo), fifo32_num_used(&s->rx_fifo));
+
+    while (!fifo32_is_empty(&s->tx_fifo)) {
+        int tx_burst = 0;
+        int index = 0;
+
+        if (s->burst_length <= 0) {
+            s->burst_length = imx_spi_burst_length(s);
+
+            DPRINTF("Burst length = %d\n", s->burst_length);
+
+            if (imx_spi_is_multiple_master_burst(s)) {
+                s->regs[ECSPI_CONREG] |= ECSPI_CONREG_XCH;
+            }
+        }
+
+        tx = fifo32_pop(&s->tx_fifo);
+
+        DPRINTF("data tx:0x%08x\n", tx);
+
+        tx_burst = MIN(s->burst_length, 32);
+
+        rx = 0;
+
+        while (tx_burst) {
+            uint8_t byte = tx & 0xff;
+
+            DPRINTF("writing 0x%02x\n", (uint32_t)byte);
+
+            /* We need to write one byte at a time */
+            byte = ssi_transfer(s->bus, byte);
+
+            DPRINTF("0x%02x read\n", (uint32_t)byte);
+
+            tx = tx >> 8;
+            rx |= (byte << (index * 8));
+
+            /* Remove 8 bits from the actual burst */
+            tx_burst -= 8;
+            s->burst_length -= 8;
+            index++;
+        }
+
+        DPRINTF("data rx:0x%08x\n", rx);
+
+        if (fifo32_is_full(&s->rx_fifo)) {
+            s->regs[ECSPI_STATREG] |= ECSPI_STATREG_RO;
+        } else {
+            fifo32_push(&s->rx_fifo, (uint8_t)rx);
+        }
+
+        if (s->burst_length <= 0) {
+            s->regs[ECSPI_CONREG] &= ~ECSPI_CONREG_XCH;
+
+            if (!imx_spi_is_multiple_master_burst(s)) {
+                s->regs[ECSPI_STATREG] |= ECSPI_STATREG_TC;
+                break;
+            }
+        }
+    }
+
+    if (fifo32_is_empty(&s->tx_fifo)) {
+        s->regs[ECSPI_STATREG] |= ECSPI_STATREG_TC;
+    }
+
+    /* TODO: We should also use TDR and RDR bits */
+
+    DPRINTF("End: TX Fifo Size = %d, RX Fifo Size = %d\n",
+            fifo32_num_used(&s->tx_fifo), fifo32_num_used(&s->rx_fifo));
+}
+
+static void imx_spi_reset(DeviceState *dev)
+{
+    IMXSPIState *s = IMX_SPI(dev);
+
+    DPRINTF("\n");
+
+    memset(s->regs, 0, sizeof(s->regs));
+
+    s->regs[ECSPI_STATREG] = 0x00000003;
+
+    imx_spi_rxfifo_reset(s);
+    imx_spi_txfifo_reset(s);
+
+    imx_spi_update_irq(s);
+
+    s->burst_length = 0;
+}
+
+static uint64_t imx_spi_read(void *opaque, hwaddr offset, unsigned size)
+{
+    uint32_t value = 0;
+    IMXSPIState *s = opaque;
+    uint32_t index = offset >> 2;
+
+    if (index >=  ECSPI_MAX) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SPI, __func__, offset);
+        return 0;
+    }
+
+    switch (index) {
+    case ECSPI_RXDATA:
+        if (!imx_spi_is_enabled(s)) {
+            value = 0;
+        } else if (fifo32_is_empty(&s->rx_fifo)) {
+            /* value is undefined */
+            value = 0xdeadbeef;
+        } else {
+            /* read from the RX FIFO */
+            value = fifo32_pop(&s->rx_fifo);
+        }
+
+        break;
+    case ECSPI_TXDATA:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read from TX FIFO\n",
+                      TYPE_IMX_SPI, __func__);
+
+        /* Reading from TXDATA gives 0 */
+
+        break;
+    case ECSPI_MSGDATA:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to read from MSG FIFO\n",
+                      TYPE_IMX_SPI, __func__);
+
+        /* Reading from MSGDATA gives 0 */
+
+        break;
+    default:
+        value = s->regs[index];
+        break;
+    }
+
+    DPRINTF("reg[%s] => 0x%" PRIx32 "\n", imx_spi_reg_name(index), value);
+
+    imx_spi_update_irq(s);
+
+    return (uint64_t)value;
+}
+
+static void imx_spi_write(void *opaque, hwaddr offset, uint64_t value,
+                           unsigned size)
+{
+    IMXSPIState *s = opaque;
+    uint32_t index = offset >> 2;
+    uint32_t change_mask;
+
+    if (index >=  ECSPI_MAX) {
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_SPI, __func__, offset);
+        return;
+    }
+
+    DPRINTF("reg[%s] <= 0x%" PRIx32 "\n", imx_spi_reg_name(index),
+            (uint32_t)value);
+
+    change_mask = s->regs[index] ^ value;
+
+    switch (index) {
+    case ECSPI_RXDATA:
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Trying to write to RX FIFO\n",
+                      TYPE_IMX_SPI, __func__);
+        break;
+    case ECSPI_TXDATA:
+    case ECSPI_MSGDATA:
+        /* Is there any difference between TXDATA and MSGDATA ? */
+        /* I'll have to look in the linux driver */
+        if (!imx_spi_is_enabled(s)) {
+            /* Ignore writes if device is disabled */
+            break;
+        } else if (fifo32_is_full(&s->tx_fifo)) {
+            /* Ignore writes if queue is full */
+            break;
+        }
+
+        fifo32_push(&s->tx_fifo, (uint32_t)value);
+
+        if (imx_spi_channel_is_master(s) &&
+            (s->regs[ECSPI_CONREG] & ECSPI_CONREG_SMC)) {
+            /*
+             * Start emitting if current channel is master and SMC bit is
+             * set.
+             */
+            imx_spi_flush_txfifo(s);
+        }
+
+        break;
+    case ECSPI_STATREG:
+        /* the RO and TC bits are write-one-to-clear */
+        value &= ECSPI_STATREG_RO | ECSPI_STATREG_TC;
+        s->regs[ECSPI_STATREG] &= ~value;
+
+        break;
+    case ECSPI_CONREG:
+        s->regs[ECSPI_CONREG] = value;
+
+        if (!imx_spi_is_enabled(s)) {
+            /* device is disabled, so this is a reset */
+            imx_spi_reset(DEVICE(s));
+            return;
+        }
+
+        if (imx_spi_channel_is_master(s)) {
+            int i;
+
+            /* We are in master mode */
+
+            for (i = 0; i < 4; i++) {
+                qemu_set_irq(s->cs_lines[i],
+                             i == imx_spi_selected_channel(s) ? 0 : 1);
+            }
+
+            if ((value & change_mask & ECSPI_CONREG_SMC) &&
+                !fifo32_is_empty(&s->tx_fifo)) {
+                /* SMC bit is set and TX FIFO has some slots filled in */
+                imx_spi_flush_txfifo(s);
+            } else if ((value & change_mask & ECSPI_CONREG_XCH) &&
+                !(value & ECSPI_CONREG_SMC)) {
+                /* This is a request to start emitting */
+                imx_spi_flush_txfifo(s);
+            }
+        }
+
+        break;
+    default:
+        s->regs[index] = value;
+
+        break;
+    }
+
+    imx_spi_update_irq(s);
+}
+
+static const struct MemoryRegionOps imx_spi_ops = {
+    .read = imx_spi_read,
+    .write = imx_spi_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        /*
+         * Our device would not work correctly if the guest was doing
+         * unaligned access. This might not be a limitation on the real
+         * device but in practice there is no reason for a guest to access
+         * this device unaligned.
+         */
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+};
+
+static void imx_spi_realize(DeviceState *dev, Error **errp)
+{
+    IMXSPIState *s = IMX_SPI(dev);
+    int i;
+
+    s->bus = ssi_create_bus(dev, "spi");
+
+    memory_region_init_io(&s->iomem, OBJECT(dev), &imx_spi_ops, s,
+                          TYPE_IMX_SPI, 0x1000);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iomem);
+    sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->irq);
+
+    ssi_auto_connect_slaves(dev, s->cs_lines, s->bus);
+
+    for (i = 0; i < 4; ++i) {
+        sysbus_init_irq(SYS_BUS_DEVICE(dev), &s->cs_lines[i]);
+    }
+
+    s->burst_length = 0;
+
+    fifo32_create(&s->tx_fifo, ECSPI_FIFO_SIZE);
+    fifo32_create(&s->rx_fifo, ECSPI_FIFO_SIZE);
+}
+
+static void imx_spi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = imx_spi_realize;
+    dc->vmsd = &vmstate_imx_spi;
+    dc->reset = imx_spi_reset;
+    dc->desc = "i.MX SPI Controller";
+}
+
+static const TypeInfo imx_spi_info = {
+    .name          = TYPE_IMX_SPI,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(IMXSPIState),
+    .class_init    = imx_spi_class_init,
+};
+
+static void imx_spi_register_types(void)
+{
+    type_register_static(&imx_spi_info);
+}
+
+type_init(imx_spi_register_types)
diff --git a/hw/ssi/omap_spi.c b/hw/ssi/omap_spi.c
index 0ed3b11..2203465 100644
--- a/hw/ssi/omap_spi.c
+++ b/hw/ssi/omap_spi.c
@@ -19,6 +19,7 @@
  * with this program; if not, write to the Free Software Foundation, Inc.,
  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/arm/omap.h"
 
@@ -226,7 +227,8 @@
     int ch = 0;
 
     if (size != 4) {
-        return omap_badwidth_write32(opaque, addr, value);
+        omap_badwidth_write32(opaque, addr, value);
+        return;
     }
 
     switch (addr) {
@@ -341,8 +343,7 @@
 struct omap_mcspi_s *omap_mcspi_init(struct omap_target_agent_s *ta, int chnum,
                 qemu_irq irq, qemu_irq *drq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_mcspi_s *s = (struct omap_mcspi_s *)
-            g_malloc0(sizeof(struct omap_mcspi_s));
+    struct omap_mcspi_s *s = g_new0(struct omap_mcspi_s, 1);
     struct omap_mcspi_ch_s *ch = s->ch;
 
     s->irq = irq;
diff --git a/hw/ssi/pl022.c b/hw/ssi/pl022.c
index 61d568f..c136801 100644
--- a/hw/ssi/pl022.c
+++ b/hw/ssi/pl022.c
@@ -7,8 +7,10 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
+#include "qemu/log.h"
 
 //#define DEBUG_PL022 1
 
diff --git a/hw/ssi/ssi.c b/hw/ssi/ssi.c
index 2aab79b..7eaaf56 100644
--- a/hw/ssi/ssi.c
+++ b/hw/ssi/ssi.c
@@ -12,7 +12,8 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "hw/ssi.h"
+#include "qemu/osdep.h"
+#include "hw/ssi/ssi.h"
 
 struct SSIBus {
     BusState parent_obj;
@@ -53,7 +54,7 @@
     return 0;
 }
 
-static int ssi_slave_init(DeviceState *dev)
+static void ssi_slave_realize(DeviceState *dev, Error **errp)
 {
     SSISlave *s = SSI_SLAVE(dev);
     SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(s);
@@ -63,7 +64,7 @@
         qdev_init_gpio_in_named(dev, ssi_cs_default, SSI_GPIO_CS, 1);
     }
 
-    return ssc->init(s);
+    ssc->realize(s, errp);
 }
 
 static void ssi_slave_class_init(ObjectClass *klass, void *data)
@@ -71,7 +72,7 @@
     SSISlaveClass *ssc = SSI_SLAVE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    dc->init = ssi_slave_init;
+    dc->realize = ssi_slave_realize;
     dc->bus_type = TYPE_SSI_BUS;
     if (!ssc->transfer_raw) {
         ssc->transfer_raw = ssi_transfer_raw_default;
diff --git a/hw/ssi/xilinx_spi.c b/hw/ssi/xilinx_spi.c
index 620573c..33482f0 100644
--- a/hw/ssi/xilinx_spi.c
+++ b/hw/ssi/xilinx_spi.c
@@ -24,12 +24,13 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "qemu/log.h"
 #include "qemu/fifo8.h"
 
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 
 #ifdef XILINX_SPI_ERR_DEBUG
 #define DB_PRINT(...) do { \
diff --git a/hw/ssi/xilinx_spips.c b/hw/ssi/xilinx_spips.c
index 0910f54..e2b77dc 100644
--- a/hw/ssi/xilinx_spips.c
+++ b/hw/ssi/xilinx_spips.c
@@ -22,13 +22,15 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "hw/ptimer.h"
 #include "qemu/log.h"
 #include "qemu/fifo8.h"
-#include "hw/ssi.h"
+#include "hw/ssi/ssi.h"
 #include "qemu/bitops.h"
+#include "hw/ssi/xilinx_spips.h"
 
 #ifndef XILINX_SPIPS_ERR_DEBUG
 #define XILINX_SPIPS_ERR_DEBUG 0
@@ -103,8 +105,6 @@
 
 #define R_MOD_ID            (0xFC / 4)
 
-#define R_MAX (R_MOD_ID+1)
-
 /* size of TXRX FIFOs */
 #define RXFF_A          32
 #define TXFF_A          32
@@ -135,30 +135,6 @@
 } FlashCMD;
 
 typedef struct {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    MemoryRegion mmlqspi;
-
-    qemu_irq irq;
-    int irqline;
-
-    uint8_t num_cs;
-    uint8_t num_busses;
-
-    uint8_t snoop_state;
-    qemu_irq *cs_lines;
-    SSIBus **spi;
-
-    Fifo8 rx_fifo;
-    Fifo8 tx_fifo;
-
-    uint8_t num_txrx_bytes;
-
-    uint32_t regs[R_MAX];
-} XilinxSPIPS;
-
-typedef struct {
     XilinxSPIPS parent_obj;
 
     uint8_t lqspi_buf[LQSPI_CACHE_SIZE];
@@ -174,19 +150,6 @@
     uint32_t tx_fifo_size;
 } XilinxSPIPSClass;
 
-#define TYPE_XILINX_SPIPS "xlnx.ps7-spi"
-#define TYPE_XILINX_QSPIPS "xlnx.ps7-qspi"
-
-#define XILINX_SPIPS(obj) \
-     OBJECT_CHECK(XilinxSPIPS, (obj), TYPE_XILINX_SPIPS)
-#define XILINX_SPIPS_CLASS(klass) \
-     OBJECT_CLASS_CHECK(XilinxSPIPSClass, (klass), TYPE_XILINX_SPIPS)
-#define XILINX_SPIPS_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(XilinxSPIPSClass, (obj), TYPE_XILINX_SPIPS)
-
-#define XILINX_QSPIPS(obj) \
-     OBJECT_CHECK(XilinxQSPIPS, (obj), TYPE_XILINX_QSPIPS)
-
 static inline int num_effective_busses(XilinxSPIPS *s)
 {
     return (s->regs[R_LQSPI_CFG] & LQSPI_CFG_SEP_BUS &&
@@ -257,7 +220,7 @@
     XilinxSPIPS *s = XILINX_SPIPS(d);
 
     int i;
-    for (i = 0; i < R_MAX; i++) {
+    for (i = 0; i < XLNX_SPIPS_R_MAX; i++) {
         s->regs[i] = 0;
     }
 
@@ -664,7 +627,7 @@
     }
 
     memory_region_init_io(&s->iomem, OBJECT(s), xsc->reg_ops, s,
-                          "spi", R_MAX*4);
+                          "spi", XLNX_SPIPS_R_MAX * 4);
     sysbus_init_mmio(sbd, &s->iomem);
 
     s->irqline = -1;
@@ -708,7 +671,7 @@
     .fields = (VMStateField[]) {
         VMSTATE_FIFO8(tx_fifo, XilinxSPIPS),
         VMSTATE_FIFO8(rx_fifo, XilinxSPIPS),
-        VMSTATE_UINT32_ARRAY(regs, XilinxSPIPS, R_MAX),
+        VMSTATE_UINT32_ARRAY(regs, XilinxSPIPS, XLNX_SPIPS_R_MAX),
         VMSTATE_UINT8(snoop_state, XilinxSPIPS),
         VMSTATE_END_OF_LIST()
     }
diff --git a/hw/timer/Makefile.objs b/hw/timer/Makefile.objs
index 4828b96..f3ad1b5 100644
--- a/hw/timer/Makefile.objs
+++ b/hw/timer/Makefile.objs
@@ -25,10 +25,13 @@
 obj-$(CONFIG_OMAP) += omap_synctimer.o
 obj-$(CONFIG_PXA2XX) += pxa2xx_timer.o
 obj-$(CONFIG_SH4) += sh_timer.o
-obj-$(CONFIG_TUSB6010) += tusb6010.o
 obj-$(CONFIG_DIGIC) += digic-timer.o
-obj-$(CONFIG_GOLDFISH) += goldfish_timer.o
+obj-$(CONFIG_MIPS_CPS) += mips_gictimer.o
 
 obj-$(CONFIG_MC146818RTC) += mc146818rtc.o
 
 obj-$(CONFIG_ALLWINNER_A10_PIT) += allwinner-a10-pit.o
+obj-$(CONFIG_GOLDFISH) += goldfish_timer.o
+
+common-obj-$(CONFIG_STM32F2XX_TIMER) += stm32f2xx_timer.o
+common-obj-$(CONFIG_ASPEED_SOC) += aspeed_timer.o
diff --git a/hw/timer/a9gtimer.c b/hw/timer/a9gtimer.c
index a0656d5..772f85f 100644
--- a/hw/timer/a9gtimer.c
+++ b/hw/timer/a9gtimer.c
@@ -20,10 +20,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/timer/a9gtimer.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/bitops.h"
 #include "qemu/log.h"
+#include "qom/cpu.h"
 
 #ifndef A9_GTIMER_ERR_DEBUG
 #define A9_GTIMER_ERR_DEBUG 0
@@ -121,7 +124,7 @@
 {
     A9GTimerState *s = A9_GTIMER(opaque);
 
-    return a9_gtimer_update(s, false);
+    a9_gtimer_update(s, false);
 }
 
 static uint64_t a9_gtimer_read(void *opaque, hwaddr addr, unsigned size)
@@ -181,7 +184,7 @@
     case R_COUNTER_LO:
         /*
          * Keep it simple - ARM docco explicitly says to disable timer before
-         * modding it, so dont bother trying to do all the difficult on the fly
+         * modding it, so don't bother trying to do all the difficult on the fly
          * timer modifications - (if they even work in real hardware??).
          */
         if (s->control & R_CONTROL_TIMER_ENABLE) {
@@ -289,7 +292,7 @@
     int i;
 
     if (s->num_cpu < 1 || s->num_cpu > A9_GTIMER_MAX_CPUS) {
-        error_setg(errp, "%s: num-cpu must be between 1 and %d\n",
+        error_setg(errp, "%s: num-cpu must be between 1 and %d",
                    __func__, A9_GTIMER_MAX_CPUS);
         return;
     }
@@ -328,7 +331,7 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_TIMER(timer, A9GTimerState),
+        VMSTATE_TIMER_PTR(timer, A9GTimerState),
         VMSTATE_UINT64(counter, A9GTimerState),
         VMSTATE_UINT64(ref_counter, A9GTimerState),
         VMSTATE_UINT64(cpu_ref_time, A9GTimerState),
diff --git a/hw/timer/allwinner-a10-pit.c b/hw/timer/allwinner-a10-pit.c
index 34124fe..3385e5d 100644
--- a/hw/timer/allwinner-a10-pit.c
+++ b/hw/timer/allwinner-a10-pit.c
@@ -15,9 +15,11 @@
  * for more details.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "hw/timer/allwinner-a10-pit.h"
+#include "qemu/log.h"
 
 static void a10_pit_update_irq(AwA10PITState *s)
 {
diff --git a/hw/timer/arm_mptimer.c b/hw/timer/arm_mptimer.c
index 35a0a23..d66bbf0 100644
--- a/hw/timer/arm_mptimer.c
+++ b/hw/timer/arm_mptimer.c
@@ -19,7 +19,9 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/timer/arm_mptimer.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qom/cpu.h"
 
@@ -38,7 +40,7 @@
 
 static inline void timerblock_update_irq(TimerBlock *tb)
 {
-    qemu_set_irq(tb->irq, tb->status);
+    qemu_set_irq(tb->irq, tb->status && (tb->control & 4));
 }
 
 /* Return conversion factor from mpcore timer ticks to qemu timer ticks.  */
@@ -122,11 +124,18 @@
     case 8: /* Control.  */
         old = tb->control;
         tb->control = value;
-        if (((old & 1) == 0) && (value & 1)) {
-            if (tb->count == 0 && (tb->control & 2)) {
+        if (value & 1) {
+            if ((old & 1) && (tb->count != 0)) {
+                /* Do nothing if timer is ticking right now.  */
+                break;
+            }
+            if (tb->control & 2) {
                 tb->count = tb->load;
             }
             timerblock_reload(tb, 1);
+        } else if (old & 1) {
+            /* Shutdown the timer.  */
+            timer_del(tb->timer);
         }
         break;
     case 12: /* Interrupt status.  */
@@ -213,8 +222,9 @@
     int i;
 
     if (s->num_cpu < 1 || s->num_cpu > ARM_MPTIMER_MAX_CPUS) {
-        hw_error("%s: num-cpu must be between 1 and %d\n",
-                 __func__, ARM_MPTIMER_MAX_CPUS);
+        error_setg(errp, "num-cpu must be between 1 and %d",
+                   ARM_MPTIMER_MAX_CPUS);
+        return;
     }
     /* We implement one timer block per CPU, and expose multiple MMIO regions:
      *  * region 0 is "timer for this core"
@@ -246,7 +256,7 @@
         VMSTATE_UINT32(control, TimerBlock),
         VMSTATE_UINT32(status, TimerBlock),
         VMSTATE_INT64(tick, TimerBlock),
-        VMSTATE_TIMER(timer, TimerBlock),
+        VMSTATE_TIMER_PTR(timer, TimerBlock),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/timer/arm_timer.c b/hw/timer/arm_timer.c
index 1452910..111a16d 100644
--- a/hw/timer/arm_timer.c
+++ b/hw/timer/arm_timer.c
@@ -7,12 +7,14 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
 #include "hw/qdev.h"
 #include "hw/ptimer.h"
 #include "qemu/main-loop.h"
+#include "qemu/log.h"
 
 /* Common timer implementation.  */
 
@@ -276,23 +278,25 @@
     }
 };
 
-static int sp804_init(SysBusDevice *sbd)
+static void sp804_init(Object *obj)
 {
-    DeviceState *dev = DEVICE(sbd);
-    SP804State *s = SP804(dev);
-    qemu_irq *qi;
+    SP804State *s = SP804(obj);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(obj);
 
-    qi = qemu_allocate_irqs(sp804_set_irq, s, 2);
     sysbus_init_irq(sbd, &s->irq);
-    s->timer[0] = arm_timer_init(s->freq0);
-    s->timer[1] = arm_timer_init(s->freq1);
-    s->timer[0]->irq = qi[0];
-    s->timer[1]->irq = qi[1];
-    memory_region_init_io(&s->iomem, OBJECT(s), &sp804_ops, s,
+    memory_region_init_io(&s->iomem, obj, &sp804_ops, s,
                           "sp804", 0x1000);
     sysbus_init_mmio(sbd, &s->iomem);
-    vmstate_register(dev, -1, &vmstate_sp804, s);
-    return 0;
+}
+
+static void sp804_realize(DeviceState *dev, Error **errp)
+{
+    SP804State *s = SP804(dev);
+
+    s->timer[0] = arm_timer_init(s->freq0);
+    s->timer[1] = arm_timer_init(s->freq1);
+    s->timer[0]->irq = qemu_allocate_irq(sp804_set_irq, s, 0);
+    s->timer[1]->irq = qemu_allocate_irq(sp804_set_irq, s, 1);
 }
 
 /* Integrator/CP timer module.  */
@@ -345,9 +349,10 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int icp_pit_init(SysBusDevice *dev)
+static void icp_pit_init(Object *obj)
 {
-    icp_pit_state *s = INTEGRATOR_PIT(dev);
+    icp_pit_state *s = INTEGRATOR_PIT(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     /* Timer 0 runs at the system clock speed (40MHz).  */
     s->timer[0] = arm_timer_init(40000000);
@@ -359,26 +364,18 @@
     sysbus_init_irq(dev, &s->timer[1]->irq);
     sysbus_init_irq(dev, &s->timer[2]->irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &icp_pit_ops, s,
+    memory_region_init_io(&s->iomem, obj, &icp_pit_ops, s,
                           "icp_pit", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
     /* This device has no state to save/restore.  The component timers will
        save themselves.  */
-    return 0;
-}
-
-static void icp_pit_class_init(ObjectClass *klass, void *data)
-{
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
-
-    sdc->init = icp_pit_init;
 }
 
 static const TypeInfo icp_pit_info = {
     .name          = TYPE_INTEGRATOR_PIT,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(icp_pit_state),
-    .class_init    = icp_pit_class_init,
+    .instance_init = icp_pit_init,
 };
 
 static Property sp804_properties[] = {
@@ -389,17 +386,18 @@
 
 static void sp804_class_init(ObjectClass *klass, void *data)
 {
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
     DeviceClass *k = DEVICE_CLASS(klass);
 
-    sdc->init = sp804_init;
+    k->realize = sp804_realize;
     k->props = sp804_properties;
+    k->vmsd = &vmstate_sp804;
 }
 
 static const TypeInfo sp804_info = {
     .name          = TYPE_SP804,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(SP804State),
+    .instance_init = sp804_init,
     .class_init    = sp804_class_init,
 };
 
diff --git a/hw/timer/aspeed_timer.c b/hw/timer/aspeed_timer.c
new file mode 100644
index 0000000..9b70ee0
--- /dev/null
+++ b/hw/timer/aspeed_timer.c
@@ -0,0 +1,514 @@
+/*
+ * ASPEED AST2400 Timer
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/sysbus.h"
+#include "hw/timer/aspeed_timer.h"
+#include "qemu-common.h"
+#include "qemu/bitops.h"
+#include "qemu/timer.h"
+#include "qemu/log.h"
+#include "trace.h"
+
+#define TIMER_NR_REGS 4
+
+#define TIMER_CTRL_BITS 4
+#define TIMER_CTRL_MASK ((1 << TIMER_CTRL_BITS) - 1)
+
+#define TIMER_CLOCK_USE_EXT true
+#define TIMER_CLOCK_EXT_HZ 1000000
+#define TIMER_CLOCK_USE_APB false
+#define TIMER_CLOCK_APB_HZ 24000000
+
+#define TIMER_REG_STATUS 0
+#define TIMER_REG_RELOAD 1
+#define TIMER_REG_MATCH_FIRST 2
+#define TIMER_REG_MATCH_SECOND 3
+
+#define TIMER_FIRST_CAP_PULSE 4
+
+enum timer_ctrl_op {
+    op_enable = 0,
+    op_external_clock,
+    op_overflow_interrupt,
+    op_pulse_enable
+};
+
+/**
+ * Avoid mutual references between AspeedTimerCtrlState and AspeedTimer
+ * structs, as it's a waste of memory. The ptimer BH callback needs to know
+ * whether a specific AspeedTimer is enabled, but this information is held in
+ * AspeedTimerCtrlState. So, provide a helper to hoist ourselves from an
+ * arbitrary AspeedTimer to AspeedTimerCtrlState.
+ */
+static inline AspeedTimerCtrlState *timer_to_ctrl(AspeedTimer *t)
+{
+    const AspeedTimer (*timers)[] = (void *)t - (t->id * sizeof(*t));
+    return container_of(timers, AspeedTimerCtrlState, timers);
+}
+
+static inline bool timer_ctrl_status(AspeedTimer *t, enum timer_ctrl_op op)
+{
+    return !!(timer_to_ctrl(t)->ctrl & BIT(t->id * TIMER_CTRL_BITS + op));
+}
+
+static inline bool timer_enabled(AspeedTimer *t)
+{
+    return timer_ctrl_status(t, op_enable);
+}
+
+static inline bool timer_overflow_interrupt(AspeedTimer *t)
+{
+    return timer_ctrl_status(t, op_overflow_interrupt);
+}
+
+static inline bool timer_can_pulse(AspeedTimer *t)
+{
+    return t->id >= TIMER_FIRST_CAP_PULSE;
+}
+
+static inline bool timer_external_clock(AspeedTimer *t)
+{
+    return timer_ctrl_status(t, op_external_clock);
+}
+
+static uint32_t clock_rates[] = { TIMER_CLOCK_APB_HZ, TIMER_CLOCK_EXT_HZ };
+
+static inline uint32_t calculate_rate(struct AspeedTimer *t)
+{
+    return clock_rates[timer_external_clock(t)];
+}
+
+static inline uint32_t calculate_ticks(struct AspeedTimer *t, uint64_t now_ns)
+{
+    uint64_t delta_ns = now_ns - MIN(now_ns, t->start);
+    uint32_t rate = calculate_rate(t);
+    uint64_t ticks = muldiv64(delta_ns, rate, NANOSECONDS_PER_SECOND);
+
+    return t->reload - MIN(t->reload, ticks);
+}
+
+static inline uint64_t calculate_time(struct AspeedTimer *t, uint32_t ticks)
+{
+    uint64_t delta_ns;
+    uint64_t delta_ticks;
+
+    delta_ticks = t->reload - MIN(t->reload, ticks);
+    delta_ns = muldiv64(delta_ticks, NANOSECONDS_PER_SECOND, calculate_rate(t));
+
+    return t->start + delta_ns;
+}
+
+static uint64_t calculate_next(struct AspeedTimer *t)
+{
+    uint64_t next = 0;
+    uint32_t rate = calculate_rate(t);
+
+    while (!next) {
+        /* We don't know the relationship between the values in the match
+         * registers, so sort using MAX/MIN/zero. We sort in that order as the
+         * timer counts down to zero. */
+        uint64_t seq[] = {
+            calculate_time(t, MAX(t->match[0], t->match[1])),
+            calculate_time(t, MIN(t->match[0], t->match[1])),
+            calculate_time(t, 0),
+        };
+        uint64_t reload_ns;
+        uint64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+        if (now < seq[0]) {
+            next = seq[0];
+        } else if (now < seq[1]) {
+            next = seq[1];
+        } else if (now < seq[2]) {
+            next = seq[2];
+        } else {
+            reload_ns = muldiv64(t->reload, NANOSECONDS_PER_SECOND, rate);
+            t->start = now - ((now - t->start) % reload_ns);
+        }
+    }
+
+    return next;
+}
+
+static void aspeed_timer_expire(void *opaque)
+{
+    AspeedTimer *t = opaque;
+    bool interrupt = false;
+    uint32_t ticks;
+
+    if (!timer_enabled(t)) {
+        return;
+    }
+
+    ticks = calculate_ticks(t, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+
+    if (!ticks) {
+        interrupt = timer_overflow_interrupt(t) || !t->match[0] || !t->match[1];
+    } else if (ticks <= MIN(t->match[0], t->match[1])) {
+        interrupt = true;
+    } else if (ticks <= MAX(t->match[0], t->match[1])) {
+        interrupt = true;
+    }
+
+    if (interrupt) {
+        t->level = !t->level;
+        qemu_set_irq(t->irq, t->level);
+    }
+
+    timer_mod(&t->timer, calculate_next(t));
+}
+
+static uint64_t aspeed_timer_get_value(AspeedTimer *t, int reg)
+{
+    uint64_t value;
+
+    switch (reg) {
+    case TIMER_REG_STATUS:
+        value = calculate_ticks(t, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+        break;
+    case TIMER_REG_RELOAD:
+        value = t->reload;
+        break;
+    case TIMER_REG_MATCH_FIRST:
+    case TIMER_REG_MATCH_SECOND:
+        value = t->match[reg - 2];
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "%s: Programming error: unexpected reg: %d\n",
+                      __func__, reg);
+        value = 0;
+        break;
+    }
+    return value;
+}
+
+static uint64_t aspeed_timer_read(void *opaque, hwaddr offset, unsigned size)
+{
+    AspeedTimerCtrlState *s = opaque;
+    const int reg = (offset & 0xf) / 4;
+    uint64_t value;
+
+    switch (offset) {
+    case 0x30: /* Control Register */
+        value = s->ctrl;
+        break;
+    case 0x34: /* Control Register 2 */
+        value = s->ctrl2;
+        break;
+    case 0x00 ... 0x2c: /* Timers 1 - 4 */
+        value = aspeed_timer_get_value(&s->timers[(offset >> 4)], reg);
+        break;
+    case 0x40 ... 0x8c: /* Timers 5 - 8 */
+        value = aspeed_timer_get_value(&s->timers[(offset >> 4) - 1], reg);
+        break;
+    /* Illegal */
+    case 0x38:
+    case 0x3C:
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                __func__, offset);
+        value = 0;
+        break;
+    }
+    trace_aspeed_timer_read(offset, size, value);
+    return value;
+}
+
+static void aspeed_timer_set_value(AspeedTimerCtrlState *s, int timer, int reg,
+                                   uint32_t value)
+{
+    AspeedTimer *t;
+
+    trace_aspeed_timer_set_value(timer, reg, value);
+    t = &s->timers[timer];
+    switch (reg) {
+    case TIMER_REG_STATUS:
+        if (timer_enabled(t)) {
+            uint64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+            int64_t delta = (int64_t) value - (int64_t) calculate_ticks(t, now);
+            uint32_t rate = calculate_rate(t);
+
+            t->start += muldiv64(delta, NANOSECONDS_PER_SECOND, rate);
+            timer_mod(&t->timer, calculate_next(t));
+        }
+        break;
+    case TIMER_REG_RELOAD:
+        t->reload = value;
+        break;
+    case TIMER_REG_MATCH_FIRST:
+    case TIMER_REG_MATCH_SECOND:
+        t->match[reg - 2] = value;
+        if (timer_enabled(t)) {
+            timer_mod(&t->timer, calculate_next(t));
+        }
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "%s: Programming error: unexpected reg: %d\n",
+                      __func__, reg);
+        break;
+    }
+}
+
+/* Control register operations are broken out into helpers that can be
+ * explicitly called on aspeed_timer_reset(), but also from
+ * aspeed_timer_ctrl_op().
+ */
+
+static void aspeed_timer_ctrl_enable(AspeedTimer *t, bool enable)
+{
+    trace_aspeed_timer_ctrl_enable(t->id, enable);
+    if (enable) {
+        t->start = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        timer_mod(&t->timer, calculate_next(t));
+    } else {
+        timer_del(&t->timer);
+    }
+}
+
+static void aspeed_timer_ctrl_external_clock(AspeedTimer *t, bool enable)
+{
+    trace_aspeed_timer_ctrl_external_clock(t->id, enable);
+}
+
+static void aspeed_timer_ctrl_overflow_interrupt(AspeedTimer *t, bool enable)
+{
+    trace_aspeed_timer_ctrl_overflow_interrupt(t->id, enable);
+}
+
+static void aspeed_timer_ctrl_pulse_enable(AspeedTimer *t, bool enable)
+{
+    if (timer_can_pulse(t)) {
+        trace_aspeed_timer_ctrl_pulse_enable(t->id, enable);
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                "%s: Timer does not support pulse mode\n", __func__);
+    }
+}
+
+/**
+ * Given the actions are fixed in number and completely described in helper
+ * functions, dispatch with a lookup table rather than manage control flow with
+ * a switch statement.
+ */
+static void (*const ctrl_ops[])(AspeedTimer *, bool) = {
+    [op_enable] = aspeed_timer_ctrl_enable,
+    [op_external_clock] = aspeed_timer_ctrl_external_clock,
+    [op_overflow_interrupt] = aspeed_timer_ctrl_overflow_interrupt,
+    [op_pulse_enable] = aspeed_timer_ctrl_pulse_enable,
+};
+
+/**
+ * Conditionally affect changes chosen by a timer's control bit.
+ *
+ * The aspeed_timer_ctrl_op() interface is convenient for the
+ * aspeed_timer_set_ctrl() function as the "no change" early exit can be
+ * calculated for all operations, which cleans up the caller code. However the
+ * interface isn't convenient for the reset function where we want to enter a
+ * specific state without artificially constructing old and new values that
+ * will fall through the change guard (and motivates extracting the actions
+ * out to helper functions).
+ *
+ * @t: The timer to manipulate
+ * @op: The type of operation to be performed
+ * @old: The old state of the timer's control bits
+ * @new: The incoming state for the timer's control bits
+ */
+static void aspeed_timer_ctrl_op(AspeedTimer *t, enum timer_ctrl_op op,
+                                 uint8_t old, uint8_t new)
+{
+    const uint8_t mask = BIT(op);
+    const bool enable = !!(new & mask);
+    const bool changed = ((old ^ new) & mask);
+    if (!changed) {
+        return;
+    }
+    ctrl_ops[op](t, enable);
+}
+
+static void aspeed_timer_set_ctrl(AspeedTimerCtrlState *s, uint32_t reg)
+{
+    int i;
+    int shift;
+    uint8_t t_old, t_new;
+    AspeedTimer *t;
+    const uint8_t enable_mask = BIT(op_enable);
+
+    /* Handle a dependency between the 'enable' and remaining three
+     * configuration bits - i.e. if more than one bit in the control set has
+     * changed, including the 'enable' bit, then we want either disable the
+     * timer and perform configuration, or perform configuration and then
+     * enable the timer
+     */
+    for (i = 0; i < ASPEED_TIMER_NR_TIMERS; i++) {
+        t = &s->timers[i];
+        shift = (i * TIMER_CTRL_BITS);
+        t_old = (s->ctrl >> shift) & TIMER_CTRL_MASK;
+        t_new = (reg >> shift) & TIMER_CTRL_MASK;
+
+        /* If we are disabling, do so first */
+        if ((t_old & enable_mask) && !(t_new & enable_mask)) {
+            aspeed_timer_ctrl_enable(t, false);
+        }
+        aspeed_timer_ctrl_op(t, op_external_clock, t_old, t_new);
+        aspeed_timer_ctrl_op(t, op_overflow_interrupt, t_old, t_new);
+        aspeed_timer_ctrl_op(t, op_pulse_enable, t_old, t_new);
+        /* If we are enabling, do so last */
+        if (!(t_old & enable_mask) && (t_new & enable_mask)) {
+            aspeed_timer_ctrl_enable(t, true);
+        }
+    }
+    s->ctrl = reg;
+}
+
+static void aspeed_timer_set_ctrl2(AspeedTimerCtrlState *s, uint32_t value)
+{
+    trace_aspeed_timer_set_ctrl2(value);
+}
+
+static void aspeed_timer_write(void *opaque, hwaddr offset, uint64_t value,
+                               unsigned size)
+{
+    const uint32_t tv = (uint32_t)(value & 0xFFFFFFFF);
+    const int reg = (offset & 0xf) / 4;
+    AspeedTimerCtrlState *s = opaque;
+
+    switch (offset) {
+    /* Control Registers */
+    case 0x30:
+        aspeed_timer_set_ctrl(s, tv);
+        break;
+    case 0x34:
+        aspeed_timer_set_ctrl2(s, tv);
+        break;
+    /* Timer Registers */
+    case 0x00 ... 0x2c:
+        aspeed_timer_set_value(s, (offset >> TIMER_NR_REGS), reg, tv);
+        break;
+    case 0x40 ... 0x8c:
+        aspeed_timer_set_value(s, (offset >> TIMER_NR_REGS) - 1, reg, tv);
+        break;
+    /* Illegal */
+    case 0x38:
+    case 0x3C:
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset 0x%" HWADDR_PRIx "\n",
+                __func__, offset);
+        break;
+    }
+}
+
+static const MemoryRegionOps aspeed_timer_ops = {
+    .read = aspeed_timer_read,
+    .write = aspeed_timer_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .valid.unaligned = false,
+};
+
+static void aspeed_init_one_timer(AspeedTimerCtrlState *s, uint8_t id)
+{
+    AspeedTimer *t = &s->timers[id];
+
+    t->id = id;
+    timer_init_ns(&t->timer, QEMU_CLOCK_VIRTUAL, aspeed_timer_expire, t);
+}
+
+static void aspeed_timer_realize(DeviceState *dev, Error **errp)
+{
+    int i;
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    AspeedTimerCtrlState *s = ASPEED_TIMER(dev);
+
+    for (i = 0; i < ASPEED_TIMER_NR_TIMERS; i++) {
+        aspeed_init_one_timer(s, i);
+        sysbus_init_irq(sbd, &s->timers[i].irq);
+    }
+    memory_region_init_io(&s->iomem, OBJECT(s), &aspeed_timer_ops, s,
+                          TYPE_ASPEED_TIMER, 0x1000);
+    sysbus_init_mmio(sbd, &s->iomem);
+}
+
+static void aspeed_timer_reset(DeviceState *dev)
+{
+    int i;
+    AspeedTimerCtrlState *s = ASPEED_TIMER(dev);
+
+    for (i = 0; i < ASPEED_TIMER_NR_TIMERS; i++) {
+        AspeedTimer *t = &s->timers[i];
+        /* Explicitly call helpers to avoid any conditional behaviour through
+         * aspeed_timer_set_ctrl().
+         */
+        aspeed_timer_ctrl_enable(t, false);
+        aspeed_timer_ctrl_external_clock(t, TIMER_CLOCK_USE_APB);
+        aspeed_timer_ctrl_overflow_interrupt(t, false);
+        aspeed_timer_ctrl_pulse_enable(t, false);
+        t->level = 0;
+        t->reload = 0;
+        t->match[0] = 0;
+        t->match[1] = 0;
+    }
+    s->ctrl = 0;
+    s->ctrl2 = 0;
+}
+
+static const VMStateDescription vmstate_aspeed_timer = {
+    .name = "aspeed.timer",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(id, AspeedTimer),
+        VMSTATE_INT32(level, AspeedTimer),
+        VMSTATE_TIMER(timer, AspeedTimer),
+        VMSTATE_UINT32(reload, AspeedTimer),
+        VMSTATE_UINT32_ARRAY(match, AspeedTimer, 2),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_aspeed_timer_state = {
+    .name = "aspeed.timerctrl",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(ctrl, AspeedTimerCtrlState),
+        VMSTATE_UINT32(ctrl2, AspeedTimerCtrlState),
+        VMSTATE_STRUCT_ARRAY(timers, AspeedTimerCtrlState,
+                             ASPEED_TIMER_NR_TIMERS, 2, vmstate_aspeed_timer,
+                             AspeedTimer),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void timer_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->realize = aspeed_timer_realize;
+    dc->reset = aspeed_timer_reset;
+    dc->desc = "ASPEED Timer";
+    dc->vmsd = &vmstate_aspeed_timer_state;
+}
+
+static const TypeInfo aspeed_timer_info = {
+    .name = TYPE_ASPEED_TIMER,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(AspeedTimerCtrlState),
+    .class_init = timer_class_init,
+};
+
+static void aspeed_timer_register_types(void)
+{
+    type_register_static(&aspeed_timer_info);
+}
+
+type_init(aspeed_timer_register_types)
diff --git a/hw/timer/cadence_ttc.c b/hw/timer/cadence_ttc.c
index d46db3c..03f5b9c 100644
--- a/hw/timer/cadence_ttc.c
+++ b/hw/timer/cadence_ttc.c
@@ -16,6 +16,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 
@@ -208,15 +209,14 @@
             s->reg_intr |= (2 << i);
         }
     }
+    if ((x < 0) || (x >= interval)) {
+        s->reg_intr |= (s->reg_count & COUNTER_CTRL_INT) ?
+            COUNTER_INTR_IV : COUNTER_INTR_OV;
+    }
     while (x < 0) {
         x += interval;
     }
     s->reg_value = (uint32_t)(x % interval);
-
-    if (s->reg_value != x) {
-        s->reg_intr |= (s->reg_count & COUNTER_CTRL_INT) ?
-            COUNTER_INTR_IV : COUNTER_INTR_OV;
-    }
     cadence_timer_update(s);
 }
 
diff --git a/hw/timer/digic-timer.c b/hw/timer/digic-timer.c
index 7e28e7e..0f21faf 100644
--- a/hw/timer/digic-timer.c
+++ b/hw/timer/digic-timer.c
@@ -26,9 +26,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/ptimer.h"
 #include "qemu/main-loop.h"
+#include "qemu/log.h"
 
 #include "hw/timer/digic-timer.h"
 
diff --git a/hw/timer/ds1338.c b/hw/timer/ds1338.c
index ec6dbee..0112949 100644
--- a/hw/timer/ds1338.c
+++ b/hw/timer/ds1338.c
@@ -10,7 +10,10 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "hw/i2c/i2c.h"
+#include "qemu/bcd.h"
 
 /* Size of NVRAM including both the user-accessible area and the
  * secondary register area.
diff --git a/hw/timer/etraxfs_timer.c b/hw/timer/etraxfs_timer.c
index aee4990..36d8f46 100644
--- a/hw/timer/etraxfs_timer.c
+++ b/hw/timer/etraxfs_timer.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
 #include "qemu/timer.h"
diff --git a/hw/timer/exynos4210_mct.c b/hw/timer/exynos4210_mct.c
index 015bbaf..ae69345 100644
--- a/hw/timer/exynos4210_mct.c
+++ b/hw/timer/exynos4210_mct.c
@@ -52,6 +52,7 @@
  * there is no way to avoid frequently events).
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu/main-loop.h"
@@ -1421,10 +1422,11 @@
 };
 
 /* MCT init */
-static int exynos4210_mct_init(SysBusDevice *dev)
+static void exynos4210_mct_init(Object *obj)
 {
     int i;
-    Exynos4210MCTState *s = EXYNOS4210_MCT(dev);
+    Exynos4210MCTState *s = EXYNOS4210_MCT(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     QEMUBH *bh[2];
 
     /* Global timer */
@@ -1449,19 +1451,15 @@
         sysbus_init_irq(dev, &s->l_timer[i].irq);
     }
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_mct_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_mct_ops, s,
                           "exynos4210-mct", MCT_SFR_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void exynos4210_mct_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_mct_init;
     dc->reset = exynos4210_mct_reset;
     dc->vmsd = &vmstate_exynos4210_mct_state;
 }
@@ -1470,6 +1468,7 @@
     .name          = TYPE_EXYNOS4210_MCT,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210MCTState),
+    .instance_init = exynos4210_mct_init,
     .class_init    = exynos4210_mct_class_init,
 };
 
diff --git a/hw/timer/exynos4210_pwm.c b/hw/timer/exynos4210_pwm.c
index 1c1a2b8..0e9e2e9 100644
--- a/hw/timer/exynos4210_pwm.c
+++ b/hw/timer/exynos4210_pwm.c
@@ -20,6 +20,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
@@ -379,9 +380,10 @@
 /*
  * PWM timer initialization
  */
-static int exynos4210_pwm_init(SysBusDevice *dev)
+static void exynos4210_pwm_init(Object *obj)
 {
-    Exynos4210PWMState *s = EXYNOS4210_PWM(dev);
+    Exynos4210PWMState *s = EXYNOS4210_PWM(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     int i;
     QEMUBH *bh;
 
@@ -393,19 +395,15 @@
         s->timer[i].parent = s;
     }
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_pwm_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_pwm_ops, s,
                           "exynos4210-pwm", EXYNOS4210_PWM_REG_MEM_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void exynos4210_pwm_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_pwm_init;
     dc->reset = exynos4210_pwm_reset;
     dc->vmsd = &vmstate_exynos4210_pwm_state;
 }
@@ -414,6 +412,7 @@
     .name          = TYPE_EXYNOS4210_PWM,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210PWMState),
+    .instance_init = exynos4210_pwm_init,
     .class_init    = exynos4210_pwm_class_init,
 };
 
diff --git a/hw/timer/exynos4210_rtc.c b/hw/timer/exynos4210_rtc.c
index bf2ee9f..da4dd45 100644
--- a/hw/timer/exynos4210_rtc.c
+++ b/hw/timer/exynos4210_rtc.c
@@ -25,9 +25,11 @@
  *  CLKOUTEN Bit[9] not used
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
+#include "qemu/bcd.h"
 #include "hw/ptimer.h"
 
 #include "hw/hw.h"
@@ -546,9 +548,10 @@
 /*
  * RTC timer initialization
  */
-static int exynos4210_rtc_init(SysBusDevice *dev)
+static void exynos4210_rtc_init(Object *obj)
 {
-    Exynos4210RTCState *s = EXYNOS4210_RTC(dev);
+    Exynos4210RTCState *s = EXYNOS4210_RTC(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     QEMUBH *bh;
 
     bh = qemu_bh_new(exynos4210_rtc_tick, s);
@@ -563,19 +566,15 @@
     sysbus_init_irq(dev, &s->alm_irq);
     sysbus_init_irq(dev, &s->tick_irq);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &exynos4210_rtc_ops, s,
+    memory_region_init_io(&s->iomem, obj, &exynos4210_rtc_ops, s,
                           "exynos4210-rtc", EXYNOS4210_RTC_REG_MEM_SIZE);
     sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static void exynos4210_rtc_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = exynos4210_rtc_init;
     dc->reset = exynos4210_rtc_reset;
     dc->vmsd = &vmstate_exynos4210_rtc_state;
 }
@@ -584,6 +583,7 @@
     .name          = TYPE_EXYNOS4210_RTC,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(Exynos4210RTCState),
+    .instance_init = exynos4210_rtc_init,
     .class_init    = exynos4210_rtc_class_init,
 };
 
diff --git a/hw/timer/goldfish_timer.c b/hw/timer/goldfish_timer.c
index 0b2f895..35a8251 100644
--- a/hw/timer/goldfish_timer.c
+++ b/hw/timer/goldfish_timer.c
@@ -9,6 +9,7 @@
 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ** GNU General Public License for more details.
 */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "cpu.h"
diff --git a/hw/timer/grlib_gptimer.c b/hw/timer/grlib_gptimer.c
index d655bb2..dd000f5 100644
--- a/hw/timer/grlib_gptimer.c
+++ b/hw/timer/grlib_gptimer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
diff --git a/hw/timer/hpet.c b/hw/timer/hpet.c
index e160e8f..a2c18b3 100644
--- a/hw/timer/hpet.c
+++ b/hw/timer/hpet.c
@@ -1,5 +1,5 @@
 /*
- *  High Precisition Event Timer emulation
+ *  High Precision Event Timer emulation
  *
  *  Copyright (c) 2007 Alexander Graf
  *  Copyright (c) 2008 IBM Corporation
@@ -24,9 +24,12 @@
  * This driver attempts to emulate an HPET device in software.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "ui/console.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "hw/timer/hpet.h"
 #include "hw/sysbus.h"
@@ -115,22 +118,22 @@
 
 static uint32_t hpet_time_after(uint64_t a, uint64_t b)
 {
-    return ((int32_t)(b) - (int32_t)(a) < 0);
+    return ((int32_t)(b - a) < 0);
 }
 
 static uint32_t hpet_time_after64(uint64_t a, uint64_t b)
 {
-    return ((int64_t)(b) - (int64_t)(a) < 0);
+    return ((int64_t)(b - a) < 0);
 }
 
 static uint64_t ticks_to_ns(uint64_t value)
 {
-    return (muldiv64(value, HPET_CLK_PERIOD, FS_PER_NS));
+    return value * HPET_CLK_PERIOD;
 }
 
 static uint64_t ns_to_ticks(uint64_t value)
 {
-    return (muldiv64(value, FS_PER_NS, HPET_CLK_PERIOD));
+    return value / HPET_CLK_PERIOD;
 }
 
 static uint64_t hpet_fixup_reg(uint64_t new, uint64_t old, uint64_t mask)
@@ -198,24 +201,15 @@
     if (!set || !timer_enabled(timer) || !hpet_enabled(timer->state)) {
         s->isr &= ~mask;
         if (!timer_fsb_route(timer)) {
-            /* fold the ICH PIRQ# pin's internal inversion logic into hpet */
-            if (route >= ISA_NUM_IRQS) {
-                qemu_irq_raise(s->irqs[route]);
-            } else {
-                qemu_irq_lower(s->irqs[route]);
-            }
+            qemu_irq_lower(s->irqs[route]);
         }
     } else if (timer_fsb_route(timer)) {
-        stl_le_phys(&address_space_memory,
-                    timer->fsb >> 32, timer->fsb & 0xffffffff);
+        address_space_stl_le(&address_space_memory, timer->fsb >> 32,
+                             timer->fsb & 0xffffffff, MEMTXATTRS_UNSPECIFIED,
+                             NULL);
     } else if (timer->config & HPET_TN_TYPE_LEVEL) {
         s->isr |= mask;
-        /* fold the ICH PIRQ# pin's internal inversion logic into hpet */
-        if (route >= ISA_NUM_IRQS) {
-            qemu_irq_lower(s->irqs[route]);
-        } else {
-            qemu_irq_raise(s->irqs[route]);
-        }
+        qemu_irq_raise(s->irqs[route]);
     } else {
         s->isr &= ~mask;
         qemu_irq_pulse(s->irqs[route]);
@@ -282,6 +276,7 @@
     .name = "hpet/rtc_irq_level",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = hpet_rtc_irq_level_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(rtc_irq_level, HPETState),
         VMSTATE_END_OF_LIST()
@@ -299,7 +294,7 @@
         VMSTATE_UINT64(fsb, HPETTimer),
         VMSTATE_UINT64(period, HPETTimer),
         VMSTATE_UINT8(wrap_flag, HPETTimer),
-        VMSTATE_TIMER(qemu_timer, HPETTimer),
+        VMSTATE_TIMER_PTR(qemu_timer, HPETTimer),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -321,13 +316,9 @@
                                     vmstate_hpet_timer, HPETTimer),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_hpet_rtc_irq_level,
-            .needed = hpet_rtc_irq_level_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_hpet_rtc_irq_level,
+        NULL
     }
 };
 
@@ -714,7 +705,7 @@
     HPETState *s = HPET(obj);
 
     /* HPET Area */
-    memory_region_init_io(&s->iomem, obj, &hpet_ram_ops, s, "hpet", 0x400);
+    memory_region_init_io(&s->iomem, obj, &hpet_ram_ops, s, "hpet", HPET_LEN);
     sysbus_init_mmio(sbd, &s->iomem);
 }
 
@@ -759,7 +750,7 @@
     /* 64-bit main counter; LegacyReplacementRoute. */
     s->capability = 0x8086a001ULL;
     s->capability |= (s->num_timers - 1) << HPET_ID_NUM_TIM_SHIFT;
-    s->capability |= ((HPET_CLK_PERIOD) << 32);
+    s->capability |= ((uint64_t)(HPET_CLK_PERIOD * FS_PER_NS) << 32);
 
     qdev_init_gpio_in(dev, hpet_handle_legacy_irq, 2);
     qdev_init_gpio_out(dev, &s->pit_enabled, 1);
diff --git a/hw/timer/i8254.c b/hw/timer/i8254.c
index 3450c98..5e61ad5 100644
--- a/hw/timer/i8254.c
+++ b/hw/timer/i8254.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -52,7 +53,7 @@
     int counter;
 
     d = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - s->count_load_time, PIT_FREQ,
-                 get_ticks_per_sec());
+                 NANOSECONDS_PER_SECOND);
     switch(s->mode) {
     case 0:
     case 1:
@@ -196,6 +197,12 @@
     PITChannelState *s;
 
     addr &= 3;
+
+    if (addr == 3) {
+        /* Mode/Command register is write only, read is ignored */
+        return 0;
+    }
+
     s = &pit->channels[addr];
     if (s->status_latched) {
         s->status_latched = 0;
@@ -256,7 +263,7 @@
 #ifdef DEBUG_PIT
     printf("irq_level=%d next_delay=%f\n",
            irq_level,
-           (double)(expire_time - current_time) / get_ticks_per_sec());
+           (double)(expire_time - current_time) / NANOSECONDS_PER_SECOND);
 #endif
     s->next_transition_time = expire_time;
     if (expire_time != -1)
diff --git a/hw/timer/i8254_common.c b/hw/timer/i8254_common.c
index 07345f6..e18299a 100644
--- a/hw/timer/i8254_common.c
+++ b/hw/timer/i8254_common.c
@@ -22,6 +22,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
@@ -46,7 +47,7 @@
     int out;
 
     d = muldiv64(current_time - s->count_load_time, PIT_FREQ,
-                 get_ticks_per_sec());
+                 NANOSECONDS_PER_SECOND);
     switch (s->mode) {
     default:
     case 0:
@@ -80,7 +81,7 @@
     int period2;
 
     d = muldiv64(current_time - s->count_load_time, PIT_FREQ,
-                 get_ticks_per_sec());
+                 NANOSECONDS_PER_SECOND);
     switch (s->mode) {
     default:
     case 0:
@@ -120,7 +121,7 @@
         break;
     }
     /* convert to timer units */
-    next_time = s->count_load_time + muldiv64(next_time, get_ticks_per_sec(),
+    next_time = s->count_load_time + muldiv64(next_time, NANOSECONDS_PER_SECOND,
                                               PIT_FREQ);
     /* fix potential rounding problems */
     /* XXX: better solution: use a clock at PIT_FREQ Hz */
diff --git a/hw/timer/imx_epit.c b/hw/timer/imx_epit.c
index ffefc22..eddf348 100644
--- a/hw/timer/imx_epit.c
+++ b/hw/timer/imx_epit.c
@@ -5,25 +5,30 @@
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
  * Updated by Peter Chubb
- * Updated by Jean-Christophe Dubois
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
-#include "hw/hw.h"
-#include "qemu/bitops.h"
-#include "qemu/timer.h"
-#include "hw/ptimer.h"
-#include "hw/sysbus.h"
-#include "hw/arm/imx.h"
+#include "qemu/osdep.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/misc/imx_ccm.h"
 #include "qemu/main-loop.h"
+#include "qemu/log.h"
 
-#define TYPE_IMX_EPIT "imx.epit"
+#ifndef DEBUG_IMX_EPIT
+#define DEBUG_IMX_EPIT 0
+#endif
 
-#define DEBUG_TIMER 0
-#if DEBUG_TIMER
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_EPIT) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_EPIT, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static char const *imx_epit_reg_name(uint32_t reg)
 {
@@ -43,76 +48,17 @@
     }
 }
 
-#  define DPRINTF(fmt, args...) \
-    do { fprintf(stderr, "%s: " fmt , __func__, ##args); } while (0)
-#else
-#  define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-          do { fprintf(stderr, "%s: " fmt, __func__, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_EPIT(obj) \
-        OBJECT_CHECK(IMXEPITState, (obj), TYPE_IMX_EPIT)
-
-/*
- * EPIT: Enhanced periodic interrupt timer
- */
-
-#define CR_EN       (1 << 0)
-#define CR_ENMOD    (1 << 1)
-#define CR_OCIEN    (1 << 2)
-#define CR_RLD      (1 << 3)
-#define CR_PRESCALE_SHIFT (4)
-#define CR_PRESCALE_MASK  (0xfff)
-#define CR_SWR      (1 << 16)
-#define CR_IOVW     (1 << 17)
-#define CR_DBGEN    (1 << 18)
-#define CR_WAITEN   (1 << 19)
-#define CR_DOZEN    (1 << 20)
-#define CR_STOPEN   (1 << 21)
-#define CR_CLKSRC_SHIFT (24)
-#define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
-
-#define EPIT_TIMER_MAX  0XFFFFFFFFUL
-
 /*
  * Exact clock frequencies vary from board to board.
  * These are typical.
  */
 static const IMXClk imx_epit_clocks[] =  {
-    0,        /* 00 disabled */
-    IPG,      /* 01 ipg_clk, ~532MHz */
-    IPG,      /* 10 ipg_clk_highfreq */
-    CLK_32k,  /* 11 ipg_clk_32k -- ~32kHz */
+    CLK_NONE,      /* 00 disabled */
+    CLK_IPG,       /* 01 ipg_clk, ~532MHz */
+    CLK_IPG_HIGH,  /* 10 ipg_clk_highfreq */
+    CLK_32k,       /* 11 ipg_clk_32k -- ~32kHz */
 };
 
-typedef struct {
-    SysBusDevice busdev;
-    ptimer_state *timer_reload;
-    ptimer_state *timer_cmp;
-    MemoryRegion iomem;
-    DeviceState *ccm;
-
-    uint32_t cr;
-    uint32_t sr;
-    uint32_t lr;
-    uint32_t cmp;
-    uint32_t cnt;
-
-    uint32_t freq;
-    qemu_irq irq;
-} IMXEPITState;
-
 /*
  * Update interrupt status
  */
@@ -129,20 +75,18 @@
 {
     uint32_t clksrc;
     uint32_t prescaler;
-    uint32_t freq;
 
     clksrc = extract32(s->cr, CR_CLKSRC_SHIFT, 2);
     prescaler = 1 + extract32(s->cr, CR_PRESCALE_SHIFT, 12);
 
-    freq = imx_clock_frequency(s->ccm, imx_epit_clocks[clksrc]) / prescaler;
+    s->freq = imx_ccm_get_clock_frequency(s->ccm,
+                                imx_epit_clocks[clksrc]) / prescaler;
 
-    s->freq = freq;
+    DPRINTF("Setting ptimer frequency to %u\n", s->freq);
 
-    DPRINTF("Setting ptimer frequency to %u\n", freq);
-
-    if (freq) {
-        ptimer_set_freq(s->timer_reload, freq);
-        ptimer_set_freq(s->timer_cmp, freq);
+    if (s->freq) {
+        ptimer_set_freq(s->timer_reload, s->freq);
+        ptimer_set_freq(s->timer_cmp, s->freq);
     }
 }
 
@@ -174,18 +118,17 @@
 
 static uint32_t imx_epit_update_count(IMXEPITState *s)
 {
-     s->cnt = ptimer_get_count(s->timer_reload);
+    s->cnt = ptimer_get_count(s->timer_reload);
 
-     return s->cnt;
+    return s->cnt;
 }
 
 static uint64_t imx_epit_read(void *opaque, hwaddr offset, unsigned size)
 {
     IMXEPITState *s = IMX_EPIT(opaque);
     uint32_t reg_value = 0;
-    uint32_t reg = offset >> 2;
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* Control Register */
         reg_value = s->cr;
         break;
@@ -208,11 +151,12 @@
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_EPIT, __func__, offset);
         break;
     }
 
-    DPRINTF("(%s) = 0x%08x\n", imx_epit_reg_name(reg), reg_value);
+    DPRINTF("(%s) = 0x%08x\n", imx_epit_reg_name(offset >> 2), reg_value);
 
     return reg_value;
 }
@@ -237,12 +181,12 @@
                            unsigned size)
 {
     IMXEPITState *s = IMX_EPIT(opaque);
-    uint32_t reg = offset >> 2;
     uint64_t oldcr;
 
-    DPRINTF("(%s, value = 0x%08x)\n", imx_epit_reg_name(reg), (uint32_t)value);
+    DPRINTF("(%s, value = 0x%08x)\n", imx_epit_reg_name(offset >> 2),
+            (uint32_t)value);
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* CR */
 
         oldcr = s->cr;
@@ -318,7 +262,8 @@
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_EPIT, __func__, offset);
 
         break;
     }
@@ -333,24 +278,14 @@
     imx_epit_update_int(s);
 }
 
-void imx_timerp_create(const hwaddr addr, qemu_irq irq, DeviceState *ccm)
-{
-    IMXEPITState *pp;
-    DeviceState *dev;
-
-    dev = sysbus_create_simple(TYPE_IMX_EPIT, addr, irq);
-    pp = IMX_EPIT(dev);
-    pp->ccm = ccm;
-}
-
 static const MemoryRegionOps imx_epit_ops = {
-  .read = imx_epit_read,
-  .write = imx_epit_write,
-  .endianness = DEVICE_NATIVE_ENDIAN,
+    .read = imx_epit_read,
+    .write = imx_epit_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
 static const VMStateDescription vmstate_imx_timer_epit = {
-    .name = "imx.epit",
+    .name = TYPE_IMX_EPIT,
     .version_id = 2,
     .minimum_version_id = 2,
     .fields = (VMStateField[]) {
diff --git a/hw/timer/imx_gpt.c b/hw/timer/imx_gpt.c
index 3b31010..82bc73c 100644
--- a/hw/timer/imx_gpt.c
+++ b/hw/timer/imx_gpt.c
@@ -5,28 +5,29 @@
  * Copyright (c) 2011 NICTA Pty Ltd
  * Originally written by Hans Jiang
  * Updated by Peter Chubb
- * Updated by Jean-Christophe Dubois
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
  *
  * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
 
-#include "hw/hw.h"
-#include "qemu/bitops.h"
-#include "qemu/timer.h"
-#include "hw/ptimer.h"
-#include "hw/sysbus.h"
-#include "hw/arm/imx.h"
+#include "qemu/osdep.h"
+#include "hw/timer/imx_gpt.h"
 #include "qemu/main-loop.h"
+#include "qemu/log.h"
 
-#define TYPE_IMX_GPT "imx.gpt"
+#ifndef DEBUG_IMX_GPT
+#define DEBUG_IMX_GPT 0
+#endif
 
-/*
- * Define to 1 for debug messages
- */
-#define DEBUG_TIMER 0
-#if DEBUG_TIMER
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_IMX_GPT) { \
+            fprintf(stderr, "[%s]%s: " fmt , TYPE_IMX_GPT, \
+                                             __func__, ##args); \
+        } \
+    } while (0)
 
 static char const *imx_gpt_reg_name(uint32_t reg)
 {
@@ -56,94 +57,8 @@
     }
 }
 
-#  define DPRINTF(fmt, args...) \
-          do { printf("%s: " fmt , __func__, ##args); } while (0)
-#else
-#  define DPRINTF(fmt, args...) do {} while (0)
-#endif
-
-/*
- * Define to 1 for messages about attempts to
- * access unimplemented registers or similar.
- */
-#define DEBUG_IMPLEMENTATION 1
-#if DEBUG_IMPLEMENTATION
-#  define IPRINTF(fmt, args...) \
-          do { fprintf(stderr, "%s: " fmt, __func__, ##args); } while (0)
-#else
-#  define IPRINTF(fmt, args...) do {} while (0)
-#endif
-
-#define IMX_GPT(obj) \
-        OBJECT_CHECK(IMXGPTState, (obj), TYPE_IMX_GPT)
-/*
- * GPT : General purpose timer
- *
- * This timer counts up continuously while it is enabled, resetting itself
- * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
- * reaches the value of one of the ocrX (in periodic mode).
- */
-
-#define GPT_TIMER_MAX  0XFFFFFFFFUL
-
-/* Control register.  Not all of these bits have any effect (yet) */
-#define GPT_CR_EN     (1 << 0)  /* GPT Enable */
-#define GPT_CR_ENMOD  (1 << 1)  /* GPT Enable Mode */
-#define GPT_CR_DBGEN  (1 << 2)  /* GPT Debug mode enable */
-#define GPT_CR_WAITEN (1 << 3)  /* GPT Wait Mode Enable  */
-#define GPT_CR_DOZEN  (1 << 4)  /* GPT Doze mode enable */
-#define GPT_CR_STOPEN (1 << 5)  /* GPT Stop Mode Enable */
-#define GPT_CR_CLKSRC_SHIFT (6)
-#define GPT_CR_CLKSRC_MASK  (0x7)
-
-#define GPT_CR_FRR    (1 << 9)  /* Freerun or Restart */
-#define GPT_CR_SWR    (1 << 15) /* Software Reset */
-#define GPT_CR_IM1    (3 << 16) /* Input capture channel 1 mode (2 bits) */
-#define GPT_CR_IM2    (3 << 18) /* Input capture channel 2 mode (2 bits) */
-#define GPT_CR_OM1    (7 << 20) /* Output Compare Channel 1 Mode (3 bits) */
-#define GPT_CR_OM2    (7 << 23) /* Output Compare Channel 2 Mode (3 bits) */
-#define GPT_CR_OM3    (7 << 26) /* Output Compare Channel 3 Mode (3 bits) */
-#define GPT_CR_FO1    (1 << 29) /* Force Output Compare Channel 1 */
-#define GPT_CR_FO2    (1 << 30) /* Force Output Compare Channel 2 */
-#define GPT_CR_FO3    (1 << 31) /* Force Output Compare Channel 3 */
-
-#define GPT_SR_OF1  (1 << 0)
-#define GPT_SR_OF2  (1 << 1)
-#define GPT_SR_OF3  (1 << 2)
-#define GPT_SR_ROV  (1 << 5)
-
-#define GPT_IR_OF1IE  (1 << 0)
-#define GPT_IR_OF2IE  (1 << 1)
-#define GPT_IR_OF3IE  (1 << 2)
-#define GPT_IR_ROVIE  (1 << 5)
-
-typedef struct {
-    SysBusDevice busdev;
-    ptimer_state *timer;
-    MemoryRegion iomem;
-    DeviceState *ccm;
-
-    uint32_t cr;
-    uint32_t pr;
-    uint32_t sr;
-    uint32_t ir;
-    uint32_t ocr1;
-    uint32_t ocr2;
-    uint32_t ocr3;
-    uint32_t icr1;
-    uint32_t icr2;
-    uint32_t cnt;
-
-    uint32_t next_timeout;
-    uint32_t next_int;
-
-    uint32_t freq;
-
-    qemu_irq irq;
-} IMXGPTState;
-
 static const VMStateDescription vmstate_imx_timer_gpt = {
-    .name = "imx.gpt",
+    .name = TYPE_IMX_GPT,
     .version_id = 3,
     .minimum_version_id = 3,
     .fields = (VMStateField[]) {
@@ -165,28 +80,50 @@
     }
 };
 
-static const IMXClk imx_gpt_clocks[] = {
-    NOCLK,    /* 000 No clock source */
-    IPG,      /* 001 ipg_clk, 532MHz*/
-    IPG,      /* 010 ipg_clk_highfreq */
-    NOCLK,    /* 011 not defined */
-    CLK_32k,  /* 100 ipg_clk_32k */
-    NOCLK,    /* 101 not defined */
-    NOCLK,    /* 110 not defined */
-    NOCLK,    /* 111 not defined */
+static const IMXClk imx25_gpt_clocks[] = {
+    CLK_NONE,      /* 000 No clock source */
+    CLK_IPG,       /* 001 ipg_clk, 532MHz*/
+    CLK_IPG_HIGH,  /* 010 ipg_clk_highfreq */
+    CLK_NONE,      /* 011 not defined */
+    CLK_32k,       /* 100 ipg_clk_32k */
+    CLK_32k,       /* 101 ipg_clk_32k */
+    CLK_32k,       /* 110 ipg_clk_32k */
+    CLK_32k,       /* 111 ipg_clk_32k */
+};
+
+static const IMXClk imx31_gpt_clocks[] = {
+    CLK_NONE,      /* 000 No clock source */
+    CLK_IPG,       /* 001 ipg_clk, 532MHz*/
+    CLK_IPG_HIGH,  /* 010 ipg_clk_highfreq */
+    CLK_NONE,      /* 011 not defined */
+    CLK_32k,       /* 100 ipg_clk_32k */
+    CLK_NONE,      /* 101 not defined */
+    CLK_NONE,      /* 110 not defined */
+    CLK_NONE,      /* 111 not defined */
+};
+
+static const IMXClk imx6_gpt_clocks[] = {
+    CLK_NONE,      /* 000 No clock source */
+    CLK_IPG,       /* 001 ipg_clk, 532MHz*/
+    CLK_IPG_HIGH,  /* 010 ipg_clk_highfreq */
+    CLK_EXT,       /* 011 External clock */
+    CLK_32k,       /* 100 ipg_clk_32k */
+    CLK_HIGH_DIV,  /* 101 reference clock / 8 */
+    CLK_NONE,      /* 110 not defined */
+    CLK_HIGH,      /* 111 reference clock */
 };
 
 static void imx_gpt_set_freq(IMXGPTState *s)
 {
     uint32_t clksrc = extract32(s->cr, GPT_CR_CLKSRC_SHIFT, 3);
-    uint32_t freq = imx_clock_frequency(s->ccm, imx_gpt_clocks[clksrc])
-                                                / (1 + s->pr);
-    s->freq = freq;
 
-    DPRINTF("Setting clksrc %d to frequency %d\n", clksrc, freq);
+    s->freq = imx_ccm_get_clock_frequency(s->ccm,
+                                          s->clocks[clksrc]) / (1 + s->pr);
 
-    if (freq) {
-        ptimer_set_freq(s->timer, freq);
+    DPRINTF("Setting clksrc %d to frequency %d\n", clksrc, s->freq);
+
+    if (s->freq) {
+        ptimer_set_freq(s->timer, s->freq);
     }
 }
 
@@ -207,7 +144,7 @@
 }
 
 static inline uint32_t imx_gpt_find_limit(uint32_t count, uint32_t reg,
-                                             uint32_t timeout)
+                                          uint32_t timeout)
 {
     if ((count < reg) && (timeout > reg)) {
         timeout = reg;
@@ -219,7 +156,7 @@
 static void imx_gpt_compute_next_timeout(IMXGPTState *s, bool event)
 {
     uint32_t timeout = GPT_TIMER_MAX;
-    uint32_t count = 0;
+    uint32_t count;
     long long limit;
 
     if (!(s->cr & GPT_CR_EN)) {
@@ -227,20 +164,23 @@
         return;
     }
 
+    /* update the count */
+    count = imx_gpt_update_count(s);
+
     if (event) {
-        /* This is a timer event  */
-
-        if ((s->cr & GPT_CR_FRR)  && (s->next_timeout != GPT_TIMER_MAX)) {
-            /*
-             * if we are in free running mode and we have not reached
-             * the GPT_TIMER_MAX limit, then update the count
+        /*
+         * This is an event (the ptimer reached 0 and stopped), and the
+         * timer counter is now equal to s->next_timeout.
+         */
+        if (!(s->cr & GPT_CR_FRR) && (count == s->ocr1)) {
+            /* We are in restart mode and we crossed the compare channel 1
+             * value. We need to reset the counter to 0.
              */
-            count = imx_gpt_update_count(s);
+            count = s->cnt = s->next_timeout = 0;
+        } else if (count == GPT_TIMER_MAX) {
+            /* We reached GPT_TIMER_MAX so we need to rollover */
+            count = s->cnt = s->next_timeout = 0;
         }
-    } else {
-        /* not a timer event, then just update the count */
-
-        count = imx_gpt_update_count(s);
     }
 
     /* now, find the next timeout related to count */
@@ -298,9 +238,8 @@
 {
     IMXGPTState *s = IMX_GPT(opaque);
     uint32_t reg_value = 0;
-    uint32_t reg = offset >> 2;
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0: /* Control Register */
         reg_value = s->cr;
         break;
@@ -330,12 +269,14 @@
         break;
 
     case 7: /* input Capture Register 1 */
-        qemu_log_mask(LOG_UNIMP, "icr1 feature is not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: icr1 feature is not implemented\n",
+                      TYPE_IMX_GPT, __func__);
         reg_value = s->icr1;
         break;
 
     case 8: /* input Capture Register 2 */
-        qemu_log_mask(LOG_UNIMP, "icr2 feature is not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "[%s]%s: icr2 feature is not implemented\n",
+                      TYPE_IMX_GPT, __func__);
         reg_value = s->icr2;
         break;
 
@@ -345,11 +286,12 @@
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPT, __func__, offset);
         break;
     }
 
-    DPRINTF("(%s) = 0x%08x\n", imx_gpt_reg_name(reg), reg_value);
+    DPRINTF("(%s) = 0x%08x\n", imx_gpt_reg_name(offset >> 2), reg_value);
 
     return reg_value;
 }
@@ -396,12 +338,11 @@
 {
     IMXGPTState *s = IMX_GPT(opaque);
     uint32_t oldreg;
-    uint32_t reg = offset >> 2;
 
-    DPRINTF("(%s, value = 0x%08x)\n", imx_gpt_reg_name(reg),
+    DPRINTF("(%s, value = 0x%08x)\n", imx_gpt_reg_name(offset >> 2),
             (uint32_t)value);
 
-    switch (reg) {
+    switch (offset >> 2) {
     case 0:
         oldreg = s->cr;
         s->cr = value & ~0x7c14;
@@ -477,7 +418,8 @@
         break;
 
     default:
-        IPRINTF("Bad offset %x\n", reg);
+        qemu_log_mask(LOG_GUEST_ERROR, "[%s]%s: Bad register at offset 0x%"
+                      HWADDR_PRIx "\n", TYPE_IMX_GPT, __func__, offset);
         break;
     }
 }
@@ -522,16 +464,6 @@
     s->timer = ptimer_init(bh);
 }
 
-void imx_timerg_create(const hwaddr addr, qemu_irq irq, DeviceState *ccm)
-{
-    IMXGPTState *pp;
-    DeviceState *dev;
-
-    dev = sysbus_create_simple(TYPE_IMX_GPT, addr, irq);
-    pp = IMX_GPT(dev);
-    pp->ccm = ccm;
-}
-
 static void imx_gpt_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -542,16 +474,52 @@
     dc->desc = "i.MX general timer";
 }
 
-static const TypeInfo imx_gpt_info = {
-    .name = TYPE_IMX_GPT,
+static void imx25_gpt_init(Object *obj)
+{
+    IMXGPTState *s = IMX_GPT(obj);
+
+    s->clocks = imx25_gpt_clocks;
+}
+
+static void imx31_gpt_init(Object *obj)
+{
+    IMXGPTState *s = IMX_GPT(obj);
+
+    s->clocks = imx31_gpt_clocks;
+}
+
+static void imx6_gpt_init(Object *obj)
+{
+    IMXGPTState *s = IMX_GPT(obj);
+
+    s->clocks = imx6_gpt_clocks;
+}
+
+static const TypeInfo imx25_gpt_info = {
+    .name = TYPE_IMX25_GPT,
     .parent = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IMXGPTState),
+    .instance_init = imx25_gpt_init,
     .class_init = imx_gpt_class_init,
 };
 
+static const TypeInfo imx31_gpt_info = {
+    .name = TYPE_IMX31_GPT,
+    .parent = TYPE_IMX25_GPT,
+    .instance_init = imx31_gpt_init,
+};
+
+static const TypeInfo imx6_gpt_info = {
+    .name = TYPE_IMX6_GPT,
+    .parent = TYPE_IMX25_GPT,
+    .instance_init = imx6_gpt_init,
+};
+
 static void imx_gpt_register_types(void)
 {
-    type_register_static(&imx_gpt_info);
+    type_register_static(&imx25_gpt_info);
+    type_register_static(&imx31_gpt_info);
+    type_register_static(&imx6_gpt_info);
 }
 
 type_init(imx_gpt_register_types)
diff --git a/hw/timer/lm32_timer.c b/hw/timer/lm32_timer.c
index d2ab1e7..e45a65b 100644
--- a/hw/timer/lm32_timer.c
+++ b/hw/timer/lm32_timer.c
@@ -21,6 +21,7 @@
  *   http://www.latticesemi.com/documents/mico32timer.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "trace.h"
@@ -175,21 +176,26 @@
     ptimer_stop(s->ptimer);
 }
 
-static int lm32_timer_init(SysBusDevice *dev)
+static void lm32_timer_init(Object *obj)
 {
-    LM32TimerState *s = LM32_TIMER(dev);
+    LM32TimerState *s = LM32_TIMER(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     sysbus_init_irq(dev, &s->irq);
 
     s->bh = qemu_bh_new(timer_hit, s);
     s->ptimer = ptimer_init(s->bh);
-    ptimer_set_freq(s->ptimer, s->freq_hz);
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &timer_ops, s,
+    memory_region_init_io(&s->iomem, obj, &timer_ops, s,
                           "timer", R_MAX * 4);
     sysbus_init_mmio(dev, &s->iomem);
+}
 
-    return 0;
+static void lm32_timer_realize(DeviceState *dev, Error **errp)
+{
+    LM32TimerState *s = LM32_TIMER(dev);
+
+    ptimer_set_freq(s->ptimer, s->freq_hz);
 }
 
 static const VMStateDescription vmstate_lm32_timer = {
@@ -212,9 +218,8 @@
 static void lm32_timer_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = lm32_timer_init;
+    dc->realize = lm32_timer_realize;
     dc->reset = timer_reset;
     dc->vmsd = &vmstate_lm32_timer;
     dc->props = lm32_timer_properties;
@@ -224,6 +229,7 @@
     .name          = TYPE_LM32_TIMER,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(LM32TimerState),
+    .instance_init = lm32_timer_init,
     .class_init    = lm32_timer_class_init,
 };
 
diff --git a/hw/timer/m48t59.c b/hw/timer/m48t59.c
index 31509d5..e46ca88 100644
--- a/hw/timer/m48t59.c
+++ b/hw/timer/m48t59.c
@@ -2,6 +2,7 @@
  * QEMU M48T59 and M48T08 NVRAM emulation for PPC PREP and Sparc platforms
  *
  * Copyright (c) 2003-2005, 2007 Jocelyn Mayer
+ * Copyright (c) 2013 Hervé Poussineau
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -21,13 +22,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/timer/m48t59.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "hw/sysbus.h"
 #include "hw/isa/isa.h"
 #include "exec/address-spaces.h"
+#include "qemu/bcd.h"
 
 //#define DEBUG_NVRAM
 
@@ -37,12 +41,35 @@
 #define NVRAM_PRINTF(fmt, ...) do { } while (0)
 #endif
 
+#define TYPE_M48TXX_SYS_BUS "sysbus-m48txx"
+#define M48TXX_SYS_BUS_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(M48txxSysBusDeviceClass, (obj), TYPE_M48TXX_SYS_BUS)
+#define M48TXX_SYS_BUS_CLASS(klass) \
+    OBJECT_CLASS_CHECK(M48txxSysBusDeviceClass, (klass), TYPE_M48TXX_SYS_BUS)
+#define M48TXX_SYS_BUS(obj) \
+    OBJECT_CHECK(M48txxSysBusState, (obj), TYPE_M48TXX_SYS_BUS)
+
+#define TYPE_M48TXX_ISA "isa-m48txx"
+#define M48TXX_ISA_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(M48txxISADeviceClass, (obj), TYPE_M48TXX_ISA)
+#define M48TXX_ISA_CLASS(klass) \
+    OBJECT_CLASS_CHECK(M48txxISADeviceClass, (klass), TYPE_M48TXX_ISA)
+#define M48TXX_ISA(obj) \
+    OBJECT_CHECK(M48txxISAState, (obj), TYPE_M48TXX_ISA)
+
 /*
  * The M48T02, M48T08 and M48T59 chips are very similar. The newer '59 has
  * alarm and a watchdog timer and related control registers. In the
  * PPC platform there is also a nvram lock function.
  */
 
+typedef struct M48txxInfo {
+    const char *isa_name;
+    const char *sysbus_name;
+    uint32_t model; /* 2 = m48t02, 8 = m48t08, 59 = m48t59 */
+    uint32_t size;
+} M48txxInfo;
+
 /*
  * Chipset docs:
  * http://www.st.com/stonline/products/literature/ds/2410/m48t02.pdf
@@ -50,12 +77,12 @@
  * http://www.st.com/stonline/products/literature/od/7001/m48t59y.pdf
  */
 
-struct M48t59State {
+typedef struct M48t59State {
     /* Hardware parameters */
     qemu_irq IRQ;
     MemoryRegion iomem;
-    uint32_t io_base;
     uint32_t size;
+    int32_t base_year;
     /* RTC management */
     time_t   time_offset;
     time_t   stop_time;
@@ -70,28 +97,51 @@
     /* NVRAM storage */
     uint16_t addr;
     uint8_t  lock;
+} M48t59State;
+
+typedef struct M48txxISAState {
+    ISADevice parent_obj;
+    M48t59State state;
+    uint32_t io_base;
+    MemoryRegion io;
+} M48txxISAState;
+
+typedef struct M48txxISADeviceClass {
+    ISADeviceClass parent_class;
+    M48txxInfo info;
+} M48txxISADeviceClass;
+
+typedef struct M48txxSysBusState {
+    SysBusDevice parent_obj;
+    M48t59State state;
+    MemoryRegion io;
+} M48txxSysBusState;
+
+typedef struct M48txxSysBusDeviceClass {
+    SysBusDeviceClass parent_class;
+    M48txxInfo info;
+} M48txxSysBusDeviceClass;
+
+static M48txxInfo m48txx_info[] = {
+    {
+        .sysbus_name = "sysbus-m48t02",
+        .model = 2,
+        .size = 0x800,
+    },{
+        .sysbus_name = "sysbus-m48t08",
+        .model = 8,
+        .size = 0x2000,
+    },{
+        .sysbus_name = "sysbus-m48t59",
+        .model = 59,
+        .size = 0x2000,
+    },{
+        .isa_name = "isa-m48t59",
+        .model = 59,
+        .size = 0x2000,
+    }
 };
 
-#define TYPE_ISA_M48T59 "m48t59_isa"
-#define ISA_M48T59(obj) \
-    OBJECT_CHECK(M48t59ISAState, (obj), TYPE_ISA_M48T59)
-
-typedef struct M48t59ISAState {
-    ISADevice parent_obj;
-
-    M48t59State state;
-    MemoryRegion io;
-} M48t59ISAState;
-
-#define SYSBUS_M48T59(obj) \
-    OBJECT_CHECK(M48t59SysBusState, (obj), TYPE_SYSBUS_M48T59)
-
-typedef struct M48t59SysBusState {
-    SysBusDevice parent_obj;
-
-    M48t59State state;
-    MemoryRegion io;
-} M48t59SysBusState;
 
 /* Fake timer functions */
 
@@ -198,9 +248,8 @@
 }
 
 /* Direct access to NVRAM */
-void m48t59_write (void *opaque, uint32_t addr, uint32_t val)
+static void m48t59_write(M48t59State *NVRAM, uint32_t addr, uint32_t val)
 {
-    M48t59State *NVRAM = opaque;
     struct tm tm;
     int tmp;
 
@@ -346,11 +395,7 @@
 	tmp = from_bcd(val);
 	if (tmp >= 0 && tmp <= 99) {
 	    get_time(NVRAM, &tm);
-            if (NVRAM->model == 8) {
-                tm.tm_year = from_bcd(val) + 68; // Base year is 1968
-            } else {
-                tm.tm_year = from_bcd(val);
-            }
+            tm.tm_year = from_bcd(val) + NVRAM->base_year - 1900;
 	    set_time(NVRAM, &tm);
 	}
         break;
@@ -368,9 +413,8 @@
     }
 }
 
-uint32_t m48t59_read (void *opaque, uint32_t addr)
+static uint32_t m48t59_read(M48t59State *NVRAM, uint32_t addr)
 {
-    M48t59State *NVRAM = opaque;
     struct tm tm;
     uint32_t retval = 0xFF;
 
@@ -453,11 +497,7 @@
     case 0x07FF:
         /* year */
         get_time(NVRAM, &tm);
-        if (NVRAM->model == 8) {
-            retval = to_bcd(tm.tm_year - 68); // Base year is 1968
-        } else {
-            retval = to_bcd(tm.tm_year);
-        }
+        retval = to_bcd((tm.tm_year + 1900 - NVRAM->base_year) % 100);
         break;
     default:
         /* Check lock registers state */
@@ -477,10 +517,8 @@
     return retval;
 }
 
-void m48t59_toggle_lock (void *opaque, int lock)
+static void m48t59_toggle_lock(M48t59State *NVRAM, int lock)
 {
-    M48t59State *NVRAM = opaque;
-
     NVRAM->lock ^= 1 << lock;
 }
 
@@ -555,10 +593,8 @@
 static uint32_t nvram_readb (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
-    uint32_t retval;
 
-    retval = m48t59_read(NVRAM, addr);
-    return retval;
+    return m48t59_read(NVRAM, addr);
 }
 
 static uint32_t nvram_readw (void *opaque, hwaddr addr)
@@ -616,7 +652,7 @@
 
 static void m48t59_reset_isa(DeviceState *d)
 {
-    M48t59ISAState *isa = ISA_M48T59(d);
+    M48txxISAState *isa = M48TXX_ISA(d);
     M48t59State *NVRAM = &isa->state;
 
     m48t59_reset_common(NVRAM);
@@ -624,7 +660,7 @@
 
 static void m48t59_reset_sysbus(DeviceState *d)
 {
-    M48t59SysBusState *sys = SYSBUS_M48T59(d);
+    M48txxSysBusState *sys = M48TXX_SYS_BUS(d);
     M48t59State *NVRAM = &sys->state;
 
     m48t59_reset_common(NVRAM);
@@ -641,58 +677,63 @@
 };
 
 /* Initialisation routine */
-M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
-                         uint32_t io_base, uint16_t size, int model)
+Nvram *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
+                   uint32_t io_base, uint16_t size, int base_year,
+                   int model)
 {
     DeviceState *dev;
     SysBusDevice *s;
-    M48t59SysBusState *d;
-    M48t59State *state;
+    int i;
 
-    dev = qdev_create(NULL, TYPE_SYSBUS_M48T59);
-    qdev_prop_set_uint32(dev, "model", model);
-    qdev_prop_set_uint32(dev, "size", size);
-    qdev_prop_set_uint32(dev, "io_base", io_base);
-    qdev_init_nofail(dev);
-    s = SYS_BUS_DEVICE(dev);
-    d = SYSBUS_M48T59(dev);
-    state = &d->state;
-    sysbus_connect_irq(s, 0, IRQ);
-    memory_region_init_io(&d->io, OBJECT(d), &m48t59_io_ops, state,
-                          "m48t59", 4);
-    if (io_base != 0) {
-        memory_region_add_subregion(get_system_io(), io_base, &d->io);
-    }
-    if (mem_base != 0) {
-        sysbus_mmio_map(s, 0, mem_base);
+    for (i = 0; i < ARRAY_SIZE(m48txx_info); i++) {
+        if (!m48txx_info[i].sysbus_name ||
+            m48txx_info[i].size != size ||
+            m48txx_info[i].model != model) {
+            continue;
+        }
+
+        dev = qdev_create(NULL, m48txx_info[i].sysbus_name);
+        qdev_prop_set_int32(dev, "base-year", base_year);
+        qdev_init_nofail(dev);
+        s = SYS_BUS_DEVICE(dev);
+        sysbus_connect_irq(s, 0, IRQ);
+        if (io_base != 0) {
+            memory_region_add_subregion(get_system_io(), io_base,
+                                        sysbus_mmio_get_region(s, 1));
+        }
+        if (mem_base != 0) {
+            sysbus_mmio_map(s, 0, mem_base);
+        }
+
+        return NVRAM(s);
     }
 
-    return state;
+    assert(false);
+    return NULL;
 }
 
-M48t59State *m48t59_init_isa(ISABus *bus, uint32_t io_base, uint16_t size,
-                             int model)
+Nvram *m48t59_init_isa(ISABus *bus, uint32_t io_base, uint16_t size,
+                       int base_year, int model)
 {
-    M48t59ISAState *d;
-    ISADevice *isadev;
     DeviceState *dev;
-    M48t59State *s;
+    int i;
 
-    isadev = isa_create(bus, TYPE_ISA_M48T59);
-    dev = DEVICE(isadev);
-    qdev_prop_set_uint32(dev, "model", model);
-    qdev_prop_set_uint32(dev, "size", size);
-    qdev_prop_set_uint32(dev, "io_base", io_base);
-    qdev_init_nofail(dev);
-    d = ISA_M48T59(isadev);
-    s = &d->state;
+    for (i = 0; i < ARRAY_SIZE(m48txx_info); i++) {
+        if (!m48txx_info[i].isa_name ||
+            m48txx_info[i].size != size ||
+            m48txx_info[i].model != model) {
+            continue;
+        }
 
-    memory_region_init_io(&d->io, OBJECT(d), &m48t59_io_ops, s, "m48t59", 4);
-    if (io_base != 0) {
-        isa_register_ioport(isadev, &d->io, io_base);
+        dev = DEVICE(isa_create(bus, m48txx_info[i].isa_name));
+        qdev_prop_set_uint32(dev, "iobase", io_base);
+        qdev_prop_set_int32(dev, "base-year", base_year);
+        qdev_init_nofail(dev);
+        return NVRAM(dev);
     }
 
-    return s;
+    assert(false);
+    return NULL;
 }
 
 static void m48t59_realize_common(M48t59State *s, Error **errp)
@@ -709,25 +750,38 @@
 
 static void m48t59_isa_realize(DeviceState *dev, Error **errp)
 {
+    M48txxISADeviceClass *u = M48TXX_ISA_GET_CLASS(dev);
     ISADevice *isadev = ISA_DEVICE(dev);
-    M48t59ISAState *d = ISA_M48T59(dev);
+    M48txxISAState *d = M48TXX_ISA(dev);
     M48t59State *s = &d->state;
 
+    s->model = u->info.model;
+    s->size = u->info.size;
     isa_init_irq(isadev, &s->IRQ, 8);
     m48t59_realize_common(s, errp);
+    memory_region_init_io(&d->io, OBJECT(dev), &m48t59_io_ops, s, "m48t59", 4);
+    if (d->io_base != 0) {
+        isa_register_ioport(isadev, &d->io, d->io_base);
+    }
 }
 
 static int m48t59_init1(SysBusDevice *dev)
 {
-    M48t59SysBusState *d = SYSBUS_M48T59(dev);
+    M48txxSysBusDeviceClass *u = M48TXX_SYS_BUS_GET_CLASS(dev);
+    M48txxSysBusState *d = M48TXX_SYS_BUS(dev);
+    Object *o = OBJECT(dev);
     M48t59State *s = &d->state;
     Error *err = NULL;
 
+    s->model = u->info.model;
+    s->size = u->info.size;
     sysbus_init_irq(dev, &s->IRQ);
 
-    memory_region_init_io(&s->iomem, OBJECT(d), &nvram_ops, s,
-                          "m48t59.nvram", s->size);
+    memory_region_init_io(&s->iomem, o, &nvram_ops, s, "m48t59.nvram",
+                          s->size);
+    memory_region_init_io(&d->io, o, &m48t59_io_ops, s, "m48t59", 4);
     sysbus_init_mmio(dev, &s->iomem);
+    sysbus_init_mmio(dev, &d->io);
     m48t59_realize_common(s, &err);
     if (err != NULL) {
         error_free(err);
@@ -737,59 +791,157 @@
     return 0;
 }
 
+static uint32_t m48txx_isa_read(Nvram *obj, uint32_t addr)
+{
+    M48txxISAState *d = M48TXX_ISA(obj);
+    return m48t59_read(&d->state, addr);
+}
+
+static void m48txx_isa_write(Nvram *obj, uint32_t addr, uint32_t val)
+{
+    M48txxISAState *d = M48TXX_ISA(obj);
+    m48t59_write(&d->state, addr, val);
+}
+
+static void m48txx_isa_toggle_lock(Nvram *obj, int lock)
+{
+    M48txxISAState *d = M48TXX_ISA(obj);
+    m48t59_toggle_lock(&d->state, lock);
+}
+
 static Property m48t59_isa_properties[] = {
-    DEFINE_PROP_UINT32("size",    M48t59ISAState, state.size,    -1),
-    DEFINE_PROP_UINT32("model",   M48t59ISAState, state.model,   -1),
-    DEFINE_PROP_UINT32("io_base", M48t59ISAState, state.io_base,  0),
+    DEFINE_PROP_INT32("base-year", M48txxISAState, state.base_year, 0),
+    DEFINE_PROP_UINT32("iobase", M48txxISAState, io_base, 0x74),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void m48t59_isa_class_init(ObjectClass *klass, void *data)
+static void m48txx_isa_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    NvramClass *nc = NVRAM_CLASS(klass);
 
     dc->realize = m48t59_isa_realize;
     dc->reset = m48t59_reset_isa;
     dc->props = m48t59_isa_properties;
-    /* Reason: needs to be wired up by m48t59_init_isa() */
-    dc->cannot_instantiate_with_device_add_yet = true;
+    nc->read = m48txx_isa_read;
+    nc->write = m48txx_isa_write;
+    nc->toggle_lock = m48txx_isa_toggle_lock;
 }
 
-static const TypeInfo m48t59_isa_info = {
-    .name          = TYPE_ISA_M48T59,
-    .parent        = TYPE_ISA_DEVICE,
-    .instance_size = sizeof(M48t59ISAState),
-    .class_init    = m48t59_isa_class_init,
-};
+static void m48txx_isa_concrete_class_init(ObjectClass *klass, void *data)
+{
+    M48txxISADeviceClass *u = M48TXX_ISA_CLASS(klass);
+    M48txxInfo *info = data;
 
-static Property m48t59_properties[] = {
-    DEFINE_PROP_UINT32("size",    M48t59SysBusState, state.size,    -1),
-    DEFINE_PROP_UINT32("model",   M48t59SysBusState, state.model,   -1),
-    DEFINE_PROP_UINT32("io_base", M48t59SysBusState, state.io_base,  0),
+    u->info = *info;
+}
+
+static uint32_t m48txx_sysbus_read(Nvram *obj, uint32_t addr)
+{
+    M48txxSysBusState *d = M48TXX_SYS_BUS(obj);
+    return m48t59_read(&d->state, addr);
+}
+
+static void m48txx_sysbus_write(Nvram *obj, uint32_t addr, uint32_t val)
+{
+    M48txxSysBusState *d = M48TXX_SYS_BUS(obj);
+    m48t59_write(&d->state, addr, val);
+}
+
+static void m48txx_sysbus_toggle_lock(Nvram *obj, int lock)
+{
+    M48txxSysBusState *d = M48TXX_SYS_BUS(obj);
+    m48t59_toggle_lock(&d->state, lock);
+}
+
+static Property m48t59_sysbus_properties[] = {
+    DEFINE_PROP_INT32("base-year", M48txxSysBusState, state.base_year, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void m48t59_class_init(ObjectClass *klass, void *data)
+static void m48txx_sysbus_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+    NvramClass *nc = NVRAM_CLASS(klass);
 
     k->init = m48t59_init1;
     dc->reset = m48t59_reset_sysbus;
-    dc->props = m48t59_properties;
+    dc->props = m48t59_sysbus_properties;
+    nc->read = m48txx_sysbus_read;
+    nc->write = m48txx_sysbus_write;
+    nc->toggle_lock = m48txx_sysbus_toggle_lock;
 }
 
-static const TypeInfo m48t59_info = {
-    .name          = TYPE_SYSBUS_M48T59,
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(M48t59SysBusState),
-    .class_init    = m48t59_class_init,
+static void m48txx_sysbus_concrete_class_init(ObjectClass *klass, void *data)
+{
+    M48txxSysBusDeviceClass *u = M48TXX_SYS_BUS_CLASS(klass);
+    M48txxInfo *info = data;
+
+    u->info = *info;
+}
+
+static const TypeInfo nvram_info = {
+    .name = TYPE_NVRAM,
+    .parent = TYPE_INTERFACE,
+    .class_size = sizeof(NvramClass),
+};
+
+static const TypeInfo m48txx_sysbus_type_info = {
+    .name = TYPE_M48TXX_SYS_BUS,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(M48txxSysBusState),
+    .abstract = true,
+    .class_init = m48txx_sysbus_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_NVRAM },
+        { }
+    }
+};
+
+static const TypeInfo m48txx_isa_type_info = {
+    .name = TYPE_M48TXX_ISA,
+    .parent = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(M48txxISAState),
+    .abstract = true,
+    .class_init = m48txx_isa_class_init,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_NVRAM },
+        { }
+    }
 };
 
 static void m48t59_register_types(void)
 {
-    type_register_static(&m48t59_info);
-    type_register_static(&m48t59_isa_info);
+    TypeInfo sysbus_type_info = {
+        .parent = TYPE_M48TXX_SYS_BUS,
+        .class_size = sizeof(M48txxSysBusDeviceClass),
+        .class_init = m48txx_sysbus_concrete_class_init,
+    };
+    TypeInfo isa_type_info = {
+        .parent = TYPE_M48TXX_ISA,
+        .class_size = sizeof(M48txxISADeviceClass),
+        .class_init = m48txx_isa_concrete_class_init,
+    };
+    int i;
+
+    type_register_static(&nvram_info);
+    type_register_static(&m48txx_sysbus_type_info);
+    type_register_static(&m48txx_isa_type_info);
+
+    for (i = 0; i < ARRAY_SIZE(m48txx_info); i++) {
+        if (m48txx_info[i].sysbus_name) {
+            sysbus_type_info.name = m48txx_info[i].sysbus_name;
+            sysbus_type_info.class_data = &m48txx_info[i];
+            type_register(&sysbus_type_info);
+        }
+
+        if (m48txx_info[i].isa_name) {
+            isa_type_info.name = m48txx_info[i].isa_name;
+            isa_type_info.class_data = &m48txx_info[i];
+            type_register(&isa_type_info);
+        }
+    }
 }
 
 type_init(m48t59_register_types)
diff --git a/hw/timer/mc146818rtc.c b/hw/timer/mc146818rtc.c
index f18d128..ea625f2 100644
--- a/hw/timer/mc146818rtc.c
+++ b/hw/timer/mc146818rtc.c
@@ -21,6 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/bcd.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
@@ -48,7 +51,6 @@
 # define DPRINTF_C(format, ...)      do { } while (0)
 #endif
 
-#define NSEC_PER_SEC    1000000000LL
 #define SEC_PER_MIN     60
 #define MIN_PER_HOUR    60
 #define SEC_PER_HOUR    3600
@@ -57,7 +59,7 @@
 
 #define RTC_REINJECT_ON_ACK_COUNT 20
 #define RTC_CLOCK_RATE            32768
-#define UIP_HOLD_LENGTH           (8 * NSEC_PER_SEC / 32768)
+#define UIP_HOLD_LENGTH           (8 * NANOSECONDS_PER_SECOND / 32768)
 
 #define MC146818_RTC(obj) OBJECT_CHECK(RTCState, (obj), TYPE_MC146818_RTC)
 
@@ -103,12 +105,10 @@
 
 static uint64_t get_guest_rtc_ns(RTCState *s)
 {
-    uint64_t guest_rtc;
     uint64_t guest_clock = qemu_clock_get_ns(rtc_clock);
 
-    guest_rtc = s->base_rtc * NSEC_PER_SEC
-                 + guest_clock - s->last_update + s->offset;
-    return guest_rtc;
+    return s->base_rtc * NANOSECONDS_PER_SECOND +
+        guest_clock - s->last_update + s->offset;
 }
 
 #ifdef TARGET_I386
@@ -120,7 +120,7 @@
         /* divide each RTC interval to 2 - 8 smaller intervals */
         int c = MIN(s->irq_coalesced, 7) + 1; 
         int64_t next_clock = qemu_clock_get_ns(rtc_clock) +
-            muldiv64(s->period / c, get_ticks_per_sec(), RTC_CLOCK_RATE);
+            muldiv64(s->period / c, NANOSECONDS_PER_SECOND, RTC_CLOCK_RATE);
         timer_mod(s->coalesced_timer, next_clock);
     }
 }
@@ -166,10 +166,12 @@
         s->period = period;
 #endif
         /* compute 32 khz clock */
-        cur_clock = muldiv64(current_time, RTC_CLOCK_RATE, get_ticks_per_sec());
+        cur_clock =
+            muldiv64(current_time, RTC_CLOCK_RATE, NANOSECONDS_PER_SECOND);
+
         next_irq_clock = (cur_clock & ~(period - 1)) + period;
-        s->next_periodic_time =
-            muldiv64(next_irq_clock, get_ticks_per_sec(), RTC_CLOCK_RATE) + 1;
+        s->next_periodic_time = muldiv64(next_irq_clock, NANOSECONDS_PER_SECOND,
+                                         RTC_CLOCK_RATE) + 1;
         timer_mod(s->periodic_timer, s->next_periodic_time);
     } else {
 #ifdef TARGET_I386
@@ -232,16 +234,17 @@
         return;
     }
 
-    guest_nsec = get_guest_rtc_ns(s) % NSEC_PER_SEC;
+    guest_nsec = get_guest_rtc_ns(s) % NANOSECONDS_PER_SECOND;
     /* if UF is clear, reprogram to next second */
     next_update_time = qemu_clock_get_ns(rtc_clock)
-        + NSEC_PER_SEC - guest_nsec;
+        + NANOSECONDS_PER_SECOND - guest_nsec;
 
     /* Compute time of next alarm.  One second is already accounted
      * for in next_update_time.
      */
     next_alarm_sec = get_next_alarm(s);
-    s->next_alarm_time = next_update_time + (next_alarm_sec - 1) * NSEC_PER_SEC;
+    s->next_alarm_time = next_update_time +
+                         (next_alarm_sec - 1) * NANOSECONDS_PER_SECOND;
 
     if (s->cmos_data[RTC_REG_C] & REG_C_UF) {
         /* UF is set, but AF is clear.  Program the timer to target
@@ -457,7 +460,7 @@
                 /* if disabling set mode, update the time */
                 if ((s->cmos_data[RTC_REG_B] & REG_B_SET) &&
                     (s->cmos_data[RTC_REG_A] & 0x70) <= 0x20) {
-                    s->offset = get_guest_rtc_ns(s) % NSEC_PER_SEC;
+                    s->offset = get_guest_rtc_ns(s) % NANOSECONDS_PER_SECOND;
                     rtc_set_time(s);
                 }
             }
@@ -581,7 +584,7 @@
     int64_t guest_nsec;
 
     guest_nsec = get_guest_rtc_ns(s);
-    guest_sec = guest_nsec / NSEC_PER_SEC;
+    guest_sec = guest_nsec / NANOSECONDS_PER_SECOND;
     gmtime_r(&guest_sec, &ret);
 
     /* Is SET flag of Register B disabled? */
@@ -609,7 +612,8 @@
 
     guest_nsec = get_guest_rtc_ns(s);
     /* UIP bit will be set at last 244us of every second. */
-    if ((guest_nsec % NSEC_PER_SEC) >= (NSEC_PER_SEC - UIP_HOLD_LENGTH)) {
+    if ((guest_nsec % NANOSECONDS_PER_SECOND) >=
+        (NANOSECONDS_PER_SECOND - UIP_HOLD_LENGTH)) {
         return 1;
     }
     return 0;
@@ -723,6 +727,12 @@
         check_update_timer(s);
     }
 
+    uint64_t now = qemu_clock_get_ns(rtc_clock);
+    if (now < s->next_periodic_time ||
+        now > (s->next_periodic_time + get_max_clock_jump())) {
+        periodic_timer_update(s, qemu_clock_get_ns(rtc_clock));
+    }
+
 #ifdef TARGET_I386
     if (version_id >= 2) {
         if (s->lost_tick_policy == LOST_TICK_POLICY_SLEW) {
@@ -733,22 +743,23 @@
     return 0;
 }
 
-static const VMStateDescription vmstate_rtc_irq_reinject_on_ack_count = {
-    .name = "irq_reinject_on_ack_count",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT16(irq_reinject_on_ack_count, RTCState),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
 static bool rtc_irq_reinject_on_ack_count_needed(void *opaque)
 {
     RTCState *s = (RTCState *)opaque;
     return s->irq_reinject_on_ack_count != 0;
 }
 
+static const VMStateDescription vmstate_rtc_irq_reinject_on_ack_count = {
+    .name = "mc146818rtc/irq_reinject_on_ack_count",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = rtc_irq_reinject_on_ack_count_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(irq_reinject_on_ack_count, RTCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_rtc = {
     .name = "mc146818rtc",
     .version_id = 3,
@@ -758,7 +769,7 @@
         VMSTATE_BUFFER(cmos_data, RTCState),
         VMSTATE_UINT8(cmos_index, RTCState),
         VMSTATE_UNUSED(7*4),
-        VMSTATE_TIMER(periodic_timer, RTCState),
+        VMSTATE_TIMER_PTR(periodic_timer, RTCState),
         VMSTATE_INT64(next_periodic_time, RTCState),
         VMSTATE_UNUSED(3*8),
         VMSTATE_UINT32_V(irq_coalesced, RTCState, 2),
@@ -766,17 +777,13 @@
         VMSTATE_UINT64_V(base_rtc, RTCState, 3),
         VMSTATE_UINT64_V(last_update, RTCState, 3),
         VMSTATE_INT64_V(offset, RTCState, 3),
-        VMSTATE_TIMER_V(update_timer, RTCState, 3),
+        VMSTATE_TIMER_PTR_V(update_timer, RTCState, 3),
         VMSTATE_UINT64_V(next_alarm_time, RTCState, 3),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_rtc_irq_reinject_on_ack_count,
-            .needed = rtc_irq_reinject_on_ack_count_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_rtc_irq_reinject_on_ack_count,
+        NULL
     }
 };
 
@@ -831,49 +838,12 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void rtc_get_date(Object *obj, Visitor *v, void *opaque,
-                         const char *name, Error **errp)
+static void rtc_get_date(Object *obj, struct tm *current_tm, Error **errp)
 {
-    Error *err = NULL;
     RTCState *s = MC146818_RTC(obj);
-    struct tm current_tm;
 
     rtc_update_time(s);
-    rtc_get_time(s, &current_tm);
-    visit_start_struct(v, NULL, "struct tm", name, 0, &err);
-    if (err) {
-        goto out;
-    }
-    visit_type_int32(v, &current_tm.tm_year, "tm_year", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_int32(v, &current_tm.tm_mon, "tm_mon", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_int32(v, &current_tm.tm_mday, "tm_mday", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_int32(v, &current_tm.tm_hour, "tm_hour", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_int32(v, &current_tm.tm_min, "tm_min", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_int32(v, &current_tm.tm_sec, "tm_sec", &err);
-    if (err) {
-        goto out_end;
-    }
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, errp);
-out:
-    error_propagate(errp, err);
+    rtc_get_time(s, current_tm);
 }
 
 static void rtc_realizefn(DeviceState *dev, Error **errp)
@@ -932,11 +902,12 @@
     qdev_set_legacy_instance_id(dev, base, 3);
     qemu_register_reset(rtc_reset, s);
 
-    object_property_add(OBJECT(s), "date", "struct tm",
-                        rtc_get_date, NULL, NULL, s, NULL);
+    object_property_add_tm(OBJECT(s), "date", rtc_get_date, NULL);
 
     object_property_add_alias(qdev_get_machine(), "rtc-time",
                               OBJECT(s), "date", NULL);
+
+    qdev_init_gpio_out(dev, &s->irq, 1);
 }
 
 ISADevice *rtc_init(ISABus *bus, int base_year, qemu_irq intercept_irq)
@@ -951,9 +922,9 @@
     qdev_prop_set_int32(dev, "base_year", base_year);
     qdev_init_nofail(dev);
     if (intercept_irq) {
-        s->irq = intercept_irq;
+        qdev_connect_gpio_out(dev, 0, intercept_irq);
     } else {
-        isa_init_irq(isadev, &s->irq, RTC_ISA_IRQ);
+        isa_connect_gpio_out(isadev, 0, RTC_ISA_IRQ);
     }
     QLIST_INSERT_HEAD(&rtc_devices, s, link);
 
diff --git a/hw/timer/milkymist-sysctl.c b/hw/timer/milkymist-sysctl.c
index 30535a4..2194832 100644
--- a/hw/timer/milkymist-sysctl.c
+++ b/hw/timer/milkymist-sysctl.c
@@ -18,9 +18,10 @@
  *
  *
  * Specification available at:
- *   http://www.milkymist.org/socdoc/sysctl.pdf
+ *   http://milkymist.walle.cc/socdoc/sysctl.pdf
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sysbus.h"
 #include "sysemu/sysemu.h"
@@ -269,9 +270,10 @@
     s->regs[R_GPIO_IN] = s->strappings;
 }
 
-static int milkymist_sysctl_init(SysBusDevice *dev)
+static void milkymist_sysctl_init(Object *obj)
 {
-    MilkymistSysctlState *s = MILKYMIST_SYSCTL(dev);
+    MilkymistSysctlState *s = MILKYMIST_SYSCTL(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     sysbus_init_irq(dev, &s->gpio_irq);
     sysbus_init_irq(dev, &s->timer0_irq);
@@ -281,14 +283,18 @@
     s->bh1 = qemu_bh_new(timer1_hit, s);
     s->ptimer0 = ptimer_init(s->bh0);
     s->ptimer1 = ptimer_init(s->bh1);
-    ptimer_set_freq(s->ptimer0, s->freq_hz);
-    ptimer_set_freq(s->ptimer1, s->freq_hz);
 
-    memory_region_init_io(&s->regs_region, OBJECT(s), &sysctl_mmio_ops, s,
+    memory_region_init_io(&s->regs_region, obj, &sysctl_mmio_ops, s,
             "milkymist-sysctl", R_MAX * 4);
     sysbus_init_mmio(dev, &s->regs_region);
+}
 
-    return 0;
+static void milkymist_sysctl_realize(DeviceState *dev, Error **errp)
+{
+    MilkymistSysctlState *s = MILKYMIST_SYSCTL(dev);
+
+    ptimer_set_freq(s->ptimer0, s->freq_hz);
+    ptimer_set_freq(s->ptimer1, s->freq_hz);
 }
 
 static const VMStateDescription vmstate_milkymist_sysctl = {
@@ -318,9 +324,8 @@
 static void milkymist_sysctl_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = milkymist_sysctl_init;
+    dc->realize = milkymist_sysctl_realize;
     dc->reset = milkymist_sysctl_reset;
     dc->vmsd = &vmstate_milkymist_sysctl;
     dc->props = milkymist_sysctl_properties;
@@ -330,6 +335,7 @@
     .name          = TYPE_MILKYMIST_SYSCTL,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(MilkymistSysctlState),
+    .instance_init = milkymist_sysctl_init,
     .class_init    = milkymist_sysctl_class_init,
 };
 
diff --git a/hw/timer/mips_gictimer.c b/hw/timer/mips_gictimer.c
new file mode 100644
index 0000000..3698889
--- /dev/null
+++ b/hw/timer/mips_gictimer.c
@@ -0,0 +1,142 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2016 Imagination Technologies
+ */
+
+#include "qemu/osdep.h"
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "qemu/timer.h"
+#include "hw/timer/mips_gictimer.h"
+
+#define TIMER_PERIOD 10 /* 10 ns period for 100 Mhz frequency */
+
+static void gic_vptimer_update(MIPSGICTimerState *gictimer,
+                                   uint32_t vp_index, uint64_t now)
+{
+    uint64_t next;
+    uint32_t wait;
+
+    wait = gictimer->vptimers[vp_index].comparelo - gictimer->sh_counterlo -
+           (uint32_t)(now / TIMER_PERIOD);
+    next = now + (uint64_t)wait * TIMER_PERIOD;
+
+    timer_mod(gictimer->vptimers[vp_index].qtimer, next);
+}
+
+static void gic_vptimer_expire(MIPSGICTimerState *gictimer, uint32_t vp_index,
+                               uint64_t now)
+{
+    if (gictimer->countstop) {
+        /* timer stopped */
+        return;
+    }
+    gictimer->cb(gictimer->opaque, vp_index);
+    gic_vptimer_update(gictimer, vp_index, now);
+}
+
+static void gic_vptimer_cb(void *opaque)
+{
+    MIPSGICTimerVPState *vptimer = opaque;
+    MIPSGICTimerState *gictimer = vptimer->gictimer;
+    gic_vptimer_expire(gictimer, vptimer->vp_index,
+                       qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+}
+
+uint32_t mips_gictimer_get_sh_count(MIPSGICTimerState *gictimer)
+{
+    int i;
+    if (gictimer->countstop) {
+        return gictimer->sh_counterlo;
+    } else {
+        uint64_t now;
+        now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        for (i = 0; i < gictimer->num_vps; i++) {
+            if (timer_pending(gictimer->vptimers[i].qtimer)
+                && timer_expired(gictimer->vptimers[i].qtimer, now)) {
+                /* The timer has already expired.  */
+                gic_vptimer_expire(gictimer, i, now);
+            }
+        }
+        return gictimer->sh_counterlo + (uint32_t)(now / TIMER_PERIOD);
+    }
+}
+
+void mips_gictimer_store_sh_count(MIPSGICTimerState *gictimer, uint64_t count)
+{
+    int i;
+    uint64_t now;
+
+    if (gictimer->countstop || !gictimer->vptimers[0].qtimer) {
+        gictimer->sh_counterlo = count;
+    } else {
+        /* Store new count register */
+        now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+        gictimer->sh_counterlo = count - (uint32_t)(now / TIMER_PERIOD);
+        /* Update timer timer */
+        for (i = 0; i < gictimer->num_vps; i++) {
+            gic_vptimer_update(gictimer, i, now);
+        }
+    }
+}
+
+uint32_t mips_gictimer_get_vp_compare(MIPSGICTimerState *gictimer,
+                                      uint32_t vp_index)
+{
+    return gictimer->vptimers[vp_index].comparelo;
+}
+
+void mips_gictimer_store_vp_compare(MIPSGICTimerState *gictimer,
+                                    uint32_t vp_index, uint64_t compare)
+{
+    gictimer->vptimers[vp_index].comparelo = (uint32_t) compare;
+    gic_vptimer_update(gictimer, vp_index,
+                       qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+}
+
+uint8_t mips_gictimer_get_countstop(MIPSGICTimerState *gictimer)
+{
+    return gictimer->countstop;
+}
+
+void mips_gictimer_start_count(MIPSGICTimerState *gictimer)
+{
+    gictimer->countstop = 0;
+    mips_gictimer_store_sh_count(gictimer, gictimer->sh_counterlo);
+}
+
+void mips_gictimer_stop_count(MIPSGICTimerState *gictimer)
+{
+    int i;
+
+    gictimer->countstop = 1;
+    /* Store the current value */
+    gictimer->sh_counterlo +=
+        (uint32_t)(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / TIMER_PERIOD);
+    for (i = 0; i < gictimer->num_vps; i++) {
+        timer_del(gictimer->vptimers[i].qtimer);
+    }
+}
+
+MIPSGICTimerState *mips_gictimer_init(void *opaque, uint32_t nvps,
+                                      MIPSGICTimerCB *cb)
+{
+    int i;
+    MIPSGICTimerState *gictimer = g_new(MIPSGICTimerState, 1);
+    gictimer->vptimers = g_new(MIPSGICTimerVPState, nvps);
+    gictimer->countstop = 1;
+    gictimer->num_vps = nvps;
+    gictimer->opaque = opaque;
+    gictimer->cb = cb;
+    for (i = 0; i < nvps; i++) {
+        gictimer->vptimers[i].gictimer = gictimer;
+        gictimer->vptimers[i].vp_index = i;
+        gictimer->vptimers[i].qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                            &gic_vptimer_cb,
+                                            &gictimer->vptimers[i]);
+    }
+    return gictimer;
+}
diff --git a/hw/timer/omap_gptimer.c b/hw/timer/omap_gptimer.c
index b7f3d49..5e3e8a6 100644
--- a/hw/timer/omap_gptimer.c
+++ b/hw/timer/omap_gptimer.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/arm/omap.h"
@@ -132,8 +133,8 @@
         timer_mod(timer->timer, timer->time + expires);
 
         if (timer->ce && timer->match_val >= timer->val) {
-            matches = muldiv64(timer->match_val - timer->val,
-                            timer->ticks_per_sec, timer->rate);
+            matches = muldiv64(timer->ticks_per_sec,
+                               timer->match_val - timer->val, timer->rate);
             timer_mod(timer->match, timer->time + matches);
         } else
             timer_del(timer->match);
@@ -401,7 +402,7 @@
         if (s->trigger == gpt_trigger_none)
             omap_gp_timer_out(s, s->scpwm);
         /* TODO: make sure this doesn't overflow 32-bits */
-        s->ticks_per_sec = get_ticks_per_sec() << (s->pre ? s->ptv + 1 : 0);
+        s->ticks_per_sec = NANOSECONDS_PER_SECOND << (s->pre ? s->ptv + 1 : 0);
         omap_gp_timer_update(s);
         break;
 
@@ -444,7 +445,7 @@
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
 
     if (addr & 2)
-        return omap_gp_timer_write(opaque, addr, (value << 16) | s->writeh);
+        omap_gp_timer_write(opaque, addr, (value << 16) | s->writeh);
     else
         s->writeh = (uint16_t) value;
 }
@@ -468,8 +469,7 @@
 struct omap_gp_timer_s *omap_gp_timer_init(struct omap_target_agent_s *ta,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk)
 {
-    struct omap_gp_timer_s *s = (struct omap_gp_timer_s *)
-            g_malloc0(sizeof(struct omap_gp_timer_s));
+    struct omap_gp_timer_s *s = g_new0(struct omap_gp_timer_s, 1);
 
     s->ta = ta;
     s->irq = irq;
diff --git a/hw/timer/omap_synctimer.c b/hw/timer/omap_synctimer.c
index 8e50488..9ee6519 100644
--- a/hw/timer/omap_synctimer.c
+++ b/hw/timer/omap_synctimer.c
@@ -17,6 +17,7 @@
  * You should have received a copy of the GNU General Public License along
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/arm/omap.h"
@@ -28,7 +29,8 @@
 
 /* 32-kHz Sync Timer of the OMAP2 */
 static uint32_t omap_synctimer_read(struct omap_synctimer_s *s) {
-    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 0x8000, get_ticks_per_sec());
+    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 0x8000,
+                    NANOSECONDS_PER_SECOND);
 }
 
 void omap_synctimer_reset(struct omap_synctimer_s *s)
diff --git a/hw/timer/pl031.c b/hw/timer/pl031.c
index 34d9b44..dbbeb9b 100644
--- a/hw/timer/pl031.c
+++ b/hw/timer/pl031.c
@@ -11,9 +11,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
+#include "qemu/log.h"
 
 //#define DEBUG_PL031
 
@@ -79,7 +82,7 @@
 static uint32_t pl031_get_count(PL031State *s)
 {
     int64_t now = qemu_clock_get_ns(rtc_clock);
-    return s->tick_offset + now / get_ticks_per_sec();
+    return s->tick_offset + now / NANOSECONDS_PER_SECOND;
 }
 
 static void pl031_set_alarm(PL031State *s)
@@ -95,7 +98,7 @@
         pl031_interrupt(s);
     } else {
         int64_t now = qemu_clock_get_ns(rtc_clock);
-        timer_mod(s->timer, now + (int64_t)ticks * get_ticks_per_sec());
+        timer_mod(s->timer, now + (int64_t)ticks * NANOSECONDS_PER_SECOND);
     }
 }
 
@@ -191,21 +194,21 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static int pl031_init(SysBusDevice *dev)
+static void pl031_init(Object *obj)
 {
-    PL031State *s = PL031(dev);
+    PL031State *s = PL031(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
     struct tm tm;
 
-    memory_region_init_io(&s->iomem, OBJECT(s), &pl031_ops, s, "pl031", 0x1000);
+    memory_region_init_io(&s->iomem, obj, &pl031_ops, s, "pl031", 0x1000);
     sysbus_init_mmio(dev, &s->iomem);
 
     sysbus_init_irq(dev, &s->irq);
     qemu_get_timedate(&tm, 0);
     s->tick_offset = mktimegm(&tm) -
-        qemu_clock_get_ns(rtc_clock) / get_ticks_per_sec();
+        qemu_clock_get_ns(rtc_clock) / NANOSECONDS_PER_SECOND;
 
     s->timer = timer_new_ns(rtc_clock, pl031_interrupt, s);
-    return 0;
 }
 
 static void pl031_pre_save(void *opaque)
@@ -215,7 +218,7 @@
     /* tick_offset is base_time - rtc_clock base time.  Instead, we want to
      * store the base time relative to the QEMU_CLOCK_VIRTUAL for backwards-compatibility.  */
     int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    s->tick_offset_vmstate = s->tick_offset + delta / get_ticks_per_sec();
+    s->tick_offset_vmstate = s->tick_offset + delta / NANOSECONDS_PER_SECOND;
 }
 
 static int pl031_post_load(void *opaque, int version_id)
@@ -223,7 +226,7 @@
     PL031State *s = opaque;
 
     int64_t delta = qemu_clock_get_ns(rtc_clock) - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-    s->tick_offset = s->tick_offset_vmstate - delta / get_ticks_per_sec();
+    s->tick_offset = s->tick_offset_vmstate - delta / NANOSECONDS_PER_SECOND;
     pl031_set_alarm(s);
     return 0;
 }
@@ -248,9 +251,7 @@
 static void pl031_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
 
-    k->init = pl031_init;
     dc->vmsd = &vmstate_pl031;
 }
 
@@ -258,6 +259,7 @@
     .name          = TYPE_PL031,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PL031State),
+    .instance_init = pl031_init,
     .class_init    = pl031_class_init,
 };
 
diff --git a/hw/timer/puv3_ost.c b/hw/timer/puv3_ost.c
index fa9eefd..93650b7 100644
--- a/hw/timer/puv3_ost.c
+++ b/hw/timer/puv3_ost.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/ptimer.h"
 #include "qemu/main-loop.h"
diff --git a/hw/timer/pxa2xx_timer.c b/hw/timer/pxa2xx_timer.c
index 130e9dc..59002b4 100644
--- a/hw/timer/pxa2xx_timer.c
+++ b/hw/timer/pxa2xx_timer.c
@@ -7,6 +7,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
@@ -118,11 +119,11 @@
     uint64_t new_qemu;
 
     now_vm = s->clock +
-            muldiv64(now_qemu - s->lastload, s->freq, get_ticks_per_sec());
+            muldiv64(now_qemu - s->lastload, s->freq, NANOSECONDS_PER_SECOND);
 
     for (i = 0; i < 4; i ++) {
         new_qemu = now_qemu + muldiv64((uint32_t) (s->timer[i].value - now_vm),
-                        get_ticks_per_sec(), s->freq);
+                        NANOSECONDS_PER_SECOND, s->freq);
         timer_mod(s->timer[i].qtimer, new_qemu);
     }
 }
@@ -147,10 +148,10 @@
 
     now_vm = s->tm4[counter].clock + muldiv64(now_qemu -
                     s->tm4[counter].lastload,
-                    s->tm4[counter].freq, get_ticks_per_sec());
+                    s->tm4[counter].freq, NANOSECONDS_PER_SECOND);
 
     new_qemu = now_qemu + muldiv64((uint32_t) (s->tm4[n].tm.value - now_vm),
-                    get_ticks_per_sec(), s->tm4[counter].freq);
+                    NANOSECONDS_PER_SECOND, s->tm4[counter].freq);
     timer_mod(s->tm4[n].tm.qtimer, new_qemu);
 }
 
@@ -189,7 +190,7 @@
         return s->tm4[tm].tm.value;
     case OSCR:
         return s->clock + muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                        s->lastload, s->freq, get_ticks_per_sec());
+                        s->lastload, s->freq, NANOSECONDS_PER_SECOND);
     case OSCR11: tm ++;
         /* fall through */
     case OSCR10: tm ++;
@@ -213,15 +214,17 @@
                 s->snapshot = s->tm4[tm - 1].clock + muldiv64(
                                 qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
                                 s->tm4[tm - 1].lastload,
-                                s->tm4[tm - 1].freq, get_ticks_per_sec());
+                                s->tm4[tm - 1].freq, NANOSECONDS_PER_SECOND);
             else
                 s->snapshot = s->tm4[tm - 1].clock;
         }
 
         if (!s->tm4[tm].freq)
             return s->tm4[tm].clock;
-        return s->tm4[tm].clock + muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
-                        s->tm4[tm].lastload, s->tm4[tm].freq, get_ticks_per_sec());
+        return s->tm4[tm].clock +
+            muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) -
+                     s->tm4[tm].lastload, s->tm4[tm].freq,
+                     NANOSECONDS_PER_SECOND);
     case OIER:
         return s->irq_enabled;
     case OSSR:	/* Status register */
@@ -432,10 +435,10 @@
     return 0;
 }
 
-static int pxa2xx_timer_init(SysBusDevice *dev)
+static void pxa2xx_timer_init(Object *obj)
 {
-    PXA2xxTimerInfo *s = PXA2XX_TIMER(dev);
-    int i;
+    PXA2xxTimerInfo *s = PXA2XX_TIMER(obj);
+    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
 
     s->irq_enabled = 0;
     s->oldclock = 0;
@@ -443,16 +446,28 @@
     s->lastload = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     s->reset3 = 0;
 
+    memory_region_init_io(&s->iomem, obj, &pxa2xx_timer_ops, s,
+                          "pxa2xx-timer", 0x00001000);
+    sysbus_init_mmio(dev, &s->iomem);
+}
+
+static void pxa2xx_timer_realize(DeviceState *dev, Error **errp)
+{
+    PXA2xxTimerInfo *s = PXA2XX_TIMER(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+    int i;
+
     for (i = 0; i < 4; i ++) {
         s->timer[i].value = 0;
-        sysbus_init_irq(dev, &s->timer[i].irq);
+        sysbus_init_irq(sbd, &s->timer[i].irq);
         s->timer[i].info = s;
         s->timer[i].num = i;
         s->timer[i].qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                        pxa2xx_timer_tick, &s->timer[i]);
+                                          pxa2xx_timer_tick, &s->timer[i]);
     }
+
     if (s->flags & (1 << PXA2XX_TIMER_HAVE_TM4)) {
-        sysbus_init_irq(dev, &s->irq4);
+        sysbus_init_irq(sbd, &s->irq4);
 
         for (i = 0; i < 8; i ++) {
             s->tm4[i].tm.value = 0;
@@ -461,15 +476,9 @@
             s->tm4[i].freq = 0;
             s->tm4[i].control = 0x0;
             s->tm4[i].tm.qtimer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                        pxa2xx_timer_tick4, &s->tm4[i]);
+                                               pxa2xx_timer_tick4, &s->tm4[i]);
         }
     }
-
-    memory_region_init_io(&s->iomem, OBJECT(s), &pxa2xx_timer_ops, s,
-                          "pxa2xx-timer", 0x00001000);
-    sysbus_init_mmio(dev, &s->iomem);
-
-    return 0;
 }
 
 static const VMStateDescription vmstate_pxa2xx_timer0_regs = {
@@ -572,9 +581,8 @@
 static void pxa2xx_timer_class_init(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
-    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(oc);
 
-    sdc->init = pxa2xx_timer_init;
+    dc->realize  = pxa2xx_timer_realize;
     dc->vmsd = &vmstate_pxa2xx_timer_regs;
 }
 
@@ -582,6 +590,7 @@
     .name          = TYPE_PXA2XX_TIMER,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(PXA2xxTimerInfo),
+    .instance_init = pxa2xx_timer_init,
     .abstract      = true,
     .class_init    = pxa2xx_timer_class_init,
 };
diff --git a/hw/timer/sh_timer.c b/hw/timer/sh_timer.c
index 07f0670..255b2fc 100644
--- a/hw/timer/sh_timer.c
+++ b/hw/timer/sh_timer.c
@@ -8,6 +8,7 @@
  * This code is licensed under the GPL.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/sh4/sh.h"
 #include "qemu/timer.h"
diff --git a/hw/timer/slavio_timer.c b/hw/timer/slavio_timer.c
index 45d97e6..fb3e08b 100644
--- a/hw/timer/slavio_timer.c
+++ b/hw/timer/slavio_timer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sparc/sun4m.h"
 #include "qemu/timer.h"
 #include "hw/ptimer.h"
diff --git a/hw/timer/stm32f2xx_timer.c b/hw/timer/stm32f2xx_timer.c
new file mode 100644
index 0000000..bf0fb28
--- /dev/null
+++ b/hw/timer/stm32f2xx_timer.c
@@ -0,0 +1,330 @@
+/*
+ * STM32F2XX Timer
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/timer/stm32f2xx_timer.h"
+#include "qemu/log.h"
+
+#ifndef STM_TIMER_ERR_DEBUG
+#define STM_TIMER_ERR_DEBUG 0
+#endif
+
+#define DB_PRINT_L(lvl, fmt, args...) do { \
+    if (STM_TIMER_ERR_DEBUG >= lvl) { \
+        qemu_log("%s: " fmt, __func__, ## args); \
+    } \
+} while (0);
+
+#define DB_PRINT(fmt, args...) DB_PRINT_L(1, fmt, ## args)
+
+static void stm32f2xx_timer_set_alarm(STM32F2XXTimerState *s, int64_t now);
+
+static void stm32f2xx_timer_interrupt(void *opaque)
+{
+    STM32F2XXTimerState *s = opaque;
+
+    DB_PRINT("Interrupt\n");
+
+    if (s->tim_dier & TIM_DIER_UIE && s->tim_cr1 & TIM_CR1_CEN) {
+        s->tim_sr |= 1;
+        qemu_irq_pulse(s->irq);
+        stm32f2xx_timer_set_alarm(s, s->hit_time);
+    }
+}
+
+static inline int64_t stm32f2xx_ns_to_ticks(STM32F2XXTimerState *s, int64_t t)
+{
+    return muldiv64(t, s->freq_hz, 1000000000ULL) / (s->tim_psc + 1);
+}
+
+static void stm32f2xx_timer_set_alarm(STM32F2XXTimerState *s, int64_t now)
+{
+    uint64_t ticks;
+    int64_t now_ticks;
+
+    if (s->tim_arr == 0) {
+        return;
+    }
+
+    DB_PRINT("Alarm set at: 0x%x\n", s->tim_cr1);
+
+    now_ticks = stm32f2xx_ns_to_ticks(s, now);
+    ticks = s->tim_arr - (now_ticks - s->tick_offset);
+
+    DB_PRINT("Alarm set in %d ticks\n", (int) ticks);
+
+    s->hit_time = muldiv64((ticks + (uint64_t) now_ticks) * (s->tim_psc + 1),
+                               1000000000ULL, s->freq_hz);
+
+    timer_mod(s->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + s->hit_time);
+    DB_PRINT("Wait Time: %" PRId64 " ticks\n", s->hit_time);
+}
+
+static void stm32f2xx_timer_reset(DeviceState *dev)
+{
+    STM32F2XXTimerState *s = STM32F2XXTIMER(dev);
+    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+    s->tim_cr1 = 0;
+    s->tim_cr2 = 0;
+    s->tim_smcr = 0;
+    s->tim_dier = 0;
+    s->tim_sr = 0;
+    s->tim_egr = 0;
+    s->tim_ccmr1 = 0;
+    s->tim_ccmr2 = 0;
+    s->tim_ccer = 0;
+    s->tim_psc = 0;
+    s->tim_arr = 0;
+    s->tim_ccr1 = 0;
+    s->tim_ccr2 = 0;
+    s->tim_ccr3 = 0;
+    s->tim_ccr4 = 0;
+    s->tim_dcr = 0;
+    s->tim_dmar = 0;
+    s->tim_or = 0;
+
+    s->tick_offset = stm32f2xx_ns_to_ticks(s, now);
+}
+
+static uint64_t stm32f2xx_timer_read(void *opaque, hwaddr offset,
+                           unsigned size)
+{
+    STM32F2XXTimerState *s = opaque;
+
+    DB_PRINT("Read 0x%"HWADDR_PRIx"\n", offset);
+
+    switch (offset) {
+    case TIM_CR1:
+        return s->tim_cr1;
+    case TIM_CR2:
+        return s->tim_cr2;
+    case TIM_SMCR:
+        return s->tim_smcr;
+    case TIM_DIER:
+        return s->tim_dier;
+    case TIM_SR:
+        return s->tim_sr;
+    case TIM_EGR:
+        return s->tim_egr;
+    case TIM_CCMR1:
+        return s->tim_ccmr1;
+    case TIM_CCMR2:
+        return s->tim_ccmr2;
+    case TIM_CCER:
+        return s->tim_ccer;
+    case TIM_CNT:
+        return stm32f2xx_ns_to_ticks(s, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL)) -
+               s->tick_offset;
+    case TIM_PSC:
+        return s->tim_psc;
+    case TIM_ARR:
+        return s->tim_arr;
+    case TIM_CCR1:
+        return s->tim_ccr1;
+    case TIM_CCR2:
+        return s->tim_ccr2;
+    case TIM_CCR3:
+        return s->tim_ccr3;
+    case TIM_CCR4:
+        return s->tim_ccr4;
+    case TIM_DCR:
+        return s->tim_dcr;
+    case TIM_DMAR:
+        return s->tim_dmar;
+    case TIM_OR:
+        return s->tim_or;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%"HWADDR_PRIx"\n", __func__, offset);
+    }
+
+    return 0;
+}
+
+static void stm32f2xx_timer_write(void *opaque, hwaddr offset,
+                        uint64_t val64, unsigned size)
+{
+    STM32F2XXTimerState *s = opaque;
+    uint32_t value = val64;
+    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+    uint32_t timer_val = 0;
+
+    DB_PRINT("Write 0x%x, 0x%"HWADDR_PRIx"\n", value, offset);
+
+    switch (offset) {
+    case TIM_CR1:
+        s->tim_cr1 = value;
+        return;
+    case TIM_CR2:
+        s->tim_cr2 = value;
+        return;
+    case TIM_SMCR:
+        s->tim_smcr = value;
+        return;
+    case TIM_DIER:
+        s->tim_dier = value;
+        return;
+    case TIM_SR:
+        /* This is set by hardware and cleared by software */
+        s->tim_sr &= value;
+        return;
+    case TIM_EGR:
+        s->tim_egr = value;
+        if (s->tim_egr & TIM_EGR_UG) {
+            timer_val = 0;
+            break;
+        }
+        return;
+    case TIM_CCMR1:
+        s->tim_ccmr1 = value;
+        return;
+    case TIM_CCMR2:
+        s->tim_ccmr2 = value;
+        return;
+    case TIM_CCER:
+        s->tim_ccer = value;
+        return;
+    case TIM_PSC:
+        timer_val = stm32f2xx_ns_to_ticks(s, now) - s->tick_offset;
+        s->tim_psc = value;
+        value = timer_val;
+        break;
+    case TIM_CNT:
+        timer_val = value;
+        break;
+    case TIM_ARR:
+        s->tim_arr = value;
+        stm32f2xx_timer_set_alarm(s, now);
+        return;
+    case TIM_CCR1:
+        s->tim_ccr1 = value;
+        return;
+    case TIM_CCR2:
+        s->tim_ccr2 = value;
+        return;
+    case TIM_CCR3:
+        s->tim_ccr3 = value;
+        return;
+    case TIM_CCR4:
+        s->tim_ccr4 = value;
+        return;
+    case TIM_DCR:
+        s->tim_dcr = value;
+        return;
+    case TIM_DMAR:
+        s->tim_dmar = value;
+        return;
+    case TIM_OR:
+        s->tim_or = value;
+        return;
+    default:
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%"HWADDR_PRIx"\n", __func__, offset);
+        return;
+    }
+
+    /* This means that a register write has affected the timer in a way that
+     * requires a refresh of both tick_offset and the alarm.
+     */
+    s->tick_offset = stm32f2xx_ns_to_ticks(s, now) - timer_val;
+    stm32f2xx_timer_set_alarm(s, now);
+}
+
+static const MemoryRegionOps stm32f2xx_timer_ops = {
+    .read = stm32f2xx_timer_read,
+    .write = stm32f2xx_timer_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static const VMStateDescription vmstate_stm32f2xx_timer = {
+    .name = TYPE_STM32F2XX_TIMER,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT64(tick_offset, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_cr1, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_cr2, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_smcr, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_dier, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_sr, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_egr, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccmr1, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccmr2, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccer, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_psc, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_arr, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccr1, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccr2, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccr3, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_ccr4, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_dcr, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_dmar, STM32F2XXTimerState),
+        VMSTATE_UINT32(tim_or, STM32F2XXTimerState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property stm32f2xx_timer_properties[] = {
+    DEFINE_PROP_UINT64("clock-frequency", struct STM32F2XXTimerState,
+                       freq_hz, 1000000000),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void stm32f2xx_timer_init(Object *obj)
+{
+    STM32F2XXTimerState *s = STM32F2XXTIMER(obj);
+
+    sysbus_init_irq(SYS_BUS_DEVICE(obj), &s->irq);
+
+    memory_region_init_io(&s->iomem, obj, &stm32f2xx_timer_ops, s,
+                          "stm32f2xx_timer", 0x4000);
+    sysbus_init_mmio(SYS_BUS_DEVICE(obj), &s->iomem);
+
+    s->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, stm32f2xx_timer_interrupt, s);
+}
+
+static void stm32f2xx_timer_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->reset = stm32f2xx_timer_reset;
+    dc->props = stm32f2xx_timer_properties;
+    dc->vmsd = &vmstate_stm32f2xx_timer;
+}
+
+static const TypeInfo stm32f2xx_timer_info = {
+    .name          = TYPE_STM32F2XX_TIMER,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(STM32F2XXTimerState),
+    .instance_init = stm32f2xx_timer_init,
+    .class_init    = stm32f2xx_timer_class_init,
+};
+
+static void stm32f2xx_timer_register_types(void)
+{
+    type_register_static(&stm32f2xx_timer_info);
+}
+
+type_init(stm32f2xx_timer_register_types)
diff --git a/hw/timer/trace-events b/hw/timer/trace-events
new file mode 100644
index 0000000..3495c41
--- /dev/null
+++ b/hw/timer/trace-events
@@ -0,0 +1,51 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/timer/slavio_timer.c
+slavio_timer_get_out(uint64_t limit, uint32_t counthigh, uint32_t count) "limit %"PRIx64" count %x%08x"
+slavio_timer_irq(uint32_t counthigh, uint32_t count) "callback: count %x%08x"
+slavio_timer_mem_readl_invalid(uint64_t addr) "invalid read address %"PRIx64
+slavio_timer_mem_readl(uint64_t addr, uint32_t ret) "read %"PRIx64" = %08x"
+slavio_timer_mem_writel(uint64_t addr, uint32_t val) "write %"PRIx64" = %08x"
+slavio_timer_mem_writel_limit(unsigned int timer_index, uint64_t count) "processor %d user timer set to %016"PRIx64
+slavio_timer_mem_writel_counter_invalid(void) "not user timer"
+slavio_timer_mem_writel_status_start(unsigned int timer_index) "processor %d user timer started"
+slavio_timer_mem_writel_status_stop(unsigned int timer_index) "processor %d user timer stopped"
+slavio_timer_mem_writel_mode_user(unsigned int timer_index) "processor %d changed from counter to user timer"
+slavio_timer_mem_writel_mode_counter(unsigned int timer_index) "processor %d changed from user timer to counter"
+slavio_timer_mem_writel_mode_invalid(void) "not system timer"
+slavio_timer_mem_writel_invalid(uint64_t addr) "invalid write address %"PRIx64
+
+# hw/timer/grlib_gptimer.c
+grlib_gptimer_enable(int id, uint32_t count) "timer:%d set count 0x%x and run"
+grlib_gptimer_disabled(int id, uint32_t config) "timer:%d Timer disable config 0x%x"
+grlib_gptimer_restart(int id, uint32_t reload) "timer:%d reload val: 0x%x"
+grlib_gptimer_set_scaler(uint32_t scaler, uint32_t freq) "scaler:0x%x freq: 0x%x"
+grlib_gptimer_hit(int id) "timer:%d HIT"
+grlib_gptimer_readl(int id, uint64_t addr, uint32_t val) "timer:%d addr 0x%"PRIx64" 0x%x"
+grlib_gptimer_writel(int id, uint64_t addr, uint32_t val) "timer:%d addr 0x%"PRIx64" 0x%x"
+
+# hw/timer/lm32_timer.c
+lm32_timer_memory_write(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
+lm32_timer_memory_read(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
+lm32_timer_hit(void) "timer hit"
+lm32_timer_irq_state(int level) "irq state %d"
+
+# hw/timer/milkymist-sysctl.c
+milkymist_sysctl_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_sysctl_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
+milkymist_sysctl_icap_write(uint32_t value) "value %08x"
+milkymist_sysctl_start_timer0(void) "Start timer0"
+milkymist_sysctl_stop_timer0(void) "Stop timer0"
+milkymist_sysctl_start_timer1(void) "Start timer1"
+milkymist_sysctl_stop_timer1(void) "Stop timer1"
+milkymist_sysctl_pulse_irq_timer0(void) "Pulse IRQ Timer0"
+milkymist_sysctl_pulse_irq_timer1(void) "Pulse IRQ Timer1"
+
+# hw/timer/aspeed_timer.c
+aspeed_timer_ctrl_enable(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_ctrl_external_clock(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_ctrl_overflow_interrupt(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_ctrl_pulse_enable(uint8_t i, bool enable) "Timer %" PRIu8 ": %d"
+aspeed_timer_set_ctrl2(uint32_t value) "Value: 0x%" PRIx32
+aspeed_timer_set_value(int timer, int reg, uint32_t value) "Timer %d register %d: 0x%" PRIx32
+aspeed_timer_read(uint64_t offset, unsigned size, uint64_t value) "From 0x%" PRIx64 ": of size %u: 0x%" PRIx64
diff --git a/hw/timer/tusb6010.c b/hw/timer/tusb6010.c
deleted file mode 100644
index 459c748..0000000
--- a/hw/timer/tusb6010.c
+++ /dev/null
@@ -1,816 +0,0 @@
-/*
- * Texas Instruments TUSB6010 emulation.
- * Based on reverse-engineering of a linux driver.
- *
- * Copyright (C) 2008 Nokia Corporation
- * Written by Andrzej Zaborowski <andrew@openedhand.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 or
- * (at your option) version 3 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-#include "qemu-common.h"
-#include "qemu/timer.h"
-#include "hw/usb.h"
-#include "hw/arm/omap.h"
-#include "hw/irq.h"
-#include "hw/devices.h"
-#include "hw/sysbus.h"
-
-#define TYPE_TUSB6010 "tusb6010"
-#define TUSB(obj) OBJECT_CHECK(TUSBState, (obj), TYPE_TUSB6010)
-
-typedef struct TUSBState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem[2];
-    qemu_irq irq;
-    MUSBState *musb;
-    QEMUTimer *otg_timer;
-    QEMUTimer *pwr_timer;
-
-    int power;
-    uint32_t scratch;
-    uint16_t test_reset;
-    uint32_t prcm_config;
-    uint32_t prcm_mngmt;
-    uint16_t otg_status;
-    uint32_t dev_config;
-    int host_mode;
-    uint32_t intr;
-    uint32_t intr_ok;
-    uint32_t mask;
-    uint32_t usbip_intr;
-    uint32_t usbip_mask;
-    uint32_t gpio_intr;
-    uint32_t gpio_mask;
-    uint32_t gpio_config;
-    uint32_t dma_intr;
-    uint32_t dma_mask;
-    uint32_t dma_map;
-    uint32_t dma_config;
-    uint32_t ep0_config;
-    uint32_t rx_config[15];
-    uint32_t tx_config[15];
-    uint32_t wkup_mask;
-    uint32_t pullup[2];
-    uint32_t control_config;
-    uint32_t otg_timer_val;
-} TUSBState;
-
-#define TUSB_DEVCLOCK			60000000	/* 60 MHz */
-
-#define TUSB_VLYNQ_CTRL			0x004
-
-/* Mentor Graphics OTG core registers.  */
-#define TUSB_BASE_OFFSET		0x400
-
-/* FIFO registers, 32-bit.  */
-#define TUSB_FIFO_BASE			0x600
-
-/* Device System & Control registers, 32-bit.  */
-#define TUSB_SYS_REG_BASE		0x800
-
-#define TUSB_DEV_CONF			(TUSB_SYS_REG_BASE + 0x000)
-#define	TUSB_DEV_CONF_USB_HOST_MODE	(1 << 16)
-#define	TUSB_DEV_CONF_PROD_TEST_MODE	(1 << 15)
-#define	TUSB_DEV_CONF_SOFT_ID		(1 << 1)
-#define	TUSB_DEV_CONF_ID_SEL		(1 << 0)
-
-#define TUSB_PHY_OTG_CTRL_ENABLE	(TUSB_SYS_REG_BASE + 0x004)
-#define TUSB_PHY_OTG_CTRL		(TUSB_SYS_REG_BASE + 0x008)
-#define	TUSB_PHY_OTG_CTRL_WRPROTECT	(0xa5 << 24)
-#define	TUSB_PHY_OTG_CTRL_O_ID_PULLUP	(1 << 23)
-#define	TUSB_PHY_OTG_CTRL_O_VBUS_DET_EN	(1 << 19)
-#define	TUSB_PHY_OTG_CTRL_O_SESS_END_EN	(1 << 18)
-#define	TUSB_PHY_OTG_CTRL_TESTM2	(1 << 17)
-#define	TUSB_PHY_OTG_CTRL_TESTM1	(1 << 16)
-#define	TUSB_PHY_OTG_CTRL_TESTM0	(1 << 15)
-#define	TUSB_PHY_OTG_CTRL_TX_DATA2	(1 << 14)
-#define	TUSB_PHY_OTG_CTRL_TX_GZ2	(1 << 13)
-#define	TUSB_PHY_OTG_CTRL_TX_ENABLE2	(1 << 12)
-#define	TUSB_PHY_OTG_CTRL_DM_PULLDOWN	(1 << 11)
-#define	TUSB_PHY_OTG_CTRL_DP_PULLDOWN	(1 << 10)
-#define	TUSB_PHY_OTG_CTRL_OSC_EN	(1 << 9)
-#define	TUSB_PHY_OTG_CTRL_PHYREF_CLK(v)	(((v) & 3) << 7)
-#define	TUSB_PHY_OTG_CTRL_PD		(1 << 6)
-#define	TUSB_PHY_OTG_CTRL_PLL_ON	(1 << 5)
-#define	TUSB_PHY_OTG_CTRL_EXT_RPU	(1 << 4)
-#define	TUSB_PHY_OTG_CTRL_PWR_GOOD	(1 << 3)
-#define	TUSB_PHY_OTG_CTRL_RESET		(1 << 2)
-#define	TUSB_PHY_OTG_CTRL_SUSPENDM	(1 << 1)
-#define	TUSB_PHY_OTG_CTRL_CLK_MODE	(1 << 0)
-
-/* OTG status register */
-#define TUSB_DEV_OTG_STAT		(TUSB_SYS_REG_BASE + 0x00c)
-#define	TUSB_DEV_OTG_STAT_PWR_CLK_GOOD	(1 << 8)
-#define	TUSB_DEV_OTG_STAT_SESS_END	(1 << 7)
-#define	TUSB_DEV_OTG_STAT_SESS_VALID	(1 << 6)
-#define	TUSB_DEV_OTG_STAT_VBUS_VALID	(1 << 5)
-#define	TUSB_DEV_OTG_STAT_VBUS_SENSE	(1 << 4)
-#define	TUSB_DEV_OTG_STAT_ID_STATUS	(1 << 3)
-#define	TUSB_DEV_OTG_STAT_HOST_DISCON	(1 << 2)
-#define	TUSB_DEV_OTG_STAT_LINE_STATE	(3 << 0)
-#define	TUSB_DEV_OTG_STAT_DP_ENABLE	(1 << 1)
-#define	TUSB_DEV_OTG_STAT_DM_ENABLE	(1 << 0)
-
-#define TUSB_DEV_OTG_TIMER		(TUSB_SYS_REG_BASE + 0x010)
-#define TUSB_DEV_OTG_TIMER_ENABLE	(1 << 31)
-#define TUSB_DEV_OTG_TIMER_VAL(v)	((v) & 0x07ffffff)
-#define TUSB_PRCM_REV			(TUSB_SYS_REG_BASE + 0x014)
-
-/* PRCM configuration register */
-#define TUSB_PRCM_CONF			(TUSB_SYS_REG_BASE + 0x018)
-#define	TUSB_PRCM_CONF_SFW_CPEN		(1 << 24)
-#define	TUSB_PRCM_CONF_SYS_CLKSEL(v)	(((v) & 3) << 16)
-
-/* PRCM management register */
-#define TUSB_PRCM_MNGMT			(TUSB_SYS_REG_BASE + 0x01c)
-#define	TUSB_PRCM_MNGMT_SRP_FIX_TMR(v)	(((v) & 0xf) << 25)
-#define	TUSB_PRCM_MNGMT_SRP_FIX_EN	(1 << 24)
-#define	TUSB_PRCM_MNGMT_VBUS_VAL_TMR(v)	(((v) & 0xf) << 20)
-#define	TUSB_PRCM_MNGMT_VBUS_VAL_FLT_EN	(1 << 19)
-#define	TUSB_PRCM_MNGMT_DFT_CLK_DIS	(1 << 18)
-#define	TUSB_PRCM_MNGMT_VLYNQ_CLK_DIS	(1 << 17)
-#define	TUSB_PRCM_MNGMT_OTG_SESS_END_EN	(1 << 10)
-#define	TUSB_PRCM_MNGMT_OTG_VBUS_DET_EN	(1 << 9)
-#define	TUSB_PRCM_MNGMT_OTG_ID_PULLUP	(1 << 8)
-#define	TUSB_PRCM_MNGMT_15_SW_EN	(1 << 4)
-#define	TUSB_PRCM_MNGMT_33_SW_EN	(1 << 3)
-#define	TUSB_PRCM_MNGMT_5V_CPEN		(1 << 2)
-#define	TUSB_PRCM_MNGMT_PM_IDLE		(1 << 1)
-#define	TUSB_PRCM_MNGMT_DEV_IDLE	(1 << 0)
-
-/* Wake-up source clear and mask registers */
-#define TUSB_PRCM_WAKEUP_SOURCE		(TUSB_SYS_REG_BASE + 0x020)
-#define TUSB_PRCM_WAKEUP_CLEAR		(TUSB_SYS_REG_BASE + 0x028)
-#define TUSB_PRCM_WAKEUP_MASK		(TUSB_SYS_REG_BASE + 0x02c)
-#define	TUSB_PRCM_WAKEUP_RESERVED_BITS	(0xffffe << 13)
-#define	TUSB_PRCM_WGPIO_7		(1 << 12)
-#define	TUSB_PRCM_WGPIO_6		(1 << 11)
-#define	TUSB_PRCM_WGPIO_5		(1 << 10)
-#define	TUSB_PRCM_WGPIO_4		(1 << 9)
-#define	TUSB_PRCM_WGPIO_3		(1 << 8)
-#define	TUSB_PRCM_WGPIO_2		(1 << 7)
-#define	TUSB_PRCM_WGPIO_1		(1 << 6)
-#define	TUSB_PRCM_WGPIO_0		(1 << 5)
-#define	TUSB_PRCM_WHOSTDISCON		(1 << 4)	/* Host disconnect */
-#define	TUSB_PRCM_WBUS			(1 << 3)	/* USB bus resume */
-#define	TUSB_PRCM_WNORCS		(1 << 2)	/* NOR chip select */
-#define	TUSB_PRCM_WVBUS			(1 << 1)	/* OTG PHY VBUS */
-#define	TUSB_PRCM_WID			(1 << 0)	/* OTG PHY ID detect */
-
-#define TUSB_PULLUP_1_CTRL		(TUSB_SYS_REG_BASE + 0x030)
-#define TUSB_PULLUP_2_CTRL		(TUSB_SYS_REG_BASE + 0x034)
-#define TUSB_INT_CTRL_REV		(TUSB_SYS_REG_BASE + 0x038)
-#define TUSB_INT_CTRL_CONF		(TUSB_SYS_REG_BASE + 0x03c)
-#define TUSB_USBIP_INT_SRC		(TUSB_SYS_REG_BASE + 0x040)
-#define TUSB_USBIP_INT_SET		(TUSB_SYS_REG_BASE + 0x044)
-#define TUSB_USBIP_INT_CLEAR		(TUSB_SYS_REG_BASE + 0x048)
-#define TUSB_USBIP_INT_MASK		(TUSB_SYS_REG_BASE + 0x04c)
-#define TUSB_DMA_INT_SRC		(TUSB_SYS_REG_BASE + 0x050)
-#define TUSB_DMA_INT_SET		(TUSB_SYS_REG_BASE + 0x054)
-#define TUSB_DMA_INT_CLEAR		(TUSB_SYS_REG_BASE + 0x058)
-#define TUSB_DMA_INT_MASK		(TUSB_SYS_REG_BASE + 0x05c)
-#define TUSB_GPIO_INT_SRC		(TUSB_SYS_REG_BASE + 0x060)
-#define TUSB_GPIO_INT_SET		(TUSB_SYS_REG_BASE + 0x064)
-#define TUSB_GPIO_INT_CLEAR		(TUSB_SYS_REG_BASE + 0x068)
-#define TUSB_GPIO_INT_MASK		(TUSB_SYS_REG_BASE + 0x06c)
-
-/* NOR flash interrupt source registers */
-#define TUSB_INT_SRC			(TUSB_SYS_REG_BASE + 0x070)
-#define TUSB_INT_SRC_SET		(TUSB_SYS_REG_BASE + 0x074)
-#define TUSB_INT_SRC_CLEAR		(TUSB_SYS_REG_BASE + 0x078)
-#define TUSB_INT_MASK			(TUSB_SYS_REG_BASE + 0x07c)
-#define	TUSB_INT_SRC_TXRX_DMA_DONE	(1 << 24)
-#define	TUSB_INT_SRC_USB_IP_CORE	(1 << 17)
-#define	TUSB_INT_SRC_OTG_TIMEOUT	(1 << 16)
-#define	TUSB_INT_SRC_VBUS_SENSE_CHNG	(1 << 15)
-#define	TUSB_INT_SRC_ID_STATUS_CHNG	(1 << 14)
-#define	TUSB_INT_SRC_DEV_WAKEUP		(1 << 13)
-#define	TUSB_INT_SRC_DEV_READY		(1 << 12)
-#define	TUSB_INT_SRC_USB_IP_TX		(1 << 9)
-#define	TUSB_INT_SRC_USB_IP_RX		(1 << 8)
-#define	TUSB_INT_SRC_USB_IP_VBUS_ERR	(1 << 7)
-#define	TUSB_INT_SRC_USB_IP_VBUS_REQ	(1 << 6)
-#define	TUSB_INT_SRC_USB_IP_DISCON	(1 << 5)
-#define	TUSB_INT_SRC_USB_IP_CONN	(1 << 4)
-#define	TUSB_INT_SRC_USB_IP_SOF		(1 << 3)
-#define	TUSB_INT_SRC_USB_IP_RST_BABBLE	(1 << 2)
-#define	TUSB_INT_SRC_USB_IP_RESUME	(1 << 1)
-#define	TUSB_INT_SRC_USB_IP_SUSPEND	(1 << 0)
-
-#define TUSB_GPIO_REV			(TUSB_SYS_REG_BASE + 0x080)
-#define TUSB_GPIO_CONF			(TUSB_SYS_REG_BASE + 0x084)
-#define TUSB_DMA_CTRL_REV		(TUSB_SYS_REG_BASE + 0x100)
-#define TUSB_DMA_REQ_CONF		(TUSB_SYS_REG_BASE + 0x104)
-#define TUSB_EP0_CONF			(TUSB_SYS_REG_BASE + 0x108)
-#define TUSB_EP_IN_SIZE			(TUSB_SYS_REG_BASE + 0x10c)
-#define TUSB_DMA_EP_MAP			(TUSB_SYS_REG_BASE + 0x148)
-#define TUSB_EP_OUT_SIZE		(TUSB_SYS_REG_BASE + 0x14c)
-#define TUSB_EP_MAX_PACKET_SIZE_OFFSET	(TUSB_SYS_REG_BASE + 0x188)
-#define TUSB_SCRATCH_PAD		(TUSB_SYS_REG_BASE + 0x1c4)
-#define TUSB_WAIT_COUNT			(TUSB_SYS_REG_BASE + 0x1c8)
-#define TUSB_PROD_TEST_RESET		(TUSB_SYS_REG_BASE + 0x1d8)
-
-#define TUSB_DIDR1_LO			(TUSB_SYS_REG_BASE + 0x1f8)
-#define TUSB_DIDR1_HI			(TUSB_SYS_REG_BASE + 0x1fc)
-
-/* Device System & Control register bitfields */
-#define TUSB_INT_CTRL_CONF_INT_RLCYC(v)	(((v) & 0x7) << 18)
-#define TUSB_INT_CTRL_CONF_INT_POLARITY	(1 << 17)
-#define TUSB_INT_CTRL_CONF_INT_MODE	(1 << 16)
-#define TUSB_GPIO_CONF_DMAREQ(v)	(((v) & 0x3f) << 24)
-#define TUSB_DMA_REQ_CONF_BURST_SIZE(v)	(((v) & 3) << 26)
-#define TUSB_DMA_REQ_CONF_DMA_RQ_EN(v)	(((v) & 0x3f) << 20)
-#define TUSB_DMA_REQ_CONF_DMA_RQ_ASR(v)	(((v) & 0xf) << 16)
-#define TUSB_EP0_CONFIG_SW_EN		(1 << 8)
-#define TUSB_EP0_CONFIG_DIR_TX		(1 << 7)
-#define TUSB_EP0_CONFIG_XFR_SIZE(v)	((v) & 0x7f)
-#define TUSB_EP_CONFIG_SW_EN		(1 << 31)
-#define TUSB_EP_CONFIG_XFR_SIZE(v)	((v) & 0x7fffffff)
-#define TUSB_PROD_TEST_RESET_VAL	0xa596
-
-static void tusb_intr_update(TUSBState *s)
-{
-    if (s->control_config & TUSB_INT_CTRL_CONF_INT_POLARITY)
-        qemu_set_irq(s->irq, s->intr & ~s->mask & s->intr_ok);
-    else
-        qemu_set_irq(s->irq, (!(s->intr & ~s->mask)) & s->intr_ok);
-}
-
-static void tusb_usbip_intr_update(TUSBState *s)
-{
-    /* TX interrupt in the MUSB */
-    if (s->usbip_intr & 0x0000ffff & ~s->usbip_mask)
-        s->intr |= TUSB_INT_SRC_USB_IP_TX;
-    else
-        s->intr &= ~TUSB_INT_SRC_USB_IP_TX;
-
-    /* RX interrupt in the MUSB */
-    if (s->usbip_intr & 0xffff0000 & ~s->usbip_mask)
-        s->intr |= TUSB_INT_SRC_USB_IP_RX;
-    else
-        s->intr &= ~TUSB_INT_SRC_USB_IP_RX;
-
-    /* XXX: What about TUSB_INT_SRC_USB_IP_CORE?  */
-
-    tusb_intr_update(s);
-}
-
-static void tusb_dma_intr_update(TUSBState *s)
-{
-    if (s->dma_intr & ~s->dma_mask)
-        s->intr |= TUSB_INT_SRC_TXRX_DMA_DONE;
-    else
-        s->intr &= ~TUSB_INT_SRC_TXRX_DMA_DONE;
-
-    tusb_intr_update(s);
-}
-
-static void tusb_gpio_intr_update(TUSBState *s)
-{
-    /* TODO: How is this signalled?  */
-}
-
-static uint32_t tusb_async_readb(void *opaque, hwaddr addr)
-{
-    TUSBState *s = (TUSBState *) opaque;
-
-    switch (addr & 0xfff) {
-    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
-        return musb_read[0](s->musb, addr & 0x1ff);
-
-    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
-        return musb_read[0](s->musb, 0x20 + ((addr >> 3) & 0x3c));
-    }
-
-    printf("%s: unknown register at %03x\n",
-                    __FUNCTION__, (int) (addr & 0xfff));
-    return 0;
-}
-
-static uint32_t tusb_async_readh(void *opaque, hwaddr addr)
-{
-    TUSBState *s = (TUSBState *) opaque;
-
-    switch (addr & 0xfff) {
-    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
-        return musb_read[1](s->musb, addr & 0x1ff);
-
-    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
-        return musb_read[1](s->musb, 0x20 + ((addr >> 3) & 0x3c));
-    }
-
-    printf("%s: unknown register at %03x\n",
-                    __FUNCTION__, (int) (addr & 0xfff));
-    return 0;
-}
-
-static uint32_t tusb_async_readw(void *opaque, hwaddr addr)
-{
-    TUSBState *s = (TUSBState *) opaque;
-    int offset = addr & 0xfff;
-    int epnum;
-    uint32_t ret;
-
-    switch (offset) {
-    case TUSB_DEV_CONF:
-        return s->dev_config;
-
-    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
-        return musb_read[2](s->musb, offset & 0x1ff);
-
-    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
-        return musb_read[2](s->musb, 0x20 + ((addr >> 3) & 0x3c));
-
-    case TUSB_PHY_OTG_CTRL_ENABLE:
-    case TUSB_PHY_OTG_CTRL:
-        return 0x00;	/* TODO */
-
-    case TUSB_DEV_OTG_STAT:
-        ret = s->otg_status;
-#if 0
-        if (!(s->prcm_mngmt & TUSB_PRCM_MNGMT_OTG_VBUS_DET_EN))
-            ret &= ~TUSB_DEV_OTG_STAT_VBUS_VALID;
-#endif
-        return ret;
-    case TUSB_DEV_OTG_TIMER:
-        return s->otg_timer_val;
-
-    case TUSB_PRCM_REV:
-        return 0x20;
-    case TUSB_PRCM_CONF:
-        return s->prcm_config;
-    case TUSB_PRCM_MNGMT:
-        return s->prcm_mngmt;
-    case TUSB_PRCM_WAKEUP_SOURCE:
-    case TUSB_PRCM_WAKEUP_CLEAR:	/* TODO: What does this one return?  */
-        return 0x00000000;
-    case TUSB_PRCM_WAKEUP_MASK:
-        return s->wkup_mask;
-
-    case TUSB_PULLUP_1_CTRL:
-        return s->pullup[0];
-    case TUSB_PULLUP_2_CTRL:
-        return s->pullup[1];
-
-    case TUSB_INT_CTRL_REV:
-        return 0x20;
-    case TUSB_INT_CTRL_CONF:
-        return s->control_config;
-
-    case TUSB_USBIP_INT_SRC:
-    case TUSB_USBIP_INT_SET:	/* TODO: What do these two return?  */
-    case TUSB_USBIP_INT_CLEAR:
-        return s->usbip_intr;
-    case TUSB_USBIP_INT_MASK:
-        return s->usbip_mask;
-
-    case TUSB_DMA_INT_SRC:
-    case TUSB_DMA_INT_SET:	/* TODO: What do these two return?  */
-    case TUSB_DMA_INT_CLEAR:
-        return s->dma_intr;
-    case TUSB_DMA_INT_MASK:
-        return s->dma_mask;
-
-    case TUSB_GPIO_INT_SRC:	/* TODO: What do these two return?  */
-    case TUSB_GPIO_INT_SET:
-    case TUSB_GPIO_INT_CLEAR:
-        return s->gpio_intr;
-    case TUSB_GPIO_INT_MASK:
-        return s->gpio_mask;
-
-    case TUSB_INT_SRC:
-    case TUSB_INT_SRC_SET:	/* TODO: What do these two return?  */
-    case TUSB_INT_SRC_CLEAR:
-        return s->intr;
-    case TUSB_INT_MASK:
-        return s->mask;
-
-    case TUSB_GPIO_REV:
-        return 0x30;
-    case TUSB_GPIO_CONF:
-        return s->gpio_config;
-
-    case TUSB_DMA_CTRL_REV:
-        return 0x30;
-    case TUSB_DMA_REQ_CONF:
-        return s->dma_config;
-    case TUSB_EP0_CONF:
-        return s->ep0_config;
-    case TUSB_EP_IN_SIZE ... (TUSB_EP_IN_SIZE + 0x3b):
-        epnum = (offset - TUSB_EP_IN_SIZE) >> 2;
-        return s->tx_config[epnum];
-    case TUSB_DMA_EP_MAP:
-        return s->dma_map;
-    case TUSB_EP_OUT_SIZE ... (TUSB_EP_OUT_SIZE + 0x3b):
-        epnum = (offset - TUSB_EP_OUT_SIZE) >> 2;
-        return s->rx_config[epnum];
-    case TUSB_EP_MAX_PACKET_SIZE_OFFSET ...
-            (TUSB_EP_MAX_PACKET_SIZE_OFFSET + 0x3b):
-        return 0x00000000;	/* TODO */
-    case TUSB_WAIT_COUNT:
-        return 0x00;		/* TODO */
-
-    case TUSB_SCRATCH_PAD:
-        return s->scratch;
-
-    case TUSB_PROD_TEST_RESET:
-        return s->test_reset;
-
-    /* DIE IDs */
-    case TUSB_DIDR1_LO:
-        return 0xa9453c59;
-    case TUSB_DIDR1_HI:
-        return 0x54059adf;
-    }
-
-    printf("%s: unknown register at %03x\n", __FUNCTION__, offset);
-    return 0;
-}
-
-static void tusb_async_writeb(void *opaque, hwaddr addr,
-                uint32_t value)
-{
-    TUSBState *s = (TUSBState *) opaque;
-
-    switch (addr & 0xfff) {
-    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
-        musb_write[0](s->musb, addr & 0x1ff, value);
-        break;
-
-    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
-        musb_write[0](s->musb, 0x20 + ((addr >> 3) & 0x3c), value);
-        break;
-
-    default:
-        printf("%s: unknown register at %03x\n",
-                        __FUNCTION__, (int) (addr & 0xfff));
-        return;
-    }
-}
-
-static void tusb_async_writeh(void *opaque, hwaddr addr,
-                uint32_t value)
-{
-    TUSBState *s = (TUSBState *) opaque;
-
-    switch (addr & 0xfff) {
-    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
-        musb_write[1](s->musb, addr & 0x1ff, value);
-        break;
-
-    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
-        musb_write[1](s->musb, 0x20 + ((addr >> 3) & 0x3c), value);
-        break;
-
-    default:
-        printf("%s: unknown register at %03x\n",
-                        __FUNCTION__, (int) (addr & 0xfff));
-        return;
-    }
-}
-
-static void tusb_async_writew(void *opaque, hwaddr addr,
-                uint32_t value)
-{
-    TUSBState *s = (TUSBState *) opaque;
-    int offset = addr & 0xfff;
-    int epnum;
-
-    switch (offset) {
-    case TUSB_VLYNQ_CTRL:
-        break;
-
-    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
-        musb_write[2](s->musb, offset & 0x1ff, value);
-        break;
-
-    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
-        musb_write[2](s->musb, 0x20 + ((addr >> 3) & 0x3c), value);
-        break;
-
-    case TUSB_DEV_CONF:
-        s->dev_config = value;
-        s->host_mode = (value & TUSB_DEV_CONF_USB_HOST_MODE);
-        if (value & TUSB_DEV_CONF_PROD_TEST_MODE)
-            hw_error("%s: Product Test mode not allowed\n", __FUNCTION__);
-        break;
-
-    case TUSB_PHY_OTG_CTRL_ENABLE:
-    case TUSB_PHY_OTG_CTRL:
-        return;		/* TODO */
-    case TUSB_DEV_OTG_TIMER:
-        s->otg_timer_val = value;
-        if (value & TUSB_DEV_OTG_TIMER_ENABLE)
-            timer_mod(s->otg_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                            muldiv64(TUSB_DEV_OTG_TIMER_VAL(value),
-                                     get_ticks_per_sec(), TUSB_DEVCLOCK));
-        else
-            timer_del(s->otg_timer);
-        break;
-
-    case TUSB_PRCM_CONF:
-        s->prcm_config = value;
-        break;
-    case TUSB_PRCM_MNGMT:
-        s->prcm_mngmt = value;
-        break;
-    case TUSB_PRCM_WAKEUP_CLEAR:
-        break;
-    case TUSB_PRCM_WAKEUP_MASK:
-        s->wkup_mask = value;
-        break;
-
-    case TUSB_PULLUP_1_CTRL:
-        s->pullup[0] = value;
-        break;
-    case TUSB_PULLUP_2_CTRL:
-        s->pullup[1] = value;
-        break;
-    case TUSB_INT_CTRL_CONF:
-        s->control_config = value;
-        tusb_intr_update(s);
-        break;
-
-    case TUSB_USBIP_INT_SET:
-        s->usbip_intr |= value;
-        tusb_usbip_intr_update(s);
-        break;
-    case TUSB_USBIP_INT_CLEAR:
-        s->usbip_intr &= ~value;
-        tusb_usbip_intr_update(s);
-        musb_core_intr_clear(s->musb, ~value);
-        break;
-    case TUSB_USBIP_INT_MASK:
-        s->usbip_mask = value;
-        tusb_usbip_intr_update(s);
-        break;
-
-    case TUSB_DMA_INT_SET:
-        s->dma_intr |= value;
-        tusb_dma_intr_update(s);
-        break;
-    case TUSB_DMA_INT_CLEAR:
-        s->dma_intr &= ~value;
-        tusb_dma_intr_update(s);
-        break;
-    case TUSB_DMA_INT_MASK:
-        s->dma_mask = value;
-        tusb_dma_intr_update(s);
-        break;
-
-    case TUSB_GPIO_INT_SET:
-        s->gpio_intr |= value;
-        tusb_gpio_intr_update(s);
-        break;
-    case TUSB_GPIO_INT_CLEAR:
-        s->gpio_intr &= ~value;
-        tusb_gpio_intr_update(s);
-        break;
-    case TUSB_GPIO_INT_MASK:
-        s->gpio_mask = value;
-        tusb_gpio_intr_update(s);
-        break;
-
-    case TUSB_INT_SRC_SET:
-        s->intr |= value;
-        tusb_intr_update(s);
-        break;
-    case TUSB_INT_SRC_CLEAR:
-        s->intr &= ~value;
-        tusb_intr_update(s);
-        break;
-    case TUSB_INT_MASK:
-        s->mask = value;
-        tusb_intr_update(s);
-        break;
-
-    case TUSB_GPIO_CONF:
-        s->gpio_config = value;
-        break;
-    case TUSB_DMA_REQ_CONF:
-        s->dma_config = value;
-        break;
-    case TUSB_EP0_CONF:
-        s->ep0_config = value & 0x1ff;
-        musb_set_size(s->musb, 0, TUSB_EP0_CONFIG_XFR_SIZE(value),
-                        value & TUSB_EP0_CONFIG_DIR_TX);
-        break;
-    case TUSB_EP_IN_SIZE ... (TUSB_EP_IN_SIZE + 0x3b):
-        epnum = (offset - TUSB_EP_IN_SIZE) >> 2;
-        s->tx_config[epnum] = value;
-        musb_set_size(s->musb, epnum + 1, TUSB_EP_CONFIG_XFR_SIZE(value), 1);
-        break;
-    case TUSB_DMA_EP_MAP:
-        s->dma_map = value;
-        break;
-    case TUSB_EP_OUT_SIZE ... (TUSB_EP_OUT_SIZE + 0x3b):
-        epnum = (offset - TUSB_EP_OUT_SIZE) >> 2;
-        s->rx_config[epnum] = value;
-        musb_set_size(s->musb, epnum + 1, TUSB_EP_CONFIG_XFR_SIZE(value), 0);
-        break;
-    case TUSB_EP_MAX_PACKET_SIZE_OFFSET ...
-            (TUSB_EP_MAX_PACKET_SIZE_OFFSET + 0x3b):
-        return;		/* TODO */
-    case TUSB_WAIT_COUNT:
-        return;		/* TODO */
-
-    case TUSB_SCRATCH_PAD:
-        s->scratch = value;
-        break;
-
-    case TUSB_PROD_TEST_RESET:
-        s->test_reset = value;
-        break;
-
-    default:
-        printf("%s: unknown register at %03x\n", __FUNCTION__, offset);
-        return;
-    }
-}
-
-static const MemoryRegionOps tusb_async_ops = {
-    .old_mmio = {
-        .read = { tusb_async_readb, tusb_async_readh, tusb_async_readw, },
-        .write =  { tusb_async_writeb, tusb_async_writeh, tusb_async_writew, },
-    },
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static void tusb_otg_tick(void *opaque)
-{
-    TUSBState *s = (TUSBState *) opaque;
-
-    s->otg_timer_val = 0;
-    s->intr |= TUSB_INT_SRC_OTG_TIMEOUT;
-    tusb_intr_update(s);
-}
-
-static void tusb_power_tick(void *opaque)
-{
-    TUSBState *s = (TUSBState *) opaque;
-
-    if (s->power) {
-        s->intr_ok = ~0;
-        tusb_intr_update(s);
-    }
-}
-
-static void tusb_musb_core_intr(void *opaque, int source, int level)
-{
-    TUSBState *s = (TUSBState *) opaque;
-    uint16_t otg_status = s->otg_status;
-
-    switch (source) {
-    case musb_set_vbus:
-        if (level)
-            otg_status |= TUSB_DEV_OTG_STAT_VBUS_VALID;
-        else
-            otg_status &= ~TUSB_DEV_OTG_STAT_VBUS_VALID;
-
-        /* XXX: only if TUSB_PHY_OTG_CTRL_OTG_VBUS_DET_EN set?  */
-        /* XXX: only if TUSB_PRCM_MNGMT_OTG_VBUS_DET_EN set?  */
-        if (s->otg_status != otg_status) {
-            s->otg_status = otg_status;
-            s->intr |= TUSB_INT_SRC_VBUS_SENSE_CHNG;
-            tusb_intr_update(s);
-        }
-        break;
-
-    case musb_set_session:
-        /* XXX: only if TUSB_PHY_OTG_CTRL_OTG_SESS_END_EN set?  */
-        /* XXX: only if TUSB_PRCM_MNGMT_OTG_SESS_END_EN set?  */
-        if (level) {
-            s->otg_status |= TUSB_DEV_OTG_STAT_SESS_VALID;
-            s->otg_status &= ~TUSB_DEV_OTG_STAT_SESS_END;
-        } else {
-            s->otg_status &= ~TUSB_DEV_OTG_STAT_SESS_VALID;
-            s->otg_status |= TUSB_DEV_OTG_STAT_SESS_END;
-        }
-
-        /* XXX: some IRQ or anything?  */
-        break;
-
-    case musb_irq_tx:
-    case musb_irq_rx:
-        s->usbip_intr = musb_core_intr_get(s->musb);
-        /* Fall through.  */
-    default:
-        if (level)
-            s->intr |= 1 << source;
-        else
-            s->intr &= ~(1 << source);
-        tusb_intr_update(s);
-        break;
-    }
-}
-
-static void tusb6010_power(TUSBState *s, int on)
-{
-    if (!on) {
-        s->power = 0;
-    } else if (!s->power && on) {
-        s->power = 1;
-        /* Pull the interrupt down after TUSB6010 comes up.  */
-        s->intr_ok = 0;
-        tusb_intr_update(s);
-        timer_mod(s->pwr_timer,
-                       qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + get_ticks_per_sec() / 2);
-    }
-}
-
-static void tusb6010_irq(void *opaque, int source, int level)
-{
-    if (source) {
-        tusb_musb_core_intr(opaque, source - 1, level);
-    } else {
-        tusb6010_power(opaque, level);
-    }
-}
-
-static void tusb6010_reset(DeviceState *dev)
-{
-    TUSBState *s = TUSB(dev);
-    int i;
-
-    s->test_reset = TUSB_PROD_TEST_RESET_VAL;
-    s->host_mode = 0;
-    s->dev_config = 0;
-    s->otg_status = 0;	/* !TUSB_DEV_OTG_STAT_ID_STATUS means host mode */
-    s->power = 0;
-    s->mask = 0xffffffff;
-    s->intr = 0x00000000;
-    s->otg_timer_val = 0;
-    s->scratch = 0;
-    s->prcm_config = 0;
-    s->prcm_mngmt = 0;
-    s->intr_ok = 0;
-    s->usbip_intr = 0;
-    s->usbip_mask = 0;
-    s->gpio_intr = 0;
-    s->gpio_mask = 0;
-    s->gpio_config = 0;
-    s->dma_intr = 0;
-    s->dma_mask = 0;
-    s->dma_map = 0;
-    s->dma_config = 0;
-    s->ep0_config = 0;
-    s->wkup_mask = 0;
-    s->pullup[0] = s->pullup[1] = 0;
-    s->control_config = 0;
-    for (i = 0; i < 15; i++) {
-        s->rx_config[i] = s->tx_config[i] = 0;
-    }
-    musb_reset(s->musb);
-}
-
-static int tusb6010_init(SysBusDevice *sbd)
-{
-    DeviceState *dev = DEVICE(sbd);
-    TUSBState *s = TUSB(dev);
-
-    s->otg_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, tusb_otg_tick, s);
-    s->pwr_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, tusb_power_tick, s);
-    memory_region_init_io(&s->iomem[1], OBJECT(s), &tusb_async_ops, s,
-                          "tusb-async", UINT32_MAX);
-    sysbus_init_mmio(sbd, &s->iomem[0]);
-    sysbus_init_mmio(sbd, &s->iomem[1]);
-    sysbus_init_irq(sbd, &s->irq);
-    qdev_init_gpio_in(dev, tusb6010_irq, musb_irq_max + 1);
-    s->musb = musb_init(dev, 1);
-    return 0;
-}
-
-static void tusb6010_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-
-    k->init = tusb6010_init;
-    dc->reset = tusb6010_reset;
-}
-
-static const TypeInfo tusb6010_info = {
-    .name          = TYPE_TUSB6010,
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(TUSBState),
-    .class_init    = tusb6010_class_init,
-};
-
-static void tusb6010_register_types(void)
-{
-    type_register_static(&tusb6010_info);
-}
-
-type_init(tusb6010_register_types)
diff --git a/hw/timer/twl92230.c b/hw/timer/twl92230.c
index 7ded4ba..7ba4e9a 100644
--- a/hw/timer/twl92230.c
+++ b/hw/timer/twl92230.c
@@ -19,11 +19,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/i2c/i2c.h"
 #include "sysemu/sysemu.h"
 #include "ui/console.h"
+#include "qemu/bcd.h"
 
 #define VERBOSE 1
 
diff --git a/hw/timer/xilinx_timer.c b/hw/timer/xilinx_timer.c
index cdb3355..2ea970d 100644
--- a/hw/timer/xilinx_timer.c
+++ b/hw/timer/xilinx_timer.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/ptimer.h"
 #include "qemu/log.h"
diff --git a/hw/tpm/Makefile.objs b/hw/tpm/Makefile.objs
index 99f5983..64cecc3 100644
--- a/hw/tpm/Makefile.objs
+++ b/hw/tpm/Makefile.objs
@@ -1,2 +1,2 @@
 common-obj-$(CONFIG_TPM_TIS) += tpm_tis.o
-common-obj-$(CONFIG_TPM_PASSTHROUGH) += tpm_passthrough.o
+common-obj-$(CONFIG_TPM_PASSTHROUGH) += tpm_passthrough.o tpm_util.o
diff --git a/hw/tpm/tpm_int.h b/hw/tpm/tpm_int.h
index 2f582ca..f2f285b 100644
--- a/hw/tpm/tpm_int.h
+++ b/hw/tpm/tpm_int.h
@@ -29,6 +29,7 @@
 
     char *backend;
     TPMBackend *be_driver;
+    TPMVersion be_tpm_version;
 };
 
 #define TPM(obj) OBJECT_CHECK(TPMState, (obj), TYPE_TPM_TIS)
@@ -62,6 +63,13 @@
 
 #define TPM_FAIL                  9
 
+#define TPM_ORD_ContinueSelfTest  0x53
 #define TPM_ORD_GetTicks          0xf1
 
+
+/* TPM2 defines */
+#define TPM2_ST_NO_SESSIONS       0x8001
+
+#define TPM2_CC_ReadClock         0x00000181
+
 #endif /* TPM_TPM_INT_H */
diff --git a/hw/tpm/tpm_passthrough.c b/hw/tpm/tpm_passthrough.c
index 56e9e0f..e88c0d2 100644
--- a/hw/tpm/tpm_passthrough.c
+++ b/hw/tpm/tpm_passthrough.c
@@ -22,10 +22,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
-#include <dirent.h>
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "qapi/error.h"
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "sysemu/tpm_backend.h"
 #include "tpm_int.h"
@@ -33,16 +32,15 @@
 #include "hw/i386/pc.h"
 #include "sysemu/tpm_backend_int.h"
 #include "tpm_tis.h"
+#include "tpm_util.h"
 
-/* #define DEBUG_TPM */
+#define DEBUG_TPM 0
 
-#ifdef DEBUG_TPM
-#define DPRINTF(fmt, ...) \
-    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
+#define DPRINTF(fmt, ...) do { \
+    if (DEBUG_TPM) { \
+        fprintf(stderr, fmt, ## __VA_ARGS__); \
+    } \
+} while (0);
 
 #define TYPE_TPM_PASSTHROUGH "tpm-passthrough"
 #define TPM_PASSTHROUGH(obj) \
@@ -71,6 +69,8 @@
     bool tpm_op_canceled;
     int cancel_fd;
     bool had_startup_error;
+
+    TPMVersion tpm_version;
 };
 
 typedef struct TPMPassthruState TPMPassthruState;
@@ -83,12 +83,37 @@
 
 static int tpm_passthrough_unix_write(int fd, const uint8_t *buf, uint32_t len)
 {
-    return send_all(fd, buf, len);
+    int ret, remain;
+
+    remain = len;
+    while (remain > 0) {
+        ret = write(fd, buf, remain);
+        if (ret < 0) {
+            if (errno != EINTR && errno != EAGAIN) {
+                return -1;
+            }
+        } else if (ret == 0) {
+            break;
+        } else {
+            buf += ret;
+            remain -= ret;
+        }
+    }
+    return len - remain;
 }
 
 static int tpm_passthrough_unix_read(int fd, uint8_t *buf, uint32_t len)
 {
-    return recv_all(fd, buf, len, true);
+    int ret;
+ reread:
+    ret = read(fd, buf, len);
+    if (ret < 0) {
+        if (errno != EINTR && errno != EAGAIN) {
+            return -1;
+        }
+        goto reread;
+    }
+    return ret;
 }
 
 static uint32_t tpm_passthrough_get_size_from_buffer(const uint8_t *buf)
@@ -112,21 +137,38 @@
     }
 }
 
+static bool tpm_passthrough_is_selftest(const uint8_t *in, uint32_t in_len)
+{
+    struct tpm_req_hdr *hdr = (struct tpm_req_hdr *)in;
+
+    if (in_len >= sizeof(*hdr)) {
+        return (be32_to_cpu(hdr->ordinal) == TPM_ORD_ContinueSelfTest);
+    }
+
+    return false;
+}
+
 static int tpm_passthrough_unix_tx_bufs(TPMPassthruState *tpm_pt,
                                         const uint8_t *in, uint32_t in_len,
-                                        uint8_t *out, uint32_t out_len)
+                                        uint8_t *out, uint32_t out_len,
+                                        bool *selftest_done)
 {
     int ret;
+    bool is_selftest;
+    const struct tpm_resp_hdr *hdr;
 
     tpm_pt->tpm_op_canceled = false;
     tpm_pt->tpm_executing = true;
+    *selftest_done = false;
+
+    is_selftest = tpm_passthrough_is_selftest(in, in_len);
 
     ret = tpm_passthrough_unix_write(tpm_pt->tpm_fd, in, in_len);
     if (ret != in_len) {
         if (!tpm_pt->tpm_op_canceled ||
             (tpm_pt->tpm_op_canceled && errno != ECANCELED)) {
             error_report("tpm_passthrough: error while transmitting data "
-                         "to TPM: %s (%i)\n",
+                         "to TPM: %s (%i)",
                          strerror(errno), errno);
         }
         goto err_exit;
@@ -139,14 +181,19 @@
         if (!tpm_pt->tpm_op_canceled ||
             (tpm_pt->tpm_op_canceled && errno != ECANCELED)) {
             error_report("tpm_passthrough: error while reading data from "
-                         "TPM: %s (%i)\n",
+                         "TPM: %s (%i)",
                          strerror(errno), errno);
         }
     } else if (ret < sizeof(struct tpm_resp_hdr) ||
                tpm_passthrough_get_size_from_buffer(out) != ret) {
         ret = -1;
         error_report("tpm_passthrough: received invalid response "
-                     "packet from TPM\n");
+                     "packet from TPM");
+    }
+
+    if (is_selftest && (ret >= sizeof(struct tpm_resp_hdr))) {
+        hdr = (struct tpm_resp_hdr *)out;
+        *selftest_done = (be32_to_cpu(hdr->errcode) == 0);
     }
 
 err_exit:
@@ -160,13 +207,15 @@
 }
 
 static int tpm_passthrough_unix_transfer(TPMPassthruState *tpm_pt,
-                                         const TPMLocality *locty_data)
+                                         const TPMLocality *locty_data,
+                                         bool *selftest_done)
 {
     return tpm_passthrough_unix_tx_bufs(tpm_pt,
                                         locty_data->w_buffer.buffer,
                                         locty_data->w_offset,
                                         locty_data->r_buffer.buffer,
-                                        locty_data->r_buffer.size);
+                                        locty_data->r_buffer.size,
+                                        selftest_done);
 }
 
 static void tpm_passthrough_worker_thread(gpointer data,
@@ -175,16 +224,19 @@
     TPMPassthruThreadParams *thr_parms = user_data;
     TPMPassthruState *tpm_pt = TPM_PASSTHROUGH(thr_parms->tb);
     TPMBackendCmd cmd = (TPMBackendCmd)data;
+    bool selftest_done = false;
 
     DPRINTF("tpm_passthrough: processing command type %d\n", cmd);
 
     switch (cmd) {
     case TPM_BACKEND_CMD_PROCESS_CMD:
         tpm_passthrough_unix_transfer(tpm_pt,
-                                      thr_parms->tpm_state->locty_data);
+                                      thr_parms->tpm_state->locty_data,
+                                      &selftest_done);
 
         thr_parms->recv_data_callback(thr_parms->tpm_state,
-                                      thr_parms->tpm_state->locty_number);
+                                      thr_parms->tpm_state->locty_number,
+                                      selftest_done);
         break;
     case TPM_BACKEND_CMD_INIT:
     case TPM_BACKEND_CMD_END:
@@ -242,6 +294,13 @@
     return false;
 }
 
+static int tpm_passthrough_reset_tpm_established_flag(TPMBackend *tb,
+                                                      uint8_t locty)
+{
+    /* only a TPM 2.0 will support this */
+    return 0;
+}
+
 static bool tpm_passthrough_get_startup_error(TPMBackend *tb)
 {
     TPMPassthruState *tpm_pt = TPM_PASSTHROUGH(tb);
@@ -282,7 +341,7 @@
         if (tpm_pt->cancel_fd >= 0) {
             n = write(tpm_pt->cancel_fd, "-", 1);
             if (n != 1) {
-                error_report("Canceling TPM command failed: %s\n",
+                error_report("Canceling TPM command failed: %s",
                              strerror(errno));
             } else {
                 tpm_pt->tpm_op_canceled = true;
@@ -299,56 +358,11 @@
     return "Passthrough TPM backend driver";
 }
 
-/*
- * A basic test of a TPM device. We expect a well formatted response header
- * (error response is fine) within one second.
- */
-static int tpm_passthrough_test_tpmdev(int fd)
+static TPMVersion tpm_passthrough_get_tpm_version(TPMBackend *tb)
 {
-    struct tpm_req_hdr req = {
-        .tag = cpu_to_be16(TPM_TAG_RQU_COMMAND),
-        .len = cpu_to_be32(sizeof(req)),
-        .ordinal = cpu_to_be32(TPM_ORD_GetTicks),
-    };
-    struct tpm_resp_hdr *resp;
-    fd_set readfds;
-    int n;
-    struct timeval tv = {
-        .tv_sec = 1,
-        .tv_usec = 0,
-    };
-    unsigned char buf[1024];
+    TPMPassthruState *tpm_pt = TPM_PASSTHROUGH(tb);
 
-    n = write(fd, &req, sizeof(req));
-    if (n < 0) {
-        return errno;
-    }
-    if (n != sizeof(req)) {
-        return EFAULT;
-    }
-
-    FD_ZERO(&readfds);
-    FD_SET(fd, &readfds);
-
-    /* wait for a second */
-    n = select(fd + 1, &readfds, NULL, NULL, &tv);
-    if (n != 1) {
-        return errno;
-    }
-
-    n = read(fd, &buf, sizeof(buf));
-    if (n < sizeof(struct tpm_resp_hdr)) {
-        return EFAULT;
-    }
-
-    resp = (struct tpm_resp_hdr *)buf;
-    /* check the header */
-    if (be16_to_cpu(resp->tag) != TPM_TAG_RSP_COMMAND ||
-        be32_to_cpu(resp->len) != n) {
-        return EBADMSG;
-    }
-
-    return 0;
+    return tpm_pt->tpm_version;
 }
 
 /*
@@ -400,9 +414,7 @@
     const char *value;
 
     value = qemu_opt_get(opts, "cancel-path");
-    if (value) {
-        tb->cancel_path = g_strdup(value);
-    }
+    tb->cancel_path = g_strdup(value);
 
     value = qemu_opt_get(opts, "path");
     if (!value) {
@@ -415,13 +427,13 @@
 
     tpm_pt->tpm_fd = qemu_open(tpm_pt->tpm_dev, O_RDWR);
     if (tpm_pt->tpm_fd < 0) {
-        error_report("Cannot access TPM device using '%s': %s\n",
+        error_report("Cannot access TPM device using '%s': %s",
                      tpm_pt->tpm_dev, strerror(errno));
         goto err_free_parameters;
     }
 
-    if (tpm_passthrough_test_tpmdev(tpm_pt->tpm_fd)) {
-        error_report("'%s' is not a TPM device.\n",
+    if (tpm_util_test_tpmdev(tpm_pt->tpm_fd, &tpm_pt->tpm_version)) {
+        error_report("'%s' is not a TPM device.",
                      tpm_pt->tpm_dev);
         goto err_close_tpmdev;
     }
@@ -517,6 +529,8 @@
     .deliver_request          = tpm_passthrough_deliver_request,
     .cancel_cmd               = tpm_passthrough_cancel_cmd,
     .get_tpm_established_flag = tpm_passthrough_get_tpm_established_flag,
+    .reset_tpm_established_flag = tpm_passthrough_reset_tpm_established_flag,
+    .get_tpm_version          = tpm_passthrough_get_tpm_version,
 };
 
 static void tpm_passthrough_inst_init(Object *obj)
diff --git a/hw/tpm/tpm_tis.c b/hw/tpm/tpm_tis.c
index c0e7cd7..381e726 100644
--- a/hw/tpm/tpm_tis.c
+++ b/hw/tpm/tpm_tis.c
@@ -14,11 +14,15 @@
  *
  * Implementation of the TIS interface according to specs found at
  * http://www.trustedcomputinggroup.org. This implementation currently
- * supports version 1.21, revision 1.0.
+ * supports version 1.3, 21 March 2013
  * In the developers menu choose the PC Client section then find the TIS
  * specification.
+ *
+ * TPM TIS for TPM 2 implementation following TCG PC Client Platform
+ * TPM Profile (PTP) Specification, Familiy 2.0, Revision 00.43
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/tpm_backend.h"
 #include "tpm_int.h"
 #include "sysemu/block-backend.h"
@@ -27,18 +31,18 @@
 #include "hw/i386/pc.h"
 #include "hw/pci/pci_ids.h"
 #include "tpm_tis.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
+#include "sysemu/tpm_backend.h"
 
-/*#define DEBUG_TIS */
+#define DEBUG_TIS 0
 
-#ifdef DEBUG_TIS
-#define DPRINTF(fmt, ...) \
-    do { fprintf(stderr, fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
+#define DPRINTF(fmt, ...) do { \
+    if (DEBUG_TIS) { \
+        printf(fmt, ## __VA_ARGS__); \
+    } \
+} while (0);
 
 /* whether the STS interrupt is supported */
 #define RAISE_STS_IRQ
@@ -51,17 +55,27 @@
 #define TPM_TIS_REG_INTF_CAPABILITY       0x14
 #define TPM_TIS_REG_STS                   0x18
 #define TPM_TIS_REG_DATA_FIFO             0x24
+#define TPM_TIS_REG_INTERFACE_ID          0x30
+#define TPM_TIS_REG_DATA_XFIFO            0x80
+#define TPM_TIS_REG_DATA_XFIFO_END        0xbc
 #define TPM_TIS_REG_DID_VID               0xf00
 #define TPM_TIS_REG_RID                   0xf04
 
 /* vendor-specific registers */
 #define TPM_TIS_REG_DEBUG                 0xf90
 
+#define TPM_TIS_STS_TPM_FAMILY_MASK         (0x3 << 26)/* TPM 2.0 */
+#define TPM_TIS_STS_TPM_FAMILY1_2           (0 << 26)  /* TPM 2.0 */
+#define TPM_TIS_STS_TPM_FAMILY2_0           (1 << 26)  /* TPM 2.0 */
+#define TPM_TIS_STS_RESET_ESTABLISHMENT_BIT (1 << 25)  /* TPM 2.0 */
+#define TPM_TIS_STS_COMMAND_CANCEL          (1 << 24)  /* TPM 2.0 */
+
 #define TPM_TIS_STS_VALID                 (1 << 7)
 #define TPM_TIS_STS_COMMAND_READY         (1 << 6)
 #define TPM_TIS_STS_TPM_GO                (1 << 5)
 #define TPM_TIS_STS_DATA_AVAILABLE        (1 << 4)
 #define TPM_TIS_STS_EXPECT                (1 << 3)
+#define TPM_TIS_STS_SELFTEST_DONE         (1 << 2)
 #define TPM_TIS_STS_RESPONSE_RETRY        (1 << 1)
 
 #define TPM_TIS_BURST_COUNT_SHIFT         8
@@ -100,9 +114,43 @@
 
 #endif
 
+#define TPM_TIS_CAP_INTERFACE_VERSION1_3 (2 << 28)
+#define TPM_TIS_CAP_INTERFACE_VERSION1_3_FOR_TPM2_0 (3 << 28)
+#define TPM_TIS_CAP_DATA_TRANSFER_64B    (3 << 9)
+#define TPM_TIS_CAP_DATA_TRANSFER_LEGACY (0 << 9)
+#define TPM_TIS_CAP_BURST_COUNT_DYNAMIC  (0 << 8)
 #define TPM_TIS_CAP_INTERRUPT_LOW_LEVEL  (1 << 4) /* support is mandatory */
-#define TPM_TIS_CAPABILITIES_SUPPORTED   (TPM_TIS_CAP_INTERRUPT_LOW_LEVEL | \
-                                          TPM_TIS_INTERRUPTS_SUPPORTED)
+#define TPM_TIS_CAPABILITIES_SUPPORTED1_3 \
+    (TPM_TIS_CAP_INTERRUPT_LOW_LEVEL | \
+     TPM_TIS_CAP_BURST_COUNT_DYNAMIC | \
+     TPM_TIS_CAP_DATA_TRANSFER_64B | \
+     TPM_TIS_CAP_INTERFACE_VERSION1_3 | \
+     TPM_TIS_INTERRUPTS_SUPPORTED)
+
+#define TPM_TIS_CAPABILITIES_SUPPORTED2_0 \
+    (TPM_TIS_CAP_INTERRUPT_LOW_LEVEL | \
+     TPM_TIS_CAP_BURST_COUNT_DYNAMIC | \
+     TPM_TIS_CAP_DATA_TRANSFER_64B | \
+     TPM_TIS_CAP_INTERFACE_VERSION1_3_FOR_TPM2_0 | \
+     TPM_TIS_INTERRUPTS_SUPPORTED)
+
+#define TPM_TIS_IFACE_ID_INTERFACE_TIS1_3   (0xf)     /* TPM 2.0 */
+#define TPM_TIS_IFACE_ID_INTERFACE_FIFO     (0x0)     /* TPM 2.0 */
+#define TPM_TIS_IFACE_ID_INTERFACE_VER_FIFO (0 << 4)  /* TPM 2.0 */
+#define TPM_TIS_IFACE_ID_CAP_5_LOCALITIES   (1 << 8)  /* TPM 2.0 */
+#define TPM_TIS_IFACE_ID_CAP_TIS_SUPPORTED  (1 << 13) /* TPM 2.0 */
+#define TPM_TIS_IFACE_ID_INT_SEL_LOCK       (1 << 19) /* TPM 2.0 */
+
+#define TPM_TIS_IFACE_ID_SUPPORTED_FLAGS1_3 \
+    (TPM_TIS_IFACE_ID_INTERFACE_TIS1_3 | \
+     (~0u << 4)/* all of it is don't care */)
+
+/* if backend was a TPM 2.0: */
+#define TPM_TIS_IFACE_ID_SUPPORTED_FLAGS2_0 \
+    (TPM_TIS_IFACE_ID_INTERFACE_FIFO | \
+     TPM_TIS_IFACE_ID_INTERFACE_VER_FIFO | \
+     TPM_TIS_IFACE_ID_CAP_5_LOCALITIES | \
+     TPM_TIS_IFACE_ID_CAP_TIS_SUPPORTED)
 
 #define TPM_TIS_TPM_DID       0x0001
 #define TPM_TIS_TPM_VID       PCI_VENDOR_ID_IBM
@@ -145,6 +193,25 @@
 }
 
 /*
+ * Set the given flags in the STS register by clearing the register but
+ * preserving the SELFTEST_DONE and TPM_FAMILY_MASK flags and then setting
+ * the new flags.
+ *
+ * The SELFTEST_DONE flag is acquired from the backend that determines it by
+ * peeking into TPM commands.
+ *
+ * A VM suspend/resume will preserve the flag by storing it into the VM
+ * device state, but the backend will not remember it when QEMU is started
+ * again. Therefore, we cache the flag here. Once set, it will not be unset
+ * except by a reset.
+ */
+static void tpm_tis_sts_set(TPMLocality *l, uint32_t flags)
+{
+    l->sts &= TPM_TIS_STS_SELFTEST_DONE | TPM_TIS_STS_TPM_FAMILY_MASK;
+    l->sts |= flags;
+}
+
+/*
  * Send a request to the TPM.
  */
 static void tpm_tis_tpm_send(TPMState *s, uint8_t locty)
@@ -255,7 +322,8 @@
      */
     if (tis->aborting_locty == tis->next_locty) {
         tis->loc[tis->aborting_locty].state = TPM_TIS_STATE_READY;
-        tis->loc[tis->aborting_locty].sts = TPM_TIS_STS_COMMAND_READY;
+        tpm_tis_sts_set(&tis->loc[tis->aborting_locty],
+                        TPM_TIS_STS_COMMAND_READY);
         tpm_tis_raise_irq(s, tis->aborting_locty, TPM_TIS_INT_COMMAND_READY);
     }
 
@@ -300,7 +368,8 @@
     TPMTISEmuState *tis = &s->s.tis;
     uint8_t locty = s->locty_number;
 
-    tis->loc[locty].sts = TPM_TIS_STS_VALID | TPM_TIS_STS_DATA_AVAILABLE;
+    tpm_tis_sts_set(&tis->loc[locty],
+                    TPM_TIS_STS_VALID | TPM_TIS_STS_DATA_AVAILABLE);
     tis->loc[locty].state = TPM_TIS_STATE_COMPLETION;
     tis->loc[locty].r_offset = 0;
     tis->loc[locty].w_offset = 0;
@@ -320,12 +389,20 @@
 /*
  * Callback from the TPM to indicate that the response was received.
  */
-static void tpm_tis_receive_cb(TPMState *s, uint8_t locty)
+static void tpm_tis_receive_cb(TPMState *s, uint8_t locty,
+                               bool is_selftest_done)
 {
     TPMTISEmuState *tis = &s->s.tis;
+    uint8_t l;
 
     assert(s->locty_number == locty);
 
+    if (is_selftest_done) {
+        for (l = 0; l < TPM_TIS_NUM_LOCALITIES; l++) {
+            tis->loc[locty].sts |= TPM_TIS_STS_SELFTEST_DONE;
+        }
+    }
+
     qemu_bh_schedule(tis->bh);
 }
 
@@ -344,7 +421,7 @@
         ret = tis->loc[locty].r_buffer.buffer[tis->loc[locty].r_offset++];
         if (tis->loc[locty].r_offset >= len) {
             /* got last byte */
-            tis->loc[locty].sts = TPM_TIS_STS_VALID;
+            tpm_tis_sts_set(&tis->loc[locty], TPM_TIS_STS_VALID);
 #ifdef RAISE_STS_IRQ
             tpm_tis_raise_irq(s, locty, TPM_TIS_INT_STS_VALID);
 #endif
@@ -383,7 +460,7 @@
 
     for (idx = 0; regs[idx] != 0xfff; idx++) {
         DPRINTF("tpm_tis: 0x%04x : 0x%08x\n", regs[idx],
-                (uint32_t)tpm_tis_mmio_read(opaque, base + regs[idx], 4));
+                (int)tpm_tis_mmio_read(opaque, base + regs[idx], 4));
     }
 
     DPRINTF("tpm_tis: read offset   : %d\n"
@@ -427,6 +504,7 @@
     uint32_t val = 0xffffffff;
     uint8_t locty = tpm_tis_locality_from_addr(addr);
     uint32_t avail;
+    uint8_t v;
 
     if (tpm_backend_had_startup_error(s->be_driver)) {
         return val;
@@ -452,7 +530,17 @@
         val = tis->loc[locty].ints;
         break;
     case TPM_TIS_REG_INTF_CAPABILITY:
-        val = TPM_TIS_CAPABILITIES_SUPPORTED;
+        switch (s->be_tpm_version) {
+        case TPM_VERSION_UNSPEC:
+            val = 0;
+            break;
+        case TPM_VERSION_1_2:
+            val = TPM_TIS_CAPABILITIES_SUPPORTED1_3;
+            break;
+        case TPM_VERSION_2_0:
+            val = TPM_TIS_CAPABILITIES_SUPPORTED2_0;
+            break;
+        }
         break;
     case TPM_TIS_REG_STS:
         if (tis->active_locty == locty) {
@@ -475,17 +563,33 @@
         }
         break;
     case TPM_TIS_REG_DATA_FIFO:
+    case TPM_TIS_REG_DATA_XFIFO ... TPM_TIS_REG_DATA_XFIFO_END:
         if (tis->active_locty == locty) {
-            switch (tis->loc[locty].state) {
-            case TPM_TIS_STATE_COMPLETION:
-                val = tpm_tis_data_read(s, locty);
-                break;
-            default:
-                val = TPM_TIS_NO_DATA_BYTE;
-                break;
+            if (size > 4 - (addr & 0x3)) {
+                /* prevent access beyond FIFO */
+                size = 4 - (addr & 0x3);
             }
+            val = 0;
+            shift = 0;
+            while (size > 0) {
+                switch (tis->loc[locty].state) {
+                case TPM_TIS_STATE_COMPLETION:
+                    v = tpm_tis_data_read(s, locty);
+                    break;
+                default:
+                    v = TPM_TIS_NO_DATA_BYTE;
+                    break;
+                }
+                val |= (v << shift);
+                shift += 8;
+                size--;
+            }
+            shift = 0; /* no more adjustments */
         }
         break;
+    case TPM_TIS_REG_INTERFACE_ID:
+        val = tis->loc[locty].iface_id;
+        break;
     case TPM_TIS_REG_DID_VID:
         val = (TPM_TIS_TPM_DID << 16) | TPM_TIS_TPM_VID;
         break;
@@ -503,7 +607,7 @@
         val >>= shift;
     }
 
-    DPRINTF("tpm_tis:  read.%u(%08x) = %08x\n", size, (int)addr, (uint32_t)val);
+    DPRINTF("tpm_tis:  read.%u(%08x) = %08x\n", size, (int)addr, (int)val);
 
     return val;
 }
@@ -518,13 +622,15 @@
 {
     TPMState *s = opaque;
     TPMTISEmuState *tis = &s->s.tis;
-    uint16_t off = addr & 0xfff;
+    uint16_t off = addr & 0xffc;
+    uint8_t shift = (addr & 0x3) * 8;
     uint8_t locty = tpm_tis_locality_from_addr(addr);
     uint8_t active_locty, l;
     int c, set_new_locty = 1;
     uint16_t len;
+    uint32_t mask = (size == 1) ? 0xff : ((size == 2) ? 0xffff : ~0);
 
-    DPRINTF("tpm_tis: write.%u(%08x) = %08x\n", size, (int)addr, (uint32_t)val);
+    DPRINTF("tpm_tis: write.%u(%08x) = %08x\n", size, (int)addr, (int)val);
 
     if (locty == 4 && !hw_access) {
         DPRINTF("tpm_tis: Access to locality 4 only allowed from hardware\n");
@@ -535,6 +641,15 @@
         return;
     }
 
+    val &= mask;
+
+    if (shift) {
+        val <<= shift;
+        mask <<= shift;
+    }
+
+    mask ^= 0xffffffff;
+
     switch (off) {
     case TPM_TIS_REG_ACCESS:
 
@@ -646,9 +761,10 @@
             break;
         }
 
-        tis->loc[locty].inte = (val & (TPM_TIS_INT_ENABLED |
-                                       TPM_TIS_INT_POLARITY_MASK |
-                                       TPM_TIS_INTERRUPTS_SUPPORTED));
+        tis->loc[locty].inte &= mask;
+        tis->loc[locty].inte |= (val & (TPM_TIS_INT_ENABLED |
+                                        TPM_TIS_INT_POLARITY_MASK |
+                                        TPM_TIS_INTERRUPTS_SUPPORTED));
         break;
     case TPM_TIS_REG_INT_VECTOR:
         /* hard wired -- ignore */
@@ -674,6 +790,25 @@
             break;
         }
 
+        if (s->be_tpm_version == TPM_VERSION_2_0) {
+            /* some flags that are only supported for TPM 2 */
+            if (val & TPM_TIS_STS_COMMAND_CANCEL) {
+                if (tis->loc[locty].state == TPM_TIS_STATE_EXECUTION) {
+                    /*
+                     * request the backend to cancel. Some backends may not
+                     * support it
+                     */
+                    tpm_backend_cancel_cmd(s->be_driver);
+                }
+            }
+
+            if (val & TPM_TIS_STS_RESET_ESTABLISHMENT_BIT) {
+                if (locty == 3 || locty == 4) {
+                    tpm_backend_reset_tpm_established_flag(s->be_driver, locty);
+                }
+            }
+        }
+
         val &= (TPM_TIS_STS_COMMAND_READY | TPM_TIS_STS_TPM_GO |
                 TPM_TIS_STS_RESPONSE_RETRY);
 
@@ -686,7 +821,7 @@
             break;
 
             case TPM_TIS_STATE_IDLE:
-                tis->loc[locty].sts = TPM_TIS_STS_COMMAND_READY;
+                tpm_tis_sts_set(&tis->loc[locty], TPM_TIS_STS_COMMAND_READY);
                 tis->loc[locty].state = TPM_TIS_STATE_READY;
                 tpm_tis_raise_irq(s, locty, TPM_TIS_INT_COMMAND_READY);
             break;
@@ -705,7 +840,8 @@
                 /* shortcut to ready state with C/R set */
                 tis->loc[locty].state = TPM_TIS_STATE_READY;
                 if (!(tis->loc[locty].sts & TPM_TIS_STS_COMMAND_READY)) {
-                    tis->loc[locty].sts   = TPM_TIS_STS_COMMAND_READY;
+                    tpm_tis_sts_set(&tis->loc[locty],
+                                    TPM_TIS_STS_COMMAND_READY);
                     tpm_tis_raise_irq(s, locty, TPM_TIS_INT_COMMAND_READY);
                 }
                 tis->loc[locty].sts &= ~(TPM_TIS_STS_DATA_AVAILABLE);
@@ -727,8 +863,9 @@
             switch (tis->loc[locty].state) {
             case TPM_TIS_STATE_COMPLETION:
                 tis->loc[locty].r_offset = 0;
-                tis->loc[locty].sts = TPM_TIS_STS_VALID |
-                                      TPM_TIS_STS_DATA_AVAILABLE;
+                tpm_tis_sts_set(&tis->loc[locty],
+                                TPM_TIS_STS_VALID|
+                                TPM_TIS_STS_DATA_AVAILABLE);
                 break;
             default:
                 /* ignore */
@@ -737,6 +874,7 @@
         }
         break;
     case TPM_TIS_REG_DATA_FIFO:
+    case TPM_TIS_REG_DATA_XFIFO ... TPM_TIS_REG_DATA_XFIFO_END:
         /* data fifo */
         if (tis->active_locty != locty) {
             break;
@@ -747,18 +885,28 @@
             tis->loc[locty].state == TPM_TIS_STATE_COMPLETION) {
             /* drop the byte */
         } else {
-            DPRINTF("tpm_tis: Byte to send to TPM: %02x\n", (uint8_t)val);
+            DPRINTF("tpm_tis: Data to send to TPM: %08x (size=%d)\n",
+                    (int)val, size);
             if (tis->loc[locty].state == TPM_TIS_STATE_READY) {
                 tis->loc[locty].state = TPM_TIS_STATE_RECEPTION;
-                tis->loc[locty].sts = TPM_TIS_STS_EXPECT | TPM_TIS_STS_VALID;
+                tpm_tis_sts_set(&tis->loc[locty],
+                                TPM_TIS_STS_EXPECT | TPM_TIS_STS_VALID);
             }
 
-            if ((tis->loc[locty].sts & TPM_TIS_STS_EXPECT)) {
+            val >>= shift;
+            if (size > 4 - (addr & 0x3)) {
+                /* prevent access beyond FIFO */
+                size = 4 - (addr & 0x3);
+            }
+
+            while ((tis->loc[locty].sts & TPM_TIS_STS_EXPECT) && size > 0) {
                 if (tis->loc[locty].w_offset < tis->loc[locty].w_buffer.size) {
                     tis->loc[locty].w_buffer.
                         buffer[tis->loc[locty].w_offset++] = (uint8_t)val;
+                    val >>= 8;
+                    size--;
                 } else {
-                    tis->loc[locty].sts = TPM_TIS_STS_VALID;
+                    tpm_tis_sts_set(&tis->loc[locty], TPM_TIS_STS_VALID);
                 }
             }
 
@@ -767,31 +915,38 @@
                 (tis->loc[locty].sts & TPM_TIS_STS_EXPECT)) {
                 /* we have a packet length - see if we have all of it */
 #ifdef RAISE_STS_IRQ
-                bool needIrq = !(tis->loc[locty].sts & TPM_TIS_STS_VALID);
+                bool need_irq = !(tis->loc[locty].sts & TPM_TIS_STS_VALID);
 #endif
                 len = tpm_tis_get_size_from_buffer(&tis->loc[locty].w_buffer);
                 if (len > tis->loc[locty].w_offset) {
-                    tis->loc[locty].sts = TPM_TIS_STS_EXPECT |
-                                          TPM_TIS_STS_VALID;
+                    tpm_tis_sts_set(&tis->loc[locty],
+                                    TPM_TIS_STS_EXPECT | TPM_TIS_STS_VALID);
                 } else {
                     /* packet complete */
-                    tis->loc[locty].sts = TPM_TIS_STS_VALID;
+                    tpm_tis_sts_set(&tis->loc[locty], TPM_TIS_STS_VALID);
                 }
 #ifdef RAISE_STS_IRQ
-                if (needIrq) {
+                if (need_irq) {
                     tpm_tis_raise_irq(s, locty, TPM_TIS_INT_STS_VALID);
                 }
 #endif
             }
         }
         break;
+    case TPM_TIS_REG_INTERFACE_ID:
+        if (val & TPM_TIS_IFACE_ID_INT_SEL_LOCK) {
+            for (l = 0; l < TPM_TIS_NUM_LOCALITIES; l++) {
+                tis->loc[l].iface_id |= TPM_TIS_IFACE_ID_INT_SEL_LOCK;
+            }
+        }
+        break;
     }
 }
 
 static void tpm_tis_mmio_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
-    return tpm_tis_mmio_write_intern(opaque, addr, val, size, false);
+    tpm_tis_mmio_write_intern(opaque, addr, val, size, false);
 }
 
 static const MemoryRegionOps tpm_tis_memory_ops = {
@@ -810,6 +965,16 @@
 }
 
 /*
+ * Get the TPMVersion of the backend device being used
+ */
+TPMVersion tpm_tis_get_tpm_version(Object *obj)
+{
+    TPMState *s = TPM(obj);
+
+    return tpm_backend_get_tpm_version(s->be_driver);
+}
+
+/*
  * This function is called when the machine starts, resets or due to
  * S3 resume.
  */
@@ -819,6 +984,8 @@
     TPMTISEmuState *tis = &s->s.tis;
     int c;
 
+    s->be_tpm_version = tpm_backend_get_tpm_version(s->be_driver);
+
     tpm_backend_reset(s->be_driver);
 
     tis->active_locty = TPM_TIS_NO_LOCALITY;
@@ -827,7 +994,18 @@
 
     for (c = 0; c < TPM_TIS_NUM_LOCALITIES; c++) {
         tis->loc[c].access = TPM_TIS_ACCESS_TPM_REG_VALID_STS;
-        tis->loc[c].sts = 0;
+        switch (s->be_tpm_version) {
+        case TPM_VERSION_UNSPEC:
+            break;
+        case TPM_VERSION_1_2:
+            tis->loc[c].sts = TPM_TIS_STS_TPM_FAMILY1_2;
+            tis->loc[c].iface_id = TPM_TIS_IFACE_ID_SUPPORTED_FLAGS1_3;
+            break;
+        case TPM_VERSION_2_0:
+            tis->loc[c].sts = TPM_TIS_STS_TPM_FAMILY2_0;
+            tis->loc[c].iface_id = TPM_TIS_IFACE_ID_SUPPORTED_FLAGS2_0;
+            break;
+        }
         tis->loc[c].inte = TPM_TIS_INT_POLARITY_LOW_LEVEL;
         tis->loc[c].ints = 0;
         tis->loc[c].state = TPM_TIS_STATE_IDLE;
@@ -875,25 +1053,25 @@
 
     if (tis->irq_num > 15) {
         error_setg(errp, "tpm_tis: IRQ %d for TPM TIS is outside valid range "
-                   "of 0 to 15.\n", tis->irq_num);
+                   "of 0 to 15", tis->irq_num);
         return;
     }
 
     tis->bh = qemu_bh_new(tpm_tis_receive_bh, s);
 
     isa_init_irq(&s->busdev, &tis->irq, tis->irq_num);
+
+    memory_region_add_subregion(isa_address_space(ISA_DEVICE(dev)),
+                                TPM_TIS_ADDR_BASE, &s->mmio);
 }
 
 static void tpm_tis_initfn(Object *obj)
 {
-    ISADevice *dev = ISA_DEVICE(obj);
     TPMState *s = TPM(obj);
 
     memory_region_init_io(&s->mmio, OBJECT(s), &tpm_tis_memory_ops,
                           s, "tpm-tis-mmio",
                           TPM_TIS_NUM_LOCALITIES << TPM_TIS_LOCALITY_SHIFT);
-    memory_region_add_subregion(isa_address_space(dev), TPM_TIS_ADDR_BASE,
-                                &s->mmio);
 }
 
 static void tpm_tis_class_init(ObjectClass *klass, void *data)
diff --git a/hw/tpm/tpm_tis.h b/hw/tpm/tpm_tis.h
index 1a0db23..a1df41f 100644
--- a/hw/tpm/tpm_tis.h
+++ b/hw/tpm/tpm_tis.h
@@ -41,7 +41,8 @@
 typedef struct TPMLocality {
     TPMTISState state;
     uint8_t access;
-    uint8_t sts;
+    uint32_t sts;
+    uint32_t iface_id;
     uint32_t inte;
     uint32_t ints;
 
diff --git a/hw/tpm/tpm_util.c b/hw/tpm/tpm_util.c
new file mode 100644
index 0000000..7b35429
--- /dev/null
+++ b/hw/tpm/tpm_util.c
@@ -0,0 +1,127 @@
+/*
+ * TPM utility functions
+ *
+ *  Copyright (c) 2010 - 2015 IBM Corporation
+ *  Authors:
+ *    Stefan Berger <stefanb@us.ibm.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu/osdep.h"
+#include "tpm_util.h"
+#include "tpm_int.h"
+
+/*
+ * A basic test of a TPM device. We expect a well formatted response header
+ * (error response is fine) within one second.
+ */
+static int tpm_util_test(int fd,
+                         unsigned char *request,
+                         size_t requestlen,
+                         uint16_t *return_tag)
+{
+    struct tpm_resp_hdr *resp;
+    fd_set readfds;
+    int n;
+    struct timeval tv = {
+        .tv_sec = 1,
+        .tv_usec = 0,
+    };
+    unsigned char buf[1024];
+
+    n = write(fd, request, requestlen);
+    if (n < 0) {
+        return errno;
+    }
+    if (n != requestlen) {
+        return EFAULT;
+    }
+
+    FD_ZERO(&readfds);
+    FD_SET(fd, &readfds);
+
+    /* wait for a second */
+    n = select(fd + 1, &readfds, NULL, NULL, &tv);
+    if (n != 1) {
+        return errno;
+    }
+
+    n = read(fd, &buf, sizeof(buf));
+    if (n < sizeof(struct tpm_resp_hdr)) {
+        return EFAULT;
+    }
+
+    resp = (struct tpm_resp_hdr *)buf;
+    /* check the header */
+    if (be32_to_cpu(resp->len) != n) {
+        return EBADMSG;
+    }
+
+    *return_tag = be16_to_cpu(resp->tag);
+
+    return 0;
+}
+
+/*
+ * Probe for the TPM device in the back
+ * Returns 0 on success with the version of the probed TPM set, 1 on failure.
+ */
+int tpm_util_test_tpmdev(int tpm_fd, TPMVersion *tpm_version)
+{
+    /*
+     * Sending a TPM1.2 command to a TPM2 should return a TPM1.2
+     * header (tag = 0xc4) and error code (TPM_BADTAG = 0x1e)
+     *
+     * Sending a TPM2 command to a TPM 2 will give a TPM 2 tag in the
+     * header.
+     * Sending a TPM2 command to a TPM 1.2 will give a TPM 1.2 tag
+     * in the header and an error code.
+     */
+    const struct tpm_req_hdr test_req = {
+        .tag = cpu_to_be16(TPM_TAG_RQU_COMMAND),
+        .len = cpu_to_be32(sizeof(test_req)),
+        .ordinal = cpu_to_be32(TPM_ORD_GetTicks),
+    };
+
+    const struct tpm_req_hdr test_req_tpm2 = {
+        .tag = cpu_to_be16(TPM2_ST_NO_SESSIONS),
+        .len = cpu_to_be32(sizeof(test_req_tpm2)),
+        .ordinal = cpu_to_be32(TPM2_CC_ReadClock),
+    };
+    uint16_t return_tag;
+    int ret;
+
+    /* Send TPM 2 command */
+    ret = tpm_util_test(tpm_fd, (unsigned char *)&test_req_tpm2,
+                        sizeof(test_req_tpm2), &return_tag);
+    /* TPM 2 would respond with a tag of TPM2_ST_NO_SESSIONS */
+    if (!ret && return_tag == TPM2_ST_NO_SESSIONS) {
+        *tpm_version = TPM_VERSION_2_0;
+        return 0;
+    }
+
+    /* Send TPM 1.2 command */
+    ret = tpm_util_test(tpm_fd, (unsigned char *)&test_req,
+                        sizeof(test_req), &return_tag);
+    if (!ret && return_tag == TPM_TAG_RSP_COMMAND) {
+        *tpm_version = TPM_VERSION_1_2;
+        /* this is a TPM 1.2 */
+        return 0;
+    }
+
+    *tpm_version = TPM_VERSION_UNSPEC;
+
+    return 1;
+}
diff --git a/hw/tpm/tpm_util.h b/hw/tpm/tpm_util.h
new file mode 100644
index 0000000..df76245
--- /dev/null
+++ b/hw/tpm/tpm_util.h
@@ -0,0 +1,29 @@
+/*
+ * TPM utility functions
+ *
+ *  Copyright (c) 2010 - 2015 IBM Corporation
+ *  Authors:
+ *    Stefan Berger <stefanb@us.ibm.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef TPM_TPM_UTIL_H
+#define TPM_TPM_UTIL_H
+
+#include "sysemu/tpm_backend.h"
+
+int tpm_util_test_tpmdev(int tpm_fd, TPMVersion *tpm_version);
+
+#endif /* TPM_TPM_UTIL_H */
diff --git a/hw/tricore/tricore_testboard.c b/hw/tricore/tricore_testboard.c
index a059a20..8d3520f 100644
--- a/hw/tricore/tricore_testboard.c
+++ b/hw/tricore/tricore_testboard.c
@@ -18,6 +18,10 @@
  */
 
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/devices.h"
 #include "net/net.h"
@@ -44,7 +48,7 @@
     kernel_size = load_elf(tricoretb_binfo.kernel_filename, NULL,
                            NULL, (uint64_t *)&entry, NULL,
                            NULL, 0,
-                           ELF_MACHINE, 1);
+                           EM_TRICORE, 1, 0);
     if (kernel_size <= 0) {
         error_report("qemu: no kernel file '%s'",
                 tricoretb_binfo.kernel_filename);
@@ -76,17 +80,23 @@
         exit(1);
     }
     env = &cpu->env;
-    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024, &error_abort);
+    memory_region_init_ram(ext_cram, NULL, "powerlink_ext_c.ram", 2*1024*1024,
+                           &error_fatal);
     vmstate_register_ram_global(ext_cram);
-    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024, &error_abort);
+    memory_region_init_ram(ext_dram, NULL, "powerlink_ext_d.ram", 4*1024*1024,
+                           &error_fatal);
     vmstate_register_ram_global(ext_dram);
-    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024, &error_abort);
+    memory_region_init_ram(int_cram, NULL, "powerlink_int_c.ram", 48*1024,
+                           &error_fatal);
     vmstate_register_ram_global(int_cram);
-    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024, &error_abort);
+    memory_region_init_ram(int_dram, NULL, "powerlink_int_d.ram", 48*1024,
+                           &error_fatal);
     vmstate_register_ram_global(int_dram);
-    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024, &error_abort);
+    memory_region_init_ram(pcp_data, NULL, "powerlink_pcp_data.ram", 16*1024,
+                           &error_fatal);
     vmstate_register_ram_global(pcp_data);
-    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024, &error_abort);
+    memory_region_init_ram(pcp_text, NULL, "powerlink_pcp_text.ram", 32*1024,
+                           &error_fatal);
     vmstate_register_ram_global(pcp_text);
 
     memory_region_add_subregion(sysmem, 0x80000000, ext_cram);
@@ -109,16 +119,11 @@
     tricore_testboard_init(machine, 0x183);
 }
 
-static QEMUMachine ttb_machine = {
-    .name = "tricore_testboard",
-    .desc = "a minimal TriCore board",
-    .init = tricoreboard_init,
-    .is_default = 0,
-};
-
-static void tricore_testboard_machine_init(void)
+static void ttb_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&ttb_machine);
+    mc->desc = "a minimal TriCore board";
+    mc->init = tricoreboard_init;
+    mc->is_default = 0;
 }
 
-machine_init(tricore_testboard_machine_init);
+DEFINE_MACHINE("tricore_testboard", ttb_machine_init)
diff --git a/hw/unicore32/puv3.c b/hw/unicore32/puv3.c
index c41499e..31cd171 100644
--- a/hw/unicore32/puv3.c
+++ b/hw/unicore32/puv3.c
@@ -9,6 +9,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "elf.h"
@@ -17,6 +21,7 @@
 #include "hw/boards.h"
 #include "hw/loader.h"
 #include "hw/i386/pc.h"
+#include "qemu/error-report.h"
 #include "sysemu/qtest.h"
 
 #undef DEBUG_PUV3
@@ -40,15 +45,15 @@
 
 static void puv3_soc_init(CPUUniCore32State *env)
 {
-    qemu_irq *cpu_intc, irqs[PUV3_IRQS_NR];
+    qemu_irq cpu_intc, irqs[PUV3_IRQS_NR];
     DeviceState *dev;
     MemoryRegion *i8042 = g_new(MemoryRegion, 1);
     int i;
 
     /* Initialize interrupt controller */
-    cpu_intc = qemu_allocate_irqs(puv3_intc_cpu_handler,
-                                  uc32_env_get_cpu(env), 1);
-    dev = sysbus_create_simple("puv3_intc", PUV3_INTC_BASE, *cpu_intc);
+    cpu_intc = qemu_allocate_irq(puv3_intc_cpu_handler,
+                                 uc32_env_get_cpu(env), 0);
+    dev = sysbus_create_simple("puv3_intc", PUV3_INTC_BASE, cpu_intc);
     for (i = 0; i < PUV3_IRQS_NR; i++) {
         irqs[i] = qdev_get_gpio_in(dev, i);
     }
@@ -75,7 +80,7 @@
 
     /* SDRAM at address zero.  */
     memory_region_init_ram(ram_memory, NULL, "puv3.ram", ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram_memory);
     memory_region_add_subregion(get_system_memory(), 0, ram_memory);
 }
@@ -95,7 +100,8 @@
     size = load_image_targphys(kernel_filename, KERNEL_LOAD_ADDR,
             KERNEL_MAX_SIZE);
     if (size < 0) {
-        hw_error("Load kernel error: '%s'\n", kernel_filename);
+        error_report("Load kernel error: '%s'", kernel_filename);
+        exit(1);
     }
 
     /* cheat curses that we have a graphic console, only under ocd console */
@@ -109,35 +115,34 @@
     const char *kernel_filename = machine->kernel_filename;
     const char *initrd_filename = machine->initrd_filename;
     CPUUniCore32State *env;
+    UniCore32CPU *cpu;
 
     if (initrd_filename) {
-        hw_error("Please use kernel built-in initramdisk.\n");
+        error_report("Please use kernel built-in initramdisk");
+        exit(1);
     }
 
     if (!cpu_model) {
         cpu_model = "UniCore-II";
     }
 
-    env = cpu_init(cpu_model);
-    if (!env) {
-        hw_error("Unable to find CPU definition\n");
+    cpu = uc32_cpu_init(cpu_model);
+    if (!cpu) {
+        error_report("Unable to find CPU definition");
+        exit(1);
     }
+    env = &cpu->env;
 
     puv3_soc_init(env);
     puv3_board_init(env, ram_size);
     puv3_load_kernel(kernel_filename);
 }
 
-static QEMUMachine puv3_machine = {
-    .name = "puv3",
-    .desc = "PKUnity Version-3 based on UniCore32",
-    .init = puv3_init,
-    .is_default = 1,
-};
-
-static void puv3_machine_init(void)
+static void puv3_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&puv3_machine);
+    mc->desc = "PKUnity Version-3 based on UniCore32";
+    mc->init = puv3_init;
+    mc->is_default = 1;
 }
 
-machine_init(puv3_machine_init)
+DEFINE_MACHINE("puv3", puv3_machine_init)
diff --git a/hw/usb/Makefile.objs b/hw/usb/Makefile.objs
index 3fe4dff..98b5c9d 100644
--- a/hw/usb/Makefile.objs
+++ b/hw/usb/Makefile.objs
@@ -1,17 +1,20 @@
 # usb subsystem core
-common-obj-y += core.o combined-packet.o bus.o desc.o desc-msos.o
-common-obj-y += libhw.o
+common-obj-y += core.o combined-packet.o bus.o libhw.o
+common-obj-$(CONFIG_USB) += desc.o desc-msos.o
 
 # usb host adapters
 common-obj-$(CONFIG_USB_UHCI) += hcd-uhci.o
 common-obj-$(CONFIG_USB_OHCI) += hcd-ohci.o
-common-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o hcd-ehci-pci.o hcd-ehci-sysbus.o
+common-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o hcd-ehci-pci.o
+common-obj-$(CONFIG_USB_EHCI_SYSBUS) += hcd-ehci-sysbus.o
 common-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
 common-obj-$(CONFIG_USB_MUSB) += hcd-musb.o
 
+obj-$(CONFIG_TUSB6010) += tusb6010.o
+
 # emulated usb devices
-common-obj-y += dev-hub.o
-common-obj-y += dev-hid.o
+common-obj-$(CONFIG_USB) += dev-hub.o
+common-obj-$(CONFIG_USB) += dev-hid.o
 common-obj-$(CONFIG_USB_TABLET_WACOM) += dev-wacom.o
 common-obj-$(CONFIG_USB_STORAGE_BOT)  += dev-storage.o
 common-obj-$(CONFIG_USB_STORAGE_UAS)  += dev-uas.o
@@ -22,9 +25,8 @@
 
 ifeq ($(CONFIG_USB_SMARTCARD),y)
 common-obj-y                          += dev-smartcard-reader.o
-common-obj-y                          += ccid-card-passthru.o
-common-obj-$(CONFIG_SMARTCARD_NSS)    += ccid-card-emulated.o
-ccid-card-emulated.o-cflags := -I$(SRC_PATH)/libcacard
+common-obj-$(CONFIG_SMARTCARD)        += ccid-card-passthru.o
+common-obj-$(CONFIG_SMARTCARD)        += ccid-card-emulated.o
 endif
 
 ifeq ($(CONFIG_POSIX),y)
@@ -36,3 +38,7 @@
 
 # usb pass-through
 common-obj-y += $(patsubst %,host-%.o,$(HOST_USB))
+
+ifeq ($(CONFIG_USB_LIBUSB),y)
+common-obj-$(CONFIG_XEN_BACKEND) += xen-usb.o
+endif
diff --git a/hw/usb/bus.c b/hw/usb/bus.c
index 986b2d8..25913ad 100644
--- a/hw/usb/bus.c
+++ b/hw/usb/bus.c
@@ -1,9 +1,13 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/usb.h"
 #include "hw/qdev.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
 #include "trace.h"
+#include "qemu/cutils.h"
 
 static void usb_bus_dev_print(Monitor *mon, DeviceState *qdev, int indent);
 
@@ -51,9 +55,9 @@
     USBDevice *dev = opaque;
 
     if (dev->state == USB_STATE_NOTATTACHED) {
-        dev->attached = 0;
+        dev->attached = false;
     } else {
-        dev->attached = 1;
+        dev->attached = true;
     }
     if (dev->setup_index < 0 ||
         dev->setup_len < 0 ||
@@ -275,6 +279,13 @@
 static void usb_qdev_unrealize(DeviceState *qdev, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(qdev);
+    USBDescString *s, *next;
+
+    QLIST_FOREACH_SAFE(s, &dev->strings, next, next) {
+        QLIST_REMOVE(s, next);
+        g_free(s->str);
+        g_free(s);
+    }
 
     if (dev->attached) {
         usb_device_detach(dev);
@@ -315,23 +326,33 @@
     return USB_DEVICE(dev);
 }
 
-USBDevice *usb_create_simple(USBBus *bus, const char *name)
+static USBDevice *usb_try_create_simple(USBBus *bus, const char *name,
+                                        Error **errp)
 {
-    USBDevice *dev = usb_create(bus, name);
-    int rc;
+    Error *err = NULL;
+    USBDevice *dev;
 
+    dev = USB_DEVICE(qdev_try_create(&bus->qbus, name));
     if (!dev) {
-        error_report("Failed to create USB device '%s'", name);
+        error_setg(errp, "Failed to create USB device '%s'", name);
         return NULL;
     }
-    rc = qdev_init(&dev->qdev);
-    if (rc < 0) {
-        error_report("Failed to initialize USB device '%s'", name);
+    object_property_set_bool(OBJECT(dev), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        error_prepend(errp, "Failed to initialize USB device '%s': ",
+                      name);
+        object_unparent(OBJECT(dev));
         return NULL;
     }
     return dev;
 }
 
+USBDevice *usb_create_simple(USBBus *bus, const char *name)
+{
+    return usb_try_create_simple(bus, name, &error_abort);
+}
+
 static void usb_fill_port(USBPort *port, void *opaque, int index,
                           USBPortOps *ops, int speedmask)
 {
@@ -350,9 +371,10 @@
     bus->nfree++;
 }
 
-int usb_register_companion(const char *masterbus, USBPort *ports[],
-                           uint32_t portcount, uint32_t firstport,
-                           void *opaque, USBPortOps *ops, int speedmask)
+void usb_register_companion(const char *masterbus, USBPort *ports[],
+                            uint32_t portcount, uint32_t firstport,
+                            void *opaque, USBPortOps *ops, int speedmask,
+                            Error **errp)
 {
     USBBus *bus;
     int i;
@@ -363,22 +385,22 @@
         }
     }
 
-    if (!bus || !bus->ops->register_companion) {
-        qerror_report(QERR_INVALID_PARAMETER_VALUE, "masterbus",
-                      "an USB masterbus");
-        if (bus) {
-            error_printf_unless_qmp(
-                "USB bus '%s' does not allow companion controllers\n",
-                masterbus);
-        }
-        return -1;
+    if (!bus) {
+        error_setg(errp, "USB bus '%s' not found", masterbus);
+        return;
+    }
+    if (!bus->ops->register_companion) {
+        error_setg(errp, "Can't use USB bus '%s' as masterbus,"
+                   " it doesn't support companion controllers",
+                   masterbus);
+        return;
     }
 
     for (i = 0; i < portcount; i++) {
         usb_fill_port(ports[i], opaque, i, ops, speedmask);
     }
 
-    return bus->ops->register_companion(bus, ports, portcount, firstport);
+    bus->ops->register_companion(bus, ports, portcount, firstport, errp);
 }
 
 void usb_port_location(USBPort *downstream, USBPort *upstream, int portnr)
@@ -416,17 +438,17 @@
             }
         }
         if (port == NULL) {
-            error_setg(errp, "Error: usb port %s (bus %s) not found (in use?)",
+            error_setg(errp, "usb port %s (bus %s) not found (in use?)",
                        dev->port_path, bus->qbus.name);
             return;
         }
     } else {
         if (bus->nfree == 1 && strcmp(object_get_typename(OBJECT(dev)), "usb-hub") != 0) {
             /* Create a new hub and chain it on */
-            usb_create_simple(bus, "usb-hub");
+            usb_try_create_simple(bus, "usb-hub", NULL);
         }
         if (bus->nfree == 0) {
-            error_setg(errp, "Error: tried to attach usb device %s to a bus "
+            error_setg(errp, "tried to attach usb device %s to a bus "
                        "with no free ports", dev->product_desc);
             return;
         }
@@ -518,7 +540,7 @@
         return;
     }
 
-    dev->attached++;
+    dev->attached = true;
     usb_attach(port);
 }
 
@@ -532,7 +554,7 @@
     trace_usb_port_detach(bus->busnr, port->path);
 
     usb_detach(port);
-    dev->attached--;
+    dev->attached = false;
     return 0;
 }
 
@@ -627,7 +649,7 @@
     return fw_path;
 }
 
-void usb_info(Monitor *mon, const QDict *qdict)
+void hmp_info_usb(Monitor *mon, const QDict *qdict)
 {
     USBBus *bus;
     USBDevice *dev;
@@ -643,9 +665,12 @@
             dev = port->dev;
             if (!dev)
                 continue;
-            monitor_printf(mon, "  Device %d.%d, Port %s, Speed %s Mb/s, Product %s\n",
-                           bus->busnr, dev->addr, port->path, usb_speed(dev->speed),
-                           dev->product_desc);
+            monitor_printf(mon, "  Device %d.%d, Port %s, Speed %s Mb/s, "
+                           "Product %s%s%s\n",
+                           bus->busnr, dev->addr, port->path,
+                           usb_speed(dev->speed), dev->product_desc,
+                           dev->qdev.id ? ", ID: " : "",
+                           dev->qdev.id ?: "");
         }
     }
 }
@@ -655,10 +680,12 @@
 {
     USBBus *bus = usb_bus_find(-1 /* any */);
     LegacyUSBFactory *f = NULL;
+    Error *err = NULL;
     GSList *i;
     char driver[32];
     const char *params;
     int len;
+    USBDevice *dev;
 
     params = strchr(cmdline,':');
     if (params) {
@@ -693,14 +720,69 @@
         return NULL;
     }
 
-    if (!f->usbdevice_init) {
+    if (f->usbdevice_init) {
+        dev = f->usbdevice_init(bus, params);
+    } else {
         if (*params) {
             error_report("usbdevice %s accepts no params", driver);
             return NULL;
         }
-        return usb_create_simple(bus, f->name);
+        dev = usb_create(bus, f->name);
     }
-    return f->usbdevice_init(bus, params);
+    if (!dev) {
+        error_report("Failed to create USB device '%s'", f->name);
+        return NULL;
+    }
+    object_property_set_bool(OBJECT(dev), true, "realized", &err);
+    if (err) {
+        error_reportf_err(err, "Failed to initialize USB device '%s': ",
+                          f->name);
+        object_unparent(OBJECT(dev));
+        return NULL;
+    }
+    return dev;
+}
+
+static bool usb_get_attached(Object *obj, Error **errp)
+{
+    USBDevice *dev = USB_DEVICE(obj);
+
+    return dev->attached;
+}
+
+static void usb_set_attached(Object *obj, bool value, Error **errp)
+{
+    USBDevice *dev = USB_DEVICE(obj);
+    Error *err = NULL;
+
+    if (dev->attached == value) {
+        return;
+    }
+
+    if (value) {
+        usb_device_attach(dev, &err);
+        if (err) {
+            error_propagate(errp, err);
+        }
+    } else {
+        usb_device_detach(dev);
+    }
+}
+
+static void usb_device_instance_init(Object *obj)
+{
+    USBDevice *dev = USB_DEVICE(obj);
+    USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+
+    if (klass->attached_settable) {
+        object_property_add_bool(obj, "attached",
+                                 usb_get_attached, usb_set_attached,
+                                 NULL);
+    } else {
+        object_property_add_bool(obj, "attached",
+                                 usb_get_attached, NULL,
+                                 NULL);
+    }
 }
 
 static void usb_device_class_init(ObjectClass *klass, void *data)
@@ -716,6 +798,7 @@
     .name = TYPE_USB_DEVICE,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(USBDevice),
+    .instance_init = usb_device_instance_init,
     .abstract = true,
     .class_size = sizeof(USBDeviceClass),
     .class_init = usb_device_class_init,
diff --git a/hw/usb/ccid-card-emulated.c b/hw/usb/ccid-card-emulated.c
index aa1c37a..3213f9f 100644
--- a/hw/usb/ccid-card-emulated.c
+++ b/hw/usb/ccid-card-emulated.c
@@ -26,6 +26,7 @@
  * the db parameter.
  */
 
+#include "qemu/osdep.h"
 #include <eventt.h>
 #include <vevent.h>
 #include <vreader.h>
@@ -33,7 +34,6 @@
 
 #include "qemu/thread.h"
 #include "sysemu/char.h"
-#include "monitor/monitor.h"
 #include "ccid.h"
 
 #define DPRINTF(card, lvl, fmt, ...) \
@@ -43,7 +43,10 @@
     } \
 } while (0)
 
-#define EMULATED_DEV_NAME "ccid-card-emulated"
+
+#define TYPE_EMULATED_CCID "ccid-card-emulated"
+#define EMULATED_CCID_CARD(obj) \
+    OBJECT_CHECK(EmulatedState, (obj), TYPE_EMULATED_CCID)
 
 #define BACKEND_NSS_EMULATED_NAME "nss-emulated"
 #define BACKEND_CERTIFICATES_NAME "certificates"
@@ -134,7 +137,7 @@
 static void emulated_apdu_from_guest(CCIDCardState *base,
     const uint8_t *apdu, uint32_t len)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
     EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent) + len);
 
     assert(event);
@@ -151,7 +154,7 @@
 
 static const uint8_t *emulated_get_atr(CCIDCardState *base, uint32_t *len)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
 
     *len = card->atr_length;
     return card->atr;
@@ -167,7 +170,7 @@
 
 static void emulated_push_type(EmulatedState *card, uint32_t type)
 {
-    EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent));
+    EmulEvent *event = g_new(EmulEvent, 1);
 
     assert(event);
     event->p.gen.type = type;
@@ -176,7 +179,7 @@
 
 static void emulated_push_error(EmulatedState *card, uint64_t code)
 {
-    EmulEvent *event = (EmulEvent *)g_malloc(sizeof(EmulEvent));
+    EmulEvent *event = g_new(EmulEvent, 1);
 
     assert(event);
     event->p.error.type = EMUL_ERROR;
@@ -404,7 +407,7 @@
         DPRINTF(card, 2, "event notifier creation failed\n");
         return -1;
     }
-    event_notifier_set_handler(&card->notifier, card_event_handler);
+    event_notifier_set_handler(&card->notifier, false, card_event_handler);
     return 0;
 }
 
@@ -479,7 +482,7 @@
 
 static int emulated_initfn(CCIDCardState *base)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
     VCardEmulError ret;
     const EnumTable *ptable;
 
@@ -515,26 +518,26 @@
             ret = emulated_initialize_vcard_from_certificates(card);
         } else {
             printf("%s: you must provide all three certs for"
-                   " certificates backend\n", EMULATED_DEV_NAME);
+                   " certificates backend\n", TYPE_EMULATED_CCID);
             return -1;
         }
     } else {
         if (card->backend != BACKEND_NSS_EMULATED) {
             printf("%s: bad backend specified. The options are:\n%s (default),"
-                " %s.\n", EMULATED_DEV_NAME, BACKEND_NSS_EMULATED_NAME,
+                " %s.\n", TYPE_EMULATED_CCID, BACKEND_NSS_EMULATED_NAME,
                 BACKEND_CERTIFICATES_NAME);
             return -1;
         }
         if (card->cert1 != NULL || card->cert2 != NULL || card->cert3 != NULL) {
             printf("%s: unexpected cert parameters to nss emulated backend\n",
-                   EMULATED_DEV_NAME);
+                   TYPE_EMULATED_CCID);
             return -1;
         }
         /* default to mirroring the local hardware readers */
         ret = wrap_vcard_emul_init(NULL);
     }
     if (ret != VCARD_EMUL_OK) {
-        printf("%s: failed to initialize vcard\n", EMULATED_DEV_NAME);
+        printf("%s: failed to initialize vcard\n", TYPE_EMULATED_CCID);
         return -1;
     }
     qemu_thread_create(&card->event_thread_id, "ccid/event", event_thread,
@@ -546,7 +549,7 @@
 
 static int emulated_exitfn(CCIDCardState *base)
 {
-    EmulatedState *card = DO_UPCAST(EmulatedState, base, base);
+    EmulatedState *card = EMULATED_CCID_CARD(base);
     VEvent *vevent = vevent_new(VEVENT_LAST, NULL, NULL);
 
     vevent_queue_vevent(vevent); /* stop vevent thread */
@@ -589,7 +592,7 @@
 }
 
 static const TypeInfo emulated_card_info = {
-    .name          = EMULATED_DEV_NAME,
+    .name          = TYPE_EMULATED_CCID,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(EmulatedState),
     .class_init    = emulated_class_initfn,
diff --git a/hw/usb/ccid-card-passthru.c b/hw/usb/ccid-card-passthru.c
index 10f1d30..c0e90e5 100644
--- a/hw/usb/ccid-card-passthru.c
+++ b/hw/usb/ccid-card-passthru.c
@@ -8,11 +8,12 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/char.h"
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
-#include "monitor/monitor.h"
 #include "ccid.h"
-#include "libcacard/vscard_common.h"
+#include "cacard/vscard_common.h"
 
 #define DPRINTF(card, lvl, fmt, ...)                    \
 do {                                                    \
@@ -38,8 +39,6 @@
  0x13, 0x08
 };
 
-
-#define PASSTHRU_DEV_NAME "ccid-card-passthru"
 #define VSCARD_IN_SIZE 65536
 
 /* maximum size of ATR - from 7816-3 */
@@ -58,6 +57,10 @@
     uint8_t  debug;
 };
 
+#define TYPE_CCID_PASSTHRU "ccid-card-passthru"
+#define PASSTHRU_CCID_CARD(obj) \
+    OBJECT_CHECK(PassthruState, (obj), TYPE_CCID_PASSTHRU)
+
 /*
  * VSCard protocol over chardev
  * This code should not depend on the card type.
@@ -316,7 +319,7 @@
 static void passthru_apdu_from_guest(
     CCIDCardState *base, const uint8_t *apdu, uint32_t len)
 {
-    PassthruState *card = DO_UPCAST(PassthruState, base, base);
+    PassthruState *card = PASSTHRU_CCID_CARD(base);
 
     if (!card->cs) {
         printf("ccid-passthru: no chardev, discarding apdu length %d\n", len);
@@ -327,7 +330,7 @@
 
 static const uint8_t *passthru_get_atr(CCIDCardState *base, uint32_t *len)
 {
-    PassthruState *card = DO_UPCAST(PassthruState, base, base);
+    PassthruState *card = PASSTHRU_CCID_CARD(base);
 
     *len = card->atr_length;
     return card->atr;
@@ -335,7 +338,7 @@
 
 static int passthru_initfn(CCIDCardState *base)
 {
-    PassthruState *card = DO_UPCAST(PassthruState, base, base);
+    PassthruState *card = PASSTHRU_CCID_CARD(base);
 
     card->vscard_in_pos = 0;
     card->vscard_in_hdr = 0;
@@ -399,7 +402,7 @@
 }
 
 static const TypeInfo passthru_card_info = {
-    .name          = PASSTHRU_DEV_NAME,
+    .name          = TYPE_CCID_PASSTHRU,
     .parent        = TYPE_CCID_CARD,
     .instance_size = sizeof(PassthruState),
     .class_init    = passthru_class_initfn,
diff --git a/hw/usb/combined-packet.c b/hw/usb/combined-packet.c
index ad77705..48cac87 100644
--- a/hw/usb/combined-packet.c
+++ b/hw/usb/combined-packet.c
@@ -19,6 +19,7 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "qemu/iov.h"
diff --git a/hw/usb/core.c b/hw/usb/core.c
index cf34755..45fa00c 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -23,6 +23,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "qemu/iov.h"
@@ -128,9 +129,16 @@
     }
 
     usb_packet_copy(p, s->setup_buf, p->iov.size);
+    s->setup_index = 0;
     p->actual_length = 0;
     s->setup_len   = (s->setup_buf[7] << 8) | s->setup_buf[6];
-    s->setup_index = 0;
+    if (s->setup_len > sizeof(s->data_buf)) {
+        fprintf(stderr,
+                "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
+                s->setup_len, sizeof(s->data_buf));
+        p->status = USB_RET_STALL;
+        return;
+    }
 
     request = (s->setup_buf[0] << 8) | s->setup_buf[1];
     value   = (s->setup_buf[3] << 8) | s->setup_buf[2];
@@ -151,13 +159,6 @@
         }
         s->setup_state = SETUP_STATE_DATA;
     } else {
-        if (s->setup_len > sizeof(s->data_buf)) {
-            fprintf(stderr,
-                "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
-                s->setup_len, sizeof(s->data_buf));
-            p->status = USB_RET_STALL;
-            return;
-        }
         if (s->setup_len == 0)
             s->setup_state = SETUP_STATE_ACK;
         else
@@ -176,7 +177,7 @@
     request = (s->setup_buf[0] << 8) | s->setup_buf[1];
     value   = (s->setup_buf[3] << 8) | s->setup_buf[2];
     index   = (s->setup_buf[5] << 8) | s->setup_buf[4];
- 
+
     switch(s->setup_state) {
     case SETUP_STATE_ACK:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
@@ -331,23 +332,6 @@
     usb_packet_complete(s, p);
 }
 
-/* XXX: fix overflow */
-int set_usb_string(uint8_t *buf, const char *str)
-{
-    int len, i;
-    uint8_t *q;
-
-    q = buf;
-    len = strlen(str);
-    *q++ = 2 * len + 2;
-    *q++ = 3;
-    for(i = 0; i < len; i++) {
-        *q++ = str[i];
-        *q++ = 0;
-    }
-    return q - buf;
-}
-
 USBDevice *usb_find_device(USBPort *port, uint8_t addr)
 {
     USBDevice *dev = port->dev;
@@ -749,12 +733,6 @@
     uep->type = type;
 }
 
-uint8_t usb_ep_get_ifnum(USBDevice *dev, int pid, int ep)
-{
-    struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
-    return uep->ifnum;
-}
-
 void usb_ep_set_ifnum(USBDevice *dev, int pid, int ep, uint8_t ifnum)
 {
     struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
@@ -782,12 +760,6 @@
     uep->max_packet_size = size * microframes;
 }
 
-int usb_ep_get_max_packet_size(USBDevice *dev, int pid, int ep)
-{
-    struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
-    return uep->max_packet_size;
-}
-
 void usb_ep_set_max_streams(USBDevice *dev, int pid, int ep, uint8_t raw)
 {
     struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
@@ -801,18 +773,6 @@
     }
 }
 
-int usb_ep_get_max_streams(USBDevice *dev, int pid, int ep)
-{
-    struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
-    return uep->max_streams;
-}
-
-void usb_ep_set_pipeline(USBDevice *dev, int pid, int ep, bool enabled)
-{
-    struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
-    uep->pipeline = enabled;
-}
-
 void usb_ep_set_halted(USBDevice *dev, int pid, int ep, bool halted)
 {
     struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
diff --git a/hw/usb/desc-msos.c b/hw/usb/desc-msos.c
index 334d1ae..3652919 100644
--- a/hw/usb/desc-msos.c
+++ b/hw/usb/desc-msos.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 
@@ -231,7 +232,7 @@
         length = len;
     }
     memcpy(dest, buf, length);
-    free(buf);
+    g_free(buf);
 
     p->actual_length = length;
     return 0;
diff --git a/hw/usb/desc.c b/hw/usb/desc.c
index b82c397..5e0e1d1 100644
--- a/hw/usb/desc.c
+++ b/hw/usb/desc.c
@@ -1,4 +1,4 @@
-#include <ctype.h>
+#include "qemu/osdep.h"
 
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
@@ -574,6 +574,7 @@
     }
     dst += snprintf(serial+dst, sizeof(serial)-dst, "-%s", dev->port->path);
     usb_desc_set_string(dev, index, serial);
+    g_free(path);
 }
 
 const char *usb_desc_get_string(USBDevice *dev, uint8_t index)
diff --git a/hw/usb/desc.h b/hw/usb/desc.h
index 8e8db03..4d81c68 100644
--- a/hw/usb/desc.h
+++ b/hw/usb/desc.h
@@ -1,7 +1,6 @@
 #ifndef QEMU_HW_USB_DESC_H
 #define QEMU_HW_USB_DESC_H
 
-#include <inttypes.h>
 #include <wchar.h>
 
 /* binary representation */
diff --git a/hw/usb/dev-audio.c b/hw/usb/dev-audio.c
index 67deffe..87cab0a 100644
--- a/hw/usb/dev-audio.c
+++ b/hw/usb/dev-audio.c
@@ -29,6 +29,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
@@ -361,6 +362,9 @@
     uint32_t buffer;
 } USBAudioState;
 
+#define TYPE_USB_AUDIO "usb-audio"
+#define USB_AUDIO(obj) OBJECT_CHECK(USBAudioState, (obj), TYPE_USB_AUDIO)
+
 static void output_callback(void *opaque, int avail)
 {
     USBAudioState *s = opaque;
@@ -506,7 +510,7 @@
                                     int request, int value, int index,
                                     int length, uint8_t *data)
 {
-    USBAudioState *s = DO_UPCAST(USBAudioState, dev, dev);
+    USBAudioState *s = USB_AUDIO(dev);
     int ret = 0;
 
     if (s->debug) {
@@ -565,7 +569,7 @@
 static void usb_audio_set_interface(USBDevice *dev, int iface,
                                     int old, int value)
 {
-    USBAudioState *s = DO_UPCAST(USBAudioState, dev, dev);
+    USBAudioState *s = USB_AUDIO(dev);
 
     if (iface == 1) {
         usb_audio_set_output_altset(s, value);
@@ -574,7 +578,7 @@
 
 static void usb_audio_handle_reset(USBDevice *dev)
 {
-    USBAudioState *s = DO_UPCAST(USBAudioState, dev, dev);
+    USBAudioState *s = USB_AUDIO(dev);
 
     if (s->debug) {
         fprintf(stderr, "usb-audio: reset\n");
@@ -615,7 +619,7 @@
 
 static void usb_audio_handle_destroy(USBDevice *dev)
 {
-    USBAudioState *s = DO_UPCAST(USBAudioState, dev, dev);
+    USBAudioState *s = USB_AUDIO(dev);
 
     if (s->debug) {
         fprintf(stderr, "usb-audio: destroy\n");
@@ -630,12 +634,12 @@
 
 static void usb_audio_realize(USBDevice *dev, Error **errp)
 {
-    USBAudioState *s = DO_UPCAST(USBAudioState, dev, dev);
+    USBAudioState *s = USB_AUDIO(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
     s->dev.opaque = s;
-    AUD_register_card("usb-audio", &s->card);
+    AUD_register_card(TYPE_USB_AUDIO, &s->card);
 
     s->out.altset        = ALTSET_OFF;
     s->out.mute          = false;
@@ -647,21 +651,21 @@
     s->out.as.endianness = 0;
     streambuf_init(&s->out.buf, s->buffer);
 
-    s->out.voice = AUD_open_out(&s->card, s->out.voice, "usb-audio",
+    s->out.voice = AUD_open_out(&s->card, s->out.voice, TYPE_USB_AUDIO,
                                 s, output_callback, &s->out.as);
     AUD_set_volume_out(s->out.voice, s->out.mute, s->out.vol[0], s->out.vol[1]);
     AUD_set_active_out(s->out.voice, 0);
 }
 
 static const VMStateDescription vmstate_usb_audio = {
-    .name = "usb-audio",
+    .name = TYPE_USB_AUDIO,
     .unmigratable = 1,
 };
 
 static Property usb_audio_properties[] = {
     DEFINE_PROP_UINT32("debug", USBAudioState, debug, 0),
     DEFINE_PROP_UINT32("buffer", USBAudioState, buffer,
-                       8 * USBAUDIO_PACKET_SIZE),
+                       32 * USBAUDIO_PACKET_SIZE),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -684,7 +688,7 @@
 }
 
 static const TypeInfo usb_audio_info = {
-    .name          = "usb-audio",
+    .name          = TYPE_USB_AUDIO,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBAudioState),
     .class_init    = usb_audio_class_init,
@@ -693,7 +697,7 @@
 static void usb_audio_register_types(void)
 {
     type_register_static(&usb_audio_info);
-    usb_legacy_register("usb-audio", "audio", NULL);
+    usb_legacy_register(TYPE_USB_AUDIO, "audio", NULL);
 }
 
 type_init(usb_audio_register_types)
diff --git a/hw/usb/dev-bluetooth.c b/hw/usb/dev-bluetooth.c
index 390d475..91a4a0b 100644
--- a/hw/usb/dev-bluetooth.c
+++ b/hw/usb/dev-bluetooth.c
@@ -18,6 +18,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/usb.h"
@@ -49,6 +50,9 @@
     } outcmd, outacl, outsco;
 };
 
+#define TYPE_USB_BT "usb-bt-dongle"
+#define USB_BT(obj) OBJECT_CHECK(struct USBBtState, (obj), TYPE_USB_BT)
+
 #define USB_EVT_EP	1
 #define USB_ACL_EP	2
 #define USB_SCO_EP	3
@@ -503,7 +507,7 @@
 
 static void usb_bt_realize(USBDevice *dev, Error **errp)
 {
-    struct USBBtState *s = DO_UPCAST(struct USBBtState, dev, dev);
+    struct USBBtState *s = USB_BT(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
@@ -523,28 +527,19 @@
     USBDevice *dev;
     struct USBBtState *s;
     HCIInfo *hci;
-    const char *name = "usb-bt-dongle";
+    const char *name = TYPE_USB_BT;
 
     if (*cmdline) {
         hci = hci_init(cmdline);
     } else {
         hci = bt_new_hci(qemu_find_bt_vlan(0));
     }
-
     if (!hci)
         return NULL;
-    dev = usb_create(bus, name);
-    if (!dev) {
-        error_report("Failed to create USB device '%s'", name);
-        return NULL;
-    }
-    s = DO_UPCAST(struct USBBtState, dev, dev);
-    s->hci = hci;
-    if (qdev_init(&dev->qdev) < 0) {
-        error_report("Failed to initialize USB device '%s'", name);
-        return NULL;
-    }
 
+    dev = usb_create(bus, name);
+    s = USB_BT(dev);
+    s->hci = hci;
     return dev;
 }
 
@@ -570,7 +565,7 @@
 }
 
 static const TypeInfo bt_info = {
-    .name          = "usb-bt-dongle",
+    .name          = TYPE_USB_BT,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(struct USBBtState),
     .class_init    = usb_bt_class_initfn,
@@ -579,7 +574,7 @@
 static void usb_bt_register_types(void)
 {
     type_register_static(&bt_info);
-    usb_legacy_register("usb-bt-dongle", "bt", usb_bt_init);
+    usb_legacy_register(TYPE_USB_BT, "bt", usb_bt_init);
 }
 
 type_init(usb_bt_register_types)
diff --git a/hw/usb/dev-hid.c b/hw/usb/dev-hid.c
index 507c966..24d05f7 100644
--- a/hw/usb/dev-hid.c
+++ b/hw/usb/dev-hid.c
@@ -22,10 +22,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/input/hid.h"
 
@@ -51,6 +53,9 @@
     uint32_t head;
 } USBHIDState;
 
+#define TYPE_USB_HID "usb-hid"
+#define USB_HID(obj) OBJECT_CHECK(USBHIDState, (obj), TYPE_USB_HID)
+
 enum {
     STR_MANUFACTURER = 1,
     STR_PRODUCT_MOUSE,
@@ -564,7 +569,7 @@
 
 static void usb_hid_handle_reset(USBDevice *dev)
 {
-    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+    USBHIDState *us = USB_HID(dev);
 
     hid_reset(&us->hid);
 }
@@ -572,7 +577,7 @@
 static void usb_hid_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
-    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+    USBHIDState *us = USB_HID(dev);
     HIDState *hs = &us->hid;
     int ret;
 
@@ -651,7 +656,7 @@
 
 static void usb_hid_handle_data(USBDevice *dev, USBPacket *p)
 {
-    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+    USBHIDState *us = USB_HID(dev);
     HIDState *hs = &us->hid;
     uint8_t buf[p->iov.size];
     int len = 0;
@@ -687,7 +692,7 @@
 
 static void usb_hid_handle_destroy(USBDevice *dev)
 {
-    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+    USBHIDState *us = USB_HID(dev);
 
     hid_free(&us->hid);
 }
@@ -696,7 +701,7 @@
                            const USBDesc *usb1, const USBDesc *usb2,
                            Error **errp)
 {
-    USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
+    USBHIDState *us = USB_HID(dev);
     switch (us->usb_version) {
     case 1:
         dev->usb_desc = usb1;
@@ -784,6 +789,14 @@
     uc->handle_attach  = usb_desc_attach;
 }
 
+static const TypeInfo usb_hid_type_info = {
+    .name = TYPE_USB_HID,
+    .parent = TYPE_USB_DEVICE,
+    .instance_size = sizeof(USBHIDState),
+    .abstract = true,
+    .class_init = usb_hid_class_initfn,
+};
+
 static Property usb_tablet_properties[] = {
         DEFINE_PROP_UINT32("usb_version", USBHIDState, usb_version, 2),
         DEFINE_PROP_STRING("display", USBHIDState, display),
@@ -796,7 +809,6 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    usb_hid_class_initfn(klass, data);
     uc->realize        = usb_tablet_realize;
     uc->product_desc   = "QEMU USB Tablet";
     dc->vmsd = &vmstate_usb_ptr;
@@ -806,8 +818,7 @@
 
 static const TypeInfo usb_tablet_info = {
     .name          = "usb-tablet",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBHIDState),
+    .parent        = TYPE_USB_HID,
     .class_init    = usb_tablet_class_initfn,
 };
 
@@ -821,7 +832,6 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    usb_hid_class_initfn(klass, data);
     uc->realize        = usb_mouse_realize;
     uc->product_desc   = "QEMU USB Mouse";
     dc->vmsd = &vmstate_usb_ptr;
@@ -831,8 +841,7 @@
 
 static const TypeInfo usb_mouse_info = {
     .name          = "usb-mouse",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBHIDState),
+    .parent        = TYPE_USB_HID,
     .class_init    = usb_mouse_class_initfn,
 };
 
@@ -847,7 +856,6 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    usb_hid_class_initfn(klass, data);
     uc->realize        = usb_keyboard_realize;
     uc->product_desc   = "QEMU USB Keyboard";
     dc->vmsd = &vmstate_usb_kbd;
@@ -857,13 +865,13 @@
 
 static const TypeInfo usb_keyboard_info = {
     .name          = "usb-kbd",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBHIDState),
+    .parent        = TYPE_USB_HID,
     .class_init    = usb_keyboard_class_initfn,
 };
 
 static void usb_hid_register_types(void)
 {
+    type_register_static(&usb_hid_type_info);
     type_register_static(&usb_tablet_info);
     usb_legacy_register("usb-tablet", "tablet", NULL);
     type_register_static(&usb_mouse_info);
diff --git a/hw/usb/dev-hub.c b/hw/usb/dev-hub.c
index 0482f58..a33f21c 100644
--- a/hw/usb/dev-hub.c
+++ b/hw/usb/dev-hub.c
@@ -21,6 +21,8 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "trace.h"
 #include "hw/usb.h"
@@ -41,6 +43,9 @@
     USBHubPort ports[NUM_PORTS];
 } USBHubState;
 
+#define TYPE_USB_HUB "usb-hub"
+#define USB_HUB(obj) OBJECT_CHECK(USBHubState, (obj), TYPE_USB_HUB)
+
 #define ClearHubFeature		(0x2000 | USB_REQ_CLEAR_FEATURE)
 #define ClearPortFeature	(0x2300 | USB_REQ_CLEAR_FEATURE)
 #define GetHubDescriptor	(0xa000 | USB_REQ_GET_DESCRIPTOR)
@@ -227,7 +232,7 @@
 
 static USBDevice *usb_hub_find_device(USBDevice *dev, uint8_t addr)
 {
-    USBHubState *s = DO_UPCAST(USBHubState, dev, dev);
+    USBHubState *s = USB_HUB(dev);
     USBHubPort *port;
     USBDevice *downstream;
     int i;
@@ -247,7 +252,7 @@
 
 static void usb_hub_handle_reset(USBDevice *dev)
 {
-    USBHubState *s = DO_UPCAST(USBHubState, dev, dev);
+    USBHubState *s = USB_HUB(dev);
     USBHubPort *port;
     int i;
 
@@ -513,7 +518,7 @@
 
 static void usb_hub_realize(USBDevice *dev, Error **errp)
 {
-    USBHubState *s = DO_UPCAST(USBHubState, dev, dev);
+    USBHubState *s = USB_HUB(dev);
     USBHubPort *port;
     int i;
 
@@ -577,7 +582,7 @@
 }
 
 static const TypeInfo hub_info = {
-    .name          = "usb-hub",
+    .name          = TYPE_USB_HUB,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBHubState),
     .class_init    = usb_hub_class_initfn,
diff --git a/hw/usb/dev-mtp.c b/hw/usb/dev-mtp.c
index 108ece8..fd5fe68 100644
--- a/hw/usb/dev-mtp.c
+++ b/hw/usb/dev-mtp.c
@@ -9,12 +9,17 @@
  * This code is licensed under the GPL v2 or later.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <wchar.h>
 #include <dirent.h>
-#include <unistd.h>
 
-#include <sys/stat.h>
 #include <sys/statvfs.h>
+#ifdef CONFIG_INOTIFY1
+#include <sys/inotify.h>
+#include "qapi/error.h"
+#include "qemu/main-loop.h"
+#endif
 
 #include "qemu-common.h"
 #include "qemu/iov.h"
@@ -62,6 +67,11 @@
     /* format codes */
     FMT_UNDEFINED_OBJECT           = 0x3000,
     FMT_ASSOCIATION                = 0x3001,
+
+    /* event codes */
+    EVT_OBJ_ADDED                  = 0x4002,
+    EVT_OBJ_REMOVED                = 0x4003,
+    EVT_OBJ_INFO_CHANGED           = 0x4007,
 };
 
 typedef struct {
@@ -84,6 +94,17 @@
     EP_EVENT,
 };
 
+#ifdef CONFIG_INOTIFY1
+typedef struct MTPMonEntry MTPMonEntry;
+
+struct MTPMonEntry {
+    uint32_t event;
+    uint32_t handle;
+
+    QTAILQ_ENTRY(MTPMonEntry) next;
+};
+#endif
+
 struct MTPControl {
     uint16_t     code;
     uint32_t     trans;
@@ -108,9 +129,14 @@
     char         *name;
     char         *path;
     struct stat  stat;
+#ifdef CONFIG_INOTIFY1
+    /* inotify watch cookie */
+    int          watchfd;
+#endif
     MTPObject    *parent;
-    MTPObject    **children;
     uint32_t     nchildren;
+    QLIST_HEAD(, MTPObject) children;
+    QLIST_ENTRY(MTPObject) list;
     bool         have_children;
     QTAILQ_ENTRY(MTPObject) next;
 };
@@ -128,8 +154,16 @@
     uint32_t     next_handle;
 
     QTAILQ_HEAD(, MTPObject) objects;
+#ifdef CONFIG_INOTIFY1
+    /* inotify descriptor */
+    int          inotifyfd;
+    QTAILQ_HEAD(events, MTPMonEntry) events;
+#endif
 };
 
+#define TYPE_USB_MTP "usb-mtp"
+#define USB_MTP(obj) OBJECT_CHECK(MTPState, (obj), TYPE_USB_MTP)
+
 #define QEMU_STORAGE_ID 0x00010001
 
 #define MTP_FLAG_WRITABLE 0
@@ -180,7 +214,7 @@
         },{
             .bEndpointAddress      = USB_DIR_IN | EP_EVENT,
             .bmAttributes          = USB_ENDPOINT_XFER_INT,
-            .wMaxPacketSize        = 8,
+            .wMaxPacketSize        = 64,
             .bInterval             = 0x0a,
         },
     }
@@ -222,7 +256,7 @@
         },{
             .bEndpointAddress      = USB_DIR_IN | EP_EVENT,
             .bmAttributes          = USB_ENDPOINT_XFER_INT,
-            .wMaxPacketSize        = 8,
+            .wMaxPacketSize        = 64,
             .bInterval             = 0x0a,
         },
     }
@@ -285,7 +319,7 @@
         o->path = g_strdup_printf("%s/%s", parent->path, name);
     }
 
-    if (lstat(o->path, &o->stat) != 0) {
+    if (qemu_lstat(o->path, &o->stat) != 0) {
         goto ignore;
     }
     if (S_ISREG(o->stat.st_mode)) {
@@ -314,15 +348,24 @@
 
 static void usb_mtp_object_free(MTPState *s, MTPObject *o)
 {
-    int i;
+    MTPObject *iter;
+
+    if (!o) {
+        return;
+    }
 
     trace_usb_mtp_object_free(s->dev.addr, o->handle, o->path);
 
     QTAILQ_REMOVE(&s->objects, o, next);
-    for (i = 0; i < o->nchildren; i++) {
-        usb_mtp_object_free(s, o->children[i]);
+    if (o->parent) {
+        QLIST_REMOVE(o, list);
+        o->parent->nchildren--;
     }
-    g_free(o->children);
+
+    while (!QLIST_EMPTY(&o->children)) {
+        iter = QLIST_FIRST(&o->children);
+        usb_mtp_object_free(s, iter);
+    }
     g_free(o->name);
     g_free(o->path);
     g_free(o);
@@ -340,6 +383,203 @@
     return NULL;
 }
 
+static MTPObject *usb_mtp_add_child(MTPState *s, MTPObject *o,
+                                    char *name)
+{
+    MTPObject *child =
+        usb_mtp_object_alloc(s, s->next_handle++, o, name);
+
+    if (child) {
+        trace_usb_mtp_add_child(s->dev.addr, child->handle, child->path);
+        QLIST_INSERT_HEAD(&o->children, child, list);
+        o->nchildren++;
+
+        if (child->format == FMT_ASSOCIATION) {
+            QLIST_INIT(&child->children);
+        }
+    }
+
+    return child;
+}
+
+#ifdef CONFIG_INOTIFY1
+static MTPObject *usb_mtp_object_lookup_name(MTPObject *parent,
+                                             char *name, int len)
+{
+    MTPObject *iter;
+
+    QLIST_FOREACH(iter, &parent->children, list) {
+        if (strncmp(iter->name, name, len) == 0) {
+            return iter;
+        }
+    }
+
+    return NULL;
+}
+
+static MTPObject *usb_mtp_object_lookup_wd(MTPState *s, int wd)
+{
+    MTPObject *iter;
+
+    QTAILQ_FOREACH(iter, &s->objects, next) {
+        if (iter->watchfd == wd) {
+            return iter;
+        }
+    }
+
+    return NULL;
+}
+
+static void inotify_watchfn(void *arg)
+{
+    MTPState *s = arg;
+    ssize_t bytes;
+    /* From the man page: atleast one event can be read */
+    int pos;
+    char buf[sizeof(struct inotify_event) + NAME_MAX + 1];
+
+    for (;;) {
+        bytes = read(s->inotifyfd, buf, sizeof(buf));
+        pos = 0;
+
+        if (bytes <= 0) {
+            /* Better luck next time */
+            return;
+        }
+
+        /*
+         * TODO: Ignore initiator initiated events.
+         * For now we are good because the store is RO
+         */
+        while (bytes > 0) {
+            char *p = buf + pos;
+            struct inotify_event *event = (struct inotify_event *)p;
+            int watchfd = 0;
+            uint32_t mask = event->mask & (IN_CREATE | IN_DELETE |
+                                           IN_MODIFY | IN_IGNORED);
+            MTPObject *parent = usb_mtp_object_lookup_wd(s, event->wd);
+            MTPMonEntry *entry = NULL;
+            MTPObject *o;
+
+            pos = pos + sizeof(struct inotify_event) + event->len;
+            bytes = bytes - pos;
+
+            if (!parent) {
+                continue;
+            }
+
+            switch (mask) {
+            case IN_CREATE:
+                if (usb_mtp_object_lookup_name
+                    (parent, event->name, event->len)) {
+                    /* Duplicate create event */
+                    continue;
+                }
+                entry = g_new0(MTPMonEntry, 1);
+                entry->handle = s->next_handle;
+                entry->event = EVT_OBJ_ADDED;
+                o = usb_mtp_add_child(s, parent, event->name);
+                if (!o) {
+                    g_free(entry);
+                    continue;
+                }
+                o->watchfd = watchfd;
+                trace_usb_mtp_inotify_event(s->dev.addr, event->name,
+                                            event->mask, "Obj Added");
+                break;
+
+            case IN_DELETE:
+                /*
+                 * The kernel issues a IN_IGNORED event
+                 * when a dir containing a watchpoint is
+                 * deleted, so we don't have to delete the
+                 * watchpoint
+                 */
+                o = usb_mtp_object_lookup_name(parent, event->name, event->len);
+                if (!o) {
+                    continue;
+                }
+                entry = g_new0(MTPMonEntry, 1);
+                entry->handle = o->handle;
+                entry->event = EVT_OBJ_REMOVED;
+                trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                      event->mask, "Obj Deleted");
+                usb_mtp_object_free(s, o);
+                break;
+
+            case IN_MODIFY:
+                o = usb_mtp_object_lookup_name(parent, event->name, event->len);
+                if (!o) {
+                    continue;
+                }
+                entry = g_new0(MTPMonEntry, 1);
+                entry->handle = o->handle;
+                entry->event = EVT_OBJ_INFO_CHANGED;
+                trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                      event->mask, "Obj Modified");
+                break;
+
+            case IN_IGNORED:
+                o = usb_mtp_object_lookup_name(parent, event->name, event->len);
+                trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                      event->mask, "Obj ignored");
+                break;
+
+            default:
+                fprintf(stderr, "usb-mtp: failed to parse inotify event\n");
+                continue;
+            }
+
+            if (entry) {
+                QTAILQ_INSERT_HEAD(&s->events, entry, next);
+            }
+        }
+    }
+}
+
+static int usb_mtp_inotify_init(MTPState *s)
+{
+    int fd;
+
+    fd = inotify_init1(IN_NONBLOCK);
+    if (fd == -1) {
+        return 1;
+    }
+
+    QTAILQ_INIT(&s->events);
+    s->inotifyfd = fd;
+
+    qemu_set_fd_handler(fd, inotify_watchfn, NULL, s);
+
+    return 0;
+}
+
+static void usb_mtp_inotify_cleanup(MTPState *s)
+{
+    MTPMonEntry *e, *p;
+
+    if (!s->inotifyfd) {
+        return;
+    }
+
+    qemu_set_fd_handler(s->inotifyfd, NULL, NULL, s);
+    close(s->inotifyfd);
+
+    QTAILQ_FOREACH_SAFE(e, &s->events, next, p) {
+        QTAILQ_REMOVE(&s->events, e, next);
+        g_free(e);
+    }
+}
+
+static int usb_mtp_add_watch(int inotifyfd, char *path)
+{
+    uint32_t mask = IN_CREATE | IN_DELETE | IN_MODIFY |
+        IN_ISDIR;
+
+    return inotify_add_watch(inotifyfd, path, mask);
+}
+#endif
+
 static void usb_mtp_object_readdir(MTPState *s, MTPObject *o)
 {
     struct dirent *entry;
@@ -354,16 +594,18 @@
     if (!dir) {
         return;
     }
+#ifdef CONFIG_INOTIFY1
+    int watchfd = usb_mtp_add_watch(s->inotifyfd, o->path);
+    if (watchfd == -1) {
+        fprintf(stderr, "usb-mtp: failed to add watch for %s\n", o->path);
+    } else {
+        trace_usb_mtp_inotify_event(s->dev.addr, o->path,
+                                    0, "Watch Added");
+        o->watchfd = watchfd;
+    }
+#endif
     while ((entry = readdir(dir)) != NULL) {
-        if ((o->nchildren % 32) == 0) {
-            o->children = g_realloc(o->children,
-                                    (o->nchildren + 32) * sizeof(MTPObject *));
-        }
-        o->children[o->nchildren] =
-            usb_mtp_object_alloc(s, s->next_handle++, o, entry->d_name);
-        if (o->children[o->nchildren] != NULL) {
-            o->nchildren++;
-        }
+        usb_mtp_add_child(s, o, entry->d_name);
     }
     closedir(dir);
 }
@@ -477,7 +719,7 @@
 static void usb_mtp_add_str(MTPData *data, const char *str)
 {
     uint32_t len = strlen(str)+1;
-    wchar_t wstr[len];
+    wchar_t *wstr = g_new(wchar_t, len);
     size_t ret;
 
     ret = mbstowcs(wstr, str, len);
@@ -486,6 +728,8 @@
     } else {
         usb_mtp_add_wstr(data, wstr);
     }
+
+    g_free(wstr);
 }
 
 static void usb_mtp_add_time(MTPData *data, time_t time)
@@ -544,8 +788,8 @@
     trace_usb_mtp_op_get_device_info(s->dev.addr);
 
     usb_mtp_add_u16(d, 100);
-    usb_mtp_add_u32(d, 0xffffffff);
-    usb_mtp_add_u16(d, 0x0101);
+    usb_mtp_add_u32(d, 0x00000006);
+    usb_mtp_add_u16(d, 0x0064);
     usb_mtp_add_wstr(d, L"");
     usb_mtp_add_u16(d, 0x0000);
 
@@ -615,13 +859,15 @@
                                            MTPObject *o)
 {
     MTPData *d = usb_mtp_data_alloc(c);
-    uint32_t i, handles[o->nchildren];
+    uint32_t i = 0, handles[o->nchildren];
+    MTPObject *iter;
 
     trace_usb_mtp_op_get_object_handles(s->dev.addr, o->handle, o->path);
 
-    for (i = 0; i < o->nchildren; i++) {
-        handles[i] = o->children[i]->handle;
+    QLIST_FOREACH(iter, &o->children, list) {
+        handles[i++] = iter->handle;
     }
+    assert(i == o->nchildren);
     usb_mtp_add_u32_array(d, o->nchildren, handles);
 
     return d;
@@ -752,11 +998,19 @@
         trace_usb_mtp_op_open_session(s->dev.addr);
         s->session = c->argv[0];
         usb_mtp_object_alloc(s, s->next_handle++, NULL, s->root);
+#ifdef CONFIG_INOTIFY1
+        if (usb_mtp_inotify_init(s)) {
+            fprintf(stderr, "usb-mtp: file monitoring init failed\n");
+        }
+#endif
         break;
     case CMD_CLOSE_SESSION:
         trace_usb_mtp_op_close_session(s->dev.addr);
         s->session = 0;
         s->next_handle = 0;
+#ifdef CONFIG_INOTIFY1
+        usb_mtp_inotify_cleanup(s);
+#endif
         usb_mtp_object_free(s, QTAILQ_FIRST(&s->objects));
         assert(QTAILQ_EMPTY(&s->objects));
         break;
@@ -878,10 +1132,14 @@
 
 static void usb_mtp_handle_reset(USBDevice *dev)
 {
-    MTPState *s = DO_UPCAST(MTPState, dev, dev);
+    MTPState *s = USB_MTP(dev);
 
     trace_usb_mtp_reset(s->dev.addr);
 
+#ifdef CONFIG_INOTIFY1
+    usb_mtp_inotify_cleanup(s);
+#endif
+    usb_mtp_object_free(s, QTAILQ_FIRST(&s->objects));
     s->session = 0;
     usb_mtp_data_free(s->data_in);
     s->data_in = NULL;
@@ -914,7 +1172,7 @@
 
 static void usb_mtp_handle_data(USBDevice *dev, USBPacket *p)
 {
-    MTPState *s = DO_UPCAST(MTPState, dev, dev);
+    MTPState *s = USB_MTP(dev);
     MTPControl cmd;
     mtp_container container;
     uint32_t params[5];
@@ -1041,6 +1299,31 @@
         }
         break;
     case EP_EVENT:
+#ifdef CONFIG_INOTIFY1
+        if (!QTAILQ_EMPTY(&s->events)) {
+            struct MTPMonEntry *e = QTAILQ_LAST(&s->events, events);
+            uint32_t handle;
+            int len = sizeof(container) + sizeof(uint32_t);
+
+            if (p->iov.size < len) {
+                trace_usb_mtp_stall(s->dev.addr,
+                                    "packet too small to send event");
+                p->status = USB_RET_STALL;
+                return;
+            }
+
+            QTAILQ_REMOVE(&s->events, e, next);
+            container.length = cpu_to_le32(len);
+            container.type = cpu_to_le32(TYPE_EVENT);
+            container.code = cpu_to_le16(e->event);
+            container.trans = 0; /* no trans specific events */
+            handle = cpu_to_le32(e->handle);
+            usb_packet_copy(p, &container, sizeof(container));
+            usb_packet_copy(p, &handle, sizeof(uint32_t));
+            g_free(e);
+            return;
+        }
+#endif
         p->status = USB_RET_NAK;
         return;
     default:
@@ -1062,12 +1345,16 @@
 
 static void usb_mtp_realize(USBDevice *dev, Error **errp)
 {
-    MTPState *s = DO_UPCAST(MTPState, dev, dev);
+    MTPState *s = USB_MTP(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
     QTAILQ_INIT(&s->objects);
     if (s->desc == NULL) {
+        if (s->root == NULL) {
+            error_setg(errp, "usb-mtp: x-root property must be configured");
+            return;
+        }
         s->desc = strrchr(s->root, '/');
         if (s->desc && s->desc[0]) {
             s->desc = g_strdup(s->desc + 1);
@@ -1113,7 +1400,7 @@
 }
 
 static TypeInfo mtp_info = {
-    .name          = "usb-mtp",
+    .name          = TYPE_USB_MTP,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(MTPState),
     .class_init    = usb_mtp_class_initfn,
diff --git a/hw/usb/dev-network.c b/hw/usb/dev-network.c
index 5b95d5c..c0f1193 100644
--- a/hw/usb/dev-network.c
+++ b/hw/usb/dev-network.c
@@ -23,6 +23,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
@@ -32,6 +34,7 @@
 #include "qemu/config-file.h"
 #include "sysemu/sysemu.h"
 #include "qemu/iov.h"
+#include "qemu/cutils.h"
 
 /*#define TRAFFIC_DEBUG*/
 /* Thanks to NetChip Technologies for donating this product ID.
@@ -648,9 +651,13 @@
     QTAILQ_HEAD(rndis_resp_head, rndis_response) rndis_resp;
 } USBNetState;
 
+#define TYPE_USB_NET "usb-net"
+#define USB_NET(obj) OBJECT_CHECK(USBNetState, (obj), TYPE_USB_NET)
+
 static int is_rndis(USBNetState *s)
 {
-    return s->dev.config->bConfigurationValue == DEV_RNDIS_CONFIG_VALUE;
+    return s->dev.config ?
+            s->dev.config->bConfigurationValue == DEV_RNDIS_CONFIG_VALUE : 0;
 }
 
 static int ndis_query(USBNetState *s, uint32_t oid,
@@ -663,48 +670,49 @@
     /* general oids (table 4-1) */
     /* mandatory */
     case OID_GEN_SUPPORTED_LIST:
-        for (i = 0; i < ARRAY_SIZE(oid_supported_list); i++)
-            ((le32 *) outbuf)[i] = cpu_to_le32(oid_supported_list[i]);
+        for (i = 0; i < ARRAY_SIZE(oid_supported_list); i++) {
+            stl_le_p(outbuf + (i * sizeof(le32)), oid_supported_list[i]);
+        }
         return sizeof(oid_supported_list);
 
     /* mandatory */
     case OID_GEN_HARDWARE_STATUS:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_MEDIA_SUPPORTED:
-        *((le32 *) outbuf) = cpu_to_le32(s->medium);
+        stl_le_p(outbuf, s->medium);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_MEDIA_IN_USE:
-        *((le32 *) outbuf) = cpu_to_le32(s->medium);
+        stl_le_p(outbuf, s->medium);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_MAXIMUM_FRAME_SIZE:
-        *((le32 *) outbuf) = cpu_to_le32(ETH_FRAME_LEN);
+        stl_le_p(outbuf, ETH_FRAME_LEN);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_LINK_SPEED:
-        *((le32 *) outbuf) = cpu_to_le32(s->speed);
+        stl_le_p(outbuf, s->speed);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_TRANSMIT_BLOCK_SIZE:
-        *((le32 *) outbuf) = cpu_to_le32(ETH_FRAME_LEN);
+        stl_le_p(outbuf, ETH_FRAME_LEN);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_RECEIVE_BLOCK_SIZE:
-        *((le32 *) outbuf) = cpu_to_le32(ETH_FRAME_LEN);
+        stl_le_p(outbuf, ETH_FRAME_LEN);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_VENDOR_ID:
-        *((le32 *) outbuf) = cpu_to_le32(s->vendorid);
+        stl_le_p(outbuf, s->vendorid);
         return sizeof(le32);
 
     /* mandatory */
@@ -713,58 +721,57 @@
         return strlen((char *)outbuf) + 1;
 
     case OID_GEN_VENDOR_DRIVER_VERSION:
-        *((le32 *) outbuf) = cpu_to_le32(1);
+        stl_le_p(outbuf, 1);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_CURRENT_PACKET_FILTER:
-        *((le32 *) outbuf) = cpu_to_le32(s->filter);
+        stl_le_p(outbuf, s->filter);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_MAXIMUM_TOTAL_SIZE:
-        *((le32 *) outbuf) = cpu_to_le32(RNDIS_MAX_TOTAL_SIZE);
+        stl_le_p(outbuf, RNDIS_MAX_TOTAL_SIZE);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_MEDIA_CONNECT_STATUS:
-        *((le32 *) outbuf) = cpu_to_le32(s->media_state);
+        stl_le_p(outbuf, s->media_state);
         return sizeof(le32);
 
     case OID_GEN_PHYSICAL_MEDIUM:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     case OID_GEN_MAC_OPTIONS:
-        *((le32 *) outbuf) = cpu_to_le32(
-                        NDIS_MAC_OPTION_RECEIVE_SERIALIZED |
-                        NDIS_MAC_OPTION_FULL_DUPLEX);
+        stl_le_p(outbuf, NDIS_MAC_OPTION_RECEIVE_SERIALIZED |
+                 NDIS_MAC_OPTION_FULL_DUPLEX);
         return sizeof(le32);
 
     /* statistics OIDs (table 4-2) */
     /* mandatory */
     case OID_GEN_XMIT_OK:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_RCV_OK:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_XMIT_ERROR:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_RCV_ERROR:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_GEN_RCV_NO_BUFFER:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* ieee802.3 OIDs (table 4-3) */
@@ -780,12 +787,12 @@
 
     /* mandatory */
     case OID_802_3_MULTICAST_LIST:
-        *((le32 *) outbuf) = cpu_to_le32(0xe0000000);
+        stl_le_p(outbuf, 0xe0000000);
         return sizeof(le32);
 
     /* mandatory */
     case OID_802_3_MAXIMUM_LIST_SIZE:
-        *((le32 *) outbuf) = cpu_to_le32(1);
+        stl_le_p(outbuf, 1);
         return sizeof(le32);
 
     case OID_802_3_MAC_OPTIONS:
@@ -794,17 +801,17 @@
     /* ieee802.3 statistics OIDs (table 4-4) */
     /* mandatory */
     case OID_802_3_RCV_ERROR_ALIGNMENT:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_802_3_XMIT_ONE_COLLISION:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     /* mandatory */
     case OID_802_3_XMIT_MORE_COLLISIONS:
-        *((le32 *) outbuf) = cpu_to_le32(0);
+        stl_le_p(outbuf, 0);
         return sizeof(le32);
 
     default:
@@ -819,7 +826,7 @@
 {
     switch (oid) {
     case OID_GEN_CURRENT_PACKET_FILTER:
-        s->filter = le32_to_cpup((le32 *) inbuf);
+        s->filter = ldl_le_p(inbuf);
         if (s->filter) {
             s->rndis_state = RNDIS_DATA_INITIALIZED;
         } else {
@@ -911,8 +918,9 @@
 
     bufoffs = le32_to_cpu(buf->InformationBufferOffset) + 8;
     buflen = le32_to_cpu(buf->InformationBufferLength);
-    if (bufoffs + buflen > length)
+    if (buflen > length || bufoffs >= length || bufoffs + buflen > length) {
         return USB_RET_STALL;
+    }
 
     infobuflen = ndis_query(s, le32_to_cpu(buf->OID),
                             bufoffs + (uint8_t *) buf, buflen, infobuf,
@@ -957,8 +965,9 @@
 
     bufoffs = le32_to_cpu(buf->InformationBufferOffset) + 8;
     buflen = le32_to_cpu(buf->InformationBufferLength);
-    if (bufoffs + buflen > length)
+    if (buflen > length || bufoffs >= length || bufoffs + buflen > length) {
         return USB_RET_STALL;
+    }
 
     ret = ndis_set(s, le32_to_cpu(buf->OID),
                     bufoffs + (uint8_t *) buf, buflen);
@@ -1017,10 +1026,7 @@
 
 static int rndis_parse(USBNetState *s, uint8_t *data, int length)
 {
-    uint32_t msg_type;
-    le32 *tmp = (le32 *) data;
-
-    msg_type = le32_to_cpup(tmp);
+    uint32_t msg_type = ldl_le_p(data);
 
     switch (msg_type) {
     case RNDIS_INITIALIZE_MSG:
@@ -1208,8 +1214,9 @@
     if (le32_to_cpu(msg->MessageType) == RNDIS_PACKET_MSG) {
         uint32_t offs = 8 + le32_to_cpu(msg->DataOffset);
         uint32_t size = le32_to_cpu(msg->DataLength);
-        if (offs + size <= len)
+        if (offs < len && size < len && offs + size <= len) {
             qemu_send_packet(qemu_get_queue(s->nic), s->out_buf + offs, size);
+        }
     }
     s->out_ptr -= len;
     memmove(s->out_buf, &s->out_buf[len], s->out_ptr);
@@ -1265,6 +1272,10 @@
     uint8_t *in_buf = s->in_buf;
     size_t total_size = size;
 
+    if (!s->dev.config) {
+        return -1;
+    }
+
     if (is_rndis(s)) {
         if (s->rndis_state != RNDIS_DATA_INITIALIZED) {
             return -1;
@@ -1306,17 +1317,6 @@
     return size;
 }
 
-static int usbnet_can_receive(NetClientState *nc)
-{
-    USBNetState *s = qemu_get_nic_opaque(nc);
-
-    if (is_rndis(s) && s->rndis_state != RNDIS_DATA_INITIALIZED) {
-        return 1;
-    }
-
-    return !s->in_len;
-}
-
 static void usbnet_cleanup(NetClientState *nc)
 {
     USBNetState *s = qemu_get_nic_opaque(nc);
@@ -1334,16 +1334,15 @@
 }
 
 static NetClientInfo net_usbnet_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NIC,
+    .type = NET_CLIENT_DRIVER_NIC,
     .size = sizeof(NICState),
-    .can_receive = usbnet_can_receive,
     .receive = usbnet_receive,
     .cleanup = usbnet_cleanup,
 };
 
 static void usb_net_realize(USBDevice *dev, Error **errrp)
 {
-    USBNetState *s = DO_UPCAST(USBNetState, dev, dev);
+    USBNetState *s = USB_NET(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
@@ -1376,7 +1375,7 @@
 static void usb_net_instance_init(Object *obj)
 {
     USBDevice *dev = USB_DEVICE(obj);
-    USBNetState *s = DO_UPCAST(USBNetState, dev, dev);
+    USBNetState *s = USB_NET(dev);
 
     device_add_bootindex_property(obj, &s->conf.bootindex,
                                   "bootindex", "/ethernet-phy@0",
@@ -1390,26 +1389,21 @@
     QemuOpts *opts;
     int idx;
 
-    opts = qemu_opts_parse(qemu_find_opts("net"), cmdline, 0);
+    opts = qemu_opts_parse_noisily(qemu_find_opts("net"), cmdline, false);
     if (!opts) {
         return NULL;
     }
-    qemu_opt_set(opts, "type", "nic");
-    qemu_opt_set(opts, "model", "usb");
+    qemu_opt_set(opts, "type", "nic", &error_abort);
+    qemu_opt_set(opts, "model", "usb", &error_abort);
 
-    idx = net_client_init(opts, 0, &local_err);
+    idx = net_client_init(opts, false, &local_err);
     if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         return NULL;
     }
 
     dev = usb_create(bus, "usb-net");
-    if (!dev) {
-        return NULL;
-    }
     qdev_set_nic_properties(&dev->qdev, &nd_table[idx]);
-    qdev_init_nofail(&dev->qdev);
     return dev;
 }
 
@@ -1442,7 +1436,7 @@
 }
 
 static const TypeInfo net_info = {
-    .name          = "usb-net",
+    .name          = TYPE_USB_NET,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBNetState),
     .class_init    = usb_net_class_initfn,
@@ -1452,7 +1446,7 @@
 static void usb_net_register_types(void)
 {
     type_register_static(&net_info);
-    usb_legacy_register("usb-net", "net", usb_net_init);
+    usb_legacy_register(TYPE_USB_NET, "net", usb_net_init);
 }
 
 type_init(usb_net_register_types)
diff --git a/hw/usb/dev-serial.c b/hw/usb/dev-serial.c
index 1cee450..ba8538e 100644
--- a/hw/usb/dev-serial.c
+++ b/hw/usb/dev-serial.c
@@ -8,8 +8,11 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
-#include "monitor/monitor.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
 #include "sysemu/char.h"
@@ -103,6 +106,9 @@
     CharDriverState *cs;
 } USBSerialState;
 
+#define TYPE_USB_SERIAL "usb-serial-dev"
+#define USB_SERIAL_DEV(obj) OBJECT_CHECK(USBSerialState, (obj), TYPE_USB_SERIAL)
+
 enum {
     STR_MANUFACTURER = 1,
     STR_PRODUCT_SERIAL,
@@ -473,7 +479,7 @@
 
 static void usb_serial_realize(USBDevice *dev, Error **errp)
 {
-    USBSerialState *s = DO_UPCAST(USBSerialState, dev, dev);
+    USBSerialState *s = USB_SERIAL_DEV(dev);
     Error *local_err = NULL;
 
     usb_desc_create_serial(dev);
@@ -544,16 +550,11 @@
         return NULL;
 
     dev = usb_create(bus, "usb-serial");
-    if (!dev) {
-        return NULL;
-    }
     qdev_prop_set_chr(&dev->qdev, "chardev", cdrv);
     if (vendorid)
         qdev_prop_set_uint16(&dev->qdev, "vendorid", vendorid);
     if (productid)
         qdev_prop_set_uint16(&dev->qdev, "productid", productid);
-    qdev_init_nofail(&dev->qdev);
-
     return dev;
 }
 
@@ -568,8 +569,6 @@
 
     dev = usb_create(bus, "usb-braille");
     qdev_prop_set_chr(&dev->qdev, "chardev", cdrv);
-    qdev_init_nofail(&dev->qdev);
-
     return dev;
 }
 
@@ -583,26 +582,40 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void usb_serial_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
+
+    uc->realize        = usb_serial_realize;
+    uc->handle_reset   = usb_serial_handle_reset;
+    uc->handle_control = usb_serial_handle_control;
+    uc->handle_data    = usb_serial_handle_data;
+    dc->vmsd = &vmstate_usb_serial;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
+}
+
+static const TypeInfo usb_serial_dev_type_info = {
+    .name = TYPE_USB_SERIAL,
+    .parent = TYPE_USB_DEVICE,
+    .instance_size = sizeof(USBSerialState),
+    .abstract = true,
+    .class_init = usb_serial_dev_class_init,
+};
+
 static void usb_serial_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->realize = usb_serial_realize;
     uc->product_desc   = "QEMU USB Serial";
     uc->usb_desc       = &desc_serial;
-    uc->handle_reset   = usb_serial_handle_reset;
-    uc->handle_control = usb_serial_handle_control;
-    uc->handle_data    = usb_serial_handle_data;
-    dc->vmsd = &vmstate_usb_serial;
     dc->props = serial_properties;
-    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo serial_info = {
     .name          = "usb-serial",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBSerialState),
+    .parent        = TYPE_USB_SERIAL,
     .class_init    = usb_serial_class_initfn,
 };
 
@@ -616,26 +629,20 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
 
-    uc->realize        = usb_serial_realize;
     uc->product_desc   = "QEMU USB Braille";
     uc->usb_desc       = &desc_braille;
-    uc->handle_reset   = usb_serial_handle_reset;
-    uc->handle_control = usb_serial_handle_control;
-    uc->handle_data    = usb_serial_handle_data;
-    dc->vmsd = &vmstate_usb_serial;
     dc->props = braille_properties;
-    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
 static const TypeInfo braille_info = {
     .name          = "usb-braille",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(USBSerialState),
+    .parent        = TYPE_USB_SERIAL,
     .class_init    = usb_braille_class_initfn,
 };
 
 static void usb_serial_register_types(void)
 {
+    type_register_static(&usb_serial_dev_type_info);
     type_register_static(&serial_info);
     usb_legacy_register("usb-serial", "serial", usb_serial_init);
     type_register_static(&braille_info);
diff --git a/hw/usb/dev-smartcard-reader.c b/hw/usb/dev-smartcard-reader.c
index 78ce681..af4b851 100644
--- a/hw/usb/dev-smartcard-reader.c
+++ b/hw/usb/dev-smartcard-reader.c
@@ -34,11 +34,12 @@
  *  Not sure which messages trigger this.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "hw/usb.h"
 #include "hw/usb/desc.h"
-#include "monitor/monitor.h"
 
 #include "ccid.h"
 
@@ -55,7 +56,7 @@
 #define D_VERBOSE 4
 
 #define CCID_DEV_NAME "usb-ccid"
-
+#define USB_CCID_DEV(obj) OBJECT_CHECK(USBCCIDState, (obj), CCID_DEV_NAME)
 /*
  * The two options for variable sized buffers:
  * make them constant size, for large enough constant,
@@ -284,6 +285,7 @@
 typedef struct USBCCIDState {
     USBDevice dev;
     USBEndpoint *intr;
+    USBEndpoint *bulk;
     CCIDBus bus;
     CCIDCardState *card;
     BulkIn bulk_in_pending[BULK_IN_PENDING_NUM]; /* circular */
@@ -649,7 +651,7 @@
 
 static void ccid_handle_reset(USBDevice *dev)
 {
-    USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     DPRINTF(s, 1, "Reset\n");
 
@@ -692,7 +694,7 @@
 static void ccid_handle_control(USBDevice *dev, USBPacket *p, int request,
                                int value, int index, int length, uint8_t *data)
 {
-    USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
+    USBCCIDState *s = USB_CCID_DEV(dev);
     int ret;
 
     DPRINTF(s, 1, "%s: got control %s (%x), value %x\n", __func__,
@@ -770,6 +772,7 @@
     h->b.bError = s->bError;
     h->bClockStatus = CLOCK_STATUS_RUNNING;
     ccid_reset_error_status(s);
+    usb_wakeup(s->bulk, 0);
 }
 
 static void ccid_write_parameters(USBCCIDState *s, CCID_Header *recv)
@@ -790,6 +793,7 @@
     h->bProtocolNum = s->bProtocolNum;
     h->abProtocolDataStructure = s->abProtocolDataStructure;
     ccid_reset_error_status(s);
+    usb_wakeup(s->bulk, 0);
 }
 
 static void ccid_write_data_block(USBCCIDState *s, uint8_t slot, uint8_t seq,
@@ -811,6 +815,7 @@
     }
     memcpy(p->abData, data, len);
     ccid_reset_error_status(s);
+    usb_wakeup(s->bulk, 0);
 }
 
 static void ccid_report_error_failed(USBCCIDState *s, uint8_t error)
@@ -1104,7 +1109,7 @@
 
 static void ccid_handle_data(USBDevice *dev, USBPacket *p)
 {
-    USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
+    USBCCIDState *s = USB_CCID_DEV(dev);
     uint8_t buf[2];
 
     switch (p->pid) {
@@ -1148,7 +1153,7 @@
 
 static void ccid_handle_destroy(USBDevice *dev)
 {
-    USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     ccid_bulk_in_clear(s);
 }
@@ -1184,8 +1189,9 @@
 void ccid_card_send_apdu_to_guest(CCIDCardState *card,
                                   uint8_t *apdu, uint32_t len)
 {
-    USBCCIDState *s = DO_UPCAST(USBCCIDState, dev.qdev,
-                                card->qdev.parent_bus->parent);
+    DeviceState *qdev = DEVICE(card);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
     Answer *answer;
 
     if (!ccid_has_pending_answers(s)) {
@@ -1206,8 +1212,9 @@
 
 void ccid_card_card_removed(CCIDCardState *card)
 {
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    DeviceState *qdev = DEVICE(card);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     ccid_on_slot_change(s, false);
     ccid_flush_pending_answers(s);
@@ -1216,8 +1223,9 @@
 
 int ccid_card_ccid_attach(CCIDCardState *card)
 {
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    DeviceState *qdev = DEVICE(card);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     DPRINTF(s, 1, "CCID Attach\n");
     if (s->migration_state == MIGRATION_MIGRATED) {
@@ -1228,8 +1236,9 @@
 
 void ccid_card_ccid_detach(CCIDCardState *card)
 {
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    DeviceState *qdev = DEVICE(card);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     DPRINTF(s, 1, "CCID Detach\n");
     if (ccid_card_inserted(s)) {
@@ -1240,8 +1249,9 @@
 
 void ccid_card_card_error(CCIDCardState *card, uint64_t error)
 {
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    DeviceState *qdev = DEVICE(card);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     s->bmCommandStatus = COMMAND_STATUS_FAILED;
     s->last_answer_error = error;
@@ -1258,8 +1268,9 @@
 
 void ccid_card_card_inserted(CCIDCardState *card)
 {
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    DeviceState *qdev = DEVICE(card);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     s->bmCommandStatus = COMMAND_STATUS_NO_ERROR;
     ccid_flush_pending_answers(s);
@@ -1270,8 +1281,8 @@
 {
     int ret = 0;
     CCIDCardState *card = CCID_CARD(qdev);
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     if (ccid_card_inserted(s)) {
         ccid_card_card_removed(card);
@@ -1284,8 +1295,8 @@
 static int ccid_card_init(DeviceState *qdev)
 {
     CCIDCardState *card = CCID_CARD(qdev);
-    USBCCIDState *s =
-        DO_UPCAST(USBCCIDState, dev.qdev, card->qdev.parent_bus->parent);
+    USBDevice *dev = USB_DEVICE(qdev->parent_bus->parent);
+    USBCCIDState *s = USB_CCID_DEV(dev);
     int ret = 0;
 
     if (card->slot != 0) {
@@ -1306,7 +1317,7 @@
 
 static void ccid_realize(USBDevice *dev, Error **errp)
 {
-    USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
+    USBCCIDState *s = USB_CCID_DEV(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
@@ -1314,6 +1325,7 @@
                         NULL);
     qbus_set_hotplug_handler(BUS(&s->bus), DEVICE(dev), &error_abort);
     s->intr = usb_ep_get(dev, USB_TOKEN_IN, CCID_INT_IN_EP);
+    s->bulk = usb_ep_get(dev, USB_TOKEN_IN, CCID_BULK_IN_EP);
     s->card = NULL;
     s->migration_state = MIGRATION_NONE;
     s->migration_target_ip = 0;
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index 4539733..c607f76 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -7,7 +7,10 @@
  * This code is licensed under the LGPL.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/error-report.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "hw/usb.h"
@@ -19,6 +22,7 @@
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 #include "qapi/visitor.h"
+#include "qemu/cutils.h"
 
 //#define DEBUG_MSD
 
@@ -64,6 +68,9 @@
     SCSIDevice *scsi_dev;
 } MSDState;
 
+#define TYPE_USB_STORAGE "usb-storage-dev"
+#define USB_STORAGE_DEV(obj) OBJECT_CHECK(MSDState, (obj), TYPE_USB_STORAGE)
+
 struct usb_msd_cbw {
     uint32_t sig;
     uint32_t tag;
@@ -385,7 +392,7 @@
 
 static void usb_msd_cancel_io(USBDevice *dev, USBPacket *p)
 {
-    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+    MSDState *s = USB_STORAGE_DEV(dev);
 
     assert(s->packet == p);
     s->packet = NULL;
@@ -549,22 +556,6 @@
     }
 }
 
-static void usb_msd_password_cb(void *opaque, int err)
-{
-    MSDState *s = opaque;
-    Error *local_err = NULL;
-
-    if (!err) {
-        usb_device_attach(&s->dev, &local_err);
-    }
-
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        qdev_unplug(&s->dev.qdev, NULL);
-    }
-}
-
 static void *usb_msd_load_request(QEMUFile *f, SCSIRequest *req)
 {
     MSDState *s = DO_UPCAST(MSDState, dev.qdev, req->bus->qbus.parent);
@@ -600,7 +591,7 @@
 
 static void usb_msd_realize_storage(USBDevice *dev, Error **errp)
 {
-    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+    MSDState *s = USB_STORAGE_DEV(dev);
     BlockBackend *blk = s->conf.blk;
     SCSIDevice *scsi_dev;
     Error *err = NULL;
@@ -611,16 +602,20 @@
     }
 
     blkconf_serial(&s->conf, &dev->serial);
+    blkconf_blocksizes(&s->conf);
+    blkconf_apply_backend_options(&s->conf);
 
     /*
      * Hack alert: this pretends to be a block device, but it's really
      * a SCSI bus that can serve only a single device, which it
      * creates automatically.  But first it needs to detach from its
      * blockdev, or else scsi_bus_legacy_add_drive() dies when it
-     * attaches again.
+     * attaches again. We also need to take another reference so that
+     * blk_detach_dev() doesn't free blk while we still need it.
      *
      * The hack is probably a bad idea.
      */
+    blk_ref(blk);
     blk_detach_dev(blk, &s->dev.qdev);
     s->conf.blk = NULL;
 
@@ -631,30 +626,26 @@
     scsi_dev = scsi_bus_legacy_add_drive(&s->bus, blk, 0, !!s->removable,
                                          s->conf.bootindex, dev->serial,
                                          &err);
+    blk_unref(blk);
     if (!scsi_dev) {
         error_propagate(errp, err);
         return;
     }
     usb_msd_handle_reset(dev);
     s->scsi_dev = scsi_dev;
-
-    if (bdrv_key_required(blk_bs(blk))) {
-        if (cur_mon) {
-            monitor_read_bdrv_key_start(cur_mon, blk_bs(blk),
-                                        usb_msd_password_cb, s);
-            s->dev.auto_attach = 0;
-        } else {
-            autostart = 0;
-        }
-    }
 }
 
 static void usb_msd_realize_bot(USBDevice *dev, Error **errp)
 {
-    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+    MSDState *s = USB_STORAGE_DEV(dev);
+    DeviceState *d = DEVICE(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    if (d->hotplugged) {
+        s->dev.auto_attach = 0;
+    }
+
     scsi_bus_new(&s->bus, sizeof(s->bus), DEVICE(dev),
                  &usb_msd_scsi_info_bot, NULL);
     usb_msd_handle_reset(dev);
@@ -663,6 +654,7 @@
 static USBDevice *usb_msd_init(USBBus *bus, const char *filename)
 {
     static int nr=0;
+    Error *err = NULL;
     char id[8];
     QemuOpts *opts;
     DriveInfo *dinfo;
@@ -683,7 +675,7 @@
         if (strstart(filename, "format=", &p2)) {
             int len = MIN(p1 - p2, sizeof(fmt));
             pstrcpy(fmt, len, p2);
-            qemu_opt_set(opts, "format", fmt);
+            qemu_opt_set(opts, "format", fmt, &error_abort);
         } else if (*filename != ':') {
             error_report("unrecognized USB mass-storage option %s", filename);
             return NULL;
@@ -694,8 +686,8 @@
         error_report("block device specification needed");
         return NULL;
     }
-    qemu_opt_set(opts, "file", filename);
-    qemu_opt_set(opts, "if", "none");
+    qemu_opt_set(opts, "file", filename, &error_abort);
+    qemu_opt_set(opts, "if", "none", &error_abort);
 
     /* create host drive */
     dinfo = drive_new(opts, 0);
@@ -706,17 +698,13 @@
 
     /* create guest device */
     dev = usb_create(bus, "usb-storage");
-    if (!dev) {
-        return NULL;
-    }
-    if (qdev_prop_set_drive(&dev->qdev, "drive",
-                            blk_by_legacy_dinfo(dinfo)) < 0) {
+    qdev_prop_set_drive(&dev->qdev, "drive", blk_by_legacy_dinfo(dinfo),
+                        &err);
+    if (err) {
+        error_report_err(err);
         object_unparent(OBJECT(dev));
         return NULL;
     }
-    if (qdev_init(&dev->qdev) < 0)
-        return NULL;
-
     return dev;
 }
 
@@ -744,7 +732,7 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void usb_msd_class_initfn_common(ObjectClass *klass)
+static void usb_msd_class_initfn_common(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
@@ -768,27 +756,26 @@
 
     uc->realize = usb_msd_realize_storage;
     dc->props = msd_properties;
-    usb_msd_class_initfn_common(klass);
 }
 
-static void usb_msd_get_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void usb_msd_get_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(obj);
-    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+    MSDState *s = USB_STORAGE_DEV(dev);
 
-    visit_type_int32(v, &s->conf.bootindex, name, errp);
+    visit_type_int32(v, name, &s->conf.bootindex, errp);
 }
 
-static void usb_msd_set_bootindex(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
+static void usb_msd_set_bootindex(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     USBDevice *dev = USB_DEVICE(obj);
-    MSDState *s = DO_UPCAST(MSDState, dev, dev);
+    MSDState *s = USB_STORAGE_DEV(dev);
     int32_t boot_index;
     Error *local_err = NULL;
 
-    visit_type_int32(v, &boot_index, name, &local_err);
+    visit_type_int32(v, name, &boot_index, &local_err);
     if (local_err) {
         goto out;
     }
@@ -806,11 +793,17 @@
     }
 
 out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
+static const TypeInfo usb_storage_dev_type_info = {
+    .name = TYPE_USB_STORAGE,
+    .parent = TYPE_USB_DEVICE,
+    .instance_size = sizeof(MSDState),
+    .abstract = true,
+    .class_init = usb_msd_class_initfn_common,
+};
+
 static void usb_msd_instance_init(Object *obj)
 {
     object_property_add(obj, "bootindex", "int32",
@@ -822,30 +815,27 @@
 static void usb_msd_class_initfn_bot(ObjectClass *klass, void *data)
 {
     USBDeviceClass *uc = USB_DEVICE_CLASS(klass);
-    DeviceClass *dc = DEVICE_CLASS(klass);
 
     uc->realize = usb_msd_realize_bot;
-    usb_msd_class_initfn_common(klass);
-    dc->hotpluggable = false;
+    uc->attached_settable = true;
 }
 
 static const TypeInfo msd_info = {
     .name          = "usb-storage",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(MSDState),
+    .parent        = TYPE_USB_STORAGE,
     .class_init    = usb_msd_class_initfn_storage,
     .instance_init = usb_msd_instance_init,
 };
 
 static const TypeInfo bot_info = {
     .name          = "usb-bot",
-    .parent        = TYPE_USB_DEVICE,
-    .instance_size = sizeof(MSDState),
+    .parent        = TYPE_USB_STORAGE,
     .class_init    = usb_msd_class_initfn_bot,
 };
 
 static void usb_msd_register_types(void)
 {
+    type_register_static(&usb_storage_dev_type_info);
     type_register_static(&msd_info);
     type_register_static(&bot_info);
     usb_legacy_register("usb-storage", "disk", usb_msd_init);
diff --git a/hw/usb/dev-uas.c b/hw/usb/dev-uas.c
index 04fc515..3a8ff18 100644
--- a/hw/usb/dev-uas.c
+++ b/hw/usb/dev-uas.c
@@ -9,6 +9,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
@@ -127,6 +128,9 @@
     USBPacket                 *status3[UAS_MAX_STREAMS + 1];
 };
 
+#define TYPE_USB_UAS "usb-uas"
+#define USB_UAS(obj) OBJECT_CHECK(UASDevice, (obj), TYPE_USB_UAS)
+
 struct UASRequest {
     uint16_t     tag;
     uint64_t     lun;
@@ -626,7 +630,7 @@
 
 static void usb_uas_handle_reset(USBDevice *dev)
 {
-    UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
+    UASDevice *uas = USB_UAS(dev);
     UASRequest *req, *nreq;
     UASStatus *st, *nst;
 
@@ -655,7 +659,7 @@
 
 static void usb_uas_cancel_io(USBDevice *dev, USBPacket *p)
 {
-    UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
+    UASDevice *uas = USB_UAS(dev);
     UASRequest *req, *nreq;
     int i;
 
@@ -797,7 +801,7 @@
 
 static void usb_uas_handle_data(USBDevice *dev, USBPacket *p)
 {
-    UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
+    UASDevice *uas = USB_UAS(dev);
     uas_iu iu;
     UASStatus *st;
     UASRequest *req;
@@ -888,17 +892,21 @@
 
 static void usb_uas_handle_destroy(USBDevice *dev)
 {
-    UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
+    UASDevice *uas = USB_UAS(dev);
 
     qemu_bh_delete(uas->status_bh);
 }
 
 static void usb_uas_realize(USBDevice *dev, Error **errp)
 {
-    UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
+    UASDevice *uas = USB_UAS(dev);
+    DeviceState *d = DEVICE(dev);
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    if (d->hotplugged) {
+        uas->dev.auto_attach = 0;
+    }
 
     QTAILQ_INIT(&uas->results);
     QTAILQ_INIT(&uas->requests);
@@ -936,6 +944,7 @@
     uc->handle_control = usb_uas_handle_control;
     uc->handle_data    = usb_uas_handle_data;
     uc->handle_destroy = usb_uas_handle_destroy;
+    uc->attached_settable = true;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->fw_name = "storage";
     dc->vmsd = &vmstate_usb_uas;
@@ -943,7 +952,7 @@
 }
 
 static const TypeInfo uas_info = {
-    .name          = "usb-uas",
+    .name          = TYPE_USB_UAS,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(UASDevice),
     .class_init    = usb_uas_class_initfn,
diff --git a/hw/usb/dev-wacom.c b/hw/usb/dev-wacom.c
index 844eafa..c4702db 100644
--- a/hw/usb/dev-wacom.c
+++ b/hw/usb/dev-wacom.c
@@ -25,6 +25,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "ui/console.h"
 #include "hw/usb.h"
@@ -56,6 +57,9 @@
     int changed;
 } USBWacomState;
 
+#define TYPE_USB_WACOM "usb-wacom-tablet"
+#define USB_WACOM(obj) OBJECT_CHECK(USBWacomState, (obj), TYPE_USB_WACOM)
+
 enum {
     STR_MANUFACTURER = 1,
     STR_PRODUCT,
@@ -337,7 +341,7 @@
 
 static void usb_wacom_realize(USBDevice *dev, Error **errp)
 {
-    USBWacomState *s = DO_UPCAST(USBWacomState, dev, dev);
+    USBWacomState *s = USB_WACOM(dev);
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
     s->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
@@ -367,7 +371,7 @@
 }
 
 static const TypeInfo wacom_info = {
-    .name          = "usb-wacom-tablet",
+    .name          = TYPE_USB_WACOM,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBWacomState),
     .class_init    = usb_wacom_class_init,
@@ -376,7 +380,7 @@
 static void usb_wacom_register_types(void)
 {
     type_register_static(&wacom_info);
-    usb_legacy_register("usb-wacom-tablet", "wacom-tablet", NULL);
+    usb_legacy_register(TYPE_USB_WACOM, "wacom-tablet", NULL);
 }
 
 type_init(usb_wacom_register_types)
diff --git a/hw/usb/hcd-ehci-pci.c b/hw/usb/hcd-ehci-pci.c
index 490f2b6..5657705 100644
--- a/hw/usb/hcd-ehci-pci.c
+++ b/hw/usb/hcd-ehci-pci.c
@@ -15,6 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/usb/hcd-ehci.h"
 #include "qemu/range.h"
 
@@ -26,7 +27,7 @@
     bool companion;
 } EHCIPCIInfo;
 
-static int usb_ehci_pci_initfn(PCIDevice *dev)
+static void usb_ehci_pci_realize(PCIDevice *dev, Error **errp)
 {
     EHCIPCIState *i = PCI_EHCI(dev);
     EHCIState *s = &i->ehci;
@@ -66,8 +67,6 @@
 
     usb_ehci_realize(s, DEVICE(dev), NULL);
     pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mem);
-
-    return 0;
 }
 
 static void usb_ehci_pci_init(Object *obj)
@@ -97,10 +96,17 @@
 
     usb_ehci_unrealize(s, DEVICE(dev), NULL);
 
-    if (s->irq) {
-        g_free(s->irq);
-        s->irq = NULL;
-    }
+    g_free(s->irq);
+    s->irq = NULL;
+}
+
+static void usb_ehci_pci_reset(DeviceState *dev)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(dev);
+    EHCIPCIState *i = PCI_EHCI(pci_dev);
+    EHCIState *s = &i->ehci;
+
+    ehci_reset(s);
 }
 
 static void usb_ehci_pci_write_config(PCIDevice *dev, uint32_t addr,
@@ -139,12 +145,13 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = usb_ehci_pci_initfn;
+    k->realize = usb_ehci_pci_realize;
     k->exit = usb_ehci_pci_exit;
     k->class_id = PCI_CLASS_SERIAL_USB;
     k->config_write = usb_ehci_pci_write_config;
     dc->vmsd = &vmstate_ehci_pci;
     dc->props = ehci_pci_properties;
+    dc->reset = usb_ehci_pci_reset;
 }
 
 static const TypeInfo ehci_pci_type_info = {
diff --git a/hw/usb/hcd-ehci-sysbus.c b/hw/usb/hcd-ehci-sysbus.c
index 19ed2c2..6c20604 100644
--- a/hw/usb/hcd-ehci-sysbus.c
+++ b/hw/usb/hcd-ehci-sysbus.c
@@ -15,6 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/usb/hcd-ehci.h"
 
 static const VMStateDescription vmstate_ehci_sysbus = {
@@ -42,6 +43,15 @@
     sysbus_init_irq(d, &s->irq);
 }
 
+static void usb_ehci_sysbus_reset(DeviceState *dev)
+{
+    SysBusDevice *d = SYS_BUS_DEVICE(dev);
+    EHCISysBusState *i = SYS_BUS_EHCI(d);
+    EHCIState *s = &i->ehci;
+
+    ehci_reset(s);
+}
+
 static void ehci_sysbus_init(Object *obj)
 {
     SysBusDevice *d = SYS_BUS_DEVICE(obj);
@@ -70,6 +80,7 @@
     dc->realize = usb_ehci_sysbus_realize;
     dc->vmsd = &vmstate_ehci_sysbus;
     dc->props = ehci_sysbus_properties;
+    dc->reset = usb_ehci_sysbus_reset;
     set_bit(DEVICE_CATEGORY_USB, dc->categories);
 }
 
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index 1cc0fc1..b093db7 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -27,12 +27,14 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/usb/ehci-regs.h"
 #include "hw/usb/hcd-ehci.h"
 #include "trace.h"
 
 #define FRAME_TIMER_FREQ 1000
-#define FRAME_TIMER_NS   (1000000000 / FRAME_TIMER_FREQ)
+#define FRAME_TIMER_NS   (NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ)
 #define UFRAME_TIMER_NS  (FRAME_TIMER_NS / 8)
 
 #define NB_MAXINTRATE    8        // Max rate at which controller issues ints
@@ -726,7 +728,7 @@
     ehci_queues_rip_device(s, port->dev, 0);
     ehci_queues_rip_device(s, port->dev, 1);
 
-    *portsc &= ~(PORTSC_CONNECT|PORTSC_PED);
+    *portsc &= ~(PORTSC_CONNECT|PORTSC_PED|PORTSC_SUSPEND);
     *portsc |= PORTSC_CSC;
 
     ehci_raise_irq(s, USBSTS_PCD);
@@ -769,30 +771,26 @@
     qemu_bh_schedule(s->async_bh);
 }
 
-static int ehci_register_companion(USBBus *bus, USBPort *ports[],
-                                   uint32_t portcount, uint32_t firstport)
+static void ehci_register_companion(USBBus *bus, USBPort *ports[],
+                                    uint32_t portcount, uint32_t firstport,
+                                    Error **errp)
 {
     EHCIState *s = container_of(bus, EHCIState, bus);
     uint32_t i;
 
     if (firstport + portcount > NB_PORTS) {
-        qerror_report(QERR_INVALID_PARAMETER_VALUE, "firstport",
-                      "firstport on masterbus");
-        error_printf_unless_qmp(
-            "firstport value of %u makes companion take ports %u - %u, which "
-            "is outside of the valid range of 0 - %u\n", firstport, firstport,
-            firstport + portcount - 1, NB_PORTS - 1);
-        return -1;
+        error_setg(errp, "firstport must be between 0 and %u",
+                   NB_PORTS - portcount);
+        return;
     }
 
     for (i = 0; i < portcount; i++) {
         if (s->companion_ports[firstport + i]) {
-            qerror_report(QERR_INVALID_PARAMETER_VALUE, "masterbus",
-                          "an USB masterbus");
-            error_printf_unless_qmp(
-                "port %u on masterbus %s already has a companion assigned\n",
-                firstport + i, bus->qbus.name);
-            return -1;
+            error_setg(errp, "firstport %u asks for ports %u-%u,"
+                       " but port %u has a companion assigned already",
+                       firstport, firstport, firstport + portcount - 1,
+                       firstport + i);
+            return;
         }
     }
 
@@ -806,8 +804,6 @@
 
     s->companion_count++;
     s->caps[0x05] = (s->companion_count << 4) | portcount;
-
-    return 0;
 }
 
 static void ehci_wakeup_endpoint(USBBus *bus, USBEndpoint *ep,
@@ -845,7 +841,7 @@
 }
 
 /* 4.1 host controller initialization */
-static void ehci_reset(void *opaque)
+void ehci_reset(void *opaque)
 {
     EHCIState *s = opaque;
     int i;
@@ -871,6 +867,7 @@
     s->usbsts = USBSTS_HALT;
     s->usbsts_pending = 0;
     s->usbsts_frindex = 0;
+    ehci_update_irq(s);
 
     s->astate = EST_INACTIVE;
     s->pstate = EST_INACTIVE;
@@ -899,6 +896,11 @@
     return s->caps[addr];
 }
 
+static void ehci_caps_write(void *ptr, hwaddr addr,
+                             uint64_t val, unsigned size)
+{
+}
+
 static uint64_t ehci_opreg_read(void *ptr, hwaddr addr,
                                 unsigned size)
 {
@@ -1410,21 +1412,23 @@
         if (itd->transact[i] & ITD_XACT_ACTIVE) {
             pg   = get_field(itd->transact[i], ITD_XACT_PGSEL);
             off  = itd->transact[i] & ITD_XACT_OFFSET_MASK;
-            ptr1 = (itd->bufptr[pg] & ITD_BUFPTR_MASK);
-            ptr2 = (itd->bufptr[pg+1] & ITD_BUFPTR_MASK);
             len  = get_field(itd->transact[i], ITD_XACT_LENGTH);
 
             if (len > max * mult) {
                 len = max * mult;
             }
-
-            if (len > BUFF_SIZE) {
+            if (len > BUFF_SIZE || pg > 6) {
                 return -1;
             }
 
+            ptr1 = (itd->bufptr[pg] & ITD_BUFPTR_MASK);
             qemu_sglist_init(&ehci->isgl, ehci->device, 2, ehci->as);
             if (off + len > 4096) {
                 /* transfer crosses page border */
+                if (pg == 6) {
+                    return -1;  /* avoid page pg + 1 */
+                }
+                ptr2 = (itd->bufptr[pg + 1] & ITD_BUFPTR_MASK);
                 uint32_t len2 = off + len - 4096;
                 uint32_t len1 = len - len2;
                 qemu_sglist_add(&ehci->isgl, ptr1 + off, len1);
@@ -2006,6 +2010,7 @@
 static void ehci_advance_state(EHCIState *ehci, int async)
 {
     EHCIQueue *q = NULL;
+    int itd_count = 0;
     int again;
 
     do {
@@ -2030,10 +2035,12 @@
 
         case EST_FETCHITD:
             again = ehci_state_fetchitd(ehci, async);
+            itd_count++;
             break;
 
         case EST_FETCHSITD:
             again = ehci_state_fetchsitd(ehci, async);
+            itd_count++;
             break;
 
         case EST_ADVANCEQUEUE:
@@ -2082,7 +2089,8 @@
             break;
         }
 
-        if (again < 0) {
+        if (again < 0 || itd_count > 16) {
+            /* TODO: notify guest (raise HSE irq?) */
             fprintf(stderr, "processing error - resetting ehci HC\n");
             ehci_reset(ehci);
             again = 0;
@@ -2198,29 +2206,28 @@
 
 static void ehci_update_frindex(EHCIState *ehci, int uframes)
 {
-    int i;
-
     if (!ehci_enabled(ehci) && ehci->pstate == EST_INACTIVE) {
         return;
     }
 
-    for (i = 0; i < uframes; i++) {
-        ehci->frindex++;
+    /* Generate FLR interrupt if frame index rolls over 0x2000 */
+    if ((ehci->frindex % 0x2000) + uframes >= 0x2000) {
+        ehci_raise_irq(ehci, USBSTS_FLR);
+    }
 
-        if (ehci->frindex == 0x00002000) {
-            ehci_raise_irq(ehci, USBSTS_FLR);
-        }
-
-        if (ehci->frindex == 0x00004000) {
-            ehci_raise_irq(ehci, USBSTS_FLR);
-            ehci->frindex = 0;
-            if (ehci->usbsts_frindex >= 0x00004000) {
-                ehci->usbsts_frindex -= 0x00004000;
-            } else {
-                ehci->usbsts_frindex = 0;
-            }
+    /* How many times will frindex roll over 0x4000 with this frame count?
+     * usbsts_frindex is decremented by 0x4000 on rollover until it reaches 0
+     */
+    int rollovers = (ehci->frindex + uframes) / 0x4000;
+    if (rollovers > 0) {
+        if (ehci->usbsts_frindex >= (rollovers * 0x4000)) {
+            ehci->usbsts_frindex -= 0x4000 * rollovers;
+        } else {
+            ehci->usbsts_frindex = 0;
         }
     }
+
+    ehci->frindex = (ehci->frindex + uframes) % 0x4000;
 }
 
 static void ehci_frame_timer(void *opaque)
@@ -2304,10 +2311,11 @@
         /* If we've raised int, we speed up the timer, so that we quickly
          * notice any new packets queued up in response */
         if (ehci->int_req_by_async && (ehci->usbsts & USBSTS_INT)) {
-            expire_time = t_now + get_ticks_per_sec() / (FRAME_TIMER_FREQ * 4);
+            expire_time = t_now +
+                NANOSECONDS_PER_SECOND / (FRAME_TIMER_FREQ * 4);
             ehci->int_req_by_async = false;
         } else {
-            expire_time = t_now + (get_ticks_per_sec()
+            expire_time = t_now + (NANOSECONDS_PER_SECOND
                                * (ehci->async_stepdown+1) / FRAME_TIMER_FREQ);
         }
         timer_mod(ehci->frame_timer, expire_time);
@@ -2316,6 +2324,7 @@
 
 static const MemoryRegionOps ehci_mmio_caps_ops = {
     .read = ehci_caps_read,
+    .write = ehci_caps_write,
     .valid.min_access_size = 1,
     .valid.max_access_size = 4,
     .impl.min_access_size = 1,
@@ -2437,7 +2446,7 @@
         VMSTATE_UINT32(portsc[4], EHCIState),
         VMSTATE_UINT32(portsc[5], EHCIState),
         /* frame timer */
-        VMSTATE_TIMER(frame_timer, EHCIState),
+        VMSTATE_TIMER_PTR(frame_timer, EHCIState),
         VMSTATE_UINT64(last_run_ns, EHCIState),
         VMSTATE_UINT32(async_stepdown, EHCIState),
         /* schedule state */
@@ -2471,7 +2480,6 @@
     s->async_bh = qemu_bh_new(ehci_frame_timer, s);
     s->device = dev;
 
-    qemu_register_reset(ehci_reset, s);
     s->vmstate = qemu_add_vm_change_state_handler(usb_ehci_vm_state_change, s);
 }
 
diff --git a/hw/usb/hcd-ehci.h b/hw/usb/hcd-ehci.h
index 2bc259c..3fd7038 100644
--- a/hw/usb/hcd-ehci.h
+++ b/hw/usb/hcd-ehci.h
@@ -14,13 +14,13 @@
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef HW_USB_EHCI_H
-#define HW_USB_EHCI_H 1
+
+#ifndef HW_USB_HCD_EHCI_H
+#define HW_USB_HCD_EHCI_H
 
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
-#include "monitor/monitor.h"
 #include "sysemu/dma.h"
 #include "sysemu/sysemu.h"
 #include "hw/pci/pci.h"
@@ -325,6 +325,7 @@
 void usb_ehci_init(EHCIState *s, DeviceState *dev);
 void usb_ehci_realize(EHCIState *s, DeviceState *dev, Error **errp);
 void usb_ehci_unrealize(EHCIState *s, DeviceState *dev, Error **errp);
+void ehci_reset(void *opaque);
 
 #define TYPE_PCI_EHCI "pci-ehci-usb"
 #define PCI_EHCI(obj) OBJECT_CHECK(EHCIPCIState, (obj), TYPE_PCI_EHCI)
diff --git a/hw/usb/hcd-musb.c b/hw/usb/hcd-musb.c
index 40809f6..27d9d0b 100644
--- a/hw/usb/hcd-musb.c
+++ b/hw/usb/hcd-musb.c
@@ -20,6 +20,7 @@
  *
  * Only host-mode and non-DMA accesses are currently supported.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
@@ -554,14 +555,16 @@
         timeout = ep->timeout[dir];
     else if (ep->interrupt[dir])
         timeout = 8;
-    else
-        return musb_cb_tick(ep);
+    else {
+        musb_cb_tick(ep);
+        return;
+    }
 
     if (!ep->intv_timer[dir])
         ep->intv_timer[dir] = timer_new_ns(QEMU_CLOCK_VIRTUAL, musb_cb_tick, ep);
 
     timer_mod(ep->intv_timer[dir], qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                   muldiv64(timeout, get_ticks_per_sec(), 8000));
+                   muldiv64(timeout, NANOSECONDS_PER_SECOND, 8000));
 }
 
 static int musb_timeout(int ttype, int speed, int val)
@@ -772,9 +775,11 @@
 
         /* NAK timeouts are only generated in Bulk transfers and
          * Data-errors in Isochronous.  */
-        if (ep->interrupt[1])
-            return musb_packet(s, ep, epnum, USB_TOKEN_IN,
-                            packey->iov.size, musb_rx_packet_complete, 1);
+        if (ep->interrupt[1]) {
+            musb_packet(s, ep, epnum, USB_TOKEN_IN,
+                        packey->iov.size, musb_rx_packet_complete, 1);
+            return;
+        }
 
         ep->csr[1] |= MGC_M_RXCSR_DATAERROR;
         if (!epnum)
@@ -864,8 +869,7 @@
          * but it doesn't make sense for us to do that.  */
     }
 
-    return musb_packet(s, ep, epnum, pid,
-                    total, musb_tx_packet_complete, 0);
+    musb_packet(s, ep, epnum, pid, total, musb_tx_packet_complete, 0);
 }
 
 static void musb_rx_req(MUSBState *s, int epnum)
@@ -929,8 +933,7 @@
     }
 #endif
 
-    return musb_packet(s, ep, epnum, USB_TOKEN_IN,
-                    total, musb_rx_packet_complete, 1);
+    musb_packet(s, ep, epnum, USB_TOKEN_IN, total, musb_rx_packet_complete, 1);
 }
 
 static uint8_t musb_read_fifo(MUSBEndPoint *ep)
diff --git a/hw/usb/hcd-ohci.c b/hw/usb/hcd-ohci.c
index 9a84eb6..fa57038 100644
--- a/hw/usb/hcd-ohci.c
+++ b/hw/usb/hcd-ohci.c
@@ -25,7 +25,9 @@
  *  o BIOS work to boot from USB storage
 */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
 #include "hw/pci/pci.h"
@@ -439,15 +441,37 @@
     }
 }
 
-/* Reset the controller */
-static void ohci_reset(void *opaque)
+static void ohci_roothub_reset(OHCIState *ohci)
 {
-    OHCIState *ohci = opaque;
     OHCIPort *port;
     int i;
 
     ohci_bus_stop(ohci);
-    ohci->ctl = 0;
+    ohci->rhdesc_a = OHCI_RHA_NPS | ohci->num_ports;
+    ohci->rhdesc_b = 0x0; /* Impl. specific */
+    ohci->rhstatus = 0;
+
+    for (i = 0; i < ohci->num_ports; i++) {
+        port = &ohci->rhport[i];
+        port->ctrl = 0;
+        if (port->port.dev && port->port.dev->attached) {
+            usb_port_reset(&port->port);
+        }
+    }
+    if (ohci->async_td) {
+        usb_cancel_packet(&ohci->usb_packet);
+        ohci->async_td = 0;
+    }
+    ohci_stop_endpoints(ohci);
+}
+
+/* Reset the controller */
+static void ohci_soft_reset(OHCIState *ohci)
+{
+    trace_usb_ohci_reset(ohci->name);
+
+    ohci_bus_stop(ohci);
+    ohci->ctl = (ohci->ctl & OHCI_CTL_IR) | OHCI_USB_SUSPEND;
     ohci->old_ctl = 0;
     ohci->status = 0;
     ohci->intr_status = 0;
@@ -470,25 +494,13 @@
     ohci->frame_number = 0;
     ohci->pstart = 0;
     ohci->lst = OHCI_LS_THRESH;
+}
 
-    ohci->rhdesc_a = OHCI_RHA_NPS | ohci->num_ports;
-    ohci->rhdesc_b = 0x0; /* Impl. specific */
-    ohci->rhstatus = 0;
-
-    for (i = 0; i < ohci->num_ports; i++)
-      {
-        port = &ohci->rhport[i];
-        port->ctrl = 0;
-        if (port->port.dev && port->port.dev->attached) {
-            usb_port_reset(&port->port);
-        }
-      }
-    if (ohci->async_td) {
-        usb_cancel_packet(&ohci->usb_packet);
-        ohci->async_td = 0;
-    }
-    ohci_stop_endpoints(ohci);
-    trace_usb_ohci_reset(ohci->name);
+static void ohci_hard_reset(OHCIState *ohci)
+{
+    ohci_soft_reset(ohci);
+    ohci->ctl = 0;
+    ohci_roothub_reset(ohci);
 }
 
 /* Get an array of dwords from main memory */
@@ -1231,11 +1243,16 @@
     return active;
 }
 
-/* Generate a SOF event, and set a timer for EOF */
-static void ohci_sof(OHCIState *ohci)
+/* set a timer for EOF */
+static void ohci_eof_timer(OHCIState *ohci)
 {
     ohci->sof_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
     timer_mod(ohci->eof_timer, ohci->sof_time + usb_frame_time);
+}
+/* Set a timer for EOF and generate a SOF event */
+static void ohci_sof(OHCIState *ohci)
+{
+    ohci_eof_timer(ohci);
     ohci_set_interrupt(ohci, OHCI_INTR_SF);
 }
 
@@ -1331,19 +1348,14 @@
  */
 static int ohci_bus_start(OHCIState *ohci)
 {
-    ohci->eof_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
-                    ohci_frame_boundary,
-                    ohci);
-
-    if (ohci->eof_timer == NULL) {
-        trace_usb_ohci_bus_eof_timer_failed(ohci->name);
-        ohci_die(ohci);
-        return 0;
-    }
-
     trace_usb_ohci_start(ohci->name);
 
-    ohci_sof(ohci);
+    /* Delay the first SOF event by one frame time as
+     * linux driver is not ready to receive it and
+     * can meet some race conditions
+     */
+
+    ohci_eof_timer(ohci);
 
     return 1;
 }
@@ -1352,11 +1364,7 @@
 static void ohci_bus_stop(OHCIState *ohci)
 {
     trace_usb_ohci_stop(ohci->name);
-    if (ohci->eof_timer) {
-        timer_del(ohci->eof_timer);
-        timer_free(ohci->eof_timer);
-    }
-    ohci->eof_timer = NULL;
+    timer_del(ohci->eof_timer);
 }
 
 /* Sets a flag in a port status register but only set it if the port is
@@ -1436,12 +1444,15 @@
         break;
     case OHCI_USB_SUSPEND:
         ohci_bus_stop(ohci);
+        /* clear pending SF otherwise linux driver loops in ohci_irq() */
+        ohci->intr_status &= ~OHCI_INTR_SF;
+        ohci_intr_update(ohci);
         break;
     case OHCI_USB_RESUME:
         trace_usb_ohci_resume(ohci->name);
         break;
     case OHCI_USB_RESET:
-        ohci_reset(ohci);
+        ohci_roothub_reset(ohci);
         break;
     }
 }
@@ -1463,7 +1474,7 @@
     if (tks >= usb_frame_time)
         return (ohci->frt << 31);
 
-    tks = muldiv64(1, tks, usb_bit_time);
+    tks = tks / usb_bit_time;
     fr = (uint16_t)(ohci->fi - tks);
 
     return (ohci->frt << 31) | fr;
@@ -1704,7 +1715,7 @@
         ohci->status |= val;
 
         if (ohci->status & OHCI_STATUS_HCR)
-            ohci_reset(ohci);
+            ohci_soft_reset(ohci);
         break;
 
     case 3: /* HcInterruptStatus */
@@ -1783,7 +1794,7 @@
     case 25: /* HcHReset */
         ohci->hreset = val & ~OHCI_HRESET_FSBIR;
         if (val & OHCI_HRESET_FSBIR)
-            ohci_reset(ohci);
+            ohci_hard_reset(ohci);
         break;
 
     case 26: /* HcHInterruptEnable */
@@ -1827,23 +1838,30 @@
 static USBBusOps ohci_bus_ops = {
 };
 
-static int usb_ohci_init(OHCIState *ohci, DeviceState *dev,
-                         int num_ports, dma_addr_t localmem_base,
-                         char *masterbus, uint32_t firstport,
-                         AddressSpace *as)
+static void usb_ohci_init(OHCIState *ohci, DeviceState *dev,
+                          int num_ports, dma_addr_t localmem_base,
+                          char *masterbus, uint32_t firstport,
+                          AddressSpace *as, Error **errp)
 {
+    Error *err = NULL;
     int i;
 
     ohci->as = as;
 
+    if (num_ports > OHCI_MAX_PORTS) {
+        error_setg(errp, "OHCI num-ports=%d is too big (limit is %d ports)",
+                   num_ports, OHCI_MAX_PORTS);
+        return;
+    }
+
     if (usb_frame_time == 0) {
 #ifdef OHCI_TIME_WARP
-        usb_frame_time = get_ticks_per_sec();
-        usb_bit_time = muldiv64(1, get_ticks_per_sec(), USB_HZ/1000);
+        usb_frame_time = NANOSECONDS_PER_SECOND;
+        usb_bit_time = NANOSECONDS_PER_SECOND / (USB_HZ / 1000);
 #else
-        usb_frame_time = muldiv64(1, get_ticks_per_sec(), 1000);
-        if (get_ticks_per_sec() >= USB_HZ) {
-            usb_bit_time = muldiv64(1, get_ticks_per_sec(), USB_HZ);
+        usb_frame_time = NANOSECONDS_PER_SECOND / 1000;
+        if (NANOSECONDS_PER_SECOND >= USB_HZ) {
+            usb_bit_time = NANOSECONDS_PER_SECOND / USB_HZ;
         } else {
             usb_bit_time = 1;
         }
@@ -1857,10 +1875,13 @@
         for(i = 0; i < num_ports; i++) {
             ports[i] = &ohci->rhport[i].port;
         }
-        if (usb_register_companion(masterbus, ports, num_ports,
-                firstport, ohci, &ohci_port_ops,
-                USB_SPEED_MASK_LOW | USB_SPEED_MASK_FULL) != 0) {
-            return -1;
+        usb_register_companion(masterbus, ports, num_ports,
+                               firstport, ohci, &ohci_port_ops,
+                               USB_SPEED_MASK_LOW | USB_SPEED_MASK_FULL,
+                               &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
         }
     } else {
         usb_bus_new(&ohci->bus, sizeof(ohci->bus), &ohci_bus_ops, dev);
@@ -1879,9 +1900,9 @@
     usb_packet_init(&ohci->usb_packet);
 
     ohci->async_td = 0;
-    qemu_register_reset(ohci_reset, ohci);
 
-    return 0;
+    ohci->eof_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
+                                   ohci_frame_boundary, ohci);
 }
 
 #define TYPE_PCI_OHCI "pci-ohci"
@@ -1914,22 +1935,24 @@
                  PCI_STATUS_DETECTED_PARITY);
 }
 
-static int usb_ohci_initfn_pci(PCIDevice *dev)
+static void usb_ohci_realize_pci(PCIDevice *dev, Error **errp)
 {
+    Error *err = NULL;
     OHCIPCIState *ohci = PCI_OHCI(dev);
 
     dev->config[PCI_CLASS_PROG] = 0x10; /* OHCI */
     dev->config[PCI_INTERRUPT_PIN] = 0x01; /* interrupt pin A */
 
-    if (usb_ohci_init(&ohci->state, DEVICE(dev), ohci->num_ports, 0,
-                      ohci->masterbus, ohci->firstport,
-                      pci_get_address_space(dev)) != 0) {
-        return -1;
+    usb_ohci_init(&ohci->state, DEVICE(dev), ohci->num_ports, 0,
+                  ohci->masterbus, ohci->firstport,
+                  pci_get_address_space(dev), &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
-    ohci->state.irq = pci_allocate_irq(dev);
 
+    ohci->state.irq = pci_allocate_irq(dev);
     pci_register_bar(dev, 0, 0, &ohci->state.mem);
-    return 0;
 }
 
 static void usb_ohci_exit(PCIDevice *dev)
@@ -1949,6 +1972,18 @@
     if (!ohci->masterbus) {
         usb_bus_release(&s->bus);
     }
+
+    timer_del(s->eof_timer);
+    timer_free(s->eof_timer);
+}
+
+static void usb_ohci_reset_pci(DeviceState *d)
+{
+    PCIDevice *dev = PCI_DEVICE(d);
+    OHCIPCIState *ohci = PCI_OHCI(dev);
+    OHCIState *s = &ohci->state;
+
+    ohci_hard_reset(s);
 }
 
 #define TYPE_SYSBUS_OHCI "sysbus-ohci"
@@ -1971,11 +2006,19 @@
 
     /* Cannot fail as we pass NULL for masterbus */
     usb_ohci_init(&s->ohci, dev, s->num_ports, s->dma_offset, NULL, 0,
-                  &address_space_memory);
+                  &address_space_memory, &error_abort);
     sysbus_init_irq(sbd, &s->ohci.irq);
     sysbus_init_mmio(sbd, &s->ohci.mem);
 }
 
+static void usb_ohci_reset_sysbus(DeviceState *dev)
+{
+    OHCISysBusState *s = SYSBUS_OHCI(dev);
+    OHCIState *ohci = &s->ohci;
+
+    ohci_hard_reset(ohci);
+}
+
 static Property ohci_pci_properties[] = {
     DEFINE_PROP_STRING("masterbus", OHCIPCIState, masterbus),
     DEFINE_PROP_UINT32("num-ports", OHCIPCIState, num_ports, 3),
@@ -1997,25 +2040,16 @@
 {
     OHCIState *ohci = opaque;
 
-    return ohci->eof_timer != NULL;
-}
-
-static int ohci_eof_timer_pre_load(void *opaque)
-{
-    OHCIState *ohci = opaque;
-
-    ohci_bus_start(ohci);
-
-    return 0;
+    return timer_pending(ohci->eof_timer);
 }
 
 static const VMStateDescription vmstate_ohci_eof_timer = {
     .name = "ohci-core/eof-timer",
     .version_id = 1,
     .minimum_version_id = 1,
-    .pre_load = ohci_eof_timer_pre_load,
+    .needed = ohci_eof_timer_needed,
     .fields = (VMStateField[]) {
-        VMSTATE_TIMER(eof_timer, OHCIState),
+        VMSTATE_TIMER_PTR(eof_timer, OHCIState),
         VMSTATE_END_OF_LIST()
     },
 };
@@ -2061,13 +2095,9 @@
         VMSTATE_BOOL(async_complete, OHCIState),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_ohci_eof_timer,
-            .needed = ohci_eof_timer_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_ohci_eof_timer,
+        NULL
     }
 };
 
@@ -2087,7 +2117,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = usb_ohci_initfn_pci;
+    k->realize = usb_ohci_realize_pci;
     k->exit = usb_ohci_exit;
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_IPID_USB;
@@ -2097,6 +2127,7 @@
     dc->props = ohci_pci_properties;
     dc->hotpluggable = false;
     dc->vmsd = &vmstate_ohci;
+    dc->reset = usb_ohci_reset_pci;
 }
 
 static const TypeInfo ohci_pci_info = {
@@ -2120,6 +2151,7 @@
     set_bit(DEVICE_CATEGORY_USB, dc->categories);
     dc->desc = "OHCI USB Controller";
     dc->props = ohci_sysbus_properties;
+    dc->reset = usb_ohci_reset_sysbus;
 }
 
 static const TypeInfo ohci_sysbus_info = {
diff --git a/hw/usb/hcd-uhci.c b/hw/usb/hcd-uhci.c
index 4a4215d..ca72a80 100644
--- a/hw/usb/hcd-uhci.c
+++ b/hw/usb/hcd-uhci.c
@@ -25,10 +25,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/usb.h"
 #include "hw/usb/uhci-regs.h"
 #include "hw/pci/pci.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/iov.h"
 #include "sysemu/dma.h"
@@ -66,7 +68,7 @@
     uint16_t   device_id;
     uint8_t    revision;
     uint8_t    irq_pin;
-    int        (*initfn)(PCIDevice *dev);
+    void       (*realize)(PCIDevice *dev, Error **errp);
     bool       unplug;
 };
 
@@ -154,6 +156,9 @@
 static void uhci_queue_fill(UHCIQueue *q, UHCI_TD *td);
 static void uhci_resume(void *opaque);
 
+#define TYPE_UHCI "pci-uhci-usb"
+#define UHCI(obj) OBJECT_CHECK(UHCIState, (obj), TYPE_UHCI)
+
 static inline int32_t uhci_queue_token(UHCI_TD *td)
 {
     if ((td->token & (0xf << 15)) == 0) {
@@ -348,9 +353,10 @@
     pci_set_irq(&s->dev, level);
 }
 
-static void uhci_reset(void *opaque)
+static void uhci_reset(DeviceState *dev)
 {
-    UHCIState *s = opaque;
+    PCIDevice *d = PCI_DEVICE(dev);
+    UHCIState *s = UHCI(d);
     uint8_t *pci_conf;
     int i;
     UHCIPort *port;
@@ -362,7 +368,7 @@
     pci_conf[0x6a] = 0x01; /* usb clock */
     pci_conf[0x6b] = 0x00;
     s->cmd = 0;
-    s->status = 0;
+    s->status = UHCI_STS_HCHALTED;
     s->status2 = 0;
     s->intr = 0;
     s->fl_base_addr = 0;
@@ -397,7 +403,7 @@
 
     if (version_id < 2) {
         s->expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-            (get_ticks_per_sec() / FRAME_TIMER_FREQ);
+            (NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ);
     }
     return 0;
 }
@@ -419,7 +425,7 @@
         VMSTATE_UINT32(fl_base_addr, UHCIState),
         VMSTATE_UINT8(sof_timing, UHCIState),
         VMSTATE_UINT8(status2, UHCIState),
-        VMSTATE_TIMER(frame_timer, UHCIState),
+        VMSTATE_TIMER_PTR(frame_timer, UHCIState),
         VMSTATE_INT64_V(expire_time, UHCIState, 2),
         VMSTATE_UINT32_V(pending_int_mask, UHCIState, 3),
         VMSTATE_END_OF_LIST()
@@ -439,7 +445,7 @@
             /* start frame processing */
             trace_usb_uhci_schedule_start();
             s->expire_time = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                (get_ticks_per_sec() / FRAME_TIMER_FREQ);
+                (NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ);
             timer_mod(s->frame_timer, s->expire_time);
             s->status &= ~UHCI_STS_HCHALTED;
         } else if (!(val & UHCI_CMD_RS)) {
@@ -454,11 +460,11 @@
                 port = &s->ports[i];
                 usb_device_reset(port->port.dev);
             }
-            uhci_reset(s);
+            uhci_reset(DEVICE(s));
             return;
         }
         if (val & UHCI_CMD_HCRESET) {
-            uhci_reset(s);
+            uhci_reset(DEVICE(s));
             return;
         }
         s->cmd = val;
@@ -768,8 +774,9 @@
     bool spd;
     bool queuing = (q != NULL);
     uint8_t pid = td->token & 0xff;
-    UHCIAsync *async = uhci_async_find_td(s, td_addr);
+    UHCIAsync *async;
 
+    async = uhci_async_find_td(s, td_addr);
     if (async) {
         if (uhci_queue_verify(async->queue, qh_addr, td, td_addr, queuing)) {
             assert(q == NULL || q == async->queue);
@@ -808,6 +815,19 @@
         return TD_RESULT_NEXT_QH;
     }
 
+    switch (pid) {
+    case USB_TOKEN_OUT:
+    case USB_TOKEN_SETUP:
+    case USB_TOKEN_IN:
+        break;
+    default:
+        /* invalid pid : frame interrupted */
+        s->status |= UHCI_STS_HCPERR;
+        s->cmd &= ~UHCI_CMD_RS;
+        uhci_update_irq(s);
+        return TD_RESULT_STOP_FRAME;
+    }
+
     if (async) {
         if (queuing) {
             /* we are busy filling the queue, we are not prepared
@@ -875,11 +895,7 @@
         break;
 
     default:
-        /* invalid pid : frame interrupted */
-        uhci_async_free(async);
-        s->status |= UHCI_STS_HCPERR;
-        uhci_update_irq(s);
-        return TD_RESULT_STOP_FRAME;
+        abort(); /* Never to execute */
     }
 
     if (async->packet.status == USB_RET_ASYNC) {
@@ -1115,7 +1131,7 @@
     UHCIState *s = opaque;
     uint64_t t_now, t_last_run;
     int i, frames;
-    const uint64_t frame_t = get_ticks_per_sec() / FRAME_TIMER_FREQ;
+    const uint64_t frame_t = NANOSECONDS_PER_SECOND / FRAME_TIMER_FREQ;
 
     s->completions_only = false;
     qemu_bh_cancel(s->bh);
@@ -1190,11 +1206,12 @@
 static USBBusOps uhci_bus_ops = {
 };
 
-static int usb_uhci_common_initfn(PCIDevice *dev)
+static void usb_uhci_common_realize(PCIDevice *dev, Error **errp)
 {
+    Error *err = NULL;
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
     UHCIPCIDeviceClass *u = container_of(pc, UHCIPCIDeviceClass, parent_class);
-    UHCIState *s = DO_UPCAST(UHCIState, dev, dev);
+    UHCIState *s = UHCI(dev);
     uint8_t *pci_conf = s->dev.config;
     int i;
 
@@ -1209,10 +1226,13 @@
         for(i = 0; i < NB_PORTS; i++) {
             ports[i] = &s->ports[i].port;
         }
-        if (usb_register_companion(s->masterbus, ports, NB_PORTS,
-                s->firstport, s, &uhci_port_ops,
-                USB_SPEED_MASK_LOW | USB_SPEED_MASK_FULL) != 0) {
-            return -1;
+        usb_register_companion(s->masterbus, ports, NB_PORTS,
+                               s->firstport, s, &uhci_port_ops,
+                               USB_SPEED_MASK_LOW | USB_SPEED_MASK_FULL,
+                               &err);
+        if (err) {
+            error_propagate(errp, err);
+            return;
         }
     } else {
         usb_bus_new(&s->bus, sizeof(s->bus), &uhci_bus_ops, DEVICE(dev));
@@ -1226,21 +1246,17 @@
     s->num_ports_vmstate = NB_PORTS;
     QTAILQ_INIT(&s->queues);
 
-    qemu_register_reset(uhci_reset, s);
-
     memory_region_init_io(&s->io_bar, OBJECT(s), &uhci_ioport_ops, s,
                           "uhci", 0x20);
 
     /* Use region 4 for consistency with real hardware.  BSD guests seem
        to rely on this.  */
     pci_register_bar(&s->dev, 4, PCI_BASE_ADDRESS_SPACE_IO, &s->io_bar);
-
-    return 0;
 }
 
-static int usb_uhci_vt82c686b_initfn(PCIDevice *dev)
+static void usb_uhci_vt82c686b_realize(PCIDevice *dev, Error **errp)
 {
-    UHCIState *s = DO_UPCAST(UHCIState, dev, dev);
+    UHCIState *s = UHCI(dev);
     uint8_t *pci_conf = s->dev.config;
 
     /* USB misc control 1/2 */
@@ -1250,12 +1266,12 @@
     /* USB legacy support  */
     pci_set_long(pci_conf + 0xc0,0x00002000);
 
-    return usb_uhci_common_initfn(dev);
+    usb_uhci_common_realize(dev, errp);
 }
 
 static void usb_uhci_exit(PCIDevice *dev)
 {
-    UHCIState *s = DO_UPCAST(UHCIState, dev, dev);
+    UHCIState *s = UHCI(dev);
 
     trace_usb_uhci_exit();
 
@@ -1293,16 +1309,34 @@
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->class_id  = PCI_CLASS_SERIAL_USB;
+    dc->vmsd = &vmstate_uhci;
+    dc->reset = uhci_reset;
+    set_bit(DEVICE_CATEGORY_USB, dc->categories);
+}
+
+static const TypeInfo uhci_pci_type_info = {
+    .name = TYPE_UHCI,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(UHCIState),
+    .class_size    = sizeof(UHCIPCIDeviceClass),
+    .abstract = true,
+    .class_init = uhci_class_init,
+};
+
+static void uhci_data_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
     UHCIPCIDeviceClass *u = container_of(k, UHCIPCIDeviceClass, parent_class);
     UHCIInfo *info = data;
 
-    k->init = info->initfn ? info->initfn : usb_uhci_common_initfn;
+    k->realize = info->realize ? info->realize : usb_uhci_common_realize;
     k->exit = info->unplug ? usb_uhci_exit : NULL;
     k->vendor_id = info->vendor_id;
     k->device_id = info->device_id;
     k->revision  = info->revision;
-    k->class_id  = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
     if (!info->unplug) {
         /* uhci controllers in companion setups can't be hotplugged */
         dc->hotpluggable = false;
@@ -1310,7 +1344,6 @@
     } else {
         dc->props = uhci_properties_standalone;
     }
-    set_bit(DEVICE_CATEGORY_USB, dc->categories);
     u->info = *info;
 }
 
@@ -1335,7 +1368,7 @@
         .device_id = PCI_DEVICE_ID_VIA_UHCI,
         .revision  = 0x01,
         .irq_pin   = 3,
-        .initfn    = usb_uhci_vt82c686b_initfn,
+        .realize   = usb_uhci_vt82c686b_realize,
         .unplug    = true,
     },{
         .name      = "ich9-usb-uhci1", /* 00:1d.0 */
@@ -1385,13 +1418,13 @@
 static void uhci_register_types(void)
 {
     TypeInfo uhci_type_info = {
-        .parent        = TYPE_PCI_DEVICE,
-        .instance_size = sizeof(UHCIState),
-        .class_size    = sizeof(UHCIPCIDeviceClass),
-        .class_init    = uhci_class_init,
+        .parent        = TYPE_UHCI,
+        .class_init    = uhci_data_class_init,
     };
     int i;
 
+    type_register_static(&uhci_pci_type_info);
+
     for (i = 0; i < ARRAY_SIZE(uhci_info); i++) {
         uhci_type_info.name = uhci_info[i].name;
         uhci_type_info.class_data = uhci_info + i;
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index df45f10..a0b9c37 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -18,6 +18,7 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/timer.h"
 #include "hw/usb.h"
@@ -25,6 +26,7 @@
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
 #include "trace.h"
+#include "qapi/error.h"
 
 //#define DEBUG_XHCI
 //#define DEBUG_DATA
@@ -460,6 +462,8 @@
     uint32_t numslots;
     uint32_t flags;
     uint32_t max_pstreams_mask;
+    OnOffAuto msi;
+    OnOffAuto msix;
 
     /* Operational Registers */
     uint32_t usbcmd;
@@ -497,9 +501,7 @@
 } XHCIEvRingSeg;
 
 enum xhci_flags {
-    XHCI_FLAG_USE_MSI = 1,
-    XHCI_FLAG_USE_MSI_X,
-    XHCI_FLAG_SS_FIRST,
+    XHCI_FLAG_SS_FIRST = 1,
     XHCI_FLAG_FORCE_PCIE_ENDCAP,
     XHCI_FLAG_ENABLE_STREAMS,
 };
@@ -697,11 +699,13 @@
                                        uint32_t *buf, size_t len)
 {
     int i;
-    uint32_t tmp[len / sizeof(uint32_t)];
+    uint32_t tmp[5];
+    uint32_t n = len / sizeof(uint32_t);
 
     assert((len % sizeof(uint32_t)) == 0);
+    assert(n <= ARRAY_SIZE(tmp));
 
-    for (i = 0; i < (len / sizeof(uint32_t)); i++) {
+    for (i = 0; i < n; i++) {
         tmp[i] = cpu_to_le32(buf[i]);
     }
     pci_dma_write(PCI_DEVICE(xhci), addr, tmp, len);
@@ -1453,9 +1457,7 @@
         t->running_retry = 0;
         killed = 1;
     }
-    if (t->trbs) {
-        g_free(t->trbs);
-    }
+    g_free(t->trbs);
 
     t->trbs = NULL;
     t->trb_count = t->trb_alloced = 0;
@@ -1530,7 +1532,10 @@
         usb_packet_cleanup(&epctx->transfers[i].packet);
     }
 
-    xhci_set_ep_state(xhci, epctx, NULL, EP_DISABLED);
+    /* only touch guest RAM if we're not resetting the HC */
+    if (xhci->dcbaap_low || xhci->dcbaap_high) {
+        xhci_set_ep_state(xhci, epctx, NULL, EP_DISABLED);
+    }
 
     timer_free(epctx->kick_timer);
     g_free(epctx);
@@ -1793,6 +1798,14 @@
                 return;
             }
         }
+
+        switch (TRB_TYPE(*trb)) {
+        case TR_SETUP:
+            reported = 0;
+            shortpkt = 0;
+            break;
+        }
+
     }
 }
 
@@ -2182,20 +2195,21 @@
             xfer->trbs = NULL;
         }
         if (!xfer->trbs) {
-            xfer->trbs = g_malloc(sizeof(XHCITRB) * length);
+            xfer->trbs = g_new(XHCITRB, length);
             xfer->trb_alloced = length;
         }
         xfer->trb_count = length;
 
         for (i = 0; i < length; i++) {
-            assert(xhci_ring_fetch(xhci, ring, &xfer->trbs[i], NULL));
+            TRBType type;
+            type = xhci_ring_fetch(xhci, ring, &xfer->trbs[i], NULL);
+            assert(type);
         }
         xfer->streamid = streamid;
 
         if (epid == 1) {
             if (xhci_fire_ctl_transfer(xhci, xfer) >= 0) {
                 epctx->next_xfer = (epctx->next_xfer + 1) % TD_QUEUE;
-                ep = xfer->packet.ep;
             } else {
                 DPRINTF("xhci: error firing CTL transfer\n");
             }
@@ -2352,6 +2366,8 @@
     slot->uport = uport;
     slot->ctx = octx;
 
+    /* Make sure device is in USB_STATE_DEFAULT state */
+    usb_device_reset(dev);
     if (bsr) {
         slot_ctx[3] = SLOT_DEFAULT << SLOT_STATE_SHIFT;
     } else {
@@ -2359,7 +2375,6 @@
         uint8_t buf[1];
 
         slot_ctx[3] = (SLOT_ADDRESSED << SLOT_STATE_SHIFT) | slotid;
-        usb_device_reset(dev);
         memset(&p, 0, sizeof(p));
         usb_packet_addbuf(&p, buf, sizeof(buf));
         usb_packet_setup(&p, USB_TOKEN_OUT,
@@ -3567,9 +3582,10 @@
     }
 }
 
-static int usb_xhci_initfn(struct PCIDevice *dev)
+static void usb_xhci_realize(struct PCIDevice *dev, Error **errp)
 {
     int i, ret;
+    Error *err = NULL;
 
     XHCIState *xhci = XHCI(dev);
 
@@ -3580,6 +3596,23 @@
 
     usb_xhci_init(xhci);
 
+    if (xhci->msi != ON_OFF_AUTO_OFF) {
+        ret = msi_init(dev, 0x70, xhci->numintrs, true, false, &err);
+        /* Any error other than -ENOTSUP(board's MSI support is broken)
+         * is a programming error */
+        assert(!ret || ret == -ENOTSUP);
+        if (ret && xhci->msi == ON_OFF_AUTO_ON) {
+            /* Can't satisfy user's explicit msi=on request, fail */
+            error_append_hint(&err, "You have to use msi=auto (default) or "
+                    "msi=off with this machine type.\n");
+            error_propagate(errp, err);
+            return;
+        }
+        assert(!err || xhci->msi == ON_OFF_AUTO_AUTO);
+        /* With msi=auto, we fall back to MSI off silently */
+        error_free(err);
+    }
+
     if (xhci->numintrs > MAXINTRS) {
         xhci->numintrs = MAXINTRS;
     }
@@ -3638,17 +3671,13 @@
         (void)ret;
     }
 
-    if (xhci_get_flag(xhci, XHCI_FLAG_USE_MSI)) {
-        msi_init(dev, 0x70, xhci->numintrs, true, false);
-    }
-    if (xhci_get_flag(xhci, XHCI_FLAG_USE_MSI_X)) {
+    if (xhci->msix != ON_OFF_AUTO_OFF) {
+        /* TODO check for errors */
         msix_init(dev, xhci->numintrs,
                   &xhci->mem, 0, OFF_MSIX_TABLE,
                   &xhci->mem, 0, OFF_MSIX_PBA,
                   0x90);
     }
-
-    return 0;
 }
 
 static void usb_xhci_exit(PCIDevice *dev)
@@ -3856,7 +3885,7 @@
 
         /* Runtime Registers & state */
         VMSTATE_INT64(mfindex_start,  XHCIState),
-        VMSTATE_TIMER(mfwrap_timer,   XHCIState),
+        VMSTATE_TIMER_PTR(mfwrap_timer,   XHCIState),
         VMSTATE_STRUCT(cmd_ring, XHCIState, 1, vmstate_xhci_ring, XHCIRing),
 
         VMSTATE_END_OF_LIST()
@@ -3864,8 +3893,8 @@
 };
 
 static Property xhci_properties[] = {
-    DEFINE_PROP_BIT("msi",      XHCIState, flags, XHCI_FLAG_USE_MSI, true),
-    DEFINE_PROP_BIT("msix",     XHCIState, flags, XHCI_FLAG_USE_MSI_X, true),
+    DEFINE_PROP_ON_OFF_AUTO("msi", XHCIState, msi, ON_OFF_AUTO_AUTO),
+    DEFINE_PROP_ON_OFF_AUTO("msix", XHCIState, msix, ON_OFF_AUTO_AUTO),
     DEFINE_PROP_BIT("superspeed-ports-first",
                     XHCIState, flags, XHCI_FLAG_SS_FIRST, true),
     DEFINE_PROP_BIT("force-pcie-endcap", XHCIState, flags,
@@ -3888,7 +3917,7 @@
     dc->props   = xhci_properties;
     dc->reset   = xhci_reset;
     set_bit(DEVICE_CATEGORY_USB, dc->categories);
-    k->init         = usb_xhci_initfn;
+    k->realize      = usb_xhci_realize;
     k->exit         = usb_xhci_exit;
     k->vendor_id    = PCI_VENDOR_ID_NEC;
     k->device_id    = PCI_DEVICE_ID_NEC_UPD720200;
diff --git a/hw/usb/host-legacy.c b/hw/usb/host-legacy.c
index 3cc9c42..3b57e21 100644
--- a/hw/usb/host-legacy.c
+++ b/hw/usb/host-legacy.c
@@ -30,6 +30,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/usb.h"
 #include "hw/usb/host.h"
@@ -128,7 +129,6 @@
     qdev_prop_set_uint32(&dev->qdev, "hostaddr",  filter.addr);
     qdev_prop_set_uint32(&dev->qdev, "vendorid",  filter.vendor_id);
     qdev_prop_set_uint32(&dev->qdev, "productid", filter.product_id);
-    qdev_init_nofail(&dev->qdev);
     return dev;
 
 fail:
diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index a5f9dab..e94672c 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -33,11 +33,16 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#ifndef CONFIG_WIN32
 #include <poll.h>
+#endif
 #include <libusb.h>
 
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "trace.h"
 
@@ -76,6 +81,7 @@
     uint32_t                         iso_urb_frames;
     uint32_t                         options;
     uint32_t                         loglevel;
+    bool                             needs_autoscan;
 
     /* state */
     QTAILQ_ENTRY(USBHostDevice)      next;
@@ -201,6 +207,8 @@
 static libusb_context *ctx;
 static uint32_t loglevel;
 
+#ifndef CONFIG_WIN32
+
 static void usb_host_handle_fd(void *opaque)
 {
     struct timeval tv = { 0, 0 };
@@ -220,10 +228,14 @@
     qemu_set_fd_handler(fd, NULL, NULL, NULL);
 }
 
+#endif /* !CONFIG_WIN32 */
+
 static int usb_host_init(void)
 {
+#ifndef CONFIG_WIN32
     const struct libusb_pollfd **poll;
-    int i, rc;
+#endif
+    int rc;
 
     if (ctx) {
         return 0;
@@ -233,17 +245,21 @@
         return -1;
     }
     libusb_set_debug(ctx, loglevel);
-
+#ifdef CONFIG_WIN32
+    /* FIXME: add support for Windows. */
+#else
     libusb_set_pollfd_notifiers(ctx, usb_host_add_fd,
                                 usb_host_del_fd,
                                 ctx);
     poll = libusb_get_pollfds(ctx);
     if (poll) {
+        int i;
         for (i = 0; poll[i] != NULL; i++) {
             usb_host_add_fd(poll[i]->fd, poll[i]->events, ctx);
         }
     }
     free(poll);
+#endif
     return 0;
 }
 
@@ -343,7 +359,7 @@
     return NULL;
 }
 
-static void usb_host_req_complete_ctrl(struct libusb_transfer *xfer)
+static void LIBUSB_CALL usb_host_req_complete_ctrl(struct libusb_transfer *xfer)
 {
     USBHostRequest *r = xfer->user_data;
     USBHostDevice  *s = r->host;
@@ -376,7 +392,7 @@
     }
 }
 
-static void usb_host_req_complete_data(struct libusb_transfer *xfer)
+static void LIBUSB_CALL usb_host_req_complete_data(struct libusb_transfer *xfer)
 {
     USBHostRequest *r = xfer->user_data;
     USBHostDevice  *s = r->host;
@@ -432,7 +448,8 @@
 
 /* ------------------------------------------------------------------------ */
 
-static void usb_host_req_complete_iso(struct libusb_transfer *transfer)
+static void LIBUSB_CALL
+usb_host_req_complete_iso(struct libusb_transfer *transfer)
 {
     USBHostIsoXfer *xfer = transfer->user_data;
 
@@ -450,6 +467,7 @@
     }
     if (xfer->ring->ep->pid == USB_TOKEN_IN) {
         QTAILQ_INSERT_TAIL(&xfer->ring->copy, xfer, next);
+        usb_wakeup(xfer->ring->ep, 0);
     } else {
         QTAILQ_INSERT_TAIL(&xfer->ring->unused, xfer, next);
     }
@@ -878,8 +896,7 @@
 
     usb_device_attach(udev, &local_err);
     if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         goto fail;
     }
 
@@ -889,6 +906,9 @@
 fail:
     trace_usb_host_open_failure(bus_num, addr);
     if (s->dh != NULL) {
+        usb_host_release_interfaces(s);
+        libusb_reset_device(s->dh);
+        usb_host_attach_kernel(s);
         libusb_close(s->dh);
         s->dh = NULL;
         s->dev = NULL;
@@ -957,9 +977,32 @@
     }
 }
 
+static libusb_device *usb_host_find_ref(int bus, int addr)
+{
+    libusb_device **devs = NULL;
+    libusb_device *ret = NULL;
+    int i, n;
+
+    if (usb_host_init() != 0) {
+        return NULL;
+    }
+    n = libusb_get_device_list(ctx, &devs);
+    for (i = 0; i < n; i++) {
+        if (libusb_get_bus_number(devs[i]) == bus &&
+            libusb_get_device_address(devs[i]) == addr) {
+            ret = libusb_ref_device(devs[i]);
+            break;
+        }
+    }
+    libusb_free_device_list(devs, 1);
+    return ret;
+}
+
 static void usb_host_realize(USBDevice *udev, Error **errp)
 {
     USBHostDevice *s = USB_HOST_DEVICE(udev);
+    libusb_device *ldev;
+    int rc;
 
     if (s->match.vendor_id > 0xffff) {
         error_setg(errp, "vendorid out of range");
@@ -980,11 +1023,33 @@
     QTAILQ_INIT(&s->requests);
     QTAILQ_INIT(&s->isorings);
 
+    if (s->match.addr && s->match.bus_num &&
+        !s->match.vendor_id &&
+        !s->match.product_id &&
+        !s->match.port) {
+        s->needs_autoscan = false;
+        ldev = usb_host_find_ref(s->match.bus_num,
+                                 s->match.addr);
+        if (!ldev) {
+            error_setg(errp, "failed to find host usb device %d:%d",
+                       s->match.bus_num, s->match.addr);
+            return;
+        }
+        rc = usb_host_open(s, ldev);
+        libusb_unref_device(ldev);
+        if (rc < 0) {
+            error_setg(errp, "failed to open host usb device %d:%d",
+                       s->match.bus_num, s->match.addr);
+            return;
+        }
+    } else {
+        s->needs_autoscan = true;
+        QTAILQ_INSERT_TAIL(&hostdevs, s, next);
+        usb_host_auto_check(NULL);
+    }
+
     s->exit.notify = usb_host_exit_notifier;
     qemu_add_exit_notifier(&s->exit);
-
-    QTAILQ_INSERT_TAIL(&hostdevs, s, next);
-    usb_host_auto_check(NULL);
 }
 
 static void usb_host_instance_init(Object *obj)
@@ -1002,7 +1067,9 @@
     USBHostDevice *s = USB_HOST_DEVICE(udev);
 
     qemu_remove_exit_notifier(&s->exit);
-    QTAILQ_REMOVE(&hostdevs, s, next);
+    if (s->needs_autoscan) {
+        QTAILQ_REMOVE(&hostdevs, s, next);
+    }
     usb_host_close(s);
 }
 
@@ -1236,8 +1303,8 @@
 
     /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
      * to work redirected to a not superspeed capable hcd */
-    if (udev->speed == USB_SPEED_SUPER &&
-        !((udev->port->speedmask & USB_SPEED_MASK_SUPER)) &&
+    if ((udev->speedmask & USB_SPEED_MASK_SUPER) &&
+        !(udev->port->speedmask & USB_SPEED_MASK_SUPER) &&
         request == 0x8006 && value == 0x100 && index == 0) {
         r->usb3ep0quirk = true;
     }
@@ -1426,7 +1493,7 @@
  * still present in the first place.  Attemping to contine where we
  * left off is impossible.
  *
- * What we are going to to to here is emulate a surprise removal of
+ * What we are going to do here is emulate a surprise removal of
  * the usb device passed through, then kick host scan so the device
  * will get re-attached (and re-initialized by the guest) in case it
  * is still present.
@@ -1637,7 +1704,7 @@
     timer_mod(usb_auto_timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + 2000);
 }
 
-void usb_host_info(Monitor *mon, const QDict *qdict)
+void hmp_info_usbhost(Monitor *mon, const QDict *qdict)
 {
     libusb_device **devs = NULL;
     struct libusb_device_descriptor ddesc;
diff --git a/hw/usb/host-stub.c b/hw/usb/host-stub.c
index 28d8032..6ba65a1 100644
--- a/hw/usb/host-stub.c
+++ b/hw/usb/host-stub.c
@@ -30,12 +30,13 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "hw/usb.h"
 #include "monitor/monitor.h"
 
-void usb_host_info(Monitor *mon, const QDict *qdict)
+void hmp_info_usbhost(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "USB host devices not supported\n");
 }
diff --git a/hw/usb/libhw.c b/hw/usb/libhw.c
index 8df11c4..73cdf0c 100644
--- a/hw/usb/libhw.c
+++ b/hw/usb/libhw.c
@@ -19,6 +19,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "hw/usb.h"
diff --git a/hw/usb/quirks.c b/hw/usb/quirks.c
index a761a96..38a9c56 100644
--- a/hw/usb/quirks.c
+++ b/hw/usb/quirks.c
@@ -12,6 +12,7 @@
  * (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "quirks.h"
 #include "hw/usb.h"
 
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 9fbd59e..444672a 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -25,21 +25,24 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
-#include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "qemu/iov.h"
 #include "sysemu/char.h"
 
-#include <dirent.h>
-#include <sys/ioctl.h>
-#include <signal.h>
 #include <usbredirparser.h>
 #include <usbredirfilter.h>
 
 #include "hw/usb.h"
 
+/* ERROR is defined below. Remove any previous definition. */
+#undef ERROR
+
 #define MAX_ENDPOINTS 32
 #define NO_INTERFACE_INFO 255 /* Valid interface_count always <= 32 */
 #define EP2I(ep_address) (((ep_address & 0x80) >> 3) | (ep_address & 0x0f))
@@ -106,6 +109,7 @@
     uint8_t debug;
     char *filter_str;
     int32_t bootindex;
+    bool enable_streams;
     /* Data passed from chardev the fd_read cb to the usbredirparser read cb */
     const uint8_t *read_buf;
     int read_buf_size;
@@ -130,6 +134,9 @@
     int compatible_speedmask;
 };
 
+#define TYPE_USB_REDIR "usb-redir"
+#define USB_REDIRECT(obj) OBJECT_CHECK(USBRedirDevice, (obj), TYPE_USB_REDIR)
+
 static void usbredir_hello(void *priv, struct usb_redir_hello_header *h);
 static void usbredir_device_connect(void *priv,
     struct usb_redir_device_connect_header *device_connect);
@@ -320,7 +327,7 @@
 
     DPRINTF("adding packet id %"PRIu64" to %s queue\n", id, q->name);
 
-    e = g_malloc0(sizeof(struct PacketIdQueueEntry));
+    e = g_new0(struct PacketIdQueueEntry, 1);
     e->id = id;
     QTAILQ_INSERT_TAIL(&q->head, e, next);
     q->size++;
@@ -360,7 +367,7 @@
 
 static void usbredir_cancel_packet(USBDevice *udev, USBPacket *p)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
     int i = USBEP2I(p->ep);
 
     if (p->combined) {
@@ -444,7 +451,7 @@
     return p;
 }
 
-static void bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
+static int bufp_alloc(USBRedirDevice *dev, uint8_t *data, uint16_t len,
     uint8_t status, uint8_t ep, void *free_on_destroy)
 {
     struct buf_packet *bufp;
@@ -461,12 +468,12 @@
         if (dev->endpoint[EP2I(ep)].bufpq_size >
                 dev->endpoint[EP2I(ep)].bufpq_target_size) {
             free(data);
-            return;
+            return -1;
         }
         dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
     }
 
-    bufp = g_malloc(sizeof(struct buf_packet));
+    bufp = g_new(struct buf_packet, 1);
     bufp->data   = data;
     bufp->len    = len;
     bufp->offset = 0;
@@ -474,6 +481,7 @@
     bufp->free_on_destroy = free_on_destroy;
     QTAILQ_INSERT_TAIL(&dev->endpoint[EP2I(ep)].bufpq, bufp, next);
     dev->endpoint[EP2I(ep)].bufpq_size++;
+    return 0;
 }
 
 static void bufp_free(USBRedirDevice *dev, struct buf_packet *bufp,
@@ -500,7 +508,7 @@
 
 static void usbredir_handle_reset(USBDevice *udev)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
 
     DPRINTF("reset device\n");
     usbredirparser_send_reset(dev->parser);
@@ -535,9 +543,9 @@
             start_iso.pkts_per_urb = 32;
         }
 
-        start_iso.no_urbs = (dev->endpoint[EP2I(ep)].bufpq_target_size +
-                             start_iso.pkts_per_urb - 1) /
-                            start_iso.pkts_per_urb;
+        start_iso.no_urbs = DIV_ROUND_UP(
+                                     dev->endpoint[EP2I(ep)].bufpq_target_size,
+                                     start_iso.pkts_per_urb);
         /* Output endpoints pre-fill only 1/2 of the packets, keeping the rest
            as overflow buffer. Also see the usbredir protocol documentation */
         if (!(ep & USB_DIR_IN)) {
@@ -907,7 +915,7 @@
 
 static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
     uint8_t ep;
 
     ep = p->ep->nr;
@@ -976,7 +984,7 @@
 
 static void usbredir_ep_stopped(USBDevice *udev, USBEndpoint *uep)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
 
     usbredir_stop_ep(dev, USBEP2I(uep));
     usbredirparser_do_write(dev->parser);
@@ -1046,7 +1054,7 @@
 static void usbredir_handle_control(USBDevice *udev, USBPacket *p,
         int request, int value, int index, int length, uint8_t *data)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
     struct usb_redir_control_packet_header control_packet;
 
     if (usbredir_already_in_flight(dev, p->id)) {
@@ -1101,7 +1109,7 @@
 static int usbredir_alloc_streams(USBDevice *udev, USBEndpoint **eps,
                                   int nr_eps, int streams)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
 #if USBREDIR_VERSION >= 0x000700
     struct usb_redir_alloc_bulk_streams_header alloc_streams;
     int i;
@@ -1140,7 +1148,7 @@
                                   int nr_eps)
 {
 #if USBREDIR_VERSION >= 0x000700
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
     struct usb_redir_free_bulk_streams_header free_streams;
     int i;
 
@@ -1222,7 +1230,9 @@
     usbredirparser_caps_set_cap(caps, usb_redir_cap_32bits_bulk_length);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_receiving);
 #if USBREDIR_VERSION >= 0x000700
-    usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_streams);
+    if (dev->enable_streams) {
+        usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_streams);
+    }
 #endif
 
     if (runstate_check(RUN_STATE_INMIGRATE)) {
@@ -1273,8 +1283,7 @@
 
     usb_device_attach(&dev->dev, &local_err);
     if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         WARNING("rejecting device due to speed mismatch\n");
         usbredir_reject_device(dev);
     }
@@ -1363,11 +1372,11 @@
 
 static void usbredir_realize(USBDevice *udev, Error **errp)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
     int i;
 
     if (dev->cs == NULL) {
-        error_set(errp, QERR_MISSING_PARAMETER, "chardev");
+        error_setg(errp, QERR_MISSING_PARAMETER, "chardev");
         return;
     }
 
@@ -1376,8 +1385,8 @@
                                            &dev->filter_rules,
                                            &dev->filter_rules_count);
         if (i) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, "filter",
-                      "a usb device filter string");
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "filter",
+                       "a usb device filter string");
             return;
         }
     }
@@ -1416,7 +1425,7 @@
 
 static void usbredir_handle_destroy(USBDevice *udev)
 {
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
 
     qemu_chr_delete(dev->cs);
     dev->cs = NULL;
@@ -2080,13 +2089,17 @@
     status = usb_redir_success;
     free_on_destroy = NULL;
     for (i = 0; i < data_len; i += len) {
+        int r;
         if (len >= (data_len - i)) {
             len = data_len - i;
             status = buffered_bulk_packet->status;
             free_on_destroy = data;
         }
         /* bufp_alloc also adds the packet to the ep queue */
-        bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+        r = bufp_alloc(dev, data + i, len, status, ep, free_on_destroy);
+        if (r) {
+            break;
+        }
     }
 
     if (dev->endpoint[EP2I(ep)].pending_async_packet) {
@@ -2233,7 +2246,7 @@
 
     endp->bufpq_size = qemu_get_be32(f);
     for (i = 0; i < endp->bufpq_size; i++) {
-        bufp = g_malloc(sizeof(struct buf_packet));
+        bufp = g_new(struct buf_packet, 1);
         bufp->len = qemu_get_be32(f);
         bufp->status = qemu_get_be32(f);
         bufp->offset = 0;
@@ -2255,16 +2268,6 @@
 
 
 /* For endp_data migration */
-static const VMStateDescription usbredir_bulk_receiving_vmstate = {
-    .name = "usb-redir-ep/bulk-receiving",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT8(bulk_receiving_started, struct endp_data),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
 static bool usbredir_bulk_receiving_needed(void *priv)
 {
     struct endp_data *endp = priv;
@@ -2272,12 +2275,13 @@
     return endp->bulk_receiving_started;
 }
 
-static const VMStateDescription usbredir_stream_vmstate = {
-    .name = "usb-redir-ep/stream-state",
+static const VMStateDescription usbredir_bulk_receiving_vmstate = {
+    .name = "usb-redir-ep/bulk-receiving",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = usbredir_bulk_receiving_needed,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT32(max_streams, struct endp_data),
+        VMSTATE_UINT8(bulk_receiving_started, struct endp_data),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -2289,6 +2293,17 @@
     return endp->max_streams;
 }
 
+static const VMStateDescription usbredir_stream_vmstate = {
+    .name = "usb-redir-ep/stream-state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = usbredir_stream_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(max_streams, struct endp_data),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription usbredir_ep_vmstate = {
     .name = "usb-redir-ep",
     .version_id = 1,
@@ -2316,16 +2331,10 @@
         VMSTATE_INT32(bufpq_target_size, struct endp_data),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &usbredir_bulk_receiving_vmstate,
-            .needed = usbredir_bulk_receiving_needed,
-        }, {
-            .vmsd = &usbredir_stream_vmstate,
-            .needed = usbredir_stream_needed,
-        }, {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &usbredir_bulk_receiving_vmstate,
+        &usbredir_stream_vmstate,
+        NULL
     }
 };
 
@@ -2438,7 +2447,7 @@
     .post_load = usbredir_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_USB_DEVICE(dev, USBRedirDevice),
-        VMSTATE_TIMER(attach_timer, USBRedirDevice),
+        VMSTATE_TIMER_PTR(attach_timer, USBRedirDevice),
         {
             .name         = "parser",
             .version_id   = 0,
@@ -2470,6 +2479,7 @@
     DEFINE_PROP_CHR("chardev", USBRedirDevice, cs),
     DEFINE_PROP_UINT8("debug", USBRedirDevice, debug, usbredirparser_warning),
     DEFINE_PROP_STRING("filter", USBRedirDevice, filter_str),
+    DEFINE_PROP_BOOL("streams", USBRedirDevice, enable_streams, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2497,7 +2507,7 @@
 static void usbredir_instance_init(Object *obj)
 {
     USBDevice *udev = USB_DEVICE(obj);
-    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    USBRedirDevice *dev = USB_REDIRECT(udev);
 
     device_add_bootindex_property(obj, &dev->bootindex,
                                   "bootindex", NULL,
@@ -2505,7 +2515,7 @@
 }
 
 static const TypeInfo usbredir_dev_info = {
-    .name          = "usb-redir",
+    .name          = TYPE_USB_REDIR,
     .parent        = TYPE_USB_DEVICE,
     .instance_size = sizeof(USBRedirDevice),
     .class_init    = usbredir_class_initfn,
diff --git a/hw/usb/trace-events b/hw/usb/trace-events
new file mode 100644
index 0000000..2d42fd4
--- /dev/null
+++ b/hw/usb/trace-events
@@ -0,0 +1,268 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/usb/core.c
+usb_packet_state_change(int bus, const char *port, int ep, void *p, const char *o, const char *n) "bus %d, port %s, ep %d, packet %p, state %s -> %s"
+usb_packet_state_fault(int bus, const char *port, int ep, void *p, const char *o, const char *n) "bus %d, port %s, ep %d, packet %p, state %s, expected %s"
+
+# hw/usb/bus.c
+usb_port_claim(int bus, const char *port) "bus %d, port %s"
+usb_port_attach(int bus, const char *port, const char *devspeed, const char *portspeed) "bus %d, port %s, devspeed %s, portspeed %s"
+usb_port_detach(int bus, const char *port) "bus %d, port %s"
+usb_port_release(int bus, const char *port) "bus %d, port %s"
+
+# hw/usb/hcd-ohci.c
+usb_ohci_iso_td_read_failed(uint32_t addr) "ISO_TD read error at %x"
+usb_ohci_iso_td_head(uint32_t head, uint32_t tail, uint32_t flags, uint32_t bp, uint32_t next, uint32_t be, uint32_t framenum, uint32_t startframe, uint32_t framecount, int rel_frame_num) "ISO_TD ED head 0x%.8x tailp 0x%.8x\n0x%.8x 0x%.8x 0x%.8x 0x%.8x\nframe_number 0x%.8x starting_frame 0x%.8x\nframe_count  0x%.8x relative %d"
+usb_ohci_iso_td_head_offset(uint32_t o0, uint32_t o1, uint32_t o2, uint32_t o3, uint32_t o4, uint32_t o5, uint32_t o6, uint32_t o7) "0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x"
+usb_ohci_iso_td_relative_frame_number_neg(int rel) "ISO_TD R=%d < 0"
+usb_ohci_iso_td_relative_frame_number_big(int rel, int count) "ISO_TD R=%d > FC=%d"
+usb_ohci_iso_td_bad_direction(int dir) "Bad direction %d"
+usb_ohci_iso_td_bad_bp_be(uint32_t bp, uint32_t be) "ISO_TD bp 0x%.8x be 0x%.8x"
+usb_ohci_iso_td_bad_cc_not_accessed(uint32_t start, uint32_t next) "ISO_TD cc != not accessed 0x%.8x 0x%.8x"
+usb_ohci_iso_td_bad_cc_overrun(uint32_t start, uint32_t next) "ISO_TD start_offset=0x%.8x > next_offset=0x%.8x"
+usb_ohci_iso_td_so(uint32_t so, uint32_t eo, uint32_t s, uint32_t e, const char *str, ssize_t len, int ret) "0x%.8x eo 0x%.8x\nsa 0x%.8x ea 0x%.8x\ndir %s len %zu ret %d"
+usb_ohci_iso_td_data_overrun(int ret, ssize_t len) "DataOverrun %d > %zu"
+usb_ohci_iso_td_data_underrun(int ret) "DataUnderrun %d"
+usb_ohci_iso_td_nak(int ret) "got NAK/STALL %d"
+usb_ohci_iso_td_bad_response(int ret) "Bad device response %d"
+usb_ohci_port_attach(int index) "port #%d"
+usb_ohci_port_detach(int index) "port #%d"
+usb_ohci_port_wakeup(int index) "port #%d"
+usb_ohci_port_suspend(int index) "port #%d"
+usb_ohci_port_reset(int index) "port #%d"
+usb_ohci_remote_wakeup(const char *s) "%s: SUSPEND->RESUME"
+usb_ohci_reset(const char *s) "%s"
+usb_ohci_start(const char *s) "%s: USB Operational"
+usb_ohci_resume(const char *s) "%s: USB Resume"
+usb_ohci_stop(const char *s) "%s: USB Suspended"
+usb_ohci_exit(const char *s) "%s"
+usb_ohci_set_ctl(const char *s, uint32_t new_state) "%s: new state 0x%x"
+usb_ohci_td_underrun(void) ""
+usb_ohci_td_dev_error(void) ""
+usb_ohci_td_nak(void) ""
+usb_ohci_td_stall(void) ""
+usb_ohci_td_babble(void) ""
+usb_ohci_td_bad_device_response(int rc) "%d"
+usb_ohci_td_read_error(uint32_t addr) "TD read error at %x"
+usb_ohci_td_bad_direction(int dir) "Bad direction %d"
+usb_ohci_td_skip_async(void) ""
+usb_ohci_td_pkt_hdr(uint32_t addr, int64_t pktlen, int64_t len, const char *s, int flag_r, uint32_t cbp, uint32_t be) " TD @ 0x%.8x %" PRId64 " of %" PRId64 " bytes %s r=%d cbp=0x%.8x be=0x%.8x"
+usb_ohci_td_pkt_short(const char *dir, const char *buf) "%s data: %s"
+usb_ohci_td_pkt_full(const char *dir, const char *buf) "%s data: %s"
+usb_ohci_td_too_many_pending(void) ""
+usb_ohci_td_packet_status(int status) "status=%d"
+usb_ohci_ed_read_error(uint32_t addr) "ED read error at %x"
+usb_ohci_ed_pkt(uint32_t cur, int h, int c, uint32_t head, uint32_t tail, uint32_t next) "ED @ 0x%.8x h=%u c=%u\n  head=0x%.8x tailp=0x%.8x next=0x%.8x"
+usb_ohci_ed_pkt_flags(uint32_t fa, uint32_t en, uint32_t d, int s, int k, int f, uint32_t mps) "fa=%u en=%u d=%u s=%u k=%u f=%u mps=%u"
+usb_ohci_hcca_read_error(uint32_t addr) "HCCA read error at %x"
+usb_ohci_mem_read_unaligned(uint32_t addr) "at %x"
+usb_ohci_mem_read_bad_offset(uint32_t addr) "%x"
+usb_ohci_mem_write_unaligned(uint32_t addr) "at %x"
+usb_ohci_mem_write_bad_offset(uint32_t addr) "%x"
+usb_ohci_process_lists(uint32_t head, uint32_t cur) "head %x, cur %x"
+usb_ohci_bus_eof_timer_failed(const char *name) "%s: timer_new_ns failed"
+usb_ohci_set_frame_interval(const char *name, uint16_t fi_x, uint16_t fi_u) "%s: FrameInterval = 0x%x (%u)"
+usb_ohci_hub_power_up(void) "powered up all ports"
+usb_ohci_hub_power_down(void) "powered down all ports"
+usb_ohci_init_time(int64_t frametime, int64_t bittime) "usb_bit_time=%" PRId64 " usb_frame_time=%" PRId64
+usb_ohci_die(void) ""
+usb_ohci_async_complete(void) ""
+
+# hw/usb/hcd-ehci.c
+usb_ehci_reset(void) "=== RESET ==="
+usb_ehci_unrealize(void) "=== UNREALIZE ==="
+usb_ehci_opreg_read(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
+usb_ehci_opreg_write(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
+usb_ehci_opreg_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
+usb_ehci_portsc_read(uint32_t addr, uint32_t port, uint32_t val) "rd mmio %04x [port %d] = %x"
+usb_ehci_portsc_write(uint32_t addr, uint32_t port, uint32_t val) "wr mmio %04x [port %d] = %x"
+usb_ehci_portsc_change(uint32_t addr, uint32_t port, uint32_t new, uint32_t old) "ch mmio %04x [port %d] = %x (old: %x)"
+usb_ehci_usbsts(const char *sts, int state) "usbsts %s %d"
+usb_ehci_state(const char *schedule, const char *state) "%s schedule %s"
+usb_ehci_qh_ptrs(void *q, uint32_t addr, uint32_t nxt, uint32_t c_qtd, uint32_t n_qtd, uint32_t a_qtd) "q %p - QH @ %08x: next %08x qtds %08x,%08x,%08x"
+usb_ehci_qh_fields(uint32_t addr, int rl, int mplen, int eps, int ep, int devaddr) "QH @ %08x - rl %d, mplen %d, eps %d, ep %d, dev %d"
+usb_ehci_qh_bits(uint32_t addr, int c, int h, int dtc, int i) "QH @ %08x - c %d, h %d, dtc %d, i %d"
+usb_ehci_qtd_ptrs(void *q, uint32_t addr, uint32_t nxt, uint32_t altnext) "q %p - QTD @ %08x: next %08x altnext %08x"
+usb_ehci_qtd_fields(uint32_t addr, int tbytes, int cpage, int cerr, int pid) "QTD @ %08x - tbytes %d, cpage %d, cerr %d, pid %d"
+usb_ehci_qtd_bits(uint32_t addr, int ioc, int active, int halt, int babble, int xacterr) "QTD @ %08x - ioc %d, active %d, halt %d, babble %d, xacterr %d"
+usb_ehci_itd(uint32_t addr, uint32_t nxt, uint32_t mplen, uint32_t mult, uint32_t ep, uint32_t devaddr) "ITD @ %08x: next %08x - mplen %d, mult %d, ep %d, dev %d"
+usb_ehci_sitd(uint32_t addr, uint32_t nxt, uint32_t active) "ITD @ %08x: next %08x - active %d"
+usb_ehci_port_attach(uint32_t port, const char *owner, const char *device) "attach port #%d, owner %s, device %s"
+usb_ehci_port_detach(uint32_t port, const char *owner) "detach port #%d, owner %s"
+usb_ehci_port_reset(uint32_t port, int enable) "reset port #%d - %d"
+usb_ehci_port_suspend(uint32_t port) "port #%d"
+usb_ehci_port_wakeup(uint32_t port) "port #%d"
+usb_ehci_port_resume(uint32_t port) "port #%d"
+usb_ehci_queue_action(void *q, const char *action) "q %p: %s"
+usb_ehci_packet_action(void *q, void *p, const char *action) "q %p p %p: %s"
+usb_ehci_irq(uint32_t level, uint32_t frindex, uint32_t sts, uint32_t mask) "level %d, frindex 0x%04x, sts 0x%x, mask 0x%x"
+usb_ehci_guest_bug(const char *reason) "%s"
+usb_ehci_doorbell_ring(void) ""
+usb_ehci_doorbell_ack(void) ""
+usb_ehci_dma_error(void) ""
+
+# hw/usb/hcd-uhci.c
+usb_uhci_reset(void) "=== RESET ==="
+usb_uhci_exit(void) "=== EXIT ==="
+usb_uhci_schedule_start(void) ""
+usb_uhci_schedule_stop(void) ""
+usb_uhci_frame_start(uint32_t num) "nr %d"
+usb_uhci_frame_stop_bandwidth(void) ""
+usb_uhci_frame_loop_stop_idle(void) ""
+usb_uhci_frame_loop_continue(void) ""
+usb_uhci_mmio_readw(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%04x"
+usb_uhci_mmio_writew(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%04x"
+usb_uhci_queue_add(uint32_t token) "token 0x%x"
+usb_uhci_queue_del(uint32_t token, const char *reason) "token 0x%x: %s"
+usb_uhci_packet_add(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_link_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_unlink_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_cancel(uint32_t token, uint32_t addr, int done) "token 0x%x, td 0x%x, done %d"
+usb_uhci_packet_complete_success(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_complete_shortxfer(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_complete_stall(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_complete_babble(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_complete_error(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_packet_del(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
+usb_uhci_qh_load(uint32_t qh) "qh 0x%x"
+usb_uhci_td_load(uint32_t qh, uint32_t td, uint32_t ctrl, uint32_t token) "qh 0x%x, td 0x%x, ctrl 0x%x, token 0x%x"
+usb_uhci_td_queue(uint32_t td, uint32_t ctrl, uint32_t token) "td 0x%x, ctrl 0x%x, token 0x%x"
+usb_uhci_td_nextqh(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
+usb_uhci_td_async(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
+usb_uhci_td_complete(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
+
+# hw/usb/hcd-xhci.c
+usb_xhci_reset(void) "=== RESET ==="
+usb_xhci_exit(void) "=== EXIT ==="
+usb_xhci_run(void) ""
+usb_xhci_stop(void) ""
+usb_xhci_cap_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
+usb_xhci_oper_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
+usb_xhci_port_read(uint32_t port, uint32_t off, uint32_t val) "port %d, off 0x%04x, ret 0x%08x"
+usb_xhci_runtime_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
+usb_xhci_doorbell_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
+usb_xhci_oper_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
+usb_xhci_port_write(uint32_t port, uint32_t off, uint32_t val) "port %d, off 0x%04x, val 0x%08x"
+usb_xhci_runtime_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
+usb_xhci_doorbell_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
+usb_xhci_irq_intx(uint32_t level) "level %d"
+usb_xhci_irq_msi(uint32_t nr) "nr %d"
+usb_xhci_irq_msix(uint32_t nr) "nr %d"
+usb_xhci_irq_msix_use(uint32_t nr) "nr %d"
+usb_xhci_irq_msix_unuse(uint32_t nr) "nr %d"
+usb_xhci_queue_event(uint32_t vector, uint32_t idx, const char *trb, const char *evt, uint64_t param, uint32_t status, uint32_t control) "v %d, idx %d, %s, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
+usb_xhci_fetch_trb(uint64_t addr, const char *name, uint64_t param, uint32_t status, uint32_t control) "addr %016" PRIx64 ", %s, p %016" PRIx64 ", s %08x, c 0x%08x"
+usb_xhci_port_reset(uint32_t port, bool warm) "port %d, warm %d"
+usb_xhci_port_link(uint32_t port, uint32_t pls) "port %d, pls %d"
+usb_xhci_port_notify(uint32_t port, uint32_t pls) "port %d, bits %x"
+usb_xhci_slot_enable(uint32_t slotid) "slotid %d"
+usb_xhci_slot_disable(uint32_t slotid) "slotid %d"
+usb_xhci_slot_address(uint32_t slotid, const char *port) "slotid %d, port %s"
+usb_xhci_slot_configure(uint32_t slotid) "slotid %d"
+usb_xhci_slot_evaluate(uint32_t slotid) "slotid %d"
+usb_xhci_slot_reset(uint32_t slotid) "slotid %d"
+usb_xhci_ep_enable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
+usb_xhci_ep_disable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
+usb_xhci_ep_set_dequeue(uint32_t slotid, uint32_t epid, uint32_t streamid, uint64_t param) "slotid %d, epid %d, streamid %d, ptr %016" PRIx64
+usb_xhci_ep_kick(uint32_t slotid, uint32_t epid, uint32_t streamid) "slotid %d, epid %d, streamid %d"
+usb_xhci_ep_stop(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
+usb_xhci_ep_reset(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
+usb_xhci_ep_state(uint32_t slotid, uint32_t epid, const char *os, const char *ns) "slotid %d, epid %d, %s -> %s"
+usb_xhci_xfer_start(void *xfer, uint32_t slotid, uint32_t epid, uint32_t streamid) "%p: slotid %d, epid %d, streamid %d"
+usb_xhci_xfer_async(void *xfer) "%p"
+usb_xhci_xfer_nak(void *xfer) "%p"
+usb_xhci_xfer_retry(void *xfer) "%p"
+usb_xhci_xfer_success(void *xfer, uint32_t bytes) "%p: len %d"
+usb_xhci_xfer_error(void *xfer, uint32_t ret) "%p: ret %d"
+usb_xhci_unimplemented(const char *item, int nr) "%s (0x%x)"
+
+# hw/usb/desc.c
+usb_desc_device(int addr, int len, int ret) "dev %d query device, len %d, ret %d"
+usb_desc_device_qualifier(int addr, int len, int ret) "dev %d query device qualifier, len %d, ret %d"
+usb_desc_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
+usb_desc_other_speed_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
+usb_desc_string(int addr, int index, int len, int ret) "dev %d query string %d, len %d, ret %d"
+usb_desc_bos(int addr, int len, int ret) "dev %d bos, len %d, ret %d"
+usb_desc_msos(int addr, int index, int len, int ret) "dev %d msos, index 0x%x, len %d, ret %d"
+usb_set_addr(int addr) "dev %d"
+usb_set_config(int addr, int config, int ret) "dev %d, config %d, ret %d"
+usb_set_interface(int addr, int iface, int alt, int ret) "dev %d, interface %d, altsetting %d, ret %d"
+usb_clear_device_feature(int addr, int feature, int ret) "dev %d, feature %d, ret %d"
+usb_set_device_feature(int addr, int feature, int ret) "dev %d, feature %d, ret %d"
+
+# hw/usb/dev-hub.c
+usb_hub_reset(int addr) "dev %d"
+usb_hub_control(int addr, int request, int value, int index, int length) "dev %d, req 0x%x, value %d, index %d, langth %d"
+usb_hub_get_port_status(int addr, int nr, int status, int changed) "dev %d, port %d, status 0x%x, changed 0x%x"
+usb_hub_set_port_feature(int addr, int nr, const char *f) "dev %d, port %d, feature %s"
+usb_hub_clear_port_feature(int addr, int nr, const char *f) "dev %d, port %d, feature %s"
+usb_hub_attach(int addr, int nr) "dev %d, port %d"
+usb_hub_detach(int addr, int nr) "dev %d, port %d"
+usb_hub_status_report(int addr, int status) "dev %d, status 0x%x"
+
+# hw/usb/dev-uas.c
+usb_uas_reset(int addr) "dev %d"
+usb_uas_command(int addr, uint16_t tag, int lun, uint32_t lun64_1, uint32_t lun64_2) "dev %d, tag 0x%x, lun %d, lun64 %08x-%08x"
+usb_uas_response(int addr, uint16_t tag, uint8_t code) "dev %d, tag 0x%x, code 0x%x"
+usb_uas_sense(int addr, uint16_t tag, uint8_t status) "dev %d, tag 0x%x, status 0x%x"
+usb_uas_read_ready(int addr, uint16_t tag) "dev %d, tag 0x%x"
+usb_uas_write_ready(int addr, uint16_t tag) "dev %d, tag 0x%x"
+usb_uas_xfer_data(int addr, uint16_t tag, uint32_t copy, uint32_t uoff, uint32_t usize, uint32_t soff, uint32_t ssize) "dev %d, tag 0x%x, copy %d, usb-pkt %d/%d, scsi-buf %d/%d"
+usb_uas_scsi_data(int addr, uint16_t tag, uint32_t bytes) "dev %d, tag 0x%x, bytes %d"
+usb_uas_scsi_complete(int addr, uint16_t tag, uint32_t status, uint32_t resid) "dev %d, tag 0x%x, status 0x%x, residue %d"
+usb_uas_tmf_abort_task(int addr, uint16_t tag, uint16_t task_tag) "dev %d, tag 0x%x, task-tag 0x%x"
+usb_uas_tmf_logical_unit_reset(int addr, uint16_t tag, int lun) "dev %d, tag 0x%x, lun %d"
+usb_uas_tmf_unsupported(int addr, uint16_t tag, uint32_t function) "dev %d, tag 0x%x, function 0x%x"
+
+# hw/usb/dev-mtp.c
+usb_mtp_reset(int addr) "dev %d"
+usb_mtp_command(int dev, uint16_t code, uint32_t trans, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4) "dev %d, code 0x%x, trans 0x%x, args 0x%x, 0x%x, 0x%x, 0x%x, 0x%x"
+usb_mtp_success(int dev, uint32_t trans, uint32_t arg0, uint32_t arg1) "dev %d, trans 0x%x, args 0x%x, 0x%x"
+usb_mtp_error(int dev, uint16_t code, uint32_t trans, uint32_t arg0, uint32_t arg1) "dev %d, code 0x%x, trans 0x%x, args 0x%x, 0x%x"
+usb_mtp_data_in(int dev, uint32_t trans, uint32_t len) "dev %d, trans 0x%x, len %d"
+usb_mtp_xfer(int dev, uint32_t ep, uint32_t dlen, uint32_t plen) "dev %d, ep %d, %d/%d"
+usb_mtp_nak(int dev, uint32_t ep) "dev %d, ep %d"
+usb_mtp_stall(int dev, const char *reason) "dev %d, reason: %s"
+usb_mtp_op_get_device_info(int dev) "dev %d"
+usb_mtp_op_open_session(int dev) "dev %d"
+usb_mtp_op_close_session(int dev) "dev %d"
+usb_mtp_op_get_storage_ids(int dev) "dev %d"
+usb_mtp_op_get_storage_info(int dev) "dev %d"
+usb_mtp_op_get_num_objects(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_op_get_object_handles(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_op_get_object_info(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_op_get_object(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_op_get_partial_object(int dev, uint32_t handle, const char *path, uint32_t offset, uint32_t length) "dev %d, handle 0x%x, path %s, off %d, len %d"
+usb_mtp_op_unknown(int dev, uint32_t code) "dev %d, command code 0x%x"
+usb_mtp_object_alloc(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_object_free(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_add_child(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
+usb_mtp_inotify_event(int dev, const char *path, uint32_t mask, const char *s) "dev %d, path %s mask 0x%x event %s"
+
+# hw/usb/host-libusb.c
+usb_host_open_started(int bus, int addr) "dev %d:%d"
+usb_host_open_success(int bus, int addr) "dev %d:%d"
+usb_host_open_failure(int bus, int addr) "dev %d:%d"
+usb_host_close(int bus, int addr) "dev %d:%d"
+usb_host_attach_kernel(int bus, int addr, int interface) "dev %d:%d, if %d"
+usb_host_detach_kernel(int bus, int addr, int interface) "dev %d:%d, if %d"
+usb_host_set_address(int bus, int addr, int config) "dev %d:%d, address %d"
+usb_host_set_config(int bus, int addr, int config) "dev %d:%d, config %d"
+usb_host_set_interface(int bus, int addr, int interface, int alt) "dev %d:%d, interface %d, alt %d"
+usb_host_claim_interface(int bus, int addr, int config, int interface) "dev %d:%d, config %d, if %d"
+usb_host_release_interface(int bus, int addr, int interface) "dev %d:%d, if %d"
+usb_host_req_control(int bus, int addr, void *p, int req, int value, int index) "dev %d:%d, packet %p, req 0x%x, value %d, index %d"
+usb_host_req_data(int bus, int addr, void *p, int in, int ep, int size) "dev %d:%d, packet %p, in %d, ep %d, size %d"
+usb_host_req_complete(int bus, int addr, void *p, int status, int length) "dev %d:%d, packet %p, status %d, length %d"
+usb_host_req_emulated(int bus, int addr, void *p, int status) "dev %d:%d, packet %p, status %d"
+usb_host_req_canceled(int bus, int addr, void *p) "dev %d:%d, packet %p"
+usb_host_iso_start(int bus, int addr, int ep) "dev %d:%d, ep %d"
+usb_host_iso_stop(int bus, int addr, int ep) "dev %d:%d, ep %d"
+usb_host_iso_out_of_bufs(int bus, int addr, int ep) "dev %d:%d, ep %d"
+usb_host_reset(int bus, int addr) "dev %d:%d"
+usb_host_auto_scan_enabled(void)
+usb_host_auto_scan_disabled(void)
+usb_host_parse_config(int bus, int addr, int value, int active) "dev %d:%d, value %d, active %d"
+usb_host_parse_interface(int bus, int addr, int num, int alt, int active) "dev %d:%d, num %d, alt %d, active %d"
+usb_host_parse_endpoint(int bus, int addr, int ep, const char *dir, const char *type, int active) "dev %d:%d, ep %d, %s, %s, active %d"
+usb_host_parse_error(int bus, int addr, const char *errmsg) "dev %d:%d, msg %s"
diff --git a/hw/usb/tusb6010.c b/hw/usb/tusb6010.c
new file mode 100644
index 0000000..8f593a6
--- /dev/null
+++ b/hw/usb/tusb6010.c
@@ -0,0 +1,817 @@
+/*
+ * Texas Instruments TUSB6010 emulation.
+ * Based on reverse-engineering of a linux driver.
+ *
+ * Copyright (C) 2008 Nokia Corporation
+ * Written by Andrzej Zaborowski <andrew@openedhand.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/timer.h"
+#include "hw/usb.h"
+#include "hw/arm/omap.h"
+#include "hw/irq.h"
+#include "hw/devices.h"
+#include "hw/sysbus.h"
+
+#define TYPE_TUSB6010 "tusb6010"
+#define TUSB(obj) OBJECT_CHECK(TUSBState, (obj), TYPE_TUSB6010)
+
+typedef struct TUSBState {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem[2];
+    qemu_irq irq;
+    MUSBState *musb;
+    QEMUTimer *otg_timer;
+    QEMUTimer *pwr_timer;
+
+    int power;
+    uint32_t scratch;
+    uint16_t test_reset;
+    uint32_t prcm_config;
+    uint32_t prcm_mngmt;
+    uint16_t otg_status;
+    uint32_t dev_config;
+    int host_mode;
+    uint32_t intr;
+    uint32_t intr_ok;
+    uint32_t mask;
+    uint32_t usbip_intr;
+    uint32_t usbip_mask;
+    uint32_t gpio_intr;
+    uint32_t gpio_mask;
+    uint32_t gpio_config;
+    uint32_t dma_intr;
+    uint32_t dma_mask;
+    uint32_t dma_map;
+    uint32_t dma_config;
+    uint32_t ep0_config;
+    uint32_t rx_config[15];
+    uint32_t tx_config[15];
+    uint32_t wkup_mask;
+    uint32_t pullup[2];
+    uint32_t control_config;
+    uint32_t otg_timer_val;
+} TUSBState;
+
+#define TUSB_DEVCLOCK			60000000	/* 60 MHz */
+
+#define TUSB_VLYNQ_CTRL			0x004
+
+/* Mentor Graphics OTG core registers.  */
+#define TUSB_BASE_OFFSET		0x400
+
+/* FIFO registers, 32-bit.  */
+#define TUSB_FIFO_BASE			0x600
+
+/* Device System & Control registers, 32-bit.  */
+#define TUSB_SYS_REG_BASE		0x800
+
+#define TUSB_DEV_CONF			(TUSB_SYS_REG_BASE + 0x000)
+#define	TUSB_DEV_CONF_USB_HOST_MODE	(1 << 16)
+#define	TUSB_DEV_CONF_PROD_TEST_MODE	(1 << 15)
+#define	TUSB_DEV_CONF_SOFT_ID		(1 << 1)
+#define	TUSB_DEV_CONF_ID_SEL		(1 << 0)
+
+#define TUSB_PHY_OTG_CTRL_ENABLE	(TUSB_SYS_REG_BASE + 0x004)
+#define TUSB_PHY_OTG_CTRL		(TUSB_SYS_REG_BASE + 0x008)
+#define	TUSB_PHY_OTG_CTRL_WRPROTECT	(0xa5 << 24)
+#define	TUSB_PHY_OTG_CTRL_O_ID_PULLUP	(1 << 23)
+#define	TUSB_PHY_OTG_CTRL_O_VBUS_DET_EN	(1 << 19)
+#define	TUSB_PHY_OTG_CTRL_O_SESS_END_EN	(1 << 18)
+#define	TUSB_PHY_OTG_CTRL_TESTM2	(1 << 17)
+#define	TUSB_PHY_OTG_CTRL_TESTM1	(1 << 16)
+#define	TUSB_PHY_OTG_CTRL_TESTM0	(1 << 15)
+#define	TUSB_PHY_OTG_CTRL_TX_DATA2	(1 << 14)
+#define	TUSB_PHY_OTG_CTRL_TX_GZ2	(1 << 13)
+#define	TUSB_PHY_OTG_CTRL_TX_ENABLE2	(1 << 12)
+#define	TUSB_PHY_OTG_CTRL_DM_PULLDOWN	(1 << 11)
+#define	TUSB_PHY_OTG_CTRL_DP_PULLDOWN	(1 << 10)
+#define	TUSB_PHY_OTG_CTRL_OSC_EN	(1 << 9)
+#define	TUSB_PHY_OTG_CTRL_PHYREF_CLK(v)	(((v) & 3) << 7)
+#define	TUSB_PHY_OTG_CTRL_PD		(1 << 6)
+#define	TUSB_PHY_OTG_CTRL_PLL_ON	(1 << 5)
+#define	TUSB_PHY_OTG_CTRL_EXT_RPU	(1 << 4)
+#define	TUSB_PHY_OTG_CTRL_PWR_GOOD	(1 << 3)
+#define	TUSB_PHY_OTG_CTRL_RESET		(1 << 2)
+#define	TUSB_PHY_OTG_CTRL_SUSPENDM	(1 << 1)
+#define	TUSB_PHY_OTG_CTRL_CLK_MODE	(1 << 0)
+
+/* OTG status register */
+#define TUSB_DEV_OTG_STAT		(TUSB_SYS_REG_BASE + 0x00c)
+#define	TUSB_DEV_OTG_STAT_PWR_CLK_GOOD	(1 << 8)
+#define	TUSB_DEV_OTG_STAT_SESS_END	(1 << 7)
+#define	TUSB_DEV_OTG_STAT_SESS_VALID	(1 << 6)
+#define	TUSB_DEV_OTG_STAT_VBUS_VALID	(1 << 5)
+#define	TUSB_DEV_OTG_STAT_VBUS_SENSE	(1 << 4)
+#define	TUSB_DEV_OTG_STAT_ID_STATUS	(1 << 3)
+#define	TUSB_DEV_OTG_STAT_HOST_DISCON	(1 << 2)
+#define	TUSB_DEV_OTG_STAT_LINE_STATE	(3 << 0)
+#define	TUSB_DEV_OTG_STAT_DP_ENABLE	(1 << 1)
+#define	TUSB_DEV_OTG_STAT_DM_ENABLE	(1 << 0)
+
+#define TUSB_DEV_OTG_TIMER		(TUSB_SYS_REG_BASE + 0x010)
+#define TUSB_DEV_OTG_TIMER_ENABLE	(1 << 31)
+#define TUSB_DEV_OTG_TIMER_VAL(v)	((v) & 0x07ffffff)
+#define TUSB_PRCM_REV			(TUSB_SYS_REG_BASE + 0x014)
+
+/* PRCM configuration register */
+#define TUSB_PRCM_CONF			(TUSB_SYS_REG_BASE + 0x018)
+#define	TUSB_PRCM_CONF_SFW_CPEN		(1 << 24)
+#define	TUSB_PRCM_CONF_SYS_CLKSEL(v)	(((v) & 3) << 16)
+
+/* PRCM management register */
+#define TUSB_PRCM_MNGMT			(TUSB_SYS_REG_BASE + 0x01c)
+#define	TUSB_PRCM_MNGMT_SRP_FIX_TMR(v)	(((v) & 0xf) << 25)
+#define	TUSB_PRCM_MNGMT_SRP_FIX_EN	(1 << 24)
+#define	TUSB_PRCM_MNGMT_VBUS_VAL_TMR(v)	(((v) & 0xf) << 20)
+#define	TUSB_PRCM_MNGMT_VBUS_VAL_FLT_EN	(1 << 19)
+#define	TUSB_PRCM_MNGMT_DFT_CLK_DIS	(1 << 18)
+#define	TUSB_PRCM_MNGMT_VLYNQ_CLK_DIS	(1 << 17)
+#define	TUSB_PRCM_MNGMT_OTG_SESS_END_EN	(1 << 10)
+#define	TUSB_PRCM_MNGMT_OTG_VBUS_DET_EN	(1 << 9)
+#define	TUSB_PRCM_MNGMT_OTG_ID_PULLUP	(1 << 8)
+#define	TUSB_PRCM_MNGMT_15_SW_EN	(1 << 4)
+#define	TUSB_PRCM_MNGMT_33_SW_EN	(1 << 3)
+#define	TUSB_PRCM_MNGMT_5V_CPEN		(1 << 2)
+#define	TUSB_PRCM_MNGMT_PM_IDLE		(1 << 1)
+#define	TUSB_PRCM_MNGMT_DEV_IDLE	(1 << 0)
+
+/* Wake-up source clear and mask registers */
+#define TUSB_PRCM_WAKEUP_SOURCE		(TUSB_SYS_REG_BASE + 0x020)
+#define TUSB_PRCM_WAKEUP_CLEAR		(TUSB_SYS_REG_BASE + 0x028)
+#define TUSB_PRCM_WAKEUP_MASK		(TUSB_SYS_REG_BASE + 0x02c)
+#define	TUSB_PRCM_WAKEUP_RESERVED_BITS	(0xffffe << 13)
+#define	TUSB_PRCM_WGPIO_7		(1 << 12)
+#define	TUSB_PRCM_WGPIO_6		(1 << 11)
+#define	TUSB_PRCM_WGPIO_5		(1 << 10)
+#define	TUSB_PRCM_WGPIO_4		(1 << 9)
+#define	TUSB_PRCM_WGPIO_3		(1 << 8)
+#define	TUSB_PRCM_WGPIO_2		(1 << 7)
+#define	TUSB_PRCM_WGPIO_1		(1 << 6)
+#define	TUSB_PRCM_WGPIO_0		(1 << 5)
+#define	TUSB_PRCM_WHOSTDISCON		(1 << 4)	/* Host disconnect */
+#define	TUSB_PRCM_WBUS			(1 << 3)	/* USB bus resume */
+#define	TUSB_PRCM_WNORCS		(1 << 2)	/* NOR chip select */
+#define	TUSB_PRCM_WVBUS			(1 << 1)	/* OTG PHY VBUS */
+#define	TUSB_PRCM_WID			(1 << 0)	/* OTG PHY ID detect */
+
+#define TUSB_PULLUP_1_CTRL		(TUSB_SYS_REG_BASE + 0x030)
+#define TUSB_PULLUP_2_CTRL		(TUSB_SYS_REG_BASE + 0x034)
+#define TUSB_INT_CTRL_REV		(TUSB_SYS_REG_BASE + 0x038)
+#define TUSB_INT_CTRL_CONF		(TUSB_SYS_REG_BASE + 0x03c)
+#define TUSB_USBIP_INT_SRC		(TUSB_SYS_REG_BASE + 0x040)
+#define TUSB_USBIP_INT_SET		(TUSB_SYS_REG_BASE + 0x044)
+#define TUSB_USBIP_INT_CLEAR		(TUSB_SYS_REG_BASE + 0x048)
+#define TUSB_USBIP_INT_MASK		(TUSB_SYS_REG_BASE + 0x04c)
+#define TUSB_DMA_INT_SRC		(TUSB_SYS_REG_BASE + 0x050)
+#define TUSB_DMA_INT_SET		(TUSB_SYS_REG_BASE + 0x054)
+#define TUSB_DMA_INT_CLEAR		(TUSB_SYS_REG_BASE + 0x058)
+#define TUSB_DMA_INT_MASK		(TUSB_SYS_REG_BASE + 0x05c)
+#define TUSB_GPIO_INT_SRC		(TUSB_SYS_REG_BASE + 0x060)
+#define TUSB_GPIO_INT_SET		(TUSB_SYS_REG_BASE + 0x064)
+#define TUSB_GPIO_INT_CLEAR		(TUSB_SYS_REG_BASE + 0x068)
+#define TUSB_GPIO_INT_MASK		(TUSB_SYS_REG_BASE + 0x06c)
+
+/* NOR flash interrupt source registers */
+#define TUSB_INT_SRC			(TUSB_SYS_REG_BASE + 0x070)
+#define TUSB_INT_SRC_SET		(TUSB_SYS_REG_BASE + 0x074)
+#define TUSB_INT_SRC_CLEAR		(TUSB_SYS_REG_BASE + 0x078)
+#define TUSB_INT_MASK			(TUSB_SYS_REG_BASE + 0x07c)
+#define	TUSB_INT_SRC_TXRX_DMA_DONE	(1 << 24)
+#define	TUSB_INT_SRC_USB_IP_CORE	(1 << 17)
+#define	TUSB_INT_SRC_OTG_TIMEOUT	(1 << 16)
+#define	TUSB_INT_SRC_VBUS_SENSE_CHNG	(1 << 15)
+#define	TUSB_INT_SRC_ID_STATUS_CHNG	(1 << 14)
+#define	TUSB_INT_SRC_DEV_WAKEUP		(1 << 13)
+#define	TUSB_INT_SRC_DEV_READY		(1 << 12)
+#define	TUSB_INT_SRC_USB_IP_TX		(1 << 9)
+#define	TUSB_INT_SRC_USB_IP_RX		(1 << 8)
+#define	TUSB_INT_SRC_USB_IP_VBUS_ERR	(1 << 7)
+#define	TUSB_INT_SRC_USB_IP_VBUS_REQ	(1 << 6)
+#define	TUSB_INT_SRC_USB_IP_DISCON	(1 << 5)
+#define	TUSB_INT_SRC_USB_IP_CONN	(1 << 4)
+#define	TUSB_INT_SRC_USB_IP_SOF		(1 << 3)
+#define	TUSB_INT_SRC_USB_IP_RST_BABBLE	(1 << 2)
+#define	TUSB_INT_SRC_USB_IP_RESUME	(1 << 1)
+#define	TUSB_INT_SRC_USB_IP_SUSPEND	(1 << 0)
+
+#define TUSB_GPIO_REV			(TUSB_SYS_REG_BASE + 0x080)
+#define TUSB_GPIO_CONF			(TUSB_SYS_REG_BASE + 0x084)
+#define TUSB_DMA_CTRL_REV		(TUSB_SYS_REG_BASE + 0x100)
+#define TUSB_DMA_REQ_CONF		(TUSB_SYS_REG_BASE + 0x104)
+#define TUSB_EP0_CONF			(TUSB_SYS_REG_BASE + 0x108)
+#define TUSB_EP_IN_SIZE			(TUSB_SYS_REG_BASE + 0x10c)
+#define TUSB_DMA_EP_MAP			(TUSB_SYS_REG_BASE + 0x148)
+#define TUSB_EP_OUT_SIZE		(TUSB_SYS_REG_BASE + 0x14c)
+#define TUSB_EP_MAX_PACKET_SIZE_OFFSET	(TUSB_SYS_REG_BASE + 0x188)
+#define TUSB_SCRATCH_PAD		(TUSB_SYS_REG_BASE + 0x1c4)
+#define TUSB_WAIT_COUNT			(TUSB_SYS_REG_BASE + 0x1c8)
+#define TUSB_PROD_TEST_RESET		(TUSB_SYS_REG_BASE + 0x1d8)
+
+#define TUSB_DIDR1_LO			(TUSB_SYS_REG_BASE + 0x1f8)
+#define TUSB_DIDR1_HI			(TUSB_SYS_REG_BASE + 0x1fc)
+
+/* Device System & Control register bitfields */
+#define TUSB_INT_CTRL_CONF_INT_RLCYC(v)	(((v) & 0x7) << 18)
+#define TUSB_INT_CTRL_CONF_INT_POLARITY	(1 << 17)
+#define TUSB_INT_CTRL_CONF_INT_MODE	(1 << 16)
+#define TUSB_GPIO_CONF_DMAREQ(v)	(((v) & 0x3f) << 24)
+#define TUSB_DMA_REQ_CONF_BURST_SIZE(v)	(((v) & 3) << 26)
+#define TUSB_DMA_REQ_CONF_DMA_RQ_EN(v)	(((v) & 0x3f) << 20)
+#define TUSB_DMA_REQ_CONF_DMA_RQ_ASR(v)	(((v) & 0xf) << 16)
+#define TUSB_EP0_CONFIG_SW_EN		(1 << 8)
+#define TUSB_EP0_CONFIG_DIR_TX		(1 << 7)
+#define TUSB_EP0_CONFIG_XFR_SIZE(v)	((v) & 0x7f)
+#define TUSB_EP_CONFIG_SW_EN		(1 << 31)
+#define TUSB_EP_CONFIG_XFR_SIZE(v)	((v) & 0x7fffffff)
+#define TUSB_PROD_TEST_RESET_VAL	0xa596
+
+static void tusb_intr_update(TUSBState *s)
+{
+    if (s->control_config & TUSB_INT_CTRL_CONF_INT_POLARITY)
+        qemu_set_irq(s->irq, s->intr & ~s->mask & s->intr_ok);
+    else
+        qemu_set_irq(s->irq, (!(s->intr & ~s->mask)) & s->intr_ok);
+}
+
+static void tusb_usbip_intr_update(TUSBState *s)
+{
+    /* TX interrupt in the MUSB */
+    if (s->usbip_intr & 0x0000ffff & ~s->usbip_mask)
+        s->intr |= TUSB_INT_SRC_USB_IP_TX;
+    else
+        s->intr &= ~TUSB_INT_SRC_USB_IP_TX;
+
+    /* RX interrupt in the MUSB */
+    if (s->usbip_intr & 0xffff0000 & ~s->usbip_mask)
+        s->intr |= TUSB_INT_SRC_USB_IP_RX;
+    else
+        s->intr &= ~TUSB_INT_SRC_USB_IP_RX;
+
+    /* XXX: What about TUSB_INT_SRC_USB_IP_CORE?  */
+
+    tusb_intr_update(s);
+}
+
+static void tusb_dma_intr_update(TUSBState *s)
+{
+    if (s->dma_intr & ~s->dma_mask)
+        s->intr |= TUSB_INT_SRC_TXRX_DMA_DONE;
+    else
+        s->intr &= ~TUSB_INT_SRC_TXRX_DMA_DONE;
+
+    tusb_intr_update(s);
+}
+
+static void tusb_gpio_intr_update(TUSBState *s)
+{
+    /* TODO: How is this signalled?  */
+}
+
+static uint32_t tusb_async_readb(void *opaque, hwaddr addr)
+{
+    TUSBState *s = (TUSBState *) opaque;
+
+    switch (addr & 0xfff) {
+    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
+        return musb_read[0](s->musb, addr & 0x1ff);
+
+    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
+        return musb_read[0](s->musb, 0x20 + ((addr >> 3) & 0x3c));
+    }
+
+    printf("%s: unknown register at %03x\n",
+                    __FUNCTION__, (int) (addr & 0xfff));
+    return 0;
+}
+
+static uint32_t tusb_async_readh(void *opaque, hwaddr addr)
+{
+    TUSBState *s = (TUSBState *) opaque;
+
+    switch (addr & 0xfff) {
+    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
+        return musb_read[1](s->musb, addr & 0x1ff);
+
+    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
+        return musb_read[1](s->musb, 0x20 + ((addr >> 3) & 0x3c));
+    }
+
+    printf("%s: unknown register at %03x\n",
+                    __FUNCTION__, (int) (addr & 0xfff));
+    return 0;
+}
+
+static uint32_t tusb_async_readw(void *opaque, hwaddr addr)
+{
+    TUSBState *s = (TUSBState *) opaque;
+    int offset = addr & 0xfff;
+    int epnum;
+    uint32_t ret;
+
+    switch (offset) {
+    case TUSB_DEV_CONF:
+        return s->dev_config;
+
+    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
+        return musb_read[2](s->musb, offset & 0x1ff);
+
+    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
+        return musb_read[2](s->musb, 0x20 + ((addr >> 3) & 0x3c));
+
+    case TUSB_PHY_OTG_CTRL_ENABLE:
+    case TUSB_PHY_OTG_CTRL:
+        return 0x00;	/* TODO */
+
+    case TUSB_DEV_OTG_STAT:
+        ret = s->otg_status;
+#if 0
+        if (!(s->prcm_mngmt & TUSB_PRCM_MNGMT_OTG_VBUS_DET_EN))
+            ret &= ~TUSB_DEV_OTG_STAT_VBUS_VALID;
+#endif
+        return ret;
+    case TUSB_DEV_OTG_TIMER:
+        return s->otg_timer_val;
+
+    case TUSB_PRCM_REV:
+        return 0x20;
+    case TUSB_PRCM_CONF:
+        return s->prcm_config;
+    case TUSB_PRCM_MNGMT:
+        return s->prcm_mngmt;
+    case TUSB_PRCM_WAKEUP_SOURCE:
+    case TUSB_PRCM_WAKEUP_CLEAR:	/* TODO: What does this one return?  */
+        return 0x00000000;
+    case TUSB_PRCM_WAKEUP_MASK:
+        return s->wkup_mask;
+
+    case TUSB_PULLUP_1_CTRL:
+        return s->pullup[0];
+    case TUSB_PULLUP_2_CTRL:
+        return s->pullup[1];
+
+    case TUSB_INT_CTRL_REV:
+        return 0x20;
+    case TUSB_INT_CTRL_CONF:
+        return s->control_config;
+
+    case TUSB_USBIP_INT_SRC:
+    case TUSB_USBIP_INT_SET:	/* TODO: What do these two return?  */
+    case TUSB_USBIP_INT_CLEAR:
+        return s->usbip_intr;
+    case TUSB_USBIP_INT_MASK:
+        return s->usbip_mask;
+
+    case TUSB_DMA_INT_SRC:
+    case TUSB_DMA_INT_SET:	/* TODO: What do these two return?  */
+    case TUSB_DMA_INT_CLEAR:
+        return s->dma_intr;
+    case TUSB_DMA_INT_MASK:
+        return s->dma_mask;
+
+    case TUSB_GPIO_INT_SRC:	/* TODO: What do these two return?  */
+    case TUSB_GPIO_INT_SET:
+    case TUSB_GPIO_INT_CLEAR:
+        return s->gpio_intr;
+    case TUSB_GPIO_INT_MASK:
+        return s->gpio_mask;
+
+    case TUSB_INT_SRC:
+    case TUSB_INT_SRC_SET:	/* TODO: What do these two return?  */
+    case TUSB_INT_SRC_CLEAR:
+        return s->intr;
+    case TUSB_INT_MASK:
+        return s->mask;
+
+    case TUSB_GPIO_REV:
+        return 0x30;
+    case TUSB_GPIO_CONF:
+        return s->gpio_config;
+
+    case TUSB_DMA_CTRL_REV:
+        return 0x30;
+    case TUSB_DMA_REQ_CONF:
+        return s->dma_config;
+    case TUSB_EP0_CONF:
+        return s->ep0_config;
+    case TUSB_EP_IN_SIZE ... (TUSB_EP_IN_SIZE + 0x3b):
+        epnum = (offset - TUSB_EP_IN_SIZE) >> 2;
+        return s->tx_config[epnum];
+    case TUSB_DMA_EP_MAP:
+        return s->dma_map;
+    case TUSB_EP_OUT_SIZE ... (TUSB_EP_OUT_SIZE + 0x3b):
+        epnum = (offset - TUSB_EP_OUT_SIZE) >> 2;
+        return s->rx_config[epnum];
+    case TUSB_EP_MAX_PACKET_SIZE_OFFSET ...
+            (TUSB_EP_MAX_PACKET_SIZE_OFFSET + 0x3b):
+        return 0x00000000;	/* TODO */
+    case TUSB_WAIT_COUNT:
+        return 0x00;		/* TODO */
+
+    case TUSB_SCRATCH_PAD:
+        return s->scratch;
+
+    case TUSB_PROD_TEST_RESET:
+        return s->test_reset;
+
+    /* DIE IDs */
+    case TUSB_DIDR1_LO:
+        return 0xa9453c59;
+    case TUSB_DIDR1_HI:
+        return 0x54059adf;
+    }
+
+    printf("%s: unknown register at %03x\n", __FUNCTION__, offset);
+    return 0;
+}
+
+static void tusb_async_writeb(void *opaque, hwaddr addr,
+                uint32_t value)
+{
+    TUSBState *s = (TUSBState *) opaque;
+
+    switch (addr & 0xfff) {
+    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
+        musb_write[0](s->musb, addr & 0x1ff, value);
+        break;
+
+    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
+        musb_write[0](s->musb, 0x20 + ((addr >> 3) & 0x3c), value);
+        break;
+
+    default:
+        printf("%s: unknown register at %03x\n",
+                        __FUNCTION__, (int) (addr & 0xfff));
+        return;
+    }
+}
+
+static void tusb_async_writeh(void *opaque, hwaddr addr,
+                uint32_t value)
+{
+    TUSBState *s = (TUSBState *) opaque;
+
+    switch (addr & 0xfff) {
+    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
+        musb_write[1](s->musb, addr & 0x1ff, value);
+        break;
+
+    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
+        musb_write[1](s->musb, 0x20 + ((addr >> 3) & 0x3c), value);
+        break;
+
+    default:
+        printf("%s: unknown register at %03x\n",
+                        __FUNCTION__, (int) (addr & 0xfff));
+        return;
+    }
+}
+
+static void tusb_async_writew(void *opaque, hwaddr addr,
+                uint32_t value)
+{
+    TUSBState *s = (TUSBState *) opaque;
+    int offset = addr & 0xfff;
+    int epnum;
+
+    switch (offset) {
+    case TUSB_VLYNQ_CTRL:
+        break;
+
+    case TUSB_BASE_OFFSET ... (TUSB_BASE_OFFSET | 0x1ff):
+        musb_write[2](s->musb, offset & 0x1ff, value);
+        break;
+
+    case TUSB_FIFO_BASE ... (TUSB_FIFO_BASE | 0x1ff):
+        musb_write[2](s->musb, 0x20 + ((addr >> 3) & 0x3c), value);
+        break;
+
+    case TUSB_DEV_CONF:
+        s->dev_config = value;
+        s->host_mode = (value & TUSB_DEV_CONF_USB_HOST_MODE);
+        if (value & TUSB_DEV_CONF_PROD_TEST_MODE)
+            hw_error("%s: Product Test mode not allowed\n", __FUNCTION__);
+        break;
+
+    case TUSB_PHY_OTG_CTRL_ENABLE:
+    case TUSB_PHY_OTG_CTRL:
+        return;		/* TODO */
+    case TUSB_DEV_OTG_TIMER:
+        s->otg_timer_val = value;
+        if (value & TUSB_DEV_OTG_TIMER_ENABLE)
+            timer_mod(s->otg_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                            muldiv64(TUSB_DEV_OTG_TIMER_VAL(value),
+                                     NANOSECONDS_PER_SECOND, TUSB_DEVCLOCK));
+        else
+            timer_del(s->otg_timer);
+        break;
+
+    case TUSB_PRCM_CONF:
+        s->prcm_config = value;
+        break;
+    case TUSB_PRCM_MNGMT:
+        s->prcm_mngmt = value;
+        break;
+    case TUSB_PRCM_WAKEUP_CLEAR:
+        break;
+    case TUSB_PRCM_WAKEUP_MASK:
+        s->wkup_mask = value;
+        break;
+
+    case TUSB_PULLUP_1_CTRL:
+        s->pullup[0] = value;
+        break;
+    case TUSB_PULLUP_2_CTRL:
+        s->pullup[1] = value;
+        break;
+    case TUSB_INT_CTRL_CONF:
+        s->control_config = value;
+        tusb_intr_update(s);
+        break;
+
+    case TUSB_USBIP_INT_SET:
+        s->usbip_intr |= value;
+        tusb_usbip_intr_update(s);
+        break;
+    case TUSB_USBIP_INT_CLEAR:
+        s->usbip_intr &= ~value;
+        tusb_usbip_intr_update(s);
+        musb_core_intr_clear(s->musb, ~value);
+        break;
+    case TUSB_USBIP_INT_MASK:
+        s->usbip_mask = value;
+        tusb_usbip_intr_update(s);
+        break;
+
+    case TUSB_DMA_INT_SET:
+        s->dma_intr |= value;
+        tusb_dma_intr_update(s);
+        break;
+    case TUSB_DMA_INT_CLEAR:
+        s->dma_intr &= ~value;
+        tusb_dma_intr_update(s);
+        break;
+    case TUSB_DMA_INT_MASK:
+        s->dma_mask = value;
+        tusb_dma_intr_update(s);
+        break;
+
+    case TUSB_GPIO_INT_SET:
+        s->gpio_intr |= value;
+        tusb_gpio_intr_update(s);
+        break;
+    case TUSB_GPIO_INT_CLEAR:
+        s->gpio_intr &= ~value;
+        tusb_gpio_intr_update(s);
+        break;
+    case TUSB_GPIO_INT_MASK:
+        s->gpio_mask = value;
+        tusb_gpio_intr_update(s);
+        break;
+
+    case TUSB_INT_SRC_SET:
+        s->intr |= value;
+        tusb_intr_update(s);
+        break;
+    case TUSB_INT_SRC_CLEAR:
+        s->intr &= ~value;
+        tusb_intr_update(s);
+        break;
+    case TUSB_INT_MASK:
+        s->mask = value;
+        tusb_intr_update(s);
+        break;
+
+    case TUSB_GPIO_CONF:
+        s->gpio_config = value;
+        break;
+    case TUSB_DMA_REQ_CONF:
+        s->dma_config = value;
+        break;
+    case TUSB_EP0_CONF:
+        s->ep0_config = value & 0x1ff;
+        musb_set_size(s->musb, 0, TUSB_EP0_CONFIG_XFR_SIZE(value),
+                        value & TUSB_EP0_CONFIG_DIR_TX);
+        break;
+    case TUSB_EP_IN_SIZE ... (TUSB_EP_IN_SIZE + 0x3b):
+        epnum = (offset - TUSB_EP_IN_SIZE) >> 2;
+        s->tx_config[epnum] = value;
+        musb_set_size(s->musb, epnum + 1, TUSB_EP_CONFIG_XFR_SIZE(value), 1);
+        break;
+    case TUSB_DMA_EP_MAP:
+        s->dma_map = value;
+        break;
+    case TUSB_EP_OUT_SIZE ... (TUSB_EP_OUT_SIZE + 0x3b):
+        epnum = (offset - TUSB_EP_OUT_SIZE) >> 2;
+        s->rx_config[epnum] = value;
+        musb_set_size(s->musb, epnum + 1, TUSB_EP_CONFIG_XFR_SIZE(value), 0);
+        break;
+    case TUSB_EP_MAX_PACKET_SIZE_OFFSET ...
+            (TUSB_EP_MAX_PACKET_SIZE_OFFSET + 0x3b):
+        return;		/* TODO */
+    case TUSB_WAIT_COUNT:
+        return;		/* TODO */
+
+    case TUSB_SCRATCH_PAD:
+        s->scratch = value;
+        break;
+
+    case TUSB_PROD_TEST_RESET:
+        s->test_reset = value;
+        break;
+
+    default:
+        printf("%s: unknown register at %03x\n", __FUNCTION__, offset);
+        return;
+    }
+}
+
+static const MemoryRegionOps tusb_async_ops = {
+    .old_mmio = {
+        .read = { tusb_async_readb, tusb_async_readh, tusb_async_readw, },
+        .write =  { tusb_async_writeb, tusb_async_writeh, tusb_async_writew, },
+    },
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void tusb_otg_tick(void *opaque)
+{
+    TUSBState *s = (TUSBState *) opaque;
+
+    s->otg_timer_val = 0;
+    s->intr |= TUSB_INT_SRC_OTG_TIMEOUT;
+    tusb_intr_update(s);
+}
+
+static void tusb_power_tick(void *opaque)
+{
+    TUSBState *s = (TUSBState *) opaque;
+
+    if (s->power) {
+        s->intr_ok = ~0;
+        tusb_intr_update(s);
+    }
+}
+
+static void tusb_musb_core_intr(void *opaque, int source, int level)
+{
+    TUSBState *s = (TUSBState *) opaque;
+    uint16_t otg_status = s->otg_status;
+
+    switch (source) {
+    case musb_set_vbus:
+        if (level)
+            otg_status |= TUSB_DEV_OTG_STAT_VBUS_VALID;
+        else
+            otg_status &= ~TUSB_DEV_OTG_STAT_VBUS_VALID;
+
+        /* XXX: only if TUSB_PHY_OTG_CTRL_OTG_VBUS_DET_EN set?  */
+        /* XXX: only if TUSB_PRCM_MNGMT_OTG_VBUS_DET_EN set?  */
+        if (s->otg_status != otg_status) {
+            s->otg_status = otg_status;
+            s->intr |= TUSB_INT_SRC_VBUS_SENSE_CHNG;
+            tusb_intr_update(s);
+        }
+        break;
+
+    case musb_set_session:
+        /* XXX: only if TUSB_PHY_OTG_CTRL_OTG_SESS_END_EN set?  */
+        /* XXX: only if TUSB_PRCM_MNGMT_OTG_SESS_END_EN set?  */
+        if (level) {
+            s->otg_status |= TUSB_DEV_OTG_STAT_SESS_VALID;
+            s->otg_status &= ~TUSB_DEV_OTG_STAT_SESS_END;
+        } else {
+            s->otg_status &= ~TUSB_DEV_OTG_STAT_SESS_VALID;
+            s->otg_status |= TUSB_DEV_OTG_STAT_SESS_END;
+        }
+
+        /* XXX: some IRQ or anything?  */
+        break;
+
+    case musb_irq_tx:
+    case musb_irq_rx:
+        s->usbip_intr = musb_core_intr_get(s->musb);
+        /* Fall through.  */
+    default:
+        if (level)
+            s->intr |= 1 << source;
+        else
+            s->intr &= ~(1 << source);
+        tusb_intr_update(s);
+        break;
+    }
+}
+
+static void tusb6010_power(TUSBState *s, int on)
+{
+    if (!on) {
+        s->power = 0;
+    } else if (!s->power && on) {
+        s->power = 1;
+        /* Pull the interrupt down after TUSB6010 comes up.  */
+        s->intr_ok = 0;
+        tusb_intr_update(s);
+        timer_mod(s->pwr_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  NANOSECONDS_PER_SECOND / 2);
+    }
+}
+
+static void tusb6010_irq(void *opaque, int source, int level)
+{
+    if (source) {
+        tusb_musb_core_intr(opaque, source - 1, level);
+    } else {
+        tusb6010_power(opaque, level);
+    }
+}
+
+static void tusb6010_reset(DeviceState *dev)
+{
+    TUSBState *s = TUSB(dev);
+    int i;
+
+    s->test_reset = TUSB_PROD_TEST_RESET_VAL;
+    s->host_mode = 0;
+    s->dev_config = 0;
+    s->otg_status = 0;	/* !TUSB_DEV_OTG_STAT_ID_STATUS means host mode */
+    s->power = 0;
+    s->mask = 0xffffffff;
+    s->intr = 0x00000000;
+    s->otg_timer_val = 0;
+    s->scratch = 0;
+    s->prcm_config = 0;
+    s->prcm_mngmt = 0;
+    s->intr_ok = 0;
+    s->usbip_intr = 0;
+    s->usbip_mask = 0;
+    s->gpio_intr = 0;
+    s->gpio_mask = 0;
+    s->gpio_config = 0;
+    s->dma_intr = 0;
+    s->dma_mask = 0;
+    s->dma_map = 0;
+    s->dma_config = 0;
+    s->ep0_config = 0;
+    s->wkup_mask = 0;
+    s->pullup[0] = s->pullup[1] = 0;
+    s->control_config = 0;
+    for (i = 0; i < 15; i++) {
+        s->rx_config[i] = s->tx_config[i] = 0;
+    }
+    musb_reset(s->musb);
+}
+
+static int tusb6010_init(SysBusDevice *sbd)
+{
+    DeviceState *dev = DEVICE(sbd);
+    TUSBState *s = TUSB(dev);
+
+    s->otg_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, tusb_otg_tick, s);
+    s->pwr_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, tusb_power_tick, s);
+    memory_region_init_io(&s->iomem[1], OBJECT(s), &tusb_async_ops, s,
+                          "tusb-async", UINT32_MAX);
+    sysbus_init_mmio(sbd, &s->iomem[0]);
+    sysbus_init_mmio(sbd, &s->iomem[1]);
+    sysbus_init_irq(sbd, &s->irq);
+    qdev_init_gpio_in(dev, tusb6010_irq, musb_irq_max + 1);
+    s->musb = musb_init(dev, 1);
+    return 0;
+}
+
+static void tusb6010_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = tusb6010_init;
+    dc->reset = tusb6010_reset;
+}
+
+static const TypeInfo tusb6010_info = {
+    .name          = TYPE_TUSB6010,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(TUSBState),
+    .class_init    = tusb6010_class_init,
+};
+
+static void tusb6010_register_types(void)
+{
+    type_register_static(&tusb6010_info);
+}
+
+type_init(tusb6010_register_types)
diff --git a/hw/usb/xen-usb.c b/hw/usb/xen-usb.c
new file mode 100644
index 0000000..174d715
--- /dev/null
+++ b/hw/usb/xen-usb.c
@@ -0,0 +1,1107 @@
+/*
+ *  xen paravirt usb device backend
+ *
+ *  (c) Juergen Gross <jgross@suse.com>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ *  Contributions after 2012-01-13 are licensed under the terms of the
+ *  GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include <libusb.h>
+#include <sys/user.h>
+
+#include "qemu-common.h"
+#include "qemu/config-file.h"
+#include "hw/sysbus.h"
+#include "hw/usb.h"
+#include "hw/xen/xen_backend.h"
+#include "monitor/qdev.h"
+#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/qint.h"
+#include "qapi/qmp/qstring.h"
+
+#include <xen/io/ring.h>
+#include <xen/io/usbif.h>
+
+/*
+ * Check for required support of usbif.h: USBIF_SHORT_NOT_OK was the last
+ * macro added we rely on.
+ */
+#ifdef USBIF_SHORT_NOT_OK
+
+#define TR(xendev, lvl, fmt, args...)                               \
+    {                                                               \
+        struct timeval tv;                                          \
+                                                                    \
+        gettimeofday(&tv, NULL);                                    \
+        xen_be_printf(xendev, lvl, "%8ld.%06ld xen-usb(%s):" fmt,   \
+                      tv.tv_sec, tv.tv_usec, __func__, ##args);     \
+    }
+#define TR_BUS(xendev, fmt, args...) TR(xendev, 2, fmt, ##args)
+#define TR_REQ(xendev, fmt, args...) TR(xendev, 3, fmt, ##args)
+
+#define USBBACK_MAXPORTS        USBIF_PIPE_PORT_MASK
+#define USB_DEV_ADDR_SIZE       (USBIF_PIPE_DEV_MASK + 1)
+
+/* USB wire protocol: structure describing control request parameter. */
+struct usbif_ctrlrequest {
+    uint8_t    bRequestType;
+    uint8_t    bRequest;
+    uint16_t   wValue;
+    uint16_t   wIndex;
+    uint16_t   wLength;
+};
+
+struct usbback_info;
+struct usbback_req;
+
+struct usbback_stub {
+    USBDevice     *dev;
+    USBPort       port;
+    unsigned int  speed;
+    bool          attached;
+    QTAILQ_HEAD(submit_q_head, usbback_req) submit_q;
+};
+
+struct usbback_req {
+    struct usbback_info      *usbif;
+    struct usbback_stub      *stub;
+    struct usbif_urb_request req;
+    USBPacket                packet;
+
+    unsigned int             nr_buffer_segs; /* # of transfer_buffer segments */
+    unsigned int             nr_extra_segs;  /* # of iso_frame_desc segments  */
+
+    QTAILQ_ENTRY(usbback_req) q;
+
+    void                     *buffer;
+    void                     *isoc_buffer;
+    struct libusb_transfer   *xfer;
+
+    bool                     cancelled;
+};
+
+struct usbback_hotplug {
+    QSIMPLEQ_ENTRY(usbback_hotplug) q;
+    unsigned                 port;
+};
+
+struct usbback_info {
+    struct XenDevice         xendev;  /* must be first */
+    USBBus                   bus;
+    void                     *urb_sring;
+    void                     *conn_sring;
+    struct usbif_urb_back_ring urb_ring;
+    struct usbif_conn_back_ring conn_ring;
+    int                      num_ports;
+    int                      usb_ver;
+    bool                     ring_error;
+    QTAILQ_HEAD(req_free_q_head, usbback_req) req_free_q;
+    QSIMPLEQ_HEAD(hotplug_q_head, usbback_hotplug) hotplug_q;
+    struct usbback_stub      ports[USBBACK_MAXPORTS];
+    struct usbback_stub      *addr_table[USB_DEV_ADDR_SIZE];
+    QEMUBH                   *bh;
+};
+
+static struct usbback_req *usbback_get_req(struct usbback_info *usbif)
+{
+    struct usbback_req *usbback_req;
+
+    if (QTAILQ_EMPTY(&usbif->req_free_q)) {
+        usbback_req = g_new0(struct usbback_req, 1);
+    } else {
+        usbback_req = QTAILQ_FIRST(&usbif->req_free_q);
+        QTAILQ_REMOVE(&usbif->req_free_q, usbback_req, q);
+    }
+    return usbback_req;
+}
+
+static void usbback_put_req(struct usbback_req *usbback_req)
+{
+    struct usbback_info *usbif;
+
+    usbif = usbback_req->usbif;
+    memset(usbback_req, 0, sizeof(*usbback_req));
+    QTAILQ_INSERT_HEAD(&usbif->req_free_q, usbback_req, q);
+}
+
+static int usbback_gnttab_map(struct usbback_req *usbback_req)
+{
+    unsigned int nr_segs, i, prot;
+    uint32_t ref[USBIF_MAX_SEGMENTS_PER_REQUEST];
+    struct usbback_info *usbif = usbback_req->usbif;
+    struct XenDevice *xendev = &usbif->xendev;
+    struct usbif_request_segment *seg;
+    void *addr;
+
+    nr_segs = usbback_req->nr_buffer_segs + usbback_req->nr_extra_segs;
+    if (!nr_segs) {
+        return 0;
+    }
+
+    if (nr_segs > USBIF_MAX_SEGMENTS_PER_REQUEST) {
+        xen_be_printf(xendev, 0, "bad number of segments in request (%d)\n",
+                      nr_segs);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < nr_segs; i++) {
+        if ((unsigned)usbback_req->req.seg[i].offset +
+            (unsigned)usbback_req->req.seg[i].length > PAGE_SIZE) {
+            xen_be_printf(xendev, 0, "segment crosses page boundary\n");
+            return -EINVAL;
+        }
+    }
+
+    if (usbback_req->nr_buffer_segs) {
+        prot = PROT_READ;
+        if (usbif_pipein(usbback_req->req.pipe)) {
+                prot |= PROT_WRITE;
+        }
+        for (i = 0; i < usbback_req->nr_buffer_segs; i++) {
+            ref[i] = usbback_req->req.seg[i].gref;
+        }
+        usbback_req->buffer = xengnttab_map_domain_grant_refs(xendev->gnttabdev,
+            usbback_req->nr_buffer_segs, xendev->dom, ref, prot);
+
+        if (!usbback_req->buffer) {
+            return -ENOMEM;
+        }
+
+        for (i = 0; i < usbback_req->nr_buffer_segs; i++) {
+            seg = usbback_req->req.seg + i;
+            addr = usbback_req->buffer + i * PAGE_SIZE + seg->offset;
+            qemu_iovec_add(&usbback_req->packet.iov, addr, seg->length);
+        }
+    }
+
+    if (!usbif_pipeisoc(usbback_req->req.pipe)) {
+        return 0;
+    }
+
+    /*
+     * Right now isoc requests are not supported.
+     * Prepare supporting those by doing the work needed on the guest
+     * interface side.
+     */
+
+    if (!usbback_req->nr_extra_segs) {
+        xen_be_printf(xendev, 0, "iso request without descriptor segments\n");
+        return -EINVAL;
+    }
+
+    prot = PROT_READ | PROT_WRITE;
+    for (i = 0; i < usbback_req->nr_extra_segs; i++) {
+        ref[i] = usbback_req->req.seg[i + usbback_req->req.nr_buffer_segs].gref;
+    }
+    usbback_req->isoc_buffer = xengnttab_map_domain_grant_refs(
+         xendev->gnttabdev, usbback_req->nr_extra_segs, xendev->dom, ref, prot);
+
+    if (!usbback_req->isoc_buffer) {
+        return -ENOMEM;
+    }
+
+    return 0;
+}
+
+static int usbback_init_packet(struct usbback_req *usbback_req)
+{
+    struct XenDevice *xendev = &usbback_req->usbif->xendev;
+    USBPacket *packet = &usbback_req->packet;
+    USBDevice *dev = usbback_req->stub->dev;
+    USBEndpoint *ep;
+    unsigned int pid, ep_nr;
+    bool sok;
+    int ret = 0;
+
+    qemu_iovec_init(&packet->iov, USBIF_MAX_SEGMENTS_PER_REQUEST);
+    pid = usbif_pipein(usbback_req->req.pipe) ? USB_TOKEN_IN : USB_TOKEN_OUT;
+    ep_nr = usbif_pipeendpoint(usbback_req->req.pipe);
+    sok = !!(usbback_req->req.transfer_flags & USBIF_SHORT_NOT_OK);
+    if (usbif_pipectrl(usbback_req->req.pipe)) {
+        ep_nr = 0;
+        sok = false;
+    }
+    ep = usb_ep_get(dev, pid, ep_nr);
+    usb_packet_setup(packet, pid, ep, 0, 1, sok, true);
+
+    switch (usbif_pipetype(usbback_req->req.pipe)) {
+    case USBIF_PIPE_TYPE_ISOC:
+        TR_REQ(xendev, "iso transfer %s: buflen: %x, %d frames\n",
+               (pid == USB_TOKEN_IN) ? "in" : "out",
+               usbback_req->req.buffer_length,
+               usbback_req->req.u.isoc.nr_frame_desc_segs);
+        ret = -EINVAL;  /* isoc not implemented yet */
+        break;
+
+    case USBIF_PIPE_TYPE_INT:
+        TR_REQ(xendev, "int transfer %s: buflen: %x\n",
+               (pid == USB_TOKEN_IN) ? "in" : "out",
+               usbback_req->req.buffer_length);
+        break;
+
+    case USBIF_PIPE_TYPE_CTRL:
+        packet->parameter = *(uint64_t *)usbback_req->req.u.ctrl;
+        TR_REQ(xendev, "ctrl parameter: %"PRIx64", buflen: %x\n",
+               packet->parameter,
+               usbback_req->req.buffer_length);
+        break;
+
+    case USBIF_PIPE_TYPE_BULK:
+        TR_REQ(xendev, "bulk transfer %s: buflen: %x\n",
+               (pid == USB_TOKEN_IN) ? "in" : "out",
+               usbback_req->req.buffer_length);
+        break;
+    default:
+        ret = -EINVAL;
+        break;
+    }
+
+    return ret;
+}
+
+static void usbback_do_response(struct usbback_req *usbback_req, int32_t status,
+                                int32_t actual_length, int32_t error_count)
+{
+    struct usbback_info *usbif;
+    struct usbif_urb_response *res;
+    struct XenDevice *xendev;
+    unsigned int notify;
+
+    usbif = usbback_req->usbif;
+    xendev = &usbif->xendev;
+
+    TR_REQ(xendev, "id %d, status %d, length %d, errcnt %d\n",
+           usbback_req->req.id, status, actual_length, error_count);
+
+    if (usbback_req->packet.iov.iov) {
+        qemu_iovec_destroy(&usbback_req->packet.iov);
+    }
+
+    if (usbback_req->buffer) {
+        xengnttab_unmap(xendev->gnttabdev, usbback_req->buffer,
+                        usbback_req->nr_buffer_segs);
+        usbback_req->buffer = NULL;
+    }
+
+    if (usbback_req->isoc_buffer) {
+        xengnttab_unmap(xendev->gnttabdev, usbback_req->isoc_buffer,
+                        usbback_req->nr_extra_segs);
+        usbback_req->isoc_buffer = NULL;
+    }
+
+    if (usbif->urb_sring) {
+        res = RING_GET_RESPONSE(&usbif->urb_ring, usbif->urb_ring.rsp_prod_pvt);
+        res->id = usbback_req->req.id;
+        res->status = status;
+        res->actual_length = actual_length;
+        res->error_count = error_count;
+        res->start_frame = 0;
+        usbif->urb_ring.rsp_prod_pvt++;
+        RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&usbif->urb_ring, notify);
+
+        if (notify) {
+            xen_be_send_notify(xendev);
+        }
+    }
+
+    if (!usbback_req->cancelled)
+        usbback_put_req(usbback_req);
+}
+
+static void usbback_do_response_ret(struct usbback_req *usbback_req,
+                                    int32_t status)
+{
+    usbback_do_response(usbback_req, status, 0, 0);
+}
+
+static int32_t usbback_xlat_status(int status)
+{
+    switch (status) {
+    case USB_RET_SUCCESS:
+        return 0;
+    case USB_RET_NODEV:
+        return -ENODEV;
+    case USB_RET_STALL:
+        return -EPIPE;
+    case USB_RET_BABBLE:
+        return -EOVERFLOW;
+    case USB_RET_IOERROR:
+        return -EPROTO;
+    }
+
+    return -ESHUTDOWN;
+}
+
+static void usbback_packet_complete(USBPacket *packet)
+{
+    struct usbback_req *usbback_req;
+    int32_t status;
+
+    usbback_req = container_of(packet, struct usbback_req, packet);
+
+    QTAILQ_REMOVE(&usbback_req->stub->submit_q, usbback_req, q);
+
+    status = usbback_xlat_status(packet->status);
+    usbback_do_response(usbback_req, status, packet->actual_length, 0);
+}
+
+static void usbback_set_address(struct usbback_info *usbif,
+                                struct usbback_stub *stub,
+                                unsigned int cur_addr, unsigned int new_addr)
+{
+    if (cur_addr) {
+        usbif->addr_table[cur_addr] = NULL;
+    }
+    if (new_addr) {
+        usbif->addr_table[new_addr] = stub;
+    }
+}
+
+static void usbback_cancel_req(struct usbback_req *usbback_req)
+{
+    if (usb_packet_is_inflight(&usbback_req->packet)) {
+        usb_cancel_packet(&usbback_req->packet);
+        QTAILQ_REMOVE(&usbback_req->stub->submit_q, usbback_req, q);
+        usbback_req->cancelled = true;
+        usbback_do_response_ret(usbback_req, -EPROTO);
+    }
+}
+
+static void usbback_process_unlink_req(struct usbback_req *usbback_req)
+{
+    struct usbback_info *usbif;
+    struct usbback_req *unlink_req;
+    unsigned int id, devnum;
+    int ret;
+
+    usbif = usbback_req->usbif;
+    ret = 0;
+    id = usbback_req->req.u.unlink.unlink_id;
+    TR_REQ(&usbif->xendev, "unlink id %d\n", id);
+    devnum = usbif_pipedevice(usbback_req->req.pipe);
+    if (unlikely(devnum == 0)) {
+        usbback_req->stub = usbif->ports +
+                            usbif_pipeportnum(usbback_req->req.pipe) - 1;
+        if (unlikely(!usbback_req->stub)) {
+            ret = -ENODEV;
+            goto fail_response;
+        }
+    } else {
+        if (unlikely(!usbif->addr_table[devnum])) {
+            ret = -ENODEV;
+            goto fail_response;
+        }
+        usbback_req->stub = usbif->addr_table[devnum];
+    }
+
+    QTAILQ_FOREACH(unlink_req, &usbback_req->stub->submit_q, q) {
+        if (unlink_req->req.id == id) {
+            usbback_cancel_req(unlink_req);
+            break;
+        }
+    }
+
+fail_response:
+    usbback_do_response_ret(usbback_req, ret);
+}
+
+/*
+ * Checks whether a request can be handled at once or should be forwarded
+ * to the usb framework.
+ * Return value is:
+ * 0 in case of usb framework is needed
+ * 1 in case of local handling (no error)
+ * The request response has been queued already if return value not 0.
+ */
+static int usbback_check_and_submit(struct usbback_req *usbback_req)
+{
+    struct usbback_info *usbif;
+    unsigned int devnum;
+    struct usbback_stub *stub;
+    struct usbif_ctrlrequest *ctrl;
+    int ret;
+    uint16_t wValue;
+
+    usbif = usbback_req->usbif;
+    stub = NULL;
+    devnum = usbif_pipedevice(usbback_req->req.pipe);
+    ctrl = (struct usbif_ctrlrequest *)usbback_req->req.u.ctrl;
+    wValue = le16_to_cpu(ctrl->wValue);
+
+    /*
+     * When the device is first connected or resetted, USB device has no
+     * address. In this initial state, following requests are sent to device
+     * address (#0),
+     *
+     *  1. GET_DESCRIPTOR (with Descriptor Type is "DEVICE") is sent,
+     *     and OS knows what device is connected to.
+     *
+     *  2. SET_ADDRESS is sent, and then device has its address.
+     *
+     * In the next step, SET_CONFIGURATION is sent to addressed device, and
+     * then the device is finally ready to use.
+     */
+    if (unlikely(devnum == 0)) {
+        stub = usbif->ports + usbif_pipeportnum(usbback_req->req.pipe) - 1;
+        if (!stub->dev || !stub->attached) {
+            ret = -ENODEV;
+            goto do_response;
+        }
+
+        switch (ctrl->bRequest) {
+        case USB_REQ_GET_DESCRIPTOR:
+            /*
+             * GET_DESCRIPTOR request to device #0.
+             * through normal transfer.
+             */
+            TR_REQ(&usbif->xendev, "devnum 0 GET_DESCRIPTOR\n");
+            usbback_req->stub = stub;
+            return 0;
+        case USB_REQ_SET_ADDRESS:
+            /*
+             * SET_ADDRESS request to device #0.
+             * add attached device to addr_table.
+             */
+            TR_REQ(&usbif->xendev, "devnum 0 SET_ADDRESS\n");
+            usbback_set_address(usbif, stub, 0, wValue);
+            ret = 0;
+            break;
+        default:
+            ret = -EINVAL;
+            break;
+        }
+        goto do_response;
+    }
+
+    if (unlikely(!usbif->addr_table[devnum])) {
+            ret = -ENODEV;
+            goto do_response;
+    }
+    usbback_req->stub = usbif->addr_table[devnum];
+
+    /*
+     * Check special request
+     */
+    if (ctrl->bRequest != USB_REQ_SET_ADDRESS) {
+        return 0;
+    }
+
+    /*
+     * SET_ADDRESS request to addressed device.
+     * change addr or remove from addr_table.
+     */
+    usbback_set_address(usbif, usbback_req->stub, devnum, wValue);
+    ret = 0;
+
+do_response:
+    usbback_do_response_ret(usbback_req, ret);
+    return 1;
+}
+
+static void usbback_dispatch(struct usbback_req *usbback_req)
+{
+    int ret;
+    unsigned int devnum;
+    struct usbback_info *usbif;
+
+    usbif = usbback_req->usbif;
+
+    TR_REQ(&usbif->xendev, "start req_id %d pipe %08x\n", usbback_req->req.id,
+           usbback_req->req.pipe);
+
+    /* unlink request */
+    if (unlikely(usbif_pipeunlink(usbback_req->req.pipe))) {
+        usbback_process_unlink_req(usbback_req);
+        return;
+    }
+
+    if (usbif_pipectrl(usbback_req->req.pipe)) {
+        if (usbback_check_and_submit(usbback_req)) {
+            return;
+        }
+    } else {
+        devnum = usbif_pipedevice(usbback_req->req.pipe);
+        usbback_req->stub = usbif->addr_table[devnum];
+
+        if (!usbback_req->stub || !usbback_req->stub->attached) {
+            ret = -ENODEV;
+            goto fail_response;
+        }
+    }
+
+    QTAILQ_INSERT_TAIL(&usbback_req->stub->submit_q, usbback_req, q);
+
+    usbback_req->nr_buffer_segs = usbback_req->req.nr_buffer_segs;
+    usbback_req->nr_extra_segs = usbif_pipeisoc(usbback_req->req.pipe) ?
+                                 usbback_req->req.u.isoc.nr_frame_desc_segs : 0;
+
+    ret = usbback_init_packet(usbback_req);
+    if (ret) {
+        xen_be_printf(&usbif->xendev, 0, "invalid request\n");
+        ret = -ESHUTDOWN;
+        goto fail_free_urb;
+    }
+
+    ret = usbback_gnttab_map(usbback_req);
+    if (ret) {
+        xen_be_printf(&usbif->xendev, 0, "invalid buffer, ret=%d\n", ret);
+        ret = -ESHUTDOWN;
+        goto fail_free_urb;
+    }
+
+    usb_handle_packet(usbback_req->stub->dev, &usbback_req->packet);
+    if (usbback_req->packet.status != USB_RET_ASYNC) {
+        usbback_packet_complete(&usbback_req->packet);
+    }
+    return;
+
+fail_free_urb:
+    QTAILQ_REMOVE(&usbback_req->stub->submit_q, usbback_req, q);
+
+fail_response:
+    usbback_do_response_ret(usbback_req, ret);
+}
+
+static void usbback_hotplug_notify(struct usbback_info *usbif)
+{
+    struct usbif_conn_back_ring *ring = &usbif->conn_ring;
+    struct usbif_conn_request req;
+    struct usbif_conn_response *res;
+    struct usbback_hotplug *usb_hp;
+    unsigned int notify;
+
+    if (!usbif->conn_sring) {
+        return;
+    }
+
+    /* Check for full ring. */
+    if ((RING_SIZE(ring) - ring->rsp_prod_pvt - ring->req_cons) == 0) {
+        xen_be_send_notify(&usbif->xendev);
+        return;
+    }
+
+    usb_hp = QSIMPLEQ_FIRST(&usbif->hotplug_q);
+    QSIMPLEQ_REMOVE_HEAD(&usbif->hotplug_q, q);
+
+    RING_COPY_REQUEST(ring, ring->req_cons, &req);
+    ring->req_cons++;
+    ring->sring->req_event = ring->req_cons + 1;
+
+    res = RING_GET_RESPONSE(ring, ring->rsp_prod_pvt);
+    res->id = req.id;
+    res->portnum = usb_hp->port;
+    res->speed = usbif->ports[usb_hp->port - 1].speed;
+    ring->rsp_prod_pvt++;
+    RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(ring, notify);
+
+    if (notify) {
+        xen_be_send_notify(&usbif->xendev);
+    }
+
+    TR_BUS(&usbif->xendev, "hotplug port %d speed %d\n", usb_hp->port,
+           res->speed);
+
+    g_free(usb_hp);
+
+    if (!QSIMPLEQ_EMPTY(&usbif->hotplug_q)) {
+        qemu_bh_schedule(usbif->bh);
+    }
+}
+
+static void usbback_bh(void *opaque)
+{
+    struct usbback_info *usbif;
+    struct usbif_urb_back_ring *urb_ring;
+    struct usbback_req *usbback_req;
+    RING_IDX rc, rp;
+    unsigned int more_to_do;
+
+    usbif = opaque;
+    if (usbif->ring_error) {
+        return;
+    }
+
+    if (!QSIMPLEQ_EMPTY(&usbif->hotplug_q)) {
+        usbback_hotplug_notify(usbif);
+    }
+
+    urb_ring = &usbif->urb_ring;
+    rc = urb_ring->req_cons;
+    rp = urb_ring->sring->req_prod;
+    xen_rmb(); /* Ensure we see queued requests up to 'rp'. */
+
+    if (RING_REQUEST_PROD_OVERFLOW(urb_ring, rp)) {
+        rc = urb_ring->rsp_prod_pvt;
+        xen_be_printf(&usbif->xendev, 0, "domU provided bogus ring requests "
+                      "(%#x - %#x = %u). Halting ring processing.\n",
+                      rp, rc, rp - rc);
+        usbif->ring_error = true;
+        return;
+    }
+
+    while (rc != rp) {
+        if (RING_REQUEST_CONS_OVERFLOW(urb_ring, rc)) {
+            break;
+        }
+        usbback_req = usbback_get_req(usbif);
+
+        RING_COPY_REQUEST(urb_ring, rc, &usbback_req->req);
+        usbback_req->usbif = usbif;
+
+        usbback_dispatch(usbback_req);
+
+        urb_ring->req_cons = ++rc;
+    }
+
+    RING_FINAL_CHECK_FOR_REQUESTS(urb_ring, more_to_do);
+    if (more_to_do) {
+        qemu_bh_schedule(usbif->bh);
+    }
+}
+
+static void usbback_hotplug_enq(struct usbback_info *usbif, unsigned port)
+{
+    struct usbback_hotplug *usb_hp;
+
+    usb_hp = g_new0(struct usbback_hotplug, 1);
+    usb_hp->port = port;
+    QSIMPLEQ_INSERT_TAIL(&usbif->hotplug_q, usb_hp, q);
+    usbback_hotplug_notify(usbif);
+}
+
+static void usbback_portid_drain(struct usbback_info *usbif, unsigned port)
+{
+    struct usbback_req *req, *tmp;
+    bool sched = false;
+
+    QTAILQ_FOREACH_SAFE(req, &usbif->ports[port - 1].submit_q, q, tmp) {
+        usbback_cancel_req(req);
+        sched = true;
+    }
+
+    if (sched) {
+        qemu_bh_schedule(usbif->bh);
+    }
+}
+
+static void usbback_portid_detach(struct usbback_info *usbif, unsigned port)
+{
+    if (!usbif->ports[port - 1].attached) {
+        return;
+    }
+
+    usbif->ports[port - 1].speed = USBIF_SPEED_NONE;
+    usbif->ports[port - 1].attached = false;
+    usbback_portid_drain(usbif, port);
+    usbback_hotplug_enq(usbif, port);
+}
+
+static void usbback_portid_remove(struct usbback_info *usbif, unsigned port)
+{
+    USBPort *p;
+
+    if (!usbif->ports[port - 1].dev) {
+        return;
+    }
+
+    p = &(usbif->ports[port - 1].port);
+    snprintf(p->path, sizeof(p->path), "%d", 99);
+
+    object_unparent(OBJECT(usbif->ports[port - 1].dev));
+    usbif->ports[port - 1].dev = NULL;
+    usbback_portid_detach(usbif, port);
+
+    TR_BUS(&usbif->xendev, "port %d removed\n", port);
+}
+
+static void usbback_portid_add(struct usbback_info *usbif, unsigned port,
+                               char *busid)
+{
+    unsigned speed;
+    char *portname;
+    USBPort *p;
+    Error *local_err = NULL;
+    QDict *qdict;
+    QemuOpts *opts;
+
+    if (usbif->ports[port - 1].dev) {
+        return;
+    }
+
+    portname = strchr(busid, '-');
+    if (!portname) {
+        xen_be_printf(&usbif->xendev, 0, "device %s illegal specification\n",
+                      busid);
+        return;
+    }
+    portname++;
+    p = &(usbif->ports[port - 1].port);
+    snprintf(p->path, sizeof(p->path), "%s", portname);
+
+    qdict = qdict_new();
+    qdict_put(qdict, "driver", qstring_from_str("usb-host"));
+    qdict_put(qdict, "hostbus", qint_from_int(atoi(busid)));
+    qdict_put(qdict, "hostport", qstring_from_str(portname));
+    opts = qemu_opts_from_qdict(qemu_find_opts("device"), qdict, &local_err);
+    if (local_err) {
+        goto err;
+    }
+    usbif->ports[port - 1].dev = USB_DEVICE(qdev_device_add(opts, &local_err));
+    if (!usbif->ports[port - 1].dev) {
+        goto err;
+    }
+    QDECREF(qdict);
+    snprintf(p->path, sizeof(p->path), "%d", port);
+    speed = usbif->ports[port - 1].dev->speed;
+    switch (speed) {
+    case USB_SPEED_LOW:
+        speed = USBIF_SPEED_LOW;
+        break;
+    case USB_SPEED_FULL:
+        speed = USBIF_SPEED_FULL;
+        break;
+    case USB_SPEED_HIGH:
+        speed = (usbif->usb_ver < USB_VER_USB20) ?
+                USBIF_SPEED_NONE : USBIF_SPEED_HIGH;
+        break;
+    default:
+        speed = USBIF_SPEED_NONE;
+        break;
+    }
+    if (speed == USBIF_SPEED_NONE) {
+        xen_be_printf(&usbif->xendev, 0, "device %s wrong speed\n", busid);
+        object_unparent(OBJECT(usbif->ports[port - 1].dev));
+        usbif->ports[port - 1].dev = NULL;
+        return;
+    }
+    usb_device_reset(usbif->ports[port - 1].dev);
+    usbif->ports[port - 1].speed = speed;
+    usbif->ports[port - 1].attached = true;
+    QTAILQ_INIT(&usbif->ports[port - 1].submit_q);
+    usbback_hotplug_enq(usbif, port);
+
+    TR_BUS(&usbif->xendev, "port %d attached\n", port);
+    return;
+
+err:
+    QDECREF(qdict);
+    snprintf(p->path, sizeof(p->path), "%d", 99);
+    xen_be_printf(&usbif->xendev, 0, "device %s could not be opened\n", busid);
+}
+
+static void usbback_process_port(struct usbback_info *usbif, unsigned port)
+{
+    char node[8];
+    char *busid;
+
+    snprintf(node, sizeof(node), "port/%d", port);
+    busid = xenstore_read_be_str(&usbif->xendev, node);
+    if (busid == NULL) {
+        xen_be_printf(&usbif->xendev, 0, "xenstore_read %s failed\n", node);
+        return;
+    }
+
+    /* Remove portid, if the port is not connected.  */
+    if (strlen(busid) == 0) {
+        usbback_portid_remove(usbif, port);
+    } else {
+        usbback_portid_add(usbif, port, busid);
+    }
+
+    g_free(busid);
+}
+
+static void usbback_disconnect(struct XenDevice *xendev)
+{
+    struct usbback_info *usbif;
+    unsigned int i;
+
+    TR_BUS(xendev, "start\n");
+
+    usbif = container_of(xendev, struct usbback_info, xendev);
+
+    xen_be_unbind_evtchn(xendev);
+
+    if (usbif->urb_sring) {
+        xengnttab_unmap(xendev->gnttabdev, usbif->urb_sring, 1);
+        usbif->urb_sring = NULL;
+    }
+    if (usbif->conn_sring) {
+        xengnttab_unmap(xendev->gnttabdev, usbif->conn_sring, 1);
+        usbif->conn_sring = NULL;
+    }
+
+    for (i = 0; i < usbif->num_ports; i++) {
+        if (usbif->ports[i].dev) {
+            usbback_portid_drain(usbif, i + 1);
+        }
+    }
+
+    TR_BUS(xendev, "finished\n");
+}
+
+static int usbback_connect(struct XenDevice *xendev)
+{
+    struct usbback_info *usbif;
+    struct usbif_urb_sring *urb_sring;
+    struct usbif_conn_sring *conn_sring;
+    int urb_ring_ref;
+    int conn_ring_ref;
+    unsigned int i;
+
+    TR_BUS(xendev, "start\n");
+
+    usbif = container_of(xendev, struct usbback_info, xendev);
+
+    if (xenstore_read_fe_int(xendev, "urb-ring-ref", &urb_ring_ref)) {
+        xen_be_printf(xendev, 0, "error reading urb-ring-ref\n");
+        return -1;
+    }
+    if (xenstore_read_fe_int(xendev, "conn-ring-ref", &conn_ring_ref)) {
+        xen_be_printf(xendev, 0, "error reading conn-ring-ref\n");
+        return -1;
+    }
+    if (xenstore_read_fe_int(xendev, "event-channel", &xendev->remote_port)) {
+        xen_be_printf(xendev, 0, "error reading event-channel\n");
+        return -1;
+    }
+
+    usbif->urb_sring = xengnttab_map_grant_ref(xendev->gnttabdev, xendev->dom,
+                                               urb_ring_ref,
+                                               PROT_READ | PROT_WRITE);
+    usbif->conn_sring = xengnttab_map_grant_ref(xendev->gnttabdev, xendev->dom,
+                                                conn_ring_ref,
+                                                PROT_READ | PROT_WRITE);
+    if (!usbif->urb_sring || !usbif->conn_sring) {
+        xen_be_printf(xendev, 0, "error mapping rings\n");
+        usbback_disconnect(xendev);
+        return -1;
+    }
+
+    urb_sring = usbif->urb_sring;
+    conn_sring = usbif->conn_sring;
+    BACK_RING_INIT(&usbif->urb_ring, urb_sring, XC_PAGE_SIZE);
+    BACK_RING_INIT(&usbif->conn_ring, conn_sring, XC_PAGE_SIZE);
+
+    xen_be_bind_evtchn(xendev);
+
+    xen_be_printf(xendev, 1, "urb-ring-ref %d, conn-ring-ref %d, "
+                  "remote port %d, local port %d\n", urb_ring_ref,
+                  conn_ring_ref, xendev->remote_port, xendev->local_port);
+
+    for (i = 1; i <= usbif->num_ports; i++) {
+        if (usbif->ports[i - 1].dev) {
+            usbback_hotplug_enq(usbif, i);
+        }
+    }
+
+    return 0;
+}
+
+static void usbback_backend_changed(struct XenDevice *xendev, const char *node)
+{
+    struct usbback_info *usbif;
+    unsigned int i;
+
+    TR_BUS(xendev, "path %s\n", node);
+
+    usbif = container_of(xendev, struct usbback_info, xendev);
+    for (i = 1; i <= usbif->num_ports; i++) {
+        usbback_process_port(usbif, i);
+    }
+}
+
+static int usbback_init(struct XenDevice *xendev)
+{
+    struct usbback_info *usbif;
+
+    TR_BUS(xendev, "start\n");
+
+    usbif = container_of(xendev, struct usbback_info, xendev);
+
+    if (xenstore_read_be_int(xendev, "num-ports", &usbif->num_ports) ||
+        usbif->num_ports < 1 || usbif->num_ports > USBBACK_MAXPORTS) {
+        xen_be_printf(xendev, 0, "num-ports not readable or out of bounds\n");
+        return -1;
+    }
+    if (xenstore_read_be_int(xendev, "usb-ver", &usbif->usb_ver) ||
+        (usbif->usb_ver != USB_VER_USB11 && usbif->usb_ver != USB_VER_USB20)) {
+        xen_be_printf(xendev, 0, "usb-ver not readable or out of bounds\n");
+        return -1;
+    }
+
+    usbback_backend_changed(xendev, "port");
+
+    TR_BUS(xendev, "finished\n");
+
+    return 0;
+}
+
+static void xen_bus_attach(USBPort *port)
+{
+    struct usbback_info *usbif;
+
+    usbif = port->opaque;
+    TR_BUS(&usbif->xendev, "\n");
+    usbif->ports[port->index].attached = true;
+    usbback_hotplug_enq(usbif, port->index + 1);
+}
+
+static void xen_bus_detach(USBPort *port)
+{
+    struct usbback_info *usbif;
+
+    usbif = port->opaque;
+    TR_BUS(&usbif->xendev, "\n");
+    usbback_portid_detach(usbif, port->index + 1);
+}
+
+static void xen_bus_child_detach(USBPort *port, USBDevice *child)
+{
+    struct usbback_info *usbif;
+
+    usbif = port->opaque;
+    TR_BUS(&usbif->xendev, "\n");
+}
+
+static void xen_bus_complete(USBPort *port, USBPacket *packet)
+{
+    struct usbback_req *usbback_req;
+    struct usbback_info *usbif;
+
+    usbback_req = container_of(packet, struct usbback_req, packet);
+    if (usbback_req->cancelled) {
+        g_free(usbback_req);
+        return;
+    }
+
+    usbif = usbback_req->usbif;
+    TR_REQ(&usbif->xendev, "\n");
+    usbback_packet_complete(packet);
+}
+
+static USBPortOps xen_usb_port_ops = {
+    .attach = xen_bus_attach,
+    .detach = xen_bus_detach,
+    .child_detach = xen_bus_child_detach,
+    .complete = xen_bus_complete,
+};
+
+static USBBusOps xen_usb_bus_ops = {
+};
+
+static void usbback_alloc(struct XenDevice *xendev)
+{
+    struct usbback_info *usbif;
+    USBPort *p;
+    unsigned int i, max_grants;
+
+    usbif = container_of(xendev, struct usbback_info, xendev);
+
+    usb_bus_new(&usbif->bus, sizeof(usbif->bus), &xen_usb_bus_ops, xen_sysdev);
+    for (i = 0; i < USBBACK_MAXPORTS; i++) {
+        p = &(usbif->ports[i].port);
+        usb_register_port(&usbif->bus, p, usbif, i, &xen_usb_port_ops,
+                          USB_SPEED_MASK_LOW | USB_SPEED_MASK_FULL |
+                          USB_SPEED_MASK_HIGH);
+        snprintf(p->path, sizeof(p->path), "%d", 99);
+    }
+
+    QTAILQ_INIT(&usbif->req_free_q);
+    QSIMPLEQ_INIT(&usbif->hotplug_q);
+    usbif->bh = qemu_bh_new(usbback_bh, usbif);
+
+    /* max_grants: for each request and for the rings (request and connect). */
+    max_grants = USBIF_MAX_SEGMENTS_PER_REQUEST * USB_URB_RING_SIZE + 2;
+    if (xengnttab_set_max_grants(xendev->gnttabdev, max_grants) < 0) {
+        xen_be_printf(xendev, 0, "xengnttab_set_max_grants failed: %s\n",
+                      strerror(errno));
+    }
+}
+
+static int usbback_free(struct XenDevice *xendev)
+{
+    struct usbback_info *usbif;
+    struct usbback_req *usbback_req;
+    struct usbback_hotplug *usb_hp;
+    unsigned int i;
+
+    TR_BUS(xendev, "start\n");
+
+    usbback_disconnect(xendev);
+    usbif = container_of(xendev, struct usbback_info, xendev);
+    for (i = 1; i <= usbif->num_ports; i++) {
+        usbback_portid_remove(usbif, i);
+    }
+
+    while (!QTAILQ_EMPTY(&usbif->req_free_q)) {
+        usbback_req = QTAILQ_FIRST(&usbif->req_free_q);
+        QTAILQ_REMOVE(&usbif->req_free_q, usbback_req, q);
+        g_free(usbback_req);
+    }
+    while (!QSIMPLEQ_EMPTY(&usbif->hotplug_q)) {
+        usb_hp = QSIMPLEQ_FIRST(&usbif->hotplug_q);
+        QSIMPLEQ_REMOVE_HEAD(&usbif->hotplug_q, q);
+        g_free(usb_hp);
+    }
+
+    qemu_bh_delete(usbif->bh);
+
+    for (i = 0; i < USBBACK_MAXPORTS; i++) {
+        usb_unregister_port(&usbif->bus, &(usbif->ports[i].port));
+    }
+
+    usb_bus_release(&usbif->bus);
+    object_unparent(OBJECT(&usbif->bus));
+
+    TR_BUS(xendev, "finished\n");
+
+    return 0;
+}
+
+static void usbback_event(struct XenDevice *xendev)
+{
+    struct usbback_info *usbif;
+
+    usbif = container_of(xendev, struct usbback_info, xendev);
+    qemu_bh_schedule(usbif->bh);
+}
+
+struct XenDevOps xen_usb_ops = {
+    .size            = sizeof(struct usbback_info),
+    .flags           = DEVOPS_FLAG_NEED_GNTDEV,
+    .init            = usbback_init,
+    .alloc           = usbback_alloc,
+    .free            = usbback_free,
+    .backend_changed = usbback_backend_changed,
+    .initialise      = usbback_connect,
+    .disconnect      = usbback_disconnect,
+    .event           = usbback_event,
+};
+
+#else /* USBIF_SHORT_NOT_OK */
+
+static int usbback_not_supported(void)
+{
+    return -EINVAL;
+}
+
+struct XenDevOps xen_usb_ops = {
+    .backend_register = usbback_not_supported,
+};
+
+#endif
diff --git a/hw/vfio/Makefile.objs b/hw/vfio/Makefile.objs
new file mode 100644
index 0000000..c25e32b
--- /dev/null
+++ b/hw/vfio/Makefile.objs
@@ -0,0 +1,8 @@
+ifeq ($(CONFIG_LINUX), y)
+obj-$(CONFIG_SOFTMMU) += common.o
+obj-$(CONFIG_PCI) += pci.o pci-quirks.o
+obj-$(CONFIG_SOFTMMU) += platform.o
+obj-$(CONFIG_SOFTMMU) += calxeda-xgmac.o
+obj-$(CONFIG_SOFTMMU) += amd-xgbe.o
+obj-$(CONFIG_SOFTMMU) += spapr.o
+endif
diff --git a/hw/vfio/amd-xgbe.c b/hw/vfio/amd-xgbe.c
new file mode 100644
index 0000000..2c60310
--- /dev/null
+++ b/hw/vfio/amd-xgbe.c
@@ -0,0 +1,56 @@
+/*
+ * AMD XGBE VFIO device
+ *
+ * Copyright Linaro Limited, 2015
+ *
+ * Authors:
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-amd-xgbe.h"
+
+static void amd_xgbe_realize(DeviceState *dev, Error **errp)
+{
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(dev);
+    VFIOAmdXgbeDeviceClass *k = VFIO_AMD_XGBE_DEVICE_GET_CLASS(dev);
+
+    vdev->compat = g_strdup("amd,xgbe-seattle-v1a");
+
+    k->parent_realize(dev, errp);
+}
+
+static const VMStateDescription vfio_platform_amd_xgbe_vmstate = {
+    .name = TYPE_VFIO_AMD_XGBE,
+    .unmigratable = 1,
+};
+
+static void vfio_amd_xgbe_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VFIOAmdXgbeDeviceClass *vcxc =
+        VFIO_AMD_XGBE_DEVICE_CLASS(klass);
+    vcxc->parent_realize = dc->realize;
+    dc->realize = amd_xgbe_realize;
+    dc->desc = "VFIO AMD XGBE";
+    dc->vmsd = &vfio_platform_amd_xgbe_vmstate;
+}
+
+static const TypeInfo vfio_amd_xgbe_dev_info = {
+    .name = TYPE_VFIO_AMD_XGBE,
+    .parent = TYPE_VFIO_PLATFORM,
+    .instance_size = sizeof(VFIOAmdXgbeDevice),
+    .class_init = vfio_amd_xgbe_class_init,
+    .class_size = sizeof(VFIOAmdXgbeDeviceClass),
+};
+
+static void register_amd_xgbe_dev_type(void)
+{
+    type_register_static(&vfio_amd_xgbe_dev_info);
+}
+
+type_init(register_amd_xgbe_dev_type)
diff --git a/hw/vfio/calxeda-xgmac.c b/hw/vfio/calxeda-xgmac.c
new file mode 100644
index 0000000..bb15d58
--- /dev/null
+++ b/hw/vfio/calxeda-xgmac.c
@@ -0,0 +1,56 @@
+/*
+ * calxeda xgmac VFIO device
+ *
+ * Copyright Linaro Limited, 2014
+ *
+ * Authors:
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hw/vfio/vfio-calxeda-xgmac.h"
+
+static void calxeda_xgmac_realize(DeviceState *dev, Error **errp)
+{
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(dev);
+    VFIOCalxedaXgmacDeviceClass *k = VFIO_CALXEDA_XGMAC_DEVICE_GET_CLASS(dev);
+
+    vdev->compat = g_strdup("calxeda,hb-xgmac");
+
+    k->parent_realize(dev, errp);
+}
+
+static const VMStateDescription vfio_platform_calxeda_xgmac_vmstate = {
+    .name = TYPE_VFIO_CALXEDA_XGMAC,
+    .unmigratable = 1,
+};
+
+static void vfio_calxeda_xgmac_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VFIOCalxedaXgmacDeviceClass *vcxc =
+        VFIO_CALXEDA_XGMAC_DEVICE_CLASS(klass);
+    vcxc->parent_realize = dc->realize;
+    dc->realize = calxeda_xgmac_realize;
+    dc->desc = "VFIO Calxeda XGMAC";
+    dc->vmsd = &vfio_platform_calxeda_xgmac_vmstate;
+}
+
+static const TypeInfo vfio_calxeda_xgmac_dev_info = {
+    .name = TYPE_VFIO_CALXEDA_XGMAC,
+    .parent = TYPE_VFIO_PLATFORM,
+    .instance_size = sizeof(VFIOCalxedaXgmacDevice),
+    .class_init = vfio_calxeda_xgmac_class_init,
+    .class_size = sizeof(VFIOCalxedaXgmacDeviceClass),
+};
+
+static void register_calxeda_xgmac_dev_type(void)
+{
+    type_register_static(&vfio_calxeda_xgmac_dev_info);
+}
+
+type_init(register_calxeda_xgmac_dev_type)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
new file mode 100644
index 0000000..b313e7c
--- /dev/null
+++ b/hw/vfio/common.c
@@ -0,0 +1,1406 @@
+/*
+ * generic functions used by VFIO devices
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on qemu-kvm device-assignment:
+ *  Adapted for KVM by Qumranet.
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+
+#include "qemu/osdep.h"
+#include <sys/ioctl.h>
+#ifdef CONFIG_KVM
+#include <linux/kvm.h>
+#endif
+#include <linux/vfio.h>
+
+#include "hw/vfio/vfio-common.h"
+#include "hw/vfio/vfio.h"
+#include "exec/address-spaces.h"
+#include "exec/memory.h"
+#include "hw/hw.h"
+#include "qemu/error-report.h"
+#include "qemu/range.h"
+#include "sysemu/kvm.h"
+#include "trace.h"
+
+struct vfio_group_head vfio_group_list =
+    QLIST_HEAD_INITIALIZER(vfio_group_list);
+struct vfio_as_head vfio_address_spaces =
+    QLIST_HEAD_INITIALIZER(vfio_address_spaces);
+
+#ifdef CONFIG_KVM
+/*
+ * We have a single VFIO pseudo device per KVM VM.  Once created it lives
+ * for the life of the VM.  Closing the file descriptor only drops our
+ * reference to it and the device's reference to kvm.  Therefore once
+ * initialized, this file descriptor is only released on QEMU exit and
+ * we'll re-use it should another vfio device be attached before then.
+ */
+static int vfio_kvm_device_fd = -1;
+#endif
+
+/*
+ * Common VFIO interrupt disable
+ */
+void vfio_disable_irqindex(VFIODevice *vbasedev, int index)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
+        .index = index,
+        .start = 0,
+        .count = 0,
+    };
+
+    ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
+        .index = index,
+        .start = 0,
+        .count = 1,
+    };
+
+    ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
+        .index = index,
+        .start = 0,
+        .count = 1,
+    };
+
+    ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+/*
+ * IO Port/MMIO - Beware of the endians, VFIO is always little endian
+ */
+void vfio_region_write(void *opaque, hwaddr addr,
+                       uint64_t data, unsigned size)
+{
+    VFIORegion *region = opaque;
+    VFIODevice *vbasedev = region->vbasedev;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+
+    switch (size) {
+    case 1:
+        buf.byte = data;
+        break;
+    case 2:
+        buf.word = cpu_to_le16(data);
+        break;
+    case 4:
+        buf.dword = cpu_to_le32(data);
+        break;
+    default:
+        hw_error("vfio: unsupported write size, %d bytes", size);
+        break;
+    }
+
+    if (pwrite(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) {
+        error_report("%s(%s:region%d+0x%"HWADDR_PRIx", 0x%"PRIx64
+                     ",%d) failed: %m",
+                     __func__, vbasedev->name, region->nr,
+                     addr, data, size);
+    }
+
+    trace_vfio_region_write(vbasedev->name, region->nr, addr, data, size);
+
+    /*
+     * A read or write to a BAR always signals an INTx EOI.  This will
+     * do nothing if not pending (including not in INTx mode).  We assume
+     * that a BAR access is in response to an interrupt and that BAR
+     * accesses will service the interrupt.  Unfortunately, we don't know
+     * which access will service the interrupt, so we're potentially
+     * getting quite a few host interrupts per guest interrupt.
+     */
+    vbasedev->ops->vfio_eoi(vbasedev);
+}
+
+uint64_t vfio_region_read(void *opaque,
+                          hwaddr addr, unsigned size)
+{
+    VFIORegion *region = opaque;
+    VFIODevice *vbasedev = region->vbasedev;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+    uint64_t data = 0;
+
+    if (pread(vbasedev->fd, &buf, size, region->fd_offset + addr) != size) {
+        error_report("%s(%s:region%d+0x%"HWADDR_PRIx", %d) failed: %m",
+                     __func__, vbasedev->name, region->nr,
+                     addr, size);
+        return (uint64_t)-1;
+    }
+    switch (size) {
+    case 1:
+        data = buf.byte;
+        break;
+    case 2:
+        data = le16_to_cpu(buf.word);
+        break;
+    case 4:
+        data = le32_to_cpu(buf.dword);
+        break;
+    default:
+        hw_error("vfio: unsupported read size, %d bytes", size);
+        break;
+    }
+
+    trace_vfio_region_read(vbasedev->name, region->nr, addr, size, data);
+
+    /* Same as write above */
+    vbasedev->ops->vfio_eoi(vbasedev);
+
+    return data;
+}
+
+const MemoryRegionOps vfio_region_ops = {
+    .read = vfio_region_read,
+    .write = vfio_region_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
+ */
+static int vfio_dma_unmap(VFIOContainer *container,
+                          hwaddr iova, ram_addr_t size)
+{
+    struct vfio_iommu_type1_dma_unmap unmap = {
+        .argsz = sizeof(unmap),
+        .flags = 0,
+        .iova = iova,
+        .size = size,
+    };
+
+    if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
+        error_report("VFIO_UNMAP_DMA: %d", -errno);
+        return -errno;
+    }
+
+    return 0;
+}
+
+static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
+                        ram_addr_t size, void *vaddr, bool readonly)
+{
+    struct vfio_iommu_type1_dma_map map = {
+        .argsz = sizeof(map),
+        .flags = VFIO_DMA_MAP_FLAG_READ,
+        .vaddr = (__u64)(uintptr_t)vaddr,
+        .iova = iova,
+        .size = size,
+    };
+
+    if (!readonly) {
+        map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
+    }
+
+    /*
+     * Try the mapping, if it fails with EBUSY, unmap the region and try
+     * again.  This shouldn't be necessary, but we sometimes see it in
+     * the VGA ROM space.
+     */
+    if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
+        (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
+         ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
+        return 0;
+    }
+
+    error_report("VFIO_MAP_DMA: %d", -errno);
+    return -errno;
+}
+
+static void vfio_host_win_add(VFIOContainer *container,
+                              hwaddr min_iova, hwaddr max_iova,
+                              uint64_t iova_pgsizes)
+{
+    VFIOHostDMAWindow *hostwin;
+
+    QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+        if (ranges_overlap(hostwin->min_iova,
+                           hostwin->max_iova - hostwin->min_iova + 1,
+                           min_iova,
+                           max_iova - min_iova + 1)) {
+            hw_error("%s: Overlapped IOMMU are not enabled", __func__);
+        }
+    }
+
+    hostwin = g_malloc0(sizeof(*hostwin));
+
+    hostwin->min_iova = min_iova;
+    hostwin->max_iova = max_iova;
+    hostwin->iova_pgsizes = iova_pgsizes;
+    QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next);
+}
+
+static int vfio_host_win_del(VFIOContainer *container, hwaddr min_iova,
+                             hwaddr max_iova)
+{
+    VFIOHostDMAWindow *hostwin;
+
+    QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+        if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) {
+            QLIST_REMOVE(hostwin, hostwin_next);
+            return 0;
+        }
+    }
+
+    return -1;
+}
+
+static bool vfio_listener_skipped_section(MemoryRegionSection *section)
+{
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
+           /*
+            * Sizing an enabled 64-bit BAR can cause spurious mappings to
+            * addresses in the upper part of the 64-bit address space.  These
+            * are never accessed by the CPU and beyond the address width of
+            * some IOMMU hardware.  TODO: VFIO should tell us the IOMMU width.
+            */
+           section->offset_within_address_space & (1ULL << 63);
+}
+
+static void vfio_iommu_map_notify(Notifier *n, void *data)
+{
+    VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
+    VFIOContainer *container = giommu->container;
+    IOMMUTLBEntry *iotlb = data;
+    hwaddr iova = iotlb->iova + giommu->iommu_offset;
+    MemoryRegion *mr;
+    hwaddr xlat;
+    hwaddr len = iotlb->addr_mask + 1;
+    void *vaddr;
+    int ret;
+
+    trace_vfio_iommu_map_notify(iova, iova + iotlb->addr_mask);
+
+    if (iotlb->target_as != &address_space_memory) {
+        error_report("Wrong target AS \"%s\", only system memory is allowed",
+                     iotlb->target_as->name ? iotlb->target_as->name : "none");
+        return;
+    }
+
+    /*
+     * The IOMMU TLB entry we have just covers translation through
+     * this IOMMU to its immediate target.  We need to translate
+     * it the rest of the way through to memory.
+     */
+    rcu_read_lock();
+    mr = address_space_translate(&address_space_memory,
+                                 iotlb->translated_addr,
+                                 &xlat, &len, iotlb->perm & IOMMU_WO);
+    if (!memory_region_is_ram(mr)) {
+        error_report("iommu map to non memory area %"HWADDR_PRIx"",
+                     xlat);
+        goto out;
+    }
+    /*
+     * Translation truncates length to the IOMMU page size,
+     * check that it did not truncate too much.
+     */
+    if (len & iotlb->addr_mask) {
+        error_report("iommu has granularity incompatible with target AS");
+        goto out;
+    }
+
+    if ((iotlb->perm & IOMMU_RW) != IOMMU_NONE) {
+        vaddr = memory_region_get_ram_ptr(mr) + xlat;
+        ret = vfio_dma_map(container, iova,
+                           iotlb->addr_mask + 1, vaddr,
+                           !(iotlb->perm & IOMMU_WO) || mr->readonly);
+        if (ret) {
+            error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+                         "0x%"HWADDR_PRIx", %p) = %d (%m)",
+                         container, iova,
+                         iotlb->addr_mask + 1, vaddr, ret);
+        }
+    } else {
+        ret = vfio_dma_unmap(container, iova, iotlb->addr_mask + 1);
+        if (ret) {
+            error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+                         "0x%"HWADDR_PRIx") = %d (%m)",
+                         container, iova,
+                         iotlb->addr_mask + 1, ret);
+        }
+    }
+out:
+    rcu_read_unlock();
+}
+
+static void vfio_listener_region_add(MemoryListener *listener,
+                                     MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+    hwaddr iova, end;
+    Int128 llend, llsize;
+    void *vaddr;
+    int ret;
+    VFIOHostDMAWindow *hostwin;
+    bool hostwin_found;
+
+    if (vfio_listener_skipped_section(section)) {
+        trace_vfio_listener_region_add_skip(
+                section->offset_within_address_space,
+                section->offset_within_address_space +
+                int128_get64(int128_sub(section->size, int128_one())));
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+    end = int128_get64(int128_sub(llend, int128_one()));
+
+    if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
+        VFIOHostDMAWindow *hostwin;
+        hwaddr pgsize = 0;
+
+        /* For now intersections are not allowed, we may relax this later */
+        QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+            if (ranges_overlap(hostwin->min_iova,
+                               hostwin->max_iova - hostwin->min_iova + 1,
+                               section->offset_within_address_space,
+                               int128_get64(section->size))) {
+                ret = -1;
+                goto fail;
+            }
+        }
+
+        ret = vfio_spapr_create_window(container, section, &pgsize);
+        if (ret) {
+            goto fail;
+        }
+
+        vfio_host_win_add(container, section->offset_within_address_space,
+                          section->offset_within_address_space +
+                          int128_get64(section->size) - 1, pgsize);
+    }
+
+    hostwin_found = false;
+    QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) {
+        if (hostwin->min_iova <= iova && end <= hostwin->max_iova) {
+            hostwin_found = true;
+            break;
+        }
+    }
+
+    if (!hostwin_found) {
+        error_report("vfio: IOMMU container %p can't map guest IOVA region"
+                     " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx,
+                     container, iova, end);
+        ret = -EFAULT;
+        goto fail;
+    }
+
+    memory_region_ref(section->mr);
+
+    if (memory_region_is_iommu(section->mr)) {
+        VFIOGuestIOMMU *giommu;
+
+        trace_vfio_listener_region_add_iommu(iova, end);
+        /*
+         * FIXME: For VFIO iommu types which have KVM acceleration to
+         * avoid bouncing all map/unmaps through qemu this way, this
+         * would be the right place to wire that up (tell the KVM
+         * device emulation the VFIO iommu handles to use).
+         */
+        giommu = g_malloc0(sizeof(*giommu));
+        giommu->iommu = section->mr;
+        giommu->iommu_offset = section->offset_within_address_space -
+                               section->offset_within_region;
+        giommu->container = container;
+        giommu->n.notify = vfio_iommu_map_notify;
+        QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
+
+        memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
+        memory_region_iommu_replay(giommu->iommu, &giommu->n, false);
+
+        return;
+    }
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    trace_vfio_listener_region_add_ram(iova, end, vaddr);
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    ret = vfio_dma_map(container, iova, int128_get64(llsize),
+                       vaddr, section->readonly);
+    if (ret) {
+        error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+                     "0x%"HWADDR_PRIx", %p) = %d (%m)",
+                     container, iova, int128_get64(llsize), vaddr, ret);
+        goto fail;
+    }
+
+    return;
+
+fail:
+    /*
+     * On the initfn path, store the first error in the container so we
+     * can gracefully fail.  Runtime, there's not much we can do other
+     * than throw a hardware error.
+     */
+    if (!container->initialized) {
+        if (!container->error) {
+            container->error = ret;
+        }
+    } else {
+        hw_error("vfio: DMA mapping failed, unable to continue");
+    }
+}
+
+static void vfio_listener_region_del(MemoryListener *listener,
+                                     MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer, listener);
+    hwaddr iova, end;
+    Int128 llend, llsize;
+    int ret;
+
+    if (vfio_listener_skipped_section(section)) {
+        trace_vfio_listener_region_del_skip(
+                section->offset_within_address_space,
+                section->offset_within_address_space +
+                int128_get64(int128_sub(section->size, int128_one())));
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    if (memory_region_is_iommu(section->mr)) {
+        VFIOGuestIOMMU *giommu;
+
+        QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
+            if (giommu->iommu == section->mr) {
+                memory_region_unregister_iommu_notifier(giommu->iommu,
+                                                        &giommu->n);
+                QLIST_REMOVE(giommu, giommu_next);
+                g_free(giommu);
+                break;
+            }
+        }
+
+        /*
+         * FIXME: We assume the one big unmap below is adequate to
+         * remove any individual page mappings in the IOMMU which
+         * might have been copied into VFIO. This works for a page table
+         * based IOMMU where a big unmap flattens a large range of IO-PTEs.
+         * That may not be true for all IOMMU types.
+         */
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
+
+    if (int128_ge(int128_make64(iova), llend)) {
+        return;
+    }
+    end = int128_get64(int128_sub(llend, int128_one()));
+
+    llsize = int128_sub(llend, int128_make64(iova));
+
+    trace_vfio_listener_region_del(iova, end);
+
+    ret = vfio_dma_unmap(container, iova, int128_get64(llsize));
+    memory_region_unref(section->mr);
+    if (ret) {
+        error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+                     "0x%"HWADDR_PRIx") = %d (%m)",
+                     container, iova, int128_get64(llsize), ret);
+    }
+
+    if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
+        vfio_spapr_remove_window(container,
+                                 section->offset_within_address_space);
+        if (vfio_host_win_del(container,
+                              section->offset_within_address_space,
+                              section->offset_within_address_space +
+                              int128_get64(section->size) - 1) < 0) {
+            hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx,
+                     __func__, section->offset_within_address_space);
+        }
+    }
+}
+
+static const MemoryListener vfio_memory_listener = {
+    .region_add = vfio_listener_region_add,
+    .region_del = vfio_listener_region_del,
+};
+
+static void vfio_listener_release(VFIOContainer *container)
+{
+    memory_listener_unregister(&container->listener);
+    if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) {
+        memory_listener_unregister(&container->prereg_listener);
+    }
+}
+
+static struct vfio_info_cap_header *
+vfio_get_region_info_cap(struct vfio_region_info *info, uint16_t id)
+{
+    struct vfio_info_cap_header *hdr;
+    void *ptr = info;
+
+    if (!(info->flags & VFIO_REGION_INFO_FLAG_CAPS)) {
+        return NULL;
+    }
+
+    for (hdr = ptr + info->cap_offset; hdr != ptr; hdr = ptr + hdr->next) {
+        if (hdr->id == id) {
+            return hdr;
+        }
+    }
+
+    return NULL;
+}
+
+static void vfio_setup_region_sparse_mmaps(VFIORegion *region,
+                                           struct vfio_region_info *info)
+{
+    struct vfio_info_cap_header *hdr;
+    struct vfio_region_info_cap_sparse_mmap *sparse;
+    int i;
+
+    hdr = vfio_get_region_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP);
+    if (!hdr) {
+        return;
+    }
+
+    sparse = container_of(hdr, struct vfio_region_info_cap_sparse_mmap, header);
+
+    trace_vfio_region_sparse_mmap_header(region->vbasedev->name,
+                                         region->nr, sparse->nr_areas);
+
+    region->nr_mmaps = sparse->nr_areas;
+    region->mmaps = g_new0(VFIOMmap, region->nr_mmaps);
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        region->mmaps[i].offset = sparse->areas[i].offset;
+        region->mmaps[i].size = sparse->areas[i].size;
+        trace_vfio_region_sparse_mmap_entry(i, region->mmaps[i].offset,
+                                            region->mmaps[i].offset +
+                                            region->mmaps[i].size);
+    }
+}
+
+int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region,
+                      int index, const char *name)
+{
+    struct vfio_region_info *info;
+    int ret;
+
+    ret = vfio_get_region_info(vbasedev, index, &info);
+    if (ret) {
+        return ret;
+    }
+
+    region->vbasedev = vbasedev;
+    region->flags = info->flags;
+    region->size = info->size;
+    region->fd_offset = info->offset;
+    region->nr = index;
+
+    if (region->size) {
+        region->mem = g_new0(MemoryRegion, 1);
+        memory_region_init_io(region->mem, obj, &vfio_region_ops,
+                              region, name, region->size);
+
+        if (!vbasedev->no_mmap &&
+            region->flags & VFIO_REGION_INFO_FLAG_MMAP &&
+            !(region->size & ~qemu_real_host_page_mask)) {
+
+            vfio_setup_region_sparse_mmaps(region, info);
+
+            if (!region->nr_mmaps) {
+                region->nr_mmaps = 1;
+                region->mmaps = g_new0(VFIOMmap, region->nr_mmaps);
+                region->mmaps[0].offset = 0;
+                region->mmaps[0].size = region->size;
+            }
+        }
+    }
+
+    g_free(info);
+
+    trace_vfio_region_setup(vbasedev->name, index, name,
+                            region->flags, region->fd_offset, region->size);
+    return 0;
+}
+
+int vfio_region_mmap(VFIORegion *region)
+{
+    int i, prot = 0;
+    char *name;
+
+    if (!region->mem) {
+        return 0;
+    }
+
+    prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0;
+    prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0;
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        region->mmaps[i].mmap = mmap(NULL, region->mmaps[i].size, prot,
+                                     MAP_SHARED, region->vbasedev->fd,
+                                     region->fd_offset +
+                                     region->mmaps[i].offset);
+        if (region->mmaps[i].mmap == MAP_FAILED) {
+            int ret = -errno;
+
+            trace_vfio_region_mmap_fault(memory_region_name(region->mem), i,
+                                         region->fd_offset +
+                                         region->mmaps[i].offset,
+                                         region->fd_offset +
+                                         region->mmaps[i].offset +
+                                         region->mmaps[i].size - 1, ret);
+
+            region->mmaps[i].mmap = NULL;
+
+            for (i--; i >= 0; i--) {
+                memory_region_del_subregion(region->mem, &region->mmaps[i].mem);
+                munmap(region->mmaps[i].mmap, region->mmaps[i].size);
+                object_unparent(OBJECT(&region->mmaps[i].mem));
+                region->mmaps[i].mmap = NULL;
+            }
+
+            return ret;
+        }
+
+        name = g_strdup_printf("%s mmaps[%d]",
+                               memory_region_name(region->mem), i);
+        memory_region_init_ram_ptr(&region->mmaps[i].mem,
+                                   memory_region_owner(region->mem),
+                                   name, region->mmaps[i].size,
+                                   region->mmaps[i].mmap);
+        g_free(name);
+        memory_region_set_skip_dump(&region->mmaps[i].mem);
+        memory_region_add_subregion(region->mem, region->mmaps[i].offset,
+                                    &region->mmaps[i].mem);
+
+        trace_vfio_region_mmap(memory_region_name(&region->mmaps[i].mem),
+                               region->mmaps[i].offset,
+                               region->mmaps[i].offset +
+                               region->mmaps[i].size - 1);
+    }
+
+    return 0;
+}
+
+void vfio_region_exit(VFIORegion *region)
+{
+    int i;
+
+    if (!region->mem) {
+        return;
+    }
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        if (region->mmaps[i].mmap) {
+            memory_region_del_subregion(region->mem, &region->mmaps[i].mem);
+        }
+    }
+
+    trace_vfio_region_exit(region->vbasedev->name, region->nr);
+}
+
+void vfio_region_finalize(VFIORegion *region)
+{
+    int i;
+
+    if (!region->mem) {
+        return;
+    }
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        if (region->mmaps[i].mmap) {
+            munmap(region->mmaps[i].mmap, region->mmaps[i].size);
+            object_unparent(OBJECT(&region->mmaps[i].mem));
+        }
+    }
+
+    object_unparent(OBJECT(region->mem));
+
+    g_free(region->mem);
+    g_free(region->mmaps);
+
+    trace_vfio_region_finalize(region->vbasedev->name, region->nr);
+}
+
+void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled)
+{
+    int i;
+
+    if (!region->mem) {
+        return;
+    }
+
+    for (i = 0; i < region->nr_mmaps; i++) {
+        if (region->mmaps[i].mmap) {
+            memory_region_set_enabled(&region->mmaps[i].mem, enabled);
+        }
+    }
+
+    trace_vfio_region_mmaps_set_enabled(memory_region_name(region->mem),
+                                        enabled);
+}
+
+void vfio_reset_handler(void *opaque)
+{
+    VFIOGroup *group;
+    VFIODevice *vbasedev;
+
+    QLIST_FOREACH(group, &vfio_group_list, next) {
+        QLIST_FOREACH(vbasedev, &group->device_list, next) {
+            vbasedev->ops->vfio_compute_needs_reset(vbasedev);
+        }
+    }
+
+    QLIST_FOREACH(group, &vfio_group_list, next) {
+        QLIST_FOREACH(vbasedev, &group->device_list, next) {
+            if (vbasedev->needs_reset) {
+                vbasedev->ops->vfio_hot_reset_multi(vbasedev);
+            }
+        }
+    }
+}
+
+static void vfio_kvm_device_add_group(VFIOGroup *group)
+{
+#ifdef CONFIG_KVM
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_GROUP,
+        .attr = KVM_DEV_VFIO_GROUP_ADD,
+        .addr = (uint64_t)(unsigned long)&group->fd,
+    };
+
+    if (!kvm_enabled()) {
+        return;
+    }
+
+    if (vfio_kvm_device_fd < 0) {
+        struct kvm_create_device cd = {
+            .type = KVM_DEV_TYPE_VFIO,
+        };
+
+        if (kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd)) {
+            error_report("Failed to create KVM VFIO device: %m");
+            return;
+        }
+
+        vfio_kvm_device_fd = cd.fd;
+    }
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to add group %d to KVM VFIO device: %m",
+                     group->groupid);
+    }
+#endif
+}
+
+static void vfio_kvm_device_del_group(VFIOGroup *group)
+{
+#ifdef CONFIG_KVM
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_VFIO_GROUP,
+        .attr = KVM_DEV_VFIO_GROUP_DEL,
+        .addr = (uint64_t)(unsigned long)&group->fd,
+    };
+
+    if (vfio_kvm_device_fd < 0) {
+        return;
+    }
+
+    if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) {
+        error_report("Failed to remove group %d from KVM VFIO device: %m",
+                     group->groupid);
+    }
+#endif
+}
+
+static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
+{
+    VFIOAddressSpace *space;
+
+    QLIST_FOREACH(space, &vfio_address_spaces, list) {
+        if (space->as == as) {
+            return space;
+        }
+    }
+
+    /* No suitable VFIOAddressSpace, create a new one */
+    space = g_malloc0(sizeof(*space));
+    space->as = as;
+    QLIST_INIT(&space->containers);
+
+    QLIST_INSERT_HEAD(&vfio_address_spaces, space, list);
+
+    return space;
+}
+
+static void vfio_put_address_space(VFIOAddressSpace *space)
+{
+    if (QLIST_EMPTY(&space->containers)) {
+        QLIST_REMOVE(space, list);
+        g_free(space);
+    }
+}
+
+static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
+{
+    VFIOContainer *container;
+    int ret, fd;
+    VFIOAddressSpace *space;
+
+    space = vfio_get_address_space(as);
+
+    QLIST_FOREACH(container, &space->containers, next) {
+        if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
+            group->container = container;
+            QLIST_INSERT_HEAD(&container->group_list, group, container_next);
+            return 0;
+        }
+    }
+
+    fd = qemu_open("/dev/vfio/vfio", O_RDWR);
+    if (fd < 0) {
+        error_report("vfio: failed to open /dev/vfio/vfio: %m");
+        ret = -errno;
+        goto put_space_exit;
+    }
+
+    ret = ioctl(fd, VFIO_GET_API_VERSION);
+    if (ret != VFIO_API_VERSION) {
+        error_report("vfio: supported vfio version: %d, "
+                     "reported version: %d", VFIO_API_VERSION, ret);
+        ret = -EINVAL;
+        goto close_fd_exit;
+    }
+
+    container = g_malloc0(sizeof(*container));
+    container->space = space;
+    container->fd = fd;
+    if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) ||
+        ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU)) {
+        bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1v2_IOMMU);
+        struct vfio_iommu_type1_info info;
+
+        ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
+        if (ret) {
+            error_report("vfio: failed to set group container: %m");
+            ret = -errno;
+            goto free_container_exit;
+        }
+
+        container->iommu_type = v2 ? VFIO_TYPE1v2_IOMMU : VFIO_TYPE1_IOMMU;
+        ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type);
+        if (ret) {
+            error_report("vfio: failed to set iommu for container: %m");
+            ret = -errno;
+            goto free_container_exit;
+        }
+
+        /*
+         * FIXME: This assumes that a Type1 IOMMU can map any 64-bit
+         * IOVA whatsoever.  That's not actually true, but the current
+         * kernel interface doesn't tell us what it can map, and the
+         * existing Type1 IOMMUs generally support any IOVA we're
+         * going to actually try in practice.
+         */
+        info.argsz = sizeof(info);
+        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, &info);
+        /* Ignore errors */
+        if (ret || !(info.flags & VFIO_IOMMU_INFO_PGSIZES)) {
+            /* Assume 4k IOVA page size */
+            info.iova_pgsizes = 4096;
+        }
+        vfio_host_win_add(container, 0, (hwaddr)-1, info.iova_pgsizes);
+    } else if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_IOMMU) ||
+               ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU)) {
+        struct vfio_iommu_spapr_tce_info info;
+        bool v2 = !!ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_SPAPR_TCE_v2_IOMMU);
+
+        ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
+        if (ret) {
+            error_report("vfio: failed to set group container: %m");
+            ret = -errno;
+            goto free_container_exit;
+        }
+        container->iommu_type =
+            v2 ? VFIO_SPAPR_TCE_v2_IOMMU : VFIO_SPAPR_TCE_IOMMU;
+        ret = ioctl(fd, VFIO_SET_IOMMU, container->iommu_type);
+        if (ret) {
+            error_report("vfio: failed to set iommu for container: %m");
+            ret = -errno;
+            goto free_container_exit;
+        }
+
+        /*
+         * The host kernel code implementing VFIO_IOMMU_DISABLE is called
+         * when container fd is closed so we do not call it explicitly
+         * in this file.
+         */
+        if (!v2) {
+            ret = ioctl(fd, VFIO_IOMMU_ENABLE);
+            if (ret) {
+                error_report("vfio: failed to enable container: %m");
+                ret = -errno;
+                goto free_container_exit;
+            }
+        } else {
+            container->prereg_listener = vfio_prereg_listener;
+
+            memory_listener_register(&container->prereg_listener,
+                                     &address_space_memory);
+            if (container->error) {
+                memory_listener_unregister(&container->prereg_listener);
+                error_report("vfio: RAM memory listener initialization failed for container");
+                goto free_container_exit;
+            }
+        }
+
+        info.argsz = sizeof(info);
+        ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info);
+        if (ret) {
+            error_report("vfio: VFIO_IOMMU_SPAPR_TCE_GET_INFO failed: %m");
+            ret = -errno;
+            if (v2) {
+                memory_listener_unregister(&container->prereg_listener);
+            }
+            goto free_container_exit;
+        }
+
+        if (v2) {
+            /*
+             * There is a default window in just created container.
+             * To make region_add/del simpler, we better remove this
+             * window now and let those iommu_listener callbacks
+             * create/remove them when needed.
+             */
+            ret = vfio_spapr_remove_window(container, info.dma32_window_start);
+            if (ret) {
+                goto free_container_exit;
+            }
+        } else {
+            /* The default table uses 4K pages */
+            vfio_host_win_add(container, info.dma32_window_start,
+                              info.dma32_window_start +
+                              info.dma32_window_size - 1,
+                              0x1000);
+        }
+    } else {
+        error_report("vfio: No available IOMMU models");
+        ret = -EINVAL;
+        goto free_container_exit;
+    }
+
+    container->listener = vfio_memory_listener;
+
+    memory_listener_register(&container->listener, container->space->as);
+
+    if (container->error) {
+        ret = container->error;
+        error_report("vfio: memory listener initialization failed for container");
+        goto listener_release_exit;
+    }
+
+    container->initialized = true;
+
+    QLIST_INIT(&container->group_list);
+    QLIST_INSERT_HEAD(&space->containers, container, next);
+
+    group->container = container;
+    QLIST_INSERT_HEAD(&container->group_list, group, container_next);
+
+    return 0;
+listener_release_exit:
+    vfio_listener_release(container);
+
+free_container_exit:
+    g_free(container);
+
+close_fd_exit:
+    close(fd);
+
+put_space_exit:
+    vfio_put_address_space(space);
+
+    return ret;
+}
+
+static void vfio_disconnect_container(VFIOGroup *group)
+{
+    VFIOContainer *container = group->container;
+
+    if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
+        error_report("vfio: error disconnecting group %d from container",
+                     group->groupid);
+    }
+
+    QLIST_REMOVE(group, container_next);
+    group->container = NULL;
+
+    if (QLIST_EMPTY(&container->group_list)) {
+        VFIOAddressSpace *space = container->space;
+        VFIOGuestIOMMU *giommu, *tmp;
+
+        vfio_listener_release(container);
+        QLIST_REMOVE(container, next);
+
+        QLIST_FOREACH_SAFE(giommu, &container->giommu_list, giommu_next, tmp) {
+            memory_region_unregister_iommu_notifier(giommu->iommu, &giommu->n);
+            QLIST_REMOVE(giommu, giommu_next);
+            g_free(giommu);
+        }
+
+        trace_vfio_disconnect_container(container->fd);
+        close(container->fd);
+        g_free(container);
+
+        vfio_put_address_space(space);
+    }
+}
+
+VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
+{
+    VFIOGroup *group;
+    char path[32];
+    struct vfio_group_status status = { .argsz = sizeof(status) };
+
+    QLIST_FOREACH(group, &vfio_group_list, next) {
+        if (group->groupid == groupid) {
+            /* Found it.  Now is it already in the right context? */
+            if (group->container->space->as == as) {
+                return group;
+            } else {
+                error_report("vfio: group %d used in multiple address spaces",
+                             group->groupid);
+                return NULL;
+            }
+        }
+    }
+
+    group = g_malloc0(sizeof(*group));
+
+    snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
+    group->fd = qemu_open(path, O_RDWR);
+    if (group->fd < 0) {
+        error_report("vfio: error opening %s: %m", path);
+        goto free_group_exit;
+    }
+
+    if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
+        error_report("vfio: error getting group status: %m");
+        goto close_fd_exit;
+    }
+
+    if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
+        error_report("vfio: error, group %d is not viable, please ensure "
+                     "all devices within the iommu_group are bound to their "
+                     "vfio bus driver.", groupid);
+        goto close_fd_exit;
+    }
+
+    group->groupid = groupid;
+    QLIST_INIT(&group->device_list);
+
+    if (vfio_connect_container(group, as)) {
+        error_report("vfio: failed to setup container for group %d", groupid);
+        goto close_fd_exit;
+    }
+
+    if (QLIST_EMPTY(&vfio_group_list)) {
+        qemu_register_reset(vfio_reset_handler, NULL);
+    }
+
+    QLIST_INSERT_HEAD(&vfio_group_list, group, next);
+
+    vfio_kvm_device_add_group(group);
+
+    return group;
+
+close_fd_exit:
+    close(group->fd);
+
+free_group_exit:
+    g_free(group);
+
+    return NULL;
+}
+
+void vfio_put_group(VFIOGroup *group)
+{
+    if (!group || !QLIST_EMPTY(&group->device_list)) {
+        return;
+    }
+
+    vfio_kvm_device_del_group(group);
+    vfio_disconnect_container(group);
+    QLIST_REMOVE(group, next);
+    trace_vfio_put_group(group->fd);
+    close(group->fd);
+    g_free(group);
+
+    if (QLIST_EMPTY(&vfio_group_list)) {
+        qemu_unregister_reset(vfio_reset_handler, NULL);
+    }
+}
+
+int vfio_get_device(VFIOGroup *group, const char *name,
+                       VFIODevice *vbasedev)
+{
+    struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
+    int ret, fd;
+
+    fd = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
+    if (fd < 0) {
+        error_report("vfio: error getting device %s from group %d: %m",
+                     name, group->groupid);
+        error_printf("Verify all devices in group %d are bound to vfio-<bus> "
+                     "or pci-stub and not already in use\n", group->groupid);
+        return fd;
+    }
+
+    ret = ioctl(fd, VFIO_DEVICE_GET_INFO, &dev_info);
+    if (ret) {
+        error_report("vfio: error getting device info: %m");
+        close(fd);
+        return ret;
+    }
+
+    vbasedev->fd = fd;
+    vbasedev->group = group;
+    QLIST_INSERT_HEAD(&group->device_list, vbasedev, next);
+
+    vbasedev->num_irqs = dev_info.num_irqs;
+    vbasedev->num_regions = dev_info.num_regions;
+    vbasedev->flags = dev_info.flags;
+
+    trace_vfio_get_device(name, dev_info.flags, dev_info.num_regions,
+                          dev_info.num_irqs);
+
+    vbasedev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
+    return 0;
+}
+
+void vfio_put_base_device(VFIODevice *vbasedev)
+{
+    if (!vbasedev->group) {
+        return;
+    }
+    QLIST_REMOVE(vbasedev, next);
+    vbasedev->group = NULL;
+    trace_vfio_put_base_device(vbasedev->fd);
+    close(vbasedev->fd);
+}
+
+int vfio_get_region_info(VFIODevice *vbasedev, int index,
+                         struct vfio_region_info **info)
+{
+    size_t argsz = sizeof(struct vfio_region_info);
+
+    *info = g_malloc0(argsz);
+
+    (*info)->index = index;
+retry:
+    (*info)->argsz = argsz;
+
+    if (ioctl(vbasedev->fd, VFIO_DEVICE_GET_REGION_INFO, *info)) {
+        g_free(*info);
+        *info = NULL;
+        return -errno;
+    }
+
+    if ((*info)->argsz > argsz) {
+        argsz = (*info)->argsz;
+        *info = g_realloc(*info, argsz);
+
+        goto retry;
+    }
+
+    return 0;
+}
+
+int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type,
+                             uint32_t subtype, struct vfio_region_info **info)
+{
+    int i;
+
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        struct vfio_info_cap_header *hdr;
+        struct vfio_region_info_cap_type *cap_type;
+
+        if (vfio_get_region_info(vbasedev, i, info)) {
+            continue;
+        }
+
+        hdr = vfio_get_region_info_cap(*info, VFIO_REGION_INFO_CAP_TYPE);
+        if (!hdr) {
+            g_free(*info);
+            continue;
+        }
+
+        cap_type = container_of(hdr, struct vfio_region_info_cap_type, header);
+
+        trace_vfio_get_dev_region(vbasedev->name, i,
+                                  cap_type->type, cap_type->subtype);
+
+        if (cap_type->type == type && cap_type->subtype == subtype) {
+            return 0;
+        }
+
+        g_free(*info);
+    }
+
+    *info = NULL;
+    return -ENODEV;
+}
+
+/*
+ * Interfaces for IBM EEH (Enhanced Error Handling)
+ */
+static bool vfio_eeh_container_ok(VFIOContainer *container)
+{
+    /*
+     * As of 2016-03-04 (linux-4.5) the host kernel EEH/VFIO
+     * implementation is broken if there are multiple groups in a
+     * container.  The hardware works in units of Partitionable
+     * Endpoints (== IOMMU groups) and the EEH operations naively
+     * iterate across all groups in the container, without any logic
+     * to make sure the groups have their state synchronized.  For
+     * certain operations (ENABLE) that might be ok, until an error
+     * occurs, but for others (GET_STATE) it's clearly broken.
+     */
+
+    /*
+     * XXX Once fixed kernels exist, test for them here
+     */
+
+    if (QLIST_EMPTY(&container->group_list)) {
+        return false;
+    }
+
+    if (QLIST_NEXT(QLIST_FIRST(&container->group_list), container_next)) {
+        return false;
+    }
+
+    return true;
+}
+
+static int vfio_eeh_container_op(VFIOContainer *container, uint32_t op)
+{
+    struct vfio_eeh_pe_op pe_op = {
+        .argsz = sizeof(pe_op),
+        .op = op,
+    };
+    int ret;
+
+    if (!vfio_eeh_container_ok(container)) {
+        error_report("vfio/eeh: EEH_PE_OP 0x%x: "
+                     "kernel requires a container with exactly one group", op);
+        return -EPERM;
+    }
+
+    ret = ioctl(container->fd, VFIO_EEH_PE_OP, &pe_op);
+    if (ret < 0) {
+        error_report("vfio/eeh: EEH_PE_OP 0x%x failed: %m", op);
+        return -errno;
+    }
+
+    return ret;
+}
+
+static VFIOContainer *vfio_eeh_as_container(AddressSpace *as)
+{
+    VFIOAddressSpace *space = vfio_get_address_space(as);
+    VFIOContainer *container = NULL;
+
+    if (QLIST_EMPTY(&space->containers)) {
+        /* No containers to act on */
+        goto out;
+    }
+
+    container = QLIST_FIRST(&space->containers);
+
+    if (QLIST_NEXT(container, next)) {
+        /* We don't yet have logic to synchronize EEH state across
+         * multiple containers */
+        container = NULL;
+        goto out;
+    }
+
+out:
+    vfio_put_address_space(space);
+    return container;
+}
+
+bool vfio_eeh_as_ok(AddressSpace *as)
+{
+    VFIOContainer *container = vfio_eeh_as_container(as);
+
+    return (container != NULL) && vfio_eeh_container_ok(container);
+}
+
+int vfio_eeh_as_op(AddressSpace *as, uint32_t op)
+{
+    VFIOContainer *container = vfio_eeh_as_container(as);
+
+    if (!container) {
+        return -ENODEV;
+    }
+    return vfio_eeh_container_op(container, op);
+}
diff --git a/hw/vfio/pci-quirks.c b/hw/vfio/pci-quirks.c
new file mode 100644
index 0000000..bec694c
--- /dev/null
+++ b/hw/vfio/pci-quirks.c
@@ -0,0 +1,1846 @@
+/*
+ * device quirks for PCI devices
+ *
+ * Copyright Red Hat, Inc. 2012-2015
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "qemu/range.h"
+#include "qapi/error.h"
+#include "hw/nvram/fw_cfg.h"
+#include "pci.h"
+#include "trace.h"
+
+/* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */
+static bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device)
+{
+    return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) &&
+           (device == PCI_ANY_ID || device == vdev->device_id);
+}
+
+static bool vfio_is_vga(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE);
+
+    return class == PCI_CLASS_DISPLAY_VGA;
+}
+
+/*
+ * List of device ids/vendor ids for which to disable
+ * option rom loading. This avoids the guest hangs during rom
+ * execution as noticed with the BCM 57810 card for lack of a
+ * more better way to handle such issues.
+ * The  user can still override by specifying a romfile or
+ * rombar=1.
+ * Please see https://bugs.launchpad.net/qemu/+bug/1284874
+ * for an analysis of the 57810 card hang. When adding
+ * a new vendor id/device id combination below, please also add
+ * your card/environment details and information that could
+ * help in debugging to the bug tracking this issue
+ */
+static const struct {
+    uint32_t vendor;
+    uint32_t device;
+} romblacklist[] = {
+    { 0x14e4, 0x168e }, /* Broadcom BCM 57810 */
+};
+
+bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0 ; i < ARRAY_SIZE(romblacklist); i++) {
+        if (vfio_pci_is(vdev, romblacklist[i].vendor, romblacklist[i].device)) {
+            trace_vfio_quirk_rom_blacklisted(vdev->vbasedev.name,
+                                             romblacklist[i].vendor,
+                                             romblacklist[i].device);
+            return true;
+        }
+    }
+    return false;
+}
+
+/*
+ * Device specific region quirks (mostly backdoors to PCI config space)
+ */
+
+/*
+ * The generic window quirks operate on an address and data register,
+ * vfio_generic_window_address_quirk handles the address register and
+ * vfio_generic_window_data_quirk handles the data register.  These ops
+ * pass reads and writes through to hardware until a value matching the
+ * stored address match/mask is written.  When this occurs, the data
+ * register access emulated PCI config space for the device rather than
+ * passing through accesses.  This enables devices where PCI config space
+ * is accessible behind a window register to maintain the virtualization
+ * provided through vfio.
+ */
+typedef struct VFIOConfigWindowMatch {
+    uint32_t match;
+    uint32_t mask;
+} VFIOConfigWindowMatch;
+
+typedef struct VFIOConfigWindowQuirk {
+    struct VFIOPCIDevice *vdev;
+
+    uint32_t address_val;
+
+    uint32_t address_offset;
+    uint32_t data_offset;
+
+    bool window_enabled;
+    uint8_t bar;
+
+    MemoryRegion *addr_mem;
+    MemoryRegion *data_mem;
+
+    uint32_t nr_matches;
+    VFIOConfigWindowMatch matches[];
+} VFIOConfigWindowQuirk;
+
+static uint64_t vfio_generic_window_quirk_address_read(void *opaque,
+                                                       hwaddr addr,
+                                                       unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+
+    return vfio_region_read(&vdev->bars[window->bar].region,
+                            addr + window->address_offset, size);
+}
+
+static void vfio_generic_window_quirk_address_write(void *opaque, hwaddr addr,
+                                                    uint64_t data,
+                                                    unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+    int i;
+
+    window->window_enabled = false;
+
+    vfio_region_write(&vdev->bars[window->bar].region,
+                      addr + window->address_offset, data, size);
+
+    for (i = 0; i < window->nr_matches; i++) {
+        if ((data & ~window->matches[i].mask) == window->matches[i].match) {
+            window->window_enabled = true;
+            window->address_val = data & window->matches[i].mask;
+            trace_vfio_quirk_generic_window_address_write(vdev->vbasedev.name,
+                                    memory_region_name(window->addr_mem), data);
+            break;
+        }
+    }
+}
+
+static const MemoryRegionOps vfio_generic_window_address_quirk = {
+    .read = vfio_generic_window_quirk_address_read,
+    .write = vfio_generic_window_quirk_address_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_generic_window_quirk_data_read(void *opaque,
+                                                    hwaddr addr, unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+    uint64_t data;
+
+    /* Always read data reg, discard if window enabled */
+    data = vfio_region_read(&vdev->bars[window->bar].region,
+                            addr + window->data_offset, size);
+
+    if (window->window_enabled) {
+        data = vfio_pci_read_config(&vdev->pdev, window->address_val, size);
+        trace_vfio_quirk_generic_window_data_read(vdev->vbasedev.name,
+                                    memory_region_name(window->data_mem), data);
+    }
+
+    return data;
+}
+
+static void vfio_generic_window_quirk_data_write(void *opaque, hwaddr addr,
+                                                 uint64_t data, unsigned size)
+{
+    VFIOConfigWindowQuirk *window = opaque;
+    VFIOPCIDevice *vdev = window->vdev;
+
+    if (window->window_enabled) {
+        vfio_pci_write_config(&vdev->pdev, window->address_val, data, size);
+        trace_vfio_quirk_generic_window_data_write(vdev->vbasedev.name,
+                                    memory_region_name(window->data_mem), data);
+        return;
+    }
+
+    vfio_region_write(&vdev->bars[window->bar].region,
+                      addr + window->data_offset, data, size);
+}
+
+static const MemoryRegionOps vfio_generic_window_data_quirk = {
+    .read = vfio_generic_window_quirk_data_read,
+    .write = vfio_generic_window_quirk_data_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * The generic mirror quirk handles devices which expose PCI config space
+ * through a region within a BAR.  When enabled, reads and writes are
+ * redirected through to emulated PCI config space.  XXX if PCI config space
+ * used memory regions, this could just be an alias.
+ */
+typedef struct VFIOConfigMirrorQuirk {
+    struct VFIOPCIDevice *vdev;
+    uint32_t offset;
+    uint8_t bar;
+    MemoryRegion *mem;
+} VFIOConfigMirrorQuirk;
+
+static uint64_t vfio_generic_quirk_mirror_read(void *opaque,
+                                               hwaddr addr, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+    uint64_t data;
+
+    /* Read and discard in case the hardware cares */
+    (void)vfio_region_read(&vdev->bars[mirror->bar].region,
+                           addr + mirror->offset, size);
+
+    data = vfio_pci_read_config(&vdev->pdev, addr, size);
+    trace_vfio_quirk_generic_mirror_read(vdev->vbasedev.name,
+                                         memory_region_name(mirror->mem),
+                                         addr, data);
+    return data;
+}
+
+static void vfio_generic_quirk_mirror_write(void *opaque, hwaddr addr,
+                                            uint64_t data, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+
+    vfio_pci_write_config(&vdev->pdev, addr, data, size);
+    trace_vfio_quirk_generic_mirror_write(vdev->vbasedev.name,
+                                          memory_region_name(mirror->mem),
+                                          addr, data);
+}
+
+static const MemoryRegionOps vfio_generic_mirror_quirk = {
+    .read = vfio_generic_quirk_mirror_read,
+    .write = vfio_generic_quirk_mirror_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/* Is range1 fully contained within range2?  */
+static bool vfio_range_contained(uint64_t first1, uint64_t len1,
+                                 uint64_t first2, uint64_t len2) {
+    return (first1 >= first2 && first1 + len1 <= first2 + len2);
+}
+
+#define PCI_VENDOR_ID_ATI               0x1002
+
+/*
+ * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR
+ * through VGA register 0x3c3.  On newer cards, the I/O port BAR is always
+ * BAR4 (older cards like the X550 used BAR1, but we don't care to support
+ * those).  Note that on bare metal, a read of 0x3c3 doesn't always return the
+ * I/O port BAR address.  Originally this was coded to return the virtual BAR
+ * address only if the physical register read returns the actual BAR address,
+ * but users have reported greater success if we return the virtual address
+ * unconditionally.
+ */
+static uint64_t vfio_ati_3c3_quirk_read(void *opaque,
+                                        hwaddr addr, unsigned size)
+{
+    VFIOPCIDevice *vdev = opaque;
+    uint64_t data = vfio_pci_read_config(&vdev->pdev,
+                                         PCI_BASE_ADDRESS_4 + 1, size);
+
+    trace_vfio_quirk_ati_3c3_read(vdev->vbasedev.name, data);
+
+    return data;
+}
+
+static const MemoryRegionOps vfio_ati_3c3_quirk = {
+    .read = vfio_ati_3c3_quirk_read,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+
+    /*
+     * As long as the BAR is >= 256 bytes it will be aligned such that the
+     * lower byte is always zero.  Filter out anything else, if it exists.
+     */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->bars[4].ioport || vdev->bars[4].region.size < 256) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+
+    memory_region_init_io(quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, vdev,
+                          "vfio-ati-3c3-quirk", 1);
+    memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                                3 /* offset 3 bytes from 0x3c0 */, quirk->mem);
+
+    QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks,
+                      quirk, next);
+
+    trace_vfio_quirk_ati_3c3_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Newer ATI/AMD devices, including HD5450 and HD7850, have a mirror to PCI
+ * config space through MMIO BAR2 at offset 0x4000.  Nothing seems to access
+ * the MMIO space directly, but a window to this space is provided through
+ * I/O port BAR4.  Offset 0x0 is the address register and offset 0x4 is the
+ * data register.  When the address is programmed to a range of 0x4000-0x4fff
+ * PCI configuration space is available.  Experimentation seems to indicate
+ * that read-only may be provided by hardware.
+ */
+static void vfio_probe_ati_bar4_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigWindowQuirk *window;
+
+    /* This windows doesn't seem to be used except by legacy VGA code */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->vga || nr != 4) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    window = quirk->data = g_malloc0(sizeof(*window) +
+                                     sizeof(VFIOConfigWindowMatch));
+    window->vdev = vdev;
+    window->address_offset = 0;
+    window->data_offset = 4;
+    window->nr_matches = 1;
+    window->matches[0].match = 0x4000;
+    window->matches[0].mask = vdev->config_size - 1;
+    window->bar = nr;
+    window->addr_mem = &quirk->mem[0];
+    window->data_mem = &quirk->mem[1];
+
+    memory_region_init_io(window->addr_mem, OBJECT(vdev),
+                          &vfio_generic_window_address_quirk, window,
+                          "vfio-ati-bar4-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->address_offset,
+                                        window->addr_mem, 1);
+
+    memory_region_init_io(window->data_mem, OBJECT(vdev),
+                          &vfio_generic_window_data_quirk, window,
+                          "vfio-ati-bar4-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->data_offset,
+                                        window->data_mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_ati_bar4_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Trap the BAR2 MMIO mirror to config space as well.
+ */
+static void vfio_probe_ati_bar2_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigMirrorQuirk *mirror;
+
+    /* Only enable on newer devices where BAR2 is 64bit */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) ||
+        !vdev->vga || nr != 2 || !vdev->bars[2].mem64) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    mirror = quirk->data = g_malloc0(sizeof(*mirror));
+    mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+    mirror->vdev = vdev;
+    mirror->offset = 0x4000;
+    mirror->bar = nr;
+
+    memory_region_init_io(mirror->mem, OBJECT(vdev),
+                          &vfio_generic_mirror_quirk, mirror,
+                          "vfio-ati-bar2-4000-quirk", PCI_CONFIG_SPACE_SIZE);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        mirror->offset, mirror->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_ati_bar2_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Older ATI/AMD cards like the X550 have a similar window to that above.
+ * I/O port BAR1 provides a window to a mirror of PCI config space located
+ * in BAR2 at offset 0xf00.  We don't care to support such older cards, but
+ * note it for future reference.
+ */
+
+#define PCI_VENDOR_ID_NVIDIA                    0x10de
+
+/*
+ * Nvidia has several different methods to get to config space, the
+ * nouveu project has several of these documented here:
+ * https://github.com/pathscale/envytools/tree/master/hwdocs
+ *
+ * The first quirk is actually not documented in envytools and is found
+ * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]).  This is an
+ * NV46 chipset.  The backdoor uses the legacy VGA I/O ports to access
+ * the mirror of PCI config space found at BAR0 offset 0x1800.  The access
+ * sequence first writes 0x338 to I/O port 0x3d4.  The target offset is
+ * then written to 0x3d0.  Finally 0x538 is written for a read and 0x738
+ * is written for a write to 0x3d4.  The BAR0 offset is then accessible
+ * through 0x3d0.  This quirk doesn't seem to be necessary on newer cards
+ * that use the I/O port BAR5 window but it doesn't hurt to leave it.
+ */
+typedef enum {NONE = 0, SELECT, WINDOW, READ, WRITE} VFIONvidia3d0State;
+static const char *nv3d0_states[] = { "NONE", "SELECT",
+                                      "WINDOW", "READ", "WRITE" };
+
+typedef struct VFIONvidia3d0Quirk {
+    VFIOPCIDevice *vdev;
+    VFIONvidia3d0State state;
+    uint32_t offset;
+} VFIONvidia3d0Quirk;
+
+static uint64_t vfio_nvidia_3d4_quirk_read(void *opaque,
+                                           hwaddr addr, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+
+    quirk->state = NONE;
+
+    return vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                         addr + 0x14, size);
+}
+
+static void vfio_nvidia_3d4_quirk_write(void *opaque, hwaddr addr,
+                                        uint64_t data, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+
+    quirk->state = NONE;
+
+    switch (data) {
+    case 0x338:
+        if (old_state == NONE) {
+            quirk->state = SELECT;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    case 0x538:
+        if (old_state == WINDOW) {
+            quirk->state = READ;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    case 0x738:
+        if (old_state == WINDOW) {
+            quirk->state = WRITE;
+            trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                              nv3d0_states[quirk->state]);
+        }
+        break;
+    }
+
+    vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                   addr + 0x14, data, size);
+}
+
+static const MemoryRegionOps vfio_nvidia_3d4_quirk = {
+    .read = vfio_nvidia_3d4_quirk_read,
+    .write = vfio_nvidia_3d4_quirk_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque,
+                                           hwaddr addr, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+    uint64_t data = vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                                  addr + 0x10, size);
+
+    quirk->state = NONE;
+
+    if (old_state == READ &&
+        (quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) {
+        uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1);
+
+        data = vfio_pci_read_config(&vdev->pdev, offset, size);
+        trace_vfio_quirk_nvidia_3d0_read(vdev->vbasedev.name,
+                                         offset, size, data);
+    }
+
+    return data;
+}
+
+static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr,
+                                        uint64_t data, unsigned size)
+{
+    VFIONvidia3d0Quirk *quirk = opaque;
+    VFIOPCIDevice *vdev = quirk->vdev;
+    VFIONvidia3d0State old_state = quirk->state;
+
+    quirk->state = NONE;
+
+    if (old_state == SELECT) {
+        quirk->offset = (uint32_t)data;
+        quirk->state = WINDOW;
+        trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name,
+                                          nv3d0_states[quirk->state]);
+    } else if (old_state == WRITE) {
+        if ((quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) {
+            uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1);
+
+            vfio_pci_write_config(&vdev->pdev, offset, data, size);
+            trace_vfio_quirk_nvidia_3d0_write(vdev->vbasedev.name,
+                                              offset, data, size);
+            return;
+        }
+    }
+
+    vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                   addr + 0x10, data, size);
+}
+
+static const MemoryRegionOps vfio_nvidia_3d0_quirk = {
+    .read = vfio_nvidia_3d0_quirk_read,
+    .write = vfio_nvidia_3d0_quirk_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+    VFIONvidia3d0Quirk *data;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vdev->bars[1].region.size) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->data = data = g_malloc0(sizeof(*data));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    data->vdev = vdev;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_nvidia_3d4_quirk,
+                          data, "vfio-nvidia-3d4-quirk", 2);
+    memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                                0x14 /* 0x3c0 + 0x14 */, &quirk->mem[0]);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_nvidia_3d0_quirk,
+                          data, "vfio-nvidia-3d0-quirk", 2);
+    memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                                0x10 /* 0x3c0 + 0x10 */, &quirk->mem[1]);
+
+    QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks,
+                      quirk, next);
+
+    trace_vfio_quirk_nvidia_3d0_probe(vdev->vbasedev.name);
+}
+
+/*
+ * The second quirk is documented in envytools.  The I/O port BAR5 is just
+ * a set of address/data ports to the MMIO BARs.  The BAR we care about is
+ * again BAR0.  This backdoor is apparently a bit newer than the one above
+ * so we need to not only trap 256 bytes @0x1800, but all of PCI config
+ * space, including extended space is available at the 4k @0x88000.
+ */
+typedef struct VFIONvidiaBAR5Quirk {
+    uint32_t master;
+    uint32_t enable;
+    MemoryRegion *addr_mem;
+    MemoryRegion *data_mem;
+    bool enabled;
+    VFIOConfigWindowQuirk window; /* last for match data */
+} VFIONvidiaBAR5Quirk;
+
+static void vfio_nvidia_bar5_enable(VFIONvidiaBAR5Quirk *bar5)
+{
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    if (((bar5->master & bar5->enable) & 0x1) == bar5->enabled) {
+        return;
+    }
+
+    bar5->enabled = !bar5->enabled;
+    trace_vfio_quirk_nvidia_bar5_state(vdev->vbasedev.name,
+                                       bar5->enabled ?  "Enable" : "Disable");
+    memory_region_set_enabled(bar5->addr_mem, bar5->enabled);
+    memory_region_set_enabled(bar5->data_mem, bar5->enabled);
+}
+
+static uint64_t vfio_nvidia_bar5_quirk_master_read(void *opaque,
+                                                   hwaddr addr, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    return vfio_region_read(&vdev->bars[5].region, addr, size);
+}
+
+static void vfio_nvidia_bar5_quirk_master_write(void *opaque, hwaddr addr,
+                                                uint64_t data, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    vfio_region_write(&vdev->bars[5].region, addr, data, size);
+
+    bar5->master = data;
+    vfio_nvidia_bar5_enable(bar5);
+}
+
+static const MemoryRegionOps vfio_nvidia_bar5_quirk_master = {
+    .read = vfio_nvidia_bar5_quirk_master_read,
+    .write = vfio_nvidia_bar5_quirk_master_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_nvidia_bar5_quirk_enable_read(void *opaque,
+                                                   hwaddr addr, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    return vfio_region_read(&vdev->bars[5].region, addr + 4, size);
+}
+
+static void vfio_nvidia_bar5_quirk_enable_write(void *opaque, hwaddr addr,
+                                                uint64_t data, unsigned size)
+{
+    VFIONvidiaBAR5Quirk *bar5 = opaque;
+    VFIOPCIDevice *vdev = bar5->window.vdev;
+
+    vfio_region_write(&vdev->bars[5].region, addr + 4, data, size);
+
+    bar5->enable = data;
+    vfio_nvidia_bar5_enable(bar5);
+}
+
+static const MemoryRegionOps vfio_nvidia_bar5_quirk_enable = {
+    .read = vfio_nvidia_bar5_quirk_enable_read,
+    .write = vfio_nvidia_bar5_quirk_enable_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_nvidia_bar5_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIONvidiaBAR5Quirk *bar5;
+    VFIOConfigWindowQuirk *window;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vdev->vga || nr != 5) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 4);
+    quirk->nr_mem = 4;
+    bar5 = quirk->data = g_malloc0(sizeof(*bar5) +
+                                   (sizeof(VFIOConfigWindowMatch) * 2));
+    window = &bar5->window;
+
+    window->vdev = vdev;
+    window->address_offset = 0x8;
+    window->data_offset = 0xc;
+    window->nr_matches = 2;
+    window->matches[0].match = 0x1800;
+    window->matches[0].mask = PCI_CONFIG_SPACE_SIZE - 1;
+    window->matches[1].match = 0x88000;
+    window->matches[1].mask = vdev->config_size - 1;
+    window->bar = nr;
+    window->addr_mem = bar5->addr_mem = &quirk->mem[0];
+    window->data_mem = bar5->data_mem = &quirk->mem[1];
+
+    memory_region_init_io(window->addr_mem, OBJECT(vdev),
+                          &vfio_generic_window_address_quirk, window,
+                          "vfio-nvidia-bar5-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->address_offset,
+                                        window->addr_mem, 1);
+    memory_region_set_enabled(window->addr_mem, false);
+
+    memory_region_init_io(window->data_mem, OBJECT(vdev),
+                          &vfio_generic_window_data_quirk, window,
+                          "vfio-nvidia-bar5-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        window->data_offset,
+                                        window->data_mem, 1);
+    memory_region_set_enabled(window->data_mem, false);
+
+    memory_region_init_io(&quirk->mem[2], OBJECT(vdev),
+                          &vfio_nvidia_bar5_quirk_master, bar5,
+                          "vfio-nvidia-bar5-master-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0, &quirk->mem[2], 1);
+
+    memory_region_init_io(&quirk->mem[3], OBJECT(vdev),
+                          &vfio_nvidia_bar5_quirk_enable, bar5,
+                          "vfio-nvidia-bar5-enable-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        4, &quirk->mem[3], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_nvidia_bar5_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Finally, BAR0 itself.  We want to redirect any accesses to either
+ * 0x1800 or 0x88000 through the PCI config space access functions.
+ */
+static void vfio_nvidia_quirk_mirror_write(void *opaque, hwaddr addr,
+                                           uint64_t data, unsigned size)
+{
+    VFIOConfigMirrorQuirk *mirror = opaque;
+    VFIOPCIDevice *vdev = mirror->vdev;
+    PCIDevice *pdev = &vdev->pdev;
+
+    vfio_generic_quirk_mirror_write(opaque, addr, data, size);
+
+    /*
+     * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the
+     * MSI capability ID register.  Both the ID and next register are
+     * read-only, so we allow writes covering either of those to real hw.
+     */
+    if ((pdev->cap_present & QEMU_PCI_CAP_MSI) &&
+        vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) {
+        vfio_region_write(&vdev->bars[mirror->bar].region,
+                          addr + mirror->offset, data, size);
+        trace_vfio_quirk_nvidia_bar0_msi_ack(vdev->vbasedev.name);
+    }
+}
+
+static const MemoryRegionOps vfio_nvidia_mirror_quirk = {
+    .read = vfio_generic_quirk_mirror_read,
+    .write = vfio_nvidia_quirk_mirror_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_nvidia_bar0_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOConfigMirrorQuirk *mirror;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) ||
+        !vfio_is_vga(vdev) || nr != 0) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    mirror = quirk->data = g_malloc0(sizeof(*mirror));
+    mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+    quirk->nr_mem = 1;
+    mirror->vdev = vdev;
+    mirror->offset = 0x88000;
+    mirror->bar = nr;
+
+    memory_region_init_io(mirror->mem, OBJECT(vdev),
+                          &vfio_nvidia_mirror_quirk, mirror,
+                          "vfio-nvidia-bar0-88000-mirror-quirk",
+                          vdev->config_size);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        mirror->offset, mirror->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    /* The 0x1800 offset mirror only seems to get used by legacy VGA */
+    if (vdev->vga) {
+        quirk = g_malloc0(sizeof(*quirk));
+        mirror = quirk->data = g_malloc0(sizeof(*mirror));
+        mirror->mem = quirk->mem = g_new0(MemoryRegion, 1);
+        quirk->nr_mem = 1;
+        mirror->vdev = vdev;
+        mirror->offset = 0x1800;
+        mirror->bar = nr;
+
+        memory_region_init_io(mirror->mem, OBJECT(vdev),
+                              &vfio_nvidia_mirror_quirk, mirror,
+                              "vfio-nvidia-bar0-1800-mirror-quirk",
+                              PCI_CONFIG_SPACE_SIZE);
+        memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                            mirror->offset, mirror->mem, 1);
+
+        QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+    }
+
+    trace_vfio_quirk_nvidia_bar0_probe(vdev->vbasedev.name);
+}
+
+/*
+ * TODO - Some Nvidia devices provide config access to their companion HDA
+ * device and even to their parent bridge via these config space mirrors.
+ * Add quirks for those regions.
+ */
+
+#define PCI_VENDOR_ID_REALTEK 0x10ec
+
+/*
+ * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
+ * offset 0x70 there is a dword data register, offset 0x74 is a dword address
+ * register.  According to the Linux r8169 driver, the MSI-X table is addressed
+ * when the "type" portion of the address register is set to 0x1.  This appears
+ * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
+ * "address latched" indicator.  Bits 12:15 are a mask field, which we can
+ * ignore because the MSI-X table should always be accessed as a dword (full
+ * mask).  Bits 0:11 is offset within the type.
+ *
+ * Example trace:
+ *
+ * Read from MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
+ *
+ * Write 0xfee00000 to MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
+ */
+typedef struct VFIOrtl8168Quirk {
+    VFIOPCIDevice *vdev;
+    uint32_t addr;
+    uint32_t data;
+    bool enabled;
+} VFIOrtl8168Quirk;
+
+static uint64_t vfio_rtl8168_quirk_address_read(void *opaque,
+                                                hwaddr addr, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+    uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size);
+
+    if (rtl->enabled) {
+        data = rtl->addr ^ 0x80000000U; /* latch/complete */
+        trace_vfio_quirk_rtl8168_fake_latch(vdev->vbasedev.name, data);
+    }
+
+    return data;
+}
+
+static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr,
+                                             uint64_t data, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+
+    rtl->enabled = false;
+
+    if ((data & 0x7fff0000) == 0x10000) { /* MSI-X table */
+        rtl->enabled = true;
+        rtl->addr = (uint32_t)data;
+
+        if (data & 0x80000000U) { /* Do write */
+            if (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
+                hwaddr offset = data & 0xfff;
+                uint64_t val = rtl->data;
+
+                trace_vfio_quirk_rtl8168_msix_write(vdev->vbasedev.name,
+                                                    (uint16_t)offset, val);
+
+                /* Write to the proper guest MSI-X table instead */
+                memory_region_dispatch_write(&vdev->pdev.msix_table_mmio,
+                                             offset, val, size,
+                                             MEMTXATTRS_UNSPECIFIED);
+            }
+            return; /* Do not write guest MSI-X data to hardware */
+        }
+    }
+
+    vfio_region_write(&vdev->bars[2].region, addr + 0x74, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl_address_quirk = {
+    .read = vfio_rtl8168_quirk_address_read,
+    .write = vfio_rtl8168_quirk_address_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_rtl8168_quirk_data_read(void *opaque,
+                                             hwaddr addr, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+    uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size);
+
+    if (rtl->enabled && (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
+        hwaddr offset = rtl->addr & 0xfff;
+        memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, offset,
+                                    &data, size, MEMTXATTRS_UNSPECIFIED);
+        trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data);
+    }
+
+    return data;
+}
+
+static void vfio_rtl8168_quirk_data_write(void *opaque, hwaddr addr,
+                                          uint64_t data, unsigned size)
+{
+    VFIOrtl8168Quirk *rtl = opaque;
+    VFIOPCIDevice *vdev = rtl->vdev;
+
+    rtl->data = (uint32_t)data;
+
+    vfio_region_write(&vdev->bars[2].region, addr + 0x70, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl_data_quirk = {
+    .read = vfio_rtl8168_quirk_data_read,
+    .write = vfio_rtl8168_quirk_data_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOQuirk *quirk;
+    VFIOrtl8168Quirk *rtl;
+
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_REALTEK, 0x8168) || nr != 2) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    quirk->data = rtl = g_malloc0(sizeof(*rtl));
+    rtl->vdev = vdev;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev),
+                          &vfio_rtl_address_quirk, rtl,
+                          "vfio-rtl8168-window-address-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0x74, &quirk->mem[0], 1);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev),
+                          &vfio_rtl_data_quirk, rtl,
+                          "vfio-rtl8168-window-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0x70, &quirk->mem[1], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name);
+}
+
+/*
+ * Intel IGD support
+ *
+ * Obviously IGD is not a discrete device, this is evidenced not only by it
+ * being integrated into the CPU, but by the various chipset and BIOS
+ * dependencies that it brings along with it.  Intel is trying to move away
+ * from this and Broadwell and newer devices can run in what Intel calls
+ * "Universal Pass-Through" mode, or UPT.  Theoretically in UPT mode, nothing
+ * more is required beyond assigning the IGD device to a VM.  There are
+ * however support limitations to this mode.  It only supports IGD as a
+ * secondary graphics device in the VM and it doesn't officially support any
+ * physical outputs.
+ *
+ * The code here attempts to enable what we'll call legacy mode assignment,
+ * IGD retains most of the capabilities we expect for it to have on bare
+ * metal.  To enable this mode, the IGD device must be assigned to the VM
+ * at PCI address 00:02.0, it must have a ROM, it very likely needs VGA
+ * support, we must have VM BIOS support for reserving and populating some
+ * of the required tables, and we need to tweak the chipset with revisions
+ * and IDs and an LPC/ISA bridge device.  The intention is to make all of
+ * this happen automatically by installing the device at the correct VM PCI
+ * bus address.  If any of the conditions are not met, we cross our fingers
+ * and hope the user knows better.
+ *
+ * NB - It is possible to enable physical outputs in UPT mode by supplying
+ * an OpRegion table.  We don't do this by default because the guest driver
+ * behaves differently if an OpRegion is provided and no monitor is attached
+ * vs no OpRegion and a monitor being attached or not.  Effectively, if a
+ * headless setup is desired, the OpRegion gets in the way of that.
+ */
+
+/*
+ * This presumes the device is already known to be an Intel VGA device, so we
+ * take liberties in which device ID bits match which generation.  This should
+ * not be taken as an indication that all the devices are supported, or even
+ * supportable, some of them don't even support VT-d.
+ * See linux:include/drm/i915_pciids.h for IDs.
+ */
+static int igd_gen(VFIOPCIDevice *vdev)
+{
+    if ((vdev->device_id & 0xfff) == 0xa84) {
+        return 8; /* Broxton */
+    }
+
+    switch (vdev->device_id & 0xff00) {
+    /* Old, untested, unavailable, unknown */
+    case 0x0000:
+    case 0x2500:
+    case 0x2700:
+    case 0x2900:
+    case 0x2a00:
+    case 0x2e00:
+    case 0x3500:
+    case 0xa000:
+        return -1;
+    /* SandyBridge, IvyBridge, ValleyView, Haswell */
+    case 0x0100:
+    case 0x0400:
+    case 0x0a00:
+    case 0x0c00:
+    case 0x0d00:
+    case 0x0f00:
+        return 6;
+    /* BroadWell, CherryView, SkyLake, KabyLake */
+    case 0x1600:
+    case 0x1900:
+    case 0x2200:
+    case 0x5900:
+        return 8;
+    }
+
+    return 8; /* Assume newer is compatible */
+}
+
+typedef struct VFIOIGDQuirk {
+    struct VFIOPCIDevice *vdev;
+    uint32_t index;
+} VFIOIGDQuirk;
+
+#define IGD_GMCH 0x50 /* Graphics Control Register */
+#define IGD_BDSM 0x5c /* Base Data of Stolen Memory */
+#define IGD_ASLS 0xfc /* ASL Storage Register */
+
+/*
+ * The OpRegion includes the Video BIOS Table, which seems important for
+ * telling the driver what sort of outputs it has.  Without this, the device
+ * may work in the guest, but we may not get output.  This also requires BIOS
+ * support to reserve and populate a section of guest memory sufficient for
+ * the table and to write the base address of that memory to the ASLS register
+ * of the IGD device.
+ */
+int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev,
+                               struct vfio_region_info *info)
+{
+    int ret;
+
+    vdev->igd_opregion = g_malloc0(info->size);
+    ret = pread(vdev->vbasedev.fd, vdev->igd_opregion,
+                info->size, info->offset);
+    if (ret != info->size) {
+        error_report("vfio: Error reading IGD OpRegion");
+        g_free(vdev->igd_opregion);
+        vdev->igd_opregion = NULL;
+        return -EINVAL;
+    }
+
+    /*
+     * Provide fw_cfg with a copy of the OpRegion which the VM firmware is to
+     * allocate 32bit reserved memory for, copy these contents into, and write
+     * the reserved memory base address to the device ASLS register at 0xFC.
+     * Alignment of this reserved region seems flexible, but using a 4k page
+     * alignment seems to work well.  This interface assumes a single IGD
+     * device, which may be at VM address 00:02.0 in legacy mode or another
+     * address in UPT mode.
+     *
+     * NB, there may be future use cases discovered where the VM should have
+     * direct interaction with the host OpRegion, in which case the write to
+     * the ASLS register would trigger MemoryRegion setup to enable that.
+     */
+    fw_cfg_add_file(fw_cfg_find(), "etc/igd-opregion",
+                    vdev->igd_opregion, info->size);
+
+    trace_vfio_pci_igd_opregion_enabled(vdev->vbasedev.name);
+
+    pci_set_long(vdev->pdev.config + IGD_ASLS, 0);
+    pci_set_long(vdev->pdev.wmask + IGD_ASLS, ~0);
+    pci_set_long(vdev->emulated_config_bits + IGD_ASLS, ~0);
+
+    return 0;
+}
+
+/*
+ * The rather short list of registers that we copy from the host devices.
+ * The LPC/ISA bridge values are definitely needed to support the vBIOS, the
+ * host bridge values may or may not be needed depending on the guest OS.
+ * Since we're only munging revision and subsystem values on the host bridge,
+ * we don't require our own device.  The LPC/ISA bridge needs to be our very
+ * own though.
+ */
+typedef struct {
+    uint8_t offset;
+    uint8_t len;
+} IGDHostInfo;
+
+static const IGDHostInfo igd_host_bridge_infos[] = {
+    {PCI_REVISION_ID,         2},
+    {PCI_SUBSYSTEM_VENDOR_ID, 2},
+    {PCI_SUBSYSTEM_ID,        2},
+};
+
+static const IGDHostInfo igd_lpc_bridge_infos[] = {
+    {PCI_VENDOR_ID,           2},
+    {PCI_DEVICE_ID,           2},
+    {PCI_REVISION_ID,         2},
+    {PCI_SUBSYSTEM_VENDOR_ID, 2},
+    {PCI_SUBSYSTEM_ID,        2},
+};
+
+static int vfio_pci_igd_copy(VFIOPCIDevice *vdev, PCIDevice *pdev,
+                             struct vfio_region_info *info,
+                             const IGDHostInfo *list, int len)
+{
+    int i, ret;
+
+    for (i = 0; i < len; i++) {
+        ret = pread(vdev->vbasedev.fd, pdev->config + list[i].offset,
+                    list[i].len, info->offset + list[i].offset);
+        if (ret != list[i].len) {
+            error_report("IGD copy failed: %m");
+            return -errno;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Stuff a few values into the host bridge.
+ */
+static int vfio_pci_igd_host_init(VFIOPCIDevice *vdev,
+                                  struct vfio_region_info *info)
+{
+    PCIBus *bus;
+    PCIDevice *host_bridge;
+    int ret;
+
+    bus = pci_device_root_bus(&vdev->pdev);
+    host_bridge = pci_find_device(bus, 0, PCI_DEVFN(0, 0));
+
+    if (!host_bridge) {
+        error_report("Can't find host bridge");
+        return -ENODEV;
+    }
+
+    ret = vfio_pci_igd_copy(vdev, host_bridge, info, igd_host_bridge_infos,
+                            ARRAY_SIZE(igd_host_bridge_infos));
+    if (!ret) {
+        trace_vfio_pci_igd_host_bridge_enabled(vdev->vbasedev.name);
+    }
+
+    return ret;
+}
+
+/*
+ * IGD LPC/ISA bridge support code.  The vBIOS needs this, but we can't write
+ * arbitrary values into just any bridge, so we must create our own.  We try
+ * to handle if the user has created it for us, which they might want to do
+ * to enable multifuction so we don't occupy the whole PCI slot.
+ */
+static void vfio_pci_igd_lpc_bridge_realize(PCIDevice *pdev, Error **errp)
+{
+    if (pdev->devfn != PCI_DEVFN(0x1f, 0)) {
+        error_setg(errp, "VFIO dummy ISA/LPC bridge must have address 1f.0");
+    }
+}
+
+static void vfio_pci_igd_lpc_bridge_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    dc->desc = "VFIO dummy ISA/LPC bridge for IGD assignment";
+    dc->hotpluggable = false;
+    k->realize = vfio_pci_igd_lpc_bridge_realize;
+    k->class_id = PCI_CLASS_BRIDGE_ISA;
+}
+
+static TypeInfo vfio_pci_igd_lpc_bridge_info = {
+    .name = "vfio-pci-igd-lpc-bridge",
+    .parent = TYPE_PCI_DEVICE,
+    .class_init = vfio_pci_igd_lpc_bridge_class_init,
+};
+
+static void vfio_pci_igd_register_types(void)
+{
+    type_register_static(&vfio_pci_igd_lpc_bridge_info);
+}
+
+type_init(vfio_pci_igd_register_types)
+
+static int vfio_pci_igd_lpc_init(VFIOPCIDevice *vdev,
+                                 struct vfio_region_info *info)
+{
+    PCIDevice *lpc_bridge;
+    int ret;
+
+    lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev),
+                                 0, PCI_DEVFN(0x1f, 0));
+    if (!lpc_bridge) {
+        lpc_bridge = pci_create_simple(pci_device_root_bus(&vdev->pdev),
+                                 PCI_DEVFN(0x1f, 0), "vfio-pci-igd-lpc-bridge");
+    }
+
+    ret = vfio_pci_igd_copy(vdev, lpc_bridge, info, igd_lpc_bridge_infos,
+                            ARRAY_SIZE(igd_lpc_bridge_infos));
+    if (!ret) {
+        trace_vfio_pci_igd_lpc_bridge_enabled(vdev->vbasedev.name);
+    }
+
+    return ret;
+}
+
+/*
+ * IGD Gen8 and newer support up to 8MB for the GTT and use a 64bit PTE
+ * entry, older IGDs use 2MB and 32bit.  Each PTE maps a 4k page.  Therefore
+ * we either have 2M/4k * 4 = 2k or 8M/4k * 8 = 16k as the maximum iobar index
+ * for programming the GTT.
+ *
+ * See linux:include/drm/i915_drm.h for shift and mask values.
+ */
+static int vfio_igd_gtt_max(VFIOPCIDevice *vdev)
+{
+    uint32_t gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch));
+    int ggms, gen = igd_gen(vdev);
+
+    gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch));
+    ggms = (gmch >> (gen < 8 ? 8 : 6)) & 0x3;
+    if (gen > 6) {
+        ggms = 1 << ggms;
+    }
+
+    ggms *= 1024 * 1024;
+
+    return (ggms / (4 * 1024)) * (gen < 8 ? 4 : 8);
+}
+
+/*
+ * The IGD ROM will make use of stolen memory (GGMS) for support of VESA modes.
+ * Somehow the host stolen memory range is used for this, but how the ROM gets
+ * it is a mystery, perhaps it's hardcoded into the ROM.  Thankfully though, it
+ * reprograms the GTT through the IOBAR where we can trap it and transpose the
+ * programming to the VM allocated buffer.  That buffer gets reserved by the VM
+ * firmware via the fw_cfg entry added below.  Here we're just monitoring the
+ * IOBAR address and data registers to detect a write sequence targeting the
+ * GTTADR.  This code is developed by observed behavior and doesn't have a
+ * direct spec reference, unfortunately.
+ */
+static uint64_t vfio_igd_quirk_data_read(void *opaque,
+                                         hwaddr addr, unsigned size)
+{
+    VFIOIGDQuirk *igd = opaque;
+    VFIOPCIDevice *vdev = igd->vdev;
+
+    igd->index = ~0;
+
+    return vfio_region_read(&vdev->bars[4].region, addr + 4, size);
+}
+
+static void vfio_igd_quirk_data_write(void *opaque, hwaddr addr,
+                                      uint64_t data, unsigned size)
+{
+    VFIOIGDQuirk *igd = opaque;
+    VFIOPCIDevice *vdev = igd->vdev;
+    uint64_t val = data;
+    int gen = igd_gen(vdev);
+
+    /*
+     * Programming the GGMS starts at index 0x1 and uses every 4th index (ie.
+     * 0x1, 0x5, 0x9, 0xd,...).  For pre-Gen8 each 4-byte write is a whole PTE
+     * entry, with 0th bit enable set.  For Gen8 and up, PTEs are 64bit, so
+     * entries 0x5 & 0xd are the high dword, in our case zero.  Each PTE points
+     * to a 4k page, which we translate to a page from the VM allocated region,
+     * pointed to by the BDSM register.  If this is not set, we fail.
+     *
+     * We trap writes to the full configured GTT size, but we typically only
+     * see the vBIOS writing up to (nearly) the 1MB barrier.  In fact it often
+     * seems to miss the last entry for an even 1MB GTT.  Doing a gratuitous
+     * write of that last entry does work, but is hopefully unnecessary since
+     * we clear the previous GTT on initialization.
+     */
+    if ((igd->index % 4 == 1) && igd->index < vfio_igd_gtt_max(vdev)) {
+        if (gen < 8 || (igd->index % 8 == 1)) {
+            uint32_t base;
+
+            base = pci_get_long(vdev->pdev.config + IGD_BDSM);
+            if (!base) {
+                hw_error("vfio-igd: Guest attempted to program IGD GTT before "
+                         "BIOS reserved stolen memory.  Unsupported BIOS?");
+            }
+
+            val = base | (data & ((1 << 20) - 1));
+        } else {
+            val = 0; /* upper 32bits of pte, we only enable below 4G PTEs */
+        }
+
+        trace_vfio_pci_igd_bar4_write(vdev->vbasedev.name,
+                                      igd->index, data, val);
+    }
+
+    vfio_region_write(&vdev->bars[4].region, addr + 4, val, size);
+
+    igd->index = ~0;
+}
+
+static const MemoryRegionOps vfio_igd_data_quirk = {
+    .read = vfio_igd_quirk_data_read,
+    .write = vfio_igd_quirk_data_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static uint64_t vfio_igd_quirk_index_read(void *opaque,
+                                          hwaddr addr, unsigned size)
+{
+    VFIOIGDQuirk *igd = opaque;
+    VFIOPCIDevice *vdev = igd->vdev;
+
+    igd->index = ~0;
+
+    return vfio_region_read(&vdev->bars[4].region, addr, size);
+}
+
+static void vfio_igd_quirk_index_write(void *opaque, hwaddr addr,
+                                       uint64_t data, unsigned size)
+{
+    VFIOIGDQuirk *igd = opaque;
+    VFIOPCIDevice *vdev = igd->vdev;
+
+    igd->index = data;
+
+    vfio_region_write(&vdev->bars[4].region, addr, data, size);
+}
+
+static const MemoryRegionOps vfio_igd_index_quirk = {
+    .read = vfio_igd_quirk_index_read,
+    .write = vfio_igd_quirk_index_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr)
+{
+    struct vfio_region_info *rom = NULL, *opregion = NULL,
+                            *host = NULL, *lpc = NULL;
+    VFIOQuirk *quirk;
+    VFIOIGDQuirk *igd;
+    PCIDevice *lpc_bridge;
+    int i, ret, ggms_mb, gms_mb = 0, gen;
+    uint64_t *bdsm_size;
+    uint32_t gmch;
+    uint16_t cmd_orig, cmd;
+
+    /*
+     * This must be an Intel VGA device at address 00:02.0 for us to even
+     * consider enabling legacy mode.  The vBIOS has dependencies on the
+     * PCI bus address.
+     */
+    if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) ||
+        !vfio_is_vga(vdev) || nr != 4 ||
+        &vdev->pdev != pci_find_device(pci_device_root_bus(&vdev->pdev),
+                                       0, PCI_DEVFN(0x2, 0))) {
+        return;
+    }
+
+    /*
+     * We need to create an LPC/ISA bridge at PCI bus address 00:1f.0 that we
+     * can stuff host values into, so if there's already one there and it's not
+     * one we can hack on, legacy mode is no-go.  Sorry Q35.
+     */
+    lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev),
+                                 0, PCI_DEVFN(0x1f, 0));
+    if (lpc_bridge && !object_dynamic_cast(OBJECT(lpc_bridge),
+                                           "vfio-pci-igd-lpc-bridge")) {
+        error_report("IGD device %s cannot support legacy mode due to existing "
+                     "devices at address 1f.0", vdev->vbasedev.name);
+        return;
+    }
+
+    /*
+     * IGD is not a standard, they like to change their specs often.  We
+     * only attempt to support back to SandBridge and we hope that newer
+     * devices maintain compatibility with generation 8.
+     */
+    gen = igd_gen(vdev);
+    if (gen != 6 && gen != 8) {
+        error_report("IGD device %s is unsupported in legacy mode, "
+                     "try SandyBridge or newer", vdev->vbasedev.name);
+        return;
+    }
+
+    /*
+     * Most of what we're doing here is to enable the ROM to run, so if
+     * there's no ROM, there's no point in setting up this quirk.
+     * NB. We only seem to get BIOS ROMs, so a UEFI VM would need CSM support.
+     */
+    ret = vfio_get_region_info(&vdev->vbasedev,
+                               VFIO_PCI_ROM_REGION_INDEX, &rom);
+    if ((ret || !rom->size) && !vdev->pdev.romfile) {
+        error_report("IGD device %s has no ROM, legacy mode disabled",
+                     vdev->vbasedev.name);
+        goto out;
+    }
+
+    /*
+     * Ignore the hotplug corner case, mark the ROM failed, we can't
+     * create the devices we need for legacy mode in the hotplug scenario.
+     */
+    if (vdev->pdev.qdev.hotplugged) {
+        error_report("IGD device %s hotplugged, ROM disabled, "
+                     "legacy mode disabled", vdev->vbasedev.name);
+        vdev->rom_read_failed = true;
+        goto out;
+    }
+
+    /*
+     * Check whether we have all the vfio device specific regions to
+     * support legacy mode (added in Linux v4.6).  If not, bail.
+     */
+    ret = vfio_get_dev_region_info(&vdev->vbasedev,
+                        VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
+                        VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
+    if (ret) {
+        error_report("IGD device %s does not support OpRegion access,"
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    ret = vfio_get_dev_region_info(&vdev->vbasedev,
+                        VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
+                        VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG, &host);
+    if (ret) {
+        error_report("IGD device %s does not support host bridge access,"
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    ret = vfio_get_dev_region_info(&vdev->vbasedev,
+                        VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
+                        VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG, &lpc);
+    if (ret) {
+        error_report("IGD device %s does not support LPC bridge access,"
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, 4);
+
+    /*
+     * If IGD VGA Disable is clear (expected) and VGA is not already enabled,
+     * try to enable it.  Probably shouldn't be using legacy mode without VGA,
+     * but also no point in us enabling VGA if disabled in hardware.
+     */
+    if (!(gmch & 0x2) && !vdev->vga && vfio_populate_vga(vdev)) {
+        error_report("IGD device %s failed to enable VGA access, "
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Create our LPC/ISA bridge */
+    ret = vfio_pci_igd_lpc_init(vdev, lpc);
+    if (ret) {
+        error_report("IGD device %s failed to create LPC bridge, "
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Stuff some host values into the VM PCI host bridge */
+    ret = vfio_pci_igd_host_init(vdev, host);
+    if (ret) {
+        error_report("IGD device %s failed to modify host bridge, "
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Setup OpRegion access */
+    ret = vfio_pci_igd_opregion_init(vdev, opregion);
+    if (ret) {
+        error_report("IGD device %s failed to setup OpRegion, "
+                     "legacy mode disabled", vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Setup our quirk to munge GTT addresses to the VM allocated buffer */
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->mem = g_new0(MemoryRegion, 2);
+    quirk->nr_mem = 2;
+    igd = quirk->data = g_malloc0(sizeof(*igd));
+    igd->vdev = vdev;
+    igd->index = ~0;
+
+    memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_igd_index_quirk,
+                          igd, "vfio-igd-index-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        0, &quirk->mem[0], 1);
+
+    memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_igd_data_quirk,
+                          igd, "vfio-igd-data-quirk", 4);
+    memory_region_add_subregion_overlap(vdev->bars[nr].region.mem,
+                                        4, &quirk->mem[1], 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    /* Determine the size of stolen memory needed for GTT */
+    ggms_mb = (gmch >> (gen < 8 ? 8 : 6)) & 0x3;
+    if (gen > 6) {
+        ggms_mb = 1 << ggms_mb;
+    }
+
+    /*
+     * Assume we have no GMS memory, but allow it to be overrided by device
+     * option (experimental).  The spec doesn't actually allow zero GMS when
+     * when IVD (IGD VGA Disable) is clear, but the claim is that it's unused,
+     * so let's not waste VM memory for it.
+     */
+    gmch &= ~((gen < 8 ? 0x1f : 0xff) << (gen < 8 ? 3 : 8));
+
+    if (vdev->igd_gms) {
+        if (vdev->igd_gms <= 0x10) {
+            gms_mb = vdev->igd_gms * 32;
+            gmch |= vdev->igd_gms << (gen < 8 ? 3 : 8);
+        } else {
+            error_report("Unsupported IGD GMS value 0x%x", vdev->igd_gms);
+            vdev->igd_gms = 0;
+        }
+    }
+
+    /*
+     * Request reserved memory for stolen memory via fw_cfg.  VM firmware
+     * must allocate a 1MB aligned reserved memory region below 4GB with
+     * the requested size (in bytes) for use by the Intel PCI class VGA
+     * device at VM address 00:02.0.  The base address of this reserved
+     * memory region must be written to the device BDSM regsiter at PCI
+     * config offset 0x5C.
+     */
+    bdsm_size = g_malloc(sizeof(*bdsm_size));
+    *bdsm_size = cpu_to_le64((ggms_mb + gms_mb) * 1024 * 1024);
+    fw_cfg_add_file(fw_cfg_find(), "etc/igd-bdsm-size",
+                    bdsm_size, sizeof(*bdsm_size));
+
+    /* GMCH is read-only, emulated */
+    pci_set_long(vdev->pdev.config + IGD_GMCH, gmch);
+    pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0);
+    pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0);
+
+    /* BDSM is read-write, emulated.  The BIOS needs to be able to write it */
+    pci_set_long(vdev->pdev.config + IGD_BDSM, 0);
+    pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0);
+    pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0);
+
+    /*
+     * This IOBAR gives us access to GTTADR, which allows us to write to
+     * the GTT itself.  So let's go ahead and write zero to all the GTT
+     * entries to avoid spurious DMA faults.  Be sure I/O access is enabled
+     * before talking to the device.
+     */
+    if (pread(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig),
+              vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) {
+        error_report("IGD device %s - failed to read PCI command register",
+                     vdev->vbasedev.name);
+    }
+
+    cmd = cmd_orig | PCI_COMMAND_IO;
+
+    if (pwrite(vdev->vbasedev.fd, &cmd, sizeof(cmd),
+               vdev->config_offset + PCI_COMMAND) != sizeof(cmd)) {
+        error_report("IGD device %s - failed to write PCI command register",
+                     vdev->vbasedev.name);
+    }
+
+    for (i = 1; i < vfio_igd_gtt_max(vdev); i += 4) {
+        vfio_region_write(&vdev->bars[4].region, 0, i, 4);
+        vfio_region_write(&vdev->bars[4].region, 4, 0, 4);
+    }
+
+    if (pwrite(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig),
+               vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) {
+        error_report("IGD device %s - failed to restore PCI command register",
+                     vdev->vbasedev.name);
+    }
+
+    trace_vfio_pci_igd_bdsm_enabled(vdev->vbasedev.name, ggms_mb + gms_mb);
+
+out:
+    g_free(rom);
+    g_free(opregion);
+    g_free(host);
+    g_free(lpc);
+}
+
+/*
+ * Common quirk probe entry points.
+ */
+void vfio_vga_quirk_setup(VFIOPCIDevice *vdev)
+{
+    vfio_vga_probe_ati_3c3_quirk(vdev);
+    vfio_vga_probe_nvidia_3d0_quirk(vdev);
+}
+
+void vfio_vga_quirk_exit(VFIOPCIDevice *vdev)
+{
+    VFIOQuirk *quirk;
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
+        QLIST_FOREACH(quirk, &vdev->vga->region[i].quirks, next) {
+            for (j = 0; j < quirk->nr_mem; j++) {
+                memory_region_del_subregion(&vdev->vga->region[i].mem,
+                                            &quirk->mem[j]);
+            }
+        }
+    }
+}
+
+void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev)
+{
+    int i, j;
+
+    for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
+        while (!QLIST_EMPTY(&vdev->vga->region[i].quirks)) {
+            VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga->region[i].quirks);
+            QLIST_REMOVE(quirk, next);
+            for (j = 0; j < quirk->nr_mem; j++) {
+                object_unparent(OBJECT(&quirk->mem[j]));
+            }
+            g_free(quirk->mem);
+            g_free(quirk->data);
+            g_free(quirk);
+        }
+    }
+}
+
+void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr)
+{
+    vfio_probe_ati_bar4_quirk(vdev, nr);
+    vfio_probe_ati_bar2_quirk(vdev, nr);
+    vfio_probe_nvidia_bar5_quirk(vdev, nr);
+    vfio_probe_nvidia_bar0_quirk(vdev, nr);
+    vfio_probe_rtl8168_bar2_quirk(vdev, nr);
+    vfio_probe_igd_bar4_quirk(vdev, nr);
+}
+
+void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    VFIOQuirk *quirk;
+    int i;
+
+    QLIST_FOREACH(quirk, &bar->quirks, next) {
+        for (i = 0; i < quirk->nr_mem; i++) {
+            memory_region_del_subregion(bar->region.mem, &quirk->mem[i]);
+        }
+    }
+}
+
+void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    int i;
+
+    while (!QLIST_EMPTY(&bar->quirks)) {
+        VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks);
+        QLIST_REMOVE(quirk, next);
+        for (i = 0; i < quirk->nr_mem; i++) {
+            object_unparent(OBJECT(&quirk->mem[i]));
+        }
+        g_free(quirk->mem);
+        g_free(quirk->data);
+        g_free(quirk);
+    }
+}
+
+/*
+ * Reset quirks
+ */
+
+/*
+ * AMD Radeon PCI config reset, based on Linux:
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running()
+ *   drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc()
+ *   drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock()
+ * IDs: include/drm/drm_pciids.h
+ * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0
+ *
+ * Bonaire and Hawaii GPUs do not respond to a bus reset.  This is a bug in the
+ * hardware that should be fixed on future ASICs.  The symptom of this is that
+ * once the accerlated driver loads, Windows guests will bsod on subsequent
+ * attmpts to load the driver, such as after VM reset or shutdown/restart.  To
+ * work around this, we do an AMD specific PCI config reset, followed by an SMC
+ * reset.  The PCI config reset only works if SMC firmware is running, so we
+ * have a dependency on the state of the device as to whether this reset will
+ * be effective.  There are still cases where we won't be able to kick the
+ * device into working, but this greatly improves the usability overall.  The
+ * config reset magic is relatively common on AMD GPUs, but the setup and SMC
+ * poking is largely ASIC specific.
+ */
+static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev)
+{
+    uint32_t clk, pc_c;
+
+    /*
+     * Registers 200h and 204h are index and data registers for accessing
+     * indirect configuration registers within the device.
+     */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
+    clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4);
+    pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+
+    return (!(clk & 1) && (0x20100 <= pc_c));
+}
+
+/*
+ * The scope of a config reset is controlled by a mode bit in the misc register
+ * and a fuse, exposed as a bit in another register.  The fuse is the default
+ * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula
+ * scope = !(misc ^ fuse), where the resulting scope is defined the same as
+ * the fuse.  A truth table therefore tells us that if misc == fuse, we need
+ * to flip the value of the bit in the misc register.
+ */
+static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev)
+{
+    uint32_t misc, fuse;
+    bool a, b;
+
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4);
+    fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    b = fuse & 64;
+
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4);
+    misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    a = misc & 2;
+
+    if (a == b) {
+        vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4);
+        vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */
+    }
+}
+
+static int vfio_radeon_reset(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    int i, ret = 0;
+    uint32_t data;
+
+    /* Defer to a kernel implemented reset */
+    if (vdev->vbasedev.reset_works) {
+        trace_vfio_quirk_ati_bonaire_reset_skipped(vdev->vbasedev.name);
+        return -ENODEV;
+    }
+
+    /* Enable only memory BAR access */
+    vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2);
+
+    /* Reset only works if SMC firmware is loaded and running */
+    if (!vfio_radeon_smc_is_running(vdev)) {
+        ret = -EINVAL;
+        trace_vfio_quirk_ati_bonaire_reset_no_smc(vdev->vbasedev.name);
+        goto out;
+    }
+
+    /* Make sure only the GFX function is reset */
+    vfio_radeon_set_gfx_only_reset(vdev);
+
+    /* AMD PCI config reset */
+    vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4);
+    usleep(100);
+
+    /* Read back the memory size to make sure we're out of reset */
+    for (i = 0; i < 100000; i++) {
+        if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) {
+            goto reset_smc;
+        }
+        usleep(1);
+    }
+
+    trace_vfio_quirk_ati_bonaire_reset_timeout(vdev->vbasedev.name);
+
+reset_smc:
+    /* Reset SMC */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4);
+    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    data |= 1;
+    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
+
+    /* Disable SMC clock */
+    vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4);
+    data = vfio_region_read(&vdev->bars[5].region, 0x204, 4);
+    data |= 1;
+    vfio_region_write(&vdev->bars[5].region, 0x204, data, 4);
+
+    trace_vfio_quirk_ati_bonaire_reset_done(vdev->vbasedev.name);
+
+out:
+    /* Restore PCI command register */
+    vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2);
+
+    return ret;
+}
+
+void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev)
+{
+    switch (vdev->vendor_id) {
+    case 0x1002:
+        switch (vdev->device_id) {
+        /* Bonaire */
+        case 0x6649: /* Bonaire [FirePro W5100] */
+        case 0x6650:
+        case 0x6651:
+        case 0x6658: /* Bonaire XTX [Radeon R7 260X] */
+        case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */
+        case 0x665d: /* Bonaire [Radeon R7 200 Series] */
+        /* Hawaii */
+        case 0x67A0: /* Hawaii XT GL [FirePro W9100] */
+        case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */
+        case 0x67A2:
+        case 0x67A8:
+        case 0x67A9:
+        case 0x67AA:
+        case 0x67B0: /* Hawaii XT [Radeon R9 290X] */
+        case 0x67B1: /* Hawaii PRO [Radeon R9 290] */
+        case 0x67B8:
+        case 0x67B9:
+        case 0x67BA:
+        case 0x67BE:
+            vdev->resetfn = vfio_radeon_reset;
+            trace_vfio_quirk_ati_bonaire_reset(vdev->vbasedev.name);
+            break;
+        }
+        break;
+    }
+}
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
new file mode 100644
index 0000000..80e68bb
--- /dev/null
+++ b/hw/vfio/pci.c
@@ -0,0 +1,2877 @@
+/*
+ * vfio based device assignment support
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on qemu-kvm device-assignment:
+ *  Adapted for KVM by Qumranet.
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+
+#include "qemu/osdep.h"
+#include <linux/vfio.h>
+#include <sys/ioctl.h>
+
+#include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
+#include "hw/pci/pci_bridge.h"
+#include "qemu/error-report.h"
+#include "qemu/range.h"
+#include "sysemu/kvm.h"
+#include "sysemu/sysemu.h"
+#include "pci.h"
+#include "trace.h"
+#include "qapi/error.h"
+
+#define MSIX_CAP_LENGTH 12
+
+static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
+static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
+
+/*
+ * Disabling BAR mmaping can be slow, but toggling it around INTx can
+ * also be a huge overhead.  We try to get the best of both worlds by
+ * waiting until an interrupt to disable mmaps (subsequent transitions
+ * to the same state are effectively no overhead).  If the interrupt has
+ * been serviced and the time gap is long enough, we re-enable mmaps for
+ * performance.  This works well for things like graphics cards, which
+ * may not use their interrupt at all and are penalized to an unusable
+ * level by read/write BAR traps.  Other devices, like NICs, have more
+ * regular interrupts and see much better latency by staying in non-mmap
+ * mode.  We therefore set the default mmap_timeout such that a ping
+ * is just enough to keep the mmap disabled.  Users can experiment with
+ * other options with the x-intx-mmap-timeout-ms parameter (a value of
+ * zero disables the timer).
+ */
+static void vfio_intx_mmap_enable(void *opaque)
+{
+    VFIOPCIDevice *vdev = opaque;
+
+    if (vdev->intx.pending) {
+        timer_mod(vdev->intx.mmap_timer,
+                       qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
+        return;
+    }
+
+    vfio_mmap_set_enabled(vdev, true);
+}
+
+static void vfio_intx_interrupt(void *opaque)
+{
+    VFIOPCIDevice *vdev = opaque;
+
+    if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
+        return;
+    }
+
+    trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
+
+    vdev->intx.pending = true;
+    pci_irq_assert(&vdev->pdev);
+    vfio_mmap_set_enabled(vdev, false);
+    if (vdev->intx.mmap_timeout) {
+        timer_mod(vdev->intx.mmap_timer,
+                       qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
+    }
+}
+
+static void vfio_intx_eoi(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
+
+    if (!vdev->intx.pending) {
+        return;
+    }
+
+    trace_vfio_intx_eoi(vbasedev->name);
+
+    vdev->intx.pending = false;
+    pci_irq_deassert(&vdev->pdev);
+    vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+}
+
+static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev)
+{
+#ifdef CONFIG_KVM
+    struct kvm_irqfd irqfd = {
+        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+        .gsi = vdev->intx.route.irq,
+        .flags = KVM_IRQFD_FLAG_RESAMPLE,
+    };
+    struct vfio_irq_set *irq_set;
+    int ret, argsz;
+    int32_t *pfd;
+
+    if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
+        vdev->intx.route.mode != PCI_INTX_ENABLED ||
+        !kvm_resamplefds_enabled()) {
+        return;
+    }
+
+    /* Get to a known interrupt state */
+    qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
+    vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+    vdev->intx.pending = false;
+    pci_irq_deassert(&vdev->pdev);
+
+    /* Get an eventfd for resample/unmask */
+    if (event_notifier_init(&vdev->intx.unmask, 0)) {
+        error_report("vfio: Error: event_notifier_init failed eoi");
+        goto fail;
+    }
+
+    /* KVM triggers it, VFIO listens for it */
+    irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
+
+    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+        error_report("vfio: Error: Failed to setup resample irqfd: %m");
+        goto fail_irqfd;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
+    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = irqfd.resamplefd;
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret) {
+        error_report("vfio: Error: Failed to setup INTx unmask fd: %m");
+        goto fail_vfio;
+    }
+
+    /* Let'em rip */
+    vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+
+    vdev->intx.kvm_accel = true;
+
+    trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
+
+    return;
+
+fail_vfio:
+    irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
+    kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
+fail_irqfd:
+    event_notifier_cleanup(&vdev->intx.unmask);
+fail:
+    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+    vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+#endif
+}
+
+static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
+{
+#ifdef CONFIG_KVM
+    struct kvm_irqfd irqfd = {
+        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+        .gsi = vdev->intx.route.irq,
+        .flags = KVM_IRQFD_FLAG_DEASSIGN,
+    };
+
+    if (!vdev->intx.kvm_accel) {
+        return;
+    }
+
+    /*
+     * Get to a known state, hardware masked, QEMU ready to accept new
+     * interrupts, QEMU IRQ de-asserted.
+     */
+    vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+    vdev->intx.pending = false;
+    pci_irq_deassert(&vdev->pdev);
+
+    /* Tell KVM to stop listening for an INTx irqfd */
+    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+        error_report("vfio: Error: Failed to disable INTx irqfd: %m");
+    }
+
+    /* We only need to close the eventfd for VFIO to cleanup the kernel side */
+    event_notifier_cleanup(&vdev->intx.unmask);
+
+    /* QEMU starts listening for interrupt events. */
+    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+
+    vdev->intx.kvm_accel = false;
+
+    /* If we've missed an event, let it re-fire through QEMU */
+    vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+
+    trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
+#endif
+}
+
+static void vfio_intx_update(PCIDevice *pdev)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    PCIINTxRoute route;
+
+    if (vdev->interrupt != VFIO_INT_INTx) {
+        return;
+    }
+
+    route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
+
+    if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
+        return; /* Nothing changed */
+    }
+
+    trace_vfio_intx_update(vdev->vbasedev.name,
+                           vdev->intx.route.irq, route.irq);
+
+    vfio_intx_disable_kvm(vdev);
+
+    vdev->intx.route = route;
+
+    if (route.mode != PCI_INTX_ENABLED) {
+        return;
+    }
+
+    vfio_intx_enable_kvm(vdev);
+
+    /* Re-enable the interrupt in cased we missed an EOI */
+    vfio_intx_eoi(&vdev->vbasedev);
+}
+
+static int vfio_intx_enable(VFIOPCIDevice *vdev)
+{
+    uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
+    int ret, argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    if (!pin) {
+        return 0;
+    }
+
+    vfio_disable_interrupts(vdev);
+
+    vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
+    pci_config_set_interrupt_pin(vdev->pdev.config, pin);
+
+#ifdef CONFIG_KVM
+    /*
+     * Only conditional to avoid generating error messages on platforms
+     * where we won't actually use the result anyway.
+     */
+    if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
+        vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
+                                                        vdev->intx.pin);
+    }
+#endif
+
+    ret = event_notifier_init(&vdev->intx.interrupt, 0);
+    if (ret) {
+        error_report("vfio: Error: event_notifier_init failed");
+        return ret;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
+    qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret) {
+        error_report("vfio: Error: Failed to setup INTx fd: %m");
+        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
+        event_notifier_cleanup(&vdev->intx.interrupt);
+        return -errno;
+    }
+
+    vfio_intx_enable_kvm(vdev);
+
+    vdev->interrupt = VFIO_INT_INTx;
+
+    trace_vfio_intx_enable(vdev->vbasedev.name);
+
+    return 0;
+}
+
+static void vfio_intx_disable(VFIOPCIDevice *vdev)
+{
+    int fd;
+
+    timer_del(vdev->intx.mmap_timer);
+    vfio_intx_disable_kvm(vdev);
+    vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
+    vdev->intx.pending = false;
+    pci_irq_deassert(&vdev->pdev);
+    vfio_mmap_set_enabled(vdev, true);
+
+    fd = event_notifier_get_fd(&vdev->intx.interrupt);
+    qemu_set_fd_handler(fd, NULL, NULL, vdev);
+    event_notifier_cleanup(&vdev->intx.interrupt);
+
+    vdev->interrupt = VFIO_INT_NONE;
+
+    trace_vfio_intx_disable(vdev->vbasedev.name);
+}
+
+/*
+ * MSI/X
+ */
+static void vfio_msi_interrupt(void *opaque)
+{
+    VFIOMSIVector *vector = opaque;
+    VFIOPCIDevice *vdev = vector->vdev;
+    MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
+    void (*notify)(PCIDevice *dev, unsigned vector);
+    MSIMessage msg;
+    int nr = vector - vdev->msi_vectors;
+
+    if (!event_notifier_test_and_clear(&vector->interrupt)) {
+        return;
+    }
+
+    if (vdev->interrupt == VFIO_INT_MSIX) {
+        get_msg = msix_get_message;
+        notify = msix_notify;
+
+        /* A masked vector firing needs to use the PBA, enable it */
+        if (msix_is_masked(&vdev->pdev, nr)) {
+            set_bit(nr, vdev->msix->pending);
+            memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
+            trace_vfio_msix_pba_enable(vdev->vbasedev.name);
+        }
+    } else if (vdev->interrupt == VFIO_INT_MSI) {
+        get_msg = msi_get_message;
+        notify = msi_notify;
+    } else {
+        abort();
+    }
+
+    msg = get_msg(&vdev->pdev, nr);
+    trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
+    notify(&vdev->pdev, nr);
+}
+
+static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
+{
+    struct vfio_irq_set *irq_set;
+    int ret = 0, i, argsz;
+    int32_t *fds;
+
+    argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = vdev->nr_vectors;
+    fds = (int32_t *)&irq_set->data;
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        int fd = -1;
+
+        /*
+         * MSI vs MSI-X - The guest has direct access to MSI mask and pending
+         * bits, therefore we always use the KVM signaling path when setup.
+         * MSI-X mask and pending bits are emulated, so we want to use the
+         * KVM signaling path only when configured and unmasked.
+         */
+        if (vdev->msi_vectors[i].use) {
+            if (vdev->msi_vectors[i].virq < 0 ||
+                (msix && msix_is_masked(&vdev->pdev, i))) {
+                fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
+            } else {
+                fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
+            }
+        }
+
+        fds[i] = fd;
+    }
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+    g_free(irq_set);
+
+    return ret;
+}
+
+static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
+                                  int vector_n, bool msix)
+{
+    int virq;
+
+    if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
+        return;
+    }
+
+    if (event_notifier_init(&vector->kvm_interrupt, 0)) {
+        return;
+    }
+
+    virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev);
+    if (virq < 0) {
+        event_notifier_cleanup(&vector->kvm_interrupt);
+        return;
+    }
+
+    if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
+                                       NULL, virq) < 0) {
+        kvm_irqchip_release_virq(kvm_state, virq);
+        event_notifier_cleanup(&vector->kvm_interrupt);
+        return;
+    }
+
+    vector->virq = virq;
+}
+
+static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
+{
+    kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
+                                          vector->virq);
+    kvm_irqchip_release_virq(kvm_state, vector->virq);
+    vector->virq = -1;
+    event_notifier_cleanup(&vector->kvm_interrupt);
+}
+
+static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
+                                     PCIDevice *pdev)
+{
+    kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
+    kvm_irqchip_commit_routes(kvm_state);
+}
+
+static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
+                                   MSIMessage *msg, IOHandler *handler)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    VFIOMSIVector *vector;
+    int ret;
+
+    trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
+
+    vector = &vdev->msi_vectors[nr];
+
+    if (!vector->use) {
+        vector->vdev = vdev;
+        vector->virq = -1;
+        if (event_notifier_init(&vector->interrupt, 0)) {
+            error_report("vfio: Error: event_notifier_init failed");
+        }
+        vector->use = true;
+        msix_vector_use(pdev, nr);
+    }
+
+    qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                        handler, NULL, vector);
+
+    /*
+     * Attempt to enable route through KVM irqchip,
+     * default to userspace handling if unavailable.
+     */
+    if (vector->virq >= 0) {
+        if (!msg) {
+            vfio_remove_kvm_msi_virq(vector);
+        } else {
+            vfio_update_kvm_msi_virq(vector, *msg, pdev);
+        }
+    } else {
+        vfio_add_kvm_msi_virq(vdev, vector, nr, true);
+    }
+
+    /*
+     * We don't want to have the host allocate all possible MSI vectors
+     * for a device if they're not in use, so we shutdown and incrementally
+     * increase them as needed.
+     */
+    if (vdev->nr_vectors < nr + 1) {
+        vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
+        vdev->nr_vectors = nr + 1;
+        ret = vfio_enable_vectors(vdev, true);
+        if (ret) {
+            error_report("vfio: failed to enable vectors, %d", ret);
+        }
+    } else {
+        int argsz;
+        struct vfio_irq_set *irq_set;
+        int32_t *pfd;
+
+        argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+        irq_set = g_malloc0(argsz);
+        irq_set->argsz = argsz;
+        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                         VFIO_IRQ_SET_ACTION_TRIGGER;
+        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+        irq_set->start = nr;
+        irq_set->count = 1;
+        pfd = (int32_t *)&irq_set->data;
+
+        if (vector->virq >= 0) {
+            *pfd = event_notifier_get_fd(&vector->kvm_interrupt);
+        } else {
+            *pfd = event_notifier_get_fd(&vector->interrupt);
+        }
+
+        ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+        g_free(irq_set);
+        if (ret) {
+            error_report("vfio: failed to modify vector, %d", ret);
+        }
+    }
+
+    /* Disable PBA emulation when nothing more is pending. */
+    clear_bit(nr, vdev->msix->pending);
+    if (find_first_bit(vdev->msix->pending,
+                       vdev->nr_vectors) == vdev->nr_vectors) {
+        memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
+        trace_vfio_msix_pba_disable(vdev->vbasedev.name);
+    }
+
+    return 0;
+}
+
+static int vfio_msix_vector_use(PCIDevice *pdev,
+                                unsigned int nr, MSIMessage msg)
+{
+    return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
+}
+
+static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    VFIOMSIVector *vector = &vdev->msi_vectors[nr];
+
+    trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
+
+    /*
+     * There are still old guests that mask and unmask vectors on every
+     * interrupt.  If we're using QEMU bypass with a KVM irqfd, leave all of
+     * the KVM setup in place, simply switch VFIO to use the non-bypass
+     * eventfd.  We'll then fire the interrupt through QEMU and the MSI-X
+     * core will mask the interrupt and set pending bits, allowing it to
+     * be re-asserted on unmask.  Nothing to do if already using QEMU mode.
+     */
+    if (vector->virq >= 0) {
+        int argsz;
+        struct vfio_irq_set *irq_set;
+        int32_t *pfd;
+
+        argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+        irq_set = g_malloc0(argsz);
+        irq_set->argsz = argsz;
+        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                         VFIO_IRQ_SET_ACTION_TRIGGER;
+        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+        irq_set->start = nr;
+        irq_set->count = 1;
+        pfd = (int32_t *)&irq_set->data;
+
+        *pfd = event_notifier_get_fd(&vector->interrupt);
+
+        ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+        g_free(irq_set);
+    }
+}
+
+static void vfio_msix_enable(VFIOPCIDevice *vdev)
+{
+    vfio_disable_interrupts(vdev);
+
+    vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
+
+    vdev->interrupt = VFIO_INT_MSIX;
+
+    /*
+     * Some communication channels between VF & PF or PF & fw rely on the
+     * physical state of the device and expect that enabling MSI-X from the
+     * guest enables the same on the host.  When our guest is Linux, the
+     * guest driver call to pci_enable_msix() sets the enabling bit in the
+     * MSI-X capability, but leaves the vector table masked.  We therefore
+     * can't rely on a vector_use callback (from request_irq() in the guest)
+     * to switch the physical device into MSI-X mode because that may come a
+     * long time after pci_enable_msix().  This code enables vector 0 with
+     * triggering to userspace, then immediately release the vector, leaving
+     * the physical device with no vectors enabled, but MSI-X enabled, just
+     * like the guest view.
+     */
+    vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
+    vfio_msix_vector_release(&vdev->pdev, 0);
+
+    if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
+                                  vfio_msix_vector_release, NULL)) {
+        error_report("vfio: msix_set_vector_notifiers failed");
+    }
+
+    trace_vfio_msix_enable(vdev->vbasedev.name);
+}
+
+static void vfio_msi_enable(VFIOPCIDevice *vdev)
+{
+    int ret, i;
+
+    vfio_disable_interrupts(vdev);
+
+    vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
+retry:
+    vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+
+        vector->vdev = vdev;
+        vector->virq = -1;
+        vector->use = true;
+
+        if (event_notifier_init(&vector->interrupt, 0)) {
+            error_report("vfio: Error: event_notifier_init failed");
+        }
+
+        qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                            vfio_msi_interrupt, NULL, vector);
+
+        /*
+         * Attempt to enable route through KVM irqchip,
+         * default to userspace handling if unavailable.
+         */
+        vfio_add_kvm_msi_virq(vdev, vector, i, false);
+    }
+
+    /* Set interrupt type prior to possible interrupts */
+    vdev->interrupt = VFIO_INT_MSI;
+
+    ret = vfio_enable_vectors(vdev, false);
+    if (ret) {
+        if (ret < 0) {
+            error_report("vfio: Error: Failed to setup MSI fds: %m");
+        } else if (ret != vdev->nr_vectors) {
+            error_report("vfio: Error: Failed to enable %d "
+                         "MSI vectors, retry with %d", vdev->nr_vectors, ret);
+        }
+
+        for (i = 0; i < vdev->nr_vectors; i++) {
+            VFIOMSIVector *vector = &vdev->msi_vectors[i];
+            if (vector->virq >= 0) {
+                vfio_remove_kvm_msi_virq(vector);
+            }
+            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                NULL, NULL, NULL);
+            event_notifier_cleanup(&vector->interrupt);
+        }
+
+        g_free(vdev->msi_vectors);
+
+        if (ret > 0 && ret != vdev->nr_vectors) {
+            vdev->nr_vectors = ret;
+            goto retry;
+        }
+        vdev->nr_vectors = 0;
+
+        /*
+         * Failing to setup MSI doesn't really fall within any specification.
+         * Let's try leaving interrupts disabled and hope the guest figures
+         * out to fall back to INTx for this device.
+         */
+        error_report("vfio: Error: Failed to enable MSI");
+        vdev->interrupt = VFIO_INT_NONE;
+
+        return;
+    }
+
+    trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
+}
+
+static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+        if (vdev->msi_vectors[i].use) {
+            if (vector->virq >= 0) {
+                vfio_remove_kvm_msi_virq(vector);
+            }
+            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                NULL, NULL, NULL);
+            event_notifier_cleanup(&vector->interrupt);
+        }
+    }
+
+    g_free(vdev->msi_vectors);
+    vdev->msi_vectors = NULL;
+    vdev->nr_vectors = 0;
+    vdev->interrupt = VFIO_INT_NONE;
+
+    vfio_intx_enable(vdev);
+}
+
+static void vfio_msix_disable(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    msix_unset_vector_notifiers(&vdev->pdev);
+
+    /*
+     * MSI-X will only release vectors if MSI-X is still enabled on the
+     * device, check through the rest and release it ourselves if necessary.
+     */
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        if (vdev->msi_vectors[i].use) {
+            vfio_msix_vector_release(&vdev->pdev, i);
+            msix_vector_unuse(&vdev->pdev, i);
+        }
+    }
+
+    if (vdev->nr_vectors) {
+        vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
+    }
+
+    vfio_msi_disable_common(vdev);
+
+    memset(vdev->msix->pending, 0,
+           BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
+
+    trace_vfio_msix_disable(vdev->vbasedev.name);
+}
+
+static void vfio_msi_disable(VFIOPCIDevice *vdev)
+{
+    vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
+    vfio_msi_disable_common(vdev);
+
+    trace_vfio_msi_disable(vdev->vbasedev.name);
+}
+
+static void vfio_update_msi(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+        MSIMessage msg;
+
+        if (!vector->use || vector->virq < 0) {
+            continue;
+        }
+
+        msg = msi_get_message(&vdev->pdev, i);
+        vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
+    }
+}
+
+static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
+{
+    struct vfio_region_info *reg_info;
+    uint64_t size;
+    off_t off = 0;
+    ssize_t bytes;
+
+    if (vfio_get_region_info(&vdev->vbasedev,
+                             VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
+        error_report("vfio: Error getting ROM info: %m");
+        return;
+    }
+
+    trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
+                            (unsigned long)reg_info->offset,
+                            (unsigned long)reg_info->flags);
+
+    vdev->rom_size = size = reg_info->size;
+    vdev->rom_offset = reg_info->offset;
+
+    g_free(reg_info);
+
+    if (!vdev->rom_size) {
+        vdev->rom_read_failed = true;
+        error_report("vfio-pci: Cannot read device rom at "
+                    "%s", vdev->vbasedev.name);
+        error_printf("Device option ROM contents are probably invalid "
+                    "(check dmesg).\nSkip option ROM probe with rombar=0, "
+                    "or load from file with romfile=\n");
+        return;
+    }
+
+    vdev->rom = g_malloc(size);
+    memset(vdev->rom, 0xff, size);
+
+    while (size) {
+        bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
+                      size, vdev->rom_offset + off);
+        if (bytes == 0) {
+            break;
+        } else if (bytes > 0) {
+            off += bytes;
+            size -= bytes;
+        } else {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            }
+            error_report("vfio: Error reading device ROM: %m");
+            break;
+        }
+    }
+
+    /*
+     * Test the ROM signature against our device, if the vendor is correct
+     * but the device ID doesn't match, store the correct device ID and
+     * recompute the checksum.  Intel IGD devices need this and are known
+     * to have bogus checksums so we can't simply adjust the checksum.
+     */
+    if (pci_get_word(vdev->rom) == 0xaa55 &&
+        pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
+        !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
+        uint16_t vid, did;
+
+        vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
+        did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
+
+        if (vid == vdev->vendor_id && did != vdev->device_id) {
+            int i;
+            uint8_t csum, *data = vdev->rom;
+
+            pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
+                         vdev->device_id);
+            data[6] = 0;
+
+            for (csum = 0, i = 0; i < vdev->rom_size; i++) {
+                csum += data[i];
+            }
+
+            data[6] = -csum;
+        }
+    }
+}
+
+static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
+{
+    VFIOPCIDevice *vdev = opaque;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } val;
+    uint64_t data = 0;
+
+    /* Load the ROM lazily when the guest tries to read it */
+    if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
+        vfio_pci_load_rom(vdev);
+    }
+
+    memcpy(&val, vdev->rom + addr,
+           (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
+
+    switch (size) {
+    case 1:
+        data = val.byte;
+        break;
+    case 2:
+        data = le16_to_cpu(val.word);
+        break;
+    case 4:
+        data = le32_to_cpu(val.dword);
+        break;
+    default:
+        hw_error("vfio: unsupported read size, %d bytes\n", size);
+        break;
+    }
+
+    trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
+
+    return data;
+}
+
+static void vfio_rom_write(void *opaque, hwaddr addr,
+                           uint64_t data, unsigned size)
+{
+}
+
+static const MemoryRegionOps vfio_rom_ops = {
+    .read = vfio_rom_read,
+    .write = vfio_rom_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
+{
+    uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
+    off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
+    DeviceState *dev = DEVICE(vdev);
+    char *name;
+    int fd = vdev->vbasedev.fd;
+
+    if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
+        /* Since pci handles romfile, just print a message and return */
+        if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
+            error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified romfile\n",
+                         vdev->vbasedev.name);
+        }
+        return;
+    }
+
+    /*
+     * Use the same size ROM BAR as the physical device.  The contents
+     * will get filled in later when the guest tries to read it.
+     */
+    if (pread(fd, &orig, 4, offset) != 4 ||
+        pwrite(fd, &size, 4, offset) != 4 ||
+        pread(fd, &size, 4, offset) != 4 ||
+        pwrite(fd, &orig, 4, offset) != 4) {
+        error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
+        return;
+    }
+
+    size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
+
+    if (!size) {
+        return;
+    }
+
+    if (vfio_blacklist_opt_rom(vdev)) {
+        if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
+            error_printf("Warning : Device at %s is known to cause system instability issues during option rom execution. Proceeding anyway since user specified non zero value for rombar\n",
+                         vdev->vbasedev.name);
+        } else {
+            error_printf("Warning : Rom loading for device at %s has been disabled due to system instability issues. Specify rombar=1 or romfile to force\n",
+                         vdev->vbasedev.name);
+            return;
+        }
+    }
+
+    trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
+
+    name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
+
+    memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
+                          &vfio_rom_ops, vdev, name, size);
+    g_free(name);
+
+    pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
+                     PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
+
+    vdev->pdev.has_rom = true;
+    vdev->rom_read_failed = false;
+}
+
+void vfio_vga_write(void *opaque, hwaddr addr,
+                           uint64_t data, unsigned size)
+{
+    VFIOVGARegion *region = opaque;
+    VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+    off_t offset = vga->fd_offset + region->offset + addr;
+
+    switch (size) {
+    case 1:
+        buf.byte = data;
+        break;
+    case 2:
+        buf.word = cpu_to_le16(data);
+        break;
+    case 4:
+        buf.dword = cpu_to_le32(data);
+        break;
+    default:
+        hw_error("vfio: unsupported write size, %d bytes", size);
+        break;
+    }
+
+    if (pwrite(vga->fd, &buf, size, offset) != size) {
+        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
+                     __func__, region->offset + addr, data, size);
+    }
+
+    trace_vfio_vga_write(region->offset + addr, data, size);
+}
+
+uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
+{
+    VFIOVGARegion *region = opaque;
+    VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+    uint64_t data = 0;
+    off_t offset = vga->fd_offset + region->offset + addr;
+
+    if (pread(vga->fd, &buf, size, offset) != size) {
+        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
+                     __func__, region->offset + addr, size);
+        return (uint64_t)-1;
+    }
+
+    switch (size) {
+    case 1:
+        data = buf.byte;
+        break;
+    case 2:
+        data = le16_to_cpu(buf.word);
+        break;
+    case 4:
+        data = le32_to_cpu(buf.dword);
+        break;
+    default:
+        hw_error("vfio: unsupported read size, %d bytes", size);
+        break;
+    }
+
+    trace_vfio_vga_read(region->offset + addr, size, data);
+
+    return data;
+}
+
+static const MemoryRegionOps vfio_vga_ops = {
+    .read = vfio_vga_read,
+    .write = vfio_vga_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * PCI config space
+ */
+uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
+
+    memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
+    emu_bits = le32_to_cpu(emu_bits);
+
+    if (emu_bits) {
+        emu_val = pci_default_read_config(pdev, addr, len);
+    }
+
+    if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
+        ssize_t ret;
+
+        ret = pread(vdev->vbasedev.fd, &phys_val, len,
+                    vdev->config_offset + addr);
+        if (ret != len) {
+            error_report("%s(%s, 0x%x, 0x%x) failed: %m",
+                         __func__, vdev->vbasedev.name, addr, len);
+            return -errno;
+        }
+        phys_val = le32_to_cpu(phys_val);
+    }
+
+    val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
+
+    trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
+
+    return val;
+}
+
+void vfio_pci_write_config(PCIDevice *pdev,
+                           uint32_t addr, uint32_t val, int len)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    uint32_t val_le = cpu_to_le32(val);
+
+    trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
+
+    /* Write everything to VFIO, let it filter out what we can't write */
+    if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
+                != len) {
+        error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
+                     __func__, vdev->vbasedev.name, addr, val, len);
+    }
+
+    /* MSI/MSI-X Enabling/Disabling */
+    if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
+        ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
+        int is_enabled, was_enabled = msi_enabled(pdev);
+
+        pci_default_write_config(pdev, addr, val, len);
+
+        is_enabled = msi_enabled(pdev);
+
+        if (!was_enabled) {
+            if (is_enabled) {
+                vfio_msi_enable(vdev);
+            }
+        } else {
+            if (!is_enabled) {
+                vfio_msi_disable(vdev);
+            } else {
+                vfio_update_msi(vdev);
+            }
+        }
+    } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
+        ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
+        int is_enabled, was_enabled = msix_enabled(pdev);
+
+        pci_default_write_config(pdev, addr, val, len);
+
+        is_enabled = msix_enabled(pdev);
+
+        if (!was_enabled && is_enabled) {
+            vfio_msix_enable(vdev);
+        } else if (was_enabled && !is_enabled) {
+            vfio_msix_disable(vdev);
+        }
+    } else {
+        /* Write everything to QEMU to keep emulated bits correct */
+        pci_default_write_config(pdev, addr, val, len);
+    }
+}
+
+/*
+ * Interrupt setup
+ */
+static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
+{
+    /*
+     * More complicated than it looks.  Disabling MSI/X transitions the
+     * device to INTx mode (if supported).  Therefore we need to first
+     * disable MSI/X and then cleanup by disabling INTx.
+     */
+    if (vdev->interrupt == VFIO_INT_MSIX) {
+        vfio_msix_disable(vdev);
+    } else if (vdev->interrupt == VFIO_INT_MSI) {
+        vfio_msi_disable(vdev);
+    }
+
+    if (vdev->interrupt == VFIO_INT_INTx) {
+        vfio_intx_disable(vdev);
+    }
+}
+
+static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos)
+{
+    uint16_t ctrl;
+    bool msi_64bit, msi_maskbit;
+    int ret, entries;
+    Error *err = NULL;
+
+    if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
+              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
+        return -errno;
+    }
+    ctrl = le16_to_cpu(ctrl);
+
+    msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
+    msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
+    entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
+
+    trace_vfio_msi_setup(vdev->vbasedev.name, pos);
+
+    ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            return 0;
+        }
+        error_prepend(&err, "vfio: msi_init failed: ");
+        error_report_err(err);
+        return ret;
+    }
+    vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
+
+    return 0;
+}
+
+static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
+{
+    off_t start, end;
+    VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
+
+    /*
+     * We expect to find a single mmap covering the whole BAR, anything else
+     * means it's either unsupported or already setup.
+     */
+    if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
+        region->size != region->mmaps[0].size) {
+        return;
+    }
+
+    /* MSI-X table start and end aligned to host page size */
+    start = vdev->msix->table_offset & qemu_real_host_page_mask;
+    end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
+                               (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
+
+    /*
+     * Does the MSI-X table cover the beginning of the BAR?  The whole BAR?
+     * NB - Host page size is necessarily a power of two and so is the PCI
+     * BAR (not counting EA yet), therefore if we have host page aligned
+     * @start and @end, then any remainder of the BAR before or after those
+     * must be at least host page sized and therefore mmap'able.
+     */
+    if (!start) {
+        if (end >= region->size) {
+            region->nr_mmaps = 0;
+            g_free(region->mmaps);
+            region->mmaps = NULL;
+            trace_vfio_msix_fixup(vdev->vbasedev.name,
+                                  vdev->msix->table_bar, 0, 0);
+        } else {
+            region->mmaps[0].offset = end;
+            region->mmaps[0].size = region->size - end;
+            trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[0].offset,
+                              region->mmaps[0].offset + region->mmaps[0].size);
+        }
+
+    /* Maybe it's aligned at the end of the BAR */
+    } else if (end >= region->size) {
+        region->mmaps[0].size = start;
+        trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[0].offset,
+                              region->mmaps[0].offset + region->mmaps[0].size);
+
+    /* Otherwise it must split the BAR */
+    } else {
+        region->nr_mmaps = 2;
+        region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
+
+        memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
+
+        region->mmaps[0].size = start;
+        trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[0].offset,
+                              region->mmaps[0].offset + region->mmaps[0].size);
+
+        region->mmaps[1].offset = end;
+        region->mmaps[1].size = region->size - end;
+        trace_vfio_msix_fixup(vdev->vbasedev.name,
+                              vdev->msix->table_bar, region->mmaps[1].offset,
+                              region->mmaps[1].offset + region->mmaps[1].size);
+    }
+}
+
+/*
+ * We don't have any control over how pci_add_capability() inserts
+ * capabilities into the chain.  In order to setup MSI-X we need a
+ * MemoryRegion for the BAR.  In order to setup the BAR and not
+ * attempt to mmap the MSI-X table area, which VFIO won't allow, we
+ * need to first look for where the MSI-X table lives.  So we
+ * unfortunately split MSI-X setup across two functions.
+ */
+static int vfio_msix_early_setup(VFIOPCIDevice *vdev)
+{
+    uint8_t pos;
+    uint16_t ctrl;
+    uint32_t table, pba;
+    int fd = vdev->vbasedev.fd;
+    VFIOMSIXInfo *msix;
+
+    pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
+    if (!pos) {
+        return 0;
+    }
+
+    if (pread(fd, &ctrl, sizeof(ctrl),
+              vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
+        return -errno;
+    }
+
+    if (pread(fd, &table, sizeof(table),
+              vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
+        return -errno;
+    }
+
+    if (pread(fd, &pba, sizeof(pba),
+              vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
+        return -errno;
+    }
+
+    ctrl = le16_to_cpu(ctrl);
+    table = le32_to_cpu(table);
+    pba = le32_to_cpu(pba);
+
+    msix = g_malloc0(sizeof(*msix));
+    msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
+    msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
+    msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
+    msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
+    msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+
+    /*
+     * Test the size of the pba_offset variable and catch if it extends outside
+     * of the specified BAR. If it is the case, we need to apply a hardware
+     * specific quirk if the device is known or we have a broken configuration.
+     */
+    if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
+        /*
+         * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
+         * adapters. The T5 hardware returns an incorrect value of 0x8000 for
+         * the VF PBA offset while the BAR itself is only 8k. The correct value
+         * is 0x1000, so we hard code that here.
+         */
+        if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
+            (vdev->device_id & 0xff00) == 0x5800) {
+            msix->pba_offset = 0x1000;
+        } else {
+            error_report("vfio: Hardware reports invalid configuration, "
+                         "MSIX PBA outside of specified BAR");
+            g_free(msix);
+            return -EINVAL;
+        }
+    }
+
+    trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
+                                msix->table_offset, msix->entries);
+    vdev->msix = msix;
+
+    vfio_pci_fixup_msix_region(vdev);
+
+    return 0;
+}
+
+static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos)
+{
+    int ret;
+
+    vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
+                                    sizeof(unsigned long));
+    ret = msix_init(&vdev->pdev, vdev->msix->entries,
+                    vdev->bars[vdev->msix->table_bar].region.mem,
+                    vdev->msix->table_bar, vdev->msix->table_offset,
+                    vdev->bars[vdev->msix->pba_bar].region.mem,
+                    vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            return 0;
+        }
+        error_report("vfio: msix_init failed");
+        return ret;
+    }
+
+    /*
+     * The PCI spec suggests that devices provide additional alignment for
+     * MSI-X structures and avoid overlapping non-MSI-X related registers.
+     * For an assigned device, this hopefully means that emulation of MSI-X
+     * structures does not affect the performance of the device.  If devices
+     * fail to provide that alignment, a significant performance penalty may
+     * result, for instance Mellanox MT27500 VFs:
+     * http://www.spinics.net/lists/kvm/msg125881.html
+     *
+     * The PBA is simply not that important for such a serious regression and
+     * most drivers do not appear to look at it.  The solution for this is to
+     * disable the PBA MemoryRegion unless it's being used.  We disable it
+     * here and only enable it if a masked vector fires through QEMU.  As the
+     * vector-use notifier is called, which occurs on unmask, we test whether
+     * PBA emulation is needed and again disable if not.
+     */
+    memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
+
+    return 0;
+}
+
+static void vfio_teardown_msi(VFIOPCIDevice *vdev)
+{
+    msi_uninit(&vdev->pdev);
+
+    if (vdev->msix) {
+        msix_uninit(&vdev->pdev,
+                    vdev->bars[vdev->msix->table_bar].region.mem,
+                    vdev->bars[vdev->msix->pba_bar].region.mem);
+        g_free(vdev->msix->pending);
+    }
+}
+
+/*
+ * Resource setup
+ */
+static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
+    }
+}
+
+static void vfio_bar_setup(VFIOPCIDevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+
+    uint32_t pci_bar;
+    uint8_t type;
+    int ret;
+
+    /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
+    if (!bar->region.size) {
+        return;
+    }
+
+    /* Determine what type of BAR this is for registration */
+    ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
+                vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
+    if (ret != sizeof(pci_bar)) {
+        error_report("vfio: Failed to read BAR %d (%m)", nr);
+        return;
+    }
+
+    pci_bar = le32_to_cpu(pci_bar);
+    bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
+    bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
+    type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
+                                    ~PCI_BASE_ADDRESS_MEM_MASK);
+
+    if (vfio_region_mmap(&bar->region)) {
+        error_report("Failed to mmap %s BAR %d. Performance may be slow",
+                     vdev->vbasedev.name, nr);
+    }
+
+    pci_register_bar(&vdev->pdev, nr, type, bar->region.mem);
+}
+
+static void vfio_bars_setup(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_bar_setup(vdev, i);
+    }
+}
+
+static void vfio_bars_exit(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_bar_quirk_exit(vdev, i);
+        vfio_region_exit(&vdev->bars[i].region);
+    }
+
+    if (vdev->vga) {
+        pci_unregister_vga(&vdev->pdev);
+        vfio_vga_quirk_exit(vdev);
+    }
+}
+
+static void vfio_bars_finalize(VFIOPCIDevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_bar_quirk_finalize(vdev, i);
+        vfio_region_finalize(&vdev->bars[i].region);
+    }
+
+    if (vdev->vga) {
+        vfio_vga_quirk_finalize(vdev);
+        for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
+            object_unparent(OBJECT(&vdev->vga->region[i].mem));
+        }
+        g_free(vdev->vga);
+    }
+}
+
+/*
+ * General setup
+ */
+static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
+{
+    uint8_t tmp;
+    uint16_t next = PCI_CONFIG_SPACE_SIZE;
+
+    for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
+         tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
+        if (tmp > pos && tmp < next) {
+            next = tmp;
+        }
+    }
+
+    return next - pos;
+}
+
+
+static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
+{
+    uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
+
+    for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
+        tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
+        if (tmp > pos && tmp < next) {
+            next = tmp;
+        }
+    }
+
+    return next - pos;
+}
+
+static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
+{
+    pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
+}
+
+static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
+                                   uint16_t val, uint16_t mask)
+{
+    vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
+    vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
+    vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
+}
+
+static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
+{
+    pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
+}
+
+static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
+                                   uint32_t val, uint32_t mask)
+{
+    vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
+    vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
+    vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
+}
+
+static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size)
+{
+    uint16_t flags;
+    uint8_t type;
+
+    flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
+    type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
+
+    if (type != PCI_EXP_TYPE_ENDPOINT &&
+        type != PCI_EXP_TYPE_LEG_END &&
+        type != PCI_EXP_TYPE_RC_END) {
+
+        error_report("vfio: Assignment of PCIe type 0x%x "
+                     "devices is not currently supported", type);
+        return -EINVAL;
+    }
+
+    if (!pci_bus_is_express(vdev->pdev.bus)) {
+        PCIBus *bus = vdev->pdev.bus;
+        PCIDevice *bridge;
+
+        /*
+         * Traditionally PCI device assignment exposes the PCIe capability
+         * as-is on non-express buses.  The reason being that some drivers
+         * simply assume that it's there, for example tg3.  However when
+         * we're running on a native PCIe machine type, like Q35, we need
+         * to hide the PCIe capability.  The reason for this is twofold;
+         * first Windows guests get a Code 10 error when the PCIe capability
+         * is exposed in this configuration.  Therefore express devices won't
+         * work at all unless they're attached to express buses in the VM.
+         * Second, a native PCIe machine introduces the possibility of fine
+         * granularity IOMMUs supporting both translation and isolation.
+         * Guest code to discover the IOMMU visibility of a device, such as
+         * IOMMU grouping code on Linux, is very aware of device types and
+         * valid transitions between bus types.  An express device on a non-
+         * express bus is not a valid combination on bare metal systems.
+         *
+         * Drivers that require a PCIe capability to make the device
+         * functional are simply going to need to have their devices placed
+         * on a PCIe bus in the VM.
+         */
+        while (!pci_bus_is_root(bus)) {
+            bridge = pci_bridge_get_device(bus);
+            bus = bridge->bus;
+        }
+
+        if (pci_bus_is_express(bus)) {
+            return 0;
+        }
+
+    } else if (pci_bus_is_root(vdev->pdev.bus)) {
+        /*
+         * On a Root Complex bus Endpoints become Root Complex Integrated
+         * Endpoints, which changes the type and clears the LNK & LNK2 fields.
+         */
+        if (type == PCI_EXP_TYPE_ENDPOINT) {
+            vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
+                                   PCI_EXP_TYPE_RC_END << 4,
+                                   PCI_EXP_FLAGS_TYPE);
+
+            /* Link Capabilities, Status, and Control goes away */
+            if (size > PCI_EXP_LNKCTL) {
+                vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
+                vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
+                vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
+
+#ifndef PCI_EXP_LNKCAP2
+#define PCI_EXP_LNKCAP2 44
+#endif
+#ifndef PCI_EXP_LNKSTA2
+#define PCI_EXP_LNKSTA2 50
+#endif
+                /* Link 2 Capabilities, Status, and Control goes away */
+                if (size > PCI_EXP_LNKCAP2) {
+                    vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
+                    vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
+                    vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
+                }
+            }
+
+        } else if (type == PCI_EXP_TYPE_LEG_END) {
+            /*
+             * Legacy endpoints don't belong on the root complex.  Windows
+             * seems to be happier with devices if we skip the capability.
+             */
+            return 0;
+        }
+
+    } else {
+        /*
+         * Convert Root Complex Integrated Endpoints to regular endpoints.
+         * These devices don't support LNK/LNK2 capabilities, so make them up.
+         */
+        if (type == PCI_EXP_TYPE_RC_END) {
+            vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
+                                   PCI_EXP_TYPE_ENDPOINT << 4,
+                                   PCI_EXP_FLAGS_TYPE);
+            vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
+                                   PCI_EXP_LNK_MLW_1 | PCI_EXP_LNK_LS_25, ~0);
+            vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
+        }
+
+        /* Mark the Link Status bits as emulated to allow virtual negotiation */
+        vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA,
+                               pci_get_word(vdev->pdev.config + pos +
+                                            PCI_EXP_LNKSTA),
+                               PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS);
+    }
+
+    pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size);
+    if (pos >= 0) {
+        vdev->pdev.exp.exp_cap = pos;
+    }
+
+    return pos;
+}
+
+static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
+{
+    uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
+
+    if (cap & PCI_EXP_DEVCAP_FLR) {
+        trace_vfio_check_pcie_flr(vdev->vbasedev.name);
+        vdev->has_flr = true;
+    }
+}
+
+static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
+{
+    uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
+
+    if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
+        trace_vfio_check_pm_reset(vdev->vbasedev.name);
+        vdev->has_pm_reset = true;
+    }
+}
+
+static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
+{
+    uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
+
+    if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
+        trace_vfio_check_af_flr(vdev->vbasedev.name);
+        vdev->has_flr = true;
+    }
+}
+
+static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint8_t cap_id, next, size;
+    int ret;
+
+    cap_id = pdev->config[pos];
+    next = pdev->config[pos + PCI_CAP_LIST_NEXT];
+
+    /*
+     * If it becomes important to configure capabilities to their actual
+     * size, use this as the default when it's something we don't recognize.
+     * Since QEMU doesn't actually handle many of the config accesses,
+     * exact size doesn't seem worthwhile.
+     */
+    size = vfio_std_cap_max_size(pdev, pos);
+
+    /*
+     * pci_add_capability always inserts the new capability at the head
+     * of the chain.  Therefore to end up with a chain that matches the
+     * physical device, we insert from the end by making this recursive.
+     * This is also why we pre-calculate size above as cached config space
+     * will be changed as we unwind the stack.
+     */
+    if (next) {
+        ret = vfio_add_std_cap(vdev, next);
+        if (ret) {
+            return ret;
+        }
+    } else {
+        /* Begin the rebuild, use QEMU emulated list bits */
+        pdev->config[PCI_CAPABILITY_LIST] = 0;
+        vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
+        vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
+    }
+
+    /* Use emulated next pointer to allow dropping caps */
+    pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
+
+    switch (cap_id) {
+    case PCI_CAP_ID_MSI:
+        ret = vfio_msi_setup(vdev, pos);
+        break;
+    case PCI_CAP_ID_EXP:
+        vfio_check_pcie_flr(vdev, pos);
+        ret = vfio_setup_pcie_cap(vdev, pos, size);
+        break;
+    case PCI_CAP_ID_MSIX:
+        ret = vfio_msix_setup(vdev, pos);
+        break;
+    case PCI_CAP_ID_PM:
+        vfio_check_pm_reset(vdev, pos);
+        vdev->pm_cap = pos;
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
+    case PCI_CAP_ID_AF:
+        vfio_check_af_flr(vdev, pos);
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
+    default:
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
+    }
+
+    if (ret < 0) {
+        error_report("vfio: %s Error adding PCI capability "
+                     "0x%x[0x%x]@0x%x: %d", vdev->vbasedev.name,
+                     cap_id, size, pos, ret);
+        return ret;
+    }
+
+    return 0;
+}
+
+static int vfio_add_ext_cap(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint32_t header;
+    uint16_t cap_id, next, size;
+    uint8_t cap_ver;
+    uint8_t *config;
+
+    /* Only add extended caps if we have them and the guest can see them */
+    if (!pci_is_express(pdev) || !pci_bus_is_express(pdev->bus) ||
+        !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
+        return 0;
+    }
+
+    /*
+     * pcie_add_capability always inserts the new capability at the tail
+     * of the chain.  Therefore to end up with a chain that matches the
+     * physical device, we cache the config space to avoid overwriting
+     * the original config space when we parse the extended capabilities.
+     */
+    config = g_memdup(pdev->config, vdev->config_size);
+
+    /*
+     * Extended capabilities are chained with each pointing to the next, so we
+     * can drop anything other than the head of the chain simply by modifying
+     * the previous next pointer.  For the head of the chain, we can modify the
+     * capability ID to something that cannot match a valid capability.  ID
+     * 0 is reserved for this since absence of capabilities is indicated by
+     * 0 for the ID, version, AND next pointer.  However, pcie_add_capability()
+     * uses ID 0 as reserved for list management and will incorrectly match and
+     * assert if we attempt to pre-load the head of the chain with with this
+     * ID.  Use ID 0xFFFF temporarily since it is also seems to be reserved in
+     * part for identifying absence of capabilities in a root complex register
+     * block.  If the ID still exists after adding capabilities, switch back to
+     * zero.  We'll mark this entire first dword as emulated for this purpose.
+     */
+    pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
+                 PCI_EXT_CAP(0xFFFF, 0, 0));
+    pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
+    pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
+
+    for (next = PCI_CONFIG_SPACE_SIZE; next;
+         next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
+        header = pci_get_long(config + next);
+        cap_id = PCI_EXT_CAP_ID(header);
+        cap_ver = PCI_EXT_CAP_VER(header);
+
+        /*
+         * If it becomes important to configure extended capabilities to their
+         * actual size, use this as the default when it's something we don't
+         * recognize. Since QEMU doesn't actually handle many of the config
+         * accesses, exact size doesn't seem worthwhile.
+         */
+        size = vfio_ext_cap_max_size(config, next);
+
+        /* Use emulated next pointer to allow dropping extended caps */
+        pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
+                                   PCI_EXT_CAP_NEXT_MASK);
+
+        switch (cap_id) {
+        case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
+        case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
+            trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
+            break;
+        default:
+            pcie_add_capability(pdev, cap_id, cap_ver, next, size);
+        }
+
+    }
+
+    /* Cleanup chain head ID if necessary */
+    if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
+        pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
+    }
+
+    g_free(config);
+    return 0;
+}
+
+static int vfio_add_capabilities(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    int ret;
+
+    if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
+        !pdev->config[PCI_CAPABILITY_LIST]) {
+        return 0; /* Nothing to add */
+    }
+
+    ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
+    if (ret) {
+        return ret;
+    }
+
+    return vfio_add_ext_cap(vdev);
+}
+
+static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint16_t cmd;
+
+    vfio_disable_interrupts(vdev);
+
+    /* Make sure the device is in D0 */
+    if (vdev->pm_cap) {
+        uint16_t pmcsr;
+        uint8_t state;
+
+        pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
+        state = pmcsr & PCI_PM_CTRL_STATE_MASK;
+        if (state) {
+            pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
+            vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
+            /* vfio handles the necessary delay here */
+            pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
+            state = pmcsr & PCI_PM_CTRL_STATE_MASK;
+            if (state) {
+                error_report("vfio: Unable to power on device, stuck in D%d",
+                             state);
+            }
+        }
+    }
+
+    /*
+     * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
+     * Also put INTx Disable in known state.
+     */
+    cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
+    cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+             PCI_COMMAND_INTX_DISABLE);
+    vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
+}
+
+static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
+{
+    vfio_intx_enable(vdev);
+}
+
+static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
+{
+    char tmp[13];
+
+    sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
+            addr->bus, addr->slot, addr->function);
+
+    return (strcmp(tmp, name) == 0);
+}
+
+static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
+{
+    VFIOGroup *group;
+    struct vfio_pci_hot_reset_info *info;
+    struct vfio_pci_dependent_device *devices;
+    struct vfio_pci_hot_reset *reset;
+    int32_t *fds;
+    int ret, i, count;
+    bool multi = false;
+
+    trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
+
+    vfio_pci_pre_reset(vdev);
+    vdev->vbasedev.needs_reset = false;
+
+    info = g_malloc0(sizeof(*info));
+    info->argsz = sizeof(*info);
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
+    if (ret && errno != ENOSPC) {
+        ret = -errno;
+        if (!vdev->has_pm_reset) {
+            error_report("vfio: Cannot reset device %s, "
+                         "no available reset mechanism.", vdev->vbasedev.name);
+        }
+        goto out_single;
+    }
+
+    count = info->count;
+    info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
+    info->argsz = sizeof(*info) + (count * sizeof(*devices));
+    devices = &info->devices[0];
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
+    if (ret) {
+        ret = -errno;
+        error_report("vfio: hot reset info failed: %m");
+        goto out_single;
+    }
+
+    trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
+
+    /* Verify that we have all the groups required */
+    for (i = 0; i < info->count; i++) {
+        PCIHostDeviceAddress host;
+        VFIOPCIDevice *tmp;
+        VFIODevice *vbasedev_iter;
+
+        host.domain = devices[i].segment;
+        host.bus = devices[i].bus;
+        host.slot = PCI_SLOT(devices[i].devfn);
+        host.function = PCI_FUNC(devices[i].devfn);
+
+        trace_vfio_pci_hot_reset_dep_devices(host.domain,
+                host.bus, host.slot, host.function, devices[i].group_id);
+
+        if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
+            continue;
+        }
+
+        QLIST_FOREACH(group, &vfio_group_list, next) {
+            if (group->groupid == devices[i].group_id) {
+                break;
+            }
+        }
+
+        if (!group) {
+            if (!vdev->has_pm_reset) {
+                error_report("vfio: Cannot reset device %s, "
+                             "depends on group %d which is not owned.",
+                             vdev->vbasedev.name, devices[i].group_id);
+            }
+            ret = -EPERM;
+            goto out;
+        }
+
+        /* Prep dependent devices for reset and clear our marker. */
+        QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+            if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
+                continue;
+            }
+            tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
+            if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
+                if (single) {
+                    ret = -EINVAL;
+                    goto out_single;
+                }
+                vfio_pci_pre_reset(tmp);
+                tmp->vbasedev.needs_reset = false;
+                multi = true;
+                break;
+            }
+        }
+    }
+
+    if (!single && !multi) {
+        ret = -EINVAL;
+        goto out_single;
+    }
+
+    /* Determine how many group fds need to be passed */
+    count = 0;
+    QLIST_FOREACH(group, &vfio_group_list, next) {
+        for (i = 0; i < info->count; i++) {
+            if (group->groupid == devices[i].group_id) {
+                count++;
+                break;
+            }
+        }
+    }
+
+    reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
+    reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
+    fds = &reset->group_fds[0];
+
+    /* Fill in group fds */
+    QLIST_FOREACH(group, &vfio_group_list, next) {
+        for (i = 0; i < info->count; i++) {
+            if (group->groupid == devices[i].group_id) {
+                fds[reset->count++] = group->fd;
+                break;
+            }
+        }
+    }
+
+    /* Bus reset! */
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
+    g_free(reset);
+
+    trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
+                                    ret ? "%m" : "Success");
+
+out:
+    /* Re-enable INTx on affected devices */
+    for (i = 0; i < info->count; i++) {
+        PCIHostDeviceAddress host;
+        VFIOPCIDevice *tmp;
+        VFIODevice *vbasedev_iter;
+
+        host.domain = devices[i].segment;
+        host.bus = devices[i].bus;
+        host.slot = PCI_SLOT(devices[i].devfn);
+        host.function = PCI_FUNC(devices[i].devfn);
+
+        if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
+            continue;
+        }
+
+        QLIST_FOREACH(group, &vfio_group_list, next) {
+            if (group->groupid == devices[i].group_id) {
+                break;
+            }
+        }
+
+        if (!group) {
+            break;
+        }
+
+        QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+            if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
+                continue;
+            }
+            tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
+            if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
+                vfio_pci_post_reset(tmp);
+                break;
+            }
+        }
+    }
+out_single:
+    vfio_pci_post_reset(vdev);
+    g_free(info);
+
+    return ret;
+}
+
+/*
+ * We want to differentiate hot reset of mulitple in-use devices vs hot reset
+ * of a single in-use device.  VFIO_DEVICE_RESET will already handle the case
+ * of doing hot resets when there is only a single device per bus.  The in-use
+ * here refers to how many VFIODevices are affected.  A hot reset that affects
+ * multiple devices, but only a single in-use device, means that we can call
+ * it from our bus ->reset() callback since the extent is effectively a single
+ * device.  This allows us to make use of it in the hotplug path.  When there
+ * are multiple in-use devices, we can only trigger the hot reset during a
+ * system reset and thus from our reset handler.  We separate _one vs _multi
+ * here so that we don't overlap and do a double reset on the system reset
+ * path where both our reset handler and ->reset() callback are used.  Calling
+ * _one() will only do a hot reset for the one in-use devices case, calling
+ * _multi() will do nothing if a _one() would have been sufficient.
+ */
+static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
+{
+    return vfio_pci_hot_reset(vdev, true);
+}
+
+static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
+    return vfio_pci_hot_reset(vdev, false);
+}
+
+static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
+{
+    VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
+    if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
+        vbasedev->needs_reset = true;
+    }
+}
+
+static VFIODeviceOps vfio_pci_ops = {
+    .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
+    .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
+    .vfio_eoi = vfio_intx_eoi,
+};
+
+int vfio_populate_vga(VFIOPCIDevice *vdev)
+{
+    VFIODevice *vbasedev = &vdev->vbasedev;
+    struct vfio_region_info *reg_info;
+    int ret;
+
+    ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
+    if (ret) {
+        return ret;
+    }
+
+    if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
+        !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
+        reg_info->size < 0xbffff + 1) {
+        error_report("vfio: Unexpected VGA info, flags 0x%lx, size 0x%lx",
+                     (unsigned long)reg_info->flags,
+                     (unsigned long)reg_info->size);
+        g_free(reg_info);
+        return -EINVAL;
+    }
+
+    vdev->vga = g_new0(VFIOVGA, 1);
+
+    vdev->vga->fd_offset = reg_info->offset;
+    vdev->vga->fd = vdev->vbasedev.fd;
+
+    g_free(reg_info);
+
+    vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
+    vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
+    QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
+
+    memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
+                          OBJECT(vdev), &vfio_vga_ops,
+                          &vdev->vga->region[QEMU_PCI_VGA_MEM],
+                          "vfio-vga-mmio@0xa0000",
+                          QEMU_PCI_VGA_MEM_SIZE);
+
+    vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
+    vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
+    QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
+
+    memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
+                          OBJECT(vdev), &vfio_vga_ops,
+                          &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
+                          "vfio-vga-io@0x3b0",
+                          QEMU_PCI_VGA_IO_LO_SIZE);
+
+    vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
+    vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
+    QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
+
+    memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
+                          OBJECT(vdev), &vfio_vga_ops,
+                          &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
+                          "vfio-vga-io@0x3c0",
+                          QEMU_PCI_VGA_IO_HI_SIZE);
+
+    pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
+                     &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
+                     &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
+
+    return 0;
+}
+
+static int vfio_populate_device(VFIOPCIDevice *vdev)
+{
+    VFIODevice *vbasedev = &vdev->vbasedev;
+    struct vfio_region_info *reg_info;
+    struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
+    int i, ret = -1;
+
+    /* Sanity check device */
+    if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
+        error_report("vfio: Um, this isn't a PCI device");
+        goto error;
+    }
+
+    if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
+        error_report("vfio: unexpected number of io regions %u",
+                     vbasedev->num_regions);
+        goto error;
+    }
+
+    if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
+        error_report("vfio: unexpected number of irqs %u", vbasedev->num_irqs);
+        goto error;
+    }
+
+    for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
+        char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
+
+        ret = vfio_region_setup(OBJECT(vdev), vbasedev,
+                                &vdev->bars[i].region, i, name);
+        g_free(name);
+
+        if (ret) {
+            error_report("vfio: Error getting region %d info: %m", i);
+            goto error;
+        }
+
+        QLIST_INIT(&vdev->bars[i].quirks);
+    }
+
+    ret = vfio_get_region_info(vbasedev,
+                               VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
+    if (ret) {
+        error_report("vfio: Error getting config info: %m");
+        goto error;
+    }
+
+    trace_vfio_populate_device_config(vdev->vbasedev.name,
+                                      (unsigned long)reg_info->size,
+                                      (unsigned long)reg_info->offset,
+                                      (unsigned long)reg_info->flags);
+
+    vdev->config_size = reg_info->size;
+    if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
+        vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
+    }
+    vdev->config_offset = reg_info->offset;
+
+    g_free(reg_info);
+
+    if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
+        ret = vfio_populate_vga(vdev);
+        if (ret) {
+            error_report(
+                "vfio: Device does not support requested feature x-vga");
+            goto error;
+        }
+    }
+
+    irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
+    if (ret) {
+        /* This can fail for an old kernel or legacy PCI dev */
+        trace_vfio_populate_device_get_irq_info_failure();
+        ret = 0;
+    } else if (irq_info.count == 1) {
+        vdev->pci_aer = true;
+    } else {
+        error_report("vfio: %s "
+                     "Could not enable error recovery for the device",
+                     vbasedev->name);
+    }
+
+error:
+    return ret;
+}
+
+static void vfio_put_device(VFIOPCIDevice *vdev)
+{
+    g_free(vdev->vbasedev.name);
+    g_free(vdev->msix);
+
+    vfio_put_base_device(&vdev->vbasedev);
+}
+
+static void vfio_err_notifier_handler(void *opaque)
+{
+    VFIOPCIDevice *vdev = opaque;
+
+    if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
+        return;
+    }
+
+    /*
+     * TBD. Retrieve the error details and decide what action
+     * needs to be taken. One of the actions could be to pass
+     * the error to the guest and have the guest driver recover
+     * from the error. This requires that PCIe capabilities be
+     * exposed to the guest. For now, we just terminate the
+     * guest to contain the error.
+     */
+
+    error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
+
+    vm_stop(RUN_STATE_INTERNAL_ERROR);
+}
+
+/*
+ * Registers error notifier for devices supporting error recovery.
+ * If we encounter a failure in this function, we report an error
+ * and continue after disabling error recovery support for the
+ * device.
+ */
+static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
+{
+    int ret;
+    int argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    if (!vdev->pci_aer) {
+        return;
+    }
+
+    if (event_notifier_init(&vdev->err_notifier, 0)) {
+        error_report("vfio: Unable to init event notifier for error detection");
+        vdev->pci_aer = false;
+        return;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                     VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = event_notifier_get_fd(&vdev->err_notifier);
+    qemu_set_fd_handler(*pfd, vfio_err_notifier_handler, NULL, vdev);
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    if (ret) {
+        error_report("vfio: Failed to set up error notification");
+        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
+        event_notifier_cleanup(&vdev->err_notifier);
+        vdev->pci_aer = false;
+    }
+    g_free(irq_set);
+}
+
+static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
+{
+    int argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+    int ret;
+
+    if (!vdev->pci_aer) {
+        return;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                     VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_ERR_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+    *pfd = -1;
+
+    ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    if (ret) {
+        error_report("vfio: Failed to de-assign error fd: %m");
+    }
+    g_free(irq_set);
+    qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
+                        NULL, NULL, vdev);
+    event_notifier_cleanup(&vdev->err_notifier);
+}
+
+static void vfio_req_notifier_handler(void *opaque)
+{
+    VFIOPCIDevice *vdev = opaque;
+
+    if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
+        return;
+    }
+
+    qdev_unplug(&vdev->pdev.qdev, NULL);
+}
+
+static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
+{
+    struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
+                                      .index = VFIO_PCI_REQ_IRQ_INDEX };
+    int argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
+        return;
+    }
+
+    if (ioctl(vdev->vbasedev.fd,
+              VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
+        return;
+    }
+
+    if (event_notifier_init(&vdev->req_notifier, 0)) {
+        error_report("vfio: Unable to init event notifier for device request");
+        return;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                     VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = event_notifier_get_fd(&vdev->req_notifier);
+    qemu_set_fd_handler(*pfd, vfio_req_notifier_handler, NULL, vdev);
+
+    if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
+        error_report("vfio: Failed to set up device request notification");
+        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
+        event_notifier_cleanup(&vdev->req_notifier);
+    } else {
+        vdev->req_enabled = true;
+    }
+
+    g_free(irq_set);
+}
+
+static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
+{
+    int argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    if (!vdev->req_enabled) {
+        return;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                     VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_REQ_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+    *pfd = -1;
+
+    if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set)) {
+        error_report("vfio: Failed to de-assign device request fd: %m");
+    }
+    g_free(irq_set);
+    qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
+                        NULL, NULL, vdev);
+    event_notifier_cleanup(&vdev->req_notifier);
+
+    vdev->req_enabled = false;
+}
+
+static int vfio_initfn(PCIDevice *pdev)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+    VFIODevice *vbasedev_iter;
+    VFIOGroup *group;
+    char *tmp, group_path[PATH_MAX], *group_name;
+    ssize_t len;
+    struct stat st;
+    int groupid;
+    int i, ret;
+
+    if (!vdev->vbasedev.sysfsdev) {
+        vdev->vbasedev.sysfsdev =
+            g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
+                            vdev->host.domain, vdev->host.bus,
+                            vdev->host.slot, vdev->host.function);
+    }
+
+    if (qemu_stat(vdev->vbasedev.sysfsdev, &st) < 0) {
+        error_report("vfio: error: no such host device: %s",
+                     vdev->vbasedev.sysfsdev);
+        return -errno;
+    }
+
+    vdev->vbasedev.name = g_strdup(basename(vdev->vbasedev.sysfsdev));
+    vdev->vbasedev.ops = &vfio_pci_ops;
+    vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
+
+    tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
+    len = readlink(tmp, group_path, sizeof(group_path));
+    g_free(tmp);
+
+    if (len <= 0 || len >= sizeof(group_path)) {
+        error_report("vfio: error no iommu_group for device");
+        return len < 0 ? -errno : -ENAMETOOLONG;
+    }
+
+    group_path[len] = 0;
+
+    group_name = basename(group_path);
+    if (sscanf(group_name, "%d", &groupid) != 1) {
+        error_report("vfio: error reading %s: %m", group_path);
+        return -errno;
+    }
+
+    trace_vfio_initfn(vdev->vbasedev.name, groupid);
+
+    group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev));
+    if (!group) {
+        error_report("vfio: failed to get group %d", groupid);
+        return -ENOENT;
+    }
+
+    QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+        if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
+            error_report("vfio: error: device %s is already attached",
+                         vdev->vbasedev.name);
+            vfio_put_group(group);
+            return -EBUSY;
+        }
+    }
+
+    ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev);
+    if (ret) {
+        error_report("vfio: failed to get device %s", vdev->vbasedev.name);
+        vfio_put_group(group);
+        return ret;
+    }
+
+    ret = vfio_populate_device(vdev);
+    if (ret) {
+        return ret;
+    }
+
+    /* Get a copy of config space */
+    ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
+                MIN(pci_config_size(&vdev->pdev), vdev->config_size),
+                vdev->config_offset);
+    if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
+        ret = ret < 0 ? -errno : -EFAULT;
+        error_report("vfio: Failed to read device config space");
+        return ret;
+    }
+
+    /* vfio emulates a lot for us, but some bits need extra love */
+    vdev->emulated_config_bits = g_malloc0(vdev->config_size);
+
+    /* QEMU can choose to expose the ROM or not */
+    memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
+
+    /*
+     * The PCI spec reserves vendor ID 0xffff as an invalid value.  The
+     * device ID is managed by the vendor and need only be a 16-bit value.
+     * Allow any 16-bit value for subsystem so they can be hidden or changed.
+     */
+    if (vdev->vendor_id != PCI_ANY_ID) {
+        if (vdev->vendor_id >= 0xffff) {
+            error_report("vfio: Invalid PCI vendor ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
+        trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
+    } else {
+        vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
+    }
+
+    if (vdev->device_id != PCI_ANY_ID) {
+        if (vdev->device_id > 0xffff) {
+            error_report("vfio: Invalid PCI device ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
+        trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
+    } else {
+        vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
+    }
+
+    if (vdev->sub_vendor_id != PCI_ANY_ID) {
+        if (vdev->sub_vendor_id > 0xffff) {
+            error_report("vfio: Invalid PCI subsystem vendor ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
+                               vdev->sub_vendor_id, ~0);
+        trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
+                                              vdev->sub_vendor_id);
+    }
+
+    if (vdev->sub_device_id != PCI_ANY_ID) {
+        if (vdev->sub_device_id > 0xffff) {
+            error_report("vfio: Invalid PCI subsystem device ID provided");
+            return -EINVAL;
+        }
+        vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
+        trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
+                                              vdev->sub_device_id);
+    }
+
+    /* QEMU can change multi-function devices to single function, or reverse */
+    vdev->emulated_config_bits[PCI_HEADER_TYPE] =
+                                              PCI_HEADER_TYPE_MULTI_FUNCTION;
+
+    /* Restore or clear multifunction, this is always controlled by QEMU */
+    if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+        vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
+    } else {
+        vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
+    }
+
+    /*
+     * Clear host resource mapping info.  If we choose not to register a
+     * BAR, such as might be the case with the option ROM, we can get
+     * confusing, unwritable, residual addresses from the host here.
+     */
+    memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
+    memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
+
+    vfio_pci_size_rom(vdev);
+
+    ret = vfio_msix_early_setup(vdev);
+    if (ret) {
+        return ret;
+    }
+
+    vfio_bars_setup(vdev);
+
+    ret = vfio_add_capabilities(vdev);
+    if (ret) {
+        goto out_teardown;
+    }
+
+    if (vdev->vga) {
+        vfio_vga_quirk_setup(vdev);
+    }
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_bar_quirk_setup(vdev, i);
+    }
+
+    if (!vdev->igd_opregion &&
+        vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
+        struct vfio_region_info *opregion;
+
+        if (vdev->pdev.qdev.hotplugged) {
+            error_report("Cannot support IGD OpRegion feature on hotplugged "
+                         "device %s", vdev->vbasedev.name);
+            ret = -EINVAL;
+            goto out_teardown;
+        }
+
+        ret = vfio_get_dev_region_info(&vdev->vbasedev,
+                        VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
+                        VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
+        if (ret) {
+            error_report("Device %s does not support requested IGD OpRegion "
+                         "feature", vdev->vbasedev.name);
+            goto out_teardown;
+        }
+
+        ret = vfio_pci_igd_opregion_init(vdev, opregion);
+        g_free(opregion);
+        if (ret) {
+            error_report("Device %s IGD OpRegion initialization failed",
+                         vdev->vbasedev.name);
+            goto out_teardown;
+        }
+    }
+
+    /* QEMU emulates all of MSI & MSIX */
+    if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
+        memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
+               MSIX_CAP_LENGTH);
+    }
+
+    if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
+        memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
+               vdev->msi_cap_size);
+    }
+
+    if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
+        vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                                  vfio_intx_mmap_enable, vdev);
+        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update);
+        ret = vfio_intx_enable(vdev);
+        if (ret) {
+            goto out_teardown;
+        }
+    }
+
+    vfio_register_err_notifier(vdev);
+    vfio_register_req_notifier(vdev);
+    vfio_setup_resetfn_quirk(vdev);
+
+    return 0;
+
+out_teardown:
+    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
+    vfio_teardown_msi(vdev);
+    vfio_bars_exit(vdev);
+    return ret;
+}
+
+static void vfio_instance_finalize(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pci_dev);
+    VFIOGroup *group = vdev->vbasedev.group;
+
+    vfio_bars_finalize(vdev);
+    g_free(vdev->emulated_config_bits);
+    g_free(vdev->rom);
+    /*
+     * XXX Leaking igd_opregion is not an oversight, we can't remove the
+     * fw_cfg entry therefore leaking this allocation seems like the safest
+     * option.
+     *
+     * g_free(vdev->igd_opregion);
+     */
+    vfio_put_device(vdev);
+    vfio_put_group(group);
+}
+
+static void vfio_exitfn(PCIDevice *pdev)
+{
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+
+    vfio_unregister_req_notifier(vdev);
+    vfio_unregister_err_notifier(vdev);
+    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
+    vfio_disable_interrupts(vdev);
+    if (vdev->intx.mmap_timer) {
+        timer_free(vdev->intx.mmap_timer);
+    }
+    vfio_teardown_msi(vdev);
+    vfio_bars_exit(vdev);
+}
+
+static void vfio_pci_reset(DeviceState *dev)
+{
+    PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, pdev);
+
+    trace_vfio_pci_reset(vdev->vbasedev.name);
+
+    vfio_pci_pre_reset(vdev);
+
+    if (vdev->resetfn && !vdev->resetfn(vdev)) {
+        goto post_reset;
+    }
+
+    if (vdev->vbasedev.reset_works &&
+        (vdev->has_flr || !vdev->has_pm_reset) &&
+        !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
+        trace_vfio_pci_reset_flr(vdev->vbasedev.name);
+        goto post_reset;
+    }
+
+    /* See if we can do our own bus reset */
+    if (!vfio_pci_hot_reset_one(vdev)) {
+        goto post_reset;
+    }
+
+    /* If nothing else works and the device supports PM reset, use it */
+    if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
+        !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
+        trace_vfio_pci_reset_pm(vdev->vbasedev.name);
+        goto post_reset;
+    }
+
+post_reset:
+    vfio_pci_post_reset(vdev);
+}
+
+static void vfio_instance_init(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    VFIOPCIDevice *vdev = DO_UPCAST(VFIOPCIDevice, pdev, PCI_DEVICE(obj));
+
+    device_add_bootindex_property(obj, &vdev->bootindex,
+                                  "bootindex", NULL,
+                                  &pci_dev->qdev, NULL);
+}
+
+static Property vfio_pci_dev_properties[] = {
+    DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
+    DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
+                       intx.mmap_timeout, 1100),
+    DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
+                    VFIO_FEATURE_ENABLE_VGA_BIT, false),
+    DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
+                    VFIO_FEATURE_ENABLE_REQ_BIT, true),
+    DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
+                    VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
+    DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
+    DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
+    DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
+    DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
+    DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
+                       sub_vendor_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
+                       sub_device_id, PCI_ANY_ID),
+    DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
+    /*
+     * TODO - support passed fds... is this necessary?
+     * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
+     * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
+     */
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription vfio_pci_vmstate = {
+    .name = "vfio-pci",
+    .unmigratable = 1,
+};
+
+static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
+
+    dc->reset = vfio_pci_reset;
+    dc->props = vfio_pci_dev_properties;
+    dc->vmsd = &vfio_pci_vmstate;
+    dc->desc = "VFIO-based PCI device assignment";
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    pdc->init = vfio_initfn;
+    pdc->exit = vfio_exitfn;
+    pdc->config_read = vfio_pci_read_config;
+    pdc->config_write = vfio_pci_write_config;
+    pdc->is_express = 1; /* We might be */
+}
+
+static const TypeInfo vfio_pci_dev_info = {
+    .name = "vfio-pci",
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VFIOPCIDevice),
+    .class_init = vfio_pci_dev_class_init,
+    .instance_init = vfio_instance_init,
+    .instance_finalize = vfio_instance_finalize,
+};
+
+static void register_vfio_pci_dev_type(void)
+{
+    type_register_static(&vfio_pci_dev_info);
+}
+
+type_init(register_vfio_pci_dev_type)
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
new file mode 100644
index 0000000..7d482d9
--- /dev/null
+++ b/hw/vfio/pci.h
@@ -0,0 +1,169 @@
+/*
+ * vfio based device assignment support - PCI devices
+ *
+ * Copyright Red Hat, Inc. 2012-2015
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+#ifndef HW_VFIO_VFIO_PCI_H
+#define HW_VFIO_VFIO_PCI_H
+
+#include "qemu-common.h"
+#include "exec/memory.h"
+#include "hw/pci/pci.h"
+#include "hw/vfio/vfio-common.h"
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+
+#define PCI_ANY_ID (~0)
+
+struct VFIOPCIDevice;
+
+typedef struct VFIOQuirk {
+    QLIST_ENTRY(VFIOQuirk) next;
+    void *data;
+    int nr_mem;
+    MemoryRegion *mem;
+} VFIOQuirk;
+
+typedef struct VFIOBAR {
+    VFIORegion region;
+    bool ioport;
+    bool mem64;
+    QLIST_HEAD(, VFIOQuirk) quirks;
+} VFIOBAR;
+
+typedef struct VFIOVGARegion {
+    MemoryRegion mem;
+    off_t offset;
+    int nr;
+    QLIST_HEAD(, VFIOQuirk) quirks;
+} VFIOVGARegion;
+
+typedef struct VFIOVGA {
+    off_t fd_offset;
+    int fd;
+    VFIOVGARegion region[QEMU_PCI_VGA_NUM_REGIONS];
+} VFIOVGA;
+
+typedef struct VFIOINTx {
+    bool pending; /* interrupt pending */
+    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+    uint8_t pin; /* which pin to pull for qemu_set_irq */
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    PCIINTxRoute route; /* routing info for QEMU bypass */
+    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
+} VFIOINTx;
+
+typedef struct VFIOMSIVector {
+    /*
+     * Two interrupt paths are configured per vector.  The first, is only used
+     * for interrupts injected via QEMU.  This is typically the non-accel path,
+     * but may also be used when we want QEMU to handle masking and pending
+     * bits.  The KVM path bypasses QEMU and is therefore higher performance,
+     * but requires masking at the device.  virq is used to track the MSI route
+     * through KVM, thus kvm_interrupt is only available when virq is set to a
+     * valid (>= 0) value.
+     */
+    EventNotifier interrupt;
+    EventNotifier kvm_interrupt;
+    struct VFIOPCIDevice *vdev; /* back pointer to device */
+    int virq;
+    bool use;
+} VFIOMSIVector;
+
+enum {
+    VFIO_INT_NONE = 0,
+    VFIO_INT_INTx = 1,
+    VFIO_INT_MSI  = 2,
+    VFIO_INT_MSIX = 3,
+};
+
+/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
+typedef struct VFIOMSIXInfo {
+    uint8_t table_bar;
+    uint8_t pba_bar;
+    uint16_t entries;
+    uint32_t table_offset;
+    uint32_t pba_offset;
+    MemoryRegion mmap_mem;
+    void *mmap;
+    unsigned long *pending;
+} VFIOMSIXInfo;
+
+typedef struct VFIOPCIDevice {
+    PCIDevice pdev;
+    VFIODevice vbasedev;
+    VFIOINTx intx;
+    unsigned int config_size;
+    uint8_t *emulated_config_bits; /* QEMU emulated bits, little-endian */
+    off_t config_offset; /* Offset of config space region within device fd */
+    unsigned int rom_size;
+    off_t rom_offset; /* Offset of ROM region within device fd */
+    void *rom;
+    int msi_cap_size;
+    VFIOMSIVector *msi_vectors;
+    VFIOMSIXInfo *msix;
+    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
+    int interrupt; /* Current interrupt type */
+    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
+    VFIOVGA *vga; /* 0xa0000, 0x3b0, 0x3c0 */
+    void *igd_opregion;
+    PCIHostDeviceAddress host;
+    EventNotifier err_notifier;
+    EventNotifier req_notifier;
+    int (*resetfn)(struct VFIOPCIDevice *);
+    uint32_t vendor_id;
+    uint32_t device_id;
+    uint32_t sub_vendor_id;
+    uint32_t sub_device_id;
+    uint32_t features;
+#define VFIO_FEATURE_ENABLE_VGA_BIT 0
+#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
+#define VFIO_FEATURE_ENABLE_REQ_BIT 1
+#define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT)
+#define VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT 2
+#define VFIO_FEATURE_ENABLE_IGD_OPREGION \
+                                (1 << VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT)
+    int32_t bootindex;
+    uint32_t igd_gms;
+    uint8_t pm_cap;
+    bool pci_aer;
+    bool req_enabled;
+    bool has_flr;
+    bool has_pm_reset;
+    bool rom_read_failed;
+    bool no_kvm_intx;
+    bool no_kvm_msi;
+    bool no_kvm_msix;
+} VFIOPCIDevice;
+
+uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
+void vfio_pci_write_config(PCIDevice *pdev,
+                           uint32_t addr, uint32_t val, int len);
+
+uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size);
+void vfio_vga_write(void *opaque, hwaddr addr, uint64_t data, unsigned size);
+
+bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_setup(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_exit(VFIOPCIDevice *vdev);
+void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev);
+void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr);
+void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr);
+void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr);
+void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev);
+
+int vfio_populate_vga(VFIOPCIDevice *vdev);
+
+int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev,
+                               struct vfio_region_info *info);
+
+#endif /* HW_VFIO_VFIO_PCI_H */
diff --git a/hw/vfio/platform.c b/hw/vfio/platform.c
new file mode 100644
index 0000000..19e189d
--- /dev/null
+++ b/hw/vfio/platform.c
@@ -0,0 +1,705 @@
+/*
+ * vfio based device assignment support - platform devices
+ *
+ * Copyright Linaro Limited, 2014
+ *
+ * Authors:
+ *  Kim Phillips <kim.phillips@linaro.org>
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on vfio based PCI device assignment support:
+ *  Copyright Red Hat, Inc. 2012
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include <sys/ioctl.h>
+#include <linux/vfio.h>
+
+#include "hw/vfio/vfio-platform.h"
+#include "qemu/error-report.h"
+#include "qemu/range.h"
+#include "sysemu/sysemu.h"
+#include "exec/memory.h"
+#include "qemu/queue.h"
+#include "hw/sysbus.h"
+#include "trace.h"
+#include "hw/platform-bus.h"
+#include "sysemu/kvm.h"
+
+/*
+ * Functions used whatever the injection method
+ */
+
+static inline bool vfio_irq_is_automasked(VFIOINTp *intp)
+{
+    return intp->flags & VFIO_IRQ_INFO_AUTOMASKED;
+}
+
+/**
+ * vfio_init_intp - allocate, initialize the IRQ struct pointer
+ * and add it into the list of IRQs
+ * @vbasedev: the VFIO device handle
+ * @info: irq info struct retrieved from VFIO driver
+ */
+static VFIOINTp *vfio_init_intp(VFIODevice *vbasedev,
+                                struct vfio_irq_info info)
+{
+    int ret;
+    VFIOPlatformDevice *vdev =
+        container_of(vbasedev, VFIOPlatformDevice, vbasedev);
+    SysBusDevice *sbdev = SYS_BUS_DEVICE(vdev);
+    VFIOINTp *intp;
+
+    intp = g_malloc0(sizeof(*intp));
+    intp->vdev = vdev;
+    intp->pin = info.index;
+    intp->flags = info.flags;
+    intp->state = VFIO_IRQ_INACTIVE;
+    intp->kvm_accel = false;
+
+    sysbus_init_irq(sbdev, &intp->qemuirq);
+
+    /* Get an eventfd for trigger */
+    intp->interrupt = g_malloc0(sizeof(EventNotifier));
+    ret = event_notifier_init(intp->interrupt, 0);
+    if (ret) {
+        g_free(intp->interrupt);
+        g_free(intp);
+        error_report("vfio: Error: trigger event_notifier_init failed ");
+        return NULL;
+    }
+    if (vfio_irq_is_automasked(intp)) {
+        /* Get an eventfd for resample/unmask */
+        intp->unmask = g_malloc0(sizeof(EventNotifier));
+        ret = event_notifier_init(intp->unmask, 0);
+        if (ret) {
+            g_free(intp->interrupt);
+            g_free(intp->unmask);
+            g_free(intp);
+            error_report("vfio: Error: resamplefd event_notifier_init failed");
+            return NULL;
+        }
+    }
+
+    QLIST_INSERT_HEAD(&vdev->intp_list, intp, next);
+    return intp;
+}
+
+/**
+ * vfio_set_trigger_eventfd - set VFIO eventfd handling
+ *
+ * @intp: IRQ struct handle
+ * @handler: handler to be called on eventfd signaling
+ *
+ * Setup VFIO signaling and attach an optional user-side handler
+ * to the eventfd
+ */
+static int vfio_set_trigger_eventfd(VFIOINTp *intp,
+                                    eventfd_user_side_handler_t handler)
+{
+    VFIODevice *vbasedev = &intp->vdev->vbasedev;
+    struct vfio_irq_set *irq_set;
+    int argsz, ret;
+    int32_t *pfd;
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = intp->pin;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+    *pfd = event_notifier_get_fd(intp->interrupt);
+    qemu_set_fd_handler(*pfd, (IOHandler *)handler, NULL, intp);
+    ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret < 0) {
+        error_report("vfio: Failed to set trigger eventfd: %m");
+        qemu_set_fd_handler(*pfd, NULL, NULL, NULL);
+    }
+    return ret;
+}
+
+/*
+ * Functions only used when eventfds are handled on user-side
+ * ie. without irqfd
+ */
+
+/**
+ * vfio_mmap_set_enabled - enable/disable the fast path mode
+ * @vdev: the VFIO platform device
+ * @enabled: the target mmap state
+ *
+ * enabled = true ~ fast path = MMIO region is mmaped (no KVM TRAP);
+ * enabled = false ~ slow path = MMIO region is trapped and region callbacks
+ * are called; slow path enables to trap the device IRQ status register reset
+*/
+
+static void vfio_mmap_set_enabled(VFIOPlatformDevice *vdev, bool enabled)
+{
+    int i;
+
+    for (i = 0; i < vdev->vbasedev.num_regions; i++) {
+        vfio_region_mmaps_set_enabled(vdev->regions[i], enabled);
+    }
+}
+
+/**
+ * vfio_intp_mmap_enable - timer function, restores the fast path
+ * if there is no more active IRQ
+ * @opaque: actually points to the VFIO platform device
+ *
+ * Called on mmap timer timout, this function checks whether the
+ * IRQ is still active and if not, restores the fast path.
+ * by construction a single eventfd is handled at a time.
+ * if the IRQ is still active, the timer is re-programmed.
+ */
+static void vfio_intp_mmap_enable(void *opaque)
+{
+    VFIOINTp *tmp;
+    VFIOPlatformDevice *vdev = (VFIOPlatformDevice *)opaque;
+
+    qemu_mutex_lock(&vdev->intp_mutex);
+    QLIST_FOREACH(tmp, &vdev->intp_list, next) {
+        if (tmp->state == VFIO_IRQ_ACTIVE) {
+            trace_vfio_platform_intp_mmap_enable(tmp->pin);
+            /* re-program the timer to check active status later */
+            timer_mod(vdev->mmap_timer,
+                      qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+                          vdev->mmap_timeout);
+            qemu_mutex_unlock(&vdev->intp_mutex);
+            return;
+        }
+    }
+    vfio_mmap_set_enabled(vdev, true);
+    qemu_mutex_unlock(&vdev->intp_mutex);
+}
+
+/**
+ * vfio_intp_inject_pending_lockheld - Injects a pending IRQ
+ * @opaque: opaque pointer, in practice the VFIOINTp handle
+ *
+ * The function is called on a previous IRQ completion, from
+ * vfio_platform_eoi, while the intp_mutex is locked.
+ * Also in such situation, the slow path already is set and
+ * the mmap timer was already programmed.
+ */
+static void vfio_intp_inject_pending_lockheld(VFIOINTp *intp)
+{
+    trace_vfio_platform_intp_inject_pending_lockheld(intp->pin,
+                              event_notifier_get_fd(intp->interrupt));
+
+    intp->state = VFIO_IRQ_ACTIVE;
+
+    /* trigger the virtual IRQ */
+    qemu_set_irq(intp->qemuirq, 1);
+}
+
+/**
+ * vfio_intp_interrupt - The user-side eventfd handler
+ * @opaque: opaque pointer which in practice is the VFIOINTp handle
+ *
+ * the function is entered in event handler context:
+ * the vIRQ is injected into the guest if there is no other active
+ * or pending IRQ.
+ */
+static void vfio_intp_interrupt(VFIOINTp *intp)
+{
+    int ret;
+    VFIOINTp *tmp;
+    VFIOPlatformDevice *vdev = intp->vdev;
+    bool delay_handling = false;
+
+    qemu_mutex_lock(&vdev->intp_mutex);
+    if (intp->state == VFIO_IRQ_INACTIVE) {
+        QLIST_FOREACH(tmp, &vdev->intp_list, next) {
+            if (tmp->state == VFIO_IRQ_ACTIVE ||
+                tmp->state == VFIO_IRQ_PENDING) {
+                delay_handling = true;
+                break;
+            }
+        }
+    }
+    if (delay_handling) {
+        /*
+         * the new IRQ gets a pending status and is pushed in
+         * the pending queue
+         */
+        intp->state = VFIO_IRQ_PENDING;
+        trace_vfio_intp_interrupt_set_pending(intp->pin);
+        QSIMPLEQ_INSERT_TAIL(&vdev->pending_intp_queue,
+                             intp, pqnext);
+        ret = event_notifier_test_and_clear(intp->interrupt);
+        qemu_mutex_unlock(&vdev->intp_mutex);
+        return;
+    }
+
+    trace_vfio_platform_intp_interrupt(intp->pin,
+                              event_notifier_get_fd(intp->interrupt));
+
+    ret = event_notifier_test_and_clear(intp->interrupt);
+    if (!ret) {
+        error_report("Error when clearing fd=%d (ret = %d)",
+                     event_notifier_get_fd(intp->interrupt), ret);
+    }
+
+    intp->state = VFIO_IRQ_ACTIVE;
+
+    /* sets slow path */
+    vfio_mmap_set_enabled(vdev, false);
+
+    /* trigger the virtual IRQ */
+    qemu_set_irq(intp->qemuirq, 1);
+
+    /*
+     * Schedule the mmap timer which will restore fastpath when no IRQ
+     * is active anymore
+     */
+    if (vdev->mmap_timeout) {
+        timer_mod(vdev->mmap_timer,
+                  qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) +
+                      vdev->mmap_timeout);
+    }
+    qemu_mutex_unlock(&vdev->intp_mutex);
+}
+
+/**
+ * vfio_platform_eoi - IRQ completion routine
+ * @vbasedev: the VFIO device handle
+ *
+ * De-asserts the active virtual IRQ and unmasks the physical IRQ
+ * (effective for level sensitive IRQ auto-masked by the  VFIO driver).
+ * Then it handles next pending IRQ if any.
+ * eoi function is called on the first access to any MMIO region
+ * after an IRQ was triggered, trapped since slow path was set.
+ * It is assumed this access corresponds to the IRQ status
+ * register reset. With such a mechanism, a single IRQ can be
+ * handled at a time since there is no way to know which IRQ
+ * was completed by the guest (we would need additional details
+ * about the IRQ status register mask).
+ */
+static void vfio_platform_eoi(VFIODevice *vbasedev)
+{
+    VFIOINTp *intp;
+    VFIOPlatformDevice *vdev =
+        container_of(vbasedev, VFIOPlatformDevice, vbasedev);
+
+    qemu_mutex_lock(&vdev->intp_mutex);
+    QLIST_FOREACH(intp, &vdev->intp_list, next) {
+        if (intp->state == VFIO_IRQ_ACTIVE) {
+            trace_vfio_platform_eoi(intp->pin,
+                                event_notifier_get_fd(intp->interrupt));
+            intp->state = VFIO_IRQ_INACTIVE;
+
+            /* deassert the virtual IRQ */
+            qemu_set_irq(intp->qemuirq, 0);
+
+            if (vfio_irq_is_automasked(intp)) {
+                /* unmasks the physical level-sensitive IRQ */
+                vfio_unmask_single_irqindex(vbasedev, intp->pin);
+            }
+
+            /* a single IRQ can be active at a time */
+            break;
+        }
+    }
+    /* in case there are pending IRQs, handle the first one */
+    if (!QSIMPLEQ_EMPTY(&vdev->pending_intp_queue)) {
+        intp = QSIMPLEQ_FIRST(&vdev->pending_intp_queue);
+        vfio_intp_inject_pending_lockheld(intp);
+        QSIMPLEQ_REMOVE_HEAD(&vdev->pending_intp_queue, pqnext);
+    }
+    qemu_mutex_unlock(&vdev->intp_mutex);
+}
+
+/**
+ * vfio_start_eventfd_injection - starts the virtual IRQ injection using
+ * user-side handled eventfds
+ * @sbdev: the sysbus device handle
+ * @irq: the qemu irq handle
+ */
+
+static void vfio_start_eventfd_injection(SysBusDevice *sbdev, qemu_irq irq)
+{
+    int ret;
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIOINTp *intp;
+
+    QLIST_FOREACH(intp, &vdev->intp_list, next) {
+        if (intp->qemuirq == irq) {
+            break;
+        }
+    }
+    assert(intp);
+
+    ret = vfio_set_trigger_eventfd(intp, vfio_intp_interrupt);
+    if (ret) {
+        error_report("vfio: failed to start eventfd signaling for IRQ %d: %m",
+                     intp->pin);
+        abort();
+    }
+}
+
+/*
+ * Functions used for irqfd
+ */
+
+/**
+ * vfio_set_resample_eventfd - sets the resamplefd for an IRQ
+ * @intp: the IRQ struct handle
+ * programs the VFIO driver to unmask this IRQ when the
+ * intp->unmask eventfd is triggered
+ */
+static int vfio_set_resample_eventfd(VFIOINTp *intp)
+{
+    VFIODevice *vbasedev = &intp->vdev->vbasedev;
+    struct vfio_irq_set *irq_set;
+    int argsz, ret;
+    int32_t *pfd;
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
+    irq_set->index = intp->pin;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+    *pfd = event_notifier_get_fd(intp->unmask);
+    qemu_set_fd_handler(*pfd, NULL, NULL, NULL);
+    ret = ioctl(vbasedev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret < 0) {
+        error_report("vfio: Failed to set resample eventfd: %m");
+    }
+    return ret;
+}
+
+/**
+ * vfio_start_irqfd_injection - starts the virtual IRQ injection using
+ * irqfd
+ *
+ * @sbdev: the sysbus device handle
+ * @irq: the qemu irq handle
+ *
+ * In case the irqfd setup fails, we fallback to userspace handled eventfd
+ */
+static void vfio_start_irqfd_injection(SysBusDevice *sbdev, qemu_irq irq)
+{
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(sbdev);
+    VFIOINTp *intp;
+
+    if (!kvm_irqfds_enabled() || !kvm_resamplefds_enabled() ||
+        !vdev->irqfd_allowed) {
+        goto fail_irqfd;
+    }
+
+    QLIST_FOREACH(intp, &vdev->intp_list, next) {
+        if (intp->qemuirq == irq) {
+            break;
+        }
+    }
+    assert(intp);
+
+    if (kvm_irqchip_add_irqfd_notifier(kvm_state, intp->interrupt,
+                                   intp->unmask, irq) < 0) {
+        goto fail_irqfd;
+    }
+
+    if (vfio_set_trigger_eventfd(intp, NULL) < 0) {
+        goto fail_vfio;
+    }
+    if (vfio_irq_is_automasked(intp)) {
+        if (vfio_set_resample_eventfd(intp) < 0) {
+            goto fail_vfio;
+        }
+        trace_vfio_platform_start_level_irqfd_injection(intp->pin,
+                                    event_notifier_get_fd(intp->interrupt),
+                                    event_notifier_get_fd(intp->unmask));
+    } else {
+        trace_vfio_platform_start_edge_irqfd_injection(intp->pin,
+                                    event_notifier_get_fd(intp->interrupt));
+    }
+
+    intp->kvm_accel = true;
+
+    return;
+fail_vfio:
+    kvm_irqchip_remove_irqfd_notifier(kvm_state, intp->interrupt, irq);
+    error_report("vfio: failed to start eventfd signaling for IRQ %d: %m",
+                 intp->pin);
+    abort();
+fail_irqfd:
+    vfio_start_eventfd_injection(sbdev, irq);
+    return;
+}
+
+/* VFIO skeleton */
+
+static void vfio_platform_compute_needs_reset(VFIODevice *vbasedev)
+{
+    vbasedev->needs_reset = true;
+}
+
+/* not implemented yet */
+static int vfio_platform_hot_reset_multi(VFIODevice *vbasedev)
+{
+    return -1;
+}
+
+/**
+ * vfio_populate_device - Allocate and populate MMIO region
+ * and IRQ structs according to driver returned information
+ * @vbasedev: the VFIO device handle
+ *
+ */
+static int vfio_populate_device(VFIODevice *vbasedev)
+{
+    VFIOINTp *intp, *tmp;
+    int i, ret = -1;
+    VFIOPlatformDevice *vdev =
+        container_of(vbasedev, VFIOPlatformDevice, vbasedev);
+
+    if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PLATFORM)) {
+        error_report("vfio: Um, this isn't a platform device");
+        return ret;
+    }
+
+    vdev->regions = g_new0(VFIORegion *, vbasedev->num_regions);
+
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        char *name = g_strdup_printf("VFIO %s region %d\n", vbasedev->name, i);
+
+        vdev->regions[i] = g_new0(VFIORegion, 1);
+        ret = vfio_region_setup(OBJECT(vdev), vbasedev,
+                                vdev->regions[i], i, name);
+        g_free(name);
+        if (ret) {
+            error_report("vfio: Error getting region %d info: %m", i);
+            goto reg_error;
+        }
+    }
+
+    vdev->mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
+                                    vfio_intp_mmap_enable, vdev);
+
+    QSIMPLEQ_INIT(&vdev->pending_intp_queue);
+
+    for (i = 0; i < vbasedev->num_irqs; i++) {
+        struct vfio_irq_info irq = { .argsz = sizeof(irq) };
+
+        irq.index = i;
+        ret = ioctl(vbasedev->fd, VFIO_DEVICE_GET_IRQ_INFO, &irq);
+        if (ret) {
+            error_report("vfio: error getting device %s irq info",
+                         vbasedev->name);
+            goto irq_err;
+        } else {
+            trace_vfio_platform_populate_interrupts(irq.index,
+                                                    irq.count,
+                                                    irq.flags);
+            intp = vfio_init_intp(vbasedev, irq);
+            if (!intp) {
+                error_report("vfio: Error installing IRQ %d up", i);
+                goto irq_err;
+            }
+        }
+    }
+    return 0;
+irq_err:
+    timer_del(vdev->mmap_timer);
+    QLIST_FOREACH_SAFE(intp, &vdev->intp_list, next, tmp) {
+        QLIST_REMOVE(intp, next);
+        g_free(intp);
+    }
+reg_error:
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        if (vdev->regions[i]) {
+            vfio_region_finalize(vdev->regions[i]);
+        }
+        g_free(vdev->regions[i]);
+    }
+    g_free(vdev->regions);
+    return ret;
+}
+
+/* specialized functions for VFIO Platform devices */
+static VFIODeviceOps vfio_platform_ops = {
+    .vfio_compute_needs_reset = vfio_platform_compute_needs_reset,
+    .vfio_hot_reset_multi = vfio_platform_hot_reset_multi,
+    .vfio_eoi = vfio_platform_eoi,
+};
+
+/**
+ * vfio_base_device_init - perform preliminary VFIO setup
+ * @vbasedev: the VFIO device handle
+ *
+ * Implement the VFIO command sequence that allows to discover
+ * assigned device resources: group extraction, device
+ * fd retrieval, resource query.
+ * Precondition: the device name must be initialized
+ */
+static int vfio_base_device_init(VFIODevice *vbasedev)
+{
+    VFIOGroup *group;
+    VFIODevice *vbasedev_iter;
+    char *tmp, group_path[PATH_MAX], *group_name;
+    ssize_t len;
+    struct stat st;
+    int groupid;
+    int ret;
+
+    /* @sysfsdev takes precedence over @host */
+    if (vbasedev->sysfsdev) {
+        g_free(vbasedev->name);
+        vbasedev->name = g_strdup(basename(vbasedev->sysfsdev));
+    } else {
+        if (!vbasedev->name || strchr(vbasedev->name, '/')) {
+            return -EINVAL;
+        }
+
+        vbasedev->sysfsdev = g_strdup_printf("/sys/bus/platform/devices/%s",
+                                             vbasedev->name);
+    }
+
+    if (qemu_stat(vbasedev->sysfsdev, &st) < 0) {
+        error_report("vfio: error: no such host device: %s",
+                     vbasedev->sysfsdev);
+        return -errno;
+    }
+
+    tmp = g_strdup_printf("%s/iommu_group", vbasedev->sysfsdev);
+    len = readlink(tmp, group_path, sizeof(group_path));
+    g_free(tmp);
+
+    if (len < 0 || len >= sizeof(group_path)) {
+        error_report("vfio: error no iommu_group for device");
+        return len < 0 ? -errno : -ENAMETOOLONG;
+    }
+
+    group_path[len] = 0;
+
+    group_name = basename(group_path);
+    if (sscanf(group_name, "%d", &groupid) != 1) {
+        error_report("vfio: error reading %s: %m", group_path);
+        return -errno;
+    }
+
+    trace_vfio_platform_base_device_init(vbasedev->name, groupid);
+
+    group = vfio_get_group(groupid, &address_space_memory);
+    if (!group) {
+        error_report("vfio: failed to get group %d", groupid);
+        return -ENOENT;
+    }
+
+    QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+        if (strcmp(vbasedev_iter->name, vbasedev->name) == 0) {
+            error_report("vfio: error: device %s is already attached",
+                         vbasedev->name);
+            vfio_put_group(group);
+            return -EBUSY;
+        }
+    }
+    ret = vfio_get_device(group, vbasedev->name, vbasedev);
+    if (ret) {
+        error_report("vfio: failed to get device %s", vbasedev->name);
+        vfio_put_group(group);
+        return ret;
+    }
+
+    ret = vfio_populate_device(vbasedev);
+    if (ret) {
+        error_report("vfio: failed to populate device %s", vbasedev->name);
+        vfio_put_group(group);
+    }
+
+    return ret;
+}
+
+/**
+ * vfio_platform_realize  - the device realize function
+ * @dev: device state pointer
+ * @errp: error
+ *
+ * initialize the device, its memory regions and IRQ structures
+ * IRQ are started separately
+ */
+static void vfio_platform_realize(DeviceState *dev, Error **errp)
+{
+    VFIOPlatformDevice *vdev = VFIO_PLATFORM_DEVICE(dev);
+    SysBusDevice *sbdev = SYS_BUS_DEVICE(dev);
+    VFIODevice *vbasedev = &vdev->vbasedev;
+    int i, ret;
+
+    vbasedev->type = VFIO_DEVICE_TYPE_PLATFORM;
+    vbasedev->ops = &vfio_platform_ops;
+
+    trace_vfio_platform_realize(vbasedev->sysfsdev ?
+                                vbasedev->sysfsdev : vbasedev->name,
+                                vdev->compat);
+
+    ret = vfio_base_device_init(vbasedev);
+    if (ret) {
+        error_setg(errp, "vfio: vfio_base_device_init failed for %s",
+                   vbasedev->name);
+        return;
+    }
+
+    for (i = 0; i < vbasedev->num_regions; i++) {
+        if (vfio_region_mmap(vdev->regions[i])) {
+            error_report("%s mmap unsupported. Performance may be slow",
+                         memory_region_name(vdev->regions[i]->mem));
+        }
+        sysbus_init_mmio(sbdev, vdev->regions[i]->mem);
+    }
+}
+
+static const VMStateDescription vfio_platform_vmstate = {
+    .name = TYPE_VFIO_PLATFORM,
+    .unmigratable = 1,
+};
+
+static Property vfio_platform_dev_properties[] = {
+    DEFINE_PROP_STRING("host", VFIOPlatformDevice, vbasedev.name),
+    DEFINE_PROP_STRING("sysfsdev", VFIOPlatformDevice, vbasedev.sysfsdev),
+    DEFINE_PROP_BOOL("x-no-mmap", VFIOPlatformDevice, vbasedev.no_mmap, false),
+    DEFINE_PROP_UINT32("mmap-timeout-ms", VFIOPlatformDevice,
+                       mmap_timeout, 1100),
+    DEFINE_PROP_BOOL("x-irqfd", VFIOPlatformDevice, irqfd_allowed, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void vfio_platform_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(klass);
+
+    dc->realize = vfio_platform_realize;
+    dc->props = vfio_platform_dev_properties;
+    dc->vmsd = &vfio_platform_vmstate;
+    dc->desc = "VFIO-based platform device assignment";
+    sbc->connect_irq_notifier = vfio_start_irqfd_injection;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
+static const TypeInfo vfio_platform_dev_info = {
+    .name = TYPE_VFIO_PLATFORM,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(VFIOPlatformDevice),
+    .class_init = vfio_platform_class_init,
+    .class_size = sizeof(VFIOPlatformDeviceClass),
+    .abstract   = true,
+};
+
+static void register_vfio_platform_dev_type(void)
+{
+    type_register_static(&vfio_platform_dev_info);
+}
+
+type_init(register_vfio_platform_dev_type)
diff --git a/hw/vfio/spapr.c b/hw/vfio/spapr.c
new file mode 100644
index 0000000..7443d34
--- /dev/null
+++ b/hw/vfio/spapr.c
@@ -0,0 +1,209 @@
+/*
+ * DMA memory preregistration
+ *
+ * Authors:
+ *  Alexey Kardashevskiy <aik@ozlabs.ru>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include <sys/ioctl.h>
+#include <linux/vfio.h>
+
+#include "hw/vfio/vfio-common.h"
+#include "hw/hw.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+
+static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section)
+{
+    if (memory_region_is_iommu(section->mr)) {
+        hw_error("Cannot possibly preregister IOMMU memory");
+    }
+
+    return !memory_region_is_ram(section->mr) ||
+            memory_region_is_skip_dump(section->mr);
+}
+
+static void *vfio_prereg_gpa_to_vaddr(MemoryRegionSection *section, hwaddr gpa)
+{
+    return memory_region_get_ram_ptr(section->mr) +
+        section->offset_within_region +
+        (gpa - section->offset_within_address_space);
+}
+
+static void vfio_prereg_listener_region_add(MemoryListener *listener,
+                                            MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer,
+                                            prereg_listener);
+    const hwaddr gpa = section->offset_within_address_space;
+    hwaddr end;
+    int ret;
+    hwaddr page_mask = qemu_real_host_page_mask;
+    struct vfio_iommu_spapr_register_memory reg = {
+        .argsz = sizeof(reg),
+        .flags = 0,
+    };
+
+    if (vfio_prereg_listener_skipped_section(section)) {
+        trace_vfio_prereg_listener_region_add_skip(
+                section->offset_within_address_space,
+                section->offset_within_address_space +
+                int128_get64(int128_sub(section->size, int128_one())));
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~page_mask) ||
+                 (section->offset_within_region & ~page_mask) ||
+                 (int128_get64(section->size) & ~page_mask))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    end = section->offset_within_address_space + int128_get64(section->size);
+    if (gpa >= end) {
+        return;
+    }
+
+    memory_region_ref(section->mr);
+
+    reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa);
+    reg.size = end - gpa;
+
+    ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, &reg);
+    trace_vfio_prereg_register(reg.vaddr, reg.size, ret ? -errno : 0);
+    if (ret) {
+        /*
+         * On the initfn path, store the first error in the container so we
+         * can gracefully fail.  Runtime, there's not much we can do other
+         * than throw a hardware error.
+         */
+        if (!container->initialized) {
+            if (!container->error) {
+                container->error = ret;
+            }
+        } else {
+            hw_error("vfio: Memory registering failed, unable to continue");
+        }
+    }
+}
+
+static void vfio_prereg_listener_region_del(MemoryListener *listener,
+                                            MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer,
+                                            prereg_listener);
+    const hwaddr gpa = section->offset_within_address_space;
+    hwaddr end;
+    int ret;
+    hwaddr page_mask = qemu_real_host_page_mask;
+    struct vfio_iommu_spapr_register_memory reg = {
+        .argsz = sizeof(reg),
+        .flags = 0,
+    };
+
+    if (vfio_prereg_listener_skipped_section(section)) {
+        trace_vfio_prereg_listener_region_del_skip(
+                section->offset_within_address_space,
+                section->offset_within_address_space +
+                int128_get64(int128_sub(section->size, int128_one())));
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~page_mask) ||
+                 (section->offset_within_region & ~page_mask) ||
+                 (int128_get64(section->size) & ~page_mask))) {
+        error_report("%s received unaligned region", __func__);
+        return;
+    }
+
+    end = section->offset_within_address_space + int128_get64(section->size);
+    if (gpa >= end) {
+        return;
+    }
+
+    reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa);
+    reg.size = end - gpa;
+
+    ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, &reg);
+    trace_vfio_prereg_unregister(reg.vaddr, reg.size, ret ? -errno : 0);
+}
+
+const MemoryListener vfio_prereg_listener = {
+    .region_add = vfio_prereg_listener_region_add,
+    .region_del = vfio_prereg_listener_region_del,
+};
+
+int vfio_spapr_create_window(VFIOContainer *container,
+                             MemoryRegionSection *section,
+                             hwaddr *pgsize)
+{
+    int ret;
+    unsigned pagesize = memory_region_iommu_get_min_page_size(section->mr);
+    unsigned entries, pages;
+    struct vfio_iommu_spapr_tce_create create = { .argsz = sizeof(create) };
+
+    /*
+     * FIXME: For VFIO iommu types which have KVM acceleration to
+     * avoid bouncing all map/unmaps through qemu this way, this
+     * would be the right place to wire that up (tell the KVM
+     * device emulation the VFIO iommu handles to use).
+     */
+    create.window_size = int128_get64(section->size);
+    create.page_shift = ctz64(pagesize);
+    /*
+     * SPAPR host supports multilevel TCE tables, there is some
+     * heuristic to decide how many levels we want for our table:
+     * 0..64 = 1; 65..4096 = 2; 4097..262144 = 3; 262145.. = 4
+     */
+    entries = create.window_size >> create.page_shift;
+    pages = MAX((entries * sizeof(uint64_t)) / getpagesize(), 1);
+    pages = MAX(pow2ceil(pages) - 1, 1); /* Round up */
+    create.levels = ctz64(pages) / 6 + 1;
+
+    ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create);
+    if (ret) {
+        error_report("Failed to create a window, ret = %d (%m)", ret);
+        return -errno;
+    }
+
+    if (create.start_addr != section->offset_within_address_space) {
+        vfio_spapr_remove_window(container, create.start_addr);
+
+        error_report("Host doesn't support DMA window at %"HWADDR_PRIx", must be %"PRIx64,
+                     section->offset_within_address_space,
+                     (uint64_t)create.start_addr);
+        return -EINVAL;
+    }
+    trace_vfio_spapr_create_window(create.page_shift,
+                                   create.window_size,
+                                   create.start_addr);
+    *pgsize = pagesize;
+
+    return 0;
+}
+
+int vfio_spapr_remove_window(VFIOContainer *container,
+                             hwaddr offset_within_address_space)
+{
+    struct vfio_iommu_spapr_tce_remove remove = {
+        .argsz = sizeof(remove),
+        .start_addr = offset_within_address_space,
+    };
+    int ret;
+
+    ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove);
+    if (ret) {
+        error_report("Failed to remove window at %"PRIx64,
+                     (uint64_t)remove.start_addr);
+        return -errno;
+    }
+
+    trace_vfio_spapr_remove_window(offset_within_address_space);
+
+    return 0;
+}
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
new file mode 100644
index 0000000..da13322
--- /dev/null
+++ b/hw/vfio/trace-events
@@ -0,0 +1,125 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/vfio/pci.c
+vfio_intx_interrupt(const char *name, char line) " (%s) Pin %c"
+vfio_intx_eoi(const char *name) " (%s) EOI"
+vfio_intx_enable_kvm(const char *name) " (%s) KVM INTx accel enabled"
+vfio_intx_disable_kvm(const char *name) " (%s) KVM INTx accel disabled"
+vfio_intx_update(const char *name, int new_irq, int target_irq) " (%s) IRQ moved %d -> %d"
+vfio_intx_enable(const char *name) " (%s)"
+vfio_intx_disable(const char *name) " (%s)"
+vfio_msi_interrupt(const char *name, int index, uint64_t addr, int data) " (%s) vector %d 0x%"PRIx64"/0x%x"
+vfio_msix_vector_do_use(const char *name, int index) " (%s) vector %d used"
+vfio_msix_vector_release(const char *name, int index) " (%s) vector %d released"
+vfio_msix_enable(const char *name) " (%s)"
+vfio_msix_pba_disable(const char *name) " (%s)"
+vfio_msix_pba_enable(const char *name) " (%s)"
+vfio_msix_disable(const char *name) " (%s)"
+vfio_msix_fixup(const char *name, int bar, uint64_t start, uint64_t end) " (%s) MSI-X region %d mmap fixup [0x%"PRIx64" - 0x%"PRIx64"]"
+vfio_msi_enable(const char *name, int nr_vectors) " (%s) Enabled %d MSI vectors"
+vfio_msi_disable(const char *name) " (%s)"
+vfio_pci_load_rom(const char *name, unsigned long size, unsigned long offset, unsigned long flags) "Device %s ROM:\n  size: 0x%lx, offset: 0x%lx, flags: 0x%lx"
+vfio_rom_read(const char *name, uint64_t addr, int size, uint64_t data) " (%s, 0x%"PRIx64", 0x%x) = 0x%"PRIx64
+vfio_pci_size_rom(const char *name, int size) "%s ROM size 0x%x"
+vfio_vga_write(uint64_t addr, uint64_t data, int size) " (0x%"PRIx64", 0x%"PRIx64", %d)"
+vfio_vga_read(uint64_t addr, int size, uint64_t data) " (0x%"PRIx64", %d) = 0x%"PRIx64
+vfio_pci_read_config(const char *name, int addr, int len, int val) " (%s, @0x%x, len=0x%x) %x"
+vfio_pci_write_config(const char *name, int addr, int val, int len) " (%s, @0x%x, 0x%x, len=0x%x)"
+vfio_msi_setup(const char *name, int pos) "%s PCI MSI CAP @0x%x"
+vfio_msix_early_setup(const char *name, int pos, int table_bar, int offset, int entries) "%s PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d"
+vfio_check_pcie_flr(const char *name) "%s Supports FLR via PCIe cap"
+vfio_check_pm_reset(const char *name) "%s Supports PM reset"
+vfio_check_af_flr(const char *name) "%s Supports FLR via AF cap"
+vfio_pci_hot_reset(const char *name, const char *type) " (%s) %s"
+vfio_pci_hot_reset_has_dep_devices(const char *name) "%s: hot reset dependent devices:"
+vfio_pci_hot_reset_dep_devices(int domain, int bus, int slot, int function, int group_id) "\t%04x:%02x:%02x.%x group %d"
+vfio_pci_hot_reset_result(const char *name, const char *result) "%s hot reset: %s"
+vfio_populate_device_config(const char *name, unsigned long size, unsigned long offset, unsigned long flags) "Device %s config:\n  size: 0x%lx, offset: 0x%lx, flags: 0x%lx"
+vfio_populate_device_get_irq_info_failure(void) "VFIO_DEVICE_GET_IRQ_INFO failure: %m"
+vfio_initfn(const char *name, int group_id) " (%s) group %d"
+vfio_add_ext_cap_dropped(const char *name, uint16_t cap, uint16_t offset) "%s %x@%x"
+vfio_pci_reset(const char *name) " (%s)"
+vfio_pci_reset_flr(const char *name) "%s FLR/VFIO_DEVICE_RESET"
+vfio_pci_reset_pm(const char *name) "%s PCI PM Reset"
+vfio_pci_emulated_vendor_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_device_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_sub_vendor_id(const char *name, uint16_t val) "%s %04x"
+vfio_pci_emulated_sub_device_id(const char *name, uint16_t val) "%s %04x"
+
+# hw/vfio/pci-quirks.
+vfio_quirk_rom_blacklisted(const char *name, uint16_t vid, uint16_t did) "%s %04x:%04x"
+vfio_quirk_generic_window_address_write(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_window_data_read(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_window_data_write(const char *name, const char * region_name, uint64_t data) "%s %s 0x%"PRIx64
+vfio_quirk_generic_mirror_read(const char *name, const char * region_name, uint64_t addr, uint64_t data) "%s %s 0x%"PRIx64": 0x%"PRIx64
+vfio_quirk_generic_mirror_write(const char *name, const char * region_name, uint64_t addr, uint64_t data) "%s %s 0x%"PRIx64": 0x%"PRIx64
+vfio_quirk_ati_3c3_read(const char *name, uint64_t data) "%s 0x%"PRIx64
+vfio_quirk_ati_3c3_probe(const char *name) "%s"
+vfio_quirk_ati_bar4_probe(const char *name) "%s"
+vfio_quirk_ati_bar2_probe(const char *name) "%s"
+vfio_quirk_nvidia_3d0_state(const char *name, const char *state) "%s %s"
+vfio_quirk_nvidia_3d0_read(const char *name, uint8_t offset, unsigned size, uint64_t val) " (%s, @0x%x, len=0x%x) %"PRIx64
+vfio_quirk_nvidia_3d0_write(const char *name, uint8_t offset, uint64_t data, unsigned size) "(%s, @0x%x, 0x%"PRIx64", len=0x%x)"
+vfio_quirk_nvidia_3d0_probe(const char *name) "%s"
+vfio_quirk_nvidia_bar5_state(const char *name, const char *state) "%s %s"
+vfio_quirk_nvidia_bar5_probe(const char *name) "%s"
+vfio_quirk_nvidia_bar0_msi_ack(const char *name) "%s"
+vfio_quirk_nvidia_bar0_probe(const char *name) "%s"
+vfio_quirk_rtl8168_fake_latch(const char *name, uint64_t val) "%s 0x%"PRIx64
+vfio_quirk_rtl8168_msix_write(const char *name, uint16_t offset, uint64_t val) "%s MSI-X table write[0x%x]: 0x%"PRIx64
+vfio_quirk_rtl8168_msix_read(const char *name, uint16_t offset, uint64_t val) "%s MSI-X table read[0x%x]: 0x%"PRIx64
+vfio_quirk_rtl8168_probe(const char *name) "%s"
+
+vfio_quirk_ati_bonaire_reset_skipped(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_no_smc(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_timeout(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset_done(const char *name) "%s"
+vfio_quirk_ati_bonaire_reset(const char *name) "%s"
+vfio_pci_igd_bar4_write(const char *name, uint32_t index, uint32_t data, uint32_t base) "%s [%03x] %08x -> %08x"
+vfio_pci_igd_bdsm_enabled(const char *name, int size) "%s %dMB"
+vfio_pci_igd_opregion_enabled(const char *name) "%s"
+vfio_pci_igd_host_bridge_enabled(const char *name) "%s"
+vfio_pci_igd_lpc_bridge_enabled(const char *name) "%s"
+
+# hw/vfio/common.c
+vfio_region_write(const char *name, int index, uint64_t addr, uint64_t data, unsigned size) " (%s:region%d+0x%"PRIx64", 0x%"PRIx64 ", %d)"
+vfio_region_read(char *name, int index, uint64_t addr, unsigned size, uint64_t data) " (%s:region%d+0x%"PRIx64", %d) = 0x%"PRIx64
+vfio_iommu_map_notify(uint64_t iova_start, uint64_t iova_end) "iommu map @ %"PRIx64" - %"PRIx64
+vfio_listener_region_add_skip(uint64_t start, uint64_t end) "SKIPPING region_add %"PRIx64" - %"PRIx64
+vfio_listener_region_add_iommu(uint64_t start, uint64_t end) "region_add [iommu] %"PRIx64" - %"PRIx64
+vfio_listener_region_add_ram(uint64_t iova_start, uint64_t iova_end, void *vaddr) "region_add [ram] %"PRIx64" - %"PRIx64" [%p]"
+vfio_listener_region_del_skip(uint64_t start, uint64_t end) "SKIPPING region_del %"PRIx64" - %"PRIx64
+vfio_listener_region_del(uint64_t start, uint64_t end) "region_del %"PRIx64" - %"PRIx64
+vfio_disconnect_container(int fd) "close container->fd=%d"
+vfio_put_group(int fd) "close group->fd=%d"
+vfio_get_device(const char * name, unsigned int flags, unsigned int num_regions, unsigned int num_irqs) "Device %s flags: %u, regions: %u, irqs: %u"
+vfio_put_base_device(int fd) "close vdev->fd=%d"
+vfio_region_setup(const char *dev, int index, const char *name, unsigned long flags, unsigned long offset, unsigned long size) "Device %s, region %d \"%s\", flags: %lx, offset: %lx, size: %lx"
+vfio_region_mmap_fault(const char *name, int index, unsigned long offset, unsigned long size, int fault) "Region %s mmaps[%d], [%lx - %lx], fault: %d"
+vfio_region_mmap(const char *name, unsigned long offset, unsigned long end) "Region %s [%lx - %lx]"
+vfio_region_exit(const char *name, int index) "Device %s, region %d"
+vfio_region_finalize(const char *name, int index) "Device %s, region %d"
+vfio_region_mmaps_set_enabled(const char *name, bool enabled) "Region %s mmaps enabled: %d"
+vfio_region_sparse_mmap_header(const char *name, int index, int nr_areas) "Device %s region %d: %d sparse mmap entries"
+vfio_region_sparse_mmap_entry(int i, unsigned long start, unsigned long end) "sparse entry %d [0x%lx - 0x%lx]"
+vfio_get_dev_region(const char *name, int index, uint32_t type, uint32_t subtype) "%s index %d, %08x/%0x8"
+
+# hw/vfio/platform.c
+vfio_platform_base_device_init(char *name, int groupid) "%s belongs to group #%d"
+vfio_platform_realize(char *name, char *compat) "vfio device %s, compat = %s"
+vfio_platform_eoi(int pin, int fd) "EOI IRQ pin %d (fd=%d)"
+vfio_platform_intp_mmap_enable(int pin) "IRQ #%d still active, stay in slow path"
+vfio_platform_intp_interrupt(int pin, int fd) "Inject IRQ #%d (fd = %d)"
+vfio_platform_intp_inject_pending_lockheld(int pin, int fd) "Inject pending IRQ #%d (fd = %d)"
+vfio_platform_populate_interrupts(int pin, int count, int flags) "- IRQ index %d: count %d, flags=0x%x"
+vfio_intp_interrupt_set_pending(int index) "irq %d is set PENDING"
+vfio_platform_start_level_irqfd_injection(int index, int fd, int resamplefd) "IRQ index=%d, fd = %d, resamplefd = %d"
+vfio_platform_start_edge_irqfd_injection(int index, int fd) "IRQ index=%d, fd = %d"
+
+# hw/vfio/spapr.c
+vfio_prereg_listener_region_add_skip(uint64_t start, uint64_t end) "%"PRIx64" - %"PRIx64
+vfio_prereg_listener_region_del_skip(uint64_t start, uint64_t end) "%"PRIx64" - %"PRIx64
+vfio_prereg_register(uint64_t va, uint64_t size, int ret) "va=%"PRIx64" size=%"PRIx64" ret=%d"
+vfio_prereg_unregister(uint64_t va, uint64_t size, int ret) "va=%"PRIx64" size=%"PRIx64" ret=%d"
+vfio_spapr_create_window(int ps, uint64_t ws, uint64_t off) "pageshift=0x%x winsize=0x%"PRIx64" offset=0x%"PRIx64
+vfio_spapr_remove_window(uint64_t off) "offset=%"PRIx64
diff --git a/hw/virtio/Makefile.objs b/hw/virtio/Makefile.objs
index d21c397..3e2b175 100644
--- a/hw/virtio/Makefile.objs
+++ b/hw/virtio/Makefile.objs
@@ -2,7 +2,6 @@
 common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
 common-obj-y += virtio-bus.o
 common-obj-y += virtio-mmio.o
-common-obj-$(CONFIG_VIRTIO) += dataplane/
 
 obj-y += virtio.o virtio-balloon.o 
 obj-$(CONFIG_LINUX) += vhost.o vhost-backend.o vhost-user.o
diff --git a/hw/virtio/dataplane/Makefile.objs b/hw/virtio/dataplane/Makefile.objs
deleted file mode 100644
index 9a8cfc0..0000000
--- a/hw/virtio/dataplane/Makefile.objs
+++ /dev/null
@@ -1 +0,0 @@
-common-obj-y += vring.o
diff --git a/hw/virtio/dataplane/vring.c b/hw/virtio/dataplane/vring.c
deleted file mode 100644
index 61f6d83..0000000
--- a/hw/virtio/dataplane/vring.c
+++ /dev/null
@@ -1,436 +0,0 @@
-/* Copyright 2012 Red Hat, Inc.
- * Copyright IBM, Corp. 2012
- *
- * Based on Linux 2.6.39 vhost code:
- * Copyright (C) 2009 Red Hat, Inc.
- * Copyright (C) 2006 Rusty Russell IBM Corporation
- *
- * Author: Michael S. Tsirkin <mst@redhat.com>
- *         Stefan Hajnoczi <stefanha@redhat.com>
- *
- * Inspiration, some code, and most witty comments come from
- * Documentation/virtual/lguest/lguest.c, by Rusty Russell
- *
- * This work is licensed under the terms of the GNU GPL, version 2.
- */
-
-#include "trace.h"
-#include "hw/hw.h"
-#include "exec/memory.h"
-#include "exec/address-spaces.h"
-#include "hw/virtio/dataplane/vring.h"
-#include "qemu/error-report.h"
-
-/* vring_map can be coupled with vring_unmap or (if you still have the
- * value returned in *mr) memory_region_unref.
- */
-static void *vring_map(MemoryRegion **mr, hwaddr phys, hwaddr len,
-                       bool is_write)
-{
-    MemoryRegionSection section = memory_region_find(get_system_memory(), phys, len);
-
-    if (!section.mr || int128_get64(section.size) < len) {
-        goto out;
-    }
-    if (is_write && section.readonly) {
-        goto out;
-    }
-    if (!memory_region_is_ram(section.mr)) {
-        goto out;
-    }
-
-    /* Ignore regions with dirty logging, we cannot mark them dirty */
-    if (memory_region_is_logging(section.mr)) {
-        goto out;
-    }
-
-    *mr = section.mr;
-    return memory_region_get_ram_ptr(section.mr) + section.offset_within_region;
-
-out:
-    memory_region_unref(section.mr);
-    *mr = NULL;
-    return NULL;
-}
-
-static void vring_unmap(void *buffer, bool is_write)
-{
-    ram_addr_t addr;
-    MemoryRegion *mr;
-
-    mr = qemu_ram_addr_from_host(buffer, &addr);
-    memory_region_unref(mr);
-}
-
-/* Map the guest's vring to host memory */
-bool vring_setup(Vring *vring, VirtIODevice *vdev, int n)
-{
-    hwaddr vring_addr = virtio_queue_get_ring_addr(vdev, n);
-    hwaddr vring_size = virtio_queue_get_ring_size(vdev, n);
-    void *vring_ptr;
-
-    vring->broken = false;
-
-    vring_ptr = vring_map(&vring->mr, vring_addr, vring_size, true);
-    if (!vring_ptr) {
-        error_report("Failed to map vring "
-                     "addr %#" HWADDR_PRIx " size %" HWADDR_PRIu,
-                     vring_addr, vring_size);
-        vring->broken = true;
-        return false;
-    }
-
-    vring_init(&vring->vr, virtio_queue_get_num(vdev, n), vring_ptr, 4096);
-
-    vring->last_avail_idx = virtio_queue_get_last_avail_idx(vdev, n);
-    vring->last_used_idx = vring->vr.used->idx;
-    vring->signalled_used = 0;
-    vring->signalled_used_valid = false;
-
-    trace_vring_setup(virtio_queue_get_ring_addr(vdev, n),
-                      vring->vr.desc, vring->vr.avail, vring->vr.used);
-    return true;
-}
-
-void vring_teardown(Vring *vring, VirtIODevice *vdev, int n)
-{
-    virtio_queue_set_last_avail_idx(vdev, n, vring->last_avail_idx);
-    virtio_queue_invalidate_signalled_used(vdev, n);
-
-    memory_region_unref(vring->mr);
-}
-
-/* Disable guest->host notifies */
-void vring_disable_notification(VirtIODevice *vdev, Vring *vring)
-{
-    if (!(vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX))) {
-        vring->vr.used->flags |= VRING_USED_F_NO_NOTIFY;
-    }
-}
-
-/* Enable guest->host notifies
- *
- * Return true if the vring is empty, false if there are more requests.
- */
-bool vring_enable_notification(VirtIODevice *vdev, Vring *vring)
-{
-    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(&vring->vr) = vring->vr.avail->idx;
-    } else {
-        vring->vr.used->flags &= ~VRING_USED_F_NO_NOTIFY;
-    }
-    smp_mb(); /* ensure update is seen before reading avail_idx */
-    return !vring_more_avail(vring);
-}
-
-/* This is stolen from linux/drivers/vhost/vhost.c:vhost_notify() */
-bool vring_should_notify(VirtIODevice *vdev, Vring *vring)
-{
-    uint16_t old, new;
-    bool v;
-    /* Flush out used index updates. This is paired
-     * with the barrier that the Guest executes when enabling
-     * interrupts. */
-    smp_mb();
-
-    if ((vdev->guest_features & VIRTIO_F_NOTIFY_ON_EMPTY) &&
-        unlikely(vring->vr.avail->idx == vring->last_avail_idx)) {
-        return true;
-    }
-
-    if (!(vdev->guest_features & VIRTIO_RING_F_EVENT_IDX)) {
-        return !(vring->vr.avail->flags & VRING_AVAIL_F_NO_INTERRUPT);
-    }
-    old = vring->signalled_used;
-    v = vring->signalled_used_valid;
-    new = vring->signalled_used = vring->last_used_idx;
-    vring->signalled_used_valid = true;
-
-    if (unlikely(!v)) {
-        return true;
-    }
-
-    return vring_need_event(vring_used_event(&vring->vr), new, old);
-}
-
-
-static int get_desc(Vring *vring, VirtQueueElement *elem,
-                    struct vring_desc *desc)
-{
-    unsigned *num;
-    struct iovec *iov;
-    hwaddr *addr;
-    MemoryRegion *mr;
-
-    if (desc->flags & VRING_DESC_F_WRITE) {
-        num = &elem->in_num;
-        iov = &elem->in_sg[*num];
-        addr = &elem->in_addr[*num];
-    } else {
-        num = &elem->out_num;
-        iov = &elem->out_sg[*num];
-        addr = &elem->out_addr[*num];
-
-        /* If it's an output descriptor, they're all supposed
-         * to come before any input descriptors. */
-        if (unlikely(elem->in_num)) {
-            error_report("Descriptor has out after in");
-            return -EFAULT;
-        }
-    }
-
-    /* Stop for now if there are not enough iovecs available. */
-    if (*num >= VIRTQUEUE_MAX_SIZE) {
-        error_report("Invalid SG num: %u", *num);
-        return -EFAULT;
-    }
-
-    /* TODO handle non-contiguous memory across region boundaries */
-    iov->iov_base = vring_map(&mr, desc->addr, desc->len,
-                              desc->flags & VRING_DESC_F_WRITE);
-    if (!iov->iov_base) {
-        error_report("Failed to map descriptor addr %#" PRIx64 " len %u",
-                     (uint64_t)desc->addr, desc->len);
-        return -EFAULT;
-    }
-
-    /* The MemoryRegion is looked up again and unref'ed later, leave the
-     * ref in place.  */
-    iov->iov_len = desc->len;
-    *addr = desc->addr;
-    *num += 1;
-    return 0;
-}
-
-/* This is stolen from linux/drivers/vhost/vhost.c. */
-static int get_indirect(Vring *vring, VirtQueueElement *elem,
-                        struct vring_desc *indirect)
-{
-    struct vring_desc desc;
-    unsigned int i = 0, count, found = 0;
-    int ret;
-
-    /* Sanity check */
-    if (unlikely(indirect->len % sizeof(desc))) {
-        error_report("Invalid length in indirect descriptor: "
-                     "len %#x not multiple of %#zx",
-                     indirect->len, sizeof(desc));
-        vring->broken = true;
-        return -EFAULT;
-    }
-
-    count = indirect->len / sizeof(desc);
-    /* Buffers are chained via a 16 bit next field, so
-     * we can have at most 2^16 of these. */
-    if (unlikely(count > USHRT_MAX + 1)) {
-        error_report("Indirect buffer length too big: %d", indirect->len);
-        vring->broken = true;
-        return -EFAULT;
-    }
-
-    do {
-        struct vring_desc *desc_ptr;
-        MemoryRegion *mr;
-
-        /* Translate indirect descriptor */
-        desc_ptr = vring_map(&mr,
-                             indirect->addr + found * sizeof(desc),
-                             sizeof(desc), false);
-        if (!desc_ptr) {
-            error_report("Failed to map indirect descriptor "
-                         "addr %#" PRIx64 " len %zu",
-                         (uint64_t)indirect->addr + found * sizeof(desc),
-                         sizeof(desc));
-            vring->broken = true;
-            return -EFAULT;
-        }
-        desc = *desc_ptr;
-        memory_region_unref(mr);
-
-        /* Ensure descriptor has been loaded before accessing fields */
-        barrier(); /* read_barrier_depends(); */
-
-        if (unlikely(++found > count)) {
-            error_report("Loop detected: last one at %u "
-                         "indirect size %u", i, count);
-            vring->broken = true;
-            return -EFAULT;
-        }
-
-        if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) {
-            error_report("Nested indirect descriptor");
-            vring->broken = true;
-            return -EFAULT;
-        }
-
-        ret = get_desc(vring, elem, &desc);
-        if (ret < 0) {
-            vring->broken |= (ret == -EFAULT);
-            return ret;
-        }
-        i = desc.next;
-    } while (desc.flags & VRING_DESC_F_NEXT);
-    return 0;
-}
-
-static void vring_unmap_element(VirtQueueElement *elem)
-{
-    int i;
-
-    /* This assumes that the iovecs, if changed, are never moved past
-     * the end of the valid area.  This is true if iovec manipulations
-     * are done with iov_discard_front and iov_discard_back.
-     */
-    for (i = 0; i < elem->out_num; i++) {
-        vring_unmap(elem->out_sg[i].iov_base, false);
-    }
-
-    for (i = 0; i < elem->in_num; i++) {
-        vring_unmap(elem->in_sg[i].iov_base, true);
-    }
-}
-
-/* This looks in the virtqueue and for the first available buffer, and converts
- * it to an iovec for convenient access.  Since descriptors consist of some
- * number of output then some number of input descriptors, it's actually two
- * iovecs, but we pack them into one and note how many of each there were.
- *
- * This function returns the descriptor number found, or vq->num (which is
- * never a valid descriptor number) if none was found.  A negative code is
- * returned on error.
- *
- * Stolen from linux/drivers/vhost/vhost.c.
- */
-int vring_pop(VirtIODevice *vdev, Vring *vring,
-              VirtQueueElement *elem)
-{
-    struct vring_desc desc;
-    unsigned int i, head, found = 0, num = vring->vr.num;
-    uint16_t avail_idx, last_avail_idx;
-    int ret;
-
-    /* Initialize elem so it can be safely unmapped */
-    elem->in_num = elem->out_num = 0;
-
-    /* If there was a fatal error then refuse operation */
-    if (vring->broken) {
-        ret = -EFAULT;
-        goto out;
-    }
-
-    /* Check it isn't doing very strange things with descriptor numbers. */
-    last_avail_idx = vring->last_avail_idx;
-    avail_idx = vring->vr.avail->idx;
-    barrier(); /* load indices now and not again later */
-
-    if (unlikely((uint16_t)(avail_idx - last_avail_idx) > num)) {
-        error_report("Guest moved used index from %u to %u",
-                     last_avail_idx, avail_idx);
-        ret = -EFAULT;
-        goto out;
-    }
-
-    /* If there's nothing new since last we looked. */
-    if (avail_idx == last_avail_idx) {
-        ret = -EAGAIN;
-        goto out;
-    }
-
-    /* Only get avail ring entries after they have been exposed by guest. */
-    smp_rmb();
-
-    /* Grab the next descriptor number they're advertising, and increment
-     * the index we've seen. */
-    head = vring->vr.avail->ring[last_avail_idx % num];
-
-    elem->index = head;
-
-    /* If their number is silly, that's an error. */
-    if (unlikely(head >= num)) {
-        error_report("Guest says index %u > %u is available", head, num);
-        ret = -EFAULT;
-        goto out;
-    }
-
-    i = head;
-    do {
-        if (unlikely(i >= num)) {
-            error_report("Desc index is %u > %u, head = %u", i, num, head);
-            ret = -EFAULT;
-            goto out;
-        }
-        if (unlikely(++found > num)) {
-            error_report("Loop detected: last one at %u vq size %u head %u",
-                         i, num, head);
-            ret = -EFAULT;
-            goto out;
-        }
-        desc = vring->vr.desc[i];
-
-        /* Ensure descriptor is loaded before accessing fields */
-        barrier();
-
-        if (desc.flags & VRING_DESC_F_INDIRECT) {
-            ret = get_indirect(vring, elem, &desc);
-            if (ret < 0) {
-                goto out;
-            }
-            continue;
-        }
-
-        ret = get_desc(vring, elem, &desc);
-        if (ret < 0) {
-            goto out;
-        }
-
-        i = desc.next;
-    } while (desc.flags & VRING_DESC_F_NEXT);
-
-    /* On success, increment avail index. */
-    vring->last_avail_idx++;
-    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(&vring->vr) = vring->last_avail_idx;
-    }
-
-    return head;
-
-out:
-    assert(ret < 0);
-    if (ret == -EFAULT) {
-        vring->broken = true;
-    }
-    vring_unmap_element(elem);
-    return ret;
-}
-
-/* After we've used one of their buffers, we tell them about it.
- *
- * Stolen from linux/drivers/vhost/vhost.c.
- */
-void vring_push(Vring *vring, VirtQueueElement *elem, int len)
-{
-    struct vring_used_elem *used;
-    unsigned int head = elem->index;
-    uint16_t new;
-
-    vring_unmap_element(elem);
-
-    /* Don't touch vring if a fatal error occurred */
-    if (vring->broken) {
-        return;
-    }
-
-    /* The virtqueue contains a ring of used buffers.  Get a pointer to the
-     * next entry in that used ring. */
-    used = &vring->vr.used->ring[vring->last_used_idx % vring->vr.num];
-    used->id = head;
-    used->len = len;
-
-    /* Make sure buffer is written before we update index. */
-    smp_wmb();
-
-    new = vring->vr.used->idx = ++vring->last_used_idx;
-    if (unlikely((int16_t)(new - vring->signalled_used) < (uint16_t)1)) {
-        vring->signalled_used_valid = false;
-    }
-}
diff --git a/hw/virtio/trace-events b/hw/virtio/trace-events
new file mode 100644
index 0000000..55184d3
--- /dev/null
+++ b/hw/virtio/trace-events
@@ -0,0 +1,16 @@
+# See docs/tracing.txt for syntax documentation.
+
+# hw/virtio/virtio.c
+virtqueue_fill(void *vq, const void *elem, unsigned int len, unsigned int idx) "vq %p elem %p len %u idx %u"
+virtqueue_flush(void *vq, unsigned int count) "vq %p count %u"
+virtqueue_pop(void *vq, void *elem, unsigned int in_num, unsigned int out_num) "vq %p elem %p in_num %u out_num %u"
+virtio_queue_notify(void *vdev, int n, void *vq) "vdev %p n %d vq %p"
+virtio_irq(void *vq) "vq %p"
+virtio_notify(void *vdev, void *vq) "vdev %p vq %p"
+virtio_set_status(void *vdev, uint8_t val) "vdev %p val %u"
+
+# hw/virtio/virtio-rng.c
+virtio_rng_guest_not_ready(void *rng) "rng %p: guest not ready"
+virtio_rng_pushed(void *rng, size_t len) "rng %p: %zd bytes pushed"
+virtio_rng_request(void *rng, size_t size, unsigned quota) "rng %p: %zd bytes requested, %u bytes quota left"
+
diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c
index ff4f200..7681f15 100644
--- a/hw/virtio/vhost-backend.c
+++ b/hw/virtio/vhost-backend.c
@@ -8,12 +8,13 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include <linux/vhost.h>
+#include <sys/ioctl.h>
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
 #include "qemu/error-report.h"
 
-#include <sys/ioctl.h>
-
 static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
                              void *arg)
 {
@@ -42,11 +43,160 @@
     return close(fd);
 }
 
+static int vhost_kernel_memslots_limit(struct vhost_dev *dev)
+{
+    int limit = 64;
+    char *s;
+
+    if (g_file_get_contents("/sys/module/vhost/parameters/max_mem_regions",
+                            &s, NULL, NULL)) {
+        uint64_t val = g_ascii_strtoull(s, NULL, 10);
+        if (!((val == G_MAXUINT64 || !val) && errno)) {
+            return val;
+        }
+        error_report("ignoring invalid max_mem_regions value in vhost module:"
+                     " %s", s);
+    }
+    return limit;
+}
+
+static int vhost_kernel_net_set_backend(struct vhost_dev *dev,
+                                        struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_NET_SET_BACKEND, file);
+}
+
+static int vhost_kernel_scsi_set_endpoint(struct vhost_dev *dev,
+                                          struct vhost_scsi_target *target)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_SET_ENDPOINT, target);
+}
+
+static int vhost_kernel_scsi_clear_endpoint(struct vhost_dev *dev,
+                                            struct vhost_scsi_target *target)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_CLEAR_ENDPOINT, target);
+}
+
+static int vhost_kernel_scsi_get_abi_version(struct vhost_dev *dev, int *version)
+{
+    return vhost_kernel_call(dev, VHOST_SCSI_GET_ABI_VERSION, version);
+}
+
+static int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log)
+{
+    return vhost_kernel_call(dev, VHOST_SET_LOG_BASE, &base);
+}
+
+static int vhost_kernel_set_mem_table(struct vhost_dev *dev,
+                                      struct vhost_memory *mem)
+{
+    return vhost_kernel_call(dev, VHOST_SET_MEM_TABLE, mem);
+}
+
+static int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr);
+}
+
+static int vhost_kernel_set_vring_endian(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_ENDIAN, ring);
+}
+
+static int vhost_kernel_set_vring_num(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_NUM, ring);
+}
+
+static int vhost_kernel_set_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_BASE, ring);
+}
+
+static int vhost_kernel_get_vring_base(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    return vhost_kernel_call(dev, VHOST_GET_VRING_BASE, ring);
+}
+
+static int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_KICK, file);
+}
+
+static int vhost_kernel_set_vring_call(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_CALL, file);
+}
+
+static int vhost_kernel_set_vring_busyloop_timeout(struct vhost_dev *dev,
+                                                   struct vhost_vring_state *s)
+{
+    return vhost_kernel_call(dev, VHOST_SET_VRING_BUSYLOOP_TIMEOUT, s);
+}
+
+static int vhost_kernel_set_features(struct vhost_dev *dev,
+                                     uint64_t features)
+{
+    return vhost_kernel_call(dev, VHOST_SET_FEATURES, &features);
+}
+
+static int vhost_kernel_get_features(struct vhost_dev *dev,
+                                     uint64_t *features)
+{
+    return vhost_kernel_call(dev, VHOST_GET_FEATURES, features);
+}
+
+static int vhost_kernel_set_owner(struct vhost_dev *dev)
+{
+    return vhost_kernel_call(dev, VHOST_SET_OWNER, NULL);
+}
+
+static int vhost_kernel_reset_device(struct vhost_dev *dev)
+{
+    return vhost_kernel_call(dev, VHOST_RESET_OWNER, NULL);
+}
+
+static int vhost_kernel_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx - dev->vq_index;
+}
+
 static const VhostOps kernel_ops = {
         .backend_type = VHOST_BACKEND_TYPE_KERNEL,
-        .vhost_call = vhost_kernel_call,
         .vhost_backend_init = vhost_kernel_init,
-        .vhost_backend_cleanup = vhost_kernel_cleanup
+        .vhost_backend_cleanup = vhost_kernel_cleanup,
+        .vhost_backend_memslots_limit = vhost_kernel_memslots_limit,
+        .vhost_net_set_backend = vhost_kernel_net_set_backend,
+        .vhost_scsi_set_endpoint = vhost_kernel_scsi_set_endpoint,
+        .vhost_scsi_clear_endpoint = vhost_kernel_scsi_clear_endpoint,
+        .vhost_scsi_get_abi_version = vhost_kernel_scsi_get_abi_version,
+        .vhost_set_log_base = vhost_kernel_set_log_base,
+        .vhost_set_mem_table = vhost_kernel_set_mem_table,
+        .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
+        .vhost_set_vring_endian = vhost_kernel_set_vring_endian,
+        .vhost_set_vring_num = vhost_kernel_set_vring_num,
+        .vhost_set_vring_base = vhost_kernel_set_vring_base,
+        .vhost_get_vring_base = vhost_kernel_get_vring_base,
+        .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
+        .vhost_set_vring_call = vhost_kernel_set_vring_call,
+        .vhost_set_vring_busyloop_timeout =
+                                vhost_kernel_set_vring_busyloop_timeout,
+        .vhost_set_features = vhost_kernel_set_features,
+        .vhost_get_features = vhost_kernel_get_features,
+        .vhost_set_owner = vhost_kernel_set_owner,
+        .vhost_reset_device = vhost_kernel_reset_device,
+        .vhost_get_vq_index = vhost_kernel_get_vq_index,
 };
 
 int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
@@ -61,7 +211,7 @@
         dev->vhost_ops = &user_ops;
         break;
     default:
-        error_report("Unknown vhost backend type\n");
+        error_report("Unknown vhost backend type");
         r = -1;
     }
 
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index aefe0bb..b57454a 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -8,22 +8,35 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/virtio/vhost.h"
 #include "hw/virtio/vhost-backend.h"
+#include "hw/virtio/virtio-net.h"
 #include "sysemu/char.h"
 #include "sysemu/kvm.h"
 #include "qemu/error-report.h"
 #include "qemu/sockets.h"
-#include "exec/ram_addr.h"
+#include "migration/migration.h"
 
-#include <fcntl.h>
-#include <unistd.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <linux/vhost.h>
 
 #define VHOST_MEMORY_MAX_NREGIONS    8
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+
+enum VhostUserProtocolFeature {
+    VHOST_USER_PROTOCOL_F_MQ = 0,
+    VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
+    VHOST_USER_PROTOCOL_F_RARP = 2,
+    VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
+
+    VHOST_USER_PROTOCOL_F_MAX
+};
+
+#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
 
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
@@ -41,6 +54,11 @@
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
+    VHOST_USER_SEND_RARP = 19,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -57,11 +75,17 @@
     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 } VhostUserMemory;
 
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
 #define VHOST_USER_VERSION_MASK     (0x3)
 #define VHOST_USER_REPLY_MASK       (0x1<<2)
+#define VHOST_USER_NEED_REPLY_MASK  (0x1 << 3)
     uint32_t flags;
     uint32_t size; /* the following payload size */
     union {
@@ -71,7 +95,8 @@
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
-    };
+        VhostUserLog log;
+    } payload;
 } QEMU_PACKED VhostUserMsg;
 
 static VhostUserMsg m __attribute__ ((unused));
@@ -89,37 +114,6 @@
     return kvm_enabled() && kvm_eventfds_enabled();
 }
 
-static unsigned long int ioctl_to_vhost_user_request[VHOST_USER_MAX] = {
-    -1,                     /* VHOST_USER_NONE */
-    VHOST_GET_FEATURES,     /* VHOST_USER_GET_FEATURES */
-    VHOST_SET_FEATURES,     /* VHOST_USER_SET_FEATURES */
-    VHOST_SET_OWNER,        /* VHOST_USER_SET_OWNER */
-    VHOST_RESET_OWNER,      /* VHOST_USER_RESET_OWNER */
-    VHOST_SET_MEM_TABLE,    /* VHOST_USER_SET_MEM_TABLE */
-    VHOST_SET_LOG_BASE,     /* VHOST_USER_SET_LOG_BASE */
-    VHOST_SET_LOG_FD,       /* VHOST_USER_SET_LOG_FD */
-    VHOST_SET_VRING_NUM,    /* VHOST_USER_SET_VRING_NUM */
-    VHOST_SET_VRING_ADDR,   /* VHOST_USER_SET_VRING_ADDR */
-    VHOST_SET_VRING_BASE,   /* VHOST_USER_SET_VRING_BASE */
-    VHOST_GET_VRING_BASE,   /* VHOST_USER_GET_VRING_BASE */
-    VHOST_SET_VRING_KICK,   /* VHOST_USER_SET_VRING_KICK */
-    VHOST_SET_VRING_CALL,   /* VHOST_USER_SET_VRING_CALL */
-    VHOST_SET_VRING_ERR     /* VHOST_USER_SET_VRING_ERR */
-};
-
-static VhostUserRequest vhost_user_request_translate(unsigned long int request)
-{
-    VhostUserRequest idx;
-
-    for (idx = 0; idx < VHOST_USER_MAX; idx++) {
-        if (ioctl_to_vhost_user_request[idx] == request) {
-            break;
-        }
-    }
-
-    return (idx == VHOST_USER_MAX) ? VHOST_USER_NONE : idx;
-}
-
 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
 {
     CharDriverState *chr = dev->opaque;
@@ -128,15 +122,15 @@
 
     r = qemu_chr_fe_read_all(chr, p, size);
     if (r != size) {
-        error_report("Failed to read msg header. Read %d instead of %d.\n", r,
-                size);
+        error_report("Failed to read msg header. Read %d instead of %d."
+                     " Original request %d.", r, size, msg->request);
         goto fail;
     }
 
     /* validate received flags */
     if (msg->flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
         error_report("Failed to read msg header."
-                " Flags 0x%x instead of 0x%x.\n", msg->flags,
+                " Flags 0x%x instead of 0x%x.", msg->flags,
                 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
         goto fail;
     }
@@ -144,7 +138,7 @@
     /* validate message size is sane */
     if (msg->size > VHOST_USER_PAYLOAD_SIZE) {
         error_report("Failed to read msg header."
-                " Size %d exceeds the maximum %zu.\n", msg->size,
+                " Size %d exceeds the maximum %zu.", msg->size,
                 VHOST_USER_PAYLOAD_SIZE);
         goto fail;
     }
@@ -155,7 +149,7 @@
         r = qemu_chr_fe_read_all(chr, p, size);
         if (r != size) {
             error_report("Failed to read msg payload."
-                         " Read %d instead of %d.\n", r, msg->size);
+                         " Read %d instead of %d.", r, msg->size);
             goto fail;
         }
     }
@@ -166,171 +160,458 @@
     return -1;
 }
 
+static int process_message_reply(struct vhost_dev *dev,
+                                 VhostUserRequest request)
+{
+    VhostUserMsg msg;
+
+    if (vhost_user_read(dev, &msg) < 0) {
+        return -1;
+    }
+
+    if (msg.request != request) {
+        error_report("Received unexpected msg type."
+                     "Expected %d received %d",
+                     request, msg.request);
+        return -1;
+    }
+
+    return msg.payload.u64 ? -1 : 0;
+}
+
+static bool vhost_user_one_time_request(VhostUserRequest request)
+{
+    switch (request) {
+    case VHOST_USER_SET_OWNER:
+    case VHOST_USER_RESET_OWNER:
+    case VHOST_USER_SET_MEM_TABLE:
+    case VHOST_USER_GET_QUEUE_NUM:
+        return true;
+    default:
+        return false;
+    }
+}
+
+/* most non-init callers ignore the error */
 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
                             int *fds, int fd_num)
 {
     CharDriverState *chr = dev->opaque;
-    int size = VHOST_USER_HDR_SIZE + msg->size;
+    int ret, size = VHOST_USER_HDR_SIZE + msg->size;
 
-    if (fd_num) {
-        qemu_chr_fe_set_msgfds(chr, fds, fd_num);
-    }
-
-    return qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size) == size ?
-            0 : -1;
-}
-
-static int vhost_user_call(struct vhost_dev *dev, unsigned long int request,
-        void *arg)
-{
-    VhostUserMsg msg;
-    VhostUserRequest msg_request;
-    struct vhost_vring_file *file = 0;
-    int need_reply = 0;
-    int fds[VHOST_MEMORY_MAX_NREGIONS];
-    int i, fd;
-    size_t fd_num = 0;
-
-    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
-
-    msg_request = vhost_user_request_translate(request);
-    msg.request = msg_request;
-    msg.flags = VHOST_USER_VERSION;
-    msg.size = 0;
-
-    switch (request) {
-    case VHOST_GET_FEATURES:
-        need_reply = 1;
-        break;
-
-    case VHOST_SET_FEATURES:
-    case VHOST_SET_LOG_BASE:
-        msg.u64 = *((__u64 *) arg);
-        msg.size = sizeof(m.u64);
-        break;
-
-    case VHOST_SET_OWNER:
-    case VHOST_RESET_OWNER:
-        break;
-
-    case VHOST_SET_MEM_TABLE:
-        for (i = 0; i < dev->mem->nregions; ++i) {
-            struct vhost_memory_region *reg = dev->mem->regions + i;
-            ram_addr_t ram_addr;
-
-            assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
-            qemu_ram_addr_from_host((void *)(uintptr_t)reg->userspace_addr, &ram_addr);
-            fd = qemu_get_ram_fd(ram_addr);
-            if (fd > 0) {
-                msg.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
-                msg.memory.regions[fd_num].memory_size  = reg->memory_size;
-                msg.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
-                msg.memory.regions[fd_num].mmap_offset = reg->userspace_addr -
-                    (uintptr_t) qemu_get_ram_block_host_ptr(ram_addr);
-                assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
-                fds[fd_num++] = fd;
-            }
-        }
-
-        msg.memory.nregions = fd_num;
-
-        if (!fd_num) {
-            error_report("Failed initializing vhost-user memory map\n"
-                    "consider using -object memory-backend-file share=on\n");
-            return -1;
-        }
-
-        msg.size = sizeof(m.memory.nregions);
-        msg.size += sizeof(m.memory.padding);
-        msg.size += fd_num * sizeof(VhostUserMemoryRegion);
-
-        break;
-
-    case VHOST_SET_LOG_FD:
-        fds[fd_num++] = *((int *) arg);
-        break;
-
-    case VHOST_SET_VRING_NUM:
-    case VHOST_SET_VRING_BASE:
-        memcpy(&msg.state, arg, sizeof(struct vhost_vring_state));
-        msg.size = sizeof(m.state);
-        break;
-
-    case VHOST_GET_VRING_BASE:
-        memcpy(&msg.state, arg, sizeof(struct vhost_vring_state));
-        msg.size = sizeof(m.state);
-        need_reply = 1;
-        break;
-
-    case VHOST_SET_VRING_ADDR:
-        memcpy(&msg.addr, arg, sizeof(struct vhost_vring_addr));
-        msg.size = sizeof(m.addr);
-        break;
-
-    case VHOST_SET_VRING_KICK:
-    case VHOST_SET_VRING_CALL:
-    case VHOST_SET_VRING_ERR:
-        file = arg;
-        msg.u64 = file->index & VHOST_USER_VRING_IDX_MASK;
-        msg.size = sizeof(m.u64);
-        if (ioeventfd_enabled() && file->fd > 0) {
-            fds[fd_num++] = file->fd;
-        } else {
-            msg.u64 |= VHOST_USER_VRING_NOFD_MASK;
-        }
-        break;
-    default:
-        error_report("vhost-user trying to send unhandled ioctl\n");
-        return -1;
-        break;
-    }
-
-    if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
+    /*
+     * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
+     * we just need send it once in the first time. For later such
+     * request, we just ignore it.
+     */
+    if (vhost_user_one_time_request(msg->request) && dev->vq_index != 0) {
         return 0;
     }
 
-    if (need_reply) {
+    if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
+        error_report("Failed to set msg fds.");
+        return -1;
+    }
+
+    ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
+    if (ret != size) {
+        error_report("Failed to write msg."
+                     " Wrote %d instead of %d.", ret, size);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
+                                   struct vhost_log *log)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    size_t fd_num = 0;
+    bool shmfd = virtio_has_feature(dev->protocol_features,
+                                    VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_LOG_BASE,
+        .flags = VHOST_USER_VERSION,
+        .payload.log.mmap_size = log->size * sizeof(*(log->log)),
+        .payload.log.mmap_offset = 0,
+        .size = sizeof(msg.payload.log),
+    };
+
+    if (shmfd && log->fd != -1) {
+        fds[fd_num++] = log->fd;
+    }
+
+    if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
+        return -1;
+    }
+
+    if (shmfd) {
+        msg.size = 0;
         if (vhost_user_read(dev, &msg) < 0) {
-            return 0;
-        }
-
-        if (msg_request != msg.request) {
-            error_report("Received unexpected msg type."
-                    " Expected %d received %d\n", msg_request, msg.request);
             return -1;
         }
 
-        switch (msg_request) {
-        case VHOST_USER_GET_FEATURES:
-            if (msg.size != sizeof(m.u64)) {
-                error_report("Received bad msg size.\n");
-                return -1;
-            }
-            *((__u64 *) arg) = msg.u64;
-            break;
-        case VHOST_USER_GET_VRING_BASE:
-            if (msg.size != sizeof(m.state)) {
-                error_report("Received bad msg size.\n");
-                return -1;
-            }
-            memcpy(arg, &msg.state, sizeof(struct vhost_vring_state));
-            break;
-        default:
-            error_report("Received unexpected msg type.\n");
+        if (msg.request != VHOST_USER_SET_LOG_BASE) {
+            error_report("Received unexpected msg type. "
+                         "Expected %d received %d",
+                         VHOST_USER_SET_LOG_BASE, msg.request);
             return -1;
-            break;
         }
     }
 
     return 0;
 }
 
+static int vhost_user_set_mem_table(struct vhost_dev *dev,
+                                    struct vhost_memory *mem)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    int i, fd;
+    size_t fd_num = 0;
+    bool reply_supported = virtio_has_feature(dev->protocol_features,
+                                              VHOST_USER_PROTOCOL_F_REPLY_ACK);
+
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_MEM_TABLE,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    if (reply_supported) {
+        msg.flags |= VHOST_USER_NEED_REPLY_MASK;
+    }
+
+    for (i = 0; i < dev->mem->nregions; ++i) {
+        struct vhost_memory_region *reg = dev->mem->regions + i;
+        ram_addr_t offset;
+        MemoryRegion *mr;
+
+        assert((uintptr_t)reg->userspace_addr == reg->userspace_addr);
+        mr = memory_region_from_host((void *)(uintptr_t)reg->userspace_addr,
+                                     &offset);
+        fd = memory_region_get_fd(mr);
+        if (fd > 0) {
+            msg.payload.memory.regions[fd_num].userspace_addr = reg->userspace_addr;
+            msg.payload.memory.regions[fd_num].memory_size  = reg->memory_size;
+            msg.payload.memory.regions[fd_num].guest_phys_addr = reg->guest_phys_addr;
+            msg.payload.memory.regions[fd_num].mmap_offset = offset;
+            assert(fd_num < VHOST_MEMORY_MAX_NREGIONS);
+            fds[fd_num++] = fd;
+        }
+    }
+
+    msg.payload.memory.nregions = fd_num;
+
+    if (!fd_num) {
+        error_report("Failed initializing vhost-user memory map, "
+                     "consider using -object memory-backend-file share=on");
+        return -1;
+    }
+
+    msg.size = sizeof(msg.payload.memory.nregions);
+    msg.size += sizeof(msg.payload.memory.padding);
+    msg.size += fd_num * sizeof(VhostUserMemoryRegion);
+
+    if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
+        return -1;
+    }
+
+    if (reply_supported) {
+        return process_message_reply(dev, msg.request);
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_vring_addr(struct vhost_dev *dev,
+                                     struct vhost_vring_addr *addr)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_VRING_ADDR,
+        .flags = VHOST_USER_VERSION,
+        .payload.addr = *addr,
+        .size = sizeof(msg.payload.addr),
+    };
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_vring_endian(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring)
+{
+    error_report("vhost-user trying to send unhandled ioctl");
+    return -1;
+}
+
+static int vhost_set_vring(struct vhost_dev *dev,
+                           unsigned long int request,
+                           struct vhost_vring_state *ring)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.state = *ring,
+        .size = sizeof(msg.payload.state),
+    };
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_vring_num(struct vhost_dev *dev,
+                                    struct vhost_vring_state *ring)
+{
+    return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
+}
+
+static int vhost_user_set_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
+}
+
+static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
+{
+    int i;
+
+    if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
+        return -1;
+    }
+
+    for (i = 0; i < dev->nvqs; ++i) {
+        struct vhost_vring_state state = {
+            .index = dev->vq_index + i,
+            .num   = enable,
+        };
+
+        vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
+    }
+
+    return 0;
+}
+
+static int vhost_user_get_vring_base(struct vhost_dev *dev,
+                                     struct vhost_vring_state *ring)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_GET_VRING_BASE,
+        .flags = VHOST_USER_VERSION,
+        .payload.state = *ring,
+        .size = sizeof(msg.payload.state),
+    };
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    if (vhost_user_read(dev, &msg) < 0) {
+        return -1;
+    }
+
+    if (msg.request != VHOST_USER_GET_VRING_BASE) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     VHOST_USER_GET_VRING_BASE, msg.request);
+        return -1;
+    }
+
+    if (msg.size != sizeof(msg.payload.state)) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
+
+    *ring = msg.payload.state;
+
+    return 0;
+}
+
+static int vhost_set_vring_file(struct vhost_dev *dev,
+                                VhostUserRequest request,
+                                struct vhost_vring_file *file)
+{
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    size_t fd_num = 0;
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
+        .size = sizeof(msg.payload.u64),
+    };
+
+    if (ioeventfd_enabled() && file->fd > 0) {
+        fds[fd_num++] = file->fd;
+    } else {
+        msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
+    }
+
+    if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_vring_kick(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
+}
+
+static int vhost_user_set_vring_call(struct vhost_dev *dev,
+                                     struct vhost_vring_file *file)
+{
+    return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
+}
+
+static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+        .payload.u64 = u64,
+        .size = sizeof(msg.payload.u64),
+    };
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_features(struct vhost_dev *dev,
+                                   uint64_t features)
+{
+    return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
+}
+
+static int vhost_user_set_protocol_features(struct vhost_dev *dev,
+                                            uint64_t features)
+{
+    return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
+}
+
+static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
+{
+    VhostUserMsg msg = {
+        .request = request,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
+        return 0;
+    }
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    if (vhost_user_read(dev, &msg) < 0) {
+        return -1;
+    }
+
+    if (msg.request != request) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     request, msg.request);
+        return -1;
+    }
+
+    if (msg.size != sizeof(msg.payload.u64)) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
+
+    *u64 = msg.payload.u64;
+
+    return 0;
+}
+
+static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
+{
+    return vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features);
+}
+
+static int vhost_user_set_owner(struct vhost_dev *dev)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_SET_OWNER,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+static int vhost_user_reset_device(struct vhost_dev *dev)
+{
+    VhostUserMsg msg = {
+        .request = VHOST_USER_RESET_OWNER,
+        .flags = VHOST_USER_VERSION,
+    };
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
 static int vhost_user_init(struct vhost_dev *dev, void *opaque)
 {
+    uint64_t features;
+    int err;
+
     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
 
     dev->opaque = opaque;
 
+    err = vhost_user_get_features(dev, &features);
+    if (err < 0) {
+        return err;
+    }
+
+    if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
+        dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
+
+        err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
+                                 &features);
+        if (err < 0) {
+            return err;
+        }
+
+        dev->protocol_features = features & VHOST_USER_PROTOCOL_FEATURE_MASK;
+        err = vhost_user_set_protocol_features(dev, dev->protocol_features);
+        if (err < 0) {
+            return err;
+        }
+
+        /* query the max queues we support if backend supports Multiple Queue */
+        if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
+            err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
+                                     &dev->max_queues);
+            if (err < 0) {
+                return err;
+            }
+        }
+    }
+
+    if (dev->migration_blocker == NULL &&
+        !virtio_has_feature(dev->protocol_features,
+                            VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
+        error_setg(&dev->migration_blocker,
+                   "Migration disabled: vhost-user backend lacks "
+                   "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
+    }
+
     return 0;
 }
 
@@ -343,9 +624,88 @@
     return 0;
 }
 
+static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
+{
+    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
+
+    return idx;
+}
+
+static int vhost_user_memslots_limit(struct vhost_dev *dev)
+{
+    return VHOST_MEMORY_MAX_NREGIONS;
+}
+
+static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
+{
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    return virtio_has_feature(dev->protocol_features,
+                              VHOST_USER_PROTOCOL_F_LOG_SHMFD);
+}
+
+static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
+{
+    VhostUserMsg msg = { 0 };
+
+    assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
+
+    /* If guest supports GUEST_ANNOUNCE do nothing */
+    if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
+        return 0;
+    }
+
+    /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
+    if (virtio_has_feature(dev->protocol_features,
+                           VHOST_USER_PROTOCOL_F_RARP)) {
+        msg.request = VHOST_USER_SEND_RARP;
+        msg.flags = VHOST_USER_VERSION;
+        memcpy((char *)&msg.payload.u64, mac_addr, 6);
+        msg.size = sizeof(msg.payload.u64);
+
+        return vhost_user_write(dev, &msg, NULL, 0);
+    }
+    return -1;
+}
+
+static bool vhost_user_can_merge(struct vhost_dev *dev,
+                                 uint64_t start1, uint64_t size1,
+                                 uint64_t start2, uint64_t size2)
+{
+    ram_addr_t offset;
+    int mfd, rfd;
+    MemoryRegion *mr;
+
+    mr = memory_region_from_host((void *)(uintptr_t)start1, &offset);
+    mfd = memory_region_get_fd(mr);
+
+    mr = memory_region_from_host((void *)(uintptr_t)start2, &offset);
+    rfd = memory_region_get_fd(mr);
+
+    return mfd == rfd;
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
-        .vhost_call = vhost_user_call,
         .vhost_backend_init = vhost_user_init,
-        .vhost_backend_cleanup = vhost_user_cleanup
-        };
+        .vhost_backend_cleanup = vhost_user_cleanup,
+        .vhost_backend_memslots_limit = vhost_user_memslots_limit,
+        .vhost_set_log_base = vhost_user_set_log_base,
+        .vhost_set_mem_table = vhost_user_set_mem_table,
+        .vhost_set_vring_addr = vhost_user_set_vring_addr,
+        .vhost_set_vring_endian = vhost_user_set_vring_endian,
+        .vhost_set_vring_num = vhost_user_set_vring_num,
+        .vhost_set_vring_base = vhost_user_set_vring_base,
+        .vhost_get_vring_base = vhost_user_get_vring_base,
+        .vhost_set_vring_kick = vhost_user_set_vring_kick,
+        .vhost_set_vring_call = vhost_user_set_vring_call,
+        .vhost_set_features = vhost_user_set_features,
+        .vhost_get_features = vhost_user_get_features,
+        .vhost_set_owner = vhost_user_set_owner,
+        .vhost_reset_device = vhost_user_reset_device,
+        .vhost_get_vq_index = vhost_user_get_vq_index,
+        .vhost_set_vring_enable = vhost_user_set_vring_enable,
+        .vhost_requires_shm_log = vhost_user_requires_shm_log,
+        .vhost_migration_done = vhost_user_migration_done,
+        .vhost_backend_can_merge = vhost_user_can_merge,
+};
diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index 698d188..0b7f8f9 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -13,24 +13,62 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "hw/virtio/vhost.h"
 #include "hw/hw.h"
 #include "qemu/atomic.h"
 #include "qemu/range.h"
+#include "qemu/error-report.h"
+#include "qemu/memfd.h"
 #include <linux/vhost.h>
 #include "exec/address-spaces.h"
 #include "hw/virtio/virtio-bus.h"
+#include "hw/virtio/virtio-access.h"
 #include "migration/migration.h"
 
+/* enabled until disconnected backend stabilizes */
+#define _VHOST_DEBUG 1
+
+#ifdef _VHOST_DEBUG
+#define VHOST_OPS_DEBUG(fmt, ...) \
+    do { error_report(fmt ": %s (%d)", ## __VA_ARGS__, \
+                      strerror(errno), errno); } while (0)
+#else
+#define VHOST_OPS_DEBUG(fmt, ...) \
+    do { } while (0)
+#endif
+
+static struct vhost_log *vhost_log;
+static struct vhost_log *vhost_log_shm;
+
+static unsigned int used_memslots;
+static QLIST_HEAD(, vhost_dev) vhost_devices =
+    QLIST_HEAD_INITIALIZER(vhost_devices);
+
+bool vhost_has_free_slot(void)
+{
+    unsigned int slots_limit = ~0U;
+    struct vhost_dev *hdev;
+
+    QLIST_FOREACH(hdev, &vhost_devices, entry) {
+        unsigned int r = hdev->vhost_ops->vhost_backend_memslots_limit(hdev);
+        slots_limit = MIN(slots_limit, r);
+    }
+    return slots_limit > used_memslots;
+}
+
 static void vhost_dev_sync_region(struct vhost_dev *dev,
                                   MemoryRegionSection *section,
                                   uint64_t mfirst, uint64_t mlast,
                                   uint64_t rfirst, uint64_t rlast)
 {
+    vhost_log_chunk_t *log = dev->log->log;
+
     uint64_t start = MAX(mfirst, rfirst);
     uint64_t end = MIN(mlast, rlast);
-    vhost_log_chunk_t *from = dev->log + start / VHOST_LOG_CHUNK;
-    vhost_log_chunk_t *to = dev->log + end / VHOST_LOG_CHUNK + 1;
+    vhost_log_chunk_t *from = log + start / VHOST_LOG_CHUNK;
+    vhost_log_chunk_t *to = log + end / VHOST_LOG_CHUNK + 1;
     uint64_t addr = (start / VHOST_LOG_CHUNK) * VHOST_LOG_CHUNK;
 
     if (end < start) {
@@ -235,6 +273,13 @@
             continue;
         }
 
+        if (dev->vhost_ops->vhost_backend_can_merge &&
+            !dev->vhost_ops->vhost_backend_can_merge(dev, uaddr, size,
+                                                     reg->userspace_addr,
+                                                     reg->memory_size)) {
+            continue;
+        }
+
         if (merged) {
             --to;
             assert(to >= 0);
@@ -281,22 +326,97 @@
     return log_size;
 }
 
-static inline void vhost_dev_log_resize(struct vhost_dev* dev, uint64_t size)
+static struct vhost_log *vhost_log_alloc(uint64_t size, bool share)
 {
-    vhost_log_chunk_t *log;
-    uint64_t log_base;
+    struct vhost_log *log;
+    uint64_t logsize = size * sizeof(*(log->log));
+    int fd = -1;
+
+    log = g_new0(struct vhost_log, 1);
+    if (share) {
+        log->log = qemu_memfd_alloc("vhost-log", logsize,
+                                    F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL,
+                                    &fd);
+        memset(log->log, 0, logsize);
+    } else {
+        log->log = g_malloc0(logsize);
+    }
+
+    log->size = size;
+    log->refcnt = 1;
+    log->fd = fd;
+
+    return log;
+}
+
+static struct vhost_log *vhost_log_get(uint64_t size, bool share)
+{
+    struct vhost_log *log = share ? vhost_log_shm : vhost_log;
+
+    if (!log || log->size != size) {
+        log = vhost_log_alloc(size, share);
+        if (share) {
+            vhost_log_shm = log;
+        } else {
+            vhost_log = log;
+        }
+    } else {
+        ++log->refcnt;
+    }
+
+    return log;
+}
+
+static void vhost_log_put(struct vhost_dev *dev, bool sync)
+{
+    struct vhost_log *log = dev->log;
+
+    if (!log) {
+        return;
+    }
+    dev->log = NULL;
+    dev->log_size = 0;
+
+    --log->refcnt;
+    if (log->refcnt == 0) {
+        /* Sync only the range covered by the old log */
+        if (dev->log_size && sync) {
+            vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
+        }
+
+        if (vhost_log == log) {
+            g_free(log->log);
+            vhost_log = NULL;
+        } else if (vhost_log_shm == log) {
+            qemu_memfd_free(log->log, log->size * sizeof(*(log->log)),
+                            log->fd);
+            vhost_log_shm = NULL;
+        }
+
+        g_free(log);
+    }
+}
+
+static bool vhost_dev_log_is_shared(struct vhost_dev *dev)
+{
+    return dev->vhost_ops->vhost_requires_shm_log &&
+           dev->vhost_ops->vhost_requires_shm_log(dev);
+}
+
+static inline void vhost_dev_log_resize(struct vhost_dev *dev, uint64_t size)
+{
+    struct vhost_log *log = vhost_log_get(size, vhost_dev_log_is_shared(dev));
+    uint64_t log_base = (uintptr_t)log->log;
     int r;
 
-    log = g_malloc0(size * sizeof *log);
-    log_base = (uint64_t)(unsigned long)log;
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_LOG_BASE, &log_base);
-    assert(r >= 0);
-    (void)r;
-    /* Sync only the range covered by the old log */
-    if (dev->log_size) {
-        vhost_log_sync_range(dev, 0, dev->log_size * VHOST_LOG_CHUNK - 1);
+    /* inform backend of log switching, this must be done before
+       releasing the current log, to ensure no logging is lost */
+    r = dev->vhost_ops->vhost_set_log_base(dev, log_base, log);
+    if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_log_base failed");
     }
-    g_free(dev->log);
+
+    vhost_log_put(dev, true);
     dev->log = log;
     dev->log_size = size;
 }
@@ -319,11 +439,11 @@
         l = vq->ring_size;
         p = cpu_physical_memory_map(vq->ring_phys, &l, 1);
         if (!p || l != vq->ring_size) {
-            fprintf(stderr, "Unable to map ring buffer for ring %d\n", i);
+            error_report("Unable to map ring buffer for ring %d", i);
             r = -ENOMEM;
         }
         if (p != vq->ring) {
-            fprintf(stderr, "Ring buffer relocated for ring %d\n", i);
+            error_report("Ring buffer relocated for ring %d", i);
             r = -EBUSY;
         }
         cpu_physical_memory_unmap(p, l, 0, 0);
@@ -378,7 +498,8 @@
                                          memory_listener);
     hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = int128_get64(section->size);
-    bool log_dirty = memory_region_is_logging(section->mr);
+    bool log_dirty =
+        memory_region_get_dirty_log_mask(section->mr) & ~(1 << DIRTY_MEMORY_MIGRATION);
     int s = offsetof(struct vhost_memory, regions) +
         (dev->mem->nregions + 1) * sizeof dev->mem->regions[0];
     void *ram;
@@ -416,6 +537,7 @@
     dev->mem_changed_start_addr = MIN(dev->mem_changed_start_addr, start_addr);
     dev->mem_changed_end_addr = MAX(dev->mem_changed_end_addr, start_addr + size - 1);
     dev->memory_changed = true;
+    used_memslots = dev->mem->nregions;
 }
 
 static bool vhost_section(MemoryRegionSection *section)
@@ -460,9 +582,10 @@
     }
 
     if (!dev->log_enabled) {
-        r = dev->vhost_ops->vhost_call(dev, VHOST_SET_MEM_TABLE, dev->mem);
-        assert(r >= 0);
-        (void)r;
+        r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
+        if (r < 0) {
+            VHOST_OPS_DEBUG("vhost_set_mem_table failed");
+        }
         dev->memory_changed = false;
         return;
     }
@@ -474,9 +597,10 @@
     if (dev->log_size < log_size) {
         vhost_dev_log_resize(dev, log_size + VHOST_LOG_BUFFER);
     }
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_MEM_TABLE, dev->mem);
-    assert(r >= 0);
-    (void)r;
+    r = dev->vhost_ops->vhost_set_mem_table(dev, dev->mem);
+    if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_mem_table failed");
+    }
     /* To log less, can only decrease log size after table update. */
     if (dev->log_size > log_size + VHOST_LOG_BUFFER) {
         vhost_dev_log_resize(dev, log_size);
@@ -543,8 +667,9 @@
         .log_guest_addr = vq->used_phys,
         .flags = enable_log ? (1 << VHOST_VRING_F_LOG) : 0,
     };
-    int r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_ADDR, &addr);
+    int r = dev->vhost_ops->vhost_set_vring_addr(dev, &addr);
     if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_vring_addr failed");
         return -errno;
     }
     return 0;
@@ -555,21 +680,25 @@
     uint64_t features = dev->acked_features;
     int r;
     if (enable_log) {
-        features |= 0x1 << VHOST_F_LOG_ALL;
+        features |= 0x1ULL << VHOST_F_LOG_ALL;
     }
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_FEATURES, &features);
+    r = dev->vhost_ops->vhost_set_features(dev, features);
+    if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_features failed");
+    }
     return r < 0 ? -errno : 0;
 }
 
 static int vhost_dev_set_log(struct vhost_dev *dev, bool enable_log)
 {
-    int r, t, i;
+    int r, i, idx;
     r = vhost_dev_set_features(dev, enable_log);
     if (r < 0) {
         goto err_features;
     }
     for (i = 0; i < dev->nvqs; ++i) {
-        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
+        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+        r = vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
                                      enable_log);
         if (r < 0) {
             goto err_vq;
@@ -578,14 +707,11 @@
     return 0;
 err_vq:
     for (; i >= 0; --i) {
-        t = vhost_virtqueue_set_addr(dev, dev->vqs + i, i,
-                                     dev->log_enabled);
-        assert(t >= 0);
-        (void)t;
+        idx = dev->vhost_ops->vhost_get_vq_index(dev, dev->vq_index + i);
+        vhost_virtqueue_set_addr(dev, dev->vqs + i, idx,
+                                 dev->log_enabled);
     }
-    t = vhost_dev_set_features(dev, dev->log_enabled);
-    assert(t >= 0);
-    (void)t;
+    vhost_dev_set_features(dev, dev->log_enabled);
 err_features:
     return r;
 }
@@ -607,9 +733,7 @@
         if (r < 0) {
             return r;
         }
-        g_free(dev->log);
-        dev->log = NULL;
-        dev->log_size = 0;
+        vhost_log_put(dev, false);
     } else {
         vhost_dev_log_resize(dev, vhost_get_log_size(dev));
         r = vhost_dev_set_log(dev, true);
@@ -642,17 +766,58 @@
 }
 
 static void vhost_log_start(MemoryListener *listener,
-                            MemoryRegionSection *section)
+                            MemoryRegionSection *section,
+                            int old, int new)
 {
     /* FIXME: implement */
 }
 
 static void vhost_log_stop(MemoryListener *listener,
-                           MemoryRegionSection *section)
+                           MemoryRegionSection *section,
+                           int old, int new)
 {
     /* FIXME: implement */
 }
 
+/* The vhost driver natively knows how to handle the vrings of non
+ * cross-endian legacy devices and modern devices. Only legacy devices
+ * exposed to a bi-endian guest may require the vhost driver to use a
+ * specific endianness.
+ */
+static inline bool vhost_needs_vring_endian(VirtIODevice *vdev)
+{
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return false;
+    }
+#ifdef HOST_WORDS_BIGENDIAN
+    return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_LITTLE;
+#else
+    return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
+#endif
+}
+
+static int vhost_virtqueue_set_vring_endian_legacy(struct vhost_dev *dev,
+                                                   bool is_big_endian,
+                                                   int vhost_vq_index)
+{
+    struct vhost_vring_state s = {
+        .index = vhost_vq_index,
+        .num = is_big_endian
+    };
+
+    if (!dev->vhost_ops->vhost_set_vring_endian(dev, &s)) {
+        return 0;
+    }
+
+    VHOST_OPS_DEBUG("vhost_set_vring_endian failed");
+    if (errno == ENOTTY) {
+        error_report("vhost does not support cross-endian");
+        return -ENOSYS;
+    }
+
+    return -errno;
+}
+
 static int vhost_virtqueue_start(struct vhost_dev *dev,
                                 struct VirtIODevice *vdev,
                                 struct vhost_virtqueue *vq,
@@ -660,7 +825,7 @@
 {
     hwaddr s, l, a;
     int r;
-    int vhost_vq_index = idx - dev->vq_index;
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
     struct vhost_vring_file file = {
         .index = vhost_vq_index
     };
@@ -669,20 +834,30 @@
     };
     struct VirtQueue *vvq = virtio_get_queue(vdev, idx);
 
-    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
 
     vq->num = state.num = virtio_queue_get_num(vdev, idx);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_NUM, &state);
+    r = dev->vhost_ops->vhost_set_vring_num(dev, &state);
     if (r) {
+        VHOST_OPS_DEBUG("vhost_set_vring_num failed");
         return -errno;
     }
 
     state.num = virtio_queue_get_last_avail_idx(vdev, idx);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_BASE, &state);
+    r = dev->vhost_ops->vhost_set_vring_base(dev, &state);
     if (r) {
+        VHOST_OPS_DEBUG("vhost_set_vring_base failed");
         return -errno;
     }
 
+    if (vhost_needs_vring_endian(vdev)) {
+        r = vhost_virtqueue_set_vring_endian_legacy(dev,
+                                                    virtio_is_big_endian(vdev),
+                                                    vhost_vq_index);
+        if (r) {
+            return -errno;
+        }
+    }
+
     s = l = virtio_queue_get_desc_size(vdev, idx);
     a = virtio_queue_get_desc_addr(vdev, idx);
     vq->desc = cpu_physical_memory_map(a, &l, 0);
@@ -720,8 +895,9 @@
     }
 
     file.fd = event_notifier_get_fd(virtio_queue_get_host_notifier(vvq));
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_KICK, &file);
+    r = dev->vhost_ops->vhost_set_vring_kick(dev, &file);
     if (r) {
+        VHOST_OPS_DEBUG("vhost_set_vring_kick failed");
         r = -errno;
         goto fail_kick;
     }
@@ -729,6 +905,14 @@
     /* Clear and discard previous events if any. */
     event_notifier_test_and_clear(&vq->masked_notifier);
 
+    /* Init vring in unmasked state, unless guest_notifier_mask
+     * will do it later.
+     */
+    if (!vdev->use_guest_notifier_mask) {
+        /* TODO: check and handle errors. */
+        vhost_virtqueue_mask(dev, vdev, idx, false);
+    }
+
     return 0;
 
 fail_kick:
@@ -753,19 +937,29 @@
                                     struct vhost_virtqueue *vq,
                                     unsigned idx)
 {
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, idx);
     struct vhost_vring_state state = {
-        .index = idx - dev->vq_index
+        .index = vhost_vq_index,
     };
     int r;
-    assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_GET_VRING_BASE, &state);
+
+    r = dev->vhost_ops->vhost_get_vring_base(dev, &state);
     if (r < 0) {
-        fprintf(stderr, "vhost VQ %d ring restore failed: %d\n", idx, r);
-        fflush(stderr);
+        VHOST_OPS_DEBUG("vhost VQ %d ring restore failed: %d", idx, r);
+    } else {
+        virtio_queue_set_last_avail_idx(vdev, idx, state.num);
     }
-    virtio_queue_set_last_avail_idx(vdev, idx, state.num);
     virtio_queue_invalidate_signalled_used(vdev, idx);
-    assert (r >= 0);
+
+    /* In the cross-endian case, we need to reset the vring endianness to
+     * native as legacy devices expect so by default.
+     */
+    if (vhost_needs_vring_endian(vdev)) {
+        vhost_virtqueue_set_vring_endian_legacy(dev,
+                                                !virtio_is_big_endian(vdev),
+                                                vhost_vq_index);
+    }
+
     cpu_physical_memory_unmap(vq->ring, virtio_queue_get_ring_size(vdev, idx),
                               0, virtio_queue_get_ring_size(vdev, idx));
     cpu_physical_memory_unmap(vq->used, virtio_queue_get_used_size(vdev, idx),
@@ -788,11 +982,35 @@
 {
 }
 
+static int vhost_virtqueue_set_busyloop_timeout(struct vhost_dev *dev,
+                                                int n, uint32_t timeout)
+{
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
+    struct vhost_vring_state state = {
+        .index = vhost_vq_index,
+        .num = timeout,
+    };
+    int r;
+
+    if (!dev->vhost_ops->vhost_set_vring_busyloop_timeout) {
+        return -EINVAL;
+    }
+
+    r = dev->vhost_ops->vhost_set_vring_busyloop_timeout(dev, &state);
+    if (r) {
+        VHOST_OPS_DEBUG("vhost_set_vring_busyloop_timeout failed");
+        return r;
+    }
+
+    return 0;
+}
+
 static int vhost_virtqueue_init(struct vhost_dev *dev,
                                 struct vhost_virtqueue *vq, int n)
 {
+    int vhost_vq_index = dev->vhost_ops->vhost_get_vq_index(dev, n);
     struct vhost_vring_file file = {
-        .index = n,
+        .index = vhost_vq_index,
     };
     int r = event_notifier_init(&vq->masked_notifier, 0);
     if (r < 0) {
@@ -800,8 +1018,9 @@
     }
 
     file.fd = event_notifier_get_fd(&vq->masked_notifier);
-    r = dev->vhost_ops->vhost_call(dev, VHOST_SET_VRING_CALL, &file);
+    r = dev->vhost_ops->vhost_set_vring_call(dev, &file);
     if (r) {
+        VHOST_OPS_DEBUG("vhost_set_vring_call failed");
         r = -errno;
         goto fail_call;
     }
@@ -817,37 +1036,57 @@
 }
 
 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
-                   VhostBackendType backend_type, bool force)
+                   VhostBackendType backend_type, uint32_t busyloop_timeout)
 {
     uint64_t features;
-    int i, r;
+    int i, r, n_initialized_vqs = 0;
 
-    if (vhost_set_backend_type(hdev, backend_type) < 0) {
-        close((uintptr_t)opaque);
-        return -1;
-    }
+    hdev->migration_blocker = NULL;
 
-    if (hdev->vhost_ops->vhost_backend_init(hdev, opaque) < 0) {
-        close((uintptr_t)opaque);
-        return -errno;
-    }
+    r = vhost_set_backend_type(hdev, backend_type);
+    assert(r >= 0);
 
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_OWNER, NULL);
+    r = hdev->vhost_ops->vhost_backend_init(hdev, opaque);
     if (r < 0) {
         goto fail;
     }
 
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_GET_FEATURES, &features);
-    if (r < 0) {
+    if (used_memslots > hdev->vhost_ops->vhost_backend_memslots_limit(hdev)) {
+        error_report("vhost backend memory slots limit is less"
+                " than current number of present memory slots");
+        r = -1;
         goto fail;
     }
 
-    for (i = 0; i < hdev->nvqs; ++i) {
-        r = vhost_virtqueue_init(hdev, hdev->vqs + i, i);
+    r = hdev->vhost_ops->vhost_set_owner(hdev);
+    if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_owner failed");
+        goto fail;
+    }
+
+    r = hdev->vhost_ops->vhost_get_features(hdev, &features);
+    if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_get_features failed");
+        goto fail;
+    }
+
+    for (i = 0; i < hdev->nvqs; ++i, ++n_initialized_vqs) {
+        r = vhost_virtqueue_init(hdev, hdev->vqs + i, hdev->vq_index + i);
         if (r < 0) {
-            goto fail_vq;
+            goto fail;
         }
     }
+
+    if (busyloop_timeout) {
+        for (i = 0; i < hdev->nvqs; ++i) {
+            r = vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i,
+                                                     busyloop_timeout);
+            if (r < 0) {
+                goto fail_busyloop;
+            }
+        }
+    }
+
     hdev->features = features;
 
     hdev->memory_listener = (MemoryListener) {
@@ -865,12 +1104,21 @@
         .eventfd_del = vhost_eventfd_del,
         .priority = 10
     };
-    hdev->migration_blocker = NULL;
-    if (!(hdev->features & (0x1 << VHOST_F_LOG_ALL))) {
-        error_setg(&hdev->migration_blocker,
-                   "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
+
+    if (hdev->migration_blocker == NULL) {
+        if (!(hdev->features & (0x1ULL << VHOST_F_LOG_ALL))) {
+            error_setg(&hdev->migration_blocker,
+                       "Migration disabled: vhost lacks VHOST_F_LOG_ALL feature.");
+        } else if (!qemu_memfd_check()) {
+            error_setg(&hdev->migration_blocker,
+                       "Migration disabled: failed to allocate shared memory");
+        }
+    }
+
+    if (hdev->migration_blocker != NULL) {
         migrate_add_blocker(hdev->migration_blocker);
     }
+
     hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
     hdev->n_mem_sections = 0;
     hdev->mem_sections = NULL;
@@ -880,43 +1128,43 @@
     hdev->started = false;
     hdev->memory_changed = false;
     memory_listener_register(&hdev->memory_listener, &address_space_memory);
-    hdev->force = force;
+    QLIST_INSERT_HEAD(&vhost_devices, hdev, entry);
     return 0;
-fail_vq:
+
+fail_busyloop:
     while (--i >= 0) {
-        vhost_virtqueue_cleanup(hdev->vqs + i);
+        vhost_virtqueue_set_busyloop_timeout(hdev, hdev->vq_index + i, 0);
     }
 fail:
-    r = -errno;
-    hdev->vhost_ops->vhost_backend_cleanup(hdev);
+    hdev->nvqs = n_initialized_vqs;
+    vhost_dev_cleanup(hdev);
     return r;
 }
 
 void vhost_dev_cleanup(struct vhost_dev *hdev)
 {
     int i;
+
     for (i = 0; i < hdev->nvqs; ++i) {
         vhost_virtqueue_cleanup(hdev->vqs + i);
     }
-    memory_listener_unregister(&hdev->memory_listener);
+    if (hdev->mem) {
+        /* those are only safe after successful init */
+        memory_listener_unregister(&hdev->memory_listener);
+        QLIST_REMOVE(hdev, entry);
+    }
     if (hdev->migration_blocker) {
         migrate_del_blocker(hdev->migration_blocker);
         error_free(hdev->migration_blocker);
     }
     g_free(hdev->mem);
     g_free(hdev->mem_sections);
-    hdev->vhost_ops->vhost_backend_cleanup(hdev);
-}
+    if (hdev->vhost_ops) {
+        hdev->vhost_ops->vhost_backend_cleanup(hdev);
+    }
+    assert(!hdev->log);
 
-bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev)
-{
-    BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
-    VirtioBusState *vbus = VIRTIO_BUS(qbus);
-    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
-
-    return !k->query_guest_notifiers ||
-           k->query_guest_notifiers(qbus->parent) ||
-           hdev->force;
+    memset(hdev, 0, sizeof(struct vhost_dev));
 }
 
 /* Stop processing guest IO notifications in qemu.
@@ -927,17 +1175,19 @@
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
     VirtioBusState *vbus = VIRTIO_BUS(qbus);
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
-    int i, r;
-    if (!k->set_host_notifier) {
-        fprintf(stderr, "binding does not support host notifiers\n");
+    int i, r, e;
+
+    if (!k->ioeventfd_started) {
+        error_report("binding does not support host notifiers");
         r = -ENOSYS;
         goto fail;
     }
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = k->set_host_notifier(qbus->parent, hdev->vq_index + i, true);
+        r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
+                                         true);
         if (r < 0) {
-            fprintf(stderr, "vhost VQ %d notifier binding failed: %d\n", i, -r);
+            error_report("vhost VQ %d notifier binding failed: %d", i, -r);
             goto fail_vq;
         }
     }
@@ -945,12 +1195,12 @@
     return 0;
 fail_vq:
     while (--i >= 0) {
-        r = k->set_host_notifier(qbus->parent, hdev->vq_index + i, false);
-        if (r < 0) {
-            fprintf(stderr, "vhost VQ %d notifier cleanup error: %d\n", i, -r);
-            fflush(stderr);
+        e = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
+                                         false);
+        if (e < 0) {
+            error_report("vhost VQ %d notifier cleanup error: %d", i, -r);
         }
-        assert (r >= 0);
+        assert (e >= 0);
     }
 fail:
     return r;
@@ -964,15 +1214,13 @@
 void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
 {
     BusState *qbus = BUS(qdev_get_parent_bus(DEVICE(vdev)));
-    VirtioBusState *vbus = VIRTIO_BUS(qbus);
-    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(vbus);
     int i, r;
 
     for (i = 0; i < hdev->nvqs; ++i) {
-        r = k->set_host_notifier(qbus->parent, hdev->vq_index + i, false);
+        r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
+                                         false);
         if (r < 0) {
-            fprintf(stderr, "vhost VQ %d notifier cleanup failed: %d\n", i, -r);
-            fflush(stderr);
+            error_report("vhost VQ %d notifier cleanup failed: %d", i, -r);
         }
         assert (r >= 0);
     }
@@ -994,28 +1242,31 @@
 {
     struct VirtQueue *vvq = virtio_get_queue(vdev, n);
     int r, index = n - hdev->vq_index;
+    struct vhost_vring_file file;
 
-    assert(n >= hdev->vq_index && n < hdev->vq_index + hdev->nvqs);
+    /* should only be called after backend is connected */
+    assert(hdev->vhost_ops);
 
-    struct vhost_vring_file file = {
-        .index = index
-    };
     if (mask) {
+        assert(vdev->use_guest_notifier_mask);
         file.fd = event_notifier_get_fd(&hdev->vqs[index].masked_notifier);
     } else {
         file.fd = event_notifier_get_fd(virtio_queue_get_guest_notifier(vvq));
     }
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_VRING_CALL, &file);
-    assert(r >= 0);
-    (void)r;
+
+    file.index = hdev->vhost_ops->vhost_get_vq_index(hdev, n);
+    r = hdev->vhost_ops->vhost_set_vring_call(hdev, &file);
+    if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_vring_call failed");
+    }
 }
 
-unsigned vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
-        unsigned features)
+uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
+                            uint64_t features)
 {
     const int *bit = feature_bits;
     while (*bit != VHOST_INVALID_FEATURE_BIT) {
-        unsigned bit_mask = (1 << *bit);
+        uint64_t bit_mask = (1ULL << *bit);
         if (!(hdev->features & bit_mask)) {
             features &= ~bit_mask;
         }
@@ -1025,11 +1276,11 @@
 }
 
 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
-        unsigned features)
+                        uint64_t features)
 {
     const int *bit = feature_bits;
     while (*bit != VHOST_INVALID_FEATURE_BIT) {
-        unsigned bit_mask = (1 << *bit);
+        uint64_t bit_mask = (1ULL << *bit);
         if (features & bit_mask) {
             hdev->acked_features |= bit_mask;
         }
@@ -1042,14 +1293,18 @@
 {
     int i, r;
 
+    /* should only be called after backend is connected */
+    assert(hdev->vhost_ops);
+
     hdev->started = true;
 
     r = vhost_dev_set_features(hdev, hdev->log_enabled);
     if (r < 0) {
         goto fail_features;
     }
-    r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_MEM_TABLE, hdev->mem);
+    r = hdev->vhost_ops->vhost_set_mem_table(hdev, hdev->mem);
     if (r < 0) {
+        VHOST_OPS_DEBUG("vhost_set_mem_table failed");
         r = -errno;
         goto fail_mem;
     }
@@ -1064,11 +1319,17 @@
     }
 
     if (hdev->log_enabled) {
+        uint64_t log_base;
+
         hdev->log_size = vhost_get_log_size(hdev);
-        hdev->log = hdev->log_size ?
-            g_malloc0(hdev->log_size * sizeof *hdev->log) : NULL;
-        r = hdev->vhost_ops->vhost_call(hdev, VHOST_SET_LOG_BASE, hdev->log);
+        hdev->log = vhost_log_get(hdev->log_size,
+                                  vhost_dev_log_is_shared(hdev));
+        log_base = (uintptr_t)hdev->log->log;
+        r = hdev->vhost_ops->vhost_set_log_base(hdev,
+                                                hdev->log_size ? log_base : 0,
+                                                hdev->log);
         if (r < 0) {
+            VHOST_OPS_DEBUG("vhost_set_log_base failed");
             r = -errno;
             goto fail_log;
         }
@@ -1076,6 +1337,7 @@
 
     return 0;
 fail_log:
+    vhost_log_put(hdev, false);
 fail_vq:
     while (--i >= 0) {
         vhost_virtqueue_stop(hdev,
@@ -1096,17 +1358,26 @@
 {
     int i;
 
+    /* should only be called after backend is connected */
+    assert(hdev->vhost_ops);
+
     for (i = 0; i < hdev->nvqs; ++i) {
         vhost_virtqueue_stop(hdev,
                              vdev,
                              hdev->vqs + i,
                              hdev->vq_index + i);
     }
-    vhost_log_sync_range(hdev, 0, ~0x0ull);
 
+    vhost_log_put(hdev, true);
     hdev->started = false;
-    g_free(hdev->log);
-    hdev->log = NULL;
-    hdev->log_size = 0;
 }
 
+int vhost_net_set_backend(struct vhost_dev *hdev,
+                          struct vhost_vring_file *file)
+{
+    if (hdev->vhost_ops->vhost_net_set_backend) {
+        return hdev->vhost_ops->vhost_net_set_backend(hdev, file);
+    }
+
+    return -1;
+}
diff --git a/hw/virtio/virtio-balloon.c b/hw/virtio/virtio-balloon.c
index 7bfbb75..5af429a 100644
--- a/hw/virtio/virtio-balloon.c
+++ b/hw/virtio/virtio-balloon.c
@@ -13,32 +13,33 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu/iov.h"
 #include "qemu/timer.h"
 #include "qemu-common.h"
 #include "hw/virtio/virtio.h"
 #include "hw/i386/pc.h"
-#include "cpu.h"
 #include "sysemu/balloon.h"
 #include "hw/virtio/virtio-balloon.h"
 #include "sysemu/kvm.h"
 #include "exec/address-spaces.h"
 #include "qapi/visitor.h"
 #include "qapi-event.h"
-
-#if defined(__linux__)
-#include <sys/mman.h>
-#endif
+#include "trace.h"
 
 #include "hw/virtio/virtio-bus.h"
 #include "hw/virtio/virtio-access.h"
 
+#define BALLOON_PAGE_SIZE  (1 << VIRTIO_BALLOON_PFN_SHIFT)
+
 static void balloon_page(void *addr, int deflate)
 {
 #if defined(__linux__)
-    if (!kvm_enabled() || kvm_has_sync_mmu())
-        qemu_madvise(addr, TARGET_PAGE_SIZE,
+    if (!qemu_balloon_is_inhibited() && (!kvm_enabled() ||
+                                         kvm_has_sync_mmu())) {
+        qemu_madvise(addr, BALLOON_PAGE_SIZE,
                 deflate ? QEMU_MADV_WILLNEED : QEMU_MADV_DONTNEED);
+    }
 #endif
 }
 
@@ -49,6 +50,7 @@
    [VIRTIO_BALLOON_S_MINFLT] = "stat-minor-faults",
    [VIRTIO_BALLOON_S_MEMFREE] = "stat-free-memory",
    [VIRTIO_BALLOON_S_MEMTOT] = "stat-total-memory",
+   [VIRTIO_BALLOON_S_AVAIL] = "stat-available-memory",
    [VIRTIO_BALLOON_S_NR] = NULL
 };
 
@@ -69,7 +71,7 @@
 static bool balloon_stats_supported(const VirtIOBalloon *s)
 {
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
-    return vdev->guest_features & (1 << VIRTIO_BALLOON_F_STATS_VQ);
+    return virtio_vdev_has_feature(vdev, VIRTIO_BALLOON_F_STATS_VQ);
 }
 
 static bool balloon_stats_enabled(const VirtIOBalloon *s)
@@ -97,69 +99,74 @@
     VirtIOBalloon *s = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
 
-    if (!balloon_stats_supported(s)) {
+    if (s->stats_vq_elem == NULL || !balloon_stats_supported(s)) {
         /* re-schedule */
         balloon_stats_change_timer(s, s->stats_poll_interval);
         return;
     }
 
-    virtqueue_push(s->svq, &s->stats_vq_elem, s->stats_vq_offset);
+    virtqueue_push(s->svq, s->stats_vq_elem, s->stats_vq_offset);
     virtio_notify(vdev, s->svq);
+    g_free(s->stats_vq_elem);
+    s->stats_vq_elem = NULL;
 }
 
-static void balloon_stats_get_all(Object *obj, struct Visitor *v,
-                                  void *opaque, const char *name, Error **errp)
+static void balloon_stats_get_all(Object *obj, Visitor *v, const char *name,
+                                  void *opaque, Error **errp)
 {
     Error *err = NULL;
     VirtIOBalloon *s = opaque;
     int i;
 
-    visit_start_struct(v, NULL, "guest-stats", name, 0, &err);
+    visit_start_struct(v, name, NULL, 0, &err);
     if (err) {
         goto out;
     }
-    visit_type_int(v, &s->stats_last_update, "last-update", &err);
+    visit_type_int(v, "last-update", &s->stats_last_update, &err);
     if (err) {
         goto out_end;
     }
 
-    visit_start_struct(v, NULL, NULL, "stats", 0, &err);
+    visit_start_struct(v, "stats", NULL, 0, &err);
     if (err) {
         goto out_end;
     }
-    for (i = 0; !err && i < VIRTIO_BALLOON_S_NR; i++) {
-        visit_type_int64(v, (int64_t *) &s->stats[i], balloon_stat_names[i],
-                         &err);
+    for (i = 0; i < VIRTIO_BALLOON_S_NR; i++) {
+        visit_type_uint64(v, balloon_stat_names[i], &s->stats[i], &err);
+        if (err) {
+            goto out_nested;
+        }
     }
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
+    visit_check_struct(v, &err);
+out_nested:
+    visit_end_struct(v, NULL);
 
+    if (!err) {
+        visit_check_struct(v, &err);
+    }
 out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
+    visit_end_struct(v, NULL);
 out:
     error_propagate(errp, err);
 }
 
-static void balloon_stats_get_poll_interval(Object *obj, struct Visitor *v,
-                                            void *opaque, const char *name,
+static void balloon_stats_get_poll_interval(Object *obj, Visitor *v,
+                                            const char *name, void *opaque,
                                             Error **errp)
 {
     VirtIOBalloon *s = opaque;
-    visit_type_int(v, &s->stats_poll_interval, name, errp);
+    visit_type_int(v, name, &s->stats_poll_interval, errp);
 }
 
-static void balloon_stats_set_poll_interval(Object *obj, struct Visitor *v,
-                                            void *opaque, const char *name,
+static void balloon_stats_set_poll_interval(Object *obj, Visitor *v,
+                                            const char *name, void *opaque,
                                             Error **errp)
 {
     VirtIOBalloon *s = opaque;
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -202,14 +209,18 @@
 static void virtio_balloon_handle_output(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBalloon *s = VIRTIO_BALLOON(vdev);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     MemoryRegionSection section;
 
-    while (virtqueue_pop(vq, &elem)) {
+    for (;;) {
         size_t offset = 0;
         uint32_t pfn;
+        elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+        if (!elem) {
+            return;
+        }
 
-        while (iov_to_buf(elem.out_sg, elem.out_num, offset, &pfn, 4) == 4) {
+        while (iov_to_buf(elem->out_sg, elem->out_num, offset, &pfn, 4) == 4) {
             ram_addr_t pa;
             ram_addr_t addr;
             int p = virtio_ldl_p(vdev, &pfn);
@@ -222,6 +233,8 @@
             if (!int128_nz(section.size) || !memory_region_is_ram(section.mr))
                 continue;
 
+            trace_virtio_balloon_handle_output(memory_region_name(section.mr),
+                                               pa);
             /* Using memory_region_get_ram_ptr is bending the rules a bit, but
                should be OK because we only want a single page.  */
             addr = section.offset_within_region;
@@ -230,23 +243,34 @@
             memory_region_unref(section.mr);
         }
 
-        virtqueue_push(vq, &elem, offset);
+        virtqueue_push(vq, elem, offset);
         virtio_notify(vdev, vq);
+        g_free(elem);
     }
 }
 
 static void virtio_balloon_receive_stats(VirtIODevice *vdev, VirtQueue *vq)
 {
     VirtIOBalloon *s = VIRTIO_BALLOON(vdev);
-    VirtQueueElement *elem = &s->stats_vq_elem;
+    VirtQueueElement *elem;
     VirtIOBalloonStat stat;
     size_t offset = 0;
     qemu_timeval tv;
 
-    if (!virtqueue_pop(vq, elem)) {
+    elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
+    if (!elem) {
         goto out;
     }
 
+    if (s->stats_vq_elem != NULL) {
+        /* This should never happen if the driver follows the spec. */
+        virtqueue_push(vq, s->stats_vq_elem, 0);
+        virtio_notify(vdev, vq);
+        g_free(s->stats_vq_elem);
+    }
+
+    s->stats_vq_elem = elem;
+
     /* Initialize the stats to get rid of any stale values.  This is only
      * needed to handle the case where a guest supports fewer stats than it
      * used to (ie. it has booted into an old kernel).
@@ -285,54 +309,91 @@
     config.num_pages = cpu_to_le32(dev->num_pages);
     config.actual = cpu_to_le32(dev->actual);
 
+    trace_virtio_balloon_get_config(config.num_pages, config.actual);
     memcpy(config_data, &config, sizeof(struct virtio_balloon_config));
 }
 
+static int build_dimm_list(Object *obj, void *opaque)
+{
+    GSList **list = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_PC_DIMM)) {
+        DeviceState *dev = DEVICE(obj);
+        if (dev->realized) { /* only realized DIMMs matter */
+            *list = g_slist_prepend(*list, dev);
+        }
+    }
+
+    object_child_foreach(obj, build_dimm_list, opaque);
+    return 0;
+}
+
+static ram_addr_t get_current_ram_size(void)
+{
+    GSList *list = NULL, *item;
+    ram_addr_t size = ram_size;
+
+    build_dimm_list(qdev_get_machine(), &list);
+    for (item = list; item; item = g_slist_next(item)) {
+        Object *obj = OBJECT(item->data);
+        if (!strcmp(object_get_typename(obj), TYPE_PC_DIMM)) {
+            size += object_property_get_int(obj, PC_DIMM_SIZE_PROP,
+                                            &error_abort);
+        }
+    }
+    g_slist_free(list);
+
+    return size;
+}
+
 static void virtio_balloon_set_config(VirtIODevice *vdev,
                                       const uint8_t *config_data)
 {
     VirtIOBalloon *dev = VIRTIO_BALLOON(vdev);
     struct virtio_balloon_config config;
     uint32_t oldactual = dev->actual;
+    ram_addr_t vm_ram_size = get_current_ram_size();
+
     memcpy(&config, config_data, sizeof(struct virtio_balloon_config));
     dev->actual = le32_to_cpu(config.actual);
     if (dev->actual != oldactual) {
-        qapi_event_send_balloon_change(ram_size -
+        qapi_event_send_balloon_change(vm_ram_size -
                         ((ram_addr_t) dev->actual << VIRTIO_BALLOON_PFN_SHIFT),
                         &error_abort);
     }
+    trace_virtio_balloon_set_config(dev->actual, oldactual);
 }
 
-static uint32_t virtio_balloon_get_features(VirtIODevice *vdev, uint32_t f)
+static uint64_t virtio_balloon_get_features(VirtIODevice *vdev, uint64_t f,
+                                            Error **errp)
 {
-    f |= (1 << VIRTIO_BALLOON_F_STATS_VQ);
+    VirtIOBalloon *dev = VIRTIO_BALLOON(vdev);
+    f |= dev->host_features;
+    virtio_add_feature(&f, VIRTIO_BALLOON_F_STATS_VQ);
     return f;
 }
 
 static void virtio_balloon_stat(void *opaque, BalloonInfo *info)
 {
     VirtIOBalloon *dev = opaque;
-    info->actual = ram_size - ((uint64_t) dev->actual <<
-                               VIRTIO_BALLOON_PFN_SHIFT);
+    info->actual = get_current_ram_size() - ((uint64_t) dev->actual <<
+                                             VIRTIO_BALLOON_PFN_SHIFT);
 }
 
 static void virtio_balloon_to_target(void *opaque, ram_addr_t target)
 {
     VirtIOBalloon *dev = VIRTIO_BALLOON(opaque);
     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
+    ram_addr_t vm_ram_size = get_current_ram_size();
 
-    if (target > ram_size) {
-        target = ram_size;
+    if (target > vm_ram_size) {
+        target = vm_ram_size;
     }
     if (target) {
-        dev->num_pages = (ram_size - target) >> VIRTIO_BALLOON_PFN_SHIFT;
+        dev->num_pages = (vm_ram_size - target) >> VIRTIO_BALLOON_PFN_SHIFT;
         virtio_notify_config(vdev);
     }
-}
-
-static void virtio_balloon_save(QEMUFile *f, void *opaque)
-{
-    virtio_save(VIRTIO_DEVICE(opaque), f);
+    trace_virtio_balloon_to_target(target, dev->num_pages);
 }
 
 static void virtio_balloon_save_device(VirtIODevice *vdev, QEMUFile *f)
@@ -343,12 +404,9 @@
     qemu_put_be32(f, s->actual);
 }
 
-static int virtio_balloon_load(QEMUFile *f, void *opaque, int version_id)
+static int virtio_balloon_load(QEMUFile *f, void *opaque, size_t size)
 {
-    if (version_id != 1)
-        return -EINVAL;
-
-    return virtio_load(VIRTIO_DEVICE(opaque), f, version_id);
+    return virtio_load(VIRTIO_DEVICE(opaque), f, 1);
 }
 
 static int virtio_balloon_load_device(VirtIODevice *vdev, QEMUFile *f,
@@ -358,6 +416,10 @@
 
     s->num_pages = qemu_get_be32(f);
     s->actual = qemu_get_be32(f);
+
+    if (balloon_stats_enabled(s)) {
+        balloon_stats_change_timer(s, s->stats_poll_interval);
+    }
     return 0;
 }
 
@@ -374,7 +436,7 @@
                                    virtio_balloon_stat, s);
 
     if (ret < 0) {
-        error_setg(errp, "Adding balloon handler failed");
+        error_setg(errp, "Only one balloon device is supported");
         virtio_cleanup(vdev);
         return;
     }
@@ -384,17 +446,6 @@
     s->svq = virtio_add_queue(vdev, 128, virtio_balloon_receive_stats);
 
     reset_stats(s);
-
-    register_savevm(dev, "virtio-balloon", -1, 1,
-                    virtio_balloon_save, virtio_balloon_load, s);
-
-    object_property_add(OBJECT(dev), "guest-stats", "guest statistics",
-                        balloon_stats_get_all, NULL, NULL, s, NULL);
-
-    object_property_add(OBJECT(dev), "guest-stats-polling-interval", "int",
-                        balloon_stats_get_poll_interval,
-                        balloon_stats_set_poll_interval,
-                        NULL, s, NULL);
 }
 
 static void virtio_balloon_device_unrealize(DeviceState *dev, Error **errp)
@@ -404,11 +455,37 @@
 
     balloon_stats_destroy_timer(s);
     qemu_remove_balloon_handler(s);
-    unregister_savevm(dev, "virtio-balloon", s);
     virtio_cleanup(vdev);
 }
 
+static void virtio_balloon_device_reset(VirtIODevice *vdev)
+{
+    VirtIOBalloon *s = VIRTIO_BALLOON(vdev);
+
+    if (s->stats_vq_elem != NULL) {
+        g_free(s->stats_vq_elem);
+        s->stats_vq_elem = NULL;
+    }
+}
+
+static void virtio_balloon_instance_init(Object *obj)
+{
+    VirtIOBalloon *s = VIRTIO_BALLOON(obj);
+
+    object_property_add(obj, "guest-stats", "guest statistics",
+                        balloon_stats_get_all, NULL, NULL, s, NULL);
+
+    object_property_add(obj, "guest-stats-polling-interval", "int",
+                        balloon_stats_get_poll_interval,
+                        balloon_stats_set_poll_interval,
+                        NULL, s, NULL);
+}
+
+VMSTATE_VIRTIO_DEVICE(balloon, 1, virtio_balloon_load, virtio_vmstate_save);
+
 static Property virtio_balloon_properties[] = {
+    DEFINE_PROP_BIT("deflate-on-oom", VirtIOBalloon, host_features,
+                    VIRTIO_BALLOON_F_DEFLATE_ON_OOM, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -418,9 +495,11 @@
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 
     dc->props = virtio_balloon_properties;
+    dc->vmsd = &vmstate_virtio_balloon;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     vdc->realize = virtio_balloon_device_realize;
     vdc->unrealize = virtio_balloon_device_unrealize;
+    vdc->reset = virtio_balloon_device_reset;
     vdc->get_config = virtio_balloon_get_config;
     vdc->set_config = virtio_balloon_set_config;
     vdc->get_features = virtio_balloon_get_features;
@@ -432,6 +511,7 @@
     .name = TYPE_VIRTIO_BALLOON,
     .parent = TYPE_VIRTIO_DEVICE,
     .instance_size = sizeof(VirtIOBalloon),
+    .instance_init = virtio_balloon_instance_init,
     .class_init = virtio_balloon_class_init,
 };
 
diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
index eb77019..a85b7c8 100644
--- a/hw/virtio/virtio-bus.c
+++ b/hw/virtio/virtio-bus.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "qemu/error-report.h"
 #include "hw/qdev.h"
@@ -38,19 +39,27 @@
 #endif
 
 /* A VirtIODevice is being plugged */
-int virtio_bus_device_plugged(VirtIODevice *vdev)
+void virtio_bus_device_plugged(VirtIODevice *vdev, Error **errp)
 {
     DeviceState *qdev = DEVICE(vdev);
     BusState *qbus = BUS(qdev_get_parent_bus(qdev));
     VirtioBusState *bus = VIRTIO_BUS(qbus);
     VirtioBusClass *klass = VIRTIO_BUS_GET_CLASS(bus);
+    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
+
     DPRINTF("%s: plug device.\n", qbus->name);
 
     if (klass->device_plugged != NULL) {
-        klass->device_plugged(qbus->parent);
+        klass->device_plugged(qbus->parent, errp);
     }
 
-    return 0;
+    /* Get the features of the plugged device. */
+    assert(vdc->get_features != NULL);
+    vdev->host_features = vdc->get_features(vdev, vdev->host_features,
+                                            errp);
+    if (klass->post_plugged != NULL) {
+        klass->post_plugged(qbus->parent, errp);
+    }
 }
 
 /* Reset the virtio_bus */
@@ -58,7 +67,7 @@
 {
     VirtIODevice *vdev = virtio_bus_get_device(bus);
 
-    DPRINTF("%s: reset device.\n", qbus->name);
+    DPRINTF("%s: reset device.\n", BUS(bus)->name);
     if (vdev != NULL) {
         virtio_reset(vdev);
     }
@@ -96,33 +105,6 @@
     return vdev->config_len;
 }
 
-/* Get the features of the plugged device. */
-uint32_t virtio_bus_get_vdev_features(VirtioBusState *bus,
-                                    uint32_t requested_features)
-{
-    VirtIODevice *vdev = virtio_bus_get_device(bus);
-    VirtioDeviceClass *k;
-
-    assert(vdev != NULL);
-    k = VIRTIO_DEVICE_GET_CLASS(vdev);
-    assert(k->get_features != NULL);
-    return k->get_features(vdev, requested_features);
-}
-
-/* Set the features of the plugged device. */
-void virtio_bus_set_vdev_features(VirtioBusState *bus,
-                                      uint32_t requested_features)
-{
-    VirtIODevice *vdev = virtio_bus_get_device(bus);
-    VirtioDeviceClass *k;
-
-    assert(vdev != NULL);
-    k = VIRTIO_DEVICE_GET_CLASS(vdev);
-    if (k->set_features != NULL) {
-        k->set_features(vdev, requested_features);
-    }
-}
-
 /* Get bad features of the plugged device. */
 uint32_t virtio_bus_get_vdev_bad_features(VirtioBusState *bus)
 {
@@ -164,6 +146,132 @@
     }
 }
 
+/*
+ * This function handles both assigning the ioeventfd handler and
+ * registering it with the kernel.
+ * assign: register/deregister ioeventfd with the kernel
+ * set_handler: use the generic ioeventfd handler
+ */
+static int set_host_notifier_internal(DeviceState *proxy, VirtioBusState *bus,
+                                      int n, bool assign, bool set_handler)
+{
+    VirtIODevice *vdev = virtio_bus_get_device(bus);
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus);
+    VirtQueue *vq = virtio_get_queue(vdev, n);
+    EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
+    int r = 0;
+
+    if (assign) {
+        r = event_notifier_init(notifier, 1);
+        if (r < 0) {
+            error_report("%s: unable to init event notifier: %d", __func__, r);
+            return r;
+        }
+        virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
+        r = k->ioeventfd_assign(proxy, notifier, n, assign);
+        if (r < 0) {
+            error_report("%s: unable to assign ioeventfd: %d", __func__, r);
+            virtio_queue_set_host_notifier_fd_handler(vq, false, false);
+            event_notifier_cleanup(notifier);
+            return r;
+        }
+    } else {
+        k->ioeventfd_assign(proxy, notifier, n, assign);
+        virtio_queue_set_host_notifier_fd_handler(vq, false, false);
+        event_notifier_cleanup(notifier);
+    }
+    return r;
+}
+
+void virtio_bus_start_ioeventfd(VirtioBusState *bus)
+{
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus);
+    DeviceState *proxy = DEVICE(BUS(bus)->parent);
+    VirtIODevice *vdev;
+    int n, r;
+
+    if (!k->ioeventfd_started || k->ioeventfd_started(proxy)) {
+        return;
+    }
+    if (k->ioeventfd_disabled(proxy)) {
+        return;
+    }
+    vdev = virtio_bus_get_device(bus);
+    for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            continue;
+        }
+        r = set_host_notifier_internal(proxy, bus, n, true, true);
+        if (r < 0) {
+            goto assign_error;
+        }
+    }
+    k->ioeventfd_set_started(proxy, true, false);
+    return;
+
+assign_error:
+    while (--n >= 0) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            continue;
+        }
+
+        r = set_host_notifier_internal(proxy, bus, n, false, false);
+        assert(r >= 0);
+    }
+    k->ioeventfd_set_started(proxy, false, true);
+    error_report("%s: failed. Fallback to userspace (slower).", __func__);
+}
+
+void virtio_bus_stop_ioeventfd(VirtioBusState *bus)
+{
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus);
+    DeviceState *proxy = DEVICE(BUS(bus)->parent);
+    VirtIODevice *vdev;
+    int n, r;
+
+    if (!k->ioeventfd_started || !k->ioeventfd_started(proxy)) {
+        return;
+    }
+    vdev = virtio_bus_get_device(bus);
+    for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            continue;
+        }
+        r = set_host_notifier_internal(proxy, bus, n, false, false);
+        assert(r >= 0);
+    }
+    k->ioeventfd_set_started(proxy, false, false);
+}
+
+/*
+ * This function switches from/to the generic ioeventfd handler.
+ * assign==false means 'use generic ioeventfd handler'.
+ */
+int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
+{
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(bus);
+    DeviceState *proxy = DEVICE(BUS(bus)->parent);
+
+    if (!k->ioeventfd_started) {
+        return -ENOSYS;
+    }
+    k->ioeventfd_set_disabled(proxy, assign);
+    if (assign) {
+        /*
+         * Stop using the generic ioeventfd, we are doing eventfd handling
+         * ourselves below
+         *
+         * FIXME: We should just switch the handler and not deassign the
+         * ioeventfd.
+         * Otherwise, there's a window where we don't have an
+         * ioeventfd and we may end up with a notification where
+         * we don't expect one.
+         */
+        virtio_bus_stop_ioeventfd(bus);
+    }
+    return set_host_notifier_internal(proxy, bus, n, assign, false);
+}
+
 static char *virtio_bus_get_dev_path(DeviceState *dev)
 {
     BusState *bus = qdev_get_parent_bus(dev);
diff --git a/hw/virtio/virtio-mmio.c b/hw/virtio/virtio-mmio.c
index 2450c13..13798b3 100644
--- a/hw/virtio/virtio-mmio.c
+++ b/hw/virtio/virtio-mmio.c
@@ -19,10 +19,13 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/sysbus.h"
 #include "hw/virtio/virtio.h"
 #include "qemu/host-utils.h"
+#include "sysemu/kvm.h"
 #include "hw/virtio/virtio-bus.h"
+#include "qemu/error-report.h"
 
 /* #define DEBUG_VIRTIO_MMIO */
 
@@ -80,15 +83,72 @@
     SysBusDevice parent_obj;
     MemoryRegion iomem;
     qemu_irq irq;
-    uint32_t host_features;
     /* Guest accessible state needing migration and reset */
     uint32_t host_features_sel;
     uint32_t guest_features_sel;
     uint32_t guest_page_shift;
     /* virtio-bus */
     VirtioBusState bus;
+    bool ioeventfd_disabled;
+    bool ioeventfd_started;
+    bool format_transport_address;
 } VirtIOMMIOProxy;
 
+static bool virtio_mmio_ioeventfd_started(DeviceState *d)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+
+    return proxy->ioeventfd_started;
+}
+
+static void virtio_mmio_ioeventfd_set_started(DeviceState *d, bool started,
+                                              bool err)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+
+    proxy->ioeventfd_started = started;
+}
+
+static bool virtio_mmio_ioeventfd_disabled(DeviceState *d)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+
+    return !kvm_eventfds_enabled() || proxy->ioeventfd_disabled;
+}
+
+static void virtio_mmio_ioeventfd_set_disabled(DeviceState *d, bool disabled)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+
+    proxy->ioeventfd_disabled = disabled;
+}
+
+static int virtio_mmio_ioeventfd_assign(DeviceState *d,
+                                        EventNotifier *notifier,
+                                        int n, bool assign)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+
+    if (assign) {
+        memory_region_add_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
+                                  true, n, notifier);
+    } else {
+        memory_region_del_eventfd(&proxy->iomem, VIRTIO_MMIO_QUEUENOTIFY, 4,
+                                  true, n, notifier);
+    }
+    return 0;
+}
+
+static void virtio_mmio_start_ioeventfd(VirtIOMMIOProxy *proxy)
+{
+    virtio_bus_start_ioeventfd(&proxy->bus);
+}
+
+static void virtio_mmio_stop_ioeventfd(VirtIOMMIOProxy *proxy)
+{
+    virtio_bus_stop_ioeventfd(&proxy->bus);
+}
+
 static uint64_t virtio_mmio_read(void *opaque, hwaddr offset, unsigned size)
 {
     VirtIOMMIOProxy *proxy = (VirtIOMMIOProxy *)opaque;
@@ -147,7 +207,7 @@
         if (proxy->host_features_sel) {
             return 0;
         }
-        return proxy->host_features;
+        return vdev->host_features;
     case VIRTIO_MMIO_QUEUENUMMAX:
         if (!virtio_queue_get_num(vdev, vdev->queue_sel)) {
             return 0;
@@ -237,15 +297,18 @@
                 proxy->guest_page_shift);
         break;
     case VIRTIO_MMIO_QUEUESEL:
-        if (value < VIRTIO_PCI_QUEUE_MAX) {
+        if (value < VIRTIO_QUEUE_MAX) {
             vdev->queue_sel = value;
         }
         break;
     case VIRTIO_MMIO_QUEUENUM:
         DPRINTF("mmio_queue write %d max %d\n", (int)value, VIRTQUEUE_MAX_SIZE);
         virtio_queue_set_num(vdev, vdev->queue_sel, value);
+        /* Note: only call this function for legacy devices */
+        virtio_queue_update_rings(vdev, vdev->queue_sel);
         break;
     case VIRTIO_MMIO_QUEUEALIGN:
+        /* Note: this is only valid for legacy devices */
         virtio_queue_set_align(vdev, vdev->queue_sel, value);
         break;
     case VIRTIO_MMIO_QUEUEPFN:
@@ -257,7 +320,7 @@
         }
         break;
     case VIRTIO_MMIO_QUEUENOTIFY:
-        if (value < VIRTIO_PCI_QUEUE_MAX) {
+        if (value < VIRTIO_QUEUE_MAX) {
             virtio_queue_notify(vdev, value);
         }
         break;
@@ -266,7 +329,16 @@
         virtio_update_irq(vdev);
         break;
     case VIRTIO_MMIO_STATUS:
+        if (!(value & VIRTIO_CONFIG_S_DRIVER_OK)) {
+            virtio_mmio_stop_ioeventfd(proxy);
+        }
+
         virtio_set_status(vdev, value & 0xff);
+
+        if (value & VIRTIO_CONFIG_S_DRIVER_OK) {
+            virtio_mmio_start_ioeventfd(proxy);
+        }
+
         if (vdev->status == 0) {
             virtio_reset(vdev);
         }
@@ -306,13 +378,6 @@
     qemu_set_irq(proxy->irq, level);
 }
 
-static unsigned int virtio_mmio_get_features(DeviceState *opaque)
-{
-    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(opaque);
-
-    return proxy->host_features;
-}
-
 static int virtio_mmio_load_config(DeviceState *opaque, QEMUFile *f)
 {
     VirtIOMMIOProxy *proxy = VIRTIO_MMIO(opaque);
@@ -336,23 +401,80 @@
 {
     VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
 
+    virtio_mmio_stop_ioeventfd(proxy);
     virtio_bus_reset(&proxy->bus);
     proxy->host_features_sel = 0;
     proxy->guest_features_sel = 0;
     proxy->guest_page_shift = 0;
 }
 
+static int virtio_mmio_set_guest_notifier(DeviceState *d, int n, bool assign,
+                                          bool with_irqfd)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
+    VirtQueue *vq = virtio_get_queue(vdev, n);
+    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
+
+    if (assign) {
+        int r = event_notifier_init(notifier, 0);
+        if (r < 0) {
+            return r;
+        }
+        virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
+    } else {
+        virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
+        event_notifier_cleanup(notifier);
+    }
+
+    if (vdc->guest_notifier_mask) {
+        vdc->guest_notifier_mask(vdev, n, !assign);
+    }
+
+    return 0;
+}
+
+static int virtio_mmio_set_guest_notifiers(DeviceState *d, int nvqs,
+                                           bool assign)
+{
+    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(d);
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    /* TODO: need to check if kvm-arm supports irqfd */
+    bool with_irqfd = false;
+    int r, n;
+
+    nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX);
+
+    for (n = 0; n < nvqs; n++) {
+        if (!virtio_queue_get_num(vdev, n)) {
+            break;
+        }
+
+        r = virtio_mmio_set_guest_notifier(d, n, assign, with_irqfd);
+        if (r < 0) {
+            goto assign_error;
+        }
+    }
+
+    return 0;
+
+assign_error:
+    /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */
+    assert(assign);
+    while (--n >= 0) {
+        virtio_mmio_set_guest_notifier(d, n, !assign, false);
+    }
+    return r;
+}
+
 /* virtio-mmio device */
 
-/* This is called by virtio-bus just after the device is plugged. */
-static void virtio_mmio_device_plugged(DeviceState *opaque)
-{
-    VirtIOMMIOProxy *proxy = VIRTIO_MMIO(opaque);
-
-    proxy->host_features |= (0x1 << VIRTIO_F_NOTIFY_ON_EMPTY);
-    proxy->host_features = virtio_bus_get_vdev_features(&proxy->bus,
-                                                        proxy->host_features);
-}
+static Property virtio_mmio_properties[] = {
+    DEFINE_PROP_BOOL("format_transport_address", VirtIOMMIOProxy,
+                     format_transport_address, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
 static void virtio_mmio_realizefn(DeviceState *d, Error **errp)
 {
@@ -367,19 +489,14 @@
     sysbus_init_mmio(sbd, &proxy->iomem);
 }
 
-static Property virtio_mmio_properties[] = {
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOMMIOProxy, host_features),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
 static void virtio_mmio_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
 
-    dc->props = virtio_mmio_properties;
     dc->realize = virtio_mmio_realizefn;
     dc->reset = virtio_mmio_reset;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->props = virtio_mmio_properties;
 }
 
 static const TypeInfo virtio_mmio_info = {
@@ -391,6 +508,46 @@
 
 /* virtio-mmio-bus. */
 
+static char *virtio_mmio_bus_get_dev_path(DeviceState *dev)
+{
+    BusState *virtio_mmio_bus;
+    VirtIOMMIOProxy *virtio_mmio_proxy;
+    char *proxy_path;
+    SysBusDevice *proxy_sbd;
+    char *path;
+
+    virtio_mmio_bus = qdev_get_parent_bus(dev);
+    virtio_mmio_proxy = VIRTIO_MMIO(virtio_mmio_bus->parent);
+    proxy_path = qdev_get_dev_path(DEVICE(virtio_mmio_proxy));
+
+    /*
+     * If @format_transport_address is false, then we just perform the same as
+     * virtio_bus_get_dev_path(): we delegate the address formatting for the
+     * device on the virtio-mmio bus to the bus that the virtio-mmio proxy
+     * (i.e., the device that implements the virtio-mmio bus) resides on. In
+     * this case the base address of the virtio-mmio transport will be
+     * invisible.
+     */
+    if (!virtio_mmio_proxy->format_transport_address) {
+        return proxy_path;
+    }
+
+    /* Otherwise, we append the base address of the transport. */
+    proxy_sbd = SYS_BUS_DEVICE(virtio_mmio_proxy);
+    assert(proxy_sbd->num_mmio == 1);
+    assert(proxy_sbd->mmio[0].memory == &virtio_mmio_proxy->iomem);
+
+    if (proxy_path) {
+        path = g_strdup_printf("%s/virtio-mmio@" TARGET_FMT_plx, proxy_path,
+                               proxy_sbd->mmio[0].addr);
+    } else {
+        path = g_strdup_printf("virtio-mmio@" TARGET_FMT_plx,
+                               proxy_sbd->mmio[0].addr);
+    }
+    g_free(proxy_path);
+    return path;
+}
+
 static void virtio_mmio_bus_class_init(ObjectClass *klass, void *data)
 {
     BusClass *bus_class = BUS_CLASS(klass);
@@ -399,10 +556,15 @@
     k->notify = virtio_mmio_update_irq;
     k->save_config = virtio_mmio_save_config;
     k->load_config = virtio_mmio_load_config;
-    k->get_features = virtio_mmio_get_features;
-    k->device_plugged = virtio_mmio_device_plugged;
+    k->set_guest_notifiers = virtio_mmio_set_guest_notifiers;
+    k->ioeventfd_started = virtio_mmio_ioeventfd_started;
+    k->ioeventfd_set_started = virtio_mmio_ioeventfd_set_started;
+    k->ioeventfd_disabled = virtio_mmio_ioeventfd_disabled;
+    k->ioeventfd_set_disabled = virtio_mmio_ioeventfd_set_disabled;
+    k->ioeventfd_assign = virtio_mmio_ioeventfd_assign;
     k->has_variable_vring_alignment = true;
     bus_class->max_dev = 1;
+    bus_class->get_dev_path = virtio_mmio_bus_get_dev_path;
 }
 
 static const TypeInfo virtio_mmio_bus_info = {
diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
index 76b9b4f..75697fc 100644
--- a/hw/virtio/virtio-pci.c
+++ b/hw/virtio/virtio-pci.c
@@ -15,15 +15,18 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
+#include "standard-headers/linux/virtio_pci.h"
 #include "hw/virtio/virtio.h"
 #include "hw/virtio/virtio-blk.h"
 #include "hw/virtio/virtio-net.h"
 #include "hw/virtio/virtio-serial.h"
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/virtio-balloon.h"
+#include "hw/virtio/virtio-input.h"
 #include "hw/pci/pci.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
 #include "hw/pci/msi.h"
 #include "hw/pci/msix.h"
@@ -35,59 +38,17 @@
 #include "hw/virtio/virtio-bus.h"
 #include "qapi/visitor.h"
 
-/* from Linux's linux/virtio_pci.h */
+#define VIRTIO_PCI_REGION_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_present(dev))
 
-/* A 32-bit r/o bitmask of the features supported by the host */
-#define VIRTIO_PCI_HOST_FEATURES        0
-
-/* A 32-bit r/w bitmask of features activated by the guest */
-#define VIRTIO_PCI_GUEST_FEATURES       4
-
-/* A 32-bit r/w PFN for the currently selected queue */
-#define VIRTIO_PCI_QUEUE_PFN            8
-
-/* A 16-bit r/o queue size for the currently selected queue */
-#define VIRTIO_PCI_QUEUE_NUM            12
-
-/* A 16-bit r/w queue selector */
-#define VIRTIO_PCI_QUEUE_SEL            14
-
-/* A 16-bit r/w queue notifier */
-#define VIRTIO_PCI_QUEUE_NOTIFY         16
-
-/* An 8-bit device status register.  */
-#define VIRTIO_PCI_STATUS               18
-
-/* An 8-bit r/o interrupt status register.  Reading the value will return the
- * current contents of the ISR and will also clear it.  This is effectively
- * a read-and-acknowledge. */
-#define VIRTIO_PCI_ISR                  19
-
-/* MSI-X registers: only enabled if MSI-X is enabled. */
-/* A 16-bit vector for configuration changes. */
-#define VIRTIO_MSI_CONFIG_VECTOR        20
-/* A 16-bit vector for selected queue notifications. */
-#define VIRTIO_MSI_QUEUE_VECTOR         22
-
-/* Config space size */
-#define VIRTIO_PCI_CONFIG_NOMSI         20
-#define VIRTIO_PCI_CONFIG_MSI           24
-#define VIRTIO_PCI_REGION_SIZE(dev)     (msix_present(dev) ? \
-                                         VIRTIO_PCI_CONFIG_MSI : \
-                                         VIRTIO_PCI_CONFIG_NOMSI)
+#undef VIRTIO_PCI_CONFIG
 
 /* The remaining space is defined by each driver as the per-driver
  * configuration space */
-#define VIRTIO_PCI_CONFIG(dev)          (msix_enabled(dev) ? \
-                                         VIRTIO_PCI_CONFIG_MSI : \
-                                         VIRTIO_PCI_CONFIG_NOMSI)
-
-/* How many bits to shift physical queue address written to QUEUE_PFN.
- * 12 is historical, and due to x86 page size. */
-#define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
+#define VIRTIO_PCI_CONFIG_SIZE(dev)     VIRTIO_PCI_CONFIG_OFF(msix_enabled(dev))
 
 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtIOPCIProxy *dev);
+static void virtio_pci_reset(DeviceState *qdev);
 
 /* virtio device */
 /* DeviceState to VirtIOPCIProxy. For use off data-path. TODO: use QOM. */
@@ -127,6 +88,129 @@
         qemu_put_be16(f, vdev->config_vector);
 }
 
+static void virtio_pci_load_modern_queue_state(VirtIOPCIQueue *vq,
+                                               QEMUFile *f)
+{
+    vq->num = qemu_get_be16(f);
+    vq->enabled = qemu_get_be16(f);
+    vq->desc[0] = qemu_get_be32(f);
+    vq->desc[1] = qemu_get_be32(f);
+    vq->avail[0] = qemu_get_be32(f);
+    vq->avail[1] = qemu_get_be32(f);
+    vq->used[0] = qemu_get_be32(f);
+    vq->used[1] = qemu_get_be32(f);
+}
+
+static bool virtio_pci_has_extra_state(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return proxy->flags & VIRTIO_PCI_FLAG_MIGRATE_EXTRA;
+}
+
+static int get_virtio_pci_modern_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIOPCIProxy *proxy = pv;
+    int i;
+
+    proxy->dfselect = qemu_get_be32(f);
+    proxy->gfselect = qemu_get_be32(f);
+    proxy->guest_features[0] = qemu_get_be32(f);
+    proxy->guest_features[1] = qemu_get_be32(f);
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        virtio_pci_load_modern_queue_state(&proxy->vqs[i], f);
+    }
+
+    return 0;
+}
+
+static void virtio_pci_save_modern_queue_state(VirtIOPCIQueue *vq,
+                                               QEMUFile *f)
+{
+    qemu_put_be16(f, vq->num);
+    qemu_put_be16(f, vq->enabled);
+    qemu_put_be32(f, vq->desc[0]);
+    qemu_put_be32(f, vq->desc[1]);
+    qemu_put_be32(f, vq->avail[0]);
+    qemu_put_be32(f, vq->avail[1]);
+    qemu_put_be32(f, vq->used[0]);
+    qemu_put_be32(f, vq->used[1]);
+}
+
+static void put_virtio_pci_modern_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIOPCIProxy *proxy = pv;
+    int i;
+
+    qemu_put_be32(f, proxy->dfselect);
+    qemu_put_be32(f, proxy->gfselect);
+    qemu_put_be32(f, proxy->guest_features[0]);
+    qemu_put_be32(f, proxy->guest_features[1]);
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        virtio_pci_save_modern_queue_state(&proxy->vqs[i], f);
+    }
+}
+
+static const VMStateInfo vmstate_info_virtio_pci_modern_state = {
+    .name = "virtqueue_state",
+    .get = get_virtio_pci_modern_state,
+    .put = put_virtio_pci_modern_state,
+};
+
+static bool virtio_pci_modern_state_needed(void *opaque)
+{
+    VirtIOPCIProxy *proxy = opaque;
+
+    return virtio_pci_modern(proxy);
+}
+
+static const VMStateDescription vmstate_virtio_pci_modern_state = {
+    .name = "virtio_pci/modern_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_pci_modern_state_needed,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "modern_state",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &vmstate_info_virtio_pci_modern_state,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_virtio_pci = {
+    .name = "virtio_pci",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_virtio_pci_modern_state,
+        NULL
+    }
+};
+
+static void virtio_pci_save_extra_state(DeviceState *d, QEMUFile *f)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    vmstate_save_state(f, &vmstate_virtio_pci, proxy, NULL);
+}
+
+static int virtio_pci_load_extra_state(DeviceState *d, QEMUFile *f)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return vmstate_load_state(f, &vmstate_virtio_pci, proxy, 1);
+}
+
 static void virtio_pci_save_queue(DeviceState *d, int n, QEMUFile *f)
 {
     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
@@ -174,93 +258,107 @@
     if (vector != VIRTIO_NO_VECTOR) {
         return msix_vector_use(&proxy->pci_dev, vector);
     }
+
     return 0;
 }
 
-static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
-                                                 int n, bool assign, bool set_handler)
+static bool virtio_pci_ioeventfd_started(DeviceState *d)
 {
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return proxy->ioeventfd_started;
+}
+
+static void virtio_pci_ioeventfd_set_started(DeviceState *d, bool started,
+                                             bool err)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    proxy->ioeventfd_started = started;
+}
+
+static bool virtio_pci_ioeventfd_disabled(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    return proxy->ioeventfd_disabled ||
+        !(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD);
+}
+
+static void virtio_pci_ioeventfd_set_disabled(DeviceState *d, bool disabled)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
+
+    proxy->ioeventfd_disabled = disabled;
+}
+
+#define QEMU_VIRTIO_PCI_QUEUE_MEM_MULT 0x1000
+
+static int virtio_pci_ioeventfd_assign(DeviceState *d, EventNotifier *notifier,
+                                       int n, bool assign)
+{
+    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
     VirtQueue *vq = virtio_get_queue(vdev, n);
-    EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
-    int r = 0;
+    bool legacy = virtio_pci_legacy(proxy);
+    bool modern = virtio_pci_modern(proxy);
+    bool fast_mmio = kvm_ioeventfd_any_length_enabled();
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
+    MemoryRegion *modern_mr = &proxy->notify.mr;
+    MemoryRegion *modern_notify_mr = &proxy->notify_pio.mr;
+    MemoryRegion *legacy_mr = &proxy->bar;
+    hwaddr modern_addr = QEMU_VIRTIO_PCI_QUEUE_MEM_MULT *
+                         virtio_get_queue_index(vq);
+    hwaddr legacy_addr = VIRTIO_PCI_QUEUE_NOTIFY;
 
     if (assign) {
-        r = event_notifier_init(notifier, 1);
-        if (r < 0) {
-            error_report("%s: unable to init event notifier: %d",
-                         __func__, r);
-            return r;
+        if (modern) {
+            if (fast_mmio) {
+                memory_region_add_eventfd(modern_mr, modern_addr, 0,
+                                          false, n, notifier);
+            } else {
+                memory_region_add_eventfd(modern_mr, modern_addr, 2,
+                                          false, n, notifier);
+            }
+            if (modern_pio) {
+                memory_region_add_eventfd(modern_notify_mr, 0, 2,
+                                              true, n, notifier);
+            }
         }
-        virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
-        memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
-                                  true, n, notifier);
+        if (legacy) {
+            memory_region_add_eventfd(legacy_mr, legacy_addr, 2,
+                                      true, n, notifier);
+        }
     } else {
-        memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
-                                  true, n, notifier);
-        virtio_queue_set_host_notifier_fd_handler(vq, false, false);
-        event_notifier_cleanup(notifier);
+        if (modern) {
+            if (fast_mmio) {
+                memory_region_del_eventfd(modern_mr, modern_addr, 0,
+                                          false, n, notifier);
+            } else {
+                memory_region_del_eventfd(modern_mr, modern_addr, 2,
+                                          false, n, notifier);
+            }
+            if (modern_pio) {
+                memory_region_del_eventfd(modern_notify_mr, 0, 2,
+                                          true, n, notifier);
+            }
+        }
+        if (legacy) {
+            memory_region_del_eventfd(legacy_mr, legacy_addr, 2,
+                                      true, n, notifier);
+        }
     }
-    return r;
+    return 0;
 }
 
 static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
 {
-    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
-    int n, r;
-
-    if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) ||
-        proxy->ioeventfd_disabled ||
-        proxy->ioeventfd_started) {
-        return;
-    }
-
-    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
-        if (!virtio_queue_get_num(vdev, n)) {
-            continue;
-        }
-
-        r = virtio_pci_set_host_notifier_internal(proxy, n, true, true);
-        if (r < 0) {
-            goto assign_error;
-        }
-    }
-    proxy->ioeventfd_started = true;
-    return;
-
-assign_error:
-    while (--n >= 0) {
-        if (!virtio_queue_get_num(vdev, n)) {
-            continue;
-        }
-
-        r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
-        assert(r >= 0);
-    }
-    proxy->ioeventfd_started = false;
-    error_report("%s: failed. Fallback to a userspace (slower).", __func__);
+    virtio_bus_start_ioeventfd(&proxy->bus);
 }
 
 static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
 {
-    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
-    int r;
-    int n;
-
-    if (!proxy->ioeventfd_started) {
-        return;
-    }
-
-    for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) {
-        if (!virtio_queue_get_num(vdev, n)) {
-            continue;
-        }
-
-        r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
-        assert(r >= 0);
-        (void)r;
-    }
-    proxy->ioeventfd_started = false;
+    virtio_bus_stop_ioeventfd(&proxy->bus);
 }
 
 static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
@@ -280,19 +378,17 @@
     case VIRTIO_PCI_QUEUE_PFN:
         pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
         if (pa == 0) {
-            virtio_pci_stop_ioeventfd(proxy);
-            virtio_reset(vdev);
-            msix_unuse_all_vectors(&proxy->pci_dev);
+            virtio_pci_reset(DEVICE(proxy));
         }
         else
             virtio_queue_set_addr(vdev, vdev->queue_sel, pa);
         break;
     case VIRTIO_PCI_QUEUE_SEL:
-        if (val < VIRTIO_PCI_QUEUE_MAX)
+        if (val < VIRTIO_QUEUE_MAX)
             vdev->queue_sel = val;
         break;
     case VIRTIO_PCI_QUEUE_NOTIFY:
-        if (val < VIRTIO_PCI_QUEUE_MAX) {
+        if (val < VIRTIO_QUEUE_MAX) {
             virtio_queue_notify(vdev, val);
         }
         break;
@@ -308,8 +404,7 @@
         }
 
         if (vdev->status == 0) {
-            virtio_reset(vdev);
-            msix_unuse_all_vectors(&proxy->pci_dev);
+            virtio_pci_reset(DEVICE(proxy));
         }
 
         /* Linux before 2.6.34 drives the device without enabling
@@ -351,7 +446,7 @@
 
     switch (addr) {
     case VIRTIO_PCI_HOST_FEATURES:
-        ret = proxy->host_features;
+        ret = vdev->host_features;
         break;
     case VIRTIO_PCI_GUEST_FEATURES:
         ret = vdev->guest_features;
@@ -393,7 +488,7 @@
 {
     VirtIOPCIProxy *proxy = opaque;
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
+    uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
     uint64_t val = 0;
     if (addr < config) {
         return virtio_ioport_read(proxy, addr);
@@ -424,7 +519,7 @@
                                     uint64_t val, unsigned size)
 {
     VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
+    uint32_t config = VIRTIO_PCI_CONFIG_SIZE(&proxy->pci_dev);
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
     if (addr < config) {
         virtio_ioport_write(proxy, addr, val);
@@ -464,11 +559,89 @@
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+/* Below are generic functions to do memcpy from/to an address space,
+ * without byteswaps, with input validation.
+ *
+ * As regular address_space_* APIs all do some kind of byteswap at least for
+ * some host/target combinations, we are forced to explicitly convert to a
+ * known-endianness integer value.
+ * It doesn't really matter which endian format to go through, so the code
+ * below selects the endian that causes the least amount of work on the given
+ * host.
+ *
+ * Note: host pointer must be aligned.
+ */
+static
+void virtio_address_space_write(AddressSpace *as, hwaddr addr,
+                                const uint8_t *buf, int len)
+{
+    uint32_t val;
+
+    /* address_space_* APIs assume an aligned address.
+     * As address is under guest control, handle illegal values.
+     */
+    addr &= ~(len - 1);
+
+    /* Make sure caller aligned buf properly */
+    assert(!(((uintptr_t)buf) & (len - 1)));
+
+    switch (len) {
+    case 1:
+        val = pci_get_byte(buf);
+        address_space_stb(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
+        break;
+    case 2:
+        val = pci_get_word(buf);
+        address_space_stw_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
+        break;
+    case 4:
+        val = pci_get_long(buf);
+        address_space_stl_le(as, addr, val, MEMTXATTRS_UNSPECIFIED, NULL);
+        break;
+    default:
+        /* As length is under guest control, handle illegal values. */
+        break;
+    }
+}
+
+static void
+virtio_address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
+{
+    uint32_t val;
+
+    /* address_space_* APIs assume an aligned address.
+     * As address is under guest control, handle illegal values.
+     */
+    addr &= ~(len - 1);
+
+    /* Make sure caller aligned buf properly */
+    assert(!(((uintptr_t)buf) & (len - 1)));
+
+    switch (len) {
+    case 1:
+        val = address_space_ldub(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+        pci_set_byte(buf, val);
+        break;
+    case 2:
+        val = address_space_lduw_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+        pci_set_word(buf, val);
+        break;
+    case 4:
+        val = address_space_ldl_le(as, addr, MEMTXATTRS_UNSPECIFIED, NULL);
+        pci_set_long(buf, val);
+        break;
+    default:
+        /* As length is under guest control, handle illegal values. */
+        break;
+    }
+}
+
 static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
                                 uint32_t val, int len)
 {
     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    struct virtio_pci_cfg_cap *cfg;
 
     pci_default_write_config(pci_dev, address, val, len);
 
@@ -477,24 +650,62 @@
         virtio_pci_stop_ioeventfd(proxy);
         virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
     }
+
+    if (proxy->config_cap &&
+        ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
+                                                                  pci_cfg_data),
+                       sizeof cfg->pci_cfg_data)) {
+        uint32_t off;
+        uint32_t len;
+
+        cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
+        off = le32_to_cpu(cfg->cap.offset);
+        len = le32_to_cpu(cfg->cap.length);
+
+        if (len == 1 || len == 2 || len == 4) {
+            assert(len <= sizeof cfg->pci_cfg_data);
+            virtio_address_space_write(&proxy->modern_as, off,
+                                       cfg->pci_cfg_data, len);
+        }
+    }
 }
 
-static unsigned virtio_pci_get_features(DeviceState *d)
+static uint32_t virtio_read_config(PCIDevice *pci_dev,
+                                   uint32_t address, int len)
 {
-    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
-    return proxy->host_features;
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+    struct virtio_pci_cfg_cap *cfg;
+
+    if (proxy->config_cap &&
+        ranges_overlap(address, len, proxy->config_cap + offsetof(struct virtio_pci_cfg_cap,
+                                                                  pci_cfg_data),
+                       sizeof cfg->pci_cfg_data)) {
+        uint32_t off;
+        uint32_t len;
+
+        cfg = (void *)(proxy->pci_dev.config + proxy->config_cap);
+        off = le32_to_cpu(cfg->cap.offset);
+        len = le32_to_cpu(cfg->cap.length);
+
+        if (len == 1 || len == 2 || len == 4) {
+            assert(len <= sizeof cfg->pci_cfg_data);
+            virtio_address_space_read(&proxy->modern_as, off,
+                                      cfg->pci_cfg_data, len);
+        }
+    }
+
+    return pci_default_read_config(pci_dev, address, len);
 }
 
 static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
                                         unsigned int queue_no,
-                                        unsigned int vector,
-                                        MSIMessage msg)
+                                        unsigned int vector)
 {
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
     int ret;
 
     if (irqfd->users == 0) {
-        ret = kvm_irqchip_add_msi_route(kvm_state, msg);
+        ret = kvm_irqchip_add_msi_route(kvm_state, vector, &proxy->pci_dev);
         if (ret < 0) {
             return ret;
         }
@@ -521,9 +732,7 @@
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
     VirtQueue *vq = virtio_get_queue(vdev, queue_no);
     EventNotifier *n = virtio_queue_get_guest_notifier(vq);
-    int ret;
-    ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, NULL, irqfd->virq);
-    return ret;
+    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, irqfd->virq);
 }
 
 static void kvm_virtio_pci_irqfd_release(VirtIOPCIProxy *proxy,
@@ -536,7 +745,7 @@
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
     int ret;
 
-    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, n, irqfd->virq);
+    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, irqfd->virq);
     assert(ret == 0);
     (void)ret;
 }
@@ -548,7 +757,6 @@
     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
     unsigned int vector;
     int ret, queue_no;
-    MSIMessage msg;
 
     for (queue_no = 0; queue_no < nvqs; queue_no++) {
         if (!virtio_queue_get_num(vdev, queue_no)) {
@@ -558,15 +766,14 @@
         if (vector >= msix_nr_vectors_allocated(dev)) {
             continue;
         }
-        msg = msix_get_message(dev, vector);
-        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector, msg);
+        ret = kvm_virtio_pci_vq_vector_use(proxy, queue_no, vector);
         if (ret < 0) {
             goto undo;
         }
         /* If guest supports masking, set up irqfd now.
          * Otherwise, delay until unmasked in the frontend.
          */
-        if (k->guest_notifier_mask) {
+        if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
             ret = kvm_virtio_pci_irqfd_use(proxy, queue_no, vector);
             if (ret < 0) {
                 kvm_virtio_pci_vq_vector_release(proxy, vector);
@@ -582,7 +789,7 @@
         if (vector >= msix_nr_vectors_allocated(dev)) {
             continue;
         }
-        if (k->guest_notifier_mask) {
+        if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
             kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
         }
         kvm_virtio_pci_vq_vector_release(proxy, vector);
@@ -609,7 +816,7 @@
         /* If guest supports masking, clean up irqfd now.
          * Otherwise, it was cleaned when masked in the frontend.
          */
-        if (k->guest_notifier_mask) {
+        if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
             kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
         }
         kvm_virtio_pci_vq_vector_release(proxy, vector);
@@ -631,17 +838,19 @@
     if (proxy->vector_irqfd) {
         irqfd = &proxy->vector_irqfd[vector];
         if (irqfd->msg.data != msg.data || irqfd->msg.address != msg.address) {
-            ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg);
+            ret = kvm_irqchip_update_msi_route(kvm_state, irqfd->virq, msg,
+                                               &proxy->pci_dev);
             if (ret < 0) {
                 return ret;
             }
+            kvm_irqchip_commit_routes(kvm_state);
         }
     }
 
     /* If guest supports masking, irqfd is already setup, unmask it.
      * Otherwise, set it up now.
      */
-    if (k->guest_notifier_mask) {
+    if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
         k->guest_notifier_mask(vdev, queue_no, false);
         /* Test after unmasking to avoid losing events. */
         if (k->guest_notifier_pending &&
@@ -664,7 +873,7 @@
     /* If guest supports masking, keep irqfd but mask it.
      * Otherwise, clean it up now.
      */ 
-    if (k->guest_notifier_mask) {
+    if (vdev->use_guest_notifier_mask && k->guest_notifier_mask) {
         k->guest_notifier_mask(vdev, queue_no, true);
     } else {
         kvm_virtio_pci_irqfd_release(proxy, queue_no, vector);
@@ -676,28 +885,35 @@
 {
     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
-    int ret, queue_no;
+    VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
+    int ret, index, unmasked = 0;
 
-    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
-        if (!virtio_queue_get_num(vdev, queue_no)) {
+    while (vq) {
+        index = virtio_get_queue_index(vq);
+        if (!virtio_queue_get_num(vdev, index)) {
             break;
         }
-        if (virtio_queue_vector(vdev, queue_no) != vector) {
-            continue;
+        if (index < proxy->nvqs_with_notifiers) {
+            ret = virtio_pci_vq_vector_unmask(proxy, index, vector, msg);
+            if (ret < 0) {
+                goto undo;
+            }
+            ++unmasked;
         }
-        ret = virtio_pci_vq_vector_unmask(proxy, queue_no, vector, msg);
-        if (ret < 0) {
-            goto undo;
-        }
+        vq = virtio_vector_next_queue(vq);
     }
+
     return 0;
 
 undo:
-    while (--queue_no >= 0) {
-        if (virtio_queue_vector(vdev, queue_no) != vector) {
-            continue;
+    vq = virtio_vector_first_queue(vdev, vector);
+    while (vq && unmasked >= 0) {
+        index = virtio_get_queue_index(vq);
+        if (index < proxy->nvqs_with_notifiers) {
+            virtio_pci_vq_vector_mask(proxy, index, vector);
+            --unmasked;
         }
-        virtio_pci_vq_vector_mask(proxy, queue_no, vector);
+        vq = virtio_vector_next_queue(vq);
     }
     return ret;
 }
@@ -706,16 +922,18 @@
 {
     VirtIOPCIProxy *proxy = container_of(dev, VirtIOPCIProxy, pci_dev);
     VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
-    int queue_no;
+    VirtQueue *vq = virtio_vector_first_queue(vdev, vector);
+    int index;
 
-    for (queue_no = 0; queue_no < proxy->nvqs_with_notifiers; queue_no++) {
-        if (!virtio_queue_get_num(vdev, queue_no)) {
+    while (vq) {
+        index = virtio_get_queue_index(vq);
+        if (!virtio_queue_get_num(vdev, index)) {
             break;
         }
-        if (virtio_queue_vector(vdev, queue_no) != vector) {
-            continue;
+        if (index < proxy->nvqs_with_notifiers) {
+            virtio_pci_vq_vector_mask(proxy, index, vector);
         }
-        virtio_pci_vq_vector_mask(proxy, queue_no, vector);
+        vq = virtio_vector_next_queue(vq);
     }
 }
 
@@ -772,7 +990,9 @@
         event_notifier_cleanup(notifier);
     }
 
-    if (!msix_enabled(&proxy->pci_dev) && vdc->guest_notifier_mask) {
+    if (!msix_enabled(&proxy->pci_dev) &&
+        vdev->use_guest_notifier_mask &&
+        vdc->guest_notifier_mask) {
         vdc->guest_notifier_mask(vdev, n, !assign);
     }
 
@@ -794,7 +1014,7 @@
     bool with_irqfd = msix_enabled(&proxy->pci_dev) &&
         kvm_msi_via_irqfd_enabled();
 
-    nvqs = MIN(nvqs, VIRTIO_PCI_QUEUE_MAX);
+    nvqs = MIN(nvqs, VIRTIO_QUEUE_MAX);
 
     /* When deassigning, pass a consistent nvqs value
      * to avoid leaking notifiers.
@@ -861,24 +1081,6 @@
     return r;
 }
 
-static int virtio_pci_set_host_notifier(DeviceState *d, int n, bool assign)
-{
-    VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
-
-    /* Stop using ioeventfd for virtqueue kick if the device starts using host
-     * notifiers.  This makes it easy to avoid stepping on each others' toes.
-     */
-    proxy->ioeventfd_disabled = assign;
-    if (assign) {
-        virtio_pci_stop_ioeventfd(proxy);
-    }
-    /* We don't need to start here: it's not needed because backend
-     * currently only stops on status change away from ok,
-     * reset, vmstop and such. If we do add code to start here,
-     * need to check vmstate, device state etc. */
-    return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
-}
-
 static void virtio_pci_vmstate_change(DeviceState *d, bool running)
 {
     VirtIOPCIProxy *proxy = to_virtio_pci_proxy(d);
@@ -902,16 +1104,13 @@
 }
 
 #ifdef CONFIG_VIRTFS
-static int virtio_9p_init_pci(VirtIOPCIProxy *vpci_dev)
+static void virtio_9p_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     V9fsPCIState *dev = VIRTIO_9P_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
 
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static Property virtio_9p_pci_properties[] = {
@@ -927,7 +1126,7 @@
     PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
     VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
 
-    k->init = virtio_9p_init_pci;
+    k->realize = virtio_9p_pci_realize;
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_9P;
     pcidev_k->revision = VIRTIO_PCI_ABI_VERSION;
@@ -957,103 +1156,715 @@
  * virtio-pci: This is the PCIDevice which has a virtio-pci-bus.
  */
 
+static int virtio_pci_query_nvectors(DeviceState *d)
+{
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+
+    return proxy->nvectors;
+}
+
+static int virtio_pci_add_mem_cap(VirtIOPCIProxy *proxy,
+                                   struct virtio_pci_cap *cap)
+{
+    PCIDevice *dev = &proxy->pci_dev;
+    int offset;
+
+    offset = pci_add_capability(dev, PCI_CAP_ID_VNDR, 0, cap->cap_len);
+    assert(offset > 0);
+
+    assert(cap->cap_len >= sizeof *cap);
+    memcpy(dev->config + offset + PCI_CAP_FLAGS, &cap->cap_len,
+           cap->cap_len - PCI_CAP_FLAGS);
+
+    return offset;
+}
+
+static uint64_t virtio_pci_common_read(void *opaque, hwaddr addr,
+                                       unsigned size)
+{
+    VirtIOPCIProxy *proxy = opaque;
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    uint32_t val = 0;
+    int i;
+
+    switch (addr) {
+    case VIRTIO_PCI_COMMON_DFSELECT:
+        val = proxy->dfselect;
+        break;
+    case VIRTIO_PCI_COMMON_DF:
+        if (proxy->dfselect <= 1) {
+            val = (vdev->host_features & ~VIRTIO_LEGACY_FEATURES) >>
+                (32 * proxy->dfselect);
+        }
+        break;
+    case VIRTIO_PCI_COMMON_GFSELECT:
+        val = proxy->gfselect;
+        break;
+    case VIRTIO_PCI_COMMON_GF:
+        if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
+            val = proxy->guest_features[proxy->gfselect];
+        }
+        break;
+    case VIRTIO_PCI_COMMON_MSIX:
+        val = vdev->config_vector;
+        break;
+    case VIRTIO_PCI_COMMON_NUMQ:
+        for (i = 0; i < VIRTIO_QUEUE_MAX; ++i) {
+            if (virtio_queue_get_num(vdev, i)) {
+                val = i + 1;
+            }
+        }
+        break;
+    case VIRTIO_PCI_COMMON_STATUS:
+        val = vdev->status;
+        break;
+    case VIRTIO_PCI_COMMON_CFGGENERATION:
+        val = vdev->generation;
+        break;
+    case VIRTIO_PCI_COMMON_Q_SELECT:
+        val = vdev->queue_sel;
+        break;
+    case VIRTIO_PCI_COMMON_Q_SIZE:
+        val = virtio_queue_get_num(vdev, vdev->queue_sel);
+        break;
+    case VIRTIO_PCI_COMMON_Q_MSIX:
+        val = virtio_queue_vector(vdev, vdev->queue_sel);
+        break;
+    case VIRTIO_PCI_COMMON_Q_ENABLE:
+        val = proxy->vqs[vdev->queue_sel].enabled;
+        break;
+    case VIRTIO_PCI_COMMON_Q_NOFF:
+        /* Simply map queues in order */
+        val = vdev->queue_sel;
+        break;
+    case VIRTIO_PCI_COMMON_Q_DESCLO:
+        val = proxy->vqs[vdev->queue_sel].desc[0];
+        break;
+    case VIRTIO_PCI_COMMON_Q_DESCHI:
+        val = proxy->vqs[vdev->queue_sel].desc[1];
+        break;
+    case VIRTIO_PCI_COMMON_Q_AVAILLO:
+        val = proxy->vqs[vdev->queue_sel].avail[0];
+        break;
+    case VIRTIO_PCI_COMMON_Q_AVAILHI:
+        val = proxy->vqs[vdev->queue_sel].avail[1];
+        break;
+    case VIRTIO_PCI_COMMON_Q_USEDLO:
+        val = proxy->vqs[vdev->queue_sel].used[0];
+        break;
+    case VIRTIO_PCI_COMMON_Q_USEDHI:
+        val = proxy->vqs[vdev->queue_sel].used[1];
+        break;
+    default:
+        val = 0;
+    }
+
+    return val;
+}
+
+static void virtio_pci_common_write(void *opaque, hwaddr addr,
+                                    uint64_t val, unsigned size)
+{
+    VirtIOPCIProxy *proxy = opaque;
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+
+    switch (addr) {
+    case VIRTIO_PCI_COMMON_DFSELECT:
+        proxy->dfselect = val;
+        break;
+    case VIRTIO_PCI_COMMON_GFSELECT:
+        proxy->gfselect = val;
+        break;
+    case VIRTIO_PCI_COMMON_GF:
+        if (proxy->gfselect < ARRAY_SIZE(proxy->guest_features)) {
+            proxy->guest_features[proxy->gfselect] = val;
+            virtio_set_features(vdev,
+                                (((uint64_t)proxy->guest_features[1]) << 32) |
+                                proxy->guest_features[0]);
+        }
+        break;
+    case VIRTIO_PCI_COMMON_MSIX:
+        msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
+        /* Make it possible for guest to discover an error took place. */
+        if (msix_vector_use(&proxy->pci_dev, val) < 0) {
+            val = VIRTIO_NO_VECTOR;
+        }
+        vdev->config_vector = val;
+        break;
+    case VIRTIO_PCI_COMMON_STATUS:
+        if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) {
+            virtio_pci_stop_ioeventfd(proxy);
+        }
+
+        virtio_set_status(vdev, val & 0xFF);
+
+        if (val & VIRTIO_CONFIG_S_DRIVER_OK) {
+            virtio_pci_start_ioeventfd(proxy);
+        }
+
+        if (vdev->status == 0) {
+            virtio_pci_reset(DEVICE(proxy));
+        }
+
+        break;
+    case VIRTIO_PCI_COMMON_Q_SELECT:
+        if (val < VIRTIO_QUEUE_MAX) {
+            vdev->queue_sel = val;
+        }
+        break;
+    case VIRTIO_PCI_COMMON_Q_SIZE:
+        proxy->vqs[vdev->queue_sel].num = val;
+        break;
+    case VIRTIO_PCI_COMMON_Q_MSIX:
+        msix_vector_unuse(&proxy->pci_dev,
+                          virtio_queue_vector(vdev, vdev->queue_sel));
+        /* Make it possible for guest to discover an error took place. */
+        if (msix_vector_use(&proxy->pci_dev, val) < 0) {
+            val = VIRTIO_NO_VECTOR;
+        }
+        virtio_queue_set_vector(vdev, vdev->queue_sel, val);
+        break;
+    case VIRTIO_PCI_COMMON_Q_ENABLE:
+        /* TODO: need a way to put num back on reset. */
+        virtio_queue_set_num(vdev, vdev->queue_sel,
+                             proxy->vqs[vdev->queue_sel].num);
+        virtio_queue_set_rings(vdev, vdev->queue_sel,
+                       ((uint64_t)proxy->vqs[vdev->queue_sel].desc[1]) << 32 |
+                       proxy->vqs[vdev->queue_sel].desc[0],
+                       ((uint64_t)proxy->vqs[vdev->queue_sel].avail[1]) << 32 |
+                       proxy->vqs[vdev->queue_sel].avail[0],
+                       ((uint64_t)proxy->vqs[vdev->queue_sel].used[1]) << 32 |
+                       proxy->vqs[vdev->queue_sel].used[0]);
+        proxy->vqs[vdev->queue_sel].enabled = 1;
+        break;
+    case VIRTIO_PCI_COMMON_Q_DESCLO:
+        proxy->vqs[vdev->queue_sel].desc[0] = val;
+        break;
+    case VIRTIO_PCI_COMMON_Q_DESCHI:
+        proxy->vqs[vdev->queue_sel].desc[1] = val;
+        break;
+    case VIRTIO_PCI_COMMON_Q_AVAILLO:
+        proxy->vqs[vdev->queue_sel].avail[0] = val;
+        break;
+    case VIRTIO_PCI_COMMON_Q_AVAILHI:
+        proxy->vqs[vdev->queue_sel].avail[1] = val;
+        break;
+    case VIRTIO_PCI_COMMON_Q_USEDLO:
+        proxy->vqs[vdev->queue_sel].used[0] = val;
+        break;
+    case VIRTIO_PCI_COMMON_Q_USEDHI:
+        proxy->vqs[vdev->queue_sel].used[1] = val;
+        break;
+    default:
+        break;
+    }
+}
+
+
+static uint64_t virtio_pci_notify_read(void *opaque, hwaddr addr,
+                                       unsigned size)
+{
+    return 0;
+}
+
+static void virtio_pci_notify_write(void *opaque, hwaddr addr,
+                                    uint64_t val, unsigned size)
+{
+    VirtIODevice *vdev = opaque;
+    unsigned queue = addr / QEMU_VIRTIO_PCI_QUEUE_MEM_MULT;
+
+    if (queue < VIRTIO_QUEUE_MAX) {
+        virtio_queue_notify(vdev, queue);
+    }
+}
+
+static void virtio_pci_notify_write_pio(void *opaque, hwaddr addr,
+                                        uint64_t val, unsigned size)
+{
+    VirtIODevice *vdev = opaque;
+    unsigned queue = val;
+
+    if (queue < VIRTIO_QUEUE_MAX) {
+        virtio_queue_notify(vdev, queue);
+    }
+}
+
+static uint64_t virtio_pci_isr_read(void *opaque, hwaddr addr,
+                                    unsigned size)
+{
+    VirtIOPCIProxy *proxy = opaque;
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
+    uint64_t val = vdev->isr;
+
+    vdev->isr = 0;
+    pci_irq_deassert(&proxy->pci_dev);
+
+    return val;
+}
+
+static void virtio_pci_isr_write(void *opaque, hwaddr addr,
+                                 uint64_t val, unsigned size)
+{
+}
+
+static uint64_t virtio_pci_device_read(void *opaque, hwaddr addr,
+                                       unsigned size)
+{
+    VirtIODevice *vdev = opaque;
+    uint64_t val = 0;
+
+    switch (size) {
+    case 1:
+        val = virtio_config_modern_readb(vdev, addr);
+        break;
+    case 2:
+        val = virtio_config_modern_readw(vdev, addr);
+        break;
+    case 4:
+        val = virtio_config_modern_readl(vdev, addr);
+        break;
+    }
+    return val;
+}
+
+static void virtio_pci_device_write(void *opaque, hwaddr addr,
+                                    uint64_t val, unsigned size)
+{
+    VirtIODevice *vdev = opaque;
+    switch (size) {
+    case 1:
+        virtio_config_modern_writeb(vdev, addr, val);
+        break;
+    case 2:
+        virtio_config_modern_writew(vdev, addr, val);
+        break;
+    case 4:
+        virtio_config_modern_writel(vdev, addr, val);
+        break;
+    }
+}
+
+static void virtio_pci_modern_regions_init(VirtIOPCIProxy *proxy)
+{
+    static const MemoryRegionOps common_ops = {
+        .read = virtio_pci_common_read,
+        .write = virtio_pci_common_write,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+    static const MemoryRegionOps isr_ops = {
+        .read = virtio_pci_isr_read,
+        .write = virtio_pci_isr_write,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+    static const MemoryRegionOps device_ops = {
+        .read = virtio_pci_device_read,
+        .write = virtio_pci_device_write,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+    static const MemoryRegionOps notify_ops = {
+        .read = virtio_pci_notify_read,
+        .write = virtio_pci_notify_write,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+    static const MemoryRegionOps notify_pio_ops = {
+        .read = virtio_pci_notify_read,
+        .write = virtio_pci_notify_write_pio,
+        .impl = {
+            .min_access_size = 1,
+            .max_access_size = 4,
+        },
+        .endianness = DEVICE_LITTLE_ENDIAN,
+    };
+
+
+    memory_region_init_io(&proxy->common.mr, OBJECT(proxy),
+                          &common_ops,
+                          proxy,
+                          "virtio-pci-common",
+                          proxy->common.size);
+
+    memory_region_init_io(&proxy->isr.mr, OBJECT(proxy),
+                          &isr_ops,
+                          proxy,
+                          "virtio-pci-isr",
+                          proxy->isr.size);
+
+    memory_region_init_io(&proxy->device.mr, OBJECT(proxy),
+                          &device_ops,
+                          virtio_bus_get_device(&proxy->bus),
+                          "virtio-pci-device",
+                          proxy->device.size);
+
+    memory_region_init_io(&proxy->notify.mr, OBJECT(proxy),
+                          &notify_ops,
+                          virtio_bus_get_device(&proxy->bus),
+                          "virtio-pci-notify",
+                          proxy->notify.size);
+
+    memory_region_init_io(&proxy->notify_pio.mr, OBJECT(proxy),
+                          &notify_pio_ops,
+                          virtio_bus_get_device(&proxy->bus),
+                          "virtio-pci-notify-pio",
+                          proxy->notify.size);
+}
+
+static void virtio_pci_modern_region_map(VirtIOPCIProxy *proxy,
+                                         VirtIOPCIRegion *region,
+                                         struct virtio_pci_cap *cap,
+                                         MemoryRegion *mr,
+                                         uint8_t bar)
+{
+    memory_region_add_subregion(mr, region->offset, &region->mr);
+
+    cap->cfg_type = region->type;
+    cap->bar = bar;
+    cap->offset = cpu_to_le32(region->offset);
+    cap->length = cpu_to_le32(region->size);
+    virtio_pci_add_mem_cap(proxy, cap);
+
+}
+
+static void virtio_pci_modern_mem_region_map(VirtIOPCIProxy *proxy,
+                                             VirtIOPCIRegion *region,
+                                             struct virtio_pci_cap *cap)
+{
+    virtio_pci_modern_region_map(proxy, region, cap,
+                                 &proxy->modern_bar, proxy->modern_mem_bar);
+}
+
+static void virtio_pci_modern_io_region_map(VirtIOPCIProxy *proxy,
+                                            VirtIOPCIRegion *region,
+                                            struct virtio_pci_cap *cap)
+{
+    virtio_pci_modern_region_map(proxy, region, cap,
+                                 &proxy->io_bar, proxy->modern_io_bar);
+}
+
+static void virtio_pci_modern_mem_region_unmap(VirtIOPCIProxy *proxy,
+                                               VirtIOPCIRegion *region)
+{
+    memory_region_del_subregion(&proxy->modern_bar,
+                                &region->mr);
+}
+
+static void virtio_pci_modern_io_region_unmap(VirtIOPCIProxy *proxy,
+                                              VirtIOPCIRegion *region)
+{
+    memory_region_del_subregion(&proxy->io_bar,
+                                &region->mr);
+}
+
 /* This is called by virtio-bus just after the device is plugged. */
-static void virtio_pci_device_plugged(DeviceState *d)
+static void virtio_pci_device_plugged(DeviceState *d, Error **errp)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
     VirtioBusState *bus = &proxy->bus;
+    bool legacy = virtio_pci_legacy(proxy);
+    bool modern = virtio_pci_modern(proxy);
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
     uint8_t *config;
     uint32_t size;
+    VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
 
     config = proxy->pci_dev.config;
     if (proxy->class_code) {
         pci_config_set_class(config, proxy->class_code);
     }
-    pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
-                 pci_get_word(config + PCI_VENDOR_ID));
-    pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
+
+    if (legacy) {
+        /* legacy and transitional */
+        pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID,
+                     pci_get_word(config + PCI_VENDOR_ID));
+        pci_set_word(config + PCI_SUBSYSTEM_ID, virtio_bus_get_vdev_id(bus));
+    } else {
+        /* pure virtio-1.0 */
+        pci_set_word(config + PCI_VENDOR_ID,
+                     PCI_VENDOR_ID_REDHAT_QUMRANET);
+        pci_set_word(config + PCI_DEVICE_ID,
+                     0x1040 + virtio_bus_get_vdev_id(bus));
+        pci_config_set_revision(config, 1);
+    }
     config[PCI_INTERRUPT_PIN] = 1;
 
-    if (proxy->nvectors &&
-        msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors, 1)) {
-        error_report("unable to init msix vectors to %" PRIu32,
-                     proxy->nvectors);
-        proxy->nvectors = 0;
+
+    if (modern) {
+        struct virtio_pci_cap cap = {
+            .cap_len = sizeof cap,
+        };
+        struct virtio_pci_notify_cap notify = {
+            .cap.cap_len = sizeof notify,
+            .notify_off_multiplier =
+                cpu_to_le32(QEMU_VIRTIO_PCI_QUEUE_MEM_MULT),
+        };
+        struct virtio_pci_cfg_cap cfg = {
+            .cap.cap_len = sizeof cfg,
+            .cap.cfg_type = VIRTIO_PCI_CAP_PCI_CFG,
+        };
+        struct virtio_pci_notify_cap notify_pio = {
+            .cap.cap_len = sizeof notify,
+            .notify_off_multiplier = cpu_to_le32(0x0),
+        };
+
+        struct virtio_pci_cfg_cap *cfg_mask;
+
+        virtio_add_feature(&vdev->host_features, VIRTIO_F_VERSION_1);
+        virtio_pci_modern_regions_init(proxy);
+
+        virtio_pci_modern_mem_region_map(proxy, &proxy->common, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->isr, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->device, &cap);
+        virtio_pci_modern_mem_region_map(proxy, &proxy->notify, &notify.cap);
+
+        if (modern_pio) {
+            memory_region_init(&proxy->io_bar, OBJECT(proxy),
+                               "virtio-pci-io", 0x4);
+
+            pci_register_bar(&proxy->pci_dev, proxy->modern_io_bar,
+                             PCI_BASE_ADDRESS_SPACE_IO, &proxy->io_bar);
+
+            virtio_pci_modern_io_region_map(proxy, &proxy->notify_pio,
+                                            &notify_pio.cap);
+        }
+
+        pci_register_bar(&proxy->pci_dev, proxy->modern_mem_bar,
+                         PCI_BASE_ADDRESS_SPACE_MEMORY |
+                         PCI_BASE_ADDRESS_MEM_PREFETCH |
+                         PCI_BASE_ADDRESS_MEM_TYPE_64,
+                         &proxy->modern_bar);
+
+        proxy->config_cap = virtio_pci_add_mem_cap(proxy, &cfg.cap);
+        cfg_mask = (void *)(proxy->pci_dev.wmask + proxy->config_cap);
+        pci_set_byte(&cfg_mask->cap.bar, ~0x0);
+        pci_set_long((uint8_t *)&cfg_mask->cap.offset, ~0x0);
+        pci_set_long((uint8_t *)&cfg_mask->cap.length, ~0x0);
+        pci_set_long(cfg_mask->pci_cfg_data, ~0x0);
+    }
+
+    if (proxy->nvectors) {
+        int err = msix_init_exclusive_bar(&proxy->pci_dev, proxy->nvectors,
+                                          proxy->msix_bar);
+        if (err) {
+            /* Notice when a system that supports MSIx can't initialize it.  */
+            if (err != -ENOTSUP) {
+                error_report("unable to init msix vectors to %" PRIu32,
+                             proxy->nvectors);
+            }
+            proxy->nvectors = 0;
+        }
     }
 
     proxy->pci_dev.config_write = virtio_write_config;
+    proxy->pci_dev.config_read = virtio_read_config;
 
-    size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
-         + virtio_bus_get_vdev_config_len(bus);
-    if (size & (size - 1)) {
-        size = 1 << qemu_fls(size);
+    if (legacy) {
+        size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev)
+            + virtio_bus_get_vdev_config_len(bus);
+        size = pow2ceil(size);
+
+        memory_region_init_io(&proxy->bar, OBJECT(proxy),
+                              &virtio_pci_config_ops,
+                              proxy, "virtio-pci", size);
+
+        pci_register_bar(&proxy->pci_dev, proxy->legacy_io_bar,
+                         PCI_BASE_ADDRESS_SPACE_IO, &proxy->bar);
     }
 
-    memory_region_init_io(&proxy->bar, OBJECT(proxy), &virtio_pci_config_ops,
-                          proxy, "virtio-pci", size);
-    pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO,
-                     &proxy->bar);
-
     if (!kvm_has_many_ioeventfds()) {
         proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD;
     }
 
-    proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY;
-    proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE;
-    proxy->host_features = virtio_bus_get_vdev_features(bus,
-                                                      proxy->host_features);
+    virtio_add_feature(&vdev->host_features, VIRTIO_F_BAD_FEATURE);
 }
 
 static void virtio_pci_device_unplugged(DeviceState *d)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(d);
+    bool modern = virtio_pci_modern(proxy);
+    bool modern_pio = proxy->flags & VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY;
 
     virtio_pci_stop_ioeventfd(proxy);
+
+    if (modern) {
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->common);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->isr);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->device);
+        virtio_pci_modern_mem_region_unmap(proxy, &proxy->notify);
+        if (modern_pio) {
+            virtio_pci_modern_io_region_unmap(proxy, &proxy->notify_pio);
+        }
+    }
 }
 
-static int virtio_pci_init(PCIDevice *pci_dev)
+static void virtio_pci_realize(PCIDevice *pci_dev, Error **errp)
 {
-    VirtIOPCIProxy *dev = VIRTIO_PCI(pci_dev);
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
     VirtioPCIClass *k = VIRTIO_PCI_GET_CLASS(pci_dev);
-    virtio_pci_bus_new(&dev->bus, sizeof(dev->bus), dev);
-    if (k->init != NULL) {
-        return k->init(dev);
+    bool pcie_port = pci_bus_is_express(pci_dev->bus) &&
+                     !pci_bus_is_root(pci_dev->bus);
+
+    /*
+     * virtio pci bar layout used by default.
+     * subclasses can re-arrange things if needed.
+     *
+     *   region 0   --  virtio legacy io bar
+     *   region 1   --  msi-x bar
+     *   region 4+5 --  virtio modern memory (64bit) bar
+     *
+     */
+    proxy->legacy_io_bar  = 0;
+    proxy->msix_bar       = 1;
+    proxy->modern_io_bar  = 2;
+    proxy->modern_mem_bar = 4;
+
+    proxy->common.offset = 0x0;
+    proxy->common.size = 0x1000;
+    proxy->common.type = VIRTIO_PCI_CAP_COMMON_CFG;
+
+    proxy->isr.offset = 0x1000;
+    proxy->isr.size = 0x1000;
+    proxy->isr.type = VIRTIO_PCI_CAP_ISR_CFG;
+
+    proxy->device.offset = 0x2000;
+    proxy->device.size = 0x1000;
+    proxy->device.type = VIRTIO_PCI_CAP_DEVICE_CFG;
+
+    proxy->notify.offset = 0x3000;
+    proxy->notify.size =
+        QEMU_VIRTIO_PCI_QUEUE_MEM_MULT * VIRTIO_QUEUE_MAX;
+    proxy->notify.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
+
+    proxy->notify_pio.offset = 0x0;
+    proxy->notify_pio.size = 0x4;
+    proxy->notify_pio.type = VIRTIO_PCI_CAP_NOTIFY_CFG;
+
+    /* subclasses can enforce modern, so do this unconditionally */
+    memory_region_init(&proxy->modern_bar, OBJECT(proxy), "virtio-pci",
+                       2 * QEMU_VIRTIO_PCI_QUEUE_MEM_MULT *
+                       VIRTIO_QUEUE_MAX);
+
+    memory_region_init_alias(&proxy->modern_cfg,
+                             OBJECT(proxy),
+                             "virtio-pci-cfg",
+                             &proxy->modern_bar,
+                             0,
+                             memory_region_size(&proxy->modern_bar));
+
+    address_space_init(&proxy->modern_as, &proxy->modern_cfg, "virtio-pci-cfg-as");
+
+    if (proxy->disable_legacy == ON_OFF_AUTO_AUTO) {
+        proxy->disable_legacy = pcie_port ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
     }
-    return 0;
+
+    if (pcie_port && pci_is_express(pci_dev)) {
+        int pos;
+
+        pos = pcie_endpoint_cap_init(pci_dev, 0);
+        assert(pos > 0);
+
+        pos = pci_add_capability(pci_dev, PCI_CAP_ID_PM, 0, PCI_PM_SIZEOF);
+        assert(pos > 0);
+
+        /*
+         * Indicates that this function complies with revision 1.2 of the
+         * PCI Power Management Interface Specification.
+         */
+        pci_set_word(pci_dev->config + pos + PCI_PM_PMC, 0x3);
+    } else {
+        /*
+         * make future invocations of pci_is_express() return false
+         * and pci_config_size() return PCI_CONFIG_SPACE_SIZE.
+         */
+        pci_dev->cap_present &= ~QEMU_PCI_CAP_EXPRESS;
+    }
+
+    virtio_pci_bus_new(&proxy->bus, sizeof(proxy->bus), proxy);
+    if (k->realize) {
+        k->realize(proxy, errp);
+    }
 }
 
 static void virtio_pci_exit(PCIDevice *pci_dev)
 {
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(pci_dev);
+
     msix_uninit_exclusive_bar(pci_dev);
+    address_space_destroy(&proxy->modern_as);
 }
 
 static void virtio_pci_reset(DeviceState *qdev)
 {
     VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
     VirtioBusState *bus = VIRTIO_BUS(&proxy->bus);
+    int i;
+
     virtio_pci_stop_ioeventfd(proxy);
     virtio_bus_reset(bus);
     msix_unuse_all_vectors(&proxy->pci_dev);
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        proxy->vqs[i].enabled = 0;
+    }
 }
 
 static Property virtio_pci_properties[] = {
     DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
                     VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
-    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
+    DEFINE_PROP_ON_OFF_AUTO("disable-legacy", VirtIOPCIProxy, disable_legacy,
+                            ON_OFF_AUTO_AUTO),
+    DEFINE_PROP_BOOL("disable-modern", VirtIOPCIProxy, disable_modern, false),
+    DEFINE_PROP_BIT("migrate-extra", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT, true),
+    DEFINE_PROP_BIT("modern-pio-notify", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT, false),
+    DEFINE_PROP_BIT("x-disable-pcie", VirtIOPCIProxy, flags,
+                    VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void virtio_pci_dc_realize(DeviceState *qdev, Error **errp)
+{
+    VirtioPCIClass *vpciklass = VIRTIO_PCI_GET_CLASS(qdev);
+    VirtIOPCIProxy *proxy = VIRTIO_PCI(qdev);
+    PCIDevice *pci_dev = &proxy->pci_dev;
+
+    if (!(proxy->flags & VIRTIO_PCI_FLAG_DISABLE_PCIE) &&
+        virtio_pci_modern(proxy)) {
+        pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
+    }
+
+    vpciklass->parent_dc_realize(qdev, errp);
+}
+
 static void virtio_pci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    VirtioPCIClass *vpciklass = VIRTIO_PCI_CLASS(klass);
 
     dc->props = virtio_pci_properties;
-    k->init = virtio_pci_init;
+    k->realize = virtio_pci_realize;
     k->exit = virtio_pci_exit;
     k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     k->revision = VIRTIO_PCI_ABI_VERSION;
     k->class_id = PCI_CLASS_OTHERS;
+    vpciklass->parent_dc_realize = dc->realize;
+    dc->realize = virtio_pci_dc_realize;
     dc->reset = virtio_pci_reset;
 }
 
@@ -1076,15 +1887,13 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static int virtio_blk_pci_init(VirtIOPCIProxy *vpci_dev)
+static void virtio_blk_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIOBlkPCI *dev = VIRTIO_BLK_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
+
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_blk_pci_class_init(ObjectClass *klass, void *data)
@@ -1095,7 +1904,7 @@
 
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->props = virtio_blk_pci_properties;
-    k->init = virtio_blk_pci_init;
+    k->realize = virtio_blk_pci_realize;
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_BLOCK;
     pcidev_k->revision = VIRTIO_PCI_ABI_VERSION;
@@ -1129,11 +1938,10 @@
                     VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true),
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors,
                        DEV_NVECTORS_UNSPECIFIED),
-    DEFINE_VIRTIO_SCSI_FEATURES(VirtIOPCIProxy, host_features),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static int virtio_scsi_pci_init_pci(VirtIOPCIProxy *vpci_dev)
+static void virtio_scsi_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIOSCSIPCI *dev = VIRTIO_SCSI_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
@@ -1156,10 +1964,7 @@
     }
 
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_scsi_pci_class_init(ObjectClass *klass, void *data)
@@ -1167,7 +1972,8 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
     PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
-    k->init = virtio_scsi_pci_init_pci;
+
+    k->realize = virtio_scsi_pci_realize;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->props = virtio_scsi_pci_properties;
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
@@ -1203,7 +2009,7 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static int vhost_scsi_pci_init_pci(VirtIOPCIProxy *vpci_dev)
+static void vhost_scsi_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VHostSCSIPCI *dev = VHOST_SCSI_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
@@ -1214,10 +2020,7 @@
     }
 
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void vhost_scsi_pci_class_init(ObjectClass *klass, void *data)
@@ -1225,7 +2028,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
     PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
-    k->init = vhost_scsi_pci_init_pci;
+    k->realize = vhost_scsi_pci_realize;
     set_bit(DEVICE_CATEGORY_STORAGE, dc->categories);
     dc->props = vhost_scsi_pci_properties;
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
@@ -1240,6 +2043,8 @@
 
     virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
                                 TYPE_VHOST_SCSI);
+    object_property_add_alias(obj, "bootindex", OBJECT(&dev->vdev),
+                              "bootindex", &error_abort);
 }
 
 static const TypeInfo vhost_scsi_pci_info = {
@@ -1253,38 +2058,12 @@
 
 /* virtio-balloon-pci */
 
-static void balloon_pci_stats_get_all(Object *obj, struct Visitor *v,
-                                      void *opaque, const char *name,
-                                      Error **errp)
-{
-    VirtIOBalloonPCI *dev = opaque;
-    object_property_get(OBJECT(&dev->vdev), v, "guest-stats", errp);
-}
-
-static void balloon_pci_stats_get_poll_interval(Object *obj, struct Visitor *v,
-                                                void *opaque, const char *name,
-                                                Error **errp)
-{
-    VirtIOBalloonPCI *dev = opaque;
-    object_property_get(OBJECT(&dev->vdev), v, "guest-stats-polling-interval",
-                        errp);
-}
-
-static void balloon_pci_stats_set_poll_interval(Object *obj, struct Visitor *v,
-                                                void *opaque, const char *name,
-                                                Error **errp)
-{
-    VirtIOBalloonPCI *dev = opaque;
-    object_property_set(OBJECT(&dev->vdev), v, "guest-stats-polling-interval",
-                        errp);
-}
-
 static Property virtio_balloon_pci_properties[] = {
     DEFINE_PROP_UINT32("class", VirtIOPCIProxy, class_code, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static int virtio_balloon_pci_init(VirtIOPCIProxy *vpci_dev)
+static void virtio_balloon_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIOBalloonPCI *dev = VIRTIO_BALLOON_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
@@ -1295,10 +2074,7 @@
     }
 
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_balloon_pci_class_init(ObjectClass *klass, void *data)
@@ -1306,7 +2082,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
     PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
-    k->init = virtio_balloon_pci_init;
+    k->realize = virtio_balloon_pci_realize;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     dc->props = virtio_balloon_pci_properties;
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
@@ -1318,17 +2094,14 @@
 static void virtio_balloon_pci_instance_init(Object *obj)
 {
     VirtIOBalloonPCI *dev = VIRTIO_BALLOON_PCI(obj);
-    object_initialize(&dev->vdev, sizeof(dev->vdev), TYPE_VIRTIO_BALLOON);
-    object_property_add_child(obj, "virtio-backend", OBJECT(&dev->vdev), NULL);
-    object_unref(OBJECT(&dev->vdev));
-    object_property_add(obj, "guest-stats", "guest statistics",
-                        balloon_pci_stats_get_all, NULL, NULL, dev,
-                        NULL);
 
-    object_property_add(obj, "guest-stats-polling-interval", "int",
-                        balloon_pci_stats_get_poll_interval,
-                        balloon_pci_stats_set_poll_interval,
-                        NULL, dev, NULL);
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_BALLOON);
+    object_property_add_alias(obj, "guest-stats", OBJECT(&dev->vdev),
+                                  "guest-stats", &error_abort);
+    object_property_add_alias(obj, "guest-stats-polling-interval",
+                              OBJECT(&dev->vdev),
+                              "guest-stats-polling-interval", &error_abort);
 }
 
 static const TypeInfo virtio_balloon_pci_info = {
@@ -1341,7 +2114,7 @@
 
 /* virtio-serial-pci */
 
-static int virtio_serial_pci_init(VirtIOPCIProxy *vpci_dev)
+static void virtio_serial_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIOSerialPCI *dev = VIRTIO_SERIAL_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
@@ -1371,10 +2144,7 @@
     }
 
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static Property virtio_serial_pci_properties[] = {
@@ -1390,7 +2160,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
     PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
-    k->init = virtio_serial_pci_init;
+    k->realize = virtio_serial_pci_realize;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
     dc->props = virtio_serial_pci_properties;
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
@@ -1421,24 +2191,19 @@
     DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags,
                     VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false),
     DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3),
-    DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features),
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static int virtio_net_pci_init(VirtIOPCIProxy *vpci_dev)
+static void virtio_net_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     DeviceState *qdev = DEVICE(vpci_dev);
     VirtIONetPCI *dev = VIRTIO_NET_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&dev->vdev);
 
-    virtio_net_set_config_size(&dev->vdev, vpci_dev->host_features);
     virtio_net_set_netclient_name(&dev->vdev, qdev->id,
                                   object_get_typename(OBJECT(qdev)));
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
-    }
-    return 0;
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
 }
 
 static void virtio_net_pci_class_init(ObjectClass *klass, void *data)
@@ -1454,7 +2219,7 @@
     k->class_id = PCI_CLASS_NETWORK_ETHERNET;
     set_bit(DEVICE_CATEGORY_NETWORK, dc->categories);
     dc->props = virtio_net_properties;
-    vpciklass->init = virtio_net_pci_init;
+    vpciklass->realize = virtio_net_pci_realize;
 }
 
 static void virtio_net_pci_instance_init(Object *obj)
@@ -1477,25 +2242,22 @@
 
 /* virtio-rng-pci */
 
-static Property virtio_rng_pci_properties[] = {
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static int virtio_rng_pci_init(VirtIOPCIProxy *vpci_dev)
+static void virtio_rng_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
 {
     VirtIORngPCI *vrng = VIRTIO_RNG_PCI(vpci_dev);
     DeviceState *vdev = DEVICE(&vrng->vdev);
+    Error *err = NULL;
 
     qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
-    if (qdev_init(vdev) < 0) {
-        return -1;
+    object_property_set_bool(OBJECT(vdev), true, "realized", &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
 
     object_property_set_link(OBJECT(vrng),
                              OBJECT(vrng->vdev.conf.rng), "rng",
                              NULL);
-
-    return 0;
 }
 
 static void virtio_rng_pci_class_init(ObjectClass *klass, void *data)
@@ -1504,9 +2266,8 @@
     VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
     PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
 
-    k->init = virtio_rng_pci_init;
+    k->realize = virtio_rng_pci_realize;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
-    dc->props = virtio_rng_pci_properties;
 
     pcidev_k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
     pcidev_k->device_id = PCI_DEVICE_ID_VIRTIO_RNG;
@@ -1532,6 +2293,130 @@
     .class_init    = virtio_rng_pci_class_init,
 };
 
+/* virtio-input-pci */
+
+static Property virtio_input_pci_properties[] = {
+    DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_input_pci_realize(VirtIOPCIProxy *vpci_dev, Error **errp)
+{
+    VirtIOInputPCI *vinput = VIRTIO_INPUT_PCI(vpci_dev);
+    DeviceState *vdev = DEVICE(&vinput->vdev);
+
+    qdev_set_parent_bus(vdev, BUS(&vpci_dev->bus));
+    virtio_pci_force_virtio_1(vpci_dev);
+    object_property_set_bool(OBJECT(vdev), true, "realized", errp);
+}
+
+static void virtio_input_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    VirtioPCIClass *k = VIRTIO_PCI_CLASS(klass);
+    PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
+
+    dc->props = virtio_input_pci_properties;
+    k->realize = virtio_input_pci_realize;
+    set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
+
+    pcidev_k->class_id = PCI_CLASS_INPUT_OTHER;
+}
+
+static void virtio_input_hid_kbd_pci_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
+
+    pcidev_k->class_id = PCI_CLASS_INPUT_KEYBOARD;
+}
+
+static void virtio_input_hid_mouse_pci_class_init(ObjectClass *klass,
+                                                  void *data)
+{
+    PCIDeviceClass *pcidev_k = PCI_DEVICE_CLASS(klass);
+
+    pcidev_k->class_id = PCI_CLASS_INPUT_MOUSE;
+}
+
+static void virtio_keyboard_initfn(Object *obj)
+{
+    VirtIOInputHIDPCI *dev = VIRTIO_INPUT_HID_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_KEYBOARD);
+}
+
+static void virtio_mouse_initfn(Object *obj)
+{
+    VirtIOInputHIDPCI *dev = VIRTIO_INPUT_HID_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_MOUSE);
+}
+
+static void virtio_tablet_initfn(Object *obj)
+{
+    VirtIOInputHIDPCI *dev = VIRTIO_INPUT_HID_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_TABLET);
+}
+
+static const TypeInfo virtio_input_pci_info = {
+    .name          = TYPE_VIRTIO_INPUT_PCI,
+    .parent        = TYPE_VIRTIO_PCI,
+    .instance_size = sizeof(VirtIOInputPCI),
+    .class_init    = virtio_input_pci_class_init,
+    .abstract      = true,
+};
+
+static const TypeInfo virtio_input_hid_pci_info = {
+    .name          = TYPE_VIRTIO_INPUT_HID_PCI,
+    .parent        = TYPE_VIRTIO_INPUT_PCI,
+    .instance_size = sizeof(VirtIOInputHIDPCI),
+    .abstract      = true,
+};
+
+static const TypeInfo virtio_keyboard_pci_info = {
+    .name          = TYPE_VIRTIO_KEYBOARD_PCI,
+    .parent        = TYPE_VIRTIO_INPUT_HID_PCI,
+    .class_init    = virtio_input_hid_kbd_pci_class_init,
+    .instance_size = sizeof(VirtIOInputHIDPCI),
+    .instance_init = virtio_keyboard_initfn,
+};
+
+static const TypeInfo virtio_mouse_pci_info = {
+    .name          = TYPE_VIRTIO_MOUSE_PCI,
+    .parent        = TYPE_VIRTIO_INPUT_HID_PCI,
+    .class_init    = virtio_input_hid_mouse_pci_class_init,
+    .instance_size = sizeof(VirtIOInputHIDPCI),
+    .instance_init = virtio_mouse_initfn,
+};
+
+static const TypeInfo virtio_tablet_pci_info = {
+    .name          = TYPE_VIRTIO_TABLET_PCI,
+    .parent        = TYPE_VIRTIO_INPUT_HID_PCI,
+    .instance_size = sizeof(VirtIOInputHIDPCI),
+    .instance_init = virtio_tablet_initfn,
+};
+
+#ifdef CONFIG_LINUX
+static void virtio_host_initfn(Object *obj)
+{
+    VirtIOInputHostPCI *dev = VIRTIO_INPUT_HOST_PCI(obj);
+
+    virtio_instance_init_common(obj, &dev->vdev, sizeof(dev->vdev),
+                                TYPE_VIRTIO_INPUT_HOST);
+}
+
+static const TypeInfo virtio_host_pci_info = {
+    .name          = TYPE_VIRTIO_INPUT_HOST_PCI,
+    .parent        = TYPE_VIRTIO_INPUT_PCI,
+    .instance_size = sizeof(VirtIOInputHostPCI),
+    .instance_init = virtio_host_initfn,
+};
+#endif
+
 /* virtio-pci-bus */
 
 static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
@@ -1554,13 +2439,20 @@
     k->load_config = virtio_pci_load_config;
     k->save_queue = virtio_pci_save_queue;
     k->load_queue = virtio_pci_load_queue;
-    k->get_features = virtio_pci_get_features;
+    k->save_extra_state = virtio_pci_save_extra_state;
+    k->load_extra_state = virtio_pci_load_extra_state;
+    k->has_extra_state = virtio_pci_has_extra_state;
     k->query_guest_notifiers = virtio_pci_query_guest_notifiers;
-    k->set_host_notifier = virtio_pci_set_host_notifier;
     k->set_guest_notifiers = virtio_pci_set_guest_notifiers;
     k->vmstate_change = virtio_pci_vmstate_change;
     k->device_plugged = virtio_pci_device_plugged;
     k->device_unplugged = virtio_pci_device_unplugged;
+    k->query_nvectors = virtio_pci_query_nvectors;
+    k->ioeventfd_started = virtio_pci_ioeventfd_started;
+    k->ioeventfd_set_started = virtio_pci_ioeventfd_set_started;
+    k->ioeventfd_disabled = virtio_pci_ioeventfd_disabled;
+    k->ioeventfd_set_disabled = virtio_pci_ioeventfd_set_disabled;
+    k->ioeventfd_assign = virtio_pci_ioeventfd_assign;
 }
 
 static const TypeInfo virtio_pci_bus_info = {
@@ -1573,6 +2465,14 @@
 static void virtio_pci_register_types(void)
 {
     type_register_static(&virtio_rng_pci_info);
+    type_register_static(&virtio_input_pci_info);
+    type_register_static(&virtio_input_hid_pci_info);
+    type_register_static(&virtio_keyboard_pci_info);
+    type_register_static(&virtio_mouse_pci_info);
+    type_register_static(&virtio_tablet_pci_info);
+#ifdef CONFIG_LINUX
+    type_register_static(&virtio_host_pci_info);
+#endif
     type_register_static(&virtio_pci_bus_info);
     type_register_static(&virtio_pci_info);
 #ifdef CONFIG_VIRTFS
diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h
index 8873b6d..25fbf8a 100644
--- a/hw/virtio/virtio-pci.h
+++ b/hw/virtio/virtio-pci.h
@@ -23,7 +23,8 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/virtio-balloon.h"
 #include "hw/virtio/virtio-bus.h"
-#include "hw/virtio/virtio-9p.h"
+#include "hw/virtio/virtio-input.h"
+#include "hw/virtio/virtio-gpu.h"
 #ifdef CONFIG_VIRTFS
 #include "hw/9pfs/virtio-9p.h"
 #endif
@@ -39,6 +40,10 @@
 typedef struct VirtIONetPCI VirtIONetPCI;
 typedef struct VHostSCSIPCI VHostSCSIPCI;
 typedef struct VirtIORngPCI VirtIORngPCI;
+typedef struct VirtIOInputPCI VirtIOInputPCI;
+typedef struct VirtIOInputHIDPCI VirtIOInputHIDPCI;
+typedef struct VirtIOInputHostPCI VirtIOInputHostPCI;
+typedef struct VirtIOGPUPCI VirtIOGPUPCI;
 
 /* virtio-pci-bus */
 
@@ -53,16 +58,32 @@
 #define VIRTIO_PCI_BUS_CLASS(klass) \
         OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
 
+enum {
+    VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT,
+    VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT,
+    VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT,
+    VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT,
+    VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT,
+};
+
 /* Need to activate work-arounds for buggy guests at vmstate load. */
-#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT  0
 #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION \
     (1 << VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT)
 
 /* Performance improves when virtqueue kick processing is decoupled from the
  * vcpu thread using ioeventfd for some devices. */
-#define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
 #define VIRTIO_PCI_FLAG_USE_IOEVENTFD   (1 << VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT)
 
+/* virtio version flags */
+#define VIRTIO_PCI_FLAG_DISABLE_PCIE (1 << VIRTIO_PCI_FLAG_DISABLE_PCIE_BIT)
+
+/* migrate extra state */
+#define VIRTIO_PCI_FLAG_MIGRATE_EXTRA (1 << VIRTIO_PCI_FLAG_MIGRATE_EXTRA_BIT)
+
+/* have pio notification for modern device ? */
+#define VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY \
+    (1 << VIRTIO_PCI_FLAG_MODERN_PIO_NOTIFY_BIT)
+
 typedef struct {
     MSIMessage msg;
     int virq;
@@ -82,16 +103,52 @@
 
 typedef struct VirtioPCIClass {
     PCIDeviceClass parent_class;
-    int (*init)(VirtIOPCIProxy *vpci_dev);
+    DeviceRealize parent_dc_realize;
+    void (*realize)(VirtIOPCIProxy *vpci_dev, Error **errp);
 } VirtioPCIClass;
 
+typedef struct VirtIOPCIRegion {
+    MemoryRegion mr;
+    uint32_t offset;
+    uint32_t size;
+    uint32_t type;
+} VirtIOPCIRegion;
+
+typedef struct VirtIOPCIQueue {
+  uint16_t num;
+  bool enabled;
+  uint32_t desc[2];
+  uint32_t avail[2];
+  uint32_t used[2];
+} VirtIOPCIQueue;
+
 struct VirtIOPCIProxy {
     PCIDevice pci_dev;
     MemoryRegion bar;
+    VirtIOPCIRegion common;
+    VirtIOPCIRegion isr;
+    VirtIOPCIRegion device;
+    VirtIOPCIRegion notify;
+    VirtIOPCIRegion notify_pio;
+    MemoryRegion modern_bar;
+    MemoryRegion io_bar;
+    MemoryRegion modern_cfg;
+    AddressSpace modern_as;
+    uint32_t legacy_io_bar;
+    uint32_t msix_bar;
+    uint32_t modern_io_bar;
+    uint32_t modern_mem_bar;
+    int config_cap;
     uint32_t flags;
+    bool disable_modern;
+    OnOffAuto disable_legacy;
     uint32_t class_code;
     uint32_t nvectors;
-    uint32_t host_features;
+    uint32_t dfselect;
+    uint32_t gfselect;
+    uint32_t guest_features[2];
+    VirtIOPCIQueue vqs[VIRTIO_QUEUE_MAX];
+
     bool ioeventfd_disabled;
     bool ioeventfd_started;
     VirtIOIRQFD *vector_irqfd;
@@ -99,6 +156,21 @@
     VirtioBusState bus;
 };
 
+static inline bool virtio_pci_modern(VirtIOPCIProxy *proxy)
+{
+    return !proxy->disable_modern;
+}
+
+static inline bool virtio_pci_legacy(VirtIOPCIProxy *proxy)
+{
+    return proxy->disable_legacy == ON_OFF_AUTO_OFF;
+}
+
+static inline void virtio_pci_force_virtio_1(VirtIOPCIProxy *proxy)
+{
+    proxy->disable_modern = false;
+    proxy->disable_legacy = ON_OFF_AUTO_ON;
+}
 
 /*
  * virtio-scsi-pci: This extends VirtioPCIProxy.
@@ -186,7 +258,7 @@
 
 typedef struct V9fsPCIState {
     VirtIOPCIProxy parent_obj;
-    V9fsState vdev;
+    V9fsVirtioState vdev;
 } V9fsPCIState;
 
 #endif
@@ -203,6 +275,55 @@
     VirtIORNG vdev;
 };
 
+/*
+ * virtio-input-pci: This extends VirtioPCIProxy.
+ */
+#define TYPE_VIRTIO_INPUT_PCI "virtio-input-pci"
+#define VIRTIO_INPUT_PCI(obj) \
+        OBJECT_CHECK(VirtIOInputPCI, (obj), TYPE_VIRTIO_INPUT_PCI)
+
+struct VirtIOInputPCI {
+    VirtIOPCIProxy parent_obj;
+    VirtIOInput vdev;
+};
+
+#define TYPE_VIRTIO_INPUT_HID_PCI "virtio-input-hid-pci"
+#define TYPE_VIRTIO_KEYBOARD_PCI  "virtio-keyboard-pci"
+#define TYPE_VIRTIO_MOUSE_PCI     "virtio-mouse-pci"
+#define TYPE_VIRTIO_TABLET_PCI    "virtio-tablet-pci"
+#define VIRTIO_INPUT_HID_PCI(obj) \
+        OBJECT_CHECK(VirtIOInputHIDPCI, (obj), TYPE_VIRTIO_INPUT_HID_PCI)
+
+struct VirtIOInputHIDPCI {
+    VirtIOPCIProxy parent_obj;
+    VirtIOInputHID vdev;
+};
+
+#ifdef CONFIG_LINUX
+
+#define TYPE_VIRTIO_INPUT_HOST_PCI "virtio-input-host-pci"
+#define VIRTIO_INPUT_HOST_PCI(obj) \
+        OBJECT_CHECK(VirtIOInputHostPCI, (obj), TYPE_VIRTIO_INPUT_HOST_PCI)
+
+struct VirtIOInputHostPCI {
+    VirtIOPCIProxy parent_obj;
+    VirtIOInputHost vdev;
+};
+
+#endif
+
+/*
+ * virtio-gpu-pci: This extends VirtioPCIProxy.
+ */
+#define TYPE_VIRTIO_GPU_PCI "virtio-gpu-pci"
+#define VIRTIO_GPU_PCI(obj) \
+        OBJECT_CHECK(VirtIOGPUPCI, (obj), TYPE_VIRTIO_GPU_PCI)
+
+struct VirtIOGPUPCI {
+    VirtIOPCIProxy parent_obj;
+    VirtIOGPU vdev;
+};
+
 /* Virtio ABI version, if we increment this, we break the guest driver. */
 #define VIRTIO_PCI_ABI_VERSION          0
 
diff --git a/hw/virtio/virtio-rng.c b/hw/virtio/virtio-rng.c
index 473c044..cd8ca10 100644
--- a/hw/virtio/virtio-rng.c
+++ b/hw/virtio/virtio-rng.c
@@ -9,9 +9,10 @@
  * top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
 #include "hw/qdev.h"
-#include "qapi/qmp/qerror.h"
 #include "hw/virtio/virtio.h"
 #include "hw/virtio/virtio-rng.h"
 #include "sysemu/rng.h"
@@ -44,7 +45,7 @@
 {
     VirtIORNG *vrng = opaque;
     VirtIODevice *vdev = VIRTIO_DEVICE(vrng);
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
     size_t len;
     int offset;
 
@@ -56,17 +57,26 @@
 
     offset = 0;
     while (offset < size) {
-        if (!virtqueue_pop(vrng->vq, &elem)) {
+        elem = virtqueue_pop(vrng->vq, sizeof(VirtQueueElement));
+        if (!elem) {
             break;
         }
-        len = iov_from_buf(elem.in_sg, elem.in_num,
+        len = iov_from_buf(elem->in_sg, elem->in_num,
                            0, buf + offset, size - offset);
         offset += len;
 
-        virtqueue_push(vrng->vq, &elem, len);
+        virtqueue_push(vrng->vq, elem, len);
         trace_virtio_rng_pushed(vrng, len);
+        g_free(elem);
     }
     virtio_notify(vdev, vrng->vq);
+
+    if (!virtio_queue_empty(vrng->vq)) {
+        /* If we didn't drain the queue, call virtio_rng_process
+         * to take care of asking for more data as appropriate.
+         */
+        virtio_rng_process(vrng);
+    }
 }
 
 static void virtio_rng_process(VirtIORNG *vrng)
@@ -78,6 +88,12 @@
         return;
     }
 
+    if (vrng->activate_timer) {
+        timer_mod(vrng->rate_limit_timer,
+                  qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vrng->conf.period_ms);
+        vrng->activate_timer = false;
+    }
+
     if (vrng->quota_remaining < 0) {
         quota = 0;
     } else {
@@ -99,27 +115,17 @@
     virtio_rng_process(vrng);
 }
 
-static uint32_t get_features(VirtIODevice *vdev, uint32_t f)
+static uint64_t get_features(VirtIODevice *vdev, uint64_t f, Error **errp)
 {
     return f;
 }
 
-static void virtio_rng_save(QEMUFile *f, void *opaque)
-{
-    VirtIODevice *vdev = opaque;
-
-    virtio_save(vdev, f);
-}
-
-static int virtio_rng_load(QEMUFile *f, void *opaque, int version_id)
+static int virtio_rng_load(QEMUFile *f, void *opaque, size_t size)
 {
     VirtIORNG *vrng = opaque;
     int ret;
 
-    if (version_id != 1) {
-        return -EINVAL;
-    }
-    ret = virtio_load(VIRTIO_DEVICE(vrng), f, version_id);
+    ret = virtio_load(VIRTIO_DEVICE(vrng), f, 1);
     if (ret != 0) {
         return ret;
     }
@@ -139,8 +145,7 @@
 
     vrng->quota_remaining = vrng->conf.max_bytes;
     virtio_rng_process(vrng);
-    timer_mod(vrng->rate_limit_timer,
-                   qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vrng->conf.period_ms);
+    vrng->activate_timer = true;
 }
 
 static void virtio_rng_device_realize(DeviceState *dev, Error **errp)
@@ -149,7 +154,7 @@
     VirtIORNG *vrng = VIRTIO_RNG(dev);
     Error *local_err = NULL;
 
-    if (!vrng->conf.period_ms > 0) {
+    if (vrng->conf.period_ms <= 0) {
         error_setg(errp, "'period' parameter expects a positive integer");
         return;
     }
@@ -196,15 +201,9 @@
 
     vrng->vq = virtio_add_queue(vdev, 8, handle_input);
     vrng->quota_remaining = vrng->conf.max_bytes;
-
     vrng->rate_limit_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
                                                check_rate_limit, vrng);
-
-    timer_mod(vrng->rate_limit_timer,
-                   qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vrng->conf.period_ms);
-
-    register_savevm(dev, "virtio-rng", -1, 1, virtio_rng_save,
-                    virtio_rng_load, vrng);
+    vrng->activate_timer = true;
 }
 
 static void virtio_rng_device_unrealize(DeviceState *dev, Error **errp)
@@ -214,12 +213,19 @@
 
     timer_del(vrng->rate_limit_timer);
     timer_free(vrng->rate_limit_timer);
-    unregister_savevm(dev, "virtio-rng", vrng);
     virtio_cleanup(vdev);
 }
 
+VMSTATE_VIRTIO_DEVICE(rng, 1, virtio_rng_load, virtio_vmstate_save);
+
 static Property virtio_rng_properties[] = {
-    DEFINE_VIRTIO_RNG_PROPERTIES(VirtIORNG, conf),
+    /* Set a default rate limit of 2^47 bytes per minute or roughly 2TB/s.  If
+     * you have an entropy source capable of generating more entropy than this
+     * and you can pass it through via virtio-rng, then hats off to you.  Until
+     * then, this is unlimited for all practical purposes.
+     */
+    DEFINE_PROP_UINT64("max-bytes", VirtIORNG, conf.max_bytes, INT64_MAX),
+    DEFINE_PROP_UINT32("period", VirtIORNG, conf.period_ms, 1 << 16),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -229,6 +235,7 @@
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
 
     dc->props = virtio_rng_properties;
+    dc->vmsd = &vmstate_virtio_rng;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     vdc->realize = virtio_rng_device_realize;
     vdc->unrealize = virtio_rng_device_unrealize;
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index 98fc79c..db90a31 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -11,8 +11,10 @@
  *
  */
 
-#include <inttypes.h>
-
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "trace.h"
 #include "exec/address-spaces.h"
 #include "qemu/error-report.h"
@@ -60,6 +62,7 @@
 typedef struct VRing
 {
     unsigned int num;
+    unsigned int num_default;
     unsigned int align;
     hwaddr desc;
     hwaddr avail;
@@ -69,8 +72,15 @@
 struct VirtQueue
 {
     VRing vring;
-    hwaddr pa;
+
+    /* Next head to pop */
     uint16_t last_avail_idx;
+
+    /* Last avail_idx read from VQ. */
+    uint16_t shadow_avail_idx;
+
+    uint16_t used_idx;
+
     /* Last used index value we have signalled on */
     uint16_t signalled_used;
 
@@ -85,53 +95,39 @@
     int inuse;
 
     uint16_t vector;
-    void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq);
+    VirtIOHandleOutput handle_output;
+    VirtIOHandleOutput handle_aio_output;
+    bool use_aio;
     VirtIODevice *vdev;
     EventNotifier guest_notifier;
     EventNotifier host_notifier;
+    QLIST_ENTRY(VirtQueue) node;
 };
 
 /* virt queue functions */
-static void virtqueue_init(VirtQueue *vq)
+void virtio_queue_update_rings(VirtIODevice *vdev, int n)
 {
-    hwaddr pa = vq->pa;
+    VRing *vring = &vdev->vq[n].vring;
 
-    vq->vring.desc = pa;
-    vq->vring.avail = pa + vq->vring.num * sizeof(VRingDesc);
-    vq->vring.used = vring_align(vq->vring.avail +
-                                 offsetof(VRingAvail, ring[vq->vring.num]),
-                                 vq->vring.align);
+    if (!vring->desc) {
+        /* not yet setup -> nothing to do */
+        return;
+    }
+    vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
+    vring->used = vring_align(vring->avail +
+                              offsetof(VRingAvail, ring[vring->num]),
+                              vring->align);
 }
 
-static inline uint64_t vring_desc_addr(VirtIODevice *vdev, hwaddr desc_pa,
-                                       int i)
+static void vring_desc_read(VirtIODevice *vdev, VRingDesc *desc,
+                            hwaddr desc_pa, int i)
 {
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
-    return virtio_ldq_phys(vdev, pa);
-}
-
-static inline uint32_t vring_desc_len(VirtIODevice *vdev, hwaddr desc_pa, int i)
-{
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
-    return virtio_ldl_phys(vdev, pa);
-}
-
-static inline uint16_t vring_desc_flags(VirtIODevice *vdev, hwaddr desc_pa,
-                                        int i)
-{
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
-    return virtio_lduw_phys(vdev, pa);
-}
-
-static inline uint16_t vring_desc_next(VirtIODevice *vdev, hwaddr desc_pa,
-                                       int i)
-{
-    hwaddr pa;
-    pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
-    return virtio_lduw_phys(vdev, pa);
+    address_space_read(&address_space_memory, desc_pa + i * sizeof(VRingDesc),
+                       MEMTXATTRS_UNSPECIFIED, (void *)desc, sizeof(VRingDesc));
+    virtio_tswap64s(vdev, &desc->addr);
+    virtio_tswap32s(vdev, &desc->len);
+    virtio_tswap16s(vdev, &desc->flags);
+    virtio_tswap16s(vdev, &desc->next);
 }
 
 static inline uint16_t vring_avail_flags(VirtQueue *vq)
@@ -145,7 +141,8 @@
 {
     hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, idx);
-    return virtio_lduw_phys(vq->vdev, pa);
+    vq->shadow_avail_idx = virtio_lduw_phys(vq->vdev, pa);
+    return vq->shadow_avail_idx;
 }
 
 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
@@ -155,23 +152,20 @@
     return virtio_lduw_phys(vq->vdev, pa);
 }
 
-static inline uint16_t vring_used_event(VirtQueue *vq)
+static inline uint16_t vring_get_used_event(VirtQueue *vq)
 {
     return vring_avail_ring(vq, vq->vring.num);
 }
 
-static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
+static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
+                                    int i)
 {
     hwaddr pa;
-    pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
-    virtio_stl_phys(vq->vdev, pa, val);
-}
-
-static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
-{
-    hwaddr pa;
-    pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
-    virtio_stl_phys(vq->vdev, pa, val);
+    virtio_tswap32s(vq->vdev, &uelem->id);
+    virtio_tswap32s(vq->vdev, &uelem->len);
+    pa = vq->vring.used + offsetof(VRingUsed, ring[i]);
+    address_space_write(&address_space_memory, pa, MEMTXATTRS_UNSPECIFIED,
+                       (void *)uelem, sizeof(VRingUsedElem));
 }
 
 static uint16_t vring_used_idx(VirtQueue *vq)
@@ -186,6 +180,7 @@
     hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, idx);
     virtio_stw_phys(vq->vdev, pa, val);
+    vq->used_idx = val;
 }
 
 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
@@ -204,7 +199,7 @@
     virtio_stw_phys(vdev, pa, virtio_lduw_phys(vdev, pa) & ~mask);
 }
 
-static inline void vring_avail_event(VirtQueue *vq, uint16_t val)
+static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
 {
     hwaddr pa;
     if (!vq->notification) {
@@ -217,8 +212,8 @@
 void virtio_queue_set_notification(VirtQueue *vq, int enable)
 {
     vq->notification = enable;
-    if (vq->vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(vq, vring_avail_idx(vq));
+    if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
+        vring_set_avail_event(vq, vring_avail_idx(vq));
     } else if (enable) {
         vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
     } else {
@@ -235,19 +230,23 @@
     return vq->vring.avail != 0;
 }
 
+/* Fetch avail_idx from VQ memory only when we really need to know if
+ * guest has added some buffers. */
 int virtio_queue_empty(VirtQueue *vq)
 {
+    if (vq->shadow_avail_idx != vq->last_avail_idx) {
+        return 0;
+    }
+
     return vring_avail_idx(vq) == vq->last_avail_idx;
 }
 
-void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
-                    unsigned int len, unsigned int idx)
+static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
+                               unsigned int len)
 {
     unsigned int offset;
     int i;
 
-    trace_virtqueue_fill(vq, elem, len, idx);
-
     offset = 0;
     for (i = 0; i < elem->in_num; i++) {
         size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
@@ -263,12 +262,30 @@
         cpu_physical_memory_unmap(elem->out_sg[i].iov_base,
                                   elem->out_sg[i].iov_len,
                                   0, elem->out_sg[i].iov_len);
+}
 
-    idx = (idx + vring_used_idx(vq)) % vq->vring.num;
+void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
+                       unsigned int len)
+{
+    vq->last_avail_idx--;
+    vq->inuse--;
+    virtqueue_unmap_sg(vq, elem, len);
+}
 
-    /* Get a pointer to the next entry in the used ring. */
-    vring_used_ring_id(vq, idx, elem->index);
-    vring_used_ring_len(vq, idx, len);
+void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
+                    unsigned int len, unsigned int idx)
+{
+    VRingUsedElem uelem;
+
+    trace_virtqueue_fill(vq, elem, len, idx);
+
+    virtqueue_unmap_sg(vq, elem, len);
+
+    idx = (idx + vq->used_idx) % vq->vring.num;
+
+    uelem.id = elem->index;
+    uelem.len = len;
+    vring_used_write(vq, &uelem, idx);
 }
 
 void virtqueue_flush(VirtQueue *vq, unsigned int count)
@@ -277,7 +294,7 @@
     /* Make sure buffer is written before we update index. */
     smp_wmb();
     trace_virtqueue_flush(vq, count);
-    old = vring_used_idx(vq);
+    old = vq->used_idx;
     new = old + count;
     vring_used_idx_set(vq, new);
     vq->inuse -= count;
@@ -299,7 +316,7 @@
     /* Check it isn't doing very strange things with descriptor numbers. */
     if (num_heads > vq->vring.num) {
         error_report("Guest moved used index from %u to %u",
-                     idx, vring_avail_idx(vq));
+                     idx, vq->shadow_avail_idx);
         exit(1);
     }
     /* On success, callers read a descriptor at vq->last_avail_idx.
@@ -328,18 +345,18 @@
     return head;
 }
 
-static unsigned virtqueue_next_desc(VirtIODevice *vdev, hwaddr desc_pa,
-                                    unsigned int i, unsigned int max)
+static unsigned virtqueue_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
+                                         hwaddr desc_pa, unsigned int max)
 {
     unsigned int next;
 
     /* If this descriptor says it doesn't chain, we're done. */
-    if (!(vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_NEXT)) {
+    if (!(desc->flags & VRING_DESC_F_NEXT)) {
         return max;
     }
 
     /* Check they're not leading us off end of descriptors. */
-    next = vring_desc_next(vdev, desc_pa, i);
+    next = desc->next;
     /* Make sure compiler knows to grab that: we don't want it changing! */
     smp_wmb();
 
@@ -348,6 +365,7 @@
         exit(1);
     }
 
+    vring_desc_read(vdev, desc, desc_pa, next);
     return next;
 }
 
@@ -364,6 +382,7 @@
     while (virtqueue_num_heads(vq, idx)) {
         VirtIODevice *vdev = vq->vdev;
         unsigned int max, num_bufs, indirect = 0;
+        VRingDesc desc;
         hwaddr desc_pa;
         int i;
 
@@ -371,9 +390,10 @@
         num_bufs = total_bufs;
         i = virtqueue_get_head(vq, idx++);
         desc_pa = vq->vring.desc;
+        vring_desc_read(vdev, &desc, desc_pa, i);
 
-        if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
-            if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
+        if (desc.flags & VRING_DESC_F_INDIRECT) {
+            if (desc.len % sizeof(VRingDesc)) {
                 error_report("Invalid size for indirect buffer table");
                 exit(1);
             }
@@ -386,9 +406,10 @@
 
             /* loop over the indirect descriptor table */
             indirect = 1;
-            max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
-            desc_pa = vring_desc_addr(vdev, desc_pa, i);
+            max = desc.len / sizeof(VRingDesc);
+            desc_pa = desc.addr;
             num_bufs = i = 0;
+            vring_desc_read(vdev, &desc, desc_pa, i);
         }
 
         do {
@@ -398,15 +419,15 @@
                 exit(1);
             }
 
-            if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
-                in_total += vring_desc_len(vdev, desc_pa, i);
+            if (desc.flags & VRING_DESC_F_WRITE) {
+                in_total += desc.len;
             } else {
-                out_total += vring_desc_len(vdev, desc_pa, i);
+                out_total += desc.len;
             }
             if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
                 goto done;
             }
-        } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
+        } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 
         if (!indirect)
             total_bufs = num_bufs;
@@ -431,98 +452,266 @@
     return in_bytes <= in_total && out_bytes <= out_total;
 }
 
-void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
-    size_t num_sg, int is_write)
+static void virtqueue_map_desc(unsigned int *p_num_sg, hwaddr *addr, struct iovec *iov,
+                               unsigned int max_num_sg, bool is_write,
+                               hwaddr pa, size_t sz)
+{
+    unsigned num_sg = *p_num_sg;
+    assert(num_sg <= max_num_sg);
+
+    if (!sz) {
+        error_report("virtio: zero sized buffers are not allowed");
+        exit(1);
+    }
+
+    while (sz) {
+        hwaddr len = sz;
+
+        if (num_sg == max_num_sg) {
+            error_report("virtio: too many write descriptors in indirect table");
+            exit(1);
+        }
+
+        iov[num_sg].iov_base = cpu_physical_memory_map(pa, &len, is_write);
+        iov[num_sg].iov_len = len;
+        addr[num_sg] = pa;
+
+        sz -= len;
+        pa += len;
+        num_sg++;
+    }
+    *p_num_sg = num_sg;
+}
+
+static void virtqueue_map_iovec(struct iovec *sg, hwaddr *addr,
+                                unsigned int *num_sg, unsigned int max_size,
+                                int is_write)
 {
     unsigned int i;
     hwaddr len;
 
-    if (num_sg > VIRTQUEUE_MAX_SIZE) {
-        error_report("virtio: map attempt out of bounds: %zd > %d",
-                     num_sg, VIRTQUEUE_MAX_SIZE);
-        exit(1);
-    }
+    /* Note: this function MUST validate input, some callers
+     * are passing in num_sg values received over the network.
+     */
+    /* TODO: teach all callers that this can fail, and return failure instead
+     * of asserting here.
+     * When we do, we might be able to re-enable NDEBUG below.
+     */
+#ifdef NDEBUG
+#error building with NDEBUG is not supported
+#endif
+    assert(*num_sg <= max_size);
 
-    for (i = 0; i < num_sg; i++) {
+    for (i = 0; i < *num_sg; i++) {
         len = sg[i].iov_len;
         sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write);
-        if (sg[i].iov_base == NULL || len != sg[i].iov_len) {
+        if (!sg[i].iov_base) {
             error_report("virtio: error trying to map MMIO memory");
             exit(1);
         }
+        if (len != sg[i].iov_len) {
+            error_report("virtio: unexpected memory split");
+            exit(1);
+        }
     }
 }
 
-int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
+void virtqueue_map(VirtQueueElement *elem)
+{
+    virtqueue_map_iovec(elem->in_sg, elem->in_addr, &elem->in_num,
+                        VIRTQUEUE_MAX_SIZE, 1);
+    virtqueue_map_iovec(elem->out_sg, elem->out_addr, &elem->out_num,
+                        VIRTQUEUE_MAX_SIZE, 0);
+}
+
+void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
+{
+    VirtQueueElement *elem;
+    size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
+    size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
+    size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
+    size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
+    size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
+    size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
+
+    assert(sz >= sizeof(VirtQueueElement));
+    elem = g_malloc(out_sg_end);
+    elem->out_num = out_num;
+    elem->in_num = in_num;
+    elem->in_addr = (void *)elem + in_addr_ofs;
+    elem->out_addr = (void *)elem + out_addr_ofs;
+    elem->in_sg = (void *)elem + in_sg_ofs;
+    elem->out_sg = (void *)elem + out_sg_ofs;
+    return elem;
+}
+
+void *virtqueue_pop(VirtQueue *vq, size_t sz)
 {
     unsigned int i, head, max;
     hwaddr desc_pa = vq->vring.desc;
     VirtIODevice *vdev = vq->vdev;
+    VirtQueueElement *elem;
+    unsigned out_num, in_num;
+    hwaddr addr[VIRTQUEUE_MAX_SIZE];
+    struct iovec iov[VIRTQUEUE_MAX_SIZE];
+    VRingDesc desc;
 
-    if (!virtqueue_num_heads(vq, vq->last_avail_idx))
-        return 0;
+    if (virtio_queue_empty(vq)) {
+        return NULL;
+    }
+    /* Needed after virtio_queue_empty(), see comment in
+     * virtqueue_num_heads(). */
+    smp_rmb();
 
     /* When we start there are none of either input nor output. */
-    elem->out_num = elem->in_num = 0;
+    out_num = in_num = 0;
 
     max = vq->vring.num;
 
-    i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
-    if (vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX)) {
-        vring_avail_event(vq, vq->last_avail_idx);
+    if (vq->inuse >= vq->vring.num) {
+        error_report("Virtqueue size exceeded");
+        exit(1);
     }
 
-    if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_INDIRECT) {
-        if (vring_desc_len(vdev, desc_pa, i) % sizeof(VRingDesc)) {
+    i = head = virtqueue_get_head(vq, vq->last_avail_idx++);
+    if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
+        vring_set_avail_event(vq, vq->last_avail_idx);
+    }
+
+    vring_desc_read(vdev, &desc, desc_pa, i);
+    if (desc.flags & VRING_DESC_F_INDIRECT) {
+        if (desc.len % sizeof(VRingDesc)) {
             error_report("Invalid size for indirect buffer table");
             exit(1);
         }
 
         /* loop over the indirect descriptor table */
-        max = vring_desc_len(vdev, desc_pa, i) / sizeof(VRingDesc);
-        desc_pa = vring_desc_addr(vdev, desc_pa, i);
+        max = desc.len / sizeof(VRingDesc);
+        desc_pa = desc.addr;
         i = 0;
+        vring_desc_read(vdev, &desc, desc_pa, i);
     }
 
     /* Collect all the descriptors */
     do {
-        struct iovec *sg;
-
-        if (vring_desc_flags(vdev, desc_pa, i) & VRING_DESC_F_WRITE) {
-            if (elem->in_num >= ARRAY_SIZE(elem->in_sg)) {
-                error_report("Too many write descriptors in indirect table");
-                exit(1);
-            }
-            elem->in_addr[elem->in_num] = vring_desc_addr(vdev, desc_pa, i);
-            sg = &elem->in_sg[elem->in_num++];
+        if (desc.flags & VRING_DESC_F_WRITE) {
+            virtqueue_map_desc(&in_num, addr + out_num, iov + out_num,
+                               VIRTQUEUE_MAX_SIZE - out_num, true, desc.addr, desc.len);
         } else {
-            if (elem->out_num >= ARRAY_SIZE(elem->out_sg)) {
-                error_report("Too many read descriptors in indirect table");
+            if (in_num) {
+                error_report("Incorrect order for descriptors");
                 exit(1);
             }
-            elem->out_addr[elem->out_num] = vring_desc_addr(vdev, desc_pa, i);
-            sg = &elem->out_sg[elem->out_num++];
+            virtqueue_map_desc(&out_num, addr, iov,
+                               VIRTQUEUE_MAX_SIZE, false, desc.addr, desc.len);
         }
 
-        sg->iov_len = vring_desc_len(vdev, desc_pa, i);
-
         /* If we've got too many, that implies a descriptor loop. */
-        if ((elem->in_num + elem->out_num) > max) {
+        if ((in_num + out_num) > max) {
             error_report("Looped descriptor");
             exit(1);
         }
-    } while ((i = virtqueue_next_desc(vdev, desc_pa, i, max)) != max);
+    } while ((i = virtqueue_read_next_desc(vdev, &desc, desc_pa, max)) != max);
 
-    /* Now map what we have collected */
-    virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1);
-    virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0);
-
+    /* Now copy what we have collected and mapped */
+    elem = virtqueue_alloc_element(sz, out_num, in_num);
     elem->index = head;
+    for (i = 0; i < out_num; i++) {
+        elem->out_addr[i] = addr[i];
+        elem->out_sg[i] = iov[i];
+    }
+    for (i = 0; i < in_num; i++) {
+        elem->in_addr[i] = addr[out_num + i];
+        elem->in_sg[i] = iov[out_num + i];
+    }
 
     vq->inuse++;
 
     trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
-    return elem->in_num + elem->out_num;
+    return elem;
+}
+
+/* Reading and writing a structure directly to QEMUFile is *awful*, but
+ * it is what QEMU has always done by mistake.  We can change it sooner
+ * or later by bumping the version number of the affected vm states.
+ * In the meanwhile, since the in-memory layout of VirtQueueElement
+ * has changed, we need to marshal to and from the layout that was
+ * used before the change.
+ */
+typedef struct VirtQueueElementOld {
+    unsigned int index;
+    unsigned int out_num;
+    unsigned int in_num;
+    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
+    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
+    struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
+    struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
+} VirtQueueElementOld;
+
+void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz)
+{
+    VirtQueueElement *elem;
+    VirtQueueElementOld data;
+    int i;
+
+    qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
+
+    elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
+    elem->index = data.index;
+
+    for (i = 0; i < elem->in_num; i++) {
+        elem->in_addr[i] = data.in_addr[i];
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        elem->out_addr[i] = data.out_addr[i];
+    }
+
+    for (i = 0; i < elem->in_num; i++) {
+        /* Base is overwritten by virtqueue_map.  */
+        elem->in_sg[i].iov_base = 0;
+        elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        /* Base is overwritten by virtqueue_map.  */
+        elem->out_sg[i].iov_base = 0;
+        elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
+    }
+
+    virtqueue_map(elem);
+    return elem;
+}
+
+void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem)
+{
+    VirtQueueElementOld data;
+    int i;
+
+    memset(&data, 0, sizeof(data));
+    data.index = elem->index;
+    data.in_num = elem->in_num;
+    data.out_num = elem->out_num;
+
+    for (i = 0; i < elem->in_num; i++) {
+        data.in_addr[i] = elem->in_addr[i];
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        data.out_addr[i] = elem->out_addr[i];
+    }
+
+    for (i = 0; i < elem->in_num; i++) {
+        /* Base is overwritten by virtqueue_map when loading.  Do not
+         * save it, as it would leak the QEMU address space layout.  */
+        data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
+    }
+
+    for (i = 0; i < elem->out_num; i++) {
+        /* Do not save iov_base as above.  */
+        data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
+    }
+    qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
 }
 
 /* virtio device */
@@ -541,15 +730,37 @@
     virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
 }
 
-void virtio_set_status(VirtIODevice *vdev, uint8_t val)
+static int virtio_validate_features(VirtIODevice *vdev)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+
+    if (k->validate_features) {
+        return k->validate_features(vdev);
+    } else {
+        return 0;
+    }
+}
+
+int virtio_set_status(VirtIODevice *vdev, uint8_t val)
 {
     VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
     trace_virtio_set_status(vdev, val);
 
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
+            val & VIRTIO_CONFIG_S_FEATURES_OK) {
+            int ret = virtio_validate_features(vdev);
+
+            if (ret) {
+                return ret;
+            }
+        }
+    }
     if (k->set_status) {
         k->set_status(vdev, val);
     }
     vdev->status = val;
+    return 0;
 }
 
 bool target_words_bigendian(void);
@@ -599,16 +810,18 @@
     vdev->config_vector = VIRTIO_NO_VECTOR;
     virtio_notify_vector(vdev, vdev->config_vector);
 
-    for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
         vdev->vq[i].vring.desc = 0;
         vdev->vq[i].vring.avail = 0;
         vdev->vq[i].vring.used = 0;
         vdev->vq[i].last_avail_idx = 0;
-        vdev->vq[i].pa = 0;
-        vdev->vq[i].vector = VIRTIO_NO_VECTOR;
+        vdev->vq[i].shadow_avail_idx = 0;
+        vdev->vq[i].used_idx = 0;
+        virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
         vdev->vq[i].signalled_used = 0;
         vdev->vq[i].signalled_used_valid = false;
         vdev->vq[i].notification = true;
+        vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
     }
 }
 
@@ -705,15 +918,119 @@
     }
 }
 
+uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint8_t val;
+
+    if (addr + sizeof(val) > vdev->config_len) {
+        return (uint32_t)-1;
+    }
+
+    k->get_config(vdev, vdev->config);
+
+    val = ldub_p(vdev->config + addr);
+    return val;
+}
+
+uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint16_t val;
+
+    if (addr + sizeof(val) > vdev->config_len) {
+        return (uint32_t)-1;
+    }
+
+    k->get_config(vdev, vdev->config);
+
+    val = lduw_le_p(vdev->config + addr);
+    return val;
+}
+
+uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint32_t val;
+
+    if (addr + sizeof(val) > vdev->config_len) {
+        return (uint32_t)-1;
+    }
+
+    k->get_config(vdev, vdev->config);
+
+    val = ldl_le_p(vdev->config + addr);
+    return val;
+}
+
+void virtio_config_modern_writeb(VirtIODevice *vdev,
+                                 uint32_t addr, uint32_t data)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint8_t val = data;
+
+    if (addr + sizeof(val) > vdev->config_len) {
+        return;
+    }
+
+    stb_p(vdev->config + addr, val);
+
+    if (k->set_config) {
+        k->set_config(vdev, vdev->config);
+    }
+}
+
+void virtio_config_modern_writew(VirtIODevice *vdev,
+                                 uint32_t addr, uint32_t data)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint16_t val = data;
+
+    if (addr + sizeof(val) > vdev->config_len) {
+        return;
+    }
+
+    stw_le_p(vdev->config + addr, val);
+
+    if (k->set_config) {
+        k->set_config(vdev, vdev->config);
+    }
+}
+
+void virtio_config_modern_writel(VirtIODevice *vdev,
+                                 uint32_t addr, uint32_t data)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint32_t val = data;
+
+    if (addr + sizeof(val) > vdev->config_len) {
+        return;
+    }
+
+    stl_le_p(vdev->config + addr, val);
+
+    if (k->set_config) {
+        k->set_config(vdev, vdev->config);
+    }
+}
+
 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
 {
-    vdev->vq[n].pa = addr;
-    virtqueue_init(&vdev->vq[n]);
+    vdev->vq[n].vring.desc = addr;
+    virtio_queue_update_rings(vdev, n);
 }
 
 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
 {
-    return vdev->vq[n].pa;
+    return vdev->vq[n].vring.desc;
+}
+
+void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
+                            hwaddr avail, hwaddr used)
+{
+    vdev->vq[n].vring.desc = desc;
+    vdev->vq[n].vring.avail = avail;
+    vdev->vq[n].vring.used = used;
 }
 
 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
@@ -727,7 +1044,16 @@
         return;
     }
     vdev->vq[n].vring.num = num;
-    virtqueue_init(&vdev->vq[n]);
+}
+
+VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
+{
+    return QLIST_FIRST(&vdev->vector_queues[vector]);
+}
+
+VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
+{
+    return QLIST_NEXT(vq, node);
 }
 
 int virtio_queue_get_num(VirtIODevice *vdev, int n)
@@ -735,11 +1061,17 @@
     return vdev->vq[n].vring.num;
 }
 
-int virtio_queue_get_id(VirtQueue *vq)
+int virtio_get_num_queues(VirtIODevice *vdev)
 {
-    VirtIODevice *vdev = vq->vdev;
-    assert(vq >= &vdev->vq[0] && vq < &vdev->vq[VIRTIO_PCI_QUEUE_MAX]);
-    return vq - &vdev->vq[0];
+    int i;
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        if (!virtio_queue_get_num(vdev, i)) {
+            break;
+        }
+    }
+
+    return i;
 }
 
 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
@@ -747,6 +1079,11 @@
     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
 
+    /* virtio-1 compliant devices cannot change the alignment */
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        error_report("tried to modify queue alignment for virtio-1 device");
+        return;
+    }
     /* Check that the transport told us it was going to do this
      * (so a buggy transport will immediately assert rather than
      * silently failing to migrate this state)
@@ -755,13 +1092,24 @@
     (void)k;
 
     vdev->vq[n].vring.align = align;
-    virtqueue_init(&vdev->vq[n]);
+    virtio_queue_update_rings(vdev, n);
 }
 
-void virtio_queue_notify_vq(VirtQueue *vq)
+static void virtio_queue_notify_aio_vq(VirtQueue *vq)
 {
-    if (vq->vring.desc) {
+    if (vq->vring.desc && vq->handle_aio_output) {
         VirtIODevice *vdev = vq->vdev;
+
+        trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
+        vq->handle_aio_output(vdev, vq);
+    }
+}
+
+static void virtio_queue_notify_vq(VirtQueue *vq)
+{
+    if (vq->vring.desc && vq->handle_output) {
+        VirtIODevice *vdev = vq->vdev;
+
         trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
         vq->handle_output(vdev, vq);
     }
@@ -774,43 +1122,76 @@
 
 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
 {
-    return n < VIRTIO_PCI_QUEUE_MAX ? vdev->vq[n].vector :
+    return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
         VIRTIO_NO_VECTOR;
 }
 
 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
 {
-    if (n < VIRTIO_PCI_QUEUE_MAX)
+    VirtQueue *vq = &vdev->vq[n];
+
+    if (n < VIRTIO_QUEUE_MAX) {
+        if (vdev->vector_queues &&
+            vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
+            QLIST_REMOVE(vq, node);
+        }
         vdev->vq[n].vector = vector;
+        if (vdev->vector_queues &&
+            vector != VIRTIO_NO_VECTOR) {
+            QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
+        }
+    }
 }
 
-VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
-                            void (*handle_output)(VirtIODevice *, VirtQueue *))
+static VirtQueue *virtio_add_queue_internal(VirtIODevice *vdev, int queue_size,
+                                            VirtIOHandleOutput handle_output,
+                                            bool use_aio)
 {
     int i;
 
-    for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
         if (vdev->vq[i].vring.num == 0)
             break;
     }
 
-    if (i == VIRTIO_PCI_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
+    if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
         abort();
 
     vdev->vq[i].vring.num = queue_size;
+    vdev->vq[i].vring.num_default = queue_size;
     vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
     vdev->vq[i].handle_output = handle_output;
+    vdev->vq[i].handle_aio_output = NULL;
+    vdev->vq[i].use_aio = use_aio;
 
     return &vdev->vq[i];
 }
 
+/* Add a virt queue and mark AIO.
+ * An AIO queue will use the AioContext based event interface instead of the
+ * default IOHandler and EventNotifier interface.
+ */
+VirtQueue *virtio_add_queue_aio(VirtIODevice *vdev, int queue_size,
+                                VirtIOHandleOutput handle_output)
+{
+    return virtio_add_queue_internal(vdev, queue_size, handle_output, true);
+}
+
+/* Add a normal virt queue (on the contrary to the AIO version above. */
+VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
+                            VirtIOHandleOutput handle_output)
+{
+    return virtio_add_queue_internal(vdev, queue_size, handle_output, false);
+}
+
 void virtio_del_queue(VirtIODevice *vdev, int n)
 {
-    if (n < 0 || n >= VIRTIO_PCI_QUEUE_MAX) {
+    if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
         abort();
     }
 
     vdev->vq[n].vring.num = 0;
+    vdev->vq[n].vring.num_default = 0;
 }
 
 void virtio_irq(VirtQueue *vq)
@@ -820,45 +1201,32 @@
     virtio_notify_vector(vq->vdev, vq->vector);
 }
 
-/* Assuming a given event_idx value from the other size, if
- * we have just incremented index from old to new_idx,
- * should we trigger an event? */
-static inline int vring_need_event(uint16_t event, uint16_t new, uint16_t old)
-{
-	/* Note: Xen has similar logic for notification hold-off
-	 * in include/xen/interface/io/ring.h with req_event and req_prod
-	 * corresponding to event_idx + 1 and new respectively.
-	 * Note also that req_event and req_prod in Xen start at 1,
-	 * event indexes in virtio start at 0. */
-	return (uint16_t)(new - event - 1) < (uint16_t)(new - old);
-}
-
-static bool vring_notify(VirtIODevice *vdev, VirtQueue *vq)
+bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
 {
     uint16_t old, new;
     bool v;
     /* We need to expose used array entries before checking used event. */
     smp_mb();
     /* Always notify when queue is empty (when feature acknowledge) */
-    if (((vdev->guest_features & (1 << VIRTIO_F_NOTIFY_ON_EMPTY)) &&
-         !vq->inuse && vring_avail_idx(vq) == vq->last_avail_idx)) {
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
+        !vq->inuse && virtio_queue_empty(vq)) {
         return true;
     }
 
-    if (!(vdev->guest_features & (1 << VIRTIO_RING_F_EVENT_IDX))) {
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
         return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
     }
 
     v = vq->signalled_used_valid;
     vq->signalled_used_valid = true;
     old = vq->signalled_used;
-    new = vq->signalled_used = vring_used_idx(vq);
-    return !v || vring_need_event(vring_used_event(vq), new, old);
+    new = vq->signalled_used = vq->used_idx;
+    return !v || vring_need_event(vring_get_used_event(vq), new, old);
 }
 
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
 {
-    if (!vring_notify(vdev, vq)) {
+    if (!virtio_should_notify(vdev, vq)) {
         return;
     }
 
@@ -873,6 +1241,7 @@
         return;
 
     vdev->isr |= 0x03;
+    vdev->generation++;
     virtio_notify_vector(vdev, vdev->config_vector);
 }
 
@@ -881,19 +1250,164 @@
     VirtIODevice *vdev = opaque;
 
     assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
-    return vdev->device_endian != virtio_default_endian();
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        return vdev->device_endian != virtio_default_endian();
+    }
+    /* Devices conforming to VIRTIO 1.0 or later are always LE. */
+    return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
 }
 
+static bool virtio_64bit_features_needed(void *opaque)
+{
+    VirtIODevice *vdev = opaque;
+
+    return (vdev->host_features >> 32) != 0;
+}
+
+static bool virtio_virtqueue_needed(void *opaque)
+{
+    VirtIODevice *vdev = opaque;
+
+    return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
+}
+
+static bool virtio_ringsize_needed(void *opaque)
+{
+    VirtIODevice *vdev = opaque;
+    int i;
+
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
+        if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool virtio_extra_state_needed(void *opaque)
+{
+    VirtIODevice *vdev = opaque;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    return k->has_extra_state &&
+        k->has_extra_state(qbus->parent);
+}
+
+static const VMStateDescription vmstate_virtqueue = {
+    .name = "virtqueue_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(vring.avail, struct VirtQueue),
+        VMSTATE_UINT64(vring.used, struct VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_virtio_virtqueues = {
+    .name = "virtio/virtqueues",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_virtqueue_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
+                      VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_ringsize = {
+    .name = "ringsize_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(vring.num_default, struct VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_virtio_ringsize = {
+    .name = "virtio/ringsize",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_ringsize_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
+                      VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static int get_extra_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    if (!k->load_extra_state) {
+        return -1;
+    } else {
+        return k->load_extra_state(qbus->parent, f);
+    }
+}
+
+static void put_extra_state(QEMUFile *f, void *pv, size_t size)
+{
+    VirtIODevice *vdev = pv;
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
+
+    k->save_extra_state(qbus->parent, f);
+}
+
+static const VMStateInfo vmstate_info_extra_state = {
+    .name = "virtqueue_extra_state",
+    .get = get_extra_state,
+    .put = put_extra_state,
+};
+
+static const VMStateDescription vmstate_virtio_extra_state = {
+    .name = "virtio/extra_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_extra_state_needed,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "extra_state",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &vmstate_info_extra_state,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_virtio_device_endian = {
     .name = "virtio/device_endian",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = &virtio_device_endian_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT8(device_endian, VirtIODevice),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static const VMStateDescription vmstate_virtio_64bit_features = {
+    .name = "virtio/64bit_features",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = &virtio_64bit_features_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(guest_features, VirtIODevice),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_virtio = {
     .name = "virtio",
     .version_id = 1,
@@ -902,12 +1416,13 @@
     .fields = (VMStateField[]) {
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_virtio_device_endian,
-            .needed = &virtio_device_endian_needed
-        },
-        { 0 }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_virtio_device_endian,
+        &vmstate_virtio_64bit_features,
+        &vmstate_virtio_virtqueues,
+        &vmstate_virtio_ringsize,
+        &vmstate_virtio_extra_state,
+        NULL
     }
 };
 
@@ -916,6 +1431,7 @@
     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
+    uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
     int i;
 
     if (k->save_config) {
@@ -925,18 +1441,18 @@
     qemu_put_8s(f, &vdev->status);
     qemu_put_8s(f, &vdev->isr);
     qemu_put_be16s(f, &vdev->queue_sel);
-    qemu_put_be32s(f, &vdev->guest_features);
+    qemu_put_be32s(f, &guest_features_lo);
     qemu_put_be32(f, vdev->config_len);
     qemu_put_buffer(f, vdev->config, vdev->config_len);
 
-    for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
         if (vdev->vq[i].vring.num == 0)
             break;
     }
 
     qemu_put_be32(f, i);
 
-    for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
         if (vdev->vq[i].vring.num == 0)
             break;
 
@@ -944,7 +1460,8 @@
         if (k->has_variable_vring_alignment) {
             qemu_put_be32(f, vdev->vq[i].vring.align);
         }
-        qemu_put_be64(f, vdev->vq[i].pa);
+        /* XXX virtio-1 devices */
+        qemu_put_be64(f, vdev->vq[i].vring.desc);
         qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
         if (k->save_queue) {
             k->save_queue(qbus->parent, i, f);
@@ -956,18 +1473,21 @@
     }
 
     /* Subsections */
-    vmstate_save_state(f, &vmstate_virtio, vdev);
+    vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
 }
 
-int virtio_set_features(VirtIODevice *vdev, uint32_t val)
+/* A wrapper for use as a VMState .put function */
+void virtio_vmstate_save(QEMUFile *f, void *opaque, size_t size)
 {
-    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
-    VirtioBusClass *vbusk = VIRTIO_BUS_GET_CLASS(qbus);
-    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
-    uint32_t supported_features = vbusk->get_features(qbus->parent);
-    bool bad = (val & ~supported_features) != 0;
+    virtio_save(VIRTIO_DEVICE(opaque), f);
+}
 
-    val &= supported_features;
+static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
+{
+    VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
+    bool bad = (val & ~(vdev->host_features)) != 0;
+
+    val &= vdev->host_features;
     if (k->set_features) {
         k->set_features(vdev, val);
     }
@@ -975,13 +1495,24 @@
     return bad ? -1 : 0;
 }
 
+int virtio_set_features(VirtIODevice *vdev, uint64_t val)
+{
+   /*
+     * The driver must not attempt to set features after feature negotiation
+     * has finished.
+     */
+    if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
+        return -EINVAL;
+    }
+    return virtio_set_features_nocheck(vdev, val);
+}
+
 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
 {
     int i, ret;
     int32_t config_len;
     uint32_t num;
     uint32_t features;
-    uint32_t supported_features;
     BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
     VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
@@ -1001,17 +1532,21 @@
     qemu_get_8s(f, &vdev->status);
     qemu_get_8s(f, &vdev->isr);
     qemu_get_be16s(f, &vdev->queue_sel);
-    if (vdev->queue_sel >= VIRTIO_PCI_QUEUE_MAX) {
+    if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
         return -1;
     }
     qemu_get_be32s(f, &features);
 
-    if (virtio_set_features(vdev, features) < 0) {
-        supported_features = k->get_features(qbus->parent);
-        error_report("Features 0x%x unsupported. Allowed features: 0x%x",
-                     features, supported_features);
-        return -1;
-    }
+    /*
+     * Temporarily set guest_features low bits - needed by
+     * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
+     * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
+     *
+     * Note: devices should always test host features in future - don't create
+     * new dependencies like this.
+     */
+    vdev->guest_features = features;
+
     config_len = qemu_get_be32(f);
 
     /*
@@ -1028,8 +1563,8 @@
 
     num = qemu_get_be32(f);
 
-    if (num > VIRTIO_PCI_QUEUE_MAX) {
-        error_report("Invalid number of PCI queues: 0x%x", num);
+    if (num > VIRTIO_QUEUE_MAX) {
+        error_report("Invalid number of virtqueues: 0x%x", num);
         return -1;
     }
 
@@ -1038,13 +1573,14 @@
         if (k->has_variable_vring_alignment) {
             vdev->vq[i].vring.align = qemu_get_be32(f);
         }
-        vdev->vq[i].pa = qemu_get_be64(f);
+        vdev->vq[i].vring.desc = qemu_get_be64(f);
         qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
         vdev->vq[i].signalled_used_valid = false;
         vdev->vq[i].notification = true;
 
-        if (vdev->vq[i].pa) {
-            virtqueue_init(&vdev->vq[i]);
+        if (vdev->vq[i].vring.desc) {
+            /* XXX virtio-1 devices */
+            virtio_queue_update_rings(vdev, i);
         } else if (vdev->vq[i].last_avail_idx) {
             error_report("VQ %d address 0x0 "
                          "inconsistent with Host index 0x%x",
@@ -1077,8 +1613,30 @@
         vdev->device_endian = virtio_default_endian();
     }
 
+    if (virtio_64bit_features_needed(vdev)) {
+        /*
+         * Subsection load filled vdev->guest_features.  Run them
+         * through virtio_set_features to sanity-check them against
+         * host_features.
+         */
+        uint64_t features64 = vdev->guest_features;
+        if (virtio_set_features_nocheck(vdev, features64) < 0) {
+            error_report("Features 0x%" PRIx64 " unsupported. "
+                         "Allowed features: 0x%" PRIx64,
+                         features64, vdev->host_features);
+            return -1;
+        }
+    } else {
+        if (virtio_set_features_nocheck(vdev, features) < 0) {
+            error_report("Features 0x%x unsupported. "
+                         "Allowed features: 0x%" PRIx64,
+                         features, vdev->host_features);
+            return -1;
+        }
+    }
+
     for (i = 0; i < num; i++) {
-        if (vdev->vq[i].pa) {
+        if (vdev->vq[i].vring.desc) {
             uint16_t nheads;
             nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
             /* Check it isn't doing strange things with descriptor numbers. */
@@ -1090,6 +1648,23 @@
                              vdev->vq[i].last_avail_idx, nheads);
                 return -1;
             }
+            vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
+            vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
+
+            /*
+             * Some devices migrate VirtQueueElements that have been popped
+             * from the avail ring but not yet returned to the used ring.
+             */
+            vdev->vq[i].inuse = vdev->vq[i].last_avail_idx -
+                                vdev->vq[i].used_idx;
+            if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
+                error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
+                             "used_idx 0x%x",
+                             i, vdev->vq[i].vring.num,
+                             vdev->vq[i].last_avail_idx,
+                             vdev->vq[i].used_idx);
+                return -1;
+            }
         }
     }
 
@@ -1101,6 +1676,7 @@
     qemu_del_vm_change_state_handler(vdev->vmstate);
     g_free(vdev->config);
     g_free(vdev->vq);
+    g_free(vdev->vector_queues);
 }
 
 static void virtio_vmstate_change(void *opaque, int running, RunState state)
@@ -1138,15 +1714,24 @@
 void virtio_init(VirtIODevice *vdev, const char *name,
                  uint16_t device_id, size_t config_size)
 {
+    BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
+    VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
     int i;
+    int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
+
+    if (nvectors) {
+        vdev->vector_queues =
+            g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
+    }
+
     vdev->device_id = device_id;
     vdev->status = 0;
     vdev->isr = 0;
     vdev->queue_sel = 0;
     vdev->config_vector = VIRTIO_NO_VECTOR;
-    vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_PCI_QUEUE_MAX);
+    vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
     vdev->vm_running = runstate_is_running();
-    for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) {
+    for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
         vdev->vq[i].vector = VIRTIO_NO_VECTOR;
         vdev->vq[i].vdev = vdev;
         vdev->vq[i].queue_index = i;
@@ -1162,6 +1747,7 @@
     vdev->vmstate = qemu_add_vm_change_state_handler(virtio_vmstate_change,
                                                      vdev);
     vdev->device_endian = virtio_default_endian();
+    vdev->use_guest_notifier_mask = true;
 }
 
 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
@@ -1192,7 +1778,7 @@
 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingAvail, ring) +
-        sizeof(uint64_t) * vdev->vq[n].vring.num;
+        sizeof(uint16_t) * vdev->vq[n].vring.num;
 }
 
 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
@@ -1215,6 +1801,7 @@
 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx)
 {
     vdev->vq[n].last_avail_idx = idx;
+    vdev->vq[n].shadow_avail_idx = idx;
 }
 
 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
@@ -1244,10 +1831,10 @@
                                                 bool with_irqfd)
 {
     if (assign && !with_irqfd) {
-        event_notifier_set_handler(&vq->guest_notifier,
+        event_notifier_set_handler(&vq->guest_notifier, false,
                                    virtio_queue_guest_notifier_read);
     } else {
-        event_notifier_set_handler(&vq->guest_notifier, NULL);
+        event_notifier_set_handler(&vq->guest_notifier, false, NULL);
     }
     if (!assign) {
         /* Test and clear notifier before closing it,
@@ -1261,6 +1848,30 @@
     return &vq->guest_notifier;
 }
 
+static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
+{
+    VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
+    if (event_notifier_test_and_clear(n)) {
+        virtio_queue_notify_aio_vq(vq);
+    }
+}
+
+void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
+                                                VirtIOHandleOutput handle_output)
+{
+    if (handle_output) {
+        vq->handle_aio_output = handle_output;
+        aio_set_event_notifier(ctx, &vq->host_notifier, true,
+                               virtio_queue_host_notifier_aio_read);
+    } else {
+        aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
+        /* Test and clear notifier before after disabling event,
+         * in case poll callback didn't have time to run. */
+        virtio_queue_host_notifier_aio_read(&vq->host_notifier);
+        vq->handle_aio_output = NULL;
+    }
+}
+
 static void virtio_queue_host_notifier_read(EventNotifier *n)
 {
     VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
@@ -1272,11 +1883,21 @@
 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                bool set_handler)
 {
+    AioContext *ctx = qemu_get_aio_context();
     if (assign && set_handler) {
-        event_notifier_set_handler(&vq->host_notifier,
+        if (vq->use_aio) {
+            aio_set_event_notifier(ctx, &vq->host_notifier, true,
                                    virtio_queue_host_notifier_read);
+        } else {
+            event_notifier_set_handler(&vq->host_notifier, true,
+                                       virtio_queue_host_notifier_read);
+        }
     } else {
-        event_notifier_set_handler(&vq->host_notifier, NULL);
+        if (vq->use_aio) {
+            aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL);
+        } else {
+            event_notifier_set_handler(&vq->host_notifier, true, NULL);
+        }
     }
     if (!assign) {
         /* Test and clear notifier before after disabling event,
@@ -1309,7 +1930,12 @@
             return;
         }
     }
-    virtio_bus_device_plugged(vdev);
+
+    virtio_bus_device_plugged(vdev, &err);
+    if (err != NULL) {
+        error_propagate(errp, err);
+        return;
+    }
 }
 
 static void virtio_device_unrealize(DeviceState *dev, Error **errp)
@@ -1332,6 +1958,11 @@
     vdev->bus_name = NULL;
 }
 
+static Property virtio_properties[] = {
+    DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void virtio_device_class_init(ObjectClass *klass, void *data)
 {
     /* Set the default value here. */
@@ -1340,6 +1971,7 @@
     dc->realize = virtio_device_realize;
     dc->unrealize = virtio_device_unrealize;
     dc->bus_type = TYPE_VIRTIO_BUS;
+    dc->props = virtio_properties;
 }
 
 static const TypeInfo virtio_device_info = {
diff --git a/hw/watchdog/Makefile.objs b/hw/watchdog/Makefile.objs
index 4b0374a..72e3ffd 100644
--- a/hw/watchdog/Makefile.objs
+++ b/hw/watchdog/Makefile.objs
@@ -1,3 +1,4 @@
 common-obj-y += watchdog.o
 common-obj-$(CONFIG_WDT_IB6300ESB) += wdt_i6300esb.o
 common-obj-$(CONFIG_WDT_IB700) += wdt_ib700.o
+common-obj-$(CONFIG_WDT_DIAG288) += wdt_diag288.o
diff --git a/hw/watchdog/watchdog.c b/hw/watchdog/watchdog.c
index c307f9b..2aeaf1f 100644
--- a/hw/watchdog/watchdog.c
+++ b/hw/watchdog/watchdog.c
@@ -19,7 +19,7 @@
  * By Richard W.M. Jones (rjones@redhat.com).
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/queue.h"
@@ -27,14 +27,8 @@
 #include "sysemu/sysemu.h"
 #include "sysemu/watchdog.h"
 #include "qapi-event.h"
-
-/* Possible values for action parameter. */
-#define WDT_RESET        1	/* Hard reset. */
-#define WDT_SHUTDOWN     2	/* Shutdown. */
-#define WDT_POWEROFF     3	/* Quit. */
-#define WDT_PAUSE        4	/* Pause. */
-#define WDT_DEBUG        5	/* Prints a message and continues running. */
-#define WDT_NONE         6	/* Do nothing. */
+#include "hw/nmi.h"
+#include "qemu/help_option.h"
 
 static int watchdog_action = WDT_RESET;
 static QLIST_HEAD(watchdog_list, WatchdogTimerModel) watchdog_list;
@@ -68,7 +62,7 @@
             /* add the device */
             opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
                                     &error_abort);
-            qemu_opt_set(opts, "driver", p);
+            qemu_opt_set(opts, "driver", p, &error_abort);
             return 0;
         }
     }
@@ -95,12 +89,19 @@
         watchdog_action = WDT_DEBUG;
     else if (strcasecmp(p, "none") == 0)
         watchdog_action = WDT_NONE;
+    else if (strcasecmp(p, "inject-nmi") == 0)
+        watchdog_action = WDT_NMI;
     else
         return -1;
 
     return 0;
 }
 
+int get_watchdog_action(void)
+{
+    return watchdog_action;
+}
+
 /* This actually performs the "action" once a watchdog has expired,
  * ie. reboot, shutdown, exit, etc.
  */
@@ -138,5 +139,11 @@
     case WDT_NONE:
         qapi_event_send_watchdog(WATCHDOG_EXPIRATION_ACTION_NONE, &error_abort);
         break;
+
+    case WDT_NMI:
+        qapi_event_send_watchdog(WATCHDOG_EXPIRATION_ACTION_INJECT_NMI,
+                                 &error_abort);
+        nmi_monitor_handle(0, NULL);
+        break;
     }
 }
diff --git a/hw/watchdog/wdt_diag288.c b/hw/watchdog/wdt_diag288.c
new file mode 100644
index 0000000..a7b64e2
--- /dev/null
+++ b/hw/watchdog/wdt_diag288.c
@@ -0,0 +1,143 @@
+/*
+ * watchdog device diag288 support
+ *
+ * Copyright IBM, Corp. 2015
+ *
+ * Authors:
+ *  Xu Wang <gesaint@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/watchdog.h"
+#include "hw/sysbus.h"
+#include "qemu/timer.h"
+#include "hw/watchdog/wdt_diag288.h"
+#include "qemu/log.h"
+
+static WatchdogTimerModel model = {
+    .wdt_name = TYPE_WDT_DIAG288,
+    .wdt_description = "diag288 device for s390x platform",
+};
+
+static const VMStateDescription vmstate_diag288 = {
+    .name = "vmstate_diag288",
+    .version_id = 0,
+    .minimum_version_id = 0,
+    .fields = (VMStateField[]) {
+        VMSTATE_TIMER_PTR(timer, DIAG288State),
+        VMSTATE_BOOL(enabled, DIAG288State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void wdt_diag288_reset(DeviceState *dev)
+{
+    DIAG288State *diag288 = DIAG288(dev);
+
+    diag288->enabled = false;
+    timer_del(diag288->timer);
+}
+
+static void diag288_reset(void *opaque)
+{
+    DeviceState *diag288 = opaque;
+
+    wdt_diag288_reset(diag288);
+}
+
+static void diag288_timer_expired(void *dev)
+{
+    qemu_log_mask(CPU_LOG_RESET, "Watchdog timer expired.\n");
+    /* Reset the watchdog only if the guest gets notified about
+     * expiry. watchdog_perform_action() may temporarily relinquish
+     * the BQL; reset before triggering the action to avoid races with
+     * diag288 instructions. */
+    switch (get_watchdog_action()) {
+    case WDT_DEBUG:
+    case WDT_NONE:
+    case WDT_PAUSE:
+        break;
+    default:
+        wdt_diag288_reset(dev);
+    }
+    watchdog_perform_action();
+}
+
+static int wdt_diag288_handle_timer(DIAG288State *diag288,
+                                     uint64_t func, uint64_t timeout)
+{
+    switch (func) {
+    case WDT_DIAG288_INIT:
+        diag288->enabled = true;
+        /* fall through */
+    case WDT_DIAG288_CHANGE:
+        if (!diag288->enabled) {
+            return -1;
+        }
+        timer_mod(diag288->timer,
+                  qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
+                  timeout * NANOSECONDS_PER_SECOND);
+        break;
+    case WDT_DIAG288_CANCEL:
+        if (!diag288->enabled) {
+            return -1;
+        }
+        diag288->enabled = false;
+        timer_del(diag288->timer);
+        break;
+    default:
+        return -1;
+    }
+
+    return 0;
+}
+
+static void wdt_diag288_realize(DeviceState *dev, Error **errp)
+{
+    DIAG288State *diag288 = DIAG288(dev);
+
+    qemu_register_reset(diag288_reset, diag288);
+    diag288->timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, diag288_timer_expired,
+                                  dev);
+}
+
+static void wdt_diag288_unrealize(DeviceState *dev, Error **errp)
+{
+    DIAG288State *diag288 = DIAG288(dev);
+
+    timer_del(diag288->timer);
+    timer_free(diag288->timer);
+}
+
+static void wdt_diag288_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    DIAG288Class *diag288 = DIAG288_CLASS(klass);
+
+    dc->realize = wdt_diag288_realize;
+    dc->unrealize = wdt_diag288_unrealize;
+    dc->reset = wdt_diag288_reset;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    dc->vmsd = &vmstate_diag288;
+    diag288->handle_timer = wdt_diag288_handle_timer;
+}
+
+static const TypeInfo wdt_diag288_info = {
+    .class_init = wdt_diag288_class_init,
+    .parent = TYPE_DEVICE,
+    .name  = TYPE_WDT_DIAG288,
+    .instance_size  = sizeof(DIAG288State),
+    .class_size = sizeof(DIAG288Class),
+};
+
+static void wdt_diag288_register_types(void)
+{
+    watchdog_add_model(&model);
+    type_register_static(&wdt_diag288_info);
+}
+
+type_init(wdt_diag288_register_types)
diff --git a/hw/watchdog/wdt_i6300esb.c b/hw/watchdog/wdt_i6300esb.c
index 687c8b1..a83d951 100644
--- a/hw/watchdog/wdt_i6300esb.c
+++ b/hw/watchdog/wdt_i6300esb.c
@@ -19,7 +19,7 @@
  * By Richard W.M. Jones (rjones@redhat.com).
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
@@ -103,6 +103,10 @@
 
 typedef struct I6300State I6300State;
 
+#define TYPE_WATCHDOG_I6300ESB_DEVICE "i6300esb"
+#define WATCHDOG_I6300ESB_DEVICE(obj) \
+    OBJECT_CHECK(I6300State, (obj), TYPE_WATCHDOG_I6300ESB_DEVICE)
+
 /* This function is called when the watchdog has either been enabled
  * (hence it starts counting down) or has been keep-alived.
  */
@@ -125,8 +129,9 @@
     else
         timeout <<= 5;
 
-    /* Get the timeout in units of ticks_per_sec. */
-    timeout = get_ticks_per_sec() * timeout / 33000000;
+    /* Get the timeout in nanoseconds. */
+
+    timeout = timeout * 30; /* on a PCI bus, 1 tick is 30 ns*/
 
     i6300esb_debug("stage %d, timeout %" PRIi64 "\n", d->stage, timeout);
 
@@ -144,7 +149,7 @@
 static void i6300esb_reset(DeviceState *dev)
 {
     PCIDevice *pdev = PCI_DEVICE(dev);
-    I6300State *d = DO_UPCAST(I6300State, dev, pdev);
+    I6300State *d = WATCHDOG_I6300ESB_DEVICE(pdev);
 
     i6300esb_debug("I6300State = %p\n", d);
 
@@ -207,7 +212,7 @@
 static void i6300esb_config_write(PCIDevice *dev, uint32_t addr,
                                   uint32_t data, int len)
 {
-    I6300State *d = DO_UPCAST(I6300State, dev, dev);
+    I6300State *d = WATCHDOG_I6300ESB_DEVICE(dev);
     int old;
 
     i6300esb_debug("addr = %x, data = %x, len = %d\n", addr, data, len);
@@ -235,7 +240,7 @@
 
 static uint32_t i6300esb_config_read(PCIDevice *dev, uint32_t addr, int len)
 {
-    I6300State *d = DO_UPCAST(I6300State, dev, dev);
+    I6300State *d = WATCHDOG_I6300ESB_DEVICE(dev);
     uint32_t data;
 
     i6300esb_debug ("addr = %x, len = %d\n", addr, len);
@@ -369,7 +374,7 @@
             i6300esb_mem_writel,
         },
     },
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static const VMStateDescription vmstate_i6300esb = {
@@ -398,7 +403,7 @@
         VMSTATE_INT32(free_run, I6300State),
         VMSTATE_INT32(locked, I6300State),
         VMSTATE_INT32(enabled, I6300State),
-        VMSTATE_TIMER(timer, I6300State),
+        VMSTATE_TIMER_PTR(timer, I6300State),
         VMSTATE_UINT32(timer1_preload, I6300State),
         VMSTATE_UINT32(timer2_preload, I6300State),
         VMSTATE_INT32(stage, I6300State),
@@ -408,9 +413,9 @@
     }
 };
 
-static int i6300esb_init(PCIDevice *dev)
+static void i6300esb_realize(PCIDevice *dev, Error **errp)
 {
-    I6300State *d = DO_UPCAST(I6300State, dev, dev);
+    I6300State *d = WATCHDOG_I6300ESB_DEVICE(dev);
 
     i6300esb_debug("I6300State = %p\n", d);
 
@@ -421,8 +426,6 @@
                           "i6300esb", 0x10);
     pci_register_bar(&d->dev, 0, 0, &d->io_mem);
     /* qemu_register_coalesced_mmio (addr, 0x10); ? */
-
-    return 0;
 }
 
 static WatchdogTimerModel model = {
@@ -437,7 +440,7 @@
 
     k->config_read = i6300esb_config_read;
     k->config_write = i6300esb_config_write;
-    k->init = i6300esb_init;
+    k->realize = i6300esb_realize;
     k->vendor_id = PCI_VENDOR_ID_INTEL;
     k->device_id = PCI_DEVICE_ID_INTEL_ESB_9;
     k->class_id = PCI_CLASS_SYSTEM_OTHER;
@@ -447,7 +450,7 @@
 }
 
 static const TypeInfo i6300esb_info = {
-    .name          = "i6300esb",
+    .name          = TYPE_WATCHDOG_I6300ESB_DEVICE,
     .parent        = TYPE_PCI_DEVICE,
     .instance_size = sizeof(I6300State),
     .class_init    = i6300esb_class_init,
diff --git a/hw/watchdog/wdt_ib700.c b/hw/watchdog/wdt_ib700.c
index 8cb9827..532afe8 100644
--- a/hw/watchdog/wdt_ib700.c
+++ b/hw/watchdog/wdt_ib700.c
@@ -19,6 +19,7 @@
  * By Richard W.M. Jones (rjones@redhat.com).
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 #include "sysemu/watchdog.h"
@@ -63,7 +64,7 @@
 
     ib700_debug("addr = %x, data = %x\n", addr, data);
 
-    timeout = (int64_t) time_map[data & 0xF] * get_ticks_per_sec();
+    timeout = (int64_t) time_map[data & 0xF] * NANOSECONDS_PER_SECOND;
     timer_mod(s->timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + timeout);
 }
 
@@ -93,7 +94,7 @@
     .version_id = 0,
     .minimum_version_id = 0,
     .fields = (VMStateField[]) {
-        VMSTATE_TIMER(timer, IB700State),
+        VMSTATE_TIMER_PTR(timer, IB700State),
         VMSTATE_END_OF_LIST()
     }
 };
diff --git a/hw/xen/Makefile.objs b/hw/xen/Makefile.objs
index a0ca0aa..d367094 100644
--- a/hw/xen/Makefile.objs
+++ b/hw/xen/Makefile.objs
@@ -2,4 +2,4 @@
 common-obj-$(CONFIG_XEN_BACKEND) += xen_backend.o xen_devconfig.o
 
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen-host-pci-device.o
-obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o
+obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_graphics.o xen_pt_msi.o
diff --git a/hw/xen/xen-host-pci-device.c b/hw/xen/xen-host-pci-device.c
index 743b37b..924e186 100644
--- a/hw/xen/xen-host-pci-device.c
+++ b/hw/xen/xen-host-pci-device.c
@@ -6,7 +6,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "xen-host-pci-device.h"
 
 #define XEN_HOST_PCI_MAX_EXT_CAP \
@@ -31,25 +34,20 @@
 #define IORESOURCE_PREFETCH     0x00001000      /* No side effects */
 #define IORESOURCE_MEM_64       0x00100000
 
-static int xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
-                                   const char *name, char *buf, ssize_t size)
+static void xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
+                                    const char *name, char *buf, ssize_t size)
 {
     int rc;
 
     rc = snprintf(buf, size, "/sys/bus/pci/devices/%04x:%02x:%02x.%d/%s",
                   d->domain, d->bus, d->dev, d->func, name);
-
-    if (rc >= size || rc < 0) {
-        /* The ouput is truncated or an other error is encountered */
-        return -ENODEV;
-    }
-    return 0;
+    assert(rc >= 0 && rc < size);
 }
 
 
 /* This size should be enough to read the first 7 lines of a resource file */
 #define XEN_HOST_PCI_RESOURCE_BUFFER_SIZE 400
-static int xen_host_pci_get_resource(XenHostPCIDevice *d)
+static void xen_host_pci_get_resource(XenHostPCIDevice *d, Error **errp)
 {
     int i, rc, fd;
     char path[PATH_MAX];
@@ -58,25 +56,22 @@
     char *endptr, *s;
     uint8_t type;
 
-    rc = xen_host_pci_sysfs_path(d, "resource", path, sizeof (path));
-    if (rc) {
-        return rc;
-    }
+    xen_host_pci_sysfs_path(d, "resource", path, sizeof(path));
+
     fd = open(path, O_RDONLY);
     if (fd == -1) {
-        XEN_HOST_PCI_LOG("Error: Can't open %s: %s\n", path, strerror(errno));
-        return -errno;
+        error_setg_file_open(errp, errno, path);
+        return;
     }
 
     do {
-        rc = read(fd, &buf, sizeof (buf) - 1);
+        rc = read(fd, &buf, sizeof(buf) - 1);
         if (rc < 0 && errno != EINTR) {
-            rc = -errno;
+            error_setg_errno(errp, errno, "read err");
             goto out;
         }
     } while (rc < 0);
     buf[rc] = 0;
-    rc = 0;
 
     s = buf;
     for (i = 0; i < PCI_NUM_REGIONS; i++) {
@@ -129,70 +124,69 @@
             d->rom.bus_flags = flags & IORESOURCE_BITS;
         }
     }
+
     if (i != PCI_NUM_REGIONS) {
-        /* Invalid format or input to short */
-        rc = -ENODEV;
+        error_setg(errp, "Invalid format or input too short: %s", buf);
     }
 
 out:
     close(fd);
-    return rc;
 }
 
 /* This size should be enough to read a long from a file */
 #define XEN_HOST_PCI_GET_VALUE_BUFFER_SIZE 22
-static int xen_host_pci_get_value(XenHostPCIDevice *d, const char *name,
-                                  unsigned int *pvalue, int base)
+static void xen_host_pci_get_value(XenHostPCIDevice *d, const char *name,
+                                   unsigned int *pvalue, int base, Error **errp)
 {
     char path[PATH_MAX];
     char buf[XEN_HOST_PCI_GET_VALUE_BUFFER_SIZE];
     int fd, rc;
     unsigned long value;
-    char *endptr;
+    const char *endptr;
 
-    rc = xen_host_pci_sysfs_path(d, name, path, sizeof (path));
-    if (rc) {
-        return rc;
-    }
+    xen_host_pci_sysfs_path(d, name, path, sizeof(path));
+
     fd = open(path, O_RDONLY);
     if (fd == -1) {
-        XEN_HOST_PCI_LOG("Error: Can't open %s: %s\n", path, strerror(errno));
-        return -errno;
+        error_setg_file_open(errp, errno, path);
+        return;
     }
+
     do {
-        rc = read(fd, &buf, sizeof (buf) - 1);
+        rc = read(fd, &buf, sizeof(buf) - 1);
         if (rc < 0 && errno != EINTR) {
-            rc = -errno;
+            error_setg_errno(errp, errno, "read err");
             goto out;
         }
     } while (rc < 0);
+
     buf[rc] = 0;
-    value = strtol(buf, &endptr, base);
-    if (endptr == buf || *endptr != '\n') {
-        rc = -1;
-    } else if ((value == LONG_MIN || value == LONG_MAX) && errno == ERANGE) {
-        rc = -errno;
-    } else {
-        rc = 0;
+    rc = qemu_strtoul(buf, &endptr, base, &value);
+    if (!rc) {
+        assert(value <= UINT_MAX);
         *pvalue = value;
+    } else {
+        error_setg_errno(errp, -rc, "failed to parse value '%s'", buf);
     }
+
 out:
     close(fd);
-    return rc;
 }
 
-static inline int xen_host_pci_get_hex_value(XenHostPCIDevice *d,
-                                             const char *name,
-                                             unsigned int *pvalue)
+static inline void xen_host_pci_get_hex_value(XenHostPCIDevice *d,
+                                              const char *name,
+                                              unsigned int *pvalue,
+                                              Error **errp)
 {
-    return xen_host_pci_get_value(d, name, pvalue, 16);
+    xen_host_pci_get_value(d, name, pvalue, 16, errp);
 }
 
-static inline int xen_host_pci_get_dec_value(XenHostPCIDevice *d,
-                                             const char *name,
-                                             unsigned int *pvalue)
+static inline void xen_host_pci_get_dec_value(XenHostPCIDevice *d,
+                                              const char *name,
+                                              unsigned int *pvalue,
+                                              Error **errp)
 {
-    return xen_host_pci_get_value(d, name, pvalue, 10);
+    xen_host_pci_get_value(d, name, pvalue, 10, errp);
 }
 
 static bool xen_host_pci_dev_is_virtfn(XenHostPCIDevice *d)
@@ -200,26 +194,21 @@
     char path[PATH_MAX];
     struct stat buf;
 
-    if (xen_host_pci_sysfs_path(d, "physfn", path, sizeof (path))) {
-        return false;
-    }
-    return !stat(path, &buf);
+    xen_host_pci_sysfs_path(d, "physfn", path, sizeof(path));
+
+    return !qemu_stat(path, &buf);
 }
 
-static int xen_host_pci_config_open(XenHostPCIDevice *d)
+static void xen_host_pci_config_open(XenHostPCIDevice *d, Error **errp)
 {
     char path[PATH_MAX];
-    int rc;
 
-    rc = xen_host_pci_sysfs_path(d, "config", path, sizeof (path));
-    if (rc) {
-        return rc;
-    }
+    xen_host_pci_sysfs_path(d, "config", path, sizeof(path));
+
     d->config_fd = open(path, O_RDWR);
-    if (d->config_fd < 0) {
-        return -errno;
+    if (d->config_fd == -1) {
+        error_setg_file_open(errp, errno, path);
     }
-    return 0;
 }
 
 static int xen_host_pci_config_read(XenHostPCIDevice *d,
@@ -341,11 +330,12 @@
     return -1;
 }
 
-int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
-                            uint8_t bus, uint8_t dev, uint8_t func)
+void xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
+                             uint8_t bus, uint8_t dev, uint8_t func,
+                             Error **errp)
 {
     unsigned int v;
-    int rc = 0;
+    Error *err = NULL;
 
     d->config_fd = -1;
     d->domain = domain;
@@ -353,38 +343,56 @@
     d->dev = dev;
     d->func = func;
 
-    rc = xen_host_pci_config_open(d);
-    if (rc) {
+    xen_host_pci_config_open(d, &err);
+    if (err) {
         goto error;
     }
-    rc = xen_host_pci_get_resource(d);
-    if (rc) {
+
+    xen_host_pci_get_resource(d, &err);
+    if (err) {
         goto error;
     }
-    rc = xen_host_pci_get_hex_value(d, "vendor", &v);
-    if (rc) {
+
+    xen_host_pci_get_hex_value(d, "vendor", &v, &err);
+    if (err) {
         goto error;
     }
     d->vendor_id = v;
-    rc = xen_host_pci_get_hex_value(d, "device", &v);
-    if (rc) {
+
+    xen_host_pci_get_hex_value(d, "device", &v, &err);
+    if (err) {
         goto error;
     }
     d->device_id = v;
-    rc = xen_host_pci_get_dec_value(d, "irq", &v);
-    if (rc) {
+
+    xen_host_pci_get_dec_value(d, "irq", &v, &err);
+    if (err) {
         goto error;
     }
     d->irq = v;
+
+    xen_host_pci_get_hex_value(d, "class", &v, &err);
+    if (err) {
+        goto error;
+    }
+    d->class_code = v;
+
     d->is_virtfn = xen_host_pci_dev_is_virtfn(d);
 
-    return 0;
+    return;
+
 error:
+    error_propagate(errp, err);
+
     if (d->config_fd >= 0) {
         close(d->config_fd);
         d->config_fd = -1;
     }
-    return rc;
+}
+
+bool xen_host_pci_device_closed(XenHostPCIDevice *d)
+{
+    return d->config_fd == -1;
 }
 
 void xen_host_pci_device_put(XenHostPCIDevice *d)
diff --git a/hw/xen/xen-host-pci-device.h b/hw/xen/xen-host-pci-device.h
index c2486f0..4d8d34e 100644
--- a/hw/xen/xen-host-pci-device.h
+++ b/hw/xen/xen-host-pci-device.h
@@ -25,6 +25,7 @@
 
     uint16_t vendor_id;
     uint16_t device_id;
+    uint32_t class_code;
     int irq;
 
     XenHostPCIIORegion io_regions[PCI_NUM_REGIONS - 1];
@@ -35,9 +36,11 @@
     int config_fd;
 } XenHostPCIDevice;
 
-int xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
-                            uint8_t bus, uint8_t dev, uint8_t func);
+void xen_host_pci_device_get(XenHostPCIDevice *d, uint16_t domain,
+                             uint8_t bus, uint8_t dev, uint8_t func,
+                             Error **errp);
 void xen_host_pci_device_put(XenHostPCIDevice *pci_dev);
+bool xen_host_pci_device_closed(XenHostPCIDevice *d);
 
 int xen_host_pci_get_byte(XenHostPCIDevice *d, int pos, uint8_t *p);
 int xen_host_pci_get_word(XenHostPCIDevice *d, int pos, uint16_t *p);
@@ -52,4 +55,4 @@
 
 int xen_host_pci_find_ext_cap_offset(XenHostPCIDevice *s, uint32_t cap);
 
-#endif /* !XEN_HOST_PCI_DEVICE_H_ */
+#endif /* XEN_HOST_PCI_DEVICE_H */
diff --git a/hw/xen/xen_backend.c b/hw/xen/xen_backend.c
index b2cb22b..69a2388 100644
--- a/hw/xen/xen_backend.c
+++ b/hw/xen/xen_backend.c
@@ -22,38 +22,60 @@
  * TODO: add some xenbus / xenstore concepts overview here.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <inttypes.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
+#include "qemu/osdep.h"
 #include <sys/signal.h>
 
 #include "hw/hw.h"
+#include "hw/sysbus.h"
 #include "sysemu/char.h"
 #include "qemu/log.h"
 #include "hw/xen/xen_backend.h"
 
 #include <xen/grant_table.h>
 
+#define TYPE_XENSYSDEV "xensysdev"
+
+DeviceState *xen_sysdev;
+
 /* ------------------------------------------------------------- */
 
 /* public */
-XenXC xen_xc = XC_HANDLER_INITIAL_VALUE;
+xc_interface *xen_xc = NULL;
+xenforeignmemory_handle *xen_fmem = NULL;
 struct xs_handle *xenstore = NULL;
 const char *xen_protocol;
 
 /* private */
+struct xs_dirs {
+    char *xs_dir;
+    QTAILQ_ENTRY(xs_dirs) list;
+};
+static QTAILQ_HEAD(xs_dirs_head, xs_dirs) xs_cleanup =
+    QTAILQ_HEAD_INITIALIZER(xs_cleanup);
+
 static QTAILQ_HEAD(XenDeviceHead, XenDevice) xendevs = QTAILQ_HEAD_INITIALIZER(xendevs);
 static int debug = 0;
 
 /* ------------------------------------------------------------- */
 
+static void xenstore_cleanup_dir(char *dir)
+{
+    struct xs_dirs *d;
+
+    d = g_malloc(sizeof(*d));
+    d->xs_dir = dir;
+    QTAILQ_INSERT_TAIL(&xs_cleanup, d, list);
+}
+
+void xen_config_cleanup(void)
+{
+    struct xs_dirs *d;
+
+    QTAILQ_FOREACH(d, &xs_cleanup, list) {
+        xs_rm(xenstore, 0, d->xs_dir);
+    }
+}
+
 int xenstore_write_str(const char *base, const char *node, const char *val)
 {
     char abspath[XEN_BUFSIZE];
@@ -82,6 +104,30 @@
     return ret;
 }
 
+int xenstore_mkdir(char *path, int p)
+{
+    struct xs_permissions perms[2] = {
+        {
+            .id    = 0, /* set owner: dom0 */
+        }, {
+            .id    = xen_domid,
+            .perms = p,
+        }
+    };
+
+    if (!xs_mkdir(xenstore, 0, path)) {
+        xen_be_printf(NULL, 0, "xs_mkdir %s: failed\n", path);
+        return -1;
+    }
+    xenstore_cleanup_dir(g_strdup(path));
+
+    if (!xs_set_permissions(xenstore, 0, path, perms, 2)) {
+        xen_be_printf(NULL, 0, "xs_set_permissions %s: failed\n", path);
+        return -1;
+    }
+    return 0;
+}
+
 int xenstore_write_int(const char *base, const char *node, int ival)
 {
     char val[12];
@@ -243,24 +289,24 @@
     xendev->debug      = debug;
     xendev->local_port = -1;
 
-    xendev->evtchndev = xen_xc_evtchn_open(NULL, 0);
-    if (xendev->evtchndev == XC_HANDLER_INITIAL_VALUE) {
+    xendev->evtchndev = xenevtchn_open(NULL, 0);
+    if (xendev->evtchndev == NULL) {
         xen_be_printf(NULL, 0, "can't open evtchn device\n");
         g_free(xendev);
         return NULL;
     }
-    fcntl(xc_evtchn_fd(xendev->evtchndev), F_SETFD, FD_CLOEXEC);
+    fcntl(xenevtchn_fd(xendev->evtchndev), F_SETFD, FD_CLOEXEC);
 
     if (ops->flags & DEVOPS_FLAG_NEED_GNTDEV) {
-        xendev->gnttabdev = xen_xc_gnttab_open(NULL, 0);
-        if (xendev->gnttabdev == XC_HANDLER_INITIAL_VALUE) {
+        xendev->gnttabdev = xengnttab_open(NULL, 0);
+        if (xendev->gnttabdev == NULL) {
             xen_be_printf(NULL, 0, "can't open gnttab device\n");
-            xc_evtchn_close(xendev->evtchndev);
+            xenevtchn_close(xendev->evtchndev);
             g_free(xendev);
             return NULL;
         }
     } else {
-        xendev->gnttabdev = XC_HANDLER_INITIAL_VALUE;
+        xendev->gnttabdev = NULL;
     }
 
     QTAILQ_INSERT_TAIL(&xendevs, xendev, next);
@@ -275,48 +321,28 @@
 /*
  * release xen backend device.
  */
-static struct XenDevice *xen_be_del_xendev(int dom, int dev)
+static void xen_be_del_xendev(struct XenDevice *xendev)
 {
-    struct XenDevice *xendev, *xnext;
-
-    /*
-     * This is pretty much like QTAILQ_FOREACH(xendev, &xendevs, next) but
-     * we save the next pointer in xnext because we might free xendev.
-     */
-    xnext = xendevs.tqh_first;
-    while (xnext) {
-        xendev = xnext;
-        xnext = xendev->next.tqe_next;
-
-        if (xendev->dom != dom) {
-            continue;
-        }
-        if (xendev->dev != dev && dev != -1) {
-            continue;
-        }
-
-        if (xendev->ops->free) {
-            xendev->ops->free(xendev);
-        }
-
-        if (xendev->fe) {
-            char token[XEN_BUFSIZE];
-            snprintf(token, sizeof(token), "fe:%p", xendev);
-            xs_unwatch(xenstore, xendev->fe, token);
-            g_free(xendev->fe);
-        }
-
-        if (xendev->evtchndev != XC_HANDLER_INITIAL_VALUE) {
-            xc_evtchn_close(xendev->evtchndev);
-        }
-        if (xendev->gnttabdev != XC_HANDLER_INITIAL_VALUE) {
-            xc_gnttab_close(xendev->gnttabdev);
-        }
-
-        QTAILQ_REMOVE(&xendevs, xendev, next);
-        g_free(xendev);
+    if (xendev->ops->free) {
+        xendev->ops->free(xendev);
     }
-    return NULL;
+
+    if (xendev->fe) {
+        char token[XEN_BUFSIZE];
+        snprintf(token, sizeof(token), "fe:%p", xendev);
+        xs_unwatch(xenstore, xendev->fe, token);
+        g_free(xendev->fe);
+    }
+
+    if (xendev->evtchndev != NULL) {
+        xenevtchn_close(xendev->evtchndev);
+    }
+    if (xendev->gnttabdev != NULL) {
+        xengnttab_close(xendev->gnttabdev);
+    }
+
+    QTAILQ_REMOVE(&xendevs, xendev, next);
+    g_free(xendev);
 }
 
 /*
@@ -636,7 +662,7 @@
     if (xendev != NULL) {
         bepath = xs_read(xenstore, 0, xendev->be, &len);
         if (bepath == NULL) {
-            xen_be_del_xendev(dom, dev);
+            xen_be_del_xendev(xendev);
         } else {
             free(bepath);
             xen_be_backend_changed(xendev, path);
@@ -691,13 +717,14 @@
     struct XenDevice *xendev = opaque;
     evtchn_port_t port;
 
-    port = xc_evtchn_pending(xendev->evtchndev);
+    port = xenevtchn_pending(xendev->evtchndev);
     if (port != xendev->local_port) {
-        xen_be_printf(xendev, 0, "xc_evtchn_pending returned %d (expected %d)\n",
+        xen_be_printf(xendev, 0,
+                      "xenevtchn_pending returned %d (expected %d)\n",
                       port, xendev->local_port);
         return;
     }
-    xc_evtchn_unmask(xendev->evtchndev, port);
+    xenevtchn_unmask(xendev->evtchndev, port);
 
     if (xendev->ops->event) {
         xendev->ops->event(xendev);
@@ -714,14 +741,16 @@
         return -1;
     }
 
-    if (qemu_set_fd_handler(xs_fileno(xenstore), xenstore_update, NULL, NULL) < 0) {
-        goto err;
-    }
+    qemu_set_fd_handler(xs_fileno(xenstore), xenstore_update, NULL, NULL);
 
-    if (xen_xc == XC_HANDLER_INITIAL_VALUE) {
+    if (xen_xc == NULL || xen_fmem == NULL) {
         /* Check if xen_init() have been called */
         goto err;
     }
+
+    xen_sysdev = qdev_create(NULL, TYPE_XENSYSDEV);
+    qdev_init_nofail(xen_sysdev);
+
     return 0;
 
 err:
@@ -734,22 +763,46 @@
 
 int xen_be_register(const char *type, struct XenDevOps *ops)
 {
+    char path[50];
+    int rc;
+
+    if (ops->backend_register) {
+        rc = ops->backend_register();
+        if (rc) {
+            return rc;
+        }
+    }
+
+    snprintf(path, sizeof(path), "device-model/%u/backends/%s", xen_domid,
+             type);
+    xenstore_mkdir(path, XS_PERM_NONE);
+
     return xenstore_scan(type, xen_domid, ops);
 }
 
+void xen_be_register_common(void)
+{
+    xen_be_register("console", &xen_console_ops);
+    xen_be_register("vkbd", &xen_kbdmouse_ops);
+    xen_be_register("qdisk", &xen_blkdev_ops);
+#ifdef CONFIG_USB_LIBUSB
+    xen_be_register("qusb", &xen_usb_ops);
+#endif
+}
+
 int xen_be_bind_evtchn(struct XenDevice *xendev)
 {
     if (xendev->local_port != -1) {
         return 0;
     }
-    xendev->local_port = xc_evtchn_bind_interdomain
+    xendev->local_port = xenevtchn_bind_interdomain
         (xendev->evtchndev, xendev->dom, xendev->remote_port);
     if (xendev->local_port == -1) {
-        xen_be_printf(xendev, 0, "xc_evtchn_bind_interdomain failed\n");
+        xen_be_printf(xendev, 0, "xenevtchn_bind_interdomain failed\n");
         return -1;
     }
     xen_be_printf(xendev, 2, "bind evtchn port %d\n", xendev->local_port);
-    qemu_set_fd_handler(xc_evtchn_fd(xendev->evtchndev),
+    qemu_set_fd_handler(xenevtchn_fd(xendev->evtchndev),
                         xen_be_evtchn_event, NULL, xendev);
     return 0;
 }
@@ -759,15 +812,15 @@
     if (xendev->local_port == -1) {
         return;
     }
-    qemu_set_fd_handler(xc_evtchn_fd(xendev->evtchndev), NULL, NULL, NULL);
-    xc_evtchn_unbind(xendev->evtchndev, xendev->local_port);
+    qemu_set_fd_handler(xenevtchn_fd(xendev->evtchndev), NULL, NULL, NULL);
+    xenevtchn_unbind(xendev->evtchndev, xendev->local_port);
     xen_be_printf(xendev, 2, "unbind evtchn port %d\n", xendev->local_port);
     xendev->local_port = -1;
 }
 
 int xen_be_send_notify(struct XenDevice *xendev)
 {
-    return xc_evtchn_notify(xendev->evtchndev, xendev->local_port);
+    return xenevtchn_notify(xendev->evtchndev, xendev->local_port);
 }
 
 /*
@@ -808,3 +861,35 @@
     }
     qemu_log_flush();
 }
+
+static int xen_sysdev_init(SysBusDevice *dev)
+{
+    return 0;
+}
+
+static Property xen_sysdev_properties[] = {
+    {/* end of property list */},
+};
+
+static void xen_sysdev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = xen_sysdev_init;
+    dc->props = xen_sysdev_properties;
+}
+
+static const TypeInfo xensysdev_info = {
+    .name          = TYPE_XENSYSDEV,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(SysBusDevice),
+    .class_init    = xen_sysdev_class_init,
+};
+
+static void xenbe_register_types(void)
+{
+    type_register_static(&xensysdev_info);
+}
+
+type_init(xenbe_register_types);
diff --git a/hw/xen/xen_devconfig.c b/hw/xen/xen_devconfig.c
index e138dbb..b7d290d 100644
--- a/hw/xen/xen_devconfig.c
+++ b/hw/xen/xen_devconfig.c
@@ -1,57 +1,10 @@
+#include "qemu/osdep.h"
 #include "hw/xen/xen_backend.h"
 #include "sysemu/block-backend.h"
 #include "sysemu/blockdev.h"
 
 /* ------------------------------------------------------------- */
 
-struct xs_dirs {
-    char *xs_dir;
-    QTAILQ_ENTRY(xs_dirs) list;
-};
-static QTAILQ_HEAD(xs_dirs_head, xs_dirs) xs_cleanup = QTAILQ_HEAD_INITIALIZER(xs_cleanup);
-
-static void xen_config_cleanup_dir(char *dir)
-{
-    struct xs_dirs *d;
-
-    d = g_malloc(sizeof(*d));
-    d->xs_dir = dir;
-    QTAILQ_INSERT_TAIL(&xs_cleanup, d, list);
-}
-
-void xen_config_cleanup(void)
-{
-    struct xs_dirs *d;
-
-    QTAILQ_FOREACH(d, &xs_cleanup, list) {
-	xs_rm(xenstore, 0, d->xs_dir);
-    }
-}
-
-/* ------------------------------------------------------------- */
-
-static int xen_config_dev_mkdir(char *dev, int p)
-{
-    struct xs_permissions perms[2] = {{
-            .id    = 0, /* set owner: dom0 */
-        },{
-            .id    = xen_domid,
-            .perms = p,
-        }};
-
-    if (!xs_mkdir(xenstore, 0, dev)) {
-	xen_be_printf(NULL, 0, "xs_mkdir %s: failed\n", dev);
-	return -1;
-    }
-    xen_config_cleanup_dir(g_strdup(dev));
-
-    if (!xs_set_permissions(xenstore, 0, dev, perms, 2)) {
-	xen_be_printf(NULL, 0, "xs_set_permissions %s: failed\n", dev);
-	return -1;
-    }
-    return 0;
-}
-
 static int xen_config_dev_dirs(const char *ftype, const char *btype, int vdev,
 			       char *fe, char *be, int len)
 {
@@ -65,8 +18,8 @@
     snprintf(be, len, "%s/backend/%s/%d/%d", dom, btype, xen_domid, vdev);
     free(dom);
 
-    xen_config_dev_mkdir(fe, XS_PERM_READ | XS_PERM_WRITE);
-    xen_config_dev_mkdir(be, XS_PERM_READ);
+    xenstore_mkdir(fe, XS_PERM_READ | XS_PERM_WRITE);
+    xenstore_mkdir(be, XS_PERM_READ);
     return 0;
 }
 
diff --git a/hw/xen/xen_pt.c b/hw/xen/xen_pt.c
index c1bf357..b6d71bb 100644
--- a/hw/xen/xen_pt.c
+++ b/hw/xen/xen_pt.c
@@ -52,10 +52,13 @@
  *         - Set entry->pirq to '-1'.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
 
 #include "hw/pci/pci.h"
 #include "hw/xen/xen.h"
+#include "hw/i386/pc.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_pt.h"
 #include "qemu/range.h"
@@ -125,7 +128,7 @@
 
 static uint32_t xen_pt_pci_read_config(PCIDevice *d, uint32_t addr, int len)
 {
-    XenPCIPassthroughState *s = DO_UPCAST(XenPCIPassthroughState, dev, d);
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
     uint32_t val = 0;
     XenPTRegGroup *reg_grp_entry = NULL;
     XenPTReg *reg_entry = NULL;
@@ -230,15 +233,16 @@
 static void xen_pt_pci_write_config(PCIDevice *d, uint32_t addr,
                                     uint32_t val, int len)
 {
-    XenPCIPassthroughState *s = DO_UPCAST(XenPCIPassthroughState, dev, d);
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
     int index = 0;
     XenPTRegGroup *reg_grp_entry = NULL;
     int rc = 0;
-    uint32_t read_val = 0;
+    uint32_t read_val = 0, wb_mask;
     int emul_len = 0;
     XenPTReg *reg_entry = NULL;
     uint32_t find_addr = addr;
     XenPTRegInfo *reg = NULL;
+    bool wp_flag = false;
 
     if (xen_pt_pci_config_access_check(d, addr, len)) {
         return;
@@ -248,10 +252,18 @@
 
     /* check unused BAR register */
     index = xen_pt_bar_offset_to_index(addr);
-    if ((index >= 0) && (val > 0 && val < XEN_PT_BAR_ALLF) &&
-        (s->bases[index].bar_flag == XEN_PT_BAR_FLAG_UNUSED)) {
-        XEN_PT_WARN(d, "Guest attempt to set address to unused Base Address "
-                    "Register. (addr: 0x%02x, len: %d)\n", addr, len);
+    if ((index >= 0) && (val != 0)) {
+        uint32_t chk = val;
+
+        if (index == PCI_ROM_SLOT)
+            chk |= (uint32_t)~PCI_ROM_ADDRESS_MASK;
+
+        if ((chk != XEN_PT_BAR_ALLF) &&
+            (s->bases[index].bar_flag == XEN_PT_BAR_FLAG_UNUSED)) {
+            XEN_PT_WARN(d, "Guest attempt to set address to unused "
+                        "Base Address Register. (addr: 0x%02x, len: %d)\n",
+                        addr, len);
+        }
     }
 
     /* find register group entry */
@@ -271,10 +283,17 @@
     if (rc < 0) {
         XEN_PT_ERR(d, "pci_read_block failed. return value: %d.\n", rc);
         memset(&read_val, 0xff, len);
+        wb_mask = 0;
+    } else {
+        wb_mask = 0xFFFFFFFF >> ((4 - len) << 3);
     }
 
     /* pass directly to the real device for passthrough type register group */
     if (reg_grp_entry == NULL) {
+        if (!s->permissive) {
+            wb_mask = 0;
+            wp_flag = true;
+        }
         goto out;
     }
 
@@ -295,9 +314,17 @@
             uint32_t real_offset = reg_grp_entry->base_offset + reg->offset;
             uint32_t valid_mask = 0xFFFFFFFF >> ((4 - emul_len) << 3);
             uint8_t *ptr_val = NULL;
+            uint32_t wp_mask = reg->emu_mask | reg->ro_mask;
 
             valid_mask <<= (find_addr - real_offset) << 3;
             ptr_val = (uint8_t *)&val + (real_offset & 3);
+            if (!s->permissive) {
+                wp_mask |= reg->res_mask;
+            }
+            if (wp_mask == (0xFFFFFFFF >> ((4 - reg->size) << 3))) {
+                wb_mask &= ~((wp_mask >> ((find_addr - real_offset) << 3))
+                             << ((len - emul_len) << 3));
+            }
 
             /* do emulation based on register size */
             switch (reg->size) {
@@ -339,24 +366,50 @@
         } else {
             /* nothing to do with passthrough type register,
              * continue to find next byte */
+            if (!s->permissive) {
+                wb_mask &= ~(0xff << ((len - emul_len) << 3));
+                /* Unused BARs will make it here, but we don't want to issue
+                 * warnings for writes to them (bogus writes get dealt with
+                 * above).
+                 */
+                if (index < 0) {
+                    wp_flag = true;
+                }
+            }
             emul_len--;
             find_addr++;
         }
     }
 
-    /* need to shift back before passing them to xen_host_pci_device */
+    /* need to shift back before passing them to xen_host_pci_set_block. */
     val >>= (addr & 3) << 3;
 
     memory_region_transaction_commit();
 
 out:
-    if (!(reg && reg->no_wb)) {
+    if (wp_flag && !s->permissive_warned) {
+        s->permissive_warned = true;
+        xen_pt_log(d, "Write-back to unknown field 0x%02x (partially) inhibited (0x%0*x)\n",
+                   addr, len * 2, wb_mask);
+        xen_pt_log(d, "If the device doesn't work, try enabling permissive mode\n");
+        xen_pt_log(d, "(unsafe) and if it helps report the problem to xen-devel\n");
+    }
+    for (index = 0; wb_mask; index += len) {
         /* unknown regs are passed through */
-        rc = xen_host_pci_set_block(&s->real_device, addr,
-                                    (uint8_t *)&val, len);
+        while (!(wb_mask & 0xff)) {
+            index++;
+            wb_mask >>= 8;
+        }
+        len = 0;
+        do {
+            len++;
+            wb_mask >>= 8;
+        } while (wb_mask & 0xff);
+        rc = xen_host_pci_set_block(&s->real_device, addr + index,
+                                    (uint8_t *)&val + index, len);
 
         if (rc < 0) {
-            XEN_PT_ERR(d, "pci_write_block failed. return value: %d.\n", rc);
+            XEN_PT_ERR(d, "xen_host_pci_set_block failed. return value: %d.\n", rc);
         }
     }
 }
@@ -388,7 +441,7 @@
     .write = xen_pt_bar_write,
 };
 
-static int xen_pt_register_regions(XenPCIPassthroughState *s)
+static int xen_pt_register_regions(XenPCIPassthroughState *s, uint16_t *cmd)
 {
     int i = 0;
     XenHostPCIDevice *d = &s->real_device;
@@ -406,6 +459,7 @@
 
         if (r->type & XEN_HOST_PCI_REGION_TYPE_IO) {
             type = PCI_BASE_ADDRESS_SPACE_IO;
+            *cmd |= PCI_COMMAND_IO;
         } else {
             type = PCI_BASE_ADDRESS_SPACE_MEMORY;
             if (r->type & XEN_HOST_PCI_REGION_TYPE_PREFETCH) {
@@ -414,6 +468,7 @@
             if (r->type & XEN_HOST_PCI_REGION_TYPE_MEM_64) {
                 type |= PCI_BASE_ADDRESS_MEM_TYPE_64;
             }
+            *cmd |= PCI_COMMAND_MEMORY;
         }
 
         memory_region_init_io(&s->bar[i], OBJECT(s), &ops, &s->dev,
@@ -450,6 +505,7 @@
                    d->rom.size, d->rom.base_addr);
     }
 
+    xen_pt_register_vga_regions(d);
     return 0;
 }
 
@@ -563,8 +619,8 @@
                                       guest_port, machine_port, size,
                                       op);
         if (rc) {
-            XEN_PT_ERR(d, "%s ioport mapping failed! (rc: %i)\n",
-                       adding ? "create new" : "remove old", rc);
+            XEN_PT_ERR(d, "%s ioport mapping failed! (err: %i)\n",
+                       adding ? "create new" : "remove old", errno);
         }
     } else {
         pcibus_t guest_addr = sec->offset_within_address_space;
@@ -577,8 +633,8 @@
                                       XEN_PFN(size + XC_PAGE_SIZE - 1),
                                       op);
         if (rc) {
-            XEN_PT_ERR(d, "%s mem mapping failed! (rc: %i)\n",
-                       adding ? "create new" : "remove old", rc);
+            XEN_PT_ERR(d, "%s mem mapping failed! (err: %i)\n",
+                       adding ? "create new" : "remove old", errno);
         }
     }
 }
@@ -631,14 +687,89 @@
     .priority = 10,
 };
 
+static void
+xen_igd_passthrough_isa_bridge_create(XenPCIPassthroughState *s,
+                                      XenHostPCIDevice *dev)
+{
+    uint16_t gpu_dev_id;
+    PCIDevice *d = &s->dev;
+
+    gpu_dev_id = dev->device_id;
+    igd_passthrough_isa_bridge_create(d->bus, gpu_dev_id);
+}
+
+/* destroy. */
+static void xen_pt_destroy(PCIDevice *d) {
+
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
+    XenHostPCIDevice *host_dev = &s->real_device;
+    uint8_t machine_irq = s->machine_irq;
+    uint8_t intx;
+    int rc;
+
+    if (machine_irq && !xen_host_pci_device_closed(&s->real_device)) {
+        intx = xen_pt_pci_intx(s);
+        rc = xc_domain_unbind_pt_irq(xen_xc, xen_domid, machine_irq,
+                                     PT_IRQ_TYPE_PCI,
+                                     pci_bus_num(d->bus),
+                                     PCI_SLOT(s->dev.devfn),
+                                     intx,
+                                     0 /* isa_irq */);
+        if (rc < 0) {
+            XEN_PT_ERR(d, "unbinding of interrupt INT%c failed."
+                       " (machine irq: %i, err: %d)"
+                       " But bravely continuing on..\n",
+                       'a' + intx, machine_irq, errno);
+        }
+    }
+
+    /* N.B. xen_pt_config_delete takes care of freeing them. */
+    if (s->msi) {
+        xen_pt_msi_disable(s);
+    }
+    if (s->msix) {
+        xen_pt_msix_disable(s);
+    }
+
+    if (machine_irq) {
+        xen_pt_mapped_machine_irq[machine_irq]--;
+
+        if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
+            rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq);
+
+            if (rc < 0) {
+                XEN_PT_ERR(d, "unmapping of interrupt %i failed. (err: %d)"
+                           " But bravely continuing on..\n",
+                           machine_irq, errno);
+            }
+        }
+        s->machine_irq = 0;
+    }
+
+    /* delete all emulated config registers */
+    xen_pt_config_delete(s);
+
+    xen_pt_unregister_vga_regions(host_dev);
+
+    if (s->listener_set) {
+        memory_listener_unregister(&s->memory_listener);
+        memory_listener_unregister(&s->io_listener);
+        s->listener_set = false;
+    }
+    if (!xen_host_pci_device_closed(&s->real_device)) {
+        xen_host_pci_device_put(&s->real_device);
+    }
+}
 /* init */
 
-static int xen_pt_initfn(PCIDevice *d)
+static void xen_pt_realize(PCIDevice *d, Error **errp)
 {
-    XenPCIPassthroughState *s = DO_UPCAST(XenPCIPassthroughState, dev, d);
-    int rc = 0;
-    uint8_t machine_irq = 0;
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(d);
+    int i, rc = 0;
+    uint8_t machine_irq = 0, scratch;
+    uint16_t cmd = 0;
     int pirq = XEN_PT_UNASSIGNED_PIRQ;
+    Error *err = NULL;
 
     /* register real device */
     XEN_PT_LOG(d, "Assigning real physical device %02x:%02x.%d"
@@ -646,12 +777,14 @@
                s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function,
                s->dev.devfn);
 
-    rc = xen_host_pci_device_get(&s->real_device,
-                                 s->hostaddr.domain, s->hostaddr.bus,
-                                 s->hostaddr.slot, s->hostaddr.function);
-    if (rc) {
-        XEN_PT_ERR(d, "Failed to \"open\" the real pci device. rc: %i\n", rc);
-        return -1;
+    xen_host_pci_device_get(&s->real_device,
+                            s->hostaddr.domain, s->hostaddr.bus,
+                            s->hostaddr.slot, s->hostaddr.function,
+                            &err);
+    if (err) {
+        error_append_hint(&err, "Failed to \"open\" the real pci device");
+        error_propagate(errp, err);
+        return;
     }
 
     s->is_virtfn = s->real_device.is_virtfn;
@@ -662,43 +795,65 @@
     }
 
     /* Initialize virtualized PCI configuration (Extended 256 Bytes) */
-    if (xen_host_pci_get_block(&s->real_device, 0, d->config,
-                               PCI_CONFIG_SPACE_SIZE) == -1) {
-        xen_host_pci_device_put(&s->real_device);
-        return -1;
-    }
+    memset(d->config, 0, PCI_CONFIG_SPACE_SIZE);
 
     s->memory_listener = xen_pt_memory_listener;
     s->io_listener = xen_pt_io_listener;
 
+    /* Setup VGA bios for passthrough GFX */
+    if ((s->real_device.domain == 0) && (s->real_device.bus == 0) &&
+        (s->real_device.dev == 2) && (s->real_device.func == 0)) {
+        if (!is_igd_vga_passthrough(&s->real_device)) {
+            error_setg(errp, "Need to enable igd-passthru if you're trying"
+                    " to passthrough IGD GFX");
+            xen_host_pci_device_put(&s->real_device);
+            return;
+        }
+
+        xen_pt_setup_vga(s, &s->real_device, &err);
+        if (err) {
+            error_append_hint(&err, "Setup VGA BIOS of passthrough"
+                    " GFX failed");
+            error_propagate(errp, err);
+            xen_host_pci_device_put(&s->real_device);
+            return;
+        }
+
+        /* Register ISA bridge for passthrough GFX. */
+        xen_igd_passthrough_isa_bridge_create(s, &s->real_device);
+    }
+
     /* Handle real device's MMIO/PIO BARs */
-    xen_pt_register_regions(s);
+    xen_pt_register_regions(s, &cmd);
 
     /* reinitialize each config register to be emulated */
-    if (xen_pt_config_init(s)) {
-        XEN_PT_ERR(d, "PCI Config space initialisation failed.\n");
-        xen_host_pci_device_put(&s->real_device);
-        return -1;
+    xen_pt_config_init(s, &err);
+    if (err) {
+        error_append_hint(&err, "PCI Config space initialisation failed");
+        error_report_err(err);
+        rc = -1;
+        goto err_out;
     }
 
     /* Bind interrupt */
-    if (!s->dev.config[PCI_INTERRUPT_PIN]) {
+    rc = xen_host_pci_get_byte(&s->real_device, PCI_INTERRUPT_PIN, &scratch);
+    if (rc) {
+        error_setg_errno(errp, errno, "Failed to read PCI_INTERRUPT_PIN");
+        goto err_out;
+    }
+    if (!scratch) {
         XEN_PT_LOG(d, "no pin interrupt\n");
         goto out;
     }
 
     machine_irq = s->real_device.irq;
     rc = xc_physdev_map_pirq(xen_xc, xen_domid, machine_irq, &pirq);
-
     if (rc < 0) {
-        XEN_PT_ERR(d, "Mapping machine irq %u to pirq %i failed, (rc: %d)\n",
-                   machine_irq, pirq, rc);
+        error_setg_errno(errp, errno, "Mapping machine irq %u to"
+                         " pirq %i failed", machine_irq, pirq);
 
         /* Disable PCI intx assertion (turn on bit10 of devctl) */
-        xen_host_pci_set_word(&s->real_device,
-                              PCI_COMMAND,
-                              pci_get_word(s->dev.config + PCI_COMMAND)
-                              | PCI_COMMAND_INTX_DISABLE);
+        cmd |= PCI_COMMAND_INTX_DISABLE;
         machine_irq = 0;
         s->machine_irq = 0;
     } else {
@@ -716,19 +871,17 @@
                                        PCI_SLOT(d->devfn),
                                        e_intx);
         if (rc < 0) {
-            XEN_PT_ERR(d, "Binding of interrupt %i failed! (rc: %d)\n",
-                       e_intx, rc);
+            error_setg_errno(errp, errno, "Binding of interrupt %u failed",
+                             e_intx);
 
             /* Disable PCI intx assertion (turn on bit10 of devctl) */
-            xen_host_pci_set_word(&s->real_device, PCI_COMMAND,
-                                  *(uint16_t *)(&s->dev.config[PCI_COMMAND])
-                                  | PCI_COMMAND_INTX_DISABLE);
+            cmd |= PCI_COMMAND_INTX_DISABLE;
             xen_pt_mapped_machine_irq[machine_irq]--;
 
             if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
                 if (xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq)) {
-                    XEN_PT_ERR(d, "Unmapping of machine interrupt %i failed!"
-                               " (rc: %d)\n", machine_irq, rc);
+                    error_setg_errno(errp, errno, "Unmapping of machine"
+                            " interrupt %u failed", machine_irq);
                 }
             }
             s->machine_irq = 0;
@@ -736,69 +889,51 @@
     }
 
 out:
-    memory_listener_register(&s->memory_listener, &address_space_memory);
-    memory_listener_register(&s->io_listener, &address_space_io);
-    XEN_PT_LOG(d,
-               "Real physical device %02x:%02x.%d registered successfully!\n",
-               s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function);
+    if (cmd) {
+        uint16_t val;
 
-    return 0;
-}
-
-static void xen_pt_unregister_device(PCIDevice *d)
-{
-    XenPCIPassthroughState *s = DO_UPCAST(XenPCIPassthroughState, dev, d);
-    uint8_t machine_irq = s->machine_irq;
-    uint8_t intx = xen_pt_pci_intx(s);
-    int rc;
-
-    if (machine_irq) {
-        rc = xc_domain_unbind_pt_irq(xen_xc, xen_domid, machine_irq,
-                                     PT_IRQ_TYPE_PCI,
-                                     pci_bus_num(d->bus),
-                                     PCI_SLOT(s->dev.devfn),
-                                     intx,
-                                     0 /* isa_irq */);
-        if (rc < 0) {
-            XEN_PT_ERR(d, "unbinding of interrupt INT%c failed."
-                       " (machine irq: %i, rc: %d)"
-                       " But bravely continuing on..\n",
-                       'a' + intx, machine_irq, rc);
-        }
-    }
-
-    if (s->msi) {
-        xen_pt_msi_disable(s);
-    }
-    if (s->msix) {
-        xen_pt_msix_disable(s);
-    }
-
-    if (machine_irq) {
-        xen_pt_mapped_machine_irq[machine_irq]--;
-
-        if (xen_pt_mapped_machine_irq[machine_irq] == 0) {
-            rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, machine_irq);
-
-            if (rc < 0) {
-                XEN_PT_ERR(d, "unmapping of interrupt %i failed. (rc: %d)"
-                           " But bravely continuing on..\n",
-                           machine_irq, rc);
+        rc = xen_host_pci_get_word(&s->real_device, PCI_COMMAND, &val);
+        if (rc) {
+            error_setg_errno(errp, errno, "Failed to read PCI_COMMAND");
+            goto err_out;
+        } else {
+            val |= cmd;
+            rc = xen_host_pci_set_word(&s->real_device, PCI_COMMAND, val);
+            if (rc) {
+                error_setg_errno(errp, errno, "Failed to write PCI_COMMAND"
+                                 " val = 0x%x", val);
+                goto err_out;
             }
         }
     }
 
-    /* delete all emulated config registers */
-    xen_pt_config_delete(s);
+    memory_listener_register(&s->memory_listener, &s->dev.bus_master_as);
+    memory_listener_register(&s->io_listener, &address_space_io);
+    s->listener_set = true;
+    XEN_PT_LOG(d,
+               "Real physical device %02x:%02x.%d registered successfully\n",
+               s->hostaddr.bus, s->hostaddr.slot, s->hostaddr.function);
 
-    memory_listener_unregister(&s->memory_listener);
-    memory_listener_unregister(&s->io_listener);
+    return;
 
-    xen_host_pci_device_put(&s->real_device);
+err_out:
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        object_unparent(OBJECT(&s->bar[i]));
+    }
+    object_unparent(OBJECT(&s->rom));
+
+    xen_pt_destroy(d);
+    assert(rc);
+}
+
+static void xen_pt_unregister_device(PCIDevice *d)
+{
+    xen_pt_destroy(d);
 }
 
 static Property xen_pci_passthrough_properties[] = {
     DEFINE_PROP_PCI_HOST_DEVADDR("hostaddr", XenPCIPassthroughState, hostaddr),
+    DEFINE_PROP_BOOL("permissive", XenPCIPassthroughState, permissive, false),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -807,7 +942,7 @@
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
-    k->init = xen_pt_initfn;
+    k->realize = xen_pt_realize;
     k->exit = xen_pt_unregister_device;
     k->config_read = xen_pt_pci_read_config;
     k->config_write = xen_pt_pci_write_config;
@@ -816,10 +951,18 @@
     dc->props = xen_pci_passthrough_properties;
 };
 
+static void xen_pci_passthrough_finalize(Object *obj)
+{
+    XenPCIPassthroughState *s = XEN_PT_DEVICE(obj);
+
+    xen_pt_msix_delete(s);
+}
+
 static const TypeInfo xen_pci_passthrough_info = {
-    .name = "xen-pci-passthrough",
+    .name = TYPE_XEN_PT_DEVICE,
     .parent = TYPE_PCI_DEVICE,
     .instance_size = sizeof(XenPCIPassthroughState),
+    .instance_finalize = xen_pci_passthrough_finalize,
     .class_init = xen_pci_passthrough_class_init,
 };
 
diff --git a/hw/xen/xen_pt.h b/hw/xen/xen_pt.h
index 942dc60..191d9ca 100644
--- a/hw/xen/xen_pt.h
+++ b/hw/xen/xen_pt.h
@@ -31,11 +31,18 @@
 /* Helper */
 #define XEN_PFN(x) ((x) >> XC_PAGE_SHIFT)
 
-typedef struct XenPTRegInfo XenPTRegInfo;
+typedef const struct XenPTRegInfo XenPTRegInfo;
 typedef struct XenPTReg XenPTReg;
 
 typedef struct XenPCIPassthroughState XenPCIPassthroughState;
 
+#define TYPE_XEN_PT_DEVICE "xen-pci-passthrough"
+#define XEN_PT_DEVICE(obj) \
+    OBJECT_CHECK(XenPCIPassthroughState, (obj), TYPE_XEN_PT_DEVICE)
+
+uint32_t igd_read_opregion(XenPCIPassthroughState *s);
+void igd_write_opregion(XenPCIPassthroughState *s, uint32_t val);
+
 /* function type for config reg */
 typedef int (*xen_pt_conf_reg_init)
     (XenPCIPassthroughState *, XenPTRegInfo *, uint32_t real_offset,
@@ -62,8 +69,9 @@
 #define XEN_PT_BAR_ALLF 0xFFFFFFFF
 #define XEN_PT_BAR_UNMAPPED (-1)
 
-#define PCI_CAP_MAX 48
+#define XEN_PCI_CAP_MAX 48
 
+#define XEN_PCI_INTEL_OPREGION 0xfc
 
 typedef enum {
     XEN_PT_GRP_TYPE_HARDWIRED = 0,  /* 0 Hardwired reg group */
@@ -101,12 +109,14 @@
     uint32_t offset;
     uint32_t size;
     uint32_t init_val;
+    /* reg reserved field mask (ON:reserved, OFF:defined) */
+    uint32_t res_mask;
     /* reg read only field mask (ON:RO/ROS, OFF:other) */
     uint32_t ro_mask;
+    /* reg read/write-1-clear field mask (ON:RW1C/RW1CS, OFF:other) */
+    uint32_t rw1c_mask;
     /* reg emulate field mask (ON:emu, OFF:passthrough) */
     uint32_t emu_mask;
-    /* no write back allowed */
-    uint32_t no_wb;
     xen_pt_conf_reg_init init;
     /* read/write function pointer
      * for double_word/word/byte size */
@@ -130,14 +140,18 @@
 struct XenPTReg {
     QLIST_ENTRY(XenPTReg) entries;
     XenPTRegInfo *reg;
-    uint32_t data; /* emulated value */
+    union {
+        uint8_t *byte;
+        uint16_t *half_word;
+        uint32_t *word;
+    } ptr; /* pointer to dev.config. */
 };
 
-typedef struct XenPTRegGroupInfo XenPTRegGroupInfo;
+typedef const struct XenPTRegGroupInfo XenPTRegGroupInfo;
 
 /* emul reg group size initialize method */
 typedef int (*xen_pt_reg_size_init_fn)
-    (XenPCIPassthroughState *, const XenPTRegGroupInfo *,
+    (XenPCIPassthroughState *, XenPTRegGroupInfo *,
      uint32_t base_offset, uint8_t *size);
 
 /* emulated register group information */
@@ -152,7 +166,7 @@
 /* emul register group management table */
 typedef struct XenPTRegGroup {
     QLIST_ENTRY(XenPTRegGroup) entries;
-    const XenPTRegGroupInfo *reg_grp;
+    XenPTRegGroupInfo *reg_grp;
     uint32_t base_offset;
     uint8_t size;
     QLIST_HEAD(, XenPTReg) reg_tbl_list;
@@ -175,12 +189,13 @@
     int pirq;
     uint64_t addr;
     uint32_t data;
-    uint32_t vector_ctrl;
+    uint32_t latch[4];
     bool updated; /* indicate whether MSI ADDR or DATA is updated */
 } XenPTMSIXEntry;
 typedef struct XenPTMSIX {
     uint32_t ctrl_offset;
     bool enabled;
+    bool maskall;
     int total_entries;
     int bar_index;
     uint64_t table_base;
@@ -196,6 +211,8 @@
 
     PCIHostDeviceAddress hostaddr;
     bool is_virtfn;
+    bool permissive;
+    bool permissive_warned;
     XenHostPCIDevice real_device;
     XenPTRegion bases[PCI_NUM_REGIONS]; /* Access regions */
     QLIST_HEAD(, XenPTRegGroup) reg_grps;
@@ -210,9 +227,10 @@
 
     MemoryListener memory_listener;
     MemoryListener io_listener;
+    bool listener_set;
 };
 
-int xen_pt_config_init(XenPCIPassthroughState *s);
+void xen_pt_config_init(XenPCIPassthroughState *s, Error **errp);
 void xen_pt_config_delete(XenPCIPassthroughState *s);
 XenPTRegGroup *xen_pt_find_reg_grp(XenPCIPassthroughState *s, uint32_t address);
 XenPTReg *xen_pt_find_reg(XenPTRegGroup *reg_grp, uint32_t address);
@@ -275,6 +293,7 @@
                    " value=%i, acceptable range is 1 - 4\n", r_val);
         r_val = 0;
     } else {
+        /* Note that if s.real_device.config_fd is closed we make 0xff. */
         r_val -= 1;
     }
 
@@ -282,13 +301,13 @@
 }
 
 /* MSI/MSI-X */
-int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool en);
 int xen_pt_msi_setup(XenPCIPassthroughState *s);
 int xen_pt_msi_update(XenPCIPassthroughState *d);
 void xen_pt_msi_disable(XenPCIPassthroughState *s);
 
 int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base);
 void xen_pt_msix_delete(XenPCIPassthroughState *s);
+void xen_pt_msix_unmap(XenPCIPassthroughState *s);
 int xen_pt_msix_update(XenPCIPassthroughState *s);
 int xen_pt_msix_update_remap(XenPCIPassthroughState *s, int bar_index);
 void xen_pt_msix_disable(XenPCIPassthroughState *s);
@@ -298,5 +317,19 @@
     return s->msix && s->msix->bar_index == bar;
 }
 
-
-#endif /* !XEN_PT_H */
+extern void *pci_assign_dev_load_option_rom(PCIDevice *dev,
+                                            struct Object *owner, int *size,
+                                            unsigned int domain,
+                                            unsigned int bus, unsigned int slot,
+                                            unsigned int function);
+extern bool has_igd_gfx_passthru;
+static inline bool is_igd_vga_passthrough(XenHostPCIDevice *dev)
+{
+    return (has_igd_gfx_passthru
+            && ((dev->class_code >> 0x8) == PCI_CLASS_DISPLAY_VGA));
+}
+int xen_pt_register_vga_regions(XenHostPCIDevice *dev);
+int xen_pt_unregister_vga_regions(XenHostPCIDevice *dev);
+void xen_pt_setup_vga(XenPCIPassthroughState *s, XenHostPCIDevice *dev,
+                     Error **errp);
+#endif /* XEN_PT_H */
diff --git a/hw/xen/xen_pt_config_init.c b/hw/xen/xen_pt_config_init.c
index de9a20f..6f18366 100644
--- a/hw/xen/xen_pt_config_init.c
+++ b/hw/xen/xen_pt_config_init.c
@@ -12,6 +12,8 @@
  * This file implements direct PCI assignment to a HVM guest
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_pt.h"
@@ -95,6 +97,17 @@
     return NULL;
 }
 
+static uint32_t get_throughable_mask(const XenPCIPassthroughState *s,
+                                     XenPTRegInfo *reg, uint32_t valid_mask)
+{
+    uint32_t throughable_mask = ~(reg->emu_mask | reg->ro_mask);
+
+    if (!s->permissive) {
+        throughable_mask &= ~reg->res_mask;
+    }
+
+    return throughable_mask & valid_mask;
+}
 
 /****************
  * general register functions
@@ -117,10 +130,11 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint8_t valid_emu_mask = 0;
+    uint8_t *data = cfg_entry->ptr.byte;
 
     /* emulate byte register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -129,10 +143,11 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t valid_emu_mask = 0;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* emulate word register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -141,10 +156,11 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t valid_emu_mask = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* emulate long register */
     valid_emu_mask = reg->emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *data, ~valid_emu_mask);
 
     return 0;
 }
@@ -157,15 +173,16 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint8_t writable_mask = 0;
-    uint8_t throughable_mask = 0;
+    uint8_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint8_t *data = cfg_entry->ptr.byte;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~reg->rw1c_mask,
+                              throughable_mask);
 
     return 0;
 }
@@ -175,15 +192,16 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
-    uint16_t throughable_mask = 0;
+    uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~reg->rw1c_mask,
+                              throughable_mask);
 
     return 0;
 }
@@ -193,15 +211,16 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t throughable_mask = 0;
+    uint32_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value & ~reg->rw1c_mask,
+                              throughable_mask);
 
     return 0;
 }
@@ -247,7 +266,7 @@
         reg_entry = xen_pt_find_reg(reg_grp_entry, PCI_CAPABILITY_LIST);
         if (reg_entry) {
             /* check Capabilities Pointer register */
-            if (reg_entry->data) {
+            if (*reg_entry->ptr.half_word) {
                 reg_field |= PCI_STATUS_CAP_LIST;
             } else {
                 reg_field &= ~PCI_STATUS_CAP_LIST;
@@ -286,43 +305,20 @@
 }
 
 /* Command register */
-static int xen_pt_cmd_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
-                               uint16_t *value, uint16_t valid_mask)
-{
-    XenPTRegInfo *reg = cfg_entry->reg;
-    uint16_t valid_emu_mask = 0;
-    uint16_t emu_mask = reg->emu_mask;
-
-    if (s->is_virtfn) {
-        emu_mask |= PCI_COMMAND_MEMORY;
-    }
-
-    /* emulate word register */
-    valid_emu_mask = emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
-
-    return 0;
-}
 static int xen_pt_cmd_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
                                 uint16_t *val, uint16_t dev_value,
                                 uint16_t valid_mask)
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
-    uint16_t throughable_mask = 0;
-    uint16_t emu_mask = reg->emu_mask;
-
-    if (s->is_virtfn) {
-        emu_mask |= PCI_COMMAND_MEMORY;
-    }
+    uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~emu_mask & valid_mask;
-
     if (*val & PCI_COMMAND_INTX_DISABLE) {
         throughable_mask |= PCI_COMMAND_INTX_DISABLE;
     } else {
@@ -360,15 +356,13 @@
 }
 
 static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s,
-                                         XenPTRegInfo *reg)
+                                         int index)
 {
     PCIDevice *d = &s->dev;
     XenPTRegion *region = NULL;
     PCIIORegion *r;
-    int index = 0;
 
     /* check 64bit BAR */
-    index = xen_pt_bar_offset_to_index(reg->offset);
     if ((0 < index) && (index < PCI_ROM_SLOT)) {
         int type = s->real_device.io_regions[index - 1].type;
 
@@ -422,7 +416,7 @@
     }
 
     /* set BAR flag */
-    s->bases[index].bar_flag = xen_pt_bar_reg_parse(s, reg);
+    s->bases[index].bar_flag = xen_pt_bar_reg_parse(s, index);
     if (s->bases[index].bar_flag == XEN_PT_BAR_FLAG_UNUSED) {
         reg_field = XEN_PT_INVALID_REG;
     }
@@ -440,7 +434,7 @@
 
     /* get BAR index */
     index = xen_pt_bar_offset_to_index(reg->offset);
-    if (index < 0 || index >= PCI_NUM_REGIONS) {
+    if (index < 0 || index >= PCI_NUM_REGIONS - 1) {
         XEN_PT_ERR(&s->dev, "Internal error: Invalid BAR index [%d].\n", index);
         return -1;
     }
@@ -465,7 +459,7 @@
 
     /* emulate BAR */
     valid_emu_mask = bar_emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
+    *value = XEN_PT_MERGE_VALUE(*value, *cfg_entry->ptr.word, ~valid_emu_mask);
 
     return 0;
 }
@@ -478,11 +472,11 @@
     PCIDevice *d = &s->dev;
     const PCIIORegion *r;
     uint32_t writable_mask = 0;
-    uint32_t throughable_mask = 0;
     uint32_t bar_emu_mask = 0;
     uint32_t bar_ro_mask = 0;
     uint32_t r_size = 0;
     int index = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     index = xen_pt_bar_offset_to_index(reg->offset);
     if (index < 0 || index >= PCI_NUM_REGIONS) {
@@ -519,7 +513,7 @@
 
     /* modify emulate register */
     writable_mask = bar_emu_mask & ~bar_ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* check whether we need to update the virtual region address or not */
     switch (s->bases[index].bar_flag) {
@@ -535,8 +529,7 @@
     }
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~bar_emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     return 0;
 }
@@ -550,10 +543,10 @@
     XenPTRegion *base = NULL;
     PCIDevice *d = (PCIDevice *)&s->dev;
     uint32_t writable_mask = 0;
-    uint32_t throughable_mask = 0;
+    uint32_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
     pcibus_t r_size = 0;
-    uint32_t bar_emu_mask = 0;
     uint32_t bar_ro_mask = 0;
+    uint32_t *data = cfg_entry->ptr.word;
 
     r_size = d->io_regions[PCI_ROM_SLOT].size;
     base = &s->bases[PCI_ROM_SLOT];
@@ -561,20 +554,34 @@
     r_size = xen_pt_get_emul_size(base->bar_flag, r_size);
 
     /* set emulate mask and read-only mask */
-    bar_emu_mask = reg->emu_mask;
     bar_ro_mask = (reg->ro_mask | (r_size - 1)) & ~PCI_ROM_ADDRESS_ENABLE;
 
     /* modify emulate register */
     writable_mask = ~bar_ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~bar_emu_mask & valid_mask;
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
 
     return 0;
 }
 
+static int xen_pt_intel_opregion_read(XenPCIPassthroughState *s,
+                                      XenPTReg *cfg_entry,
+                                      uint32_t *value, uint32_t valid_mask)
+{
+    *value = igd_read_opregion(s);
+    return 0;
+}
+
+static int xen_pt_intel_opregion_write(XenPCIPassthroughState *s,
+                                       XenPTReg *cfg_entry, uint32_t *value,
+                                       uint32_t dev_value, uint32_t valid_mask)
+{
+    igd_write_opregion(s, *value);
+    return 0;
+}
+
 /* Header Type0 reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_header0[] = {
     /* Vendor ID reg */
@@ -604,10 +611,10 @@
         .offset     = PCI_COMMAND,
         .size       = 2,
         .init_val   = 0x0000,
-        .ro_mask    = 0xF880,
-        .emu_mask   = 0x0740,
+        .res_mask   = 0xF880,
+        .emu_mask   = 0x0743,
         .init       = xen_pt_common_reg_init,
-        .u.w.read   = xen_pt_cmd_reg_read,
+        .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_cmd_reg_write,
     },
     /* Capabilities Pointer reg */
@@ -629,7 +636,9 @@
         .offset     = PCI_STATUS,
         .size       = 2,
         .init_val   = 0x0000,
-        .ro_mask    = 0x06FF,
+        .res_mask   = 0x0007,
+        .ro_mask    = 0x06F8,
+        .rw1c_mask  = 0xF900,
         .emu_mask   = 0x0010,
         .init       = xen_pt_status_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
@@ -750,8 +759,8 @@
         .offset     = PCI_ROM_ADDRESS,
         .size       = 4,
         .init_val   = 0x00000000,
-        .ro_mask    = 0x000007FE,
-        .emu_mask   = 0xFFFFF800,
+        .ro_mask    = ~PCI_ROM_ADDRESS_MASK & ~PCI_ROM_ADDRESS_ENABLE,
+        .emu_mask   = (uint32_t)PCI_ROM_ADDRESS_MASK,
         .init       = xen_pt_bar_reg_init,
         .u.dw.read  = xen_pt_long_reg_read,
         .u.dw.write = xen_pt_exp_rom_bar_reg_write,
@@ -779,6 +788,15 @@
         .u.b.write  = xen_pt_byte_reg_write,
     },
     {
+        .offset     = PCI_VPD_ADDR,
+        .size       = 2,
+        .ro_mask    = 0x0003,
+        .emu_mask   = 0x0003,
+        .init       = xen_pt_common_reg_init,
+        .u.w.read   = xen_pt_word_reg_read,
+        .u.w.write  = xen_pt_word_reg_write,
+    },
+    {
         .size = 0,
     },
 };
@@ -813,15 +831,21 @@
 static inline uint8_t get_capability_version(XenPCIPassthroughState *s,
                                              uint32_t offset)
 {
-    uint8_t flags = pci_get_byte(s->dev.config + offset + PCI_EXP_FLAGS);
-    return flags & PCI_EXP_FLAGS_VERS;
+    uint8_t flag;
+    if (xen_host_pci_get_byte(&s->real_device, offset + PCI_EXP_FLAGS, &flag)) {
+        return 0;
+    }
+    return flag & PCI_EXP_FLAGS_VERS;
 }
 
 static inline uint8_t get_device_type(XenPCIPassthroughState *s,
                                       uint32_t offset)
 {
-    uint8_t flags = pci_get_byte(s->dev.config + offset + PCI_EXP_FLAGS);
-    return (flags & PCI_EXP_FLAGS_TYPE) >> 4;
+    uint8_t flag;
+    if (xen_host_pci_get_byte(&s->real_device, offset + PCI_EXP_FLAGS, &flag)) {
+        return 0;
+    }
+    return (flag & PCI_EXP_FLAGS_TYPE) >> 4;
 }
 
 /* initialize Link Control register */
@@ -870,8 +894,14 @@
         reg_field = XEN_PT_INVALID_REG;
     } else {
         /* set Supported Link Speed */
-        uint8_t lnkcap = pci_get_byte(s->dev.config + real_offset - reg->offset
-                                      + PCI_EXP_LNKCAP);
+        uint8_t lnkcap;
+        int rc;
+        rc = xen_host_pci_get_byte(&s->real_device,
+                                   real_offset - reg->offset + PCI_EXP_LNKCAP,
+                                   &lnkcap);
+        if (rc) {
+            return rc;
+        }
         reg_field |= PCI_EXP_LNKCAP_SLS & lnkcap;
     }
 
@@ -897,7 +927,7 @@
         .offset     = PCI_EXP_DEVCAP,
         .size       = 4,
         .init_val   = 0x00000000,
-        .ro_mask    = 0x1FFCFFFF,
+        .ro_mask    = 0xFFFFFFFF,
         .emu_mask   = 0x10000000,
         .init       = xen_pt_common_reg_init,
         .u.dw.read  = xen_pt_long_reg_read,
@@ -914,6 +944,17 @@
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_word_reg_write,
     },
+    /* Device Status reg */
+    {
+        .offset     = PCI_EXP_DEVSTA,
+        .size       = 2,
+        .res_mask   = 0xFFC0,
+        .ro_mask    = 0x0030,
+        .rw1c_mask  = 0x000F,
+        .init       = xen_pt_common_reg_init,
+        .u.w.read   = xen_pt_word_reg_read,
+        .u.w.write  = xen_pt_word_reg_write,
+    },
     /* Link Control reg */
     {
         .offset     = PCI_EXP_LNKCTL,
@@ -925,6 +966,16 @@
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_word_reg_write,
     },
+    /* Link Status reg */
+    {
+        .offset     = PCI_EXP_LNKSTA,
+        .size       = 2,
+        .ro_mask    = 0x3FFF,
+        .rw1c_mask  = 0xC000,
+        .init       = xen_pt_common_reg_init,
+        .u.w.read   = xen_pt_word_reg_read,
+        .u.w.write  = xen_pt_word_reg_write,
+    },
     /* Device Control 2 reg */
     {
         .offset     = 0x28,
@@ -957,43 +1008,6 @@
  * Power Management Capability
  */
 
-/* read Power Management Control/Status register */
-static int xen_pt_pmcsr_reg_read(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
-                                 uint16_t *value, uint16_t valid_mask)
-{
-    XenPTRegInfo *reg = cfg_entry->reg;
-    uint16_t valid_emu_mask = reg->emu_mask;
-
-    valid_emu_mask |= PCI_PM_CTRL_STATE_MASK | PCI_PM_CTRL_NO_SOFT_RESET;
-
-    valid_emu_mask = valid_emu_mask & valid_mask;
-    *value = XEN_PT_MERGE_VALUE(*value, cfg_entry->data, ~valid_emu_mask);
-
-    return 0;
-}
-/* write Power Management Control/Status register */
-static int xen_pt_pmcsr_reg_write(XenPCIPassthroughState *s,
-                                  XenPTReg *cfg_entry, uint16_t *val,
-                                  uint16_t dev_value, uint16_t valid_mask)
-{
-    XenPTRegInfo *reg = cfg_entry->reg;
-    uint16_t emu_mask = reg->emu_mask;
-    uint16_t writable_mask = 0;
-    uint16_t throughable_mask = 0;
-
-    emu_mask |= PCI_PM_CTRL_STATE_MASK | PCI_PM_CTRL_NO_SOFT_RESET;
-
-    /* modify emulate register */
-    writable_mask = emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-
-    /* create value for writing to I/O device register */
-    throughable_mask = ~emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
-
-    return 0;
-}
-
 /* Power Management Capability reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_pm[] = {
     /* Next Pointer reg */
@@ -1023,11 +1037,13 @@
         .offset     = PCI_PM_CTRL,
         .size       = 2,
         .init_val   = 0x0008,
-        .ro_mask    = 0xE1FC,
-        .emu_mask   = 0x8100,
+        .res_mask   = 0x00F0,
+        .ro_mask    = 0x610C,
+        .rw1c_mask  = 0x8000,
+        .emu_mask   = 0x810B,
         .init       = xen_pt_common_reg_init,
-        .u.w.read   = xen_pt_pmcsr_reg_read,
-        .u.w.write  = xen_pt_pmcsr_reg_write,
+        .u.w.read   = xen_pt_word_reg_read,
+        .u.w.write  = xen_pt_word_reg_write,
     },
     {
         .size = 0,
@@ -1040,26 +1056,24 @@
  */
 
 /* Helper */
-static bool xen_pt_msgdata_check_type(uint32_t offset, uint16_t flags)
-{
-    /* check the offset whether matches the type or not */
-    bool is_32 = (offset == PCI_MSI_DATA_32) && !(flags & PCI_MSI_FLAGS_64BIT);
-    bool is_64 = (offset == PCI_MSI_DATA_64) &&  (flags & PCI_MSI_FLAGS_64BIT);
-    return is_32 || is_64;
-}
+#define xen_pt_msi_check_type(offset, flags, what) \
+        ((offset) == ((flags) & PCI_MSI_FLAGS_64BIT ? \
+                      PCI_MSI_##what##_64 : PCI_MSI_##what##_32))
 
 /* Message Control register */
 static int xen_pt_msgctrl_reg_init(XenPCIPassthroughState *s,
                                    XenPTRegInfo *reg, uint32_t real_offset,
                                    uint32_t *data)
 {
-    PCIDevice *d = &s->dev;
     XenPTMSI *msi = s->msi;
-    uint16_t reg_field = 0;
+    uint16_t reg_field;
+    int rc;
 
     /* use I/O device register's value as initial value */
-    reg_field = pci_get_word(d->config + real_offset);
-
+    rc = xen_host_pci_get_word(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     if (reg_field & PCI_MSI_FLAGS_ENABLE) {
         XEN_PT_LOG(&s->dev, "MSI already enabled, disabling it first\n");
         xen_host_pci_set_word(&s->real_device, real_offset,
@@ -1080,8 +1094,8 @@
     XenPTRegInfo *reg = cfg_entry->reg;
     XenPTMSI *msi = s->msi;
     uint16_t writable_mask = 0;
-    uint16_t throughable_mask = 0;
-    uint16_t raw_val;
+    uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* Currently no support for multi-vector */
     if (*val & PCI_MSI_FLAGS_QSIZE) {
@@ -1090,20 +1104,18 @@
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-    msi->flags |= cfg_entry->data & ~PCI_MSI_FLAGS_ENABLE;
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
+    msi->flags |= *data & ~PCI_MSI_FLAGS_ENABLE;
 
     /* create value for writing to I/O device register */
-    raw_val = *val;
-    throughable_mask = ~reg->emu_mask & valid_mask;
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
 
     /* update MSI */
-    if (raw_val & PCI_MSI_FLAGS_ENABLE) {
+    if (*val & PCI_MSI_FLAGS_ENABLE) {
         /* setup MSI pirq for the first time */
         if (!msi->initialized) {
             /* Init physical one */
-            XEN_PT_LOG(&s->dev, "setup MSI\n");
+            XEN_PT_LOG(&s->dev, "setup MSI (register: %x).\n", *val);
             if (xen_pt_msi_setup(s)) {
                 /* We do not broadcast the error to the framework code, so
                  * that MSI errors are contained in MSI emulation code and
@@ -1111,12 +1123,12 @@
                  * Guest MSI would be actually not working.
                  */
                 *val &= ~PCI_MSI_FLAGS_ENABLE;
-                XEN_PT_WARN(&s->dev, "Can not map MSI.\n");
+                XEN_PT_WARN(&s->dev, "Can not map MSI (register: %x)!\n", *val);
                 return 0;
             }
             if (xen_pt_msi_update(s)) {
                 *val &= ~PCI_MSI_FLAGS_ENABLE;
-                XEN_PT_WARN(&s->dev, "Can not bind MSI\n");
+                XEN_PT_WARN(&s->dev, "Can not bind MSI (register: %x)!\n", *val);
                 return 0;
             }
             msi->initialized = true;
@@ -1127,10 +1139,6 @@
         xen_pt_msi_disable(s);
     }
 
-    /* pass through MSI_ENABLE bit */
-    *val &= ~PCI_MSI_FLAGS_ENABLE;
-    *val |= raw_val & PCI_MSI_FLAGS_ENABLE;
-
     return 0;
 }
 
@@ -1158,7 +1166,45 @@
     uint32_t offset = reg->offset;
 
     /* check the offset whether matches the type or not */
-    if (xen_pt_msgdata_check_type(offset, flags)) {
+    if (xen_pt_msi_check_type(offset, flags, DATA)) {
+        *data = reg->init_val;
+    } else {
+        *data = XEN_PT_INVALID_REG;
+    }
+    return 0;
+}
+
+/* this function will be called twice (for 32 bit and 64 bit type) */
+/* initialize Mask register */
+static int xen_pt_mask_reg_init(XenPCIPassthroughState *s,
+                                XenPTRegInfo *reg, uint32_t real_offset,
+                                uint32_t *data)
+{
+    uint32_t flags = s->msi->flags;
+
+    /* check the offset whether matches the type or not */
+    if (!(flags & PCI_MSI_FLAGS_MASKBIT)) {
+        *data = XEN_PT_INVALID_REG;
+    } else if (xen_pt_msi_check_type(reg->offset, flags, MASK)) {
+        *data = reg->init_val;
+    } else {
+        *data = XEN_PT_INVALID_REG;
+    }
+    return 0;
+}
+
+/* this function will be called twice (for 32 bit and 64 bit type) */
+/* initialize Pending register */
+static int xen_pt_pending_reg_init(XenPCIPassthroughState *s,
+                                   XenPTRegInfo *reg, uint32_t real_offset,
+                                   uint32_t *data)
+{
+    uint32_t flags = s->msi->flags;
+
+    /* check the offset whether matches the type or not */
+    if (!(flags & PCI_MSI_FLAGS_MASKBIT)) {
+        *data = XEN_PT_INVALID_REG;
+    } else if (xen_pt_msi_check_type(reg->offset, flags, PENDING)) {
         *data = reg->init_val;
     } else {
         *data = XEN_PT_INVALID_REG;
@@ -1173,20 +1219,19 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t throughable_mask = 0;
-    uint32_t old_addr = cfg_entry->data;
+    uint32_t old_addr = *cfg_entry->ptr.word;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
-    s->msi->addr_lo = cfg_entry->data;
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
+    s->msi->addr_lo = *data;
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_addr) {
+    if (*data != old_addr) {
         if (s->msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1201,8 +1246,8 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint32_t writable_mask = 0;
-    uint32_t throughable_mask = 0;
-    uint32_t old_addr = cfg_entry->data;
+    uint32_t old_addr = *cfg_entry->ptr.word;
+    uint32_t *data = cfg_entry->ptr.word;
 
     /* check whether the type is 64 bit or not */
     if (!(s->msi->flags & PCI_MSI_FLAGS_64BIT)) {
@@ -1213,16 +1258,15 @@
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
     /* update the msi_info too */
-    s->msi->addr_hi = cfg_entry->data;
+    s->msi->addr_hi = *data;
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_addr) {
+    if (*data != old_addr) {
         if (s->msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1241,12 +1285,12 @@
     XenPTRegInfo *reg = cfg_entry->reg;
     XenPTMSI *msi = s->msi;
     uint16_t writable_mask = 0;
-    uint16_t throughable_mask = 0;
-    uint16_t old_data = cfg_entry->data;
+    uint16_t old_data = *cfg_entry->ptr.half_word;
     uint32_t offset = reg->offset;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* check the offset whether matches the type or not */
-    if (!xen_pt_msgdata_check_type(offset, msi->flags)) {
+    if (!xen_pt_msi_check_type(offset, msi->flags, DATA)) {
         /* exit I/O emulator */
         XEN_PT_ERR(&s->dev, "the offset does not match the 32/64 bit type!\n");
         return -1;
@@ -1254,16 +1298,15 @@
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
     /* update the msi_info too */
-    msi->data = cfg_entry->data;
+    msi->data = *data;
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
-    *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
+    *val = XEN_PT_MERGE_VALUE(*val, dev_value, 0);
 
     /* update MSI */
-    if (cfg_entry->data != old_data) {
+    if (*data != old_data) {
         if (msi->mapped) {
             xen_pt_msi_update(s);
         }
@@ -1290,8 +1333,9 @@
         .offset     = PCI_MSI_FLAGS,
         .size       = 2,
         .init_val   = 0x0000,
-        .ro_mask    = 0xFF8E,
-        .emu_mask   = 0x007F,
+        .res_mask   = 0xFE00,
+        .ro_mask    = 0x018E,
+        .emu_mask   = 0x017E,
         .init       = xen_pt_msgctrl_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_msgctrl_reg_write,
@@ -1303,7 +1347,6 @@
         .init_val   = 0x00000000,
         .ro_mask    = 0x00000003,
         .emu_mask   = 0xFFFFFFFF,
-        .no_wb      = 1,
         .init       = xen_pt_common_reg_init,
         .u.dw.read  = xen_pt_long_reg_read,
         .u.dw.write = xen_pt_msgaddr32_reg_write,
@@ -1315,7 +1358,6 @@
         .init_val   = 0x00000000,
         .ro_mask    = 0x00000000,
         .emu_mask   = 0xFFFFFFFF,
-        .no_wb      = 1,
         .init       = xen_pt_msgaddr64_reg_init,
         .u.dw.read  = xen_pt_long_reg_read,
         .u.dw.write = xen_pt_msgaddr64_reg_write,
@@ -1327,7 +1369,6 @@
         .init_val   = 0x0000,
         .ro_mask    = 0x0000,
         .emu_mask   = 0xFFFF,
-        .no_wb      = 1,
         .init       = xen_pt_msgdata_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_msgdata_reg_write,
@@ -1339,11 +1380,54 @@
         .init_val   = 0x0000,
         .ro_mask    = 0x0000,
         .emu_mask   = 0xFFFF,
-        .no_wb      = 1,
         .init       = xen_pt_msgdata_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
         .u.w.write  = xen_pt_msgdata_reg_write,
     },
+    /* Mask reg (if PCI_MSI_FLAGS_MASKBIT set, for 32-bit devices) */
+    {
+        .offset     = PCI_MSI_MASK_32,
+        .size       = 4,
+        .init_val   = 0x00000000,
+        .ro_mask    = 0xFFFFFFFF,
+        .emu_mask   = 0xFFFFFFFF,
+        .init       = xen_pt_mask_reg_init,
+        .u.dw.read  = xen_pt_long_reg_read,
+        .u.dw.write = xen_pt_long_reg_write,
+    },
+    /* Mask reg (if PCI_MSI_FLAGS_MASKBIT set, for 64-bit devices) */
+    {
+        .offset     = PCI_MSI_MASK_64,
+        .size       = 4,
+        .init_val   = 0x00000000,
+        .ro_mask    = 0xFFFFFFFF,
+        .emu_mask   = 0xFFFFFFFF,
+        .init       = xen_pt_mask_reg_init,
+        .u.dw.read  = xen_pt_long_reg_read,
+        .u.dw.write = xen_pt_long_reg_write,
+    },
+    /* Pending reg (if PCI_MSI_FLAGS_MASKBIT set, for 32-bit devices) */
+    {
+        .offset     = PCI_MSI_MASK_32 + 4,
+        .size       = 4,
+        .init_val   = 0x00000000,
+        .ro_mask    = 0xFFFFFFFF,
+        .emu_mask   = 0x00000000,
+        .init       = xen_pt_pending_reg_init,
+        .u.dw.read  = xen_pt_long_reg_read,
+        .u.dw.write = xen_pt_long_reg_write,
+    },
+    /* Pending reg (if PCI_MSI_FLAGS_MASKBIT set, for 64-bit devices) */
+    {
+        .offset     = PCI_MSI_MASK_64 + 4,
+        .size       = 4,
+        .init_val   = 0x00000000,
+        .ro_mask    = 0xFFFFFFFF,
+        .emu_mask   = 0x00000000,
+        .init       = xen_pt_pending_reg_init,
+        .u.dw.read  = xen_pt_long_reg_read,
+        .u.dw.write = xen_pt_long_reg_write,
+    },
     {
         .size = 0,
     },
@@ -1359,14 +1443,16 @@
                                     XenPTRegInfo *reg, uint32_t real_offset,
                                     uint32_t *data)
 {
-    PCIDevice *d = &s->dev;
-    uint16_t reg_field = 0;
+    uint16_t reg_field;
+    int rc;
 
     /* use I/O device register's value as initial value */
-    reg_field = pci_get_word(d->config + real_offset);
-
+    rc = xen_host_pci_get_word(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     if (reg_field & PCI_MSIX_FLAGS_ENABLE) {
-        XEN_PT_LOG(d, "MSIX already enabled, disabling it first\n");
+        XEN_PT_LOG(&s->dev, "MSIX already enabled, disabling it first\n");
         xen_host_pci_set_word(&s->real_device, real_offset,
                               reg_field & ~PCI_MSIX_FLAGS_ENABLE);
     }
@@ -1382,15 +1468,15 @@
 {
     XenPTRegInfo *reg = cfg_entry->reg;
     uint16_t writable_mask = 0;
-    uint16_t throughable_mask = 0;
+    uint16_t throughable_mask = get_throughable_mask(s, reg, valid_mask);
     int debug_msix_enabled_old;
+    uint16_t *data = cfg_entry->ptr.half_word;
 
     /* modify emulate register */
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
-    cfg_entry->data = XEN_PT_MERGE_VALUE(*val, cfg_entry->data, writable_mask);
+    *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
     /* create value for writing to I/O device register */
-    throughable_mask = ~reg->emu_mask & valid_mask;
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
 
     /* update MSI-X */
@@ -1401,6 +1487,8 @@
         xen_pt_msix_disable(s);
     }
 
+    s->msix->maskall = *val & PCI_MSIX_FLAGS_MASKALL;
+
     debug_msix_enabled_old = s->msix->enabled;
     s->msix->enabled = !!(*val & PCI_MSIX_FLAGS_ENABLE);
     if (s->msix->enabled != debug_msix_enabled_old) {
@@ -1429,7 +1517,8 @@
         .offset     = PCI_MSI_FLAGS,
         .size       = 2,
         .init_val   = 0x0000,
-        .ro_mask    = 0x3FFF,
+        .res_mask   = 0x3800,
+        .ro_mask    = 0x07FF,
         .emu_mask   = 0x0000,
         .init       = xen_pt_msixctrl_reg_init,
         .u.w.read   = xen_pt_word_reg_read,
@@ -1440,6 +1529,19 @@
     },
 };
 
+static XenPTRegInfo xen_pt_emu_reg_igd_opregion[] = {
+    /* Intel IGFX OpRegion reg */
+    {
+        .offset     = 0x0,
+        .size       = 4,
+        .init_val   = 0,
+        .u.dw.read   = xen_pt_intel_opregion_read,
+        .u.dw.write  = xen_pt_intel_opregion_write,
+    },
+    {
+        .size = 0,
+    },
+};
 
 /****************************
  * Capabilities
@@ -1459,8 +1561,7 @@
                                    const XenPTRegGroupInfo *grp_reg,
                                    uint32_t base_offset, uint8_t *size)
 {
-    *size = pci_get_byte(s->dev.config + base_offset + 0x02);
-    return 0;
+    return xen_host_pci_get_byte(&s->real_device, base_offset + 0x02, size);
 }
 /* get PCI Express Capability Structure register group size */
 static int xen_pt_pcie_size_init(XenPCIPassthroughState *s,
@@ -1539,12 +1640,15 @@
                                 const XenPTRegGroupInfo *grp_reg,
                                 uint32_t base_offset, uint8_t *size)
 {
-    PCIDevice *d = &s->dev;
     uint16_t msg_ctrl = 0;
     uint8_t msi_size = 0xa;
+    int rc;
 
-    msg_ctrl = pci_get_word(d->config + (base_offset + PCI_MSI_FLAGS));
-
+    rc = xen_host_pci_get_word(&s->real_device, base_offset + PCI_MSI_FLAGS,
+                               &msg_ctrl);
+    if (rc) {
+        return rc;
+    }
     /* check if 64-bit address is capable of per-vector masking */
     if (msg_ctrl & PCI_MSI_FLAGS_64BIT) {
         msi_size += 4;
@@ -1677,6 +1781,14 @@
         .size_init   = xen_pt_msix_size_init,
         .emu_regs = xen_pt_emu_reg_msix,
     },
+    /* Intel IGD Opregion group */
+    {
+        .grp_id      = XEN_PCI_INTEL_OPREGION,
+        .grp_type    = XEN_PT_GRP_TYPE_EMU,
+        .grp_size    = 0x4,
+        .size_init   = xen_pt_reg_grp_size_init,
+        .emu_regs    = xen_pt_emu_reg_igd_opregion,
+    },
     {
         .grp_size = 0,
     },
@@ -1687,11 +1799,14 @@
                                XenPTRegInfo *reg, uint32_t real_offset,
                                uint32_t *data)
 {
-    int i;
-    uint8_t *config = s->dev.config;
-    uint32_t reg_field = pci_get_byte(config + real_offset);
+    int i, rc;
+    uint8_t reg_field;
     uint8_t cap_id = 0;
 
+    rc = xen_host_pci_get_byte(&s->real_device, real_offset, &reg_field);
+    if (rc) {
+        return rc;
+    }
     /* find capability offset */
     while (reg_field) {
         for (i = 0; xen_pt_emu_reg_grps[i].grp_size != 0; i++) {
@@ -1700,7 +1815,13 @@
                 continue;
             }
 
-            cap_id = pci_get_byte(config + reg_field + PCI_CAP_LIST_ID);
+            rc = xen_host_pci_get_byte(&s->real_device,
+                                       reg_field + PCI_CAP_LIST_ID, &cap_id);
+            if (rc) {
+                XEN_PT_ERR(&s->dev, "Failed to read capability @0x%x (rc:%d)\n",
+                           reg_field + PCI_CAP_LIST_ID, rc);
+                return rc;
+            }
             if (xen_pt_emu_reg_grps[i].grp_id == cap_id) {
                 if (xen_pt_emu_reg_grps[i].grp_type == XEN_PT_GRP_TYPE_EMU) {
                     goto out;
@@ -1711,7 +1832,11 @@
         }
 
         /* next capability */
-        reg_field = pci_get_byte(config + reg_field + PCI_CAP_LIST_NEXT);
+        rc = xen_host_pci_get_byte(&s->real_device,
+                                   reg_field + PCI_CAP_LIST_NEXT, &reg_field);
+        if (rc) {
+            return rc;
+        }
     }
 
 out:
@@ -1727,7 +1852,7 @@
 static uint8_t find_cap_offset(XenPCIPassthroughState *s, uint8_t cap)
 {
     uint8_t id;
-    unsigned max_cap = PCI_CAP_MAX;
+    unsigned max_cap = XEN_PCI_CAP_MAX;
     uint8_t pos = PCI_CAPABILITY_LIST;
     uint8_t status = 0;
 
@@ -1764,8 +1889,9 @@
     return 0;
 }
 
-static int xen_pt_config_reg_init(XenPCIPassthroughState *s,
-                                  XenPTRegGroup *reg_grp, XenPTRegInfo *reg)
+static void xen_pt_config_reg_init(XenPCIPassthroughState *s,
+                                   XenPTRegGroup *reg_grp, XenPTRegInfo *reg,
+                                   Error **errp)
 {
     XenPTReg *reg_entry;
     uint32_t data = 0;
@@ -1775,30 +1901,94 @@
     reg_entry->reg = reg;
 
     if (reg->init) {
+        uint32_t host_mask, size_mask;
+        unsigned int offset;
+        uint32_t val;
+
         /* initialize emulate register */
         rc = reg->init(s, reg_entry->reg,
                        reg_grp->base_offset + reg->offset, &data);
         if (rc < 0) {
             g_free(reg_entry);
-            return rc;
+            error_setg(errp, "Init emulate register fail");
+            return;
         }
         if (data == XEN_PT_INVALID_REG) {
             /* free unused BAR register entry */
             g_free(reg_entry);
-            return 0;
+            return;
         }
-        /* set register value */
-        reg_entry->data = data;
+        /* Sync up the data to dev.config */
+        offset = reg_grp->base_offset + reg->offset;
+        size_mask = 0xFFFFFFFF >> ((4 - reg->size) << 3);
+
+        switch (reg->size) {
+        case 1: rc = xen_host_pci_get_byte(&s->real_device, offset, (uint8_t *)&val);
+                break;
+        case 2: rc = xen_host_pci_get_word(&s->real_device, offset, (uint16_t *)&val);
+                break;
+        case 4: rc = xen_host_pci_get_long(&s->real_device, offset, &val);
+                break;
+        default: abort();
+        }
+        if (rc) {
+            /* Serious issues when we cannot read the host values! */
+            g_free(reg_entry);
+            error_setg(errp, "Cannot read host values");
+            return;
+        }
+        /* Set bits in emu_mask are the ones we emulate. The dev.config shall
+         * contain the emulated view of the guest - therefore we flip the mask
+         * to mask out the host values (which dev.config initially has) . */
+        host_mask = size_mask & ~reg->emu_mask;
+
+        if ((data & host_mask) != (val & host_mask)) {
+            uint32_t new_val;
+
+            /* Mask out host (including past size). */
+            new_val = val & host_mask;
+            /* Merge emulated ones (excluding the non-emulated ones). */
+            new_val |= data & host_mask;
+            /* Leave intact host and emulated values past the size - even though
+             * we do not care as we write per reg->size granularity, but for the
+             * logging below lets have the proper value. */
+            new_val |= ((val | data)) & ~size_mask;
+            XEN_PT_LOG(&s->dev,"Offset 0x%04x mismatch! Emulated=0x%04x, host=0x%04x, syncing to 0x%04x.\n",
+                       offset, data, val, new_val);
+            val = new_val;
+        } else
+            val = data;
+
+        if (val & ~size_mask) {
+            error_setg(errp, "Offset 0x%04x:0x%04x expands past"
+                    " register size (%d)", offset, val, reg->size);
+            g_free(reg_entry);
+            return;
+        }
+        /* This could be just pci_set_long as we don't modify the bits
+         * past reg->size, but in case this routine is run in parallel or the
+         * init value is larger, we do not want to over-write registers. */
+        switch (reg->size) {
+        case 1: pci_set_byte(s->dev.config + offset, (uint8_t)val);
+                break;
+        case 2: pci_set_word(s->dev.config + offset, (uint16_t)val);
+                break;
+        case 4: pci_set_long(s->dev.config + offset, val);
+                break;
+        default: abort();
+        }
+        /* set register value pointer to the data. */
+        reg_entry->ptr.byte = s->dev.config + offset;
+
     }
     /* list add register entry */
     QLIST_INSERT_HEAD(&reg_grp->reg_tbl_list, reg_entry, entries);
-
-    return 0;
 }
 
-int xen_pt_config_init(XenPCIPassthroughState *s)
+void xen_pt_config_init(XenPCIPassthroughState *s, Error **errp)
 {
     int i, rc;
+    Error *err = NULL;
 
     QLIST_INIT(&s->reg_grps);
 
@@ -1806,7 +1996,8 @@
         uint32_t reg_grp_offset = 0;
         XenPTRegGroup *reg_grp_entry = NULL;
 
-        if (xen_pt_emu_reg_grps[i].grp_id != 0xFF) {
+        if (xen_pt_emu_reg_grps[i].grp_id != 0xFF
+            && xen_pt_emu_reg_grps[i].grp_id != XEN_PCI_INTEL_OPREGION) {
             if (xen_pt_hide_dev_cap(&s->real_device,
                                     xen_pt_emu_reg_grps[i].grp_id)) {
                 continue;
@@ -1819,6 +2010,15 @@
             }
         }
 
+        /*
+         * By default we will trap up to 0x40 in the cfg space.
+         * If an intel device is pass through we need to trap 0xfc,
+         * therefore the size should be 0xff.
+         */
+        if (xen_pt_emu_reg_grps[i].grp_id == XEN_PCI_INTEL_OPREGION) {
+            reg_grp_offset = XEN_PCI_INTEL_OPREGION;
+        }
+
         reg_grp_entry = g_new0(XenPTRegGroup, 1);
         QLIST_INIT(&reg_grp_entry->reg_tbl_list);
         QLIST_INSERT_HEAD(&s->reg_grps, reg_grp_entry, entries);
@@ -1831,8 +2031,12 @@
                                                   reg_grp_offset,
                                                   &reg_grp_entry->size);
             if (rc < 0) {
+                error_setg(&err, "Failed to initialize %d/%zu, type = 0x%x,"
+                           " rc: %d", i, ARRAY_SIZE(xen_pt_emu_reg_grps),
+                           xen_pt_emu_reg_grps[i].grp_type, rc);
+                error_propagate(errp, err);
                 xen_pt_config_delete(s);
-                return rc;
+                return;
             }
         }
 
@@ -1840,20 +2044,23 @@
             if (xen_pt_emu_reg_grps[i].emu_regs) {
                 int j = 0;
                 XenPTRegInfo *regs = xen_pt_emu_reg_grps[i].emu_regs;
+
                 /* initialize capability register */
                 for (j = 0; regs->size != 0; j++, regs++) {
-                    /* initialize capability register */
-                    rc = xen_pt_config_reg_init(s, reg_grp_entry, regs);
-                    if (rc < 0) {
+                    xen_pt_config_reg_init(s, reg_grp_entry, regs, &err);
+                    if (err) {
+                        error_append_hint(&err, "Failed to init register %d"
+                                " offsets 0x%x in grp_type = 0x%x (%d/%zu)", j,
+                                regs->offset, xen_pt_emu_reg_grps[i].grp_type,
+                                i, ARRAY_SIZE(xen_pt_emu_reg_grps));
+                        error_propagate(errp, err);
                         xen_pt_config_delete(s);
-                        return rc;
+                        return;
                     }
                 }
             }
         }
     }
-
-    return 0;
 }
 
 /* delete all emulate register */
@@ -1864,11 +2071,9 @@
 
     /* free MSI/MSI-X info table */
     if (s->msix) {
-        xen_pt_msix_delete(s);
+        xen_pt_msix_unmap(s);
     }
-    if (s->msi) {
-        g_free(s->msi);
-    }
+    g_free(s->msi);
 
     /* free all register group entry */
     QLIST_FOREACH_SAFE(reg_group, &s->reg_grps, entries, next_grp) {
diff --git a/hw/xen/xen_pt_graphics.c b/hw/xen/xen_pt_graphics.c
new file mode 100644
index 0000000..0f4c8d7
--- /dev/null
+++ b/hw/xen/xen_pt_graphics.c
@@ -0,0 +1,275 @@
+/*
+ * graphics passthrough
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "xen_pt.h"
+#include "xen-host-pci-device.h"
+#include "hw/xen/xen_backend.h"
+
+static unsigned long igd_guest_opregion;
+static unsigned long igd_host_opregion;
+
+#define XEN_PCI_INTEL_OPREGION_MASK 0xfff
+
+typedef struct VGARegion {
+    int type;           /* Memory or port I/O */
+    uint64_t guest_base_addr;
+    uint64_t machine_base_addr;
+    uint64_t size;    /* size of the region */
+    int rc;
+} VGARegion;
+
+#define IORESOURCE_IO           0x00000100
+#define IORESOURCE_MEM          0x00000200
+
+static struct VGARegion vga_args[] = {
+    {
+        .type = IORESOURCE_IO,
+        .guest_base_addr = 0x3B0,
+        .machine_base_addr = 0x3B0,
+        .size = 0xC,
+        .rc = -1,
+    },
+    {
+        .type = IORESOURCE_IO,
+        .guest_base_addr = 0x3C0,
+        .machine_base_addr = 0x3C0,
+        .size = 0x20,
+        .rc = -1,
+    },
+    {
+        .type = IORESOURCE_MEM,
+        .guest_base_addr = 0xa0000 >> XC_PAGE_SHIFT,
+        .machine_base_addr = 0xa0000 >> XC_PAGE_SHIFT,
+        .size = 0x20,
+        .rc = -1,
+    },
+};
+
+/*
+ * register VGA resources for the domain with assigned gfx
+ */
+int xen_pt_register_vga_regions(XenHostPCIDevice *dev)
+{
+    int i = 0;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return 0;
+    }
+
+    for (i = 0 ; i < ARRAY_SIZE(vga_args); i++) {
+        if (vga_args[i].type == IORESOURCE_IO) {
+            vga_args[i].rc = xc_domain_ioport_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_ADD_MAPPING);
+        } else {
+            vga_args[i].rc = xc_domain_memory_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_ADD_MAPPING);
+        }
+
+        if (vga_args[i].rc) {
+            XEN_PT_ERR(NULL, "VGA %s mapping failed! (rc: %i)\n",
+                    vga_args[i].type == IORESOURCE_IO ? "ioport" : "memory",
+                    vga_args[i].rc);
+            return vga_args[i].rc;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * unregister VGA resources for the domain with assigned gfx
+ */
+int xen_pt_unregister_vga_regions(XenHostPCIDevice *dev)
+{
+    int i = 0;
+    int ret = 0;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        return 0;
+    }
+
+    for (i = 0 ; i < ARRAY_SIZE(vga_args); i++) {
+        if (vga_args[i].type == IORESOURCE_IO) {
+            vga_args[i].rc = xc_domain_ioport_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_REMOVE_MAPPING);
+        } else {
+            vga_args[i].rc = xc_domain_memory_mapping(xen_xc, xen_domid,
+                            vga_args[i].guest_base_addr,
+                            vga_args[i].machine_base_addr,
+                            vga_args[i].size, DPCI_REMOVE_MAPPING);
+        }
+
+        if (vga_args[i].rc) {
+            XEN_PT_ERR(NULL, "VGA %s unmapping failed! (rc: %i)\n",
+                    vga_args[i].type == IORESOURCE_IO ? "ioport" : "memory",
+                    vga_args[i].rc);
+            return vga_args[i].rc;
+        }
+    }
+
+    if (igd_guest_opregion) {
+        ret = xc_domain_memory_mapping(xen_xc, xen_domid,
+                (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT),
+                (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                3,
+                DPCI_REMOVE_MAPPING);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+static void *get_vgabios(XenPCIPassthroughState *s, int *size,
+                       XenHostPCIDevice *dev)
+{
+    return pci_assign_dev_load_option_rom(&s->dev, OBJECT(&s->dev), size,
+                                          dev->domain, dev->bus,
+                                          dev->dev, dev->func);
+}
+
+/* Refer to Seabios. */
+struct rom_header {
+    uint16_t signature;
+    uint8_t size;
+    uint8_t initVector[4];
+    uint8_t reserved[17];
+    uint16_t pcioffset;
+    uint16_t pnpoffset;
+} __attribute__((packed));
+
+struct pci_data {
+    uint32_t signature;
+    uint16_t vendor;
+    uint16_t device;
+    uint16_t vitaldata;
+    uint16_t dlen;
+    uint8_t drevision;
+    uint8_t class_lo;
+    uint16_t class_hi;
+    uint16_t ilen;
+    uint16_t irevision;
+    uint8_t type;
+    uint8_t indicator;
+    uint16_t reserved;
+} __attribute__((packed));
+
+void xen_pt_setup_vga(XenPCIPassthroughState *s, XenHostPCIDevice *dev,
+                     Error **errp)
+{
+    unsigned char *bios = NULL;
+    struct rom_header *rom;
+    int bios_size;
+    char *c = NULL;
+    char checksum = 0;
+    uint32_t len = 0;
+    struct pci_data *pd = NULL;
+
+    if (!is_igd_vga_passthrough(dev)) {
+        error_setg(errp, "Need to enable igd-passthrough");
+        return;
+    }
+
+    bios = get_vgabios(s, &bios_size, dev);
+    if (!bios) {
+        error_setg(errp, "VGA: Can't get VBIOS");
+        return;
+    }
+
+    /* Currently we fixed this address as a primary. */
+    rom = (struct rom_header *)bios;
+    pd = (void *)(bios + (unsigned char)rom->pcioffset);
+
+    /* We may need to fixup Device Identification. */
+    if (pd->device != s->real_device.device_id) {
+        pd->device = s->real_device.device_id;
+
+        len = rom->size * 512;
+        /* Then adjust the bios checksum */
+        for (c = (char *)bios; c < ((char *)bios + len); c++) {
+            checksum += *c;
+        }
+        if (checksum) {
+            bios[len - 1] -= checksum;
+            XEN_PT_LOG(&s->dev, "vga bios checksum is adjusted %x!\n",
+                       checksum);
+        }
+    }
+
+    /* Currently we fixed this address as a primary for legacy BIOS. */
+    cpu_physical_memory_rw(0xc0000, bios, bios_size, 1);
+}
+
+uint32_t igd_read_opregion(XenPCIPassthroughState *s)
+{
+    uint32_t val = 0;
+
+    if (!igd_guest_opregion) {
+        return val;
+    }
+
+    val = igd_guest_opregion;
+
+    XEN_PT_LOG(&s->dev, "Read opregion val=%x\n", val);
+    return val;
+}
+
+#define XEN_PCI_INTEL_OPREGION_PAGES 0x3
+#define XEN_PCI_INTEL_OPREGION_ENABLE_ACCESSED 0x1
+void igd_write_opregion(XenPCIPassthroughState *s, uint32_t val)
+{
+    int ret;
+
+    if (igd_guest_opregion) {
+        XEN_PT_LOG(&s->dev, "opregion register already been set, ignoring %x\n",
+                   val);
+        return;
+    }
+
+    /* We just work with LE. */
+    xen_host_pci_get_block(&s->real_device, XEN_PCI_INTEL_OPREGION,
+            (uint8_t *)&igd_host_opregion, 4);
+    igd_guest_opregion = (unsigned long)(val & ~XEN_PCI_INTEL_OPREGION_MASK)
+                            | (igd_host_opregion & XEN_PCI_INTEL_OPREGION_MASK);
+
+    ret = xc_domain_iomem_permission(xen_xc, xen_domid,
+            (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+            XEN_PCI_INTEL_OPREGION_PAGES,
+            XEN_PCI_INTEL_OPREGION_ENABLE_ACCESSED);
+
+    if (ret) {
+        XEN_PT_ERR(&s->dev, "[%d]:Can't enable to access IGD host opregion:"
+                    " 0x%lx.\n", ret,
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT)),
+        igd_guest_opregion = 0;
+        return;
+    }
+
+    ret = xc_domain_memory_mapping(xen_xc, xen_domid,
+            (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT),
+            (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+            XEN_PCI_INTEL_OPREGION_PAGES,
+            DPCI_ADD_MAPPING);
+
+    if (ret) {
+        XEN_PT_ERR(&s->dev, "[%d]:Can't map IGD host opregion:0x%lx to"
+                    " guest opregion:0x%lx.\n", ret,
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                    (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT));
+        igd_guest_opregion = 0;
+        return;
+    }
+
+    XEN_PT_LOG(&s->dev, "Map OpRegion: 0x%lx -> 0x%lx\n",
+                    (unsigned long)(igd_host_opregion >> XC_PAGE_SHIFT),
+                    (unsigned long)(igd_guest_opregion >> XC_PAGE_SHIFT));
+}
diff --git a/hw/xen/xen_pt_msi.c b/hw/xen/xen_pt_msi.c
index 9ed9321..62add06 100644
--- a/hw/xen/xen_pt_msi.c
+++ b/hw/xen/xen_pt_msi.c
@@ -9,7 +9,7 @@
  * This file implements direct PCI assignment to a HVM guest
  */
 
-#include <sys/mman.h>
+#include "qemu/osdep.h"
 
 #include "hw/xen/xen_backend.h"
 #include "xen_pt.h"
@@ -25,6 +25,7 @@
 #define XEN_PT_GFLAGSSHIFT_DELIV_MODE     12
 #define XEN_PT_GFLAGSSHIFT_TRG_MODE       15
 
+#define latch(fld) latch[PCI_MSIX_ENTRY_##fld / sizeof(uint32_t)]
 
 /*
  * Helpers
@@ -75,19 +76,29 @@
                            bool enable)
 {
     uint16_t val = 0;
+    int rc;
 
     if (!address) {
         return -1;
     }
 
-    xen_host_pci_get_word(&s->real_device, address, &val);
+    rc = xen_host_pci_get_word(&s->real_device, address, &val);
+    if (rc) {
+        XEN_PT_ERR(&s->dev, "Failed to read MSI/MSI-X register (0x%x), rc:%d\n",
+                   address, rc);
+        return rc;
+    }
     if (enable) {
         val |= flag;
     } else {
         val &= ~flag;
     }
-    xen_host_pci_set_word(&s->real_device, address, val);
-    return 0;
+    rc = xen_host_pci_set_word(&s->real_device, address, val);
+    if (rc) {
+        XEN_PT_ERR(&s->dev, "Failed to write MSI/MSI-X register (0x%x), rc:%d\n",
+                   address, rc);
+    }
+    return rc;
 }
 
 static int msi_msix_setup(XenPCIPassthroughState *s,
@@ -103,9 +114,7 @@
 
     assert((!is_msix && msix_entry == 0) || is_msix);
 
-    if (gvec == 0) {
-        /* if gvec is 0, the guest is asking for a particular pirq that
-         * is passed as dest_id */
+    if (xen_is_pirq_msi(data)) {
         *ppirq = msi_ext_dest_id(addr >> 32) | msi_dest_id(addr);
         if (!*ppirq) {
             /* this probably identifies an misconfiguration of the guest,
@@ -132,8 +141,8 @@
                                      msix_entry, table_base);
         if (rc) {
             XEN_PT_ERR(&s->dev,
-                       "Mapping of MSI%s (rc: %i, vec: %#x, entry %#x)\n",
-                       is_msix ? "-X" : "", rc, gvec, msix_entry);
+                       "Mapping of MSI%s (err: %i, vec: %#x, entry %#x)\n",
+                       is_msix ? "-X" : "", errno, gvec, msix_entry);
             return rc;
         }
     }
@@ -166,12 +175,12 @@
                                   pirq, gflags, table_addr);
 
     if (rc) {
-        XEN_PT_ERR(d, "Updating of MSI%s failed. (rc: %d)\n",
-                   is_msix ? "-X" : "", rc);
+        XEN_PT_ERR(d, "Updating of MSI%s failed. (err: %d)\n",
+                   is_msix ? "-X" : "", errno);
 
         if (xc_physdev_unmap_pirq(xen_xc, xen_domid, *old_pirq)) {
-            XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed.\n",
-                       is_msix ? "-X" : "", *old_pirq);
+            XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (err: %d)\n",
+                       is_msix ? "-X" : "", *old_pirq, errno);
         }
         *old_pirq = XEN_PT_UNASSIGNED_PIRQ;
     }
@@ -199,8 +208,8 @@
                    is_msix ? "-X" : "", pirq, gvec);
         rc = xc_domain_unbind_msi_irq(xen_xc, xen_domid, gvec, pirq, gflags);
         if (rc) {
-            XEN_PT_ERR(d, "Unbinding of MSI%s failed. (pirq: %d, gvec: %#x)\n",
-                       is_msix ? "-X" : "", pirq, gvec);
+            XEN_PT_ERR(d, "Unbinding of MSI%s failed. (err: %d, pirq: %d, gvec: %#x)\n",
+                       is_msix ? "-X" : "", errno, pirq, gvec);
             return rc;
         }
     }
@@ -208,8 +217,8 @@
     XEN_PT_LOG(d, "Unmap MSI%s pirq %d\n", is_msix ? "-X" : "", pirq);
     rc = xc_physdev_unmap_pirq(xen_xc, xen_domid, pirq);
     if (rc) {
-        XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (rc: %i)\n",
-                   is_msix ? "-X" : "", pirq, rc);
+        XEN_PT_ERR(d, "Unmapping of MSI%s pirq %d failed. (err: %i)\n",
+                   is_msix ? "-X" : "", pirq, errno);
         return rc;
     }
 
@@ -220,7 +229,7 @@
  * MSI virtualization functions
  */
 
-int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
+static int xen_pt_msi_set_enable(XenPCIPassthroughState *s, bool enable)
 {
     XEN_PT_LOG(&s->dev, "%s MSI.\n", enable ? "enabling" : "disabling");
 
@@ -276,7 +285,7 @@
         return;
     }
 
-    xen_pt_msi_set_enable(s, false);
+    (void)xen_pt_msi_set_enable(s, false);
 
     msi_msix_disable(s, msi_addr64(msi), msi->data, msi->pirq, false,
                      msi->initialized);
@@ -304,7 +313,8 @@
                            enabled);
 }
 
-static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr)
+static int xen_pt_msix_update_one(XenPCIPassthroughState *s, int entry_nr,
+                                  uint32_t vec_ctrl)
 {
     XenPTMSIXEntry *entry = NULL;
     int pirq;
@@ -322,6 +332,19 @@
 
     pirq = entry->pirq;
 
+    /*
+     * Update the entry addr and data to the latest values only when the
+     * entry is masked or they are all masked, as required by the spec.
+     * Addr and data changes while the MSI-X entry is unmasked get deferred
+     * until the next masked -> unmasked transition.
+     */
+    if (pirq == XEN_PT_UNASSIGNED_PIRQ || s->msix->maskall ||
+        (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
+        entry->addr = entry->latch(LOWER_ADDR) |
+                      ((uint64_t)entry->latch(UPPER_ADDR) << 32);
+        entry->data = entry->latch(DATA);
+    }
+
     rc = msi_msix_setup(s, entry->addr, entry->data, &pirq, true, entry_nr,
                         entry->pirq == XEN_PT_UNASSIGNED_PIRQ);
     if (rc) {
@@ -347,7 +370,7 @@
     int i;
 
     for (i = 0; i < msix->total_entries; i++) {
-        xen_pt_msix_update_one(s, i);
+        xen_pt_msix_update_one(s, i, msix->msix_entry[i].latch(VECTOR_CTRL));
     }
 
     return 0;
@@ -385,8 +408,8 @@
             ret = xc_domain_unbind_pt_irq(xen_xc, xen_domid, entry->pirq,
                                           PT_IRQ_TYPE_MSI, 0, 0, 0, 0);
             if (ret) {
-                XEN_PT_ERR(&s->dev, "unbind MSI-X entry %d failed\n",
-                           entry->pirq);
+                XEN_PT_ERR(&s->dev, "unbind MSI-X entry %d failed (err: %d)\n",
+                           entry->pirq, errno);
             }
             entry->updated = true;
         }
@@ -396,36 +419,14 @@
 
 static uint32_t get_entry_value(XenPTMSIXEntry *e, int offset)
 {
-    switch (offset) {
-    case PCI_MSIX_ENTRY_LOWER_ADDR:
-        return e->addr & UINT32_MAX;
-    case PCI_MSIX_ENTRY_UPPER_ADDR:
-        return e->addr >> 32;
-    case PCI_MSIX_ENTRY_DATA:
-        return e->data;
-    case PCI_MSIX_ENTRY_VECTOR_CTRL:
-        return e->vector_ctrl;
-    default:
-        return 0;
-    }
+    assert(!(offset % sizeof(*e->latch)));
+    return e->latch[offset / sizeof(*e->latch)];
 }
 
 static void set_entry_value(XenPTMSIXEntry *e, int offset, uint32_t val)
 {
-    switch (offset) {
-    case PCI_MSIX_ENTRY_LOWER_ADDR:
-        e->addr = (e->addr & ((uint64_t)UINT32_MAX << 32)) | val;
-        break;
-    case PCI_MSIX_ENTRY_UPPER_ADDR:
-        e->addr = (uint64_t)val << 32 | (e->addr & UINT32_MAX);
-        break;
-    case PCI_MSIX_ENTRY_DATA:
-        e->data = val;
-        break;
-    case PCI_MSIX_ENTRY_VECTOR_CTRL:
-        e->vector_ctrl = val;
-        break;
-    }
+    assert(!(offset % sizeof(*e->latch)));
+    e->latch[offset / sizeof(*e->latch)] = val;
 }
 
 static void pci_msix_write(void *opaque, hwaddr addr,
@@ -434,47 +435,36 @@
     XenPCIPassthroughState *s = opaque;
     XenPTMSIX *msix = s->msix;
     XenPTMSIXEntry *entry;
-    int entry_nr, offset;
+    unsigned int entry_nr, offset;
 
     entry_nr = addr / PCI_MSIX_ENTRY_SIZE;
-    if (entry_nr < 0 || entry_nr >= msix->total_entries) {
-        XEN_PT_ERR(&s->dev, "asked MSI-X entry '%i' invalid!\n", entry_nr);
+    if (entry_nr >= msix->total_entries) {
         return;
     }
     entry = &msix->msix_entry[entry_nr];
     offset = addr % PCI_MSIX_ENTRY_SIZE;
 
     if (offset != PCI_MSIX_ENTRY_VECTOR_CTRL) {
-        const volatile uint32_t *vec_ctrl;
-
         if (get_entry_value(entry, offset) == val
             && entry->pirq != XEN_PT_UNASSIGNED_PIRQ) {
             return;
         }
 
+        entry->updated = true;
+    } else if (msix->enabled && entry->updated &&
+               !(val & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
+        const volatile uint32_t *vec_ctrl;
+
         /*
          * If Xen intercepts the mask bit access, entry->vec_ctrl may not be
          * up-to-date. Read from hardware directly.
          */
         vec_ctrl = s->msix->phys_iomem_base + entry_nr * PCI_MSIX_ENTRY_SIZE
             + PCI_MSIX_ENTRY_VECTOR_CTRL;
-
-        if (msix->enabled && !(*vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
-            XEN_PT_ERR(&s->dev, "Can't update msix entry %d since MSI-X is"
-                       " already enabled.\n", entry_nr);
-            return;
-        }
-
-        entry->updated = true;
+        xen_pt_msix_update_one(s, entry_nr, *vec_ctrl);
     }
 
     set_entry_value(entry, offset, val);
-
-    if (offset == PCI_MSIX_ENTRY_VECTOR_CTRL) {
-        if (msix->enabled && !(val & PCI_MSIX_ENTRY_CTRL_MASKBIT)) {
-            xen_pt_msix_update_one(s, entry_nr);
-        }
-    }
 }
 
 static uint64_t pci_msix_read(void *opaque, hwaddr addr,
@@ -500,6 +490,12 @@
     }
 }
 
+static bool pci_msix_accepts(void *opaque, hwaddr addr,
+                             unsigned size, bool is_write)
+{
+    return !(addr & (size - 1));
+}
+
 static const MemoryRegionOps pci_msix_ops = {
     .read = pci_msix_read,
     .write = pci_msix_write,
@@ -508,7 +504,13 @@
         .min_access_size = 4,
         .max_access_size = 4,
         .unaligned = false,
+        .accepts = pci_msix_accepts
     },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false
+    }
 };
 
 int xen_pt_msix_init(XenPCIPassthroughState *s, uint32_t base)
@@ -598,7 +600,7 @@
     return rc;
 }
 
-void xen_pt_msix_delete(XenPCIPassthroughState *s)
+void xen_pt_msix_unmap(XenPCIPassthroughState *s)
 {
     XenPTMSIX *msix = s->msix;
 
@@ -615,6 +617,17 @@
     }
 
     memory_region_del_subregion(&s->bar[msix->bar_index], &msix->mmio);
+}
+
+void xen_pt_msix_delete(XenPCIPassthroughState *s)
+{
+    XenPTMSIX *msix = s->msix;
+
+    if (!msix) {
+        return;
+    }
+
+    object_unparent(OBJECT(&msix->mmio));
 
     g_free(s->msix);
     s->msix = NULL;
diff --git a/hw/xenpv/Makefile.objs b/hw/xenpv/Makefile.objs
index 49f6e9e..bbf5873 100644
--- a/hw/xenpv/Makefile.objs
+++ b/hw/xenpv/Makefile.objs
@@ -1,2 +1,4 @@
 # Xen PV machine support
-obj-$(CONFIG_XEN) += xen_domainbuild.o xen_machine_pv.o
+obj-$(CONFIG_XEN) += xen_machine_pv.o
+# Xen PV machine builder support
+obj-$(CONFIG_XEN_PV_DOMAIN_BUILD) += xen_domainbuild.o
diff --git a/hw/xenpv/xen_domainbuild.c b/hw/xenpv/xen_domainbuild.c
index c0ab753..5a9f5ac 100644
--- a/hw/xenpv/xen_domainbuild.c
+++ b/hw/xenpv/xen_domainbuild.c
@@ -1,4 +1,4 @@
-#include <signal.h>
+#include "qemu/osdep.h"
 #include "hw/xen/xen_backend.h"
 #include "xen_domainbuild.h"
 #include "qemu/timer.h"
@@ -174,12 +174,15 @@
     for (i = 3; i < n; i++) {
         if (i == fd[0])
             continue;
-        if (i == xc_fd(xen_xc)) {
-            continue;
-        }
         close(i);
     }
 
+    /*
+     * Reopen xc interface, since the original is unsafe after fork
+     * and was closed above.
+     */
+    xen_xc = xc_interface_open(0, 0, 0);
+
     /* ignore term signals */
     signal(SIGINT,  SIG_IGN);
     signal(SIGTERM, SIG_IGN);
@@ -234,7 +237,7 @@
     int rc;
 
     memcpy(uuid, qemu_uuid, sizeof(uuid));
-    rc = xc_domain_create(xen_xc, ssidref, uuid, flags, &xen_domid);
+    rc = xen_domain_create(xen_xc, ssidref, uuid, flags, &xen_domid);
     if (rc < 0) {
         fprintf(stderr, "xen: xc_domain_create() failed\n");
         goto err;
diff --git a/hw/xenpv/xen_domainbuild.h b/hw/xenpv/xen_domainbuild.h
index 29a91ea..652d9b4 100644
--- a/hw/xenpv/xen_domainbuild.h
+++ b/hw/xenpv/xen_domainbuild.h
@@ -1,5 +1,5 @@
 #ifndef QEMU_HW_XEN_DOMAINBUILD_H
-#define QEMU_HW_XEN_DOMAINBUILD_H 1
+#define QEMU_HW_XEN_DOMAINBUILD_H
 
 #include "hw/xen/xen_common.h"
 
diff --git a/hw/xenpv/xen_machine_pv.c b/hw/xenpv/xen_machine_pv.c
index 2e545d2..79aef4e 100644
--- a/hw/xenpv/xen_machine_pv.c
+++ b/hw/xenpv/xen_machine_pv.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "hw/xen/xen_backend.h"
@@ -30,9 +31,6 @@
 
 static void xen_init_pv(MachineState *machine)
 {
-    const char *kernel_filename = machine->kernel_filename;
-    const char *kernel_cmdline = machine->kernel_cmdline;
-    const char *initrd_filename = machine->initrd_filename;
     DriveInfo *dinfo;
     int i;
 
@@ -46,23 +44,31 @@
     case XEN_ATTACH:
         /* nothing to do, xend handles everything */
         break;
-    case XEN_CREATE:
+#ifdef CONFIG_XEN_PV_DOMAIN_BUILD
+    case XEN_CREATE: {
+        const char *kernel_filename = machine->kernel_filename;
+        const char *kernel_cmdline = machine->kernel_cmdline;
+        const char *initrd_filename = machine->initrd_filename;
         if (xen_domain_build_pv(kernel_filename, initrd_filename,
                                 kernel_cmdline) < 0) {
             fprintf(stderr, "xen pv domain creation failed\n");
             exit(1);
         }
         break;
+    }
+#endif
     case XEN_EMULATE:
         fprintf(stderr, "xen emulation not implemented (yet)\n");
         exit(1);
         break;
+    default:
+        fprintf(stderr, "unhandled xen_mode %d\n", xen_mode);
+        exit(1);
+        break;
     }
 
-    xen_be_register("console", &xen_console_ops);
-    xen_be_register("vkbd", &xen_kbdmouse_ops);
+    xen_be_register_common();
     xen_be_register("vfb", &xen_framebuffer_ops);
-    xen_be_register("qdisk", &xen_blkdev_ops);
     xen_be_register("qnic", &xen_netdev_ops);
 
     /* configure framebuffer */
@@ -93,17 +99,12 @@
     xen_init_display(xen_domid);
 }
 
-static QEMUMachine xenpv_machine = {
-    .name = "xenpv",
-    .desc = "Xen Para-virtualized PC",
-    .init = xen_init_pv,
-    .max_cpus = 1,
-    .default_machine_opts = "accel=xen",
-};
-
-static void xenpv_machine_init(void)
+static void xenpv_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xenpv_machine);
+    mc->desc = "Xen Para-virtualized PC";
+    mc->init = xen_init_pv;
+    mc->max_cpus = 1;
+    mc->default_machine_opts = "accel=xen";
 }
 
-machine_init(xenpv_machine_init);
+DEFINE_MACHINE("xenpv", xenpv_machine_init)
diff --git a/hw/xtensa/bootparam.h b/hw/xtensa/bootparam.h
index 955f4e8..ade7891 100644
--- a/hw/xtensa/bootparam.h
+++ b/hw/xtensa/bootparam.h
@@ -1,5 +1,5 @@
-#ifndef HW_XTENSA_BOOTPARAM
-#define HW_XTENSA_BOOTPARAM
+#ifndef HW_XTENSA_BOOTPARAM_H
+#define HW_XTENSA_BOOTPARAM_H
 
 #define BP_TAG_COMMAND_LINE     0x1001  /* command line (0-terminated string)*/
 #define BP_TAG_INITRD           0x1002  /* ramdisk addr and size (bp_meminfo) */
diff --git a/hw/xtensa/pic_cpu.c b/hw/xtensa/pic_cpu.c
index 18825d1..2bed64f 100644
--- a/hw/xtensa/pic_cpu.c
+++ b/hw/xtensa/pic_cpu.c
@@ -25,6 +25,8 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "qemu/log.h"
 #include "qemu/timer.h"
@@ -120,8 +122,8 @@
     }
     env->wake_ccount = wake_ccount;
     timer_mod(env->ccompare_timer, env->halt_clock +
-            muldiv64(wake_ccount - env->sregs[CCOUNT],
-                1000000, env->config->clock_freq_khz));
+            (uint64_t)(wake_ccount - env->sregs[CCOUNT]) *
+            1000000 / env->config->clock_freq_khz);
 }
 
 static void xtensa_ccompare_cb(void *opaque)
diff --git a/hw/xtensa/sim.c b/hw/xtensa/sim.c
index 37ea9ae..5e94004 100644
--- a/hw/xtensa/sim.c
+++ b/hw/xtensa/sim.c
@@ -25,6 +25,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -64,7 +68,7 @@
     for (n = 0; n < smp_cpus; n++) {
         cpu = cpu_xtensa_init(cpu_model);
         if (cpu == NULL) {
-            error_report("unable to find CPU definition '%s'\n",
+            error_report("unable to find CPU definition '%s'",
                          cpu_model);
             exit(EXIT_FAILURE);
         }
@@ -79,12 +83,12 @@
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_abort);
+    memory_region_init_ram(ram, NULL, "xtensa.sram", ram_size, &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(get_system_memory(), 0, ram);
 
     rom = g_malloc(sizeof(*rom));
-    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_abort);
+    memory_region_init_ram(rom, NULL, "xtensa.rom", 0x1000, &error_fatal);
     vmstate_register_ram_global(rom);
     memory_region_add_subregion(get_system_memory(), 0xfe000000, rom);
 
@@ -93,10 +97,10 @@
         uint64_t elf_lowaddr;
 #ifdef TARGET_WORDS_BIGENDIAN
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, 1, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, 1, EM_XTENSA, 0, 0);
 #else
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, 0, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, 0, EM_XTENSA, 0, 0);
 #endif
         if (success > 0) {
             env->pc = elf_entry;
@@ -104,17 +108,12 @@
     }
 }
 
-static QEMUMachine xtensa_sim_machine = {
-    .name = "sim",
-    .desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .is_default = true,
-    .init = xtensa_sim_init,
-    .max_cpus = 4,
-};
-
-static void xtensa_sim_machine_init(void)
+static void xtensa_sim_machine_init(MachineClass *mc)
 {
-    qemu_register_machine(&xtensa_sim_machine);
+    mc->desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->is_default = true;
+    mc->init = xtensa_sim_init;
+    mc->max_cpus = 4;
 }
 
-machine_init(xtensa_sim_machine_init);
+DEFINE_MACHINE("sim", xtensa_sim_machine_init)
diff --git a/hw/xtensa/xtfpga.c b/hw/xtensa/xtfpga.c
index e5a6bba..ac75949 100644
--- a/hw/xtensa/xtfpga.c
+++ b/hw/xtensa/xtfpga.c
@@ -25,6 +25,10 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/sysemu.h"
 #include "hw/boards.h"
 #include "hw/loader.h"
@@ -143,11 +147,34 @@
             sysbus_mmio_get_region(s, 1));
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384, &error_abort);
+    memory_region_init_ram(ram, OBJECT(s), "open_eth.ram", 16384,
+                           &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(address_space, buffers, ram);
 }
 
+static pflash_t *xtfpga_flash_init(MemoryRegion *address_space,
+                                   const LxBoardDesc *board,
+                                   DriveInfo *dinfo, int be)
+{
+    SysBusDevice *s;
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+
+    qdev_prop_set_drive(dev, "drive", blk_by_legacy_dinfo(dinfo),
+                        &error_abort);
+    qdev_prop_set_uint32(dev, "num-blocks",
+                         board->flash_size / board->flash_sector_size);
+    qdev_prop_set_uint64(dev, "sector-length", board->flash_sector_size);
+    qdev_prop_set_uint8(dev, "width", 2);
+    qdev_prop_set_bit(dev, "big-endian", be);
+    qdev_prop_set_string(dev, "name", "lx60.io.flash");
+    qdev_init_nofail(dev);
+    s = SYS_BUS_DEVICE(dev);
+    memory_region_add_subregion(address_space, board->flash_base,
+                                sysbus_mmio_get_region(s, 0));
+    return OBJECT_CHECK(pflash_t, (dev), "cfi.pflash01");
+}
+
 static uint64_t translate_phys_addr(void *opaque, uint64_t addr)
 {
     XtensaCPU *cpu = opaque;
@@ -162,6 +189,23 @@
     cpu_reset(CPU(cpu));
 }
 
+static uint64_t lx60_io_read(void *opaque, hwaddr addr,
+        unsigned size)
+{
+    return 0;
+}
+
+static void lx60_io_write(void *opaque, hwaddr addr,
+        uint64_t val, unsigned size)
+{
+}
+
+static const MemoryRegionOps lx60_io_ops = {
+    .read = lx60_io_read,
+    .write = lx60_io_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
 static void lx_init(const LxBoardDesc *board, MachineState *machine)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -190,7 +234,7 @@
     for (n = 0; n < smp_cpus; n++) {
         cpu = cpu_xtensa_init(cpu_model);
         if (cpu == NULL) {
-            error_report("unable to find CPU definition '%s'\n",
+            error_report("unable to find CPU definition '%s'",
                          cpu_model);
             exit(EXIT_FAILURE);
         }
@@ -206,12 +250,13 @@
 
     ram = g_malloc(sizeof(*ram));
     memory_region_init_ram(ram, NULL, "lx60.dram", machine->ram_size,
-                           &error_abort);
+                           &error_fatal);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
     system_io = g_malloc(sizeof(*system_io));
-    memory_region_init(system_io, NULL, "lx60.io", 224 * 1024 * 1024);
+    memory_region_init_io(system_io, NULL, &lx60_io_ops, NULL, "lx60.io",
+                          224 * 1024 * 1024);
     memory_region_add_subregion(system_memory, 0xf0000000, system_io);
     lx60_fpga_init(system_io, 0x0d020000);
     if (nd_table[0].used) {
@@ -228,16 +273,7 @@
 
     dinfo = drive_get(IF_PFLASH, 0, 0);
     if (dinfo) {
-        flash = pflash_cfi01_register(board->flash_base,
-                NULL, "lx60.io.flash", board->flash_size,
-                blk_by_legacy_dinfo(dinfo),
-                board->flash_sector_size,
-                board->flash_size / board->flash_sector_size,
-                4, 0x0000, 0x0000, 0x0000, 0x0000, be);
-        if (flash == NULL) {
-            error_report("unable to mount pflash\n");
-            exit(EXIT_FAILURE);
-        }
+        flash = xtfpga_flash_init(system_io, board, dinfo, be);
     }
 
     /* Use presence of kernel file name as 'boot from SRAM' switch. */
@@ -257,7 +293,7 @@
 
         rom = g_malloc(sizeof(*rom));
         memory_region_init_ram(rom, NULL, "lx60.sram", board->sram_size,
-                               &error_abort);
+                               &error_fatal);
         vmstate_register_ram_global(rom);
         memory_region_add_subregion(system_memory, 0xfe000000, rom);
 
@@ -287,7 +323,7 @@
             uint32_t dtb_addr = tswap32(cur_lowmem);
 
             if (!fdt) {
-                error_report("could not load DTB '%s'\n", dtb_filename);
+                error_report("could not load DTB '%s'", dtb_filename);
                 exit(EXIT_FAILURE);
             }
 
@@ -307,7 +343,7 @@
                                                   lowmem_end - cur_lowmem);
             }
             if (initrd_size < 0) {
-                error_report("could not load initrd '%s'\n", initrd_filename);
+                error_report("could not load initrd '%s'", initrd_filename);
                 exit(EXIT_FAILURE);
             }
             initrd_location.start = tswap32(cur_lowmem);
@@ -322,7 +358,7 @@
         uint64_t elf_entry;
         uint64_t elf_lowaddr;
         int success = load_elf(kernel_filename, translate_phys_addr, cpu,
-                &elf_entry, &elf_lowaddr, NULL, be, ELF_MACHINE, 0);
+                &elf_entry, &elf_lowaddr, NULL, be, EM_XTENSA, 0, 0);
         if (success > 0) {
             entry_point = elf_entry;
         } else {
@@ -333,7 +369,7 @@
             if (success > 0 && is_linux) {
                 entry_point = ep;
             } else {
-                error_report("could not load kernel '%s'\n",
+                error_report("could not load kernel '%s'",
                              kernel_filename);
                 exit(EXIT_FAILURE);
             }
@@ -367,7 +403,7 @@
 static void xtensa_lx60_init(MachineState *machine)
 {
     static const LxBoardDesc lx60_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x00400000,
         .flash_sector_size = 0x10000,
         .sram_size = 0x20000,
@@ -378,7 +414,7 @@
 static void xtensa_lx200_init(MachineState *machine)
 {
     static const LxBoardDesc lx200_board = {
-        .flash_base = 0xf8000000,
+        .flash_base = 0x08000000,
         .flash_size = 0x01000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
@@ -389,8 +425,8 @@
 static void xtensa_ml605_init(MachineState *machine)
 {
     static const LxBoardDesc ml605_board = {
-        .flash_base = 0xf8000000,
-        .flash_size = 0x02000000,
+        .flash_base = 0x08000000,
+        .flash_size = 0x01000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
     };
@@ -400,7 +436,7 @@
 static void xtensa_kc705_init(MachineState *machine)
 {
     static const LxBoardDesc kc705_board = {
-        .flash_base = 0xf0000000,
+        .flash_base = 0x00000000,
         .flash_size = 0x08000000,
         .flash_boot_base = 0x06000000,
         .flash_sector_size = 0x20000,
@@ -409,40 +445,72 @@
     lx_init(&kc705_board, machine);
 }
 
-static QEMUMachine xtensa_lx60_machine = {
-    .name = "lx60",
-    .desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_lx60_init,
-    .max_cpus = 4,
+static void xtensa_lx60_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lx60 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_lx60_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_lx60_type = {
+    .name = MACHINE_TYPE_NAME("lx60"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_lx60_class_init,
 };
 
-static QEMUMachine xtensa_lx200_machine = {
-    .name = "lx200",
-    .desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_lx200_init,
-    .max_cpus = 4,
+static void xtensa_lx200_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "lx200 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_lx200_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_lx200_type = {
+    .name = MACHINE_TYPE_NAME("lx200"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_lx200_class_init,
 };
 
-static QEMUMachine xtensa_ml605_machine = {
-    .name = "ml605",
-    .desc = "ml605 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_ml605_init,
-    .max_cpus = 4,
+static void xtensa_ml605_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "ml605 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_ml605_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_ml605_type = {
+    .name = MACHINE_TYPE_NAME("ml605"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_ml605_class_init,
 };
 
-static QEMUMachine xtensa_kc705_machine = {
-    .name = "kc705",
-    .desc = "kc705 EVB (" XTENSA_DEFAULT_CPU_MODEL ")",
-    .init = xtensa_kc705_init,
-    .max_cpus = 4,
+static void xtensa_kc705_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+
+    mc->desc = "kc705 EVB (" XTENSA_DEFAULT_CPU_MODEL ")";
+    mc->init = xtensa_kc705_init;
+    mc->max_cpus = 4;
+}
+
+static const TypeInfo xtensa_kc705_type = {
+    .name = MACHINE_TYPE_NAME("kc705"),
+    .parent = TYPE_MACHINE,
+    .class_init = xtensa_kc705_class_init,
 };
 
 static void xtensa_lx_machines_init(void)
 {
-    qemu_register_machine(&xtensa_lx60_machine);
-    qemu_register_machine(&xtensa_lx200_machine);
-    qemu_register_machine(&xtensa_ml605_machine);
-    qemu_register_machine(&xtensa_kc705_machine);
+    type_register_static(&xtensa_lx60_type);
+    type_register_static(&xtensa_lx200_type);
+    type_register_static(&xtensa_ml605_type);
+    type_register_static(&xtensa_kc705_type);
 }
 
-machine_init(xtensa_lx_machines_init);
+type_init(xtensa_lx_machines_init)
diff --git a/include/block/accounting.h b/include/block/accounting.h
index 50b42b3..2089163 100644
--- a/include/block/accounting.h
+++ b/include/block/accounting.h
@@ -2,6 +2,7 @@
  * QEMU System Emulator block accounting
  *
  * Copyright (c) 2011 Christoph Hellwig
+ * Copyright (c) 2015 Igalia, S.L.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -24,9 +25,9 @@
 #ifndef BLOCK_ACCOUNTING_H
 #define BLOCK_ACCOUNTING_H
 
-#include <stdint.h>
+#include "qemu/timed-average.h"
 
-#include "qemu/typedefs.h"
+typedef struct BlockAcctTimedStats BlockAcctTimedStats;
 
 enum BlockAcctType {
     BLOCK_ACCT_READ,
@@ -35,11 +36,23 @@
     BLOCK_MAX_IOTYPE,
 };
 
+struct BlockAcctTimedStats {
+    TimedAverage latency[BLOCK_MAX_IOTYPE];
+    unsigned interval_length; /* in seconds */
+    QSLIST_ENTRY(BlockAcctTimedStats) entries;
+};
+
 typedef struct BlockAcctStats {
     uint64_t nr_bytes[BLOCK_MAX_IOTYPE];
     uint64_t nr_ops[BLOCK_MAX_IOTYPE];
+    uint64_t invalid_ops[BLOCK_MAX_IOTYPE];
+    uint64_t failed_ops[BLOCK_MAX_IOTYPE];
     uint64_t total_time_ns[BLOCK_MAX_IOTYPE];
-    uint64_t wr_highest_sector;
+    uint64_t merged[BLOCK_MAX_IOTYPE];
+    int64_t last_access_time_ns;
+    QSLIST_HEAD(, BlockAcctTimedStats) intervals;
+    bool account_invalid;
+    bool account_failed;
 } BlockAcctStats;
 
 typedef struct BlockAcctCookie {
@@ -48,10 +61,21 @@
     enum BlockAcctType type;
 } BlockAcctCookie;
 
+void block_acct_init(BlockAcctStats *stats, bool account_invalid,
+                     bool account_failed);
+void block_acct_cleanup(BlockAcctStats *stats);
+void block_acct_add_interval(BlockAcctStats *stats, unsigned interval_length);
+BlockAcctTimedStats *block_acct_interval_next(BlockAcctStats *stats,
+                                              BlockAcctTimedStats *s);
 void block_acct_start(BlockAcctStats *stats, BlockAcctCookie *cookie,
                       int64_t bytes, enum BlockAcctType type);
 void block_acct_done(BlockAcctStats *stats, BlockAcctCookie *cookie);
-void block_acct_highest_sector(BlockAcctStats *stats, int64_t sector_num,
-                               unsigned int nb_sectors);
+void block_acct_failed(BlockAcctStats *stats, BlockAcctCookie *cookie);
+void block_acct_invalid(BlockAcctStats *stats, enum BlockAcctType type);
+void block_acct_merge_done(BlockAcctStats *stats, enum BlockAcctType type,
+                           int num_requests);
+int64_t block_acct_idle_time_ns(BlockAcctStats *stats);
+double block_acct_queue_depth(BlockAcctTimedStats *stats,
+                              enum BlockAcctType type);
 
 #endif
diff --git a/include/block/aio.h b/include/block/aio.h
index 6bf0e04..173c1ed 100644
--- a/include/block/aio.h
+++ b/include/block/aio.h
@@ -14,7 +14,6 @@
 #ifndef QEMU_AIO_H
 #define QEMU_AIO_H
 
-#include "qemu/typedefs.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "qemu/event_notifier.h"
@@ -48,6 +47,9 @@
 typedef void QEMUBHFunc(void *opaque);
 typedef void IOHandler(void *opaque);
 
+struct ThreadPool;
+struct LinuxAioState;
+
 struct AioContext {
     GSource source;
 
@@ -63,10 +65,30 @@
      */
     int walking_handlers;
 
-    /* Used to avoid unnecessary event_notifier_set calls in aio_notify.
-     * Writes protected by lock or BQL, reads are lockless.
+    /* Used to avoid unnecessary event_notifier_set calls in aio_notify;
+     * accessed with atomic primitives.  If this field is 0, everything
+     * (file descriptors, bottom halves, timers) will be re-evaluated
+     * before the next blocking poll(), thus the event_notifier_set call
+     * can be skipped.  If it is non-zero, you may need to wake up a
+     * concurrent aio_poll or the glib main event loop, making
+     * event_notifier_set necessary.
+     *
+     * Bit 0 is reserved for GSource usage of the AioContext, and is 1
+     * between a call to aio_ctx_prepare and the next call to aio_ctx_check.
+     * Bits 1-31 simply count the number of active calls to aio_poll
+     * that are in the prepare or poll phase.
+     *
+     * The GSource and aio_poll must use a different mechanism because
+     * there is no certainty that a call to GSource's prepare callback
+     * (via g_main_context_prepare) is indeed followed by check and
+     * dispatch.  It's not clear whether this would be a bug, but let's
+     * play safe and allow it---it will just cause extra calls to
+     * event_notifier_set until the next call to dispatch.
+     *
+     * Instead, the aio_poll calls include both the prepare and the
+     * dispatch phase, hence a simple counter is enough for them.
      */
-    bool dispatching;
+    uint32_t notify_me;
 
     /* lock to protect between bh's adders and deleter */
     QemuMutex bh_lock;
@@ -79,21 +101,44 @@
      */
     int walking_bh;
 
-    /* Used for aio_notify.  */
+    /* Used by aio_notify.
+     *
+     * "notified" is used to avoid expensive event_notifier_test_and_clear
+     * calls.  When it is clear, the EventNotifier is clear, or one thread
+     * is going to clear "notified" before processing more events.  False
+     * positives are possible, i.e. "notified" could be set even though the
+     * EventNotifier is clear.
+     *
+     * Note that event_notifier_set *cannot* be optimized the same way.  For
+     * more information on the problem that would result, see "#ifdef BUG2"
+     * in the docs/aio_notify_accept.promela formal model.
+     */
+    bool notified;
     EventNotifier notifier;
 
-    /* GPollFDs for aio_poll() */
-    GArray *pollfds;
+    /* Scheduling this BH forces the event loop it iterate */
+    QEMUBH *notify_dummy_bh;
 
     /* Thread pool for performing work and receiving completion callbacks */
     struct ThreadPool *thread_pool;
 
+#ifdef CONFIG_LINUX_AIO
+    /* State for native Linux AIO.  Uses aio_context_acquire/release for
+     * locking.
+     */
+    struct LinuxAioState *linux_aio;
+#endif
+
     /* TimerLists for calling timers - one per clock type */
     QEMUTimerListGroup tlg;
-};
 
-/* Used internally to synchronize aio_poll against qemu_bh_schedule.  */
-void aio_set_dispatching(AioContext *ctx, bool dispatching);
+    int external_disable_cnt;
+
+    /* epoll(7) state used when built with CONFIG_EPOLL */
+    int epollfd;
+    bool epoll_enabled;
+    bool epoll_available;
+};
 
 /**
  * aio_context_new: Allocate a new AioContext.
@@ -121,13 +166,14 @@
 void aio_context_unref(AioContext *ctx);
 
 /* Take ownership of the AioContext.  If the AioContext will be shared between
- * threads, a thread must have ownership when calling aio_poll().
+ * threads, and a thread does not want to be interrupted, it will have to
+ * take ownership around calls to aio_poll().  Otherwise, aio_poll()
+ * automatically takes care of calling aio_context_acquire and
+ * aio_context_release.
  *
- * Note that multiple threads calling aio_poll() means timers, BHs, and
- * callbacks may be invoked from a different thread than they were registered
- * from.  Therefore, code must use AioContext acquire/release or use
- * fine-grained synchronization to protect shared state if other threads will
- * be accessing it simultaneously.
+ * Access to timers and BHs from a thread that has not acquired AioContext
+ * is possible.  Access to callbacks for now must be done while the AioContext
+ * is owned by the thread (FIXME).
  */
 void aio_context_acquire(AioContext *ctx);
 
@@ -159,6 +205,29 @@
 void aio_notify(AioContext *ctx);
 
 /**
+ * aio_notify_accept: Acknowledge receiving an aio_notify.
+ *
+ * aio_notify() uses an EventNotifier in order to wake up a sleeping
+ * aio_poll() or g_main_context_iteration().  Calls to aio_notify() are
+ * usually rare, but the AioContext has to clear the EventNotifier on
+ * every aio_poll() or g_main_context_iteration() in order to avoid
+ * busy waiting.  This event_notifier_test_and_clear() cannot be done
+ * using the usual aio_context_set_event_notifier(), because it must
+ * be done before processing all events (file descriptors, bottom halves,
+ * timers).
+ *
+ * aio_notify_accept() is an optimized event_notifier_test_and_clear()
+ * that is specific to an AioContext's notifier; it is used internally
+ * to clear the EventNotifier only if aio_notify() had been called.
+ */
+void aio_notify_accept(AioContext *ctx);
+
+/**
+ * aio_bh_call: Executes callback function of the specified BH.
+ */
+void aio_bh_call(QEMUBH *bh);
+
+/**
  * aio_bh_poll: Poll bottom halves for an AioContext.
  *
  * These are internal functions used by the QEMU main loop.
@@ -243,7 +312,7 @@
 bool aio_poll(AioContext *ctx, bool blocking);
 
 /* Register a file descriptor and associated callbacks.  Behaves very similarly
- * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
+ * to qemu_set_fd_handler.  Unlike qemu_set_fd_handler, these callbacks will
  * be invoked when using aio_poll().
  *
  * Code that invokes AIO completion functions should rely on this function
@@ -251,6 +320,7 @@
  */
 void aio_set_fd_handler(AioContext *ctx,
                         int fd,
+                        bool is_external,
                         IOHandler *io_read,
                         IOHandler *io_write,
                         void *opaque);
@@ -264,6 +334,7 @@
  */
 void aio_set_event_notifier(AioContext *ctx,
                             EventNotifier *notifier,
+                            bool is_external,
                             EventNotifierHandler *io_read);
 
 /* Return a GSource that lets the main loop poll the file descriptors attached
@@ -274,6 +345,9 @@
 /* Return the ThreadPool bound to this AioContext */
 struct ThreadPool *aio_get_thread_pool(AioContext *ctx);
 
+/* Return the LinuxAioState bound to this AioContext */
+struct LinuxAioState *aio_get_linux_aio(AioContext *ctx);
+
 /**
  * aio_timer_new:
  * @ctx: the aio context
@@ -314,7 +388,7 @@
                                   int scale,
                                   QEMUTimerCB *cb, void *opaque)
 {
-    timer_init(ts, ctx->tlg.tl[type], scale, cb, opaque);
+    timer_init_tl(ts, ctx->tlg.tl[type], scale, cb, opaque);
 }
 
 /**
@@ -325,4 +399,59 @@
  */
 int64_t aio_compute_timeout(AioContext *ctx);
 
+/**
+ * aio_disable_external:
+ * @ctx: the aio context
+ *
+ * Disable the further processing of external clients.
+ */
+static inline void aio_disable_external(AioContext *ctx)
+{
+    atomic_inc(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_enable_external:
+ * @ctx: the aio context
+ *
+ * Enable the processing of external clients.
+ */
+static inline void aio_enable_external(AioContext *ctx)
+{
+    assert(ctx->external_disable_cnt > 0);
+    atomic_dec(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_external_disabled:
+ * @ctx: the aio context
+ *
+ * Return true if the external clients are disabled.
+ */
+static inline bool aio_external_disabled(AioContext *ctx)
+{
+    return atomic_read(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_node_check:
+ * @ctx: the aio context
+ * @is_external: Whether or not the checked node is an external event source.
+ *
+ * Check if the node's is_external flag is okay to be polled by the ctx at this
+ * moment. True means green light.
+ */
+static inline bool aio_node_check(AioContext *ctx, bool is_external)
+{
+    return !is_external || !atomic_read(&ctx->external_disable_cnt);
+}
+
+/**
+ * aio_context_setup:
+ * @ctx: the aio context
+ *
+ * Initialize the aio context.
+ */
+void aio_context_setup(AioContext *ctx);
+
 #endif
diff --git a/include/block/block.h b/include/block/block.h
index 131473d..4dc9b4a 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -2,16 +2,24 @@
 #define BLOCK_H
 
 #include "block/aio.h"
-#include "qemu-common.h"
+#include "qemu/iov.h"
 #include "qemu/option.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "block/accounting.h"
+#include "block/dirty-bitmap.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi-types.h"
+#include "qemu/hbitmap.h"
+
+// Avoid conflict with AndroidEmu function with the same name.
+#define path_is_absolute qemu_path_is_absolute
 
 /* block.c */
 typedef struct BlockDriver BlockDriver;
 typedef struct BlockJob BlockJob;
+typedef struct BdrvChild BdrvChild;
+typedef struct BdrvChildRole BdrvChildRole;
+typedef struct BlockJobTxn BlockJobTxn;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
@@ -21,14 +29,14 @@
     bool is_dirty;
     /*
      * True if unallocated blocks read back as zeroes. This is equivalent
-     * to the the LBPRZ flag in the SCSI logical block provisioning page.
+     * to the LBPRZ flag in the SCSI logical block provisioning page.
      */
     bool unallocated_blocks_are_zero;
     /*
      * True if the driver can optimize writing zeroes by unmapping
      * sectors. This is equivalent to the BLKDISCARDZEROES ioctl in Linux
      * with the difference that in qemu a discard is allowed to silently
-     * fail. Therefore we have to use bdrv_write_zeroes with the
+     * fail. Therefore we have to use bdrv_pwrite_zeroes with the
      * BDRV_REQ_MAY_UNMAP flag for an optimized zero write with unmapping.
      * After this call the driver has to guarantee that the contents read
      * back as zero. It is additionally required that the block device is
@@ -49,45 +57,73 @@
 } BlockFragInfo;
 
 typedef enum {
-    BDRV_REQ_COPY_ON_READ = 0x1,
-    BDRV_REQ_ZERO_WRITE   = 0x2,
+    BDRV_REQ_COPY_ON_READ       = 0x1,
+    BDRV_REQ_ZERO_WRITE         = 0x2,
     /* The BDRV_REQ_MAY_UNMAP flag is used to indicate that the block driver
      * is allowed to optimize a write zeroes request by unmapping (discarding)
      * blocks if it is guaranteed that the result will read back as
      * zeroes. The flag is only passed to the driver if the block device is
      * opened with BDRV_O_UNMAP.
      */
-    BDRV_REQ_MAY_UNMAP    = 0x4,
+    BDRV_REQ_MAY_UNMAP          = 0x4,
+    BDRV_REQ_NO_SERIALISING     = 0x8,
+    BDRV_REQ_FUA                = 0x10,
+
+    /* Mask of valid flags */
+    BDRV_REQ_MASK               = 0x1f,
 } BdrvRequestFlags;
 
+typedef struct BlockSizes {
+    uint32_t phys;
+    uint32_t log;
+} BlockSizes;
+
+typedef struct HDGeometry {
+    uint32_t heads;
+    uint32_t sectors;
+    uint32_t cylinders;
+} HDGeometry;
+
 #define BDRV_O_RDWR        0x0002
 #define BDRV_O_SNAPSHOT    0x0008 /* open the file read only and save writes in a snapshot */
 #define BDRV_O_TEMPORARY   0x0010 /* delete the file after use */
 #define BDRV_O_NOCACHE     0x0020 /* do not use the host page cache */
-#define BDRV_O_CACHE_WB    0x0040 /* use write-back caching */
 #define BDRV_O_NATIVE_AIO  0x0080 /* use native AIO instead of the thread pool */
 #define BDRV_O_NO_BACKING  0x0100 /* don't open the backing file */
 #define BDRV_O_NO_FLUSH    0x0200 /* disable flushing on this disk */
 #define BDRV_O_COPY_ON_READ 0x0400 /* copy read backing sectors into image */
-#define BDRV_O_INCOMING    0x0800  /* consistency hint for incoming migration */
+#define BDRV_O_INACTIVE    0x0800  /* consistency hint for migration handoff */
 #define BDRV_O_CHECK       0x1000  /* open solely for consistency check */
 #define BDRV_O_ALLOW_RDWR  0x2000  /* allow reopen to change from r/o to r/w */
 #define BDRV_O_UNMAP       0x4000  /* execute guest UNMAP/TRIM operations */
 #define BDRV_O_PROTOCOL    0x8000  /* if no block driver is explicitly given:
                                       select an appropriate protocol driver,
                                       ignoring the format layer */
+#define BDRV_O_NO_IO       0x10000 /* don't initialize for I/O */
 
-#define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH)
+#define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_NO_FLUSH)
+
+
+/* Option names of options parsed by the block layer */
+
+#define BDRV_OPT_CACHE_WB       "cache.writeback"
+#define BDRV_OPT_CACHE_DIRECT   "cache.direct"
+#define BDRV_OPT_CACHE_NO_FLUSH "cache.no-flush"
+
 
 #define BDRV_SECTOR_BITS   9
 #define BDRV_SECTOR_SIZE   (1ULL << BDRV_SECTOR_BITS)
 #define BDRV_SECTOR_MASK   ~(BDRV_SECTOR_SIZE - 1)
 
+#define BDRV_REQUEST_MAX_SECTORS MIN(SIZE_MAX >> BDRV_SECTOR_BITS, \
+                                     INT_MAX >> BDRV_SECTOR_BITS)
+
 /*
  * Allocation status flags
- * BDRV_BLOCK_DATA: data is read from bs->file or another file
+ * BDRV_BLOCK_DATA: data is read from a file returned by bdrv_get_block_status.
  * BDRV_BLOCK_ZERO: sectors read as zero
- * BDRV_BLOCK_OFFSET_VALID: sector stored in bs->file as raw data
+ * BDRV_BLOCK_OFFSET_VALID: sector stored as raw data in a file returned by
+ *                          bdrv_get_block_status.
  * BDRV_BLOCK_ALLOCATED: the content of the block is determined by this
  *                       layer (as opposed to the backing file)
  * BDRV_BLOCK_RAW: used internally to indicate that the request
@@ -123,6 +159,8 @@
 typedef struct BDRVReopenState {
     BlockDriverState *bs;
     int flags;
+    QDict *options;
+    QDict *explicit_options;
     void *opaque;
 } BDRVReopenState;
 
@@ -133,103 +171,89 @@
     BLOCK_OP_TYPE_BACKUP_SOURCE,
     BLOCK_OP_TYPE_BACKUP_TARGET,
     BLOCK_OP_TYPE_CHANGE,
-    BLOCK_OP_TYPE_COMMIT,
+    BLOCK_OP_TYPE_COMMIT_SOURCE,
+    BLOCK_OP_TYPE_COMMIT_TARGET,
     BLOCK_OP_TYPE_DATAPLANE,
     BLOCK_OP_TYPE_DRIVE_DEL,
     BLOCK_OP_TYPE_EJECT,
     BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
     BLOCK_OP_TYPE_INTERNAL_SNAPSHOT,
     BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE,
-    BLOCK_OP_TYPE_MIRROR,
+    BLOCK_OP_TYPE_MIRROR_SOURCE,
+    BLOCK_OP_TYPE_MIRROR_TARGET,
     BLOCK_OP_TYPE_RESIZE,
     BLOCK_OP_TYPE_STREAM,
     BLOCK_OP_TYPE_REPLACE,
     BLOCK_OP_TYPE_MAX,
 } BlockOpType;
 
-void bdrv_iostatus_enable(BlockDriverState *bs);
-void bdrv_iostatus_reset(BlockDriverState *bs);
-void bdrv_iostatus_disable(BlockDriverState *bs);
-bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
-void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
 void bdrv_info_print(Monitor *mon, const QObject *data);
 void bdrv_info(Monitor *mon, QObject **ret_data);
 void bdrv_stats_print(Monitor *mon, const QObject *data);
 void bdrv_info_stats(Monitor *mon, QObject **ret_data);
 
 /* disk I/O throttling */
-void bdrv_io_limits_enable(BlockDriverState *bs);
-void bdrv_io_limits_disable(BlockDriverState *bs);
-
 void bdrv_init(void);
 void bdrv_init_with_whitelist(void);
+bool bdrv_uses_whitelist(void);
 BlockDriver *bdrv_find_protocol(const char *filename,
-                                bool allow_protocol_prefix);
+                                bool allow_protocol_prefix,
+                                Error **errp);
 BlockDriver *bdrv_find_format(const char *format_name);
-BlockDriver *bdrv_find_whitelisted_format(const char *format_name,
-                                          bool readonly);
 int bdrv_create(BlockDriver *drv, const char* filename,
                 QemuOpts *opts, Error **errp);
 int bdrv_create_file(const char *filename, QemuOpts *opts, Error **errp);
-BlockDriverState *bdrv_new_root(void);
 BlockDriverState *bdrv_new(void);
-void bdrv_make_anon(BlockDriverState *bs);
-void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old);
 void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
-int bdrv_parse_cache_flags(const char *mode, int *flags);
+void bdrv_replace_in_backing_chain(BlockDriverState *old,
+                                   BlockDriverState *new);
+
+int bdrv_parse_cache_mode(const char *mode, int *flags, bool *writethrough);
 int bdrv_parse_discard_flags(const char *mode, int *flags);
-int bdrv_open_image(BlockDriverState **pbs, const char *filename,
-                    QDict *options, const char *bdref_key, int flags,
-                    bool allow_none, Error **errp);
+BdrvChild *bdrv_open_child(const char *filename,
+                           QDict *options, const char *bdref_key,
+                           BlockDriverState* parent,
+                           const BdrvChildRole *child_role,
+                           bool allow_none, Error **errp);
 void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd);
-int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp);
-int bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp);
-int bdrv_open(BlockDriverState **pbs, const char *filename,
-              const char *reference, QDict *options, int flags,
-              BlockDriver *drv, Error **errp);
+int bdrv_open_backing_file(BlockDriverState *bs, QDict *parent_options,
+                           const char *bdref_key, Error **errp);
+BlockDriverState *bdrv_open(const char *filename, const char *reference,
+                            QDict *options, int flags, Error **errp);
 BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
-                                    BlockDriverState *bs, int flags);
+                                    BlockDriverState *bs,
+                                    QDict *options, int flags);
 int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
 int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
 int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
                         BlockReopenQueue *queue, Error **errp);
 void bdrv_reopen_commit(BDRVReopenState *reopen_state);
 void bdrv_reopen_abort(BDRVReopenState *reopen_state);
-void bdrv_close(BlockDriverState *bs);
-void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
-int bdrv_read(BlockDriverState *bs, int64_t sector_num,
+int bdrv_read(BdrvChild *child, int64_t sector_num,
               uint8_t *buf, int nb_sectors);
-int bdrv_read_unthrottled(BlockDriverState *bs, int64_t sector_num,
-                          uint8_t *buf, int nb_sectors);
-int bdrv_write(BlockDriverState *bs, int64_t sector_num,
+int bdrv_write(BdrvChild *child, int64_t sector_num,
                const uint8_t *buf, int nb_sectors);
-int bdrv_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-               int nb_sectors, BdrvRequestFlags flags);
-BlockAIOCB *bdrv_aio_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-                                  int nb_sectors, BdrvRequestFlags flags,
-                                  BlockCompletionFunc *cb, void *opaque);
-int bdrv_make_zero(BlockDriverState *bs, BdrvRequestFlags flags);
-int bdrv_pread(BlockDriverState *bs, int64_t offset,
-               void *buf, int count);
-int bdrv_pwrite(BlockDriverState *bs, int64_t offset,
-                const void *buf, int count);
-int bdrv_pwritev(BlockDriverState *bs, int64_t offset, QEMUIOVector *qiov);
-int bdrv_pwrite_sync(BlockDriverState *bs, int64_t offset,
-    const void *buf, int count);
-int coroutine_fn bdrv_co_readv(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov);
-int coroutine_fn bdrv_co_copy_on_readv(BlockDriverState *bs,
-    int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
-int coroutine_fn bdrv_co_writev(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, QEMUIOVector *qiov);
+int bdrv_pwrite_zeroes(BdrvChild *child, int64_t offset,
+                       int count, BdrvRequestFlags flags);
+int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags);
+int bdrv_pread(BdrvChild *child, int64_t offset, void *buf, int bytes);
+int bdrv_preadv(BdrvChild *child, int64_t offset, QEMUIOVector *qiov);
+int bdrv_pwrite(BdrvChild *child, int64_t offset, const void *buf, int bytes);
+int bdrv_pwritev(BdrvChild *child, int64_t offset, QEMUIOVector *qiov);
+int bdrv_pwrite_sync(BdrvChild *child, int64_t offset,
+                     const void *buf, int count);
+int coroutine_fn bdrv_co_readv(BdrvChild *child, int64_t sector_num,
+                               int nb_sectors, QEMUIOVector *qiov);
+int coroutine_fn bdrv_co_writev(BdrvChild *child, int64_t sector_num,
+                               int nb_sectors, QEMUIOVector *qiov);
 /*
  * Efficiently zero a region of the disk image.  Note that this is a regular
  * I/O request like read or write and should have a reasonable size.  This
  * function is not suitable for zeroing the entire image in a single request
  * because it may allocate memory for the entire region.
  */
-int coroutine_fn bdrv_co_write_zeroes(BlockDriverState *bs, int64_t sector_num,
-    int nb_sectors, BdrvRequestFlags flags);
+int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
+                                       int count, BdrvRequestFlags flags);
 BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
     const char *backing_file);
 int bdrv_get_backing_file_depth(BlockDriverState *bs);
@@ -241,7 +265,6 @@
 void bdrv_get_geometry(BlockDriverState *bs, uint64_t *nb_sectors_ptr);
 void bdrv_refresh_limits(BlockDriverState *bs, Error **errp);
 int bdrv_commit(BlockDriverState *bs);
-int bdrv_commit_all(void);
 int bdrv_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt);
 void bdrv_register(BlockDriver *bdrv);
@@ -274,9 +297,9 @@
  * block driver; total_work_size may change during the course of the amendment
  * operation */
 typedef void BlockDriverAmendStatusCB(BlockDriverState *bs, int64_t offset,
-                                      int64_t total_work_size);
+                                      int64_t total_work_size, void *opaque);
 int bdrv_amend_options(BlockDriverState *bs_new, QemuOpts *opts,
-                       BlockDriverAmendStatusCB *status_cb);
+                       BlockDriverAmendStatusCB *status_cb, void *cb_opaque);
 
 /* external snapshots */
 bool bdrv_recurse_is_first_non_filter(BlockDriverState *bs,
@@ -284,41 +307,24 @@
 bool bdrv_is_first_non_filter(BlockDriverState *candidate);
 
 /* check if a named node can be replaced when doing drive-mirror */
-BlockDriverState *check_to_replace_node(const char *node_name, Error **errp);
+BlockDriverState *check_to_replace_node(BlockDriverState *parent_bs,
+                                        const char *node_name, Error **errp);
 
 /* async block I/O */
-typedef void BlockDriverDirtyHandler(BlockDriverState *bs, int64_t sector,
-                                     int sector_num);
-BlockAIOCB *bdrv_aio_readv(BlockDriverState *bs, int64_t sector_num,
+BlockAIOCB *bdrv_aio_readv(BdrvChild *child, int64_t sector_num,
                            QEMUIOVector *iov, int nb_sectors,
                            BlockCompletionFunc *cb, void *opaque);
-BlockAIOCB *bdrv_aio_writev(BlockDriverState *bs, int64_t sector_num,
+BlockAIOCB *bdrv_aio_writev(BdrvChild *child, int64_t sector_num,
                             QEMUIOVector *iov, int nb_sectors,
                             BlockCompletionFunc *cb, void *opaque);
 BlockAIOCB *bdrv_aio_flush(BlockDriverState *bs,
                            BlockCompletionFunc *cb, void *opaque);
-BlockAIOCB *bdrv_aio_discard(BlockDriverState *bs,
-                             int64_t sector_num, int nb_sectors,
-                             BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *bdrv_aio_pdiscard(BlockDriverState *bs,
+                              int64_t offset, int count,
+                              BlockCompletionFunc *cb, void *opaque);
 void bdrv_aio_cancel(BlockAIOCB *acb);
 void bdrv_aio_cancel_async(BlockAIOCB *acb);
 
-typedef struct BlockRequest {
-    /* Fields to be filled by multiwrite caller */
-    int64_t sector;
-    int nb_sectors;
-    int flags;
-    QEMUIOVector *qiov;
-    BlockCompletionFunc *cb;
-    void *opaque;
-
-    /* Filled by multiwrite implementation */
-    int error;
-} BlockRequest;
-
-int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs,
-    int num_reqs);
-
 /* sg packet commands */
 int bdrv_ioctl(BlockDriverState *bs, unsigned long int req, void *buf);
 BlockAIOCB *bdrv_aio_ioctl(BlockDriverState *bs,
@@ -328,81 +334,105 @@
 /* Invalidate any cached metadata used by image formats */
 void bdrv_invalidate_cache(BlockDriverState *bs, Error **errp);
 void bdrv_invalidate_cache_all(Error **errp);
-
-void bdrv_clear_incoming_migration_all(void);
+int bdrv_inactivate_all(void);
 
 /* Ensure contents are flushed to disk.  */
 int bdrv_flush(BlockDriverState *bs);
 int coroutine_fn bdrv_co_flush(BlockDriverState *bs);
-int bdrv_flush_all(void);
 void bdrv_close_all(void);
 void bdrv_drain(BlockDriverState *bs);
+void coroutine_fn bdrv_co_drain(BlockDriverState *bs);
 void bdrv_drain_all(void);
 
-int bdrv_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
-int bdrv_co_discard(BlockDriverState *bs, int64_t sector_num, int nb_sectors);
+int bdrv_pdiscard(BlockDriverState *bs, int64_t offset, int count);
+int bdrv_co_pdiscard(BlockDriverState *bs, int64_t offset, int count);
 int bdrv_has_zero_init_1(BlockDriverState *bs);
 int bdrv_has_zero_init(BlockDriverState *bs);
 bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs);
 bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs);
 int64_t bdrv_get_block_status(BlockDriverState *bs, int64_t sector_num,
-                              int nb_sectors, int *pnum);
+                              int nb_sectors, int *pnum,
+                              BlockDriverState **file);
+int64_t bdrv_get_block_status_above(BlockDriverState *bs,
+                                    BlockDriverState *base,
+                                    int64_t sector_num,
+                                    int nb_sectors, int *pnum,
+                                    BlockDriverState **file);
 int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
                       int *pnum);
 int bdrv_is_allocated_above(BlockDriverState *top, BlockDriverState *base,
                             int64_t sector_num, int nb_sectors, int *pnum);
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
-                       BlockdevOnError on_write_error);
-BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
-BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
-void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
-                       bool is_read, int error);
-int bdrv_is_read_only(BlockDriverState *bs);
-int bdrv_is_sg(BlockDriverState *bs);
-int bdrv_enable_write_cache(BlockDriverState *bs);
-void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce);
-int bdrv_is_inserted(BlockDriverState *bs);
+bool bdrv_is_read_only(BlockDriverState *bs);
+bool bdrv_is_sg(BlockDriverState *bs);
+bool bdrv_is_inserted(BlockDriverState *bs);
 int bdrv_media_changed(BlockDriverState *bs);
 void bdrv_lock_medium(BlockDriverState *bs, bool locked);
 void bdrv_eject(BlockDriverState *bs, bool eject_flag);
 const char *bdrv_get_format_name(BlockDriverState *bs);
-BlockDriverState *bdrv_find(const char *name);
 BlockDriverState *bdrv_find_node(const char *node_name);
-BlockDeviceInfoList *bdrv_named_nodes_list(void);
+BlockDeviceInfoList *bdrv_named_nodes_list(Error **errp);
 BlockDriverState *bdrv_lookup_bs(const char *device,
                                  const char *node_name,
                                  Error **errp);
 bool bdrv_chain_contains(BlockDriverState *top, BlockDriverState *base);
-BlockDriverState *bdrv_next(BlockDriverState *bs);
-int bdrv_is_encrypted(BlockDriverState *bs);
-int bdrv_key_required(BlockDriverState *bs);
+BlockDriverState *bdrv_next_node(BlockDriverState *bs);
+
+typedef struct BdrvNextIterator {
+    enum {
+        BDRV_NEXT_BACKEND_ROOTS,
+        BDRV_NEXT_MONITOR_OWNED,
+    } phase;
+    BlockBackend *blk;
+    BlockDriverState *bs;
+} BdrvNextIterator;
+
+BlockDriverState *bdrv_first(BdrvNextIterator *it);
+BlockDriverState *bdrv_next(BdrvNextIterator *it);
+
+BlockDriverState *bdrv_next_monitor_owned(BlockDriverState *bs);
+bool bdrv_is_encrypted(BlockDriverState *bs);
+bool bdrv_key_required(BlockDriverState *bs);
 int bdrv_set_key(BlockDriverState *bs, const char *key);
+void bdrv_add_key(BlockDriverState *bs, const char *key, Error **errp);
 int bdrv_query_missing_keys(void);
 void bdrv_iterate_format(void (*it)(void *opaque, const char *name),
                          void *opaque);
+const char *bdrv_get_node_name(const BlockDriverState *bs);
 const char *bdrv_get_device_name(const BlockDriverState *bs);
+const char *bdrv_get_device_or_node_name(const BlockDriverState *bs);
 int bdrv_get_flags(BlockDriverState *bs);
 int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
                           const uint8_t *buf, int nb_sectors);
 int bdrv_get_info(BlockDriverState *bs, BlockDriverInfo *bdi);
 ImageInfoSpecific *bdrv_get_specific_info(BlockDriverState *bs);
+void bdrv_round_sectors_to_clusters(BlockDriverState *bs,
+                                    int64_t sector_num, int nb_sectors,
+                                    int64_t *cluster_sector_num,
+                                    int *cluster_nb_sectors);
 void bdrv_round_to_clusters(BlockDriverState *bs,
-                            int64_t sector_num, int nb_sectors,
-                            int64_t *cluster_sector_num,
-                            int *cluster_nb_sectors);
+                            int64_t offset, unsigned int bytes,
+                            int64_t *cluster_offset,
+                            unsigned int *cluster_bytes);
 
 const char *bdrv_get_encrypted_filename(BlockDriverState *bs);
 void bdrv_get_backing_filename(BlockDriverState *bs,
                                char *filename, int filename_size);
 void bdrv_get_full_backing_filename(BlockDriverState *bs,
-                                    char *dest, size_t sz);
+                                    char *dest, size_t sz, Error **errp);
+void bdrv_get_full_backing_filename_from_filename(const char *backed,
+                                                  const char *backing,
+                                                  char *dest, size_t sz,
+                                                  Error **errp);
 int bdrv_is_snapshot(BlockDriverState *bs);
 
+int path_has_protocol(const char *path);
+int path_is_absolute(const char *path);
 void path_combine(char *dest, int dest_size,
                   const char *base_path,
                   const char *filename);
 
+int bdrv_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos);
 int bdrv_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos);
 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
                       int64_t pos, int size);
@@ -417,32 +447,25 @@
 
 /* Returns the alignment in bytes that is required so that no bounce buffer
  * is required throughout the stack */
+size_t bdrv_min_mem_align(BlockDriverState *bs);
+/* Returns optimal alignment in bytes for bounce buffer */
 size_t bdrv_opt_mem_align(BlockDriverState *bs);
-void bdrv_set_guest_block_size(BlockDriverState *bs, int align);
 void *qemu_blockalign(BlockDriverState *bs, size_t size);
 void *qemu_blockalign0(BlockDriverState *bs, size_t size);
 void *qemu_try_blockalign(BlockDriverState *bs, size_t size);
 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size);
 bool bdrv_qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov);
 
-struct HBitmapIter;
-typedef struct BdrvDirtyBitmap BdrvDirtyBitmap;
-BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs, int granularity,
-                                          Error **errp);
-void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
-BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs);
-int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap, int64_t sector);
-void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
-void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
-void bdrv_dirty_iter_init(BlockDriverState *bs,
-                          BdrvDirtyBitmap *bitmap, struct HBitmapIter *hbi);
-int64_t bdrv_get_dirty_count(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
-
 void bdrv_enable_copy_on_read(BlockDriverState *bs);
 void bdrv_disable_copy_on_read(BlockDriverState *bs);
 
 void bdrv_ref(BlockDriverState *bs);
 void bdrv_unref(BlockDriverState *bs);
+void bdrv_unref_child(BlockDriverState *parent, BdrvChild *child);
+BdrvChild *bdrv_attach_child(BlockDriverState *parent_bs,
+                             BlockDriverState *child_bs,
+                             const char *child_name,
+                             const BdrvChildRole *child_role);
 
 bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp);
 void bdrv_op_block(BlockDriverState *bs, BlockOpType op, Error *reason);
@@ -451,68 +474,14 @@
 void bdrv_op_unblock_all(BlockDriverState *bs, Error *reason);
 bool bdrv_op_blocker_is_empty(BlockDriverState *bs);
 
-typedef enum {
-    BLKDBG_L1_UPDATE,
+#define BLKDBG_EVENT(child, evt) \
+    do { \
+        if (child) { \
+            bdrv_debug_event(child->bs, evt); \
+        } \
+    } while (0)
 
-    BLKDBG_L1_GROW_ALLOC_TABLE,
-    BLKDBG_L1_GROW_WRITE_TABLE,
-    BLKDBG_L1_GROW_ACTIVATE_TABLE,
-
-    BLKDBG_L2_LOAD,
-    BLKDBG_L2_UPDATE,
-    BLKDBG_L2_UPDATE_COMPRESSED,
-    BLKDBG_L2_ALLOC_COW_READ,
-    BLKDBG_L2_ALLOC_WRITE,
-
-    BLKDBG_READ_AIO,
-    BLKDBG_READ_BACKING_AIO,
-    BLKDBG_READ_COMPRESSED,
-
-    BLKDBG_WRITE_AIO,
-    BLKDBG_WRITE_COMPRESSED,
-
-    BLKDBG_VMSTATE_LOAD,
-    BLKDBG_VMSTATE_SAVE,
-
-    BLKDBG_COW_READ,
-    BLKDBG_COW_WRITE,
-
-    BLKDBG_REFTABLE_LOAD,
-    BLKDBG_REFTABLE_GROW,
-    BLKDBG_REFTABLE_UPDATE,
-
-    BLKDBG_REFBLOCK_LOAD,
-    BLKDBG_REFBLOCK_UPDATE,
-    BLKDBG_REFBLOCK_UPDATE_PART,
-    BLKDBG_REFBLOCK_ALLOC,
-    BLKDBG_REFBLOCK_ALLOC_HOOKUP,
-    BLKDBG_REFBLOCK_ALLOC_WRITE,
-    BLKDBG_REFBLOCK_ALLOC_WRITE_BLOCKS,
-    BLKDBG_REFBLOCK_ALLOC_WRITE_TABLE,
-    BLKDBG_REFBLOCK_ALLOC_SWITCH_TABLE,
-
-    BLKDBG_CLUSTER_ALLOC,
-    BLKDBG_CLUSTER_ALLOC_BYTES,
-    BLKDBG_CLUSTER_FREE,
-
-    BLKDBG_FLUSH_TO_OS,
-    BLKDBG_FLUSH_TO_DISK,
-
-    BLKDBG_PWRITEV_RMW_HEAD,
-    BLKDBG_PWRITEV_RMW_AFTER_HEAD,
-    BLKDBG_PWRITEV_RMW_TAIL,
-    BLKDBG_PWRITEV_RMW_AFTER_TAIL,
-    BLKDBG_PWRITEV,
-    BLKDBG_PWRITEV_ZERO,
-    BLKDBG_PWRITEV_DONE,
-
-    BLKDBG_EMPTY_IMAGE_PREPARE,
-
-    BLKDBG_EVENT_MAX,
-} BlkDebugEvent;
-
-#define BLKDBG_EVENT(bs, evt) bdrv_debug_event(bs, evt)
-void bdrv_debug_event(BlockDriverState *bs, BlkDebugEvent event);
+void bdrv_debug_event(BlockDriverState *bs, BlkdebugEvent event);
 
 int bdrv_debug_breakpoint(BlockDriverState *bs, const char *event,
                            const char *tag);
@@ -533,15 +502,38 @@
  * Changes the #AioContext used for fd handlers, timers, and BHs by this
  * BlockDriverState and all its children.
  *
- * This function must be called from the old #AioContext or with a lock held so
- * the old #AioContext is not executing.
+ * This function must be called with iothread lock held.
  */
 void bdrv_set_aio_context(BlockDriverState *bs, AioContext *new_context);
+int bdrv_probe_blocksizes(BlockDriverState *bs, BlockSizes *bsz);
+int bdrv_probe_geometry(BlockDriverState *bs, HDGeometry *geo);
 
 void bdrv_io_plug(BlockDriverState *bs);
 void bdrv_io_unplug(BlockDriverState *bs);
-void bdrv_flush_io_queue(BlockDriverState *bs);
+void bdrv_io_unplugged_begin(BlockDriverState *bs);
+void bdrv_io_unplugged_end(BlockDriverState *bs);
 
-BlockAcctStats *bdrv_get_stats(BlockDriverState *bs);
+/**
+ * bdrv_drained_begin:
+ *
+ * Begin a quiesced section for exclusive access to the BDS, by disabling
+ * external request sources including NBD server and device model. Note that
+ * this doesn't block timers or coroutines from submitting more requests, which
+ * means block_job_pause is still necessary.
+ *
+ * This function can be recursive.
+ */
+void bdrv_drained_begin(BlockDriverState *bs);
+
+/**
+ * bdrv_drained_end:
+ *
+ * End a quiescent section started by bdrv_drained_begin().
+ */
+void bdrv_drained_end(BlockDriverState *bs);
+
+void bdrv_add_child(BlockDriverState *parent, BlockDriverState *child,
+                    Error **errp);
+void bdrv_del_child(BlockDriverState *parent, BdrvChild *child, Error **errp);
 
 #endif
diff --git a/include/block/block_int.h b/include/block/block_int.h
index a1c17b9..1e939de 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -28,23 +28,21 @@
 #include "block/block.h"
 #include "qemu/option.h"
 #include "qemu/queue.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "qemu/timer.h"
 #include "qapi-types.h"
-#include "qapi/qmp/qerror.h"
-#include "monitor/monitor.h"
 #include "qemu/hbitmap.h"
 #include "block/snapshot.h"
 #include "qemu/main-loop.h"
 #include "qemu/throttle.h"
 
 #define BLOCK_FLAG_ENCRYPT          1
-#define BLOCK_FLAG_COMPAT6          4
 #define BLOCK_FLAG_LAZY_REFCOUNTS   8
 
 #define BLOCK_OPT_SIZE              "size"
 #define BLOCK_OPT_ENCRYPT           "encryption"
 #define BLOCK_OPT_COMPAT6           "compat6"
+#define BLOCK_OPT_HWVERSION         "hwversion"
 #define BLOCK_OPT_BACKING_FILE      "backing_file"
 #define BLOCK_OPT_BACKING_FMT       "backing_fmt"
 #define BLOCK_OPT_CLUSTER_SIZE      "cluster_size"
@@ -56,12 +54,24 @@
 #define BLOCK_OPT_ADAPTER_TYPE      "adapter_type"
 #define BLOCK_OPT_REDUNDANCY        "redundancy"
 #define BLOCK_OPT_NOCOW             "nocow"
+#define BLOCK_OPT_OBJECT_SIZE       "object_size"
+#define BLOCK_OPT_REFCOUNT_BITS     "refcount_bits"
+
+#define BLOCK_PROBE_BUF_SIZE        512
+
+enum BdrvTrackedRequestType {
+    BDRV_TRACKED_READ,
+    BDRV_TRACKED_WRITE,
+    BDRV_TRACKED_FLUSH,
+    BDRV_TRACKED_IOCTL,
+    BDRV_TRACKED_DISCARD,
+};
 
 typedef struct BdrvTrackedRequest {
     BlockDriverState *bs;
     int64_t offset;
     unsigned int bytes;
-    bool is_write;
+    enum BdrvTrackedRequestType type;
 
     bool serialising;
     int64_t overlap_offset;
@@ -110,22 +120,18 @@
                                BlockReopenQueue *queue, Error **errp);
     void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
     void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
+    void (*bdrv_join_options)(QDict *options, QDict *old_options);
 
     int (*bdrv_open)(BlockDriverState *bs, QDict *options, int flags,
                      Error **errp);
     int (*bdrv_file_open)(BlockDriverState *bs, QDict *options, int flags,
                           Error **errp);
-    int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num,
-                     uint8_t *buf, int nb_sectors);
-    int (*bdrv_write)(BlockDriverState *bs, int64_t sector_num,
-                      const uint8_t *buf, int nb_sectors);
     void (*bdrv_close)(BlockDriverState *bs);
-    void (*bdrv_rebind)(BlockDriverState *bs);
     int (*bdrv_create)(const char *filename, QemuOpts *opts, Error **errp);
     int (*bdrv_set_key)(BlockDriverState *bs, const char *key);
     int (*bdrv_make_empty)(BlockDriverState *bs);
 
-    void (*bdrv_refresh_filename)(BlockDriverState *bs);
+    void (*bdrv_refresh_filename)(BlockDriverState *bs, QDict *options);
 
     /* aio */
     BlockAIOCB *(*bdrv_aio_readv)(BlockDriverState *bs,
@@ -136,31 +142,47 @@
         BlockCompletionFunc *cb, void *opaque);
     BlockAIOCB *(*bdrv_aio_flush)(BlockDriverState *bs,
         BlockCompletionFunc *cb, void *opaque);
-    BlockAIOCB *(*bdrv_aio_discard)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors,
+    BlockAIOCB *(*bdrv_aio_pdiscard)(BlockDriverState *bs,
+        int64_t offset, int count,
         BlockCompletionFunc *cb, void *opaque);
 
     int coroutine_fn (*bdrv_co_readv)(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+    int coroutine_fn (*bdrv_co_preadv)(BlockDriverState *bs,
+        uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags);
     int coroutine_fn (*bdrv_co_writev)(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, QEMUIOVector *qiov);
+    int coroutine_fn (*bdrv_co_writev_flags)(BlockDriverState *bs,
+        int64_t sector_num, int nb_sectors, QEMUIOVector *qiov, int flags);
+    int coroutine_fn (*bdrv_co_pwritev)(BlockDriverState *bs,
+        uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags);
+
     /*
      * Efficiently zero a region of the disk image.  Typically an image format
      * would use a compact metadata representation to implement this.  This
-     * function pointer may be NULL and .bdrv_co_writev() will be called
-     * instead.
+     * function pointer may be NULL or return -ENOSUP and .bdrv_co_writev()
+     * will be called instead.
      */
-    int coroutine_fn (*bdrv_co_write_zeroes)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, BdrvRequestFlags flags);
-    int coroutine_fn (*bdrv_co_discard)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors);
+    int coroutine_fn (*bdrv_co_pwrite_zeroes)(BlockDriverState *bs,
+        int64_t offset, int count, BdrvRequestFlags flags);
+    int coroutine_fn (*bdrv_co_pdiscard)(BlockDriverState *bs,
+        int64_t offset, int count);
     int64_t coroutine_fn (*bdrv_co_get_block_status)(BlockDriverState *bs,
-        int64_t sector_num, int nb_sectors, int *pnum);
+        int64_t sector_num, int nb_sectors, int *pnum,
+        BlockDriverState **file);
 
     /*
      * Invalidate any cached meta-data.
      */
     void (*bdrv_invalidate_cache)(BlockDriverState *bs, Error **errp);
+    int (*bdrv_inactivate)(BlockDriverState *bs);
+
+    /*
+     * Flushes all data for all layers by calling bdrv_co_flush for underlying
+     * layers, if needed. This function is needed for deterministic
+     * synchronization of the flush finishing callback.
+     */
+    int coroutine_fn (*bdrv_co_flush)(BlockDriverState *bs);
 
     /*
      * Flushes all data that was already written to the OS all the way down to
@@ -202,22 +224,23 @@
     int (*bdrv_get_info)(BlockDriverState *bs, BlockDriverInfo *bdi);
     ImageInfoSpecific *(*bdrv_get_specific_info)(BlockDriverState *bs);
 
-    int (*bdrv_save_vmstate)(BlockDriverState *bs, QEMUIOVector *qiov,
-                             int64_t pos);
-    int (*bdrv_load_vmstate)(BlockDriverState *bs, uint8_t *buf,
-                             int64_t pos, int size);
+    int coroutine_fn (*bdrv_save_vmstate)(BlockDriverState *bs,
+                                          QEMUIOVector *qiov,
+                                          int64_t pos);
+    int coroutine_fn (*bdrv_load_vmstate)(BlockDriverState *bs,
+                                          QEMUIOVector *qiov,
+                                          int64_t pos);
 
     int (*bdrv_change_backing_file)(BlockDriverState *bs,
         const char *backing_file, const char *backing_fmt);
 
     /* removable device specific */
-    int (*bdrv_is_inserted)(BlockDriverState *bs);
+    bool (*bdrv_is_inserted)(BlockDriverState *bs);
     int (*bdrv_media_changed)(BlockDriverState *bs);
     void (*bdrv_eject)(BlockDriverState *bs, bool eject_flag);
     void (*bdrv_lock_medium)(BlockDriverState *bs, bool locked);
 
     /* to control generic scsi devices */
-    int (*bdrv_ioctl)(BlockDriverState *bs, unsigned long int req, void *buf);
     BlockAIOCB *(*bdrv_aio_ioctl)(BlockDriverState *bs,
         unsigned long int req, void *buf,
         BlockCompletionFunc *cb, void *opaque);
@@ -233,9 +256,10 @@
         BdrvCheckMode fix);
 
     int (*bdrv_amend_options)(BlockDriverState *bs, QemuOpts *opts,
-                              BlockDriverAmendStatusCB *status_cb);
+                              BlockDriverAmendStatusCB *status_cb,
+                              void *cb_opaque);
 
-    void (*bdrv_debug_event)(BlockDriverState *bs, BlkDebugEvent event);
+    void (*bdrv_debug_event)(BlockDriverState *bs, BlkdebugEvent event);
 
     /* TODO Better pass a option string/QDict/QemuOpts to add any rule? */
     int (*bdrv_debug_breakpoint)(BlockDriverState *bs, const char *event,
@@ -269,32 +293,86 @@
     /* io queue for linux-aio */
     void (*bdrv_io_plug)(BlockDriverState *bs);
     void (*bdrv_io_unplug)(BlockDriverState *bs);
-    void (*bdrv_flush_io_queue)(BlockDriverState *bs);
+
+    /**
+     * Try to get @bs's logical and physical block size.
+     * On success, store them in @bsz and return zero.
+     * On failure, return negative errno.
+     */
+    int (*bdrv_probe_blocksizes)(BlockDriverState *bs, BlockSizes *bsz);
+    /**
+     * Try to get @bs's geometry (cyls, heads, sectors)
+     * On success, store them in @geo and return 0.
+     * On failure return -errno.
+     * Only drivers that want to override guest geometry implement this
+     * callback; see hd_geometry_guess().
+     */
+    int (*bdrv_probe_geometry)(BlockDriverState *bs, HDGeometry *geo);
+
+    /**
+     * Drain and stop any internal sources of requests in the driver, and
+     * remain so until next I/O callback (e.g. bdrv_co_writev) is called.
+     */
+    void (*bdrv_drain)(BlockDriverState *bs);
+
+    void (*bdrv_add_child)(BlockDriverState *parent, BlockDriverState *child,
+                           Error **errp);
+    void (*bdrv_del_child)(BlockDriverState *parent, BdrvChild *child,
+                           Error **errp);
 
     QLIST_ENTRY(BlockDriver) list;
 };
 
 typedef struct BlockLimits {
-    /* maximum number of sectors that can be discarded at once */
-    int max_discard;
+    /* Alignment requirement, in bytes, for offset/length of I/O
+     * requests. Must be a power of 2 less than INT_MAX; defaults to
+     * 1 for drivers with modern byte interfaces, and to 512
+     * otherwise. */
+    uint32_t request_alignment;
 
-    /* optimal alignment for discard requests in sectors */
-    int64_t discard_alignment;
+    /* Maximum number of bytes that can be discarded at once (since it
+     * is signed, it must be < 2G, if set). Must be multiple of
+     * pdiscard_alignment, but need not be power of 2. May be 0 if no
+     * inherent 32-bit limit */
+    int32_t max_pdiscard;
 
-    /* maximum number of sectors that can zeroized at once */
-    int max_write_zeroes;
+    /* Optimal alignment for discard requests in bytes. A power of 2
+     * is best but not mandatory.  Must be a multiple of
+     * bl.request_alignment, and must be less than max_pdiscard if
+     * that is set. May be 0 if bl.request_alignment is good enough */
+    uint32_t pdiscard_alignment;
 
-    /* optimal alignment for write zeroes requests in sectors */
-    int64_t write_zeroes_alignment;
+    /* Maximum number of bytes that can zeroized at once (since it is
+     * signed, it must be < 2G, if set). Must be multiple of
+     * pwrite_zeroes_alignment. May be 0 if no inherent 32-bit limit */
+    int32_t max_pwrite_zeroes;
 
-    /* optimal transfer length in sectors */
-    int opt_transfer_length;
+    /* Optimal alignment for write zeroes requests in bytes. A power
+     * of 2 is best but not mandatory.  Must be a multiple of
+     * bl.request_alignment, and must be less than max_pwrite_zeroes
+     * if that is set. May be 0 if bl.request_alignment is good
+     * enough */
+    uint32_t pwrite_zeroes_alignment;
 
-    /* maximal transfer length in sectors */
-    int max_transfer_length;
+    /* Optimal transfer length in bytes.  A power of 2 is best but not
+     * mandatory.  Must be a multiple of bl.request_alignment, or 0 if
+     * no preferred size */
+    uint32_t opt_transfer;
 
-    /* memory alignment so that no bounce buffer is needed */
+    /* Maximal transfer length in bytes.  Need not be power of 2, but
+     * must be multiple of opt_transfer and bl.request_alignment, or 0
+     * for no 32-bit limit.  For now, anything larger than INT_MAX is
+     * clamped down. */
+    uint32_t max_transfer;
+
+    /* memory alignment, in bytes so that no bounce buffer is needed */
+    size_t min_mem_alignment;
+
+    /* memory alignment, in bytes, for bounce buffer */
     size_t opt_mem_alignment;
+
+    /* maximum number of iovec elements */
+    int max_iov;
 } BlockLimits;
 
 typedef struct BdrvOpBlocker BdrvOpBlocker;
@@ -304,10 +382,47 @@
     void (*detach_aio_context)(void *opaque);
 
     void *opaque;
+    bool deleted;
 
     QLIST_ENTRY(BdrvAioNotifier) list;
 } BdrvAioNotifier;
 
+struct BdrvChildRole {
+    void (*inherit_options)(int *child_flags, QDict *child_options,
+                            int parent_flags, QDict *parent_options);
+
+    void (*change_media)(BdrvChild *child, bool load);
+    void (*resize)(BdrvChild *child);
+
+    /* Returns a name that is supposedly more useful for human users than the
+     * node name for identifying the node in question (in particular, a BB
+     * name), or NULL if the parent can't provide a better name. */
+    const char* (*get_name)(BdrvChild *child);
+
+    /*
+     * If this pair of functions is implemented, the parent doesn't issue new
+     * requests after returning from .drained_begin() until .drained_end() is
+     * called.
+     *
+     * Note that this can be nested. If drained_begin() was called twice, new
+     * I/O is allowed only after drained_end() was called twice, too.
+     */
+    void (*drained_begin)(BdrvChild *child);
+    void (*drained_end)(BdrvChild *child);
+};
+
+extern const BdrvChildRole child_file;
+extern const BdrvChildRole child_format;
+
+struct BdrvChild {
+    BlockDriverState *bs;
+    char *name;
+    const BdrvChildRole *role;
+    void *opaque;
+    QLIST_ENTRY(BdrvChild) next;
+    QLIST_ENTRY(BdrvChild) next_parent;
+};
+
 /*
  * Note: the function bdrv_append() copies and swaps contents of
  * BlockDriverStates, so if you add new fields to this struct, please
@@ -317,37 +432,41 @@
 struct BlockDriverState {
     int64_t total_sectors; /* if we are reading a disk image, give its
                               size in sectors */
-    int read_only; /* if true, the media is read only */
     int open_flags; /* flags used to open the file, re-used for re-open */
-    int encrypted; /* if true, the media is encrypted */
-    int valid_key; /* if true, a valid encryption key has been set */
-    int sg;        /* if true, the device is a /dev/sg* */
-    int copy_on_read; /* if true, copy read backing sectors into image
+    bool read_only; /* if true, the media is read only */
+    bool encrypted; /* if true, the media is encrypted */
+    bool valid_key; /* if true, a valid encryption key has been set */
+    bool sg;        /* if true, the device is a /dev/sg* */
+    bool probed;    /* if true, format was probed rather than specified */
+
+    int copy_on_read; /* if nonzero, copy read backing sectors into image.
                          note this is a reference count */
 
+    CoQueue flush_queue;            /* Serializing flush queue */
+    BdrvTrackedRequest *active_flush_req; /* Flush request in flight */
+    unsigned int write_gen;         /* Current data generation */
+    unsigned int flushed_gen;       /* Flushed write generation */
+
     BlockDriver *drv; /* NULL means no media */
     void *opaque;
 
-    BlockBackend *blk;          /* owning backend, if any */
-
     AioContext *aio_context; /* event loop used for fd handlers, timers, etc */
     /* long-running tasks intended to always use the same AioContext as this
      * BDS may register themselves in this list to be notified of changes
      * regarding this BDS's context */
     QLIST_HEAD(, BdrvAioNotifier) aio_notifiers;
+    bool walking_aio_notifiers; /* to make removal during iteration safe */
 
-    char filename[1024];
-    char backing_file[1024]; /* if non zero, the image is a diff of
-                                this file image */
+    char filename[PATH_MAX];
+    char backing_file[PATH_MAX]; /* if non zero, the image is a diff of
+                                    this file image */
     char backing_format[16]; /* if non-zero and backing_file exists */
 
     QDict *full_open_options;
-    char exact_filename[1024];
+    char exact_filename[PATH_MAX];
 
-    BlockDriverState *backing_hd;
-    BlockDriverState *file;
-
-    NotifierList close_notifiers;
+    BdrvChild *backing;
+    BdrvChild *file;
 
     /* Callback before write request is processed */
     NotifierWithReturnList before_write_notifiers;
@@ -355,44 +474,26 @@
     /* number of in-flight serialising requests */
     unsigned int serialising_in_flight;
 
-    /* I/O throttling */
-    ThrottleState throttle_state;
-    CoQueue      throttled_reqs[2];
-    bool         io_limits_enabled;
-
-    /* I/O stats (display with "info blockstats"). */
-    BlockAcctStats stats;
+    /* Offset after the highest byte written to */
+    uint64_t wr_highest_offset;
 
     /* I/O Limits */
     BlockLimits bl;
 
-    /* Whether the disk can expand beyond total_sectors */
-    int growable;
-
-    /* Whether produces zeros when read beyond eof */
-    bool zero_beyond_eof;
-
-    /* Alignment requirement for offset/length of I/O requests */
-    unsigned int request_alignment;
-
-    /* the block size for which the guest device expects atomicity */
-    int guest_block_size;
-
-    /* do we need to tell the quest if we have a volatile write cache? */
-    int enable_write_cache;
-
-    /* NOTE: the following infos are only hints for real hardware
-       drivers. They are not used by the block driver */
-    BlockdevOnError on_read_error, on_write_error;
-    bool iostatus_enabled;
-    BlockDeviceIoStatus iostatus;
+    /* Flags honored during pwrite (so far: BDRV_REQ_FUA) */
+    unsigned int supported_write_flags;
+    /* Flags honored during pwrite_zeroes (so far: BDRV_REQ_FUA,
+     * BDRV_REQ_MAY_UNMAP) */
+    unsigned int supported_zero_flags;
 
     /* the following member gives a name to every node on the bs graph. */
     char node_name[32];
     /* element of the list of named nodes building the graph */
     QTAILQ_ENTRY(BlockDriverState) node_list;
-    /* element of the list of "drives" the guest sees */
-    QTAILQ_ENTRY(BlockDriverState) device_list;
+    /* element of the list of all BlockDriverStates (all_bdrv_states) */
+    QTAILQ_ENTRY(BlockDriverState) bs_list;
+    /* element of the list of monitor-owned BDS */
+    QTAILQ_ENTRY(BlockDriverState) monitor_list;
     QLIST_HEAD(, BdrvDirtyBitmap) dirty_bitmaps;
     int refcnt;
 
@@ -404,17 +505,82 @@
     /* long-running background operation */
     BlockJob *job;
 
+    /* The node that this node inherited default options from (and a reopen on
+     * which can affect this node by changing these defaults). This is always a
+     * parent node of this node. */
+    BlockDriverState *inherits_from;
+    QLIST_HEAD(, BdrvChild) children;
+    QLIST_HEAD(, BdrvChild) parents;
+
     QDict *options;
+    QDict *explicit_options;
     BlockdevDetectZeroesOptions detect_zeroes;
 
     /* The error object in use for blocking operations on backing_hd */
     Error *backing_blocker;
+
+    /* threshold limit for writes, in bytes. "High water mark". */
+    uint64_t write_threshold_offset;
+    NotifierWithReturn write_threshold_notifier;
+
+    /* counters for nested bdrv_io_plug and bdrv_io_unplugged_begin */
+    unsigned io_plugged;
+    unsigned io_plug_disabled;
+
+    int quiesce_counter;
 };
 
-int get_tmp_filename(char *filename, int size);
+struct BlockBackendRootState {
+    int open_flags;
+    bool read_only;
+    BlockdevDetectZeroesOptions detect_zeroes;
+};
 
-void bdrv_set_io_limits(BlockDriverState *bs,
-                        ThrottleConfig *cfg);
+typedef enum BlockMirrorBackingMode {
+    /* Reuse the existing backing chain from the source for the target.
+     * - sync=full: Set backing BDS to NULL.
+     * - sync=top:  Use source's backing BDS.
+     * - sync=none: Use source as the backing BDS. */
+    MIRROR_SOURCE_BACKING_CHAIN,
+
+    /* Open the target's backing chain completely anew */
+    MIRROR_OPEN_BACKING_CHAIN,
+
+    /* Do not change the target's backing BDS after job completion */
+    MIRROR_LEAVE_BACKING_CHAIN,
+} BlockMirrorBackingMode;
+
+static inline BlockDriverState *backing_bs(BlockDriverState *bs)
+{
+    return bs->backing ? bs->backing->bs : NULL;
+}
+
+
+/* Essential block drivers which must always be statically linked into qemu, and
+ * which therefore can be accessed without using bdrv_find_format() */
+extern BlockDriver bdrv_file;
+extern BlockDriver bdrv_raw;
+extern BlockDriver bdrv_qcow2;
+
+/**
+ * bdrv_setup_io_funcs:
+ *
+ * Prepare a #BlockDriver for I/O request processing by populating
+ * unimplemented coroutine and AIO interfaces with generic wrapper functions
+ * that fall back to implemented interfaces.
+ */
+void bdrv_setup_io_funcs(BlockDriver *bdrv);
+
+int coroutine_fn bdrv_co_preadv(BdrvChild *child,
+    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
+    int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
+    BdrvRequestFlags flags);
+
+int get_tmp_filename(char *filename, int size);
+BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,
+                            const char *filename);
 
 
 /**
@@ -431,7 +597,7 @@
  *
  * May be called from .bdrv_detach_aio_context() to detach children from the
  * current #AioContext.  This is only needed by block drivers that manage their
- * own children.  Both ->file and ->backing_hd are automatically handled and
+ * own children.  Both ->file and ->backing are automatically handled and
  * block drivers should not call this function on them explicitly.
  */
 void bdrv_detach_aio_context(BlockDriverState *bs);
@@ -441,7 +607,7 @@
  *
  * May be called from .bdrv_attach_aio_context() to attach children to the new
  * #AioContext.  This is only needed by block drivers that manage their own
- * children.  Both ->file and ->backing_hd are automatically handled and block
+ * children.  Both ->file and ->backing are automatically handled and block
  * drivers should not call this function on them explicitly.
  */
 void bdrv_attach_aio_context(BlockDriverState *bs,
@@ -481,11 +647,13 @@
 
 /**
  * stream_start:
+ * @job_id: The id of the newly-created job, or %NULL to use the
+ * device name of @bs.
  * @bs: Block device to operate on.
  * @base: Block device that will become the new base, or %NULL to
  * flatten the whole backing file chain onto @bs.
- * @base_id: The file name that will be written to @bs as the new
- * backing file if the job completes.  Ignored if @base is %NULL.
+ * @backing_file_str: The file name that will be written to @bs as the
+ * the new backing file if the job completes. Ignored if @base is %NULL.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
  * @on_error: The action to take upon error.
  * @cb: Completion function for the job.
@@ -496,15 +664,18 @@
  * in @bs, but allocated in any image between @base and @bs (both
  * exclusive) will be written to @bs.  At the end of a successful
  * streaming job, the backing file of @bs will be changed to
- * @base_id in the written image and to @base in the live BlockDriverState.
+ * @backing_file_str in the written image and to @base in the live
+ * BlockDriverState.
  */
-void stream_start(BlockDriverState *bs, BlockDriverState *base,
-                  const char *base_id, int64_t speed, BlockdevOnError on_error,
-                  BlockCompletionFunc *cb,
-                  void *opaque, Error **errp);
+void stream_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *base, const char *backing_file_str,
+                  int64_t speed, BlockdevOnError on_error,
+                  BlockCompletionFunc *cb, void *opaque, Error **errp);
 
 /**
  * commit_start:
+ * @job_id: The id of the newly-created job, or %NULL to use the
+ * device name of @bs.
  * @bs: Active block device.
  * @top: Top block device to be committed.
  * @base: Block device that will be written into, and become the new top.
@@ -516,12 +687,14 @@
  * @errp: Error object.
  *
  */
-void commit_start(BlockDriverState *bs, BlockDriverState *base,
-                 BlockDriverState *top, int64_t speed,
-                 BlockdevOnError on_error, BlockCompletionFunc *cb,
-                 void *opaque, const char *backing_file_str, Error **errp);
+void commit_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *base, BlockDriverState *top, int64_t speed,
+                  BlockdevOnError on_error, BlockCompletionFunc *cb,
+                  void *opaque, const char *backing_file_str, Error **errp);
 /**
  * commit_active_start:
+ * @job_id: The id of the newly-created job, or %NULL to use the
+ * device name of @bs.
  * @bs: Active block device to be committed.
  * @base: Block device that will be written into, and become the new top.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
@@ -531,13 +704,15 @@
  * @errp: Error object.
  *
  */
-void commit_active_start(BlockDriverState *bs, BlockDriverState *base,
-                         int64_t speed,
+void commit_active_start(const char *job_id, BlockDriverState *bs,
+                         BlockDriverState *base, int64_t speed,
                          BlockdevOnError on_error,
                          BlockCompletionFunc *cb,
                          void *opaque, Error **errp);
 /*
  * mirror_start:
+ * @job_id: The id of the newly-created job, or %NULL to use the
+ * device name of @bs.
  * @bs: Block device to operate on.
  * @target: Block device to write to.
  * @replaces: Block graph node name to replace once the mirror is done. Can
@@ -546,8 +721,10 @@
  * @granularity: The chosen granularity for the dirty bitmap.
  * @buf_size: The amount of data that can be in flight at one time.
  * @mode: Whether to collapse all images in the chain to the target.
+ * @backing_mode: How to establish the target's backing chain after completion.
  * @on_source_error: The action to take upon error reading from the source.
  * @on_target_error: The action to take upon error writing to the target.
+ * @unmap: Whether to unmap target where source sectors only contain zeroes.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
@@ -557,40 +734,64 @@
  * manually completed.  At the end of a successful mirroring job,
  * @bs will be switched to read from @target.
  */
-void mirror_start(BlockDriverState *bs, BlockDriverState *target,
-                  const char *replaces,
-                  int64_t speed, int64_t granularity, int64_t buf_size,
-                  MirrorSyncMode mode, BlockdevOnError on_source_error,
+void mirror_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *target, const char *replaces,
+                  int64_t speed, uint32_t granularity, int64_t buf_size,
+                  MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
+                  BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
+                  bool unmap,
                   BlockCompletionFunc *cb,
                   void *opaque, Error **errp);
 
 /*
  * backup_start:
+ * @job_id: The id of the newly-created job, or %NULL to use the
+ * device name of @bs.
  * @bs: Block device to operate on.
  * @target: Block device to write to.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
  * @sync_mode: What parts of the disk image should be copied to the destination.
+ * @sync_bitmap: The dirty bitmap if sync_mode is MIRROR_SYNC_MODE_INCREMENTAL.
  * @on_source_error: The action to take upon error reading from the source.
  * @on_target_error: The action to take upon error writing to the target.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
+ * @txn: Transaction that this job is part of (may be NULL).
  *
  * Start a backup operation on @bs.  Clusters in @bs are written to @target
  * until the job is cancelled or manually completed.
  */
-void backup_start(BlockDriverState *bs, BlockDriverState *target,
-                  int64_t speed, MirrorSyncMode sync_mode,
+void backup_start(const char *job_id, BlockDriverState *bs,
+                  BlockDriverState *target, int64_t speed,
+                  MirrorSyncMode sync_mode, BdrvDirtyBitmap *sync_bitmap,
                   BlockdevOnError on_source_error,
                   BlockdevOnError on_target_error,
                   BlockCompletionFunc *cb, void *opaque,
-                  Error **errp);
+                  BlockJobTxn *txn, Error **errp);
 
+void hmp_drive_add_node(Monitor *mon, const char *optstr);
+
+BdrvChild *bdrv_root_attach_child(BlockDriverState *child_bs,
+                                  const char *child_name,
+                                  const BdrvChildRole *child_role,
+                                  void *opaque);
+void bdrv_root_unref_child(BdrvChild *child);
+
+const char *bdrv_get_parent_name(const BlockDriverState *bs);
 void blk_dev_change_media_cb(BlockBackend *blk, bool load);
 bool blk_dev_has_removable_media(BlockBackend *blk);
+bool blk_dev_has_tray(BlockBackend *blk);
 void blk_dev_eject_request(BlockBackend *blk, bool force);
 bool blk_dev_is_tray_open(BlockBackend *blk);
 bool blk_dev_is_medium_locked(BlockBackend *blk);
-void blk_dev_resize_cb(BlockBackend *blk);
+
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int64_t nr_sect);
+bool bdrv_requests_pending(BlockDriverState *bs);
+
+void bdrv_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap **out);
+void bdrv_undo_clear_dirty_bitmap(BdrvDirtyBitmap *bitmap, HBitmap *in);
+
+void blockdev_close_all_bdrv_states(void);
 
 #endif /* BLOCK_INT_H */
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index b6d4ebb..4ddb4ae 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -22,8 +22,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef BLOCKJOB_H
-#define BLOCKJOB_H 1
+#define BLOCKJOB_H
 
 #include "block/block.h"
 
@@ -50,6 +51,47 @@
      * manually.
      */
     void (*complete)(BlockJob *job, Error **errp);
+
+    /**
+     * If the callback is not NULL, it will be invoked when all the jobs
+     * belonging to the same transaction complete; or upon this job's
+     * completion if it is not in a transaction. Skipped if NULL.
+     *
+     * All jobs will complete with a call to either .commit() or .abort() but
+     * never both.
+     */
+    void (*commit)(BlockJob *job);
+
+    /**
+     * If the callback is not NULL, it will be invoked when any job in the
+     * same transaction fails; or upon this job's failure (due to error or
+     * cancellation) if it is not in a transaction. Skipped if NULL.
+     *
+     * All jobs will complete with a call to either .commit() or .abort() but
+     * never both.
+     */
+    void (*abort)(BlockJob *job);
+
+    /**
+     * If the callback is not NULL, it will be invoked when the job transitions
+     * into the paused state.  Paused jobs must not perform any asynchronous
+     * I/O or event loop activity.  This callback is used to quiesce jobs.
+     */
+    void coroutine_fn (*pause)(BlockJob *job);
+
+    /**
+     * If the callback is not NULL, it will be invoked when the job transitions
+     * out of the paused state.  Any asynchronous I/O or event loop activity
+     * should be restarted from this callback.
+     */
+    void coroutine_fn (*resume)(BlockJob *job);
+
+    /*
+     * If the callback is not NULL, it will be invoked before the job is
+     * resumed in a new AioContext.  This is the place to move any resources
+     * besides job->blk to the new AioContext.
+     */
+    void (*attached_aio_context)(BlockJob *job, AioContext *new_context);
 } BlockJobDriver;
 
 /**
@@ -62,7 +104,12 @@
     const BlockJobDriver *driver;
 
     /** The block device on which the job is operating.  */
-    BlockDriverState *bs;
+    BlockBackend *blk;
+
+    /**
+     * The ID of the block job.
+     */
+    char *id;
 
     /**
      * The coroutine that executes the job.  If not NULL, it is
@@ -79,23 +126,43 @@
     bool cancelled;
 
     /**
-     * Set to true if the job is either paused, or will pause itself
-     * as soon as possible (if busy == true).
+     * Counter for pause request. If non-zero, the block job is either paused,
+     * or if busy == true will pause itself as soon as possible.
      */
-    bool paused;
+    int pause_count;
 
     /**
-     * Set to false by the job while it is in a quiescent state, where
-     * no I/O is pending and the job has yielded on any condition
-     * that is not detected by #aio_poll, such as a timer.
+     * Set to true if the job is paused by user.  Can be unpaused with the
+     * block-job-resume QMP command.
+     */
+    bool user_paused;
+
+    /**
+     * Set to false by the job while the coroutine has yielded and may be
+     * re-entered by block_job_enter().  There may still be I/O or event loop
+     * activity pending.
      */
     bool busy;
 
     /**
+     * Set to true by the job while it is in a quiescent state, where
+     * no I/O or event loop activity is pending.
+     */
+    bool paused;
+
+    /**
      * Set to true when the job is ready to be completed.
      */
     bool ready;
 
+    /**
+     * Set to true when the job has deferred work to the main loop.
+     */
+    bool deferred_to_main_loop;
+
+    /** Element of the list of block jobs */
+    QLIST_ENTRY(BlockJob) job_list;
+
     /** Status that is published by the query-block-jobs QMP API */
     BlockDeviceIoStatus iostatus;
 
@@ -116,10 +183,48 @@
 
     /** The opaque value that is passed to the completion function.  */
     void *opaque;
+
+    /** Reference count of the block job */
+    int refcnt;
+
+    /* True if this job has reported completion by calling block_job_completed.
+     */
+    bool completed;
+
+    /* ret code passed to block_job_completed.
+     */
+    int ret;
+
+    /** Non-NULL if this job is part of a transaction */
+    BlockJobTxn *txn;
+    QLIST_ENTRY(BlockJob) txn_list;
 };
 
 /**
+ * block_job_next:
+ * @job: A block job, or %NULL.
+ *
+ * Get the next element from the list of block jobs after @job, or the
+ * first one if @job is %NULL.
+ *
+ * Returns the requested job, or %NULL if there are no more jobs left.
+ */
+BlockJob *block_job_next(BlockJob *job);
+
+/**
+ * block_job_get:
+ * @id: The id of the block job.
+ *
+ * Get the block job identified by @id (which must not be %NULL).
+ *
+ * Returns the requested job, or %NULL if it doesn't exist.
+ */
+BlockJob *block_job_get(const char *id);
+
+/**
  * block_job_create:
+ * @job_id: The id of the newly-created job, or %NULL to have one
+ * generated automatically.
  * @job_type: The class object for the newly-created job.
  * @bs: The block
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
@@ -136,9 +241,9 @@
  * This function is not part of the public job interface; it should be
  * called from a wrapper that is specific to the job type.
  */
-void *block_job_create(const BlockJobDriver *driver, BlockDriverState *bs,
-                       int64_t speed, BlockCompletionFunc *cb,
-                       void *opaque, Error **errp);
+void *block_job_create(const char *job_id, const BlockJobDriver *driver,
+                       BlockDriverState *bs, int64_t speed,
+                       BlockCompletionFunc *cb, void *opaque, Error **errp);
 
 /**
  * block_job_sleep_ns:
@@ -160,6 +265,23 @@
 void block_job_yield(BlockJob *job);
 
 /**
+ * block_job_ref:
+ * @bs: The block device.
+ *
+ * Grab a reference to the block job. Should be paired with block_job_unref.
+ */
+void block_job_ref(BlockJob *job);
+
+/**
+ * block_job_unref:
+ * @bs: The block device.
+ *
+ * Release reference to the block job and release resources if it is the last
+ * reference.
+ */
+void block_job_unref(BlockJob *job);
+
+/**
  * block_job_completed:
  * @job: The job being completed.
  * @ret: The status code.
@@ -214,6 +336,15 @@
 BlockJobInfo *block_job_query(BlockJob *job);
 
 /**
+ * block_job_pause_point:
+ * @job: The job that is ready to pause.
+ *
+ * Pause now if block_job_pause() has been called.  Block jobs that perform
+ * lots of I/O must call this between requests so that the job can be paused.
+ */
+void coroutine_fn block_job_pause_point(BlockJob *job);
+
+/**
  * block_job_pause:
  * @job: The job to be paused.
  *
@@ -225,11 +356,19 @@
  * block_job_resume:
  * @job: The job to be resumed.
  *
- * Resume the specified job.
+ * Resume the specified job.  Must be paired with a preceding block_job_pause.
  */
 void block_job_resume(BlockJob *job);
 
 /**
+ * block_job_enter:
+ * @job: The job to enter.
+ *
+ * Continue the specified job by entering the coroutine.
+ */
+void block_job_enter(BlockJob *job);
+
+/**
  * block_job_event_cancelled:
  * @job: The job whose information is requested.
  *
@@ -255,15 +394,6 @@
 void block_job_event_ready(BlockJob *job);
 
 /**
- * block_job_is_paused:
- * @job: The job being queried.
- *
- * Returns whether the job is currently paused, or will pause
- * as soon as it reaches a sleeping point.
- */
-bool block_job_is_paused(BlockJob *job);
-
-/**
  * block_job_cancel_sync:
  * @job: The job to be canceled.
  *
@@ -278,6 +408,13 @@
 int block_job_cancel_sync(BlockJob *job);
 
 /**
+ * block_job_cancel_sync_all:
+ *
+ * Synchronously cancels all jobs using block_job_cancel_sync().
+ */
+void block_job_cancel_sync_all(void);
+
+/**
  * block_job_complete_sync:
  * @job: The job to be completed.
  * @errp: Error object which may be set by block_job_complete(); this is not
@@ -297,14 +434,13 @@
  * @job: The job whose I/O status should be reset.
  *
  * Reset I/O status on @job and on BlockDriverState objects it uses,
- * other than job->bs.
+ * other than job->blk.
  */
 void block_job_iostatus_reset(BlockJob *job);
 
 /**
  * block_job_error_action:
  * @job: The job to signal an error for.
- * @bs: The block device on which to set an I/O error.
  * @on_err: The error action setting.
  * @is_read: Whether the operation was a read.
  * @error: The error that was reported.
@@ -312,8 +448,7 @@
  * Report an I/O error for a block job and possibly stop the VM.  Return the
  * action that was selected based on @on_err and @error.
  */
-BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
-                                        BlockdevOnError on_err,
+BlockErrorAction block_job_error_action(BlockJob *job, BlockdevOnError on_err,
                                         int is_read, int error);
 
 typedef void BlockJobDeferToMainLoopFn(BlockJob *job, void *opaque);
@@ -334,4 +469,39 @@
                                   BlockJobDeferToMainLoopFn *fn,
                                   void *opaque);
 
+/**
+ * block_job_txn_new:
+ *
+ * Allocate and return a new block job transaction.  Jobs can be added to the
+ * transaction using block_job_txn_add_job().
+ *
+ * The transaction is automatically freed when the last job completes or is
+ * cancelled.
+ *
+ * All jobs in the transaction either complete successfully or fail/cancel as a
+ * group.  Jobs wait for each other before completing.  Cancelling one job
+ * cancels all jobs in the transaction.
+ */
+BlockJobTxn *block_job_txn_new(void);
+
+/**
+ * block_job_txn_unref:
+ *
+ * Release a reference that was previously acquired with block_job_txn_add_job
+ * or block_job_txn_new. If it's the last reference to the object, it will be
+ * freed.
+ */
+void block_job_txn_unref(BlockJobTxn *txn);
+
+/**
+ * block_job_txn_add_job:
+ * @txn: The transaction (may be NULL)
+ * @job: Job to add to the transaction
+ *
+ * Add @job to the transaction.  The @job must not already be in a transaction.
+ * The caller must call either block_job_txn_unref() or block_job_completed()
+ * to release the reference that is automatically grabbed here.
+ */
+void block_job_txn_add_job(BlockJobTxn *txn, BlockJob *job);
+
 #endif
diff --git a/include/block/coroutine.h b/include/block/coroutine.h
deleted file mode 100644
index 793df0e..0000000
--- a/include/block/coroutine.h
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * QEMU coroutine implementation
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
- *  Kevin Wolf         <kwolf@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef QEMU_COROUTINE_H
-#define QEMU_COROUTINE_H
-
-#include <stdbool.h>
-#include "qemu/typedefs.h"
-#include "qemu/queue.h"
-#include "qemu/timer.h"
-
-/**
- * Coroutines are a mechanism for stack switching and can be used for
- * cooperative userspace threading.  These functions provide a simple but
- * useful flavor of coroutines that is suitable for writing sequential code,
- * rather than callbacks, for operations that need to give up control while
- * waiting for events to complete.
- *
- * These functions are re-entrant and may be used outside the global mutex.
- */
-
-/**
- * Mark a function that executes in coroutine context
- *
- * Functions that execute in coroutine context cannot be called directly from
- * normal functions.  In the future it would be nice to enable compiler or
- * static checker support for catching such errors.  This annotation might make
- * it possible and in the meantime it serves as documentation.
- *
- * For example:
- *
- *   static void coroutine_fn foo(void) {
- *       ....
- *   }
- */
-#define coroutine_fn
-
-typedef struct Coroutine Coroutine;
-
-/**
- * Coroutine entry point
- *
- * When the coroutine is entered for the first time, opaque is passed in as an
- * argument.
- *
- * When this function returns, the coroutine is destroyed automatically and
- * execution continues in the caller who last entered the coroutine.
- */
-typedef void coroutine_fn CoroutineEntry(void *opaque);
-
-/**
- * Create a new coroutine
- *
- * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
- */
-Coroutine *qemu_coroutine_create(CoroutineEntry *entry);
-
-/**
- * Transfer control to a coroutine
- *
- * The opaque argument is passed as the argument to the entry point when
- * entering the coroutine for the first time.  It is subsequently ignored.
- */
-void qemu_coroutine_enter(Coroutine *coroutine, void *opaque);
-
-/**
- * Transfer control back to a coroutine's caller
- *
- * This function does not return until the coroutine is re-entered using
- * qemu_coroutine_enter().
- */
-void coroutine_fn qemu_coroutine_yield(void);
-
-/**
- * Get the currently executing coroutine
- */
-Coroutine *coroutine_fn qemu_coroutine_self(void);
-
-/**
- * Return whether or not currently inside a coroutine
- *
- * This can be used to write functions that work both when in coroutine context
- * and when not in coroutine context.  Note that such functions cannot use the
- * coroutine_fn annotation since they work outside coroutine context.
- */
-bool qemu_in_coroutine(void);
-
-
-
-/**
- * CoQueues are a mechanism to queue coroutines in order to continue executing
- * them later. They provide the fundamental primitives on which coroutine locks
- * are built.
- */
-typedef struct CoQueue {
-    QTAILQ_HEAD(, Coroutine) entries;
-} CoQueue;
-
-/**
- * Initialise a CoQueue. This must be called before any other operation is used
- * on the CoQueue.
- */
-void qemu_co_queue_init(CoQueue *queue);
-
-/**
- * Adds the current coroutine to the CoQueue and transfers control to the
- * caller of the coroutine.
- */
-void coroutine_fn qemu_co_queue_wait(CoQueue *queue);
-
-/**
- * Restarts the next coroutine in the CoQueue and removes it from the queue.
- *
- * Returns true if a coroutine was restarted, false if the queue is empty.
- */
-bool coroutine_fn qemu_co_queue_next(CoQueue *queue);
-
-/**
- * Restarts all coroutines in the CoQueue and leaves the queue empty.
- */
-void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue);
-
-/**
- * Enter the next coroutine in the queue
- */
-bool qemu_co_enter_next(CoQueue *queue);
-
-/**
- * Checks if the CoQueue is empty.
- */
-bool qemu_co_queue_empty(CoQueue *queue);
-
-
-/**
- * Provides a mutex that can be used to synchronise coroutines
- */
-typedef struct CoMutex {
-    bool locked;
-    CoQueue queue;
-} CoMutex;
-
-/**
- * Initialises a CoMutex. This must be called before any other operation is used
- * on the CoMutex.
- */
-void qemu_co_mutex_init(CoMutex *mutex);
-
-/**
- * Locks the mutex. If the lock cannot be taken immediately, control is
- * transferred to the caller of the current coroutine.
- */
-void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex);
-
-/**
- * Unlocks the mutex and schedules the next coroutine that was waiting for this
- * lock to be run.
- */
-void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex);
-
-typedef struct CoRwlock {
-    bool writer;
-    int reader;
-    CoQueue queue;
-} CoRwlock;
-
-/**
- * Initialises a CoRwlock. This must be called before any other operation
- * is used on the CoRwlock
- */
-void qemu_co_rwlock_init(CoRwlock *lock);
-
-/**
- * Read locks the CoRwlock. If the lock cannot be taken immediately because
- * of a parallel writer, control is transferred to the caller of the current
- * coroutine.
- */
-void qemu_co_rwlock_rdlock(CoRwlock *lock);
-
-/**
- * Write Locks the mutex. If the lock cannot be taken immediately because
- * of a parallel reader, control is transferred to the caller of the current
- * coroutine.
- */
-void qemu_co_rwlock_wrlock(CoRwlock *lock);
-
-/**
- * Unlocks the read/write lock and schedules the next coroutine that was
- * waiting for this lock to be run.
- */
-void qemu_co_rwlock_unlock(CoRwlock *lock);
-
-/**
- * Yield the coroutine for a given duration
- *
- * Behaves similarly to co_sleep_ns(), but the sleeping coroutine will be
- * resumed when using aio_poll().
- */
-void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
-                                  int64_t ns);
-
-/**
- * Yield until a file descriptor becomes readable
- *
- * Note that this function clobbers the handlers for the file descriptor.
- */
-void coroutine_fn yield_until_fd_readable(int fd);
-
-/**
- * Add or subtract from the coroutine pool size
- *
- * The coroutine implementation keeps a pool of coroutines to be reused by
- * qemu_coroutine_create().  This makes coroutine creation cheap.  Heavy
- * coroutine users should call this to reserve pool space.  Call it again with
- * a negative number to release pool space.
- */
-void qemu_coroutine_adjust_pool_size(int n);
-
-#endif /* QEMU_COROUTINE_H */
diff --git a/include/block/coroutine_int.h b/include/block/coroutine_int.h
deleted file mode 100644
index f133d65..0000000
--- a/include/block/coroutine_int.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Coroutine internals
- *
- * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#ifndef QEMU_COROUTINE_INT_H
-#define QEMU_COROUTINE_INT_H
-
-#include "qemu/queue.h"
-#include "block/coroutine.h"
-
-typedef enum {
-    COROUTINE_YIELD = 1,
-    COROUTINE_TERMINATE = 2,
-} CoroutineAction;
-
-struct Coroutine {
-    CoroutineEntry *entry;
-    void *entry_arg;
-    Coroutine *caller;
-    QSLIST_ENTRY(Coroutine) pool_next;
-
-    /* Coroutines that should be woken up when we yield or terminate */
-    QTAILQ_HEAD(, Coroutine) co_queue_wakeup;
-    QTAILQ_ENTRY(Coroutine) co_queue_next;
-};
-
-Coroutine *qemu_coroutine_new(void);
-void qemu_coroutine_delete(Coroutine *co);
-CoroutineAction qemu_coroutine_switch(Coroutine *from, Coroutine *to,
-                                      CoroutineAction action);
-void coroutine_fn qemu_co_queue_run_restart(Coroutine *co);
-
-#endif
diff --git a/include/block/dirty-bitmap.h b/include/block/dirty-bitmap.h
new file mode 100644
index 0000000..ee3388f
--- /dev/null
+++ b/include/block/dirty-bitmap.h
@@ -0,0 +1,44 @@
+#ifndef BLOCK_DIRTY_BITMAP_H
+#define BLOCK_DIRTY_BITMAP_H
+
+#include "qemu-common.h"
+#include "qemu/hbitmap.h"
+
+BdrvDirtyBitmap *bdrv_create_dirty_bitmap(BlockDriverState *bs,
+                                          uint32_t granularity,
+                                          const char *name,
+                                          Error **errp);
+int bdrv_dirty_bitmap_create_successor(BlockDriverState *bs,
+                                       BdrvDirtyBitmap *bitmap,
+                                       Error **errp);
+BdrvDirtyBitmap *bdrv_dirty_bitmap_abdicate(BlockDriverState *bs,
+                                            BdrvDirtyBitmap *bitmap,
+                                            Error **errp);
+BdrvDirtyBitmap *bdrv_reclaim_dirty_bitmap(BlockDriverState *bs,
+                                           BdrvDirtyBitmap *bitmap,
+                                           Error **errp);
+BdrvDirtyBitmap *bdrv_find_dirty_bitmap(BlockDriverState *bs,
+                                        const char *name);
+void bdrv_dirty_bitmap_make_anon(BdrvDirtyBitmap *bitmap);
+void bdrv_release_dirty_bitmap(BlockDriverState *bs, BdrvDirtyBitmap *bitmap);
+void bdrv_release_named_dirty_bitmaps(BlockDriverState *bs);
+void bdrv_disable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
+void bdrv_enable_dirty_bitmap(BdrvDirtyBitmap *bitmap);
+BlockDirtyInfoList *bdrv_query_dirty_bitmaps(BlockDriverState *bs);
+uint32_t bdrv_get_default_bitmap_granularity(BlockDriverState *bs);
+uint32_t bdrv_dirty_bitmap_granularity(BdrvDirtyBitmap *bitmap);
+bool bdrv_dirty_bitmap_enabled(BdrvDirtyBitmap *bitmap);
+bool bdrv_dirty_bitmap_frozen(BdrvDirtyBitmap *bitmap);
+DirtyBitmapStatus bdrv_dirty_bitmap_status(BdrvDirtyBitmap *bitmap);
+int bdrv_get_dirty(BlockDriverState *bs, BdrvDirtyBitmap *bitmap,
+                   int64_t sector);
+void bdrv_set_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                           int64_t cur_sector, int64_t nr_sectors);
+void bdrv_reset_dirty_bitmap(BdrvDirtyBitmap *bitmap,
+                             int64_t cur_sector, int64_t nr_sectors);
+void bdrv_dirty_iter_init(BdrvDirtyBitmap *bitmap, struct HBitmapIter *hbi);
+void bdrv_set_dirty_iter(struct HBitmapIter *hbi, int64_t offset);
+int64_t bdrv_get_dirty_count(BdrvDirtyBitmap *bitmap);
+void bdrv_dirty_bitmap_truncate(BlockDriverState *bs);
+
+#endif
diff --git a/include/block/nbd.h b/include/block/nbd.h
index 9e835d2..1897557 100644
--- a/include/block/nbd.h
+++ b/include/block/nbd.h
@@ -19,24 +19,26 @@
 #ifndef NBD_H
 #define NBD_H
 
-#include <sys/types.h>
 
 #include "qemu-common.h"
 #include "qemu/option.h"
+#include "io/channel-socket.h"
+#include "crypto/tlscreds.h"
 
+/* Note: these are _NOT_ the same as the network representation of an NBD
+ * request and reply!
+ */
 struct nbd_request {
-    uint32_t magic;
-    uint32_t type;
     uint64_t handle;
     uint64_t from;
     uint32_t len;
-} QEMU_PACKED;
+    uint32_t type;
+};
 
 struct nbd_reply {
-    uint32_t magic;
-    uint32_t error;
     uint64_t handle;
-} QEMU_PACKED;
+    uint32_t error;
+};
 
 #define NBD_FLAG_HAS_FLAGS      (1 << 0)        /* Flags are there */
 #define NBD_FLAG_READ_ONLY      (1 << 1)        /* Device is read-only */
@@ -54,8 +56,11 @@
 /* Reply types. */
 #define NBD_REP_ACK             (1)             /* Data sending finished. */
 #define NBD_REP_SERVER          (2)             /* Export description. */
-#define NBD_REP_ERR_UNSUP       ((1 << 31) | 1) /* Unknown option. */
-#define NBD_REP_ERR_INVALID     ((1 << 31) | 3) /* Invalid length. */
+#define NBD_REP_ERR_UNSUP       ((UINT32_C(1) << 31) | 1) /* Unknown option. */
+#define NBD_REP_ERR_POLICY      ((UINT32_C(1) << 31) | 2) /* Server denied */
+#define NBD_REP_ERR_INVALID     ((UINT32_C(1) << 31) | 3) /* Invalid length. */
+#define NBD_REP_ERR_TLS_REQD    ((UINT32_C(1) << 31) | 5) /* TLS required */
+
 
 #define NBD_CMD_MASK_COMMAND	0x0000ffff
 #define NBD_CMD_FLAG_FUA	(1 << 16)
@@ -73,34 +78,49 @@
 /* Maximum size of a single READ/WRITE data buffer */
 #define NBD_MAX_BUFFER_SIZE (32 * 1024 * 1024)
 
-ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read);
-int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
-                          off_t *size, size_t *blocksize);
-int nbd_init(int fd, int csock, uint32_t flags, off_t size, size_t blocksize);
-ssize_t nbd_send_request(int csock, struct nbd_request *request);
-ssize_t nbd_receive_reply(int csock, struct nbd_reply *reply);
+/* Maximum size of an export name. The NBD spec requires 256 and
+ * suggests that servers support up to 4096, but we stick to only the
+ * required size so that we can stack-allocate the names, and because
+ * going larger would require an audit of more code to make sure we
+ * aren't overflowing some other buffer. */
+#define NBD_MAX_NAME_SIZE 256
+
+ssize_t nbd_wr_syncv(QIOChannel *ioc,
+                     struct iovec *iov,
+                     size_t niov,
+                     size_t length,
+                     bool do_read);
+int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint16_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
+                          off_t *size, Error **errp);
+int nbd_init(int fd, QIOChannelSocket *sioc, uint16_t flags, off_t size);
+ssize_t nbd_send_request(QIOChannel *ioc, struct nbd_request *request);
+ssize_t nbd_receive_reply(QIOChannel *ioc, struct nbd_reply *reply);
 int nbd_client(int fd);
 int nbd_disconnect(int fd);
 
 typedef struct NBDExport NBDExport;
 typedef struct NBDClient NBDClient;
 
-NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
-                          off_t size, uint32_t nbdflags,
-                          void (*close)(NBDExport *));
+NBDExport *nbd_export_new(BlockBackend *blk, off_t dev_offset, off_t size,
+                          uint16_t nbdflags, void (*close)(NBDExport *),
+                          Error **errp);
 void nbd_export_close(NBDExport *exp);
 void nbd_export_get(NBDExport *exp);
 void nbd_export_put(NBDExport *exp);
 
-BlockDriverState *nbd_export_get_blockdev(NBDExport *exp);
+BlockBackend *nbd_export_get_blockdev(NBDExport *exp);
 
 NBDExport *nbd_export_find(const char *name);
 void nbd_export_set_name(NBDExport *exp, const char *name);
 void nbd_export_close_all(void);
 
-NBDClient *nbd_client_new(NBDExport *exp, int csock,
-                          void (*close)(NBDClient *));
-void nbd_client_close(NBDClient *client);
+void nbd_client_new(NBDExport *exp,
+                    QIOChannelSocket *sioc,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *tlsaclname,
+                    void (*close)(NBDClient *));
 void nbd_client_get(NBDClient *client);
 void nbd_client_put(NBDClient *client);
 
diff --git a/include/block/qapi.h b/include/block/qapi.h
index 168d788..82ba4b6 100644
--- a/include/block/qapi.h
+++ b/include/block/qapi.h
@@ -29,7 +29,8 @@
 #include "block/block.h"
 #include "block/snapshot.h"
 
-BlockDeviceInfo *bdrv_block_device_info(BlockDriverState *bs);
+BlockDeviceInfo *bdrv_block_device_info(BlockBackend *blk,
+                                        BlockDriverState *bs, Error **errp);
 int bdrv_query_snapshot_info_list(BlockDriverState *bs,
                                   SnapshotInfoList **p_list,
                                   Error **errp);
diff --git a/include/block/raw-aio.h b/include/block/raw-aio.h
new file mode 100644
index 0000000..a4cdbbf
--- /dev/null
+++ b/include/block/raw-aio.h
@@ -0,0 +1,68 @@
+/*
+ * Declarations for AIO in the raw protocol
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#ifndef QEMU_RAW_AIO_H
+#define QEMU_RAW_AIO_H
+
+#include "qemu/coroutine.h"
+#include "qemu/iov.h"
+
+/* AIO request types */
+#define QEMU_AIO_READ         0x0001
+#define QEMU_AIO_WRITE        0x0002
+#define QEMU_AIO_IOCTL        0x0004
+#define QEMU_AIO_FLUSH        0x0008
+#define QEMU_AIO_DISCARD      0x0010
+#define QEMU_AIO_WRITE_ZEROES 0x0020
+#define QEMU_AIO_TYPE_MASK \
+        (QEMU_AIO_READ|QEMU_AIO_WRITE|QEMU_AIO_IOCTL|QEMU_AIO_FLUSH| \
+         QEMU_AIO_DISCARD|QEMU_AIO_WRITE_ZEROES)
+
+/* AIO flags */
+#define QEMU_AIO_MISALIGNED   0x1000
+#define QEMU_AIO_BLKDEV       0x2000
+
+
+/* linux-aio.c - Linux native implementation */
+#ifdef CONFIG_LINUX_AIO
+typedef struct LinuxAioState LinuxAioState;
+LinuxAioState *laio_init(void);
+void laio_cleanup(LinuxAioState *s);
+int coroutine_fn laio_co_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
+                                uint64_t offset, QEMUIOVector *qiov, int type);
+BlockAIOCB *laio_submit(BlockDriverState *bs, LinuxAioState *s, int fd,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockCompletionFunc *cb, void *opaque, int type);
+void laio_detach_aio_context(LinuxAioState *s, AioContext *old_context);
+void laio_attach_aio_context(LinuxAioState *s, AioContext *new_context);
+void laio_io_plug(BlockDriverState *bs, LinuxAioState *s);
+void laio_io_unplug(BlockDriverState *bs, LinuxAioState *s);
+#endif
+
+#ifdef _WIN32
+typedef struct QEMUWin32AIOState QEMUWin32AIOState;
+QEMUWin32AIOState *win32_aio_init(void);
+void win32_aio_cleanup(QEMUWin32AIOState *aio);
+int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile);
+BlockAIOCB *win32_aio_submit(BlockDriverState *bs,
+        QEMUWin32AIOState *aio, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockCompletionFunc *cb, void *opaque, int type);
+void win32_aio_detach_aio_context(QEMUWin32AIOState *aio,
+                                  AioContext *old_context);
+void win32_aio_attach_aio_context(QEMUWin32AIOState *aio,
+                                  AioContext *new_context);
+#endif
+
+#endif /* QEMU_RAW_AIO_H */
diff --git a/include/block/scsi.h b/include/block/scsi.h
index edde960..cdf0a58 100644
--- a/include/block/scsi.h
+++ b/include/block/scsi.h
@@ -19,8 +19,9 @@
  * This header file contains public constants and structures used by
  * the scsi code for linux.
  */
-#ifndef HW_SCSI_DEFS_H
-#define HW_SCSI_DEFS_H 1
+
+#ifndef BLOCK_SCSI_H
+#define BLOCK_SCSI_H
 
 /*
  *      SCSI opcodes
@@ -48,13 +49,17 @@
 #define ERASE                 0x19
 #define MODE_SENSE            0x1a
 #define LOAD_UNLOAD           0x1b
+#define SCAN                  0x1b
 #define START_STOP            0x1b
 #define RECEIVE_DIAGNOSTIC    0x1c
 #define SEND_DIAGNOSTIC       0x1d
 #define ALLOW_MEDIUM_REMOVAL  0x1e
+#define SET_WINDOW            0x24
 #define READ_CAPACITY_10      0x25
+#define GET_WINDOW            0x25
 #define READ_10               0x28
 #define WRITE_10              0x2a
+#define SEND                  0x2a
 #define SEEK_10               0x2b
 #define LOCATE_10             0x2b
 #define POSITION_TO_ELEMENT   0x2b
@@ -62,10 +67,12 @@
 #define VERIFY_10             0x2f
 #define SEARCH_HIGH           0x30
 #define SEARCH_EQUAL          0x31
+#define OBJECT_POSITION       0x31
 #define SEARCH_LOW            0x32
 #define SET_LIMITS            0x33
 #define PRE_FETCH             0x34
 #define READ_POSITION         0x34
+#define GET_DATA_BUFFER_STATUS 0x34
 #define SYNCHRONIZE_CACHE     0x35
 #define LOCK_UNLOCK_CACHE     0x36
 #define INITIALIZE_ELEMENT_STATUS_WITH_RANGE 0x37
@@ -229,7 +236,7 @@
 #define MODE_PAGE_TO_PROTECT                  0x1d
 #define MODE_PAGE_CAPABILITIES                0x2a
 #define MODE_PAGE_ALLS                        0x3f
-/* Not in Mt. Fuji, but in ATAPI 2.6 -- depricated now in favor
+/* Not in Mt. Fuji, but in ATAPI 2.6 -- deprecated now in favor
  * of MODE_PAGE_SENSE_POWER */
 #define MODE_PAGE_CDROM                       0x0d
 
diff --git a/include/block/snapshot.h b/include/block/snapshot.h
index 770d9bb..e5c0553 100644
--- a/include/block/snapshot.h
+++ b/include/block/snapshot.h
@@ -26,7 +26,6 @@
 #define SNAPSHOT_H
 
 #include "qemu-common.h"
-#include "qapi/error.h"
 #include "qemu/option.h"
 
 
@@ -63,9 +62,9 @@
                          const char *snapshot_id,
                          const char *name,
                          Error **errp);
-void bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
-                                        const char *id_or_name,
-                                        Error **errp);
+int bdrv_snapshot_delete_by_id_or_name(BlockDriverState *bs,
+                                       const char *id_or_name,
+                                       Error **errp);
 int bdrv_snapshot_list(BlockDriverState *bs,
                        QEMUSnapshotInfo **psn_info);
 int bdrv_snapshot_load_tmp(BlockDriverState *bs,
@@ -75,4 +74,22 @@
 int bdrv_snapshot_load_tmp_by_id_or_name(BlockDriverState *bs,
                                          const char *id_or_name,
                                          Error **errp);
+
+
+/* Group operations. All block drivers are involved.
+ * These functions will properly handle dataplane (take aio_context_acquire
+ * when appropriate for appropriate block drivers */
+
+bool bdrv_all_can_snapshot(BlockDriverState **first_bad_bs);
+int bdrv_all_delete_snapshot(const char *name, BlockDriverState **first_bsd_bs,
+                             Error **err);
+int bdrv_all_goto_snapshot(const char *name, BlockDriverState **first_bsd_bs);
+int bdrv_all_find_snapshot(const char *name, BlockDriverState **first_bad_bs);
+int bdrv_all_create_snapshot(QEMUSnapshotInfo *sn,
+                             BlockDriverState *vm_state_bs,
+                             uint64_t vm_state_size,
+                             BlockDriverState **first_bad_bs);
+
+BlockDriverState *bdrv_all_find_vmstate_bs(void);
+
 #endif
diff --git a/include/block/thread-pool.h b/include/block/thread-pool.h
index 42eb5e8..7dd7d73 100644
--- a/include/block/thread-pool.h
+++ b/include/block/thread-pool.h
@@ -16,7 +16,7 @@
  */
 
 #ifndef QEMU_THREAD_POOL_H
-#define QEMU_THREAD_POOL_H 1
+#define QEMU_THREAD_POOL_H
 
 #include "block/block.h"
 
diff --git a/include/block/throttle-groups.h b/include/block/throttle-groups.h
new file mode 100644
index 0000000..d983d34
--- /dev/null
+++ b/include/block/throttle-groups.h
@@ -0,0 +1,47 @@
+/*
+ * QEMU block throttling group infrastructure
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef THROTTLE_GROUPS_H
+#define THROTTLE_GROUPS_H
+
+#include "qemu/throttle.h"
+#include "block/block_int.h"
+
+const char *throttle_group_get_name(BlockBackend *blk);
+
+ThrottleState *throttle_group_incref(const char *name);
+void throttle_group_unref(ThrottleState *ts);
+
+void throttle_group_config(BlockBackend *blk, ThrottleConfig *cfg);
+void throttle_group_get_config(BlockBackend *blk, ThrottleConfig *cfg);
+
+void throttle_group_register_blk(BlockBackend *blk, const char *groupname);
+void throttle_group_unregister_blk(BlockBackend *blk);
+void throttle_group_restart_blk(BlockBackend *blk);
+
+void coroutine_fn throttle_group_co_io_limits_intercept(BlockBackend *blk,
+                                                        unsigned int bytes,
+                                                        bool is_write);
+
+#endif
diff --git a/include/block/write-threshold.h b/include/block/write-threshold.h
new file mode 100644
index 0000000..234d219
--- /dev/null
+++ b/include/block/write-threshold.h
@@ -0,0 +1,61 @@
+/*
+ * QEMU System Emulator block write threshold notification
+ *
+ * Copyright Red Hat, Inc. 2014
+ *
+ * Authors:
+ *  Francesco Romani <fromani@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+#ifndef BLOCK_WRITE_THRESHOLD_H
+#define BLOCK_WRITE_THRESHOLD_H
+
+#include "qemu-common.h"
+
+/*
+ * bdrv_write_threshold_set:
+ *
+ * Set the write threshold for block devices, in bytes.
+ * Notify when a write exceeds the threshold, meaning the device
+ * is becoming full, so it can be transparently resized.
+ * To be used with thin-provisioned block devices.
+ *
+ * Use threshold_bytes == 0 to disable.
+ */
+void bdrv_write_threshold_set(BlockDriverState *bs, uint64_t threshold_bytes);
+
+/*
+ * bdrv_write_threshold_get
+ *
+ * Get the configured write threshold, in bytes.
+ * Zero means no threshold configured.
+ */
+uint64_t bdrv_write_threshold_get(const BlockDriverState *bs);
+
+/*
+ * bdrv_write_threshold_is_set
+ *
+ * Tell if a write threshold is set for a given BDS.
+ */
+bool bdrv_write_threshold_is_set(const BlockDriverState *bs);
+
+/*
+ * bdrv_write_threshold_exceeded
+ *
+ * Return the extent of a write request that exceeded the threshold,
+ * or zero if the request is below the threshold.
+ * Return zero also if the threshold was not set.
+ *
+ * NOTE: here we assume the following holds for each request this code
+ * deals with:
+ *
+ * assert((req->offset + req->bytes) <= UINT64_MAX)
+ *
+ * Please not there is *not* an actual C assert().
+ */
+uint64_t bdrv_write_threshold_exceeded(const BlockDriverState *bs,
+                                       const BdrvTrackedRequest *req);
+
+#endif
diff --git a/include/config.h b/include/config.h
deleted file mode 100644
index e20f786..0000000
--- a/include/config.h
+++ /dev/null
@@ -1,2 +0,0 @@
-#include "config-host.h"
-#include "config-target.h"
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
new file mode 100644
index 0000000..12fb321
--- /dev/null
+++ b/include/crypto/aes.h
@@ -0,0 +1,67 @@
+#ifndef QEMU_AES_H
+#define QEMU_AES_H
+
+#define AES_MAXNR 14
+#define AES_BLOCK_SIZE 16
+
+struct aes_key_st {
+    uint32_t rd_key[4 *(AES_MAXNR + 1)];
+    int rounds;
+};
+typedef struct aes_key_st AES_KEY;
+
+/* FreeBSD/OpenSSL have their own AES functions with the same names in -lcrypto
+ * (which might be pulled in via curl), so redefine to avoid conflicts. */
+#define AES_set_encrypt_key QEMU_AES_set_encrypt_key
+#define AES_set_decrypt_key QEMU_AES_set_decrypt_key
+#define AES_encrypt QEMU_AES_encrypt
+#define AES_decrypt QEMU_AES_decrypt
+#define AES_cbc_encrypt QEMU_AES_cbc_encrypt
+
+int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
+	AES_KEY *key);
+int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
+	AES_KEY *key);
+
+void AES_encrypt(const unsigned char *in, unsigned char *out,
+	const AES_KEY *key);
+void AES_decrypt(const unsigned char *in, unsigned char *out,
+	const AES_KEY *key);
+void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
+		     const unsigned long length, const AES_KEY *key,
+		     unsigned char *ivec, const int enc);
+
+extern const uint8_t AES_sbox[256];
+extern const uint8_t AES_isbox[256];
+
+/* AES ShiftRows and InvShiftRows */
+extern const uint8_t AES_shifts[16];
+extern const uint8_t AES_ishifts[16];
+
+/* AES InvMixColumns */
+/* AES_imc[x][0] = [x].[0e, 09, 0d, 0b]; */
+/* AES_imc[x][1] = [x].[0b, 0e, 09, 0d]; */
+/* AES_imc[x][2] = [x].[0d, 0b, 0e, 09]; */
+/* AES_imc[x][3] = [x].[09, 0d, 0b, 0e]; */
+extern const uint32_t AES_imc[256][4];
+
+/*
+AES_Te0[x] = S [x].[02, 01, 01, 03];
+AES_Te1[x] = S [x].[03, 02, 01, 01];
+AES_Te2[x] = S [x].[01, 03, 02, 01];
+AES_Te3[x] = S [x].[01, 01, 03, 02];
+AES_Te4[x] = S [x].[01, 01, 01, 01];
+
+AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
+AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
+AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
+AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
+AES_Td4[x] = Si[x].[01, 01, 01, 01];
+*/
+
+extern const uint32_t AES_Te0[256], AES_Te1[256], AES_Te2[256],
+                      AES_Te3[256], AES_Te4[256];
+extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256],
+                      AES_Td3[256], AES_Td4[256];
+
+#endif
diff --git a/include/crypto/afsplit.h b/include/crypto/afsplit.h
new file mode 100644
index 0000000..7dd21f0
--- /dev/null
+++ b/include/crypto/afsplit.h
@@ -0,0 +1,135 @@
+/*
+ * QEMU Crypto anti forensic information splitter
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_AFSPLIT_H
+#define QCRYPTO_AFSPLIT_H
+
+#include "crypto/hash.h"
+
+/**
+ * This module implements the anti-forensic splitter that is specified
+ * as part of the LUKS format:
+ *
+ *   http://clemens.endorphin.org/cryptography
+ *   http://clemens.endorphin.org/TKS1-draft.pdf
+ *
+ * The core idea is to take a short piece of data (key material)
+ * and process it to expand it to a much larger piece of data.
+ * The expansion process is reversible, to obtain the original
+ * short data. The key property of the expansion is that if any
+ * byte in the larger data set is changed / missing, it should be
+ * impossible to recreate the original short data.
+ *
+ * <example>
+ *    <title>Creating a large split key for storage</title>
+ *    <programlisting>
+ * size_t nkey = 32;
+ * uint32_t stripes = 32768; // To produce a 1 MB split key
+ * uint8_t *masterkey = ....a 32-byte AES key...
+ * uint8_t *splitkey;
+ *
+ * splitkey = g_new0(uint8_t, nkey * stripes);
+ *
+ * if (qcrypto_afsplit_encode(QCRYPTO_HASH_ALG_SHA256,
+ *                            nkey, stripes,
+ *                            masterkey, splitkey, errp) < 0) {
+ *     g_free(splitkey);
+ *     g_free(masterkey);
+ *     return -1;
+ * }
+ *
+ * ...store splitkey somewhere...
+ *
+ * g_free(splitkey);
+ * g_free(masterkey);
+ *    </programlisting>
+ * </example>
+ *
+ * <example>
+ *    <title>Retrieving a master key from storage</title>
+ *    <programlisting>
+ * size_t nkey = 32;
+ * uint32_t stripes = 32768; // To produce a 1 MB split key
+ * uint8_t *masterkey;
+ * uint8_t *splitkey = .... read in 1 MB of data...
+ *
+ * masterkey = g_new0(uint8_t, nkey);
+ *
+ * if (qcrypto_afsplit_decode(QCRYPTO_HASH_ALG_SHA256,
+ *                            nkey, stripes,
+ *                            splitkey, masterkey, errp) < 0) {
+ *     g_free(splitkey);
+ *     g_free(masterkey);
+ *     return -1;
+ * }
+ *
+ * ..decrypt data with masterkey...
+ *
+ * g_free(splitkey);
+ * g_free(masterkey);
+ *    </programlisting>
+ * </example>
+ */
+
+/**
+ * qcrypto_afsplit_encode:
+ * @hash: the hash algorithm to use for data expansion
+ * @blocklen: the size of @in in bytes
+ * @stripes: the number of times to expand @in in size
+ * @in: the master key to be expanded in size
+ * @out: preallocated buffer to hold the split key
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Split the data in @in, which is @blocklen bytes in
+ * size, to form a larger piece of data @out, which is
+ * @blocklen * @stripes bytes in size.
+ *
+ * Returns: 0 on success, -1 on error;
+ */
+int qcrypto_afsplit_encode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp);
+
+/**
+ * qcrypto_afsplit_decode:
+ * @hash: the hash algorithm to use for data compression
+ * @blocklen: the size of @out in bytes
+ * @stripes: the number of times to decrease @in in size
+ * @in: the split key to be recombined
+ * @out: preallocated buffer to hold the master key
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Join the data in @in, which is @blocklen * @stripes
+ * bytes in size, to form the original small piece of
+ * data @out, which is @blocklen bytes in size.
+ *
+ * Returns: 0 on success, -1 on error;
+ */
+int qcrypto_afsplit_decode(QCryptoHashAlgorithm hash,
+                           size_t blocklen,
+                           uint32_t stripes,
+                           const uint8_t *in,
+                           uint8_t *out,
+                           Error **errp);
+
+#endif /* QCRYPTO_AFSPLIT_H */
diff --git a/include/crypto/block.h b/include/crypto/block.h
new file mode 100644
index 0000000..b6971de
--- /dev/null
+++ b/include/crypto/block.h
@@ -0,0 +1,248 @@
+/*
+ * QEMU Crypto block device encryption
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_BLOCK_H
+#define QCRYPTO_BLOCK_H
+
+#include "crypto/cipher.h"
+#include "crypto/ivgen.h"
+
+typedef struct QCryptoBlock QCryptoBlock;
+
+/* See also QCryptoBlockFormat, QCryptoBlockCreateOptions
+ * and QCryptoBlockOpenOptions in qapi/crypto.json */
+
+typedef ssize_t (*QCryptoBlockReadFunc)(QCryptoBlock *block,
+                                        size_t offset,
+                                        uint8_t *buf,
+                                        size_t buflen,
+                                        Error **errp,
+                                        void *opaque);
+
+typedef ssize_t (*QCryptoBlockInitFunc)(QCryptoBlock *block,
+                                        size_t headerlen,
+                                        Error **errp,
+                                        void *opaque);
+
+typedef ssize_t (*QCryptoBlockWriteFunc)(QCryptoBlock *block,
+                                         size_t offset,
+                                         const uint8_t *buf,
+                                         size_t buflen,
+                                         Error **errp,
+                                         void *opaque);
+
+/**
+ * qcrypto_block_has_format:
+ * @format: the encryption format
+ * @buf: the data from head of the volume
+ * @len: the length of @buf in bytes
+ *
+ * Given @len bytes of data from the head of a storage volume
+ * in @buf, probe to determine if the volume has the encryption
+ * format specified in @format.
+ *
+ * Returns: true if the data in @buf matches @format
+ */
+bool qcrypto_block_has_format(QCryptoBlockFormat format,
+                              const uint8_t *buf,
+                              size_t buflen);
+
+typedef enum {
+    QCRYPTO_BLOCK_OPEN_NO_IO = (1 << 0),
+} QCryptoBlockOpenFlags;
+
+/**
+ * qcrypto_block_open:
+ * @options: the encryption options
+ * @readfunc: callback for reading data from the volume
+ * @opaque: data to pass to @readfunc
+ * @flags: bitmask of QCryptoBlockOpenFlags values
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new block encryption object for an existing
+ * storage volume encrypted with format identified by
+ * the parameters in @options.
+ *
+ * This will use @readfunc to initialize the encryption
+ * context based on the volume header(s), extracting the
+ * master key(s) as required.
+ *
+ * If @flags contains QCRYPTO_BLOCK_OPEN_NO_IO then
+ * the open process will be optimized to skip any parts
+ * that are only required to perform I/O. In particular
+ * this would usually avoid the need to decrypt any
+ * master keys. The only thing that can be done with
+ * the resulting QCryptoBlock object would be to query
+ * metadata such as the payload offset. There will be
+ * no cipher or ivgen objects available.
+ *
+ * If any part of initializing the encryption context
+ * fails an error will be returned. This could be due
+ * to the volume being in the wrong format, a cipher
+ * or IV generator algorithm that is not supported,
+ * or incorrect passphrases.
+ *
+ * Returns: a block encryption format, or NULL on error
+ */
+QCryptoBlock *qcrypto_block_open(QCryptoBlockOpenOptions *options,
+                                 QCryptoBlockReadFunc readfunc,
+                                 void *opaque,
+                                 unsigned int flags,
+                                 Error **errp);
+
+/**
+ * qcrypto_block_create:
+ * @format: the encryption format
+ * @initfunc: callback for initializing volume header
+ * @writefunc: callback for writing data to the volume header
+ * @opaque: data to pass to @initfunc and @writefunc
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new block encryption object for initializing
+ * a storage volume to be encrypted with format identified
+ * by the parameters in @options.
+ *
+ * This method will allocate space for a new volume header
+ * using @initfunc and then write header data using @writefunc,
+ * generating new master keys, etc as required. Any existing
+ * data present on the volume will be irrevocably destroyed.
+ *
+ * If any part of initializing the encryption context
+ * fails an error will be returned. This could be due
+ * to the volume being in the wrong format, a cipher
+ * or IV generator algorithm that is not supported,
+ * or incorrect passphrases.
+ *
+ * Returns: a block encryption format, or NULL on error
+ */
+QCryptoBlock *qcrypto_block_create(QCryptoBlockCreateOptions *options,
+                                   QCryptoBlockInitFunc initfunc,
+                                   QCryptoBlockWriteFunc writefunc,
+                                   void *opaque,
+                                   Error **errp);
+
+
+/**
+ * qcrypto_block_get_info:
+ * @block: the block encryption object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Get information about the configuration options for the
+ * block encryption object. This includes details such as
+ * the cipher algorithms, modes, and initialization vector
+ * generators.
+ *
+ * Returns: a block encryption info object, or NULL on error
+ */
+QCryptoBlockInfo *qcrypto_block_get_info(QCryptoBlock *block,
+                                         Error **errp);
+
+/**
+ * @qcrypto_block_decrypt:
+ * @block: the block encryption object
+ * @startsector: the sector from which @buf was read
+ * @buf: the buffer to decrypt
+ * @len: the length of @buf in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Decrypt @len bytes of cipher text in @buf, writing
+ * plain text back into @buf
+ *
+ * Returns 0 on success, -1 on failure
+ */
+int qcrypto_block_decrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp);
+
+/**
+ * @qcrypto_block_encrypt:
+ * @block: the block encryption object
+ * @startsector: the sector to which @buf will be written
+ * @buf: the buffer to decrypt
+ * @len: the length of @buf in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Encrypt @len bytes of plain text in @buf, writing
+ * cipher text back into @buf
+ *
+ * Returns 0 on success, -1 on failure
+ */
+int qcrypto_block_encrypt(QCryptoBlock *block,
+                          uint64_t startsector,
+                          uint8_t *buf,
+                          size_t len,
+                          Error **errp);
+
+/**
+ * qcrypto_block_get_cipher:
+ * @block: the block encryption object
+ *
+ * Get the cipher to use for payload encryption
+ *
+ * Returns: the cipher object
+ */
+QCryptoCipher *qcrypto_block_get_cipher(QCryptoBlock *block);
+
+/**
+ * qcrypto_block_get_ivgen:
+ * @block: the block encryption object
+ *
+ * Get the initialization vector generator to use for
+ * payload encryption
+ *
+ * Returns: the IV generator object
+ */
+QCryptoIVGen *qcrypto_block_get_ivgen(QCryptoBlock *block);
+
+
+/**
+ * qcrypto_block_get_kdf_hash:
+ * @block: the block encryption object
+ *
+ * Get the hash algorithm used with the key derivation
+ * function
+ *
+ * Returns: the hash algorithm
+ */
+QCryptoHashAlgorithm qcrypto_block_get_kdf_hash(QCryptoBlock *block);
+
+/**
+ * qcrypto_block_get_payload_offset:
+ * @block: the block encryption object
+ *
+ * Get the offset to the payload indicated by the
+ * encryption header, in bytes.
+ *
+ * Returns: the payload offset in bytes
+ */
+uint64_t qcrypto_block_get_payload_offset(QCryptoBlock *block);
+
+/**
+ * qcrypto_block_free:
+ * @block: the block encryption object
+ *
+ * Release all resources associated with the encryption
+ * object
+ */
+void qcrypto_block_free(QCryptoBlock *block);
+
+#endif /* QCRYPTO_BLOCK_H */
diff --git a/include/crypto/cipher.h b/include/crypto/cipher.h
new file mode 100644
index 0000000..376654d
--- /dev/null
+++ b/include/crypto/cipher.h
@@ -0,0 +1,233 @@
+/*
+ * QEMU Crypto cipher algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_CIPHER_H
+#define QCRYPTO_CIPHER_H
+
+#include "qapi-types.h"
+
+typedef struct QCryptoCipher QCryptoCipher;
+
+/* See also "QCryptoCipherAlgorithm" and "QCryptoCipherMode"
+ * enums defined in qapi/crypto.json */
+
+/**
+ * QCryptoCipher:
+ *
+ * The QCryptoCipher object provides a way to perform encryption
+ * and decryption of data, with a standard API, regardless of the
+ * algorithm used. It further isolates the calling code from the
+ * details of the specific underlying implementation, whether
+ * built-in, libgcrypt or nettle.
+ *
+ * Each QCryptoCipher object is capable of performing both
+ * encryption and decryption, and can operate in a number
+ * or modes including ECB, CBC.
+ *
+ * <example>
+ *   <title>Encrypting data with AES-128 in CBC mode</title>
+ *   <programlisting>
+ * QCryptoCipher *cipher;
+ * uint8_t key = ....;
+ * size_t keylen = 16;
+ * uint8_t iv = ....;
+ *
+ * if (!qcrypto_cipher_supports(QCRYPTO_CIPHER_ALG_AES_128)) {
+ *    error_report(errp, "Feature <blah> requires AES cipher support");
+ *    return -1;
+ * }
+ *
+ * cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+ *                             QCRYPTO_CIPHER_MODE_CBC,
+ *                             key, keylen,
+ *                             errp);
+ * if (!cipher) {
+ *    return -1;
+ * }
+ *
+ * if (qcrypto_cipher_set_iv(cipher, iv, keylen, errp) < 0) {
+ *    return -1;
+ * }
+ *
+ * if (qcrypto_cipher_encrypt(cipher, rawdata, encdata, datalen, errp) < 0) {
+ *    return -1;
+ * }
+ *
+ * qcrypto_cipher_free(cipher);
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+struct QCryptoCipher {
+    QCryptoCipherAlgorithm alg;
+    QCryptoCipherMode mode;
+    void *opaque;
+};
+
+/**
+ * qcrypto_cipher_supports:
+ * @alg: the cipher algorithm
+ *
+ * Determine if @alg cipher algorithm is supported by the
+ * current configured build
+ *
+ * Returns: true if the algorithm is supported, false otherwise
+ */
+bool qcrypto_cipher_supports(QCryptoCipherAlgorithm alg);
+
+/**
+ * qcrypto_cipher_get_block_len:
+ * @alg: the cipher algorithm
+ *
+ * Get the required data block size in bytes. When
+ * encrypting data, it must be a multiple of the
+ * block size.
+ *
+ * Returns: the block size in bytes
+ */
+size_t qcrypto_cipher_get_block_len(QCryptoCipherAlgorithm alg);
+
+
+/**
+ * qcrypto_cipher_get_key_len:
+ * @alg: the cipher algorithm
+ *
+ * Get the required key size in bytes.
+ *
+ * Returns: the key size in bytes
+ */
+size_t qcrypto_cipher_get_key_len(QCryptoCipherAlgorithm alg);
+
+
+/**
+ * qcrypto_cipher_get_iv_len:
+ * @alg: the cipher algorithm
+ * @mode: the cipher mode
+ *
+ * Get the required initialization vector size
+ * in bytes, if one is required.
+ *
+ * Returns: the IV size in bytes, or 0 if no IV is permitted
+ */
+size_t qcrypto_cipher_get_iv_len(QCryptoCipherAlgorithm alg,
+                                 QCryptoCipherMode mode);
+
+
+/**
+ * qcrypto_cipher_new:
+ * @alg: the cipher algorithm
+ * @mode: the cipher usage mode
+ * @key: the private key bytes
+ * @nkey: the length of @key
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Creates a new cipher object for encrypting/decrypting
+ * data with the algorithm @alg in the usage mode @mode.
+ *
+ * The @key parameter provides the bytes representing
+ * the encryption/decryption key to use. The @nkey parameter
+ * specifies the length of @key in bytes. Each algorithm has
+ * one or more valid key lengths, and it is an error to provide
+ * a key of the incorrect length.
+ *
+ * The returned cipher object must be released with
+ * qcrypto_cipher_free() when no longer required
+ *
+ * Returns: a new cipher object, or NULL on error
+ */
+QCryptoCipher *qcrypto_cipher_new(QCryptoCipherAlgorithm alg,
+                                  QCryptoCipherMode mode,
+                                  const uint8_t *key, size_t nkey,
+                                  Error **errp);
+
+/**
+ * qcrypto_cipher_free:
+ * @cipher: the cipher object
+ *
+ * Release the memory associated with @cipher that
+ * was previously allocated by qcrypto_cipher_new()
+ */
+void qcrypto_cipher_free(QCryptoCipher *cipher);
+
+/**
+ * qcrypto_cipher_encrypt:
+ * @cipher: the cipher object
+ * @in: buffer holding the plain text input data
+ * @out: buffer to fill with the cipher text output data
+ * @len: the length of @in and @out buffers
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Encrypts the plain text stored in @in, filling
+ * @out with the resulting ciphered text. Both the
+ * @in and @out buffers must have the same size,
+ * given by @len.
+ *
+ * Returns: 0 on success, or -1 on error
+ */
+int qcrypto_cipher_encrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp);
+
+
+/**
+ * qcrypto_cipher_decrypt:
+ * @cipher: the cipher object
+ * @in: buffer holding the cipher text input data
+ * @out: buffer to fill with the plain text output data
+ * @len: the length of @in and @out buffers
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Decrypts the cipher text stored in @in, filling
+ * @out with the resulting plain text. Both the
+ * @in and @out buffers must have the same size,
+ * given by @len.
+ *
+ * Returns: 0 on success, or -1 on error
+ */
+int qcrypto_cipher_decrypt(QCryptoCipher *cipher,
+                           const void *in,
+                           void *out,
+                           size_t len,
+                           Error **errp);
+
+/**
+ * qcrypto_cipher_setiv:
+ * @cipher: the cipher object
+ * @iv: the initialization vector bytes
+ * @niv: the length of @iv
+ * @errpr: pointer to a NULL-initialized error object
+ *
+ * If the @cipher object is setup to use a mode that requires
+ * initialization vectors, this sets the initialization vector
+ * bytes. The @iv data should have the same length as the
+ * cipher key used when originally constructing the cipher
+ * object. It is an error to set an initialization vector
+ * if the cipher mode does not require one.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_cipher_setiv(QCryptoCipher *cipher,
+                         const uint8_t *iv, size_t niv,
+                         Error **errp);
+
+#endif /* QCRYPTO_CIPHER_H */
diff --git a/include/crypto/desrfb.h b/include/crypto/desrfb.h
new file mode 100644
index 0000000..7ca596c
--- /dev/null
+++ b/include/crypto/desrfb.h
@@ -0,0 +1,50 @@
+/*
+ * This is D3DES (V5.09) by Richard Outerbridge with the double and
+ * triple-length support removed for use in VNC.
+ *
+ * These changes are:
+ *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
+ *
+ * This software is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+#ifndef QCRYPTO_DESRFB_H
+#define QCRYPTO_DESRFB_H
+
+/* d3des.h -
+ *
+ *	Headers and defines for d3des.c
+ *	Graven Imagery, 1992.
+ *
+ * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge
+ *	(GEnie : OUTER; CIS : [71755,204])
+ */
+
+#define EN0	0	/* MODE == encrypt */
+#define DE1	1	/* MODE == decrypt */
+
+void deskey(unsigned char *, int);
+/*		      hexkey[8]     MODE
+ * Sets the internal key register according to the hexadecimal
+ * key contained in the 8 bytes of hexkey, according to the DES,
+ * for encryption or decryption according to MODE.
+ */
+
+void usekey(unsigned long *);
+/*		    cookedkey[32]
+ * Loads the internal key register with the data in cookedkey.
+ */
+
+void des(unsigned char *, unsigned char *);
+/*		    from[8]	      to[8]
+ * Encrypts/Decrypts (according to the key currently loaded in the
+ * internal key register) one block of eight bytes at address 'from'
+ * into the block at address 'to'.  They can be the same.
+ */
+
+/* d3des.h V5.09 rwo 9208.04 15:06 Graven Imagery
+ ********************************************************************/
+
+#endif
diff --git a/include/crypto/hash.h b/include/crypto/hash.h
new file mode 100644
index 0000000..ca3267f
--- /dev/null
+++ b/include/crypto/hash.h
@@ -0,0 +1,192 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_HASH_H
+#define QCRYPTO_HASH_H
+
+#include "qapi-types.h"
+
+/* See also "QCryptoHashAlgorithm" defined in qapi/crypto.json */
+
+/**
+ * qcrypto_hash_supports:
+ * @alg: the hash algorithm
+ *
+ * Determine if @alg hash algorithm is supported by the
+ * current configured build.
+ *
+ * Returns: true if the algorithm is supported, false otherwise
+ */
+gboolean qcrypto_hash_supports(QCryptoHashAlgorithm alg);
+
+
+/**
+ * qcrypto_hash_digest_len:
+ * @alg: the hash algorithm
+ *
+ * Determine the size of the hash digest in bytes
+ *
+ * Returns: the digest length in bytes
+ */
+size_t qcrypto_hash_digest_len(QCryptoHashAlgorithm alg);
+
+/**
+ * qcrypto_hash_bytesv:
+ * @alg: the hash algorithm
+ * @iov: the array of memory regions to hash
+ * @niov: the length of @iov
+ * @result: pointer to hold output hash
+ * @resultlen: pointer to hold length of @result
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Computes the hash across all the memory regions
+ * present in @iov. The @result pointer will be
+ * filled with raw bytes representing the computed
+ * hash, which will have length @resultlen. The
+ * memory pointer in @result must be released
+ * with a call to g_free() when no longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_hash_bytesv(QCryptoHashAlgorithm alg,
+                        const struct iovec *iov,
+                        size_t niov,
+                        uint8_t **result,
+                        size_t *resultlen,
+                        Error **errp);
+
+/**
+ * qcrypto_hash_bytes:
+ * @alg: the hash algorithm
+ * @buf: the memory region to hash
+ * @len: the length of @buf
+ * @result: pointer to hold output hash
+ * @resultlen: pointer to hold length of @result
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Computes the hash across all the memory region
+ * @buf of length @len. The @result pointer will be
+ * filled with raw bytes representing the computed
+ * hash, which will have length @resultlen. The
+ * memory pointer in @result must be released
+ * with a call to g_free() when no longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_hash_bytes(QCryptoHashAlgorithm alg,
+                       const char *buf,
+                       size_t len,
+                       uint8_t **result,
+                       size_t *resultlen,
+                       Error **errp);
+
+/**
+ * qcrypto_hash_digestv:
+ * @alg: the hash algorithm
+ * @iov: the array of memory regions to hash
+ * @niov: the length of @iov
+ * @digest: pointer to hold output hash
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Computes the hash across all the memory regions
+ * present in @iov. The @digest pointer will be
+ * filled with the printable hex digest of the computed
+ * hash, which will be terminated by '\0'. The
+ * memory pointer in @digest must be released
+ * with a call to g_free() when no longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_hash_digestv(QCryptoHashAlgorithm alg,
+                         const struct iovec *iov,
+                         size_t niov,
+                         char **digest,
+                         Error **errp);
+
+/**
+ * qcrypto_hash_digest:
+ * @alg: the hash algorithm
+ * @buf: the memory region to hash
+ * @len: the length of @buf
+ * @digest: pointer to hold output hash
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Computes the hash across all the memory region
+ * @buf of length @len. The @digest pointer will be
+ * filled with the printable hex digest of the computed
+ * hash, which will be terminated by '\0'. The
+ * memory pointer in @digest must be released
+ * with a call to g_free() when no longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_hash_digest(QCryptoHashAlgorithm alg,
+                        const char *buf,
+                        size_t len,
+                        char **digest,
+                        Error **errp);
+
+/**
+ * qcrypto_hash_base64v:
+ * @alg: the hash algorithm
+ * @iov: the array of memory regions to hash
+ * @niov: the length of @iov
+ * @base64: pointer to hold output hash
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Computes the hash across all the memory regions
+ * present in @iov. The @base64 pointer will be
+ * filled with the base64 encoding of the computed
+ * hash, which will be terminated by '\0'. The
+ * memory pointer in @base64 must be released
+ * with a call to g_free() when no longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_hash_base64v(QCryptoHashAlgorithm alg,
+                         const struct iovec *iov,
+                         size_t niov,
+                         char **base64,
+                         Error **errp);
+
+/**
+ * qcrypto_hash_base64:
+ * @alg: the hash algorithm
+ * @buf: the memory region to hash
+ * @len: the length of @buf
+ * @base64: pointer to hold output hash
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Computes the hash across all the memory region
+ * @buf of length @len. The @base64 pointer will be
+ * filled with the base64 encoding of the computed
+ * hash, which will be terminated by '\0'. The
+ * memory pointer in @base64 must be released
+ * with a call to g_free() when no longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_hash_base64(QCryptoHashAlgorithm alg,
+                        const char *buf,
+                        size_t len,
+                        char **base64,
+                        Error **errp);
+
+#endif /* QCRYPTO_HASH_H */
diff --git a/include/crypto/init.h b/include/crypto/init.h
new file mode 100644
index 0000000..04c1edf
--- /dev/null
+++ b/include/crypto/init.h
@@ -0,0 +1,26 @@
+/*
+ * QEMU Crypto initialization
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_INIT_H
+#define QCRYPTO_INIT_H
+
+int qcrypto_init(Error **errp);
+
+#endif /* QCRYPTO_INIT_H */
diff --git a/include/crypto/ivgen.h b/include/crypto/ivgen.h
new file mode 100644
index 0000000..0350cd2
--- /dev/null
+++ b/include/crypto/ivgen.h
@@ -0,0 +1,206 @@
+/*
+ * QEMU Crypto block IV generator
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_IVGEN_H
+#define QCRYPTO_IVGEN_H
+
+#include "crypto/cipher.h"
+#include "crypto/hash.h"
+
+/**
+ * This module provides a framework for generating initialization
+ * vectors for block encryption schemes using chained cipher modes
+ * CBC. The principle is that each disk sector is assigned a unique
+ * initialization vector for use for encryption of data in that
+ * sector.
+ *
+ * <example>
+ *   <title>Encrypting block data with initialiation vectors</title>
+ *   <programlisting>
+ * uint8_t *data = ....data to encrypt...
+ * size_t ndata = XXX;
+ * uint8_t *key = ....some encryption key...
+ * size_t nkey = XXX;
+ * uint8_t *iv;
+ * size_t niv;
+ * size_t sector = 0;
+ *
+ * g_assert((ndata % 512) == 0);
+ *
+ * QCryptoIVGen *ivgen = qcrypto_ivgen_new(QCRYPTO_IVGEN_ALG_ESSIV,
+ *                                         QCRYPTO_CIPHER_ALG_AES_128,
+ *                                         QCRYPTO_HASH_ALG_SHA256,
+ *                                         key, nkey, errp);
+ * if (!ivgen) {
+ *    return -1;
+ * }
+ *
+ * QCryptoCipher *cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+ *                                            QCRYPTO_CIPHER_MODE_CBC,
+ *                                            key, nkey, errp);
+ * if (!cipher) {
+ *     goto error;
+ * }
+ *
+ * niv =  qcrypto_cipher_get_iv_len(QCRYPTO_CIPHER_ALG_AES_128,
+ *                                  QCRYPTO_CIPHER_MODE_CBC);
+ * iv = g_new0(uint8_t, niv);
+ *
+ *
+ * while (ndata) {
+ *     if (qcrypto_ivgen_calculate(ivgen, sector, iv, niv, errp) < 0) {
+ *         goto error;
+ *     }
+ *     if (qcrypto_cipher_setiv(cipher, iv, niv, errp) < 0) {
+ *         goto error;
+ *     }
+ *     if (qcrypto_cipher_encrypt(cipher,
+ *                                data + (sector * 512),
+ *                                data + (sector * 512),
+ *                                512, errp) < 0) {
+ *         goto error;
+ *     }
+ *     sector++;
+ *     ndata -= 512;
+ * }
+ *
+ * g_free(iv);
+ * qcrypto_ivgen_free(ivgen);
+ * qcrypto_cipher_free(cipher);
+ * return 0;
+ *
+ *error:
+ * g_free(iv);
+ * qcrypto_ivgen_free(ivgen);
+ * qcrypto_cipher_free(cipher);
+ * return -1;
+ *   </programlisting>
+ * </example>
+ */
+
+typedef struct QCryptoIVGen QCryptoIVGen;
+
+/* See also QCryptoIVGenAlgorithm enum in qapi/crypto.json */
+
+
+/**
+ * qcrypto_ivgen_new:
+ * @alg: the initialization vector generation algorithm
+ * @cipheralg: the cipher algorithm or 0
+ * @hash: the hash algorithm or 0
+ * @key: the encryption key or NULL
+ * @nkey: the size of @key in bytes
+ *
+ * Create a new initialization vector generator that uses
+ * the algorithm @alg. Whether the remaining parameters
+ * are required or not depends on the choice of @alg
+ * requested.
+ *
+ * - QCRYPTO_IVGEN_ALG_PLAIN
+ *
+ * The IVs are generated by the 32-bit truncated sector
+ * number. This should never be used for block devices
+ * that are larger than 2^32 sectors in size.
+ * All the other parameters are unused.
+ *
+ * - QCRYPTO_IVGEN_ALG_PLAIN64
+ *
+ * The IVs are generated by the 64-bit sector number.
+ * All the other parameters are unused.
+ *
+ * - QCRYPTO_IVGEN_ALG_ESSIV:
+ *
+ * The IVs are generated by encrypting the 64-bit sector
+ * number with a hash of an encryption key. The @cipheralg,
+ * @hash, @key and @nkey parameters are all required.
+ *
+ * Returns: a new IV generator, or NULL on error
+ */
+QCryptoIVGen *qcrypto_ivgen_new(QCryptoIVGenAlgorithm alg,
+                                QCryptoCipherAlgorithm cipheralg,
+                                QCryptoHashAlgorithm hash,
+                                const uint8_t *key, size_t nkey,
+                                Error **errp);
+
+/**
+ * qcrypto_ivgen_calculate:
+ * @ivgen: the IV generator object
+ * @sector: the 64-bit sector number
+ * @iv: a pre-allocated buffer to hold the generated IV
+ * @niv: the number of bytes in @iv
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Calculate a new initialiation vector for the data
+ * to be stored in sector @sector. The IV will be
+ * written into the buffer @iv of size @niv.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_ivgen_calculate(QCryptoIVGen *ivgen,
+                            uint64_t sector,
+                            uint8_t *iv, size_t niv,
+                            Error **errp);
+
+
+/**
+ * qcrypto_ivgen_get_algorithm:
+ * @ivgen: the IV generator object
+ *
+ * Get the algorithm used by this IV generator
+ *
+ * Returns: the IV generator algorithm
+ */
+QCryptoIVGenAlgorithm qcrypto_ivgen_get_algorithm(QCryptoIVGen *ivgen);
+
+
+/**
+ * qcrypto_ivgen_get_cipher:
+ * @ivgen: the IV generator object
+ *
+ * Get the cipher algorithm used by this IV generator (if
+ * applicable)
+ *
+ * Returns: the cipher algorithm
+ */
+QCryptoCipherAlgorithm qcrypto_ivgen_get_cipher(QCryptoIVGen *ivgen);
+
+
+/**
+ * qcrypto_ivgen_get_hash:
+ * @ivgen: the IV generator object
+ *
+ * Get the hash algorithm used by this IV generator (if
+ * applicable)
+ *
+ * Returns: the hash algorithm
+ */
+QCryptoHashAlgorithm qcrypto_ivgen_get_hash(QCryptoIVGen *ivgen);
+
+
+/**
+ * qcrypto_ivgen_free:
+ * @ivgen: the IV generator object
+ *
+ * Release all resources associated with @ivgen, or a no-op
+ * if @ivgen is NULL
+ */
+void qcrypto_ivgen_free(QCryptoIVGen *ivgen);
+
+#endif /* QCRYPTO_IVGEN_H */
diff --git a/include/crypto/pbkdf.h b/include/crypto/pbkdf.h
new file mode 100644
index 0000000..e9e4cec
--- /dev/null
+++ b/include/crypto/pbkdf.h
@@ -0,0 +1,152 @@
+/*
+ * QEMU Crypto PBKDF support (Password-Based Key Derivation Function)
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_PBKDF_H
+#define QCRYPTO_PBKDF_H
+
+#include "crypto/hash.h"
+
+/**
+ * This module provides an interface to the PBKDF2 algorithm
+ *
+ *   https://en.wikipedia.org/wiki/PBKDF2
+ *
+ * <example>
+ *   <title>Generating an AES encryption key from a user password</title>
+ *   <programlisting>
+ * #include "crypto/cipher.h"
+ * #include "crypto/random.h"
+ * #include "crypto/pbkdf.h"
+ *
+ * ....
+ *
+ * char *password = "a-typical-awful-user-password";
+ * size_t nkey = qcrypto_cipher_get_key_len(QCRYPTO_CIPHER_ALG_AES_128);
+ * uint8_t *salt = g_new0(uint8_t, nkey);
+ * uint8_t *key = g_new0(uint8_t, nkey);
+ * int iterations;
+ * QCryptoCipher *cipher;
+ *
+ * if (qcrypto_random_bytes(salt, nkey, errp) < 0) {
+ *     g_free(key);
+ *     g_free(salt);
+ *     return -1;
+ * }
+ *
+ * iterations = qcrypto_pbkdf2_count_iters(QCRYPTO_HASH_ALG_SHA256,
+ *                                         (const uint8_t *)password,
+ *                                         strlen(password),
+ *                                         salt, nkey, errp);
+ * if (iterations < 0) {
+ *     g_free(key);
+ *     g_free(salt);
+ *     return -1;
+ * }
+ *
+ * if (qcrypto_pbkdf2(QCRYPTO_HASH_ALG_SHA256,
+ *                    (const uint8_t *)password, strlen(password),
+ *                    salt, nkey, iterations, key, nkey, errp) < 0) {
+ *     g_free(key);
+ *     g_free(salt);
+ *     return -1;
+ * }
+ *
+ * g_free(salt);
+ *
+ * cipher = qcrypto_cipher_new(QCRYPTO_CIPHER_ALG_AES_128,
+ *                             QCRYPTO_CIPHER_MODE_ECB,
+ *                             key, nkey, errp);
+ * g_free(key);
+ *
+ * ....encrypt some data...
+ *
+ * qcrypto_cipher_free(cipher);
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+/**
+ * qcrypto_pbkdf2_supports:
+ * @hash: the hash algorithm
+ *
+ * Determine if the current build supports the PBKDF2 algorithm
+ * in combination with the hash @hash.
+ *
+ * Returns true if supported, false otherwise
+ */
+bool qcrypto_pbkdf2_supports(QCryptoHashAlgorithm hash);
+
+
+/**
+ * qcrypto_pbkdf2:
+ * @hash: the hash algorithm to use
+ * @key: the user password / key
+ * @nkey: the length of @key in bytes
+ * @salt: a random salt
+ * @nsalt: length of @salt in bytes
+ * @iterations: the number of iterations to compute
+ * @out: pointer to pre-allocated buffer to hold output
+ * @nout: length of @out in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Apply the PBKDF2 algorithm to derive an encryption
+ * key from a user password provided in @key. The
+ * @salt parameter is used to perturb the algorithm.
+ * The @iterations count determines how many times
+ * the hashing process is run, which influences how
+ * hard it is to crack the key. The number of @iterations
+ * should be large enough such that the algorithm takes
+ * 1 second or longer to derive a key. The derived key
+ * will be stored in the preallocated buffer @out.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qcrypto_pbkdf2(QCryptoHashAlgorithm hash,
+                   const uint8_t *key, size_t nkey,
+                   const uint8_t *salt, size_t nsalt,
+                   unsigned int iterations,
+                   uint8_t *out, size_t nout,
+                   Error **errp);
+
+/**
+ * qcrypto_pbkdf2_count_iters:
+ * @hash: the hash algorithm to use
+ * @key: the user password / key
+ * @nkey: the length of @key in bytes
+ * @salt: a random salt
+ * @nsalt: length of @salt in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Time the PBKDF2 algorithm to determine how many
+ * iterations are required to derive an encryption
+ * key from a user password provided in @key in 1
+ * second of compute time. The result of this can
+ * be used as a the @iterations parameter of a later
+ * call to qcrypto_pbkdf2().
+ *
+ * Returns: number of iterations in 1 second, -1 on error
+ */
+int qcrypto_pbkdf2_count_iters(QCryptoHashAlgorithm hash,
+                               const uint8_t *key, size_t nkey,
+                               const uint8_t *salt, size_t nsalt,
+                               Error **errp);
+
+#endif /* QCRYPTO_PBKDF_H */
diff --git a/include/crypto/random.h b/include/crypto/random.h
new file mode 100644
index 0000000..a101353
--- /dev/null
+++ b/include/crypto/random.h
@@ -0,0 +1,44 @@
+/*
+ * QEMU Crypto random number provider
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_RANDOM_H
+#define QCRYPTO_RANDOM_H
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+
+
+/**
+ * qcrypto_random_bytes:
+ * @buf: the buffer to fill
+ * @buflen: length of @buf in bytes
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Fill @buf with @buflen bytes of cryptographically strong
+ * random data
+ *
+ * Returns 0 on success, -1 on error
+ */
+int qcrypto_random_bytes(uint8_t *buf,
+                         size_t buflen,
+                         Error **errp);
+
+
+#endif /* QCRYPTO_RANDOM_H */
diff --git a/include/crypto/secret.h b/include/crypto/secret.h
new file mode 100644
index 0000000..07a963e
--- /dev/null
+++ b/include/crypto/secret.h
@@ -0,0 +1,146 @@
+/*
+ * QEMU crypto secret support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_SECRET_H
+#define QCRYPTO_SECRET_H
+
+#include "qom/object.h"
+
+#define TYPE_QCRYPTO_SECRET "secret"
+#define QCRYPTO_SECRET(obj)                  \
+    OBJECT_CHECK(QCryptoSecret, (obj), TYPE_QCRYPTO_SECRET)
+
+typedef struct QCryptoSecret QCryptoSecret;
+typedef struct QCryptoSecretClass QCryptoSecretClass;
+
+/**
+ * QCryptoSecret:
+ *
+ * The QCryptoSecret object provides storage of secrets,
+ * which may be user passwords, encryption keys or any
+ * other kind of sensitive data that is represented as
+ * a sequence of bytes.
+ *
+ * The sensitive data associated with the secret can
+ * be provided directly via the 'data' property, or
+ * indirectly via the 'file' property. In the latter
+ * case there is support for file descriptor passing
+ * via the usual /dev/fdset/NN syntax that QEMU uses.
+ *
+ * The data for a secret can be provided in two formats,
+ * either as a UTF-8 string (the default), or as base64
+ * encoded 8-bit binary data. The latter is appropriate
+ * for raw encryption keys, while the former is appropriate
+ * for user entered passwords.
+ *
+ * The data may be optionally encrypted with AES-256-CBC,
+ * and the decryption key provided by another
+ * QCryptoSecret instance identified by the 'keyid'
+ * property. When passing sensitive data directly
+ * via the 'data' property it is strongly recommended
+ * to use the AES encryption facility to prevent the
+ * sensitive data being exposed in the process listing
+ * or system log files.
+ *
+ * Providing data directly, insecurely (suitable for
+ * ad hoc developer testing only)
+ *
+ *  $QEMU -object secret,id=sec0,data=letmein
+ *
+ * Providing data indirectly:
+ *
+ *  # printf "letmein" > password.txt
+ *  # $QEMU \
+ *      -object secret,id=sec0,file=password.txt
+ *
+ * Using a master encryption key with data.
+ *
+ * The master key needs to be created as 32 secure
+ * random bytes (optionally base64 encoded)
+ *
+ *  # openssl rand -base64 32 > key.b64
+ *  # KEY=$(base64 -d key.b64 | hexdump  -v -e '/1 "%02X"')
+ *
+ * Each secret to be encrypted needs to have a random
+ * initialization vector generated. These do not need
+ * to be kept secret
+ *
+ *  # openssl rand -base64 16 > iv.b64
+ *  # IV=$(base64 -d iv.b64 | hexdump  -v -e '/1 "%02X"')
+ *
+ * A secret to be defined can now be encrypted
+ *
+ *  # SECRET=$(printf "letmein" |
+ *             openssl enc -aes-256-cbc -a -K $KEY -iv $IV)
+ *
+ * When launching QEMU, create a master secret pointing
+ * to key.b64 and specify that to be used to decrypt
+ * the user password
+ *
+ *  # $QEMU \
+ *      -object secret,id=secmaster0,format=base64,file=key.b64 \
+ *      -object secret,id=sec0,keyid=secmaster0,format=base64,\
+ *          data=$SECRET,iv=$(<iv.b64)
+ *
+ * When encrypting, the data can still be provided via an
+ * external file, in which case it is possible to use either
+ * raw binary data, or base64 encoded. This example uses
+ * raw format
+ *
+ *  # printf "letmein" |
+ *       openssl enc -aes-256-cbc -K $KEY -iv $IV -o pw.aes
+ *  # $QEMU \
+ *      -object secret,id=secmaster0,format=base64,file=key.b64 \
+ *      -object secret,id=sec0,keyid=secmaster0,\
+ *          file=pw.aes,iv=$(<iv.b64)
+ *
+ * Note that the ciphertext can be in either raw or base64
+ * format, as indicated by the 'format' parameter, but the
+ * plaintext resulting from decryption is expected to always
+ * be in raw format.
+ */
+
+struct QCryptoSecret {
+    Object parent_obj;
+    uint8_t *rawdata;
+    size_t rawlen;
+    QCryptoSecretFormat format;
+    char *data;
+    char *file;
+    char *keyid;
+    char *iv;
+};
+
+
+struct QCryptoSecretClass {
+    ObjectClass parent_class;
+};
+
+
+extern int qcrypto_secret_lookup(const char *secretid,
+                                 uint8_t **data,
+                                 size_t *datalen,
+                                 Error **errp);
+extern char *qcrypto_secret_lookup_as_utf8(const char *secretid,
+                                           Error **errp);
+extern char *qcrypto_secret_lookup_as_base64(const char *secretid,
+                                             Error **errp);
+
+#endif /* QCRYPTO_SECRET_H */
diff --git a/include/crypto/tlscreds.h b/include/crypto/tlscreds.h
new file mode 100644
index 0000000..ad47d88
--- /dev/null
+++ b/include/crypto/tlscreds.h
@@ -0,0 +1,66 @@
+/*
+ * QEMU crypto TLS credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCREDS_H
+#define QCRYPTO_TLSCREDS_H
+
+#include "qom/object.h"
+
+#ifdef CONFIG_GNUTLS
+#include <gnutls/gnutls.h>
+#endif
+
+#define TYPE_QCRYPTO_TLS_CREDS "tls-creds"
+#define QCRYPTO_TLS_CREDS(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCreds, (obj), TYPE_QCRYPTO_TLS_CREDS)
+
+typedef struct QCryptoTLSCreds QCryptoTLSCreds;
+typedef struct QCryptoTLSCredsClass QCryptoTLSCredsClass;
+
+#define QCRYPTO_TLS_CREDS_DH_PARAMS "dh-params.pem"
+
+
+/**
+ * QCryptoTLSCreds:
+ *
+ * The QCryptoTLSCreds object is an abstract base for different
+ * types of TLS handshake credentials. Most commonly the
+ * QCryptoTLSCredsX509 subclass will be used to provide x509
+ * certificate credentials.
+ */
+
+struct QCryptoTLSCreds {
+    Object parent_obj;
+    char *dir;
+    QCryptoTLSCredsEndpoint endpoint;
+#ifdef CONFIG_GNUTLS
+    gnutls_dh_params_t dh_params;
+#endif
+    bool verifyPeer;
+    char *priority;
+};
+
+
+struct QCryptoTLSCredsClass {
+    ObjectClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCREDS_H */
diff --git a/include/crypto/tlscredsanon.h b/include/crypto/tlscredsanon.h
new file mode 100644
index 0000000..4d6b7e4
--- /dev/null
+++ b/include/crypto/tlscredsanon.h
@@ -0,0 +1,111 @@
+/*
+ * QEMU crypto TLS anonymous credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCREDSANON_H
+#define QCRYPTO_TLSCREDSANON_H
+
+#include "crypto/tlscreds.h"
+
+#define TYPE_QCRYPTO_TLS_CREDS_ANON "tls-creds-anon"
+#define QCRYPTO_TLS_CREDS_ANON(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCredsAnon, (obj), TYPE_QCRYPTO_TLS_CREDS_ANON)
+
+
+typedef struct QCryptoTLSCredsAnon QCryptoTLSCredsAnon;
+typedef struct QCryptoTLSCredsAnonClass QCryptoTLSCredsAnonClass;
+
+/**
+ * QCryptoTLSCredsAnon:
+ *
+ * The QCryptoTLSCredsAnon object provides a representation
+ * of anonymous credentials used perform a TLS handshake.
+ * This is primarily provided for backwards compatibility and
+ * its use is discouraged as it has poor security characteristics
+ * due to lacking MITM attack protection amongst other problems.
+ *
+ * This is a user creatable object, which can be instantiated
+ * via object_new_propv():
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects in code</title>
+ *   <programlisting>
+ *   Object *obj;
+ *   Error *err = NULL;
+ *   obj = object_new_propv(TYPE_QCRYPTO_TLS_CREDS_ANON,
+ *                          "tlscreds0",
+ *                          &err,
+ *                          "endpoint", "server",
+ *                          "dir", "/path/x509/cert/dir",
+ *                          "verify-peer", "yes",
+ *                          NULL);
+ *   </programlisting>
+ * </example>
+ *
+ * Or via QMP:
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects via QMP</title>
+ *   <programlisting>
+ *    {
+ *       "execute": "object-add", "arguments": {
+ *          "id": "tlscreds0",
+ *          "qom-type": "tls-creds-anon",
+ *          "props": {
+ *             "endpoint": "server",
+ *             "dir": "/path/to/x509/cert/dir",
+ *             "verify-peer": false
+ *          }
+ *       }
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ *
+ * Or via the CLI:
+ *
+ * <example>
+ *   <title>Creating anonymous TLS credential objects via CLI</title>
+ *   <programlisting>
+ *  qemu-system-x86_64 -object tls-creds-anon,id=tlscreds0,\
+ *          endpoint=server,verify-peer=off,\
+ *          dir=/path/to/x509/certdir/
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+
+struct QCryptoTLSCredsAnon {
+    QCryptoTLSCreds parent_obj;
+#ifdef CONFIG_GNUTLS
+    union {
+        gnutls_anon_server_credentials_t server;
+        gnutls_anon_client_credentials_t client;
+    } data;
+#endif
+};
+
+
+struct QCryptoTLSCredsAnonClass {
+    QCryptoTLSCredsClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCREDSANON_H */
diff --git a/include/crypto/tlscredsx509.h b/include/crypto/tlscredsx509.h
new file mode 100644
index 0000000..66ad6a7
--- /dev/null
+++ b/include/crypto/tlscredsx509.h
@@ -0,0 +1,113 @@
+/*
+ * QEMU crypto TLS x509 credential support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSCREDSX509_H
+#define QCRYPTO_TLSCREDSX509_H
+
+#include "crypto/tlscreds.h"
+
+#define TYPE_QCRYPTO_TLS_CREDS_X509 "tls-creds-x509"
+#define QCRYPTO_TLS_CREDS_X509(obj)                  \
+    OBJECT_CHECK(QCryptoTLSCredsX509, (obj), TYPE_QCRYPTO_TLS_CREDS_X509)
+
+typedef struct QCryptoTLSCredsX509 QCryptoTLSCredsX509;
+typedef struct QCryptoTLSCredsX509Class QCryptoTLSCredsX509Class;
+
+#define QCRYPTO_TLS_CREDS_X509_CA_CERT "ca-cert.pem"
+#define QCRYPTO_TLS_CREDS_X509_CA_CRL "ca-crl.pem"
+#define QCRYPTO_TLS_CREDS_X509_SERVER_KEY "server-key.pem"
+#define QCRYPTO_TLS_CREDS_X509_SERVER_CERT "server-cert.pem"
+#define QCRYPTO_TLS_CREDS_X509_CLIENT_KEY "client-key.pem"
+#define QCRYPTO_TLS_CREDS_X509_CLIENT_CERT "client-cert.pem"
+
+
+/**
+ * QCryptoTLSCredsX509:
+ *
+ * The QCryptoTLSCredsX509 object provides a representation
+ * of x509 credentials used to perform a TLS handshake.
+ *
+ * This is a user creatable object, which can be instantiated
+ * via object_new_propv():
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects in code</title>
+ *   <programlisting>
+ *   Object *obj;
+ *   Error *err = NULL;
+ *   obj = object_new_propv(TYPE_QCRYPTO_TLS_CREDS_X509,
+ *                          "tlscreds0",
+ *                          &err,
+ *                          "endpoint", "server",
+ *                          "dir", "/path/x509/cert/dir",
+ *                          "verify-peer", "yes",
+ *                          NULL);
+ *   </programlisting>
+ * </example>
+ *
+ * Or via QMP:
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects via QMP</title>
+ *   <programlisting>
+ *    {
+ *       "execute": "object-add", "arguments": {
+ *          "id": "tlscreds0",
+ *          "qom-type": "tls-creds-x509",
+ *          "props": {
+ *             "endpoint": "server",
+ *             "dir": "/path/to/x509/cert/dir",
+ *             "verify-peer": false
+ *          }
+ *       }
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ *
+ * Or via the CLI:
+ *
+ * <example>
+ *   <title>Creating x509 TLS credential objects via CLI</title>
+ *   <programlisting>
+ *  qemu-system-x86_64 -object tls-creds-x509,id=tlscreds0,\
+ *          endpoint=server,verify-peer=off,\
+ *          dir=/path/to/x509/certdir/
+ *   </programlisting>
+ * </example>
+ *
+ */
+
+struct QCryptoTLSCredsX509 {
+    QCryptoTLSCreds parent_obj;
+#ifdef CONFIG_GNUTLS
+    gnutls_certificate_credentials_t data;
+#endif
+    bool sanityCheck;
+    char *passwordid;
+};
+
+
+struct QCryptoTLSCredsX509Class {
+    QCryptoTLSCredsClass parent_class;
+};
+
+
+#endif /* QCRYPTO_TLSCREDSX509_H */
diff --git a/include/crypto/tlssession.h b/include/crypto/tlssession.h
new file mode 100644
index 0000000..1c7414e
--- /dev/null
+++ b/include/crypto/tlssession.h
@@ -0,0 +1,322 @@
+/*
+ * QEMU crypto TLS session support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QCRYPTO_TLSSESSION_H
+#define QCRYPTO_TLSSESSION_H
+
+#include "crypto/tlscreds.h"
+
+/**
+ * QCryptoTLSSession:
+ *
+ * The QCryptoTLSSession object encapsulates the
+ * logic to integrate with a TLS providing library such
+ * as GNUTLS, to setup and run TLS sessions.
+ *
+ * The API is designed such that it has no assumption about
+ * the type of transport it is running over. It may be a
+ * traditional TCP socket, or something else entirely. The
+ * only requirement is a full-duplex stream of some kind.
+ *
+ * <example>
+ *   <title>Using TLS session objects</title>
+ *   <programlisting>
+ * static ssize_t mysock_send(const char *buf, size_t len,
+ *                            void *opaque)
+ * {
+ *    int fd = GPOINTER_TO_INT(opaque);
+ *
+ *    return write(*fd, buf, len);
+ * }
+ *
+ * static ssize_t mysock_recv(const char *buf, size_t len,
+ *                            void *opaque)
+ * {
+ *    int fd = GPOINTER_TO_INT(opaque);
+ *
+ *    return read(*fd, buf, len);
+ * }
+ *
+ * static int mysock_run_tls(int sockfd,
+ *                           QCryptoTLSCreds *creds,
+ *                           Error *errp)
+ * {
+ *    QCryptoTLSSession *sess;
+ *
+ *    sess = qcrypto_tls_session_new(creds,
+ *                                   "vnc.example.com",
+ *                                   NULL,
+ *                                   QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+ *                                   errp);
+ *    if (sess == NULL) {
+ *       return -1;
+ *    }
+ *
+ *    qcrypto_tls_session_set_callbacks(sess,
+ *                                      mysock_send,
+ *                                      mysock_recv
+ *                                      GINT_TO_POINTER(fd));
+ *
+ *    while (1) {
+ *       if (qcrypto_tls_session_handshake(sess, errp) < 0) {
+ *           qcrypto_tls_session_free(sess);
+ *           return -1;
+ *       }
+ *
+ *       switch(qcrypto_tls_session_get_handshake_status(sess)) {
+ *       case QCRYPTO_TLS_HANDSHAKE_COMPLETE:
+ *           if (qcrypto_tls_session_check_credentials(sess, errp) < )) {
+ *               qcrypto_tls_session_free(sess);
+ *               return -1;
+ *           }
+ *           goto done;
+ *       case QCRYPTO_TLS_HANDSHAKE_RECVING:
+ *           ...wait for GIO_IN event on fd...
+ *           break;
+ *       case QCRYPTO_TLS_HANDSHAKE_SENDING:
+ *           ...wait for GIO_OUT event on fd...
+ *           break;
+ *       }
+ *    }
+ *   done:
+ *
+ *    ....send/recv payload data on sess...
+ *
+ *    qcrypto_tls_session_free(sess):
+ * }
+ *   </programlisting>
+ * </example>
+ */
+
+typedef struct QCryptoTLSSession QCryptoTLSSession;
+
+
+/**
+ * qcrypto_tls_session_new:
+ * @creds: pointer to a TLS credentials object
+ * @hostname: optional hostname to validate
+ * @aclname: optional ACL to validate peer credentials against
+ * @endpoint: role of the TLS session, client or server
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new TLS session object that will be used to
+ * negotiate a TLS session over an arbitrary data channel.
+ * The session object can operate as either the server or
+ * client, according to the value of the @endpoint argument.
+ *
+ * For clients, the @hostname parameter should hold the full
+ * unmodified hostname as requested by the user. This will
+ * be used to verify the against the hostname reported in
+ * the server's credentials (aka x509 certificate).
+ *
+ * The @aclname parameter (optionally) specifies the name
+ * of an access control list that will be used to validate
+ * the peer's credentials. For x509 credentials, the ACL
+ * will be matched against the CommonName shown in the peer's
+ * certificate. If the session is acting as a server, setting
+ * an ACL will require that the client provide a validate
+ * x509 client certificate.
+ *
+ * After creating the session object, the I/O callbacks
+ * must be set using the qcrypto_tls_session_set_callbacks()
+ * method. A TLS handshake sequence must then be completed
+ * using qcrypto_tls_session_handshake(), before payload
+ * data is permitted to be sent/received.
+ *
+ * The session object must be released by calling
+ * qcrypto_tls_session_free() when no longer required
+ *
+ * Returns: a TLS session object, or NULL on error.
+ */
+QCryptoTLSSession *qcrypto_tls_session_new(QCryptoTLSCreds *creds,
+                                           const char *hostname,
+                                           const char *aclname,
+                                           QCryptoTLSCredsEndpoint endpoint,
+                                           Error **errp);
+
+/**
+ * qcrypto_tls_session_free:
+ * @sess: the TLS session object
+ *
+ * Release all memory associated with the TLS session
+ * object previously allocated by qcrypto_tls_session_new()
+ */
+void qcrypto_tls_session_free(QCryptoTLSSession *sess);
+
+/**
+ * qcrypto_tls_session_check_credentials:
+ * @sess: the TLS session object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Validate the peer's credentials after a successful
+ * TLS handshake. It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns 0 if the credentials validated, -1 on error
+ */
+int qcrypto_tls_session_check_credentials(QCryptoTLSSession *sess,
+                                          Error **errp);
+
+typedef ssize_t (*QCryptoTLSSessionWriteFunc)(const char *buf,
+                                              size_t len,
+                                              void *opaque);
+typedef ssize_t (*QCryptoTLSSessionReadFunc)(char *buf,
+                                             size_t len,
+                                             void *opaque);
+
+/**
+ * qcrypto_tls_session_set_callbacks:
+ * @sess: the TLS session object
+ * @writeFunc: callback for sending data
+ * @readFunc: callback to receiving data
+ * @opaque: data to pass to callbacks
+ *
+ * Sets the callback functions that are to be used for sending
+ * and receiving data on the underlying data channel. Typically
+ * the callbacks to write/read to/from a TCP socket, but there
+ * is no assumption made about the type of channel used.
+ *
+ * The @writeFunc callback will be passed the encrypted
+ * data to send to the remote peer.
+ *
+ * The @readFunc callback will be passed a pointer to fill
+ * with encrypted data received from the remote peer
+ */
+void qcrypto_tls_session_set_callbacks(QCryptoTLSSession *sess,
+                                       QCryptoTLSSessionWriteFunc writeFunc,
+                                       QCryptoTLSSessionReadFunc readFunc,
+                                       void *opaque);
+
+/**
+ * qcrypto_tls_session_write:
+ * @sess: the TLS session object
+ * @buf: the plain text to send
+ * @len: the length of @buf
+ *
+ * Encrypt @len bytes of the data in @buf and send
+ * it to the remote peer using the callback previously
+ * registered with qcrypto_tls_session_set_callbacks()
+ *
+ * It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns: the number of bytes sent, or -1 on error
+ */
+ssize_t qcrypto_tls_session_write(QCryptoTLSSession *sess,
+                                  const char *buf,
+                                  size_t len);
+
+/**
+ * qcrypto_tls_session_read:
+ * @sess: the TLS session object
+ * @buf: to fill with plain text received
+ * @len: the length of @buf
+ *
+ * Receive up to @len bytes of data from the remote peer
+ * using the callback previously registered with
+ * qcrypto_tls_session_set_callbacks(), decrypt it and
+ * store it in @buf.
+ *
+ * It is an error to call this before
+ * qcrypto_tls_session_get_handshake_status() returns
+ * QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ *
+ * Returns: the number of bytes received, or -1 on error
+ */
+ssize_t qcrypto_tls_session_read(QCryptoTLSSession *sess,
+                                 char *buf,
+                                 size_t len);
+
+/**
+ * qcrypto_tls_session_handshake:
+ * @sess: the TLS session object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Start, or continue, a TLS handshake sequence. If
+ * the underlying data channel is non-blocking, then
+ * this method may return control before the handshake
+ * is complete. On non-blocking channels the
+ * qcrypto_tls_session_get_handshake_status() method
+ * should be used to determine whether the handshake
+ * has completed, or is waiting to send or receive
+ * data. In the latter cases, the caller should setup
+ * an event loop watch and call this method again
+ * once the underlying data channel is ready to read
+ * or write again
+ */
+int qcrypto_tls_session_handshake(QCryptoTLSSession *sess,
+                                  Error **errp);
+
+typedef enum {
+    QCRYPTO_TLS_HANDSHAKE_COMPLETE,
+    QCRYPTO_TLS_HANDSHAKE_SENDING,
+    QCRYPTO_TLS_HANDSHAKE_RECVING,
+} QCryptoTLSSessionHandshakeStatus;
+
+/**
+ * qcrypto_tls_session_get_handshake_status:
+ * @sess: the TLS session object
+ *
+ * Check the status of the TLS handshake. This
+ * is used with non-blocking data channels to
+ * determine whether the handshake is waiting
+ * to send or receive further data to/from the
+ * remote peer.
+ *
+ * Once this returns QCRYPTO_TLS_HANDSHAKE_COMPLETE
+ * it is permitted to send/receive payload data on
+ * the channel
+ */
+QCryptoTLSSessionHandshakeStatus
+qcrypto_tls_session_get_handshake_status(QCryptoTLSSession *sess);
+
+/**
+ * qcrypto_tls_session_get_key_size:
+ * @sess: the TLS session object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Check the size of the data channel encryption key
+ *
+ * Returns: the length in bytes of the encryption key
+ * or -1 on error
+ */
+int qcrypto_tls_session_get_key_size(QCryptoTLSSession *sess,
+                                     Error **errp);
+
+/**
+ * qcrypto_tls_session_get_peer_name:
+ * @sess: the TLS session object
+ *
+ * Get the identified name of the remote peer. If the
+ * TLS session was negotiated using x509 certificate
+ * credentials, this will return the CommonName from
+ * the peer's certificate. If no identified name is
+ * available it will return NULL.
+ *
+ * The returned data must be released with g_free()
+ * when no longer required.
+ *
+ * Returns: the peer's name or NULL.
+ */
+char *qcrypto_tls_session_get_peer_name(QCryptoTLSSession *sess);
+
+#endif /* QCRYPTO_TLSSESSION_H */
diff --git a/include/crypto/xts.h b/include/crypto/xts.h
new file mode 100644
index 0000000..da32ab8
--- /dev/null
+++ b/include/crypto/xts.h
@@ -0,0 +1,85 @@
+/*
+ * QEMU Crypto XTS cipher mode
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This code is originally derived from public domain / WTFPL code in
+ * LibTomCrypt crytographic library http://libtom.org. The XTS code
+ * was donated by Elliptic Semiconductor Inc (www.ellipticsemi.com)
+ * to the LibTom Projects
+ *
+ */
+
+#ifndef QCRYPTO_XTS_H
+#define QCRYPTO_XTS_H
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+
+
+#define XTS_BLOCK_SIZE 16
+
+typedef void xts_cipher_func(const void *ctx,
+                             size_t length,
+                             uint8_t *dst,
+                             const uint8_t *src);
+
+/**
+ * xts_decrypt:
+ * @datactx: the cipher context for data decryption
+ * @tweakctx: the cipher context for tweak decryption
+ * @encfunc: the cipher function for encryption
+ * @decfunc: the cipher function for decryption
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ * @length: the length of @dst and @src
+ * @dst: buffer to hold the decrypted plaintext
+ * @src: buffer providing the ciphertext
+ *
+ * Decrypts @src into @dst
+ */
+void xts_decrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src);
+
+/**
+ * xts_decrypt:
+ * @datactx: the cipher context for data encryption
+ * @tweakctx: the cipher context for tweak encryption
+ * @encfunc: the cipher function for encryption
+ * @decfunc: the cipher function for decryption
+ * @iv: the initialization vector tweak of XTS_BLOCK_SIZE bytes
+ * @length: the length of @dst and @src
+ * @dst: buffer to hold the encrypted ciphertext
+ * @src: buffer providing the plaintext
+ *
+ * Decrypts @src into @dst
+ */
+void xts_encrypt(const void *datactx,
+                 const void *tweakctx,
+                 xts_cipher_func *encfunc,
+                 xts_cipher_func *decfunc,
+                 uint8_t *iv,
+                 size_t length,
+                 uint8_t *dst,
+                 const uint8_t *src);
+
+
+#endif /* QCRYPTO_XTS_H */
diff --git a/include/disas/bfd.h b/include/disas/bfd.h
index 85ad998..8a3488c 100644
--- a/include/disas/bfd.h
+++ b/include/disas/bfd.h
@@ -6,14 +6,10 @@
    interface, for making instruction-processing programs more independent
    of the instruction set being processed.  */
 
-#ifndef DIS_ASM_H
-#define DIS_ASM_H
+#ifndef DISAS_BFD_H
+#define DISAS_BFD_H
 
-#include <stdbool.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
+#include "qemu/fprintf-fn.h"
 
 typedef void *PTR;
 typedef uint64_t bfd_vma;
@@ -22,12 +18,6 @@
 #define sprintf_vma(s,x) sprintf (s, "%0" PRIx64, x)
 #define snprintf_vma(s,ss,x) snprintf (s, ss, "%0" PRIx64, x)
 
-typedef int (*fprintf_function)(FILE *f, const char *fmt, ...);
-
-#ifndef ARRAY_SIZE
-#define ARRAY_SIZE(x)  (sizeof(x) / sizeof((x)[0]))
-#endif
-
 #define BFD64
 
 enum bfd_flavour {
@@ -323,6 +313,11 @@
   void (*print_address_func)
     (bfd_vma addr, struct disassemble_info *info);
 
+    /* Function called to print an instruction. The function is architecture
+     * specific.
+     */
+    int (*print_insn)(bfd_vma addr, struct disassemble_info *info);
+
   /* Function called to determine if there is a symbol at the given ADDR.
      If there is, the function returns 1, otherwise it returns 0.
      This is used by ports which support an overlay manager where
@@ -473,6 +468,7 @@
   (INFO).read_memory_func = buffer_read_memory, \
   (INFO).memory_error_func = perror_memory, \
   (INFO).print_address_func = generic_print_address, \
+  (INFO).print_insn = NULL, \
   (INFO).symbol_at_address_func = generic_symbol_at_address, \
   (INFO).flags = 0, \
   (INFO).bytes_per_line = 0, \
@@ -481,8 +477,9 @@
   (INFO).disassembler_options = NULL, \
   (INFO).insn_info_valid = 0
 
-#define _(x) x
+#ifndef ATTRIBUTE_UNUSED
 #define ATTRIBUTE_UNUSED __attribute__((unused))
+#endif
 
 /* from libbfd */
 
@@ -493,4 +490,4 @@
 bfd_vma bfd_getb16 (const bfd_byte *addr);
 typedef bool bfd_boolean;
 
-#endif /* ! defined (DIS_ASM_H) */
+#endif /* DISAS_BFD_H */
diff --git a/include/disas/disas.h b/include/disas/disas.h
index c13ca9a..e549ca2 100644
--- a/include/disas/disas.h
+++ b/include/disas/disas.h
@@ -1,15 +1,17 @@
-#ifndef _QEMU_DISAS_H
-#define _QEMU_DISAS_H
+#ifndef QEMU_DISAS_H
+#define QEMU_DISAS_H
 
 #include "qemu-common.h"
 
 #ifdef NEED_CPU_H
+#include "cpu.h"
+
 /* Disassemble this for me please... (debugging). */
 void disas(FILE *out, void *code, unsigned long size);
-void target_disas(FILE *out, CPUArchState *env, target_ulong code,
+void target_disas(FILE *out, CPUState *cpu, target_ulong code,
                   target_ulong size, int flags);
 
-void monitor_disas(Monitor *mon, CPUArchState *env,
+void monitor_disas(Monitor *mon, CPUState *cpu,
                    target_ulong pc, int nb_insn, int is_physical, int flags);
 
 /* Look up symbol for debugging purpose.  Returns "" if unknown. */
@@ -40,4 +42,4 @@
 /* Filled in by elfload.c.  Simplistic, but will do for now. */
 extern struct syminfo *syminfos;
 
-#endif /* _QEMU_DISAS_H */
+#endif /* QEMU_DISAS_H */
diff --git a/include/elf.h b/include/elf.h
index a516584..1c2975d 100644
--- a/include/elf.h
+++ b/include/elf.h
@@ -1,7 +1,5 @@
-#ifndef _QEMU_ELF_H
-#define _QEMU_ELF_H
-
-#include <inttypes.h>
+#ifndef QEMU_ELF_H
+#define QEMU_ELF_H
 
 /* 32-bit ELF base types. */
 typedef uint32_t Elf32_Addr;
@@ -54,6 +52,8 @@
 #define EF_MIPS_OPTIONS_FIRST	0x00000080
 #define EF_MIPS_32BITMODE	0x00000100
 #define EF_MIPS_ABI		0x0000f000
+#define EF_MIPS_FP64      0x00000200
+#define EF_MIPS_NAN2008   0x00000400
 #define EF_MIPS_ARCH      0xf0000000
 
 /* These constants define the different elf file types */
@@ -133,6 +133,11 @@
 
 #define EM_AARCH64  183
 
+#define EM_TILEGX   191 /* TILE-Gx */
+
+#define EM_MOXIE           223     /* Moxie processor family */
+#define EM_MOXIE_OLD       0xFEED
+
 /* This is the info that is needed to parse the dynamic section of the file */
 #define DT_NULL		0
 #define DT_NEEDED	1
@@ -473,6 +478,19 @@
 #define PPC_FEATURE_TRUE_LE             0x00000002
 #define PPC_FEATURE_PPC_LE              0x00000001
 
+/* Bits present in AT_HWCAP2 for PowerPC.  */
+
+#define PPC_FEATURE2_ARCH_2_07          0x80000000
+#define PPC_FEATURE2_HAS_HTM            0x40000000
+#define PPC_FEATURE2_HAS_DSCR           0x20000000
+#define PPC_FEATURE2_HAS_EBB            0x10000000
+#define PPC_FEATURE2_HAS_ISEL           0x08000000
+#define PPC_FEATURE2_HAS_TAR            0x04000000
+#define PPC_FEATURE2_HAS_VEC_CRYPTO     0x02000000
+#define PPC_FEATURE2_HTM_NOSC           0x01000000
+#define PPC_FEATURE2_ARCH_3_00          0x00800000
+#define PPC_FEATURE2_HAS_IEEE128        0x00400000
+
 /* Bits present in AT_HWCAP for Sparc.  */
 
 #define HWCAP_SPARC_FLUSH               0x00000001
@@ -1456,6 +1474,8 @@
 #define NT_TASKSTRUCT	4
 #define NT_AUXV		6
 #define NT_PRXFPREG     0x46e62b7f      /* copied from gdb5.1/include/elf/common.h */
+#define NT_S390_VXRS_HIGH 0x30a         /* s390 vector registers 16-31 */
+#define NT_S390_VXRS_LOW  0x309         /* s390 vector registers 0-15 (lower half) */
 #define NT_S390_PREFIX  0x305           /* s390 prefix register */
 #define NT_S390_CTRS    0x304           /* s390 control registers */
 #define NT_S390_TODPREG 0x303           /* s390 TOD programmable register */
@@ -1464,6 +1484,11 @@
 #define NT_PPC_VMX       0x100          /* PowerPC Altivec/VMX registers */
 #define NT_PPC_SPE       0x101          /* PowerPC SPE/EVR registers */
 #define NT_PPC_VSX       0x102          /* PowerPC VSX registers */
+#define NT_ARM_VFP      0x400           /* ARM VFP/NEON registers */
+#define NT_ARM_TLS      0x401           /* ARM TLS register */
+#define NT_ARM_HW_BREAK 0x402           /* ARM hardware breakpoint registers */
+#define NT_ARM_HW_WATCH 0x403           /* ARM hardware watchpoint registers */
+#define NT_ARM_SYSTEM_CALL      0x404   /* ARM system call number */
 
 
 /* Note header in a PT_NOTE section */
@@ -1508,6 +1533,7 @@
 #define elf_shdr	elf32_shdr
 #define elf_sym		elf32_sym
 #define elf_addr_t	Elf32_Off
+#define elf_rela  elf32_rela
 
 #ifdef ELF_USES_RELOCA
 # define ELF_RELOC      Elf32_Rela
@@ -1523,6 +1549,7 @@
 #define elf_shdr	elf64_shdr
 #define elf_sym		elf64_sym
 #define elf_addr_t	Elf64_Off
+#define elf_rela  elf64_rela
 
 #ifdef ELF_USES_RELOCA
 # define ELF_RELOC      Elf64_Rela
@@ -1545,4 +1572,4 @@
 #endif /* ELF_CLASS */
 
 
-#endif /* _QEMU_ELF_H */
+#endif /* QEMU_ELF_H */
diff --git a/include/exec/address-spaces.h b/include/exec/address-spaces.h
index 3d12cdd..db8bfa9 100644
--- a/include/exec/address-spaces.h
+++ b/include/exec/address-spaces.h
@@ -11,8 +11,8 @@
  *
  */
 
-#ifndef EXEC_MEMORY_H
-#define EXEC_MEMORY_H
+#ifndef EXEC_ADDRESS_SPACES_H
+#define EXEC_ADDRESS_SPACES_H
 
 /*
  * Internal interfaces between memory.c/exec.c/vl.c.  Do not #include unless
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index 24aa095..b6a7059 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -24,17 +24,19 @@
 #include "exec/memory.h"
 #include "qemu/thread.h"
 #include "qom/cpu.h"
+#include "qemu/rcu.h"
+
+#define EXCP_INTERRUPT 	0x10000 /* async interruption */
+#define EXCP_HLT        0x10001 /* hlt instruction reached */
+#define EXCP_DEBUG      0x10002 /* cpu stopped after a breakpoint or singlestep */
+#define EXCP_HALTED     0x10003 /* cpu is halted (waiting for external event) */
+#define EXCP_YIELD      0x10004 /* cpu wants to yield timeslice to another */
 
 /* some important defines:
  *
- * WORDS_ALIGNED : if defined, the host cpu can only make word aligned
- * memory accesses.
- *
  * HOST_WORDS_BIGENDIAN : if defined, the host cpu is big endian and
  * otherwise little endian.
  *
- * (TARGET_WORDS_ALIGNED : same for target cpu (not supported yet))
- *
  * TARGET_WORDS_BIGENDIAN : same for target cpu
  */
 
@@ -115,43 +117,9 @@
 #define bswaptls(s) bswap64s(s)
 #endif
 
-/* CPU memory access without any memory or io remapping */
-
-/*
- * the generic syntax for the memory accesses is:
- *
- * load: ld{type}{sign}{size}{endian}_{access_type}(ptr)
- *
- * store: st{type}{size}{endian}_{access_type}(ptr, val)
- *
- * type is:
- * (empty): integer access
- *   f    : float access
- *
- * sign is:
- * (empty): for floats or 32 bit size
- *   u    : unsigned
- *   s    : signed
- *
- * size is:
- *   b: 8 bits
- *   w: 16 bits
- *   l: 32 bits
- *   q: 64 bits
- *
- * endian is:
- * (empty): target cpu endianness or 8 bit access
- *   r    : reversed target cpu endianness (not implemented yet)
- *   be   : big endian (not implemented yet)
- *   le   : little endian (not implemented yet)
- *
- * access_type is:
- *   raw    : host memory access
- *   user   : user mode access using soft MMU
- *   kernel : kernel mode access using soft MMU
+/* Target-endianness CPU memory access functions. These fit into the
+ * {ld,st}{type}{sign}{size}{endian}_p naming scheme described in bswap.h.
  */
-
-/* target-endianness CPU memory access functions */
 #if defined(TARGET_WORDS_BIGENDIAN)
 #define lduw_p(p) lduw_be_p(p)
 #define ldsw_p(p) ldsw_be_p(p)
@@ -181,25 +149,42 @@
 /* MMU memory access macros */
 
 #if defined(CONFIG_USER_ONLY)
-#include <assert.h>
 #include "exec/user/abitypes.h"
 
 /* On some host systems the guest address space is reserved on the host.
  * This allows the guest address space to be offset to a convenient location.
  */
-#if defined(CONFIG_USE_GUEST_BASE)
 extern unsigned long guest_base;
 extern int have_guest_base;
 extern unsigned long reserved_va;
-#define GUEST_BASE guest_base
-#define RESERVED_VA reserved_va
-#else
-#define GUEST_BASE 0ul
-#define RESERVED_VA 0ul
-#endif
 
-#define GUEST_ADDR_MAX (RESERVED_VA ? RESERVED_VA : \
+#define GUEST_ADDR_MAX (reserved_va ? reserved_va : \
                                     (1ul << TARGET_VIRT_ADDR_SPACE_BITS) - 1)
+#else
+
+#include "exec/hwaddr.h"
+uint32_t lduw_phys(AddressSpace *as, hwaddr addr);
+uint32_t ldl_phys(AddressSpace *as, hwaddr addr);
+uint64_t ldq_phys(AddressSpace *as, hwaddr addr);
+void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val);
+void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val);
+void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val);
+void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val);
+
+uint32_t address_space_lduw(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint32_t address_space_ldl(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint64_t address_space_ldq(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stw(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stl(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stq(AddressSpace *as, hwaddr addr, uint64_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
 #endif
 
 /* page related stuff */
@@ -208,12 +193,17 @@
 #define TARGET_PAGE_MASK ~(TARGET_PAGE_SIZE - 1)
 #define TARGET_PAGE_ALIGN(addr) (((addr) + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK)
 
-/* ??? These should be the larger of uintptr_t and target_ulong.  */
+/* Using intptr_t ensures that qemu_*_page_mask is sign-extended even
+ * when intptr_t is 32-bit and we are aligning a long long.
+ */
 extern uintptr_t qemu_real_host_page_size;
+extern intptr_t qemu_real_host_page_mask;
 extern uintptr_t qemu_host_page_size;
-extern uintptr_t qemu_host_page_mask;
+extern intptr_t qemu_host_page_mask;
 
 #define HOST_PAGE_ALIGN(addr) (((addr) + qemu_host_page_size - 1) & qemu_host_page_mask)
+#define REAL_HOST_PAGE_ALIGN(addr) (((addr) + qemu_real_host_page_size - 1) & \
+                                    qemu_real_host_page_mask)
 
 /* same as PROT_xxx */
 #define PAGE_READ      0x0001
@@ -297,33 +287,6 @@
 
 #if !defined(CONFIG_USER_ONLY)
 
-/* memory API */
-
-typedef struct RAMBlock {
-    struct MemoryRegion *mr;
-    uint8_t *host;
-    ram_addr_t offset;
-    ram_addr_t length;
-    uint32_t flags;
-    char idstr[256];
-    /* Reads can take either the iothread or the ramlist lock.
-     * Writes must take both locks.
-     */
-    QTAILQ_ENTRY(RAMBlock) next;
-    int fd;
-} RAMBlock;
-
-typedef struct RAMList {
-    QemuMutex mutex;
-    /* Protected by the iothread lock.  */
-    unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
-    RAMBlock *mru_block;
-    /* Protected by the ramlist lock.  */
-    QTAILQ_HEAD(, RAMBlock) blocks;
-    uint32_t version;
-} RAMList;
-extern RAMList ram_list;
-
 /* Flags stored in the low bits of the TLB virtual address.  These are
  * defined so that fast path ram access is all zeros.
  * The flags all must be between TARGET_PAGE_BITS and
@@ -343,12 +306,12 @@
 #define TLB_FLAGS_MASK  (TLB_INVALID_MASK | TLB_NOTDIRTY | TLB_MMIO)
 
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
-ram_addr_t last_ram_offset(void);
-void qemu_mutex_lock_ramlist(void);
-void qemu_mutex_unlock_ramlist(void);
+void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf);
 #endif /* !CONFIG_USER_ONLY */
 
 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
                         uint8_t *buf, int len, int is_write);
 
+int cpu_exec(CPUState *cpu);
+
 #endif /* CPU_ALL_H */
diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h
index 7b7659e..69fd730 100644
--- a/include/exec/cpu-common.h
+++ b/include/exec/cpu-common.h
@@ -1,5 +1,5 @@
 #ifndef CPU_COMMON_H
-#define CPU_COMMON_H 1
+#define CPU_COMMON_H
 
 /* CPU interfaces that are target independent.  */
 
@@ -7,12 +7,9 @@
 #include "exec/hwaddr.h"
 #endif
 
-#ifndef NEED_CPU_H
-#include "exec/poison.h"
-#endif
-
 #include "qemu/bswap.h"
 #include "qemu/queue.h"
+#include "qemu/fprintf-fn.h"
 
 /**
  * CPUListState:
@@ -26,12 +23,6 @@
     FILE *file;
 } CPUListState;
 
-typedef enum MMUAccessType {
-    MMU_DATA_LOAD  = 0,
-    MMU_DATA_STORE = 1,
-    MMU_INST_FETCH = 2
-} MMUAccessType;
-
 #if !defined(CONFIG_USER_ONLY)
 
 enum device_endian {
@@ -60,9 +51,13 @@
 
 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
 /* This should not be used by devices.  */
-MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
-void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
-void qemu_ram_unset_idstr(ram_addr_t addr);
+ram_addr_t qemu_ram_addr_from_host(void *ptr);
+RAMBlock *qemu_ram_block_by_name(const char *name);
+RAMBlock *qemu_ram_block_from_host(void *ptr, bool round_offset,
+                                   ram_addr_t *offset);
+void qemu_ram_set_idstr(RAMBlock *block, const char *name, DeviceState *dev);
+void qemu_ram_unset_idstr(RAMBlock *block);
+const char *qemu_ram_get_idstr(RAMBlock *rb);
 
 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                             int len, int is_write);
@@ -81,7 +76,8 @@
                               int is_write);
 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
                                int is_write, hwaddr access_len);
-void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque));
+void cpu_register_map_client(QEMUBH *bh);
+void cpu_unregister_map_client(QEMUBH *bh);
 
 bool cpu_physical_memory_is_io(hwaddr phys_addr);
 
@@ -107,16 +103,6 @@
 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val);
 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val);
 
-#ifdef NEED_CPU_H
-uint32_t lduw_phys(AddressSpace *as, hwaddr addr);
-uint32_t ldl_phys(AddressSpace *as, hwaddr addr);
-uint64_t ldq_phys(AddressSpace *as, hwaddr addr);
-void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val);
-void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val);
-void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val);
-void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val);
-#endif
-
 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
                                    const uint8_t *buf, int len);
 void cpu_flush_icache_range(hwaddr start, int len);
@@ -124,11 +110,11 @@
 extern struct MemoryRegion io_mem_rom;
 extern struct MemoryRegion io_mem_notdirty;
 
-typedef void (RAMBlockIterFunc)(void *host_addr,
+typedef int (RAMBlockIterFunc)(const char *block_name, void *host_addr,
     ram_addr_t offset, ram_addr_t length, void *opaque);
 
-void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);
+int qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque);
 
 #endif
 
-#endif /* !CPU_COMMON_H */
+#endif /* CPU_COMMON_H */
diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h
index 0ca6f0b..5f4e303 100644
--- a/include/exec/cpu-defs.h
+++ b/include/exec/cpu-defs.h
@@ -23,13 +23,13 @@
 #error cpu.h included from common code
 #endif
 
-#include "config.h"
-#include <inttypes.h>
-#include "qemu/osdep.h"
+#include "qemu/host-utils.h"
 #include "qemu/queue.h"
+#include "tcg-target.h"
 #ifndef CONFIG_USER_ONLY
 #include "exec/hwaddr.h"
 #endif
+#include "exec/memattrs.h"
 
 #ifndef TARGET_LONG_BITS
 #error TARGET_LONG_BITS must be defined before including this header
@@ -54,23 +54,7 @@
 #error TARGET_LONG_SIZE undefined
 #endif
 
-#define EXCP_INTERRUPT 	0x10000 /* async interruption */
-#define EXCP_HLT        0x10001 /* hlt instruction reached */
-#define EXCP_DEBUG      0x10002 /* cpu stopped after a breakpoint or singlestep */
-#define EXCP_HALTED     0x10003 /* cpu is halted (waiting for external event) */
-#define EXCP_YIELD      0x10004 /* cpu wants to yield timeslice to another */
-
-/* Only the bottom TB_JMP_PAGE_BITS of the jump cache hash bits vary for
-   addresses on the same page.  The top bits are the same.  This allows
-   TLB invalidation to quickly clear a subset of the hash table.  */
-#define TB_JMP_PAGE_BITS (TB_JMP_CACHE_BITS / 2)
-#define TB_JMP_PAGE_SIZE (1 << TB_JMP_PAGE_BITS)
-#define TB_JMP_ADDR_MASK (TB_JMP_PAGE_SIZE - 1)
-#define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE)
-
 #if !defined(CONFIG_USER_ONLY)
-#define CPU_TLB_BITS 8
-#define CPU_TLB_SIZE (1 << CPU_TLB_BITS)
 /* use a fully associative victim tlb of 8 entries */
 #define CPU_VTLB_SIZE 8
 
@@ -80,6 +64,38 @@
 #define CPU_TLB_ENTRY_BITS 5
 #endif
 
+/* TCG_TARGET_TLB_DISPLACEMENT_BITS is used in CPU_TLB_BITS to ensure that
+ * the TLB is not unnecessarily small, but still small enough for the
+ * TLB lookup instruction sequence used by the TCG target.
+ *
+ * TCG will have to generate an operand as large as the distance between
+ * env and the tlb_table[NB_MMU_MODES - 1][0].addend.  For simplicity,
+ * the TCG targets just round everything up to the next power of two, and
+ * count bits.  This works because: 1) the size of each TLB is a largish
+ * power of two, 2) and because the limit of the displacement is really close
+ * to a power of two, 3) the offset of tlb_table[0][0] inside env is smaller
+ * than the size of a TLB.
+ *
+ * For example, the maximum displacement 0xFFF0 on PPC and MIPS, but TCG
+ * just says "the displacement is 16 bits".  TCG_TARGET_TLB_DISPLACEMENT_BITS
+ * then ensures that tlb_table at least 0x8000 bytes large ("not unnecessarily
+ * small": 2^15).  The operand then will come up smaller than 0xFFF0 without
+ * any particular care, because the TLB for a single MMU mode is larger than
+ * 0x10000-0xFFF0=16 bytes.  In the end, the maximum value of the operand
+ * could be something like 0xC000 (the offset of the last TLB table) plus
+ * 0x18 (the offset of the addend field in each TLB entry) plus the offset
+ * of tlb_table inside env (which is non-trivial but not huge).
+ */
+#define CPU_TLB_BITS                                             \
+    MIN(8,                                                       \
+        TCG_TARGET_TLB_DISPLACEMENT_BITS - CPU_TLB_ENTRY_BITS -  \
+        (NB_MMU_MODES <= 1 ? 0 :                                 \
+         NB_MMU_MODES <= 2 ? 1 :                                 \
+         NB_MMU_MODES <= 4 ? 2 :                                 \
+         NB_MMU_MODES <= 8 ? 3 : 4))
+
+#define CPU_TLB_SIZE (1 << CPU_TLB_BITS)
+
 typedef struct CPUTLBEntry {
     /* bit TARGET_LONG_BITS to TARGET_PAGE_BITS : virtual address
        bit TARGET_PAGE_BITS-1..4  : Nonzero for accesses that should not
@@ -87,27 +103,38 @@
        bit 3                      : indicates that the entry is invalid
        bit 2..0                   : zero
     */
-    target_ulong addr_read;
-    target_ulong addr_write;
-    target_ulong addr_code;
-    /* Addend to virtual address to get host address.  IO accesses
-       use the corresponding iotlb value.  */
-    uintptr_t addend;
-    /* padding to get a power of two size */
-    uint8_t dummy[(1 << CPU_TLB_ENTRY_BITS) -
-                  (sizeof(target_ulong) * 3 +
-                   ((-sizeof(target_ulong) * 3) & (sizeof(uintptr_t) - 1)) +
-                   sizeof(uintptr_t))];
+    union {
+        struct {
+            target_ulong addr_read;
+            target_ulong addr_write;
+            target_ulong addr_code;
+            /* Addend to virtual address to get host address.  IO accesses
+               use the corresponding iotlb value.  */
+            uintptr_t addend;
+        };
+        /* padding to get a power of two size */
+        uint8_t dummy[1 << CPU_TLB_ENTRY_BITS];
+    };
 } CPUTLBEntry;
 
 QEMU_BUILD_BUG_ON(sizeof(CPUTLBEntry) != (1 << CPU_TLB_ENTRY_BITS));
 
+/* The IOTLB is not accessed directly inline by generated TCG code,
+ * so the CPUIOTLBEntry layout is not as critical as that of the
+ * CPUTLBEntry. (This is also why we don't want to combine the two
+ * structs into one.)
+ */
+typedef struct CPUIOTLBEntry {
+    hwaddr addr;
+    MemTxAttrs attrs;
+} CPUIOTLBEntry;
+
 #define CPU_COMMON_TLB \
     /* The meaning of the MMU modes is defined in the target code. */   \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
     CPUTLBEntry tlb_v_table[NB_MMU_MODES][CPU_VTLB_SIZE];               \
-    hwaddr iotlb[NB_MMU_MODES][CPU_TLB_SIZE];                           \
-    hwaddr iotlb_v[NB_MMU_MODES][CPU_VTLB_SIZE];                        \
+    CPUIOTLBEntry iotlb[NB_MMU_MODES][CPU_TLB_SIZE];                    \
+    CPUIOTLBEntry iotlb_v[NB_MMU_MODES][CPU_VTLB_SIZE];                 \
     target_ulong tlb_flush_addr;                                        \
     target_ulong tlb_flush_mask;                                        \
     target_ulong vtlb_index;                                            \
@@ -119,7 +146,6 @@
 #endif
 
 
-#define CPU_TEMP_BUF_NLONGS 128
 #define CPU_COMMON                                                      \
     /* soft mmu support */                                              \
     CPU_COMMON_TLB                                                      \
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
index e5550e7..b573df5 100644
--- a/include/exec/cpu_ldst.h
+++ b/include/exec/cpu_ldst.h
@@ -23,27 +23,46 @@
  *
  * Used by target op helpers.
  *
- * MMU mode suffixes are defined in target cpu.h.
+ * The syntax for the accessors is:
+ *
+ * load: cpu_ld{sign}{size}_{mmusuffix}(env, ptr)
+ *
+ * store: cpu_st{sign}{size}_{mmusuffix}(env, ptr, val)
+ *
+ * sign is:
+ * (empty): for 32 and 64 bit sizes
+ *   u    : unsigned
+ *   s    : signed
+ *
+ * size is:
+ *   b: 8 bits
+ *   w: 16 bits
+ *   l: 32 bits
+ *   q: 64 bits
+ *
+ * mmusuffix is one of the generic suffixes "data" or "code", or
+ * (for softmmu configs)  a target-specific MMU mode suffix as defined
+ * in target cpu.h.
  */
 #ifndef CPU_LDST_H
 #define CPU_LDST_H
 
 #if defined(CONFIG_USER_ONLY)
 /* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
-#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + GUEST_BASE))
+#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + guest_base))
 
 #if HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS
 #define h2g_valid(x) 1
 #else
 #define h2g_valid(x) ({ \
-    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
+    unsigned long __guest = (unsigned long)(x) - guest_base; \
     (__guest < (1ul << TARGET_VIRT_ADDR_SPACE_BITS)) && \
-    (!RESERVED_VA || (__guest < RESERVED_VA)); \
+    (!reserved_va || (__guest < reserved_va)); \
 })
 #endif
 
 #define h2g_nocheck(x) ({ \
-    unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
+    unsigned long __ret = (unsigned long)(x) - guest_base; \
     (abi_ulong)__ret; \
 })
 
@@ -53,135 +72,48 @@
     h2g_nocheck(x); \
 })
 
-#define saddr(x) g2h(x)
-#define laddr(x) g2h(x)
-
-#else /* !CONFIG_USER_ONLY */
-/* NOTE: we use double casts if pointers and target_ulong have
-   different sizes */
-#define saddr(x) (uint8_t *)(intptr_t)(x)
-#define laddr(x) (uint8_t *)(intptr_t)(x)
 #endif
 
-#define ldub_raw(p) ldub_p(laddr((p)))
-#define ldsb_raw(p) ldsb_p(laddr((p)))
-#define lduw_raw(p) lduw_p(laddr((p)))
-#define ldsw_raw(p) ldsw_p(laddr((p)))
-#define ldl_raw(p) ldl_p(laddr((p)))
-#define ldq_raw(p) ldq_p(laddr((p)))
-#define ldfl_raw(p) ldfl_p(laddr((p)))
-#define ldfq_raw(p) ldfq_p(laddr((p)))
-#define stb_raw(p, v) stb_p(saddr((p)), v)
-#define stw_raw(p, v) stw_p(saddr((p)), v)
-#define stl_raw(p, v) stl_p(saddr((p)), v)
-#define stq_raw(p, v) stq_p(saddr((p)), v)
-#define stfl_raw(p, v) stfl_p(saddr((p)), v)
-#define stfq_raw(p, v) stfq_p(saddr((p)), v)
-
-
 #if defined(CONFIG_USER_ONLY)
 
-/* if user mode, no other memory access functions */
-#define ldub(p) ldub_raw(p)
-#define ldsb(p) ldsb_raw(p)
-#define lduw(p) lduw_raw(p)
-#define ldsw(p) ldsw_raw(p)
-#define ldl(p) ldl_raw(p)
-#define ldq(p) ldq_raw(p)
-#define ldfl(p) ldfl_raw(p)
-#define ldfq(p) ldfq_raw(p)
-#define stb(p, v) stb_raw(p, v)
-#define stw(p, v) stw_raw(p, v)
-#define stl(p, v) stl_raw(p, v)
-#define stq(p, v) stq_raw(p, v)
-#define stfl(p, v) stfl_raw(p, v)
-#define stfq(p, v) stfq_raw(p, v)
+/* In user-only mode we provide only the _code and _data accessors. */
 
-#define cpu_ldub_code(env1, p) ldub_raw(p)
-#define cpu_ldsb_code(env1, p) ldsb_raw(p)
-#define cpu_lduw_code(env1, p) lduw_raw(p)
-#define cpu_ldsw_code(env1, p) ldsw_raw(p)
-#define cpu_ldl_code(env1, p) ldl_raw(p)
-#define cpu_ldq_code(env1, p) ldq_raw(p)
+#define MEMSUFFIX _data
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_useronly_template.h"
 
-#define cpu_ldub_data(env, addr) ldub_raw(addr)
-#define cpu_lduw_data(env, addr) lduw_raw(addr)
-#define cpu_ldsw_data(env, addr) ldsw_raw(addr)
-#define cpu_ldl_data(env, addr) ldl_raw(addr)
-#define cpu_ldq_data(env, addr) ldq_raw(addr)
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_useronly_template.h"
 
-#define cpu_stb_data(env, addr, data) stb_raw(addr, data)
-#define cpu_stw_data(env, addr, data) stw_raw(addr, data)
-#define cpu_stl_data(env, addr, data) stl_raw(addr, data)
-#define cpu_stq_data(env, addr, data) stq_raw(addr, data)
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_useronly_template.h"
 
-#define cpu_ldub_kernel(env, addr) ldub_raw(addr)
-#define cpu_lduw_kernel(env, addr) lduw_raw(addr)
-#define cpu_ldsw_kernel(env, addr) ldsw_raw(addr)
-#define cpu_ldl_kernel(env, addr) ldl_raw(addr)
-#define cpu_ldq_kernel(env, addr) ldq_raw(addr)
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_useronly_template.h"
+#undef MEMSUFFIX
 
-#define cpu_stb_kernel(env, addr, data) stb_raw(addr, data)
-#define cpu_stw_kernel(env, addr, data) stw_raw(addr, data)
-#define cpu_stl_kernel(env, addr, data) stl_raw(addr, data)
-#define cpu_stq_kernel(env, addr, data) stq_raw(addr, data)
+#define MEMSUFFIX _code
+#define CODE_ACCESS
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_useronly_template.h"
 
-#define ldub_kernel(p) ldub_raw(p)
-#define ldsb_kernel(p) ldsb_raw(p)
-#define lduw_kernel(p) lduw_raw(p)
-#define ldsw_kernel(p) ldsw_raw(p)
-#define ldl_kernel(p) ldl_raw(p)
-#define ldq_kernel(p) ldq_raw(p)
-#define ldfl_kernel(p) ldfl_raw(p)
-#define ldfq_kernel(p) ldfq_raw(p)
-#define stb_kernel(p, v) stb_raw(p, v)
-#define stw_kernel(p, v) stw_raw(p, v)
-#define stl_kernel(p, v) stl_raw(p, v)
-#define stq_kernel(p, v) stq_raw(p, v)
-#define stfl_kernel(p, v) stfl_raw(p, v)
-#define stfq_kernel(p, vt) stfq_raw(p, v)
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_useronly_template.h"
 
-#define cpu_ldub_data(env, addr) ldub_raw(addr)
-#define cpu_lduw_data(env, addr) lduw_raw(addr)
-#define cpu_ldl_data(env, addr) ldl_raw(addr)
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_useronly_template.h"
 
-#define cpu_stb_data(env, addr, data) stb_raw(addr, data)
-#define cpu_stw_data(env, addr, data) stw_raw(addr, data)
-#define cpu_stl_data(env, addr, data) stl_raw(addr, data)
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_useronly_template.h"
+#undef MEMSUFFIX
+#undef CODE_ACCESS
 
 #else
 
-/* XXX: find something cleaner.
- * Furthermore, this is false for 64 bits targets
- */
-#define ldul_user       ldl_user
-#define ldul_kernel     ldl_kernel
-#define ldul_hypv       ldl_hypv
-#define ldul_executive  ldl_executive
-#define ldul_supervisor ldl_supervisor
-
 /* The memory helpers for tcg-generated code need tcg_target_long etc.  */
 #include "tcg.h"
 
-uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
-void helper_stb_mmu(CPUArchState *env, target_ulong addr,
-                    uint8_t val, int mmu_idx);
-void helper_stw_mmu(CPUArchState *env, target_ulong addr,
-                    uint16_t val, int mmu_idx);
-void helper_stl_mmu(CPUArchState *env, target_ulong addr,
-                    uint32_t val, int mmu_idx);
-void helper_stq_mmu(CPUArchState *env, target_ulong addr,
-                    uint64_t val, int mmu_idx);
-
-uint8_t helper_ldb_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
+#ifdef MMU_MODE0_SUFFIX
 #define CPU_MMU_INDEX 0
 #define MEMSUFFIX MMU_MODE0_SUFFIX
 #define DATA_SIZE 1
@@ -197,7 +129,9 @@
 #include "exec/cpu_ldst_template.h"
 #undef CPU_MMU_INDEX
 #undef MEMSUFFIX
+#endif
 
+#if (NB_MMU_MODES >= 2) && defined(MMU_MODE1_SUFFIX)
 #define CPU_MMU_INDEX 1
 #define MEMSUFFIX MMU_MODE1_SUFFIX
 #define DATA_SIZE 1
@@ -213,8 +147,9 @@
 #include "exec/cpu_ldst_template.h"
 #undef CPU_MMU_INDEX
 #undef MEMSUFFIX
+#endif
 
-#if (NB_MMU_MODES >= 3)
+#if (NB_MMU_MODES >= 3) && defined(MMU_MODE2_SUFFIX)
 
 #define CPU_MMU_INDEX 2
 #define MEMSUFFIX MMU_MODE2_SUFFIX
@@ -233,7 +168,7 @@
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 3) */
 
-#if (NB_MMU_MODES >= 4)
+#if (NB_MMU_MODES >= 4) && defined(MMU_MODE3_SUFFIX)
 
 #define CPU_MMU_INDEX 3
 #define MEMSUFFIX MMU_MODE3_SUFFIX
@@ -252,7 +187,7 @@
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 4) */
 
-#if (NB_MMU_MODES >= 5)
+#if (NB_MMU_MODES >= 5) && defined(MMU_MODE4_SUFFIX)
 
 #define CPU_MMU_INDEX 4
 #define MEMSUFFIX MMU_MODE4_SUFFIX
@@ -271,7 +206,7 @@
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 5) */
 
-#if (NB_MMU_MODES >= 6)
+#if (NB_MMU_MODES >= 6) && defined(MMU_MODE5_SUFFIX)
 
 #define CPU_MMU_INDEX 5
 #define MEMSUFFIX MMU_MODE5_SUFFIX
@@ -290,12 +225,126 @@
 #undef MEMSUFFIX
 #endif /* (NB_MMU_MODES >= 6) */
 
-#if (NB_MMU_MODES > 6)
-#error "NB_MMU_MODES > 6 is not supported for now"
-#endif /* (NB_MMU_MODES > 6) */
+#if (NB_MMU_MODES >= 7) && defined(MMU_MODE6_SUFFIX)
+
+#define CPU_MMU_INDEX 6
+#define MEMSUFFIX MMU_MODE6_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 7) */
+
+#if (NB_MMU_MODES >= 8) && defined(MMU_MODE7_SUFFIX)
+
+#define CPU_MMU_INDEX 7
+#define MEMSUFFIX MMU_MODE7_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 8) */
+
+#if (NB_MMU_MODES >= 9) && defined(MMU_MODE8_SUFFIX)
+
+#define CPU_MMU_INDEX 8
+#define MEMSUFFIX MMU_MODE8_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 9) */
+
+#if (NB_MMU_MODES >= 10) && defined(MMU_MODE9_SUFFIX)
+
+#define CPU_MMU_INDEX 9
+#define MEMSUFFIX MMU_MODE9_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 10) */
+
+#if (NB_MMU_MODES >= 11) && defined(MMU_MODE10_SUFFIX)
+
+#define CPU_MMU_INDEX 10
+#define MEMSUFFIX MMU_MODE10_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 11) */
+
+#if (NB_MMU_MODES >= 12) && defined(MMU_MODE11_SUFFIX)
+
+#define CPU_MMU_INDEX 11
+#define MEMSUFFIX MMU_MODE11_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 12) */
+
+#if (NB_MMU_MODES > 12)
+#error "NB_MMU_MODES > 12 is not supported for now"
+#endif /* (NB_MMU_MODES > 12) */
 
 /* these access are slower, they must be as rare as possible */
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define CPU_MMU_INDEX (cpu_mmu_index(env, false))
 #define MEMSUFFIX _data
 #define DATA_SIZE 1
 #include "exec/cpu_ldst_template.h"
@@ -311,19 +360,7 @@
 #undef CPU_MMU_INDEX
 #undef MEMSUFFIX
 
-#define ldub(p) ldub_data(p)
-#define ldsb(p) ldsb_data(p)
-#define lduw(p) lduw_data(p)
-#define ldsw(p) ldsw_data(p)
-#define ldl(p) ldl_data(p)
-#define ldq(p) ldq_data(p)
-
-#define stb(p, v) stb_data(p, v)
-#define stw(p, v) stw_data(p, v)
-#define stl(p, v) stl_data(p, v)
-#define stq(p, v) stq_data(p, v)
-
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define CPU_MMU_INDEX (cpu_mmu_index(env, true))
 #define MEMSUFFIX _code
 #define SOFTMMU_CODE_ACCESS
 
@@ -343,6 +380,8 @@
 #undef MEMSUFFIX
 #undef SOFTMMU_CODE_ACCESS
 
+#endif /* defined(CONFIG_USER_ONLY) */
+
 /**
  * tlb_vaddr_to_host:
  * @env: CPUArchState
@@ -361,6 +400,9 @@
 static inline void *tlb_vaddr_to_host(CPUArchState *env, target_ulong addr,
                                       int access_type, int mmu_idx)
 {
+#if defined(CONFIG_USER_ONLY)
+    return g2h(vaddr);
+#else
     int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index];
     target_ulong tlb_addr;
@@ -393,8 +435,7 @@
 
     haddr = addr + env->tlb_table[mmu_idx][index].addend;
     return (void *)haddr;
-}
-
 #endif /* defined(CONFIG_USER_ONLY) */
+}
 
 #endif /* CPU_LDST_H */
diff --git a/include/exec/cpu_ldst_template.h b/include/exec/cpu_ldst_template.h
index 006093a..eaf69a1 100644
--- a/include/exec/cpu_ldst_template.h
+++ b/include/exec/cpu_ldst_template.h
@@ -4,9 +4,7 @@
  * Generate inline load/store functions for one MMU mode and data
  * size.
  *
- * Generate a store function as well as signed and unsigned loads. For
- * 32 and 64 bit cases, also generate floating point functions with
- * the same size.
+ * Generate a store function as well as signed and unsigned loads.
  *
  * Not used directly but included from cpu_ldst.h.
  *
@@ -25,24 +23,35 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+
+#if !defined(SOFTMMU_CODE_ACCESS)
+#include "trace.h"
+#endif
+
+#include "trace/mem.h"
+
 #if DATA_SIZE == 8
 #define SUFFIX q
 #define USUFFIX q
 #define DATA_TYPE uint64_t
+#define SHIFT 3
 #elif DATA_SIZE == 4
 #define SUFFIX l
 #define USUFFIX l
 #define DATA_TYPE uint32_t
+#define SHIFT 2
 #elif DATA_SIZE == 2
 #define SUFFIX w
 #define USUFFIX uw
 #define DATA_TYPE uint16_t
 #define DATA_STYPE int16_t
+#define SHIFT 1
 #elif DATA_SIZE == 1
 #define SUFFIX b
 #define USUFFIX ub
 #define DATA_TYPE uint8_t
 #define DATA_STYPE int8_t
+#define SHIFT 0
 #else
 #error unsupported data size
 #endif
@@ -56,55 +65,92 @@
 #ifdef SOFTMMU_CODE_ACCESS
 #define ADDR_READ addr_code
 #define MMUSUFFIX _cmmu
+#define URETSUFFIX SUFFIX
+#define SRETSUFFIX SUFFIX
 #else
 #define ADDR_READ addr_read
 #define MMUSUFFIX _mmu
+#define URETSUFFIX USUFFIX
+#define SRETSUFFIX glue(s, SUFFIX)
 #endif
 
 /* generic load/store macros */
 
 static inline RES_TYPE
-glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
 {
     int page_index;
     RES_TYPE res;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
+
+#if !defined(SOFTMMU_CODE_ACCESS)
+    trace_guest_mem_before_exec(
+        ENV_GET_CPU(env), ptr,
+        trace_mem_build_info(SHIFT, false, MO_TE, false));
+#endif
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        res = glue(glue(helper_ret_ld, URETSUFFIX), MMUSUFFIX)(env, addr,
+                                                            oi, retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
-        res = glue(glue(ld, USUFFIX), _raw)(hostaddr);
+        res = glue(glue(ld, USUFFIX), _p)((uint8_t *)hostaddr);
     }
     return res;
 }
 
+static inline RES_TYPE
+glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+    return glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(env, ptr, 0);
+}
+
 #if DATA_SIZE <= 2
 static inline int
-glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
 {
     int res, page_index;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
+
+#if !defined(SOFTMMU_CODE_ACCESS)
+    trace_guest_mem_before_exec(
+        ENV_GET_CPU(env), ptr,
+        trace_mem_build_info(SHIFT, true, MO_TE, false));
+#endif
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = (DATA_STYPE)glue(glue(helper_ld, SUFFIX),
-                               MMUSUFFIX)(env, addr, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        res = (DATA_STYPE)glue(glue(helper_ret_ld, SRETSUFFIX),
+                               MMUSUFFIX)(env, addr, oi, retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
-        res = glue(glue(lds, SUFFIX), _raw)(hostaddr);
+        res = glue(glue(lds, SUFFIX), _p)((uint8_t *)hostaddr);
     }
     return res;
 }
+
+static inline int
+glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+    return glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(env, ptr, 0);
+}
 #endif
 
 #ifndef SOFTMMU_CODE_ACCESS
@@ -112,75 +158,42 @@
 /* generic store macro */
 
 static inline void
-glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
-                                      RES_TYPE v)
+glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                 target_ulong ptr,
+                                                 RES_TYPE v, uintptr_t retaddr)
 {
     int page_index;
     target_ulong addr;
     int mmu_idx;
+    TCGMemOpIdx oi;
+
+#if !defined(SOFTMMU_CODE_ACCESS)
+    trace_guest_mem_before_exec(
+        ENV_GET_CPU(env), ptr,
+        trace_mem_build_info(SHIFT, false, MO_TE, true));
+#endif
 
     addr = ptr;
     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        glue(glue(helper_st, SUFFIX), MMUSUFFIX)(env, addr, v, mmu_idx);
+        oi = make_memop_idx(SHIFT, mmu_idx);
+        glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi,
+                                                     retaddr);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
-        glue(glue(st, SUFFIX), _raw)(hostaddr, v);
+        glue(glue(st, SUFFIX), _p)((uint8_t *)hostaddr, v);
     }
 }
 
-
-
-#if DATA_SIZE == 8
-static inline float64 glue(cpu_ldfq, MEMSUFFIX)(CPUArchState *env,
-                                                target_ulong ptr)
+static inline void
+glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
+                                      RES_TYPE v)
 {
-    union {
-        float64 d;
-        uint64_t i;
-    } u;
-    u.i = glue(cpu_ldq, MEMSUFFIX)(env, ptr);
-    return u.d;
+    glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(env, ptr, v, 0);
 }
 
-static inline void glue(cpu_stfq, MEMSUFFIX)(CPUArchState *env,
-                                             target_ulong ptr, float64 v)
-{
-    union {
-        float64 d;
-        uint64_t i;
-    } u;
-    u.d = v;
-    glue(cpu_stq, MEMSUFFIX)(env, ptr, u.i);
-}
-#endif /* DATA_SIZE == 8 */
-
-#if DATA_SIZE == 4
-static inline float32 glue(cpu_ldfl, MEMSUFFIX)(CPUArchState *env,
-                                                target_ulong ptr)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.i = glue(cpu_ldl, MEMSUFFIX)(env, ptr);
-    return u.f;
-}
-
-static inline void glue(cpu_stfl, MEMSUFFIX)(CPUArchState *env,
-                                             target_ulong ptr, float32 v)
-{
-    union {
-        float32 f;
-        uint32_t i;
-    } u;
-    u.f = v;
-    glue(cpu_stl, MEMSUFFIX)(env, ptr, u.i);
-}
-#endif /* DATA_SIZE == 4 */
-
 #endif /* !SOFTMMU_CODE_ACCESS */
 
 #undef RES_TYPE
@@ -191,3 +204,6 @@
 #undef DATA_SIZE
 #undef MMUSUFFIX
 #undef ADDR_READ
+#undef URETSUFFIX
+#undef SRETSUFFIX
+#undef SHIFT
diff --git a/include/exec/cpu_ldst_useronly_template.h b/include/exec/cpu_ldst_useronly_template.h
new file mode 100644
index 0000000..b1378bf
--- /dev/null
+++ b/include/exec/cpu_ldst_useronly_template.h
@@ -0,0 +1,128 @@
+/*
+ *  User-only accessor function support
+ *
+ * Generate inline load/store functions for one data size.
+ *
+ * Generate a store function as well as signed and unsigned loads.
+ *
+ * Not used directly but included from cpu_ldst.h.
+ *
+ *  Copyright (c) 2015 Linaro Limited
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#if !defined(CODE_ACCESS)
+#include "trace.h"
+#endif
+
+#include "trace/mem.h"
+
+#if DATA_SIZE == 8
+#define SUFFIX q
+#define USUFFIX q
+#define DATA_TYPE uint64_t
+#elif DATA_SIZE == 4
+#define SUFFIX l
+#define USUFFIX l
+#define DATA_TYPE uint32_t
+#elif DATA_SIZE == 2
+#define SUFFIX w
+#define USUFFIX uw
+#define DATA_TYPE uint16_t
+#define DATA_STYPE int16_t
+#elif DATA_SIZE == 1
+#define SUFFIX b
+#define USUFFIX ub
+#define DATA_TYPE uint8_t
+#define DATA_STYPE int8_t
+#else
+#error unsupported data size
+#endif
+
+#if DATA_SIZE == 8
+#define RES_TYPE uint64_t
+#else
+#define RES_TYPE uint32_t
+#endif
+
+static inline RES_TYPE
+glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+#if !defined(CODE_ACCESS)
+    trace_guest_mem_before_exec(
+        ENV_GET_CPU(env), ptr,
+        trace_mem_build_info(DATA_SIZE, false, MO_TE, false));
+#endif
+    return glue(glue(ld, USUFFIX), _p)(g2h(ptr));
+}
+
+static inline RES_TYPE
+glue(glue(glue(cpu_ld, USUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
+{
+    return glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(env, ptr);
+}
+
+#if DATA_SIZE <= 2
+static inline int
+glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
+{
+#if !defined(CODE_ACCESS)
+    trace_guest_mem_before_exec(
+        ENV_GET_CPU(env), ptr,
+        trace_mem_build_info(DATA_SIZE, true, MO_TE, false));
+#endif
+    return glue(glue(lds, SUFFIX), _p)(g2h(ptr));
+}
+
+static inline int
+glue(glue(glue(cpu_lds, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  uintptr_t retaddr)
+{
+    return glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(env, ptr);
+}
+#endif
+
+#ifndef CODE_ACCESS
+static inline void
+glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
+                                      RES_TYPE v)
+{
+#if !defined(CODE_ACCESS)
+    trace_guest_mem_before_exec(
+        ENV_GET_CPU(env), ptr,
+        trace_mem_build_info(DATA_SIZE, false, MO_TE, true));
+#endif
+    glue(glue(st, SUFFIX), _p)(g2h(ptr), v);
+}
+
+static inline void
+glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env,
+                                                  target_ulong ptr,
+                                                  RES_TYPE v,
+                                                  uintptr_t retaddr)
+{
+    glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(env, ptr, v);
+}
+#endif
+
+#undef RES_TYPE
+#undef DATA_TYPE
+#undef DATA_STYPE
+#undef SUFFIX
+#undef USUFFIX
+#undef DATA_SIZE
diff --git a/include/exec/cputlb.h b/include/exec/cputlb.h
index b8ecd6f..d454c00 100644
--- a/include/exec/cputlb.h
+++ b/include/exec/cputlb.h
@@ -22,27 +22,10 @@
 #if !defined(CONFIG_USER_ONLY)
 /* cputlb.c */
 void tlb_protect_code(ram_addr_t ram_addr);
-void tlb_unprotect_code_phys(CPUState *cpu, ram_addr_t ram_addr,
-                             target_ulong vaddr);
+void tlb_unprotect_code(ram_addr_t ram_addr);
 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
                            uintptr_t length);
-void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);
-void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);
 extern int tlb_flush_count;
 
-/* exec.c */
-void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
-
-MemoryRegionSection *
-address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
-                                  hwaddr *plen);
-hwaddr memory_region_section_get_iotlb(CPUState *cpu,
-                                       MemoryRegionSection *section,
-                                       target_ulong vaddr,
-                                       hwaddr paddr, hwaddr xlat,
-                                       int prot,
-                                       target_ulong *address);
-bool memory_region_is_unassigned(MemoryRegion *mr);
-
 #endif
 #endif
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 9954e2b..d008296 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -17,10 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef _EXEC_ALL_H_
-#define _EXEC_ALL_H_
+#ifndef EXEC_ALL_H
+#define EXEC_ALL_H
 
 #include "qemu-common.h"
+#include "exec/tb-context.h"
 
 /* allow to see translation results - the slowdown should be negligible, so we leave it */
 #define DEBUG_DISAS
@@ -40,66 +41,130 @@
 #define DISAS_UPDATE  2 /* cpu state was modified dynamically */
 #define DISAS_TB_JUMP 3 /* only pc was modified statically */
 
-struct TranslationBlock;
-typedef struct TranslationBlock TranslationBlock;
-
-/* XXX: make safe guess about sizes */
-#define MAX_OP_PER_INSTR 266
-
-#if HOST_LONG_BITS == 32
-#define MAX_OPC_PARAM_PER_ARG 2
-#else
-#define MAX_OPC_PARAM_PER_ARG 1
-#endif
-#define MAX_OPC_PARAM_IARGS 5
-#define MAX_OPC_PARAM_OARGS 1
-#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
-
-/* A Call op needs up to 4 + 2N parameters on 32-bit archs,
- * and up to 4 + N parameters on 64-bit archs
- * (N = number of input arguments + output arguments).  */
-#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
-#define OPC_BUF_SIZE 640
-#define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
-
-/* Maximum size a TCG op can expand to.  This is complicated because a
-   single op may require several host instructions and register reloads.
-   For now take a wild guess at 192 bytes, which should allow at least
-   a couple of fixup instructions per argument.  */
-#define TCG_MAX_OP_SIZE 192
-
-#define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
-
 #include "qemu/log.h"
 
 void gen_intermediate_code(CPUArchState *env, struct TranslationBlock *tb);
-void gen_intermediate_code_pc(CPUArchState *env, struct TranslationBlock *tb);
 void restore_state_to_opc(CPUArchState *env, struct TranslationBlock *tb,
-                          int pc_pos);
+                          target_ulong *data);
 
 void cpu_gen_init(void);
-int cpu_gen_code(CPUArchState *env, struct TranslationBlock *tb,
-                 int *gen_code_size_ptr);
 bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc);
-void page_size_init(void);
 
-void QEMU_NORETURN cpu_resume_from_signal(CPUState *cpu, void *puc);
+void QEMU_NORETURN cpu_loop_exit_noexc(CPUState *cpu);
 void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
 TranslationBlock *tb_gen_code(CPUState *cpu,
-                              target_ulong pc, target_ulong cs_base, int flags,
+                              target_ulong pc, target_ulong cs_base,
+                              uint32_t flags,
                               int cflags);
-void cpu_exec_init(CPUArchState *env);
+#if defined(CONFIG_USER_ONLY)
+void cpu_list_lock(void);
+void cpu_list_unlock(void);
+#else
+static inline void cpu_list_unlock(void)
+{
+}
+static inline void cpu_list_lock(void)
+{
+}
+#endif
+
+void cpu_exec_init(CPUState *cpu, Error **errp);
 void QEMU_NORETURN cpu_loop_exit(CPUState *cpu);
-int page_unprotect(target_ulong address, uintptr_t pc, void *puc);
-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
-                                   int is_cpu_write_access);
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
-                              int is_cpu_write_access);
+void QEMU_NORETURN cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc);
+
 #if !defined(CONFIG_USER_ONLY)
-void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as);
+void cpu_reloading_memory_map(void);
+/**
+ * cpu_address_space_init:
+ * @cpu: CPU to add this address space to
+ * @as: address space to add
+ * @asidx: integer index of this address space
+ *
+ * Add the specified address space to the CPU's cpu_ases list.
+ * The address space added with @asidx 0 is the one used for the
+ * convenience pointer cpu->as.
+ * The target-specific code which registers ASes is responsible
+ * for defining what semantics address space 0, 1, 2, etc have.
+ *
+ * Before the first call to this function, the caller must set
+ * cpu->num_ases to the total number of address spaces it needs
+ * to support.
+ *
+ * Note that with KVM only one address space is supported.
+ */
+void cpu_address_space_init(CPUState *cpu, AddressSpace *as, int asidx);
 /* cputlb.c */
+/**
+ * tlb_flush_page:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of page to be flushed
+ *
+ * Flush one page from the TLB of the specified CPU, for all
+ * MMU indexes.
+ */
 void tlb_flush_page(CPUState *cpu, target_ulong addr);
+/**
+ * tlb_flush:
+ * @cpu: CPU whose TLB should be flushed
+ * @flush_global: ignored
+ *
+ * Flush the entire TLB for the specified CPU.
+ * The flush_global flag is in theory an indicator of whether the whole
+ * TLB should be flushed, or only those entries not marked global.
+ * In practice QEMU does not implement any global/not global flag for
+ * TLB entries, and the argument is ignored.
+ */
 void tlb_flush(CPUState *cpu, int flush_global);
+/**
+ * tlb_flush_page_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of page to be flushed
+ * @...: list of MMU indexes to flush, terminated by a negative value
+ *
+ * Flush one page from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...);
+/**
+ * tlb_flush_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @...: list of MMU indexes to flush, terminated by a negative value
+ *
+ * Flush all entries from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_by_mmuidx(CPUState *cpu, ...);
+/**
+ * tlb_set_page_with_attrs:
+ * @cpu: CPU to add this TLB entry for
+ * @vaddr: virtual address of page to add entry for
+ * @paddr: physical address of the page
+ * @attrs: memory transaction attributes
+ * @prot: access permissions (PAGE_READ/PAGE_WRITE/PAGE_EXEC bits)
+ * @mmu_idx: MMU index to insert TLB entry for
+ * @size: size of the page in bytes
+ *
+ * Add an entry to this CPU's TLB (a mapping from virtual address
+ * @vaddr to physical address @paddr) with the specified memory
+ * transaction attributes. This is generally called by the target CPU
+ * specific code after it has been called through the tlb_fill()
+ * entry point and performed a successful page table walk to find
+ * the physical address and attributes for the virtual address
+ * which provoked the TLB miss.
+ *
+ * At most one entry for a given virtual address is permitted. Only a
+ * single TARGET_PAGE_SIZE region is mapped; the supplied @size is only
+ * used by tlb_flush_page.
+ */
+void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
+                             hwaddr paddr, MemTxAttrs attrs,
+                             int prot, int mmu_idx, target_ulong size);
+/* tlb_set_page:
+ *
+ * This function is equivalent to calling tlb_set_page_with_attrs()
+ * with an @attrs argument of MEMTXATTRS_UNSPECIFIED. It's provided
+ * as a convenience for CPUs which don't use memory transaction attributes.
+ */
 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
                   hwaddr paddr, int prot,
                   int mmu_idx, target_ulong size);
@@ -114,20 +179,27 @@
 static inline void tlb_flush(CPUState *cpu, int flush_global)
 {
 }
+
+static inline void tlb_flush_page_by_mmuidx(CPUState *cpu,
+                                            target_ulong addr, ...)
+{
+}
+
+static inline void tlb_flush_by_mmuidx(CPUState *cpu, ...)
+{
+}
 #endif
 
 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
 
-#define CODE_GEN_PHYS_HASH_BITS     15
-#define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
-
-/* estimated block size for TB allocation */
-/* XXX: use a per code average code fragment size and modulate it
-   according to the host CPU */
+/* Estimated block size for TB allocation.  */
+/* ??? The following is based on a 2015 survey of x86_64 host output.
+   Better would seem to be some sort of dynamically sized TB array,
+   adapting to the block sizes actually being produced.  */
 #if defined(CONFIG_SOFTMMU)
-#define CODE_GEN_AVG_BLOCK_SIZE 128
+#define CODE_GEN_AVG_BLOCK_SIZE 400
 #else
-#define CODE_GEN_AVG_BLOCK_SIZE 64
+#define CODE_GEN_AVG_BLOCK_SIZE 150
 #endif
 
 #if defined(__arm__) || defined(_ARCH_PPC) \
@@ -135,85 +207,65 @@
     || defined(__sparc__) || defined(__aarch64__) \
     || defined(__s390x__) || defined(__mips__) \
     || defined(CONFIG_TCG_INTERPRETER)
+/* NOTE: Direct jump patching must be atomic to be thread-safe. */
 #define USE_DIRECT_JUMP
 #endif
 
 struct TranslationBlock {
     target_ulong pc;   /* simulated PC corresponding to this block (EIP + CS base) */
     target_ulong cs_base; /* CS base for this block */
-    uint64_t flags; /* flags defining in which context the code was generated */
+    uint32_t flags; /* flags defining in which context the code was generated */
     uint16_t size;      /* size of target code for this block (1 <=
                            size <= TARGET_PAGE_SIZE) */
-    uint16_t cflags;    /* compile flags */
+    uint16_t icount;
+    uint32_t cflags;    /* compile flags */
 #define CF_COUNT_MASK  0x7fff
 #define CF_LAST_IO     0x8000 /* Last insn may be an IO access.  */
+#define CF_NOCACHE     0x10000 /* To be freed after execution */
+#define CF_USE_ICOUNT  0x20000
+#define CF_IGNORE_ICOUNT 0x40000 /* Do not generate icount code */
 
     void *tc_ptr;    /* pointer to the translated code */
-    /* next matching tb for physical address. */
-    struct TranslationBlock *phys_hash_next;
+    uint8_t *tc_search;  /* pointer to search data */
+    /* original tb when cflags has CF_NOCACHE */
+    struct TranslationBlock *orig_tb;
     /* first and second physical page containing code. The lower bit
        of the pointer tells the index in page_next[] */
     struct TranslationBlock *page_next[2];
     tb_page_addr_t page_addr[2];
 
-    /* the following data are used to directly call another TB from
-       the code of this one. */
-    uint16_t tb_next_offset[2]; /* offset of original jump target */
+    /* The following data are used to directly call another TB from
+     * the code of this one. This can be done either by emitting direct or
+     * indirect native jump instructions. These jumps are reset so that the TB
+     * just continue its execution. The TB can be linked to another one by
+     * setting one of the jump targets (or patching the jump instruction). Only
+     * two of such jumps are supported.
+     */
+    uint16_t jmp_reset_offset[2]; /* offset of original jump target */
+#define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */
 #ifdef USE_DIRECT_JUMP
-    uint16_t tb_jmp_offset[2]; /* offset of jump instruction */
+    uint16_t jmp_insn_offset[2]; /* offset of native jump instruction */
 #else
-    uintptr_t tb_next[2]; /* address of jump generated code */
+    uintptr_t jmp_target_addr[2]; /* target address for indirect jump */
 #endif
-    /* list of TBs jumping to this one. This is a circular list using
-       the two least significant bits of the pointers to tell what is
-       the next pointer: 0 = jmp_next[0], 1 = jmp_next[1], 2 =
-       jmp_first */
-    struct TranslationBlock *jmp_next[2];
-    struct TranslationBlock *jmp_first;
-    uint32_t icount;
+    /* Each TB has an assosiated circular list of TBs jumping to this one.
+     * jmp_list_first points to the first TB jumping to this one.
+     * jmp_list_next is used to point to the next TB in a list.
+     * Since each TB can have two jumps, it can participate in two lists.
+     * jmp_list_first and jmp_list_next are 4-byte aligned pointers to a
+     * TranslationBlock structure, but the two least significant bits of
+     * them are used to encode which data field of the pointed TB should
+     * be used to traverse the list further from that TB:
+     * 0 => jmp_list_next[0], 1 => jmp_list_next[1], 2 => jmp_list_first.
+     * In other words, 0/1 tells which jump is used in the pointed TB,
+     * and 2 means that this is a pointer back to the target TB of this list.
+     */
+    uintptr_t jmp_list_next[2];
+    uintptr_t jmp_list_first;
 };
 
-#include "exec/spinlock.h"
-
-typedef struct TBContext TBContext;
-
-struct TBContext {
-
-    TranslationBlock *tbs;
-    TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
-    int nb_tbs;
-    /* any access to the tbs or the page table must use this lock */
-    spinlock_t tb_lock;
-
-    /* statistics */
-    int tb_flush_count;
-    int tb_phys_invalidate_count;
-
-    int tb_invalidated_flag;
-};
-
-static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc)
-{
-    target_ulong tmp;
-    tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
-    return (tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK;
-}
-
-static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
-{
-    target_ulong tmp;
-    tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
-    return (((tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK)
-	    | (tmp & TB_JMP_ADDR_MASK));
-}
-
-static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc)
-{
-    return (pc >> 2) & (CODE_GEN_PHYS_HASH_SIZE - 1);
-}
-
 void tb_free(TranslationBlock *tb);
-void tb_flush(CPUArchState *env);
+void tb_flush(CPUState *cpu);
 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
 
 #if defined(USE_DIRECT_JUMP)
@@ -222,7 +274,7 @@
 static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
 {
     /* patch the branch destination */
-    *(uint32_t *)jmp_addr = addr - (jmp_addr + 4);
+    atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
     /* no need to flush icache explicitly */
 }
 #elif defined(_ARCH_PPC)
@@ -232,7 +284,7 @@
 static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
 {
     /* patch the branch destination */
-    stl_le_p((void*)jmp_addr, addr - (jmp_addr + 4));
+    atomic_set((int32_t *)jmp_addr, addr - (jmp_addr + 4));
     /* no need to flush icache explicitly */
 }
 #elif defined(__s390x__)
@@ -240,36 +292,15 @@
 {
     /* patch the branch destination */
     intptr_t disp = addr - (jmp_addr - 2);
-    stl_be_p((void*)jmp_addr, disp / 2);
+    atomic_set((int32_t *)jmp_addr, disp / 2);
     /* no need to flush icache explicitly */
 }
 #elif defined(__aarch64__)
 void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr);
 #define tb_set_jmp_target1 aarch64_tb_set_jmp_target
 #elif defined(__arm__)
-static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
-{
-#if !QEMU_GNUC_PREREQ(4, 1)
-    register unsigned long _beg __asm ("a1");
-    register unsigned long _end __asm ("a2");
-    register unsigned long _flg __asm ("a3");
-#endif
-
-    /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */
-    *(uint32_t *)jmp_addr =
-        (*(uint32_t *)jmp_addr & ~0xffffff)
-        | (((addr - (jmp_addr + 8)) >> 2) & 0xffffff);
-
-#if QEMU_GNUC_PREREQ(4, 1)
-    __builtin___clear_cache((char *) jmp_addr, (char *) jmp_addr + 4);
-#else
-    /* flush icache */
-    _beg = jmp_addr;
-    _end = jmp_addr + 4;
-    _flg = 0;
-    __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
-#endif
-}
+void arm_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr);
+#define tb_set_jmp_target1 arm_tb_set_jmp_target
 #elif defined(__sparc__) || defined(__mips__)
 void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr);
 #else
@@ -279,7 +310,7 @@
 static inline void tb_set_jmp_target(TranslationBlock *tb,
                                      int n, uintptr_t addr)
 {
-    uint16_t offset = tb->tb_jmp_offset[n];
+    uint16_t offset = tb->jmp_insn_offset[n];
     tb_set_jmp_target1((uintptr_t)(tb->tc_ptr + offset), addr);
 }
 
@@ -289,7 +320,7 @@
 static inline void tb_set_jmp_target(TranslationBlock *tb,
                                      int n, uintptr_t addr)
 {
-    tb->tb_next[n] = addr;
+    tb->jmp_target_addr[n] = addr;
 }
 
 #endif
@@ -297,15 +328,23 @@
 static inline void tb_add_jump(TranslationBlock *tb, int n,
                                TranslationBlock *tb_next)
 {
-    /* NOTE: this test is only needed for thread safety */
-    if (!tb->jmp_next[n]) {
-        /* patch the native jump address */
-        tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr);
-
-        /* add in TB jmp circular list */
-        tb->jmp_next[n] = tb_next->jmp_first;
-        tb_next->jmp_first = (TranslationBlock *)((uintptr_t)(tb) | (n));
+    if (tb->jmp_list_next[n]) {
+        /* Another thread has already done this while we were
+         * outside of the lock; nothing to do in this case */
+        return;
     }
+    qemu_log_mask_and_addr(CPU_LOG_EXEC, tb->pc,
+                           "Linking TBs %p [" TARGET_FMT_lx
+                           "] index %d -> %p [" TARGET_FMT_lx "]\n",
+                           tb->tc_ptr, tb->pc, n,
+                           tb_next->tc_ptr, tb_next->pc);
+
+    /* patch the native jump address */
+    tb_set_jmp_target(tb, n, (uintptr_t)tb_next->tc_ptr);
+
+    /* add in TB jmp circular list */
+    tb->jmp_list_next[n] = tb_next->jmp_list_first;
+    tb_next->jmp_list_first = (uintptr_t)tb | n;
 }
 
 /* GETRA is the true target of the return instruction that we'll execute,
@@ -325,64 +364,59 @@
    to indicate the compressed mode; subtracting two works around that.  It
    is also the case that there are no host isas that contain a call insn
    smaller than 4 bytes, so we don't worry about special-casing this.  */
-#if defined(CONFIG_TCG_INTERPRETER)
-# define GETPC_ADJ   0
-#else
-# define GETPC_ADJ   2
-#endif
+#define GETPC_ADJ   2
 
 #define GETPC()  (GETRA() - GETPC_ADJ)
 
 #if !defined(CONFIG_USER_ONLY)
 
-void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
+struct MemoryRegion *iotlb_to_region(CPUState *cpu,
+                                     hwaddr index, MemTxAttrs attrs);
 
-struct MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index);
-bool io_mem_read(struct MemoryRegion *mr, hwaddr addr,
-                 uint64_t *pvalue, unsigned size);
-bool io_mem_write(struct MemoryRegion *mr, hwaddr addr,
-                  uint64_t value, unsigned size);
-
-void tlb_fill(CPUState *cpu, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr);
+void tlb_fill(CPUState *cpu, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr);
 
 #endif
 
 #if defined(CONFIG_USER_ONLY)
+void mmap_lock(void);
+void mmap_unlock(void);
+
 static inline tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 {
     return addr;
 }
 #else
+static inline void mmap_lock(void) {}
+static inline void mmap_unlock(void) {}
+
 /* cputlb.c */
 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr);
+
+void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length);
+void tlb_set_dirty(CPUState *cpu, target_ulong vaddr);
+
+/* exec.c */
+void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
+
+MemoryRegionSection *
+address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
+                                  hwaddr *xlat, hwaddr *plen);
+hwaddr memory_region_section_get_iotlb(CPUState *cpu,
+                                       MemoryRegionSection *section,
+                                       target_ulong vaddr,
+                                       hwaddr paddr, hwaddr xlat,
+                                       int prot,
+                                       target_ulong *address);
+bool memory_region_is_unassigned(MemoryRegion *mr);
+
 #endif
 
 /* vl.c */
 extern int singlestep;
 
-/* cpu-exec.c */
-extern volatile sig_atomic_t exit_request;
-
-/**
- * cpu_can_do_io:
- * @cpu: The CPU for which to check IO.
- *
- * Deterministic execution requires that IO only be performed on the last
- * instruction of a TB so that interrupts take effect immediately.
- *
- * Returns: %true if memory-mapped IO is safe, %false otherwise.
- */
-static inline bool cpu_can_do_io(CPUState *cpu)
-{
-    if (!use_icount) {
-        return true;
-    }
-    /* If not executing code then assume we are ok.  */
-    if (cpu->current_tb == NULL) {
-        return true;
-    }
-    return cpu->can_do_io != 0;
-}
+/* cpu-exec.c, accessed with atomic_mb_read/atomic_mb_set */
+extern CPUState *tcg_current_cpu;
+extern bool exit_request;
 
 #endif
diff --git a/include/exec/gdbstub.h b/include/exec/gdbstub.h
index a608a26..f9708bb 100644
--- a/include/exec/gdbstub.h
+++ b/include/exec/gdbstub.h
@@ -11,18 +11,46 @@
 #define GDB_WATCHPOINT_ACCESS    4
 
 #ifdef NEED_CPU_H
+#include "cpu.h"
+
 typedef void (*gdb_syscall_complete_cb)(CPUState *cpu,
                                         target_ulong ret, target_ulong err);
 
+/**
+ * gdb_do_syscall:
+ * @cb: function to call when the system call has completed
+ * @fmt: gdb syscall format string
+ * ...: list of arguments to interpolate into @fmt
+ *
+ * Send a GDB syscall request. This function will return immediately;
+ * the callback function will be called later when the remote system
+ * call has completed.
+ *
+ * @fmt should be in the 'call-id,parameter,parameter...' format documented
+ * for the F request packet in the GDB remote protocol. A limited set of
+ * printf-style format specifiers is supported:
+ *   %x  - target_ulong argument printed in hex
+ *   %lx - 64-bit argument printed in hex
+ *   %s  - string pointer (target_ulong) and length (int) pair
+ */
 void gdb_do_syscall(gdb_syscall_complete_cb cb, const char *fmt, ...);
+/**
+ * gdb_do_syscallv:
+ * @cb: function to call when the system call has completed
+ * @fmt: gdb syscall format string
+ * @va: arguments to interpolate into @fmt
+ *
+ * As gdb_do_syscall, but taking a va_list rather than a variable
+ * argument list.
+ */
+void gdb_do_syscallv(gdb_syscall_complete_cb cb, const char *fmt, va_list va);
 int use_gdb_syscalls(void);
 void gdb_set_stop_cpu(CPUState *cpu);
 void gdb_exit(CPUArchState *, int);
 #ifdef CONFIG_USER_ONLY
-int gdb_queuesig (void);
 int gdb_handlesig(CPUState *, int);
 void gdb_signalled(CPUArchState *, int);
-void gdbserver_fork(CPUArchState *);
+void gdbserver_fork(CPUState *);
 #endif
 /* Get or set a register.  Returns the size of the register.  */
 typedef int (*gdb_reg_cb)(CPUArchState *env, uint8_t *buf, int reg);
diff --git a/include/exec/gen-icount.h b/include/exec/gen-icount.h
index 79af464..050de59 100644
--- a/include/exec/gen-icount.h
+++ b/include/exec/gen-icount.h
@@ -1,18 +1,17 @@
 #ifndef GEN_ICOUNT_H
-#define GEN_ICOUNT_H 1
+#define GEN_ICOUNT_H
 
 #include "qemu/timer.h"
 
 /* Helpers for instruction counting code generation.  */
 
-static TCGArg *icount_arg;
-static int icount_label;
-static int exitreq_label;
+static int icount_start_insn_idx;
+static TCGLabel *icount_label;
+static TCGLabel *exitreq_label;
 
-static inline void gen_tb_start(void)
+static inline void gen_tb_start(TranslationBlock *tb)
 {
     TCGv_i32 count, flag, imm;
-    int i;
 
     exitreq_label = gen_new_label();
     flag = tcg_temp_new_i32();
@@ -21,8 +20,9 @@
     tcg_gen_brcondi_i32(TCG_COND_NE, flag, 0, exitreq_label);
     tcg_temp_free_i32(flag);
 
-    if (!use_icount)
+    if (!(tb->cflags & CF_USE_ICOUNT)) {
         return;
+    }
 
     icount_label = gen_new_label();
     count = tcg_temp_local_new_i32();
@@ -30,13 +30,12 @@
                    -ENV_OFFSET + offsetof(CPUState, icount_decr.u32));
 
     imm = tcg_temp_new_i32();
+    /* We emit a movi with a dummy immediate argument. Keep the insn index
+     * of the movi so that we later (when we know the actual insn count)
+     * can update the immediate argument with the actual insn count.  */
+    icount_start_insn_idx = tcg_op_buf_count();
     tcg_gen_movi_i32(imm, 0xdeadbeef);
 
-    /* This is a horrid hack to allow fixing up the value later.  */
-    i = tcg_ctx.gen_last_op_idx;
-    i = tcg_ctx.gen_op_buf[i].args;
-    icount_arg = &tcg_ctx.gen_opparam_buf[i + 1];
-
     tcg_gen_sub_i32(count, count, imm);
     tcg_temp_free_i32(imm);
 
@@ -51,14 +50,16 @@
     gen_set_label(exitreq_label);
     tcg_gen_exit_tb((uintptr_t)tb + TB_EXIT_REQUESTED);
 
-    if (use_icount) {
-        *icount_arg = num_insns;
+    if (tb->cflags & CF_USE_ICOUNT) {
+        /* Update the num_insn immediate parameter now that we know
+         * the actual insn count.  */
+        tcg_set_insn_param(icount_start_insn_idx, 1, num_insns);
         gen_set_label(icount_label);
         tcg_gen_exit_tb((uintptr_t)tb + TB_EXIT_ICOUNT_EXPIRED);
     }
 
     /* Terminate the linked list.  */
-    tcg_ctx.gen_op_buf[tcg_ctx.gen_last_op_idx].next = -1;
+    tcg_ctx.gen_op_buf[tcg_ctx.gen_op_buf[0].prev].next = 0;
 }
 
 static inline void gen_io_start(void)
diff --git a/include/exec/helper-gen.h b/include/exec/helper-gen.h
index 0d0da3a..8239ffc 100644
--- a/include/exec/helper-gen.h
+++ b/include/exec/helper-gen.h
@@ -2,9 +2,9 @@
    This one expands generation functions for tcg opcodes.  */
 
 #ifndef HELPER_GEN_H
-#define HELPER_GEN_H 1
+#define HELPER_GEN_H
 
-#include <exec/helper-head.h>
+#include "exec/helper-head.h"
 
 #define DEF_HELPER_FLAGS_0(name, flags, ret)                            \
 static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret))        \
diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h
index b009ccb..1cfc43b 100644
--- a/include/exec/helper-head.h
+++ b/include/exec/helper-head.h
@@ -15,10 +15,8 @@
     GEN_HELPER 2 to do runtime registration helper functions.
  */
 
-#ifndef DEF_HELPER_H
-#define DEF_HELPER_H 1
-
-#include "qemu/osdep.h"
+#ifndef EXEC_HELPER_HEAD_H
+#define EXEC_HELPER_HEAD_H
 
 #define HELPER(name) glue(helper_, name)
 
@@ -34,17 +32,9 @@
 #define dh_alias_s64 i64
 #define dh_alias_f32 i32
 #define dh_alias_f64 i64
-#ifdef TARGET_LONG_BITS
-# if TARGET_LONG_BITS == 32
-#  define dh_alias_tl i32
-# else
-#  define dh_alias_tl i64
-# endif
-#endif
 #define dh_alias_ptr ptr
 #define dh_alias_void void
 #define dh_alias_noreturn noreturn
-#define dh_alias_env ptr
 #define dh_alias(t) glue(dh_alias_, t)
 
 #define dh_ctype_i32 uint32_t
@@ -54,13 +44,24 @@
 #define dh_ctype_s64 int64_t
 #define dh_ctype_f32 float32
 #define dh_ctype_f64 float64
-#define dh_ctype_tl target_ulong
 #define dh_ctype_ptr void *
 #define dh_ctype_void void
 #define dh_ctype_noreturn void QEMU_NORETURN
-#define dh_ctype_env CPUArchState *
 #define dh_ctype(t) dh_ctype_##t
 
+#ifdef NEED_CPU_H
+# ifdef TARGET_LONG_BITS
+#  if TARGET_LONG_BITS == 32
+#   define dh_alias_tl i32
+#  else
+#   define dh_alias_tl i64
+#  endif
+# endif
+# define dh_alias_env ptr
+# define dh_ctype_tl target_ulong
+# define dh_ctype_env CPUArchState *
+#endif
+
 /* We can't use glue() here because it falls foul of C preprocessor
    recursive expansion rules.  */
 #define dh_retvar_decl0_void void
@@ -131,4 +132,4 @@
 
 /* MAX_OPC_PARAM_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */
 
-#endif /* DEF_HELPER_H */
+#endif /* EXEC_HELPER_HEAD_H */
diff --git a/include/exec/helper-proto.h b/include/exec/helper-proto.h
index effdd43..954bef8 100644
--- a/include/exec/helper-proto.h
+++ b/include/exec/helper-proto.h
@@ -2,9 +2,9 @@
    This one expands prototypes for the helper functions.  */
 
 #ifndef HELPER_PROTO_H
-#define HELPER_PROTO_H 1
+#define HELPER_PROTO_H
 
-#include <exec/helper-head.h>
+#include "exec/helper-head.h"
 
 #define DEF_HELPER_FLAGS_0(name, flags, ret) \
 dh_ctype(ret) HELPER(name) (void);
diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h
index 79fa3c8..bb92877 100644
--- a/include/exec/helper-tcg.h
+++ b/include/exec/helper-tcg.h
@@ -2,9 +2,9 @@
    This one defines data structures private to tcg.c.  */
 
 #ifndef HELPER_TCG_H
-#define HELPER_TCG_H 1
+#define HELPER_TCG_H
 
-#include <exec/helper-head.h>
+#include "exec/helper-head.h"
 
 #define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \
   { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
diff --git a/include/exec/hwaddr.h b/include/exec/hwaddr.h
index c9eb78f..a71c93c 100644
--- a/include/exec/hwaddr.h
+++ b/include/exec/hwaddr.h
@@ -3,6 +3,7 @@
 #ifndef HWADDR_H
 #define HWADDR_H
 
+
 #define HWADDR_BITS 64
 /* hwaddr is the type of a physical address (its size can
    be different from 'target_ulong').  */
diff --git a/include/exec/ioport.h b/include/exec/ioport.h
index 3bd6722..a298b89 100644
--- a/include/exec/ioport.h
+++ b/include/exec/ioport.h
@@ -24,13 +24,6 @@
 #ifndef IOPORT_H
 #define IOPORT_H
 
-#include "qemu-common.h"
-#include "qom/object.h"
-#include "exec/memory.h"
-
-typedef uint32_t pio_addr_t;
-#define FMT_pioaddr     PRIx32
-
 #define MAX_IOPORTS     (64 * 1024)
 #define IOPORTS_MASK    (MAX_IOPORTS - 1)
 
@@ -49,12 +42,12 @@
 extern const MemoryRegionOps unassigned_io_ops;
 #endif
 
-void cpu_outb(pio_addr_t addr, uint8_t val);
-void cpu_outw(pio_addr_t addr, uint16_t val);
-void cpu_outl(pio_addr_t addr, uint32_t val);
-uint8_t cpu_inb(pio_addr_t addr);
-uint16_t cpu_inw(pio_addr_t addr);
-uint32_t cpu_inl(pio_addr_t addr);
+void cpu_outb(uint32_t addr, uint8_t val);
+void cpu_outw(uint32_t addr, uint16_t val);
+void cpu_outl(uint32_t addr, uint32_t val);
+uint8_t cpu_inb(uint32_t addr);
+uint16_t cpu_inw(uint32_t addr);
+uint32_t cpu_inl(uint32_t addr);
 
 typedef struct PortioList {
     const struct MemoryRegionPortio *ports;
diff --git a/include/exec/log.h b/include/exec/log.h
new file mode 100644
index 0000000..ba1c9b5
--- /dev/null
+++ b/include/exec/log.h
@@ -0,0 +1,60 @@
+#ifndef QEMU_EXEC_LOG_H
+#define QEMU_EXEC_LOG_H
+
+#include "qemu/log.h"
+#include "qom/cpu.h"
+#include "disas/disas.h"
+
+/* cpu_dump_state() logging functions: */
+/**
+ * log_cpu_state:
+ * @cpu: The CPU whose state is to be logged.
+ * @flags: Flags what to log.
+ *
+ * Logs the output of cpu_dump_state().
+ */
+static inline void log_cpu_state(CPUState *cpu, int flags)
+{
+    if (qemu_log_enabled()) {
+        cpu_dump_state(cpu, qemu_logfile, fprintf, flags);
+    }
+}
+
+/**
+ * log_cpu_state_mask:
+ * @mask: Mask when to log.
+ * @cpu: The CPU whose state is to be logged.
+ * @flags: Flags what to log.
+ *
+ * Logs the output of cpu_dump_state() if loglevel includes @mask.
+ */
+static inline void log_cpu_state_mask(int mask, CPUState *cpu, int flags)
+{
+    if (qemu_loglevel & mask) {
+        log_cpu_state(cpu, flags);
+    }
+}
+
+#ifdef NEED_CPU_H
+/* disas() and target_disas() to qemu_logfile: */
+static inline void log_target_disas(CPUState *cpu, target_ulong start,
+                                    target_ulong len, int flags)
+{
+    target_disas(qemu_logfile, cpu, start, len, flags);
+}
+
+static inline void log_disas(void *code, unsigned long size)
+{
+    disas(qemu_logfile, code, size);
+}
+
+#if defined(CONFIG_USER_ONLY)
+/* page_dump() output to the log file: */
+static inline void log_page_dump(void)
+{
+    page_dump(qemu_logfile);
+}
+#endif
+#endif
+
+#endif
diff --git a/include/exec/memattrs.h b/include/exec/memattrs.h
new file mode 100644
index 0000000..e601061
--- /dev/null
+++ b/include/exec/memattrs.h
@@ -0,0 +1,49 @@
+/*
+ * Memory transaction attributes
+ *
+ * Copyright (c) 2015 Linaro Limited.
+ *
+ * Authors:
+ *  Peter Maydell <peter.maydell@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef MEMATTRS_H
+#define MEMATTRS_H
+
+/* Every memory transaction has associated with it a set of
+ * attributes. Some of these are generic (such as the ID of
+ * the bus master); some are specific to a particular kind of
+ * bus (such as the ARM Secure/NonSecure bit). We define them
+ * all as non-overlapping bitfields in a single struct to avoid
+ * confusion if different parts of QEMU used the same bit for
+ * different semantics.
+ */
+typedef struct MemTxAttrs {
+    /* Bus masters which don't specify any attributes will get this
+     * (via the MEMTXATTRS_UNSPECIFIED constant), so that we can
+     * distinguish "all attributes deliberately clear" from
+     * "didn't specify" if necessary.
+     */
+    unsigned int unspecified:1;
+    /* ARM/AMBA: TrustZone Secure access
+     * x86: System Management Mode access
+     */
+    unsigned int secure:1;
+    /* Memory access is usermode (unprivileged) */
+    unsigned int user:1;
+    /* Requester ID (for MSI for example) */
+    unsigned int requester_id:16;
+} MemTxAttrs;
+
+/* Bus masters which don't specify any attributes will get this,
+ * which has all attribute bits clear except the topmost one
+ * (so that we can distinguish "all attributes deliberately clear"
+ * from "didn't specify" if necessary).
+ */
+#define MEMTXATTRS_UNSPECIFIED ((MemTxAttrs) { .unspecified = 1 })
+
+#endif
diff --git a/include/exec/memory-internal.h b/include/exec/memory-internal.h
index 25c43c0..fb467ac 100644
--- a/include/exec/memory-internal.h
+++ b/include/exec/memory-internal.h
@@ -23,6 +23,7 @@
 typedef struct AddressSpaceDispatch AddressSpaceDispatch;
 
 void address_space_init_dispatch(AddressSpace *as);
+void address_space_unregister(AddressSpace *as);
 void address_space_destroy_dispatch(AddressSpace *as);
 
 extern const MemoryRegionOps unassigned_mem_ops;
diff --git a/include/exec/memory.h b/include/exec/memory.h
index f64ab5e..3e4d416 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -21,18 +21,18 @@
 #define DIRTY_MEMORY_MIGRATION 2
 #define DIRTY_MEMORY_NUM       3        /* num of dirty bits */
 
-#include <stdint.h>
-#include <stdbool.h>
-#include "qemu-common.h"
 #include "exec/cpu-common.h"
 #ifndef CONFIG_USER_ONLY
 #include "exec/hwaddr.h"
 #endif
+#include "exec/memattrs.h"
 #include "qemu/queue.h"
 #include "qemu/int128.h"
 #include "qemu/notify.h"
-#include "qapi/error.h"
 #include "qom/object.h"
+#include "qemu/rcu.h"
+
+#define RAM_ADDR_INVALID (~(ram_addr_t)0)
 
 #define MAX_PHYS_ADDR_SPACE_BITS 62
 #define MAX_PHYS_ADDR            (((hwaddr)1 << MAX_PHYS_ADDR_SPACE_BITS) - 1)
@@ -67,6 +67,16 @@
     IOMMUAccessFlags perm;
 };
 
+/* New-style MMIO accessors can indicate that the transaction failed.
+ * A zero (MEMTX_OK) response means success; anything else is a failure
+ * of some kind. The memory subsystem will bitwise-OR together results
+ * if it is synthesizing an operation from multiple smaller accesses.
+ */
+#define MEMTX_OK 0
+#define MEMTX_ERROR             (1U << 0) /* device returned an error */
+#define MEMTX_DECODE_ERROR      (1U << 1) /* nothing at that address */
+typedef uint32_t MemTxResult;
+
 /*
  * Memory region callbacks
  */
@@ -83,6 +93,17 @@
                   uint64_t data,
                   unsigned size);
 
+    MemTxResult (*read_with_attrs)(void *opaque,
+                                   hwaddr addr,
+                                   uint64_t *data,
+                                   unsigned size,
+                                   MemTxAttrs attrs);
+    MemTxResult (*write_with_attrs)(void *opaque,
+                                    hwaddr addr,
+                                    uint64_t data,
+                                    unsigned size,
+                                    MemTxAttrs attrs);
+
     enum device_endian endianness;
     /* Guest-visible constraints: */
     struct {
@@ -130,6 +151,12 @@
 struct MemoryRegionIOMMUOps {
     /* Return a TLB entry that contains a given address. */
     IOMMUTLBEntry (*translate)(MemoryRegion *iommu, hwaddr addr, bool is_write);
+    /* Returns minimum supported page size */
+    uint64_t (*get_min_page_size)(MemoryRegion *iommu);
+    /* Called when the first notifier is set */
+    void (*notify_started)(MemoryRegion *iommu);
+    /* Called when the last notifier is removed */
+    void (*notify_stopped)(MemoryRegion *iommu);
 };
 
 typedef struct CoalescedMemoryRange CoalescedMemoryRange;
@@ -137,35 +164,41 @@
 
 struct MemoryRegion {
     Object parent_obj;
+
     /* All fields are private - violators will be prosecuted */
-    const MemoryRegionOps *ops;
+
+    /* The following fields should fit in a cache line */
+    bool romd_mode;
+    bool ram;
+    bool subpage;
+    bool readonly; /* For RAM regions */
+    bool rom_device;
+    bool flush_coalesced_mmio;
+    bool global_locking;
+    uint8_t dirty_log_mask;
+    RAMBlock *ram_block;
+    Object *owner;
     const MemoryRegionIOMMUOps *iommu_ops;
+
+    const MemoryRegionOps *ops;
     void *opaque;
     MemoryRegion *container;
     Int128 size;
     hwaddr addr;
     void (*destructor)(MemoryRegion *mr);
-    ram_addr_t ram_addr;
     uint64_t align;
-    bool subpage;
     bool terminates;
-    bool romd_mode;
-    bool ram;
     bool skip_dump;
-    bool readonly; /* For RAM regions */
     bool enabled;
-    bool rom_device;
     bool warning_printed; /* For reservations */
-    bool flush_coalesced_mmio;
+    uint8_t vga_logging_count;
     MemoryRegion *alias;
     hwaddr alias_offset;
     int32_t priority;
-    bool may_overlap;
     QTAILQ_HEAD(subregions, MemoryRegion) subregions;
     QTAILQ_ENTRY(MemoryRegion) subregions_link;
     QTAILQ_HEAD(coalesced_ranges, CoalescedMemoryRange) coalesced;
     const char *name;
-    uint8_t dirty_log_mask;
     unsigned ioeventfd_nb;
     MemoryRegionIoeventfd *ioeventfds;
     NotifierList iommu_notify;
@@ -183,8 +216,10 @@
     void (*region_add)(MemoryListener *listener, MemoryRegionSection *section);
     void (*region_del)(MemoryListener *listener, MemoryRegionSection *section);
     void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section);
-    void (*log_start)(MemoryListener *listener, MemoryRegionSection *section);
-    void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section);
+    void (*log_start)(MemoryListener *listener, MemoryRegionSection *section,
+                      int old, int new);
+    void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section,
+                     int old, int new);
     void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section);
     void (*log_global_start)(MemoryListener *listener);
     void (*log_global_stop)(MemoryListener *listener);
@@ -207,9 +242,15 @@
  */
 struct AddressSpace {
     /* All fields are private. */
+    struct rcu_head rcu;
     char *name;
     MemoryRegion *root;
+    int ref_count;
+    bool malloced;
+
+    /* Accessed via RCU.  */
     struct FlatView *current_map;
+
     int ioeventfd_nb;
     struct MemoryRegionIoeventfd *ioeventfds;
     struct AddressSpaceDispatch *dispatch;
@@ -294,7 +335,7 @@
  * @owner: the object that tracks the region's reference count
  * @ops: a structure containing read and write callbacks to be used when
  *       I/O is performed on the region.
- * @opaque: passed to to the read and write callbacks of the @ops structure.
+ * @opaque: passed to the read and write callbacks of the @ops structure.
  * @name: used for debugging; not visible to the user or ABI
  * @size: size of the region.
  */
@@ -321,6 +362,30 @@
                             uint64_t size,
                             Error **errp);
 
+/**
+ * memory_region_init_resizeable_ram:  Initialize memory region with resizeable
+ *                                     RAM.  Accesses into the region will
+ *                                     modify memory directly.  Only an initial
+ *                                     portion of this RAM is actually used.
+ *                                     The used size can change across reboots.
+ *
+ * @mr: the #MemoryRegion to be initialized.
+ * @owner: the object that tracks the region's reference count
+ * @name: the name of the region.
+ * @size: used size of the region.
+ * @max_size: max size of the region.
+ * @resized: callback to notify owner about used size change.
+ * @errp: pointer to Error*, to store an error if it happens.
+ */
+void memory_region_init_resizeable_ram(MemoryRegion *mr,
+                                       struct Object *owner,
+                                       const char *name,
+                                       uint64_t size,
+                                       uint64_t max_size,
+                                       void (*resized)(const char*,
+                                                       uint64_t length,
+                                                       void *host),
+                                       Error **errp);
 #ifdef __linux__
 /**
  * memory_region_init_ram_from_file:  Initialize RAM memory region with a
@@ -380,12 +445,31 @@
                               uint64_t size);
 
 /**
+ * memory_region_init_rom: Initialize a ROM memory region.
+ *
+ * This has the same effect as calling memory_region_init_ram()
+ * and then marking the resulting region read-only with
+ * memory_region_set_readonly().
+ *
+ * @mr: the #MemoryRegion to be initialized.
+ * @owner: the object that tracks the region's reference count
+ * @name: the name of the region.
+ * @size: size of the region.
+ * @errp: pointer to Error*, to store an error if it happens.
+ */
+void memory_region_init_rom(MemoryRegion *mr,
+                            struct Object *owner,
+                            const char *name,
+                            uint64_t size,
+                            Error **errp);
+
+/**
  * memory_region_init_rom_device:  Initialize a ROM memory region.  Writes are
  *                                 handled via callbacks.
  *
  * @mr: the #MemoryRegion to be initialized.
  * @owner: the object that tracks the region's reference count
- * @ops: callbacks for write access handling.
+ * @ops: callbacks for write access handling (must not be NULL).
  * @name: the name of the region.
  * @size: size of the region.
  * @errp: pointer to Error*, to store an error if it happens.
@@ -405,16 +489,21 @@
  * A reservation region primariy serves debugging purposes.  It claims I/O
  * space that is not supposed to be handled by QEMU itself.  Any access via
  * the memory API will cause an abort().
+ * This function is deprecated. Use memory_region_init_io() with NULL
+ * callbacks instead.
  *
  * @mr: the #MemoryRegion to be initialized
  * @owner: the object that tracks the region's reference count
  * @name: used for debugging; not visible to the user or ABI
  * @size: size of the region.
  */
-void memory_region_init_reservation(MemoryRegion *mr,
-                                    struct Object *owner,
+static inline void memory_region_init_reservation(MemoryRegion *mr,
+                                    Object *owner,
                                     const char *name,
-                                    uint64_t size);
+                                    uint64_t size)
+{
+    memory_region_init_io(mr, owner, NULL, mr, name, size);
+}
 
 /**
  * memory_region_init_iommu: Initialize a memory region that translates
@@ -456,7 +545,10 @@
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_ram(MemoryRegion *mr);
+static inline bool memory_region_is_ram(MemoryRegion *mr)
+{
+    return mr->ram;
+}
 
 /**
  * memory_region_is_skip_dump: check whether a memory region should not be
@@ -496,7 +588,21 @@
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_iommu(MemoryRegion *mr);
+static inline bool memory_region_is_iommu(MemoryRegion *mr)
+{
+    return mr->iommu_ops;
+}
+
+
+/**
+ * memory_region_iommu_get_min_page_size: get minimum supported page size
+ * for an iommu
+ *
+ * Returns minimum supported page size for an iommu.
+ *
+ * @mr: the memory region being queried
+ */
+uint64_t memory_region_iommu_get_min_page_size(MemoryRegion *mr);
 
 /**
  * memory_region_notify_iommu: notify a change in an IOMMU translation entry.
@@ -521,12 +627,26 @@
 void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n);
 
 /**
+ * memory_region_iommu_replay: replay existing IOMMU translations to
+ * a notifier with the minimum page granularity returned by
+ * mr->iommu_ops->get_page_size().
+ *
+ * @mr: the memory region to observe
+ * @n: the notifier to which to replay iommu mappings
+ * @is_write: Whether to treat the replay as a translate "write"
+ *     through the iommu
+ */
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n, bool is_write);
+
+/**
  * memory_region_unregister_iommu_notifier: unregister a notifier for
  * changes to IOMMU translation entries.
  *
+ * @mr: the memory region which was observed and for which notity_stopped()
+ *      needs to be called
  * @n: the notifier to be removed.
  */
-void memory_region_unregister_iommu_notifier(Notifier *n);
+void memory_region_unregister_iommu_notifier(MemoryRegion *mr, Notifier *n);
 
 /**
  * memory_region_name: get a memory region's name
@@ -540,11 +660,23 @@
 /**
  * memory_region_is_logging: return whether a memory region is logging writes
  *
- * Returns %true if the memory region is logging writes
+ * Returns %true if the memory region is logging writes for the given client
+ *
+ * @mr: the memory region being queried
+ * @client: the client being queried
+ */
+bool memory_region_is_logging(MemoryRegion *mr, uint8_t client);
+
+/**
+ * memory_region_get_dirty_log_mask: return the clients for which a
+ * memory region is logging writes.
+ *
+ * Returns a bitmap of clients, in which the DIRTY_MEMORY_* constants
+ * are the bit indices.
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_logging(MemoryRegion *mr);
+uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr);
 
 /**
  * memory_region_is_rom: check whether a memory region is ROM
@@ -553,7 +685,11 @@
  *
  * @mr: the memory region being queried
  */
-bool memory_region_is_rom(MemoryRegion *mr);
+static inline bool memory_region_is_rom(MemoryRegion *mr)
+{
+    return mr->ram && mr->readonly;
+}
+
 
 /**
  * memory_region_get_fd: Get a file descriptor backing a RAM memory region.
@@ -566,16 +702,62 @@
 int memory_region_get_fd(MemoryRegion *mr);
 
 /**
+ * memory_region_set_fd: Mark a RAM memory region as backed by a
+ * file descriptor.
+ *
+ * This function is typically used after memory_region_init_ram_ptr().
+ *
+ * @mr: the memory region being queried.
+ * @fd: the file descriptor that backs @mr.
+ */
+void memory_region_set_fd(MemoryRegion *mr, int fd);
+
+/**
+ * memory_region_from_host: Convert a pointer into a RAM memory region
+ * and an offset within it.
+ *
+ * Given a host pointer inside a RAM memory region (created with
+ * memory_region_init_ram() or memory_region_init_ram_ptr()), return
+ * the MemoryRegion and the offset within it.
+ *
+ * Use with care; by the time this function returns, the returned pointer is
+ * not protected by RCU anymore.  If the caller is not within an RCU critical
+ * section and does not hold the iothread lock, it must have other means of
+ * protecting the pointer, such as a reference to the region that includes
+ * the incoming ram_addr_t.
+ *
+ * @mr: the memory region being queried.
+ */
+MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset);
+
+/**
  * memory_region_get_ram_ptr: Get a pointer into a RAM memory region.
  *
  * Returns a host pointer to a RAM memory region (created with
- * memory_region_init_ram() or memory_region_init_ram_ptr()).  Use with
- * care.
+ * memory_region_init_ram() or memory_region_init_ram_ptr()).
+ *
+ * Use with care; by the time this function returns, the returned pointer is
+ * not protected by RCU anymore.  If the caller is not within an RCU critical
+ * section and does not hold the iothread lock, it must have other means of
+ * protecting the pointer, such as a reference to the region that includes
+ * the incoming ram_addr_t.
  *
  * @mr: the memory region being queried.
  */
 void *memory_region_get_ram_ptr(MemoryRegion *mr);
 
+/* memory_region_ram_resize: Resize a RAM region.
+ *
+ * Only legal before guest might have detected the memory size: e.g. on
+ * incoming migration, or right after reset.
+ *
+ * @mr: a memory region created with @memory_region_init_resizeable_ram.
+ * @newsize: the new size the region
+ * @errp: pointer to Error*, to store an error if it happens.
+ */
+void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize,
+                              Error **errp);
+
 /**
  * memory_region_set_log: Turn dirty logging on or off for a region.
  *
@@ -584,8 +766,7 @@
  *
  * @mr: the memory region being updated.
  * @log: whether dirty logging is to be enabled or disabled.
- * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
- *          %DIRTY_MEMORY_VGA.
+ * @client: the user of the logging information; %DIRTY_MEMORY_VGA only.
  */
 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client);
 
@@ -749,6 +930,31 @@
 void memory_region_clear_flush_coalesced(MemoryRegion *mr);
 
 /**
+ * memory_region_set_global_locking: Declares the access processing requires
+ *                                   QEMU's global lock.
+ *
+ * When this is invoked, accesses to the memory region will be processed while
+ * holding the global lock of QEMU. This is the default behavior of memory
+ * regions.
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_set_global_locking(MemoryRegion *mr);
+
+/**
+ * memory_region_clear_global_locking: Declares that access processing does
+ *                                     not depend on the QEMU global lock.
+ *
+ * By clearing this property, accesses to the memory region will be processed
+ * outside of QEMU's global lock (unless the lock is held on when issuing the
+ * access request). In this case, the device model implementing the access
+ * handlers is responsible for synchronization of concurrency.
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_clear_global_locking(MemoryRegion *mr);
+
+/**
  * memory_region_add_eventfd: Request an eventfd to be triggered when a word
  *                            is written to a location.
  *
@@ -833,9 +1039,6 @@
 /**
  * memory_region_get_ram_addr: Get the ram address associated with a memory
  *                             region
- *
- * DO NOT USE THIS FUNCTION.  This is a temporary workaround while the Xen
- * code is being reworked.
  */
 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr);
 
@@ -878,6 +1081,16 @@
 void memory_region_set_address(MemoryRegion *mr, hwaddr addr);
 
 /*
+ * memory_region_set_size: dynamically update the size of a region.
+ *
+ * Dynamically updates the size of a region.
+ *
+ * @mr: the region to be updated
+ * @size: used size of the region.
+ */
+void memory_region_set_size(MemoryRegion *mr, uint64_t size);
+
+/*
  * memory_region_set_alias_offset: dynamically update a memory alias's offset
  *
  * Dynamically updates the offset into the target region that an alias points
@@ -992,15 +1205,62 @@
 void mtree_info(fprintf_function mon_printf, void *f);
 
 /**
+ * memory_region_dispatch_read: perform a read directly to the specified
+ * MemoryRegion.
+ *
+ * @mr: #MemoryRegion to access
+ * @addr: address within that region
+ * @pval: pointer to uint64_t which the data is written to
+ * @size: size of the access in bytes
+ * @attrs: memory transaction attributes to use for the access
+ */
+MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
+                                        hwaddr addr,
+                                        uint64_t *pval,
+                                        unsigned size,
+                                        MemTxAttrs attrs);
+/**
+ * memory_region_dispatch_write: perform a write directly to the specified
+ * MemoryRegion.
+ *
+ * @mr: #MemoryRegion to access
+ * @addr: address within that region
+ * @data: data to write
+ * @size: size of the access in bytes
+ * @attrs: memory transaction attributes to use for the access
+ */
+MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
+                                         hwaddr addr,
+                                         uint64_t data,
+                                         unsigned size,
+                                         MemTxAttrs attrs);
+
+/**
  * address_space_init: initializes an address space
  *
  * @as: an uninitialized #AddressSpace
- * @root: a #MemoryRegion that routes addesses for the address space
+ * @root: a #MemoryRegion that routes addresses for the address space
  * @name: an address space name.  The name is only used for debugging
  *        output.
  */
 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name);
 
+/**
+ * address_space_init_shareable: return an address space for a memory region,
+ *                               creating it if it does not already exist
+ *
+ * @root: a #MemoryRegion that routes addresses for the address space
+ * @name: an address space name.  The name is only used for debugging
+ *        output.
+ *
+ * This function will return a pointer to an existing AddressSpace
+ * which was initialized with the specified MemoryRegion, or it will
+ * create and initialize one if it does not already exist. The ASes
+ * are reference-counted, so the memory will be freed automatically
+ * when the AddressSpace is destroyed via address_space_destroy.
+ */
+AddressSpace *address_space_init_shareable(MemoryRegion *root,
+                                           const char *name);
 
 /**
  * address_space_destroy: destroy an address space
@@ -1016,44 +1276,89 @@
 /**
  * address_space_rw: read from or write to an address space.
  *
- * Return true if the operation hit any unassigned memory or encountered an
- * IOMMU fault.
+ * Return a MemTxResult indicating whether the operation succeeded
+ * or failed (eg unassigned memory, device rejected the transaction,
+ * IOMMU fault).
  *
  * @as: #AddressSpace to be accessed
  * @addr: address within that address space
+ * @attrs: memory transaction attributes
  * @buf: buffer with the data transferred
  * @is_write: indicates the transfer direction
  */
-bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
-                      int len, bool is_write);
+MemTxResult address_space_rw(AddressSpace *as, hwaddr addr,
+                             MemTxAttrs attrs, uint8_t *buf,
+                             int len, bool is_write);
 
 /**
  * address_space_write: write to address space.
  *
- * Return true if the operation hit any unassigned memory or encountered an
- * IOMMU fault.
+ * Return a MemTxResult indicating whether the operation succeeded
+ * or failed (eg unassigned memory, device rejected the transaction,
+ * IOMMU fault).
  *
  * @as: #AddressSpace to be accessed
  * @addr: address within that address space
+ * @attrs: memory transaction attributes
  * @buf: buffer with the data transferred
  */
-bool address_space_write(AddressSpace *as, hwaddr addr,
-                         const uint8_t *buf, int len);
+MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
+                                MemTxAttrs attrs,
+                                const uint8_t *buf, int len);
 
-/**
- * address_space_read: read from an address space.
+/* address_space_ld*: load from an address space
+ * address_space_st*: store to an address space
  *
- * Return true if the operation hit any unassigned memory or encountered an
- * IOMMU fault.
+ * These functions perform a load or store of the byte, word,
+ * longword or quad to the specified address within the AddressSpace.
+ * The _le suffixed functions treat the data as little endian;
+ * _be indicates big endian; no suffix indicates "same endianness
+ * as guest CPU".
  *
- * @as: #AddressSpace to be accessed
+ * The "guest CPU endianness" accessors are deprecated for use outside
+ * target-* code; devices should be CPU-agnostic and use either the LE
+ * or the BE accessors.
+ *
+ * @as #AddressSpace to be accessed
  * @addr: address within that address space
- * @buf: buffer with the data transferred
+ * @val: data value, for stores
+ * @attrs: memory transaction attributes
+ * @result: location to write the success/failure of the transaction;
+ *   if NULL, this information is discarded
  */
-bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len);
+uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
+void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
+                            MemTxAttrs attrs, MemTxResult *result);
 
 /* address_space_translate: translate an address range into an address space
- * into a MemoryRegion and an address range into that section
+ * into a MemoryRegion and an address range into that section.  Should be
+ * called from an RCU critical section, to avoid that the last reference
+ * to the returned region disappears after address_space_translate returns.
  *
  * @as: #AddressSpace to be accessed
  * @addr: address within that address space
@@ -1114,6 +1419,65 @@
                          int is_write, hwaddr access_len);
 
 
+/* Internal functions, part of the implementation of address_space_read.  */
+MemTxResult address_space_read_continue(AddressSpace *as, hwaddr addr,
+                                        MemTxAttrs attrs, uint8_t *buf,
+                                        int len, hwaddr addr1, hwaddr l,
+					MemoryRegion *mr);
+MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
+                                    MemTxAttrs attrs, uint8_t *buf, int len);
+void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr);
+
+static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
+{
+    if (is_write) {
+        return memory_region_is_ram(mr) && !mr->readonly;
+    } else {
+        return memory_region_is_ram(mr) || memory_region_is_romd(mr);
+    }
+}
+
+/**
+ * address_space_read: read from an address space.
+ *
+ * Return a MemTxResult indicating whether the operation succeeded
+ * or failed (eg unassigned memory, device rejected the transaction,
+ * IOMMU fault).
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @attrs: memory transaction attributes
+ * @buf: buffer with the data transferred
+ */
+static inline __attribute__((__always_inline__))
+MemTxResult address_space_read(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                               uint8_t *buf, int len)
+{
+    MemTxResult result = MEMTX_OK;
+    hwaddr l, addr1;
+    void *ptr;
+    MemoryRegion *mr;
+
+    if (__builtin_constant_p(len)) {
+        if (len) {
+            rcu_read_lock();
+            l = len;
+            mr = address_space_translate(as, addr, &addr1, &l, false);
+            if (len == l && memory_access_is_direct(mr, false)) {
+                ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
+                memcpy(buf, ptr, len);
+            } else {
+                result = address_space_read_continue(as, addr, attrs, buf, len,
+                                                     addr1, l, mr);
+            }
+            rcu_read_unlock();
+        }
+    } else {
+        result = address_space_read_full(as, addr, attrs, buf, len);
+    }
+    return result;
+}
+
 #endif
 
 #endif
diff --git a/include/exec/poison.h b/include/exec/poison.h
index a4b1eca..3ca7929 100644
--- a/include/exec/poison.h
+++ b/include/exec/poison.h
@@ -37,14 +37,6 @@
 
 #pragma GCC poison CPUArchState
 
-#pragma GCC poison lduw_phys
-#pragma GCC poison ldl_phys
-#pragma GCC poison ldq_phys
-#pragma GCC poison stl_phys_notdirty
-#pragma GCC poison stw_phys
-#pragma GCC poison stl_phys
-#pragma GCC poison stq_phys
-
 #pragma GCC poison CPU_INTERRUPT_HARD
 #pragma GCC poison CPU_INTERRUPT_EXITTB
 #pragma GCC poison CPU_INTERRUPT_HALT
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index 8fc75cd..2a9465d 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -22,46 +22,176 @@
 #ifndef CONFIG_USER_ONLY
 #include "hw/xen/xen.h"
 
-ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
-                                    bool share, const char *mem_path,
-                                    Error **errp);
-ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
-                                   MemoryRegion *mr, Error **errp);
-ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
-int qemu_get_ram_fd(ram_addr_t addr);
-void *qemu_get_ram_block_host_ptr(ram_addr_t addr);
-void *qemu_get_ram_ptr(ram_addr_t addr);
-void qemu_ram_free(ram_addr_t addr);
-void qemu_ram_free_from_ptr(ram_addr_t addr);
+struct RAMBlock {
+    struct rcu_head rcu;
+    struct MemoryRegion *mr;
+    uint8_t *host;
+    ram_addr_t offset;
+    ram_addr_t used_length;
+    ram_addr_t max_length;
+    void (*resized)(const char*, uint64_t length, void *host);
+    uint32_t flags;
+    /* Protected by iothread lock.  */
+    char idstr[256];
+    /* RCU-enabled, writes protected by the ramlist lock */
+    QLIST_ENTRY(RAMBlock) next;
+    int fd;
+};
+
+static inline bool offset_in_ramblock(RAMBlock *b, ram_addr_t offset)
+{
+    return (b && b->host && offset < b->used_length) ? true : false;
+}
+
+static inline void *ramblock_ptr(RAMBlock *block, ram_addr_t offset)
+{
+    assert(offset_in_ramblock(block, offset));
+    return (char *)block->host + offset;
+}
+
+/* The dirty memory bitmap is split into fixed-size blocks to allow growth
+ * under RCU.  The bitmap for a block can be accessed as follows:
+ *
+ *   rcu_read_lock();
+ *
+ *   DirtyMemoryBlocks *blocks =
+ *       atomic_rcu_read(&ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION]);
+ *
+ *   ram_addr_t idx = (addr >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
+ *   unsigned long *block = blocks.blocks[idx];
+ *   ...access block bitmap...
+ *
+ *   rcu_read_unlock();
+ *
+ * Remember to check for the end of the block when accessing a range of
+ * addresses.  Move on to the next block if you reach the end.
+ *
+ * Organization into blocks allows dirty memory to grow (but not shrink) under
+ * RCU.  When adding new RAMBlocks requires the dirty memory to grow, a new
+ * DirtyMemoryBlocks array is allocated with pointers to existing blocks kept
+ * the same.  Other threads can safely access existing blocks while dirty
+ * memory is being grown.  When no threads are using the old DirtyMemoryBlocks
+ * anymore it is freed by RCU (but the underlying blocks stay because they are
+ * pointed to from the new DirtyMemoryBlocks).
+ */
+#define DIRTY_MEMORY_BLOCK_SIZE ((ram_addr_t)256 * 1024 * 8)
+typedef struct {
+    struct rcu_head rcu;
+    unsigned long *blocks[];
+} DirtyMemoryBlocks;
+
+typedef struct RAMList {
+    QemuMutex mutex;
+    RAMBlock *mru_block;
+    /* RCU-enabled, writes protected by the ramlist lock. */
+    QLIST_HEAD(, RAMBlock) blocks;
+    DirtyMemoryBlocks *dirty_memory[DIRTY_MEMORY_NUM];
+    uint32_t version;
+} RAMList;
+extern RAMList ram_list;
+
+ram_addr_t last_ram_offset(void);
+void qemu_mutex_lock_ramlist(void);
+void qemu_mutex_unlock_ramlist(void);
+
+RAMBlock *qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
+                                   bool share, const char *mem_path,
+                                   Error **errp);
+RAMBlock *qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
+                                  MemoryRegion *mr, Error **errp);
+RAMBlock *qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr, Error **errp);
+RAMBlock *qemu_ram_alloc_resizeable(ram_addr_t size, ram_addr_t max_size,
+                                    void (*resized)(const char*,
+                                                    uint64_t length,
+                                                    void *host),
+                                    MemoryRegion *mr, Error **errp);
+void qemu_ram_free(RAMBlock *block);
+
+int qemu_ram_resize(RAMBlock *block, ram_addr_t newsize, Error **errp);
+
+#define DIRTY_CLIENTS_ALL     ((1 << DIRTY_MEMORY_NUM) - 1)
+#define DIRTY_CLIENTS_NOCODE  (DIRTY_CLIENTS_ALL & ~(1 << DIRTY_MEMORY_CODE))
 
 static inline bool cpu_physical_memory_get_dirty(ram_addr_t start,
                                                  ram_addr_t length,
                                                  unsigned client)
 {
-    unsigned long end, page, next;
+    DirtyMemoryBlocks *blocks;
+    unsigned long end, page;
+    unsigned long idx, offset, base;
+    bool dirty = false;
 
     assert(client < DIRTY_MEMORY_NUM);
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    next = find_next_bit(ram_list.dirty_memory[client], end, page);
 
-    return next < end;
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+    base = page - offset;
+    while (page < end) {
+        unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
+        unsigned long num = next - base;
+        unsigned long found = find_next_bit(blocks->blocks[idx], num, offset);
+        if (found < num) {
+            dirty = true;
+            break;
+        }
+
+        page = next;
+        idx++;
+        offset = 0;
+        base += DIRTY_MEMORY_BLOCK_SIZE;
+    }
+
+    rcu_read_unlock();
+
+    return dirty;
 }
 
-static inline bool cpu_physical_memory_get_clean(ram_addr_t start,
+static inline bool cpu_physical_memory_all_dirty(ram_addr_t start,
                                                  ram_addr_t length,
                                                  unsigned client)
 {
-    unsigned long end, page, next;
+    DirtyMemoryBlocks *blocks;
+    unsigned long end, page;
+    unsigned long idx, offset, base;
+    bool dirty = true;
 
     assert(client < DIRTY_MEMORY_NUM);
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    next = find_next_zero_bit(ram_list.dirty_memory[client], end, page);
 
-    return next < end;
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+    base = page - offset;
+    while (page < end) {
+        unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
+        unsigned long num = next - base;
+        unsigned long found = find_next_zero_bit(blocks->blocks[idx], num, offset);
+        if (found < num) {
+            dirty = false;
+            break;
+        }
+
+        page = next;
+        idx++;
+        offset = 0;
+        base += DIRTY_MEMORY_BLOCK_SIZE;
+    }
+
+    rcu_read_unlock();
+
+    return dirty;
 }
 
 static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
@@ -79,44 +209,97 @@
     return !(vga && code && migration);
 }
 
-static inline bool cpu_physical_memory_range_includes_clean(ram_addr_t start,
-                                                            ram_addr_t length)
+static inline uint8_t cpu_physical_memory_range_includes_clean(ram_addr_t start,
+                                                               ram_addr_t length,
+                                                               uint8_t mask)
 {
-    bool vga = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_VGA);
-    bool code = cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_CODE);
-    bool migration =
-        cpu_physical_memory_get_clean(start, length, DIRTY_MEMORY_MIGRATION);
-    return vga || code || migration;
+    uint8_t ret = 0;
+
+    if (mask & (1 << DIRTY_MEMORY_VGA) &&
+        !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_VGA)) {
+        ret |= (1 << DIRTY_MEMORY_VGA);
+    }
+    if (mask & (1 << DIRTY_MEMORY_CODE) &&
+        !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_CODE)) {
+        ret |= (1 << DIRTY_MEMORY_CODE);
+    }
+    if (mask & (1 << DIRTY_MEMORY_MIGRATION) &&
+        !cpu_physical_memory_all_dirty(start, length, DIRTY_MEMORY_MIGRATION)) {
+        ret |= (1 << DIRTY_MEMORY_MIGRATION);
+    }
+    return ret;
 }
 
 static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
                                                       unsigned client)
 {
+    unsigned long page, idx, offset;
+    DirtyMemoryBlocks *blocks;
+
     assert(client < DIRTY_MEMORY_NUM);
-    set_bit(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]);
-}
 
-static inline void cpu_physical_memory_set_dirty_range_nocode(ram_addr_t start,
-                                                              ram_addr_t length)
-{
-    unsigned long end, page;
+    page = addr >> TARGET_PAGE_BITS;
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
 
-    end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
-    page = start >> TARGET_PAGE_BITS;
-    bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page);
-    bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page);
+    rcu_read_lock();
+
+    blocks = atomic_rcu_read(&ram_list.dirty_memory[client]);
+
+    set_bit_atomic(offset, blocks->blocks[idx]);
+
+    rcu_read_unlock();
 }
 
 static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
-                                                       ram_addr_t length)
+                                                       ram_addr_t length,
+                                                       uint8_t mask)
 {
+    DirtyMemoryBlocks *blocks[DIRTY_MEMORY_NUM];
     unsigned long end, page;
+    unsigned long idx, offset, base;
+    int i;
+
+    if (!mask && !xen_enabled()) {
+        return;
+    }
 
     end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
     page = start >> TARGET_PAGE_BITS;
-    bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page);
-    bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page);
-    bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page);
+
+    rcu_read_lock();
+
+    for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+        blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i]);
+    }
+
+    idx = page / DIRTY_MEMORY_BLOCK_SIZE;
+    offset = page % DIRTY_MEMORY_BLOCK_SIZE;
+    base = page - offset;
+    while (page < end) {
+        unsigned long next = MIN(end, base + DIRTY_MEMORY_BLOCK_SIZE);
+
+        if (likely(mask & (1 << DIRTY_MEMORY_MIGRATION))) {
+            bitmap_set_atomic(blocks[DIRTY_MEMORY_MIGRATION]->blocks[idx],
+                              offset, next - page);
+        }
+        if (unlikely(mask & (1 << DIRTY_MEMORY_VGA))) {
+            bitmap_set_atomic(blocks[DIRTY_MEMORY_VGA]->blocks[idx],
+                              offset, next - page);
+        }
+        if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
+            bitmap_set_atomic(blocks[DIRTY_MEMORY_CODE]->blocks[idx],
+                              offset, next - page);
+        }
+
+        page = next;
+        idx++;
+        offset = 0;
+        base += DIRTY_MEMORY_BLOCK_SIZE;
+    }
+
+    rcu_read_unlock();
+
     xen_modified_memory(start, length);
 }
 
@@ -136,20 +319,44 @@
     /* start address is aligned at the start of a word? */
     if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
         (hpratio == 1)) {
+        unsigned long **blocks[DIRTY_MEMORY_NUM];
+        unsigned long idx;
+        unsigned long offset;
         long k;
         long nr = BITS_TO_LONGS(pages);
 
+        idx = (start >> TARGET_PAGE_BITS) / DIRTY_MEMORY_BLOCK_SIZE;
+        offset = BIT_WORD((start >> TARGET_PAGE_BITS) %
+                          DIRTY_MEMORY_BLOCK_SIZE);
+
+        rcu_read_lock();
+
+        for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+            blocks[i] = atomic_rcu_read(&ram_list.dirty_memory[i])->blocks;
+        }
+
         for (k = 0; k < nr; k++) {
             if (bitmap[k]) {
                 unsigned long temp = leul_to_cpu(bitmap[k]);
 
-                ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION][page + k] |= temp;
-                ram_list.dirty_memory[DIRTY_MEMORY_VGA][page + k] |= temp;
-                ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp;
+                atomic_or(&blocks[DIRTY_MEMORY_MIGRATION][idx][offset], temp);
+                atomic_or(&blocks[DIRTY_MEMORY_VGA][idx][offset], temp);
+                if (tcg_enabled()) {
+                    atomic_or(&blocks[DIRTY_MEMORY_CODE][idx][offset], temp);
+                }
+            }
+
+            if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
+                offset = 0;
+                idx++;
             }
         }
-        xen_modified_memory(start, pages);
+
+        rcu_read_unlock();
+
+        xen_modified_memory(start, pages << TARGET_PAGE_BITS);
     } else {
+        uint8_t clients = tcg_enabled() ? DIRTY_CLIENTS_ALL : DIRTY_CLIENTS_NOCODE;
         /*
          * bitmap-traveling is faster than memory-traveling (for addr...)
          * especially when most of the memory is not dirty.
@@ -164,7 +371,7 @@
                     addr = page_number * TARGET_PAGE_SIZE;
                     ram_addr = start + addr;
                     cpu_physical_memory_set_dirty_range(ram_addr,
-                                       TARGET_PAGE_SIZE * hpratio);
+                                       TARGET_PAGE_SIZE * hpratio, clients);
                 } while (c != 0);
             }
         }
@@ -172,20 +379,76 @@
 }
 #endif /* not _WIN32 */
 
-static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
-                                                         ram_addr_t length,
-                                                         unsigned client)
-{
-    unsigned long end, page;
+bool cpu_physical_memory_test_and_clear_dirty(ram_addr_t start,
+                                              ram_addr_t length,
+                                              unsigned client);
 
-    assert(client < DIRTY_MEMORY_NUM);
-    end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
-    page = start >> TARGET_PAGE_BITS;
-    bitmap_clear(ram_list.dirty_memory[client], page, end - page);
+static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
+                                                         ram_addr_t length)
+{
+    cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_MIGRATION);
+    cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_VGA);
+    cpu_physical_memory_test_and_clear_dirty(start, length, DIRTY_MEMORY_CODE);
 }
 
-void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
-                                     unsigned client);
 
+static inline
+uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned long *dest,
+                                               ram_addr_t start,
+                                               ram_addr_t length)
+{
+    ram_addr_t addr;
+    unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
+    uint64_t num_dirty = 0;
+
+    /* start address is aligned at the start of a word? */
+    if (((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) {
+        int k;
+        int nr = BITS_TO_LONGS(length >> TARGET_PAGE_BITS);
+        unsigned long * const *src;
+        unsigned long idx = (page * BITS_PER_LONG) / DIRTY_MEMORY_BLOCK_SIZE;
+        unsigned long offset = BIT_WORD((page * BITS_PER_LONG) %
+                                        DIRTY_MEMORY_BLOCK_SIZE);
+
+        rcu_read_lock();
+
+        src = atomic_rcu_read(
+                &ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION])->blocks;
+
+        for (k = page; k < page + nr; k++) {
+            if (src[idx][offset]) {
+                unsigned long bits = atomic_xchg(&src[idx][offset], 0);
+                unsigned long new_dirty;
+                new_dirty = ~dest[k];
+                dest[k] |= bits;
+                new_dirty &= bits;
+                num_dirty += ctpopl(new_dirty);
+            }
+
+            if (++offset >= BITS_TO_LONGS(DIRTY_MEMORY_BLOCK_SIZE)) {
+                offset = 0;
+                idx++;
+            }
+        }
+
+        rcu_read_unlock();
+    } else {
+        for (addr = 0; addr < length; addr += TARGET_PAGE_SIZE) {
+            if (cpu_physical_memory_test_and_clear_dirty(
+                        start + addr,
+                        TARGET_PAGE_SIZE,
+                        DIRTY_MEMORY_MIGRATION)) {
+                long k = (start + addr) >> TARGET_PAGE_BITS;
+                if (!test_and_set_bit(k, dest)) {
+                    num_dirty++;
+                }
+            }
+        }
+    }
+
+    return num_dirty;
+}
+
+void migration_bitmap_extend(ram_addr_t old, ram_addr_t new);
 #endif
 #endif
diff --git a/include/exec/semihost.h b/include/exec/semihost.h
new file mode 100644
index 0000000..5980939
--- /dev/null
+++ b/include/exec/semihost.h
@@ -0,0 +1,62 @@
+/*
+ * Semihosting support
+ *
+ * Copyright (c) 2015 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SEMIHOST_H
+#define SEMIHOST_H
+
+typedef enum SemihostingTarget {
+    SEMIHOSTING_TARGET_AUTO = 0,
+    SEMIHOSTING_TARGET_NATIVE,
+    SEMIHOSTING_TARGET_GDB
+} SemihostingTarget;
+
+#ifdef CONFIG_USER_ONLY
+static inline bool semihosting_enabled(void)
+{
+    return true;
+}
+
+static inline SemihostingTarget semihosting_get_target(void)
+{
+    return SEMIHOSTING_TARGET_AUTO;
+}
+
+static inline const char *semihosting_get_arg(int i)
+{
+    return NULL;
+}
+
+static inline int semihosting_get_argc(void)
+{
+    return 0;
+}
+
+static inline const char *semihosting_get_cmdline(void)
+{
+    return NULL;
+}
+#else
+bool semihosting_enabled(void);
+SemihostingTarget semihosting_get_target(void);
+const char *semihosting_get_arg(int i);
+int semihosting_get_argc(void);
+const char *semihosting_get_cmdline(void);
+#endif
+
+#endif
diff --git a/include/exec/softmmu-semi.h b/include/exec/softmmu-semi.h
index 8401f7d..7eefad8 100644
--- a/include/exec/softmmu-semi.h
+++ b/include/exec/softmmu-semi.h
@@ -6,17 +6,27 @@
  *
  * This code is licensed under the GPL
  */
-#ifndef SOFTMMU_SEMI_H
-#define SOFTMMU_SEMI_H 1
 
-static inline uint32_t softmmu_tget32(CPUArchState *env, uint32_t addr)
+#ifndef SOFTMMU_SEMI_H
+#define SOFTMMU_SEMI_H
+
+static inline uint64_t softmmu_tget64(CPUArchState *env, target_ulong addr)
+{
+    uint64_t val;
+
+    cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 8, 0);
+    return tswap64(val);
+}
+
+static inline uint32_t softmmu_tget32(CPUArchState *env, target_ulong addr)
 {
     uint32_t val;
 
     cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 4, 0);
     return tswap32(val);
 }
-static inline uint32_t softmmu_tget8(CPUArchState *env, uint32_t addr)
+
+static inline uint32_t softmmu_tget8(CPUArchState *env, target_ulong addr)
 {
     uint8_t val;
 
@@ -24,20 +34,30 @@
     return val;
 }
 
+#define get_user_u64(arg, p) ({ arg = softmmu_tget64(env, p); 0; })
 #define get_user_u32(arg, p) ({ arg = softmmu_tget32(env, p) ; 0; })
 #define get_user_u8(arg, p) ({ arg = softmmu_tget8(env, p) ; 0; })
 #define get_user_ual(arg, p) get_user_u32(arg, p)
 
-static inline void softmmu_tput32(CPUArchState *env, uint32_t addr, uint32_t val)
+static inline void softmmu_tput64(CPUArchState *env,
+                                  target_ulong addr, uint64_t val)
+{
+    val = tswap64(val);
+    cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 8, 1);
+}
+
+static inline void softmmu_tput32(CPUArchState *env,
+                                  target_ulong addr, uint32_t val)
 {
     val = tswap32(val);
     cpu_memory_rw_debug(ENV_GET_CPU(env), addr, (uint8_t *)&val, 4, 1);
 }
+#define put_user_u64(arg, p) ({ softmmu_tput64(env, p, arg) ; 0; })
 #define put_user_u32(arg, p) ({ softmmu_tput32(env, p, arg) ; 0; })
 #define put_user_ual(arg, p) put_user_u32(arg, p)
 
-static void *softmmu_lock_user(CPUArchState *env, uint32_t addr, uint32_t len,
-                               int copy)
+static void *softmmu_lock_user(CPUArchState *env,
+                               target_ulong addr, target_ulong len, int copy)
 {
     uint8_t *p;
     /* TODO: Make this something that isn't fixed size.  */
@@ -48,7 +68,7 @@
     return p;
 }
 #define lock_user(type, p, len, copy) softmmu_lock_user(env, p, len, copy)
-static char *softmmu_lock_user_string(CPUArchState *env, uint32_t addr)
+static char *softmmu_lock_user_string(CPUArchState *env, target_ulong addr)
 {
     char *p;
     char *s;
diff --git a/include/exec/spinlock.h b/include/exec/spinlock.h
deleted file mode 100644
index a72edda..0000000
--- a/include/exec/spinlock.h
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-
-/* configure guarantees us that we have pthreads on any host except
- * mingw32, which doesn't support any of the user-only targets.
- * So we can simply assume we have pthread mutexes here.
- */
-#if defined(CONFIG_USER_ONLY)
-
-#include <pthread.h>
-#define spin_lock pthread_mutex_lock
-#define spin_unlock pthread_mutex_unlock
-#define spinlock_t pthread_mutex_t
-#define SPIN_LOCK_UNLOCKED PTHREAD_MUTEX_INITIALIZER
-
-#else
-
-/* Empty implementations, on the theory that system mode emulation
- * is single-threaded. This means that these functions should only
- * be used from code run in the TCG cpu thread, and cannot protect
- * data structures which might also be accessed from the IO thread
- * or from signal handlers.
- */
-typedef int spinlock_t;
-#define SPIN_LOCK_UNLOCKED 0
-
-static inline void spin_lock(spinlock_t *lock)
-{
-}
-
-static inline void spin_unlock(spinlock_t *lock)
-{
-}
-
-#endif
diff --git a/include/exec/tb-context.h b/include/exec/tb-context.h
new file mode 100644
index 0000000..dce95d9
--- /dev/null
+++ b/include/exec/tb-context.h
@@ -0,0 +1,45 @@
+/*
+ * Internal structs that QEMU exports to TCG
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_TB_CONTEXT_H
+#define QEMU_TB_CONTEXT_H
+
+#include "qemu/thread.h"
+#include "qemu/qht.h"
+
+#define CODE_GEN_HTABLE_BITS     15
+#define CODE_GEN_HTABLE_SIZE     (1 << CODE_GEN_HTABLE_BITS)
+
+typedef struct TranslationBlock TranslationBlock;
+typedef struct TBContext TBContext;
+
+struct TBContext {
+
+    TranslationBlock *tbs;
+    struct qht htable;
+    int nb_tbs;
+    /* any access to the tbs or the page table must use this lock */
+    QemuMutex tb_lock;
+
+    /* statistics */
+    int tb_flush_count;
+    int tb_phys_invalidate_count;
+};
+
+#endif
diff --git a/include/exec/tb-hash-xx.h b/include/exec/tb-hash-xx.h
new file mode 100644
index 0000000..2c40b5c
--- /dev/null
+++ b/include/exec/tb-hash-xx.h
@@ -0,0 +1,95 @@
+/*
+ * xxHash - Fast Hash algorithm
+ * Copyright (C) 2012-2016, Yann Collet
+ *
+ * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * + Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * + Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * You can contact the author at :
+ * - xxHash source repository : https://github.com/Cyan4973/xxHash
+ */
+
+#ifndef EXEC_TB_HASH_XX_H
+#define EXEC_TB_HASH_XX_H
+
+#include "qemu/bitops.h"
+
+#define PRIME32_1   2654435761U
+#define PRIME32_2   2246822519U
+#define PRIME32_3   3266489917U
+#define PRIME32_4    668265263U
+#define PRIME32_5    374761393U
+
+#define TB_HASH_XX_SEED 1
+
+/*
+ * xxhash32, customized for input variables that are not guaranteed to be
+ * contiguous in memory.
+ */
+static inline
+uint32_t tb_hash_func5(uint64_t a0, uint64_t b0, uint32_t e)
+{
+    uint32_t v1 = TB_HASH_XX_SEED + PRIME32_1 + PRIME32_2;
+    uint32_t v2 = TB_HASH_XX_SEED + PRIME32_2;
+    uint32_t v3 = TB_HASH_XX_SEED + 0;
+    uint32_t v4 = TB_HASH_XX_SEED - PRIME32_1;
+    uint32_t a = a0 >> 32;
+    uint32_t b = a0;
+    uint32_t c = b0 >> 32;
+    uint32_t d = b0;
+    uint32_t h32;
+
+    v1 += a * PRIME32_2;
+    v1 = rol32(v1, 13);
+    v1 *= PRIME32_1;
+
+    v2 += b * PRIME32_2;
+    v2 = rol32(v2, 13);
+    v2 *= PRIME32_1;
+
+    v3 += c * PRIME32_2;
+    v3 = rol32(v3, 13);
+    v3 *= PRIME32_1;
+
+    v4 += d * PRIME32_2;
+    v4 = rol32(v4, 13);
+    v4 *= PRIME32_1;
+
+    h32 = rol32(v1, 1) + rol32(v2, 7) + rol32(v3, 12) + rol32(v4, 18);
+    h32 += 20;
+
+    h32 += e * PRIME32_3;
+    h32  = rol32(h32, 17) * PRIME32_4;
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+#endif /* EXEC_TB_HASH_XX_H */
diff --git a/include/exec/tb-hash.h b/include/exec/tb-hash.h
new file mode 100644
index 0000000..2c27490
--- /dev/null
+++ b/include/exec/tb-hash.h
@@ -0,0 +1,54 @@
+/*
+ * internal execution defines for qemu
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef EXEC_TB_HASH_H
+#define EXEC_TB_HASH_H
+
+#include "exec/tb-hash-xx.h"
+
+/* Only the bottom TB_JMP_PAGE_BITS of the jump cache hash bits vary for
+   addresses on the same page.  The top bits are the same.  This allows
+   TLB invalidation to quickly clear a subset of the hash table.  */
+#define TB_JMP_PAGE_BITS (TB_JMP_CACHE_BITS / 2)
+#define TB_JMP_PAGE_SIZE (1 << TB_JMP_PAGE_BITS)
+#define TB_JMP_ADDR_MASK (TB_JMP_PAGE_SIZE - 1)
+#define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE)
+
+static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc)
+{
+    target_ulong tmp;
+    tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
+    return (tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK;
+}
+
+static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc)
+{
+    target_ulong tmp;
+    tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS));
+    return (((tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK)
+           | (tmp & TB_JMP_ADDR_MASK));
+}
+
+static inline
+uint32_t tb_hash_func(tb_page_addr_t phys_pc, target_ulong pc, uint32_t flags)
+{
+    return tb_hash_func5(phys_pc, pc, flags);
+}
+
+#endif
diff --git a/include/exec/user/abitypes.h b/include/exec/user/abitypes.h
index 80eedac..ba18860 100644
--- a/include/exec/user/abitypes.h
+++ b/include/exec/user/abitypes.h
@@ -1,5 +1,6 @@
-#ifndef QEMU_TYPES_H
-#define QEMU_TYPES_H
+#ifndef EXEC_USER_ABITYPES_H
+#define EXEC_USER_ABITYPES_H
+
 #include "cpu.h"
 
 #ifdef TARGET_ABI32
@@ -14,6 +15,10 @@
 #define ABI_LLONG_ALIGNMENT 2
 #endif
 
+#if defined(TARGET_I386) && !defined(TARGET_X86_64)
+#define ABI_LLONG_ALIGNMENT 4
+#endif
+
 #ifndef ABI_SHORT_ALIGNMENT
 #define ABI_SHORT_ALIGNMENT 2
 #endif
diff --git a/include/exec/user/thunk.h b/include/exec/user/thunk.h
index 87025c3..f19ef4b 100644
--- a/include/exec/user/thunk.h
+++ b/include/exec/user/thunk.h
@@ -19,7 +19,6 @@
 #ifndef THUNK_H
 #define THUNK_H
 
-#include <inttypes.h>
 #include "cpu.h"
 
 /* types enums definitions */
@@ -61,10 +60,10 @@
 
 /* Translation table for bitmasks... */
 typedef struct bitmask_transtbl {
-	unsigned int	x86_mask;
-	unsigned int	x86_bits;
-	unsigned int	alpha_mask;
-	unsigned int	alpha_bits;
+    unsigned int target_mask;
+    unsigned int target_bits;
+    unsigned int host_mask;
+    unsigned int host_bits;
 } bitmask_transtbl;
 
 void thunk_register_struct(int id, const char *name, const argtype *types);
@@ -72,9 +71,8 @@
                                   const StructEntry *se1);
 const argtype *thunk_convert(void *dst, const void *src,
                              const argtype *type_ptr, int to_host);
-#ifndef NO_THUNK_TYPE_SIZE
 
-extern StructEntry struct_entries[];
+extern StructEntry *struct_entries;
 
 int thunk_type_size_array(const argtype *type_ptr, int is_host);
 int thunk_type_align_array(const argtype *type_ptr, int is_host);
@@ -179,11 +177,11 @@
     }
 }
 
-#endif /* NO_THUNK_TYPE_SIZE */
+unsigned int target_to_host_bitmask(unsigned int target_mask,
+                                    const bitmask_transtbl * trans_tbl);
+unsigned int host_to_target_bitmask(unsigned int host_mask,
+                                    const bitmask_transtbl * trans_tbl);
 
-unsigned int target_to_host_bitmask(unsigned int x86_mask,
-                                    const bitmask_transtbl * trans_tbl);
-unsigned int host_to_target_bitmask(unsigned int alpha_mask,
-                                    const bitmask_transtbl * trans_tbl);
+void thunk_init(unsigned int max_structs);
 
 #endif
diff --git a/include/fpu/softfloat.h b/include/fpu/softfloat.h
index e32e25d..1bde349 100644
--- a/include/fpu/softfloat.h
+++ b/include/fpu/softfloat.h
@@ -1,13 +1,24 @@
 /*
  * QEMU float support
  *
- * Derived from SoftFloat.
+ * The code in this source file is derived from release 2a of the SoftFloat
+ * IEC/IEEE Floating-point Arithmetic Package. Those parts of the code (and
+ * some later contributions) are provided under that license, as detailed below.
+ * It has subsequently been modified by contributors to the QEMU Project,
+ * so some portions are provided under:
+ *  the SoftFloat-2a license
+ *  the BSD license
+ *  GPL-v2-or-later
+ *
+ * Any future contributions to this file after December 1st 2014 will be
+ * taken to be licensed under the Softfloat-2a license unless specifically
+ * indicated otherwise.
  */
 
-/*============================================================================
-
-This C header file is part of the SoftFloat IEC/IEEE Floating-point Arithmetic
-Package, Release 2b.
+/*
+===============================================================================
+This C header file is part of the SoftFloat IEC/IEEE Floating-point
+Arithmetic Package, Release 2a.
 
 Written by John R. Hauser.  This work was made possible in part by the
 International Computer Science Institute, located at Suite 600, 1947 Center
@@ -16,24 +27,57 @@
 of this code was written as part of a project to build a fixed-point vector
 processor in collaboration with the University of California at Berkeley,
 overseen by Profs. Nelson Morgan and John Wawrzynek.  More information
-is available through the Web page `http://www.cs.berkeley.edu/~jhauser/
+is available through the Web page `http://HTTP.CS.Berkeley.EDU/~jhauser/
 arithmetic/SoftFloat.html'.
 
-THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort has
-been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT TIMES
-RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO PERSONS
-AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ALL LOSSES,
-COSTS, OR OTHER PROBLEMS THEY INCUR DUE TO THE SOFTWARE, AND WHO FURTHERMORE
-EFFECTIVELY INDEMNIFY JOHN HAUSER AND THE INTERNATIONAL COMPUTER SCIENCE
-INSTITUTE (possibly via similar legal warning) AGAINST ALL LOSSES, COSTS, OR
-OTHER PROBLEMS INCURRED BY THEIR CUSTOMERS AND CLIENTS DUE TO THE SOFTWARE.
+THIS SOFTWARE IS DISTRIBUTED AS IS, FOR FREE.  Although reasonable effort
+has been made to avoid it, THIS SOFTWARE MAY CONTAIN FAULTS THAT WILL AT
+TIMES RESULT IN INCORRECT BEHAVIOR.  USE OF THIS SOFTWARE IS RESTRICTED TO
+PERSONS AND ORGANIZATIONS WHO CAN AND WILL TAKE FULL RESPONSIBILITY FOR ANY
+AND ALL LOSSES, COSTS, OR OTHER PROBLEMS ARISING FROM ITS USE.
 
 Derivative works are acceptable, even for commercial purposes, so long as
-(1) the source code for the derivative work includes prominent notice that
-the work is derivative, and (2) the source code includes prominent notice with
-these four paragraphs for those parts of this code that are retained.
+(1) they include prominent notice that the work is derivative, and (2) they
+include prominent notice akin to these four paragraphs for those parts of
+this code that are retained.
 
-=============================================================================*/
+===============================================================================
+*/
+
+/* BSD licensing:
+ * Copyright (c) 2006, Fabrice Bellard
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ * may be used to endorse or promote products derived from this software without
+ * specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* Portions of this work are licensed under the terms of the GNU GPL,
+ * version 2 or later. See the COPYING file in the top-level directory.
+ */
 
 #ifndef SOFTFLOAT_H
 #define SOFTFLOAT_H
@@ -42,32 +86,15 @@
 #include <sunmath.h>
 #endif
 
-#include <inttypes.h>
-#include "config-host.h"
-#include "qemu/osdep.h"
 
-/*----------------------------------------------------------------------------
-| Each of the following `typedef's defines the most convenient type that holds
-| integers of at least as many bits as specified.  For example, `uint8' should
-| be the most convenient type that can hold unsigned integers of as many as
-| 8 bits.  The `flag' type must be able to hold either a 0 or 1.  For most
-| implementations of C, `flag', `uint8', and `int8' should all be `typedef'ed
-| to the same as `int'.
-*----------------------------------------------------------------------------*/
+/* This 'flag' type must be able to hold at least 0 and 1. It should
+ * probably be replaced with 'bool' but the uses would need to be audited
+ * to check that they weren't accidentally relying on it being a larger type.
+ */
 typedef uint8_t flag;
-typedef uint8_t uint8;
-typedef int8_t int8;
-typedef unsigned int uint32;
-typedef signed int int32;
-typedef uint64_t uint64;
-typedef int64_t int64;
 
 #define LIT64( a ) a##LL
 
-#define STATUS_PARAM , float_status *status
-#define STATUS(field) status->field
-#define STATUS_VAR , status
-
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE floating-point ordering relations
 *----------------------------------------------------------------------------*/
@@ -171,84 +198,90 @@
 typedef struct float_status {
     signed char float_detect_tininess;
     signed char float_rounding_mode;
-    signed char float_exception_flags;
+    uint8_t     float_exception_flags;
     signed char floatx80_rounding_precision;
     /* should denormalised results go to zero and set the inexact flag? */
     flag flush_to_zero;
     /* should denormalised inputs go to zero and set the input_denormal flag? */
     flag flush_inputs_to_zero;
     flag default_nan_mode;
+    flag snan_bit_is_one;
 } float_status;
 
-static inline void set_float_detect_tininess(int val STATUS_PARAM)
+static inline void set_float_detect_tininess(int val, float_status *status)
 {
-    STATUS(float_detect_tininess) = val;
+    status->float_detect_tininess = val;
 }
-static inline void set_float_rounding_mode(int val STATUS_PARAM)
+static inline void set_float_rounding_mode(int val, float_status *status)
 {
-    STATUS(float_rounding_mode) = val;
+    status->float_rounding_mode = val;
 }
-static inline void set_float_exception_flags(int val STATUS_PARAM)
+static inline void set_float_exception_flags(int val, float_status *status)
 {
-    STATUS(float_exception_flags) = val;
+    status->float_exception_flags = val;
 }
-static inline void set_floatx80_rounding_precision(int val STATUS_PARAM)
+static inline void set_floatx80_rounding_precision(int val,
+                                                   float_status *status)
 {
-    STATUS(floatx80_rounding_precision) = val;
+    status->floatx80_rounding_precision = val;
 }
-static inline void set_flush_to_zero(flag val STATUS_PARAM)
+static inline void set_flush_to_zero(flag val, float_status *status)
 {
-    STATUS(flush_to_zero) = val;
+    status->flush_to_zero = val;
 }
-static inline void set_flush_inputs_to_zero(flag val STATUS_PARAM)
+static inline void set_flush_inputs_to_zero(flag val, float_status *status)
 {
-    STATUS(flush_inputs_to_zero) = val;
+    status->flush_inputs_to_zero = val;
 }
-static inline void set_default_nan_mode(flag val STATUS_PARAM)
+static inline void set_default_nan_mode(flag val, float_status *status)
 {
-    STATUS(default_nan_mode) = val;
+    status->default_nan_mode = val;
+}
+static inline void set_snan_bit_is_one(flag val, float_status *status)
+{
+    status->snan_bit_is_one = val;
 }
 static inline int get_float_detect_tininess(float_status *status)
 {
-    return STATUS(float_detect_tininess);
+    return status->float_detect_tininess;
 }
 static inline int get_float_rounding_mode(float_status *status)
 {
-    return STATUS(float_rounding_mode);
+    return status->float_rounding_mode;
 }
 static inline int get_float_exception_flags(float_status *status)
 {
-    return STATUS(float_exception_flags);
+    return status->float_exception_flags;
 }
 static inline int get_floatx80_rounding_precision(float_status *status)
 {
-    return STATUS(floatx80_rounding_precision);
+    return status->floatx80_rounding_precision;
 }
 static inline flag get_flush_to_zero(float_status *status)
 {
-    return STATUS(flush_to_zero);
+    return status->flush_to_zero;
 }
 static inline flag get_flush_inputs_to_zero(float_status *status)
 {
-    return STATUS(flush_inputs_to_zero);
+    return status->flush_inputs_to_zero;
 }
 static inline flag get_default_nan_mode(float_status *status)
 {
-    return STATUS(default_nan_mode);
+    return status->default_nan_mode;
 }
 
 /*----------------------------------------------------------------------------
 | Routine to raise any or all of the software IEC/IEEE floating-point
 | exception flags.
 *----------------------------------------------------------------------------*/
-void float_raise( int8 flags STATUS_PARAM);
+void float_raise(uint8_t flags, float_status *status);
 
 /*----------------------------------------------------------------------------
 | If `a' is denormal and we are in flush-to-zero mode then set the
 | input-denormal exception and return zero. Otherwise just return the value.
 *----------------------------------------------------------------------------*/
-float32 float32_squash_input_denormal(float32 a STATUS_PARAM);
-float64 float64_squash_input_denormal(float64 a STATUS_PARAM);
+float32 float32_squash_input_denormal(float32 a, float_status *status);
+float64 float64_squash_input_denormal(float64 a, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Options to indicate which negations to perform in float*_muladd()
@@ -268,55 +301,55 @@
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE integer-to-floating-point conversion routines.
 *----------------------------------------------------------------------------*/
-float32 int32_to_float32(int32_t STATUS_PARAM);
-float64 int32_to_float64(int32_t STATUS_PARAM);
-float32 uint32_to_float32(uint32_t STATUS_PARAM);
-float64 uint32_to_float64(uint32_t STATUS_PARAM);
-floatx80 int32_to_floatx80(int32_t STATUS_PARAM);
-float128 int32_to_float128(int32_t STATUS_PARAM);
-float32 int64_to_float32(int64_t STATUS_PARAM);
-float32 uint64_to_float32(uint64_t STATUS_PARAM);
-float64 int64_to_float64(int64_t STATUS_PARAM);
-float64 uint64_to_float64(uint64_t STATUS_PARAM);
-floatx80 int64_to_floatx80(int64_t STATUS_PARAM);
-float128 int64_to_float128(int64_t STATUS_PARAM);
-float128 uint64_to_float128(uint64_t STATUS_PARAM);
+float32 int32_to_float32(int32_t, float_status *status);
+float64 int32_to_float64(int32_t, float_status *status);
+float32 uint32_to_float32(uint32_t, float_status *status);
+float64 uint32_to_float64(uint32_t, float_status *status);
+floatx80 int32_to_floatx80(int32_t, float_status *status);
+float128 int32_to_float128(int32_t, float_status *status);
+float32 int64_to_float32(int64_t, float_status *status);
+float64 int64_to_float64(int64_t, float_status *status);
+floatx80 int64_to_floatx80(int64_t, float_status *status);
+float128 int64_to_float128(int64_t, float_status *status);
+float32 uint64_to_float32(uint64_t, float_status *status);
+float64 uint64_to_float64(uint64_t, float_status *status);
+float128 uint64_to_float128(uint64_t, float_status *status);
 
 /* We provide the int16 versions for symmetry of API with float-to-int */
-static inline float32 int16_to_float32(int16_t v STATUS_PARAM)
+static inline float32 int16_to_float32(int16_t v, float_status *status)
 {
-    return int32_to_float32(v STATUS_VAR);
+    return int32_to_float32(v, status);
 }
 
-static inline float32 uint16_to_float32(uint16_t v STATUS_PARAM)
+static inline float32 uint16_to_float32(uint16_t v, float_status *status)
 {
-    return uint32_to_float32(v STATUS_VAR);
+    return uint32_to_float32(v, status);
 }
 
-static inline float64 int16_to_float64(int16_t v STATUS_PARAM)
+static inline float64 int16_to_float64(int16_t v, float_status *status)
 {
-    return int32_to_float64(v STATUS_VAR);
+    return int32_to_float64(v, status);
 }
 
-static inline float64 uint16_to_float64(uint16_t v STATUS_PARAM)
+static inline float64 uint16_to_float64(uint16_t v, float_status *status)
 {
-    return uint32_to_float64(v STATUS_VAR);
+    return uint32_to_float64(v, status);
 }
 
 /*----------------------------------------------------------------------------
 | Software half-precision conversion routines.
 *----------------------------------------------------------------------------*/
-float16 float32_to_float16( float32, flag STATUS_PARAM );
-float32 float16_to_float32( float16, flag STATUS_PARAM );
-float16 float64_to_float16(float64 a, flag ieee STATUS_PARAM);
-float64 float16_to_float64(float16 a, flag ieee STATUS_PARAM);
+float16 float32_to_float16(float32, flag, float_status *status);
+float32 float16_to_float32(float16, flag, float_status *status);
+float16 float64_to_float16(float64 a, flag ieee, float_status *status);
+float64 float16_to_float64(float16 a, flag ieee, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software half-precision operations.
 *----------------------------------------------------------------------------*/
-int float16_is_quiet_nan( float16 );
-int float16_is_signaling_nan( float16 );
-float16 float16_maybe_silence_nan( float16 );
+int float16_is_quiet_nan(float16, float_status *status);
+int float16_is_signaling_nan(float16, float_status *status);
+float16 float16_maybe_silence_nan(float16, float_status *status);
 
 static inline int float16_is_any_nan(float16 a)
 {
@@ -326,60 +359,60 @@
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
-extern const float16 float16_default_nan;
+float16 float16_default_nan(float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE single-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int_fast16_t float32_to_int16(float32 STATUS_PARAM);
-uint_fast16_t float32_to_uint16(float32 STATUS_PARAM);
-int_fast16_t float32_to_int16_round_to_zero(float32 STATUS_PARAM);
-uint_fast16_t float32_to_uint16_round_to_zero(float32 STATUS_PARAM);
-int32 float32_to_int32( float32 STATUS_PARAM );
-int32 float32_to_int32_round_to_zero( float32 STATUS_PARAM );
-uint32 float32_to_uint32( float32 STATUS_PARAM );
-uint32 float32_to_uint32_round_to_zero( float32 STATUS_PARAM );
-int64 float32_to_int64( float32 STATUS_PARAM );
-uint64 float32_to_uint64(float32 STATUS_PARAM);
-uint64 float32_to_uint64_round_to_zero(float32 STATUS_PARAM);
-int64 float32_to_int64_round_to_zero( float32 STATUS_PARAM );
-float64 float32_to_float64( float32 STATUS_PARAM );
-floatx80 float32_to_floatx80( float32 STATUS_PARAM );
-float128 float32_to_float128( float32 STATUS_PARAM );
+int16_t float32_to_int16(float32, float_status *status);
+uint16_t float32_to_uint16(float32, float_status *status);
+int16_t float32_to_int16_round_to_zero(float32, float_status *status);
+uint16_t float32_to_uint16_round_to_zero(float32, float_status *status);
+int32_t float32_to_int32(float32, float_status *status);
+int32_t float32_to_int32_round_to_zero(float32, float_status *status);
+uint32_t float32_to_uint32(float32, float_status *status);
+uint32_t float32_to_uint32_round_to_zero(float32, float_status *status);
+int64_t float32_to_int64(float32, float_status *status);
+uint64_t float32_to_uint64(float32, float_status *status);
+uint64_t float32_to_uint64_round_to_zero(float32, float_status *status);
+int64_t float32_to_int64_round_to_zero(float32, float_status *status);
+float64 float32_to_float64(float32, float_status *status);
+floatx80 float32_to_floatx80(float32, float_status *status);
+float128 float32_to_float128(float32, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE single-precision operations.
 *----------------------------------------------------------------------------*/
-float32 float32_round_to_int( float32 STATUS_PARAM );
-float32 float32_add( float32, float32 STATUS_PARAM );
-float32 float32_sub( float32, float32 STATUS_PARAM );
-float32 float32_mul( float32, float32 STATUS_PARAM );
-float32 float32_div( float32, float32 STATUS_PARAM );
-float32 float32_rem( float32, float32 STATUS_PARAM );
-float32 float32_muladd(float32, float32, float32, int STATUS_PARAM);
-float32 float32_sqrt( float32 STATUS_PARAM );
-float32 float32_exp2( float32 STATUS_PARAM );
-float32 float32_log2( float32 STATUS_PARAM );
-int float32_eq( float32, float32 STATUS_PARAM );
-int float32_le( float32, float32 STATUS_PARAM );
-int float32_lt( float32, float32 STATUS_PARAM );
-int float32_unordered( float32, float32 STATUS_PARAM );
-int float32_eq_quiet( float32, float32 STATUS_PARAM );
-int float32_le_quiet( float32, float32 STATUS_PARAM );
-int float32_lt_quiet( float32, float32 STATUS_PARAM );
-int float32_unordered_quiet( float32, float32 STATUS_PARAM );
-int float32_compare( float32, float32 STATUS_PARAM );
-int float32_compare_quiet( float32, float32 STATUS_PARAM );
-float32 float32_min(float32, float32 STATUS_PARAM);
-float32 float32_max(float32, float32 STATUS_PARAM);
-float32 float32_minnum(float32, float32 STATUS_PARAM);
-float32 float32_maxnum(float32, float32 STATUS_PARAM);
-float32 float32_minnummag(float32, float32 STATUS_PARAM);
-float32 float32_maxnummag(float32, float32 STATUS_PARAM);
-int float32_is_quiet_nan( float32 );
-int float32_is_signaling_nan( float32 );
-float32 float32_maybe_silence_nan( float32 );
-float32 float32_scalbn( float32, int STATUS_PARAM );
+float32 float32_round_to_int(float32, float_status *status);
+float32 float32_add(float32, float32, float_status *status);
+float32 float32_sub(float32, float32, float_status *status);
+float32 float32_mul(float32, float32, float_status *status);
+float32 float32_div(float32, float32, float_status *status);
+float32 float32_rem(float32, float32, float_status *status);
+float32 float32_muladd(float32, float32, float32, int, float_status *status);
+float32 float32_sqrt(float32, float_status *status);
+float32 float32_exp2(float32, float_status *status);
+float32 float32_log2(float32, float_status *status);
+int float32_eq(float32, float32, float_status *status);
+int float32_le(float32, float32, float_status *status);
+int float32_lt(float32, float32, float_status *status);
+int float32_unordered(float32, float32, float_status *status);
+int float32_eq_quiet(float32, float32, float_status *status);
+int float32_le_quiet(float32, float32, float_status *status);
+int float32_lt_quiet(float32, float32, float_status *status);
+int float32_unordered_quiet(float32, float32, float_status *status);
+int float32_compare(float32, float32, float_status *status);
+int float32_compare_quiet(float32, float32, float_status *status);
+float32 float32_min(float32, float32, float_status *status);
+float32 float32_max(float32, float32, float_status *status);
+float32 float32_minnum(float32, float32, float_status *status);
+float32 float32_maxnum(float32, float32, float_status *status);
+float32 float32_minnummag(float32, float32, float_status *status);
+float32 float32_maxnummag(float32, float32, float_status *status);
+int float32_is_quiet_nan(float32, float_status *status);
+int float32_is_signaling_nan(float32, float_status *status);
+float32 float32_maybe_silence_nan(float32, float_status *status);
+float32 float32_scalbn(float32, int, float_status *status);
 
 static inline float32 float32_abs(float32 a)
 {
@@ -438,60 +471,60 @@
 /*----------------------------------------------------------------------------
 | The pattern for a default generated single-precision NaN.
 *----------------------------------------------------------------------------*/
-extern const float32 float32_default_nan;
+float32 float32_default_nan(float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE double-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int_fast16_t float64_to_int16(float64 STATUS_PARAM);
-uint_fast16_t float64_to_uint16(float64 STATUS_PARAM);
-int_fast16_t float64_to_int16_round_to_zero(float64 STATUS_PARAM);
-uint_fast16_t float64_to_uint16_round_to_zero(float64 STATUS_PARAM);
-int32 float64_to_int32( float64 STATUS_PARAM );
-int32 float64_to_int32_round_to_zero( float64 STATUS_PARAM );
-uint32 float64_to_uint32( float64 STATUS_PARAM );
-uint32 float64_to_uint32_round_to_zero( float64 STATUS_PARAM );
-int64 float64_to_int64( float64 STATUS_PARAM );
-int64 float64_to_int64_round_to_zero( float64 STATUS_PARAM );
-uint64 float64_to_uint64 (float64 a STATUS_PARAM);
-uint64 float64_to_uint64_round_to_zero (float64 a STATUS_PARAM);
-float32 float64_to_float32( float64 STATUS_PARAM );
-floatx80 float64_to_floatx80( float64 STATUS_PARAM );
-float128 float64_to_float128( float64 STATUS_PARAM );
+int16_t float64_to_int16(float64, float_status *status);
+uint16_t float64_to_uint16(float64, float_status *status);
+int16_t float64_to_int16_round_to_zero(float64, float_status *status);
+uint16_t float64_to_uint16_round_to_zero(float64, float_status *status);
+int32_t float64_to_int32(float64, float_status *status);
+int32_t float64_to_int32_round_to_zero(float64, float_status *status);
+uint32_t float64_to_uint32(float64, float_status *status);
+uint32_t float64_to_uint32_round_to_zero(float64, float_status *status);
+int64_t float64_to_int64(float64, float_status *status);
+int64_t float64_to_int64_round_to_zero(float64, float_status *status);
+uint64_t float64_to_uint64(float64 a, float_status *status);
+uint64_t float64_to_uint64_round_to_zero(float64 a, float_status *status);
+float32 float64_to_float32(float64, float_status *status);
+floatx80 float64_to_floatx80(float64, float_status *status);
+float128 float64_to_float128(float64, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE double-precision operations.
 *----------------------------------------------------------------------------*/
-float64 float64_round_to_int( float64 STATUS_PARAM );
-float64 float64_trunc_to_int( float64 STATUS_PARAM );
-float64 float64_add( float64, float64 STATUS_PARAM );
-float64 float64_sub( float64, float64 STATUS_PARAM );
-float64 float64_mul( float64, float64 STATUS_PARAM );
-float64 float64_div( float64, float64 STATUS_PARAM );
-float64 float64_rem( float64, float64 STATUS_PARAM );
-float64 float64_muladd(float64, float64, float64, int STATUS_PARAM);
-float64 float64_sqrt( float64 STATUS_PARAM );
-float64 float64_log2( float64 STATUS_PARAM );
-int float64_eq( float64, float64 STATUS_PARAM );
-int float64_le( float64, float64 STATUS_PARAM );
-int float64_lt( float64, float64 STATUS_PARAM );
-int float64_unordered( float64, float64 STATUS_PARAM );
-int float64_eq_quiet( float64, float64 STATUS_PARAM );
-int float64_le_quiet( float64, float64 STATUS_PARAM );
-int float64_lt_quiet( float64, float64 STATUS_PARAM );
-int float64_unordered_quiet( float64, float64 STATUS_PARAM );
-int float64_compare( float64, float64 STATUS_PARAM );
-int float64_compare_quiet( float64, float64 STATUS_PARAM );
-float64 float64_min(float64, float64 STATUS_PARAM);
-float64 float64_max(float64, float64 STATUS_PARAM);
-float64 float64_minnum(float64, float64 STATUS_PARAM);
-float64 float64_maxnum(float64, float64 STATUS_PARAM);
-float64 float64_minnummag(float64, float64 STATUS_PARAM);
-float64 float64_maxnummag(float64, float64 STATUS_PARAM);
-int float64_is_quiet_nan( float64 a );
-int float64_is_signaling_nan( float64 );
-float64 float64_maybe_silence_nan( float64 );
-float64 float64_scalbn( float64, int STATUS_PARAM );
+float64 float64_round_to_int(float64, float_status *status);
+float64 float64_trunc_to_int(float64, float_status *status);
+float64 float64_add(float64, float64, float_status *status);
+float64 float64_sub(float64, float64, float_status *status);
+float64 float64_mul(float64, float64, float_status *status);
+float64 float64_div(float64, float64, float_status *status);
+float64 float64_rem(float64, float64, float_status *status);
+float64 float64_muladd(float64, float64, float64, int, float_status *status);
+float64 float64_sqrt(float64, float_status *status);
+float64 float64_log2(float64, float_status *status);
+int float64_eq(float64, float64, float_status *status);
+int float64_le(float64, float64, float_status *status);
+int float64_lt(float64, float64, float_status *status);
+int float64_unordered(float64, float64, float_status *status);
+int float64_eq_quiet(float64, float64, float_status *status);
+int float64_le_quiet(float64, float64, float_status *status);
+int float64_lt_quiet(float64, float64, float_status *status);
+int float64_unordered_quiet(float64, float64, float_status *status);
+int float64_compare(float64, float64, float_status *status);
+int float64_compare_quiet(float64, float64, float_status *status);
+float64 float64_min(float64, float64, float_status *status);
+float64 float64_max(float64, float64, float_status *status);
+float64 float64_minnum(float64, float64, float_status *status);
+float64 float64_maxnum(float64, float64, float_status *status);
+float64 float64_minnummag(float64, float64, float_status *status);
+float64 float64_maxnummag(float64, float64, float_status *status);
+int float64_is_quiet_nan(float64 a, float_status *status);
+int float64_is_signaling_nan(float64, float_status *status);
+float64 float64_maybe_silence_nan(float64, float_status *status);
+float64 float64_scalbn(float64, int, float_status *status);
 
 static inline float64 float64_abs(float64 a)
 {
@@ -550,43 +583,43 @@
 /*----------------------------------------------------------------------------
 | The pattern for a default generated double-precision NaN.
 *----------------------------------------------------------------------------*/
-extern const float64 float64_default_nan;
+float64 float64_default_nan(float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE extended double-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int32 floatx80_to_int32( floatx80 STATUS_PARAM );
-int32 floatx80_to_int32_round_to_zero( floatx80 STATUS_PARAM );
-int64 floatx80_to_int64( floatx80 STATUS_PARAM );
-int64 floatx80_to_int64_round_to_zero( floatx80 STATUS_PARAM );
-float32 floatx80_to_float32( floatx80 STATUS_PARAM );
-float64 floatx80_to_float64( floatx80 STATUS_PARAM );
-float128 floatx80_to_float128( floatx80 STATUS_PARAM );
+int32_t floatx80_to_int32(floatx80, float_status *status);
+int32_t floatx80_to_int32_round_to_zero(floatx80, float_status *status);
+int64_t floatx80_to_int64(floatx80, float_status *status);
+int64_t floatx80_to_int64_round_to_zero(floatx80, float_status *status);
+float32 floatx80_to_float32(floatx80, float_status *status);
+float64 floatx80_to_float64(floatx80, float_status *status);
+float128 floatx80_to_float128(floatx80, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE extended double-precision operations.
 *----------------------------------------------------------------------------*/
-floatx80 floatx80_round_to_int( floatx80 STATUS_PARAM );
-floatx80 floatx80_add( floatx80, floatx80 STATUS_PARAM );
-floatx80 floatx80_sub( floatx80, floatx80 STATUS_PARAM );
-floatx80 floatx80_mul( floatx80, floatx80 STATUS_PARAM );
-floatx80 floatx80_div( floatx80, floatx80 STATUS_PARAM );
-floatx80 floatx80_rem( floatx80, floatx80 STATUS_PARAM );
-floatx80 floatx80_sqrt( floatx80 STATUS_PARAM );
-int floatx80_eq( floatx80, floatx80 STATUS_PARAM );
-int floatx80_le( floatx80, floatx80 STATUS_PARAM );
-int floatx80_lt( floatx80, floatx80 STATUS_PARAM );
-int floatx80_unordered( floatx80, floatx80 STATUS_PARAM );
-int floatx80_eq_quiet( floatx80, floatx80 STATUS_PARAM );
-int floatx80_le_quiet( floatx80, floatx80 STATUS_PARAM );
-int floatx80_lt_quiet( floatx80, floatx80 STATUS_PARAM );
-int floatx80_unordered_quiet( floatx80, floatx80 STATUS_PARAM );
-int floatx80_compare( floatx80, floatx80 STATUS_PARAM );
-int floatx80_compare_quiet( floatx80, floatx80 STATUS_PARAM );
-int floatx80_is_quiet_nan( floatx80 );
-int floatx80_is_signaling_nan( floatx80 );
-floatx80 floatx80_maybe_silence_nan( floatx80 );
-floatx80 floatx80_scalbn( floatx80, int STATUS_PARAM );
+floatx80 floatx80_round_to_int(floatx80, float_status *status);
+floatx80 floatx80_add(floatx80, floatx80, float_status *status);
+floatx80 floatx80_sub(floatx80, floatx80, float_status *status);
+floatx80 floatx80_mul(floatx80, floatx80, float_status *status);
+floatx80 floatx80_div(floatx80, floatx80, float_status *status);
+floatx80 floatx80_rem(floatx80, floatx80, float_status *status);
+floatx80 floatx80_sqrt(floatx80, float_status *status);
+int floatx80_eq(floatx80, floatx80, float_status *status);
+int floatx80_le(floatx80, floatx80, float_status *status);
+int floatx80_lt(floatx80, floatx80, float_status *status);
+int floatx80_unordered(floatx80, floatx80, float_status *status);
+int floatx80_eq_quiet(floatx80, floatx80, float_status *status);
+int floatx80_le_quiet(floatx80, floatx80, float_status *status);
+int floatx80_lt_quiet(floatx80, floatx80, float_status *status);
+int floatx80_unordered_quiet(floatx80, floatx80, float_status *status);
+int floatx80_compare(floatx80, floatx80, float_status *status);
+int floatx80_compare_quiet(floatx80, floatx80, float_status *status);
+int floatx80_is_quiet_nan(floatx80, float_status *status);
+int floatx80_is_signaling_nan(floatx80, float_status *status);
+floatx80 floatx80_maybe_silence_nan(floatx80, float_status *status);
+floatx80 floatx80_scalbn(floatx80, int, float_status *status);
 
 static inline floatx80 floatx80_abs(floatx80 a)
 {
@@ -635,43 +668,43 @@
 /*----------------------------------------------------------------------------
 | The pattern for a default generated extended double-precision NaN.
 *----------------------------------------------------------------------------*/
-extern const floatx80 floatx80_default_nan;
+floatx80 floatx80_default_nan(float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE quadruple-precision conversion routines.
 *----------------------------------------------------------------------------*/
-int32 float128_to_int32( float128 STATUS_PARAM );
-int32 float128_to_int32_round_to_zero( float128 STATUS_PARAM );
-int64 float128_to_int64( float128 STATUS_PARAM );
-int64 float128_to_int64_round_to_zero( float128 STATUS_PARAM );
-float32 float128_to_float32( float128 STATUS_PARAM );
-float64 float128_to_float64( float128 STATUS_PARAM );
-floatx80 float128_to_floatx80( float128 STATUS_PARAM );
+int32_t float128_to_int32(float128, float_status *status);
+int32_t float128_to_int32_round_to_zero(float128, float_status *status);
+int64_t float128_to_int64(float128, float_status *status);
+int64_t float128_to_int64_round_to_zero(float128, float_status *status);
+float32 float128_to_float32(float128, float_status *status);
+float64 float128_to_float64(float128, float_status *status);
+floatx80 float128_to_floatx80(float128, float_status *status);
 
 /*----------------------------------------------------------------------------
 | Software IEC/IEEE quadruple-precision operations.
 *----------------------------------------------------------------------------*/
-float128 float128_round_to_int( float128 STATUS_PARAM );
-float128 float128_add( float128, float128 STATUS_PARAM );
-float128 float128_sub( float128, float128 STATUS_PARAM );
-float128 float128_mul( float128, float128 STATUS_PARAM );
-float128 float128_div( float128, float128 STATUS_PARAM );
-float128 float128_rem( float128, float128 STATUS_PARAM );
-float128 float128_sqrt( float128 STATUS_PARAM );
-int float128_eq( float128, float128 STATUS_PARAM );
-int float128_le( float128, float128 STATUS_PARAM );
-int float128_lt( float128, float128 STATUS_PARAM );
-int float128_unordered( float128, float128 STATUS_PARAM );
-int float128_eq_quiet( float128, float128 STATUS_PARAM );
-int float128_le_quiet( float128, float128 STATUS_PARAM );
-int float128_lt_quiet( float128, float128 STATUS_PARAM );
-int float128_unordered_quiet( float128, float128 STATUS_PARAM );
-int float128_compare( float128, float128 STATUS_PARAM );
-int float128_compare_quiet( float128, float128 STATUS_PARAM );
-int float128_is_quiet_nan( float128 );
-int float128_is_signaling_nan( float128 );
-float128 float128_maybe_silence_nan( float128 );
-float128 float128_scalbn( float128, int STATUS_PARAM );
+float128 float128_round_to_int(float128, float_status *status);
+float128 float128_add(float128, float128, float_status *status);
+float128 float128_sub(float128, float128, float_status *status);
+float128 float128_mul(float128, float128, float_status *status);
+float128 float128_div(float128, float128, float_status *status);
+float128 float128_rem(float128, float128, float_status *status);
+float128 float128_sqrt(float128, float_status *status);
+int float128_eq(float128, float128, float_status *status);
+int float128_le(float128, float128, float_status *status);
+int float128_lt(float128, float128, float_status *status);
+int float128_unordered(float128, float128, float_status *status);
+int float128_eq_quiet(float128, float128, float_status *status);
+int float128_le_quiet(float128, float128, float_status *status);
+int float128_lt_quiet(float128, float128, float_status *status);
+int float128_unordered_quiet(float128, float128, float_status *status);
+int float128_compare(float128, float128, float_status *status);
+int float128_compare_quiet(float128, float128, float_status *status);
+int float128_is_quiet_nan(float128, float_status *status);
+int float128_is_signaling_nan(float128, float_status *status);
+float128 float128_maybe_silence_nan(float128, float_status *status);
+float128 float128_scalbn(float128, int, float_status *status);
 
 static inline float128 float128_abs(float128 a)
 {
@@ -716,6 +749,6 @@
 /*----------------------------------------------------------------------------
 | The pattern for a default generated quadruple-precision NaN.
 *----------------------------------------------------------------------------*/
-extern const float128 float128_default_nan;
+float128 float128_default_nan(float_status *status);
 
-#endif /* !SOFTFLOAT_H */
+#endif /* SOFTFLOAT_H */
diff --git a/include/glib-compat.h b/include/glib-compat.h
index f0615c9..8d5a7f3 100644
--- a/include/glib-compat.h
+++ b/include/glib-compat.h
@@ -23,16 +23,8 @@
 #define G_TIME_SPAN_SECOND              (G_GINT64_CONSTANT(1000000))
 #endif
 
-#if !GLIB_CHECK_VERSION(2, 14, 0)
-static inline guint g_timeout_add_seconds(guint interval, GSourceFunc function,
-                                          gpointer data)
-{
-    return g_timeout_add(interval * 1000, function, data);
-}
-#endif
-
 #if !GLIB_CHECK_VERSION(2, 28, 0)
-static inline gint64 g_get_monotonic_time(void)
+static inline gint64 qemu_g_get_monotonic_time(void)
 {
     /* g_get_monotonic_time() is best-effort so we can use the wall clock as a
      * fallback.
@@ -43,23 +35,8 @@
 
     return time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
 }
-#endif
-
-#if !GLIB_CHECK_VERSION(2, 16, 0)
-static inline int g_strcmp0(const char *str1, const char *str2)
-{
-    int result;
-
-    if (!str1) {
-        result = -(str1 != str2);
-    } else if (!str2) {
-        result = (str1 != str2);
-    } else {
-        result = strcmp(str1, str2);
-    }
-
-    return result;
-}
+/* work around distro backports of this interface */
+#define g_get_monotonic_time() qemu_g_get_monotonic_time()
 #endif
 
 #ifdef _WIN32
@@ -69,18 +46,28 @@
  */
 #define g_poll(fds, nfds, timeout) g_poll_fixed(fds, nfds, timeout)
 gint g_poll_fixed(GPollFD *fds, guint nfds, gint timeout);
-#elif !GLIB_CHECK_VERSION(2, 20, 0)
-/*
- * Glib before 2.20.0 doesn't implement g_poll, so wrap it to compile properly
- * on older systems.
- */
-static inline gint g_poll(GPollFD *fds, guint nfds, gint timeout)
-{
-    GMainContext *ctx = g_main_context_default();
-    return g_main_context_get_poll_func(ctx)(fds, nfds, timeout);
-}
 #endif
 
+#if !GLIB_CHECK_VERSION(2, 30, 0)
+/* Not a 100% compatible implementation, but good enough for most
+ * cases. Placeholders are only supported at the end of the
+ * template. */
+static inline gchar *qemu_g_dir_make_tmp(gchar const *tmpl, GError **error)
+{
+    gchar *path = g_build_filename(g_get_tmp_dir(), tmpl ?: ".XXXXXX", NULL);
+
+    if (mkdtemp(path) != NULL) {
+        return path;
+    }
+    /* Error occurred, clean up. */
+    g_set_error(error, G_FILE_ERROR, g_file_error_from_errno(errno),
+                "mkdtemp() failed");
+    g_free(path);
+    return NULL;
+}
+#define g_dir_make_tmp(tmpl, error) qemu_g_dir_make_tmp(tmpl, error)
+#endif /* glib 2.30 */
+
 #if !GLIB_CHECK_VERSION(2, 31, 0)
 /* before glib-2.31, GMutex and GCond was dynamic-only (there was a separate
  * GStaticMutex, but it didn't work with condition variables).
@@ -113,7 +100,7 @@
 
 static inline void g_mutex_clear(CompatGMutex *mutex)
 {
-    assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
+    g_assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
     if (mutex->once.retval) {
         g_mutex_free((GMutex *) mutex->once.retval);
     }
@@ -153,7 +140,7 @@
 
 static inline void g_cond_clear(CompatGCond *cond)
 {
-    assert(cond->once.status != G_ONCE_STATUS_PROGRESS);
+    g_assert(cond->once.status != G_ONCE_STATUS_PROGRESS);
     if (cond->once.retval) {
         g_cond_free((GCond *) cond->once.retval);
     }
@@ -162,7 +149,7 @@
 
 static inline void (g_cond_wait)(CompatGCond *cond, CompatGMutex *mutex)
 {
-    assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
+    g_assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
     g_once(&cond->once, do_g_cond_new, NULL);
     g_cond_wait((GCond *) cond->once.retval, (GMutex *) mutex->once.retval);
 }
@@ -182,6 +169,32 @@
 }
 #undef g_cond_signal
 
+static inline gboolean (g_cond_timed_wait)(CompatGCond *cond,
+                                           CompatGMutex *mutex,
+                                           GTimeVal *time)
+{
+    g_assert(mutex->once.status != G_ONCE_STATUS_PROGRESS);
+    g_once(&cond->once, do_g_cond_new, NULL);
+    return g_cond_timed_wait((GCond *) cond->once.retval,
+                             (GMutex *) mutex->once.retval, time);
+}
+#undef g_cond_timed_wait
+
+/* This is not a macro, because it didn't exist until 2.32.  */
+static inline gboolean g_cond_wait_until(CompatGCond *cond, CompatGMutex *mutex,
+                                         gint64 end_time)
+{
+    GTimeVal time;
+
+    /* Convert from monotonic to CLOCK_REALTIME.  */
+    end_time -= g_get_monotonic_time();
+    g_get_current_time(&time);
+    end_time += time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
+
+    time.tv_sec = end_time / G_TIME_SPAN_SECOND;
+    time.tv_usec = end_time % G_TIME_SPAN_SECOND;
+    return g_cond_timed_wait(cond, mutex, &time);
+}
 
 /* before 2.31 there was no g_thread_new() */
 static inline GThread *g_thread_new(const char *name,
@@ -198,4 +211,73 @@
 #define CompatGCond GCond
 #endif /* glib 2.31 */
 
+#if !GLIB_CHECK_VERSION(2, 32, 0)
+/* Beware, function returns gboolean since 2.39.2, see GLib commit 9101915 */
+static inline void g_hash_table_add(GHashTable *hash_table, gpointer key)
+{
+    g_hash_table_replace(hash_table, key, key);
+}
+#endif
+
+#ifndef g_assert_true
+#define g_assert_true(expr)                                                    \
+    do {                                                                       \
+        if (G_LIKELY(expr)) {                                                  \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be TRUE");                 \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_false
+#define g_assert_false(expr)                                                   \
+    do {                                                                       \
+        if (G_LIKELY(!(expr))) {                                               \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be FALSE");                \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_null
+#define g_assert_null(expr)                                                    \
+    do {                                                                       \
+        if (G_LIKELY((expr) == NULL)) {                                        \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should be NULL");                 \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_nonnull
+#define g_assert_nonnull(expr)                                                 \
+    do {                                                                       \
+        if (G_LIKELY((expr) != NULL)) {                                        \
+        } else {                                                               \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "'" #expr "' should not be NULL");             \
+        }                                                                      \
+    } while (0)
+#endif
+
+#ifndef g_assert_cmpmem
+#define g_assert_cmpmem(m1, l1, m2, l2)                                        \
+    do {                                                                       \
+        gconstpointer __m1 = m1, __m2 = m2;                                    \
+        int __l1 = l1, __l2 = l2;                                              \
+        if (__l1 != __l2) {                                                    \
+            g_assertion_message_cmpnum(                                        \
+                G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,                   \
+                #l1 " (len(" #m1 ")) == " #l2 " (len(" #m2 "))", __l1, "==",   \
+                __l2, 'i');                                                    \
+        } else if (memcmp(__m1, __m2, __l1) != 0) {                            \
+            g_assertion_message(G_LOG_DOMAIN, __FILE__, __LINE__, G_STRFUNC,   \
+                                "assertion failed (" #m1 " == " #m2 ")");      \
+        }                                                                      \
+    } while (0)
+#endif
+
 #endif
diff --git a/include/hw/acpi/acpi-defs.h b/include/hw/acpi/acpi-defs.h
new file mode 100644
index 0000000..41c1d95
--- /dev/null
+++ b/include/hw/acpi/acpi-defs.h
@@ -0,0 +1,601 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef QEMU_ACPI_DEFS_H
+#define QEMU_ACPI_DEFS_H
+
+enum {
+    ACPI_FADT_F_WBINVD,
+    ACPI_FADT_F_WBINVD_FLUSH,
+    ACPI_FADT_F_PROC_C1,
+    ACPI_FADT_F_P_LVL2_UP,
+    ACPI_FADT_F_PWR_BUTTON,
+    ACPI_FADT_F_SLP_BUTTON,
+    ACPI_FADT_F_FIX_RTC,
+    ACPI_FADT_F_RTC_S4,
+    ACPI_FADT_F_TMR_VAL_EXT,
+    ACPI_FADT_F_DCK_CAP,
+    ACPI_FADT_F_RESET_REG_SUP,
+    ACPI_FADT_F_SEALED_CASE,
+    ACPI_FADT_F_HEADLESS,
+    ACPI_FADT_F_CPU_SW_SLP,
+    ACPI_FADT_F_PCI_EXP_WAK,
+    ACPI_FADT_F_USE_PLATFORM_CLOCK,
+    ACPI_FADT_F_S4_RTC_STS_VALID,
+    ACPI_FADT_F_REMOTE_POWER_ON_CAPABLE,
+    ACPI_FADT_F_FORCE_APIC_CLUSTER_MODEL,
+    ACPI_FADT_F_FORCE_APIC_PHYSICAL_DESTINATION_MODE,
+    ACPI_FADT_F_HW_REDUCED_ACPI,
+    ACPI_FADT_F_LOW_POWER_S0_IDLE_CAPABLE,
+};
+
+/*
+ * ACPI 2.0 Generic Address Space definition.
+ */
+struct Acpi20GenericAddress {
+    uint8_t  address_space_id;
+    uint8_t  register_bit_width;
+    uint8_t  register_bit_offset;
+    uint8_t  reserved;
+    uint64_t address;
+} QEMU_PACKED;
+typedef struct Acpi20GenericAddress Acpi20GenericAddress;
+
+struct AcpiRsdpDescriptor {        /* Root System Descriptor Pointer */
+    uint64_t signature;              /* ACPI signature, contains "RSD PTR " */
+    uint8_t  checksum;               /* To make sum of struct == 0 */
+    uint8_t  oem_id [6];             /* OEM identification */
+    uint8_t  revision;               /* Must be 0 for 1.0, 2 for 2.0 */
+    uint32_t rsdt_physical_address;  /* 32-bit physical address of RSDT */
+    uint32_t length;                 /* XSDT Length in bytes including hdr */
+    uint64_t xsdt_physical_address;  /* 64-bit physical address of XSDT */
+    uint8_t  extended_checksum;      /* Checksum of entire table */
+    uint8_t  reserved [3];           /* Reserved field must be 0 */
+} QEMU_PACKED;
+typedef struct AcpiRsdpDescriptor AcpiRsdpDescriptor;
+
+/* Table structure from Linux kernel (the ACPI tables are under the
+   BSD license) */
+
+
+#define ACPI_TABLE_HEADER_DEF   /* ACPI common table header */ \
+    uint32_t signature;          /* ACPI signature (4 ASCII characters) */ \
+    uint32_t length;                 /* Length of table, in bytes, including header */ \
+    uint8_t  revision;               /* ACPI Specification minor version # */ \
+    uint8_t  checksum;               /* To make sum of entire table == 0 */ \
+    uint8_t  oem_id [6];             /* OEM identification */ \
+    uint8_t  oem_table_id [8];       /* OEM table identification */ \
+    uint32_t oem_revision;           /* OEM revision number */ \
+    uint8_t  asl_compiler_id [4];    /* ASL compiler vendor ID */ \
+    uint32_t asl_compiler_revision;  /* ASL compiler revision number */
+
+
+struct AcpiTableHeader         /* ACPI common table header */
+{
+    ACPI_TABLE_HEADER_DEF
+} QEMU_PACKED;
+typedef struct AcpiTableHeader AcpiTableHeader;
+
+/*
+ * ACPI Fixed ACPI Description Table (FADT)
+ */
+#define ACPI_FADT_COMMON_DEF /* FADT common definition */ \
+    ACPI_TABLE_HEADER_DEF    /* ACPI common table header */ \
+    uint32_t firmware_ctrl;  /* Physical address of FACS */ \
+    uint32_t dsdt;         /* Physical address of DSDT */ \
+    uint8_t  model;        /* System Interrupt Model */ \
+    uint8_t  reserved1;    /* Reserved */ \
+    uint16_t sci_int;      /* System vector of SCI interrupt */ \
+    uint32_t smi_cmd;      /* Port address of SMI command port */ \
+    uint8_t  acpi_enable;  /* Value to write to smi_cmd to enable ACPI */ \
+    uint8_t  acpi_disable; /* Value to write to smi_cmd to disable ACPI */ \
+    /* Value to write to SMI CMD to enter S4BIOS state */ \
+    uint8_t  S4bios_req; \
+    uint8_t  reserved2;    /* Reserved - must be zero */ \
+    /* Port address of Power Mgt 1a acpi_event Reg Blk */ \
+    uint32_t pm1a_evt_blk; \
+    /* Port address of Power Mgt 1b acpi_event Reg Blk */ \
+    uint32_t pm1b_evt_blk; \
+    uint32_t pm1a_cnt_blk; /* Port address of Power Mgt 1a Control Reg Blk */ \
+    uint32_t pm1b_cnt_blk; /* Port address of Power Mgt 1b Control Reg Blk */ \
+    uint32_t pm2_cnt_blk;  /* Port address of Power Mgt 2 Control Reg Blk */ \
+    uint32_t pm_tmr_blk;   /* Port address of Power Mgt Timer Ctrl Reg Blk */ \
+    /* Port addr of General Purpose acpi_event 0 Reg Blk */ \
+    uint32_t gpe0_blk; \
+    /* Port addr of General Purpose acpi_event 1 Reg Blk */ \
+    uint32_t gpe1_blk; \
+    uint8_t  pm1_evt_len;  /* Byte length of ports at pm1_x_evt_blk */ \
+    uint8_t  pm1_cnt_len;  /* Byte length of ports at pm1_x_cnt_blk */ \
+    uint8_t  pm2_cnt_len;  /* Byte Length of ports at pm2_cnt_blk */ \
+    uint8_t  pm_tmr_len;   /* Byte Length of ports at pm_tm_blk */ \
+    uint8_t  gpe0_blk_len; /* Byte Length of ports at gpe0_blk */ \
+    uint8_t  gpe1_blk_len; /* Byte Length of ports at gpe1_blk */ \
+    uint8_t  gpe1_base;    /* Offset in gpe model where gpe1 events start */ \
+    uint8_t  reserved3;    /* Reserved */ \
+    uint16_t plvl2_lat;    /* Worst case HW latency to enter/exit C2 state */ \
+    uint16_t plvl3_lat;    /* Worst case HW latency to enter/exit C3 state */ \
+    uint16_t flush_size;   /* Size of area read to flush caches */ \
+    uint16_t flush_stride; /* Stride used in flushing caches */ \
+    uint8_t  duty_offset;  /* Bit location of duty cycle field in p_cnt reg */ \
+    uint8_t  duty_width;   /* Bit width of duty cycle field in p_cnt reg */ \
+    uint8_t  day_alrm;     /* Index to day-of-month alarm in RTC CMOS RAM */ \
+    uint8_t  mon_alrm;     /* Index to month-of-year alarm in RTC CMOS RAM */ \
+    uint8_t  century;      /* Index to century in RTC CMOS RAM */
+
+struct AcpiFadtDescriptorRev1
+{
+    ACPI_FADT_COMMON_DEF
+    uint8_t  reserved4;              /* Reserved */
+    uint8_t  reserved4a;             /* Reserved */
+    uint8_t  reserved4b;             /* Reserved */
+    uint32_t flags;
+} QEMU_PACKED;
+typedef struct AcpiFadtDescriptorRev1 AcpiFadtDescriptorRev1;
+
+struct AcpiGenericAddress {
+    uint8_t space_id;        /* Address space where struct or register exists */
+    uint8_t bit_width;       /* Size in bits of given register */
+    uint8_t bit_offset;      /* Bit offset within the register */
+    uint8_t access_width;    /* Minimum Access size (ACPI 3.0) */
+    uint64_t address;        /* 64-bit address of struct or register */
+} QEMU_PACKED;
+
+struct AcpiFadtDescriptorRev5_1 {
+    ACPI_FADT_COMMON_DEF
+    /* IA-PC Boot Architecture Flags (see below for individual flags) */
+    uint16_t boot_flags;
+    uint8_t reserved;    /* Reserved, must be zero */
+    /* Miscellaneous flag bits (see below for individual flags) */
+    uint32_t flags;
+    /* 64-bit address of the Reset register */
+    struct AcpiGenericAddress reset_register;
+    /* Value to write to the reset_register port to reset the system */
+    uint8_t reset_value;
+    /* ARM-Specific Boot Flags (see below for individual flags) (ACPI 5.1) */
+    uint16_t arm_boot_flags;
+    uint8_t minor_revision;  /* FADT Minor Revision (ACPI 5.1) */
+    uint64_t Xfacs;          /* 64-bit physical address of FACS */
+    uint64_t Xdsdt;          /* 64-bit physical address of DSDT */
+    /* 64-bit Extended Power Mgt 1a Event Reg Blk address */
+    struct AcpiGenericAddress xpm1a_event_block;
+    /* 64-bit Extended Power Mgt 1b Event Reg Blk address */
+    struct AcpiGenericAddress xpm1b_event_block;
+    /* 64-bit Extended Power Mgt 1a Control Reg Blk address */
+    struct AcpiGenericAddress xpm1a_control_block;
+    /* 64-bit Extended Power Mgt 1b Control Reg Blk address */
+    struct AcpiGenericAddress xpm1b_control_block;
+    /* 64-bit Extended Power Mgt 2 Control Reg Blk address */
+    struct AcpiGenericAddress xpm2_control_block;
+    /* 64-bit Extended Power Mgt Timer Ctrl Reg Blk address */
+    struct AcpiGenericAddress xpm_timer_block;
+    /* 64-bit Extended General Purpose Event 0 Reg Blk address */
+    struct AcpiGenericAddress xgpe0_block;
+    /* 64-bit Extended General Purpose Event 1 Reg Blk address */
+    struct AcpiGenericAddress xgpe1_block;
+    /* 64-bit Sleep Control register (ACPI 5.0) */
+    struct AcpiGenericAddress sleep_control;
+    /* 64-bit Sleep Status register (ACPI 5.0) */
+    struct AcpiGenericAddress sleep_status;
+} QEMU_PACKED;
+
+typedef struct AcpiFadtDescriptorRev5_1 AcpiFadtDescriptorRev5_1;
+
+enum {
+    ACPI_FADT_ARM_USE_PSCI_G_0_2 = 0,
+    ACPI_FADT_ARM_PSCI_USE_HVC = 1,
+};
+
+/*
+ * Serial Port Console Redirection Table (SPCR), Rev. 1.02
+ *
+ * For .interface_type see Debug Port Table 2 (DBG2) serial port
+ * subtypes in Table 3, Rev. May 22, 2012
+ */
+struct AcpiSerialPortConsoleRedirection {
+    ACPI_TABLE_HEADER_DEF
+    uint8_t  interface_type;
+    uint8_t  reserved1[3];
+    struct AcpiGenericAddress base_address;
+    uint8_t  interrupt_types;
+    uint8_t  irq;
+    uint32_t gsi;
+    uint8_t  baud;
+    uint8_t  parity;
+    uint8_t  stopbits;
+    uint8_t  flowctrl;
+    uint8_t  term_type;
+    uint8_t  reserved2;
+    uint16_t pci_device_id;
+    uint16_t pci_vendor_id;
+    uint8_t  pci_bus;
+    uint8_t  pci_slot;
+    uint8_t  pci_func;
+    uint32_t pci_flags;
+    uint8_t  pci_seg;
+    uint32_t reserved3;
+} QEMU_PACKED;
+typedef struct AcpiSerialPortConsoleRedirection
+               AcpiSerialPortConsoleRedirection;
+
+/*
+ * ACPI 1.0 Root System Description Table (RSDT)
+ */
+struct AcpiRsdtDescriptorRev1
+{
+    ACPI_TABLE_HEADER_DEF       /* ACPI common table header */
+    uint32_t table_offset_entry[0];  /* Array of pointers to other */
+    /* ACPI tables */
+} QEMU_PACKED;
+typedef struct AcpiRsdtDescriptorRev1 AcpiRsdtDescriptorRev1;
+
+/*
+ * ACPI 1.0 Firmware ACPI Control Structure (FACS)
+ */
+struct AcpiFacsDescriptorRev1
+{
+    uint32_t signature;           /* ACPI Signature */
+    uint32_t length;                 /* Length of structure, in bytes */
+    uint32_t hardware_signature;     /* Hardware configuration signature */
+    uint32_t firmware_waking_vector; /* ACPI OS waking vector */
+    uint32_t global_lock;            /* Global Lock */
+    uint32_t flags;
+    uint8_t  resverved3 [40];        /* Reserved - must be zero */
+} QEMU_PACKED;
+typedef struct AcpiFacsDescriptorRev1 AcpiFacsDescriptorRev1;
+
+/*
+ * Differentiated System Description Table (DSDT)
+ */
+
+/*
+ * MADT values and structures
+ */
+
+/* Values for MADT PCATCompat */
+
+#define ACPI_DUAL_PIC                0
+#define ACPI_MULTIPLE_APIC           1
+
+/* Master MADT */
+
+struct AcpiMultipleApicTable
+{
+    ACPI_TABLE_HEADER_DEF     /* ACPI common table header */
+    uint32_t local_apic_address;     /* Physical address of local APIC */
+    uint32_t flags;
+} QEMU_PACKED;
+typedef struct AcpiMultipleApicTable AcpiMultipleApicTable;
+
+/* Values for Type in APIC sub-headers */
+
+#define ACPI_APIC_PROCESSOR          0
+#define ACPI_APIC_IO                 1
+#define ACPI_APIC_XRUPT_OVERRIDE     2
+#define ACPI_APIC_NMI                3
+#define ACPI_APIC_LOCAL_NMI          4
+#define ACPI_APIC_ADDRESS_OVERRIDE   5
+#define ACPI_APIC_IO_SAPIC           6
+#define ACPI_APIC_LOCAL_SAPIC        7
+#define ACPI_APIC_XRUPT_SOURCE       8
+#define ACPI_APIC_LOCAL_X2APIC       9
+#define ACPI_APIC_LOCAL_X2APIC_NMI      10
+#define ACPI_APIC_GENERIC_INTERRUPT     11
+#define ACPI_APIC_GENERIC_DISTRIBUTOR   12
+#define ACPI_APIC_GENERIC_MSI_FRAME     13
+#define ACPI_APIC_GENERIC_REDISTRIBUTOR 14
+#define ACPI_APIC_RESERVED              15   /* 15 and greater are reserved */
+
+/*
+ * MADT sub-structures (Follow MULTIPLE_APIC_DESCRIPTION_TABLE)
+ */
+#define ACPI_SUB_HEADER_DEF   /* Common ACPI sub-structure header */\
+    uint8_t  type;                               \
+    uint8_t  length;
+
+/* Sub-structures for MADT */
+
+struct AcpiMadtProcessorApic
+{
+    ACPI_SUB_HEADER_DEF
+    uint8_t  processor_id;           /* ACPI processor id */
+    uint8_t  local_apic_id;          /* Processor's local APIC id */
+    uint32_t flags;
+} QEMU_PACKED;
+typedef struct AcpiMadtProcessorApic AcpiMadtProcessorApic;
+
+struct AcpiMadtIoApic
+{
+    ACPI_SUB_HEADER_DEF
+    uint8_t  io_apic_id;             /* I/O APIC ID */
+    uint8_t  reserved;               /* Reserved - must be zero */
+    uint32_t address;                /* APIC physical address */
+    uint32_t interrupt;              /* Global system interrupt where INTI
+                                 * lines start */
+} QEMU_PACKED;
+typedef struct AcpiMadtIoApic AcpiMadtIoApic;
+
+struct AcpiMadtIntsrcovr {
+    ACPI_SUB_HEADER_DEF
+    uint8_t  bus;
+    uint8_t  source;
+    uint32_t gsi;
+    uint16_t flags;
+} QEMU_PACKED;
+typedef struct AcpiMadtIntsrcovr AcpiMadtIntsrcovr;
+
+struct AcpiMadtLocalNmi {
+    ACPI_SUB_HEADER_DEF
+    uint8_t  processor_id;           /* ACPI processor id */
+    uint16_t flags;                  /* MPS INTI flags */
+    uint8_t  lint;                   /* Local APIC LINT# */
+} QEMU_PACKED;
+typedef struct AcpiMadtLocalNmi AcpiMadtLocalNmi;
+
+struct AcpiMadtGenericInterrupt {
+    ACPI_SUB_HEADER_DEF
+    uint16_t reserved;
+    uint32_t cpu_interface_number;
+    uint32_t uid;
+    uint32_t flags;
+    uint32_t parking_version;
+    uint32_t performance_interrupt;
+    uint64_t parked_address;
+    uint64_t base_address;
+    uint64_t gicv_base_address;
+    uint64_t gich_base_address;
+    uint32_t vgic_interrupt;
+    uint64_t gicr_base_address;
+    uint64_t arm_mpidr;
+} QEMU_PACKED;
+
+typedef struct AcpiMadtGenericInterrupt AcpiMadtGenericInterrupt;
+
+struct AcpiMadtGenericDistributor {
+    ACPI_SUB_HEADER_DEF
+    uint16_t reserved;
+    uint32_t gic_id;
+    uint64_t base_address;
+    uint32_t global_irq_base;
+    /* ACPI 5.1 Errata 1228 Present GIC version in MADT table */
+    uint8_t version;
+    uint8_t reserved2[3];
+} QEMU_PACKED;
+
+typedef struct AcpiMadtGenericDistributor AcpiMadtGenericDistributor;
+
+struct AcpiMadtGenericMsiFrame {
+    ACPI_SUB_HEADER_DEF
+    uint16_t reserved;
+    uint32_t gic_msi_frame_id;
+    uint64_t base_address;
+    uint32_t flags;
+    uint16_t spi_count;
+    uint16_t spi_base;
+} QEMU_PACKED;
+
+typedef struct AcpiMadtGenericMsiFrame AcpiMadtGenericMsiFrame;
+
+struct AcpiMadtGenericRedistributor {
+    ACPI_SUB_HEADER_DEF
+    uint16_t reserved;
+    uint64_t base_address;
+    uint32_t range_length;
+} QEMU_PACKED;
+
+typedef struct AcpiMadtGenericRedistributor AcpiMadtGenericRedistributor;
+
+/*
+ * Generic Timer Description Table (GTDT)
+ */
+
+#define ACPI_GTDT_INTERRUPT_MODE        (1 << 0)
+#define ACPI_GTDT_INTERRUPT_POLARITY    (1 << 1)
+#define ACPI_GTDT_ALWAYS_ON             (1 << 2)
+
+/* Triggering */
+
+#define ACPI_LEVEL_SENSITIVE            ((uint8_t) 0x00)
+#define ACPI_EDGE_SENSITIVE             ((uint8_t) 0x01)
+
+/* Polarity */
+
+#define ACPI_ACTIVE_HIGH                ((uint8_t) 0x00)
+#define ACPI_ACTIVE_LOW                 ((uint8_t) 0x01)
+#define ACPI_ACTIVE_BOTH                ((uint8_t) 0x02)
+
+struct AcpiGenericTimerTable {
+    ACPI_TABLE_HEADER_DEF
+    uint64_t counter_block_addresss;
+    uint32_t reserved;
+    uint32_t secure_el1_interrupt;
+    uint32_t secure_el1_flags;
+    uint32_t non_secure_el1_interrupt;
+    uint32_t non_secure_el1_flags;
+    uint32_t virtual_timer_interrupt;
+    uint32_t virtual_timer_flags;
+    uint32_t non_secure_el2_interrupt;
+    uint32_t non_secure_el2_flags;
+    uint64_t counter_read_block_address;
+    uint32_t platform_timer_count;
+    uint32_t platform_timer_offset;
+} QEMU_PACKED;
+typedef struct AcpiGenericTimerTable AcpiGenericTimerTable;
+
+/*
+ * HPET Description Table
+ */
+struct Acpi20Hpet {
+    ACPI_TABLE_HEADER_DEF                    /* ACPI common table header */
+    uint32_t           timer_block_id;
+    Acpi20GenericAddress addr;
+    uint8_t            hpet_number;
+    uint16_t           min_tick;
+    uint8_t            page_protect;
+} QEMU_PACKED;
+typedef struct Acpi20Hpet Acpi20Hpet;
+
+/*
+ * SRAT (NUMA topology description) table
+ */
+
+struct AcpiSystemResourceAffinityTable
+{
+    ACPI_TABLE_HEADER_DEF
+    uint32_t    reserved1;
+    uint32_t    reserved2[2];
+} QEMU_PACKED;
+typedef struct AcpiSystemResourceAffinityTable AcpiSystemResourceAffinityTable;
+
+#define ACPI_SRAT_PROCESSOR_APIC     0
+#define ACPI_SRAT_MEMORY             1
+#define ACPI_SRAT_PROCESSOR_x2APIC   2
+#define ACPI_SRAT_PROCESSOR_GICC     3
+
+struct AcpiSratProcessorAffinity
+{
+    ACPI_SUB_HEADER_DEF
+    uint8_t     proximity_lo;
+    uint8_t     local_apic_id;
+    uint32_t    flags;
+    uint8_t     local_sapic_eid;
+    uint8_t     proximity_hi[3];
+    uint32_t    reserved;
+} QEMU_PACKED;
+typedef struct AcpiSratProcessorAffinity AcpiSratProcessorAffinity;
+
+struct AcpiSratMemoryAffinity
+{
+    ACPI_SUB_HEADER_DEF
+    uint32_t    proximity;
+    uint16_t    reserved1;
+    uint64_t    base_addr;
+    uint64_t    range_length;
+    uint32_t    reserved2;
+    uint32_t    flags;
+    uint32_t    reserved3[2];
+} QEMU_PACKED;
+typedef struct AcpiSratMemoryAffinity AcpiSratMemoryAffinity;
+
+struct AcpiSratProcessorGiccAffinity
+{
+    ACPI_SUB_HEADER_DEF
+    uint32_t    proximity;
+    uint32_t    acpi_processor_uid;
+    uint32_t    flags;
+    uint32_t    clock_domain;
+} QEMU_PACKED;
+
+typedef struct AcpiSratProcessorGiccAffinity AcpiSratProcessorGiccAffinity;
+
+/* PCI fw r3.0 MCFG table. */
+/* Subtable */
+struct AcpiMcfgAllocation {
+    uint64_t address;                /* Base address, processor-relative */
+    uint16_t pci_segment;            /* PCI segment group number */
+    uint8_t start_bus_number;       /* Starting PCI Bus number */
+    uint8_t end_bus_number;         /* Final PCI Bus number */
+    uint32_t reserved;
+} QEMU_PACKED;
+typedef struct AcpiMcfgAllocation AcpiMcfgAllocation;
+
+struct AcpiTableMcfg {
+    ACPI_TABLE_HEADER_DEF;
+    uint8_t reserved[8];
+    AcpiMcfgAllocation allocation[0];
+} QEMU_PACKED;
+typedef struct AcpiTableMcfg AcpiTableMcfg;
+
+/*
+ * TCPA Description Table
+ *
+ * Following Level 00, Rev 00.37 of specs:
+ * http://www.trustedcomputinggroup.org/resources/tcg_acpi_specification
+ */
+struct Acpi20Tcpa {
+    ACPI_TABLE_HEADER_DEF                    /* ACPI common table header */
+    uint16_t platform_class;
+    uint32_t log_area_minimum_length;
+    uint64_t log_area_start_address;
+} QEMU_PACKED;
+typedef struct Acpi20Tcpa Acpi20Tcpa;
+
+/*
+ * TPM2
+ *
+ * Following Level 00, Rev 00.37 of specs:
+ * http://www.trustedcomputinggroup.org/resources/tcg_acpi_specification
+ */
+struct Acpi20TPM2 {
+    ACPI_TABLE_HEADER_DEF
+    uint16_t platform_class;
+    uint16_t reserved;
+    uint64_t control_area_address;
+    uint32_t start_method;
+} QEMU_PACKED;
+typedef struct Acpi20TPM2 Acpi20TPM2;
+
+/* DMAR - DMA Remapping table r2.2 */
+struct AcpiTableDmar {
+    ACPI_TABLE_HEADER_DEF
+    uint8_t host_address_width; /* Maximum DMA physical addressability */
+    uint8_t flags;
+    uint8_t reserved[10];
+} QEMU_PACKED;
+typedef struct AcpiTableDmar AcpiTableDmar;
+
+/* Masks for Flags field above */
+#define ACPI_DMAR_INTR_REMAP        1
+#define ACPI_DMAR_X2APIC_OPT_OUT    (1 << 1)
+
+/* Values for sub-structure type for DMAR */
+enum {
+    ACPI_DMAR_TYPE_HARDWARE_UNIT = 0,       /* DRHD */
+    ACPI_DMAR_TYPE_RESERVED_MEMORY = 1,     /* RMRR */
+    ACPI_DMAR_TYPE_ATSR = 2,                /* ATSR */
+    ACPI_DMAR_TYPE_HARDWARE_AFFINITY = 3,   /* RHSR */
+    ACPI_DMAR_TYPE_ANDD = 4,                /* ANDD */
+    ACPI_DMAR_TYPE_RESERVED = 5             /* Reserved for furture use */
+};
+
+/*
+ * Sub-structures for DMAR
+ */
+
+/* Device scope structure for DRHD. */
+struct AcpiDmarDeviceScope {
+    uint8_t entry_type;
+    uint8_t length;
+    uint16_t reserved;
+    uint8_t enumeration_id;
+    uint8_t bus;
+    uint16_t path[0];           /* list of dev:func pairs */
+} QEMU_PACKED;
+typedef struct AcpiDmarDeviceScope AcpiDmarDeviceScope;
+
+/* Type 0: Hardware Unit Definition */
+struct AcpiDmarHardwareUnit {
+    uint16_t type;
+    uint16_t length;
+    uint8_t flags;
+    uint8_t reserved;
+    uint16_t pci_segment;   /* The PCI Segment associated with this unit */
+    uint64_t address;   /* Base address of remapping hardware register-set */
+    AcpiDmarDeviceScope scope[0];
+} QEMU_PACKED;
+typedef struct AcpiDmarHardwareUnit AcpiDmarHardwareUnit;
+
+/* Masks for Flags field above */
+#define ACPI_DMAR_INCLUDE_PCI_ALL   1
+
+#endif
diff --git a/include/hw/acpi/acpi.h b/include/hw/acpi/acpi.h
index 1f678b4..7b3d93c 100644
--- a/include/hw/acpi/acpi.h
+++ b/include/hw/acpi/acpi.h
@@ -1,5 +1,6 @@
 #ifndef QEMU_HW_ACPI_H
 #define QEMU_HW_ACPI_H
+
 /*
  *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
  *                     VA Linux Systems Japan K.K.
@@ -19,12 +20,11 @@
  * <http://www.gnu.org/licenses/>.
  */
 
-#include "qapi/error.h"
-#include "qemu/typedefs.h"
 #include "qemu/notify.h"
 #include "qemu/option.h"
 #include "exec/memory.h"
 #include "hw/irq.h"
+#include "hw/acpi/acpi_dev_interface.h"
 
 /*
  * current device naming scheme supports up to 256 memory devices
@@ -145,13 +145,6 @@
                       MemoryRegion *parent);
 void acpi_pm_tmr_reset(ACPIREGS *ar);
 
-#include "qemu/timer.h"
-static inline int64_t acpi_pm_tmr_get_clock(void)
-{
-    return muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), PM_TIMER_FREQUENCY,
-                    get_ticks_per_sec());
-}
-
 /* PM1a_EVT: piix and ich9 don't implement PM1b. */
 uint16_t acpi_pm1_evt_get_sts(ACPIREGS *ar);
 void acpi_pm1_evt_power_down(ACPIREGS *ar);
@@ -160,7 +153,8 @@
                        MemoryRegion *parent);
 
 /* PM1a_CNT: piix and ich9 don't implement PM1b CNT. */
-void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent, uint8_t s4_val);
+void acpi_pm1_cnt_init(ACPIREGS *ar, MemoryRegion *parent,
+                       bool disable_s3, bool disable_s4, uint8_t s4_val);
 void acpi_pm1_cnt_update(ACPIREGS *ar,
                          bool sci_enable, bool sci_disable);
 void acpi_pm1_cnt_reset(ACPIREGS *ar);
@@ -172,6 +166,9 @@
 void acpi_gpe_ioport_writeb(ACPIREGS *ar, uint32_t addr, uint32_t val);
 uint32_t acpi_gpe_ioport_readb(ACPIREGS *ar, uint32_t addr);
 
+void acpi_send_gpe_event(ACPIREGS *ar, qemu_irq irq,
+                         AcpiEventStatusBits status);
+
 void acpi_update_sci(ACPIREGS *acpi_regs, qemu_irq irq);
 
 /* acpi.c */
@@ -185,4 +182,11 @@
 void acpi_table_add(const QemuOpts *opts, Error **errp);
 void acpi_table_add_builtin(const QemuOpts *opts, Error **errp);
 
-#endif /* !QEMU_HW_ACPI_H */
+typedef struct AcpiSlicOem AcpiSlicOem;
+struct AcpiSlicOem {
+  char *id;
+  char *table_id;
+};
+int acpi_get_slic_oem(AcpiSlicOem *oem);
+
+#endif /* QEMU_HW_ACPI_H */
diff --git a/include/hw/acpi/acpi_dev_interface.h b/include/hw/acpi/acpi_dev_interface.h
index f245f8d..da4ef7f 100644
--- a/include/hw/acpi/acpi_dev_interface.h
+++ b/include/hw/acpi/acpi_dev_interface.h
@@ -3,6 +3,14 @@
 
 #include "qom/object.h"
 #include "qapi-types.h"
+#include "hw/boards.h"
+
+/* These values are part of guest ABI, and can not be changed */
+typedef enum {
+    ACPI_PCI_HOTPLUG_STATUS = 2,
+    ACPI_CPU_HOTPLUG_STATUS = 4,
+    ACPI_MEMORY_HOTPLUG_STATUS = 8,
+} AcpiEventStatusBits;
 
 #define TYPE_ACPI_DEVICE_IF "acpi-device-interface"
 
@@ -22,11 +30,18 @@
     Object Parent;
 } AcpiDeviceIf;
 
+void acpi_send_event(DeviceState *dev, AcpiEventStatusBits event);
+
 /**
  * AcpiDeviceIfClass:
  *
  * ospm_status: returns status of ACPI device objects, reported
  *              via _OST method if device supports it.
+ * send_event: inject a specified event into guest
+ * madt_cpu: fills @entry with Interrupt Controller Structure
+ *           for CPU indexed by @uid in @apic_ids array,
+ *           returned structure types are:
+ *           0 - Local APIC, 9 - Local x2APIC, 0xB - GICC
  *
  * Interface is designed for providing unified interface
  * to generic ACPI functionality that could be used without
@@ -39,5 +54,8 @@
 
     /* <public> */
     void (*ospm_status)(AcpiDeviceIf *adev, ACPIOSTInfoList ***list);
+    void (*send_event)(AcpiDeviceIf *adev, AcpiEventStatusBits ev);
+    void (*madt_cpu)(AcpiDeviceIf *adev, int uid,
+                     CPUArchIdList *apic_ids, GArray *entry);
 } AcpiDeviceIfClass;
 #endif
diff --git a/include/hw/acpi/aml-build.h b/include/hw/acpi/aml-build.h
new file mode 100644
index 0000000..e5f0878
--- /dev/null
+++ b/include/hw/acpi/aml-build.h
@@ -0,0 +1,390 @@
+#ifndef HW_ACPI_AML_BUILD_H
+#define HW_ACPI_AML_BUILD_H
+
+#include "hw/acpi/acpi-defs.h"
+#include "hw/acpi/bios-linker-loader.h"
+
+/* Reserve RAM space for tables: add another order of magnitude. */
+#define ACPI_BUILD_TABLE_MAX_SIZE         0x200000
+
+#define ACPI_BUILD_APPNAME6 "BOCHS "
+#define ACPI_BUILD_APPNAME4 "BXPC"
+
+#define ACPI_BUILD_TABLE_FILE "etc/acpi/tables"
+#define ACPI_BUILD_RSDP_FILE "etc/acpi/rsdp"
+#define ACPI_BUILD_TPMLOG_FILE "etc/tpm/log"
+
+#define AML_NOTIFY_METHOD "NTFY"
+
+typedef enum {
+    AML_NO_OPCODE = 0,/* has only data */
+    AML_OPCODE,       /* has opcode optionally followed by data */
+    AML_PACKAGE,      /* has opcode and uses PkgLength for its length */
+    AML_EXT_PACKAGE,  /* Same as AML_PACKAGE but also has 'ExOpPrefix' */
+    AML_BUFFER,       /* data encoded as 'DefBuffer' */
+    AML_RES_TEMPLATE, /* encoded as ResourceTemplate macro */
+} AmlBlockFlags;
+
+struct Aml {
+    GArray *buf;
+
+    /*< private >*/
+    uint8_t op;
+    AmlBlockFlags block_flags;
+};
+typedef struct Aml Aml;
+
+typedef enum {
+    AML_COMPATIBILITY = 0,
+    AML_TYPEA = 1,
+    AML_TYPEB = 2,
+    AML_TYPEF = 3,
+} AmlDmaType;
+
+typedef enum {
+    AML_NOTBUSMASTER = 0,
+    AML_BUSMASTER = 1,
+} AmlDmaBusMaster;
+
+typedef enum {
+    AML_TRANSFER8 = 0,
+    AML_TRANSFER8_16 = 1,
+    AML_TRANSFER16 = 2,
+} AmlTransferSize;
+
+typedef enum {
+    AML_DECODE10 = 0,
+    AML_DECODE16 = 1,
+} AmlIODecode;
+
+typedef enum {
+    AML_ANY_ACC = 0,
+    AML_BYTE_ACC = 1,
+    AML_WORD_ACC = 2,
+    AML_DWORD_ACC = 3,
+    AML_QWORD_ACC = 4,
+    AML_BUFFER_ACC = 5,
+} AmlAccessType;
+
+typedef enum {
+    AML_NOLOCK = 0,
+    AML_LOCK = 1,
+} AmlLockRule;
+
+typedef enum {
+    AML_PRESERVE = 0,
+    AML_WRITE_AS_ONES = 1,
+    AML_WRITE_AS_ZEROS = 2,
+} AmlUpdateRule;
+
+typedef enum {
+    AML_SYSTEM_MEMORY = 0X00,
+    AML_SYSTEM_IO = 0X01,
+    AML_PCI_CONFIG = 0X02,
+} AmlRegionSpace;
+
+typedef enum {
+    AML_MEMORY_RANGE = 0,
+    AML_IO_RANGE = 1,
+    AML_BUS_NUMBER_RANGE = 2,
+} AmlResourceType;
+
+typedef enum {
+    AML_SUB_DECODE = 1 << 1,
+    AML_POS_DECODE = 0
+} AmlDecode;
+
+typedef enum {
+    AML_MAX_FIXED = 1 << 3,
+    AML_MAX_NOT_FIXED = 0,
+} AmlMaxFixed;
+
+typedef enum {
+    AML_MIN_FIXED = 1 << 2,
+    AML_MIN_NOT_FIXED = 0
+} AmlMinFixed;
+
+/*
+ * ACPI 1.0b: Table 6-26 I/O Resource Flag (Resource Type = 1) Definitions
+ * _RNG field definition
+ */
+typedef enum {
+    AML_ISA_ONLY = 1,
+    AML_NON_ISA_ONLY = 2,
+    AML_ENTIRE_RANGE = 3,
+} AmlISARanges;
+
+/*
+ * ACPI 1.0b: Table 6-25 Memory Resource Flag (Resource Type = 0) Definitions
+ * _MEM field definition
+ */
+typedef enum {
+    AML_NON_CACHEABLE = 0,
+    AML_CACHEABLE = 1,
+    AML_WRITE_COMBINING = 2,
+    AML_PREFETCHABLE = 3,
+} AmlCacheable;
+
+/*
+ * ACPI 1.0b: Table 6-25 Memory Resource Flag (Resource Type = 0) Definitions
+ * _RW field definition
+ */
+typedef enum {
+    AML_READ_ONLY = 0,
+    AML_READ_WRITE = 1,
+} AmlReadAndWrite;
+
+/*
+ * ACPI 5.0: Table 6-187 Extended Interrupt Descriptor Definition
+ * Interrupt Vector Flags Bits[0] Consumer/Producer
+ */
+typedef enum {
+    AML_CONSUMER_PRODUCER = 0,
+    AML_CONSUMER = 1,
+} AmlConsumerAndProducer;
+
+/*
+ * ACPI 5.0: Table 6-187 Extended Interrupt Descriptor Definition
+ * _HE field definition
+ */
+typedef enum {
+    AML_LEVEL = 0,
+    AML_EDGE = 1,
+} AmlLevelAndEdge;
+
+/*
+ * ACPI 5.0: Table 6-187 Extended Interrupt Descriptor Definition
+ * _LL field definition
+ */
+typedef enum {
+    AML_ACTIVE_HIGH = 0,
+    AML_ACTIVE_LOW = 1,
+} AmlActiveHighAndLow;
+
+/*
+ * ACPI 5.0: Table 6-187 Extended Interrupt Descriptor Definition
+ * _SHR field definition
+ */
+typedef enum {
+    AML_EXCLUSIVE = 0,
+    AML_SHARED = 1,
+    AML_EXCLUSIVE_AND_WAKE = 2,
+    AML_SHARED_AND_WAKE = 3,
+} AmlShared;
+
+/* ACPI 1.0b: 16.2.5.2 Named Objects Encoding: MethodFlags */
+typedef enum {
+    AML_NOTSERIALIZED = 0,
+    AML_SERIALIZED = 1,
+} AmlSerializeFlag;
+
+/*
+ * ACPI 5.0: Table 6-189 GPIO Connection Descriptor Definition
+ * GPIO Connection Type
+ */
+typedef enum {
+    AML_INTERRUPT_CONNECTION = 0,
+    AML_IO_CONNECTION = 1,
+} AmlGpioConnectionType;
+
+/*
+ * ACPI 5.0: Table 6-189 GPIO Connection Descriptor Definition
+ * _PPI field definition
+ */
+typedef enum {
+    AML_PULL_DEFAULT = 0,
+    AML_PULL_UP = 1,
+    AML_PULL_DOWN = 2,
+    AML_PULL_NONE = 3,
+} AmlPinConfig;
+
+typedef enum {
+    MEM_AFFINITY_NOFLAGS      = 0,
+    MEM_AFFINITY_ENABLED      = (1 << 0),
+    MEM_AFFINITY_HOTPLUGGABLE = (1 << 1),
+    MEM_AFFINITY_NON_VOLATILE = (1 << 2),
+} MemoryAffinityFlags;
+
+typedef
+struct AcpiBuildTables {
+    GArray *table_data;
+    GArray *rsdp;
+    GArray *tcpalog;
+    BIOSLinker *linker;
+} AcpiBuildTables;
+
+/**
+ * init_aml_allocator:
+ *
+ * Called for initializing API allocator which allow to use
+ * AML API.
+ * Returns: toplevel container which accumulates all other
+ * AML elements for a table.
+ */
+Aml *init_aml_allocator(void);
+
+/**
+ * free_aml_allocator:
+ *
+ * Releases all elements used by AML API, frees associated memory
+ * and invalidates AML allocator. After this call @init_aml_allocator
+ * should be called again if AML API is to be used again.
+ */
+void free_aml_allocator(void);
+
+/**
+ * aml_append:
+ * @parent_ctx: context to which @child element is added
+ * @child: element that is copied into @parent_ctx context
+ *
+ * Joins Aml elements together and helps to construct AML tables
+ * Examle of usage:
+ *   Aml *table = aml_def_block("SSDT", ...);
+ *   Aml *sb = aml_scope("\\_SB");
+ *   Aml *dev = aml_device("PCI0");
+ *
+ *   aml_append(dev, aml_name_decl("HID", aml_eisaid("PNP0A03")));
+ *   aml_append(sb, dev);
+ *   aml_append(table, sb);
+ */
+void aml_append(Aml *parent_ctx, Aml *child);
+
+/* non block AML object primitives */
+Aml *aml_name(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
+Aml *aml_name_decl(const char *name, Aml *val);
+Aml *aml_debug(void);
+Aml *aml_return(Aml *val);
+Aml *aml_int(const uint64_t val);
+Aml *aml_arg(int pos);
+Aml *aml_to_integer(Aml *arg);
+Aml *aml_to_hexstring(Aml *src, Aml *dst);
+Aml *aml_to_buffer(Aml *src, Aml *dst);
+Aml *aml_store(Aml *val, Aml *target);
+Aml *aml_and(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_or(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_lor(Aml *arg1, Aml *arg2);
+Aml *aml_shiftleft(Aml *arg1, Aml *count);
+Aml *aml_shiftright(Aml *arg1, Aml *count, Aml *dst);
+Aml *aml_lless(Aml *arg1, Aml *arg2);
+Aml *aml_add(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_subtract(Aml *arg1, Aml *arg2, Aml *dst);
+Aml *aml_increment(Aml *arg);
+Aml *aml_decrement(Aml *arg);
+Aml *aml_index(Aml *arg1, Aml *idx);
+Aml *aml_notify(Aml *arg1, Aml *arg2);
+Aml *aml_call0(const char *method);
+Aml *aml_call1(const char *method, Aml *arg1);
+Aml *aml_call2(const char *method, Aml *arg1, Aml *arg2);
+Aml *aml_call3(const char *method, Aml *arg1, Aml *arg2, Aml *arg3);
+Aml *aml_call4(const char *method, Aml *arg1, Aml *arg2, Aml *arg3, Aml *arg4);
+Aml *aml_call5(const char *method, Aml *arg1, Aml *arg2, Aml *arg3, Aml *arg4,
+               Aml *arg5);
+Aml *aml_gpio_int(AmlConsumerAndProducer con_and_pro,
+                  AmlLevelAndEdge edge_level,
+                  AmlActiveHighAndLow active_level, AmlShared shared,
+                  AmlPinConfig pin_config, uint16_t debounce_timeout,
+                  const uint32_t pin_list[], uint32_t pin_count,
+                  const char *resource_source_name,
+                  const uint8_t *vendor_data, uint16_t vendor_data_len);
+Aml *aml_memory32_fixed(uint32_t addr, uint32_t size,
+                        AmlReadAndWrite read_and_write);
+Aml *aml_interrupt(AmlConsumerAndProducer con_and_pro,
+                   AmlLevelAndEdge level_and_edge,
+                   AmlActiveHighAndLow high_and_low, AmlShared shared,
+                   uint32_t *irq_list, uint8_t irq_count);
+Aml *aml_io(AmlIODecode dec, uint16_t min_base, uint16_t max_base,
+            uint8_t aln, uint8_t len);
+Aml *aml_operation_region(const char *name, AmlRegionSpace rs,
+                          Aml *offset, uint32_t len);
+Aml *aml_irq_no_flags(uint8_t irq);
+Aml *aml_named_field(const char *name, unsigned length);
+Aml *aml_reserved_field(unsigned length);
+Aml *aml_local(int num);
+Aml *aml_string(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
+Aml *aml_lnot(Aml *arg);
+Aml *aml_equal(Aml *arg1, Aml *arg2);
+Aml *aml_lgreater(Aml *arg1, Aml *arg2);
+Aml *aml_lgreater_equal(Aml *arg1, Aml *arg2);
+Aml *aml_processor(uint8_t proc_id, uint32_t pblk_addr, uint8_t pblk_len,
+                   const char *name_format, ...) GCC_FMT_ATTR(4, 5);
+Aml *aml_eisaid(const char *str);
+Aml *aml_word_bus_number(AmlMinFixed min_fixed, AmlMaxFixed max_fixed,
+                         AmlDecode dec, uint16_t addr_gran,
+                         uint16_t addr_min, uint16_t addr_max,
+                         uint16_t addr_trans, uint16_t len);
+Aml *aml_word_io(AmlMinFixed min_fixed, AmlMaxFixed max_fixed,
+                 AmlDecode dec, AmlISARanges isa_ranges,
+                 uint16_t addr_gran, uint16_t addr_min,
+                 uint16_t addr_max, uint16_t addr_trans,
+                 uint16_t len);
+Aml *aml_dword_io(AmlMinFixed min_fixed, AmlMaxFixed max_fixed,
+                 AmlDecode dec, AmlISARanges isa_ranges,
+                 uint32_t addr_gran, uint32_t addr_min,
+                 uint32_t addr_max, uint32_t addr_trans,
+                 uint32_t len);
+Aml *aml_dword_memory(AmlDecode dec, AmlMinFixed min_fixed,
+                      AmlMaxFixed max_fixed, AmlCacheable cacheable,
+                      AmlReadAndWrite read_and_write,
+                      uint32_t addr_gran, uint32_t addr_min,
+                      uint32_t addr_max, uint32_t addr_trans,
+                      uint32_t len);
+Aml *aml_qword_memory(AmlDecode dec, AmlMinFixed min_fixed,
+                      AmlMaxFixed max_fixed, AmlCacheable cacheable,
+                      AmlReadAndWrite read_and_write,
+                      uint64_t addr_gran, uint64_t addr_min,
+                      uint64_t addr_max, uint64_t addr_trans,
+                      uint64_t len);
+Aml *aml_dma(AmlDmaType typ, AmlDmaBusMaster bm, AmlTransferSize sz,
+             uint8_t channel);
+Aml *aml_sleep(uint64_t msec);
+
+/* Block AML object primitives */
+Aml *aml_scope(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
+Aml *aml_device(const char *name_format, ...) GCC_FMT_ATTR(1, 2);
+Aml *aml_method(const char *name, int arg_count, AmlSerializeFlag sflag);
+Aml *aml_if(Aml *predicate);
+Aml *aml_else(void);
+Aml *aml_while(Aml *predicate);
+Aml *aml_package(uint8_t num_elements);
+Aml *aml_buffer(int buffer_size, uint8_t *byte_list);
+Aml *aml_resource_template(void);
+Aml *aml_field(const char *name, AmlAccessType type, AmlLockRule lock,
+               AmlUpdateRule rule);
+Aml *aml_mutex(const char *name, uint8_t sync_level);
+Aml *aml_acquire(Aml *mutex, uint16_t timeout);
+Aml *aml_release(Aml *mutex);
+Aml *aml_alias(const char *source_object, const char *alias_object);
+Aml *aml_create_field(Aml *srcbuf, Aml *bit_index, Aml *num_bits,
+                      const char *name);
+Aml *aml_create_dword_field(Aml *srcbuf, Aml *index, const char *name);
+Aml *aml_create_qword_field(Aml *srcbuf, Aml *index, const char *name);
+Aml *aml_varpackage(uint32_t num_elements);
+Aml *aml_touuid(const char *uuid);
+Aml *aml_unicode(const char *str);
+Aml *aml_refof(Aml *arg);
+Aml *aml_derefof(Aml *arg);
+Aml *aml_sizeof(Aml *arg);
+Aml *aml_concatenate(Aml *source1, Aml *source2, Aml *target);
+Aml *aml_object_type(Aml *object);
+
+void
+build_header(BIOSLinker *linker, GArray *table_data,
+             AcpiTableHeader *h, const char *sig, int len, uint8_t rev,
+             const char *oem_id, const char *oem_table_id);
+void *acpi_data_push(GArray *table_data, unsigned size);
+unsigned acpi_data_len(GArray *table);
+void acpi_add_table(GArray *table_offsets, GArray *table_data);
+void acpi_build_tables_init(AcpiBuildTables *tables);
+void acpi_build_tables_cleanup(AcpiBuildTables *tables, bool mfre);
+void
+build_rsdt(GArray *table_data, BIOSLinker *linker, GArray *table_offsets,
+           const char *oem_id, const char *oem_table_id);
+
+int
+build_append_named_dword(GArray *array, const char *name_format, ...)
+GCC_FMT_ATTR(2, 3);
+
+void build_srat_memory(AcpiSratMemoryAffinity *numamem, uint64_t base,
+                       uint64_t len, int node, MemoryAffinityFlags flags);
+
+#endif
diff --git a/include/hw/acpi/bios-linker-loader.h b/include/hw/acpi/bios-linker-loader.h
new file mode 100644
index 0000000..fa1e5d1
--- /dev/null
+++ b/include/hw/acpi/bios-linker-loader.h
@@ -0,0 +1,30 @@
+#ifndef BIOS_LINKER_LOADER_H
+#define BIOS_LINKER_LOADER_H
+
+
+typedef struct BIOSLinker {
+    GArray *cmd_blob;
+    GArray *file_list;
+} BIOSLinker;
+
+BIOSLinker *bios_linker_loader_init(void);
+
+void bios_linker_loader_alloc(BIOSLinker *linker,
+                              const char *file_name,
+                              GArray *file_blob,
+                              uint32_t alloc_align,
+                              bool alloc_fseg);
+
+void bios_linker_loader_add_checksum(BIOSLinker *linker, const char *file,
+                                     unsigned start_offset, unsigned size,
+                                     unsigned checksum_offset);
+
+void bios_linker_loader_add_pointer(BIOSLinker *linker,
+                                    const char *dest_file,
+                                    uint32_t dst_patched_offset,
+                                    uint8_t dst_patched_size,
+                                    const char *src_file,
+                                    uint32_t src_offset);
+
+void bios_linker_loader_cleanup(BIOSLinker *linker);
+#endif
diff --git a/include/hw/acpi/cpu.h b/include/hw/acpi/cpu.h
new file mode 100644
index 0000000..89ce172
--- /dev/null
+++ b/include/hw/acpi/cpu.h
@@ -0,0 +1,67 @@
+/*
+ * QEMU ACPI hotplug utilities
+ *
+ * Copyright (C) 2016 Red Hat Inc
+ *
+ * Authors:
+ *   Igor Mammedov <imammedo@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef ACPI_CPU_H
+#define ACPI_CPU_H
+
+#include "hw/qdev-core.h"
+#include "hw/acpi/acpi.h"
+#include "hw/acpi/aml-build.h"
+#include "hw/hotplug.h"
+
+typedef struct AcpiCpuStatus {
+    struct CPUState *cpu;
+    uint64_t arch_id;
+    bool is_inserting;
+    bool is_removing;
+    uint32_t ost_event;
+    uint32_t ost_status;
+} AcpiCpuStatus;
+
+typedef struct CPUHotplugState {
+    MemoryRegion ctrl_reg;
+    uint32_t selector;
+    uint8_t command;
+    uint32_t dev_count;
+    AcpiCpuStatus *devs;
+} CPUHotplugState;
+
+void acpi_cpu_plug_cb(HotplugHandler *hotplug_dev,
+                      CPUHotplugState *cpu_st, DeviceState *dev, Error **errp);
+
+void acpi_cpu_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                CPUHotplugState *cpu_st,
+                                DeviceState *dev, Error **errp);
+
+void acpi_cpu_unplug_cb(CPUHotplugState *cpu_st,
+                        DeviceState *dev, Error **errp);
+
+void cpu_hotplug_hw_init(MemoryRegion *as, Object *owner,
+                         CPUHotplugState *state, hwaddr base_addr);
+
+typedef struct CPUHotplugFeatures {
+    bool apci_1_compatible;
+    bool has_legacy_cphp;
+} CPUHotplugFeatures;
+
+void build_cpus_aml(Aml *table, MachineState *machine, CPUHotplugFeatures opts,
+                    hwaddr io_base,
+                    const char *res_root,
+                    const char *event_handler_method);
+
+void acpi_cpu_ospm_status(CPUHotplugState *cpu_st, ACPIOSTInfoList ***list);
+
+extern const VMStateDescription vmstate_cpu_hotplug;
+#define VMSTATE_CPU_HOTPLUG(cpuhp, state) \
+    VMSTATE_STRUCT(cpuhp, state, 1, \
+                   vmstate_cpu_hotplug, CPUHotplugState)
+
+#endif
diff --git a/include/hw/acpi/cpu_hotplug.h b/include/hw/acpi/cpu_hotplug.h
index f6d358d..3b932ab 100644
--- a/include/hw/acpi/cpu_hotplug.h
+++ b/include/hw/acpi/cpu_hotplug.h
@@ -9,20 +9,32 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#ifndef ACPI_HOTPLUG_H
-#define ACPI_HOTPLUG_H
+
+#ifndef HW_ACPI_CPU_HOTPLUG_H
+#define HW_ACPI_CPU_HOTPLUG_H
 
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/pc-hotplug.h"
+#include "hw/acpi/aml-build.h"
+#include "hw/hotplug.h"
+#include "hw/acpi/cpu.h"
 
 typedef struct AcpiCpuHotplug {
+    Object *device;
     MemoryRegion io;
     uint8_t sts[ACPI_GPE_PROC_LEN];
 } AcpiCpuHotplug;
 
-void acpi_cpu_plug_cb(ACPIREGS *ar, qemu_irq irq,
-                      AcpiCpuHotplug *g, DeviceState *dev, Error **errp);
+void legacy_acpi_cpu_plug_cb(HotplugHandler *hotplug_dev,
+                             AcpiCpuHotplug *g, DeviceState *dev, Error **errp);
 
-void acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
-                           AcpiCpuHotplug *gpe_cpu, uint16_t base);
+void legacy_acpi_cpu_hotplug_init(MemoryRegion *parent, Object *owner,
+                                  AcpiCpuHotplug *gpe_cpu, uint16_t base);
+
+void acpi_switch_to_modern_cphp(AcpiCpuHotplug *gpe_cpu,
+                                CPUHotplugState *cpuhp_state,
+                                uint16_t io_port);
+
+void build_legacy_cpu_hotplug_aml(Aml *ctx, MachineState *machine,
+                                  uint16_t io_base);
 #endif
diff --git a/include/hw/acpi/goldfish_defs.h b/include/hw/acpi/goldfish_defs.h
index 90555c6..a3ce5a7 100644
--- a/include/hw/acpi/goldfish_defs.h
+++ b/include/hw/acpi/goldfish_defs.h
@@ -23,38 +23,38 @@
 #define ACPI_GOLDFISH_DEFS_H
 
 /* goldfish battery */
-#define GF_BATTERY_IOMEM_BASE   0xff010000
-#define GF_BATTERY_IOMEM_SIZE   0x00001000
-#define GF_BATTERY_IRQ          16
+#define GOLDFISH_BATTERY_IOMEM_BASE   0xff010000
+#define GOLDFISH_BATTERY_IOMEM_SIZE   0x00001000
+#define GOLDFISH_BATTERY_IRQ          16
 
 /* goldfish events */
-#define GF_EVENTS_IOMEM_BASE    0xff011000
-#define GF_EVENTS_IOMEM_SIZE    0x00001000
-#define GF_EVENTS_IRQ           17
+#define GOLDFISH_EVENTS_IOMEM_BASE    0xff011000
+#define GOLDFISH_EVENTS_IOMEM_SIZE    0x00001000
+#define GOLDFISH_EVENTS_IRQ           17
 
 /* android pipe */
-#define GF_PIPE_IOMEM_BASE      0xff001000
-#define GF_PIPE_IOMEM_SIZE      0x00002000
-#define GF_PIPE_IRQ             18
+#define GOLDFISH_PIPE_IOMEM_BASE      0xff001000
+#define GOLDFISH_PIPE_IOMEM_SIZE      0x00002000
+#define GOLDFISH_PIPE_IRQ             18
 
 /* goldfish framebuffer */
-#define GF_FB_IOMEM_BASE        0xff012000
-#define GF_FB_IOMEM_SIZE        0x00000100
-#define GF_FB_IRQ               19
+#define GOLDFISH_FB_IOMEM_BASE        0xff012000
+#define GOLDFISH_FB_IOMEM_SIZE        0x00000100
+#define GOLDFISH_FB_IRQ               19
 
 /* goldfish audio */
-#define GF_AUDIO_IOMEM_BASE     0xff013000
-#define GF_AUDIO_IOMEM_SIZE     0x00000100
-#define GF_AUDIO_IRQ            20
+#define GOLDFISH_AUDIO_IOMEM_BASE     0xff013000
+#define GOLDFISH_AUDIO_IOMEM_SIZE     0x00000100
+#define GOLDFISH_AUDIO_IRQ            20
 
 /* goldfish sync */
-#define GF_SYNC_IOMEM_BASE      0xff014000
-#define GF_SYNC_IOMEM_SIZE      0x00002000
-#define GF_SYNC_IRQ             21
+#define GOLDFISH_SYNC_IOMEM_BASE      0xff014000
+#define GOLDFISH_SYNC_IOMEM_SIZE      0x00002000
+#define GOLDFISH_SYNC_IRQ             21
 
 /* goldfish rtc */
-#define GF_RTC_IOMEM_BASE       0xff016000
-#define GF_RTC_IOMEM_SIZE       0x00001000
-#define GF_RTC_IRQ              22
+#define GOLDFISH_RTC_IOMEM_BASE       0xff016000
+#define GOLDFISH_RTC_IOMEM_SIZE       0x00001000
+#define GOLDFISH_RTC_IRQ              22
 
 #endif  /* !ACPI_GOLDFISH_DEFS_H */
diff --git a/include/hw/acpi/ich9.h b/include/hw/acpi/ich9.h
index fe975e6..a352c94 100644
--- a/include/hw/acpi/ich9.h
+++ b/include/hw/acpi/ich9.h
@@ -23,8 +23,10 @@
 
 #include "hw/acpi/acpi.h"
 #include "hw/acpi/cpu_hotplug.h"
+#include "hw/acpi/cpu.h"
 #include "hw/acpi/memory_hotplug.h"
 #include "hw/acpi/acpi_dev_interface.h"
+#include "hw/acpi/tco.h"
 
 typedef struct ICH9LPCPMRegs {
     /*
@@ -39,6 +41,7 @@
     MemoryRegion io_smi;
 
     uint32_t smi_en;
+    uint32_t smi_en_wmask;
     uint32_t smi_sts;
 
     qemu_irq irq;      /* SCI */
@@ -46,19 +49,35 @@
     uint32_t pm_io_base;
     Notifier powerdown_notifier;
 
+    bool cpu_hotplug_legacy;
     AcpiCpuHotplug gpe_cpu;
+    CPUHotplugState cpuhp_state;
 
     MemHotplugState acpi_memory_hotplug;
+
+    uint8_t disable_s3;
+    uint8_t disable_s4;
+    uint8_t s4_val;
+    uint8_t smm_enabled;
+    bool enable_tco;
+    TCOIORegs tco_regs;
 } ICH9LPCPMRegs;
 
 void ich9_pm_init(PCIDevice *lpc_pci, ICH9LPCPMRegs *pm,
+                  bool smm_enabled,
                   qemu_irq sci_irq);
+
 void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base);
 extern const VMStateDescription vmstate_ich9_pm;
 
 void ich9_pm_add_properties(Object *obj, ICH9LPCPMRegs *pm, Error **errp);
 
-void ich9_pm_device_plug_cb(ICH9LPCPMRegs *pm, DeviceState *dev, Error **errp);
+void ich9_pm_device_plug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
+                            Error **errp);
+void ich9_pm_device_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                      DeviceState *dev, Error **errp);
+void ich9_pm_device_unplug_cb(HotplugHandler *hotplug_dev, DeviceState *dev,
+                              Error **errp);
 
 void ich9_pm_ospm_status(AcpiDeviceIf *adev, ACPIOSTInfoList ***list);
 #endif /* HW_ACPI_ICH9_H */
diff --git a/include/hw/acpi/ipmi.h b/include/hw/acpi/ipmi.h
new file mode 100644
index 0000000..ab2bb29
--- /dev/null
+++ b/include/hw/acpi/ipmi.h
@@ -0,0 +1,22 @@
+/*
+ * QEMU IPMI ACPI handling
+ *
+ * Copyright (c) 2015,2016 Corey Minyard <cminyard@mvista.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef HW_ACPI_IPMI_H
+#define HW_ACPI_IPMI_H
+
+#include "qemu/osdep.h"
+#include "hw/acpi/aml-build.h"
+
+/*
+ * Add ACPI IPMI entries for all registered IPMI devices whose parent
+ * bus matches the given bus.  The resource is the ACPI resource that
+ * contains the IPMI device, this is required for the I2C CRS.
+ */
+void build_acpi_ipmi_devices(Aml *table, BusState *bus);
+
+#endif /* HW_ACPI_IPMI_H */
diff --git a/include/hw/acpi/memory_hotplug.h b/include/hw/acpi/memory_hotplug.h
index 7bbf8a0..d2c7452 100644
--- a/include/hw/acpi/memory_hotplug.h
+++ b/include/hw/acpi/memory_hotplug.h
@@ -4,13 +4,19 @@
 #include "hw/qdev-core.h"
 #include "hw/acpi/acpi.h"
 #include "migration/vmstate.h"
+#include "hw/acpi/aml-build.h"
 
-#define ACPI_MEMORY_HOTPLUG_STATUS 8
-
+/**
+ * MemStatus:
+ * @is_removing: the memory device in slot has been requested to be ejected.
+ *
+ * This structure stores memory device's status.
+ */
 typedef struct MemStatus {
     DeviceState *dimm;
     bool is_enabled;
     bool is_inserting;
+    bool is_removing;
     uint32_t ost_event;
     uint32_t ost_status;
 } MemStatus;
@@ -26,8 +32,13 @@
 void acpi_memory_hotplug_init(MemoryRegion *as, Object *owner,
                               MemHotplugState *state);
 
-void acpi_memory_plug_cb(ACPIREGS *ar, qemu_irq irq, MemHotplugState *mem_st,
+void acpi_memory_plug_cb(HotplugHandler *hotplug_dev, MemHotplugState *mem_st,
                          DeviceState *dev, Error **errp);
+void acpi_memory_unplug_request_cb(HotplugHandler *hotplug_dev,
+                                   MemHotplugState *mem_st,
+                                   DeviceState *dev, Error **errp);
+void acpi_memory_unplug_cb(MemHotplugState *mem_st,
+                           DeviceState *dev, Error **errp);
 
 extern const VMStateDescription vmstate_memory_hotplug;
 #define VMSTATE_MEMORY_HOTPLUG(memhp, state) \
@@ -35,4 +46,12 @@
                    vmstate_memory_hotplug, MemHotplugState)
 
 void acpi_memory_ospm_status(MemHotplugState *mem_st, ACPIOSTInfoList ***list);
+
+#define MEMORY_HOTPLUG_DEVICE        "MHPD"
+#define MEMORY_SLOT_SCAN_METHOD      "MSCN"
+#define MEMORY_HOTPLUG_HANDLER_PATH "\\_SB.PCI0." \
+     MEMORY_HOTPLUG_DEVICE "." MEMORY_SLOT_SCAN_METHOD
+
+void build_memory_hotplug_aml(Aml *ctx, uint32_t nr_mem,
+                              uint16_t io_base, uint16_t io_len);
 #endif
diff --git a/include/hw/acpi/pc-hotplug.h b/include/hw/acpi/pc-hotplug.h
index b9db295..6a8d268 100644
--- a/include/hw/acpi/pc-hotplug.h
+++ b/include/hw/acpi/pc-hotplug.h
@@ -16,7 +16,6 @@
  * ONLY DEFINEs are permited in this file since it's shared
  * between C and ASL code.
  */
-#define ACPI_CPU_HOTPLUG_STATUS 4
 
 /* Limit for CPU arch IDs for CPU hotplug. All hotpluggable CPUs should
  * have CPUClass.get_arch_id() < ACPI_CPU_HOTPLUG_ID_LIMIT.
@@ -28,29 +27,31 @@
 
 #define ICH9_CPU_HOTPLUG_IO_BASE 0x0CD8
 #define PIIX4_CPU_HOTPLUG_IO_BASE 0xaf00
+#define CPU_HOTPLUG_RESOURCE_DEVICE PRES
 
 #define ACPI_MEMORY_HOTPLUG_IO_LEN 24
 #define ACPI_MEMORY_HOTPLUG_BASE 0x0a00
 
-#define MEMORY_HOTPLUG_DEVICE        MHPD
-#define MEMORY_SLOTS_NUMBER          MDNR
-#define MEMORY_HOTPLUG_IO_REGION     HPMR
-#define MEMORY_SLOT_ADDR_LOW         MRBL
-#define MEMORY_SLOT_ADDR_HIGH        MRBH
-#define MEMORY_SLOT_SIZE_LOW         MRLL
-#define MEMORY_SLOT_SIZE_HIGH        MRLH
-#define MEMORY_SLOT_PROXIMITY        MPX
-#define MEMORY_SLOT_ENABLED          MES
-#define MEMORY_SLOT_INSERT_EVENT     MINS
-#define MEMORY_SLOT_SLECTOR          MSEL
-#define MEMORY_SLOT_OST_EVENT        MOEV
-#define MEMORY_SLOT_OST_STATUS       MOSC
-#define MEMORY_SLOT_LOCK             MLCK
-#define MEMORY_SLOT_STATUS_METHOD    MRST
-#define MEMORY_SLOT_CRS_METHOD       MCRS
-#define MEMORY_SLOT_OST_METHOD       MOST
-#define MEMORY_SLOT_PROXIMITY_METHOD MPXM
-#define MEMORY_SLOT_NOTIFY_METHOD    MTFY
-#define MEMORY_SLOT_SCAN_METHOD      MSCN
+#define MEMORY_SLOTS_NUMBER          "MDNR"
+#define MEMORY_HOTPLUG_IO_REGION     "HPMR"
+#define MEMORY_SLOT_ADDR_LOW         "MRBL"
+#define MEMORY_SLOT_ADDR_HIGH        "MRBH"
+#define MEMORY_SLOT_SIZE_LOW         "MRLL"
+#define MEMORY_SLOT_SIZE_HIGH        "MRLH"
+#define MEMORY_SLOT_PROXIMITY        "MPX"
+#define MEMORY_SLOT_ENABLED          "MES"
+#define MEMORY_SLOT_INSERT_EVENT     "MINS"
+#define MEMORY_SLOT_REMOVE_EVENT     "MRMV"
+#define MEMORY_SLOT_EJECT            "MEJ"
+#define MEMORY_SLOT_SLECTOR          "MSEL"
+#define MEMORY_SLOT_OST_EVENT        "MOEV"
+#define MEMORY_SLOT_OST_STATUS       "MOSC"
+#define MEMORY_SLOT_LOCK             "MLCK"
+#define MEMORY_SLOT_STATUS_METHOD    "MRST"
+#define MEMORY_SLOT_CRS_METHOD       "MCRS"
+#define MEMORY_SLOT_OST_METHOD       "MOST"
+#define MEMORY_SLOT_PROXIMITY_METHOD "MPXM"
+#define MEMORY_SLOT_EJECT_METHOD     "MEJ0"
+#define MEMORY_SLOT_NOTIFY_METHOD    "MTFY"
 
 #endif
diff --git a/include/hw/acpi/pcihp.h b/include/hw/acpi/pcihp.h
index 9323838..04528b7 100644
--- a/include/hw/acpi/pcihp.h
+++ b/include/hw/acpi/pcihp.h
@@ -27,10 +27,12 @@
 #ifndef HW_ACPI_PCIHP_H
 #define HW_ACPI_PCIHP_H
 
-#include <inttypes.h>
-#include <qemu/typedefs.h>
 #include "hw/acpi/acpi.h"
 #include "migration/vmstate.h"
+#include "hw/hotplug.h"
+
+#define ACPI_PCIHP_IO_BASE_PROP "acpi-pcihp-io-base"
+#define ACPI_PCIHP_IO_LEN_PROP "acpi-pcihp-io-len"
 
 typedef struct AcpiPciHpPciStatus {
     uint32_t up;
@@ -48,14 +50,16 @@
     PCIBus *root;
     MemoryRegion io;
     bool legacy_piix;
+    uint16_t io_base;
+    uint16_t io_len;
 } AcpiPciHpState;
 
-void acpi_pcihp_init(AcpiPciHpState *, PCIBus *root,
+void acpi_pcihp_init(Object *owner, AcpiPciHpState *, PCIBus *root,
                      MemoryRegion *address_space_io, bool bridges_enabled);
 
-void acpi_pcihp_device_plug_cb(ACPIREGS *ar, qemu_irq irq, AcpiPciHpState *s,
+void acpi_pcihp_device_plug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s,
                                DeviceState *dev, Error **errp);
-void acpi_pcihp_device_unplug_cb(ACPIREGS *ar, qemu_irq irq, AcpiPciHpState *s,
+void acpi_pcihp_device_unplug_cb(HotplugHandler *hotplug_dev, AcpiPciHpState *s,
                                  DeviceState *dev, Error **errp);
 
 /* Called on reset */
diff --git a/include/hw/acpi/piix4.h b/include/hw/acpi/piix4.h
index 65e6fd7..26c2370 100644
--- a/include/hw/acpi/piix4.h
+++ b/include/hw/acpi/piix4.h
@@ -1,8 +1,6 @@
 #ifndef HW_ACPI_PIIX4_H
 #define HW_ACPI_PIIX4_H
 
-#include "qemu/typedefs.h"
-
 Object *piix4_pm_find(void);
 
 #endif
diff --git a/include/hw/acpi/tco.h b/include/hw/acpi/tco.h
new file mode 100644
index 0000000..52ad767
--- /dev/null
+++ b/include/hw/acpi/tco.h
@@ -0,0 +1,81 @@
+/*
+ * QEMU ICH9 TCO emulation
+ *
+ * Copyright (c) 2015 Paulo Alcantara <pcacjr@zytor.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef HW_ACPI_TCO_H
+#define HW_ACPI_TCO_H
+
+#include "qemu-common.h"
+
+/* As per ICH9 spec, the internal timer has an error of ~0.6s on every tick */
+#define TCO_TICK_NSEC 600000000LL
+
+/* TCO I/O register offsets */
+enum {
+    TCO_RLD           = 0x00,
+    TCO_DAT_IN        = 0x02,
+    TCO_DAT_OUT       = 0x03,
+    TCO1_STS          = 0x04,
+    TCO2_STS          = 0x06,
+    TCO1_CNT          = 0x08,
+    TCO2_CNT          = 0x0a,
+    TCO_MESSAGE1      = 0x0c,
+    TCO_MESSAGE2      = 0x0d,
+    TCO_WDCNT         = 0x0e,
+    SW_IRQ_GEN        = 0x10,
+    TCO_TMR           = 0x12,
+};
+
+/* TCO I/O register control/status bits */
+enum {
+    SW_TCO_SMI           = 1 << 1,
+    TCO_INT_STS          = 1 << 2,
+    TCO_LOCK             = 1 << 12,
+    TCO_TMR_HLT          = 1 << 11,
+    TCO_TIMEOUT          = 1 << 3,
+    TCO_SECOND_TO_STS    = 1 << 1,
+    TCO_BOOT_STS         = 1 << 2,
+};
+
+/* TCO I/O registers mask bits */
+enum {
+    TCO_RLD_MASK     = 0x3ff,
+    TCO1_STS_MASK    = 0xe870,
+    TCO2_STS_MASK    = 0xfff8,
+    TCO1_CNT_MASK    = 0xfeff,
+    TCO_TMR_MASK     = 0x3ff,
+};
+
+typedef struct TCOIORegs {
+    struct {
+        uint16_t rld;
+        uint8_t din;
+        uint8_t dout;
+        uint16_t sts1;
+        uint16_t sts2;
+        uint16_t cnt1;
+        uint16_t cnt2;
+        uint8_t msg1;
+        uint8_t msg2;
+        uint8_t wdcnt;
+        uint16_t tmr;
+    } tco;
+    uint8_t sw_irq_gen;
+
+    QEMUTimer *tco_timer;
+    int64_t expire_time;
+    uint8_t timeouts_no;
+
+    MemoryRegion io;
+} TCOIORegs;
+
+/* tco.c */
+void acpi_pm_tco_init(TCOIORegs *tr, MemoryRegion *parent);
+
+extern const VMStateDescription vmstate_tco_io_sts;
+
+#endif /* HW_ACPI_TCO_H */
diff --git a/include/hw/acpi/tpm.h b/include/hw/acpi/tpm.h
index 792fcbf..6d516c6 100644
--- a/include/hw/acpi/tpm.h
+++ b/include/hw/acpi/tpm.h
@@ -26,4 +26,9 @@
 #define TPM_TCPA_ACPI_CLASS_CLIENT  0
 #define TPM_TCPA_ACPI_CLASS_SERVER  1
 
+#define TPM2_ACPI_CLASS_CLIENT      0
+#define TPM2_ACPI_CLASS_SERVER      1
+
+#define TPM2_START_METHOD_MMIO      6
+
 #endif /* HW_ACPI_TPM_H */
diff --git a/include/hw/arm/allwinner-a10.h b/include/hw/arm/allwinner-a10.h
index 01a189b..6b32a99 100644
--- a/include/hw/arm/allwinner-a10.h
+++ b/include/hw/arm/allwinner-a10.h
@@ -7,6 +7,8 @@
 #include "hw/timer/allwinner-a10-pit.h"
 #include "hw/intc/allwinner-a10-pic.h"
 #include "hw/net/allwinner_emac.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
 
 #include "sysemu/sysemu.h"
 #include "exec/address-spaces.h"
@@ -16,6 +18,7 @@
 #define AW_A10_PIT_REG_BASE     0x01c20c00
 #define AW_A10_UART0_REG_BASE   0x01c28000
 #define AW_A10_EMAC_BASE        0x01c0b000
+#define AW_A10_SATA_BASE        0x01c18000
 
 #define AW_A10_SDRAM_BASE       0x40000000
 
@@ -32,6 +35,7 @@
     AwA10PITState timer;
     AwA10PICState intc;
     AwEmacState emac;
+    AllwinnerAHCIState sata;
 } AwA10State;
 
 #define ALLWINNER_H_
diff --git a/include/hw/arm/arm.h b/include/hw/arm/arm.h
index cefc9e6..aeeebfe 100644
--- a/include/hw/arm/arm.h
+++ b/include/hw/arm/arm.h
@@ -8,17 +8,34 @@
  *
  */
 
-#ifndef ARM_MISC_H
-#define ARM_MISC_H 1
+#ifndef HW_ARM_H
+#define HW_ARM_H
 
 #include "exec/memory.h"
+#include "target-arm/cpu-qom.h"
 #include "hw/irq.h"
+#include "qemu/notify.h"
+
+typedef enum {
+    ARM_ENDIANNESS_UNKNOWN = 0,
+    ARM_ENDIANNESS_LE,
+    ARM_ENDIANNESS_BE8,
+    ARM_ENDIANNESS_BE32,
+} arm_endianness;
 
 /* armv7m.c */
-qemu_irq *armv7m_init(MemoryRegion *system_memory,
-                      int flash_size, int sram_size,
+DeviceState *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq,
                       const char *kernel_filename, const char *cpu_model);
 
+/*
+ * struct used as a parameter of the arm_load_kernel machine init
+ * done notifier
+ */
+typedef struct {
+    Notifier notifier; /* actual notifier */
+    ARMCPU *cpu; /* handle to the first cpu object */
+} ArmLoadKernelNotifier;
+
 /* arm_boot.c */
 struct arm_boot_info {
     uint64_t ram_size;
@@ -37,6 +54,10 @@
     hwaddr gic_cpu_if_addr;
     int nb_cpus;
     int board_id;
+    /* ARM machines that support the ARM Security Extensions use this field to
+     * control whether Linux is booted as secure(true) or non-secure(false).
+     */
+    bool secure_boot;
     int (*atag_board)(const struct arm_boot_info *info, void *p);
     /* multicore boards that use the default secondary core boot functions
      * can ignore these two function calls. If the default functions won't
@@ -61,16 +82,62 @@
      * the user it should implement this hook.
      */
     void (*modify_dtb)(const struct arm_boot_info *info, void *fdt);
+    /* machine init done notifier executing arm_load_dtb */
+    ArmLoadKernelNotifier load_kernel_notifier;
     /* Used internally by arm_boot.c */
     int is_linux;
     hwaddr initrd_start;
     hwaddr initrd_size;
     hwaddr entry;
+
+    /* Boot firmware has been loaded, typically at address 0, with -bios or
+     * -pflash. It also implies that fw_cfg_find() will succeed.
+     */
+    bool firmware_loaded;
+
+    /* Address at which board specific loader/setup code exists. If enabled,
+     * this code-blob will run before anything else. It must return to the
+     * caller via the link register. There is no stack set up. Enabled by
+     * defining write_board_setup, which is responsible for loading the blob
+     * to the specified address.
+     */
+    hwaddr board_setup_addr;
+    void (*write_board_setup)(ARMCPU *cpu,
+                              const struct arm_boot_info *info);
+
+    /* If set, the board specific loader/setup blob will be run from secure
+     * mode, regardless of secure_boot. The blob becomes responsible for
+     * changing to non-secure state if implementing a non-secure boot
+     */
+    bool secure_board_setup;
+
+    arm_endianness endianness;
 };
+
+/**
+ * arm_load_kernel - Loads memory with everything needed to boot
+ *
+ * @cpu: handle to the first CPU object
+ * @info: handle to the boot info struct
+ * Registers a machine init done notifier that copies to memory
+ * everything needed to boot, depending on machine and user options:
+ * kernel image, boot loaders, initrd, dtb. Also registers the CPU
+ * reset handler.
+ *
+ * In case the machine file supports the platform bus device and its
+ * dynamically instantiable sysbus devices, this function must be called
+ * before sysbus-fdt arm_register_platform_bus_fdt_creator. Indeed the
+ * machine init done notifiers are called in registration reverse order.
+ */
 void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info);
 
+/* Write a secure board setup routine with a dummy handler for SMCs */
+void arm_write_secure_board_setup_dummy_smc(ARMCPU *cpu,
+                                            const struct arm_boot_info *info,
+                                            hwaddr mvbar_addr);
+
 /* Multiplication factor to convert from system clock ticks to qemu timer
    ticks.  */
 extern int system_clock_scale;
 
-#endif /* !ARM_MISC_H */
+#endif /* HW_ARM_H */
diff --git a/include/hw/arm/ast2400.h b/include/hw/arm/ast2400.h
new file mode 100644
index 0000000..7833bc7
--- /dev/null
+++ b/include/hw/arm/ast2400.h
@@ -0,0 +1,42 @@
+/*
+ * ASPEED AST2400 SoC
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef AST2400_H
+#define AST2400_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/aspeed_vic.h"
+#include "hw/misc/aspeed_scu.h"
+#include "hw/timer/aspeed_timer.h"
+#include "hw/i2c/aspeed_i2c.h"
+#include "hw/ssi/aspeed_smc.h"
+
+typedef struct AST2400State {
+    /*< private >*/
+    DeviceState parent;
+
+    /*< public >*/
+    ARMCPU *cpu;
+    MemoryRegion iomem;
+    AspeedVICState vic;
+    AspeedTimerCtrlState timerctrl;
+    AspeedI2CState i2c;
+    AspeedSCUState scu;
+    AspeedSMCState smc;
+    AspeedSMCState spi;
+} AST2400State;
+
+#define TYPE_AST2400 "ast2400"
+#define AST2400(obj) OBJECT_CHECK(AST2400State, (obj), TYPE_AST2400)
+
+#define AST2400_SDRAM_BASE       0x40000000
+
+#endif /* AST2400_H */
diff --git a/include/hw/arm/bcm2835_peripherals.h b/include/hw/arm/bcm2835_peripherals.h
new file mode 100644
index 0000000..e12ae37
--- /dev/null
+++ b/include/hw/arm/bcm2835_peripherals.h
@@ -0,0 +1,48 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_PERIPHERALS_H
+#define BCM2835_PERIPHERALS_H
+
+#include "qemu-common.h"
+#include "exec/address-spaces.h"
+#include "hw/sysbus.h"
+#include "hw/char/bcm2835_aux.h"
+#include "hw/display/bcm2835_fb.h"
+#include "hw/dma/bcm2835_dma.h"
+#include "hw/intc/bcm2835_ic.h"
+#include "hw/misc/bcm2835_property.h"
+#include "hw/misc/bcm2835_mbox.h"
+#include "hw/sd/sdhci.h"
+
+#define TYPE_BCM2835_PERIPHERALS "bcm2835-peripherals"
+#define BCM2835_PERIPHERALS(obj) \
+    OBJECT_CHECK(BCM2835PeripheralState, (obj), TYPE_BCM2835_PERIPHERALS)
+
+typedef struct BCM2835PeripheralState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion peri_mr, peri_mr_alias, gpu_bus_mr, mbox_mr;
+    MemoryRegion ram_alias[4];
+    qemu_irq irq, fiq;
+
+    SysBusDevice *uart0;
+    BCM2835AuxState aux;
+    BCM2835FBState fb;
+    BCM2835DMAState dma;
+    BCM2835ICState ic;
+    BCM2835PropertyState property;
+    BCM2835MboxState mboxes;
+    SDHCIState sdhci;
+} BCM2835PeripheralState;
+
+#endif /* BCM2835_PERIPHERALS_H */
diff --git a/include/hw/arm/bcm2836.h b/include/hw/arm/bcm2836.h
new file mode 100644
index 0000000..76de199
--- /dev/null
+++ b/include/hw/arm/bcm2836.h
@@ -0,0 +1,35 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2836_H
+#define BCM2836_H
+
+#include "hw/arm/arm.h"
+#include "hw/arm/bcm2835_peripherals.h"
+#include "hw/intc/bcm2836_control.h"
+
+#define TYPE_BCM2836 "bcm2836"
+#define BCM2836(obj) OBJECT_CHECK(BCM2836State, (obj), TYPE_BCM2836)
+
+#define BCM2836_NCPUS 4
+
+typedef struct BCM2836State {
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
+
+    uint32_t enabled_cpus;
+
+    ARMCPU cpus[BCM2836_NCPUS];
+    BCM2836ControlState control;
+    BCM2835PeripheralState peripherals;
+} BCM2836State;
+
+#endif /* BCM2836_H */
diff --git a/include/hw/arm/digic.h b/include/hw/arm/digic.h
index a739d6a..63785ba 100644
--- a/include/hw/arm/digic.h
+++ b/include/hw/arm/digic.h
@@ -19,7 +19,6 @@
 #define HW_ARM_DIGIC_H
 
 #include "cpu.h"
-
 #include "hw/timer/digic-timer.h"
 #include "hw/char/digic-uart.h"
 
diff --git a/include/hw/arm/exynos4210.h b/include/hw/arm/exynos4210.h
index 5c1820f..29fef8b 100644
--- a/include/hw/arm/exynos4210.h
+++ b/include/hw/arm/exynos4210.h
@@ -22,12 +22,12 @@
  *
  */
 
-
-#ifndef EXYNOS4210_H_
-#define EXYNOS4210_H_
+#ifndef EXYNOS4210_H
+#define EXYNOS4210_H
 
 #include "qemu-common.h"
 #include "exec/memory.h"
+#include "target-arm/cpu-qom.h"
 
 #define EXYNOS4210_NCPUS                    2
 
@@ -134,4 +134,4 @@
                                     CharDriverState *chr,
                                     qemu_irq irq);
 
-#endif /* EXYNOS4210_H_ */
+#endif /* EXYNOS4210_H */
diff --git a/include/hw/arm/fdt.h b/include/hw/arm/fdt.h
new file mode 100644
index 0000000..c3d5015
--- /dev/null
+++ b/include/hw/arm/fdt.h
@@ -0,0 +1,34 @@
+/*
+ *
+ * Copyright (c) 2015 Linaro Limited
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Define macros useful when building ARM device tree nodes
+ */
+
+#ifndef QEMU_ARM_FDT_H
+#define QEMU_ARM_FDT_H
+
+#define GIC_FDT_IRQ_TYPE_SPI 0
+#define GIC_FDT_IRQ_TYPE_PPI 1
+
+#define GIC_FDT_IRQ_FLAGS_EDGE_LO_HI 1
+#define GIC_FDT_IRQ_FLAGS_EDGE_HI_LO 2
+#define GIC_FDT_IRQ_FLAGS_LEVEL_HI 4
+#define GIC_FDT_IRQ_FLAGS_LEVEL_LO 8
+
+#define GIC_FDT_IRQ_PPI_CPU_START 8
+#define GIC_FDT_IRQ_PPI_CPU_WIDTH 8
+
+#endif
diff --git a/include/hw/arm/fsl-imx25.h b/include/hw/arm/fsl-imx25.h
new file mode 100644
index 0000000..d0e8e9d
--- /dev/null
+++ b/include/hw/arm/fsl-imx25.h
@@ -0,0 +1,249 @@
+/*
+ * Freescale i.MX25 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX25_H
+#define FSL_IMX25_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/imx_avic.h"
+#include "hw/misc/imx25_ccm.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/net/imx_fec.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "exec/memory.h"
+
+#define TYPE_FSL_IMX25 "fsl,imx25"
+#define FSL_IMX25(obj) OBJECT_CHECK(FslIMX25State, (obj), TYPE_FSL_IMX25)
+
+#define FSL_IMX25_NUM_UARTS 5
+#define FSL_IMX25_NUM_GPTS 4
+#define FSL_IMX25_NUM_EPITS 2
+#define FSL_IMX25_NUM_I2CS 3
+#define FSL_IMX25_NUM_GPIOS 4
+
+typedef struct FslIMX25State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu;
+    IMXAVICState   avic;
+    IMX25CCMState  ccm;
+    IMXSerialState uart[FSL_IMX25_NUM_UARTS];
+    IMXGPTState    gpt[FSL_IMX25_NUM_GPTS];
+    IMXEPITState   epit[FSL_IMX25_NUM_EPITS];
+    IMXFECState    fec;
+    IMXI2CState    i2c[FSL_IMX25_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX25_NUM_GPIOS];
+    MemoryRegion   rom[2];
+    MemoryRegion   iram;
+    MemoryRegion   iram_alias;
+} FslIMX25State;
+
+/**
+ * i.MX25 memory map
+ ****************************************************************
+ * 0x0000_0000 0x0000_3FFF 16 Kbytes    ROM (36 Kbytes)
+ * 0x0000_4000 0x0040_3FFF 4 Mbytes     Reserved
+ * 0x0040_4000 0x0040_8FFF 20 Kbytes    ROM (36 Kbytes)
+ * 0x0040_9000 0x0FFF_FFFF 252 Mbytes (minus 36 Kbytes) Reserved
+ * 0x1000_0000 0x1FFF_FFFF 256 Mbytes   Reserved
+ * 0x2000_0000 0x2FFF_FFFF 256 Mbytes   Reserved
+ * 0x3000_0000 0x3FFF_FFFF 256 Mbytes   Reserved
+ * 0x4000_0000 0x43EF_FFFF 63 Mbytes    Reserved
+ * 0x43F0_0000 0x43F0_3FFF 16 Kbytes    AIPS A control registers
+ * 0x43F0_4000 0x43F0_7FFF 16 Kbytes    ARM926 platform MAX
+ * 0x43F0_8000 0x43F0_BFFF 16 Kbytes    ARM926 platform CLKCTL
+ * 0x43F0_C000 0x43F0_FFFF 16 Kbytes    ARM926 platform ETB registers
+ * 0x43F1_0000 0x43F1_3FFF 16 Kbytes    ARM926 platform ETB memory
+ * 0x43F1_4000 0x43F1_7FFF 16 Kbytes    ARM926 platform AAPE registers
+ * 0x43F1_8000 0x43F7_FFFF 416 Kbytes   Reserved
+ * 0x43F8_0000 0x43F8_3FFF 16 Kbytes    I2C-1
+ * 0x43F8_4000 0x43F8_7FFF 16 Kbytes    I2C-3
+ * 0x43F8_8000 0x43F8_BFFF 16 Kbytes    CAN-1
+ * 0x43F8_C000 0x43F8_FFFF 16 Kbytes    CAN-2
+ * 0x43F9_0000 0x43F9_3FFF 16 Kbytes    UART-1
+ * 0x43F9_4000 0x43F9_7FFF 16 Kbytes    UART-2
+ * 0x43F9_8000 0x43F9_BFFF 16 Kbytes    I2C-2
+ * 0x43F9_C000 0x43F9_FFFF 16 Kbytes    1-Wire
+ * 0x43FA_0000 0x43FA_3FFF 16 Kbytes    ATA (CPU side)
+ * 0x43FA_4000 0x43FA_7FFF 16 Kbytes    CSPI-1
+ * 0x43FA_8000 0x43FA_BFFF 16 Kbytes    KPP
+ * 0x43FA_C000 0x43FA_FFFF 16 Kbytes    IOMUXC
+ * 0x43FB_0000 0x43FB_3FFF 16 Kbytes    AUDMUX
+ * 0x43FB_4000 0x43FB_7FFF 16 Kbytes    Reserved
+ * 0x43FB_8000 0x43FB_BFFF 16 Kbytes    ECT (IP BUS A)
+ * 0x43FB_C000 0x43FB_FFFF 16 Kbytes    ECT (IP BUS B)
+ * 0x43FC_0000 0x43FF_FFFF 256 Kbytes   Reserved AIPS A off-platform slots
+ * 0x4400_0000 0x4FFF_FFFF 192 Mbytes   Reserved
+ * 0x5000_0000 0x5000_3FFF 16 Kbytes    SPBA base address
+ * 0x5000_4000 0x5000_7FFF 16 Kbytes    CSPI-3
+ * 0x5000_8000 0x5000_BFFF 16 Kbytes    UART-4
+ * 0x5000_C000 0x5000_FFFF 16 Kbytes    UART-3
+ * 0x5001_0000 0x5001_3FFF 16 Kbytes    CSPI-2
+ * 0x5001_4000 0x5001_7FFF 16 Kbytes    SSI-2
+ * 0x5001_C000 0x5001_FFFF 16 Kbytes    Reserved
+ * 0x5002_0000 0x5002_3FFF 16 Kbytes    ATA
+ * 0x5002_4000 0x5002_7FFF 16 Kbytes    SIM-1
+ * 0x5002_8000 0x5002_BFFF 16 Kbytes    SIM-2
+ * 0x5002_C000 0x5002_FFFF 16 Kbytes    UART-5
+ * 0x5003_0000 0x5003_3FFF 16 Kbytes    TSC
+ * 0x5003_4000 0x5003_7FFF 16 Kbytes    SSI-1
+ * 0x5003_8000 0x5003_BFFF 16 Kbytes    FEC
+ * 0x5003_C000 0x5003_FFFF 16 Kbytes    SPBA registers
+ * 0x5004_0000 0x51FF_FFFF 32 Mbytes (minus 256 Kbytes)
+ * 0x5200_0000 0x53EF_FFFF 31 Mbytes    Reserved
+ * 0x53F0_0000 0x53F0_3FFF 16 Kbytes    AIPS B control registers
+ * 0x53F0_4000 0x53F7_FFFF 496 Kbytes   Reserved
+ * 0x53F8_0000 0x53F8_3FFF 16 Kbytes    CCM
+ * 0x53F8_4000 0x53F8_7FFF 16 Kbytes    GPT-4
+ * 0x53F8_8000 0x53F8_BFFF 16 Kbytes    GPT-3
+ * 0x53F8_C000 0x53F8_FFFF 16 Kbytes    GPT-2
+ * 0x53F9_0000 0x53F9_3FFF 16 Kbytes    GPT-1
+ * 0x53F9_4000 0x53F9_7FFF 16 Kbytes    EPIT-1
+ * 0x53F9_8000 0x53F9_BFFF 16 Kbytes    EPIT-2
+ * 0x53F9_C000 0x53F9_FFFF 16 Kbytes    GPIO-4
+ * 0x53FA_0000 0x53FA_3FFF 16 Kbytes    PWM-2
+ * 0x53FA_4000 0x53FA_7FFF 16 Kbytes    GPIO-3
+ * 0x53FA_8000 0x53FA_BFFF 16 Kbytes    PWM-3
+ * 0x53FA_C000 0x53FA_FFFF 16 Kbytes    SCC
+ * 0x53FB_0000 0x53FB_3FFF 16 Kbytes    RNGB
+ * 0x53FB_4000 0x53FB_7FFF 16 Kbytes    eSDHC-1
+ * 0x53FB_8000 0x53FB_BFFF 16 Kbytes    eSDHC-2
+ * 0x53FB_C000 0x53FB_FFFF 16 Kbytes    LCDC
+ * 0x53FC_0000 0x53FC_3FFF 16 Kbytes    SLCDC
+ * 0x53FC_4000 0x53FC_7FFF 16 Kbytes    Reserved
+ * 0x53FC_8000 0x53FC_BFFF 16 Kbytes    PWM-4
+ * 0x53FC_C000 0x53FC_FFFF 16 Kbytes    GPIO-1
+ * 0x53FD_0000 0x53FD_3FFF 16 Kbytes    GPIO-2
+ * 0x53FD_4000 0x53FD_7FFF 16 Kbytes    SDMA
+ * 0x53FD_8000 0x53FD_BFFF 16 Kbytes    Reserved
+ * 0x53FD_C000 0x53FD_FFFF 16 Kbytes    WDOG
+ * 0x53FE_0000 0x53FE_3FFF 16 Kbytes    PWM-1
+ * 0x53FE_4000 0x53FE_7FFF 16 Kbytes    Reserved
+ * 0x53FE_8000 0x53FE_BFFF 16 Kbytes    Reserved
+ * 0x53FE_C000 0x53FE_FFFF 16 Kbytes    RTICv3
+ * 0x53FF_0000 0x53FF_3FFF 16 Kbytes    IIM
+ * 0x53FF_4000 0x53FF_7FFF 16 Kbytes    USB
+ * 0x53FF_8000 0x53FF_BFFF 16 Kbytes    CSI
+ * 0x53FF_C000 0x53FF_FFFF 16 Kbytes    DryIce
+ * 0x5400_0000 0x5FFF_FFFF 192 Mbytes   Reserved (aliased AIPS B slots)
+ * 0x6000_0000 0x67FF_FFFF 128 Mbytes   ARM926 platform ROMPATCH
+ * 0x6800_0000 0x6FFF_FFFF 128 Mbytes   ARM926 platform ASIC
+ * 0x7000_0000 0x77FF_FFFF 128 Mbytes   Reserved
+ * 0x7800_0000 0x7801_FFFF 128 Kbytes   RAM
+ * 0x7802_0000 0x7FFF_FFFF 128 Mbytes (minus 128 Kbytes)
+ * 0x8000_0000 0x8FFF_FFFF 256 Mbytes   SDRAM bank 0
+ * 0x9000_0000 0x9FFF_FFFF 256 Mbytes   SDRAM bank 1
+ * 0xA000_0000 0xA7FF_FFFF 128 Mbytes   WEIM CS0 (flash 128) 1
+ * 0xA800_0000 0xAFFF_FFFF 128 Mbytes   WEIM CS1 (flash 64) 1
+ * 0xB000_0000 0xB1FF_FFFF 32 Mbytes    WEIM CS2 (SRAM)
+ * 0xB200_0000 0xB3FF_FFFF 32 Mbytes    WEIM CS3 (SRAM)
+ * 0xB400_0000 0xB5FF_FFFF 32 Mbytes    WEIM CS4
+ * 0xB600_0000 0xB7FF_FFFF 32 Mbytes    Reserved
+ * 0xB800_0000 0xB800_0FFF 4 Kbytes     Reserved
+ * 0xB800_1000 0xB800_1FFF 4 Kbytes     SDRAM control registers
+ * 0xB800_2000 0xB800_2FFF 4 Kbytes     WEIM control registers
+ * 0xB800_3000 0xB800_3FFF 4 Kbytes     M3IF control registers
+ * 0xB800_4000 0xB800_4FFF 4 Kbytes     EMI control registers
+ * 0xB800_5000 0xBAFF_FFFF 32 Mbytes (minus 20 Kbytes)
+ * 0xBB00_0000 0xBB00_0FFF 4 Kbytes     NAND flash main area buffer
+ * 0xBB00_1000 0xBB00_11FF 512 B        NAND flash spare area buffer
+ * 0xBB00_1200 0xBB00_1DFF 3 Kbytes     Reserved
+ * 0xBB00_1E00 0xBB00_1FFF 512 B        NAND flash control regisers
+ * 0xBB01_2000 0xBFFF_FFFF 96 Mbytes (minus 8 Kbytes) Reserved
+ * 0xC000_0000 0xFFFF_FFFF 1024 Mbytes  Reserved
+ */
+
+#define FSL_IMX25_ROM0_ADDR     0x00000000
+#define FSL_IMX25_ROM0_SIZE     0x4000
+#define FSL_IMX25_ROM1_ADDR     0x00404000
+#define FSL_IMX25_ROM1_SIZE     0x4000
+#define FSL_IMX25_I2C1_ADDR     0x43F80000
+#define FSL_IMX25_I2C1_SIZE     0x4000
+#define FSL_IMX25_I2C3_ADDR     0x43F84000
+#define FSL_IMX25_I2C3_SIZE     0x4000
+#define FSL_IMX25_UART1_ADDR    0x43F90000
+#define FSL_IMX25_UART1_SIZE    0x4000
+#define FSL_IMX25_UART2_ADDR    0x43F94000
+#define FSL_IMX25_UART2_SIZE    0x4000
+#define FSL_IMX25_I2C2_ADDR     0x43F98000
+#define FSL_IMX25_I2C2_SIZE     0x4000
+#define FSL_IMX25_UART4_ADDR    0x50008000
+#define FSL_IMX25_UART4_SIZE    0x4000
+#define FSL_IMX25_UART3_ADDR    0x5000C000
+#define FSL_IMX25_UART3_SIZE    0x4000
+#define FSL_IMX25_UART5_ADDR    0x5002C000
+#define FSL_IMX25_UART5_SIZE    0x4000
+#define FSL_IMX25_FEC_ADDR      0x50038000
+#define FSL_IMX25_FEC_SIZE      0x4000
+#define FSL_IMX25_CCM_ADDR      0x53F80000
+#define FSL_IMX25_CCM_SIZE      0x4000
+#define FSL_IMX25_GPT4_ADDR     0x53F84000
+#define FSL_IMX25_GPT4_SIZE     0x4000
+#define FSL_IMX25_GPT3_ADDR     0x53F88000
+#define FSL_IMX25_GPT3_SIZE     0x4000
+#define FSL_IMX25_GPT2_ADDR     0x53F8C000
+#define FSL_IMX25_GPT2_SIZE     0x4000
+#define FSL_IMX25_GPT1_ADDR     0x53F90000
+#define FSL_IMX25_GPT1_SIZE     0x4000
+#define FSL_IMX25_EPIT1_ADDR    0x53F94000
+#define FSL_IMX25_EPIT1_SIZE    0x4000
+#define FSL_IMX25_EPIT2_ADDR    0x53F98000
+#define FSL_IMX25_EPIT2_SIZE    0x4000
+#define FSL_IMX25_GPIO4_ADDR    0x53F9C000
+#define FSL_IMX25_GPIO4_SIZE    0x4000
+#define FSL_IMX25_GPIO3_ADDR    0x53FA4000
+#define FSL_IMX25_GPIO3_SIZE    0x4000
+#define FSL_IMX25_GPIO1_ADDR    0x53FCC000
+#define FSL_IMX25_GPIO1_SIZE    0x4000
+#define FSL_IMX25_GPIO2_ADDR    0x53FD0000
+#define FSL_IMX25_GPIO2_SIZE    0x4000
+#define FSL_IMX25_AVIC_ADDR     0x68000000
+#define FSL_IMX25_AVIC_SIZE     0x4000
+#define FSL_IMX25_IRAM_ADDR     0x78000000
+#define FSL_IMX25_IRAM_SIZE     0x20000
+#define FSL_IMX25_IRAM_ALIAS_ADDR     0x78020000
+#define FSL_IMX25_IRAM_ALIAS_SIZE     0x7FE0000
+#define FSL_IMX25_SDRAM0_ADDR   0x80000000
+#define FSL_IMX25_SDRAM0_SIZE   0x10000000
+#define FSL_IMX25_SDRAM1_ADDR   0x90000000
+#define FSL_IMX25_SDRAM1_SIZE   0x10000000
+
+#define FSL_IMX25_UART1_IRQ     45
+#define FSL_IMX25_UART2_IRQ     32
+#define FSL_IMX25_UART3_IRQ     18
+#define FSL_IMX25_UART4_IRQ     5
+#define FSL_IMX25_UART5_IRQ     40
+#define FSL_IMX25_GPT1_IRQ      54
+#define FSL_IMX25_GPT2_IRQ      53
+#define FSL_IMX25_GPT3_IRQ      29
+#define FSL_IMX25_GPT4_IRQ      1
+#define FSL_IMX25_EPIT1_IRQ     28
+#define FSL_IMX25_EPIT2_IRQ     27
+#define FSL_IMX25_FEC_IRQ       57
+#define FSL_IMX25_I2C1_IRQ      3
+#define FSL_IMX25_I2C2_IRQ      4
+#define FSL_IMX25_I2C3_IRQ      10
+#define FSL_IMX25_GPIO1_IRQ     52
+#define FSL_IMX25_GPIO2_IRQ     51
+#define FSL_IMX25_GPIO3_IRQ     16
+#define FSL_IMX25_GPIO4_IRQ     23
+
+#endif /* FSL_IMX25_H */
diff --git a/include/hw/arm/fsl-imx31.h b/include/hw/arm/fsl-imx31.h
new file mode 100644
index 0000000..d408abb
--- /dev/null
+++ b/include/hw/arm/fsl-imx31.h
@@ -0,0 +1,122 @@
+/*
+ * Freescale i.MX31 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX31_H
+#define FSL_IMX31_H
+
+#include "hw/arm/arm.h"
+#include "hw/intc/imx_avic.h"
+#include "hw/misc/imx31_ccm.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "exec/memory.h"
+
+#define TYPE_FSL_IMX31 "fsl,imx31"
+#define FSL_IMX31(obj) OBJECT_CHECK(FslIMX31State, (obj), TYPE_FSL_IMX31)
+
+#define FSL_IMX31_NUM_UARTS 2
+#define FSL_IMX31_NUM_EPITS 2
+#define FSL_IMX31_NUM_I2CS 3
+#define FSL_IMX31_NUM_GPIOS 3
+
+typedef struct FslIMX31State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu;
+    IMXAVICState   avic;
+    IMX31CCMState  ccm;
+    IMXSerialState uart[FSL_IMX31_NUM_UARTS];
+    IMXGPTState    gpt;
+    IMXEPITState   epit[FSL_IMX31_NUM_EPITS];
+    IMXI2CState    i2c[FSL_IMX31_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX31_NUM_GPIOS];
+    MemoryRegion   secure_rom;
+    MemoryRegion   rom;
+    MemoryRegion   iram;
+    MemoryRegion   iram_alias;
+} FslIMX31State;
+
+#define FSL_IMX31_SECURE_ROM_ADDR       0x00000000
+#define FSL_IMX31_SECURE_ROM_SIZE       0x4000
+#define FSL_IMX31_ROM_ADDR              0x00404000
+#define FSL_IMX31_ROM_SIZE              0x4000
+#define FSL_IMX31_IRAM_ALIAS_ADDR       0x10000000
+#define FSL_IMX31_IRAM_ALIAS_SIZE       0xFFC0000
+#define FSL_IMX31_IRAM_ADDR             0x1FFFC000
+#define FSL_IMX31_IRAM_SIZE             0x4000
+#define FSL_IMX31_I2C1_ADDR             0x43F80000
+#define FSL_IMX31_I2C1_SIZE             0x4000
+#define FSL_IMX31_I2C3_ADDR             0x43F84000
+#define FSL_IMX31_I2C3_SIZE             0x4000
+#define FSL_IMX31_UART1_ADDR            0x43F90000
+#define FSL_IMX31_UART1_SIZE            0x4000
+#define FSL_IMX31_UART2_ADDR            0x43F94000
+#define FSL_IMX31_UART2_SIZE            0x4000
+#define FSL_IMX31_I2C2_ADDR             0x43F98000
+#define FSL_IMX31_I2C2_SIZE             0x4000
+#define FSL_IMX31_CCM_ADDR              0x53F80000
+#define FSL_IMX31_CCM_SIZE              0x4000
+#define FSL_IMX31_GPT_ADDR              0x53F90000
+#define FSL_IMX31_GPT_SIZE              0x4000
+#define FSL_IMX31_EPIT1_ADDR            0x53F94000
+#define FSL_IMX31_EPIT1_SIZE            0x4000
+#define FSL_IMX31_EPIT2_ADDR            0x53F98000
+#define FSL_IMX31_EPIT2_SIZE            0x4000
+#define FSL_IMX31_GPIO3_ADDR            0x53FA4000
+#define FSL_IMX31_GPIO3_SIZE            0x4000
+#define FSL_IMX31_GPIO1_ADDR            0x53FCC000
+#define FSL_IMX31_GPIO1_SIZE            0x4000
+#define FSL_IMX31_GPIO2_ADDR            0x53FD0000
+#define FSL_IMX31_GPIO2_SIZE            0x4000
+#define FSL_IMX31_AVIC_ADDR             0x68000000
+#define FSL_IMX31_AVIC_SIZE             0x100
+#define FSL_IMX31_SDRAM0_ADDR           0x80000000
+#define FSL_IMX31_SDRAM0_SIZE           0x10000000
+#define FSL_IMX31_SDRAM1_ADDR           0x90000000
+#define FSL_IMX31_SDRAM1_SIZE           0x10000000
+#define FSL_IMX31_FLASH0_ADDR           0xA0000000
+#define FSL_IMX31_FLASH0_SIZE           0x8000000
+#define FSL_IMX31_FLASH1_ADDR           0xA8000000
+#define FSL_IMX31_FLASH1_SIZE           0x8000000
+#define FSL_IMX31_CS2_ADDR              0xB0000000
+#define FSL_IMX31_CS2_SIZE              0x2000000
+#define FSL_IMX31_CS3_ADDR              0xB2000000
+#define FSL_IMX31_CS3_SIZE              0x2000000
+#define FSL_IMX31_CS4_ADDR              0xB4000000
+#define FSL_IMX31_CS4_SIZE              0x2000000
+#define FSL_IMX31_CS5_ADDR              0xB6000000
+#define FSL_IMX31_CS5_SIZE              0x2000000
+#define FSL_IMX31_NAND_ADDR             0xB8000000
+#define FSL_IMX31_NAND_SIZE             0x1000
+
+#define FSL_IMX31_EPIT2_IRQ             27
+#define FSL_IMX31_EPIT1_IRQ             28
+#define FSL_IMX31_GPT_IRQ               29
+#define FSL_IMX31_UART2_IRQ             32
+#define FSL_IMX31_UART1_IRQ             45
+#define FSL_IMX31_I2C1_IRQ              10
+#define FSL_IMX31_I2C2_IRQ              4
+#define FSL_IMX31_I2C3_IRQ              3
+#define FSL_IMX31_GPIO1_IRQ             52
+#define FSL_IMX31_GPIO2_IRQ             51
+#define FSL_IMX31_GPIO3_IRQ             56
+
+#endif /* FSL_IMX31_H */
diff --git a/include/hw/arm/fsl-imx6.h b/include/hw/arm/fsl-imx6.h
new file mode 100644
index 0000000..ec6c509
--- /dev/null
+++ b/include/hw/arm/fsl-imx6.h
@@ -0,0 +1,453 @@
+/*
+ * Freescale i.MX31 SoC emulation
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef FSL_IMX6_H
+#define FSL_IMX6_H
+
+#include "hw/arm/arm.h"
+#include "hw/cpu/a9mpcore.h"
+#include "hw/misc/imx6_ccm.h"
+#include "hw/misc/imx6_src.h"
+#include "hw/char/imx_serial.h"
+#include "hw/timer/imx_gpt.h"
+#include "hw/timer/imx_epit.h"
+#include "hw/i2c/imx_i2c.h"
+#include "hw/gpio/imx_gpio.h"
+#include "hw/sd/sdhci.h"
+#include "hw/ssi/imx_spi.h"
+#include "hw/net/imx_fec.h"
+#include "exec/memory.h"
+#include "cpu.h"
+
+#define TYPE_FSL_IMX6 "fsl,imx6"
+#define FSL_IMX6(obj) OBJECT_CHECK(FslIMX6State, (obj), TYPE_FSL_IMX6)
+
+#define FSL_IMX6_NUM_CPUS 4
+#define FSL_IMX6_NUM_UARTS 5
+#define FSL_IMX6_NUM_EPITS 2
+#define FSL_IMX6_NUM_I2CS 3
+#define FSL_IMX6_NUM_GPIOS 7
+#define FSL_IMX6_NUM_ESDHCS 4
+#define FSL_IMX6_NUM_ECSPIS 5
+
+typedef struct FslIMX6State {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU         cpu[FSL_IMX6_NUM_CPUS];
+    A9MPPrivState  a9mpcore;
+    IMX6CCMState   ccm;
+    IMX6SRCState   src;
+    IMXSerialState uart[FSL_IMX6_NUM_UARTS];
+    IMXGPTState    gpt;
+    IMXEPITState   epit[FSL_IMX6_NUM_EPITS];
+    IMXI2CState    i2c[FSL_IMX6_NUM_I2CS];
+    IMXGPIOState   gpio[FSL_IMX6_NUM_GPIOS];
+    SDHCIState     esdhc[FSL_IMX6_NUM_ESDHCS];
+    IMXSPIState    spi[FSL_IMX6_NUM_ECSPIS];
+    IMXFECState    eth;
+    MemoryRegion   rom;
+    MemoryRegion   caam;
+    MemoryRegion   ocram;
+    MemoryRegion   ocram_alias;
+} FslIMX6State;
+
+
+#define FSL_IMX6_MMDC_ADDR 0x10000000
+#define FSL_IMX6_MMDC_SIZE 0xF0000000
+#define FSL_IMX6_EIM_MEM_ADDR 0x08000000
+#define FSL_IMX6_EIM_MEM_SIZE 0x8000000
+#define FSL_IMX6_IPU_2_ADDR 0x02800000
+#define FSL_IMX6_IPU_2_SIZE 0x400000
+#define FSL_IMX6_IPU_1_ADDR 0x02400000
+#define FSL_IMX6_IPU_1_SIZE 0x400000
+#define FSL_IMX6_MIPI_HSI_ADDR 0x02208000
+#define FSL_IMX6_MIPI_HSI_SIZE 0x4000
+#define FSL_IMX6_OPENVG_ADDR 0x02204000
+#define FSL_IMX6_OPENVG_SIZE 0x4000
+#define FSL_IMX6_SATA_ADDR 0x02200000
+#define FSL_IMX6_SATA_SIZE 0x4000
+#define FSL_IMX6_AIPS_2_ADDR 0x02100000
+#define FSL_IMX6_AIPS_2_SIZE 0x100000
+/* AIPS2 */
+#define FSL_IMX6_UART5_ADDR 0x021F4000
+#define FSL_IMX6_UART5_SIZE 0x4000
+#define FSL_IMX6_UART4_ADDR 0x021F0000
+#define FSL_IMX6_UART4_SIZE 0x4000
+#define FSL_IMX6_UART3_ADDR 0x021EC000
+#define FSL_IMX6_UART3_SIZE 0x4000
+#define FSL_IMX6_UART2_ADDR 0x021E8000
+#define FSL_IMX6_UART2_SIZE 0x4000
+#define FSL_IMX6_VDOA_ADDR 0x021E4000
+#define FSL_IMX6_VDOA_SIZE 0x4000
+#define FSL_IMX6_MIPI_DSI_ADDR 0x021E0000
+#define FSL_IMX6_MIPI_DSI_SIZE 0x4000
+#define FSL_IMX6_MIPI_CSI_ADDR 0x021DC000
+#define FSL_IMX6_MIPI_CSI_SIZE 0x4000
+#define FSL_IMX6_AUDMUX_ADDR 0x021D8000
+#define FSL_IMX6_AUDMUX_SIZE 0x4000
+#define FSL_IMX6_TZASC2_ADDR 0x021D4000
+#define FSL_IMX6_TZASC2_SIZE 0x4000
+#define FSL_IMX6_TZASC1_ADDR 0x021D0000
+#define FSL_IMX6_TZASC1_SIZE 0x4000
+#define FSL_IMX6_CSU_ADDR 0x021C0000
+#define FSL_IMX6_CSU_SIZE 0x4000
+#define FSL_IMX6_OCOTPCTRL_ADDR 0x021BC000
+#define FSL_IMX6_OCOTPCTRL_SIZE 0x4000
+#define FSL_IMX6_EIM_ADDR 0x021B8000
+#define FSL_IMX6_EIM_SIZE 0x4000
+#define FSL_IMX6_MMDC1_ADDR 0x021B4000
+#define FSL_IMX6_MMDC1_SIZE 0x4000
+#define FSL_IMX6_MMDC0_ADDR 0x021B0000
+#define FSL_IMX6_MMDC0_SIZE 0x4000
+#define FSL_IMX6_ROMCP_ADDR 0x021AC000
+#define FSL_IMX6_ROMCP_SIZE 0x4000
+#define FSL_IMX6_I2C3_ADDR 0x021A8000
+#define FSL_IMX6_I2C3_SIZE 0x4000
+#define FSL_IMX6_I2C2_ADDR 0x021A4000
+#define FSL_IMX6_I2C2_SIZE 0x4000
+#define FSL_IMX6_I2C1_ADDR 0x021A0000
+#define FSL_IMX6_I2C1_SIZE 0x4000
+#define FSL_IMX6_uSDHC4_ADDR 0x0219C000
+#define FSL_IMX6_uSDHC4_SIZE 0x4000
+#define FSL_IMX6_uSDHC3_ADDR 0x02198000
+#define FSL_IMX6_uSDHC3_SIZE 0x4000
+#define FSL_IMX6_uSDHC2_ADDR 0x02194000
+#define FSL_IMX6_uSDHC2_SIZE 0x4000
+#define FSL_IMX6_uSDHC1_ADDR 0x02190000
+#define FSL_IMX6_uSDHC1_SIZE 0x4000
+#define FSL_IMX6_MLB150_ADDR 0x0218C000
+#define FSL_IMX6_MLB150_SIZE 0x4000
+#define FSL_IMX6_ENET_ADDR 0x02188000
+#define FSL_IMX6_ENET_SIZE 0x4000
+#define FSL_IMX6_USBOH3_USB_ADDR 0x02184000
+#define FSL_IMX6_USBOH3_USB_SIZE 0x4000
+#define FSL_IMX6_AIPS2_CFG_ADDR 0x0217C000
+#define FSL_IMX6_AIPS2_CFG_SIZE 0x4000
+/* DAP */
+#define FSL_IMX6_PTF_CTRL_ADDR 0x02160000
+#define FSL_IMX6_PTF_CTRL_SIZE 0x1000
+#define FSL_IMX6_PTM3_ADDR 0x0215F000
+#define FSL_IMX6_PTM3_SIZE 0x1000
+#define FSL_IMX6_PTM2_ADDR 0x0215E000
+#define FSL_IMX6_PTM2_SIZE 0x1000
+#define FSL_IMX6_PTM1_ADDR 0x0215D000
+#define FSL_IMX6_PTM1_SIZE 0x1000
+#define FSL_IMX6_PTM0_ADDR 0x0215C000
+#define FSL_IMX6_PTM0_SIZE 0x1000
+#define FSL_IMX6_CTI3_ADDR 0x0215B000
+#define FSL_IMX6_CTI3_SIZE 0x1000
+#define FSL_IMX6_CTI2_ADDR 0x0215A000
+#define FSL_IMX6_CTI2_SIZE 0x1000
+#define FSL_IMX6_CTI1_ADDR 0x02159000
+#define FSL_IMX6_CTI1_SIZE 0x1000
+#define FSL_IMX6_CTI0_ADDR 0x02158000
+#define FSL_IMX6_CTI0_SIZE 0x1000
+#define FSL_IMX6_CPU3_PMU_ADDR 0x02157000
+#define FSL_IMX6_CPU3_PMU_SIZE 0x1000
+#define FSL_IMX6_CPU3_DEBUG_IF_ADDR 0x02156000
+#define FSL_IMX6_CPU3_DEBUG_IF_SIZE 0x1000
+#define FSL_IMX6_CPU2_PMU_ADDR 0x02155000
+#define FSL_IMX6_CPU2_PMU_SIZE 0x1000
+#define FSL_IMX6_CPU2_DEBUG_IF_ADDR 0x02154000
+#define FSL_IMX6_CPU2_DEBUG_IF_SIZE 0x1000
+#define FSL_IMX6_CPU1_PMU_ADDR 0x02153000
+#define FSL_IMX6_CPU1_PMU_SIZE 0x1000
+#define FSL_IMX6_CPU1_DEBUG_IF_ADDR 0x02152000
+#define FSL_IMX6_CPU1_DEBUG_IF_SIZE 0x1000
+#define FSL_IMX6_CPU0_PMU_ADDR 0x02151000
+#define FSL_IMX6_CPU0_PMU_SIZE 0x1000
+#define FSL_IMX6_CPU0_DEBUG_IF_ADDR 0x02150000
+#define FSL_IMX6_CPU0_DEBUG_IF_SIZE 0x1000
+#define FSL_IMX6_CA9_INTEG_ADDR 0x0214F000
+#define FSL_IMX6_CA9_INTEG_SIZE 0x1000
+#define FSL_IMX6_FUNNEL_ADDR 0x02144000
+#define FSL_IMX6_FUNNEL_SIZE 0x1000
+#define FSL_IMX6_TPIU_ADDR 0x02143000
+#define FSL_IMX6_TPIU_SIZE 0x1000
+#define FSL_IMX6_EXT_CTI_ADDR 0x02142000
+#define FSL_IMX6_EXT_CTI_SIZE 0x1000
+#define FSL_IMX6_ETB_ADDR 0x02141000
+#define FSL_IMX6_ETB_SIZE 0x1000
+#define FSL_IMX6_DAP_ROM_TABLE_ADDR 0x02140000
+#define FSL_IMX6_DAP_ROM_TABLE_SIZE 0x1000
+/* DAP end */
+#define FSL_IMX6_CAAM_ADDR 0x02100000
+#define FSL_IMX6_CAAM_SIZE 0x10000
+/* AIPS2 end */
+#define FSL_IMX6_AIPS_1_ADDR 0x02000000
+#define FSL_IMX6_AIPS_1_SIZE 0x100000
+/* AIPS1 */
+#define FSL_IMX6_SDMA_ADDR 0x020EC000
+#define FSL_IMX6_SDMA_SIZE 0x4000
+#define FSL_IMX6_DCIC2_ADDR 0x020E8000
+#define FSL_IMX6_DCIC2_SIZE 0x4000
+#define FSL_IMX6_DCIC1_ADDR 0x020E4000
+#define FSL_IMX6_DCIC1_SIZE 0x4000
+#define FSL_IMX6_IOMUXC_ADDR 0x020E0000
+#define FSL_IMX6_IOMUXC_SIZE 0x4000
+#define FSL_IMX6_PGCARM_ADDR 0x020DCA00
+#define FSL_IMX6_PGCARM_SIZE 0x20
+#define FSL_IMX6_PGCPU_ADDR 0x020DC260
+#define FSL_IMX6_PGCPU_SIZE 0x20
+#define FSL_IMX6_GPC_ADDR 0x020DC000
+#define FSL_IMX6_GPC_SIZE 0x4000
+#define FSL_IMX6_SRC_ADDR 0x020D8000
+#define FSL_IMX6_SRC_SIZE 0x4000
+#define FSL_IMX6_EPIT2_ADDR 0x020D4000
+#define FSL_IMX6_EPIT2_SIZE 0x4000
+#define FSL_IMX6_EPIT1_ADDR 0x020D0000
+#define FSL_IMX6_EPIT1_SIZE 0x4000
+#define FSL_IMX6_SNVSHP_ADDR 0x020CC000
+#define FSL_IMX6_SNVSHP_SIZE 0x4000
+#define FSL_IMX6_USBPHY2_ADDR 0x020CA000
+#define FSL_IMX6_USBPHY2_SIZE 0x1000
+#define FSL_IMX6_USBPHY1_ADDR 0x020C9000
+#define FSL_IMX6_USBPHY1_SIZE 0x1000
+#define FSL_IMX6_ANALOG_ADDR 0x020C8000
+#define FSL_IMX6_ANALOG_SIZE 0x1000
+#define FSL_IMX6_CCM_ADDR 0x020C4000
+#define FSL_IMX6_CCM_SIZE 0x4000
+#define FSL_IMX6_WDOG2_ADDR 0x020C0000
+#define FSL_IMX6_WDOG2_SIZE 0x4000
+#define FSL_IMX6_WDOG1_ADDR 0x020BC000
+#define FSL_IMX6_WDOG1_SIZE 0x4000
+#define FSL_IMX6_KPP_ADDR 0x020B8000
+#define FSL_IMX6_KPP_SIZE 0x4000
+#define FSL_IMX6_GPIO7_ADDR 0x020B4000
+#define FSL_IMX6_GPIO7_SIZE 0x4000
+#define FSL_IMX6_GPIO6_ADDR 0x020B0000
+#define FSL_IMX6_GPIO6_SIZE 0x4000
+#define FSL_IMX6_GPIO5_ADDR 0x020AC000
+#define FSL_IMX6_GPIO5_SIZE 0x4000
+#define FSL_IMX6_GPIO4_ADDR 0x020A8000
+#define FSL_IMX6_GPIO4_SIZE 0x4000
+#define FSL_IMX6_GPIO3_ADDR 0x020A4000
+#define FSL_IMX6_GPIO3_SIZE 0x4000
+#define FSL_IMX6_GPIO2_ADDR 0x020A0000
+#define FSL_IMX6_GPIO2_SIZE 0x4000
+#define FSL_IMX6_GPIO1_ADDR 0x0209C000
+#define FSL_IMX6_GPIO1_SIZE 0x4000
+#define FSL_IMX6_GPT_ADDR 0x02098000
+#define FSL_IMX6_GPT_SIZE 0x4000
+#define FSL_IMX6_CAN2_ADDR 0x02094000
+#define FSL_IMX6_CAN2_SIZE 0x4000
+#define FSL_IMX6_CAN1_ADDR 0x02090000
+#define FSL_IMX6_CAN1_SIZE 0x4000
+#define FSL_IMX6_PWM4_ADDR 0x0208C000
+#define FSL_IMX6_PWM4_SIZE 0x4000
+#define FSL_IMX6_PWM3_ADDR 0x02088000
+#define FSL_IMX6_PWM3_SIZE 0x4000
+#define FSL_IMX6_PWM2_ADDR 0x02084000
+#define FSL_IMX6_PWM2_SIZE 0x4000
+#define FSL_IMX6_PWM1_ADDR 0x02080000
+#define FSL_IMX6_PWM1_SIZE 0x4000
+#define FSL_IMX6_AIPS1_CFG_ADDR 0x0207C000
+#define FSL_IMX6_AIPS1_CFG_SIZE 0x4000
+#define FSL_IMX6_VPU_ADDR 0x02040000
+#define FSL_IMX6_VPU_SIZE 0x3C000
+#define FSL_IMX6_AIPS1_SPBA_ADDR 0x0203C000
+#define FSL_IMX6_AIPS1_SPBA_SIZE 0x4000
+#define FSL_IMX6_ASRC_ADDR 0x02034000
+#define FSL_IMX6_ASRC_SIZE 0x4000
+#define FSL_IMX6_SSI3_ADDR 0x02030000
+#define FSL_IMX6_SSI3_SIZE 0x4000
+#define FSL_IMX6_SSI2_ADDR 0x0202C000
+#define FSL_IMX6_SSI2_SIZE 0x4000
+#define FSL_IMX6_SSI1_ADDR 0x02028000
+#define FSL_IMX6_SSI1_SIZE 0x4000
+#define FSL_IMX6_ESAI_ADDR 0x02024000
+#define FSL_IMX6_ESAI_SIZE 0x4000
+#define FSL_IMX6_UART1_ADDR 0x02020000
+#define FSL_IMX6_UART1_SIZE 0x4000
+#define FSL_IMX6_eCSPI5_ADDR 0x02018000
+#define FSL_IMX6_eCSPI5_SIZE 0x4000
+#define FSL_IMX6_eCSPI4_ADDR 0x02014000
+#define FSL_IMX6_eCSPI4_SIZE 0x4000
+#define FSL_IMX6_eCSPI3_ADDR 0x02010000
+#define FSL_IMX6_eCSPI3_SIZE 0x4000
+#define FSL_IMX6_eCSPI2_ADDR 0x0200C000
+#define FSL_IMX6_eCSPI2_SIZE 0x4000
+#define FSL_IMX6_eCSPI1_ADDR 0x02008000
+#define FSL_IMX6_eCSPI1_SIZE 0x4000
+#define FSL_IMX6_SPDIF_ADDR 0x02004000
+#define FSL_IMX6_SPDIF_SIZE 0x4000
+/* AIPS1 end */
+#define FSL_IMX6_PCIe_REG_ADDR 0x01FFC000
+#define FSL_IMX6_PCIe_REG_SIZE 0x4000
+#define FSL_IMX6_PCIe_ADDR 0x01000000
+#define FSL_IMX6_PCIe_SIZE 0xFFC000
+#define FSL_IMX6_GPV_1_PL301_CFG_ADDR 0x00C00000
+#define FSL_IMX6_GPV_1_PL301_CFG_SIZE 0x100000
+#define FSL_IMX6_GPV_0_PL301_CFG_ADDR 0x00B00000
+#define FSL_IMX6_GPV_0_PL301_CFG_SIZE 0x100000
+#define FSL_IMX6_PL310_ADDR 0x00A02000
+#define FSL_IMX6_PL310_SIZE 0x1000
+#define FSL_IMX6_A9MPCORE_ADDR 0x00A00000
+#define FSL_IMX6_A9MPCORE_SIZE 0x2000
+#define FSL_IMX6_OCRAM_ALIAS_ADDR 0x00940000
+#define FSL_IMX6_OCRAM_ALIAS_SIZE 0xC0000
+#define FSL_IMX6_OCRAM_ADDR 0x00900000
+#define FSL_IMX6_OCRAM_SIZE 0x40000
+#define FSL_IMX6_GPV_4_PL301_CFG_ADDR 0x00800000
+#define FSL_IMX6_GPV_4_PL301_CFG_SIZE 0x100000
+#define FSL_IMX6_GPV_3_PL301_CFG_ADDR 0x00300000
+#define FSL_IMX6_GPV_3_PL301_CFG_SIZE 0x100000
+#define FSL_IMX6_GPV_2_PL301_CFG_ADDR 0x00200000
+#define FSL_IMX6_GPV_2_PL301_CFG_SIZE 0x100000
+#define FSL_IMX6_DTCP_ADDR 0x00138000
+#define FSL_IMX6_DTCP_SIZE 0x4000
+#define FSL_IMX6_GPU_2D_ADDR 0x00134000
+#define FSL_IMX6_GPU_2D_SIZE 0x4000
+#define FSL_IMX6_GPU_3D_ADDR 0x00130000
+#define FSL_IMX6_GPU_3D_SIZE 0x4000
+#define FSL_IMX6_HDMI_ADDR 0x00120000
+#define FSL_IMX6_HDMI_SIZE 0x9000
+#define FSL_IMX6_BCH_ADDR 0x00114000
+#define FSL_IMX6_BCH_SIZE 0x4000
+#define FSL_IMX6_GPMI_ADDR 0x00112000
+#define FSL_IMX6_GPMI_SIZE 0x2000
+#define FSL_IMX6_APBH_BRIDGE_DMA_ADDR 0x00110000
+#define FSL_IMX6_APBH_BRIDGE_DMA_SIZE 0x2000
+#define FSL_IMX6_CAAM_MEM_ADDR 0x00100000
+#define FSL_IMX6_CAAM_MEM_SIZE 0x4000
+#define FSL_IMX6_ROM_ADDR 0x00000000
+#define FSL_IMX6_ROM_SIZE 0x18000
+
+#define FSL_IMX6_IOMUXC_IRQ 0
+#define FSL_IMX6_DAP_IRQ 1
+#define FSL_IMX6_SDMA_IRQ 2
+#define FSL_IMX6_VPU_JPEG_IRQ 3
+#define FSL_IMX6_SNVS_PMIC_IRQ 4
+#define FSL_IMX6_IPU1_ERROR_IRQ 5
+#define FSL_IMX6_IPU1_SYNC_IRQ 6
+#define FSL_IMX6_IPU2_ERROR_IRQ 7
+#define FSL_IMX6_IPU2_SYNC_IRQ 8
+#define FSL_IMX6_GPU3D_IRQ 9
+#define FSL_IMX6_R2D_IRQ 10
+#define FSL_IMX6_V2D_IRQ 11
+#define FSL_IMX6_VPU_IRQ 12
+#define FSL_IMX6_APBH_BRIDGE_DMA_IRQ 13
+#define FSL_IMX6_EIM_IRQ 14
+#define FSL_IMX6_BCH_IRQ 15
+#define FSL_IMX6_GPMI_IRQ 16
+#define FSL_IMX6_DTCP_IRQ 17
+#define FSL_IMX6_VDOA_IRQ 18
+#define FSL_IMX6_SNVS_CONS_IRQ 19
+#define FSL_IMX6_SNVS_SEC_IRQ 20
+#define FSL_IMX6_CSU_IRQ 21
+#define FSL_IMX6_uSDHC1_IRQ 22
+#define FSL_IMX6_uSDHC2_IRQ 23
+#define FSL_IMX6_uSDHC3_IRQ 24
+#define FSL_IMX6_uSDHC4_IRQ 25
+#define FSL_IMX6_UART1_IRQ 26
+#define FSL_IMX6_UART2_IRQ 27
+#define FSL_IMX6_UART3_IRQ 28
+#define FSL_IMX6_UART4_IRQ 29
+#define FSL_IMX6_UART5_IRQ 30
+#define FSL_IMX6_ECSPI1_IRQ 31
+#define FSL_IMX6_ECSPI2_IRQ 32
+#define FSL_IMX6_ECSPI3_IRQ 33
+#define FSL_IMX6_ECSPI4_IRQ 34
+#define FSL_IMX6_ECSPI5_IRQ 35
+#define FSL_IMX6_I2C1_IRQ 36
+#define FSL_IMX6_I2C2_IRQ 37
+#define FSL_IMX6_I2C3_IRQ 38
+#define FSL_IMX6_SATA_IRQ 39
+#define FSL_IMX6_USB_HOST1_IRQ 40
+#define FSL_IMX6_USB_HOST2_IRQ 41
+#define FSL_IMX6_USB_HOST3_IRQ 42
+#define FSL_IMX6_USB_OTG_IRQ 43
+#define FSL_IMX6_USB_PHY_UTMI0_IRQ 44
+#define FSL_IMX6_USB_PHY_UTMI1_IRQ 45
+#define FSL_IMX6_SSI1_IRQ 46
+#define FSL_IMX6_SSI2_IRQ 47
+#define FSL_IMX6_SSI3_IRQ 48
+#define FSL_IMX6_TEMP_IRQ 49
+#define FSL_IMX6_ASRC_IRQ 50
+#define FSL_IMX6_ESAI_IRQ 51
+#define FSL_IMX6_SPDIF_IRQ 52
+#define FSL_IMX6_MLB150_IRQ 53
+#define FSL_IMX6_PMU1_IRQ 54
+#define FSL_IMX6_GPT_IRQ 55
+#define FSL_IMX6_EPIT1_IRQ 56
+#define FSL_IMX6_EPIT2_IRQ 57
+#define FSL_IMX6_GPIO1_INT7_IRQ 58
+#define FSL_IMX6_GPIO1_INT6_IRQ 59
+#define FSL_IMX6_GPIO1_INT5_IRQ 60
+#define FSL_IMX6_GPIO1_INT4_IRQ 61
+#define FSL_IMX6_GPIO1_INT3_IRQ 62
+#define FSL_IMX6_GPIO1_INT2_IRQ 63
+#define FSL_IMX6_GPIO1_INT1_IRQ 64
+#define FSL_IMX6_GPIO1_INT0_IRQ 65
+#define FSL_IMX6_GPIO1_LOW_IRQ 66
+#define FSL_IMX6_GPIO1_HIGH_IRQ 67
+#define FSL_IMX6_GPIO2_LOW_IRQ 68
+#define FSL_IMX6_GPIO2_HIGH_IRQ 69
+#define FSL_IMX6_GPIO3_LOW_IRQ 70
+#define FSL_IMX6_GPIO3_HIGH_IRQ 71
+#define FSL_IMX6_GPIO4_LOW_IRQ 72
+#define FSL_IMX6_GPIO4_HIGH_IRQ 73
+#define FSL_IMX6_GPIO5_LOW_IRQ 74
+#define FSL_IMX6_GPIO5_HIGH_IRQ 75
+#define FSL_IMX6_GPIO6_LOW_IRQ 76
+#define FSL_IMX6_GPIO6_HIGH_IRQ 77
+#define FSL_IMX6_GPIO7_LOW_IRQ 78
+#define FSL_IMX6_GPIO7_HIGH_IRQ 79
+#define FSL_IMX6_WDOG1_IRQ 80
+#define FSL_IMX6_WDOG2_IRQ 81
+#define FSL_IMX6_KPP_IRQ 82
+#define FSL_IMX6_PWM1_IRQ 83
+#define FSL_IMX6_PWM2_IRQ 84
+#define FSL_IMX6_PWM3_IRQ 85
+#define FSL_IMX6_PWM4_IRQ 86
+#define FSL_IMX6_CCM1_IRQ 87
+#define FSL_IMX6_CCM2_IRQ 88
+#define FSL_IMX6_GPC_IRQ 89
+#define FSL_IMX6_SRC_IRQ 91
+#define FSL_IMX6_CPU_L2_IRQ 92
+#define FSL_IMX6_CPU_PARITY_IRQ 93
+#define FSL_IMX6_CPU_PERF_IRQ 94
+#define FSL_IMX6_CPU_CTI_IRQ 95
+#define FSL_IMX6_SRC_COMB_IRQ 96
+#define FSL_IMX6_MIPI_CSI1_IRQ 100
+#define FSL_IMX6_MIPI_CSI2_IRQ 101
+#define FSL_IMX6_MIPI_DSI_IRQ 102
+#define FSL_IMX6_MIPI_HSI_IRQ 103
+#define FSL_IMX6_SJC_IRQ 104
+#define FSL_IMX6_CAAM0_IRQ 105
+#define FSL_IMX6_CAAM1_IRQ 106
+#define FSL_IMX6_ASC1_IRQ 108
+#define FSL_IMX6_ASC2_IRQ 109
+#define FSL_IMX6_FLEXCAN1_IRQ 110
+#define FSL_IMX6_FLEXCAN2_IRQ 111
+#define FSL_IMX6_HDMI_MASTER_IRQ 115
+#define FSL_IMX6_HDMI_CEC_IRQ 116
+#define FSL_IMX6_MLB150_LOW_IRQ 117
+#define FSL_IMX6_ENET_MAC_1588_IRQ 118
+#define FSL_IMX6_ENET_MAC_IRQ 119
+#define FSL_IMX6_PCIE1_IRQ 120
+#define FSL_IMX6_PCIE2_IRQ 121
+#define FSL_IMX6_PCIE3_IRQ 122
+#define FSL_IMX6_PCIE4_IRQ 123
+#define FSL_IMX6_DCIC1_IRQ 124
+#define FSL_IMX6_DCIC2_IRQ 125
+#define FSL_IMX6_MLB150_HIGH_IRQ 126
+#define FSL_IMX6_PMU2_IRQ 127
+#define FSL_IMX6_MAX_IRQ 128
+
+#endif /* FSL_IMX6_H */
diff --git a/include/hw/arm/imx.h b/include/hw/arm/imx.h
deleted file mode 100644
index ea9e093..0000000
--- a/include/hw/arm/imx.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * i.MX31 emulation
- *
- * Copyright (C) 2012 Peter Chubb
- * NICTA
- *
- * This code is released under the GPL, version 2.0 or later
- * See the file `../COPYING' for details.
- */
-
-#ifndef IMX_H
-#define IMX_H
-
-void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq);
-
-typedef enum  {
-    NOCLK,
-    MCU,
-    HSP,
-    IPG,
-    CLK_32k
-} IMXClk;
-
-uint32_t imx_clock_frequency(DeviceState *s, IMXClk clock);
-
-void imx_timerp_create(const hwaddr addr,
-                      qemu_irq irq,
-                      DeviceState *ccm);
-void imx_timerg_create(const hwaddr addr,
-                      qemu_irq irq,
-                      DeviceState *ccm);
-
-
-#endif /* IMX_H */
diff --git a/include/hw/arm/linux-boot-if.h b/include/hw/arm/linux-boot-if.h
new file mode 100644
index 0000000..aba4479
--- /dev/null
+++ b/include/hw/arm/linux-boot-if.h
@@ -0,0 +1,43 @@
+/*
+ * hw/arm/linux-boot-if.h : interface for devices which need to behave
+ * specially for direct boot of an ARM Linux kernel
+ */
+
+#ifndef HW_ARM_LINUX_BOOT_IF_H
+#define HW_ARM_LINUX_BOOT_IF_H
+
+#include "qom/object.h"
+
+#define TYPE_ARM_LINUX_BOOT_IF "arm-linux-boot-if"
+#define ARM_LINUX_BOOT_IF_CLASS(klass) \
+    OBJECT_CLASS_CHECK(ARMLinuxBootIfClass, (klass), TYPE_ARM_LINUX_BOOT_IF)
+#define ARM_LINUX_BOOT_IF_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(ARMLinuxBootIfClass, (obj), TYPE_ARM_LINUX_BOOT_IF)
+#define ARM_LINUX_BOOT_IF(obj) \
+    INTERFACE_CHECK(ARMLinuxBootIf, (obj), TYPE_ARM_LINUX_BOOT_IF)
+
+typedef struct ARMLinuxBootIf {
+    /*< private >*/
+    Object parent_obj;
+} ARMLinuxBootIf;
+
+typedef struct ARMLinuxBootIfClass {
+    /*< private >*/
+    InterfaceClass parent_class;
+
+    /*< public >*/
+    /** arm_linux_init: configure the device for a direct boot
+     * of an ARM Linux kernel (so that device reset puts it into
+     * the state the kernel expects after firmware initialization,
+     * rather than the true hardware reset state). This callback is
+     * called once after machine construction is complete (before the
+     * first system reset).
+     *
+     * @obj: the object implementing this interface
+     * @secure_boot: true if we are booting Secure, false for NonSecure
+     * (or for a CPU which doesn't support TrustZone)
+     */
+    void (*arm_linux_init)(ARMLinuxBootIf *obj, bool secure_boot);
+} ARMLinuxBootIfClass;
+
+#endif
diff --git a/include/hw/arm/omap.h b/include/hw/arm/omap.h
index 0ad5fb8..f026c8d 100644
--- a/include/hw/arm/omap.h
+++ b/include/hw/arm/omap.h
@@ -20,6 +20,7 @@
 #include "exec/memory.h"
 # define hw_omap_h		"omap.h"
 #include "hw/irq.h"
+#include "target-arm/cpu-qom.h"
 
 # define OMAP_EMIFS_BASE	0x00000000
 # define OMAP2_Q0_BASE		0x00000000
diff --git a/include/hw/arm/pxa.h b/include/hw/arm/pxa.h
index 259b852..dd1a48b 100644
--- a/include/hw/arm/pxa.h
+++ b/include/hw/arm/pxa.h
@@ -6,10 +6,12 @@
  *
  * This code is licensed under the GNU GPL v2.
  */
+
 #ifndef PXA_H
-# define PXA_H			"pxa.h"
+#define PXA_H
 
 #include "exec/memory.h"
+#include "target-arm/cpu-qom.h"
 
 /* Interrupt numbers */
 # define PXA2XX_PIC_SSP3	0
@@ -188,4 +190,4 @@
                          const char *revision);
 PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size);
 
-#endif	/* PXA_H */
+#endif /* PXA_H */
diff --git a/include/hw/arm/raspi_platform.h b/include/hw/arm/raspi_platform.h
new file mode 100644
index 0000000..6467e88
--- /dev/null
+++ b/include/hw/arm/raspi_platform.h
@@ -0,0 +1,128 @@
+/*
+ * bcm2708 aka bcm2835/2836 aka Raspberry Pi/Pi2 SoC platform defines
+ *
+ * These definitions are derived from those in Raspbian Linux at
+ * arch/arm/mach-{bcm2708,bcm2709}/include/mach/platform.h
+ * where they carry the following notice:
+ *
+ * Copyright (C) 2010 Broadcom
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+#define MCORE_OFFSET            0x0000   /* Fake frame buffer device
+                                          * (the multicore sync block) */
+#define IC0_OFFSET              0x2000
+#define ST_OFFSET               0x3000   /* System Timer */
+#define MPHI_OFFSET             0x6000   /* Message-based Parallel Host Intf. */
+#define DMA_OFFSET              0x7000   /* DMA controller, channels 0-14 */
+#define ARM_OFFSET              0xB000   /* BCM2708 ARM control block */
+#define ARMCTRL_OFFSET          (ARM_OFFSET + 0x000)
+#define ARMCTRL_IC_OFFSET       (ARM_OFFSET + 0x200) /* Interrupt controller */
+#define ARMCTRL_TIMER0_1_OFFSET (ARM_OFFSET + 0x400) /* Timer 0 and 1 */
+#define ARMCTRL_0_SBM_OFFSET    (ARM_OFFSET + 0x800) /* User 0 (ARM) Semaphores
+                                                      * Doorbells & Mailboxes */
+#define PM_OFFSET               0x100000 /* Power Management, Reset controller
+                                          * and Watchdog registers */
+#define PCM_CLOCK_OFFSET        0x101098
+#define RNG_OFFSET              0x104000
+#define GPIO_OFFSET             0x200000
+#define UART0_OFFSET            0x201000
+#define MMCI0_OFFSET            0x202000
+#define I2S_OFFSET              0x203000
+#define SPI0_OFFSET             0x204000
+#define BSC0_OFFSET             0x205000 /* BSC0 I2C/TWI */
+#define UART1_OFFSET            0x215000
+#define EMMC_OFFSET             0x300000
+#define SMI_OFFSET              0x600000
+#define BSC1_OFFSET             0x804000 /* BSC1 I2C/TWI */
+#define USB_OFFSET              0x980000 /* DTC_OTG USB controller */
+#define DMA15_OFFSET            0xE05000 /* DMA controller, channel 15 */
+
+/* GPU interrupts */
+#define INTERRUPT_TIMER0               0
+#define INTERRUPT_TIMER1               1
+#define INTERRUPT_TIMER2               2
+#define INTERRUPT_TIMER3               3
+#define INTERRUPT_CODEC0               4
+#define INTERRUPT_CODEC1               5
+#define INTERRUPT_CODEC2               6
+#define INTERRUPT_JPEG                 7
+#define INTERRUPT_ISP                  8
+#define INTERRUPT_USB                  9
+#define INTERRUPT_3D                   10
+#define INTERRUPT_TRANSPOSER           11
+#define INTERRUPT_MULTICORESYNC0       12
+#define INTERRUPT_MULTICORESYNC1       13
+#define INTERRUPT_MULTICORESYNC2       14
+#define INTERRUPT_MULTICORESYNC3       15
+#define INTERRUPT_DMA0                 16
+#define INTERRUPT_DMA1                 17
+#define INTERRUPT_DMA2                 18
+#define INTERRUPT_DMA3                 19
+#define INTERRUPT_DMA4                 20
+#define INTERRUPT_DMA5                 21
+#define INTERRUPT_DMA6                 22
+#define INTERRUPT_DMA7                 23
+#define INTERRUPT_DMA8                 24
+#define INTERRUPT_DMA9                 25
+#define INTERRUPT_DMA10                26
+#define INTERRUPT_DMA11                27
+#define INTERRUPT_DMA12                28
+#define INTERRUPT_AUX                  29
+#define INTERRUPT_ARM                  30
+#define INTERRUPT_VPUDMA               31
+#define INTERRUPT_HOSTPORT             32
+#define INTERRUPT_VIDEOSCALER          33
+#define INTERRUPT_CCP2TX               34
+#define INTERRUPT_SDC                  35
+#define INTERRUPT_DSI0                 36
+#define INTERRUPT_AVE                  37
+#define INTERRUPT_CAM0                 38
+#define INTERRUPT_CAM1                 39
+#define INTERRUPT_HDMI0                40
+#define INTERRUPT_HDMI1                41
+#define INTERRUPT_PIXELVALVE1          42
+#define INTERRUPT_I2CSPISLV            43
+#define INTERRUPT_DSI1                 44
+#define INTERRUPT_PWA0                 45
+#define INTERRUPT_PWA1                 46
+#define INTERRUPT_CPR                  47
+#define INTERRUPT_SMI                  48
+#define INTERRUPT_GPIO0                49
+#define INTERRUPT_GPIO1                50
+#define INTERRUPT_GPIO2                51
+#define INTERRUPT_GPIO3                52
+#define INTERRUPT_I2C                  53
+#define INTERRUPT_SPI                  54
+#define INTERRUPT_I2SPCM               55
+#define INTERRUPT_SDIO                 56
+#define INTERRUPT_UART                 57
+#define INTERRUPT_SLIMBUS              58
+#define INTERRUPT_VEC                  59
+#define INTERRUPT_CPG                  60
+#define INTERRUPT_RNG                  61
+#define INTERRUPT_ARASANSDIO           62
+#define INTERRUPT_AVSPMON              63
+
+/* ARM CPU IRQs use a private number space */
+#define INTERRUPT_ARM_TIMER            0
+#define INTERRUPT_ARM_MAILBOX          1
+#define INTERRUPT_ARM_DOORBELL_0       2
+#define INTERRUPT_ARM_DOORBELL_1       3
+#define INTERRUPT_VPU0_HALTED          4
+#define INTERRUPT_VPU1_HALTED          5
+#define INTERRUPT_ILLEGAL_TYPE0        6
+#define INTERRUPT_ILLEGAL_TYPE1        7
diff --git a/include/hw/arm/soc_dma.h b/include/hw/arm/soc_dma.h
index 7379731..fae3229 100644
--- a/include/hw/arm/soc_dma.h
+++ b/include/hw/arm/soc_dma.h
@@ -19,8 +19,7 @@
  */
 
 #ifndef HW_SOC_DMA_H
-#define HW_SOC_DMA_H 1
-
+#define HW_SOC_DMA_H
 
 #include "exec/memory.h"
 #include "hw/irq.h"
diff --git a/include/hw/arm/stm32f205_soc.h b/include/hw/arm/stm32f205_soc.h
new file mode 100644
index 0000000..779b5da
--- /dev/null
+++ b/include/hw/arm/stm32f205_soc.h
@@ -0,0 +1,57 @@
+/*
+ * STM32F205 SoC
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef HW_ARM_STM32F205_SOC_H
+#define HW_ARM_STM32F205_SOC_H
+
+#include "hw/misc/stm32f2xx_syscfg.h"
+#include "hw/timer/stm32f2xx_timer.h"
+#include "hw/char/stm32f2xx_usart.h"
+
+#define TYPE_STM32F205_SOC "stm32f205-soc"
+#define STM32F205_SOC(obj) \
+    OBJECT_CHECK(STM32F205State, (obj), TYPE_STM32F205_SOC)
+
+#define STM_NUM_USARTS 6
+#define STM_NUM_TIMERS 4
+
+#define FLASH_BASE_ADDRESS 0x08000000
+#define FLASH_SIZE (1024 * 1024)
+#define SRAM_BASE_ADDRESS 0x20000000
+#define SRAM_SIZE (128 * 1024)
+
+typedef struct STM32F205State {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    char *kernel_filename;
+    char *cpu_model;
+
+    STM32F2XXSyscfgState syscfg;
+    STM32F2XXUsartState usart[STM_NUM_USARTS];
+    STM32F2XXTimerState timer[STM_NUM_TIMERS];
+} STM32F205State;
+
+#endif
diff --git a/include/hw/arm/sysbus-fdt.h b/include/hw/arm/sysbus-fdt.h
new file mode 100644
index 0000000..e15bb81
--- /dev/null
+++ b/include/hw/arm/sysbus-fdt.h
@@ -0,0 +1,60 @@
+/*
+ * Dynamic sysbus device tree node generation API
+ *
+ * Copyright Linaro Limited, 2014
+ *
+ * Authors:
+ *  Alex Graf <agraf@suse.de>
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef HW_ARM_SYSBUS_FDT_H
+#define HW_ARM_SYSBUS_FDT_H
+
+#include "hw/arm/arm.h"
+#include "qemu-common.h"
+#include "hw/sysbus.h"
+
+/*
+ * struct that contains dimensioning parameters of the platform bus
+ */
+typedef struct {
+    hwaddr platform_bus_base; /* start address of the bus */
+    hwaddr platform_bus_size; /* size of the bus */
+    int platform_bus_first_irq; /* first hwirq assigned to the bus */
+    int platform_bus_num_irqs; /* number of hwirq assigned to the bus */
+} ARMPlatformBusSystemParams;
+
+/*
+ * struct that contains all relevant info to build the fdt nodes of
+ * platform bus and attached dynamic sysbus devices
+ * in the future might be augmented with additional info
+ * such as PHY, CLK handles ...
+ */
+typedef struct {
+    const ARMPlatformBusSystemParams *system_params;
+    struct arm_boot_info *binfo;
+    const char *intc; /* parent interrupt controller name */
+} ARMPlatformBusFDTParams;
+
+/**
+ * arm_register_platform_bus_fdt_creator - register a machine init done
+ * notifier that creates the device tree nodes of the platform bus and
+ * associated dynamic sysbus devices
+ */
+void arm_register_platform_bus_fdt_creator(ARMPlatformBusFDTParams *fdt_params);
+
+#endif
diff --git a/include/hw/arm/virt-acpi-build.h b/include/hw/arm/virt-acpi-build.h
new file mode 100644
index 0000000..e43330a
--- /dev/null
+++ b/include/hw/arm/virt-acpi-build.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright (c) 2015 HUAWEI TECHNOLOGIES CO.,LTD.
+ *
+ * Author: Shannon Zhao <zhaoshenglong@huawei.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef QEMU_VIRT_ACPI_BUILD_H
+#define QEMU_VIRT_ACPI_BUILD_H
+
+#include "qemu-common.h"
+#include "hw/arm/virt.h"
+#include "qemu/notify.h"
+
+#define ACPI_GICC_ENABLED 1
+
+typedef struct VirtGuestInfo {
+    int smp_cpus;
+    FWCfgState *fw_cfg;
+    const MemMapEntry *memmap;
+    const int *irqmap;
+    bool use_highmem;
+    int gic_version;
+} VirtGuestInfo;
+
+
+typedef struct VirtGuestInfoState {
+    VirtGuestInfo info;
+    Notifier machine_done;
+} VirtGuestInfoState;
+
+void virt_acpi_setup(VirtGuestInfo *guest_info);
+
+#endif
diff --git a/include/hw/arm/virt.h b/include/hw/arm/virt.h
new file mode 100644
index 0000000..9650193
--- /dev/null
+++ b/include/hw/arm/virt.h
@@ -0,0 +1,78 @@
+/*
+ *
+ * Copyright (c) 2015 Linaro Limited
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * Emulate a virtual board which works by passing Linux all the information
+ * it needs about what devices are present via the device tree.
+ * There are some restrictions about what we can do here:
+ *  + we can only present devices whose Linux drivers will work based
+ *    purely on the device tree with no platform data at all
+ *  + we want to present a very stripped-down minimalist platform,
+ *    both because this reduces the security attack surface from the guest
+ *    and also because it reduces our exposure to being broken when
+ *    the kernel updates its device tree bindings and requires further
+ *    information in a device binding that we aren't providing.
+ * This is essentially the same approach kvmtool uses.
+ */
+
+#ifndef QEMU_ARM_VIRT_H
+#define QEMU_ARM_VIRT_H
+
+#include "qemu-common.h"
+#include "exec/hwaddr.h"
+
+#define NUM_GICV2M_SPIS       64
+#define NUM_VIRTIO_TRANSPORTS 32
+
+#define ARCH_TIMER_VIRT_IRQ   11
+#define ARCH_TIMER_S_EL1_IRQ  13
+#define ARCH_TIMER_NS_EL1_IRQ 14
+#define ARCH_TIMER_NS_EL2_IRQ 10
+
+#define VIRTUAL_PMU_IRQ 7
+
+#define PPI(irq) ((irq) + 16)
+
+enum {
+    VIRT_FLASH,
+    VIRT_MEM,
+    VIRT_CPUPERIPHS,
+    VIRT_GIC_DIST,
+    VIRT_GIC_CPU,
+    VIRT_GIC_V2M,
+    VIRT_GIC_ITS,
+    VIRT_GIC_REDIST,
+    VIRT_UART,
+    VIRT_MMIO,
+    VIRT_RTC,
+    VIRT_FW_CFG,
+    VIRT_PCIE,
+    VIRT_PCIE_MMIO,
+    VIRT_PCIE_PIO,
+    VIRT_PCIE_ECAM,
+    VIRT_PLATFORM_BUS,
+    VIRT_PCIE_MMIO_HIGH,
+    VIRT_GPIO,
+    VIRT_SECURE_UART,
+    VIRT_SECURE_MEM,
+};
+
+typedef struct MemMapEntry {
+    hwaddr base;
+    hwaddr size;
+} MemMapEntry;
+
+
+#endif
diff --git a/include/hw/arm/xlnx-zynqmp.h b/include/hw/arm/xlnx-zynqmp.h
new file mode 100644
index 0000000..c2931bf
--- /dev/null
+++ b/include/hw/arm/xlnx-zynqmp.h
@@ -0,0 +1,99 @@
+/*
+ * Xilinx Zynq MPSoC emulation
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * for more details.
+ */
+
+#ifndef XLNX_ZYNQMP_H
+
+#include "qemu-common.h"
+#include "hw/arm/arm.h"
+#include "hw/intc/arm_gic.h"
+#include "hw/net/cadence_gem.h"
+#include "hw/char/cadence_uart.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
+#include "hw/sd/sdhci.h"
+#include "hw/ssi/xilinx_spips.h"
+#include "hw/dma/xlnx_dpdma.h"
+#include "hw/display/xlnx_dp.h"
+
+#define TYPE_XLNX_ZYNQMP "xlnx,zynqmp"
+#define XLNX_ZYNQMP(obj) OBJECT_CHECK(XlnxZynqMPState, (obj), \
+                                       TYPE_XLNX_ZYNQMP)
+
+#define XLNX_ZYNQMP_NUM_APU_CPUS 4
+#define XLNX_ZYNQMP_NUM_RPU_CPUS 2
+#define XLNX_ZYNQMP_NUM_GEMS 4
+#define XLNX_ZYNQMP_NUM_UARTS 2
+#define XLNX_ZYNQMP_NUM_SDHCI 2
+#define XLNX_ZYNQMP_NUM_SPIS 2
+
+#define XLNX_ZYNQMP_NUM_OCM_BANKS 4
+#define XLNX_ZYNQMP_OCM_RAM_0_ADDRESS 0xFFFC0000
+#define XLNX_ZYNQMP_OCM_RAM_SIZE 0x10000
+
+#define XLNX_ZYNQMP_GIC_REGIONS 2
+
+/* ZynqMP maps the ARM GIC regions (GICC, GICD ...) at consecutive 64k offsets
+ * and under-decodes the 64k region. This mirrors the 4k regions to every 4k
+ * aligned address in the 64k region. To implement each GIC region needs a
+ * number of memory region aliases.
+ */
+
+#define XLNX_ZYNQMP_GIC_REGION_SIZE 0x1000
+#define XLNX_ZYNQMP_GIC_ALIASES     (0x10000 / XLNX_ZYNQMP_GIC_REGION_SIZE - 1)
+
+#define XLNX_ZYNQMP_MAX_LOW_RAM_SIZE    0x80000000ull
+
+#define XLNX_ZYNQMP_MAX_HIGH_RAM_SIZE   0x800000000ull
+#define XLNX_ZYNQMP_HIGH_RAM_START      0x800000000ull
+
+#define XLNX_ZYNQMP_MAX_RAM_SIZE (XLNX_ZYNQMP_MAX_LOW_RAM_SIZE + \
+                                  XLNX_ZYNQMP_MAX_HIGH_RAM_SIZE)
+
+typedef struct XlnxZynqMPState {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    ARMCPU apu_cpu[XLNX_ZYNQMP_NUM_APU_CPUS];
+    ARMCPU rpu_cpu[XLNX_ZYNQMP_NUM_RPU_CPUS];
+    GICState gic;
+    MemoryRegion gic_mr[XLNX_ZYNQMP_GIC_REGIONS][XLNX_ZYNQMP_GIC_ALIASES];
+
+    MemoryRegion ocm_ram[XLNX_ZYNQMP_NUM_OCM_BANKS];
+
+    MemoryRegion *ddr_ram;
+    MemoryRegion ddr_ram_low, ddr_ram_high;
+
+    CadenceGEMState gem[XLNX_ZYNQMP_NUM_GEMS];
+    CadenceUARTState uart[XLNX_ZYNQMP_NUM_UARTS];
+    SysbusAHCIState sata;
+    SDHCIState sdhci[XLNX_ZYNQMP_NUM_SDHCI];
+    XilinxSPIPS spi[XLNX_ZYNQMP_NUM_SPIS];
+    XlnxDPState dp;
+    XlnxDPDMAState dpdma;
+
+    char *boot_cpu;
+    ARMCPU *boot_cpu_ptr;
+
+    /* Has the ARM Security extensions?  */
+    bool secure;
+    /* Has the RPU subsystem?  */
+    bool has_rpu;
+}  XlnxZynqMPState;
+
+#define XLNX_ZYNQMP_H
+#endif
diff --git a/include/hw/audio/audio.h b/include/hw/audio/audio.h
index b28abdd..55d40f7 100644
--- a/include/hw/audio/audio.h
+++ b/include/hw/audio/audio.h
@@ -1,5 +1,5 @@
-#ifndef HW_AUDIODEV_H
-#define HW_AUDIODEV_H 1
+#ifndef HW_AUDIO_H
+#define HW_AUDIO_H
 
 void isa_register_soundhw(const char *name, const char *descr,
                           int (*init_isa)(ISABus *bus));
diff --git a/include/hw/audio/pcspk.h b/include/hw/audio/pcspk.h
index ef95dd1..172afbf 100644
--- a/include/hw/audio/pcspk.h
+++ b/include/hw/audio/pcspk.h
@@ -38,10 +38,10 @@
     isadev = isa_create(bus, TYPE_PC_SPEAKER);
     dev = DEVICE(isadev);
     qdev_prop_set_uint32(dev, "iobase", 0x61);
-    qdev_prop_set_ptr(dev, "pit", pit);
+    object_property_set_link(OBJECT(dev), OBJECT(pit), "pit", NULL);
     qdev_init_nofail(dev);
 
     return isadev;
 }
 
-#endif /* !HW_PCSPK_H */
+#endif /* HW_PCSPK_H */
diff --git a/include/hw/block/block.h b/include/hw/block/block.h
index 0d0ce9a..df9d207 100644
--- a/include/hw/block/block.h
+++ b/include/hw/block/block.h
@@ -8,11 +8,10 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#ifndef HW_BLOCK_COMMON_H
-#define HW_BLOCK_COMMON_H
+#ifndef HW_BLOCK_H
+#define HW_BLOCK_H
 
 #include "qemu-common.h"
-#include "qapi/error.h"
 
 /* Configuration */
 
@@ -26,6 +25,9 @@
     uint32_t discard_granularity;
     /* geometry, not all devices use this */
     uint32_t cyls, heads, secs;
+    OnOffAuto wce;
+    BlockdevOnError rerror;
+    BlockdevOnError werror;
 } BlockConf;
 
 static inline unsigned int get_physical_block_exp(BlockConf *conf)
@@ -44,25 +46,34 @@
 #define DEFINE_BLOCK_PROPERTIES(_state, _conf)                          \
     DEFINE_PROP_DRIVE("drive", _state, _conf.blk),                      \
     DEFINE_PROP_BLOCKSIZE("logical_block_size", _state,                 \
-                          _conf.logical_block_size, 512),               \
+                          _conf.logical_block_size),                    \
     DEFINE_PROP_BLOCKSIZE("physical_block_size", _state,                \
-                          _conf.physical_block_size, 512),              \
+                          _conf.physical_block_size),                   \
     DEFINE_PROP_UINT16("min_io_size", _state, _conf.min_io_size, 0),  \
     DEFINE_PROP_UINT32("opt_io_size", _state, _conf.opt_io_size, 0),    \
     DEFINE_PROP_UINT32("discard_granularity", _state, \
-                       _conf.discard_granularity, -1)
+                       _conf.discard_granularity, -1), \
+    DEFINE_PROP_ON_OFF_AUTO("write-cache", _state, _conf.wce, ON_OFF_AUTO_AUTO)
 
 #define DEFINE_BLOCK_CHS_PROPERTIES(_state, _conf)      \
     DEFINE_PROP_UINT32("cyls", _state, _conf.cyls, 0),  \
     DEFINE_PROP_UINT32("heads", _state, _conf.heads, 0), \
     DEFINE_PROP_UINT32("secs", _state, _conf.secs, 0)
 
+#define DEFINE_BLOCK_ERROR_PROPERTIES(_state, _conf)                    \
+    DEFINE_PROP_BLOCKDEV_ON_ERROR("rerror", _state, _conf.rerror,       \
+                                  BLOCKDEV_ON_ERROR_AUTO),              \
+    DEFINE_PROP_BLOCKDEV_ON_ERROR("werror", _state, _conf.werror,       \
+                                  BLOCKDEV_ON_ERROR_AUTO)
+
 /* Configuration helpers */
 
 void blkconf_serial(BlockConf *conf, char **serial);
 void blkconf_geometry(BlockConf *conf, int *trans,
                       unsigned cyls_max, unsigned heads_max, unsigned secs_max,
                       Error **errp);
+void blkconf_blocksizes(BlockConf *conf);
+void blkconf_apply_backend_options(BlockConf *conf);
 
 /* Hard disk geometry */
 
diff --git a/include/hw/block/fdc.h b/include/hw/block/fdc.h
index d48b2f8..1749dab 100644
--- a/include/hw/block/fdc.h
+++ b/include/hw/block/fdc.h
@@ -6,13 +6,6 @@
 /* fdc.c */
 #define MAX_FD 2
 
-typedef enum FDriveType {
-    FDRIVE_DRV_144  = 0x00,   /* 1.44 MB 3"5 drive      */
-    FDRIVE_DRV_288  = 0x01,   /* 2.88 MB 3"5 drive      */
-    FDRIVE_DRV_120  = 0x02,   /* 1.2  MB 5"25 drive     */
-    FDRIVE_DRV_NONE = 0x03,   /* No drive connected     */
-} FDriveType;
-
 #define TYPE_ISA_FDC "isa-fdc"
 
 ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds);
@@ -21,6 +14,8 @@
 void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
                        DriveInfo **fds, qemu_irq *fdc_tc);
 
-FDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i);
+FloppyDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i);
+void isa_fdc_get_drive_max_chs(FloppyDriveType type,
+                               uint8_t *maxc, uint8_t *maxh, uint8_t *maxs);
 
 #endif
diff --git a/include/hw/block/flash.h b/include/hw/block/flash.h
index 50ccbbc..67c3aa3 100644
--- a/include/hw/block/flash.h
+++ b/include/hw/block/flash.h
@@ -1,10 +1,13 @@
 #ifndef HW_FLASH_H
-#define HW_FLASH_H 1
+#define HW_FLASH_H
 
 /* NOR flash devices */
 
 #include "exec/memory.h"
 
+#define TYPE_CFI_PFLASH01 "cfi.pflash01"
+#define TYPE_CFI_PFLASH02 "cfi.pflash02"
+
 typedef struct pflash_t pflash_t;
 
 /* pflash_cfi01.c */
diff --git a/include/hw/boards.h b/include/hw/boards.h
index e0a6790..3e69eca 100644
--- a/include/hw/boards.h
+++ b/include/hw/boards.h
@@ -3,56 +3,23 @@
 #ifndef HW_BOARDS_H
 #define HW_BOARDS_H
 
-#include "qemu/typedefs.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/accel.h"
 #include "hw/qdev.h"
 #include "qom/object.h"
-
-
-typedef void QEMUMachineInitFunc(MachineState *ms);
-
-typedef void QEMUMachineResetFunc(void);
-
-typedef void QEMUMachineHotAddCPUFunc(const int64_t id, Error **errp);
-
-typedef int QEMUMachineGetKvmtypeFunc(const char *arg);
-
-struct QEMUMachine {
-    const char *family; /* NULL iff @name identifies a standalone machtype */
-    const char *name;
-    const char *alias;
-    const char *desc;
-    QEMUMachineInitFunc *init;
-    QEMUMachineResetFunc *reset;
-    QEMUMachineHotAddCPUFunc *hot_add_cpu;
-    QEMUMachineGetKvmtypeFunc *kvm_type;
-    BlockInterfaceType block_default_type;
-    int units_per_default_bus;
-    int max_cpus;
-    unsigned int no_serial:1,
-        no_parallel:1,
-        use_virtcon:1,
-        use_sclp:1,
-        no_floppy:1,
-        no_cdrom:1,
-        no_sdcard:1,
-        has_dynamic_sysbus:1;
-    int is_default;
-    const char *default_machine_opts;
-    const char *default_boot_order;
-    const char *default_display;
-    GlobalProperty *compat_props;
-    const char *hw_version;
-};
+#include "qom/cpu.h"
 
 void memory_region_allocate_system_memory(MemoryRegion *mr, Object *owner,
                                           const char *name,
                                           uint64_t ram_size);
 
-int qemu_register_machine(QEMUMachine *m);
-
 #define TYPE_MACHINE_SUFFIX "-machine"
+
+/* Machine class name that needs to be used for class-name-based machine
+ * type lookup to work.
+ */
+#define MACHINE_TYPE_NAME(machinename) (machinename TYPE_MACHINE_SUFFIX)
+
 #define TYPE_MACHINE "machine"
 #undef MACHINE  /* BSD defines it and QEMU does not use it */
 #define MACHINE(obj) \
@@ -65,14 +32,60 @@
 MachineClass *find_default_machine(void);
 extern MachineState *current_machine;
 
+bool machine_usb(MachineState *machine);
+bool machine_kernel_irqchip_allowed(MachineState *machine);
+bool machine_kernel_irqchip_required(MachineState *machine);
+bool machine_kernel_irqchip_split(MachineState *machine);
+int machine_kvm_shadow_mem(MachineState *machine);
+int machine_phandle_start(MachineState *machine);
+bool machine_dump_guest_core(MachineState *machine);
+bool machine_mem_merge(MachineState *machine);
+void machine_register_compat_props(MachineState *machine);
+
+/**
+ * CPUArchId:
+ * @arch_id - architecture-dependent CPU ID of present or possible CPU
+ * @cpu - pointer to corresponding CPU object if it's present on NULL otherwise
+ */
+typedef struct {
+    uint64_t arch_id;
+    struct CPUState *cpu;
+} CPUArchId;
+
+/**
+ * CPUArchIdList:
+ * @len - number of @CPUArchId items in @cpus array
+ * @cpus - array of present or possible CPUs for current machine configuration
+ */
+typedef struct {
+    int len;
+    CPUArchId cpus[0];
+} CPUArchIdList;
+
 /**
  * MachineClass:
- * @qemu_machine: #QEMUMachine
  * @get_hotplug_handler: this function is called during bus-less
  *    device hotplug. If defined it returns pointer to an instance
  *    of HotplugHandler object, which handles hotplug operation
  *    for a given @dev. It may return NULL if @dev doesn't require
  *    any actions to be performed by hotplug handler.
+ * @cpu_index_to_socket_id:
+ *    used to provide @cpu_index to socket number mapping, allowing
+ *    a machine to group CPU threads belonging to the same socket/package
+ *    Returns: socket number given cpu_index belongs to.
+ * @hw_version:
+ *    Value of QEMU_VERSION when the machine was added to QEMU.
+ *    Set only by old machines because they need to keep
+ *    compatibility on code that exposed QEMU_VERSION to guests in
+ *    the past (and now use qemu_hw_version()).
+ * @possible_cpu_arch_ids:
+ *    Returns an array of @CPUArchId architecture-dependent CPU IDs
+ *    which includes CPU IDs for present and possible to hotplug CPUs.
+ *    Caller is responsible for freeing returned list.
+ * @query_hotpluggable_cpus:
+ *    Returns a @HotpluggableCPUList, which describes CPUs objects which
+ *    could be added with -device/device_add.
+ *    Caller is responsible for freeing returned list.
  */
 struct MachineClass {
     /*< private >*/
@@ -99,16 +112,24 @@
         no_floppy:1,
         no_cdrom:1,
         no_sdcard:1,
-        has_dynamic_sysbus:1;
+        has_dynamic_sysbus:1,
+        pci_allow_0_address:1,
+        legacy_fw_cfg_order:1;
     int is_default;
     const char *default_machine_opts;
     const char *default_boot_order;
     const char *default_display;
-    GlobalProperty *compat_props;
+    GArray *compat_props;
     const char *hw_version;
+    ram_addr_t default_ram_size;
+    bool option_rom_has_mr;
+    bool rom_file_has_mr;
 
     HotplugHandler *(*get_hotplug_handler)(MachineState *machine,
                                            DeviceState *dev);
+    unsigned (*cpu_index_to_socket_id)(unsigned cpu_index);
+    CPUArchIdList *(*possible_cpu_arch_ids)(MachineState *machine);
+    HotpluggableCPUList *(*query_hotpluggable_cpus)(MachineState *machine);
 };
 
 /**
@@ -122,7 +143,9 @@
     /*< public >*/
 
     char *accel;
-    bool kernel_irqchip;
+    bool kernel_irqchip_allowed;
+    bool kernel_irqchip_required;
+    bool kernel_irqchip_split;
     int kvm_shadow_mem;
     char *dtb;
     char *dumpdtb;
@@ -131,8 +154,13 @@
     bool dump_guest_core;
     bool mem_merge;
     bool usb;
+    bool usb_disabled;
+    bool igd_gfx_passthru;
     char *firmware;
     bool iommu;
+    bool suppress_vmdesc;
+    bool enforce_config_section;
+    bool enable_graphics;
 
     ram_addr_t ram_size;
     ram_addr_t maxram_size;
@@ -145,4 +173,37 @@
     AccelState *accelerator;
 };
 
+#define DEFINE_MACHINE(namestr, machine_initfn) \
+    static void machine_initfn##_class_init(ObjectClass *oc, void *data) \
+    { \
+        MachineClass *mc = MACHINE_CLASS(oc); \
+        machine_initfn(mc); \
+    } \
+    static const TypeInfo machine_initfn##_typeinfo = { \
+        .name       = MACHINE_TYPE_NAME(namestr), \
+        .parent     = TYPE_MACHINE, \
+        .class_init = machine_initfn##_class_init, \
+    }; \
+    static void machine_initfn##_register_types(void) \
+    { \
+        type_register_static(&machine_initfn##_typeinfo); \
+    } \
+    type_init(machine_initfn##_register_types)
+
+#define SET_MACHINE_COMPAT(m, COMPAT) \
+    do {                              \
+        int i;                        \
+        static GlobalProperty props[] = {       \
+            COMPAT                              \
+            { /* end of list */ }               \
+        };                                      \
+        if (!m->compat_props) { \
+            m->compat_props = g_array_new(false, false, sizeof(void *)); \
+        } \
+        for (i = 0; props[i].driver != NULL; i++) {    \
+            GlobalProperty *prop = &props[i];          \
+            g_array_append_val(m->compat_props, prop); \
+        }                                              \
+    } while (0)
+
 #endif
diff --git a/include/hw/bt.h b/include/hw/bt.h
index cb2a7e6..185e79d 100644
--- a/include/hw/bt.h
+++ b/include/hw/bt.h
@@ -24,7 +24,7 @@
  */
 
 #ifndef HW_BT_H
-#define HW_BT_H 1
+#define HW_BT_H
 
 #include "hw/irq.h"
 
@@ -504,7 +504,6 @@
 
 #define OCF_CREATE_CONN_CANCEL		0x0008
 typedef struct {
-    uint8_t	status;
     bdaddr_t	bdaddr;
 } QEMU_PACKED create_conn_cancel_cp;
 #define CREATE_CONN_CANCEL_CP_SIZE 6
@@ -1266,13 +1265,13 @@
     uint8_t	status;
     uint16_t	handle;
 } QEMU_PACKED reset_failed_contact_counter_rp;
-#define RESET_FAILED_CONTACT_COUNTER_RP_SIZE 4
+#define RESET_FAILED_CONTACT_COUNTER_RP_SIZE 3
 
 #define OCF_READ_LINK_QUALITY		0x0003
 typedef struct {
     uint16_t	handle;
 } QEMU_PACKED read_link_quality_cp;
-#define READ_LINK_QUALITY_CP_SIZE 4
+#define READ_LINK_QUALITY_CP_SIZE 2
 
 typedef struct {
     uint8_t	status;
@@ -1332,7 +1331,7 @@
     uint8_t	dev_class[3];
     uint16_t	clock_offset;
 } QEMU_PACKED inquiry_info;
-#define INQUIRY_INFO_SIZE 14
+#define INQUIRY_INFO_SIZE 15
 
 #define EVT_CONN_COMPLETE		0x03
 typedef struct {
@@ -1381,7 +1380,7 @@
     uint16_t	handle;
     uint8_t	encrypt;
 } QEMU_PACKED evt_encrypt_change;
-#define EVT_ENCRYPT_CHANGE_SIZE 5
+#define EVT_ENCRYPT_CHANGE_SIZE 4
 
 #define EVT_CHANGE_CONN_LINK_KEY_COMPLETE	0x09
 typedef struct {
@@ -1629,18 +1628,6 @@
 } QEMU_PACKED evt_sniff_subrate;
 #define EVT_SNIFF_SUBRATE_SIZE 11
 
-#define EVT_EXTENDED_INQUIRY_RESULT	0x2F
-typedef struct {
-    bdaddr_t	bdaddr;
-    uint8_t	pscan_rep_mode;
-    uint8_t	pscan_period_mode;
-    uint8_t	dev_class[3];
-    uint16_t	clock_offset;
-    int8_t	rssi;
-    uint8_t	data[240];
-} QEMU_PACKED extended_inquiry_info;
-#define EXTENDED_INQUIRY_INFO_SIZE 254
-
 #define EVT_TESTING			0xFE
 
 #define EVT_VENDOR			0xFF
diff --git a/include/hw/char/bcm2835_aux.h b/include/hw/char/bcm2835_aux.h
new file mode 100644
index 0000000..42f0ee7
--- /dev/null
+++ b/include/hw/char/bcm2835_aux.h
@@ -0,0 +1,33 @@
+/*
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_AUX_H
+#define BCM2835_AUX_H
+
+#include "hw/sysbus.h"
+#include "sysemu/char.h"
+
+#define TYPE_BCM2835_AUX "bcm2835-aux"
+#define BCM2835_AUX(obj) OBJECT_CHECK(BCM2835AuxState, (obj), TYPE_BCM2835_AUX)
+
+#define BCM2835_AUX_RX_FIFO_LEN 8
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion iomem;
+    CharDriverState *chr;
+    qemu_irq irq;
+
+    uint8_t read_fifo[BCM2835_AUX_RX_FIFO_LEN];
+    uint8_t read_pos, read_count;
+    uint8_t ier, iir;
+} BCM2835AuxState;
+
+#endif
diff --git a/include/hw/char/cadence_uart.h b/include/hw/char/cadence_uart.h
new file mode 100644
index 0000000..a12773c
--- /dev/null
+++ b/include/hw/char/cadence_uart.h
@@ -0,0 +1,70 @@
+/*
+ * Device model for Cadence UART
+ *
+ * Copyright (c) 2010 Xilinx Inc.
+ * Copyright (c) 2012 Peter A.G. Crosthwaite (peter.crosthwaite@petalogix.com)
+ * Copyright (c) 2012 PetaLogix Pty Ltd.
+ * Written by Haibing Ma
+ *            M.Habib
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef CADENCE_UART_H
+
+#include "hw/sysbus.h"
+#include "sysemu/char.h"
+#include "qemu/timer.h"
+
+#define CADENCE_UART_RX_FIFO_SIZE           16
+#define CADENCE_UART_TX_FIFO_SIZE           16
+
+#define CADENCE_UART_R_MAX (0x48/4)
+
+#define TYPE_CADENCE_UART "cadence_uart"
+#define CADENCE_UART(obj) OBJECT_CHECK(CadenceUARTState, (obj), \
+                                       TYPE_CADENCE_UART)
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    uint32_t r[CADENCE_UART_R_MAX];
+    uint8_t rx_fifo[CADENCE_UART_RX_FIFO_SIZE];
+    uint8_t tx_fifo[CADENCE_UART_TX_FIFO_SIZE];
+    uint32_t rx_wpos;
+    uint32_t rx_count;
+    uint32_t tx_count;
+    uint64_t char_tx_time;
+    CharDriverState *chr;
+    qemu_irq irq;
+    QEMUTimer *fifo_trigger_handle;
+} CadenceUARTState;
+
+static inline DeviceState *cadence_uart_create(hwaddr addr,
+                                        qemu_irq irq,
+                                        CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+
+    dev = qdev_create(NULL, TYPE_CADENCE_UART);
+    s = SYS_BUS_DEVICE(dev);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(s, 0, addr);
+    sysbus_connect_irq(s, 0, irq);
+
+    return dev;
+}
+
+#define CADENCE_UART_H
+#endif
diff --git a/include/hw/char/digic-uart.h b/include/hw/char/digic-uart.h
index ef83a30..7b3f145 100644
--- a/include/hw/char/digic-uart.h
+++ b/include/hw/char/digic-uart.h
@@ -19,7 +19,6 @@
 #define HW_CHAR_DIGIC_UART_H
 
 #include "hw/sysbus.h"
-#include "qemu/typedefs.h"
 
 #define TYPE_DIGIC_UART "digic-uart"
 #define DIGIC_UART(obj) \
diff --git a/include/hw/char/escc.h b/include/hw/char/escc.h
index 2742d70..297e2eb 100644
--- a/include/hw/char/escc.h
+++ b/include/hw/char/escc.h
@@ -1,5 +1,5 @@
 #ifndef HW_ESCC_H
-#define HW_ESCC_H 1
+#define HW_ESCC_H
 
 /* escc.c */
 #define TYPE_ESCC "escc"
diff --git a/include/hw/char/imx_serial.h b/include/hw/char/imx_serial.h
new file mode 100644
index 0000000..6cd75c0
--- /dev/null
+++ b/include/hw/char/imx_serial.h
@@ -0,0 +1,102 @@
+/*
+ * Device model for i.MX UART
+ *
+ * Copyright (c) 2008 OKL
+ * Originally Written by Hans Jiang
+ * Copyright (c) 2011 NICTA Pty Ltd.
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_SERIAL_H
+#define IMX_SERIAL_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_SERIAL "imx.serial"
+#define IMX_SERIAL(obj) OBJECT_CHECK(IMXSerialState, (obj), TYPE_IMX_SERIAL)
+
+#define URXD_CHARRDY    (1<<15)   /* character read is valid */
+#define URXD_ERR        (1<<14)   /* Character has error */
+#define URXD_BRK        (1<<11)   /* Break received */
+
+#define USR1_PARTYER    (1<<15)   /* Parity Error */
+#define USR1_RTSS       (1<<14)   /* RTS pin status */
+#define USR1_TRDY       (1<<13)   /* Tx ready */
+#define USR1_RTSD       (1<<12)   /* RTS delta: pin changed state */
+#define USR1_ESCF       (1<<11)   /* Escape sequence interrupt */
+#define USR1_FRAMERR    (1<<10)   /* Framing error  */
+#define USR1_RRDY       (1<<9)    /* receiver ready */
+#define USR1_AGTIM      (1<<8)    /* Aging timer interrupt */
+#define USR1_DTRD       (1<<7)    /* DTR changed */
+#define USR1_RXDS       (1<<6)    /* Receiver is idle */
+#define USR1_AIRINT     (1<<5)    /* Aysnch IR interrupt */
+#define USR1_AWAKE      (1<<4)    /* Falling edge detected on RXd pin */
+
+#define USR2_ADET       (1<<15)   /* Autobaud complete */
+#define USR2_TXFE       (1<<14)   /* Transmit FIFO empty */
+#define USR2_DTRF       (1<<13)   /* DTR/DSR transition */
+#define USR2_IDLE       (1<<12)   /* UART has been idle for too long */
+#define USR2_ACST       (1<<11)   /* Autobaud counter stopped */
+#define USR2_RIDELT     (1<<10)   /* Ring Indicator delta */
+#define USR2_RIIN       (1<<9)    /* Ring Indicator Input */
+#define USR2_IRINT      (1<<8)    /* Serial Infrared Interrupt */
+#define USR2_WAKE       (1<<7)    /* Start bit detected */
+#define USR2_DCDDELT    (1<<6)    /* Data Carrier Detect delta */
+#define USR2_DCDIN      (1<<5)    /* Data Carrier Detect Input */
+#define USR2_RTSF       (1<<4)    /* RTS transition */
+#define USR2_TXDC       (1<<3)    /* Transmission complete */
+#define USR2_BRCD       (1<<2)    /* Break condition detected */
+#define USR2_ORE        (1<<1)    /* Overrun error */
+#define USR2_RDR        (1<<0)    /* Receive data ready */
+
+#define UCR1_TRDYEN     (1<<13)   /* Tx Ready Interrupt Enable */
+#define UCR1_RRDYEN     (1<<9)    /* Rx Ready Interrupt Enable */
+#define UCR1_TXMPTYEN   (1<<6)    /* Tx Empty Interrupt Enable */
+#define UCR1_UARTEN     (1<<0)    /* UART Enable */
+
+#define UCR2_TXEN       (1<<2)    /* Transmitter enable */
+#define UCR2_RXEN       (1<<1)    /* Receiver enable */
+#define UCR2_SRST       (1<<0)    /* Reset complete */
+
+#define UTS1_TXEMPTY    (1<<6)
+#define UTS1_RXEMPTY    (1<<5)
+#define UTS1_TXFULL     (1<<4)
+#define UTS1_RXFULL     (1<<3)
+
+typedef struct IMXSerialState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    int32_t readbuff;
+
+    uint32_t usr1;
+    uint32_t usr2;
+    uint32_t ucr1;
+    uint32_t ucr2;
+    uint32_t uts1;
+
+    /*
+     * The registers below are implemented just so that the
+     * guest OS sees what it has written
+     */
+    uint32_t onems;
+    uint32_t ufcr;
+    uint32_t ubmr;
+    uint32_t ubrc;
+    uint32_t ucr3;
+
+    qemu_irq irq;
+    CharDriverState *chr;
+} IMXSerialState;
+
+#endif
diff --git a/include/hw/char/lm32_juart.h b/include/hw/char/lm32_juart.h
index 70dc416..e7c6fb5 100644
--- a/include/hw/char/lm32_juart.h
+++ b/include/hw/char/lm32_juart.h
@@ -10,4 +10,4 @@
 void lm32_juart_set_jtx(DeviceState *d, uint32_t jtx);
 void lm32_juart_set_jrx(DeviceState *d, uint32_t jrx);
 
-#endif /* QEMU_HW_LM32_JUART_H */
+#endif /* QEMU_HW_CHAR_LM32_JUART_H */
diff --git a/include/hw/char/pl011.h b/include/hw/char/pl011.h
new file mode 100644
index 0000000..0ca7c19
--- /dev/null
+++ b/include/hw/char/pl011.h
@@ -0,0 +1,52 @@
+/*
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HW_PL011_H
+#define HW_PL011_H
+
+static inline DeviceState *pl011_create(hwaddr addr,
+                                        qemu_irq irq,
+                                        CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+
+    dev = qdev_create(NULL, "pl011");
+    s = SYS_BUS_DEVICE(dev);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(s, 0, addr);
+    sysbus_connect_irq(s, 0, irq);
+
+    return dev;
+}
+
+static inline DeviceState *pl011_luminary_create(hwaddr addr,
+                                                 qemu_irq irq,
+                                                 CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+
+    dev = qdev_create(NULL, "pl011_luminary");
+    s = SYS_BUS_DEVICE(dev);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(s, 0, addr);
+    sysbus_connect_irq(s, 0, irq);
+
+    return dev;
+}
+
+#endif
diff --git a/include/hw/char/serial.h b/include/hw/char/serial.h
index f431764..a4fd3d5 100644
--- a/include/hw/char/serial.h
+++ b/include/hw/char/serial.h
@@ -22,8 +22,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef HW_SERIAL_H
-#define HW_SERIAL_H 1
+#define HW_SERIAL_H
 
 #include "hw/hw.h"
 #include "sysemu/sysemu.h"
@@ -55,7 +56,8 @@
     int last_break_enable;
     int it_shift;
     int baudbase;
-    int tsr_retry;
+    uint32_t tsr_retry;
+    guint watch_tag;
     uint32_t wakeup;
 
     /* Time when the last byte was successfully sent out of the tsr */
@@ -92,6 +94,6 @@
 
 /* serial-isa.c */
 #define TYPE_ISA_SERIAL "isa-serial"
-bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr);
+void serial_hds_isa_init(ISABus *bus, int n);
 
 #endif
diff --git a/include/hw/char/stm32f2xx_usart.h b/include/hw/char/stm32f2xx_usart.h
new file mode 100644
index 0000000..b97f192
--- /dev/null
+++ b/include/hw/char/stm32f2xx_usart.h
@@ -0,0 +1,73 @@
+/*
+ * STM32F2XX USART
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef HW_STM32F2XX_USART_H
+#define HW_STM32F2XX_USART_H
+
+#include "hw/sysbus.h"
+#include "sysemu/char.h"
+#include "hw/hw.h"
+
+#define USART_SR   0x00
+#define USART_DR   0x04
+#define USART_BRR  0x08
+#define USART_CR1  0x0C
+#define USART_CR2  0x10
+#define USART_CR3  0x14
+#define USART_GTPR 0x18
+
+#define USART_SR_RESET 0x00C00000
+
+#define USART_SR_TXE  (1 << 7)
+#define USART_SR_TC   (1 << 6)
+#define USART_SR_RXNE (1 << 5)
+
+#define USART_CR1_UE  (1 << 13)
+#define USART_CR1_RXNEIE  (1 << 5)
+#define USART_CR1_TE  (1 << 3)
+#define USART_CR1_RE  (1 << 2)
+
+#define TYPE_STM32F2XX_USART "stm32f2xx-usart"
+#define STM32F2XX_USART(obj) \
+    OBJECT_CHECK(STM32F2XXUsartState, (obj), TYPE_STM32F2XX_USART)
+
+typedef struct {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+    MemoryRegion mmio;
+
+    uint32_t usart_sr;
+    uint32_t usart_dr;
+    uint32_t usart_brr;
+    uint32_t usart_cr1;
+    uint32_t usart_cr2;
+    uint32_t usart_cr3;
+    uint32_t usart_gtpr;
+
+    CharDriverState *chr;
+    qemu_irq irq;
+} STM32F2XXUsartState;
+#endif /* HW_STM32F2XX_USART_H */
diff --git a/include/hw/char/xilinx_uartlite.h b/include/hw/char/xilinx_uartlite.h
new file mode 100644
index 0000000..8b4fc54
--- /dev/null
+++ b/include/hw/char/xilinx_uartlite.h
@@ -0,0 +1,35 @@
+/*
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef XILINX_UARTLITE_H
+#define XILINX_UARTLITE_H
+
+static inline DeviceState *xilinx_uartlite_create(hwaddr addr,
+                                        qemu_irq irq,
+                                        CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+
+    dev = qdev_create(NULL, "xlnx.xps-uartlite");
+    s = SYS_BUS_DEVICE(dev);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(s, 0, addr);
+    sysbus_connect_irq(s, 0, irq);
+
+    return dev;
+}
+
+#endif
diff --git a/include/hw/compat.h b/include/hw/compat.h
index 313682a..7ee7299 100644
--- a/include/hw/compat.h
+++ b/include/hw/compat.h
@@ -1,35 +1,131 @@
 #ifndef HW_COMPAT_H
 #define HW_COMPAT_H
 
+#define HW_COMPAT_2_6 \
+    {\
+        .driver   = "virtio-mmio",\
+        .property = "format_transport_address",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "disable-modern",\
+        .value    = "on",\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "disable-legacy",\
+        .value    = "off",\
+    },
+
+#define HW_COMPAT_2_5 \
+    {\
+        .driver   = "isa-fdc",\
+        .property = "fallback",\
+        .value    = "144",\
+    },{\
+        .driver   = "pvscsi",\
+        .property = "x-old-pci-configuration",\
+        .value    = "on",\
+    },{\
+        .driver   = "pvscsi",\
+        .property = "x-disable-pcie",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "vmxnet3",\
+        .property = "x-old-msi-offsets",\
+        .value    = "on",\
+    },{\
+        .driver   = "vmxnet3",\
+        .property = "x-disable-pcie",\
+        .value    = "on",\
+    },
+
+#define HW_COMPAT_2_4 \
+    {\
+        .driver   = "virtio-blk-device",\
+        .property = "scsi",\
+        .value    = "true",\
+    },{\
+        .driver   = "e1000",\
+        .property = "extra_mac_registers",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "x-disable-pcie",\
+        .value    = "on",\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "migrate-extra",\
+        .value    = "off",\
+    },{\
+        .driver   = "fw_cfg_mem",\
+        .property = "dma_enabled",\
+        .value    = "off",\
+    },{\
+        .driver   = "fw_cfg_io",\
+        .property = "dma_enabled",\
+        .value    = "off",\
+    },
+
+#define HW_COMPAT_2_3 \
+    {\
+        .driver   = "virtio-blk-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-balloon-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-serial-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-9p-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-rng-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = TYPE_PCI_DEVICE,\
+        .property = "x-pcie-lnksta-dllla",\
+        .value    = "off",\
+    },
+
+#define HW_COMPAT_2_2 \
+    /* empty */
+
 #define HW_COMPAT_2_1 \
-        {\
-            .driver   = "intel-hda",\
-            .property = "old_msi_addr",\
-            .value    = "on",\
-        },{\
-            .driver   = "VGA",\
-            .property = "qemu-extended-regs",\
-            .value    = "off",\
-        },{\
-            .driver   = "secondary-vga",\
-            .property = "qemu-extended-regs",\
-            .value    = "off",\
-        },{\
-            .driver   = "virtio-scsi-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "usb-mouse",\
-            .property = "usb_version",\
-            .value    = stringify(1),\
-        },{\
-            .driver   = "usb-kbd",\
-            .property = "usb_version",\
-            .value    = stringify(1),\
-        },{\
-            .driver   = "virtio-pci",\
-            .property = "virtio-pci-bus-master-bug-migration",\
-            .value    = "on",\
-        }
+    {\
+        .driver   = "intel-hda",\
+        .property = "old_msi_addr",\
+        .value    = "on",\
+    },{\
+        .driver   = "VGA",\
+        .property = "qemu-extended-regs",\
+        .value    = "off",\
+    },{\
+        .driver   = "secondary-vga",\
+        .property = "qemu-extended-regs",\
+        .value    = "off",\
+    },{\
+        .driver   = "virtio-scsi-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "usb-mouse",\
+        .property = "usb_version",\
+        .value    = stringify(1),\
+    },{\
+        .driver   = "usb-kbd",\
+        .property = "usb_version",\
+        .value    = stringify(1),\
+    },{\
+        .driver   = "virtio-pci",\
+        .property = "virtio-pci-bus-master-bug-migration",\
+        .value    = "on",\
+    },
 
 #endif /* HW_COMPAT_H */
diff --git a/include/hw/cpu/core.h b/include/hw/cpu/core.h
new file mode 100644
index 0000000..79ac79c
--- /dev/null
+++ b/include/hw/cpu/core.h
@@ -0,0 +1,34 @@
+/*
+ * CPU core abstract device
+ *
+ * Copyright (C) 2016 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef HW_CPU_CORE_H
+#define HW_CPU_CORE_H
+
+#include "qemu/osdep.h"
+#include "hw/qdev.h"
+
+#define TYPE_CPU_CORE "cpu-core"
+
+#define CPU_CORE(obj) \
+    OBJECT_CHECK(CPUCore, (obj), TYPE_CPU_CORE)
+
+typedef struct CPUCore {
+    /*< private >*/
+    DeviceState parent_obj;
+
+    /*< public >*/
+    int core_id;
+    int nr_threads;
+} CPUCore;
+
+/* Note: topology field names need to be kept in sync with
+ * 'CpuInstanceProperties' */
+
+#define CPU_CORE_PROP_CORE_ID "core-id"
+
+#endif
diff --git a/include/hw/cpu/icc_bus.h b/include/hw/cpu/icc_bus.h
deleted file mode 100644
index 98a979f..0000000
--- a/include/hw/cpu/icc_bus.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* icc_bus.h
- * emulate x86 ICC (Interrupt Controller Communications) bus
- *
- * Copyright (c) 2013 Red Hat, Inc
- *
- * Authors:
- *     Igor Mammedov <imammedo@redhat.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>
- */
-#ifndef ICC_BUS_H
-#define ICC_BUS_H
-
-#include "exec/memory.h"
-#include "hw/qdev-core.h"
-
-#define TYPE_ICC_BUS "icc-bus"
-
-#ifndef CONFIG_USER_ONLY
-
-/**
- * ICCBus:
- *
- * ICC bus
- */
-typedef struct ICCBus {
-    /*< private >*/
-    BusState parent_obj;
-    /*< public >*/
-
-    MemoryRegion *apic_address_space;
-} ICCBus;
-
-#define ICC_BUS(obj) OBJECT_CHECK(ICCBus, (obj), TYPE_ICC_BUS)
-
-/**
- * ICCDevice:
- *
- * ICC device
- */
-typedef struct ICCDevice {
-    /*< private >*/
-    DeviceState qdev;
-    /*< public >*/
-} ICCDevice;
-
-/**
- * ICCDeviceClass:
- * @init: Initialization callback for derived classes.
- *
- * ICC device class
- */
-typedef struct ICCDeviceClass {
-    /*< private >*/
-    DeviceClass parent_class;
-    /*< public >*/
-
-    DeviceRealize realize;
-} ICCDeviceClass;
-
-#define TYPE_ICC_DEVICE "icc-device"
-#define ICC_DEVICE(obj) OBJECT_CHECK(ICCDevice, (obj), TYPE_ICC_DEVICE)
-#define ICC_DEVICE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(ICCDeviceClass, (klass), TYPE_ICC_DEVICE)
-#define ICC_DEVICE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(ICCDeviceClass, (obj), TYPE_ICC_DEVICE)
-
-#define TYPE_ICC_BRIDGE "icc-bridge"
-
-#endif /* CONFIG_USER_ONLY */
-#endif
diff --git a/include/hw/cris/etraxfs.h b/include/hw/cris/etraxfs.h
index 73a6134..723a275 100644
--- a/include/hw/cris/etraxfs.h
+++ b/include/hw/cris/etraxfs.h
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef HW_EXTRAXFS_H
-#define HW_EXTRAXFS_H 1
+#ifndef HW_ETRAXFS_H
+#define HW_ETRAXFS_H
 
 #include "net/net.h"
 #include "hw/cris/etraxfs_dma.h"
@@ -46,4 +46,20 @@
     return dev;
 }
 
+static inline DeviceState *etraxfs_ser_create(hwaddr addr,
+                                              qemu_irq irq,
+                                              CharDriverState *chr)
+{
+    DeviceState *dev;
+    SysBusDevice *s;
+
+    dev = qdev_create(NULL, "etraxfs,serial");
+    s = SYS_BUS_DEVICE(dev);
+    qdev_prop_set_chr(dev, "chardev", chr);
+    qdev_init_nofail(dev);
+    sysbus_mmio_map(s, 0, addr);
+    sysbus_connect_irq(s, 0, irq);
+    return dev;
+}
+
 #endif
diff --git a/include/hw/cris/etraxfs_dma.h b/include/hw/cris/etraxfs_dma.h
index 38104a6..f6f33e0 100644
--- a/include/hw/cris/etraxfs_dma.h
+++ b/include/hw/cris/etraxfs_dma.h
@@ -1,5 +1,5 @@
 #ifndef HW_ETRAXFS_DMA_H
-#define HW_ETRAXFS_DMA_H 1
+#define HW_ETRAXFS_DMA_H
 
 struct dma_context_metadata {
 	/* data descriptor md */
diff --git a/include/hw/display/bcm2835_fb.h b/include/hw/display/bcm2835_fb.h
new file mode 100644
index 0000000..9a12d7a
--- /dev/null
+++ b/include/hw/display/bcm2835_fb.h
@@ -0,0 +1,47 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_FB_H
+#define BCM2835_FB_H
+
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+#include "ui/console.h"
+
+#define TYPE_BCM2835_FB "bcm2835-fb"
+#define BCM2835_FB(obj) OBJECT_CHECK(BCM2835FBState, (obj), TYPE_BCM2835_FB)
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    uint32_t vcram_base, vcram_size;
+    MemoryRegion *dma_mr;
+    AddressSpace dma_as;
+    MemoryRegion iomem;
+    MemoryRegionSection fbsection;
+    QemuConsole *con;
+    qemu_irq mbox_irq;
+
+    bool lock, invalidate, pending;
+    uint32_t xres, yres;
+    uint32_t xres_virtual, yres_virtual;
+    uint32_t xoffset, yoffset;
+    uint32_t bpp;
+    uint32_t base, pitch, size;
+    uint32_t pixo, alpha;
+} BCM2835FBState;
+
+void bcm2835_fb_reconfigure(BCM2835FBState *s, uint32_t *xres, uint32_t *yres,
+                            uint32_t *xoffset, uint32_t *yoffset, uint32_t *bpp,
+                            uint32_t *pixo, uint32_t *alpha);
+
+#endif
diff --git a/include/hw/display/dpcd.h b/include/hw/display/dpcd.h
new file mode 100644
index 0000000..6880ee3
--- /dev/null
+++ b/include/hw/display/dpcd.h
@@ -0,0 +1,105 @@
+/*
+ * dpcd.h
+ *
+ *  Copyright (C)2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option)any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef DPCD_H
+#define DPCD_H
+
+typedef struct DPCDState DPCDState;
+
+#define TYPE_DPCD "dpcd"
+#define DPCD(obj) OBJECT_CHECK(DPCDState, (obj), TYPE_DPCD)
+
+/* DCPD Revision. */
+#define DPCD_REVISION                           0x00
+#define DPCD_REV_1_0                            0x10
+#define DPCD_REV_1_1                            0x11
+
+/* DCPD Max Link Rate. */
+#define DPCD_MAX_LINK_RATE                      0x01
+#define DPCD_1_62GBPS                           0x06
+#define DPCD_2_7GBPS                            0x0A
+#define DPCD_5_4GBPS                            0x14
+
+#define DPCD_MAX_LANE_COUNT                     0x02
+#define DPCD_ONE_LANE                           0x01
+#define DPCD_TWO_LANES                          0x02
+#define DPCD_FOUR_LANES                         0x04
+
+/* DCPD Max down spread. */
+#define DPCD_UP_TO_0_5                          0x01
+#define DPCD_NO_AUX_HANDSHAKE_LINK_TRAINING     0x40
+
+/* DCPD Downstream port type. */
+#define DPCD_DISPLAY_PORT                       0x00
+#define DPCD_ANALOG                             0x02
+#define DPCD_DVI_HDMI                           0x04
+#define DPCD_OTHER                              0x06
+
+/* DPCD Format conversion. */
+#define DPCD_FORMAT_CONVERSION                  0x08
+
+/* Main link channel coding. */
+#define DPCD_ANSI_8B_10B                        0x01
+
+/* Down stream port count. */
+#define DPCD_OUI_SUPPORTED                      0x80
+
+/* Receiver port capability. */
+#define DPCD_RECEIVE_PORT0_CAP_0                0x08
+#define DPCD_RECEIVE_PORT0_CAP_1                0x09
+#define DPCD_EDID_PRESENT                       0x02
+#define DPCD_ASSOCIATED_TO_PRECEDING_PORT       0x04
+
+/* Down stream port capability. */
+#define DPCD_CAP_DISPLAY_PORT                   0x000
+#define DPCD_CAP_ANALOG_VGA                     0x001
+#define DPCD_CAP_DVI                            0x002
+#define DPCD_CAP_HDMI                           0x003
+#define DPCD_CAP_OTHER                          0x100
+
+#define DPCD_LANE0_1_STATUS                     0x202
+#define DPCD_LANE0_CR_DONE                      (1 << 0)
+#define DPCD_LANE0_CHANNEL_EQ_DONE              (1 << 1)
+#define DPCD_LANE0_SYMBOL_LOCKED                (1 << 2)
+#define DPCD_LANE1_CR_DONE                      (1 << 4)
+#define DPCD_LANE1_CHANNEL_EQ_DONE              (1 << 5)
+#define DPCD_LANE1_SYMBOL_LOCKED                (1 << 6)
+
+#define DPCD_LANE2_3_STATUS                     0x203
+#define DPCD_LANE2_CR_DONE                      (1 << 0)
+#define DPCD_LANE2_CHANNEL_EQ_DONE              (1 << 1)
+#define DPCD_LANE2_SYMBOL_LOCKED                (1 << 2)
+#define DPCD_LANE3_CR_DONE                      (1 << 4)
+#define DPCD_LANE3_CHANNEL_EQ_DONE              (1 << 5)
+#define DPCD_LANE3_SYMBOL_LOCKED                (1 << 6)
+
+#define DPCD_LANE_ALIGN_STATUS_UPDATED          0x204
+#define DPCD_INTERLANE_ALIGN_DONE               0x01
+#define DPCD_DOWNSTREAM_PORT_STATUS_CHANGED     0x40
+#define DPCD_LINK_STATUS_UPDATED                0x80
+
+#define DPCD_SINK_STATUS                        0x205
+#define DPCD_RECEIVE_PORT_0_STATUS              0x01
+
+#endif /* DPCD_H */
diff --git a/include/hw/display/xlnx_dp.h b/include/hw/display/xlnx_dp.h
new file mode 100644
index 0000000..ee046a5
--- /dev/null
+++ b/include/hw/display/xlnx_dp.h
@@ -0,0 +1,109 @@
+/*
+ * xlnx_dp.h
+ *
+ *  Copyright (C) 2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "hw/sysbus.h"
+#include "ui/console.h"
+#include "hw/misc/auxbus.h"
+#include "hw/i2c/i2c.h"
+#include "hw/display/dpcd.h"
+#include "hw/i2c/i2c-ddc.h"
+#include "qemu/fifo8.h"
+#include "hw/dma/xlnx_dpdma.h"
+#include "audio/audio.h"
+
+#ifndef XLNX_DP_H
+#define XLNX_DP_H
+
+#define AUD_CHBUF_MAX_DEPTH                 32768
+#define MAX_QEMU_BUFFER_SIZE                4096
+
+#define DP_CORE_REG_ARRAY_SIZE              (0x3AF >> 2)
+#define DP_AVBUF_REG_ARRAY_SIZE             (0x238 >> 2)
+#define DP_VBLEND_REG_ARRAY_SIZE            (0x1DF >> 2)
+#define DP_AUDIO_REG_ARRAY_SIZE             (0x50 >> 2)
+
+struct PixmanPlane {
+    pixman_format_code_t format;
+    DisplaySurface *surface;
+};
+
+typedef struct XlnxDPState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /* < public >*/
+    MemoryRegion container;
+
+    uint32_t core_registers[DP_CORE_REG_ARRAY_SIZE];
+    MemoryRegion core_iomem;
+
+    uint32_t avbufm_registers[DP_AVBUF_REG_ARRAY_SIZE];
+    MemoryRegion avbufm_iomem;
+
+    uint32_t vblend_registers[DP_VBLEND_REG_ARRAY_SIZE];
+    MemoryRegion vblend_iomem;
+
+    uint32_t audio_registers[DP_AUDIO_REG_ARRAY_SIZE];
+    MemoryRegion audio_iomem;
+
+    QemuConsole *console;
+
+    /*
+     * This is the planes used to display in console. When the blending is
+     * enabled bout_plane is displayed in console else it's g_plane.
+     */
+    struct PixmanPlane g_plane;
+    struct PixmanPlane v_plane;
+    struct PixmanPlane bout_plane;
+
+    QEMUSoundCard aud_card;
+    SWVoiceOut *amixer_output_stream;
+    int16_t audio_buffer_0[AUD_CHBUF_MAX_DEPTH];
+    int16_t audio_buffer_1[AUD_CHBUF_MAX_DEPTH];
+    size_t audio_data_available[2];
+    int64_t temp_buffer[AUD_CHBUF_MAX_DEPTH];
+    int16_t out_buffer[AUD_CHBUF_MAX_DEPTH];
+    size_t byte_left; /* byte available in out_buffer. */
+    size_t data_ptr;  /* next byte to be sent to QEMU. */
+
+    /* Associated DPDMA controller. */
+    XlnxDPDMAState *dpdma;
+
+    qemu_irq irq;
+
+    AUXBus *aux_bus;
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    /*
+     * XXX: This should be in an other module.
+     */
+    DPCDState *dpcd;
+    I2CDDCState *edid;
+} XlnxDPState;
+
+#define TYPE_XLNX_DP "xlnx.v-dp"
+#define XLNX_DP(obj) OBJECT_CHECK(XlnxDPState, (obj), TYPE_XLNX_DP)
+
+#endif /* !XLNX_DP_H */
diff --git a/include/hw/dma/bcm2835_dma.h b/include/hw/dma/bcm2835_dma.h
new file mode 100644
index 0000000..75312e2
--- /dev/null
+++ b/include/hw/dma/bcm2835_dma.h
@@ -0,0 +1,47 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_DMA_H
+#define BCM2835_DMA_H
+
+#include "qemu-common.h"
+#include "exec/address-spaces.h"
+#include "hw/sysbus.h"
+
+typedef struct {
+    uint32_t cs;
+    uint32_t conblk_ad;
+    uint32_t ti;
+    uint32_t source_ad;
+    uint32_t dest_ad;
+    uint32_t txfr_len;
+    uint32_t stride;
+    uint32_t nextconbk;
+    uint32_t debug;
+
+    qemu_irq irq;
+} BCM2835DMAChan;
+
+#define TYPE_BCM2835_DMA "bcm2835-dma"
+#define BCM2835_DMA(obj) \
+        OBJECT_CHECK(BCM2835DMAState, (obj), TYPE_BCM2835_DMA)
+
+#define BCM2835_DMA_NCHANS 16
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    MemoryRegion iomem0, iomem15;
+    MemoryRegion *dma_mr;
+    AddressSpace dma_as;
+
+    BCM2835DMAChan chan[BCM2835_DMA_NCHANS];
+    uint32_t int_status;
+    uint32_t enable;
+} BCM2835DMAState;
+
+#endif
diff --git a/include/hw/dma/xlnx-zynq-devcfg.h b/include/hw/dma/xlnx-zynq-devcfg.h
new file mode 100644
index 0000000..d40e5c8
--- /dev/null
+++ b/include/hw/dma/xlnx-zynq-devcfg.h
@@ -0,0 +1,62 @@
+/*
+ * QEMU model of the Xilinx Devcfg Interface
+ *
+ * (C) 2011 PetaLogix Pty Ltd
+ * (C) 2014 Xilinx Inc.
+ * Written by Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef XLNX_ZYNQ_DEVCFG_H
+
+#include "hw/register.h"
+#include "hw/sysbus.h"
+
+#define TYPE_XLNX_ZYNQ_DEVCFG "xlnx.ps7-dev-cfg"
+
+#define XLNX_ZYNQ_DEVCFG(obj) \
+    OBJECT_CHECK(XlnxZynqDevcfg, (obj), TYPE_XLNX_ZYNQ_DEVCFG)
+
+#define XLNX_ZYNQ_DEVCFG_R_MAX 0x118
+
+#define XLNX_ZYNQ_DEVCFG_DMA_CMD_FIFO_LEN 10
+
+typedef struct XlnxZynqDevcfgDMACmd {
+    uint32_t src_addr;
+    uint32_t dest_addr;
+    uint32_t src_len;
+    uint32_t dest_len;
+} XlnxZynqDevcfgDMACmd;
+
+typedef struct XlnxZynqDevcfg {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+    qemu_irq irq;
+
+    XlnxZynqDevcfgDMACmd dma_cmd_fifo[XLNX_ZYNQ_DEVCFG_DMA_CMD_FIFO_LEN];
+    uint8_t dma_cmd_fifo_num;
+
+    uint32_t regs[XLNX_ZYNQ_DEVCFG_R_MAX];
+    RegisterInfo regs_info[XLNX_ZYNQ_DEVCFG_R_MAX];
+} XlnxZynqDevcfg;
+
+#define XLNX_ZYNQ_DEVCFG_H
+#endif
diff --git a/include/hw/dma/xlnx_dpdma.h b/include/hw/dma/xlnx_dpdma.h
new file mode 100644
index 0000000..664df28
--- /dev/null
+++ b/include/hw/dma/xlnx_dpdma.h
@@ -0,0 +1,85 @@
+/*
+ * xlnx_dpdma.h
+ *
+ *  Copyright (C) 2015 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef XLNX_DPDMA_H
+#define XLNX_DPDMA_H
+
+#include "hw/sysbus.h"
+#include "ui/console.h"
+#include "sysemu/dma.h"
+
+#define XLNX_DPDMA_REG_ARRAY_SIZE (0x1000 >> 2)
+
+struct XlnxDPDMAState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+    MemoryRegion iomem;
+    uint32_t registers[XLNX_DPDMA_REG_ARRAY_SIZE];
+    uint8_t *data[6];
+    bool operation_finished[6];
+    qemu_irq irq;
+};
+
+typedef struct XlnxDPDMAState XlnxDPDMAState;
+
+#define TYPE_XLNX_DPDMA "xlnx.dpdma"
+#define XLNX_DPDMA(obj) OBJECT_CHECK(XlnxDPDMAState, (obj), TYPE_XLNX_DPDMA)
+
+/*
+ * xlnx_dpdma_start_operation: Start the operation on the specified channel. The
+ *                             DPDMA gets the current descriptor and retrieves
+ *                             data to the buffer specified by
+ *                             dpdma_set_host_data_location().
+ *
+ * Returns The number of bytes transfered by the DPDMA or 0 if an error occured.
+ *
+ * @s The DPDMA state.
+ * @channel The channel to start.
+ */
+size_t xlnx_dpdma_start_operation(XlnxDPDMAState *s, uint8_t channel,
+                                  bool one_desc);
+
+/*
+ * xlnx_dpdma_set_host_data_location: Set the location in the host memory where
+ *                                    to store the data out from the dma
+ *                                    channel.
+ *
+ * @s The DPDMA state.
+ * @channel The channel associated to the pointer.
+ * @p The buffer where to store the data.
+ */
+/* XXX: add a maximum size arg and send an interrupt in case of overflow. */
+void xlnx_dpdma_set_host_data_location(XlnxDPDMAState *s, uint8_t channel,
+                                       void *p);
+
+/*
+ * xlnx_dpdma_trigger_vsync_irq: Trigger a VSYNC IRQ when the display is
+ *                               updated.
+ *
+ * @s The DPDMA state.
+ */
+void xlnx_dpdma_trigger_vsync_irq(XlnxDPDMAState *s);
+
+#endif /* XLNX_DPDMA_H */
diff --git a/include/hw/elf_ops.h b/include/hw/elf_ops.h
index a517753..f510e7e 100644
--- a/include/hw/elf_ops.h
+++ b/include/hw/elf_ops.h
@@ -49,6 +49,13 @@
     bswap16s(&sym->st_shndx);
 }
 
+static void glue(bswap_rela, SZ)(struct elf_rela *rela)
+{
+    bswapSZs(&rela->r_offset);
+    bswapSZs(&rela->r_info);
+    bswapSZs((elf_word *)&rela->r_addend);
+}
+
 static struct elf_shdr *glue(find_section, SZ)(struct elf_shdr *shdr_table,
                                                int n, int type)
 {
@@ -182,12 +189,81 @@
     return -1;
 }
 
+static int glue(elf_reloc, SZ)(struct elfhdr *ehdr, int fd, int must_swab,
+                               uint64_t (*translate_fn)(void *, uint64_t),
+                               void *translate_opaque, uint8_t *data,
+                               struct elf_phdr *ph, int elf_machine)
+{
+    struct elf_shdr *reltab, *shdr_table = NULL;
+    struct elf_rela *rels = NULL;
+    int nrels, i, ret = -1;
+    elf_word wordval;
+    void *addr;
+
+    shdr_table = load_at(fd, ehdr->e_shoff,
+                         sizeof(struct elf_shdr) * ehdr->e_shnum);
+    if (!shdr_table) {
+        return -1;
+    }
+    if (must_swab) {
+        for (i = 0; i < ehdr->e_shnum; i++) {
+            glue(bswap_shdr, SZ)(&shdr_table[i]);
+        }
+    }
+
+    reltab = glue(find_section, SZ)(shdr_table, ehdr->e_shnum, SHT_RELA);
+    if (!reltab) {
+        goto fail;
+    }
+    rels = load_at(fd, reltab->sh_offset, reltab->sh_size);
+    if (!rels) {
+        goto fail;
+    }
+    nrels = reltab->sh_size / sizeof(struct elf_rela);
+
+    for (i = 0; i < nrels; i++) {
+        if (must_swab) {
+            glue(bswap_rela, SZ)(&rels[i]);
+        }
+        if (rels[i].r_offset < ph->p_vaddr ||
+            rels[i].r_offset >= ph->p_vaddr + ph->p_filesz) {
+            continue;
+        }
+        addr = &data[rels[i].r_offset - ph->p_vaddr];
+        switch (elf_machine) {
+        case EM_S390:
+            switch (rels[i].r_info) {
+            case R_390_RELATIVE:
+                wordval = *(elf_word *)addr;
+                if (must_swab) {
+                    bswapSZs(&wordval);
+                }
+                wordval = translate_fn(translate_opaque, wordval);
+                if (must_swab) {
+                    bswapSZs(&wordval);
+                }
+                *(elf_word *)addr = wordval;
+                break;
+            default:
+                fprintf(stderr, "Unsupported relocation type %i!\n",
+                        (int)rels[i].r_info);
+            }
+        }
+    }
+
+    ret = 0;
+fail:
+    g_free(rels);
+    g_free(shdr_table);
+    return ret;
+}
+
 static int glue(load_elf, SZ)(const char *name, int fd,
                               uint64_t (*translate_fn)(void *, uint64_t),
                               void *translate_opaque,
                               int must_swab, uint64_t *pentry,
                               uint64_t *lowaddr, uint64_t *highaddr,
-                              int elf_machine, int clear_lsb)
+                              int elf_machine, int clear_lsb, int data_swab)
 {
     struct elfhdr ehdr;
     struct elf_phdr *phdr = NULL, *ph;
@@ -206,25 +282,36 @@
 
     switch (elf_machine) {
         case EM_PPC64:
-            if (EM_PPC64 != ehdr.e_machine)
-                if (EM_PPC != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_PPC64) {
+                if (ehdr.e_machine != EM_PPC) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         case EM_X86_64:
-            if (EM_X86_64 != ehdr.e_machine)
-                if (EM_386 != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_X86_64) {
+                if (ehdr.e_machine != EM_386) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
             break;
         case EM_MICROBLAZE:
-            if (EM_MICROBLAZE != ehdr.e_machine)
-                if (EM_MICROBLAZE_OLD != ehdr.e_machine) {
+            if (ehdr.e_machine != EM_MICROBLAZE) {
+                if (ehdr.e_machine != EM_MICROBLAZE_OLD) {
                     ret = ELF_LOAD_WRONG_ARCH;
                     goto fail;
                 }
+            }
+            break;
+        case EM_MOXIE:
+            if (ehdr.e_machine != EM_MOXIE) {
+                if (ehdr.e_machine != EM_MOXIE_OLD) {
+                    ret = ELF_LOAD_WRONG_ARCH;
+                    goto fail;
+                }
+            }
             break;
         default:
             if (elf_machine != ehdr.e_machine) {
@@ -239,7 +326,9 @@
     glue(load_symbols, SZ)(&ehdr, fd, must_swab, clear_lsb);
 
     size = ehdr.e_phnum * sizeof(phdr[0]);
-    lseek(fd, ehdr.e_phoff, SEEK_SET);
+    if (lseek(fd, ehdr.e_phoff, SEEK_SET) != ehdr.e_phoff) {
+        goto fail;
+    }
     phdr = g_malloc0(size);
     if (!phdr)
         goto fail;
@@ -271,10 +360,32 @@
                linked at the wrong physical address.  */
             if (translate_fn) {
                 addr = translate_fn(translate_opaque, ph->p_paddr);
+                glue(elf_reloc, SZ)(&ehdr, fd, must_swab,  translate_fn,
+                                    translate_opaque, data, ph, elf_machine);
             } else {
                 addr = ph->p_paddr;
             }
 
+            if (data_swab) {
+                int j;
+                for (j = 0; j < file_size; j += (1 << data_swab)) {
+                    uint8_t *dp = data + j;
+                    switch (data_swab) {
+                    case (1):
+                        *(uint16_t *)dp = bswap16(*(uint16_t *)dp);
+                        break;
+                    case (2):
+                        *(uint32_t *)dp = bswap32(*(uint32_t *)dp);
+                        break;
+                    case (3):
+                        *(uint64_t *)dp = bswap64(*(uint64_t *)dp);
+                        break;
+                    default:
+                        g_assert_not_reached();
+                    }
+                }
+            }
+
             /* the entry pointer in the ELF header is a virtual
              * address, if the text segments paddr and vaddr differ
              * we need to adjust the entry */
diff --git a/include/hw/empty_slot.h b/include/hw/empty_slot.h
index 6079602..123a9f8 100644
--- a/include/hw/empty_slot.h
+++ b/include/hw/empty_slot.h
@@ -1,5 +1,5 @@
 #ifndef HW_EMPTY_SLOT_H
-#define HW_EMPTY_SLOT_H 1
+#define HW_EMPTY_SLOT_H
 
 /* empty_slot.c */
 void empty_slot_init(hwaddr addr, uint64_t slot_size);
diff --git a/include/hw/fw-path-provider.h b/include/hw/fw-path-provider.h
index 7afaec0..050cb05 100644
--- a/include/hw/fw-path-provider.h
+++ b/include/hw/fw-path-provider.h
@@ -16,7 +16,7 @@
  */
 
 #ifndef FW_PATH_PROVIDER_H
-#define FW_PATH_PROVIDER_H 1
+#define FW_PATH_PROVIDER_H
 
 #include "qemu-common.h"
 #include "qom/object.h"
diff --git a/include/hw/gpio/imx_gpio.h b/include/hw/gpio/imx_gpio.h
new file mode 100644
index 0000000..ffab437
--- /dev/null
+++ b/include/hw/gpio/imx_gpio.h
@@ -0,0 +1,63 @@
+/*
+ * i.MX processors GPIO registers definition.
+ *
+ * Copyright (C) 2015 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_GPIO_H
+#define IMX_GPIO_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_GPIO "imx.gpio"
+#define IMX_GPIO(obj) OBJECT_CHECK(IMXGPIOState, (obj), TYPE_IMX_GPIO)
+
+#define IMX_GPIO_MEM_SIZE 0x20
+
+/* i.MX GPIO memory map */
+#define DR_ADDR             0x00 /* DATA REGISTER */
+#define GDIR_ADDR           0x04 /* DIRECTION REGISTER */
+#define PSR_ADDR            0x08 /* PAD STATUS REGISTER */
+#define ICR1_ADDR           0x0c /* INTERRUPT CONFIGURATION REGISTER 1 */
+#define ICR2_ADDR           0x10 /* INTERRUPT CONFIGURATION REGISTER 2 */
+#define IMR_ADDR            0x14 /* INTERRUPT MASK REGISTER */
+#define ISR_ADDR            0x18 /* INTERRUPT STATUS REGISTER */
+#define EDGE_SEL_ADDR       0x1c /* EDGE SEL REGISTER */
+
+#define IMX_GPIO_PIN_COUNT 32
+
+typedef struct IMXGPIOState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t dr;
+    uint32_t gdir;
+    uint32_t psr;
+    uint64_t icr;
+    uint32_t imr;
+    uint32_t isr;
+    bool has_edge_sel;
+    uint32_t edge_sel;
+    bool has_upper_pin_irq;
+
+    qemu_irq irq[2];
+    qemu_irq output[IMX_GPIO_PIN_COUNT];
+} IMXGPIOState;
+
+#endif /* IMX_GPIO_H */
diff --git a/include/hw/hotplug.h b/include/hw/hotplug.h
index 050d2f0..c0db869 100644
--- a/include/hw/hotplug.h
+++ b/include/hw/hotplug.h
@@ -13,7 +13,6 @@
 #define HOTPLUG_H
 
 #include "qom/object.h"
-#include "qemu/typedefs.h"
 
 #define TYPE_HOTPLUG_HANDLER "hotplug-handler"
 
@@ -46,19 +45,21 @@
  * hardware (un)plug functions.
  *
  * @parent: Opaque parent interface.
- * @plug: plug callback.
+ * @pre_plug: pre plug callback called at start of device.realize(true)
+ * @plug: plug callback called at end of device.realize(true).
  * @unplug_request: unplug request callback.
  *                  Used as a means to initiate device unplug for devices that
  *                  require asynchronous unplug handling.
  * @unplug: unplug callback.
  *          Used for device removal with devices that implement
- *          asynchronous and synchronous (suprise) removal.
+ *          asynchronous and synchronous (surprise) removal.
  */
 typedef struct HotplugHandlerClass {
     /* <private> */
     InterfaceClass parent;
 
     /* <public> */
+    hotplug_fn pre_plug;
     hotplug_fn plug;
     hotplug_fn unplug_request;
     hotplug_fn unplug;
@@ -74,6 +75,16 @@
                           Error **errp);
 
 /**
+ * hotplug_handler_pre_plug:
+ *
+ * Call #HotplugHandlerClass.pre_plug callback of @plug_handler.
+ */
+void hotplug_handler_pre_plug(HotplugHandler *plug_handler,
+                              DeviceState *plugged_dev,
+                              Error **errp);
+
+
+/**
  * hotplug_handler_unplug_request:
  *
  * Calls #HotplugHandlerClass.unplug_request callback of @plug_handler.
diff --git a/include/hw/hw.h b/include/hw/hw.h
index 33bdb92..3669ebd 100644
--- a/include/hw/hw.h
+++ b/include/hw/hw.h
@@ -2,74 +2,22 @@
 #ifndef QEMU_HW_H
 #define QEMU_HW_H
 
-#include "qemu-common.h"
+#ifdef CONFIG_USER_ONLY
+#error Cannot include hw/hw.h from user emulation
+#endif
 
-#if !defined(CONFIG_USER_ONLY) && !defined(NEED_CPU_H)
 #include "exec/cpu-common.h"
-#endif
-
-#include "exec/ioport.h"
+#include "qom/object.h"
+#include "exec/memory.h"
 #include "hw/irq.h"
-#include "block/aio.h"
 #include "migration/vmstate.h"
-#include "qemu/log.h"
-
-#ifdef NEED_CPU_H
-#if TARGET_LONG_BITS == 64
-#define qemu_put_betl qemu_put_be64
-#define qemu_get_betl qemu_get_be64
-#define qemu_put_betls qemu_put_be64s
-#define qemu_get_betls qemu_get_be64s
-#define qemu_put_sbetl qemu_put_sbe64
-#define qemu_get_sbetl qemu_get_sbe64
-#define qemu_put_sbetls qemu_put_sbe64s
-#define qemu_get_sbetls qemu_get_sbe64s
-#else
-#define qemu_put_betl qemu_put_be32
-#define qemu_get_betl qemu_get_be32
-#define qemu_put_betls qemu_put_be32s
-#define qemu_get_betls qemu_get_be32s
-#define qemu_put_sbetl qemu_put_sbe32
-#define qemu_get_sbetl qemu_get_sbe32
-#define qemu_put_sbetls qemu_put_sbe32s
-#define qemu_get_sbetls qemu_get_sbe32s
-#endif
-#endif
+#include "qemu/module.h"
 
 typedef void QEMUResetHandler(void *opaque);
 
 void qemu_register_reset(QEMUResetHandler *func, void *opaque);
 void qemu_unregister_reset(QEMUResetHandler *func, void *opaque);
 
-/* handler to set the boot_device order for a specific type of QEMUMachine */
-/* return 0 if success */
-typedef int QEMUBootSetHandler(void *opaque, const char *boot_order);
-void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque);
-int qemu_boot_set(const char *boot_order);
-
-#ifdef NEED_CPU_H
-#if TARGET_LONG_BITS == 64
-#define VMSTATE_UINTTL_V(_f, _s, _v)                                  \
-    VMSTATE_UINT64_V(_f, _s, _v)
-#define VMSTATE_UINTTL_EQUAL_V(_f, _s, _v)                            \
-    VMSTATE_UINT64_EQUAL_V(_f, _s, _v)
-#define VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, _v)                        \
-    VMSTATE_UINT64_ARRAY_V(_f, _s, _n, _v)
-#else
-#define VMSTATE_UINTTL_V(_f, _s, _v)                                  \
-    VMSTATE_UINT32_V(_f, _s, _v)
-#define VMSTATE_UINTTL_EQUAL_V(_f, _s, _v)                            \
-    VMSTATE_UINT32_EQUAL_V(_f, _s, _v)
-#define VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, _v)                        \
-    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, _v)
-#endif
-#define VMSTATE_UINTTL(_f, _s)                                        \
-    VMSTATE_UINTTL_V(_f, _s, 0)
-#define VMSTATE_UINTTL_EQUAL(_f, _s)                                  \
-    VMSTATE_UINTTL_EQUAL_V(_f, _s, 0)
-#define VMSTATE_UINTTL_ARRAY(_f, _s, _n)                              \
-    VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, 0)
-
-#endif
+void QEMU_NORETURN hw_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 
 #endif
diff --git a/include/hw/i2c/aspeed_i2c.h b/include/hw/i2c/aspeed_i2c.h
new file mode 100644
index 0000000..f9020ac
--- /dev/null
+++ b/include/hw/i2c/aspeed_i2c.h
@@ -0,0 +1,62 @@
+/*
+ *  ASPEED AST2400 I2C Controller
+ *
+ *  Copyright (C) 2016 IBM Corp.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#ifndef ASPEED_I2C_H
+#define ASPEED_I2C_H
+
+#include "hw/i2c/i2c.h"
+
+#define TYPE_ASPEED_I2C "aspeed.i2c"
+#define ASPEED_I2C(obj) \
+    OBJECT_CHECK(AspeedI2CState, (obj), TYPE_ASPEED_I2C)
+
+#define ASPEED_I2C_NR_BUSSES 14
+
+struct AspeedI2CState;
+
+typedef struct AspeedI2CBus {
+    struct AspeedI2CState *controller;
+
+    MemoryRegion mr;
+
+    I2CBus *bus;
+    uint8_t id;
+
+    uint32_t ctrl;
+    uint32_t timing[2];
+    uint32_t intr_ctrl;
+    uint32_t intr_status;
+    uint32_t cmd;
+    uint32_t buf;
+} AspeedI2CBus;
+
+typedef struct AspeedI2CState {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+    qemu_irq irq;
+
+    uint32_t intr_status;
+
+    AspeedI2CBus busses[ASPEED_I2C_NR_BUSSES];
+} AspeedI2CState;
+
+I2CBus *aspeed_i2c_get_bus(DeviceState *dev, int busnr);
+
+#endif /* ASPEED_I2C_H */
diff --git a/include/hw/i2c/i2c-ddc.h b/include/hw/i2c/i2c-ddc.h
new file mode 100644
index 0000000..d9b5f33
--- /dev/null
+++ b/include/hw/i2c/i2c-ddc.h
@@ -0,0 +1,38 @@
+/* A simple I2C slave for returning monitor EDID data via DDC.
+ *
+ * Copyright (c) 2011 Linaro Limited
+ * Written by Peter Maydell
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License version 2 as
+ *  published by the Free Software Foundation.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef I2C_DDC_H
+#define I2C_DDC_H
+
+/* A simple I2C slave which just returns the contents of its EDID blob. */
+
+struct I2CDDCState {
+    /*< private >*/
+    I2CSlave i2c;
+    /*< public >*/
+    bool firstbyte;
+    uint8_t reg;
+    uint8_t edid_blob[128];
+};
+
+typedef struct I2CDDCState I2CDDCState;
+
+#define TYPE_I2CDDC "i2c-ddc"
+#define I2CDDC(obj) OBJECT_CHECK(I2CDDCState, (obj), TYPE_I2CDDC)
+
+#endif /* I2C_DDC_H */
diff --git a/include/hw/i2c/i2c.h b/include/hw/i2c/i2c.h
index 4986ebc..c4085aa 100644
--- a/include/hw/i2c/i2c.h
+++ b/include/hw/i2c/i2c.h
@@ -56,6 +56,7 @@
 int i2c_start_transfer(I2CBus *bus, uint8_t address, int recv);
 void i2c_end_transfer(I2CBus *bus);
 void i2c_nack(I2CBus *bus);
+int i2c_send_recv(I2CBus *bus, uint8_t *data, bool send);
 int i2c_send(I2CBus *bus, uint8_t data);
 int i2c_recv(I2CBus *bus);
 
diff --git a/include/hw/i2c/imx_i2c.h b/include/hw/i2c/imx_i2c.h
new file mode 100644
index 0000000..7c73a1f
--- /dev/null
+++ b/include/hw/i2c/imx_i2c.h
@@ -0,0 +1,87 @@
+/*
+ *  i.MX I2C Bus Serial Interface registers definition
+ *
+ *  Copyright (C) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef IMX_I2C_H
+#define IMX_I2C_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_I2C "imx.i2c"
+#define IMX_I2C(obj) OBJECT_CHECK(IMXI2CState, (obj), TYPE_IMX_I2C)
+
+#define IMX_I2C_MEM_SIZE           0x14
+
+/* i.MX I2C memory map */
+#define IADR_ADDR                  0x00  /* address register */
+#define IFDR_ADDR                  0x04  /* frequency divider register */
+#define I2CR_ADDR                  0x08  /* control register */
+#define I2SR_ADDR                  0x0c  /* status register */
+#define I2DR_ADDR                  0x10  /* data register */
+
+#define IADR_MASK                  0xFE
+#define IADR_RESET                 0
+
+#define IFDR_MASK                  0x3F
+#define IFDR_RESET                 0
+
+#define I2CR_IEN                   (1 << 7)
+#define I2CR_IIEN                  (1 << 6)
+#define I2CR_MSTA                  (1 << 5)
+#define I2CR_MTX                   (1 << 4)
+#define I2CR_TXAK                  (1 << 3)
+#define I2CR_RSTA                  (1 << 2)
+#define I2CR_MASK                  0xFC
+#define I2CR_RESET                 0
+
+#define I2SR_ICF                   (1 << 7)
+#define I2SR_IAAF                  (1 << 6)
+#define I2SR_IBB                   (1 << 5)
+#define I2SR_IAL                   (1 << 4)
+#define I2SR_SRW                   (1 << 2)
+#define I2SR_IIF                   (1 << 1)
+#define I2SR_RXAK                  (1 << 0)
+#define I2SR_MASK                  0xE9
+#define I2SR_RESET                 0x81
+
+#define I2DR_MASK                  0xFF
+#define I2DR_RESET                 0
+
+#define ADDR_RESET                 0xFF00
+
+typedef struct IMXI2CState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    I2CBus *bus;
+    qemu_irq irq;
+
+    uint16_t  address;
+
+    uint16_t iadr;
+    uint16_t ifdr;
+    uint16_t i2cr;
+    uint16_t i2sr;
+    uint16_t i2dr_read;
+    uint16_t i2dr_write;
+} IMXI2CState;
+
+#endif /* IMX_I2C_H */
diff --git a/include/hw/i2c/pm_smbus.h b/include/hw/i2c/pm_smbus.h
index 926603f..2a837af 100644
--- a/include/hw/i2c/pm_smbus.h
+++ b/include/hw/i2c/pm_smbus.h
@@ -17,4 +17,4 @@
 
 void pm_smbus_init(DeviceState *parent, PMSMBus *smb);
 
-#endif /* !PM_SMBUS_H */
+#endif /* PM_SMBUS_H */
diff --git a/include/hw/i386/apic-msidef.h b/include/hw/i386/apic-msidef.h
index 6e2eb71..8b4d4cc 100644
--- a/include/hw/i386/apic-msidef.h
+++ b/include/hw/i386/apic-msidef.h
@@ -25,6 +25,7 @@
 #define MSI_ADDR_REDIRECTION_SHIFT      3
 
 #define MSI_ADDR_DEST_ID_SHIFT          12
+#define MSI_ADDR_DEST_IDX_SHIFT         4
 #define  MSI_ADDR_DEST_ID_MASK          0x00ffff0
 
 #endif /* HW_APIC_MSIDEF_H */
diff --git a/include/hw/i386/apic.h b/include/hw/i386/apic.h
index 1d48e02..ea48ea9 100644
--- a/include/hw/i386/apic.h
+++ b/include/hw/i386/apic.h
@@ -18,15 +18,10 @@
 uint8_t cpu_get_apic_tpr(DeviceState *s);
 void apic_init_reset(DeviceState *s);
 void apic_sipi(DeviceState *s);
-void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip,
-                                   TPRAccess access);
 void apic_poll_irq(DeviceState *d);
-void apic_designate_bsp(DeviceState *d);
+void apic_designate_bsp(DeviceState *d, bool bsp);
 
 /* pc.c */
 DeviceState *cpu_get_current_apic(void);
 
-/* cpu.c */
-bool cpu_is_bsp(X86CPU *cpu);
-
 #endif
diff --git a/include/hw/i386/apic_internal.h b/include/hw/i386/apic_internal.h
index 83e2a42..06c4e9f 100644
--- a/include/hw/i386/apic_internal.h
+++ b/include/hw/i386/apic_internal.h
@@ -17,11 +17,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
+
 #ifndef QEMU_APIC_INTERNAL_H
 #define QEMU_APIC_INTERNAL_H
 
+#include "cpu.h"
 #include "exec/memory.h"
-#include "hw/cpu/icc_bus.h"
 #include "qemu/timer.h"
 
 /* APIC Local Vector Table */
@@ -49,23 +50,77 @@
 #define APIC_TRIGGER_EDGE               0
 #define APIC_TRIGGER_LEVEL              1
 
-#define APIC_LVT_TIMER_PERIODIC         (1<<17)
-#define APIC_LVT_MASKED                 (1<<16)
-#define APIC_LVT_LEVEL_TRIGGER          (1<<15)
-#define APIC_LVT_REMOTE_IRR             (1<<14)
-#define APIC_INPUT_POLARITY             (1<<13)
-#define APIC_SEND_PENDING               (1<<12)
+#define APIC_VECTOR_MASK                0xff
+#define APIC_DCR_MASK                   0xf
 
-#define ESR_ILLEGAL_ADDRESS (1 << 7)
+#define APIC_LVT_TIMER_SHIFT            17
+#define APIC_LVT_MASKED_SHIFT           16
+#define APIC_LVT_LEVEL_TRIGGER_SHIFT    15
+#define APIC_LVT_REMOTE_IRR_SHIFT       14
+#define APIC_LVT_INT_POLARITY_SHIFT     13
+#define APIC_LVT_DELIV_STS_SHIFT        12
+#define APIC_LVT_DELIV_MOD_SHIFT        8
 
-#define APIC_SV_DIRECTED_IO             (1<<12)
-#define APIC_SV_ENABLE                  (1<<8)
+#define APIC_LVT_TIMER_TSCDEADLINE      (2 << APIC_LVT_TIMER_SHIFT)
+#define APIC_LVT_TIMER_PERIODIC         (1 << APIC_LVT_TIMER_SHIFT)
+#define APIC_LVT_MASKED                 (1 << APIC_LVT_MASKED_SHIFT)
+#define APIC_LVT_LEVEL_TRIGGER          (1 << APIC_LVT_LEVEL_TRIGGER_SHIFT)
+#define APIC_LVT_REMOTE_IRR             (1 << APIC_LVT_REMOTE_IRR_SHIFT)
+#define APIC_LVT_INT_POLARITY           (1 << APIC_LVT_INT_POLARITY_SHIFT)
+#define APIC_LVT_DELIV_STS              (1 << APIC_LVT_DELIV_STS_SHIFT)
+#define APIC_LVT_DELIV_MOD              (7 << APIC_LVT_DELIV_MOD_SHIFT)
+
+#define APIC_ESR_ILL_ADDRESS_SHIFT      7
+#define APIC_ESR_RECV_ILL_VECT_SHIFT    6
+#define APIC_ESR_SEND_ILL_VECT_SHIFT    5
+#define APIC_ESR_RECV_ACCEPT_SHIFT      3
+#define APIC_ESR_SEND_ACCEPT_SHIFT      2
+#define APIC_ESR_RECV_CHECK_SUM_SHIFT   1
+
+#define APIC_ESR_ILLEGAL_ADDRESS        (1 << APIC_ESR_ILL_ADDRESS_SHIFT)
+#define APIC_ESR_RECV_ILLEGAL_VECT      (1 << APIC_ESR_RECV_ILL_VECT_SHIFT)
+#define APIC_ESR_SEND_ILLEGAL_VECT      (1 << APIC_ESR_SEND_ILL_VECT_SHIFT)
+#define APIC_ESR_RECV_ACCEPT            (1 << APIC_ESR_RECV_ACCEPT_SHIFT)
+#define APIC_ESR_SEND_ACCEPT            (1 << APIC_ESR_SEND_ACCEPT_SHIFT)
+#define APIC_ESR_RECV_CHECK_SUM         (1 << APIC_ESR_RECV_CHECK_SUM_SHIFT)
+#define APIC_ESR_SEND_CHECK_SUM         1
+
+#define APIC_ICR_DEST_SHIFT             24
+#define APIC_ICR_DEST_SHORT_SHIFT       18
+#define APIC_ICR_TRIGGER_MOD_SHIFT      15
+#define APIC_ICR_LEVEL_SHIFT            14
+#define APIC_ICR_DELIV_STS_SHIFT        12
+#define APIC_ICR_DEST_MOD_SHIFT         11
+#define APIC_ICR_DELIV_MOD_SHIFT        8
+
+#define APIC_ICR_DEST_SHORT             (3 << APIC_ICR_DEST_SHORT_SHIFT)
+#define APIC_ICR_TRIGGER_MOD            (1 << APIC_ICR_TRIGGER_MOD_SHIFT)
+#define APIC_ICR_LEVEL                  (1 << APIC_ICR_LEVEL_SHIFT)
+#define APIC_ICR_DELIV_STS              (1 << APIC_ICR_DELIV_STS_SHIFT)
+#define APIC_ICR_DEST_MOD               (1 << APIC_ICR_DEST_MOD_SHIFT)
+#define APIC_ICR_DELIV_MOD              (7 << APIC_ICR_DELIV_MOD_SHIFT)
+
+#define APIC_PR_CLASS_SHIFT             4
+#define APIC_PR_SUB_CLASS               0xf
+
+#define APIC_LOGDEST_XAPIC_SHIFT        4
+#define APIC_LOGDEST_XAPIC_ID           0xf
+
+#define APIC_LOGDEST_X2APIC_SHIFT       16
+#define APIC_LOGDEST_X2APIC_ID          0xffff
+
+#define APIC_SPURIO_FOCUS_SHIFT         9
+#define APIC_SPURIO_ENABLED_SHIFT       8
+
+#define APIC_SPURIO_FOCUS               (1 << APIC_SPURIO_FOCUS_SHIFT)
+#define APIC_SPURIO_ENABLED             (1 << APIC_SPURIO_ENABLED_SHIFT)
+
+#define APIC_SV_DIRECTED_IO             (1 << 12)
+#define APIC_SV_ENABLE                  (1 << 8)
 
 #define VAPIC_ENABLE_BIT                0
 #define VAPIC_ENABLE_MASK               (1 << VAPIC_ENABLE_BIT)
 
-#define MAX_APICS 255
-
 typedef struct APICCommonState APICCommonState;
 
 #define TYPE_APIC_COMMON "apic-common"
@@ -78,9 +133,10 @@
 
 typedef struct APICCommonClass
 {
-    ICCDeviceClass parent_class;
+    DeviceClass parent_class;
 
     DeviceRealize realize;
+    DeviceUnrealize unrealize;
     void (*set_base)(APICCommonState *s, uint64_t val);
     void (*set_tpr)(APICCommonState *s, uint8_t val);
     uint8_t (*get_tpr)(APICCommonState *s);
@@ -89,10 +145,13 @@
     void (*external_nmi)(APICCommonState *s);
     void (*pre_save)(APICCommonState *s);
     void (*post_load)(APICCommonState *s);
+    void (*reset)(APICCommonState *s);
 } APICCommonClass;
 
 struct APICCommonState {
-    ICCDevice busdev;
+    /*< private >*/
+    DeviceState parent_obj;
+    /*< public >*/
 
     MemoryRegion io_memory;
     X86CPU *cpu;
@@ -116,7 +175,6 @@
     uint32_t initial_count;
     int64_t initial_count_load_time;
     int64_t next_time;
-    int idx;
     QEMUTimer *timer;
     int64_t timer_expiry;
     int sipi_vector;
@@ -125,6 +183,7 @@
     uint32_t vapic_control;
     DeviceState *vapic;
     hwaddr vapic_paddr; /* note: persistence via kvmvapic */
+    bool legacy_instance_id;
 };
 
 typedef struct VAPICState {
@@ -145,4 +204,22 @@
 void vapic_report_tpr_access(DeviceState *dev, CPUState *cpu, target_ulong ip,
                              TPRAccess access);
 
-#endif /* !QEMU_APIC_INTERNAL_H */
+int apic_get_ppr(APICCommonState *s);
+
+static inline void apic_set_bit(uint32_t *tab, int index)
+{
+    int i, mask;
+    i = index >> 5;
+    mask = 1 << (index & 0x1f);
+    tab[i] |= mask;
+}
+
+static inline int apic_get_bit(uint32_t *tab, int index)
+{
+    int i, mask;
+    i = index >> 5;
+    mask = 1 << (index & 0x1f);
+    return !!(tab[i] & mask);
+}
+
+#endif /* QEMU_APIC_INTERNAL_H */
diff --git a/include/hw/i386/ich9.h b/include/hw/i386/ich9.h
index 59ea25b..5fd7e97 100644
--- a/include/hw/i386/ich9.h
+++ b/include/hw/i386/ich9.h
@@ -2,7 +2,6 @@
 #define HW_ICH9_H
 
 #include "hw/hw.h"
-#include "qemu/range.h"
 #include "hw/isa/isa.h"
 #include "hw/sysbus.h"
 #include "hw/i386/pc.h"
@@ -18,11 +17,13 @@
 void ich9_lpc_set_irq(void *opaque, int irq_num, int level);
 int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx);
 PCIINTxRoute ich9_route_intx_pin_to_irq(void *opaque, int pirq_pin);
-void ich9_lpc_pm_init(PCIDevice *pci_lpc);
-PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus);
+void ich9_lpc_pm_init(PCIDevice *pci_lpc, bool smm_enabled);
 I2CBus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base);
 
-#define ICH9_CC_SIZE                            (16 * 1024)     /* 16KB */
+void ich9_generate_smi(void);
+void ich9_generate_nmi(void);
+
+#define ICH9_CC_SIZE (16 * 1024) /* 16KB. Chipset configuration registers */
 
 #define TYPE_ICH9_LPC_DEVICE "ICH9-LPC"
 #define ICH9_LPC_DEVICE(obj) \
@@ -34,7 +35,7 @@
 
     /* (pci device, intx) -> pirq
      * In real chipset case, the unused slots are never used
-     * as ICH9 supports only D25-D32 irq routing.
+     * as ICH9 supports only D25-D31 irq routing.
      * On the other hand in qemu case, any slot/function can be populated
      * via command line option.
      * So fallback interrupt routing for any devices in any slots is necessary.
@@ -44,6 +45,12 @@
     APMState apm;
     ICH9LPCPMRegs pm;
     uint32_t sci_level; /* track sci level */
+    uint8_t sci_gsi;
+
+    /* 2.24 Pin Straps */
+    struct {
+        bool spkr_hi;
+    } pin_strap;
 
     /* 10.1 Chipset Configuration registers(Memory Space)
      which is pointed by RCBA */
@@ -59,11 +66,10 @@
 
     /* isa bus */
     ISABus *isa_bus;
-    MemoryRegion rbca_mem;
+    MemoryRegion rcrb_mem; /* root complex register block */
     Notifier machine_ready;
 
-    qemu_irq *pic;
-    qemu_irq *ioapic;
+    qemu_irq gsi[GSI_NUM_PINS];
 } ICH9LPCState;
 
 Object *ich9_lpc_find(void);
@@ -92,6 +98,9 @@
 #define ICH9_CC_DIR_MASK                        0x7
 #define ICH9_CC_OIC                             0x31FF
 #define ICH9_CC_OIC_AEN                         0x1
+#define ICH9_CC_GCS                             0x3410
+#define ICH9_CC_GCS_DEFAULT                     0x00000020
+#define ICH9_CC_GCS_NO_REBOOT                   (1 << 5)
 
 /* D28:F[0-5] */
 #define ICH9_PCIE_DEV                           28
@@ -154,6 +163,12 @@
 #define ICH9_LPC_PIRQ_ROUT_MASK                 Q35_MASK(8, 3, 0)
 #define ICH9_LPC_PIRQ_ROUT_DEFAULT              0x80
 
+#define ICH9_LPC_GEN_PMCON_1                    0xa0
+#define ICH9_LPC_GEN_PMCON_1_SMI_LOCK           (1 << 4)
+#define ICH9_LPC_GEN_PMCON_2                    0xa2
+#define ICH9_LPC_GEN_PMCON_3                    0xa4
+#define ICH9_LPC_GEN_PMCON_LOCK                 0xa6
+
 #define ICH9_LPC_RCBA                           0xf0
 #define ICH9_LPC_RCBA_BA_MASK                   Q35_MASK(32, 31, 14)
 #define ICH9_LPC_RCBA_EN                        0x1
@@ -162,11 +177,13 @@
 #define ICH9_LPC_PIC_NUM_PINS                   16
 #define ICH9_LPC_IOAPIC_NUM_PINS                24
 
+#define ICH9_GPIO_GSI "gsi"
+
 /* D31:F2 SATA Controller #1 */
 #define ICH9_SATA1_DEV                          31
 #define ICH9_SATA1_FUNC                         2
 
-/* D30:F1 power management I/O registers
+/* D31:F0 power management I/O registers
    offset from the address ICH9_LPC_PMBASE */
 
 /* ICH9 LPC PM I/O registers are 128 ports and 128-aligned */
@@ -182,7 +199,10 @@
 #define ICH9_PMIO_GPE0_LEN                      16
 #define ICH9_PMIO_SMI_EN                        0x30
 #define ICH9_PMIO_SMI_EN_APMC_EN                (1 << 5)
+#define ICH9_PMIO_SMI_EN_TCO_EN                 (1 << 13)
 #define ICH9_PMIO_SMI_STS                       0x34
+#define ICH9_PMIO_TCO_RLD                       0x60
+#define ICH9_PMIO_TCO_LEN                       32
 
 /* FADT ACPI_ENABLE/ACPI_DISABLE */
 #define ICH9_APM_ACPI_ENABLE                    0x2
@@ -190,6 +210,8 @@
 
 
 /* D31:F3 SMBus controller */
+#define TYPE_ICH9_SMB_DEVICE "ICH9 SMB"
+
 #define ICH9_A2_SMB_REVISION                    0x02
 #define ICH9_SMB_PI                             0x00
 
diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h
index e321ee4..a42dbd7 100644
--- a/include/hw/i386/intel_iommu.h
+++ b/include/hw/i386/intel_iommu.h
@@ -23,6 +23,10 @@
 #define INTEL_IOMMU_H
 #include "hw/qdev.h"
 #include "sysemu/dma.h"
+#include "hw/i386/x86-iommu.h"
+#include "hw/i386/ioapic.h"
+#include "hw/pci/msi.h"
+#include "hw/sysbus.h"
 
 #define TYPE_INTEL_IOMMU_DEVICE "intel-iommu"
 #define INTEL_IOMMU_DEVICE(obj) \
@@ -34,7 +38,6 @@
 #define VTD_PCI_BUS_MAX             256
 #define VTD_PCI_SLOT_MAX            32
 #define VTD_PCI_FUNC_MAX            8
-#define VTD_PCI_DEVFN_MAX           256
 #define VTD_PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)
 #define VTD_PCI_FUNC(devfn)         ((devfn) & 0x07)
 #define VTD_SID_TO_BUS(sid)         (((sid) >> 8) & 0xff)
@@ -44,11 +47,22 @@
 #define VTD_HOST_ADDRESS_WIDTH      39
 #define VTD_HAW_MASK                ((1ULL << VTD_HOST_ADDRESS_WIDTH) - 1)
 
+#define DMAR_REPORT_F_INTR          (1)
+
+#define  VTD_MSI_ADDR_HI_MASK        (0xffffffff00000000ULL)
+#define  VTD_MSI_ADDR_HI_SHIFT       (32)
+#define  VTD_MSI_ADDR_LO_MASK        (0x00000000ffffffffULL)
+
 typedef struct VTDContextEntry VTDContextEntry;
 typedef struct VTDContextCacheEntry VTDContextCacheEntry;
 typedef struct IntelIOMMUState IntelIOMMUState;
 typedef struct VTDAddressSpace VTDAddressSpace;
 typedef struct VTDIOTLBEntry VTDIOTLBEntry;
+typedef struct VTDBus VTDBus;
+typedef union VTD_IR_TableEntry VTD_IR_TableEntry;
+typedef union VTD_IR_MSIAddress VTD_IR_MSIAddress;
+typedef struct VTDIrq VTDIrq;
+typedef struct VTD_MSIMessage VTD_MSIMessage;
 
 /* Context-Entry */
 struct VTDContextEntry {
@@ -65,25 +79,178 @@
 };
 
 struct VTDAddressSpace {
-    uint8_t bus_num;
+    PCIBus *bus;
     uint8_t devfn;
     AddressSpace as;
     MemoryRegion iommu;
+    MemoryRegion iommu_ir;      /* Interrupt region: 0xfeeXXXXX */
     IntelIOMMUState *iommu_state;
     VTDContextCacheEntry context_cache_entry;
 };
 
+struct VTDBus {
+    PCIBus* bus;		/* A reference to the bus to provide translation for */
+    VTDAddressSpace *dev_as[0];	/* A table of VTDAddressSpace objects indexed by devfn */
+};
+
 struct VTDIOTLBEntry {
     uint64_t gfn;
     uint16_t domain_id;
     uint64_t slpte;
+    uint64_t mask;
     bool read_flags;
     bool write_flags;
 };
 
+/* VT-d Source-ID Qualifier types */
+enum {
+    VTD_SQ_FULL = 0x00,     /* Full SID verification */
+    VTD_SQ_IGN_3 = 0x01,    /* Ignore bit 3 */
+    VTD_SQ_IGN_2_3 = 0x02,  /* Ignore bits 2 & 3 */
+    VTD_SQ_IGN_1_3 = 0x03,  /* Ignore bits 1-3 */
+    VTD_SQ_MAX,
+};
+
+/* VT-d Source Validation Types */
+enum {
+    VTD_SVT_NONE = 0x00,    /* No validation */
+    VTD_SVT_ALL = 0x01,     /* Do full validation */
+    VTD_SVT_BUS = 0x02,     /* Validate bus range */
+    VTD_SVT_MAX,
+};
+
+/* Interrupt Remapping Table Entry Definition */
+union VTD_IR_TableEntry {
+    struct {
+#ifdef HOST_WORDS_BIGENDIAN
+        uint32_t dest_id:32;         /* Destination ID */
+        uint32_t __reserved_1:8;     /* Reserved 1 */
+        uint32_t vector:8;           /* Interrupt Vector */
+        uint32_t irte_mode:1;        /* IRTE Mode */
+        uint32_t __reserved_0:3;     /* Reserved 0 */
+        uint32_t __avail:4;          /* Available spaces for software */
+        uint32_t delivery_mode:3;    /* Delivery Mode */
+        uint32_t trigger_mode:1;     /* Trigger Mode */
+        uint32_t redir_hint:1;       /* Redirection Hint */
+        uint32_t dest_mode:1;        /* Destination Mode */
+        uint32_t fault_disable:1;    /* Fault Processing Disable */
+        uint32_t present:1;          /* Whether entry present/available */
+#else
+        uint32_t present:1;          /* Whether entry present/available */
+        uint32_t fault_disable:1;    /* Fault Processing Disable */
+        uint32_t dest_mode:1;        /* Destination Mode */
+        uint32_t redir_hint:1;       /* Redirection Hint */
+        uint32_t trigger_mode:1;     /* Trigger Mode */
+        uint32_t delivery_mode:3;    /* Delivery Mode */
+        uint32_t __avail:4;          /* Available spaces for software */
+        uint32_t __reserved_0:3;     /* Reserved 0 */
+        uint32_t irte_mode:1;        /* IRTE Mode */
+        uint32_t vector:8;           /* Interrupt Vector */
+        uint32_t __reserved_1:8;     /* Reserved 1 */
+        uint32_t dest_id:32;         /* Destination ID */
+#endif
+        uint16_t source_id:16;       /* Source-ID */
+#ifdef HOST_WORDS_BIGENDIAN
+        uint64_t __reserved_2:44;    /* Reserved 2 */
+        uint64_t sid_vtype:2;        /* Source-ID Validation Type */
+        uint64_t sid_q:2;            /* Source-ID Qualifier */
+#else
+        uint64_t sid_q:2;            /* Source-ID Qualifier */
+        uint64_t sid_vtype:2;        /* Source-ID Validation Type */
+        uint64_t __reserved_2:44;    /* Reserved 2 */
+#endif
+    } QEMU_PACKED irte;
+    uint64_t data[2];
+};
+
+#define VTD_IR_INT_FORMAT_COMPAT     (0) /* Compatible Interrupt */
+#define VTD_IR_INT_FORMAT_REMAP      (1) /* Remappable Interrupt */
+
+/* Programming format for MSI/MSI-X addresses */
+union VTD_IR_MSIAddress {
+    struct {
+#ifdef HOST_WORDS_BIGENDIAN
+        uint32_t __head:12;          /* Should always be: 0x0fee */
+        uint32_t index_l:15;         /* Interrupt index bit 14-0 */
+        uint32_t int_mode:1;         /* Interrupt format */
+        uint32_t sub_valid:1;        /* SHV: Sub-Handle Valid bit */
+        uint32_t index_h:1;          /* Interrupt index bit 15 */
+        uint32_t __not_care:2;
+#else
+        uint32_t __not_care:2;
+        uint32_t index_h:1;          /* Interrupt index bit 15 */
+        uint32_t sub_valid:1;        /* SHV: Sub-Handle Valid bit */
+        uint32_t int_mode:1;         /* Interrupt format */
+        uint32_t index_l:15;         /* Interrupt index bit 14-0 */
+        uint32_t __head:12;          /* Should always be: 0x0fee */
+#endif
+    } QEMU_PACKED addr;
+    uint32_t data;
+};
+
+/* Generic IRQ entry information */
+struct VTDIrq {
+    /* Used by both IOAPIC/MSI interrupt remapping */
+    uint8_t trigger_mode;
+    uint8_t vector;
+    uint8_t delivery_mode;
+    uint32_t dest;
+    uint8_t dest_mode;
+
+    /* only used by MSI interrupt remapping */
+    uint8_t redir_hint;
+    uint8_t msi_addr_last_bits;
+};
+
+struct VTD_MSIMessage {
+    union {
+        struct {
+#ifdef HOST_WORDS_BIGENDIAN
+            uint32_t __addr_head:12; /* 0xfee */
+            uint32_t dest:8;
+            uint32_t __reserved:8;
+            uint32_t redir_hint:1;
+            uint32_t dest_mode:1;
+            uint32_t __not_used:2;
+#else
+            uint32_t __not_used:2;
+            uint32_t dest_mode:1;
+            uint32_t redir_hint:1;
+            uint32_t __reserved:8;
+            uint32_t dest:8;
+            uint32_t __addr_head:12; /* 0xfee */
+#endif
+            uint32_t __addr_hi:32;
+        } QEMU_PACKED;
+        uint64_t msi_addr;
+    };
+    union {
+        struct {
+#ifdef HOST_WORDS_BIGENDIAN
+            uint16_t trigger_mode:1;
+            uint16_t level:1;
+            uint16_t __resved:3;
+            uint16_t delivery_mode:3;
+            uint16_t vector:8;
+#else
+            uint16_t vector:8;
+            uint16_t delivery_mode:3;
+            uint16_t __resved:3;
+            uint16_t level:1;
+            uint16_t trigger_mode:1;
+#endif
+            uint16_t __resved1:16;
+        } QEMU_PACKED;
+        uint32_t msi_data;
+    };
+};
+
+/* When IR is enabled, all MSI/MSI-X data bits should be zero */
+#define VTD_IR_MSI_DATA          (0)
+
 /* The iommu (DMAR) device state struct */
 struct IntelIOMMUState {
-    SysBusDevice busdev;
+    X86IOMMUState x86_iommu;
     MemoryRegion csrmem;
     uint8_t csr[DMAR_REG_SIZE];     /* register values */
     uint8_t wmask[DMAR_REG_SIZE];   /* R/W bytes */
@@ -114,7 +281,19 @@
     GHashTable *iotlb;              /* IOTLB */
 
     MemoryRegionIOMMUOps iommu_ops;
-    VTDAddressSpace **address_spaces[VTD_PCI_BUS_MAX];
+    GHashTable *vtd_as_by_busptr;   /* VTDBus objects indexed by PCIBus* reference */
+    VTDBus *vtd_as_by_bus_num[VTD_PCI_BUS_MAX]; /* VTDBus objects indexed by bus number */
+
+    /* interrupt remapping */
+    bool intr_enabled;              /* Whether guest enabled IR */
+    dma_addr_t intr_root;           /* Interrupt remapping table pointer */
+    uint32_t intr_size;             /* Number of IR table entries */
+    bool intr_eime;                 /* Extended interrupt mode enabled */
 };
 
+/* Find the VTD Address space associated with the given bus pointer,
+ * create a new one if none exists
+ */
+VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn);
+
 #endif
diff --git a/include/hw/i386/ioapic.h b/include/hw/i386/ioapic.h
index 6245388..9c8816f 100644
--- a/include/hw/i386/ioapic.h
+++ b/include/hw/i386/ioapic.h
@@ -25,4 +25,4 @@
 
 void ioapic_eoi_broadcast(int vector);
 
-#endif /* !HW_IOAPIC_H */
+#endif /* HW_IOAPIC_H */
diff --git a/include/hw/i386/ioapic_internal.h b/include/hw/i386/ioapic_internal.h
index 3be3352..a11d86d 100644
--- a/include/hw/i386/ioapic_internal.h
+++ b/include/hw/i386/ioapic_internal.h
@@ -25,12 +25,12 @@
 #include "hw/hw.h"
 #include "exec/memory.h"
 #include "hw/sysbus.h"
+#include "qemu/notify.h"
 
 #define MAX_IOAPICS                     1
 
-#define IOAPIC_VERSION                  0x11
-
 #define IOAPIC_LVT_DEST_SHIFT           56
+#define IOAPIC_LVT_DEST_IDX_SHIFT       48
 #define IOAPIC_LVT_MASKED_SHIFT         16
 #define IOAPIC_LVT_TRIGGER_MODE_SHIFT   15
 #define IOAPIC_LVT_REMOTE_IRR_SHIFT     14
@@ -40,7 +40,17 @@
 #define IOAPIC_LVT_DELIV_MODE_SHIFT     8
 
 #define IOAPIC_LVT_MASKED               (1 << IOAPIC_LVT_MASKED_SHIFT)
+#define IOAPIC_LVT_TRIGGER_MODE         (1 << IOAPIC_LVT_TRIGGER_MODE_SHIFT)
 #define IOAPIC_LVT_REMOTE_IRR           (1 << IOAPIC_LVT_REMOTE_IRR_SHIFT)
+#define IOAPIC_LVT_POLARITY             (1 << IOAPIC_LVT_POLARITY_SHIFT)
+#define IOAPIC_LVT_DELIV_STATUS         (1 << IOAPIC_LVT_DELIV_STATUS_SHIFT)
+#define IOAPIC_LVT_DEST_MODE            (1 << IOAPIC_LVT_DEST_MODE_SHIFT)
+#define IOAPIC_LVT_DELIV_MODE           (7 << IOAPIC_LVT_DELIV_MODE_SHIFT)
+
+/* Bits that are read-only for IOAPIC entry */
+#define IOAPIC_RO_BITS                  (IOAPIC_LVT_REMOTE_IRR | \
+                                         IOAPIC_LVT_DELIV_STATUS)
+#define IOAPIC_RW_BITS                  (~(uint64_t)IOAPIC_RO_BITS)
 
 #define IOAPIC_TRIGGER_EDGE             0
 #define IOAPIC_TRIGGER_LEVEL            1
@@ -59,6 +69,7 @@
 
 #define IOAPIC_IOREGSEL                 0x00
 #define IOAPIC_IOWIN                    0x10
+#define IOAPIC_EOI                      0x40
 
 #define IOAPIC_REG_ID                   0x00
 #define IOAPIC_REG_VER                  0x01
@@ -96,8 +107,12 @@
     uint8_t ioregsel;
     uint32_t irr;
     uint64_t ioredtbl[IOAPIC_NUM_PINS];
+    Notifier machine_done;
+    uint8_t version;
 };
 
 void ioapic_reset_common(DeviceState *dev);
 
-#endif /* !QEMU_IOAPIC_INTERNAL_H */
+void ioapic_print_redtbl(Monitor *mon, IOAPICCommonState *s);
+
+#endif /* QEMU_IOAPIC_INTERNAL_H */
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index 69d9cf8..74c175c 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -15,55 +15,134 @@
 #include "hw/pci/pci.h"
 #include "hw/boards.h"
 #include "hw/compat.h"
+#include "hw/mem/pc-dimm.h"
+#include "hw/mem/nvdimm.h"
+#include "hw/acpi/acpi_dev_interface.h"
 
 #define HPET_INTCAP "hpet-intcap"
 
+#ifdef CONFIG_KVM
+#define kvm_pit_in_kernel() \
+    (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_pic_in_kernel()  \
+    (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#define kvm_ioapic_in_kernel() \
+    (kvm_irqchip_in_kernel() && !kvm_irqchip_is_split())
+#else
+#define kvm_pit_in_kernel()      0
+#define kvm_pic_in_kernel()      0
+#define kvm_ioapic_in_kernel()   0
+#endif
+
 /**
  * PCMachineState:
- * @hotplug_memory_base: address in guest RAM address space where hotplug memory
- * address space begins.
- * @hotplug_memory: hotplug memory addess space container
  * @acpi_dev: link to ACPI PM device that performs ACPI hotplug handling
- * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
- *                        backend's alignment value if provided
  */
 struct PCMachineState {
     /*< private >*/
     MachineState parent_obj;
 
     /* <public> */
-    ram_addr_t hotplug_memory_base;
-    MemoryRegion hotplug_memory;
 
+    /* State for other subsystems/APIs: */
+    MemoryHotplugState hotplug_memory;
+    Notifier machine_done;
+
+    /* Pointers to devices and objects: */
     HotplugHandler *acpi_dev;
     ISADevice *rtc;
+    PCIBus *bus;
+    FWCfgState *fw_cfg;
 
+    /* Configuration options: */
     uint64_t max_ram_below_4g;
     OnOffAuto vmport;
-    bool enforce_aligned_dimm;
+    OnOffAuto smm;
+
+    AcpiNVDIMMState acpi_nvdimm_state;
+
+    /* RAM information (sizes, addresses, configuration): */
+    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+
+    /* CPU and apic information: */
+    bool apic_xrupt_override;
+    unsigned apic_id_limit;
+    CPUArchIdList *possible_cpus;
+
+    /* NUMA information: */
+    uint64_t numa_nodes;
+    uint64_t *node_mem;
+
+    /* Address space used by IOAPIC device. All IOAPIC interrupts
+     * will be translated to MSI messages in the address space. */
+    AddressSpace *ioapic_as;
 };
 
 #define PC_MACHINE_ACPI_DEVICE_PROP "acpi-device"
 #define PC_MACHINE_MEMHP_REGION_SIZE "hotplug-memory-region-size"
 #define PC_MACHINE_MAX_RAM_BELOW_4G "max-ram-below-4g"
 #define PC_MACHINE_VMPORT           "vmport"
-#define PC_MACHINE_ENFORCE_ALIGNED_DIMM "enforce-aligned-dimm"
+#define PC_MACHINE_SMM              "smm"
+#define PC_MACHINE_NVDIMM           "nvdimm"
 
 /**
  * PCMachineClass:
+ *
+ * Methods:
+ *
  * @get_hotplug_handler: pointer to parent class callback @get_hotplug_handler
+ *
+ * Compat fields:
+ *
+ * @enforce_aligned_dimm: check that DIMM's address/size is aligned by
+ *                        backend's alignment value if provided
+ * @acpi_data_size: Size of the chunk of memory at the top of RAM
+ *                  for the BIOS ACPI tables and other BIOS
+ *                  datastructures.
+ * @gigabyte_align: Make sure that guest addresses aligned at
+ *                  1Gbyte boundaries get mapped to host
+ *                  addresses aligned at 1Gbyte boundaries. This
+ *                  way we can use 1GByte pages in the host.
+ *
  */
 struct PCMachineClass {
     /*< private >*/
     MachineClass parent_class;
 
     /*< public >*/
+
+    /* Methods: */
     HotplugHandler *(*get_hotplug_handler)(MachineState *machine,
                                            DeviceState *dev);
-};
 
-typedef struct PCMachineState PCMachineState;
-typedef struct PCMachineClass PCMachineClass;
+    /* Device configuration: */
+    bool pci_enabled;
+    bool kvmclock_enabled;
+
+    /* Compat options: */
+
+    /* ACPI compat: */
+    bool has_acpi_build;
+    bool rsdp_in_ram;
+    int legacy_acpi_table_size;
+    unsigned acpi_data_size;
+
+    /* SMBIOS compat: */
+    bool smbios_defaults;
+    bool smbios_legacy_mode;
+    bool smbios_uuid_encoded;
+
+    /* RAM / address space compat: */
+    bool gigabyte_align;
+    bool has_reserved_memory;
+    bool enforce_aligned_dimm;
+    bool broken_reserved_end;
+
+    /* TSC rate migration: */
+    bool save_tsc_khz;
+    /* generate legacy CPU hotplug AML */
+    bool legacy_cpu_hotplug;
+};
 
 #define TYPE_PC_MACHINE "generic-pc-machine"
 #define PC_MACHINE(obj) \
@@ -73,15 +152,8 @@
 #define PC_MACHINE_CLASS(klass) \
     OBJECT_CLASS_CHECK(PCMachineClass, (klass), TYPE_PC_MACHINE)
 
-void qemu_register_pc_machine(QEMUMachine *m);
-
 /* PC-style peripherals (also used by other machines).  */
 
-typedef struct PcPciInfo {
-    Range w32;
-    Range w64;
-} PcPciInfo;
-
 #define ACPI_PM_PROP_S3_DISABLED "disable_s3"
 #define ACPI_PM_PROP_S4_DISABLED "disable_s4"
 #define ACPI_PM_PROP_S4_VAL "s4_val"
@@ -91,39 +163,11 @@
 #define ACPI_PM_PROP_PM_IO_BASE "pm_io_base"
 #define ACPI_PM_PROP_GPE0_BLK "gpe0_blk"
 #define ACPI_PM_PROP_GPE0_BLK_LEN "gpe0_blk_len"
-
-struct PcGuestInfo {
-    bool isapc_ram_fw;
-    hwaddr ram_size, ram_size_below_4g;
-    unsigned apic_id_limit;
-    bool apic_xrupt_override;
-    uint64_t numa_nodes;
-    uint64_t *node_mem;
-    uint64_t *node_cpu;
-    FWCfgState *fw_cfg;
-    int legacy_acpi_table_size;
-    bool has_acpi_build;
-    bool has_reserved_memory;
-};
+#define ACPI_PM_PROP_TCO_ENABLED "enable_tco"
 
 /* parallel.c */
-static inline bool parallel_init(ISABus *bus, int index, CharDriverState *chr)
-{
-    DeviceState *dev;
-    ISADevice *isadev;
 
-    isadev = isa_try_create(bus, "isa-parallel");
-    if (!isadev) {
-        return false;
-    }
-    dev = DEVICE(isadev);
-    qdev_prop_set_uint32(dev, "index", index);
-    qdev_prop_set_chr(dev, "chardev", chr);
-    if (qdev_init(dev) < 0) {
-        return false;
-    }
-    return true;
-}
+void parallel_hds_isa_init(ISABus *bus, int n);
 
 bool parallel_mm_init(MemoryRegion *address_space,
                       hwaddr base, int it_shift, qemu_irq irq,
@@ -136,8 +180,13 @@
 qemu_irq *kvm_i8259_init(ISABus *bus);
 int pic_read_irq(DeviceState *d);
 int pic_get_output(DeviceState *d);
-void pic_info(Monitor *mon, const QDict *qdict);
-void irq_info(Monitor *mon, const QDict *qdict);
+void hmp_info_pic(Monitor *mon, const QDict *qdict);
+void hmp_info_irq(Monitor *mon, const QDict *qdict);
+
+/* ioapic.c */
+
+void kvm_ioapic_dump_state(Monitor *mon, const QDict *qdict);
+void ioapic_dump_state(Monitor *mon, const QDict *qdict);
 
 /* Global System Interrupts */
 
@@ -151,11 +200,12 @@
 void gsi_handler(void *opaque, int n, int level);
 
 /* vmport.c */
+#define TYPE_VMPORT "vmport"
 typedef uint32_t (VMPortReadFunc)(void *opaque, uint32_t address);
 
 static inline void vmport_init(ISABus *bus)
 {
-    isa_create_simple(bus, "vmport");
+    isa_create_simple(bus, TYPE_VMPORT);
 }
 
 void vmport_register(unsigned char command, VMPortReadFunc *func, void *opaque);
@@ -163,6 +213,7 @@
 void vmmouse_set_data(const uint32_t *data);
 
 /* pckbd.c */
+#define I8042_A20_LINE "a20"
 
 void i8042_init(qemu_irq kbd_irq, qemu_irq mouse_irq, uint32_t io_base);
 void i8042_mm_init(qemu_irq kbd_irq, qemu_irq mouse_irq,
@@ -174,67 +225,64 @@
 /* pc.c */
 extern int fd_bootchk;
 
+bool pc_machine_is_smm_enabled(PCMachineState *pcms);
 void pc_register_ferr_irq(qemu_irq irq);
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level);
 
-void pc_cpus_init(const char *cpu_model, DeviceState *icc_bridge);
+void pc_cpus_init(PCMachineState *pcms);
 void pc_hot_add_cpu(const int64_t id, Error **errp);
 void pc_acpi_init(const char *default_dsdt);
 
-PcGuestInfo *pc_guest_info_init(ram_addr_t below_4g_mem_size,
-                                ram_addr_t above_4g_mem_size);
-
-void pc_set_legacy_acpi_data_size(void);
+void pc_guest_info_init(PCMachineState *pcms);
 
 #define PCI_HOST_PROP_PCI_HOLE_START   "pci-hole-start"
 #define PCI_HOST_PROP_PCI_HOLE_END     "pci-hole-end"
 #define PCI_HOST_PROP_PCI_HOLE64_START "pci-hole64-start"
 #define PCI_HOST_PROP_PCI_HOLE64_END   "pci-hole64-end"
 #define PCI_HOST_PROP_PCI_HOLE64_SIZE  "pci-hole64-size"
+#define PCI_HOST_BELOW_4G_MEM_SIZE     "below-4g-mem-size"
+#define PCI_HOST_ABOVE_4G_MEM_SIZE     "above-4g-mem-size"
 #define DEFAULT_PCI_HOLE64_SIZE (~0x0ULL)
 
 
 void pc_pci_as_mapping_init(Object *owner, MemoryRegion *system_memory,
                             MemoryRegion *pci_address_space);
 
-FWCfgState *xen_load_linux(const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           ram_addr_t below_4g_mem_size,
-                           PcGuestInfo *guest_info);
-FWCfgState *pc_memory_init(MachineState *machine,
-                           MemoryRegion *system_memory,
-                           ram_addr_t below_4g_mem_size,
-                           ram_addr_t above_4g_mem_size,
-                           MemoryRegion *rom_memory,
-                           MemoryRegion **ram_memory,
-                           PcGuestInfo *guest_info);
-qemu_irq *pc_allocate_cpu_irq(void);
+void xen_load_linux(PCMachineState *pcms);
+void pc_memory_init(PCMachineState *pcms,
+                    MemoryRegion *system_memory,
+                    MemoryRegion *rom_memory,
+                    MemoryRegion **ram_memory);
+qemu_irq pc_allocate_cpu_irq(void);
 DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus);
 void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
                           ISADevice **rtc_state,
-                          ISADevice **floppy,
+                          bool create_fdctrl,
                           bool no_vmport,
-                          uint32 hpet_irqs);
+                          uint32_t hpet_irqs);
 void pc_init_ne2k_isa(ISABus *bus, NICInfo *nd);
-void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
-                  const char *boot_device, MachineState *machine,
-                  ISADevice *floppy, BusState *ide0, BusState *ide1,
+void pc_cmos_init(PCMachineState *pcms,
+                  BusState *ide0, BusState *ide1,
                   ISADevice *s);
 void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus);
 void pc_pci_device_init(PCIBus *pci_bus);
 
 typedef void (*cpu_set_smm_t)(int smm, void *arg);
-void cpu_smm_register(cpu_set_smm_t callback, void *arg);
 
 void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name);
 
+ISADevice *pc_find_fdc0(void);
+int cmos_get_fd_drive_type(FloppyDriveType fd0);
+
+#define FW_CFG_IO_BASE     0x510
+
+#define PORT92_A20_LINE "a20"
+
 /* acpi_piix.c */
 
 I2CBus *piix4_pm_init(PCIBus *bus, int devfn, uint32_t smb_io_base,
                       qemu_irq sci_irq, qemu_irq smi_irq,
-                      int kvm_enabled, FWCfgState *fw_cfg,
-                      DeviceState **piix4_pm);
+                      int smm_enabled, DeviceState **piix4_pm);
 void piix4_smbus_register_device(SMBusDevice *dev, uint8_t addr);
 
 /* hpet.c */
@@ -244,7 +292,13 @@
 struct PCII440FXState;
 typedef struct PCII440FXState PCII440FXState;
 
-PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix_devfn,
+#define TYPE_I440FX_PCI_HOST_BRIDGE "i440FX-pcihost"
+#define TYPE_I440FX_PCI_DEVICE "i440FX"
+
+#define TYPE_IGD_PASSTHROUGH_I440FX_PCI_DEVICE "igd-passthrough-i440FX"
+
+PCIBus *i440fx_init(const char *host_type, const char *pci_type,
+                    PCII440FXState **pi440fx_state, int *piix_devfn,
                     ISABus **isa_bus, qemu_irq *pic,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
@@ -298,6 +352,10 @@
 /* pvpanic.c */
 uint16_t pvpanic_port(void);
 
+/* acpi-build.c */
+void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid,
+                       CPUArchIdList *apic_ids, GArray *entry);
+
 /* e820 types */
 #define E820_RAM        1
 #define E820_RESERVED   2
@@ -309,223 +367,555 @@
 int e820_get_num_entries(void);
 bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
 
+#define PC_COMPAT_2_6 \
+    HW_COMPAT_2_6 \
+    {\
+        .driver   = "fw_cfg_io",\
+        .property = "dma_enabled",\
+        .value    = "off",\
+    },{\
+        .driver   = TYPE_X86_CPU,\
+        .property = "cpuid-0xb",\
+        .value    = "off",\
+    },{\
+        .driver   = "vmxnet3",\
+        .property = "romfile",\
+        .value    = "",\
+    },\
+    {\
+        .driver = TYPE_X86_CPU,\
+        .property = "fill-mtrr-mask",\
+        .value = "off",\
+    },\
+    {\
+        .driver   = "apic-common",\
+        .property = "legacy-instance-id",\
+        .value    = "on",\
+    },
+
+#define PC_COMPAT_2_5 \
+    PC_COMPAT_2_6 \
+    HW_COMPAT_2_5
+
+/* Helper for setting model-id for CPU models that changed model-id
+ * depending on QEMU versions up to QEMU 2.4.
+ */
+#define PC_CPU_MODEL_IDS(v) \
+    {\
+        .driver   = "qemu32-" TYPE_X86_CPU,\
+        .property = "model-id",\
+        .value    = "QEMU Virtual CPU version " v,\
+    },\
+    {\
+        .driver   = "qemu64-" TYPE_X86_CPU,\
+        .property = "model-id",\
+        .value    = "QEMU Virtual CPU version " v,\
+    },\
+    {\
+        .driver   = "athlon-" TYPE_X86_CPU,\
+        .property = "model-id",\
+        .value    = "QEMU Virtual CPU version " v,\
+    },
+
+#define PC_COMPAT_2_4 \
+    HW_COMPAT_2_4 \
+    PC_CPU_MODEL_IDS("2.4.0") \
+    {\
+        .driver   = "Haswell-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "Haswell-noTSX-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "Broadwell-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "Broadwell-noTSX-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "host" "-" TYPE_X86_CPU,\
+        .property = "host-cache-info",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = TYPE_X86_CPU,\
+        .property = "check",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "sse4a",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "abm",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "popcnt",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "qemu32" "-" TYPE_X86_CPU,\
+        .property = "popcnt",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G2" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G3" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G4" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },{\
+        .driver   = "Opteron_G5" "-" TYPE_X86_CPU,\
+        .property = "rdtscp",\
+        .value    = "on",\
+    },
+
+
+#define PC_COMPAT_2_3 \
+    HW_COMPAT_2_3 \
+    PC_CPU_MODEL_IDS("2.3.0") \
+    {\
+        .driver   = TYPE_X86_CPU,\
+        .property = "arat",\
+        .value    = "off",\
+    },{\
+        .driver   = "qemu64" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "kvm64" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(5),\
+    },{\
+        .driver   = "pentium3" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "n270" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(5),\
+    },{\
+        .driver   = "Conroe" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "Penryn" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "Nehalem" "-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(4),\
+    },{\
+        .driver   = "n270" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Penryn" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Conroe" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Nehalem" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Westmere" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "SandyBridge" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "IvyBridge" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Haswell-noTSX" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },{\
+        .driver   = "Broadwell-noTSX" "-" TYPE_X86_CPU,\
+        .property = "xlevel",\
+        .value    = stringify(0x8000000a),\
+    },
+
+#define PC_COMPAT_2_2 \
+    HW_COMPAT_2_2 \
+    PC_CPU_MODEL_IDS("2.3.0") \
+    {\
+        .driver = "kvm64" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "kvm32" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Conroe" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Penryn" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Nehalem" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Westmere" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "SandyBridge" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G1" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G2" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G3" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G4" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Opteron_G5" "-" TYPE_X86_CPU,\
+        .property = "vme",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "f16c",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Haswell" "-" TYPE_X86_CPU,\
+        .property = "rdrand",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "f16c",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Broadwell" "-" TYPE_X86_CPU,\
+        .property = "rdrand",\
+        .value = "off",\
+    },
+
+#define PC_COMPAT_2_1 \
+    HW_COMPAT_2_1 \
+    PC_CPU_MODEL_IDS("2.1.0") \
+    {\
+        .driver = "coreduo" "-" TYPE_X86_CPU,\
+        .property = "vmx",\
+        .value = "on",\
+    },\
+    {\
+        .driver = "core2duo" "-" TYPE_X86_CPU,\
+        .property = "vmx",\
+        .value = "on",\
+    },
+
 #define PC_COMPAT_2_0 \
-        HW_COMPAT_2_1, \
-        {\
-            .driver   = "virtio-scsi-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver   = "PIIX4_PM",\
-            .property = "memory-hotplug-support",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "apic",\
-            .property = "version",\
-            .value    = stringify(0x11),\
-        },\
-        {\
-            .driver   = "nec-usb-xhci",\
-            .property = "superspeed-ports-first",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "nec-usb-xhci",\
-            .property = "force-pcie-endcap",\
-            .value    = "on",\
-        },\
-        {\
-            .driver   = "pci-serial",\
-            .property = "prog_if",\
-            .value    = stringify(0),\
-        },\
-        {\
-            .driver   = "pci-serial-2x",\
-            .property = "prog_if",\
-            .value    = stringify(0),\
-        },\
-        {\
-            .driver   = "pci-serial-4x",\
-            .property = "prog_if",\
-            .value    = stringify(0),\
-        },\
-        {\
-            .driver   = "virtio-net-pci",\
-            .property = "guest_announce",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "ICH9-LPC",\
-            .property = "memory-hotplug-support",\
-            .value    = "off",\
-        },{\
-            .driver   = "xio3130-downstream",\
-            .property = COMPAT_PROP_PCP,\
-            .value    = "off",\
-        },{\
-            .driver   = "ioh3420",\
-            .property = COMPAT_PROP_PCP,\
-            .value    = "off",\
-        }
+    PC_CPU_MODEL_IDS("2.0.0") \
+    {\
+        .driver   = "virtio-scsi-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver   = "PIIX4_PM",\
+        .property = "memory-hotplug-support",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "apic",\
+        .property = "version",\
+        .value    = stringify(0x11),\
+    },\
+    {\
+        .driver   = "nec-usb-xhci",\
+        .property = "superspeed-ports-first",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "nec-usb-xhci",\
+        .property = "force-pcie-endcap",\
+        .value    = "on",\
+    },\
+    {\
+        .driver   = "pci-serial",\
+        .property = "prog_if",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver   = "pci-serial-2x",\
+        .property = "prog_if",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver   = "pci-serial-4x",\
+        .property = "prog_if",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver   = "virtio-net-pci",\
+        .property = "guest_announce",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "ICH9-LPC",\
+        .property = "memory-hotplug-support",\
+        .value    = "off",\
+    },{\
+        .driver   = "xio3130-downstream",\
+        .property = COMPAT_PROP_PCP,\
+        .value    = "off",\
+    },{\
+        .driver   = "ioh3420",\
+        .property = COMPAT_PROP_PCP,\
+        .value    = "off",\
+    },
 
 #define PC_COMPAT_1_7 \
-        PC_COMPAT_2_0, \
-        {\
-            .driver   = TYPE_USB_DEVICE,\
-            .property = "msos-desc",\
-            .value    = "no",\
-        },\
-        {\
-            .driver   = "PIIX4_PM",\
-            .property = "acpi-pci-hotplug-with-bridge-support",\
-            .value    = "off",\
-        },\
-        {\
-            .driver   = "hpet",\
-            .property = HPET_INTCAP,\
-            .value    = stringify(4),\
-        }
+    PC_CPU_MODEL_IDS("1.7.0") \
+    {\
+        .driver   = TYPE_USB_DEVICE,\
+        .property = "msos-desc",\
+        .value    = "no",\
+    },\
+    {\
+        .driver   = "PIIX4_PM",\
+        .property = "acpi-pci-hotplug-with-bridge-support",\
+        .value    = "off",\
+    },\
+    {\
+        .driver   = "hpet",\
+        .property = HPET_INTCAP,\
+        .value    = stringify(4),\
+    },
 
 #define PC_COMPAT_1_6 \
-        PC_COMPAT_1_7, \
-        {\
-            .driver   = "e1000",\
-            .property = "mitigation",\
-            .value    = "off",\
-        },{\
-            .driver   = "qemu64-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "qemu32-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(3),\
-        },{\
-            .driver   = "i440FX-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(1),\
-        },{\
-            .driver   = "q35-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(1),\
-        }
+    PC_CPU_MODEL_IDS("1.6.0") \
+    {\
+        .driver   = "e1000",\
+        .property = "mitigation",\
+        .value    = "off",\
+    },{\
+        .driver   = "qemu64-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "qemu32-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(3),\
+    },{\
+        .driver   = "i440FX-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(1),\
+    },{\
+        .driver   = "q35-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(1),\
+    },
 
 #define PC_COMPAT_1_5 \
-        PC_COMPAT_1_6, \
-        {\
-            .driver   = "Conroe-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Conroe-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Penryn-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Penryn-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Nehalem-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "Nehalem-" TYPE_X86_CPU,\
-            .property = "level",\
-            .value    = stringify(2),\
-        },{\
-            .driver   = "virtio-net-pci",\
-            .property = "any_layout",\
-            .value    = "off",\
-        },{\
-            .driver = TYPE_X86_CPU,\
-            .property = "pmu",\
-            .value = "on",\
-        },{\
-            .driver   = "i440FX-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "q35-pcihost",\
-            .property = "short_root_bus",\
-            .value    = stringify(0),\
-        }
+    PC_CPU_MODEL_IDS("1.5.0") \
+    {\
+        .driver   = "Conroe-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Conroe-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Penryn-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Penryn-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Nehalem-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "Nehalem-" TYPE_X86_CPU,\
+        .property = "level",\
+        .value    = stringify(2),\
+    },{\
+        .driver   = "virtio-net-pci",\
+        .property = "any_layout",\
+        .value    = "off",\
+    },{\
+        .driver = TYPE_X86_CPU,\
+        .property = "pmu",\
+        .value = "on",\
+    },{\
+        .driver   = "i440FX-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "q35-pcihost",\
+        .property = "short_root_bus",\
+        .value    = stringify(0),\
+    },
 
 #define PC_COMPAT_1_4 \
-        PC_COMPAT_1_5, \
-        {\
-            .driver   = "scsi-hd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-	},{\
-            .driver   = "scsi-cd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-	},{\
-            .driver   = "scsi-disk",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-	},{\
-            .driver   = "ide-hd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-	},{\
-            .driver   = "ide-cd",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-	},{\
-            .driver   = "ide-drive",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-        },{\
-            .driver   = "virtio-blk-pci",\
-            .property = "discard_granularity",\
-            .value    = stringify(0),\
-	},{\
-            .driver   = "virtio-serial-pci",\
-            .property = "vectors",\
-            /* DEV_NVECTORS_UNSPECIFIED as a uint32_t string */\
-            .value    = stringify(0xFFFFFFFF),\
-        },{ \
-            .driver   = "virtio-net-pci", \
-            .property = "ctrl_guest_offloads", \
-            .value    = "off", \
-        },{\
-            .driver   = "e1000",\
-            .property = "romfile",\
-            .value    = "pxe-e1000.rom",\
-        },{\
-            .driver   = "ne2k_pci",\
-            .property = "romfile",\
-            .value    = "pxe-ne2k_pci.rom",\
-        },{\
-            .driver   = "pcnet",\
-            .property = "romfile",\
-            .value    = "pxe-pcnet.rom",\
-        },{\
-            .driver   = "rtl8139",\
-            .property = "romfile",\
-            .value    = "pxe-rtl8139.rom",\
-        },{\
-            .driver   = "virtio-net-pci",\
-            .property = "romfile",\
-            .value    = "pxe-virtio.rom",\
-        },{\
-            .driver   = "486-" TYPE_X86_CPU,\
-            .property = "model",\
-            .value    = stringify(0),\
-        }
+    PC_CPU_MODEL_IDS("1.4.0") \
+    {\
+        .driver   = "scsi-hd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "scsi-cd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "scsi-disk",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-hd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-cd",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "ide-drive",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "virtio-blk-pci",\
+        .property = "discard_granularity",\
+        .value    = stringify(0),\
+    },{\
+        .driver   = "virtio-serial-pci",\
+        .property = "vectors",\
+        /* DEV_NVECTORS_UNSPECIFIED as a uint32_t string */\
+        .value    = stringify(0xFFFFFFFF),\
+    },{ \
+        .driver   = "virtio-net-pci", \
+        .property = "ctrl_guest_offloads", \
+        .value    = "off", \
+    },{\
+        .driver   = "e1000",\
+        .property = "romfile",\
+        .value    = "pxe-e1000.rom",\
+    },{\
+        .driver   = "ne2k_pci",\
+        .property = "romfile",\
+        .value    = "pxe-ne2k_pci.rom",\
+    },{\
+        .driver   = "pcnet",\
+        .property = "romfile",\
+        .value    = "pxe-pcnet.rom",\
+    },{\
+        .driver   = "rtl8139",\
+        .property = "romfile",\
+        .value    = "pxe-rtl8139.rom",\
+    },{\
+        .driver   = "virtio-net-pci",\
+        .property = "romfile",\
+        .value    = "pxe-virtio.rom",\
+    },{\
+        .driver   = "486-" TYPE_X86_CPU,\
+        .property = "model",\
+        .value    = stringify(0),\
+    },\
+    {\
+        .driver = "n270" "-" TYPE_X86_CPU,\
+        .property = "movbe",\
+        .value = "off",\
+    },\
+    {\
+        .driver = "Westmere" "-" TYPE_X86_CPU,\
+        .property = "pclmulqdq",\
+        .value = "off",\
+    },
 
-#define PC_COMMON_MACHINE_OPTIONS \
-    .default_boot_order = "cad"
+#define DEFINE_PC_MACHINE(suffix, namestr, initfn, optsfn) \
+    static void pc_machine_##suffix##_class_init(ObjectClass *oc, void *data) \
+    { \
+        MachineClass *mc = MACHINE_CLASS(oc); \
+        optsfn(mc); \
+        mc->name = namestr; \
+        mc->init = initfn; \
+    } \
+    static const TypeInfo pc_machine_type_##suffix = { \
+        .name       = namestr TYPE_MACHINE_SUFFIX, \
+        .parent     = TYPE_PC_MACHINE, \
+        .class_init = pc_machine_##suffix##_class_init, \
+    }; \
+    static void pc_machine_init_##suffix(void) \
+    { \
+        type_register(&pc_machine_type_##suffix); \
+    } \
+    type_init(pc_machine_init_##suffix)
 
-#define PC_DEFAULT_MACHINE_OPTIONS \
-    PC_COMMON_MACHINE_OPTIONS, \
-    .hot_add_cpu = pc_hot_add_cpu, \
-    .max_cpus = 255
-
+extern void igd_passthrough_isa_bridge_create(PCIBus *bus, uint16_t gpu_dev_id);
 #endif
diff --git a/include/hw/i386/smbios.h b/include/hw/i386/smbios.h
deleted file mode 100644
index d2850be..0000000
--- a/include/hw/i386/smbios.h
+++ /dev/null
@@ -1,227 +0,0 @@
-#ifndef QEMU_SMBIOS_H
-#define QEMU_SMBIOS_H
-/*
- * SMBIOS Support
- *
- * Copyright (C) 2009 Hewlett-Packard Development Company, L.P.
- *
- * Authors:
- *  Alex Williamson <alex.williamson@hp.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#include "qemu/option.h"
-
-#define SMBIOS_MAX_TYPE 127
-
-void smbios_entry_add(QemuOpts *opts);
-void smbios_set_cpuid(uint32_t version, uint32_t features);
-void smbios_set_defaults(const char *manufacturer, const char *product,
-                         const char *version, bool legacy_mode,
-                         bool uuid_encoded);
-uint8_t *smbios_get_table_legacy(size_t *length);
-void smbios_get_tables(uint8_t **tables, size_t *tables_len,
-                       uint8_t **anchor, size_t *anchor_len);
-
-/*
- * SMBIOS spec defined tables
- */
-
-/* SMBIOS entry point (anchor).
- * BIOS must place this at a 16-bit-aligned address between 0xf0000 and 0xfffff.
- */
-struct smbios_entry_point {
-    uint8_t anchor_string[4];
-    uint8_t checksum;
-    uint8_t length;
-    uint8_t smbios_major_version;
-    uint8_t smbios_minor_version;
-    uint16_t max_structure_size;
-    uint8_t entry_point_revision;
-    uint8_t formatted_area[5];
-    uint8_t intermediate_anchor_string[5];
-    uint8_t intermediate_checksum;
-    uint16_t structure_table_length;
-    uint32_t structure_table_address;
-    uint16_t number_of_structures;
-    uint8_t smbios_bcd_revision;
-} QEMU_PACKED;
-
-/* This goes at the beginning of every SMBIOS structure. */
-struct smbios_structure_header {
-    uint8_t type;
-    uint8_t length;
-    uint16_t handle;
-} QEMU_PACKED;
-
-/* SMBIOS type 0 - BIOS Information */
-struct smbios_type_0 {
-    struct smbios_structure_header header;
-    uint8_t vendor_str;
-    uint8_t bios_version_str;
-    uint16_t bios_starting_address_segment;
-    uint8_t bios_release_date_str;
-    uint8_t bios_rom_size;
-    uint64_t bios_characteristics;
-    uint8_t bios_characteristics_extension_bytes[2];
-    uint8_t system_bios_major_release;
-    uint8_t system_bios_minor_release;
-    uint8_t embedded_controller_major_release;
-    uint8_t embedded_controller_minor_release;
-} QEMU_PACKED;
-
-/* UUID encoding. The time_* fields are little-endian, as specified by SMBIOS
- * version 2.6.
- */
-struct smbios_uuid {
-    uint32_t time_low;
-    uint16_t time_mid;
-    uint16_t time_hi_and_version;
-    uint8_t clock_seq_hi_and_reserved;
-    uint8_t clock_seq_low;
-    uint8_t node[6];
-} QEMU_PACKED;
-
-/* SMBIOS type 1 - System Information */
-struct smbios_type_1 {
-    struct smbios_structure_header header;
-    uint8_t manufacturer_str;
-    uint8_t product_name_str;
-    uint8_t version_str;
-    uint8_t serial_number_str;
-    struct smbios_uuid uuid;
-    uint8_t wake_up_type;
-    uint8_t sku_number_str;
-    uint8_t family_str;
-} QEMU_PACKED;
-
-/* SMBIOS type 2 - Base Board */
-struct smbios_type_2 {
-    struct smbios_structure_header header;
-    uint8_t manufacturer_str;
-    uint8_t product_str;
-    uint8_t version_str;
-    uint8_t serial_number_str;
-    uint8_t asset_tag_number_str;
-    uint8_t feature_flags;
-    uint8_t location_str;
-    uint16_t chassis_handle;
-    uint8_t board_type;
-    uint8_t contained_element_count;
-    /* contained elements follow */
-} QEMU_PACKED;
-
-/* SMBIOS type 3 - System Enclosure (v2.7) */
-struct smbios_type_3 {
-    struct smbios_structure_header header;
-    uint8_t manufacturer_str;
-    uint8_t type;
-    uint8_t version_str;
-    uint8_t serial_number_str;
-    uint8_t asset_tag_number_str;
-    uint8_t boot_up_state;
-    uint8_t power_supply_state;
-    uint8_t thermal_state;
-    uint8_t security_status;
-    uint32_t oem_defined;
-    uint8_t height;
-    uint8_t number_of_power_cords;
-    uint8_t contained_element_count;
-    uint8_t sku_number_str;
-    /* contained elements follow */
-} QEMU_PACKED;
-
-/* SMBIOS type 4 - Processor Information (v2.6) */
-struct smbios_type_4 {
-    struct smbios_structure_header header;
-    uint8_t socket_designation_str;
-    uint8_t processor_type;
-    uint8_t processor_family;
-    uint8_t processor_manufacturer_str;
-    uint32_t processor_id[2];
-    uint8_t processor_version_str;
-    uint8_t voltage;
-    uint16_t external_clock;
-    uint16_t max_speed;
-    uint16_t current_speed;
-    uint8_t status;
-    uint8_t processor_upgrade;
-    uint16_t l1_cache_handle;
-    uint16_t l2_cache_handle;
-    uint16_t l3_cache_handle;
-    uint8_t serial_number_str;
-    uint8_t asset_tag_number_str;
-    uint8_t part_number_str;
-    uint8_t core_count;
-    uint8_t core_enabled;
-    uint8_t thread_count;
-    uint16_t processor_characteristics;
-    uint16_t processor_family2;
-} QEMU_PACKED;
-
-/* SMBIOS type 16 - Physical Memory Array (v2.7) */
-struct smbios_type_16 {
-    struct smbios_structure_header header;
-    uint8_t location;
-    uint8_t use;
-    uint8_t error_correction;
-    uint32_t maximum_capacity;
-    uint16_t memory_error_information_handle;
-    uint16_t number_of_memory_devices;
-    uint64_t extended_maximum_capacity;
-} QEMU_PACKED;
-
-/* SMBIOS type 17 - Memory Device (v2.8) */
-struct smbios_type_17 {
-    struct smbios_structure_header header;
-    uint16_t physical_memory_array_handle;
-    uint16_t memory_error_information_handle;
-    uint16_t total_width;
-    uint16_t data_width;
-    uint16_t size;
-    uint8_t form_factor;
-    uint8_t device_set;
-    uint8_t device_locator_str;
-    uint8_t bank_locator_str;
-    uint8_t memory_type;
-    uint16_t type_detail;
-    uint16_t speed;
-    uint8_t manufacturer_str;
-    uint8_t serial_number_str;
-    uint8_t asset_tag_number_str;
-    uint8_t part_number_str;
-    uint8_t attributes;
-    uint32_t extended_size;
-    uint16_t configured_clock_speed;
-    uint16_t minimum_voltage;
-    uint16_t maximum_voltage;
-    uint16_t configured_voltage;
-} QEMU_PACKED;
-
-/* SMBIOS type 19 - Memory Array Mapped Address (v2.7) */
-struct smbios_type_19 {
-    struct smbios_structure_header header;
-    uint32_t starting_address;
-    uint32_t ending_address;
-    uint16_t memory_array_handle;
-    uint8_t partition_width;
-    uint64_t extended_starting_address;
-    uint64_t extended_ending_address;
-} QEMU_PACKED;
-
-/* SMBIOS type 32 - System Boot Information */
-struct smbios_type_32 {
-    struct smbios_structure_header header;
-    uint8_t reserved[6];
-    uint8_t boot_status;
-} QEMU_PACKED;
-
-/* SMBIOS type 127 -- End-of-table */
-struct smbios_type_127 {
-    struct smbios_structure_header header;
-} QEMU_PACKED;
-
-#endif /*QEMU_SMBIOS_H */
diff --git a/include/hw/i386/topology.h b/include/hw/i386/topology.h
new file mode 100644
index 0000000..1ebaee0
--- /dev/null
+++ b/include/hw/i386/topology.h
@@ -0,0 +1,148 @@
+/*
+ *  x86 CPU topology data structures and functions
+ *
+ *  Copyright (c) 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef HW_I386_TOPOLOGY_H
+#define HW_I386_TOPOLOGY_H
+
+/* This file implements the APIC-ID-based CPU topology enumeration logic,
+ * documented at the following document:
+ *   Intel® 64 Architecture Processor Topology Enumeration
+ *   http://software.intel.com/en-us/articles/intel-64-architecture-processor-topology-enumeration/
+ *
+ * This code should be compatible with AMD's "Extended Method" described at:
+ *   AMD CPUID Specification (Publication #25481)
+ *   Section 3: Multiple Core Calcuation
+ * as long as:
+ *  nr_threads is set to 1;
+ *  OFFSET_IDX is assumed to be 0;
+ *  CPUID Fn8000_0008_ECX[ApicIdCoreIdSize[3:0]] is set to apicid_core_width().
+ */
+
+
+#include "qemu/bitops.h"
+
+/* APIC IDs can be 32-bit, but beware: APIC IDs > 255 require x2APIC support
+ */
+typedef uint32_t apic_id_t;
+
+typedef struct X86CPUTopoInfo {
+    unsigned pkg_id;
+    unsigned core_id;
+    unsigned smt_id;
+} X86CPUTopoInfo;
+
+/* Return the bit width needed for 'count' IDs
+ */
+static unsigned apicid_bitwidth_for_count(unsigned count)
+{
+    g_assert(count >= 1);
+    count -= 1;
+    return count ? 32 - clz32(count) : 0;
+}
+
+/* Bit width of the SMT_ID (thread ID) field on the APIC ID
+ */
+static inline unsigned apicid_smt_width(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_bitwidth_for_count(nr_threads);
+}
+
+/* Bit width of the Core_ID field
+ */
+static inline unsigned apicid_core_width(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_bitwidth_for_count(nr_cores);
+}
+
+/* Bit offset of the Core_ID field
+ */
+static inline unsigned apicid_core_offset(unsigned nr_cores,
+                                          unsigned nr_threads)
+{
+    return apicid_smt_width(nr_cores, nr_threads);
+}
+
+/* Bit offset of the Pkg_ID (socket ID) field
+ */
+static inline unsigned apicid_pkg_offset(unsigned nr_cores, unsigned nr_threads)
+{
+    return apicid_core_offset(nr_cores, nr_threads) +
+           apicid_core_width(nr_cores, nr_threads);
+}
+
+/* Make APIC ID for the CPU based on Pkg_ID, Core_ID, SMT_ID
+ *
+ * The caller must make sure core_id < nr_cores and smt_id < nr_threads.
+ */
+static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
+                                             unsigned nr_threads,
+                                             const X86CPUTopoInfo *topo)
+{
+    return (topo->pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
+           (topo->core_id << apicid_core_offset(nr_cores, nr_threads)) |
+           topo->smt_id;
+}
+
+/* Calculate thread/core/package IDs for a specific topology,
+ * based on (contiguous) CPU index
+ */
+static inline void x86_topo_ids_from_idx(unsigned nr_cores,
+                                         unsigned nr_threads,
+                                         unsigned cpu_index,
+                                         X86CPUTopoInfo *topo)
+{
+    unsigned core_index = cpu_index / nr_threads;
+    topo->smt_id = cpu_index % nr_threads;
+    topo->core_id = core_index % nr_cores;
+    topo->pkg_id = core_index / nr_cores;
+}
+
+/* Calculate thread/core/package IDs for a specific topology,
+ * based on APIC ID
+ */
+static inline void x86_topo_ids_from_apicid(apic_id_t apicid,
+                                            unsigned nr_cores,
+                                            unsigned nr_threads,
+                                            X86CPUTopoInfo *topo)
+{
+    topo->smt_id = apicid &
+                   ~(0xFFFFFFFFUL << apicid_smt_width(nr_cores, nr_threads));
+    topo->core_id = (apicid >> apicid_core_offset(nr_cores, nr_threads)) &
+                   ~(0xFFFFFFFFUL << apicid_core_width(nr_cores, nr_threads));
+    topo->pkg_id = apicid >> apicid_pkg_offset(nr_cores, nr_threads);
+}
+
+/* Make APIC ID for the CPU 'cpu_index'
+ *
+ * 'cpu_index' is a sequential, contiguous ID for the CPU.
+ */
+static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_cores,
+                                                unsigned nr_threads,
+                                                unsigned cpu_index)
+{
+    X86CPUTopoInfo topo;
+    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index, &topo);
+    return apicid_from_topo_ids(nr_cores, nr_threads, &topo);
+}
+
+#endif /* HW_I386_TOPOLOGY_H */
diff --git a/include/hw/i386/x86-iommu.h b/include/hw/i386/x86-iommu.h
new file mode 100644
index 0000000..c48e8dd
--- /dev/null
+++ b/include/hw/i386/x86-iommu.h
@@ -0,0 +1,100 @@
+/*
+ * Common IOMMU interface for X86 platform
+ *
+ * Copyright (C) 2016 Peter Xu, Red Hat <peterx@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IOMMU_COMMON_H
+#define IOMMU_COMMON_H
+
+#include "hw/sysbus.h"
+#include "hw/pci/pci.h"
+
+#define  TYPE_X86_IOMMU_DEVICE  ("x86-iommu")
+#define  X86_IOMMU_DEVICE(obj) \
+    OBJECT_CHECK(X86IOMMUState, (obj), TYPE_X86_IOMMU_DEVICE)
+#define  X86_IOMMU_CLASS(klass) \
+    OBJECT_CLASS_CHECK(X86IOMMUClass, (klass), TYPE_X86_IOMMU_DEVICE)
+#define  X86_IOMMU_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(X86IOMMUClass, obj, TYPE_X86_IOMMU_DEVICE)
+
+#define X86_IOMMU_PCI_DEVFN_MAX           256
+#define X86_IOMMU_SID_INVALID             (0xffff)
+
+typedef struct X86IOMMUState X86IOMMUState;
+typedef struct X86IOMMUClass X86IOMMUClass;
+
+struct X86IOMMUClass {
+    SysBusDeviceClass parent;
+    /* Intel/AMD specific realize() hook */
+    DeviceRealize realize;
+    /* MSI-based interrupt remapping */
+    int (*int_remap)(X86IOMMUState *iommu, MSIMessage *src,
+                     MSIMessage *dst, uint16_t sid);
+};
+
+/**
+ * iec_notify_fn - IEC (Interrupt Entry Cache) notifier hook,
+ *                 triggered when IR invalidation happens.
+ * @private: private data
+ * @global: whether this is a global IEC invalidation
+ * @index: IRTE index to invalidate (start from)
+ * @mask: invalidation mask
+ */
+typedef void (*iec_notify_fn)(void *private, bool global,
+                              uint32_t index, uint32_t mask);
+
+struct IEC_Notifier {
+    iec_notify_fn iec_notify;
+    void *private;
+    QLIST_ENTRY(IEC_Notifier) list;
+};
+typedef struct IEC_Notifier IEC_Notifier;
+
+struct X86IOMMUState {
+    SysBusDevice busdev;
+    bool intr_supported;        /* Whether vIOMMU supports IR */
+    QLIST_HEAD(, IEC_Notifier) iec_notifiers; /* IEC notify list */
+};
+
+/**
+ * x86_iommu_get_default - get default IOMMU device
+ * @return: pointer to default IOMMU device
+ */
+X86IOMMUState *x86_iommu_get_default(void);
+
+/**
+ * x86_iommu_iec_register_notifier - register IEC (Interrupt Entry
+ *                                   Cache) notifiers
+ * @iommu: IOMMU device to register
+ * @fn: IEC notifier hook function
+ * @data: notifier private data
+ */
+void x86_iommu_iec_register_notifier(X86IOMMUState *iommu,
+                                     iec_notify_fn fn, void *data);
+
+/**
+ * x86_iommu_iec_notify_all - Notify IEC invalidations
+ * @iommu: IOMMU device that sends the notification
+ * @global: whether this is a global invalidation. If true, @index
+ *          and @mask are undefined.
+ * @index: starting index of interrupt entry to invalidate
+ * @mask: index mask for the invalidation
+ */
+void x86_iommu_iec_notify_all(X86IOMMUState *iommu, bool global,
+                              uint32_t index, uint32_t mask);
+
+#endif
diff --git a/include/hw/ide/ahci.h b/include/hw/ide/ahci.h
new file mode 100644
index 0000000..0ca7c65
--- /dev/null
+++ b/include/hw/ide/ahci.h
@@ -0,0 +1,405 @@
+/*
+ * QEMU AHCI Emulation
+ *
+ * Copyright (c) 2010 qiaochong@loongson.cn
+ * Copyright (c) 2010 Roland Elek <elek.roland@gmail.com>
+ * Copyright (c) 2010 Sebastian Herbszt <herbszt@gmx.de>
+ * Copyright (c) 2010 Alexander Graf <agraf@suse.de>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef HW_IDE_AHCI_H
+#define HW_IDE_AHCI_H
+
+#include "hw/sysbus.h"
+
+#define AHCI_MEM_BAR_SIZE         0x1000
+#define AHCI_MAX_PORTS            32
+#define AHCI_MAX_SG               168 /* hardware max is 64K */
+#define AHCI_DMA_BOUNDARY         0xffffffff
+#define AHCI_USE_CLUSTERING       0
+#define AHCI_MAX_CMDS             32
+#define AHCI_CMD_SZ               32
+#define AHCI_CMD_SLOT_SZ          (AHCI_MAX_CMDS * AHCI_CMD_SZ)
+#define AHCI_RX_FIS_SZ            256
+#define AHCI_CMD_TBL_CDB          0x40
+#define AHCI_CMD_TBL_HDR_SZ       0x80
+#define AHCI_CMD_TBL_SZ           (AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16))
+#define AHCI_CMD_TBL_AR_SZ        (AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS)
+#define AHCI_PORT_PRIV_DMA_SZ     (AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ + \
+                                   AHCI_RX_FIS_SZ)
+
+#define AHCI_IRQ_ON_SG            (1U << 31)
+#define AHCI_CMD_ATAPI            (1 << 5)
+#define AHCI_CMD_WRITE            (1 << 6)
+#define AHCI_CMD_PREFETCH         (1 << 7)
+#define AHCI_CMD_RESET            (1 << 8)
+#define AHCI_CMD_CLR_BUSY         (1 << 10)
+
+#define RX_FIS_D2H_REG            0x40 /* offset of D2H Register FIS data */
+#define RX_FIS_SDB                0x58 /* offset of SDB FIS data */
+#define RX_FIS_UNK                0x60 /* offset of Unknown FIS data */
+
+/* global controller registers */
+#define HOST_CAP                  0x00 /* host capabilities */
+#define HOST_CTL                  0x04 /* global host control */
+#define HOST_IRQ_STAT             0x08 /* interrupt status */
+#define HOST_PORTS_IMPL           0x0c /* bitmap of implemented ports */
+#define HOST_VERSION              0x10 /* AHCI spec. version compliancy */
+
+/* HOST_CTL bits */
+#define HOST_CTL_RESET            (1 << 0)  /* reset controller; self-clear */
+#define HOST_CTL_IRQ_EN           (1 << 1)  /* global IRQ enable */
+#define HOST_CTL_AHCI_EN          (1U << 31) /* AHCI enabled */
+
+/* HOST_CAP bits */
+#define HOST_CAP_SSC              (1 << 14) /* Slumber capable */
+#define HOST_CAP_AHCI             (1 << 18) /* AHCI only */
+#define HOST_CAP_CLO              (1 << 24) /* Command List Override support */
+#define HOST_CAP_SSS              (1 << 27) /* Staggered Spin-up */
+#define HOST_CAP_NCQ              (1 << 30) /* Native Command Queueing */
+#define HOST_CAP_64               (1U << 31) /* PCI DAC (64-bit DMA) support */
+
+/* registers for each SATA port */
+#define PORT_LST_ADDR             0x00 /* command list DMA addr */
+#define PORT_LST_ADDR_HI          0x04 /* command list DMA addr hi */
+#define PORT_FIS_ADDR             0x08 /* FIS rx buf addr */
+#define PORT_FIS_ADDR_HI          0x0c /* FIS rx buf addr hi */
+#define PORT_IRQ_STAT             0x10 /* interrupt status */
+#define PORT_IRQ_MASK             0x14 /* interrupt enable/disable mask */
+#define PORT_CMD                  0x18 /* port command */
+#define PORT_TFDATA               0x20 /* taskfile data */
+#define PORT_SIG                  0x24 /* device TF signature */
+#define PORT_SCR_STAT             0x28 /* SATA phy register: SStatus */
+#define PORT_SCR_CTL              0x2c /* SATA phy register: SControl */
+#define PORT_SCR_ERR              0x30 /* SATA phy register: SError */
+#define PORT_SCR_ACT              0x34 /* SATA phy register: SActive */
+#define PORT_CMD_ISSUE            0x38 /* command issue */
+#define PORT_RESERVED             0x3c /* reserved */
+
+/* PORT_IRQ_{STAT,MASK} bits */
+#define PORT_IRQ_COLD_PRES        (1U << 31) /* cold presence detect */
+#define PORT_IRQ_TF_ERR           (1 << 30) /* task file error */
+#define PORT_IRQ_HBUS_ERR         (1 << 29) /* host bus fatal error */
+#define PORT_IRQ_HBUS_DATA_ERR    (1 << 28) /* host bus data error */
+#define PORT_IRQ_IF_ERR           (1 << 27) /* interface fatal error */
+#define PORT_IRQ_IF_NONFATAL      (1 << 26) /* interface non-fatal error */
+#define PORT_IRQ_OVERFLOW         (1 << 24) /* xfer exhausted available S/G */
+#define PORT_IRQ_BAD_PMP          (1 << 23) /* incorrect port multiplier */
+
+#define PORT_IRQ_PHYRDY           (1 << 22) /* PhyRdy changed */
+#define PORT_IRQ_DEV_ILCK         (1 << 7) /* device interlock */
+#define PORT_IRQ_CONNECT          (1 << 6) /* port connect change status */
+#define PORT_IRQ_SG_DONE          (1 << 5) /* descriptor processed */
+#define PORT_IRQ_UNK_FIS          (1 << 4) /* unknown FIS rx'd */
+#define PORT_IRQ_SDB_FIS          (1 << 3) /* Set Device Bits FIS rx'd */
+#define PORT_IRQ_DMAS_FIS         (1 << 2) /* DMA Setup FIS rx'd */
+#define PORT_IRQ_PIOS_FIS         (1 << 1) /* PIO Setup FIS rx'd */
+#define PORT_IRQ_D2H_REG_FIS      (1 << 0) /* D2H Register FIS rx'd */
+
+#define PORT_IRQ_FREEZE           (PORT_IRQ_HBUS_ERR | PORT_IRQ_IF_ERR |   \
+                                   PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY |    \
+                                   PORT_IRQ_UNK_FIS)
+#define PORT_IRQ_ERROR            (PORT_IRQ_FREEZE | PORT_IRQ_TF_ERR |     \
+                                   PORT_IRQ_HBUS_DATA_ERR)
+#define DEF_PORT_IRQ              (PORT_IRQ_ERROR | PORT_IRQ_SG_DONE |     \
+                                   PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS |  \
+                                   PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS)
+
+/* PORT_CMD bits */
+#define PORT_CMD_ATAPI            (1 << 24) /* Device is ATAPI */
+#define PORT_CMD_LIST_ON          (1 << 15) /* cmd list DMA engine running */
+#define PORT_CMD_FIS_ON           (1 << 14) /* FIS DMA engine running */
+#define PORT_CMD_FIS_RX           (1 << 4) /* Enable FIS receive DMA engine */
+#define PORT_CMD_CLO              (1 << 3) /* Command list override */
+#define PORT_CMD_POWER_ON         (1 << 2) /* Power up device */
+#define PORT_CMD_SPIN_UP          (1 << 1) /* Spin up device */
+#define PORT_CMD_START            (1 << 0) /* Enable port DMA engine */
+
+#define PORT_CMD_ICC_MASK        (0xfU << 28) /* i/f ICC state mask */
+#define PORT_CMD_ICC_ACTIVE       (0x1 << 28) /* Put i/f in active state */
+#define PORT_CMD_ICC_PARTIAL      (0x2 << 28) /* Put i/f in partial state */
+#define PORT_CMD_ICC_SLUMBER      (0x6 << 28) /* Put i/f in slumber state */
+
+#define PORT_CMD_RO_MASK          0x007dffe0 /* Which CMD bits are read only? */
+
+/* ap->flags bits */
+#define AHCI_FLAG_NO_NCQ                  (1 << 24)
+#define AHCI_FLAG_IGN_IRQ_IF_ERR          (1 << 25) /* ignore IRQ_IF_ERR */
+#define AHCI_FLAG_HONOR_PI                (1 << 26) /* honor PORTS_IMPL */
+#define AHCI_FLAG_IGN_SERR_INTERNAL       (1 << 27) /* ignore SERR_INTERNAL */
+#define AHCI_FLAG_32BIT_ONLY              (1 << 28) /* force 32bit */
+
+#define ATA_SRST                          (1 << 2)  /* software reset */
+
+#define STATE_RUN                         0
+#define STATE_RESET                       1
+
+#define SATA_SCR_SSTATUS_DET_NODEV        0x0
+#define SATA_SCR_SSTATUS_DET_DEV_PRESENT_PHY_UP 0x3
+
+#define SATA_SCR_SSTATUS_SPD_NODEV        0x00
+#define SATA_SCR_SSTATUS_SPD_GEN1         0x10
+
+#define SATA_SCR_SSTATUS_IPM_NODEV        0x000
+#define SATA_SCR_SSTATUS_IPM_ACTIVE       0X100
+
+#define AHCI_SCR_SCTL_DET                 0xf
+
+#define SATA_FIS_TYPE_REGISTER_H2D        0x27
+#define   SATA_FIS_REG_H2D_UPDATE_COMMAND_REGISTER 0x80
+#define SATA_FIS_TYPE_REGISTER_D2H        0x34
+#define SATA_FIS_TYPE_PIO_SETUP           0x5f
+#define SATA_FIS_TYPE_SDB                 0xA1
+
+#define AHCI_CMD_HDR_CMD_FIS_LEN           0x1f
+#define AHCI_CMD_HDR_PRDT_LEN              16
+
+#define SATA_SIGNATURE_CDROM               0xeb140101
+#define SATA_SIGNATURE_DISK                0x00000101
+
+#define AHCI_GENERIC_HOST_CONTROL_REGS_MAX_ADDR 0x20
+                                            /* Shouldn't this be 0x2c? */
+
+#define AHCI_PORT_REGS_START_ADDR          0x100
+#define AHCI_PORT_ADDR_OFFSET_MASK         0x7f
+#define AHCI_PORT_ADDR_OFFSET_LEN          0x80
+
+#define AHCI_NUM_COMMAND_SLOTS             31
+#define AHCI_SUPPORTED_SPEED               20
+#define AHCI_SUPPORTED_SPEED_GEN1          1
+#define AHCI_VERSION_1_0                   0x10000
+
+#define AHCI_PROGMODE_MAJOR_REV_1          1
+
+#define AHCI_COMMAND_TABLE_ACMD            0x40
+
+#define AHCI_PRDT_SIZE_MASK                0x3fffff
+
+#define IDE_FEATURE_DMA                    1
+
+#define READ_FPDMA_QUEUED                  0x60
+#define WRITE_FPDMA_QUEUED                 0x61
+#define NCQ_NON_DATA                       0x63
+#define RECEIVE_FPDMA_QUEUED               0x65
+#define SEND_FPDMA_QUEUED                  0x64
+
+#define NCQ_FIS_FUA_MASK                   0x80
+#define NCQ_FIS_RARC_MASK                  0x01
+
+#define RES_FIS_DSFIS                      0x00
+#define RES_FIS_PSFIS                      0x20
+#define RES_FIS_RFIS                       0x40
+#define RES_FIS_SDBFIS                     0x58
+#define RES_FIS_UFIS                       0x60
+
+#define SATA_CAP_SIZE           0x8
+#define SATA_CAP_REV            0x2
+#define SATA_CAP_BAR            0x4
+
+typedef struct AHCIControlRegs {
+    uint32_t    cap;
+    uint32_t    ghc;
+    uint32_t    irqstatus;
+    uint32_t    impl;
+    uint32_t    version;
+} AHCIControlRegs;
+
+typedef struct AHCIPortRegs {
+    uint32_t    lst_addr;
+    uint32_t    lst_addr_hi;
+    uint32_t    fis_addr;
+    uint32_t    fis_addr_hi;
+    uint32_t    irq_stat;
+    uint32_t    irq_mask;
+    uint32_t    cmd;
+    uint32_t    unused0;
+    uint32_t    tfdata;
+    uint32_t    sig;
+    uint32_t    scr_stat;
+    uint32_t    scr_ctl;
+    uint32_t    scr_err;
+    uint32_t    scr_act;
+    uint32_t    cmd_issue;
+    uint32_t    reserved;
+} AHCIPortRegs;
+
+typedef struct AHCICmdHdr {
+    uint16_t    opts;
+    uint16_t    prdtl;
+    uint32_t    status;
+    uint64_t    tbl_addr;
+    uint32_t    reserved[4];
+} QEMU_PACKED AHCICmdHdr;
+
+typedef struct AHCI_SG {
+    uint64_t    addr;
+    uint32_t    reserved;
+    uint32_t    flags_size;
+} QEMU_PACKED AHCI_SG;
+
+typedef struct AHCIDevice AHCIDevice;
+
+typedef struct NCQTransferState {
+    AHCIDevice *drive;
+    BlockAIOCB *aiocb;
+    AHCICmdHdr *cmdh;
+    QEMUSGList sglist;
+    BlockAcctCookie acct;
+    uint32_t sector_count;
+    uint64_t lba;
+    uint8_t tag;
+    uint8_t cmd;
+    uint8_t slot;
+    bool used;
+    bool halt;
+} NCQTransferState;
+
+struct AHCIDevice {
+    IDEDMA dma;
+    IDEBus port;
+    int port_no;
+    uint32_t port_state;
+    uint32_t finished;
+    AHCIPortRegs port_regs;
+    struct AHCIState *hba;
+    QEMUBH *check_bh;
+    uint8_t *lst;
+    uint8_t *res_fis;
+    bool done_atapi_packet;
+    int32_t busy_slot;
+    bool init_d2h_sent;
+    AHCICmdHdr *cur_cmd;
+    NCQTransferState ncq_tfs[AHCI_MAX_CMDS];
+};
+
+typedef struct AHCIState {
+    DeviceState *container;
+
+    AHCIDevice *dev;
+    AHCIControlRegs control_regs;
+    MemoryRegion mem;
+    MemoryRegion idp;       /* Index-Data Pair I/O port space */
+    unsigned idp_offset;    /* Offset of index in I/O port space */
+    uint32_t idp_index;     /* Current IDP index */
+    int32_t ports;
+    qemu_irq irq;
+    AddressSpace *as;
+} AHCIState;
+
+typedef struct AHCIPCIState {
+    /*< private >*/
+    PCIDevice parent_obj;
+    /*< public >*/
+
+    AHCIState ahci;
+} AHCIPCIState;
+
+#define TYPE_ICH9_AHCI "ich9-ahci"
+
+#define ICH_AHCI(obj) \
+    OBJECT_CHECK(AHCIPCIState, (obj), TYPE_ICH9_AHCI)
+
+extern const VMStateDescription vmstate_ahci;
+
+#define VMSTATE_AHCI(_field, _state) {                               \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(AHCIState),                                 \
+    .vmsd       = &vmstate_ahci,                                     \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, AHCIState),   \
+}
+
+/**
+ * NCQFrame is the same as a Register H2D FIS (described in SATA 3.2),
+ * but some fields have been re-mapped and re-purposed, as seen in
+ * SATA 3.2 section 13.6.4.1 ("READ FPDMA QUEUED")
+ *
+ * cmd_fis[3], feature 7:0, becomes sector count 7:0.
+ * cmd_fis[7], device 7:0, uses bit 7 as the Force Unit Access bit.
+ * cmd_fis[11], feature 15:8, becomes sector count 15:8.
+ * cmd_fis[12], count 7:0, becomes the NCQ TAG (7:3) and RARC bit (0)
+ * cmd_fis[13], count 15:8, becomes the priority value (7:6)
+ * bytes 16-19 become an le32 "auxiliary" field.
+ */
+typedef struct NCQFrame {
+    uint8_t fis_type;
+    uint8_t c;
+    uint8_t command;
+    uint8_t sector_count_low;  /* (feature 7:0) */
+    uint8_t lba0;
+    uint8_t lba1;
+    uint8_t lba2;
+    uint8_t fua;               /* (device 7:0) */
+    uint8_t lba3;
+    uint8_t lba4;
+    uint8_t lba5;
+    uint8_t sector_count_high; /* (feature 15:8) */
+    uint8_t tag;               /* (count 0:7) */
+    uint8_t prio;              /* (count 15:8) */
+    uint8_t icc;
+    uint8_t control;
+    uint8_t aux0;
+    uint8_t aux1;
+    uint8_t aux2;
+    uint8_t aux3;
+} QEMU_PACKED NCQFrame;
+
+typedef struct SDBFIS {
+    uint8_t type;
+    uint8_t flags;
+    uint8_t status;
+    uint8_t error;
+    uint32_t payload;
+} QEMU_PACKED SDBFIS;
+
+void ahci_realize(AHCIState *s, DeviceState *qdev, AddressSpace *as, int ports);
+void ahci_init(AHCIState *s, DeviceState *qdev);
+void ahci_uninit(AHCIState *s);
+
+void ahci_reset(AHCIState *s);
+
+void ahci_ide_create_devs(PCIDevice *dev, DriveInfo **hd);
+
+#define TYPE_SYSBUS_AHCI "sysbus-ahci"
+#define SYSBUS_AHCI(obj) OBJECT_CHECK(SysbusAHCIState, (obj), TYPE_SYSBUS_AHCI)
+
+typedef struct SysbusAHCIState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    AHCIState ahci;
+    uint32_t num_ports;
+} SysbusAHCIState;
+
+#define TYPE_ALLWINNER_AHCI "allwinner-ahci"
+#define ALLWINNER_AHCI(obj) OBJECT_CHECK(AllwinnerAHCIState, (obj), \
+                       TYPE_ALLWINNER_AHCI)
+
+#define ALLWINNER_AHCI_MMIO_OFF  0x80
+#define ALLWINNER_AHCI_MMIO_SIZE 0x80
+
+struct AllwinnerAHCIState {
+    /*< private >*/
+    SysbusAHCIState parent_obj;
+    /*< public >*/
+
+    MemoryRegion mmio;
+    uint32_t regs[ALLWINNER_AHCI_MMIO_SIZE/4];
+};
+
+#endif /* HW_IDE_AHCI_H */
diff --git a/include/hw/ide/internal.h b/include/hw/ide/internal.h
new file mode 100644
index 0000000..7824bc3
--- /dev/null
+++ b/include/hw/ide/internal.h
@@ -0,0 +1,635 @@
+#ifndef HW_IDE_INTERNAL_H
+#define HW_IDE_INTERNAL_H
+
+/*
+ * QEMU IDE Emulation -- internal header file
+ * only files in hw/ide/ are supposed to include this file.
+ * non-internal declarations are in hw/ide.h
+ */
+#include "hw/ide.h"
+#include "hw/isa/isa.h"
+#include "sysemu/dma.h"
+#include "sysemu/sysemu.h"
+#include "hw/block/block.h"
+#include "block/scsi.h"
+
+/* debug IDE devices */
+//#define DEBUG_IDE
+//#define DEBUG_IDE_ATAPI
+//#define DEBUG_AIO
+#define USE_DMA_CDROM
+
+typedef struct IDEBus IDEBus;
+typedef struct IDEDevice IDEDevice;
+typedef struct IDEState IDEState;
+typedef struct IDEDMA IDEDMA;
+typedef struct IDEDMAOps IDEDMAOps;
+
+#define TYPE_IDE_BUS "IDE"
+#define IDE_BUS(obj) OBJECT_CHECK(IDEBus, (obj), TYPE_IDE_BUS)
+
+/* Bits of HD_STATUS */
+#define ERR_STAT		0x01
+#define INDEX_STAT		0x02
+#define ECC_STAT		0x04	/* Corrected error */
+#define DRQ_STAT		0x08
+#define SEEK_STAT		0x10
+#define SRV_STAT		0x10
+#define WRERR_STAT		0x20
+#define READY_STAT		0x40
+#define BUSY_STAT		0x80
+
+/* Bits for HD_ERROR */
+#define MARK_ERR		0x01	/* Bad address mark */
+#define TRK0_ERR		0x02	/* couldn't find track 0 */
+#define ABRT_ERR		0x04	/* Command aborted */
+#define MCR_ERR			0x08	/* media change request */
+#define ID_ERR			0x10	/* ID field not found */
+#define MC_ERR			0x20	/* media changed */
+#define ECC_ERR			0x40	/* Uncorrectable ECC error */
+#define BBD_ERR			0x80	/* pre-EIDE meaning:  block marked bad */
+#define ICRC_ERR		0x80	/* new meaning:  CRC error during transfer */
+
+/* Bits of HD_NSECTOR */
+#define CD			0x01
+#define IO			0x02
+#define REL			0x04
+#define TAG_MASK		0xf8
+
+#define IDE_CMD_RESET           0x04
+#define IDE_CMD_DISABLE_IRQ     0x02
+
+/* ACS-2 T13/2015-D Table B.2 Command codes */
+#define WIN_NOP				0x00
+/* reserved                             0x01..0x02 */
+#define CFA_REQ_EXT_ERROR_CODE		0x03 /* CFA Request Extended Error Code */
+/* reserved                             0x04..0x05 */
+#define WIN_DSM                         0x06
+/* reserved                             0x07 */
+#define WIN_DEVICE_RESET		0x08
+/* reserved                             0x09..0x0a */
+/* REQUEST SENSE DATA EXT               0x0B */
+/* reserved                             0x0C..0x0F */
+#define WIN_RECAL                       0x10 /* obsolete since ATA4 */
+/* obsolete since ATA3, retired in ATA4 0x11..0x1F */
+#define WIN_READ			0x20 /* 28-Bit */
+#define WIN_READ_ONCE                   0x21 /* 28-Bit w/o retries, obsolete since ATA5 */
+/* obsolete since ATA4                  0x22..0x23 */
+#define WIN_READ_EXT			0x24 /* 48-Bit */
+#define WIN_READDMA_EXT			0x25 /* 48-Bit */
+#define WIN_READDMA_QUEUED_EXT          0x26 /* 48-Bit, obsolete since ACS2 */
+#define WIN_READ_NATIVE_MAX_EXT		0x27 /* 48-Bit */
+/* reserved                             0x28 */
+#define WIN_MULTREAD_EXT		0x29 /* 48-Bit */
+/* READ STREAM DMA EXT                  0x2A */
+/* READ STREAM EXT                      0x2B */
+/* reserved                             0x2C..0x2E */
+/* READ LOG EXT                         0x2F */
+#define WIN_WRITE			0x30 /* 28-Bit */
+#define WIN_WRITE_ONCE                  0x31 /* 28-Bit w/o retries, obsolete since ATA5 */
+/* obsolete since ATA4                  0x32..0x33 */
+#define WIN_WRITE_EXT			0x34 /* 48-Bit */
+#define WIN_WRITEDMA_EXT		0x35 /* 48-Bit */
+#define WIN_WRITEDMA_QUEUED_EXT		0x36 /* 48-Bit */
+#define WIN_SET_MAX_EXT                 0x37 /* 48-Bit, obsolete since ACS2 */
+#define WIN_SET_MAX_EXT			0x37 /* 48-Bit */
+#define CFA_WRITE_SECT_WO_ERASE		0x38 /* CFA Write Sectors without erase */
+#define WIN_MULTWRITE_EXT		0x39 /* 48-Bit */
+/* WRITE STREAM DMA EXT                 0x3A */
+/* WRITE STREAM EXT                     0x3B */
+#define WIN_WRITE_VERIFY                0x3C /* 28-Bit, obsolete since ATA4 */
+/* WRITE DMA FUA EXT                    0x3D */
+/* obsolete since ACS2                  0x3E */
+/* WRITE LOG EXT                        0x3F */
+#define WIN_VERIFY			0x40 /* 28-Bit - Read Verify Sectors */
+#define WIN_VERIFY_ONCE                 0x41 /* 28-Bit - w/o retries, obsolete since ATA5 */
+#define WIN_VERIFY_EXT			0x42 /* 48-Bit */
+/* reserved                             0x43..0x44 */
+/* WRITE UNCORRECTABLE EXT              0x45 */
+/* reserved                             0x46 */
+/* READ LOG DMA EXT                     0x47 */
+/* reserved                             0x48..0x4F */
+/* obsolete since ATA4                  0x50 */
+/* CONFIGURE STREAM                     0x51 */
+/* reserved                             0x52..0x56 */
+/* WRITE LOG DMA EXT                    0x57 */
+/* reserved                             0x58..0x5A */
+/* TRUSTED NON DATA                     0x5B */
+/* TRUSTED RECEIVE                      0x5C */
+/* TRUSTED RECEIVE DMA                  0x5D */
+/* TRUSTED SEND                         0x5E */
+/* TRUSTED SEND DMA                     0x5F */
+/* READ FPDMA QUEUED                    0x60 */
+/* WRITE FPDMA QUEUED                   0x61 */
+/* reserved                             0x62->0x6F */
+#define WIN_SEEK                        0x70 /* obsolete since ATA7 */
+/* reserved                             0x71-0x7F */
+/* vendor specific                      0x80-0x86 */
+#define CFA_TRANSLATE_SECTOR		0x87 /* CFA Translate Sector */
+/* vendor specific                      0x88-0x8F */
+#define WIN_DIAGNOSE			0x90
+#define WIN_SPECIFY                     0x91 /* set drive geometry translation, obsolete since ATA6 */
+#define WIN_DOWNLOAD_MICROCODE		0x92
+/* DOWNLOAD MICROCODE DMA               0x93 */
+#define WIN_STANDBYNOW2                 0x94 /* retired in ATA4 */
+#define WIN_IDLEIMMEDIATE2              0x95 /* force drive to become "ready", retired in ATA4 */
+#define WIN_STANDBY2                    0x96 /* retired in ATA4 */
+#define WIN_SETIDLE2                    0x97 /* retired in ATA4 */
+#define WIN_CHECKPOWERMODE2             0x98 /* retired in ATA4 */
+#define WIN_SLEEPNOW2                   0x99 /* retired in ATA4 */
+/* vendor specific                      0x9A */
+/* reserved                             0x9B..0x9F */
+#define WIN_PACKETCMD			0xA0 /* Send a packet command. */
+#define WIN_PIDENTIFY			0xA1 /* identify ATAPI device	*/
+#define WIN_QUEUED_SERVICE              0xA2 /* obsolete since ACS2 */
+/* reserved                             0xA3..0xAF */
+#define WIN_SMART			0xB0 /* self-monitoring and reporting */
+/* Device Configuration Overlay         0xB1 */
+/* reserved                             0xB2..0xB3 */
+/* Sanitize Device                      0xB4 */
+/* reserved                             0xB5 */
+/* NV Cache                             0xB6 */
+/* reserved for CFA                     0xB7..0xBB */
+#define CFA_ACCESS_METADATA_STORAGE	0xB8
+/* reserved                             0xBC..0xBF */
+#define CFA_ERASE_SECTORS       	0xC0 /* microdrives implement as NOP */
+/* vendor specific                      0xC1..0xC3 */
+#define WIN_MULTREAD			0xC4 /* read sectors using multiple mode*/
+#define WIN_MULTWRITE			0xC5 /* write sectors using multiple mode */
+#define WIN_SETMULT			0xC6 /* enable/disable multiple mode */
+#define WIN_READDMA_QUEUED              0xC7 /* read sectors using Queued DMA transfers, obsolete since ACS2 */
+#define WIN_READDMA			0xC8 /* read sectors using DMA transfers */
+#define WIN_READDMA_ONCE                0xC9 /* 28-Bit - w/o retries, obsolete since ATA5 */
+#define WIN_WRITEDMA			0xCA /* write sectors using DMA transfers */
+#define WIN_WRITEDMA_ONCE               0xCB /* 28-Bit - w/o retries, obsolete since ATA5 */
+#define WIN_WRITEDMA_QUEUED		0xCC /* write sectors using Queued DMA transfers, obsolete since ACS2 */
+#define CFA_WRITE_MULTI_WO_ERASE	0xCD /* CFA Write multiple without erase */
+/* WRITE MULTIPLE FUA EXT               0xCE */
+/* reserved                             0xCF..0xDO */
+/* CHECK MEDIA CARD TYPE                0xD1 */
+/* reserved for media card pass through 0xD2..0xD4 */
+/* reserved                             0xD5..0xD9 */
+#define WIN_GETMEDIASTATUS              0xDA /* obsolete since ATA8 */
+/* obsolete since ATA3, retired in ATA4 0xDB..0xDD */
+#define WIN_DOORLOCK                    0xDE /* lock door on removable drives, obsolete since ATA8 */
+#define WIN_DOORUNLOCK                  0xDF /* unlock door on removable drives, obsolete since ATA8 */
+#define WIN_STANDBYNOW1			0xE0
+#define WIN_IDLEIMMEDIATE		0xE1 /* force drive to become "ready" */
+#define WIN_STANDBY             	0xE2 /* Set device in Standby Mode */
+#define WIN_SETIDLE1			0xE3
+#define WIN_READ_BUFFER			0xE4 /* force read only 1 sector */
+#define WIN_CHECKPOWERMODE1		0xE5
+#define WIN_SLEEPNOW1			0xE6
+#define WIN_FLUSH_CACHE			0xE7
+#define WIN_WRITE_BUFFER		0xE8 /* force write only 1 sector */
+/* READ BUFFER DMA                      0xE9 */
+#define WIN_FLUSH_CACHE_EXT		0xEA /* 48-Bit */
+/* WRITE BUFFER DMA                     0xEB */
+#define WIN_IDENTIFY			0xEC /* ask drive to identify itself	*/
+#define WIN_MEDIAEJECT                  0xED /* obsolete since ATA8 */
+/* obsolete since ATA4                  0xEE */
+#define WIN_SETFEATURES			0xEF /* set special drive features */
+#define IBM_SENSE_CONDITION             0xF0 /* measure disk temperature, vendor specific */
+#define WIN_SECURITY_SET_PASS		0xF1
+#define WIN_SECURITY_UNLOCK		0xF2
+#define WIN_SECURITY_ERASE_PREPARE	0xF3
+#define WIN_SECURITY_ERASE_UNIT		0xF4
+#define WIN_SECURITY_FREEZE_LOCK	0xF5
+#define CFA_WEAR_LEVEL                  0xF5 /* microdrives implement as NOP; not specified in T13! */
+#define WIN_SECURITY_DISABLE		0xF6
+/* vendor specific                      0xF7 */
+#define WIN_READ_NATIVE_MAX		0xF8 /* return the native maximum address */
+#define WIN_SET_MAX			0xF9
+/* vendor specific                      0xFA..0xFF */
+
+/* set to 1 set disable mult support */
+#define MAX_MULT_SECTORS 16
+
+#define IDE_DMA_BUF_SECTORS 256
+
+/* feature values for Data Set Management */
+#define DSM_TRIM                        0x01
+
+#if (IDE_DMA_BUF_SECTORS < MAX_MULT_SECTORS)
+#error "IDE_DMA_BUF_SECTORS must be bigger or equal to MAX_MULT_SECTORS"
+#endif
+
+/* ATAPI defines */
+
+#define ATAPI_PACKET_SIZE 12
+
+/* The generic packet command opcodes for CD/DVD Logical Units,
+ * From Table 57 of the SFF8090 Ver. 3 (Mt. Fuji) draft standard. */
+#define GPCMD_BLANK			    0xa1
+#define GPCMD_CLOSE_TRACK		    0x5b
+#define GPCMD_FLUSH_CACHE		    0x35
+#define GPCMD_FORMAT_UNIT		    0x04
+#define GPCMD_GET_CONFIGURATION		    0x46
+#define GPCMD_GET_EVENT_STATUS_NOTIFICATION 0x4a
+#define GPCMD_GET_PERFORMANCE		    0xac
+#define GPCMD_INQUIRY			    0x12
+#define GPCMD_LOAD_UNLOAD		    0xa6
+#define GPCMD_MECHANISM_STATUS		    0xbd
+#define GPCMD_MODE_SELECT_10		    0x55
+#define GPCMD_MODE_SENSE_10		    0x5a
+#define GPCMD_PAUSE_RESUME		    0x4b
+#define GPCMD_PLAY_AUDIO_10		    0x45
+#define GPCMD_PLAY_AUDIO_MSF		    0x47
+#define GPCMD_PLAY_AUDIO_TI		    0x48
+#define GPCMD_PLAY_CD			    0xbc
+#define GPCMD_PREVENT_ALLOW_MEDIUM_REMOVAL  0x1e
+#define GPCMD_READ_10			    0x28
+#define GPCMD_READ_12			    0xa8
+#define GPCMD_READ_CDVD_CAPACITY	    0x25
+#define GPCMD_READ_CD			    0xbe
+#define GPCMD_READ_CD_MSF		    0xb9
+#define GPCMD_READ_DISC_INFO		    0x51
+#define GPCMD_READ_DVD_STRUCTURE	    0xad
+#define GPCMD_READ_FORMAT_CAPACITIES	    0x23
+#define GPCMD_READ_HEADER		    0x44
+#define GPCMD_READ_TRACK_RZONE_INFO	    0x52
+#define GPCMD_READ_SUBCHANNEL		    0x42
+#define GPCMD_READ_TOC_PMA_ATIP		    0x43
+#define GPCMD_REPAIR_RZONE_TRACK	    0x58
+#define GPCMD_REPORT_KEY		    0xa4
+#define GPCMD_REQUEST_SENSE		    0x03
+#define GPCMD_RESERVE_RZONE_TRACK	    0x53
+#define GPCMD_SCAN			    0xba
+#define GPCMD_SEEK			    0x2b
+#define GPCMD_SEND_DVD_STRUCTURE	    0xad
+#define GPCMD_SEND_EVENT		    0xa2
+#define GPCMD_SEND_KEY			    0xa3
+#define GPCMD_SEND_OPC			    0x54
+#define GPCMD_SET_READ_AHEAD		    0xa7
+#define GPCMD_SET_STREAMING		    0xb6
+#define GPCMD_START_STOP_UNIT		    0x1b
+#define GPCMD_STOP_PLAY_SCAN		    0x4e
+#define GPCMD_TEST_UNIT_READY		    0x00
+#define GPCMD_VERIFY_10			    0x2f
+#define GPCMD_WRITE_10			    0x2a
+#define GPCMD_WRITE_AND_VERIFY_10	    0x2e
+/* This is listed as optional in ATAPI 2.6, but is (curiously)
+ * missing from Mt. Fuji, Table 57.  It _is_ mentioned in Mt. Fuji
+ * Table 377 as an MMC command for SCSi devices though...  Most ATAPI
+ * drives support it. */
+#define GPCMD_SET_SPEED			    0xbb
+/* This seems to be a SCSI specific CD-ROM opcode
+ * to play data at track/index */
+#define GPCMD_PLAYAUDIO_TI		    0x48
+/*
+ * From MS Media Status Notification Support Specification. For
+ * older drives only.
+ */
+#define GPCMD_GET_MEDIA_STATUS		    0xda
+#define GPCMD_MODE_SENSE_6		    0x1a
+
+#define ATAPI_INT_REASON_CD             0x01 /* 0 = data transfer */
+#define ATAPI_INT_REASON_IO             0x02 /* 1 = transfer to the host */
+#define ATAPI_INT_REASON_REL            0x04
+#define ATAPI_INT_REASON_TAG            0xf8
+
+/* same constants as bochs */
+#define ASC_NO_SEEK_COMPLETE                 0x02
+#define ASC_ILLEGAL_OPCODE                   0x20
+#define ASC_LOGICAL_BLOCK_OOR                0x21
+#define ASC_INV_FIELD_IN_CMD_PACKET          0x24
+#define ASC_MEDIUM_MAY_HAVE_CHANGED          0x28
+#define ASC_INCOMPATIBLE_FORMAT              0x30
+#define ASC_MEDIUM_NOT_PRESENT               0x3a
+#define ASC_SAVING_PARAMETERS_NOT_SUPPORTED  0x39
+#define ASC_DATA_PHASE_ERROR                 0x4b
+#define ASC_MEDIA_REMOVAL_PREVENTED          0x53
+
+#define CFA_NO_ERROR            0x00
+#define CFA_MISC_ERROR          0x09
+#define CFA_INVALID_COMMAND     0x20
+#define CFA_INVALID_ADDRESS     0x21
+#define CFA_ADDRESS_OVERFLOW    0x2f
+
+#define SMART_READ_DATA       0xd0
+#define SMART_READ_THRESH     0xd1
+#define SMART_ATTR_AUTOSAVE   0xd2
+#define SMART_SAVE_ATTR       0xd3
+#define SMART_EXECUTE_OFFLINE 0xd4
+#define SMART_READ_LOG        0xd5
+#define SMART_WRITE_LOG       0xd6
+#define SMART_ENABLE          0xd8
+#define SMART_DISABLE         0xd9
+#define SMART_STATUS          0xda
+
+typedef enum { IDE_HD, IDE_CD, IDE_CFATA } IDEDriveKind;
+
+typedef void EndTransferFunc(IDEState *);
+
+typedef void DMAStartFunc(IDEDMA *, IDEState *, BlockCompletionFunc *);
+typedef void DMAVoidFunc(IDEDMA *);
+typedef int DMAIntFunc(IDEDMA *, int);
+typedef int32_t DMAInt32Func(IDEDMA *, int32_t len);
+typedef void DMAu32Func(IDEDMA *, uint32_t);
+typedef void DMAStopFunc(IDEDMA *, bool);
+typedef void DMARestartFunc(void *, int, RunState);
+
+struct unreported_events {
+    bool eject_request;
+    bool new_media;
+};
+
+enum ide_dma_cmd {
+    IDE_DMA_READ,
+    IDE_DMA_WRITE,
+    IDE_DMA_TRIM,
+    IDE_DMA_ATAPI,
+};
+
+#define ide_cmd_is_read(s) \
+	((s)->dma_cmd == IDE_DMA_READ)
+
+typedef struct IDEBufferedRequest {
+    QLIST_ENTRY(IDEBufferedRequest) list;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    QEMUIOVector *original_qiov;
+    BlockCompletionFunc *original_cb;
+    void *original_opaque;
+    bool orphaned;
+} IDEBufferedRequest;
+
+/* NOTE: IDEState represents in fact one drive */
+struct IDEState {
+    IDEBus *bus;
+    uint8_t unit;
+    /* ide config */
+    IDEDriveKind drive_kind;
+    int cylinders, heads, sectors, chs_trans;
+    int64_t nb_sectors;
+    int mult_sectors;
+    int identify_set;
+    uint8_t identify_data[512];
+    int drive_serial;
+    char drive_serial_str[21];
+    char drive_model_str[41];
+    uint64_t wwn;
+    /* ide regs */
+    uint8_t feature;
+    uint8_t error;
+    uint32_t nsector;
+    uint8_t sector;
+    uint8_t lcyl;
+    uint8_t hcyl;
+    /* other part of tf for lba48 support */
+    uint8_t hob_feature;
+    uint8_t hob_nsector;
+    uint8_t hob_sector;
+    uint8_t hob_lcyl;
+    uint8_t hob_hcyl;
+
+    uint8_t select;
+    uint8_t status;
+
+    /* set for lba48 access */
+    uint8_t lba48;
+    BlockBackend *blk;
+    char version[9];
+    /* ATAPI specific */
+    struct unreported_events events;
+    uint8_t sense_key;
+    uint8_t asc;
+    bool tray_open;
+    bool tray_locked;
+    uint8_t cdrom_changed;
+    int packet_transfer_size;
+    int elementary_transfer_size;
+    int32_t io_buffer_index;
+    int lba;
+    int cd_sector_size;
+    int atapi_dma; /* true if dma is requested for the packet cmd */
+    BlockAcctCookie acct;
+    BlockAIOCB *pio_aiocb;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    QLIST_HEAD(, IDEBufferedRequest) buffered_requests;
+    /* ATA DMA state */
+    uint64_t io_buffer_offset;
+    int32_t io_buffer_size;
+    QEMUSGList sg;
+    /* PIO transfer handling */
+    int req_nb_sectors; /* number of sectors per interrupt */
+    EndTransferFunc *end_transfer_func;
+    uint8_t *data_ptr;
+    uint8_t *data_end;
+    uint8_t *io_buffer;
+    /* PIO save/restore */
+    int32_t io_buffer_total_len;
+    int32_t cur_io_buffer_offset;
+    int32_t cur_io_buffer_len;
+    uint8_t end_transfer_fn_idx;
+    QEMUTimer *sector_write_timer; /* only used for win2k install hack */
+    uint32_t irq_count; /* counts IRQs when using win2k install hack */
+    /* CF-ATA extended error */
+    uint8_t ext_error;
+    /* CF-ATA metadata storage */
+    uint32_t mdata_size;
+    uint8_t *mdata_storage;
+    int media_changed;
+    enum ide_dma_cmd dma_cmd;
+    /* SMART */
+    uint8_t smart_enabled;
+    uint8_t smart_autosave;
+    int smart_errors;
+    uint8_t smart_selftest_count;
+    uint8_t *smart_selftest_data;
+    /* AHCI */
+    int ncq_queues;
+};
+
+struct IDEDMAOps {
+    DMAStartFunc *start_dma;
+    DMAVoidFunc *start_transfer;
+    DMAInt32Func *prepare_buf;
+    DMAu32Func *commit_buf;
+    DMAIntFunc *rw_buf;
+    DMAVoidFunc *restart;
+    DMAVoidFunc *restart_dma;
+    DMAStopFunc *set_inactive;
+    DMAVoidFunc *cmd_done;
+    DMAVoidFunc *reset;
+};
+
+struct IDEDMA {
+    const struct IDEDMAOps *ops;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    BlockAIOCB *aiocb;
+};
+
+struct IDEBus {
+    BusState qbus;
+    IDEDevice *master;
+    IDEDevice *slave;
+    IDEState ifs[2];
+    QEMUBH *bh;
+
+    int bus_id;
+    int max_units;
+    IDEDMA *dma;
+    uint8_t unit;
+    uint8_t cmd;
+    qemu_irq irq;
+
+    int error_status;
+    uint8_t retry_unit;
+    int64_t retry_sector_num;
+    uint32_t retry_nsector;
+};
+
+#define TYPE_IDE_DEVICE "ide-device"
+#define IDE_DEVICE(obj) \
+     OBJECT_CHECK(IDEDevice, (obj), TYPE_IDE_DEVICE)
+#define IDE_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(IDEDeviceClass, (klass), TYPE_IDE_DEVICE)
+#define IDE_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IDEDeviceClass, (obj), TYPE_IDE_DEVICE)
+
+typedef struct IDEDeviceClass {
+    DeviceClass parent_class;
+    int (*init)(IDEDevice *dev);
+} IDEDeviceClass;
+
+struct IDEDevice {
+    DeviceState qdev;
+    uint32_t unit;
+    BlockConf conf;
+    int chs_trans;
+    char *version;
+    char *serial;
+    char *model;
+    uint64_t wwn;
+};
+
+/* These are used for the error_status field of IDEBus */
+#define IDE_RETRY_MASK 0xf8
+#define IDE_RETRY_DMA  0x08
+#define IDE_RETRY_PIO  0x10
+#define IDE_RETRY_ATAPI 0x20 /* reused IDE_RETRY_READ bit */
+#define IDE_RETRY_READ  0x20
+#define IDE_RETRY_FLUSH 0x40
+#define IDE_RETRY_TRIM 0x80
+#define IDE_RETRY_HBA  0x100
+
+#define IS_IDE_RETRY_DMA(_status) \
+    ((_status) & IDE_RETRY_DMA)
+
+#define IS_IDE_RETRY_PIO(_status) \
+    ((_status) & IDE_RETRY_PIO)
+
+/*
+ * The method of the IDE_RETRY_ATAPI determination is to use a previously
+ * impossible bit combination as a new status value.
+ */
+#define IS_IDE_RETRY_ATAPI(_status)   \
+    (((_status) & IDE_RETRY_MASK) == IDE_RETRY_ATAPI)
+
+static inline uint8_t ide_dma_cmd_to_retry(uint8_t dma_cmd)
+{
+    switch (dma_cmd) {
+    case IDE_DMA_READ:
+        return IDE_RETRY_DMA | IDE_RETRY_READ;
+    case IDE_DMA_WRITE:
+        return IDE_RETRY_DMA;
+    case IDE_DMA_TRIM:
+        return IDE_RETRY_DMA | IDE_RETRY_TRIM;
+    case IDE_DMA_ATAPI:
+        return IDE_RETRY_ATAPI;
+    default:
+        break;
+    }
+    return 0;
+}
+
+static inline IDEState *idebus_active_if(IDEBus *bus)
+{
+    return bus->ifs + bus->unit;
+}
+
+static inline void ide_set_irq(IDEBus *bus)
+{
+    if (!(bus->cmd & IDE_CMD_DISABLE_IRQ)) {
+        qemu_irq_raise(bus->irq);
+    }
+}
+
+/* hw/ide/core.c */
+extern const VMStateDescription vmstate_ide_bus;
+
+#define VMSTATE_IDE_BUS(_field, _state)                          \
+    VMSTATE_STRUCT(_field, _state, 1, vmstate_ide_bus, IDEBus)
+
+#define VMSTATE_IDE_BUS_ARRAY(_field, _state, _num)              \
+    VMSTATE_STRUCT_ARRAY(_field, _state, _num, 1, vmstate_ide_bus, IDEBus)
+
+extern const VMStateDescription vmstate_ide_drive;
+
+#define VMSTATE_IDE_DRIVES(_field, _state) \
+    VMSTATE_STRUCT_ARRAY(_field, _state, 2, 3, vmstate_ide_drive, IDEState)
+
+#define VMSTATE_IDE_DRIVE(_field, _state) \
+    VMSTATE_STRUCT(_field, _state, 1, vmstate_ide_drive, IDEState)
+
+void ide_bus_reset(IDEBus *bus);
+int64_t ide_get_sector(IDEState *s);
+void ide_set_sector(IDEState *s, int64_t sector_num);
+
+void ide_start_dma(IDEState *s, BlockCompletionFunc *cb);
+void dma_buf_commit(IDEState *s, uint32_t tx_bytes);
+void ide_dma_error(IDEState *s);
+void ide_abort_command(IDEState *s);
+
+void ide_atapi_cmd_ok(IDEState *s);
+void ide_atapi_cmd_error(IDEState *s, int sense_key, int asc);
+void ide_atapi_dma_restart(IDEState *s);
+void ide_atapi_io_error(IDEState *s, int ret);
+
+void ide_ioport_write(void *opaque, uint32_t addr, uint32_t val);
+uint32_t ide_ioport_read(void *opaque, uint32_t addr1);
+uint32_t ide_status_read(void *opaque, uint32_t addr);
+void ide_cmd_write(void *opaque, uint32_t addr, uint32_t val);
+void ide_data_writew(void *opaque, uint32_t addr, uint32_t val);
+uint32_t ide_data_readw(void *opaque, uint32_t addr);
+void ide_data_writel(void *opaque, uint32_t addr, uint32_t val);
+uint32_t ide_data_readl(void *opaque, uint32_t addr);
+
+int ide_init_drive(IDEState *s, BlockBackend *blk, IDEDriveKind kind,
+                   const char *version, const char *serial, const char *model,
+                   uint64_t wwn,
+                   uint32_t cylinders, uint32_t heads, uint32_t secs,
+                   int chs_trans);
+void ide_init2(IDEBus *bus, qemu_irq irq);
+void ide_init_ioport(IDEBus *bus, ISADevice *isa, int iobase, int iobase2);
+void ide_register_restart_cb(IDEBus *bus);
+
+void ide_exec_cmd(IDEBus *bus, uint32_t val);
+
+void ide_transfer_start(IDEState *s, uint8_t *buf, int size,
+                        EndTransferFunc *end_transfer_func);
+void ide_transfer_stop(IDEState *s);
+void ide_set_inactive(IDEState *s, bool more);
+BlockAIOCB *ide_issue_trim(
+        int64_t offset, QEMUIOVector *qiov,
+        BlockCompletionFunc *cb, void *cb_opaque, void *opaque);
+BlockAIOCB *ide_buffered_readv(IDEState *s, int64_t sector_num,
+                               QEMUIOVector *iov, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque);
+void ide_cancel_dma_sync(IDEState *s);
+
+/* hw/ide/atapi.c */
+void ide_atapi_cmd(IDEState *s);
+void ide_atapi_cmd_reply_end(IDEState *s);
+
+/* hw/ide/qdev.c */
+void ide_bus_new(IDEBus *idebus, size_t idebus_size, DeviceState *dev,
+                 int bus_id, int max_units);
+IDEDevice *ide_create_drive(IDEBus *bus, int unit, DriveInfo *drive);
+
+int ide_handle_rw_error(IDEState *s, int error, int op);
+
+#endif /* HW_IDE_INTERNAL_H */
diff --git a/include/hw/ide/pci.h b/include/hw/ide/pci.h
new file mode 100644
index 0000000..dbc6a03
--- /dev/null
+++ b/include/hw/ide/pci.h
@@ -0,0 +1,76 @@
+#ifndef HW_IDE_PCI_H
+#define HW_IDE_PCI_H
+
+#include "hw/ide/internal.h"
+
+#define BM_STATUS_DMAING 0x01
+#define BM_STATUS_ERROR  0x02
+#define BM_STATUS_INT    0x04
+
+#define BM_CMD_START     0x01
+#define BM_CMD_READ      0x08
+
+typedef struct BMDMAState {
+    IDEDMA dma;
+    uint8_t cmd;
+    uint8_t status;
+    uint32_t addr;
+
+    IDEBus *bus;
+    /* current transfer state */
+    uint32_t cur_addr;
+    uint32_t cur_prd_last;
+    uint32_t cur_prd_addr;
+    uint32_t cur_prd_len;
+    BlockCompletionFunc *dma_cb;
+    MemoryRegion addr_ioport;
+    MemoryRegion extra_io;
+    qemu_irq irq;
+
+    /* Bit 0-2 and 7:   BM status register
+     * Bit 3-6:         bus->error_status */
+    uint8_t migration_compat_status;
+    uint8_t migration_retry_unit;
+    int64_t migration_retry_sector_num;
+    uint32_t migration_retry_nsector;
+
+    struct PCIIDEState *pci_dev;
+} BMDMAState;
+
+typedef struct CMD646BAR {
+    MemoryRegion cmd;
+    MemoryRegion data;
+    IDEBus *bus;
+    struct PCIIDEState *pci_dev;
+} CMD646BAR;
+
+#define TYPE_PCI_IDE "pci-ide"
+#define PCI_IDE(obj) OBJECT_CHECK(PCIIDEState, (obj), TYPE_PCI_IDE)
+
+typedef struct PCIIDEState {
+    /*< private >*/
+    PCIDevice parent_obj;
+    /*< public >*/
+
+    IDEBus bus[2];
+    BMDMAState bmdma[2];
+    uint32_t secondary; /* used only for cmd646 */
+    MemoryRegion bmdma_bar;
+    CMD646BAR cmd646_bar[2]; /* used only for cmd646 */
+} PCIIDEState;
+
+
+static inline IDEState *bmdma_active_if(BMDMAState *bmdma)
+{
+    assert(bmdma->bus->retry_unit != (uint8_t)-1);
+    return bmdma->bus->ifs + bmdma->bus->retry_unit;
+}
+
+
+void bmdma_init(IDEBus *bus, BMDMAState *bm, PCIIDEState *d);
+void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val);
+extern MemoryRegionOps bmdma_addr_ioport_ops;
+void pci_ide_create_devs(PCIDevice *dev, DriveInfo **hd_table);
+
+extern const VMStateDescription vmstate_ide_pci;
+#endif
diff --git a/include/hw/input/adb.h b/include/hw/input/adb.h
index bdfccd4..3ae8445 100644
--- a/include/hw/input/adb.h
+++ b/include/hw/input/adb.h
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
-#if !defined(__ADB_H__)
-#define __ADB_H__
+#ifndef ADB_H
+#define ADB_H
 
 #include "hw/qdev.h"
 
@@ -79,9 +79,9 @@
 
 int adb_request(ADBBusState *s, uint8_t *buf_out,
                 const uint8_t *buf, int len);
-int adb_poll(ADBBusState *s, uint8_t *buf_out);
+int adb_poll(ADBBusState *s, uint8_t *buf_out, uint16_t poll_mask);
 
 #define TYPE_ADB_KEYBOARD "adb-keyboard"
 #define TYPE_ADB_MOUSE "adb-mouse"
 
-#endif /* !defined(__ADB_H__) */
+#endif /* ADB_H */
diff --git a/include/hw/input/goldfish_sensors.h b/include/hw/input/goldfish_sensors.h
deleted file mode 100644
index e5f0257..0000000
--- a/include/hw/input/goldfish_sensors.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Goldfish Sensors public declarations.
- *
- * Copyright (C) 2014 Alex Bennée <alex.bennee@linaor.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- */
-
-#ifndef HW_INPUT_GOLDFISH_SENSORS_FB_H
-#define HW_INPUT_GOLDFISH_SENSORS_FB_H
-
-void goldfish_sensors_set_rotation(int rotation);
-
-#endif /* HW_INPUT_GOLDFISH_SENSORS_FB_H */
diff --git a/include/hw/input/ps2.h b/include/hw/input/ps2.h
index 7c45ce7..b9ceee4 100644
--- a/include/hw/input/ps2.h
+++ b/include/hw/input/ps2.h
@@ -35,4 +35,4 @@
 void ps2_keyboard_set_translation(void *opaque, int mode);
 void ps2_mouse_fake_event(void *opaque);
 
-#endif /* !HW_PS2_H */
+#endif /* HW_PS2_H */
diff --git a/include/hw/intc/allwinner-a10-pic.h b/include/hw/intc/allwinner-a10-pic.h
index 5721b2e..1d314a7 100644
--- a/include/hw/intc/allwinner-a10-pic.h
+++ b/include/hw/intc/allwinner-a10-pic.h
@@ -1,5 +1,5 @@
-#ifndef AW_A10_PIC_H
-#define AW_A10_PIC_H
+#ifndef ALLWINNER_A10_PIC_H
+#define ALLWINNER_A10_PIC_H
 
 #define TYPE_AW_A10_PIC  "allwinner-a10-pic"
 #define AW_A10_PIC(obj) OBJECT_CHECK(AwA10PICState, (obj), TYPE_AW_A10_PIC)
diff --git a/include/hw/intc/arm_gic.h b/include/hw/intc/arm_gic.h
index 0971e37..42bb535 100644
--- a/include/hw/intc/arm_gic.h
+++ b/include/hw/intc/arm_gic.h
@@ -23,6 +23,9 @@
 
 #include "arm_gic_common.h"
 
+/* Number of SGI target-list bits */
+#define GIC_TARGETLIST_BITS 8
+
 #define TYPE_ARM_GIC "arm_gic"
 #define ARM_GIC(obj) \
      OBJECT_CHECK(GICState, (obj), TYPE_ARM_GIC)
diff --git a/include/hw/intc/arm_gic_common.h b/include/hw/intc/arm_gic_common.h
index f6887ed..f4c349a 100644
--- a/include/hw/intc/arm_gic_common.h
+++ b/include/hw/intc/arm_gic_common.h
@@ -34,6 +34,9 @@
 #define MAX_NR_GROUP_PRIO 128
 #define GIC_NR_APRS (MAX_NR_GROUP_PRIO / 32)
 
+#define GIC_MIN_BPR 0
+#define GIC_MIN_ABPR (GIC_MIN_BPR + 1)
+
 typedef struct gic_irq_state {
     /* The enable bits are only banked for per-cpu interrupts.  */
     uint8_t enabled;
@@ -42,6 +45,7 @@
     uint8_t level;
     bool model; /* 0 = N:N, 1 = 1:N */
     bool edge_trigger; /* true: edge-triggered, false: level-triggered  */
+    uint8_t group;
 } gic_irq_state;
 
 typedef struct GICState {
@@ -50,14 +54,20 @@
     /*< public >*/
 
     qemu_irq parent_irq[GIC_NCPU];
-    bool enabled;
-    bool cpu_enabled[GIC_NCPU];
+    qemu_irq parent_fiq[GIC_NCPU];
+    /* GICD_CTLR; for a GIC with the security extensions the NS banked version
+     * of this register is just an alias of bit 1 of the S banked version.
+     */
+    uint32_t ctlr;
+    /* GICC_CTLR; again, the NS banked version is just aliases of bits of
+     * the S banked register, so our state only needs to store the S version.
+     */
+    uint32_t cpu_ctlr[GIC_NCPU];
 
     gic_irq_state irq_state[GIC_MAXIRQ];
     uint8_t irq_target[GIC_MAXIRQ];
     uint8_t priority1[GIC_INTERNAL][GIC_NCPU];
     uint8_t priority2[GIC_MAXIRQ - GIC_INTERNAL];
-    uint16_t last_active[GIC_MAXIRQ][GIC_NCPU];
     /* For each SGI on the target CPU, we store 8 bits
      * indicating which source CPUs have made this SGI
      * pending on the target CPU. These correspond to
@@ -67,13 +77,14 @@
     uint8_t sgi_pending[GIC_NR_SGIS][GIC_NCPU];
 
     uint16_t priority_mask[GIC_NCPU];
-    uint16_t running_irq[GIC_NCPU];
     uint16_t running_priority[GIC_NCPU];
     uint16_t current_pending[GIC_NCPU];
 
-    /* We present the GICv2 without security extensions to a guest and
-     * therefore the guest can configure the GICC_CTLR to configure group 1
-     * binary point in the abpr.
+    /* If we present the GICv2 without security extensions to a guest,
+     * the guest can configure the GICC_CTLR to configure group 1 binary point
+     * in the abpr.
+     * For a GIC with Security Extensions we use use bpr for the
+     * secure copy and abpr as storage for the non-secure copy of the register.
      */
     uint8_t  bpr[GIC_NCPU];
     uint8_t  abpr[GIC_NCPU];
@@ -83,16 +94,9 @@
      * If an interrupt for preemption level X is active, then
      *   APRn[X mod 32] == 0b1,  where n = X / 32
      * otherwise the bit is clear.
-     *
-     * TODO: rewrite the interrupt acknowlege/complete routines to use
-     * the APR registers to track the necessary information to update
-     * s->running_priority[] on interrupt completion (ie completely remove
-     * last_active[][] and running_irq[]). This will be necessary if we ever
-     * want to support TCG<->KVM migration, or TCG guests which can
-     * do power management involving powering down and restarting
-     * the GIC.
      */
     uint32_t apr[GIC_NR_APRS][GIC_NCPU];
+    uint32_t nsapr[GIC_NR_APRS][GIC_NCPU];
 
     uint32_t num_cpu;
 
@@ -104,7 +108,10 @@
     MemoryRegion cpuiomem[GIC_NCPU + 1]; /* CPU interfaces */
     uint32_t num_irq;
     uint32_t revision;
+    bool security_extn;
+    bool irq_reset_nonsecure; /* configure IRQs as group 1 (NS) on reset? */
     int dev_fd; /* kvm device fd if backed by kvm vgic support */
+    Error *migration_blocker;
 } GICState;
 
 #define TYPE_ARM_GIC_COMMON "arm_gic_common"
@@ -124,4 +131,7 @@
     void (*post_load)(GICState *s);
 } ARMGICCommonClass;
 
+void gic_init_irqs_and_mmio(GICState *s, qemu_irq_handler handler,
+                            const MemoryRegionOps *ops);
+
 #endif
diff --git a/include/hw/intc/arm_gicv3.h b/include/hw/intc/arm_gicv3.h
new file mode 100644
index 0000000..4a6fd85
--- /dev/null
+++ b/include/hw/intc/arm_gicv3.h
@@ -0,0 +1,32 @@
+/*
+ * ARM Generic Interrupt Controller v3
+ *
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2016 Linaro Limited
+ * Written by Shlomo Pongratz, Peter Maydell
+ *
+ * This code is licensed under the GPL, version 2 or (at your option)
+ * any later version.
+ */
+
+#ifndef HW_ARM_GICV3_H
+#define HW_ARM_GICV3_H
+
+#include "arm_gicv3_common.h"
+
+#define TYPE_ARM_GICV3 "arm-gicv3"
+#define ARM_GICV3(obj) OBJECT_CHECK(GICv3State, (obj), TYPE_ARM_GICV3)
+#define ARM_GICV3_CLASS(klass) \
+     OBJECT_CLASS_CHECK(ARMGICv3Class, (klass), TYPE_ARM_GICV3)
+#define ARM_GICV3_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(ARMGICv3Class, (obj), TYPE_ARM_GICV3)
+
+typedef struct ARMGICv3Class {
+    /*< private >*/
+    ARMGICv3CommonClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+} ARMGICv3Class;
+
+#endif
diff --git a/include/hw/intc/arm_gicv3_common.h b/include/hw/intc/arm_gicv3_common.h
new file mode 100644
index 0000000..341a311
--- /dev/null
+++ b/include/hw/intc/arm_gicv3_common.h
@@ -0,0 +1,274 @@
+/*
+ * ARM GIC support
+ *
+ * Copyright (c) 2012 Linaro Limited
+ * Copyright (c) 2015 Huawei.
+ * Copyright (c) 2015 Samsung Electronics Co., Ltd.
+ * Written by Peter Maydell
+ * Reworked for GICv3 by Shlomo Pongratz and Pavel Fedin
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HW_ARM_GICV3_COMMON_H
+#define HW_ARM_GICV3_COMMON_H
+
+#include "hw/sysbus.h"
+#include "hw/intc/arm_gic_common.h"
+
+/*
+ * Maximum number of possible interrupts, determined by the GIC architecture.
+ * Note that this does not include LPIs. When implemented, these should be
+ * dealt with separately.
+ */
+#define GICV3_MAXIRQ 1020
+#define GICV3_MAXSPI (GICV3_MAXIRQ - GIC_INTERNAL)
+
+/* Number of SGI target-list bits */
+#define GICV3_TARGETLIST_BITS 16
+
+/* Minimum BPR for Secure, or when security not enabled */
+#define GIC_MIN_BPR 0
+/* Minimum BPR for Nonsecure when security is enabled */
+#define GIC_MIN_BPR_NS (GIC_MIN_BPR + 1)
+
+/* For some distributor fields we want to model the array of 32-bit
+ * register values which hold various bitmaps corresponding to enabled,
+ * pending, etc bits. These macros and functions facilitate that; the
+ * APIs are generally modelled on the generic bitmap.h functions
+ * (which are unsuitable here because they use 'unsigned long' as the
+ * underlying storage type, which is very awkward when you need to
+ * access the data as 32-bit values.)
+ * Each bitmap contains a bit for each interrupt. Although there is
+ * space for the PPIs and SGIs, those bits (the first 32) are never
+ * used as that state lives in the redistributor. The unused bits are
+ * provided purely so that interrupt X's state is always in bit X; this
+ * avoids bugs where we forget to subtract GIC_INTERNAL from an
+ * interrupt number.
+ */
+#define GICV3_BMP_SIZE (DIV_ROUND_UP(GICV3_MAXIRQ, 32))
+
+#define GIC_DECLARE_BITMAP(name) \
+    uint32_t name[GICV3_BMP_SIZE]
+
+#define GIC_BIT_MASK(nr) (1U << ((nr) % 32))
+#define GIC_BIT_WORD(nr) ((nr) / 32)
+
+static inline void gic_bmp_set_bit(int nr, uint32_t *addr)
+{
+    uint32_t mask = GIC_BIT_MASK(nr);
+    uint32_t *p = addr + GIC_BIT_WORD(nr);
+
+    *p |= mask;
+}
+
+static inline void gic_bmp_clear_bit(int nr, uint32_t *addr)
+{
+    uint32_t mask = GIC_BIT_MASK(nr);
+    uint32_t *p = addr + GIC_BIT_WORD(nr);
+
+    *p &= ~mask;
+}
+
+static inline int gic_bmp_test_bit(int nr, const uint32_t *addr)
+{
+    return 1U & (addr[GIC_BIT_WORD(nr)] >> (nr & 31));
+}
+
+static inline void gic_bmp_replace_bit(int nr, uint32_t *addr, int val)
+{
+    uint32_t mask = GIC_BIT_MASK(nr);
+    uint32_t *p = addr + GIC_BIT_WORD(nr);
+
+    *p &= ~mask;
+    *p |= (val & 1U) << (nr % 32);
+}
+
+/* Return a pointer to the 32-bit word containing the specified bit. */
+static inline uint32_t *gic_bmp_ptr32(uint32_t *addr, int nr)
+{
+    return addr + GIC_BIT_WORD(nr);
+}
+
+typedef struct GICv3State GICv3State;
+typedef struct GICv3CPUState GICv3CPUState;
+
+/* Some CPU interface registers come in three flavours:
+ * Group0, Group1 (Secure) and Group1 (NonSecure)
+ * (where the latter two are exposed as a single banked system register).
+ * In the state struct they are implemented as a 3-element array which
+ * can be indexed into by the GICV3_G0, GICV3_G1 and GICV3_G1NS constants.
+ * If the CPU doesn't support EL3 then the G1 element is unused.
+ *
+ * These constants are also used to communicate the group to use for
+ * an interrupt or SGI when it is passed between the cpu interface and
+ * the redistributor or distributor. For those purposes the receiving end
+ * must be prepared to cope with a Group 1 Secure interrupt even if it does
+ * not have security support enabled, because security can be disabled
+ * independently in the CPU and in the GIC. In that case the receiver should
+ * treat an incoming Group 1 Secure interrupt as if it were Group 0.
+ * (This architectural requirement is why the _G1 element is the unused one
+ * in a no-EL3 CPU:  we would otherwise have to translate back and forth
+ * between (G0, G1NS) from the distributor and (G0, G1) in the CPU i/f.)
+ */
+#define GICV3_G0 0
+#define GICV3_G1 1
+#define GICV3_G1NS 2
+
+/* ICC_CTLR_EL1, GICD_STATUSR and GICR_STATUSR are banked but not
+ * group-related, so those indices are just 0 for S and 1 for NS.
+ * (If the CPU or the GIC, respectively, don't support the Security
+ * extensions then the S element is unused.)
+ */
+#define GICV3_S 0
+#define GICV3_NS 1
+
+typedef struct {
+    int irq;
+    uint8_t prio;
+    int grp;
+} PendingIrq;
+
+struct GICv3CPUState {
+    GICv3State *gic;
+    CPUState *cpu;
+    qemu_irq parent_irq;
+    qemu_irq parent_fiq;
+
+    /* Redistributor */
+    uint32_t level;                  /* Current IRQ level */
+    /* RD_base page registers */
+    uint32_t gicr_ctlr;
+    uint64_t gicr_typer;
+    uint32_t gicr_statusr[2];
+    uint32_t gicr_waker;
+    uint64_t gicr_propbaser;
+    uint64_t gicr_pendbaser;
+    /* SGI_base page registers */
+    uint32_t gicr_igroupr0;
+    uint32_t gicr_ienabler0;
+    uint32_t gicr_ipendr0;
+    uint32_t gicr_iactiver0;
+    uint32_t edge_trigger; /* ICFGR0 and ICFGR1 even bits */
+    uint32_t gicr_igrpmodr0;
+    uint32_t gicr_nsacr;
+    uint8_t gicr_ipriorityr[GIC_INTERNAL];
+
+    /* CPU interface */
+    uint64_t icc_ctlr_el1[2];
+    uint64_t icc_pmr_el1;
+    uint64_t icc_bpr[3];
+    uint64_t icc_apr[3][4];
+    uint64_t icc_igrpen[3];
+    uint64_t icc_ctlr_el3;
+
+    /* Current highest priority pending interrupt for this CPU.
+     * This is cached information that can be recalculated from the
+     * real state above; it doesn't need to be migrated.
+     */
+    PendingIrq hppi;
+    /* This is temporary working state, to avoid a malloc in gicv3_update() */
+    bool seenbetter;
+};
+
+struct GICv3State {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    MemoryRegion iomem_dist; /* Distributor */
+    MemoryRegion iomem_redist; /* Redistributors */
+
+    uint32_t num_cpu;
+    uint32_t num_irq;
+    uint32_t revision;
+    bool security_extn;
+    bool irq_reset_nonsecure;
+
+    int dev_fd; /* kvm device fd if backed by kvm vgic support */
+    Error *migration_blocker;
+
+    /* Distributor */
+
+    /* for a GIC with the security extensions the NS banked version of this
+     * register is just an alias of bit 1 of the S banked version.
+     */
+    uint32_t gicd_ctlr;
+    uint32_t gicd_statusr[2];
+    GIC_DECLARE_BITMAP(group);        /* GICD_IGROUPR */
+    GIC_DECLARE_BITMAP(grpmod);       /* GICD_IGRPMODR */
+    GIC_DECLARE_BITMAP(enabled);      /* GICD_ISENABLER */
+    GIC_DECLARE_BITMAP(pending);      /* GICD_ISPENDR */
+    GIC_DECLARE_BITMAP(active);       /* GICD_ISACTIVER */
+    GIC_DECLARE_BITMAP(level);        /* Current level */
+    GIC_DECLARE_BITMAP(edge_trigger); /* GICD_ICFGR even bits */
+    uint8_t gicd_ipriority[GICV3_MAXIRQ];
+    uint64_t gicd_irouter[GICV3_MAXIRQ];
+    /* Cached information: pointer to the cpu i/f for the CPUs specified
+     * in the IROUTER registers
+     */
+    GICv3CPUState *gicd_irouter_target[GICV3_MAXIRQ];
+    uint32_t gicd_nsacr[DIV_ROUND_UP(GICV3_MAXIRQ, 16)];
+
+    GICv3CPUState *cpu;
+};
+
+#define GICV3_BITMAP_ACCESSORS(BMP)                                     \
+    static inline void gicv3_gicd_##BMP##_set(GICv3State *s, int irq)   \
+    {                                                                   \
+        gic_bmp_set_bit(irq, s->BMP);                                   \
+    }                                                                   \
+    static inline int gicv3_gicd_##BMP##_test(GICv3State *s, int irq)   \
+    {                                                                   \
+        return gic_bmp_test_bit(irq, s->BMP);                           \
+    }                                                                   \
+    static inline void gicv3_gicd_##BMP##_clear(GICv3State *s, int irq) \
+    {                                                                   \
+        gic_bmp_clear_bit(irq, s->BMP);                                 \
+    }                                                                   \
+    static inline void gicv3_gicd_##BMP##_replace(GICv3State *s,        \
+                                                  int irq, int value)   \
+    {                                                                   \
+        gic_bmp_replace_bit(irq, s->BMP, value);                        \
+    }
+
+GICV3_BITMAP_ACCESSORS(group)
+GICV3_BITMAP_ACCESSORS(grpmod)
+GICV3_BITMAP_ACCESSORS(enabled)
+GICV3_BITMAP_ACCESSORS(pending)
+GICV3_BITMAP_ACCESSORS(active)
+GICV3_BITMAP_ACCESSORS(level)
+GICV3_BITMAP_ACCESSORS(edge_trigger)
+
+#define TYPE_ARM_GICV3_COMMON "arm-gicv3-common"
+#define ARM_GICV3_COMMON(obj) \
+     OBJECT_CHECK(GICv3State, (obj), TYPE_ARM_GICV3_COMMON)
+#define ARM_GICV3_COMMON_CLASS(klass) \
+     OBJECT_CLASS_CHECK(ARMGICv3CommonClass, (klass), TYPE_ARM_GICV3_COMMON)
+#define ARM_GICV3_COMMON_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(ARMGICv3CommonClass, (obj), TYPE_ARM_GICV3_COMMON)
+
+typedef struct ARMGICv3CommonClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+
+    void (*pre_save)(GICv3State *s);
+    void (*post_load)(GICv3State *s);
+} ARMGICv3CommonClass;
+
+void gicv3_init_irqs_and_mmio(GICv3State *s, qemu_irq_handler handler,
+                              const MemoryRegionOps *ops);
+
+#endif
diff --git a/include/hw/intc/aspeed_vic.h b/include/hw/intc/aspeed_vic.h
new file mode 100644
index 0000000..107ff17
--- /dev/null
+++ b/include/hw/intc/aspeed_vic.h
@@ -0,0 +1,48 @@
+/*
+ * ASPEED Interrupt Controller (New)
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Need to add SVIC and CVIC support
+ */
+#ifndef ASPEED_VIC_H
+#define ASPEED_VIC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_ASPEED_VIC "aspeed.vic"
+#define ASPEED_VIC(obj) OBJECT_CHECK(AspeedVICState, (obj), TYPE_ASPEED_VIC)
+
+#define ASPEED_VIC_NR_IRQS 51
+
+typedef struct AspeedVICState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    qemu_irq irq;
+    qemu_irq fiq;
+
+    uint64_t level;
+    uint64_t raw;
+    uint64_t select;
+    uint64_t enable;
+    uint64_t trigger;
+
+    /* 0=edge, 1=level */
+    uint64_t sense;
+
+    /* 0=single-edge, 1=dual-edge */
+    uint64_t dual_edge;
+
+    /* 0=low-sensitive/falling-edge, 1=high-sensitive/rising-edge */
+    uint64_t event;
+} AspeedVICState;
+
+#endif /* ASPEED_VIC_H */
diff --git a/include/hw/intc/bcm2835_ic.h b/include/hw/intc/bcm2835_ic.h
new file mode 100644
index 0000000..fb75fa0
--- /dev/null
+++ b/include/hw/intc/bcm2835_ic.h
@@ -0,0 +1,33 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_IC_H
+#define BCM2835_IC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_BCM2835_IC "bcm2835-ic"
+#define BCM2835_IC(obj) OBJECT_CHECK(BCM2835ICState, (obj), TYPE_BCM2835_IC)
+
+#define BCM2835_IC_GPU_IRQ "gpu-irq"
+#define BCM2835_IC_ARM_IRQ "arm-irq"
+
+typedef struct BCM2835ICState {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    MemoryRegion iomem;
+    qemu_irq irq;
+    qemu_irq fiq;
+
+    /* 64 GPU IRQs + 8 ARM IRQs = 72 total (GPU first) */
+    uint64_t gpu_irq_level, gpu_irq_enable;
+    uint8_t arm_irq_level, arm_irq_enable;
+    bool fiq_enable;
+    uint8_t fiq_select;
+} BCM2835ICState;
+
+#endif
diff --git a/include/hw/intc/bcm2836_control.h b/include/hw/intc/bcm2836_control.h
new file mode 100644
index 0000000..613f3c4
--- /dev/null
+++ b/include/hw/intc/bcm2836_control.h
@@ -0,0 +1,51 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * Upstreaming code cleanup [including bcm2835_*] (c) 2013 Jan Petrous
+ *
+ * Rasperry Pi 2 emulation and refactoring Copyright (c) 2015, Microsoft
+ * Written by Andrew Baumann
+ *
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2836_CONTROL_H
+#define BCM2836_CONTROL_H
+
+#include "hw/sysbus.h"
+
+/* 4 mailboxes per core, for 16 total */
+#define BCM2836_NCORES 4
+#define BCM2836_MBPERCORE 4
+
+#define TYPE_BCM2836_CONTROL "bcm2836-control"
+#define BCM2836_CONTROL(obj) \
+    OBJECT_CHECK(BCM2836ControlState, (obj), TYPE_BCM2836_CONTROL)
+
+typedef struct BCM2836ControlState {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+    MemoryRegion iomem;
+
+    /* mailbox state */
+    uint32_t mailboxes[BCM2836_NCORES * BCM2836_MBPERCORE];
+
+    /* interrupt routing/control registers */
+    uint8_t route_gpu_irq, route_gpu_fiq;
+    uint32_t timercontrol[BCM2836_NCORES];
+    uint32_t mailboxcontrol[BCM2836_NCORES];
+
+    /* interrupt status regs (derived from input pins; not visible to user) */
+    bool gpu_irq, gpu_fiq;
+    uint8_t timerirqs[BCM2836_NCORES];
+
+    /* interrupt source registers, post-routing (also input-derived; visible) */
+    uint32_t irqsrc[BCM2836_NCORES];
+    uint32_t fiqsrc[BCM2836_NCORES];
+
+    /* outputs to CPU cores */
+    qemu_irq irq[BCM2836_NCORES];
+    qemu_irq fiq[BCM2836_NCORES];
+} BCM2836ControlState;
+
+#endif
diff --git a/include/hw/intc/imx_avic.h b/include/hw/intc/imx_avic.h
new file mode 100644
index 0000000..1b80769
--- /dev/null
+++ b/include/hw/intc/imx_avic.h
@@ -0,0 +1,55 @@
+/*
+ * i.MX31 Vectored Interrupt Controller
+ *
+ * Note this is NOT the PL192 provided by ARM, but
+ * a custom implementation by Freescale.
+ *
+ * Copyright (c) 2008 OKL
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ *
+ * TODO: implement vectors.
+ */
+#ifndef IMX_AVIC_H
+#define IMX_AVIC_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_IMX_AVIC "imx.avic"
+#define IMX_AVIC(obj) OBJECT_CHECK(IMXAVICState, (obj), TYPE_IMX_AVIC)
+
+#define IMX_AVIC_NUM_IRQS 64
+
+/* Interrupt Control Bits */
+#define ABFLAG (1<<25)
+#define ABFEN  (1<<24)
+#define NIDIS  (1<<22) /* Normal Interrupt disable */
+#define FIDIS  (1<<21) /* Fast interrupt disable */
+#define NIAD   (1<<20) /* Normal Interrupt Arbiter Rise ARM level */
+#define FIAD   (1<<19) /* Fast Interrupt Arbiter Rise ARM level */
+#define NM     (1<<18) /* Normal interrupt mode */
+
+#define PRIO_PER_WORD (sizeof(uint32_t) * 8 / 4)
+#define PRIO_WORDS (IMX_AVIC_NUM_IRQS/PRIO_PER_WORD)
+
+typedef struct IMXAVICState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    uint64_t pending;
+    uint64_t enabled;
+    uint64_t is_fiq;
+    uint32_t intcntl;
+    uint32_t intmask;
+    qemu_irq irq;
+    qemu_irq fiq;
+    uint32_t prio[PRIO_WORDS]; /* Priorities are 4-bits each */
+} IMXAVICState;
+
+#endif /* IMX_AVIC_H */
diff --git a/include/hw/intc/mips_gic.h b/include/hw/intc/mips_gic.h
new file mode 100644
index 0000000..b98d500
--- /dev/null
+++ b/include/hw/intc/mips_gic.h
@@ -0,0 +1,216 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2000, 07 MIPS Technologies, Inc.
+ * Copyright (C) 2016 Imagination Technologies
+ *
+ */
+
+#ifndef MIPS_GIC_H
+#define MIPS_GIC_H
+
+#include "hw/timer/mips_gictimer.h"
+#include "cpu.h"
+/*
+ * GIC Specific definitions
+ */
+
+/* The MIPS default location */
+#define GIC_BASE_ADDR           0x1bdc0000ULL
+#define GIC_ADDRSPACE_SZ        (128 * 1024)
+
+/* Constants */
+#define GIC_POL_POS     1
+#define GIC_POL_NEG     0
+#define GIC_TRIG_EDGE   1
+#define GIC_TRIG_LEVEL  0
+
+#define MSK(n)              ((1ULL << (n)) - 1)
+
+/* GIC Address Space */
+#define SHARED_SECTION_OFS          0x0000
+#define SHARED_SECTION_SIZE         0x8000
+#define VP_LOCAL_SECTION_OFS        0x8000
+#define VP_LOCAL_SECTION_SIZE       0x4000
+#define VP_OTHER_SECTION_OFS        0xc000
+#define VP_OTHER_SECTION_SIZE       0x4000
+#define USM_VISIBLE_SECTION_OFS     0x10000
+#define USM_VISIBLE_SECTION_SIZE    0x10000
+
+/* Register Map for Shared Section */
+
+#define GIC_SH_CONFIG_OFS           0x0000
+
+/* Shared Global Counter */
+#define GIC_SH_COUNTERLO_OFS        0x0010
+#define GIC_SH_COUNTERHI_OFS        0x0014
+#define GIC_SH_REVISIONID_OFS       0x0020
+
+/* Set/Clear corresponding bit in Edge Detect Register */
+#define GIC_SH_WEDGE_OFS            0x0280
+
+/* Reset Mask - Disables Interrupt */
+#define GIC_SH_RMASK_OFS            0x0300
+#define GIC_SH_RMASK_LAST_OFS       0x031c
+
+/* Set Mask (WO) - Enables Interrupt */
+#define GIC_SH_SMASK_OFS            0x0380
+#define GIC_SH_SMASK_LAST_OFS       0x039c
+
+/* Global Interrupt Mask Register (RO) - Bit Set == Interrupt enabled */
+#define GIC_SH_MASK_OFS             0x0400
+#define GIC_SH_MASK_LAST_OFS        0x041c
+
+/* Pending Global Interrupts (RO) */
+#define GIC_SH_PEND_OFS             0x0480
+#define GIC_SH_PEND_LAST_OFS        0x049c
+
+#define GIC_SH_MAP0_PIN_OFS         0x0500
+#define GIC_SH_MAP255_PIN_OFS       0x08fc
+
+#define GIC_SH_MAP0_VP_OFS          0x2000
+#define GIC_SH_MAP255_VP_LAST_OFS   0x3fe4
+
+/* Register Map for Local Section */
+#define GIC_VP_CTL_OFS              0x0000
+#define GIC_VP_PEND_OFS             0x0004
+#define GIC_VP_MASK_OFS             0x0008
+#define GIC_VP_RMASK_OFS            0x000c
+#define GIC_VP_SMASK_OFS            0x0010
+#define GIC_VP_WD_MAP_OFS           0x0040
+#define GIC_VP_COMPARE_MAP_OFS      0x0044
+#define GIC_VP_TIMER_MAP_OFS        0x0048
+#define GIC_VP_FDC_MAP_OFS          0x004c
+#define GIC_VP_PERFCTR_MAP_OFS      0x0050
+#define GIC_VP_SWINT0_MAP_OFS       0x0054
+#define GIC_VP_SWINT1_MAP_OFS       0x0058
+#define GIC_VP_OTHER_ADDR_OFS       0x0080
+#define GIC_VP_IDENT_OFS            0x0088
+#define GIC_VP_WD_CONFIG0_OFS       0x0090
+#define GIC_VP_WD_COUNT0_OFS        0x0094
+#define GIC_VP_WD_INITIAL0_OFS      0x0098
+#define GIC_VP_COMPARE_LO_OFS       0x00a0
+#define GIC_VP_COMPARE_HI_OFS       0x00a4
+#define GIC_VL_BRK_GROUP            0x3080
+
+/* User-Mode Visible Section Register */
+/* Read-only alias for GIC Shared CounterLo */
+#define GIC_USER_MODE_COUNTERLO     0x0000
+/* Read-only alias for GIC Shared CounterHi */
+#define GIC_USER_MODE_COUNTERHI     0x0004
+
+/* Masks */
+#define GIC_SH_CONFIG_COUNTSTOP_SHF     28
+#define GIC_SH_CONFIG_COUNTSTOP_MSK     (MSK(1) << GIC_SH_CONFIG_COUNTSTOP_SHF)
+#define GIC_SH_CONFIG_COUNTBITS_SHF     24
+#define GIC_SH_CONFIG_COUNTBITS_MSK     (MSK(4) << GIC_SH_CONFIG_COUNTBITS_SHF)
+#define GIC_SH_CONFIG_NUMINTRS_SHF      16
+#define GIC_SH_CONFIG_NUMINTRS_MSK      (MSK(8) << GIC_SH_CONFIG_NUMINTRS_SHF)
+#define GIC_SH_CONFIG_PVPS_SHF          0
+#define GIC_SH_CONFIG_PVPS_MSK          (MSK(8) << GIC_SH_CONFIG_NUMVPS_SHF)
+
+#define GIC_SH_WEDGE_RW_SHF             31
+#define GIC_SH_WEDGE_RW_MSK             (MSK(1) << GIC_SH_WEDGE_RW_SHF)
+
+#define GIC_MAP_TO_PIN_SHF              31
+#define GIC_MAP_TO_PIN_MSK              (MSK(1) << GIC_MAP_TO_PIN_SHF)
+#define GIC_MAP_TO_NMI_SHF              30
+#define GIC_MAP_TO_NMI_MSK              (MSK(1) << GIC_MAP_TO_NMI_SHF)
+#define GIC_MAP_TO_YQ_SHF               29
+#define GIC_MAP_TO_YQ_MSK               (MSK(1) << GIC_MAP_TO_YQ_SHF)
+#define GIC_MAP_SHF                     0
+#define GIC_MAP_MSK                     (MSK(6) << GIC_MAP_SHF)
+#define GIC_MAP_TO_PIN_REG_MSK          \
+    (GIC_MAP_TO_PIN_MSK | GIC_MAP_TO_NMI_MSK | GIC_MAP_TO_YQ_MSK | GIC_MAP_MSK)
+
+/* GIC_VP_CTL Masks */
+#define GIC_VP_CTL_FDC_RTBL_SHF         4
+#define GIC_VP_CTL_FDC_RTBL_MSK         (MSK(1) << GIC_VP_CTL_FDC_RTBL_SHF)
+#define GIC_VP_CTL_SWINT_RTBL_SHF       3
+#define GIC_VP_CTL_SWINT_RTBL_MSK       (MSK(1) << GIC_VP_CTL_SWINT_RTBL_SHF)
+#define GIC_VP_CTL_PERFCNT_RTBL_SHF     2
+#define GIC_VP_CTL_PERFCNT_RTBL_MSK     (MSK(1) << GIC_VP_CTL_PERFCNT_RTBL_SHF)
+#define GIC_VP_CTL_TIMER_RTBL_SHF       1
+#define GIC_VP_CTL_TIMER_RTBL_MSK       (MSK(1) << GIC_VP_CTL_TIMER_RTBL_SHF)
+#define GIC_VP_CTL_EIC_MODE_SHF         0
+#define GIC_VP_CTL_EIC_MODE_MSK         (MSK(1) << GIC_VP_CTL_EIC_MODE_SHF)
+
+/* GIC_VP_MASK Masks */
+#define GIC_VP_MASK_FDC_SHF         6
+#define GIC_VP_MASK_FDC_MSK         (MSK(1) << GIC_VP_MASK_FDC_SHF)
+#define GIC_VP_MASK_SWINT1_SHF      5
+#define GIC_VP_MASK_SWINT1_MSK      (MSK(1) << GIC_VP_MASK_SWINT1_SHF)
+#define GIC_VP_MASK_SWINT0_SHF      4
+#define GIC_VP_MASK_SWINT0_MSK      (MSK(1) << GIC_VP_MASK_SWINT0_SHF)
+#define GIC_VP_MASK_PERFCNT_SHF     3
+#define GIC_VP_MASK_PERFCNT_MSK     (MSK(1) << GIC_VP_MASK_PERFCNT_SHF)
+#define GIC_VP_MASK_TIMER_SHF       2
+#define GIC_VP_MASK_TIMER_MSK       (MSK(1) << GIC_VP_MASK_TIMER_SHF)
+#define GIC_VP_MASK_CMP_SHF         1
+#define GIC_VP_MASK_CMP_MSK         (MSK(1) << GIC_VP_MASK_CMP_SHF)
+#define GIC_VP_MASK_WD_SHF          0
+#define GIC_VP_MASK_WD_MSK          (MSK(1) << GIC_VP_MASK_WD_SHF)
+#define GIC_VP_SET_RESET_MSK        (MSK(7) << GIC_VP_MASK_WD_SHF)
+
+#define GIC_CPU_INT_MAX             5 /* Core Interrupt 7 */
+#define GIC_CPU_PIN_OFFSET          2
+
+/* Local GIC interrupts. */
+#define GIC_NUM_LOCAL_INTRS     7
+#define GIC_LOCAL_INT_FDC       6 /* CPU fast debug channel */
+#define GIC_LOCAL_INT_SWINT1    5 /* CPU software interrupt 1 */
+#define GIC_LOCAL_INT_SWINT0    4 /* CPU software interrupt 0 */
+#define GIC_LOCAL_INT_PERFCTR   3 /* CPU performance counter */
+#define GIC_LOCAL_INT_TIMER     2 /* CPU timer interrupt */
+#define GIC_LOCAL_INT_COMPARE   1 /* GIC count and compare timer */
+#define GIC_LOCAL_INT_WD        0 /* GIC watchdog */
+
+#define TYPE_MIPS_GIC "mips-gic"
+#define MIPS_GIC(obj) OBJECT_CHECK(MIPSGICState, (obj), TYPE_MIPS_GIC)
+
+/* Support up to 32 VPs and 256 IRQs */
+#define GIC_MAX_VPS             32
+#define GIC_MAX_INTRS           256
+
+typedef struct MIPSGICState MIPSGICState;
+typedef struct MIPSGICIRQState MIPSGICIRQState;
+typedef struct MIPSGICVPState MIPSGICVPState;
+
+struct MIPSGICIRQState {
+    uint8_t enabled;
+    uint8_t pending;
+    uint32_t map_pin;
+    int32_t map_vp;
+    qemu_irq irq;
+};
+
+struct MIPSGICVPState {
+    uint32_t ctl;
+    uint32_t pend;
+    uint32_t mask;
+    uint32_t compare_map;
+    uint32_t other_addr;
+    CPUMIPSState *env;
+};
+
+struct MIPSGICState {
+    SysBusDevice parent_obj;
+    MemoryRegion mr;
+
+    /* Shared Section Registers */
+    uint32_t sh_config;
+    MIPSGICIRQState *irq_state;
+
+    /* VP Local/Other Section Registers */
+    MIPSGICVPState *vps;
+
+    /* GIC VP Timer */
+    MIPSGICTimerState *gic_timer;
+
+    int32_t num_vps;
+    int32_t num_irq;
+};
+
+#endif /* MIPS_GIC_H */
diff --git a/include/hw/ipack/ipack.h b/include/hw/ipack/ipack.h
index e95ffe8..e33e032 100644
--- a/include/hw/ipack/ipack.h
+++ b/include/hw/ipack/ipack.h
@@ -2,7 +2,7 @@
  * QEMU IndustryPack emulation
  *
  * Copyright (C) 2012 Igalia, S.L.
- * Author: Alberto Garcia <agarcia@igalia.com>
+ * Author: Alberto Garcia <berto@igalia.com>
  *
  * This code is licensed under the GNU GPL v2 or (at your option) any
  * later version.
diff --git a/include/hw/ipmi/ipmi.h b/include/hw/ipmi/ipmi.h
new file mode 100644
index 0000000..91b83b5
--- /dev/null
+++ b/include/hw/ipmi/ipmi.h
@@ -0,0 +1,262 @@
+/*
+ * IPMI base class
+ *
+ * Copyright (c) 2015 Corey Minyard, MontaVista Software, LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef HW_IPMI_H
+#define HW_IPMI_H
+
+#include "exec/memory.h"
+#include "qemu-common.h"
+#include "hw/qdev.h"
+
+#define MAX_IPMI_MSG_SIZE 300
+
+enum ipmi_op {
+    IPMI_RESET_CHASSIS,
+    IPMI_POWEROFF_CHASSIS,
+    IPMI_POWERON_CHASSIS,
+    IPMI_POWERCYCLE_CHASSIS,
+    IPMI_PULSE_DIAG_IRQ,
+    IPMI_SHUTDOWN_VIA_ACPI_OVERTEMP,
+    IPMI_SEND_NMI
+};
+
+#define IPMI_CC_INVALID_CMD                              0xc1
+#define IPMI_CC_COMMAND_INVALID_FOR_LUN                  0xc2
+#define IPMI_CC_TIMEOUT                                  0xc3
+#define IPMI_CC_OUT_OF_SPACE                             0xc4
+#define IPMI_CC_INVALID_RESERVATION                      0xc5
+#define IPMI_CC_REQUEST_DATA_TRUNCATED                   0xc6
+#define IPMI_CC_REQUEST_DATA_LENGTH_INVALID              0xc7
+#define IPMI_CC_PARM_OUT_OF_RANGE                        0xc9
+#define IPMI_CC_CANNOT_RETURN_REQ_NUM_BYTES              0xca
+#define IPMI_CC_REQ_ENTRY_NOT_PRESENT                    0xcb
+#define IPMI_CC_INVALID_DATA_FIELD                       0xcc
+#define IPMI_CC_BMC_INIT_IN_PROGRESS                     0xd2
+#define IPMI_CC_COMMAND_NOT_SUPPORTED                    0xd5
+
+#define IPMI_NETFN_APP                0x06
+
+#define IPMI_DEBUG 1
+
+/* Specified in the SMBIOS spec. */
+#define IPMI_SMBIOS_KCS         0x01
+#define IPMI_SMBIOS_SMIC        0x02
+#define IPMI_SMBIOS_BT          0x03
+#define IPMI_SMBIOS_SSIF        0x04
+
+/*
+ * Used for transferring information to interfaces that add
+ * entries to firmware tables.
+ */
+typedef struct IPMIFwInfo {
+    const char *interface_name;
+    int interface_type;
+    uint8_t ipmi_spec_major_revision;
+    uint8_t ipmi_spec_minor_revision;
+    uint8_t i2c_slave_address;
+    uint32_t uuid;
+
+    uint64_t base_address;
+    uint64_t register_length;
+    uint8_t register_spacing;
+    enum {
+        IPMI_MEMSPACE_IO,
+        IPMI_MEMSPACE_MEM32,
+        IPMI_MEMSPACE_MEM64,
+        IPMI_MEMSPACE_SMBUS
+    } memspace;
+
+    int interrupt_number;
+    enum {
+        IPMI_LEVEL_IRQ,
+        IPMI_EDGE_IRQ
+    } irq_type;
+} IPMIFwInfo;
+
+/*
+ * Called by each instantiated IPMI interface device to get it's uuid.
+ */
+uint32_t ipmi_next_uuid(void);
+
+/* IPMI Interface types (KCS, SMIC, BT) are prefixed with this */
+#define TYPE_IPMI_INTERFACE_PREFIX "ipmi-interface-"
+
+/*
+ * An IPMI Interface, the interface for talking between the target
+ * and the BMC.
+ */
+#define TYPE_IPMI_INTERFACE "ipmi-interface"
+#define IPMI_INTERFACE(obj) \
+     INTERFACE_CHECK(IPMIInterface, (obj), TYPE_IPMI_INTERFACE)
+#define IPMI_INTERFACE_CLASS(class) \
+     OBJECT_CLASS_CHECK(IPMIInterfaceClass, (class), TYPE_IPMI_INTERFACE)
+#define IPMI_INTERFACE_GET_CLASS(class) \
+     OBJECT_GET_CLASS(IPMIInterfaceClass, (class), TYPE_IPMI_INTERFACE)
+
+typedef struct IPMIInterface {
+    Object parent;
+} IPMIInterface;
+
+typedef struct IPMIInterfaceClass {
+    InterfaceClass parent;
+
+    void (*init)(struct IPMIInterface *s, Error **errp);
+
+    /*
+     * Perform various operations on the hardware.  If checkonly is
+     * true, it will return if the operation can be performed, but it
+     * will not do the operation.
+     */
+    int (*do_hw_op)(struct IPMIInterface *s, enum ipmi_op op, int checkonly);
+
+    /*
+     * Enable/disable irqs on the interface when the BMC requests this.
+     */
+    void (*set_irq_enable)(struct IPMIInterface *s, int val);
+
+    /*
+     * Handle an event that occurred on the interface, generally the.
+     * target writing to a register.
+     */
+    void (*handle_if_event)(struct IPMIInterface *s);
+
+    /*
+     * The interfaces use this to perform certain ops
+     */
+    void (*set_atn)(struct IPMIInterface *s, int val, int irq);
+
+    /*
+     * Got an IPMI warm/cold reset.
+     */
+    void (*reset)(struct IPMIInterface *s, bool is_cold);
+
+    /*
+     * Handle a response from the bmc.
+     */
+    void (*handle_rsp)(struct IPMIInterface *s, uint8_t msg_id,
+                       unsigned char *rsp, unsigned int rsp_len);
+
+    /*
+     * Set by the owner to hold the backend data for the interface.
+     */
+    void *(*get_backend_data)(struct IPMIInterface *s);
+
+    /*
+     * Return the firmware info for a device.
+     */
+    void (*get_fwinfo)(struct IPMIInterface *s, IPMIFwInfo *info);
+} IPMIInterfaceClass;
+
+/*
+ * Define a BMC simulator (or perhaps a connection to a real BMC)
+ */
+#define TYPE_IPMI_BMC "ipmi-bmc"
+#define IPMI_BMC(obj) \
+     OBJECT_CHECK(IPMIBmc, (obj), TYPE_IPMI_BMC)
+#define IPMI_BMC_CLASS(obj_class) \
+     OBJECT_CLASS_CHECK(IPMIBmcClass, (obj_class), TYPE_IPMI_BMC)
+#define IPMI_BMC_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IPMIBmcClass, (obj), TYPE_IPMI_BMC)
+
+typedef struct IPMIBmc {
+    DeviceState parent;
+
+    uint8_t slave_addr;
+
+    IPMIInterface *intf;
+} IPMIBmc;
+
+typedef struct IPMIBmcClass {
+    DeviceClass parent;
+
+    /* Called when the system resets to report to the bmc. */
+    void (*handle_reset)(struct IPMIBmc *s);
+
+    /*
+     * Handle a command to the bmc.
+     */
+    void (*handle_command)(struct IPMIBmc *s,
+                           uint8_t *cmd, unsigned int cmd_len,
+                           unsigned int max_cmd_len,
+                           uint8_t msg_id);
+} IPMIBmcClass;
+
+/*
+ * Add a link property to obj that points to a BMC.
+ */
+void ipmi_bmc_find_and_link(Object *obj, Object **bmc);
+
+#ifdef IPMI_DEBUG
+#define ipmi_debug(fs, ...) \
+    fprintf(stderr, "IPMI (%s): " fs, __func__, ##__VA_ARGS__)
+#else
+#define ipmi_debug(fs, ...)
+#endif
+
+struct ipmi_sdr_header {
+    uint8_t  rec_id[2];
+    uint8_t  sdr_version;               /* 0x51 */
+    uint8_t  rec_type;
+    uint8_t  rec_length;
+};
+#define IPMI_SDR_HEADER_SIZE     sizeof(struct ipmi_sdr_header)
+
+#define ipmi_sdr_recid(sdr) ((sdr)->rec_id[0] | ((sdr)->rec_id[1] << 8))
+#define ipmi_sdr_length(sdr) ((sdr)->rec_length + IPMI_SDR_HEADER_SIZE)
+
+/*
+ * 43.2 SDR Type 02h. Compact Sensor Record
+ */
+#define IPMI_SDR_COMPACT_TYPE    2
+
+struct ipmi_sdr_compact {
+    struct ipmi_sdr_header header;
+
+    uint8_t  sensor_owner_id;
+    uint8_t  sensor_owner_lun;
+    uint8_t  sensor_owner_number;       /* byte 8 */
+    uint8_t  entity_id;
+    uint8_t  entity_instance;
+    uint8_t  sensor_init;
+    uint8_t  sensor_caps;
+    uint8_t  sensor_type;
+    uint8_t  reading_type;
+    uint8_t  assert_mask[2];            /* byte 16 */
+    uint8_t  deassert_mask[2];
+    uint8_t  discrete_mask[2];
+    uint8_t  sensor_unit1;
+    uint8_t  sensor_unit2;
+    uint8_t  sensor_unit3;
+    uint8_t  sensor_direction[2];       /* byte 24 */
+    uint8_t  positive_threshold;
+    uint8_t  negative_threshold;
+    uint8_t  reserved[3];
+    uint8_t  oem;
+    uint8_t  id_str_len;                /* byte 32 */
+    uint8_t  id_string[16];
+};
+
+typedef uint8_t ipmi_sdr_compact_buffer[sizeof(struct ipmi_sdr_compact)];
+
+#endif
diff --git a/include/hw/isa/apm.h b/include/hw/isa/apm.h
index 3edea5f..4839ff1 100644
--- a/include/hw/isa/apm.h
+++ b/include/hw/isa/apm.h
@@ -1,7 +1,6 @@
 #ifndef APM_H
 #define APM_H
 
-#include <stdint.h>
 #include "qemu-common.h"
 #include "hw/hw.h"
 #include "exec/memory.h"
diff --git a/include/hw/isa/i8257.h b/include/hw/isa/i8257.h
new file mode 100644
index 0000000..aa211c0
--- /dev/null
+++ b/include/hw/isa/i8257.h
@@ -0,0 +1,41 @@
+#ifndef HW_I8257_H
+#define HW_I8257_H
+
+#define TYPE_I8257 "i8257"
+
+typedef struct I8257Regs {
+    int now[2];
+    uint16_t base[2];
+    uint8_t mode;
+    uint8_t page;
+    uint8_t pageh;
+    uint8_t dack;
+    uint8_t eop;
+    IsaDmaTransferHandler transfer_handler;
+    void *opaque;
+} I8257Regs;
+
+typedef struct I8257State {
+    /* <private> */
+    ISADevice parent_obj;
+
+    /* <public> */
+    int32_t base;
+    int32_t page_base;
+    int32_t pageh_base;
+    int32_t dshift;
+
+    uint8_t status;
+    uint8_t command;
+    uint8_t mask;
+    uint8_t flip_flop;
+    I8257Regs regs[4];
+    MemoryRegion channel_io;
+    MemoryRegion cont_io;
+
+    QEMUBH *dma_bh;
+    bool dma_bh_scheduled;
+    int running;
+} I8257State;
+
+#endif
diff --git a/include/hw/isa/i8259_internal.h b/include/hw/isa/i8259_internal.h
index cded509..6954b6e 100644
--- a/include/hw/isa/i8259_internal.h
+++ b/include/hw/isa/i8259_internal.h
@@ -80,4 +80,4 @@
 ISADevice *i8259_init_chip(const char *name, ISABus *bus, bool master);
 
 
-#endif /* !QEMU_I8259_INTERNAL_H */
+#endif /* QEMU_I8259_INTERNAL_H */
diff --git a/include/hw/isa/isa.h b/include/hw/isa/isa.h
index e0c749f..7693ac5 100644
--- a/include/hw/isa/isa.h
+++ b/include/hw/isa/isa.h
@@ -3,8 +3,8 @@
 
 /* ISA bus */
 
-#include "exec/ioport.h"
 #include "exec/memory.h"
+#include "exec/ioport.h"
 #include "hw/qdev.h"
 
 #define ISA_NUM_IRQS 16
@@ -21,12 +21,57 @@
 #define ISA_BUS(obj) OBJECT_CHECK(ISABus, (obj), TYPE_ISA_BUS)
 
 #define TYPE_APPLE_SMC "isa-applesmc"
+#define APPLESMC_MAX_DATA_LENGTH       32
+#define APPLESMC_PROP_IO_BASE "iobase"
 
-static inline bool applesmc_find(void)
+static inline uint16_t applesmc_port(void)
 {
-    return object_resolve_path_type("", TYPE_APPLE_SMC, NULL);
+    Object *obj = object_resolve_path_type("", TYPE_APPLE_SMC, NULL);
+
+    if (obj) {
+        return object_property_get_int(obj, APPLESMC_PROP_IO_BASE, NULL);
+    }
+    return 0;
 }
 
+#define TYPE_ISADMA "isa-dma"
+
+#define ISADMA_CLASS(klass) \
+    OBJECT_CLASS_CHECK(IsaDmaClass, (klass), TYPE_ISADMA)
+#define ISADMA_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(IsaDmaClass, (obj), TYPE_ISADMA)
+#define ISADMA(obj) \
+    INTERFACE_CHECK(IsaDma, (obj), TYPE_ISADMA)
+
+struct IsaDma {
+    Object parent;
+};
+
+typedef enum {
+    ISADMA_TRANSFER_VERIFY,
+    ISADMA_TRANSFER_READ,
+    ISADMA_TRANSFER_WRITE,
+    ISADMA_TRANSFER_ILLEGAL,
+} IsaDmaTransferMode;
+
+typedef int (*IsaDmaTransferHandler)(void *opaque, int nchan, int pos,
+                                     int size);
+
+typedef struct IsaDmaClass {
+    InterfaceClass parent;
+
+    IsaDmaTransferMode (*get_transfer_mode)(IsaDma *obj, int nchan);
+    bool (*has_autoinitialization)(IsaDma *obj, int nchan);
+    int (*read_memory)(IsaDma *obj, int nchan, void *buf, int pos, int len);
+    int (*write_memory)(IsaDma *obj, int nchan, void *buf, int pos, int len);
+    void (*hold_DREQ)(IsaDma *obj, int nchan);
+    void (*release_DREQ)(IsaDma *obj, int nchan);
+    void (*schedule)(IsaDma *obj);
+    void (*register_channel)(IsaDma *obj, int nchan,
+                             IsaDmaTransferHandler transfer_handler,
+                             void *opaque);
+} IsaDmaClass;
+
 typedef struct ISADeviceClass {
     DeviceClass parent_class;
 } ISADeviceClass;
@@ -36,8 +81,10 @@
     BusState parent_obj;
     /*< public >*/
 
+    MemoryRegion *address_space;
     MemoryRegion *address_space_io;
     qemu_irq *irqs;
+    IsaDma *dma[2];
 };
 
 struct ISADevice {
@@ -50,10 +97,14 @@
     int ioport_id;
 };
 
-ISABus *isa_bus_new(DeviceState *dev, MemoryRegion *address_space_io);
+ISABus *isa_bus_new(DeviceState *dev, MemoryRegion *address_space,
+                    MemoryRegion *address_space_io, Error **errp);
 void isa_bus_irqs(ISABus *bus, qemu_irq *irqs);
 qemu_irq isa_get_irq(ISADevice *dev, int isairq);
 void isa_init_irq(ISADevice *dev, qemu_irq *p, int isairq);
+void isa_connect_gpio_out(ISADevice *isadev, int gpioirq, int isairq);
+void isa_bus_dma(ISABus *bus, IsaDma *dma8, IsaDma *dma16);
+IsaDma *isa_get_dma(ISABus *bus, int nchan);
 MemoryRegion *isa_address_space(ISADevice *dev);
 MemoryRegion *isa_address_space_io(ISADevice *dev);
 ISADevice *isa_create(ISABus *bus, const char *name);
@@ -97,17 +148,6 @@
     return ISA_BUS(qdev_get_parent_bus(DEVICE(d)));
 }
 
-extern hwaddr isa_mem_base;
-
-/* dma.c */
-int DMA_get_channel_mode (int nchan);
-int DMA_read_memory (int nchan, void *buf, int pos, int size);
-int DMA_write_memory (int nchan, void *buf, int pos, int size);
-void DMA_hold_DREQ (int nchan);
-void DMA_release_DREQ (int nchan);
-void DMA_schedule(int nchan);
-void DMA_init(int high_page_enable, qemu_irq *cpu_request_exit);
-void DMA_register_channel (int nchan,
-                           DMA_transfer_handler transfer_handler,
-                           void *opaque);
+/* i8257.c */
+void DMA_init(ISABus *bus, int high_page_enable);
 #endif
diff --git a/include/hw/lm32/lm32_pic.h b/include/hw/lm32/lm32_pic.h
index 5556803..189fa38 100644
--- a/include/hw/lm32/lm32_pic.h
+++ b/include/hw/lm32/lm32_pic.h
@@ -8,7 +8,7 @@
 void lm32_pic_set_ip(DeviceState *d, uint32_t ip);
 void lm32_pic_set_im(DeviceState *d, uint32_t im);
 
-void lm32_do_pic_info(Monitor *mon, const QDict *qdict);
-void lm32_irq_info(Monitor *mon, const QDict *qdict);
+void lm32_hmp_info_pic(Monitor *mon, const QDict *qdict);
+void lm32_hmp_info_irq(Monitor *mon, const QDict *qdict);
 
 #endif /* QEMU_HW_LM32_PIC_H */
diff --git a/include/hw/loader.h b/include/hw/loader.h
index 6481639..4879b63 100644
--- a/include/hw/loader.h
+++ b/include/hw/loader.h
@@ -16,6 +16,27 @@
 ssize_t load_image_size(const char *filename, void *addr, size_t size);
 int load_image_targphys(const char *filename, hwaddr,
                         uint64_t max_sz);
+/**
+ * load_image_mr: load an image into a memory region
+ * @filename: Path to the image file
+ * @mr: Memory Region to load into
+ *
+ * Load the specified file into the memory region.
+ * The file loaded is registered as a ROM, so its contents will be
+ * reinstated whenever the system is reset.
+ * If the file is larger than the memory region's size the call will fail.
+ * Returns -1 on failure, or the size of the file.
+ */
+int load_image_mr(const char *filename, MemoryRegion *mr);
+
+/* This is the limit on the maximum uncompressed image size that
+ * load_image_gzipped_buffer() and load_image_gzipped() will read. It prevents
+ * g_malloc() in those functions from allocating a huge amount of memory.
+ */
+#define LOAD_IMAGE_MAX_GUNZIP_BYTES (256 << 20)
+
+int load_image_gzipped_buffer(const char *filename, uint64_t max_sz,
+                              uint8_t **buffer);
 int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz);
 
 #define ELF_LOAD_FAILED       -1
@@ -23,10 +44,49 @@
 #define ELF_LOAD_WRONG_ARCH   -3
 #define ELF_LOAD_WRONG_ENDIAN -4
 const char *load_elf_strerror(int error);
+
+/** load_elf:
+ * @filename: Path of ELF file
+ * @translate_fn: optional function to translate load addresses
+ * @translate_opaque: opaque data passed to @translate_fn
+ * @pentry: Populated with program entry point. Ignored if NULL.
+ * @lowaddr: Populated with lowest loaded address. Ignored if NULL.
+ * @highaddr: Populated with highest loaded address. Ignored if NULL.
+ * @bigendian: Expected ELF endianness. 0 for LE otherwise BE
+ * @elf_machine: Expected ELF machine type
+ * @clear_lsb: Set to mask off LSB of addresses (Some architectures use
+ *             this for non-address data)
+ * @data_swab: Set to order of byte swapping for data. 0 for no swap, 1
+ *             for swapping bytes within halfwords, 2 for bytes within
+ *             words and 3 for within doublewords.
+ *
+ * Load an ELF file's contents to the emulated system's address space.
+ * Clients may optionally specify a callback to perform address
+ * translations. @pentry, @lowaddr and @highaddr are optional pointers
+ * which will be populated with various load information. @bigendian and
+ * @elf_machine give the expected endianness and machine for the ELF the
+ * load will fail if the target ELF does not match. Some architectures
+ * have some architecture-specific behaviours that come into effect when
+ * their particular values for @elf_machine are set.
+ */
+
 int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
              void *translate_opaque, uint64_t *pentry, uint64_t *lowaddr,
              uint64_t *highaddr, int big_endian, int elf_machine,
-             int clear_lsb);
+             int clear_lsb, int data_swab);
+
+/** load_elf_hdr:
+ * @filename: Path of ELF file
+ * @hdr: Buffer to populate with header data. Header data will not be
+ * filled if set to NULL.
+ * @is64: Set to true if the ELF is 64bit. Ignored if set to NULL
+ * @errp: Populated with an error in failure cases
+ *
+ * Inspect an ELF file's header. Read its full header contents into a
+ * buffer and/or determine if the ELF is 64bit.
+ */
+void load_elf_hdr(const char *filename, void *hdr, bool *is64, Error **errp);
+
 int load_aout(const char *filename, hwaddr addr, int max_sz,
               int bswap_needed, hwaddr target_page_size);
 int load_uimage(const char *filename, hwaddr *ep,
@@ -58,23 +118,28 @@
 
 int rom_add_file(const char *file, const char *fw_dir,
                  hwaddr addr, int32_t bootindex,
-                 bool option_rom);
-ram_addr_t rom_add_blob(const char *name, const void *blob, size_t len,
-                   hwaddr addr, const char *fw_file_name,
-                   FWCfgReadCallback fw_callback, void *callback_opaque);
+                 bool option_rom, MemoryRegion *mr);
+MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len,
+                           size_t max_len, hwaddr addr,
+                           const char *fw_file_name,
+                           FWCfgReadCallback fw_callback,
+                           void *callback_opaque);
 int rom_add_elf_program(const char *name, void *data, size_t datasize,
                         size_t romsize, hwaddr addr);
-int rom_load_all(void);
-void rom_load_done(void);
+int rom_check_and_register_reset(void);
 void rom_set_fw(FWCfgState *f);
+void rom_set_order_override(int order);
+void rom_reset_order_override(void);
 int rom_copy(uint8_t *dest, hwaddr addr, size_t size);
 void *rom_ptr(hwaddr addr);
-void do_info_roms(Monitor *mon, const QDict *qdict);
+void hmp_info_roms(Monitor *mon, const QDict *qdict);
 
 #define rom_add_file_fixed(_f, _a, _i)          \
-    rom_add_file(_f, NULL, _a, _i, false)
+    rom_add_file(_f, NULL, _a, _i, false, NULL)
 #define rom_add_blob_fixed(_f, _b, _l, _a)      \
-    rom_add_blob(_f, _b, _l, _a, NULL, NULL, NULL)
+    rom_add_blob(_f, _b, _l, _l, _a, NULL, NULL, NULL)
+#define rom_add_file_mr(_f, _mr, _i)            \
+    rom_add_file(_f, NULL, 0, _i, false, _mr)
 
 #define PC_ROM_MIN_VGA     0xc0000
 #define PC_ROM_MIN_OPTION  0xc8000
diff --git a/include/hw/m68k/mcf.h b/include/hw/m68k/mcf.h
index fbc8dc2..0f0d228 100644
--- a/include/hw/m68k/mcf.h
+++ b/include/hw/m68k/mcf.h
@@ -2,6 +2,8 @@
 #define HW_MCF_H
 /* Motorola ColdFire device prototypes.  */
 
+#include "target-m68k/cpu-qom.h"
+
 struct MemoryRegion;
 
 /* mcf_uart.c */
diff --git a/include/hw/mem/nvdimm.h b/include/hw/mem/nvdimm.h
new file mode 100644
index 0000000..1cfe9e0
--- /dev/null
+++ b/include/hw/mem/nvdimm.h
@@ -0,0 +1,116 @@
+/*
+ * Non-Volatile Dual In-line Memory Module Virtualization Implementation
+ *
+ * Copyright(C) 2015 Intel Corporation.
+ *
+ * Author:
+ *  Xiao Guangrong <guangrong.xiao@linux.intel.com>
+ *
+ * NVDIMM specifications and some documents can be found at:
+ * NVDIMM ACPI device and NFIT are introduced in ACPI 6:
+ *      http://www.uefi.org/sites/default/files/resources/ACPI_6.0.pdf
+ * NVDIMM Namespace specification:
+ *      http://pmem.io/documents/NVDIMM_Namespace_Spec.pdf
+ * DSM Interface Example:
+ *      http://pmem.io/documents/NVDIMM_DSM_Interface_Example.pdf
+ * Driver Writer's Guide:
+ *      http://pmem.io/documents/NVDIMM_Driver_Writers_Guide.pdf
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_NVDIMM_H
+#define QEMU_NVDIMM_H
+
+#include "hw/mem/pc-dimm.h"
+#include "hw/acpi/bios-linker-loader.h"
+
+#define NVDIMM_DEBUG 0
+#define nvdimm_debug(fmt, ...)                                \
+    do {                                                      \
+        if (NVDIMM_DEBUG) {                                   \
+            fprintf(stderr, "nvdimm: " fmt, ## __VA_ARGS__);  \
+        }                                                     \
+    } while (0)
+
+/*
+ * The minimum label data size is required by NVDIMM Namespace
+ * specification, see the chapter 2 Namespaces:
+ *   "NVDIMMs following the NVDIMM Block Mode Specification use an area
+ *    at least 128KB in size, which holds around 1000 labels."
+ */
+#define MIN_NAMESPACE_LABEL_SIZE      (128UL << 10)
+
+#define TYPE_NVDIMM      "nvdimm"
+#define NVDIMM(obj)      OBJECT_CHECK(NVDIMMDevice, (obj), TYPE_NVDIMM)
+#define NVDIMM_CLASS(oc) OBJECT_CLASS_CHECK(NVDIMMClass, (oc), TYPE_NVDIMM)
+#define NVDIMM_GET_CLASS(obj) OBJECT_GET_CLASS(NVDIMMClass, (obj), \
+                                               TYPE_NVDIMM)
+struct NVDIMMDevice {
+    /* private */
+    PCDIMMDevice parent_obj;
+
+    /* public */
+
+    /*
+     * the size of label data in NVDIMM device which is presented to
+     * guest via __DSM "Get Namespace Label Size" function.
+     */
+    uint64_t label_size;
+
+    /*
+     * the address of label data which is read by __DSM "Get Namespace
+     * Label Data" function and written by __DSM "Set Namespace Label
+     * Data" function.
+     */
+    void *label_data;
+
+    /*
+     * it's the PMEM region in NVDIMM device, which is presented to
+     * guest via ACPI NFIT and _FIT method if NVDIMM hotplug is supported.
+     */
+    MemoryRegion nvdimm_mr;
+};
+typedef struct NVDIMMDevice NVDIMMDevice;
+
+struct NVDIMMClass {
+    /* private */
+    PCDIMMDeviceClass parent_class;
+
+    /* public */
+
+    /* read @size bytes from NVDIMM label data at @offset into @buf. */
+    void (*read_label_data)(NVDIMMDevice *nvdimm, void *buf,
+                            uint64_t size, uint64_t offset);
+    /* write @size bytes from @buf to NVDIMM label data at @offset. */
+    void (*write_label_data)(NVDIMMDevice *nvdimm, const void *buf,
+                             uint64_t size, uint64_t offset);
+};
+typedef struct NVDIMMClass NVDIMMClass;
+
+#define NVDIMM_DSM_MEM_FILE     "etc/acpi/nvdimm-mem"
+
+/*
+ * 32 bits IO port starting from 0x0a18 in guest is reserved for
+ * NVDIMM ACPI emulation.
+ */
+#define NVDIMM_ACPI_IO_BASE     0x0a18
+#define NVDIMM_ACPI_IO_LEN      4
+
+struct AcpiNVDIMMState {
+    /* detect if NVDIMM support is enabled. */
+    bool is_enabled;
+
+    /* the data of the fw_cfg file NVDIMM_DSM_MEM_FILE. */
+    GArray *dsm_mem;
+    /* the IO region used by OSPM to transfer control to QEMU. */
+    MemoryRegion io_mr;
+};
+typedef struct AcpiNVDIMMState AcpiNVDIMMState;
+
+void nvdimm_init_acpi_state(AcpiNVDIMMState *state, MemoryRegion *io,
+                            FWCfgState *fw_cfg, Object *owner);
+void nvdimm_build_acpi(GArray *table_offsets, GArray *table_data,
+                       BIOSLinker *linker, GArray *dsm_dma_arrea);
+#endif
diff --git a/include/hw/mem/pc-dimm.h b/include/hw/mem/pc-dimm.h
index e1dcbbc..1e483f2 100644
--- a/include/hw/mem/pc-dimm.h
+++ b/include/hw/mem/pc-dimm.h
@@ -20,8 +20,6 @@
 #include "sysemu/hostmem.h"
 #include "hw/qdev.h"
 
-#define DEFAULT_PC_DIMMSIZE (1024*1024*1024)
-
 #define TYPE_PC_DIMM "pc-dimm"
 #define PC_DIMM(obj) \
     OBJECT_CHECK(PCDIMMDevice, (obj), TYPE_PC_DIMM)
@@ -60,16 +58,34 @@
 
 /**
  * PCDIMMDeviceClass:
- * @get_memory_region: returns #MemoryRegion associated with @dimm
+ * @realize: called after common dimm is realized so that the dimm based
+ * devices get the chance to do specified operations.
+ * @get_memory_region: returns #MemoryRegion associated with @dimm which
+ * is directly mapped into the physical address space of guest.
+ * @get_vmstate_memory_region: returns #MemoryRegion which indicates the
+ * memory of @dimm should be kept during live migration.
  */
 typedef struct PCDIMMDeviceClass {
     /* private */
     DeviceClass parent_class;
 
     /* public */
+    void (*realize)(PCDIMMDevice *dimm, Error **errp);
     MemoryRegion *(*get_memory_region)(PCDIMMDevice *dimm);
+    MemoryRegion *(*get_vmstate_memory_region)(PCDIMMDevice *dimm);
 } PCDIMMDeviceClass;
 
+/**
+ * MemoryHotplugState:
+ * @base: address in guest physical address space where hotplug memory
+ * address space begins.
+ * @mr: hotplug memory address space container
+ */
+typedef struct MemoryHotplugState {
+    hwaddr base;
+    MemoryRegion mr;
+} MemoryHotplugState;
+
 uint64_t pc_dimm_get_free_addr(uint64_t address_space_start,
                                uint64_t address_space_size,
                                uint64_t *hint, uint64_t align, uint64_t size,
@@ -78,4 +94,9 @@
 int pc_dimm_get_free_slot(const int *hint, int max_slots, Error **errp);
 
 int qmp_pc_dimm_device_list(Object *obj, void *opaque);
+uint64_t pc_existing_dimms_capacity(Error **errp);
+void pc_dimm_memory_plug(DeviceState *dev, MemoryHotplugState *hpms,
+                         MemoryRegion *mr, uint64_t align, Error **errp);
+void pc_dimm_memory_unplug(DeviceState *dev, MemoryHotplugState *hpms,
+                           MemoryRegion *mr);
 #endif
diff --git a/include/hw/mips/cps.h b/include/hw/mips/cps.h
new file mode 100644
index 0000000..526b8d0
--- /dev/null
+++ b/include/hw/mips/cps.h
@@ -0,0 +1,48 @@
+/*
+ * Coherent Processing System emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MIPS_CPS_H
+#define MIPS_CPS_H
+
+#include "hw/sysbus.h"
+#include "hw/misc/mips_cmgcr.h"
+#include "hw/intc/mips_gic.h"
+#include "hw/misc/mips_cpc.h"
+#include "hw/misc/mips_itu.h"
+
+#define TYPE_MIPS_CPS "mips-cps"
+#define MIPS_CPS(obj) OBJECT_CHECK(MIPSCPSState, (obj), TYPE_MIPS_CPS)
+
+typedef struct MIPSCPSState {
+    SysBusDevice parent_obj;
+
+    uint32_t num_vp;
+    uint32_t num_irq;
+    char *cpu_model;
+
+    MemoryRegion container;
+    MIPSGCRState gcr;
+    MIPSGICState gic;
+    MIPSCPCState cpc;
+    MIPSITUState itu;
+} MIPSCPSState;
+
+qemu_irq get_cps_irq(MIPSCPSState *cps, int pin_number);
+
+#endif
diff --git a/include/hw/mips/cpudevs.h b/include/hw/mips/cpudevs.h
index b2626f2..8673daa 100644
--- a/include/hw/mips/cpudevs.h
+++ b/include/hw/mips/cpudevs.h
@@ -1,5 +1,8 @@
 #ifndef HW_MIPS_CPUDEVS_H
 #define HW_MIPS_CPUDEVS_H
+
+#include "target-mips/cpu-qom.h"
+
 /* Definitions for MIPS CPU internal devices.  */
 
 /* mips_addr.c */
@@ -9,9 +12,9 @@
 
 
 /* mips_int.c */
-void cpu_mips_irq_init_cpu(CPUMIPSState *env);
+void cpu_mips_irq_init_cpu(MIPSCPU *cpu);
 
 /* mips_timer.c */
-void cpu_mips_clock_init(CPUMIPSState *);
+void cpu_mips_clock_init(MIPSCPU *cpu);
 
 #endif
diff --git a/include/hw/mips/mips.h b/include/hw/mips/mips.h
index 2a7a9c9..e0065ce 100644
--- a/include/hw/mips/mips.h
+++ b/include/hw/mips/mips.h
@@ -15,18 +15,9 @@
 
 /* rc4030.c */
 typedef struct rc4030DMAState *rc4030_dma;
-void rc4030_dma_memory_rw(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write);
 void rc4030_dma_read(void *dma, uint8_t *buf, int len);
 void rc4030_dma_write(void *dma, uint8_t *buf, int len);
 
-void *rc4030_init(qemu_irq timer, qemu_irq jazz_bus,
-                  qemu_irq **irqs, rc4030_dma **dmas,
-                  MemoryRegion *sysmem);
-
-/* dp8393x.c */
-void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
-                  MemoryRegion *address_space,
-                  qemu_irq irq, void* mem_opaque,
-                  void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write));
+DeviceState *rc4030_init(rc4030_dma **dmas, MemoryRegion **dma_mr);
 
 #endif
diff --git a/include/hw/misc/android_qemud.h b/include/hw/misc/android_qemud.h
deleted file mode 100644
index 327ad9a..0000000
--- a/include/hw/misc/android_qemud.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* Copyright (C) 2015 Linaro Limited
-**
-** This software is licensed under the terms of the GNU General Public
-** License version 2, as published by the Free Software Foundation, and
-** may be copied, distributed, and modified under those terms.
-**
-** This program is distributed in the hope that it will be useful,
-** but WITHOUT ANY WARRANTY; without even the implied warranty of
-** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-** GNU General Public License for more details.
-**
-** Simple QEMUD framing wrappers
-*/
-
-#ifndef ANDROID_QEMUD_H
-#define ANDROID_QEMUD_H
-
-#include <glib.h>
-
-/* Read Android buffers into an array of terminated strings */
-int qemud_pipe_read_buffers(const AndroidPipeBuffer *buf, int cnt,
-                            GPtrArray *msgs, GString *remaining);
-
-/* Write an array of strings into a series on Android pipe buffers */
-int qemud_pipe_write_buffers(GPtrArray *msgs, GString *remaining,
-                             AndroidPipeBuffer *buf, int cnt);
-
-#endif /* ANDROID_QEMUD_H */
diff --git a/include/hw/misc/arm_integrator_debug.h b/include/hw/misc/arm_integrator_debug.h
index 37789b6..0077dac 100644
--- a/include/hw/misc/arm_integrator_debug.h
+++ b/include/hw/misc/arm_integrator_debug.h
@@ -10,8 +10,9 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#ifndef QEMU_INTEGRATOR_DEBUG_H
-#define QEMU_INTEGRATOR_DEBUG_H
+
+#ifndef ARM_INTEGRATOR_DEBUG_H
+#define ARM_INTEGRATOR_DEBUG_H
 
 #define TYPE_INTEGRATOR_DEBUG "integrator_debug"
 
diff --git a/include/hw/misc/aspeed_scu.h b/include/hw/misc/aspeed_scu.h
new file mode 100644
index 0000000..fdfd982
--- /dev/null
+++ b/include/hw/misc/aspeed_scu.h
@@ -0,0 +1,39 @@
+/*
+ * ASPEED System Control Unit
+ *
+ * Andrew Jeffery <andrew@aj.id.au>
+ *
+ * Copyright 2016 IBM Corp.
+ *
+ * This code is licensed under the GPL version 2 or later.  See
+ * the COPYING file in the top-level directory.
+ */
+#ifndef ASPEED_SCU_H
+#define ASPEED_SCU_H
+
+#include "hw/sysbus.h"
+
+#define TYPE_ASPEED_SCU "aspeed.scu"
+#define ASPEED_SCU(obj) OBJECT_CHECK(AspeedSCUState, (obj), TYPE_ASPEED_SCU)
+
+#define ASPEED_SCU_NR_REGS (0x1A8 >> 2)
+
+typedef struct AspeedSCUState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t regs[ASPEED_SCU_NR_REGS];
+    uint32_t silicon_rev;
+    uint32_t hw_strap1;
+    uint32_t hw_strap2;
+} AspeedSCUState;
+
+#define AST2400_A0_SILICON_REV   0x02000303U
+#define AST2500_A0_SILICON_REV   0x04000303U
+
+extern bool is_supported_silicon_rev(uint32_t silicon_rev);
+
+#endif /* ASPEED_SCU_H */
diff --git a/include/hw/misc/auxbus.h b/include/hw/misc/auxbus.h
new file mode 100644
index 0000000..68ade8a
--- /dev/null
+++ b/include/hw/misc/auxbus.h
@@ -0,0 +1,128 @@
+/*
+ * auxbus.h
+ *
+ *  Copyright (C)2014 : GreenSocs Ltd
+ *      http://www.greensocs.com/ , email: info@greensocs.com
+ *
+ *  Developed by :
+ *  Frederic Konrad   <fred.konrad@greensocs.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option)any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef HW_MISC_AUXBUS_H
+#define HW_MISC_AUXBUS_H
+
+#include "hw/qdev.h"
+
+typedef struct AUXBus AUXBus;
+typedef struct AUXSlave AUXSlave;
+typedef enum AUXCommand AUXCommand;
+typedef enum AUXReply AUXReply;
+typedef struct AUXTOI2CState AUXTOI2CState;
+
+enum AUXCommand {
+    WRITE_I2C = 0,
+    READ_I2C = 1,
+    WRITE_I2C_STATUS = 2,
+    WRITE_I2C_MOT = 4,
+    READ_I2C_MOT = 5,
+    WRITE_AUX = 8,
+    READ_AUX = 9
+};
+
+enum AUXReply {
+    AUX_I2C_ACK = 0,
+    AUX_NACK = 1,
+    AUX_DEFER = 2,
+    AUX_I2C_NACK = 4,
+    AUX_I2C_DEFER = 8
+};
+
+#define TYPE_AUX_BUS "aux-bus"
+#define AUX_BUS(obj) OBJECT_CHECK(AUXBus, (obj), TYPE_AUX_BUS)
+
+struct AUXBus {
+    /* < private > */
+    BusState qbus;
+
+    /* < public > */
+    AUXSlave *current_dev;
+    AUXSlave *dev;
+    uint32_t last_i2c_address;
+    AUXCommand last_transaction;
+
+    AUXTOI2CState *bridge;
+
+    MemoryRegion *aux_io;
+    AddressSpace aux_addr_space;
+};
+
+#define TYPE_AUX_SLAVE "aux-slave"
+#define AUX_SLAVE(obj) \
+     OBJECT_CHECK(AUXSlave, (obj), TYPE_AUX_SLAVE)
+
+struct AUXSlave {
+    /* < private > */
+    DeviceState parent_obj;
+
+    /* < public > */
+    MemoryRegion *mmio;
+};
+
+/**
+ * aux_init_bus: Initialize an AUX bus.
+ *
+ * Returns the new AUX bus created.
+ *
+ * @parent The device where this bus is located.
+ * @name The name of the bus.
+ */
+AUXBus *aux_init_bus(DeviceState *parent, const char *name);
+
+/*
+ * aux_request: Make a request on the bus.
+ *
+ * Returns the reply of the request.
+ *
+ * @bus Ths bus where the request happen.
+ * @cmd The command requested.
+ * @address The 20bits address of the slave.
+ * @len The length of the read or write.
+ * @data The data array which will be filled or read during transfer.
+ */
+AUXReply aux_request(AUXBus *bus, AUXCommand cmd, uint32_t address,
+                              uint8_t len, uint8_t *data);
+
+/*
+ * aux_get_i2c_bus: Get the i2c bus for I2C over AUX command.
+ *
+ * Returns the i2c bus associated to this AUX bus.
+ *
+ * @bus The AUX bus.
+ */
+I2CBus *aux_get_i2c_bus(AUXBus *bus);
+
+/*
+ * aux_init_mmio: Init an mmio for an AUX slave.
+ *
+ * @aux_slave The AUX slave.
+ * @mmio The mmio to be registered.
+ */
+void aux_init_mmio(AUXSlave *aux_slave, MemoryRegion *mmio);
+
+DeviceState *aux_create_slave(AUXBus *bus, const char *name, uint32_t addr);
+
+#endif /* HW_MISC_AUXBUS_H */
diff --git a/include/hw/misc/bcm2835_mbox.h b/include/hw/misc/bcm2835_mbox.h
new file mode 100644
index 0000000..f4e9ff9
--- /dev/null
+++ b/include/hw/misc/bcm2835_mbox.h
@@ -0,0 +1,38 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_MBOX_H
+#define BCM2835_MBOX_H
+
+#include "bcm2835_mbox_defs.h"
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+
+#define TYPE_BCM2835_MBOX "bcm2835-mbox"
+#define BCM2835_MBOX(obj) \
+        OBJECT_CHECK(BCM2835MboxState, (obj), TYPE_BCM2835_MBOX)
+
+typedef struct {
+    uint32_t reg[MBOX_SIZE];
+    uint32_t count;
+    uint32_t status;
+    uint32_t config;
+} BCM2835Mbox;
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+    MemoryRegion *mbox_mr;
+    AddressSpace mbox_as;
+    MemoryRegion iomem;
+    qemu_irq arm_irq;
+
+    bool mbox_irq_disabled;
+    bool available[MBOX_CHAN_COUNT];
+    BCM2835Mbox mbox[2];
+} BCM2835MboxState;
+
+#endif
diff --git a/include/hw/misc/bcm2835_mbox_defs.h b/include/hw/misc/bcm2835_mbox_defs.h
new file mode 100644
index 0000000..a18e520
--- /dev/null
+++ b/include/hw/misc/bcm2835_mbox_defs.h
@@ -0,0 +1,27 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_MBOX_DEFS_H
+#define BCM2835_MBOX_DEFS_H
+
+/* Constants shared with the ARM identifying separate mailbox channels */
+#define MBOX_CHAN_POWER    0 /* for use by the power management interface */
+#define MBOX_CHAN_FB       1 /* for use by the frame buffer */
+#define MBOX_CHAN_VCHIQ    3 /* for use by the VCHIQ interface */
+#define MBOX_CHAN_PROPERTY 8 /* for use by the property channel */
+#define MBOX_CHAN_COUNT    9
+
+#define MBOX_SIZE          32
+#define MBOX_INVALID_DATA  0x0f
+
+/* Layout of the private address space used for communication between
+ * the mbox device emulation, and child devices: each channel occupies
+ * 16 bytes of address space, but only two registers are presently defined.
+ */
+#define MBOX_AS_CHAN_SHIFT 4
+#define MBOX_AS_DATA       0 /* request / response data (RW at offset 0) */
+#define MBOX_AS_PENDING    4 /* pending response status (RO at offset 4) */
+
+#endif /* BCM2835_MBOX_DEFS_H */
diff --git a/include/hw/misc/bcm2835_property.h b/include/hw/misc/bcm2835_property.h
new file mode 100644
index 0000000..edcab60
--- /dev/null
+++ b/include/hw/misc/bcm2835_property.h
@@ -0,0 +1,35 @@
+/*
+ * Raspberry Pi emulation (c) 2012 Gregory Estrade
+ * This code is licensed under the GNU GPLv2 and later.
+ */
+
+#ifndef BCM2835_PROPERTY_H
+#define BCM2835_PROPERTY_H
+
+#include "hw/sysbus.h"
+#include "exec/address-spaces.h"
+#include "net/net.h"
+#include "hw/display/bcm2835_fb.h"
+
+#define TYPE_BCM2835_PROPERTY "bcm2835-property"
+#define BCM2835_PROPERTY(obj) \
+        OBJECT_CHECK(BCM2835PropertyState, (obj), TYPE_BCM2835_PROPERTY)
+
+typedef struct {
+    /*< private >*/
+    SysBusDevice busdev;
+    /*< public >*/
+
+    MemoryRegion *dma_mr;
+    AddressSpace dma_as;
+    MemoryRegion iomem;
+    qemu_irq mbox_irq;
+    BCM2835FBState *fbdev;
+
+    MACAddr macaddr;
+    uint32_t board_rev;
+    uint32_t addr;
+    bool pending;
+} BCM2835PropertyState;
+
+#endif
diff --git a/include/hw/misc/goldfish_sync.h b/include/hw/misc/goldfish_sync.h
index 26d714a..bb85349 100644
--- a/include/hw/misc/goldfish_sync.h
+++ b/include/hw/misc/goldfish_sync.h
@@ -32,14 +32,14 @@
  * and will be called by the virtual device at runtime. */
 typedef struct {
     /* Called when the guest sends the result of a previous command to the
-     * host. See goldfish_send_send_command(). */
+     * host. */
     void (*receive_hostcmd_result)(uint32_t cmd,
                                    uint64_t handle,
                                    uint32_t time_arg,
                                    uint64_t hostcmd_handle);
 
     /* Called when the guest wants to trigger a host-side wait for a
-     * specific glsync and thread pointer pair. */
+     * specific glsync and thread pointer pair */
     void (*trigger_host_wait)(uint64_t glsync_ptr,
                               uint64_t thread_ptr,
                               uint64_t timeline);
@@ -49,9 +49,7 @@
  * must be called at emulation setup time before the device runs. */
 void goldfish_sync_set_service_ops(const GoldfishSyncServiceOps *ops);
 
-/* Send a command to the guest through the goldfish_sync device.
- * The result will be sent asynchronously through the |receive_hostcmd_result|
- * callback. */
+/* Send a command to the guest through the goldfish_sync device. */
 void goldfish_sync_send_command(uint32_t cmd,
                                 uint64_t handle,
                                 uint32_t time_arg,
diff --git a/include/hw/misc/imx25_ccm.h b/include/hw/misc/imx25_ccm.h
new file mode 100644
index 0000000..296321c
--- /dev/null
+++ b/include/hw/misc/imx25_ccm.h
@@ -0,0 +1,79 @@
+/*
+ * IMX25 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX25_CCM_H
+#define IMX25_CCM_H
+
+#include "hw/misc/imx_ccm.h"
+
+#define IMX25_CCM_MPCTL_REG  0
+#define IMX25_CCM_UPCTL_REG  1
+#define IMX25_CCM_CCTL_REG   2
+#define IMX25_CCM_CGCR0_REG  3
+#define IMX25_CCM_CGCR1_REG  4
+#define IMX25_CCM_CGCR2_REG  5
+#define IMX25_CCM_PCDR0_REG  6
+#define IMX25_CCM_PCDR1_REG  7
+#define IMX25_CCM_PCDR2_REG  8
+#define IMX25_CCM_PCDR3_REG  9
+#define IMX25_CCM_RCSR_REG   10
+#define IMX25_CCM_CRDR_REG   11
+#define IMX25_CCM_DCVR0_REG  12
+#define IMX25_CCM_DCVR1_REG  13
+#define IMX25_CCM_DCVR2_REG  14
+#define IMX25_CCM_DCVR3_REG  15
+#define IMX25_CCM_LTR0_REG   16
+#define IMX25_CCM_LTR1_REG   17
+#define IMX25_CCM_LTR2_REG   18
+#define IMX25_CCM_LTR3_REG   19
+#define IMX25_CCM_LTBR0_REG  20
+#define IMX25_CCM_LTBR1_REG  21
+#define IMX25_CCM_PMCR0_REG  22
+#define IMX25_CCM_PMCR1_REG  23
+#define IMX25_CCM_PMCR2_REG  24
+#define IMX25_CCM_MCR_REG    25
+#define IMX25_CCM_LPIMR0_REG 26
+#define IMX25_CCM_LPIMR1_REG 27
+#define IMX25_CCM_MAX_REG    28
+
+/* CCTL */
+#define CCTL_ARM_CLK_DIV_SHIFT (30)
+#define CCTL_ARM_CLK_DIV_MASK  (0x3)
+#define CCTL_AHB_CLK_DIV_SHIFT (28)
+#define CCTL_AHB_CLK_DIV_MASK  (0x3)
+#define CCTL_MPLL_BYPASS_SHIFT (22)
+#define CCTL_MPLL_BYPASS_MASK  (0x1)
+#define CCTL_USB_DIV_SHIFT     (16)
+#define CCTL_USB_DIV_MASK      (0x3F)
+#define CCTL_ARM_SRC_SHIFT     (13)
+#define CCTL_ARM_SRC_MASK      (0x1)
+#define CCTL_UPLL_DIS_SHIFT    (23)
+#define CCTL_UPLL_DIS_MASK     (0x1)
+
+#define EXTRACT(value, name) (((value) >> CCTL_##name##_SHIFT) \
+                              & CCTL_##name##_MASK)
+#define INSERT(value, name) (((value) & CCTL_##name##_MASK) << \
+                             CCTL_##name##_SHIFT)
+
+#define TYPE_IMX25_CCM "imx25.ccm"
+#define IMX25_CCM(obj) OBJECT_CHECK(IMX25CCMState, (obj), TYPE_IMX25_CCM)
+
+typedef struct IMX25CCMState {
+    /* <private> */
+    IMXCCMState parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    uint32_t reg[IMX25_CCM_MAX_REG];
+
+} IMX25CCMState;
+
+#endif /* IMX25_CCM_H */
diff --git a/include/hw/misc/imx31_ccm.h b/include/hw/misc/imx31_ccm.h
new file mode 100644
index 0000000..c376fad
--- /dev/null
+++ b/include/hw/misc/imx31_ccm.h
@@ -0,0 +1,88 @@
+/*
+ * IMX31 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX31_CCM_H
+#define IMX31_CCM_H
+
+#include "hw/misc/imx_ccm.h"
+
+#define IMX31_CCM_CCMR_REG  0
+#define IMX31_CCM_PDR0_REG  1
+#define IMX31_CCM_PDR1_REG  2
+#define IMX31_CCM_RCSR_REG  3
+#define IMX31_CCM_MPCTL_REG 4
+#define IMX31_CCM_UPCTL_REG 5
+#define IMX31_CCM_SPCTL_REG 6
+#define IMX31_CCM_COSR_REG  7
+#define IMX31_CCM_CGR0_REG  8
+#define IMX31_CCM_CGR1_REG  9
+#define IMX31_CCM_CGR2_REG  10
+#define IMX31_CCM_WIMR_REG  11
+#define IMX31_CCM_LDC_REG   12
+#define IMX31_CCM_DCVR0_REG 13
+#define IMX31_CCM_DCVR1_REG 14
+#define IMX31_CCM_DCVR2_REG 15
+#define IMX31_CCM_DCVR3_REG 16
+#define IMX31_CCM_LTR0_REG  17
+#define IMX31_CCM_LTR1_REG  18
+#define IMX31_CCM_LTR2_REG  19
+#define IMX31_CCM_LTR3_REG  20
+#define IMX31_CCM_LTBR0_REG 21
+#define IMX31_CCM_LTBR1_REG 22
+#define IMX31_CCM_PMCR0_REG 23
+#define IMX31_CCM_PMCR1_REG 24
+#define IMX31_CCM_PDR2_REG  25
+#define IMX31_CCM_MAX_REG   26
+
+/* CCMR */
+#define CCMR_FPME    (1<<0)
+#define CCMR_MPE     (1<<3)
+#define CCMR_MDS     (1<<7)
+#define CCMR_FPMF    (1<<26)
+#define CCMR_PRCS    (3<<1)
+
+#define PMCR0_DFSUP1 (1<<31)
+
+/* PDR0 */
+#define PDR0_MCU_PODF_SHIFT (0)
+#define PDR0_MCU_PODF_MASK (0x7)
+#define PDR0_MAX_PODF_SHIFT (3)
+#define PDR0_MAX_PODF_MASK (0x7)
+#define PDR0_IPG_PODF_SHIFT (6)
+#define PDR0_IPG_PODF_MASK (0x3)
+#define PDR0_NFC_PODF_SHIFT (8)
+#define PDR0_NFC_PODF_MASK (0x7)
+#define PDR0_HSP_PODF_SHIFT (11)
+#define PDR0_HSP_PODF_MASK (0x7)
+#define PDR0_PER_PODF_SHIFT (16)
+#define PDR0_PER_PODF_MASK (0x1f)
+#define PDR0_CSI_PODF_SHIFT (23)
+#define PDR0_CSI_PODF_MASK (0x1ff)
+
+#define EXTRACT(value, name) (((value) >> PDR0_##name##_PODF_SHIFT) \
+                              & PDR0_##name##_PODF_MASK)
+#define INSERT(value, name) (((value) & PDR0_##name##_PODF_MASK) << \
+                             PDR0_##name##_PODF_SHIFT)
+
+#define TYPE_IMX31_CCM "imx31.ccm"
+#define IMX31_CCM(obj) OBJECT_CHECK(IMX31CCMState, (obj), TYPE_IMX31_CCM)
+
+typedef struct IMX31CCMState {
+    /* <private> */
+    IMXCCMState parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    uint32_t reg[IMX31_CCM_MAX_REG];
+
+} IMX31CCMState;
+
+#endif /* IMX31_CCM_H */
diff --git a/include/hw/misc/imx6_ccm.h b/include/hw/misc/imx6_ccm.h
new file mode 100644
index 0000000..8050580
--- /dev/null
+++ b/include/hw/misc/imx6_ccm.h
@@ -0,0 +1,197 @@
+/*
+ * IMX6 Clock Control Module
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX6_CCM_H
+#define IMX6_CCM_H
+
+#include "hw/misc/imx_ccm.h"
+#include "qemu/bitops.h"
+
+#define CCM_CCR 0
+#define CCM_CCDR 1
+#define CCM_CSR 2
+#define CCM_CCSR 3
+#define CCM_CACRR 4
+#define CCM_CBCDR 5
+#define CCM_CBCMR 6
+#define CCM_CSCMR1 7
+#define CCM_CSCMR2 8
+#define CCM_CSCDR1 9
+#define CCM_CS1CDR 10
+#define CCM_CS2CDR 11
+#define CCM_CDCDR 12
+#define CCM_CHSCCDR 13
+#define CCM_CSCDR2 14
+#define CCM_CSCDR3 15
+#define CCM_CDHIPR 18
+#define CCM_CTOR 20
+#define CCM_CLPCR 21
+#define CCM_CISR 22
+#define CCM_CIMR 23
+#define CCM_CCOSR 24
+#define CCM_CGPR 25
+#define CCM_CCGR0 26
+#define CCM_CCGR1 27
+#define CCM_CCGR2 28
+#define CCM_CCGR3 29
+#define CCM_CCGR4 30
+#define CCM_CCGR5 31
+#define CCM_CCGR6 32
+#define CCM_CMEOR 34
+#define CCM_MAX 35
+
+#define CCM_ANALOG_PLL_ARM 0
+#define CCM_ANALOG_PLL_ARM_SET 1
+#define CCM_ANALOG_PLL_ARM_CLR 2
+#define CCM_ANALOG_PLL_ARM_TOG 3
+#define CCM_ANALOG_PLL_USB1 4
+#define CCM_ANALOG_PLL_USB1_SET 5
+#define CCM_ANALOG_PLL_USB1_CLR 6
+#define CCM_ANALOG_PLL_USB1_TOG 7
+#define CCM_ANALOG_PLL_USB2 8
+#define CCM_ANALOG_PLL_USB2_SET 9
+#define CCM_ANALOG_PLL_USB2_CLR 10
+#define CCM_ANALOG_PLL_USB2_TOG 11
+#define CCM_ANALOG_PLL_SYS 12
+#define CCM_ANALOG_PLL_SYS_SET 13
+#define CCM_ANALOG_PLL_SYS_CLR 14
+#define CCM_ANALOG_PLL_SYS_TOG 15
+#define CCM_ANALOG_PLL_SYS_SS 16
+#define CCM_ANALOG_PLL_SYS_NUM 20
+#define CCM_ANALOG_PLL_SYS_DENOM 24
+#define CCM_ANALOG_PLL_AUDIO 28
+#define CCM_ANALOG_PLL_AUDIO_SET 29
+#define CCM_ANALOG_PLL_AUDIO_CLR 30
+#define CCM_ANALOG_PLL_AUDIO_TOG 31
+#define CCM_ANALOG_PLL_AUDIO_NUM 32
+#define CCM_ANALOG_PLL_AUDIO_DENOM 36
+#define CCM_ANALOG_PLL_VIDEO 40
+#define CCM_ANALOG_PLL_VIDEO_SET 41
+#define CCM_ANALOG_PLL_VIDEO_CLR 42
+#define CCM_ANALOG_PLL_VIDEO_TOG 44
+#define CCM_ANALOG_PLL_VIDEO_NUM 46
+#define CCM_ANALOG_PLL_VIDEO_DENOM 48
+#define CCM_ANALOG_PLL_MLB 52
+#define CCM_ANALOG_PLL_MLB_SET 53
+#define CCM_ANALOG_PLL_MLB_CLR 54
+#define CCM_ANALOG_PLL_MLB_TOG 55
+#define CCM_ANALOG_PLL_ENET 56
+#define CCM_ANALOG_PLL_ENET_SET 57
+#define CCM_ANALOG_PLL_ENET_CLR 58
+#define CCM_ANALOG_PLL_ENET_TOG 59
+#define CCM_ANALOG_PFD_480 60
+#define CCM_ANALOG_PFD_480_SET 61
+#define CCM_ANALOG_PFD_480_CLR 62
+#define CCM_ANALOG_PFD_480_TOG 63
+#define CCM_ANALOG_PFD_528 64
+#define CCM_ANALOG_PFD_528_SET 65
+#define CCM_ANALOG_PFD_528_CLR 66
+#define CCM_ANALOG_PFD_528_TOG 67
+
+/* PMU registers */
+#define PMU_REG_1P1 68
+#define PMU_REG_3P0 72
+#define PMU_REG_2P5 76
+#define PMU_REG_CORE 80
+
+#define CCM_ANALOG_MISC0 84
+#define PMU_MISC0 84
+#define CCM_ANALOG_MISC0_SET 85
+#define CCM_ANALOG_MISC0_CLR 86
+#define CCM_ANALOG_MISC0_TOG 87
+
+#define PMU_MISC1 88
+#define PMU_MISC1_SET 89
+#define PMU_MISC1_CLR 90
+#define PMU_MISC1_TOG 91
+
+#define CCM_ANALOG_MISC2 92
+#define PMU_MISC2 92
+#define CCM_ANALOG_MISC2_SET 93
+#define CCM_ANALOG_MISC2_CLR 94
+#define CCM_ANALOG_MISC2_TOG 95
+
+#define USB_ANALOG_USB1_VBUS_DETECT 104
+#define USB_ANALOG_USB1_VBUS_DETECT_SET 105
+#define USB_ANALOG_USB1_VBUS_DETECT_CLR 106
+#define USB_ANALOG_USB1_VBUS_DETECT_TOG 107
+#define USB_ANALOG_USB1_CHRG_DETECT 108
+#define USB_ANALOG_USB1_CHRG_DETECT_SET 109
+#define USB_ANALOG_USB1_CHRG_DETECT_CLR 110
+#define USB_ANALOG_USB1_CHRG_DETECT_TOG 111
+#define USB_ANALOG_USB1_VBUS_DETECT_STAT 112
+#define USB_ANALOG_USB1_CHRG_DETECT_STAT 116
+#define USB_ANALOG_USB1_MISC 124
+#define USB_ANALOG_USB1_MISC_SET 125
+#define USB_ANALOG_USB1_MISC_CLR 126
+#define USB_ANALOG_USB1_MISC_TOG 127
+#define USB_ANALOG_USB2_VBUS_DETECT 128
+#define USB_ANALOG_USB2_VBUS_DETECT_SET 129
+#define USB_ANALOG_USB2_VBUS_DETECT_CLR 130
+#define USB_ANALOG_USB2_VBUS_DETECT_TOG 131
+#define USB_ANALOG_USB2_CHRG_DETECT 132
+#define USB_ANALOG_USB2_CHRG_DETECT_SET 133
+#define USB_ANALOG_USB2_CHRG_DETECT_CLR 134
+#define USB_ANALOG_USB2_CHRG_DETECT_TOG 135
+#define USB_ANALOG_USB2_VBUS_DETECT_STAT 136
+#define USB_ANALOG_USB2_CHRG_DETECT_STAT 140
+#define USB_ANALOG_USB2_MISC 148
+#define USB_ANALOG_USB2_MISC_SET 149
+#define USB_ANALOG_USB2_MISC_CLR 150
+#define USB_ANALOG_USB2_MISC_TOG 151
+#define USB_ANALOG_DIGPROG 152
+#define CCM_ANALOG_MAX 153
+
+/* CCM_CBCMR */
+#define PRE_PERIPH_CLK_SEL_SHIFT  (18)
+#define PRE_PERIPH_CLK_SEL_LENGTH (2)
+
+/* CCM_CBCDR */
+#define AHB_PODF_SHIFT           (10)
+#define AHB_PODF_LENGTH          (3)
+#define IPG_PODF_SHIFT           (8)
+#define IPG_PODF_LENGTH          (2)
+
+/* CCM_CSCMR1 */
+#define PERCLK_PODF_SHIFT        (0)
+#define PERCLK_PODF_LENGTH       (6)
+
+/* CCM_ANALOG_PFD_528 */
+#define PFD0_FRAC_SHIFT          (0)
+#define PFD0_FRAC_LENGTH         (6)
+#define PFD2_FRAC_SHIFT          (16)
+#define PFD2_FRAC_LENGTH         (6)
+
+/* CCM_ANALOG_PLL_SYS */
+#define DIV_SELECT_SHIFT         (0)
+#define DIV_SELECT_LENGTH        (1)
+
+#define CCM_ANALOG_PLL_LOCK      (1 << 31);
+
+#define EXTRACT(value, name) extract32(value, name##_SHIFT, name##_LENGTH)
+
+#define TYPE_IMX6_CCM "imx6.ccm"
+#define IMX6_CCM(obj) OBJECT_CHECK(IMX6CCMState, (obj), TYPE_IMX6_CCM)
+
+typedef struct IMX6CCMState {
+    /* <private> */
+    IMXCCMState parent_obj;
+
+    /* <public> */
+    MemoryRegion container;
+    MemoryRegion ioccm;
+    MemoryRegion ioanalog;
+
+    uint32_t ccm[CCM_MAX];
+    uint32_t analog[CCM_ANALOG_MAX];
+
+} IMX6CCMState;
+
+#endif /* IMX6_CCM_H */
diff --git a/include/hw/misc/imx6_src.h b/include/hw/misc/imx6_src.h
new file mode 100644
index 0000000..eb36407
--- /dev/null
+++ b/include/hw/misc/imx6_src.h
@@ -0,0 +1,73 @@
+/*
+ * IMX6 System Reset Controller
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX6_SRC_H
+#define IMX6_SRC_H
+
+#include "hw/sysbus.h"
+#include "qemu/bitops.h"
+
+#define SRC_SCR 0
+#define SRC_SBMR1 1
+#define SRC_SRSR 2
+#define SRC_SISR 5
+#define SRC_SIMR 6
+#define SRC_SBMR2 7
+#define SRC_GPR1 8
+#define SRC_GPR2 9
+#define SRC_GPR3 10
+#define SRC_GPR4 11
+#define SRC_GPR5 12
+#define SRC_GPR6 13
+#define SRC_GPR7 14
+#define SRC_GPR8 15
+#define SRC_GPR9 16
+#define SRC_GPR10 17
+#define SRC_MAX 18
+
+/* SRC_SCR */
+#define CORE3_ENABLE_SHIFT     24
+#define CORE3_ENABLE_LENGTH    1
+#define CORE2_ENABLE_SHIFT     23
+#define CORE2_ENABLE_LENGTH    1
+#define CORE1_ENABLE_SHIFT     22
+#define CORE1_ENABLE_LENGTH    1
+#define CORE3_RST_SHIFT        16
+#define CORE3_RST_LENGTH       1
+#define CORE2_RST_SHIFT        15
+#define CORE2_RST_LENGTH       1
+#define CORE1_RST_SHIFT        14
+#define CORE1_RST_LENGTH       1
+#define CORE0_RST_SHIFT        13
+#define CORE0_RST_LENGTH       1
+#define SW_IPU1_RST_SHIFT      3
+#define SW_IPU1_RST_LENGTH     1
+#define SW_IPU2_RST_SHIFT      12
+#define SW_IPU2_RST_LENGTH     1
+#define WARM_RST_ENABLE_SHIFT  0
+#define WARM_RST_ENABLE_LENGTH 1
+
+#define EXTRACT(value, name) extract32(value, name##_SHIFT, name##_LENGTH)
+
+#define TYPE_IMX6_SRC "imx6.src"
+#define IMX6_SRC(obj) OBJECT_CHECK(IMX6SRCState, (obj), TYPE_IMX6_SRC)
+
+typedef struct IMX6SRCState {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    uint32_t regs[SRC_MAX];
+
+} IMX6SRCState;
+
+#endif /* IMX6_SRC_H */
diff --git a/include/hw/misc/imx_ccm.h b/include/hw/misc/imx_ccm.h
new file mode 100644
index 0000000..33cbc09
--- /dev/null
+++ b/include/hw/misc/imx_ccm.h
@@ -0,0 +1,67 @@
+/*
+ * IMX Clock Control Module base class
+ *
+ * Copyright (C) 2012 NICTA
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX_CCM_H
+#define IMX_CCM_H
+
+#include "hw/sysbus.h"
+
+#define CKIL_FREQ 32768 /* nominal 32khz clock */
+
+/* PLL control registers */
+#define PD(v) (((v) >> 26) & 0xf)
+#define MFD(v) (((v) >> 16) & 0x3ff)
+#define MFI(v) (((v) >> 10) & 0xf);
+#define MFN(v) ((v) & 0x3ff)
+
+#define PLL_PD(x)               (((x) & 0xf) << 26)
+#define PLL_MFD(x)              (((x) & 0x3ff) << 16)
+#define PLL_MFI(x)              (((x) & 0xf) << 10)
+#define PLL_MFN(x)              (((x) & 0x3ff) << 0)
+
+#define TYPE_IMX_CCM "imx.ccm"
+#define IMX_CCM(obj) \
+     OBJECT_CHECK(IMXCCMState, (obj), TYPE_IMX_CCM)
+#define IMX_CCM_CLASS(klass) \
+     OBJECT_CLASS_CHECK(IMXCCMClass, (klass), TYPE_IMX_CCM)
+#define IMX_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(IMXCCMClass, (obj), TYPE_IMX_CCM)
+
+typedef struct IMXCCMState {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+
+} IMXCCMState;
+
+typedef enum  {
+    CLK_NONE,
+    CLK_IPG,
+    CLK_IPG_HIGH,
+    CLK_32k,
+    CLK_EXT,
+    CLK_HIGH_DIV,
+    CLK_HIGH,
+} IMXClk;
+
+typedef struct IMXCCMClass {
+    /* <private> */
+    SysBusDeviceClass parent_class;
+
+    /* <public> */
+    uint32_t (*get_clock_frequency)(IMXCCMState *s, IMXClk clk);
+} IMXCCMClass;
+
+uint32_t imx_ccm_calc_pll(uint32_t pllreg, uint32_t base_freq);
+
+uint32_t imx_ccm_get_clock_frequency(IMXCCMState *s, IMXClk clock);
+
+#endif /* IMX_CCM_H */
diff --git a/include/hw/misc/ivshmem.h b/include/hw/misc/ivshmem.h
new file mode 100644
index 0000000..433ef53
--- /dev/null
+++ b/include/hw/misc/ivshmem.h
@@ -0,0 +1,25 @@
+
+/*
+ * Inter-VM Shared Memory PCI device.
+ *
+ * Author:
+ *      Cam Macdonell <cam@cs.ualberta.ca>
+ *
+ * Based On: cirrus_vga.c
+ *          Copyright (c) 2004 Fabrice Bellard
+ *          Copyright (c) 2004 Makoto Suzuki (suzu)
+ *
+ *      and rtl8139.c
+ *          Copyright (c) 2006 Igor Kovalenko
+ *
+ * This code is licensed under the GNU GPL v2.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#ifndef IVSHMEM_H
+#define IVSHMEM_H
+
+#define IVSHMEM_PROTOCOL_VERSION 0
+
+#endif /* IVSHMEM_H */
diff --git a/include/hw/misc/mips_cmgcr.h b/include/hw/misc/mips_cmgcr.h
new file mode 100644
index 0000000..a209d91
--- /dev/null
+++ b/include/hw/misc/mips_cmgcr.h
@@ -0,0 +1,86 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2015 Imagination Technologies
+ *
+ */
+
+#ifndef MIPS_CMGCR_H
+#define MIPS_CMGCR_H
+
+#define TYPE_MIPS_GCR "mips-gcr"
+#define MIPS_GCR(obj) OBJECT_CHECK(MIPSGCRState, (obj), TYPE_MIPS_GCR)
+
+#define GCR_BASE_ADDR           0x1fbf8000ULL
+#define GCR_ADDRSPACE_SZ        0x8000
+
+/* Offsets to register blocks */
+#define MIPS_GCB_OFS        0x0000 /* Global Control Block */
+#define MIPS_CLCB_OFS       0x2000 /* Core Local Control Block */
+#define MIPS_COCB_OFS       0x4000 /* Core Other Control Block */
+#define MIPS_GDB_OFS        0x6000 /* Global Debug Block */
+
+/* Global Control Block Register Map */
+#define GCR_CONFIG_OFS      0x0000
+#define GCR_BASE_OFS        0x0008
+#define GCR_REV_OFS         0x0030
+#define GCR_GIC_BASE_OFS    0x0080
+#define GCR_CPC_BASE_OFS    0x0088
+#define GCR_GIC_STATUS_OFS  0x00D0
+#define GCR_CPC_STATUS_OFS  0x00F0
+#define GCR_L2_CONFIG_OFS   0x0130
+
+/* Core Local and Core Other Block Register Map */
+#define GCR_CL_CONFIG_OFS   0x0010
+#define GCR_CL_OTHER_OFS    0x0018
+#define GCR_CL_RESETBASE_OFS 0x0020
+
+/* GCR_L2_CONFIG register fields */
+#define GCR_L2_CONFIG_BYPASS_SHF    20
+#define GCR_L2_CONFIG_BYPASS_MSK    ((0x1ULL) << GCR_L2_CONFIG_BYPASS_SHF)
+
+/* GCR_GIC_BASE register fields */
+#define GCR_GIC_BASE_GICEN_MSK   1
+#define GCR_GIC_BASE_GICBASE_MSK 0xFFFFFFFE0000ULL
+#define GCR_GIC_BASE_MSK (GCR_GIC_BASE_GICEN_MSK | GCR_GIC_BASE_GICBASE_MSK)
+
+/* GCR_CPC_BASE register fields */
+#define GCR_CPC_BASE_CPCEN_MSK   1
+#define GCR_CPC_BASE_CPCBASE_MSK 0xFFFFFFFF8000ULL
+#define GCR_CPC_BASE_MSK (GCR_CPC_BASE_CPCEN_MSK | GCR_CPC_BASE_CPCBASE_MSK)
+
+/* GCR_CL_OTHER_OFS register fields */
+#define GCR_CL_OTHER_VPOTHER_MSK 0x7
+#define GCR_CL_OTHER_MSK GCR_CL_OTHER_VPOTHER_MSK
+
+/* GCR_CL_RESETBASE_OFS register fields */
+#define GCR_CL_RESET_BASE_RESETBASE_MSK 0xFFFFF000U
+#define GCR_CL_RESET_BASE_MSK GCR_CL_RESET_BASE_RESETBASE_MSK
+
+typedef struct MIPSGCRVPState MIPSGCRVPState;
+struct MIPSGCRVPState {
+    uint32_t other;
+    uint64_t reset_base;
+};
+
+typedef struct MIPSGCRState MIPSGCRState;
+struct MIPSGCRState {
+    SysBusDevice parent_obj;
+
+    int32_t gcr_rev;
+    int32_t num_vps;
+    hwaddr gcr_base;
+    MemoryRegion iomem;
+    MemoryRegion *cpc_mr;
+    MemoryRegion *gic_mr;
+
+    uint64_t cpc_base;
+    uint64_t gic_base;
+
+    /* VP Local/Other Registers */
+    MIPSGCRVPState *vps;
+};
+
+#endif /* MIPS_CMGCR_H */
diff --git a/include/hw/misc/mips_cpc.h b/include/hw/misc/mips_cpc.h
new file mode 100644
index 0000000..72c834e
--- /dev/null
+++ b/include/hw/misc/mips_cpc.h
@@ -0,0 +1,47 @@
+/*
+ * Cluster Power Controller emulation
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MIPS_CPC_H
+#define MIPS_CPC_H
+
+#define CPC_ADDRSPACE_SZ    0x6000
+
+/* CPC blocks offsets relative to base address */
+#define CPC_CL_BASE_OFS     0x2000
+#define CPC_CO_BASE_OFS     0x4000
+
+/* CPC register offsets relative to block offsets */
+#define CPC_VP_STOP_OFS     0x20
+#define CPC_VP_RUN_OFS      0x28
+#define CPC_VP_RUNNING_OFS  0x30
+
+#define TYPE_MIPS_CPC "mips-cpc"
+#define MIPS_CPC(obj) OBJECT_CHECK(MIPSCPCState, (obj), TYPE_MIPS_CPC)
+
+typedef struct MIPSCPCState {
+    SysBusDevice parent_obj;
+
+    uint32_t num_vp;
+    uint64_t vp_start_running; /* VPs running from restart */
+
+    MemoryRegion mr;
+    uint64_t vp_running; /* Indicates which VPs are in the run state */
+} MIPSCPCState;
+
+#endif /* MIPS_CPC_H */
diff --git a/include/hw/misc/mips_itu.h b/include/hw/misc/mips_itu.h
new file mode 100644
index 0000000..b3a4532
--- /dev/null
+++ b/include/hw/misc/mips_itu.h
@@ -0,0 +1,72 @@
+/*
+ * Inter-Thread Communication Unit emulation.
+ *
+ * Copyright (c) 2016 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MIPS_ITU_H
+#define MIPS_ITU_H
+
+#define TYPE_MIPS_ITU "mips-itu"
+#define MIPS_ITU(obj) OBJECT_CHECK(MIPSITUState, (obj), TYPE_MIPS_ITU)
+
+#define ITC_CELL_DEPTH_SHIFT 2
+#define ITC_CELL_DEPTH (1u << ITC_CELL_DEPTH_SHIFT)
+
+typedef struct ITCStorageCell {
+    struct {
+        uint8_t FIFODepth; /* Log2 of the cell depth */
+        uint8_t FIFOPtr; /* Number of elements in a FIFO cell */
+        uint8_t FIFO; /* 1 - FIFO cell, 0 - Semaphore cell */
+        uint8_t T; /* Trap Bit */
+        uint8_t F; /* Full Bit */
+        uint8_t E; /* Empty Bit */
+    } tag;
+
+    /* Index of the oldest element in the queue */
+    uint8_t fifo_out;
+
+    /* Circular buffer for FIFO. Semaphore cells use index 0 only */
+    uint64_t data[ITC_CELL_DEPTH];
+
+    /* Bitmap tracking blocked threads on the cell.
+       TODO: support >64 threads ? */
+    uint64_t blocked_threads;
+} ITCStorageCell;
+
+#define ITC_ADDRESSMAP_NUM 2
+
+typedef struct MIPSITUState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+    /*< public >*/
+
+    int32_t num_fifo;
+    int32_t num_semaphores;
+
+    /* ITC Storage */
+    ITCStorageCell *cell;
+    MemoryRegion storage_io;
+
+    /* ITC Configuration Tags */
+    uint64_t ITCAddressMap[ITC_ADDRESSMAP_NUM];
+    MemoryRegion tag_io;
+} MIPSITUState;
+
+/* Get ITC Configuration Tag memory region. */
+MemoryRegion *mips_itu_get_tag_region(MIPSITUState *itu);
+
+#endif /* MIPS_ITU_H */
diff --git a/include/hw/misc/stm32f2xx_syscfg.h b/include/hw/misc/stm32f2xx_syscfg.h
new file mode 100644
index 0000000..69e6a30
--- /dev/null
+++ b/include/hw/misc/stm32f2xx_syscfg.h
@@ -0,0 +1,61 @@
+/*
+ * STM32F2XX SYSCFG
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef HW_STM32F2XX_SYSCFG_H
+#define HW_STM32F2XX_SYSCFG_H
+
+#include "hw/sysbus.h"
+#include "hw/hw.h"
+
+#define SYSCFG_MEMRMP  0x00
+#define SYSCFG_PMC     0x04
+#define SYSCFG_EXTICR1 0x08
+#define SYSCFG_EXTICR2 0x0C
+#define SYSCFG_EXTICR3 0x10
+#define SYSCFG_EXTICR4 0x14
+#define SYSCFG_CMPCR   0x20
+
+#define TYPE_STM32F2XX_SYSCFG "stm32f2xx-syscfg"
+#define STM32F2XX_SYSCFG(obj) \
+    OBJECT_CHECK(STM32F2XXSyscfgState, (obj), TYPE_STM32F2XX_SYSCFG)
+
+typedef struct {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+    MemoryRegion mmio;
+
+    uint32_t syscfg_memrmp;
+    uint32_t syscfg_pmc;
+    uint32_t syscfg_exticr1;
+    uint32_t syscfg_exticr2;
+    uint32_t syscfg_exticr3;
+    uint32_t syscfg_exticr4;
+    uint32_t syscfg_cmpcr;
+
+    qemu_irq irq;
+} STM32F2XXSyscfgState;
+
+#endif /* HW_STM32F2XX_SYSCFG_H */
diff --git a/include/hw/misc/tmp105_regs.h b/include/hw/misc/tmp105_regs.h
index 9b55aba..ef015ee 100644
--- a/include/hw/misc/tmp105_regs.h
+++ b/include/hw/misc/tmp105_regs.h
@@ -11,8 +11,9 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or
  * later. See the COPYING file in the top-level directory.
  */
-#ifndef QEMU_TMP105_MSGS_H
-#define QEMU_TMP105_MSGS_H
+
+#ifndef TMP105_REGS_H
+#define TMP105_REGS_H
 
 /**
  * TMP105Reg:
diff --git a/include/hw/misc/vfio.h b/include/hw/misc/vfio.h
deleted file mode 100644
index 0b26cd8..0000000
--- a/include/hw/misc/vfio.h
+++ /dev/null
@@ -1,9 +0,0 @@
-#ifndef VFIO_API_H
-#define VFIO_API_H
-
-#include "qemu/typedefs.h"
-
-extern int vfio_container_ioctl(AddressSpace *as, int32_t groupid,
-                                int req, void *param);
-
-#endif
diff --git a/include/hw/misc/zynq-xadc.h b/include/hw/misc/zynq-xadc.h
new file mode 100644
index 0000000..f1a410a
--- /dev/null
+++ b/include/hw/misc/zynq-xadc.h
@@ -0,0 +1,46 @@
+/*
+ * Device model for Zynq ADC controller
+ *
+ * Copyright (c) 2015 Guenter Roeck <linux@roeck-us.net>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef ZYNQ_XADC_H
+#define ZYNQ_XADC_H
+
+#include "hw/sysbus.h"
+
+#define ZYNQ_XADC_MMIO_SIZE     0x0020
+#define ZYNQ_XADC_NUM_IO_REGS   (ZYNQ_XADC_MMIO_SIZE / 4)
+#define ZYNQ_XADC_NUM_ADC_REGS  128
+#define ZYNQ_XADC_FIFO_DEPTH    15
+
+#define TYPE_ZYNQ_XADC          "xlnx,zynq-xadc"
+#define ZYNQ_XADC(obj) \
+    OBJECT_CHECK(ZynqXADCState, (obj), TYPE_ZYNQ_XADC)
+
+typedef struct ZynqXADCState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t regs[ZYNQ_XADC_NUM_IO_REGS];
+    uint16_t xadc_regs[ZYNQ_XADC_NUM_ADC_REGS];
+    uint16_t xadc_read_reg_previous;
+    uint16_t xadc_dfifo[ZYNQ_XADC_FIFO_DEPTH];
+    uint16_t xadc_dfifo_entries;
+
+    struct IRQState *qemu_irq;
+
+} ZynqXADCState;
+
+#endif /* ZYNQ_XADC_H */
diff --git a/include/hw/net/allwinner_emac.h b/include/hw/net/allwinner_emac.h
index 5ae7717..4cc8aab 100644
--- a/include/hw/net/allwinner_emac.h
+++ b/include/hw/net/allwinner_emac.h
@@ -19,11 +19,13 @@
  * GNU General Public License for more details.
  *
  */
-#ifndef AW_EMAC_H
-#define AW_EMAC_H
+
+#ifndef ALLWINNER_EMAC_H
+#define ALLWINNER_EMAC_H
 
 #include "net/net.h"
 #include "qemu/fifo8.h"
+#include "hw/net/mii.h"
 
 #define TYPE_AW_EMAC "allwinner-emac"
 #define AW_EMAC(obj) OBJECT_CHECK(AwEmacState, (obj), TYPE_AW_EMAC)
@@ -118,45 +120,6 @@
 #define EMAC_RX_IO_DATA_STATUS_OK       (1 << 7)
 #define EMAC_UNDOCUMENTED_MAGIC         0x0143414d  /* header for RX frames */
 
-/* PHY registers */
-#define MII_BMCR            0
-#define MII_BMSR            1
-#define MII_PHYID1          2
-#define MII_PHYID2          3
-#define MII_ANAR            4
-#define MII_ANLPAR          5
-#define MII_ANER            6
-#define MII_NSR             16
-#define MII_LBREMR          17
-#define MII_REC             18
-#define MII_SNRDR           19
-#define MII_TEST            25
-
-/* PHY registers fields */
-#define MII_BMCR_RESET      (1 << 15)
-#define MII_BMCR_LOOPBACK   (1 << 14)
-#define MII_BMCR_SPEED      (1 << 13)
-#define MII_BMCR_AUTOEN     (1 << 12)
-#define MII_BMCR_FD         (1 << 8)
-
-#define MII_BMSR_100TX_FD   (1 << 14)
-#define MII_BMSR_100TX_HD   (1 << 13)
-#define MII_BMSR_10T_FD     (1 << 12)
-#define MII_BMSR_10T_HD     (1 << 11)
-#define MII_BMSR_MFPS       (1 << 6)
-#define MII_BMSR_AN_COMP    (1 << 5)
-#define MII_BMSR_AUTONEG    (1 << 3)
-#define MII_BMSR_LINK_ST    (1 << 2)
-
-#define MII_ANAR_TXFD       (1 << 8)
-#define MII_ANAR_TX         (1 << 7)
-#define MII_ANAR_10FD       (1 << 6)
-#define MII_ANAR_10         (1 << 5)
-#define MII_ANAR_CSMACD     (1 << 0)
-
-#define RTL8201CP_PHYID1    0x0000
-#define RTL8201CP_PHYID2    0x8201
-
 /* INT CTL and INT STA registers fields */
 #define EMAC_INT_TX_CHAN(x) (1 << (x))
 #define EMAC_INT_RX         (1 << 8)
diff --git a/include/hw/net/cadence_gem.h b/include/hw/net/cadence_gem.h
new file mode 100644
index 0000000..f2e08e3
--- /dev/null
+++ b/include/hw/net/cadence_gem.h
@@ -0,0 +1,73 @@
+/*
+ * QEMU Cadence GEM emulation
+ *
+ * Copyright (c) 2011 Xilinx, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef CADENCE_GEM_H
+
+#define TYPE_CADENCE_GEM "cadence_gem"
+#define CADENCE_GEM(obj) OBJECT_CHECK(CadenceGEMState, (obj), TYPE_CADENCE_GEM)
+
+#include "net/net.h"
+#include "hw/sysbus.h"
+
+#define CADENCE_GEM_MAXREG        (0x00000640/4) /* Last valid GEM address */
+
+typedef struct CadenceGEMState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    MemoryRegion iomem;
+    NICState *nic;
+    NICConf conf;
+    qemu_irq irq;
+
+    /* GEM registers backing store */
+    uint32_t regs[CADENCE_GEM_MAXREG];
+    /* Mask of register bits which are write only */
+    uint32_t regs_wo[CADENCE_GEM_MAXREG];
+    /* Mask of register bits which are read only */
+    uint32_t regs_ro[CADENCE_GEM_MAXREG];
+    /* Mask of register bits which are clear on read */
+    uint32_t regs_rtc[CADENCE_GEM_MAXREG];
+    /* Mask of register bits which are write 1 to clear */
+    uint32_t regs_w1c[CADENCE_GEM_MAXREG];
+
+    /* PHY registers backing store */
+    uint16_t phy_regs[32];
+
+    uint8_t phy_loop; /* Are we in phy loopback? */
+
+    /* The current DMA descriptor pointers */
+    uint32_t rx_desc_addr;
+    uint32_t tx_desc_addr;
+
+    uint8_t can_rx_state; /* Debug only */
+
+    unsigned rx_desc[2];
+
+    bool sar_active[4];
+} CadenceGEMState;
+
+#define CADENCE_GEM_H
+#endif
diff --git a/include/hw/net/imx_fec.h b/include/hw/net/imx_fec.h
new file mode 100644
index 0000000..62ad473
--- /dev/null
+++ b/include/hw/net/imx_fec.h
@@ -0,0 +1,257 @@
+/*
+ * i.MX FEC/ENET Ethernet Controller emulation.
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois. <jcd@tribudubois.net>
+ *
+ * Based on Coldfire Fast Ethernet Controller emulation.
+ *
+ * Copyright (c) 2007 CodeSourcery.
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef IMX_FEC_H
+#define IMX_FEC_H
+
+#define TYPE_IMX_FEC "imx.fec"
+#define IMX_FEC(obj) OBJECT_CHECK(IMXFECState, (obj), TYPE_IMX_FEC)
+
+#define TYPE_IMX_ENET "imx.enet"
+
+#include "hw/sysbus.h"
+#include "net/net.h"
+
+#define ENET_EIR               1
+#define ENET_EIMR              2
+#define ENET_RDAR              4
+#define ENET_TDAR              5
+#define ENET_ECR               9
+#define ENET_MMFR              16
+#define ENET_MSCR              17
+#define ENET_MIBC              25
+#define ENET_RCR               33
+#define ENET_TCR               49
+#define ENET_PALR              57
+#define ENET_PAUR              58
+#define ENET_OPD               59
+#define ENET_IAUR              70
+#define ENET_IALR              71
+#define ENET_GAUR              72
+#define ENET_GALR              73
+#define ENET_TFWR              81
+#define ENET_FRBR              83
+#define ENET_FRSR              84
+#define ENET_RDSR              96
+#define ENET_TDSR              97
+#define ENET_MRBR              98
+#define ENET_RSFL              100
+#define ENET_RSEM              101
+#define ENET_RAEM              102
+#define ENET_RAFL              103
+#define ENET_TSEM              104
+#define ENET_TAEM              105
+#define ENET_TAFL              106
+#define ENET_TIPG              107
+#define ENET_FTRL              108
+#define ENET_TACC              112
+#define ENET_RACC              113
+#define ENET_MIIGSK_CFGR       192
+#define ENET_MIIGSK_ENR        194
+#define ENET_ATCR              256
+#define ENET_ATVR              257
+#define ENET_ATOFF             258
+#define ENET_ATPER             259
+#define ENET_ATCOR             260
+#define ENET_ATINC             261
+#define ENET_ATSTMP            262
+#define ENET_TGSR              385
+#define ENET_TCSR0             386
+#define ENET_TCCR0             387
+#define ENET_TCSR1             388
+#define ENET_TCCR1             389
+#define ENET_TCSR2             390
+#define ENET_TCCR2             391
+#define ENET_TCSR3             392
+#define ENET_TCCR3             393
+#define ENET_MAX               400
+
+#define ENET_MAX_FRAME_SIZE    2032
+
+/* EIR and EIMR */
+#define ENET_INT_HB            (1 << 31)
+#define ENET_INT_BABR          (1 << 30)
+#define ENET_INT_BABT          (1 << 29)
+#define ENET_INT_GRA           (1 << 28)
+#define ENET_INT_TXF           (1 << 27)
+#define ENET_INT_TXB           (1 << 26)
+#define ENET_INT_RXF           (1 << 25)
+#define ENET_INT_RXB           (1 << 24)
+#define ENET_INT_MII           (1 << 23)
+#define ENET_INT_EBERR         (1 << 22)
+#define ENET_INT_LC            (1 << 21)
+#define ENET_INT_RL            (1 << 20)
+#define ENET_INT_UN            (1 << 19)
+#define ENET_INT_PLR           (1 << 18)
+#define ENET_INT_WAKEUP        (1 << 17)
+#define ENET_INT_TS_AVAIL      (1 << 16)
+#define ENET_INT_TS_TIMER      (1 << 15)
+
+#define ENET_INT_MAC           (ENET_INT_HB | ENET_INT_BABR | ENET_INT_BABT | \
+                                ENET_INT_GRA | ENET_INT_TXF | ENET_INT_TXB | \
+                                ENET_INT_RXF | ENET_INT_RXB | ENET_INT_MII | \
+                                ENET_INT_EBERR | ENET_INT_LC | ENET_INT_RL | \
+                                ENET_INT_UN | ENET_INT_PLR | ENET_INT_WAKEUP | \
+                                ENET_INT_TS_AVAIL)
+
+/* RDAR */
+#define ENET_RDAR_RDAR         (1 << 24)
+
+/* TDAR */
+#define ENET_TDAR_TDAR         (1 << 24)
+
+/* ECR */
+#define ENET_ECR_RESET         (1 << 0)
+#define ENET_ECR_ETHEREN       (1 << 1)
+#define ENET_ECR_MAGICEN       (1 << 2)
+#define ENET_ECR_SLEEP         (1 << 3)
+#define ENET_ECR_EN1588        (1 << 4)
+#define ENET_ECR_SPEED         (1 << 5)
+#define ENET_ECR_DBGEN         (1 << 6)
+#define ENET_ECR_STOPEN        (1 << 7)
+#define ENET_ECR_DSBWP         (1 << 8)
+
+/* MIBC */
+#define ENET_MIBC_MIB_DIS      (1 << 31)
+#define ENET_MIBC_MIB_IDLE     (1 << 30)
+#define ENET_MIBC_MIB_CLEAR    (1 << 29)
+
+/* RCR */
+#define ENET_RCR_LOOP          (1 << 0)
+#define ENET_RCR_DRT           (1 << 1)
+#define ENET_RCR_MII_MODE      (1 << 2)
+#define ENET_RCR_PROM          (1 << 3)
+#define ENET_RCR_BC_REJ        (1 << 4)
+#define ENET_RCR_FCE           (1 << 5)
+#define ENET_RCR_RGMII_EN      (1 << 6)
+#define ENET_RCR_RMII_MODE     (1 << 8)
+#define ENET_RCR_RMII_10T      (1 << 9)
+#define ENET_RCR_PADEN         (1 << 12)
+#define ENET_RCR_PAUFWD        (1 << 13)
+#define ENET_RCR_CRCFWD        (1 << 14)
+#define ENET_RCR_CFEN          (1 << 15)
+#define ENET_RCR_MAX_FL_SHIFT  (16)
+#define ENET_RCR_MAX_FL_LENGTH (14)
+#define ENET_RCR_NLC           (1 << 30)
+#define ENET_RCR_GRS           (1 << 31)
+
+/* TCR */
+#define ENET_TCR_GTS           (1 << 0)
+#define ENET_TCR_FDEN          (1 << 2)
+#define ENET_TCR_TFC_PAUSE     (1 << 3)
+#define ENET_TCR_RFC_PAUSE     (1 << 4)
+#define ENET_TCR_ADDSEL_SHIFT  (5)
+#define ENET_TCR_ADDSEL_LENGTH (3)
+#define ENET_TCR_CRCFWD        (1 << 9)
+
+/* RDSR */
+#define ENET_TWFR_TFWR_SHIFT   (0)
+#define ENET_TWFR_TFWR_LENGTH  (6)
+#define ENET_TWFR_STRFWD       (1 << 8)
+
+/* Buffer Descriptor.  */
+typedef struct {
+    uint16_t length;
+    uint16_t flags;
+    uint32_t data;
+} IMXFECBufDesc;
+
+#define ENET_BD_R              (1 << 15)
+#define ENET_BD_E              (1 << 15)
+#define ENET_BD_O1             (1 << 14)
+#define ENET_BD_W              (1 << 13)
+#define ENET_BD_O2             (1 << 12)
+#define ENET_BD_L              (1 << 11)
+#define ENET_BD_TC             (1 << 10)
+#define ENET_BD_ABC            (1 << 9)
+#define ENET_BD_M              (1 << 8)
+#define ENET_BD_BC             (1 << 7)
+#define ENET_BD_MC             (1 << 6)
+#define ENET_BD_LG             (1 << 5)
+#define ENET_BD_NO             (1 << 4)
+#define ENET_BD_CR             (1 << 2)
+#define ENET_BD_OV             (1 << 1)
+#define ENET_BD_TR             (1 << 0)
+
+typedef struct {
+    uint16_t length;
+    uint16_t flags;
+    uint32_t data;
+    uint16_t status;
+    uint16_t option;
+    uint16_t checksum;
+    uint16_t head_proto;
+    uint32_t last_buffer;
+    uint32_t timestamp;
+    uint32_t reserved[2];
+} IMXENETBufDesc;
+
+#define ENET_BD_ME             (1 << 15)
+#define ENET_BD_TX_INT         (1 << 14)
+#define ENET_BD_TS             (1 << 13)
+#define ENET_BD_PINS           (1 << 12)
+#define ENET_BD_IINS           (1 << 11)
+#define ENET_BD_PE             (1 << 10)
+#define ENET_BD_CE             (1 << 9)
+#define ENET_BD_UC             (1 << 8)
+#define ENET_BD_RX_INT         (1 << 7)
+
+#define ENET_BD_TXE            (1 << 15)
+#define ENET_BD_UE             (1 << 13)
+#define ENET_BD_EE             (1 << 12)
+#define ENET_BD_FE             (1 << 11)
+#define ENET_BD_LCE            (1 << 10)
+#define ENET_BD_OE             (1 << 9)
+#define ENET_BD_TSE            (1 << 8)
+#define ENET_BD_ICE            (1 << 5)
+#define ENET_BD_PCR            (1 << 4)
+#define ENET_BD_VLAN           (1 << 2)
+#define ENET_BD_IPV6           (1 << 1)
+#define ENET_BD_FRAG           (1 << 0)
+
+#define ENET_BD_BDU            (1 << 31)
+
+typedef struct IMXFECState {
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    NICState *nic;
+    NICConf conf;
+    qemu_irq irq[2];
+    MemoryRegion iomem;
+
+    uint32_t regs[ENET_MAX];
+    uint32_t rx_descriptor;
+    uint32_t tx_descriptor;
+
+    uint32_t phy_status;
+    uint32_t phy_control;
+    uint32_t phy_advertise;
+    uint32_t phy_int;
+    uint32_t phy_int_mask;
+
+    bool is_fec;
+} IMXFECState;
+
+#endif
diff --git a/include/hw/net/mii.h b/include/hw/net/mii.h
new file mode 100644
index 0000000..9fdd7bb
--- /dev/null
+++ b/include/hw/net/mii.h
@@ -0,0 +1,76 @@
+/*
+ * Common network MII address and register definitions.
+ *
+ * Copyright (C) 2014 Beniamino Galvani <b.galvani@gmail.com>
+ *
+ * Allwinner EMAC register definitions from Linux kernel are:
+ *   Copyright 2012 Stefan Roese <sr@denx.de>
+ *   Copyright 2013 Maxime Ripard <maxime.ripard@free-electrons.com>
+ *   Copyright 1997 Sten Wang
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ */
+#ifndef MII_H
+#define MII_H
+
+/* PHY registers */
+#define MII_BMCR            0
+#define MII_BMSR            1
+#define MII_PHYID1          2
+#define MII_PHYID2          3
+#define MII_ANAR            4
+#define MII_ANLPAR          5
+#define MII_ANER            6
+#define MII_NSR             16
+#define MII_LBREMR          17
+#define MII_REC             18
+#define MII_SNRDR           19
+#define MII_TEST            25
+
+/* PHY registers fields */
+#define MII_BMCR_RESET      (1 << 15)
+#define MII_BMCR_LOOPBACK   (1 << 14)
+#define MII_BMCR_SPEED      (1 << 13)
+#define MII_BMCR_AUTOEN     (1 << 12)
+#define MII_BMCR_FD         (1 << 8)
+
+#define MII_BMSR_100TX_FD   (1 << 14)
+#define MII_BMSR_100TX_HD   (1 << 13)
+#define MII_BMSR_10T_FD     (1 << 12)
+#define MII_BMSR_10T_HD     (1 << 11)
+#define MII_BMSR_MFPS       (1 << 6)
+#define MII_BMSR_AN_COMP    (1 << 5)
+#define MII_BMSR_AUTONEG    (1 << 3)
+#define MII_BMSR_LINK_ST    (1 << 2)
+
+#define MII_ANAR_TXFD       (1 << 8)
+#define MII_ANAR_TX         (1 << 7)
+#define MII_ANAR_10FD       (1 << 6)
+#define MII_ANAR_10         (1 << 5)
+#define MII_ANAR_CSMACD     (1 << 0)
+
+#define MII_ANLPAR_ACK      (1 << 14)
+#define MII_ANLPAR_TXFD     (1 << 8)
+#define MII_ANLPAR_TX       (1 << 7)
+#define MII_ANLPAR_10FD     (1 << 6)
+#define MII_ANLPAR_10       (1 << 5)
+#define MII_ANLPAR_CSMACD   (1 << 0)
+
+/* List of vendor identifiers */
+/* RealTek 8201 */
+#define RTL8201CP_PHYID1    0x0000
+#define RTL8201CP_PHYID2    0x8201
+
+/* National Semiconductor DP83848 */
+#define DP83848_PHYID1      0x2000
+#define DP83848_PHYID2      0x5c90
+
+#endif /* MII_H */
diff --git a/include/hw/nmi.h b/include/hw/nmi.h
index b541772..d092c68 100644
--- a/include/hw/nmi.h
+++ b/include/hw/nmi.h
@@ -20,7 +20,7 @@
  */
 
 #ifndef NMI_H
-#define NMI_H 1
+#define NMI_H
 
 #include "qemu-common.h"
 #include "qom/object.h"
diff --git a/include/hw/nvram/fw_cfg.h b/include/hw/nvram/fw_cfg.h
index 56e1ed7..5c27a1f 100644
--- a/include/hw/nvram/fw_cfg.h
+++ b/include/hw/nvram/fw_cfg.h
@@ -1,54 +1,9 @@
 #ifndef FW_CFG_H
 #define FW_CFG_H
 
-#ifndef NO_QEMU_PROTOS
-#include <stdint.h>
-#include <stddef.h>
-
 #include "exec/hwaddr.h"
-#include "qemu/typedefs.h"
-#endif
+#include "hw/nvram/fw_cfg_keys.h"
 
-#define FW_CFG_SIGNATURE        0x00
-#define FW_CFG_ID               0x01
-#define FW_CFG_UUID             0x02
-#define FW_CFG_RAM_SIZE         0x03
-#define FW_CFG_NOGRAPHIC        0x04
-#define FW_CFG_NB_CPUS          0x05
-#define FW_CFG_MACHINE_ID       0x06
-#define FW_CFG_KERNEL_ADDR      0x07
-#define FW_CFG_KERNEL_SIZE      0x08
-#define FW_CFG_KERNEL_CMDLINE   0x09
-#define FW_CFG_INITRD_ADDR      0x0a
-#define FW_CFG_INITRD_SIZE      0x0b
-#define FW_CFG_BOOT_DEVICE      0x0c
-#define FW_CFG_NUMA             0x0d
-#define FW_CFG_BOOT_MENU        0x0e
-#define FW_CFG_MAX_CPUS         0x0f
-#define FW_CFG_KERNEL_ENTRY     0x10
-#define FW_CFG_KERNEL_DATA      0x11
-#define FW_CFG_INITRD_DATA      0x12
-#define FW_CFG_CMDLINE_ADDR     0x13
-#define FW_CFG_CMDLINE_SIZE     0x14
-#define FW_CFG_CMDLINE_DATA     0x15
-#define FW_CFG_SETUP_ADDR       0x16
-#define FW_CFG_SETUP_SIZE       0x17
-#define FW_CFG_SETUP_DATA       0x18
-#define FW_CFG_FILE_DIR         0x19
-
-#define FW_CFG_FILE_FIRST       0x20
-#define FW_CFG_FILE_SLOTS       0x10
-#define FW_CFG_MAX_ENTRY        (FW_CFG_FILE_FIRST+FW_CFG_FILE_SLOTS)
-
-#define FW_CFG_WRITE_CHANNEL    0x4000
-#define FW_CFG_ARCH_LOCAL       0x8000
-#define FW_CFG_ENTRY_MASK       ~(FW_CFG_WRITE_CHANNEL | FW_CFG_ARCH_LOCAL)
-
-#define FW_CFG_INVALID          0xffff
-
-#define FW_CFG_MAX_FILE_PATH    56
-
-#ifndef NO_QEMU_PROTOS
 typedef struct FWCfgFile {
     uint32_t  size;        /* file size */
     uint16_t  select;      /* write this to 0x510 to read it */
@@ -56,33 +11,177 @@
     char      name[FW_CFG_MAX_FILE_PATH];
 } FWCfgFile;
 
+#define FW_CFG_ORDER_OVERRIDE_VGA    70
+#define FW_CFG_ORDER_OVERRIDE_NIC    80
+#define FW_CFG_ORDER_OVERRIDE_USER   100
+#define FW_CFG_ORDER_OVERRIDE_DEVICE 110
+
+void fw_cfg_set_order_override(FWCfgState *fw_cfg, int order);
+void fw_cfg_reset_order_override(FWCfgState *fw_cfg);
+
 typedef struct FWCfgFiles {
     uint32_t  count;
     FWCfgFile f[];
 } FWCfgFiles;
 
-typedef void (*FWCfgCallback)(void *opaque, uint8_t *data);
-typedef void (*FWCfgReadCallback)(void *opaque, uint32_t offset);
+/* Control as first field allows for different structures selected by this
+ * field, which might be useful in the future
+ */
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} QEMU_PACKED FWCfgDmaAccess;
 
+typedef void (*FWCfgReadCallback)(void *opaque);
+
+/**
+ * fw_cfg_add_bytes:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Add a new fw_cfg item, available by selecting the given key, as a raw
+ * "blob" of the given size. The data referenced by the starting pointer
+ * is only linked, NOT copied, into the data structure of the fw_cfg device.
+ */
 void fw_cfg_add_bytes(FWCfgState *s, uint16_t key, void *data, size_t len);
+
+/**
+ * fw_cfg_add_string:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: NUL-terminated ascii string
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the provided string,
+ * including its NUL terminator.
+ */
 void fw_cfg_add_string(FWCfgState *s, uint16_t key, const char *value);
+
+/**
+ * fw_cfg_add_i16:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 16-bit integer
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the given 16-bit
+ * value, converted to little-endian representation.
+ */
 void fw_cfg_add_i16(FWCfgState *s, uint16_t key, uint16_t value);
+
+/**
+ * fw_cfg_modify_i16:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 16-bit integer
+ *
+ * Replace the fw_cfg item available by selecting the given key. The new
+ * data will consist of a dynamically allocated copy of the given 16-bit
+ * value, converted to little-endian representation. The data being replaced,
+ * assumed to have been dynamically allocated during an earlier call to
+ * either fw_cfg_add_i16() or fw_cfg_modify_i16(), is freed before returning.
+ */
+void fw_cfg_modify_i16(FWCfgState *s, uint16_t key, uint16_t value);
+
+/**
+ * fw_cfg_add_i32:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 32-bit integer
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the given 32-bit
+ * value, converted to little-endian representation.
+ */
 void fw_cfg_add_i32(FWCfgState *s, uint16_t key, uint32_t value);
+
+/**
+ * fw_cfg_add_i64:
+ * @s: fw_cfg device being modified
+ * @key: selector key value for new fw_cfg item
+ * @value: 64-bit integer
+ *
+ * Add a new fw_cfg item, available by selecting the given key. The item
+ * data will consist of a dynamically allocated copy of the given 64-bit
+ * value, converted to little-endian representation.
+ */
 void fw_cfg_add_i64(FWCfgState *s, uint16_t key, uint64_t value);
-void fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
-                         void *callback_opaque, void *data, size_t len);
+
+/**
+ * fw_cfg_add_file:
+ * @s: fw_cfg device being modified
+ * @filename: name of new fw_cfg file item
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Add a new NAMED fw_cfg item as a raw "blob" of the given size. The data
+ * referenced by the starting pointer is only linked, NOT copied, into the
+ * data structure of the fw_cfg device.
+ * The next available (unused) selector key starting at FW_CFG_FILE_FIRST
+ * will be used; also, a new entry will be added to the file directory
+ * structure residing at key value FW_CFG_FILE_DIR, containing the item name,
+ * data size, and assigned selector key value.
+ */
 void fw_cfg_add_file(FWCfgState *s, const char *filename, void *data,
                      size_t len);
+
+/**
+ * fw_cfg_add_file_callback:
+ * @s: fw_cfg device being modified
+ * @filename: name of new fw_cfg file item
+ * @callback: callback function
+ * @callback_opaque: argument to be passed into callback function
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Add a new NAMED fw_cfg item as a raw "blob" of the given size. The data
+ * referenced by the starting pointer is only linked, NOT copied, into the
+ * data structure of the fw_cfg device.
+ * The next available (unused) selector key starting at FW_CFG_FILE_FIRST
+ * will be used; also, a new entry will be added to the file directory
+ * structure residing at key value FW_CFG_FILE_DIR, containing the item name,
+ * data size, and assigned selector key value.
+ * Additionally, set a callback function (and argument) to be called each
+ * time this item is selected (by having its selector key either written to
+ * the fw_cfg control register, or passed to QEMU in FWCfgDmaAccess.control
+ * with FW_CFG_DMA_CTL_SELECT).
+ */
 void fw_cfg_add_file_callback(FWCfgState *s, const char *filename,
                               FWCfgReadCallback callback, void *callback_opaque,
                               void *data, size_t len);
+
+/**
+ * fw_cfg_modify_file:
+ * @s: fw_cfg device being modified
+ * @filename: name of new fw_cfg file item
+ * @data: pointer to start of item data
+ * @len: size of item data
+ *
+ * Replace a NAMED fw_cfg item. If an existing item is found, its callback
+ * information will be cleared, and a pointer to its data will be returned
+ * to the caller, so that it may be freed if necessary. If an existing item
+ * is not found, this call defaults to fw_cfg_add_file(), and NULL is
+ * returned to the caller.
+ * In either case, the new item data is only linked, NOT copied, into the
+ * data structure of the fw_cfg device.
+ *
+ * Returns: pointer to old item's data, or NULL if old item does not exist.
+ */
 void *fw_cfg_modify_file(FWCfgState *s, const char *filename, void *data,
                          size_t len);
-FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
-                        hwaddr crl_addr, hwaddr data_addr);
+
+FWCfgState *fw_cfg_init_io_dma(uint32_t iobase, uint32_t dma_iobase,
+                                AddressSpace *dma_as);
+FWCfgState *fw_cfg_init_io(uint32_t iobase);
+FWCfgState *fw_cfg_init_mem(hwaddr ctl_addr, hwaddr data_addr);
+FWCfgState *fw_cfg_init_mem_wide(hwaddr ctl_addr,
+                                 hwaddr data_addr, uint32_t data_width,
+                                 hwaddr dma_addr, AddressSpace *dma_as);
 
 FWCfgState *fw_cfg_find(void);
-
-#endif /* NO_QEMU_PROTOS */
+bool fw_cfg_dma_enabled(void *opaque);
 
 #endif
diff --git a/include/hw/nvram/fw_cfg_keys.h b/include/hw/nvram/fw_cfg_keys.h
new file mode 100644
index 0000000..0f3e871
--- /dev/null
+++ b/include/hw/nvram/fw_cfg_keys.h
@@ -0,0 +1,46 @@
+#ifndef FW_CFG_KEYS_H
+#define FW_CFG_KEYS_H
+
+#define FW_CFG_SIGNATURE        0x00
+#define FW_CFG_ID               0x01
+#define FW_CFG_UUID             0x02
+#define FW_CFG_RAM_SIZE         0x03
+#define FW_CFG_NOGRAPHIC        0x04
+#define FW_CFG_NB_CPUS          0x05
+#define FW_CFG_MACHINE_ID       0x06
+#define FW_CFG_KERNEL_ADDR      0x07
+#define FW_CFG_KERNEL_SIZE      0x08
+#define FW_CFG_KERNEL_CMDLINE   0x09
+#define FW_CFG_INITRD_ADDR      0x0a
+#define FW_CFG_INITRD_SIZE      0x0b
+#define FW_CFG_BOOT_DEVICE      0x0c
+#define FW_CFG_NUMA             0x0d
+#define FW_CFG_BOOT_MENU        0x0e
+#define FW_CFG_MAX_CPUS         0x0f
+#define FW_CFG_KERNEL_ENTRY     0x10
+#define FW_CFG_KERNEL_DATA      0x11
+#define FW_CFG_INITRD_DATA      0x12
+#define FW_CFG_CMDLINE_ADDR     0x13
+#define FW_CFG_CMDLINE_SIZE     0x14
+#define FW_CFG_CMDLINE_DATA     0x15
+#define FW_CFG_SETUP_ADDR       0x16
+#define FW_CFG_SETUP_SIZE       0x17
+#define FW_CFG_SETUP_DATA       0x18
+#define FW_CFG_FILE_DIR         0x19
+
+#define FW_CFG_FILE_FIRST       0x20
+#define FW_CFG_FILE_SLOTS       0x10
+#define FW_CFG_MAX_ENTRY        (FW_CFG_FILE_FIRST + FW_CFG_FILE_SLOTS)
+
+#define FW_CFG_WRITE_CHANNEL    0x4000
+#define FW_CFG_ARCH_LOCAL       0x8000
+#define FW_CFG_ENTRY_MASK       (~(FW_CFG_WRITE_CHANNEL | FW_CFG_ARCH_LOCAL))
+
+#define FW_CFG_INVALID          0xffff
+
+/* width in bytes of fw_cfg control register */
+#define FW_CFG_CTL_SIZE         0x02
+
+#define FW_CFG_MAX_FILE_PATH    56
+
+#endif
diff --git a/include/hw/nvram/openbios_firmware_abi.h b/include/hw/nvram/openbios_firmware_abi.h
index c66ee22..74cfd56 100644
--- a/include/hw/nvram/openbios_firmware_abi.h
+++ b/include/hw/nvram/openbios_firmware_abi.h
@@ -1,5 +1,5 @@
-#ifndef FIRMWARE_ABI_H
-#define FIRMWARE_ABI_H
+#ifndef OPENBIOS_FIRMWARE_ABI_H
+#define OPENBIOS_FIRMWARE_ABI_H
 
 /* OpenBIOS NVRAM partition */
 struct OpenBIOS_nvpart_v1 {
@@ -72,4 +72,4 @@
 
     header->checksum = tmp;
 }
-#endif /* FIRMWARE_ABI_H */
+#endif /* OPENBIOS_FIRMWARE_ABI_H */
diff --git a/include/hw/pci-host/apb.h b/include/hw/pci-host/apb.h
index 736db61..b19bd55 100644
--- a/include/hw/pci-host/apb.h
+++ b/include/hw/pci-host/apb.h
@@ -1,5 +1,5 @@
-#ifndef APB_PCI_H
-#define APB_PCI_H
+#ifndef PCI_HOST_APB_H
+#define PCI_HOST_APB_H
 
 #include "qemu-common.h"
 
diff --git a/include/hw/pci-host/gpex.h b/include/hw/pci-host/gpex.h
new file mode 100644
index 0000000..68c9348
--- /dev/null
+++ b/include/hw/pci-host/gpex.h
@@ -0,0 +1,56 @@
+/*
+ * QEMU Generic PCI Express Bridge Emulation
+ *
+ * Copyright (C) 2015 Alexander Graf <agraf@suse.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef HW_GPEX_H
+#define HW_GPEX_H
+
+#include "hw/hw.h"
+#include "hw/sysbus.h"
+#include "hw/pci/pci.h"
+#include "hw/pci/pcie_host.h"
+
+#define TYPE_GPEX_HOST "gpex-pcihost"
+#define GPEX_HOST(obj) \
+     OBJECT_CHECK(GPEXHost, (obj), TYPE_GPEX_HOST)
+
+#define TYPE_GPEX_ROOT_DEVICE "gpex-root"
+#define MCH_PCI_DEVICE(obj) \
+     OBJECT_CHECK(GPEXRootState, (obj), TYPE_GPEX_ROOT_DEVICE)
+
+#define GPEX_NUM_IRQS 4
+
+typedef struct GPEXRootState {
+    /*< private >*/
+    PCIDevice parent_obj;
+    /*< public >*/
+} GPEXRootState;
+
+typedef struct GPEXHost {
+    /*< private >*/
+    PCIExpressHost parent_obj;
+    /*< public >*/
+
+    GPEXRootState gpex_root;
+
+    MemoryRegion io_ioport;
+    MemoryRegion io_mmio;
+    qemu_irq irq[GPEX_NUM_IRQS];
+} GPEXHost;
+
+#endif /* HW_GPEX_H */
diff --git a/include/hw/pci-host/pam.h b/include/hw/pci-host/pam.h
index 4d03e4b..6116c63 100644
--- a/include/hw/pci-host/pam.h
+++ b/include/hw/pci-host/pam.h
@@ -86,10 +86,6 @@
     unsigned current;
 } PAMMemoryRegion;
 
-void smram_update(MemoryRegion *smram_region, uint8_t smram,
-                  uint8_t smm_enabled);
-void smram_set_smm(uint8_t *host_smm_enabled, int smm, uint8_t smram,
-                   MemoryRegion *smram_region);
 void init_pam(DeviceState *dev, MemoryRegion *ram, MemoryRegion *system,
               MemoryRegion *pci, PAMMemoryRegion *mem, uint32_t start, uint32_t size);
 void pam_update(PAMMemoryRegion *mem, int idx, uint8_t val);
diff --git a/include/hw/pci-host/ppce500.h b/include/hw/pci-host/ppce500.h
index 61f773e..e3a3742 100644
--- a/include/hw/pci-host/ppce500.h
+++ b/include/hw/pci-host/ppce500.h
@@ -1,5 +1,5 @@
-#ifndef PPCE500_PCI_H
-#define PPCE500_PCI_H
+#ifndef PCI_HOST_PPCE500_H
+#define PCI_HOST_PPCE500_H
 
 static inline int ppce500_pci_map_irq_slot(int devno, int irq_num)
 {
diff --git a/include/hw/pci-host/q35.h b/include/hw/pci-host/q35.h
index 025d6e6..94486fd 100644
--- a/include/hw/pci-host/q35.h
+++ b/include/hw/pci-host/q35.h
@@ -23,7 +23,6 @@
 #define HW_Q35_H
 
 #include "hw/hw.h"
-#include "qemu/range.h"
 #include "hw/isa/isa.h"
 #include "hw/sysbus.h"
 #include "hw/i386/pc.h"
@@ -53,15 +52,14 @@
     MemoryRegion *system_memory;
     MemoryRegion *address_space_io;
     PAMMemoryRegion pam_regions[13];
-    MemoryRegion smram_region;
-    PcPciInfo pci_info;
-    uint8_t smm_enabled;
-    ram_addr_t below_4g_mem_size;
-    ram_addr_t above_4g_mem_size;
+    MemoryRegion smram_region, open_high_smram;
+    MemoryRegion smram, low_smram, high_smram;
+    MemoryRegion tseg_blackhole, tseg_window;
+    Range pci_hole;
+    uint64_t below_4g_mem_size;
+    uint64_t above_4g_mem_size;
     uint64_t pci_hole64_size;
-    PcGuestInfo *guest_info;
     uint32_t short_root_bus;
-    IntelIOMMUState *iommu;
 } MCHPCIState;
 
 typedef struct Q35PCIHost {
@@ -79,6 +77,11 @@
  * gmch part
  */
 
+#define MCH_HOST_PROP_RAM_MEM "ram-mem"
+#define MCH_HOST_PROP_PCI_MEM "pci-mem"
+#define MCH_HOST_PROP_SYSTEM_MEM "system-mem"
+#define MCH_HOST_PROP_IO_MEM "io-mem"
+
 /* PCI configuration */
 #define MCH_HOST_BRIDGE                        "MCH"
 
@@ -128,8 +131,7 @@
 #define MCH_HOST_BRIDGE_PAM_MASK               ((uint8_t)0x3)
 
 #define MCH_HOST_BRIDGE_SMRAM                  0x9d
-#define MCH_HOST_BRIDGE_SMRAM_SIZE             1
-#define MCH_HOST_BRIDGE_SMRAM_DEFAULT          ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_SMRAM_SIZE             2
 #define MCH_HOST_BRIDGE_SMRAM_D_OPEN           ((uint8_t)(1 << 6))
 #define MCH_HOST_BRIDGE_SMRAM_D_CLS            ((uint8_t)(1 << 5))
 #define MCH_HOST_BRIDGE_SMRAM_D_LCK            ((uint8_t)(1 << 4))
@@ -140,18 +142,36 @@
 #define MCH_HOST_BRIDGE_SMRAM_C_END            0xc0000
 #define MCH_HOST_BRIDGE_SMRAM_C_SIZE           0x20000
 #define MCH_HOST_BRIDGE_UPPER_SYSTEM_BIOS_END  0x100000
+#define MCH_HOST_BRIDGE_SMRAM_DEFAULT           \
+    MCH_HOST_BRIDGE_SMRAM_C_BASE_SEG
+#define MCH_HOST_BRIDGE_SMRAM_WMASK             \
+    (MCH_HOST_BRIDGE_SMRAM_D_OPEN |             \
+     MCH_HOST_BRIDGE_SMRAM_D_CLS |              \
+     MCH_HOST_BRIDGE_SMRAM_D_LCK |              \
+     MCH_HOST_BRIDGE_SMRAM_G_SMRAME)
+#define MCH_HOST_BRIDGE_SMRAM_WMASK_LCK         \
+    MCH_HOST_BRIDGE_SMRAM_D_CLS
 
 #define MCH_HOST_BRIDGE_ESMRAMC                0x9e
-#define MCH_HOST_BRIDGE_ESMRAMC_H_SMRAME       ((uint8_t)(1 << 6))
-#define MCH_HOST_BRIDGE_ESMRAMC_E_SMERR        ((uint8_t)(1 << 5))
-#define MCH_HOST_BRIDGE_ESMRAMC_SM_CACHE       ((uint8_t)(1 << 4))
-#define MCH_HOST_BRIDGE_ESMRAMC_SM_L1          ((uint8_t)(1 << 3))
-#define MCH_HOST_BRIDGE_ESMRAMC_SM_L2          ((uint8_t)(1 << 2))
+#define MCH_HOST_BRIDGE_ESMRAMC_H_SMRAME       ((uint8_t)(1 << 7))
+#define MCH_HOST_BRIDGE_ESMRAMC_E_SMERR        ((uint8_t)(1 << 6))
+#define MCH_HOST_BRIDGE_ESMRAMC_SM_CACHE       ((uint8_t)(1 << 5))
+#define MCH_HOST_BRIDGE_ESMRAMC_SM_L1          ((uint8_t)(1 << 4))
+#define MCH_HOST_BRIDGE_ESMRAMC_SM_L2          ((uint8_t)(1 << 3))
 #define MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_MASK   ((uint8_t)(0x3 << 1))
 #define MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_1MB    ((uint8_t)(0x0 << 1))
 #define MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_2MB    ((uint8_t)(0x1 << 1))
 #define MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_8MB    ((uint8_t)(0x2 << 1))
 #define MCH_HOST_BRIDGE_ESMRAMC_T_EN           ((uint8_t)1)
+#define MCH_HOST_BRIDGE_ESMRAMC_DEFAULT \
+    (MCH_HOST_BRIDGE_ESMRAMC_SM_CACHE | \
+     MCH_HOST_BRIDGE_ESMRAMC_SM_L1 |    \
+     MCH_HOST_BRIDGE_ESMRAMC_SM_L2)
+#define MCH_HOST_BRIDGE_ESMRAMC_WMASK               \
+    (MCH_HOST_BRIDGE_ESMRAMC_H_SMRAME |             \
+     MCH_HOST_BRIDGE_ESMRAMC_TSEG_SZ_MASK |         \
+     MCH_HOST_BRIDGE_ESMRAMC_T_EN)
+#define MCH_HOST_BRIDGE_ESMRAMC_WMASK_LCK     0
 
 /* D1:F0 PCIE* port*/
 #define MCH_PCIE_DEV                           1
@@ -159,4 +179,12 @@
 
 uint64_t mch_mcfg_base(void);
 
+/*
+ * Arbitary but unique BNF number for IOAPIC device.
+ *
+ * TODO: make sure there would have no conflict with real PCI bus
+ */
+#define Q35_PSEUDO_BUS_PLATFORM         (0xff)
+#define Q35_PSEUDO_DEVFN_IOAPIC         (0x00)
+
 #endif /* HW_Q35_H */
diff --git a/include/hw/pci-host/spapr.h b/include/hw/pci-host/spapr.h
index 4ea2a0d..5adc603 100644
--- a/include/hw/pci-host/spapr.h
+++ b/include/hw/pci-host/spapr.h
@@ -16,40 +16,23 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#if !defined(__HW_SPAPR_H__)
-#error Please include spapr.h before this file!
-#endif
 
-#if !defined(__HW_SPAPR_PCI_H__)
-#define __HW_SPAPR_PCI_H__
+#ifndef PCI_HOST_SPAPR_H
+#define PCI_HOST_SPAPR_H
 
+#include "hw/ppc/spapr.h"
 #include "hw/pci/pci.h"
 #include "hw/pci/pci_host.h"
 #include "hw/ppc/xics.h"
 
 #define TYPE_SPAPR_PCI_HOST_BRIDGE "spapr-pci-host-bridge"
-#define TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE "spapr-pci-vfio-host-bridge"
 
 #define SPAPR_PCI_HOST_BRIDGE(obj) \
     OBJECT_CHECK(sPAPRPHBState, (obj), TYPE_SPAPR_PCI_HOST_BRIDGE)
 
-#define SPAPR_PCI_VFIO_HOST_BRIDGE(obj) \
-    OBJECT_CHECK(sPAPRPHBVFIOState, (obj), TYPE_SPAPR_PCI_VFIO_HOST_BRIDGE)
+#define SPAPR_PCI_DMA_MAX_WINDOWS    2
 
-#define SPAPR_PCI_HOST_BRIDGE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(sPAPRPHBClass, (klass), TYPE_SPAPR_PCI_HOST_BRIDGE)
-#define SPAPR_PCI_HOST_BRIDGE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(sPAPRPHBClass, (obj), TYPE_SPAPR_PCI_HOST_BRIDGE)
-
-typedef struct sPAPRPHBClass sPAPRPHBClass;
 typedef struct sPAPRPHBState sPAPRPHBState;
-typedef struct sPAPRPHBVFIOState sPAPRPHBVFIOState;
-
-struct sPAPRPHBClass {
-    PCIHostBridgeClass parent_class;
-
-    void (*finish_realize)(sPAPRPHBState *sphb, Error **errp);
-};
 
 typedef struct spapr_pci_msi {
     uint32_t first_irq;
@@ -64,15 +47,17 @@
 struct sPAPRPHBState {
     PCIHostState parent_obj;
 
-    int32_t index;
+    uint32_t index;
     uint64_t buid;
     char *dtbusname;
+    bool dr_enabled;
 
     MemoryRegion memspace, iospace;
     hwaddr mem_win_addr, mem_win_size, io_win_addr, io_win_size;
     MemoryRegion memwindow, iowindow, msiwindow;
 
-    uint32_t dma_liobn;
+    uint32_t dma_liobn[SPAPR_PCI_DMA_MAX_WINDOWS];
+    hwaddr dma_win_addr, dma_win_size;
     AddressSpace iommu_as;
     MemoryRegion iommu_root;
 
@@ -86,40 +71,86 @@
     spapr_pci_msi_mig *msi_devs;
 
     QLIST_ENTRY(sPAPRPHBState) list;
+
+    bool ddw_enabled;
+    uint64_t page_size_mask;
+    uint64_t dma64_win_addr;
 };
 
-struct sPAPRPHBVFIOState {
-    sPAPRPHBState phb;
-
-    int32_t iommugroupid;
-};
+#define SPAPR_PCI_MAX_INDEX          255
 
 #define SPAPR_PCI_BASE_BUID          0x800000020000000ULL
 
+#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
+
 #define SPAPR_PCI_WINDOW_BASE        0x10000000000ULL
 #define SPAPR_PCI_WINDOW_SPACING     0x1000000000ULL
 #define SPAPR_PCI_MMIO_WIN_OFF       0xA0000000
-#define SPAPR_PCI_MMIO_WIN_SIZE      0x20000000
+#define SPAPR_PCI_MMIO_WIN_SIZE      (SPAPR_PCI_WINDOW_SPACING - \
+                                     SPAPR_PCI_MEM_WIN_BUS_OFFSET)
 #define SPAPR_PCI_IO_WIN_OFF         0x80000000
 #define SPAPR_PCI_IO_WIN_SIZE        0x10000
 
 #define SPAPR_PCI_MSI_WINDOW         0x40000000000ULL
 
-#define SPAPR_PCI_MEM_WIN_BUS_OFFSET 0x80000000ULL
-
 static inline qemu_irq spapr_phb_lsi_qirq(struct sPAPRPHBState *phb, int pin)
 {
-    return xics_get_qirq(spapr->icp, phb->lsi_table[pin].irq);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+
+    return xics_get_qirq(spapr->xics, phb->lsi_table[pin].irq);
 }
 
-PCIHostState *spapr_create_phb(sPAPREnvironment *spapr, int index);
+PCIHostState *spapr_create_phb(sPAPRMachineState *spapr, int index);
 
 int spapr_populate_pci_dt(sPAPRPHBState *phb,
                           uint32_t xics_phandle,
                           void *fdt);
 
-void spapr_pci_msi_init(sPAPREnvironment *spapr, hwaddr addr);
+void spapr_pci_msi_init(sPAPRMachineState *spapr, hwaddr addr);
 
 void spapr_pci_rtas_init(void);
 
-#endif /* __HW_SPAPR_PCI_H__ */
+sPAPRPHBState *spapr_pci_find_phb(sPAPRMachineState *spapr, uint64_t buid);
+PCIDevice *spapr_pci_find_dev(sPAPRMachineState *spapr, uint64_t buid,
+                              uint32_t config_addr);
+
+/* VFIO EEH hooks */
+#ifdef CONFIG_LINUX
+bool spapr_phb_eeh_available(sPAPRPHBState *sphb);
+int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
+                                  unsigned int addr, int option);
+int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb, int *state);
+int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option);
+int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb);
+void spapr_phb_vfio_reset(DeviceState *qdev);
+#else
+static inline bool spapr_phb_eeh_available(sPAPRPHBState *sphb)
+{
+    return false;
+}
+static inline int spapr_phb_vfio_eeh_set_option(sPAPRPHBState *sphb,
+                                                unsigned int addr, int option)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline int spapr_phb_vfio_eeh_get_state(sPAPRPHBState *sphb,
+                                               int *state)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline int spapr_phb_vfio_eeh_reset(sPAPRPHBState *sphb, int option)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline int spapr_phb_vfio_eeh_configure(sPAPRPHBState *sphb)
+{
+    return RTAS_OUT_HW_ERROR;
+}
+static inline void spapr_phb_vfio_reset(DeviceState *qdev)
+{
+}
+#endif
+
+void spapr_phb_dma_reset(sPAPRPHBState *sphb);
+
+#endif /* PCI_HOST_SPAPR_H */
diff --git a/include/hw/pci/msi.h b/include/hw/pci/msi.h
index 81a3848..4837bcf 100644
--- a/include/hw/pci/msi.h
+++ b/include/hw/pci/msi.h
@@ -29,16 +29,18 @@
     uint32_t data;
 };
 
-extern bool msi_supported;
+extern bool msi_nonbroken;
 
 void msi_set_message(PCIDevice *dev, MSIMessage msg);
 MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector);
 bool msi_enabled(const PCIDevice *dev);
 int msi_init(struct PCIDevice *dev, uint8_t offset,
-             unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask);
+             unsigned int nr_vectors, bool msi64bit,
+             bool msi_per_vector_mask, Error **errp);
 void msi_uninit(struct PCIDevice *dev);
 void msi_reset(PCIDevice *dev);
 void msi_notify(PCIDevice *dev, unsigned int vector);
+void msi_send_message(PCIDevice *dev, MSIMessage msg);
 void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len);
 unsigned int msi_nr_vectors_allocated(const PCIDevice *dev);
 
diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
index 954d82b..048a29d 100644
--- a/include/hw/pci/msix.h
+++ b/include/hw/pci/msix.h
@@ -29,6 +29,7 @@
 
 bool msix_is_masked(PCIDevice *dev, unsigned vector);
 void msix_set_pending(PCIDevice *dev, unsigned vector);
+void msix_clr_pending(PCIDevice *dev, int vector);
 
 int msix_vector_use(PCIDevice *dev, unsigned vector);
 void msix_vector_unuse(PCIDevice *dev, unsigned vector);
@@ -46,12 +47,16 @@
 
 extern const VMStateDescription vmstate_msix;
 
-#define VMSTATE_MSIX(_field, _state) {                               \
-    .name       = (stringify(_field)),                               \
-    .size       = sizeof(PCIDevice),                                 \
-    .vmsd       = &vmstate_msix,                                     \
-    .flags      = VMS_STRUCT,                                        \
-    .offset     = vmstate_offset_value(_state, _field, PCIDevice),   \
+#define VMSTATE_MSIX_TEST(_field, _state, _test) {                   \
+    .name         = (stringify(_field)),                             \
+    .size         = sizeof(PCIDevice),                               \
+    .vmsd         = &vmstate_msix,                                   \
+    .flags        = VMS_STRUCT,                                      \
+    .offset       = vmstate_offset_value(_state, _field, PCIDevice), \
+    .field_exists = (_test)                                          \
 }
 
+#define VMSTATE_MSIX(_f, _s)                                         \
+    VMSTATE_MSIX_TEST(_f, _s, NULL)
+
 #endif
diff --git a/include/hw/pci/pci-assign.h b/include/hw/pci/pci-assign.h
new file mode 100644
index 0000000..55f42c5
--- /dev/null
+++ b/include/hw/pci/pci-assign.h
@@ -0,0 +1,27 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Just split from hw/i386/kvm/pci-assign.c.
+ */
+#ifndef PCI_ASSIGN_H
+#define PCI_ASSIGN_H
+
+#include "hw/pci/pci.h"
+
+//#define DEVICE_ASSIGNMENT_DEBUG
+
+#ifdef DEVICE_ASSIGNMENT_DEBUG
+#define DEBUG(fmt, ...)                                       \
+    do {                                                      \
+        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
+    } while (0)
+#else
+#define DEBUG(fmt, ...)
+#endif
+
+void *pci_assign_dev_load_option_rom(PCIDevice *dev, struct Object *owner,
+                                     int *size, unsigned int domain,
+                                     unsigned int bus, unsigned int slot,
+                                     unsigned int function);
+#endif /* PCI_ASSIGN_H */
diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h
index c352c7b..929ec2f 100644
--- a/include/hw/pci/pci.h
+++ b/include/hw/pci/pci.h
@@ -1,12 +1,9 @@
 #ifndef QEMU_PCI_H
 #define QEMU_PCI_H
 
-#include "qemu-common.h"
-
 #include "hw/qdev.h"
 #include "exec/memory.h"
 #include "sysemu/dma.h"
-#include "qapi/error.h"
 
 /* PCI includes legacy ISA access.  */
 #include "hw/isa/isa.h"
@@ -18,6 +15,7 @@
 #define PCI_DEVFN(slot, func)   ((((slot) & 0x1f) << 3) | ((func) & 0x07))
 #define PCI_SLOT(devfn)         (((devfn) >> 3) & 0x1f)
 #define PCI_FUNC(devfn)         ((devfn) & 0x07)
+#define PCI_BUILD_BDF(bus, devfn)     ((bus << 8) | (devfn))
 #define PCI_SLOT_MAX            32
 #define PCI_FUNC_MAX            8
 
@@ -88,10 +86,25 @@
 #define PCI_DEVICE_ID_REDHAT_SERIAL2     0x0003
 #define PCI_DEVICE_ID_REDHAT_SERIAL4     0x0004
 #define PCI_DEVICE_ID_REDHAT_TEST        0x0005
+#define PCI_DEVICE_ID_REDHAT_ROCKER      0x0006
+#define PCI_DEVICE_ID_REDHAT_SDHCI       0x0007
+#define PCI_DEVICE_ID_REDHAT_PCIE_HOST   0x0008
+#define PCI_DEVICE_ID_REDHAT_PXB         0x0009
+#define PCI_DEVICE_ID_REDHAT_BRIDGE_SEAT 0x000a
+#define PCI_DEVICE_ID_REDHAT_PXB_PCIE    0x000b
 #define PCI_DEVICE_ID_REDHAT_QXL         0x0100
 
 #define FMT_PCIBUS                      PRIx64
 
+typedef uint64_t pcibus_t;
+
+struct PCIHostDeviceAddress {
+    unsigned int domain;
+    unsigned int bus;
+    unsigned int slot;
+    unsigned int function;
+};
+
 typedef void PCIConfigWriteFunc(PCIDevice *pci_dev,
                                 uint32_t address, uint32_t data, int len);
 typedef uint32_t PCIConfigReadFunc(PCIDevice *pci_dev,
@@ -135,7 +148,7 @@
 #define PCI_CONFIG_HEADER_SIZE 0x40
 /* Size of the standard PCI config space */
 #define PCI_CONFIG_SPACE_SIZE 0x100
-/* Size of the standart PCIe config space: 4KB */
+/* Size of the standard PCIe config space: 4KB */
 #define PCIE_CONFIG_SPACE_SIZE  0x1000
 
 #define PCI_NUM_PINS 4 /* A-D */
@@ -161,6 +174,9 @@
     /* PCI Express capability - Power Controller Present */
 #define QEMU_PCIE_SLTCAP_PCP_BITNR 7
     QEMU_PCIE_SLTCAP_PCP = (1 << QEMU_PCIE_SLTCAP_PCP_BITNR),
+    /* Link active status in endpoint capability is always set */
+#define QEMU_PCIE_LNKSTA_DLLLA_BITNR 8
+    QEMU_PCIE_LNKSTA_DLLLA = (1 << QEMU_PCIE_LNKSTA_DLLLA_BITNR),
 };
 
 #define TYPE_PCI_DEVICE "pci-device"
@@ -183,7 +199,8 @@
 typedef struct PCIDeviceClass {
     DeviceClass parent_class;
 
-    int (*init)(PCIDevice *dev);
+    void (*realize)(PCIDevice *dev, Error **errp);
+    int (*init)(PCIDevice *dev);/* TODO convert to realize() and remove */
     PCIUnregisterFunc *exit;
     PCIConfigReadFunc *config_read;
     PCIConfigWriteFunc *config_write;
@@ -217,6 +234,20 @@
                                       unsigned int vector_start,
                                       unsigned int vector_end);
 
+enum PCIReqIDType {
+    PCI_REQ_ID_INVALID = 0,
+    PCI_REQ_ID_BDF,
+    PCI_REQ_ID_SECONDARY_BUS,
+    PCI_REQ_ID_MAX,
+};
+typedef enum PCIReqIDType PCIReqIDType;
+
+struct PCIReqIDCache {
+    PCIDevice *dev;
+    PCIReqIDType type;
+};
+typedef struct PCIReqIDCache PCIReqIDCache;
+
 struct PCIDevice {
     DeviceState qdev;
 
@@ -239,6 +270,11 @@
     /* the following fields are read only */
     PCIBus *bus;
     int32_t devfn;
+    /* Cached device to fetch requester ID from, to avoid the PCI
+     * tree walking every time we invoke PCI request (e.g.,
+     * MSI). For conventional PCI root complex, this field is
+     * meaningless. */
+    PCIReqIDCache requester_id_cache;
     char name[64];
     PCIIORegion io_regions[PCI_NUM_REGIONS];
     AddressSpace bus_master_as;
@@ -330,12 +366,20 @@
 MemoryRegion *pci_address_space(PCIDevice *dev);
 MemoryRegion *pci_address_space_io(PCIDevice *dev);
 
+/*
+ * Should not normally be used by devices. For use by sPAPR target
+ * where QEMU emulates firmware.
+ */
+int pci_bar(PCIDevice *d, int reg);
+
 typedef void (*pci_set_irq_fn)(void *opaque, int irq_num, int level);
 typedef int (*pci_map_irq_fn)(PCIDevice *pci_dev, int irq_num);
 typedef PCIINTxRoute (*pci_route_irq_fn)(void *opaque, int pin);
 
 #define TYPE_PCI_BUS "PCI"
 #define PCI_BUS(obj) OBJECT_CHECK(PCIBus, (obj), TYPE_PCI_BUS)
+#define PCI_BUS_CLASS(klass) OBJECT_CLASS_CHECK(PCIBusClass, (klass), TYPE_PCI_BUS)
+#define PCI_BUS_GET_CLASS(obj) OBJECT_GET_CLASS(PCIBusClass, (obj), TYPE_PCI_BUS)
 #define TYPE_PCIE_BUS "PCIE"
 
 bool pci_bus_is_express(PCIBus *bus);
@@ -368,9 +412,6 @@
                                           PCIINTxRoutingNotifier notifier);
 void pci_device_reset(PCIDevice *dev);
 
-PCIDevice *pci_nic_init(NICInfo *nd, PCIBus *rootbus,
-                        const char *default_model,
-                        const char *default_devaddr);
 PCIDevice *pci_nic_init_nofail(NICInfo *nd, PCIBus *rootbus,
                                const char *default_model,
                                const char *default_devaddr);
@@ -378,6 +419,7 @@
 PCIDevice *pci_vga_init(PCIBus *bus);
 
 int pci_bus_num(PCIBus *s);
+int pci_bus_numa_node(PCIBus *bus);
 void pci_for_each_device(PCIBus *bus, int bus_num,
                          void (*fn)(PCIBus *bus, PCIDevice *d, void *opaque),
                          void *opaque);
@@ -385,6 +427,7 @@
                                   void *(*begin)(PCIBus *bus, void *parent_state),
                                   void (*end)(PCIBus *bus, void *state),
                                   void *parent_state);
+PCIDevice *pci_get_function_0(PCIDevice *pci_dev);
 
 /* Use this wrapper when specific scan order is not required. */
 static inline
@@ -400,12 +443,8 @@
 const char *pci_root_bus_path(PCIDevice *dev);
 PCIDevice *pci_find_device(PCIBus *bus, int bus_num, uint8_t devfn);
 int pci_qdev_find_device(const char *id, PCIDevice **pdev);
-PCIBus *pci_get_bus_devfn(int *devfnp, PCIBus *root, const char *devaddr);
 void pci_bus_get_w64_range(PCIBus *bus, Range *range);
 
-int pci_parse_devaddr(const char *addr, int *domp, int *busp,
-                      unsigned int *slotp, unsigned int *funcp);
-
 void pci_device_deassert_intx(PCIDevice *dev);
 
 typedef AddressSpace *(*PCIIOMMUFunc)(PCIBus *, void *, int);
@@ -449,16 +488,23 @@
     return ldl_le_p(config);
 }
 
+/*
+ * PCI capabilities and/or their fields
+ * are generally DWORD aligned only so
+ * mechanism used by pci_set/get_quad()
+ * must be tolerant to unaligned pointers
+ *
+ */
 static inline void
 pci_set_quad(uint8_t *config, uint64_t val)
 {
-    cpu_to_le64w((uint64_t *)config, val);
+    stq_le_p(config, val);
 }
 
 static inline uint64_t
 pci_get_quad(const uint8_t *config)
 {
-    return le64_to_cpup((const uint64_t *)config);
+    return ldq_le_p(config);
 }
 
 static inline void
@@ -572,7 +618,7 @@
 pci_set_byte_by_mask(uint8_t *config, uint8_t mask, uint8_t reg)
 {
     uint8_t val = pci_get_byte(config);
-    uint8_t rval = reg << (ffs(mask) - 1);
+    uint8_t rval = reg << ctz32(mask);
     pci_set_byte(config, (~mask & val) | (mask & rval));
 }
 
@@ -580,14 +626,14 @@
 pci_get_byte_by_mask(uint8_t *config, uint8_t mask)
 {
     uint8_t val = pci_get_byte(config);
-    return (val & mask) >> (ffs(mask) - 1);
+    return (val & mask) >> ctz32(mask);
 }
 
 static inline void
 pci_set_word_by_mask(uint8_t *config, uint16_t mask, uint16_t reg)
 {
     uint16_t val = pci_get_word(config);
-    uint16_t rval = reg << (ffs(mask) - 1);
+    uint16_t rval = reg << ctz32(mask);
     pci_set_word(config, (~mask & val) | (mask & rval));
 }
 
@@ -595,14 +641,14 @@
 pci_get_word_by_mask(uint8_t *config, uint16_t mask)
 {
     uint16_t val = pci_get_word(config);
-    return (val & mask) >> (ffs(mask) - 1);
+    return (val & mask) >> ctz32(mask);
 }
 
 static inline void
 pci_set_long_by_mask(uint8_t *config, uint32_t mask, uint32_t reg)
 {
     uint32_t val = pci_get_long(config);
-    uint32_t rval = reg << (ffs(mask) - 1);
+    uint32_t rval = reg << ctz32(mask);
     pci_set_long(config, (~mask & val) | (mask & rval));
 }
 
@@ -610,14 +656,14 @@
 pci_get_long_by_mask(uint8_t *config, uint32_t mask)
 {
     uint32_t val = pci_get_long(config);
-    return (val & mask) >> (ffs(mask) - 1);
+    return (val & mask) >> ctz32(mask);
 }
 
 static inline void
 pci_set_quad_by_mask(uint8_t *config, uint64_t mask, uint64_t reg)
 {
     uint64_t val = pci_get_quad(config);
-    uint64_t rval = reg << (ffs(mask) - 1);
+    uint64_t rval = reg << ctz32(mask);
     pci_set_quad(config, (~mask & val) | (mask & rval));
 }
 
@@ -625,7 +671,7 @@
 pci_get_quad_by_mask(uint8_t *config, uint64_t mask)
 {
     uint64_t val = pci_get_quad(config);
-    return (val & mask) >> (ffs(mask) - 1);
+    return (val & mask) >> ctz32(mask);
 }
 
 PCIDevice *pci_create_multifunction(PCIBus *bus, int devfn, bool multifunction,
@@ -669,6 +715,13 @@
     return pci_is_express(d) ? PCIE_CONFIG_SPACE_SIZE : PCI_CONFIG_SPACE_SIZE;
 }
 
+static inline uint16_t pci_get_bdf(PCIDevice *dev)
+{
+    return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn);
+}
+
+uint16_t pci_requester_id(PCIDevice *dev);
+
 /* DMA access functions */
 static inline AddressSpace *pci_get_address_space(PCIDevice *dev)
 {
@@ -755,4 +808,6 @@
     .offset     = vmstate_offset_pointer(_state, _field, PCIDevice), \
 }
 
+MSIMessage pci_get_msi_message(PCIDevice *dev, int vector);
+
 #endif
diff --git a/include/hw/pci/pci_bridge.h b/include/hw/pci/pci_bridge.h
index 1d8f997..847fd7d 100644
--- a/include/hw/pci/pci_bridge.h
+++ b/include/hw/pci/pci_bridge.h
@@ -28,6 +28,10 @@
 
 #include "hw/pci/pci.h"
 
+#define PCI_BRIDGE_DEV_PROP_CHASSIS_NR "chassis_nr"
+#define PCI_BRIDGE_DEV_PROP_MSI        "msi"
+#define PCI_BRIDGE_DEV_PROP_SHPC       "shpc"
+
 int pci_bridge_ssvid_init(PCIDevice *dev, uint8_t offset,
                           uint16_t svid, uint16_t ssid);
 
@@ -44,7 +48,7 @@
 void pci_bridge_reset_reg(PCIDevice *dev);
 void pci_bridge_reset(DeviceState *qdev);
 
-int pci_bridge_initfn(PCIDevice *pci_dev, const char *typename);
+void pci_bridge_initfn(PCIDevice *pci_dev, const char *typename);
 void pci_bridge_exitfn(PCIDevice *pci_dev);
 
 
@@ -63,4 +67,4 @@
 #define  PCI_BRIDGE_CTL_DISCARD_STATUS	0x400	/* Discard timer status */
 #define  PCI_BRIDGE_CTL_DISCARD_SERR	0x800	/* Discard timer SERR# enable */
 
-#endif  /* QEMU_PCI_BRIDGE_H */
+#endif /* QEMU_PCI_BRIDGE_H */
diff --git a/include/hw/pci/pci_bus.h b/include/hw/pci/pci_bus.h
index fabaeee..5484a9b 100644
--- a/include/hw/pci/pci_bus.h
+++ b/include/hw/pci/pci_bus.h
@@ -8,6 +8,16 @@
  * use accessor functions in pci.h, pci_bridge.h
  */
 
+typedef struct PCIBusClass {
+    /*< private >*/
+    BusClass parent_class;
+    /*< public >*/
+
+    bool (*is_root)(PCIBus *bus);
+    int (*bus_num)(PCIBus *bus);
+    uint16_t (*numa_node)(PCIBus *bus);
+} PCIBusClass;
+
 struct PCIBus {
     BusState qbus;
     PCIIOMMUFunc iommu_fn;
@@ -29,6 +39,8 @@
        Keep a count of the number of devices with raised IRQs.  */
     int nirq;
     int *irq_count;
+
+    Notifier machine_done;
 };
 
 typedef struct PCIBridgeWindows PCIBridgeWindows;
diff --git a/include/hw/pci/pci_ids.h b/include/hw/pci/pci_ids.h
index 321d622..d77ca60 100644
--- a/include/hw/pci/pci_ids.h
+++ b/include/hw/pci/pci_ids.h
@@ -7,8 +7,9 @@
  *
  *      QEMU-specific definitions belong in pci.h
  */
+
 #ifndef HW_PCI_IDS_H
-#define HW_PCI_IDS_H 1
+#define HW_PCI_IDS_H
 
 /* Device classes and subclasses */
 
@@ -23,6 +24,7 @@
 #define PCI_CLASS_STORAGE_OTHER          0x0180
 
 #define PCI_CLASS_NETWORK_ETHERNET       0x0200
+#define PCI_CLASS_NETWORK_OTHER          0x0280
 
 #define PCI_CLASS_DISPLAY_VGA            0x0300
 #define PCI_CLASS_DISPLAY_OTHER          0x0380
@@ -31,6 +33,7 @@
 
 #define PCI_CLASS_MEMORY_RAM             0x0500
 
+#define PCI_CLASS_SYSTEM_SDHCI           0x0805
 #define PCI_CLASS_SYSTEM_OTHER           0x0880
 
 #define PCI_CLASS_SERIAL_USB             0x0c03
@@ -45,6 +48,13 @@
 #define PCI_CLASS_COMMUNICATION_SERIAL   0x0700
 #define PCI_CLASS_COMMUNICATION_OTHER    0x0780
 
+#define PCI_CLASS_INPUT_KEYBOARD         0x0900
+#define PCI_CLASS_INPUT_PEN              0x0901
+#define PCI_CLASS_INPUT_MOUSE            0x0902
+#define PCI_CLASS_INPUT_SCANNER          0x0903
+#define PCI_CLASS_INPUT_GAMEPORT         0x0904
+#define PCI_CLASS_INPUT_OTHER            0x0980
+
 #define PCI_CLASS_PROCESSOR_CO           0x0b40
 #define PCI_CLASS_PROCESSOR_POWERPC      0x0b20
 
@@ -55,6 +65,7 @@
 #define PCI_VENDOR_ID_LSI_LOGIC          0x1000
 #define PCI_DEVICE_ID_LSI_53C810         0x0001
 #define PCI_DEVICE_ID_LSI_53C895A        0x0012
+#define PCI_DEVICE_ID_LSI_SAS1068        0x0054
 #define PCI_DEVICE_ID_LSI_SAS1078        0x0060
 #define PCI_DEVICE_ID_LSI_SAS0079        0x0079
 
@@ -105,6 +116,8 @@
 #define PCI_VENDOR_ID_ENSONIQ            0x1274
 #define PCI_DEVICE_ID_ENSONIQ_ES1370     0x5000
 
+#define PCI_VENDOR_ID_CHELSIO            0x1425
+
 #define PCI_VENDOR_ID_FREESCALE          0x1957
 #define PCI_DEVICE_ID_MPC8533E           0x0030
 
diff --git a/include/hw/pci/pci_regs.h b/include/hw/pci/pci_regs.h
index 56a404b..7a83142 100644
--- a/include/hw/pci/pci_regs.h
+++ b/include/hw/pci/pci_regs.h
@@ -1,717 +1,3 @@
-/*
- *	pci_regs.h
- *
- *	PCI standard defines
- *	Copyright 1994, Drew Eckhardt
- *	Copyright 1997--1999 Martin Mares <mj@ucw.cz>
- *
- *	For more information, please consult the following manuals (look at
- *	http://www.pcisig.com/ for how to get them):
- *
- *	PCI BIOS Specification
- *	PCI Local Bus Specification
- *	PCI to PCI Bridge Specification
- *	PCI System Design Guide
- *
- * 	For hypertransport information, please consult the following manuals
- * 	from http://www.hypertransport.org
- *
- *	The Hypertransport I/O Link Specification
- */
+#include "standard-headers/linux/pci_regs.h"
 
-#ifndef LINUX_PCI_REGS_H
-#define LINUX_PCI_REGS_H
-
-/*
- * Under PCI, each device has 256 bytes of configuration address space,
- * of which the first 64 bytes are standardized as follows:
- */
-#define PCI_VENDOR_ID		0x00	/* 16 bits */
-#define PCI_DEVICE_ID		0x02	/* 16 bits */
-#define PCI_COMMAND		0x04	/* 16 bits */
-#define  PCI_COMMAND_IO		0x1	/* Enable response in I/O space */
-#define  PCI_COMMAND_MEMORY	0x2	/* Enable response in Memory space */
-#define  PCI_COMMAND_MASTER	0x4	/* Enable bus mastering */
-#define  PCI_COMMAND_SPECIAL	0x8	/* Enable response to special cycles */
-#define  PCI_COMMAND_INVALIDATE	0x10	/* Use memory write and invalidate */
-#define  PCI_COMMAND_VGA_PALETTE 0x20	/* Enable palette snooping */
-#define  PCI_COMMAND_PARITY	0x40	/* Enable parity checking */
-#define  PCI_COMMAND_WAIT 	0x80	/* Enable address/data stepping */
-#define  PCI_COMMAND_SERR	0x100	/* Enable SERR */
-#define  PCI_COMMAND_FAST_BACK	0x200	/* Enable back-to-back writes */
-#define  PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */
-
-#define PCI_STATUS		0x06	/* 16 bits */
-#define  PCI_STATUS_INTERRUPT	0x08	/* Interrupt status */
-#define  PCI_STATUS_CAP_LIST	0x10	/* Support Capability List */
-#define  PCI_STATUS_66MHZ	0x20	/* Support 66 Mhz PCI 2.1 bus */
-#define  PCI_STATUS_UDF		0x40	/* Support User Definable Features [obsolete] */
-#define  PCI_STATUS_FAST_BACK	0x80	/* Accept fast-back to back */
-#define  PCI_STATUS_PARITY	0x100	/* Detected parity error */
-#define  PCI_STATUS_DEVSEL_MASK	0x600	/* DEVSEL timing */
-#define  PCI_STATUS_DEVSEL_FAST		0x000
-#define  PCI_STATUS_DEVSEL_MEDIUM	0x200
-#define  PCI_STATUS_DEVSEL_SLOW		0x400
-#define  PCI_STATUS_SIG_TARGET_ABORT	0x800 /* Set on target abort */
-#define  PCI_STATUS_REC_TARGET_ABORT	0x1000 /* Master ack of " */
-#define  PCI_STATUS_REC_MASTER_ABORT	0x2000 /* Set on master abort */
-#define  PCI_STATUS_SIG_SYSTEM_ERROR	0x4000 /* Set when we drive SERR */
-#define  PCI_STATUS_DETECTED_PARITY	0x8000 /* Set on parity error */
-
-#define PCI_CLASS_REVISION	0x08	/* High 24 bits are class, low 8 revision */
-#define PCI_REVISION_ID		0x08	/* Revision ID */
-#define PCI_CLASS_PROG		0x09	/* Reg. Level Programming Interface */
-#define PCI_CLASS_DEVICE	0x0a	/* Device class */
-
-#define PCI_CACHE_LINE_SIZE	0x0c	/* 8 bits */
-#define PCI_LATENCY_TIMER	0x0d	/* 8 bits */
-#define PCI_HEADER_TYPE		0x0e	/* 8 bits */
-#define  PCI_HEADER_TYPE_NORMAL		0
-#define  PCI_HEADER_TYPE_BRIDGE		1
-#define  PCI_HEADER_TYPE_CARDBUS	2
-
-#define PCI_BIST		0x0f	/* 8 bits */
-#define  PCI_BIST_CODE_MASK	0x0f	/* Return result */
-#define  PCI_BIST_START		0x40	/* 1 to start BIST, 2 secs or less */
-#define  PCI_BIST_CAPABLE	0x80	/* 1 if BIST capable */
-
-/*
- * Base addresses specify locations in memory or I/O space.
- * Decoded size can be determined by writing a value of
- * 0xffffffff to the register, and reading it back.  Only
- * 1 bits are decoded.
- */
-#define PCI_BASE_ADDRESS_0	0x10	/* 32 bits */
-#define PCI_BASE_ADDRESS_1	0x14	/* 32 bits [htype 0,1 only] */
-#define PCI_BASE_ADDRESS_2	0x18	/* 32 bits [htype 0 only] */
-#define PCI_BASE_ADDRESS_3	0x1c	/* 32 bits */
-#define PCI_BASE_ADDRESS_4	0x20	/* 32 bits */
-#define PCI_BASE_ADDRESS_5	0x24	/* 32 bits */
-#define  PCI_BASE_ADDRESS_SPACE		0x01	/* 0 = memory, 1 = I/O */
-#define  PCI_BASE_ADDRESS_SPACE_IO	0x01
-#define  PCI_BASE_ADDRESS_SPACE_MEMORY	0x00
-#define  PCI_BASE_ADDRESS_MEM_TYPE_MASK	0x06
-#define  PCI_BASE_ADDRESS_MEM_TYPE_32	0x00	/* 32 bit address */
-#define  PCI_BASE_ADDRESS_MEM_TYPE_1M	0x02	/* Below 1M [obsolete] */
-#define  PCI_BASE_ADDRESS_MEM_TYPE_64	0x04	/* 64 bit address */
-#define  PCI_BASE_ADDRESS_MEM_PREFETCH	0x08	/* prefetchable? */
-#define  PCI_BASE_ADDRESS_MEM_MASK	(~0x0fUL)
-#define  PCI_BASE_ADDRESS_IO_MASK	(~0x03UL)
-/* bit 1 is reserved if address_space = 1 */
-
-/* Header type 0 (normal devices) */
-#define PCI_CARDBUS_CIS		0x28
-#define PCI_SUBSYSTEM_VENDOR_ID	0x2c
-#define PCI_SUBSYSTEM_ID	0x2e
-#define PCI_ROM_ADDRESS		0x30	/* Bits 31..11 are address, 10..1 reserved */
-#define  PCI_ROM_ADDRESS_ENABLE	0x01
-#define PCI_ROM_ADDRESS_MASK	(~0x7ffUL)
-
-#define PCI_CAPABILITY_LIST	0x34	/* Offset of first capability list entry */
-
-/* 0x35-0x3b are reserved */
-#define PCI_INTERRUPT_LINE	0x3c	/* 8 bits */
-#define PCI_INTERRUPT_PIN	0x3d	/* 8 bits */
-#define PCI_MIN_GNT		0x3e	/* 8 bits */
-#define PCI_MAX_LAT		0x3f	/* 8 bits */
-
-/* Header type 1 (PCI-to-PCI bridges) */
-#define PCI_PRIMARY_BUS		0x18	/* Primary bus number */
-#define PCI_SECONDARY_BUS	0x19	/* Secondary bus number */
-#define PCI_SUBORDINATE_BUS	0x1a	/* Highest bus number behind the bridge */
-#define PCI_SEC_LATENCY_TIMER	0x1b	/* Latency timer for secondary interface */
-#define PCI_IO_BASE		0x1c	/* I/O range behind the bridge */
-#define PCI_IO_LIMIT		0x1d
-#define  PCI_IO_RANGE_TYPE_MASK	0x0fUL	/* I/O bridging type */
-#define  PCI_IO_RANGE_TYPE_16	0x00
-#define  PCI_IO_RANGE_TYPE_32	0x01
-#define  PCI_IO_RANGE_MASK	(~0x0fUL)
-#define PCI_SEC_STATUS		0x1e	/* Secondary status register, only bit 14 used */
-#define PCI_MEMORY_BASE		0x20	/* Memory range behind */
-#define PCI_MEMORY_LIMIT	0x22
-#define  PCI_MEMORY_RANGE_TYPE_MASK 0x0fUL
-#define  PCI_MEMORY_RANGE_MASK	(~0x0fUL)
-#define PCI_PREF_MEMORY_BASE	0x24	/* Prefetchable memory range behind */
-#define PCI_PREF_MEMORY_LIMIT	0x26
-#define  PCI_PREF_RANGE_TYPE_MASK 0x0fUL
-#define  PCI_PREF_RANGE_TYPE_32	0x00
-#define  PCI_PREF_RANGE_TYPE_64	0x01
-#define  PCI_PREF_RANGE_MASK	(~0x0fUL)
-#define PCI_PREF_BASE_UPPER32	0x28	/* Upper half of prefetchable memory range */
-#define PCI_PREF_LIMIT_UPPER32	0x2c
-#define PCI_IO_BASE_UPPER16	0x30	/* Upper half of I/O addresses */
-#define PCI_IO_LIMIT_UPPER16	0x32
-/* 0x34 same as for htype 0 */
-/* 0x35-0x3b is reserved */
-#define PCI_ROM_ADDRESS1	0x38	/* Same as PCI_ROM_ADDRESS, but for htype 1 */
-/* 0x3c-0x3d are same as for htype 0 */
-#define PCI_BRIDGE_CONTROL	0x3e
-#define  PCI_BRIDGE_CTL_PARITY	0x01	/* Enable parity detection on secondary interface */
-#define  PCI_BRIDGE_CTL_SERR	0x02	/* The same for SERR forwarding */
-#define  PCI_BRIDGE_CTL_ISA	0x04	/* Enable ISA mode */
-#define  PCI_BRIDGE_CTL_VGA	0x08	/* Forward VGA addresses */
-#define  PCI_BRIDGE_CTL_MASTER_ABORT	0x20  /* Report master aborts */
-#define  PCI_BRIDGE_CTL_BUS_RESET	0x40	/* Secondary bus reset */
-#define  PCI_BRIDGE_CTL_FAST_BACK	0x80	/* Fast Back2Back enabled on secondary interface */
-
-/* Header type 2 (CardBus bridges) */
-#define PCI_CB_CAPABILITY_LIST	0x14
-/* 0x15 reserved */
-#define PCI_CB_SEC_STATUS	0x16	/* Secondary status */
-#define PCI_CB_PRIMARY_BUS	0x18	/* PCI bus number */
-#define PCI_CB_CARD_BUS		0x19	/* CardBus bus number */
-#define PCI_CB_SUBORDINATE_BUS	0x1a	/* Subordinate bus number */
-#define PCI_CB_LATENCY_TIMER	0x1b	/* CardBus latency timer */
-#define PCI_CB_MEMORY_BASE_0	0x1c
-#define PCI_CB_MEMORY_LIMIT_0	0x20
-#define PCI_CB_MEMORY_BASE_1	0x24
-#define PCI_CB_MEMORY_LIMIT_1	0x28
-#define PCI_CB_IO_BASE_0	0x2c
-#define PCI_CB_IO_BASE_0_HI	0x2e
-#define PCI_CB_IO_LIMIT_0	0x30
-#define PCI_CB_IO_LIMIT_0_HI	0x32
-#define PCI_CB_IO_BASE_1	0x34
-#define PCI_CB_IO_BASE_1_HI	0x36
-#define PCI_CB_IO_LIMIT_1	0x38
-#define PCI_CB_IO_LIMIT_1_HI	0x3a
-#define  PCI_CB_IO_RANGE_MASK	(~0x03UL)
-/* 0x3c-0x3d are same as for htype 0 */
-#define PCI_CB_BRIDGE_CONTROL	0x3e
-#define  PCI_CB_BRIDGE_CTL_PARITY	0x01	/* Similar to standard bridge control register */
-#define  PCI_CB_BRIDGE_CTL_SERR		0x02
-#define  PCI_CB_BRIDGE_CTL_ISA		0x04
-#define  PCI_CB_BRIDGE_CTL_VGA		0x08
-#define  PCI_CB_BRIDGE_CTL_MASTER_ABORT	0x20
-#define  PCI_CB_BRIDGE_CTL_CB_RESET	0x40	/* CardBus reset */
-#define  PCI_CB_BRIDGE_CTL_16BIT_INT	0x80	/* Enable interrupt for 16-bit cards */
-#define  PCI_CB_BRIDGE_CTL_PREFETCH_MEM0 0x100	/* Prefetch enable for both memory regions */
-#define  PCI_CB_BRIDGE_CTL_PREFETCH_MEM1 0x200
-#define  PCI_CB_BRIDGE_CTL_POST_WRITES	0x400
-#define PCI_CB_SUBSYSTEM_VENDOR_ID	0x40
-#define PCI_CB_SUBSYSTEM_ID		0x42
-#define PCI_CB_LEGACY_MODE_BASE		0x44	/* 16-bit PC Card legacy mode base address (ExCa) */
-/* 0x48-0x7f reserved */
-
-/* Capability lists */
-
-#define PCI_CAP_LIST_ID		0	/* Capability ID */
-#define  PCI_CAP_ID_PM		0x01	/* Power Management */
-#define  PCI_CAP_ID_AGP		0x02	/* Accelerated Graphics Port */
-#define  PCI_CAP_ID_VPD		0x03	/* Vital Product Data */
-#define  PCI_CAP_ID_SLOTID	0x04	/* Slot Identification */
-#define  PCI_CAP_ID_MSI		0x05	/* Message Signalled Interrupts */
-#define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
-#define  PCI_CAP_ID_PCIX	0x07	/* PCI-X */
-#define  PCI_CAP_ID_HT		0x08	/* HyperTransport */
-#define  PCI_CAP_ID_VNDR	0x09	/* Vendor specific */
-#define  PCI_CAP_ID_DBG		0x0A	/* Debug port */
-#define  PCI_CAP_ID_CCRC	0x0B	/* CompactPCI Central Resource Control */
-#define  PCI_CAP_ID_SHPC 	0x0C	/* PCI Standard Hot-Plug Controller */
-#define  PCI_CAP_ID_SSVID	0x0D	/* Bridge subsystem vendor/device ID */
-#define  PCI_CAP_ID_AGP3	0x0E	/* AGP Target PCI-PCI bridge */
-#define  PCI_CAP_ID_EXP 	0x10	/* PCI Express */
-#define  PCI_CAP_ID_MSIX	0x11	/* MSI-X */
-#define  PCI_CAP_ID_SATA	0x12	/* Serial ATA */
-#define  PCI_CAP_ID_AF		0x13	/* PCI Advanced Features */
-#define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
-#define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
-#define PCI_CAP_SIZEOF		4
-
-/* Power Management Registers */
-
-#define PCI_PM_PMC		2	/* PM Capabilities Register */
-#define  PCI_PM_CAP_VER_MASK	0x0007	/* Version */
-#define  PCI_PM_CAP_PME_CLOCK	0x0008	/* PME clock required */
-#define  PCI_PM_CAP_RESERVED    0x0010  /* Reserved field */
-#define  PCI_PM_CAP_DSI		0x0020	/* Device specific initialization */
-#define  PCI_PM_CAP_AUX_POWER	0x01C0	/* Auxiliary power support mask */
-#define  PCI_PM_CAP_D1		0x0200	/* D1 power state support */
-#define  PCI_PM_CAP_D2		0x0400	/* D2 power state support */
-#define  PCI_PM_CAP_PME		0x0800	/* PME pin supported */
-#define  PCI_PM_CAP_PME_MASK	0xF800	/* PME Mask of all supported states */
-#define  PCI_PM_CAP_PME_D0	0x0800	/* PME# from D0 */
-#define  PCI_PM_CAP_PME_D1	0x1000	/* PME# from D1 */
-#define  PCI_PM_CAP_PME_D2	0x2000	/* PME# from D2 */
-#define  PCI_PM_CAP_PME_D3	0x4000	/* PME# from D3 (hot) */
-#define  PCI_PM_CAP_PME_D3cold	0x8000	/* PME# from D3 (cold) */
-#define  PCI_PM_CAP_PME_SHIFT	11	/* Start of the PME Mask in PMC */
-#define PCI_PM_CTRL		4	/* PM control and status register */
-#define  PCI_PM_CTRL_STATE_MASK	0x0003	/* Current power state (D0 to D3) */
-#define  PCI_PM_CTRL_NO_SOFT_RESET	0x0008	/* No reset for D3hot->D0 */
-#define  PCI_PM_CTRL_PME_ENABLE	0x0100	/* PME pin enable */
-#define  PCI_PM_CTRL_DATA_SEL_MASK	0x1e00	/* Data select (??) */
-#define  PCI_PM_CTRL_DATA_SCALE_MASK	0x6000	/* Data scale (??) */
-#define  PCI_PM_CTRL_PME_STATUS	0x8000	/* PME pin status */
-#define PCI_PM_PPB_EXTENSIONS	6	/* PPB support extensions (??) */
-#define  PCI_PM_PPB_B2_B3	0x40	/* Stop clock when in D3hot (??) */
-#define  PCI_PM_BPCC_ENABLE	0x80	/* Bus power/clock control enable (??) */
-#define PCI_PM_DATA_REGISTER	7	/* (??) */
-#define PCI_PM_SIZEOF		8
-
-/* AGP registers */
-
-#define PCI_AGP_VERSION		2	/* BCD version number */
-#define PCI_AGP_RFU		3	/* Rest of capability flags */
-#define PCI_AGP_STATUS		4	/* Status register */
-#define  PCI_AGP_STATUS_RQ_MASK	0xff000000	/* Maximum number of requests - 1 */
-#define  PCI_AGP_STATUS_SBA	0x0200	/* Sideband addressing supported */
-#define  PCI_AGP_STATUS_64BIT	0x0020	/* 64-bit addressing supported */
-#define  PCI_AGP_STATUS_FW	0x0010	/* FW transfers supported */
-#define  PCI_AGP_STATUS_RATE4	0x0004	/* 4x transfer rate supported */
-#define  PCI_AGP_STATUS_RATE2	0x0002	/* 2x transfer rate supported */
-#define  PCI_AGP_STATUS_RATE1	0x0001	/* 1x transfer rate supported */
-#define PCI_AGP_COMMAND		8	/* Control register */
-#define  PCI_AGP_COMMAND_RQ_MASK 0xff000000  /* Master: Maximum number of requests */
-#define  PCI_AGP_COMMAND_SBA	0x0200	/* Sideband addressing enabled */
-#define  PCI_AGP_COMMAND_AGP	0x0100	/* Allow processing of AGP transactions */
-#define  PCI_AGP_COMMAND_64BIT	0x0020 	/* Allow processing of 64-bit addresses */
-#define  PCI_AGP_COMMAND_FW	0x0010 	/* Force FW transfers */
-#define  PCI_AGP_COMMAND_RATE4	0x0004	/* Use 4x rate */
-#define  PCI_AGP_COMMAND_RATE2	0x0002	/* Use 2x rate */
-#define  PCI_AGP_COMMAND_RATE1	0x0001	/* Use 1x rate */
-#define PCI_AGP_SIZEOF		12
-
-/* Vital Product Data */
-
-#define PCI_VPD_ADDR		2	/* Address to access (15 bits!) */
-#define  PCI_VPD_ADDR_MASK	0x7fff	/* Address mask */
-#define  PCI_VPD_ADDR_F		0x8000	/* Write 0, 1 indicates completion */
-#define PCI_VPD_DATA		4	/* 32-bits of data returned here */
-
-/* Slot Identification */
-
-#define PCI_SID_ESR		2	/* Expansion Slot Register */
-#define  PCI_SID_ESR_NSLOTS	0x1f	/* Number of expansion slots available */
-#define  PCI_SID_ESR_FIC	0x20	/* First In Chassis Flag */
-#define PCI_SID_CHASSIS_NR	3	/* Chassis Number */
-
-/* Message Signalled Interrupts registers */
-
-#define PCI_MSI_FLAGS		2	/* Various flags */
-#define  PCI_MSI_FLAGS_64BIT	0x80	/* 64-bit addresses allowed */
-#define  PCI_MSI_FLAGS_QSIZE	0x70	/* Message queue size configured */
-#define  PCI_MSI_FLAGS_QMASK	0x0e	/* Maximum queue size available */
-#define  PCI_MSI_FLAGS_ENABLE	0x01	/* MSI feature enabled */
-#define  PCI_MSI_FLAGS_MASKBIT	0x100	/* 64-bit mask bits allowed */
-#define PCI_MSI_RFU		3	/* Rest of capability flags */
-#define PCI_MSI_ADDRESS_LO	4	/* Lower 32 bits */
-#define PCI_MSI_ADDRESS_HI	8	/* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */
-#define PCI_MSI_DATA_32		8	/* 16 bits of data for 32-bit devices */
-#define PCI_MSI_MASK_32		12	/* Mask bits register for 32-bit devices */
-#define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
-#define PCI_MSI_MASK_64		16	/* Mask bits register for 64-bit devices */
-
-/* MSI-X registers */
-#define PCI_MSIX_FLAGS		2
-#define  PCI_MSIX_FLAGS_QSIZE	0x7FF
-#define  PCI_MSIX_FLAGS_ENABLE	(1 << 15)
-#define  PCI_MSIX_FLAGS_MASKALL	(1 << 14)
-#define PCI_MSIX_TABLE		4
-#define PCI_MSIX_PBA		8
-#define  PCI_MSIX_FLAGS_BIRMASK	(7 << 0)
-
-/* MSI-X entry's format */
-#define PCI_MSIX_ENTRY_SIZE		16
-#define  PCI_MSIX_ENTRY_LOWER_ADDR	0
-#define  PCI_MSIX_ENTRY_UPPER_ADDR	4
-#define  PCI_MSIX_ENTRY_DATA		8
-#define  PCI_MSIX_ENTRY_VECTOR_CTRL	12
-#define   PCI_MSIX_ENTRY_CTRL_MASKBIT	1
-
-/* CompactPCI Hotswap Register */
-
-#define PCI_CHSWP_CSR		2	/* Control and Status Register */
-#define  PCI_CHSWP_DHA		0x01	/* Device Hiding Arm */
-#define  PCI_CHSWP_EIM		0x02	/* ENUM# Signal Mask */
-#define  PCI_CHSWP_PIE		0x04	/* Pending Insert or Extract */
-#define  PCI_CHSWP_LOO		0x08	/* LED On / Off */
-#define  PCI_CHSWP_PI		0x30	/* Programming Interface */
-#define  PCI_CHSWP_EXT		0x40	/* ENUM# status - extraction */
-#define  PCI_CHSWP_INS		0x80	/* ENUM# status - insertion */
-
-/* PCI Advanced Feature registers */
-
-#define PCI_AF_LENGTH		2
-#define PCI_AF_CAP		3
-#define  PCI_AF_CAP_TP		0x01
-#define  PCI_AF_CAP_FLR		0x02
-#define PCI_AF_CTRL		4
-#define  PCI_AF_CTRL_FLR	0x01
-#define PCI_AF_STATUS		5
-#define  PCI_AF_STATUS_TP	0x01
-
-/* PCI-X registers */
-
-#define PCI_X_CMD		2	/* Modes & Features */
-#define  PCI_X_CMD_DPERR_E	0x0001	/* Data Parity Error Recovery Enable */
-#define  PCI_X_CMD_ERO		0x0002	/* Enable Relaxed Ordering */
-#define  PCI_X_CMD_READ_512	0x0000	/* 512 byte maximum read byte count */
-#define  PCI_X_CMD_READ_1K	0x0004	/* 1Kbyte maximum read byte count */
-#define  PCI_X_CMD_READ_2K	0x0008	/* 2Kbyte maximum read byte count */
-#define  PCI_X_CMD_READ_4K	0x000c	/* 4Kbyte maximum read byte count */
-#define  PCI_X_CMD_MAX_READ	0x000c	/* Max Memory Read Byte Count */
-				/* Max # of outstanding split transactions */
-#define  PCI_X_CMD_SPLIT_1	0x0000	/* Max 1 */
-#define  PCI_X_CMD_SPLIT_2	0x0010	/* Max 2 */
-#define  PCI_X_CMD_SPLIT_3	0x0020	/* Max 3 */
-#define  PCI_X_CMD_SPLIT_4	0x0030	/* Max 4 */
-#define  PCI_X_CMD_SPLIT_8	0x0040	/* Max 8 */
-#define  PCI_X_CMD_SPLIT_12	0x0050	/* Max 12 */
-#define  PCI_X_CMD_SPLIT_16	0x0060	/* Max 16 */
-#define  PCI_X_CMD_SPLIT_32	0x0070	/* Max 32 */
-#define  PCI_X_CMD_MAX_SPLIT	0x0070	/* Max Outstanding Split Transactions */
-#define  PCI_X_CMD_VERSION(x) 	(((x) >> 12) & 3) /* Version */
-#define PCI_X_STATUS		4	/* PCI-X capabilities */
-#define  PCI_X_STATUS_DEVFN	0x000000ff	/* A copy of devfn */
-#define  PCI_X_STATUS_BUS	0x0000ff00	/* A copy of bus nr */
-#define  PCI_X_STATUS_64BIT	0x00010000	/* 64-bit device */
-#define  PCI_X_STATUS_133MHZ	0x00020000	/* 133 MHz capable */
-#define  PCI_X_STATUS_SPL_DISC	0x00040000	/* Split Completion Discarded */
-#define  PCI_X_STATUS_UNX_SPL	0x00080000	/* Unexpected Split Completion */
-#define  PCI_X_STATUS_COMPLEX	0x00100000	/* Device Complexity */
-#define  PCI_X_STATUS_MAX_READ	0x00600000	/* Designed Max Memory Read Count */
-#define  PCI_X_STATUS_MAX_SPLIT	0x03800000	/* Designed Max Outstanding Split Transactions */
-#define  PCI_X_STATUS_MAX_CUM	0x1c000000	/* Designed Max Cumulative Read Size */
-#define  PCI_X_STATUS_SPL_ERR	0x20000000	/* Rcvd Split Completion Error Msg */
-#define  PCI_X_STATUS_266MHZ	0x40000000	/* 266 MHz capable */
-#define  PCI_X_STATUS_533MHZ	0x80000000	/* 533 MHz capable */
-
-/* PCI Bridge Subsystem ID registers */
-
-#define PCI_SSVID_VENDOR_ID     4	/* PCI-Bridge subsystem vendor id register */
-#define PCI_SSVID_DEVICE_ID     6	/* PCI-Bridge subsystem device id register */
-
-/* PCI Express capability registers */
-
-#define PCI_EXP_FLAGS		2	/* Capabilities register */
-#define PCI_EXP_FLAGS_VERS	0x000f	/* Capability version */
-#define PCI_EXP_FLAGS_TYPE	0x00f0	/* Device/Port type */
-#define  PCI_EXP_TYPE_ENDPOINT	0x0	/* Express Endpoint */
-#define  PCI_EXP_TYPE_LEG_END	0x1	/* Legacy Endpoint */
-#define  PCI_EXP_TYPE_ROOT_PORT 0x4	/* Root Port */
-#define  PCI_EXP_TYPE_UPSTREAM	0x5	/* Upstream Port */
-#define  PCI_EXP_TYPE_DOWNSTREAM 0x6	/* Downstream Port */
-#define  PCI_EXP_TYPE_PCI_BRIDGE 0x7	/* PCI/PCI-X Bridge */
-#define  PCI_EXP_TYPE_PCIE_BRIDGE 0x8   /* PCI/PCI-X to PCIE Bridge */
-#define  PCI_EXP_TYPE_RC_END	0x9	/* Root Complex Integrated Endpoint */
-#define  PCI_EXP_TYPE_RC_EC     0xa     /* Root Complex Event Collector */
-#define PCI_EXP_FLAGS_SLOT	0x0100	/* Slot implemented */
-#define PCI_EXP_FLAGS_IRQ	0x3e00	/* Interrupt message number */
-#define PCI_EXP_DEVCAP		4	/* Device capabilities */
-#define  PCI_EXP_DEVCAP_PAYLOAD	0x07	/* Max_Payload_Size */
-#define  PCI_EXP_DEVCAP_PHANTOM	0x18	/* Phantom functions */
-#define  PCI_EXP_DEVCAP_EXT_TAG	0x20	/* Extended tags */
-#define  PCI_EXP_DEVCAP_L0S	0x1c0	/* L0s Acceptable Latency */
-#define  PCI_EXP_DEVCAP_L1	0xe00	/* L1 Acceptable Latency */
-#define  PCI_EXP_DEVCAP_ATN_BUT	0x1000	/* Attention Button Present */
-#define  PCI_EXP_DEVCAP_ATN_IND	0x2000	/* Attention Indicator Present */
-#define  PCI_EXP_DEVCAP_PWR_IND	0x4000	/* Power Indicator Present */
-#define  PCI_EXP_DEVCAP_RBER	0x8000	/* Role-Based Error Reporting */
-#define  PCI_EXP_DEVCAP_PWR_VAL	0x3fc0000 /* Slot Power Limit Value */
-#define  PCI_EXP_DEVCAP_PWR_SCL	0xc000000 /* Slot Power Limit Scale */
-#define  PCI_EXP_DEVCAP_FLR     0x10000000 /* Function Level Reset */
-#define PCI_EXP_DEVCTL		8	/* Device Control */
-#define  PCI_EXP_DEVCTL_CERE	0x0001	/* Correctable Error Reporting En. */
-#define  PCI_EXP_DEVCTL_NFERE	0x0002	/* Non-Fatal Error Reporting Enable */
-#define  PCI_EXP_DEVCTL_FERE	0x0004	/* Fatal Error Reporting Enable */
-#define  PCI_EXP_DEVCTL_URRE	0x0008	/* Unsupported Request Reporting En. */
-#define  PCI_EXP_DEVCTL_RELAX_EN 0x0010 /* Enable relaxed ordering */
-#define  PCI_EXP_DEVCTL_PAYLOAD	0x00e0	/* Max_Payload_Size */
-#define  PCI_EXP_DEVCTL_EXT_TAG	0x0100	/* Extended Tag Field Enable */
-#define  PCI_EXP_DEVCTL_PHANTOM	0x0200	/* Phantom Functions Enable */
-#define  PCI_EXP_DEVCTL_AUX_PME	0x0400	/* Auxiliary Power PM Enable */
-#define  PCI_EXP_DEVCTL_NOSNOOP_EN 0x0800  /* Enable No Snoop */
-#define  PCI_EXP_DEVCTL_READRQ	0x7000	/* Max_Read_Request_Size */
-#define  PCI_EXP_DEVCTL_BCR_FLR 0x8000  /* Bridge Configuration Retry / FLR */
-#define PCI_EXP_DEVSTA		10	/* Device Status */
-#define  PCI_EXP_DEVSTA_CED	0x01	/* Correctable Error Detected */
-#define  PCI_EXP_DEVSTA_NFED	0x02	/* Non-Fatal Error Detected */
-#define  PCI_EXP_DEVSTA_FED	0x04	/* Fatal Error Detected */
-#define  PCI_EXP_DEVSTA_URD	0x08	/* Unsupported Request Detected */
-#define  PCI_EXP_DEVSTA_AUXPD	0x10	/* AUX Power Detected */
-#define  PCI_EXP_DEVSTA_TRPND	0x20	/* Transactions Pending */
-#define PCI_EXP_LNKCAP		12	/* Link Capabilities */
-#define  PCI_EXP_LNKCAP_SLS	0x0000000f /* Supported Link Speeds */
-#define  PCI_EXP_LNKCAP_MLW	0x000003f0 /* Maximum Link Width */
-#define  PCI_EXP_LNKCAP_ASPMS	0x00000c00 /* ASPM Support */
-#define  PCI_EXP_LNKCAP_L0SEL	0x00007000 /* L0s Exit Latency */
-#define  PCI_EXP_LNKCAP_L1EL	0x00038000 /* L1 Exit Latency */
-#define  PCI_EXP_LNKCAP_CLKPM	0x00040000 /* L1 Clock Power Management */
-#define  PCI_EXP_LNKCAP_SDERC	0x00080000 /* Surprise Down Error Reporting Capable */
-#define  PCI_EXP_LNKCAP_DLLLARC	0x00100000 /* Data Link Layer Link Active Reporting Capable */
-#define  PCI_EXP_LNKCAP_LBNC	0x00200000 /* Link Bandwidth Notification Capability */
-#define  PCI_EXP_LNKCAP_PN	0xff000000 /* Port Number */
-#define PCI_EXP_LNKCTL		16	/* Link Control */
-#define  PCI_EXP_LNKCTL_ASPMC	0x0003	/* ASPM Control */
-#define  PCI_EXP_LNKCTL_RCB	0x0008	/* Read Completion Boundary */
-#define  PCI_EXP_LNKCTL_LD	0x0010	/* Link Disable */
-#define  PCI_EXP_LNKCTL_RL	0x0020	/* Retrain Link */
-#define  PCI_EXP_LNKCTL_CCC	0x0040	/* Common Clock Configuration */
-#define  PCI_EXP_LNKCTL_ES	0x0080	/* Extended Synch */
-#define  PCI_EXP_LNKCTL_CLKREQ_EN 0x100	/* Enable clkreq */
-#define  PCI_EXP_LNKCTL_HAWD	0x0200	/* Hardware Autonomous Width Disable */
-#define  PCI_EXP_LNKCTL_LBMIE	0x0400	/* Link Bandwidth Management Interrupt Enable */
-#define  PCI_EXP_LNKCTL_LABIE	0x0800	/* Lnk Autonomous Bandwidth Interrupt Enable */
-#define PCI_EXP_LNKSTA		18	/* Link Status */
-#define  PCI_EXP_LNKSTA_CLS	0x000f	/* Current Link Speed */
-#define  PCI_EXP_LNKSTA_CLS_2_5GB 0x01	/* Current Link Speed 2.5GT/s */
-#define  PCI_EXP_LNKSTA_CLS_5_0GB 0x02	/* Current Link Speed 5.0GT/s */
-#define  PCI_EXP_LNKSTA_NLW	0x03f0	/* Nogotiated Link Width */
-#define  PCI_EXP_LNKSTA_NLW_SHIFT 4	/* start of NLW mask in link status */
-#define  PCI_EXP_LNKSTA_LT	0x0800	/* Link Training */
-#define  PCI_EXP_LNKSTA_SLC	0x1000	/* Slot Clock Configuration */
-#define  PCI_EXP_LNKSTA_DLLLA	0x2000	/* Data Link Layer Link Active */
-#define  PCI_EXP_LNKSTA_LBMS	0x4000	/* Link Bandwidth Management Status */
-#define  PCI_EXP_LNKSTA_LABS	0x8000	/* Link Autonomous Bandwidth Status */
-#define PCI_EXP_SLTCAP		20	/* Slot Capabilities */
-#define  PCI_EXP_SLTCAP_ABP	0x00000001 /* Attention Button Present */
-#define  PCI_EXP_SLTCAP_PCP	0x00000002 /* Power Controller Present */
-#define  PCI_EXP_SLTCAP_MRLSP	0x00000004 /* MRL Sensor Present */
-#define  PCI_EXP_SLTCAP_AIP	0x00000008 /* Attention Indicator Present */
-#define  PCI_EXP_SLTCAP_PIP	0x00000010 /* Power Indicator Present */
-#define  PCI_EXP_SLTCAP_HPS	0x00000020 /* Hot-Plug Surprise */
-#define  PCI_EXP_SLTCAP_HPC	0x00000040 /* Hot-Plug Capable */
-#define  PCI_EXP_SLTCAP_SPLV	0x00007f80 /* Slot Power Limit Value */
-#define  PCI_EXP_SLTCAP_SPLS	0x00018000 /* Slot Power Limit Scale */
-#define  PCI_EXP_SLTCAP_EIP	0x00020000 /* Electromechanical Interlock Present */
-#define  PCI_EXP_SLTCAP_NCCS	0x00040000 /* No Command Completed Support */
-#define  PCI_EXP_SLTCAP_PSN	0xfff80000 /* Physical Slot Number */
-#define PCI_EXP_SLTCTL		24	/* Slot Control */
-#define  PCI_EXP_SLTCTL_ABPE	0x0001	/* Attention Button Pressed Enable */
-#define  PCI_EXP_SLTCTL_PFDE	0x0002	/* Power Fault Detected Enable */
-#define  PCI_EXP_SLTCTL_MRLSCE	0x0004	/* MRL Sensor Changed Enable */
-#define  PCI_EXP_SLTCTL_PDCE	0x0008	/* Presence Detect Changed Enable */
-#define  PCI_EXP_SLTCTL_CCIE	0x0010	/* Command Completed Interrupt Enable */
-#define  PCI_EXP_SLTCTL_HPIE	0x0020	/* Hot-Plug Interrupt Enable */
-#define  PCI_EXP_SLTCTL_AIC	0x00c0	/* Attention Indicator Control */
-#define  PCI_EXP_SLTCTL_PIC	0x0300	/* Power Indicator Control */
-#define  PCI_EXP_SLTCTL_PCC	0x0400	/* Power Controller Control */
-#define  PCI_EXP_SLTCTL_EIC	0x0800	/* Electromechanical Interlock Control */
-#define  PCI_EXP_SLTCTL_DLLSCE	0x1000	/* Data Link Layer State Changed Enable */
-#define PCI_EXP_SLTSTA		26	/* Slot Status */
-#define  PCI_EXP_SLTSTA_ABP	0x0001	/* Attention Button Pressed */
-#define  PCI_EXP_SLTSTA_PFD	0x0002	/* Power Fault Detected */
-#define  PCI_EXP_SLTSTA_MRLSC	0x0004	/* MRL Sensor Changed */
-#define  PCI_EXP_SLTSTA_PDC	0x0008	/* Presence Detect Changed */
-#define  PCI_EXP_SLTSTA_CC	0x0010	/* Command Completed */
-#define  PCI_EXP_SLTSTA_MRLSS	0x0020	/* MRL Sensor State */
-#define  PCI_EXP_SLTSTA_PDS	0x0040	/* Presence Detect State */
-#define  PCI_EXP_SLTSTA_EIS	0x0080	/* Electromechanical Interlock Status */
-#define  PCI_EXP_SLTSTA_DLLSC	0x0100	/* Data Link Layer State Changed */
-#define PCI_EXP_RTCTL		28	/* Root Control */
-#define  PCI_EXP_RTCTL_SECEE	0x01	/* System Error on Correctable Error */
-#define  PCI_EXP_RTCTL_SENFEE	0x02	/* System Error on Non-Fatal Error */
-#define  PCI_EXP_RTCTL_SEFEE	0x04	/* System Error on Fatal Error */
-#define  PCI_EXP_RTCTL_PMEIE	0x08	/* PME Interrupt Enable */
-#define  PCI_EXP_RTCTL_CRSSVE	0x10	/* CRS Software Visibility Enable */
-#define PCI_EXP_RTCAP		30	/* Root Capabilities */
-#define PCI_EXP_RTSTA		32	/* Root Status */
-#define PCI_EXP_RTSTA_PME	0x10000 /* PME status */
-#define PCI_EXP_RTSTA_PENDING	0x20000 /* PME pending */
-#define PCI_EXP_DEVCAP2		36	/* Device Capabilities 2 */
-#define  PCI_EXP_DEVCAP2_ARI	0x20	/* Alternative Routing-ID */
-#define  PCI_EXP_DEVCAP2_LTR	0x800	/* Latency tolerance reporting */
-#define  PCI_EXP_OBFF_MASK	0xc0000 /* OBFF support mechanism */
-#define  PCI_EXP_OBFF_MSG	0x40000 /* New message signaling */
-#define  PCI_EXP_OBFF_WAKE	0x80000 /* Re-use WAKE# for OBFF */
-#define PCI_EXP_DEVCTL2		40	/* Device Control 2 */
-#define  PCI_EXP_DEVCTL2_ARI	0x20	/* Alternative Routing-ID */
-#define  PCI_EXP_IDO_REQ_EN	0x100	/* ID-based ordering request enable */
-#define  PCI_EXP_IDO_CMP_EN	0x200	/* ID-based ordering completion enable */
-#define  PCI_EXP_LTR_EN		0x400	/* Latency tolerance reporting */
-#define  PCI_EXP_OBFF_MSGA_EN	0x2000	/* OBFF enable with Message type A */
-#define  PCI_EXP_OBFF_MSGB_EN	0x4000	/* OBFF enable with Message type B */
-#define  PCI_EXP_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
-#define PCI_EXP_LNKCTL2		48	/* Link Control 2 */
-#define PCI_EXP_SLTCTL2		56	/* Slot Control 2 */
-
-/* Extended Capabilities (PCI-X 2.0 and Express) */
-#define PCI_EXT_CAP_ID(header)		(header & 0x0000ffff)
-#define PCI_EXT_CAP_VER(header)		((header >> 16) & 0xf)
-#define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
-
-#define PCI_EXT_CAP_ID_ERR	1
-#define PCI_EXT_CAP_ID_VC	2
-#define PCI_EXT_CAP_ID_DSN	3
-#define PCI_EXT_CAP_ID_PWR	4
-#define PCI_EXT_CAP_ID_VNDR	11
-#define PCI_EXT_CAP_ID_ACS	13
-#define PCI_EXT_CAP_ID_ARI	14
-#define PCI_EXT_CAP_ID_ATS	15
-#define PCI_EXT_CAP_ID_SRIOV	16
-#define PCI_EXT_CAP_ID_LTR	24
-
-/* Advanced Error Reporting */
-#define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
-#define  PCI_ERR_UNC_TRAIN	0x00000001	/* Training */
-#define  PCI_ERR_UNC_DLP	0x00000010	/* Data Link Protocol */
-#define  PCI_ERR_UNC_POISON_TLP	0x00001000	/* Poisoned TLP */
-#define  PCI_ERR_UNC_FCP	0x00002000	/* Flow Control Protocol */
-#define  PCI_ERR_UNC_COMP_TIME	0x00004000	/* Completion Timeout */
-#define  PCI_ERR_UNC_COMP_ABORT	0x00008000	/* Completer Abort */
-#define  PCI_ERR_UNC_UNX_COMP	0x00010000	/* Unexpected Completion */
-#define  PCI_ERR_UNC_RX_OVER	0x00020000	/* Receiver Overflow */
-#define  PCI_ERR_UNC_MALF_TLP	0x00040000	/* Malformed TLP */
-#define  PCI_ERR_UNC_ECRC	0x00080000	/* ECRC Error Status */
-#define  PCI_ERR_UNC_UNSUP	0x00100000	/* Unsupported Request */
-#define PCI_ERR_UNCOR_MASK	8	/* Uncorrectable Error Mask */
-	/* Same bits as above */
-#define PCI_ERR_UNCOR_SEVER	12	/* Uncorrectable Error Severity */
-	/* Same bits as above */
-#define PCI_ERR_COR_STATUS	16	/* Correctable Error Status */
-#define  PCI_ERR_COR_RCVR	0x00000001	/* Receiver Error Status */
-#define  PCI_ERR_COR_BAD_TLP	0x00000040	/* Bad TLP Status */
-#define  PCI_ERR_COR_BAD_DLLP	0x00000080	/* Bad DLLP Status */
-#define  PCI_ERR_COR_REP_ROLL	0x00000100	/* REPLAY_NUM Rollover */
-#define  PCI_ERR_COR_REP_TIMER	0x00001000	/* Replay Timer Timeout */
-#define PCI_ERR_COR_MASK	20	/* Correctable Error Mask */
-	/* Same bits as above */
-#define PCI_ERR_CAP		24	/* Advanced Error Capabilities */
-#define  PCI_ERR_CAP_FEP(x)	((x) & 31)	/* First Error Pointer */
-#define  PCI_ERR_CAP_ECRC_GENC	0x00000020	/* ECRC Generation Capable */
-#define  PCI_ERR_CAP_ECRC_GENE	0x00000040	/* ECRC Generation Enable */
-#define  PCI_ERR_CAP_ECRC_CHKC	0x00000080	/* ECRC Check Capable */
-#define  PCI_ERR_CAP_ECRC_CHKE	0x00000100	/* ECRC Check Enable */
-#define PCI_ERR_HEADER_LOG	28	/* Header Log Register (16 bytes) */
-#define PCI_ERR_ROOT_COMMAND	44	/* Root Error Command */
-/* Correctable Err Reporting Enable */
-#define PCI_ERR_ROOT_CMD_COR_EN		0x00000001
-/* Non-fatal Err Reporting Enable */
-#define PCI_ERR_ROOT_CMD_NONFATAL_EN	0x00000002
-/* Fatal Err Reporting Enable */
-#define PCI_ERR_ROOT_CMD_FATAL_EN	0x00000004
-#define PCI_ERR_ROOT_STATUS	48
-#define PCI_ERR_ROOT_COR_RCV		0x00000001	/* ERR_COR Received */
-/* Multi ERR_COR Received */
-#define PCI_ERR_ROOT_MULTI_COR_RCV	0x00000002
-/* ERR_FATAL/NONFATAL Recevied */
-#define PCI_ERR_ROOT_UNCOR_RCV		0x00000004
-/* Multi ERR_FATAL/NONFATAL Recevied */
-#define PCI_ERR_ROOT_MULTI_UNCOR_RCV	0x00000008
-#define PCI_ERR_ROOT_FIRST_FATAL	0x00000010	/* First Fatal */
-#define PCI_ERR_ROOT_NONFATAL_RCV	0x00000020	/* Non-Fatal Received */
-#define PCI_ERR_ROOT_FATAL_RCV		0x00000040	/* Fatal Received */
-#define PCI_ERR_ROOT_ERR_SRC	52	/* Error Source Identification */
-
-/* Virtual Channel */
-#define PCI_VC_PORT_REG1	4
-#define PCI_VC_PORT_REG2	8
-#define PCI_VC_PORT_CTRL	12
-#define PCI_VC_PORT_STATUS	14
-#define PCI_VC_RES_CAP		16
-#define PCI_VC_RES_CTRL		20
-#define PCI_VC_RES_STATUS	26
-
-/* Power Budgeting */
-#define PCI_PWR_DSR		4	/* Data Select Register */
-#define PCI_PWR_DATA		8	/* Data Register */
-#define  PCI_PWR_DATA_BASE(x)	((x) & 0xff)	    /* Base Power */
-#define  PCI_PWR_DATA_SCALE(x)	(((x) >> 8) & 3)    /* Data Scale */
-#define  PCI_PWR_DATA_PM_SUB(x)	(((x) >> 10) & 7)   /* PM Sub State */
-#define  PCI_PWR_DATA_PM_STATE(x) (((x) >> 13) & 3) /* PM State */
-#define  PCI_PWR_DATA_TYPE(x)	(((x) >> 15) & 7)   /* Type */
-#define  PCI_PWR_DATA_RAIL(x)	(((x) >> 18) & 7)   /* Power Rail */
-#define PCI_PWR_CAP		12	/* Capability */
-#define  PCI_PWR_CAP_BUDGET(x)	((x) & 1)	/* Included in system budget */
-
-/*
- * Hypertransport sub capability types
- *
- * Unfortunately there are both 3 bit and 5 bit capability types defined
- * in the HT spec, catering for that is a little messy. You probably don't
- * want to use these directly, just use pci_find_ht_capability() and it
- * will do the right thing for you.
- */
-#define HT_3BIT_CAP_MASK	0xE0
-#define HT_CAPTYPE_SLAVE	0x00	/* Slave/Primary link configuration */
-#define HT_CAPTYPE_HOST		0x20	/* Host/Secondary link configuration */
-
-#define HT_5BIT_CAP_MASK	0xF8
-#define HT_CAPTYPE_IRQ		0x80	/* IRQ Configuration */
-#define HT_CAPTYPE_REMAPPING_40	0xA0	/* 40 bit address remapping */
-#define HT_CAPTYPE_REMAPPING_64 0xA2	/* 64 bit address remapping */
-#define HT_CAPTYPE_UNITID_CLUMP	0x90	/* Unit ID clumping */
-#define HT_CAPTYPE_EXTCONF	0x98	/* Extended Configuration Space Access */
-#define HT_CAPTYPE_MSI_MAPPING	0xA8	/* MSI Mapping Capability */
-#define  HT_MSI_FLAGS		0x02		/* Offset to flags */
-#define  HT_MSI_FLAGS_ENABLE	0x1		/* Mapping enable */
-#define  HT_MSI_FLAGS_FIXED	0x2		/* Fixed mapping only */
-#define  HT_MSI_FIXED_ADDR	0x00000000FEE00000ULL	/* Fixed addr */
-#define  HT_MSI_ADDR_LO		0x04		/* Offset to low addr bits */
-#define  HT_MSI_ADDR_LO_MASK	0xFFF00000	/* Low address bit mask */
-#define  HT_MSI_ADDR_HI		0x08		/* Offset to high addr bits */
-#define HT_CAPTYPE_DIRECT_ROUTE	0xB0	/* Direct routing configuration */
-#define HT_CAPTYPE_VCSET	0xB8	/* Virtual Channel configuration */
-#define HT_CAPTYPE_ERROR_RETRY	0xC0	/* Retry on error configuration */
-#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 hypertransport configuration */
-#define HT_CAPTYPE_PM		0xE0	/* Hypertransport powermanagement configuration */
-
-/* Alternative Routing-ID Interpretation */
-#define PCI_ARI_CAP		0x04	/* ARI Capability Register */
-#define  PCI_ARI_CAP_MFVC	0x0001	/* MFVC Function Groups Capability */
-#define  PCI_ARI_CAP_ACS	0x0002	/* ACS Function Groups Capability */
-#define  PCI_ARI_CAP_NFN(x)	(((x) >> 8) & 0xff) /* Next Function Number */
-#define PCI_ARI_CTRL		0x06	/* ARI Control Register */
-#define  PCI_ARI_CTRL_MFVC	0x0001	/* MFVC Function Groups Enable */
-#define  PCI_ARI_CTRL_ACS	0x0002	/* ACS Function Groups Enable */
-#define  PCI_ARI_CTRL_FG(x)	(((x) >> 4) & 7) /* Function Group */
-
-/* Address Translation Service */
-#define PCI_ATS_CAP		0x04	/* ATS Capability Register */
-#define  PCI_ATS_CAP_QDEP(x)	((x) & 0x1f)	/* Invalidate Queue Depth */
-#define  PCI_ATS_MAX_QDEP	32	/* Max Invalidate Queue Depth */
-#define PCI_ATS_CTRL		0x06	/* ATS Control Register */
-#define  PCI_ATS_CTRL_ENABLE	0x8000	/* ATS Enable */
-#define  PCI_ATS_CTRL_STU(x)	((x) & 0x1f)	/* Smallest Translation Unit */
-#define  PCI_ATS_MIN_STU	12	/* shift of minimum STU block */
-
-/* Single Root I/O Virtualization */
-#define PCI_SRIOV_CAP		0x04	/* SR-IOV Capabilities */
-#define  PCI_SRIOV_CAP_VFM	0x01	/* VF Migration Capable */
-#define  PCI_SRIOV_CAP_INTR(x)	((x) >> 21) /* Interrupt Message Number */
-#define PCI_SRIOV_CTRL		0x08	/* SR-IOV Control */
-#define  PCI_SRIOV_CTRL_VFE	0x01	/* VF Enable */
-#define  PCI_SRIOV_CTRL_VFM	0x02	/* VF Migration Enable */
-#define  PCI_SRIOV_CTRL_INTR	0x04	/* VF Migration Interrupt Enable */
-#define  PCI_SRIOV_CTRL_MSE	0x08	/* VF Memory Space Enable */
-#define  PCI_SRIOV_CTRL_ARI	0x10	/* ARI Capable Hierarchy */
-#define PCI_SRIOV_STATUS	0x0a	/* SR-IOV Status */
-#define  PCI_SRIOV_STATUS_VFM	0x01	/* VF Migration Status */
-#define PCI_SRIOV_INITIAL_VF	0x0c	/* Initial VFs */
-#define PCI_SRIOV_TOTAL_VF	0x0e	/* Total VFs */
-#define PCI_SRIOV_NUM_VF	0x10	/* Number of VFs */
-#define PCI_SRIOV_FUNC_LINK	0x12	/* Function Dependency Link */
-#define PCI_SRIOV_VF_OFFSET	0x14	/* First VF Offset */
-#define PCI_SRIOV_VF_STRIDE	0x16	/* Following VF Stride */
-#define PCI_SRIOV_VF_DID	0x1a	/* VF Device ID */
-#define PCI_SRIOV_SUP_PGSIZE	0x1c	/* Supported Page Sizes */
-#define PCI_SRIOV_SYS_PGSIZE	0x20	/* System Page Size */
-#define PCI_SRIOV_BAR		0x24	/* VF BAR0 */
-#define  PCI_SRIOV_NUM_BARS	6	/* Number of VF BARs */
-#define PCI_SRIOV_VFM		0x3c	/* VF Migration State Array Offset*/
-#define  PCI_SRIOV_VFM_BIR(x)	((x) & 7)	/* State BIR */
-#define  PCI_SRIOV_VFM_OFFSET(x) ((x) & ~7)	/* State Offset */
-#define  PCI_SRIOV_VFM_UA	0x0	/* Inactive.Unavailable */
-#define  PCI_SRIOV_VFM_MI	0x1	/* Dormant.MigrateIn */
-#define  PCI_SRIOV_VFM_MO	0x2	/* Active.MigrateOut */
-#define  PCI_SRIOV_VFM_AV	0x3	/* Active.Available */
-
-#define PCI_LTR_MAX_SNOOP_LAT	0x4
-#define PCI_LTR_MAX_NOSNOOP_LAT	0x6
-#define  PCI_LTR_VALUE_MASK	0x000003ff
-#define  PCI_LTR_SCALE_MASK	0x00001c00
-#define  PCI_LTR_SCALE_SHIFT	10
-
-/* Access Control Service */
-#define PCI_ACS_CAP		0x04	/* ACS Capability Register */
-#define  PCI_ACS_SV		0x01	/* Source Validation */
-#define  PCI_ACS_TB		0x02	/* Translation Blocking */
-#define  PCI_ACS_RR		0x04	/* P2P Request Redirect */
-#define  PCI_ACS_CR		0x08	/* P2P Completion Redirect */
-#define  PCI_ACS_UF		0x10	/* Upstream Forwarding */
-#define  PCI_ACS_EC		0x20	/* P2P Egress Control */
-#define  PCI_ACS_DT		0x40	/* Direct Translated P2P */
-#define PCI_ACS_CTRL		0x06	/* ACS Control Register */
-#define PCI_ACS_EGRESS_CTL_V	0x08	/* ACS Egress Control Vector */
-
-#endif /* LINUX_PCI_REGS_H */
+#define  PCI_PM_CAP_VER_1_1     0x0002  /* PCI PM spec ver. 1.1 */
diff --git a/include/hw/pci/pcie.h b/include/hw/pci/pcie.h
index b48a7a2..056d25e 100644
--- a/include/hw/pci/pcie.h
+++ b/include/hw/pci/pcie.h
@@ -80,8 +80,12 @@
 
 /* PCI express capability helper functions */
 int pcie_cap_init(PCIDevice *dev, uint8_t offset, uint8_t type, uint8_t port);
+int pcie_cap_v1_init(PCIDevice *dev, uint8_t offset,
+                     uint8_t type, uint8_t port);
 int pcie_endpoint_cap_init(PCIDevice *dev, uint8_t offset);
 void pcie_cap_exit(PCIDevice *dev);
+int pcie_endpoint_cap_v1_init(PCIDevice *dev, uint8_t offset);
+void pcie_cap_v1_exit(PCIDevice *dev);
 uint8_t pcie_cap_get_type(const PCIDevice *dev);
 void pcie_cap_flags_set_vector(PCIDevice *dev, uint8_t vector);
 uint8_t pcie_cap_flags_get_vector(PCIDevice *dev);
@@ -115,6 +119,7 @@
                          uint16_t offset, uint16_t size);
 
 void pcie_ari_init(PCIDevice *dev, uint16_t offset, uint16_t nextfn);
+void pcie_dev_ser_num_init(PCIDevice *dev, uint16_t offset, uint64_t ser_num);
 
 extern const VMStateDescription vmstate_pcie_device;
 
diff --git a/include/hw/pci/pcie_aer.h b/include/hw/pci/pcie_aer.h
index bcac80a..c2ee4e2 100644
--- a/include/hw/pci/pcie_aer.h
+++ b/include/hw/pci/pcie_aer.h
@@ -51,7 +51,7 @@
     PCIEAERErr *log;
 };
 
-/* aer error message: error signaling message has only error sevirity and
+/* aer error message: error signaling message has only error severity and
    source id. See 2.2.8.3 error signaling messages */
 struct PCIEAERMsg {
     /*
@@ -87,7 +87,7 @@
 
 extern const VMStateDescription vmstate_pcie_aer_log;
 
-int pcie_aer_init(PCIDevice *dev, uint16_t offset);
+int pcie_aer_init(PCIDevice *dev, uint16_t offset, uint16_t size);
 void pcie_aer_exit(PCIDevice *dev);
 void pcie_aer_write_config(PCIDevice *dev,
                            uint32_t addr, uint32_t val, int len);
@@ -102,5 +102,6 @@
 
 /* error injection */
 int pcie_aer_inject_error(PCIDevice *dev, const PCIEAERErr *err);
+void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg);
 
 #endif /* QEMU_PCIE_AER_H */
diff --git a/include/hw/pci/pcie_host.h b/include/hw/pci/pcie_host.h
index ff44ef6..4d23c80 100644
--- a/include/hw/pci/pcie_host.h
+++ b/include/hw/pci/pcie_host.h
@@ -50,6 +50,7 @@
 };
 
 void pcie_host_mmcfg_unmap(PCIExpressHost *e);
+void pcie_host_mmcfg_init(PCIExpressHost *e, uint32_t size);
 void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr, uint32_t size);
 void pcie_host_mmcfg_update(PCIExpressHost *e,
                             int enable,
diff --git a/include/hw/pci/pcie_regs.h b/include/hw/pci/pcie_regs.h
index 652d9fc..a95522a 100644
--- a/include/hw/pci/pcie_regs.h
+++ b/include/hw/pci/pcie_regs.h
@@ -11,6 +11,7 @@
 
 /* express capability */
 
+#define PCI_EXP_VER1_SIZEOF             0x14 /* express capability of ver. 1 */
 #define PCI_EXP_VER2_SIZEOF             0x3c /* express capability of ver. 2 */
 #define PCI_EXT_CAP_VER_SHIFT           16
 #define PCI_EXT_CAP_NEXT_SHIFT          20
@@ -26,35 +27,35 @@
     (((x) + PCI_EXT_CAP_ALIGN - 1) & ~(PCI_EXT_CAP_ALIGN - 1))
 
 /* PCI_EXP_FLAGS */
-#define PCI_EXP_FLAGS_VER2              2 /* for now, supports only ver. 2 */
-#define PCI_EXP_FLAGS_IRQ_SHIFT         (ffs(PCI_EXP_FLAGS_IRQ) - 1)
-#define PCI_EXP_FLAGS_TYPE_SHIFT        (ffs(PCI_EXP_FLAGS_TYPE) - 1)
-
+#define PCI_EXP_FLAGS_VER1              1
+#define PCI_EXP_FLAGS_VER2              2
+#define PCI_EXP_FLAGS_IRQ_SHIFT         ctz32(PCI_EXP_FLAGS_IRQ)
+#define PCI_EXP_FLAGS_TYPE_SHIFT        ctz32(PCI_EXP_FLAGS_TYPE)
 
 /* PCI_EXP_LINK{CAP, STA} */
 /* link speed */
 #define PCI_EXP_LNK_LS_25               1
 
-#define PCI_EXP_LNK_MLW_SHIFT           (ffs(PCI_EXP_LNKCAP_MLW) - 1)
+#define PCI_EXP_LNK_MLW_SHIFT           ctz32(PCI_EXP_LNKCAP_MLW)
 #define PCI_EXP_LNK_MLW_1               (1 << PCI_EXP_LNK_MLW_SHIFT)
 
 /* PCI_EXP_LINKCAP */
-#define PCI_EXP_LNKCAP_ASPMS_SHIFT      (ffs(PCI_EXP_LNKCAP_ASPMS) - 1)
+#define PCI_EXP_LNKCAP_ASPMS_SHIFT      ctz32(PCI_EXP_LNKCAP_ASPMS)
 #define PCI_EXP_LNKCAP_ASPMS_0S         (1 << PCI_EXP_LNKCAP_ASPMS_SHIFT)
 
-#define PCI_EXP_LNKCAP_PN_SHIFT         (ffs(PCI_EXP_LNKCAP_PN) - 1)
+#define PCI_EXP_LNKCAP_PN_SHIFT         ctz32(PCI_EXP_LNKCAP_PN)
 
-#define PCI_EXP_SLTCAP_PSN_SHIFT        (ffs(PCI_EXP_SLTCAP_PSN) - 1)
+#define PCI_EXP_SLTCAP_PSN_SHIFT        ctz32(PCI_EXP_SLTCAP_PSN)
 
 #define PCI_EXP_SLTCTL_IND_RESERVED     0x0
 #define PCI_EXP_SLTCTL_IND_ON           0x1
 #define PCI_EXP_SLTCTL_IND_BLINK        0x2
 #define PCI_EXP_SLTCTL_IND_OFF          0x3
-#define PCI_EXP_SLTCTL_AIC_SHIFT        (ffs(PCI_EXP_SLTCTL_AIC) - 1)
+#define PCI_EXP_SLTCTL_AIC_SHIFT        ctz32(PCI_EXP_SLTCTL_AIC)
 #define PCI_EXP_SLTCTL_AIC_OFF                          \
     (PCI_EXP_SLTCTL_IND_OFF << PCI_EXP_SLTCTL_AIC_SHIFT)
 
-#define PCI_EXP_SLTCTL_PIC_SHIFT        (ffs(PCI_EXP_SLTCTL_PIC) - 1)
+#define PCI_EXP_SLTCTL_PIC_SHIFT        ctz32(PCI_EXP_SLTCTL_PIC)
 #define PCI_EXP_SLTCTL_PIC_OFF                          \
     (PCI_EXP_SLTCTL_IND_OFF << PCI_EXP_SLTCTL_PIC_SHIFT)
 #define PCI_EXP_SLTCTL_PIC_ON                          \
@@ -72,7 +73,7 @@
 #define PCI_EXP_DEVCAP2_EFF             0x100000
 #define PCI_EXP_DEVCAP2_EETLPP          0x200000
 
-#define PCI_EXP_DEVCTL2_EETLPPB         0x80
+#define PCI_EXP_DEVCTL2_EETLPPB         0x8000
 
 /* ARI */
 #define PCI_ARI_VER                     1
@@ -109,7 +110,7 @@
 
 #define PCI_ERR_ROOT_IRQ_MAX            32
 #define PCI_ERR_ROOT_IRQ                0xf8000000
-#define PCI_ERR_ROOT_IRQ_SHIFT          (ffs(PCI_ERR_ROOT_IRQ) - 1)
+#define PCI_ERR_ROOT_IRQ_SHIFT          ctz32(PCI_ERR_ROOT_IRQ)
 #define PCI_ERR_ROOT_STATUS_REPORT_MASK (PCI_ERR_ROOT_COR_RCV |         \
                                          PCI_ERR_ROOT_MULTI_COR_RCV |   \
                                          PCI_ERR_ROOT_UNCOR_RCV |       \
diff --git a/include/hw/pci/shpc.h b/include/hw/pci/shpc.h
index 025bc5b..b208554 100644
--- a/include/hw/pci/shpc.h
+++ b/include/hw/pci/shpc.h
@@ -4,8 +4,8 @@
 #include "qemu-common.h"
 #include "exec/memory.h"
 #include "migration/vmstate.h"
-#include "qapi/error.h"
 #include "hw/hotplug.h"
+#include "hw/pci/pci.h"
 
 struct SHPCDevice {
     /* Capability offset in device's config space */
@@ -41,6 +41,7 @@
 int shpc_bar_size(PCIDevice *dev);
 int shpc_init(PCIDevice *dev, PCIBus *sec_bus, MemoryRegion *bar, unsigned off);
 void shpc_cleanup(PCIDevice *dev, MemoryRegion *bar);
+void shpc_free(PCIDevice *dev);
 void shpc_cap_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int len);
 
 
@@ -50,7 +51,13 @@
                                        DeviceState *dev, Error **errp);
 
 extern VMStateInfo shpc_vmstate_info;
-#define SHPC_VMSTATE(_field, _type) \
-    VMSTATE_BUFFER_UNSAFE_INFO(_field, _type, 0, shpc_vmstate_info, 0)
+#define SHPC_VMSTATE(_field, _type,  _test) \
+    VMSTATE_BUFFER_UNSAFE_INFO_TEST(_field, _type, _test, 0, \
+                                    shpc_vmstate_info, 0)
+
+static inline bool shpc_present(const PCIDevice *dev)
+{
+    return dev->cap_present & QEMU_PCI_CAP_SHPC;
+}
 
 #endif
diff --git a/include/hw/pcmcia.h b/include/hw/pcmcia.h
index 98406ff..79cac9c 100644
--- a/include/hw/pcmcia.h
+++ b/include/hw/pcmcia.h
@@ -1,5 +1,5 @@
 #ifndef HW_PCMCIA_H
-#define HW_PCMCIA_H 1
+#define HW_PCMCIA_H
 
 /* PCMCIA/Cardbus */
 
diff --git a/include/hw/platform-bus.h b/include/hw/platform-bus.h
index bd42b83..a00775c 100644
--- a/include/hw/platform-bus.h
+++ b/include/hw/platform-bus.h
@@ -1,5 +1,5 @@
 #ifndef HW_PLATFORM_BUS_H
-#define HW_PLATFORM_BUS_H 1
+#define HW_PLATFORM_BUS_H
 
 /*
  *  Platform Bus device to support dynamic Sysbus devices
@@ -54,4 +54,4 @@
 hwaddr platform_bus_get_mmio_addr(PlatformBusDevice *pbus, SysBusDevice *sbdev,
                                   int n);
 
-#endif /* !HW_PLATFORM_BUS_H */
+#endif /* HW_PLATFORM_BUS_H */
diff --git a/include/hw/ppc/mac_dbdma.h b/include/hw/ppc/mac_dbdma.h
index d7db06c..a860387 100644
--- a/include/hw/ppc/mac_dbdma.h
+++ b/include/hw/ppc/mac_dbdma.h
@@ -19,10 +19,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef HW_MAC_DBDMA_H
-#define HW_MAC_DBDMA_H 1
+#define HW_MAC_DBDMA_H
 
 #include "exec/memory.h"
+#include "qemu/iov.h"
+#include "sysemu/dma.h"
 
 typedef struct DBDMA_io DBDMA_io;
 
@@ -40,13 +43,13 @@
     /* DMA is in progress, don't start another one */
     bool processing;
     /* unaligned last sector of a request */
-    uint8_t remainder[0x200];
-    int remainder_len;
+    uint8_t head_remainder[0x200];
+    uint8_t tail_remainder[0x200];
     QEMUIOVector iov;
-    bool finish_remain_read;
-    hwaddr finish_addr;
-    hwaddr finish_len;
-    int requests;
+    /* DMA request */
+    void *dma_mem;
+    dma_addr_t dma_len;
+    DMADirection dir;
 };
 
 /*
diff --git a/include/hw/ppc/openpic.h b/include/hw/ppc/openpic.h
index ee67098..6137e2d 100644
--- a/include/hw/ppc/openpic.h
+++ b/include/hw/ppc/openpic.h
@@ -1,8 +1,9 @@
-#if !defined(__OPENPIC_H__)
-#define __OPENPIC_H__
+#ifndef OPENPIC_H
+#define OPENPIC_H
 
 #include "qemu-common.h"
-#include "hw/qdev.h"
+#include "hw/qdev-core.h"
+#include "qom/cpu.h"
 
 #define TYPE_OPENPIC "openpic"
 
@@ -29,4 +30,4 @@
 #define TYPE_KVM_OPENPIC "kvm-openpic"
 int kvm_openpic_connect_vcpu(DeviceState *d, CPUState *cs);
 
-#endif /* __OPENPIC_H__ */
+#endif /* OPENPIC_H */
diff --git a/include/hw/ppc/ppc.h b/include/hw/ppc/ppc.h
index 14efd0c..00c1fb1 100644
--- a/include/hw/ppc/ppc.h
+++ b/include/hw/ppc/ppc.h
@@ -1,5 +1,7 @@
 #ifndef HW_PPC_H
-#define HW_PPC_H 1
+#define HW_PPC_H
+
+#include "target-ppc/cpu-qom.h"
 
 void ppc_set_irq(PowerPCCPU *cpu, int n_IRQ, int level);
 
@@ -64,17 +66,21 @@
 void ppc40x_core_reset(PowerPCCPU *cpu);
 void ppc40x_chip_reset(PowerPCCPU *cpu);
 void ppc40x_system_reset(PowerPCCPU *cpu);
-void PREP_debug_write (void *opaque, uint32_t addr, uint32_t val);
-
-extern CPUWriteMemoryFunc * const PPC_io_write[];
-extern CPUReadMemoryFunc * const PPC_io_read[];
 void PPC_debug_write (void *opaque, uint32_t addr, uint32_t val);
 
-void ppc40x_irq_init (CPUPPCState *env);
-void ppce500_irq_init (CPUPPCState *env);
-void ppc6xx_irq_init (CPUPPCState *env);
-void ppc970_irq_init (CPUPPCState *env);
-void ppcPOWER7_irq_init (CPUPPCState *env);
+#if defined(CONFIG_USER_ONLY)
+static inline void ppc40x_irq_init(PowerPCCPU *cpu) {}
+static inline void ppc6xx_irq_init(PowerPCCPU *cpu) {}
+static inline void ppc970_irq_init(PowerPCCPU *cpu) {}
+static inline void ppcPOWER7_irq_init(PowerPCCPU *cpu) {}
+static inline void ppce500_irq_init(PowerPCCPU *cpu) {}
+#else
+void ppc40x_irq_init(PowerPCCPU *cpu);
+void ppce500_irq_init(PowerPCCPU *cpu);
+void ppc6xx_irq_init(PowerPCCPU *cpu);
+void ppc970_irq_init(PowerPCCPU *cpu);
+void ppcPOWER7_irq_init(PowerPCCPU *cpu);
+#endif
 
 /* PPC machines for OpenBIOS */
 enum {
@@ -100,4 +106,5 @@
 /* ppc_booke.c */
 void ppc_booke_timers_init(PowerPCCPU *cpu, uint32_t freq, uint32_t flags);
 
+void ppc_cpu_parse_features(const char *cpu_model);
 #endif
diff --git a/include/hw/ppc/ppc4xx.h b/include/hw/ppc/ppc4xx.h
index 91d84ba..3b01ae8 100644
--- a/include/hw/ppc/ppc4xx.h
+++ b/include/hw/ppc/ppc4xx.h
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#if !defined(PPC_4XX_H)
-#define PPC_4XX_H
+#ifndef PPC4XX_H
+#define PPC4XX_H
 
 #include "hw/pci/pci.h"
 
@@ -61,4 +61,4 @@
                         hwaddr special_cycle,
                         hwaddr registers);
 
-#endif /* !defined(PPC_4XX_H) */
+#endif /* PPC4XX_H */
diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index 749daf4..caf7be9 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -1,22 +1,59 @@
-#if !defined(__HW_SPAPR_H__)
-#define __HW_SPAPR_H__
+#ifndef HW_SPAPR_H
+#define HW_SPAPR_H
 
 #include "sysemu/dma.h"
+#include "hw/boards.h"
 #include "hw/ppc/xics.h"
+#include "hw/ppc/spapr_drc.h"
+#include "hw/mem/pc-dimm.h"
 
 struct VIOsPAPRBus;
 struct sPAPRPHBState;
 struct sPAPRNVRAM;
+typedef struct sPAPRConfigureConnectorState sPAPRConfigureConnectorState;
+typedef struct sPAPREventLogEntry sPAPREventLogEntry;
 
 #define HPTE64_V_HPTE_DIRTY     0x0000000000000040ULL
+#define SPAPR_ENTRY_POINT       0x100
 
-typedef struct sPAPREnvironment {
+#define SPAPR_TIMEBASE_FREQ     512000000ULL
+
+typedef struct sPAPRMachineClass sPAPRMachineClass;
+typedef struct sPAPRMachineState sPAPRMachineState;
+
+#define TYPE_SPAPR_MACHINE      "spapr-machine"
+#define SPAPR_MACHINE(obj) \
+    OBJECT_CHECK(sPAPRMachineState, (obj), TYPE_SPAPR_MACHINE)
+#define SPAPR_MACHINE_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(sPAPRMachineClass, obj, TYPE_SPAPR_MACHINE)
+#define SPAPR_MACHINE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(sPAPRMachineClass, klass, TYPE_SPAPR_MACHINE)
+
+/**
+ * sPAPRMachineClass:
+ */
+struct sPAPRMachineClass {
+    /*< private >*/
+    MachineClass parent_class;
+
+    /*< public >*/
+    bool dr_lmb_enabled;       /* enable dynamic-reconfig/hotplug of LMBs */
+    bool use_ohci_by_default;  /* use USB-OHCI instead of XHCI */
+};
+
+/**
+ * sPAPRMachineState:
+ */
+struct sPAPRMachineState {
+    /*< private >*/
+    MachineState parent_obj;
+
     struct VIOsPAPRBus *vio_bus;
     QLIST_HEAD(, sPAPRPHBState) phbs;
     struct sPAPRNVRAM *nvram;
-    XICSState *icp;
+    XICSState *xics;
+    DeviceState *rtc;
 
-    hwaddr ram_limit;
     void *htab;
     uint32_t htab_shift;
     hwaddr rma_size;
@@ -25,19 +62,27 @@
     ssize_t rtas_size;
     void *rtas_blob;
     void *fdt_skel;
-    target_ulong entry_point;
-    uint64_t rtc_offset;
+    uint64_t rtc_offset; /* Now used only during incoming migration */
     struct PPCTimebase tb;
     bool has_graphics;
 
-    uint32_t epow_irq;
+    uint32_t check_exception_irq;
     Notifier epow_notifier;
+    QTAILQ_HEAD(, sPAPREventLogEntry) pending_events;
 
     /* Migration state */
     int htab_save_index;
     bool htab_first_pass;
     int htab_fd;
-} sPAPREnvironment;
+
+    /* RTAS state */
+    QTAILQ_HEAD(, sPAPRConfigureConnectorState) ccs_list;
+
+    /*< public >*/
+    char *kvm_type;
+    MemoryHotplugState hotplug_memory;
+    Object **cores;
+};
 
 #define H_SUCCESS         0
 #define H_BUSY            1        /* Hardware busy -- retry later */
@@ -162,11 +207,6 @@
 #define H_SET_MODE_ENDIAN_BIG    0
 #define H_SET_MODE_ENDIAN_LITTLE 1
 
-/* Flags for H_SET_MODE_RESOURCE_ADDR_TRANS_MODE */
-#define H_SET_MODE_ADDR_TRANS_NONE                  0
-#define H_SET_MODE_ADDR_TRANS_0001_8000             2
-#define H_SET_MODE_ADDR_TRANS_C000_0000_0000_4000   3
-
 /* VASI States */
 #define H_VASI_INVALID    0
 #define H_VASI_ENABLED    1
@@ -292,6 +332,7 @@
 #define H_SET_MPP               0x2D0
 #define H_GET_MPP               0x2D4
 #define H_XIRR_X                0x2FC
+#define H_RANDOM                0x300
 #define H_SET_MODE              0x31C
 #define MAX_HCALL_OPCODE        H_SET_MODE
 
@@ -310,23 +351,16 @@
 #define KVMPPC_H_CAS            (KVMPPC_HCALL_BASE + 0x2)
 #define KVMPPC_HCALL_MAX        KVMPPC_H_CAS
 
-extern sPAPREnvironment *spapr;
-
 typedef struct sPAPRDeviceTreeUpdateHeader {
     uint32_t version_id;
 } sPAPRDeviceTreeUpdateHeader;
 
-/*#define DEBUG_SPAPR_HCALLS*/
-
-#ifdef DEBUG_SPAPR_HCALLS
 #define hcall_dprintf(fmt, ...) \
-    do { fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); } while (0)
-#else
-#define hcall_dprintf(fmt, ...) \
-    do { } while (0)
-#endif
+    do { \
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: " fmt, __func__, ## __VA_ARGS__); \
+    } while (0)
 
-typedef target_ulong (*spapr_hcall_fn)(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+typedef target_ulong (*spapr_hcall_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm,
                                        target_ulong opcode,
                                        target_ulong *args);
 
@@ -337,14 +371,59 @@
 int spapr_allocate_irq(int hint, bool lsi);
 int spapr_allocate_irq_block(int num, bool lsi, bool msi);
 
+/* ibm,set-eeh-option */
+#define RTAS_EEH_DISABLE                 0
+#define RTAS_EEH_ENABLE                  1
+#define RTAS_EEH_THAW_IO                 2
+#define RTAS_EEH_THAW_DMA                3
+
+/* ibm,get-config-addr-info2 */
+#define RTAS_GET_PE_ADDR                 0
+#define RTAS_GET_PE_MODE                 1
+#define RTAS_PE_MODE_NONE                0
+#define RTAS_PE_MODE_NOT_SHARED          1
+#define RTAS_PE_MODE_SHARED              2
+
+/* ibm,read-slot-reset-state2 */
+#define RTAS_EEH_PE_STATE_NORMAL         0
+#define RTAS_EEH_PE_STATE_RESET          1
+#define RTAS_EEH_PE_STATE_STOPPED_IO_DMA 2
+#define RTAS_EEH_PE_STATE_STOPPED_DMA    4
+#define RTAS_EEH_PE_STATE_UNAVAIL        5
+#define RTAS_EEH_NOT_SUPPORT             0
+#define RTAS_EEH_SUPPORT                 1
+#define RTAS_EEH_PE_UNAVAIL_INFO         1000
+#define RTAS_EEH_PE_RECOVER_INFO         0
+
+/* ibm,set-slot-reset */
+#define RTAS_SLOT_RESET_DEACTIVATE       0
+#define RTAS_SLOT_RESET_HOT              1
+#define RTAS_SLOT_RESET_FUNDAMENTAL      3
+
+/* ibm,slot-error-detail */
+#define RTAS_SLOT_TEMP_ERR_LOG           1
+#define RTAS_SLOT_PERM_ERR_LOG           2
+
 /* RTAS return codes */
-#define RTAS_OUT_SUCCESS            0
-#define RTAS_OUT_NO_ERRORS_FOUND    1
-#define RTAS_OUT_HW_ERROR           -1
-#define RTAS_OUT_BUSY               -2
-#define RTAS_OUT_PARAM_ERROR        -3
-#define RTAS_OUT_NOT_SUPPORTED      -3
-#define RTAS_OUT_NOT_AUTHORIZED     -9002
+#define RTAS_OUT_SUCCESS                        0
+#define RTAS_OUT_NO_ERRORS_FOUND                1
+#define RTAS_OUT_HW_ERROR                       -1
+#define RTAS_OUT_BUSY                           -2
+#define RTAS_OUT_PARAM_ERROR                    -3
+#define RTAS_OUT_NOT_SUPPORTED                  -3
+#define RTAS_OUT_NO_SUCH_INDICATOR              -3
+#define RTAS_OUT_NOT_AUTHORIZED                 -9002
+#define RTAS_OUT_SYSPARM_PARAM_ERROR            -9999
+
+/* DDW pagesize mask values from ibm,query-pe-dma-window */
+#define RTAS_DDW_PGSIZE_4K       0x01
+#define RTAS_DDW_PGSIZE_64K      0x02
+#define RTAS_DDW_PGSIZE_16M      0x04
+#define RTAS_DDW_PGSIZE_32M      0x08
+#define RTAS_DDW_PGSIZE_64M      0x10
+#define RTAS_DDW_PGSIZE_128M     0x20
+#define RTAS_DDW_PGSIZE_256M     0x40
+#define RTAS_DDW_PGSIZE_16G      0x80
 
 /* RTAS tokens */
 #define RTAS_TOKEN_BASE      0x2000
@@ -381,14 +460,35 @@
 #define RTAS_GET_SENSOR_STATE                   (RTAS_TOKEN_BASE + 0x1D)
 #define RTAS_IBM_CONFIGURE_CONNECTOR            (RTAS_TOKEN_BASE + 0x1E)
 #define RTAS_IBM_OS_TERM                        (RTAS_TOKEN_BASE + 0x1F)
+#define RTAS_IBM_SET_EEH_OPTION                 (RTAS_TOKEN_BASE + 0x20)
+#define RTAS_IBM_GET_CONFIG_ADDR_INFO2          (RTAS_TOKEN_BASE + 0x21)
+#define RTAS_IBM_READ_SLOT_RESET_STATE2         (RTAS_TOKEN_BASE + 0x22)
+#define RTAS_IBM_SET_SLOT_RESET                 (RTAS_TOKEN_BASE + 0x23)
+#define RTAS_IBM_CONFIGURE_PE                   (RTAS_TOKEN_BASE + 0x24)
+#define RTAS_IBM_SLOT_ERROR_DETAIL              (RTAS_TOKEN_BASE + 0x25)
+#define RTAS_IBM_QUERY_PE_DMA_WINDOW            (RTAS_TOKEN_BASE + 0x26)
+#define RTAS_IBM_CREATE_PE_DMA_WINDOW           (RTAS_TOKEN_BASE + 0x27)
+#define RTAS_IBM_REMOVE_PE_DMA_WINDOW           (RTAS_TOKEN_BASE + 0x28)
+#define RTAS_IBM_RESET_PE_DMA_WINDOW            (RTAS_TOKEN_BASE + 0x29)
 
-#define RTAS_TOKEN_MAX                          (RTAS_TOKEN_BASE + 0x20)
+#define RTAS_TOKEN_MAX                          (RTAS_TOKEN_BASE + 0x2A)
 
 /* RTAS ibm,get-system-parameter token values */
 #define RTAS_SYSPARM_SPLPAR_CHARACTERISTICS      20
 #define RTAS_SYSPARM_DIAGNOSTICS_RUN_MODE        42
 #define RTAS_SYSPARM_UUID                        48
 
+/* RTAS indicator/sensor types
+ *
+ * as defined by PAPR+ 2.7 7.3.5.4, Table 41
+ *
+ * NOTE: currently only DR-related sensors are implemented here
+ */
+#define RTAS_SENSOR_TYPE_ISOLATION_STATE        9001
+#define RTAS_SENSOR_TYPE_DR                     9002
+#define RTAS_SENSOR_TYPE_ALLOCATION_STATE       9003
+#define RTAS_SENSOR_TYPE_ENTITY_SENSE RTAS_SENSOR_TYPE_ALLOCATION_STATE
+
 /* Possible values for the platform-processor-diagnostics-run-mode parameter
  * of the RTAS ibm,get-system-parameter call.
  */
@@ -407,30 +507,22 @@
     return ldl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n));
 }
 
+static inline uint64_t rtas_ldq(target_ulong phys, int n)
+{
+    return (uint64_t)rtas_ld(phys, n) << 32 | rtas_ld(phys, n + 1);
+}
+
 static inline void rtas_st(target_ulong phys, int n, uint32_t val)
 {
     stl_be_phys(&address_space_memory, ppc64_phys_to_real(phys + 4*n), val);
 }
 
-
-static inline void rtas_st_buffer(target_ulong phys, target_ulong phys_len,
-                                  uint8_t *buffer, uint16_t buffer_len)
-{
-    if (phys_len < 2) {
-        return;
-    }
-    stw_be_phys(&address_space_memory,
-                ppc64_phys_to_real(phys), buffer_len);
-    cpu_physical_memory_write(ppc64_phys_to_real(phys + 2),
-                              buffer, MIN(buffer_len, phys_len - 2));
-}
-
-typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+typedef void (*spapr_rtas_fn)(PowerPCCPU *cpu, sPAPRMachineState *sm,
                               uint32_t token,
                               uint32_t nargs, target_ulong args,
                               uint32_t nret, target_ulong rets);
 void spapr_rtas_register(int token, const char *name, spapr_rtas_fn fn);
-target_ulong spapr_rtas_call(PowerPCCPU *cpu, sPAPREnvironment *spapr,
+target_ulong spapr_rtas_call(PowerPCCPU *cpu, sPAPRMachineState *sm,
                              uint32_t token, uint32_t nargs, target_ulong args,
                              uint32_t nret, target_ulong rets);
 int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
@@ -441,10 +533,16 @@
 #define SPAPR_TCE_PAGE_MASK    (SPAPR_TCE_PAGE_SIZE - 1)
 
 #define SPAPR_VIO_BASE_LIOBN    0x00000000
-#define SPAPR_PCI_BASE_LIOBN    0x80000000
+#define SPAPR_VIO_LIOBN(reg)    (0x00000000 | (reg))
+#define SPAPR_PCI_LIOBN(phb_index, window_num) \
+    (0x80000000 | ((phb_index) << 8) | (window_num))
+#define SPAPR_IS_PCI_LIOBN(liobn)   (!!((liobn) & 0x80000000))
+#define SPAPR_PCI_DMA_WINDOW_NUM(liobn) ((liobn) & 0xff)
 
 #define RTAS_ERROR_LOG_MAX      2048
 
+#define RTAS_EVENT_SCAN_RATE    1
+
 typedef struct sPAPRTCETable sPAPRTCETable;
 
 #define TYPE_SPAPR_TCE_TABLE "spapr-tce-table"
@@ -458,26 +556,95 @@
     uint64_t bus_offset;
     uint32_t page_shift;
     uint64_t *table;
+    uint32_t mig_nb_table;
+    uint64_t *mig_table;
     bool bypass;
-    bool vfio_accel;
+    bool need_vfio;
     int fd;
-    MemoryRegion iommu;
+    MemoryRegion root, iommu;
+    struct VIOsPAPRDevice *vdev; /* for @bypass migration compatibility only */
     QLIST_ENTRY(sPAPRTCETable) list;
 };
 
-void spapr_events_init(sPAPREnvironment *spapr);
+sPAPRTCETable *spapr_tce_find_by_liobn(target_ulong liobn);
+
+struct sPAPREventLogEntry {
+    int log_type;
+    bool exception;
+    void *data;
+    QTAILQ_ENTRY(sPAPREventLogEntry) next;
+};
+
+void spapr_events_init(sPAPRMachineState *sm);
 void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq);
-int spapr_h_cas_compose_response(target_ulong addr, target_ulong size);
-sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn,
-                                   uint64_t bus_offset,
-                                   uint32_t page_shift,
-                                   uint32_t nb_table,
-                                   bool vfio_accel);
+int spapr_h_cas_compose_response(sPAPRMachineState *sm,
+                                 target_ulong addr, target_ulong size,
+                                 bool cpu_update, bool memory_update);
+sPAPRTCETable *spapr_tce_new_table(DeviceState *owner, uint32_t liobn);
+void spapr_tce_table_enable(sPAPRTCETable *tcet,
+                            uint32_t page_shift, uint64_t bus_offset,
+                            uint32_t nb_table);
+void spapr_tce_table_disable(sPAPRTCETable *tcet);
+void spapr_tce_set_need_vfio(sPAPRTCETable *tcet, bool need_vfio);
+
 MemoryRegion *spapr_tce_get_iommu(sPAPRTCETable *tcet);
-void spapr_tce_set_bypass(sPAPRTCETable *tcet, bool bypass);
 int spapr_dma_dt(void *fdt, int node_off, const char *propname,
                  uint32_t liobn, uint64_t window, uint32_t size);
 int spapr_tcet_dma_dt(void *fdt, int node_off, const char *propname,
                       sPAPRTCETable *tcet);
+void spapr_pci_switch_vga(bool big_endian);
+void spapr_hotplug_req_add_by_index(sPAPRDRConnector *drc);
+void spapr_hotplug_req_remove_by_index(sPAPRDRConnector *drc);
+void spapr_hotplug_req_add_by_count(sPAPRDRConnectorType drc_type,
+                                       uint32_t count);
+void spapr_hotplug_req_remove_by_count(sPAPRDRConnectorType drc_type,
+                                          uint32_t count);
+void spapr_cpu_init(sPAPRMachineState *spapr, PowerPCCPU *cpu, Error **errp);
+void *spapr_populate_hotplug_cpu_dt(CPUState *cs, int *fdt_offset,
+                                    sPAPRMachineState *spapr);
 
-#endif /* !defined (__HW_SPAPR_H__) */
+/* rtas-configure-connector state */
+struct sPAPRConfigureConnectorState {
+    uint32_t drc_index;
+    int fdt_offset;
+    int fdt_depth;
+    QTAILQ_ENTRY(sPAPRConfigureConnectorState) next;
+};
+
+void spapr_ccs_reset_hook(void *opaque);
+
+#define TYPE_SPAPR_RTC "spapr-rtc"
+#define TYPE_SPAPR_RNG "spapr-rng"
+
+void spapr_rtc_read(DeviceState *dev, struct tm *tm, uint32_t *ns);
+int spapr_rtc_import_offset(DeviceState *dev, int64_t legacy_offset);
+
+int spapr_rng_populate_dt(void *fdt);
+
+#define SPAPR_MEMORY_BLOCK_SIZE (1 << 28) /* 256MB */
+
+/*
+ * This defines the maximum number of DIMM slots we can have for sPAPR
+ * guest. This is not defined by sPAPR but we are defining it to 32 slots
+ * based on default number of slots provided by PowerPC kernel.
+ */
+#define SPAPR_MAX_RAM_SLOTS     32
+
+/* 1GB alignment for hotplug memory region */
+#define SPAPR_HOTPLUG_MEM_ALIGN (1ULL << 30)
+
+/*
+ * Number of 32 bit words in each LMB list entry in ibm,dynamic-memory
+ * property under ibm,dynamic-reconfiguration-memory node.
+ */
+#define SPAPR_DR_LMB_LIST_ENTRY_SIZE 6
+
+/*
+ * Defines for flag value in ibm,dynamic-memory property under
+ * ibm,dynamic-reconfiguration-memory node.
+ */
+#define SPAPR_LMB_FLAGS_ASSIGNED 0x00000008
+#define SPAPR_LMB_FLAGS_DRC_INVALID 0x00000020
+#define SPAPR_LMB_FLAGS_RESERVED 0x00000080
+
+#endif /* HW_SPAPR_H */
diff --git a/include/hw/ppc/spapr_cpu_core.h b/include/hw/ppc/spapr_cpu_core.h
new file mode 100644
index 0000000..1c9b319
--- /dev/null
+++ b/include/hw/ppc/spapr_cpu_core.h
@@ -0,0 +1,36 @@
+/*
+ * sPAPR CPU core device.
+ *
+ * Copyright (C) 2016 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef HW_SPAPR_CPU_CORE_H
+#define HW_SPAPR_CPU_CORE_H
+
+#include "hw/qdev.h"
+#include "hw/cpu/core.h"
+#include "target-ppc/cpu-qom.h"
+
+#define TYPE_SPAPR_CPU_CORE "spapr-cpu-core"
+#define SPAPR_CPU_CORE(obj) \
+    OBJECT_CHECK(sPAPRCPUCore, (obj), TYPE_SPAPR_CPU_CORE)
+
+typedef struct sPAPRCPUCore {
+    /*< private >*/
+    CPUCore parent_obj;
+
+    /*< public >*/
+    void *threads;
+    ObjectClass *cpu_class;
+} sPAPRCPUCore;
+
+void spapr_core_pre_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                         Error **errp);
+char *spapr_get_cpu_core_type(const char *model);
+void spapr_core_plug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                     Error **errp);
+void spapr_core_unplug(HotplugHandler *hotplug_dev, DeviceState *dev,
+                       Error **errp);
+#endif
diff --git a/include/hw/ppc/spapr_drc.h b/include/hw/ppc/spapr_drc.h
new file mode 100644
index 0000000..fa531d5
--- /dev/null
+++ b/include/hw/ppc/spapr_drc.h
@@ -0,0 +1,206 @@
+/*
+ * QEMU SPAPR Dynamic Reconfiguration Connector Implementation
+ *
+ * Copyright IBM Corp. 2014
+ *
+ * Authors:
+ *  Michael Roth      <mdroth@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef HW_SPAPR_DRC_H
+#define HW_SPAPR_DRC_H
+
+#include <libfdt.h>
+#include "qom/object.h"
+#include "hw/qdev.h"
+
+#define TYPE_SPAPR_DR_CONNECTOR "spapr-dr-connector"
+#define SPAPR_DR_CONNECTOR_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(sPAPRDRConnectorClass, obj, TYPE_SPAPR_DR_CONNECTOR)
+#define SPAPR_DR_CONNECTOR_CLASS(klass) \
+        OBJECT_CLASS_CHECK(sPAPRDRConnectorClass, klass, \
+                           TYPE_SPAPR_DR_CONNECTOR)
+#define SPAPR_DR_CONNECTOR(obj) OBJECT_CHECK(sPAPRDRConnector, (obj), \
+                                             TYPE_SPAPR_DR_CONNECTOR)
+
+/*
+ * Various hotplug types managed by sPAPRDRConnector
+ *
+ * these are somewhat arbitrary, but to make things easier
+ * when generating DRC indexes later we've aligned the bit
+ * positions with the values used to assign DRC indexes on
+ * pSeries. we use those values as bit shifts to allow for
+ * the OR'ing of these values in various QEMU routines, but
+ * for values exposed to the guest (via DRC indexes for
+ * instance) we will use the shift amounts.
+ */
+typedef enum {
+    SPAPR_DR_CONNECTOR_TYPE_SHIFT_CPU = 1,
+    SPAPR_DR_CONNECTOR_TYPE_SHIFT_PHB = 2,
+    SPAPR_DR_CONNECTOR_TYPE_SHIFT_VIO = 3,
+    SPAPR_DR_CONNECTOR_TYPE_SHIFT_PCI = 4,
+    SPAPR_DR_CONNECTOR_TYPE_SHIFT_LMB = 8,
+} sPAPRDRConnectorTypeShift;
+
+typedef enum {
+    SPAPR_DR_CONNECTOR_TYPE_ANY = ~0,
+    SPAPR_DR_CONNECTOR_TYPE_CPU = 1 << SPAPR_DR_CONNECTOR_TYPE_SHIFT_CPU,
+    SPAPR_DR_CONNECTOR_TYPE_PHB = 1 << SPAPR_DR_CONNECTOR_TYPE_SHIFT_PHB,
+    SPAPR_DR_CONNECTOR_TYPE_VIO = 1 << SPAPR_DR_CONNECTOR_TYPE_SHIFT_VIO,
+    SPAPR_DR_CONNECTOR_TYPE_PCI = 1 << SPAPR_DR_CONNECTOR_TYPE_SHIFT_PCI,
+    SPAPR_DR_CONNECTOR_TYPE_LMB = 1 << SPAPR_DR_CONNECTOR_TYPE_SHIFT_LMB,
+} sPAPRDRConnectorType;
+
+/*
+ * set via set-indicator RTAS calls
+ * as documented by PAPR+ 2.7 13.5.3.4, Table 177
+ *
+ * isolated: put device under firmware control
+ * unisolated: claim OS control of device (may or may not be in use)
+ */
+typedef enum {
+    SPAPR_DR_ISOLATION_STATE_ISOLATED   = 0,
+    SPAPR_DR_ISOLATION_STATE_UNISOLATED = 1
+} sPAPRDRIsolationState;
+
+/*
+ * set via set-indicator RTAS calls
+ * as documented by PAPR+ 2.7 13.5.3.4, Table 177
+ *
+ * unusable: mark device as unavailable to OS
+ * usable: mark device as available to OS
+ * exchange: (currently unused)
+ * recover: (currently unused)
+ */
+typedef enum {
+    SPAPR_DR_ALLOCATION_STATE_UNUSABLE  = 0,
+    SPAPR_DR_ALLOCATION_STATE_USABLE    = 1,
+    SPAPR_DR_ALLOCATION_STATE_EXCHANGE  = 2,
+    SPAPR_DR_ALLOCATION_STATE_RECOVER   = 3
+} sPAPRDRAllocationState;
+
+/*
+ * LED/visual indicator state
+ *
+ * set via set-indicator RTAS calls
+ * as documented by PAPR+ 2.7 13.5.3.4, Table 177,
+ * and PAPR+ 2.7 13.5.4.1, Table 180
+ *
+ * inactive: hotpluggable entity inactive and safely removable
+ * active: hotpluggable entity in use and not safely removable
+ * identify: (currently unused)
+ * action: (currently unused)
+ */
+typedef enum {
+    SPAPR_DR_INDICATOR_STATE_INACTIVE   = 0,
+    SPAPR_DR_INDICATOR_STATE_ACTIVE     = 1,
+    SPAPR_DR_INDICATOR_STATE_IDENTIFY   = 2,
+    SPAPR_DR_INDICATOR_STATE_ACTION     = 3,
+} sPAPRDRIndicatorState;
+
+/*
+ * returned via get-sensor-state RTAS calls
+ * as documented by PAPR+ 2.7 13.5.3.3, Table 175:
+ *
+ * empty: connector slot empty (e.g. empty hotpluggable PCI slot)
+ * present: connector slot populated and device available to OS
+ * unusable: device not currently available to OS
+ * exchange: (currently unused)
+ * recover: (currently unused)
+ */
+typedef enum {
+    SPAPR_DR_ENTITY_SENSE_EMPTY     = 0,
+    SPAPR_DR_ENTITY_SENSE_PRESENT   = 1,
+    SPAPR_DR_ENTITY_SENSE_UNUSABLE  = 2,
+    SPAPR_DR_ENTITY_SENSE_EXCHANGE  = 3,
+    SPAPR_DR_ENTITY_SENSE_RECOVER   = 4,
+} sPAPRDREntitySense;
+
+typedef enum {
+    SPAPR_DR_CC_RESPONSE_NEXT_SIB         = 1, /* currently unused */
+    SPAPR_DR_CC_RESPONSE_NEXT_CHILD       = 2,
+    SPAPR_DR_CC_RESPONSE_NEXT_PROPERTY    = 3,
+    SPAPR_DR_CC_RESPONSE_PREV_PARENT      = 4,
+    SPAPR_DR_CC_RESPONSE_SUCCESS          = 0,
+    SPAPR_DR_CC_RESPONSE_ERROR            = -1,
+    SPAPR_DR_CC_RESPONSE_CONTINUE         = -2,
+    SPAPR_DR_CC_RESPONSE_NOT_CONFIGURABLE = -9003,
+} sPAPRDRCCResponse;
+
+typedef void (spapr_drc_detach_cb)(DeviceState *d, void *opaque);
+
+typedef struct sPAPRDRConnector {
+    /*< private >*/
+    DeviceState parent;
+
+    sPAPRDRConnectorType type;
+    uint32_t id;
+    Object *owner;
+    const char *name;
+
+    /* sensor/indicator states */
+    uint32_t isolation_state;
+    uint32_t allocation_state;
+    uint32_t indicator_state;
+
+    /* configure-connector state */
+    void *fdt;
+    int fdt_start_offset;
+    bool configured;
+
+    bool awaiting_release;
+    bool signalled;
+    bool awaiting_allocation;
+
+    /* device pointer, via link property */
+    DeviceState *dev;
+    spapr_drc_detach_cb *detach_cb;
+    void *detach_cb_opaque;
+} sPAPRDRConnector;
+
+typedef struct sPAPRDRConnectorClass {
+    /*< private >*/
+    DeviceClass parent;
+
+    /*< public >*/
+
+    /* accessors for guest-visible (generally via RTAS) DR state */
+    uint32_t (*set_isolation_state)(sPAPRDRConnector *drc,
+                                    sPAPRDRIsolationState state);
+    uint32_t (*set_indicator_state)(sPAPRDRConnector *drc,
+                                    sPAPRDRIndicatorState state);
+    uint32_t (*set_allocation_state)(sPAPRDRConnector *drc,
+                                     sPAPRDRAllocationState state);
+    uint32_t (*get_index)(sPAPRDRConnector *drc);
+    uint32_t (*get_type)(sPAPRDRConnector *drc);
+    const char *(*get_name)(sPAPRDRConnector *drc);
+
+    uint32_t (*entity_sense)(sPAPRDRConnector *drc, sPAPRDREntitySense *state);
+
+    /* QEMU interfaces for managing FDT/configure-connector */
+    const void *(*get_fdt)(sPAPRDRConnector *drc, int *fdt_start_offset);
+    void (*set_configured)(sPAPRDRConnector *drc);
+
+    /* QEMU interfaces for managing hotplug operations */
+    void (*attach)(sPAPRDRConnector *drc, DeviceState *d, void *fdt,
+                   int fdt_start_offset, bool coldplug, Error **errp);
+    void (*detach)(sPAPRDRConnector *drc, DeviceState *d,
+                   spapr_drc_detach_cb *detach_cb,
+                   void *detach_cb_opaque, Error **errp);
+    bool (*release_pending)(sPAPRDRConnector *drc);
+    void (*set_signalled)(sPAPRDRConnector *drc);
+} sPAPRDRConnectorClass;
+
+sPAPRDRConnector *spapr_dr_connector_new(Object *owner,
+                                         sPAPRDRConnectorType type,
+                                         uint32_t id);
+sPAPRDRConnector *spapr_dr_connector_by_index(uint32_t index);
+sPAPRDRConnector *spapr_dr_connector_by_id(sPAPRDRConnectorType type,
+                                           uint32_t id);
+int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner,
+                          uint32_t drc_type_mask);
+
+#endif /* HW_SPAPR_DRC_H */
diff --git a/include/hw/ppc/spapr_vio.h b/include/hw/ppc/spapr_vio.h
index 46edc2a..d4a1e2c 100644
--- a/include/hw/ppc/spapr_vio.h
+++ b/include/hw/ppc/spapr_vio.h
@@ -1,5 +1,6 @@
-#ifndef _HW_SPAPR_VIO_H
-#define _HW_SPAPR_VIO_H
+#ifndef HW_SPAPR_VIO_H
+#define HW_SPAPR_VIO_H
+
 /*
  * QEMU sPAPR VIO bus definitions
  *
@@ -34,7 +35,7 @@
 #define TYPE_SPAPR_VIO_BUS "spapr-vio-bus"
 #define SPAPR_VIO_BUS(obj) OBJECT_CHECK(VIOsPAPRBus, (obj), TYPE_SPAPR_VIO_BUS)
 
-struct VIOsPAPRDevice;
+#define TYPE_SPAPR_VIO_BRIDGE "spapr-vio-bridge"
 
 typedef struct VIOsPAPR_CRQ {
     uint64_t qladdr;
@@ -52,7 +53,7 @@
     const char *dt_name, *dt_type, *dt_compatible;
     target_ulong signal_mask;
     uint32_t rtce_window_size;
-    int (*init)(VIOsPAPRDevice *dev);
+    void (*realize)(VIOsPAPRDevice *dev, Error **errp);
     void (*reset)(VIOsPAPRDevice *dev);
     int (*devnode)(VIOsPAPRDevice *dev, void *fdt, int node_off);
 } VIOsPAPRDeviceClass;
@@ -61,9 +62,11 @@
     DeviceState qdev;
     uint32_t reg;
     uint32_t irq;
-    target_ulong signal_state;
+    uint64_t signal_state;
     VIOsPAPR_CRQ crq;
     AddressSpace as;
+    MemoryRegion mrroot;
+    MemoryRegion mrbypass;
     sPAPRTCETable *tcet;
 };
 
@@ -86,7 +89,9 @@
 
 static inline qemu_irq spapr_vio_qirq(VIOsPAPRDevice *dev)
 {
-    return xics_get_qirq(spapr->icp, dev->irq);
+    sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
+
+    return xics_get_qirq(spapr->xics, dev->irq);
 }
 
 static inline bool spapr_vio_dma_valid(VIOsPAPRDevice *dev, uint64_t taddr,
@@ -124,7 +129,7 @@
 
 int spapr_vio_send_crq(VIOsPAPRDevice *dev, uint8_t *crq);
 
-VIOsPAPRDevice *vty_lookup(sPAPREnvironment *spapr, target_ulong reg);
+VIOsPAPRDevice *vty_lookup(sPAPRMachineState *spapr, target_ulong reg);
 void vty_putchars(VIOsPAPRDevice *sdev, uint8_t *buf, int len);
 void spapr_vty_create(VIOsPAPRBus *bus, CharDriverState *chardev);
 void spapr_vlan_create(VIOsPAPRBus *bus, NICInfo *nd);
@@ -139,4 +144,6 @@
 #define VMSTATE_SPAPR_VIO(_f, _s) \
     VMSTATE_STRUCT(_f, _s, 0, vmstate_spapr_vio, VIOsPAPRDevice)
 
-#endif /* _HW_SPAPR_VIO_H */
+void spapr_vio_set_bypass(VIOsPAPRDevice *dev, bool bypass);
+
+#endif /* HW_SPAPR_VIO_H */
diff --git a/include/hw/ppc/xics.h b/include/hw/ppc/xics.h
index a214dd7..2db9f93 100644
--- a/include/hw/ppc/xics.h
+++ b/include/hw/ppc/xics.h
@@ -24,28 +24,34 @@
  * THE SOFTWARE.
  *
  */
-#if !defined(__XICS_H__)
-#define __XICS_H__
+
+#ifndef XICS_H
+#define XICS_H
 
 #include "hw/sysbus.h"
 
 #define TYPE_XICS_COMMON "xics-common"
 #define XICS_COMMON(obj) OBJECT_CHECK(XICSState, (obj), TYPE_XICS_COMMON)
 
-#define TYPE_XICS "xics"
-#define XICS(obj) OBJECT_CHECK(XICSState, (obj), TYPE_XICS)
+/*
+ * Retain xics as the type name to be compatible for migration. Rest all the
+ * functions, class and variables are renamed as xics_spapr.
+ */
+#define TYPE_XICS_SPAPR "xics"
+#define XICS_SPAPR(obj) OBJECT_CHECK(XICSState, (obj), TYPE_XICS_SPAPR)
 
-#define TYPE_KVM_XICS "xics-kvm"
-#define KVM_XICS(obj) OBJECT_CHECK(KVMXICSState, (obj), TYPE_KVM_XICS)
+#define TYPE_XICS_SPAPR_KVM "xics-spapr-kvm"
+#define XICS_SPAPR_KVM(obj) \
+     OBJECT_CHECK(KVMXICSState, (obj), TYPE_XICS_SPAPR_KVM)
 
 #define XICS_COMMON_CLASS(klass) \
      OBJECT_CLASS_CHECK(XICSStateClass, (klass), TYPE_XICS_COMMON)
-#define XICS_CLASS(klass) \
-     OBJECT_CLASS_CHECK(XICSStateClass, (klass), TYPE_XICS)
+#define XICS_SPAPR_CLASS(klass) \
+     OBJECT_CLASS_CHECK(XICSStateClass, (klass), TYPE_XICS_SPAPR)
 #define XICS_COMMON_GET_CLASS(obj) \
      OBJECT_GET_CLASS(XICSStateClass, (obj), TYPE_XICS_COMMON)
-#define XICS_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(XICSStateClass, (obj), TYPE_XICS)
+#define XICS_SPAPR_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(XICSStateClass, (obj), TYPE_XICS_SPAPR)
 
 #define XICS_IPI        0x2
 #define XICS_BUID       0x1
@@ -109,6 +115,7 @@
     uint8_t pending_priority;
     uint8_t mfrr;
     qemu_irq output;
+    bool cap_irq_xics_enabled;
 };
 
 #define TYPE_ICS "ics"
@@ -137,9 +144,15 @@
     uint32_t offset;
     qemu_irq *qirqs;
     ICSIRQState *irqs;
-    XICSState *icp;
+    XICSState *xics;
 };
 
+static inline bool ics_valid_irq(ICSState *ics, uint32_t nr)
+{
+    return (nr >= ics->offset)
+        && (nr < (ics->offset + ics->nr_irqs));
+}
+
 struct ICSIRQState {
     uint32_t server;
     uint8_t priority;
@@ -156,14 +169,32 @@
     uint8_t flags;
 };
 
-#define XICS_IRQS               1024
+#define XICS_IRQS_SPAPR               1024
 
 qemu_irq xics_get_qirq(XICSState *icp, int irq);
-void xics_set_irq_type(XICSState *icp, int irq, bool lsi);
-int xics_alloc(XICSState *icp, int src, int irq_hint, bool lsi);
-int xics_alloc_block(XICSState *icp, int src, int num, bool lsi, bool align);
-void xics_free(XICSState *icp, int irq, int num);
+int xics_spapr_alloc(XICSState *icp, int src, int irq_hint, bool lsi,
+                     Error **errp);
+int xics_spapr_alloc_block(XICSState *icp, int src, int num, bool lsi,
+                           bool align, Error **errp);
+void xics_spapr_free(XICSState *icp, int irq, int num);
 
 void xics_cpu_setup(XICSState *icp, PowerPCCPU *cpu);
+void xics_cpu_destroy(XICSState *icp, PowerPCCPU *cpu);
 
-#endif /* __XICS_H__ */
+/* Internal XICS interfaces */
+int xics_get_cpu_index_by_dt_id(int cpu_dt_id);
+
+void icp_set_cppr(XICSState *icp, int server, uint8_t cppr);
+void icp_set_mfrr(XICSState *icp, int server, uint8_t mfrr);
+uint32_t icp_accept(ICPState *ss);
+uint32_t icp_ipoll(ICPState *ss, uint32_t *mfrr);
+void icp_eoi(XICSState *icp, int server, uint32_t xirr);
+
+void ics_write_xive(ICSState *ics, int nr, int server,
+                    uint8_t priority, uint8_t saved_priority);
+
+void ics_set_irq_type(ICSState *ics, int srcno, bool lsi);
+
+int xics_find_source(XICSState *icp, int irq);
+
+#endif /* XICS_H */
diff --git a/include/hw/ptimer.h b/include/hw/ptimer.h
index 8ebacbb..e397db5 100644
--- a/include/hw/ptimer.h
+++ b/include/hw/ptimer.h
@@ -19,6 +19,7 @@
 ptimer_state *ptimer_init(QEMUBH *bh);
 void ptimer_set_period(ptimer_state *s, int64_t period);
 void ptimer_set_freq(ptimer_state *s, uint32_t freq);
+uint64_t ptimer_get_limit(ptimer_state *s);
 void ptimer_set_limit(ptimer_state *s, uint64_t limit, int reload);
 uint64_t ptimer_get_count(ptimer_state *s);
 void ptimer_set_count(ptimer_state *s, uint64_t count);
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index 589bbe7..4b4b33b 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -3,11 +3,9 @@
 
 #include "qemu/queue.h"
 #include "qemu/option.h"
-#include "qemu/typedefs.h"
 #include "qemu/bitmap.h"
 #include "qom/object.h"
 #include "hw/irq.h"
-#include "qapi/error.h"
 #include "hw/hotplug.h"
 
 enum {
@@ -65,8 +63,8 @@
  * Operations depending on @props static properties should go into @realize.
  * After successful realization, setting static properties will fail.
  *
- * As an interim step, the #DeviceState:realized property is set by deprecated
- * functions qdev_init() and qdev_init_nofail().
+ * As an interim step, the #DeviceState:realized property can also be
+ * set with qdev_init_nofail().
  * In the future, devices will propagate this state change to their children
  * and along busses they expose.
  * The point in time will be deferred to machine creation, so that values
@@ -114,6 +112,19 @@
      * TODO remove once we're there
      */
     bool cannot_instantiate_with_device_add_yet;
+    /*
+     * Does this device model survive object_unref(object_new(TNAME))?
+     * All device models should, and this flag shouldn't exist.  Some
+     * devices crash in object_new(), some crash or hang in
+     * object_unref().  Makes introspecting properties with
+     * qmp_device_list_properties() dangerous.  Bad, because it's used
+     * by -device FOO,help.  This flag serves to protect that code.
+     * It should never be set without a comment explaining why it is
+     * set.
+     * TODO remove once we're there
+     */
+    bool cannot_destroy_with_object_finalize_yet;
+
     bool hotpluggable;
 
     /* callbacks */
@@ -165,6 +176,12 @@
     int alias_required_for_version;
 };
 
+struct DeviceListener {
+    void (*realize)(DeviceListener *listener, DeviceState *dev);
+    void (*unrealize)(DeviceListener *listener, DeviceState *dev);
+    QTAILQ_ENTRY(DeviceListener) link;
+};
+
 #define TYPE_BUS "bus"
 #define BUS(obj) OBJECT_CHECK(BusState, (obj), TYPE_BUS)
 #define BUS_CLASS(klass) OBJECT_CLASS_CHECK(BusClass, (klass), TYPE_BUS)
@@ -218,9 +235,9 @@
 struct Property {
     const char   *name;
     PropertyInfo *info;
-    int          offset;
+    ptrdiff_t    offset;
     uint8_t      bitnr;
-    uint8_t      qtype;
+    QType        qtype;
     int64_t      defval;
     int          arrayoffset;
     PropertyInfo *arrayinfo;
@@ -230,7 +247,7 @@
 struct PropertyInfo {
     const char *name;
     const char *description;
-    const char **enum_table;
+    const char * const *enum_table;
     int (*print)(DeviceState *dev, Property *prop, char *dest, size_t len);
     ObjectPropertyAccessor *get;
     ObjectPropertyAccessor *set;
@@ -242,6 +259,11 @@
  * @user_provided: Set to true if property comes from user-provided config
  * (command-line or config file).
  * @used: Set to true if property was used when initializing a device.
+ * @errp: Error destination, used like first argument of error_setg()
+ *        in case property setting fails later. If @errp is NULL, we
+ *        print warnings instead of ignoring errors silently. For
+ *        hotplugged devices, errp is always ignored and warnings are
+ *        printed instead.
  */
 typedef struct GlobalProperty {
     const char *driver;
@@ -249,17 +271,17 @@
     const char *value;
     bool user_provided;
     bool used;
-    QTAILQ_ENTRY(GlobalProperty) next;
+    Error **errp;
 } GlobalProperty;
 
 /*** Board API.  This should go away once we have a machine config file.  ***/
 
 DeviceState *qdev_create(BusState *bus, const char *name);
 DeviceState *qdev_try_create(BusState *bus, const char *name);
-int qdev_init(DeviceState *dev) QEMU_WARN_UNUSED_RESULT;
 void qdev_init_nofail(DeviceState *dev);
 void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
                                  int required_for_version);
+HotplugHandler *qdev_get_hotplug_handler(DeviceState *dev);
 void qdev_unplug(DeviceState *dev, Error **errp);
 void qdev_simple_device_unplug_cb(HotplugHandler *hotplug_dev,
                                   DeviceState *dev, Error **errp);
@@ -318,6 +340,7 @@
                        void *opaque);
 
 void qdev_reset_all(DeviceState *dev);
+void qdev_reset_all_fn(void *opaque);
 
 /**
  * @qbus_reset_all:
@@ -336,6 +359,7 @@
 BusState *sysbus_get_default(void);
 
 char *qdev_get_fw_dev_path(DeviceState *dev);
+char *qdev_get_own_fw_dev_path_from_handler(BusState *bus, DeviceState *dev);
 
 /**
  * @qdev_machine_init
@@ -376,4 +400,8 @@
 {
    return bus->hotplug_handler;
 }
+
+void device_listener_register(DeviceListener *listener);
+void device_listener_unregister(DeviceListener *listener);
+
 #endif
diff --git a/include/hw/qdev-properties.h b/include/hw/qdev-properties.h
index 070006c..2a9d2f9 100644
--- a/include/hw/qdev-properties.h
+++ b/include/hw/qdev-properties.h
@@ -6,6 +6,7 @@
 /*** qdev-properties.c ***/
 
 extern PropertyInfo qdev_prop_bit;
+extern PropertyInfo qdev_prop_bit64;
 extern PropertyInfo qdev_prop_bool;
 extern PropertyInfo qdev_prop_uint8;
 extern PropertyInfo qdev_prop_uint16;
@@ -17,8 +18,11 @@
 extern PropertyInfo qdev_prop_chr;
 extern PropertyInfo qdev_prop_ptr;
 extern PropertyInfo qdev_prop_macaddr;
+extern PropertyInfo qdev_prop_on_off_auto;
 extern PropertyInfo qdev_prop_losttickpolicy;
+extern PropertyInfo qdev_prop_blockdev_on_error;
 extern PropertyInfo qdev_prop_bios_chs_trans;
+extern PropertyInfo qdev_prop_fdc_drive_type;
 extern PropertyInfo qdev_prop_drive;
 extern PropertyInfo qdev_prop_netdev;
 extern PropertyInfo qdev_prop_vlan;
@@ -50,6 +54,15 @@
         .qtype     = QTYPE_QBOOL,                                \
         .defval    = (bool)_defval,                              \
         }
+#define DEFINE_PROP_BIT64(_name, _state, _field, _bit, _defval) {       \
+        .name      = (_name),                                           \
+        .info      = &(qdev_prop_bit64),                                \
+        .bitnr    = (_bit),                                             \
+        .offset    = offsetof(_state, _field)                           \
+            + type_check(uint64_t, typeof_field(_state, _field)),       \
+        .qtype     = QTYPE_QBOOL,                                       \
+        .defval    = (bool)_defval,                                     \
+        }
 
 #define DEFINE_PROP_BOOL(_name, _state, _field, _defval) {       \
         .name      = (_name),                                    \
@@ -144,13 +157,18 @@
     DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockBackend *)
 #define DEFINE_PROP_MACADDR(_n, _s, _f)         \
     DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_ON_OFF_AUTO(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_on_off_auto, OnOffAuto)
 #define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
     DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_losttickpolicy, \
                         LostTickPolicy)
+#define DEFINE_PROP_BLOCKDEV_ON_ERROR(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_blockdev_on_error, \
+                        BlockdevOnError)
 #define DEFINE_PROP_BIOS_CHS_TRANS(_n, _s, _f, _d) \
     DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_bios_chs_trans, int)
-#define DEFINE_PROP_BLOCKSIZE(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_blocksize, uint16_t)
+#define DEFINE_PROP_BLOCKSIZE(_n, _s, _f) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, 0, qdev_prop_blocksize, uint16_t)
 #define DEFINE_PROP_PCI_HOST_DEVADDR(_n, _s, _f) \
     DEFINE_PROP(_n, _s, _f, qdev_prop_pci_host_devaddr, PCIHostDeviceAddress)
 
@@ -168,10 +186,8 @@
 void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value);
 void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value);
 void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
-int qdev_prop_set_drive(DeviceState *dev, const char *name,
-                        BlockBackend *value) QEMU_WARN_UNUSED_RESULT;
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name,
-                                BlockBackend *value);
+void qdev_prop_set_drive(DeviceState *dev, const char *name,
+                         BlockBackend *value, Error **errp);
 void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
 void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
 /* FIXME: Remove opaque pointer properties.  */
@@ -180,15 +196,19 @@
 void qdev_prop_register_global(GlobalProperty *prop);
 void qdev_prop_register_global_list(GlobalProperty *props);
 int qdev_prop_check_globals(void);
-void qdev_prop_set_globals(DeviceState *dev, Error **errp);
-void qdev_prop_set_globals_for_type(DeviceState *dev, const char *typename,
-                                    Error **errp);
+void qdev_prop_set_globals(DeviceState *dev);
 void error_set_from_qdev_prop_error(Error **errp, int ret, DeviceState *dev,
                                     Property *prop, const char *value);
 
 /**
- * @qdev_property_add_static - add a @Property to a device referencing a
- * field in a struct.
+ * qdev_property_add_static:
+ * @dev: Device to add the property to.
+ * @prop: The qdev property definition.
+ * @errp: location to store error information.
+ *
+ * Add a static QOM property to @dev for qdev property @prop.
+ * On error, store error in @errp.  Static properties access data in a struct.
+ * The type of the QOM property is derived from prop->info.
  */
 void qdev_property_add_static(DeviceState *dev, Property *prop, Error **errp);
 
diff --git a/include/hw/register.h b/include/hw/register.h
new file mode 100644
index 0000000..8c12233
--- /dev/null
+++ b/include/hw/register.h
@@ -0,0 +1,255 @@
+/*
+ * Register Definition API
+ *
+ * Copyright (c) 2016 Xilinx Inc.
+ * Copyright (c) 2013 Peter Crosthwaite <peter.crosthwaite@xilinx.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef REGISTER_H
+#define REGISTER_H
+
+#include "hw/qdev-core.h"
+#include "exec/memory.h"
+
+typedef struct RegisterInfo RegisterInfo;
+typedef struct RegisterAccessInfo RegisterAccessInfo;
+typedef struct RegisterInfoArray RegisterInfoArray;
+
+/**
+ * Access description for a register that is part of guest accessible device
+ * state.
+ *
+ * @name: String name of the register
+ * @ro: whether or not the bit is read-only
+ * @w1c: bits with the common write 1 to clear semantic.
+ * @reset: reset value.
+ * @cor: Bits that are clear on read
+ * @rsvd: Bits that are reserved and should not be changed
+ *
+ * @pre_write: Pre write callback. Passed the value that's to be written,
+ * immediately before the actual write. The returned value is what is written,
+ * giving the handler a chance to modify the written value.
+ * @post_write: Post write callback. Passed the written value. Most write side
+ * effects should be implemented here.
+ *
+ * @post_read: Post read callback. Passes the value that is about to be returned
+ * for a read. The return value from this function is what is ultimately read,
+ * allowing this function to modify the value before return to the client.
+ */
+
+struct RegisterAccessInfo {
+    const char *name;
+    uint64_t ro;
+    uint64_t w1c;
+    uint64_t reset;
+    uint64_t cor;
+    uint64_t rsvd;
+    uint64_t unimp;
+
+    uint64_t (*pre_write)(RegisterInfo *reg, uint64_t val);
+    void (*post_write)(RegisterInfo *reg, uint64_t val);
+
+    uint64_t (*post_read)(RegisterInfo *reg, uint64_t val);
+
+    hwaddr addr;
+};
+
+/**
+ * A register that is part of guest accessible state
+ * @data: pointer to the register data. Will be cast
+ * to the relevant uint type depending on data_size.
+ * @data_size: Size of the register in bytes. Must be
+ * 1, 2, 4 or 8
+ *
+ * @access: Access description of this register
+ *
+ * @debug: Whether or not verbose debug is enabled
+ * @prefix: String prefix for log and debug messages
+ *
+ * @opaque: Opaque data for the register
+ */
+
+struct RegisterInfo {
+    /* <private> */
+    DeviceState parent_obj;
+
+    /* <public> */
+    void *data;
+    int data_size;
+
+    const RegisterAccessInfo *access;
+
+    void *opaque;
+};
+
+#define TYPE_REGISTER "qemu,register"
+#define REGISTER(obj) OBJECT_CHECK(RegisterInfo, (obj), TYPE_REGISTER)
+
+/**
+ * This structure is used to group all of the individual registers which are
+ * modeled using the RegisterInfo structure.
+ *
+ * @r is an aray containing of all the relevent RegisterInfo structures.
+ *
+ * @num_elements is the number of elements in the array r
+ *
+ * @mem: optional Memory region for the register
+ */
+
+struct RegisterInfoArray {
+    MemoryRegion mem;
+
+    int num_elements;
+    RegisterInfo **r;
+
+    bool debug;
+    const char *prefix;
+};
+
+/**
+ * write a value to a register, subject to its restrictions
+ * @reg: register to write to
+ * @val: value to write
+ * @we: write enable mask
+ * @prefix: The device prefix that should be printed before the register name
+ * @debug: Should the write operation debug information be printed?
+ */
+
+void register_write(RegisterInfo *reg, uint64_t val, uint64_t we,
+                    const char *prefix, bool debug);
+
+/**
+ * read a value from a register, subject to its restrictions
+ * @reg: register to read from
+ * @re: read enable mask
+ * @prefix: The device prefix that should be printed before the register name
+ * @debug: Should the read operation debug information be printed?
+ * returns: value read
+ */
+
+uint64_t register_read(RegisterInfo *reg, uint64_t re, const char* prefix,
+                       bool debug);
+
+/**
+ * reset a register
+ * @reg: register to reset
+ */
+
+void register_reset(RegisterInfo *reg);
+
+/**
+ * Initialize a register.
+ * @reg: Register to initialize
+ */
+
+void register_init(RegisterInfo *reg);
+
+/**
+ * Memory API MMIO write handler that will write to a Register API register.
+ * @opaque: RegisterInfo to write to
+ * @addr: Address to write
+ * @value: Value to write
+ * @size: Number of bytes to write
+ */
+
+void register_write_memory(void *opaque, hwaddr addr, uint64_t value,
+                           unsigned size);
+
+/**
+ * Memory API MMIO read handler that will read from a Register API register.
+ * @opaque: RegisterInfo to read from
+ * @addr: Address to read
+ * @size: Number of bytes to read
+ * returns: Value read from register
+ */
+
+uint64_t register_read_memory(void *opaque, hwaddr addr, unsigned size);
+
+/**
+ * Init a block of registers into a container MemoryRegion. A
+ * number of constant register definitions are parsed to create a corresponding
+ * array of RegisterInfo's.
+ *
+ * @owner: device owning the registers
+ * @rae: Register definitions to init
+ * @num: number of registers to init (length of @rae)
+ * @ri: Register array to init, must already be allocated
+ * @data: Array to use for register data, must already be allocated
+ * @ops: Memory region ops to access registers.
+ * @debug enabled: turn on/off verbose debug information
+ * returns: A structure containing all of the registers and an initialized
+ *          memory region (r_array->mem) the caller should add to a container.
+ */
+
+RegisterInfoArray *register_init_block32(DeviceState *owner,
+                                         const RegisterAccessInfo *rae,
+                                         int num, RegisterInfo *ri,
+                                         uint32_t *data,
+                                         const MemoryRegionOps *ops,
+                                         bool debug_enabled,
+                                         uint64_t memory_size);
+
+/**
+ * This function should be called to cleanup the registers that were initialized
+ * when calling register_init_block32(). This function should only be called
+ * from the device's instance_finalize function.
+ *
+ * Any memory operations that the device performed that require cleanup (such
+ * as creating subregions) need to be called before calling this function.
+ *
+ * @r_array: A structure containing all of the registers, as returned by
+ *           register_init_block32()
+ */
+
+void register_finalize_block(RegisterInfoArray *r_array);
+
+/* Define constants for a 32 bit register */
+
+/* This macro will define A_FOO, for the byte address of a register
+ * as well as R_FOO for the uint32_t[] register number (A_FOO / 4).
+ */
+#define REG32(reg, addr)                                                  \
+    enum { A_ ## reg = (addr) };                                          \
+    enum { R_ ## reg = (addr) / 4 };
+
+/* Define SHIFT, LENGTH and MASK constants for a field within a register */
+
+/* This macro will define FOO_BAR_MASK, FOO_BAR_SHIFT and FOO_BAR_LENGTH 
+ * constants for field BAR in register FOO.
+ */
+#define FIELD(reg, field, shift, length)                                  \
+    enum { R_ ## reg ## _ ## field ## _SHIFT = (shift)};                  \
+    enum { R_ ## reg ## _ ## field ## _LENGTH = (length)};                \
+    enum { R_ ## reg ## _ ## field ## _MASK =                             \
+                                        MAKE_64BIT_MASK(shift, length)};
+
+/* Extract a field from a register */
+#define FIELD_EX32(storage, reg, field)                                   \
+    extract32((storage), R_ ## reg ## _ ## field ## _SHIFT,               \
+              R_ ## reg ## _ ## field ## _LENGTH)
+
+/* Extract a field from an array of registers */
+#define ARRAY_FIELD_EX32(regs, reg, field)                                \
+    FIELD_EX32((regs)[R_ ## reg], reg, field)
+
+/* Deposit a register field.
+ * Assigning values larger then the target field will result in
+ * compilation warnings.
+ */
+#define FIELD_DP32(storage, reg, field, val) ({                           \
+    struct {                                                              \
+        unsigned int v:R_ ## reg ## _ ## field ## _LENGTH;                \
+    } v = { .v = val };                                                   \
+    uint32_t d;                                                           \
+    d = deposit32((storage), R_ ## reg ## _ ## field ## _SHIFT,           \
+                  R_ ## reg ## _ ## field ## _LENGTH, v.v);               \
+    d; })
+
+/* Deposit a field to array of registers.  */
+#define ARRAY_FIELD_DP32(regs, reg, field, val)                           \
+    (regs)[R_ ## reg] = FIELD_DP32((regs)[R_ ## reg], reg, field, val);
+
+#endif
diff --git a/include/hw/s390x/css-bridge.h b/include/hw/s390x/css-bridge.h
new file mode 100644
index 0000000..5a0203b
--- /dev/null
+++ b/include/hw/s390x/css-bridge.h
@@ -0,0 +1,38 @@
+/*
+ * virtual css bridge definition
+ *
+ * Copyright 2012,2016 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *            Pierre Morel <pmorel@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390X_CSS_BRIDGE_H
+#define HW_S390X_CSS_BRIDGE_H
+#include "qom/object.h"
+#include "hw/qdev-core.h"
+
+/* virtual css bridge */
+typedef struct VirtualCssBridge {
+    SysBusDevice sysbus_dev;
+    bool css_dev_path;
+} VirtualCssBridge;
+
+#define TYPE_VIRTUAL_CSS_BRIDGE "virtual-css-bridge"
+#define VIRTUAL_CSS_BRIDGE(obj) \
+    OBJECT_CHECK(VirtualCssBridge, (obj), TYPE_VIRTUAL_CSS_BRIDGE)
+
+/* virtual css bus type */
+typedef struct VirtualCssBus {
+    BusState parent_obj;
+} VirtualCssBus;
+
+#define TYPE_VIRTUAL_CSS_BUS "virtual-css-bus"
+#define VIRTUAL_CSS_BUS(obj) \
+     OBJECT_CHECK(VirtualCssBus, (obj), TYPE_VIRTUAL_CSS_BUS)
+VirtualCssBus *virtual_css_bus_init(void);
+
+#endif
diff --git a/include/hw/s390x/css.h b/include/hw/s390x/css.h
new file mode 100644
index 0000000..1da63e3
--- /dev/null
+++ b/include/hw/s390x/css.h
@@ -0,0 +1,190 @@
+/*
+ * Channel subsystem structures and definitions.
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef CSS_H
+#define CSS_H
+
+#include "hw/s390x/adapter.h"
+#include "hw/s390x/s390_flic.h"
+#include "hw/s390x/ioinst.h"
+
+/* Channel subsystem constants. */
+#define MAX_DEVNO 65535
+#define MAX_SCHID 65535
+#define MAX_SSID 3
+#define MAX_CSSID 254 /* 255 is reserved */
+#define MAX_CHPID 255
+
+#define MAX_CIWS 62
+
+#define VIRTUAL_CSSID 0xfe
+
+typedef struct CIW {
+    uint8_t type;
+    uint8_t command;
+    uint16_t count;
+} QEMU_PACKED CIW;
+
+typedef struct SenseId {
+    /* common part */
+    uint8_t reserved;        /* always 0x'FF' */
+    uint16_t cu_type;        /* control unit type */
+    uint8_t cu_model;        /* control unit model */
+    uint16_t dev_type;       /* device type */
+    uint8_t dev_model;       /* device model */
+    uint8_t unused;          /* padding byte */
+    /* extended part */
+    CIW ciw[MAX_CIWS];       /* variable # of CIWs */
+} QEMU_PACKED SenseId;
+
+/* Channel measurements, from linux/drivers/s390/cio/cmf.c. */
+typedef struct CMB {
+    uint16_t ssch_rsch_count;
+    uint16_t sample_count;
+    uint32_t device_connect_time;
+    uint32_t function_pending_time;
+    uint32_t device_disconnect_time;
+    uint32_t control_unit_queuing_time;
+    uint32_t device_active_only_time;
+    uint32_t reserved[2];
+} QEMU_PACKED CMB;
+
+typedef struct CMBE {
+    uint32_t ssch_rsch_count;
+    uint32_t sample_count;
+    uint32_t device_connect_time;
+    uint32_t function_pending_time;
+    uint32_t device_disconnect_time;
+    uint32_t control_unit_queuing_time;
+    uint32_t device_active_only_time;
+    uint32_t device_busy_time;
+    uint32_t initial_command_response_time;
+    uint32_t reserved[7];
+} QEMU_PACKED CMBE;
+
+typedef struct SubchDev SubchDev;
+struct SubchDev {
+    /* channel-subsystem related things: */
+    uint8_t cssid;
+    uint8_t ssid;
+    uint16_t schid;
+    uint16_t devno;
+    SCHIB curr_status;
+    uint8_t sense_data[32];
+    hwaddr channel_prog;
+    CCW1 last_cmd;
+    bool last_cmd_valid;
+    bool ccw_fmt_1;
+    bool thinint_active;
+    uint8_t ccw_no_data_cnt;
+    /* transport-provided data: */
+    int (*ccw_cb) (SubchDev *, CCW1);
+    void (*disable_cb)(SubchDev *);
+    SenseId id;
+    void *driver_data;
+};
+
+typedef struct IndAddr {
+    hwaddr addr;
+    uint64_t map;
+    unsigned long refcnt;
+    int len;
+    QTAILQ_ENTRY(IndAddr) sibling;
+} IndAddr;
+
+IndAddr *get_indicator(hwaddr ind_addr, int len);
+void release_indicator(AdapterInfo *adapter, IndAddr *indicator);
+int map_indicator(AdapterInfo *adapter, IndAddr *indicator);
+
+typedef SubchDev *(*css_subch_cb_func)(uint8_t m, uint8_t cssid, uint8_t ssid,
+                                       uint16_t schid);
+void subch_device_save(SubchDev *s, QEMUFile *f);
+int subch_device_load(SubchDev *s, QEMUFile *f);
+int css_create_css_image(uint8_t cssid, bool default_image);
+bool css_devno_used(uint8_t cssid, uint8_t ssid, uint16_t devno);
+void css_subch_assign(uint8_t cssid, uint8_t ssid, uint16_t schid,
+                      uint16_t devno, SubchDev *sch);
+void css_sch_build_virtual_schib(SubchDev *sch, uint8_t chpid, uint8_t type);
+uint16_t css_build_subchannel_id(SubchDev *sch);
+void css_reset(void);
+void css_reset_sch(SubchDev *sch);
+void css_queue_crw(uint8_t rsc, uint8_t erc, int chain, uint16_t rsid);
+void css_generate_sch_crws(uint8_t cssid, uint8_t ssid, uint16_t schid,
+                           int hotplugged, int add);
+void css_generate_chp_crws(uint8_t cssid, uint8_t chpid);
+void css_generate_css_crws(uint8_t cssid);
+void css_clear_sei_pending(void);
+void css_adapter_interrupt(uint8_t isc);
+
+#define CSS_IO_ADAPTER_VIRTIO 1
+int css_register_io_adapter(uint8_t type, uint8_t isc, bool swap,
+                            bool maskable, uint32_t *id);
+
+#ifndef CONFIG_USER_ONLY
+SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
+                         uint16_t schid);
+bool css_subch_visible(SubchDev *sch);
+void css_conditional_io_interrupt(SubchDev *sch);
+int css_do_stsch(SubchDev *sch, SCHIB *schib);
+bool css_schid_final(int m, uint8_t cssid, uint8_t ssid, uint16_t schid);
+int css_do_msch(SubchDev *sch, const SCHIB *schib);
+int css_do_xsch(SubchDev *sch);
+int css_do_csch(SubchDev *sch);
+int css_do_hsch(SubchDev *sch);
+int css_do_ssch(SubchDev *sch, ORB *orb);
+int css_do_tsch_get_irb(SubchDev *sch, IRB *irb, int *irb_len);
+void css_do_tsch_update_subch(SubchDev *sch);
+int css_do_stcrw(CRW *crw);
+void css_undo_stcrw(CRW *crw);
+int css_do_tpi(IOIntCode *int_code, int lowcore);
+int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid, uint8_t l_chpid,
+                         int rfmt, void *buf);
+void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo);
+int css_enable_mcsse(void);
+int css_enable_mss(void);
+int css_do_rsch(SubchDev *sch);
+int css_do_rchp(uint8_t cssid, uint8_t chpid);
+bool css_present(uint8_t cssid);
+#endif
+/*
+ * Identify a device within the channel subsystem.
+ * Note that this can be used to identify either the subchannel or
+ * the attached I/O device, as there's always one I/O device per
+ * subchannel.
+ */
+typedef struct CssDevId {
+    uint8_t cssid;
+    uint8_t ssid;
+    uint16_t devid;
+    bool valid;
+} CssDevId;
+
+extern PropertyInfo css_devid_propinfo;
+
+#define DEFINE_PROP_CSS_DEV_ID(_n, _s, _f) \
+    DEFINE_PROP(_n, _s, _f, css_devid_propinfo, CssDevId)
+
+/**
+ * Create a subchannel for the given bus id.
+ *
+ * If @p bus_id is valid, verify that it uses the virtual channel
+ * subsystem id and is not already in use, and find a free subchannel
+ * id for it. If @p bus_id is not valid, find a free subchannel id and
+ * device number across all subchannel sets. If either of the former
+ * actions succeed, allocate a subchannel structure, initialise it
+ * with the bus id, subchannel id and device number, register it with
+ * the CSS and return it. Otherwise return NULL.
+ *
+ * The caller becomes owner of the returned subchannel structure and
+ * is responsible for unregistering and freeing it.
+ */
+SubchDev *css_create_virtual_sch(CssDevId bus_id, Error **errp);
+#endif
diff --git a/include/hw/s390x/ebcdic.h b/include/hw/s390x/ebcdic.h
index 1d6fde9..69a04ca 100644
--- a/include/hw/s390x/ebcdic.h
+++ b/include/hw/s390x/ebcdic.h
@@ -9,8 +9,8 @@
  *
  */
 
-#ifndef EBCDIC_H_
-#define EBCDIC_H_
+#ifndef EBCDIC_H
+#define EBCDIC_H
 
 /* EBCDIC handling */
 static const uint8_t ebcdic2ascii[] = {
@@ -101,4 +101,4 @@
     }
 }
 
-#endif /* EBCDIC_H_ */
+#endif /* EBCDIC_H */
diff --git a/include/hw/s390x/event-facility.h b/include/hw/s390x/event-facility.h
index 6a062b6..def1bb0 100644
--- a/include/hw/s390x/event-facility.h
+++ b/include/hw/s390x/event-facility.h
@@ -15,7 +15,7 @@
 #ifndef HW_S390_SCLP_EVENT_FACILITY_H
 #define HW_S390_SCLP_EVENT_FACILITY_H
 
-#include <hw/qdev.h>
+#include "hw/qdev.h"
 #include "qemu/thread.h"
 #include "hw/s390x/sclp.h"
 
@@ -47,6 +47,7 @@
      OBJECT_GET_CLASS(SCLPEventClass, (obj), TYPE_SCLP_EVENT)
 
 #define TYPE_SCLP_CPU_HOTPLUG "sclp-cpu-hotplug"
+#define TYPE_SCLP_QUIESCE "sclpquiesce"
 
 typedef struct WriteEventMask {
     SCCBHeader h;
@@ -146,8 +147,10 @@
 
 typedef struct ReadEventData {
     SCCBHeader h;
-    EventBufferHeader ebh;
-    uint32_t mask;
+    union {
+        uint32_t mask;
+        EventBufferHeader ebh;
+    };
 } QEMU_PACKED ReadEventData;
 
 typedef struct SCLPEvent {
@@ -186,11 +189,8 @@
      OBJECT_GET_CLASS(SCLPEventFacilityClass, (obj), \
                       TYPE_SCLP_EVENT_FACILITY)
 
-typedef struct SCLPEventFacility SCLPEventFacility;
-
 typedef struct SCLPEventFacilityClass {
-    DeviceClass parent_class;
-    int (*init)(SCLPEventFacility *ef);
+    SysBusDeviceClass parent_class;
     void (*command_handler)(SCLPEventFacility *ef, SCCB *sccb, uint64_t code);
     bool (*event_pending)(SCLPEventFacility *ef);
 } SCLPEventFacilityClass;
diff --git a/include/hw/s390x/ioinst.h b/include/hw/s390x/ioinst.h
new file mode 100644
index 0000000..c559f53
--- /dev/null
+++ b/include/hw/s390x/ioinst.h
@@ -0,0 +1,232 @@
+/*
+ * S/390 channel I/O instructions
+ *
+ * Copyright 2012 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+*/
+
+#ifndef S390X_IOINST_H
+#define S390X_IOINST_H
+
+/*
+ * Channel I/O related definitions, as defined in the Principles
+ * Of Operation (and taken from the Linux implementation).
+ */
+
+/* subchannel status word (command mode only) */
+typedef struct SCSW {
+    uint16_t flags;
+    uint16_t ctrl;
+    uint32_t cpa;
+    uint8_t dstat;
+    uint8_t cstat;
+    uint16_t count;
+} QEMU_PACKED SCSW;
+
+#define SCSW_FLAGS_MASK_KEY 0xf000
+#define SCSW_FLAGS_MASK_SCTL 0x0800
+#define SCSW_FLAGS_MASK_ESWF 0x0400
+#define SCSW_FLAGS_MASK_CC 0x0300
+#define SCSW_FLAGS_MASK_FMT 0x0080
+#define SCSW_FLAGS_MASK_PFCH 0x0040
+#define SCSW_FLAGS_MASK_ISIC 0x0020
+#define SCSW_FLAGS_MASK_ALCC 0x0010
+#define SCSW_FLAGS_MASK_SSI 0x0008
+#define SCSW_FLAGS_MASK_ZCC 0x0004
+#define SCSW_FLAGS_MASK_ECTL 0x0002
+#define SCSW_FLAGS_MASK_PNO 0x0001
+
+#define SCSW_CTRL_MASK_FCTL 0x7000
+#define SCSW_CTRL_MASK_ACTL 0x0fe0
+#define SCSW_CTRL_MASK_STCTL 0x001f
+
+#define SCSW_FCTL_CLEAR_FUNC 0x1000
+#define SCSW_FCTL_HALT_FUNC 0x2000
+#define SCSW_FCTL_START_FUNC 0x4000
+
+#define SCSW_ACTL_SUSP 0x0020
+#define SCSW_ACTL_DEVICE_ACTIVE 0x0040
+#define SCSW_ACTL_SUBCH_ACTIVE 0x0080
+#define SCSW_ACTL_CLEAR_PEND 0x0100
+#define SCSW_ACTL_HALT_PEND  0x0200
+#define SCSW_ACTL_START_PEND 0x0400
+#define SCSW_ACTL_RESUME_PEND 0x0800
+
+#define SCSW_STCTL_STATUS_PEND 0x0001
+#define SCSW_STCTL_SECONDARY 0x0002
+#define SCSW_STCTL_PRIMARY 0x0004
+#define SCSW_STCTL_INTERMEDIATE 0x0008
+#define SCSW_STCTL_ALERT 0x0010
+
+#define SCSW_DSTAT_ATTENTION     0x80
+#define SCSW_DSTAT_STAT_MOD      0x40
+#define SCSW_DSTAT_CU_END        0x20
+#define SCSW_DSTAT_BUSY          0x10
+#define SCSW_DSTAT_CHANNEL_END   0x08
+#define SCSW_DSTAT_DEVICE_END    0x04
+#define SCSW_DSTAT_UNIT_CHECK    0x02
+#define SCSW_DSTAT_UNIT_EXCEP    0x01
+
+#define SCSW_CSTAT_PCI           0x80
+#define SCSW_CSTAT_INCORR_LEN    0x40
+#define SCSW_CSTAT_PROG_CHECK    0x20
+#define SCSW_CSTAT_PROT_CHECK    0x10
+#define SCSW_CSTAT_DATA_CHECK    0x08
+#define SCSW_CSTAT_CHN_CTRL_CHK  0x04
+#define SCSW_CSTAT_INTF_CTRL_CHK 0x02
+#define SCSW_CSTAT_CHAIN_CHECK   0x01
+
+/* path management control word */
+typedef struct PMCW {
+    uint32_t intparm;
+    uint16_t flags;
+    uint16_t devno;
+    uint8_t  lpm;
+    uint8_t  pnom;
+    uint8_t  lpum;
+    uint8_t  pim;
+    uint16_t mbi;
+    uint8_t  pom;
+    uint8_t  pam;
+    uint8_t  chpid[8];
+    uint32_t chars;
+} QEMU_PACKED PMCW;
+
+#define PMCW_FLAGS_MASK_QF 0x8000
+#define PMCW_FLAGS_MASK_W 0x4000
+#define PMCW_FLAGS_MASK_ISC 0x3800
+#define PMCW_FLAGS_MASK_ENA 0x0080
+#define PMCW_FLAGS_MASK_LM 0x0060
+#define PMCW_FLAGS_MASK_MME 0x0018
+#define PMCW_FLAGS_MASK_MP 0x0004
+#define PMCW_FLAGS_MASK_TF 0x0002
+#define PMCW_FLAGS_MASK_DNV 0x0001
+#define PMCW_FLAGS_MASK_INVALID 0x0700
+
+#define PMCW_CHARS_MASK_ST 0x00e00000
+#define PMCW_CHARS_MASK_MBFC 0x00000004
+#define PMCW_CHARS_MASK_XMWME 0x00000002
+#define PMCW_CHARS_MASK_CSENSE 0x00000001
+#define PMCW_CHARS_MASK_INVALID 0xff1ffff8
+
+/* subchannel information block */
+typedef struct SCHIB {
+    PMCW pmcw;
+    SCSW scsw;
+    uint64_t mba;
+    uint8_t mda[4];
+} QEMU_PACKED SCHIB;
+
+/* interruption response block */
+typedef struct IRB {
+    SCSW scsw;
+    uint32_t esw[5];
+    uint32_t ecw[8];
+    uint32_t emw[8];
+} QEMU_PACKED IRB;
+
+/* operation request block */
+typedef struct ORB {
+    uint32_t intparm;
+    uint16_t ctrl0;
+    uint8_t lpm;
+    uint8_t ctrl1;
+    uint32_t cpa;
+} QEMU_PACKED ORB;
+
+#define ORB_CTRL0_MASK_KEY 0xf000
+#define ORB_CTRL0_MASK_SPND 0x0800
+#define ORB_CTRL0_MASK_STR 0x0400
+#define ORB_CTRL0_MASK_MOD 0x0200
+#define ORB_CTRL0_MASK_SYNC 0x0100
+#define ORB_CTRL0_MASK_FMT 0x0080
+#define ORB_CTRL0_MASK_PFCH 0x0040
+#define ORB_CTRL0_MASK_ISIC 0x0020
+#define ORB_CTRL0_MASK_ALCC 0x0010
+#define ORB_CTRL0_MASK_SSIC 0x0008
+#define ORB_CTRL0_MASK_C64 0x0002
+#define ORB_CTRL0_MASK_I2K 0x0001
+#define ORB_CTRL0_MASK_INVALID 0x0004
+
+#define ORB_CTRL1_MASK_ILS 0x80
+#define ORB_CTRL1_MASK_MIDAW 0x40
+#define ORB_CTRL1_MASK_ORBX 0x01
+#define ORB_CTRL1_MASK_INVALID 0x3e
+
+/* channel command word (type 0) */
+typedef struct CCW0 {
+        uint8_t cmd_code;
+        uint8_t cda0;
+        uint16_t cda1;
+        uint8_t flags;
+        uint8_t reserved;
+        uint16_t count;
+} QEMU_PACKED CCW0;
+
+/* channel command word (type 1) */
+typedef struct CCW1 {
+    uint8_t cmd_code;
+    uint8_t flags;
+    uint16_t count;
+    uint32_t cda;
+} QEMU_PACKED CCW1;
+
+#define CCW_FLAG_DC              0x80
+#define CCW_FLAG_CC              0x40
+#define CCW_FLAG_SLI             0x20
+#define CCW_FLAG_SKIP            0x10
+#define CCW_FLAG_PCI             0x08
+#define CCW_FLAG_IDA             0x04
+#define CCW_FLAG_SUSPEND         0x02
+
+#define CCW_CMD_NOOP             0x03
+#define CCW_CMD_BASIC_SENSE      0x04
+#define CCW_CMD_TIC              0x08
+#define CCW_CMD_SENSE_ID         0xe4
+
+typedef struct CRW {
+    uint16_t flags;
+    uint16_t rsid;
+} QEMU_PACKED CRW;
+
+#define CRW_FLAGS_MASK_S 0x4000
+#define CRW_FLAGS_MASK_R 0x2000
+#define CRW_FLAGS_MASK_C 0x1000
+#define CRW_FLAGS_MASK_RSC 0x0f00
+#define CRW_FLAGS_MASK_A 0x0080
+#define CRW_FLAGS_MASK_ERC 0x003f
+
+#define CRW_ERC_INIT 0x02
+#define CRW_ERC_IPI  0x04
+
+#define CRW_RSC_SUBCH 0x3
+#define CRW_RSC_CHP   0x4
+#define CRW_RSC_CSS   0xb
+
+/* I/O interruption code */
+typedef struct IOIntCode {
+    uint32_t subsys_id;
+    uint32_t intparm;
+    uint32_t interrupt_id;
+} QEMU_PACKED IOIntCode;
+
+/* schid disintegration */
+#define IOINST_SCHID_ONE(_schid)   ((_schid & 0x00010000) >> 16)
+#define IOINST_SCHID_M(_schid)     ((_schid & 0x00080000) >> 19)
+#define IOINST_SCHID_CSSID(_schid) ((_schid & 0xff000000) >> 24)
+#define IOINST_SCHID_SSID(_schid)  ((_schid & 0x00060000) >> 17)
+#define IOINST_SCHID_NR(_schid)    (_schid & 0x0000ffff)
+
+#define IO_INT_WORD_ISC(_int_word) ((_int_word & 0x38000000) >> 27)
+#define ISC_TO_ISC_BITS(_isc)      ((0x80 >> _isc) << 24)
+
+#define IO_INT_WORD_AI 0x80000000
+
+int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
+                                 int *schid);
+
+#endif
diff --git a/include/hw/s390x/s390-virtio-ccw.h b/include/hw/s390x/s390-virtio-ccw.h
new file mode 100644
index 0000000..a0c1fc8
--- /dev/null
+++ b/include/hw/s390x/s390-virtio-ccw.h
@@ -0,0 +1,44 @@
+/*
+ * virtio ccw machine definitions
+ *
+ * Copyright 2012, 2016 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+#ifndef HW_S390X_S390_VIRTIO_CCW_H
+#define HW_S390X_S390_VIRTIO_CCW_H
+
+#include "hw/boards.h"
+
+#define TYPE_S390_CCW_MACHINE               "s390-ccw-machine"
+
+#define S390_CCW_MACHINE(obj) \
+    OBJECT_CHECK(S390CcwMachineState, (obj), TYPE_S390_CCW_MACHINE)
+
+#define S390_MACHINE_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390CcwMachineClass, (klass), TYPE_S390_CCW_MACHINE)
+
+typedef struct S390CcwMachineState {
+    /*< private >*/
+    MachineState parent_obj;
+
+    /*< public >*/
+    bool aes_key_wrap;
+    bool dea_key_wrap;
+} S390CcwMachineState;
+
+typedef struct S390CcwMachineClass {
+    /*< private >*/
+    MachineClass parent_class;
+
+    /*< public >*/
+    bool ri_allowed;
+} S390CcwMachineClass;
+
+/* runtime-instrumentation allowed by the machine */
+bool ri_allowed(void);
+
+#endif
diff --git a/include/hw/s390x/s390_flic.h b/include/hw/s390x/s390_flic.h
index 489d73b..9094eda 100644
--- a/include/hw/s390x/s390_flic.h
+++ b/include/hw/s390x/s390_flic.h
@@ -10,17 +10,20 @@
  * directory.
  */
 
-#ifndef __HW_S390_FLIC_H
-#define __HW_S390_FLIC_H
+#ifndef HW_S390_FLIC_H
+#define HW_S390_FLIC_H
 
 #include "hw/sysbus.h"
 #include "hw/s390x/adapter.h"
 #include "hw/virtio/virtio.h"
 
+#define ADAPTER_ROUTES_MAX_GSI 64
+#define VIRTIO_CCW_QUEUE_MAX ADAPTER_ROUTES_MAX_GSI
+
 typedef struct AdapterRoutes {
     AdapterInfo adapter;
     int num_routes;
-    int gsi[VIRTIO_PCI_QUEUE_MAX];
+    int gsi[ADAPTER_ROUTES_MAX_GSI];
 } AdapterRoutes;
 
 #define TYPE_S390_FLIC_COMMON "s390-flic"
@@ -46,6 +49,8 @@
                           bool do_map);
     int (*add_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
     void (*release_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
+    int (*clear_io_irq)(S390FLICState *fs, uint16_t subchannel_id,
+                        uint16_t subchannel_nr);
 } S390FLICStateClass;
 
 #define TYPE_KVM_S390_FLIC "s390-flic-kvm"
@@ -73,4 +78,4 @@
 }
 #endif
 
-#endif /* __HW_S390_FLIC_H */
+#endif /* HW_S390_FLIC_H */
diff --git a/include/hw/s390x/sclp.h b/include/hw/s390x/sclp.h
index ec07a11..ba28d1d 100644
--- a/include/hw/s390x/sclp.h
+++ b/include/hw/s390x/sclp.h
@@ -14,8 +14,8 @@
 #ifndef HW_S390_SCLP_H
 #define HW_S390_SCLP_H
 
-#include <hw/sysbus.h>
-#include <hw/qdev.h>
+#include "hw/sysbus.h"
+#include "hw/qdev.h"
 
 #define SCLP_CMD_CODE_MASK                      0xffff00ff
 
@@ -43,14 +43,23 @@
 #define SCLP_CMDW_CONFIGURE_CPU                 0x00110001
 #define SCLP_CMDW_DECONFIGURE_CPU               0x00100001
 
+/* SCLP PCI codes */
+#define SCLP_HAS_PCI_RECONFIG                   0x0000000040000000ULL
+#define SCLP_CMDW_CONFIGURE_PCI                 0x001a0001
+#define SCLP_CMDW_DECONFIGURE_PCI               0x001b0001
+#define SCLP_RECONFIG_PCI_ATPYE                 2
+
 /* SCLP response codes */
 #define SCLP_RC_NORMAL_READ_COMPLETION          0x0010
 #define SCLP_RC_NORMAL_COMPLETION               0x0020
 #define SCLP_RC_SCCB_BOUNDARY_VIOLATION         0x0100
+#define SCLP_RC_NO_ACTION_REQUIRED              0x0120
 #define SCLP_RC_INVALID_SCLP_COMMAND            0x01f0
 #define SCLP_RC_CONTAINED_EQUIPMENT_CHECK       0x0340
 #define SCLP_RC_INSUFFICIENT_SCCB_LENGTH        0x0300
 #define SCLP_RC_STANDBY_READ_COMPLETION         0x0410
+#define SCLP_RC_ADAPTER_IN_RESERVED_STATE       0x05f0
+#define SCLP_RC_ADAPTER_ID_NOT_RECOGNIZED       0x09f0
 #define SCLP_RC_INVALID_FUNCTION                0x40f0
 #define SCLP_RC_NO_EVENT_BUFFERS_STORED         0x60f0
 #define SCLP_RC_INVALID_SELECTION_MASK          0x70f0
@@ -155,6 +164,39 @@
     char data[SCCB_DATA_LEN];
  } QEMU_PACKED SCCB;
 
+#define TYPE_SCLP "sclp"
+#define SCLP(obj) OBJECT_CHECK(SCLPDevice, (obj), TYPE_SCLP)
+#define SCLP_CLASS(oc) OBJECT_CLASS_CHECK(SCLPDeviceClass, (oc), TYPE_SCLP)
+#define SCLP_GET_CLASS(obj) OBJECT_GET_CLASS(SCLPDeviceClass, (obj), TYPE_SCLP)
+
+typedef struct SCLPEventFacility SCLPEventFacility;
+
+typedef struct SCLPDevice {
+    /* private */
+    DeviceState parent_obj;
+    SCLPEventFacility *event_facility;
+    int increment_size;
+
+    /* public */
+} SCLPDevice;
+
+typedef struct SCLPDeviceClass {
+    /* private */
+    DeviceClass parent_class;
+    void (*read_SCP_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_storage_element0_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_storage_element1_info)(SCLPDevice *sclp, SCCB *sccb);
+    void (*attach_storage_element)(SCLPDevice *sclp, SCCB *sccb,
+                                   uint16_t element);
+    void (*assign_storage)(SCLPDevice *sclp, SCCB *sccb);
+    void (*unassign_storage)(SCLPDevice *sclp, SCCB *sccb);
+    void (*read_cpu_info)(SCLPDevice *sclp, SCCB *sccb);
+
+    /* public */
+    void (*execute)(SCLPDevice *sclp, SCCB *sccb, uint32_t code);
+    void (*service_interrupt)(SCLPDevice *sclp, uint32_t sccb);
+} SCLPDeviceClass;
+
 typedef struct sclpMemoryHotplugDev sclpMemoryHotplugDev;
 
 #define TYPE_SCLP_MEMORY_HOTPLUG_DEV "sclp-memory-hotplug-dev"
diff --git a/include/hw/s390x/storage-keys.h b/include/hw/s390x/storage-keys.h
new file mode 100644
index 0000000..62df48e
--- /dev/null
+++ b/include/hw/s390x/storage-keys.h
@@ -0,0 +1,60 @@
+/*
+ * s390 storage key device
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Jason J. Herne <jjherne@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef S390_STORAGE_KEYS_H
+#define S390_STORAGE_KEYS_H
+
+#include "hw/qdev.h"
+#include "monitor/monitor.h"
+
+#define TYPE_S390_SKEYS "s390-skeys"
+#define S390_SKEYS(obj) \
+    OBJECT_CHECK(S390SKeysState, (obj), TYPE_S390_SKEYS)
+
+typedef struct S390SKeysState {
+    DeviceState parent_obj;
+    bool migration_enabled;
+
+} S390SKeysState;
+
+#define S390_SKEYS_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390SKeysClass, (klass), TYPE_S390_SKEYS)
+#define S390_SKEYS_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390SKeysClass, (obj), TYPE_S390_SKEYS)
+
+typedef struct S390SKeysClass {
+    DeviceClass parent_class;
+    int (*skeys_enabled)(S390SKeysState *ks);
+    int (*get_skeys)(S390SKeysState *ks, uint64_t start_gfn, uint64_t count,
+                     uint8_t *keys);
+    int (*set_skeys)(S390SKeysState *ks, uint64_t start_gfn, uint64_t count,
+                     uint8_t *keys);
+} S390SKeysClass;
+
+#define TYPE_KVM_S390_SKEYS "s390-skeys-kvm"
+#define TYPE_QEMU_S390_SKEYS "s390-skeys-qemu"
+#define QEMU_S390_SKEYS(obj) \
+    OBJECT_CHECK(QEMUS390SKeysState, (obj), TYPE_QEMU_S390_SKEYS)
+
+typedef struct QEMUS390SKeysState {
+    S390SKeysState parent_obj;
+    uint8_t *keydata;
+    uint32_t key_count;
+} QEMUS390SKeysState;
+
+void s390_skeys_init(void);
+
+S390SKeysState *s390_get_skeys_device(void);
+
+void hmp_dump_skeys(Monitor *mon, const QDict *qdict);
+void hmp_info_skeys(Monitor *mon, const QDict *qdict);
+
+#endif /* S390_STORAGE_KEYS_H */
diff --git a/include/hw/scsi/esp.h b/include/hw/scsi/esp.h
index 6c79527..d2c4886 100644
--- a/include/hw/scsi/esp.h
+++ b/include/hw/scsi/esp.h
@@ -14,6 +14,7 @@
 
 #define ESP_REGS 16
 #define TI_BUFSZ 16
+#define ESP_CMDBUF_SZ 32
 
 typedef struct ESPState ESPState;
 
@@ -31,7 +32,7 @@
     SCSIBus bus;
     SCSIDevice *current_dev;
     SCSIRequest *current_req;
-    uint8_t cmdbuf[TI_BUFSZ];
+    uint8_t cmdbuf[ESP_CMDBUF_SZ];
     uint32_t cmdlen;
     uint32_t do_cmd;
 
diff --git a/include/hw/scsi/scsi.h b/include/hw/scsi/scsi.h
index cdaf0f8..94d7868 100644
--- a/include/hw/scsi/scsi.h
+++ b/include/hw/scsi/scsi.h
@@ -2,16 +2,16 @@
 #define QEMU_HW_SCSI_H
 
 #include "hw/qdev.h"
-#include "qemu/typedefs.h"
 #include "hw/block/block.h"
 #include "sysemu/sysemu.h"
 #include "qemu/notify.h"
 
 #define MAX_SCSI_DEVS	255
 
-#define SCSI_CMD_BUF_SIZE     16
-#define SCSI_SENSE_LEN      18
-#define SCSI_INQUIRY_LEN    36
+#define SCSI_CMD_BUF_SIZE      16
+#define SCSI_SENSE_LEN         18
+#define SCSI_SENSE_LEN_SCANNER 32
+#define SCSI_INQUIRY_LEN       36
 
 typedef struct SCSIBus SCSIBus;
 typedef struct SCSIBusInfo SCSIBusInfo;
@@ -108,6 +108,8 @@
     int blocksize;
     int type;
     uint64_t max_lba;
+    uint64_t wwn;
+    uint64_t port_wwn;
 };
 
 extern const VMStateDescription vmstate_scsi_device;
@@ -250,7 +252,6 @@
 SCSIRequest *scsi_req_new(SCSIDevice *d, uint32_t tag, uint32_t lun,
                           uint8_t *buf, void *hba_private);
 int32_t scsi_req_enqueue(SCSIRequest *req);
-void scsi_req_free(SCSIRequest *req);
 SCSIRequest *scsi_req_ref(SCSIRequest *req);
 void scsi_req_unref(SCSIRequest *req);
 
@@ -272,6 +273,7 @@
 void scsi_device_set_ua(SCSIDevice *sdev, SCSISense sense);
 void scsi_device_report_change(SCSIDevice *dev, SCSISense sense);
 void scsi_device_unit_attention_reported(SCSIDevice *dev);
+void scsi_generic_read_device_identification(SCSIDevice *dev);
 int scsi_device_get_sense(SCSIDevice *dev, uint8_t *buf, int len, bool fixed);
 SCSIDevice *scsi_device_find(SCSIBus *bus, int channel, int target, int lun);
 
diff --git a/include/hw/sd.h b/include/hw/sd.h
deleted file mode 100644
index 79adb5b..0000000
--- a/include/hw/sd.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * SD Memory Card emulation.  Mostly correct for MMC too.
- *
- * Copyright (c) 2006 Andrzej Zaborowski  <balrog@zabor.org>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in
- *    the documentation and/or other materials provided with the
- *    distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
- * PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
- * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#ifndef __hw_sd_h
-#define __hw_sd_h		1
-
-#define OUT_OF_RANGE		(1 << 31)
-#define ADDRESS_ERROR		(1 << 30)
-#define BLOCK_LEN_ERROR		(1 << 29)
-#define ERASE_SEQ_ERROR		(1 << 28)
-#define ERASE_PARAM		(1 << 27)
-#define WP_VIOLATION		(1 << 26)
-#define CARD_IS_LOCKED		(1 << 25)
-#define LOCK_UNLOCK_FAILED	(1 << 24)
-#define COM_CRC_ERROR		(1 << 23)
-#define ILLEGAL_COMMAND		(1 << 22)
-#define CARD_ECC_FAILED		(1 << 21)
-#define CC_ERROR		(1 << 20)
-#define SD_ERROR		(1 << 19)
-#define CID_CSD_OVERWRITE	(1 << 16)
-#define WP_ERASE_SKIP		(1 << 15)
-#define CARD_ECC_DISABLED	(1 << 14)
-#define ERASE_RESET		(1 << 13)
-#define CURRENT_STATE		(7 << 9)
-#define READY_FOR_DATA		(1 << 8)
-#define APP_CMD			(1 << 5)
-#define AKE_SEQ_ERROR		(1 << 3)
-#define OCR_CCS_BITN        30
-
-typedef enum {
-    sd_none = -1,
-    sd_bc = 0,	/* broadcast -- no response */
-    sd_bcr,	/* broadcast with response */
-    sd_ac,	/* addressed -- no data transfer */
-    sd_adtc,	/* addressed with data transfer */
-} sd_cmd_type_t;
-
-typedef struct {
-    uint8_t cmd;
-    uint32_t arg;
-    uint8_t crc;
-} SDRequest;
-
-typedef struct SDState SDState;
-
-SDState *sd_init(BlockBackend *bs, bool is_spi);
-int sd_do_command(SDState *sd, SDRequest *req,
-                  uint8_t *response);
-void sd_write_data(SDState *sd, uint8_t value);
-uint8_t sd_read_data(SDState *sd);
-void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert);
-bool sd_data_ready(SDState *sd);
-void sd_enable(SDState *sd, bool enable);
-
-#endif	/* __hw_sd_h */
diff --git a/include/hw/sd/sd.h b/include/hw/sd/sd.h
new file mode 100644
index 0000000..79909b2
--- /dev/null
+++ b/include/hw/sd/sd.h
@@ -0,0 +1,148 @@
+/*
+ * SD Memory Card emulation.  Mostly correct for MMC too.
+ *
+ * Copyright (c) 2006 Andrzej Zaborowski  <balrog@zabor.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef HW_SD_H
+#define HW_SD_H
+
+#include "hw/qdev.h"
+
+#define OUT_OF_RANGE		(1 << 31)
+#define ADDRESS_ERROR		(1 << 30)
+#define BLOCK_LEN_ERROR		(1 << 29)
+#define ERASE_SEQ_ERROR		(1 << 28)
+#define ERASE_PARAM		(1 << 27)
+#define WP_VIOLATION		(1 << 26)
+#define CARD_IS_LOCKED		(1 << 25)
+#define LOCK_UNLOCK_FAILED	(1 << 24)
+#define COM_CRC_ERROR		(1 << 23)
+#define ILLEGAL_COMMAND		(1 << 22)
+#define CARD_ECC_FAILED		(1 << 21)
+#define CC_ERROR		(1 << 20)
+#define SD_ERROR		(1 << 19)
+#define CID_CSD_OVERWRITE	(1 << 16)
+#define WP_ERASE_SKIP		(1 << 15)
+#define CARD_ECC_DISABLED	(1 << 14)
+#define ERASE_RESET		(1 << 13)
+#define CURRENT_STATE		(7 << 9)
+#define READY_FOR_DATA		(1 << 8)
+#define APP_CMD			(1 << 5)
+#define AKE_SEQ_ERROR		(1 << 3)
+#define OCR_CCS_BITN        30
+
+typedef enum {
+    sd_none = -1,
+    sd_bc = 0,	/* broadcast -- no response */
+    sd_bcr,	/* broadcast with response */
+    sd_ac,	/* addressed -- no data transfer */
+    sd_adtc,	/* addressed with data transfer */
+} sd_cmd_type_t;
+
+typedef struct {
+    uint8_t cmd;
+    uint32_t arg;
+    uint8_t crc;
+} SDRequest;
+
+typedef struct SDState SDState;
+typedef struct SDBus SDBus;
+
+#define TYPE_SD_CARD "sd-card"
+#define SD_CARD(obj) OBJECT_CHECK(SDState, (obj), TYPE_SD_CARD)
+#define SD_CARD_CLASS(klass) \
+    OBJECT_CLASS_CHECK(SDCardClass, (klass), TYPE_SD_CARD)
+#define SD_CARD_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(SDCardClass, (obj), TYPE_SD_CARD)
+
+typedef struct {
+    /*< private >*/
+    DeviceClass parent_class;
+    /*< public >*/
+
+    int (*do_command)(SDState *sd, SDRequest *req, uint8_t *response);
+    void (*write_data)(SDState *sd, uint8_t value);
+    uint8_t (*read_data)(SDState *sd);
+    bool (*data_ready)(SDState *sd);
+    void (*enable)(SDState *sd, bool enable);
+    bool (*get_inserted)(SDState *sd);
+    bool (*get_readonly)(SDState *sd);
+} SDCardClass;
+
+#define TYPE_SD_BUS "sd-bus"
+#define SD_BUS(obj) OBJECT_CHECK(SDBus, (obj), TYPE_SD_BUS)
+#define SD_BUS_CLASS(klass) OBJECT_CLASS_CHECK(SDBusClass, (klass), TYPE_SD_BUS)
+#define SD_BUS_GET_CLASS(obj) OBJECT_GET_CLASS(SDBusClass, (obj), TYPE_SD_BUS)
+
+struct SDBus {
+    BusState qbus;
+};
+
+typedef struct {
+    /*< private >*/
+    BusClass parent_class;
+    /*< public >*/
+
+    /* These methods are called by the SD device to notify the controller
+     * when the card insertion or readonly status changes
+     */
+    void (*set_inserted)(DeviceState *dev, bool inserted);
+    void (*set_readonly)(DeviceState *dev, bool readonly);
+} SDBusClass;
+
+/* Legacy functions to be used only by non-qdevified callers */
+SDState *sd_init(BlockBackend *bs, bool is_spi);
+int sd_do_command(SDState *sd, SDRequest *req,
+                  uint8_t *response);
+void sd_write_data(SDState *sd, uint8_t value);
+uint8_t sd_read_data(SDState *sd);
+void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert);
+bool sd_data_ready(SDState *sd);
+/* sd_enable should not be used -- it is only used on the nseries boards,
+ * where it is part of a broken implementation of the MMC card slot switch
+ * (there should be two card slots which are multiplexed to a single MMC
+ * controller, but instead we model it with one card and controller and
+ * disable the card when the second slot is selected, so it looks like the
+ * second slot is always empty).
+ */
+void sd_enable(SDState *sd, bool enable);
+
+/* Functions to be used by qdevified callers (working via
+ * an SDBus rather than directly with SDState)
+ */
+int sdbus_do_command(SDBus *sd, SDRequest *req, uint8_t *response);
+void sdbus_write_data(SDBus *sd, uint8_t value);
+uint8_t sdbus_read_data(SDBus *sd);
+bool sdbus_data_ready(SDBus *sd);
+bool sdbus_get_inserted(SDBus *sd);
+bool sdbus_get_readonly(SDBus *sd);
+
+/* Functions to be used by SD devices to report back to qdevified controllers */
+void sdbus_set_inserted(SDBus *sd, bool inserted);
+void sdbus_set_readonly(SDBus *sd, bool inserted);
+
+#endif /* HW_SD_H */
diff --git a/include/hw/sd/sdhci.h b/include/hw/sd/sdhci.h
new file mode 100644
index 0000000..0f0c3f1
--- /dev/null
+++ b/include/hw/sd/sdhci.h
@@ -0,0 +1,95 @@
+/*
+ * SD Association Host Standard Specification v2.0 controller emulation
+ *
+ * Copyright (c) 2011 Samsung Electronics Co., Ltd.
+ * Mitsyanko Igor <i.mitsyanko@samsung.com>
+ * Peter A.G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ *
+ * Based on MMC controller for Samsung S5PC1xx-based board emulation
+ * by Alexey Merkulov and Vladimir Monakhov.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU _General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef SDHCI_H
+#define SDHCI_H
+
+#include "qemu-common.h"
+#include "hw/block/block.h"
+#include "hw/pci/pci.h"
+#include "hw/sysbus.h"
+#include "hw/sd/sd.h"
+
+/* SD/MMC host controller state */
+typedef struct SDHCIState {
+    union {
+        PCIDevice pcidev;
+        SysBusDevice busdev;
+    };
+    SDBus sdbus;
+    MemoryRegion iomem;
+
+    QEMUTimer *insert_timer;       /* timer for 'changing' sd card. */
+    QEMUTimer *transfer_timer;
+    qemu_irq eject_cb;
+    qemu_irq ro_cb;
+    qemu_irq irq;
+
+    uint32_t sdmasysad;    /* SDMA System Address register */
+    uint16_t blksize;      /* Host DMA Buff Boundary and Transfer BlkSize Reg */
+    uint16_t blkcnt;       /* Blocks count for current transfer */
+    uint32_t argument;     /* Command Argument Register */
+    uint16_t trnmod;       /* Transfer Mode Setting Register */
+    uint16_t cmdreg;       /* Command Register */
+    uint32_t rspreg[4];    /* Response Registers 0-3 */
+    uint32_t prnsts;       /* Present State Register */
+    uint8_t  hostctl;      /* Host Control Register */
+    uint8_t  pwrcon;       /* Power control Register */
+    uint8_t  blkgap;       /* Block Gap Control Register */
+    uint8_t  wakcon;       /* WakeUp Control Register */
+    uint16_t clkcon;       /* Clock control Register */
+    uint8_t  timeoutcon;   /* Timeout Control Register */
+    uint8_t  admaerr;      /* ADMA Error Status Register */
+    uint16_t norintsts;    /* Normal Interrupt Status Register */
+    uint16_t errintsts;    /* Error Interrupt Status Register */
+    uint16_t norintstsen;  /* Normal Interrupt Status Enable Register */
+    uint16_t errintstsen;  /* Error Interrupt Status Enable Register */
+    uint16_t norintsigen;  /* Normal Interrupt Signal Enable Register */
+    uint16_t errintsigen;  /* Error Interrupt Signal Enable Register */
+    uint16_t acmd12errsts; /* Auto CMD12 error status register */
+    uint64_t admasysaddr;  /* ADMA System Address Register */
+
+    uint32_t capareg;      /* Capabilities Register */
+    uint32_t maxcurr;      /* Maximum Current Capabilities Register */
+    uint8_t  *fifo_buffer; /* SD host i/o FIFO buffer */
+    uint32_t buf_maxsz;
+    uint16_t data_count;   /* current element in FIFO buffer */
+    uint8_t  stopped_state;/* Current SDHC state */
+    bool     pending_insert_quirk;/* Quirk for Raspberry Pi card insert int */
+    bool     pending_insert_state;
+    /* Buffer Data Port Register - virtual access point to R and W buffers */
+    /* Software Reset Register - always reads as 0 */
+    /* Force Event Auto CMD12 Error Interrupt Reg - write only */
+    /* Force Event Error Interrupt Register- write only */
+    /* RO Host Controller Version Register always reads as 0x2401 */
+} SDHCIState;
+
+#define TYPE_PCI_SDHCI "sdhci-pci"
+#define PCI_SDHCI(obj) OBJECT_CHECK(SDHCIState, (obj), TYPE_PCI_SDHCI)
+
+#define TYPE_SYSBUS_SDHCI "generic-sdhci"
+#define SYSBUS_SDHCI(obj)                               \
+     OBJECT_CHECK(SDHCIState, (obj), TYPE_SYSBUS_SDHCI)
+
+#endif /* SDHCI_H */
diff --git a/include/hw/sh4/sh.h b/include/hw/sh4/sh.h
index e61de9a..070312d 100644
--- a/include/hw/sh4/sh.h
+++ b/include/hw/sh4/sh.h
@@ -3,6 +3,7 @@
 /* Definitions for SH board emulation.  */
 
 #include "hw/sh4/sh_intc.h"
+#include "target-sh4/cpu-qom.h"
 
 #define A7ADDR(x) ((x) & 0x1fffffff)
 #define P4ADDR(x) ((x) | 0xe0000000)
diff --git a/include/hw/sh4/sh_intc.h b/include/hw/sh4/sh_intc.h
index b7ddcb0..7913bc4 100644
--- a/include/hw/sh4/sh_intc.h
+++ b/include/hw/sh4/sh_intc.h
@@ -1,5 +1,5 @@
-#ifndef __SH_INTC_H__
-#define __SH_INTC_H__
+#ifndef SH_INTC_H
+#define SH_INTC_H
 
 #include "qemu-common.h"
 #include "hw/irq.h"
@@ -80,4 +80,4 @@
 
 void sh_intc_set_irl(void *opaque, int n, int level);
 
-#endif /* __SH_INTC_H__ */
+#endif /* SH_INTC_H */
diff --git a/include/hw/smbios/ipmi.h b/include/hw/smbios/ipmi.h
new file mode 100644
index 0000000..1c9aae3
--- /dev/null
+++ b/include/hw/smbios/ipmi.h
@@ -0,0 +1,15 @@
+/*
+ * IPMI SMBIOS firmware handling
+ *
+ * Copyright (c) 2015,2016 Corey Minyard, MontaVista Software, LLC
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_SMBIOS_IPMI_H
+#define QEMU_SMBIOS_IPMI_H
+
+void smbios_build_type_38_table(void);
+
+#endif /* QEMU_SMBIOS_IPMI_H */
diff --git a/include/hw/smbios/smbios.h b/include/hw/smbios/smbios.h
new file mode 100644
index 0000000..1cd53cc
--- /dev/null
+++ b/include/hw/smbios/smbios.h
@@ -0,0 +1,270 @@
+#ifndef QEMU_SMBIOS_H
+#define QEMU_SMBIOS_H
+
+/*
+ * SMBIOS Support
+ *
+ * Copyright (C) 2009 Hewlett-Packard Development Company, L.P.
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@hp.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/option.h"
+
+#define SMBIOS_MAX_TYPE 127
+
+/* memory area description, used by type 19 table */
+struct smbios_phys_mem_area {
+    uint64_t address;
+    uint64_t length;
+};
+
+/*
+ * SMBIOS spec defined tables
+ */
+typedef enum SmbiosEntryPointType {
+    SMBIOS_ENTRY_POINT_21,
+    SMBIOS_ENTRY_POINT_30,
+} SmbiosEntryPointType;
+
+/* SMBIOS Entry Point
+ * There are two types of entry points defined in the SMBIOS specification
+ * (see below). BIOS must place the entry point(s) at a 16-byte-aligned
+ * address between 0xf0000 and 0xfffff. Note that either entry point type
+ * can be used in a 64-bit target system, except that SMBIOS 2.1 entry point
+ * only allows the SMBIOS struct table to reside below 4GB address space.
+ */
+
+/* SMBIOS 2.1 (32-bit) Entry Point
+ *  - introduced since SMBIOS 2.1
+ *  - supports structure table below 4GB only
+ */
+struct smbios_21_entry_point {
+    uint8_t anchor_string[4];
+    uint8_t checksum;
+    uint8_t length;
+    uint8_t smbios_major_version;
+    uint8_t smbios_minor_version;
+    uint16_t max_structure_size;
+    uint8_t entry_point_revision;
+    uint8_t formatted_area[5];
+    uint8_t intermediate_anchor_string[5];
+    uint8_t intermediate_checksum;
+    uint16_t structure_table_length;
+    uint32_t structure_table_address;
+    uint16_t number_of_structures;
+    uint8_t smbios_bcd_revision;
+} QEMU_PACKED;
+
+/* SMBIOS 3.0 (64-bit) Entry Point
+ *  - introduced since SMBIOS 3.0
+ *  - supports structure table at 64-bit address space
+ */
+struct smbios_30_entry_point {
+    uint8_t anchor_string[5];
+    uint8_t checksum;
+    uint8_t length;
+    uint8_t smbios_major_version;
+    uint8_t smbios_minor_version;
+    uint8_t smbios_doc_rev;
+    uint8_t entry_point_revision;
+    uint8_t reserved;
+    uint32_t structure_table_max_size;
+    uint64_t structure_table_address;
+} QEMU_PACKED;
+
+typedef union {
+    struct smbios_21_entry_point ep21;
+    struct smbios_30_entry_point ep30;
+} QEMU_PACKED SmbiosEntryPoint;
+
+/* This goes at the beginning of every SMBIOS structure. */
+struct smbios_structure_header {
+    uint8_t type;
+    uint8_t length;
+    uint16_t handle;
+} QEMU_PACKED;
+
+/* SMBIOS type 0 - BIOS Information */
+struct smbios_type_0 {
+    struct smbios_structure_header header;
+    uint8_t vendor_str;
+    uint8_t bios_version_str;
+    uint16_t bios_starting_address_segment;
+    uint8_t bios_release_date_str;
+    uint8_t bios_rom_size;
+    uint64_t bios_characteristics;
+    uint8_t bios_characteristics_extension_bytes[2];
+    uint8_t system_bios_major_release;
+    uint8_t system_bios_minor_release;
+    uint8_t embedded_controller_major_release;
+    uint8_t embedded_controller_minor_release;
+} QEMU_PACKED;
+
+/* UUID encoding. The time_* fields are little-endian, as specified by SMBIOS
+ * version 2.6.
+ */
+struct smbios_uuid {
+    uint32_t time_low;
+    uint16_t time_mid;
+    uint16_t time_hi_and_version;
+    uint8_t clock_seq_hi_and_reserved;
+    uint8_t clock_seq_low;
+    uint8_t node[6];
+} QEMU_PACKED;
+
+/* SMBIOS type 1 - System Information */
+struct smbios_type_1 {
+    struct smbios_structure_header header;
+    uint8_t manufacturer_str;
+    uint8_t product_name_str;
+    uint8_t version_str;
+    uint8_t serial_number_str;
+    struct smbios_uuid uuid;
+    uint8_t wake_up_type;
+    uint8_t sku_number_str;
+    uint8_t family_str;
+} QEMU_PACKED;
+
+/* SMBIOS type 2 - Base Board */
+struct smbios_type_2 {
+    struct smbios_structure_header header;
+    uint8_t manufacturer_str;
+    uint8_t product_str;
+    uint8_t version_str;
+    uint8_t serial_number_str;
+    uint8_t asset_tag_number_str;
+    uint8_t feature_flags;
+    uint8_t location_str;
+    uint16_t chassis_handle;
+    uint8_t board_type;
+    uint8_t contained_element_count;
+    /* contained elements follow */
+} QEMU_PACKED;
+
+/* SMBIOS type 3 - System Enclosure (v2.7) */
+struct smbios_type_3 {
+    struct smbios_structure_header header;
+    uint8_t manufacturer_str;
+    uint8_t type;
+    uint8_t version_str;
+    uint8_t serial_number_str;
+    uint8_t asset_tag_number_str;
+    uint8_t boot_up_state;
+    uint8_t power_supply_state;
+    uint8_t thermal_state;
+    uint8_t security_status;
+    uint32_t oem_defined;
+    uint8_t height;
+    uint8_t number_of_power_cords;
+    uint8_t contained_element_count;
+    uint8_t sku_number_str;
+    /* contained elements follow */
+} QEMU_PACKED;
+
+/* SMBIOS type 4 - Processor Information (v2.6) */
+struct smbios_type_4 {
+    struct smbios_structure_header header;
+    uint8_t socket_designation_str;
+    uint8_t processor_type;
+    uint8_t processor_family;
+    uint8_t processor_manufacturer_str;
+    uint32_t processor_id[2];
+    uint8_t processor_version_str;
+    uint8_t voltage;
+    uint16_t external_clock;
+    uint16_t max_speed;
+    uint16_t current_speed;
+    uint8_t status;
+    uint8_t processor_upgrade;
+    uint16_t l1_cache_handle;
+    uint16_t l2_cache_handle;
+    uint16_t l3_cache_handle;
+    uint8_t serial_number_str;
+    uint8_t asset_tag_number_str;
+    uint8_t part_number_str;
+    uint8_t core_count;
+    uint8_t core_enabled;
+    uint8_t thread_count;
+    uint16_t processor_characteristics;
+    uint16_t processor_family2;
+} QEMU_PACKED;
+
+/* SMBIOS type 16 - Physical Memory Array (v2.7) */
+struct smbios_type_16 {
+    struct smbios_structure_header header;
+    uint8_t location;
+    uint8_t use;
+    uint8_t error_correction;
+    uint32_t maximum_capacity;
+    uint16_t memory_error_information_handle;
+    uint16_t number_of_memory_devices;
+    uint64_t extended_maximum_capacity;
+} QEMU_PACKED;
+
+/* SMBIOS type 17 - Memory Device (v2.8) */
+struct smbios_type_17 {
+    struct smbios_structure_header header;
+    uint16_t physical_memory_array_handle;
+    uint16_t memory_error_information_handle;
+    uint16_t total_width;
+    uint16_t data_width;
+    uint16_t size;
+    uint8_t form_factor;
+    uint8_t device_set;
+    uint8_t device_locator_str;
+    uint8_t bank_locator_str;
+    uint8_t memory_type;
+    uint16_t type_detail;
+    uint16_t speed;
+    uint8_t manufacturer_str;
+    uint8_t serial_number_str;
+    uint8_t asset_tag_number_str;
+    uint8_t part_number_str;
+    uint8_t attributes;
+    uint32_t extended_size;
+    uint16_t configured_clock_speed;
+    uint16_t minimum_voltage;
+    uint16_t maximum_voltage;
+    uint16_t configured_voltage;
+} QEMU_PACKED;
+
+/* SMBIOS type 19 - Memory Array Mapped Address (v2.7) */
+struct smbios_type_19 {
+    struct smbios_structure_header header;
+    uint32_t starting_address;
+    uint32_t ending_address;
+    uint16_t memory_array_handle;
+    uint8_t partition_width;
+    uint64_t extended_starting_address;
+    uint64_t extended_ending_address;
+} QEMU_PACKED;
+
+/* SMBIOS type 32 - System Boot Information */
+struct smbios_type_32 {
+    struct smbios_structure_header header;
+    uint8_t reserved[6];
+    uint8_t boot_status;
+} QEMU_PACKED;
+
+/* SMBIOS type 127 -- End-of-table */
+struct smbios_type_127 {
+    struct smbios_structure_header header;
+} QEMU_PACKED;
+
+void smbios_entry_add(QemuOpts *opts);
+void smbios_set_cpuid(uint32_t version, uint32_t features);
+void smbios_set_defaults(const char *manufacturer, const char *product,
+                         const char *version, bool legacy_mode,
+                         bool uuid_encoded, SmbiosEntryPointType ep_type);
+uint8_t *smbios_get_table_legacy(size_t *length);
+void smbios_get_tables(const struct smbios_phys_mem_area *mem_array,
+                       const unsigned int mem_array_size,
+                       uint8_t **tables, size_t *tables_len,
+                       uint8_t **anchor, size_t *anchor_len);
+#endif /* QEMU_SMBIOS_H */
diff --git a/include/hw/sparc/grlib.h b/include/hw/sparc/grlib.h
index 470ce72..afbb9bc 100644
--- a/include/hw/sparc/grlib.h
+++ b/include/hw/sparc/grlib.h
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef _GRLIB_H_
-#define _GRLIB_H_
+#ifndef GRLIB_H
+#define GRLIB_H
 
 #include "hw/qdev.h"
 #include "hw/sysbus.h"
@@ -55,9 +55,7 @@
     qdev_prop_set_ptr(dev, "set_pil_in", set_pil_in);
     qdev_prop_set_ptr(dev, "set_pil_in_opaque", env);
 
-    if (qdev_init(dev)) {
-        return NULL;
-    }
+    qdev_init_nofail(dev);
 
     env->irq_manager = dev;
 
@@ -87,9 +85,7 @@
     qdev_prop_set_uint32(dev, "frequency", freq);
     qdev_prop_set_uint32(dev, "irq-line", base_irq);
 
-    if (qdev_init(dev)) {
-        return NULL;
-    }
+    qdev_init_nofail(dev);
 
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
@@ -112,9 +108,7 @@
     dev = qdev_create(NULL, "grlib,apbuart");
     qdev_prop_set_chr(dev, "chrdev", serial);
 
-    if (qdev_init(dev)) {
-        return NULL;
-    }
+    qdev_init_nofail(dev);
 
     sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, base);
 
@@ -123,4 +117,4 @@
     return dev;
 }
 
-#endif /* ! _GRLIB_H_ */
+#endif /* GRLIB_H */
diff --git a/include/hw/sparc/sun4m.h b/include/hw/sparc/sun4m.h
index a587700..9c17425 100644
--- a/include/hw/sparc/sun4m.h
+++ b/include/hw/sparc/sun4m.h
@@ -29,8 +29,8 @@
 void slavio_irq_info(Monitor *mon, DeviceState *dev);
 
 /* sun4m.c */
-void sun4m_pic_info(Monitor *mon, const QDict *qdict);
-void sun4m_irq_info(Monitor *mon, const QDict *qdict);
+void sun4m_hmp_info_pic(Monitor *mon, const QDict *qdict);
+void sun4m_hmp_info_irq(Monitor *mon, const QDict *qdict);
 
 /* sparc32_dma.c */
 #include "hw/sparc/sparc32_dma.h"
diff --git a/include/hw/ssi.h b/include/hw/ssi.h
deleted file mode 100644
index df0f838..0000000
--- a/include/hw/ssi.h
+++ /dev/null
@@ -1,94 +0,0 @@
-/* QEMU Synchronous Serial Interface support.  */
-
-/* In principle SSI is a point-point interface.  As such the qemu
-   implementation has a single slave device on a "bus".
-   However it is fairly common for boards to have multiple slaves
-   connected to a single master, and select devices with an external
-   chip select.  This is implemented in qemu by having an explicit mux device.
-   It is assumed that master and slave are both using the same transfer width.
-   */
-
-#ifndef QEMU_SSI_H
-#define QEMU_SSI_H
-
-#include "hw/qdev.h"
-
-typedef struct SSISlave SSISlave;
-
-#define TYPE_SSI_SLAVE "ssi-slave"
-#define SSI_SLAVE(obj) \
-     OBJECT_CHECK(SSISlave, (obj), TYPE_SSI_SLAVE)
-#define SSI_SLAVE_CLASS(klass) \
-     OBJECT_CLASS_CHECK(SSISlaveClass, (klass), TYPE_SSI_SLAVE)
-#define SSI_SLAVE_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(SSISlaveClass, (obj), TYPE_SSI_SLAVE)
-
-#define SSI_GPIO_CS "ssi-gpio-cs"
-
-typedef enum {
-    SSI_CS_NONE = 0,
-    SSI_CS_LOW,
-    SSI_CS_HIGH,
-} SSICSMode;
-
-/* Slave devices.  */
-typedef struct SSISlaveClass {
-    DeviceClass parent_class;
-
-    int (*init)(SSISlave *dev);
-
-    /* if you have standard or no CS behaviour, just override transfer.
-     * This is called when the device cs is active (true by default).
-     */
-    uint32_t (*transfer)(SSISlave *dev, uint32_t val);
-    /* called when the CS line changes. Optional, devices only need to implement
-     * this if they have side effects associated with the cs line (beyond
-     * tristating the txrx lines).
-     */
-    int (*set_cs)(SSISlave *dev, bool select);
-    /* define whether or not CS exists and is active low/high */
-    SSICSMode cs_polarity;
-
-    /* if you have non-standard CS behaviour override this to take control
-     * of the CS behaviour at the device level. transfer, set_cs, and
-     * cs_polarity are unused if this is overwritten. Transfer_raw will
-     * always be called for the device for every txrx access to the parent bus
-     */
-    uint32_t (*transfer_raw)(SSISlave *dev, uint32_t val);
-} SSISlaveClass;
-
-struct SSISlave {
-    DeviceState parent_obj;
-
-    /* Chip select state */
-    bool cs;
-};
-
-#define FROM_SSI_SLAVE(type, dev) DO_UPCAST(type, ssidev, dev)
-
-extern const VMStateDescription vmstate_ssi_slave;
-
-#define VMSTATE_SSI_SLAVE(_field, _state) {                          \
-    .name       = (stringify(_field)),                               \
-    .size       = sizeof(SSISlave),                                  \
-    .vmsd       = &vmstate_ssi_slave,                                \
-    .flags      = VMS_STRUCT,                                        \
-    .offset     = vmstate_offset_value(_state, _field, SSISlave),    \
-}
-
-DeviceState *ssi_create_slave(SSIBus *bus, const char *name);
-DeviceState *ssi_create_slave_no_init(SSIBus *bus, const char *name);
-
-/* Master interface.  */
-SSIBus *ssi_create_bus(DeviceState *parent, const char *name);
-
-uint32_t ssi_transfer(SSIBus *bus, uint32_t val);
-
-/* Automatically connect all children nodes a spi controller as slaves */
-void ssi_auto_connect_slaves(DeviceState *parent, qemu_irq *cs_lines,
-                             SSIBus *bus);
-
-/* max111x.c */
-void max111x_set_input(DeviceState *dev, int line, uint8_t value);
-
-#endif
diff --git a/include/hw/ssi/aspeed_smc.h b/include/hw/ssi/aspeed_smc.h
new file mode 100644
index 0000000..def3b45
--- /dev/null
+++ b/include/hw/ssi/aspeed_smc.h
@@ -0,0 +1,100 @@
+/*
+ * ASPEED AST2400 SMC Controller (SPI Flash Only)
+ *
+ * Copyright (C) 2016 IBM Corp.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef ASPEED_SMC_H
+#define ASPEED_SMC_H
+
+#include "hw/ssi/ssi.h"
+
+typedef struct AspeedSegments {
+    hwaddr addr;
+    uint32_t size;
+} AspeedSegments;
+
+struct AspeedSMCState;
+typedef struct AspeedSMCController {
+    const char *name;
+    uint8_t r_conf;
+    uint8_t r_ce_ctrl;
+    uint8_t r_ctrl0;
+    uint8_t r_timings;
+    uint8_t conf_enable_w0;
+    uint8_t max_slaves;
+    const AspeedSegments *segments;
+    uint32_t mapping_window_size;
+} AspeedSMCController;
+
+typedef struct AspeedSMCFlash {
+    const struct AspeedSMCState *controller;
+
+    uint8_t id;
+    uint32_t size;
+
+    MemoryRegion mmio;
+    DeviceState *flash;
+} AspeedSMCFlash;
+
+#define TYPE_ASPEED_SMC "aspeed.smc"
+#define ASPEED_SMC(obj) OBJECT_CHECK(AspeedSMCState, (obj), TYPE_ASPEED_SMC)
+#define ASPEED_SMC_CLASS(klass) \
+     OBJECT_CLASS_CHECK(AspeedSMCClass, (klass), TYPE_ASPEED_SMC)
+#define ASPEED_SMC_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(AspeedSMCClass, (obj), TYPE_ASPEED_SMC)
+
+typedef struct  AspeedSMCClass {
+    SysBusDevice parent_obj;
+    const AspeedSMCController *ctrl;
+}  AspeedSMCClass;
+
+#define ASPEED_SMC_R_MAX        (0x100 / 4)
+
+typedef struct AspeedSMCState {
+    SysBusDevice parent_obj;
+
+    const AspeedSMCController *ctrl;
+
+    MemoryRegion mmio;
+    MemoryRegion mmio_flash;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint32_t num_cs;
+    qemu_irq *cs_lines;
+
+    SSIBus *spi;
+
+    uint32_t regs[ASPEED_SMC_R_MAX];
+
+    /* depends on the controller type */
+    uint8_t r_conf;
+    uint8_t r_ce_ctrl;
+    uint8_t r_ctrl0;
+    uint8_t r_timings;
+    uint8_t conf_enable_w0;
+
+    AspeedSMCFlash *flashes;
+} AspeedSMCState;
+
+#endif /* ASPEED_SMC_H */
diff --git a/include/hw/ssi/imx_spi.h b/include/hw/ssi/imx_spi.h
new file mode 100644
index 0000000..7103953
--- /dev/null
+++ b/include/hw/ssi/imx_spi.h
@@ -0,0 +1,103 @@
+/*
+ * IMX SPI Controller
+ *
+ * Copyright 2016 Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IMX_SPI_H
+#define IMX_SPI_H
+
+#include "hw/sysbus.h"
+#include "hw/ssi/ssi.h"
+#include "qemu/bitops.h"
+#include "qemu/fifo32.h"
+
+#define ECSPI_FIFO_SIZE 64
+
+#define ECSPI_RXDATA 0
+#define ECSPI_TXDATA 1
+#define ECSPI_CONREG 2
+#define ECSPI_CONFIGREG 3
+#define ECSPI_INTREG 4
+#define ECSPI_DMAREG 5
+#define ECSPI_STATREG 6
+#define ECSPI_PERIODREG 7
+#define ECSPI_TESTREG 8
+#define ECSPI_MSGDATA 16
+#define ECSPI_MAX 17
+
+/* ECSPI_CONREG */
+#define ECSPI_CONREG_EN (1 << 0)
+#define ECSPI_CONREG_HT (1 << 1)
+#define ECSPI_CONREG_XCH (1 << 2)
+#define ECSPI_CONREG_SMC (1 << 3)
+#define ECSPI_CONREG_CHANNEL_MODE_SHIFT 4
+#define ECSPI_CONREG_CHANNEL_MODE_LENGTH 4
+#define ECSPI_CONREG_DRCTL_SHIFT 16
+#define ECSPI_CONREG_DRCTL_LENGTH 2
+#define ECSPI_CONREG_CHANNEL_SELECT_SHIFT 18
+#define ECSPI_CONREG_CHANNEL_SELECT_LENGTH 2
+#define ECSPI_CONREG_BURST_LENGTH_SHIFT 20
+#define ECSPI_CONREG_BURST_LENGTH_LENGTH 12
+
+/* ECSPI_CONFIGREG */
+#define ECSPI_CONFIGREG_SS_CTL_SHIFT 8
+#define ECSPI_CONFIGREG_SS_CTL_LENGTH 4
+
+/* ECSPI_INTREG */
+#define ECSPI_INTREG_TEEN (1 << 0)
+#define ECSPI_INTREG_TDREN (1 << 1)
+#define ECSPI_INTREG_TFEN (1 << 2)
+#define ECSPI_INTREG_RREN (1 << 3)
+#define ECSPI_INTREG_RDREN (1 << 4)
+#define ECSPI_INTREG_RFEN (1 << 5)
+#define ECSPI_INTREG_ROEN (1 << 6)
+#define ECSPI_INTREG_TCEN (1 << 7)
+
+/* ECSPI_DMAREG */
+#define ECSPI_DMAREG_RXTDEN (1 << 31)
+#define ECSPI_DMAREG_RXDEN (1 << 23)
+#define ECSPI_DMAREG_TEDEN (1 << 7)
+#define ECSPI_DMAREG_RX_THRESHOLD_SHIFT 16
+#define ECSPI_DMAREG_RX_THRESHOLD_LENGTH 6
+
+/* ECSPI_STATREG */
+#define ECSPI_STATREG_TE (1 << 0)
+#define ECSPI_STATREG_TDR (1 << 1)
+#define ECSPI_STATREG_TF (1 << 2)
+#define ECSPI_STATREG_RR (1 << 3)
+#define ECSPI_STATREG_RDR (1 << 4)
+#define ECSPI_STATREG_RF (1 << 5)
+#define ECSPI_STATREG_RO (1 << 6)
+#define ECSPI_STATREG_TC (1 << 7)
+
+#define EXTRACT(value, name) extract32(value, name##_SHIFT, name##_LENGTH)
+
+#define TYPE_IMX_SPI "imx.spi"
+#define IMX_SPI(obj) OBJECT_CHECK(IMXSPIState, (obj), TYPE_IMX_SPI)
+
+typedef struct IMXSPIState {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+
+    qemu_irq irq;
+
+    qemu_irq cs_lines[4];
+
+    SSIBus *bus;
+
+    uint32_t regs[ECSPI_MAX];
+
+    Fifo32 rx_fifo;
+    Fifo32 tx_fifo;
+
+    int16_t burst_length;
+} IMXSPIState;
+
+#endif /* IMX_SPI_H */
diff --git a/include/hw/ssi/ssi.h b/include/hw/ssi/ssi.h
new file mode 100644
index 0000000..6a0c3c3
--- /dev/null
+++ b/include/hw/ssi/ssi.h
@@ -0,0 +1,96 @@
+/* QEMU Synchronous Serial Interface support.  */
+
+/* In principle SSI is a point-point interface.  As such the qemu
+   implementation has a single slave device on a "bus".
+   However it is fairly common for boards to have multiple slaves
+   connected to a single master, and select devices with an external
+   chip select.  This is implemented in qemu by having an explicit mux device.
+   It is assumed that master and slave are both using the same transfer width.
+   */
+
+#ifndef QEMU_SSI_H
+#define QEMU_SSI_H
+
+#include "hw/qdev.h"
+
+typedef struct SSISlave SSISlave;
+typedef struct SSISlaveClass SSISlaveClass;
+typedef enum SSICSMode SSICSMode;
+
+#define TYPE_SSI_SLAVE "ssi-slave"
+#define SSI_SLAVE(obj) \
+     OBJECT_CHECK(SSISlave, (obj), TYPE_SSI_SLAVE)
+#define SSI_SLAVE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(SSISlaveClass, (klass), TYPE_SSI_SLAVE)
+#define SSI_SLAVE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(SSISlaveClass, (obj), TYPE_SSI_SLAVE)
+
+#define SSI_GPIO_CS "ssi-gpio-cs"
+
+enum SSICSMode {
+    SSI_CS_NONE = 0,
+    SSI_CS_LOW,
+    SSI_CS_HIGH,
+};
+
+/* Slave devices.  */
+struct SSISlaveClass {
+    DeviceClass parent_class;
+
+    void (*realize)(SSISlave *dev, Error **errp);
+
+    /* if you have standard or no CS behaviour, just override transfer.
+     * This is called when the device cs is active (true by default).
+     */
+    uint32_t (*transfer)(SSISlave *dev, uint32_t val);
+    /* called when the CS line changes. Optional, devices only need to implement
+     * this if they have side effects associated with the cs line (beyond
+     * tristating the txrx lines).
+     */
+    int (*set_cs)(SSISlave *dev, bool select);
+    /* define whether or not CS exists and is active low/high */
+    SSICSMode cs_polarity;
+
+    /* if you have non-standard CS behaviour override this to take control
+     * of the CS behaviour at the device level. transfer, set_cs, and
+     * cs_polarity are unused if this is overwritten. Transfer_raw will
+     * always be called for the device for every txrx access to the parent bus
+     */
+    uint32_t (*transfer_raw)(SSISlave *dev, uint32_t val);
+};
+
+struct SSISlave {
+    DeviceState parent_obj;
+
+    /* Chip select state */
+    bool cs;
+};
+
+#define FROM_SSI_SLAVE(type, dev) DO_UPCAST(type, ssidev, dev)
+
+extern const VMStateDescription vmstate_ssi_slave;
+
+#define VMSTATE_SSI_SLAVE(_field, _state) {                          \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(SSISlave),                                  \
+    .vmsd       = &vmstate_ssi_slave,                                \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, SSISlave),    \
+}
+
+DeviceState *ssi_create_slave(SSIBus *bus, const char *name);
+DeviceState *ssi_create_slave_no_init(SSIBus *bus, const char *name);
+
+/* Master interface.  */
+SSIBus *ssi_create_bus(DeviceState *parent, const char *name);
+
+uint32_t ssi_transfer(SSIBus *bus, uint32_t val);
+
+/* Automatically connect all children nodes a spi controller as slaves */
+void ssi_auto_connect_slaves(DeviceState *parent, qemu_irq *cs_lines,
+                             SSIBus *bus);
+
+/* max111x.c */
+void max111x_set_input(DeviceState *dev, int line, uint8_t value);
+
+#endif
diff --git a/include/hw/ssi/xilinx_spips.h b/include/hw/ssi/xilinx_spips.h
new file mode 100644
index 0000000..06aa096
--- /dev/null
+++ b/include/hw/ssi/xilinx_spips.h
@@ -0,0 +1,72 @@
+/*
+ * Header file for the Xilinx Zynq SPI controller
+ *
+ * Copyright (C) 2015 Xilinx Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef XILINX_SPIPS_H
+#define XILINX_SPIPS_H
+
+#include "hw/ssi/ssi.h"
+#include "qemu/fifo8.h"
+
+typedef struct XilinxSPIPS XilinxSPIPS;
+
+#define XLNX_SPIPS_R_MAX        (0x100 / 4)
+
+struct XilinxSPIPS {
+    SysBusDevice parent_obj;
+
+    MemoryRegion iomem;
+    MemoryRegion mmlqspi;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint8_t num_cs;
+    uint8_t num_busses;
+
+    uint8_t snoop_state;
+    qemu_irq *cs_lines;
+    SSIBus **spi;
+
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    uint8_t num_txrx_bytes;
+
+    uint32_t regs[XLNX_SPIPS_R_MAX];
+};
+
+#define TYPE_XILINX_SPIPS "xlnx.ps7-spi"
+#define TYPE_XILINX_QSPIPS "xlnx.ps7-qspi"
+
+#define XILINX_SPIPS(obj) \
+     OBJECT_CHECK(XilinxSPIPS, (obj), TYPE_XILINX_SPIPS)
+#define XILINX_SPIPS_CLASS(klass) \
+     OBJECT_CLASS_CHECK(XilinxSPIPSClass, (klass), TYPE_XILINX_SPIPS)
+#define XILINX_SPIPS_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(XilinxSPIPSClass, (obj), TYPE_XILINX_SPIPS)
+
+#define XILINX_QSPIPS(obj) \
+     OBJECT_CHECK(XilinxQSPIPS, (obj), TYPE_XILINX_QSPIPS)
+
+#endif /* XILINX_SPIPS_H */
diff --git a/include/hw/stream.h b/include/hw/stream.h
index 30ccc56..c370ba0 100644
--- a/include/hw/stream.h
+++ b/include/hw/stream.h
@@ -1,5 +1,5 @@
 #ifndef STREAM_H
-#define STREAM_H 1
+#define STREAM_H
 
 #include "qemu-common.h"
 #include "qom/object.h"
diff --git a/include/hw/sysbus.h b/include/hw/sysbus.h
index d1f3f00..e73a5b2 100644
--- a/include/hw/sysbus.h
+++ b/include/hw/sysbus.h
@@ -1,5 +1,5 @@
 #ifndef HW_SYSBUS_H
-#define HW_SYSBUS_H 1
+#define HW_SYSBUS_H
 
 /* Devices attached directly to the main system bus.  */
 
@@ -41,6 +41,24 @@
     /*< public >*/
 
     int (*init)(SysBusDevice *dev);
+
+    /*
+     * Let the sysbus device format its own non-PIO, non-MMIO unit address.
+     *
+     * Sometimes a class of SysBusDevices has neither MMIO nor PIO resources,
+     * yet instances of it would like to distinguish themselves, in
+     * OpenFirmware device paths, from other instances of the same class on the
+     * sysbus. For that end we expose this callback.
+     *
+     * The implementation is not supposed to change *@dev, or incur other
+     * observable change.
+     *
+     * The function returns a dynamically allocated string. On error, NULL
+     * should be returned; the unit address portion of the OFW node will be
+     * omitted then. (This is not considered a fatal error.)
+     */
+    char *(*explicit_ofw_unit_address)(const SysBusDevice *dev);
+    void (*connect_irq_notifier)(SysBusDevice *dev, qemu_irq irq);
 } SysBusDeviceClass;
 
 struct SysBusDevice {
@@ -54,7 +72,7 @@
         MemoryRegion *memory;
     } mmio[QDEV_MAX_MMIO];
     int num_pio;
-    pio_addr_t pio[QDEV_MAX_PIO];
+    uint32_t pio[QDEV_MAX_PIO];
 };
 
 typedef int FindSysbusDeviceFunc(SysBusDevice *sbdev, void *opaque);
@@ -63,7 +81,7 @@
 MemoryRegion *sysbus_mmio_get_region(SysBusDevice *dev, int n);
 void sysbus_init_irq(SysBusDevice *dev, qemu_irq *p);
 void sysbus_pass_irq(SysBusDevice *dev, SysBusDevice *target);
-void sysbus_init_ioports(SysBusDevice *dev, pio_addr_t ioport, pio_addr_t size);
+void sysbus_init_ioports(SysBusDevice *dev, uint32_t ioport, uint32_t size);
 
 
 bool sysbus_has_irq(SysBusDevice *dev, int n);
@@ -100,4 +118,4 @@
     return sysbus_try_create_varargs(name, addr, irq, NULL);
 }
 
-#endif /* !HW_SYSBUS_H */
+#endif /* HW_SYSBUS_H */
diff --git a/include/hw/timer/a9gtimer.h b/include/hw/timer/a9gtimer.h
index b88c02a..81c4388 100644
--- a/include/hw/timer/a9gtimer.h
+++ b/include/hw/timer/a9gtimer.h
@@ -20,8 +20,8 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef HW_TIMER_A9_GTIMER_H_H
-#define HW_TIMER_A9_GTIMER_H_H
+#ifndef A9GTIMER_H
+#define A9GTIMER_H
 
 #include "hw/sysbus.h"
 
@@ -37,7 +37,7 @@
 #define R_CONTROL_TIMER_ENABLE      (1 << 0)
 #define R_CONTROL_COMP_ENABLE       (1 << 1)
 #define R_CONTROL_IRQ_ENABLE        (1 << 2)
-#define R_CONTROL_AUTO_INCREMENT    (1 << 2)
+#define R_CONTROL_AUTO_INCREMENT    (1 << 3)
 #define R_CONTROL_PRESCALER_SHIFT   8
 #define R_CONTROL_PRESCALER_LEN     8
 #define R_CONTROL_PRESCALER_MASK    (((1 << R_CONTROL_PRESCALER_LEN) - 1) << \
@@ -94,4 +94,4 @@
     uint64_t new;
 } A9GTimerUpdate;
 
-#endif /* #ifdef HW_TIMER_A9_GTIMER_H_H */
+#endif /* A9GTIMER_H */
diff --git a/include/hw/timer/allwinner-a10-pit.h b/include/hw/timer/allwinner-a10-pit.h
index 770bdc0..c0cc3e2 100644
--- a/include/hw/timer/allwinner-a10-pit.h
+++ b/include/hw/timer/allwinner-a10-pit.h
@@ -1,5 +1,5 @@
-#ifndef AW_A10_PIT_H
-#define AW_A10_PIT_H
+#ifndef ALLWINNER_A10_PIT_H
+#define ALLWINNER_A10_PIT_H
 
 #include "hw/ptimer.h"
 
diff --git a/include/hw/timer/aspeed_timer.h b/include/hw/timer/aspeed_timer.h
new file mode 100644
index 0000000..bd6c1a7
--- /dev/null
+++ b/include/hw/timer/aspeed_timer.h
@@ -0,0 +1,60 @@
+/*
+ *  ASPEED AST2400 Timer
+ *
+ *  Andrew Jeffery <andrew@aj.id.au>
+ *
+ *  Copyright (C) 2016 IBM Corp.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, write to the Free Software Foundation, Inc.,
+ *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+#ifndef ASPEED_TIMER_H
+#define ASPEED_TIMER_H
+
+#include "qemu/timer.h"
+
+#define ASPEED_TIMER(obj) \
+    OBJECT_CHECK(AspeedTimerCtrlState, (obj), TYPE_ASPEED_TIMER);
+#define TYPE_ASPEED_TIMER "aspeed.timer"
+#define ASPEED_TIMER_NR_TIMERS 8
+
+typedef struct AspeedTimer {
+    qemu_irq irq;
+
+    uint8_t id;
+    QEMUTimer timer;
+
+    /**
+     * Track the line level as the ASPEED timers implement edge triggered
+     * interrupts, signalling with both the rising and falling edge.
+     */
+    int32_t level;
+    uint32_t reload;
+    uint32_t match[2];
+    uint64_t start;
+} AspeedTimer;
+
+typedef struct AspeedTimerCtrlState {
+    /*< private >*/
+    SysBusDevice parent;
+
+    /*< public >*/
+    MemoryRegion iomem;
+
+    uint32_t ctrl;
+    uint32_t ctrl2;
+    AspeedTimer timers[ASPEED_TIMER_NR_TIMERS];
+} AspeedTimerCtrlState;
+
+#endif /* ASPEED_TIMER_H */
diff --git a/include/hw/timer/digic-timer.h b/include/hw/timer/digic-timer.h
index ae91348..d9e67fe 100644
--- a/include/hw/timer/digic-timer.h
+++ b/include/hw/timer/digic-timer.h
@@ -19,7 +19,6 @@
 #define HW_TIMER_DIGIC_TIMER_H
 
 #include "hw/sysbus.h"
-#include "qemu/typedefs.h"
 #include "hw/ptimer.h"
 
 #define TYPE_DIGIC_TIMER "digic-timer"
diff --git a/include/hw/timer/hpet.h b/include/hw/timer/hpet.h
index 773953b..f04c4d3 100644
--- a/include/hw/timer/hpet.h
+++ b/include/hw/timer/hpet.h
@@ -10,15 +10,17 @@
  * the COPYING file in the top-level directory.
  *
  */
-#ifndef QEMU_HPET_EMUL_H
-#define QEMU_HPET_EMUL_H
+
+#ifndef HW_HPET_H
+#define HW_HPET_H
 
 #include "qom/object.h"
 
 #define HPET_BASE               0xfed00000
-#define HPET_CLK_PERIOD         10000000ULL /* 10000000 femtoseconds == 10ns*/
+#define HPET_LEN                0x400
+#define HPET_CLK_PERIOD         10 /* 10 ns*/
 
-#define FS_PER_NS 1000000
+#define FS_PER_NS 1000000       /* 1000000 femtoseconds == 1 ns */
 #define HPET_MIN_TIMERS         3
 #define HPET_MAX_TIMERS         32
 
diff --git a/include/hw/timer/i8254.h b/include/hw/timer/i8254.h
index 4349033..5adae9f 100644
--- a/include/hw/timer/i8254.h
+++ b/include/hw/timer/i8254.h
@@ -37,6 +37,14 @@
     int out;
 } PITChannelInfo;
 
+#define TYPE_PIT_COMMON "pit-common"
+#define PIT_COMMON(obj) \
+     OBJECT_CHECK(PITCommonState, (obj), TYPE_PIT_COMMON)
+#define PIT_COMMON_CLASS(klass) \
+     OBJECT_CLASS_CHECK(PITCommonClass, (klass), TYPE_PIT_COMMON)
+#define PIT_COMMON_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(PITCommonClass, (obj), TYPE_PIT_COMMON)
+
 #define TYPE_I8254 "isa-pit"
 #define TYPE_KVM_I8254 "kvm-pit"
 
@@ -72,4 +80,4 @@
 void pit_set_gate(ISADevice *dev, int channel, int val);
 void pit_get_channel_info(ISADevice *dev, int channel, PITChannelInfo *info);
 
-#endif /* !HW_I8254_H */
+#endif /* HW_I8254_H */
diff --git a/include/hw/timer/i8254_internal.h b/include/hw/timer/i8254_internal.h
index 61a1bfb..dc09cc0 100644
--- a/include/hw/timer/i8254_internal.h
+++ b/include/hw/timer/i8254_internal.h
@@ -57,14 +57,6 @@
     PITChannelState channels[3];
 } PITCommonState;
 
-#define TYPE_PIT_COMMON "pit-common"
-#define PIT_COMMON(obj) \
-     OBJECT_CHECK(PITCommonState, (obj), TYPE_PIT_COMMON)
-#define PIT_COMMON_CLASS(klass) \
-     OBJECT_CLASS_CHECK(PITCommonClass, (klass), TYPE_PIT_COMMON)
-#define PIT_COMMON_GET_CLASS(obj) \
-     OBJECT_GET_CLASS(PITCommonClass, (obj), TYPE_PIT_COMMON)
-
 typedef struct PITCommonClass {
     ISADeviceClass parent_class;
 
@@ -81,4 +73,4 @@
                                  PITChannelInfo *info);
 void pit_reset_common(PITCommonState *s);
 
-#endif /* !QEMU_I8254_INTERNAL_H */
+#endif /* QEMU_I8254_INTERNAL_H */
diff --git a/include/hw/timer/imx_epit.h b/include/hw/timer/imx_epit.h
new file mode 100644
index 0000000..0730ac3
--- /dev/null
+++ b/include/hw/timer/imx_epit.h
@@ -0,0 +1,80 @@
+/*
+ * i.MX EPIT Timer
+ *
+ * Copyright (c) 2008 OK Labs
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Peter Chubb
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef IMX_EPIT_H
+#define IMX_EPIT_H
+
+#include "hw/sysbus.h"
+#include "hw/ptimer.h"
+#include "hw/misc/imx_ccm.h"
+
+/*
+ * EPIT: Enhanced periodic interrupt timer
+ */
+
+#define CR_EN       (1 << 0)
+#define CR_ENMOD    (1 << 1)
+#define CR_OCIEN    (1 << 2)
+#define CR_RLD      (1 << 3)
+#define CR_PRESCALE_SHIFT (4)
+#define CR_PRESCALE_MASK  (0xfff)
+#define CR_SWR      (1 << 16)
+#define CR_IOVW     (1 << 17)
+#define CR_DBGEN    (1 << 18)
+#define CR_WAITEN   (1 << 19)
+#define CR_DOZEN    (1 << 20)
+#define CR_STOPEN   (1 << 21)
+#define CR_CLKSRC_SHIFT (24)
+#define CR_CLKSRC_MASK  (0x3 << CR_CLKSRC_SHIFT)
+
+#define EPIT_TIMER_MAX  0XFFFFFFFFUL
+
+#define TYPE_IMX_EPIT "imx.epit"
+#define IMX_EPIT(obj) OBJECT_CHECK(IMXEPITState, (obj), TYPE_IMX_EPIT)
+
+typedef struct IMXEPITState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    ptimer_state *timer_reload;
+    ptimer_state *timer_cmp;
+    MemoryRegion  iomem;
+    IMXCCMState  *ccm;
+
+    uint32_t cr;
+    uint32_t sr;
+    uint32_t lr;
+    uint32_t cmp;
+    uint32_t cnt;
+
+    uint32_t freq;
+    qemu_irq irq;
+} IMXEPITState;
+
+#endif /* IMX_EPIT_H */
diff --git a/include/hw/timer/imx_gpt.h b/include/hw/timer/imx_gpt.h
new file mode 100644
index 0000000..eac59b2
--- /dev/null
+++ b/include/hw/timer/imx_gpt.h
@@ -0,0 +1,115 @@
+/*
+ * i.MX GPT Timer
+ *
+ * Copyright (c) 2008 OK Labs
+ * Copyright (c) 2011 NICTA Pty Ltd
+ * Originally written by Hans Jiang
+ * Updated by Peter Chubb
+ * Updated by Jean-Christophe Dubois <jcd@tribudubois.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef IMX_GPT_H
+#define IMX_GPT_H
+
+#include "hw/sysbus.h"
+#include "hw/ptimer.h"
+#include "hw/misc/imx_ccm.h"
+
+/*
+ * GPT : General purpose timer
+ *
+ * This timer counts up continuously while it is enabled, resetting itself
+ * to 0 when it reaches GPT_TIMER_MAX (in freerun mode) or when it
+ * reaches the value of one of the ocrX (in periodic mode).
+ */
+
+#define GPT_TIMER_MAX  0XFFFFFFFFUL
+
+/* Control register.  Not all of these bits have any effect (yet) */
+#define GPT_CR_EN     (1 << 0)  /* GPT Enable */
+#define GPT_CR_ENMOD  (1 << 1)  /* GPT Enable Mode */
+#define GPT_CR_DBGEN  (1 << 2)  /* GPT Debug mode enable */
+#define GPT_CR_WAITEN (1 << 3)  /* GPT Wait Mode Enable  */
+#define GPT_CR_DOZEN  (1 << 4)  /* GPT Doze mode enable */
+#define GPT_CR_STOPEN (1 << 5)  /* GPT Stop Mode Enable */
+#define GPT_CR_CLKSRC_SHIFT (6)
+#define GPT_CR_CLKSRC_MASK  (0x7)
+
+#define GPT_CR_FRR    (1 << 9)  /* Freerun or Restart */
+#define GPT_CR_SWR    (1 << 15) /* Software Reset */
+#define GPT_CR_IM1    (3 << 16) /* Input capture channel 1 mode (2 bits) */
+#define GPT_CR_IM2    (3 << 18) /* Input capture channel 2 mode (2 bits) */
+#define GPT_CR_OM1    (7 << 20) /* Output Compare Channel 1 Mode (3 bits) */
+#define GPT_CR_OM2    (7 << 23) /* Output Compare Channel 2 Mode (3 bits) */
+#define GPT_CR_OM3    (7 << 26) /* Output Compare Channel 3 Mode (3 bits) */
+#define GPT_CR_FO1    (1 << 29) /* Force Output Compare Channel 1 */
+#define GPT_CR_FO2    (1 << 30) /* Force Output Compare Channel 2 */
+#define GPT_CR_FO3    (1 << 31) /* Force Output Compare Channel 3 */
+
+#define GPT_SR_OF1  (1 << 0)
+#define GPT_SR_OF2  (1 << 1)
+#define GPT_SR_OF3  (1 << 2)
+#define GPT_SR_ROV  (1 << 5)
+
+#define GPT_IR_OF1IE  (1 << 0)
+#define GPT_IR_OF2IE  (1 << 1)
+#define GPT_IR_OF3IE  (1 << 2)
+#define GPT_IR_ROVIE  (1 << 5)
+
+#define TYPE_IMX25_GPT "imx25.gpt"
+#define TYPE_IMX31_GPT "imx31.gpt"
+#define TYPE_IMX6_GPT "imx6.gpt"
+
+#define TYPE_IMX_GPT TYPE_IMX25_GPT
+
+#define IMX_GPT(obj) OBJECT_CHECK(IMXGPTState, (obj), TYPE_IMX_GPT)
+
+typedef struct IMXGPTState{
+    /*< private >*/
+    SysBusDevice parent_obj;
+
+    /*< public >*/
+    ptimer_state *timer;
+    MemoryRegion  iomem;
+    IMXCCMState  *ccm;
+
+    uint32_t cr;
+    uint32_t pr;
+    uint32_t sr;
+    uint32_t ir;
+    uint32_t ocr1;
+    uint32_t ocr2;
+    uint32_t ocr3;
+    uint32_t icr1;
+    uint32_t icr2;
+    uint32_t cnt;
+
+    uint32_t next_timeout;
+    uint32_t next_int;
+
+    uint32_t freq;
+
+    qemu_irq irq;
+
+    const IMXClk *clocks;
+} IMXGPTState;
+
+#endif /* IMX_GPT_H */
diff --git a/include/hw/timer/m48t59.h b/include/hw/timer/m48t59.h
index 8217522..db5e43a 100644
--- a/include/hw/timer/m48t59.h
+++ b/include/hw/timer/m48t59.h
@@ -1,37 +1,34 @@
-#ifndef NVRAM_H
-#define NVRAM_H
+#ifndef HW_M48T59_H
+#define HW_M48T59_H
 
-/* NVRAM helpers */
-typedef uint32_t (*nvram_read_t)(void *private, uint32_t addr);
-typedef void (*nvram_write_t)(void *private, uint32_t addr, uint32_t val);
-typedef struct nvram_t {
-    void *opaque;
-    nvram_read_t read_fn;
-    nvram_write_t write_fn;
-} nvram_t;
+#include "qemu-common.h"
+#include "qom/object.h"
 
-uint32_t NVRAM_get_lword (nvram_t *nvram, uint32_t addr);
-int NVRAM_get_string (nvram_t *nvram, uint8_t *dst, uint16_t addr, int max);
+#define TYPE_NVRAM "nvram"
 
-int PPC_NVRAM_set_params (nvram_t *nvram, uint16_t NVRAM_size,
-                          const char *arch,
-                          uint32_t RAM_size, int boot_device,
-                          uint32_t kernel_image, uint32_t kernel_size,
-                          const char *cmdline,
-                          uint32_t initrd_image, uint32_t initrd_size,
-                          uint32_t NVRAM_image,
-                          int width, int height, int depth);
+#define NVRAM_CLASS(klass) \
+    OBJECT_CLASS_CHECK(NvramClass, (klass), TYPE_NVRAM)
+#define NVRAM_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(NvramClass, (obj), TYPE_NVRAM)
+#define NVRAM(obj) \
+    INTERFACE_CHECK(Nvram, (obj), TYPE_NVRAM)
 
-#define TYPE_SYSBUS_M48T59 "m48t59"
+typedef struct Nvram {
+    Object parent;
+} Nvram;
 
-typedef struct M48t59State M48t59State;
+typedef struct NvramClass {
+    InterfaceClass parent;
 
-void m48t59_write (void *private, uint32_t addr, uint32_t val);
-uint32_t m48t59_read (void *private, uint32_t addr);
-void m48t59_toggle_lock (void *private, int lock);
-M48t59State *m48t59_init_isa(ISABus *bus, uint32_t io_base, uint16_t size,
-                             int type);
-M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
-                         uint32_t io_base, uint16_t size, int type);
+    uint32_t (*read)(Nvram *obj, uint32_t addr);
+    void (*write)(Nvram *obj, uint32_t addr, uint32_t val);
+    void (*toggle_lock)(Nvram *obj, int lock);
+} NvramClass;
 
-#endif /* !NVRAM_H */
+Nvram *m48t59_init_isa(ISABus *bus, uint32_t io_base, uint16_t size,
+                       int base_year, int type);
+Nvram *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
+                   uint32_t io_base, uint16_t size, int base_year,
+                   int type);
+
+#endif /* HW_M48T59_H */
diff --git a/include/hw/timer/mc146818rtc.h b/include/hw/timer/mc146818rtc.h
index eaf6497..7c8e64b 100644
--- a/include/hw/timer/mc146818rtc.h
+++ b/include/hw/timer/mc146818rtc.h
@@ -10,4 +10,4 @@
 void rtc_set_memory(ISADevice *dev, int addr, int val);
 int rtc_get_memory(ISADevice *dev, int addr);
 
-#endif /* !MC146818RTC_H */
+#endif /* MC146818RTC_H */
diff --git a/include/hw/timer/mc146818rtc_regs.h b/include/hw/timer/mc146818rtc_regs.h
index ccdee42..6ede6c8 100644
--- a/include/hw/timer/mc146818rtc_regs.h
+++ b/include/hw/timer/mc146818rtc_regs.h
@@ -21,8 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef RTC_REGS_H
-#define RTC_REGS_H
+
+#ifndef MC146818RTC_REGS_H
+#define MC146818RTC_REGS_H
 
 #define RTC_ISA_IRQ 8
 
diff --git a/include/hw/timer/mips_gictimer.h b/include/hw/timer/mips_gictimer.h
new file mode 100644
index 0000000..c8bc5d2
--- /dev/null
+++ b/include/hw/timer/mips_gictimer.h
@@ -0,0 +1,46 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 2016 Imagination Technologies
+ *
+ */
+
+#ifndef MIPS_GICTIMER_H
+#define MIPS_GICTIMER_H
+
+typedef struct MIPSGICTimerVPState MIPSGICTimerVPState;
+typedef struct MIPSGICTimerState MIPSGICTimerState;
+
+typedef void MIPSGICTimerCB(void *opaque, uint32_t vp_index);
+
+struct MIPSGICTimerVPState {
+    QEMUTimer *qtimer;
+    uint32_t vp_index;
+    uint32_t comparelo;
+    MIPSGICTimerState *gictimer;
+};
+
+struct MIPSGICTimerState {
+    void *opaque;
+    uint8_t countstop;
+    uint32_t sh_counterlo;
+    int32_t num_vps;
+    MIPSGICTimerVPState *vptimers;
+    MIPSGICTimerCB *cb;
+};
+
+uint32_t mips_gictimer_get_sh_count(MIPSGICTimerState *gic);
+void mips_gictimer_store_sh_count(MIPSGICTimerState *gic, uint64_t count);
+uint32_t mips_gictimer_get_vp_compare(MIPSGICTimerState *gictimer,
+                                      uint32_t vp_index);
+void mips_gictimer_store_vp_compare(MIPSGICTimerState *gic, uint32_t vp_index,
+                                    uint64_t compare);
+uint8_t mips_gictimer_get_countstop(MIPSGICTimerState *gic);
+void mips_gictimer_start_count(MIPSGICTimerState *gic);
+void mips_gictimer_stop_count(MIPSGICTimerState *gic);
+MIPSGICTimerState *mips_gictimer_init(void *opaque, uint32_t nvps,
+                                      MIPSGICTimerCB *cb);
+
+#endif /* MIPS_GICTIMER_H */
diff --git a/include/hw/timer/stm32f2xx_timer.h b/include/hw/timer/stm32f2xx_timer.h
new file mode 100644
index 0000000..e6a8323
--- /dev/null
+++ b/include/hw/timer/stm32f2xx_timer.h
@@ -0,0 +1,101 @@
+/*
+ * STM32F2XX Timer
+ *
+ * Copyright (c) 2014 Alistair Francis <alistair@alistair23.me>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef HW_STM32F2XX_TIMER_H
+#define HW_STM32F2XX_TIMER_H
+
+#include "hw/sysbus.h"
+#include "qemu/timer.h"
+#include "sysemu/sysemu.h"
+
+#define TIM_CR1      0x00
+#define TIM_CR2      0x04
+#define TIM_SMCR     0x08
+#define TIM_DIER     0x0C
+#define TIM_SR       0x10
+#define TIM_EGR      0x14
+#define TIM_CCMR1    0x18
+#define TIM_CCMR2    0x1C
+#define TIM_CCER     0x20
+#define TIM_CNT      0x24
+#define TIM_PSC      0x28
+#define TIM_ARR      0x2C
+#define TIM_CCR1     0x34
+#define TIM_CCR2     0x38
+#define TIM_CCR3     0x3C
+#define TIM_CCR4     0x40
+#define TIM_DCR      0x48
+#define TIM_DMAR     0x4C
+#define TIM_OR       0x50
+
+#define TIM_CR1_CEN   1
+
+#define TIM_EGR_UG 1
+
+#define TIM_CCER_CC2E   (1 << 4)
+#define TIM_CCMR1_OC2M2 (1 << 14)
+#define TIM_CCMR1_OC2M1 (1 << 13)
+#define TIM_CCMR1_OC2M0 (1 << 12)
+#define TIM_CCMR1_OC2PE (1 << 11)
+
+#define TIM_DIER_UIE  1
+
+#define TYPE_STM32F2XX_TIMER "stm32f2xx-timer"
+#define STM32F2XXTIMER(obj) OBJECT_CHECK(STM32F2XXTimerState, \
+                            (obj), TYPE_STM32F2XX_TIMER)
+
+typedef struct STM32F2XXTimerState {
+    /* <private> */
+    SysBusDevice parent_obj;
+
+    /* <public> */
+    MemoryRegion iomem;
+    QEMUTimer *timer;
+    qemu_irq irq;
+
+    int64_t tick_offset;
+    uint64_t hit_time;
+    uint64_t freq_hz;
+
+    uint32_t tim_cr1;
+    uint32_t tim_cr2;
+    uint32_t tim_smcr;
+    uint32_t tim_dier;
+    uint32_t tim_sr;
+    uint32_t tim_egr;
+    uint32_t tim_ccmr1;
+    uint32_t tim_ccmr2;
+    uint32_t tim_ccer;
+    uint32_t tim_psc;
+    uint32_t tim_arr;
+    uint32_t tim_ccr1;
+    uint32_t tim_ccr2;
+    uint32_t tim_ccr3;
+    uint32_t tim_ccr4;
+    uint32_t tim_dcr;
+    uint32_t tim_dmar;
+    uint32_t tim_or;
+} STM32F2XXTimerState;
+
+#endif /* HW_STM32F2XX_TIMER_H */
diff --git a/include/hw/tricore/tricore.h b/include/hw/tricore/tricore.h
index 5f13252..89ef922 100644
--- a/include/hw/tricore/tricore.h
+++ b/include/hw/tricore/tricore.h
@@ -1,5 +1,5 @@
-#ifndef TRICORE_MISC_H
-#define TRICORE_MISC_H 1
+#ifndef HW_TRICORE_H
+#define HW_TRICORE_H
 
 #include "exec/memory.h"
 #include "hw/irq.h"
diff --git a/include/hw/unicore32/puv3.h b/include/hw/unicore32/puv3.h
index f37adcb..5a4839f 100644
--- a/include/hw/unicore32/puv3.h
+++ b/include/hw/unicore32/puv3.h
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+
 #ifndef QEMU_HW_PUV3_H
 #define QEMU_HW_PUV3_H
 
@@ -46,4 +47,4 @@
 #define DPRINTF(fmt, ...) do {} while (0)
 #endif
 
-#endif /* !QEMU_HW_PUV3_H */
+#endif /* QEMU_HW_PUV3_H */
diff --git a/include/hw/usb.h b/include/hw/usb.h
index b20b959..847c9de 100644
--- a/include/hw/usb.h
+++ b/include/hw/usb.h
@@ -26,6 +26,7 @@
  */
 
 #include "hw/qdev.h"
+#include "qemu/iov.h"
 #include "qemu/queue.h"
 
 /* Constants related to the USB / PCI interaction */
@@ -234,7 +235,7 @@
     uint8_t addr;
     char product_desc[32];
     int auto_attach;
-    int attached;
+    bool attached;
 
     int32_t state;
     uint8_t setup_buf[8];
@@ -346,6 +347,7 @@
 
     const char *product_desc;
     const USBDesc *usb_desc;
+    bool attached_settable;
 } USBDeviceClass;
 
 typedef struct USBPortOps {
@@ -445,15 +447,11 @@
 void usb_ep_dump(USBDevice *dev);
 struct USBEndpoint *usb_ep_get(USBDevice *dev, int pid, int ep);
 uint8_t usb_ep_get_type(USBDevice *dev, int pid, int ep);
-uint8_t usb_ep_get_ifnum(USBDevice *dev, int pid, int ep);
 void usb_ep_set_type(USBDevice *dev, int pid, int ep, uint8_t type);
 void usb_ep_set_ifnum(USBDevice *dev, int pid, int ep, uint8_t ifnum);
 void usb_ep_set_max_packet_size(USBDevice *dev, int pid, int ep,
                                 uint16_t raw);
-int usb_ep_get_max_packet_size(USBDevice *dev, int pid, int ep);
 void usb_ep_set_max_streams(USBDevice *dev, int pid, int ep, uint8_t raw);
-int usb_ep_get_max_streams(USBDevice *dev, int pid, int ep);
-void usb_ep_set_pipeline(USBDevice *dev, int pid, int ep, bool enabled);
 void usb_ep_set_halted(USBDevice *dev, int pid, int ep, bool halted);
 USBPacket *usb_ep_find_packet_by_id(USBDevice *dev, int pid, int ep,
                                     uint64_t id);
@@ -469,11 +467,10 @@
 void usb_device_reset(USBDevice *dev);
 void usb_wakeup(USBEndpoint *ep, unsigned int stream);
 void usb_generic_async_ctrl_complete(USBDevice *s, USBPacket *p);
-int set_usb_string(uint8_t *buf, const char *str);
 
 /* usb-linux.c */
 USBDevice *usb_host_device_open(USBBus *bus, const char *devname);
-void usb_host_info(Monitor *mon, const QDict *qdict);
+void hmp_info_usbhost(Monitor *mon, const QDict *qdict);
 
 /* usb ports of the VM */
 
@@ -526,8 +523,9 @@
 };
 
 struct USBBusOps {
-    int (*register_companion)(USBBus *bus, USBPort *ports[],
-                              uint32_t portcount, uint32_t firstport);
+    void (*register_companion)(USBBus *bus, USBPort *ports[],
+                               uint32_t portcount, uint32_t firstport,
+                               Error **errp);
     void (*wakeup_endpoint)(USBBus *bus, USBEndpoint *ep, unsigned int stream);
 };
 
@@ -543,9 +541,10 @@
 USBDevice *usbdevice_create(const char *cmdline);
 void usb_register_port(USBBus *bus, USBPort *port, void *opaque, int index,
                        USBPortOps *ops, int speedmask);
-int usb_register_companion(const char *masterbus, USBPort *ports[],
-                           uint32_t portcount, uint32_t firstport,
-                           void *opaque, USBPortOps *ops, int speedmask);
+void usb_register_companion(const char *masterbus, USBPort *ports[],
+                            uint32_t portcount, uint32_t firstport,
+                            void *opaque, USBPortOps *ops, int speedmask,
+                            Error **errp);
 void usb_port_location(USBPort *downstream, USBPort *upstream, int portnr);
 void usb_unregister_port(USBBus *bus, USBPort *port);
 void usb_claim_port(USBDevice *dev, Error **errp);
diff --git a/include/hw/usb/ehci-regs.h b/include/hw/usb/ehci-regs.h
index 616f1b8..3e91b8e 100644
--- a/include/hw/usb/ehci-regs.h
+++ b/include/hw/usb/ehci-regs.h
@@ -1,5 +1,5 @@
 #ifndef HW_USB_EHCI_REGS_H
-#define HW_USB_EHCI_REGS_H 1
+#define HW_USB_EHCI_REGS_H
 
 /* Capability Registers Base Address - section 2.2 */
 #define CAPLENGTH        0x0000  /* 1-byte, 0x0001 reserved */
diff --git a/include/hw/usb/uhci-regs.h b/include/hw/usb/uhci-regs.h
index c7315c5..fd45d29 100644
--- a/include/hw/usb/uhci-regs.h
+++ b/include/hw/usb/uhci-regs.h
@@ -1,5 +1,5 @@
 #ifndef HW_USB_UHCI_REGS_H
-#define HW_USB_UHCI_REGS_H 1
+#define HW_USB_UHCI_REGS_H
 
 #define UHCI_CMD_FGR      (1 << 4)
 #define UHCI_CMD_EGSM     (1 << 3)
diff --git a/include/hw/vfio/vfio-amd-xgbe.h b/include/hw/vfio/vfio-amd-xgbe.h
new file mode 100644
index 0000000..9fff65e
--- /dev/null
+++ b/include/hw/vfio/vfio-amd-xgbe.h
@@ -0,0 +1,51 @@
+/*
+ * VFIO AMD XGBE device
+ *
+ * Copyright Linaro Limited, 2015
+ *
+ * Authors:
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VFIO_VFIO_AMD_XGBE_H
+#define HW_VFIO_VFIO_AMD_XGBE_H
+
+#include "hw/vfio/vfio-platform.h"
+
+#define TYPE_VFIO_AMD_XGBE "vfio-amd-xgbe"
+
+/**
+ * This device exposes:
+ * - 5 MMIO regions: MAC, PCS, SerDes Rx/Tx regs,
+     SerDes Integration Registers 1/2 & 2/2
+ * - 2 level sensitive IRQs and optional DMA channel IRQs
+ */
+struct VFIOAmdXgbeDevice {
+    VFIOPlatformDevice vdev;
+};
+
+typedef struct VFIOAmdXgbeDevice VFIOAmdXgbeDevice;
+
+struct VFIOAmdXgbeDeviceClass {
+    /*< private >*/
+    VFIOPlatformDeviceClass parent_class;
+    /*< public >*/
+    DeviceRealize parent_realize;
+};
+
+typedef struct VFIOAmdXgbeDeviceClass VFIOAmdXgbeDeviceClass;
+
+#define VFIO_AMD_XGBE_DEVICE(obj) \
+     OBJECT_CHECK(VFIOAmdXgbeDevice, (obj), TYPE_VFIO_AMD_XGBE)
+#define VFIO_AMD_XGBE_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(VFIOAmdXgbeDeviceClass, (klass), \
+                        TYPE_VFIO_AMD_XGBE)
+#define VFIO_AMD_XGBE_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(VFIOAmdXgbeDeviceClass, (obj), \
+                      TYPE_VFIO_AMD_XGBE)
+
+#endif
diff --git a/include/hw/vfio/vfio-calxeda-xgmac.h b/include/hw/vfio/vfio-calxeda-xgmac.h
new file mode 100644
index 0000000..f994775
--- /dev/null
+++ b/include/hw/vfio/vfio-calxeda-xgmac.h
@@ -0,0 +1,46 @@
+/*
+ * VFIO calxeda xgmac device
+ *
+ * Copyright Linaro Limited, 2014
+ *
+ * Authors:
+ *  Eric Auger <eric.auger@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_VFIO_VFIO_CALXEDA_XGMAC_H
+#define HW_VFIO_VFIO_CALXEDA_XGMAC_H
+
+#include "hw/vfio/vfio-platform.h"
+
+#define TYPE_VFIO_CALXEDA_XGMAC "vfio-calxeda-xgmac"
+
+/**
+ * This device exposes:
+ * - a single MMIO region corresponding to its register space
+ * - 3 IRQS (main and 2 power related IRQs)
+ */
+typedef struct VFIOCalxedaXgmacDevice {
+    VFIOPlatformDevice vdev;
+} VFIOCalxedaXgmacDevice;
+
+typedef struct VFIOCalxedaXgmacDeviceClass {
+    /*< private >*/
+    VFIOPlatformDeviceClass parent_class;
+    /*< public >*/
+    DeviceRealize parent_realize;
+} VFIOCalxedaXgmacDeviceClass;
+
+#define VFIO_CALXEDA_XGMAC_DEVICE(obj) \
+     OBJECT_CHECK(VFIOCalxedaXgmacDevice, (obj), TYPE_VFIO_CALXEDA_XGMAC)
+#define VFIO_CALXEDA_XGMAC_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(VFIOCalxedaXgmacDeviceClass, (klass), \
+                        TYPE_VFIO_CALXEDA_XGMAC)
+#define VFIO_CALXEDA_XGMAC_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(VFIOCalxedaXgmacDeviceClass, (obj), \
+                      TYPE_VFIO_CALXEDA_XGMAC)
+
+#endif
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
new file mode 100644
index 0000000..94dfae3
--- /dev/null
+++ b/include/hw/vfio/vfio-common.h
@@ -0,0 +1,178 @@
+/*
+ * common header for vfio based device assignment support
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on qemu-kvm device-assignment:
+ *  Adapted for KVM by Qumranet.
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+
+#ifndef HW_VFIO_VFIO_COMMON_H
+#define HW_VFIO_VFIO_COMMON_H
+
+#include "qemu-common.h"
+#include "exec/address-spaces.h"
+#include "exec/memory.h"
+#include "qemu/queue.h"
+#include "qemu/notify.h"
+#ifdef CONFIG_LINUX
+#include <linux/vfio.h>
+#endif
+
+/*#define DEBUG_VFIO*/
+#ifdef DEBUG_VFIO
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+enum {
+    VFIO_DEVICE_TYPE_PCI = 0,
+    VFIO_DEVICE_TYPE_PLATFORM = 1,
+};
+
+typedef struct VFIOMmap {
+    MemoryRegion mem;
+    void *mmap;
+    off_t offset;
+    size_t size;
+} VFIOMmap;
+
+typedef struct VFIORegion {
+    struct VFIODevice *vbasedev;
+    off_t fd_offset; /* offset of region within device fd */
+    MemoryRegion *mem; /* slow, read/write access */
+    size_t size;
+    uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
+    uint32_t nr_mmaps;
+    VFIOMmap *mmaps;
+    uint8_t nr; /* cache the region number for debug */
+} VFIORegion;
+
+typedef struct VFIOAddressSpace {
+    AddressSpace *as;
+    QLIST_HEAD(, VFIOContainer) containers;
+    QLIST_ENTRY(VFIOAddressSpace) list;
+} VFIOAddressSpace;
+
+struct VFIOGroup;
+
+typedef struct VFIOContainer {
+    VFIOAddressSpace *space;
+    int fd; /* /dev/vfio/vfio, empowered by the attached groups */
+    MemoryListener listener;
+    MemoryListener prereg_listener;
+    unsigned iommu_type;
+    int error;
+    bool initialized;
+    /*
+     * This assumes the host IOMMU can support only a single
+     * contiguous IOVA window.  We may need to generalize that in
+     * future
+     */
+    QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
+    QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
+    QLIST_HEAD(, VFIOGroup) group_list;
+    QLIST_ENTRY(VFIOContainer) next;
+} VFIOContainer;
+
+typedef struct VFIOGuestIOMMU {
+    VFIOContainer *container;
+    MemoryRegion *iommu;
+    hwaddr iommu_offset;
+    Notifier n;
+    QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
+} VFIOGuestIOMMU;
+
+typedef struct VFIOHostDMAWindow {
+    hwaddr min_iova;
+    hwaddr max_iova;
+    uint64_t iova_pgsizes;
+    QLIST_ENTRY(VFIOHostDMAWindow) hostwin_next;
+} VFIOHostDMAWindow;
+
+typedef struct VFIODeviceOps VFIODeviceOps;
+
+typedef struct VFIODevice {
+    QLIST_ENTRY(VFIODevice) next;
+    struct VFIOGroup *group;
+    char *sysfsdev;
+    char *name;
+    int fd;
+    int type;
+    bool reset_works;
+    bool needs_reset;
+    bool no_mmap;
+    VFIODeviceOps *ops;
+    unsigned int num_irqs;
+    unsigned int num_regions;
+    unsigned int flags;
+} VFIODevice;
+
+struct VFIODeviceOps {
+    void (*vfio_compute_needs_reset)(VFIODevice *vdev);
+    int (*vfio_hot_reset_multi)(VFIODevice *vdev);
+    void (*vfio_eoi)(VFIODevice *vdev);
+};
+
+typedef struct VFIOGroup {
+    int fd;
+    int groupid;
+    VFIOContainer *container;
+    QLIST_HEAD(, VFIODevice) device_list;
+    QLIST_ENTRY(VFIOGroup) next;
+    QLIST_ENTRY(VFIOGroup) container_next;
+} VFIOGroup;
+
+void vfio_put_base_device(VFIODevice *vbasedev);
+void vfio_disable_irqindex(VFIODevice *vbasedev, int index);
+void vfio_unmask_single_irqindex(VFIODevice *vbasedev, int index);
+void vfio_mask_single_irqindex(VFIODevice *vbasedev, int index);
+void vfio_region_write(void *opaque, hwaddr addr,
+                           uint64_t data, unsigned size);
+uint64_t vfio_region_read(void *opaque,
+                          hwaddr addr, unsigned size);
+int vfio_region_setup(Object *obj, VFIODevice *vbasedev, VFIORegion *region,
+                      int index, const char *name);
+int vfio_region_mmap(VFIORegion *region);
+void vfio_region_mmaps_set_enabled(VFIORegion *region, bool enabled);
+void vfio_region_exit(VFIORegion *region);
+void vfio_region_finalize(VFIORegion *region);
+void vfio_reset_handler(void *opaque);
+VFIOGroup *vfio_get_group(int groupid, AddressSpace *as);
+void vfio_put_group(VFIOGroup *group);
+int vfio_get_device(VFIOGroup *group, const char *name,
+                    VFIODevice *vbasedev);
+
+extern const MemoryRegionOps vfio_region_ops;
+extern QLIST_HEAD(vfio_group_head, VFIOGroup) vfio_group_list;
+extern QLIST_HEAD(vfio_as_head, VFIOAddressSpace) vfio_address_spaces;
+
+#ifdef CONFIG_LINUX
+int vfio_get_region_info(VFIODevice *vbasedev, int index,
+                         struct vfio_region_info **info);
+int vfio_get_dev_region_info(VFIODevice *vbasedev, uint32_t type,
+                             uint32_t subtype, struct vfio_region_info **info);
+#endif
+extern const MemoryListener vfio_prereg_listener;
+
+int vfio_spapr_create_window(VFIOContainer *container,
+                             MemoryRegionSection *section,
+                             hwaddr *pgsize);
+int vfio_spapr_remove_window(VFIOContainer *container,
+                             hwaddr offset_within_address_space);
+
+#endif /* HW_VFIO_VFIO_COMMON_H */
diff --git a/include/hw/vfio/vfio-platform.h b/include/hw/vfio/vfio-platform.h
new file mode 100644
index 0000000..9baaa2d
--- /dev/null
+++ b/include/hw/vfio/vfio-platform.h
@@ -0,0 +1,77 @@
+/*
+ * vfio based device assignment support - platform devices
+ *
+ * Copyright Linaro Limited, 2014
+ *
+ * Authors:
+ *  Kim Phillips <kim.phillips@linaro.org>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on vfio based PCI device assignment support:
+ *  Copyright Red Hat, Inc. 2012
+ */
+
+#ifndef HW_VFIO_VFIO_PLATFORM_H
+#define HW_VFIO_VFIO_PLATFORM_H
+
+#include "hw/sysbus.h"
+#include "hw/vfio/vfio-common.h"
+#include "qemu/event_notifier.h"
+#include "qemu/queue.h"
+#include "hw/irq.h"
+
+#define TYPE_VFIO_PLATFORM "vfio-platform"
+
+enum {
+    VFIO_IRQ_INACTIVE = 0,
+    VFIO_IRQ_PENDING = 1,
+    VFIO_IRQ_ACTIVE = 2,
+    /* VFIO_IRQ_ACTIVE_AND_PENDING cannot happen with VFIO */
+};
+
+typedef struct VFIOINTp {
+    QLIST_ENTRY(VFIOINTp) next; /* entry for IRQ list */
+    QSIMPLEQ_ENTRY(VFIOINTp) pqnext; /* entry for pending IRQ queue */
+    EventNotifier *interrupt; /* eventfd triggered on interrupt */
+    EventNotifier *unmask; /* eventfd for unmask on QEMU bypass */
+    qemu_irq qemuirq;
+    struct VFIOPlatformDevice *vdev; /* back pointer to device */
+    int state; /* inactive, pending, active */
+    uint8_t pin; /* index */
+    uint32_t flags; /* IRQ info flags */
+    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+} VFIOINTp;
+
+/* function type for user side eventfd handler */
+typedef void (*eventfd_user_side_handler_t)(VFIOINTp *intp);
+
+typedef struct VFIOPlatformDevice {
+    SysBusDevice sbdev;
+    VFIODevice vbasedev; /* not a QOM object */
+    VFIORegion **regions;
+    QLIST_HEAD(, VFIOINTp) intp_list; /* list of IRQs */
+    /* queue of pending IRQs */
+    QSIMPLEQ_HEAD(pending_intp_queue, VFIOINTp) pending_intp_queue;
+    char *compat; /* compatibility string */
+    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+    QEMUTimer *mmap_timer; /* allows fast-path resume after IRQ hit */
+    QemuMutex intp_mutex; /* protect the intp_list IRQ state */
+    bool irqfd_allowed; /* debug option to force irqfd on/off */
+} VFIOPlatformDevice;
+
+typedef struct VFIOPlatformDeviceClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+} VFIOPlatformDeviceClass;
+
+#define VFIO_PLATFORM_DEVICE(obj) \
+     OBJECT_CHECK(VFIOPlatformDevice, (obj), TYPE_VFIO_PLATFORM)
+#define VFIO_PLATFORM_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(VFIOPlatformDeviceClass, (klass), TYPE_VFIO_PLATFORM)
+#define VFIO_PLATFORM_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(VFIOPlatformDeviceClass, (obj), TYPE_VFIO_PLATFORM)
+
+#endif /* HW_VFIO_VFIO_PLATFORM_H */
diff --git a/include/hw/vfio/vfio.h b/include/hw/vfio/vfio.h
new file mode 100644
index 0000000..86248f5
--- /dev/null
+++ b/include/hw/vfio/vfio.h
@@ -0,0 +1,7 @@
+#ifndef HW_VFIO_H
+#define HW_VFIO_H
+
+bool vfio_eeh_as_ok(AddressSpace *as);
+int vfio_eeh_as_op(AddressSpace *as, uint32_t op);
+
+#endif
diff --git a/include/hw/virtio/dataplane/vring.h b/include/hw/virtio/dataplane/vring.h
deleted file mode 100644
index d3e086a..0000000
--- a/include/hw/virtio/dataplane/vring.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/* Copyright 2012 Red Hat, Inc. and/or its affiliates
- * Copyright IBM, Corp. 2012
- *
- * Based on Linux 2.6.39 vhost code:
- * Copyright (C) 2009 Red Hat, Inc.
- * Copyright (C) 2006 Rusty Russell IBM Corporation
- *
- * Author: Michael S. Tsirkin <mst@redhat.com>
- *         Stefan Hajnoczi <stefanha@redhat.com>
- *
- * Inspiration, some code, and most witty comments come from
- * Documentation/virtual/lguest/lguest.c, by Rusty Russell
- *
- * This work is licensed under the terms of the GNU GPL, version 2.
- */
-
-#ifndef VRING_H
-#define VRING_H
-
-#include "qemu-common.h"
-#include "hw/virtio/virtio_ring.h"
-#include "hw/virtio/virtio.h"
-
-typedef struct {
-    MemoryRegion *mr;               /* memory region containing the vring */
-    struct vring vr;                /* virtqueue vring mapped to host memory */
-    uint16_t last_avail_idx;        /* last processed avail ring index */
-    uint16_t last_used_idx;         /* last processed used ring index */
-    uint16_t signalled_used;        /* EVENT_IDX state */
-    bool signalled_used_valid;
-    bool broken;                    /* was there a fatal error? */
-} Vring;
-
-static inline unsigned int vring_get_num(Vring *vring)
-{
-    return vring->vr.num;
-}
-
-/* Are there more descriptors available? */
-static inline bool vring_more_avail(Vring *vring)
-{
-    return vring->vr.avail->idx != vring->last_avail_idx;
-}
-
-/* Fail future vring_pop() and vring_push() calls until reset */
-static inline void vring_set_broken(Vring *vring)
-{
-    vring->broken = true;
-}
-
-bool vring_setup(Vring *vring, VirtIODevice *vdev, int n);
-void vring_teardown(Vring *vring, VirtIODevice *vdev, int n);
-void vring_disable_notification(VirtIODevice *vdev, Vring *vring);
-bool vring_enable_notification(VirtIODevice *vdev, Vring *vring);
-bool vring_should_notify(VirtIODevice *vdev, Vring *vring);
-int vring_pop(VirtIODevice *vdev, Vring *vring, VirtQueueElement *elem);
-void vring_push(Vring *vring, VirtQueueElement *elem, int len);
-
-#endif /* VRING_H */
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index e472f29..cf7f0b5 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -8,8 +8,8 @@
  *
  */
 
-#ifndef VHOST_BACKEND_H_
-#define VHOST_BACKEND_H_
+#ifndef VHOST_BACKEND_H
+#define VHOST_BACKEND_H
 
 typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_NONE = 0,
@@ -19,17 +19,89 @@
 } VhostBackendType;
 
 struct vhost_dev;
+struct vhost_log;
+struct vhost_memory;
+struct vhost_vring_file;
+struct vhost_vring_state;
+struct vhost_vring_addr;
+struct vhost_scsi_target;
 
-typedef int (*vhost_call)(struct vhost_dev *dev, unsigned long int request,
-             void *arg);
 typedef int (*vhost_backend_init)(struct vhost_dev *dev, void *opaque);
 typedef int (*vhost_backend_cleanup)(struct vhost_dev *dev);
+typedef int (*vhost_backend_memslots_limit)(struct vhost_dev *dev);
+
+typedef int (*vhost_net_set_backend_op)(struct vhost_dev *dev,
+                                struct vhost_vring_file *file);
+typedef int (*vhost_scsi_set_endpoint_op)(struct vhost_dev *dev,
+                                  struct vhost_scsi_target *target);
+typedef int (*vhost_scsi_clear_endpoint_op)(struct vhost_dev *dev,
+                                    struct vhost_scsi_target *target);
+typedef int (*vhost_scsi_get_abi_version_op)(struct vhost_dev *dev,
+                                             int *version);
+typedef int (*vhost_set_log_base_op)(struct vhost_dev *dev, uint64_t base,
+                                     struct vhost_log *log);
+typedef int (*vhost_set_mem_table_op)(struct vhost_dev *dev,
+                                      struct vhost_memory *mem);
+typedef int (*vhost_set_vring_addr_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_addr *addr);
+typedef int (*vhost_set_vring_endian_op)(struct vhost_dev *dev,
+                                         struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_num_op)(struct vhost_dev *dev,
+                                      struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_base_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+typedef int (*vhost_get_vring_base_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_state *ring);
+typedef int (*vhost_set_vring_kick_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+typedef int (*vhost_set_vring_call_op)(struct vhost_dev *dev,
+                                       struct vhost_vring_file *file);
+typedef int (*vhost_set_vring_busyloop_timeout_op)(struct vhost_dev *dev,
+                                                   struct vhost_vring_state *r);
+typedef int (*vhost_set_features_op)(struct vhost_dev *dev,
+                                     uint64_t features);
+typedef int (*vhost_get_features_op)(struct vhost_dev *dev,
+                                     uint64_t *features);
+typedef int (*vhost_set_owner_op)(struct vhost_dev *dev);
+typedef int (*vhost_reset_device_op)(struct vhost_dev *dev);
+typedef int (*vhost_get_vq_index_op)(struct vhost_dev *dev, int idx);
+typedef int (*vhost_set_vring_enable_op)(struct vhost_dev *dev,
+                                         int enable);
+typedef bool (*vhost_requires_shm_log_op)(struct vhost_dev *dev);
+typedef int (*vhost_migration_done_op)(struct vhost_dev *dev,
+                                       char *mac_addr);
+typedef bool (*vhost_backend_can_merge_op)(struct vhost_dev *dev,
+                                           uint64_t start1, uint64_t size1,
+                                           uint64_t start2, uint64_t size2);
 
 typedef struct VhostOps {
     VhostBackendType backend_type;
-    vhost_call vhost_call;
     vhost_backend_init vhost_backend_init;
     vhost_backend_cleanup vhost_backend_cleanup;
+    vhost_backend_memslots_limit vhost_backend_memslots_limit;
+    vhost_net_set_backend_op vhost_net_set_backend;
+    vhost_scsi_set_endpoint_op vhost_scsi_set_endpoint;
+    vhost_scsi_clear_endpoint_op vhost_scsi_clear_endpoint;
+    vhost_scsi_get_abi_version_op vhost_scsi_get_abi_version;
+    vhost_set_log_base_op vhost_set_log_base;
+    vhost_set_mem_table_op vhost_set_mem_table;
+    vhost_set_vring_addr_op vhost_set_vring_addr;
+    vhost_set_vring_endian_op vhost_set_vring_endian;
+    vhost_set_vring_num_op vhost_set_vring_num;
+    vhost_set_vring_base_op vhost_set_vring_base;
+    vhost_get_vring_base_op vhost_get_vring_base;
+    vhost_set_vring_kick_op vhost_set_vring_kick;
+    vhost_set_vring_call_op vhost_set_vring_call;
+    vhost_set_vring_busyloop_timeout_op vhost_set_vring_busyloop_timeout;
+    vhost_set_features_op vhost_set_features;
+    vhost_get_features_op vhost_get_features;
+    vhost_set_owner_op vhost_set_owner;
+    vhost_reset_device_op vhost_reset_device;
+    vhost_get_vq_index_op vhost_get_vq_index;
+    vhost_set_vring_enable_op vhost_set_vring_enable;
+    vhost_requires_shm_log_op vhost_requires_shm_log;
+    vhost_migration_done_op vhost_migration_done;
+    vhost_backend_can_merge_op vhost_backend_can_merge;
 } VhostOps;
 
 extern const VhostOps user_ops;
@@ -37,4 +109,4 @@
 int vhost_set_backend_type(struct vhost_dev *dev,
                            VhostBackendType backend_type);
 
-#endif /* VHOST_BACKEND_H_ */
+#endif /* VHOST_BACKEND_H */
diff --git a/include/hw/virtio/vhost-scsi.h b/include/hw/virtio/vhost-scsi.h
index 85cc031..9fd63df 100644
--- a/include/hw/virtio/vhost-scsi.h
+++ b/include/hw/virtio/vhost-scsi.h
@@ -19,37 +19,12 @@
 #include "hw/virtio/virtio-scsi.h"
 #include "hw/virtio/vhost.h"
 
-/*
- * Used by QEMU userspace to ensure a consistent vhost-scsi ABI.
- *
- * ABI Rev 0: July 2012 version starting point for v3.6-rc merge candidate +
- *            RFC-v2 vhost-scsi userspace.  Add GET_ABI_VERSION ioctl usage
- * ABI Rev 1: January 2013. Ignore vhost_tpgt filed in struct vhost_scsi_target.
- * 	      All the targets under vhost_wwpn can be seen and used by guest.
- */
-
-#define VHOST_SCSI_ABI_VERSION 1
-
-/* TODO #include <linux/vhost.h> properly */
-/* For VHOST_SCSI_SET_ENDPOINT/VHOST_SCSI_CLEAR_ENDPOINT ioctl */
-struct vhost_scsi_target {
-    int abi_version;
-    char vhost_wwpn[224];
-    unsigned short vhost_tpgt;
-    unsigned short reserved;
-};
-
 enum vhost_scsi_vq_list {
     VHOST_SCSI_VQ_CONTROL = 0,
     VHOST_SCSI_VQ_EVENT = 1,
     VHOST_SCSI_VQ_NUM_FIXED = 2,
 };
 
-#define VHOST_VIRTIO 0xAF
-#define VHOST_SCSI_SET_ENDPOINT _IOW(VHOST_VIRTIO, 0x40, struct vhost_scsi_target)
-#define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target)
-#define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int)
-
 #define TYPE_VHOST_SCSI "vhost-scsi"
 #define VHOST_SCSI(obj) \
         OBJECT_CHECK(VHostSCSI, (obj), TYPE_VHOST_SCSI)
@@ -60,14 +35,10 @@
     Error *migration_blocker;
 
     struct vhost_dev dev;
+    int32_t bootindex;
+    int channel;
+    int target;
+    int lun;
 } VHostSCSI;
 
-#define DEFINE_VHOST_SCSI_PROPERTIES(_state, _conf_field) \
-    DEFINE_PROP_STRING("vhostfd", _state, _conf_field.vhostfd), \
-    DEFINE_PROP_STRING("wwpn", _state, _conf_field.wwpn), \
-    DEFINE_PROP_UINT32("num_queues", _state, _conf_field.num_queues, 1), \
-    DEFINE_PROP_UINT32("max_sectors", _state, _conf_field.max_sectors, 0xFFFF), \
-    DEFINE_PROP_UINT32("cmd_per_lun", _state, _conf_field.cmd_per_lun, 128)
-
-
 #endif
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index d5593d1..e433089 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -28,6 +28,13 @@
 #define VHOST_LOG_CHUNK (VHOST_LOG_PAGE * VHOST_LOG_BITS)
 #define VHOST_INVALID_FEATURE_BIT   (0xff)
 
+struct vhost_log {
+    unsigned long long size;
+    int refcnt;
+    int fd;
+    vhost_log_chunk_t *log;
+};
+
 struct vhost_memory;
 struct vhost_dev {
     MemoryListener memory_listener;
@@ -36,28 +43,30 @@
     MemoryRegionSection *mem_sections;
     struct vhost_virtqueue *vqs;
     int nvqs;
-    /* the first virtuque which would be used by this vhost dev */
+    /* the first virtqueue which would be used by this vhost dev */
     int vq_index;
-    unsigned long long features;
-    unsigned long long acked_features;
-    unsigned long long backend_features;
+    uint64_t features;
+    uint64_t acked_features;
+    uint64_t backend_features;
+    uint64_t protocol_features;
+    uint64_t max_queues;
     bool started;
     bool log_enabled;
-    vhost_log_chunk_t *log;
-    unsigned long long log_size;
+    uint64_t log_size;
     Error *migration_blocker;
-    bool force;
     bool memory_changed;
     hwaddr mem_changed_start_addr;
     hwaddr mem_changed_end_addr;
     const VhostOps *vhost_ops;
     void *opaque;
+    struct vhost_log *log;
+    QLIST_ENTRY(vhost_dev) entry;
 };
 
 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
-                   VhostBackendType backend_type, bool force);
+                   VhostBackendType backend_type,
+                   uint32_t busyloop_timeout);
 void vhost_dev_cleanup(struct vhost_dev *hdev);
-bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev);
 void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev);
@@ -72,8 +81,13 @@
  */
 void vhost_virtqueue_mask(struct vhost_dev *hdev, VirtIODevice *vdev, int n,
                           bool mask);
-unsigned vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
-        unsigned features);
+uint64_t vhost_get_features(struct vhost_dev *hdev, const int *feature_bits,
+                            uint64_t features);
 void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
-        unsigned features);
+                        uint64_t features);
+bool vhost_has_free_slot(void);
+
+int vhost_net_set_backend(struct vhost_dev *hdev,
+                          struct vhost_vring_file *file);
+
 #endif
diff --git a/include/hw/virtio/virtio-9p.h b/include/hw/virtio/virtio-9p.h
deleted file mode 100644
index 65789db..0000000
--- a/include/hw/virtio/virtio-9p.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Virtio 9p
- *
- * Copyright IBM, Corp. 2010
- *
- * Authors:
- *  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-#ifndef QEMU_VIRTIO_9P_DEVICE_H
-#define QEMU_VIRTIO_9P_DEVICE_H
-
-typedef struct V9fsConf
-{
-    /* tag name for the device */
-    char *tag;
-    char *fsdev_id;
-} V9fsConf;
-
-#endif
diff --git a/include/hw/virtio/virtio-access.h b/include/hw/virtio/virtio-access.h
index 46456fd..440b455 100644
--- a/include/hw/virtio/virtio-access.h
+++ b/include/hw/virtio/virtio-access.h
@@ -12,16 +12,26 @@
  * (at your option) any later version.
  *
  */
-#ifndef _QEMU_VIRTIO_ACCESS_H
-#define _QEMU_VIRTIO_ACCESS_H
+
+#ifndef QEMU_VIRTIO_ACCESS_H
+#define QEMU_VIRTIO_ACCESS_H
+
 #include "hw/virtio/virtio.h"
 #include "exec/address-spaces.h"
 
+#if defined(TARGET_PPC64) || defined(TARGET_ARM)
+#define LEGACY_VIRTIO_IS_BIENDIAN 1
+#endif
+
 static inline bool virtio_access_is_big_endian(VirtIODevice *vdev)
 {
-#if defined(TARGET_IS_BIENDIAN)
+#if defined(LEGACY_VIRTIO_IS_BIENDIAN)
     return virtio_is_big_endian(vdev);
 #elif defined(TARGET_WORDS_BIGENDIAN)
+    if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        /* Devices conforming to VIRTIO 1.0 or later are always LE. */
+        return false;
+    }
     return true;
 #else
     return false;
@@ -167,4 +177,4 @@
 {
     *s = virtio_tswap64(vdev, *s);
 }
-#endif /* _QEMU_VIRTIO_ACCESS_H */
+#endif /* QEMU_VIRTIO_ACCESS_H */
diff --git a/include/hw/virtio/virtio-balloon.h b/include/hw/virtio/virtio-balloon.h
index f863bfe..1ea13bd 100644
--- a/include/hw/virtio/virtio-balloon.h
+++ b/include/hw/virtio/virtio-balloon.h
@@ -12,9 +12,10 @@
  *
  */
 
-#ifndef _QEMU_VIRTIO_BALLOON_H
-#define _QEMU_VIRTIO_BALLOON_H
+#ifndef QEMU_VIRTIO_BALLOON_H
+#define QEMU_VIRTIO_BALLOON_H
 
+#include "standard-headers/linux/virtio_balloon.h"
 #include "hw/virtio/virtio.h"
 #include "hw/pci/pci.h"
 
@@ -22,39 +23,13 @@
 #define VIRTIO_BALLOON(obj) \
         OBJECT_CHECK(VirtIOBalloon, (obj), TYPE_VIRTIO_BALLOON)
 
-/* from Linux's linux/virtio_balloon.h */
+typedef struct virtio_balloon_stat VirtIOBalloonStat;
 
-/* The ID for virtio_balloon */
-#define VIRTIO_ID_BALLOON 5
-
-/* The feature bitmap for virtio balloon */
-#define VIRTIO_BALLOON_F_MUST_TELL_HOST 0 /* Tell before reclaiming pages */
-#define VIRTIO_BALLOON_F_STATS_VQ 1       /* Memory stats virtqueue */
-
-/* Size of a PFN in the balloon interface. */
-#define VIRTIO_BALLOON_PFN_SHIFT 12
-
-struct virtio_balloon_config
-{
-    /* Number of pages host wants Guest to give up. */
-    uint32_t num_pages;
-    /* Number of pages we've actually got in balloon. */
-    uint32_t actual;
-};
-
-/* Memory Statistics */
-#define VIRTIO_BALLOON_S_SWAP_IN  0   /* Amount of memory swapped in */
-#define VIRTIO_BALLOON_S_SWAP_OUT 1   /* Amount of memory swapped out */
-#define VIRTIO_BALLOON_S_MAJFLT   2   /* Number of major faults */
-#define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
-#define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
-#define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
-#define VIRTIO_BALLOON_S_NR       6
-
-typedef struct VirtIOBalloonStat {
-    uint16_t tag;
-    uint64_t val;
-} QEMU_PACKED VirtIOBalloonStat;
+typedef struct virtio_balloon_stat_modern {
+       uint16_t tag;
+       uint8_t reserved[6];
+       uint64_t val;
+} VirtIOBalloonStatModern;
 
 typedef struct VirtIOBalloon {
     VirtIODevice parent_obj;
@@ -62,11 +37,12 @@
     uint32_t num_pages;
     uint32_t actual;
     uint64_t stats[VIRTIO_BALLOON_S_NR];
-    VirtQueueElement stats_vq_elem;
+    VirtQueueElement *stats_vq_elem;
     size_t stats_vq_offset;
     QEMUTimer *stats_timer;
     int64_t stats_last_update;
     int64_t stats_poll_interval;
+    uint32_t host_features;
 } VirtIOBalloon;
 
 #endif
diff --git a/include/hw/virtio/virtio-blk.h b/include/hw/virtio/virtio-blk.h
index 3979dc4..180bd8d 100644
--- a/include/hw/virtio/virtio-blk.h
+++ b/include/hw/virtio/virtio-blk.h
@@ -11,9 +11,10 @@
  *
  */
 
-#ifndef _QEMU_VIRTIO_BLK_H
-#define _QEMU_VIRTIO_BLK_H
+#ifndef QEMU_VIRTIO_BLK_H
+#define QEMU_VIRTIO_BLK_H
 
+#include "standard-headers/linux/virtio_blk.h"
 #include "hw/virtio/virtio.h"
 #include "hw/block/block.h"
 #include "sysemu/iothread.h"
@@ -23,88 +24,12 @@
 #define VIRTIO_BLK(obj) \
         OBJECT_CHECK(VirtIOBlock, (obj), TYPE_VIRTIO_BLK)
 
-/* from Linux's linux/virtio_blk.h */
-
-/* The ID for virtio_block */
-#define VIRTIO_ID_BLOCK 2
-
-/* Feature bits */
-#define VIRTIO_BLK_F_BARRIER    0       /* Does host support barriers? */
-#define VIRTIO_BLK_F_SIZE_MAX   1       /* Indicates maximum segment size */
-#define VIRTIO_BLK_F_SEG_MAX    2       /* Indicates maximum # of segments */
-#define VIRTIO_BLK_F_GEOMETRY   4       /* Indicates support of legacy geometry */
-#define VIRTIO_BLK_F_RO         5       /* Disk is read-only */
-#define VIRTIO_BLK_F_BLK_SIZE   6       /* Block size of disk is available*/
-#define VIRTIO_BLK_F_SCSI       7       /* Supports scsi command passthru */
-/* #define VIRTIO_BLK_F_IDENTIFY   8       ATA IDENTIFY supported, DEPRECATED */
-#define VIRTIO_BLK_F_WCE        9       /* write cache enabled */
-#define VIRTIO_BLK_F_TOPOLOGY   10      /* Topology information is available */
-#define VIRTIO_BLK_F_CONFIG_WCE 11      /* write cache configurable */
-
-#define VIRTIO_BLK_ID_BYTES     20      /* ID string length */
-
-struct virtio_blk_config
-{
-    uint64_t capacity;
-    uint32_t size_max;
-    uint32_t seg_max;
-    uint16_t cylinders;
-    uint8_t heads;
-    uint8_t sectors;
-    uint32_t blk_size;
-    uint8_t physical_block_exp;
-    uint8_t alignment_offset;
-    uint16_t min_io_size;
-    uint32_t opt_io_size;
-    uint8_t wce;
-} QEMU_PACKED;
-
-/* These two define direction. */
-#define VIRTIO_BLK_T_IN         0
-#define VIRTIO_BLK_T_OUT        1
-
-/* This bit says it's a scsi command, not an actual read or write. */
-#define VIRTIO_BLK_T_SCSI_CMD   2
-
-/* Flush the volatile write cache */
-#define VIRTIO_BLK_T_FLUSH      4
-
-/* return the device ID string */
-#define VIRTIO_BLK_T_GET_ID     8
-
-/* Barrier before this op. */
-#define VIRTIO_BLK_T_BARRIER    0x80000000
-
-/* This is the first element of the read scatter-gather list. */
-struct virtio_blk_outhdr
-{
-    /* VIRTIO_BLK_T* */
-    uint32_t type;
-    /* io priority. */
-    uint32_t ioprio;
-    /* Sector (ie. 512 byte offset) */
-    uint64_t sector;
-};
-
-#define VIRTIO_BLK_S_OK         0
-#define VIRTIO_BLK_S_IOERR      1
-#define VIRTIO_BLK_S_UNSUPP     2
-
 /* This is the last element of the write scatter-gather list */
 struct virtio_blk_inhdr
 {
     unsigned char status;
 };
 
-/* SCSI pass-through header */
-struct virtio_scsi_inhdr
-{
-    uint32_t errors;
-    uint32_t data_len;
-    uint32_t sense_len;
-    uint32_t residual;
-};
-
 struct VirtIOBlkConf
 {
     BlockConf conf;
@@ -112,7 +37,8 @@
     char *serial;
     uint32_t scsi;
     uint32_t config_wce;
-    uint32_t data_plane;
+    uint32_t request_merging;
+    uint16_t num_queues;
 };
 
 struct VirtIOBlockDataPlane;
@@ -121,43 +47,47 @@
 typedef struct VirtIOBlock {
     VirtIODevice parent_obj;
     BlockBackend *blk;
-    VirtQueue *vq;
     void *rq;
     QEMUBH *bh;
     VirtIOBlkConf conf;
     unsigned short sector_mask;
     bool original_wce;
     VMChangeStateEntry *change;
-    /* Function to push to vq and notify guest */
-    void (*complete_request)(struct VirtIOBlockReq *req, unsigned char status);
-    Notifier migration_state_notifier;
+    bool dataplane_disabled;
+    bool dataplane_started;
     struct VirtIOBlockDataPlane *dataplane;
 } VirtIOBlock;
 
-typedef struct MultiReqBuffer {
-    BlockRequest        blkreq[32];
-    unsigned int        num_writes;
-} MultiReqBuffer;
-
 typedef struct VirtIOBlockReq {
-    VirtIOBlock *dev;
     VirtQueueElement elem;
+    int64_t sector_num;
+    VirtIOBlock *dev;
+    VirtQueue *vq;
     struct virtio_blk_inhdr *in;
     struct virtio_blk_outhdr out;
     QEMUIOVector qiov;
+    size_t in_len;
     struct VirtIOBlockReq *next;
+    struct VirtIOBlockReq *mr_next;
     BlockAcctCookie acct;
 } VirtIOBlockReq;
 
-VirtIOBlockReq *virtio_blk_alloc_request(VirtIOBlock *s);
+#define VIRTIO_BLK_MAX_MERGE_REQS 32
 
+typedef struct MultiReqBuffer {
+    VirtIOBlockReq *reqs[VIRTIO_BLK_MAX_MERGE_REQS];
+    unsigned int num_reqs;
+    bool is_write;
+} MultiReqBuffer;
+
+void virtio_blk_init_request(VirtIOBlock *s, VirtQueue *vq,
+                             VirtIOBlockReq *req);
 void virtio_blk_free_request(VirtIOBlockReq *req);
 
-int virtio_blk_handle_scsi_req(VirtIOBlock *blk,
-                               VirtQueueElement *elem);
-
 void virtio_blk_handle_request(VirtIOBlockReq *req, MultiReqBuffer *mrb);
 
-void virtio_submit_multiwrite(BlockBackend *blk, MultiReqBuffer *mrb);
+void virtio_blk_submit_multireq(BlockBackend *blk, MultiReqBuffer *mrb);
+
+void virtio_blk_handle_vq(VirtIOBlock *s, VirtQueue *vq);
 
 #endif
diff --git a/include/hw/virtio/virtio-bus.h b/include/hw/virtio/virtio-bus.h
index 0756545..f3e5ef3 100644
--- a/include/hw/virtio/virtio-bus.h
+++ b/include/hw/virtio/virtio-bus.h
@@ -44,24 +44,54 @@
     void (*notify)(DeviceState *d, uint16_t vector);
     void (*save_config)(DeviceState *d, QEMUFile *f);
     void (*save_queue)(DeviceState *d, int n, QEMUFile *f);
+    void (*save_extra_state)(DeviceState *d, QEMUFile *f);
     int (*load_config)(DeviceState *d, QEMUFile *f);
     int (*load_queue)(DeviceState *d, int n, QEMUFile *f);
     int (*load_done)(DeviceState *d, QEMUFile *f);
-    unsigned (*get_features)(DeviceState *d);
+    int (*load_extra_state)(DeviceState *d, QEMUFile *f);
+    bool (*has_extra_state)(DeviceState *d);
     bool (*query_guest_notifiers)(DeviceState *d);
     int (*set_guest_notifiers)(DeviceState *d, int nvqs, bool assign);
-    int (*set_host_notifier)(DeviceState *d, int n, bool assigned);
     void (*vmstate_change)(DeviceState *d, bool running);
     /*
      * transport independent init function.
      * This is called by virtio-bus just after the device is plugged.
      */
-    void (*device_plugged)(DeviceState *d);
+    void (*device_plugged)(DeviceState *d, Error **errp);
+    /*
+     * Re-evaluate setup after feature bits have been validated
+     * by the device backend.
+     */
+    void (*post_plugged)(DeviceState *d, Error **errp);
     /*
      * transport independent exit function.
      * This is called by virtio-bus just before the device is unplugged.
      */
     void (*device_unplugged)(DeviceState *d);
+    int (*query_nvectors)(DeviceState *d);
+    /*
+     * ioeventfd handling: if the transport implements ioeventfd_started,
+     * it must implement the other ioeventfd callbacks as well
+     */
+    /* Returns true if the ioeventfd has been started for the device. */
+    bool (*ioeventfd_started)(DeviceState *d);
+    /*
+     * Sets the 'ioeventfd started' state after the ioeventfd has been
+     * started/stopped for the device. err signifies whether an error
+     * had occurred.
+     */
+    void (*ioeventfd_set_started)(DeviceState *d, bool started, bool err);
+    /* Returns true if the ioeventfd has been disabled for the device. */
+    bool (*ioeventfd_disabled)(DeviceState *d);
+    /* Sets the 'ioeventfd disabled' state for the device. */
+    void (*ioeventfd_set_disabled)(DeviceState *d, bool disabled);
+    /*
+     * Assigns/deassigns the ioeventfd backing for the transport on
+     * the device for queue number n. Returns an error value on
+     * failure.
+     */
+    int (*ioeventfd_assign)(DeviceState *d, EventNotifier *notifier,
+                            int n, bool assign);
     /*
      * Does the transport have variable vring alignment?
      * (ie can it ever call virtio_queue_set_align()?)
@@ -74,7 +104,7 @@
     BusState parent_obj;
 };
 
-int virtio_bus_device_plugged(VirtIODevice *vdev);
+void virtio_bus_device_plugged(VirtIODevice *vdev, Error **errp);
 void virtio_bus_reset(VirtioBusState *bus);
 void virtio_bus_device_unplugged(VirtIODevice *bus);
 /* Get the device id of the plugged device. */
@@ -84,9 +114,6 @@
 /* Get the features of the plugged device. */
 uint32_t virtio_bus_get_vdev_features(VirtioBusState *bus,
                                     uint32_t requested_features);
-/* Set the features of the plugged device. */
-void virtio_bus_set_vdev_features(VirtioBusState *bus,
-                                  uint32_t requested_features);
 /* Get bad features of the plugged device. */
 uint32_t virtio_bus_get_vdev_bad_features(VirtioBusState *bus);
 /* Get config of the plugged device. */
@@ -106,4 +133,11 @@
     return (VirtIODevice *)qdev;
 }
 
+/* Start the ioeventfd. */
+void virtio_bus_start_ioeventfd(VirtioBusState *bus);
+/* Stop the ioeventfd. */
+void virtio_bus_stop_ioeventfd(VirtioBusState *bus);
+/* Switch from/to the generic ioeventfd handler */
+int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign);
+
 #endif /* VIRTIO_BUS_H */
diff --git a/include/hw/virtio/virtio-gpu.h b/include/hw/virtio/virtio-gpu.h
new file mode 100644
index 0000000..20d1cd6
--- /dev/null
+++ b/include/hw/virtio/virtio-gpu.h
@@ -0,0 +1,169 @@
+/*
+ * Virtio GPU Device
+ *
+ * Copyright Red Hat, Inc. 2013-2014
+ *
+ * Authors:
+ *     Dave Airlie <airlied@redhat.com>
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef HW_VIRTIO_GPU_H
+#define HW_VIRTIO_GPU_H
+
+#include "qemu/queue.h"
+#include "ui/qemu-pixman.h"
+#include "ui/console.h"
+#include "hw/virtio/virtio.h"
+#include "hw/pci/pci.h"
+#include "qemu/log.h"
+
+#include "standard-headers/linux/virtio_gpu.h"
+#define TYPE_VIRTIO_GPU "virtio-gpu-device"
+#define VIRTIO_GPU(obj)                                        \
+        OBJECT_CHECK(VirtIOGPU, (obj), TYPE_VIRTIO_GPU)
+
+#define VIRTIO_ID_GPU 16
+
+struct virtio_gpu_simple_resource {
+    uint32_t resource_id;
+    uint32_t width;
+    uint32_t height;
+    uint32_t format;
+    uint64_t *addrs;
+    struct iovec *iov;
+    unsigned int iov_cnt;
+    uint32_t scanout_bitmask;
+    pixman_image_t *image;
+    QTAILQ_ENTRY(virtio_gpu_simple_resource) next;
+};
+
+struct virtio_gpu_scanout {
+    QemuConsole *con;
+    DisplaySurface *ds;
+    uint32_t width, height;
+    int x, y;
+    int invalidate;
+    uint32_t resource_id;
+    struct virtio_gpu_update_cursor cursor;
+    QEMUCursor *current_cursor;
+};
+
+struct virtio_gpu_requested_state {
+    uint32_t width, height;
+    int x, y;
+};
+
+enum virtio_gpu_conf_flags {
+    VIRTIO_GPU_FLAG_VIRGL_ENABLED = 1,
+    VIRTIO_GPU_FLAG_STATS_ENABLED,
+};
+
+#define virtio_gpu_virgl_enabled(_cfg) \
+    (_cfg.flags & (1 << VIRTIO_GPU_FLAG_VIRGL_ENABLED))
+#define virtio_gpu_stats_enabled(_cfg) \
+    (_cfg.flags & (1 << VIRTIO_GPU_FLAG_STATS_ENABLED))
+
+struct virtio_gpu_conf {
+    uint32_t max_outputs;
+    uint32_t flags;
+};
+
+struct virtio_gpu_ctrl_command {
+    VirtQueueElement elem;
+    VirtQueue *vq;
+    struct virtio_gpu_ctrl_hdr cmd_hdr;
+    uint32_t error;
+    bool waiting;
+    bool finished;
+    QTAILQ_ENTRY(virtio_gpu_ctrl_command) next;
+};
+
+typedef struct VirtIOGPU {
+    VirtIODevice parent_obj;
+
+    QEMUBH *ctrl_bh;
+    QEMUBH *cursor_bh;
+    VirtQueue *ctrl_vq;
+    VirtQueue *cursor_vq;
+
+    int enable;
+
+    int config_size;
+    DeviceState *qdev;
+
+    QTAILQ_HEAD(, virtio_gpu_simple_resource) reslist;
+    QTAILQ_HEAD(, virtio_gpu_ctrl_command) cmdq;
+    QTAILQ_HEAD(, virtio_gpu_ctrl_command) fenceq;
+
+    struct virtio_gpu_scanout scanout[VIRTIO_GPU_MAX_SCANOUTS];
+    struct virtio_gpu_requested_state req_state[VIRTIO_GPU_MAX_SCANOUTS];
+
+    struct virtio_gpu_conf conf;
+    int enabled_output_bitmask;
+    struct virtio_gpu_config virtio_config;
+
+    bool use_virgl_renderer;
+    bool renderer_inited;
+    int renderer_blocked;
+    QEMUTimer *fence_poll;
+    QEMUTimer *print_stats;
+
+    uint32_t inflight;
+    struct {
+        uint32_t max_inflight;
+        uint32_t requests;
+        uint32_t req_3d;
+        uint32_t bytes_3d;
+    } stats;
+
+    Error *migration_blocker;
+} VirtIOGPU;
+
+extern const GraphicHwOps virtio_gpu_ops;
+
+/* to share between PCI and VGA */
+#define DEFINE_VIRTIO_GPU_PCI_PROPERTIES(_state)               \
+    DEFINE_PROP_BIT("ioeventfd", _state, flags,                \
+                    VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false), \
+    DEFINE_PROP_UINT32("vectors", _state, nvectors, 3)
+
+#define VIRTIO_GPU_FILL_CMD(out) do {                                   \
+        size_t s;                                                       \
+        s = iov_to_buf(cmd->elem.out_sg, cmd->elem.out_num, 0,          \
+                       &out, sizeof(out));                              \
+        if (s != sizeof(out)) {                                         \
+            qemu_log_mask(LOG_GUEST_ERROR,                              \
+                          "%s: command size incorrect %zu vs %zu\n",    \
+                          __func__, s, sizeof(out));                    \
+            return;                                                     \
+        }                                                               \
+    } while (0)
+
+/* virtio-gpu.c */
+void virtio_gpu_ctrl_response(VirtIOGPU *g,
+                              struct virtio_gpu_ctrl_command *cmd,
+                              struct virtio_gpu_ctrl_hdr *resp,
+                              size_t resp_len);
+void virtio_gpu_ctrl_response_nodata(VirtIOGPU *g,
+                                     struct virtio_gpu_ctrl_command *cmd,
+                                     enum virtio_gpu_ctrl_type type);
+void virtio_gpu_get_display_info(VirtIOGPU *g,
+                                 struct virtio_gpu_ctrl_command *cmd);
+int virtio_gpu_create_mapping_iov(struct virtio_gpu_resource_attach_backing *ab,
+                                  struct virtio_gpu_ctrl_command *cmd,
+                                  uint64_t **addr, struct iovec **iov);
+void virtio_gpu_cleanup_mapping_iov(struct iovec *iov, uint32_t count);
+void virtio_gpu_process_cmdq(VirtIOGPU *g);
+
+/* virtio-gpu-3d.c */
+void virtio_gpu_virgl_process_cmd(VirtIOGPU *g,
+                                  struct virtio_gpu_ctrl_command *cmd);
+void virtio_gpu_virgl_fence_poll(VirtIOGPU *g);
+void virtio_gpu_virgl_reset(VirtIOGPU *g);
+int virtio_gpu_virgl_init(VirtIOGPU *g);
+
+#endif
diff --git a/include/hw/virtio/virtio-input.h b/include/hw/virtio/virtio-input.h
new file mode 100644
index 0000000..55db310
--- /dev/null
+++ b/include/hw/virtio/virtio-input.h
@@ -0,0 +1,108 @@
+#ifndef QEMU_VIRTIO_INPUT_H
+#define QEMU_VIRTIO_INPUT_H
+
+#include "ui/input.h"
+
+/* ----------------------------------------------------------------- */
+/* virtio input protocol                                             */
+
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_input.h"
+
+typedef struct virtio_input_absinfo virtio_input_absinfo;
+typedef struct virtio_input_config virtio_input_config;
+typedef struct virtio_input_event virtio_input_event;
+
+/* ----------------------------------------------------------------- */
+/* qemu internals                                                    */
+
+#define TYPE_VIRTIO_INPUT "virtio-input-device"
+#define VIRTIO_INPUT(obj) \
+        OBJECT_CHECK(VirtIOInput, (obj), TYPE_VIRTIO_INPUT)
+#define VIRTIO_INPUT_GET_PARENT_CLASS(obj) \
+        OBJECT_GET_PARENT_CLASS(obj, TYPE_VIRTIO_INPUT)
+#define VIRTIO_INPUT_GET_CLASS(obj) \
+        OBJECT_GET_CLASS(VirtIOInputClass, obj, TYPE_VIRTIO_INPUT)
+#define VIRTIO_INPUT_CLASS(klass) \
+        OBJECT_CLASS_CHECK(VirtIOInputClass, klass, TYPE_VIRTIO_INPUT)
+
+#define TYPE_VIRTIO_INPUT_HID "virtio-input-hid-device"
+#define TYPE_VIRTIO_KEYBOARD  "virtio-keyboard-device"
+#define TYPE_VIRTIO_MOUSE     "virtio-mouse-device"
+#define TYPE_VIRTIO_TABLET    "virtio-tablet-device"
+
+#define VIRTIO_INPUT_HID(obj) \
+        OBJECT_CHECK(VirtIOInputHID, (obj), TYPE_VIRTIO_INPUT_HID)
+#define VIRTIO_INPUT_HID_GET_PARENT_CLASS(obj) \
+        OBJECT_GET_PARENT_CLASS(obj, TYPE_VIRTIO_INPUT_HID)
+
+#define TYPE_VIRTIO_INPUT_HOST   "virtio-input-host-device"
+#define VIRTIO_INPUT_HOST(obj) \
+        OBJECT_CHECK(VirtIOInputHost, (obj), TYPE_VIRTIO_INPUT_HOST)
+#define VIRTIO_INPUT_HOST_GET_PARENT_CLASS(obj) \
+        OBJECT_GET_PARENT_CLASS(obj, TYPE_VIRTIO_INPUT_HOST)
+
+typedef struct VirtIOInput VirtIOInput;
+typedef struct VirtIOInputClass VirtIOInputClass;
+typedef struct VirtIOInputConfig VirtIOInputConfig;
+typedef struct VirtIOInputHID VirtIOInputHID;
+typedef struct VirtIOInputHost VirtIOInputHost;
+
+struct VirtIOInputConfig {
+    virtio_input_config               config;
+    QTAILQ_ENTRY(VirtIOInputConfig)   node;
+};
+
+struct VirtIOInput {
+    VirtIODevice                      parent_obj;
+    uint8_t                           cfg_select;
+    uint8_t                           cfg_subsel;
+    uint32_t                          cfg_size;
+    QTAILQ_HEAD(, VirtIOInputConfig)  cfg_list;
+    VirtQueue                         *evt, *sts;
+    char                              *serial;
+
+    virtio_input_event                *queue;
+    uint32_t                          qindex, qsize;
+
+    bool                              active;
+};
+
+struct VirtIOInputClass {
+    /*< private >*/
+    VirtioDeviceClass parent;
+    /*< public >*/
+
+    DeviceRealize realize;
+    DeviceUnrealize unrealize;
+    void (*change_active)(VirtIOInput *vinput);
+    void (*handle_status)(VirtIOInput *vinput, virtio_input_event *event);
+};
+
+struct VirtIOInputHID {
+    VirtIOInput                       parent_obj;
+    char                              *display;
+    uint32_t                          head;
+    QemuInputHandler                  *handler;
+    QemuInputHandlerState             *hs;
+    int                               ledstate;
+};
+
+struct VirtIOInputHost {
+    VirtIOInput                       parent_obj;
+    char                              *evdev;
+    int                               fd;
+};
+
+void virtio_input_send(VirtIOInput *vinput, virtio_input_event *event);
+void virtio_input_init_config(VirtIOInput *vinput,
+                              virtio_input_config *config);
+virtio_input_config *virtio_input_find_config(VirtIOInput *vinput,
+                                              uint8_t select,
+                                              uint8_t subsel);
+void virtio_input_add_config(VirtIOInput *vinput,
+                             virtio_input_config *config);
+void virtio_input_idstr_config(VirtIOInput *vinput,
+                               uint8_t select, const char *string);
+
+#endif /* QEMU_VIRTIO_INPUT_H */
diff --git a/include/hw/virtio/virtio-net.h b/include/hw/virtio/virtio-net.h
index 6ceb5aa..91ed97c 100644
--- a/include/hw/virtio/virtio-net.h
+++ b/include/hw/virtio/virtio-net.h
@@ -11,53 +11,16 @@
  *
  */
 
-#ifndef _QEMU_VIRTIO_NET_H
-#define _QEMU_VIRTIO_NET_H
+#ifndef QEMU_VIRTIO_NET_H
+#define QEMU_VIRTIO_NET_H
 
+#include "standard-headers/linux/virtio_net.h"
 #include "hw/virtio/virtio.h"
-#include "hw/pci/pci.h"
 
 #define TYPE_VIRTIO_NET "virtio-net-device"
 #define VIRTIO_NET(obj) \
         OBJECT_CHECK(VirtIONet, (obj), TYPE_VIRTIO_NET)
 
-#define ETH_ALEN    6
-
-/* from Linux's virtio_net.h */
-
-/* The ID for virtio_net */
-#define VIRTIO_ID_NET   1
-
-/* The feature bitmap for virtio net */
-#define VIRTIO_NET_F_CSUM       0       /* Host handles pkts w/ partial csum */
-#define VIRTIO_NET_F_GUEST_CSUM 1       /* Guest handles pkts w/ partial csum */
-#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 2 /* Control channel offload
-                                         * configuration support */
-#define VIRTIO_NET_F_MAC        5       /* Host has given MAC address. */
-#define VIRTIO_NET_F_GSO        6       /* Host handles pkts w/ any GSO type */
-#define VIRTIO_NET_F_GUEST_TSO4 7       /* Guest can handle TSOv4 in. */
-#define VIRTIO_NET_F_GUEST_TSO6 8       /* Guest can handle TSOv6 in. */
-#define VIRTIO_NET_F_GUEST_ECN  9       /* Guest can handle TSO[6] w/ ECN in. */
-#define VIRTIO_NET_F_GUEST_UFO  10      /* Guest can handle UFO in. */
-#define VIRTIO_NET_F_HOST_TSO4  11      /* Host can handle TSOv4 in. */
-#define VIRTIO_NET_F_HOST_TSO6  12      /* Host can handle TSOv6 in. */
-#define VIRTIO_NET_F_HOST_ECN   13      /* Host can handle TSO[6] w/ ECN in. */
-#define VIRTIO_NET_F_HOST_UFO   14      /* Host can handle UFO in. */
-#define VIRTIO_NET_F_MRG_RXBUF  15      /* Host can merge receive buffers. */
-#define VIRTIO_NET_F_STATUS     16      /* virtio_net_config.status available */
-#define VIRTIO_NET_F_CTRL_VQ    17      /* Control channel available */
-#define VIRTIO_NET_F_CTRL_RX    18      /* Control channel RX mode support */
-#define VIRTIO_NET_F_CTRL_VLAN  19      /* Control channel VLAN filtering */
-#define VIRTIO_NET_F_CTRL_RX_EXTRA 20   /* Extra RX mode control support */
-#define VIRTIO_NET_F_GUEST_ANNOUNCE 21  /* Guest can announce itself */
-#define VIRTIO_NET_F_MQ         22      /* Device supports Receive Flow
-                                         * Steering */
-
-#define VIRTIO_NET_F_CTRL_MAC_ADDR   23 /* Set MAC address */
-
-#define VIRTIO_NET_S_LINK_UP    1       /* Link is up */
-#define VIRTIO_NET_S_ANNOUNCE   2       /* Announcement is needed */
-
 #define TX_TIMER_INTERVAL 150000 /* 150 us */
 
 /* Limit the number of packets that can be sent via a single flush
@@ -77,72 +40,6 @@
 /* Maximum packet size we can receive from tap device: header + 64k */
 #define VIRTIO_NET_MAX_BUFSIZE (sizeof(struct virtio_net_hdr) + (64 << 10))
 
-struct virtio_net_config
-{
-    /* The config defining mac address ($ETH_ALEN bytes) */
-    uint8_t mac[ETH_ALEN];
-    /* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */
-    uint16_t status;
-    /* Max virtqueue pairs supported by the device */
-    uint16_t max_virtqueue_pairs;
-} QEMU_PACKED;
-
-/*
- * Control virtqueue data structures
- *
- * The control virtqueue expects a header in the first sg entry
- * and an ack/status response in the last entry.  Data for the
- * command goes in between.
- */
-struct virtio_net_ctrl_hdr {
-    uint8_t class;
-    uint8_t cmd;
-};
-
-typedef uint8_t virtio_net_ctrl_ack;
-
-#define VIRTIO_NET_OK     0
-#define VIRTIO_NET_ERR    1
-
-/*
- * Control the RX mode, ie. promisucous, allmulti, etc...
- * All commands require an "out" sg entry containing a 1 byte
- * state value, zero = disable, non-zero = enable.  Commands
- * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
- * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
- */
-#define VIRTIO_NET_CTRL_RX    0
- #define VIRTIO_NET_CTRL_RX_PROMISC      0
- #define VIRTIO_NET_CTRL_RX_ALLMULTI     1
- #define VIRTIO_NET_CTRL_RX_ALLUNI       2
- #define VIRTIO_NET_CTRL_RX_NOMULTI      3
- #define VIRTIO_NET_CTRL_RX_NOUNI        4
- #define VIRTIO_NET_CTRL_RX_NOBCAST      5
-
-/*
- * Control the MAC
- *
- * The MAC filter table is managed by the hypervisor, the guest should
- * assume the size is infinite.  Filtering should be considered
- * non-perfect, ie. based on hypervisor resources, the guest may
- * received packets from sources not specified in the filter list.
- *
- * In addition to the class/cmd header, the TABLE_SET command requires
- * two out scatterlists.  Each contains a 4 byte count of entries followed
- * by a concatenated byte stream of the ETH_ALEN MAC addresses.  The
- * first sg list contains unicast addresses, the second is for multicast.
- * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
- * is available.
- *
- * The ADDR_SET command requests one out scatterlist, it contains a
- * 6 bytes MAC address. This functionality is present if the
- * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
- */
-struct virtio_net_ctrl_mac {
-    uint32_t entries;
-    uint8_t macs[][ETH_ALEN];
-};
-
 typedef struct VirtIONetQueue {
     VirtQueue *rx_vq;
     VirtQueue *tx_vq;
@@ -150,8 +47,7 @@
     QEMUBH *tx_bh;
     int tx_waiting;
     struct {
-        VirtQueueElement elem;
-        ssize_t len;
+        VirtQueueElement *elem;
     } async_tx;
     struct VirtIONet *n;
 } VirtIONetQueue;
@@ -168,6 +64,7 @@
     uint32_t has_vnet_hdr;
     size_t host_hdr_len;
     size_t guest_hdr_len;
+    uint32_t host_features;
     uint8_t has_ufo;
     int mergeable_rx_bufs;
     uint8_t promisc;
@@ -197,96 +94,9 @@
     uint64_t curr_guest_offloads;
     QEMUTimer *announce_timer;
     int announce_counter;
+    bool needs_vnet_hdr_swap;
 } VirtIONet;
 
-#define VIRTIO_NET_CTRL_MAC    1
- #define VIRTIO_NET_CTRL_MAC_TABLE_SET        0
- #define VIRTIO_NET_CTRL_MAC_ADDR_SET         1
-
-/*
- * Control VLAN filtering
- *
- * The VLAN filter table is controlled via a simple ADD/DEL interface.
- * VLAN IDs not added may be filterd by the hypervisor.  Del is the
- * opposite of add.  Both commands expect an out entry containing a 2
- * byte VLAN ID.  VLAN filterting is available with the
- * VIRTIO_NET_F_CTRL_VLAN feature bit.
- */
-#define VIRTIO_NET_CTRL_VLAN       2
- #define VIRTIO_NET_CTRL_VLAN_ADD             0
- #define VIRTIO_NET_CTRL_VLAN_DEL             1
-
-/*
- * Control link announce acknowledgement
- *
- * VIRTIO_NET_S_ANNOUNCE bit in the status field requests link announcement from
- * guest driver. The driver is notified by config space change interrupt.  The
- * command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that the driver has
- * received the notification. It makes the device clear the bit
- * VIRTIO_NET_S_ANNOUNCE in the status field.
- */
-#define VIRTIO_NET_CTRL_ANNOUNCE       3
- #define VIRTIO_NET_CTRL_ANNOUNCE_ACK         0
-
-/*
- * Control Multiqueue
- *
- * The command VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
- * enables multiqueue, specifying the number of the transmit and
- * receive queues that will be used. After the command is consumed and acked by
- * the device, the device will not steer new packets on receive virtqueues
- * other than specified nor read from transmit virtqueues other than specified.
- * Accordingly, driver should not transmit new packets  on virtqueues other than
- * specified.
- */
-struct virtio_net_ctrl_mq {
-    uint16_t virtqueue_pairs;
-};
-
-#define VIRTIO_NET_CTRL_MQ   4
- #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET        0
- #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN        1
- #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX        0x8000
-
-/*
- * Control network offloads
- *
- * Dynamic offloads are available with the
- * VIRTIO_NET_F_CTRL_GUEST_OFFLOADS feature bit.
- */
-#define VIRTIO_NET_CTRL_GUEST_OFFLOADS   5
- #define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET        0
-
-#define DEFINE_VIRTIO_NET_FEATURES(_state, _field) \
-        DEFINE_PROP_BIT("any_layout", _state, _field, VIRTIO_F_ANY_LAYOUT, true), \
-        DEFINE_PROP_BIT("csum", _state, _field, VIRTIO_NET_F_CSUM, true), \
-        DEFINE_PROP_BIT("guest_csum", _state, _field, VIRTIO_NET_F_GUEST_CSUM, true), \
-        DEFINE_PROP_BIT("gso", _state, _field, VIRTIO_NET_F_GSO, true), \
-        DEFINE_PROP_BIT("guest_tso4", _state, _field, VIRTIO_NET_F_GUEST_TSO4, true), \
-        DEFINE_PROP_BIT("guest_tso6", _state, _field, VIRTIO_NET_F_GUEST_TSO6, true), \
-        DEFINE_PROP_BIT("guest_ecn", _state, _field, VIRTIO_NET_F_GUEST_ECN, true), \
-        DEFINE_PROP_BIT("guest_ufo", _state, _field, VIRTIO_NET_F_GUEST_UFO, true), \
-        DEFINE_PROP_BIT("guest_announce", _state, _field, VIRTIO_NET_F_GUEST_ANNOUNCE, true), \
-        DEFINE_PROP_BIT("host_tso4", _state, _field, VIRTIO_NET_F_HOST_TSO4, true), \
-        DEFINE_PROP_BIT("host_tso6", _state, _field, VIRTIO_NET_F_HOST_TSO6, true), \
-        DEFINE_PROP_BIT("host_ecn", _state, _field, VIRTIO_NET_F_HOST_ECN, true), \
-        DEFINE_PROP_BIT("host_ufo", _state, _field, VIRTIO_NET_F_HOST_UFO, true), \
-        DEFINE_PROP_BIT("mrg_rxbuf", _state, _field, VIRTIO_NET_F_MRG_RXBUF, true), \
-        DEFINE_PROP_BIT("status", _state, _field, VIRTIO_NET_F_STATUS, true), \
-        DEFINE_PROP_BIT("ctrl_vq", _state, _field, VIRTIO_NET_F_CTRL_VQ, true), \
-        DEFINE_PROP_BIT("ctrl_rx", _state, _field, VIRTIO_NET_F_CTRL_RX, true), \
-        DEFINE_PROP_BIT("ctrl_vlan", _state, _field, VIRTIO_NET_F_CTRL_VLAN, true), \
-        DEFINE_PROP_BIT("ctrl_rx_extra", _state, _field, VIRTIO_NET_F_CTRL_RX_EXTRA, true), \
-        DEFINE_PROP_BIT("ctrl_mac_addr", _state, _field, VIRTIO_NET_F_CTRL_MAC_ADDR, true), \
-        DEFINE_PROP_BIT("ctrl_guest_offloads", _state, _field, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, true), \
-        DEFINE_PROP_BIT("mq", _state, _field, VIRTIO_NET_F_MQ, false)
-
-#define DEFINE_VIRTIO_NET_PROPERTIES(_state, _field)                           \
-    DEFINE_PROP_UINT32("x-txtimer", _state, _field.txtimer, TX_TIMER_INTERVAL),\
-    DEFINE_PROP_INT32("x-txburst", _state, _field.txburst, TX_BURST),          \
-    DEFINE_PROP_STRING("tx", _state, _field.tx)
-
-void virtio_net_set_config_size(VirtIONet *n, uint32_t host_features);
 void virtio_net_set_netclient_name(VirtIONet *n, const char *name,
                                    const char *type);
 
diff --git a/include/hw/virtio/virtio-rng.h b/include/hw/virtio/virtio-rng.h
index 14e85a5..2d40abd 100644
--- a/include/hw/virtio/virtio-rng.h
+++ b/include/hw/virtio/virtio-rng.h
@@ -9,11 +9,12 @@
  * top-level directory.
  */
 
-#ifndef _QEMU_VIRTIO_RNG_H
-#define _QEMU_VIRTIO_RNG_H
+#ifndef QEMU_VIRTIO_RNG_H
+#define QEMU_VIRTIO_RNG_H
 
 #include "sysemu/rng.h"
 #include "sysemu/rng-random.h"
+#include "standard-headers/linux/virtio_rng.h"
 
 #define TYPE_VIRTIO_RNG "virtio-rng-device"
 #define VIRTIO_RNG(obj) \
@@ -21,14 +22,11 @@
 #define VIRTIO_RNG_GET_PARENT_CLASS(obj) \
         OBJECT_GET_PARENT_CLASS(obj, TYPE_VIRTIO_RNG)
 
-/* The Virtio ID for the virtio rng device */
-#define VIRTIO_ID_RNG    4
-
 struct VirtIORNGConf {
     RngBackend *rng;
     uint64_t max_bytes;
     uint32_t period_ms;
-    RndRandom *default_backend;
+    RngRandom *default_backend;
 };
 
 typedef struct VirtIORNG {
@@ -46,16 +44,7 @@
      */
     QEMUTimer *rate_limit_timer;
     int64_t quota_remaining;
+    bool activate_timer;
 } VirtIORNG;
 
-/* Set a default rate limit of 2^47 bytes per minute or roughly 2TB/s.  If
-   you have an entropy source capable of generating more entropy than this
-   and you can pass it through via virtio-rng, then hats off to you.  Until
-   then, this is unlimited for all practical purposes.
-*/
-#define DEFINE_VIRTIO_RNG_PROPERTIES(_state, _conf_field)                    \
-        DEFINE_PROP_UINT64("max-bytes", _state, _conf_field.max_bytes,       \
-                           INT64_MAX),                                       \
-        DEFINE_PROP_UINT32("period", _state, _conf_field.period_ms, 1 << 16)
-
 #endif
diff --git a/include/hw/virtio/virtio-scsi.h b/include/hw/virtio/virtio-scsi.h
index bf17cc9..a1e0cfb 100644
--- a/include/hw/virtio/virtio-scsi.h
+++ b/include/hw/virtio/virtio-scsi.h
@@ -11,14 +11,17 @@
  *
  */
 
-#ifndef _QEMU_VIRTIO_SCSI_H
-#define _QEMU_VIRTIO_SCSI_H
+#ifndef QEMU_VIRTIO_SCSI_H
+#define QEMU_VIRTIO_SCSI_H
 
+/* Override CDB/sense data size: they are dynamic (guest controlled) in QEMU */
+#define VIRTIO_SCSI_CDB_SIZE 0
+#define VIRTIO_SCSI_SENSE_SIZE 0
+#include "standard-headers/linux/virtio_scsi.h"
 #include "hw/virtio/virtio.h"
 #include "hw/pci/pci.h"
 #include "hw/scsi/scsi.h"
 #include "sysemu/iothread.h"
-#include "hw/virtio/dataplane/vring.h"
 
 #define TYPE_VIRTIO_SCSI_COMMON "virtio-scsi-common"
 #define VIRTIO_SCSI_COMMON(obj) \
@@ -28,124 +31,19 @@
 #define VIRTIO_SCSI(obj) \
         OBJECT_CHECK(VirtIOSCSI, (obj), TYPE_VIRTIO_SCSI)
 
-
-/* The ID for virtio_scsi */
-#define VIRTIO_ID_SCSI  8
-
-/* Feature Bits */
-#define VIRTIO_SCSI_F_INOUT                    0
-#define VIRTIO_SCSI_F_HOTPLUG                  1
-#define VIRTIO_SCSI_F_CHANGE                   2
-
 #define VIRTIO_SCSI_VQ_SIZE     128
-#define VIRTIO_SCSI_CDB_SIZE    32
-#define VIRTIO_SCSI_SENSE_SIZE  96
 #define VIRTIO_SCSI_MAX_CHANNEL 0
 #define VIRTIO_SCSI_MAX_TARGET  255
 #define VIRTIO_SCSI_MAX_LUN     16383
 
-/* Response codes */
-#define VIRTIO_SCSI_S_OK                       0
-#define VIRTIO_SCSI_S_OVERRUN                  1
-#define VIRTIO_SCSI_S_ABORTED                  2
-#define VIRTIO_SCSI_S_BAD_TARGET               3
-#define VIRTIO_SCSI_S_RESET                    4
-#define VIRTIO_SCSI_S_BUSY                     5
-#define VIRTIO_SCSI_S_TRANSPORT_FAILURE        6
-#define VIRTIO_SCSI_S_TARGET_FAILURE           7
-#define VIRTIO_SCSI_S_NEXUS_FAILURE            8
-#define VIRTIO_SCSI_S_FAILURE                  9
-#define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED       10
-#define VIRTIO_SCSI_S_FUNCTION_REJECTED        11
-#define VIRTIO_SCSI_S_INCORRECT_LUN            12
-
-/* Controlq type codes.  */
-#define VIRTIO_SCSI_T_TMF                      0
-#define VIRTIO_SCSI_T_AN_QUERY                 1
-#define VIRTIO_SCSI_T_AN_SUBSCRIBE             2
-
-/* Valid TMF subtypes.  */
-#define VIRTIO_SCSI_T_TMF_ABORT_TASK           0
-#define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET       1
-#define VIRTIO_SCSI_T_TMF_CLEAR_ACA            2
-#define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET       3
-#define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET      4
-#define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET   5
-#define VIRTIO_SCSI_T_TMF_QUERY_TASK           6
-#define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET       7
-
-/* Events.  */
-#define VIRTIO_SCSI_T_EVENTS_MISSED            0x80000000
-#define VIRTIO_SCSI_T_NO_EVENT                 0
-#define VIRTIO_SCSI_T_TRANSPORT_RESET          1
-#define VIRTIO_SCSI_T_ASYNC_NOTIFY             2
-#define VIRTIO_SCSI_T_PARAM_CHANGE             3
-
-/* Reasons for transport reset event */
-#define VIRTIO_SCSI_EVT_RESET_HARD             0
-#define VIRTIO_SCSI_EVT_RESET_RESCAN           1
-#define VIRTIO_SCSI_EVT_RESET_REMOVED          2
-
-/* SCSI command request, followed by CDB and data-out */
-typedef struct {
-    uint8_t lun[8];              /* Logical Unit Number */
-    uint64_t tag;                /* Command identifier */
-    uint8_t task_attr;           /* Task attribute */
-    uint8_t prio;
-    uint8_t crn;
-} QEMU_PACKED VirtIOSCSICmdReq;
-
-/* Response, followed by sense data and data-in */
-typedef struct {
-    uint32_t sense_len;          /* Sense data length */
-    uint32_t resid;              /* Residual bytes in data buffer */
-    uint16_t status_qualifier;   /* Status qualifier */
-    uint8_t status;              /* Command completion status */
-    uint8_t response;            /* Response values */
-} QEMU_PACKED VirtIOSCSICmdResp;
-
-/* Task Management Request */
-typedef struct {
-    uint32_t type;
-    uint32_t subtype;
-    uint8_t lun[8];
-    uint64_t tag;
-} QEMU_PACKED VirtIOSCSICtrlTMFReq;
-
-typedef struct {
-    uint8_t response;
-} QEMU_PACKED VirtIOSCSICtrlTMFResp;
-
-/* Asynchronous notification query/subscription */
-typedef struct {
-    uint32_t type;
-    uint8_t lun[8];
-    uint32_t event_requested;
-} QEMU_PACKED VirtIOSCSICtrlANReq;
-
-typedef struct {
-    uint32_t event_actual;
-    uint8_t response;
-} QEMU_PACKED VirtIOSCSICtrlANResp;
-
-typedef struct {
-    uint32_t event;
-    uint8_t lun[8];
-    uint32_t reason;
-} QEMU_PACKED VirtIOSCSIEvent;
-
-typedef struct {
-    uint32_t num_queues;
-    uint32_t seg_max;
-    uint32_t max_sectors;
-    uint32_t cmd_per_lun;
-    uint32_t event_info_size;
-    uint32_t sense_size;
-    uint32_t cdb_size;
-    uint16_t max_channel;
-    uint16_t max_target;
-    uint32_t max_lun;
-} QEMU_PACKED VirtIOSCSIConfig;
+typedef struct virtio_scsi_cmd_req VirtIOSCSICmdReq;
+typedef struct virtio_scsi_cmd_resp VirtIOSCSICmdResp;
+typedef struct virtio_scsi_ctrl_tmf_req VirtIOSCSICtrlTMFReq;
+typedef struct virtio_scsi_ctrl_tmf_resp VirtIOSCSICtrlTMFResp;
+typedef struct virtio_scsi_ctrl_an_req VirtIOSCSICtrlANReq;
+typedef struct virtio_scsi_ctrl_an_resp VirtIOSCSICtrlANResp;
+typedef struct virtio_scsi_event VirtIOSCSIEvent;
+typedef struct virtio_scsi_config VirtIOSCSIConfig;
 
 struct VirtIOSCSIConf {
     uint32_t num_queues;
@@ -153,18 +51,12 @@
     uint32_t cmd_per_lun;
     char *vhostfd;
     char *wwpn;
+    uint32_t boot_tpgt;
     IOThread *iothread;
 };
 
 struct VirtIOSCSI;
 
-typedef struct {
-    struct VirtIOSCSI *parent;
-    Vring vring;
-    EventNotifier host_notifier;
-    EventNotifier guest_notifier;
-} VirtIOSCSIVring;
-
 typedef struct VirtIOSCSICommon {
     VirtIODevice parent_obj;
     VirtIOSCSIConf conf;
@@ -186,37 +78,25 @@
     /* Fields for dataplane below */
     AioContext *ctx; /* one iothread per virtio-scsi-pci for now */
 
-    /* Vring is used instead of vq in dataplane code, because of the underlying
-     * memory layer thread safety */
-    VirtIOSCSIVring *ctrl_vring;
-    VirtIOSCSIVring *event_vring;
-    VirtIOSCSIVring **cmd_vrings;
     bool dataplane_started;
     bool dataplane_starting;
     bool dataplane_stopping;
-    bool dataplane_disabled;
     bool dataplane_fenced;
-    Error *blocker;
-    Notifier migration_state_notifier;
+    uint32_t host_features;
 } VirtIOSCSI;
 
 typedef struct VirtIOSCSIReq {
+    /* Note:
+     * - fields up to resp_iov are initialized by virtio_scsi_init_req;
+     * - fields starting at vring are zeroed by virtio_scsi_init_req.
+     * */
+    VirtQueueElement elem;
+
     VirtIOSCSI *dev;
     VirtQueue *vq;
     QEMUSGList qsgl;
     QEMUIOVector resp_iov;
 
-    /* Note:
-     * - fields before elem are initialized by virtio_scsi_init_req;
-     * - elem is uninitialized at the time of allocation.
-     * - fields after elem (except the ending cdb[]) are zeroed by
-     *   virtio_scsi_init_req.
-     * */
-
-    VirtQueueElement elem;
-    /* Set by dataplane code. */
-    VirtIOSCSIVring *vring;
-
     union {
         /* Used for two-stage request submission */
         QTAILQ_ENTRY(VirtIOSCSIReq) next;
@@ -235,42 +115,21 @@
         VirtIOSCSIEvent       event;
     } resp;
     union {
-        struct {
-            VirtIOSCSICmdReq  cmd;
-            uint8_t           cdb[];
-        } QEMU_PACKED;
+        VirtIOSCSICmdReq      cmd;
         VirtIOSCSICtrlTMFReq  tmf;
         VirtIOSCSICtrlANReq   an;
     } req;
 } VirtIOSCSIReq;
 
-QEMU_BUILD_BUG_ON(offsetof(VirtIOSCSIReq, req.cdb) !=
-                  offsetof(VirtIOSCSIReq, req.cmd) + sizeof(VirtIOSCSICmdReq));
-
-#define DEFINE_VIRTIO_SCSI_PROPERTIES(_state, _conf_field)                     \
-    DEFINE_PROP_UINT32("num_queues", _state, _conf_field.num_queues, 1),       \
-    DEFINE_PROP_UINT32("max_sectors", _state, _conf_field.max_sectors, 0xFFFF),\
-    DEFINE_PROP_UINT32("cmd_per_lun", _state, _conf_field.cmd_per_lun, 128)
-
-#define DEFINE_VIRTIO_SCSI_FEATURES(_state, _feature_field)                    \
-    DEFINE_PROP_BIT("any_layout", _state, _feature_field,                      \
-                    VIRTIO_F_ANY_LAYOUT, true),                                \
-    DEFINE_PROP_BIT("hotplug", _state, _feature_field, VIRTIO_SCSI_F_HOTPLUG,  \
-                                                       true),                  \
-    DEFINE_PROP_BIT("param_change", _state, _feature_field,                    \
-                                            VIRTIO_SCSI_F_CHANGE, true)
-
-typedef void (*HandleOutput)(VirtIODevice *, VirtQueue *);
-
 void virtio_scsi_common_realize(DeviceState *dev, Error **errp,
-                                HandleOutput ctrl, HandleOutput evt,
-                                HandleOutput cmd);
+                                VirtIOHandleOutput ctrl, VirtIOHandleOutput evt,
+                                VirtIOHandleOutput cmd);
 
 void virtio_scsi_common_unrealize(DeviceState *dev, Error **errp);
-void virtio_scsi_handle_ctrl_req(VirtIOSCSI *s, VirtIOSCSIReq *req);
-bool virtio_scsi_handle_cmd_req_prepare(VirtIOSCSI *s, VirtIOSCSIReq *req);
-void virtio_scsi_handle_cmd_req_submit(VirtIOSCSI *s, VirtIOSCSIReq *req);
-VirtIOSCSIReq *virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_handle_event_vq(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_handle_cmd_vq(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_handle_ctrl_vq(VirtIOSCSI *s, VirtQueue *vq);
+void virtio_scsi_init_req(VirtIOSCSI *s, VirtQueue *vq, VirtIOSCSIReq *req);
 void virtio_scsi_free_req(VirtIOSCSIReq *req);
 void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
                             uint32_t event, uint32_t reason);
@@ -278,8 +137,6 @@
 void virtio_scsi_set_iothread(VirtIOSCSI *s, IOThread *iothread);
 void virtio_scsi_dataplane_start(VirtIOSCSI *s);
 void virtio_scsi_dataplane_stop(VirtIOSCSI *s);
-void virtio_scsi_vring_push_notify(VirtIOSCSIReq *req);
-VirtIOSCSIReq *virtio_scsi_pop_req_vring(VirtIOSCSI *s,
-                                         VirtIOSCSIVring *vring);
+void virtio_scsi_dataplane_notify(VirtIODevice *vdev, VirtIOSCSIReq *req);
 
-#endif /* _QEMU_VIRTIO_SCSI_H */
+#endif /* QEMU_VIRTIO_SCSI_H */
diff --git a/include/hw/virtio/virtio-serial.h b/include/hw/virtio/virtio-serial.h
index b434f78..730c88d 100644
--- a/include/hw/virtio/virtio-serial.h
+++ b/include/hw/virtio/virtio-serial.h
@@ -12,56 +12,19 @@
  * the COPYING file in the top-level directory.
  *
  */
-#ifndef _QEMU_VIRTIO_SERIAL_H
-#define _QEMU_VIRTIO_SERIAL_H
 
+#ifndef QEMU_VIRTIO_SERIAL_H
+#define QEMU_VIRTIO_SERIAL_H
+
+#include "standard-headers/linux/virtio_console.h"
 #include "hw/qdev.h"
 #include "hw/virtio/virtio.h"
 
-/* == Interface shared between the guest kernel and qemu == */
-
-/* The Virtio ID for virtio console / serial ports */
-#define VIRTIO_ID_CONSOLE		3
-
-/* Features supported */
-#define VIRTIO_CONSOLE_F_MULTIPORT	1
-
-#define VIRTIO_CONSOLE_BAD_ID           (~(uint32_t)0)
-
-struct virtio_console_config {
-    /*
-     * These two fields are used by VIRTIO_CONSOLE_F_SIZE which
-     * isn't implemented here yet
-     */
-    uint16_t cols;
-    uint16_t rows;
-
-    uint32_t max_nr_ports;
-} QEMU_PACKED;
-
-struct virtio_console_control {
-    uint32_t id;		/* Port number */
-    uint16_t event;		/* The kind of control event (see below) */
-    uint16_t value;		/* Extra information for the key */
-};
-
 struct virtio_serial_conf {
     /* Max. number of ports we can have for a virtio-serial device */
     uint32_t max_virtserial_ports;
 };
 
-/* Some events for the internal messages (control packets) */
-#define VIRTIO_CONSOLE_DEVICE_READY	0
-#define VIRTIO_CONSOLE_PORT_ADD		1
-#define VIRTIO_CONSOLE_PORT_REMOVE	2
-#define VIRTIO_CONSOLE_PORT_READY	3
-#define VIRTIO_CONSOLE_CONSOLE_PORT	4
-#define VIRTIO_CONSOLE_RESIZE		5
-#define VIRTIO_CONSOLE_PORT_OPEN	6
-#define VIRTIO_CONSOLE_PORT_NAME	7
-
-/* == In-qemu interface == */
-
 #define TYPE_VIRTIO_SERIAL_PORT "virtio-serial-port"
 #define VIRTIO_SERIAL_PORT(obj) \
      OBJECT_CHECK(VirtIOSerialPort, (obj), TYPE_VIRTIO_SERIAL_PORT)
@@ -98,7 +61,15 @@
         /* Guest is now ready to accept data (virtqueues set up). */
     void (*guest_ready)(VirtIOSerialPort *port);
 
-        /* Guest vq became writable again */
+        /*
+         * Guest has enqueued a buffer for the host to write into.
+         * Called each time a buffer is enqueued by the guest;
+         * irrespective of whether there already were free buffers the
+         * host could have consumed.
+         *
+         * This is dependent on both the guest and host end being
+         * connected.
+         */
     void (*guest_writable)(VirtIOSerialPort *port);
 
     /*
@@ -152,7 +123,7 @@
      * element popped and continue consuming it once the backend
      * becomes writable again.
      */
-    VirtQueueElement elem;
+    VirtQueueElement *elem;
 
     /*
      * The index and the offset into the iov buffer that was popped in
@@ -210,8 +181,6 @@
     /* bitmap for identifying active ports */
     uint32_t *ports_map;
 
-    struct virtio_console_config config;
-
     struct VirtIOSerialPostLoad *post_load;
 
     virtio_serial_conf serial;
@@ -253,7 +222,4 @@
 #define VIRTIO_SERIAL(obj) \
         OBJECT_CHECK(VirtIOSerial, (obj), TYPE_VIRTIO_SERIAL)
 
-#define DEFINE_VIRTIO_SERIAL_PROPERTIES(_state, _field) \
-        DEFINE_PROP_UINT32("max_ports", _state, _field.max_virtserial_ports, 31)
-
 #endif
diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
index 0726d76..d2490c1 100644
--- a/include/hw/virtio/virtio.h
+++ b/include/hw/virtio/virtio.h
@@ -11,63 +11,23 @@
  *
  */
 
-#ifndef _QEMU_VIRTIO_H
-#define _QEMU_VIRTIO_H
+#ifndef QEMU_VIRTIO_H
+#define QEMU_VIRTIO_H
 
 #include "hw/hw.h"
 #include "net/net.h"
 #include "hw/qdev.h"
 #include "sysemu/sysemu.h"
 #include "qemu/event_notifier.h"
-#ifdef CONFIG_VIRTFS
-#include "hw/virtio/virtio-9p.h"
-#endif
+#include "standard-headers/linux/virtio_config.h"
+#include "standard-headers/linux/virtio_ring.h"
 
-/* from Linux's linux/virtio_config.h */
-
-/* Status byte for guest to report progress, and synchronize features. */
-/* We have seen device and processed generic fields (VIRTIO_CONFIG_F_VIRTIO) */
-#define VIRTIO_CONFIG_S_ACKNOWLEDGE     1
-/* We have found a driver for the device. */
-#define VIRTIO_CONFIG_S_DRIVER          2
-/* Driver has used its parts of the config, and is happy */
-#define VIRTIO_CONFIG_S_DRIVER_OK       4
-/* We've given up on this device. */
-#define VIRTIO_CONFIG_S_FAILED          0x80
-
-/* Some virtio feature bits (currently bits 28 through 31) are reserved for the
- * transport being used (eg. virtio_ring), the rest are per-device feature bits. */
-#define VIRTIO_TRANSPORT_F_START        28
-#define VIRTIO_TRANSPORT_F_END          32
-
-/* We notify when the ring is completely used, even if the guest is suppressing
- * callbacks */
-#define VIRTIO_F_NOTIFY_ON_EMPTY        24
-/* Can the device handle any descriptor layout? */
-#define VIRTIO_F_ANY_LAYOUT             27
-/* We support indirect buffer descriptors */
-#define VIRTIO_RING_F_INDIRECT_DESC     28
-/* The Guest publishes the used index for which it expects an interrupt
- * at the end of the avail ring. Host should ignore the avail->flags field. */
-/* The Host publishes the avail index for which it expects a kick
- * at the end of the used ring. Guest should ignore the used->flags field. */
-#define VIRTIO_RING_F_EVENT_IDX         29
 /* A guest should never accept this.  It implies negotiation is broken. */
 #define VIRTIO_F_BAD_FEATURE		30
 
-/* from Linux's linux/virtio_ring.h */
-
-/* This marks a buffer as continuing via the next field. */
-#define VRING_DESC_F_NEXT       1
-/* This marks a buffer as write-only (otherwise read-only). */
-#define VRING_DESC_F_WRITE      2
-/* This means the buffer contains a list of buffer descriptors. */
-#define VRING_DESC_F_INDIRECT  4
-
-/* This means don't notify other side when buffer added. */
-#define VRING_USED_F_NO_NOTIFY  1
-/* This means don't interrupt guest when buffer consumed. */
-#define VRING_AVAIL_F_NO_INTERRUPT      1
+#define VIRTIO_LEGACY_FEATURES ((0x1ULL << VIRTIO_F_BAD_FEATURE) | \
+                                (0x1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | \
+                                (0x1ULL << VIRTIO_F_ANY_LAYOUT))
 
 struct VirtQueue;
 
@@ -86,13 +46,13 @@
     unsigned int index;
     unsigned int out_num;
     unsigned int in_num;
-    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
-    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
-    struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
-    struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
+    hwaddr *in_addr;
+    hwaddr *out_addr;
+    struct iovec *in_sg;
+    struct iovec *out_sg;
 } VirtQueueElement;
 
-#define VIRTIO_PCI_QUEUE_MAX 64
+#define VIRTIO_QUEUE_MAX 1024
 
 #define VIRTIO_NO_VECTOR 0xffff
 
@@ -117,10 +77,12 @@
     uint8_t status;
     uint8_t isr;
     uint16_t queue_sel;
-    uint32_t guest_features;
+    uint64_t guest_features;
+    uint64_t host_features;
     size_t config_len;
     void *config;
     uint16_t config_vector;
+    uint32_t generation;
     int nvectors;
     VirtQueue *vq;
     uint16_t device_id;
@@ -128,6 +90,8 @@
     VMChangeStateEntry *vmstate;
     char *bus_name;
     uint8_t device_endian;
+    bool use_guest_notifier_mask;
+    QLIST_HEAD(, VirtQueue) *vector_queues;
 };
 
 typedef struct VirtioDeviceClass {
@@ -138,9 +102,12 @@
     /* This is what a VirtioDevice must implement */
     DeviceRealize realize;
     DeviceUnrealize unrealize;
-    uint32_t (*get_features)(VirtIODevice *vdev, uint32_t requested_features);
-    uint32_t (*bad_features)(VirtIODevice *vdev);
-    void (*set_features)(VirtIODevice *vdev, uint32_t val);
+    uint64_t (*get_features)(VirtIODevice *vdev,
+                             uint64_t requested_features,
+                             Error **errp);
+    uint64_t (*bad_features)(VirtIODevice *vdev);
+    void (*set_features)(VirtIODevice *vdev, uint64_t val);
+    int (*validate_features)(VirtIODevice *vdev);
     void (*get_config)(VirtIODevice *vdev, uint8_t *config);
     void (*set_config)(VirtIODevice *vdev, const uint8_t *config);
     void (*reset)(VirtIODevice *vdev);
@@ -171,30 +138,59 @@
 /* Set the child bus name. */
 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name);
 
+typedef void (*VirtIOHandleOutput)(VirtIODevice *, VirtQueue *);
+
 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
-                            void (*handle_output)(VirtIODevice *,
-                                                  VirtQueue *));
+                            VirtIOHandleOutput handle_output);
+
+VirtQueue *virtio_add_queue_aio(VirtIODevice *vdev, int queue_size,
+                                VirtIOHandleOutput handle_output);
 
 void virtio_del_queue(VirtIODevice *vdev, int n);
 
+void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num);
 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len);
 void virtqueue_flush(VirtQueue *vq, unsigned int count);
+void virtqueue_discard(VirtQueue *vq, const VirtQueueElement *elem,
+                       unsigned int len);
 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len, unsigned int idx);
 
-void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
-    size_t num_sg, int is_write);
-int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem);
+void virtqueue_map(VirtQueueElement *elem);
+void *virtqueue_pop(VirtQueue *vq, size_t sz);
+void *qemu_get_virtqueue_element(QEMUFile *f, size_t sz);
+void qemu_put_virtqueue_element(QEMUFile *f, VirtQueueElement *elem);
 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
                           unsigned int out_bytes);
 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
                                unsigned int *out_bytes,
                                unsigned max_in_bytes, unsigned max_out_bytes);
 
+bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq);
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq);
 
 void virtio_save(VirtIODevice *vdev, QEMUFile *f);
+void virtio_vmstate_save(QEMUFile *f, void *opaque, size_t size);
+
+#define VMSTATE_VIRTIO_DEVICE(devname, v, getf, putf) \
+    static const VMStateDescription vmstate_virtio_ ## devname = { \
+        .name = "virtio-" #devname ,          \
+        .minimum_version_id = v,              \
+        .version_id = v,                      \
+        .fields = (VMStateField[]) {          \
+            {                                 \
+                .name = "virtio",             \
+                .info = &(const VMStateInfo) {\
+                        .name = "virtio",     \
+                        .get = getf,          \
+                        .put = putf,          \
+                    },                        \
+                .flags = VMS_SINGLE,          \
+            },                                \
+            VMSTATE_END_OF_LIST()             \
+        }                                     \
+    }
 
 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id);
 
@@ -214,34 +210,49 @@
 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data);
 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data);
 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data);
+uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr);
+uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr);
+uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr);
+void virtio_config_modern_writeb(VirtIODevice *vdev,
+                                 uint32_t addr, uint32_t data);
+void virtio_config_modern_writew(VirtIODevice *vdev,
+                                 uint32_t addr, uint32_t data);
+void virtio_config_modern_writel(VirtIODevice *vdev,
+                                 uint32_t addr, uint32_t data);
 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr);
 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n);
 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num);
 int virtio_queue_get_num(VirtIODevice *vdev, int n);
+int virtio_get_num_queues(VirtIODevice *vdev);
+void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
+                            hwaddr avail, hwaddr used);
+void virtio_queue_update_rings(VirtIODevice *vdev, int n);
 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align);
 void virtio_queue_notify(VirtIODevice *vdev, int n);
 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n);
 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector);
-void virtio_set_status(VirtIODevice *vdev, uint8_t val);
+int virtio_set_status(VirtIODevice *vdev, uint8_t val);
 void virtio_reset(void *opaque);
 void virtio_update_irq(VirtIODevice *vdev);
-int virtio_set_features(VirtIODevice *vdev, uint32_t val);
+int virtio_set_features(VirtIODevice *vdev, uint64_t val);
 
 /* Base devices.  */
 typedef struct VirtIOBlkConf VirtIOBlkConf;
 struct virtio_net_conf;
-VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
-                              struct virtio_net_conf *net,
-                              uint32_t host_features);
 typedef struct virtio_serial_conf virtio_serial_conf;
+typedef struct virtio_input_conf virtio_input_conf;
 typedef struct VirtIOSCSIConf VirtIOSCSIConf;
 typedef struct VirtIORNGConf VirtIORNGConf;
 
 #define DEFINE_VIRTIO_COMMON_FEATURES(_state, _field) \
-	DEFINE_PROP_BIT("indirect_desc", _state, _field, \
-			VIRTIO_RING_F_INDIRECT_DESC, true), \
-	DEFINE_PROP_BIT("event_idx", _state, _field, \
-			VIRTIO_RING_F_EVENT_IDX, true)
+    DEFINE_PROP_BIT64("indirect_desc", _state, _field,    \
+                      VIRTIO_RING_F_INDIRECT_DESC, true), \
+    DEFINE_PROP_BIT64("event_idx", _state, _field,        \
+                      VIRTIO_RING_F_EVENT_IDX, true),     \
+    DEFINE_PROP_BIT64("notify_on_empty", _state, _field,  \
+                      VIRTIO_F_NOTIFY_ON_EMPTY, true), \
+    DEFINE_PROP_BIT64("any_layout", _state, _field, \
+                      VIRTIO_F_ANY_LAYOUT, true)
 
 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n);
 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n);
@@ -256,19 +267,56 @@
 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n);
 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n);
 uint16_t virtio_get_queue_index(VirtQueue *vq);
-int virtio_queue_get_id(VirtQueue *vq);
 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq);
 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                 bool with_irqfd);
 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
 void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                bool set_handler);
-void virtio_queue_notify_vq(VirtQueue *vq);
+void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
+                                                void (*fn)(VirtIODevice *,
+                                                           VirtQueue *));
 void virtio_irq(VirtQueue *vq);
+VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector);
+VirtQueue *virtio_vector_next_queue(VirtQueue *vq);
+
+static inline void virtio_add_feature(uint64_t *features, unsigned int fbit)
+{
+    assert(fbit < 64);
+    *features |= (1ULL << fbit);
+}
+
+static inline void virtio_clear_feature(uint64_t *features, unsigned int fbit)
+{
+    assert(fbit < 64);
+    *features &= ~(1ULL << fbit);
+}
+
+static inline bool virtio_has_feature(uint64_t features, unsigned int fbit)
+{
+    assert(fbit < 64);
+    return !!(features & (1ULL << fbit));
+}
+
+static inline bool virtio_vdev_has_feature(VirtIODevice *vdev,
+                                           unsigned int fbit)
+{
+    return virtio_has_feature(vdev->guest_features, fbit);
+}
+
+static inline bool virtio_host_has_feature(VirtIODevice *vdev,
+                                           unsigned int fbit)
+{
+    return virtio_has_feature(vdev->host_features, fbit);
+}
 
 static inline bool virtio_is_big_endian(VirtIODevice *vdev)
 {
-    assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
-    return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
+    if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
+        assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
+        return vdev->device_endian == VIRTIO_DEVICE_ENDIAN_BIG;
+    }
+    /* Devices conforming to VIRTIO 1.0 or later are always LE. */
+    return false;
 }
 #endif
diff --git a/include/hw/virtio/virtio_ring.h b/include/hw/virtio/virtio_ring.h
deleted file mode 100644
index 0b42e6e..0000000
--- a/include/hw/virtio/virtio_ring.h
+++ /dev/null
@@ -1,167 +0,0 @@
-#ifndef _LINUX_VIRTIO_RING_H
-#define _LINUX_VIRTIO_RING_H
-/*
- * This file is copied from /usr/include/linux while converting __uNN types
- * to uXX_t, __inline__ to inline, and tab to spaces.
- * */
-
-/* An interface for efficient virtio implementation, currently for use by KVM
- * and lguest, but hopefully others soon.  Do NOT change this since it will
- * break existing servers and clients.
- *
- * This header is BSD licensed so anyone can use the definitions to implement
- * compatible drivers/servers.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of IBM nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Copyright Rusty Russell IBM Corporation 2007. */
-
-/* This marks a buffer as continuing via the next field. */
-#define VRING_DESC_F_NEXT   1
-/* This marks a buffer as write-only (otherwise read-only). */
-#define VRING_DESC_F_WRITE  2
-/* This means the buffer contains a list of buffer descriptors. */
-#define VRING_DESC_F_INDIRECT   4
-
-/* The Host uses this in used->flags to advise the Guest: don't kick me when
- * you add a buffer.  It's unreliable, so it's simply an optimization.  Guest
- * will still kick if it's out of buffers. */
-#define VRING_USED_F_NO_NOTIFY  1
-/* The Guest uses this in avail->flags to advise the Host: don't interrupt me
- * when you consume a buffer.  It's unreliable, so it's simply an
- * optimization.  */
-#define VRING_AVAIL_F_NO_INTERRUPT  1
-
-/* We support indirect buffer descriptors */
-#define VIRTIO_RING_F_INDIRECT_DESC 28
-
-/* The Guest publishes the used index for which it expects an interrupt
- * at the end of the avail ring. Host should ignore the avail->flags field. */
-/* The Host publishes the avail index for which it expects a kick
- * at the end of the used ring. Guest should ignore the used->flags field. */
-#define VIRTIO_RING_F_EVENT_IDX     29
-
-/* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
-struct vring_desc {
-    /* Address (guest-physical). */
-    uint64_t addr;
-    /* Length. */
-    uint32_t len;
-    /* The flags as indicated above. */
-    uint16_t flags;
-    /* We chain unused descriptors via this, too */
-    uint16_t next;
-};
-
-struct vring_avail {
-    uint16_t flags;
-    uint16_t idx;
-    uint16_t ring[];
-};
-
-/* u32 is used here for ids for padding reasons. */
-struct vring_used_elem {
-    /* Index of start of used descriptor chain. */
-    uint32_t id;
-    /* Total length of the descriptor chain which was used (written to) */
-    uint32_t len;
-};
-
-struct vring_used {
-    uint16_t flags;
-    uint16_t idx;
-    struct vring_used_elem ring[];
-};
-
-struct vring {
-    unsigned int num;
-
-    struct vring_desc *desc;
-
-    struct vring_avail *avail;
-
-    struct vring_used *used;
-};
-
-/* The standard layout for the ring is a continuous chunk of memory which looks
- * like this.  We assume num is a power of 2.
- *
- * struct vring
- * {
- *  // The actual descriptors (16 bytes each)
- *  struct vring_desc desc[num];
- *
- *  // A ring of available descriptor heads with free-running index.
- *  uint16_t avail_flags;
- *  uint16_t avail_idx;
- *  uint16_t available[num];
- *  uint16_t used_event_idx;
- *
- *  // Padding to the next align boundary.
- *  char pad[];
- *
- *  // A ring of used descriptor heads with free-running index.
- *  uint16_t used_flags;
- *  uint16_t used_idx;
- *  struct vring_used_elem used[num];
- *  uint16_t avail_event_idx;
- * };
- */
-/* We publish the used event index at the end of the available ring, and vice
- * versa. They are at the end for backwards compatibility. */
-#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num])
-#define vring_avail_event(vr) (*(uint16_t *)&(vr)->used->ring[(vr)->num])
-
-static inline void vring_init(struct vring *vr, unsigned int num, void *p,
-                  unsigned long align)
-{
-    vr->num = num;
-    vr->desc = p;
-    vr->avail = p + num*sizeof(struct vring_desc);
-    vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(uint16_t)
-        + align - 1) & ~(align - 1));
-}
-
-static inline unsigned vring_size(unsigned int num, unsigned long align)
-{
-    return ((sizeof(struct vring_desc) * num + sizeof(uint16_t) * (3 + num)
-         + align - 1) & ~(align - 1))
-        + sizeof(uint16_t) * 3 + sizeof(struct vring_used_elem) * num;
-}
-
-/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */
-/* Assuming a given event_idx value from the other size, if
- * we have just incremented index from old to new_idx,
- * should we trigger an event? */
-static inline int vring_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
-{
-    /* Note: Xen has similar logic for notification hold-off
-     * in include/xen/interface/io/ring.h with req_event and req_prod
-     * corresponding to event_idx + 1 and new_idx respectively.
-     * Note also that req_event and req_prod in Xen start at 1,
-     * event indexes in virtio start at 0. */
-    return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old);
-}
-
-#endif /* _LINUX_VIRTIO_RING_H */
diff --git a/include/hw/watchdog/wdt_diag288.h b/include/hw/watchdog/wdt_diag288.h
new file mode 100644
index 0000000..706d96b
--- /dev/null
+++ b/include/hw/watchdog/wdt_diag288.h
@@ -0,0 +1,36 @@
+#ifndef WDT_DIAG288_H
+#define WDT_DIAG288_H
+
+#include "hw/qdev.h"
+
+#define TYPE_WDT_DIAG288 "diag288"
+#define DIAG288(obj) \
+    OBJECT_CHECK(DIAG288State, (obj), TYPE_WDT_DIAG288)
+#define DIAG288_CLASS(klass) \
+    OBJECT_CLASS_CHECK(DIAG288Class, (klass), TYPE_WDT_DIAG288)
+#define DIAG288_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(DIAG288Class, (obj), TYPE_WDT_DIAG288)
+
+#define WDT_DIAG288_INIT      0
+#define WDT_DIAG288_CHANGE    1
+#define WDT_DIAG288_CANCEL    2
+
+typedef struct DIAG288State {
+    /*< private >*/
+    DeviceState parent_obj;
+    QEMUTimer *timer;
+    bool enabled;
+
+    /*< public >*/
+} DIAG288State;
+
+typedef struct DIAG288Class {
+    /*< private >*/
+    DeviceClass parent_class;
+
+    /*< public >*/
+    int (*handle_timer)(DIAG288State *dev,
+                        uint64_t func, uint64_t timeout);
+} DIAG288Class;
+
+#endif /* WDT_DIAG288_H */
diff --git a/include/hw/xen/xen.h b/include/hw/xen/xen.h
index b0ed04c..a8f3afb 100644
--- a/include/hw/xen/xen.h
+++ b/include/hw/xen/xen.h
@@ -1,15 +1,16 @@
 #ifndef QEMU_HW_XEN_H
-#define QEMU_HW_XEN_H 1
+#define QEMU_HW_XEN_H
+
 /*
  * public xen header
  *   stuff needed outside xen-*.c, i.e. interfaces to qemu.
  *   must not depend on any xen headers being present in
  *   /usr/include/xen, so it can be included unconditionally.
  */
-#include <inttypes.h>
 
-#include "hw/irq.h"
 #include "qemu-common.h"
+#include "exec/cpu-common.h"
+#include "hw/irq.h"
 
 /* xen-machine.c */
 enum xen_mode {
@@ -32,24 +33,18 @@
 void xen_piix3_set_irq(void *opaque, int irq_num, int level);
 void xen_piix_pci_write_config_client(uint32_t address, uint32_t val, int len);
 void xen_hvm_inject_msi(uint64_t addr, uint32_t data);
-void xen_cmos_set_s3_resume(void *opaque, int irq, int level);
+int xen_is_pirq_msi(uint32_t msi_data);
 
 qemu_irq *xen_interrupt_controller_init(void);
 
 void xenstore_store_pv_console_info(int i, struct CharDriverState *chr);
 
-#if defined(NEED_CPU_H) && !defined(CONFIG_USER_ONLY)
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory);
+void xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory);
+
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
-                   struct MemoryRegion *mr);
+                   struct MemoryRegion *mr, Error **errp);
 void xen_modified_memory(ram_addr_t start, ram_addr_t length);
-#endif
 
 void xen_register_framebuffer(struct MemoryRegion *mr);
 
-#if defined(CONFIG_XEN) && CONFIG_XEN_CTRL_INTERFACE_VERSION < 400
-#  define HVM_MAX_VCPUS 32
-#endif
-
 #endif /* QEMU_HW_XEN_H */
diff --git a/include/hw/xen/xen_backend.h b/include/hw/xen/xen_backend.h
index 3b4125e..0df282a 100644
--- a/include/hw/xen/xen_backend.h
+++ b/include/hw/xen/xen_backend.h
@@ -1,5 +1,5 @@
 #ifndef QEMU_HW_XEN_BACKEND_H
-#define QEMU_HW_XEN_BACKEND_H 1
+#define QEMU_HW_XEN_BACKEND_H
 
 #include "hw/xen/xen_common.h"
 #include "sysemu/sysemu.h"
@@ -28,6 +28,7 @@
     int       (*free)(struct XenDevice *xendev);
     void      (*backend_changed)(struct XenDevice *xendev, const char *node);
     void      (*frontend_changed)(struct XenDevice *xendev, const char *node);
+    int       (*backend_register)(void);
 };
 
 struct XenDevice {
@@ -46,8 +47,8 @@
     int                remote_port;
     int                local_port;
 
-    XenEvtchn          evtchndev;
-    XenGnttab          gnttabdev;
+    xenevtchn_handle   *evtchndev;
+    xengnttab_handle   *gnttabdev;
 
     struct XenDevOps   *ops;
     QTAILQ_ENTRY(XenDevice) next;
@@ -56,11 +57,14 @@
 /* ------------------------------------------------------------- */
 
 /* variables */
-extern XenXC xen_xc;
+extern xc_interface *xen_xc;
+extern xenforeignmemory_handle *xen_fmem;
 extern struct xs_handle *xenstore;
 extern const char *xen_protocol;
+extern DeviceState *xen_sysdev;
 
 /* xenstore helper functions */
+int xenstore_mkdir(char *path, int p);
 int xenstore_write_str(const char *base, const char *node, const char *val);
 int xenstore_write_int(const char *base, const char *node, int ival);
 int xenstore_write_int64(const char *base, const char *node, int64_t ival);
@@ -83,6 +87,7 @@
 
 /* xen backend driver bits */
 int xen_be_init(void);
+void xen_be_register_common(void);
 int xen_be_register(const char *type, struct XenDevOps *ops);
 int xen_be_set_state(struct XenDevice *xendev, enum xenbus_state state);
 int xen_be_bind_evtchn(struct XenDevice *xendev);
@@ -97,6 +102,9 @@
 extern struct XenDevOps xen_framebuffer_ops;  /* xen_framebuffer.c */
 extern struct XenDevOps xen_blkdev_ops;       /* xen_disk.c        */
 extern struct XenDevOps xen_netdev_ops;       /* xen_nic.c         */
+#ifdef CONFIG_USB_LIBUSB
+extern struct XenDevOps xen_usb_ops;          /* xen-usb.c         */
+#endif
 
 void xen_init_display(int domid);
 
diff --git a/include/hw/xen/xen_common.h b/include/hw/xen/xen_common.h
index 95612a4..bd39287 100644
--- a/include/hw/xen/xen_common.h
+++ b/include/hw/xen/xen_common.h
@@ -1,162 +1,76 @@
 #ifndef QEMU_HW_XEN_COMMON_H
-#define QEMU_HW_XEN_COMMON_H 1
+#define QEMU_HW_XEN_COMMON_H
 
-#include "config-host.h"
-
-#include <stddef.h>
-#include <inttypes.h>
+/*
+ * If we have new enough libxenctrl then we do not want/need these compat
+ * interfaces, despite what the user supplied cflags might say. They
+ * must be undefined before including xenctrl.h
+ */
+#undef XC_WANT_COMPAT_EVTCHN_API
+#undef XC_WANT_COMPAT_GNTTAB_API
+#undef XC_WANT_COMPAT_MAP_FOREIGN_API
 
 #include <xenctrl.h>
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 420
-#  include <xs.h>
-#else
-#  include <xenstore.h>
-#endif
+#include <xenstore.h>
 #include <xen/io/xenbus.h>
 
 #include "hw/hw.h"
 #include "hw/xen/xen.h"
+#include "hw/pci/pci.h"
 #include "qemu/queue.h"
+#include "trace.h"
 
 /*
- * We don't support Xen prior to 3.3.0.
+ * We don't support Xen prior to 4.2.0.
  */
 
-/* Xen before 4.0 */
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 400
-static inline void *xc_map_foreign_bulk(int xc_handle, uint32_t dom, int prot,
-                                        xen_pfn_t *arr, int *err,
-                                        unsigned int num)
+/* Xen 4.2 through 4.6 */
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 471
+
+typedef xc_interface xenforeignmemory_handle;
+typedef xc_evtchn xenevtchn_handle;
+typedef xc_gnttab xengnttab_handle;
+
+#define xenevtchn_open(l, f) xc_evtchn_open(l, f);
+#define xenevtchn_close(h) xc_evtchn_close(h)
+#define xenevtchn_fd(h) xc_evtchn_fd(h)
+#define xenevtchn_pending(h) xc_evtchn_pending(h)
+#define xenevtchn_notify(h, p) xc_evtchn_notify(h, p)
+#define xenevtchn_bind_interdomain(h, d, p) xc_evtchn_bind_interdomain(h, d, p)
+#define xenevtchn_unmask(h, p) xc_evtchn_unmask(h, p)
+#define xenevtchn_unbind(h, p) xc_evtchn_unbind(h, p)
+
+#define xengnttab_open(l, f) xc_gnttab_open(l, f)
+#define xengnttab_close(h) xc_gnttab_close(h)
+#define xengnttab_set_max_grants(h, n) xc_gnttab_set_max_grants(h, n)
+#define xengnttab_map_grant_ref(h, d, r, p) xc_gnttab_map_grant_ref(h, d, r, p)
+#define xengnttab_unmap(h, a, n) xc_gnttab_munmap(h, a, n)
+#define xengnttab_map_grant_refs(h, c, d, r, p) \
+    xc_gnttab_map_grant_refs(h, c, d, r, p)
+#define xengnttab_map_domain_grant_refs(h, c, d, r, p) \
+    xc_gnttab_map_domain_grant_refs(h, c, d, r, p)
+
+#define xenforeignmemory_open(l, f) xen_xc
+
+static inline void *xenforeignmemory_map(xc_interface *h, uint32_t dom,
+                                         int prot, size_t pages,
+                                         const xen_pfn_t arr[/*pages*/],
+                                         int err[/*pages*/])
 {
-    return xc_map_foreign_batch(xc_handle, dom, prot, arr, num);
-}
-#endif
-
-
-/* Xen before 4.1 */
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 410
-
-typedef int XenXC;
-typedef int XenEvtchn;
-typedef int XenGnttab;
-
-#  define XC_INTERFACE_FMT "%i"
-#  define XC_HANDLER_INITIAL_VALUE    -1
-
-static inline XenEvtchn xen_xc_evtchn_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_evtchn_open();
+    if (err)
+        return xc_map_foreign_bulk(h, dom, prot, arr, err, pages);
+    else
+        return xc_map_foreign_pages(h, dom, prot, arr, pages);
 }
 
-static inline XenGnttab xen_xc_gnttab_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_gnttab_open();
-}
+#define xenforeignmemory_unmap(h, p, s) munmap(p, s * XC_PAGE_SIZE)
 
-static inline XenXC xen_xc_interface_open(void *logger, void *dombuild_logger,
-                                          unsigned int open_flags)
-{
-    return xc_interface_open();
-}
+#else /* CONFIG_XEN_CTRL_INTERFACE_VERSION >= 471 */
 
-static inline int xc_fd(int xen_xc)
-{
-    return xen_xc;
-}
+#include <xenevtchn.h>
+#include <xengnttab.h>
+#include <xenforeignmemory.h>
 
-
-static inline int xc_domain_populate_physmap_exact
-    (XenXC xc_handle, uint32_t domid, unsigned long nr_extents,
-     unsigned int extent_order, unsigned int mem_flags, xen_pfn_t *extent_start)
-{
-    return xc_domain_memory_populate_physmap
-        (xc_handle, domid, nr_extents, extent_order, mem_flags, extent_start);
-}
-
-static inline int xc_domain_add_to_physmap(int xc_handle, uint32_t domid,
-                                           unsigned int space, unsigned long idx,
-                                           xen_pfn_t gpfn)
-{
-    struct xen_add_to_physmap xatp = {
-        .domid = domid,
-        .space = space,
-        .idx = idx,
-        .gpfn = gpfn,
-    };
-
-    return xc_memory_op(xc_handle, XENMEM_add_to_physmap, &xatp);
-}
-
-static inline struct xs_handle *xs_open(unsigned long flags)
-{
-    return xs_daemon_open();
-}
-
-static inline void xs_close(struct xs_handle *xsh)
-{
-    if (xsh != NULL) {
-        xs_daemon_close(xsh);
-    }
-}
-
-
-/* Xen 4.1 */
-#else
-
-typedef xc_interface *XenXC;
-typedef xc_evtchn *XenEvtchn;
-typedef xc_gnttab *XenGnttab;
-
-#  define XC_INTERFACE_FMT "%p"
-#  define XC_HANDLER_INITIAL_VALUE    NULL
-
-static inline XenEvtchn xen_xc_evtchn_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_evtchn_open(logger, open_flags);
-}
-
-static inline XenGnttab xen_xc_gnttab_open(void *logger,
-                                           unsigned int open_flags)
-{
-    return xc_gnttab_open(logger, open_flags);
-}
-
-static inline XenXC xen_xc_interface_open(void *logger, void *dombuild_logger,
-                                          unsigned int open_flags)
-{
-    return xc_interface_open(logger, dombuild_logger, open_flags);
-}
-
-/* FIXME There is now way to have the xen fd */
-static inline int xc_fd(xc_interface *xen_xc)
-{
-    return -1;
-}
-#endif
-
-/* Xen before 4.2 */
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 420
-static inline int xen_xc_hvm_inject_msi(XenXC xen_xc, domid_t dom,
-        uint64_t addr, uint32_t data)
-{
-    return -ENOSYS;
-}
-/* The followings are only to compile op_discard related code on older
- * Xen releases. */
-#define BLKIF_OP_DISCARD 5
-struct blkif_request_discard {
-    uint64_t nr_sectors;
-    uint64_t sector_number;
-};
-#else
-static inline int xen_xc_hvm_inject_msi(XenXC xen_xc, domid_t dom,
-        uint64_t addr, uint32_t data)
-{
-    return xc_hvm_inject_msi(xen_xc, dom, addr, data);
-}
 #endif
 
 void destroy_hvm_domain(bool reboot);
@@ -165,18 +79,349 @@
 void xen_shutdown_fatal_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 
 #ifdef HVM_PARAM_VMPORT_REGS_PFN
-static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
-                                          unsigned long *vmport_regs_pfn)
+static inline int xen_get_vmport_regs_pfn(xc_interface *xc, domid_t dom,
+                                          xen_pfn_t *vmport_regs_pfn)
 {
-    return xc_get_hvm_param(xc, dom, HVM_PARAM_VMPORT_REGS_PFN,
-                            vmport_regs_pfn);
+    int rc;
+    uint64_t value;
+    rc = xc_hvm_param_get(xc, dom, HVM_PARAM_VMPORT_REGS_PFN, &value);
+    if (rc >= 0) {
+        *vmport_regs_pfn = (xen_pfn_t) value;
+    }
+    return rc;
 }
 #else
-static inline int xen_get_vmport_regs_pfn(XenXC xc, domid_t dom,
-                                          unsigned long *vmport_regs_pfn)
+static inline int xen_get_vmport_regs_pfn(xc_interface *xc, domid_t dom,
+                                          xen_pfn_t *vmport_regs_pfn)
 {
     return -ENOSYS;
 }
 #endif
 
+/* Xen before 4.6 */
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 460
+
+#ifndef HVM_IOREQSRV_BUFIOREQ_ATOMIC
+#define HVM_IOREQSRV_BUFIOREQ_ATOMIC 2
+#endif
+
+#endif
+
+static inline int xen_get_default_ioreq_server_info(xc_interface *xc,
+                                                    domid_t dom,
+                                                    xen_pfn_t *ioreq_pfn,
+                                                    xen_pfn_t *bufioreq_pfn,
+                                                    evtchn_port_t
+                                                        *bufioreq_evtchn)
+{
+    unsigned long param;
+    int rc;
+
+    rc = xc_get_hvm_param(xc, dom, HVM_PARAM_IOREQ_PFN, &param);
+    if (rc < 0) {
+        fprintf(stderr, "failed to get HVM_PARAM_IOREQ_PFN\n");
+        return -1;
+    }
+
+    *ioreq_pfn = param;
+
+    rc = xc_get_hvm_param(xc, dom, HVM_PARAM_BUFIOREQ_PFN, &param);
+    if (rc < 0) {
+        fprintf(stderr, "failed to get HVM_PARAM_BUFIOREQ_PFN\n");
+        return -1;
+    }
+
+    *bufioreq_pfn = param;
+
+    rc = xc_get_hvm_param(xc, dom, HVM_PARAM_BUFIOREQ_EVTCHN,
+                          &param);
+    if (rc < 0) {
+        fprintf(stderr, "failed to get HVM_PARAM_BUFIOREQ_EVTCHN\n");
+        return -1;
+    }
+
+    *bufioreq_evtchn = param;
+
+    return 0;
+}
+
+/* Xen before 4.5 */
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 450
+
+#ifndef HVM_PARAM_BUFIOREQ_EVTCHN
+#define HVM_PARAM_BUFIOREQ_EVTCHN 26
+#endif
+
+#define IOREQ_TYPE_PCI_CONFIG 2
+
+typedef uint16_t ioservid_t;
+
+static inline void xen_map_memory_section(xc_interface *xc, domid_t dom,
+                                          ioservid_t ioservid,
+                                          MemoryRegionSection *section)
+{
+}
+
+static inline void xen_unmap_memory_section(xc_interface *xc, domid_t dom,
+                                            ioservid_t ioservid,
+                                            MemoryRegionSection *section)
+{
+}
+
+static inline void xen_map_io_section(xc_interface *xc, domid_t dom,
+                                      ioservid_t ioservid,
+                                      MemoryRegionSection *section)
+{
+}
+
+static inline void xen_unmap_io_section(xc_interface *xc, domid_t dom,
+                                        ioservid_t ioservid,
+                                        MemoryRegionSection *section)
+{
+}
+
+static inline void xen_map_pcidev(xc_interface *xc, domid_t dom,
+                                  ioservid_t ioservid,
+                                  PCIDevice *pci_dev)
+{
+}
+
+static inline void xen_unmap_pcidev(xc_interface *xc, domid_t dom,
+                                    ioservid_t ioservid,
+                                    PCIDevice *pci_dev)
+{
+}
+
+static inline void xen_create_ioreq_server(xc_interface *xc, domid_t dom,
+                                           ioservid_t *ioservid)
+{
+}
+
+static inline void xen_destroy_ioreq_server(xc_interface *xc, domid_t dom,
+                                            ioservid_t ioservid)
+{
+}
+
+static inline int xen_get_ioreq_server_info(xc_interface *xc, domid_t dom,
+                                            ioservid_t ioservid,
+                                            xen_pfn_t *ioreq_pfn,
+                                            xen_pfn_t *bufioreq_pfn,
+                                            evtchn_port_t *bufioreq_evtchn)
+{
+    return xen_get_default_ioreq_server_info(xc, dom, ioreq_pfn, bufioreq_pfn,
+                                             bufioreq_evtchn);
+}
+
+static inline int xen_set_ioreq_server_state(xc_interface *xc, domid_t dom,
+                                             ioservid_t ioservid,
+                                             bool enable)
+{
+    return 0;
+}
+
+/* Xen 4.5 */
+#else
+
+static bool use_default_ioreq_server;
+
+static inline void xen_map_memory_section(xc_interface *xc, domid_t dom,
+                                          ioservid_t ioservid,
+                                          MemoryRegionSection *section)
+{
+    hwaddr start_addr = section->offset_within_address_space;
+    ram_addr_t size = int128_get64(section->size);
+    hwaddr end_addr = start_addr + size - 1;
+
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+    trace_xen_map_mmio_range(ioservid, start_addr, end_addr);
+    xc_hvm_map_io_range_to_ioreq_server(xc, dom, ioservid, 1,
+                                        start_addr, end_addr);
+}
+
+static inline void xen_unmap_memory_section(xc_interface *xc, domid_t dom,
+                                            ioservid_t ioservid,
+                                            MemoryRegionSection *section)
+{
+    hwaddr start_addr = section->offset_within_address_space;
+    ram_addr_t size = int128_get64(section->size);
+    hwaddr end_addr = start_addr + size - 1;
+
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+
+    trace_xen_unmap_mmio_range(ioservid, start_addr, end_addr);
+    xc_hvm_unmap_io_range_from_ioreq_server(xc, dom, ioservid, 1,
+                                            start_addr, end_addr);
+}
+
+static inline void xen_map_io_section(xc_interface *xc, domid_t dom,
+                                      ioservid_t ioservid,
+                                      MemoryRegionSection *section)
+{
+    hwaddr start_addr = section->offset_within_address_space;
+    ram_addr_t size = int128_get64(section->size);
+    hwaddr end_addr = start_addr + size - 1;
+
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+
+    trace_xen_map_portio_range(ioservid, start_addr, end_addr);
+    xc_hvm_map_io_range_to_ioreq_server(xc, dom, ioservid, 0,
+                                        start_addr, end_addr);
+}
+
+static inline void xen_unmap_io_section(xc_interface *xc, domid_t dom,
+                                        ioservid_t ioservid,
+                                        MemoryRegionSection *section)
+{
+    hwaddr start_addr = section->offset_within_address_space;
+    ram_addr_t size = int128_get64(section->size);
+    hwaddr end_addr = start_addr + size - 1;
+
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+    trace_xen_unmap_portio_range(ioservid, start_addr, end_addr);
+    xc_hvm_unmap_io_range_from_ioreq_server(xc, dom, ioservid, 0,
+                                            start_addr, end_addr);
+}
+
+static inline void xen_map_pcidev(xc_interface *xc, domid_t dom,
+                                  ioservid_t ioservid,
+                                  PCIDevice *pci_dev)
+{
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+    trace_xen_map_pcidev(ioservid, pci_bus_num(pci_dev->bus),
+                         PCI_SLOT(pci_dev->devfn), PCI_FUNC(pci_dev->devfn));
+    xc_hvm_map_pcidev_to_ioreq_server(xc, dom, ioservid,
+                                      0, pci_bus_num(pci_dev->bus),
+                                      PCI_SLOT(pci_dev->devfn),
+                                      PCI_FUNC(pci_dev->devfn));
+}
+
+static inline void xen_unmap_pcidev(xc_interface *xc, domid_t dom,
+                                    ioservid_t ioservid,
+                                    PCIDevice *pci_dev)
+{
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+    trace_xen_unmap_pcidev(ioservid, pci_bus_num(pci_dev->bus),
+                           PCI_SLOT(pci_dev->devfn), PCI_FUNC(pci_dev->devfn));
+    xc_hvm_unmap_pcidev_from_ioreq_server(xc, dom, ioservid,
+                                          0, pci_bus_num(pci_dev->bus),
+                                          PCI_SLOT(pci_dev->devfn),
+                                          PCI_FUNC(pci_dev->devfn));
+}
+
+static inline void xen_create_ioreq_server(xc_interface *xc, domid_t dom,
+                                           ioservid_t *ioservid)
+{
+    int rc = xc_hvm_create_ioreq_server(xc, dom, HVM_IOREQSRV_BUFIOREQ_ATOMIC,
+                                        ioservid);
+
+    if (rc == 0) {
+        trace_xen_ioreq_server_create(*ioservid);
+        return;
+    }
+
+    *ioservid = 0;
+    use_default_ioreq_server = true;
+    trace_xen_default_ioreq_server();
+}
+
+static inline void xen_destroy_ioreq_server(xc_interface *xc, domid_t dom,
+                                            ioservid_t ioservid)
+{
+    if (use_default_ioreq_server) {
+        return;
+    }
+
+    trace_xen_ioreq_server_destroy(ioservid);
+    xc_hvm_destroy_ioreq_server(xc, dom, ioservid);
+}
+
+static inline int xen_get_ioreq_server_info(xc_interface *xc, domid_t dom,
+                                            ioservid_t ioservid,
+                                            xen_pfn_t *ioreq_pfn,
+                                            xen_pfn_t *bufioreq_pfn,
+                                            evtchn_port_t *bufioreq_evtchn)
+{
+    if (use_default_ioreq_server) {
+        return xen_get_default_ioreq_server_info(xc, dom, ioreq_pfn,
+                                                 bufioreq_pfn,
+                                                 bufioreq_evtchn);
+    }
+
+    return xc_hvm_get_ioreq_server_info(xc, dom, ioservid,
+                                        ioreq_pfn, bufioreq_pfn,
+                                        bufioreq_evtchn);
+}
+
+static inline int xen_set_ioreq_server_state(xc_interface *xc, domid_t dom,
+                                             ioservid_t ioservid,
+                                             bool enable)
+{
+    if (use_default_ioreq_server) {
+        return 0;
+    }
+
+    trace_xen_ioreq_server_state(ioservid, enable);
+    return xc_hvm_set_ioreq_server_state(xc, dom, ioservid, enable);
+}
+
+#endif
+
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 460
+static inline int xen_xc_domain_add_to_physmap(xc_interface *xch, uint32_t domid,
+                                               unsigned int space,
+                                               unsigned long idx,
+                                               xen_pfn_t gpfn)
+{
+    return xc_domain_add_to_physmap(xch, domid, space, idx, gpfn);
+}
+#else
+static inline int xen_xc_domain_add_to_physmap(xc_interface *xch, uint32_t domid,
+                                               unsigned int space,
+                                               unsigned long idx,
+                                               xen_pfn_t gpfn)
+{
+    /* In Xen 4.6 rc is -1 and errno contains the error value. */
+    int rc = xc_domain_add_to_physmap(xch, domid, space, idx, gpfn);
+    if (rc == -1)
+        return errno;
+    return rc;
+}
+#endif
+
+#ifdef CONFIG_XEN_PV_DOMAIN_BUILD
+#if CONFIG_XEN_CTRL_INTERFACE_VERSION < 470
+static inline int xen_domain_create(xc_interface *xc, uint32_t ssidref,
+                                    xen_domain_handle_t handle, uint32_t flags,
+                                    uint32_t *pdomid)
+{
+    return xc_domain_create(xc, ssidref, handle, flags, pdomid);
+}
+#else
+static inline int xen_domain_create(xc_interface *xc, uint32_t ssidref,
+                                    xen_domain_handle_t handle, uint32_t flags,
+                                    uint32_t *pdomid)
+{
+    return xc_domain_create(xc, ssidref, handle, flags, pdomid, NULL);
+}
+#endif
+#endif
+
 #endif /* QEMU_HW_XEN_COMMON_H */
diff --git a/include/io/channel-buffer.h b/include/io/channel-buffer.h
new file mode 100644
index 0000000..3f4b3f2
--- /dev/null
+++ b/include/io/channel-buffer.h
@@ -0,0 +1,60 @@
+/*
+ * QEMU I/O channels memory buffer driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_BUFFER_H
+#define QIO_CHANNEL_BUFFER_H
+
+#include "io/channel.h"
+
+#define TYPE_QIO_CHANNEL_BUFFER "qio-channel-buffer"
+#define QIO_CHANNEL_BUFFER(obj)                                     \
+    OBJECT_CHECK(QIOChannelBuffer, (obj), TYPE_QIO_CHANNEL_BUFFER)
+
+typedef struct QIOChannelBuffer QIOChannelBuffer;
+
+/**
+ * QIOChannelBuffer:
+ *
+ * The QIOChannelBuffer object provides a channel implementation
+ * that is able to perform I/O to/from a memory buffer.
+ *
+ */
+
+struct QIOChannelBuffer {
+    QIOChannel parent;
+    size_t capacity; /* Total allocated memory */
+    size_t usage;    /* Current size of data */
+    size_t offset;   /* Offset for future I/O ops */
+    uint8_t *data;
+};
+
+
+/**
+ * qio_channel_buffer_new:
+ * @capacity: the initial buffer capacity to allocate
+ *
+ * Allocate a new buffer which is initially empty
+ *
+ * Returns: the new channel object
+ */
+QIOChannelBuffer *
+qio_channel_buffer_new(size_t capacity);
+
+#endif /* QIO_CHANNEL_BUFFER_H */
diff --git a/include/io/channel-command.h b/include/io/channel-command.h
new file mode 100644
index 0000000..336d47f
--- /dev/null
+++ b/include/io/channel-command.h
@@ -0,0 +1,91 @@
+/*
+ * QEMU I/O channels external command driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_COMMAND_H
+#define QIO_CHANNEL_COMMAND_H
+
+#include "io/channel.h"
+
+#define TYPE_QIO_CHANNEL_COMMAND "qio-channel-command"
+#define QIO_CHANNEL_COMMAND(obj)                                     \
+    OBJECT_CHECK(QIOChannelCommand, (obj), TYPE_QIO_CHANNEL_COMMAND)
+
+typedef struct QIOChannelCommand QIOChannelCommand;
+
+
+/**
+ * QIOChannelCommand:
+ *
+ * The QIOChannelCommand class provides a channel implementation
+ * that can transport data with an externally running command
+ * via its stdio streams.
+ */
+
+struct QIOChannelCommand {
+    QIOChannel parent;
+    int writefd;
+    int readfd;
+    pid_t pid;
+};
+
+
+/**
+ * qio_channel_command_new_pid:
+ * @writefd: the FD connected to the command's stdin
+ * @readfd: the FD connected to the command's stdout
+ * @pid: the PID of the running child command
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O with the
+ * previously spawned command identified by @pid.
+ * The two file descriptors provide the connection
+ * to command's stdio streams, either one or which
+ * may be -1 to indicate that stream is not open.
+ *
+ * The channel will take ownership of the process
+ * @pid and will kill it when closing the channel.
+ * Similarly it will take responsibility for
+ * closing the file descriptors @writefd and @readfd.
+ *
+ * Returns: the command channel object, or NULL on error
+ */
+QIOChannelCommand *
+qio_channel_command_new_pid(int writefd,
+                            int readfd,
+                            pid_t pid);
+
+/**
+ * qio_channel_command_new_spawn:
+ * @argv: the NULL terminated list of command arguments
+ * @flags: the I/O mode, one of O_RDONLY, O_WRONLY, O_RDWR
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O with the
+ * command to be spawned with arguments @argv.
+ *
+ * Returns: the command channel object, or NULL on error
+ */
+QIOChannelCommand *
+qio_channel_command_new_spawn(const char *const argv[],
+                              int flags,
+                              Error **errp);
+
+
+#endif /* QIO_CHANNEL_COMMAND_H */
diff --git a/include/io/channel-file.h b/include/io/channel-file.h
new file mode 100644
index 0000000..d2462c2
--- /dev/null
+++ b/include/io/channel-file.h
@@ -0,0 +1,93 @@
+/*
+ * QEMU I/O channels files driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_FILE_H
+#define QIO_CHANNEL_FILE_H
+
+#include "io/channel.h"
+
+#define TYPE_QIO_CHANNEL_FILE "qio-channel-file"
+#define QIO_CHANNEL_FILE(obj)                                     \
+    OBJECT_CHECK(QIOChannelFile, (obj), TYPE_QIO_CHANNEL_FILE)
+
+typedef struct QIOChannelFile QIOChannelFile;
+
+/**
+ * QIOChannelFile:
+ *
+ * The QIOChannelFile object provides a channel implementation
+ * that is able to perform I/O on block devices, character
+ * devices, FIFOs, pipes and plain files. While it is technically
+ * able to work on sockets too on the UNIX platform, this is not
+ * portable to Windows and lacks some extra sockets specific
+ * functionality. So the QIOChannelSocket object is recommended
+ * for that use case.
+ *
+ */
+
+struct QIOChannelFile {
+    QIOChannel parent;
+    int fd;
+};
+
+
+/**
+ * qio_channel_file_new_fd:
+ * @fd: the file descriptor
+ *
+ * Create a new IO channel object for a file represented
+ * by the @fd parameter. @fd can be associated with a
+ * block device, character device, fifo, pipe, or a
+ * regular file. For sockets, the QIOChannelSocket class
+ * should be used instead, as this provides greater
+ * functionality and cross platform portability.
+ *
+ * The channel will own the passed in file descriptor
+ * and will take responsibility for closing it, so the
+ * caller must not close it. If appropriate the caller
+ * should dup() its FD before opening the channel.
+ *
+ * Returns: the new channel object
+ */
+QIOChannelFile *
+qio_channel_file_new_fd(int fd);
+
+/**
+ * qio_channel_file_new_path:
+ * @fd: the file descriptor
+ * @flags: the open flags (O_RDONLY|O_WRONLY|O_RDWR, etc)
+ * @mode: the file creation mode if O_WRONLY is set in @flags
+ * @errp: pointer to initialized error object
+ *
+ * Create a new IO channel object for a file represented
+ * by the @path parameter. @path can point to any
+ * type of file on which sequential I/O can be
+ * performed, whether it be a plain file, character
+ * device or block device.
+ *
+ * Returns: the new channel object
+ */
+QIOChannelFile *
+qio_channel_file_new_path(const char *path,
+                          int flags,
+                          mode_t mode,
+                          Error **errp);
+
+#endif /* QIO_CHANNEL_FILE_H */
diff --git a/include/io/channel-socket.h b/include/io/channel-socket.h
new file mode 100644
index 0000000..711f8bf
--- /dev/null
+++ b/include/io/channel-socket.h
@@ -0,0 +1,251 @@
+/*
+ * QEMU I/O channels sockets driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_SOCKET_H
+#define QIO_CHANNEL_SOCKET_H
+
+#include "io/channel.h"
+#include "io/task.h"
+#include "qemu/sockets.h"
+
+#define TYPE_QIO_CHANNEL_SOCKET "qio-channel-socket"
+#define QIO_CHANNEL_SOCKET(obj)                                     \
+    OBJECT_CHECK(QIOChannelSocket, (obj), TYPE_QIO_CHANNEL_SOCKET)
+
+typedef struct QIOChannelSocket QIOChannelSocket;
+
+/**
+ * QIOChannelSocket:
+ *
+ * The QIOChannelSocket class provides a channel implementation
+ * that can transport data over a UNIX socket or TCP socket.
+ * Beyond the core channel API, it also provides functionality
+ * for accepting client connections, tuning some socket
+ * parameters and getting socket address strings.
+ */
+
+struct QIOChannelSocket {
+    QIOChannel parent;
+    int fd;
+    struct sockaddr_storage localAddr;
+    socklen_t localAddrLen;
+    struct sockaddr_storage remoteAddr;
+    socklen_t remoteAddrLen;
+};
+
+
+/**
+ * qio_channel_socket_new:
+ *
+ * Create a channel for performing I/O on a socket
+ * connection, that is initially closed. After
+ * creating the socket, it must be setup as a client
+ * connection or server.
+ *
+ * Returns: the socket channel object
+ */
+QIOChannelSocket *
+qio_channel_socket_new(void);
+
+/**
+ * qio_channel_socket_new_fd:
+ * @fd: the socket file descriptor
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O on the socket
+ * connection represented by the file descriptor @fd.
+ *
+ * Returns: the socket channel object, or NULL on error
+ */
+QIOChannelSocket *
+qio_channel_socket_new_fd(int fd,
+                          Error **errp);
+
+
+/**
+ * qio_channel_socket_connect_sync:
+ * @ioc: the socket channel object
+ * @addr: the address to connect to
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to connect to the address @addr. This method
+ * will run in the foreground so the caller will not regain
+ * execution control until the connection is established or
+ * an error occurs.
+ */
+int qio_channel_socket_connect_sync(QIOChannelSocket *ioc,
+                                    SocketAddress *addr,
+                                    Error **errp);
+
+/**
+ * qio_channel_socket_connect_async:
+ * @ioc: the socket channel object
+ * @addr: the address to connect to
+ * @callback: the function to invoke on completion
+ * @opaque: user data to pass to @callback
+ * @destroy: the function to free @opaque
+ *
+ * Attempt to connect to the address @addr. This method
+ * will run in the background so the caller will regain
+ * execution control immediately. The function @callback
+ * will be invoked on completion or failure. The @addr
+ * parameter will be copied, so may be freed as soon
+ * as this function returns without waiting for completion.
+ */
+void qio_channel_socket_connect_async(QIOChannelSocket *ioc,
+                                      SocketAddress *addr,
+                                      QIOTaskFunc callback,
+                                      gpointer opaque,
+                                      GDestroyNotify destroy);
+
+
+/**
+ * qio_channel_socket_listen_sync:
+ * @ioc: the socket channel object
+ * @addr: the address to listen to
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to listen to the address @addr. This method
+ * will run in the foreground so the caller will not regain
+ * execution control until the connection is established or
+ * an error occurs.
+ */
+int qio_channel_socket_listen_sync(QIOChannelSocket *ioc,
+                                   SocketAddress *addr,
+                                   Error **errp);
+
+/**
+ * qio_channel_socket_listen_async:
+ * @ioc: the socket channel object
+ * @addr: the address to listen to
+ * @callback: the function to invoke on completion
+ * @opaque: user data to pass to @callback
+ * @destroy: the function to free @opaque
+ *
+ * Attempt to listen to the address @addr. This method
+ * will run in the background so the caller will regain
+ * execution control immediately. The function @callback
+ * will be invoked on completion or failure. The @addr
+ * parameter will be copied, so may be freed as soon
+ * as this function returns without waiting for completion.
+ */
+void qio_channel_socket_listen_async(QIOChannelSocket *ioc,
+                                     SocketAddress *addr,
+                                     QIOTaskFunc callback,
+                                     gpointer opaque,
+                                     GDestroyNotify destroy);
+
+
+/**
+ * qio_channel_socket_dgram_sync:
+ * @ioc: the socket channel object
+ * @localAddr: the address to local bind address
+ * @remoteAddr: the address to remote peer address
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to initialize a datagram socket bound to
+ * @localAddr and communicating with peer @remoteAddr.
+ * This method will run in the foreground so the caller
+ * will not regain execution control until the socket
+ * is established or an error occurs.
+ */
+int qio_channel_socket_dgram_sync(QIOChannelSocket *ioc,
+                                  SocketAddress *localAddr,
+                                  SocketAddress *remoteAddr,
+                                  Error **errp);
+
+/**
+ * qio_channel_socket_dgram_async:
+ * @ioc: the socket channel object
+ * @localAddr: the address to local bind address
+ * @remoteAddr: the address to remote peer address
+ * @callback: the function to invoke on completion
+ * @opaque: user data to pass to @callback
+ * @destroy: the function to free @opaque
+ *
+ * Attempt to initialize a datagram socket bound to
+ * @localAddr and communicating with peer @remoteAddr.
+ * This method will run in the background so the caller
+ * will regain execution control immediately. The function
+ * @callback will be invoked on completion or failure.
+ * The @localAddr and @remoteAddr parameters will be copied,
+ * so may be freed as soon as this function returns without
+ * waiting for completion.
+ */
+void qio_channel_socket_dgram_async(QIOChannelSocket *ioc,
+                                    SocketAddress *localAddr,
+                                    SocketAddress *remoteAddr,
+                                    QIOTaskFunc callback,
+                                    gpointer opaque,
+                                    GDestroyNotify destroy);
+
+
+/**
+ * qio_channel_socket_get_local_address:
+ * @ioc: the socket channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+SocketAddress *
+qio_channel_socket_get_local_address(QIOChannelSocket *ioc,
+                                     Error **errp);
+
+/**
+ * qio_channel_socket_get_remote_address:
+ * @ioc: the socket channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *
+qio_channel_socket_get_remote_address(QIOChannelSocket *ioc,
+                                      Error **errp);
+
+
+/**
+ * qio_channel_socket_accept:
+ * @ioc: the socket channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * If the socket represents a server, then this accepts
+ * a new client connection. The returned channel will
+ * represent the connected client socket.
+ *
+ * Returns: the new client channel, or NULL on error
+ */
+QIOChannelSocket *
+qio_channel_socket_accept(QIOChannelSocket *ioc,
+                          Error **errp);
+
+
+#endif /* QIO_CHANNEL_SOCKET_H */
diff --git a/include/io/channel-tls.h b/include/io/channel-tls.h
new file mode 100644
index 0000000..d157eb1
--- /dev/null
+++ b/include/io/channel-tls.h
@@ -0,0 +1,142 @@
+/*
+ * QEMU I/O channels TLS driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_TLS_H
+#define QIO_CHANNEL_TLS_H
+
+#include "io/channel.h"
+#include "io/task.h"
+#include "crypto/tlssession.h"
+
+#define TYPE_QIO_CHANNEL_TLS "qio-channel-tls"
+#define QIO_CHANNEL_TLS(obj)                                     \
+    OBJECT_CHECK(QIOChannelTLS, (obj), TYPE_QIO_CHANNEL_TLS)
+
+typedef struct QIOChannelTLS QIOChannelTLS;
+
+/**
+ * QIOChannelTLS
+ *
+ * The QIOChannelTLS class provides a channel wrapper which
+ * can transparently run the TLS encryption protocol. It is
+ * usually used over a TCP socket, but there is actually no
+ * technical restriction on which type of master channel is
+ * used as the transport.
+ *
+ * This channel object is capable of running as either a
+ * TLS server or TLS client.
+ */
+
+struct QIOChannelTLS {
+    QIOChannel parent;
+    QIOChannel *master;
+    QCryptoTLSSession *session;
+};
+
+/**
+ * qio_channel_tls_new_server:
+ * @master: the underlying channel object
+ * @creds: the credentials to use for TLS handshake
+ * @aclname: the access control list for validating clients
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new TLS channel that runs the server side of
+ * a TLS session. The TLS session handshake will use the
+ * credentials provided in @creds. If the @aclname parameter
+ * is non-NULL, then the client will have to provide
+ * credentials (ie a x509 client certificate) which will
+ * then be validated against the ACL.
+ *
+ * After creating the channel, it is mandatory to call
+ * the qio_channel_tls_handshake() method before attempting
+ * todo any I/O on the channel.
+ *
+ * Once the handshake has completed, all I/O should be done
+ * via the new TLS channel object and not the original
+ * master channel
+ *
+ * Returns: the new TLS channel object, or NULL
+ */
+QIOChannelTLS *
+qio_channel_tls_new_server(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *aclname,
+                           Error **errp);
+
+/**
+ * qio_channel_tls_new_client:
+ * @master: the underlying channel object
+ * @creds: the credentials to use for TLS handshake
+ * @hostname: the user specified server hostname
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a new TLS channel that runs the client side of
+ * a TLS session. The TLS session handshake will use the
+ * credentials provided in @creds. The @hostname parameter
+ * should provide the user specified hostname of the server
+ * and will be validated against the server's credentials
+ * (ie CommonName of the x509 certificate)
+ *
+ * After creating the channel, it is mandatory to call
+ * the qio_channel_tls_handshake() method before attempting
+ * todo any I/O on the channel.
+ *
+ * Once the handshake has completed, all I/O should be done
+ * via the new TLS channel object and not the original
+ * master channel
+ *
+ * Returns: the new TLS channel object, or NULL
+ */
+QIOChannelTLS *
+qio_channel_tls_new_client(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *hostname,
+                           Error **errp);
+
+/**
+ * qio_channel_tls_handshake:
+ * @ioc: the TLS channel object
+ * @func: the callback to invoke when completed
+ * @opaque: opaque data to pass to @func
+ * @destroy: optional callback to free @opaque
+ *
+ * Perform the TLS session handshake. This method
+ * will return immediately and the handshake will
+ * continue in the background, provided the main
+ * loop is running. When the handshake is complete,
+ * or fails, the @func callback will be invoked.
+ */
+void qio_channel_tls_handshake(QIOChannelTLS *ioc,
+                               QIOTaskFunc func,
+                               gpointer opaque,
+                               GDestroyNotify destroy);
+
+/**
+ * qio_channel_tls_get_session:
+ * @ioc: the TLS channel object
+ *
+ * Get the TLS session used by the channel.
+ *
+ * Returns: the TLS session
+ */
+QCryptoTLSSession *
+qio_channel_tls_get_session(QIOChannelTLS *ioc);
+
+#endif /* QIO_CHANNEL_TLS_H */
diff --git a/include/io/channel-util.h b/include/io/channel-util.h
new file mode 100644
index 0000000..c0b79cf
--- /dev/null
+++ b/include/io/channel-util.h
@@ -0,0 +1,52 @@
+/*
+ * QEMU I/O channels utility APIs
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_UTIL_H
+#define QIO_CHANNEL_UTIL_H
+
+#include "io/channel.h"
+
+/*
+ * This module provides helper functions that are useful when dealing
+ * with QIOChannel objects
+ */
+
+
+/**
+ * qio_channel_new_fd:
+ * @fd: the file descriptor
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Create a channel for performing I/O on the file
+ * descriptor @fd. The particular subclass of QIOChannel
+ * that is returned will depend on what underlying object
+ * the file descriptor is associated with. It may be either
+ * a QIOChannelSocket or a QIOChannelFile instance. Upon
+ * success, the returned QIOChannel instance will own
+ * the @fd file descriptor, and take responsibility for
+ * closing it when no longer required. On failure, the
+ * caller is responsible for closing @fd.
+ *
+ * Returns: the channel object, or NULL on error
+ */
+QIOChannel *qio_channel_new_fd(int fd,
+                               Error **errp);
+
+#endif /* QIO_CHANNEL_UTIL_H */
diff --git a/include/io/channel-watch.h b/include/io/channel-watch.h
new file mode 100644
index 0000000..63bc4ae
--- /dev/null
+++ b/include/io/channel-watch.h
@@ -0,0 +1,90 @@
+/*
+ * QEMU I/O channels watch helper APIs
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_WATCH_H
+#define QIO_CHANNEL_WATCH_H
+
+#include "io/channel.h"
+
+/*
+ * This module provides helper functions that will be needed by
+ * the various QIOChannel implementations, for creating watches
+ * on file descriptors / sockets
+ */
+
+/**
+ * qio_channel_create_fd_watch:
+ * @ioc: the channel object
+ * @fd: the file descriptor
+ * @condition: the I/O condition
+ *
+ * Create a new main loop source that is able to
+ * monitor the file descriptor @fd for the
+ * I/O conditions in @condition. This is able
+ * monitor block devices, character devices,
+ * pipes but not plain files or, on Win32, sockets.
+ *
+ * Returns: the new main loop source
+ */
+GSource *qio_channel_create_fd_watch(QIOChannel *ioc,
+                                     int fd,
+                                     GIOCondition condition);
+
+/**
+ * qio_channel_create_socket_watch:
+ * @ioc: the channel object
+ * @fd: the file descriptor
+ * @condition: the I/O condition
+ *
+ * Create a new main loop source that is able to
+ * monitor the file descriptor @fd for the
+ * I/O conditions in @condition. This is equivalent
+ * to qio_channel_create_fd_watch on POSIX systems
+ * but not on Windows.
+ *
+ * Returns: the new main loop source
+ */
+GSource *qio_channel_create_socket_watch(QIOChannel *ioc,
+                                         int fd,
+                                         GIOCondition condition);
+
+/**
+ * qio_channel_create_fd_pair_watch:
+ * @ioc: the channel object
+ * @fdread: the file descriptor for reading
+ * @fdwrite: the file descriptor for writing
+ * @condition: the I/O condition
+ *
+ * Create a new main loop source that is able to
+ * monitor the pair of file descriptors @fdread
+ * and @fdwrite for the I/O conditions in @condition.
+ * This is intended for monitoring unidirectional
+ * file descriptors such as pipes, where a pair
+ * of descriptors is required for bidirectional
+ * I/O
+ *
+ * Returns: the new main loop source
+ */
+GSource *qio_channel_create_fd_pair_watch(QIOChannel *ioc,
+                                          int fdread,
+                                          int fdwrite,
+                                          GIOCondition condition);
+
+#endif /* QIO_CHANNEL_WATCH_H */
diff --git a/include/io/channel-websock.h b/include/io/channel-websock.h
new file mode 100644
index 0000000..3c9ff84
--- /dev/null
+++ b/include/io/channel-websock.h
@@ -0,0 +1,108 @@
+/*
+ * QEMU I/O channels driver websockets
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_WEBSOCK_H
+#define QIO_CHANNEL_WEBSOCK_H
+
+#include "io/channel.h"
+#include "qemu/buffer.h"
+#include "io/task.h"
+
+#define TYPE_QIO_CHANNEL_WEBSOCK "qio-channel-websock"
+#define QIO_CHANNEL_WEBSOCK(obj)                                     \
+    OBJECT_CHECK(QIOChannelWebsock, (obj), TYPE_QIO_CHANNEL_WEBSOCK)
+
+typedef struct QIOChannelWebsock QIOChannelWebsock;
+typedef union QIOChannelWebsockMask QIOChannelWebsockMask;
+
+union QIOChannelWebsockMask {
+    char c[4];
+    uint32_t u;
+};
+
+/**
+ * QIOChannelWebsock
+ *
+ * The QIOChannelWebsock class provides a channel wrapper which
+ * can transparently run the HTTP websockets protocol. This is
+ * usually used over a TCP socket, but there is actually no
+ * technical restriction on which type of master channel is
+ * used as the transport.
+ *
+ * This channel object is currently only capable of running as
+ * a websocket server and is a pretty crude implementation
+ * of it, not supporting the full websockets protocol feature
+ * set. It is sufficient to use with a simple websockets
+ * client for encapsulating VNC for noVNC in-browser client.
+ */
+
+struct QIOChannelWebsock {
+    QIOChannel parent;
+    QIOChannel *master;
+    Buffer encinput;
+    Buffer encoutput;
+    Buffer rawinput;
+    Buffer rawoutput;
+    size_t payload_remain;
+    QIOChannelWebsockMask mask;
+    guint io_tag;
+    Error *io_err;
+    gboolean io_eof;
+};
+
+/**
+ * qio_channel_websock_new_server:
+ * @master: the underlying channel object
+ *
+ * Create a new websockets channel that runs the server
+ * side of the protocol.
+ *
+ * After creating the channel, it is mandatory to call
+ * the qio_channel_websock_handshake() method before attempting
+ * todo any I/O on the channel.
+ *
+ * Once the handshake has completed, all I/O should be done
+ * via the new websocket channel object and not the original
+ * master channel
+ *
+ * Returns: the new websockets channel object
+ */
+QIOChannelWebsock *
+qio_channel_websock_new_server(QIOChannel *master);
+
+/**
+ * qio_channel_websock_handshake:
+ * @ioc: the websocket channel object
+ * @func: the callback to invoke when completed
+ * @opaque: opaque data to pass to @func
+ * @destroy: optional callback to free @opaque
+ *
+ * Perform the websocket handshake. This method
+ * will return immediately and the handshake will
+ * continue in the background, provided the main
+ * loop is running. When the handshake is complete,
+ * or fails, the @func callback will be invoked.
+ */
+void qio_channel_websock_handshake(QIOChannelWebsock *ioc,
+                                   QIOTaskFunc func,
+                                   gpointer opaque,
+                                   GDestroyNotify destroy);
+
+#endif /* QIO_CHANNEL_WEBSOCK_H */
diff --git a/include/io/channel.h b/include/io/channel.h
new file mode 100644
index 0000000..3cff64d
--- /dev/null
+++ b/include/io/channel.h
@@ -0,0 +1,506 @@
+/*
+ * QEMU I/O channels
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_CHANNEL_H
+#define QIO_CHANNEL_H
+
+#include "qemu-common.h"
+#include "qom/object.h"
+
+#define TYPE_QIO_CHANNEL "qio-channel"
+#define QIO_CHANNEL(obj)                                    \
+    OBJECT_CHECK(QIOChannel, (obj), TYPE_QIO_CHANNEL)
+#define QIO_CHANNEL_CLASS(klass)                                    \
+    OBJECT_CLASS_CHECK(QIOChannelClass, klass, TYPE_QIO_CHANNEL)
+#define QIO_CHANNEL_GET_CLASS(obj)                                  \
+    OBJECT_GET_CLASS(QIOChannelClass, obj, TYPE_QIO_CHANNEL)
+
+typedef struct QIOChannel QIOChannel;
+typedef struct QIOChannelClass QIOChannelClass;
+
+#define QIO_CHANNEL_ERR_BLOCK -2
+
+#ifndef __cplusplus
+typedef enum QIOChannelFeature QIOChannelFeature;
+#endif
+enum QIOChannelFeature {
+    QIO_CHANNEL_FEATURE_FD_PASS  = (1 << 0),
+    QIO_CHANNEL_FEATURE_SHUTDOWN = (1 << 1),
+    QIO_CHANNEL_FEATURE_LISTEN   = (1 << 2),
+};
+
+#ifndef __cplusplus
+typedef enum QIOChannelShutdown QIOChannelShutdown;
+#endif
+enum QIOChannelShutdown {
+    QIO_CHANNEL_SHUTDOWN_BOTH,
+    QIO_CHANNEL_SHUTDOWN_READ,
+    QIO_CHANNEL_SHUTDOWN_WRITE,
+};
+
+typedef gboolean (*QIOChannelFunc)(QIOChannel *ioc,
+                                   GIOCondition condition,
+                                   gpointer data);
+
+/**
+ * QIOChannel:
+ *
+ * The QIOChannel defines the core API for a generic I/O channel
+ * class hierarchy. It is inspired by GIOChannel, but has the
+ * following differences
+ *
+ *  - Use QOM to properly support arbitrary subclassing
+ *  - Support use of iovecs for efficient I/O with multiple blocks
+ *  - None of the character set translation, binary data exclusively
+ *  - Direct support for QEMU Error object reporting
+ *  - File descriptor passing
+ *
+ * This base class is abstract so cannot be instantiated. There
+ * will be subclasses for dealing with sockets, files, and higher
+ * level protocols such as TLS, WebSocket, etc.
+ */
+
+struct QIOChannel {
+    Object parent;
+    unsigned int features; /* bitmask of QIOChannelFeatures */
+#ifdef _WIN32
+    HANDLE event; /* For use with GSource on Win32 */
+#endif
+};
+
+/**
+ * QIOChannelClass:
+ *
+ * This class defines the contract that all subclasses
+ * must follow to provide specific channel implementations.
+ * The first five callbacks are mandatory to support, others
+ * provide additional optional features.
+ *
+ * Consult the corresponding public API docs for a description
+ * of the semantics of each callback
+ */
+struct QIOChannelClass {
+    ObjectClass parent;
+
+    /* Mandatory callbacks */
+    ssize_t (*io_writev)(QIOChannel *ioc,
+                         const struct iovec *iov,
+                         size_t niov,
+                         int *fds,
+                         size_t nfds,
+                         Error **errp);
+    ssize_t (*io_readv)(QIOChannel *ioc,
+                        const struct iovec *iov,
+                        size_t niov,
+                        int **fds,
+                        size_t *nfds,
+                        Error **errp);
+    int (*io_close)(QIOChannel *ioc,
+                    Error **errp);
+    GSource * (*io_create_watch)(QIOChannel *ioc,
+                                 GIOCondition condition);
+    int (*io_set_blocking)(QIOChannel *ioc,
+                           bool enabled,
+                           Error **errp);
+
+    /* Optional callbacks */
+    int (*io_shutdown)(QIOChannel *ioc,
+                       QIOChannelShutdown how,
+                       Error **errp);
+    void (*io_set_cork)(QIOChannel *ioc,
+                        bool enabled);
+    void (*io_set_delay)(QIOChannel *ioc,
+                         bool enabled);
+    off_t (*io_seek)(QIOChannel *ioc,
+                     off_t offset,
+                     int whence,
+                     Error **errp);
+};
+
+/* General I/O handling functions */
+
+/**
+ * qio_channel_has_feature:
+ * @ioc: the channel object
+ * @feature: the feature to check support of
+ *
+ * Determine whether the channel implementation supports
+ * the optional feature named in @feature.
+ *
+ * Returns: true if supported, false otherwise.
+ */
+bool qio_channel_has_feature(QIOChannel *ioc,
+                             QIOChannelFeature feature);
+
+/**
+ * qio_channel_readv_full:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to read data into
+ * @niov: the length of the @iov array
+ * @fds: pointer to an array that will received file handles
+ * @nfds: pointer filled with number of elements in @fds on return
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Read data from the IO channel, storing it in the
+ * memory regions referenced by @iov. Each element
+ * in the @iov will be fully populated with data
+ * before the next one is used. The @niov parameter
+ * specifies the total number of elements in @iov.
+ *
+ * It is not required for all @iov to be filled with
+ * data. If the channel is in blocking mode, at least
+ * one byte of data will be read, but no more is
+ * guaranteed. If the channel is non-blocking and no
+ * data is available, it will return QIO_CHANNEL_ERR_BLOCK
+ *
+ * If the channel has passed any file descriptors,
+ * the @fds array pointer will be allocated and
+ * the elements filled with the received file
+ * descriptors. The @nfds pointer will be updated
+ * to indicate the size of the @fds array that
+ * was allocated. It is the callers responsibility
+ * to call close() on each file descriptor and to
+ * call g_free() on the array pointer in @fds.
+ *
+ * It is an error to pass a non-NULL @fds parameter
+ * unless qio_channel_has_feature() returns a true
+ * value for the QIO_CHANNEL_FEATURE_FD_PASS constant.
+ *
+ * Returns: the number of bytes read, or -1 on error,
+ * or QIO_CHANNEL_ERR_BLOCK if no data is available
+ * and the channel is non-blocking
+ */
+ssize_t qio_channel_readv_full(QIOChannel *ioc,
+                               const struct iovec *iov,
+                               size_t niov,
+                               int **fds,
+                               size_t *nfds,
+                               Error **errp);
+
+
+/**
+ * qio_channel_writev_full:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to write data from
+ * @niov: the length of the @iov array
+ * @fds: an array of file handles to send
+ * @nfds: number of file handles in @fds
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Write data to the IO channel, reading it from the
+ * memory regions referenced by @iov. Each element
+ * in the @iov will be fully sent, before the next
+ * one is used. The @niov parameter specifies the
+ * total number of elements in @iov.
+ *
+ * It is not required for all @iov data to be fully
+ * sent. If the channel is in blocking mode, at least
+ * one byte of data will be sent, but no more is
+ * guaranteed. If the channel is non-blocking and no
+ * data can be sent, it will return QIO_CHANNEL_ERR_BLOCK
+ *
+ * If there are file descriptors to send, the @fds
+ * array should be non-NULL and provide the handles.
+ * All file descriptors will be sent if at least one
+ * byte of data was sent.
+ *
+ * It is an error to pass a non-NULL @fds parameter
+ * unless qio_channel_has_feature() returns a true
+ * value for the QIO_CHANNEL_FEATURE_FD_PASS constant.
+ *
+ * Returns: the number of bytes sent, or -1 on error,
+ * or QIO_CHANNEL_ERR_BLOCK if no data is can be sent
+ * and the channel is non-blocking
+ */
+ssize_t qio_channel_writev_full(QIOChannel *ioc,
+                                const struct iovec *iov,
+                                size_t niov,
+                                int *fds,
+                                size_t nfds,
+                                Error **errp);
+
+/**
+ * qio_channel_readv:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to read data into
+ * @niov: the length of the @iov array
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_readv_full() but does not support
+ * receiving of file handles.
+ */
+ssize_t qio_channel_readv(QIOChannel *ioc,
+                          const struct iovec *iov,
+                          size_t niov,
+                          Error **errp);
+
+/**
+ * qio_channel_writev:
+ * @ioc: the channel object
+ * @iov: the array of memory regions to write data from
+ * @niov: the length of the @iov array
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_writev_full() but does not support
+ * sending of file handles.
+ */
+ssize_t qio_channel_writev(QIOChannel *ioc,
+                           const struct iovec *iov,
+                           size_t niov,
+                           Error **errp);
+
+/**
+ * qio_channel_readv:
+ * @ioc: the channel object
+ * @buf: the memory region to read data into
+ * @buflen: the length of @buf
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_readv_full() but does not support
+ * receiving of file handles, and only supports reading into
+ * a single memory region.
+ */
+ssize_t qio_channel_read(QIOChannel *ioc,
+                         char *buf,
+                         size_t buflen,
+                         Error **errp);
+
+/**
+ * qio_channel_writev:
+ * @ioc: the channel object
+ * @buf: the memory regions to send data from
+ * @buflen: the length of @buf
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Behaves as qio_channel_writev_full() but does not support
+ * sending of file handles, and only supports writing from a
+ * single memory region.
+ */
+ssize_t qio_channel_write(QIOChannel *ioc,
+                          const char *buf,
+                          size_t buflen,
+                          Error **errp);
+
+/**
+ * qio_channel_set_blocking:
+ * @ioc: the channel object
+ * @enabled: the blocking flag state
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * If @enabled is true, then the channel is put into
+ * blocking mode, otherwise it will be non-blocking.
+ *
+ * In non-blocking mode, read/write operations may
+ * return QIO_CHANNEL_ERR_BLOCK if they would otherwise
+ * block on I/O
+ */
+int qio_channel_set_blocking(QIOChannel *ioc,
+                             bool enabled,
+                             Error **errp);
+
+/**
+ * qio_channel_close:
+ * @ioc: the channel object
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Close the channel, flushing any pending I/O
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qio_channel_close(QIOChannel *ioc,
+                      Error **errp);
+
+/**
+ * qio_channel_shutdown:
+ * @ioc: the channel object
+ * @how: the direction to shutdown
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Shutdowns transmission and/or receiving of data
+ * without closing the underlying transport.
+ *
+ * Not all implementations will support this facility,
+ * so may report an error. To avoid errors, the
+ * caller may check for the feature flag
+ * QIO_CHANNEL_FEATURE_SHUTDOWN prior to calling
+ * this method.
+ *
+ * Returns: 0 on success, -1 on error
+ */
+int qio_channel_shutdown(QIOChannel *ioc,
+                         QIOChannelShutdown how,
+                         Error **errp);
+
+/**
+ * qio_channel_set_delay:
+ * @ioc: the channel object
+ * @enabled: the new flag state
+ *
+ * Controls whether the underlying transport is
+ * permitted to delay writes in order to merge
+ * small packets. If @enabled is true, then the
+ * writes may be delayed in order to opportunistically
+ * merge small packets into larger ones. If @enabled
+ * is false, writes are dispatched immediately with
+ * no delay.
+ *
+ * When @enabled is false, applications may wish to
+ * use the qio_channel_set_cork() method to explicitly
+ * control write merging.
+ *
+ * On channels which are backed by a socket, this
+ * API corresponds to the inverse of TCP_NODELAY flag,
+ * controlling whether the Nagle algorithm is active.
+ *
+ * This setting is merely a hint, so implementations are
+ * free to ignore this without it being considered an
+ * error.
+ */
+void qio_channel_set_delay(QIOChannel *ioc,
+                           bool enabled);
+
+/**
+ * qio_channel_set_cork:
+ * @ioc: the channel object
+ * @enabled: the new flag state
+ *
+ * Controls whether the underlying transport is
+ * permitted to dispatch data that is written.
+ * If @enabled is true, then any data written will
+ * be queued in local buffers until @enabled is
+ * set to false once again.
+ *
+ * This feature is typically used when the automatic
+ * write coalescing facility is disabled via the
+ * qio_channel_set_delay() method.
+ *
+ * On channels which are backed by a socket, this
+ * API corresponds to the TCP_CORK flag.
+ *
+ * This setting is merely a hint, so implementations are
+ * free to ignore this without it being considered an
+ * error.
+ */
+void qio_channel_set_cork(QIOChannel *ioc,
+                          bool enabled);
+
+
+/**
+ * qio_channel_seek:
+ * @ioc: the channel object
+ * @offset: the position to seek to, relative to @whence
+ * @whence: one of the (POSIX) SEEK_* constants listed below
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Moves the current I/O position within the channel
+ * @ioc, to be @offset. The value of @offset is
+ * interpreted relative to @whence:
+ *
+ * SEEK_SET - the position is set to @offset bytes
+ * SEEK_CUR - the position is moved by @offset bytes
+ * SEEK_END - the position is set to end of the file plus @offset bytes
+ *
+ * Not all implementations will support this facility,
+ * so may report an error.
+ *
+ * Returns: the new position on success, (off_t)-1 on failure
+ */
+off_t qio_channel_io_seek(QIOChannel *ioc,
+                          off_t offset,
+                          int whence,
+                          Error **errp);
+
+
+/**
+ * qio_channel_create_watch:
+ * @ioc: the channel object
+ * @condition: the I/O condition to monitor
+ *
+ * Create a new main loop source that is used to watch
+ * for the I/O condition @condition. Typically the
+ * qio_channel_add_watch() method would be used instead
+ * of this, since it directly attaches a callback to
+ * the source
+ *
+ * Returns: the new main loop source.
+ */
+GSource *qio_channel_create_watch(QIOChannel *ioc,
+                                  GIOCondition condition);
+
+/**
+ * qio_channel_add_watch:
+ * @ioc: the channel object
+ * @condition: the I/O condition to monitor
+ * @func: callback to invoke when the source becomes ready
+ * @user_data: opaque data to pass to @func
+ * @notify: callback to free @user_data
+ *
+ * Create a new main loop source that is used to watch
+ * for the I/O condition @condition. The callback @func
+ * will be registered against the source, to be invoked
+ * when the source becomes ready. The optional @user_data
+ * will be passed to @func when it is invoked. The @notify
+ * callback will be used to free @user_data when the
+ * watch is deleted
+ *
+ * The returned source ID can be used with g_source_remove()
+ * to remove and free the source when no longer required.
+ * Alternatively the @func callback can return a FALSE
+ * value.
+ *
+ * Returns: the source ID
+ */
+guint qio_channel_add_watch(QIOChannel *ioc,
+                            GIOCondition condition,
+                            QIOChannelFunc func,
+                            gpointer user_data,
+                            GDestroyNotify notify);
+
+
+/**
+ * qio_channel_yield:
+ * @ioc: the channel object
+ * @condition: the I/O condition to wait for
+ *
+ * Yields execution from the current coroutine until
+ * the condition indicated by @condition becomes
+ * available.
+ *
+ * This must only be called from coroutine context
+ */
+void qio_channel_yield(QIOChannel *ioc,
+                       GIOCondition condition);
+
+/**
+ * qio_channel_wait:
+ * @ioc: the channel object
+ * @condition: the I/O condition to wait for
+ *
+ * Block execution from the current thread until
+ * the condition indicated by @condition becomes
+ * available.
+ *
+ * This will enter a nested event loop to perform
+ * the wait.
+ */
+void qio_channel_wait(QIOChannel *ioc,
+                      GIOCondition condition);
+
+#endif /* QIO_CHANNEL_H */
diff --git a/include/io/task.h b/include/io/task.h
new file mode 100644
index 0000000..42028cb
--- /dev/null
+++ b/include/io/task.h
@@ -0,0 +1,255 @@
+/*
+ * QEMU I/O task
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QIO_TASK_H
+#define QIO_TASK_H
+
+#include "qemu-common.h"
+#include "qom/object.h"
+
+typedef struct QIOTask QIOTask;
+
+typedef void (*QIOTaskFunc)(Object *source,
+                            Error *err,
+                            gpointer opaque);
+
+typedef int (*QIOTaskWorker)(QIOTask *task,
+                             Error **errp,
+                             gpointer opaque);
+
+/**
+ * QIOTask:
+ *
+ * The QIOTask object provides a simple mechanism for reporting
+ * success / failure of long running background operations.
+ *
+ * A object on which the operation is to be performed could have
+ * a public API which accepts a task callback:
+ *
+ * <example>
+ *   <title>Task callback function signature</title>
+ *   <programlisting>
+ *  void myobject_operation(QMyObject *obj,
+ *                          QIOTaskFunc *func,
+ *                          gpointer opaque,
+ *                          GDestroyNotify *notify);
+ *   </programlisting>
+ * </example>
+ *
+ * The 'func' parameter is the callback to be invoked, and 'opaque'
+ * is data to pass to it. The optional 'notify' function is used
+ * to free 'opaque' when no longer needed.
+ *
+ * Now, lets say the implementation of this method wants to set
+ * a timer to run once a second checking for completion of some
+ * activity. It would do something like
+ *
+ * <example>
+ *   <title>Task callback function implementation</title>
+ *   <programlisting>
+ *    void myobject_operation(QMyObject *obj,
+ *                            QIOTaskFunc *func,
+ *                            gpointer opaque,
+ *                            GDestroyNotify *notify)
+ *    {
+ *      QIOTask *task;
+ *
+ *      task = qio_task_new(OBJECT(obj), func, opaque, notify);
+ *
+ *      g_timeout_add_full(G_PRIORITY_DEFAULT,
+ *                         1000,
+ *                         myobject_operation_timer,
+ *                         task,
+ *                         NULL);
+ *    }
+ *   </programlisting>
+ * </example>
+ *
+ * It could equally have setup a watch on a file descriptor or
+ * created a background thread, or something else entirely.
+ * Notice that the source object is passed to the task, and
+ * QIOTask will hold a reference on that. This ensure that
+ * the QMyObject instance cannot be garbage collected while
+ * the async task is still in progress.
+ *
+ * In this case, myobject_operation_timer will fire after
+ * 3 secs and do
+ *
+ * <example>
+ *   <title>Task timer function</title>
+ *   <programlisting>
+ *   gboolean myobject_operation_timer(gpointer opaque)
+ *   {
+ *      QIOTask *task = QIO_TASK(opaque);
+ *      Error *err;*
+ *
+ *      ...check something important...
+ *       if (err) {
+ *           qio_task_abort(task, err);
+ *           error_free(task);
+ *           return FALSE;
+ *       } else if (...work is completed ...) {
+ *           qio_task_complete(task);
+ *           return FALSE;
+ *       }
+ *       ...carry on polling ...
+ *       return TRUE;
+ *   }
+ *   </programlisting>
+ * </example>
+ *
+ * Once this function returns false, object_unref will be called
+ * automatically on the task causing it to be released and the
+ * ref on QMyObject dropped too.
+ *
+ * The QIOTask module can also be used to perform operations
+ * in a background thread context, while still reporting the
+ * results in the main event thread. This allows code which
+ * cannot easily be rewritten to be asychronous (such as DNS
+ * lookups) to be easily run non-blocking. Reporting the
+ * results in the main thread context means that the caller
+ * typically does not need to be concerned about thread
+ * safety wrt the QEMU global mutex.
+ *
+ * For example, the socket_listen() method will block the caller
+ * while DNS lookups take place if given a name, instead of IP
+ * address. The C library often do not provide a practical async
+ * DNS API, so the to get non-blocking DNS lookups in a portable
+ * manner requires use of a thread. So achieve a non-blocking
+ * socket listen using QIOTask would require:
+ *
+ * <example>
+ *    static int myobject_listen_worker(QIOTask *task,
+ *                                      Error **errp,
+ *                                      gpointer opaque)
+ *    {
+ *       QMyObject obj = QMY_OBJECT(qio_task_get_source(task));
+ *       SocketAddress *addr = opaque;
+ *
+ *       obj->fd = socket_listen(addr, errp);
+ *       if (obj->fd < 0) {
+ *          return -1;
+ *       }
+ *       return 0;
+ *    }
+ *
+ *    void myobject_listen_async(QMyObject *obj,
+ *                               SocketAddress *addr,
+ *                               QIOTaskFunc *func,
+ *                               gpointer opaque,
+ *                               GDestroyNotify *notify)
+ *    {
+ *      QIOTask *task;
+ *      SocketAddress *addrCopy;
+ *
+ *      addrCopy = QAPI_CLONE(SocketAddress, addr);
+ *      task = qio_task_new(OBJECT(obj), func, opaque, notify);
+ *
+ *      qio_task_run_in_thread(task, myobject_listen_worker,
+ *                             addrCopy,
+ *                             qapi_free_SocketAddress);
+ *    }
+ * </example>
+ *
+ * NB, The 'func' callback passed into myobject_listen_async
+ * will be invoked from the main event thread, despite the
+ * actual operation being performed in a different thread.
+ */
+
+/**
+ * qio_task_new:
+ * @source: the object on which the operation is invoked
+ * @func: the callback to invoke when the task completes
+ * @opaque: opaque data to pass to @func when invoked
+ * @destroy: optional callback to free @opaque
+ *
+ * Creates a new task struct to track completion of a
+ * background operation running on the object @source.
+ * When the operation completes or fails, the callback
+ * @func will be invoked. The callback can access the
+ * 'err' attribute in the task object to determine if
+ * the operation was successful or not.
+ *
+ * The returned task will be released when one of
+ * qio_task_abort() or qio_task_complete() are invoked.
+ *
+ * Returns: the task struct
+ */
+QIOTask *qio_task_new(Object *source,
+                      QIOTaskFunc func,
+                      gpointer opaque,
+                      GDestroyNotify destroy);
+
+/**
+ * qio_task_run_in_thread:
+ * @task: the task struct
+ * @worker: the function to invoke in a thread
+ * @opaque: opaque data to pass to @worker
+ * @destroy: function to free @opaque
+ *
+ * Run a task in a background thread. If @worker
+ * returns 0 it will call qio_task_complete() in
+ * the main event thread context. If @worker
+ * returns -1 it will call qio_task_abort() in
+ * the main event thread context.
+ */
+void qio_task_run_in_thread(QIOTask *task,
+                            QIOTaskWorker worker,
+                            gpointer opaque,
+                            GDestroyNotify destroy);
+
+/**
+ * qio_task_complete:
+ * @task: the task struct
+ *
+ * Mark the operation as successfully completed
+ * and free the memory for @task.
+ */
+void qio_task_complete(QIOTask *task);
+
+/**
+ * qio_task_abort:
+ * @task: the task struct
+ * @err: the error to record for the operation
+ *
+ * Mark the operation as failed, with @err providing
+ * details about the failure. The @err may be freed
+ * afer the function returns, as the notification
+ * callback is invoked synchronously. The @task will
+ * be freed when this call completes.
+ */
+void qio_task_abort(QIOTask *task,
+                    Error *err);
+
+
+/**
+ * qio_task_get_source:
+ * @task: the task struct
+ *
+ * Get the source object associated with the background
+ * task. This returns a new reference to the object,
+ * which the caller must released with object_unref()
+ * when no longer required.
+ *
+ * Returns: the source object
+ */
+Object *qio_task_get_source(QIOTask *task);
+
+#endif /* QIO_TASK_H */
diff --git a/include/libdecnumber/dconfig.h b/include/libdecnumber/dconfig.h
index 2f0455a..0f7dcce 100644
--- a/include/libdecnumber/dconfig.h
+++ b/include/libdecnumber/dconfig.h
@@ -27,7 +27,6 @@
    Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.  */
 
-#include "config-host.h"
 
 #if defined(HOST_WORDS_BIGENDIAN)
 #define WORDS_BIGENDIAN 1
diff --git a/include/libdecnumber/decContext.h b/include/libdecnumber/decContext.h
index c3e46f4..cea6e42 100644
--- a/include/libdecnumber/decContext.h
+++ b/include/libdecnumber/decContext.h
@@ -50,15 +50,13 @@
 /*								      */
 /* ------------------------------------------------------------------ */
 
-#if !defined(DECCONTEXT)
-  #define DECCONTEXT
+#ifndef DECCONTEXT_H
+#define DECCONTEXT_H
+
   #define DECCNAME     "decContext"			/* Short name */
   #define DECCFULLNAME "Decimal Context Descriptor"   /* Verbose name */
   #define DECCAUTHOR   "Mike Cowlishaw"		      /* Who to blame */
 
-  #include <stdint.h>
-  #include <stdio.h>		   /* for printf, etc.		      */
-  #include <signal.h>		   /* for traps			      */
 
   /* Extended flags setting -- set this to 0 to use only IEEE flags   */
   #define DECEXTFLAG 1		   /* 1=enable extended flags	      */
diff --git a/include/libdecnumber/decNumber.h b/include/libdecnumber/decNumber.h
index 9fa4e6a..aa115fe 100644
--- a/include/libdecnumber/decNumber.h
+++ b/include/libdecnumber/decNumber.h
@@ -32,15 +32,14 @@
 /* Decimal Number arithmetic module header			      */
 /* ------------------------------------------------------------------ */
 
-#if !defined(DECNUMBER)
-  #define DECNUMBER
+#ifndef DECNUMBER_H
+#define DECNUMBER_H
+
   #define DECNAME     "decNumber"			/* Short name */
   #define DECFULLNAME "Decimal Number Module"	      /* Verbose name */
   #define DECAUTHOR   "Mike Cowlishaw"		      /* Who to blame */
 
-  #if !defined(DECCONTEXT)
-    #include "libdecnumber/decContext.h"
-  #endif
+  #include "libdecnumber/decContext.h"
 
   /* Bit settings for decNumber.bits				      */
   #define DECNEG    0x80      /* Sign; 1=negative, 0=positive or zero */
diff --git a/include/libdecnumber/decNumberLocal.h b/include/libdecnumber/decNumberLocal.h
index 71ed77b..12cf1d8 100644
--- a/include/libdecnumber/decNumberLocal.h
+++ b/include/libdecnumber/decNumberLocal.h
@@ -37,13 +37,12 @@
 /* decNumber.h or one of decDouble (etc.) must be included first.     */
 /* ------------------------------------------------------------------ */
 
-#if !defined(DECNUMBERLOC)
-  #define DECNUMBERLOC
+#ifndef DECNUMBERLOCAL_H
+#define DECNUMBERLOCAL_H
+
   #define DECVERSION	"decNumber 3.53" /* Package Version [16 max.] */
   #define DECNLAUTHOR	"Mike Cowlishaw"	      /* Who to blame */
 
-  #include <stdlib.h>	      /* for abs			      */
-  #include <string.h>	      /* for memset, strcpy		      */
   #include "libdecnumber/dconfig.h"
 
   /* Conditional code flag -- set this to match hardware platform     */
@@ -660,6 +659,4 @@
   /* [end of format-dependent macros and constants]		      */
   #endif
 
-#else
-  #error decNumberLocal included more than once
 #endif
diff --git a/include/libdecnumber/dpd/decimal128.h b/include/libdecnumber/dpd/decimal128.h
index 7d9ee24..aff261e 100644
--- a/include/libdecnumber/dpd/decimal128.h
+++ b/include/libdecnumber/dpd/decimal128.h
@@ -32,8 +32,9 @@
 /* Decimal 128-bit format module header				      */
 /* ------------------------------------------------------------------ */
 
-#if !defined(DECIMAL128)
-  #define DECIMAL128
+#ifndef DECIMAL128_H
+#define DECIMAL128_H
+
   #define DEC128NAME	 "decimal128"		      /* Short name   */
   #define DEC128FULLNAME "Decimal 128-bit Number"     /* Verbose name */
   #define DEC128AUTHOR	 "Mike Cowlishaw"	      /* Who to blame */
@@ -59,9 +60,7 @@
   #ifndef DECNUMDIGITS
     #define DECNUMDIGITS DECIMAL128_Pmax /* size if not already defined*/
   #endif
-  #ifndef DECNUMBER
-    #include "libdecnumber/decNumber.h"
-  #endif
+  #include "libdecnumber/decNumber.h"
 
   /* Decimal 128-bit type, accessible by bytes			      */
   typedef struct {
diff --git a/include/libdecnumber/dpd/decimal32.h b/include/libdecnumber/dpd/decimal32.h
index de313e0..6cb9e43 100644
--- a/include/libdecnumber/dpd/decimal32.h
+++ b/include/libdecnumber/dpd/decimal32.h
@@ -32,8 +32,9 @@
 /* Decimal 32-bit format module header				      */
 /* ------------------------------------------------------------------ */
 
-#if !defined(DECIMAL32)
-  #define DECIMAL32
+#ifndef DECIMAL32_H
+#define DECIMAL32_H
+
   #define DEC32NAME	"decimal32"		      /* Short name   */
   #define DEC32FULLNAME "Decimal 32-bit Number"	      /* Verbose name */
   #define DEC32AUTHOR	"Mike Cowlishaw"	      /* Who to blame */
@@ -59,9 +60,7 @@
   #ifndef DECNUMDIGITS
     #define DECNUMDIGITS DECIMAL32_Pmax /* size if not already defined*/
   #endif
-  #ifndef DECNUMBER
-    #include "libdecnumber/decNumber.h"
-  #endif
+  #include "libdecnumber/decNumber.h"
 
   /* Decimal 32-bit type, accessible by bytes */
   typedef struct {
diff --git a/include/libdecnumber/dpd/decimal64.h b/include/libdecnumber/dpd/decimal64.h
index 2f6c049..f29e570 100644
--- a/include/libdecnumber/dpd/decimal64.h
+++ b/include/libdecnumber/dpd/decimal64.h
@@ -32,8 +32,9 @@
 /* Decimal 64-bit format module header				      */
 /* ------------------------------------------------------------------ */
 
-#if !defined(DECIMAL64)
-  #define DECIMAL64
+#ifndef DECIMAL64_H
+#define DECIMAL64_H
+
   #define DEC64NAME	"decimal64"		      /* Short name   */
   #define DEC64FULLNAME "Decimal 64-bit Number"	      /* Verbose name */
   #define DEC64AUTHOR	"Mike Cowlishaw"	      /* Who to blame */
@@ -61,9 +62,7 @@
   #ifndef DECNUMDIGITS
     #define DECNUMDIGITS DECIMAL64_Pmax /* size if not already defined*/
   #endif
-  #ifndef DECNUMBER
-    #include "libdecnumber/decNumber.h"
-  #endif
+  #include "libdecnumber/decNumber.h"
 
   /* Decimal 64-bit type, accessible by bytes			      */
   typedef struct {
diff --git a/include/migration/block.h b/include/migration/block.h
index ffa8ac0..41a1ac8 100644
--- a/include/migration/block.h
+++ b/include/migration/block.h
@@ -11,8 +11,8 @@
  *
  */
 
-#ifndef BLOCK_MIGRATION_H
-#define BLOCK_MIGRATION_H
+#ifndef MIGRATION_BLOCK_H
+#define MIGRATION_BLOCK_H
 
 void blk_mig_init(void);
 int blk_mig_active(void);
@@ -20,4 +20,4 @@
 uint64_t blk_mig_bytes_remaining(void);
 uint64_t blk_mig_bytes_total(void);
 
-#endif /* BLOCK_MIGRATION_H */
+#endif /* MIGRATION_BLOCK_H */
diff --git a/include/migration/cpu.h b/include/migration/cpu.h
new file mode 100644
index 0000000..f3abbab
--- /dev/null
+++ b/include/migration/cpu.h
@@ -0,0 +1,48 @@
+/* Declarations for use for CPU state serialization.  */
+#ifndef MIGRATION_CPU_H
+#define MIGRATION_CPU_H
+
+#if TARGET_LONG_BITS == 64
+#define qemu_put_betl qemu_put_be64
+#define qemu_get_betl qemu_get_be64
+#define qemu_put_betls qemu_put_be64s
+#define qemu_get_betls qemu_get_be64s
+#define qemu_put_sbetl qemu_put_sbe64
+#define qemu_get_sbetl qemu_get_sbe64
+#define qemu_put_sbetls qemu_put_sbe64s
+#define qemu_get_sbetls qemu_get_sbe64s
+
+#define VMSTATE_UINTTL_V(_f, _s, _v)                                  \
+    VMSTATE_UINT64_V(_f, _s, _v)
+#define VMSTATE_UINTTL_EQUAL_V(_f, _s, _v)                            \
+    VMSTATE_UINT64_EQUAL_V(_f, _s, _v)
+#define VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, _v)                        \
+    VMSTATE_UINT64_ARRAY_V(_f, _s, _n, _v)
+#define vmstate_info_uinttl vmstate_info_uint64
+#else
+#define qemu_put_betl qemu_put_be32
+#define qemu_get_betl qemu_get_be32
+#define qemu_put_betls qemu_put_be32s
+#define qemu_get_betls qemu_get_be32s
+#define qemu_put_sbetl qemu_put_sbe32
+#define qemu_get_sbetl qemu_get_sbe32
+#define qemu_put_sbetls qemu_put_sbe32s
+#define qemu_get_sbetls qemu_get_sbe32s
+
+#define VMSTATE_UINTTL_V(_f, _s, _v)                                  \
+    VMSTATE_UINT32_V(_f, _s, _v)
+#define VMSTATE_UINTTL_EQUAL_V(_f, _s, _v)                            \
+    VMSTATE_UINT32_EQUAL_V(_f, _s, _v)
+#define VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, _v)                        \
+    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, _v)
+#define vmstate_info_uinttl vmstate_info_uint32
+#endif
+
+#define VMSTATE_UINTTL(_f, _s)                                        \
+    VMSTATE_UINTTL_V(_f, _s, 0)
+#define VMSTATE_UINTTL_EQUAL(_f, _s)                                  \
+    VMSTATE_UINTTL_EQUAL_V(_f, _s, 0)
+#define VMSTATE_UINTTL_ARRAY(_f, _s, _n)                              \
+    VMSTATE_UINTTL_ARRAY_V(_f, _s, _n, 0)
+
+#endif
diff --git a/include/migration/migration.h b/include/migration/migration.h
index 3cb5ba8..3c96623 100644
--- a/include/migration/migration.h
+++ b/include/migration/migration.h
@@ -18,7 +18,6 @@
 #include "qemu-common.h"
 #include "qemu/thread.h"
 #include "qemu/notify.h"
-#include "qapi/error.h"
 #include "migration/vmstate.h"
 #include "qapi-types.h"
 #include "exec/cpu-common.h"
@@ -33,13 +32,100 @@
 #define QEMU_VM_SECTION_END          0x03
 #define QEMU_VM_SECTION_FULL         0x04
 #define QEMU_VM_SUBSECTION           0x05
+#define QEMU_VM_VMDESCRIPTION        0x06
+#define QEMU_VM_CONFIGURATION        0x07
+#define QEMU_VM_COMMAND              0x08
+#define QEMU_VM_SECTION_FOOTER       0x7e
 
 struct MigrationParams {
     bool blk;
     bool shared;
 };
 
-typedef struct MigrationState MigrationState;
+/* Messages sent on the return path from destination to source */
+enum mig_rp_message_type {
+    MIG_RP_MSG_INVALID = 0,  /* Must be 0 */
+    MIG_RP_MSG_SHUT,         /* sibling will not send any more RP messages */
+    MIG_RP_MSG_PONG,         /* Response to a PING; data (seq: be32 ) */
+
+    MIG_RP_MSG_REQ_PAGES_ID, /* data (start: be64, len: be32, id: string) */
+    MIG_RP_MSG_REQ_PAGES,    /* data (start: be64, len: be32) */
+
+    MIG_RP_MSG_MAX
+};
+
+typedef QLIST_HEAD(, LoadStateEntry) LoadStateEntry_Head;
+
+/* The current postcopy state is read/set by postcopy_state_get/set
+ * which update it atomically.
+ * The state is updated as postcopy messages are received, and
+ * in general only one thread should be writing to the state at any one
+ * time, initially the main thread and then the listen thread;
+ * Corner cases are where either thread finishes early and/or errors.
+ * The state is checked as messages are received to ensure that
+ * the source is sending us messages in the correct order.
+ * The state is also used by the RAM reception code to know if it
+ * has to place pages atomically, and the cleanup code at the end of
+ * the main thread to know if it has to delay cleanup until the end
+ * of postcopy.
+ */
+typedef enum {
+    POSTCOPY_INCOMING_NONE = 0,  /* Initial state - no postcopy */
+    POSTCOPY_INCOMING_ADVISE,
+    POSTCOPY_INCOMING_DISCARD,
+    POSTCOPY_INCOMING_LISTENING,
+    POSTCOPY_INCOMING_RUNNING,
+    POSTCOPY_INCOMING_END
+} PostcopyState;
+
+/* State for the incoming migration */
+struct MigrationIncomingState {
+    QEMUFile *from_src_file;
+
+    /*
+     * Free at the start of the main state load, set as the main thread finishes
+     * loading state.
+     */
+    QemuEvent main_thread_load_event;
+
+    bool           have_fault_thread;
+    QemuThread     fault_thread;
+    QemuSemaphore  fault_thread_sem;
+
+    bool           have_listen_thread;
+    QemuThread     listen_thread;
+    QemuSemaphore  listen_thread_sem;
+
+    /* For the kernel to send us notifications */
+    int       userfault_fd;
+    /* To tell the fault_thread to quit */
+    int       userfault_quit_fd;
+    QEMUFile *to_src_file;
+    QemuMutex rp_mutex;    /* We send replies from multiple threads */
+    void     *postcopy_tmp_page;
+
+    QEMUBH *bh;
+
+    int state;
+    /* See savevm.c */
+    LoadStateEntry_Head loadvm_handlers;
+};
+
+MigrationIncomingState *migration_incoming_get_current(void);
+MigrationIncomingState *migration_incoming_state_new(QEMUFile *f);
+void migration_incoming_state_destroy(void);
+
+/*
+ * An outstanding page request, on the source, having been received
+ * and queued
+ */
+struct MigrationSrcPageRequest {
+    RAMBlock *rb;
+    hwaddr    offset;
+    hwaddr    len;
+
+    QSIMPLEQ_ENTRY(MigrationSrcPageRequest) next_req;
+};
 
 struct MigrationState
 {
@@ -48,32 +134,77 @@
     size_t xfer_limit;
     QemuThread thread;
     QEMUBH *cleanup_bh;
-    QEMUFile *file;
+    QEMUFile *to_dst_file;
+
+    /* New style params from 'migrate-set-parameters' */
+    MigrationParameters parameters;
 
     int state;
+    /* Old style params from 'migrate' command */
     MigrationParams params;
+
+    /* State related to return path */
+    struct {
+        QEMUFile     *from_dst_file;
+        QemuThread    rp_thread;
+        bool          error;
+    } rp_state;
+
     double mbps;
     int64_t total_time;
     int64_t downtime;
     int64_t expected_downtime;
     int64_t dirty_pages_rate;
     int64_t dirty_bytes_rate;
-    bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
+    bool enabled_capabilities[MIGRATION_CAPABILITY__MAX];
     int64_t xbzrle_cache_size;
     int64_t setup_time;
     int64_t dirty_sync_count;
+    /* Count of requests incoming from destination */
+    int64_t postcopy_requests;
+
+    /* Flag set once the migration has been asked to enter postcopy */
+    bool start_postcopy;
+    /* Flag set after postcopy has sent the device state */
+    bool postcopy_after_devices;
+
+    /* Flag set once the migration thread is running (and needs joining) */
+    bool migration_thread_running;
+
+    /* Queue of outstanding page requests from the destination */
+    QemuMutex src_page_req_mutex;
+    QSIMPLEQ_HEAD(src_page_requests, MigrationSrcPageRequest) src_page_requests;
+    /* The RAMBlock used in the last src_page_request */
+    RAMBlock *last_req_rb;
+
+    /* The last error that occurred */
+    Error *error;
 };
 
-void process_incoming_migration(QEMUFile *f);
+void migrate_set_state(int *state, int old_state, int new_state);
+
+void migration_fd_process_incoming(QEMUFile *f);
 
 void qemu_start_incoming_migration(const char *uri, Error **errp);
 
+void migration_channel_process_incoming(MigrationState *s,
+                                        QIOChannel *ioc);
+
+void migration_tls_channel_process_incoming(MigrationState *s,
+                                            QIOChannel *ioc,
+                                            Error **errp);
+
+void migration_channel_connect(MigrationState *s,
+                               QIOChannel *ioc,
+                               const char *hostname);
+
+void migration_tls_channel_connect(MigrationState *s,
+                                   QIOChannel *ioc,
+                                   const char *hostname,
+                                   Error **errp);
+
 uint64_t migrate_max_downtime(void);
 
-void do_info_migrate_print(Monitor *mon, const QObject *data);
-
-void do_info_migrate(Monitor *mon, QObject **ret_data);
-
 void exec_start_incoming_migration(const char *host_port, Error **errp);
 
 void exec_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
@@ -94,7 +225,7 @@
 
 void rdma_start_incoming_migration(const char *host_port, Error **errp);
 
-void migrate_fd_error(MigrationState *s);
+void migrate_fd_error(MigrationState *s, const Error *error);
 
 void migrate_fd_connect(MigrationState *s);
 
@@ -102,11 +233,21 @@
 
 void add_migration_state_change_notifier(Notifier *notify);
 void remove_migration_state_change_notifier(Notifier *notify);
+MigrationState *migrate_init(const MigrationParams *params);
+bool migration_is_blocked(Error **errp);
 bool migration_in_setup(MigrationState *);
 bool migration_has_finished(MigrationState *);
 bool migration_has_failed(MigrationState *);
+/* True if outgoing migration has entered postcopy phase */
+bool migration_in_postcopy(MigrationState *);
+/* ...and after the device transmission */
+bool migration_in_postcopy_after_devices(MigrationState *);
 MigrationState *migrate_get_current(void);
 
+void migrate_compress_threads_create(void);
+void migrate_compress_threads_join(void);
+void migrate_decompress_threads_create(void);
+void migrate_decompress_threads_join(void);
 uint64_t ram_bytes_remaining(void);
 uint64_t ram_bytes_transferred(void);
 uint64_t ram_bytes_total(void);
@@ -127,6 +268,13 @@
 double xbzrle_mig_cache_miss_rate(void);
 
 void ram_handle_compressed(void *host, uint8_t ch, uint64_t size);
+void ram_debug_dump_bitmap(unsigned long *todump, bool expected);
+/* For outgoing discard bitmap */
+int ram_postcopy_send_discard_bitmap(MigrationState *ms);
+/* For incoming postcopy discard */
+int ram_discard_range(MigrationIncomingState *mis, const char *block_name,
+                      uint64_t start, size_t length);
+int ram_postcopy_incoming_init(MigrationIncomingState *mis);
 
 /**
  * @migrate_add_blocker - prevent migration from proceeding
@@ -142,7 +290,7 @@
  */
 void migrate_del_blocker(Error *reason);
 
-bool migrate_rdma_pin_all(void);
+bool migrate_postcopy_ram(void);
 bool migrate_zero_blocks(void);
 
 bool migrate_auto_converge(void);
@@ -156,9 +304,26 @@
 
 int64_t xbzrle_cache_resize(int64_t new_size);
 
+bool migrate_use_compression(void);
+int migrate_compress_level(void);
+int migrate_compress_threads(void);
+int migrate_decompress_threads(void);
+bool migrate_use_events(void);
+
+/* Sending on the return path - generic and then for each message type */
+void migrate_send_rp_message(MigrationIncomingState *mis,
+                             enum mig_rp_message_type message_type,
+                             uint16_t len, void *data);
+void migrate_send_rp_shut(MigrationIncomingState *mis,
+                          uint32_t value);
+void migrate_send_rp_pong(MigrationIncomingState *mis,
+                          uint32_t value);
+void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char* rbname,
+                              ram_addr_t start, size_t len);
+
 void ram_control_before_iterate(QEMUFile *f, uint64_t flags);
 void ram_control_after_iterate(QEMUFile *f, uint64_t flags);
-void ram_control_load_hook(QEMUFile *f, uint64_t flags);
+void ram_control_load_hook(QEMUFile *f, uint64_t flags, void *data);
 
 /* Whenever this is found in the data stream, the flags
  * will be passed to ram_control_load_hook in the incoming-migration
@@ -172,6 +337,21 @@
 
 size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset,
                              ram_addr_t offset, size_t size,
-                             int *bytes_sent);
+                             uint64_t *bytes_sent);
 
+void ram_mig_init(void);
+void savevm_skip_section_footers(void);
+void register_global_state(void);
+void global_state_set_optional(void);
+void savevm_skip_configuration(void);
+int global_state_store(void);
+void global_state_store_running(void);
+
+void flush_page_queue(MigrationState *ms);
+int ram_save_queue_pages(MigrationState *ms, const char *rbname,
+                         ram_addr_t start, ram_addr_t len);
+
+PostcopyState postcopy_state_get(void);
+/* Set the state and return the old state */
+PostcopyState postcopy_state_set(PostcopyState new_state);
 #endif
diff --git a/include/migration/page_cache.h b/include/migration/page_cache.h
index 2d5ce2d..10ed532 100644
--- a/include/migration/page_cache.h
+++ b/include/migration/page_cache.h
@@ -43,8 +43,10 @@
  *
  * @cache pointer to the PageCache struct
  * @addr: page addr
+ * @current_age: current bitmap generation
  */
-bool cache_is_cached(const PageCache *cache, uint64_t addr);
+bool cache_is_cached(const PageCache *cache, uint64_t addr,
+                     uint64_t current_age);
 
 /**
  * get_cached_data: Get the data cached for an addr
@@ -60,13 +62,15 @@
  * cache_insert: insert the page into the cache. the page cache
  * will dup the data on insert. the previous value will be overwritten
  *
- * Returns -1 on error
+ * Returns -1 when the page isn't inserted into cache
  *
  * @cache pointer to the PageCache struct
  * @addr: page address
  * @pdata: pointer to the page
+ * @current_age: current bitmap generation
  */
-int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata);
+int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata,
+                 uint64_t current_age);
 
 /**
  * cache_resize: resize the page cache. In case of size reduction the extra
diff --git a/include/migration/postcopy-ram.h b/include/migration/postcopy-ram.h
new file mode 100644
index 0000000..b6a7491
--- /dev/null
+++ b/include/migration/postcopy-ram.h
@@ -0,0 +1,99 @@
+/*
+ * Postcopy migration for RAM
+ *
+ * Copyright 2013 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Dave Gilbert  <dgilbert@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#ifndef QEMU_POSTCOPY_RAM_H
+#define QEMU_POSTCOPY_RAM_H
+
+/* Return true if the host supports everything we need to do postcopy-ram */
+bool postcopy_ram_supported_by_host(void);
+
+/*
+ * Make all of RAM sensitive to accesses to areas that haven't yet been written
+ * and wire up anything necessary to deal with it.
+ */
+int postcopy_ram_enable_notify(MigrationIncomingState *mis);
+
+/*
+ * Initialise postcopy-ram, setting the RAM to a state where we can go into
+ * postcopy later; must be called prior to any precopy.
+ * called from ram.c's similarly named ram_postcopy_incoming_init
+ */
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages);
+
+/*
+ * At the end of a migration where postcopy_ram_incoming_init was called.
+ */
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis);
+
+/*
+ * Discard the contents of 'length' bytes from 'start'
+ * We can assume that if we've been called postcopy_ram_hosttest returned true
+ */
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length);
+
+/*
+ * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
+ * however leaving it until after precopy means that most of the precopy
+ * data is still THPd
+ */
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis);
+
+/*
+ * Called at the start of each RAMBlock by the bitmap code.
+ * 'offset' is the bitmap offset of the named RAMBlock in the migration
+ * bitmap.
+ * Returns a new PDS
+ */
+PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms,
+                                                 unsigned long offset,
+                                                 const char *name);
+
+/*
+ * Called by the bitmap code for each chunk to discard.
+ * May send a discard message, may just leave it queued to
+ * be sent later.
+ * @start,@length: a range of pages in the migration bitmap in the
+ *  RAM block passed to postcopy_discard_send_init() (length=1 is one page)
+ */
+void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds,
+                                 unsigned long start, unsigned long length);
+
+/*
+ * Called at the end of each RAMBlock by the bitmap code.
+ * Sends any outstanding discard messages, frees the PDS.
+ */
+void postcopy_discard_send_finish(MigrationState *ms,
+                                  PostcopyDiscardState *pds);
+
+/*
+ * Place a page (from) at (host) efficiently
+ *    There are restrictions on how 'from' must be mapped, in general best
+ *    to use other postcopy_ routines to allocate.
+ * returns 0 on success
+ */
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from);
+
+/*
+ * Place a zero page at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host);
+
+/*
+ * Allocate a page of memory that can be mapped at a later point in time
+ * using postcopy_place_page
+ * Returns: Pointer to allocated page
+ */
+void *postcopy_get_tmp_page(MigrationIncomingState *mis);
+
+#endif
diff --git a/include/migration/qemu-file.h b/include/migration/qemu-file.h
index 401676b..abedd46 100644
--- a/include/migration/qemu-file.h
+++ b/include/migration/qemu-file.h
@@ -21,25 +21,21 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef QEMU_FILE_H
-#define QEMU_FILE_H 1
+#define QEMU_FILE_H
+
+#include "qemu-common.h"
 #include "exec/cpu-common.h"
+#include "io/channel.h"
 
-#include <stdint.h>
-
-/* This function writes a chunk of data to a file at the given position.
- * The pos argument can be ignored if the file is only being used for
- * streaming.  The handler should try to write all of the data it can.
- */
-typedef int (QEMUFilePutBufferFunc)(void *opaque, const uint8_t *buf,
-                                    int64_t pos, int size);
 
 /* Read a chunk of data from a file at the given position.  The pos argument
  * can be ignored if the file is only be used for streaming.  The number of
  * bytes actually read should be returned.
  */
-typedef int (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
-                                    int64_t pos, int size);
+typedef ssize_t (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
+                                        int64_t pos, size_t size);
 
 /* Close a file
  *
@@ -54,8 +50,13 @@
  */
 typedef int (QEMUFileGetFD)(void *opaque);
 
+/* Called to change the blocking mode of the file
+ */
+typedef int (QEMUFileSetBlocking)(void *opaque, bool enabled);
+
 /*
- * This function writes an iovec to file.
+ * This function writes an iovec to file. The handler must write all
+ * of the data or return a negative errno value.
  */
 typedef ssize_t (QEMUFileWritevBufferFunc)(void *opaque, struct iovec *iov,
                                            int iovcnt, int64_t pos);
@@ -63,16 +64,20 @@
 /*
  * This function provides hooks around different
  * stages of RAM migration.
+ * 'opaque' is the backend specific data in QEMUFile
+ * 'data' is call specific data associated with the 'flags' value
  */
-typedef int (QEMURamHookFunc)(QEMUFile *f, void *opaque, uint64_t flags);
+typedef int (QEMURamHookFunc)(QEMUFile *f, void *opaque, uint64_t flags,
+                              void *data);
 
 /*
  * Constants used by ram_control_* hooks
  */
-#define RAM_CONTROL_SETUP    0
-#define RAM_CONTROL_ROUND    1
-#define RAM_CONTROL_HOOK     2
-#define RAM_CONTROL_FINISH   3
+#define RAM_CONTROL_SETUP     0
+#define RAM_CONTROL_ROUND     1
+#define RAM_CONTROL_HOOK      2
+#define RAM_CONTROL_FINISH    3
+#define RAM_CONTROL_BLOCK_REG 4
 
 /*
  * This function allows override of where the RAM page
@@ -82,61 +87,55 @@
                                ram_addr_t block_offset,
                                ram_addr_t offset,
                                size_t size,
-                               int *bytes_sent);
+                               uint64_t *bytes_sent);
+
+/*
+ * Return a QEMUFile for comms in the opposite direction
+ */
+typedef QEMUFile *(QEMURetPathFunc)(void *opaque);
+
+/*
+ * Stop any read or write (depending on flags) on the underlying
+ * transport on the QEMUFile.
+ * Existing blocking reads/writes must be woken
+ * Returns 0 on success, -err on error
+ */
+typedef int (QEMUFileShutdownFunc)(void *opaque, bool rd, bool wr);
 
 typedef struct QEMUFileOps {
-    QEMUFilePutBufferFunc *put_buffer;
     QEMUFileGetBufferFunc *get_buffer;
     QEMUFileCloseFunc *close;
-    QEMUFileGetFD *get_fd;
+    QEMUFileSetBlocking *set_blocking;
     QEMUFileWritevBufferFunc *writev_buffer;
+    QEMURetPathFunc *get_return_path;
+    QEMUFileShutdownFunc *shut_down;
+} QEMUFileOps;
+
+typedef struct QEMUFileHooks {
     QEMURamHookFunc *before_ram_iterate;
     QEMURamHookFunc *after_ram_iterate;
     QEMURamHookFunc *hook_ram_load;
     QEMURamSaveFunc *save_page;
-} QEMUFileOps;
-
-struct QEMUSizedBuffer {
-    struct iovec *iov;
-    size_t n_iov;
-    size_t size; /* total allocated size in all iov's */
-    size_t used; /* number of used bytes */
-};
+} QEMUFileHooks;
 
 QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops);
-QEMUFile *qemu_fopen(const char *filename, const char *mode);
-QEMUFile *qemu_fdopen(int fd, const char *mode);
-QEMUFile *qemu_fopen_socket(int fd, const char *mode);
-QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
-QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input);
+QEMUFile *qemu_fopen_channel_input(QIOChannel *ioc);
+QEMUFile *qemu_fopen_channel_output(QIOChannel *ioc);
+void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks);
 int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 int64_t qemu_ftell(QEMUFile *f);
-void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
+int64_t qemu_ftell_fast(QEMUFile *f);
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size);
 void qemu_put_byte(QEMUFile *f, int v);
 /*
  * put_buffer without copying the buffer.
  * The buffer should be available till it is sent asynchronously.
  */
-void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size);
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size);
 bool qemu_file_mode_is_not_valid(const char *mode);
 bool qemu_file_is_writable(QEMUFile *f);
 
-QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len);
-QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *);
-void qsb_free(QEMUSizedBuffer *);
-size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t length);
-size_t qsb_get_length(const QEMUSizedBuffer *qsb);
-ssize_t qsb_get_buffer(const QEMUSizedBuffer *, off_t start, size_t count,
-                       uint8_t *buf);
-ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *buf,
-                     off_t pos, size_t count);
-
-
-/*
- * For use on files opened with qemu_bufopen
- */
-const QEMUSizedBuffer *qemu_buf_get(QEMUFile *f);
 
 static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
 {
@@ -148,8 +147,13 @@
 void qemu_put_be16(QEMUFile *f, unsigned int v);
 void qemu_put_be32(QEMUFile *f, unsigned int v);
 void qemu_put_be64(QEMUFile *f, uint64_t v);
-int qemu_peek_buffer(QEMUFile *f, uint8_t *buf, int size, size_t offset);
-int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size);
+size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset);
+size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size);
+size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size);
+ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
+                                  int level);
+int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src);
+
 /*
  * Note that you can only peek continuous bytes from where the current pointer
  * is; you aren't guaranteed to be able to peak to +n bytes unless you've
@@ -177,7 +181,10 @@
 int64_t qemu_file_get_rate_limit(QEMUFile *f);
 int qemu_file_get_error(QEMUFile *f);
 void qemu_file_set_error(QEMUFile *f, int ret);
+int qemu_file_shutdown(QEMUFile *f);
+QEMUFile *qemu_file_get_return_path(QEMUFile *f);
 void qemu_fflush(QEMUFile *f);
+void qemu_file_set_blocking(QEMUFile *f, bool block);
 
 static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
 {
@@ -220,7 +227,7 @@
 }
 
 // Signed versions for type safety
-static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, int size)
+static inline void qemu_put_sbuffer(QEMUFile *f, const int8_t *buf, size_t size)
 {
     qemu_put_buffer(f, (const uint8_t *)buf, size);
 }
@@ -299,4 +306,7 @@
 {
     qemu_get_be64s(f, (uint64_t *)pv);
 }
+
+size_t qemu_get_counted_string(QEMUFile *f, char buf[256]);
+
 #endif
diff --git a/include/migration/qjson.h b/include/migration/qjson.h
new file mode 100644
index 0000000..2978b5f
--- /dev/null
+++ b/include/migration/qjson.h
@@ -0,0 +1,29 @@
+/*
+ * QEMU JSON writer
+ *
+ * Copyright Alexander Graf
+ *
+ * Authors:
+ *  Alexander Graf <agraf@suse.de>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+#ifndef QEMU_QJSON_H
+#define QEMU_QJSON_H
+
+typedef struct QJSON QJSON;
+
+QJSON *qjson_new(void);
+void qjson_destroy(QJSON *json);
+void json_prop_str(QJSON *json, const char *name, const char *str);
+void json_prop_int(QJSON *json, const char *name, int64_t val);
+void json_end_array(QJSON *json);
+void json_start_array(QJSON *json, const char *name);
+void json_end_object(QJSON *json);
+void json_start_object(QJSON *json, const char *name);
+const char *qjson_get_str(QJSON *json);
+void qjson_finish(QJSON *json);
+
+#endif /* QEMU_QJSON_H */
diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 436fcca..85d1849 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -23,12 +23,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+
 #ifndef QEMU_VMSTATE_H
-#define QEMU_VMSTATE_H 1
+#define QEMU_VMSTATE_H
 
 #ifndef CONFIG_USER_ONLY
-#include <migration/qemu-file.h>
+#include "migration/qemu-file.h"
 #endif
+#include "migration/qjson.h"
 
 typedef void SaveStateHandler(QEMUFile *f, void *opaque);
 typedef int LoadStateHandler(QEMUFile *f, void *opaque, int version_id);
@@ -39,8 +41,9 @@
     void (*set_params)(const MigrationParams *params, void * opaque);
     SaveStateHandler *save_state;
 
-    void (*cancel)(void *opaque);
-    int (*save_live_complete)(QEMUFile *f, void *opaque);
+    void (*cleanup)(void *opaque);
+    int (*save_live_complete_postcopy)(QEMUFile *f, void *opaque);
+    int (*save_live_complete_precopy)(QEMUFile *f, void *opaque);
 
     /* This runs both outside and inside the iothread lock.  */
     bool (*is_active)(void *opaque);
@@ -54,10 +57,10 @@
 
     /* This runs outside the iothread lock!  */
     int (*save_live_setup)(QEMUFile *f, void *opaque);
-    uint64_t (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size);
-
+    void (*save_live_pending)(QEMUFile *f, void *opaque, uint64_t max_size,
+                              uint64_t *non_postcopiable_pending,
+                              uint64_t *postcopiable_pending);
     LoadStateHandler *load_state;
-
     PostLoadHandler *post_load;
 } SaveVMHandlers;
 
@@ -87,9 +90,6 @@
                          void *opaque);
 
 void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque);
-void register_device_unmigratable(DeviceState *dev, const char *idstr,
-                                                                void *opaque);
-
 
 typedef struct VMStateInfo VMStateInfo;
 typedef struct VMStateDescription VMStateDescription;
@@ -101,20 +101,101 @@
 };
 
 enum VMStateFlags {
+    /* Ignored */
     VMS_SINGLE           = 0x001,
+
+    /* The struct member at opaque + VMStateField.offset is a pointer
+     * to the actual field (e.g. struct a { uint8_t *b;
+     * }). Dereference the pointer before using it as basis for
+     * further pointer arithmetic (see e.g. VMS_ARRAY). Does not
+     * affect the meaning of VMStateField.num_offset or
+     * VMStateField.size_offset; see VMS_VARRAY* and VMS_VBUFFER for
+     * those. */
     VMS_POINTER          = 0x002,
+
+    /* The field is an array of fixed size. VMStateField.num contains
+     * the number of entries in the array. The size of each entry is
+     * given by VMStateField.size and / or opaque +
+     * VMStateField.size_offset; see VMS_VBUFFER and
+     * VMS_MULTIPLY. Each array entry will be processed individually
+     * (VMStateField.info.get()/put() if VMS_STRUCT is not set,
+     * recursion into VMStateField.vmsd if VMS_STRUCT is set). May not
+     * be combined with VMS_VARRAY*. */
     VMS_ARRAY            = 0x004,
+
+    /* The field is itself a struct, containing one or more
+     * fields. Recurse into VMStateField.vmsd. Most useful in
+     * combination with VMS_ARRAY / VMS_VARRAY*, recursing into each
+     * array entry. */
     VMS_STRUCT           = 0x008,
-    VMS_VARRAY_INT32     = 0x010,  /* Array with size in int32_t field*/
-    VMS_BUFFER           = 0x020,  /* static sized buffer */
+
+    /* The field is an array of variable size. The int32_t at opaque +
+     * VMStateField.num_offset contains the number of entries in the
+     * array. See the VMS_ARRAY description regarding array handling
+     * in general. May not be combined with VMS_ARRAY or any other
+     * VMS_VARRAY*. */
+    VMS_VARRAY_INT32     = 0x010,
+
+    /* Ignored */
+    VMS_BUFFER           = 0x020,
+
+    /* The field is a (fixed-size or variable-size) array of pointers
+     * (e.g. struct a { uint8_t *b[]; }). Dereference each array entry
+     * before using it. Note: Does not imply any one of VMS_ARRAY /
+     * VMS_VARRAY*; these need to be set explicitly. */
     VMS_ARRAY_OF_POINTER = 0x040,
-    VMS_VARRAY_UINT16    = 0x080,  /* Array with size in uint16_t field */
-    VMS_VBUFFER          = 0x100,  /* Buffer with size in int32_t field */
-    VMS_MULTIPLY         = 0x200,  /* multiply "size" field by field_size */
-    VMS_VARRAY_UINT8     = 0x400,  /* Array with size in uint8_t field*/
-    VMS_VARRAY_UINT32    = 0x800,  /* Array with size in uint32_t field*/
-    VMS_MUST_EXIST       = 0x1000, /* Field must exist in input */
-    VMS_ALLOC            = 0x2000, /* Alloc a buffer on the destination */
+
+    /* The field is an array of variable size. The uint16_t at opaque
+     * + VMStateField.num_offset (subject to VMS_MULTIPLY_ELEMENTS)
+     * contains the number of entries in the array. See the VMS_ARRAY
+     * description regarding array handling in general. May not be
+     * combined with VMS_ARRAY or any other VMS_VARRAY*. */
+    VMS_VARRAY_UINT16    = 0x080,
+
+    /* The size of the individual entries (a single array entry if
+     * VMS_ARRAY or any of VMS_VARRAY* are set, or the field itself if
+     * neither is set) is variable (i.e. not known at compile-time),
+     * but the same for all entries. Use the int32_t at opaque +
+     * VMStateField.size_offset (subject to VMS_MULTIPLY) to determine
+     * the size of each (and every) entry. */
+    VMS_VBUFFER          = 0x100,
+
+    /* Multiply the entry size given by the int32_t at opaque +
+     * VMStateField.size_offset (see VMS_VBUFFER description) with
+     * VMStateField.size to determine the number of bytes to be
+     * allocated. Only valid in combination with VMS_VBUFFER. */
+    VMS_MULTIPLY         = 0x200,
+
+    /* The field is an array of variable size. The uint8_t at opaque +
+     * VMStateField.num_offset (subject to VMS_MULTIPLY_ELEMENTS)
+     * contains the number of entries in the array. See the VMS_ARRAY
+     * description regarding array handling in general. May not be
+     * combined with VMS_ARRAY or any other VMS_VARRAY*. */
+    VMS_VARRAY_UINT8     = 0x400,
+
+    /* The field is an array of variable size. The uint32_t at opaque
+     * + VMStateField.num_offset (subject to VMS_MULTIPLY_ELEMENTS)
+     * contains the number of entries in the array. See the VMS_ARRAY
+     * description regarding array handling in general. May not be
+     * combined with VMS_ARRAY or any other VMS_VARRAY*. */
+    VMS_VARRAY_UINT32    = 0x800,
+
+    /* Fail loading the serialised VM state if this field is missing
+     * from the input. */
+    VMS_MUST_EXIST       = 0x1000,
+
+    /* When loading serialised VM state, allocate memory for the
+     * (entire) field. Only valid in combination with
+     * VMS_POINTER. Note: Not all combinations with other flags are
+     * currently supported, e.g. VMS_ALLOC|VMS_ARRAY_OF_POINTER won't
+     * cause the individual entries to be allocated. */
+    VMS_ALLOC            = 0x2000,
+
+    /* Multiply the number of entries given by the integer at opaque +
+     * VMStateField.num_offset (see VMS_VARRAY*) with VMStateField.num
+     * to determine the number of entries in the array. Only valid in
+     * combination with one of VMS_VARRAY*. */
+    VMS_MULTIPLY_ELEMENTS = 0x4000,
 };
 
 typedef struct {
@@ -132,11 +213,6 @@
     bool (*field_exists)(void *opaque, int version_id);
 } VMStateField;
 
-typedef struct VMStateSubsection {
-    const VMStateDescription *vmsd;
-    bool (*needed)(void *opaque);
-} VMStateSubsection;
-
 struct VMStateDescription {
     const char *name;
     int unmigratable;
@@ -147,13 +223,12 @@
     int (*pre_load)(void *opaque);
     int (*post_load)(void *opaque, int version_id);
     void (*pre_save)(void *opaque);
+    bool (*needed)(void *opaque);
     VMStateField *fields;
-    const VMStateSubsection *subsections;
+    const VMStateDescription **subsections;
 };
 
-#ifdef CONFIG_USER_ONLY
 extern const VMStateDescription vmstate_dummy;
-#endif
 
 extern const VMStateInfo vmstate_info_bool;
 
@@ -175,6 +250,7 @@
 extern const VMStateInfo vmstate_info_uint64;
 
 extern const VMStateInfo vmstate_info_float64;
+extern const VMStateInfo vmstate_info_cpudouble;
 
 extern const VMStateInfo vmstate_info_timer;
 extern const VMStateInfo vmstate_info_buffer;
@@ -202,7 +278,7 @@
      type_check_2darray(_type, typeof_field(_state, _field), _n1, _n2))
 
 #define vmstate_offset_sub_array(_state, _field, _type, _start)      \
-    (offsetof(_state, _field[_start]))
+    vmstate_offset_value(_state, _field[_start], _type)
 
 #define vmstate_offset_buffer(_state, _field)                        \
     vmstate_offset_array(_state, _field, uint8_t,                    \
@@ -264,6 +340,16 @@
     .offset     = vmstate_offset_2darray(_state, _field, _type, _n1, _n2),  \
 }
 
+#define VMSTATE_VARRAY_MULTIPLY(_field, _state, _field_num, _multiply, _info, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint32_t),\
+    .num        = (_multiply),                                       \
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_UINT32|VMS_MULTIPLY_ELEMENTS,           \
+    .offset     = offsetof(_state, _field),                          \
+}
+
 #define VMSTATE_ARRAY_TEST(_field, _state, _num, _test, _info, _type) {\
     .name         = (stringify(_field)),                              \
     .field_exists = (_test),                                          \
@@ -313,6 +399,16 @@
     .offset     = vmstate_offset_pointer(_state, _field, _type),     \
 }
 
+#define VMSTATE_VARRAY_UINT32_ALLOC(_field, _state, _field_num, _version, _info, _type) {\
+    .name       = (stringify(_field)),                               \
+    .version_id = (_version),                                        \
+    .num_offset = vmstate_offset_value(_state, _field_num, uint32_t),\
+    .info       = &(_info),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_VARRAY_UINT32|VMS_POINTER|VMS_ALLOC,           \
+    .offset     = vmstate_offset_pointer(_state, _field, _type),     \
+}
+
 #define VMSTATE_VARRAY_UINT16_UNSAFE(_field, _state, _field_num, _version, _info, _type) {\
     .name       = (stringify(_field)),                               \
     .version_id = (_version),                                        \
@@ -372,6 +468,16 @@
     .offset     = vmstate_offset_array(_s, _f, _type*, _n),          \
 }
 
+#define VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, _num, _version, _vmsd, _type) { \
+    .name       = (stringify(_field)),                                     \
+    .version_id = (_version),                                              \
+    .num        = (_num),                                                  \
+    .vmsd       = &(_vmsd),                                                \
+    .size       = sizeof(_type),                                           \
+    .flags      = VMS_STRUCT|VMS_ARRAY,                                    \
+    .offset     = vmstate_offset_sub_array(_state, _field, _type, _start), \
+}
+
 #define VMSTATE_STRUCT_ARRAY_TEST(_field, _state, _num, _test, _version, _vmsd, _type) { \
     .name         = (stringify(_field)),                             \
     .num          = (_num),                                          \
@@ -393,6 +499,19 @@
     .offset     = offsetof(_state, _field),                          \
 }
 
+/* a variable length array (i.e. _type *_field) but we know the
+ * length
+ */
+#define VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(_field, _state, _num, _version, _vmsd, _type) { \
+    .name       = (stringify(_field)),                               \
+    .num          = (_num),                                          \
+    .version_id = (_version),                                        \
+    .vmsd       = &(_vmsd),                                          \
+    .size       = sizeof(_type),                                     \
+    .flags      = VMS_STRUCT|VMS_ARRAY|VMS_POINTER,                  \
+    .offset     = offsetof(_state, _field),                          \
+}
+
 #define VMSTATE_STRUCT_VARRAY_POINTER_INT32(_field, _state, _field_num, _vmsd, _type) { \
     .name       = (stringify(_field)),                               \
     .version_id = 0,                                                 \
@@ -508,9 +627,10 @@
     .start        = (_start),                                        \
 }
 
-#define VMSTATE_BUFFER_UNSAFE_INFO(_field, _state, _version, _info, _size) { \
+#define VMSTATE_BUFFER_UNSAFE_INFO_TEST(_field, _state, _test, _version, _info, _size) { \
     .name       = (stringify(_field)),                               \
     .version_id = (_version),                                        \
+    .field_exists = (_test),                                         \
     .size       = (_size),                                           \
     .info       = &(_info),                                          \
     .flags      = VMS_BUFFER,                                        \
@@ -570,6 +690,10 @@
     VMSTATE_STRUCT_ARRAY_TEST(_field, _state, _num, NULL, _version,   \
             _vmsd, _type)
 
+#define VMSTATE_BUFFER_UNSAFE_INFO(_field, _state, _version, _info, _size) \
+    VMSTATE_BUFFER_UNSAFE_INFO_TEST(_field, _state, NULL, _version, _info, \
+            _size)
+
 #define VMSTATE_BOOL_V(_f, _s, _v)                                    \
     VMSTATE_SINGLE(_f, _s, _v, vmstate_info_bool, bool)
 
@@ -639,6 +763,18 @@
 #define VMSTATE_INT32_POSITIVE_LE(_f, _s)                             \
     VMSTATE_SINGLE(_f, _s, 0, vmstate_info_int32_le, int32_t)
 
+#define VMSTATE_INT8_TEST(_f, _s, _t)                               \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_int8, int8_t)
+
+#define VMSTATE_INT16_TEST(_f, _s, _t)                               \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_int16, int16_t)
+
+#define VMSTATE_INT32_TEST(_f, _s, _t)                                  \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_int32, int32_t)
+
+#define VMSTATE_INT64_TEST(_f, _s, _t)                                  \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_int64, int64_t)
+
 #define VMSTATE_UINT8_TEST(_f, _s, _t)                               \
     VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_uint8, uint8_t)
 
@@ -648,6 +784,9 @@
 #define VMSTATE_UINT32_TEST(_f, _s, _t)                                  \
     VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_uint32, uint32_t)
 
+#define VMSTATE_UINT64_TEST(_f, _s, _t)                                  \
+    VMSTATE_SINGLE_TEST(_f, _s, _t, 0, vmstate_info_uint64, uint64_t)
+
 
 #define VMSTATE_FLOAT64_V(_f, _s, _v)                                 \
     VMSTATE_SINGLE(_f, _s, _v, vmstate_info_float64, float64)
@@ -655,17 +794,29 @@
 #define VMSTATE_FLOAT64(_f, _s)                                       \
     VMSTATE_FLOAT64_V(_f, _s, 0)
 
-#define VMSTATE_TIMER_TEST(_f, _s, _test)                             \
+#define VMSTATE_TIMER_PTR_TEST(_f, _s, _test)                             \
     VMSTATE_POINTER_TEST(_f, _s, _test, vmstate_info_timer, QEMUTimer *)
 
-#define VMSTATE_TIMER_V(_f, _s, _v)                                   \
+#define VMSTATE_TIMER_PTR_V(_f, _s, _v)                                   \
     VMSTATE_POINTER(_f, _s, _v, vmstate_info_timer, QEMUTimer *)
 
+#define VMSTATE_TIMER_PTR(_f, _s)                                         \
+    VMSTATE_TIMER_PTR_V(_f, _s, 0)
+
+#define VMSTATE_TIMER_PTR_ARRAY(_f, _s, _n)                              \
+    VMSTATE_ARRAY_OF_POINTER(_f, _s, _n, 0, vmstate_info_timer, QEMUTimer *)
+
+#define VMSTATE_TIMER_TEST(_f, _s, _test)                             \
+    VMSTATE_SINGLE_TEST(_f, _s, _test, 0, vmstate_info_timer, QEMUTimer)
+
+#define VMSTATE_TIMER_V(_f, _s, _v)                                   \
+    VMSTATE_SINGLE(_f, _s, _v, vmstate_info_timer, QEMUTimer)
+
 #define VMSTATE_TIMER(_f, _s)                                         \
     VMSTATE_TIMER_V(_f, _s, 0)
 
 #define VMSTATE_TIMER_ARRAY(_f, _s, _n)                              \
-    VMSTATE_ARRAY_OF_POINTER(_f, _s, _n, 0, vmstate_info_timer, QEMUTimer *)
+    VMSTATE_ARRAY(_f, _s, _n, 0, vmstate_info_timer, QEMUTimer)
 
 #define VMSTATE_BOOL_ARRAY_V(_f, _s, _n, _v)                         \
     VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_bool, bool)
@@ -718,6 +869,12 @@
 #define VMSTATE_UINT64_ARRAY(_f, _s, _n)                              \
     VMSTATE_UINT64_ARRAY_V(_f, _s, _n, 0)
 
+#define VMSTATE_UINT64_2DARRAY(_f, _s, _n1, _n2)                      \
+    VMSTATE_UINT64_2DARRAY_V(_f, _s, _n1, _n2, 0)
+
+#define VMSTATE_UINT64_2DARRAY_V(_f, _s, _n1, _n2, _v)                 \
+    VMSTATE_2DARRAY(_f, _s, _n1, _n2, _v, vmstate_info_uint64, uint64_t)
+
 #define VMSTATE_INT16_ARRAY_V(_f, _s, _n, _v)                         \
     VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int16, int16_t)
 
@@ -733,9 +890,6 @@
 #define VMSTATE_UINT32_SUB_ARRAY(_f, _s, _start, _num)                \
     VMSTATE_SUB_ARRAY(_f, _s, _start, _num, 0, vmstate_info_uint32, uint32_t)
 
-#define VMSTATE_UINT32_ARRAY(_f, _s, _n)                              \
-    VMSTATE_UINT32_ARRAY_V(_f, _s, _n, 0)
-
 #define VMSTATE_INT64_ARRAY_V(_f, _s, _n, _v)                         \
     VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_int64, int64_t)
 
@@ -748,6 +902,12 @@
 #define VMSTATE_FLOAT64_ARRAY(_f, _s, _n)                             \
     VMSTATE_FLOAT64_ARRAY_V(_f, _s, _n, 0)
 
+#define VMSTATE_CPUDOUBLE_ARRAY_V(_f, _s, _n, _v)                     \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_cpudouble, CPU_DoubleU)
+
+#define VMSTATE_CPUDOUBLE_ARRAY(_f, _s, _n)                           \
+    VMSTATE_CPUDOUBLE_ARRAY_V(_f, _s, _n, 0)
+
 #define VMSTATE_BUFFER_V(_f, _s, _v)                                  \
     VMSTATE_STATIC_BUFFER(_f, _s, _v, NULL, 0, sizeof(typeof_field(_s, _f)))
 
@@ -757,8 +917,11 @@
 #define VMSTATE_PARTIAL_BUFFER(_f, _s, _size)                         \
     VMSTATE_STATIC_BUFFER(_f, _s, 0, NULL, 0, _size)
 
+#define VMSTATE_BUFFER_START_MIDDLE_V(_f, _s, _start, _v) \
+    VMSTATE_STATIC_BUFFER(_f, _s, _v, NULL, _start, sizeof(typeof_field(_s, _f)))
+
 #define VMSTATE_BUFFER_START_MIDDLE(_f, _s, _start) \
-    VMSTATE_STATIC_BUFFER(_f, _s, 0, NULL, _start, sizeof(typeof_field(_s, _f)))
+    VMSTATE_BUFFER_START_MIDDLE_V(_f, _s, _start, 0)
 
 #define VMSTATE_PARTIAL_VBUFFER(_f, _s, _size)                        \
     VMSTATE_VBUFFER(_f, _s, 0, NULL, 0, _size)
@@ -789,10 +952,14 @@
 
 #define SELF_ANNOUNCE_ROUNDS 5
 
+void loadvm_free_handlers(MigrationIncomingState *mis);
+
 int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
                        void *opaque, int version_id);
 void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
-                        void *opaque);
+                        void *opaque, QJSON *vmdesc);
+
+bool vmstate_save_needed(const VMStateDescription *vmsd, void *opaque);
 
 int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
                                    const VMStateDescription *vmsd,
diff --git a/include/monitor/hmp-target.h b/include/monitor/hmp-target.h
new file mode 100644
index 0000000..454e8ed
--- /dev/null
+++ b/include/monitor/hmp-target.h
@@ -0,0 +1,50 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef MONITOR_HMP_TARGET_H
+#define MONITOR_HMP_TARGET_H
+
+#define MD_TLONG 0
+#define MD_I32   1
+
+struct MonitorDef {
+    const char *name;
+    int offset;
+    target_long (*get_value)(const struct MonitorDef *md, int val);
+    int type;
+};
+
+const MonitorDef *target_monitor_defs(void);
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval);
+
+CPUArchState *mon_get_cpu_env(void);
+CPUState *mon_get_cpu(void);
+
+void hmp_info_mem(Monitor *mon, const QDict *qdict);
+void hmp_info_tlb(Monitor *mon, const QDict *qdict);
+void hmp_mce(Monitor *mon, const QDict *qdict);
+void hmp_info_local_apic(Monitor *mon, const QDict *qdict);
+void hmp_info_io_apic(Monitor *mon, const QDict *qdict);
+
+#endif /* MONITOR_HMP_TARGET_H */
diff --git a/include/monitor/monitor.h b/include/monitor/monitor.h
index 6714fff..a714d8e 100644
--- a/include/monitor/monitor.h
+++ b/include/monitor/monitor.h
@@ -2,60 +2,22 @@
 #define MONITOR_H
 
 #include "qemu-common.h"
-#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qdict.h"
 #include "block/block.h"
 #include "qemu/readline.h"
 
 extern Monitor *cur_mon;
-extern Monitor *default_mon;
-
-typedef void (MonitorCompletion)(void *opaque, QObject *ret_data);
-
-typedef union cmd_table_t {
-    struct mon_cmd_t *static_table;
-    GArray           *dynamic_table;
-} cmd_table_t;
-
-typedef struct mon_cmd_t {
-    const char *name;
-    const char *args_type;
-    const char *params;
-    const char *help;
-    void (*user_print)(Monitor *mon, const QObject *data);
-    union {
-        void (*cmd)(Monitor *mon, const QDict *qdict);
-        int  (*cmd_new)(Monitor *mon, const QDict *params, QObject **ret_data);
-        int  (*cmd_async)(Monitor *mon, const QDict *params,
-                          MonitorCompletion *cb, void *opaque);
-    } mhandler;
-    int flags;
-    /* @sub_cmd is a list of 2nd level of commands. If it do not exist,
-     * mhandler should be used. If it exist, sub_table[?].mhandler should be
-     * used, and mhandler of 1st level plays the role of help
-     * function.
-     *
-     * .hx defined sub_cmds can only be static.
-     */
-    cmd_table_t sub_cmds;
-
-    void (*command_completion)(ReadLineState *rs, int nb_args, const char *str);
-} mon_cmd_t;
 
 /* flags for monitor_init */
 #define MONITOR_IS_DEFAULT    0x01
 #define MONITOR_USE_READLINE  0x02
 #define MONITOR_USE_CONTROL   0x04
 #define MONITOR_USE_PRETTY    0x08
-#define MONITOR_DYNAMIC_CMDS  0x10
 
-/* flags for monitor commands */
-#define MONITOR_CMD_ASYNC       0x0001
+bool monitor_cur_is_qmp(void);
 
-int monitor_cur_is_qmp(void);
-
-Monitor * monitor_init(CharDriverState *chr, int flags);
-void monitor_add_command(Monitor *mon, mon_cmd_t *cmd);
+void monitor_init(CharDriverState *chr, int flags);
+void monitor_cleanup(void);
 
 int monitor_suspend(Monitor *mon);
 void monitor_resume(Monitor *mon);
@@ -68,29 +30,20 @@
                                   void *opaque);
 
 int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp);
-int monitor_handle_fd_param(Monitor *mon, const char *fdname);
-int monitor_handle_fd_param2(Monitor *mon, const char *fdname, Error **errp);
+int monitor_fd_param(Monitor *mon, const char *fdname, Error **errp);
 
 void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
     GCC_FMT_ATTR(2, 0);
 void monitor_printf(Monitor *mon, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
+int monitor_fprintf(FILE *stream, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
 void monitor_flush(Monitor *mon);
 int monitor_set_cpu(int cpu_index);
 int monitor_get_cpu_index(void);
 
-void monitor_set_error(Monitor *mon, QError *qerror);
 void monitor_read_command(Monitor *mon, int show_prompt);
-ReadLineState *monitor_get_rs(Monitor *mon);
 int monitor_read_password(Monitor *mon, ReadLineFunc *readline_func,
                           void *opaque);
 
-int qmp_qom_set(Monitor *mon, const QDict *qdict, QObject **ret);
-
-int qmp_qom_get(Monitor *mon, const QDict *qdict, QObject **ret);
-int qmp_object_add(Monitor *mon, const QDict *qdict, QObject **ret);
-void object_add(const char *type, const char *id, const QDict *qdict,
-                Visitor *v, Error **errp);
-
 AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
                                 bool has_opaque, const char *opaque,
                                 Error **errp);
@@ -99,4 +52,4 @@
 void monitor_fdset_dup_fd_remove(int dup_fd);
 int monitor_fdset_dup_fd_find(int dup_fd);
 
-#endif /* !MONITOR_H */
+#endif /* MONITOR_H */
diff --git a/include/monitor/qdev.h b/include/monitor/qdev.h
index 8d16e11..8e504bc 100644
--- a/include/monitor/qdev.h
+++ b/include/monitor/qdev.h
@@ -1,15 +1,16 @@
-#ifndef QEMU_QDEV_MONITOR_H
-#define QEMU_QDEV_MONITOR_H
+#ifndef MONITOR_QDEV_H
+#define MONITOR_QDEV_H
 
 #include "hw/qdev-core.h"
-#include "monitor/monitor.h"
 
 /*** monitor commands ***/
 
-void do_info_qtree(Monitor *mon, const QDict *qdict);
-void do_info_qdm(Monitor *mon, const QDict *qdict);
-int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data);
+void hmp_info_qtree(Monitor *mon, const QDict *qdict);
+void hmp_info_qdm(Monitor *mon, const QDict *qdict);
+void hmp_info_qom_tree(Monitor *mon, const QDict *dict);
+void qmp_device_add(QDict *qdict, QObject **ret_data, Error **errp);
+
 int qdev_device_help(QemuOpts *opts);
-DeviceState *qdev_device_add(QemuOpts *opts);
+DeviceState *qdev_device_add(QemuOpts *opts, Error **errp);
 
 #endif
diff --git a/include/net/checksum.h b/include/net/checksum.h
index 2d7a363..7df472c 100644
--- a/include/net/checksum.h
+++ b/include/net/checksum.h
@@ -18,7 +18,7 @@
 #ifndef QEMU_NET_CHECKSUM_H
 #define QEMU_NET_CHECKSUM_H
 
-#include <stdint.h>
+#include "qemu/bswap.h"
 struct iovec;
 
 uint32_t net_checksum_add_cont(int len, uint8_t *buf, int seq);
@@ -46,9 +46,55 @@
  * @iov_cnt: number of array elements
  * @iov_off: starting iov offset for checksumming
  * @size: length of data to be checksummed
+ * @csum_offset: offset of the checksum chunk
  */
 uint32_t net_checksum_add_iov(const struct iovec *iov,
                               const unsigned int iov_cnt,
-                              uint32_t iov_off, uint32_t size);
+                              uint32_t iov_off, uint32_t size,
+                              uint32_t csum_offset);
+
+typedef struct toeplitz_key_st {
+    uint32_t leftmost_32_bits;
+    uint8_t *next_byte;
+} net_toeplitz_key;
+
+static inline
+void net_toeplitz_key_init(net_toeplitz_key *key, uint8_t *key_bytes)
+{
+    key->leftmost_32_bits = be32_to_cpu(*(uint32_t *)key_bytes);
+    key->next_byte = key_bytes + sizeof(uint32_t);
+}
+
+static inline
+void net_toeplitz_add(uint32_t *result,
+                      uint8_t *input,
+                      uint32_t len,
+                      net_toeplitz_key *key)
+{
+    register uint32_t accumulator = *result;
+    register uint32_t leftmost_32_bits = key->leftmost_32_bits;
+    register uint32_t byte;
+
+    for (byte = 0; byte < len; byte++) {
+        register uint8_t input_byte = input[byte];
+        register uint8_t key_byte = *(key->next_byte++);
+        register uint8_t bit;
+
+        for (bit = 0; bit < 8; bit++) {
+            if (input_byte & (1 << 7)) {
+                accumulator ^= leftmost_32_bits;
+            }
+
+            leftmost_32_bits =
+                (leftmost_32_bits << 1) | ((key_byte & (1 << 7)) >> 7);
+
+            input_byte <<= 1;
+            key_byte <<= 1;
+        }
+    }
+
+    key->leftmost_32_bits = leftmost_32_bits;
+    *result = accumulator;
+}
 
 #endif /* QEMU_NET_CHECKSUM_H */
diff --git a/include/net/eth.h b/include/net/eth.h
index b3273b8..2013175 100644
--- a/include/net/eth.h
+++ b/include/net/eth.h
@@ -26,12 +26,11 @@
 #ifndef QEMU_ETH_H
 #define QEMU_ETH_H
 
-#include <sys/types.h>
-#include <string.h>
 #include "qemu/bswap.h"
 #include "qemu/iov.h"
 
 #define ETH_ALEN 6
+#define ETH_HLEN 14
 
 struct eth_header {
     uint8_t  h_dest[ETH_ALEN];   /* destination eth addr */
@@ -68,6 +67,16 @@
     uint16_t th_urp;            /* urgent pointer */
 } tcp_header;
 
+#define TCP_FLAGS_ONLY(flags) ((flags) & 0x3f)
+
+#define TCP_HEADER_FLAGS(tcp) \
+    TCP_FLAGS_ONLY(be16_to_cpu((tcp)->th_offset_flags))
+
+#define TCP_FLAG_ACK  0x10
+
+#define TCP_HEADER_DATA_OFFSET(tcp) \
+    (((be16_to_cpu((tcp)->th_offset_flags) >> 12) & 0xf) << 2)
+
 typedef struct udp_header {
     uint16_t uh_sport; /* source port */
     uint16_t uh_dport; /* destination port */
@@ -109,11 +118,34 @@
     struct in6_address ip6_dst;    /* destination address */
 };
 
+typedef struct ip6_pseudo_header {
+    struct in6_address ip6_src;
+    struct in6_address ip6_dst;
+    uint32_t           len;
+    uint8_t            zero[3];
+    uint8_t            next_hdr;
+} ip6_pseudo_header;
+
 struct ip6_ext_hdr {
     uint8_t        ip6r_nxt;   /* next header */
     uint8_t        ip6r_len;   /* length in units of 8 octets */
 };
 
+struct ip6_ext_hdr_routing {
+    uint8_t     nxt;
+    uint8_t     len;
+    uint8_t     rtype;
+    uint8_t     segleft;
+    uint8_t     rsvd[4];
+};
+
+struct ip6_option_hdr {
+#define IP6_OPT_PAD1   (0x00)
+#define IP6_OPT_HOME   (0xC9)
+    uint8_t type;
+    uint8_t len;
+};
+
 struct udp_hdr {
   uint16_t uh_sport;           /* source port */
   uint16_t uh_dport;           /* destination port */
@@ -162,18 +194,22 @@
 #define PKT_GET_IP_HDR(p)         \
     ((struct ip_header *)(((uint8_t *)(p)) + eth_get_l2_hdr_length(p)))
 #define IP_HDR_GET_LEN(p)         \
-    ((((struct ip_header *)p)->ip_ver_len & 0x0F) << 2)
+    ((((struct ip_header *)(p))->ip_ver_len & 0x0F) << 2)
 #define PKT_GET_IP_HDR_LEN(p)     \
     (IP_HDR_GET_LEN(PKT_GET_IP_HDR(p)))
 #define PKT_GET_IP6_HDR(p)        \
     ((struct ip6_header *) (((uint8_t *)(p)) + eth_get_l2_hdr_length(p)))
 #define IP_HEADER_VERSION(ip)     \
-    ((ip->ip_ver_len >> 4)&0xf)
+    (((ip)->ip_ver_len >> 4) & 0xf)
+#define IP4_IS_FRAGMENT(ip) \
+    ((be16_to_cpu((ip)->ip_off) & (IP_OFFMASK | IP_MF)) != 0)
 
-#define ETH_P_IP                  (0x0800)
+#define ETH_P_IP                  (0x0800)      /* Internet Protocol packet  */
+#define ETH_P_ARP                 (0x0806)      /* Address Resolution packet */
 #define ETH_P_IPV6                (0x86dd)
 #define ETH_P_VLAN                (0x8100)
 #define ETH_P_DVLAN               (0x88a8)
+#define ETH_P_UNKNOWN             (0xffff)
 #define VLAN_VID_MASK             0x0fff
 #define IP_HEADER_VERSION_4       (4)
 #define IP_HEADER_VERSION_6       (6)
@@ -258,7 +294,7 @@
     case ETH_P_VLAN:
         return sizeof(struct eth_header) + sizeof(struct vlan_header);
     case ETH_P_DVLAN:
-        if (hvlan->h_proto == ETH_P_VLAN) {
+        if (be16_to_cpu(hvlan->h_proto) == ETH_P_VLAN) {
             return sizeof(struct eth_header) + 2 * sizeof(struct vlan_header);
         } else {
             return sizeof(struct eth_header) + sizeof(struct vlan_header);
@@ -268,6 +304,19 @@
     }
 }
 
+static inline uint32_t
+eth_get_l2_hdr_length_iov(const struct iovec *iov, int iovcnt)
+{
+    uint8_t p[sizeof(struct eth_header) + sizeof(struct vlan_header)];
+    size_t copied = iov_to_buf(iov, iovcnt, 0, p, ARRAY_SIZE(p));
+
+    if (copied < ARRAY_SIZE(p)) {
+        return copied;
+    }
+
+    return eth_get_l2_hdr_length(p);
+}
+
 static inline uint16_t
 eth_get_pkt_tci(const void *p)
 {
@@ -282,51 +331,67 @@
     }
 }
 
-static inline bool
-eth_strip_vlan(const void *p, uint8_t *new_ehdr_buf,
-               uint16_t *payload_offset, uint16_t *tci)
-{
-    uint16_t proto = be16_to_cpu(PKT_GET_ETH_HDR(p)->h_proto);
-    struct vlan_header *hvlan = PKT_GET_VLAN_HDR(p);
-    struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf;
+bool
+eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff,
+               uint8_t *new_ehdr_buf,
+               uint16_t *payload_offset, uint16_t *tci);
 
-    switch (proto) {
-    case ETH_P_VLAN:
-    case ETH_P_DVLAN:
-        memcpy(new_ehdr->h_source, PKT_GET_ETH_HDR(p)->h_source, ETH_ALEN);
-        memcpy(new_ehdr->h_dest, PKT_GET_ETH_HDR(p)->h_dest, ETH_ALEN);
-        new_ehdr->h_proto = hvlan->h_proto;
-        *tci = be16_to_cpu(hvlan->h_tci);
-        *payload_offset =
-            sizeof(struct eth_header) + sizeof(struct vlan_header);
-        if (be16_to_cpu(new_ehdr->h_proto) == ETH_P_VLAN) {
-            memcpy(PKT_GET_VLAN_HDR(new_ehdr),
-                   PKT_GET_DVLAN_HDR(p),
-                   sizeof(struct vlan_header));
-            *payload_offset += sizeof(struct vlan_header);
-        }
-        return true;
-    default:
-        return false;
-    }
+bool
+eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff,
+                  uint16_t vet, uint8_t *new_ehdr_buf,
+                  uint16_t *payload_offset, uint16_t *tci);
+
+uint16_t
+eth_get_l3_proto(const struct iovec *l2hdr_iov, int iovcnt, size_t l2hdr_len);
+
+void eth_setup_vlan_headers_ex(struct eth_header *ehdr, uint16_t vlan_tag,
+    uint16_t vlan_ethtype, bool *is_new);
+
+static inline void
+eth_setup_vlan_headers(struct eth_header *ehdr, uint16_t vlan_tag,
+    bool *is_new)
+{
+    eth_setup_vlan_headers_ex(ehdr, vlan_tag, ETH_P_VLAN, is_new);
 }
 
-static inline uint16_t
-eth_get_l3_proto(const void *l2hdr, size_t l2hdr_len)
-{
-    uint8_t *proto_ptr = (uint8_t *) l2hdr + l2hdr_len - sizeof(uint16_t);
-    return be16_to_cpup((uint16_t *)proto_ptr);
-}
-
-void eth_setup_vlan_headers(struct eth_header *ehdr, uint16_t vlan_tag,
-    bool *is_new);
 
 uint8_t eth_get_gso_type(uint16_t l3_proto, uint8_t *l3_hdr, uint8_t l4proto);
 
-void eth_get_protocols(const uint8_t *headers,
-                       uint32_t hdr_length,
+typedef struct eth_ip6_hdr_info_st {
+    uint8_t l4proto;
+    size_t  full_hdr_len;
+    struct  ip6_header ip6_hdr;
+    bool    has_ext_hdrs;
+    bool    rss_ex_src_valid;
+    struct  in6_address rss_ex_src;
+    bool    rss_ex_dst_valid;
+    struct  in6_address rss_ex_dst;
+    bool    fragment;
+} eth_ip6_hdr_info;
+
+typedef struct eth_ip4_hdr_info_st {
+    struct ip_header ip4_hdr;
+    bool   fragment;
+} eth_ip4_hdr_info;
+
+typedef struct eth_l4_hdr_info_st {
+    union {
+        struct tcp_header tcp;
+        struct udp_header udp;
+    } hdr;
+
+    bool has_tcp_data;
+} eth_l4_hdr_info;
+
+void eth_get_protocols(const struct iovec *iov, int iovcnt,
                        bool *isip4, bool *isip6,
-                       bool *isudp, bool *istcp);
+                       bool *isudp, bool *istcp,
+                       size_t *l3hdr_off,
+                       size_t *l4hdr_off,
+                       size_t *l5hdr_off,
+                       eth_ip6_hdr_info *ip6hdr_info,
+                       eth_ip4_hdr_info *ip4hdr_info,
+                       eth_l4_hdr_info  *l4hdr_info);
 
 void eth_setup_ip4_fragmentation(const void *l2hdr, size_t l2hdr_len,
                                  void *l3hdr, size_t l3hdr_len,
@@ -337,11 +402,18 @@
 eth_fix_ip4_checksum(void *l3hdr, size_t l3hdr_len);
 
 uint32_t
-eth_calc_pseudo_hdr_csum(struct ip_header *iphdr, uint16_t csl);
+eth_calc_ip4_pseudo_hdr_csum(struct ip_header *iphdr,
+                             uint16_t csl,
+                             uint32_t *cso);
+
+uint32_t
+eth_calc_ip6_pseudo_hdr_csum(struct ip6_header *iphdr,
+                             uint16_t csl,
+                             uint8_t l4_proto,
+                             uint32_t *cso);
 
 bool
-eth_parse_ipv6_hdr(struct iovec *pkt, int pkt_frags,
-                   size_t ip6hdr_off, uint8_t *l4proto,
-                   size_t *full_hdr_len);
+eth_parse_ipv6_hdr(const struct iovec *pkt, int pkt_frags,
+                   size_t ip6hdr_off, eth_ip6_hdr_info *info);
 
 #endif
diff --git a/include/net/filter.h b/include/net/filter.h
new file mode 100644
index 0000000..0c4a2ea
--- /dev/null
+++ b/include/net/filter.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_NET_FILTER_H
+#define QEMU_NET_FILTER_H
+
+#include "qom/object.h"
+#include "qemu-common.h"
+#include "net/queue.h"
+
+#define TYPE_NETFILTER "netfilter"
+#define NETFILTER(obj) \
+    OBJECT_CHECK(NetFilterState, (obj), TYPE_NETFILTER)
+#define NETFILTER_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(NetFilterClass, (obj), TYPE_NETFILTER)
+#define NETFILTER_CLASS(klass) \
+    OBJECT_CLASS_CHECK(NetFilterClass, (klass), TYPE_NETFILTER)
+
+typedef void (FilterSetup) (NetFilterState *nf, Error **errp);
+typedef void (FilterCleanup) (NetFilterState *nf);
+/*
+ * Return:
+ *   0: finished handling the packet, we should continue
+ *   size: filter stolen this packet, we stop pass this packet further
+ */
+typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
+                                   NetClientState *sender,
+                                   unsigned flags,
+                                   const struct iovec *iov,
+                                   int iovcnt,
+                                   NetPacketSent *sent_cb);
+
+typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
+
+typedef struct NetFilterClass {
+    ObjectClass parent_class;
+
+    /* optional */
+    FilterSetup *setup;
+    FilterCleanup *cleanup;
+    FilterStatusChanged *status_changed;
+    /* mandatory */
+    FilterReceiveIOV *receive_iov;
+} NetFilterClass;
+
+
+struct NetFilterState {
+    /* private */
+    Object parent;
+
+    /* protected */
+    char *netdev_id;
+    NetClientState *netdev;
+    NetFilterDirection direction;
+    bool on;
+    QTAILQ_ENTRY(NetFilterState) next;
+};
+
+ssize_t qemu_netfilter_receive(NetFilterState *nf,
+                               NetFilterDirection direction,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb);
+
+/* pass the packet to the next filter */
+ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
+                                    unsigned flags,
+                                    const struct iovec *iov,
+                                    int iovcnt,
+                                    void *opaque);
+
+#endif /* QEMU_NET_FILTER_H */
diff --git a/include/net/net.h b/include/net/net.h
index 008d610..e8d9e9e 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -9,6 +9,11 @@
 #include "migration/vmstate.h"
 #include "qapi-types.h"
 
+#define MAC_FMT "%02X:%02X:%02X:%02X:%02X:%02X"
+#define MAC_ARG(x) ((uint8_t *)(x))[0], ((uint8_t *)(x))[1], \
+                   ((uint8_t *)(x))[2], ((uint8_t *)(x))[3], \
+                   ((uint8_t *)(x))[4], ((uint8_t *)(x))[5]
+
 #define MAX_QUEUE_NUM 1024
 
 /* Maximum GSO packet size (64k) plus plenty of room for
@@ -55,9 +60,13 @@
 typedef void (UsingVnetHdr)(NetClientState *, bool);
 typedef void (SetOffload)(NetClientState *, int, int, int, int, int);
 typedef void (SetVnetHdrLen)(NetClientState *, int);
+typedef int (SetVnetLE)(NetClientState *, bool);
+typedef int (SetVnetBE)(NetClientState *, bool);
+typedef struct SocketReadState SocketReadState;
+typedef void (SocketReadStateFinalize)(SocketReadState *rs);
 
 typedef struct NetClientInfo {
-    NetClientOptionsKind type;
+    NetClientDriver type;
     size_t size;
     NetReceive *receive;
     NetReceive *receive_raw;
@@ -73,6 +82,8 @@
     UsingVnetHdr *using_vnet_hdr;
     SetOffload *set_offload;
     SetVnetHdrLen *set_vnet_hdr_len;
+    SetVnetLE *set_vnet_le;
+    SetVnetBE *set_vnet_be;
 } NetClientInfo;
 
 struct NetClientState {
@@ -88,6 +99,8 @@
     NetClientDestructor *destructor;
     unsigned int queue_index;
     unsigned rxfilter_notify_enabled:1;
+    int vring_enable;
+    QTAILQ_HEAD(NetFilterHead, NetFilterState) filters;
 };
 
 typedef struct NICState {
@@ -97,9 +110,19 @@
     bool peer_deleted;
 } NICState;
 
+struct SocketReadState {
+    int state; /* 0 = getting length, 1 = getting data */
+    uint32_t index;
+    uint32_t packet_len;
+    uint8_t buf[NET_BUFSIZE];
+    SocketReadStateFinalize *finalize;
+};
+
+int net_fill_rstate(SocketReadState *rs, const uint8_t *buf, int size);
+char *qemu_mac_strdup_printf(const uint8_t *macaddr);
 NetClientState *qemu_find_netdev(const char *id);
 int qemu_find_net_clients_except(const char *id, NetClientState **ncs,
-                                 NetClientOptionsKind type, int max);
+                                 NetClientDriver type, int max);
 NetClientState *qemu_new_net_client(NetClientInfo *info,
                                     NetClientState *peer,
                                     const char *model,
@@ -138,17 +161,14 @@
 void qemu_set_offload(NetClientState *nc, int csum, int tso4, int tso6,
                       int ecn, int ufo);
 void qemu_set_vnet_hdr_len(NetClientState *nc, int len);
+int qemu_set_vnet_le(NetClientState *nc, bool is_le);
+int qemu_set_vnet_be(NetClientState *nc, bool is_be);
 void qemu_macaddr_default_if_unset(MACAddr *macaddr);
 int qemu_show_nic_models(const char *arg, const char *const *models);
 void qemu_check_nic_model(NICInfo *nd, const char *model);
 int qemu_find_nic_model(NICInfo *nd, const char * const *models,
                         const char *default_model);
 
-ssize_t qemu_deliver_packet(NetClientState *sender,
-                            unsigned flags,
-                            const uint8_t *data,
-                            size_t size,
-                            void *opaque);
 ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                             unsigned flags,
                             const struct iovec *iov,
@@ -156,7 +176,9 @@
                             void *opaque);
 
 void print_net_client(Monitor *mon, NetClientState *nc);
-void do_info_network(Monitor *mon, const QDict *qdict);
+void hmp_info_network(Monitor *mon, const QDict *qdict);
+void net_socket_rs_init(SocketReadState *rs,
+                        SocketReadStateFinalize *finalize);
 
 /* NIC info */
 
@@ -175,22 +197,21 @@
 
 extern int nb_nics;
 extern NICInfo nd_table[MAX_NICS];
-extern int default_net;
 extern const char *host_net_devices[];
 
 /* from net.c */
 extern const char *legacy_tftp_prefix;
 extern const char *legacy_bootp_filename;
 
-int net_client_init(QemuOpts *opts, int is_netdev, Error **errp);
+int net_client_init(QemuOpts *opts, bool is_netdev, Error **errp);
 int net_client_parse(QemuOptsList *opts_list, const char *str);
 int net_init_clients(void);
 void net_check_clients(void);
 void net_cleanup(void);
-void net_host_device_add(Monitor *mon, const QDict *qdict);
-void net_host_device_remove(Monitor *mon, const QDict *qdict);
+void hmp_host_net_add(Monitor *mon, const QDict *qdict);
+void hmp_host_net_remove(Monitor *mon, const QDict *qdict);
 void netdev_add(QemuOpts *opts, Error **errp);
-int qmp_netdev_add(Monitor *mon, const QDict *qdict, QObject **ret);
+void qmp_netdev_add(QDict *qdict, QObject **ret, Error **errp);
 
 int net_hub_id_for_client(NetClientState *nc, int *id);
 NetClientState *net_hub_port_find(int hub_id);
diff --git a/include/net/queue.h b/include/net/queue.h
index fc02b33..5469fdb 100644
--- a/include/net/queue.h
+++ b/include/net/queue.h
@@ -34,7 +34,25 @@
 #define QEMU_NET_PACKET_FLAG_NONE  0
 #define QEMU_NET_PACKET_FLAG_RAW  (1<<0)
 
-NetQueue *qemu_new_net_queue(void *opaque);
+/* Returns:
+ *   >0 - success
+ *    0 - queue packet for future redelivery
+ *   <0 - failure (discard packet)
+ */
+typedef ssize_t (NetQueueDeliverFunc)(NetClientState *sender,
+                                      unsigned flags,
+                                      const struct iovec *iov,
+                                      int iovcnt,
+                                      void *opaque);
+
+NetQueue *qemu_new_net_queue(NetQueueDeliverFunc *deliver, void *opaque);
+
+void qemu_net_queue_append_iov(NetQueue *queue,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb);
 
 void qemu_del_net_queue(NetQueue *queue);
 
diff --git a/include/net/slirp.h b/include/net/slirp.h
index eb12e6b..8999201 100644
--- a/include/net/slirp.h
+++ b/include/net/slirp.h
@@ -31,8 +31,8 @@
 
 #ifdef CONFIG_SLIRP
 
-void net_slirp_hostfwd_add(Monitor *mon, const QDict *qdict);
-void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict);
+void hmp_hostfwd_add(Monitor *mon, const QDict *qdict);
+void hmp_hostfwd_remove(Monitor *mon, const QDict *qdict);
 
 int net_slirp_redir(const char *redir_str);
 
@@ -40,14 +40,22 @@
 
 int net_slirp_smb(const char *exported_dir);
 
-void do_info_usernet(Monitor *mon, const QDict *qdict);
+void hmp_info_usernet(Monitor *mon, const QDict *qdict);
 
-/* Given a VLAN ID and stack name string, look up the corresponding
- * Slirp backend.
- */
-Slirp *net_slirp_lookup(const char *vlan, const char *stack, Error **errp);
+/* Functions to be called by |out_send| and |in_send| implementations below */
+/* |opaque| is the result of net_slirp_set_shapers(). */
+void net_slirp_output_raw(void *opaque, const uint8_t *pkt, int pkt_len);
+void net_slirp_receive_raw(void* opaque, const uint8_t *buf, size_t size);
 
-int net_slirp_is_inited(void);
+/* Return a Slirp instance, or NULL if the network stack is not initialized */
+void* net_slirp_state(void);
+
+typedef void (*SlirpShaperSendFunc)(void* opaque, const void* data, int len);
+
+void* net_slirp_set_shapers(void* out_opaque,
+                            SlirpShaperSendFunc out_send,
+                            void* in_opaque,
+                            SlirpShaperSendFunc in_send);
 
 #endif
 
diff --git a/include/net/tap.h b/include/net/tap.h
index 6daeb42..5da4edc 100644
--- a/include/net/tap.h
+++ b/include/net/tap.h
@@ -28,6 +28,7 @@
 
 #include "qemu-common.h"
 #include "qapi-types.h"
+#include "standard-headers/linux/virtio_net.h"
 
 int tap_enable(NetClientState *nc);
 int tap_disable(NetClientState *nc);
@@ -37,27 +38,4 @@
 struct vhost_net;
 struct vhost_net *tap_get_vhost_net(NetClientState *nc);
 
-struct virtio_net_hdr
-{
-#define VIRTIO_NET_HDR_F_NEEDS_CSUM     1       // Use csum_start, csum_offset
-#define VIRTIO_NET_HDR_F_DATA_VALID    2       // Csum is valid
-    uint8_t flags;
-#define VIRTIO_NET_HDR_GSO_NONE         0       // Not a GSO frame
-#define VIRTIO_NET_HDR_GSO_TCPV4        1       // GSO frame, IPv4 TCP (TSO)
-#define VIRTIO_NET_HDR_GSO_UDP          3       // GSO frame, IPv4 UDP (UFO)
-#define VIRTIO_NET_HDR_GSO_TCPV6        4       // GSO frame, IPv6 TCP
-#define VIRTIO_NET_HDR_GSO_ECN          0x80    // TCP has ECN set
-    uint8_t gso_type;
-    uint16_t hdr_len;
-    uint16_t gso_size;
-    uint16_t csum_start;
-    uint16_t csum_offset;
-};
-
-struct virtio_net_hdr_mrg_rxbuf
-{
-    struct virtio_net_hdr hdr;
-    uint16_t num_buffers;   /* Number of merged rx buffers */
-};
-
 #endif /* QEMU_NET_TAP_H */
diff --git a/include/net/vhost-user.h b/include/net/vhost-user.h
index 85109f6..5bcd8a6 100644
--- a/include/net/vhost-user.h
+++ b/include/net/vhost-user.h
@@ -8,10 +8,11 @@
  *
  */
 
-#ifndef VHOST_USER_H_
-#define VHOST_USER_H_
+#ifndef VHOST_USER_H
+#define VHOST_USER_H
 
 struct vhost_net;
 struct vhost_net *vhost_user_get_vhost_net(NetClientState *nc);
+uint64_t vhost_user_get_acked_features(NetClientState *nc);
 
-#endif /* VHOST_USER_H_ */
+#endif /* VHOST_USER_H */
diff --git a/include/net/vhost_net.h b/include/net/vhost_net.h
index b1c18a3..5a08eff 100644
--- a/include/net/vhost_net.h
+++ b/include/net/vhost_net.h
@@ -10,23 +10,29 @@
 typedef struct VhostNetOptions {
     VhostBackendType backend_type;
     NetClientState *net_backend;
+    uint32_t busyloop_timeout;
     void *opaque;
-    bool force;
 } VhostNetOptions;
 
+uint64_t vhost_net_get_max_queues(VHostNetState *net);
 struct vhost_net *vhost_net_init(VhostNetOptions *options);
 
-bool vhost_net_query(VHostNetState *net, VirtIODevice *dev);
 int vhost_net_start(VirtIODevice *dev, NetClientState *ncs, int total_queues);
 void vhost_net_stop(VirtIODevice *dev, NetClientState *ncs, int total_queues);
 
 void vhost_net_cleanup(VHostNetState *net);
 
-unsigned vhost_net_get_features(VHostNetState *net, unsigned features);
-void vhost_net_ack_features(VHostNetState *net, unsigned features);
+uint64_t vhost_net_get_features(VHostNetState *net, uint64_t features);
+void vhost_net_ack_features(VHostNetState *net, uint64_t features);
 
 bool vhost_net_virtqueue_pending(VHostNetState *net, int n);
 void vhost_net_virtqueue_mask(VHostNetState *net, VirtIODevice *dev,
                               int idx, bool mask);
+int vhost_net_notify_migration_done(VHostNetState *net, char* mac_addr);
 VHostNetState *get_vhost_net(NetClientState *nc);
+
+int vhost_set_vring_enable(NetClientState * nc, int enable);
+
+uint64_t vhost_net_get_acked_features(VHostNetState *net);
+
 #endif
diff --git a/include/qapi/clone-visitor.h b/include/qapi/clone-visitor.h
new file mode 100644
index 0000000..b16177e
--- /dev/null
+++ b/include/qapi/clone-visitor.h
@@ -0,0 +1,39 @@
+/*
+ * Clone Visitor
+ *
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QAPI_CLONE_VISITOR_H
+#define QAPI_CLONE_VISITOR_H
+
+#include "qemu/typedefs.h"
+#include "qapi/visitor.h"
+#include "qapi-visit.h"
+
+/*
+ * The clone visitor is for direct use only by the QAPI_CLONE() macro;
+ * it requires that the root visit occur on an object, list, or
+ * alternate, and is not usable directly on built-in QAPI types.
+ */
+typedef struct QapiCloneVisitor QapiCloneVisitor;
+
+void *qapi_clone(const void *src, void (*visit_type)(Visitor *, const char *,
+                                                     void **, Error **));
+
+/*
+ * Deep-clone QAPI object @src of the given @type, and return the result.
+ *
+ * Not usable on QAPI scalars (integers, strings, enums), nor on a
+ * QAPI object that references the 'any' type.  Safe when @src is NULL.
+ */
+#define QAPI_CLONE(type, src)                                           \
+    ((type *)qapi_clone(src,                                            \
+                        (void (*)(Visitor *, const char *, void**,      \
+                                  Error **))visit_type_ ## type))
+
+#endif
diff --git a/include/qapi/dealloc-visitor.h b/include/qapi/dealloc-visitor.h
index cf4c36d..b3e5c85 100644
--- a/include/qapi/dealloc-visitor.h
+++ b/include/qapi/dealloc-visitor.h
@@ -18,9 +18,11 @@
 
 typedef struct QapiDeallocVisitor QapiDeallocVisitor;
 
-QapiDeallocVisitor *qapi_dealloc_visitor_new(void);
-void qapi_dealloc_visitor_cleanup(QapiDeallocVisitor *d);
-
-Visitor *qapi_dealloc_get_visitor(QapiDeallocVisitor *v);
+/*
+ * The dealloc visitor is primarly used only by generated
+ * qapi_free_FOO() functions, and is the only visitor designed to work
+ * correctly in the face of a partially-constructed QAPI tree.
+ */
+Visitor *qapi_dealloc_visitor_new(void);
 
 #endif
diff --git a/include/qapi/error.h b/include/qapi/error.h
index 822ad56..42aedec 100644
--- a/include/qapi/error.h
+++ b/include/qapi/error.h
@@ -2,105 +2,307 @@
  * QEMU Error Objects
  *
  * Copyright IBM, Corp. 2011
+ * Copyright (C) 2011-2015 Red Hat, Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Markus Armbruster <armbru@redhat.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.  See
  * the COPYING.LIB file in the top-level directory.
  */
+
+/*
+ * Error reporting system loosely patterned after Glib's GError.
+ *
+ * Create an error:
+ *     error_setg(&err, "situation normal, all fouled up");
+ *
+ * Create an error and add additional explanation:
+ *     error_setg(&err, "invalid quark");
+ *     error_append_hint(&err, "Valid quarks are up, down, strange, "
+ *                       "charm, top, bottom.\n");
+ *
+ * Do *not* contract this to
+ *     error_setg(&err, "invalid quark\n"
+ *                "Valid quarks are up, down, strange, charm, top, bottom.");
+ *
+ * Report an error to the current monitor if we have one, else stderr:
+ *     error_report_err(err);
+ * This frees the error object.
+ *
+ * Likewise, but with additional text prepended:
+ *     error_reportf_err(err, "Could not frobnicate '%s': ", name);
+ *
+ * Report an error somewhere else:
+ *     const char *msg = error_get_pretty(err);
+ *     do with msg what needs to be done...
+ *     error_free(err);
+ * Note that this loses hints added with error_append_hint().
+ *
+ * Handle an error without reporting it (just for completeness):
+ *     error_free(err);
+ *
+ * Assert that an expected error occurred, but clean it up without
+ * reporting it (primarily useful in testsuites):
+ *     error_free_or_abort(&err);
+ *
+ * Pass an existing error to the caller:
+ *     error_propagate(errp, err);
+ * where Error **errp is a parameter, by convention the last one.
+ *
+ * Pass an existing error to the caller with the message modified:
+ *     error_propagate(errp, err);
+ *     error_prepend(errp, "Could not frobnicate '%s': ", name);
+ *
+ * Create a new error and pass it to the caller:
+ *     error_setg(errp, "situation normal, all fouled up");
+ *
+ * Call a function and receive an error from it:
+ *     Error *err = NULL;
+ *     foo(arg, &err);
+ *     if (err) {
+ *         handle the error...
+ *     }
+ *
+ * Call a function ignoring errors:
+ *     foo(arg, NULL);
+ *
+ * Call a function aborting on errors:
+ *     foo(arg, &error_abort);
+ *
+ * Call a function treating errors as fatal:
+ *     foo(arg, &error_fatal);
+ *
+ * Receive an error and pass it on to the caller:
+ *     Error *err = NULL;
+ *     foo(arg, &err);
+ *     if (err) {
+ *         handle the error...
+ *         error_propagate(errp, err);
+ *     }
+ * where Error **errp is a parameter, by convention the last one.
+ *
+ * Do *not* "optimize" this to
+ *     foo(arg, errp);
+ *     if (*errp) { // WRONG!
+ *         handle the error...
+ *     }
+ * because errp may be NULL!
+ *
+ * But when all you do with the error is pass it on, please use
+ *     foo(arg, errp);
+ * for readability.
+ *
+ * Receive and accumulate multiple errors (first one wins):
+ *     Error *err = NULL, *local_err = NULL;
+ *     foo(arg, &err);
+ *     bar(arg, &local_err);
+ *     error_propagate(&err, local_err);
+ *     if (err) {
+ *         handle the error...
+ *     }
+ *
+ * Do *not* "optimize" this to
+ *     foo(arg, &err);
+ *     bar(arg, &err); // WRONG!
+ *     if (err) {
+ *         handle the error...
+ *     }
+ * because this may pass a non-null err to bar().
+ */
+
 #ifndef ERROR_H
 #define ERROR_H
 
-#include "qemu/compiler.h"
 #include "qapi-types.h"
-#include <stdbool.h>
-
-/**
- * A class representing internal errors within QEMU.  An error has a ErrorClass
- * code and a human message.
- */
-typedef struct Error Error;
-
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message.  This function is not meant to be used outside
- * of QEMU.
- */
-void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
-    GCC_FMT_ATTR(3, 4);
-
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message, followed by a strerror() string if
- * @os_error is not zero.
- */
-void error_set_errno(Error **errp, int os_error, ErrorClass err_class,
-                     const char *fmt, ...) GCC_FMT_ATTR(4, 5);
-
-#ifdef _WIN32
-/**
- * Set an indirect pointer to an error given a ErrorClass value and a
- * printf-style human message, followed by a g_win32_error_message() string if
- * @win32_err is not zero.
- */
-void error_set_win32(Error **errp, int win32_err, ErrorClass err_class,
-                     const char *fmt, ...) GCC_FMT_ATTR(4, 5);
-#endif
-
-/**
- * Same as error_set(), but sets a generic error
- */
-#define error_setg(errp, fmt, ...) \
-    error_set(errp, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
-#define error_setg_errno(errp, os_error, fmt, ...) \
-    error_set_errno(errp, os_error, ERROR_CLASS_GENERIC_ERROR, \
-                    fmt, ## __VA_ARGS__)
-#ifdef _WIN32
-#define error_setg_win32(errp, win32_err, fmt, ...) \
-    error_set_win32(errp, win32_err, ERROR_CLASS_GENERIC_ERROR, \
-                    fmt, ## __VA_ARGS__)
-#endif
-
-/**
- * Helper for open() errors
- */
-void error_setg_file_open(Error **errp, int os_errno, const char *filename);
 
 /*
- * Get the error class of an error object.
+ * Overall category of an error.
+ * Based on the qapi type QapiErrorClass, but reproduced here for nicer
+ * enum names.
+ */
+typedef enum ErrorClass {
+    ERROR_CLASS_GENERIC_ERROR = QAPI_ERROR_CLASS_GENERICERROR,
+    ERROR_CLASS_COMMAND_NOT_FOUND = QAPI_ERROR_CLASS_COMMANDNOTFOUND,
+    ERROR_CLASS_DEVICE_ENCRYPTED = QAPI_ERROR_CLASS_DEVICEENCRYPTED,
+    ERROR_CLASS_DEVICE_NOT_ACTIVE = QAPI_ERROR_CLASS_DEVICENOTACTIVE,
+    ERROR_CLASS_DEVICE_NOT_FOUND = QAPI_ERROR_CLASS_DEVICENOTFOUND,
+    ERROR_CLASS_KVM_MISSING_CAP = QAPI_ERROR_CLASS_KVMMISSINGCAP,
+} ErrorClass;
+
+/*
+ * Get @err's human-readable error message.
+ */
+const char *error_get_pretty(const Error *err);
+
+/*
+ * Get @err's error class.
+ * Note: use of error classes other than ERROR_CLASS_GENERIC_ERROR is
+ * strongly discouraged.
  */
 ErrorClass error_get_class(const Error *err);
 
-/**
- * Returns an exact copy of the error passed as an argument.
+/*
+ * Create a new error object and assign it to *@errp.
+ * If @errp is NULL, the error is ignored.  Don't bother creating one
+ * then.
+ * If @errp is &error_abort, print a suitable message and abort().
+ * If @errp is &error_fatal, print a suitable message and exit(1).
+ * If @errp is anything else, *@errp must be NULL.
+ * The new error's class is ERROR_CLASS_GENERIC_ERROR, and its
+ * human-readable error message is made from printf-style @fmt, ...
+ * The resulting message should be a single phrase, with no newline or
+ * trailing punctuation.
+ * Please don't error_setg(&error_fatal, ...), use error_report() and
+ * exit(), because that's more obvious.
+ * Likewise, don't error_setg(&error_abort, ...), use assert().
  */
-Error *error_copy(const Error *err);
+#define error_setg(errp, fmt, ...)                              \
+    error_setg_internal((errp), __FILE__, __LINE__, __func__,   \
+                        (fmt), ## __VA_ARGS__)
+void error_setg_internal(Error **errp,
+                         const char *src, int line, const char *func,
+                         const char *fmt, ...)
+    GCC_FMT_ATTR(5, 6);
 
-/**
- * Get a human readable representation of an error object.
+/*
+ * Just like error_setg(), with @os_error info added to the message.
+ * If @os_error is non-zero, ": " + strerror(os_error) is appended to
+ * the human-readable error message.
  */
-const char *error_get_pretty(Error *err);
+#define error_setg_errno(errp, os_error, fmt, ...)                      \
+    error_setg_errno_internal((errp), __FILE__, __LINE__, __func__,     \
+                              (os_error), (fmt), ## __VA_ARGS__)
+void error_setg_errno_internal(Error **errp,
+                               const char *fname, int line, const char *func,
+                               int os_error, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 
-/**
- * Propagate an error to an indirect pointer to an error.  This function will
- * always transfer ownership of the error reference and handles the case where
- * dst_err is NULL correctly.  Errors after the first are discarded.
+#ifdef _WIN32
+/*
+ * Just like error_setg(), with @win32_error info added to the message.
+ * If @win32_error is non-zero, ": " + g_win32_error_message(win32_err)
+ * is appended to the human-readable error message.
+ */
+#define error_setg_win32(errp, win32_err, fmt, ...)                     \
+    error_setg_win32_internal((errp), __FILE__, __LINE__, __func__,     \
+                              (win32_err), (fmt), ## __VA_ARGS__)
+void error_setg_win32_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int win32_err, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
+#endif
+
+/*
+ * Propagate error object (if any) from @local_err to @dst_errp.
+ * If @local_err is NULL, do nothing (because there's nothing to
+ * propagate).
+ * Else, if @dst_errp is NULL, errors are being ignored.  Free the
+ * error object.
+ * Else, if @dst_errp is &error_abort, print a suitable message and
+ * abort().
+ * Else, if @dst_errp is &error_fatal, print a suitable message and
+ * exit(1).
+ * Else, if @dst_errp already contains an error, ignore this one: free
+ * the error object.
+ * Else, move the error object from @local_err to *@dst_errp.
+ * On return, @local_err is invalid.
+ * Please don't error_propagate(&error_fatal, ...), use
+ * error_report_err() and exit(), because that's more obvious.
  */
 void error_propagate(Error **dst_errp, Error *local_err);
 
-/**
- * Free an error object.
+/*
+ * Prepend some text to @errp's human-readable error message.
+ * The text is made by formatting @fmt, @ap like vprintf().
+ */
+void error_vprepend(Error **errp, const char *fmt, va_list ap);
+
+/*
+ * Prepend some text to @errp's human-readable error message.
+ * The text is made by formatting @fmt, ... like printf().
+ */
+void error_prepend(Error **errp, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
+
+/*
+ * Append a printf-style human-readable explanation to an existing error.
+ * @errp may be NULL, but not &error_fatal or &error_abort.
+ * Trivially the case if you call it only after error_setg() or
+ * error_propagate().
+ * May be called multiple times.  The resulting hint should end with a
+ * newline.
+ */
+void error_append_hint(Error **errp, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
+
+/*
+ * Convenience function to report open() failure.
+ */
+#define error_setg_file_open(errp, os_errno, filename)                  \
+    error_setg_file_open_internal((errp), __FILE__, __LINE__, __func__, \
+                                  (os_errno), (filename))
+void error_setg_file_open_internal(Error **errp,
+                                   const char *src, int line, const char *func,
+                                   int os_errno, const char *filename);
+
+/*
+ * Return an exact copy of @err.
+ */
+Error *error_copy(const Error *err);
+
+/*
+ * Free @err.
+ * @err may be NULL.
  */
 void error_free(Error *err);
 
-/**
- * If passed to error_set and friends, abort().
+/*
+ * Convenience function to assert that *@errp is set, then silently free it.
  */
+void error_free_or_abort(Error **errp);
 
+/*
+ * Convenience function to error_report() and free @err.
+ */
+void error_report_err(Error *err);
+
+/*
+ * Convenience function to error_prepend(), error_report() and free @err.
+ */
+void error_reportf_err(Error *err, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
+
+/*
+ * Just like error_setg(), except you get to specify the error class.
+ * Note: use of error classes other than ERROR_CLASS_GENERIC_ERROR is
+ * strongly discouraged.
+ */
+#define error_set(errp, err_class, fmt, ...)                    \
+    error_set_internal((errp), __FILE__, __LINE__, __func__,    \
+                       (err_class), (fmt), ## __VA_ARGS__)
+void error_set_internal(Error **errp,
+                        const char *src, int line, const char *func,
+                        ErrorClass err_class, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
+
+/*
+ * Special error destination to abort on error.
+ * See error_setg() and error_propagate() for details.
+ */
 extern Error *error_abort;
-#ifdef CONFIG_ANDROID
-extern Error *android_init_error_with_message;
-#endif  // CONFIG_ANDROID
+
+/*
+ * Special error destination to exit(1) on error.
+ * See error_setg() and error_propagate() for details.
+ */
+extern Error *error_fatal;
+
+/*
+ * Special error destination for errors that happen during initialization,
+ * inside functions that do not take an Error** parameter to report them.
+ */
+extern Error *error_during_init;
 
 #endif
diff --git a/include/qapi/opts-visitor.h b/include/qapi/opts-visitor.h
index fd48c14..6462c96 100644
--- a/include/qapi/opts-visitor.h
+++ b/include/qapi/opts-visitor.h
@@ -29,9 +29,12 @@
  * - string representations of negative numbers yield negative values,
  * - values below INT64_MIN or LLONG_MIN are rejected,
  * - values above INT64_MAX or LLONG_MAX are rejected.
+ *
+ * The Opts input visitor does not implement support for visiting QAPI
+ * alternates, numbers (other than integers), null, or arbitrary
+ * QTypes.  It also requires a non-null list argument to
+ * visit_start_list().
  */
-OptsVisitor *opts_visitor_new(const QemuOpts *opts);
-void opts_visitor_cleanup(OptsVisitor *nv);
-Visitor *opts_get_visitor(OptsVisitor *nv);
+Visitor *opts_visitor_new(const QemuOpts *opts);
 
 #endif
diff --git a/include/qapi/qmp-event.h b/include/qapi/qmp-event.h
index 8a8ffb5..40fe3cb 100644
--- a/include/qapi/qmp-event.h
+++ b/include/qapi/qmp-event.h
@@ -14,7 +14,6 @@
 #ifndef QMP_EVENT_H
 #define QMP_EVENT_H
 
-#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 
 typedef void (*QMPEventFuncEmit)(unsigned event, QDict *dict, Error **errp);
diff --git a/include/qapi/qmp-input-visitor.h b/include/qapi/qmp-input-visitor.h
index 3ed499c..f3ff5f3 100644
--- a/include/qapi/qmp-input-visitor.h
+++ b/include/qapi/qmp-input-visitor.h
@@ -19,11 +19,12 @@
 
 typedef struct QmpInputVisitor QmpInputVisitor;
 
-QmpInputVisitor *qmp_input_visitor_new(QObject *obj);
-QmpInputVisitor *qmp_input_visitor_new_strict(QObject *obj);
-
-void qmp_input_visitor_cleanup(QmpInputVisitor *v);
-
-Visitor *qmp_input_get_visitor(QmpInputVisitor *v);
+/*
+ * Return a new input visitor that converts QMP to QAPI.
+ *
+ * Set @strict to reject a parse that doesn't consume all keys of a
+ * dictionary; otherwise excess input is ignored.
+ */
+Visitor *qmp_input_visitor_new(QObject *obj, bool strict);
 
 #endif
diff --git a/include/qapi/qmp-output-visitor.h b/include/qapi/qmp-output-visitor.h
index 2266770..040fdda 100644
--- a/include/qapi/qmp-output-visitor.h
+++ b/include/qapi/qmp-output-visitor.h
@@ -19,10 +19,12 @@
 
 typedef struct QmpOutputVisitor QmpOutputVisitor;
 
-QmpOutputVisitor *qmp_output_visitor_new(void);
-void qmp_output_visitor_cleanup(QmpOutputVisitor *v);
-
-QObject *qmp_output_get_qobject(QmpOutputVisitor *v);
-Visitor *qmp_output_get_visitor(QmpOutputVisitor *v);
+/*
+ * Create a new QMP output visitor.
+ *
+ * If everything else succeeds, pass @result to visit_complete() to
+ * collect the result of the visit.
+ */
+Visitor *qmp_output_visitor_new(QObject **result);
 
 #endif
diff --git a/include/qapi/qmp/dispatch.h b/include/qapi/qmp/dispatch.h
index e389697..48c11b6 100644
--- a/include/qapi/qmp/dispatch.h
+++ b/include/qapi/qmp/dispatch.h
@@ -11,20 +11,14 @@
  *
  */
 
-#ifndef QMP_CORE_H
-#define QMP_CORE_H
+#ifndef QAPI_QMP_DISPATCH_H
+#define QAPI_QMP_DISPATCH_H
 
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qdict.h"
-#include "qapi/error.h"
 
 typedef void (QmpCommandFunc)(QDict *, QObject **, Error **);
 
-typedef enum QmpCommandType
-{
-    QCT_NORMAL,
-} QmpCommandType;
-
 typedef enum QmpCommandOptions
 {
     QCO_NO_OPTIONS = 0x0,
@@ -34,7 +28,6 @@
 typedef struct QmpCommand
 {
     const char *name;
-    QmpCommandType type;
     QmpCommandFunc *fn;
     QmpCommandOptions options;
     QTAILQ_ENTRY(QmpCommand) node;
@@ -55,4 +48,3 @@
 void qmp_for_each_command(qmp_cmd_callback_fn fn, void *opaque);
 
 #endif
-
diff --git a/include/qapi/qmp/json-lexer.h b/include/qapi/qmp/json-lexer.h
index cdff046..afee782 100644
--- a/include/qapi/qmp/json-lexer.h
+++ b/include/qapi/qmp/json-lexer.h
@@ -14,11 +14,15 @@
 #ifndef QEMU_JSON_LEXER_H
 #define QEMU_JSON_LEXER_H
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qlist.h"
 
 typedef enum json_token_type {
-    JSON_OPERATOR = 100,
+    JSON_MIN = 100,
+    JSON_LCURLY = JSON_MIN,
+    JSON_RCURLY,
+    JSON_LSQUARE,
+    JSON_RSQUARE,
+    JSON_COLON,
+    JSON_COMMA,
     JSON_INTEGER,
     JSON_FLOAT,
     JSON_KEYWORD,
@@ -30,13 +34,14 @@
 
 typedef struct JSONLexer JSONLexer;
 
-typedef void (JSONLexerEmitter)(JSONLexer *, QString *, JSONTokenType, int x, int y);
+typedef void (JSONLexerEmitter)(JSONLexer *, GString *,
+                                JSONTokenType, int x, int y);
 
 struct JSONLexer
 {
     JSONLexerEmitter *emit;
     int state;
-    QString *token;
+    GString *token;
     int x, y;
 };
 
diff --git a/include/qapi/qmp/json-parser.h b/include/qapi/qmp/json-parser.h
index 44d88f3..9987f8c 100644
--- a/include/qapi/qmp/json-parser.h
+++ b/include/qapi/qmp/json-parser.h
@@ -16,9 +16,8 @@
 
 #include "qemu-common.h"
 #include "qapi/qmp/qlist.h"
-#include "qapi/error.h"
 
-QObject *json_parser_parse(QList *tokens, va_list *ap);
-QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp);
+QObject *json_parser_parse(GQueue *tokens, va_list *ap);
+QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp);
 
 #endif
diff --git a/include/qapi/qmp/json-streamer.h b/include/qapi/qmp/json-streamer.h
index 823f7d7..00d8a23 100644
--- a/include/qapi/qmp/json-streamer.h
+++ b/include/qapi/qmp/json-streamer.h
@@ -14,21 +14,27 @@
 #ifndef QEMU_JSON_STREAMER_H
 #define QEMU_JSON_STREAMER_H
 
-#include "qapi/qmp/qlist.h"
 #include "qapi/qmp/json-lexer.h"
 
+typedef struct JSONToken {
+    int type;
+    int x;
+    int y;
+    char str[];
+} JSONToken;
+
 typedef struct JSONMessageParser
 {
-    void (*emit)(struct JSONMessageParser *parser, QList *tokens);
+    void (*emit)(struct JSONMessageParser *parser, GQueue *tokens);
     JSONLexer lexer;
     int brace_count;
     int bracket_count;
-    QList *tokens;
+    GQueue *tokens;
     uint64_t token_size;
 } JSONMessageParser;
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, QList *));
+                              void (*func)(JSONMessageParser *, GQueue *));
 
 int json_message_parser_feed(JSONMessageParser *parser,
                              const char *buffer, size_t size);
diff --git a/include/qapi/qmp/qbool.h b/include/qapi/qmp/qbool.h
index c4eaab9..a41111c 100644
--- a/include/qapi/qmp/qbool.h
+++ b/include/qapi/qmp/qbool.h
@@ -14,16 +14,16 @@
 #ifndef QBOOL_H
 #define QBOOL_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QBool {
-    QObject_HEAD;
-    int value;
+    QObject base;
+    bool value;
 } QBool;
 
-QBool *qbool_from_int(int value);
-int qbool_get_int(const QBool *qb);
+QBool *qbool_from_bool(bool value);
+bool qbool_get_bool(const QBool *qb);
 QBool *qobject_to_qbool(const QObject *obj);
+void qbool_destroy_obj(QObject *obj);
 
 #endif /* QBOOL_H */
diff --git a/include/qapi/qmp/qdict.h b/include/qapi/qmp/qdict.h
index d68f4eb..71b8eb0 100644
--- a/include/qapi/qmp/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -16,8 +16,6 @@
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qlist.h"
 #include "qemu/queue.h"
-#include <stdbool.h>
-#include <stdint.h>
 
 #define QDICT_BUCKET_MAX 512
 
@@ -28,7 +26,7 @@
 } QDictEntry;
 
 typedef struct QDict {
-    QObject_HEAD;
+    QObject base;
     size_t size;
     QLIST_HEAD(,QDictEntry) table[QDICT_BUCKET_MAX];
 } QDict;
@@ -48,6 +46,7 @@
                 void *opaque);
 const QDictEntry *qdict_first(const QDict *qdict);
 const QDictEntry *qdict_next(const QDict *qdict, const QDictEntry *entry);
+void qdict_destroy_obj(QObject *obj);
 
 /* Helper to qdict_put_obj(), accepts any object */
 #define qdict_put(qdict, key, obj) \
@@ -56,20 +55,24 @@
 /* High level helpers */
 double qdict_get_double(const QDict *qdict, const char *key);
 int64_t qdict_get_int(const QDict *qdict, const char *key);
-int qdict_get_bool(const QDict *qdict, const char *key);
+bool qdict_get_bool(const QDict *qdict, const char *key);
 QList *qdict_get_qlist(const QDict *qdict, const char *key);
 QDict *qdict_get_qdict(const QDict *qdict, const char *key);
 const char *qdict_get_str(const QDict *qdict, const char *key);
 int64_t qdict_get_try_int(const QDict *qdict, const char *key,
                           int64_t def_value);
-int qdict_get_try_bool(const QDict *qdict, const char *key, int def_value);
+bool qdict_get_try_bool(const QDict *qdict, const char *key, bool def_value);
 const char *qdict_get_try_str(const QDict *qdict, const char *key);
 
+void qdict_copy_default(QDict *dst, QDict *src, const char *key);
+void qdict_set_default_str(QDict *dst, const char *key, const char *val);
+
 QDict *qdict_clone_shallow(const QDict *src);
 void qdict_flatten(QDict *qdict);
 
 void qdict_extract_subqdict(QDict *src, QDict **dst, const char *start);
 void qdict_array_split(QDict *src, QList **dst);
+int qdict_array_entries(QDict *src, const char *subqdict);
 
 void qdict_join(QDict *dest, QDict *src, bool overwrite);
 
diff --git a/include/qapi/qmp/qerror.h b/include/qapi/qmp/qerror.h
index 0ca6cbd..6586c9f 100644
--- a/include/qapi/qmp/qerror.h
+++ b/include/qapi/qmp/qerror.h
@@ -12,146 +12,95 @@
 #ifndef QERROR_H
 #define QERROR_H
 
-#include "qapi/qmp/qstring.h"
-#include "qemu/error-report.h"
-#include "qapi/error.h"
-#include "qapi-types.h"
-#include <stdarg.h>
-
-typedef struct QError {
-    QObject_HEAD;
-    Location loc;
-    char *err_msg;
-    ErrorClass err_class;
-} QError;
-
-QString *qerror_human(const QError *qerror);
-void qerror_report(ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(2, 3);
-void qerror_report_err(Error *err);
-
 /*
- * QError class list
- * Please keep the definitions in alphabetical order.
- * Use scripts/check-qerror.sh to check.
+ * These macros will go away, please don't use in new code, and do not
+ * add new ones!
  */
 #define QERR_BASE_NOT_FOUND \
-    ERROR_CLASS_GENERIC_ERROR, "Base '%s' not found"
-
-#define QERR_BLOCK_JOB_NOT_ACTIVE \
-    ERROR_CLASS_DEVICE_NOT_ACTIVE, "No active block job on device '%s'"
-
-#define QERR_BLOCK_JOB_NOT_READY \
-    ERROR_CLASS_GENERIC_ERROR, "The active block job for device '%s' cannot be completed"
-
-#define QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED \
-    ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by device '%s' does not support feature '%s'"
+    "Base '%s' not found"
 
 #define QERR_BUS_NO_HOTPLUG \
-    ERROR_CLASS_GENERIC_ERROR, "Bus '%s' does not support hotplugging"
-
-#define QERR_BUS_NOT_FOUND \
-    ERROR_CLASS_GENERIC_ERROR, "Bus '%s' not found"
-
-#define QERR_COMMAND_NOT_FOUND \
-    ERROR_CLASS_COMMAND_NOT_FOUND, "The command %s has not been found"
-
-#define QERR_DEVICE_ENCRYPTED \
-    ERROR_CLASS_DEVICE_ENCRYPTED, "'%s' (%s) is encrypted"
+    "Bus '%s' does not support hotplugging"
 
 #define QERR_DEVICE_HAS_NO_MEDIUM \
-    ERROR_CLASS_GENERIC_ERROR, "Device '%s' has no medium"
+    "Device '%s' has no medium"
 
 #define QERR_DEVICE_INIT_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Device '%s' could not be initialized"
+    "Device '%s' could not be initialized"
 
 #define QERR_DEVICE_IN_USE \
-    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is in use"
-
-#define QERR_DEVICE_IS_READ_ONLY \
-    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is read only"
+    "Device '%s' is in use"
 
 #define QERR_DEVICE_NO_HOTPLUG \
-    ERROR_CLASS_GENERIC_ERROR, "Device '%s' does not support hotplugging"
-
-#define QERR_DEVICE_NOT_ACTIVE \
-    ERROR_CLASS_DEVICE_NOT_ACTIVE, "No %s device has been activated"
-
-#define QERR_DEVICE_NOT_ENCRYPTED \
-    ERROR_CLASS_GENERIC_ERROR, "Device '%s' is not encrypted"
-
-#define QERR_DEVICE_NOT_FOUND \
-    ERROR_CLASS_DEVICE_NOT_FOUND, "Device '%s' not found"
+    "Device '%s' does not support hotplugging"
 
 #define QERR_FD_NOT_FOUND \
-    ERROR_CLASS_GENERIC_ERROR, "File descriptor named '%s' not found"
+    "File descriptor named '%s' not found"
 
 #define QERR_FD_NOT_SUPPLIED \
-    ERROR_CLASS_GENERIC_ERROR, "No file descriptor supplied via SCM_RIGHTS"
+    "No file descriptor supplied via SCM_RIGHTS"
 
 #define QERR_FEATURE_DISABLED \
-    ERROR_CLASS_GENERIC_ERROR, "The feature '%s' is not enabled"
+    "The feature '%s' is not enabled"
 
 #define QERR_INVALID_BLOCK_FORMAT \
-    ERROR_CLASS_GENERIC_ERROR, "Invalid block format '%s'"
+    "Invalid block format '%s'"
 
 #define QERR_INVALID_PARAMETER \
-    ERROR_CLASS_GENERIC_ERROR, "Invalid parameter '%s'"
+    "Invalid parameter '%s'"
 
 #define QERR_INVALID_PARAMETER_TYPE \
-    ERROR_CLASS_GENERIC_ERROR, "Invalid parameter type for '%s', expected: %s"
+    "Invalid parameter type for '%s', expected: %s"
 
 #define QERR_INVALID_PARAMETER_VALUE \
-    ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' expects %s"
+    "Parameter '%s' expects %s"
 
 #define QERR_INVALID_PASSWORD \
-    ERROR_CLASS_GENERIC_ERROR, "Password incorrect"
+    "Password incorrect"
 
 #define QERR_IO_ERROR \
-    ERROR_CLASS_GENERIC_ERROR, "An IO error has occurred"
+    "An IO error has occurred"
 
 #define QERR_JSON_PARSING \
-    ERROR_CLASS_GENERIC_ERROR, "Invalid JSON syntax"
-
-#define QERR_KVM_MISSING_CAP \
-    ERROR_CLASS_KVM_MISSING_CAP, "Using KVM without %s, %s unavailable"
+    "Invalid JSON syntax"
 
 #define QERR_MIGRATION_ACTIVE \
-    ERROR_CLASS_GENERIC_ERROR, "There's a migration process in progress"
+    "There's a migration process in progress"
 
 #define QERR_MISSING_PARAMETER \
-    ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' is missing"
+    "Parameter '%s' is missing"
 
 #define QERR_PERMISSION_DENIED \
-    ERROR_CLASS_GENERIC_ERROR, "Insufficient permission to perform this operation"
+    "Insufficient permission to perform this operation"
 
 #define QERR_PROPERTY_VALUE_BAD \
-    ERROR_CLASS_GENERIC_ERROR, "Property '%s.%s' doesn't take value '%s'"
+    "Property '%s.%s' doesn't take value '%s'"
 
 #define QERR_PROPERTY_VALUE_OUT_OF_RANGE \
-    ERROR_CLASS_GENERIC_ERROR, "Property %s.%s doesn't take value %" PRId64 " (minimum: %" PRId64 ", maximum: %" PRId64 ")"
+    "Property %s.%s doesn't take value %" PRId64 " (minimum: %" PRId64 ", maximum: %" PRId64 ")"
 
 #define QERR_QGA_COMMAND_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Guest agent command failed, error was '%s'"
+    "Guest agent command failed, error was '%s'"
 
 #define QERR_QMP_BAD_INPUT_OBJECT \
-    ERROR_CLASS_GENERIC_ERROR, "Expected '%s' in QMP input"
+    "Expected '%s' in QMP input"
 
 #define QERR_QMP_BAD_INPUT_OBJECT_MEMBER \
-    ERROR_CLASS_GENERIC_ERROR, "QMP input object member '%s' expects '%s'"
+    "QMP input object member '%s' expects '%s'"
 
 #define QERR_QMP_EXTRA_MEMBER \
-    ERROR_CLASS_GENERIC_ERROR, "QMP input object member '%s' is unexpected"
+    "QMP input object member '%s' is unexpected"
 
 #define QERR_SET_PASSWD_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Could not set password"
+    "Could not set password"
 
 #define QERR_UNDEFINED_ERROR \
-    ERROR_CLASS_GENERIC_ERROR, "An undefined error has occurred"
-
-#define QERR_UNKNOWN_BLOCK_FORMAT_FEATURE \
-    ERROR_CLASS_GENERIC_ERROR, "'%s' uses a %s feature which is not supported by this qemu version: %s"
+    "An undefined error has occurred"
 
 #define QERR_UNSUPPORTED \
-    ERROR_CLASS_GENERIC_ERROR, "this feature or command is not currently supported"
+    "this feature or command is not currently supported"
+
+#define QERR_REPLAY_NOT_SUPPORTED \
+    "Record/replay feature is not supported for '%s'"
 
 #endif /* QERROR_H */
diff --git a/include/qapi/qmp/qfloat.h b/include/qapi/qmp/qfloat.h
index a865844..b5d1583 100644
--- a/include/qapi/qmp/qfloat.h
+++ b/include/qapi/qmp/qfloat.h
@@ -14,16 +14,16 @@
 #ifndef QFLOAT_H
 #define QFLOAT_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QFloat {
-    QObject_HEAD;
+    QObject base;
     double value;
 } QFloat;
 
 QFloat *qfloat_from_double(double value);
 double qfloat_get_double(const QFloat *qi);
 QFloat *qobject_to_qfloat(const QObject *obj);
+void qfloat_destroy_obj(QObject *obj);
 
 #endif /* QFLOAT_H */
diff --git a/include/qapi/qmp/qint.h b/include/qapi/qmp/qint.h
index 48a41b0..3aaff76 100644
--- a/include/qapi/qmp/qint.h
+++ b/include/qapi/qmp/qint.h
@@ -13,16 +13,16 @@
 #ifndef QINT_H
 #define QINT_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QInt {
-    QObject_HEAD;
+    QObject base;
     int64_t value;
 } QInt;
 
 QInt *qint_from_int(int64_t value);
 int64_t qint_get_int(const QInt *qi);
 QInt *qobject_to_qint(const QObject *obj);
+void qint_destroy_obj(QObject *obj);
 
 #endif /* QINT_H */
diff --git a/include/qapi/qmp/qjson.h b/include/qapi/qmp/qjson.h
index ee4d31a..02b1f2c 100644
--- a/include/qapi/qmp/qjson.h
+++ b/include/qapi/qmp/qjson.h
@@ -14,8 +14,6 @@
 #ifndef QJSON_H
 #define QJSON_H
 
-#include <stdarg.h>
-#include "qemu/compiler.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qstring.h"
 
diff --git a/include/qapi/qmp/qlist.h b/include/qapi/qmp/qlist.h
index 6cc4831..a84117e 100644
--- a/include/qapi/qmp/qlist.h
+++ b/include/qapi/qmp/qlist.h
@@ -22,7 +22,7 @@
 } QListEntry;
 
 typedef struct QList {
-    QObject_HEAD;
+    QObject base;
     QTAILQ_HEAD(,QListEntry) head;
 } QList;
 
@@ -49,6 +49,7 @@
 int qlist_empty(const QList *qlist);
 size_t qlist_size(const QList *qlist);
 QList *qobject_to_qlist(const QObject *obj);
+void qlist_destroy_obj(QObject *obj);
 
 static inline const QListEntry *qlist_first(const QList *qlist)
 {
diff --git a/include/qapi/qmp/qobject.h b/include/qapi/qmp/qobject.h
index d0bbc7c..b8ddbca 100644
--- a/include/qapi/qmp/qobject.h
+++ b/include/qapi/qmp/qobject.h
@@ -3,7 +3,7 @@
  *
  * Based on ideas by Avi Kivity <avi@redhat.com>
  *
- * Copyright (C) 2009 Red Hat Inc.
+ * Copyright (C) 2009, 2015 Red Hat Inc.
  *
  * Authors:
  *  Luiz Capitulino <lcapitulino@redhat.com>
@@ -32,36 +32,12 @@
 #ifndef QOBJECT_H
 #define QOBJECT_H
 
-#include <stddef.h>
-#include <assert.h>
+#include "qapi-types.h"
 
-typedef enum {
-    QTYPE_NONE,
-    QTYPE_QINT,
-    QTYPE_QSTRING,
-    QTYPE_QDICT,
-    QTYPE_QLIST,
-    QTYPE_QFLOAT,
-    QTYPE_QBOOL,
-    QTYPE_QERROR,
-    QTYPE_MAX,
-} qtype_code;
-
-struct QObject;
-
-typedef struct QType {
-    qtype_code code;
-    void (*destroy)(struct QObject *);
-} QType;
-
-typedef struct QObject {
-    const QType *type;
+struct QObject {
+    QType type;
     size_t refcnt;
-} QObject;
-
-/* Objects definitions must include this */
-#define QObject_HEAD  \
-    QObject base
+};
 
 /* Get the 'base' part of an object */
 #define QOBJECT(obj) (&(obj)->base)
@@ -75,9 +51,12 @@
     qobject_decref(obj ? QOBJECT(obj) : NULL)
 
 /* Initialize an object to default values */
-#define QOBJECT_INIT(obj, qtype_type)   \
-    obj->base.refcnt = 1;               \
-    obj->base.type   = qtype_type
+static inline void qobject_init(QObject *obj, QType type)
+{
+    assert(QTYPE_NONE < type && type < QTYPE__MAX);
+    obj->refcnt = 1;
+    obj->type = type;
+}
 
 /**
  * qobject_incref(): Increment QObject's reference count
@@ -89,25 +68,37 @@
 }
 
 /**
+ * qobject_destroy(): Free resources used by the object
+ */
+void qobject_destroy(QObject *obj);
+
+/**
  * qobject_decref(): Decrement QObject's reference count, deallocate
  * when it reaches zero
  */
 static inline void qobject_decref(QObject *obj)
 {
+    assert(!obj || obj->refcnt);
     if (obj && --obj->refcnt == 0) {
-        assert(obj->type != NULL);
-        assert(obj->type->destroy != NULL);
-        obj->type->destroy(obj);
+        qobject_destroy(obj);
     }
 }
 
 /**
  * qobject_type(): Return the QObject's type
  */
-static inline qtype_code qobject_type(const QObject *obj)
+static inline QType qobject_type(const QObject *obj)
 {
-    assert(obj->type != NULL);
-    return obj->type->code;
+    assert(QTYPE_NONE < obj->type && obj->type < QTYPE__MAX);
+    return obj->type;
+}
+
+extern QObject qnull_;
+
+static inline QObject *qnull(void)
+{
+    qobject_incref(&qnull_);
+    return &qnull_;
 }
 
 #endif /* QOBJECT_H */
diff --git a/include/qapi/qmp/qstring.h b/include/qapi/qmp/qstring.h
index 1bc3666..10076b7 100644
--- a/include/qapi/qmp/qstring.h
+++ b/include/qapi/qmp/qstring.h
@@ -13,11 +13,10 @@
 #ifndef QSTRING_H
 #define QSTRING_H
 
-#include <stdint.h>
 #include "qapi/qmp/qobject.h"
 
 typedef struct QString {
-    QObject_HEAD;
+    QObject base;
     char *string;
     size_t length;
     size_t capacity;
@@ -32,5 +31,6 @@
 void qstring_append(QString *qstring, const char *str);
 void qstring_append_chr(QString *qstring, int c);
 QString *qobject_to_qstring(const QObject *obj);
+void qstring_destroy_obj(QObject *obj);
 
 #endif /* QSTRING_H */
diff --git a/include/qapi/qmp/types.h b/include/qapi/qmp/types.h
index 7782ec5..27cfbd8 100644
--- a/include/qapi/qmp/types.h
+++ b/include/qapi/qmp/types.h
@@ -10,8 +10,8 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#ifndef QEMU_OBJECTS_H
-#define QEMU_OBJECTS_H
+#ifndef QAPI_QMP_TYPES_H
+#define QAPI_QMP_TYPES_H
 
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qint.h"
@@ -20,6 +20,5 @@
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qdict.h"
 #include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qjson.h"
 
-#endif /* QEMU_OBJECTS_H */
+#endif /* QAPI_QMP_TYPES_H */
diff --git a/include/qapi/string-input-visitor.h b/include/qapi/string-input-visitor.h
index 089243c..3355134 100644
--- a/include/qapi/string-input-visitor.h
+++ b/include/qapi/string-input-visitor.h
@@ -17,9 +17,11 @@
 
 typedef struct StringInputVisitor StringInputVisitor;
 
-StringInputVisitor *string_input_visitor_new(const char *str);
-void string_input_visitor_cleanup(StringInputVisitor *v);
-
-Visitor *string_input_get_visitor(StringInputVisitor *v);
+/*
+ * The string input visitor does not implement support for visiting
+ * QAPI structs, alternates, null, or arbitrary QTypes.  It also
+ * requires a non-null list argument to visit_start_list().
+ */
+Visitor *string_input_visitor_new(const char *str);
 
 #endif
diff --git a/include/qapi/string-output-visitor.h b/include/qapi/string-output-visitor.h
index d99717f..268dfe9 100644
--- a/include/qapi/string-output-visitor.h
+++ b/include/qapi/string-output-visitor.h
@@ -17,10 +17,19 @@
 
 typedef struct StringOutputVisitor StringOutputVisitor;
 
-StringOutputVisitor *string_output_visitor_new(bool human);
-void string_output_visitor_cleanup(StringOutputVisitor *v);
-
-char *string_output_get_string(StringOutputVisitor *v);
-Visitor *string_output_get_visitor(StringOutputVisitor *v);
+/*
+ * Create a new string output visitor.
+ *
+ * Using @human creates output that is a bit easier for humans to read
+ * (for example, showing integer values in both decimal and hex).
+ *
+ * If everything else succeeds, pass @result to visit_complete() to
+ * collect the result of the visit.
+ *
+ * The string output visitor does not implement support for visiting
+ * QAPI structs, alternates, null, or arbitrary QTypes.  It also
+ * requires a non-null list argument to visit_start_list().
+ */
+Visitor *string_output_visitor_new(bool human, char **result);
 
 #endif
diff --git a/include/qapi/util.h b/include/qapi/util.h
index de9238b..7ad26c0 100644
--- a/include/qapi/util.h
+++ b/include/qapi/util.h
@@ -11,7 +11,7 @@
 #ifndef QAPI_UTIL_H
 #define QAPI_UTIL_H
 
-int qapi_enum_parse(const char *lookup[], const char *buf,
+int qapi_enum_parse(const char * const lookup[], const char *buf,
                     int max, int def, Error **errp);
 
 #endif
diff --git a/include/qapi/visitor-impl.h b/include/qapi/visitor-impl.h
index 09bb0fd..8bd47ee 100644
--- a/include/qapi/visitor-impl.h
+++ b/include/qapi/visitor-impl.h
@@ -1,7 +1,7 @@
 /*
  * Core Definitions for QAPI Visitor implementations
  *
- * Copyright (C) 2012 Red Hat, Inc.
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  *
  * Author: Paolo Bonizni <pbonzini@redhat.com>
  *
@@ -12,56 +12,108 @@
 #ifndef QAPI_VISITOR_IMPL_H
 #define QAPI_VISITOR_IMPL_H
 
-#include "qapi/error.h"
 #include "qapi/visitor.h"
 
+/*
+ * This file describes the callback interface for implementing a QAPI
+ * visitor.  For the client interface, see visitor.h.  When
+ * implementing the callbacks, it is easiest to declare a struct with
+ * 'Visitor visitor;' as the first member.  A callback's contract
+ * matches the corresponding public functions' contract unless stated
+ * otherwise.  In the comments below, some callbacks are marked "must
+ * be set for $TYPE visits to work"; if a visitor implementation omits
+ * that callback, it should also document that it is only useful for a
+ * subset of QAPI.
+ */
+
+/*
+ * There are four classes of visitors; setting the class determines
+ * how QAPI enums are visited, as well as what additional restrictions
+ * can be asserted.  The values are intentionally chosen so as to
+ * permit some assertions based on whether a given bit is set (that
+ * is, some assertions apply to input and clone visitors, some
+ * assertions apply to output and clone visitors).
+ */
+typedef enum VisitorType {
+    VISITOR_INPUT = 1,
+    VISITOR_OUTPUT = 2,
+    VISITOR_CLONE = 3,
+    VISITOR_DEALLOC = 4,
+} VisitorType;
+
 struct Visitor
 {
+    /* Must be set to visit structs */
+    void (*start_struct)(Visitor *v, const char *name, void **obj,
+                         size_t size, Error **errp);
+
+    /* Optional; intended for input visitors */
+    void (*check_struct)(Visitor *v, Error **errp);
+
+    /* Must be set to visit structs */
+    void (*end_struct)(Visitor *v, void **obj);
+
+    /* Must be set; implementations may require @list to be non-null,
+     * but must document it. */
+    void (*start_list)(Visitor *v, const char *name, GenericList **list,
+                       size_t size, Error **errp);
+
     /* Must be set */
-    void (*start_struct)(Visitor *v, void **obj, const char *kind,
-                         const char *name, size_t size, Error **errp);
-    void (*end_struct)(Visitor *v, Error **errp);
+    GenericList *(*next_list)(Visitor *v, GenericList *tail, size_t size);
 
-    void (*start_implicit_struct)(Visitor *v, void **obj, size_t size,
-                                  Error **errp);
-    void (*end_implicit_struct)(Visitor *v, Error **errp);
+    /* Must be set */
+    void (*end_list)(Visitor *v, void **list);
 
-    void (*start_list)(Visitor *v, const char *name, Error **errp);
-    GenericList *(*next_list)(Visitor *v, GenericList **list, Error **errp);
-    void (*end_list)(Visitor *v, Error **errp);
+    /* Must be set by input and dealloc visitors to visit alternates;
+     * optional for output visitors. */
+    void (*start_alternate)(Visitor *v, const char *name,
+                            GenericAlternate **obj, size_t size,
+                            bool promote_int, Error **errp);
 
-    void (*type_enum)(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name, Error **errp);
-    void (*get_next_type)(Visitor *v, int *kind, const int *qobjects,
-                          const char *name, Error **errp);
+    /* Optional, needed for dealloc visitor */
+    void (*end_alternate)(Visitor *v, void **obj);
 
-    void (*type_int)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    void (*type_bool)(Visitor *v, bool *obj, const char *name, Error **errp);
-    void (*type_str)(Visitor *v, char **obj, const char *name, Error **errp);
-    void (*type_number)(Visitor *v, double *obj, const char *name,
+    /* Must be set */
+    void (*type_int64)(Visitor *v, const char *name, int64_t *obj,
+                       Error **errp);
+
+    /* Must be set */
+    void (*type_uint64)(Visitor *v, const char *name, uint64_t *obj,
                         Error **errp);
 
-    /* May be NULL */
-    void (*optional)(Visitor *v, bool *present, const char *name,
+    /* Optional; fallback is type_uint64() */
+    void (*type_size)(Visitor *v, const char *name, uint64_t *obj,
+                      Error **errp);
+
+    /* Must be set */
+    void (*type_bool)(Visitor *v, const char *name, bool *obj, Error **errp);
+
+    /* Must be set */
+    void (*type_str)(Visitor *v, const char *name, char **obj, Error **errp);
+
+    /* Must be set to visit numbers */
+    void (*type_number)(Visitor *v, const char *name, double *obj,
+                        Error **errp);
+
+    /* Must be set to visit arbitrary QTypes */
+    void (*type_any)(Visitor *v, const char *name, QObject **obj,
                      Error **errp);
 
-    void (*type_uint8)(Visitor *v, uint8_t *obj, const char *name, Error **errp);
-    void (*type_uint16)(Visitor *v, uint16_t *obj, const char *name, Error **errp);
-    void (*type_uint32)(Visitor *v, uint32_t *obj, const char *name, Error **errp);
-    void (*type_uint64)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-    void (*type_int8)(Visitor *v, int8_t *obj, const char *name, Error **errp);
-    void (*type_int16)(Visitor *v, int16_t *obj, const char *name, Error **errp);
-    void (*type_int32)(Visitor *v, int32_t *obj, const char *name, Error **errp);
-    void (*type_int64)(Visitor *v, int64_t *obj, const char *name, Error **errp);
-    /* visit_type_size() falls back to (*type_uint64)() if type_size is unset */
-    void (*type_size)(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-    bool (*start_union)(Visitor *v, bool data_present, Error **errp);
-    void (*end_union)(Visitor *v, bool data_present, Error **errp);
-};
+    /* Must be set to visit explicit null values.  */
+    void (*type_null)(Visitor *v, const char *name, Error **errp);
 
-void input_type_enum(Visitor *v, int *obj, const char *strings[],
-                     const char *kind, const char *name, Error **errp);
-void output_type_enum(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name, Error **errp);
+    /* Must be set for input visitors, optional otherwise.  The core
+     * takes care of the return type in the public interface. */
+    void (*optional)(Visitor *v, const char *name, bool *present);
+
+    /* Must be set */
+    VisitorType type;
+
+    /* Must be set for output visitors, optional otherwise. */
+    void (*complete)(Visitor *v, void *opaque);
+
+    /* Must be set */
+    void (*free)(Visitor *v);
+};
 
 #endif
diff --git a/include/qapi/visitor.h b/include/qapi/visitor.h
index 5934f59..6c77a91 100644
--- a/include/qapi/visitor.h
+++ b/include/qapi/visitor.h
@@ -1,6 +1,7 @@
 /*
  * Core Definitions for QAPI Visitor Classes
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -10,55 +11,606 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
-#ifndef QAPI_VISITOR_CORE_H
-#define QAPI_VISITOR_CORE_H
 
-#include "qemu/typedefs.h"
+#ifndef QAPI_VISITOR_H
+#define QAPI_VISITOR_H
+
 #include "qapi/qmp/qobject.h"
-#include "qapi/error.h"
-#include <stdlib.h>
 
-typedef struct GenericList
-{
-    union {
-        void *value;
-        uint64_t padding;
-    };
+/*
+ * The QAPI schema defines both a set of C data types, and a QMP wire
+ * format.  QAPI objects can contain references to other QAPI objects,
+ * resulting in a directed acyclic graph.  QAPI also generates visitor
+ * functions to walk these graphs.  This file represents the interface
+ * for doing work at each node of a QAPI graph; it can also be used
+ * for a virtual walk, where there is no actual QAPI C struct.
+ *
+ * There are four kinds of visitor classes: input visitors (QMP,
+ * string, and QemuOpts) parse an external representation and build
+ * the corresponding QAPI graph, output visitors (QMP and string) take
+ * a completed QAPI graph and generate an external representation, the
+ * dealloc visitor can take a QAPI graph (possibly partially
+ * constructed) and recursively free its resources, and the clone
+ * visitor performs a deep clone of one QAPI object to another.  While
+ * the dealloc and QMP input/output visitors are general, the string,
+ * QemuOpts, and clone visitors have some implementation limitations;
+ * see the documentation for each visitor for more details on what it
+ * supports.  Also, see visitor-impl.h for the callback contracts
+ * implemented by each visitor, and docs/qapi-code-gen.txt for more
+ * about the QAPI code generator.
+ *
+ * All of the visitors are created via:
+ *
+ * Visitor *subtype_visitor_new(parameters...);
+ *
+ * A visitor should be used for exactly one top-level visit_type_FOO()
+ * or virtual walk; if that is successful, the caller can optionally
+ * call visit_complete() (for now, useful only for output visits, but
+ * safe to call on all visits).  Then, regardless of success or
+ * failure, the user should call visit_free() to clean up resources.
+ * It is okay to free the visitor without completing the visit, if
+ * some other error is detected in the meantime.
+ *
+ * All QAPI types have a corresponding function with a signature
+ * roughly compatible with this:
+ *
+ * void visit_type_FOO(Visitor *v, const char *name, T obj, Error **errp);
+ *
+ * where T is FOO for scalar types, and FOO * otherwise.  The scalar
+ * visitors are declared here; the remaining visitors are generated in
+ * qapi-visit.h.
+ *
+ * The @name parameter of visit_type_FOO() describes the relation
+ * between this QAPI value and its parent container.  When visiting
+ * the root of a tree, @name is ignored; when visiting a member of an
+ * object, @name is the key associated with the value; and when
+ * visiting a member of a list, @name is NULL.
+ *
+ * FIXME: Clients must pass NULL for @name when visiting a member of a
+ * list, but this leads to poor error messages; it might be nicer to
+ * require a non-NULL name such as "key.0" for '{ "key": [ "value" ]
+ * }' if an error is encountered on "value" (or to have the visitor
+ * core auto-generate the nicer name).
+ *
+ * The visit_type_FOO() functions expect a non-null @obj argument;
+ * they allocate *@obj during input visits, leave it unchanged on
+ * output visits, and recursively free any resources during a dealloc
+ * visit.  Each function also takes the customary @errp argument (see
+ * qapi/error.h for details), for reporting any errors (such as if a
+ * member @name is not present, or is present but not the specified
+ * type).
+ *
+ * If an error is detected during visit_type_FOO() with an input
+ * visitor, then *@obj will be NULL for pointer types, and left
+ * unchanged for scalar types.  Using an output or clone visitor with
+ * an incomplete object has undefined behavior (other than a special
+ * case for visit_type_str() treating NULL like ""), while the dealloc
+ * visitor safely handles incomplete objects.  Since input visitors
+ * never produce an incomplete object, such an object is possible only
+ * by manual construction.
+ *
+ * For the QAPI object types (structs, unions, and alternates), there
+ * is an additional generated function in qapi-visit.h compatible
+ * with:
+ *
+ * void visit_type_FOO_members(Visitor *v, FOO *obj, Error **errp);
+ *
+ * for visiting the members of a type without also allocating the QAPI
+ * struct.
+ *
+ * Additionally, in qapi-types.h, all QAPI pointer types (structs,
+ * unions, alternates, and lists) have a generated function compatible
+ * with:
+ *
+ * void qapi_free_FOO(FOO *obj);
+ *
+ * where behaves like free() in that @obj may be NULL.  Such objects
+ * may also be used with the following macro, provided alongside the
+ * clone visitor:
+ *
+ * Type *QAPI_CLONE(Type, src);
+ *
+ * in order to perform a deep clone of @src.  Because of the generated
+ * qapi_free functions and the QAPI_CLONE() macro, the clone and
+ * dealloc visitor should not be used directly outside of QAPI code.
+ *
+ * QAPI types can also inherit from a base class; when this happens, a
+ * function is generated for easily going from the derived type to the
+ * base type:
+ *
+ * BASE *qapi_CHILD_base(CHILD *obj);
+ *
+ * For a real QAPI struct, typical input usage involves:
+ *
+ * <example>
+ *  Foo *f;
+ *  Error *err = NULL;
+ *  Visitor *v;
+ *
+ *  v = FOO_visitor_new(...);
+ *  visit_type_Foo(v, NULL, &f, &err);
+ *  if (err) {
+ *      ...handle error...
+ *  } else {
+ *      ...use f...
+ *  }
+ *  visit_free(v);
+ *  qapi_free_Foo(f);
+ * </example>
+ *
+ * For a list, it is:
+ * <example>
+ *  FooList *l;
+ *  Error *err = NULL;
+ *  Visitor *v;
+ *
+ *  v = FOO_visitor_new(...);
+ *  visit_type_FooList(v, NULL, &l, &err);
+ *  if (err) {
+ *      ...handle error...
+ *  } else {
+ *      for ( ; l; l = l->next) {
+ *          ...use l->value...
+ *      }
+ *  }
+ *  visit_free(v);
+ *  qapi_free_FooList(l);
+ * </example>
+ *
+ * Similarly, typical output usage is:
+ *
+ * <example>
+ *  Foo *f = ...obtain populated object...
+ *  Error *err = NULL;
+ *  Visitor *v;
+ *  Type *result;
+ *
+ *  v = FOO_visitor_new(..., &result);
+ *  visit_type_Foo(v, NULL, &f, &err);
+ *  if (err) {
+ *      ...handle error...
+ *  } else {
+ *      visit_complete(v, &result);
+ *      ...use result...
+ *  }
+ *  visit_free(v);
+ * </example>
+ *
+ * When visiting a real QAPI struct, this file provides several
+ * helpers that rely on in-tree information to control the walk:
+ * visit_optional() for the 'has_member' field associated with
+ * optional 'member' in the C struct; and visit_next_list() for
+ * advancing through a FooList linked list.  Similarly, the
+ * visit_is_input() helper makes it possible to write code that is
+ * visitor-agnostic everywhere except for cleanup.  Only the generated
+ * visit_type functions need to use these helpers.
+ *
+ * It is also possible to use the visitors to do a virtual walk, where
+ * no actual QAPI struct is present.  In this situation, decisions
+ * about what needs to be walked are made by the calling code, and
+ * structured visits are split between pairs of start and end methods
+ * (where the end method must be called if the start function
+ * succeeded, even if an intermediate visit encounters an error).
+ * Thus, a virtual walk corresponding to '{ "list": [1, 2] }' looks
+ * like:
+ *
+ * <example>
+ *  Visitor *v;
+ *  Error *err = NULL;
+ *  int value;
+ *
+ *  v = FOO_visitor_new(...);
+ *  visit_start_struct(v, NULL, NULL, 0, &err);
+ *  if (err) {
+ *      goto out;
+ *  }
+ *  visit_start_list(v, "list", NULL, 0, &err);
+ *  if (err) {
+ *      goto outobj;
+ *  }
+ *  value = 1;
+ *  visit_type_int(v, NULL, &value, &err);
+ *  if (err) {
+ *      goto outlist;
+ *  }
+ *  value = 2;
+ *  visit_type_int(v, NULL, &value, &err);
+ *  if (err) {
+ *      goto outlist;
+ *  }
+ * outlist:
+ *  visit_end_list(v, NULL);
+ *  if (!err) {
+ *      visit_check_struct(v, &err);
+ *  }
+ * outobj:
+ *  visit_end_struct(v, NULL);
+ * out:
+ *  error_propagate(errp, err);
+ *  visit_free(v);
+ * </example>
+ */
+
+/*** Useful types ***/
+
+/* This struct is layout-compatible with all other *List structs
+ * created by the QAPI generator.  It is used as a typical
+ * singly-linked list. */
+typedef struct GenericList {
     struct GenericList *next;
+    char padding[];
 } GenericList;
 
-void visit_start_handle(Visitor *v, void **obj, const char *kind,
-                        const char *name, Error **errp);
-void visit_end_handle(Visitor *v, Error **errp);
-void visit_start_struct(Visitor *v, void **obj, const char *kind,
-                        const char *name, size_t size, Error **errp);
-void visit_end_struct(Visitor *v, Error **errp);
-void visit_start_implicit_struct(Visitor *v, void **obj, size_t size,
-                                 Error **errp);
-void visit_end_implicit_struct(Visitor *v, Error **errp);
-void visit_start_list(Visitor *v, const char *name, Error **errp);
-GenericList *visit_next_list(Visitor *v, GenericList **list, Error **errp);
-void visit_end_list(Visitor *v, Error **errp);
-void visit_optional(Visitor *v, bool *present, const char *name,
-                    Error **errp);
-void visit_get_next_type(Visitor *v, int *obj, const int *qtypes,
-                         const char *name, Error **errp);
-void visit_type_enum(Visitor *v, int *obj, const char *strings[],
-                     const char *kind, const char *name, Error **errp);
-void visit_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp);
-void visit_type_uint8(Visitor *v, uint8_t *obj, const char *name, Error **errp);
-void visit_type_uint16(Visitor *v, uint16_t *obj, const char *name, Error **errp);
-void visit_type_uint32(Visitor *v, uint32_t *obj, const char *name, Error **errp);
-void visit_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-void visit_type_int8(Visitor *v, int8_t *obj, const char *name, Error **errp);
-void visit_type_int16(Visitor *v, int16_t *obj, const char *name, Error **errp);
-void visit_type_int32(Visitor *v, int32_t *obj, const char *name, Error **errp);
-void visit_type_int64(Visitor *v, int64_t *obj, const char *name, Error **errp);
-void visit_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp);
-void visit_type_bool(Visitor *v, bool *obj, const char *name, Error **errp);
-void visit_type_str(Visitor *v, char **obj, const char *name, Error **errp);
-void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp);
-bool visit_start_union(Visitor *v, bool data_present, Error **errp);
-void visit_end_union(Visitor *v, bool data_present, Error **errp);
+/* This struct is layout-compatible with all Alternate types
+ * created by the QAPI generator. */
+typedef struct GenericAlternate {
+    QType type;
+    char padding[];
+} GenericAlternate;
+
+/*** Visitor cleanup ***/
+
+/*
+ * Complete the visit, collecting any output.
+ *
+ * May only be called only once after a successful top-level
+ * visit_type_FOO() or visit_end_ITEM(), and marks the end of the
+ * visit.  The @opaque pointer should match the output parameter
+ * passed to the subtype_visitor_new() used to create an output
+ * visitor, or NULL for any other visitor.  Needed for output
+ * visitors, but may also be called with other visitors.
+ */
+void visit_complete(Visitor *v, void *opaque);
+
+/*
+ * Free @v and any resources it has tied up.
+ *
+ * May be called whether or not the visit has been successfully
+ * completed, but should not be called until a top-level
+ * visit_type_FOO() or visit_start_ITEM() has been performed on the
+ * visitor.  Safe if @v is NULL.
+ */
+void visit_free(Visitor *v);
+
+
+/*** Visiting structures ***/
+
+/*
+ * Start visiting an object @obj (struct or union).
+ *
+ * @name expresses the relationship of this object to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL for a real walk, in which case @size
+ * determines how much memory an input or clone visitor will allocate
+ * into *@obj.  @obj may also be NULL for a virtual walk, in which
+ * case @size is ignored.
+ *
+ * @errp obeys typical error usage, and reports failures such as a
+ * member @name is not present, or present but not an object.  On
+ * error, input visitors set *@obj to NULL.
+ *
+ * After visit_start_struct() succeeds, the caller may visit its
+ * members one after the other, passing the member's name and address
+ * within the struct.  Finally, visit_end_struct() needs to be called
+ * with the same @obj to clean up, even if intermediate visits fail.
+ * See the examples above.
+ *
+ * FIXME Should this be named visit_start_object, since it is also
+ * used for QAPI unions, and maps to JSON objects?
+ */
+void visit_start_struct(Visitor *v, const char *name, void **obj,
+                        size_t size, Error **errp);
+
+/*
+ * Prepare for completing an object visit.
+ *
+ * @errp obeys typical error usage, and reports failures such as
+ * unparsed keys remaining in the input stream.
+ *
+ * Should be called prior to visit_end_struct() if all other
+ * intermediate visit steps were successful, to allow the visitor one
+ * last chance to report errors.  May be skipped on a cleanup path,
+ * where there is no need to check for further errors.
+ */
+void visit_check_struct(Visitor *v, Error **errp);
+
+/*
+ * Complete an object visit started earlier.
+ *
+ * @obj must match what was passed to the paired visit_start_struct().
+ *
+ * Must be called after any successful use of visit_start_struct(),
+ * even if intermediate processing was skipped due to errors, to allow
+ * the backend to release any resources.  Destroying the visitor early
+ * with visit_free() behaves as if this was implicitly called.
+ */
+void visit_end_struct(Visitor *v, void **obj);
+
+
+/*** Visiting lists ***/
+
+/*
+ * Start visiting a list.
+ *
+ * @name expresses the relationship of this list to its parent
+ * container; see the general description of @name above.
+ *
+ * @list must be non-NULL for a real walk, in which case @size
+ * determines how much memory an input or clone visitor will allocate
+ * into *@list (at least sizeof(GenericList)).  Some visitors also
+ * allow @list to be NULL for a virtual walk, in which case @size is
+ * ignored.
+ *
+ * @errp obeys typical error usage, and reports failures such as a
+ * member @name is not present, or present but not a list.  On error,
+ * input visitors set *@list to NULL.
+ *
+ * After visit_start_list() succeeds, the caller may visit its members
+ * one after the other.  A real visit (where @obj is non-NULL) uses
+ * visit_next_list() for traversing the linked list, while a virtual
+ * visit (where @obj is NULL) uses other means.  For each list
+ * element, call the appropriate visit_type_FOO() with name set to
+ * NULL and obj set to the address of the value member of the list
+ * element.  Finally, visit_end_list() needs to be called with the
+ * same @list to clean up, even if intermediate visits fail.  See the
+ * examples above.
+ */
+void visit_start_list(Visitor *v, const char *name, GenericList **list,
+                      size_t size, Error **errp);
+
+/*
+ * Iterate over a GenericList during a non-virtual list visit.
+ *
+ * @size represents the size of a linked list node (at least
+ * sizeof(GenericList)).
+ *
+ * @tail must not be NULL; on the first call, @tail is the value of
+ * *list after visit_start_list(), and on subsequent calls @tail must
+ * be the previously returned value.  Should be called in a loop until
+ * a NULL return or error occurs; for each non-NULL return, the caller
+ * then calls the appropriate visit_type_*() for the element type of
+ * the list, with that function's name parameter set to NULL and obj
+ * set to the address of @tail->value.
+ */
+GenericList *visit_next_list(Visitor *v, GenericList *tail, size_t size);
+
+/*
+ * Complete a list visit started earlier.
+ *
+ * @list must match what was passed to the paired visit_start_list().
+ *
+ * Must be called after any successful use of visit_start_list(), even
+ * if intermediate processing was skipped due to errors, to allow the
+ * backend to release any resources.  Destroying the visitor early
+ * with visit_free() behaves as if this was implicitly called.
+ */
+void visit_end_list(Visitor *v, void **list);
+
+
+/*** Visiting alternates ***/
+
+/*
+ * Start the visit of an alternate @obj.
+ *
+ * @name expresses the relationship of this alternate to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must not be NULL. Input and clone visitors use @size to
+ * determine how much memory to allocate into *@obj, then determine
+ * the qtype of the next thing to be visited, stored in (*@obj)->type.
+ * Other visitors will leave @obj unchanged.
+ *
+ * If @promote_int, treat integers as QTYPE_FLOAT.
+ *
+ * If successful, this must be paired with visit_end_alternate() with
+ * the same @obj to clean up, even if visiting the contents of the
+ * alternate fails.
+ */
+void visit_start_alternate(Visitor *v, const char *name,
+                           GenericAlternate **obj, size_t size,
+                           bool promote_int, Error **errp);
+
+/*
+ * Finish visiting an alternate type.
+ *
+ * @obj must match what was passed to the paired visit_start_alternate().
+ *
+ * Must be called after any successful use of visit_start_alternate(),
+ * even if intermediate processing was skipped due to errors, to allow
+ * the backend to release any resources.  Destroying the visitor early
+ * with visit_free() behaves as if this was implicitly called.
+ *
+ */
+void visit_end_alternate(Visitor *v, void **obj);
+
+
+/*** Other helpers ***/
+
+/*
+ * Does optional struct member @name need visiting?
+ *
+ * @name must not be NULL.  This function is only useful between
+ * visit_start_struct() and visit_end_struct(), since only objects
+ * have optional keys.
+ *
+ * @present points to the address of the optional member's has_ flag.
+ *
+ * Input visitors set *@present according to input; other visitors
+ * leave it unchanged.  In either case, return *@present for
+ * convenience.
+ */
+bool visit_optional(Visitor *v, const char *name, bool *present);
+
+/*
+ * Visit an enum value.
+ *
+ * @name expresses the relationship of this enum to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL.  Input visitors parse input and set *@obj to
+ * the enumeration value, leaving @obj unchanged on error; other
+ * visitors use *@obj but leave it unchanged.
+ *
+ * Currently, all input visitors parse text input, and all output
+ * visitors produce text output.  The mapping between enumeration
+ * values and strings is done by the visitor core, using @strings; it
+ * should be the ENUM_lookup array from visit-types.h.
+ *
+ * May call visit_type_str() under the hood, and the enum visit may
+ * fail even if the corresponding string visit succeeded; this implies
+ * that visit_type_str() must have no unwelcome side effects.
+ */
+void visit_type_enum(Visitor *v, const char *name, int *obj,
+                     const char *const strings[], Error **errp);
+
+/*
+ * Check if visitor is an input visitor.
+ */
+bool visit_is_input(Visitor *v);
+
+/*** Visiting built-in types ***/
+
+/*
+ * Visit an integer value.
+ *
+ * @name expresses the relationship of this integer to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL.  Input visitors set *@obj to the value;
+ * other visitors will leave *@obj unchanged.
+ */
+void visit_type_int(Visitor *v, const char *name, int64_t *obj, Error **errp);
+
+/*
+ * Visit a uint8_t value.
+ * Like visit_type_int(), except clamps the value to uint8_t range.
+ */
+void visit_type_uint8(Visitor *v, const char *name, uint8_t *obj,
+                      Error **errp);
+
+/*
+ * Visit a uint16_t value.
+ * Like visit_type_int(), except clamps the value to uint16_t range.
+ */
+void visit_type_uint16(Visitor *v, const char *name, uint16_t *obj,
+                       Error **errp);
+
+/*
+ * Visit a uint32_t value.
+ * Like visit_type_int(), except clamps the value to uint32_t range.
+ */
+void visit_type_uint32(Visitor *v, const char *name, uint32_t *obj,
+                       Error **errp);
+
+/*
+ * Visit a uint64_t value.
+ * Like visit_type_int(), except clamps the value to uint64_t range,
+ * that is, ensures it is unsigned.
+ */
+void visit_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                       Error **errp);
+
+/*
+ * Visit an int8_t value.
+ * Like visit_type_int(), except clamps the value to int8_t range.
+ */
+void visit_type_int8(Visitor *v, const char *name, int8_t *obj, Error **errp);
+
+/*
+ * Visit an int16_t value.
+ * Like visit_type_int(), except clamps the value to int16_t range.
+ */
+void visit_type_int16(Visitor *v, const char *name, int16_t *obj,
+                      Error **errp);
+
+/*
+ * Visit an int32_t value.
+ * Like visit_type_int(), except clamps the value to int32_t range.
+ */
+void visit_type_int32(Visitor *v, const char *name, int32_t *obj,
+                      Error **errp);
+
+/*
+ * Visit an int64_t value.
+ * Identical to visit_type_int().
+ */
+void visit_type_int64(Visitor *v, const char *name, int64_t *obj,
+                      Error **errp);
+
+/*
+ * Visit a uint64_t value.
+ * Like visit_type_uint64(), except that some visitors may choose to
+ * recognize additional syntax, such as suffixes for easily scaling
+ * values.
+ */
+void visit_type_size(Visitor *v, const char *name, uint64_t *obj,
+                     Error **errp);
+
+/*
+ * Visit a boolean value.
+ *
+ * @name expresses the relationship of this boolean to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL.  Input visitors set *@obj to the value;
+ * other visitors will leave *@obj unchanged.
+ */
+void visit_type_bool(Visitor *v, const char *name, bool *obj, Error **errp);
+
+/*
+ * Visit a string value.
+ *
+ * @name expresses the relationship of this string to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL.  Input and clone visitors set *@obj to the
+ * value (always using "" rather than NULL for an empty string).
+ * Other visitors leave *@obj unchanged, and commonly treat NULL like
+ * "".
+ *
+ * It is safe to cast away const when preparing a (const char *) value
+ * into @obj for use by an output visitor.
+ *
+ * FIXME: Callers that try to output NULL *obj should not be allowed.
+ */
+void visit_type_str(Visitor *v, const char *name, char **obj, Error **errp);
+
+/*
+ * Visit a number (i.e. double) value.
+ *
+ * @name expresses the relationship of this number to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL.  Input visitors set *@obj to the value;
+ * other visitors will leave *@obj unchanged.  Visitors should
+ * document if infinity or NaN are not permitted.
+ */
+void visit_type_number(Visitor *v, const char *name, double *obj,
+                       Error **errp);
+
+/*
+ * Visit an arbitrary value.
+ *
+ * @name expresses the relationship of this value to its parent
+ * container; see the general description of @name above.
+ *
+ * @obj must be non-NULL.  Input visitors set *@obj to the value;
+ * other visitors will leave *@obj unchanged.  *@obj must be non-NULL
+ * for output visitors.
+ */
+void visit_type_any(Visitor *v, const char *name, QObject **obj, Error **errp);
+
+/*
+ * Visit a JSON null value.
+ *
+ * @name expresses the relationship of the null value to its parent
+ * container; see the general description of @name above.
+ *
+ * Unlike all other visit_type_* functions, no obj parameter is
+ * needed; rather, this is a witness that an explicit null value is
+ * expected rather than any other type.
+ */
+void visit_type_null(Visitor *v, const char *name, Error **errp);
 
 #endif
diff --git a/include/qemu-common.h b/include/qemu-common.h
index 47958c5..14f0f38 100644
--- a/include/qemu-common.h
+++ b/include/qemu-common.h
@@ -12,126 +12,17 @@
 #ifndef QEMU_COMMON_H
 #define QEMU_COMMON_H
 
-#include "qemu/compiler.h"
-#include "config-host.h"
-#include "qemu/typedefs.h"
-
-#if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__)
-#define WORDS_ALIGNED
-#endif
+#include "qemu/fprintf-fn.h"
 
 #define TFR(expr) do { if ((expr) != -1) break; } while (errno == EINTR)
 
-/* we put basic includes here to avoid repeating them in device drivers */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <string.h>
-#include <strings.h>
-#include <inttypes.h>
-#include <limits.h>
-#include <time.h>
-#include <ctype.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/stat.h>
-#include <sys/time.h>
-#include <assert.h>
-#include <signal.h>
-#include "glib-compat.h"
 #include "qemu/option.h"
 
-#ifdef _WIN32
-#include "sysemu/os-win32.h"
-#endif
+#include <stdbool.h>
 
-#ifdef CONFIG_POSIX
-#include "sysemu/os-posix.h"
-#endif
-
-#ifndef O_LARGEFILE
-#define O_LARGEFILE 0
-#endif
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-#ifndef ENOMEDIUM
-#define ENOMEDIUM ENODEV
-#endif
-#if !defined(ENOTSUP)
-#define ENOTSUP 4096
-#endif
-#if !defined(ECANCELED)
-#define ECANCELED 4097
-#endif
-#if !defined(EMEDIUMTYPE)
-#define EMEDIUMTYPE 4098
-#endif
-#ifndef TIME_MAX
-#define TIME_MAX LONG_MAX
-#endif
-
-/* HOST_LONG_BITS is the size of a native pointer in bits. */
-#if UINTPTR_MAX == UINT32_MAX
-# define HOST_LONG_BITS 32
-#elif UINTPTR_MAX == UINT64_MAX
-# define HOST_LONG_BITS 64
-#else
-# error Unknown pointer size
-#endif
-
-typedef int (*fprintf_function)(FILE *f, const char *fmt, ...)
-    GCC_FMT_ATTR(2, 3);
-
-#ifdef _WIN32
-#define fsync _commit
-#if !defined(lseek)
-# define lseek _lseeki64
-#endif
-int qemu_ftruncate64(int, int64_t);
-#if !defined(ftruncate)
-# define ftruncate qemu_ftruncate64
-#endif
-
-#ifdef CONFIG_ANDROID
-char *realpath_with_length(const char *path, char *resolved_path, int length);
-static inline char *realpath(const char *path, char *resolved_path)
-{
-    realpath_with_length(path, resolved_path, _MAX_PATH);
-    return resolved_path;
-}
-#else  // !CONFIG_ANDROID
-static inline char *realpath(const char *path, char *resolved_path)
-{
-    _fullpath(resolved_path, path, _MAX_PATH);
-    return resolved_path;
-}
-#endif  // CONFIG_ANDROID
-#endif
-
-void cpu_ticks_init(void);
-
-/* icount */
-void configure_icount(QemuOpts *opts, Error **errp);
-extern int use_icount;
-extern int icount_align_option;
-/* drift information for info jit command */
-extern int64_t max_delay;
-extern int64_t max_advance;
-void dump_drift_info(FILE *f, fprintf_function cpu_fprintf);
-
-#include "qemu/osdep.h"
-#include "qemu/bswap.h"
-
-/* FIXME: Remove NEED_CPU_H.  */
-#ifdef NEED_CPU_H
-#include "cpu.h"
-#endif /* !defined(NEED_CPU_H) */
+/* Copyright string for -version arguments, About dialogs, etc */
+#define QEMU_COPYRIGHT "Copyright (c) 2003-2016 " \
+    "Fabrice Bellard and the QEMU Project developers"
 
 /* main function, renamed */
 #if defined(CONFIG_COCOA)
@@ -141,71 +32,6 @@
 void qemu_get_timedate(struct tm *tm, int offset);
 int qemu_timedate_diff(struct tm *tm);
 
-/**
- * is_help_option:
- * @s: string to test
- *
- * Check whether @s is one of the standard strings which indicate
- * that the user is asking for a list of the valid values for a
- * command option like -cpu or -M. The current accepted strings
- * are 'help' and '?'. '?' is deprecated (it is a shell wildcard
- * which makes it annoying to use in a reliable way) but provided
- * for backwards compatibility.
- *
- * Returns: true if @s is a request for a list.
- */
-static inline bool is_help_option(const char *s)
-{
-    return !strcmp(s, "?") || !strcmp(s, "help");
-}
-
-/* cutils.c */
-void pstrcpy(char *buf, int buf_size, const char *str);
-void strpadcpy(char *buf, int buf_size, const char *str, char pad);
-char *pstrcat(char *buf, int buf_size, const char *s);
-int strstart(const char *str, const char *val, const char **ptr);
-int stristart(const char *str, const char *val, const char **ptr);
-int qemu_strnlen(const char *s, int max_len);
-char *qemu_strsep(char **input, const char *delim);
-time_t mktimegm(struct tm *tm);
-int qemu_fls(int i);
-int qemu_fdatasync(int fd);
-int fcntl_setfl(int fd, int flag);
-int qemu_parse_fd(const char *param);
-
-int parse_uint(const char *s, unsigned long long *value, char **endptr,
-               int base);
-int parse_uint_full(const char *s, unsigned long long *value, int base);
-
-/*
- * strtosz() suffixes used to specify the default treatment of an
- * argument passed to strtosz() without an explicit suffix.
- * These should be defined using upper case characters in the range
- * A-Z, as strtosz() will use qemu_toupper() on the given argument
- * prior to comparison.
- */
-#define STRTOSZ_DEFSUFFIX_EB	'E'
-#define STRTOSZ_DEFSUFFIX_PB	'P'
-#define STRTOSZ_DEFSUFFIX_TB	'T'
-#define STRTOSZ_DEFSUFFIX_GB	'G'
-#define STRTOSZ_DEFSUFFIX_MB	'M'
-#define STRTOSZ_DEFSUFFIX_KB	'K'
-#define STRTOSZ_DEFSUFFIX_B	'B'
-int64_t strtosz(const char *nptr, char **end);
-int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix);
-int64_t strtosz_suffix_unit(const char *nptr, char **end,
-                            const char default_suffix, int64_t unit);
-
-/* used to print char* safely */
-#define STR_OR_NULL(str) ((str) ? (str) : "null")
-
-/* id.c */
-bool id_wellformed(const char *id);
-
-/* path.c */
-void init_paths(const char *prefix);
-const char *path(const char *pathname);
-
 #define qemu_isalnum(c)		isalnum((unsigned char)(c))
 #define qemu_isalpha(c)		isalpha((unsigned char)(c))
 #define qemu_iscntrl(c)		iscntrl((unsigned char)(c))
@@ -226,10 +52,6 @@
 
 ssize_t qemu_write_full(int fd, const void *buf, size_t count)
     QEMU_WARN_UNUSED_RESULT;
-ssize_t qemu_send_full(int fd, const void *buf, size_t count, int flags)
-    QEMU_WARN_UNUSED_RESULT;
-ssize_t qemu_recv_full(int fd, void *buf, size_t count, int flags)
-    QEMU_WARN_UNUSED_RESULT;
 
 #ifndef _WIN32
 int qemu_pipe(int pipefd[2]);
@@ -256,54 +78,28 @@
     sendto(sockfd, buf, len, flags, destaddr, addrlen)
 #endif
 
-/* Error handling.  */
-
-void QEMU_NORETURN hw_error(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
-
-struct ParallelIOArg {
-    void *buffer;
-    int count;
-};
-
-typedef int (*DMA_transfer_handler) (void *opaque, int nchan, int pos, int size);
-
-typedef uint64_t pcibus_t;
-
-typedef struct PCIHostDeviceAddress {
-    unsigned int domain;
-    unsigned int bus;
-    unsigned int slot;
-    unsigned int function;
-} PCIHostDeviceAddress;
+/* ANDROID_BEGIN */
+#ifdef _WIN32
+/* For the Android emulator, filepath are UTF-8 encoded, which the default
+ * stat() doesn't support properly, so use an alternative implementation
+ * for Win32. */
+int qemu_stat(const char* filepath, struct stat* st);
+int qemu_lstat(const char* filepath, struct stat* st);
+#else
+#define qemu_stat stat
+#define qemu_lstat lstat
+#endif
+/* ANDROID_END */
 
 void tcg_exec_init(unsigned long tb_size);
 
 extern bool g_tcg_enabled;
-static inline bool tcg_enabled(void) {
+static __inline__ bool tcg_enabled(void) {
     return g_tcg_enabled;
 }
 
 void cpu_exec_init_all(void);
 
-/* CPU save/load.  */
-#ifdef CPU_SAVE_VERSION
-void cpu_save(QEMUFile *f, void *opaque);
-int cpu_load(QEMUFile *f, void *opaque, int version_id);
-#endif
-
-/* Unblock cpu */
-void qemu_cpu_kick_self(void);
-
-/* work queue */
-struct qemu_work_item {
-    struct qemu_work_item *next;
-    void (*func)(void *data);
-    void *data;
-    int done;
-    bool free;
-};
-
-
 /**
  * Sends a (part of) iovec down a socket, yielding when the socket is full, or
  * Receives data into a (part of) iovec from a socket,
@@ -327,36 +123,6 @@
 #define qemu_co_send(sockfd, buf, bytes) \
   qemu_co_send_recv(sockfd, buf, bytes, true)
 
-typedef struct QEMUIOVector {
-    struct iovec *iov;
-    int niov;
-    int nalloc;
-    size_t size;
-} QEMUIOVector;
-
-void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
-void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
-void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
-void qemu_iovec_concat(QEMUIOVector *dst,
-                       QEMUIOVector *src, size_t soffset, size_t sbytes);
-size_t qemu_iovec_concat_iov(QEMUIOVector *dst,
-                             struct iovec *src_iov, unsigned int src_cnt,
-                             size_t soffset, size_t sbytes);
-bool qemu_iovec_is_zero(QEMUIOVector *qiov);
-void qemu_iovec_destroy(QEMUIOVector *qiov);
-void qemu_iovec_reset(QEMUIOVector *qiov);
-size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
-                         void *buf, size_t bytes);
-size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
-                           const void *buf, size_t bytes);
-size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
-                         int fillc, size_t bytes);
-ssize_t qemu_iovec_compare(QEMUIOVector *a, QEMUIOVector *b);
-void qemu_iovec_clone(QEMUIOVector *dest, const QEMUIOVector *src, void *buf);
-void qemu_iovec_discard_back(QEMUIOVector *qiov, size_t bytes);
-
-bool buffer_is_zero(const void *buf, size_t len);
-
 void qemu_progress_init(int enabled, float min_skip);
 void qemu_progress_end(void);
 void qemu_progress_print(float delta, int max);
@@ -371,120 +137,24 @@
 char *os_find_datadir(void);
 void os_parse_cmd_args(int index, const char *optarg);
 
-/* Convert a byte between binary and BCD.  */
-static inline uint8_t to_bcd(uint8_t val)
-{
-    return ((val / 10) << 4) | (val % 10);
-}
-
-static inline uint8_t from_bcd(uint8_t val)
-{
-    return ((val >> 4) * 10) + (val & 0x0f);
-}
-
-/* compute with 96 bit intermediate result: (a*b)/c */
-static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
-{
-    union {
-        uint64_t ll;
-        struct {
-#ifdef HOST_WORDS_BIGENDIAN
-            uint32_t high, low;
-#else
-            uint32_t low, high;
-#endif
-        } l;
-    } u, res;
-    uint64_t rl, rh;
-
-    u.ll = a;
-    rl = (uint64_t)u.l.low * (uint64_t)b;
-    rh = (uint64_t)u.l.high * (uint64_t)b;
-    rh += (rl >> 32);
-    res.l.high = rh / c;
-    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
-    return res.ll;
-}
-
-/* Round number down to multiple */
-#define QEMU_ALIGN_DOWN(n, m) ((n) / (m) * (m))
-
-/* Round number up to multiple */
-#define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
-
-static inline bool is_power_of_2(uint64_t value)
-{
-    if (!value) {
-        return 0;
-    }
-
-    return !(value & (value - 1));
-}
-
-/* round down to the nearest power of 2*/
-int64_t pow2floor(int64_t value);
-
 #include "qemu/module.h"
 
 /*
- * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
- * Input is limited to 14-bit numbers
- */
-
-int uleb128_encode_small(uint8_t *out, uint32_t n);
-int uleb128_decode_small(const uint8_t *in, uint32_t *n);
-
-/* unicode.c */
-int mod_utf8_codepoint(const char *s, size_t n, char **end);
-
-/*
  * Hexdump a buffer to a file. An optional string prefix is added to every line
  */
 
 void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size);
 
-/* vector definitions */
-#ifdef __ALTIVEC__
-#include <altivec.h>
-/* The altivec.h header says we're allowed to undef these for
- * C++ compatibility.  Here we don't care about C++, but we
- * undef them anyway to avoid namespace pollution.
- */
-#undef vector
-#undef pixel
-#undef bool
-#define VECTYPE        __vector unsigned char
-#define SPLAT(p)       vec_splat(vec_ld(0, p), 0)
-#define ALL_EQ(v1, v2) vec_all_eq(v1, v2)
-/* altivec.h may redefine the bool macro as vector type.
- * Reset it to POSIX semantics. */
-#define bool _Bool
-#elif defined __SSE2__
-#include <emmintrin.h>
-#define VECTYPE        __m128i
-#define SPLAT(p)       _mm_set1_epi8(*(p))
-#define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF)
-#else
-#define VECTYPE        unsigned long
-#define SPLAT(p)       (*(p) * (~0UL / 255))
-#define ALL_EQ(v1, v2) ((v1) == (v2))
-#endif
-
-#define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8
-static inline bool
-can_use_buffer_find_nonzero_offset(const void *buf, size_t len)
-{
-    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
-                   * sizeof(VECTYPE)) == 0
-            && ((uintptr_t) buf) % sizeof(VECTYPE) == 0);
-}
-size_t buffer_find_nonzero_offset(const void *buf, size_t len);
-
 /*
  * helper to parse debug environment variables
  */
 int parse_debug_env(const char *name, int max, int initial);
 
 const char *qemu_ether_ntoa(const MACAddr *mac);
+void page_size_init(void);
+
+/* returns non-zero if dump is in progress, otherwise zero is
+ * returned. */
+bool dump_in_progress(void);
 
 #endif
diff --git a/include/qemu-io.h b/include/qemu-io.h
index 5d6006f..4d402b9 100644
--- a/include/qemu-io.h
+++ b/include/qemu-io.h
@@ -22,7 +22,7 @@
 
 #define CMD_FLAG_GLOBAL ((int)0x80000000) /* don't iterate "args" */
 
-typedef int (*cfunc_t)(BlockDriverState *bs, int argc, char **argv);
+typedef int (*cfunc_t)(BlockBackend *blk, int argc, char **argv);
 typedef void (*helpfunc_t)(void);
 
 typedef struct cmdinfo {
@@ -40,7 +40,7 @@
 
 extern bool qemuio_misalign;
 
-bool qemuio_command(BlockDriverState *bs, const char *cmd);
+bool qemuio_command(BlockBackend *blk, const char *cmd);
 
 void qemuio_add_command(const cmdinfo_t *ci);
 int qemuio_command_usage(const cmdinfo_t *ci);
diff --git a/include/qemu/abort.h b/include/qemu/abort.h
new file mode 100644
index 0000000..ab9819b
--- /dev/null
+++ b/include/qemu/abort.h
@@ -0,0 +1,54 @@
+/*
+ * QEMU abort handler
+ *
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_ABORT_H
+#define QEMU_ABORT_H
+
+#include "qemu/compiler.h"
+
+#include <stdarg.h>
+#include <stdbool.h>
+
+/* Abort the current program, |format| and following arguments are a printf-
+ * style formatting string explaining the reason for the failure */
+void QEMU_NORETURN qemu_abort(const char* format, ...)
+        GCC_FMT_ATTR(1, 2);
+
+/* Pointer to a function that takes a formatted string that explains why
+ * QEMU needs to abort. |format| is a printf-style string, followed by
+ * optional arguments in |args|.
+ */
+typedef void QEMU_NORETURN (*QemuAbortHandler)(const char *format,
+                                               va_list args);
+
+/* Change the QEMU abort handler to a specific implementation. The defaults
+ * prints the output to stderr() then calls abort(). NOTE: The |abort_handler|
+ * should not return. */
+void qemu_abort_set_handler(QemuAbortHandler abort_handler);
+
+/* Return true iff qemu_abort_set_handler() was called with a non-NULL
+ * |abort_handler| value. */
+bool qemu_abort_has_custom_handler(void);
+
+#endif  /* QEMU_ABORT_H */
diff --git a/include/qemu/acl.h b/include/qemu/acl.h
index 116487e..7c44119 100644
--- a/include/qemu/acl.h
+++ b/include/qemu/acl.h
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef __QEMU_ACL_H__
-#define __QEMU_ACL_H__
+#ifndef QEMU_ACL_H
+#define QEMU_ACL_H
 
 #include "qemu/queue.h"
 
@@ -63,12 +63,4 @@
 int qemu_acl_remove(qemu_acl *acl,
 		    const char *match);
 
-#endif /* __QEMU_ACL_H__ */
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- *  tab-width: 8
- * End:
- */
+#endif /* QEMU_ACL_H */
diff --git a/include/qemu/aes.h b/include/qemu/aes.h
deleted file mode 100644
index 0664dfb..0000000
--- a/include/qemu/aes.h
+++ /dev/null
@@ -1,69 +0,0 @@
-#ifndef QEMU_AES_H
-#define QEMU_AES_H
-
-#define AES_MAXNR 14
-#define AES_BLOCK_SIZE 16
-
-struct aes_key_st {
-    uint32_t rd_key[4 *(AES_MAXNR + 1)];
-    int rounds;
-};
-typedef struct aes_key_st AES_KEY;
-
-// FreeBSD has its own AES_set_decrypt_key in -lcrypto, avoid conflicts
-// AndroidEmu library also links against libcrypto
-#if defined(__FreeBSD__) || (defined(CONFIG_ANDROID))
-#define AES_set_encrypt_key QEMU_AES_set_encrypt_key
-#define AES_set_decrypt_key QEMU_AES_set_decrypt_key
-#define AES_encrypt QEMU_AES_encrypt
-#define AES_decrypt QEMU_AES_decrypt
-#define AES_cbc_encrypt QEMU_AES_cbc_encrypt
-#endif
-
-int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
-	AES_KEY *key);
-int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
-	AES_KEY *key);
-
-void AES_encrypt(const unsigned char *in, unsigned char *out,
-	const AES_KEY *key);
-void AES_decrypt(const unsigned char *in, unsigned char *out,
-	const AES_KEY *key);
-void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
-		     const unsigned long length, const AES_KEY *key,
-		     unsigned char *ivec, const int enc);
-
-extern const uint8_t AES_sbox[256];
-extern const uint8_t AES_isbox[256];
-
-/* AES ShiftRows and InvShiftRows */
-extern const uint8_t AES_shifts[16];
-extern const uint8_t AES_ishifts[16];
-
-/* AES InvMixColumns */
-/* AES_imc[x][0] = [x].[0e, 09, 0d, 0b]; */
-/* AES_imc[x][1] = [x].[0b, 0e, 09, 0d]; */
-/* AES_imc[x][2] = [x].[0d, 0b, 0e, 09]; */
-/* AES_imc[x][3] = [x].[09, 0d, 0b, 0e]; */
-extern const uint32_t AES_imc[256][4];
-
-/*
-AES_Te0[x] = S [x].[02, 01, 01, 03];
-AES_Te1[x] = S [x].[03, 02, 01, 01];
-AES_Te2[x] = S [x].[01, 03, 02, 01];
-AES_Te3[x] = S [x].[01, 01, 03, 02];
-AES_Te4[x] = S [x].[01, 01, 01, 01];
-
-AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
-AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
-AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
-AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
-AES_Td4[x] = Si[x].[01, 01, 01, 01];
-*/
-
-extern const uint32_t AES_Te0[256], AES_Te1[256], AES_Te2[256],
-                      AES_Te3[256], AES_Te4[256];
-extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256],
-                      AES_Td3[256], AES_Td4[256];
-
-#endif
diff --git a/include/qemu/atomic.h b/include/qemu/atomic.h
index 492bce1..04ad8de 100644
--- a/include/qemu/atomic.h
+++ b/include/qemu/atomic.h
@@ -8,19 +8,213 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
+ * See docs/atomics.txt for discussion about the guarantees each
+ * atomic primitive is meant to provide.
  */
 
-#ifndef __QEMU_ATOMIC_H
-#define __QEMU_ATOMIC_H 1
-
-#include "qemu/compiler.h"
-
-/* For C11 atomic ops */
+#ifndef QEMU_ATOMIC_H
+#define QEMU_ATOMIC_H
 
 /* Compiler barrier */
 #define barrier()   ({ asm volatile("" ::: "memory"); (void)0; })
 
-#ifndef __ATOMIC_RELAXED
+/* The variable that receives the old value of an atomically-accessed
+ * variable must be non-qualified, because atomic builtins return values
+ * through a pointer-type argument as in __atomic_load(&var, &old, MODEL).
+ *
+ * This macro has to handle types smaller than int manually, because of
+ * implicit promotion.  int and larger types, as well as pointers, can be
+ * converted to a non-qualified type just by applying a binary operator.
+ */
+#ifdef __cplusplus
+#define typeof_strip_qual(expr) std::remove_cv<__typeof__(expr)>::type
+#else  /* !__cplusplus */
+#define typeof_strip_qual(expr)                                                    \
+  typeof(                                                                          \
+    __builtin_choose_expr(                                                         \
+      __builtin_types_compatible_p(typeof(expr), bool) ||                          \
+        __builtin_types_compatible_p(typeof(expr), const bool) ||                  \
+        __builtin_types_compatible_p(typeof(expr), volatile bool) ||               \
+        __builtin_types_compatible_p(typeof(expr), const volatile bool),           \
+        (bool)1,                                                                   \
+    __builtin_choose_expr(                                                         \
+      __builtin_types_compatible_p(typeof(expr), signed char) ||                   \
+        __builtin_types_compatible_p(typeof(expr), const signed char) ||           \
+        __builtin_types_compatible_p(typeof(expr), volatile signed char) ||        \
+        __builtin_types_compatible_p(typeof(expr), const volatile signed char),    \
+        (signed char)1,                                                            \
+    __builtin_choose_expr(                                                         \
+      __builtin_types_compatible_p(typeof(expr), unsigned char) ||                 \
+        __builtin_types_compatible_p(typeof(expr), const unsigned char) ||         \
+        __builtin_types_compatible_p(typeof(expr), volatile unsigned char) ||      \
+        __builtin_types_compatible_p(typeof(expr), const volatile unsigned char),  \
+        (unsigned char)1,                                                          \
+    __builtin_choose_expr(                                                         \
+      __builtin_types_compatible_p(typeof(expr), signed short) ||                  \
+        __builtin_types_compatible_p(typeof(expr), const signed short) ||          \
+        __builtin_types_compatible_p(typeof(expr), volatile signed short) ||       \
+        __builtin_types_compatible_p(typeof(expr), const volatile signed short),   \
+        (signed short)1,                                                           \
+    __builtin_choose_expr(                                                         \
+      __builtin_types_compatible_p(typeof(expr), unsigned short) ||                \
+        __builtin_types_compatible_p(typeof(expr), const unsigned short) ||        \
+        __builtin_types_compatible_p(typeof(expr), volatile unsigned short) ||     \
+        __builtin_types_compatible_p(typeof(expr), const volatile unsigned short), \
+        (unsigned short)1,                                                         \
+      (expr)+0))))))
+#endif  /* !__cplusplus */
+
+#ifdef __ATOMIC_RELAXED
+/* For C11 atomic ops */
+
+/* Manual memory barriers
+ *
+ *__atomic_thread_fence does not include a compiler barrier; instead,
+ * the barrier is part of __atomic_load/__atomic_store's "volatile-like"
+ * semantics. If smp_wmb() is a no-op, absence of the barrier means that
+ * the compiler is free to reorder stores on each side of the barrier.
+ * Add one here, and similarly in smp_rmb() and smp_read_barrier_depends().
+ */
+
+#define smp_mb()    ({ barrier(); __atomic_thread_fence(__ATOMIC_SEQ_CST); barrier(); })
+#define smp_wmb()   ({ barrier(); __atomic_thread_fence(__ATOMIC_RELEASE); barrier(); })
+#define smp_rmb()   ({ barrier(); __atomic_thread_fence(__ATOMIC_ACQUIRE); barrier(); })
+
+/* Most compilers currently treat consume and acquire the same, but really
+ * no processors except Alpha need a barrier here.  Leave it in if
+ * using Thread Sanitizer to avoid warnings, otherwise optimize it away.
+ */
+#if defined(__SANITIZE_THREAD__)
+#define smp_read_barrier_depends() ({ barrier(); __atomic_thread_fence(__ATOMIC_CONSUME); barrier(); })
+#elsif defined(__alpha__)
+#define smp_read_barrier_depends()   asm volatile("mb":::"memory")
+#else
+#define smp_read_barrier_depends()   barrier()
+#endif
+
+
+/* Weak atomic operations prevent the compiler moving other
+ * loads/stores past the atomic operation load/store. However there is
+ * no explicit memory barrier for the processor.
+ */
+#define atomic_read(ptr)                              \
+    ({                                                \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof_strip_qual(*ptr) _val;                     \
+     __atomic_load(ptr, &_val, __ATOMIC_RELAXED);     \
+    _val;                                             \
+    })
+
+#define atomic_set(ptr, i)  do {                      \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof(*ptr) _val = (i);                          \
+    __atomic_store(ptr, &_val, __ATOMIC_RELAXED);     \
+} while(0)
+
+/* See above: most compilers currently treat consume and acquire the
+ * same, but this slows down atomic_rcu_read unnecessarily.
+ */
+#ifdef __SANITIZE_THREAD__
+#define atomic_rcu_read__nocheck(ptr, valptr)           \
+    __atomic_load(ptr, valptr, __ATOMIC_CONSUME);
+#else
+#define atomic_rcu_read__nocheck(ptr, valptr)           \
+    __atomic_load(ptr, valptr, __ATOMIC_RELAXED);       \
+    smp_read_barrier_depends();
+#endif
+
+#define atomic_rcu_read(ptr)                          \
+    ({                                                \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof_strip_qual(*ptr) _val;                     \
+    atomic_rcu_read__nocheck(ptr, &_val);             \
+    _val;                                             \
+    })
+
+#define atomic_rcu_set(ptr, i) do {                   \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *)); \
+    typeof(*ptr) _val = (i);                          \
+    __atomic_store(ptr, &_val, __ATOMIC_RELEASE);     \
+} while(0)
+
+/* atomic_mb_read/set semantics map Java volatile variables. They are
+ * less expensive on some platforms (notably POWER & ARMv7) than fully
+ * sequentially consistent operations.
+ *
+ * As long as they are used as paired operations they are safe to
+ * use. See docs/atomic.txt for more discussion.
+ */
+
+#if defined(_ARCH_PPC)
+#define atomic_mb_read(ptr)                             \
+    ({                                                  \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof_strip_qual(*ptr) _val;                       \
+     __atomic_load(ptr, &_val, __ATOMIC_RELAXED);       \
+     smp_rmb();                                         \
+    _val;                                               \
+    })
+
+#define atomic_mb_set(ptr, i)  do {                     \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof(*ptr) _val = (i);                            \
+    smp_wmb();                                          \
+    __atomic_store(ptr, &_val, __ATOMIC_RELAXED);       \
+    smp_mb();                                           \
+} while(0)
+#else
+#define atomic_mb_read(ptr)                             \
+    ({                                                  \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof_strip_qual(*ptr) _val;                       \
+    __atomic_load(ptr, &_val, __ATOMIC_SEQ_CST);        \
+    _val;                                               \
+    })
+
+#define atomic_mb_set(ptr, i)  do {                     \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));   \
+    typeof(*ptr) _val = (i);                            \
+    __atomic_store(ptr, &_val, __ATOMIC_SEQ_CST);       \
+} while(0)
+#endif
+
+
+/* All the remaining operations are fully sequentially consistent */
+
+#define atomic_xchg(ptr, i)    ({                           \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));       \
+    typeof_strip_qual(*ptr) _new = (i), _old;               \
+    __atomic_exchange(ptr, &_new, &_old, __ATOMIC_SEQ_CST); \
+    _old;                                                   \
+})
+
+/* Returns the eventual value, failed or not */
+#define atomic_cmpxchg(ptr, old, new)                                   \
+    ({                                                                  \
+    QEMU_BUILD_BUG_ON(sizeof(*ptr) > sizeof(void *));                   \
+    typeof_strip_qual(*ptr) _old = (old), _new = (new);                 \
+    __atomic_compare_exchange(ptr, &_old, &_new, false,                 \
+                              __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);      \
+    _old;                                                               \
+    })
+
+/* Provide shorter names for GCC atomic builtins, return old value */
+#define atomic_fetch_inc(ptr)  __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST)
+#define atomic_fetch_dec(ptr)  __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST)
+#define atomic_fetch_add(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST)
+#define atomic_fetch_sub(ptr, n) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST)
+#define atomic_fetch_and(ptr, n) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST)
+#define atomic_fetch_or(ptr, n)  __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST)
+
+/* And even shorter names that return void.  */
+#define atomic_inc(ptr)    ((void) __atomic_fetch_add(ptr, 1, __ATOMIC_SEQ_CST))
+#define atomic_dec(ptr)    ((void) __atomic_fetch_sub(ptr, 1, __ATOMIC_SEQ_CST))
+#define atomic_add(ptr, n) ((void) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST))
+#define atomic_sub(ptr, n) ((void) __atomic_fetch_sub(ptr, n, __ATOMIC_SEQ_CST))
+#define atomic_and(ptr, n) ((void) __atomic_fetch_and(ptr, n, __ATOMIC_SEQ_CST))
+#define atomic_or(ptr, n)  ((void) __atomic_fetch_or(ptr, n, __ATOMIC_SEQ_CST))
+
+#else /* __ATOMIC_RELAXED */
 
 /*
  * We use GCC builtin if it's available, as that can use mfence on
@@ -85,8 +279,6 @@
 
 #endif /* _ARCH_PPC */
 
-#endif /* C11 atomics */
-
 /*
  * For (host) platforms we don't have explicit barrier definitions
  * for, we use the gcc __sync_synchronize() primitive to generate a
@@ -98,36 +290,62 @@
 #endif
 
 #ifndef smp_wmb
-#ifdef __ATOMIC_RELEASE
-#define smp_wmb()   __atomic_thread_fence(__ATOMIC_RELEASE)
-#else
 #define smp_wmb()   __sync_synchronize()
 #endif
-#endif
 
 #ifndef smp_rmb
-#ifdef __ATOMIC_ACQUIRE
-#define smp_rmb()   __atomic_thread_fence(__ATOMIC_ACQUIRE)
-#else
 #define smp_rmb()   __sync_synchronize()
 #endif
-#endif
 
 #ifndef smp_read_barrier_depends
-#ifdef __ATOMIC_CONSUME
-#define smp_read_barrier_depends()   __atomic_thread_fence(__ATOMIC_CONSUME)
-#else
 #define smp_read_barrier_depends()   barrier()
 #endif
-#endif
 
-#ifndef atomic_read
-#define atomic_read(ptr)       (*(__typeof__(*ptr) *volatile) (ptr))
-#endif
+/* These will only be atomic if the processor does the fetch or store
+ * in a single issue memory operation
+ */
+#define atomic_read(ptr)       (*(__typeof__(*ptr) volatile*) (ptr))
+#define atomic_set(ptr, i)     ((*(__typeof__(*ptr) volatile*) (ptr)) = (i))
 
-#ifndef atomic_set
-#define atomic_set(ptr, i)     ((*(__typeof__(*ptr) *volatile) (ptr)) = (i))
-#endif
+/**
+ * atomic_rcu_read - reads a RCU-protected pointer to a local variable
+ * into a RCU read-side critical section. The pointer can later be safely
+ * dereferenced within the critical section.
+ *
+ * This ensures that the pointer copy is invariant thorough the whole critical
+ * section.
+ *
+ * Inserts memory barriers on architectures that require them (currently only
+ * Alpha) and documents which pointers are protected by RCU.
+ *
+ * atomic_rcu_read also includes a compiler barrier to ensure that
+ * value-speculative optimizations (e.g. VSS: Value Speculation
+ * Scheduling) does not perform the data read before the pointer read
+ * by speculating the value of the pointer.
+ *
+ * Should match atomic_rcu_set(), atomic_xchg(), atomic_cmpxchg().
+ */
+#define atomic_rcu_read(ptr)    ({                \
+    typeof(*ptr) _val = atomic_read(ptr);         \
+    smp_read_barrier_depends();                   \
+    _val;                                         \
+})
+
+/**
+ * atomic_rcu_set - assigns (publicizes) a pointer to a new data structure
+ * meant to be read by RCU read-side critical sections.
+ *
+ * Documents which pointers will be dereferenced by RCU read-side critical
+ * sections and adds the required memory barriers on architectures requiring
+ * them. It also makes sure the compiler does not reorder code initializing the
+ * data structure before its publication.
+ *
+ * Should match atomic_rcu_read().
+ */
+#define atomic_rcu_set(ptr, i)  do {              \
+    smp_wmb();                                    \
+    atomic_set(ptr, i);                           \
+} while (0)
 
 /* These have the same semantics as Java volatile variables.
  * See http://gee.cs.oswego.edu/dl/jmm/cookbook.html:
@@ -151,13 +369,11 @@
  * (see docs/atomics.txt), and I'm not sure that __ATOMIC_ACQ_REL is enough.
  * Just always use the barriers manually by the rules above.
  */
-#ifndef atomic_mb_read
 #define atomic_mb_read(ptr)    ({           \
     typeof(*ptr) _val = atomic_read(ptr);   \
     smp_rmb();                              \
     _val;                                   \
 })
-#endif
 
 #ifndef atomic_mb_set
 #define atomic_mb_set(ptr, i)  do {         \
@@ -170,12 +386,6 @@
 #ifndef atomic_xchg
 #if defined(__clang__)
 #define atomic_xchg(ptr, i)    __sync_swap(ptr, i)
-#elif defined(__ATOMIC_SEQ_CST)
-#define atomic_xchg(ptr, i)    ({                           \
-    typeof(*ptr) _new = (i), _old;                          \
-    __atomic_exchange(ptr, &_new, &_old, __ATOMIC_SEQ_CST); \
-    _old;                                                   \
-})
 #else
 /* __sync_lock_test_and_set() is documented to be an acquire barrier only.  */
 #define atomic_xchg(ptr, i)    (smp_mb(), __sync_lock_test_and_set(ptr, i))
@@ -199,4 +409,5 @@
 #define atomic_and(ptr, n)     ((void) __sync_fetch_and_and(ptr, n))
 #define atomic_or(ptr, n)      ((void) __sync_fetch_and_or(ptr, n))
 
-#endif
+#endif /* __ATOMIC_RELAXED */
+#endif /* QEMU_ATOMIC_H */
diff --git a/include/qemu/base64.h b/include/qemu/base64.h
new file mode 100644
index 0000000..815d852
--- /dev/null
+++ b/include/qemu/base64.h
@@ -0,0 +1,58 @@
+/*
+ * QEMU base64 helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QEMU_BASE64_H
+#define QEMU_BASE64_H
+
+#include "qemu-common.h"
+
+
+/**
+ * qbase64_decode:
+ * @input: the (possibly) base64 encoded text
+ * @in_len: length of @input or -1 if NUL terminated
+ * @out_len: filled with length of decoded data
+ * @errp: pointer to a NULL-initialized error object
+ *
+ * Attempt to decode the (possibly) base64 encoded
+ * text provided in @input. If the @input text may
+ * contain embedded NUL characters, or may not be
+ * NUL terminated, then @in_len must be set to the
+ * known size of the @input buffer.
+ *
+ * Note that embedded NULs, or lack of a NUL terminator
+ * are considered invalid base64 data and errors
+ * will be reported to this effect.
+ *
+ * If decoding is successful, the decoded data will
+ * be returned and @out_len set to indicate the
+ * number of bytes in the decoded data. The caller
+ * must use g_free() to free the returned data when
+ * it is no longer required.
+ *
+ * Returns: the decoded data or NULL
+ */
+uint8_t *qbase64_decode(const char *input,
+                        size_t in_len,
+                        size_t *out_len,
+                        Error **errp);
+
+
+#endif /* QEMU_BASE64_H */
diff --git a/include/qemu/bcd.h b/include/qemu/bcd.h
new file mode 100644
index 0000000..dfebacf
--- /dev/null
+++ b/include/qemu/bcd.h
@@ -0,0 +1,15 @@
+#ifndef QEMU_BCD_H
+#define QEMU_BCD_H
+
+/* Convert a byte between binary and BCD.  */
+static inline uint8_t to_bcd(uint8_t val)
+{
+    return ((val / 10) << 4) | (val % 10);
+}
+
+static inline uint8_t from_bcd(uint8_t val)
+{
+    return ((val >> 4) * 10) + (val & 0x0f);
+}
+
+#endif
diff --git a/include/qemu/bitmap.h b/include/qemu/bitmap.h
index f0273c9..ec5146f 100644
--- a/include/qemu/bitmap.h
+++ b/include/qemu/bitmap.h
@@ -12,11 +12,7 @@
 #ifndef BITMAP_H
 #define BITMAP_H
 
-#include <glib.h>
-#include <string.h>
-#include <stdlib.h>
 
-#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
 /*
@@ -39,7 +35,9 @@
  * bitmap_empty(src, nbits)			Are all bits zero in *src?
  * bitmap_full(src, nbits)			Are all bits set in *src?
  * bitmap_set(dst, pos, nbits)			Set specified bit area
+ * bitmap_set_atomic(dst, pos, nbits)   Set specified bit area with atomic ops
  * bitmap_clear(dst, pos, nbits)		Clear specified bit area
+ * bitmap_test_and_clear_atomic(dst, pos, nbits)    Test and clear area
  * bitmap_find_next_zero_area(buf, len, pos, n, mask)	Find bit free area
  */
 
@@ -226,7 +224,9 @@
 }
 
 void bitmap_set(unsigned long *map, long i, long len);
+void bitmap_set_atomic(unsigned long *map, long i, long len);
 void bitmap_clear(unsigned long *map, long start, long nr);
+bool bitmap_test_and_clear_atomic(unsigned long *map, long start, long nr);
 unsigned long bitmap_find_next_zero_area(unsigned long *map,
                                          unsigned long size,
                                          unsigned long start,
diff --git a/include/qemu/bitops.h b/include/qemu/bitops.h
index 181bd46..98fb005 100644
--- a/include/qemu/bitops.h
+++ b/include/qemu/bitops.h
@@ -12,18 +12,20 @@
 #ifndef BITOPS_H
 #define BITOPS_H
 
-#include <stdint.h>
-#include <assert.h>
 
 #include "host-utils.h"
+#include "atomic.h"
 
 #define BITS_PER_BYTE           CHAR_BIT
 #define BITS_PER_LONG           (sizeof (unsigned long) * BITS_PER_BYTE)
 
-#define BIT(nr)			(1UL << (nr))
-#define BIT_MASK(nr)		(1UL << ((nr) % BITS_PER_LONG))
-#define BIT_WORD(nr)		((nr) / BITS_PER_LONG)
-#define BITS_TO_LONGS(nr)	DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
+#define BIT(nr)                 (1UL << (nr))
+#define BIT_MASK(nr)            (1UL << ((nr) % BITS_PER_LONG))
+#define BIT_WORD(nr)            ((nr) / BITS_PER_LONG)
+#define BITS_TO_LONGS(nr)       DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
+
+#define MAKE_64BIT_MASK(shift, length) \
+    (((~0ULL) >> (64 - (length))) << (shift))
 
 /**
  * set_bit - Set a bit in memory
@@ -32,10 +34,23 @@
  */
 static inline void set_bit(long nr, unsigned long *addr)
 {
-	unsigned long mask = BIT_MASK(nr);
-        unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
 
-	*p  |= mask;
+    *p  |= mask;
+}
+
+/**
+ * set_bit_atomic - Set a bit in memory atomically
+ * @nr: the bit to set
+ * @addr: the address to start counting from
+ */
+static inline void set_bit_atomic(long nr, unsigned long *addr)
+{
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
+
+    atomic_or(p, mask);
 }
 
 /**
@@ -45,10 +60,10 @@
  */
 static inline void clear_bit(long nr, unsigned long *addr)
 {
-	unsigned long mask = BIT_MASK(nr);
-        unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
 
-	*p &= ~mask;
+    *p &= ~mask;
 }
 
 /**
@@ -58,10 +73,10 @@
  */
 static inline void change_bit(long nr, unsigned long *addr)
 {
-	unsigned long mask = BIT_MASK(nr);
-        unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
 
-	*p ^= mask;
+    *p ^= mask;
 }
 
 /**
@@ -71,12 +86,12 @@
  */
 static inline int test_and_set_bit(long nr, unsigned long *addr)
 {
-	unsigned long mask = BIT_MASK(nr);
-        unsigned long *p = addr + BIT_WORD(nr);
-	unsigned long old = *p;
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long old = *p;
 
-	*p = old | mask;
-	return (old & mask) != 0;
+    *p = old | mask;
+    return (old & mask) != 0;
 }
 
 /**
@@ -86,12 +101,12 @@
  */
 static inline int test_and_clear_bit(long nr, unsigned long *addr)
 {
-	unsigned long mask = BIT_MASK(nr);
-        unsigned long *p = addr + BIT_WORD(nr);
-	unsigned long old = *p;
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long old = *p;
 
-	*p = old & ~mask;
-	return (old & mask) != 0;
+    *p = old & ~mask;
+    return (old & mask) != 0;
 }
 
 /**
@@ -101,12 +116,12 @@
  */
 static inline int test_and_change_bit(long nr, unsigned long *addr)
 {
-	unsigned long mask = BIT_MASK(nr);
-        unsigned long *p = addr + BIT_WORD(nr);
-	unsigned long old = *p;
+    unsigned long mask = BIT_MASK(nr);
+    unsigned long *p = addr + BIT_WORD(nr);
+    unsigned long old = *p;
 
-	*p = old ^ mask;
-	return (old & mask) != 0;
+    *p = old ^ mask;
+    return (old & mask) != 0;
 }
 
 /**
@@ -116,7 +131,7 @@
  */
 static inline int test_bit(long nr, const unsigned long *addr)
 {
-	return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
+    return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
 }
 
 /**
@@ -136,7 +151,8 @@
  * @size: The bitmap size in bits
  */
 unsigned long find_next_bit(const unsigned long *addr,
-				   unsigned long size, unsigned long offset);
+                            unsigned long size,
+                            unsigned long offset);
 
 /**
  * find_next_zero_bit - find the next cleared bit in a memory region
@@ -354,7 +370,7 @@
  * Returns: the sign extended value of the bit field extracted from the
  * input value.
  */
-static inline uint64_t sextract64(uint64_t value, int start, int length)
+static inline int64_t sextract64(uint64_t value, int start, int length)
 {
     assert(start >= 0 && length > 0 && length <= 64 - start);
     /* Note that this implementation relies on right shift of signed
@@ -415,4 +431,112 @@
     return (value & ~mask) | ((fieldval << start) & mask);
 }
 
+/**
+ * half_shuffle32:
+ * @value: 32-bit value (of which only the bottom 16 bits are of interest)
+ *
+ * Given an input value:
+ *  xxxx xxxx xxxx xxxx ABCD EFGH IJKL MNOP
+ * return the value where the bottom 16 bits are spread out into
+ * the odd bits in the word, and the even bits are zeroed:
+ *  0A0B 0C0D 0E0F 0G0H 0I0J 0K0L 0M0N 0O0P
+ *
+ * Any bits set in the top half of the input are ignored.
+ *
+ * Returns: the shuffled bits.
+ */
+static inline uint32_t half_shuffle32(uint32_t x)
+{
+    /* This algorithm is from _Hacker's Delight_ section 7-2 "Shuffling Bits".
+     * It ignores any bits set in the top half of the input.
+     */
+    x = ((x & 0xFF00) << 8) | (x & 0x00FF);
+    x = ((x << 4) | x) & 0x0F0F0F0F;
+    x = ((x << 2) | x) & 0x33333333;
+    x = ((x << 1) | x) & 0x55555555;
+    return x;
+}
+
+/**
+ * half_shuffle64:
+ * @value: 64-bit value (of which only the bottom 32 bits are of interest)
+ *
+ * Given an input value:
+ *  xxxx xxxx xxxx .... xxxx xxxx ABCD EFGH IJKL MNOP QRST UVWX YZab cdef
+ * return the value where the bottom 32 bits are spread out into
+ * the odd bits in the word, and the even bits are zeroed:
+ *  0A0B 0C0D 0E0F 0G0H 0I0J 0K0L 0M0N .... 0U0V 0W0X 0Y0Z 0a0b 0c0d 0e0f
+ *
+ * Any bits set in the top half of the input are ignored.
+ *
+ * Returns: the shuffled bits.
+ */
+static inline uint64_t half_shuffle64(uint64_t x)
+{
+    /* This algorithm is from _Hacker's Delight_ section 7-2 "Shuffling Bits".
+     * It ignores any bits set in the top half of the input.
+     */
+    x = ((x & 0xFFFF0000ULL) << 16) | (x & 0xFFFF);
+    x = ((x << 8) | x) & 0x00FF00FF00FF00FFULL;
+    x = ((x << 4) | x) & 0x0F0F0F0F0F0F0F0FULL;
+    x = ((x << 2) | x) & 0x3333333333333333ULL;
+    x = ((x << 1) | x) & 0x5555555555555555ULL;
+    return x;
+}
+
+/**
+ * half_unshuffle32:
+ * @value: 32-bit value (of which only the odd bits are of interest)
+ *
+ * Given an input value:
+ *  xAxB xCxD xExF xGxH xIxJ xKxL xMxN xOxP
+ * return the value where all the odd bits are compressed down
+ * into the low half of the word, and the high half is zeroed:
+ *  0000 0000 0000 0000 ABCD EFGH IJKL MNOP
+ *
+ * Any even bits set in the input are ignored.
+ *
+ * Returns: the unshuffled bits.
+ */
+static inline uint32_t half_unshuffle32(uint32_t x)
+{
+    /* This algorithm is from _Hacker's Delight_ section 7-2 "Shuffling Bits".
+     * where it is called an inverse half shuffle.
+     */
+    x &= 0x55555555;
+    x = ((x >> 1) | x) & 0x33333333;
+    x = ((x >> 2) | x) & 0x0F0F0F0F;
+    x = ((x >> 4) | x) & 0x00FF00FF;
+    x = ((x >> 8) | x) & 0x0000FFFF;
+    return x;
+}
+
+/**
+ * half_unshuffle64:
+ * @value: 64-bit value (of which only the odd bits are of interest)
+ *
+ * Given an input value:
+ *  xAxB xCxD xExF xGxH xIxJ xKxL xMxN .... xUxV xWxX xYxZ xaxb xcxd xexf
+ * return the value where all the odd bits are compressed down
+ * into the low half of the word, and the high half is zeroed:
+ *  0000 0000 0000 .... 0000 0000 ABCD EFGH IJKL MNOP QRST UVWX YZab cdef
+ *
+ * Any even bits set in the input are ignored.
+ *
+ * Returns: the unshuffled bits.
+ */
+static inline uint64_t half_unshuffle64(uint64_t x)
+{
+    /* This algorithm is from _Hacker's Delight_ section 7-2 "Shuffling Bits".
+     * where it is called an inverse half shuffle.
+     */
+    x &= 0x5555555555555555ULL;
+    x = ((x >> 1) | x) & 0x3333333333333333ULL;
+    x = ((x >> 2) | x) & 0x0F0F0F0F0F0F0F0FULL;
+    x = ((x >> 4) | x) & 0x00FF00FF00FF00FFULL;
+    x = ((x >> 8) | x) & 0x0000FFFF0000FFFFULL;
+    x = ((x >> 16) | x) & 0x00000000FFFFFFFFULL;
+    return x;
+}
+
 #endif
diff --git a/include/qemu/bswap.h b/include/qemu/bswap.h
index 78c1ced..09c78fd 100644
--- a/include/qemu/bswap.h
+++ b/include/qemu/bswap.h
@@ -1,15 +1,10 @@
 #ifndef BSWAP_H
 #define BSWAP_H
 
-#include "config-host.h"
-#include <inttypes.h>
-#include <limits.h>
-#include <string.h>
 #include "fpu/softfloat.h"
 
 #ifdef CONFIG_MACHINE_BSWAP_H
 # include <sys/endian.h>
-# include <sys/types.h>
 # include <machine/bswap.h>
 #elif defined(__FreeBSD__)
 # include <sys/endian.h>
@@ -85,6 +80,64 @@
 #define be_bswaps(p, size) do { *p = glue(bswap, size)(*p); } while(0)
 #endif
 
+/**
+ * Endianness conversion functions between host cpu and specified endianness.
+ * (We list the complete set of prototypes produced by the macros below
+ * to assist people who search the headers to find their definitions.)
+ *
+ * uint16_t le16_to_cpu(uint16_t v);
+ * uint32_t le32_to_cpu(uint32_t v);
+ * uint64_t le64_to_cpu(uint64_t v);
+ * uint16_t be16_to_cpu(uint16_t v);
+ * uint32_t be32_to_cpu(uint32_t v);
+ * uint64_t be64_to_cpu(uint64_t v);
+ *
+ * Convert the value @v from the specified format to the native
+ * endianness of the host CPU by byteswapping if necessary, and
+ * return the converted value.
+ *
+ * uint16_t cpu_to_le16(uint16_t v);
+ * uint32_t cpu_to_le32(uint32_t v);
+ * uint64_t cpu_to_le64(uint64_t v);
+ * uint16_t cpu_to_be16(uint16_t v);
+ * uint32_t cpu_to_be32(uint32_t v);
+ * uint64_t cpu_to_be64(uint64_t v);
+ *
+ * Convert the value @v from the native endianness of the host CPU to
+ * the specified format by byteswapping if necessary, and return
+ * the converted value.
+ *
+ * void le16_to_cpus(uint16_t *v);
+ * void le32_to_cpus(uint32_t *v);
+ * void le64_to_cpus(uint64_t *v);
+ * void be16_to_cpus(uint16_t *v);
+ * void be32_to_cpus(uint32_t *v);
+ * void be64_to_cpus(uint64_t *v);
+ *
+ * Do an in-place conversion of the value pointed to by @v from the
+ * specified format to the native endianness of the host CPU.
+ *
+ * void cpu_to_le16s(uint16_t *v);
+ * void cpu_to_le32s(uint32_t *v);
+ * void cpu_to_le64s(uint64_t *v);
+ * void cpu_to_be16s(uint16_t *v);
+ * void cpu_to_be32s(uint32_t *v);
+ * void cpu_to_be64s(uint64_t *v);
+ *
+ * Do an in-place conversion of the value pointed to by @v from the
+ * native endianness of the host CPU to the specified format.
+ *
+ * Both X_to_cpu() and cpu_to_X() perform the same operation; you
+ * should use whichever one is better documenting of the function your
+ * code is performing.
+ *
+ * Do not use these functions for conversion of values which are in guest
+ * memory, since the data may not be sufficiently aligned for the host CPU's
+ * load and store instructions. Instead you should use the ld*_p() and
+ * st*_p() functions, which perform loads and stores of data of any
+ * required size and endianness and handle possible misalignment.
+ */
+
 #define CPU_CONVERT(endian, size, type)\
 static inline type endian ## size ## _to_cpu(type v)\
 {\
@@ -104,16 +157,6 @@
 static inline void cpu_to_ ## endian ## size ## s(type *p)\
 {\
     glue(endian, _bswaps)(p, size);\
-}\
-\
-static inline type endian ## size ## _to_cpup(const type *p)\
-{\
-    return glue(glue(endian, size), _to_cpu)(*p);\
-}\
-\
-static inline void cpu_to_ ## endian ## size ## w(type *p, type v)\
-{\
-    *p = glue(glue(cpu_to_, endian), size)(v);\
 }
 
 CPU_CONVERT(be, 16, uint16_t)
@@ -130,6 +173,25 @@
     return bswap32(value) >> (32 - 8 * len);
 }
 
+/*
+ * Same as cpu_to_le{16,32}, except that gcc will figure the result is
+ * a compile-time constant if you pass in a constant.  So this can be
+ * used to initialize static variables.
+ */
+#if defined(HOST_WORDS_BIGENDIAN)
+# define const_le32(_x)                          \
+    ((((_x) & 0x000000ffU) << 24) |              \
+     (((_x) & 0x0000ff00U) <<  8) |              \
+     (((_x) & 0x00ff0000U) >>  8) |              \
+     (((_x) & 0xff000000U) >> 24))
+# define const_le16(_x)                          \
+    ((((_x) & 0x00ff) << 8) |                    \
+     (((_x) & 0xff00) >> 8))
+#else
+# define const_le32(_x) (_x)
+# define const_le16(_x) (_x)
+#endif
+
 /* Unions for reinterpreting between floats and integers.  */
 
 typedef union {
@@ -204,7 +266,7 @@
  *   f    : float access
  *
  * sign is:
- * (empty): for floats or 32 bit size
+ * (empty): for 32 or 64 bit sizes (including floats and doubles)
  *   u    : unsigned
  *   s    : signed
  *
@@ -218,7 +280,16 @@
  *   he   : host endian
  *   be   : big endian
  *   le   : little endian
+ *   te   : target endian
  * (except for byte accesses, which have no endian infix).
+ *
+ * The target endian accessors are obviously only available to source
+ * files which are built per-target; they are defined in cpu-all.h.
+ *
+ * In all cases these functions take a host pointer.
+ * For accessors that take a guest address rather than a
+ * host address, see the cpu_{ld,st}_* accessors defined in
+ * cpu_ldst.h.
  */
 
 static inline int ldub_p(const void *ptr)
@@ -415,11 +486,9 @@
 
 static inline unsigned long leul_to_cpu(unsigned long v)
 {
-    /* In order to break an include loop between here and
-       qemu-common.h, don't rely on HOST_LONG_BITS.  */
-#if ULONG_MAX == UINT32_MAX
+#if HOST_LONG_BITS == 32
     return le_bswap(v, 32);
-#elif ULONG_MAX == UINT64_MAX
+#elif HOST_LONG_BITS == 64
     return le_bswap(v, 64);
 #else
 # error Unknown sizeof long
diff --git a/include/qemu/buffer.h b/include/qemu/buffer.h
new file mode 100644
index 0000000..b2ead1f
--- /dev/null
+++ b/include/qemu/buffer.h
@@ -0,0 +1,161 @@
+/*
+ * QEMU generic buffers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#ifndef QEMU_BUFFER_H
+#define QEMU_BUFFER_H
+
+#include "qemu-common.h"
+
+typedef struct Buffer Buffer;
+
+/**
+ * Buffer:
+ *
+ * The Buffer object provides a simple dynamically resizing
+ * array, with separate tracking of capacity and usage. This
+ * is typically useful when buffering I/O or processing data.
+ */
+
+struct Buffer {
+    char *name;
+    size_t capacity;
+    size_t offset;
+    uint64_t avg_size;
+    uint8_t *buffer;
+};
+
+/**
+ * buffer_init:
+ * @buffer: the buffer object
+ * @name: buffer name
+ *
+ * Optionally attach a name to the buffer, to make it easier
+ * to identify in debug traces.
+ */
+void buffer_init(Buffer *buffer, const char *name, ...)
+        GCC_FMT_ATTR(2, 3);
+
+/**
+ * buffer_shrink:
+ * @buffer: the buffer object
+ *
+ * Try to shrink the buffer.  Checks current buffer capacity and size
+ * and reduces capacity in case only a fraction of the buffer is
+ * actually used.
+ */
+void buffer_shrink(Buffer *buffer);
+
+/**
+ * buffer_reserve:
+ * @buffer: the buffer object
+ * @len: the minimum required free space
+ *
+ * Ensure that the buffer has space allocated for at least
+ * @len bytes. If the current buffer is too small, it will
+ * be reallocated, possibly to a larger size than requested.
+ */
+void buffer_reserve(Buffer *buffer, size_t len);
+
+/**
+ * buffer_reset:
+ * @buffer: the buffer object
+ *
+ * Reset the length of the stored data to zero, but do
+ * not free / reallocate the memory buffer
+ */
+void buffer_reset(Buffer *buffer);
+
+/**
+ * buffer_free:
+ * @buffer: the buffer object
+ *
+ * Reset the length of the stored data to zero and also
+ * free the internal memory buffer
+ */
+void buffer_free(Buffer *buffer);
+
+/**
+ * buffer_append:
+ * @buffer: the buffer object
+ * @data: the data block to append
+ * @len: the length of @data in bytes
+ *
+ * Append the contents of @data to the end of the buffer.
+ * The caller must ensure that the buffer has sufficient
+ * free space for @len bytes, typically by calling the
+ * buffer_reserve() method prior to appending.
+ */
+void buffer_append(Buffer *buffer, const void *data, size_t len);
+
+/**
+ * buffer_advance:
+ * @buffer: the buffer object
+ * @len: the number of bytes to skip
+ *
+ * Remove @len bytes of data from the head of the buffer.
+ * The internal buffer will not be reallocated, so will
+ * have at least @len bytes of free space after this
+ * call completes
+ */
+void buffer_advance(Buffer *buffer, size_t len);
+
+/**
+ * buffer_end:
+ * @buffer: the buffer object
+ *
+ * Get a pointer to the tail end of the internal buffer
+ * The returned pointer is only valid until the next
+ * call to buffer_reserve().
+ *
+ * Returns: the tail of the buffer
+ */
+uint8_t *buffer_end(Buffer *buffer);
+
+/**
+ * buffer_empty:
+ * @buffer: the buffer object
+ *
+ * Determine if the buffer contains any current data
+ *
+ * Returns: true if the buffer holds data, false otherwise
+ */
+gboolean buffer_empty(Buffer *buffer);
+
+/**
+ * buffer_move_empty:
+ * @to: destination buffer object
+ * @from: source buffer object
+ *
+ * Moves buffer, without copying data.  'to' buffer must be empty.
+ * 'from' buffer is empty and zero-sized on return.
+ */
+void buffer_move_empty(Buffer *to, Buffer *from);
+
+/**
+ * buffer_move:
+ * @to: destination buffer object
+ * @from: source buffer object
+ *
+ * Moves buffer, copying data (unless 'to' buffer happens to be empty).
+ * 'from' buffer is empty and zero-sized on return.
+ */
+void buffer_move(Buffer *to, Buffer *from);
+
+#endif /* QEMU_BUFFER_H */
diff --git a/include/qemu/compatfd.h b/include/qemu/compatfd.h
index 6b04877..aa12ee9 100644
--- a/include/qemu/compatfd.h
+++ b/include/qemu/compatfd.h
@@ -14,7 +14,6 @@
 #ifndef QEMU_COMPATFD_H
 #define QEMU_COMPATFD_H
 
-#include <signal.h>
 
 struct qemu_signalfd_siginfo {
     uint32_t ssi_signo;   /* Signal number */
@@ -39,6 +38,5 @@
 };
 
 int qemu_signalfd(const sigset_t *mask);
-bool qemu_signalfd_available(void);
 
 #endif
diff --git a/include/qemu/compiler.h b/include/qemu/compiler.h
index ac7c4c4..cf16d79 100644
--- a/include/qemu/compiler.h
+++ b/include/qemu/compiler.h
@@ -3,7 +3,9 @@
 #ifndef COMPILER_H
 #define COMPILER_H
 
-#include "config-host.h"
+#if defined __clang_analyzer__ || defined __COVERITY__
+#define QEMU_STATIC_ANALYSIS 1
+#endif
 
 /*----------------------------------------------------------------------------
 | The macro QEMU_GNUC_PREREQ tests for minimum version of the GNU C compiler.
@@ -24,6 +26,12 @@
 #define QEMU_WARN_UNUSED_RESULT
 #endif
 
+#if QEMU_GNUC_PREREQ(4, 0)
+#define QEMU_SENTINEL __attribute__((sentinel))
+#else
+#define QEMU_SENTINEL
+#endif
+
 #if QEMU_GNUC_PREREQ(4, 3)
 #define QEMU_ARTIFICIAL __attribute__((always_inline, artificial))
 #else
@@ -36,10 +44,45 @@
 # define QEMU_PACKED __attribute__((packed))
 #endif
 
-#define cat(x,y) x ## y
-#define cat2(x,y) cat(x,y)
+#define QEMU_ALIGNED(X) __attribute__((aligned(X)))
+
+#ifndef glue
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+#endif
+
+#ifndef likely
+#if __GNUC__ < 3
+#define __builtin_expect(x, n) (x)
+#endif
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x)   __builtin_expect(!!(x), 0)
+#endif
+
+#ifndef container_of
+#define container_of(ptr, type, member) ({                      \
+        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
+        (type *) ((char *) __mptr - offsetof(type, member));})
+#endif
+
+/* Convert from a base type to a parent type, with compile time checking.  */
+#ifdef __GNUC__
+#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
+    char __attribute__((unused)) offset_must_be_zero[ \
+        -offsetof(type, field)]; \
+    container_of(dev, type, field);}))
+#else
+#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+#endif
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
 #define QEMU_BUILD_BUG_ON(x) \
-    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+    typedef char glue(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
 
 #if defined __GNUC__
 # if !QEMU_GNUC_PREREQ(4, 4)
@@ -58,4 +101,11 @@
 #define GCC_FMT_ATTR(n, m)
 #endif
 
+/* Don't allow CONFIG_ANDROID to be used, if POISON_CONFIG_ANDROID is
+ * defined from the command-line. This is useful to ensure that the code
+ * that should not depend on Android-specific features does not. */
+#ifdef POISON_CONFIG_ANDROID
+#pragma GCC poison CONFIG_ANDROID
+#endif
+
 #endif /* COMPILER_H */
diff --git a/include/qemu/config-file.h b/include/qemu/config-file.h
index d4ba20e..8d4b2b6 100644
--- a/include/qemu/config-file.h
+++ b/include/qemu/config-file.h
@@ -1,9 +1,7 @@
-#ifndef QEMU_CONFIG_H
-#define QEMU_CONFIG_H
+#ifndef QEMU_CONFIG_FILE_H
+#define QEMU_CONFIG_FILE_H
 
-#include <stdio.h>
 #include "qemu/option.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 
 QemuOptsList *qemu_find_opts(const char *group);
@@ -14,7 +12,6 @@
 void qemu_add_drive_opts(QemuOptsList *list);
 int qemu_set_option(const char *str);
 int qemu_global_option(const char *str);
-void qemu_add_globals(void);
 
 void qemu_config_write(FILE *fp);
 int qemu_config_parse(FILE *fp, QemuOptsList **lists, const char *fname);
@@ -30,4 +27,4 @@
  */
 int qemu_read_default_config_files(bool userconfig);
 
-#endif /* QEMU_CONFIG_H */
+#endif /* QEMU_CONFIG_FILE_H */
diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h
new file mode 100644
index 0000000..ac8d4c9
--- /dev/null
+++ b/include/qemu/coroutine.h
@@ -0,0 +1,215 @@
+/*
+ * QEMU coroutine implementation
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
+ *  Kevin Wolf         <kwolf@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_COROUTINE_H
+#define QEMU_COROUTINE_H
+
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+
+/**
+ * Coroutines are a mechanism for stack switching and can be used for
+ * cooperative userspace threading.  These functions provide a simple but
+ * useful flavor of coroutines that is suitable for writing sequential code,
+ * rather than callbacks, for operations that need to give up control while
+ * waiting for events to complete.
+ *
+ * These functions are re-entrant and may be used outside the global mutex.
+ */
+
+/**
+ * Mark a function that executes in coroutine context
+ *
+ * Functions that execute in coroutine context cannot be called directly from
+ * normal functions.  In the future it would be nice to enable compiler or
+ * static checker support for catching such errors.  This annotation might make
+ * it possible and in the meantime it serves as documentation.
+ *
+ * For example:
+ *
+ *   static void coroutine_fn foo(void) {
+ *       ....
+ *   }
+ */
+#define coroutine_fn
+
+typedef struct Coroutine Coroutine;
+
+/**
+ * Coroutine entry point
+ *
+ * When the coroutine is entered for the first time, opaque is passed in as an
+ * argument.
+ *
+ * When this function returns, the coroutine is destroyed automatically and
+ * execution continues in the caller who last entered the coroutine.
+ */
+typedef void coroutine_fn CoroutineEntry(void *opaque);
+
+/**
+ * Create a new coroutine
+ *
+ * Use qemu_coroutine_enter() to actually transfer control to the coroutine.
+ * The opaque argument is passed as the argument to the entry point.
+ */
+Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque);
+
+/**
+ * Transfer control to a coroutine
+ */
+void qemu_coroutine_enter(Coroutine *coroutine);
+
+/**
+ * Transfer control back to a coroutine's caller
+ *
+ * This function does not return until the coroutine is re-entered using
+ * qemu_coroutine_enter().
+ */
+void coroutine_fn qemu_coroutine_yield(void);
+
+/**
+ * Get the currently executing coroutine
+ */
+Coroutine *coroutine_fn qemu_coroutine_self(void);
+
+/**
+ * Return whether or not currently inside a coroutine
+ *
+ * This can be used to write functions that work both when in coroutine context
+ * and when not in coroutine context.  Note that such functions cannot use the
+ * coroutine_fn annotation since they work outside coroutine context.
+ */
+bool qemu_in_coroutine(void);
+
+
+
+/**
+ * CoQueues are a mechanism to queue coroutines in order to continue executing
+ * them later. They provide the fundamental primitives on which coroutine locks
+ * are built.
+ */
+typedef struct CoQueue {
+    QSIMPLEQ_HEAD(, Coroutine) entries;
+} CoQueue;
+
+/**
+ * Initialise a CoQueue. This must be called before any other operation is used
+ * on the CoQueue.
+ */
+void qemu_co_queue_init(CoQueue *queue);
+
+/**
+ * Adds the current coroutine to the CoQueue and transfers control to the
+ * caller of the coroutine.
+ */
+void coroutine_fn qemu_co_queue_wait(CoQueue *queue);
+
+/**
+ * Restarts the next coroutine in the CoQueue and removes it from the queue.
+ *
+ * Returns true if a coroutine was restarted, false if the queue is empty.
+ */
+bool coroutine_fn qemu_co_queue_next(CoQueue *queue);
+
+/**
+ * Restarts all coroutines in the CoQueue and leaves the queue empty.
+ */
+void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue);
+
+/**
+ * Enter the next coroutine in the queue
+ */
+bool qemu_co_enter_next(CoQueue *queue);
+
+/**
+ * Checks if the CoQueue is empty.
+ */
+bool qemu_co_queue_empty(CoQueue *queue);
+
+
+/**
+ * Provides a mutex that can be used to synchronise coroutines
+ */
+typedef struct CoMutex {
+    bool locked;
+    CoQueue queue;
+} CoMutex;
+
+/**
+ * Initialises a CoMutex. This must be called before any other operation is used
+ * on the CoMutex.
+ */
+void qemu_co_mutex_init(CoMutex *mutex);
+
+/**
+ * Locks the mutex. If the lock cannot be taken immediately, control is
+ * transferred to the caller of the current coroutine.
+ */
+void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex);
+
+/**
+ * Unlocks the mutex and schedules the next coroutine that was waiting for this
+ * lock to be run.
+ */
+void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex);
+
+typedef struct CoRwlock {
+    bool writer;
+    int reader;
+    CoQueue queue;
+} CoRwlock;
+
+/**
+ * Initialises a CoRwlock. This must be called before any other operation
+ * is used on the CoRwlock
+ */
+void qemu_co_rwlock_init(CoRwlock *lock);
+
+/**
+ * Read locks the CoRwlock. If the lock cannot be taken immediately because
+ * of a parallel writer, control is transferred to the caller of the current
+ * coroutine.
+ */
+void qemu_co_rwlock_rdlock(CoRwlock *lock);
+
+/**
+ * Write Locks the mutex. If the lock cannot be taken immediately because
+ * of a parallel reader, control is transferred to the caller of the current
+ * coroutine.
+ */
+void qemu_co_rwlock_wrlock(CoRwlock *lock);
+
+/**
+ * Unlocks the read/write lock and schedules the next coroutine that was
+ * waiting for this lock to be run.
+ */
+void qemu_co_rwlock_unlock(CoRwlock *lock);
+
+/**
+ * Yield the coroutine for a given duration
+ *
+ * Behaves similarly to co_sleep_ns(), but the sleeping coroutine will be
+ * resumed when using aio_poll().
+ */
+void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
+                                  int64_t ns);
+
+/**
+ * Yield until a file descriptor becomes readable
+ *
+ * Note that this function clobbers the handlers for the file descriptor.
+ */
+void coroutine_fn yield_until_fd_readable(int fd);
+
+#endif /* QEMU_COROUTINE_H */
diff --git a/include/qemu/coroutine_int.h b/include/qemu/coroutine_int.h
new file mode 100644
index 0000000..581a7f5
--- /dev/null
+++ b/include/qemu/coroutine_int.h
@@ -0,0 +1,54 @@
+/*
+ * Coroutine internals
+ *
+ * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef QEMU_COROUTINE_INT_H
+#define QEMU_COROUTINE_INT_H
+
+#include "qemu/queue.h"
+#include "qemu/coroutine.h"
+
+typedef enum {
+    COROUTINE_YIELD = 1,
+    COROUTINE_TERMINATE = 2,
+    COROUTINE_ENTER = 3,
+} CoroutineAction;
+
+struct Coroutine {
+    CoroutineEntry *entry;
+    void *entry_arg;
+    Coroutine *caller;
+    QSLIST_ENTRY(Coroutine) pool_next;
+
+    /* Coroutines that should be woken up when we yield or terminate */
+    QSIMPLEQ_HEAD(, Coroutine) co_queue_wakeup;
+    QSIMPLEQ_ENTRY(Coroutine) co_queue_next;
+};
+
+Coroutine *qemu_coroutine_new(void);
+void qemu_coroutine_delete(Coroutine *co);
+CoroutineAction qemu_coroutine_switch(Coroutine *from, Coroutine *to,
+                                      CoroutineAction action);
+void coroutine_fn qemu_co_queue_run_restart(Coroutine *co);
+
+#endif
diff --git a/include/qemu/cutils.h b/include/qemu/cutils.h
new file mode 100644
index 0000000..3e4ea23
--- /dev/null
+++ b/include/qemu/cutils.h
@@ -0,0 +1,183 @@
+#ifndef QEMU_CUTILS_H
+#define QEMU_CUTILS_H
+
+#include "qemu/fprintf-fn.h"
+
+/**
+ * pstrcpy:
+ * @buf: buffer to copy string into
+ * @buf_size: size of @buf in bytes
+ * @str: string to copy
+ *
+ * Copy @str into @buf, including the trailing NUL, but do not
+ * write more than @buf_size bytes. The resulting buffer is
+ * always NUL terminated (even if the source string was too long).
+ * If @buf_size is zero or negative then no bytes are copied.
+ *
+ * This function is similar to strncpy(), but avoids two of that
+ * function's problems:
+ *  * if @str fits in the buffer, pstrcpy() does not zero-fill the
+ *    remaining space at the end of @buf
+ *  * if @str is too long, pstrcpy() will copy the first @buf_size-1
+ *    bytes and then add a NUL
+ */
+void pstrcpy(char *buf, int buf_size, const char *str);
+/**
+ * strpadcpy:
+ * @buf: buffer to copy string into
+ * @buf_size: size of @buf in bytes
+ * @str: string to copy
+ * @pad: character to pad the remainder of @buf with
+ *
+ * Copy @str into @buf (but *not* its trailing NUL!), and then pad the
+ * rest of the buffer with the @pad character. If @str is too large
+ * for the buffer then it is truncated, so that @buf contains the
+ * first @buf_size characters of @str, with no terminator.
+ */
+void strpadcpy(char *buf, int buf_size, const char *str, char pad);
+/**
+ * pstrcat:
+ * @buf: buffer containing existing string
+ * @buf_size: size of @buf in bytes
+ * @s: string to concatenate to @buf
+ *
+ * Append a copy of @s to the string already in @buf, but do not
+ * allow the buffer to overflow. If the existing contents of @buf
+ * plus @str would total more than @buf_size bytes, then write
+ * as much of @str as will fit followed by a NUL terminator.
+ *
+ * @buf must already contain a NUL-terminated string, or the
+ * behaviour is undefined.
+ *
+ * Returns: @buf.
+ */
+char *pstrcat(char *buf, int buf_size, const char *s);
+/**
+ * strstart:
+ * @str: string to test
+ * @val: prefix string to look for
+ * @ptr: NULL, or pointer to be written to indicate start of
+ *       the remainder of the string
+ *
+ * Test whether @str starts with the prefix @val.
+ * If it does (including the degenerate case where @str and @val
+ * are equal) then return true. If @ptr is not NULL then a
+ * pointer to the first character following the prefix is written
+ * to it. If @val is not a prefix of @str then return false (and
+ * @ptr is not written to).
+ *
+ * Returns: true if @str starts with prefix @val, false otherwise.
+ */
+int strstart(const char *str, const char *val, const char **ptr);
+/**
+ * stristart:
+ * @str: string to test
+ * @val: prefix string to look for
+ * @ptr: NULL, or pointer to be written to indicate start of
+ *       the remainder of the string
+ *
+ * Test whether @str starts with the case-insensitive prefix @val.
+ * This function behaves identically to strstart(), except that the
+ * comparison is made after calling qemu_toupper() on each pair of
+ * characters.
+ *
+ * Returns: true if @str starts with case-insensitive prefix @val,
+ *          false otherwise.
+ */
+int stristart(const char *str, const char *val, const char **ptr);
+/**
+ * qemu_strnlen:
+ * @s: string
+ * @max_len: maximum number of bytes in @s to scan
+ *
+ * Return the length of the string @s, like strlen(), but do not
+ * examine more than @max_len bytes of the memory pointed to by @s.
+ * If no NUL terminator is found within @max_len bytes, then return
+ * @max_len instead.
+ *
+ * This function has the same behaviour as the POSIX strnlen()
+ * function.
+ *
+ * Returns: length of @s in bytes, or @max_len, whichever is smaller.
+ */
+int qemu_strnlen(const char *s, int max_len);
+/**
+ * qemu_strsep:
+ * @input: pointer to string to parse
+ * @delim: string containing delimiter characters to search for
+ *
+ * Locate the first occurrence of any character in @delim within
+ * the string referenced by @input, and replace it with a NUL.
+ * The location of the next character after the delimiter character
+ * is stored into @input.
+ * If the end of the string was reached without finding a delimiter
+ * character, then NULL is stored into @input.
+ * If @input points to a NULL pointer on entry, return NULL.
+ * The return value is always the original value of *@input (and
+ * so now points to a NUL-terminated string corresponding to the
+ * part of the input up to the first delimiter).
+ *
+ * This function has the same behaviour as the BSD strsep() function.
+ *
+ * Returns: the pointer originally in @input.
+ */
+char *qemu_strsep(char **input, const char *delim);
+time_t mktimegm(struct tm *tm);
+int qemu_fdatasync(int fd);
+int fcntl_setfl(int fd, int flag);
+int qemu_parse_fd(const char *param);
+int qemu_strtol(const char *nptr, const char **endptr, int base,
+                long *result);
+int qemu_strtoul(const char *nptr, const char **endptr, int base,
+                 unsigned long *result);
+int qemu_strtoll(const char *nptr, const char **endptr, int base,
+                 int64_t *result);
+int qemu_strtoull(const char *nptr, const char **endptr, int base,
+                  uint64_t *result);
+
+int parse_uint(const char *s, unsigned long long *value, char **endptr,
+               int base);
+int parse_uint_full(const char *s, unsigned long long *value, int base);
+
+/*
+ * qemu_strtosz() suffixes used to specify the default treatment of an
+ * argument passed to qemu_strtosz() without an explicit suffix.
+ * These should be defined using upper case characters in the range
+ * A-Z, as qemu_strtosz() will use qemu_toupper() on the given argument
+ * prior to comparison.
+ */
+#define QEMU_STRTOSZ_DEFSUFFIX_EB 'E'
+#define QEMU_STRTOSZ_DEFSUFFIX_PB 'P'
+#define QEMU_STRTOSZ_DEFSUFFIX_TB 'T'
+#define QEMU_STRTOSZ_DEFSUFFIX_GB 'G'
+#define QEMU_STRTOSZ_DEFSUFFIX_MB 'M'
+#define QEMU_STRTOSZ_DEFSUFFIX_KB 'K'
+#define QEMU_STRTOSZ_DEFSUFFIX_B 'B'
+int64_t qemu_strtosz(const char *nptr, char **end);
+int64_t qemu_strtosz_suffix(const char *nptr, char **end,
+                            const char default_suffix);
+int64_t qemu_strtosz_suffix_unit(const char *nptr, char **end,
+                            const char default_suffix, int64_t unit);
+#define K_BYTE     (1ULL << 10)
+#define M_BYTE     (1ULL << 20)
+#define G_BYTE     (1ULL << 30)
+#define T_BYTE     (1ULL << 40)
+#define P_BYTE     (1ULL << 50)
+#define E_BYTE     (1ULL << 60)
+
+/* used to print char* safely */
+#define STR_OR_NULL(str) ((str) ? (str) : "null")
+
+bool can_use_buffer_find_nonzero_offset(const void *buf, size_t len);
+size_t buffer_find_nonzero_offset(const void *buf, size_t len);
+bool buffer_is_zero(const void *buf, size_t len);
+
+/*
+ * Implementation of ULEB128 (http://en.wikipedia.org/wiki/LEB128)
+ * Input is limited to 14-bit numbers
+ */
+
+int uleb128_encode_small(uint8_t *out, uint32_t n);
+int uleb128_decode_small(const uint8_t *in, uint32_t *n);
+
+#endif
diff --git a/include/qemu/error-report.h b/include/qemu/error-report.h
index 5971245..6ebcf3e 100644
--- a/include/qemu/error-report.h
+++ b/include/qemu/error-report.h
@@ -10,12 +10,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#ifndef QEMU_ERROR_H
-#define QEMU_ERROR_H
-
-#include <stdarg.h>
-#include <stdbool.h>
-#include "qemu/compiler.h"
+#ifndef QEMU_ERROR_REPORT_H
+#define QEMU_ERROR_REPORT_H
 
 typedef struct Location {
     /* all members are private to qemu-error.c */
diff --git a/include/qemu/event_notifier.h b/include/qemu/event_notifier.h
index 88b57af..e326990 100644
--- a/include/qemu/event_notifier.h
+++ b/include/qemu/event_notifier.h
@@ -34,11 +34,13 @@
 void event_notifier_cleanup(EventNotifier *);
 int event_notifier_set(EventNotifier *);
 int event_notifier_test_and_clear(EventNotifier *);
-int event_notifier_set_handler(EventNotifier *, EventNotifierHandler *);
+int event_notifier_set_handler(EventNotifier *,
+                               bool is_external,
+                               EventNotifierHandler *);
 
 #ifdef CONFIG_POSIX
 void event_notifier_init_fd(EventNotifier *, int fd);
-int event_notifier_get_fd(EventNotifier *);
+int event_notifier_get_fd(const EventNotifier *);
 #else
 HANDLE event_notifier_get_handle(EventNotifier *);
 #endif
diff --git a/include/qemu/fifo32.h b/include/qemu/fifo32.h
new file mode 100644
index 0000000..4e9fd1b
--- /dev/null
+++ b/include/qemu/fifo32.h
@@ -0,0 +1,190 @@
+/*
+ * Generic FIFO32 component, based on FIFO8.
+ *
+ * Copyright (c) 2016 Jean-Christophe Dubois
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef FIFO32_H
+#define FIFO32_H
+
+#include "qemu/fifo8.h"
+
+typedef struct {
+    Fifo8 fifo;
+} Fifo32;
+
+/**
+ * fifo32_create:
+ * @fifo: struct Fifo32 to initialise with new FIFO
+ * @capacity: capacity of the newly created FIFO expressed in 32 bit words
+ *
+ * Create a FIFO of the specified size. Clients should call fifo32_destroy()
+ * when finished using the fifo. The FIFO is initially empty.
+ */
+
+static inline void fifo32_create(Fifo32 *fifo, uint32_t capacity)
+{
+    fifo8_create(&fifo->fifo, capacity * sizeof(uint32_t));
+}
+
+/**
+ * fifo32_destroy:
+ * @fifo: FIFO to cleanup
+ *
+ * Cleanup a FIFO created with fifo32_create(). Frees memory created for FIFO
+ * storage. The FIFO is no longer usable after this has been called.
+ */
+
+static inline void fifo32_destroy(Fifo32 *fifo)
+{
+    fifo8_destroy(&fifo->fifo);
+}
+
+/**
+ * fifo32_num_free:
+ * @fifo: FIFO to check
+ *
+ * Return the number of free uint32_t slots in the FIFO.
+ *
+ * Returns: Number of free 32 bit words.
+ */
+
+static inline uint32_t fifo32_num_free(Fifo32 *fifo)
+{
+    return DIV_ROUND_UP(fifo8_num_free(&fifo->fifo), sizeof(uint32_t));
+}
+
+/**
+ * fifo32_num_used:
+ * @fifo: FIFO to check
+ *
+ * Return the number of used uint32_t slots in the FIFO.
+ *
+ * Returns: Number of used 32 bit words.
+ */
+
+static inline uint32_t fifo32_num_used(Fifo32 *fifo)
+{
+    return DIV_ROUND_UP(fifo8_num_used(&fifo->fifo), sizeof(uint32_t));
+}
+
+/**
+ * fifo32_push:
+ * @fifo: FIFO to push to
+ * @data: 32 bits data word to push
+ *
+ * Push a 32 bits data word to the FIFO. Behaviour is undefined if the FIFO
+ * is full. Clients are responsible for checking for fullness using
+ * fifo32_is_full().
+ */
+
+static inline void fifo32_push(Fifo32 *fifo, uint32_t data)
+{
+    int i;
+
+    for (i = 0; i < sizeof(data); i++) {
+        fifo8_push(&fifo->fifo, data & 0xff);
+        data >>= 8;
+    }
+}
+
+/**
+ * fifo32_push_all:
+ * @fifo: FIFO to push to
+ * @data: data to push
+ * @size: number of 32 bit words to push
+ *
+ * Push a 32 bit word array to the FIFO. Behaviour is undefined if the FIFO
+ * is full. Clients are responsible for checking the space left in the FIFO
+ * using fifo32_num_free().
+ */
+
+static inline void fifo32_push_all(Fifo32 *fifo, const uint32_t *data,
+                                   uint32_t num)
+{
+    int i;
+
+    for (i = 0; i < num; i++) {
+        fifo32_push(fifo, data[i]);
+    }
+}
+
+/**
+ * fifo32_pop:
+ * @fifo: fifo to pop from
+ *
+ * Pop a 32 bits data word from the FIFO. Behaviour is undefined if the FIFO
+ * is empty. Clients are responsible for checking for emptiness using
+ * fifo32_is_empty().
+ *
+ * Returns: The popped 32 bits data word.
+ */
+
+static inline uint32_t fifo32_pop(Fifo32 *fifo)
+{
+    uint32_t ret = 0;
+    int i;
+
+    for (i = 0; i < sizeof(uint32_t); i++) {
+        ret |= (fifo8_pop(&fifo->fifo) << (i * 8));
+    }
+
+    return ret;
+}
+
+/**
+ * There is no fifo32_pop_buf() because the data is not stored in the buffer
+ * as a set of native-order words.
+ */
+
+/**
+ * fifo32_reset:
+ * @fifo: FIFO to reset
+ *
+ * Reset a FIFO. All data is discarded and the FIFO is emptied.
+ */
+
+static inline void fifo32_reset(Fifo32 *fifo)
+{
+    fifo8_reset(&fifo->fifo);
+}
+
+/**
+ * fifo32_is_empty:
+ * @fifo: FIFO to check
+ *
+ * Check if a FIFO is empty.
+ *
+ * Returns: True if the fifo is empty, false otherwise.
+ */
+
+static inline bool fifo32_is_empty(Fifo32 *fifo)
+{
+    return fifo8_is_empty(&fifo->fifo);
+}
+
+/**
+ * fifo32_is_full:
+ * @fifo: FIFO to check
+ *
+ * Check if a FIFO is full.
+ *
+ * Returns: True if the fifo is full, false otherwise.
+ */
+
+static inline bool fifo32_is_full(Fifo32 *fifo)
+{
+    return fifo8_num_free(&fifo->fifo) < sizeof(uint32_t);
+}
+
+#define VMSTATE_FIFO32(_field, _state) VMSTATE_FIFO8(_field.fifo, _state)
+
+#endif /* FIFO32_H */
diff --git a/include/qemu/fifo8.h b/include/qemu/fifo8.h
index 8820780..24b3644 100644
--- a/include/qemu/fifo8.h
+++ b/include/qemu/fifo8.h
@@ -1,5 +1,5 @@
-#ifndef FIFO_H
-#define FIFO_H
+#ifndef QEMU_FIFO8_H
+#define QEMU_FIFO8_H
 
 #include "migration/vmstate.h"
 
@@ -157,4 +157,4 @@
     .offset     = vmstate_offset_value(_state, _field, Fifo8),       \
 }
 
-#endif /* FIFO_H */
+#endif /* QEMU_FIFO8_H */
diff --git a/include/qemu/fprintf-fn.h b/include/qemu/fprintf-fn.h
new file mode 100644
index 0000000..9068a96
--- /dev/null
+++ b/include/qemu/fprintf-fn.h
@@ -0,0 +1,14 @@
+/*
+ * Typedef for fprintf-alike function pointers.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_FPRINTF_FN_H
+#define QEMU_FPRINTF_FN_H
+
+typedef int (*fprintf_function)(FILE *f, const char *fmt, ...)
+    GCC_FMT_ATTR(2, 3);
+
+#endif
diff --git a/include/qemu/hbitmap.h b/include/qemu/hbitmap.h
index 550d7ce..8ab721e 100644
--- a/include/qemu/hbitmap.h
+++ b/include/qemu/hbitmap.h
@@ -10,11 +10,8 @@
  */
 
 #ifndef HBITMAP_H
-#define HBITMAP_H 1
+#define HBITMAP_H
 
-#include <limits.h>
-#include <stdint.h>
-#include <stdbool.h>
 #include "bitops.h"
 #include "host-utils.h"
 
@@ -65,6 +62,29 @@
 HBitmap *hbitmap_alloc(uint64_t size, int granularity);
 
 /**
+ * hbitmap_truncate:
+ * @hb: The bitmap to change the size of.
+ * @size: The number of elements to change the bitmap to accommodate.
+ *
+ * truncate or grow an existing bitmap to accommodate a new number of elements.
+ * This may invalidate existing HBitmapIterators.
+ */
+void hbitmap_truncate(HBitmap *hb, uint64_t size);
+
+/**
+ * hbitmap_merge:
+ * @a: The bitmap to store the result in.
+ * @b: The bitmap to merge into @a.
+ * @return true if the merge was successful,
+ *         false if it was not attempted.
+ *
+ * Merge two bitmaps together.
+ * A := A (BITOR) B.
+ * B is left unmodified.
+ */
+bool hbitmap_merge(HBitmap *a, const HBitmap *b);
+
+/**
  * hbitmap_empty:
  * @hb: HBitmap to operate on.
  *
@@ -109,6 +129,14 @@
 void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count);
 
 /**
+ * hbitmap_reset_all:
+ * @hb: HBitmap to operate on.
+ *
+ * Reset all bits in an HBitmap.
+ */
+void hbitmap_reset_all(HBitmap *hb);
+
+/**
  * hbitmap_get:
  * @hb: HBitmap to operate on.
  * @item: Bit to query (0-based).
diff --git a/include/qemu/help_option.h b/include/qemu/help_option.h
new file mode 100644
index 0000000..328d2a8
--- /dev/null
+++ b/include/qemu/help_option.h
@@ -0,0 +1,22 @@
+#ifndef QEMU_HELP_OPTION_H
+#define QEMU_HELP_OPTION_H
+
+/**
+ * is_help_option:
+ * @s: string to test
+ *
+ * Check whether @s is one of the standard strings which indicate
+ * that the user is asking for a list of the valid values for a
+ * command option like -cpu or -M. The current accepted strings
+ * are 'help' and '?'. '?' is deprecated (it is a shell wildcard
+ * which makes it annoying to use in a reliable way) but provided
+ * for backwards compatibility.
+ *
+ * Returns: true if @s is a request for a list.
+ */
+static inline bool is_help_option(const char *s)
+{
+    return !strcmp(s, "?") || !strcmp(s, "help");
+}
+
+#endif
diff --git a/include/qemu/host-utils.h b/include/qemu/host-utils.h
index d4f21c9..46187bb 100644
--- a/include/qemu/host-utils.h
+++ b/include/qemu/host-utils.h
@@ -22,11 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef HOST_UTILS_H
-#define HOST_UTILS_H 1
 
-#include "qemu/compiler.h"   /* QEMU_GNUC_PREREQ */
-#include <limits.h>
+#ifndef HOST_UTILS_H
+#define HOST_UTILS_H
+
+#include "qemu/bswap.h"
 
 #ifdef CONFIG_INT128
 static inline void mulu64(uint64_t *plow, uint64_t *phigh,
@@ -45,6 +45,12 @@
     *phigh = r >> 64;
 }
 
+/* compute with 96 bit intermediate result: (a*b)/c */
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    return (__int128_t)a * b / c;
+}
+
 static inline int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor)
 {
     if (divisor == 0) {
@@ -75,6 +81,29 @@
 void mulu64(uint64_t *phigh, uint64_t *plow, uint64_t a, uint64_t b);
 int divu128(uint64_t *plow, uint64_t *phigh, uint64_t divisor);
 int divs128(int64_t *plow, int64_t *phigh, int64_t divisor);
+
+static inline uint64_t muldiv64(uint64_t a, uint32_t b, uint32_t c)
+{
+    union {
+        uint64_t ll;
+        struct {
+#ifdef HOST_WORDS_BIGENDIAN
+            uint32_t high, low;
+#else
+            uint32_t low, high;
+#endif
+        } l;
+    } u, res;
+    uint64_t rl, rh;
+
+    u.ll = a;
+    rl = (uint64_t)u.l.low * (uint64_t)b;
+    rh = (uint64_t)u.l.high * (uint64_t)b;
+    rh += (rl >> 32);
+    res.l.high = rh / c;
+    res.l.low = (((rh % c) << 32) + (rl & 0xffffffff)) / c;
+    return res.ll;
+}
 #endif
 
 /**
@@ -361,6 +390,80 @@
 #endif
 }
 
+/**
+ * revbit8 - reverse the bits in an 8-bit value.
+ * @x: The value to modify.
+ */
+static inline uint8_t revbit8(uint8_t x)
+{
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0) >> 4)
+      | ((x & 0x0f) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x88) >> 3)
+      | ((x & 0x44) >> 1)
+      | ((x & 0x22) << 1)
+      | ((x & 0x11) << 3);
+    return x;
+}
+
+/**
+ * revbit16 - reverse the bits in a 16-bit value.
+ * @x: The value to modify.
+ */
+static inline uint16_t revbit16(uint16_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap16(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0) >> 4)
+      | ((x & 0x0f0f) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x8888) >> 3)
+      | ((x & 0x4444) >> 1)
+      | ((x & 0x2222) << 1)
+      | ((x & 0x1111) << 3);
+    return x;
+}
+
+/**
+ * revbit32 - reverse the bits in a 32-bit value.
+ * @x: The value to modify.
+ */
+static inline uint32_t revbit32(uint32_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap32(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0f0f0u) >> 4)
+      | ((x & 0x0f0f0f0fu) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x88888888u) >> 3)
+      | ((x & 0x44444444u) >> 1)
+      | ((x & 0x22222222u) << 1)
+      | ((x & 0x11111111u) << 3);
+    return x;
+}
+
+/**
+ * revbit64 - reverse the bits in a 64-bit value.
+ * @x: The value to modify.
+ */
+static inline uint64_t revbit64(uint64_t x)
+{
+    /* Assign the correct byte position.  */
+    x = bswap64(x);
+    /* Assign the correct nibble position.  */
+    x = ((x & 0xf0f0f0f0f0f0f0f0ull) >> 4)
+      | ((x & 0x0f0f0f0f0f0f0f0full) << 4);
+    /* Assign the correct bit position.  */
+    x = ((x & 0x8888888888888888ull) >> 3)
+      | ((x & 0x4444444444444444ull) >> 1)
+      | ((x & 0x2222222222222222ull) << 1)
+      | ((x & 0x1111111111111111ull) << 3);
+    return x;
+}
+
 /* Host type specific sizes of these routines.  */
 
 #if ULONG_MAX == UINT32_MAX
@@ -369,14 +472,48 @@
 # define clol   clo32
 # define ctol   cto32
 # define ctpopl ctpop32
+# define revbitl revbit32
 #elif ULONG_MAX == UINT64_MAX
 # define clzl   clz64
 # define ctzl   ctz64
 # define clol   clo64
 # define ctol   cto64
 # define ctpopl ctpop64
+# define revbitl revbit64
 #else
 # error Unknown sizeof long
 #endif
 
+static inline bool is_power_of_2(uint64_t value)
+{
+    if (!value) {
+        return false;
+    }
+
+    return !(value & (value - 1));
+}
+
+/* round down to the nearest power of 2*/
+static inline int64_t pow2floor(int64_t value)
+{
+    if (!is_power_of_2(value)) {
+        value = 0x8000000000000000ULL >> clz64(value);
+    }
+    return value;
+}
+
+/* round up to the nearest power of 2 (0 if overflow) */
+static inline uint64_t pow2ceil(uint64_t value)
+{
+    uint8_t nlz = clz64(value);
+
+    if (is_power_of_2(value)) {
+        return value;
+    }
+    if (!nlz) {
+        return 0;
+    }
+    return 1ULL << (64 - nlz);
+}
+
 #endif
diff --git a/include/qemu/id.h b/include/qemu/id.h
new file mode 100644
index 0000000..40c7010
--- /dev/null
+++ b/include/qemu/id.h
@@ -0,0 +1,13 @@
+#ifndef QEMU_ID_H
+#define QEMU_ID_H
+
+typedef enum IdSubSystems {
+    ID_QDEV,
+    ID_BLOCK,
+    ID_MAX      /* last element, used as array size */
+} IdSubSystems;
+
+char *id_generate(IdSubSystems id);
+bool id_wellformed(const char *id);
+
+#endif
diff --git a/include/qemu/int128.h b/include/qemu/int128.h
index fb782aa..c598881 100644
--- a/include/qemu/int128.h
+++ b/include/qemu/int128.h
@@ -1,9 +1,6 @@
 #ifndef INT128_H
 #define INT128_H
 
-#include <assert.h>
-#include <stdint.h>
-#include <stdbool.h>
 
 typedef struct Int128 Int128;
 
diff --git a/include/qemu/iov.h b/include/qemu/iov.h
index 68d25f2..bd9fd55 100644
--- a/include/qemu/iov.h
+++ b/include/qemu/iov.h
@@ -14,8 +14,6 @@
 #ifndef IOV_H
 #define IOV_H
 
-#include "qemu-common.h"
-
 /**
  * count and return data size, in bytes, of an iovec
  * starting at `iov' of `iov_cnt' number of elements.
@@ -39,10 +37,36 @@
  * such "large" value is -1 (sinice size_t is unsigned),
  * so specifying `-1' as `bytes' means 'up to the end of iovec'.
  */
-size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
-                    size_t offset, const void *buf, size_t bytes);
-size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
-                  size_t offset, void *buf, size_t bytes);
+size_t iov_from_buf_full(const struct iovec *iov, unsigned int iov_cnt,
+                         size_t offset, const void *buf, size_t bytes);
+size_t iov_to_buf_full(const struct iovec *iov, const unsigned int iov_cnt,
+		       size_t offset, void *buf, size_t bytes);
+
+static inline size_t
+iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
+             size_t offset, const void *buf, size_t bytes)
+{
+    if (__builtin_constant_p(bytes) && iov_cnt &&
+        offset <= iov[0].iov_len && bytes <= iov[0].iov_len - offset) {
+        memcpy(iov[0].iov_base + offset, buf, bytes);
+        return bytes;
+    } else {
+        return iov_from_buf_full(iov, iov_cnt, offset, buf, bytes);
+    }
+}
+
+static inline size_t
+iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
+           size_t offset, void *buf, size_t bytes)
+{
+    if (__builtin_constant_p(bytes) && iov_cnt &&
+        offset <= iov[0].iov_len && bytes <= iov[0].iov_len - offset) {
+        memcpy(buf, iov[0].iov_base + offset, bytes);
+        return bytes;
+    } else {
+        return iov_to_buf_full(iov, iov_cnt, offset, buf, bytes);
+    }
+}
 
 /**
  * Set data bytes pointed out by iovec `iov' of size `iov_cnt' elements,
@@ -75,7 +99,7 @@
  * For iov_send_recv() _whole_ area being sent or received
  * should be within the iovec, not only beginning of it.
  */
-ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
+ssize_t iov_send_recv(int sockfd, const struct iovec *iov, unsigned iov_cnt,
                       size_t offset, size_t bytes, bool do_send);
 #define iov_recv(sockfd, iov, iov_cnt, offset, bytes) \
   iov_send_recv(sockfd, iov, iov_cnt, offset, bytes, false)
@@ -112,4 +136,32 @@
 size_t iov_discard_back(struct iovec *iov, unsigned int *iov_cnt,
                         size_t bytes);
 
+typedef struct QEMUIOVector {
+    struct iovec *iov;
+    int niov;
+    int nalloc;
+    size_t size;
+} QEMUIOVector;
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
+void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes);
+size_t qemu_iovec_concat_iov(QEMUIOVector *dst,
+                             struct iovec *src_iov, unsigned int src_cnt,
+                             size_t soffset, size_t sbytes);
+bool qemu_iovec_is_zero(QEMUIOVector *qiov);
+void qemu_iovec_destroy(QEMUIOVector *qiov);
+void qemu_iovec_reset(QEMUIOVector *qiov);
+size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
+                         void *buf, size_t bytes);
+size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
+                           const void *buf, size_t bytes);
+size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
+                         int fillc, size_t bytes);
+ssize_t qemu_iovec_compare(QEMUIOVector *a, QEMUIOVector *b);
+void qemu_iovec_clone(QEMUIOVector *dest, const QEMUIOVector *src, void *buf);
+void qemu_iovec_discard_back(QEMUIOVector *qiov, size_t bytes);
+
 #endif
diff --git a/include/qemu/log.h b/include/qemu/log.h
index d515424..00bf37f 100644
--- a/include/qemu/log.h
+++ b/include/qemu/log.h
@@ -1,14 +1,6 @@
 #ifndef QEMU_LOG_H
 #define QEMU_LOG_H
 
-#include <stdarg.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include "qemu/compiler.h"
-#include "qom/cpu.h"
-#ifdef NEED_CPU_H
-#include "disas/disas.h"
-#endif
 
 /* Private global variables, don't use */
 extern FILE *qemu_logfile;
@@ -28,6 +20,13 @@
     return qemu_logfile != NULL;
 }
 
+/* Returns true if qemu_log() will write somewhere else than stderr
+ */
+static inline bool qemu_log_separate(void)
+{
+    return qemu_logfile != NULL && qemu_logfile != stderr;
+}
+
 #define CPU_LOG_TB_OUT_ASM (1 << 0)
 #define CPU_LOG_TB_IN_ASM  (1 << 1)
 #define CPU_LOG_TB_OP      (1 << 2)
@@ -35,11 +34,15 @@
 #define CPU_LOG_INT        (1 << 4)
 #define CPU_LOG_EXEC       (1 << 5)
 #define CPU_LOG_PCALL      (1 << 6)
-#define CPU_LOG_IOPORT     (1 << 7)
 #define CPU_LOG_TB_CPU     (1 << 8)
 #define CPU_LOG_RESET      (1 << 9)
 #define LOG_UNIMP          (1 << 10)
 #define LOG_GUEST_ERROR    (1 << 11)
+#define CPU_LOG_MMU        (1 << 12)
+#define CPU_LOG_TB_NOCHAIN (1 << 13)
+#define CPU_LOG_PAGE       (1 << 14)
+#define LOG_TRACE          (1 << 15)
+#define CPU_LOG_TB_OP_IND  (1 << 16)
 
 /* Returns true if a bit is set in the current loglevel mask
  */
@@ -52,7 +55,7 @@
 
 /* main logging function
  */
-void GCC_FMT_ATTR(1, 2) qemu_log(const char *fmt, ...);
+int GCC_FMT_ATTR(1, 2) qemu_log(const char *fmt, ...);
 
 /* vfprintf-like logging function
  */
@@ -64,91 +67,35 @@
     }
 }
 
+/* log only if a bit is set on the current loglevel mask:
+ * @mask: bit to check in the mask
+ * @fmt: printf-style format string
+ * @args: optional arguments for format string
+ */
+#define qemu_log_mask(MASK, FMT, ...)                   \
+    do {                                                \
+        if (unlikely(qemu_loglevel_mask(MASK))) {       \
+            qemu_log(FMT, ## __VA_ARGS__);              \
+        }                                               \
+    } while (0)
+
 /* log only if a bit is set on the current loglevel mask
+ * and we are in the address range we care about:
+ * @mask: bit to check in the mask
+ * @addr: address to check in dfilter
+ * @fmt: printf-style format string
+ * @args: optional arguments for format string
  */
-void GCC_FMT_ATTR(2, 3) qemu_log_mask(int mask, const char *fmt, ...);
-
-
-/* Special cases: */
-
-/* cpu_dump_state() logging functions: */
-/**
- * log_cpu_state:
- * @cpu: The CPU whose state is to be logged.
- * @flags: Flags what to log.
- *
- * Logs the output of cpu_dump_state().
- */
-static inline void log_cpu_state(CPUState *cpu, int flags)
-{
-    if (qemu_log_enabled()) {
-        cpu_dump_state(cpu, qemu_logfile, fprintf, flags);
-    }
-}
-
-/**
- * log_cpu_state_mask:
- * @mask: Mask when to log.
- * @cpu: The CPU whose state is to be logged.
- * @flags: Flags what to log.
- *
- * Logs the output of cpu_dump_state() if loglevel includes @mask.
- */
-static inline void log_cpu_state_mask(int mask, CPUState *cpu, int flags)
-{
-    if (qemu_loglevel & mask) {
-        log_cpu_state(cpu, flags);
-    }
-}
-
-#ifdef NEED_CPU_H
-/* disas() and target_disas() to qemu_logfile: */
-static inline void log_target_disas(CPUArchState *env, target_ulong start,
-                                    target_ulong len, int flags)
-{
-    target_disas(qemu_logfile, env, start, len, flags);
-}
-
-static inline void log_disas(void *code, unsigned long size)
-{
-    disas(qemu_logfile, code, size);
-}
-
-#if defined(CONFIG_USER_ONLY)
-/* page_dump() output to the log file: */
-static inline void log_page_dump(void)
-{
-    page_dump(qemu_logfile);
-}
-#endif
-#endif
-
+#define qemu_log_mask_and_addr(MASK, ADDR, FMT, ...)    \
+    do {                                                \
+        if (unlikely(qemu_loglevel_mask(MASK)) &&       \
+                     qemu_log_in_addr_range(ADDR)) {    \
+            qemu_log(FMT, ## __VA_ARGS__);              \
+        }                                               \
+    } while (0)
 
 /* Maintenance: */
 
-/* fflush() the log file */
-static inline void qemu_log_flush(void)
-{
-    fflush(qemu_logfile);
-}
-
-/* Close the log file */
-static inline void qemu_log_close(void)
-{
-    if (qemu_logfile) {
-        if (qemu_logfile != stderr) {
-            fclose(qemu_logfile);
-        }
-        qemu_logfile = NULL;
-    }
-}
-
-/* Set up a new log file */
-static inline void qemu_log_set_file(FILE *f)
-{
-    qemu_logfile = f;
-}
-
 /* define log items */
 typedef struct QEMULogItem {
     int mask;
@@ -158,22 +105,11 @@
 
 extern const QEMULogItem qemu_log_items[];
 
-/* This is the function that actually does the work of
- * changing the log level; it should only be accessed via
- * the qemu_set_log() wrapper.
- */
-void do_qemu_set_log(int log_flags, bool use_own_buffers);
-
-static inline void qemu_set_log(int log_flags)
-{
-#ifdef CONFIG_USER_ONLY
-    do_qemu_set_log(log_flags, true);
-#else
-    do_qemu_set_log(log_flags, false);
-#endif
-}
-
-void qemu_set_log_filename(const char *filename);
+void qemu_set_log(int log_flags);
+void qemu_log_needs_buffers(void);
+void qemu_set_log_filename(const char *filename, Error **errp);
+void qemu_set_dfilter_ranges(const char *ranges, Error **errp);
+bool qemu_log_in_addr_range(uint64_t addr);
 int qemu_str_to_log_mask(const char *str);
 
 /* Print a usage message listing all the valid logging categories
@@ -181,4 +117,9 @@
  */
 void qemu_print_log_usage(FILE *f);
 
+/* fflush() the log file */
+void qemu_log_flush(void);
+/* Close the log file */
+void qemu_log_close(void);
+
 #endif
diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h
index e014dc5..7436211 100644
--- a/include/qemu/main-loop.h
+++ b/include/qemu/main-loop.h
@@ -23,7 +23,7 @@
  */
 
 #ifndef QEMU_MAIN_LOOP_H
-#define QEMU_MAIN_LOOP_H 1
+#define QEMU_MAIN_LOOP_H
 
 #include "block/aio.h"
 
@@ -64,11 +64,11 @@
  *
  *     void enter_co_bh(void *opaque) {
  *         QEMUCoroutine *co = opaque;
- *         qemu_coroutine_enter(co, NULL);
+ *         qemu_coroutine_enter(co);
  *     }
  *
  *     ...
- *     QEMUCoroutine *co = qemu_coroutine_create(coroutine_entry);
+ *     QEMUCoroutine *co = qemu_coroutine_create(coroutine_entry, NULL);
  *     QEMUBH *start_bh = qemu_bh_new(enter_co_bh, co);
  *     qemu_bh_schedule(start_bh);
  *     while (...) {
@@ -82,6 +82,14 @@
 int main_loop_wait(int nonblocking);
 
 /**
+ * main_loop_register_poll_callback: Register a custom function that is called
+ * each time the main-loop needs to be polled. Used by the Android emulator
+ * to implement charpipe connectivity.
+ */
+typedef void (*MainLoopPollCallback)(void);
+void main_loop_register_poll_callback(MainLoopPollCallback poll_func);
+
+/**
  * qemu_get_aio_context: Return the main loop's AioContext
  */
 AioContext *qemu_get_aio_context(void);
@@ -96,8 +104,7 @@
  * that the main loop waits for.
  *
  * Calling qemu_notify_event is rarely necessary, because main loop
- * services (bottom halves and timers) call it themselves.  One notable
- * exception occurs when using qemu_set_fd_handler2 (see below).
+ * services (bottom halves and timers) call it themselves.
  */
 void qemu_notify_event(void);
 
@@ -172,52 +179,6 @@
 typedef int IOCanReadHandler(void *opaque);
 
 /**
- * qemu_set_fd_handler2: Register a file descriptor with the main loop
- *
- * This function tells the main loop to wake up whenever one of the
- * following conditions is true:
- *
- * 1) if @fd_write is not %NULL, when the file descriptor is writable;
- *
- * 2) if @fd_read is not %NULL, when the file descriptor is readable.
- *
- * @fd_read_poll can be used to disable the @fd_read callback temporarily.
- * This is useful to avoid calling qemu_set_fd_handler2 every time the
- * client becomes interested in reading (or dually, stops being interested).
- * A typical example is when @fd is a listening socket and you want to bound
- * the number of active clients.  Remember to call qemu_notify_event whenever
- * the condition may change from %false to %true.
- *
- * The callbacks that are set up by qemu_set_fd_handler2 are level-triggered.
- * If @fd_read does not read from @fd, or @fd_write does not write to @fd
- * until its buffers are full, they will be called again on the next
- * iteration.
- *
- * @fd: The file descriptor to be observed.  Under Windows it must be
- * a #SOCKET.
- *
- * @fd_read_poll: A function that returns 1 if the @fd_read callback
- * should be fired.  If the function returns 0, the main loop will not
- * end its iteration even if @fd becomes readable.
- *
- * @fd_read: A level-triggered callback that is fired if @fd is readable
- * at the beginning of a main loop iteration, or if it becomes readable
- * during one.
- *
- * @fd_write: A level-triggered callback that is fired when @fd is writable
- * at the beginning of a main loop iteration, or if it becomes writable
- * during one.
- *
- * @opaque: A pointer-sized value that is passed to @fd_read_poll,
- * @fd_read and @fd_write.
- */
-int qemu_set_fd_handler2(int fd,
-                         IOCanReadHandler *fd_read_poll,
-                         IOHandler *fd_read,
-                         IOHandler *fd_write,
-                         void *opaque);
-
-/**
  * qemu_set_fd_handler: Register a file descriptor with the main loop
  *
  * This function tells the main loop to wake up whenever one of the
@@ -245,11 +206,13 @@
  *
  * @opaque: A pointer-sized value that is passed to @fd_read and @fd_write.
  */
-int qemu_set_fd_handler(int fd,
-                        IOHandler *fd_read,
-                        IOHandler *fd_write,
-                        void *opaque);
+void qemu_set_fd_handler(int fd,
+                         IOHandler *fd_read,
+                         IOHandler *fd_write,
+                         void *opaque);
 
+GSource *iohandler_get_g_source(void);
+AioContext *iohandler_get_aio_context(void);
 #ifdef CONFIG_POSIX
 /**
  * qemu_add_child_watch: Register a child process for reaping.
@@ -270,6 +233,16 @@
 #endif
 
 /**
+ * qemu_mutex_iothread_locked: Return lock status of the main loop mutex.
+ *
+ * The main loop mutex is the coarsest lock in QEMU, and as such it
+ * must always be taken outside other locks.  This function helps
+ * functions take different paths depending on whether the current
+ * thread is running within the main loop mutex.
+ */
+bool qemu_mutex_iothread_locked(void);
+
+/**
  * qemu_mutex_lock_iothread: Lock the main loop mutex.
  *
  * This function locks the main loop mutex.  The mutex is taken by
@@ -299,17 +272,9 @@
  */
 void qemu_mutex_unlock_iothread(void);
 
-#ifdef CONFIG_ANDROID
-// This returns true iff the current thread currently holds the
-// main loop mutex (used to protect global VM state).
-bool qemu_mutex_check_iothread(void);
-#endif
-
 /* internal interfaces */
 
 void qemu_fd_register(int fd);
-void qemu_iohandler_fill(GArray *pollfds);
-void qemu_iohandler_poll(GArray *pollfds, int rc);
 
 QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
 void qemu_bh_schedule_idle(QEMUBH *bh);
diff --git a/include/qemu/memfd.h b/include/qemu/memfd.h
new file mode 100644
index 0000000..745a8c5
--- /dev/null
+++ b/include/qemu/memfd.h
@@ -0,0 +1,24 @@
+#ifndef QEMU_MEMFD_H
+#define QEMU_MEMFD_H
+
+
+#ifndef F_LINUX_SPECIFIC_BASE
+#define F_LINUX_SPECIFIC_BASE 1024
+#endif
+
+#ifndef F_ADD_SEALS
+#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
+#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
+
+#define F_SEAL_SEAL     0x0001  /* prevent further seals from being set */
+#define F_SEAL_SHRINK   0x0002  /* prevent file from shrinking */
+#define F_SEAL_GROW     0x0004  /* prevent file from growing */
+#define F_SEAL_WRITE    0x0008  /* prevent writes */
+#endif
+
+void *qemu_memfd_alloc(const char *name, size_t size, unsigned int seals,
+                       int *fd);
+void qemu_memfd_free(void *ptr, size_t size, int fd);
+bool qemu_memfd_check(void);
+
+#endif /* QEMU_MEMFD_H */
diff --git a/include/qemu/mmap-alloc.h b/include/qemu/mmap-alloc.h
new file mode 100644
index 0000000..933c024
--- /dev/null
+++ b/include/qemu/mmap-alloc.h
@@ -0,0 +1,12 @@
+#ifndef QEMU_MMAP_ALLOC_H
+#define QEMU_MMAP_ALLOC_H
+
+#include "qemu-common.h"
+
+size_t qemu_fd_getpagesize(int fd);
+
+void *qemu_ram_mmap(int fd, size_t size, size_t align, bool shared);
+
+void qemu_ram_munmap(void *ptr, size_t size);
+
+#endif
diff --git a/include/qemu/module.h b/include/qemu/module.h
index 72d9498..2370708 100644
--- a/include/qemu/module.h
+++ b/include/qemu/module.h
@@ -14,7 +14,6 @@
 #ifndef QEMU_MODULE_H
 #define QEMU_MODULE_H
 
-#include "qemu/osdep.h"
 
 #define DSO_STAMP_FUN         glue(qemu_stamp, CONFIG_STAMP)
 #define DSO_STAMP_FUN_STR     stringify(DSO_STAMP_FUN)
@@ -42,14 +41,14 @@
 
 typedef enum {
     MODULE_INIT_BLOCK,
-    MODULE_INIT_MACHINE,
+    MODULE_INIT_OPTS,
     MODULE_INIT_QAPI,
     MODULE_INIT_QOM,
     MODULE_INIT_MAX
 } module_init_type;
 
 #define block_init(function) module_init(function, MODULE_INIT_BLOCK)
-#define machine_init(function) module_init(function, MODULE_INIT_MACHINE)
+#define opts_init(function) module_init(function, MODULE_INIT_OPTS)
 #define qapi_init(function) module_init(function, MODULE_INIT_QAPI)
 #define type_init(function) module_init(function, MODULE_INIT_QOM)
 
diff --git a/include/qemu/option.h b/include/qemu/option.h
index 59bea75..1f9e3f9 100644
--- a/include/qemu/option.h
+++ b/include/qemu/option.h
@@ -23,12 +23,10 @@
  * THE SOFTWARE.
  */
 
-#ifndef QEMU_OPTIONS_H
-#define QEMU_OPTIONS_H
+#ifndef QEMU_OPTION_H
+#define QEMU_OPTION_H
 
-#include <stdint.h>
 #include "qemu/queue.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 
 const char *get_opt_name(char *buf, int buf_size, const char *p, char delim);
@@ -44,10 +42,6 @@
 bool has_help_option(const char *param);
 bool is_valid_option_list(const char *param);
 
-typedef struct QemuOpt QemuOpt;
-typedef struct QemuOpts QemuOpts;
-typedef struct QemuOptsList QemuOptsList;
-
 enum QemuOptType {
     QEMU_OPT_STRING = 0,  /* no parsing (use string as-is)                        */
     QEMU_OPT_BOOL,        /* on/off                                               */
@@ -94,28 +88,35 @@
 uint64_t qemu_opt_get_size_del(QemuOpts *opts, const char *name,
                                uint64_t defval);
 int qemu_opt_unset(QemuOpts *opts, const char *name);
-int qemu_opt_set(QemuOpts *opts, const char *name, const char *value);
-void qemu_opt_set_err(QemuOpts *opts, const char *name, const char *value,
-                      Error **errp);
-int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val);
-int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val);
-typedef int (*qemu_opt_loopfunc)(const char *name, const char *value, void *opaque);
+void qemu_opt_set(QemuOpts *opts, const char *name, const char *value,
+                  Error **errp);
+void qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val,
+                       Error **errp);
+void qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val,
+                         Error **errp);
+typedef int (*qemu_opt_loopfunc)(void *opaque,
+                                 const char *name, const char *value,
+                                 Error **errp);
 int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
-                     int abort_on_failure);
+                     Error **errp);
 
 QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id);
 QemuOpts *qemu_opts_create(QemuOptsList *list, const char *id,
                            int fail_if_exists, Error **errp);
 void qemu_opts_reset(QemuOptsList *list);
 void qemu_opts_loc_restore(QemuOpts *opts);
-int qemu_opts_set(QemuOptsList *list, const char *id,
-                  const char *name, const char *value);
+void qemu_opts_set(QemuOptsList *list, const char *id,
+                   const char *name, const char *value, Error **errp);
 const char *qemu_opts_id(QemuOpts *opts);
 void qemu_opts_set_id(QemuOpts *opts, char *id);
 void qemu_opts_del(QemuOpts *opts);
 void qemu_opts_validate(QemuOpts *opts, const QemuOptDesc *desc, Error **errp);
-int qemu_opts_do_parse(QemuOpts *opts, const char *params, const char *firstname);
-QemuOpts *qemu_opts_parse(QemuOptsList *list, const char *params, int permit_abbrev);
+void qemu_opts_do_parse(QemuOpts *opts, const char *params,
+                        const char *firstname, Error **errp);
+QemuOpts *qemu_opts_parse_noisily(QemuOptsList *list, const char *params,
+                                  bool permit_abbrev);
+QemuOpts *qemu_opts_parse(QemuOptsList *list, const char *params,
+                          bool permit_abbrev, Error **errp);
 void qemu_opts_set_defaults(QemuOptsList *list, const char *params,
                             int permit_abbrev);
 QemuOpts *qemu_opts_from_qdict(QemuOptsList *list, const QDict *qdict,
@@ -123,10 +124,10 @@
 QDict *qemu_opts_to_qdict(QemuOpts *opts, QDict *qdict);
 void qemu_opts_absorb_qdict(QemuOpts *opts, QDict *qdict, Error **errp);
 
-typedef int (*qemu_opts_loopfunc)(QemuOpts *opts, void *opaque);
-void qemu_opts_print(QemuOpts *opts);
-int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func, void *opaque,
-                      int abort_on_failure);
+typedef int (*qemu_opts_loopfunc)(void *opaque, QemuOpts *opts, Error **errp);
+int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func,
+                      void *opaque, Error **errp);
+void qemu_opts_print(QemuOpts *opts, const char *sep);
 void qemu_opts_print_help(QemuOptsList *list);
 void qemu_opts_free(QemuOptsList *list);
 QemuOptsList *qemu_opts_append(QemuOptsList *dst, QemuOptsList *list);
diff --git a/include/qemu/option_int.h b/include/qemu/option_int.h
index 6432c1a..26b1d9e 100644
--- a/include/qemu/option_int.h
+++ b/include/qemu/option_int.h
@@ -23,8 +23,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef QEMU_OPTIONS_INTERNAL_H
-#define QEMU_OPTIONS_INTERNAL_H
+#ifndef QEMU_OPTION_INT_H
+#define QEMU_OPTION_INT_H
 
 #include "qemu/option.h"
 #include "qemu/error-report.h"
diff --git a/include/qemu/osdep.h b/include/qemu/osdep.h
index b3300cc..3b9c94d 100644
--- a/include/qemu/osdep.h
+++ b/include/qemu/osdep.h
@@ -1,12 +1,90 @@
+/*
+ * OS includes and handling of OS dependencies
+ *
+ * This header exists to pull in some common system headers that
+ * most code in QEMU will want, and to fix up some possible issues with
+ * it (missing defines, Windows weirdness, and so on).
+ *
+ * To avoid getting into possible circular include dependencies, this
+ * file should not include any other QEMU headers, with the exceptions
+ * of config-host.h, config-target.h, qemu/compiler.h,
+ * sysemu/os-posix.h, sysemu/os-win32.h, glib-compat.h and
+ * qemu/typedefs.h, all of which are doing a similar job to this file
+ * and are under similar constraints.
+ *
+ * This header also contains prototypes for functions defined in
+ * os-*.c and util/oslib-*.c; those would probably be better split
+ * out into separate header files.
+ *
+ * In an ideal world this header would contain only:
+ *  (1) things which everybody needs
+ *  (2) things without which code would work on most platforms but
+ *      fail to compile or misbehave on a minority of host OSes
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
 #ifndef QEMU_OSDEP_H
 #define QEMU_OSDEP_H
 
 #include "config-host.h"
+#ifdef NEED_CPU_H
+#include "config-target.h"
+#else
+#include "exec/poison.h"
+#endif
+#include "qemu/compiler.h"
+
+/* Older versions of C++ don't get definitions of various macros from
+ * stdlib.h unless we define these macros before first inclusion of
+ * that system header.
+ */
+#ifndef __STDC_CONSTANT_MACROS
+#define __STDC_CONSTANT_MACROS
+#endif
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS
+#endif
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+/* The following block of code temporarily renames the daemon() function so the
+ * compiler does not see the warning associated with it in stdlib.h on OSX
+ */
+#ifdef __APPLE__
+#define daemon qemu_fake_daemon_function
+#include <stdlib.h>
+#undef daemon
+extern int daemon(int, int);
+#endif
+
 #include <stdarg.h>
 #include <stddef.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <sys/types.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <strings.h>
+#include <inttypes.h>
+#include <limits.h>
+/* Put unistd.h before time.h as that triggers localtime_r/gmtime_r
+ * function availability on recentish Mingw-w64 platforms. */
+#include <unistd.h>
+#include <time.h>
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <assert.h>
+/* setjmp must be declared before sysemu/os-win32.h
+ * because it is redefined there. */
+#include <setjmp.h>
+#include <signal.h>
+
 #ifdef __OpenBSD__
 #include <sys/signal.h>
 #endif
@@ -18,50 +96,51 @@
 #define WEXITSTATUS(x) (x)
 #endif
 
-#include <sys/time.h>
-
-#if defined(CONFIG_SOLARIS) && CONFIG_SOLARIS_VERSION < 10
-/* [u]int_fast*_t not in <sys/int_types.h> */
-typedef unsigned char           uint_fast8_t;
-typedef unsigned int            uint_fast16_t;
-typedef signed int              int_fast16_t;
+#ifdef _WIN32
+#include "sysemu/os-win32.h"
 #endif
 
-#ifndef glue
-#define xglue(x, y) x ## y
-#define glue(x, y) xglue(x, y)
-#define stringify(s)	tostring(s)
-#define tostring(s)	#s
+#ifdef CONFIG_POSIX
+#include "sysemu/os-posix.h"
 #endif
 
-#ifndef likely
-#if __GNUC__ < 3
-#define __builtin_expect(x, n) (x)
+#include "glib-compat.h"
+#include "qemu/typedefs.h"
+
+#ifndef O_LARGEFILE
+#define O_LARGEFILE 0
+#endif
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+#ifndef ENOMEDIUM
+#define ENOMEDIUM ENODEV
+#endif
+#if !defined(ENOTSUP)
+#define ENOTSUP 4096
+#endif
+#if !defined(ECANCELED)
+#define ECANCELED 4097
+#endif
+#if !defined(EMEDIUMTYPE)
+#define EMEDIUMTYPE 4098
+#endif
+#ifndef TIME_MAX
+#define TIME_MAX LONG_MAX
 #endif
 
-#define likely(x)   __builtin_expect(!!(x), 1)
-#define unlikely(x)   __builtin_expect(!!(x), 0)
-#endif
-
-#ifndef container_of
-#define container_of(ptr, type, member) ({                      \
-        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
-        (type *) ((char *) __mptr - offsetof(type, member));})
-#endif
-
-/* Convert from a base type to a parent type, with compile time checking.  */
-#ifdef __GNUC__
-#define DO_UPCAST(type, field, dev) ( __extension__ ( { \
-    char __attribute__((unused)) offset_must_be_zero[ \
-        -offsetof(type, field)]; \
-    container_of(dev, type, field);}))
+/* HOST_LONG_BITS is the size of a native pointer in bits. */
+#if UINTPTR_MAX == UINT32_MAX
+# define HOST_LONG_BITS 32
+#elif UINTPTR_MAX == UINT64_MAX
+# define HOST_LONG_BITS 64
 #else
-#define DO_UPCAST(type, field, dev) container_of(dev, type, field)
+# error Unknown pointer size
 #endif
 
-#define typeof_field(type, field) typeof(((type *)0)->field)
-#define type_check(t1,t2) ((t1*)0 - (t2*)0)
-
 #ifndef MIN
 #define MIN(a, b) (((a) < (b)) ? (a) : (b))
 #endif
@@ -72,9 +151,34 @@
 /* Minimum function that returns zero only iff both values are zero.
  * Intended for use with unsigned values only. */
 #ifndef MIN_NON_ZERO
-#define MIN_NON_ZERO(a, b) (((a) != 0 && (a) < (b)) ? (a) : (b))
+#define MIN_NON_ZERO(a, b) ((a) == 0 ? (b) : \
+                                ((b) == 0 ? (a) : (MIN(a, b))))
 #endif
 
+/* Round number down to multiple */
+#define QEMU_ALIGN_DOWN(n, m) ((n) / (m) * (m))
+
+/* Round number up to multiple. Safe when m is not a power of 2 (see
+ * ROUND_UP for a faster version when a power of 2 is guaranteed) */
+#define QEMU_ALIGN_UP(n, m) QEMU_ALIGN_DOWN((n) + (m) - 1, (m))
+
+/* Check if n is a multiple of m */
+#define QEMU_IS_ALIGNED(n, m) (((n) % (m)) == 0)
+
+/* n-byte align pointer down */
+#define QEMU_ALIGN_PTR_DOWN(p, n) \
+    ((typeof(p))QEMU_ALIGN_DOWN((uintptr_t)(p), (n)))
+
+/* n-byte align pointer up */
+#define QEMU_ALIGN_PTR_UP(p, n) \
+    ((typeof(p))QEMU_ALIGN_UP((uintptr_t)(p), (n)))
+
+/* Check if pointer p is n-bytes aligned */
+#define QEMU_PTR_IS_ALIGNED(p, n) QEMU_IS_ALIGNED((uintptr_t)(p), (n))
+
+/* Round number up to multiple. Requires that d be a power of 2 (see
+ * QEMU_ALIGN_UP for a safer but slower version on arbitrary
+ * numbers) */
 #ifndef ROUND_UP
 #define ROUND_UP(n,d) (((n) + (d) - 1) & -(d))
 #endif
@@ -87,20 +191,6 @@
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 #endif
 
-#ifndef always_inline
-#if !((__GNUC__ < 3) || defined(__APPLE__))
-#ifdef __OPTIMIZE__
-#undef inline
-#define inline __attribute__ (( always_inline )) __inline__
-#endif
-#endif
-#else
-#undef inline
-#define inline always_inline
-#endif
-
-#define qemu_printf printf
-
 int qemu_daemon(int nochdir, int noclose);
 void *qemu_try_memalign(size_t alignment, size_t size);
 void *qemu_memalign(size_t alignment, size_t size);
@@ -144,6 +234,11 @@
 #else
 #define QEMU_MADV_HUGEPAGE QEMU_MADV_INVALID
 #endif
+#ifdef MADV_NOHUGEPAGE
+#define QEMU_MADV_NOHUGEPAGE MADV_NOHUGEPAGE
+#else
+#define QEMU_MADV_NOHUGEPAGE QEMU_MADV_INVALID
+#endif
 
 #elif defined(CONFIG_POSIX_MADVISE)
 
@@ -155,6 +250,7 @@
 #define QEMU_MADV_DODUMP QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+#define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
 
 #else /* no-op */
 
@@ -166,13 +262,30 @@
 #define QEMU_MADV_DODUMP QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
+#define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
 
 #endif
 
+#if defined(__linux__) && \
+    (defined(__x86_64__) || defined(__arm__) || defined(__aarch64__))
+   /* Use 2 MiB alignment so transparent hugepages can be used by KVM.
+      Valgrind does not support alignments larger than 1 MiB,
+      therefore we need special code which handles running on Valgrind. */
+#  define QEMU_VMALLOC_ALIGN (512 * 4096)
+#elif defined(__linux__) && defined(__s390x__)
+   /* Use 1 MiB (segment size) alignment so gmap can be used by KVM. */
+#  define QEMU_VMALLOC_ALIGN (256 * 4096)
+#else
+#  define QEMU_VMALLOC_ALIGN getpagesize()
+#endif
+
 int qemu_madvise(void *addr, size_t len, int advice);
 
 int qemu_open(const char *name, int flags, ...);
 int qemu_close(int fd);
+#ifndef _WIN32
+int qemu_dup(int fd);
+#endif
 
 #if defined(__HAIKU__) && defined(__i386__)
 #define FMT_pid "%ld"
@@ -220,8 +333,21 @@
 
 void qemu_set_cloexec(int fd);
 
-void qemu_set_version(const char *);
-const char *qemu_get_version(void);
+/* Starting on QEMU 2.5, qemu_hw_version() returns "2.5+" by default
+ * instead of QEMU_VERSION, so setting hw_version on MachineClass
+ * is no longer mandatory.
+ *
+ * Do NOT change this string, or it will break compatibility on all
+ * machine classes that don't set hw_version.
+ */
+#define QEMU_HW_VERSION "2.5+"
+
+/* QEMU "hardware version" setting. Used to replace code that exposed
+ * QEMU_VERSION to guests in the past and need to keep compatibility.
+ * Do not use qemu_hw_version() in new code.
+ */
+void qemu_set_hw_version(const char *);
+const char *qemu_hw_version(void);
 
 void fips_set_state(bool requested);
 bool fips_get_state(void);
@@ -257,6 +383,26 @@
 
 void qemu_set_tty_echo(int fd, bool echo);
 
-void os_mem_prealloc(int fd, char *area, size_t sz);
+void os_mem_prealloc(int fd, char *area, size_t sz, Error **errp);
+
+int qemu_read_password(char *buf, int buf_size);
+
+/**
+ * qemu_fork:
+ *
+ * A version of fork that avoids signal handler race
+ * conditions that can lead to child process getting
+ * signals that are otherwise only expected by the
+ * parent. It also resets all signal handlers to the
+ * default settings.
+ *
+ * Returns 0 to child process, pid number to parent
+ * or -1 on failure.
+ */
+pid_t qemu_fork(Error **errp);
+
+/* Use a custom handler for user-initiated exits (e.g. Ctrl-C).
+ * The default is to call qemu_system_shutdown_request(). */
+void qemu_set_ctrlc_handler(void(*handler)(void));
 
 #endif
diff --git a/include/qemu/path.h b/include/qemu/path.h
new file mode 100644
index 0000000..c6292a9
--- /dev/null
+++ b/include/qemu/path.h
@@ -0,0 +1,7 @@
+#ifndef QEMU_PATH_H
+#define QEMU_PATH_H
+
+void init_paths(const char *prefix);
+const char *path(const char *pathname);
+
+#endif
diff --git a/include/qemu/processor.h b/include/qemu/processor.h
new file mode 100644
index 0000000..8b25702
--- /dev/null
+++ b/include/qemu/processor.h
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2.
+ *   See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_PROCESSOR_H
+#define QEMU_PROCESSOR_H
+
+#include "qemu/atomic.h"
+
+#if defined(__i386__) || defined(__x86_64__)
+# define cpu_relax() asm volatile("rep; nop" ::: "memory")
+
+#elif defined(__ia64__)
+# define cpu_relax() asm volatile("hint @pause" ::: "memory")
+
+#elif defined(__aarch64__)
+# define cpu_relax() asm volatile("yield" ::: "memory")
+
+#elif defined(__powerpc64__)
+/* set Hardware Multi-Threading (HMT) priority to low; then back to medium */
+# define cpu_relax() asm volatile("or 1, 1, 1;" \
+                                  "or 2, 2, 2;" ::: "memory")
+
+#else
+# define cpu_relax() barrier()
+#endif
+
+#endif /* QEMU_PROCESSOR_H */
diff --git a/include/qemu/qdist.h b/include/qemu/qdist.h
new file mode 100644
index 0000000..54ece76
--- /dev/null
+++ b/include/qemu/qdist.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_QDIST_H
+#define QEMU_QDIST_H
+
+#include "qemu-common.h"
+#include "qemu/bitops.h"
+
+/*
+ * Samples with the same 'x value' end up in the same qdist_entry,
+ * e.g. inc(0.1) and inc(0.1) end up as {x=0.1, count=2}.
+ *
+ * Binning happens only at print time, so that we retain the flexibility to
+ * choose the binning. This might not be ideal for workloads that do not care
+ * much about precision and insert many samples all with different x values;
+ * in that case, pre-binning (e.g. entering both 0.115 and 0.097 as 0.1)
+ * should be considered.
+ */
+struct qdist_entry {
+    double x;
+    unsigned long count;
+};
+
+struct qdist {
+    struct qdist_entry *entries;
+    size_t n;
+    size_t size;
+};
+
+#define QDIST_PR_BORDER     BIT(0)
+#define QDIST_PR_LABELS     BIT(1)
+/* the remaining options only work if PR_LABELS is set */
+#define QDIST_PR_NODECIMAL  BIT(2)
+#define QDIST_PR_PERCENT    BIT(3)
+#define QDIST_PR_100X       BIT(4)
+#define QDIST_PR_NOBINRANGE BIT(5)
+
+void qdist_init(struct qdist *dist);
+void qdist_destroy(struct qdist *dist);
+
+void qdist_add(struct qdist *dist, double x, long count);
+void qdist_inc(struct qdist *dist, double x);
+double qdist_xmin(const struct qdist *dist);
+double qdist_xmax(const struct qdist *dist);
+double qdist_avg(const struct qdist *dist);
+unsigned long qdist_sample_count(const struct qdist *dist);
+size_t qdist_unique_entries(const struct qdist *dist);
+
+/* callers must free the returned string with g_free() */
+char *qdist_pr_plain(const struct qdist *dist, size_t n_groups);
+
+/* callers must free the returned string with g_free() */
+char *qdist_pr(const struct qdist *dist, size_t n_groups, uint32_t opt);
+
+/* Only qdist code and test code should ever call this function */
+void qdist_bin__internal(struct qdist *to, const struct qdist *from, size_t n);
+
+#endif /* QEMU_QDIST_H */
diff --git a/include/qemu/qht.h b/include/qemu/qht.h
new file mode 100644
index 0000000..311139b
--- /dev/null
+++ b/include/qemu/qht.h
@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_QHT_H
+#define QEMU_QHT_H
+
+#include "qemu/seqlock.h"
+#include "qemu/thread.h"
+#include "qemu/qdist.h"
+
+struct qht {
+    struct qht_map *map;
+    QemuMutex lock; /* serializes setters of ht->map */
+    unsigned int mode;
+};
+
+/**
+ * struct qht_stats - Statistics of a QHT
+ * @head_buckets: number of head buckets
+ * @used_head_buckets: number of non-empty head buckets
+ * @entries: total number of entries
+ * @chain: frequency distribution representing the number of buckets in each
+ *         chain, excluding empty chains.
+ * @occupancy: frequency distribution representing chain occupancy rate.
+ *             Valid range: from 0.0 (empty) to 1.0 (full occupancy).
+ *
+ * An entry is a pointer-hash pair.
+ * Each bucket can host several entries.
+ * Chains are chains of buckets, whose first link is always a head bucket.
+ */
+struct qht_stats {
+    size_t head_buckets;
+    size_t used_head_buckets;
+    size_t entries;
+    struct qdist chain;
+    struct qdist occupancy;
+};
+
+typedef bool (*qht_lookup_func_t)(const void *obj, const void *userp);
+typedef void (*qht_iter_func_t)(struct qht *ht, void *p, uint32_t h, void *up);
+
+#define QHT_MODE_AUTO_RESIZE 0x1 /* auto-resize when heavily loaded */
+
+/**
+ * qht_init - Initialize a QHT
+ * @ht: QHT to be initialized
+ * @n_elems: number of entries the hash table should be optimized for.
+ * @mode: bitmask with OR'ed QHT_MODE_*
+ */
+void qht_init(struct qht *ht, size_t n_elems, unsigned int mode);
+
+/**
+ * qht_destroy - destroy a previously initialized QHT
+ * @ht: QHT to be destroyed
+ *
+ * Call only when there are no readers/writers left.
+ */
+void qht_destroy(struct qht *ht);
+
+/**
+ * qht_insert - Insert a pointer into the hash table
+ * @ht: QHT to insert to
+ * @p: pointer to be inserted
+ * @hash: hash corresponding to @p
+ *
+ * Attempting to insert a NULL @p is a bug.
+ * Inserting the same pointer @p with different @hash values is a bug.
+ *
+ * In case of successful operation, smp_wmb() is implied before the pointer is
+ * inserted into the hash table.
+ *
+ * Returns true on sucess.
+ * Returns false if the @p-@hash pair already exists in the hash table.
+ */
+bool qht_insert(struct qht *ht, void *p, uint32_t hash);
+
+/**
+ * qht_lookup - Look up a pointer in a QHT
+ * @ht: QHT to be looked up
+ * @func: function to compare existing pointers against @userp
+ * @userp: pointer to pass to @func
+ * @hash: hash of the pointer to be looked up
+ *
+ * Needs to be called under an RCU read-critical section.
+ *
+ * smp_read_barrier_depends() is implied before the call to @func.
+ *
+ * The user-provided @func compares pointers in QHT against @userp.
+ * If the function returns true, a match has been found.
+ *
+ * Returns the corresponding pointer when a match is found.
+ * Returns NULL otherwise.
+ */
+void *qht_lookup(struct qht *ht, qht_lookup_func_t func, const void *userp,
+                 uint32_t hash);
+
+/**
+ * qht_remove - remove a pointer from the hash table
+ * @ht: QHT to remove from
+ * @p: pointer to be removed
+ * @hash: hash corresponding to @p
+ *
+ * Attempting to remove a NULL @p is a bug.
+ *
+ * Just-removed @p pointers cannot be immediately freed; they need to remain
+ * valid until the end of the RCU grace period in which qht_remove() is called.
+ * This guarantees that concurrent lookups will always compare against valid
+ * data.
+ *
+ * Returns true on success.
+ * Returns false if the @p-@hash pair was not found.
+ */
+bool qht_remove(struct qht *ht, const void *p, uint32_t hash);
+
+/**
+ * qht_reset - reset a QHT
+ * @ht: QHT to be reset
+ *
+ * All entries in the hash table are reset. No resizing is performed.
+ *
+ * If concurrent readers may exist, the objects pointed to by the hash table
+ * must remain valid for the existing RCU grace period -- see qht_remove().
+ * See also: qht_reset_size()
+ */
+void qht_reset(struct qht *ht);
+
+/**
+ * qht_reset_size - reset and resize a QHT
+ * @ht: QHT to be reset and resized
+ * @n_elems: number of entries the resized hash table should be optimized for.
+ *
+ * Returns true if the resize was necessary and therefore performed.
+ * Returns false otherwise.
+ *
+ * If concurrent readers may exist, the objects pointed to by the hash table
+ * must remain valid for the existing RCU grace period -- see qht_remove().
+ * See also: qht_reset(), qht_resize().
+ */
+bool qht_reset_size(struct qht *ht, size_t n_elems);
+
+/**
+ * qht_resize - resize a QHT
+ * @ht: QHT to be resized
+ * @n_elems: number of entries the resized hash table should be optimized for
+ *
+ * Returns true on success.
+ * Returns false if the resize was not necessary and therefore not performed.
+ * See also: qht_reset_size().
+ */
+bool qht_resize(struct qht *ht, size_t n_elems);
+
+/**
+ * qht_iter - Iterate over a QHT
+ * @ht: QHT to be iterated over
+ * @func: function to be called for each entry in QHT
+ * @userp: additional pointer to be passed to @func
+ *
+ * Each time it is called, user-provided @func is passed a pointer-hash pair,
+ * plus @userp.
+ */
+void qht_iter(struct qht *ht, qht_iter_func_t func, void *userp);
+
+/**
+ * qht_statistics_init - Gather statistics from a QHT
+ * @ht: QHT to gather statistics from
+ * @stats: pointer to a struct qht_stats to be filled in
+ *
+ * Does NOT need to be called under an RCU read-critical section,
+ * since it does not dereference any pointers stored in the hash table.
+ *
+ * When done with @stats, pass the struct to qht_statistics_destroy().
+ * Failing to do this will leak memory.
+ */
+void qht_statistics_init(struct qht *ht, struct qht_stats *stats);
+
+/**
+ * qht_statistics_destroy - Destroy a struct qht_stats
+ * @stats: stuct qht_stats to be destroyed
+ *
+ * See also: qht_statistics_init().
+ */
+void qht_statistics_destroy(struct qht_stats *stats);
+
+#endif /* QEMU_QHT_H */
diff --git a/include/qemu/queue.h b/include/qemu/queue.h
index d433b90..c2b6c81 100644
--- a/include/qemu/queue.h
+++ b/include/qemu/queue.h
@@ -37,8 +37,8 @@
  *      @(#)queue.h     8.5 (Berkeley) 8/20/94
  */
 
-#ifndef QEMU_SYS_QUEUE_H_
-#define QEMU_SYS_QUEUE_H_
+#ifndef QEMU_SYS_QUEUE_H
+#define QEMU_SYS_QUEUE_H
 
 /*
  * This file defines four types of data structures: singly-linked lists,
@@ -104,6 +104,19 @@
         (head)->lh_first = NULL;                                        \
 } while (/*CONSTCOND*/0)
 
+#define QLIST_SWAP(dstlist, srclist, field) do {                        \
+        void *tmplist;                                                  \
+        tmplist = (srclist)->lh_first;                                  \
+        (srclist)->lh_first = (dstlist)->lh_first;                      \
+        if ((srclist)->lh_first != NULL) {                              \
+            (srclist)->lh_first->field.le_prev = &(srclist)->lh_first;  \
+        }                                                               \
+        (dstlist)->lh_first = tmplist;                                  \
+        if ((dstlist)->lh_first != NULL) {                              \
+            (dstlist)->lh_first->field.le_prev = &(dstlist)->lh_first;  \
+        }                                                               \
+} while (/*CONSTCOND*/0)
+
 #define QLIST_INSERT_AFTER(listelm, elm, field) do {                    \
         if (((elm)->field.le_next = (listelm)->field.le_next) != NULL)  \
                 (listelm)->field.le_next->field.le_prev =               \
@@ -126,17 +139,6 @@
         (elm)->field.le_prev = &(head)->lh_first;                       \
 } while (/*CONSTCOND*/0)
 
-#define QLIST_INSERT_HEAD_RCU(head, elm, field) do {                    \
-        (elm)->field.le_prev = &(head)->lh_first;                       \
-        (elm)->field.le_next = (head)->lh_first;                        \
-        smp_wmb(); /* fill elm before linking it */                     \
-        if ((head)->lh_first != NULL)  {                                \
-            (head)->lh_first->field.le_prev = &(elm)->field.le_next;    \
-        }                                                               \
-        (head)->lh_first = (elm);                                       \
-        smp_wmb();                                                      \
-} while (/* CONSTCOND*/0)
-
 #define QLIST_REMOVE(elm, field) do {                                   \
         if ((elm)->field.le_next != NULL)                               \
                 (elm)->field.le_next->field.le_prev =                   \
@@ -191,8 +193,20 @@
 } while (/*CONSTCOND*/0)
 
 #define QSLIST_INSERT_HEAD(head, elm, field) do {                        \
-        (elm)->field.sle_next = (head)->slh_first;                      \
-        (head)->slh_first = (elm);                                      \
+        (elm)->field.sle_next = (head)->slh_first;                       \
+        (head)->slh_first = (elm);                                       \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_INSERT_HEAD_ATOMIC(head, elm, field) do {                     \
+        typeof(elm) save_sle_next;                                           \
+        do {                                                                 \
+            save_sle_next = (elm)->field.sle_next = (head)->slh_first;       \
+        } while (atomic_cmpxchg(&(head)->slh_first, save_sle_next, (elm)) != \
+                 save_sle_next);                                             \
+} while (/*CONSTCOND*/0)
+
+#define QSLIST_MOVE_ATOMIC(dest, src) do {                               \
+        (dest)->slh_first = atomic_xchg(&(src)->slh_first, NULL);        \
 } while (/*CONSTCOND*/0)
 
 #define QSLIST_REMOVE_HEAD(head, field) do {                             \
@@ -268,6 +282,17 @@
         (head)->sqh_last = &(head)->sqh_first;                          \
 } while (/*CONSTCOND*/0)
 
+#define QSIMPLEQ_SPLIT_AFTER(head, elm, field, removed) do {            \
+    QSIMPLEQ_INIT(removed);                                             \
+    if (((removed)->sqh_first = (head)->sqh_first) != NULL) {           \
+        if (((head)->sqh_first = (elm)->field.sqe_next) == NULL) {      \
+            (head)->sqh_last = &(head)->sqh_first;                      \
+        }                                                               \
+        (removed)->sqh_last = &(elm)->field.sqe_next;                   \
+        (elm)->field.sqe_next = NULL;                                   \
+    }                                                                   \
+} while (/*CONSTCOND*/0)
+
 #define QSIMPLEQ_REMOVE(head, elm, type, field) do {                    \
     if ((head)->sqh_first == (elm)) {                                   \
         QSIMPLEQ_REMOVE_HEAD((head), field);                            \
@@ -411,4 +436,4 @@
 #define QTAILQ_PREV(elm, headname, field) \
         (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
 
-#endif  /* !QEMU_SYS_QUEUE_H_ */
+#endif /* QEMU_SYS_QUEUE_H */
diff --git a/include/qemu/range.h b/include/qemu/range.h
index cfa021f..f28f0c1 100644
--- a/include/qemu/range.h
+++ b/include/qemu/range.h
@@ -1,39 +1,136 @@
+/*
+ * QEMU 64-bit address ranges
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
 #ifndef QEMU_RANGE_H
 #define QEMU_RANGE_H
 
-#include <inttypes.h>
-#include <qemu/typedefs.h>
 #include "qemu/queue.h"
 
 /*
  * Operations on 64 bit address ranges.
  * Notes:
- *   - ranges must not wrap around 0, but can include the last byte ~0x0LL.
- *   - this can not represent a full 0 to ~0x0LL range.
+ * - Ranges must not wrap around 0, but can include UINT64_MAX.
  */
 
-/* A structure representing a range of addresses. */
 struct Range {
-    uint64_t begin; /* First byte of the range, or 0 if empty. */
-    uint64_t end;   /* 1 + the last byte. 0 if range empty or ends at ~0x0LL. */
+    /*
+     * Do not access members directly, use the functions!
+     * A non-empty range has @lob <= @upb.
+     * An empty range has @lob == @upb + 1.
+     */
+    uint64_t lob;        /* inclusive lower bound */
+    uint64_t upb;        /* inclusive upper bound */
 };
 
+static inline void range_invariant(Range *range)
+{
+    assert(range->lob <= range->upb || range->lob == range->upb + 1);
+}
+
+/* Compound literal encoding the empty range */
+#define range_empty ((Range){ .lob = 1, .upb = 0 })
+
+/* Is @range empty? */
+static inline bool range_is_empty(Range *range)
+{
+    range_invariant(range);
+    return range->lob > range->upb;
+}
+
+/* Does @range contain @val? */
+static inline bool range_contains(Range *range, uint64_t val)
+{
+    return val >= range->lob && val <= range->upb;
+}
+
+/* Initialize @range to the empty range */
+static inline void range_make_empty(Range *range)
+{
+    *range = range_empty;
+    assert(range_is_empty(range));
+}
+
+/*
+ * Initialize @range to span the interval [@lob,@upb].
+ * Both bounds are inclusive.
+ * The interval must not be empty, i.e. @lob must be less than or
+ * equal @upb.
+ */
+static inline void range_set_bounds(Range *range, uint64_t lob, uint64_t upb)
+{
+    range->lob = lob;
+    range->upb = upb;
+    assert(!range_is_empty(range));
+}
+
+/*
+ * Initialize @range to span the interval [@lob,@upb_plus1).
+ * The lower bound is inclusive, the upper bound is exclusive.
+ * Zero @upb_plus1 is special: if @lob is also zero, set @range to the
+ * empty range.  Else, set @range to [@lob,UINT64_MAX].
+ */
+static inline void range_set_bounds1(Range *range,
+                                     uint64_t lob, uint64_t upb_plus1)
+{
+    if (!lob && !upb_plus1) {
+        *range = range_empty;
+    } else {
+        range->lob = lob;
+        range->upb = upb_plus1 - 1;
+    }
+    range_invariant(range);
+}
+
+/* Return @range's lower bound.  @range must not be empty. */
+static inline uint64_t range_lob(Range *range)
+{
+    assert(!range_is_empty(range));
+    return range->lob;
+}
+
+/* Return @range's upper bound.  @range must not be empty. */
+static inline uint64_t range_upb(Range *range)
+{
+    assert(!range_is_empty(range));
+    return range->upb;
+}
+
+/*
+ * Extend @range to the smallest interval that includes @extend_by, too.
+ */
 static inline void range_extend(Range *range, Range *extend_by)
 {
-    if (!extend_by->begin && !extend_by->end) {
+    if (range_is_empty(extend_by)) {
         return;
     }
-    if (!range->begin && !range->end) {
+    if (range_is_empty(range)) {
         *range = *extend_by;
         return;
     }
-    if (range->begin > extend_by->begin) {
-        range->begin = extend_by->begin;
+    if (range->lob > extend_by->lob) {
+        range->lob = extend_by->lob;
     }
-    /* Compare last byte in case region ends at ~0x0LL */
-    if (range->end - 1 < extend_by->end - 1) {
-        range->end = extend_by->end;
+    if (range->upb < extend_by->upb) {
+        range->upb = extend_by->upb;
     }
+    range_invariant(range);
 }
 
 /* Get last byte of a range from offset + length.
@@ -61,75 +158,6 @@
     return !(last2 < first1 || last1 < first2);
 }
 
-/* 0,1 can merge with 1,2 but don't overlap */
-static inline bool ranges_can_merge(Range *range1, Range *range2)
-{
-    return !(range1->end < range2->begin || range2->end < range1->begin);
-}
-
-static inline int range_merge(Range *range1, Range *range2)
-{
-    if (ranges_can_merge(range1, range2)) {
-        if (range1->end < range2->end) {
-            range1->end = range2->end;
-        }
-        if (range1->begin > range2->begin) {
-            range1->begin = range2->begin;
-        }
-        return 0;
-    }
-
-    return -1;
-}
-
-static inline GList *g_list_insert_sorted_merged(GList *list,
-                                                 gpointer data,
-                                                 GCompareFunc func)
-{
-    GList *l, *next = NULL;
-    Range *r, *nextr;
-
-    if (!list) {
-        list = g_list_insert_sorted(list, data, func);
-        return list;
-    }
-
-    nextr = data;
-    l = list;
-    while (l && l != next && nextr) {
-        r = l->data;
-        if (ranges_can_merge(r, nextr)) {
-            range_merge(r, nextr);
-            l = g_list_remove_link(l, next);
-            next = g_list_next(l);
-            if (next) {
-                nextr = next->data;
-            } else {
-                nextr = NULL;
-            }
-        } else {
-            l = g_list_next(l);
-        }
-    }
-
-    if (!l) {
-        list = g_list_insert_sorted(list, data, func);
-    }
-
-    return list;
-}
-
-static inline gint range_compare(gconstpointer a, gconstpointer b)
-{
-    Range *ra = (Range *)a, *rb = (Range *)b;
-    if (ra->begin == rb->begin && ra->end == rb->end) {
-        return 0;
-    } else if (range_get_last(ra->begin, ra->end) <
-               range_get_last(rb->begin, rb->end)) {
-        return -1;
-    } else {
-        return 1;
-    }
-}
+GList *range_list_insert(GList *list, Range *data);
 
 #endif
diff --git a/include/qemu/ratelimit.h b/include/qemu/ratelimit.h
index d413a4a..8da1232 100644
--- a/include/qemu/ratelimit.h
+++ b/include/qemu/ratelimit.h
@@ -12,37 +12,62 @@
  */
 
 #ifndef QEMU_RATELIMIT_H
-#define QEMU_RATELIMIT_H 1
+#define QEMU_RATELIMIT_H
 
 typedef struct {
-    int64_t next_slice_time;
+    int64_t slice_start_time;
+    int64_t slice_end_time;
     uint64_t slice_quota;
     uint64_t slice_ns;
     uint64_t dispatched;
 } RateLimit;
 
+/** Calculate and return delay for next request in ns
+ *
+ * Record that we sent @p n data units. If we may send more data units
+ * in the current time slice, return 0 (i.e. no delay). Otherwise
+ * return the amount of time (in ns) until the start of the next time
+ * slice that will permit sending the next chunk of data.
+ *
+ * Recording sent data units even after exceeding the quota is
+ * permitted; the time slice will be extended accordingly.
+ */
 static inline int64_t ratelimit_calculate_delay(RateLimit *limit, uint64_t n)
 {
     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    uint64_t delay_slices;
 
-    if (limit->next_slice_time < now) {
-        limit->next_slice_time = now + limit->slice_ns;
+    assert(limit->slice_quota && limit->slice_ns);
+
+    if (limit->slice_end_time < now) {
+        /* Previous, possibly extended, time slice finished; reset the
+         * accounting. */
+        limit->slice_start_time = now;
+        limit->slice_end_time = now + limit->slice_ns;
         limit->dispatched = 0;
     }
-    if (limit->dispatched == 0 || limit->dispatched + n <= limit->slice_quota) {
-        limit->dispatched += n;
+
+    limit->dispatched += n;
+    if (limit->dispatched < limit->slice_quota) {
+        /* We may send further data within the current time slice, no
+         * need to delay the next request. */
         return 0;
-    } else {
-        limit->dispatched = n;
-        return limit->next_slice_time - now;
     }
+
+    /* Quota exceeded. Calculate the next time slice we may start
+     * sending data again. */
+    delay_slices = (limit->dispatched + limit->slice_quota - 1) /
+        limit->slice_quota;
+    limit->slice_end_time = limit->slice_start_time +
+        delay_slices * limit->slice_ns;
+    return limit->slice_end_time - now;
 }
 
 static inline void ratelimit_set_speed(RateLimit *limit, uint64_t speed,
                                        uint64_t slice_ns)
 {
     limit->slice_ns = slice_ns;
-    limit->slice_quota = ((double)speed * slice_ns)/1000000000ULL;
+    limit->slice_quota = MAX(((double)speed * slice_ns) / 1000000000ULL, 1);
 }
 
 #endif
diff --git a/include/qemu/rcu.h b/include/qemu/rcu.h
new file mode 100644
index 0000000..83ae280
--- /dev/null
+++ b/include/qemu/rcu.h
@@ -0,0 +1,144 @@
+#ifndef QEMU_RCU_H
+#define QEMU_RCU_H
+
+/*
+ * urcu-mb.h
+ *
+ * Userspace RCU header with explicit memory barrier.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+
+#include "qemu/thread.h"
+#include "qemu/queue.h"
+#include "qemu/atomic.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+ * Important !
+ *
+ * Each thread containing read-side critical sections must be registered
+ * with rcu_register_thread() before calling rcu_read_lock().
+ * rcu_unregister_thread() should be called before the thread exits.
+ */
+
+#ifdef DEBUG_RCU
+#define rcu_assert(args...)    assert(args)
+#else
+#define rcu_assert(args...)
+#endif
+
+/*
+ * Global quiescent period counter with low-order bits unused.
+ * Using a int rather than a char to eliminate false register dependencies
+ * causing stalls on some architectures.
+ */
+extern unsigned long rcu_gp_ctr;
+
+extern QemuEvent rcu_gp_event;
+
+struct rcu_reader_data {
+    /* Data used by both reader and synchronize_rcu() */
+    unsigned long ctr;
+    bool waiting;
+
+    /* Data used by reader only */
+    unsigned depth;
+
+    /* Data used for registry, protected by rcu_registry_lock */
+    QLIST_ENTRY(rcu_reader_data) node;
+};
+
+extern __thread struct rcu_reader_data rcu_reader;
+
+static inline void rcu_read_lock(void)
+{
+    struct rcu_reader_data *p_rcu_reader = &rcu_reader;
+    unsigned ctr;
+
+    if (p_rcu_reader->depth++ > 0) {
+        return;
+    }
+
+    ctr = atomic_read(&rcu_gp_ctr);
+    atomic_xchg(&p_rcu_reader->ctr, ctr);
+}
+
+static inline void rcu_read_unlock(void)
+{
+    struct rcu_reader_data *p_rcu_reader = &rcu_reader;
+
+    assert(p_rcu_reader->depth != 0);
+    if (--p_rcu_reader->depth > 0) {
+        return;
+    }
+
+    atomic_xchg(&p_rcu_reader->ctr, 0);
+    if (unlikely(atomic_read(&p_rcu_reader->waiting))) {
+        atomic_set(&p_rcu_reader->waiting, false);
+        qemu_event_set(&rcu_gp_event);
+    }
+}
+
+extern void synchronize_rcu(void);
+
+/*
+ * Reader thread registration.
+ */
+extern void rcu_register_thread(void);
+extern void rcu_unregister_thread(void);
+extern void rcu_after_fork(void);
+
+struct rcu_head;
+typedef void RCUCBFunc(struct rcu_head *head);
+
+struct rcu_head {
+    struct rcu_head *next;
+    RCUCBFunc *func;
+};
+
+extern void call_rcu1(struct rcu_head *head, RCUCBFunc *func);
+
+/* The operands of the minus operator must have the same type,
+ * which must be the one that we specify in the cast.
+ */
+#define call_rcu(head, func, field)                                      \
+    call_rcu1(({                                                         \
+         char __attribute__((unused))                                    \
+            offset_must_be_zero[-offsetof(typeof(*(head)), field)],      \
+            func_type_invalid = (func) - (void (*)(typeof(head)))(func); \
+         &(head)->field;                                                 \
+      }),                                                                \
+      (RCUCBFunc *)(func))
+
+#define g_free_rcu(obj, field) \
+    call_rcu1(({                                                         \
+        char __attribute__((unused))                                     \
+            offset_must_be_zero[-offsetof(typeof(*(obj)), field)];       \
+        &(obj)->field;                                                   \
+      }),                                                                \
+      (RCUCBFunc *)g_free);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* QEMU_RCU_H */
diff --git a/include/qemu/rcu_queue.h b/include/qemu/rcu_queue.h
new file mode 100644
index 0000000..01be774
--- /dev/null
+++ b/include/qemu/rcu_queue.h
@@ -0,0 +1,134 @@
+#ifndef QEMU_RCU_QUEUE_H
+#define QEMU_RCU_QUEUE_H
+
+/*
+ * rcu_queue.h
+ *
+ * RCU-friendly versions of the queue.h primitives.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * Copyright (c) 2013 Mike D. Day, IBM Corporation.
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include "qemu/queue.h"
+#include "qemu/atomic.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/*
+ * List access methods.
+ */
+#define QLIST_EMPTY_RCU(head) (atomic_rcu_read(&(head)->lh_first) == NULL)
+#define QLIST_FIRST_RCU(head) (atomic_rcu_read(&(head)->lh_first))
+#define QLIST_NEXT_RCU(elm, field) (atomic_rcu_read(&(elm)->field.le_next))
+
+/*
+ * List functions.
+ */
+
+
+/*
+ *  The difference between atomic_read/set and atomic_rcu_read/set
+ *  is in the including of a read/write memory barrier to the volatile
+ *  access. atomic_rcu_* macros include the memory barrier, the
+ *  plain atomic macros do not. Therefore, it should be correct to
+ *  issue a series of reads or writes to the same element using only
+ *  the atomic_* macro, until the last read or write, which should be
+ *  atomic_rcu_* to introduce a read or write memory barrier as
+ *  appropriate.
+ */
+
+/* Upon publication of the listelm->next value, list readers
+ * will see the new node when following next pointers from
+ * antecedent nodes, but may not see the new node when following
+ * prev pointers from subsequent nodes until after the RCU grace
+ * period expires.
+ * see linux/include/rculist.h __list_add_rcu(new, prev, next)
+ */
+#define QLIST_INSERT_AFTER_RCU(listelm, elm, field) do {    \
+    (elm)->field.le_next = (listelm)->field.le_next;        \
+    (elm)->field.le_prev = &(listelm)->field.le_next;       \
+    atomic_rcu_set(&(listelm)->field.le_next, (elm));       \
+    if ((elm)->field.le_next != NULL) {                     \
+       (elm)->field.le_next->field.le_prev =                \
+        &(elm)->field.le_next;                              \
+    }                                                       \
+} while (/*CONSTCOND*/0)
+
+/* Upon publication of the listelm->prev->next value, list
+ * readers will see the new element when following prev pointers
+ * from subsequent elements, but may not see the new element
+ * when following next pointers from antecedent elements
+ * until after the RCU grace period expires.
+ */
+#define QLIST_INSERT_BEFORE_RCU(listelm, elm, field) do {   \
+    (elm)->field.le_prev = (listelm)->field.le_prev;        \
+    (elm)->field.le_next = (listelm);                       \
+    atomic_rcu_set((listelm)->field.le_prev, (elm));        \
+    (listelm)->field.le_prev = &(elm)->field.le_next;       \
+} while (/*CONSTCOND*/0)
+
+/* Upon publication of the head->first value, list readers
+ * will see the new element when following the head, but may
+ * not see the new element when following prev pointers from
+ * subsequent elements until after the RCU grace period has
+ * expired.
+ */
+#define QLIST_INSERT_HEAD_RCU(head, elm, field) do {    \
+    (elm)->field.le_prev = &(head)->lh_first;           \
+    (elm)->field.le_next = (head)->lh_first;            \
+    atomic_rcu_set((&(head)->lh_first), (elm));         \
+    if ((elm)->field.le_next != NULL) {                 \
+       (elm)->field.le_next->field.le_prev =            \
+        &(elm)->field.le_next;                          \
+    }                                                   \
+} while (/*CONSTCOND*/0)
+
+
+/* prior to publication of the elm->prev->next value, some list
+ * readers may still see the removed element when following
+ * the antecedent's next pointer.
+ */
+#define QLIST_REMOVE_RCU(elm, field) do {           \
+    if ((elm)->field.le_next != NULL) {             \
+       (elm)->field.le_next->field.le_prev =        \
+        (elm)->field.le_prev;                       \
+    }                                               \
+    *(elm)->field.le_prev =  (elm)->field.le_next;  \
+} while (/*CONSTCOND*/0)
+
+/* List traversal must occur within an RCU critical section.  */
+#define QLIST_FOREACH_RCU(var, head, field)                 \
+        for ((var) = atomic_rcu_read(&(head)->lh_first);    \
+                (var);                                      \
+                (var) = atomic_rcu_read(&(var)->field.le_next))
+
+/* List traversal must occur within an RCU critical section.  */
+#define QLIST_FOREACH_SAFE_RCU(var, head, field, next_var)           \
+    for ((var) = (atomic_rcu_read(&(head)->lh_first));               \
+      (var) &&                                                       \
+          ((next_var) = atomic_rcu_read(&(var)->field.le_next), 1);  \
+           (var) = (next_var))
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* QEMU_RCU_QUEUE_H */
diff --git a/include/qemu/readline.h b/include/qemu/readline.h
index 49efe4e..c08cf74 100644
--- a/include/qemu/readline.h
+++ b/include/qemu/readline.h
@@ -60,4 +60,4 @@
                              void *opaque,
                              ReadLineCompletionFunc *completion_finder);
 
-#endif /* !READLINE_H */
+#endif /* READLINE_H */
diff --git a/include/qemu/seqlock.h b/include/qemu/seqlock.h
index 3ff118a..2e2be4c 100644
--- a/include/qemu/seqlock.h
+++ b/include/qemu/seqlock.h
@@ -10,63 +10,56 @@
  * See the COPYING file in the top-level directory.
  *
  */
-#ifndef QEMU_SEQLOCK_H
-#define QEMU_SEQLOCK_H 1
 
-#include <qemu/atomic.h>
-#include <qemu/thread.h>
+#ifndef QEMU_SEQLOCK_H
+#define QEMU_SEQLOCK_H
+
+#include "qemu/atomic.h"
+#include "qemu/thread.h"
 
 typedef struct QemuSeqLock QemuSeqLock;
 
 struct QemuSeqLock {
-    QemuMutex *mutex;
     unsigned sequence;
 };
 
-static inline void seqlock_init(QemuSeqLock *sl, QemuMutex *mutex)
+static inline void seqlock_init(QemuSeqLock *sl)
 {
-    sl->mutex = mutex;
     sl->sequence = 0;
 }
 
 /* Lock out other writers and update the count.  */
-static inline void seqlock_write_lock(QemuSeqLock *sl)
+static inline void seqlock_write_begin(QemuSeqLock *sl)
 {
-    if (sl->mutex) {
-        qemu_mutex_lock(sl->mutex);
-    }
     ++sl->sequence;
 
     /* Write sequence before updating other fields.  */
     smp_wmb();
 }
 
-static inline void seqlock_write_unlock(QemuSeqLock *sl)
+static inline void seqlock_write_end(QemuSeqLock *sl)
 {
     /* Write other fields before finalizing sequence.  */
     smp_wmb();
 
     ++sl->sequence;
-    if (sl->mutex) {
-        qemu_mutex_unlock(sl->mutex);
-    }
 }
 
 static inline unsigned seqlock_read_begin(QemuSeqLock *sl)
 {
     /* Always fail if a write is in progress.  */
-    unsigned ret = sl->sequence & ~1;
+    unsigned ret = atomic_read(&sl->sequence);
 
     /* Read sequence before reading other fields.  */
     smp_rmb();
-    return ret;
+    return ret & ~1;
 }
 
-static int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
+static inline int seqlock_read_retry(const QemuSeqLock *sl, unsigned start)
 {
     /* Read other fields before reading final sequence.  */
     smp_rmb();
-    return unlikely(sl->sequence != start);
+    return unlikely(atomic_read(&sl->sequence) != start);
 }
 
 #endif
diff --git a/include/qemu/shared-library.h b/include/qemu/shared-library.h
deleted file mode 100644
index 09d9929..0000000
--- a/include/qemu/shared-library.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* Copyright (C) 2015 The Android Open Source Project
-**
-** This software is licensed under the terms of the GNU General Public
-** License version 2, as published by the Free Software Foundation, and
-** may be copied, distributed, and modified under those terms.
-**
-** This program is distributed in the hope that it will be useful,
-** but WITHOUT ANY WARRANTY; without even the implied warranty of
-** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-** GNU General Public License for more details.
-*/
-#ifndef ANDROID_SHARED_LIBRARY_H
-#define ANDROID_SHARED_LIBRARY_H
-
-#include "qapi/error.h"
-
-// Opaque handle to a shared library instance.
-typedef struct SharedLibrary SharedLibrary;
-
-// Open a shared library. |name| is either its name or full path.
-// On success, return a non-NULL value. On failure, sets |*error| and
-// return NULL.
-//
-// A library can be opened several times, and will only be loaded once
-// in the system. However, there is no guarantee that the corresponding
-// SharedLibrary* value will be the same for each call.
-//
-SharedLibrary* shared_library_open(const char* name, Error **error);
-
-// Probe shared |library| for a specific dynamic |symbol| and return its
-// address in memory, or NULL if not found.
-void* shared_library_find(SharedLibrary* library, const char* symbol);
-
-// Close shared |library| instance. This will unload the library when its
-// internal reference count reaches 0.
-void shared_library_close(SharedLibrary* library);
-
-#endif  // ANDROID_SHARED_LIBRARY_H
diff --git a/include/qemu/sockets.h b/include/qemu/sockets.h
index 5c37b52..3657b73 100644
--- a/include/qemu/sockets.h
+++ b/include/qemu/sockets.h
@@ -1,37 +1,21 @@
 /* headers to use the BSD sockets */
-#ifndef QEMU_SOCKET_H
-#define QEMU_SOCKET_H
+
+#ifndef QEMU_SOCKETS_H
+#define QEMU_SOCKETS_H
 
 #ifdef _WIN32
-#include <windows.h>
-#include <winsock2.h>
-#include <ws2tcpip.h>
-
-#define socket_error() WSAGetLastError()
 
 int inet_aton(const char *cp, struct in_addr *ia);
 
-#else
-
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <netdb.h>
-#include <sys/un.h>
-
-#define socket_error() errno
-#define closesocket(s) close(s)
-
 #endif /* !_WIN32 */
 
-#include "qemu/option.h"
-#include "qapi/error.h"
-#include "qapi/qmp/qerror.h"
 #include "qapi-types.h"
 
-extern QemuOptsList socket_optslist;
+// Avoid conflicts with AndroidEmu functions of the same name.
+#define socket_connect qemu_socket_connect
+#define socket_listen qemu_socket_listen
+#define socket_set_nodelay qemu_socket_set_nodelay
+#define socket_set_cork qemu_socket_set_cork
 
 /* misc helpers */
 int qemu_socket(int domain, int type, int protocol);
@@ -41,45 +25,101 @@
 void qemu_set_block(int fd);
 void qemu_set_nonblock(int fd);
 int socket_set_fast_reuse(int fd);
-int send_all(int fd, const void *buf, int len1);
-int recv_all(int fd, void *buf, int len1, bool single_read);
+
+#ifdef WIN32
+/* Windows has different names for the same constants with the same values */
+#define SHUT_RD   0
+#define SHUT_WR   1
+#define SHUT_RDWR 2
+#endif
 
 /* callback function for nonblocking connect
  * valid fd on success, negative error code on failure
  */
-typedef void NonBlockingConnectHandler(int fd, Error *errp, void *opaque);
+typedef void NonBlockingConnectHandler(int fd, Error *err, void *opaque);
 
 InetSocketAddress *inet_parse(const char *str, Error **errp);
-int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
-int inet_listen(const char *str, char *ostr, int olen,
-                int socktype, int port_offset, Error **errp);
-int inet_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque);
 int inet_connect(const char *str, Error **errp);
-int inet_nonblocking_connect(const char *str,
-                             NonBlockingConnectHandler *callback,
-                             void *opaque, Error **errp);
 
-int inet_dgram_opts(QemuOpts *opts, Error **errp);
 NetworkAddressFamily inet_netfamily(int family);
 
-int unix_listen_opts(QemuOpts *opts, Error **errp);
 int unix_listen(const char *path, char *ostr, int olen, Error **errp);
-int unix_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque);
 int unix_connect(const char *path, Error **errp);
-int unix_nonblocking_connect(const char *str,
-                             NonBlockingConnectHandler *callback,
-                             void *opaque, Error **errp);
 
 SocketAddress *socket_parse(const char *str, Error **errp);
-int socket_connect_addr(SocketAddress *addr, Error **errp,
-                        NonBlockingConnectHandler *callback, void *opaque);
-int socket_listen_addr(SocketAddress *addr, Error **errp);
+
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque);
+int socket_listen(SocketAddress *addr, Error **errp);
+void socket_listen_cleanup(int fd, Error **errp);
 int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp);
 
 /* Old, ipv4 only bits.  Don't use for new code. */
 int parse_host_port(struct sockaddr_in *saddr, const char *str);
 int socket_init(void);
 
-#endif /* QEMU_SOCKET_H */
+/**
+ * socket_sockaddr_to_address:
+ * @sa: socket address struct
+ * @salen: size of @sa struct
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *
+socket_sockaddr_to_address(struct sockaddr_storage *sa,
+                           socklen_t salen,
+                           Error **errp);
+
+/**
+ * socket_local_address:
+ * @fd: the socket file handle
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the local socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *socket_local_address(int fd, Error **errp);
+
+/**
+ * socket_remote_address:
+ * @fd: the socket file handle
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the remote socket
+ * address. A pointer to the allocated address information
+ * struct will be returned, which the caller is required to
+ * release with a call qapi_free_SocketAddress when no
+ * longer required.
+ *
+ * Returns: the socket address struct, or NULL on error
+ */
+SocketAddress *socket_remote_address(int fd, Error **errp);
+
+/**
+ * socket_address_to_string:
+ * @addr: the socket address struct
+ * @errp: pointer to uninitialized error object
+ *
+ * Get the string representation of the socket
+ * address. A pointer to the char array containing
+ * string format will be returned, the caller is
+ * required to release the returned value when no
+ * longer required with g_free.
+ *
+ * Returns: the socket address in string format, or NULL on error
+ */
+char *socket_address_to_string(struct SocketAddress *addr, Error **errp);
+
+#endif /* QEMU_SOCKETS_H */
diff --git a/include/qemu/thread-posix.h b/include/qemu/thread-posix.h
index eb5c7a1..aa03567 100644
--- a/include/qemu/thread-posix.h
+++ b/include/qemu/thread-posix.h
@@ -1,6 +1,7 @@
-#ifndef __QEMU_THREAD_POSIX_H
-#define __QEMU_THREAD_POSIX_H 1
-#include "pthread.h"
+#ifndef QEMU_THREAD_POSIX_H
+#define QEMU_THREAD_POSIX_H
+
+#include <pthread.h>
 #include <semaphore.h>
 
 struct QemuMutex {
diff --git a/include/qemu/thread-win32.h b/include/qemu/thread-win32.h
index 3d58081..c7ce8dc 100644
--- a/include/qemu/thread-win32.h
+++ b/include/qemu/thread-win32.h
@@ -1,6 +1,7 @@
-#ifndef __QEMU_THREAD_WIN32_H
-#define __QEMU_THREAD_WIN32_H 1
-#include "windows.h"
+#ifndef QEMU_THREAD_WIN32_H
+#define QEMU_THREAD_WIN32_H
+
+#include <windows.h>
 
 struct QemuMutex {
     CRITICAL_SECTION lock;
@@ -18,6 +19,7 @@
 };
 
 struct QemuEvent {
+    int value;
     HANDLE event;
 };
 
diff --git a/include/qemu/thread.h b/include/qemu/thread.h
index f7e3b9b..1d230c4 100644
--- a/include/qemu/thread.h
+++ b/include/qemu/thread.h
@@ -1,8 +1,8 @@
-#ifndef __QEMU_THREAD_H
-#define __QEMU_THREAD_H 1
+#ifndef QEMU_THREAD_H
+#define QEMU_THREAD_H
 
-#include <inttypes.h>
-#include <stdbool.h>
+#include "qemu/processor.h"
+#include "qemu/atomic.h"
 
 typedef struct QemuMutex QemuMutex;
 typedef struct QemuCond QemuCond;
@@ -25,9 +25,6 @@
 int qemu_mutex_trylock(QemuMutex *mutex);
 void qemu_mutex_unlock(QemuMutex *mutex);
 
-#define rcu_read_lock() do { } while (0)
-#define rcu_read_unlock() do { } while (0)
-
 void qemu_cond_init(QemuCond *cond);
 void qemu_cond_destroy(QemuCond *cond);
 
@@ -61,4 +58,44 @@
 void qemu_thread_exit(void *retval);
 void qemu_thread_naming(bool enable);
 
+struct Notifier;
+void qemu_thread_atexit_add(struct Notifier *notifier);
+void qemu_thread_atexit_remove(struct Notifier *notifier);
+
+typedef void (*QemuThreadSetupFunc)(void);
+void qemu_thread_register_setup_callback(QemuThreadSetupFunc setup_func);
+
+typedef struct QemuSpin {
+    int value;
+} QemuSpin;
+
+static inline void qemu_spin_init(QemuSpin *spin)
+{
+    __sync_lock_release(&spin->value);
+}
+
+static inline void qemu_spin_lock(QemuSpin *spin)
+{
+    while (unlikely(__sync_lock_test_and_set(&spin->value, true))) {
+        while (atomic_read(&spin->value)) {
+            cpu_relax();
+        }
+    }
+}
+
+static inline bool qemu_spin_trylock(QemuSpin *spin)
+{
+    return __sync_lock_test_and_set(&spin->value, true);
+}
+
+static inline bool qemu_spin_locked(QemuSpin *spin)
+{
+    return atomic_read(&spin->value);
+}
+
+static inline void qemu_spin_unlock(QemuSpin *spin)
+{
+    __sync_lock_release(&spin->value);
+}
+
 #endif
diff --git a/include/qemu/throttle.h b/include/qemu/throttle.h
index b890613..9109657 100644
--- a/include/qemu/throttle.h
+++ b/include/qemu/throttle.h
@@ -1,10 +1,12 @@
 /*
  * QEMU throttling infrastructure
  *
- * Copyright (C) Nodalink, SARL. 2013
+ * Copyright (C) Nodalink, EURL. 2013-2014
+ * Copyright (C) Igalia, S.L. 2015-2016
  *
- * Author:
- *   Benoît Canet <benoit.canet@irqsave.net>
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -23,11 +25,10 @@
 #ifndef THROTTLE_H
 #define THROTTLE_H
 
-#include <stdint.h>
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
-#define NANOSECONDS_PER_SECOND  1000000000.0
+#define THROTTLE_VALUE_MAX 1000000000000000LL
 
 typedef enum {
     THROTTLE_BPS_TOTAL,
@@ -40,16 +41,47 @@
 } BucketType;
 
 /*
- * The max parameter of the leaky bucket throttling algorithm can be used to
- * allow the guest to do bursts.
- * The max value is a pool of I/O that the guest can use without being throttled
- * at all. Throttling is triggered once this pool is empty.
+ * This module implements I/O limits using the leaky bucket
+ * algorithm. The code is independent of the I/O units, but it is
+ * currently used for bytes per second and operations per second.
+ *
+ * Three parameters can be set by the user:
+ *
+ * - avg: the desired I/O limits in units per second.
+ * - max: the limit during bursts, also in units per second.
+ * - burst_length: the maximum length of the burst period, in seconds.
+ *
+ * Here's how it works:
+ *
+ * - The bucket level (number of performed I/O units) is kept in
+ *   bkt.level and leaks at a rate of bkt.avg units per second.
+ *
+ * - The size of the bucket is bkt.max * bkt.burst_length. Once the
+ *   bucket is full no more I/O is performed until the bucket leaks
+ *   again. This is what makes the I/O rate bkt.avg.
+ *
+ * - The bkt.avg rate does not apply until the bucket is full,
+ *   allowing the user to do bursts until then. The I/O limit during
+ *   bursts is bkt.max. To enforce this limit we keep an additional
+ *   bucket in bkt.burst_length that leaks at a rate of bkt.max units
+ *   per second.
+ *
+ * - Because of all of the above, the user can perform I/O at a
+ *   maximum of bkt.max units per second for at most bkt.burst_length
+ *   seconds in a row. After that the bucket will be full and the I/O
+ *   rate will go down to bkt.avg.
+ *
+ * - Since the bucket always leaks at a rate of bkt.avg, this also
+ *   determines how much the user needs to wait before being able to
+ *   do bursts again.
  */
 
 typedef struct LeakyBucket {
     double  avg;              /* average goal in units per second */
     double  max;              /* leaky bucket max burst in units */
     double  level;            /* bucket level in units */
+    double  burst_level;      /* bucket level in units (for computing bursts) */
+    unsigned burst_length;    /* max length of the burst period, in seconds */
 } LeakyBucket;
 
 /* The following structure is used to configure a ThrottleState
@@ -65,55 +97,59 @@
 typedef struct ThrottleState {
     ThrottleConfig cfg;       /* configuration */
     int64_t previous_leak;    /* timestamp of the last leak done */
-    QEMUTimer * timers[2];    /* timers used to do the throttling */
+} ThrottleState;
+
+typedef struct ThrottleTimers {
+    QEMUTimer *timers[2];     /* timers used to do the throttling */
     QEMUClockType clock_type; /* the clock used */
 
     /* Callbacks */
     QEMUTimerCB *read_timer_cb;
     QEMUTimerCB *write_timer_cb;
     void *timer_opaque;
-} ThrottleState;
+} ThrottleTimers;
 
 /* operations on single leaky buckets */
 void throttle_leak_bucket(LeakyBucket *bkt, int64_t delta);
 
 int64_t throttle_compute_wait(LeakyBucket *bkt);
 
-/* expose timer computation function for unit tests */
-bool throttle_compute_timer(ThrottleState *ts,
-                            bool is_write,
-                            int64_t now,
-                            int64_t *next_timestamp);
-
 /* init/destroy cycle */
-void throttle_init(ThrottleState *ts,
-                   AioContext *aio_context,
-                   QEMUClockType clock_type,
-                   void (read_timer)(void *),
-                   void (write_timer)(void *),
-                   void *timer_opaque);
+void throttle_init(ThrottleState *ts);
 
-void throttle_destroy(ThrottleState *ts);
+void throttle_timers_init(ThrottleTimers *tt,
+                          AioContext *aio_context,
+                          QEMUClockType clock_type,
+                          QEMUTimerCB *read_timer_cb,
+                          QEMUTimerCB *write_timer_cb,
+                          void *timer_opaque);
 
-void throttle_detach_aio_context(ThrottleState *ts);
+void throttle_timers_destroy(ThrottleTimers *tt);
 
-void throttle_attach_aio_context(ThrottleState *ts, AioContext *new_context);
+void throttle_timers_detach_aio_context(ThrottleTimers *tt);
 
-bool throttle_have_timer(ThrottleState *ts);
+void throttle_timers_attach_aio_context(ThrottleTimers *tt,
+                                        AioContext *new_context);
+
+bool throttle_timers_are_initialized(ThrottleTimers *tt);
 
 /* configuration */
 bool throttle_enabled(ThrottleConfig *cfg);
 
-bool throttle_conflicting(ThrottleConfig *cfg);
+bool throttle_is_valid(ThrottleConfig *cfg, Error **errp);
 
-bool throttle_is_valid(ThrottleConfig *cfg);
-
-void throttle_config(ThrottleState *ts, ThrottleConfig *cfg);
+void throttle_config(ThrottleState *ts,
+                     ThrottleTimers *tt,
+                     ThrottleConfig *cfg);
 
 void throttle_get_config(ThrottleState *ts, ThrottleConfig *cfg);
 
+void throttle_config_init(ThrottleConfig *cfg);
+
 /* usage */
-bool throttle_schedule_timer(ThrottleState *ts, bool is_write);
+bool throttle_schedule_timer(ThrottleState *ts,
+                             ThrottleTimers *tt,
+                             bool is_write);
 
 void throttle_account(ThrottleState *ts, bool is_write, uint64_t size);
 
diff --git a/include/qemu/timed-average.h b/include/qemu/timed-average.h
new file mode 100644
index 0000000..08245e7
--- /dev/null
+++ b/include/qemu/timed-average.h
@@ -0,0 +1,63 @@
+/*
+ * QEMU timed average computation
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) version 3 or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TIMED_AVERAGE_H
+#define TIMED_AVERAGE_H
+
+
+#include "qemu/timer.h"
+
+typedef struct TimedAverageWindow TimedAverageWindow;
+typedef struct TimedAverage TimedAverage;
+
+/* All fields of both structures are private */
+
+struct TimedAverageWindow {
+    uint64_t      min;             /* minimum value accounted in the window */
+    uint64_t      max;             /* maximum value accounted in the window */
+    uint64_t      sum;             /* sum of all values */
+    uint64_t      count;           /* number of values */
+    int64_t       expiration;      /* the end of the current window in ns */
+};
+
+struct TimedAverage {
+    uint64_t           period;     /* period in nanoseconds */
+    TimedAverageWindow windows[2]; /* two overlapping windows of with
+                                    * an offset of period / 2 between them */
+    unsigned           current;    /* the current window index: it's also the
+                                    * oldest window index */
+    QEMUClockType      clock_type; /* the clock used */
+};
+
+void timed_average_init(TimedAverage *ta, QEMUClockType clock_type,
+                        uint64_t period);
+
+void timed_average_account(TimedAverage *ta, uint64_t value);
+
+uint64_t timed_average_min(TimedAverage *ta);
+uint64_t timed_average_avg(TimedAverage *ta);
+uint64_t timed_average_max(TimedAverage *ta);
+uint64_t timed_average_sum(TimedAverage *ta, uint64_t *elapsed);
+
+#endif
diff --git a/include/qemu/timer.h b/include/qemu/timer.h
index 10f519b..e910103 100644
--- a/include/qemu/timer.h
+++ b/include/qemu/timer.h
@@ -1,9 +1,12 @@
 #ifndef QEMU_TIMER_H
 #define QEMU_TIMER_H
 
-#include "qemu/typedefs.h"
 #include "qemu-common.h"
 #include "qemu/notify.h"
+#include "qemu/host-utils.h"
+#include "sysemu/cpus.h"
+
+#define NANOSECONDS_PER_SECOND 1000000000LL
 
 /* timers */
 
@@ -36,12 +39,20 @@
  * is suspended, and it will reflect system time changes the host may
  * undergo (e.g. due to NTP). The host clock has the same precision as
  * the virtual clock.
+ *
+ * @QEMU_CLOCK_VIRTUAL_RT: realtime clock used for icount warp
+ *
+ * Outside icount mode, this clock is the same as @QEMU_CLOCK_VIRTUAL.
+ * In icount mode, this clock counts nanoseconds while the virtual
+ * machine is running.  It is used to increase @QEMU_CLOCK_VIRTUAL
+ * while the CPUs are sleeping and thus not executing instructions.
  */
 
 typedef enum {
     QEMU_CLOCK_REALTIME = 0,
     QEMU_CLOCK_VIRTUAL = 1,
     QEMU_CLOCK_HOST = 2,
+    QEMU_CLOCK_VIRTUAL_RT = 3,
     QEMU_CLOCK_MAX
 } QEMUClockType;
 
@@ -199,12 +210,11 @@
 void qemu_clock_enable(QEMUClockType type, bool enabled);
 
 /**
- * qemu_clock_warp:
- * @type: the clock type
+ * qemu_start_warp_timer:
  *
- * Warp a clock to a new value
+ * Starts a timer for virtual clock update
  */
-void qemu_clock_warp(QEMUClockType type);
+void qemu_start_warp_timer(void);
 
 /**
  * qemu_clock_register_reset_notifier:
@@ -402,7 +412,7 @@
  */
 
 /**
- * timer_init:
+ * timer_init_tl:
  * @ts: the timer to be initialised
  * @timer_list: the timer list to attach the timer to
  * @scale: the scale value for the timer
@@ -415,9 +425,82 @@
  * You need not call an explicit deinit call. Simply make
  * sure it is not on a list with timer_del.
  */
-void timer_init(QEMUTimer *ts,
-                QEMUTimerList *timer_list, int scale,
-                QEMUTimerCB *cb, void *opaque);
+void timer_init_tl(QEMUTimer *ts,
+                   QEMUTimerList *timer_list, int scale,
+                   QEMUTimerCB *cb, void *opaque);
+
+/**
+ * timer_init:
+ * @type: the clock to associate with the timer
+ * @scale: the scale value for the timer
+ * @cb: the callback to call when the timer expires
+ * @opaque: the opaque pointer to pass to the callback
+ *
+ * Initialize a timer with the given scale on the default timer list
+ * associated with the clock.
+ *
+ * You need not call an explicit deinit call. Simply make
+ * sure it is not on a list with timer_del.
+ */
+static inline void timer_init(QEMUTimer *ts, QEMUClockType type, int scale,
+                              QEMUTimerCB *cb, void *opaque)
+{
+    timer_init_tl(ts, main_loop_tlg.tl[type], scale, cb, opaque);
+}
+
+/**
+ * timer_init_ns:
+ * @type: the clock to associate with the timer
+ * @cb: the callback to call when the timer expires
+ * @opaque: the opaque pointer to pass to the callback
+ *
+ * Initialize a timer with nanosecond scale on the default timer list
+ * associated with the clock.
+ *
+ * You need not call an explicit deinit call. Simply make
+ * sure it is not on a list with timer_del.
+ */
+static inline void timer_init_ns(QEMUTimer *ts, QEMUClockType type,
+                                 QEMUTimerCB *cb, void *opaque)
+{
+    timer_init(ts, type, SCALE_NS, cb, opaque);
+}
+
+/**
+ * timer_init_us:
+ * @type: the clock to associate with the timer
+ * @cb: the callback to call when the timer expires
+ * @opaque: the opaque pointer to pass to the callback
+ *
+ * Initialize a timer with microsecond scale on the default timer list
+ * associated with the clock.
+ *
+ * You need not call an explicit deinit call. Simply make
+ * sure it is not on a list with timer_del.
+ */
+static inline void timer_init_us(QEMUTimer *ts, QEMUClockType type,
+                                 QEMUTimerCB *cb, void *opaque)
+{
+    timer_init(ts, type, SCALE_US, cb, opaque);
+}
+
+/**
+ * timer_init_ms:
+ * @type: the clock to associate with the timer
+ * @cb: the callback to call when the timer expires
+ * @opaque: the opaque pointer to pass to the callback
+ *
+ * Initialize a timer with millisecond scale on the default timer list
+ * associated with the clock.
+ *
+ * You need not call an explicit deinit call. Simply make
+ * sure it is not on a list with timer_del.
+ */
+static inline void timer_init_ms(QEMUTimer *ts, QEMUClockType type,
+                                 QEMUTimerCB *cb, void *opaque)
+{
+    timer_init(ts, type, SCALE_MS, cb, opaque);
+}
 
 /**
  * timer_new_tl:
@@ -439,8 +522,8 @@
                                       QEMUTimerCB *cb,
                                       void *opaque)
 {
-    QEMUTimer *ts = (QEMUTimer*)g_malloc0(sizeof(QEMUTimer));
-    timer_init(ts, timer_list, scale, cb, opaque);
+    QEMUTimer *ts = (QEMUTimer *) g_malloc0(sizeof(QEMUTimer));
+    timer_init_tl(ts, timer_list, scale, cb, opaque);
     return ts;
 }
 
@@ -514,6 +597,17 @@
 }
 
 /**
+ * timer_deinit:
+ * @ts: the timer to be de-initialised
+ *
+ * Deassociate the timer from any timerlist.  You should
+ * call timer_del before.  After this call, any further
+ * timer_del call cannot cause dangling pointer accesses
+ * even if the previously used timerlist is freed.
+ */
+void timer_deinit(QEMUTimer *ts);
+
+/**
  * timer_free:
  * @ts: the timer
  *
@@ -690,9 +784,13 @@
 /* Caller must hold BQL */
 void cpu_disable_ticks(void);
 
-static inline int64_t get_ticks_per_sec(void)
+static inline int64_t get_max_clock_jump(void)
 {
-    return 1000000000LL;
+    /* This should be small enough to prevent excessive interrupts from being
+     * generated by the RTC on clock jumps, but large enough to avoid frequent
+     * unnecessary resets in idle VMs.
+     */
+    return 60 * NANOSECONDS_PER_SECOND;
 }
 
 /*
@@ -718,7 +816,7 @@
 {
     LARGE_INTEGER ti;
     QueryPerformanceCounter(&ti);
-    return muldiv64(ti.QuadPart, get_ticks_per_sec(), clock_freq);
+    return muldiv64(ti.QuadPart, NANOSECONDS_PER_SECOND, clock_freq);
 }
 
 #else
@@ -743,9 +841,9 @@
 #endif
 
 /* icount */
+int64_t cpu_get_icount_raw(void);
 int64_t cpu_get_icount(void);
 int64_t cpu_get_clock(void);
-int64_t cpu_get_clock_offset(void);
 int64_t cpu_icount_to_ns(int64_t icount);
 
 /*******************************************/
@@ -753,7 +851,7 @@
 
 #if defined(_ARCH_PPC)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t retval;
 #ifdef _ARCH_PPC64
@@ -779,7 +877,7 @@
 
 #elif defined(__i386__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile ("rdtsc" : "=A" (val));
@@ -788,7 +886,7 @@
 
 #elif defined(__x86_64__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     uint32_t low,high;
     int64_t val;
@@ -801,7 +899,7 @@
 
 #elif defined(__hppa__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int val;
     asm volatile ("mfctl %%cr16, %0" : "=r"(val));
@@ -810,7 +908,7 @@
 
 #elif defined(__ia64)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile ("mov %0 = ar.itc" : "=r"(val) :: "memory");
@@ -819,7 +917,7 @@
 
 #elif defined(__s390__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     int64_t val;
     asm volatile("stck 0(%1)" : "=m" (val) : "a" (&val) : "cc");
@@ -828,7 +926,7 @@
 
 #elif defined(__sparc__)
 
-static inline int64_t cpu_get_real_ticks (void)
+static inline int64_t cpu_get_host_ticks (void)
 {
 #if defined(_LP64)
     uint64_t        rval;
@@ -866,7 +964,7 @@
                               : "=r" (value));          \
     }
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     /* On kernels >= 2.6.25 rdhwr <reg>, $2 and $3 are emulated */
     uint32_t count;
@@ -882,7 +980,7 @@
 
 #elif defined(__alpha__)
 
-static inline int64_t cpu_get_real_ticks(void)
+static inline int64_t cpu_get_host_ticks(void)
 {
     uint64_t cc;
     uint32_t cur, ofs;
@@ -897,7 +995,7 @@
 /* The host CPU doesn't have an easily accessible cycle counter.
    Just return a monotonically increasing value.  This will be
    totally wrong, but hopefully better than nothing.  */
-static inline int64_t cpu_get_real_ticks (void)
+static inline int64_t cpu_get_host_ticks (void)
 {
     static int64_t ticks = 0;
     return ticks++;
@@ -907,11 +1005,10 @@
 #ifdef CONFIG_PROFILER
 static inline int64_t profile_getclock(void)
 {
-    return cpu_get_real_ticks();
+    return get_clock();
 }
 
-extern int64_t qemu_time, qemu_time_start;
-extern int64_t tlb_flush_time;
+extern int64_t tcg_time;
 extern int64_t dev_time;
 #endif
 
diff --git a/include/qemu/tls.h b/include/qemu/tls.h
deleted file mode 100644
index b92ea9d..0000000
--- a/include/qemu/tls.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Abstraction layer for defining and using TLS variables
- *
- * Copyright (c) 2011 Red Hat, Inc
- * Copyright (c) 2011 Linaro Limited
- *
- * Authors:
- *  Paolo Bonzini <pbonzini@redhat.com>
- *  Peter Maydell <peter.maydell@linaro.org>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License as
- * published by the Free Software Foundation; either version 2 of
- * the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef QEMU_TLS_H
-#define QEMU_TLS_H
-
-/* Per-thread variables. Note that we only have implementations
- * which are really thread-local on Linux; the dummy implementations
- * define plain global variables.
- *
- * This means that for the moment use should be restricted to
- * per-VCPU variables, which are OK because:
- *  - the only -user mode supporting multiple VCPU threads is linux-user
- *  - TCG system mode is single-threaded regarding VCPUs
- *  - KVM system mode is multi-threaded but limited to Linux
- *
- * TODO: proper implementations via Win32 .tls sections and
- * POSIX pthread_getspecific.
- */
-#ifdef __linux__
-#define DECLARE_TLS(type, x) extern DEFINE_TLS(type, x)
-#define DEFINE_TLS(type, x)  __thread __typeof__(type) tls__##x
-#define tls_var(x)           tls__##x
-#else
-/* Dummy implementations which define plain global variables */
-#define DECLARE_TLS(type, x) extern DEFINE_TLS(type, x)
-#define DEFINE_TLS(type, x)  __typeof__(type) tls__##x
-#define tls_var(x)           tls__##x
-#endif
-
-#endif
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
index d619fb6..b113fcf 100644
--- a/include/qemu/typedefs.h
+++ b/include/qemu/typedefs.h
@@ -3,84 +3,96 @@
 
 /* A load of opaque types so that device init declarations don't have to
    pull in all the real definitions.  */
-typedef struct QEMUTimer QEMUTimer;
-typedef struct QEMUTimerListGroup QEMUTimerListGroup;
-typedef struct QEMUFile QEMUFile;
-typedef struct QEMUBH QEMUBH;
 
-typedef struct AioContext AioContext;
-
-typedef struct Visitor Visitor;
-
-struct Monitor;
-typedef struct Monitor Monitor;
-typedef struct MigrationParams MigrationParams;
-
-typedef struct Property Property;
-typedef struct PropertyInfo PropertyInfo;
-typedef struct CompatProperty CompatProperty;
-typedef struct DeviceState DeviceState;
-typedef struct BusState BusState;
-typedef struct BusClass BusClass;
-
+/* Please keep this list in alphabetical order */
+typedef struct AdapterInfo AdapterInfo;
 typedef struct AddressSpace AddressSpace;
-typedef struct MemoryRegion MemoryRegion;
-typedef struct MemoryRegionSection MemoryRegionSection;
-typedef struct MemoryListener MemoryListener;
-
-typedef struct MemoryMappingList MemoryMappingList;
-
-typedef struct QEMUMachine QEMUMachine;
-typedef struct MachineClass MachineClass;
-typedef struct MachineState MachineState;
-typedef struct NICInfo NICInfo;
-typedef struct HCIInfo HCIInfo;
+typedef struct AioContext AioContext;
+typedef struct AllwinnerAHCIState AllwinnerAHCIState;
 typedef struct AudioState AudioState;
+typedef struct BdrvDirtyBitmap BdrvDirtyBitmap;
 typedef struct BlockBackend BlockBackend;
+typedef struct BlockBackendRootState BlockBackendRootState;
 typedef struct BlockDriverState BlockDriverState;
-typedef struct DriveInfo DriveInfo;
-typedef struct DisplayState DisplayState;
-typedef struct DisplayChangeListener DisplayChangeListener;
-typedef struct DisplayUpdateListener DisplayUpdateListener;
-typedef struct DisplaySurface DisplaySurface;
-typedef struct PixelFormat PixelFormat;
-typedef struct QemuConsole QemuConsole;
+typedef struct BusClass BusClass;
+typedef struct BusState BusState;
 typedef struct CharDriverState CharDriverState;
-typedef struct MACAddr MACAddr;
-typedef struct NetClientState NetClientState;
+typedef struct CompatProperty CompatProperty;
+typedef struct CPUAddressSpace CPUAddressSpace;
+typedef struct CPUState CPUState;
+typedef struct DeviceListener DeviceListener;
+typedef struct DeviceState DeviceState;
+typedef struct DisplayChangeListener DisplayChangeListener;
+typedef struct DisplayState DisplayState;
+typedef struct DisplaySurface DisplaySurface;
+typedef struct DriveInfo DriveInfo;
+typedef struct Error Error;
+typedef struct EventNotifier EventNotifier;
+typedef struct FWCfgIoState FWCfgIoState;
+typedef struct FWCfgMemState FWCfgMemState;
+typedef struct FWCfgState FWCfgState;
+typedef struct HCIInfo HCIInfo;
 typedef struct I2CBus I2CBus;
+typedef struct I2SCodec I2SCodec;
 typedef struct ISABus ISABus;
 typedef struct ISADevice ISADevice;
-typedef struct SMBusDevice SMBusDevice;
-typedef struct PCIHostState PCIHostState;
-typedef struct PCIExpressHost PCIExpressHost;
+typedef struct IsaDma IsaDma;
+typedef struct LoadStateEntry LoadStateEntry;
+typedef struct MACAddr MACAddr;
+typedef struct MachineClass MachineClass;
+typedef struct MachineState MachineState;
+typedef struct MemoryListener MemoryListener;
+typedef struct MemoryMappingList MemoryMappingList;
+typedef struct MemoryRegion MemoryRegion;
+typedef struct MemoryRegionSection MemoryRegionSection;
+typedef struct MigrationIncomingState MigrationIncomingState;
+typedef struct MigrationParams MigrationParams;
+typedef struct MigrationState MigrationState;
+typedef struct Monitor Monitor;
+typedef struct MonitorDef MonitorDef;
+typedef struct MouseTransformInfo MouseTransformInfo;
+typedef struct MSIMessage MSIMessage;
+typedef struct NetClientState NetClientState;
+typedef struct NetFilterState NetFilterState;
+typedef struct NICInfo NICInfo;
+typedef struct PcGuestInfo PcGuestInfo;
+typedef struct PCIBridge PCIBridge;
 typedef struct PCIBus PCIBus;
 typedef struct PCIDevice PCIDevice;
-typedef struct PCIExpressDevice PCIExpressDevice;
-typedef struct PCIBridge PCIBridge;
-typedef struct PCIEAERMsg PCIEAERMsg;
-typedef struct PCIEAERLog PCIEAERLog;
 typedef struct PCIEAERErr PCIEAERErr;
+typedef struct PCIEAERLog PCIEAERLog;
+typedef struct PCIEAERMsg PCIEAERMsg;
 typedef struct PCIEPort PCIEPort;
 typedef struct PCIESlot PCIESlot;
-typedef struct MSIMessage MSIMessage;
-typedef struct SerialState SerialState;
+typedef struct PCIExpressDevice PCIExpressDevice;
+typedef struct PCIExpressHost PCIExpressHost;
+typedef struct PCIHostDeviceAddress PCIHostDeviceAddress;
+typedef struct PCIHostState PCIHostState;
+typedef struct PCMachineClass PCMachineClass;
+typedef struct PCMachineState PCMachineState;
 typedef struct PCMCIACardState PCMCIACardState;
-typedef struct MouseTransformInfo MouseTransformInfo;
-typedef struct uWireSlave uWireSlave;
-typedef struct I2SCodec I2SCodec;
-typedef struct SSIBus SSIBus;
-typedef struct EventNotifier EventNotifier;
-typedef struct VirtIODevice VirtIODevice;
+typedef struct PixelFormat PixelFormat;
+typedef struct PostcopyDiscardState PostcopyDiscardState;
+typedef struct Property Property;
+typedef struct PropertyInfo PropertyInfo;
+typedef struct QEMUBH QEMUBH;
+typedef struct QemuConsole QemuConsole;
+typedef struct QEMUFile QEMUFile;
+typedef struct QemuOpt QemuOpt;
+typedef struct QemuOpts QemuOpts;
+typedef struct QemuOptsList QemuOptsList;
 typedef struct QEMUSGList QEMUSGList;
-typedef struct QEMUSizedBuffer QEMUSizedBuffer;
-typedef struct SHPCDevice SHPCDevice;
-typedef struct FWCfgState FWCfgState;
-typedef struct PcGuestInfo PcGuestInfo;
+typedef struct QEMUTimer QEMUTimer;
+typedef struct QEMUTimerListGroup QEMUTimerListGroup;
+typedef struct QObject QObject;
+typedef struct RAMBlock RAMBlock;
 typedef struct Range Range;
-typedef struct AdapterInfo AdapterInfo;
-
-struct Slirp;
-typedef struct Slirp Slirp;
+typedef struct SerialState SerialState;
+typedef struct SHPCDevice SHPCDevice;
+typedef struct SMBusDevice SMBusDevice;
+typedef struct SSIBus SSIBus;
+typedef struct uWireSlave uWireSlave;
+typedef struct VirtIODevice VirtIODevice;
+typedef struct Visitor Visitor;
 
 #endif /* QEMU_TYPEDEFS_H */
diff --git a/include/qemu/unicode.h b/include/qemu/unicode.h
new file mode 100644
index 0000000..71c72db
--- /dev/null
+++ b/include/qemu/unicode.h
@@ -0,0 +1,6 @@
+#ifndef QEMU_UNICODE_H
+#define QEMU_UNICODE_H
+
+int mod_utf8_codepoint(const char *s, size_t n, char **end);
+
+#endif
diff --git a/include/qemu/xattr.h b/include/qemu/xattr.h
index f910d96..83cf98c 100644
--- a/include/qemu/xattr.h
+++ b/include/qemu/xattr.h
@@ -18,7 +18,6 @@
  * in /usr/include/sys, and don't have ENOATTR.
  */
 
-#include "config-host.h"
 
 #ifdef CONFIG_LIBATTR
 #  include <attr/xattr.h>
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index 859d4c9..7c1199c 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -20,14 +20,14 @@
 #ifndef QEMU_CPU_H
 #define QEMU_CPU_H
 
-#include <signal.h>
-#include <setjmp.h>
 #include "hw/qdev-core.h"
+#include "disas/bfd.h"
 #include "exec/hwaddr.h"
+#include "exec/memattrs.h"
+#include "qemu/bitmap.h"
 #include "qemu/queue.h"
 #include "qemu/thread.h"
-#include "qemu/tls.h"
-#include "qemu/typedefs.h"
+#include "trace/generated-events.h"
 
 typedef int (*WriteCoreDumpFunction)(const void *buf, size_t size,
                                      void *opaque);
@@ -62,7 +62,13 @@
 #define CPU_CLASS(class) OBJECT_CLASS_CHECK(CPUClass, (class), TYPE_CPU)
 #define CPU_GET_CLASS(obj) OBJECT_GET_CLASS(CPUClass, (obj), TYPE_CPU)
 
-typedef struct CPUState CPUState;
+typedef enum MMUAccessType {
+    MMU_DATA_LOAD  = 0,
+    MMU_DATA_STORE = 1,
+    MMU_INST_FETCH = 2
+} MMUAccessType;
+
+typedef struct CPUWatchpoint CPUWatchpoint;
 
 typedef void (*CPUUnassignedAccess)(CPUState *cpu, hwaddr addr,
                                     bool is_write, bool is_exec, int opaque,
@@ -82,6 +88,10 @@
  * @do_unassigned_access: Callback for unassigned access handling.
  * @do_unaligned_access: Callback for unaligned access handling, if
  * the target defines #ALIGNED_ONLY.
+ * @virtio_is_big_endian: Callback to return %true if a CPU which supports
+ * runtime configurable endianness is currently big-endian. Non-configurable
+ * CPUs can use the default implementation of this method. This method should
+ * not be used by any callers other than the pre-1.0 virtio devices.
  * @memory_rw_debug: Callback for GDB memory access.
  * @dump_state: Callback for dumping state.
  * @dump_statistics: Callback for dumping statistics.
@@ -93,9 +103,25 @@
  * #TranslationBlock.
  * @handle_mmu_fault: Callback for handling an MMU fault.
  * @get_phys_page_debug: Callback for obtaining a physical address.
+ * @get_phys_page_attrs_debug: Callback for obtaining a physical address and the
+ *       associated memory transaction attributes to use for the access.
+ *       CPUs which use memory transaction attributes should implement this
+ *       instead of get_phys_page_debug.
+ * @asidx_from_attrs: Callback to return the CPU AddressSpace to use for
+ *       a memory access with the specified memory transaction attributes.
  * @gdb_read_register: Callback for letting GDB read a register.
  * @gdb_write_register: Callback for letting GDB write a register.
+ * @debug_check_watchpoint: Callback: return true if the architectural
+ *       watchpoint whose address has matched should really fire.
  * @debug_excp_handler: Callback for handling debug exceptions.
+ * @write_elf64_note: Callback for writing a CPU-specific ELF note to a
+ * 64-bit VM coredump.
+ * @write_elf32_qemunote: Callback for writing a CPU- and QEMU-specific ELF
+ * note to a 32-bit VM coredump.
+ * @write_elf32_note: Callback for writing a CPU-specific ELF note to a
+ * 32-bit VM coredump.
+ * @write_elf32_qemunote: Callback for writing a CPU- and QEMU-specific ELF
+ * note to a 32-bit VM coredump.
  * @vmsd: State description for migration.
  * @gdb_num_core_regs: Number of core registers accessible to GDB.
  * @gdb_core_xml_file: File name for core registers GDB XML description.
@@ -106,6 +132,7 @@
  * @cpu_exec_enter: Callback for cpu_exec preparation.
  * @cpu_exec_exit: Callback for cpu_exec cleanup.
  * @cpu_exec_interrupt: Callback for processing interrupts in cpu_exec.
+ * @disas_set_info: Setup architecture specific components of disassembly info
  *
  * Represents a CPU family or model.
  */
@@ -115,7 +142,7 @@
     /*< public >*/
 
     ObjectClass *(*class_by_name)(const char *cpu_model);
-    void (*parse_features)(CPUState *cpu, char *str, Error **errp);
+    void (*parse_features)(const char *typename, char *str, Error **errp);
 
     void (*reset)(CPUState *cpu);
     int reset_dump_flags;
@@ -123,7 +150,9 @@
     void (*do_interrupt)(CPUState *cpu);
     CPUUnassignedAccess do_unassigned_access;
     void (*do_unaligned_access)(CPUState *cpu, vaddr addr,
-                                int is_write, int is_user, uintptr_t retaddr);
+                                MMUAccessType access_type,
+                                int mmu_idx, uintptr_t retaddr,
+                                unsigned size);
     bool (*virtio_is_big_endian)(CPUState *cpu);
     int (*memory_rw_debug)(CPUState *cpu, vaddr addr,
                            uint8_t *buf, int len, bool is_write);
@@ -140,8 +169,12 @@
     int (*handle_mmu_fault)(CPUState *cpu, vaddr address, int rw,
                             int mmu_index);
     hwaddr (*get_phys_page_debug)(CPUState *cpu, vaddr addr);
+    hwaddr (*get_phys_page_attrs_debug)(CPUState *cpu, vaddr addr,
+                                        MemTxAttrs *attrs);
+    int (*asidx_from_attrs)(CPUState *cpu, MemTxAttrs attrs);
     int (*gdb_read_register)(CPUState *cpu, uint8_t *buf, int reg);
     int (*gdb_write_register)(CPUState *cpu, uint8_t *buf, int reg);
+    bool (*debug_check_watchpoint)(CPUState *cpu, CPUWatchpoint *wp);
     void (*debug_excp_handler)(CPUState *cpu);
 
     int (*write_elf64_note)(WriteCoreDumpFunction f, CPUState *cpu,
@@ -162,6 +195,8 @@
     void (*cpu_exec_enter)(CPUState *cpu);
     void (*cpu_exec_exit)(CPUState *cpu);
     bool (*cpu_exec_interrupt)(CPUState *cpu, int interrupt_request);
+
+    void (*disas_set_info)(CPUState *cpu, disassemble_info *info);
 } CPUClass;
 
 #ifdef HOST_WORDS_BIGENDIAN
@@ -182,13 +217,14 @@
     QTAILQ_ENTRY(CPUBreakpoint) entry;
 } CPUBreakpoint;
 
-typedef struct CPUWatchpoint {
+struct CPUWatchpoint {
     vaddr vaddr;
     vaddr len;
     vaddr hitaddr;
+    MemTxAttrs hitattrs;
     int flags; /* BP_* */
     QTAILQ_ENTRY(CPUWatchpoint) entry;
-} CPUWatchpoint;
+};
 
 struct KVMState;
 struct kvm_run;
@@ -200,6 +236,15 @@
 #define TB_JMP_CACHE_BITS 12
 #define TB_JMP_CACHE_SIZE (1 << TB_JMP_CACHE_BITS)
 
+/* work queue */
+struct qemu_work_item {
+    struct qemu_work_item *next;
+    void (*func)(void *data);
+    void *data;
+    int done;
+    bool free;
+};
+
 /**
  * CPUState:
  * @cpu_index: CPU index (informative).
@@ -213,16 +258,25 @@
  * @halted: Nonzero if the CPU is in suspended state.
  * @stop: Indicates a pending stop request.
  * @stopped: Indicates the CPU has been artificially stopped.
+ * @unplug: Indicates a pending CPU unplug request.
+ * @crash_occurred: Indicates the OS reported a crash (panic) for this CPU
  * @tcg_exit_req: Set to force TCG to stop executing linked TBs for this
  *           CPU and return to its top level loop.
+ * @tb_flushed: Indicates the translation buffer has been flushed.
  * @singlestep_enabled: Flags for single-stepping.
  * @icount_extra: Instructions until next timer event.
  * @icount_decr: Number of cycles left, with interrupt flag in high bit.
  * This allows a single read-compare-cbranch-write sequence to test
  * for both decrementer underflow and exceptions.
- * @can_do_io: Nonzero if memory-mapped IO is safe.
+ * @can_do_io: Nonzero if memory-mapped IO is safe. Deterministic execution
+ * requires that IO only be performed on the last instruction of a TB
+ * so that interrupts take effect immediately.
+ * @cpu_ases: Pointer to array of CPUAddressSpaces (which define the
+ *            AddressSpaces this CPU has)
+ * @num_ases: number of CPUAddressSpaces in @cpu_ases
+ * @as: Pointer to the first AddressSpace, for the convenience of targets which
+ *      only have a single AddressSpace
  * @env_ptr: Pointer to subclass-specific CPUArchState field.
- * @current_tb: Currently executing TB.
  * @gdb_regs: Additional GDB registers.
  * @gdb_num_regs: Number of total registers accessible to GDB.
  * @gdb_num_g_regs: Number of registers in GDB 'g' packets.
@@ -231,6 +285,9 @@
  * @mem_io_pc: Host Program Counter at which the memory was accessed.
  * @mem_io_vaddr: Target virtual address at which the memory was accessed.
  * @kvm_fd: vCPU file descriptor for KVM.
+ * @work_mutex: Lock to prevent multiple access to queued_work_*.
+ * @queued_work_first: First asynchronous work pending.
+ * @trace_dstate: Dynamic tracing state of events for this vCPU (bitmask).
  *
  * State of one CPU core or thread.
  */
@@ -251,22 +308,28 @@
     uint32_t host_tid;
     bool running;
     struct QemuCond *halt_cond;
-    struct qemu_work_item *queued_work_first, *queued_work_last;
     bool thread_kicked;
     bool created;
     bool stop;
     bool stopped;
-    volatile sig_atomic_t exit_request;
+    bool unplug;
+    bool crash_occurred;
+    bool exit_request;
+    bool tb_flushed;
     uint32_t interrupt_request;
     int singlestep_enabled;
     int64_t icount_extra;
     sigjmp_buf jmp_env;
 
+    QemuMutex work_mutex;
+    struct qemu_work_item *queued_work_first, *queued_work_last;
+
+    CPUAddressSpace *cpu_ases;
+    int num_ases;
     AddressSpace *as;
-    MemoryListener *tcg_as_listener;
+    MemoryRegion *memory;
 
     void *env_ptr; /* CPUArchState */
-    struct TranslationBlock *current_tb;
     struct TranslationBlock *tb_jmp_cache[TB_JMP_CACHE_SIZE];
     struct GDBRegisterState *gdb_regs;
     int gdb_num_regs;
@@ -292,6 +355,9 @@
     struct KVMState *kvm_state;
     struct kvm_run *kvm_run;
 
+    /* Used for events with 'vcpu' and *without* the 'disabled' properties */
+    DECLARE_BITMAP(trace_dstate, TRACE_VCPU_EVENT_COUNT);
+
     /* TODO Move common fields from CPUArchState here. */
     int cpu_index; /* used by alpha TCG */
     uint32_t halted; /* used by alpha, cris, ppc TCG */
@@ -302,12 +368,17 @@
     uint32_t can_do_io;
     int32_t exception_index; /* used by m68k TCG */
 
+    /* Used to keep track of an outstanding cpu throttle thread for migration
+     * autoconverge
+     */
+    bool throttle_thread_scheduled;
+
     /* Note that this is accessed at the start of every TB via a negative
        offset from AREG0.  Leave this field at the end so as to make the
        (absolute value) offset as small as possible.  This reduces code
        size, especially for hosts without large memory offsets.  */
-    volatile sig_atomic_t tcg_exit_req;
-
+    uint32_t tcg_exit_req;
+    
 #ifdef CONFIG_HAX
     bool hax_vcpu_dirty;
     struct hax_vcpu_state *hax_vcpu;
@@ -320,10 +391,11 @@
 #define CPU_FOREACH(cpu) QTAILQ_FOREACH(cpu, &cpus, node)
 #define CPU_FOREACH_SAFE(cpu, next_cpu) \
     QTAILQ_FOREACH_SAFE(cpu, &cpus, node, next_cpu)
+#define CPU_FOREACH_REVERSE(cpu) \
+    QTAILQ_FOREACH_REVERSE(cpu, &cpus, CPUTailQ, node)
 #define first_cpu QTAILQ_FIRST(&cpus)
 
-DECLARE_TLS(CPUState *, current_cpu);
-#define current_cpu tls_var(current_cpu)
+extern __thread CPUState *current_cpu;
 
 /**
  * cpu_paging_enabled:
@@ -420,6 +492,32 @@
 
 #ifndef CONFIG_USER_ONLY
 /**
+ * cpu_get_phys_page_attrs_debug:
+ * @cpu: The CPU to obtain the physical page address for.
+ * @addr: The virtual address.
+ * @attrs: Updated on return with the memory transaction attributes to use
+ *         for this access.
+ *
+ * Obtains the physical page corresponding to a virtual one, together
+ * with the corresponding memory transaction attributes to use for the access.
+ * Use it only for debugging because no protection checks are done.
+ *
+ * Returns: Corresponding physical page address or -1 if no page found.
+ */
+static inline hwaddr cpu_get_phys_page_attrs_debug(CPUState *cpu, vaddr addr,
+                                                   MemTxAttrs *attrs)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    if (cc->get_phys_page_attrs_debug) {
+        return cc->get_phys_page_attrs_debug(cpu, addr, attrs);
+    }
+    /* Fallback for CPUs which don't implement the _attrs_ hook */
+    *attrs = MEMTXATTRS_UNSPECIFIED;
+    return cc->get_phys_page_debug(cpu, addr);
+}
+
+/**
  * cpu_get_phys_page_debug:
  * @cpu: The CPU to obtain the physical page address for.
  * @addr: The virtual address.
@@ -431,9 +529,26 @@
  */
 static inline hwaddr cpu_get_phys_page_debug(CPUState *cpu, vaddr addr)
 {
+    MemTxAttrs attrs = {};
+
+    return cpu_get_phys_page_attrs_debug(cpu, addr, &attrs);
+}
+
+/** cpu_asidx_from_attrs:
+ * @cpu: CPU
+ * @attrs: memory transaction attributes
+ *
+ * Returns the address space index specifying the CPU AddressSpace
+ * to use for a memory access with the given transaction attributes.
+ */
+static inline int cpu_asidx_from_attrs(CPUState *cpu, MemTxAttrs attrs)
+{
     CPUClass *cc = CPU_GET_CLASS(cpu);
 
-    return cc->get_phys_page_debug(cpu, addr);
+    if (cc->asidx_from_attrs) {
+        return cc->asidx_from_attrs(cpu, attrs);
+    }
+    return 0;
 }
 #endif
 
@@ -550,6 +665,43 @@
  */
 bool cpu_exists(int64_t id);
 
+/**
+ * cpu_throttle_set:
+ * @new_throttle_pct: Percent of sleep time. Valid range is 1 to 99.
+ *
+ * Throttles all vcpus by forcing them to sleep for the given percentage of
+ * time. A throttle_percentage of 25 corresponds to a 75% duty cycle roughly.
+ * (example: 10ms sleep for every 30ms awake).
+ *
+ * cpu_throttle_set can be called as needed to adjust new_throttle_pct.
+ * Once the throttling starts, it will remain in effect until cpu_throttle_stop
+ * is called.
+ */
+void cpu_throttle_set(int new_throttle_pct);
+
+/**
+ * cpu_throttle_stop:
+ *
+ * Stops the vcpu throttling started by cpu_throttle_set.
+ */
+void cpu_throttle_stop(void);
+
+/**
+ * cpu_throttle_active:
+ *
+ * Returns: %true if the vcpus are currently being throttled, %false otherwise.
+ */
+bool cpu_throttle_active(void);
+
+/**
+ * cpu_throttle_get_percentage:
+ *
+ * Returns the vcpu throttle percentage. See cpu_throttle_set for details.
+ *
+ * Returns: The throttle percentage in range 1 to 99.
+ */
+int cpu_throttle_get_percentage(void);
+
 #ifndef CONFIG_USER_ONLY
 
 typedef void (*CPUInterruptHandler)(CPUState *, int);
@@ -587,16 +739,31 @@
 }
 
 static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
-                                        int is_write, int is_user,
-                                        uintptr_t retaddr)
+                                        MMUAccessType access_type,
+                                        int mmu_idx, uintptr_t retaddr,
+                                        unsigned size)
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
 
-    return cc->do_unaligned_access(cpu, addr, is_write, is_user, retaddr);
+    cc->do_unaligned_access(cpu, addr, access_type, mmu_idx, retaddr, size);
 }
 #endif
 
 /**
+ * cpu_set_pc:
+ * @cpu: The CPU to set the program counter for.
+ * @addr: Program counter value.
+ *
+ * Sets the program counter for a CPU.
+ */
+static inline void cpu_set_pc(CPUState *cpu, vaddr addr)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    cc->set_pc(cpu, addr);
+}
+
+/**
  * cpu_reset_interrupt:
  * @cpu: The CPU to clear the interrupt on.
  * @mask: The interrupt mask to clear.
@@ -622,6 +789,22 @@
 void cpu_resume(CPUState *cpu);
 
 /**
+ * cpu_remove:
+ * @cpu: The CPU to remove.
+ *
+ * Requests the CPU to be removed.
+ */
+void cpu_remove(CPUState *cpu);
+
+ /**
+ * cpu_remove_sync:
+ * @cpu: The CPU to remove.
+ *
+ * Requests the CPU to be removed and waits till it is removed.
+ */
+void cpu_remove_sync(CPUState *cpu);
+
+/**
  * qemu_init_vcpu:
  * @cpu: The vCPU to initialize.
  *
@@ -650,6 +833,7 @@
 /* 0x08 currently unused */
 #define BP_GDB                0x10
 #define BP_CPU                0x20
+#define BP_ANY                (BP_GDB | BP_CPU)
 #define BP_WATCHPOINT_HIT_READ 0x40
 #define BP_WATCHPOINT_HIT_WRITE 0x80
 #define BP_WATCHPOINT_HIT (BP_WATCHPOINT_HIT_READ | BP_WATCHPOINT_HIT_WRITE)
@@ -660,6 +844,21 @@
 void cpu_breakpoint_remove_by_ref(CPUState *cpu, CPUBreakpoint *breakpoint);
 void cpu_breakpoint_remove_all(CPUState *cpu, int mask);
 
+/* Return true if PC matches an installed breakpoint.  */
+static inline bool cpu_breakpoint_test(CPUState *cpu, vaddr pc, int mask)
+{
+    CPUBreakpoint *bp;
+
+    if (unlikely(!QTAILQ_EMPTY(&cpu->breakpoints))) {
+        QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
+            if (bp->pc == pc && (bp->flags & mask)) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 int cpu_watchpoint_insert(CPUState *cpu, vaddr addr, vaddr len,
                           int flags, CPUWatchpoint **watchpoint);
 int cpu_watchpoint_remove(CPUState *cpu, vaddr addr,
@@ -667,8 +866,19 @@
 void cpu_watchpoint_remove_by_ref(CPUState *cpu, CPUWatchpoint *watchpoint);
 void cpu_watchpoint_remove_all(CPUState *cpu, int mask);
 
+/**
+ * cpu_get_address_space:
+ * @cpu: CPU to get address space from
+ * @asidx: index identifying which address space to get
+ *
+ * Return the requested address space of this CPU. @asidx
+ * specifies which address space to read.
+ */
+AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx);
+
 void QEMU_NORETURN cpu_abort(CPUState *cpu, const char *fmt, ...)
     GCC_FMT_ATTR(2, 3);
+void cpu_exec_exit(CPUState *cpu);
 
 #ifdef CONFIG_SOFTMMU
 extern const struct VMStateDescription vmstate_cpu_common;
@@ -684,4 +894,6 @@
     .offset = 0,                                                            \
 }
 
+#define UNASSIGNED_CPU_INDEX -1
+
 #endif
diff --git a/include/qom/object.h b/include/qom/object.h
index fcf4c70..853417f 100644
--- a/include/qom/object.h
+++ b/include/qom/object.h
@@ -14,13 +14,8 @@
 #ifndef QEMU_OBJECT_H
 #define QEMU_OBJECT_H
 
-#include <glib.h>
-#include <stdint.h>
-#include <stdbool.h>
+#include "qapi-types.h"
 #include "qemu/queue.h"
-#include "qapi/error.h"
-
-struct Visitor;
 
 struct TypeImpl;
 typedef struct TypeImpl *Type;
@@ -273,7 +268,7 @@
  *     .name = TYPE_DERIVED,
  *     .parent = TYPE_MY,
  *     .class_size = sizeof(DerivedClass),
- *     .class_init = my_class_init,
+ *     .class_init = derived_class_init,
  * };
  *   </programlisting>
  * </example>
@@ -291,16 +286,16 @@
  * ObjectPropertyAccessor:
  * @obj: the object that owns the property
  * @v: the visitor that contains the property data
- * @opaque: the object property opaque
  * @name: the name of the property
+ * @opaque: the object property opaque
  * @errp: a pointer to an Error that is filled if getting/setting fails.
  *
  * Called when trying to get/set a property.
  */
 typedef void (ObjectPropertyAccessor)(Object *obj,
-                                      struct Visitor *v,
-                                      void *opaque,
+                                      Visitor *v,
                                       const char *name,
+                                      void *opaque,
                                       Error **errp);
 
 /**
@@ -344,8 +339,6 @@
     ObjectPropertyResolve *resolve;
     ObjectPropertyRelease *release;
     void *opaque;
-
-    QTAILQ_ENTRY(ObjectProperty) node;
 } ObjectProperty;
 
 /**
@@ -383,6 +376,8 @@
     const char *class_cast_cache[OBJECT_CLASS_CAST_CACHE];
 
     ObjectUnparent *unparent;
+
+    GHashTable *properties;
 };
 
 /**
@@ -396,16 +391,13 @@
  * As a result, #Object contains a reference to the objects type as its
  * first member.  This allows identification of the real type of the object at
  * run time.
- *
- * #Object also contains a list of #Interfaces that this object
- * implements.
  */
 struct Object
 {
     /*< private >*/
-    ObjectClass *klass;
+    ObjectClass *clazz;
     ObjectFree *free;
-    QTAILQ_HEAD(, ObjectProperty) properties;
+    GHashTable *properties;
     uint32_t ref;
     Object *parent;
 };
@@ -510,16 +502,16 @@
 
 /**
  * OBJECT_CLASS_CHECK:
- * @class: The C type to use for the return value.
- * @obj: A derivative of @type to cast.
- * @name: the QOM typename of @class.
+ * @class_type: The C type to use for the return value.
+ * @class: A derivative class of @class_type to cast.
+ * @name: the QOM typename of @class_type.
  *
  * A type safe version of @object_class_dynamic_cast_assert.  This macro is
  * typically wrapped by each type to perform type safe casts of a class to a
  * specific class type.
  */
-#define OBJECT_CLASS_CHECK(class, obj, name) \
-    ((class *)object_class_dynamic_cast_assert(OBJECT_CLASS(obj), (name), \
+#define OBJECT_CLASS_CHECK(class_type, class, name) \
+    ((class_type *)object_class_dynamic_cast_assert(OBJECT_CLASS(class), (name), \
                                                __FILE__, __LINE__, __func__))
 
 /**
@@ -592,7 +584,7 @@
  *
  * Returns: The newly allocated and instantiated object.
  */
-Object *object_new(const char *typenam);
+Object *object_new(const char *typename_);
 
 /**
  * object_new_with_type:
@@ -607,6 +599,134 @@
 Object *object_new_with_type(Type type);
 
 /**
+ * object_new_with_props:
+ * @typename:  The name of the type of the object to instantiate.
+ * @parent: the parent object
+ * @id: The unique ID of the object
+ * @errp: pointer to error object
+ * @...: list of property names and values
+ *
+ * This function will initialize a new object using heap allocated memory.
+ * The returned object has a reference count of 1, and will be freed when
+ * the last reference is dropped.
+ *
+ * The @id parameter will be used when registering the object as a
+ * child of @parent in the composition tree.
+ *
+ * The variadic parameters are a list of pairs of (propname, propvalue)
+ * strings. The propname of %NULL indicates the end of the property
+ * list. If the object implements the user creatable interface, the
+ * object will be marked complete once all the properties have been
+ * processed.
+ *
+ * <example>
+ *   <title>Creating an object with properties</title>
+ *   <programlisting>
+ *   Error *err = NULL;
+ *   Object *obj;
+ *
+ *   obj = object_new_with_props(TYPE_MEMORY_BACKEND_FILE,
+ *                               object_get_objects_root(),
+ *                               "hostmem0",
+ *                               &err,
+ *                               "share", "yes",
+ *                               "mem-path", "/dev/shm/somefile",
+ *                               "prealloc", "yes",
+ *                               "size", "1048576",
+ *                               NULL);
+ *
+ *   if (!obj) {
+ *     g_printerr("Cannot create memory backend: %s\n",
+ *                error_get_pretty(err));
+ *   }
+ *   </programlisting>
+ * </example>
+ *
+ * The returned object will have one stable reference maintained
+ * for as long as it is present in the object hierarchy.
+ *
+ * Returns: The newly allocated, instantiated & initialized object.
+ */
+Object *object_new_with_props(const char *typename_,
+                              Object *parent,
+                              const char *id,
+                              Error **errp,
+                              ...) QEMU_SENTINEL;
+
+/**
+ * object_new_with_propv:
+ * @typename:  The name of the type of the object to instantiate.
+ * @parent: the parent object
+ * @id: The unique ID of the object
+ * @errp: pointer to error object
+ * @vargs: list of property names and values
+ *
+ * See object_new_with_props() for documentation.
+ */
+Object *object_new_with_propv(const char *typename_,
+                              Object *parent,
+                              const char *id,
+                              Error **errp,
+                              va_list vargs);
+
+/**
+ * object_set_props:
+ * @obj: the object instance to set properties on
+ * @errp: pointer to error object
+ * @...: list of property names and values
+ *
+ * This function will set a list of properties on an existing object
+ * instance.
+ *
+ * The variadic parameters are a list of pairs of (propname, propvalue)
+ * strings. The propname of %NULL indicates the end of the property
+ * list.
+ *
+ * <example>
+ *   <title>Update an object's properties</title>
+ *   <programlisting>
+ *   Error *err = NULL;
+ *   Object *obj = ...get / create object...;
+ *
+ *   obj = object_set_props(obj,
+ *                          &err,
+ *                          "share", "yes",
+ *                          "mem-path", "/dev/shm/somefile",
+ *                          "prealloc", "yes",
+ *                          "size", "1048576",
+ *                          NULL);
+ *
+ *   if (!obj) {
+ *     g_printerr("Cannot set properties: %s\n",
+ *                error_get_pretty(err));
+ *   }
+ *   </programlisting>
+ * </example>
+ *
+ * The returned object will have one stable reference maintained
+ * for as long as it is present in the object hierarchy.
+ *
+ * Returns: -1 on error, 0 on success
+ */
+int object_set_props(Object *obj,
+                     Error **errp,
+                     ...) QEMU_SENTINEL;
+
+/**
+ * object_set_propv:
+ * @obj: the object instance to set properties on
+ * @errp: pointer to error object
+ * @vargs: list of property names and values
+ *
+ * See object_set_props() for documentation.
+ *
+ * Returns: -1 on error, 0 on success
+ */
+int object_set_propv(Object *obj,
+                     Error **errp,
+                     va_list vargs);
+
+/**
  * object_initialize_with_type:
  * @data: A pointer to the memory to be used for the object.
  * @size: The maximum size available at @data for the object.
@@ -628,7 +748,7 @@
  * have already been allocated.  The returned object has a reference count of 1,
  * and will be finalized when the last reference is dropped.
  */
-void object_initialize(void *obj, size_t size, const char *typenam);
+void object_initialize(void *obj, size_t size, const char *typename_);
 
 /**
  * object_dynamic_cast:
@@ -640,7 +760,7 @@
  *
  * Returns: This function returns @obj on success or #NULL on failure.
  */
-Object *object_dynamic_cast(Object *obj, const char *typenam);
+Object *object_dynamic_cast(Object *obj, const char *typename_);
 
 /**
  * object_dynamic_cast_assert:
@@ -651,7 +771,7 @@
  * This function is not meant to be called directly, but only through
  * the wrapper macro OBJECT_CHECK.
  */
-Object *object_dynamic_cast_assert(Object *obj, const char *typenam,
+Object *object_dynamic_cast_assert(Object *obj, const char *typename_,
                                    const char *file, int line, const char *func);
 
 /**
@@ -704,7 +824,7 @@
  * the wrapper macros OBJECT_CLASS_CHECK and INTERFACE_CHECK.
  */
 ObjectClass *object_class_dynamic_cast_assert(ObjectClass *klass,
-                                              const char *typenam,
+                                              const char *typename_,
                                               const char *file, int line,
                                               const char *func);
 
@@ -723,7 +843,7 @@
  * it.  (FIXME: perhaps this can be detected at type definition time?)
  */
 ObjectClass *object_class_dynamic_cast(ObjectClass *klass,
-                                       const char *typenam);
+                                       const char *typename_);
 
 /**
  * object_class_get_parent:
@@ -755,7 +875,7 @@
  *
  * Returns: The class for @typename or %NULL if not found.
  */
-ObjectClass *object_class_by_name(const char *typenam);
+ObjectClass *object_class_by_name(const char *typename_);
 
 void object_class_foreach(void (*fn)(ObjectClass *klass, void *opaque),
                           const char *implements_type, bool include_abstract,
@@ -781,7 +901,7 @@
 void object_ref(Object *obj);
 
 /**
- * qdef_unref:
+ * object_unref:
  * @obj: the object
  *
  * Decrease the reference count of a object.  A object cannot be freed as long
@@ -821,6 +941,13 @@
 
 void object_property_del(Object *obj, const char *name, Error **errp);
 
+ObjectProperty *object_class_property_add(ObjectClass *klass, const char *name,
+                                          const char *type,
+                                          ObjectPropertyAccessor *get,
+                                          ObjectPropertyAccessor *set,
+                                          ObjectPropertyRelease *release,
+                                          void *opaque, Error **errp);
+
 /**
  * object_property_find:
  * @obj: the object
@@ -831,6 +958,55 @@
  */
 ObjectProperty *object_property_find(Object *obj, const char *name,
                                      Error **errp);
+ObjectProperty *object_class_property_find(ObjectClass *klass, const char *name,
+                                           Error **errp);
+
+typedef struct ObjectPropertyIterator {
+    ObjectClass *nextclass;
+    GHashTableIter iter;
+} ObjectPropertyIterator;
+
+/**
+ * object_property_iter_init:
+ * @obj: the object
+ *
+ * Initializes an iterator for traversing all properties
+ * registered against an object instance, its class and all parent classes.
+ *
+ * It is forbidden to modify the property list while iterating,
+ * whether removing or adding properties.
+ *
+ * Typical usage pattern would be
+ *
+ * <example>
+ *   <title>Using object property iterators</title>
+ *   <programlisting>
+ *   ObjectProperty *prop;
+ *   ObjectPropertyIterator iter;
+ *
+ *   object_property_iter_init(&iter, obj);
+ *   while ((prop = object_property_iter_next(&iter))) {
+ *     ... do something with prop ...
+ *   }
+ *   </programlisting>
+ * </example>
+ */
+void object_property_iter_init(ObjectPropertyIterator *iter,
+                               Object *obj);
+
+/**
+ * object_property_iter_next:
+ * @iter: the iterator instance
+ *
+ * Return the next available property. If no further properties
+ * are available, a %NULL value will be returned and the @iter
+ * pointer should not be used again after this point without
+ * re-initializing it.
+ *
+ * Returns: the next property, or %NULL when all properties
+ * have been traversed.
+ */
+ObjectProperty *object_property_iter_next(ObjectPropertyIterator *iter);
 
 void object_unparent(Object *obj);
 
@@ -844,7 +1020,7 @@
  *
  * Reads a property from a object.
  */
-void object_property_get(Object *obj, struct Visitor *v, const char *name,
+void object_property_get(Object *obj, Visitor *v, const char *name,
                          Error **errp);
 
 /**
@@ -935,7 +1111,7 @@
  * @name: the name of the property
  * @errp: returns an error if this function fails
  *
- * Returns: the value of the property, converted to an integer, or NULL if
+ * Returns: the value of the property, converted to an integer, or negative if
  * an error occurs (including when the property value is not an integer).
  */
 int64_t object_property_get_int(Object *obj, const char *name,
@@ -945,7 +1121,7 @@
  * object_property_get_enum:
  * @obj: the object
  * @name: the name of the property
- * @strings: strings corresponding to enums
+ * @typename: the name of the enum data type
  * @errp: returns an error if this function fails
  *
  * Returns: the value of the property, converted to an integer, or
@@ -953,7 +1129,7 @@
  * an enum).
  */
 int object_property_get_enum(Object *obj, const char *name,
-                             const char *strings[], Error **errp);
+                             const char *typename_, Error **errp);
 
 /**
  * object_property_get_uint16List:
@@ -980,7 +1156,7 @@
  *
  * Writes a property to a object.
  */
-void object_property_set(Object *obj, struct Visitor *v, const char *name,
+void object_property_set(Object *obj, Visitor *v, const char *name,
                          Error **errp);
 
 /**
@@ -1026,6 +1202,18 @@
  */
 Object *object_get_root(void);
 
+
+/**
+ * object_get_objects_root:
+ *
+ * Get the container object that holds user created
+ * object instances. This is the object at path
+ * "/objects"
+ *
+ * Returns: the user object container
+ */
+Object *object_get_objects_root(void);
+
 /**
  * object_get_canonical_path_component:
  *
@@ -1085,7 +1273,7 @@
  *
  * Returns: The matched object or NULL on path lookup failure.
  */
-Object *object_resolve_path_type(const char *path, const char *typenam,
+Object *object_resolve_path_type(const char *path, const char *typename_,
                                  bool *ambiguous);
 
 /**
@@ -1187,6 +1375,12 @@
                              void (*set)(Object *, const char *, Error **),
                              Error **errp);
 
+void object_class_property_add_str(ObjectClass *klass, const char *name,
+                                   char *(*get)(Object *, Error **),
+                                   void (*set)(Object *, const char *,
+                                               Error **),
+                                   Error **errp);
+
 /**
  * object_property_add_bool:
  * @obj: the object to add a property to
@@ -1203,6 +1397,55 @@
                               void (*set)(Object *, bool, Error **),
                               Error **errp);
 
+void object_class_property_add_bool(ObjectClass *klass, const char *name,
+                                    bool (*get)(Object *, Error **),
+                                    void (*set)(Object *, bool, Error **),
+                                    Error **errp);
+
+/**
+ * object_property_add_enum:
+ * @obj: the object to add a property to
+ * @name: the name of the property
+ * @typename: the name of the enum data type
+ * @get: the getter or %NULL if the property is write-only.
+ * @set: the setter or %NULL if the property is read-only
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Add an enum property using getters/setters.  This function will add a
+ * property of type '@typename'.
+ */
+void object_property_add_enum(Object *obj, const char *name,
+                              const char *typename_,
+                              const char * const *strings,
+                              int (*get)(Object *, Error **),
+                              void (*set)(Object *, int, Error **),
+                              Error **errp);
+
+void object_class_property_add_enum(ObjectClass *klass, const char *name,
+                                    const char *typename_,
+                                    const char * const *strings,
+                                    int (*get)(Object *, Error **),
+                                    void (*set)(Object *, int, Error **),
+                                    Error **errp);
+
+/**
+ * object_property_add_tm:
+ * @obj: the object to add a property to
+ * @name: the name of the property
+ * @get: the getter or NULL if the property is write-only.
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Add a read-only struct tm valued property using a getter function.
+ * This function will add a property of type 'struct tm'.
+ */
+void object_property_add_tm(Object *obj, const char *name,
+                            void (*get)(Object *, struct tm *, Error **),
+                            Error **errp);
+
+void object_class_property_add_tm(ObjectClass *klass, const char *name,
+                                  void (*get)(Object *, struct tm *, Error **),
+                                  Error **errp);
+
 /**
  * object_property_add_uint8_ptr:
  * @obj: the object to add a property to
@@ -1215,6 +1458,8 @@
  */
 void object_property_add_uint8_ptr(Object *obj, const char *name,
                                    const uint8_t *v, Error **errp);
+void object_class_property_add_uint8_ptr(ObjectClass *klass, const char *name,
+                                         const uint8_t *v, Error **errp);
 
 /**
  * object_property_add_uint16_ptr:
@@ -1228,6 +1473,8 @@
  */
 void object_property_add_uint16_ptr(Object *obj, const char *name,
                                     const uint16_t *v, Error **errp);
+void object_class_property_add_uint16_ptr(ObjectClass *klass, const char *name,
+                                          const uint16_t *v, Error **errp);
 
 /**
  * object_property_add_uint32_ptr:
@@ -1241,6 +1488,8 @@
  */
 void object_property_add_uint32_ptr(Object *obj, const char *name,
                                     const uint32_t *v, Error **errp);
+void object_class_property_add_uint32_ptr(ObjectClass *klass, const char *name,
+                                          const uint32_t *v, Error **errp);
 
 /**
  * object_property_add_uint64_ptr:
@@ -1254,6 +1503,8 @@
  */
 void object_property_add_uint64_ptr(Object *obj, const char *name,
                                     const uint64_t *v, Error **Errp);
+void object_class_property_add_uint64_ptr(ObjectClass *klass, const char *name,
+                                          const uint64_t *v, Error **Errp);
 
 /**
  * object_property_add_alias:
@@ -1276,6 +1527,24 @@
                                Error **errp);
 
 /**
+ * object_property_add_const_link:
+ * @obj: the object to add a property to
+ * @name: the name of the property
+ * @target: the object to be referred by the link
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Add an unmodifiable link for a property on an object.  This function will
+ * add a property of type link<TYPE> where TYPE is the type of @target.
+ *
+ * The caller must ensure that @target stays alive as long as
+ * this property exists.  In the case @target is a child of @obj,
+ * this will be the case.  Otherwise, the caller is responsible for
+ * taking a reference.
+ */
+void object_property_add_const_link(Object *obj, const char *name,
+                                    Object *target, Error **errp);
+
+/**
  * object_property_set_description:
  * @obj: the object owning the property
  * @name: the name of the property
@@ -1287,6 +1556,9 @@
  */
 void object_property_set_description(Object *obj, const char *name,
                                      const char *description, Error **errp);
+void object_class_property_set_description(ObjectClass *klass, const char *name,
+                                           const char *description,
+                                           Error **errp);
 
 /**
  * object_child_foreach:
@@ -1297,12 +1569,33 @@
  * Call @fn passing each child of @obj and @opaque to it, until @fn returns
  * non-zero.
  *
+ * It is forbidden to add or remove children from @obj from the @fn
+ * callback.
+ *
  * Returns: The last value returned by @fn, or 0 if there is no child.
  */
 int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
                          void *opaque);
 
 /**
+ * object_child_foreach_recursive:
+ * @obj: the object whose children will be navigated
+ * @fn: the iterator function to be called
+ * @opaque: an opaque value that will be passed to the iterator
+ *
+ * Call @fn passing each child of @obj and @opaque to it, until @fn returns
+ * non-zero. Calls recursively, all child nodes of @obj will also be passed
+ * all the way down to the leaf nodes of the tree. Depth first ordering.
+ *
+ * It is forbidden to add or remove children from @obj (or its
+ * child nodes) from the @fn callback.
+ *
+ * Returns: The last value returned by @fn, or 0 if there is no child.
+ */
+int object_child_foreach_recursive(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque);
+/**
  * container_get:
  * @root: root of the #path, e.g., object_get_root()
  * @path: path to the container
@@ -1314,5 +1607,11 @@
  */
 Object *container_get(Object *root, const char *path);
 
-
+/**
+ * object_type_get_instance_size:
+ * @typename: Name of the Type whose instance_size is required
+ *
+ * Returns the instance_size of the given @typename.
+ */
+size_t object_type_get_instance_size(const char *typename_);
 #endif
diff --git a/include/qom/object_interfaces.h b/include/qom/object_interfaces.h
index b792283..8b17f4d 100644
--- a/include/qom/object_interfaces.h
+++ b/include/qom/object_interfaces.h
@@ -2,6 +2,8 @@
 #define OBJECT_INTERFACES_H
 
 #include "qom/object.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/visitor.h"
 
 #define TYPE_USER_CREATABLE "user-creatable"
 
@@ -25,6 +27,8 @@
  * UserCreatableClass:
  * @parent_class: the base class
  * @complete: callback to be called after @obj's properties are set.
+ * @can_be_deleted: callback to be called before an object is removed
+ * to check if @obj can be removed safely.
  *
  * Interface is designed to work with -object/object-add/object_add
  * commands.
@@ -47,6 +51,7 @@
 
     /* <public> */
     void (*complete)(UserCreatable *uc, Error **errp);
+    bool (*can_be_deleted)(UserCreatable *uc, Error **errp);
 } UserCreatableClass;
 
 /**
@@ -59,4 +64,105 @@
  * nothing.
  */
 void user_creatable_complete(Object *obj, Error **errp);
+
+/**
+ * user_creatable_can_be_deleted:
+ * @uc: the object whose can_be_deleted() method is called if implemented
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Wrapper to call can_be_deleted() method if one of types it's inherited
+ * from implements USER_CREATABLE interface.
+ */
+bool user_creatable_can_be_deleted(UserCreatable *uc, Error **errp);
+
+/**
+ * user_creatable_add:
+ * @qdict: the object definition
+ * @v: the visitor
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Create an instance of the user creatable object whose type
+ * is defined in @qdict by the 'qom-type' field, placing it
+ * in the object composition tree with name provided by the
+ * 'id' field. The remaining fields in @qdict are used to
+ * initialize the object properties.
+ *
+ * Returns: the newly created object or NULL on error
+ */
+Object *user_creatable_add(const QDict *qdict,
+                           Visitor *v, Error **errp);
+
+/**
+ * user_creatable_add_type:
+ * @type: the object type name
+ * @id: the unique ID for the object
+ * @qdict: the object properties
+ * @v: the visitor
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Create an instance of the user creatable object @type, placing
+ * it in the object composition tree with name @id, initializing
+ * it with properties from @qdict
+ *
+ * Returns: the newly created object or NULL on error
+ */
+Object *user_creatable_add_type(const char *type, const char *id,
+                                const QDict *qdict,
+                                Visitor *v, Error **errp);
+
+/**
+ * user_creatable_add_opts:
+ * @opts: the object definition
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Create an instance of the user creatable object whose type
+ * is defined in @opts by the 'qom-type' option, placing it
+ * in the object composition tree with name provided by the
+ * 'id' field. The remaining options in @opts are used to
+ * initialize the object properties.
+ *
+ * Returns: the newly created object or NULL on error
+ */
+Object *user_creatable_add_opts(QemuOpts *opts, Error **errp);
+
+
+/**
+ * user_creatable_add_opts_predicate:
+ * @type: the QOM type to be added
+ *
+ * A callback function to determine whether an object
+ * of type @type should be created. Instances of this
+ * callback should be passed to user_creatable_add_opts_foreach
+ */
+typedef bool (*user_creatable_add_opts_predicate)(const char *type);
+
+/**
+ * user_creatable_add_opts_foreach:
+ * @opaque: a user_creatable_add_opts_predicate callback or NULL
+ * @opts: options to create
+ * @errp: unused
+ *
+ * An iterator callback to be used in conjunction with
+ * the qemu_opts_foreach() method for creating a list of
+ * objects from a set of QemuOpts
+ *
+ * The @opaque parameter can be passed a user_creatable_add_opts_predicate
+ * callback to filter which types of object are created during iteration.
+ * When it fails, report the error.
+ *
+ * Returns: 0 on success, -1 when an error was reported.
+ */
+int user_creatable_add_opts_foreach(void *opaque,
+                                    QemuOpts *opts, Error **errp);
+
+/**
+ * user_creatable_del:
+ * @id: the unique ID for the object
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Delete an instance of the user creatable object identified
+ * by @id.
+ */
+void user_creatable_del(const char *id, Error **errp);
+
 #endif
diff --git a/include/standard-headers/asm-s390/kvm_virtio.h b/include/standard-headers/asm-s390/kvm_virtio.h
new file mode 100644
index 0000000..daad324
--- /dev/null
+++ b/include/standard-headers/asm-s390/kvm_virtio.h
@@ -0,0 +1,64 @@
+/*
+ * definition for virtio for kvm on s390
+ *
+ * Copyright IBM Corp. 2008
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2 only)
+ * as published by the Free Software Foundation.
+ *
+ *    Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
+ */
+
+#ifndef __KVM_S390_VIRTIO_H
+#define __KVM_S390_VIRTIO_H
+
+#include "standard-headers/linux/types.h"
+
+struct kvm_device_desc {
+	/* The device type: console, network, disk etc.  Type 0 terminates. */
+	uint8_t type;
+	/* The number of virtqueues (first in config array) */
+	uint8_t num_vq;
+	/*
+	 * The number of bytes of feature bits.  Multiply by 2: one for host
+	 * features and one for guest acknowledgements.
+	 */
+	uint8_t feature_len;
+	/* The number of bytes of the config array after virtqueues. */
+	uint8_t config_len;
+	/* A status byte, written by the Guest. */
+	uint8_t status;
+	uint8_t config[0];
+};
+
+/*
+ * This is how we expect the device configuration field for a virtqueue
+ * to be laid out in config space.
+ */
+struct kvm_vqconfig {
+	/* The token returned with an interrupt. Set by the guest */
+	uint64_t token;
+	/* The address of the virtio ring */
+	uint64_t address;
+	/* The number of entries in the virtio_ring */
+	uint16_t num;
+
+};
+
+#define KVM_S390_VIRTIO_NOTIFY		0
+#define KVM_S390_VIRTIO_RESET		1
+#define KVM_S390_VIRTIO_SET_STATUS	2
+
+/* The alignment to use between consumer and producer parts of vring.
+ * This is pagesize for historical reasons. */
+#define KVM_S390_VIRTIO_RING_ALIGN	4096
+
+
+/* These values are supposed to be in ext_params on an interrupt */
+#define VIRTIO_PARAM_MASK		0xff
+#define VIRTIO_PARAM_VRING_INTERRUPT	0x0
+#define VIRTIO_PARAM_CONFIG_CHANGED	0x1
+#define VIRTIO_PARAM_DEV_ADD		0x2
+
+#endif
diff --git a/include/standard-headers/asm-s390/virtio-ccw.h b/include/standard-headers/asm-s390/virtio-ccw.h
new file mode 100644
index 0000000..a9a4ebf
--- /dev/null
+++ b/include/standard-headers/asm-s390/virtio-ccw.h
@@ -0,0 +1,21 @@
+/*
+ * Definitions for virtio-ccw devices.
+ *
+ * Copyright IBM Corp. 2013
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License (version 2 only)
+ * as published by the Free Software Foundation.
+ *
+ *  Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ */
+#ifndef __KVM_VIRTIO_CCW_H
+#define __KVM_VIRTIO_CCW_H
+
+/* Alignment of vring buffers. */
+#define KVM_VIRTIO_CCW_RING_ALIGN 4096
+
+/* Subcode for diagnose 500 (virtio hypercall). */
+#define KVM_S390_VIRTIO_CCW_NOTIFY 3
+
+#endif
diff --git a/include/standard-headers/asm-x86/hyperv.h b/include/standard-headers/asm-x86/hyperv.h
new file mode 100644
index 0000000..47b38fb
--- /dev/null
+++ b/include/standard-headers/asm-x86/hyperv.h
@@ -0,0 +1,366 @@
+#ifndef _ASM_X86_HYPERV_H
+#define _ASM_X86_HYPERV_H
+
+#include "standard-headers/linux/types.h"
+
+/*
+ * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
+ * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
+ */
+#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
+#define HYPERV_CPUID_INTERFACE			0x40000001
+#define HYPERV_CPUID_VERSION			0x40000002
+#define HYPERV_CPUID_FEATURES			0x40000003
+#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
+#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
+
+#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
+#define HYPERV_CPUID_MIN			0x40000005
+#define HYPERV_CPUID_MAX			0x4000ffff
+
+/*
+ * Feature identification. EAX indicates which features are available
+ * to the partition based upon the current partition privileges.
+ */
+
+/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */
+#define HV_X64_MSR_VP_RUNTIME_AVAILABLE		(1 << 0)
+/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
+#define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
+/* Partition reference TSC MSR is available */
+#define HV_X64_MSR_REFERENCE_TSC_AVAILABLE              (1 << 9)
+
+/* A partition's reference time stamp counter (TSC) page */
+#define HV_X64_MSR_REFERENCE_TSC		0x40000021
+
+/*
+ * There is a single feature flag that signifies the presence of the MSR
+ * that can be used to retrieve both the local APIC Timer frequency as
+ * well as the TSC frequency.
+ */
+
+/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
+#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
+
+/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
+#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
+
+/*
+ * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
+ * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
+ */
+#define HV_X64_MSR_SYNIC_AVAILABLE		(1 << 2)
+/*
+ * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through
+ * HV_X64_MSR_STIMER3_COUNT) available
+ */
+#define HV_X64_MSR_SYNTIMER_AVAILABLE		(1 << 3)
+/*
+ * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR)
+ * are available
+ */
+#define HV_X64_MSR_APIC_ACCESS_AVAILABLE	(1 << 4)
+/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/
+#define HV_X64_MSR_HYPERCALL_AVAILABLE		(1 << 5)
+/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/
+#define HV_X64_MSR_VP_INDEX_AVAILABLE		(1 << 6)
+/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/
+#define HV_X64_MSR_RESET_AVAILABLE		(1 << 7)
+ /*
+  * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE,
+  * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE,
+  * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available
+  */
+#define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
+
+/*
+ * Feature identification: EBX indicates which flags were specified at
+ * partition creation. The format is the same as the partition creation
+ * flag structure defined in section Partition Creation Flags.
+ */
+#define HV_X64_CREATE_PARTITIONS		(1 << 0)
+#define HV_X64_ACCESS_PARTITION_ID		(1 << 1)
+#define HV_X64_ACCESS_MEMORY_POOL		(1 << 2)
+#define HV_X64_ADJUST_MESSAGE_BUFFERS		(1 << 3)
+#define HV_X64_POST_MESSAGES			(1 << 4)
+#define HV_X64_SIGNAL_EVENTS			(1 << 5)
+#define HV_X64_CREATE_PORT			(1 << 6)
+#define HV_X64_CONNECT_PORT			(1 << 7)
+#define HV_X64_ACCESS_STATS			(1 << 8)
+#define HV_X64_DEBUGGING			(1 << 11)
+#define HV_X64_CPU_POWER_MANAGEMENT		(1 << 12)
+#define HV_X64_CONFIGURE_PROFILER		(1 << 13)
+
+/*
+ * Feature identification. EDX indicates which miscellaneous features
+ * are available to the partition.
+ */
+/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
+#define HV_X64_MWAIT_AVAILABLE				(1 << 0)
+/* Guest debugging support is available */
+#define HV_X64_GUEST_DEBUGGING_AVAILABLE		(1 << 1)
+/* Performance Monitor support is available*/
+#define HV_X64_PERF_MONITOR_AVAILABLE			(1 << 2)
+/* Support for physical CPU dynamic partitioning events is available*/
+#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	(1 << 3)
+/*
+ * Support for passing hypercall input parameter block via XMM
+ * registers is available
+ */
+#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE		(1 << 4)
+/* Support for a virtual guest idle state is available */
+#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		(1 << 5)
+/* Guest crash data handler available */
+#define HV_X64_GUEST_CRASH_MSR_AVAILABLE		(1 << 10)
+
+/*
+ * Implementation recommendations. Indicates which behaviors the hypervisor
+ * recommends the OS implement for optimal performance.
+ */
+ /*
+  * Recommend using hypercall for address space switches rather
+  * than MOV to CR3 instruction
+  */
+#define HV_X64_MWAIT_RECOMMENDED		(1 << 0)
+/* Recommend using hypercall for local TLB flushes rather
+ * than INVLPG or MOV to CR3 instructions */
+#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED	(1 << 1)
+/*
+ * Recommend using hypercall for remote TLB flushes rather
+ * than inter-processor interrupts
+ */
+#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED	(1 << 2)
+/*
+ * Recommend using MSRs for accessing APIC registers
+ * EOI, ICR and TPR rather than their memory-mapped counterparts
+ */
+#define HV_X64_APIC_ACCESS_RECOMMENDED		(1 << 3)
+/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
+#define HV_X64_SYSTEM_RESET_RECOMMENDED		(1 << 4)
+/*
+ * Recommend using relaxed timing for this partition. If used,
+ * the VM should disable any watchdog timeouts that rely on the
+ * timely delivery of external interrupts
+ */
+#define HV_X64_RELAXED_TIMING_RECOMMENDED	(1 << 5)
+
+/* MSR used to identify the guest OS. */
+#define HV_X64_MSR_GUEST_OS_ID			0x40000000
+
+/* MSR used to setup pages used to communicate with the hypervisor. */
+#define HV_X64_MSR_HYPERCALL			0x40000001
+
+/* MSR used to provide vcpu index */
+#define HV_X64_MSR_VP_INDEX			0x40000002
+
+/* MSR used to reset the guest OS. */
+#define HV_X64_MSR_RESET			0x40000003
+
+/* MSR used to provide vcpu runtime in 100ns units */
+#define HV_X64_MSR_VP_RUNTIME			0x40000010
+
+/* MSR used to read the per-partition time reference counter */
+#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
+
+/* MSR used to retrieve the TSC frequency */
+#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
+
+/* MSR used to retrieve the local APIC timer frequency */
+#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
+
+/* Define the virtual APIC registers */
+#define HV_X64_MSR_EOI				0x40000070
+#define HV_X64_MSR_ICR				0x40000071
+#define HV_X64_MSR_TPR				0x40000072
+#define HV_X64_MSR_APIC_ASSIST_PAGE		0x40000073
+
+/* Define synthetic interrupt controller model specific registers. */
+#define HV_X64_MSR_SCONTROL			0x40000080
+#define HV_X64_MSR_SVERSION			0x40000081
+#define HV_X64_MSR_SIEFP			0x40000082
+#define HV_X64_MSR_SIMP				0x40000083
+#define HV_X64_MSR_EOM				0x40000084
+#define HV_X64_MSR_SINT0			0x40000090
+#define HV_X64_MSR_SINT1			0x40000091
+#define HV_X64_MSR_SINT2			0x40000092
+#define HV_X64_MSR_SINT3			0x40000093
+#define HV_X64_MSR_SINT4			0x40000094
+#define HV_X64_MSR_SINT5			0x40000095
+#define HV_X64_MSR_SINT6			0x40000096
+#define HV_X64_MSR_SINT7			0x40000097
+#define HV_X64_MSR_SINT8			0x40000098
+#define HV_X64_MSR_SINT9			0x40000099
+#define HV_X64_MSR_SINT10			0x4000009A
+#define HV_X64_MSR_SINT11			0x4000009B
+#define HV_X64_MSR_SINT12			0x4000009C
+#define HV_X64_MSR_SINT13			0x4000009D
+#define HV_X64_MSR_SINT14			0x4000009E
+#define HV_X64_MSR_SINT15			0x4000009F
+
+/*
+ * Synthetic Timer MSRs. Four timers per vcpu.
+ */
+#define HV_X64_MSR_STIMER0_CONFIG		0x400000B0
+#define HV_X64_MSR_STIMER0_COUNT		0x400000B1
+#define HV_X64_MSR_STIMER1_CONFIG		0x400000B2
+#define HV_X64_MSR_STIMER1_COUNT		0x400000B3
+#define HV_X64_MSR_STIMER2_CONFIG		0x400000B4
+#define HV_X64_MSR_STIMER2_COUNT		0x400000B5
+#define HV_X64_MSR_STIMER3_CONFIG		0x400000B6
+#define HV_X64_MSR_STIMER3_COUNT		0x400000B7
+
+/* Hyper-V guest crash notification MSR's */
+#define HV_X64_MSR_CRASH_P0			0x40000100
+#define HV_X64_MSR_CRASH_P1			0x40000101
+#define HV_X64_MSR_CRASH_P2			0x40000102
+#define HV_X64_MSR_CRASH_P3			0x40000103
+#define HV_X64_MSR_CRASH_P4			0x40000104
+#define HV_X64_MSR_CRASH_CTL			0x40000105
+#define HV_X64_MSR_CRASH_CTL_NOTIFY		(1ULL << 63)
+#define HV_X64_MSR_CRASH_PARAMS		\
+		(1 + (HV_X64_MSR_CRASH_P4 - HV_X64_MSR_CRASH_P0))
+
+#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
+#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
+		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
+
+/* Declare the various hypercall operations. */
+#define HVCALL_NOTIFY_LONG_SPIN_WAIT		0x0008
+#define HVCALL_POST_MESSAGE			0x005c
+#define HVCALL_SIGNAL_EVENT			0x005d
+
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE		0x00000001
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT	12
+#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
+		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
+
+#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
+#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
+
+#define HV_PROCESSOR_POWER_STATE_C0		0
+#define HV_PROCESSOR_POWER_STATE_C1		1
+#define HV_PROCESSOR_POWER_STATE_C2		2
+#define HV_PROCESSOR_POWER_STATE_C3		3
+
+/* hypercall status code */
+#define HV_STATUS_SUCCESS			0
+#define HV_STATUS_INVALID_HYPERCALL_CODE	2
+#define HV_STATUS_INVALID_HYPERCALL_INPUT	3
+#define HV_STATUS_INVALID_ALIGNMENT		4
+#define HV_STATUS_INSUFFICIENT_MEMORY		11
+#define HV_STATUS_INVALID_CONNECTION_ID		18
+#define HV_STATUS_INSUFFICIENT_BUFFERS		19
+
+typedef struct _HV_REFERENCE_TSC_PAGE {
+	uint32_t tsc_sequence;
+	uint32_t res1;
+	uint64_t tsc_scale;
+	int64_t tsc_offset;
+} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
+
+/* Define the number of synthetic interrupt sources. */
+#define HV_SYNIC_SINT_COUNT		(16)
+/* Define the expected SynIC version. */
+#define HV_SYNIC_VERSION_1		(0x1)
+
+#define HV_SYNIC_CONTROL_ENABLE		(1ULL << 0)
+#define HV_SYNIC_SIMP_ENABLE		(1ULL << 0)
+#define HV_SYNIC_SIEFP_ENABLE		(1ULL << 0)
+#define HV_SYNIC_SINT_MASKED		(1ULL << 16)
+#define HV_SYNIC_SINT_AUTO_EOI		(1ULL << 17)
+#define HV_SYNIC_SINT_VECTOR_MASK	(0xFF)
+
+#define HV_SYNIC_STIMER_COUNT		(4)
+
+/* Define synthetic interrupt controller message constants. */
+#define HV_MESSAGE_SIZE			(256)
+#define HV_MESSAGE_PAYLOAD_BYTE_COUNT	(240)
+#define HV_MESSAGE_PAYLOAD_QWORD_COUNT	(30)
+
+/* Define hypervisor message types. */
+enum hv_message_type {
+	HVMSG_NONE			= 0x00000000,
+
+	/* Memory access messages. */
+	HVMSG_UNMAPPED_GPA		= 0x80000000,
+	HVMSG_GPA_INTERCEPT		= 0x80000001,
+
+	/* Timer notification messages. */
+	HVMSG_TIMER_EXPIRED			= 0x80000010,
+
+	/* Error messages. */
+	HVMSG_INVALID_VP_REGISTER_VALUE	= 0x80000020,
+	HVMSG_UNRECOVERABLE_EXCEPTION	= 0x80000021,
+	HVMSG_UNSUPPORTED_FEATURE		= 0x80000022,
+
+	/* Trace buffer complete messages. */
+	HVMSG_EVENTLOG_BUFFERCOMPLETE	= 0x80000040,
+
+	/* Platform-specific processor intercept messages. */
+	HVMSG_X64_IOPORT_INTERCEPT		= 0x80010000,
+	HVMSG_X64_MSR_INTERCEPT		= 0x80010001,
+	HVMSG_X64_CPUID_INTERCEPT		= 0x80010002,
+	HVMSG_X64_EXCEPTION_INTERCEPT	= 0x80010003,
+	HVMSG_X64_APIC_EOI			= 0x80010004,
+	HVMSG_X64_LEGACY_FP_ERROR		= 0x80010005
+};
+
+/* Define synthetic interrupt controller message flags. */
+union hv_message_flags {
+	uint8_t asu8;
+	struct {
+		uint8_t msg_pending:1;
+		uint8_t reserved:7;
+	};
+};
+
+/* Define port identifier type. */
+union hv_port_id {
+	uint32_t asu32;
+	struct {
+		uint32_t id:24;
+		uint32_t reserved:8;
+	} u;
+};
+
+/* Define synthetic interrupt controller message header. */
+struct hv_message_header {
+	uint32_t message_type;
+	uint8_t payload_size;
+	union hv_message_flags message_flags;
+	uint8_t reserved[2];
+	union {
+		uint64_t sender;
+		union hv_port_id port;
+	};
+};
+
+/* Define synthetic interrupt controller message format. */
+struct hv_message {
+	struct hv_message_header header;
+	union {
+		uint64_t payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT];
+	} u;
+};
+
+/* Define the synthetic interrupt message page layout. */
+struct hv_message_page {
+	struct hv_message sint_message[HV_SYNIC_SINT_COUNT];
+};
+
+/* Define timer message payload structure. */
+struct hv_timer_message_payload {
+	uint32_t timer_index;
+	uint32_t reserved;
+	uint64_t expiration_time;	/* When the timer expired */
+	uint64_t delivery_time;	/* When the message was delivered */
+};
+
+#define HV_STIMER_ENABLE		(1ULL << 0)
+#define HV_STIMER_PERIODIC		(1ULL << 1)
+#define HV_STIMER_LAZY			(1ULL << 2)
+#define HV_STIMER_AUTOENABLE		(1ULL << 3)
+#define HV_STIMER_SINT(config)		(uint8_t)(((config) >> 16) & 0x0F)
+
+#endif
diff --git a/include/standard-headers/linux/if_ether.h b/include/standard-headers/linux/if_ether.h
new file mode 100644
index 0000000..91cf735
--- /dev/null
+++ b/include/standard-headers/linux/if_ether.h
@@ -0,0 +1 @@
+#define ETH_ALEN    6
diff --git a/include/standard-headers/linux/input-event-codes.h b/include/standard-headers/linux/input-event-codes.h
new file mode 100644
index 0000000..354f0de
--- /dev/null
+++ b/include/standard-headers/linux/input-event-codes.h
@@ -0,0 +1,805 @@
+/*
+ * Input event codes
+ *
+ *    *** IMPORTANT ***
+ * This file is not only included from C-code but also from devicetree source
+ * files. As such this file MUST only contain comments and defines.
+ *
+ * Copyright (c) 1999-2002 Vojtech Pavlik
+ * Copyright (c) 2015 Hans de Goede <hdegoede@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+#ifndef _INPUT_EVENT_CODES_H
+#define _INPUT_EVENT_CODES_H
+
+/*
+ * Device properties and quirks
+ */
+
+#define INPUT_PROP_POINTER		0x00	/* needs a pointer */
+#define INPUT_PROP_DIRECT		0x01	/* direct input devices */
+#define INPUT_PROP_BUTTONPAD		0x02	/* has button(s) under pad */
+#define INPUT_PROP_SEMI_MT		0x03	/* touch rectangle only */
+#define INPUT_PROP_TOPBUTTONPAD		0x04	/* softbuttons at top of pad */
+#define INPUT_PROP_POINTING_STICK	0x05	/* is a pointing stick */
+#define INPUT_PROP_ACCELEROMETER	0x06	/* has accelerometer */
+
+#define INPUT_PROP_MAX			0x1f
+#define INPUT_PROP_CNT			(INPUT_PROP_MAX + 1)
+
+/*
+ * Event types
+ */
+
+#define EV_SYN			0x00
+#define EV_KEY			0x01
+#define EV_REL			0x02
+#define EV_ABS			0x03
+#define EV_MSC			0x04
+#define EV_SW			0x05
+#define EV_LED			0x11
+#define EV_SND			0x12
+#define EV_REP			0x14
+#define EV_FF			0x15
+#define EV_PWR			0x16
+#define EV_FF_STATUS		0x17
+#define EV_MAX			0x1f
+#define EV_CNT			(EV_MAX+1)
+
+/*
+ * Synchronization events.
+ */
+
+#define SYN_REPORT		0
+#define SYN_CONFIG		1
+#define SYN_MT_REPORT		2
+#define SYN_DROPPED		3
+#define SYN_MAX			0xf
+#define SYN_CNT			(SYN_MAX+1)
+
+/*
+ * Keys and buttons
+ *
+ * Most of the keys/buttons are modeled after USB HUT 1.12
+ * (see http://www.usb.org/developers/hidpage).
+ * Abbreviations in the comments:
+ * AC - Application Control
+ * AL - Application Launch Button
+ * SC - System Control
+ */
+
+#define KEY_RESERVED		0
+#define KEY_ESC			1
+#define KEY_1			2
+#define KEY_2			3
+#define KEY_3			4
+#define KEY_4			5
+#define KEY_5			6
+#define KEY_6			7
+#define KEY_7			8
+#define KEY_8			9
+#define KEY_9			10
+#define KEY_0			11
+#define KEY_MINUS		12
+#define KEY_EQUAL		13
+#define KEY_BACKSPACE		14
+#define KEY_TAB			15
+#define KEY_Q			16
+#define KEY_W			17
+#define KEY_E			18
+#define KEY_R			19
+#define KEY_T			20
+#define KEY_Y			21
+#define KEY_U			22
+#define KEY_I			23
+#define KEY_O			24
+#define KEY_P			25
+#define KEY_LEFTBRACE		26
+#define KEY_RIGHTBRACE		27
+#define KEY_ENTER		28
+#define KEY_LEFTCTRL		29
+#define KEY_A			30
+#define KEY_S			31
+#define KEY_D			32
+#define KEY_F			33
+#define KEY_G			34
+#define KEY_H			35
+#define KEY_J			36
+#define KEY_K			37
+#define KEY_L			38
+#define KEY_SEMICOLON		39
+#define KEY_APOSTROPHE		40
+#define KEY_GRAVE		41
+#define KEY_LEFTSHIFT		42
+#define KEY_BACKSLASH		43
+#define KEY_Z			44
+#define KEY_X			45
+#define KEY_C			46
+#define KEY_V			47
+#define KEY_B			48
+#define KEY_N			49
+#define KEY_M			50
+#define KEY_COMMA		51
+#define KEY_DOT			52
+#define KEY_SLASH		53
+#define KEY_RIGHTSHIFT		54
+#define KEY_KPASTERISK		55
+#define KEY_LEFTALT		56
+#define KEY_SPACE		57
+#define KEY_CAPSLOCK		58
+#define KEY_F1			59
+#define KEY_F2			60
+#define KEY_F3			61
+#define KEY_F4			62
+#define KEY_F5			63
+#define KEY_F6			64
+#define KEY_F7			65
+#define KEY_F8			66
+#define KEY_F9			67
+#define KEY_F10			68
+#define KEY_NUMLOCK		69
+#define KEY_SCROLLLOCK		70
+#define KEY_KP7			71
+#define KEY_KP8			72
+#define KEY_KP9			73
+#define KEY_KPMINUS		74
+#define KEY_KP4			75
+#define KEY_KP5			76
+#define KEY_KP6			77
+#define KEY_KPPLUS		78
+#define KEY_KP1			79
+#define KEY_KP2			80
+#define KEY_KP3			81
+#define KEY_KP0			82
+#define KEY_KPDOT		83
+
+#define KEY_ZENKAKUHANKAKU	85
+#define KEY_102ND		86
+#define KEY_F11			87
+#define KEY_F12			88
+#define KEY_RO			89
+#define KEY_KATAKANA		90
+#define KEY_HIRAGANA		91
+#define KEY_HENKAN		92
+#define KEY_KATAKANAHIRAGANA	93
+#define KEY_MUHENKAN		94
+#define KEY_KPJPCOMMA		95
+#define KEY_KPENTER		96
+#define KEY_RIGHTCTRL		97
+#define KEY_KPSLASH		98
+#define KEY_SYSRQ		99
+#define KEY_RIGHTALT		100
+#define KEY_LINEFEED		101
+#define KEY_HOME		102
+#define KEY_UP			103
+#define KEY_PAGEUP		104
+#define KEY_LEFT		105
+#define KEY_RIGHT		106
+#define KEY_END			107
+#define KEY_DOWN		108
+#define KEY_PAGEDOWN		109
+#define KEY_INSERT		110
+#define KEY_DELETE		111
+#define KEY_MACRO		112
+#define KEY_MUTE		113
+#define KEY_VOLUMEDOWN		114
+#define KEY_VOLUMEUP		115
+#define KEY_POWER		116	/* SC System Power Down */
+#define KEY_KPEQUAL		117
+#define KEY_KPPLUSMINUS		118
+#define KEY_PAUSE		119
+#define KEY_SCALE		120	/* AL Compiz Scale (Expose) */
+
+#define KEY_KPCOMMA		121
+#define KEY_HANGEUL		122
+#define KEY_HANGUEL		KEY_HANGEUL
+#define KEY_HANJA		123
+#define KEY_YEN			124
+#define KEY_LEFTMETA		125
+#define KEY_RIGHTMETA		126
+#define KEY_COMPOSE		127
+
+#define KEY_STOP		128	/* AC Stop */
+#define KEY_AGAIN		129
+#define KEY_PROPS		130	/* AC Properties */
+#define KEY_UNDO		131	/* AC Undo */
+#define KEY_FRONT		132
+#define KEY_COPY		133	/* AC Copy */
+#define KEY_OPEN		134	/* AC Open */
+#define KEY_PASTE		135	/* AC Paste */
+#define KEY_FIND		136	/* AC Search */
+#define KEY_CUT			137	/* AC Cut */
+#define KEY_HELP		138	/* AL Integrated Help Center */
+#define KEY_MENU		139	/* Menu (show menu) */
+#define KEY_CALC		140	/* AL Calculator */
+#define KEY_SETUP		141
+#define KEY_SLEEP		142	/* SC System Sleep */
+#define KEY_WAKEUP		143	/* System Wake Up */
+#define KEY_FILE		144	/* AL Local Machine Browser */
+#define KEY_SENDFILE		145
+#define KEY_DELETEFILE		146
+#define KEY_XFER		147
+#define KEY_PROG1		148
+#define KEY_PROG2		149
+#define KEY_WWW			150	/* AL Internet Browser */
+#define KEY_MSDOS		151
+#define KEY_COFFEE		152	/* AL Terminal Lock/Screensaver */
+#define KEY_SCREENLOCK		KEY_COFFEE
+#define KEY_ROTATE_DISPLAY	153	/* Display orientation for e.g. tablets */
+#define KEY_DIRECTION		KEY_ROTATE_DISPLAY
+#define KEY_CYCLEWINDOWS	154
+#define KEY_MAIL		155
+#define KEY_BOOKMARKS		156	/* AC Bookmarks */
+#define KEY_COMPUTER		157
+#define KEY_BACK		158	/* AC Back */
+#define KEY_FORWARD		159	/* AC Forward */
+#define KEY_CLOSECD		160
+#define KEY_EJECTCD		161
+#define KEY_EJECTCLOSECD	162
+#define KEY_NEXTSONG		163
+#define KEY_PLAYPAUSE		164
+#define KEY_PREVIOUSSONG	165
+#define KEY_STOPCD		166
+#define KEY_RECORD		167
+#define KEY_REWIND		168
+#define KEY_PHONE		169	/* Media Select Telephone */
+#define KEY_ISO			170
+#define KEY_CONFIG		171	/* AL Consumer Control Configuration */
+#define KEY_HOMEPAGE		172	/* AC Home */
+#define KEY_REFRESH		173	/* AC Refresh */
+#define KEY_EXIT		174	/* AC Exit */
+#define KEY_MOVE		175
+#define KEY_EDIT		176
+#define KEY_SCROLLUP		177
+#define KEY_SCROLLDOWN		178
+#define KEY_KPLEFTPAREN		179
+#define KEY_KPRIGHTPAREN	180
+#define KEY_NEW			181	/* AC New */
+#define KEY_REDO		182	/* AC Redo/Repeat */
+
+#define KEY_F13			183
+#define KEY_F14			184
+#define KEY_F15			185
+#define KEY_F16			186
+#define KEY_F17			187
+#define KEY_F18			188
+#define KEY_F19			189
+#define KEY_F20			190
+#define KEY_F21			191
+#define KEY_F22			192
+#define KEY_F23			193
+#define KEY_F24			194
+
+#define KEY_PLAYCD		200
+#define KEY_PAUSECD		201
+#define KEY_PROG3		202
+#define KEY_PROG4		203
+#define KEY_DASHBOARD		204	/* AL Dashboard */
+#define KEY_SUSPEND		205
+#define KEY_CLOSE		206	/* AC Close */
+#define KEY_PLAY		207
+#define KEY_FASTFORWARD		208
+#define KEY_BASSBOOST		209
+#define KEY_PRINT		210	/* AC Print */
+#define KEY_HP			211
+#define KEY_CAMERA		212
+#define KEY_SOUND		213
+#define KEY_QUESTION		214
+#define KEY_EMAIL		215
+#define KEY_CHAT		216
+#define KEY_SEARCH		217
+#define KEY_CONNECT		218
+#define KEY_FINANCE		219	/* AL Checkbook/Finance */
+#define KEY_SPORT		220
+#define KEY_SHOP		221
+#define KEY_ALTERASE		222
+#define KEY_CANCEL		223	/* AC Cancel */
+#define KEY_BRIGHTNESSDOWN	224
+#define KEY_BRIGHTNESSUP	225
+#define KEY_MEDIA		226
+
+#define KEY_SWITCHVIDEOMODE	227	/* Cycle between available video
+					   outputs (Monitor/LCD/TV-out/etc) */
+#define KEY_KBDILLUMTOGGLE	228
+#define KEY_KBDILLUMDOWN	229
+#define KEY_KBDILLUMUP		230
+
+#define KEY_SEND		231	/* AC Send */
+#define KEY_REPLY		232	/* AC Reply */
+#define KEY_FORWARDMAIL		233	/* AC Forward Msg */
+#define KEY_SAVE		234	/* AC Save */
+#define KEY_DOCUMENTS		235
+
+#define KEY_BATTERY		236
+
+#define KEY_BLUETOOTH		237
+#define KEY_WLAN		238
+#define KEY_UWB			239
+
+#define KEY_UNKNOWN		240
+
+#define KEY_VIDEO_NEXT		241	/* drive next video source */
+#define KEY_VIDEO_PREV		242	/* drive previous video source */
+#define KEY_BRIGHTNESS_CYCLE	243	/* brightness up, after max is min */
+#define KEY_BRIGHTNESS_AUTO	244	/* Set Auto Brightness: manual
+					  brightness control is off,
+					  rely on ambient */
+#define KEY_BRIGHTNESS_ZERO	KEY_BRIGHTNESS_AUTO
+#define KEY_DISPLAY_OFF		245	/* display device to off state */
+
+#define KEY_WWAN		246	/* Wireless WAN (LTE, UMTS, GSM, etc.) */
+#define KEY_WIMAX		KEY_WWAN
+#define KEY_RFKILL		247	/* Key that controls all radios */
+
+#define KEY_MICMUTE		248	/* Mute / unmute the microphone */
+
+/* Code 255 is reserved for special needs of AT keyboard driver */
+
+#define BTN_MISC		0x100
+#define BTN_0			0x100
+#define BTN_1			0x101
+#define BTN_2			0x102
+#define BTN_3			0x103
+#define BTN_4			0x104
+#define BTN_5			0x105
+#define BTN_6			0x106
+#define BTN_7			0x107
+#define BTN_8			0x108
+#define BTN_9			0x109
+
+#define BTN_MOUSE		0x110
+#define BTN_LEFT		0x110
+#define BTN_RIGHT		0x111
+#define BTN_MIDDLE		0x112
+#define BTN_SIDE		0x113
+#define BTN_EXTRA		0x114
+#define BTN_FORWARD		0x115
+#define BTN_BACK		0x116
+#define BTN_TASK		0x117
+
+#define BTN_JOYSTICK		0x120
+#define BTN_TRIGGER		0x120
+#define BTN_THUMB		0x121
+#define BTN_THUMB2		0x122
+#define BTN_TOP			0x123
+#define BTN_TOP2		0x124
+#define BTN_PINKIE		0x125
+#define BTN_BASE		0x126
+#define BTN_BASE2		0x127
+#define BTN_BASE3		0x128
+#define BTN_BASE4		0x129
+#define BTN_BASE5		0x12a
+#define BTN_BASE6		0x12b
+#define BTN_DEAD		0x12f
+
+#define BTN_GAMEPAD		0x130
+#define BTN_SOUTH		0x130
+#define BTN_A			BTN_SOUTH
+#define BTN_EAST		0x131
+#define BTN_B			BTN_EAST
+#define BTN_C			0x132
+#define BTN_NORTH		0x133
+#define BTN_X			BTN_NORTH
+#define BTN_WEST		0x134
+#define BTN_Y			BTN_WEST
+#define BTN_Z			0x135
+#define BTN_TL			0x136
+#define BTN_TR			0x137
+#define BTN_TL2			0x138
+#define BTN_TR2			0x139
+#define BTN_SELECT		0x13a
+#define BTN_START		0x13b
+#define BTN_MODE		0x13c
+#define BTN_THUMBL		0x13d
+#define BTN_THUMBR		0x13e
+
+#define BTN_DIGI		0x140
+#define BTN_TOOL_PEN		0x140
+#define BTN_TOOL_RUBBER		0x141
+#define BTN_TOOL_BRUSH		0x142
+#define BTN_TOOL_PENCIL		0x143
+#define BTN_TOOL_AIRBRUSH	0x144
+#define BTN_TOOL_FINGER		0x145
+#define BTN_TOOL_MOUSE		0x146
+#define BTN_TOOL_LENS		0x147
+#define BTN_TOOL_QUINTTAP	0x148	/* Five fingers on trackpad */
+#define BTN_TOUCH		0x14a
+#define BTN_STYLUS		0x14b
+#define BTN_STYLUS2		0x14c
+#define BTN_TOOL_DOUBLETAP	0x14d
+#define BTN_TOOL_TRIPLETAP	0x14e
+#define BTN_TOOL_QUADTAP	0x14f	/* Four fingers on trackpad */
+
+#define BTN_WHEEL		0x150
+#define BTN_GEAR_DOWN		0x150
+#define BTN_GEAR_UP		0x151
+
+#define KEY_OK			0x160
+#define KEY_SELECT		0x161
+#define KEY_GOTO		0x162
+#define KEY_CLEAR		0x163
+#define KEY_POWER2		0x164
+#define KEY_OPTION		0x165
+#define KEY_INFO		0x166	/* AL OEM Features/Tips/Tutorial */
+#define KEY_TIME		0x167
+#define KEY_VENDOR		0x168
+#define KEY_ARCHIVE		0x169
+#define KEY_PROGRAM		0x16a	/* Media Select Program Guide */
+#define KEY_CHANNEL		0x16b
+#define KEY_FAVORITES		0x16c
+#define KEY_EPG			0x16d
+#define KEY_PVR			0x16e	/* Media Select Home */
+#define KEY_MHP			0x16f
+#define KEY_LANGUAGE		0x170
+#define KEY_TITLE		0x171
+#define KEY_SUBTITLE		0x172
+#define KEY_ANGLE		0x173
+#define KEY_ZOOM		0x174
+#define KEY_MODE		0x175
+#define KEY_KEYBOARD		0x176
+#define KEY_SCREEN		0x177
+#define KEY_PC			0x178	/* Media Select Computer */
+#define KEY_TV			0x179	/* Media Select TV */
+#define KEY_TV2			0x17a	/* Media Select Cable */
+#define KEY_VCR			0x17b	/* Media Select VCR */
+#define KEY_VCR2		0x17c	/* VCR Plus */
+#define KEY_SAT			0x17d	/* Media Select Satellite */
+#define KEY_SAT2		0x17e
+#define KEY_CD			0x17f	/* Media Select CD */
+#define KEY_TAPE		0x180	/* Media Select Tape */
+#define KEY_RADIO		0x181
+#define KEY_TUNER		0x182	/* Media Select Tuner */
+#define KEY_PLAYER		0x183
+#define KEY_TEXT		0x184
+#define KEY_DVD			0x185	/* Media Select DVD */
+#define KEY_AUX			0x186
+#define KEY_MP3			0x187
+#define KEY_AUDIO		0x188	/* AL Audio Browser */
+#define KEY_VIDEO		0x189	/* AL Movie Browser */
+#define KEY_DIRECTORY		0x18a
+#define KEY_LIST		0x18b
+#define KEY_MEMO		0x18c	/* Media Select Messages */
+#define KEY_CALENDAR		0x18d
+#define KEY_RED			0x18e
+#define KEY_GREEN		0x18f
+#define KEY_YELLOW		0x190
+#define KEY_BLUE		0x191
+#define KEY_CHANNELUP		0x192	/* Channel Increment */
+#define KEY_CHANNELDOWN		0x193	/* Channel Decrement */
+#define KEY_FIRST		0x194
+#define KEY_LAST		0x195	/* Recall Last */
+#define KEY_AB			0x196
+#define KEY_NEXT		0x197
+#define KEY_RESTART		0x198
+#define KEY_SLOW		0x199
+#define KEY_SHUFFLE		0x19a
+#define KEY_BREAK		0x19b
+#define KEY_PREVIOUS		0x19c
+#define KEY_DIGITS		0x19d
+#define KEY_TEEN		0x19e
+#define KEY_TWEN		0x19f
+#define KEY_VIDEOPHONE		0x1a0	/* Media Select Video Phone */
+#define KEY_GAMES		0x1a1	/* Media Select Games */
+#define KEY_ZOOMIN		0x1a2	/* AC Zoom In */
+#define KEY_ZOOMOUT		0x1a3	/* AC Zoom Out */
+#define KEY_ZOOMRESET		0x1a4	/* AC Zoom */
+#define KEY_WORDPROCESSOR	0x1a5	/* AL Word Processor */
+#define KEY_EDITOR		0x1a6	/* AL Text Editor */
+#define KEY_SPREADSHEET		0x1a7	/* AL Spreadsheet */
+#define KEY_GRAPHICSEDITOR	0x1a8	/* AL Graphics Editor */
+#define KEY_PRESENTATION	0x1a9	/* AL Presentation App */
+#define KEY_DATABASE		0x1aa	/* AL Database App */
+#define KEY_NEWS		0x1ab	/* AL Newsreader */
+#define KEY_VOICEMAIL		0x1ac	/* AL Voicemail */
+#define KEY_ADDRESSBOOK		0x1ad	/* AL Contacts/Address Book */
+#define KEY_MESSENGER		0x1ae	/* AL Instant Messaging */
+#define KEY_DISPLAYTOGGLE	0x1af	/* Turn display (LCD) on and off */
+#define KEY_BRIGHTNESS_TOGGLE	KEY_DISPLAYTOGGLE
+#define KEY_SPELLCHECK		0x1b0   /* AL Spell Check */
+#define KEY_LOGOFF		0x1b1   /* AL Logoff */
+
+#define KEY_DOLLAR		0x1b2
+#define KEY_EURO		0x1b3
+
+#define KEY_FRAMEBACK		0x1b4	/* Consumer - transport controls */
+#define KEY_FRAMEFORWARD	0x1b5
+#define KEY_CONTEXT_MENU	0x1b6	/* GenDesc - system context menu */
+#define KEY_MEDIA_REPEAT	0x1b7	/* Consumer - transport control */
+#define KEY_10CHANNELSUP	0x1b8	/* 10 channels up (10+) */
+#define KEY_10CHANNELSDOWN	0x1b9	/* 10 channels down (10-) */
+#define KEY_IMAGES		0x1ba	/* AL Image Browser */
+
+#define KEY_DEL_EOL		0x1c0
+#define KEY_DEL_EOS		0x1c1
+#define KEY_INS_LINE		0x1c2
+#define KEY_DEL_LINE		0x1c3
+
+#define KEY_FN			0x1d0
+#define KEY_FN_ESC		0x1d1
+#define KEY_FN_F1		0x1d2
+#define KEY_FN_F2		0x1d3
+#define KEY_FN_F3		0x1d4
+#define KEY_FN_F4		0x1d5
+#define KEY_FN_F5		0x1d6
+#define KEY_FN_F6		0x1d7
+#define KEY_FN_F7		0x1d8
+#define KEY_FN_F8		0x1d9
+#define KEY_FN_F9		0x1da
+#define KEY_FN_F10		0x1db
+#define KEY_FN_F11		0x1dc
+#define KEY_FN_F12		0x1dd
+#define KEY_FN_1		0x1de
+#define KEY_FN_2		0x1df
+#define KEY_FN_D		0x1e0
+#define KEY_FN_E		0x1e1
+#define KEY_FN_F		0x1e2
+#define KEY_FN_S		0x1e3
+#define KEY_FN_B		0x1e4
+
+#define KEY_BRL_DOT1		0x1f1
+#define KEY_BRL_DOT2		0x1f2
+#define KEY_BRL_DOT3		0x1f3
+#define KEY_BRL_DOT4		0x1f4
+#define KEY_BRL_DOT5		0x1f5
+#define KEY_BRL_DOT6		0x1f6
+#define KEY_BRL_DOT7		0x1f7
+#define KEY_BRL_DOT8		0x1f8
+#define KEY_BRL_DOT9		0x1f9
+#define KEY_BRL_DOT10		0x1fa
+
+#define KEY_NUMERIC_0		0x200	/* used by phones, remote controls, */
+#define KEY_NUMERIC_1		0x201	/* and other keypads */
+#define KEY_NUMERIC_2		0x202
+#define KEY_NUMERIC_3		0x203
+#define KEY_NUMERIC_4		0x204
+#define KEY_NUMERIC_5		0x205
+#define KEY_NUMERIC_6		0x206
+#define KEY_NUMERIC_7		0x207
+#define KEY_NUMERIC_8		0x208
+#define KEY_NUMERIC_9		0x209
+#define KEY_NUMERIC_STAR	0x20a
+#define KEY_NUMERIC_POUND	0x20b
+#define KEY_NUMERIC_A		0x20c	/* Phone key A - HUT Telephony 0xb9 */
+#define KEY_NUMERIC_B		0x20d
+#define KEY_NUMERIC_C		0x20e
+#define KEY_NUMERIC_D		0x20f
+
+#define KEY_CAMERA_FOCUS	0x210
+#define KEY_WPS_BUTTON		0x211	/* WiFi Protected Setup key */
+
+#define KEY_TOUCHPAD_TOGGLE	0x212	/* Request switch touchpad on or off */
+#define KEY_TOUCHPAD_ON		0x213
+#define KEY_TOUCHPAD_OFF	0x214
+
+#define KEY_CAMERA_ZOOMIN	0x215
+#define KEY_CAMERA_ZOOMOUT	0x216
+#define KEY_CAMERA_UP		0x217
+#define KEY_CAMERA_DOWN		0x218
+#define KEY_CAMERA_LEFT		0x219
+#define KEY_CAMERA_RIGHT	0x21a
+
+#define KEY_ATTENDANT_ON	0x21b
+#define KEY_ATTENDANT_OFF	0x21c
+#define KEY_ATTENDANT_TOGGLE	0x21d	/* Attendant call on or off */
+#define KEY_LIGHTS_TOGGLE	0x21e	/* Reading light on or off */
+
+#define BTN_DPAD_UP		0x220
+#define BTN_DPAD_DOWN		0x221
+#define BTN_DPAD_LEFT		0x222
+#define BTN_DPAD_RIGHT		0x223
+
+#define KEY_ALS_TOGGLE		0x230	/* Ambient light sensor */
+
+#define KEY_BUTTONCONFIG		0x240	/* AL Button Configuration */
+#define KEY_TASKMANAGER		0x241	/* AL Task/Project Manager */
+#define KEY_JOURNAL		0x242	/* AL Log/Journal/Timecard */
+#define KEY_CONTROLPANEL		0x243	/* AL Control Panel */
+#define KEY_APPSELECT		0x244	/* AL Select Task/Application */
+#define KEY_SCREENSAVER		0x245	/* AL Screen Saver */
+#define KEY_VOICECOMMAND		0x246	/* Listening Voice Command */
+
+#define KEY_BRIGHTNESS_MIN		0x250	/* Set Brightness to Minimum */
+#define KEY_BRIGHTNESS_MAX		0x251	/* Set Brightness to Maximum */
+
+#define KEY_KBDINPUTASSIST_PREV		0x260
+#define KEY_KBDINPUTASSIST_NEXT		0x261
+#define KEY_KBDINPUTASSIST_PREVGROUP		0x262
+#define KEY_KBDINPUTASSIST_NEXTGROUP		0x263
+#define KEY_KBDINPUTASSIST_ACCEPT		0x264
+#define KEY_KBDINPUTASSIST_CANCEL		0x265
+
+#define BTN_TRIGGER_HAPPY		0x2c0
+#define BTN_TRIGGER_HAPPY1		0x2c0
+#define BTN_TRIGGER_HAPPY2		0x2c1
+#define BTN_TRIGGER_HAPPY3		0x2c2
+#define BTN_TRIGGER_HAPPY4		0x2c3
+#define BTN_TRIGGER_HAPPY5		0x2c4
+#define BTN_TRIGGER_HAPPY6		0x2c5
+#define BTN_TRIGGER_HAPPY7		0x2c6
+#define BTN_TRIGGER_HAPPY8		0x2c7
+#define BTN_TRIGGER_HAPPY9		0x2c8
+#define BTN_TRIGGER_HAPPY10		0x2c9
+#define BTN_TRIGGER_HAPPY11		0x2ca
+#define BTN_TRIGGER_HAPPY12		0x2cb
+#define BTN_TRIGGER_HAPPY13		0x2cc
+#define BTN_TRIGGER_HAPPY14		0x2cd
+#define BTN_TRIGGER_HAPPY15		0x2ce
+#define BTN_TRIGGER_HAPPY16		0x2cf
+#define BTN_TRIGGER_HAPPY17		0x2d0
+#define BTN_TRIGGER_HAPPY18		0x2d1
+#define BTN_TRIGGER_HAPPY19		0x2d2
+#define BTN_TRIGGER_HAPPY20		0x2d3
+#define BTN_TRIGGER_HAPPY21		0x2d4
+#define BTN_TRIGGER_HAPPY22		0x2d5
+#define BTN_TRIGGER_HAPPY23		0x2d6
+#define BTN_TRIGGER_HAPPY24		0x2d7
+#define BTN_TRIGGER_HAPPY25		0x2d8
+#define BTN_TRIGGER_HAPPY26		0x2d9
+#define BTN_TRIGGER_HAPPY27		0x2da
+#define BTN_TRIGGER_HAPPY28		0x2db
+#define BTN_TRIGGER_HAPPY29		0x2dc
+#define BTN_TRIGGER_HAPPY30		0x2dd
+#define BTN_TRIGGER_HAPPY31		0x2de
+#define BTN_TRIGGER_HAPPY32		0x2df
+#define BTN_TRIGGER_HAPPY33		0x2e0
+#define BTN_TRIGGER_HAPPY34		0x2e1
+#define BTN_TRIGGER_HAPPY35		0x2e2
+#define BTN_TRIGGER_HAPPY36		0x2e3
+#define BTN_TRIGGER_HAPPY37		0x2e4
+#define BTN_TRIGGER_HAPPY38		0x2e5
+#define BTN_TRIGGER_HAPPY39		0x2e6
+#define BTN_TRIGGER_HAPPY40		0x2e7
+
+/* We avoid low common keys in module aliases so they don't get huge. */
+#define KEY_MIN_INTERESTING	KEY_MUTE
+#define KEY_MAX			0x2ff
+#define KEY_CNT			(KEY_MAX+1)
+
+/*
+ * Relative axes
+ */
+
+#define REL_X			0x00
+#define REL_Y			0x01
+#define REL_Z			0x02
+#define REL_RX			0x03
+#define REL_RY			0x04
+#define REL_RZ			0x05
+#define REL_HWHEEL		0x06
+#define REL_DIAL		0x07
+#define REL_WHEEL		0x08
+#define REL_MISC		0x09
+#define REL_MAX			0x0f
+#define REL_CNT			(REL_MAX+1)
+
+/*
+ * Absolute axes
+ */
+
+#define ABS_X			0x00
+#define ABS_Y			0x01
+#define ABS_Z			0x02
+#define ABS_RX			0x03
+#define ABS_RY			0x04
+#define ABS_RZ			0x05
+#define ABS_THROTTLE		0x06
+#define ABS_RUDDER		0x07
+#define ABS_WHEEL		0x08
+#define ABS_GAS			0x09
+#define ABS_BRAKE		0x0a
+#define ABS_HAT0X		0x10
+#define ABS_HAT0Y		0x11
+#define ABS_HAT1X		0x12
+#define ABS_HAT1Y		0x13
+#define ABS_HAT2X		0x14
+#define ABS_HAT2Y		0x15
+#define ABS_HAT3X		0x16
+#define ABS_HAT3Y		0x17
+#define ABS_PRESSURE		0x18
+#define ABS_DISTANCE		0x19
+#define ABS_TILT_X		0x1a
+#define ABS_TILT_Y		0x1b
+#define ABS_TOOL_WIDTH		0x1c
+
+#define ABS_VOLUME		0x20
+
+#define ABS_MISC		0x28
+
+#define ABS_MT_SLOT		0x2f	/* MT slot being modified */
+#define ABS_MT_TOUCH_MAJOR	0x30	/* Major axis of touching ellipse */
+#define ABS_MT_TOUCH_MINOR	0x31	/* Minor axis (omit if circular) */
+#define ABS_MT_WIDTH_MAJOR	0x32	/* Major axis of approaching ellipse */
+#define ABS_MT_WIDTH_MINOR	0x33	/* Minor axis (omit if circular) */
+#define ABS_MT_ORIENTATION	0x34	/* Ellipse orientation */
+#define ABS_MT_POSITION_X	0x35	/* Center X touch position */
+#define ABS_MT_POSITION_Y	0x36	/* Center Y touch position */
+#define ABS_MT_TOOL_TYPE	0x37	/* Type of touching device */
+#define ABS_MT_BLOB_ID		0x38	/* Group a set of packets as a blob */
+#define ABS_MT_TRACKING_ID	0x39	/* Unique ID of initiated contact */
+#define ABS_MT_PRESSURE		0x3a	/* Pressure on contact area */
+#define ABS_MT_DISTANCE		0x3b	/* Contact hover distance */
+#define ABS_MT_TOOL_X		0x3c	/* Center X tool position */
+#define ABS_MT_TOOL_Y		0x3d	/* Center Y tool position */
+
+
+#define ABS_MAX			0x3f
+#define ABS_CNT			(ABS_MAX+1)
+
+/*
+ * Switch events
+ */
+
+#define SW_LID			0x00  /* set = lid shut */
+#define SW_TABLET_MODE		0x01  /* set = tablet mode */
+#define SW_HEADPHONE_INSERT	0x02  /* set = inserted */
+#define SW_RFKILL_ALL		0x03  /* rfkill master switch, type "any"
+					 set = radio enabled */
+#define SW_RADIO		SW_RFKILL_ALL	/* deprecated */
+#define SW_MICROPHONE_INSERT	0x04  /* set = inserted */
+#define SW_DOCK			0x05  /* set = plugged into dock */
+#define SW_LINEOUT_INSERT	0x06  /* set = inserted */
+#define SW_JACK_PHYSICAL_INSERT 0x07  /* set = mechanical switch set */
+#define SW_VIDEOOUT_INSERT	0x08  /* set = inserted */
+#define SW_CAMERA_LENS_COVER	0x09  /* set = lens covered */
+#define SW_KEYPAD_SLIDE		0x0a  /* set = keypad slide out */
+#define SW_FRONT_PROXIMITY	0x0b  /* set = front proximity sensor active */
+#define SW_ROTATE_LOCK		0x0c  /* set = rotate locked/disabled */
+#define SW_LINEIN_INSERT	0x0d  /* set = inserted */
+#define SW_MUTE_DEVICE		0x0e  /* set = device disabled */
+#define SW_MAX_			0x0f
+#define SW_CNT			(SW_MAX_+1)
+
+/*
+ * Misc events
+ */
+
+#define MSC_SERIAL		0x00
+#define MSC_PULSELED		0x01
+#define MSC_GESTURE		0x02
+#define MSC_RAW			0x03
+#define MSC_SCAN		0x04
+#define MSC_TIMESTAMP		0x05
+#define MSC_MAX			0x07
+#define MSC_CNT			(MSC_MAX+1)
+
+/*
+ * LEDs
+ */
+
+#define LED_NUML		0x00
+#define LED_CAPSL		0x01
+#define LED_SCROLLL		0x02
+#define LED_COMPOSE		0x03
+#define LED_KANA		0x04
+#define LED_SLEEP		0x05
+#define LED_SUSPEND		0x06
+#define LED_MUTE		0x07
+#define LED_MISC		0x08
+#define LED_MAIL		0x09
+#define LED_CHARGING		0x0a
+#define LED_MAX			0x0f
+#define LED_CNT			(LED_MAX+1)
+
+/*
+ * Autorepeat values
+ */
+
+#define REP_DELAY		0x00
+#define REP_PERIOD		0x01
+#define REP_MAX			0x01
+#define REP_CNT			(REP_MAX+1)
+
+/*
+ * Sounds
+ */
+
+#define SND_CLICK		0x00
+#define SND_BELL		0x01
+#define SND_TONE		0x02
+#define SND_MAX			0x07
+#define SND_CNT			(SND_MAX+1)
+
+#endif
diff --git a/include/standard-headers/linux/input.h b/include/standard-headers/linux/input.h
new file mode 100644
index 0000000..a52b202
--- /dev/null
+++ b/include/standard-headers/linux/input.h
@@ -0,0 +1,486 @@
+/*
+ * Copyright (c) 1999-2002 Vojtech Pavlik
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ */
+#ifndef _INPUT_H
+#define _INPUT_H
+
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include "standard-headers/linux/types.h"
+
+#include "standard-headers/linux/input-event-codes.h"
+
+/*
+ * The event structure itself
+ */
+
+struct input_event {
+	struct timeval time;
+	uint16_t type;
+	uint16_t code;
+	int32_t value;
+};
+
+/*
+ * Protocol version.
+ */
+
+#define EV_VERSION		0x010001
+
+/*
+ * IOCTLs (0x00 - 0x7f)
+ */
+
+struct input_id {
+	uint16_t bustype;
+	uint16_t vendor;
+	uint16_t product;
+	uint16_t version;
+};
+
+/**
+ * struct input_absinfo - used by EVIOCGABS/EVIOCSABS ioctls
+ * @value: latest reported value for the axis.
+ * @minimum: specifies minimum value for the axis.
+ * @maximum: specifies maximum value for the axis.
+ * @fuzz: specifies fuzz value that is used to filter noise from
+ *	the event stream.
+ * @flat: values that are within this value will be discarded by
+ *	joydev interface and reported as 0 instead.
+ * @resolution: specifies resolution for the values reported for
+ *	the axis.
+ *
+ * Note that input core does not clamp reported values to the
+ * [minimum, maximum] limits, such task is left to userspace.
+ *
+ * Resolution for main axes (ABS_X, ABS_Y, ABS_Z) is reported in
+ * units per millimeter (units/mm), resolution for rotational axes
+ * (ABS_RX, ABS_RY, ABS_RZ) is reported in units per radian.
+ */
+struct input_absinfo {
+	int32_t value;
+	int32_t minimum;
+	int32_t maximum;
+	int32_t fuzz;
+	int32_t flat;
+	int32_t resolution;
+};
+
+/**
+ * struct input_keymap_entry - used by EVIOCGKEYCODE/EVIOCSKEYCODE ioctls
+ * @scancode: scancode represented in machine-endian form.
+ * @len: length of the scancode that resides in @scancode buffer.
+ * @index: index in the keymap, may be used instead of scancode
+ * @flags: allows to specify how kernel should handle the request. For
+ *	example, setting INPUT_KEYMAP_BY_INDEX flag indicates that kernel
+ *	should perform lookup in keymap by @index instead of @scancode
+ * @keycode: key code assigned to this scancode
+ *
+ * The structure is used to retrieve and modify keymap data. Users have
+ * option of performing lookup either by @scancode itself or by @index
+ * in keymap entry. EVIOCGKEYCODE will also return scancode or index
+ * (depending on which element was used to perform lookup).
+ */
+struct input_keymap_entry {
+#define INPUT_KEYMAP_BY_INDEX	(1 << 0)
+	uint8_t  flags;
+	uint8_t  len;
+	uint16_t index;
+	uint32_t keycode;
+	uint8_t  scancode[32];
+};
+
+struct input_mask {
+	uint32_t type;
+	uint32_t codes_size;
+	uint64_t codes_ptr;
+};
+
+#define EVIOCGVERSION		_IOR('E', 0x01, int)			/* get driver version */
+#define EVIOCGID		_IOR('E', 0x02, struct input_id)	/* get device ID */
+#define EVIOCGREP		_IOR('E', 0x03, unsigned int[2])	/* get repeat settings */
+#define EVIOCSREP		_IOW('E', 0x03, unsigned int[2])	/* set repeat settings */
+
+#define EVIOCGKEYCODE		_IOR('E', 0x04, unsigned int[2])        /* get keycode */
+#define EVIOCGKEYCODE_V2	_IOR('E', 0x04, struct input_keymap_entry)
+#define EVIOCSKEYCODE		_IOW('E', 0x04, unsigned int[2])        /* set keycode */
+#define EVIOCSKEYCODE_V2	_IOW('E', 0x04, struct input_keymap_entry)
+
+#define EVIOCGNAME(len)		_IOC(_IOC_READ, 'E', 0x06, len)		/* get device name */
+#define EVIOCGPHYS(len)		_IOC(_IOC_READ, 'E', 0x07, len)		/* get physical location */
+#define EVIOCGUNIQ(len)		_IOC(_IOC_READ, 'E', 0x08, len)		/* get unique identifier */
+#define EVIOCGPROP(len)		_IOC(_IOC_READ, 'E', 0x09, len)		/* get device properties */
+
+/**
+ * EVIOCGMTSLOTS(len) - get MT slot values
+ * @len: size of the data buffer in bytes
+ *
+ * The ioctl buffer argument should be binary equivalent to
+ *
+ * struct input_mt_request_layout {
+ *	uint32_t code;
+ *	int32_t values[num_slots];
+ * };
+ *
+ * where num_slots is the (arbitrary) number of MT slots to extract.
+ *
+ * The ioctl size argument (len) is the size of the buffer, which
+ * should satisfy len = (num_slots + 1) * sizeof(int32_t).  If len is
+ * too small to fit all available slots, the first num_slots are
+ * returned.
+ *
+ * Before the call, code is set to the wanted ABS_MT event type. On
+ * return, values[] is filled with the slot values for the specified
+ * ABS_MT code.
+ *
+ * If the request code is not an ABS_MT value, -EINVAL is returned.
+ */
+#define EVIOCGMTSLOTS(len)	_IOC(_IOC_READ, 'E', 0x0a, len)
+
+#define EVIOCGKEY(len)		_IOC(_IOC_READ, 'E', 0x18, len)		/* get global key state */
+#define EVIOCGLED(len)		_IOC(_IOC_READ, 'E', 0x19, len)		/* get all LEDs */
+#define EVIOCGSND(len)		_IOC(_IOC_READ, 'E', 0x1a, len)		/* get all sounds status */
+#define EVIOCGSW(len)		_IOC(_IOC_READ, 'E', 0x1b, len)		/* get all switch states */
+
+#define EVIOCGBIT(ev,len)	_IOC(_IOC_READ, 'E', 0x20 + (ev), len)	/* get event bits */
+#define EVIOCGABS(abs)		_IOR('E', 0x40 + (abs), struct input_absinfo)	/* get abs value/limits */
+#define EVIOCSABS(abs)		_IOW('E', 0xc0 + (abs), struct input_absinfo)	/* set abs value/limits */
+
+#define EVIOCSFF		_IOW('E', 0x80, struct ff_effect)	/* send a force effect to a force feedback device */
+#define EVIOCRMFF		_IOW('E', 0x81, int)			/* Erase a force effect */
+#define EVIOCGEFFECTS		_IOR('E', 0x84, int)			/* Report number of effects playable at the same time */
+
+#define EVIOCGRAB		_IOW('E', 0x90, int)			/* Grab/Release device */
+#define EVIOCREVOKE		_IOW('E', 0x91, int)			/* Revoke device access */
+
+/**
+ * EVIOCGMASK - Retrieve current event mask
+ *
+ * This ioctl allows user to retrieve the current event mask for specific
+ * event type. The argument must be of type "struct input_mask" and
+ * specifies the event type to query, the address of the receive buffer and
+ * the size of the receive buffer.
+ *
+ * The event mask is a per-client mask that specifies which events are
+ * forwarded to the client. Each event code is represented by a single bit
+ * in the event mask. If the bit is set, the event is passed to the client
+ * normally. Otherwise, the event is filtered and will never be queued on
+ * the client's receive buffer.
+ *
+ * Event masks do not affect global state of the input device. They only
+ * affect the file descriptor they are applied to.
+ *
+ * The default event mask for a client has all bits set, i.e. all events
+ * are forwarded to the client. If the kernel is queried for an unknown
+ * event type or if the receive buffer is larger than the number of
+ * event codes known to the kernel, the kernel returns all zeroes for those
+ * codes.
+ *
+ * At maximum, codes_size bytes are copied.
+ *
+ * This ioctl may fail with ENODEV in case the file is revoked, EFAULT
+ * if the receive-buffer points to invalid memory, or EINVAL if the kernel
+ * does not implement the ioctl.
+ */
+#define EVIOCGMASK		_IOR('E', 0x92, struct input_mask)	/* Get event-masks */
+
+/**
+ * EVIOCSMASK - Set event mask
+ *
+ * This ioctl is the counterpart to EVIOCGMASK. Instead of receiving the
+ * current event mask, this changes the client's event mask for a specific
+ * type.  See EVIOCGMASK for a description of event-masks and the
+ * argument-type.
+ *
+ * This ioctl provides full forward compatibility. If the passed event type
+ * is unknown to the kernel, or if the number of event codes specified in
+ * the mask is bigger than what is known to the kernel, the ioctl is still
+ * accepted and applied. However, any unknown codes are left untouched and
+ * stay cleared. That means, the kernel always filters unknown codes
+ * regardless of what the client requests.  If the new mask doesn't cover
+ * all known event-codes, all remaining codes are automatically cleared and
+ * thus filtered.
+ *
+ * This ioctl may fail with ENODEV in case the file is revoked. EFAULT is
+ * returned if the receive-buffer points to invalid memory. EINVAL is returned
+ * if the kernel does not implement the ioctl.
+ */
+#define EVIOCSMASK		_IOW('E', 0x93, struct input_mask)	/* Set event-masks */
+
+#define EVIOCSCLOCKID		_IOW('E', 0xa0, int)			/* Set clockid to be used for timestamps */
+
+/*
+ * IDs.
+ */
+
+#define ID_BUS			0
+#define ID_VENDOR		1
+#define ID_PRODUCT		2
+#define ID_VERSION		3
+
+#define BUS_PCI			0x01
+#define BUS_ISAPNP		0x02
+#define BUS_USB			0x03
+#define BUS_HIL			0x04
+#define BUS_BLUETOOTH		0x05
+#define BUS_VIRTUAL		0x06
+
+#define BUS_ISA			0x10
+#define BUS_I8042		0x11
+#define BUS_XTKBD		0x12
+#define BUS_RS232		0x13
+#define BUS_GAMEPORT		0x14
+#define BUS_PARPORT		0x15
+#define BUS_AMIGA		0x16
+#define BUS_ADB			0x17
+#define BUS_I2C			0x18
+#define BUS_HOST		0x19
+#define BUS_GSC			0x1A
+#define BUS_ATARI		0x1B
+#define BUS_SPI			0x1C
+#define BUS_RMI			0x1D
+
+/*
+ * MT_TOOL types
+ */
+#define MT_TOOL_FINGER		0
+#define MT_TOOL_PEN		1
+#define MT_TOOL_PALM		2
+#define MT_TOOL_MAX		2
+
+/*
+ * Values describing the status of a force-feedback effect
+ */
+#define FF_STATUS_STOPPED	0x00
+#define FF_STATUS_PLAYING	0x01
+#define FF_STATUS_MAX		0x01
+
+/*
+ * Structures used in ioctls to upload effects to a device
+ * They are pieces of a bigger structure (called ff_effect)
+ */
+
+/*
+ * All duration values are expressed in ms. Values above 32767 ms (0x7fff)
+ * should not be used and have unspecified results.
+ */
+
+/**
+ * struct ff_replay - defines scheduling of the force-feedback effect
+ * @length: duration of the effect
+ * @delay: delay before effect should start playing
+ */
+struct ff_replay {
+	uint16_t length;
+	uint16_t delay;
+};
+
+/**
+ * struct ff_trigger - defines what triggers the force-feedback effect
+ * @button: number of the button triggering the effect
+ * @interval: controls how soon the effect can be re-triggered
+ */
+struct ff_trigger {
+	uint16_t button;
+	uint16_t interval;
+};
+
+/**
+ * struct ff_envelope - generic force-feedback effect envelope
+ * @attack_length: duration of the attack (ms)
+ * @attack_level: level at the beginning of the attack
+ * @fade_length: duration of fade (ms)
+ * @fade_level: level at the end of fade
+ *
+ * The @attack_level and @fade_level are absolute values; when applying
+ * envelope force-feedback core will convert to positive/negative
+ * value based on polarity of the default level of the effect.
+ * Valid range for the attack and fade levels is 0x0000 - 0x7fff
+ */
+struct ff_envelope {
+	uint16_t attack_length;
+	uint16_t attack_level;
+	uint16_t fade_length;
+	uint16_t fade_level;
+};
+
+/**
+ * struct ff_constant_effect - defines parameters of a constant force-feedback effect
+ * @level: strength of the effect; may be negative
+ * @envelope: envelope data
+ */
+struct ff_constant_effect {
+	int16_t level;
+	struct ff_envelope envelope;
+};
+
+/**
+ * struct ff_ramp_effect - defines parameters of a ramp force-feedback effect
+ * @start_level: beginning strength of the effect; may be negative
+ * @end_level: final strength of the effect; may be negative
+ * @envelope: envelope data
+ */
+struct ff_ramp_effect {
+	int16_t start_level;
+	int16_t end_level;
+	struct ff_envelope envelope;
+};
+
+/**
+ * struct ff_condition_effect - defines a spring or friction force-feedback effect
+ * @right_saturation: maximum level when joystick moved all way to the right
+ * @left_saturation: same for the left side
+ * @right_coeff: controls how fast the force grows when the joystick moves
+ *	to the right
+ * @left_coeff: same for the left side
+ * @deadband: size of the dead zone, where no force is produced
+ * @center: position of the dead zone
+ */
+struct ff_condition_effect {
+	uint16_t right_saturation;
+	uint16_t left_saturation;
+
+	int16_t right_coeff;
+	int16_t left_coeff;
+
+	uint16_t deadband;
+	int16_t center;
+};
+
+/**
+ * struct ff_periodic_effect - defines parameters of a periodic force-feedback effect
+ * @waveform: kind of the effect (wave)
+ * @period: period of the wave (ms)
+ * @magnitude: peak value
+ * @offset: mean value of the wave (roughly)
+ * @phase: 'horizontal' shift
+ * @envelope: envelope data
+ * @custom_len: number of samples (FF_CUSTOM only)
+ * @custom_data: buffer of samples (FF_CUSTOM only)
+ *
+ * Known waveforms - FF_SQUARE, FF_TRIANGLE, FF_SINE, FF_SAW_UP,
+ * FF_SAW_DOWN, FF_CUSTOM. The exact syntax FF_CUSTOM is undefined
+ * for the time being as no driver supports it yet.
+ *
+ * Note: the data pointed by custom_data is copied by the driver.
+ * You can therefore dispose of the memory after the upload/update.
+ */
+struct ff_periodic_effect {
+	uint16_t waveform;
+	uint16_t period;
+	int16_t magnitude;
+	int16_t offset;
+	uint16_t phase;
+
+	struct ff_envelope envelope;
+
+	uint32_t custom_len;
+	int16_t *custom_data;
+};
+
+/**
+ * struct ff_rumble_effect - defines parameters of a periodic force-feedback effect
+ * @strong_magnitude: magnitude of the heavy motor
+ * @weak_magnitude: magnitude of the light one
+ *
+ * Some rumble pads have two motors of different weight. Strong_magnitude
+ * represents the magnitude of the vibration generated by the heavy one.
+ */
+struct ff_rumble_effect {
+	uint16_t strong_magnitude;
+	uint16_t weak_magnitude;
+};
+
+/**
+ * struct ff_effect - defines force feedback effect
+ * @type: type of the effect (FF_CONSTANT, FF_PERIODIC, FF_RAMP, FF_SPRING,
+ *	FF_FRICTION, FF_DAMPER, FF_RUMBLE, FF_INERTIA, or FF_CUSTOM)
+ * @id: an unique id assigned to an effect
+ * @direction: direction of the effect
+ * @trigger: trigger conditions (struct ff_trigger)
+ * @replay: scheduling of the effect (struct ff_replay)
+ * @u: effect-specific structure (one of ff_constant_effect, ff_ramp_effect,
+ *	ff_periodic_effect, ff_condition_effect, ff_rumble_effect) further
+ *	defining effect parameters
+ *
+ * This structure is sent through ioctl from the application to the driver.
+ * To create a new effect application should set its @id to -1; the kernel
+ * will return assigned @id which can later be used to update or delete
+ * this effect.
+ *
+ * Direction of the effect is encoded as follows:
+ *	0 deg -> 0x0000 (down)
+ *	90 deg -> 0x4000 (left)
+ *	180 deg -> 0x8000 (up)
+ *	270 deg -> 0xC000 (right)
+ */
+struct ff_effect {
+	uint16_t type;
+	int16_t id;
+	uint16_t direction;
+	struct ff_trigger trigger;
+	struct ff_replay replay;
+
+	union {
+		struct ff_constant_effect constant;
+		struct ff_ramp_effect ramp;
+		struct ff_periodic_effect periodic;
+		struct ff_condition_effect condition[2]; /* One for each axis */
+		struct ff_rumble_effect rumble;
+	} u;
+};
+
+/*
+ * Force feedback effect types
+ */
+
+#define FF_RUMBLE	0x50
+#define FF_PERIODIC	0x51
+#define FF_CONSTANT	0x52
+#define FF_SPRING	0x53
+#define FF_FRICTION	0x54
+#define FF_DAMPER	0x55
+#define FF_INERTIA	0x56
+#define FF_RAMP		0x57
+
+#define FF_EFFECT_MIN	FF_RUMBLE
+#define FF_EFFECT_MAX	FF_RAMP
+
+/*
+ * Force feedback periodic effect types
+ */
+
+#define FF_SQUARE	0x58
+#define FF_TRIANGLE	0x59
+#define FF_SINE		0x5a
+#define FF_SAW_UP	0x5b
+#define FF_SAW_DOWN	0x5c
+#define FF_CUSTOM	0x5d
+
+#define FF_WAVEFORM_MIN	FF_SQUARE
+#define FF_WAVEFORM_MAX	FF_CUSTOM
+
+/*
+ * Set ff device properties
+ */
+
+#define FF_GAIN		0x60
+#define FF_AUTOCENTER	0x61
+
+/*
+ * ff->playback(effect_id = FF_GAIN) is the first effect_id to
+ * cause a collision with another ff method, in this case ff->set_gain().
+ * Therefore the greatest safe value for effect_id is FF_GAIN - 1,
+ * and thus the total number of effects should never exceed FF_GAIN.
+ */
+#define FF_MAX_EFFECTS	FF_GAIN
+
+#define FF_MAX		0x7f
+#define FF_CNT		(FF_MAX+1)
+
+#endif /* _INPUT_H */
diff --git a/include/standard-headers/linux/pci_regs.h b/include/standard-headers/linux/pci_regs.h
new file mode 100644
index 0000000..4040951
--- /dev/null
+++ b/include/standard-headers/linux/pci_regs.h
@@ -0,0 +1,967 @@
+/*
+ *	pci_regs.h
+ *
+ *	PCI standard defines
+ *	Copyright 1994, Drew Eckhardt
+ *	Copyright 1997--1999 Martin Mares <mj@ucw.cz>
+ *
+ *	For more information, please consult the following manuals (look at
+ *	http://www.pcisig.com/ for how to get them):
+ *
+ *	PCI BIOS Specification
+ *	PCI Local Bus Specification
+ *	PCI to PCI Bridge Specification
+ *	PCI System Design Guide
+ *
+ *	For HyperTransport information, please consult the following manuals
+ *	from http://www.hypertransport.org
+ *
+ *	The HyperTransport I/O Link Specification
+ */
+
+#ifndef LINUX_PCI_REGS_H
+#define LINUX_PCI_REGS_H
+
+/*
+ * Under PCI, each device has 256 bytes of configuration address space,
+ * of which the first 64 bytes are standardized as follows:
+ */
+#define PCI_STD_HEADER_SIZEOF	64
+#define PCI_VENDOR_ID		0x00	/* 16 bits */
+#define PCI_DEVICE_ID		0x02	/* 16 bits */
+#define PCI_COMMAND		0x04	/* 16 bits */
+#define  PCI_COMMAND_IO		0x1	/* Enable response in I/O space */
+#define  PCI_COMMAND_MEMORY	0x2	/* Enable response in Memory space */
+#define  PCI_COMMAND_MASTER	0x4	/* Enable bus mastering */
+#define  PCI_COMMAND_SPECIAL	0x8	/* Enable response to special cycles */
+#define  PCI_COMMAND_INVALIDATE	0x10	/* Use memory write and invalidate */
+#define  PCI_COMMAND_VGA_PALETTE 0x20	/* Enable palette snooping */
+#define  PCI_COMMAND_PARITY	0x40	/* Enable parity checking */
+#define  PCI_COMMAND_WAIT	0x80	/* Enable address/data stepping */
+#define  PCI_COMMAND_SERR	0x100	/* Enable SERR */
+#define  PCI_COMMAND_FAST_BACK	0x200	/* Enable back-to-back writes */
+#define  PCI_COMMAND_INTX_DISABLE 0x400 /* INTx Emulation Disable */
+
+#define PCI_STATUS		0x06	/* 16 bits */
+#define  PCI_STATUS_INTERRUPT	0x08	/* Interrupt status */
+#define  PCI_STATUS_CAP_LIST	0x10	/* Support Capability List */
+#define  PCI_STATUS_66MHZ	0x20	/* Support 66 MHz PCI 2.1 bus */
+#define  PCI_STATUS_UDF		0x40	/* Support User Definable Features [obsolete] */
+#define  PCI_STATUS_FAST_BACK	0x80	/* Accept fast-back to back */
+#define  PCI_STATUS_PARITY	0x100	/* Detected parity error */
+#define  PCI_STATUS_DEVSEL_MASK	0x600	/* DEVSEL timing */
+#define  PCI_STATUS_DEVSEL_FAST		0x000
+#define  PCI_STATUS_DEVSEL_MEDIUM	0x200
+#define  PCI_STATUS_DEVSEL_SLOW		0x400
+#define  PCI_STATUS_SIG_TARGET_ABORT	0x800 /* Set on target abort */
+#define  PCI_STATUS_REC_TARGET_ABORT	0x1000 /* Master ack of " */
+#define  PCI_STATUS_REC_MASTER_ABORT	0x2000 /* Set on master abort */
+#define  PCI_STATUS_SIG_SYSTEM_ERROR	0x4000 /* Set when we drive SERR */
+#define  PCI_STATUS_DETECTED_PARITY	0x8000 /* Set on parity error */
+
+#define PCI_CLASS_REVISION	0x08	/* High 24 bits are class, low 8 revision */
+#define PCI_REVISION_ID		0x08	/* Revision ID */
+#define PCI_CLASS_PROG		0x09	/* Reg. Level Programming Interface */
+#define PCI_CLASS_DEVICE	0x0a	/* Device class */
+
+#define PCI_CACHE_LINE_SIZE	0x0c	/* 8 bits */
+#define PCI_LATENCY_TIMER	0x0d	/* 8 bits */
+#define PCI_HEADER_TYPE		0x0e	/* 8 bits */
+#define  PCI_HEADER_TYPE_NORMAL		0
+#define  PCI_HEADER_TYPE_BRIDGE		1
+#define  PCI_HEADER_TYPE_CARDBUS	2
+
+#define PCI_BIST		0x0f	/* 8 bits */
+#define  PCI_BIST_CODE_MASK	0x0f	/* Return result */
+#define  PCI_BIST_START		0x40	/* 1 to start BIST, 2 secs or less */
+#define  PCI_BIST_CAPABLE	0x80	/* 1 if BIST capable */
+
+/*
+ * Base addresses specify locations in memory or I/O space.
+ * Decoded size can be determined by writing a value of
+ * 0xffffffff to the register, and reading it back.  Only
+ * 1 bits are decoded.
+ */
+#define PCI_BASE_ADDRESS_0	0x10	/* 32 bits */
+#define PCI_BASE_ADDRESS_1	0x14	/* 32 bits [htype 0,1 only] */
+#define PCI_BASE_ADDRESS_2	0x18	/* 32 bits [htype 0 only] */
+#define PCI_BASE_ADDRESS_3	0x1c	/* 32 bits */
+#define PCI_BASE_ADDRESS_4	0x20	/* 32 bits */
+#define PCI_BASE_ADDRESS_5	0x24	/* 32 bits */
+#define  PCI_BASE_ADDRESS_SPACE		0x01	/* 0 = memory, 1 = I/O */
+#define  PCI_BASE_ADDRESS_SPACE_IO	0x01
+#define  PCI_BASE_ADDRESS_SPACE_MEMORY	0x00
+#define  PCI_BASE_ADDRESS_MEM_TYPE_MASK	0x06
+#define  PCI_BASE_ADDRESS_MEM_TYPE_32	0x00	/* 32 bit address */
+#define  PCI_BASE_ADDRESS_MEM_TYPE_1M	0x02	/* Below 1M [obsolete] */
+#define  PCI_BASE_ADDRESS_MEM_TYPE_64	0x04	/* 64 bit address */
+#define  PCI_BASE_ADDRESS_MEM_PREFETCH	0x08	/* prefetchable? */
+#define  PCI_BASE_ADDRESS_MEM_MASK	(~0x0fUL)
+#define  PCI_BASE_ADDRESS_IO_MASK	(~0x03UL)
+/* bit 1 is reserved if address_space = 1 */
+
+/* Header type 0 (normal devices) */
+#define PCI_CARDBUS_CIS		0x28
+#define PCI_SUBSYSTEM_VENDOR_ID	0x2c
+#define PCI_SUBSYSTEM_ID	0x2e
+#define PCI_ROM_ADDRESS		0x30	/* Bits 31..11 are address, 10..1 reserved */
+#define  PCI_ROM_ADDRESS_ENABLE	0x01
+#define PCI_ROM_ADDRESS_MASK	(~0x7ffUL)
+
+#define PCI_CAPABILITY_LIST	0x34	/* Offset of first capability list entry */
+
+/* 0x35-0x3b are reserved */
+#define PCI_INTERRUPT_LINE	0x3c	/* 8 bits */
+#define PCI_INTERRUPT_PIN	0x3d	/* 8 bits */
+#define PCI_MIN_GNT		0x3e	/* 8 bits */
+#define PCI_MAX_LAT		0x3f	/* 8 bits */
+
+/* Header type 1 (PCI-to-PCI bridges) */
+#define PCI_PRIMARY_BUS		0x18	/* Primary bus number */
+#define PCI_SECONDARY_BUS	0x19	/* Secondary bus number */
+#define PCI_SUBORDINATE_BUS	0x1a	/* Highest bus number behind the bridge */
+#define PCI_SEC_LATENCY_TIMER	0x1b	/* Latency timer for secondary interface */
+#define PCI_IO_BASE		0x1c	/* I/O range behind the bridge */
+#define PCI_IO_LIMIT		0x1d
+#define  PCI_IO_RANGE_TYPE_MASK	0x0fUL	/* I/O bridging type */
+#define  PCI_IO_RANGE_TYPE_16	0x00
+#define  PCI_IO_RANGE_TYPE_32	0x01
+#define  PCI_IO_RANGE_MASK	(~0x0fUL) /* Standard 4K I/O windows */
+#define  PCI_IO_1K_RANGE_MASK	(~0x03UL) /* Intel 1K I/O windows */
+#define PCI_SEC_STATUS		0x1e	/* Secondary status register, only bit 14 used */
+#define PCI_MEMORY_BASE		0x20	/* Memory range behind */
+#define PCI_MEMORY_LIMIT	0x22
+#define  PCI_MEMORY_RANGE_TYPE_MASK 0x0fUL
+#define  PCI_MEMORY_RANGE_MASK	(~0x0fUL)
+#define PCI_PREF_MEMORY_BASE	0x24	/* Prefetchable memory range behind */
+#define PCI_PREF_MEMORY_LIMIT	0x26
+#define  PCI_PREF_RANGE_TYPE_MASK 0x0fUL
+#define  PCI_PREF_RANGE_TYPE_32	0x00
+#define  PCI_PREF_RANGE_TYPE_64	0x01
+#define  PCI_PREF_RANGE_MASK	(~0x0fUL)
+#define PCI_PREF_BASE_UPPER32	0x28	/* Upper half of prefetchable memory range */
+#define PCI_PREF_LIMIT_UPPER32	0x2c
+#define PCI_IO_BASE_UPPER16	0x30	/* Upper half of I/O addresses */
+#define PCI_IO_LIMIT_UPPER16	0x32
+/* 0x34 same as for htype 0 */
+/* 0x35-0x3b is reserved */
+#define PCI_ROM_ADDRESS1	0x38	/* Same as PCI_ROM_ADDRESS, but for htype 1 */
+/* 0x3c-0x3d are same as for htype 0 */
+#define PCI_BRIDGE_CONTROL	0x3e
+#define  PCI_BRIDGE_CTL_PARITY	0x01	/* Enable parity detection on secondary interface */
+#define  PCI_BRIDGE_CTL_SERR	0x02	/* The same for SERR forwarding */
+#define  PCI_BRIDGE_CTL_ISA	0x04	/* Enable ISA mode */
+#define  PCI_BRIDGE_CTL_VGA	0x08	/* Forward VGA addresses */
+#define  PCI_BRIDGE_CTL_MASTER_ABORT	0x20  /* Report master aborts */
+#define  PCI_BRIDGE_CTL_BUS_RESET	0x40	/* Secondary bus reset */
+#define  PCI_BRIDGE_CTL_FAST_BACK	0x80	/* Fast Back2Back enabled on secondary interface */
+
+/* Header type 2 (CardBus bridges) */
+#define PCI_CB_CAPABILITY_LIST	0x14
+/* 0x15 reserved */
+#define PCI_CB_SEC_STATUS	0x16	/* Secondary status */
+#define PCI_CB_PRIMARY_BUS	0x18	/* PCI bus number */
+#define PCI_CB_CARD_BUS		0x19	/* CardBus bus number */
+#define PCI_CB_SUBORDINATE_BUS	0x1a	/* Subordinate bus number */
+#define PCI_CB_LATENCY_TIMER	0x1b	/* CardBus latency timer */
+#define PCI_CB_MEMORY_BASE_0	0x1c
+#define PCI_CB_MEMORY_LIMIT_0	0x20
+#define PCI_CB_MEMORY_BASE_1	0x24
+#define PCI_CB_MEMORY_LIMIT_1	0x28
+#define PCI_CB_IO_BASE_0	0x2c
+#define PCI_CB_IO_BASE_0_HI	0x2e
+#define PCI_CB_IO_LIMIT_0	0x30
+#define PCI_CB_IO_LIMIT_0_HI	0x32
+#define PCI_CB_IO_BASE_1	0x34
+#define PCI_CB_IO_BASE_1_HI	0x36
+#define PCI_CB_IO_LIMIT_1	0x38
+#define PCI_CB_IO_LIMIT_1_HI	0x3a
+#define  PCI_CB_IO_RANGE_MASK	(~0x03UL)
+/* 0x3c-0x3d are same as for htype 0 */
+#define PCI_CB_BRIDGE_CONTROL	0x3e
+#define  PCI_CB_BRIDGE_CTL_PARITY	0x01	/* Similar to standard bridge control register */
+#define  PCI_CB_BRIDGE_CTL_SERR		0x02
+#define  PCI_CB_BRIDGE_CTL_ISA		0x04
+#define  PCI_CB_BRIDGE_CTL_VGA		0x08
+#define  PCI_CB_BRIDGE_CTL_MASTER_ABORT	0x20
+#define  PCI_CB_BRIDGE_CTL_CB_RESET	0x40	/* CardBus reset */
+#define  PCI_CB_BRIDGE_CTL_16BIT_INT	0x80	/* Enable interrupt for 16-bit cards */
+#define  PCI_CB_BRIDGE_CTL_PREFETCH_MEM0 0x100	/* Prefetch enable for both memory regions */
+#define  PCI_CB_BRIDGE_CTL_PREFETCH_MEM1 0x200
+#define  PCI_CB_BRIDGE_CTL_POST_WRITES	0x400
+#define PCI_CB_SUBSYSTEM_VENDOR_ID	0x40
+#define PCI_CB_SUBSYSTEM_ID		0x42
+#define PCI_CB_LEGACY_MODE_BASE		0x44	/* 16-bit PC Card legacy mode base address (ExCa) */
+/* 0x48-0x7f reserved */
+
+/* Capability lists */
+
+#define PCI_CAP_LIST_ID		0	/* Capability ID */
+#define  PCI_CAP_ID_PM		0x01	/* Power Management */
+#define  PCI_CAP_ID_AGP		0x02	/* Accelerated Graphics Port */
+#define  PCI_CAP_ID_VPD		0x03	/* Vital Product Data */
+#define  PCI_CAP_ID_SLOTID	0x04	/* Slot Identification */
+#define  PCI_CAP_ID_MSI		0x05	/* Message Signalled Interrupts */
+#define  PCI_CAP_ID_CHSWP	0x06	/* CompactPCI HotSwap */
+#define  PCI_CAP_ID_PCIX	0x07	/* PCI-X */
+#define  PCI_CAP_ID_HT		0x08	/* HyperTransport */
+#define  PCI_CAP_ID_VNDR	0x09	/* Vendor-Specific */
+#define  PCI_CAP_ID_DBG		0x0A	/* Debug port */
+#define  PCI_CAP_ID_CCRC	0x0B	/* CompactPCI Central Resource Control */
+#define  PCI_CAP_ID_SHPC	0x0C	/* PCI Standard Hot-Plug Controller */
+#define  PCI_CAP_ID_SSVID	0x0D	/* Bridge subsystem vendor/device ID */
+#define  PCI_CAP_ID_AGP3	0x0E	/* AGP Target PCI-PCI bridge */
+#define  PCI_CAP_ID_SECDEV	0x0F	/* Secure Device */
+#define  PCI_CAP_ID_EXP		0x10	/* PCI Express */
+#define  PCI_CAP_ID_MSIX	0x11	/* MSI-X */
+#define  PCI_CAP_ID_SATA	0x12	/* SATA Data/Index Conf. */
+#define  PCI_CAP_ID_AF		0x13	/* PCI Advanced Features */
+#define  PCI_CAP_ID_EA		0x14	/* PCI Enhanced Allocation */
+#define  PCI_CAP_ID_MAX		PCI_CAP_ID_EA
+#define PCI_CAP_LIST_NEXT	1	/* Next capability in the list */
+#define PCI_CAP_FLAGS		2	/* Capability defined flags (16 bits) */
+#define PCI_CAP_SIZEOF		4
+
+/* Power Management Registers */
+
+#define PCI_PM_PMC		2	/* PM Capabilities Register */
+#define  PCI_PM_CAP_VER_MASK	0x0007	/* Version */
+#define  PCI_PM_CAP_PME_CLOCK	0x0008	/* PME clock required */
+#define  PCI_PM_CAP_RESERVED    0x0010  /* Reserved field */
+#define  PCI_PM_CAP_DSI		0x0020	/* Device specific initialization */
+#define  PCI_PM_CAP_AUX_POWER	0x01C0	/* Auxiliary power support mask */
+#define  PCI_PM_CAP_D1		0x0200	/* D1 power state support */
+#define  PCI_PM_CAP_D2		0x0400	/* D2 power state support */
+#define  PCI_PM_CAP_PME		0x0800	/* PME pin supported */
+#define  PCI_PM_CAP_PME_MASK	0xF800	/* PME Mask of all supported states */
+#define  PCI_PM_CAP_PME_D0	0x0800	/* PME# from D0 */
+#define  PCI_PM_CAP_PME_D1	0x1000	/* PME# from D1 */
+#define  PCI_PM_CAP_PME_D2	0x2000	/* PME# from D2 */
+#define  PCI_PM_CAP_PME_D3	0x4000	/* PME# from D3 (hot) */
+#define  PCI_PM_CAP_PME_D3cold	0x8000	/* PME# from D3 (cold) */
+#define  PCI_PM_CAP_PME_SHIFT	11	/* Start of the PME Mask in PMC */
+#define PCI_PM_CTRL		4	/* PM control and status register */
+#define  PCI_PM_CTRL_STATE_MASK	0x0003	/* Current power state (D0 to D3) */
+#define  PCI_PM_CTRL_NO_SOFT_RESET	0x0008	/* No reset for D3hot->D0 */
+#define  PCI_PM_CTRL_PME_ENABLE	0x0100	/* PME pin enable */
+#define  PCI_PM_CTRL_DATA_SEL_MASK	0x1e00	/* Data select (??) */
+#define  PCI_PM_CTRL_DATA_SCALE_MASK	0x6000	/* Data scale (??) */
+#define  PCI_PM_CTRL_PME_STATUS	0x8000	/* PME pin status */
+#define PCI_PM_PPB_EXTENSIONS	6	/* PPB support extensions (??) */
+#define  PCI_PM_PPB_B2_B3	0x40	/* Stop clock when in D3hot (??) */
+#define  PCI_PM_BPCC_ENABLE	0x80	/* Bus power/clock control enable (??) */
+#define PCI_PM_DATA_REGISTER	7	/* (??) */
+#define PCI_PM_SIZEOF		8
+
+/* AGP registers */
+
+#define PCI_AGP_VERSION		2	/* BCD version number */
+#define PCI_AGP_RFU		3	/* Rest of capability flags */
+#define PCI_AGP_STATUS		4	/* Status register */
+#define  PCI_AGP_STATUS_RQ_MASK	0xff000000	/* Maximum number of requests - 1 */
+#define  PCI_AGP_STATUS_SBA	0x0200	/* Sideband addressing supported */
+#define  PCI_AGP_STATUS_64BIT	0x0020	/* 64-bit addressing supported */
+#define  PCI_AGP_STATUS_FW	0x0010	/* FW transfers supported */
+#define  PCI_AGP_STATUS_RATE4	0x0004	/* 4x transfer rate supported */
+#define  PCI_AGP_STATUS_RATE2	0x0002	/* 2x transfer rate supported */
+#define  PCI_AGP_STATUS_RATE1	0x0001	/* 1x transfer rate supported */
+#define PCI_AGP_COMMAND		8	/* Control register */
+#define  PCI_AGP_COMMAND_RQ_MASK 0xff000000  /* Master: Maximum number of requests */
+#define  PCI_AGP_COMMAND_SBA	0x0200	/* Sideband addressing enabled */
+#define  PCI_AGP_COMMAND_AGP	0x0100	/* Allow processing of AGP transactions */
+#define  PCI_AGP_COMMAND_64BIT	0x0020	/* Allow processing of 64-bit addresses */
+#define  PCI_AGP_COMMAND_FW	0x0010	/* Force FW transfers */
+#define  PCI_AGP_COMMAND_RATE4	0x0004	/* Use 4x rate */
+#define  PCI_AGP_COMMAND_RATE2	0x0002	/* Use 2x rate */
+#define  PCI_AGP_COMMAND_RATE1	0x0001	/* Use 1x rate */
+#define PCI_AGP_SIZEOF		12
+
+/* Vital Product Data */
+
+#define PCI_VPD_ADDR		2	/* Address to access (15 bits!) */
+#define  PCI_VPD_ADDR_MASK	0x7fff	/* Address mask */
+#define  PCI_VPD_ADDR_F		0x8000	/* Write 0, 1 indicates completion */
+#define PCI_VPD_DATA		4	/* 32-bits of data returned here */
+#define PCI_CAP_VPD_SIZEOF	8
+
+/* Slot Identification */
+
+#define PCI_SID_ESR		2	/* Expansion Slot Register */
+#define  PCI_SID_ESR_NSLOTS	0x1f	/* Number of expansion slots available */
+#define  PCI_SID_ESR_FIC	0x20	/* First In Chassis Flag */
+#define PCI_SID_CHASSIS_NR	3	/* Chassis Number */
+
+/* Message Signalled Interrupts registers */
+
+#define PCI_MSI_FLAGS		2	/* Message Control */
+#define  PCI_MSI_FLAGS_ENABLE	0x0001	/* MSI feature enabled */
+#define  PCI_MSI_FLAGS_QMASK	0x000e	/* Maximum queue size available */
+#define  PCI_MSI_FLAGS_QSIZE	0x0070	/* Message queue size configured */
+#define  PCI_MSI_FLAGS_64BIT	0x0080	/* 64-bit addresses allowed */
+#define  PCI_MSI_FLAGS_MASKBIT	0x0100	/* Per-vector masking capable */
+#define PCI_MSI_RFU		3	/* Rest of capability flags */
+#define PCI_MSI_ADDRESS_LO	4	/* Lower 32 bits */
+#define PCI_MSI_ADDRESS_HI	8	/* Upper 32 bits (if PCI_MSI_FLAGS_64BIT set) */
+#define PCI_MSI_DATA_32		8	/* 16 bits of data for 32-bit devices */
+#define PCI_MSI_MASK_32		12	/* Mask bits register for 32-bit devices */
+#define PCI_MSI_PENDING_32	16	/* Pending intrs for 32-bit devices */
+#define PCI_MSI_DATA_64		12	/* 16 bits of data for 64-bit devices */
+#define PCI_MSI_MASK_64		16	/* Mask bits register for 64-bit devices */
+#define PCI_MSI_PENDING_64	20	/* Pending intrs for 64-bit devices */
+
+/* MSI-X registers */
+#define PCI_MSIX_FLAGS		2	/* Message Control */
+#define  PCI_MSIX_FLAGS_QSIZE	0x07FF	/* Table size */
+#define  PCI_MSIX_FLAGS_MASKALL	0x4000	/* Mask all vectors for this function */
+#define  PCI_MSIX_FLAGS_ENABLE	0x8000	/* MSI-X enable */
+#define PCI_MSIX_TABLE		4	/* Table offset */
+#define  PCI_MSIX_TABLE_BIR	0x00000007 /* BAR index */
+#define  PCI_MSIX_TABLE_OFFSET	0xfffffff8 /* Offset into specified BAR */
+#define PCI_MSIX_PBA		8	/* Pending Bit Array offset */
+#define  PCI_MSIX_PBA_BIR	0x00000007 /* BAR index */
+#define  PCI_MSIX_PBA_OFFSET	0xfffffff8 /* Offset into specified BAR */
+#define PCI_MSIX_FLAGS_BIRMASK	PCI_MSIX_PBA_BIR /* deprecated */
+#define PCI_CAP_MSIX_SIZEOF	12	/* size of MSIX registers */
+
+/* MSI-X Table entry format */
+#define PCI_MSIX_ENTRY_SIZE		16
+#define  PCI_MSIX_ENTRY_LOWER_ADDR	0
+#define  PCI_MSIX_ENTRY_UPPER_ADDR	4
+#define  PCI_MSIX_ENTRY_DATA		8
+#define  PCI_MSIX_ENTRY_VECTOR_CTRL	12
+#define   PCI_MSIX_ENTRY_CTRL_MASKBIT	1
+
+/* CompactPCI Hotswap Register */
+
+#define PCI_CHSWP_CSR		2	/* Control and Status Register */
+#define  PCI_CHSWP_DHA		0x01	/* Device Hiding Arm */
+#define  PCI_CHSWP_EIM		0x02	/* ENUM# Signal Mask */
+#define  PCI_CHSWP_PIE		0x04	/* Pending Insert or Extract */
+#define  PCI_CHSWP_LOO		0x08	/* LED On / Off */
+#define  PCI_CHSWP_PI		0x30	/* Programming Interface */
+#define  PCI_CHSWP_EXT		0x40	/* ENUM# status - extraction */
+#define  PCI_CHSWP_INS		0x80	/* ENUM# status - insertion */
+
+/* PCI Advanced Feature registers */
+
+#define PCI_AF_LENGTH		2
+#define PCI_AF_CAP		3
+#define  PCI_AF_CAP_TP		0x01
+#define  PCI_AF_CAP_FLR		0x02
+#define PCI_AF_CTRL		4
+#define  PCI_AF_CTRL_FLR	0x01
+#define PCI_AF_STATUS		5
+#define  PCI_AF_STATUS_TP	0x01
+#define PCI_CAP_AF_SIZEOF	6	/* size of AF registers */
+
+/* PCI Enhanced Allocation registers */
+
+#define PCI_EA_NUM_ENT		2	/* Number of Capability Entries */
+#define  PCI_EA_NUM_ENT_MASK	0x3f	/* Num Entries Mask */
+#define PCI_EA_FIRST_ENT	4	/* First EA Entry in List */
+#define PCI_EA_FIRST_ENT_BRIDGE	8	/* First EA Entry for Bridges */
+#define  PCI_EA_ES		0x00000007 /* Entry Size */
+#define  PCI_EA_BEI		0x000000f0 /* BAR Equivalent Indicator */
+/* 0-5 map to BARs 0-5 respectively */
+#define   PCI_EA_BEI_BAR0		0
+#define   PCI_EA_BEI_BAR5		5
+#define   PCI_EA_BEI_BRIDGE		6	/* Resource behind bridge */
+#define   PCI_EA_BEI_ENI		7	/* Equivalent Not Indicated */
+#define   PCI_EA_BEI_ROM		8	/* Expansion ROM */
+/* 9-14 map to VF BARs 0-5 respectively */
+#define   PCI_EA_BEI_VF_BAR0		9
+#define   PCI_EA_BEI_VF_BAR5		14
+#define   PCI_EA_BEI_RESERVED		15	/* Reserved - Treat like ENI */
+#define  PCI_EA_PP		0x0000ff00	/* Primary Properties */
+#define  PCI_EA_SP		0x00ff0000	/* Secondary Properties */
+#define   PCI_EA_P_MEM			0x00	/* Non-Prefetch Memory */
+#define   PCI_EA_P_MEM_PREFETCH		0x01	/* Prefetchable Memory */
+#define   PCI_EA_P_IO			0x02	/* I/O Space */
+#define   PCI_EA_P_VF_MEM_PREFETCH	0x03	/* VF Prefetchable Memory */
+#define   PCI_EA_P_VF_MEM		0x04	/* VF Non-Prefetch Memory */
+#define   PCI_EA_P_BRIDGE_MEM		0x05	/* Bridge Non-Prefetch Memory */
+#define   PCI_EA_P_BRIDGE_MEM_PREFETCH	0x06	/* Bridge Prefetchable Memory */
+#define   PCI_EA_P_BRIDGE_IO		0x07	/* Bridge I/O Space */
+/* 0x08-0xfc reserved */
+#define   PCI_EA_P_MEM_RESERVED		0xfd	/* Reserved Memory */
+#define   PCI_EA_P_IO_RESERVED		0xfe	/* Reserved I/O Space */
+#define   PCI_EA_P_UNAVAILABLE		0xff	/* Entry Unavailable */
+#define  PCI_EA_WRITABLE	0x40000000	/* Writable: 1 = RW, 0 = HwInit */
+#define  PCI_EA_ENABLE		0x80000000	/* Enable for this entry */
+#define PCI_EA_BASE		4		/* Base Address Offset */
+#define PCI_EA_MAX_OFFSET	8		/* MaxOffset (resource length) */
+/* bit 0 is reserved */
+#define  PCI_EA_IS_64		0x00000002	/* 64-bit field flag */
+#define  PCI_EA_FIELD_MASK	0xfffffffc	/* For Base & Max Offset */
+
+/* PCI-X registers (Type 0 (non-bridge) devices) */
+
+#define PCI_X_CMD		2	/* Modes & Features */
+#define  PCI_X_CMD_DPERR_E	0x0001	/* Data Parity Error Recovery Enable */
+#define  PCI_X_CMD_ERO		0x0002	/* Enable Relaxed Ordering */
+#define  PCI_X_CMD_READ_512	0x0000	/* 512 byte maximum read byte count */
+#define  PCI_X_CMD_READ_1K	0x0004	/* 1Kbyte maximum read byte count */
+#define  PCI_X_CMD_READ_2K	0x0008	/* 2Kbyte maximum read byte count */
+#define  PCI_X_CMD_READ_4K	0x000c	/* 4Kbyte maximum read byte count */
+#define  PCI_X_CMD_MAX_READ	0x000c	/* Max Memory Read Byte Count */
+				/* Max # of outstanding split transactions */
+#define  PCI_X_CMD_SPLIT_1	0x0000	/* Max 1 */
+#define  PCI_X_CMD_SPLIT_2	0x0010	/* Max 2 */
+#define  PCI_X_CMD_SPLIT_3	0x0020	/* Max 3 */
+#define  PCI_X_CMD_SPLIT_4	0x0030	/* Max 4 */
+#define  PCI_X_CMD_SPLIT_8	0x0040	/* Max 8 */
+#define  PCI_X_CMD_SPLIT_12	0x0050	/* Max 12 */
+#define  PCI_X_CMD_SPLIT_16	0x0060	/* Max 16 */
+#define  PCI_X_CMD_SPLIT_32	0x0070	/* Max 32 */
+#define  PCI_X_CMD_MAX_SPLIT	0x0070	/* Max Outstanding Split Transactions */
+#define  PCI_X_CMD_VERSION(x)	(((x) >> 12) & 3) /* Version */
+#define PCI_X_STATUS		4	/* PCI-X capabilities */
+#define  PCI_X_STATUS_DEVFN	0x000000ff	/* A copy of devfn */
+#define  PCI_X_STATUS_BUS	0x0000ff00	/* A copy of bus nr */
+#define  PCI_X_STATUS_64BIT	0x00010000	/* 64-bit device */
+#define  PCI_X_STATUS_133MHZ	0x00020000	/* 133 MHz capable */
+#define  PCI_X_STATUS_SPL_DISC	0x00040000	/* Split Completion Discarded */
+#define  PCI_X_STATUS_UNX_SPL	0x00080000	/* Unexpected Split Completion */
+#define  PCI_X_STATUS_COMPLEX	0x00100000	/* Device Complexity */
+#define  PCI_X_STATUS_MAX_READ	0x00600000	/* Designed Max Memory Read Count */
+#define  PCI_X_STATUS_MAX_SPLIT	0x03800000	/* Designed Max Outstanding Split Transactions */
+#define  PCI_X_STATUS_MAX_CUM	0x1c000000	/* Designed Max Cumulative Read Size */
+#define  PCI_X_STATUS_SPL_ERR	0x20000000	/* Rcvd Split Completion Error Msg */
+#define  PCI_X_STATUS_266MHZ	0x40000000	/* 266 MHz capable */
+#define  PCI_X_STATUS_533MHZ	0x80000000	/* 533 MHz capable */
+#define PCI_X_ECC_CSR		8	/* ECC control and status */
+#define PCI_CAP_PCIX_SIZEOF_V0	8	/* size of registers for Version 0 */
+#define PCI_CAP_PCIX_SIZEOF_V1	24	/* size for Version 1 */
+#define PCI_CAP_PCIX_SIZEOF_V2	PCI_CAP_PCIX_SIZEOF_V1	/* Same for v2 */
+
+/* PCI-X registers (Type 1 (bridge) devices) */
+
+#define PCI_X_BRIDGE_SSTATUS	2	/* Secondary Status */
+#define  PCI_X_SSTATUS_64BIT	0x0001	/* Secondary AD interface is 64 bits */
+#define  PCI_X_SSTATUS_133MHZ	0x0002	/* 133 MHz capable */
+#define  PCI_X_SSTATUS_FREQ	0x03c0	/* Secondary Bus Mode and Frequency */
+#define  PCI_X_SSTATUS_VERS	0x3000	/* PCI-X Capability Version */
+#define  PCI_X_SSTATUS_V1	0x1000	/* Mode 2, not Mode 1 */
+#define  PCI_X_SSTATUS_V2	0x2000	/* Mode 1 or Modes 1 and 2 */
+#define  PCI_X_SSTATUS_266MHZ	0x4000	/* 266 MHz capable */
+#define  PCI_X_SSTATUS_533MHZ	0x8000	/* 533 MHz capable */
+#define PCI_X_BRIDGE_STATUS	4	/* Bridge Status */
+
+/* PCI Bridge Subsystem ID registers */
+
+#define PCI_SSVID_VENDOR_ID     4	/* PCI Bridge subsystem vendor ID */
+#define PCI_SSVID_DEVICE_ID     6	/* PCI Bridge subsystem device ID */
+
+/* PCI Express capability registers */
+
+#define PCI_EXP_FLAGS		2	/* Capabilities register */
+#define PCI_EXP_FLAGS_VERS	0x000f	/* Capability version */
+#define PCI_EXP_FLAGS_TYPE	0x00f0	/* Device/Port type */
+#define  PCI_EXP_TYPE_ENDPOINT	0x0	/* Express Endpoint */
+#define  PCI_EXP_TYPE_LEG_END	0x1	/* Legacy Endpoint */
+#define  PCI_EXP_TYPE_ROOT_PORT 0x4	/* Root Port */
+#define  PCI_EXP_TYPE_UPSTREAM	0x5	/* Upstream Port */
+#define  PCI_EXP_TYPE_DOWNSTREAM 0x6	/* Downstream Port */
+#define  PCI_EXP_TYPE_PCI_BRIDGE 0x7	/* PCIe to PCI/PCI-X Bridge */
+#define  PCI_EXP_TYPE_PCIE_BRIDGE 0x8	/* PCI/PCI-X to PCIe Bridge */
+#define  PCI_EXP_TYPE_RC_END	0x9	/* Root Complex Integrated Endpoint */
+#define  PCI_EXP_TYPE_RC_EC	0xa	/* Root Complex Event Collector */
+#define PCI_EXP_FLAGS_SLOT	0x0100	/* Slot implemented */
+#define PCI_EXP_FLAGS_IRQ	0x3e00	/* Interrupt message number */
+#define PCI_EXP_DEVCAP		4	/* Device capabilities */
+#define  PCI_EXP_DEVCAP_PAYLOAD	0x00000007 /* Max_Payload_Size */
+#define  PCI_EXP_DEVCAP_PHANTOM	0x00000018 /* Phantom functions */
+#define  PCI_EXP_DEVCAP_EXT_TAG	0x00000020 /* Extended tags */
+#define  PCI_EXP_DEVCAP_L0S	0x000001c0 /* L0s Acceptable Latency */
+#define  PCI_EXP_DEVCAP_L1	0x00000e00 /* L1 Acceptable Latency */
+#define  PCI_EXP_DEVCAP_ATN_BUT	0x00001000 /* Attention Button Present */
+#define  PCI_EXP_DEVCAP_ATN_IND	0x00002000 /* Attention Indicator Present */
+#define  PCI_EXP_DEVCAP_PWR_IND	0x00004000 /* Power Indicator Present */
+#define  PCI_EXP_DEVCAP_RBER	0x00008000 /* Role-Based Error Reporting */
+#define  PCI_EXP_DEVCAP_PWR_VAL	0x03fc0000 /* Slot Power Limit Value */
+#define  PCI_EXP_DEVCAP_PWR_SCL	0x0c000000 /* Slot Power Limit Scale */
+#define  PCI_EXP_DEVCAP_FLR     0x10000000 /* Function Level Reset */
+#define PCI_EXP_DEVCTL		8	/* Device Control */
+#define  PCI_EXP_DEVCTL_CERE	0x0001	/* Correctable Error Reporting En. */
+#define  PCI_EXP_DEVCTL_NFERE	0x0002	/* Non-Fatal Error Reporting Enable */
+#define  PCI_EXP_DEVCTL_FERE	0x0004	/* Fatal Error Reporting Enable */
+#define  PCI_EXP_DEVCTL_URRE	0x0008	/* Unsupported Request Reporting En. */
+#define  PCI_EXP_DEVCTL_RELAX_EN 0x0010 /* Enable relaxed ordering */
+#define  PCI_EXP_DEVCTL_PAYLOAD	0x00e0	/* Max_Payload_Size */
+#define  PCI_EXP_DEVCTL_EXT_TAG	0x0100	/* Extended Tag Field Enable */
+#define  PCI_EXP_DEVCTL_PHANTOM	0x0200	/* Phantom Functions Enable */
+#define  PCI_EXP_DEVCTL_AUX_PME	0x0400	/* Auxiliary Power PM Enable */
+#define  PCI_EXP_DEVCTL_NOSNOOP_EN 0x0800  /* Enable No Snoop */
+#define  PCI_EXP_DEVCTL_READRQ	0x7000	/* Max_Read_Request_Size */
+#define  PCI_EXP_DEVCTL_READRQ_128B  0x0000 /* 128 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_256B  0x1000 /* 256 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_512B  0x2000 /* 512 Bytes */
+#define  PCI_EXP_DEVCTL_READRQ_1024B 0x3000 /* 1024 Bytes */
+#define  PCI_EXP_DEVCTL_BCR_FLR 0x8000  /* Bridge Configuration Retry / FLR */
+#define PCI_EXP_DEVSTA		10	/* Device Status */
+#define  PCI_EXP_DEVSTA_CED	0x0001	/* Correctable Error Detected */
+#define  PCI_EXP_DEVSTA_NFED	0x0002	/* Non-Fatal Error Detected */
+#define  PCI_EXP_DEVSTA_FED	0x0004	/* Fatal Error Detected */
+#define  PCI_EXP_DEVSTA_URD	0x0008	/* Unsupported Request Detected */
+#define  PCI_EXP_DEVSTA_AUXPD	0x0010	/* AUX Power Detected */
+#define  PCI_EXP_DEVSTA_TRPND	0x0020	/* Transactions Pending */
+#define PCI_EXP_LNKCAP		12	/* Link Capabilities */
+#define  PCI_EXP_LNKCAP_SLS	0x0000000f /* Supported Link Speeds */
+#define  PCI_EXP_LNKCAP_SLS_2_5GB 0x00000001 /* LNKCAP2 SLS Vector bit 0 */
+#define  PCI_EXP_LNKCAP_SLS_5_0GB 0x00000002 /* LNKCAP2 SLS Vector bit 1 */
+#define  PCI_EXP_LNKCAP_MLW	0x000003f0 /* Maximum Link Width */
+#define  PCI_EXP_LNKCAP_ASPMS	0x00000c00 /* ASPM Support */
+#define  PCI_EXP_LNKCAP_L0SEL	0x00007000 /* L0s Exit Latency */
+#define  PCI_EXP_LNKCAP_L1EL	0x00038000 /* L1 Exit Latency */
+#define  PCI_EXP_LNKCAP_CLKPM	0x00040000 /* Clock Power Management */
+#define  PCI_EXP_LNKCAP_SDERC	0x00080000 /* Surprise Down Error Reporting Capable */
+#define  PCI_EXP_LNKCAP_DLLLARC	0x00100000 /* Data Link Layer Link Active Reporting Capable */
+#define  PCI_EXP_LNKCAP_LBNC	0x00200000 /* Link Bandwidth Notification Capability */
+#define  PCI_EXP_LNKCAP_PN	0xff000000 /* Port Number */
+#define PCI_EXP_LNKCTL		16	/* Link Control */
+#define  PCI_EXP_LNKCTL_ASPMC	0x0003	/* ASPM Control */
+#define  PCI_EXP_LNKCTL_ASPM_L0S 0x0001	/* L0s Enable */
+#define  PCI_EXP_LNKCTL_ASPM_L1  0x0002	/* L1 Enable */
+#define  PCI_EXP_LNKCTL_RCB	0x0008	/* Read Completion Boundary */
+#define  PCI_EXP_LNKCTL_LD	0x0010	/* Link Disable */
+#define  PCI_EXP_LNKCTL_RL	0x0020	/* Retrain Link */
+#define  PCI_EXP_LNKCTL_CCC	0x0040	/* Common Clock Configuration */
+#define  PCI_EXP_LNKCTL_ES	0x0080	/* Extended Synch */
+#define  PCI_EXP_LNKCTL_CLKREQ_EN 0x0100 /* Enable clkreq */
+#define  PCI_EXP_LNKCTL_HAWD	0x0200	/* Hardware Autonomous Width Disable */
+#define  PCI_EXP_LNKCTL_LBMIE	0x0400	/* Link Bandwidth Management Interrupt Enable */
+#define  PCI_EXP_LNKCTL_LABIE	0x0800	/* Link Autonomous Bandwidth Interrupt Enable */
+#define PCI_EXP_LNKSTA		18	/* Link Status */
+#define  PCI_EXP_LNKSTA_CLS	0x000f	/* Current Link Speed */
+#define  PCI_EXP_LNKSTA_CLS_2_5GB 0x0001 /* Current Link Speed 2.5GT/s */
+#define  PCI_EXP_LNKSTA_CLS_5_0GB 0x0002 /* Current Link Speed 5.0GT/s */
+#define  PCI_EXP_LNKSTA_CLS_8_0GB 0x0003 /* Current Link Speed 8.0GT/s */
+#define  PCI_EXP_LNKSTA_NLW	0x03f0	/* Negotiated Link Width */
+#define  PCI_EXP_LNKSTA_NLW_X1	0x0010	/* Current Link Width x1 */
+#define  PCI_EXP_LNKSTA_NLW_X2	0x0020	/* Current Link Width x2 */
+#define  PCI_EXP_LNKSTA_NLW_X4	0x0040	/* Current Link Width x4 */
+#define  PCI_EXP_LNKSTA_NLW_X8	0x0080	/* Current Link Width x8 */
+#define  PCI_EXP_LNKSTA_NLW_SHIFT 4	/* start of NLW mask in link status */
+#define  PCI_EXP_LNKSTA_LT	0x0800	/* Link Training */
+#define  PCI_EXP_LNKSTA_SLC	0x1000	/* Slot Clock Configuration */
+#define  PCI_EXP_LNKSTA_DLLLA	0x2000	/* Data Link Layer Link Active */
+#define  PCI_EXP_LNKSTA_LBMS	0x4000	/* Link Bandwidth Management Status */
+#define  PCI_EXP_LNKSTA_LABS	0x8000	/* Link Autonomous Bandwidth Status */
+#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V1	20	/* v1 endpoints end here */
+#define PCI_EXP_SLTCAP		20	/* Slot Capabilities */
+#define  PCI_EXP_SLTCAP_ABP	0x00000001 /* Attention Button Present */
+#define  PCI_EXP_SLTCAP_PCP	0x00000002 /* Power Controller Present */
+#define  PCI_EXP_SLTCAP_MRLSP	0x00000004 /* MRL Sensor Present */
+#define  PCI_EXP_SLTCAP_AIP	0x00000008 /* Attention Indicator Present */
+#define  PCI_EXP_SLTCAP_PIP	0x00000010 /* Power Indicator Present */
+#define  PCI_EXP_SLTCAP_HPS	0x00000020 /* Hot-Plug Surprise */
+#define  PCI_EXP_SLTCAP_HPC	0x00000040 /* Hot-Plug Capable */
+#define  PCI_EXP_SLTCAP_SPLV	0x00007f80 /* Slot Power Limit Value */
+#define  PCI_EXP_SLTCAP_SPLS	0x00018000 /* Slot Power Limit Scale */
+#define  PCI_EXP_SLTCAP_EIP	0x00020000 /* Electromechanical Interlock Present */
+#define  PCI_EXP_SLTCAP_NCCS	0x00040000 /* No Command Completed Support */
+#define  PCI_EXP_SLTCAP_PSN	0xfff80000 /* Physical Slot Number */
+#define PCI_EXP_SLTCTL		24	/* Slot Control */
+#define  PCI_EXP_SLTCTL_ABPE	0x0001	/* Attention Button Pressed Enable */
+#define  PCI_EXP_SLTCTL_PFDE	0x0002	/* Power Fault Detected Enable */
+#define  PCI_EXP_SLTCTL_MRLSCE	0x0004	/* MRL Sensor Changed Enable */
+#define  PCI_EXP_SLTCTL_PDCE	0x0008	/* Presence Detect Changed Enable */
+#define  PCI_EXP_SLTCTL_CCIE	0x0010	/* Command Completed Interrupt Enable */
+#define  PCI_EXP_SLTCTL_HPIE	0x0020	/* Hot-Plug Interrupt Enable */
+#define  PCI_EXP_SLTCTL_AIC	0x00c0	/* Attention Indicator Control */
+#define  PCI_EXP_SLTCTL_ATTN_IND_ON    0x0040 /* Attention Indicator on */
+#define  PCI_EXP_SLTCTL_ATTN_IND_BLINK 0x0080 /* Attention Indicator blinking */
+#define  PCI_EXP_SLTCTL_ATTN_IND_OFF   0x00c0 /* Attention Indicator off */
+#define  PCI_EXP_SLTCTL_PIC	0x0300	/* Power Indicator Control */
+#define  PCI_EXP_SLTCTL_PWR_IND_ON     0x0100 /* Power Indicator on */
+#define  PCI_EXP_SLTCTL_PWR_IND_BLINK  0x0200 /* Power Indicator blinking */
+#define  PCI_EXP_SLTCTL_PWR_IND_OFF    0x0300 /* Power Indicator off */
+#define  PCI_EXP_SLTCTL_PCC	0x0400	/* Power Controller Control */
+#define  PCI_EXP_SLTCTL_PWR_ON         0x0000 /* Power On */
+#define  PCI_EXP_SLTCTL_PWR_OFF        0x0400 /* Power Off */
+#define  PCI_EXP_SLTCTL_EIC	0x0800	/* Electromechanical Interlock Control */
+#define  PCI_EXP_SLTCTL_DLLSCE	0x1000	/* Data Link Layer State Changed Enable */
+#define PCI_EXP_SLTSTA		26	/* Slot Status */
+#define  PCI_EXP_SLTSTA_ABP	0x0001	/* Attention Button Pressed */
+#define  PCI_EXP_SLTSTA_PFD	0x0002	/* Power Fault Detected */
+#define  PCI_EXP_SLTSTA_MRLSC	0x0004	/* MRL Sensor Changed */
+#define  PCI_EXP_SLTSTA_PDC	0x0008	/* Presence Detect Changed */
+#define  PCI_EXP_SLTSTA_CC	0x0010	/* Command Completed */
+#define  PCI_EXP_SLTSTA_MRLSS	0x0020	/* MRL Sensor State */
+#define  PCI_EXP_SLTSTA_PDS	0x0040	/* Presence Detect State */
+#define  PCI_EXP_SLTSTA_EIS	0x0080	/* Electromechanical Interlock Status */
+#define  PCI_EXP_SLTSTA_DLLSC	0x0100	/* Data Link Layer State Changed */
+#define PCI_EXP_RTCTL		28	/* Root Control */
+#define  PCI_EXP_RTCTL_SECEE	0x0001	/* System Error on Correctable Error */
+#define  PCI_EXP_RTCTL_SENFEE	0x0002	/* System Error on Non-Fatal Error */
+#define  PCI_EXP_RTCTL_SEFEE	0x0004	/* System Error on Fatal Error */
+#define  PCI_EXP_RTCTL_PMEIE	0x0008	/* PME Interrupt Enable */
+#define  PCI_EXP_RTCTL_CRSSVE	0x0010	/* CRS Software Visibility Enable */
+#define PCI_EXP_RTCAP		30	/* Root Capabilities */
+#define  PCI_EXP_RTCAP_CRSVIS	0x0001	/* CRS Software Visibility capability */
+#define PCI_EXP_RTSTA		32	/* Root Status */
+#define PCI_EXP_RTSTA_PME	0x00010000 /* PME status */
+#define PCI_EXP_RTSTA_PENDING	0x00020000 /* PME pending */
+/*
+ * The Device Capabilities 2, Device Status 2, Device Control 2,
+ * Link Capabilities 2, Link Status 2, Link Control 2,
+ * Slot Capabilities 2, Slot Status 2, and Slot Control 2 registers
+ * are only present on devices with PCIe Capability version 2.
+ * Use pcie_capability_read_word() and similar interfaces to use them
+ * safely.
+ */
+#define PCI_EXP_DEVCAP2		36	/* Device Capabilities 2 */
+#define  PCI_EXP_DEVCAP2_ARI		0x00000020 /* Alternative Routing-ID */
+#define  PCI_EXP_DEVCAP2_LTR		0x00000800 /* Latency tolerance reporting */
+#define  PCI_EXP_DEVCAP2_OBFF_MASK	0x000c0000 /* OBFF support mechanism */
+#define  PCI_EXP_DEVCAP2_OBFF_MSG	0x00040000 /* New message signaling */
+#define  PCI_EXP_DEVCAP2_OBFF_WAKE	0x00080000 /* Re-use WAKE# for OBFF */
+#define PCI_EXP_DEVCTL2		40	/* Device Control 2 */
+#define  PCI_EXP_DEVCTL2_COMP_TIMEOUT	0x000f	/* Completion Timeout Value */
+#define  PCI_EXP_DEVCTL2_ARI		0x0020	/* Alternative Routing-ID */
+#define  PCI_EXP_DEVCTL2_IDO_REQ_EN	0x0100	/* Allow IDO for requests */
+#define  PCI_EXP_DEVCTL2_IDO_CMP_EN	0x0200	/* Allow IDO for completions */
+#define  PCI_EXP_DEVCTL2_LTR_EN		0x0400	/* Enable LTR mechanism */
+#define  PCI_EXP_DEVCTL2_OBFF_MSGA_EN	0x2000	/* Enable OBFF Message type A */
+#define  PCI_EXP_DEVCTL2_OBFF_MSGB_EN	0x4000	/* Enable OBFF Message type B */
+#define  PCI_EXP_DEVCTL2_OBFF_WAKE_EN	0x6000	/* OBFF using WAKE# signaling */
+#define PCI_EXP_DEVSTA2		42	/* Device Status 2 */
+#define PCI_CAP_EXP_ENDPOINT_SIZEOF_V2	44	/* v2 endpoints end here */
+#define PCI_EXP_LNKCAP2		44	/* Link Capabilities 2 */
+#define  PCI_EXP_LNKCAP2_SLS_2_5GB	0x00000002 /* Supported Speed 2.5GT/s */
+#define  PCI_EXP_LNKCAP2_SLS_5_0GB	0x00000004 /* Supported Speed 5.0GT/s */
+#define  PCI_EXP_LNKCAP2_SLS_8_0GB	0x00000008 /* Supported Speed 8.0GT/s */
+#define  PCI_EXP_LNKCAP2_CROSSLINK	0x00000100 /* Crosslink supported */
+#define PCI_EXP_LNKCTL2		48	/* Link Control 2 */
+#define PCI_EXP_LNKSTA2		50	/* Link Status 2 */
+#define PCI_EXP_SLTCAP2		52	/* Slot Capabilities 2 */
+#define PCI_EXP_SLTCTL2		56	/* Slot Control 2 */
+#define PCI_EXP_SLTSTA2		58	/* Slot Status 2 */
+
+/* Extended Capabilities (PCI-X 2.0 and Express) */
+#define PCI_EXT_CAP_ID(header)		(header & 0x0000ffff)
+#define PCI_EXT_CAP_VER(header)		((header >> 16) & 0xf)
+#define PCI_EXT_CAP_NEXT(header)	((header >> 20) & 0xffc)
+
+#define PCI_EXT_CAP_ID_ERR	0x01	/* Advanced Error Reporting */
+#define PCI_EXT_CAP_ID_VC	0x02	/* Virtual Channel Capability */
+#define PCI_EXT_CAP_ID_DSN	0x03	/* Device Serial Number */
+#define PCI_EXT_CAP_ID_PWR	0x04	/* Power Budgeting */
+#define PCI_EXT_CAP_ID_RCLD	0x05	/* Root Complex Link Declaration */
+#define PCI_EXT_CAP_ID_RCILC	0x06	/* Root Complex Internal Link Control */
+#define PCI_EXT_CAP_ID_RCEC	0x07	/* Root Complex Event Collector */
+#define PCI_EXT_CAP_ID_MFVC	0x08	/* Multi-Function VC Capability */
+#define PCI_EXT_CAP_ID_VC9	0x09	/* same as _VC */
+#define PCI_EXT_CAP_ID_RCRB	0x0A	/* Root Complex RB? */
+#define PCI_EXT_CAP_ID_VNDR	0x0B	/* Vendor-Specific */
+#define PCI_EXT_CAP_ID_CAC	0x0C	/* Config Access - obsolete */
+#define PCI_EXT_CAP_ID_ACS	0x0D	/* Access Control Services */
+#define PCI_EXT_CAP_ID_ARI	0x0E	/* Alternate Routing ID */
+#define PCI_EXT_CAP_ID_ATS	0x0F	/* Address Translation Services */
+#define PCI_EXT_CAP_ID_SRIOV	0x10	/* Single Root I/O Virtualization */
+#define PCI_EXT_CAP_ID_MRIOV	0x11	/* Multi Root I/O Virtualization */
+#define PCI_EXT_CAP_ID_MCAST	0x12	/* Multicast */
+#define PCI_EXT_CAP_ID_PRI	0x13	/* Page Request Interface */
+#define PCI_EXT_CAP_ID_AMD_XXX	0x14	/* Reserved for AMD */
+#define PCI_EXT_CAP_ID_REBAR	0x15	/* Resizable BAR */
+#define PCI_EXT_CAP_ID_DPA	0x16	/* Dynamic Power Allocation */
+#define PCI_EXT_CAP_ID_TPH	0x17	/* TPH Requester */
+#define PCI_EXT_CAP_ID_LTR	0x18	/* Latency Tolerance Reporting */
+#define PCI_EXT_CAP_ID_SECPCI	0x19	/* Secondary PCIe Capability */
+#define PCI_EXT_CAP_ID_PMUX	0x1A	/* Protocol Multiplexing */
+#define PCI_EXT_CAP_ID_PASID	0x1B	/* Process Address Space ID */
+#define PCI_EXT_CAP_ID_DPC	0x1D	/* Downstream Port Containment */
+#define PCI_EXT_CAP_ID_MAX	PCI_EXT_CAP_ID_DPC
+
+#define PCI_EXT_CAP_DSN_SIZEOF	12
+#define PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF 40
+
+/* Advanced Error Reporting */
+#define PCI_ERR_UNCOR_STATUS	4	/* Uncorrectable Error Status */
+#define  PCI_ERR_UNC_UND	0x00000001	/* Undefined */
+#define  PCI_ERR_UNC_DLP	0x00000010	/* Data Link Protocol */
+#define  PCI_ERR_UNC_SURPDN	0x00000020	/* Surprise Down */
+#define  PCI_ERR_UNC_POISON_TLP	0x00001000	/* Poisoned TLP */
+#define  PCI_ERR_UNC_FCP	0x00002000	/* Flow Control Protocol */
+#define  PCI_ERR_UNC_COMP_TIME	0x00004000	/* Completion Timeout */
+#define  PCI_ERR_UNC_COMP_ABORT	0x00008000	/* Completer Abort */
+#define  PCI_ERR_UNC_UNX_COMP	0x00010000	/* Unexpected Completion */
+#define  PCI_ERR_UNC_RX_OVER	0x00020000	/* Receiver Overflow */
+#define  PCI_ERR_UNC_MALF_TLP	0x00040000	/* Malformed TLP */
+#define  PCI_ERR_UNC_ECRC	0x00080000	/* ECRC Error Status */
+#define  PCI_ERR_UNC_UNSUP	0x00100000	/* Unsupported Request */
+#define  PCI_ERR_UNC_ACSV	0x00200000	/* ACS Violation */
+#define  PCI_ERR_UNC_INTN	0x00400000	/* internal error */
+#define  PCI_ERR_UNC_MCBTLP	0x00800000	/* MC blocked TLP */
+#define  PCI_ERR_UNC_ATOMEG	0x01000000	/* Atomic egress blocked */
+#define  PCI_ERR_UNC_TLPPRE	0x02000000	/* TLP prefix blocked */
+#define PCI_ERR_UNCOR_MASK	8	/* Uncorrectable Error Mask */
+	/* Same bits as above */
+#define PCI_ERR_UNCOR_SEVER	12	/* Uncorrectable Error Severity */
+	/* Same bits as above */
+#define PCI_ERR_COR_STATUS	16	/* Correctable Error Status */
+#define  PCI_ERR_COR_RCVR	0x00000001	/* Receiver Error Status */
+#define  PCI_ERR_COR_BAD_TLP	0x00000040	/* Bad TLP Status */
+#define  PCI_ERR_COR_BAD_DLLP	0x00000080	/* Bad DLLP Status */
+#define  PCI_ERR_COR_REP_ROLL	0x00000100	/* REPLAY_NUM Rollover */
+#define  PCI_ERR_COR_REP_TIMER	0x00001000	/* Replay Timer Timeout */
+#define  PCI_ERR_COR_ADV_NFAT	0x00002000	/* Advisory Non-Fatal */
+#define  PCI_ERR_COR_INTERNAL	0x00004000	/* Corrected Internal */
+#define  PCI_ERR_COR_LOG_OVER	0x00008000	/* Header Log Overflow */
+#define PCI_ERR_COR_MASK	20	/* Correctable Error Mask */
+	/* Same bits as above */
+#define PCI_ERR_CAP		24	/* Advanced Error Capabilities */
+#define  PCI_ERR_CAP_FEP(x)	((x) & 31)	/* First Error Pointer */
+#define  PCI_ERR_CAP_ECRC_GENC	0x00000020	/* ECRC Generation Capable */
+#define  PCI_ERR_CAP_ECRC_GENE	0x00000040	/* ECRC Generation Enable */
+#define  PCI_ERR_CAP_ECRC_CHKC	0x00000080	/* ECRC Check Capable */
+#define  PCI_ERR_CAP_ECRC_CHKE	0x00000100	/* ECRC Check Enable */
+#define PCI_ERR_HEADER_LOG	28	/* Header Log Register (16 bytes) */
+#define PCI_ERR_ROOT_COMMAND	44	/* Root Error Command */
+/* Correctable Err Reporting Enable */
+#define PCI_ERR_ROOT_CMD_COR_EN		0x00000001
+/* Non-fatal Err Reporting Enable */
+#define PCI_ERR_ROOT_CMD_NONFATAL_EN	0x00000002
+/* Fatal Err Reporting Enable */
+#define PCI_ERR_ROOT_CMD_FATAL_EN	0x00000004
+#define PCI_ERR_ROOT_STATUS	48
+#define PCI_ERR_ROOT_COR_RCV		0x00000001	/* ERR_COR Received */
+/* Multi ERR_COR Received */
+#define PCI_ERR_ROOT_MULTI_COR_RCV	0x00000002
+/* ERR_FATAL/NONFATAL Received */
+#define PCI_ERR_ROOT_UNCOR_RCV		0x00000004
+/* Multi ERR_FATAL/NONFATAL Received */
+#define PCI_ERR_ROOT_MULTI_UNCOR_RCV	0x00000008
+#define PCI_ERR_ROOT_FIRST_FATAL	0x00000010	/* First Fatal */
+#define PCI_ERR_ROOT_NONFATAL_RCV	0x00000020	/* Non-Fatal Received */
+#define PCI_ERR_ROOT_FATAL_RCV		0x00000040	/* Fatal Received */
+#define PCI_ERR_ROOT_ERR_SRC	52	/* Error Source Identification */
+
+/* Virtual Channel */
+#define PCI_VC_PORT_CAP1	4
+#define  PCI_VC_CAP1_EVCC	0x00000007	/* extended VC count */
+#define  PCI_VC_CAP1_LPEVCC	0x00000070	/* low prio extended VC count */
+#define  PCI_VC_CAP1_ARB_SIZE	0x00000c00
+#define PCI_VC_PORT_CAP2	8
+#define  PCI_VC_CAP2_32_PHASE		0x00000002
+#define  PCI_VC_CAP2_64_PHASE		0x00000004
+#define  PCI_VC_CAP2_128_PHASE		0x00000008
+#define  PCI_VC_CAP2_ARB_OFF		0xff000000
+#define PCI_VC_PORT_CTRL	12
+#define  PCI_VC_PORT_CTRL_LOAD_TABLE	0x00000001
+#define PCI_VC_PORT_STATUS	14
+#define  PCI_VC_PORT_STATUS_TABLE	0x00000001
+#define PCI_VC_RES_CAP		16
+#define  PCI_VC_RES_CAP_32_PHASE	0x00000002
+#define  PCI_VC_RES_CAP_64_PHASE	0x00000004
+#define  PCI_VC_RES_CAP_128_PHASE	0x00000008
+#define  PCI_VC_RES_CAP_128_PHASE_TB	0x00000010
+#define  PCI_VC_RES_CAP_256_PHASE	0x00000020
+#define  PCI_VC_RES_CAP_ARB_OFF		0xff000000
+#define PCI_VC_RES_CTRL		20
+#define  PCI_VC_RES_CTRL_LOAD_TABLE	0x00010000
+#define  PCI_VC_RES_CTRL_ARB_SELECT	0x000e0000
+#define  PCI_VC_RES_CTRL_ID		0x07000000
+#define  PCI_VC_RES_CTRL_ENABLE		0x80000000
+#define PCI_VC_RES_STATUS	26
+#define  PCI_VC_RES_STATUS_TABLE	0x00000001
+#define  PCI_VC_RES_STATUS_NEGO		0x00000002
+#define PCI_CAP_VC_BASE_SIZEOF		0x10
+#define PCI_CAP_VC_PER_VC_SIZEOF	0x0C
+
+/* Power Budgeting */
+#define PCI_PWR_DSR		4	/* Data Select Register */
+#define PCI_PWR_DATA		8	/* Data Register */
+#define  PCI_PWR_DATA_BASE(x)	((x) & 0xff)	    /* Base Power */
+#define  PCI_PWR_DATA_SCALE(x)	(((x) >> 8) & 3)    /* Data Scale */
+#define  PCI_PWR_DATA_PM_SUB(x)	(((x) >> 10) & 7)   /* PM Sub State */
+#define  PCI_PWR_DATA_PM_STATE(x) (((x) >> 13) & 3) /* PM State */
+#define  PCI_PWR_DATA_TYPE(x)	(((x) >> 15) & 7)   /* Type */
+#define  PCI_PWR_DATA_RAIL(x)	(((x) >> 18) & 7)   /* Power Rail */
+#define PCI_PWR_CAP		12	/* Capability */
+#define  PCI_PWR_CAP_BUDGET(x)	((x) & 1)	/* Included in system budget */
+#define PCI_EXT_CAP_PWR_SIZEOF	16
+
+/* Vendor-Specific (VSEC, PCI_EXT_CAP_ID_VNDR) */
+#define PCI_VNDR_HEADER		4	/* Vendor-Specific Header */
+#define  PCI_VNDR_HEADER_ID(x)	((x) & 0xffff)
+#define  PCI_VNDR_HEADER_REV(x)	(((x) >> 16) & 0xf)
+#define  PCI_VNDR_HEADER_LEN(x)	(((x) >> 20) & 0xfff)
+
+/*
+ * HyperTransport sub capability types
+ *
+ * Unfortunately there are both 3 bit and 5 bit capability types defined
+ * in the HT spec, catering for that is a little messy. You probably don't
+ * want to use these directly, just use pci_find_ht_capability() and it
+ * will do the right thing for you.
+ */
+#define HT_3BIT_CAP_MASK	0xE0
+#define HT_CAPTYPE_SLAVE	0x00	/* Slave/Primary link configuration */
+#define HT_CAPTYPE_HOST		0x20	/* Host/Secondary link configuration */
+
+#define HT_5BIT_CAP_MASK	0xF8
+#define HT_CAPTYPE_IRQ		0x80	/* IRQ Configuration */
+#define HT_CAPTYPE_REMAPPING_40	0xA0	/* 40 bit address remapping */
+#define HT_CAPTYPE_REMAPPING_64 0xA2	/* 64 bit address remapping */
+#define HT_CAPTYPE_UNITID_CLUMP	0x90	/* Unit ID clumping */
+#define HT_CAPTYPE_EXTCONF	0x98	/* Extended Configuration Space Access */
+#define HT_CAPTYPE_MSI_MAPPING	0xA8	/* MSI Mapping Capability */
+#define  HT_MSI_FLAGS		0x02		/* Offset to flags */
+#define  HT_MSI_FLAGS_ENABLE	0x1		/* Mapping enable */
+#define  HT_MSI_FLAGS_FIXED	0x2		/* Fixed mapping only */
+#define  HT_MSI_FIXED_ADDR	0x00000000FEE00000ULL	/* Fixed addr */
+#define  HT_MSI_ADDR_LO		0x04		/* Offset to low addr bits */
+#define  HT_MSI_ADDR_LO_MASK	0xFFF00000	/* Low address bit mask */
+#define  HT_MSI_ADDR_HI		0x08		/* Offset to high addr bits */
+#define HT_CAPTYPE_DIRECT_ROUTE	0xB0	/* Direct routing configuration */
+#define HT_CAPTYPE_VCSET	0xB8	/* Virtual Channel configuration */
+#define HT_CAPTYPE_ERROR_RETRY	0xC0	/* Retry on error configuration */
+#define HT_CAPTYPE_GEN3		0xD0	/* Generation 3 HyperTransport configuration */
+#define HT_CAPTYPE_PM		0xE0	/* HyperTransport power management configuration */
+#define HT_CAP_SIZEOF_LONG	28	/* slave & primary */
+#define HT_CAP_SIZEOF_SHORT	24	/* host & secondary */
+
+/* Alternative Routing-ID Interpretation */
+#define PCI_ARI_CAP		0x04	/* ARI Capability Register */
+#define  PCI_ARI_CAP_MFVC	0x0001	/* MFVC Function Groups Capability */
+#define  PCI_ARI_CAP_ACS	0x0002	/* ACS Function Groups Capability */
+#define  PCI_ARI_CAP_NFN(x)	(((x) >> 8) & 0xff) /* Next Function Number */
+#define PCI_ARI_CTRL		0x06	/* ARI Control Register */
+#define  PCI_ARI_CTRL_MFVC	0x0001	/* MFVC Function Groups Enable */
+#define  PCI_ARI_CTRL_ACS	0x0002	/* ACS Function Groups Enable */
+#define  PCI_ARI_CTRL_FG(x)	(((x) >> 4) & 7) /* Function Group */
+#define PCI_EXT_CAP_ARI_SIZEOF	8
+
+/* Address Translation Service */
+#define PCI_ATS_CAP		0x04	/* ATS Capability Register */
+#define  PCI_ATS_CAP_QDEP(x)	((x) & 0x1f)	/* Invalidate Queue Depth */
+#define  PCI_ATS_MAX_QDEP	32	/* Max Invalidate Queue Depth */
+#define PCI_ATS_CTRL		0x06	/* ATS Control Register */
+#define  PCI_ATS_CTRL_ENABLE	0x8000	/* ATS Enable */
+#define  PCI_ATS_CTRL_STU(x)	((x) & 0x1f)	/* Smallest Translation Unit */
+#define  PCI_ATS_MIN_STU	12	/* shift of minimum STU block */
+#define PCI_EXT_CAP_ATS_SIZEOF	8
+
+/* Page Request Interface */
+#define PCI_PRI_CTRL		0x04	/* PRI control register */
+#define  PCI_PRI_CTRL_ENABLE	0x01	/* Enable */
+#define  PCI_PRI_CTRL_RESET	0x02	/* Reset */
+#define PCI_PRI_STATUS		0x06	/* PRI status register */
+#define  PCI_PRI_STATUS_RF	0x001	/* Response Failure */
+#define  PCI_PRI_STATUS_UPRGI	0x002	/* Unexpected PRG index */
+#define  PCI_PRI_STATUS_STOPPED	0x100	/* PRI Stopped */
+#define PCI_PRI_MAX_REQ		0x08	/* PRI max reqs supported */
+#define PCI_PRI_ALLOC_REQ	0x0c	/* PRI max reqs allowed */
+#define PCI_EXT_CAP_PRI_SIZEOF	16
+
+/* Process Address Space ID */
+#define PCI_PASID_CAP		0x04    /* PASID feature register */
+#define  PCI_PASID_CAP_EXEC	0x02	/* Exec permissions Supported */
+#define  PCI_PASID_CAP_PRIV	0x04	/* Privilege Mode Supported */
+#define PCI_PASID_CTRL		0x06    /* PASID control register */
+#define  PCI_PASID_CTRL_ENABLE	0x01	/* Enable bit */
+#define  PCI_PASID_CTRL_EXEC	0x02	/* Exec permissions Enable */
+#define  PCI_PASID_CTRL_PRIV	0x04	/* Privilege Mode Enable */
+#define PCI_EXT_CAP_PASID_SIZEOF	8
+
+/* Single Root I/O Virtualization */
+#define PCI_SRIOV_CAP		0x04	/* SR-IOV Capabilities */
+#define  PCI_SRIOV_CAP_VFM	0x01	/* VF Migration Capable */
+#define  PCI_SRIOV_CAP_INTR(x)	((x) >> 21) /* Interrupt Message Number */
+#define PCI_SRIOV_CTRL		0x08	/* SR-IOV Control */
+#define  PCI_SRIOV_CTRL_VFE	0x01	/* VF Enable */
+#define  PCI_SRIOV_CTRL_VFM	0x02	/* VF Migration Enable */
+#define  PCI_SRIOV_CTRL_INTR	0x04	/* VF Migration Interrupt Enable */
+#define  PCI_SRIOV_CTRL_MSE	0x08	/* VF Memory Space Enable */
+#define  PCI_SRIOV_CTRL_ARI	0x10	/* ARI Capable Hierarchy */
+#define PCI_SRIOV_STATUS	0x0a	/* SR-IOV Status */
+#define  PCI_SRIOV_STATUS_VFM	0x01	/* VF Migration Status */
+#define PCI_SRIOV_INITIAL_VF	0x0c	/* Initial VFs */
+#define PCI_SRIOV_TOTAL_VF	0x0e	/* Total VFs */
+#define PCI_SRIOV_NUM_VF	0x10	/* Number of VFs */
+#define PCI_SRIOV_FUNC_LINK	0x12	/* Function Dependency Link */
+#define PCI_SRIOV_VF_OFFSET	0x14	/* First VF Offset */
+#define PCI_SRIOV_VF_STRIDE	0x16	/* Following VF Stride */
+#define PCI_SRIOV_VF_DID	0x1a	/* VF Device ID */
+#define PCI_SRIOV_SUP_PGSIZE	0x1c	/* Supported Page Sizes */
+#define PCI_SRIOV_SYS_PGSIZE	0x20	/* System Page Size */
+#define PCI_SRIOV_BAR		0x24	/* VF BAR0 */
+#define  PCI_SRIOV_NUM_BARS	6	/* Number of VF BARs */
+#define PCI_SRIOV_VFM		0x3c	/* VF Migration State Array Offset*/
+#define  PCI_SRIOV_VFM_BIR(x)	((x) & 7)	/* State BIR */
+#define  PCI_SRIOV_VFM_OFFSET(x) ((x) & ~7)	/* State Offset */
+#define  PCI_SRIOV_VFM_UA	0x0	/* Inactive.Unavailable */
+#define  PCI_SRIOV_VFM_MI	0x1	/* Dormant.MigrateIn */
+#define  PCI_SRIOV_VFM_MO	0x2	/* Active.MigrateOut */
+#define  PCI_SRIOV_VFM_AV	0x3	/* Active.Available */
+#define PCI_EXT_CAP_SRIOV_SIZEOF 64
+
+#define PCI_LTR_MAX_SNOOP_LAT	0x4
+#define PCI_LTR_MAX_NOSNOOP_LAT	0x6
+#define  PCI_LTR_VALUE_MASK	0x000003ff
+#define  PCI_LTR_SCALE_MASK	0x00001c00
+#define  PCI_LTR_SCALE_SHIFT	10
+#define PCI_EXT_CAP_LTR_SIZEOF	8
+
+/* Access Control Service */
+#define PCI_ACS_CAP		0x04	/* ACS Capability Register */
+#define  PCI_ACS_SV		0x01	/* Source Validation */
+#define  PCI_ACS_TB		0x02	/* Translation Blocking */
+#define  PCI_ACS_RR		0x04	/* P2P Request Redirect */
+#define  PCI_ACS_CR		0x08	/* P2P Completion Redirect */
+#define  PCI_ACS_UF		0x10	/* Upstream Forwarding */
+#define  PCI_ACS_EC		0x20	/* P2P Egress Control */
+#define  PCI_ACS_DT		0x40	/* Direct Translated P2P */
+#define PCI_ACS_EGRESS_BITS	0x05	/* ACS Egress Control Vector Size */
+#define PCI_ACS_CTRL		0x06	/* ACS Control Register */
+#define PCI_ACS_EGRESS_CTL_V	0x08	/* ACS Egress Control Vector */
+
+#define PCI_VSEC_HDR		4	/* extended cap - vendor-specific */
+#define  PCI_VSEC_HDR_LEN_SHIFT	20	/* shift for length field */
+
+/* SATA capability */
+#define PCI_SATA_REGS		4	/* SATA REGs specifier */
+#define  PCI_SATA_REGS_MASK	0xF	/* location - BAR#/inline */
+#define  PCI_SATA_REGS_INLINE	0xF	/* REGS in config space */
+#define PCI_SATA_SIZEOF_SHORT	8
+#define PCI_SATA_SIZEOF_LONG	16
+
+/* Resizable BARs */
+#define PCI_REBAR_CTRL		8	/* control register */
+#define  PCI_REBAR_CTRL_NBAR_MASK	(7 << 5)	/* mask for # bars */
+#define  PCI_REBAR_CTRL_NBAR_SHIFT	5	/* shift for # bars */
+
+/* Dynamic Power Allocation */
+#define PCI_DPA_CAP		4	/* capability register */
+#define  PCI_DPA_CAP_SUBSTATE_MASK	0x1F	/* # substates - 1 */
+#define PCI_DPA_BASE_SIZEOF	16	/* size with 0 substates */
+
+/* TPH Requester */
+#define PCI_TPH_CAP		4	/* capability register */
+#define  PCI_TPH_CAP_LOC_MASK	0x600	/* location mask */
+#define   PCI_TPH_LOC_NONE	0x000	/* no location */
+#define   PCI_TPH_LOC_CAP	0x200	/* in capability */
+#define   PCI_TPH_LOC_MSIX	0x400	/* in MSI-X */
+#define PCI_TPH_CAP_ST_MASK	0x07FF0000	/* st table mask */
+#define PCI_TPH_CAP_ST_SHIFT	16	/* st table shift */
+#define PCI_TPH_BASE_SIZEOF	12	/* size with no st table */
+
+/* Downstream Port Containment */
+#define PCI_EXP_DPC_CAP			4	/* DPC Capability */
+#define  PCI_EXP_DPC_CAP_RP_EXT		0x20	/* Root Port Extensions for DPC */
+#define  PCI_EXP_DPC_CAP_POISONED_TLP	0x40	/* Poisoned TLP Egress Blocking Supported */
+#define  PCI_EXP_DPC_CAP_SW_TRIGGER	0x80	/* Software Triggering Supported */
+#define  PCI_EXP_DPC_CAP_DL_ACTIVE	0x1000	/* ERR_COR signal on DL_Active supported */
+
+#define PCI_EXP_DPC_CTL			6	/* DPC control */
+#define  PCI_EXP_DPC_CTL_EN_NONFATAL 	0x02	/* Enable trigger on ERR_NONFATAL message */
+#define  PCI_EXP_DPC_CTL_INT_EN 	0x08	/* DPC Interrupt Enable */
+
+#define PCI_EXP_DPC_STATUS		8	/* DPC Status */
+#define  PCI_EXP_DPC_STATUS_TRIGGER	0x01	/* Trigger Status */
+#define  PCI_EXP_DPC_STATUS_INTERRUPT	0x08	/* Interrupt Status */
+
+#define PCI_EXP_DPC_SOURCE_ID		10	/* DPC Source Identifier */
+
+#endif /* LINUX_PCI_REGS_H */
diff --git a/include/standard-headers/linux/types.h b/include/standard-headers/linux/types.h
new file mode 100644
index 0000000..9dbbc73
--- /dev/null
+++ b/include/standard-headers/linux/types.h
@@ -0,0 +1,3 @@
+/* For QEMU all types are already defined via osdep.h, so this
+ * header does not need to do anything.
+ */
diff --git a/include/standard-headers/linux/virtio_9p.h b/include/standard-headers/linux/virtio_9p.h
new file mode 100644
index 0000000..e68f71d
--- /dev/null
+++ b/include/standard-headers/linux/virtio_9p.h
@@ -0,0 +1,44 @@
+#ifndef _LINUX_VIRTIO_9P_H
+#define _LINUX_VIRTIO_9P_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+#include "standard-headers/linux/types.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+
+/* The feature bitmap for virtio 9P */
+
+/* The mount point is specified in a config variable */
+#define VIRTIO_9P_MOUNT_TAG 0
+
+struct virtio_9p_config {
+	/* length of the tag name */
+	uint16_t tag_len;
+	/* non-NULL terminated tag name */
+	uint8_t tag[0];
+} QEMU_PACKED;
+
+#endif /* _LINUX_VIRTIO_9P_H */
diff --git a/include/standard-headers/linux/virtio_balloon.h b/include/standard-headers/linux/virtio_balloon.h
new file mode 100644
index 0000000..9d06ccd
--- /dev/null
+++ b/include/standard-headers/linux/virtio_balloon.h
@@ -0,0 +1,85 @@
+#ifndef _LINUX_VIRTIO_BALLOON_H
+#define _LINUX_VIRTIO_BALLOON_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+#include "standard-headers/linux/types.h"
+#include "standard-headers/linux/virtio_types.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+
+/* The feature bitmap for virtio balloon */
+#define VIRTIO_BALLOON_F_MUST_TELL_HOST	0 /* Tell before reclaiming pages */
+#define VIRTIO_BALLOON_F_STATS_VQ	1 /* Memory Stats virtqueue */
+#define VIRTIO_BALLOON_F_DEFLATE_ON_OOM	2 /* Deflate balloon on OOM */
+
+/* Size of a PFN in the balloon interface. */
+#define VIRTIO_BALLOON_PFN_SHIFT 12
+
+struct virtio_balloon_config {
+	/* Number of pages host wants Guest to give up. */
+	uint32_t num_pages;
+	/* Number of pages we've actually got in balloon. */
+	uint32_t actual;
+};
+
+#define VIRTIO_BALLOON_S_SWAP_IN  0   /* Amount of memory swapped in */
+#define VIRTIO_BALLOON_S_SWAP_OUT 1   /* Amount of memory swapped out */
+#define VIRTIO_BALLOON_S_MAJFLT   2   /* Number of major faults */
+#define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
+#define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
+#define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
+#define VIRTIO_BALLOON_S_AVAIL    6   /* Available memory as in /proc */
+#define VIRTIO_BALLOON_S_NR       7
+
+/*
+ * Memory statistics structure.
+ * Driver fills an array of these structures and passes to device.
+ *
+ * NOTE: fields are laid out in a way that would make compiler add padding
+ * between and after fields, so we have to use compiler-specific attributes to
+ * pack it, to disable this padding. This also often causes compiler to
+ * generate suboptimal code.
+ *
+ * We maintain this statistics structure format for backwards compatibility,
+ * but don't follow this example.
+ *
+ * If implementing a similar structure, do something like the below instead:
+ *     struct virtio_balloon_stat {
+ *         __virtio16 tag;
+ *         uint8_t reserved[6];
+ *         __virtio64 val;
+ *     };
+ *
+ * In other words, add explicit reserved fields to align field and
+ * structure boundaries at field size, avoiding compiler padding
+ * without the packed attribute.
+ */
+struct virtio_balloon_stat {
+	__virtio16 tag;
+	__virtio64 val;
+} QEMU_PACKED;
+
+#endif /* _LINUX_VIRTIO_BALLOON_H */
diff --git a/include/standard-headers/linux/virtio_blk.h b/include/standard-headers/linux/virtio_blk.h
new file mode 100644
index 0000000..ab16ec5
--- /dev/null
+++ b/include/standard-headers/linux/virtio_blk.h
@@ -0,0 +1,147 @@
+#ifndef _LINUX_VIRTIO_BLK_H
+#define _LINUX_VIRTIO_BLK_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+#include "standard-headers/linux/types.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+#include "standard-headers/linux/virtio_types.h"
+
+/* Feature bits */
+#define VIRTIO_BLK_F_SIZE_MAX	1	/* Indicates maximum segment size */
+#define VIRTIO_BLK_F_SEG_MAX	2	/* Indicates maximum # of segments */
+#define VIRTIO_BLK_F_GEOMETRY	4	/* Legacy geometry available  */
+#define VIRTIO_BLK_F_RO		5	/* Disk is read-only */
+#define VIRTIO_BLK_F_BLK_SIZE	6	/* Block size of disk is available*/
+#define VIRTIO_BLK_F_TOPOLOGY	10	/* Topology information is available */
+#define VIRTIO_BLK_F_MQ		12	/* support more than one vq */
+
+/* Legacy feature bits */
+#ifndef VIRTIO_BLK_NO_LEGACY
+#define VIRTIO_BLK_F_BARRIER	0	/* Does host support barriers? */
+#define VIRTIO_BLK_F_SCSI	7	/* Supports scsi command passthru */
+#define VIRTIO_BLK_F_FLUSH	9	/* Flush command supported */
+#define VIRTIO_BLK_F_CONFIG_WCE	11	/* Writeback mode available in config */
+/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
+#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
+#endif /* !VIRTIO_BLK_NO_LEGACY */
+
+#define VIRTIO_BLK_ID_BYTES	20	/* ID string length */
+
+struct virtio_blk_config {
+	/* The capacity (in 512-byte sectors). */
+	uint64_t capacity;
+	/* The maximum segment size (if VIRTIO_BLK_F_SIZE_MAX) */
+	uint32_t size_max;
+	/* The maximum number of segments (if VIRTIO_BLK_F_SEG_MAX) */
+	uint32_t seg_max;
+	/* geometry of the device (if VIRTIO_BLK_F_GEOMETRY) */
+	struct virtio_blk_geometry {
+		uint16_t cylinders;
+		uint8_t heads;
+		uint8_t sectors;
+	} geometry;
+
+	/* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
+	uint32_t blk_size;
+
+	/* the next 4 entries are guarded by VIRTIO_BLK_F_TOPOLOGY  */
+	/* exponent for physical block per logical block. */
+	uint8_t physical_block_exp;
+	/* alignment offset in logical blocks. */
+	uint8_t alignment_offset;
+	/* minimum I/O size without performance penalty in logical blocks. */
+	uint16_t min_io_size;
+	/* optimal sustained I/O size in logical blocks. */
+	uint32_t opt_io_size;
+
+	/* writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
+	uint8_t wce;
+	uint8_t unused;
+
+	/* number of vqs, only available when VIRTIO_BLK_F_MQ is set */
+	uint16_t num_queues;
+} QEMU_PACKED;
+
+/*
+ * Command types
+ *
+ * Usage is a bit tricky as some bits are used as flags and some are not.
+ *
+ * Rules:
+ *   VIRTIO_BLK_T_OUT may be combined with VIRTIO_BLK_T_SCSI_CMD or
+ *   VIRTIO_BLK_T_BARRIER.  VIRTIO_BLK_T_FLUSH is a command of its own
+ *   and may not be combined with any of the other flags.
+ */
+
+/* These two define direction. */
+#define VIRTIO_BLK_T_IN		0
+#define VIRTIO_BLK_T_OUT	1
+
+#ifndef VIRTIO_BLK_NO_LEGACY
+/* This bit says it's a scsi command, not an actual read or write. */
+#define VIRTIO_BLK_T_SCSI_CMD	2
+#endif /* VIRTIO_BLK_NO_LEGACY */
+
+/* Cache flush command */
+#define VIRTIO_BLK_T_FLUSH	4
+
+/* Get device ID command */
+#define VIRTIO_BLK_T_GET_ID    8
+
+#ifndef VIRTIO_BLK_NO_LEGACY
+/* Barrier before this op. */
+#define VIRTIO_BLK_T_BARRIER	0x80000000
+#endif /* !VIRTIO_BLK_NO_LEGACY */
+
+/*
+ * This comes first in the read scatter-gather list.
+ * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated,
+ * this is the first element of the read scatter-gather list.
+ */
+struct virtio_blk_outhdr {
+	/* VIRTIO_BLK_T* */
+	__virtio32 type;
+	/* io priority. */
+	__virtio32 ioprio;
+	/* Sector (ie. 512 byte offset) */
+	__virtio64 sector;
+};
+
+#ifndef VIRTIO_BLK_NO_LEGACY
+struct virtio_scsi_inhdr {
+	__virtio32 errors;
+	__virtio32 data_len;
+	__virtio32 sense_len;
+	__virtio32 residual;
+};
+#endif /* !VIRTIO_BLK_NO_LEGACY */
+
+/* And this is the final byte of the write scatter-gather list. */
+#define VIRTIO_BLK_S_OK		0
+#define VIRTIO_BLK_S_IOERR	1
+#define VIRTIO_BLK_S_UNSUPP	2
+#endif /* _LINUX_VIRTIO_BLK_H */
diff --git a/include/standard-headers/linux/virtio_config.h b/include/standard-headers/linux/virtio_config.h
new file mode 100644
index 0000000..b30d0cb
--- /dev/null
+++ b/include/standard-headers/linux/virtio_config.h
@@ -0,0 +1,66 @@
+#ifndef _LINUX_VIRTIO_CONFIG_H
+#define _LINUX_VIRTIO_CONFIG_H
+/* This header, excluding the #ifdef __KERNEL__ part, is BSD licensed so
+ * anyone can use the definitions to implement compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+
+/* Virtio devices use a standardized configuration space to define their
+ * features and pass configuration information, but each implementation can
+ * store and access that space differently. */
+#include "standard-headers/linux/types.h"
+
+/* Status byte for guest to report progress, and synchronize features. */
+/* We have seen device and processed generic fields (VIRTIO_CONFIG_F_VIRTIO) */
+#define VIRTIO_CONFIG_S_ACKNOWLEDGE	1
+/* We have found a driver for the device. */
+#define VIRTIO_CONFIG_S_DRIVER		2
+/* Driver has used its parts of the config, and is happy */
+#define VIRTIO_CONFIG_S_DRIVER_OK	4
+/* Driver has finished configuring features */
+#define VIRTIO_CONFIG_S_FEATURES_OK	8
+/* Device entered invalid state, driver must reset it */
+#define VIRTIO_CONFIG_S_NEEDS_RESET	0x40
+/* We've given up on this device. */
+#define VIRTIO_CONFIG_S_FAILED		0x80
+
+/* Some virtio feature bits (currently bits 28 through 32) are reserved for the
+ * transport being used (eg. virtio_ring), the rest are per-device feature
+ * bits. */
+#define VIRTIO_TRANSPORT_F_START	28
+#define VIRTIO_TRANSPORT_F_END		33
+
+#ifndef VIRTIO_CONFIG_NO_LEGACY
+/* Do we get callbacks when the ring is completely used, even if we've
+ * suppressed them? */
+#define VIRTIO_F_NOTIFY_ON_EMPTY	24
+
+/* Can the device handle any descriptor layout? */
+#define VIRTIO_F_ANY_LAYOUT		27
+#endif /* VIRTIO_CONFIG_NO_LEGACY */
+
+/* v1.0 compliant. */
+#define VIRTIO_F_VERSION_1		32
+
+#endif /* _LINUX_VIRTIO_CONFIG_H */
diff --git a/include/standard-headers/linux/virtio_console.h b/include/standard-headers/linux/virtio_console.h
new file mode 100644
index 0000000..0dedc9e
--- /dev/null
+++ b/include/standard-headers/linux/virtio_console.h
@@ -0,0 +1,78 @@
+/*
+ * This header, excluding the #ifdef __KERNEL__ part, is BSD licensed so
+ * anyone can use the definitions to implement compatible drivers/servers:
+ *
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Copyright (C) Red Hat, Inc., 2009, 2010, 2011
+ * Copyright (C) Amit Shah <amit.shah@redhat.com>, 2009, 2010, 2011
+ */
+#ifndef _LINUX_VIRTIO_CONSOLE_H
+#define _LINUX_VIRTIO_CONSOLE_H
+#include "standard-headers/linux/types.h"
+#include "standard-headers/linux/virtio_types.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+
+/* Feature bits */
+#define VIRTIO_CONSOLE_F_SIZE	0	/* Does host provide console size? */
+#define VIRTIO_CONSOLE_F_MULTIPORT 1	/* Does host provide multiple ports? */
+#define VIRTIO_CONSOLE_F_EMERG_WRITE 2 /* Does host support emergency write? */
+
+#define VIRTIO_CONSOLE_BAD_ID		(~(uint32_t)0)
+
+struct virtio_console_config {
+	/* colums of the screens */
+	uint16_t cols;
+	/* rows of the screens */
+	uint16_t rows;
+	/* max. number of ports this device can hold */
+	uint32_t max_nr_ports;
+	/* emergency write register */
+	uint32_t emerg_wr;
+} QEMU_PACKED;
+
+/*
+ * A message that's passed between the Host and the Guest for a
+ * particular port.
+ */
+struct virtio_console_control {
+	__virtio32 id;		/* Port number */
+	__virtio16 event;	/* The kind of control event (see below) */
+	__virtio16 value;	/* Extra information for the key */
+};
+
+/* Some events for control messages */
+#define VIRTIO_CONSOLE_DEVICE_READY	0
+#define VIRTIO_CONSOLE_PORT_ADD		1
+#define VIRTIO_CONSOLE_PORT_REMOVE	2
+#define VIRTIO_CONSOLE_PORT_READY	3
+#define VIRTIO_CONSOLE_CONSOLE_PORT	4
+#define VIRTIO_CONSOLE_RESIZE		5
+#define VIRTIO_CONSOLE_PORT_OPEN	6
+#define VIRTIO_CONSOLE_PORT_NAME	7
+
+
+#endif /* _LINUX_VIRTIO_CONSOLE_H */
diff --git a/include/standard-headers/linux/virtio_gpu.h b/include/standard-headers/linux/virtio_gpu.h
new file mode 100644
index 0000000..c1c8f07
--- /dev/null
+++ b/include/standard-headers/linux/virtio_gpu.h
@@ -0,0 +1,316 @@
+/*
+ * Virtio GPU Device
+ *
+ * Copyright Red Hat, Inc. 2013-2014
+ *
+ * Authors:
+ *     Dave Airlie <airlied@redhat.com>
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This header is BSD licensed so anyone can use the definitions
+ * to implement compatible drivers/servers:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef VIRTIO_GPU_HW_H
+#define VIRTIO_GPU_HW_H
+
+#include "standard-headers/linux/types.h"
+
+#define VIRTIO_GPU_F_VIRGL 0
+
+enum virtio_gpu_ctrl_type {
+	VIRTIO_GPU_UNDEFINED = 0,
+
+	/* 2d commands */
+	VIRTIO_GPU_CMD_GET_DISPLAY_INFO = 0x0100,
+	VIRTIO_GPU_CMD_RESOURCE_CREATE_2D,
+	VIRTIO_GPU_CMD_RESOURCE_UNREF,
+	VIRTIO_GPU_CMD_SET_SCANOUT,
+	VIRTIO_GPU_CMD_RESOURCE_FLUSH,
+	VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D,
+	VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING,
+	VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING,
+	VIRTIO_GPU_CMD_GET_CAPSET_INFO,
+	VIRTIO_GPU_CMD_GET_CAPSET,
+
+	/* 3d commands */
+	VIRTIO_GPU_CMD_CTX_CREATE = 0x0200,
+	VIRTIO_GPU_CMD_CTX_DESTROY,
+	VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE,
+	VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE,
+	VIRTIO_GPU_CMD_RESOURCE_CREATE_3D,
+	VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D,
+	VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D,
+	VIRTIO_GPU_CMD_SUBMIT_3D,
+
+	/* cursor commands */
+	VIRTIO_GPU_CMD_UPDATE_CURSOR = 0x0300,
+	VIRTIO_GPU_CMD_MOVE_CURSOR,
+
+	/* success responses */
+	VIRTIO_GPU_RESP_OK_NODATA = 0x1100,
+	VIRTIO_GPU_RESP_OK_DISPLAY_INFO,
+	VIRTIO_GPU_RESP_OK_CAPSET_INFO,
+	VIRTIO_GPU_RESP_OK_CAPSET,
+
+	/* error responses */
+	VIRTIO_GPU_RESP_ERR_UNSPEC = 0x1200,
+	VIRTIO_GPU_RESP_ERR_OUT_OF_MEMORY,
+	VIRTIO_GPU_RESP_ERR_INVALID_SCANOUT_ID,
+	VIRTIO_GPU_RESP_ERR_INVALID_RESOURCE_ID,
+	VIRTIO_GPU_RESP_ERR_INVALID_CONTEXT_ID,
+	VIRTIO_GPU_RESP_ERR_INVALID_PARAMETER,
+};
+
+#define VIRTIO_GPU_FLAG_FENCE (1 << 0)
+
+struct virtio_gpu_ctrl_hdr {
+	uint32_t type;
+	uint32_t flags;
+	uint64_t fence_id;
+	uint32_t ctx_id;
+	uint32_t padding;
+};
+
+/* data passed in the cursor vq */
+
+struct virtio_gpu_cursor_pos {
+	uint32_t scanout_id;
+	uint32_t x;
+	uint32_t y;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_UPDATE_CURSOR, VIRTIO_GPU_CMD_MOVE_CURSOR */
+struct virtio_gpu_update_cursor {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_cursor_pos pos;  /* update & move */
+	uint32_t resource_id;           /* update only */
+	uint32_t hot_x;                 /* update only */
+	uint32_t hot_y;                 /* update only */
+	uint32_t padding;
+};
+
+/* data passed in the control vq, 2d related */
+
+struct virtio_gpu_rect {
+	uint32_t x;
+	uint32_t y;
+	uint32_t width;
+	uint32_t height;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_UNREF */
+struct virtio_gpu_resource_unref {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_CREATE_2D: create a 2d resource with a format */
+struct virtio_gpu_resource_create_2d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t format;
+	uint32_t width;
+	uint32_t height;
+};
+
+/* VIRTIO_GPU_CMD_SET_SCANOUT */
+struct virtio_gpu_set_scanout {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_rect r;
+	uint32_t scanout_id;
+	uint32_t resource_id;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_FLUSH */
+struct virtio_gpu_resource_flush {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_rect r;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_TRANSFER_TO_HOST_2D: simple transfer to_host */
+struct virtio_gpu_transfer_to_host_2d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_rect r;
+	uint64_t offset;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+struct virtio_gpu_mem_entry {
+	uint64_t addr;
+	uint32_t length;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_ATTACH_BACKING */
+struct virtio_gpu_resource_attach_backing {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t nr_entries;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_DETACH_BACKING */
+struct virtio_gpu_resource_detach_backing {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_RESP_OK_DISPLAY_INFO */
+#define VIRTIO_GPU_MAX_SCANOUTS 16
+struct virtio_gpu_resp_display_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_display_one {
+		struct virtio_gpu_rect r;
+		uint32_t enabled;
+		uint32_t flags;
+	} pmodes[VIRTIO_GPU_MAX_SCANOUTS];
+};
+
+/* data passed in the control vq, 3d related */
+
+struct virtio_gpu_box {
+	uint32_t x, y, z;
+	uint32_t w, h, d;
+};
+
+/* VIRTIO_GPU_CMD_TRANSFER_TO_HOST_3D, VIRTIO_GPU_CMD_TRANSFER_FROM_HOST_3D */
+struct virtio_gpu_transfer_host_3d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	struct virtio_gpu_box box;
+	uint64_t offset;
+	uint32_t resource_id;
+	uint32_t level;
+	uint32_t stride;
+	uint32_t layer_stride;
+};
+
+/* VIRTIO_GPU_CMD_RESOURCE_CREATE_3D */
+#define VIRTIO_GPU_RESOURCE_FLAG_Y_0_TOP (1 << 0)
+struct virtio_gpu_resource_create_3d {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t target;
+	uint32_t format;
+	uint32_t bind;
+	uint32_t width;
+	uint32_t height;
+	uint32_t depth;
+	uint32_t array_size;
+	uint32_t last_level;
+	uint32_t nr_samples;
+	uint32_t flags;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_CTX_CREATE */
+struct virtio_gpu_ctx_create {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t nlen;
+	uint32_t padding;
+	char debug_name[64];
+};
+
+/* VIRTIO_GPU_CMD_CTX_DESTROY */
+struct virtio_gpu_ctx_destroy {
+	struct virtio_gpu_ctrl_hdr hdr;
+};
+
+/* VIRTIO_GPU_CMD_CTX_ATTACH_RESOURCE, VIRTIO_GPU_CMD_CTX_DETACH_RESOURCE */
+struct virtio_gpu_ctx_resource {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t resource_id;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_SUBMIT_3D */
+struct virtio_gpu_cmd_submit {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t size;
+	uint32_t padding;
+};
+
+#define VIRTIO_GPU_CAPSET_VIRGL 1
+
+/* VIRTIO_GPU_CMD_GET_CAPSET_INFO */
+struct virtio_gpu_get_capset_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_index;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET_INFO */
+struct virtio_gpu_resp_capset_info {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_id;
+	uint32_t capset_max_version;
+	uint32_t capset_max_size;
+	uint32_t padding;
+};
+
+/* VIRTIO_GPU_CMD_GET_CAPSET */
+struct virtio_gpu_get_capset {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint32_t capset_id;
+	uint32_t capset_version;
+};
+
+/* VIRTIO_GPU_RESP_OK_CAPSET */
+struct virtio_gpu_resp_capset {
+	struct virtio_gpu_ctrl_hdr hdr;
+	uint8_t capset_data[];
+};
+
+#define VIRTIO_GPU_EVENT_DISPLAY (1 << 0)
+
+struct virtio_gpu_config {
+	uint32_t events_read;
+	uint32_t events_clear;
+	uint32_t num_scanouts;
+	uint32_t num_capsets;
+};
+
+/* simple formats for fbcon/X use */
+enum virtio_gpu_formats {
+	VIRTIO_GPU_FORMAT_B8G8R8A8_UNORM  = 1,
+	VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM  = 2,
+	VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM  = 3,
+	VIRTIO_GPU_FORMAT_X8R8G8B8_UNORM  = 4,
+
+	VIRTIO_GPU_FORMAT_R8G8B8A8_UNORM  = 67,
+	VIRTIO_GPU_FORMAT_X8B8G8R8_UNORM  = 68,
+
+	VIRTIO_GPU_FORMAT_A8B8G8R8_UNORM  = 121,
+	VIRTIO_GPU_FORMAT_R8G8B8X8_UNORM  = 134,
+};
+
+#endif
diff --git a/include/standard-headers/linux/virtio_ids.h b/include/standard-headers/linux/virtio_ids.h
new file mode 100644
index 0000000..77925f5
--- /dev/null
+++ b/include/standard-headers/linux/virtio_ids.h
@@ -0,0 +1,45 @@
+#ifndef _LINUX_VIRTIO_IDS_H
+#define _LINUX_VIRTIO_IDS_H
+/*
+ * Virtio IDs
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+
+#define VIRTIO_ID_NET		1 /* virtio net */
+#define VIRTIO_ID_BLOCK		2 /* virtio block */
+#define VIRTIO_ID_CONSOLE	3 /* virtio console */
+#define VIRTIO_ID_RNG		4 /* virtio rng */
+#define VIRTIO_ID_BALLOON	5 /* virtio balloon */
+#define VIRTIO_ID_RPMSG		7 /* virtio remote processor messaging */
+#define VIRTIO_ID_SCSI		8 /* virtio scsi */
+#define VIRTIO_ID_9P		9 /* 9p virtio console */
+#define VIRTIO_ID_RPROC_SERIAL 11 /* virtio remoteproc serial link */
+#define VIRTIO_ID_CAIF	       12 /* Virtio caif */
+#define VIRTIO_ID_GPU          16 /* virtio GPU */
+#define VIRTIO_ID_INPUT        18 /* virtio input */
+
+#endif /* _LINUX_VIRTIO_IDS_H */
diff --git a/include/standard-headers/linux/virtio_input.h b/include/standard-headers/linux/virtio_input.h
new file mode 100644
index 0000000..a98a797
--- /dev/null
+++ b/include/standard-headers/linux/virtio_input.h
@@ -0,0 +1,76 @@
+#ifndef _LINUX_VIRTIO_INPUT_H
+#define _LINUX_VIRTIO_INPUT_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+
+#include "standard-headers/linux/types.h"
+
+enum virtio_input_config_select {
+	VIRTIO_INPUT_CFG_UNSET      = 0x00,
+	VIRTIO_INPUT_CFG_ID_NAME    = 0x01,
+	VIRTIO_INPUT_CFG_ID_SERIAL  = 0x02,
+	VIRTIO_INPUT_CFG_ID_DEVIDS  = 0x03,
+	VIRTIO_INPUT_CFG_PROP_BITS  = 0x10,
+	VIRTIO_INPUT_CFG_EV_BITS    = 0x11,
+	VIRTIO_INPUT_CFG_ABS_INFO   = 0x12,
+};
+
+struct virtio_input_absinfo {
+	uint32_t min;
+	uint32_t max;
+	uint32_t fuzz;
+	uint32_t flat;
+	uint32_t res;
+};
+
+struct virtio_input_devids {
+	uint16_t bustype;
+	uint16_t vendor;
+	uint16_t product;
+	uint16_t version;
+};
+
+struct virtio_input_config {
+	uint8_t    select;
+	uint8_t    subsel;
+	uint8_t    size;
+	uint8_t    reserved[5];
+	union {
+		char string[128];
+		uint8_t bitmap[128];
+		struct virtio_input_absinfo abs;
+		struct virtio_input_devids ids;
+	} u;
+};
+
+struct virtio_input_event {
+	uint16_t type;
+	uint16_t code;
+	uint32_t value;
+};
+
+#endif /* _LINUX_VIRTIO_INPUT_H */
diff --git a/include/standard-headers/linux/virtio_net.h b/include/standard-headers/linux/virtio_net.h
new file mode 100644
index 0000000..a78f33e
--- /dev/null
+++ b/include/standard-headers/linux/virtio_net.h
@@ -0,0 +1,245 @@
+#ifndef _LINUX_VIRTIO_NET_H
+#define _LINUX_VIRTIO_NET_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE. */
+#include "standard-headers/linux/types.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+#include "standard-headers/linux/virtio_types.h"
+#include "standard-headers/linux/if_ether.h"
+
+/* The feature bitmap for virtio net */
+#define VIRTIO_NET_F_CSUM	0	/* Host handles pkts w/ partial csum */
+#define VIRTIO_NET_F_GUEST_CSUM	1	/* Guest handles pkts w/ partial csum */
+#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 2 /* Dynamic offload configuration. */
+#define VIRTIO_NET_F_MAC	5	/* Host has given MAC address. */
+#define VIRTIO_NET_F_GUEST_TSO4	7	/* Guest can handle TSOv4 in. */
+#define VIRTIO_NET_F_GUEST_TSO6	8	/* Guest can handle TSOv6 in. */
+#define VIRTIO_NET_F_GUEST_ECN	9	/* Guest can handle TSO[6] w/ ECN in. */
+#define VIRTIO_NET_F_GUEST_UFO	10	/* Guest can handle UFO in. */
+#define VIRTIO_NET_F_HOST_TSO4	11	/* Host can handle TSOv4 in. */
+#define VIRTIO_NET_F_HOST_TSO6	12	/* Host can handle TSOv6 in. */
+#define VIRTIO_NET_F_HOST_ECN	13	/* Host can handle TSO[6] w/ ECN in. */
+#define VIRTIO_NET_F_HOST_UFO	14	/* Host can handle UFO in. */
+#define VIRTIO_NET_F_MRG_RXBUF	15	/* Host can merge receive buffers. */
+#define VIRTIO_NET_F_STATUS	16	/* virtio_net_config.status available */
+#define VIRTIO_NET_F_CTRL_VQ	17	/* Control channel available */
+#define VIRTIO_NET_F_CTRL_RX	18	/* Control channel RX mode support */
+#define VIRTIO_NET_F_CTRL_VLAN	19	/* Control channel VLAN filtering */
+#define VIRTIO_NET_F_CTRL_RX_EXTRA 20	/* Extra RX mode control support */
+#define VIRTIO_NET_F_GUEST_ANNOUNCE 21	/* Guest can announce device on the
+					 * network */
+#define VIRTIO_NET_F_MQ	22	/* Device supports Receive Flow
+					 * Steering */
+#define VIRTIO_NET_F_CTRL_MAC_ADDR 23	/* Set MAC address */
+
+#ifndef VIRTIO_NET_NO_LEGACY
+#define VIRTIO_NET_F_GSO	6	/* Host handles pkts w/ any GSO type */
+#endif /* VIRTIO_NET_NO_LEGACY */
+
+#define VIRTIO_NET_S_LINK_UP	1	/* Link is up */
+#define VIRTIO_NET_S_ANNOUNCE	2	/* Announcement is needed */
+
+struct virtio_net_config {
+	/* The config defining mac address (if VIRTIO_NET_F_MAC) */
+	uint8_t mac[ETH_ALEN];
+	/* See VIRTIO_NET_F_STATUS and VIRTIO_NET_S_* above */
+	uint16_t status;
+	/* Maximum number of each of transmit and receive queues;
+	 * see VIRTIO_NET_F_MQ and VIRTIO_NET_CTRL_MQ.
+	 * Legal values are between 1 and 0x8000
+	 */
+	uint16_t max_virtqueue_pairs;
+} QEMU_PACKED;
+
+/*
+ * This header comes first in the scatter-gather list.  If you don't
+ * specify GSO or CSUM features, you can simply ignore the header.
+ *
+ * This is bitwise-equivalent to the legacy struct virtio_net_hdr_mrg_rxbuf,
+ * only flattened.
+ */
+struct virtio_net_hdr_v1 {
+#define VIRTIO_NET_HDR_F_NEEDS_CSUM	1	/* Use csum_start, csum_offset */
+#define VIRTIO_NET_HDR_F_DATA_VALID	2	/* Csum is valid */
+	uint8_t flags;
+#define VIRTIO_NET_HDR_GSO_NONE		0	/* Not a GSO frame */
+#define VIRTIO_NET_HDR_GSO_TCPV4	1	/* GSO frame, IPv4 TCP (TSO) */
+#define VIRTIO_NET_HDR_GSO_UDP		3	/* GSO frame, IPv4 UDP (UFO) */
+#define VIRTIO_NET_HDR_GSO_TCPV6	4	/* GSO frame, IPv6 TCP */
+#define VIRTIO_NET_HDR_GSO_ECN		0x80	/* TCP has ECN set */
+	uint8_t gso_type;
+	__virtio16 hdr_len;	/* Ethernet + IP + tcp/udp hdrs */
+	__virtio16 gso_size;	/* Bytes to append to hdr_len per frame */
+	__virtio16 csum_start;	/* Position to start checksumming from */
+	__virtio16 csum_offset;	/* Offset after that to place checksum */
+	__virtio16 num_buffers;	/* Number of merged rx buffers */
+};
+
+#ifndef VIRTIO_NET_NO_LEGACY
+/* This header comes first in the scatter-gather list.
+ * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated, it must
+ * be the first element of the scatter-gather list.  If you don't
+ * specify GSO or CSUM features, you can simply ignore the header. */
+struct virtio_net_hdr {
+	/* See VIRTIO_NET_HDR_F_* */
+	uint8_t flags;
+	/* See VIRTIO_NET_HDR_GSO_* */
+	uint8_t gso_type;
+	__virtio16 hdr_len;		/* Ethernet + IP + tcp/udp hdrs */
+	__virtio16 gso_size;		/* Bytes to append to hdr_len per frame */
+	__virtio16 csum_start;	/* Position to start checksumming from */
+	__virtio16 csum_offset;	/* Offset after that to place checksum */
+};
+
+/* This is the version of the header to use when the MRG_RXBUF
+ * feature has been negotiated. */
+struct virtio_net_hdr_mrg_rxbuf {
+	struct virtio_net_hdr hdr;
+	__virtio16 num_buffers;	/* Number of merged rx buffers */
+};
+#endif /* ...VIRTIO_NET_NO_LEGACY */
+
+/*
+ * Control virtqueue data structures
+ *
+ * The control virtqueue expects a header in the first sg entry
+ * and an ack/status response in the last entry.  Data for the
+ * command goes in between.
+ */
+struct virtio_net_ctrl_hdr {
+	uint8_t class;
+	uint8_t cmd;
+} QEMU_PACKED;
+
+typedef uint8_t virtio_net_ctrl_ack;
+
+#define VIRTIO_NET_OK     0
+#define VIRTIO_NET_ERR    1
+
+/*
+ * Control the RX mode, ie. promisucous, allmulti, etc...
+ * All commands require an "out" sg entry containing a 1 byte
+ * state value, zero = disable, non-zero = enable.  Commands
+ * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
+ * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
+ */
+#define VIRTIO_NET_CTRL_RX    0
+ #define VIRTIO_NET_CTRL_RX_PROMISC      0
+ #define VIRTIO_NET_CTRL_RX_ALLMULTI     1
+ #define VIRTIO_NET_CTRL_RX_ALLUNI       2
+ #define VIRTIO_NET_CTRL_RX_NOMULTI      3
+ #define VIRTIO_NET_CTRL_RX_NOUNI        4
+ #define VIRTIO_NET_CTRL_RX_NOBCAST      5
+
+/*
+ * Control the MAC
+ *
+ * The MAC filter table is managed by the hypervisor, the guest should
+ * assume the size is infinite.  Filtering should be considered
+ * non-perfect, ie. based on hypervisor resources, the guest may
+ * received packets from sources not specified in the filter list.
+ *
+ * In addition to the class/cmd header, the TABLE_SET command requires
+ * two out scatterlists.  Each contains a 4 byte count of entries followed
+ * by a concatenated byte stream of the ETH_ALEN MAC addresses.  The
+ * first sg list contains unicast addresses, the second is for multicast.
+ * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
+ * is available.
+ *
+ * The ADDR_SET command requests one out scatterlist, it contains a
+ * 6 bytes MAC address. This functionality is present if the
+ * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
+ */
+struct virtio_net_ctrl_mac {
+	__virtio32 entries;
+	uint8_t macs[][ETH_ALEN];
+} QEMU_PACKED;
+
+#define VIRTIO_NET_CTRL_MAC    1
+ #define VIRTIO_NET_CTRL_MAC_TABLE_SET        0
+ #define VIRTIO_NET_CTRL_MAC_ADDR_SET         1
+
+/*
+ * Control VLAN filtering
+ *
+ * The VLAN filter table is controlled via a simple ADD/DEL interface.
+ * VLAN IDs not added may be filterd by the hypervisor.  Del is the
+ * opposite of add.  Both commands expect an out entry containing a 2
+ * byte VLAN ID.  VLAN filterting is available with the
+ * VIRTIO_NET_F_CTRL_VLAN feature bit.
+ */
+#define VIRTIO_NET_CTRL_VLAN       2
+ #define VIRTIO_NET_CTRL_VLAN_ADD             0
+ #define VIRTIO_NET_CTRL_VLAN_DEL             1
+
+/*
+ * Control link announce acknowledgement
+ *
+ * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that
+ * driver has recevied the notification; device would clear the
+ * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives
+ * this command.
+ */
+#define VIRTIO_NET_CTRL_ANNOUNCE       3
+ #define VIRTIO_NET_CTRL_ANNOUNCE_ACK         0
+
+/*
+ * Control Receive Flow Steering
+ *
+ * The command VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
+ * enables Receive Flow Steering, specifying the number of the transmit and
+ * receive queues that will be used. After the command is consumed and acked by
+ * the device, the device will not steer new packets on receive virtqueues
+ * other than specified nor read from transmit virtqueues other than specified.
+ * Accordingly, driver should not transmit new packets  on virtqueues other than
+ * specified.
+ */
+struct virtio_net_ctrl_mq {
+	__virtio16 virtqueue_pairs;
+};
+
+#define VIRTIO_NET_CTRL_MQ   4
+ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET        0
+ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN        1
+ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX        0x8000
+
+/*
+ * Control network offloads
+ *
+ * Reconfigures the network offloads that Guest can handle.
+ *
+ * Available with the VIRTIO_NET_F_CTRL_GUEST_OFFLOADS feature bit.
+ *
+ * Command data format matches the feature bit mask exactly.
+ *
+ * See VIRTIO_NET_F_GUEST_* for the list of offloads
+ * that can be enabled/disabled.
+ */
+#define VIRTIO_NET_CTRL_GUEST_OFFLOADS   5
+#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET        0
+
+#endif /* _LINUX_VIRTIO_NET_H */
diff --git a/include/standard-headers/linux/virtio_pci.h b/include/standard-headers/linux/virtio_pci.h
new file mode 100644
index 0000000..9262acd
--- /dev/null
+++ b/include/standard-headers/linux/virtio_pci.h
@@ -0,0 +1,199 @@
+/*
+ * Virtio PCI driver
+ *
+ * This module allows virtio devices to be used over a virtual PCI device.
+ * This can be used with QEMU based VMMs like KVM or Xen.
+ *
+ * Copyright IBM Corp. 2007
+ *
+ * Authors:
+ *  Anthony Liguori  <aliguori@us.ibm.com>
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _LINUX_VIRTIO_PCI_H
+#define _LINUX_VIRTIO_PCI_H
+
+#include "standard-headers/linux/types.h"
+
+#ifndef VIRTIO_PCI_NO_LEGACY
+
+/* A 32-bit r/o bitmask of the features supported by the host */
+#define VIRTIO_PCI_HOST_FEATURES	0
+
+/* A 32-bit r/w bitmask of features activated by the guest */
+#define VIRTIO_PCI_GUEST_FEATURES	4
+
+/* A 32-bit r/w PFN for the currently selected queue */
+#define VIRTIO_PCI_QUEUE_PFN		8
+
+/* A 16-bit r/o queue size for the currently selected queue */
+#define VIRTIO_PCI_QUEUE_NUM		12
+
+/* A 16-bit r/w queue selector */
+#define VIRTIO_PCI_QUEUE_SEL		14
+
+/* A 16-bit r/w queue notifier */
+#define VIRTIO_PCI_QUEUE_NOTIFY		16
+
+/* An 8-bit device status register.  */
+#define VIRTIO_PCI_STATUS		18
+
+/* An 8-bit r/o interrupt status register.  Reading the value will return the
+ * current contents of the ISR and will also clear it.  This is effectively
+ * a read-and-acknowledge. */
+#define VIRTIO_PCI_ISR			19
+
+/* MSI-X registers: only enabled if MSI-X is enabled. */
+/* A 16-bit vector for configuration changes. */
+#define VIRTIO_MSI_CONFIG_VECTOR        20
+/* A 16-bit vector for selected queue notifications. */
+#define VIRTIO_MSI_QUEUE_VECTOR         22
+
+/* The remaining space is defined by each driver as the per-driver
+ * configuration space */
+#define VIRTIO_PCI_CONFIG_OFF(msix_enabled)	((msix_enabled) ? 24 : 20)
+/* Deprecated: please use VIRTIO_PCI_CONFIG_OFF instead */
+#define VIRTIO_PCI_CONFIG(dev)	VIRTIO_PCI_CONFIG_OFF((dev)->msix_enabled)
+
+/* Virtio ABI version, this must match exactly */
+#define VIRTIO_PCI_ABI_VERSION		0
+
+/* How many bits to shift physical queue address written to QUEUE_PFN.
+ * 12 is historical, and due to x86 page size. */
+#define VIRTIO_PCI_QUEUE_ADDR_SHIFT	12
+
+/* The alignment to use between consumer and producer parts of vring.
+ * x86 pagesize again. */
+#define VIRTIO_PCI_VRING_ALIGN		4096
+
+#endif /* VIRTIO_PCI_NO_LEGACY */
+
+/* The bit of the ISR which indicates a device configuration change. */
+#define VIRTIO_PCI_ISR_CONFIG		0x2
+/* Vector value used to disable MSI for queue */
+#define VIRTIO_MSI_NO_VECTOR            0xffff
+
+#ifndef VIRTIO_PCI_NO_MODERN
+
+/* IDs for different capabilities.  Must all exist. */
+
+/* Common configuration */
+#define VIRTIO_PCI_CAP_COMMON_CFG	1
+/* Notifications */
+#define VIRTIO_PCI_CAP_NOTIFY_CFG	2
+/* ISR access */
+#define VIRTIO_PCI_CAP_ISR_CFG		3
+/* Device specific configuration */
+#define VIRTIO_PCI_CAP_DEVICE_CFG	4
+/* PCI configuration access */
+#define VIRTIO_PCI_CAP_PCI_CFG		5
+
+/* This is the PCI capability header: */
+struct virtio_pci_cap {
+	uint8_t cap_vndr;		/* Generic PCI field: PCI_CAP_ID_VNDR */
+	uint8_t cap_next;		/* Generic PCI field: next ptr. */
+	uint8_t cap_len;		/* Generic PCI field: capability length */
+	uint8_t cfg_type;		/* Identifies the structure. */
+	uint8_t bar;		/* Where to find it. */
+	uint8_t padding[3];	/* Pad to full dword. */
+	uint32_t offset;		/* Offset within bar. */
+	uint32_t length;		/* Length of the structure, in bytes. */
+};
+
+struct virtio_pci_notify_cap {
+	struct virtio_pci_cap cap;
+	uint32_t notify_off_multiplier;	/* Multiplier for queue_notify_off. */
+};
+
+/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */
+struct virtio_pci_common_cfg {
+	/* About the whole device. */
+	uint32_t device_feature_select;	/* read-write */
+	uint32_t device_feature;		/* read-only */
+	uint32_t guest_feature_select;	/* read-write */
+	uint32_t guest_feature;		/* read-write */
+	uint16_t msix_config;		/* read-write */
+	uint16_t num_queues;		/* read-only */
+	uint8_t device_status;		/* read-write */
+	uint8_t config_generation;		/* read-only */
+
+	/* About a specific virtqueue. */
+	uint16_t queue_select;		/* read-write */
+	uint16_t queue_size;		/* read-write, power of 2. */
+	uint16_t queue_msix_vector;	/* read-write */
+	uint16_t queue_enable;		/* read-write */
+	uint16_t queue_notify_off;	/* read-only */
+	uint32_t queue_desc_lo;		/* read-write */
+	uint32_t queue_desc_hi;		/* read-write */
+	uint32_t queue_avail_lo;		/* read-write */
+	uint32_t queue_avail_hi;		/* read-write */
+	uint32_t queue_used_lo;		/* read-write */
+	uint32_t queue_used_hi;		/* read-write */
+};
+
+/* Fields in VIRTIO_PCI_CAP_PCI_CFG: */
+struct virtio_pci_cfg_cap {
+	struct virtio_pci_cap cap;
+	uint8_t pci_cfg_data[4]; /* Data for BAR access. */
+};
+
+/* Macro versions of offsets for the Old Timers! */
+#define VIRTIO_PCI_CAP_VNDR		0
+#define VIRTIO_PCI_CAP_NEXT		1
+#define VIRTIO_PCI_CAP_LEN		2
+#define VIRTIO_PCI_CAP_CFG_TYPE		3
+#define VIRTIO_PCI_CAP_BAR		4
+#define VIRTIO_PCI_CAP_OFFSET		8
+#define VIRTIO_PCI_CAP_LENGTH		12
+
+#define VIRTIO_PCI_NOTIFY_CAP_MULT	16
+
+#define VIRTIO_PCI_COMMON_DFSELECT	0
+#define VIRTIO_PCI_COMMON_DF		4
+#define VIRTIO_PCI_COMMON_GFSELECT	8
+#define VIRTIO_PCI_COMMON_GF		12
+#define VIRTIO_PCI_COMMON_MSIX		16
+#define VIRTIO_PCI_COMMON_NUMQ		18
+#define VIRTIO_PCI_COMMON_STATUS	20
+#define VIRTIO_PCI_COMMON_CFGGENERATION	21
+#define VIRTIO_PCI_COMMON_Q_SELECT	22
+#define VIRTIO_PCI_COMMON_Q_SIZE	24
+#define VIRTIO_PCI_COMMON_Q_MSIX	26
+#define VIRTIO_PCI_COMMON_Q_ENABLE	28
+#define VIRTIO_PCI_COMMON_Q_NOFF	30
+#define VIRTIO_PCI_COMMON_Q_DESCLO	32
+#define VIRTIO_PCI_COMMON_Q_DESCHI	36
+#define VIRTIO_PCI_COMMON_Q_AVAILLO	40
+#define VIRTIO_PCI_COMMON_Q_AVAILHI	44
+#define VIRTIO_PCI_COMMON_Q_USEDLO	48
+#define VIRTIO_PCI_COMMON_Q_USEDHI	52
+
+#endif /* VIRTIO_PCI_NO_MODERN */
+
+#endif
diff --git a/include/standard-headers/linux/virtio_ring.h b/include/standard-headers/linux/virtio_ring.h
new file mode 100644
index 0000000..023c6db
--- /dev/null
+++ b/include/standard-headers/linux/virtio_ring.h
@@ -0,0 +1,172 @@
+#ifndef _LINUX_VIRTIO_RING_H
+#define _LINUX_VIRTIO_RING_H
+/* An interface for efficient virtio implementation, currently for use by KVM
+ * and lguest, but hopefully others soon.  Do NOT change this since it will
+ * break existing servers and clients.
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Copyright Rusty Russell IBM Corporation 2007. */
+#include <stdint.h>
+#include "standard-headers/linux/types.h"
+#include "standard-headers/linux/virtio_types.h"
+
+/* This marks a buffer as continuing via the next field. */
+#define VRING_DESC_F_NEXT	1
+/* This marks a buffer as write-only (otherwise read-only). */
+#define VRING_DESC_F_WRITE	2
+/* This means the buffer contains a list of buffer descriptors. */
+#define VRING_DESC_F_INDIRECT	4
+
+/* The Host uses this in used->flags to advise the Guest: don't kick me when
+ * you add a buffer.  It's unreliable, so it's simply an optimization.  Guest
+ * will still kick if it's out of buffers. */
+#define VRING_USED_F_NO_NOTIFY	1
+/* The Guest uses this in avail->flags to advise the Host: don't interrupt me
+ * when you consume a buffer.  It's unreliable, so it's simply an
+ * optimization.  */
+#define VRING_AVAIL_F_NO_INTERRUPT	1
+
+/* We support indirect buffer descriptors */
+#define VIRTIO_RING_F_INDIRECT_DESC	28
+
+/* The Guest publishes the used index for which it expects an interrupt
+ * at the end of the avail ring. Host should ignore the avail->flags field. */
+/* The Host publishes the avail index for which it expects a kick
+ * at the end of the used ring. Guest should ignore the used->flags field. */
+#define VIRTIO_RING_F_EVENT_IDX		29
+
+/* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
+struct vring_desc {
+	/* Address (guest-physical). */
+	__virtio64 addr;
+	/* Length. */
+	__virtio32 len;
+	/* The flags as indicated above. */
+	__virtio16 flags;
+	/* We chain unused descriptors via this, too */
+	__virtio16 next;
+};
+
+struct vring_avail {
+	__virtio16 flags;
+	__virtio16 idx;
+	__virtio16 ring[];
+};
+
+/* u32 is used here for ids for padding reasons. */
+struct vring_used_elem {
+	/* Index of start of used descriptor chain. */
+	__virtio32 id;
+	/* Total length of the descriptor chain which was used (written to) */
+	__virtio32 len;
+};
+
+struct vring_used {
+	__virtio16 flags;
+	__virtio16 idx;
+	struct vring_used_elem ring[];
+};
+
+struct vring {
+	unsigned int num;
+
+	struct vring_desc *desc;
+
+	struct vring_avail *avail;
+
+	struct vring_used *used;
+};
+
+/* Alignment requirements for vring elements.
+ * When using pre-virtio 1.0 layout, these fall out naturally.
+ */
+#define VRING_AVAIL_ALIGN_SIZE 2
+#define VRING_USED_ALIGN_SIZE 4
+#define VRING_DESC_ALIGN_SIZE 16
+
+/* The standard layout for the ring is a continuous chunk of memory which looks
+ * like this.  We assume num is a power of 2.
+ *
+ * struct vring
+ * {
+ *	// The actual descriptors (16 bytes each)
+ *	struct vring_desc desc[num];
+ *
+ *	// A ring of available descriptor heads with free-running index.
+ *	__virtio16 avail_flags;
+ *	__virtio16 avail_idx;
+ *	__virtio16 available[num];
+ *	__virtio16 used_event_idx;
+ *
+ *	// Padding to the next align boundary.
+ *	char pad[];
+ *
+ *	// A ring of used descriptor heads with free-running index.
+ *	__virtio16 used_flags;
+ *	__virtio16 used_idx;
+ *	struct vring_used_elem used[num];
+ *	__virtio16 avail_event_idx;
+ * };
+ */
+/* We publish the used event index at the end of the available ring, and vice
+ * versa. They are at the end for backwards compatibility. */
+#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num])
+#define vring_avail_event(vr) (*(__virtio16 *)&(vr)->used->ring[(vr)->num])
+
+static inline void vring_init(struct vring *vr, unsigned int num, void *p,
+			      unsigned long align)
+{
+	vr->num = num;
+	vr->desc = p;
+	vr->avail = p + num*sizeof(struct vring_desc);
+	vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + sizeof(__virtio16)
+		+ align-1) & ~(align - 1));
+}
+
+static inline unsigned vring_size(unsigned int num, unsigned long align)
+{
+	return ((sizeof(struct vring_desc) * num + sizeof(__virtio16) * (3 + num)
+		 + align - 1) & ~(align - 1))
+		+ sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num;
+}
+
+/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */
+/* Assuming a given event_idx value from the other side, if
+ * we have just incremented index from old to new_idx,
+ * should we trigger an event? */
+static inline int vring_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
+{
+	/* Note: Xen has similar logic for notification hold-off
+	 * in include/xen/interface/io/ring.h with req_event and req_prod
+	 * corresponding to event_idx + 1 and new_idx respectively.
+	 * Note also that req_event and req_prod in Xen start at 1,
+	 * event indexes in virtio start at 0. */
+	return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old);
+}
+
+#endif /* _LINUX_VIRTIO_RING_H */
diff --git a/include/standard-headers/linux/virtio_rng.h b/include/standard-headers/linux/virtio_rng.h
new file mode 100644
index 0000000..60fc798
--- /dev/null
+++ b/include/standard-headers/linux/virtio_rng.h
@@ -0,0 +1,8 @@
+#ifndef _LINUX_VIRTIO_RNG_H
+#define _LINUX_VIRTIO_RNG_H
+/* This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers. */
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+
+#endif /* _LINUX_VIRTIO_RNG_H */
diff --git a/include/standard-headers/linux/virtio_scsi.h b/include/standard-headers/linux/virtio_scsi.h
new file mode 100644
index 0000000..ab66166
--- /dev/null
+++ b/include/standard-headers/linux/virtio_scsi.h
@@ -0,0 +1,172 @@
+/*
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _LINUX_VIRTIO_SCSI_H
+#define _LINUX_VIRTIO_SCSI_H
+
+#include "standard-headers/linux/virtio_types.h"
+
+/* Default values of the CDB and sense data size configuration fields */
+#define VIRTIO_SCSI_CDB_DEFAULT_SIZE   32
+#define VIRTIO_SCSI_SENSE_DEFAULT_SIZE 96
+
+#ifndef VIRTIO_SCSI_CDB_SIZE
+#define VIRTIO_SCSI_CDB_SIZE VIRTIO_SCSI_CDB_DEFAULT_SIZE
+#endif
+#ifndef VIRTIO_SCSI_SENSE_SIZE
+#define VIRTIO_SCSI_SENSE_SIZE VIRTIO_SCSI_SENSE_DEFAULT_SIZE
+#endif
+
+/* SCSI command request, followed by data-out */
+struct virtio_scsi_cmd_req {
+	uint8_t lun[8];		/* Logical Unit Number */
+	__virtio64 tag;		/* Command identifier */
+	uint8_t task_attr;		/* Task attribute */
+	uint8_t prio;		/* SAM command priority field */
+	uint8_t crn;
+	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
+} QEMU_PACKED;
+
+/* SCSI command request, followed by protection information */
+struct virtio_scsi_cmd_req_pi {
+	uint8_t lun[8];		/* Logical Unit Number */
+	__virtio64 tag;		/* Command identifier */
+	uint8_t task_attr;		/* Task attribute */
+	uint8_t prio;		/* SAM command priority field */
+	uint8_t crn;
+	__virtio32 pi_bytesout;	/* DataOUT PI Number of bytes */
+	__virtio32 pi_bytesin;		/* DataIN PI Number of bytes */
+	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
+} QEMU_PACKED;
+
+/* Response, followed by sense data and data-in */
+struct virtio_scsi_cmd_resp {
+	__virtio32 sense_len;		/* Sense data length */
+	__virtio32 resid;		/* Residual bytes in data buffer */
+	__virtio16 status_qualifier;	/* Status qualifier */
+	uint8_t status;		/* Command completion status */
+	uint8_t response;		/* Response values */
+	uint8_t sense[VIRTIO_SCSI_SENSE_SIZE];
+} QEMU_PACKED;
+
+/* Task Management Request */
+struct virtio_scsi_ctrl_tmf_req {
+	__virtio32 type;
+	__virtio32 subtype;
+	uint8_t lun[8];
+	__virtio64 tag;
+} QEMU_PACKED;
+
+struct virtio_scsi_ctrl_tmf_resp {
+	uint8_t response;
+} QEMU_PACKED;
+
+/* Asynchronous notification query/subscription */
+struct virtio_scsi_ctrl_an_req {
+	__virtio32 type;
+	uint8_t lun[8];
+	__virtio32 event_requested;
+} QEMU_PACKED;
+
+struct virtio_scsi_ctrl_an_resp {
+	__virtio32 event_actual;
+	uint8_t response;
+} QEMU_PACKED;
+
+struct virtio_scsi_event {
+	__virtio32 event;
+	uint8_t lun[8];
+	__virtio32 reason;
+} QEMU_PACKED;
+
+struct virtio_scsi_config {
+	uint32_t num_queues;
+	uint32_t seg_max;
+	uint32_t max_sectors;
+	uint32_t cmd_per_lun;
+	uint32_t event_info_size;
+	uint32_t sense_size;
+	uint32_t cdb_size;
+	uint16_t max_channel;
+	uint16_t max_target;
+	uint32_t max_lun;
+} QEMU_PACKED;
+
+/* Feature Bits */
+#define VIRTIO_SCSI_F_INOUT                    0
+#define VIRTIO_SCSI_F_HOTPLUG                  1
+#define VIRTIO_SCSI_F_CHANGE                   2
+#define VIRTIO_SCSI_F_T10_PI                   3
+
+/* Response codes */
+#define VIRTIO_SCSI_S_OK                       0
+#define VIRTIO_SCSI_S_OVERRUN                  1
+#define VIRTIO_SCSI_S_ABORTED                  2
+#define VIRTIO_SCSI_S_BAD_TARGET               3
+#define VIRTIO_SCSI_S_RESET                    4
+#define VIRTIO_SCSI_S_BUSY                     5
+#define VIRTIO_SCSI_S_TRANSPORT_FAILURE        6
+#define VIRTIO_SCSI_S_TARGET_FAILURE           7
+#define VIRTIO_SCSI_S_NEXUS_FAILURE            8
+#define VIRTIO_SCSI_S_FAILURE                  9
+#define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED       10
+#define VIRTIO_SCSI_S_FUNCTION_REJECTED        11
+#define VIRTIO_SCSI_S_INCORRECT_LUN            12
+
+/* Controlq type codes.  */
+#define VIRTIO_SCSI_T_TMF                      0
+#define VIRTIO_SCSI_T_AN_QUERY                 1
+#define VIRTIO_SCSI_T_AN_SUBSCRIBE             2
+
+/* Valid TMF subtypes.  */
+#define VIRTIO_SCSI_T_TMF_ABORT_TASK           0
+#define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET       1
+#define VIRTIO_SCSI_T_TMF_CLEAR_ACA            2
+#define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET       3
+#define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET      4
+#define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET   5
+#define VIRTIO_SCSI_T_TMF_QUERY_TASK           6
+#define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET       7
+
+/* Events.  */
+#define VIRTIO_SCSI_T_EVENTS_MISSED            0x80000000
+#define VIRTIO_SCSI_T_NO_EVENT                 0
+#define VIRTIO_SCSI_T_TRANSPORT_RESET          1
+#define VIRTIO_SCSI_T_ASYNC_NOTIFY             2
+#define VIRTIO_SCSI_T_PARAM_CHANGE             3
+
+/* Reasons of transport reset event */
+#define VIRTIO_SCSI_EVT_RESET_HARD             0
+#define VIRTIO_SCSI_EVT_RESET_RESCAN           1
+#define VIRTIO_SCSI_EVT_RESET_REMOVED          2
+
+#define VIRTIO_SCSI_S_SIMPLE                   0
+#define VIRTIO_SCSI_S_ORDERED                  1
+#define VIRTIO_SCSI_S_HEAD                     2
+#define VIRTIO_SCSI_S_ACA                      3
+
+
+#endif /* _LINUX_VIRTIO_SCSI_H */
diff --git a/include/standard-headers/linux/virtio_types.h b/include/standard-headers/linux/virtio_types.h
new file mode 100644
index 0000000..fd0d351
--- /dev/null
+++ b/include/standard-headers/linux/virtio_types.h
@@ -0,0 +1,46 @@
+#ifndef _LINUX_VIRTIO_TYPES_H
+#define _LINUX_VIRTIO_TYPES_H
+/* Type definitions for virtio implementations.
+ *
+ * This header is BSD licensed so anyone can use the definitions to implement
+ * compatible drivers/servers.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of IBM nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Copyright (C) 2014 Red Hat, Inc.
+ * Author: Michael S. Tsirkin <mst@redhat.com>
+ */
+#include "standard-headers/linux/types.h"
+
+/*
+ * __virtio{16,32,64} have the following meaning:
+ * - __u{16,32,64} for virtio devices in legacy mode, accessed in native endian
+ * - __le{16,32,64} for standard-compliant virtio devices
+ */
+
+typedef uint16_t  __virtio16;
+typedef uint32_t  __virtio32;
+typedef uint64_t  __virtio64;
+
+#endif /* _LINUX_VIRTIO_TYPES_H */
diff --git a/include/sysemu/accel.h b/include/sysemu/accel.h
index 997720f..a74b2fa 100644
--- a/include/sysemu/accel.h
+++ b/include/sysemu/accel.h
@@ -23,7 +23,6 @@
 #ifndef HW_ACCEL_H
 #define HW_ACCEL_H
 
-#include "qemu/typedefs.h"
 #include "qom/object.h"
 
 typedef struct AccelState {
diff --git a/include/sysemu/arch_init.h b/include/sysemu/arch_init.h
index 4592ca0..0aba314 100644
--- a/include/sysemu/arch_init.h
+++ b/include/sysemu/arch_init.h
@@ -32,8 +32,6 @@
 void select_soundhw(const char *optarg);
 void do_acpitable_option(const QemuOpts *opts);
 void do_smbios_option(QemuOpts *opts);
-void ram_mig_init(void);
-void cpudef_init(void);
 bool audio_init(void);
 int kvm_available(void);
 int xen_available(void);
diff --git a/include/sysemu/balloon.h b/include/sysemu/balloon.h
index 0345e01..af49e19 100644
--- a/include/sysemu/balloon.h
+++ b/include/sysemu/balloon.h
@@ -11,10 +11,9 @@
  *
  */
 
-#ifndef _QEMU_BALLOON_H
-#define _QEMU_BALLOON_H
+#ifndef QEMU_BALLOON_H
+#define QEMU_BALLOON_H
 
-#include "monitor/monitor.h"
 #include "qapi-types.h"
 
 typedef void (QEMUBalloonEvent)(void *opaque, ram_addr_t target);
@@ -23,5 +22,7 @@
 int qemu_add_balloon_handler(QEMUBalloonEvent *event_func,
 			     QEMUBalloonStatus *stat_func, void *opaque);
 void qemu_remove_balloon_handler(void *opaque);
+bool qemu_balloon_is_inhibited(void);
+void qemu_balloon_inhibit(bool state);
 
 #endif
diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
index 52d13c1..2da4905 100644
--- a/include/sysemu/block-backend.h
+++ b/include/sysemu/block-backend.h
@@ -1,7 +1,7 @@
 /*
  * QEMU Block backends
  *
- * Copyright (C) 2014 Red Hat, Inc.
+ * Copyright (C) 2014-2016 Red Hat, Inc.
  *
  * Authors:
  *  Markus Armbruster <armbru@redhat.com>,
@@ -13,8 +13,8 @@
 #ifndef BLOCK_BACKEND_H
 #define BLOCK_BACKEND_H
 
-#include "qemu/typedefs.h"
-#include "qapi/error.h"
+#include "qemu/iov.h"
+#include "block/throttle-groups.h"
 
 /*
  * TODO Have to include block/block.h for a bunch of block layer
@@ -60,57 +60,101 @@
     void (*resize_cb)(void *opaque);
 } BlockDevOps;
 
-BlockBackend *blk_new(const char *name, Error **errp);
-BlockBackend *blk_new_with_bs(const char *name, Error **errp);
+/* This struct is embedded in (the private) BlockBackend struct and contains
+ * fields that must be public. This is in particular for QLIST_ENTRY() and
+ * friends so that BlockBackends can be kept in lists outside block-backend.c */
+typedef struct BlockBackendPublic {
+    /* I/O throttling.
+     * throttle_state tells us if this BlockBackend has I/O limits configured.
+     * io_limits_disabled tells us if they are currently being enforced */
+    CoQueue      throttled_reqs[2];
+    unsigned int io_limits_disabled;
+
+    /* The following fields are protected by the ThrottleGroup lock.
+     * See the ThrottleGroup documentation for details. */
+    ThrottleState *throttle_state;
+    ThrottleTimers throttle_timers;
+    unsigned       pending_reqs[2];
+    QLIST_ENTRY(BlockBackendPublic) round_robin;
+} BlockBackendPublic;
+
+BlockBackend *blk_new(void);
+BlockBackend *blk_new_open(const char *filename, const char *reference,
+                           QDict *options, int flags, Error **errp);
+int blk_get_refcnt(BlockBackend *blk);
 void blk_ref(BlockBackend *blk);
 void blk_unref(BlockBackend *blk);
+void blk_remove_all_bs(void);
 const char *blk_name(BlockBackend *blk);
 BlockBackend *blk_by_name(const char *name);
 BlockBackend *blk_next(BlockBackend *blk);
+bool monitor_add_blk(BlockBackend *blk, const char *name, Error **errp);
+void monitor_remove_blk(BlockBackend *blk);
+
+BlockBackendPublic *blk_get_public(BlockBackend *blk);
+BlockBackend *blk_by_public(BlockBackendPublic *public);
 
 BlockDriverState *blk_bs(BlockBackend *blk);
+void blk_remove_bs(BlockBackend *blk);
+void blk_insert_bs(BlockBackend *blk, BlockDriverState *bs);
+bool bdrv_has_blk(BlockDriverState *bs);
 
-void blk_hide_on_behalf_of_do_drive_del(BlockBackend *blk);
-
+void blk_set_allow_write_beyond_eof(BlockBackend *blk, bool allow);
 void blk_iostatus_enable(BlockBackend *blk);
+bool blk_iostatus_is_enabled(const BlockBackend *blk);
+BlockDeviceIoStatus blk_iostatus(const BlockBackend *blk);
+void blk_iostatus_disable(BlockBackend *blk);
+void blk_iostatus_reset(BlockBackend *blk);
+void blk_iostatus_set_err(BlockBackend *blk, int error);
 int blk_attach_dev(BlockBackend *blk, void *dev);
 void blk_attach_dev_nofail(BlockBackend *blk, void *dev);
 void blk_detach_dev(BlockBackend *blk, void *dev);
 void *blk_get_attached_dev(BlockBackend *blk);
 void blk_set_dev_ops(BlockBackend *blk, const BlockDevOps *ops, void *opaque);
-int blk_read(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
-             int nb_sectors);
-int blk_read_unthrottled(BlockBackend *blk, int64_t sector_num, uint8_t *buf,
-                         int nb_sectors);
-int blk_write(BlockBackend *blk, int64_t sector_num, const uint8_t *buf,
-              int nb_sectors);
-BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
-                                 int nb_sectors, BdrvRequestFlags flags,
-                                 BlockCompletionFunc *cb, void *opaque);
+int blk_pread_unthrottled(BlockBackend *blk, int64_t offset, uint8_t *buf,
+                          int count);
+int coroutine_fn blk_co_preadv(BlockBackend *blk, int64_t offset,
+                               unsigned int bytes, QEMUIOVector *qiov,
+                               BdrvRequestFlags flags);
+int coroutine_fn blk_co_pwritev(BlockBackend *blk, int64_t offset,
+                               unsigned int bytes, QEMUIOVector *qiov,
+                               BdrvRequestFlags flags);
+int blk_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                      int count, BdrvRequestFlags flags);
+BlockAIOCB *blk_aio_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                                  int count, BdrvRequestFlags flags,
+                                  BlockCompletionFunc *cb, void *opaque);
+int blk_make_zero(BlockBackend *blk, BdrvRequestFlags flags);
 int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count);
-int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count);
+int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count,
+               BdrvRequestFlags flags);
 int64_t blk_getlength(BlockBackend *blk);
 void blk_get_geometry(BlockBackend *blk, uint64_t *nb_sectors_ptr);
-BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
-                          QEMUIOVector *iov, int nb_sectors,
-                          BlockCompletionFunc *cb, void *opaque);
-BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
-                           QEMUIOVector *iov, int nb_sectors,
+int64_t blk_nb_sectors(BlockBackend *blk);
+BlockAIOCB *blk_aio_preadv(BlockBackend *blk, int64_t offset,
+                           QEMUIOVector *qiov, BdrvRequestFlags flags,
                            BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_pwritev(BlockBackend *blk, int64_t offset,
+                            QEMUIOVector *qiov, BdrvRequestFlags flags,
+                            BlockCompletionFunc *cb, void *opaque);
 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
                           BlockCompletionFunc *cb, void *opaque);
-BlockAIOCB *blk_aio_discard(BlockBackend *blk,
-                            int64_t sector_num, int nb_sectors,
-                            BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_pdiscard(BlockBackend *blk, int64_t offset, int count,
+                             BlockCompletionFunc *cb, void *opaque);
 void blk_aio_cancel(BlockAIOCB *acb);
 void blk_aio_cancel_async(BlockAIOCB *acb);
-int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs);
 int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf);
 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
                           BlockCompletionFunc *cb, void *opaque);
+int blk_co_pdiscard(BlockBackend *blk, int64_t offset, int count);
+int blk_co_flush(BlockBackend *blk);
 int blk_flush(BlockBackend *blk);
 int blk_flush_all(void);
+int blk_commit_all(void);
+void blk_drain(BlockBackend *blk);
 void blk_drain_all(void);
+void blk_set_on_error(BlockBackend *blk, BlockdevOnError on_read_error,
+                      BlockdevOnError on_write_error);
 BlockdevOnError blk_get_on_error(BlockBackend *blk, bool is_read);
 BlockErrorAction blk_get_error_action(BlockBackend *blk, bool is_read,
                                       int error);
@@ -120,11 +164,16 @@
 int blk_is_sg(BlockBackend *blk);
 int blk_enable_write_cache(BlockBackend *blk);
 void blk_set_enable_write_cache(BlockBackend *blk, bool wce);
-int blk_is_inserted(BlockBackend *blk);
+void blk_invalidate_cache(BlockBackend *blk, Error **errp);
+bool blk_is_inserted(BlockBackend *blk);
+bool blk_is_available(BlockBackend *blk);
 void blk_lock_medium(BlockBackend *blk, bool locked);
 void blk_eject(BlockBackend *blk, bool eject_flag);
 int blk_get_flags(BlockBackend *blk);
+uint32_t blk_get_max_transfer(BlockBackend *blk);
+int blk_get_max_iov(BlockBackend *blk);
 void blk_set_guest_block_size(BlockBackend *blk, int align);
+void *blk_try_blockalign(BlockBackend *blk, size_t size);
 void *blk_blockalign(BlockBackend *blk, size_t size);
 bool blk_op_is_blocked(BlockBackend *blk, BlockOpType op, Error **errp);
 void blk_op_unblock(BlockBackend *blk, BlockOpType op, Error *reason);
@@ -132,11 +181,44 @@
 void blk_op_unblock_all(BlockBackend *blk, Error *reason);
 AioContext *blk_get_aio_context(BlockBackend *blk);
 void blk_set_aio_context(BlockBackend *blk, AioContext *new_context);
+void blk_add_aio_context_notifier(BlockBackend *blk,
+        void (*attached_aio_context)(AioContext *new_context, void *opaque),
+        void (*detach_aio_context)(void *opaque), void *opaque);
+void blk_remove_aio_context_notifier(BlockBackend *blk,
+                                     void (*attached_aio_context)(AioContext *,
+                                                                  void *),
+                                     void (*detach_aio_context)(void *),
+                                     void *opaque);
+void blk_add_remove_bs_notifier(BlockBackend *blk, Notifier *notify);
+void blk_add_insert_bs_notifier(BlockBackend *blk, Notifier *notify);
 void blk_io_plug(BlockBackend *blk);
 void blk_io_unplug(BlockBackend *blk);
 BlockAcctStats *blk_get_stats(BlockBackend *blk);
+BlockBackendRootState *blk_get_root_state(BlockBackend *blk);
+void blk_update_root_state(BlockBackend *blk);
+void blk_apply_root_state(BlockBackend *blk, BlockDriverState *bs);
+int blk_get_open_flags_from_root_state(BlockBackend *blk);
 
 void *blk_aio_get(const AIOCBInfo *aiocb_info, BlockBackend *blk,
                   BlockCompletionFunc *cb, void *opaque);
+int coroutine_fn blk_co_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                                      int count, BdrvRequestFlags flags);
+int blk_write_compressed(BlockBackend *blk, int64_t sector_num,
+                         const uint8_t *buf, int nb_sectors);
+int blk_truncate(BlockBackend *blk, int64_t offset);
+int blk_pdiscard(BlockBackend *blk, int64_t offset, int count);
+int blk_save_vmstate(BlockBackend *blk, const uint8_t *buf,
+                     int64_t pos, int size);
+int blk_load_vmstate(BlockBackend *blk, uint8_t *buf, int64_t pos, int size);
+int blk_probe_blocksizes(BlockBackend *blk, BlockSizes *bsz);
+int blk_probe_geometry(BlockBackend *blk, HDGeometry *geo);
+BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
+                                  BlockCompletionFunc *cb,
+                                  void *opaque, int ret);
+
+void blk_set_io_limits(BlockBackend *blk, ThrottleConfig *cfg);
+void blk_io_limits_disable(BlockBackend *blk);
+void blk_io_limits_enable(BlockBackend *blk, const char *group);
+void blk_io_limits_update_group(BlockBackend *blk, const char *group);
 
 #endif
diff --git a/include/sysemu/blockdev.h b/include/sysemu/blockdev.h
index 09d1e30..16432f3 100644
--- a/include/sysemu/blockdev.h
+++ b/include/sysemu/blockdev.h
@@ -11,7 +11,6 @@
 #define BLOCKDEV_H
 
 #include "block/block.h"
-#include "qapi/error.h"
 #include "qemu/queue.h"
 
 void blockdev_mark_auto_del(BlockBackend *blk);
@@ -20,7 +19,7 @@
 typedef enum {
     IF_DEFAULT = -1,            /* for use with drive_add() only */
     /*
-     * IF_IDE must be zero, because we want QEMUMachine member
+     * IF_IDE must be zero, because we want MachineClass member
      * block_default_type to default-initialize to IF_IDE
      */
     IF_IDE = 0,
@@ -63,10 +62,6 @@
 
 /* device-hotplug */
 
-DriveInfo *add_init_drive(const char *opts);
-
-void qmp_change_blockdev(const char *device, const char *filename,
-                         const char *format, Error **errp);
-void do_commit(Monitor *mon, const QDict *qdict);
-int do_drive_del(Monitor *mon, const QDict *qdict, QObject **ret_data);
+void hmp_commit(Monitor *mon, const QDict *qdict);
+void hmp_drive_del(Monitor *mon, const QDict *qdict);
 #endif
diff --git a/include/sysemu/bt.h b/include/sysemu/bt.h
index 2bc6d53..ddb05cd 100644
--- a/include/sysemu/bt.h
+++ b/include/sysemu/bt.h
@@ -1,5 +1,5 @@
-#ifndef BT_HOST_H
-#define BT_HOST_H
+#ifndef SYSEMU_BT_H
+#define SYSEMU_BT_H
 
 /* BT HCI info */
 
diff --git a/include/sysemu/char.h b/include/sysemu/char.h
index b6b2785..ee7e554 100644
--- a/include/sysemu/char.h
+++ b/include/sysemu/char.h
@@ -41,6 +41,11 @@
 #define CHR_IOCTL_PP_EPP_WRITE       11
 #define CHR_IOCTL_PP_DATA_DIR        12
 
+struct ParallelIOArg {
+    void *buffer;
+    int count;
+};
+
 #define CHR_IOCTL_SERIAL_SET_TIOCM   13
 #define CHR_IOCTL_SERIAL_GET_TIOCM   14
 
@@ -65,12 +70,13 @@
     int (*get_msgfds)(struct CharDriverState *s, int* fds, int num);
     int (*set_msgfds)(struct CharDriverState *s, int *fds, int num);
     int (*chr_add_client)(struct CharDriverState *chr, int fd);
-    int (*chr_del_client)(struct CharDriverState *chr);
+    int (*chr_wait_connected)(struct CharDriverState *chr, Error **errp);
     IOEventHandler *chr_event;
     IOCanReadHandler *chr_can_read;
     IOReadHandler *chr_read;
     void *handler_opaque;
     void (*chr_close)(struct CharDriverState *chr);
+    void (*chr_disconnect)(struct CharDriverState *chr);
     void (*chr_accept_input)(struct CharDriverState *chr);
     void (*chr_set_echo)(struct CharDriverState *chr, bool echo);
     void (*chr_set_fe_open)(struct CharDriverState *chr, int fe_open);
@@ -78,6 +84,7 @@
     void *opaque;
     char *label;
     char *filename;
+    int logfd;
     int be_open;
     int fe_open;
     int explicit_fe_open;
@@ -86,17 +93,20 @@
     int is_mux;
     guint fd_in_tag;
     QemuOpts *opts;
+    bool replay;
     QTAILQ_ENTRY(CharDriverState) next;
 };
 
 /**
- * @qemu_chr_alloc:
+ * qemu_chr_alloc:
+ * @backend: the common backend config
+ * @errp: pointer to a NULL-initialized error object
  *
  * Allocate and initialize a new CharDriverState.
  *
- * Returns: a newly allocated CharDriverState.
+ * Returns: a newly allocated CharDriverState, or NULL on error.
  */
-CharDriverState *qemu_chr_alloc(void);
+CharDriverState *qemu_chr_alloc(ChardevCommon *backend, Error **errp);
 
 /**
  * @qemu_chr_new_from_opts:
@@ -113,6 +123,16 @@
                                     Error **errp);
 
 /**
+ * @qemu_chr_parse_common:
+ *
+ * Parse the common options available to all character backends.
+ *
+ * @opts the options that still need parsing
+ * @backend a new backend
+ */
+void qemu_chr_parse_common(QemuOpts *opts, ChardevCommon *backend);
+
+/**
  * @qemu_chr_new:
  *
  * Create a new character backend from a URI.
@@ -125,15 +145,59 @@
  */
 CharDriverState *qemu_chr_new(const char *label, const char *filename,
                               void (*init)(struct CharDriverState *s));
+/**
+ * @qemu_chr_disconnect:
+ *
+ * Close a fd accpeted by character backend.
+ */
+void qemu_chr_disconnect(CharDriverState *chr);
+
+/**
+ * @qemu_chr_cleanup:
+ *
+ * Delete all chardevs (when leaving qemu)
+ */
+void qemu_chr_cleanup(void);
+
+/**
+ * @qemu_chr_wait_connected:
+ *
+ * Wait for characted backend to be connected.
+ */
+int qemu_chr_wait_connected(CharDriverState *chr, Error **errp);
+
+/**
+ * @qemu_chr_new_noreplay:
+ *
+ * Create a new character backend from a URI.
+ * Character device communications are not written
+ * into the replay log.
+ *
+ * @label the name of the backend
+ * @filename the URI
+ * @init not sure..
+ *
+ * Returns: a new character backend
+ */
+CharDriverState *qemu_chr_new_noreplay(const char *label, const char *filename,
+                                       void (*init)(struct CharDriverState *s));
 
 /**
  * @qemu_chr_delete:
  *
- * Destroy a character backend.
+ * Destroy a character backend and remove it from the list of
+ * identified character backends.
  */
 void qemu_chr_delete(CharDriverState *chr);
 
 /**
+ * @qemu_chr_free:
+ *
+ * Destroy a character backend.
+ */
+void qemu_chr_free(CharDriverState *chr);
+
+/**
  * @qemu_chr_fe_set_echo:
  *
  * Ask the backend to override its normal echo setting.  This only really
@@ -172,8 +236,20 @@
 void qemu_chr_fe_printf(CharDriverState *s, const char *fmt, ...)
     GCC_FMT_ATTR(2, 3);
 
-int qemu_chr_fe_add_watch(CharDriverState *s, GIOCondition cond,
-                          GIOFunc func, void *user_data);
+/**
+ * @qemu_chr_fe_add_watch:
+ *
+ * If the backend is connected, create and add a #GSource that fires
+ * when the given condition (typically G_IO_OUT|G_IO_HUP or G_IO_HUP)
+ * is active; return the #GSource's tag.  If it is disconnected,
+ * return 0.
+ *
+ * @cond the condition to poll for
+ * @func the function to call when the condition happens
+ * @user_data the opaque pointer to pass to @func
+ */
+guint qemu_chr_fe_add_watch(CharDriverState *s, GIOCondition cond,
+                            GIOFunc func, void *user_data);
 
 /**
  * @qemu_chr_fe_write:
@@ -321,6 +397,15 @@
  */
 void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len);
 
+/**
+ * @qemu_chr_be_write_impl:
+ *
+ * Implementation of back end writing. Used by replay module.
+ *
+ * @buf a buffer to receive data from the front end
+ * @len the number of bytes to receive from the front end
+ */
+void qemu_chr_be_write_impl(CharDriverState *s, uint8_t *buf, int len);
 
 /**
  * @qemu_chr_be_event:
@@ -340,43 +425,21 @@
 void qemu_chr_be_generic_open(CharDriverState *s);
 void qemu_chr_accept_input(CharDriverState *s);
 int qemu_chr_add_client(CharDriverState *s, int fd);
-
-/**
- * @qemu_chr_del_client:
- *
- * Delete (disconnect) the current client from the char backend,
- * as if the remote end had closed itself. This is most useful for TCP
- * listening sockets, where it means that we will close the connection to
- * the current client but remain listening for new ones.
- */
-int qemu_chr_del_client(CharDriverState *s);
 CharDriverState *qemu_chr_find(const char *name);
 bool chr_is_ringbuf(const CharDriverState *chr);
 
 QemuOpts *qemu_chr_parse_compat(const char *label, const char *filename);
 
 void register_char_driver(const char *name, ChardevBackendKind kind,
-        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp));
-
-/* add an eventfd to the qemu devices that are polled */
-CharDriverState *qemu_chr_open_eventfd(int eventfd);
+        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp),
+        CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                                   ChardevReturn *ret, Error **errp));
 
 extern int term_escape_char;
 
-CharDriverState *qemu_char_get_next_serial(void);
-
-/* msmouse */
-CharDriverState *qemu_chr_open_msmouse(void);
-
-/* testdev.c */
-CharDriverState *chr_testdev_init(void);
-
-/* baum.c */
-CharDriverState *chr_baum_init(void);
 
 /* console.c */
-typedef CharDriverState *(VcHandler)(ChardevVC *vc);
-
+typedef CharDriverState *(VcHandler)(ChardevVC *vc, Error **errp);
 void register_vc_handler(VcHandler *handler);
-CharDriverState *vc_init(ChardevVC *vc);
+
 #endif
diff --git a/include/sysemu/cpus.h b/include/sysemu/cpus.h
index 3f162a9..fe992a8 100644
--- a/include/sysemu/cpus.h
+++ b/include/sysemu/cpus.h
@@ -2,15 +2,28 @@
 #define QEMU_CPUS_H
 
 /* cpus.c */
+bool qemu_in_vcpu_thread(void);
 void qemu_init_cpu_loop(void);
 void resume_all_vcpus(void);
 void pause_all_vcpus(void);
 void cpu_stop_current(void);
+void cpu_ticks_init(void);
+
+void configure_icount(QemuOpts *opts, Error **errp);
+extern int use_icount;
+extern int icount_align_option;
+
+/* drift information for info jit command */
+extern int64_t max_delay;
+extern int64_t max_advance;
+void dump_drift_info(FILE *f, fprintf_function cpu_fprintf);
+
+/* Unblock cpu */
+void qemu_cpu_kick_self(void);
 
 void cpu_synchronize_all_states(void);
 void cpu_synchronize_all_post_reset(void);
 void cpu_synchronize_all_post_init(void);
-void cpu_clean_all_dirty(void);
 
 void qtest_clock_warp(int64_t dest);
 
diff --git a/include/sysemu/device_tree.h b/include/sysemu/device_tree.h
index 899f05c..e22e5be 100644
--- a/include/sysemu/device_tree.h
+++ b/include/sysemu/device_tree.h
@@ -11,11 +11,37 @@
  *
  */
 
-#ifndef __DEVICE_TREE_H__
-#define __DEVICE_TREE_H__
+#ifndef DEVICE_TREE_H
+#define DEVICE_TREE_H
 
 void *create_device_tree(int *sizep);
 void *load_device_tree(const char *filename_path, int *sizep);
+#ifdef CONFIG_LINUX
+/**
+ * load_device_tree_from_sysfs: reads the device tree information in the
+ * /proc/device-tree directory and return the corresponding binary blob
+ * buffer pointer. Asserts in case of error.
+ */
+void *load_device_tree_from_sysfs(void);
+#endif
+
+/**
+ * qemu_fdt_node_path: return the paths of nodes matching a given
+ * name and compat string
+ * @fdt: pointer to the dt blob
+ * @name: node name
+ * @compat: compatibility string
+ * @errp: handle to an error object
+ *
+ * returns a newly allocated NULL-terminated array of node paths.
+ * Use g_strfreev() to free it. If one or more nodes were found, the
+ * array contains the path of each node and the last element equals to
+ * NULL. If there is no error but no matching node was found, the
+ * returned array contains a single element equal to NULL. If an error
+ * was encountered when parsing the blob, the function returns NULL
+ */
+char **qemu_fdt_node_path(void *fdt, const char *name, char *compat,
+                          Error **errp);
 
 int qemu_fdt_setprop(void *fdt, const char *node_path,
                      const char *property, const void *val, int size);
@@ -28,10 +54,33 @@
 int qemu_fdt_setprop_phandle(void *fdt, const char *node_path,
                              const char *property,
                              const char *target_node_path);
+/**
+ * qemu_fdt_getprop: retrieve the value of a given property
+ * @fdt: pointer to the device tree blob
+ * @node_path: node path
+ * @property: name of the property to find
+ * @lenp: fdt error if any or length of the property on success
+ * @errp: handle to an error object
+ *
+ * returns a pointer to the property on success and NULL on failure
+ */
 const void *qemu_fdt_getprop(void *fdt, const char *node_path,
-                             const char *property, int *lenp);
+                             const char *property, int *lenp,
+                             Error **errp);
+/**
+ * qemu_fdt_getprop_cell: retrieve the value of a given 4 byte property
+ * @fdt: pointer to the device tree blob
+ * @node_path: node path
+ * @property: name of the property to find
+ * @lenp: fdt error if any or -EINVAL if the property size is different from
+ *        4 bytes, or 4 (expected length of the property) upon success.
+ * @errp: handle to an error object
+ *
+ * returns the property value on success
+ */
 uint32_t qemu_fdt_getprop_cell(void *fdt, const char *node_path,
-                               const char *property);
+                               const char *property, int *lenp,
+                               Error **errp);
 uint32_t qemu_fdt_get_phandle(void *fdt, const char *path);
 uint32_t qemu_fdt_alloc_phandle(void *fdt);
 int qemu_fdt_nop_node(void *fdt, const char *node_path);
@@ -110,4 +159,13 @@
                                                 qdt_tmp);                 \
     })
 
-#endif /* __DEVICE_TREE_H__ */
+#define FDT_PCI_RANGE_RELOCATABLE          0x80000000
+#define FDT_PCI_RANGE_PREFETCHABLE         0x40000000
+#define FDT_PCI_RANGE_ALIASED              0x20000000
+#define FDT_PCI_RANGE_TYPE_MASK            0x03000000
+#define FDT_PCI_RANGE_MMIO_64BIT           0x03000000
+#define FDT_PCI_RANGE_MMIO                 0x02000000
+#define FDT_PCI_RANGE_IOPORT               0x01000000
+#define FDT_PCI_RANGE_CONFIG               0x00000000
+
+#endif /* DEVICE_TREE_H */
diff --git a/include/sysemu/dma.h b/include/sysemu/dma.h
index 3f2f4c8..34c8eaf 100644
--- a/include/sysemu/dma.h
+++ b/include/sysemu/dma.h
@@ -10,13 +10,11 @@
 #ifndef DMA_H
 #define DMA_H
 
-#include <stdio.h>
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "hw/hw.h"
 #include "block/block.h"
 #include "block/accounting.h"
-#include "sysemu/kvm.h"
 
 typedef struct ScatterGatherEntry ScatterGatherEntry;
 
@@ -68,9 +66,7 @@
      * use lighter barriers based on the direction of the
      * transfer, the DMA context, etc...
      */
-    if (kvm_enabled()) {
-        smp_mb();
-    }
+    smp_mb();
 }
 
 /* Checks that the given range of addresses is valid for DMA.  This is
@@ -88,7 +84,8 @@
                                         void *buf, dma_addr_t len,
                                         DMADirection dir)
 {
-    return address_space_rw(as, addr, buf, len, dir == DMA_DIRECTION_FROM_DEVICE);
+    return (bool)address_space_rw(as, addr, MEMTXATTRS_UNSPECIFIED,
+                                  buf, len, dir == DMA_DIRECTION_FROM_DEVICE);
 }
 
 static inline int dma_memory_read_relaxed(AddressSpace *as, dma_addr_t addr,
@@ -197,19 +194,19 @@
 void qemu_sglist_destroy(QEMUSGList *qsg);
 #endif
 
-typedef BlockAIOCB *DMAIOFunc(BlockBackend *blk, int64_t sector_num,
-                              QEMUIOVector *iov, int nb_sectors,
-                              BlockCompletionFunc *cb, void *opaque);
+typedef BlockAIOCB *DMAIOFunc(int64_t offset, QEMUIOVector *iov,
+                              BlockCompletionFunc *cb, void *cb_opaque,
+                              void *opaque);
 
-BlockAIOCB *dma_blk_io(BlockBackend *blk,
-                       QEMUSGList *sg, uint64_t sector_num,
-                       DMAIOFunc *io_func, BlockCompletionFunc *cb,
-                       void *opaque, DMADirection dir);
+BlockAIOCB *dma_blk_io(AioContext *ctx,
+                       QEMUSGList *sg, uint64_t offset,
+                       DMAIOFunc *io_func, void *io_func_opaque,
+                       BlockCompletionFunc *cb, void *opaque, DMADirection dir);
 BlockAIOCB *dma_blk_read(BlockBackend *blk,
-                         QEMUSGList *sg, uint64_t sector,
+                         QEMUSGList *sg, uint64_t offset,
                          BlockCompletionFunc *cb, void *opaque);
 BlockAIOCB *dma_blk_write(BlockBackend *blk,
-                          QEMUSGList *sg, uint64_t sector,
+                          QEMUSGList *sg, uint64_t offset,
                           BlockCompletionFunc *cb, void *opaque);
 uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg);
 uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg);
diff --git a/include/sysemu/dump-arch.h b/include/sysemu/dump-arch.h
index 9c95ced..e25b02e 100644
--- a/include/sysemu/dump-arch.h
+++ b/include/sysemu/dump-arch.h
@@ -15,9 +15,12 @@
 #define DUMP_ARCH_H
 
 typedef struct ArchDumpInfo {
-    int d_machine;  /* Architecture */
-    int d_endian;   /* ELFDATA2LSB or ELFDATA2MSB */
-    int d_class;    /* ELFCLASS32 or ELFCLASS64 */
+    int d_machine;           /* Architecture */
+    int d_endian;            /* ELFDATA2LSB or ELFDATA2MSB */
+    int d_class;             /* ELFCLASS32 or ELFCLASS64 */
+    uint32_t page_size;      /* The target's page size. If it's variable and
+                              * unknown, then this should be the maximum. */
+    uint64_t phys_base;      /* The target's physmem base. */
 } ArchDumpInfo;
 
 struct GuestPhysBlockList; /* memory_mapping.h */
diff --git a/include/sysemu/dump.h b/include/sysemu/dump.h
index 7e4ec5c..ef931be 100644
--- a/include/sysemu/dump.h
+++ b/include/sysemu/dump.h
@@ -20,12 +20,9 @@
 #define VERSION_FLAT_HEADER         (1)    /* version of flattened format */
 #define END_FLAG_FLAT_HEADER        (-1)
 
+#ifndef ARCH_PFN_OFFSET
 #define ARCH_PFN_OFFSET             (0)
-
-#define paddr_to_pfn(X) \
-    (((unsigned long long)(X) >> TARGET_PAGE_BITS) - ARCH_PFN_OFFSET)
-#define pfn_to_paddr(X) \
-    (((unsigned long long)(X) + ARCH_PFN_OFFSET) << TARGET_PAGE_BITS)
+#endif
 
 /*
  * flag for compressed format
@@ -36,15 +33,12 @@
 
 #define KDUMP_SIGNATURE             "KDUMP   "
 #define SIG_LEN                     (sizeof(KDUMP_SIGNATURE) - 1)
-#define PHYS_BASE                   (0)
 #define DUMP_LEVEL                  (1)
 #define DISKDUMP_HEADER_BLOCKS      (1)
-#define BUFSIZE_BITMAP              (TARGET_PAGE_SIZE)
-#define PFN_BUFBITMAP               (CHAR_BIT * BUFSIZE_BITMAP)
-#define BUFSIZE_DATA_CACHE          (TARGET_PAGE_SIZE * 4)
 
 #include "sysemu/dump-arch.h"
 #include "sysemu/memory_mapping.h"
+#include "qapi-types.h"
 
 typedef struct QEMU_PACKED MakedumpfileHeader {
     char signature[16];     /* = "makedumpfile" */
@@ -183,6 +177,20 @@
     off_t offset_page;          /* offset of page part in vmcore */
     size_t num_dumpable;        /* number of page that can be dumped */
     uint32_t flag_compress;     /* indicate the compression format */
+    DumpStatus status;          /* current dump status */
+
+    bool has_format;              /* whether format is provided */
+    DumpGuestMemoryFormat format; /* valid only if has_format == true */
+    QemuThread dump_thread;       /* thread for detached dump */
+
+    int64_t total_size;          /* total memory size (in bytes) to
+                                  * be dumped. When filter is
+                                  * enabled, this will only count
+                                  * those to be written. */
+    int64_t written_size;        /* written memory size (in bytes),
+                                  * this could be used to calculate
+                                  * how much work we have
+                                  * finished. */
 } DumpState;
 
 uint16_t cpu_to_dump16(DumpState *s, uint16_t val);
diff --git a/include/sysemu/hax.h b/include/sysemu/hax.h
index 55fb2f6..d0ad6d9 100644
--- a/include/sysemu/hax.h
+++ b/include/sysemu/hax.h
@@ -22,13 +22,27 @@
 #define _HAX_H
 
 #include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
+/* Returns 1 if HAX is available and enabled, 0 otherwise. */
 int hax_enabled(void);
+
+/* Disable HAX if |disable| is 1, otherwise, enable it iff it is supported by the host CPU.
+ * Use hax_enabled() after this to get the result. */
 void hax_disable(int disable);
+
+/* Returns non-0 if the host CPU supports the VMX "unrestricted guest" feature which
+ * allows the virtual CPU to directly run in "real mode". If true, this allows QEMU to run
+ * several vCPU threads in parallel (see cpus.c). Otherwise, only a a single TCG thread
+ * can run, and it will call HAX to run the current instructions, except in case of
+ * "real mode" (paging disabled, typically at boot time), or MMIO operations. */
 int hax_ug_platform(void);
+
 int hax_pre_init(uint64_t ram_size);
+
 int hax_sync_vcpus(void);
+
 /* get the max haxm ram even before haxm library is initialized */
 int hax_get_max_ram(uint64_t *max_ram);
 
@@ -37,6 +51,7 @@
 #include "hw/hw.h"
 #include "qemu/bitops.h"
 #include "exec/memory.h"
+
 int hax_init_vcpu(CPUState *cpu);
 int hax_vcpu_exec(CPUState *cpu);
 int hax_smp_cpu_exec(CPUState *cpu);
@@ -50,8 +65,8 @@
 int hax_vcpu_destroy(CPUState *cpu);
 void hax_raise_event(CPUState *cpu);
 void hax_reset_vcpu_state(void *opaque);
-#include "target-i386/hax-interface.h"
-#include "target-i386/hax-i386.h"
+// #include "target-i386/hax-interface.h"
+// #include "target-i386/hax-i386.h"
 
 #endif
 
diff --git a/include/sysemu/hostmem.h b/include/sysemu/hostmem.h
index 1ce4394..678232a 100644
--- a/include/sysemu/hostmem.h
+++ b/include/sysemu/hostmem.h
@@ -9,12 +9,12 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#ifndef QEMU_RAM_H
-#define QEMU_RAM_H
+
+#ifndef SYSEMU_HOSTMEM_H
+#define SYSEMU_HOSTMEM_H
 
 #include "sysemu/sysemu.h" /* for MAX_NODES */
 #include "qom/object.h"
-#include "qapi/error.h"
 #include "exec/memory.h"
 #include "qemu/option.h"
 #include "qemu/bitmap.h"
@@ -45,7 +45,6 @@
  *
  * @parent: opaque parent object container
  * @size: amount of memory backend provides
- * @id: unique identification string in memdev namespace
  * @mr: MemoryRegion representing host memory belonging to backend
  */
 struct HostMemoryBackend {
@@ -55,7 +54,7 @@
     /* protected */
     uint64_t size;
     bool merge, dump;
-    bool prealloc, force_prealloc;
+    bool prealloc, force_prealloc, is_mapped;
     DECLARE_BITMAP(host_nodes, MAX_NODES + 1);
     HostMemPolicy policy;
 
@@ -65,4 +64,6 @@
 MemoryRegion *host_memory_backend_get_memory(HostMemoryBackend *backend,
                                              Error **errp);
 
+void host_memory_backend_set_mapped(HostMemoryBackend *backend, bool mapped);
+bool host_memory_backend_is_mapped(HostMemoryBackend *backend);
 #endif
diff --git a/include/sysemu/iothread.h b/include/sysemu/iothread.h
index 7c01a61..2eefea1 100644
--- a/include/sysemu/iothread.h
+++ b/include/sysemu/iothread.h
@@ -33,7 +33,6 @@
 #define IOTHREAD(obj) \
    OBJECT_CHECK(IOThread, obj, TYPE_IOTHREAD)
 
-IOThread *iothread_find(const char *id);
 char *iothread_get_id(IOThread *iothread);
 AioContext *iothread_get_aio_context(IOThread *iothread);
 
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index 22e42ef..c9c2436 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -14,10 +14,10 @@
 #ifndef QEMU_KVM_H
 #define QEMU_KVM_H
 
-#include <errno.h>
-#include "config-host.h"
 #include "qemu/queue.h"
 #include "qom/cpu.h"
+#include "exec/memattrs.h"
+#include "hw/irq.h"
 
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
@@ -41,14 +41,18 @@
 
 extern bool kvm_allowed;
 extern bool kvm_kernel_irqchip;
+extern bool kvm_split_irqchip;
 extern bool kvm_async_interrupts_allowed;
 extern bool kvm_halt_in_kernel_allowed;
 extern bool kvm_eventfds_allowed;
 extern bool kvm_irqfds_allowed;
+extern bool kvm_resamplefds_allowed;
 extern bool kvm_msi_via_irqfd_allowed;
 extern bool kvm_gsi_routing_allowed;
 extern bool kvm_gsi_direct_mapping;
 extern bool kvm_readonly_mem_allowed;
+extern bool kvm_direct_msi_allowed;
+extern bool kvm_ioeventfd_any_length_allowed;
 
 #if defined CONFIG_KVM || !defined NEED_CPU_H
 #define kvm_enabled()           (kvm_allowed)
@@ -66,6 +70,16 @@
 #define kvm_irqchip_in_kernel() (kvm_kernel_irqchip)
 
 /**
+ * kvm_irqchip_is_split:
+ *
+ * Returns: true if the user asked us to split the irqchip
+ * implementation between user and kernel space. The details are
+ * architecture and machine specific. On PC, it means that the PIC,
+ * IOAPIC, and PIT are in user space while the LAPIC is in the kernel.
+ */
+#define kvm_irqchip_is_split() (kvm_split_irqchip)
+
+/**
  * kvm_async_interrupts_enabled:
  *
  * Returns: true if we can deliver interrupts to KVM
@@ -102,6 +116,15 @@
 #define kvm_irqfds_enabled() (kvm_irqfds_allowed)
 
 /**
+ * kvm_resamplefds_enabled:
+ *
+ * Returns: true if we can use resamplefds to inject interrupts into
+ * a KVM CPU (ie the kernel supports resamplefds and we are running
+ * with a configuration where it is meaningful to use them).
+ */
+#define kvm_resamplefds_enabled() (kvm_resamplefds_allowed)
+
+/**
  * kvm_msi_via_irqfd_enabled:
  *
  * Returns: true if we can route a PCI MSI (Message Signaled Interrupt)
@@ -133,21 +156,39 @@
  */
 #define kvm_readonly_mem_enabled() (kvm_readonly_mem_allowed)
 
+/**
+ * kvm_direct_msi_enabled:
+ *
+ * Returns: true if KVM allows direct MSI injection.
+ */
+#define kvm_direct_msi_enabled() (kvm_direct_msi_allowed)
+
+/**
+ * kvm_ioeventfd_any_length_enabled:
+ * Returns: true if KVM allows any length io eventfd.
+ */
+#define kvm_ioeventfd_any_length_enabled() (kvm_ioeventfd_any_length_allowed)
+
 #else
 #define kvm_enabled()           (0)
 #define kvm_irqchip_in_kernel() (false)
+#define kvm_irqchip_is_split() (false)
 #define kvm_async_interrupts_enabled() (false)
 #define kvm_halt_in_kernel() (false)
 #define kvm_eventfds_enabled() (false)
 #define kvm_irqfds_enabled() (false)
+#define kvm_resamplefds_enabled() (false)
 #define kvm_msi_via_irqfd_enabled() (false)
 #define kvm_gsi_routing_allowed() (false)
 #define kvm_gsi_direct_mapping() (false)
 #define kvm_readonly_mem_enabled() (false)
+#define kvm_direct_msi_enabled() (false)
+#define kvm_ioeventfd_any_length_enabled() (false)
 #endif
 
 struct kvm_run;
 struct kvm_lapic_state;
+struct kvm_irq_routing_entry;
 
 typedef struct KVMCapabilityInfo {
     const char *name;
@@ -168,8 +209,6 @@
 int kvm_has_vcpu_events(void);
 int kvm_has_robust_singlestep(void);
 int kvm_has_debugregs(void);
-int kvm_has_xsave(void);
-int kvm_has_xcrs(void);
 int kvm_has_pit_state2(void);
 int kvm_has_many_ioeventfds(void);
 int kvm_has_gsi_routing(void);
@@ -177,8 +216,10 @@
 
 int kvm_init_vcpu(CPUState *cpu);
 int kvm_cpu_exec(CPUState *cpu);
+int kvm_destroy_vcpu(CPUState *cpu);
 
 #ifdef NEED_CPU_H
+#include "cpu.h"
 
 void kvm_setup_guest_memory(void *start, size_t size);
 void kvm_flush_coalesced_mmio_buffer(void);
@@ -196,6 +237,10 @@
 int kvm_on_sigbus_vcpu(CPUState *cpu, int code, void *addr);
 int kvm_on_sigbus(int code, void *addr);
 
+/* interface with exec.c */
+
+void phys_mem_set_alloc(void *(*alloc)(size_t, uint64_t *align));
+
 /* internal API */
 
 int kvm_ioctl(KVMState *s, int type, ...);
@@ -214,6 +259,44 @@
 int kvm_device_ioctl(int fd, int type, ...);
 
 /**
+ * kvm_vm_check_attr - check for existence of a specific vm attribute
+ * @s: The KVMState pointer
+ * @group: the group
+ * @attr: the attribute of that group to query for
+ *
+ * Returns: 1 if the attribute exists
+ *          0 if the attribute either does not exist or if the vm device
+ *            interface is unavailable
+ */
+int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr);
+
+/**
+ * kvm_device_check_attr - check for existence of a specific device attribute
+ * @fd: The device file descriptor
+ * @group: the group
+ * @attr: the attribute of that group to query for
+ *
+ * Returns: 1 if the attribute exists
+ *          0 if the attribute either does not exist or if the vm device
+ *            interface is unavailable
+ */
+int kvm_device_check_attr(int fd, uint32_t group, uint64_t attr);
+
+/**
+ * kvm_device_access - set or get value of a specific vm attribute
+ * @fd: The device file descriptor
+ * @group: the group
+ * @attr: the attribute of that group to set or get
+ * @val: pointer to a storage area for the value
+ * @write: true for set and false for get operation
+ *
+ * This function is not allowed to fail. Use kvm_device_check_attr()
+ * in order to check for the availability of optional attributes.
+ */
+void kvm_device_access(int fd, int group, uint64_t attr,
+                       void *val, bool write);
+
+/**
  * kvm_create_device - create a KVM device for the device control API
  * @KVMState: The KVMState pointer
  * @type: The KVM device type (see Documentation/virtual/kvm/devices in the
@@ -225,16 +308,27 @@
  */
 int kvm_create_device(KVMState *s, uint64_t type, bool test);
 
+/**
+ * kvm_device_supported - probe whether KVM supports specific device
+ *
+ * @vmfd: The fd handler for VM
+ * @type: type of device
+ *
+ * @return: true if supported, otherwise false.
+ */
+bool kvm_device_supported(int vmfd, uint64_t type);
 
 /* Arch specific hooks */
 
 extern const KVMCapabilityInfo kvm_arch_required_capabilities[];
 
 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run);
-void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run);
+MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run);
 
 int kvm_arch_handle_exit(CPUState *cpu, struct kvm_run *run);
 
+int kvm_arch_handle_ioapic_eoi(CPUState *cpu, struct kvm_run *run);
+
 int kvm_arch_process_async_events(CPUState *cpu);
 
 int kvm_arch_get_registers(CPUState *cpu);
@@ -248,10 +342,12 @@
 
 int kvm_arch_put_registers(CPUState *cpu, int level);
 
-int kvm_arch_init(KVMState *s);
+int kvm_arch_init(MachineState *ms, KVMState *s);
 
 int kvm_arch_init_vcpu(CPUState *cpu);
 
+bool kvm_vcpu_id_is_valid(int vcpu_id);
+
 /* Returns VCPU ID to be used on KVM_CREATE_VCPU ioctl() */
 unsigned long kvm_arch_vcpu_id(CPUState *cpu);
 
@@ -260,11 +356,21 @@
 
 void kvm_arch_init_irq_routing(KVMState *s);
 
+int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
+                             uint64_t address, uint32_t data, PCIDevice *dev);
+
+/* Notify arch about newly added MSI routes */
+int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
+                                int vector, PCIDevice *dev);
+/* Notify arch about released MSI routes */
+int kvm_arch_release_virq_post(int virq);
+
+int kvm_arch_msi_data_to_gsi(uint32_t data);
+
 int kvm_set_irq(KVMState *s, int irq, int level);
 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg);
 
 void kvm_irqchip_add_irq_route(KVMState *s, int gsi, int irqchip, int pin);
-void kvm_irqchip_commit_routes(KVMState *s);
 
 void kvm_put_apic_state(DeviceState *d, struct kvm_lapic_state *kapic);
 void kvm_get_apic_state(DeviceState *d, struct kvm_lapic_state *kapic);
@@ -349,7 +455,6 @@
 void kvm_cpu_synchronize_state(CPUState *cpu);
 void kvm_cpu_synchronize_post_reset(CPUState *cpu);
 void kvm_cpu_synchronize_post_init(CPUState *cpu);
-void kvm_cpu_clean_state(CPUState *cpu);
 
 /* generic hooks - to be moved/refactored once there are more users */
 
@@ -374,22 +479,35 @@
     }
 }
 
-static inline void cpu_clean_state(CPUState *cpu)
-{
-    if (kvm_enabled()) {
-        kvm_cpu_clean_state(cpu);
-    }
-}
-
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
+/**
+ * kvm_irqchip_add_msi_route - Add MSI route for specific vector
+ * @s:      KVM state
+ * @vector: which vector to add. This can be either MSI/MSIX
+ *          vector. The function will automatically detect whether
+ *          MSI/MSIX is enabled, and fetch corresponding MSI
+ *          message.
+ * @dev:    Owner PCI device to add the route. If @dev is specified
+ *          as @NULL, an empty MSI message will be inited.
+ * @return: virq (>=0) when success, errno (<0) when failed.
+ */
+int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev);
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev);
+void kvm_irqchip_commit_routes(KVMState *s);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
 
 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter);
+int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint);
 
+int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
+                                       EventNotifier *rn, int virq);
+int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
+                                          int virq);
 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
-                                   EventNotifier *rn, int virq);
-int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
+                                   EventNotifier *rn, qemu_irq irq);
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
+                                      qemu_irq irq);
+void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi);
 void kvm_pc_gsi_handler(void *opaque, int n, int level);
 void kvm_pc_setup_irq_routing(bool pci_enabled);
 void kvm_init_irq_routing(KVMState *s);
@@ -397,6 +515,7 @@
 /**
  * kvm_arch_irqchip_create:
  * @KVMState: The KVMState pointer
+ * @MachineState: The MachineState pointer
  *
  * Allow architectures to create an in-kernel irq chip themselves.
  *
@@ -404,7 +523,7 @@
  *            0: irq chip was not created
  *          > 0: irq chip was created
  */
-int kvm_arch_irqchip_create(KVMState *s);
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s);
 
 /**
  * kvm_set_one_reg - set a register value in KVM via KVM_SET_ONE_REG ioctl
@@ -425,4 +544,5 @@
  * Returns: 0 on success, or a negative errno on failure.
  */
 int kvm_get_one_reg(CPUState *cs, uint64_t id, void *target);
+int kvm_get_max_memslots(void);
 #endif
diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h
new file mode 100644
index 0000000..888557a
--- /dev/null
+++ b/include/sysemu/kvm_int.h
@@ -0,0 +1,39 @@
+/*
+ * Internal definitions for a target's KVM support
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_KVM_INT_H
+#define QEMU_KVM_INT_H
+
+#include "sysemu/sysemu.h"
+#include "sysemu/accel.h"
+#include "sysemu/kvm.h"
+
+typedef struct KVMSlot
+{
+    hwaddr start_addr;
+    ram_addr_t memory_size;
+    void *ram;
+    int slot;
+    int flags;
+} KVMSlot;
+
+typedef struct KVMMemoryListener {
+    MemoryListener listener;
+    KVMSlot *slots;
+    int as_id;
+} KVMMemoryListener;
+
+#define TYPE_KVM_ACCEL ACCEL_CLASS_NAME("kvm")
+
+#define KVM_STATE(obj) \
+    OBJECT_CHECK(KVMState, (obj), TYPE_KVM_ACCEL)
+
+void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
+                                  AddressSpace *as, int as_id);
+
+#endif
diff --git a/include/sysemu/memory_mapping.h b/include/sysemu/memory_mapping.h
index a75d59a..706152d 100644
--- a/include/sysemu/memory_mapping.h
+++ b/include/sysemu/memory_mapping.h
@@ -15,7 +15,7 @@
 #define MEMORY_MAPPING_H
 
 #include "qemu/queue.h"
-#include "qemu/typedefs.h"
+#include "exec/memory.h"
 
 typedef struct GuestPhysBlock {
     /* visible to guest, reflects PCI hole, etc */
@@ -27,6 +27,9 @@
     /* points into host memory */
     uint8_t *host_addr;
 
+    /* points to the MemoryRegion that this block belongs to */
+    MemoryRegion *mr;
+
     QTAILQ_ENTRY(GuestPhysBlock) next;
 } GuestPhysBlock;
 
diff --git a/include/sysemu/numa.h b/include/sysemu/numa.h
new file mode 100644
index 0000000..bb184c9
--- /dev/null
+++ b/include/sysemu/numa.h
@@ -0,0 +1,35 @@
+#ifndef SYSEMU_NUMA_H
+#define SYSEMU_NUMA_H
+
+#include "qemu/bitmap.h"
+#include "qemu/option.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/hostmem.h"
+#include "hw/boards.h"
+
+extern int nb_numa_nodes;   /* Number of NUMA nodes */
+
+struct numa_addr_range {
+    ram_addr_t mem_start;
+    ram_addr_t mem_end;
+    QLIST_ENTRY(numa_addr_range) entry;
+};
+
+typedef struct node_info {
+    uint64_t node_mem;
+    DECLARE_BITMAP(node_cpu, MAX_CPUMASK_BITS);
+    struct HostMemoryBackend *node_memdev;
+    bool present;
+    QLIST_HEAD(, numa_addr_range) addr; /* List to store address ranges */
+} NodeInfo;
+
+extern NodeInfo numa_info[MAX_NODES];
+void parse_numa_opts(MachineClass *mc);
+void numa_post_machine_init(void);
+void query_numa_node_mem(uint64_t node_mem[]);
+extern QemuOptsList qemu_numa_opts;
+void numa_set_mem_node_id(ram_addr_t addr, uint64_t size, uint32_t node);
+void numa_unset_mem_node_id(ram_addr_t addr, uint64_t size, uint32_t node);
+uint32_t numa_get_node(ram_addr_t addr, Error **errp);
+
+#endif
diff --git a/include/sysemu/os-posix.h b/include/sysemu/os-posix.h
index f131521..9c7dfdf 100644
--- a/include/sysemu/os-posix.h
+++ b/include/sysemu/os-posix.h
@@ -26,7 +26,13 @@
 #ifndef QEMU_OS_POSIX_H
 #define QEMU_OS_POSIX_H
 
-#include <sys/time.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/tcp.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <sys/un.h>
 
 void os_set_line_buffering(void);
 void os_set_proc_name(const char *s);
@@ -35,6 +41,9 @@
 void os_setup_post(void);
 int os_mlock(void);
 
+#define closesocket(s) close(s)
+#define ioctlsocket(s, r, v) ioctl(s, r, v)
+
 typedef struct timeval qemu_timeval;
 #define qemu_gettimeofday(tp) gettimeofday(tp, NULL)
 
diff --git a/include/sysemu/os-win32.h b/include/sysemu/os-win32.h
index 8340db3..5dc0b90 100644
--- a/include/sysemu/os-win32.h
+++ b/include/sysemu/os-win32.h
@@ -26,34 +26,9 @@
 #ifndef QEMU_OS_WIN32_H
 #define QEMU_OS_WIN32_H
 
-#include <windows.h>
 #include <winsock2.h>
-
-/* Workaround for older versions of MinGW. */
-#ifndef ECONNREFUSED
-# define ECONNREFUSED WSAECONNREFUSED
-#endif
-#ifndef EINPROGRESS
-# define EINPROGRESS  WSAEINPROGRESS
-#endif
-#ifndef EHOSTUNREACH
-# define EHOSTUNREACH WSAEHOSTUNREACH
-#endif
-#ifndef EINTR
-# define EINTR        WSAEINTR
-#endif
-#ifndef EINPROGRESS
-# define EINPROGRESS  WSAEINPROGRESS
-#endif
-#ifndef ENETUNREACH
-# define ENETUNREACH  WSAENETUNREACH
-#endif
-#ifndef ENOTCONN
-# define ENOTCONN     WSAENOTCONN
-#endif
-#ifndef EWOULDBLOCK
-# define EWOULDBLOCK  WSAEWOULDBLOCK
-#endif
+#include <windows.h>
+#include <ws2tcpip.h>
 
 #if defined(_WIN64)
 /* On w64, setjmp is implemented by _setjmp which needs a second parameter.
@@ -72,49 +47,13 @@
 #define sigsetjmp(env, savemask) setjmp(env)
 #define siglongjmp(env, val) longjmp(env, val)
 
-/* Declaration of ffs() is missing in MinGW's strings.h. */
-/* ANDROID EMULATOR BUILD HACK: Our mingw environment doesn't have any library
- * that provides ffs and friends.  This results in linking errors for these
- * functions when compiling with debugging enabled.  Without debug, gcc seems to
- * inline from the builtin.
- */
-#define ffs(i) __builtin_ffs(i)
-#define ffsl(i) __builtin_ffsl(i)
-#define ffsll(i) __builtin_ffsll(i)
-
 /* Missing POSIX functions. Don't use MinGW-w64 macros. */
-
-#ifndef __cplusplus
-// NOTE: Disabled for C++ to avoid compiler conflicts when building the Android
-// emulator.
-//
-// The root of the issue is the following:
-//
-// * <time.h> in newest Mingw headers declare functions like gmtime_r() as
-//   inline _outside_ of an extern "C" {}. They are thus declared with 'C++
-//   linkage', when <time.h> is included directly by a C++ source file.
-//
-// * The emulator's C++ source files include this header _within_ an
-//   extern "C" block (to avoid plenty of other kinds of conflicts), and
-//   the re-declaration below has 'C linkage'.
-//
-// The compiler complains that both declarations are incompatible due to
-// different linkage type (name mangling, really) and exits with an error.
-//
-// Since only QEMU's C source files use these functions, it is safe to avoid
-// re-declaring them here when __cplusplus is defined. This works with both
-// the old and new versions of <time.h> / Mingw headers.
-
+#ifndef CONFIG_LOCALTIME_R
 #undef gmtime_r
 struct tm *gmtime_r(const time_t *timep, struct tm *result);
-
 #undef localtime_r
 struct tm *localtime_r(const time_t *timep, struct tm *result);
-
-#undef strtok_r
-char *strtok_r(char *str, const char *delim, char **saveptr);
-
-#endif  /* !__cplusplus */
+#endif /* CONFIG_LOCALTIME_R */
 
 static inline void os_setup_signal_handling(void) {}
 static inline void os_daemonize(void) {}
@@ -122,7 +61,7 @@
 void os_set_line_buffering(void);
 static inline void os_set_proc_name(const char *dummy) {}
 
-size_t getpagesize(void);
+int getpagesize(void);
 
 #if !defined(EPROTONOSUPPORT)
 # define EPROTONOSUPPORT EINVAL
@@ -146,4 +85,123 @@
     return -ENOSYS;
 }
 
+#define fsync _commit
+
+#if !defined(lseek)
+# define lseek _lseeki64
+#endif
+
+int qemu_ftruncate64(int, int64_t);
+
+#if !defined(ftruncate)
+# define ftruncate qemu_ftruncate64
+#endif
+
+static inline char *realpath(const char *path, char *resolved_path)
+{
+    _fullpath(resolved_path, path, _MAX_PATH);
+    return resolved_path;
+}
+
+
+/* We wrap all the sockets functions so that we can
+ * set errno based on WSAGetLastError()
+ */
+
+#undef connect
+#define connect qemu_connect_wrap
+int qemu_connect_wrap(int sockfd, const struct sockaddr *addr,
+                      socklen_t addrlen);
+
+#undef listen
+#define listen qemu_listen_wrap
+int qemu_listen_wrap(int sockfd, int backlog);
+
+#undef bind
+#define bind qemu_bind_wrap
+int qemu_bind_wrap(int sockfd, const struct sockaddr *addr,
+                   socklen_t addrlen);
+
+#undef socket
+#define socket qemu_socket_wrap
+int qemu_socket_wrap(int domain, int type, int protocol);
+
+#undef accept
+#define accept qemu_accept_wrap
+int qemu_accept_wrap(int sockfd, struct sockaddr *addr,
+                     socklen_t *addrlen);
+
+#undef shutdown
+#define shutdown qemu_shutdown_wrap
+int qemu_shutdown_wrap(int sockfd, int how);
+
+#undef ioctlsocket
+#define ioctlsocket qemu_ioctlsocket_wrap
+int qemu_ioctlsocket_wrap(int fd, int req, void *val);
+
+#undef closesocket
+#define closesocket qemu_closesocket_wrap
+int qemu_closesocket_wrap(int fd);
+
+#undef getsockopt
+#define getsockopt qemu_getsockopt_wrap
+int qemu_getsockopt_wrap(int sockfd, int level, int optname,
+                         void *optval, socklen_t *optlen);
+
+#undef setsockopt
+#define setsockopt qemu_setsockopt_wrap
+int qemu_setsockopt_wrap(int sockfd, int level, int optname,
+                         const void *optval, socklen_t optlen);
+
+#undef getpeername
+#define getpeername qemu_getpeername_wrap
+int qemu_getpeername_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen);
+
+#undef getsockname
+#define getsockname qemu_getsockname_wrap
+int qemu_getsockname_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen);
+
+#undef send
+#define send qemu_send_wrap
+ssize_t qemu_send_wrap(int sockfd, const void *buf, size_t len, int flags);
+
+#undef sendto
+#define sendto qemu_sendto_wrap
+ssize_t qemu_sendto_wrap(int sockfd, const void *buf, size_t len, int flags,
+                         const struct sockaddr *addr, socklen_t addrlen);
+
+#undef recv
+#define recv qemu_recv_wrap
+ssize_t qemu_recv_wrap(int sockfd, void *buf, size_t len, int flags);
+
+#undef recvfrom
+#define recvfrom qemu_recvfrom_wrap
+ssize_t qemu_recvfrom_wrap(int sockfd, void *buf, size_t len, int flags,
+                           struct sockaddr *addr, socklen_t *addrlen);
+
+// ANDROID_BEGIN
+/* These are wrappers around Win32 functions. When building against the
+ * Android emulator, they will treat file names as UTF-8 encoded strings,
+ * instead of ANSI ones. */
+HANDLE win32CreateFile(
+        LPCTSTR               lpFileName,
+        DWORD                 dwDesiredAccess,
+        DWORD                 dwShareMode,
+        LPSECURITY_ATTRIBUTES lpSecurityAttributes,
+        DWORD                 dwCreationDisposition,
+        DWORD                 dwFlagsAndAttributes,
+        HANDLE                hTemplateFile);
+
+DWORD win32GetCurrentDirectory(
+        DWORD  nBufferLength,
+        LPTSTR lpBuffer);
+
+DWORD win32GetModuleFileName(
+        HMODULE hModule,
+        LPTSTR  lpFilename,
+        DWORD   nSize);
+// ANDROID_END
+
 #endif
diff --git a/include/sysemu/qtest.h b/include/sysemu/qtest.h
index 05473b7..70aa40a 100644
--- a/include/sysemu/qtest.h
+++ b/include/sysemu/qtest.h
@@ -15,7 +15,6 @@
 #define QTEST_H
 
 #include "qemu-common.h"
-#include "qapi/error.h"
 
 extern bool qtest_allowed;
 
diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h
new file mode 100644
index 0000000..0a88393
--- /dev/null
+++ b/include/sysemu/replay.h
@@ -0,0 +1,136 @@
+#ifndef REPLAY_H
+#define REPLAY_H
+
+/*
+ * replay.h
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qapi-types.h"
+
+/* replay clock kinds */
+enum ReplayClockKind {
+    /* host_clock */
+    REPLAY_CLOCK_HOST,
+    /* virtual_rt_clock */
+    REPLAY_CLOCK_VIRTUAL_RT,
+    REPLAY_CLOCK_COUNT
+};
+typedef enum ReplayClockKind ReplayClockKind;
+
+/* IDs of the checkpoints */
+enum ReplayCheckpoint {
+    CHECKPOINT_CLOCK_WARP_START,
+    CHECKPOINT_CLOCK_WARP_ACCOUNT,
+    CHECKPOINT_RESET_REQUESTED,
+    CHECKPOINT_SUSPEND_REQUESTED,
+    CHECKPOINT_CLOCK_VIRTUAL,
+    CHECKPOINT_CLOCK_HOST,
+    CHECKPOINT_CLOCK_VIRTUAL_RT,
+    CHECKPOINT_INIT,
+    CHECKPOINT_RESET,
+    CHECKPOINT_COUNT
+};
+typedef enum ReplayCheckpoint ReplayCheckpoint;
+
+extern ReplayMode replay_mode;
+
+/* Replay process control functions */
+
+/*! Enables recording or saving event log with specified parameters */
+void replay_configure(struct QemuOpts *opts);
+/*! Initializes timers used for snapshotting and enables events recording */
+void replay_start(void);
+/*! Closes replay log file and frees other resources. */
+void replay_finish(void);
+/*! Adds replay blocker with the specified error description */
+void replay_add_blocker(Error *reason);
+
+/* Processing the instructions */
+
+/*! Returns number of executed instructions. */
+uint64_t replay_get_current_step(void);
+/*! Returns number of instructions to execute in replay mode. */
+int replay_get_instructions(void);
+/*! Updates instructions counter in replay mode. */
+void replay_account_executed_instructions(void);
+
+/* Interrupts and exceptions */
+
+/*! Called by exception handler to write or read
+    exception processing events. */
+bool replay_exception(void);
+/*! Used to determine that exception is pending.
+    Does not proceed to the next event in the log. */
+bool replay_has_exception(void);
+/*! Called by interrupt handlers to write or read
+    interrupt processing events.
+    \return true if interrupt should be processed */
+bool replay_interrupt(void);
+/*! Tries to read interrupt event from the file.
+    Returns true, when interrupt request is pending */
+bool replay_has_interrupt(void);
+
+/* Processing clocks and other time sources */
+
+/*! Save the specified clock */
+int64_t replay_save_clock(ReplayClockKind kind, int64_t clock);
+/*! Read the specified clock from the log or return cached data */
+int64_t replay_read_clock(ReplayClockKind kind);
+/*! Saves or reads the clock depending on the current replay mode. */
+#define REPLAY_CLOCK(clock, value)                                      \
+    (replay_mode == REPLAY_MODE_PLAY ? replay_read_clock((clock))       \
+        : replay_mode == REPLAY_MODE_RECORD                             \
+            ? replay_save_clock((clock), (value))                       \
+        : (value))
+
+/* Events */
+
+/*! Called when qemu shutdown is requested. */
+void replay_shutdown_request(void);
+/*! Should be called at check points in the execution.
+    These check points are skipped, if they were not met.
+    Saves checkpoint in the SAVE mode and validates in the PLAY mode.
+    Returns 0 in PLAY mode if checkpoint was not found.
+    Returns 1 in all other cases. */
+bool replay_checkpoint(ReplayCheckpoint checkpoint);
+
+/* Asynchronous events queue */
+
+/*! Disables storing events in the queue */
+void replay_disable_events(void);
+/*! Returns true when saving events is enabled */
+bool replay_events_enabled(void);
+/*! Adds bottom half event to the queue */
+void replay_bh_schedule_event(QEMUBH *bh);
+/*! Adds input event to the queue */
+void replay_input_event(QemuConsole *src, InputEvent *evt);
+/*! Adds input sync event to the queue */
+void replay_input_sync_event(void);
+/*! Adds block layer event to the queue */
+void replay_block_event(QEMUBH *bh, uint64_t id);
+
+/* Character device */
+
+/*! Registers char driver to save it's events */
+void replay_register_char_driver(struct CharDriverState *chr);
+/*! Saves write to char device event to the log */
+void replay_chr_be_write(struct CharDriverState *s, uint8_t *buf, int len);
+/*! Writes char write return value to the replay log. */
+void replay_char_write_event_save(int res, int offset);
+/*! Reads char write return value from the replay log. */
+void replay_char_write_event_load(int *res, int *offset);
+/*! Reads information about read_all character event. */
+int replay_char_read_all_load(uint8_t *buf);
+/*! Writes character read_all error code into the replay log. */
+void replay_char_read_all_save_error(int res);
+/*! Writes character read_all execution result into the replay log. */
+void replay_char_read_all_save_buf(uint8_t *buf, int offset);
+
+#endif
diff --git a/include/sysemu/rng-random.h b/include/sysemu/rng-random.h
index 4332772..38186fe 100644
--- a/include/sysemu/rng-random.h
+++ b/include/sysemu/rng-random.h
@@ -15,8 +15,8 @@
 #include "qom/object.h"
 
 #define TYPE_RNG_RANDOM "rng-random"
-#define RNG_RANDOM(obj) OBJECT_CHECK(RndRandom, (obj), TYPE_RNG_RANDOM)
+#define RNG_RANDOM(obj) OBJECT_CHECK(RngRandom, (obj), TYPE_RNG_RANDOM)
 
-typedef struct RndRandom RndRandom;
+typedef struct RngRandom RngRandom;
 
 #endif
diff --git a/include/sysemu/rng.h b/include/sysemu/rng.h
index 0a27c9b..45629c4 100644
--- a/include/sysemu/rng.h
+++ b/include/sysemu/rng.h
@@ -15,7 +15,6 @@
 
 #include "qom/object.h"
 #include "qemu-common.h"
-#include "qapi/error.h"
 
 #define TYPE_RNG_BACKEND "rng-backend"
 #define RNG_BACKEND(obj) \
@@ -25,6 +24,7 @@
 #define RNG_BACKEND_CLASS(klass) \
     OBJECT_CLASS_CHECK(RngBackendClass, (klass), TYPE_RNG_BACKEND)
 
+typedef struct RngRequest RngRequest;
 typedef struct RngBackendClass RngBackendClass;
 typedef struct RngBackend RngBackend;
 
@@ -32,13 +32,21 @@
                                   const void *data,
                                   size_t size);
 
+struct RngRequest
+{
+    EntropyReceiveFunc *receive_entropy;
+    uint8_t *data;
+    void *opaque;
+    size_t offset;
+    size_t size;
+    QSIMPLEQ_ENTRY(RngRequest) next;
+};
+
 struct RngBackendClass
 {
     ObjectClass parent_class;
 
-    void (*request_entropy)(RngBackend *s, size_t size,
-                            EntropyReceiveFunc *receive_entropy, void *opaque);
-    void (*cancel_requests)(RngBackend *s);
+    void (*request_entropy)(RngBackend *s, RngRequest *req);
 
     void (*opened)(RngBackend *s, Error **errp);
 };
@@ -49,8 +57,10 @@
 
     /*< protected >*/
     bool opened;
+    QSIMPLEQ_HEAD(requests, RngRequest) requests;
 };
 
+
 /**
  * rng_backend_request_entropy:
  * @s: the backend to request entropy from
@@ -71,12 +81,13 @@
                                  void *opaque);
 
 /**
- * rng_backend_cancel_requests:
- * @s: the backend to cancel all pending requests in
+ * rng_backend_free_request:
+ * @s: the backend that created the request
+ * @req: the request to finalize
  *
- * Cancels all pending requests submitted by @rng_backend_request_entropy.  This
- * should be used by a device during reset or in preparation for live migration
- * to stop tracking any request.
+ * Used by child rng backend classes to finalize requests once they've been
+ * processed. The request is removed from the list of active requests and
+ * deleted.
  */
-void rng_backend_cancel_requests(RngBackend *s);
+void rng_backend_finalize_request(RngBackend *s, RngRequest *req);
 #endif
diff --git a/include/sysemu/seccomp.h b/include/sysemu/seccomp.h
index 1189fa2..cfc0600 100644
--- a/include/sysemu/seccomp.h
+++ b/include/sysemu/seccomp.h
@@ -16,7 +16,6 @@
 #define QEMU_SECCOMP_H
 
 #include <seccomp.h>
-#include "qemu/osdep.h"
 
 int seccomp_start(void);
 #endif
diff --git a/include/sysemu/sysemu.h b/include/sysemu/sysemu.h
index ae3e81c..ee7c760 100644
--- a/include/sysemu/sysemu.h
+++ b/include/sysemu/sysemu.h
@@ -2,7 +2,6 @@
 #define SYSEMU_H
 /* Misc. things related to the system emulator.  */
 
-#include "qemu/typedefs.h"
 #include "qemu/option.h"
 #include "qemu/queue.h"
 #include "qemu/timer.h"
@@ -28,6 +27,7 @@
 void runstate_set(RunState new_state);
 int runstate_is_running(void);
 bool runstate_needs_reset(void);
+bool runstate_store(char *str, size_t size);
 typedef struct vm_change_state_entry VMChangeStateEntry;
 typedef void VMChangeStateHandler(void *opaque, int running, RunState state);
 
@@ -68,47 +68,76 @@
 void qemu_system_killed(int signal, pid_t pid);
 void qemu_devices_reset(void);
 void qemu_system_reset(bool report);
+void qemu_system_guest_panicked(void);
+size_t qemu_target_page_bits(void);
 
 void qemu_add_exit_notifier(Notifier *notify);
 void qemu_remove_exit_notifier(Notifier *notify);
 
 void qemu_add_machine_init_done_notifier(Notifier *notify);
+void qemu_remove_machine_init_done_notifier(Notifier *notify);
 
-void do_savevm(Monitor *mon, const QDict *qdict);
+void hmp_savevm(Monitor *mon, const QDict *qdict);
 int load_vmstate(const char *name);
-void do_delvm(Monitor *mon, const QDict *qdict);
-void do_info_snapshots(Monitor *mon, const QDict *qdict);
+void hmp_delvm(Monitor *mon, const QDict *qdict);
+void hmp_info_snapshots(Monitor *mon, const QDict *qdict);
 
 void qemu_announce_self(void);
 
+/* Subcommands for QEMU_VM_COMMAND */
+enum qemu_vm_cmd {
+    MIG_CMD_INVALID = 0,   /* Must be 0 */
+    MIG_CMD_OPEN_RETURN_PATH,  /* Tell the dest to open the Return path */
+    MIG_CMD_PING,              /* Request a PONG on the RP */
+
+    MIG_CMD_POSTCOPY_ADVISE,       /* Prior to any page transfers, just
+                                      warn we might want to do PC */
+    MIG_CMD_POSTCOPY_LISTEN,       /* Start listening for incoming
+                                      pages as it's running. */
+    MIG_CMD_POSTCOPY_RUN,          /* Start execution */
+
+    MIG_CMD_POSTCOPY_RAM_DISCARD,  /* A list of pages to discard that
+                                      were previously sent during
+                                      precopy but are dirty. */
+    MIG_CMD_PACKAGED,          /* Send a wrapped stream within this stream */
+    MIG_CMD_MAX
+};
+
+#define MAX_VM_CMD_PACKAGED_SIZE (1ul << 24)
+
 bool qemu_savevm_state_blocked(Error **errp);
 void qemu_savevm_state_begin(QEMUFile *f,
                              const MigrationParams *params);
-int qemu_savevm_state_iterate(QEMUFile *f);
-void qemu_savevm_state_complete(QEMUFile *f);
-void qemu_savevm_state_cancel(void);
-uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size);
+void qemu_savevm_state_header(QEMUFile *f);
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy);
+void qemu_savevm_state_cleanup(void);
+void qemu_savevm_state_complete_postcopy(QEMUFile *f);
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only);
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable);
+void qemu_savevm_command_send(QEMUFile *f, enum qemu_vm_cmd command,
+                              uint16_t len, uint8_t *data);
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value);
+void qemu_savevm_send_open_return_path(QEMUFile *f);
+int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len);
+void qemu_savevm_send_postcopy_advise(QEMUFile *f);
+void qemu_savevm_send_postcopy_listen(QEMUFile *f);
+void qemu_savevm_send_postcopy_run(QEMUFile *f);
+
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list);
+
 int qemu_loadvm_state(QEMUFile *f);
 
-/* SLIRP */
-void do_info_slirp(Monitor *mon);
-
-typedef enum DisplayType
-{
-    DT_DEFAULT,
-    DT_CURSES,
-    DT_SDL,
-    DT_GTK,
-    DT_VNC,
-    DT_NOGRAPHIC,
-    DT_NONE,
-} DisplayType;
-
 extern int autostart;
 
 typedef enum {
     VGA_NONE, VGA_STD, VGA_CIRRUS, VGA_VMWARE, VGA_XENFB, VGA_QXL,
-    VGA_TCX, VGA_CG3, VGA_DEVICE
+    VGA_TCX, VGA_CG3, VGA_DEVICE, VGA_VIRTIO,
+    VGA_TYPE_MAX,
 } VGAInterfaceType;
 
 extern int vga_interface_type;
@@ -117,7 +146,7 @@
 extern int graphic_width;
 extern int graphic_height;
 extern int graphic_depth;
-extern DisplayType display_type;
+extern int display_opengl;
 extern const char *keyboard_layout;
 extern int win2k_install_hack;
 extern int alt_grab;
@@ -128,18 +157,19 @@
 extern int graphic_rotate;
 extern int no_quit;
 extern int no_shutdown;
-extern int semihosting_enabled;
 extern int old_param;
 extern int boot_menu;
 extern bool boot_strict;
 extern uint8_t *boot_splash_filedata;
 extern size_t boot_splash_filedata_size;
+extern bool enable_mlock;
 extern uint8_t qemu_extra_params_fw[2];
 extern QEMUClockType rtc_clock;
 extern const char *mem_path;
 extern int mem_prealloc;
 
 #define MAX_NODES 128
+#define NUMA_NODE_UNASSIGNED MAX_NODES
 
 /* The following shall be true for all CPUs:
  *   cpu->cpu_index < max_cpus <= MAX_CPUMASK_BITS
@@ -148,24 +178,6 @@
  */
 #define MAX_CPUMASK_BITS 255
 
-extern int nb_numa_nodes;   /* Number of NUMA nodes */
-extern int max_numa_nodeid; /* Highest specified NUMA node ID, plus one.
-                             * For all nodes, nodeid < max_numa_nodeid
-                             */
-
-typedef struct node_info {
-    uint64_t node_mem;
-    DECLARE_BITMAP(node_cpu, MAX_CPUMASK_BITS);
-    struct HostMemoryBackend *node_memdev;
-    bool present;
-} NodeInfo;
-extern NodeInfo numa_info[MAX_NODES];
-void set_numa_nodes(void);
-void set_numa_modes(void);
-void query_numa_node_mem(uint64_t node_mem[]);
-extern QemuOptsList qemu_numa_opts;
-int numa_init_func(QemuOpts *opts, void *opaque);
-
 #define MAX_OPTION_ROMS 16
 typedef struct QEMUOptionRom {
     const char *name;
@@ -178,18 +190,11 @@
 extern const char *prom_envs[MAX_PROM_ENVS];
 extern unsigned int nb_prom_envs;
 
-/* pci-hotplug */
-void pci_device_hot_add(Monitor *mon, const QDict *qdict);
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo);
-void do_pci_device_hot_remove(Monitor *mon, const QDict *qdict);
-
 /* generic hotplug */
-void drive_hot_add(Monitor *mon, const QDict *qdict);
+void hmp_drive_add(Monitor *mon, const QDict *qdict);
 
 /* pcie aer error injection */
-void pcie_aer_inject_error_print(Monitor *mon, const QObject *data);
-int do_pcie_aer_inject_error(Monitor *mon,
-                             const QDict *qdict, QObject **ret_data);
+void hmp_pcie_aer_inject_error(Monitor *mon, const QDict *qdict);
 
 /* serial ports */
 
@@ -203,9 +208,9 @@
 
 extern CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 
-void do_usb_add(Monitor *mon, const QDict *qdict);
-void do_usb_del(Monitor *mon, const QDict *qdict);
-void usb_info(Monitor *mon, const QDict *qdict);
+void hmp_usb_add(Monitor *mon, const QDict *qdict);
+void hmp_usb_del(Monitor *mon, const QDict *qdict);
+void hmp_info_usb(Monitor *mon, const QDict *qdict);
 
 void add_boot_device_path(int32_t bootindex, DeviceState *dev,
                           const char *suffix);
@@ -217,10 +222,18 @@
 void device_add_bootindex_property(Object *obj, int32_t *bootindex,
                                    const char *name, const char *suffix,
                                    DeviceState *dev, Error **errp);
+void restore_boot_order(void *opaque);
+void validate_bootdevices(const char *devices, Error **errp);
+
+/* handler to set the boot_device order for a specific type of MachineClass */
+typedef void QEMUBootSetHandler(void *opaque, const char *boot_order,
+                                Error **errp);
+void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque);
+void qemu_boot_set(const char *boot_order, Error **errp);
 
 QemuOpts *qemu_get_machine_opts(void);
 
-bool usb_enabled(bool default_usb);
+bool defaults_enabled(void);
 
 extern QemuOptsList qemu_legacy_drive_opts;
 extern QemuOptsList qemu_common_drive_opts;
diff --git a/include/sysemu/tpm.h b/include/sysemu/tpm.h
index 9b81ce9..c8afa17 100644
--- a/include/sysemu/tpm.h
+++ b/include/sysemu/tpm.h
@@ -20,11 +20,26 @@
 int tpm_init(void);
 void tpm_cleanup(void);
 
+typedef enum  TPMVersion {
+    TPM_VERSION_UNSPEC = 0,
+    TPM_VERSION_1_2 = 1,
+    TPM_VERSION_2_0 = 2,
+} TPMVersion;
+
+TPMVersion tpm_tis_get_tpm_version(Object *obj);
+
 #define TYPE_TPM_TIS                "tpm-tis"
 
-static inline bool tpm_find(void)
+static inline TPMVersion tpm_get_version(void)
 {
-    return object_resolve_path_type("", TYPE_TPM_TIS, NULL);
+#ifdef CONFIG_TPM
+    Object *obj = object_resolve_path_type("", TYPE_TPM_TIS, NULL);
+
+    if (obj) {
+        return tpm_tis_get_tpm_version(obj);
+    }
+#endif
+    return TPM_VERSION_UNSPEC;
 }
 
 #endif /* QEMU_TPM_H */
diff --git a/include/sysemu/tpm_backend.h b/include/sysemu/tpm_backend.h
index 825f33b..b58f52d 100644
--- a/include/sysemu/tpm_backend.h
+++ b/include/sysemu/tpm_backend.h
@@ -10,12 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#ifndef _QEMU_TPM_H
-#define _QEMU_TPM_H
+#ifndef TPM_BACKEND_H
+#define TPM_BACKEND_H
 
 #include "qom/object.h"
 #include "qemu-common.h"
-#include "qapi/error.h"
 #include "qapi-types.h"
 #include "qemu/option.h"
 #include "sysemu/tpm.h"
@@ -56,7 +55,7 @@
     QLIST_ENTRY(TPMBackend) list;
 };
 
-typedef void (TPMRecvDataCB)(TPMState *, uint8_t locty);
+typedef void (TPMRecvDataCB)(TPMState *, uint8_t locty, bool selftest_done);
 
 typedef struct TPMSizedBuffer {
     uint32_t size;
@@ -88,6 +87,10 @@
     void (*cancel_cmd)(TPMBackend *t);
 
     bool (*get_tpm_established_flag)(TPMBackend *t);
+
+    int (*reset_tpm_established_flag)(TPMBackend *t, uint8_t locty);
+
+    TPMVersion (*get_tpm_version)(TPMBackend *t);
 };
 
 
@@ -192,6 +195,15 @@
 bool tpm_backend_get_tpm_established_flag(TPMBackend *s);
 
 /**
+ * tpm_backend_reset_tpm_established_flag:
+ * @s: the backend
+ * @locty: the locality number
+ *
+ * Reset the TPM establishment flag.
+ */
+int tpm_backend_reset_tpm_established_flag(TPMBackend *s, uint8_t locty);
+
+/**
  * tpm_backend_open:
  * @s: the backend to open
  * @errp: a pointer to return the #Error object if an error occurs.
@@ -201,6 +213,16 @@
  */
 void tpm_backend_open(TPMBackend *s, Error **errp);
 
+/**
+ * tpm_backend_get_tpm_version:
+ * @s: the backend to call into
+ *
+ * Get the TPM Version that is emulated at the backend.
+ *
+ * Returns TPMVersion.
+ */
+TPMVersion tpm_backend_get_tpm_version(TPMBackend *s);
+
 TPMBackend *qemu_find_tpm(const char *id);
 
 const TPMDriverOps *tpm_get_backend_driver(const char *type);
diff --git a/include/sysemu/tpm_backend_int.h b/include/sysemu/tpm_backend_int.h
index 05d94d0..00639dd 100644
--- a/include/sysemu/tpm_backend_int.h
+++ b/include/sysemu/tpm_backend_int.h
@@ -19,10 +19,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>
  */
 
-#ifndef TPM_TPM_BACKEND_H
-#define TPM_TPM_BACKEND_H
-
-#include <glib.h>
+#ifndef TPM_BACKEND_INT_H
+#define TPM_BACKEND_INT_H
 
 typedef struct TPMBackendThread {
     GThreadPool *pool;
@@ -32,8 +30,6 @@
 void tpm_backend_thread_create(TPMBackendThread *tbt,
                                GFunc func, gpointer user_data);
 void tpm_backend_thread_end(TPMBackendThread *tbt);
-void tpm_backend_thread_tpm_reset(TPMBackendThread *tbt,
-                                  GFunc func, gpointer user_data);
 
 typedef enum TPMBackendCmd {
     TPM_BACKEND_CMD_INIT = 1,
@@ -42,4 +38,4 @@
     TPM_BACKEND_CMD_TPM_RESET,
 } TPMBackendCmd;
 
-#endif /* TPM_TPM_BACKEND_H */
+#endif /* TPM_BACKEND_INT_H */
diff --git a/include/sysemu/watchdog.h b/include/sysemu/watchdog.h
index 3e9a970..72a4da0 100644
--- a/include/sysemu/watchdog.h
+++ b/include/sysemu/watchdog.h
@@ -24,6 +24,15 @@
 
 #include "qemu/queue.h"
 
+/* Possible values for action parameter. */
+#define WDT_RESET        1      /* Hard reset. */
+#define WDT_SHUTDOWN     2      /* Shutdown. */
+#define WDT_POWEROFF     3      /* Quit. */
+#define WDT_PAUSE        4      /* Pause. */
+#define WDT_DEBUG        5      /* Prints a message and continues running. */
+#define WDT_NONE         6      /* Do nothing. */
+#define WDT_NMI          7      /* Inject nmi into the guest. */
+
 struct WatchdogTimerModel {
     QLIST_ENTRY(WatchdogTimerModel) entry;
 
@@ -37,6 +46,7 @@
 /* in hw/watchdog.c */
 int select_watchdog(const char *p);
 int select_watchdog_action(const char *action);
+int get_watchdog_action(void);
 void watchdog_add_model(WatchdogTimerModel *model);
 void watchdog_perform_action(void);
 
diff --git a/include/sysemu/xen-mapcache.h b/include/sysemu/xen-mapcache.h
index c598040..b8c93b9 100644
--- a/include/sysemu/xen-mapcache.h
+++ b/include/sysemu/xen-mapcache.h
@@ -9,8 +9,6 @@
 #ifndef XEN_MAPCACHE_H
 #define XEN_MAPCACHE_H
 
-#include <stdlib.h>
-
 typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr start_addr,
                                                      ram_addr_t size,
                                                      void *opaque);
@@ -53,4 +51,4 @@
 
 #endif
 
-#endif /* !XEN_MAPCACHE_H */
+#endif /* XEN_MAPCACHE_H */
diff --git a/include/trace-tcg.h b/include/trace-tcg.h
index 6f6bdbb..edab4b1 100644
--- a/include/trace-tcg.h
+++ b/include/trace-tcg.h
@@ -4,4 +4,4 @@
 #include "trace/generated-tcg-tracers.h"
 #include "trace/generated-events.h"
 
-#endif  /* TRACE_TCG_H */
+#endif /* TRACE_TCG_H */
diff --git a/include/trace.h b/include/trace.h
index 44a1f1f..9a01e44 100644
--- a/include/trace.h
+++ b/include/trace.h
@@ -4,4 +4,4 @@
 #include "trace/generated-tracers.h"
 #include "trace/generated-events.h"
 
-#endif  /* TRACE_H */
+#endif /* TRACE_H */
diff --git a/include/ui/console.h b/include/ui/console.h
index d9eaf19..b5374d9 100644
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -5,10 +5,14 @@
 #include "qom/object.h"
 #include "qapi/qmp/qdict.h"
 #include "qemu/notify.h"
-#include "monitor/monitor.h"
 #include "qapi-types.h"
+#include "qemu/error-report.h"
 #include "qapi/error.h"
 
+#ifdef CONFIG_OPENGL
+# include <epoxy/gl.h>
+#endif
+
 #include <stdbool.h>
 
 /* keyboard/mouse support */
@@ -28,6 +32,21 @@
 #define GUI_REFRESH_INTERVAL_DEFAULT    30
 #define GUI_REFRESH_INTERVAL_IDLE     3000
 
+/* Color number is match to standard vga palette */
+enum qemu_color_names {
+    QEMU_COLOR_BLACK   = 0,
+    QEMU_COLOR_BLUE    = 1,
+    QEMU_COLOR_GREEN   = 2,
+    QEMU_COLOR_CYAN    = 3,
+    QEMU_COLOR_RED     = 4,
+    QEMU_COLOR_MAGENTA = 5,
+    QEMU_COLOR_YELLOW  = 6,
+    QEMU_COLOR_WHITE   = 7
+};
+/* Convert to curses char attributes */
+#define ATTR2CHTYPE(c, fg, bg, bold) \
+    ((bold) << 21 | (bg) << 11 | (fg) << 8 | (c))
+
 typedef void QEMUPutKBDEvent(void *opaque, int keycode);
 typedef void QEMUPutLEDEvent(void *opaque, int ledstate);
 typedef void QEMUPutMouseEvent(void *opaque, int dx, int dy, int dz, int buttons_state);
@@ -38,7 +57,6 @@
 
 QEMUPutKbdEntry *qemu_add_kbd_event_handler(QEMUPutKBDEvent *func,
                                             void *opaque);
-void qemu_remove_kbd_event_handler(QEMUPutKbdEntry *entry);
 QEMUPutMouseEntry *qemu_add_mouse_event_handler(QEMUPutMouseEvent *func,
                                                 void *opaque, int absolute,
                                                 const char *name);
@@ -60,7 +78,7 @@
     int a[7];
 };
 
-void do_mouse_set(Monitor *mon, const QDict *qdict);
+void hmp_mouse_set(Monitor *mon, const QDict *qdict);
 
 /* keysym is a unicode code except for special keys (see QEMU_KEY_xxx
    constants) */
@@ -123,6 +141,11 @@
     pixman_format_code_t format;
     pixman_image_t *image;
     uint8_t flags;
+#ifdef CONFIG_OPENGL
+    GLenum glformat;
+    GLenum gltype;
+    GLuint texture;
+#endif
 };
 
 typedef struct QemuUIInfo {
@@ -154,6 +177,14 @@
 void cursor_get_mono_image(QEMUCursor *c, int foreground, uint8_t *mask);
 void cursor_get_mono_mask(QEMUCursor *c, int transparent, uint8_t *mask);
 
+typedef void *QEMUGLContext;
+typedef struct QEMUGLParams QEMUGLParams;
+
+struct QEMUGLParams {
+    int major_ver;
+    int minor_ver;
+};
+
 typedef struct DisplayChangeListenerOps {
     const char *dpy_name;
 
@@ -166,6 +197,8 @@
     void (*dpy_gfx_copy)(DisplayChangeListener *dcl,
                          int src_x, int src_y,
                          int dst_x, int dst_y, int w, int h);
+    bool (*dpy_gfx_check_format)(DisplayChangeListener *dcl,
+                                 pixman_format_code_t format);
 
     void (*dpy_text_cursor)(DisplayChangeListener *dcl,
                             int x, int y);
@@ -178,6 +211,22 @@
                           int x, int y, int on);
     void (*dpy_cursor_define)(DisplayChangeListener *dcl,
                               QEMUCursor *cursor);
+
+    QEMUGLContext (*dpy_gl_ctx_create)(DisplayChangeListener *dcl,
+                                       QEMUGLParams *params);
+    void (*dpy_gl_ctx_destroy)(DisplayChangeListener *dcl,
+                               QEMUGLContext ctx);
+    int (*dpy_gl_ctx_make_current)(DisplayChangeListener *dcl,
+                                   QEMUGLContext ctx);
+    QEMUGLContext (*dpy_gl_ctx_get_current)(DisplayChangeListener *dcl);
+
+    void (*dpy_gl_scanout)(DisplayChangeListener *dcl,
+                           uint32_t backing_id, bool backing_y_0_top,
+                           uint32_t backing_width, uint32_t backing_height,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+    void (*dpy_gl_update)(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
 } DisplayChangeListenerOps;
 
 struct DisplayChangeListener {
@@ -189,23 +238,15 @@
     QLIST_ENTRY(DisplayChangeListener) next;
 };
 
-struct DisplayUpdateListener {
-    void* opaque;
-    QemuConsole* con;
-
-    void (*dpy_gfx_update)(DisplayUpdateListener *dcl,
-                           int x, int y, int w, int h);
-};
-
 DisplayState *init_displaystate(void);
 DisplaySurface *qemu_create_displaysurface_from(int width, int height,
                                                 pixman_format_code_t format,
                                                 int linesize, uint8_t *data);
+DisplaySurface *qemu_create_displaysurface_pixman(pixman_image_t *image);
 DisplaySurface *qemu_create_displaysurface_guestmem(int width, int height,
                                                     pixman_format_code_t format,
                                                     int linesize,
                                                     uint64_t addr);
-PixelFormat qemu_different_endianness_pixelformat(int bpp);
 PixelFormat qemu_default_pixelformat(int bpp);
 
 DisplaySurface *qemu_create_displaysurface(int width, int height);
@@ -226,13 +267,12 @@
     return !(surface->flags & QEMU_ALLOCATED_FLAG);
 }
 
-void register_displayupdatelistener(DisplayUpdateListener *dul);
-
 void register_displaychangelistener(DisplayChangeListener *dcl);
 void update_displaychangelistener(DisplayChangeListener *dcl,
                                   uint64_t interval);
 void unregister_displaychangelistener(DisplayChangeListener *dcl);
 
+bool dpy_ui_info_supported(QemuConsole *con);
 int dpy_set_ui_info(QemuConsole *con, QemuUIInfo *info);
 
 void dpy_gfx_update(QemuConsole *con, int x, int y, int w, int h);
@@ -246,12 +286,25 @@
 void dpy_mouse_set(QemuConsole *con, int x, int y, int on);
 void dpy_cursor_define(QemuConsole *con, QEMUCursor *cursor);
 bool dpy_cursor_define_supported(QemuConsole *con);
-void dpy_gfx_update_dirty(QemuConsole *con,
-                          MemoryRegion *address_space,
-                          uint64_t base,
-                          bool invalidate);
+bool dpy_gfx_check_format(QemuConsole *con,
+                          pixman_format_code_t format);
 
-// run the display update and input processing ASAP
+void dpy_gl_scanout(QemuConsole *con,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t backing_width, uint32_t backing_height,
+                    uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void dpy_gl_update(QemuConsole *con,
+                   uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
+QEMUGLContext dpy_gl_ctx_create(QemuConsole *con,
+                                QEMUGLParams *params);
+void dpy_gl_ctx_destroy(QemuConsole *con, QEMUGLContext ctx);
+int dpy_gl_ctx_make_current(QemuConsole *con, QEMUGLContext ctx);
+QEMUGLContext dpy_gl_ctx_get_current(QemuConsole *con);
+
+bool console_has_gl(QemuConsole *con);
+
+/* Run the display update and input processing ASAP. */
 void dpy_run_update(QemuConsole* con);
 
 static inline int surface_stride(DisplaySurface *s)
@@ -286,16 +339,31 @@
     return (bits + 7) / 8;
 }
 
+static inline pixman_format_code_t surface_format(DisplaySurface *s)
+{
+    return s->format;
+}
+
 #ifdef CONFIG_CURSES
 #include <curses.h>
 typedef chtype console_ch_t;
+extern chtype vga_to_curses[];
 #else
 typedef unsigned long console_ch_t;
 #endif
 static inline void console_write_ch(console_ch_t *dest, uint32_t ch)
 {
-    if (!(ch & 0xff))
+    uint8_t c = ch;
+#ifdef CONFIG_CURSES
+    if (vga_to_curses[c]) {
+        ch &= ~(console_ch_t)0xff;
+        ch |= vga_to_curses[c];
+    }
+#else
+    if (c == '\0') {
         ch |= ' ';
+    }
+#endif
     *dest = ch;
 }
 
@@ -305,6 +373,7 @@
     void (*text_update)(void *opaque, console_ch_t *text);
     void (*update_interval)(void *opaque, uint64_t interval);
     int (*ui_info)(void *opaque, uint32_t head, QemuUIInfo *info);
+    void (*gl_block)(void *opaque, bool block);
 } GraphicHwOps;
 
 QemuConsole *graphic_console_init(DeviceState *dev, uint32_t head,
@@ -317,12 +386,16 @@
 void graphic_hw_update(QemuConsole *con);
 void graphic_hw_invalidate(QemuConsole *con);
 void graphic_hw_text_update(QemuConsole *con, console_ch_t *chardata);
+void graphic_hw_gl_block(QemuConsole *con, bool block);
 
 QemuConsole *qemu_console_lookup_by_index(unsigned int index);
 QemuConsole *qemu_console_lookup_by_device(DeviceState *dev, uint32_t head);
+QemuConsole *qemu_console_lookup_by_device_name(const char *device_id,
+                                                uint32_t head, Error **errp);
 bool qemu_console_is_visible(QemuConsole *con);
 bool qemu_console_is_graphic(QemuConsole *con);
 bool qemu_console_is_fixedsize(QemuConsole *con);
+char *qemu_console_get_label(QemuConsole *con);
 int qemu_console_get_index(QemuConsole *con);
 uint32_t qemu_console_get_head(QemuConsole *con);
 QemuUIInfo *qemu_console_get_ui_info(QemuConsole *con);
@@ -336,41 +409,124 @@
 void qemu_console_copy(QemuConsole *con, int src_x, int src_y,
                        int dst_x, int dst_y, int w, int h);
 DisplaySurface *qemu_console_surface(QemuConsole *con);
-DisplayState *qemu_console_displaystate(QemuConsole *console);
+
+/* console-gl.c */
+typedef struct ConsoleGLState ConsoleGLState;
+#ifdef CONFIG_OPENGL
+ConsoleGLState *console_gl_init_context(void);
+void console_gl_fini_context(ConsoleGLState *gls);
+bool console_gl_check_format(DisplayChangeListener *dcl,
+                             pixman_format_code_t format);
+void surface_gl_create_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface);
+void surface_gl_update_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface,
+                               int x, int y, int w, int h);
+void surface_gl_render_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface);
+void surface_gl_destroy_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface);
+void surface_gl_setup_viewport(ConsoleGLState *gls,
+                               DisplaySurface *surface,
+                               int ww, int wh);
+#endif
 
 /* sdl.c */
+#ifdef CONFIG_SDL
+void sdl_display_early_init(int opengl);
 bool sdl_display_init(DisplayState *ds, int full_screen, int no_frame);
+#else
+static inline void sdl_display_early_init(int opengl)
+{
+    /* This must never be called if CONFIG_SDL is disabled */
+    error_report("SDL support is disabled");
+    abort();
+}
+static inline bool sdl_display_init(DisplayState *ds, int full_screen,
+                                    int no_frame)
+{
+    /* This must never be called if CONFIG_SDL is disabled */
+    error_report("SDL support is disabled");
+    return false;
+}
+#endif
 
 /* cocoa.m */
+#ifdef CONFIG_COCOA
 void cocoa_display_init(DisplayState *ds, int full_screen);
+#else
+static inline void cocoa_display_init(DisplayState *ds, int full_screen)
+{
+    /* This must never be called if CONFIG_COCOA is disabled */
+    error_report("Cocoa support is disabled");
+    abort();
+}
+#endif
 
 /* vnc.c */
-void vnc_display_init(DisplayState *ds);
-void vnc_display_open(DisplayState *ds, const char *display, Error **errp);
-void vnc_display_add_client(DisplayState *ds, int csock, bool skipauth);
-char *vnc_display_local_addr(DisplayState *ds);
+void vnc_display_init(const char *id);
+void vnc_display_open(const char *id, Error **errp);
+void vnc_display_add_client(const char *id, int csock, bool skipauth);
 #ifdef CONFIG_VNC
-int vnc_display_password(DisplayState *ds, const char *password);
-int vnc_display_pw_expire(DisplayState *ds, time_t expires);
+int vnc_display_password(const char *id, const char *password);
+int vnc_display_pw_expire(const char *id, time_t expires);
+QemuOpts *vnc_parse(const char *str, Error **errp);
+int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp);
 #else
-static inline int vnc_display_password(DisplayState *ds, const char *password)
+static inline int vnc_display_password(const char *id, const char *password)
 {
     return -ENODEV;
 }
-static inline int vnc_display_pw_expire(DisplayState *ds, time_t expires)
+static inline int vnc_display_pw_expire(const char *id, time_t expires)
 {
     return -ENODEV;
 };
+static inline QemuOpts *vnc_parse(const char *str, Error **errp)
+{
+    error_setg(errp, "VNC support is disabled");
+    return NULL;
+}
+static inline int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp)
+{
+    error_setg(errp, "VNC support is disabled");
+    return -1;
+}
 #endif
 
 /* curses.c */
+#ifdef CONFIG_CURSES
 void curses_display_init(DisplayState *ds, int full_screen);
+#else
+static inline void curses_display_init(DisplayState *ds, int full_screen)
+{
+    /* This must never be called if CONFIG_CURSES is disabled */
+    error_report("curses support is disabled");
+    abort();
+}
+#endif
 
 /* input.c */
-int index_from_key(const char *key);
+int index_from_key(const char *key, size_t key_length);
 
 /* gtk.c */
-void early_gtk_display_init(void);
+#ifdef CONFIG_GTK
+void early_gtk_display_init(int opengl);
 void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover);
+#else
+static inline void gtk_display_init(DisplayState *ds, bool full_screen,
+                                    bool grab_on_hover)
+{
+    /* This must never be called if CONFIG_GTK is disabled */
+    error_report("GTK support is disabled");
+    abort();
+}
+
+static inline void early_gtk_display_init(int opengl)
+{
+    /* This must never be called if CONFIG_GTK is disabled */
+    error_report("GTK support is disabled");
+    abort();
+}
+#endif
 
 #endif
diff --git a/include/ui/egl-context.h b/include/ui/egl-context.h
new file mode 100644
index 0000000..f004ce1
--- /dev/null
+++ b/include/ui/egl-context.h
@@ -0,0 +1,14 @@
+#ifndef EGL_CONTEXT_H
+#define EGL_CONTEXT_H
+
+#include "ui/console.h"
+#include "ui/egl-helpers.h"
+
+QEMUGLContext qemu_egl_create_context(DisplayChangeListener *dcl,
+                                      QEMUGLParams *params);
+void qemu_egl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx);
+int qemu_egl_make_context_current(DisplayChangeListener *dcl,
+                                  QEMUGLContext ctx);
+QEMUGLContext qemu_egl_get_current_context(DisplayChangeListener *dcl);
+
+#endif /* EGL_CONTEXT_H */
diff --git a/include/ui/egl-helpers.h b/include/ui/egl-helpers.h
new file mode 100644
index 0000000..03fcf4b
--- /dev/null
+++ b/include/ui/egl-helpers.h
@@ -0,0 +1,28 @@
+#ifndef EGL_HELPERS_H
+#define EGL_HELPERS_H
+
+#include <epoxy/gl.h>
+#include <epoxy/egl.h>
+#include <gbm.h>
+
+extern EGLDisplay *qemu_egl_display;
+extern EGLConfig qemu_egl_config;
+
+#ifdef CONFIG_OPENGL_DMABUF
+
+extern int qemu_egl_rn_fd;
+extern struct gbm_device *qemu_egl_rn_gbm_dev;
+extern EGLContext qemu_egl_rn_ctx;
+
+int qemu_egl_rendernode_open(void);
+int egl_rendernode_init(void);
+int egl_get_fd_for_texture(uint32_t tex_id, EGLint *stride, EGLint *fourcc);
+
+#endif
+
+EGLSurface qemu_egl_init_surface_x11(EGLContext ectx, Window win);
+
+int qemu_egl_init_dpy(EGLNativeDisplayType dpy, bool gles, bool debug);
+EGLContext qemu_egl_init_ctx(void);
+
+#endif /* EGL_HELPERS_H */
diff --git a/include/ui/gtk.h b/include/ui/gtk.h
new file mode 100644
index 0000000..42ca0fe
--- /dev/null
+++ b/include/ui/gtk.h
@@ -0,0 +1,137 @@
+#ifndef UI_GTK_H
+#define UI_GTK_H
+
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+/* Work around an -Wstrict-prototypes warning in GTK headers */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wstrict-prototypes"
+#endif
+#include <gtk/gtk.h>
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic pop
+#endif
+
+#include <gdk/gdkkeysyms.h>
+
+#ifdef GDK_WINDOWING_X11
+#include <gdk/gdkx.h>
+#include <X11/XKBlib.h>
+#endif
+
+#if defined(CONFIG_OPENGL)
+#include "ui/egl-helpers.h"
+#include "ui/egl-context.h"
+#endif
+
+/* Compatibility define to let us build on both Gtk2 and Gtk3 */
+#if GTK_CHECK_VERSION(3, 0, 0)
+static inline void gdk_drawable_get_size(GdkWindow *w, gint *ww, gint *wh)
+{
+    *ww = gdk_window_get_width(w);
+    *wh = gdk_window_get_height(w);
+}
+#endif
+
+typedef struct GtkDisplayState GtkDisplayState;
+
+typedef struct VirtualGfxConsole {
+    GtkWidget *drawing_area;
+    DisplayChangeListener dcl;
+    DisplaySurface *ds;
+    pixman_image_t *convert;
+    cairo_surface_t *surface;
+    double scale_x;
+    double scale_y;
+#if defined(CONFIG_OPENGL)
+    ConsoleGLState *gls;
+    EGLContext ectx;
+    EGLSurface esurface;
+    int glupdates;
+    int x, y, w, h;
+    GLuint tex_id;
+    GLuint fbo_id;
+    bool y0_top;
+    bool scanout_mode;
+#endif
+} VirtualGfxConsole;
+
+#if defined(CONFIG_VTE)
+typedef struct VirtualVteConsole {
+    GtkWidget *box;
+    GtkWidget *scrollbar;
+    GtkWidget *terminal;
+    CharDriverState *chr;
+    bool echo;
+} VirtualVteConsole;
+#endif
+
+typedef enum VirtualConsoleType {
+    GD_VC_GFX,
+    GD_VC_VTE,
+} VirtualConsoleType;
+
+typedef struct VirtualConsole {
+    GtkDisplayState *s;
+    char *label;
+    GtkWidget *window;
+    GtkWidget *menu_item;
+    GtkWidget *tab_item;
+    GtkWidget *focus;
+    VirtualConsoleType type;
+    union {
+        VirtualGfxConsole gfx;
+#if defined(CONFIG_VTE)
+        VirtualVteConsole vte;
+#endif
+    };
+} VirtualConsole;
+
+/* ui/gtk.c */
+void gd_update_windowsize(VirtualConsole *vc);
+
+/* ui/gtk-egl.c */
+void gd_egl_init(VirtualConsole *vc);
+void gd_egl_draw(VirtualConsole *vc);
+void gd_egl_update(DisplayChangeListener *dcl,
+                   int x, int y, int w, int h);
+void gd_egl_refresh(DisplayChangeListener *dcl);
+void gd_egl_switch(DisplayChangeListener *dcl,
+                   DisplaySurface *surface);
+QEMUGLContext gd_egl_create_context(DisplayChangeListener *dcl,
+                                    QEMUGLParams *params);
+void gd_egl_scanout(DisplayChangeListener *dcl,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t backing_width, uint32_t backing_height,
+                    uint32_t x, uint32_t y,
+                    uint32_t w, uint32_t h);
+void gd_egl_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void gtk_egl_init(void);
+int gd_egl_make_current(DisplayChangeListener *dcl,
+                        QEMUGLContext ctx);
+
+/* ui/gtk-gl-area.c */
+void gd_gl_area_init(VirtualConsole *vc);
+void gd_gl_area_draw(VirtualConsole *vc);
+void gd_gl_area_update(DisplayChangeListener *dcl,
+                       int x, int y, int w, int h);
+void gd_gl_area_refresh(DisplayChangeListener *dcl);
+void gd_gl_area_switch(DisplayChangeListener *dcl,
+                       DisplaySurface *surface);
+QEMUGLContext gd_gl_area_create_context(DisplayChangeListener *dcl,
+                                        QEMUGLParams *params);
+void gd_gl_area_destroy_context(DisplayChangeListener *dcl,
+                                QEMUGLContext ctx);
+void gd_gl_area_scanout(DisplayChangeListener *dcl,
+                        uint32_t backing_id, bool backing_y_0_top,
+                        uint32_t backing_width, uint32_t backing_height,
+                        uint32_t x, uint32_t y,
+                        uint32_t w, uint32_t h);
+void gd_gl_area_scanout_flush(DisplayChangeListener *dcl,
+                              uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+void gtk_gl_area_init(void);
+QEMUGLContext gd_gl_area_get_current_context(DisplayChangeListener *dcl);
+int gd_gl_area_make_current(DisplayChangeListener *dcl,
+                            QEMUGLContext ctx);
+
+#endif /* UI_GTK_H */
diff --git a/include/ui/input.h b/include/ui/input.h
index 5d5ac00..102d8a3 100644
--- a/include/ui/input.h
+++ b/include/ui/input.h
@@ -33,7 +33,9 @@
                              const char *device_id, int head,
                              Error **errp);
 void qemu_input_event_send(QemuConsole *src, InputEvent *evt);
+void qemu_input_event_send_impl(QemuConsole *src, InputEvent *evt);
 void qemu_input_event_sync(void);
+void qemu_input_event_sync_impl(void);
 
 InputEvent *qemu_input_event_new_key(KeyValue *key, bool down);
 void qemu_input_event_send_key(QemuConsole *src, KeyValue *key, bool down);
@@ -63,4 +65,6 @@
 void qemu_add_mouse_mode_change_notifier(Notifier *notify);
 void qemu_remove_mouse_mode_change_notifier(Notifier *notify);
 
+int input_linux_init(void *opaque, QemuOpts *opts, Error **errp);
+
 #endif /* INPUT_H */
diff --git a/include/ui/qemu-pixman.h b/include/ui/qemu-pixman.h
index 48981a0..3a19649 100644
--- a/include/ui/qemu-pixman.h
+++ b/include/ui/qemu-pixman.h
@@ -16,7 +16,7 @@
 #pragma GCC diagnostic pop
 #endif
 
-#include "qemu/typedefs.h"
+#include <stdbool.h>
 
 #include <stdbool.h>
 
@@ -29,8 +29,26 @@
 
 #ifdef HOST_WORDS_BIGENDIAN
 # define PIXMAN_BE_r8g8b8     PIXMAN_r8g8b8
+# define PIXMAN_BE_x8r8g8b8   PIXMAN_x8r8g8b8
+# define PIXMAN_BE_a8r8g8b8   PIXMAN_a8r8g8b8
+# define PIXMAN_BE_b8g8r8x8   PIXMAN_b8g8r8x8
+# define PIXMAN_BE_b8g8r8a8   PIXMAN_b8g8r8a8
+# define PIXMAN_BE_r8g8b8x8   PIXMAN_r8g8b8x8
+# define PIXMAN_BE_r8g8b8a8   PIXMAN_r8g8b8a8
+# define PIXMAN_BE_x8b8g8r8   PIXMAN_x8b8g8r8
+# define PIXMAN_BE_a8b8g8r8   PIXMAN_a8b8g8r8
+# define PIXMAN_LE_x8r8g8b8   PIXMAN_b8g8r8x8
 #else
 # define PIXMAN_BE_r8g8b8     PIXMAN_b8g8r8
+# define PIXMAN_BE_x8r8g8b8   PIXMAN_b8g8r8x8
+# define PIXMAN_BE_a8r8g8b8   PIXMAN_b8g8r8a8
+# define PIXMAN_BE_b8g8r8x8   PIXMAN_x8r8g8b8
+# define PIXMAN_BE_b8g8r8a8   PIXMAN_a8r8g8b8
+# define PIXMAN_BE_r8g8b8x8   PIXMAN_x8b8g8r8
+# define PIXMAN_BE_r8g8b8a8   PIXMAN_a8b8g8r8
+# define PIXMAN_BE_x8b8g8r8   PIXMAN_r8g8b8x8
+# define PIXMAN_BE_a8b8g8r8   PIXMAN_r8g8b8a8
+# define PIXMAN_LE_x8r8g8b8   PIXMAN_x8r8g8b8
 #endif
 
 /* -------------------------------------------------------------------- */
@@ -39,6 +57,8 @@
 pixman_format_code_t qemu_default_pixman_format(int bpp, bool native_endian);
 int qemu_pixman_get_type(int rshift, int gshift, int bshift);
 pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf);
+bool qemu_pixman_check_format(DisplayChangeListener *dcl,
+                              pixman_format_code_t format);
 
 pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
                                            int width);
diff --git a/include/ui/qemu-spice.h b/include/ui/qemu-spice.h
index a93b4b2..75e1239 100644
--- a/include/ui/qemu-spice.h
+++ b/include/ui/qemu-spice.h
@@ -18,15 +18,13 @@
 #ifndef QEMU_SPICE_H
 #define QEMU_SPICE_H
 
-#include "config-host.h"
+#include "qapi/error.h"
 
 #ifdef CONFIG_SPICE
 
 #include <spice.h>
-
 #include "qemu/option.h"
 #include "qemu/config-file.h"
-#include "monitor/monitor.h"
 
 extern int using_spice;
 
@@ -42,15 +40,15 @@
                           bool fail_if_connected, bool disconnect_if_connected);
 int qemu_spice_set_pw_expire(time_t expires);
 int qemu_spice_migrate_info(const char *hostname, int port, int tls_port,
-                            const char *subject,
-                            MonitorCompletion cb, void *opaque);
+                            const char *subject);
 
-void do_info_spice_print(Monitor *mon, const QObject *data);
-void do_info_spice(Monitor *mon, QObject **ret_data);
+#if !defined(SPICE_SERVER_VERSION) || (SPICE_SERVER_VERSION < 0xc06)
+#define SPICE_NEEDS_SET_MM_TIME 1
+#else
+#define SPICE_NEEDS_SET_MM_TIME 0
+#endif
 
-CharDriverState *qemu_chr_open_spice_vmc(const char *type);
 #if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name);
 void qemu_spice_register_ports(void);
 #else
 static inline CharDriverState *qemu_chr_open_spice_port(const char *name)
@@ -58,7 +56,8 @@
 #endif
 
 #else  /* CONFIG_SPICE */
-#include "monitor/monitor.h"
+
+#include "qemu/error-report.h"
 
 #define using_spice 0
 #define spice_displays 0
@@ -73,10 +72,8 @@
     return -1;
 }
 static inline int qemu_spice_migrate_info(const char *h, int p, int t,
-                                          const char *s,
-                                          MonitorCompletion cb, void *opaque)
+                                          const char *s)
 {
-    cb(opaque, NULL);
     return -1;
 }
 
@@ -86,6 +83,27 @@
     return -1;
 }
 
+static inline void qemu_spice_display_init(void)
+{
+    /* This must never be called if CONFIG_SPICE is disabled */
+    error_report("spice support is disabled");
+    abort();
+}
+
+static inline void qemu_spice_init(void)
+{
+}
+
 #endif /* CONFIG_SPICE */
 
+static inline bool qemu_using_spice(Error **errp)
+{
+    if (!using_spice) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_ACTIVE,
+                  "SPICE is not in use");
+        return false;
+    }
+    return true;
+}
+
 #endif /* QEMU_SPICE_H */
diff --git a/include/ui/sdl2.h b/include/ui/sdl2.h
new file mode 100644
index 0000000..683bb6a
--- /dev/null
+++ b/include/ui/sdl2.h
@@ -0,0 +1,73 @@
+#ifndef SDL2_H
+#define SDL2_H
+
+/* Avoid compiler warning because macro is redefined in SDL_syswm.h. */
+#undef WIN32_LEAN_AND_MEAN
+
+#include <SDL.h>
+#include <SDL_syswm.h>
+
+struct sdl2_console {
+    DisplayChangeListener dcl;
+    DisplaySurface *surface;
+    SDL_Texture *texture;
+    SDL_Window *real_window;
+    SDL_Renderer *real_renderer;
+    int idx;
+    int last_vm_running; /* per console for caption reasons */
+    int x, y, w, h;
+    int hidden;
+    int opengl;
+    int updates;
+    int idle_counter;
+    SDL_GLContext winctx;
+#ifdef CONFIG_OPENGL
+    ConsoleGLState *gls;
+    GLuint tex_id;
+    GLuint fbo_id;
+    bool y0_top;
+    bool scanout_mode;
+#endif
+};
+
+void sdl2_window_create(struct sdl2_console *scon);
+void sdl2_window_destroy(struct sdl2_console *scon);
+void sdl2_window_resize(struct sdl2_console *scon);
+void sdl2_poll_events(struct sdl2_console *scon);
+
+void sdl2_reset_keys(struct sdl2_console *scon);
+void sdl2_process_key(struct sdl2_console *scon,
+                      SDL_KeyboardEvent *ev);
+
+void sdl2_2d_update(DisplayChangeListener *dcl,
+                    int x, int y, int w, int h);
+void sdl2_2d_switch(DisplayChangeListener *dcl,
+                    DisplaySurface *new_surface);
+void sdl2_2d_refresh(DisplayChangeListener *dcl);
+void sdl2_2d_redraw(struct sdl2_console *scon);
+bool sdl2_2d_check_format(DisplayChangeListener *dcl,
+                          pixman_format_code_t format);
+
+void sdl2_gl_update(DisplayChangeListener *dcl,
+                    int x, int y, int w, int h);
+void sdl2_gl_switch(DisplayChangeListener *dcl,
+                    DisplaySurface *new_surface);
+void sdl2_gl_refresh(DisplayChangeListener *dcl);
+void sdl2_gl_redraw(struct sdl2_console *scon);
+
+QEMUGLContext sdl2_gl_create_context(DisplayChangeListener *dcl,
+                                     QEMUGLParams *params);
+void sdl2_gl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx);
+int sdl2_gl_make_context_current(DisplayChangeListener *dcl,
+                                 QEMUGLContext ctx);
+QEMUGLContext sdl2_gl_get_current_context(DisplayChangeListener *dcl);
+
+void sdl2_gl_scanout(DisplayChangeListener *dcl,
+                     uint32_t backing_id, bool backing_y_0_top,
+                     uint32_t backing_width, uint32_t backing_height,
+                     uint32_t x, uint32_t y,
+                     uint32_t w, uint32_t h);
+void sdl2_gl_scanout_flush(DisplayChangeListener *dcl,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h);
+
+#endif /* SDL2_H */
diff --git a/include/ui/shader.h b/include/ui/shader.h
new file mode 100644
index 0000000..f7d8618
--- /dev/null
+++ b/include/ui/shader.h
@@ -0,0 +1,15 @@
+#ifndef QEMU_SHADER_H
+#define QEMU_SHADER_H
+
+#include <epoxy/gl.h>
+
+GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog);
+void qemu_gl_run_texture_blit(GLint texture_blit_prog,
+                              GLint texture_blit_vao);
+
+GLuint qemu_gl_create_compile_shader(GLenum type, const GLchar *src);
+GLuint qemu_gl_create_link_program(GLuint vert, GLuint frag);
+GLuint qemu_gl_create_compile_link_program(const GLchar *vert_src,
+                                           const GLchar *frag_src);
+
+#endif /* QEMU_SHADER_H */
diff --git a/include/ui/spice-display.h b/include/ui/spice-display.h
index 4252ab8..568b64a 100644
--- a/include/ui/spice-display.h
+++ b/include/ui/spice-display.h
@@ -24,6 +24,14 @@
 #include "ui/console.h"
 #include "sysemu/sysemu.h"
 
+#if defined(CONFIG_OPENGL_DMABUF)
+# if SPICE_SERVER_VERSION >= 0x000d01 /* release 0.13.1 */
+#  define HAVE_SPICE_GL 1
+#  include "ui/egl-helpers.h"
+#  include "ui/egl-context.h"
+# endif
+#endif
+
 #define NUM_MEMSLOTS 8
 #define MEMSLOT_GENERATION_BITS 8
 #define MEMSLOT_SLOT_BITS 8
@@ -50,6 +58,7 @@
     QXL_COOKIE_TYPE_IO,
     QXL_COOKIE_TYPE_RENDER_UPDATE_AREA,
     QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
+    QXL_COOKIE_TYPE_GL_DRAW_DONE,
 };
 
 typedef struct QXLCookie {
@@ -62,6 +71,7 @@
             QXLRect area;
             int redraw;
         } render;
+        void *data;
     } u;
 } QXLCookie;
 
@@ -97,11 +107,20 @@
     /* cursor (without qxl): displaychangelistener -> spice server */
     SimpleSpiceCursor *ptr_define;
     SimpleSpiceCursor *ptr_move;
-    uint16_t ptr_x, ptr_y;
+    int16_t ptr_x, ptr_y;
+    int16_t hot_x, hot_y;
 
     /* cursor (with qxl): qxl local renderer -> displaychangelistener */
     QEMUCursor *cursor;
     int mouse_x, mouse_y;
+    QEMUBH *cursor_bh;
+
+#ifdef HAVE_SPICE_GL
+    /* opengl rendering */
+    QEMUBH *gl_unblock_bh;
+    QEMUTimer *gl_unblock_timer;
+    int dmabuf_fd;
+#endif
 };
 
 struct SimpleSpiceUpdate {
@@ -134,7 +153,7 @@
 void qemu_spice_display_switch(SimpleSpiceDisplay *ssd,
                                DisplaySurface *surface);
 void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd);
-void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd);
+void qemu_spice_cursor_refresh_bh(void *opaque);
 
 void qemu_spice_add_memslot(SimpleSpiceDisplay *ssd, QXLDevMemSlot *memslot,
                             qxl_async_io async);
diff --git a/io/Makefile.objs b/io/Makefile.objs
new file mode 100644
index 0000000..9d8337d
--- /dev/null
+++ b/io/Makefile.objs
@@ -0,0 +1,10 @@
+io-obj-y = channel.o
+io-obj-y += channel-buffer.o
+io-obj-y += channel-command.o
+io-obj-y += channel-file.o
+io-obj-y += channel-socket.o
+io-obj-y += channel-tls.o
+io-obj-y += channel-watch.o
+io-obj-y += channel-websock.o
+io-obj-y += channel-util.o
+io-obj-y += task.o
diff --git a/io/channel-buffer.c b/io/channel-buffer.c
new file mode 100644
index 0000000..43d7959
--- /dev/null
+++ b/io/channel-buffer.c
@@ -0,0 +1,250 @@
+/*
+ * QEMU I/O channels memory buffer driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-buffer.h"
+#include "io/channel-watch.h"
+#include "qemu/sockets.h"
+#include "trace.h"
+
+QIOChannelBuffer *
+qio_channel_buffer_new(size_t capacity)
+{
+    QIOChannelBuffer *ioc;
+
+    ioc = QIO_CHANNEL_BUFFER(object_new(TYPE_QIO_CHANNEL_BUFFER));
+
+    if (capacity) {
+        ioc->data = g_new0(uint8_t, capacity);
+        ioc->capacity = capacity;
+    }
+
+    return ioc;
+}
+
+
+static void qio_channel_buffer_finalize(Object *obj)
+{
+    QIOChannelBuffer *ioc = QIO_CHANNEL_BUFFER(obj);
+    g_free(ioc->data);
+    ioc->capacity = ioc->usage = ioc->offset = 0;
+}
+
+
+static ssize_t qio_channel_buffer_readv(QIOChannel *ioc,
+                                        const struct iovec *iov,
+                                        size_t niov,
+                                        int **fds,
+                                        size_t *nfds,
+                                        Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+    ssize_t ret = 0;
+    size_t i;
+
+    for (i = 0; i < niov; i++) {
+        size_t want = iov[i].iov_len;
+        if (bioc->offset >= bioc->usage) {
+            break;
+        }
+        if ((bioc->offset + want) > bioc->usage)  {
+            want = bioc->usage - bioc->offset;
+        }
+        memcpy(iov[i].iov_base, bioc->data + bioc->offset, want);
+        ret += want;
+        bioc->offset += want;
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_buffer_writev(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int *fds,
+                                         size_t nfds,
+                                         Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+    ssize_t ret = 0;
+    size_t i;
+    size_t towrite = 0;
+
+    for (i = 0; i < niov; i++) {
+        towrite += iov[i].iov_len;
+    }
+
+    if ((bioc->offset + towrite) > bioc->capacity) {
+        bioc->capacity = bioc->offset + towrite;
+        bioc->data = g_realloc(bioc->data, bioc->capacity);
+    }
+
+    if (bioc->offset > bioc->usage) {
+        memset(bioc->data, 0, bioc->offset - bioc->usage);
+        bioc->usage = bioc->offset;
+    }
+
+    for (i = 0; i < niov; i++) {
+        memcpy(bioc->data + bioc->usage,
+               iov[i].iov_base,
+               iov[i].iov_len);
+        bioc->usage += iov[i].iov_len;
+        bioc->offset += iov[i].iov_len;
+        ret += iov[i].iov_len;
+    }
+
+    return ret;
+}
+
+static int qio_channel_buffer_set_blocking(QIOChannel *ioc G_GNUC_UNUSED,
+                                           bool enabled G_GNUC_UNUSED,
+                                           Error **errp G_GNUC_UNUSED)
+{
+    return 0;
+}
+
+
+static off_t qio_channel_buffer_seek(QIOChannel *ioc,
+                                     off_t offset,
+                                     int whence,
+                                     Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+
+    bioc->offset = offset;
+
+    return offset;
+}
+
+
+static int qio_channel_buffer_close(QIOChannel *ioc,
+                                    Error **errp)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+
+    g_free(bioc->data);
+    bioc->data = NULL;
+    bioc->capacity = bioc->usage = bioc->offset = 0;
+
+    return 0;
+}
+
+
+typedef struct QIOChannelBufferSource QIOChannelBufferSource;
+struct QIOChannelBufferSource {
+    GSource parent;
+    QIOChannelBuffer *bioc;
+    GIOCondition condition;
+};
+
+static gboolean
+qio_channel_buffer_source_prepare(GSource *source,
+                                  gint *timeout)
+{
+    QIOChannelBufferSource *bsource = (QIOChannelBufferSource *)source;
+
+    *timeout = -1;
+
+    return (G_IO_IN | G_IO_OUT) & bsource->condition;
+}
+
+static gboolean
+qio_channel_buffer_source_check(GSource *source)
+{
+    QIOChannelBufferSource *bsource = (QIOChannelBufferSource *)source;
+
+    return (G_IO_IN | G_IO_OUT) & bsource->condition;
+}
+
+static gboolean
+qio_channel_buffer_source_dispatch(GSource *source,
+                                   GSourceFunc callback,
+                                   gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelBufferSource *bsource = (QIOChannelBufferSource *)source;
+
+    return (*func)(QIO_CHANNEL(bsource->bioc),
+                   ((G_IO_IN | G_IO_OUT) & bsource->condition),
+                   user_data);
+}
+
+static void
+qio_channel_buffer_source_finalize(GSource *source)
+{
+    QIOChannelBufferSource *ssource = (QIOChannelBufferSource *)source;
+
+    object_unref(OBJECT(ssource->bioc));
+}
+
+GSourceFuncs qio_channel_buffer_source_funcs = {
+    qio_channel_buffer_source_prepare,
+    qio_channel_buffer_source_check,
+    qio_channel_buffer_source_dispatch,
+    qio_channel_buffer_source_finalize
+};
+
+static GSource *qio_channel_buffer_create_watch(QIOChannel *ioc,
+                                                GIOCondition condition)
+{
+    QIOChannelBuffer *bioc = QIO_CHANNEL_BUFFER(ioc);
+    QIOChannelBufferSource *ssource;
+    GSource *source;
+
+    source = g_source_new(&qio_channel_buffer_source_funcs,
+                          sizeof(QIOChannelBufferSource));
+    ssource = (QIOChannelBufferSource *)source;
+
+    ssource->bioc = bioc;
+    object_ref(OBJECT(bioc));
+
+    ssource->condition = condition;
+
+    return source;
+}
+
+
+static void qio_channel_buffer_class_init(ObjectClass *klass,
+                                          void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_buffer_writev;
+    ioc_klass->io_readv = qio_channel_buffer_readv;
+    ioc_klass->io_set_blocking = qio_channel_buffer_set_blocking;
+    ioc_klass->io_seek = qio_channel_buffer_seek;
+    ioc_klass->io_close = qio_channel_buffer_close;
+    ioc_klass->io_create_watch = qio_channel_buffer_create_watch;
+}
+
+static const TypeInfo qio_channel_buffer_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_BUFFER,
+    .instance_size = sizeof(QIOChannelBuffer),
+    .instance_finalize = qio_channel_buffer_finalize,
+    .class_init = qio_channel_buffer_class_init,
+};
+
+static void qio_channel_buffer_register_types(void)
+{
+    type_register_static(&qio_channel_buffer_info);
+}
+
+type_init(qio_channel_buffer_register_types);
diff --git a/io/channel-command.c b/io/channel-command.c
new file mode 100644
index 0000000..ad25313
--- /dev/null
+++ b/io/channel-command.c
@@ -0,0 +1,368 @@
+/*
+ * QEMU I/O channels external command driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-command.h"
+#include "io/channel-watch.h"
+#include "qapi/error.h"
+#include "qemu/sockets.h"
+#include "trace.h"
+
+
+QIOChannelCommand *
+qio_channel_command_new_pid(int writefd,
+                            int readfd,
+                            pid_t pid)
+{
+    QIOChannelCommand *ioc;
+
+    ioc = QIO_CHANNEL_COMMAND(object_new(TYPE_QIO_CHANNEL_COMMAND));
+
+    ioc->readfd = readfd;
+    ioc->writefd = writefd;
+    ioc->pid = pid;
+
+    trace_qio_channel_command_new_pid(ioc, writefd, readfd, pid);
+    return ioc;
+}
+
+
+#ifndef WIN32
+QIOChannelCommand *
+qio_channel_command_new_spawn(const char *const argv[],
+                              int flags,
+                              Error **errp)
+{
+    pid_t pid = -1;
+    int stdinfd[2] = { -1, -1 };
+    int stdoutfd[2] = { -1, -1 };
+    int devnull = -1;
+    bool stdinnull = false, stdoutnull = false;
+    QIOChannelCommand *ioc;
+
+    flags = flags & O_ACCMODE;
+
+    if (flags == O_RDONLY) {
+        stdinnull = true;
+    }
+    if (flags == O_WRONLY) {
+        stdoutnull = true;
+    }
+
+    if (stdinnull || stdoutnull) {
+        devnull = open("/dev/null", O_RDWR);
+        if (devnull < 0) {
+            error_setg_errno(errp, errno,
+                             "Unable to open /dev/null");
+            goto error;
+        }
+    }
+
+    if ((!stdinnull && pipe(stdinfd) < 0) ||
+        (!stdoutnull && pipe(stdoutfd) < 0)) {
+        error_setg_errno(errp, errno,
+                         "Unable to open pipe");
+        goto error;
+    }
+
+    pid = qemu_fork(errp);
+    if (pid < 0) {
+        goto error;
+    }
+
+    if (pid == 0) { /* child */
+        dup2(stdinnull ? devnull : stdinfd[0], STDIN_FILENO);
+        dup2(stdoutnull ? devnull : stdoutfd[1], STDOUT_FILENO);
+        /* Leave stderr connected to qemu's stderr */
+
+        if (!stdinnull) {
+            close(stdinfd[0]);
+            close(stdinfd[1]);
+        }
+        if (!stdoutnull) {
+            close(stdoutfd[0]);
+            close(stdoutfd[1]);
+        }
+        if (devnull != -1) {
+            close(devnull);
+        }
+
+        execv(argv[0], (char * const *)argv);
+        _exit(1);
+    }
+
+    if (!stdinnull) {
+        close(stdinfd[0]);
+    }
+    if (!stdoutnull) {
+        close(stdoutfd[1]);
+    }
+
+    ioc = qio_channel_command_new_pid(stdinnull ? devnull : stdinfd[1],
+                                      stdoutnull ? devnull : stdoutfd[0],
+                                      pid);
+    trace_qio_channel_command_new_spawn(ioc, argv[0], flags);
+    return ioc;
+
+ error:
+    if (devnull != -1) {
+        close(devnull);
+    }
+    if (stdinfd[0] != -1) {
+        close(stdinfd[0]);
+    }
+    if (stdinfd[1] != -1) {
+        close(stdinfd[1]);
+    }
+    if (stdoutfd[0] != -1) {
+        close(stdoutfd[0]);
+    }
+    if (stdoutfd[1] != -1) {
+        close(stdoutfd[1]);
+    }
+    return NULL;
+}
+
+#else /* WIN32 */
+QIOChannelCommand *
+qio_channel_command_new_spawn(const char *const argv[],
+                              int flags,
+                              Error **errp)
+{
+    error_setg_errno(errp, ENOSYS,
+                     "Command spawn not supported on this platform");
+    return NULL;
+}
+#endif /* WIN32 */
+
+#ifndef WIN32
+static int qio_channel_command_abort(QIOChannelCommand *ioc,
+                                     Error **errp)
+{
+    pid_t ret;
+    int status;
+    int step = 0;
+
+    /* See if intermediate process has exited; if not, try a nice
+     * SIGTERM followed by a more severe SIGKILL.
+     */
+ rewait:
+    trace_qio_channel_command_abort(ioc, ioc->pid);
+    ret = waitpid(ioc->pid, &status, WNOHANG);
+    trace_qio_channel_command_wait(ioc, ioc->pid, ret, status);
+    if (ret == (pid_t)-1) {
+        if (errno == EINTR) {
+            goto rewait;
+        } else {
+            error_setg_errno(errp, errno,
+                             "Cannot wait on pid %llu",
+                             (unsigned long long)ioc->pid);
+            return -1;
+        }
+    } else if (ret == 0) {
+        if (step == 0) {
+            kill(ioc->pid, SIGTERM);
+        } else if (step == 1) {
+            kill(ioc->pid, SIGKILL);
+        } else {
+            error_setg(errp,
+                       "Process %llu refused to die",
+                       (unsigned long long)ioc->pid);
+            return -1;
+        }
+        step++;
+        usleep(10 * 1000);
+        goto rewait;
+    }
+
+    return 0;
+}
+#endif /* ! WIN32 */
+
+
+static void qio_channel_command_init(Object *obj)
+{
+    QIOChannelCommand *ioc = QIO_CHANNEL_COMMAND(obj);
+    ioc->readfd = -1;
+    ioc->writefd = -1;
+    ioc->pid = -1;
+}
+
+static void qio_channel_command_finalize(Object *obj)
+{
+    QIOChannelCommand *ioc = QIO_CHANNEL_COMMAND(obj);
+    if (ioc->readfd != -1) {
+        close(ioc->readfd);
+    }
+    if (ioc->writefd != -1 &&
+        ioc->writefd != ioc->readfd) {
+        close(ioc->writefd);
+    }
+    ioc->writefd = ioc->readfd = -1;
+    if (ioc->pid > 0) {
+#ifndef WIN32
+        qio_channel_command_abort(ioc, NULL);
+#endif
+    }
+}
+
+
+static ssize_t qio_channel_command_readv(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int **fds,
+                                         size_t *nfds,
+                                         Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = readv(cioc->readfd, iov, niov);
+    if (ret < 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+
+        error_setg_errno(errp, errno,
+                         "Unable to read from command");
+        return -1;
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_command_writev(QIOChannel *ioc,
+                                          const struct iovec *iov,
+                                          size_t niov,
+                                          int *fds,
+                                          size_t nfds,
+                                          Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = writev(cioc->writefd, iov, niov);
+    if (ret <= 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to write to command");
+        return -1;
+    }
+    return ret;
+}
+
+static int qio_channel_command_set_blocking(QIOChannel *ioc,
+                                            bool enabled,
+                                            Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+
+    if (enabled) {
+        qemu_set_block(cioc->writefd);
+        qemu_set_block(cioc->readfd);
+    } else {
+        qemu_set_nonblock(cioc->writefd);
+        qemu_set_nonblock(cioc->readfd);
+    }
+
+    return 0;
+}
+
+
+static int qio_channel_command_close(QIOChannel *ioc,
+                                     Error **errp)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    int rv = 0;
+
+    /* We close FDs before killing, because that
+     * gives a better chance of clean shutdown
+     */
+    if (cioc->readfd != -1 &&
+        close(cioc->readfd) < 0) {
+        rv = -1;
+    }
+    if (cioc->writefd != -1 &&
+        cioc->writefd != cioc->readfd &&
+        close(cioc->writefd) < 0) {
+        rv = -1;
+    }
+    cioc->writefd = cioc->readfd = -1;
+#ifndef WIN32
+    if (qio_channel_command_abort(cioc, errp) < 0) {
+        return -1;
+    }
+#endif
+    if (rv < 0) {
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to close command");
+    }
+    return rv;
+}
+
+
+static GSource *qio_channel_command_create_watch(QIOChannel *ioc,
+                                                 GIOCondition condition)
+{
+    QIOChannelCommand *cioc = QIO_CHANNEL_COMMAND(ioc);
+    return qio_channel_create_fd_pair_watch(ioc,
+                                            cioc->readfd,
+                                            cioc->writefd,
+                                            condition);
+}
+
+
+static void qio_channel_command_class_init(ObjectClass *klass,
+                                           void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_command_writev;
+    ioc_klass->io_readv = qio_channel_command_readv;
+    ioc_klass->io_set_blocking = qio_channel_command_set_blocking;
+    ioc_klass->io_close = qio_channel_command_close;
+    ioc_klass->io_create_watch = qio_channel_command_create_watch;
+}
+
+static const TypeInfo qio_channel_command_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_COMMAND,
+    .instance_size = sizeof(QIOChannelCommand),
+    .instance_init = qio_channel_command_init,
+    .instance_finalize = qio_channel_command_finalize,
+    .class_init = qio_channel_command_class_init,
+};
+
+static void qio_channel_command_register_types(void)
+{
+    type_register_static(&qio_channel_command_info);
+}
+
+type_init(qio_channel_command_register_types);
diff --git a/io/channel-file.c b/io/channel-file.c
new file mode 100644
index 0000000..e1da243
--- /dev/null
+++ b/io/channel-file.c
@@ -0,0 +1,225 @@
+/*
+ * QEMU I/O channels files driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-file.h"
+#include "io/channel-watch.h"
+#include "qapi/error.h"
+#include "qemu/sockets.h"
+#include "trace.h"
+
+QIOChannelFile *
+qio_channel_file_new_fd(int fd)
+{
+    QIOChannelFile *ioc;
+
+    ioc = QIO_CHANNEL_FILE(object_new(TYPE_QIO_CHANNEL_FILE));
+
+    ioc->fd = fd;
+
+    trace_qio_channel_file_new_fd(ioc, fd);
+
+    return ioc;
+}
+
+
+QIOChannelFile *
+qio_channel_file_new_path(const char *path,
+                          int flags,
+                          mode_t mode,
+                          Error **errp)
+{
+    QIOChannelFile *ioc;
+
+    ioc = QIO_CHANNEL_FILE(object_new(TYPE_QIO_CHANNEL_FILE));
+
+    if (flags & O_WRONLY) {
+        ioc->fd = open(path, flags, mode);
+    } else {
+        ioc->fd = open(path, flags);
+    }
+    if (ioc->fd < 0) {
+        object_unref(OBJECT(ioc));
+        error_setg_errno(errp, errno,
+                         "Unable to open %s", path);
+        return NULL;
+    }
+
+    trace_qio_channel_file_new_path(ioc, path, flags, mode, ioc->fd);
+
+    return ioc;
+}
+
+
+static void qio_channel_file_init(Object *obj)
+{
+    QIOChannelFile *ioc = QIO_CHANNEL_FILE(obj);
+    ioc->fd = -1;
+}
+
+static void qio_channel_file_finalize(Object *obj)
+{
+    QIOChannelFile *ioc = QIO_CHANNEL_FILE(obj);
+    if (ioc->fd != -1) {
+        close(ioc->fd);
+        ioc->fd = -1;
+    }
+}
+
+
+static ssize_t qio_channel_file_readv(QIOChannel *ioc,
+                                      const struct iovec *iov,
+                                      size_t niov,
+                                      int **fds,
+                                      size_t *nfds,
+                                      Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = readv(fioc->fd, iov, niov);
+    if (ret < 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+
+        error_setg_errno(errp, errno,
+                         "Unable to read from file");
+        return -1;
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_file_writev(QIOChannel *ioc,
+                                       const struct iovec *iov,
+                                       size_t niov,
+                                       int *fds,
+                                       size_t nfds,
+                                       Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    ssize_t ret;
+
+ retry:
+    ret = writev(fioc->fd, iov, niov);
+    if (ret <= 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+        error_setg_errno(errp, errno,
+                         "Unable to write to file");
+        return -1;
+    }
+    return ret;
+}
+
+static int qio_channel_file_set_blocking(QIOChannel *ioc,
+                                         bool enabled,
+                                         Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+
+    if (enabled) {
+        qemu_set_block(fioc->fd);
+    } else {
+        qemu_set_nonblock(fioc->fd);
+    }
+    return 0;
+}
+
+
+static off_t qio_channel_file_seek(QIOChannel *ioc,
+                                   off_t offset,
+                                   int whence,
+                                   Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    off_t ret;
+
+    ret = lseek(fioc->fd, offset, whence);
+    if (ret == (off_t)-1) {
+        error_setg_errno(errp, errno,
+                         "Unable to seek to offset %lld whence %d in file",
+                         (long long int)offset, whence);
+        return -1;
+    }
+    return ret;
+}
+
+
+static int qio_channel_file_close(QIOChannel *ioc,
+                                  Error **errp)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+
+    if (close(fioc->fd) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to close file");
+        return -1;
+    }
+    return 0;
+}
+
+
+static GSource *qio_channel_file_create_watch(QIOChannel *ioc,
+                                              GIOCondition condition)
+{
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(ioc);
+    return qio_channel_create_fd_watch(ioc,
+                                       fioc->fd,
+                                       condition);
+}
+
+static void qio_channel_file_class_init(ObjectClass *klass,
+                                        void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_file_writev;
+    ioc_klass->io_readv = qio_channel_file_readv;
+    ioc_klass->io_set_blocking = qio_channel_file_set_blocking;
+    ioc_klass->io_seek = qio_channel_file_seek;
+    ioc_klass->io_close = qio_channel_file_close;
+    ioc_klass->io_create_watch = qio_channel_file_create_watch;
+}
+
+static const TypeInfo qio_channel_file_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_FILE,
+    .instance_size = sizeof(QIOChannelFile),
+    .instance_init = qio_channel_file_init,
+    .instance_finalize = qio_channel_file_finalize,
+    .class_init = qio_channel_file_class_init,
+};
+
+static void qio_channel_file_register_types(void)
+{
+    type_register_static(&qio_channel_file_info);
+}
+
+type_init(qio_channel_file_register_types);
diff --git a/io/channel-socket.c b/io/channel-socket.c
new file mode 100644
index 0000000..196a4f1
--- /dev/null
+++ b/io/channel-socket.c
@@ -0,0 +1,790 @@
+/*
+ * QEMU I/O channels sockets driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "io/channel-socket.h"
+#include "io/channel-watch.h"
+#include "trace.h"
+#include "qapi/clone-visitor.h"
+
+#define SOCKET_MAX_FDS 16
+
+SocketAddress *
+qio_channel_socket_get_local_address(QIOChannelSocket *ioc,
+                                     Error **errp)
+{
+    return socket_sockaddr_to_address(&ioc->localAddr,
+                                      ioc->localAddrLen,
+                                      errp);
+}
+
+SocketAddress *
+qio_channel_socket_get_remote_address(QIOChannelSocket *ioc,
+                                      Error **errp)
+{
+    return socket_sockaddr_to_address(&ioc->remoteAddr,
+                                      ioc->remoteAddrLen,
+                                      errp);
+}
+
+QIOChannelSocket *
+qio_channel_socket_new(void)
+{
+    QIOChannelSocket *sioc;
+    QIOChannel *ioc;
+
+    sioc = QIO_CHANNEL_SOCKET(object_new(TYPE_QIO_CHANNEL_SOCKET));
+    sioc->fd = -1;
+
+    ioc = QIO_CHANNEL(sioc);
+    ioc->features |= (1 << QIO_CHANNEL_FEATURE_SHUTDOWN);
+
+#ifdef WIN32
+    ioc->event = CreateEvent(NULL, FALSE, FALSE, NULL);
+#endif
+
+    trace_qio_channel_socket_new(sioc);
+
+    return sioc;
+}
+
+
+static int
+qio_channel_socket_set_fd(QIOChannelSocket *sioc,
+                          int fd,
+                          Error **errp)
+{
+    int val;
+    socklen_t len = sizeof(val);
+
+    if (sioc->fd != -1) {
+        error_setg(errp, "Socket is already open");
+        return -1;
+    }
+
+    sioc->fd = fd;
+    sioc->remoteAddrLen = sizeof(sioc->remoteAddr);
+    sioc->localAddrLen = sizeof(sioc->localAddr);
+
+
+    if (getpeername(fd, (struct sockaddr *)&sioc->remoteAddr,
+                    &sioc->remoteAddrLen) < 0) {
+        if (errno == ENOTCONN) {
+            memset(&sioc->remoteAddr, 0, sizeof(sioc->remoteAddr));
+            sioc->remoteAddrLen = sizeof(sioc->remoteAddr);
+        } else {
+            error_setg_errno(errp, errno,
+                             "Unable to query remote socket address");
+            goto error;
+        }
+    }
+
+    if (getsockname(fd, (struct sockaddr *)&sioc->localAddr,
+                    &sioc->localAddrLen) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to query local socket address");
+        goto error;
+    }
+
+#ifndef WIN32
+    if (sioc->localAddr.ss_family == AF_UNIX) {
+        QIOChannel *ioc = QIO_CHANNEL(sioc);
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_FD_PASS);
+    }
+#endif /* WIN32 */
+    if (getsockopt(fd, SOL_SOCKET, SO_ACCEPTCONN, &val, &len) == 0 && val) {
+        QIOChannel *ioc = QIO_CHANNEL(sioc);
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_LISTEN);
+    }
+
+    return 0;
+
+ error:
+    sioc->fd = -1; /* Let the caller close FD on failure */
+    return -1;
+}
+
+QIOChannelSocket *
+qio_channel_socket_new_fd(int fd,
+                          Error **errp)
+{
+    QIOChannelSocket *ioc;
+
+    ioc = qio_channel_socket_new();
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        object_unref(OBJECT(ioc));
+        return NULL;
+    }
+
+    trace_qio_channel_socket_new_fd(ioc, fd);
+
+    return ioc;
+}
+
+
+int qio_channel_socket_connect_sync(QIOChannelSocket *ioc,
+                                    SocketAddress *addr,
+                                    Error **errp)
+{
+    int fd;
+
+    trace_qio_channel_socket_connect_sync(ioc, addr);
+    fd = socket_connect(addr, errp, NULL, NULL);
+    if (fd < 0) {
+        trace_qio_channel_socket_connect_fail(ioc);
+        return -1;
+    }
+
+    trace_qio_channel_socket_connect_complete(ioc, fd);
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int qio_channel_socket_connect_worker(QIOTask *task,
+                                             Error **errp,
+                                             gpointer opaque)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(qio_task_get_source(task));
+    SocketAddress *addr = opaque;
+    int ret;
+
+    ret = qio_channel_socket_connect_sync(ioc,
+                                          addr,
+                                          errp);
+
+    object_unref(OBJECT(ioc));
+    return ret;
+}
+
+
+void qio_channel_socket_connect_async(QIOChannelSocket *ioc,
+                                      SocketAddress *addr,
+                                      QIOTaskFunc callback,
+                                      gpointer opaque,
+                                      GDestroyNotify destroy)
+{
+    QIOTask *task = qio_task_new(
+        OBJECT(ioc), callback, opaque, destroy);
+    SocketAddress *addrCopy;
+
+    addrCopy = QAPI_CLONE(SocketAddress, addr);
+
+    /* socket_connect() does a non-blocking connect(), but it
+     * still blocks in DNS lookups, so we must use a thread */
+    trace_qio_channel_socket_connect_async(ioc, addr);
+    qio_task_run_in_thread(task,
+                           qio_channel_socket_connect_worker,
+                           addrCopy,
+                           (GDestroyNotify)qapi_free_SocketAddress);
+}
+
+
+int qio_channel_socket_listen_sync(QIOChannelSocket *ioc,
+                                   SocketAddress *addr,
+                                   Error **errp)
+{
+    int fd;
+
+    trace_qio_channel_socket_listen_sync(ioc, addr);
+    fd = socket_listen(addr, errp);
+    if (fd < 0) {
+        trace_qio_channel_socket_listen_fail(ioc);
+        return -1;
+    }
+
+    trace_qio_channel_socket_listen_complete(ioc, fd);
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static int qio_channel_socket_listen_worker(QIOTask *task,
+                                            Error **errp,
+                                            gpointer opaque)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(qio_task_get_source(task));
+    SocketAddress *addr = opaque;
+    int ret;
+
+    ret = qio_channel_socket_listen_sync(ioc,
+                                         addr,
+                                         errp);
+
+    object_unref(OBJECT(ioc));
+    return ret;
+}
+
+
+void qio_channel_socket_listen_async(QIOChannelSocket *ioc,
+                                     SocketAddress *addr,
+                                     QIOTaskFunc callback,
+                                     gpointer opaque,
+                                     GDestroyNotify destroy)
+{
+    QIOTask *task = qio_task_new(
+        OBJECT(ioc), callback, opaque, destroy);
+    SocketAddress *addrCopy;
+
+    addrCopy = QAPI_CLONE(SocketAddress, addr);
+
+    /* socket_listen() blocks in DNS lookups, so we must use a thread */
+    trace_qio_channel_socket_listen_async(ioc, addr);
+    qio_task_run_in_thread(task,
+                           qio_channel_socket_listen_worker,
+                           addrCopy,
+                           (GDestroyNotify)qapi_free_SocketAddress);
+}
+
+
+int qio_channel_socket_dgram_sync(QIOChannelSocket *ioc,
+                                  SocketAddress *localAddr,
+                                  SocketAddress *remoteAddr,
+                                  Error **errp)
+{
+    int fd;
+
+    trace_qio_channel_socket_dgram_sync(ioc, localAddr, remoteAddr);
+    fd = socket_dgram(remoteAddr, localAddr, errp);
+    if (fd < 0) {
+        trace_qio_channel_socket_dgram_fail(ioc);
+        return -1;
+    }
+
+    trace_qio_channel_socket_dgram_complete(ioc, fd);
+    if (qio_channel_socket_set_fd(ioc, fd, errp) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+
+struct QIOChannelSocketDGramWorkerData {
+    SocketAddress *localAddr;
+    SocketAddress *remoteAddr;
+};
+
+
+static void qio_channel_socket_dgram_worker_free(gpointer opaque)
+{
+    struct QIOChannelSocketDGramWorkerData *data = opaque;
+    qapi_free_SocketAddress(data->localAddr);
+    qapi_free_SocketAddress(data->remoteAddr);
+    g_free(data);
+}
+
+static int qio_channel_socket_dgram_worker(QIOTask *task,
+                                           Error **errp,
+                                           gpointer opaque)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(qio_task_get_source(task));
+    struct QIOChannelSocketDGramWorkerData *data = opaque;
+    int ret;
+
+    /* socket_dgram() blocks in DNS lookups, so we must use a thread */
+    ret = qio_channel_socket_dgram_sync(ioc,
+                                        data->localAddr,
+                                        data->remoteAddr,
+                                        errp);
+
+    object_unref(OBJECT(ioc));
+    return ret;
+}
+
+
+void qio_channel_socket_dgram_async(QIOChannelSocket *ioc,
+                                    SocketAddress *localAddr,
+                                    SocketAddress *remoteAddr,
+                                    QIOTaskFunc callback,
+                                    gpointer opaque,
+                                    GDestroyNotify destroy)
+{
+    QIOTask *task = qio_task_new(
+        OBJECT(ioc), callback, opaque, destroy);
+    struct QIOChannelSocketDGramWorkerData *data = g_new0(
+        struct QIOChannelSocketDGramWorkerData, 1);
+
+    data->localAddr = QAPI_CLONE(SocketAddress, localAddr);
+    data->remoteAddr = QAPI_CLONE(SocketAddress, remoteAddr);
+
+    trace_qio_channel_socket_dgram_async(ioc, localAddr, remoteAddr);
+    qio_task_run_in_thread(task,
+                           qio_channel_socket_dgram_worker,
+                           data,
+                           qio_channel_socket_dgram_worker_free);
+}
+
+
+QIOChannelSocket *
+qio_channel_socket_accept(QIOChannelSocket *ioc,
+                          Error **errp)
+{
+    QIOChannelSocket *cioc;
+
+    cioc = QIO_CHANNEL_SOCKET(object_new(TYPE_QIO_CHANNEL_SOCKET));
+    cioc->fd = -1;
+    cioc->remoteAddrLen = sizeof(ioc->remoteAddr);
+    cioc->localAddrLen = sizeof(ioc->localAddr);
+
+#ifdef WIN32
+    QIO_CHANNEL(cioc)->event = CreateEvent(NULL, FALSE, FALSE, NULL);
+#endif
+
+
+ retry:
+    trace_qio_channel_socket_accept(ioc);
+    cioc->fd = qemu_accept(ioc->fd, (struct sockaddr *)&cioc->remoteAddr,
+                           &cioc->remoteAddrLen);
+    if (cioc->fd < 0) {
+        trace_qio_channel_socket_accept_fail(ioc);
+        if (errno == EINTR) {
+            goto retry;
+        }
+        goto error;
+    }
+
+    if (getsockname(cioc->fd, (struct sockaddr *)&cioc->localAddr,
+                    &cioc->localAddrLen) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to query local socket address");
+        goto error;
+    }
+
+#ifndef WIN32
+    if (cioc->localAddr.ss_family == AF_UNIX) {
+        QIO_CHANNEL(cioc)->features |= (1 << QIO_CHANNEL_FEATURE_FD_PASS);
+    }
+#endif /* WIN32 */
+
+    trace_qio_channel_socket_accept_complete(ioc, cioc, cioc->fd);
+    return cioc;
+
+ error:
+    object_unref(OBJECT(cioc));
+    return NULL;
+}
+
+static void qio_channel_socket_init(Object *obj)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(obj);
+    ioc->fd = -1;
+}
+
+static void qio_channel_socket_finalize(Object *obj)
+{
+    QIOChannelSocket *ioc = QIO_CHANNEL_SOCKET(obj);
+
+    if (ioc->fd != -1) {
+        if (QIO_CHANNEL(ioc)->features & QIO_CHANNEL_FEATURE_LISTEN) {
+            Error *err = NULL;
+
+            socket_listen_cleanup(ioc->fd, &err);
+            if (err) {
+                error_report_err(err);
+                err = NULL;
+            }
+        }
+#ifdef WIN32
+        WSAEventSelect(ioc->fd, NULL, 0);
+#endif
+        closesocket(ioc->fd);
+        ioc->fd = -1;
+    }
+}
+
+
+#ifndef WIN32
+static void qio_channel_socket_copy_fds(struct msghdr *msg,
+                                        int **fds, size_t *nfds)
+{
+    struct cmsghdr *cmsg;
+
+    *nfds = 0;
+    *fds = NULL;
+
+    for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
+        int fd_size, i;
+        int gotfds;
+
+        if (cmsg->cmsg_len < CMSG_LEN(sizeof(int)) ||
+            cmsg->cmsg_level != SOL_SOCKET ||
+            cmsg->cmsg_type != SCM_RIGHTS) {
+            continue;
+        }
+
+        fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+
+        if (!fd_size) {
+            continue;
+        }
+
+        gotfds = fd_size / sizeof(int);
+        *fds = g_renew(int, *fds, *nfds + gotfds);
+        memcpy(*fds + *nfds, CMSG_DATA(cmsg), fd_size);
+
+        for (i = 0; i < gotfds; i++) {
+            int fd = (*fds)[*nfds + i];
+            if (fd < 0) {
+                continue;
+            }
+
+            /* O_NONBLOCK is preserved across SCM_RIGHTS so reset it */
+            qemu_set_block(fd);
+
+#ifndef MSG_CMSG_CLOEXEC
+            qemu_set_cloexec(fd);
+#endif
+        }
+        *nfds += gotfds;
+    }
+}
+
+
+static ssize_t qio_channel_socket_readv(QIOChannel *ioc,
+                                        const struct iovec *iov,
+                                        size_t niov,
+                                        int **fds,
+                                        size_t *nfds,
+                                        Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t ret;
+    struct msghdr msg = { NULL, };
+    char control[CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS)];
+    int sflags = 0;
+
+    memset(control, 0, CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS));
+
+#ifdef MSG_CMSG_CLOEXEC
+    sflags |= MSG_CMSG_CLOEXEC;
+#endif
+
+    msg.msg_iov = (struct iovec *)iov;
+    msg.msg_iovlen = niov;
+    if (fds && nfds) {
+        msg.msg_control = control;
+        msg.msg_controllen = sizeof(control);
+    }
+
+ retry:
+    ret = recvmsg(sioc->fd, &msg, sflags);
+    if (ret < 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+
+        error_setg_errno(errp, errno,
+                         "Unable to read from socket");
+        return -1;
+    }
+
+    if (fds && nfds) {
+        qio_channel_socket_copy_fds(&msg, fds, nfds);
+    }
+
+    return ret;
+}
+
+static ssize_t qio_channel_socket_writev(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int *fds,
+                                         size_t nfds,
+                                         Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t ret;
+    struct msghdr msg = { NULL, };
+    char control[CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS)];
+    size_t fdsize = sizeof(int) * nfds;
+    struct cmsghdr *cmsg;
+
+    memset(control, 0, CMSG_SPACE(sizeof(int) * SOCKET_MAX_FDS));
+
+    msg.msg_iov = (struct iovec *)iov;
+    msg.msg_iovlen = niov;
+
+    if (nfds) {
+        if (nfds > SOCKET_MAX_FDS) {
+            error_setg_errno(errp, EINVAL,
+                             "Only %d FDs can be sent, got %zu",
+                             SOCKET_MAX_FDS, nfds);
+            return -1;
+        }
+
+        msg.msg_control = control;
+        msg.msg_controllen = CMSG_SPACE(sizeof(int) * nfds);
+
+        cmsg = CMSG_FIRSTHDR(&msg);
+        cmsg->cmsg_len = CMSG_LEN(fdsize);
+        cmsg->cmsg_level = SOL_SOCKET;
+        cmsg->cmsg_type = SCM_RIGHTS;
+        memcpy(CMSG_DATA(cmsg), fds, fdsize);
+    }
+
+ retry:
+    ret = sendmsg(sioc->fd, &msg, 0);
+    if (ret <= 0) {
+        if (errno == EAGAIN) {
+            return QIO_CHANNEL_ERR_BLOCK;
+        }
+        if (errno == EINTR) {
+            goto retry;
+        }
+        error_setg_errno(errp, errno,
+                         "Unable to write to socket");
+        return -1;
+    }
+    return ret;
+}
+#else /* WIN32 */
+static ssize_t qio_channel_socket_readv(QIOChannel *ioc,
+                                        const struct iovec *iov,
+                                        size_t niov,
+                                        int **fds,
+                                        size_t *nfds,
+                                        Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t done = 0;
+    ssize_t i;
+
+    for (i = 0; i < niov; i++) {
+        ssize_t ret;
+    retry:
+        ret = recv(sioc->fd,
+                   iov[i].iov_base,
+                   iov[i].iov_len,
+                   0);
+        if (ret < 0) {
+            if (errno == EAGAIN) {
+                if (done) {
+                    return done;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            } else if (errno == EINTR) {
+                goto retry;
+            } else {
+                error_setg_errno(errp, errno,
+                                 "Unable to read from socket");
+                return -1;
+            }
+        }
+        done += ret;
+        if (ret < iov[i].iov_len) {
+            return done;
+        }
+    }
+
+    return done;
+}
+
+static ssize_t qio_channel_socket_writev(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int *fds,
+                                         size_t nfds,
+                                         Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    ssize_t done = 0;
+    ssize_t i;
+
+    for (i = 0; i < niov; i++) {
+        ssize_t ret;
+    retry:
+        ret = send(sioc->fd,
+                   iov[i].iov_base,
+                   iov[i].iov_len,
+                   0);
+        if (ret < 0) {
+            if (errno == EAGAIN) {
+                if (done) {
+                    return done;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            } else if (errno == EINTR) {
+                goto retry;
+            } else {
+                error_setg_errno(errp, errno,
+                                 "Unable to write to socket");
+                return -1;
+            }
+        }
+        done += ret;
+        if (ret < iov[i].iov_len) {
+            return done;
+        }
+    }
+
+    return done;
+}
+#endif /* WIN32 */
+
+static int
+qio_channel_socket_set_blocking(QIOChannel *ioc,
+                                bool enabled,
+                                Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+
+    if (enabled) {
+        qemu_set_block(sioc->fd);
+    } else {
+        qemu_set_nonblock(sioc->fd);
+#ifdef WIN32
+        WSAEventSelect(sioc->fd, ioc->event,
+                       FD_READ | FD_ACCEPT | FD_CLOSE |
+                       FD_CONNECT | FD_WRITE | FD_OOB);
+#endif
+    }
+    return 0;
+}
+
+
+static void
+qio_channel_socket_set_delay(QIOChannel *ioc,
+                             bool enabled)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    int v = enabled ? 0 : 1;
+
+    qemu_setsockopt(sioc->fd,
+                    IPPROTO_TCP, TCP_NODELAY,
+                    &v, sizeof(v));
+}
+
+
+static void
+qio_channel_socket_set_cork(QIOChannel *ioc,
+                            bool enabled)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    int v = enabled ? 1 : 0;
+
+    socket_set_cork(sioc->fd, v);
+}
+
+
+static int
+qio_channel_socket_close(QIOChannel *ioc,
+                         Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+
+    if (sioc->fd != -1) {
+#ifdef WIN32
+        WSAEventSelect(sioc->fd, NULL, 0);
+#endif
+        if (closesocket(sioc->fd) < 0) {
+            sioc->fd = -1;
+            error_setg_errno(errp, errno,
+                             "Unable to close socket");
+            return -1;
+        }
+        sioc->fd = -1;
+    }
+    return 0;
+}
+
+static int
+qio_channel_socket_shutdown(QIOChannel *ioc,
+                            QIOChannelShutdown how,
+                            Error **errp)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    int sockhow;
+
+    switch (how) {
+    case QIO_CHANNEL_SHUTDOWN_READ:
+        sockhow = SHUT_RD;
+        break;
+    case QIO_CHANNEL_SHUTDOWN_WRITE:
+        sockhow = SHUT_WR;
+        break;
+    case QIO_CHANNEL_SHUTDOWN_BOTH:
+    default:
+        sockhow = SHUT_RDWR;
+        break;
+    }
+
+    if (shutdown(sioc->fd, sockhow) < 0) {
+        error_setg_errno(errp, errno,
+                         "Unable to shutdown socket");
+        return -1;
+    }
+    return 0;
+}
+
+static GSource *qio_channel_socket_create_watch(QIOChannel *ioc,
+                                                GIOCondition condition)
+{
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(ioc);
+    return qio_channel_create_socket_watch(ioc,
+                                           sioc->fd,
+                                           condition);
+}
+
+static void qio_channel_socket_class_init(ObjectClass *klass,
+                                          void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_socket_writev;
+    ioc_klass->io_readv = qio_channel_socket_readv;
+    ioc_klass->io_set_blocking = qio_channel_socket_set_blocking;
+    ioc_klass->io_close = qio_channel_socket_close;
+    ioc_klass->io_shutdown = qio_channel_socket_shutdown;
+    ioc_klass->io_set_cork = qio_channel_socket_set_cork;
+    ioc_klass->io_set_delay = qio_channel_socket_set_delay;
+    ioc_klass->io_create_watch = qio_channel_socket_create_watch;
+}
+
+static const TypeInfo qio_channel_socket_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_SOCKET,
+    .instance_size = sizeof(QIOChannelSocket),
+    .instance_init = qio_channel_socket_init,
+    .instance_finalize = qio_channel_socket_finalize,
+    .class_init = qio_channel_socket_class_init,
+};
+
+static void qio_channel_socket_register_types(void)
+{
+    type_register_static(&qio_channel_socket_info);
+}
+
+type_init(qio_channel_socket_register_types);
diff --git a/io/channel-tls.c b/io/channel-tls.c
new file mode 100644
index 0000000..9a8525c
--- /dev/null
+++ b/io/channel-tls.c
@@ -0,0 +1,395 @@
+/*
+ * QEMU I/O channels TLS driver
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "io/channel-tls.h"
+#include "trace.h"
+
+
+static ssize_t qio_channel_tls_write_handler(const char *buf,
+                                             size_t len,
+                                             void *opaque)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(opaque);
+    ssize_t ret;
+
+    ret = qio_channel_write(tioc->master, buf, len, NULL);
+    if (ret == QIO_CHANNEL_ERR_BLOCK) {
+        errno = EAGAIN;
+        return -1;
+    } else if (ret < 0) {
+        errno = EIO;
+        return -1;
+    }
+    return ret;
+}
+
+static ssize_t qio_channel_tls_read_handler(char *buf,
+                                            size_t len,
+                                            void *opaque)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(opaque);
+    ssize_t ret;
+
+    ret = qio_channel_read(tioc->master, buf, len, NULL);
+    if (ret == QIO_CHANNEL_ERR_BLOCK) {
+        errno = EAGAIN;
+        return -1;
+    } else if (ret < 0) {
+        errno = EIO;
+        return -1;
+    }
+    return ret;
+}
+
+
+QIOChannelTLS *
+qio_channel_tls_new_server(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *aclname,
+                           Error **errp)
+{
+    QIOChannelTLS *ioc;
+
+    ioc = QIO_CHANNEL_TLS(object_new(TYPE_QIO_CHANNEL_TLS));
+
+    ioc->master = master;
+    object_ref(OBJECT(master));
+
+    ioc->session = qcrypto_tls_session_new(
+        creds,
+        NULL,
+        aclname,
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        errp);
+    if (!ioc->session) {
+        goto error;
+    }
+
+    qcrypto_tls_session_set_callbacks(
+        ioc->session,
+        qio_channel_tls_write_handler,
+        qio_channel_tls_read_handler,
+        ioc);
+
+    trace_qio_channel_tls_new_server(ioc, master, creds, aclname);
+    return ioc;
+
+ error:
+    object_unref(OBJECT(ioc));
+    return NULL;
+}
+
+QIOChannelTLS *
+qio_channel_tls_new_client(QIOChannel *master,
+                           QCryptoTLSCreds *creds,
+                           const char *hostname,
+                           Error **errp)
+{
+    QIOChannelTLS *tioc;
+    QIOChannel *ioc;
+
+    tioc = QIO_CHANNEL_TLS(object_new(TYPE_QIO_CHANNEL_TLS));
+    ioc = QIO_CHANNEL(tioc);
+
+    tioc->master = master;
+    if (master->features & (1 << QIO_CHANNEL_FEATURE_SHUTDOWN)) {
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_SHUTDOWN);
+    }
+    object_ref(OBJECT(master));
+
+    tioc->session = qcrypto_tls_session_new(
+        creds,
+        hostname,
+        NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        errp);
+    if (!tioc->session) {
+        goto error;
+    }
+
+    qcrypto_tls_session_set_callbacks(
+        tioc->session,
+        qio_channel_tls_write_handler,
+        qio_channel_tls_read_handler,
+        tioc);
+
+    trace_qio_channel_tls_new_client(tioc, master, creds, hostname);
+    return tioc;
+
+ error:
+    object_unref(OBJECT(tioc));
+    return NULL;
+}
+
+
+static gboolean qio_channel_tls_handshake_io(QIOChannel *ioc,
+                                             GIOCondition condition,
+                                             gpointer user_data);
+
+static void qio_channel_tls_handshake_task(QIOChannelTLS *ioc,
+                                           QIOTask *task)
+{
+    Error *err = NULL;
+    QCryptoTLSSessionHandshakeStatus status;
+
+    if (qcrypto_tls_session_handshake(ioc->session, &err) < 0) {
+        trace_qio_channel_tls_handshake_fail(ioc);
+        qio_task_abort(task, err);
+        goto cleanup;
+    }
+
+    status = qcrypto_tls_session_get_handshake_status(ioc->session);
+    if (status == QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+        trace_qio_channel_tls_handshake_complete(ioc);
+        if (qcrypto_tls_session_check_credentials(ioc->session,
+                                                  &err) < 0) {
+            trace_qio_channel_tls_credentials_deny(ioc);
+            qio_task_abort(task, err);
+            goto cleanup;
+        }
+        trace_qio_channel_tls_credentials_allow(ioc);
+        qio_task_complete(task);
+    } else {
+        GIOCondition condition;
+        if (status == QCRYPTO_TLS_HANDSHAKE_SENDING) {
+            condition = G_IO_OUT;
+        } else {
+            condition = G_IO_IN;
+        }
+
+        trace_qio_channel_tls_handshake_pending(ioc, status);
+        qio_channel_add_watch(ioc->master,
+                              condition,
+                              qio_channel_tls_handshake_io,
+                              task,
+                              NULL);
+    }
+
+ cleanup:
+    error_free(err);
+}
+
+
+static gboolean qio_channel_tls_handshake_io(QIOChannel *ioc,
+                                             GIOCondition condition,
+                                             gpointer user_data)
+{
+    QIOTask *task = user_data;
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(
+        qio_task_get_source(task));
+
+    qio_channel_tls_handshake_task(
+       tioc, task);
+
+    object_unref(OBJECT(tioc));
+
+    return FALSE;
+}
+
+void qio_channel_tls_handshake(QIOChannelTLS *ioc,
+                               QIOTaskFunc func,
+                               gpointer opaque,
+                               GDestroyNotify destroy)
+{
+    QIOTask *task;
+
+    task = qio_task_new(OBJECT(ioc),
+                        func, opaque, destroy);
+
+    trace_qio_channel_tls_handshake_start(ioc);
+    qio_channel_tls_handshake_task(ioc, task);
+}
+
+
+static void qio_channel_tls_init(Object *obj G_GNUC_UNUSED)
+{
+}
+
+
+static void qio_channel_tls_finalize(Object *obj)
+{
+    QIOChannelTLS *ioc = QIO_CHANNEL_TLS(obj);
+
+    object_unref(OBJECT(ioc->master));
+    qcrypto_tls_session_free(ioc->session);
+}
+
+
+static ssize_t qio_channel_tls_readv(QIOChannel *ioc,
+                                     const struct iovec *iov,
+                                     size_t niov,
+                                     int **fds,
+                                     size_t *nfds,
+                                     Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+    size_t i;
+    ssize_t got = 0;
+
+    for (i = 0 ; i < niov ; i++) {
+        ssize_t ret = qcrypto_tls_session_read(tioc->session,
+                                               iov[i].iov_base,
+                                               iov[i].iov_len);
+        if (ret < 0) {
+            if (errno == EAGAIN) {
+                if (got) {
+                    return got;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            }
+
+            error_setg_errno(errp, errno,
+                             "Cannot read from TLS channel");
+            return -1;
+        }
+        got += ret;
+        if (ret < iov[i].iov_len) {
+            break;
+        }
+    }
+    return got;
+}
+
+
+static ssize_t qio_channel_tls_writev(QIOChannel *ioc,
+                                      const struct iovec *iov,
+                                      size_t niov,
+                                      int *fds,
+                                      size_t nfds,
+                                      Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+    size_t i;
+    ssize_t done = 0;
+
+    for (i = 0 ; i < niov ; i++) {
+        ssize_t ret = qcrypto_tls_session_write(tioc->session,
+                                                iov[i].iov_base,
+                                                iov[i].iov_len);
+        if (ret <= 0) {
+            if (errno == EAGAIN) {
+                if (done) {
+                    return done;
+                } else {
+                    return QIO_CHANNEL_ERR_BLOCK;
+                }
+            }
+
+            error_setg_errno(errp, errno,
+                             "Cannot write to TLS channel");
+            return -1;
+        }
+        done += ret;
+        if (ret < iov[i].iov_len) {
+            break;
+        }
+    }
+    return done;
+}
+
+static int qio_channel_tls_set_blocking(QIOChannel *ioc,
+                                        bool enabled,
+                                        Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_set_blocking(tioc->master, enabled, errp);
+}
+
+static void qio_channel_tls_set_delay(QIOChannel *ioc,
+                                      bool enabled)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    qio_channel_set_delay(tioc->master, enabled);
+}
+
+static void qio_channel_tls_set_cork(QIOChannel *ioc,
+                                     bool enabled)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    qio_channel_set_cork(tioc->master, enabled);
+}
+
+static int qio_channel_tls_shutdown(QIOChannel *ioc,
+                                    QIOChannelShutdown how,
+                                    Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_shutdown(tioc->master, how, errp);
+}
+
+static int qio_channel_tls_close(QIOChannel *ioc,
+                                 Error **errp)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_close(tioc->master, errp);
+}
+
+static GSource *qio_channel_tls_create_watch(QIOChannel *ioc,
+                                             GIOCondition condition)
+{
+    QIOChannelTLS *tioc = QIO_CHANNEL_TLS(ioc);
+
+    return qio_channel_create_watch(tioc->master, condition);
+}
+
+QCryptoTLSSession *
+qio_channel_tls_get_session(QIOChannelTLS *ioc)
+{
+    return ioc->session;
+}
+
+static void qio_channel_tls_class_init(ObjectClass *klass,
+                                       void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_tls_writev;
+    ioc_klass->io_readv = qio_channel_tls_readv;
+    ioc_klass->io_set_blocking = qio_channel_tls_set_blocking;
+    ioc_klass->io_set_delay = qio_channel_tls_set_delay;
+    ioc_klass->io_set_cork = qio_channel_tls_set_cork;
+    ioc_klass->io_close = qio_channel_tls_close;
+    ioc_klass->io_shutdown = qio_channel_tls_shutdown;
+    ioc_klass->io_create_watch = qio_channel_tls_create_watch;
+}
+
+static const TypeInfo qio_channel_tls_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_TLS,
+    .instance_size = sizeof(QIOChannelTLS),
+    .instance_init = qio_channel_tls_init,
+    .instance_finalize = qio_channel_tls_finalize,
+    .class_init = qio_channel_tls_class_init,
+};
+
+static void qio_channel_tls_register_types(void)
+{
+    type_register_static(&qio_channel_tls_info);
+}
+
+type_init(qio_channel_tls_register_types);
diff --git a/io/channel-util.c b/io/channel-util.c
new file mode 100644
index 0000000..0fb4bd0
--- /dev/null
+++ b/io/channel-util.c
@@ -0,0 +1,51 @@
+/*
+ * QEMU I/O channels utility APIs
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-util.h"
+#include "io/channel-file.h"
+#include "io/channel-socket.h"
+
+
+static bool fd_is_socket(int fd)
+{
+    int optval;
+    socklen_t optlen;
+    optlen = sizeof(optval);
+    return qemu_getsockopt(fd,
+                           SOL_SOCKET,
+                           SO_TYPE,
+                           (char *)&optval,
+                           &optlen) == 0;
+}
+
+
+QIOChannel *qio_channel_new_fd(int fd,
+                               Error **errp)
+{
+    QIOChannel *ioc;
+
+    if (fd_is_socket(fd)) {
+        ioc = QIO_CHANNEL(qio_channel_socket_new_fd(fd, errp));
+    } else {
+        ioc = QIO_CHANNEL(qio_channel_file_new_fd(fd));
+    }
+    return ioc;
+}
diff --git a/io/channel-watch.c b/io/channel-watch.c
new file mode 100644
index 0000000..cf1cdff
--- /dev/null
+++ b/io/channel-watch.c
@@ -0,0 +1,347 @@
+/*
+ * QEMU I/O channels watch helper APIs
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-watch.h"
+
+typedef struct QIOChannelFDSource QIOChannelFDSource;
+struct QIOChannelFDSource {
+    GSource parent;
+    GPollFD fd;
+    QIOChannel *ioc;
+    GIOCondition condition;
+};
+
+
+#ifdef CONFIG_WIN32
+typedef struct QIOChannelSocketSource QIOChannelSocketSource;
+struct QIOChannelSocketSource {
+    GSource parent;
+    GPollFD fd;
+    QIOChannel *ioc;
+    SOCKET socket;
+    int revents;
+    GIOCondition condition;
+};
+
+#endif
+
+
+typedef struct QIOChannelFDPairSource QIOChannelFDPairSource;
+struct QIOChannelFDPairSource {
+    GSource parent;
+    GPollFD fdread;
+    GPollFD fdwrite;
+    QIOChannel *ioc;
+    GIOCondition condition;
+};
+
+
+static gboolean
+qio_channel_fd_source_prepare(GSource *source G_GNUC_UNUSED,
+                              gint *timeout)
+{
+    *timeout = -1;
+
+    return FALSE;
+}
+
+
+static gboolean
+qio_channel_fd_source_check(GSource *source)
+{
+    QIOChannelFDSource *ssource = (QIOChannelFDSource *)source;
+
+    return ssource->fd.revents & ssource->condition;
+}
+
+
+static gboolean
+qio_channel_fd_source_dispatch(GSource *source,
+                               GSourceFunc callback,
+                               gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelFDSource *ssource = (QIOChannelFDSource *)source;
+
+    return (*func)(ssource->ioc,
+                   ssource->fd.revents & ssource->condition,
+                   user_data);
+}
+
+
+static void
+qio_channel_fd_source_finalize(GSource *source)
+{
+    QIOChannelFDSource *ssource = (QIOChannelFDSource *)source;
+
+    object_unref(OBJECT(ssource->ioc));
+}
+
+
+#ifdef CONFIG_WIN32
+static gboolean
+qio_channel_socket_source_prepare(GSource *source G_GNUC_UNUSED,
+                                  gint *timeout)
+{
+    *timeout = -1;
+
+    return FALSE;
+}
+
+
+/*
+ * NB, this impl only works when the socket is in non-blocking
+ * mode on Win32
+ */
+static gboolean
+qio_channel_socket_source_check(GSource *source)
+{
+    static struct timeval tv0;
+
+    QIOChannelSocketSource *ssource = (QIOChannelSocketSource *)source;
+    WSANETWORKEVENTS ev;
+    fd_set rfds, wfds, xfds;
+
+    if (!ssource->condition) {
+        return 0;
+    }
+
+    WSAEnumNetworkEvents(ssource->socket, ssource->ioc->event, &ev);
+
+    FD_ZERO(&rfds);
+    FD_ZERO(&wfds);
+    FD_ZERO(&xfds);
+    if (ssource->condition & G_IO_IN) {
+        FD_SET((SOCKET)ssource->socket, &rfds);
+    }
+    if (ssource->condition & G_IO_OUT) {
+        FD_SET((SOCKET)ssource->socket, &wfds);
+    }
+    if (ssource->condition & G_IO_PRI) {
+        FD_SET((SOCKET)ssource->socket, &xfds);
+    }
+    ssource->revents = 0;
+    if (select(0, &rfds, &wfds, &xfds, &tv0) == 0) {
+        return 0;
+    }
+
+    if (FD_ISSET(ssource->socket, &rfds)) {
+        ssource->revents |= G_IO_IN;
+    }
+    if (FD_ISSET(ssource->socket, &wfds)) {
+        ssource->revents |= G_IO_OUT;
+    }
+    if (FD_ISSET(ssource->socket, &xfds)) {
+        ssource->revents |= G_IO_PRI;
+    }
+
+    return ssource->revents;
+}
+
+
+static gboolean
+qio_channel_socket_source_dispatch(GSource *source,
+                                   GSourceFunc callback,
+                                   gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelSocketSource *ssource = (QIOChannelSocketSource *)source;
+
+    return (*func)(ssource->ioc, ssource->revents, user_data);
+}
+
+
+static void
+qio_channel_socket_source_finalize(GSource *source)
+{
+    QIOChannelSocketSource *ssource = (QIOChannelSocketSource *)source;
+
+    object_unref(OBJECT(ssource->ioc));
+}
+
+
+GSourceFuncs qio_channel_socket_source_funcs = {
+    qio_channel_socket_source_prepare,
+    qio_channel_socket_source_check,
+    qio_channel_socket_source_dispatch,
+    qio_channel_socket_source_finalize
+};
+#endif
+
+
+static gboolean
+qio_channel_fd_pair_source_prepare(GSource *source G_GNUC_UNUSED,
+                                   gint *timeout)
+{
+    *timeout = -1;
+
+    return FALSE;
+}
+
+
+static gboolean
+qio_channel_fd_pair_source_check(GSource *source)
+{
+    QIOChannelFDPairSource *ssource = (QIOChannelFDPairSource *)source;
+    GIOCondition poll_condition = ssource->fdread.revents |
+        ssource->fdwrite.revents;
+
+    return poll_condition & ssource->condition;
+}
+
+
+static gboolean
+qio_channel_fd_pair_source_dispatch(GSource *source,
+                                    GSourceFunc callback,
+                                    gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelFDPairSource *ssource = (QIOChannelFDPairSource *)source;
+    GIOCondition poll_condition = ssource->fdread.revents |
+        ssource->fdwrite.revents;
+
+    return (*func)(ssource->ioc,
+                   poll_condition & ssource->condition,
+                   user_data);
+}
+
+
+static void
+qio_channel_fd_pair_source_finalize(GSource *source)
+{
+    QIOChannelFDPairSource *ssource = (QIOChannelFDPairSource *)source;
+
+    object_unref(OBJECT(ssource->ioc));
+}
+
+
+GSourceFuncs qio_channel_fd_source_funcs = {
+    qio_channel_fd_source_prepare,
+    qio_channel_fd_source_check,
+    qio_channel_fd_source_dispatch,
+    qio_channel_fd_source_finalize
+};
+
+
+GSourceFuncs qio_channel_fd_pair_source_funcs = {
+    qio_channel_fd_pair_source_prepare,
+    qio_channel_fd_pair_source_check,
+    qio_channel_fd_pair_source_dispatch,
+    qio_channel_fd_pair_source_finalize
+};
+
+
+GSource *qio_channel_create_fd_watch(QIOChannel *ioc,
+                                     int fd,
+                                     GIOCondition condition)
+{
+    GSource *source;
+    QIOChannelFDSource *ssource;
+
+    source = g_source_new(&qio_channel_fd_source_funcs,
+                          sizeof(QIOChannelFDSource));
+    ssource = (QIOChannelFDSource *)source;
+
+    ssource->ioc = ioc;
+    object_ref(OBJECT(ioc));
+
+    ssource->condition = condition;
+
+#ifdef CONFIG_WIN32
+    ssource->fd.fd = (gint64)_get_osfhandle(fd);
+#else
+    ssource->fd.fd = fd;
+#endif
+    ssource->fd.events = condition;
+
+    g_source_add_poll(source, &ssource->fd);
+
+    return source;
+}
+
+#ifdef CONFIG_WIN32
+GSource *qio_channel_create_socket_watch(QIOChannel *ioc,
+                                         int socket,
+                                         GIOCondition condition)
+{
+    GSource *source;
+    QIOChannelSocketSource *ssource;
+
+    source = g_source_new(&qio_channel_socket_source_funcs,
+                          sizeof(QIOChannelSocketSource));
+    ssource = (QIOChannelSocketSource *)source;
+
+    ssource->ioc = ioc;
+    object_ref(OBJECT(ioc));
+
+    ssource->condition = condition;
+    ssource->socket = socket;
+    ssource->revents = 0;
+
+    ssource->fd.fd = (gintptr)ioc->event;
+    ssource->fd.events = G_IO_IN;
+
+    g_source_add_poll(source, &ssource->fd);
+
+    return source;
+}
+#else
+GSource *qio_channel_create_socket_watch(QIOChannel *ioc,
+                                         int socket,
+                                         GIOCondition condition)
+{
+    return qio_channel_create_fd_watch(ioc, socket, condition);
+}
+#endif
+
+GSource *qio_channel_create_fd_pair_watch(QIOChannel *ioc,
+                                          int fdread,
+                                          int fdwrite,
+                                          GIOCondition condition)
+{
+    GSource *source;
+    QIOChannelFDPairSource *ssource;
+
+    source = g_source_new(&qio_channel_fd_pair_source_funcs,
+                          sizeof(QIOChannelFDPairSource));
+    ssource = (QIOChannelFDPairSource *)source;
+
+    ssource->ioc = ioc;
+    object_ref(OBJECT(ioc));
+
+    ssource->condition = condition;
+
+#ifdef CONFIG_WIN32
+    ssource->fdread.fd = (gint64)_get_osfhandle(fdread);
+    ssource->fdwrite.fd = (gint64)_get_osfhandle(fdwrite);
+#else
+    ssource->fdread.fd = fdread;
+    ssource->fdwrite.fd = fdwrite;
+#endif
+
+    ssource->fdread.events = condition & G_IO_IN;
+    ssource->fdwrite.events = condition & G_IO_OUT;
+
+    g_source_add_poll(source, &ssource->fdread);
+    g_source_add_poll(source, &ssource->fdwrite);
+
+    return source;
+}
diff --git a/io/channel-websock.c b/io/channel-websock.c
new file mode 100644
index 0000000..533bd4b
--- /dev/null
+++ b/io/channel-websock.c
@@ -0,0 +1,964 @@
+/*
+ * QEMU I/O channels driver websockets
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/bswap.h"
+#include "io/channel-websock.h"
+#include "crypto/hash.h"
+#include "trace.h"
+
+
+/* Max amount to allow in rawinput/rawoutput buffers */
+#define QIO_CHANNEL_WEBSOCK_MAX_BUFFER 8192
+
+#define QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN 24
+#define QIO_CHANNEL_WEBSOCK_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
+#define QIO_CHANNEL_WEBSOCK_GUID_LEN strlen(QIO_CHANNEL_WEBSOCK_GUID)
+
+#define QIO_CHANNEL_WEBSOCK_HEADER_PROTOCOL "Sec-WebSocket-Protocol"
+#define QIO_CHANNEL_WEBSOCK_HEADER_VERSION "Sec-WebSocket-Version"
+#define QIO_CHANNEL_WEBSOCK_HEADER_KEY "Sec-WebSocket-Key"
+
+#define QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY "binary"
+
+#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_RESPONSE  \
+    "HTTP/1.1 101 Switching Protocols\r\n"      \
+    "Upgrade: websocket\r\n"                    \
+    "Connection: Upgrade\r\n"                   \
+    "Sec-WebSocket-Accept: %s\r\n"              \
+    "Sec-WebSocket-Protocol: binary\r\n"        \
+    "\r\n"
+#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_DELIM "\r\n"
+#define QIO_CHANNEL_WEBSOCK_HANDSHAKE_END "\r\n\r\n"
+#define QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION "13"
+
+/* The websockets packet header is variable length
+ * depending on the size of the payload... */
+
+/* ...length when using 7-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT 6
+/* ...length when using 16-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT 8
+/* ...length when using 64-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT 14
+
+/* Length of the optional data mask field in header */
+#define QIO_CHANNEL_WEBSOCK_HEADER_LEN_MASK 4
+
+/* Maximum length that can fit in 7-bit payload size */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_7_BIT 126
+/* Maximum length that can fit in 16-bit payload size */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_16_BIT 65536
+
+/* Magic 7-bit length to indicate use of 16-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT 126
+/* Magic 7-bit length to indicate use of 64-bit payload length */
+#define QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_64_BIT 127
+
+/* Bitmasks & shifts for accessing header fields */
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_FIN 0x80
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_OPCODE 0x0f
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_HAS_MASK 0x80
+#define QIO_CHANNEL_WEBSOCK_HEADER_FIELD_PAYLOAD_LEN 0x7f
+#define QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_FIN 7
+#define QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_HAS_MASK 7
+
+typedef struct QIOChannelWebsockHeader QIOChannelWebsockHeader;
+
+struct QEMU_PACKED QIOChannelWebsockHeader {
+    unsigned char b0;
+    unsigned char b1;
+    union {
+        struct QEMU_PACKED {
+            uint16_t l16;
+            QIOChannelWebsockMask m16;
+        } s16;
+        struct QEMU_PACKED {
+            uint64_t l64;
+            QIOChannelWebsockMask m64;
+        } s64;
+        QIOChannelWebsockMask m;
+    } u;
+};
+
+enum {
+    QIO_CHANNEL_WEBSOCK_OPCODE_CONTINUATION = 0x0,
+    QIO_CHANNEL_WEBSOCK_OPCODE_TEXT_FRAME = 0x1,
+    QIO_CHANNEL_WEBSOCK_OPCODE_BINARY_FRAME = 0x2,
+    QIO_CHANNEL_WEBSOCK_OPCODE_CLOSE = 0x8,
+    QIO_CHANNEL_WEBSOCK_OPCODE_PING = 0x9,
+    QIO_CHANNEL_WEBSOCK_OPCODE_PONG = 0xA
+};
+
+static char *qio_channel_websock_handshake_entry(const char *handshake,
+                                                 size_t handshake_len,
+                                                 const char *name)
+{
+    char *begin, *end, *ret = NULL;
+    char *line = g_strdup_printf("%s%s: ",
+                                 QIO_CHANNEL_WEBSOCK_HANDSHAKE_DELIM,
+                                 name);
+    begin = g_strstr_len(handshake, handshake_len, line);
+    if (begin != NULL) {
+        begin += strlen(line);
+        end = g_strstr_len(begin, handshake_len - (begin - handshake),
+                QIO_CHANNEL_WEBSOCK_HANDSHAKE_DELIM);
+        if (end != NULL) {
+            ret = g_strndup(begin, end - begin);
+        }
+    }
+    g_free(line);
+    return ret;
+}
+
+
+static int qio_channel_websock_handshake_send_response(QIOChannelWebsock *ioc,
+                                                       const char *key,
+                                                       Error **errp)
+{
+    char combined_key[QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN +
+                      QIO_CHANNEL_WEBSOCK_GUID_LEN + 1];
+    char *accept = NULL, *response = NULL;
+    size_t responselen;
+
+    g_strlcpy(combined_key, key, QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN + 1);
+    g_strlcat(combined_key, QIO_CHANNEL_WEBSOCK_GUID,
+              QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN +
+              QIO_CHANNEL_WEBSOCK_GUID_LEN + 1);
+
+    /* hash and encode it */
+    if (qcrypto_hash_base64(QCRYPTO_HASH_ALG_SHA1,
+                            combined_key,
+                            QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN +
+                            QIO_CHANNEL_WEBSOCK_GUID_LEN,
+                            &accept,
+                            errp) < 0) {
+        return -1;
+    }
+
+    response = g_strdup_printf(QIO_CHANNEL_WEBSOCK_HANDSHAKE_RESPONSE, accept);
+    responselen = strlen(response);
+    buffer_reserve(&ioc->encoutput, responselen);
+    buffer_append(&ioc->encoutput, response, responselen);
+
+    g_free(accept);
+    g_free(response);
+
+    return 0;
+}
+
+static int qio_channel_websock_handshake_process(QIOChannelWebsock *ioc,
+                                                 const char *line,
+                                                 size_t size,
+                                                 Error **errp)
+{
+    int ret = -1;
+    char *protocols = qio_channel_websock_handshake_entry(
+        line, size, QIO_CHANNEL_WEBSOCK_HEADER_PROTOCOL);
+    char *version = qio_channel_websock_handshake_entry(
+        line, size, QIO_CHANNEL_WEBSOCK_HEADER_VERSION);
+    char *key = qio_channel_websock_handshake_entry(
+        line, size, QIO_CHANNEL_WEBSOCK_HEADER_KEY);
+
+    if (!protocols) {
+        error_setg(errp, "Missing websocket protocol header data");
+        goto cleanup;
+    }
+
+    if (!version) {
+        error_setg(errp, "Missing websocket version header data");
+        goto cleanup;
+    }
+
+    if (!key) {
+        error_setg(errp, "Missing websocket key header data");
+        goto cleanup;
+    }
+
+    if (!g_strrstr(protocols, QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY)) {
+        error_setg(errp, "No '%s' protocol is supported by client '%s'",
+                   QIO_CHANNEL_WEBSOCK_PROTOCOL_BINARY, protocols);
+        goto cleanup;
+    }
+
+    if (!g_str_equal(version, QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION)) {
+        error_setg(errp, "Version '%s' is not supported by client '%s'",
+                   QIO_CHANNEL_WEBSOCK_SUPPORTED_VERSION, version);
+        goto cleanup;
+    }
+
+    if (strlen(key) != QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN) {
+        error_setg(errp, "Key length '%zu' was not as expected '%d'",
+                   strlen(key), QIO_CHANNEL_WEBSOCK_CLIENT_KEY_LEN);
+        goto cleanup;
+    }
+
+    ret = qio_channel_websock_handshake_send_response(ioc, key, errp);
+
+ cleanup:
+    g_free(protocols);
+    g_free(version);
+    g_free(key);
+    return ret;
+}
+
+static int qio_channel_websock_handshake_read(QIOChannelWebsock *ioc,
+                                              Error **errp)
+{
+    char *handshake_end;
+    ssize_t ret;
+    /* Typical HTTP headers from novnc are 512 bytes, so limiting
+     * total header size to 4096 is easily enough. */
+    size_t want = 4096 - ioc->encinput.offset;
+    buffer_reserve(&ioc->encinput, want);
+    ret = qio_channel_read(ioc->master,
+                           (char *)buffer_end(&ioc->encinput), want, errp);
+    if (ret < 0) {
+        return -1;
+    }
+    ioc->encinput.offset += ret;
+
+    handshake_end = g_strstr_len((char *)ioc->encinput.buffer,
+                                 ioc->encinput.offset,
+                                 QIO_CHANNEL_WEBSOCK_HANDSHAKE_END);
+    if (!handshake_end) {
+        if (ioc->encinput.offset >= 4096) {
+            error_setg(errp,
+                       "End of headers not found in first 4096 bytes");
+            return -1;
+        } else {
+            return 0;
+        }
+    }
+
+    if (qio_channel_websock_handshake_process(ioc,
+                                              (char *)ioc->encinput.buffer,
+                                              ioc->encinput.offset,
+                                              errp) < 0) {
+        return -1;
+    }
+
+    buffer_advance(&ioc->encinput,
+                   handshake_end - (char *)ioc->encinput.buffer +
+                   strlen(QIO_CHANNEL_WEBSOCK_HANDSHAKE_END));
+    return 1;
+}
+
+static gboolean qio_channel_websock_handshake_send(QIOChannel *ioc,
+                                                   GIOCondition condition,
+                                                   gpointer user_data)
+{
+    QIOTask *task = user_data;
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(
+        qio_task_get_source(task));
+    Error *err = NULL;
+    ssize_t ret;
+
+    ret = qio_channel_write(wioc->master,
+                            (char *)wioc->encoutput.buffer,
+                            wioc->encoutput.offset,
+                            &err);
+
+    if (ret < 0) {
+        trace_qio_channel_websock_handshake_fail(ioc);
+        qio_task_abort(task, err);
+        error_free(err);
+        return FALSE;
+    }
+
+    buffer_advance(&wioc->encoutput, ret);
+    if (wioc->encoutput.offset == 0) {
+        trace_qio_channel_websock_handshake_complete(ioc);
+        qio_task_complete(task);
+        return FALSE;
+    }
+    trace_qio_channel_websock_handshake_pending(ioc, G_IO_OUT);
+    return TRUE;
+}
+
+static gboolean qio_channel_websock_handshake_io(QIOChannel *ioc,
+                                                 GIOCondition condition,
+                                                 gpointer user_data)
+{
+    QIOTask *task = user_data;
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(
+        qio_task_get_source(task));
+    Error *err = NULL;
+    int ret;
+
+    ret = qio_channel_websock_handshake_read(wioc, &err);
+    if (ret < 0) {
+        trace_qio_channel_websock_handshake_fail(ioc);
+        qio_task_abort(task, err);
+        error_free(err);
+        return FALSE;
+    }
+    if (ret == 0) {
+        trace_qio_channel_websock_handshake_pending(ioc, G_IO_IN);
+        /* need more data still */
+        return TRUE;
+    }
+
+    trace_qio_channel_websock_handshake_reply(ioc);
+    qio_channel_add_watch(
+        wioc->master,
+        G_IO_OUT,
+        qio_channel_websock_handshake_send,
+        task,
+        NULL);
+    return FALSE;
+}
+
+
+static void qio_channel_websock_encode(QIOChannelWebsock *ioc)
+{
+    size_t header_size;
+    union {
+        char buf[QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT];
+        QIOChannelWebsockHeader ws;
+    } header;
+
+    if (!ioc->rawoutput.offset) {
+        return;
+    }
+
+    header.ws.b0 = (1 << QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_FIN) |
+        (QIO_CHANNEL_WEBSOCK_OPCODE_BINARY_FRAME &
+         QIO_CHANNEL_WEBSOCK_HEADER_FIELD_OPCODE);
+    if (ioc->rawoutput.offset <
+        QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_7_BIT) {
+        header.ws.b1 = (uint8_t)ioc->rawoutput.offset;
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT;
+    } else if (ioc->rawoutput.offset <
+               QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_THRESHOLD_16_BIT) {
+        header.ws.b1 = QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT;
+        header.ws.u.s16.l16 = cpu_to_be16((uint16_t)ioc->rawoutput.offset);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT;
+    } else {
+        header.ws.b1 = QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_64_BIT;
+        header.ws.u.s64.l64 = cpu_to_be64(ioc->rawoutput.offset);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT;
+    }
+    header_size -= QIO_CHANNEL_WEBSOCK_HEADER_LEN_MASK;
+
+    buffer_reserve(&ioc->encoutput, header_size + ioc->rawoutput.offset);
+    buffer_append(&ioc->encoutput, header.buf, header_size);
+    buffer_append(&ioc->encoutput, ioc->rawoutput.buffer,
+                  ioc->rawoutput.offset);
+    buffer_reset(&ioc->rawoutput);
+}
+
+
+static ssize_t qio_channel_websock_decode_header(QIOChannelWebsock *ioc,
+                                                 Error **errp)
+{
+    unsigned char opcode, fin, has_mask;
+    size_t header_size;
+    size_t payload_len;
+    QIOChannelWebsockHeader *header =
+        (QIOChannelWebsockHeader *)ioc->encinput.buffer;
+
+    if (ioc->payload_remain) {
+        error_setg(errp,
+                   "Decoding header but %zu bytes of payload remain",
+                   ioc->payload_remain);
+        return -1;
+    }
+    if (ioc->encinput.offset < QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT) {
+        /* header not complete */
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    fin = (header->b0 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_FIN) >>
+        QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_FIN;
+    opcode = header->b0 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_OPCODE;
+    has_mask = (header->b1 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_HAS_MASK) >>
+        QIO_CHANNEL_WEBSOCK_HEADER_SHIFT_HAS_MASK;
+    payload_len = header->b1 & QIO_CHANNEL_WEBSOCK_HEADER_FIELD_PAYLOAD_LEN;
+
+    if (opcode == QIO_CHANNEL_WEBSOCK_OPCODE_CLOSE) {
+        /* disconnect */
+        return 0;
+    }
+
+    /* Websocket frame sanity check:
+     * * Websocket fragmentation is not supported.
+     * * All  websockets frames sent by a client have to be masked.
+     * * Only binary encoding is supported.
+     */
+    if (!fin) {
+        error_setg(errp, "websocket fragmentation is not supported");
+        return -1;
+    }
+    if (!has_mask) {
+        error_setg(errp, "websocket frames must be masked");
+        return -1;
+    }
+    if (opcode != QIO_CHANNEL_WEBSOCK_OPCODE_BINARY_FRAME) {
+        error_setg(errp, "only binary websocket frames are supported");
+        return -1;
+    }
+
+    if (payload_len < QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT) {
+        ioc->payload_remain = payload_len;
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_7_BIT;
+        ioc->mask = header->u.m;
+    } else if (payload_len == QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_16_BIT &&
+               ioc->encinput.offset >= QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT) {
+        ioc->payload_remain = be16_to_cpu(header->u.s16.l16);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_16_BIT;
+        ioc->mask = header->u.s16.m16;
+    } else if (payload_len == QIO_CHANNEL_WEBSOCK_PAYLOAD_LEN_MAGIC_64_BIT &&
+               ioc->encinput.offset >= QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT) {
+        ioc->payload_remain = be64_to_cpu(header->u.s64.l64);
+        header_size = QIO_CHANNEL_WEBSOCK_HEADER_LEN_64_BIT;
+        ioc->mask = header->u.s64.m64;
+    } else {
+        /* header not complete */
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    buffer_advance(&ioc->encinput, header_size);
+    return 1;
+}
+
+
+static ssize_t qio_channel_websock_decode_payload(QIOChannelWebsock *ioc,
+                                                  Error **errp)
+{
+    size_t i;
+    size_t payload_len;
+    uint32_t *payload32;
+
+    if (!ioc->payload_remain) {
+        error_setg(errp,
+                   "Decoding payload but no bytes of payload remain");
+        return -1;
+    }
+
+    /* If we aren't at the end of the payload, then drop
+     * off the last bytes, so we're always multiple of 4
+     * for purpose of unmasking, except at end of payload
+     */
+    if (ioc->encinput.offset < ioc->payload_remain) {
+        payload_len = ioc->encinput.offset - (ioc->encinput.offset % 4);
+    } else {
+        payload_len = ioc->payload_remain;
+    }
+    if (payload_len == 0) {
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    ioc->payload_remain -= payload_len;
+
+    /* unmask frame */
+    /* process 1 frame (32 bit op) */
+    payload32 = (uint32_t *)ioc->encinput.buffer;
+    for (i = 0; i < payload_len / 4; i++) {
+        payload32[i] ^= ioc->mask.u;
+    }
+    /* process the remaining bytes (if any) */
+    for (i *= 4; i < payload_len; i++) {
+        ioc->encinput.buffer[i] ^= ioc->mask.c[i % 4];
+    }
+
+    buffer_reserve(&ioc->rawinput, payload_len);
+    buffer_append(&ioc->rawinput, ioc->encinput.buffer, payload_len);
+    buffer_advance(&ioc->encinput, payload_len);
+    return payload_len;
+}
+
+
+QIOChannelWebsock *
+qio_channel_websock_new_server(QIOChannel *master)
+{
+    QIOChannelWebsock *wioc;
+    QIOChannel *ioc;
+
+    wioc = QIO_CHANNEL_WEBSOCK(object_new(TYPE_QIO_CHANNEL_WEBSOCK));
+    ioc = QIO_CHANNEL(wioc);
+
+    wioc->master = master;
+    if (master->features & (1 << QIO_CHANNEL_FEATURE_SHUTDOWN)) {
+        ioc->features |= (1 << QIO_CHANNEL_FEATURE_SHUTDOWN);
+    }
+    object_ref(OBJECT(master));
+
+    trace_qio_channel_websock_new_server(wioc, master);
+    return wioc;
+}
+
+void qio_channel_websock_handshake(QIOChannelWebsock *ioc,
+                                   QIOTaskFunc func,
+                                   gpointer opaque,
+                                   GDestroyNotify destroy)
+{
+    QIOTask *task;
+
+    task = qio_task_new(OBJECT(ioc),
+                        func,
+                        opaque,
+                        destroy);
+
+    trace_qio_channel_websock_handshake_start(ioc);
+    trace_qio_channel_websock_handshake_pending(ioc, G_IO_IN);
+    qio_channel_add_watch(ioc->master,
+                          G_IO_IN,
+                          qio_channel_websock_handshake_io,
+                          task,
+                          NULL);
+}
+
+
+static void qio_channel_websock_finalize(Object *obj)
+{
+    QIOChannelWebsock *ioc = QIO_CHANNEL_WEBSOCK(obj);
+
+    buffer_free(&ioc->encinput);
+    buffer_free(&ioc->encoutput);
+    buffer_free(&ioc->rawinput);
+    buffer_free(&ioc->rawoutput);
+    object_unref(OBJECT(ioc->master));
+    if (ioc->io_tag) {
+        g_source_remove(ioc->io_tag);
+    }
+    if (ioc->io_err) {
+        error_free(ioc->io_err);
+    }
+}
+
+
+static ssize_t qio_channel_websock_read_wire(QIOChannelWebsock *ioc,
+                                             Error **errp)
+{
+    ssize_t ret;
+
+    if (ioc->encinput.offset < 4096) {
+        size_t want = 4096 - ioc->encinput.offset;
+
+        buffer_reserve(&ioc->encinput, want);
+        ret = qio_channel_read(ioc->master,
+                               (char *)ioc->encinput.buffer +
+                               ioc->encinput.offset,
+                               want,
+                               errp);
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret == 0 &&
+            ioc->encinput.offset == 0) {
+            return 0;
+        }
+        ioc->encinput.offset += ret;
+    }
+
+    if (ioc->payload_remain == 0) {
+        ret = qio_channel_websock_decode_header(ioc, errp);
+        if (ret < 0) {
+            return ret;
+        }
+        if (ret == 0) {
+            return 0;
+        }
+    }
+
+    ret = qio_channel_websock_decode_payload(ioc, errp);
+    if (ret < 0) {
+        return ret;
+    }
+    return ret;
+}
+
+
+static ssize_t qio_channel_websock_write_wire(QIOChannelWebsock *ioc,
+                                              Error **errp)
+{
+    ssize_t ret;
+    ssize_t done = 0;
+    qio_channel_websock_encode(ioc);
+
+    while (ioc->encoutput.offset > 0) {
+        ret = qio_channel_write(ioc->master,
+                                (char *)ioc->encoutput.buffer,
+                                ioc->encoutput.offset,
+                                errp);
+        if (ret < 0) {
+            if (ret == QIO_CHANNEL_ERR_BLOCK &&
+                done > 0) {
+                return done;
+            } else {
+                return ret;
+            }
+        }
+        buffer_advance(&ioc->encoutput, ret);
+        done += ret;
+    }
+    return done;
+}
+
+
+static void qio_channel_websock_flush_free(gpointer user_data)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(user_data);
+    object_unref(OBJECT(wioc));
+}
+
+static void qio_channel_websock_set_watch(QIOChannelWebsock *ioc);
+
+static gboolean qio_channel_websock_flush(QIOChannel *ioc,
+                                          GIOCondition condition,
+                                          gpointer user_data)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(user_data);
+    ssize_t ret;
+
+    if (condition & G_IO_OUT) {
+        ret = qio_channel_websock_write_wire(wioc, &wioc->io_err);
+        if (ret < 0) {
+            goto cleanup;
+        }
+    }
+
+    if (condition & G_IO_IN) {
+        ret = qio_channel_websock_read_wire(wioc, &wioc->io_err);
+        if (ret < 0) {
+            goto cleanup;
+        }
+        if (ret == 0) {
+            wioc->io_eof = TRUE;
+        }
+    }
+
+ cleanup:
+    qio_channel_websock_set_watch(wioc);
+    return FALSE;
+}
+
+
+static void qio_channel_websock_unset_watch(QIOChannelWebsock *ioc)
+{
+    if (ioc->io_tag) {
+        g_source_remove(ioc->io_tag);
+        ioc->io_tag = 0;
+    }
+}
+
+static void qio_channel_websock_set_watch(QIOChannelWebsock *ioc)
+{
+    GIOCondition cond = 0;
+
+    qio_channel_websock_unset_watch(ioc);
+
+    if (ioc->io_err) {
+        return;
+    }
+
+    if (ioc->encoutput.offset) {
+        cond |= G_IO_OUT;
+    }
+    if (ioc->encinput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER &&
+        !ioc->io_eof) {
+        cond |= G_IO_IN;
+    }
+
+    if (cond) {
+        object_ref(OBJECT(ioc));
+        ioc->io_tag =
+            qio_channel_add_watch(ioc->master,
+                                  cond,
+                                  qio_channel_websock_flush,
+                                  ioc,
+                                  qio_channel_websock_flush_free);
+    }
+}
+
+
+static ssize_t qio_channel_websock_readv(QIOChannel *ioc,
+                                         const struct iovec *iov,
+                                         size_t niov,
+                                         int **fds,
+                                         size_t *nfds,
+                                         Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+    size_t i;
+    ssize_t got = 0;
+    ssize_t ret;
+
+    if (wioc->io_err) {
+        *errp = error_copy(wioc->io_err);
+        return -1;
+    }
+
+    if (!wioc->rawinput.offset) {
+        ret = qio_channel_websock_read_wire(QIO_CHANNEL_WEBSOCK(ioc), errp);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    for (i = 0 ; i < niov ; i++) {
+        size_t want = iov[i].iov_len;
+        if (want > (wioc->rawinput.offset - got)) {
+            want = (wioc->rawinput.offset - got);
+        }
+
+        memcpy(iov[i].iov_base,
+               wioc->rawinput.buffer + got,
+               want);
+        got += want;
+
+        if (want < iov[i].iov_len) {
+            break;
+        }
+    }
+
+    buffer_advance(&wioc->rawinput, got);
+    qio_channel_websock_set_watch(wioc);
+    return got;
+}
+
+
+static ssize_t qio_channel_websock_writev(QIOChannel *ioc,
+                                          const struct iovec *iov,
+                                          size_t niov,
+                                          int *fds,
+                                          size_t nfds,
+                                          Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+    size_t i;
+    ssize_t done = 0;
+    ssize_t ret;
+
+    if (wioc->io_err) {
+        *errp = error_copy(wioc->io_err);
+        return -1;
+    }
+
+    if (wioc->io_eof) {
+        error_setg(errp, "%s", "Broken pipe");
+        return -1;
+    }
+
+    for (i = 0; i < niov; i++) {
+        size_t want = iov[i].iov_len;
+        if ((want + wioc->rawoutput.offset) > QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+            want = (QIO_CHANNEL_WEBSOCK_MAX_BUFFER - wioc->rawoutput.offset);
+        }
+        if (want == 0) {
+            goto done;
+        }
+
+        buffer_reserve(&wioc->rawoutput, want);
+        buffer_append(&wioc->rawoutput, iov[i].iov_base, want);
+        done += want;
+        if (want < iov[i].iov_len) {
+            break;
+        }
+    }
+
+ done:
+    ret = qio_channel_websock_write_wire(wioc, errp);
+    if (ret < 0 &&
+        ret != QIO_CHANNEL_ERR_BLOCK) {
+        qio_channel_websock_unset_watch(wioc);
+        return -1;
+    }
+
+    qio_channel_websock_set_watch(wioc);
+
+    if (done == 0) {
+        return QIO_CHANNEL_ERR_BLOCK;
+    }
+
+    return done;
+}
+
+static int qio_channel_websock_set_blocking(QIOChannel *ioc,
+                                            bool enabled,
+                                            Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    qio_channel_set_blocking(wioc->master, enabled, errp);
+    return 0;
+}
+
+static void qio_channel_websock_set_delay(QIOChannel *ioc,
+                                          bool enabled)
+{
+    QIOChannelWebsock *tioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    qio_channel_set_delay(tioc->master, enabled);
+}
+
+static void qio_channel_websock_set_cork(QIOChannel *ioc,
+                                         bool enabled)
+{
+    QIOChannelWebsock *tioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    qio_channel_set_cork(tioc->master, enabled);
+}
+
+static int qio_channel_websock_shutdown(QIOChannel *ioc,
+                                        QIOChannelShutdown how,
+                                        Error **errp)
+{
+    QIOChannelWebsock *tioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    return qio_channel_shutdown(tioc->master, how, errp);
+}
+
+static int qio_channel_websock_close(QIOChannel *ioc,
+                                     Error **errp)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+
+    return qio_channel_close(wioc->master, errp);
+}
+
+typedef struct QIOChannelWebsockSource QIOChannelWebsockSource;
+struct QIOChannelWebsockSource {
+    GSource parent;
+    QIOChannelWebsock *wioc;
+    GIOCondition condition;
+};
+
+static gboolean
+qio_channel_websock_source_prepare(GSource *source,
+                                   gint *timeout)
+{
+    QIOChannelWebsockSource *wsource = (QIOChannelWebsockSource *)source;
+    GIOCondition cond = 0;
+    *timeout = -1;
+
+    if (wsource->wioc->rawinput.offset) {
+        cond |= G_IO_IN;
+    }
+    if (wsource->wioc->rawoutput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+        cond |= G_IO_OUT;
+    }
+
+    return cond & wsource->condition;
+}
+
+static gboolean
+qio_channel_websock_source_check(GSource *source)
+{
+    QIOChannelWebsockSource *wsource = (QIOChannelWebsockSource *)source;
+    GIOCondition cond = 0;
+
+    if (wsource->wioc->rawinput.offset) {
+        cond |= G_IO_IN;
+    }
+    if (wsource->wioc->rawoutput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+        cond |= G_IO_OUT;
+    }
+
+    return cond & wsource->condition;
+}
+
+static gboolean
+qio_channel_websock_source_dispatch(GSource *source,
+                                    GSourceFunc callback,
+                                    gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelWebsockSource *wsource = (QIOChannelWebsockSource *)source;
+    GIOCondition cond = 0;
+
+    if (wsource->wioc->rawinput.offset) {
+        cond |= G_IO_IN;
+    }
+    if (wsource->wioc->rawoutput.offset < QIO_CHANNEL_WEBSOCK_MAX_BUFFER) {
+        cond |= G_IO_OUT;
+    }
+
+    return (*func)(QIO_CHANNEL(wsource->wioc),
+                   (cond & wsource->condition),
+                   user_data);
+}
+
+static void
+qio_channel_websock_source_finalize(GSource *source)
+{
+    QIOChannelWebsockSource *ssource = (QIOChannelWebsockSource *)source;
+
+    object_unref(OBJECT(ssource->wioc));
+}
+
+GSourceFuncs qio_channel_websock_source_funcs = {
+    qio_channel_websock_source_prepare,
+    qio_channel_websock_source_check,
+    qio_channel_websock_source_dispatch,
+    qio_channel_websock_source_finalize
+};
+
+static GSource *qio_channel_websock_create_watch(QIOChannel *ioc,
+                                                 GIOCondition condition)
+{
+    QIOChannelWebsock *wioc = QIO_CHANNEL_WEBSOCK(ioc);
+    QIOChannelWebsockSource *ssource;
+    GSource *source;
+
+    source = g_source_new(&qio_channel_websock_source_funcs,
+                          sizeof(QIOChannelWebsockSource));
+    ssource = (QIOChannelWebsockSource *)source;
+
+    ssource->wioc = wioc;
+    object_ref(OBJECT(wioc));
+
+    ssource->condition = condition;
+
+    qio_channel_websock_set_watch(wioc);
+    return source;
+}
+
+static void qio_channel_websock_class_init(ObjectClass *klass,
+                                           void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_websock_writev;
+    ioc_klass->io_readv = qio_channel_websock_readv;
+    ioc_klass->io_set_blocking = qio_channel_websock_set_blocking;
+    ioc_klass->io_set_cork = qio_channel_websock_set_cork;
+    ioc_klass->io_set_delay = qio_channel_websock_set_delay;
+    ioc_klass->io_close = qio_channel_websock_close;
+    ioc_klass->io_shutdown = qio_channel_websock_shutdown;
+    ioc_klass->io_create_watch = qio_channel_websock_create_watch;
+}
+
+static const TypeInfo qio_channel_websock_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_WEBSOCK,
+    .instance_size = sizeof(QIOChannelWebsock),
+    .instance_finalize = qio_channel_websock_finalize,
+    .class_init = qio_channel_websock_class_init,
+};
+
+static void qio_channel_websock_register_types(void)
+{
+    type_register_static(&qio_channel_websock_info);
+}
+
+type_init(qio_channel_websock_register_types);
diff --git a/io/channel.c b/io/channel.c
new file mode 100644
index 0000000..923c465
--- /dev/null
+++ b/io/channel.c
@@ -0,0 +1,307 @@
+/*
+ * QEMU I/O channels
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel.h"
+#include "qapi/error.h"
+#include "qemu/coroutine.h"
+
+bool qio_channel_has_feature(QIOChannel *ioc,
+                             QIOChannelFeature feature)
+{
+    return ioc->features & (1 << feature);
+}
+
+
+ssize_t qio_channel_readv_full(QIOChannel *ioc,
+                               const struct iovec *iov,
+                               size_t niov,
+                               int **fds,
+                               size_t *nfds,
+                               Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if ((fds || nfds) &&
+        !(ioc->features & (1 << QIO_CHANNEL_FEATURE_FD_PASS))) {
+        error_setg_errno(errp, EINVAL,
+                         "Channel does not support file descriptor passing");
+        return -1;
+    }
+
+    return klass->io_readv(ioc, iov, niov, fds, nfds, errp);
+}
+
+
+ssize_t qio_channel_writev_full(QIOChannel *ioc,
+                                const struct iovec *iov,
+                                size_t niov,
+                                int *fds,
+                                size_t nfds,
+                                Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if ((fds || nfds) &&
+        !(ioc->features & (1 << QIO_CHANNEL_FEATURE_FD_PASS))) {
+        error_setg_errno(errp, EINVAL,
+                         "Channel does not support file descriptor passing");
+        return -1;
+    }
+
+    return klass->io_writev(ioc, iov, niov, fds, nfds, errp);
+}
+
+
+ssize_t qio_channel_readv(QIOChannel *ioc,
+                          const struct iovec *iov,
+                          size_t niov,
+                          Error **errp)
+{
+    return qio_channel_readv_full(ioc, iov, niov, NULL, NULL, errp);
+}
+
+
+ssize_t qio_channel_writev(QIOChannel *ioc,
+                           const struct iovec *iov,
+                           size_t niov,
+                           Error **errp)
+{
+    return qio_channel_writev_full(ioc, iov, niov, NULL, 0, errp);
+}
+
+
+ssize_t qio_channel_read(QIOChannel *ioc,
+                         char *buf,
+                         size_t buflen,
+                         Error **errp)
+{
+    struct iovec iov = { .iov_base = buf, .iov_len = buflen };
+    return qio_channel_readv_full(ioc, &iov, 1, NULL, NULL, errp);
+}
+
+
+ssize_t qio_channel_write(QIOChannel *ioc,
+                          const char *buf,
+                          size_t buflen,
+                          Error **errp)
+{
+    struct iovec iov = { .iov_base = (char *)buf, .iov_len = buflen };
+    return qio_channel_writev_full(ioc, &iov, 1, NULL, 0, errp);
+}
+
+
+int qio_channel_set_blocking(QIOChannel *ioc,
+                              bool enabled,
+                              Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+    return klass->io_set_blocking(ioc, enabled, errp);
+}
+
+
+int qio_channel_close(QIOChannel *ioc,
+                      Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+    return klass->io_close(ioc, errp);
+}
+
+
+GSource *qio_channel_create_watch(QIOChannel *ioc,
+                                  GIOCondition condition)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+    return klass->io_create_watch(ioc, condition);
+}
+
+
+guint qio_channel_add_watch(QIOChannel *ioc,
+                            GIOCondition condition,
+                            QIOChannelFunc func,
+                            gpointer user_data,
+                            GDestroyNotify notify)
+{
+    GSource *source;
+    guint id;
+
+    source = qio_channel_create_watch(ioc, condition);
+
+    g_source_set_callback(source, (GSourceFunc)func, user_data, notify);
+
+    id = g_source_attach(source, NULL);
+    g_source_unref(source);
+
+    return id;
+}
+
+
+int qio_channel_shutdown(QIOChannel *ioc,
+                         QIOChannelShutdown how,
+                         Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (!klass->io_shutdown) {
+        error_setg(errp, "Data path shutdown not supported");
+        return -1;
+    }
+
+    return klass->io_shutdown(ioc, how, errp);
+}
+
+
+void qio_channel_set_delay(QIOChannel *ioc,
+                           bool enabled)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (klass->io_set_delay) {
+        klass->io_set_delay(ioc, enabled);
+    }
+}
+
+
+void qio_channel_set_cork(QIOChannel *ioc,
+                          bool enabled)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (klass->io_set_cork) {
+        klass->io_set_cork(ioc, enabled);
+    }
+}
+
+
+off_t qio_channel_io_seek(QIOChannel *ioc,
+                          off_t offset,
+                          int whence,
+                          Error **errp)
+{
+    QIOChannelClass *klass = QIO_CHANNEL_GET_CLASS(ioc);
+
+    if (!klass->io_seek) {
+        error_setg(errp, "Channel does not support random access");
+        return -1;
+    }
+
+    return klass->io_seek(ioc, offset, whence, errp);
+}
+
+
+typedef struct QIOChannelYieldData QIOChannelYieldData;
+struct QIOChannelYieldData {
+    QIOChannel *ioc;
+    Coroutine *co;
+};
+
+
+static gboolean qio_channel_yield_enter(QIOChannel *ioc,
+                                        GIOCondition condition,
+                                        gpointer opaque)
+{
+    QIOChannelYieldData *data = opaque;
+    qemu_coroutine_enter(data->co);
+    return FALSE;
+}
+
+
+void coroutine_fn qio_channel_yield(QIOChannel *ioc,
+                                    GIOCondition condition)
+{
+    QIOChannelYieldData data;
+
+    assert(qemu_in_coroutine());
+    data.ioc = ioc;
+    data.co = qemu_coroutine_self();
+    qio_channel_add_watch(ioc,
+                          condition,
+                          qio_channel_yield_enter,
+                          &data,
+                          NULL);
+    qemu_coroutine_yield();
+}
+
+
+static gboolean qio_channel_wait_complete(QIOChannel *ioc,
+                                          GIOCondition condition,
+                                          gpointer opaque)
+{
+    GMainLoop *loop = opaque;
+
+    g_main_loop_quit(loop);
+    return FALSE;
+}
+
+
+void qio_channel_wait(QIOChannel *ioc,
+                      GIOCondition condition)
+{
+    GMainContext *ctxt = g_main_context_new();
+    GMainLoop *loop = g_main_loop_new(ctxt, TRUE);
+    GSource *source;
+
+    source = qio_channel_create_watch(ioc, condition);
+
+    g_source_set_callback(source,
+                          (GSourceFunc)qio_channel_wait_complete,
+                          loop,
+                          NULL);
+
+    g_source_attach(source, ctxt);
+
+    g_main_loop_run(loop);
+
+    g_source_unref(source);
+    g_main_loop_unref(loop);
+    g_main_context_unref(ctxt);
+}
+
+
+#ifdef _WIN32
+static void qio_channel_finalize(Object *obj)
+{
+    QIOChannel *ioc = QIO_CHANNEL(obj);
+
+    if (ioc->event) {
+        CloseHandle(ioc->event);
+    }
+}
+#endif
+
+static const TypeInfo qio_channel_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_QIO_CHANNEL,
+    .instance_size = sizeof(QIOChannel),
+#ifdef _WIN32
+    .instance_finalize = qio_channel_finalize,
+#endif
+    .abstract = true,
+    .class_size = sizeof(QIOChannelClass),
+};
+
+
+static void qio_channel_register_types(void)
+{
+    type_register_static(&qio_channel_info);
+}
+
+
+type_init(qio_channel_register_types);
diff --git a/io/task.c b/io/task.c
new file mode 100644
index 0000000..c7f97a9
--- /dev/null
+++ b/io/task.c
@@ -0,0 +1,161 @@
+/*
+ * QEMU I/O task
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/task.h"
+#include "qapi/error.h"
+#include "qemu/thread.h"
+#include "trace.h"
+
+struct QIOTask {
+    Object *source;
+    QIOTaskFunc func;
+    gpointer opaque;
+    GDestroyNotify destroy;
+};
+
+
+QIOTask *qio_task_new(Object *source,
+                      QIOTaskFunc func,
+                      gpointer opaque,
+                      GDestroyNotify destroy)
+{
+    QIOTask *task;
+
+    task = g_new0(QIOTask, 1);
+
+    task->source = source;
+    object_ref(source);
+    task->func = func;
+    task->opaque = opaque;
+    task->destroy = destroy;
+
+    trace_qio_task_new(task, source, func, opaque);
+
+    return task;
+}
+
+static void qio_task_free(QIOTask *task)
+{
+    if (task->destroy) {
+        task->destroy(task->opaque);
+    }
+    object_unref(task->source);
+
+    g_free(task);
+}
+
+
+struct QIOTaskThreadData {
+    QIOTask *task;
+    QIOTaskWorker worker;
+    gpointer opaque;
+    GDestroyNotify destroy;
+    Error *err;
+    int ret;
+};
+
+
+static gboolean gio_task_thread_result(gpointer opaque)
+{
+    struct QIOTaskThreadData *data = opaque;
+
+    trace_qio_task_thread_result(data->task);
+    if (data->ret == 0) {
+        qio_task_complete(data->task);
+    } else {
+        qio_task_abort(data->task, data->err);
+    }
+
+    error_free(data->err);
+    if (data->destroy) {
+        data->destroy(data->opaque);
+    }
+
+    g_free(data);
+
+    return FALSE;
+}
+
+
+static gpointer qio_task_thread_worker(gpointer opaque)
+{
+    struct QIOTaskThreadData *data = opaque;
+
+    trace_qio_task_thread_run(data->task);
+    data->ret = data->worker(data->task, &data->err, data->opaque);
+    if (data->ret < 0 && data->err == NULL) {
+        error_setg(&data->err, "Task worker failed but did not set an error");
+    }
+
+    /* We're running in the background thread, and must only
+     * ever report the task results in the main event loop
+     * thread. So we schedule an idle callback to report
+     * the worker results
+     */
+    trace_qio_task_thread_exit(data->task);
+    g_idle_add(gio_task_thread_result, data);
+    return NULL;
+}
+
+
+void qio_task_run_in_thread(QIOTask *task,
+                            QIOTaskWorker worker,
+                            gpointer opaque,
+                            GDestroyNotify destroy)
+{
+    struct QIOTaskThreadData *data = g_new0(struct QIOTaskThreadData, 1);
+    QemuThread thread;
+
+    data->task = task;
+    data->worker = worker;
+    data->opaque = opaque;
+    data->destroy = destroy;
+
+    trace_qio_task_thread_start(task, worker, opaque);
+    qemu_thread_create(&thread,
+                       "io-task-worker",
+                       qio_task_thread_worker,
+                       data,
+                       QEMU_THREAD_DETACHED);
+}
+
+
+void qio_task_complete(QIOTask *task)
+{
+    task->func(task->source, NULL, task->opaque);
+    trace_qio_task_complete(task);
+    qio_task_free(task);
+}
+
+void qio_task_abort(QIOTask *task,
+                    Error *err)
+{
+    task->func(task->source, err, task->opaque);
+    trace_qio_task_abort(task);
+    qio_task_free(task);
+}
+
+
+Object *qio_task_get_source(QIOTask *task)
+{
+    object_ref(task->source);
+    return task->source;
+}
diff --git a/io/trace-events b/io/trace-events
new file mode 100644
index 0000000..d064665
--- /dev/null
+++ b/io/trace-events
@@ -0,0 +1,63 @@
+# See docs/tracing.txt for syntax documentation.
+
+# io/buffer.c
+buffer_resize(const char *buf, size_t olen, size_t len) "%s: old %zd, new %zd"
+buffer_move_empty(const char *buf, size_t len, const char *from) "%s: %zd bytes from %s"
+buffer_move(const char *buf, size_t len, const char *from) "%s: %zd bytes from %s"
+buffer_free(const char *buf, size_t len) "%s: capacity %zd"
+
+# io/task.c
+qio_task_new(void *task, void *source, void *func, void *opaque) "Task new task=%p source=%p func=%p opaque=%p"
+qio_task_complete(void *task) "Task complete task=%p"
+qio_task_abort(void *task) "Task abort task=%p"
+qio_task_thread_start(void *task, void *worker, void *opaque) "Task thread start task=%p worker=%p opaque=%p"
+qio_task_thread_run(void *task) "Task thread run task=%p"
+qio_task_thread_exit(void *task) "Task thread exit task=%p"
+qio_task_thread_result(void *task) "Task thread result task=%p"
+
+# io/channel-socket.c
+qio_channel_socket_new(void *ioc) "Socket new ioc=%p"
+qio_channel_socket_new_fd(void *ioc, int fd) "Socket new ioc=%p fd=%d"
+qio_channel_socket_connect_sync(void *ioc, void *addr) "Socket connect sync ioc=%p addr=%p"
+qio_channel_socket_connect_async(void *ioc, void *addr) "Socket connect async ioc=%p addr=%p"
+qio_channel_socket_connect_fail(void *ioc) "Socket connect fail ioc=%p"
+qio_channel_socket_connect_complete(void *ioc, int fd) "Socket connect complete ioc=%p fd=%d"
+qio_channel_socket_listen_sync(void *ioc, void *addr) "Socket listen sync ioc=%p addr=%p"
+qio_channel_socket_listen_async(void *ioc, void *addr) "Socket listen async ioc=%p addr=%p"
+qio_channel_socket_listen_fail(void *ioc) "Socket listen fail ioc=%p"
+qio_channel_socket_listen_complete(void *ioc, int fd) "Socket listen complete ioc=%p fd=%d"
+qio_channel_socket_dgram_sync(void *ioc, void *localAddr, void *remoteAddr) "Socket dgram sync ioc=%p localAddr=%p remoteAddr=%p"
+qio_channel_socket_dgram_async(void *ioc, void *localAddr, void *remoteAddr) "Socket dgram async ioc=%p localAddr=%p remoteAddr=%p"
+qio_channel_socket_dgram_fail(void *ioc) "Socket dgram fail ioc=%p"
+qio_channel_socket_dgram_complete(void *ioc, int fd) "Socket dgram complete ioc=%p fd=%d"
+qio_channel_socket_accept(void *ioc) "Socket accept start ioc=%p"
+qio_channel_socket_accept_fail(void *ioc) "Socket accept fail ioc=%p"
+qio_channel_socket_accept_complete(void *ioc, void *cioc, int fd) "Socket accept complete ioc=%p cioc=%p fd=%d"
+
+# io/channel-file.c
+qio_channel_file_new_fd(void *ioc, int fd) "File new fd ioc=%p fd=%d"
+qio_channel_file_new_path(void *ioc, const char *path, int flags, int mode, int fd) "File new fd ioc=%p path=%s flags=%d mode=%d fd=%d"
+
+# io/channel-tls.c
+qio_channel_tls_new_client(void *ioc, void *master, void *creds, const char *hostname) "TLS new client ioc=%p master=%p creds=%p hostname=%s"
+qio_channel_tls_new_server(void *ioc, void *master, void *creds, const char *aclname) "TLS new client ioc=%p master=%p creds=%p acltname=%s"
+qio_channel_tls_handshake_start(void *ioc) "TLS handshake start ioc=%p"
+qio_channel_tls_handshake_pending(void *ioc, int status) "TLS handshake pending ioc=%p status=%d"
+qio_channel_tls_handshake_fail(void *ioc) "TLS handshake fail ioc=%p"
+qio_channel_tls_handshake_complete(void *ioc) "TLS handshake complete ioc=%p"
+qio_channel_tls_credentials_allow(void *ioc) "TLS credentials allow ioc=%p"
+qio_channel_tls_credentials_deny(void *ioc) "TLS credentials deny ioc=%p"
+
+# io/channel-websock.c
+qio_channel_websock_new_server(void *ioc, void *master) "Websock new client ioc=%p master=%p"
+qio_channel_websock_handshake_start(void *ioc) "Websock handshake start ioc=%p"
+qio_channel_websock_handshake_pending(void *ioc, int status) "Websock handshake pending ioc=%p status=%d"
+qio_channel_websock_handshake_reply(void *ioc) "Websock handshake reply ioc=%p"
+qio_channel_websock_handshake_fail(void *ioc) "Websock handshake fail ioc=%p"
+qio_channel_websock_handshake_complete(void *ioc) "Websock handshake complete ioc=%p"
+
+# io/channel-command.c
+qio_channel_command_new_pid(void *ioc, int writefd, int readfd, int pid) "Command new pid ioc=%p writefd=%d readfd=%d pid=%d"
+qio_channel_command_new_spawn(void *ioc, const char *binary, int flags) "Command new spawn ioc=%p binary=%s flags=%d"
+qio_channel_command_abort(void *ioc, int pid) "Command abort ioc=%p pid=%d"
+qio_channel_command_wait(void *ioc, int pid, int ret, int status) "Command abort ioc=%p pid=%d ret=%d status=%d"
diff --git a/iohandler.c b/iohandler.c
index cca614f..f2fc8a9 100644
--- a/iohandler.c
+++ b/iohandler.c
@@ -22,7 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "block/aio.h"
@@ -32,128 +33,37 @@
 #include <sys/wait.h>
 #endif
 
-typedef struct IOHandlerRecord {
-    IOCanReadHandler *fd_read_poll;
-    IOHandler *fd_read;
-    IOHandler *fd_write;
-    void *opaque;
-    QLIST_ENTRY(IOHandlerRecord) next;
-    int fd;
-    int pollfds_idx;
-    bool deleted;
-} IOHandlerRecord;
+/* This context runs on top of main loop. We can't reuse qemu_aio_context
+ * because iohandlers mustn't be polled by aio_poll(qemu_aio_context). */
+static AioContext *iohandler_ctx;
 
-static QLIST_HEAD(, IOHandlerRecord) io_handlers =
-    QLIST_HEAD_INITIALIZER(io_handlers);
+static void iohandler_init(void)
+{
+    if (!iohandler_ctx) {
+        iohandler_ctx = aio_context_new(&error_abort);
+    }
+}
 
+AioContext *iohandler_get_aio_context(void)
+{
+    iohandler_init();
+    return iohandler_ctx;
+}
 
-/* XXX: fd_read_poll should be suppressed, but an API change is
-   necessary in the character devices to suppress fd_can_read(). */
-int qemu_set_fd_handler2(int fd,
-                         IOCanReadHandler *fd_read_poll,
+GSource *iohandler_get_g_source(void)
+{
+    iohandler_init();
+    return aio_get_g_source(iohandler_ctx);
+}
+
+void qemu_set_fd_handler(int fd,
                          IOHandler *fd_read,
                          IOHandler *fd_write,
                          void *opaque)
 {
-    IOHandlerRecord *ioh;
-
-    assert(fd >= 0);
-
-    if (!fd_read && !fd_write) {
-        QLIST_FOREACH(ioh, &io_handlers, next) {
-            if (ioh->fd == fd) {
-                ioh->deleted = 1;
-                break;
-            }
-        }
-    } else {
-        QLIST_FOREACH(ioh, &io_handlers, next) {
-            if (ioh->fd == fd)
-                goto found;
-        }
-        ioh = g_malloc0(sizeof(IOHandlerRecord));
-        QLIST_INSERT_HEAD(&io_handlers, ioh, next);
-    found:
-        ioh->fd = fd;
-        ioh->fd_read_poll = fd_read_poll;
-        ioh->fd_read = fd_read;
-        ioh->fd_write = fd_write;
-        ioh->opaque = opaque;
-        ioh->pollfds_idx = -1;
-        ioh->deleted = 0;
-        qemu_notify_event();
-    }
-    return 0;
-}
-
-int qemu_set_fd_handler(int fd,
-                        IOHandler *fd_read,
-                        IOHandler *fd_write,
-                        void *opaque)
-{
-    return qemu_set_fd_handler2(fd, NULL, fd_read, fd_write, opaque);
-}
-
-void qemu_iohandler_fill(GArray *pollfds)
-{
-    IOHandlerRecord *ioh;
-
-    QLIST_FOREACH(ioh, &io_handlers, next) {
-        int events = 0;
-
-        if (ioh->deleted)
-            continue;
-        if (ioh->fd_read &&
-            (!ioh->fd_read_poll ||
-             ioh->fd_read_poll(ioh->opaque) != 0)) {
-            events |= G_IO_IN | G_IO_HUP | G_IO_ERR;
-        }
-        if (ioh->fd_write) {
-            events |= G_IO_OUT | G_IO_ERR;
-        }
-        if (events) {
-            GPollFD pfd = {
-                .fd = ioh->fd,
-                .events = events,
-            };
-            ioh->pollfds_idx = pollfds->len;
-            g_array_append_val(pollfds, pfd);
-        } else {
-            ioh->pollfds_idx = -1;
-        }
-    }
-}
-
-void qemu_iohandler_poll(GArray *pollfds, int ret)
-{
-    if (ret > 0) {
-        IOHandlerRecord *pioh, *ioh;
-
-        QLIST_FOREACH_SAFE(ioh, &io_handlers, next, pioh) {
-            int revents = 0;
-
-            if (!ioh->deleted && ioh->pollfds_idx != -1) {
-                GPollFD *pfd = &g_array_index(pollfds, GPollFD,
-                                              ioh->pollfds_idx);
-                revents = pfd->revents;
-            }
-
-            if (!ioh->deleted && ioh->fd_read &&
-                (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR))) {
-                ioh->fd_read(ioh->opaque);
-            }
-            if (!ioh->deleted && ioh->fd_write &&
-                (revents & (G_IO_OUT | G_IO_ERR))) {
-                ioh->fd_write(ioh->opaque);
-            }
-
-            /* Do this last in case read/write handlers marked it for deletion */
-            if (ioh->deleted) {
-                QLIST_REMOVE(ioh, next);
-                g_free(ioh);
-            }
-        }
-    }
+    iohandler_init();
+    aio_set_fd_handler(iohandler_ctx, fd, false,
+                       fd_read, fd_write, opaque);
 }
 
 /* reaping of zombies.  right now we're not passing the status to
diff --git a/ioport.c b/ioport.c
index 783a3ae..94e08ab 100644
--- a/ioport.c
+++ b/ioport.c
@@ -25,19 +25,14 @@
  * splitted out ioport related stuffs from vl.c.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "exec/ioport.h"
 #include "trace.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 
-//#define DEBUG_IOPORT
-
-#ifdef DEBUG_IOPORT
-#  define LOG_IOPORT(...) qemu_log_mask(CPU_LOG_IOPORT, ## __VA_ARGS__)
-#else
-#  define LOG_IOPORT(...) do { } while (0)
-#endif
-
 typedef struct MemoryRegionPortioList {
     MemoryRegion mr;
     void *portio_opaque;
@@ -60,64 +55,62 @@
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void cpu_outb(pio_addr_t addr, uint8_t val)
+void cpu_outb(uint32_t addr, uint8_t val)
 {
-    LOG_IOPORT("outb: %04"FMT_pioaddr" %02"PRIx8"\n", addr, val);
-    trace_cpu_out(addr, val);
-    address_space_write(&address_space_io, addr, &val, 1);
+    trace_cpu_out(addr, 'b', val);
+    address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
+                        &val, 1);
 }
 
-void cpu_outw(pio_addr_t addr, uint16_t val)
+void cpu_outw(uint32_t addr, uint16_t val)
 {
     uint8_t buf[2];
 
-    LOG_IOPORT("outw: %04"FMT_pioaddr" %04"PRIx16"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'w', val);
     stw_p(buf, val);
-    address_space_write(&address_space_io, addr, buf, 2);
+    address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
+                        buf, 2);
 }
 
-void cpu_outl(pio_addr_t addr, uint32_t val)
+void cpu_outl(uint32_t addr, uint32_t val)
 {
     uint8_t buf[4];
 
-    LOG_IOPORT("outl: %04"FMT_pioaddr" %08"PRIx32"\n", addr, val);
-    trace_cpu_out(addr, val);
+    trace_cpu_out(addr, 'l', val);
     stl_p(buf, val);
-    address_space_write(&address_space_io, addr, buf, 4);
+    address_space_write(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
+                        buf, 4);
 }
 
-uint8_t cpu_inb(pio_addr_t addr)
+uint8_t cpu_inb(uint32_t addr)
 {
     uint8_t val;
 
-    address_space_read(&address_space_io, addr, &val, 1);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inb : %04"FMT_pioaddr" %02"PRIx8"\n", addr, val);
+    address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED,
+                       &val, 1);
+    trace_cpu_in(addr, 'b', val);
     return val;
 }
 
-uint16_t cpu_inw(pio_addr_t addr)
+uint16_t cpu_inw(uint32_t addr)
 {
     uint8_t buf[2];
     uint16_t val;
 
-    address_space_read(&address_space_io, addr, buf, 2);
+    address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED, buf, 2);
     val = lduw_p(buf);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inw : %04"FMT_pioaddr" %04"PRIx16"\n", addr, val);
+    trace_cpu_in(addr, 'w', val);
     return val;
 }
 
-uint32_t cpu_inl(pio_addr_t addr)
+uint32_t cpu_inl(uint32_t addr)
 {
     uint8_t buf[4];
     uint32_t val;
 
-    address_space_read(&address_space_io, addr, buf, 4);
+    address_space_read(&address_space_io, addr, MEMTXATTRS_UNSPECIFIED, buf, 4);
     val = ldl_p(buf);
-    trace_cpu_in(addr, val);
-    LOG_IOPORT("inl : %04"FMT_pioaddr" %08"PRIx32"\n", addr, val);
+    trace_cpu_in(addr, 'l', val);
     return val;
 }
 
@@ -187,9 +180,14 @@
         data = mrp->read(mrpio->portio_opaque, mrp->base + addr);
     } else if (size == 2) {
         mrp = find_portio(mrpio, addr, 1, false);
-        assert(mrp);
-        data = mrp->read(mrpio->portio_opaque, mrp->base + addr) |
-                (mrp->read(mrpio->portio_opaque, mrp->base + addr + 1) << 8);
+        if (mrp) {
+            data = mrp->read(mrpio->portio_opaque, mrp->base + addr);
+            if (addr + 1 < mrp->offset + mrp->len) {
+                data |= mrp->read(mrpio->portio_opaque, mrp->base + addr + 1) << 8;
+            } else {
+                data |= 0xff00;
+            }
+        }
     }
     return data;
 }
@@ -204,9 +202,12 @@
         mrp->write(mrpio->portio_opaque, mrp->base + addr, data);
     } else if (size == 2) {
         mrp = find_portio(mrpio, addr, 1, true);
-        assert(mrp);
-        mrp->write(mrpio->portio_opaque, mrp->base + addr, data & 0xff);
-        mrp->write(mrpio->portio_opaque, mrp->base + addr + 1, data >> 8);
+        if (mrp) {
+            mrp->write(mrpio->portio_opaque, mrp->base + addr, data & 0xff);
+            if (addr + 1 < mrp->offset + mrp->len) {
+                mrp->write(mrpio->portio_opaque, mrp->base + addr + 1, data >> 8);
+            }
+        }
     }
 }
 
@@ -239,10 +240,6 @@
         mrpio->ports[i].base = start + off_low;
     }
 
-    /*
-     * Use an alias so that the callback is called with an absolute address,
-     * rather than an offset relative to to start + off_low.
-     */
     memory_region_init_io(&mrpio->mr, piolist->owner, &portio_ops, mrpio,
                           piolist->name, off_high - off_low);
     if (piolist->flush_coalesced_mmio) {
@@ -265,7 +262,7 @@
 
     /* Handle the first entry specially.  */
     off_last = off_low = pio_start->offset;
-    off_high = off_low + pio_start->len;
+    off_high = off_low + pio_start->len + pio_start->size - 1;
     count = 1;
 
     for (pio = pio_start + 1; pio->size != 0; pio++, count++) {
@@ -280,10 +277,10 @@
             /* ... and start collecting anew.  */
             pio_start = pio;
             off_low = off_last;
-            off_high = off_low + pio->len;
+            off_high = off_low + pio->len + pio_start->size - 1;
             count = 0;
         } else if (off_last + pio->len > off_high) {
-            off_high = off_last + pio->len;
+            off_high = off_last + pio->len + pio_start->size - 1;
         }
     }
 
diff --git a/iothread.c b/iothread.c
index 342a23f..f183d38 100644
--- a/iothread.c
+++ b/iothread.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "qom/object_interfaces.h"
 #include "qemu/module.h"
@@ -18,8 +19,7 @@
 #include "sysemu/iothread.h"
 #include "qmp-commands.h"
 #include "qemu/error-report.h"
-
-#define IOTHREADS_PATH "/objects"
+#include "qemu/rcu.h"
 
 typedef ObjectClass IOThreadClass;
 
@@ -33,6 +33,8 @@
     IOThread *iothread = opaque;
     bool blocking;
 
+    rcu_register_thread();
+
     qemu_mutex_lock(&iothread->init_done_lock);
     iothread->thread_id = qemu_get_thread_id();
     qemu_cond_signal(&iothread->init_done_cond);
@@ -47,6 +49,8 @@
         }
         aio_context_release(iothread->ctx);
     }
+
+    rcu_unregister_thread();
     return NULL;
 }
 
@@ -69,6 +73,7 @@
 {
     Error *local_error = NULL;
     IOThread *iothread = IOTHREAD(obj);
+    char *name, *thread_name;
 
     iothread->stopping = false;
     iothread->thread_id = -1;
@@ -84,8 +89,12 @@
     /* This assumes we are called from a thread with useful CPU affinity for us
      * to inherit.
      */
-    qemu_thread_create(&iothread->thread, "iothread", iothread_run,
+    name = object_get_canonical_path_component(OBJECT(obj));
+    thread_name = g_strdup_printf("IO %s", name);
+    qemu_thread_create(&iothread->thread, thread_name, iothread_run,
                        iothread, QEMU_THREAD_JOINABLE);
+    g_free(thread_name);
+    g_free(name);
 
     /* Wait for initialization to complete */
     qemu_mutex_lock(&iothread->init_done_lock);
@@ -121,18 +130,6 @@
 
 type_init(iothread_register_types)
 
-IOThread *iothread_find(const char *id)
-{
-    Object *container = container_get(object_get_root(), IOTHREADS_PATH);
-    Object *child;
-
-    child = object_property_get_link(container, id, NULL);
-    if (!child) {
-        return NULL;
-    }
-    return (IOThread *)object_dynamic_cast(child, TYPE_IOTHREAD);
-}
-
 char *iothread_get_id(IOThread *iothread)
 {
     return object_get_canonical_path_component(OBJECT(iothread));
@@ -172,7 +169,7 @@
 {
     IOThreadInfoList *head = NULL;
     IOThreadInfoList **prev = &head;
-    Object *container = container_get(object_get_root(), IOTHREADS_PATH);
+    Object *container = object_get_objects_root();
 
     object_child_foreach(container, query_one_iothread, &prev);
     return head;
diff --git a/kvm-all.c b/kvm-all.c
index 0c07d3f..8de8f71 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -13,30 +13,30 @@
  *
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <stdarg.h>
 
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
+#include "qemu/abort.h"
 #include "qemu/atomic.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
-#include "sysemu/sysemu.h"
-#include "sysemu/accel.h"
+#include "qemu/error-report.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
+#include "hw/pci/msix.h"
 #include "hw/s390x/adapter.h"
 #include "exec/gdbstub.h"
-#include "sysemu/kvm.h"
+#include "sysemu/kvm_int.h"
 #include "qemu/bswap.h"
 #include "exec/memory.h"
 #include "exec/ram_addr.h"
 #include "exec/address-spaces.h"
 #include "qemu/event_notifier.h"
 #include "trace.h"
+#include "hw/irq.h"
 
 #include "hw/boards.h"
 
@@ -45,8 +45,10 @@
 #include <sys/eventfd.h>
 #endif
 
-/* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
-#define PAGE_SIZE TARGET_PAGE_SIZE
+/* KVM uses PAGE_SIZE in its definition of KVM_COALESCED_MMIO_MAX. We
+ * need to use the real host PAGE_SIZE, as that's what KVM will use.
+ */
+#define PAGE_SIZE getpagesize()
 
 //#define DEBUG_KVM
 
@@ -60,38 +62,16 @@
 
 #define KVM_MSI_HASHTAB_SIZE    256
 
-#ifdef CONFIG_ANDROID
-#include "android/crashreport/crash-handler.h"
-
-static void __attribute__((noreturn)) die(const char* message,
-                                          const char* function) {
-    crashhandler_die_format("KVM operation failed: %s (%s)", message, function);
-    abort(); // just in case
-}
-#else
-static void __attribute__((noreturn)) die(const char* message,
-                                          const char* function) {
-    fprintf(stderr, "KVM operation failed: %s (%s)\n", message, function);
-    abort();
-}
-#endif
-
-typedef struct KVMSlot
-{
-    hwaddr start_addr;
-    ram_addr_t memory_size;
-    void *ram;
-    int slot;
-    int flags;
-} KVMSlot;
-
-typedef struct kvm_dirty_log KVMDirtyLog;
+struct KVMParkedVcpu {
+    unsigned long vcpu_id;
+    int kvm_fd;
+    QLIST_ENTRY(KVMParkedVcpu) node;
+};
 
 struct KVMState
 {
     AccelState parent_obj;
 
-    KVMSlot *slots;
     int nr_slots;
     int fd;
     int vmfd;
@@ -99,15 +79,12 @@
     struct kvm_coalesced_mmio_ring *coalesced_mmio_ring;
     bool coalesced_flush_in_progress;
     int broken_set_mem_region;
-    int migration_log;
     int vcpu_events;
     int robust_singlestep;
     int debugregs;
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     struct kvm_sw_breakpoint_head kvm_sw_breakpoints;
 #endif
-    int pit_state2;
-    int xsave, xcrs;
     int many_ioeventfds;
     int intx_set_mask;
     /* The man page (and posix) say ioctl numbers are signed int, but
@@ -115,32 +92,34 @@
      * unsigned, and treating them as signed here can break things */
     unsigned irq_set_ioctl;
     unsigned int sigmask_len;
+    GHashTable *gsimap;
 #ifdef KVM_CAP_IRQ_ROUTING
     struct kvm_irq_routing *irq_routes;
     int nr_allocated_irq_routes;
-    uint32_t *used_gsi_bitmap;
+    unsigned long *used_gsi_bitmap;
     unsigned int gsi_count;
     QTAILQ_HEAD(msi_hashtab, KVMMSIRoute) msi_hashtab[KVM_MSI_HASHTAB_SIZE];
-    bool direct_msi;
 #endif
+    KVMMemoryListener memory_listener;
+    QLIST_HEAD(, KVMParkedVcpu) kvm_parked_vcpus;
 };
 
-#define TYPE_KVM_ACCEL ACCEL_CLASS_NAME("kvm")
-
-#define KVM_STATE(obj) \
-    OBJECT_CHECK(KVMState, (obj), TYPE_KVM_ACCEL)
-
 KVMState *kvm_state;
 bool kvm_kernel_irqchip;
+bool kvm_split_irqchip;
 bool kvm_async_interrupts_allowed;
 bool kvm_halt_in_kernel_allowed;
 bool kvm_eventfds_allowed;
 bool kvm_irqfds_allowed;
+bool kvm_resamplefds_allowed;
 bool kvm_msi_via_irqfd_allowed;
 bool kvm_gsi_routing_allowed;
 bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
+bool kvm_vm_attributes_allowed;
+bool kvm_direct_msi_allowed;
+bool kvm_ioeventfd_any_length_allowed;
 
 static const KVMCapabilityInfo kvm_required_capabilites[] = {
     KVM_CAP_INFO(USER_MEMORY),
@@ -148,13 +127,21 @@
     KVM_CAP_LAST_INFO
 };
 
-static KVMSlot *kvm_get_free_slot(KVMState *s)
+int kvm_get_max_memslots(void)
 {
+    KVMState *s = KVM_STATE(current_machine->accelerator);
+
+    return s->nr_slots;
+}
+
+static KVMSlot *kvm_get_free_slot(KVMMemoryListener *kml)
+{
+    KVMState *s = kvm_state;
     int i;
 
     for (i = 0; i < s->nr_slots; i++) {
-        if (s->slots[i].memory_size == 0) {
-            return &s->slots[i];
+        if (kml->slots[i].memory_size == 0) {
+            return &kml->slots[i];
         }
     }
 
@@ -163,28 +150,31 @@
 
 bool kvm_has_free_slot(MachineState *ms)
 {
-    return kvm_get_free_slot(KVM_STATE(ms->accelerator));
+    KVMState *s = KVM_STATE(ms->accelerator);
+
+    return kvm_get_free_slot(&s->memory_listener);
 }
 
-static KVMSlot *kvm_alloc_slot(KVMState *s)
+static KVMSlot *kvm_alloc_slot(KVMMemoryListener *kml)
 {
-    KVMSlot *slot = kvm_get_free_slot(s);
+    KVMSlot *slot = kvm_get_free_slot(kml);
 
     if (slot) {
         return slot;
     }
 
-    die("no free slot available", __func__);
+    qemu_abort("%s: no free slot available\n", __func__);
 }
 
-static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
+static KVMSlot *kvm_lookup_matching_slot(KVMMemoryListener *kml,
                                          hwaddr start_addr,
                                          hwaddr end_addr)
 {
+    KVMState *s = kvm_state;
     int i;
 
     for (i = 0; i < s->nr_slots; i++) {
-        KVMSlot *mem = &s->slots[i];
+        KVMSlot *mem = &kml->slots[i];
 
         if (start_addr == mem->start_addr &&
             end_addr == mem->start_addr + mem->memory_size) {
@@ -198,15 +188,16 @@
 /*
  * Find overlapping slot with lowest start address
  */
-static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
+static KVMSlot *kvm_lookup_overlapping_slot(KVMMemoryListener *kml,
                                             hwaddr start_addr,
                                             hwaddr end_addr)
 {
+    KVMState *s = kvm_state;
     KVMSlot *found = NULL;
     int i;
 
     for (i = 0; i < s->nr_slots; i++) {
-        KVMSlot *mem = &s->slots[i];
+        KVMSlot *mem = &kml->slots[i];
 
         if (mem->memory_size == 0 ||
             (found && found->start_addr < mem->start_addr)) {
@@ -225,10 +216,11 @@
 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
                                        hwaddr *phys_addr)
 {
+    KVMMemoryListener *kml = &s->memory_listener;
     int i;
 
     for (i = 0; i < s->nr_slots; i++) {
-        KVMSlot *mem = &s->slots[i];
+        KVMSlot *mem = &kml->slots[i];
 
         if (ram >= mem->ram && ram < mem->ram + mem->memory_size) {
             *phys_addr = mem->start_addr + (ram - mem->ram);
@@ -239,17 +231,15 @@
     return 0;
 }
 
-static int kvm_set_user_memory_region(KVMState *s, KVMSlot *slot)
+static int kvm_set_user_memory_region(KVMMemoryListener *kml, KVMSlot *slot)
 {
+    KVMState *s = kvm_state;
     struct kvm_userspace_memory_region mem;
 
-    mem.slot = slot->slot;
+    mem.slot = slot->slot | (kml->as_id << 16);
     mem.guest_phys_addr = slot->start_addr;
     mem.userspace_addr = (unsigned long)slot->ram;
     mem.flags = slot->flags;
-    if (s->migration_log) {
-        mem.flags |= KVM_MEM_LOG_DIRTY_PAGES;
-    }
 
     if (slot->memory_size && mem.flags & KVM_MEM_READONLY) {
         /* Set the slot size to 0 before setting the slot to the desired
@@ -261,6 +251,53 @@
     return kvm_vm_ioctl(s, KVM_SET_USER_MEMORY_REGION, &mem);
 }
 
+int kvm_destroy_vcpu(CPUState *cpu)
+{
+    KVMState *s = kvm_state;
+    long mmap_size;
+    struct KVMParkedVcpu *vcpu = NULL;
+    int ret = 0;
+
+    DPRINTF("kvm_destroy_vcpu\n");
+
+    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE, 0);
+    if (mmap_size < 0) {
+        ret = mmap_size;
+        DPRINTF("KVM_GET_VCPU_MMAP_SIZE failed\n");
+        goto err;
+    }
+
+    ret = munmap(cpu->kvm_run, mmap_size);
+    if (ret < 0) {
+        goto err;
+    }
+
+    vcpu = g_malloc0(sizeof(*vcpu));
+    vcpu->vcpu_id = kvm_arch_vcpu_id(cpu);
+    vcpu->kvm_fd = cpu->kvm_fd;
+    QLIST_INSERT_HEAD(&kvm_state->kvm_parked_vcpus, vcpu, node);
+err:
+    return ret;
+}
+
+static int kvm_get_vcpu(KVMState *s, unsigned long vcpu_id)
+{
+    struct KVMParkedVcpu *cpu;
+
+    QLIST_FOREACH(cpu, &s->kvm_parked_vcpus, node) {
+        if (cpu->vcpu_id == vcpu_id) {
+            int kvm_fd;
+
+            QLIST_REMOVE(cpu, node);
+            kvm_fd = cpu->kvm_fd;
+            g_free(cpu);
+            return kvm_fd;
+        }
+    }
+
+    return kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)vcpu_id);
+}
+
 int kvm_init_vcpu(CPUState *cpu)
 {
     KVMState *s = kvm_state;
@@ -269,7 +306,7 @@
 
     DPRINTF("kvm_init_vcpu\n");
 
-    ret = kvm_vm_ioctl(s, KVM_CREATE_VCPU, (void *)kvm_arch_vcpu_id(cpu));
+    ret = kvm_get_vcpu(s, kvm_arch_vcpu_id(cpu));
     if (ret < 0) {
         DPRINTF("kvm_create_vcpu failed\n");
         goto err;
@@ -308,108 +345,90 @@
  * dirty pages logging control
  */
 
-static int kvm_mem_flags(KVMState *s, bool log_dirty, bool readonly)
+static int kvm_mem_flags(MemoryRegion *mr)
 {
+    bool readonly = mr->readonly || memory_region_is_romd(mr);
     int flags = 0;
-    flags = log_dirty ? KVM_MEM_LOG_DIRTY_PAGES : 0;
+
+    if (memory_region_get_dirty_log_mask(mr) != 0) {
+        flags |= KVM_MEM_LOG_DIRTY_PAGES;
+    }
     if (readonly && kvm_readonly_mem_allowed) {
         flags |= KVM_MEM_READONLY;
     }
     return flags;
 }
 
-static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
+static int kvm_slot_update_flags(KVMMemoryListener *kml, KVMSlot *mem,
+                                 MemoryRegion *mr)
 {
-    KVMState *s = kvm_state;
-    int flags, mask = KVM_MEM_LOG_DIRTY_PAGES;
     int old_flags;
 
     old_flags = mem->flags;
-
-    flags = (mem->flags & ~mask) | kvm_mem_flags(s, log_dirty, false);
-    mem->flags = flags;
+    mem->flags = kvm_mem_flags(mr);
 
     /* If nothing changed effectively, no need to issue ioctl */
-    if (s->migration_log) {
-        flags |= KVM_MEM_LOG_DIRTY_PAGES;
-    }
-
-    if (flags == old_flags) {
+    if (mem->flags == old_flags) {
         return 0;
     }
 
-    return kvm_set_user_memory_region(s, mem);
+    return kvm_set_user_memory_region(kml, mem);
 }
 
-static int kvm_dirty_pages_log_change(hwaddr phys_addr,
-                                      ram_addr_t size, bool log_dirty)
+static int kvm_section_update_flags(KVMMemoryListener *kml,
+                                    MemoryRegionSection *section)
 {
-    KVMState *s = kvm_state;
-    KVMSlot *mem = kvm_lookup_matching_slot(s, phys_addr, phys_addr + size);
+    hwaddr phys_addr = section->offset_within_address_space;
+    ram_addr_t size = int128_get64(section->size);
+    KVMSlot *mem = kvm_lookup_matching_slot(kml, phys_addr, phys_addr + size);
 
     if (mem == NULL)  {
-        fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
-                TARGET_FMT_plx "\n", __func__, phys_addr,
-                (hwaddr)(phys_addr + size - 1));
-        return -EINVAL;
+        return 0;
+    } else {
+        return kvm_slot_update_flags(kml, mem, section->mr);
     }
-    return kvm_slot_dirty_pages_log_change(mem, log_dirty);
 }
 
 static void kvm_log_start(MemoryListener *listener,
-                          MemoryRegionSection *section)
+                          MemoryRegionSection *section,
+                          int old, int new)
 {
+    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
     int r;
 
-    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
-                                   int128_get64(section->size), true);
+    if (old != 0) {
+        return;
+    }
+
+    r = kvm_section_update_flags(kml, section);
     if (r < 0) {
-        die("dirty pages log change", __func__);
+        qemu_abort("%s: dirty pages log change\n", __func__);
     }
 }
 
 static void kvm_log_stop(MemoryListener *listener,
-                          MemoryRegionSection *section)
+                          MemoryRegionSection *section,
+                          int old, int new)
 {
+    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
     int r;
 
-    r = kvm_dirty_pages_log_change(section->offset_within_address_space,
-                                   int128_get64(section->size), false);
+    if (new != 0) {
+        return;
+    }
+
+    r = kvm_section_update_flags(kml, section);
     if (r < 0) {
-        die("dirty pages log change", __func__);
+        qemu_abort("%s: dirty pages log change\n", __func__);
     }
 }
 
-static int kvm_set_migration_log(int enable)
-{
-    KVMState *s = kvm_state;
-    KVMSlot *mem;
-    int i, err;
-
-    s->migration_log = enable;
-
-    for (i = 0; i < s->nr_slots; i++) {
-        mem = &s->slots[i];
-
-        if (!mem->memory_size) {
-            continue;
-        }
-        if (!!(mem->flags & KVM_MEM_LOG_DIRTY_PAGES) == enable) {
-            continue;
-        }
-        err = kvm_set_user_memory_region(s, mem);
-        if (err) {
-            return err;
-        }
-    }
-    return 0;
-}
-
 /* get kvm's dirty pages bitmap and update qemu's */
 static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
                                          unsigned long *bitmap)
 {
-    ram_addr_t start = section->offset_within_region + section->mr->ram_addr;
+    ram_addr_t start = section->offset_within_region +
+                       memory_region_get_ram_addr(section->mr);
     ram_addr_t pages = int128_get64(section->size) / getpagesize();
 
     cpu_physical_memory_set_dirty_lebitmap(bitmap, start, pages);
@@ -427,11 +446,12 @@
  * @start_add: start of logged region.
  * @end_addr: end of logged region.
  */
-static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
+static int kvm_physical_sync_dirty_bitmap(KVMMemoryListener *kml,
+                                          MemoryRegionSection *section)
 {
     KVMState *s = kvm_state;
     unsigned long size, allocated_size = 0;
-    KVMDirtyLog d;
+    struct kvm_dirty_log d = {};
     KVMSlot *mem;
     int ret = 0;
     hwaddr start_addr = section->offset_within_address_space;
@@ -439,7 +459,7 @@
 
     d.dirty_bitmap = NULL;
     while (start_addr < end_addr) {
-        mem = kvm_lookup_overlapping_slot(s, start_addr, end_addr);
+        mem = kvm_lookup_overlapping_slot(kml, start_addr, end_addr);
         if (mem == NULL) {
             break;
         }
@@ -454,7 +474,7 @@
          * userspace memory corruption (which is not detectable by valgrind
          * too, in most cases).
          * So for now, let's align to 64 instead of HOST_LONG_BITS here, in
-         * a hope that sizeof(long) wont become >8 any time soon.
+         * a hope that sizeof(long) won't become >8 any time soon.
          */
         size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
                      /*HOST_LONG_BITS*/ 64) / 8;
@@ -466,8 +486,7 @@
         allocated_size = size;
         memset(d.dirty_bitmap, 0, allocated_size);
 
-        d.slot = mem->slot;
-
+        d.slot = mem->slot | (kml->as_id << 16);
         if (kvm_vm_ioctl(s, KVM_GET_DIRTY_LOG, &d) == -1) {
             DPRINTF("ioctl failed %d\n", errno);
             ret = -1;
@@ -541,17 +560,37 @@
     return ret;
 }
 
+static uint32_t adjust_ioeventfd_endianness(uint32_t val, uint32_t size)
+{
+#if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
+    /* The kernel expects ioeventfd values in HOST_WORDS_BIGENDIAN
+     * endianness, but the memory core hands them in target endianness.
+     * For example, PPC is always treated as big-endian even if running
+     * on KVM and on PPC64LE.  Correct here.
+     */
+    switch (size) {
+    case 2:
+        val = bswap16(val);
+        break;
+    case 4:
+        val = bswap32(val);
+        break;
+    }
+#endif
+    return val;
+}
+
 static int kvm_set_ioeventfd_mmio(int fd, hwaddr addr, uint32_t val,
                                   bool assign, uint32_t size, bool datamatch)
 {
     int ret;
-    struct kvm_ioeventfd iofd;
-
-    iofd.datamatch = datamatch ? val : 0;
-    iofd.addr = addr;
-    iofd.len = size;
-    iofd.flags = 0;
-    iofd.fd = fd;
+    struct kvm_ioeventfd iofd = {
+        .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
+        .addr = addr,
+        .len = size,
+        .flags = 0,
+        .fd = fd,
+    };
 
     if (!kvm_enabled()) {
         return -ENOSYS;
@@ -577,7 +616,7 @@
                                  bool assign, uint32_t size, bool datamatch)
 {
     struct kvm_ioeventfd kick = {
-        .datamatch = datamatch ? val : 0,
+        .datamatch = datamatch ? adjust_ioeventfd_endianness(val, size) : 0,
         .addr = addr,
         .flags = KVM_IOEVENTFD_FLAG_PIO,
         .len = size,
@@ -650,15 +689,14 @@
     return NULL;
 }
 
-static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
+static void kvm_set_phys_mem(KVMMemoryListener *kml,
+                             MemoryRegionSection *section, bool add)
 {
     KVMState *s = kvm_state;
     KVMSlot *mem, old;
     int err;
     MemoryRegion *mr = section->mr;
-    bool log_dirty = memory_region_is_logging(mr);
     bool writeable = !mr->readonly && !mr->rom_device;
-    bool readonly_flag = mr->readonly || memory_region_is_romd(mr);
     hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = int128_get64(section->size);
     void *ram = NULL;
@@ -667,15 +705,15 @@
     /* kvm works in page size chunks, but the function may be called
        with sub-page size and unaligned start address. Pad the start
        address to next and truncate size to previous page boundary. */
-    delta = (TARGET_PAGE_SIZE - (start_addr & ~TARGET_PAGE_MASK));
-    delta &= ~TARGET_PAGE_MASK;
+    delta = qemu_real_host_page_size - (start_addr & ~qemu_real_host_page_mask);
+    delta &= ~qemu_real_host_page_mask;
     if (delta > size) {
         return;
     }
     start_addr += delta;
     size -= delta;
-    size &= TARGET_PAGE_MASK;
-    if (!size || (start_addr & ~TARGET_PAGE_MASK)) {
+    size &= qemu_real_host_page_mask;
+    if (!size || (start_addr & ~qemu_real_host_page_mask)) {
         return;
     }
 
@@ -692,7 +730,7 @@
     ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta;
 
     while (1) {
-        mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size);
+        mem = kvm_lookup_overlapping_slot(kml, start_addr, start_addr + size);
         if (!mem) {
             break;
         }
@@ -702,23 +740,22 @@
             (ram - start_addr == mem->ram - mem->start_addr)) {
             /* The new slot fits into the existing one and comes with
              * identical parameters - update flags and done. */
-            kvm_slot_dirty_pages_log_change(mem, log_dirty);
+            kvm_slot_update_flags(kml, mem, mr);
             return;
         }
 
         old = *mem;
 
         if (mem->flags & KVM_MEM_LOG_DIRTY_PAGES) {
-            kvm_physical_sync_dirty_bitmap(section);
+            kvm_physical_sync_dirty_bitmap(kml, section);
         }
 
         /* unregister the overlapping slot */
         mem->memory_size = 0;
-        err = kvm_set_user_memory_region(s, mem);
+        err = kvm_set_user_memory_region(kml, mem);
         if (err) {
-            fprintf(stderr, "%s: error unregistering overlapping slot: %s\n",
+            qemu_abort("%s: error unregistering overlapping slot: %s\n",
                     __func__, strerror(-err));
-            die("error unregistering overlapping slot", __func__);
         }
 
         /* Workaround for older KVM versions: we can't join slots, even not by
@@ -731,18 +768,16 @@
          * - and actually require a recent KVM version. */
         if (s->broken_set_mem_region &&
             old.start_addr == start_addr && old.memory_size < size && add) {
-            mem = kvm_alloc_slot(s);
+            mem = kvm_alloc_slot(kml);
             mem->memory_size = old.memory_size;
             mem->start_addr = old.start_addr;
             mem->ram = old.ram;
-            mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
+            mem->flags = kvm_mem_flags(mr);
 
-            err = kvm_set_user_memory_region(s, mem);
+            err = kvm_set_user_memory_region(kml, mem);
             if (err) {
-                fprintf(stderr, "%s: error updating slot: %s\n", __func__,
+                qemu_abort("%s: error updating slot: %s\n", __func__,
                         strerror(-err));
-                die("failed to set user memory region "
-                    "(most probably out of memory)", __func__);
             }
 
             start_addr += old.memory_size;
@@ -753,23 +788,21 @@
 
         /* register prefix slot */
         if (old.start_addr < start_addr) {
-            mem = kvm_alloc_slot(s);
+            mem = kvm_alloc_slot(kml);
             mem->memory_size = start_addr - old.start_addr;
             mem->start_addr = old.start_addr;
             mem->ram = old.ram;
-            mem->flags =  kvm_mem_flags(s, log_dirty, readonly_flag);
+            mem->flags =  kvm_mem_flags(mr);
 
-            err = kvm_set_user_memory_region(s, mem);
+            err = kvm_set_user_memory_region(kml, mem);
             if (err) {
-                fprintf(stderr, "%s: error registering prefix slot: %s\n",
-                        __func__, strerror(-err));
+                qemu_abort("%s: error registering prefix slot: %s\n"
 #ifdef TARGET_PPC
-                fprintf(stderr, "%s: This is probably because your kernel's " \
+                                "This is probably because your kernel's " \
                                 "PAGE_SIZE is too big. Please try to use 4k " \
-                                "PAGE_SIZE!\n", __func__);
+                                "PAGE_SIZE!\n"
 #endif
-                die("failed to set user memory region "
-                    "(most probably out of memory)", __func__);
+                        ,__func__ , strerror(-err));
             }
         }
 
@@ -777,19 +810,17 @@
         if (old.start_addr + old.memory_size > start_addr + size) {
             ram_addr_t size_delta;
 
-            mem = kvm_alloc_slot(s);
+            mem = kvm_alloc_slot(kml);
             mem->start_addr = start_addr + size;
             size_delta = mem->start_addr - old.start_addr;
             mem->memory_size = old.memory_size - size_delta;
             mem->ram = old.ram + size_delta;
-            mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
+            mem->flags = kvm_mem_flags(mr);
 
-            err = kvm_set_user_memory_region(s, mem);
+            err = kvm_set_user_memory_region(kml, mem);
             if (err) {
-                fprintf(stderr, "%s: error registering suffix slot: %s\n",
+                qemu_abort("%s: error registering suffix slot: %s\n",
                         __func__, strerror(-err));
-                die("failed to set user memory region "
-                    "(most probably out of memory)", __func__);
             }
         }
     }
@@ -801,63 +832,49 @@
     if (!add) {
         return;
     }
-    mem = kvm_alloc_slot(s);
+    mem = kvm_alloc_slot(kml);
     mem->memory_size = size;
     mem->start_addr = start_addr;
     mem->ram = ram;
-    mem->flags = kvm_mem_flags(s, log_dirty, readonly_flag);
+    mem->flags = kvm_mem_flags(mr);
 
-    err = kvm_set_user_memory_region(s, mem);
+    err = kvm_set_user_memory_region(kml, mem);
     if (err) {
-        fprintf(stderr, "%s: error registering slot: %s\n", __func__,
+        qemu_abort("%s: error registering slot: %s\n", __func__,
                 strerror(-err));
-        die("error registering slot", __func__);
     }
 }
 
 static void kvm_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
+    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
+
     memory_region_ref(section->mr);
-    kvm_set_phys_mem(section, true);
+    kvm_set_phys_mem(kml, section, true);
 }
 
 static void kvm_region_del(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
-    kvm_set_phys_mem(section, false);
+    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
+
+    kvm_set_phys_mem(kml, section, false);
     memory_region_unref(section->mr);
 }
 
 static void kvm_log_sync(MemoryListener *listener,
                          MemoryRegionSection *section)
 {
+    KVMMemoryListener *kml = container_of(listener, KVMMemoryListener, listener);
     int r;
 
-    r = kvm_physical_sync_dirty_bitmap(section);
+    r = kvm_physical_sync_dirty_bitmap(kml, section);
     if (r < 0) {
-        die("sync dirty bitmap", __func__);
+        qemu_abort("%s: sync dirty bitmap\n", __func__);
     }
 }
 
-static void kvm_log_global_start(struct MemoryListener *listener)
-{
-    int r;
-
-    r = kvm_set_migration_log(1);
-    assert(r >= 0);
-    (void)r;
-}
-
-static void kvm_log_global_stop(struct MemoryListener *listener)
-{
-    int r;
-
-    r = kvm_set_migration_log(0);
-    assert(r >= 0);
-    (void)r;
-}
-
 static void kvm_mem_ioeventfd_add(MemoryListener *listener,
                                   MemoryRegionSection *section,
                                   bool match_data, uint64_t data,
@@ -870,9 +887,8 @@
                                data, true, int128_get64(section->size),
                                match_data);
     if (r < 0) {
-        fprintf(stderr, "%s: error adding ioeventfd: %s\n",
+        qemu_abort("%s: error adding ioeventfd: %s\n",
                 __func__, strerror(-r));
-        die("error adding ioeventf", __func__);
     }
 }
 
@@ -888,7 +904,7 @@
                                data, false, int128_get64(section->size),
                                match_data);
     if (r < 0) {
-        die("set ioeventfd mmio", __func__);
+        qemu_abort("%s: set ioeventfd mmio", __func__);
     }
 }
 
@@ -904,9 +920,8 @@
                               data, true, int128_get64(section->size),
                               match_data);
     if (r < 0) {
-        fprintf(stderr, "%s: error adding ioeventfd: %s\n",
+        qemu_abort("%s: error adding ioeventfd: %s\n",
                 __func__, strerror(-r));
-        die("error adding ioeventfd", __func__);
     }
 }
 
@@ -923,24 +938,31 @@
                               data, false, int128_get64(section->size),
                               match_data);
     if (r < 0) {
-        die("set ioeventfd pio", __func__);
+        qemu_abort("%s: set ioeventfd pio", __func__);
     }
 }
 
-static MemoryListener kvm_memory_listener = {
-    .region_add = kvm_region_add,
-    .region_del = kvm_region_del,
-    .log_start = kvm_log_start,
-    .log_stop = kvm_log_stop,
-    .log_sync = kvm_log_sync,
-    .log_global_start = kvm_log_global_start,
-    .log_global_stop = kvm_log_global_stop,
-    .eventfd_add = kvm_mem_ioeventfd_add,
-    .eventfd_del = kvm_mem_ioeventfd_del,
-    .coalesced_mmio_add = kvm_coalesce_mmio_region,
-    .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
-    .priority = 10,
-};
+void kvm_memory_listener_register(KVMState *s, KVMMemoryListener *kml,
+                                  AddressSpace *as, int as_id)
+{
+    int i;
+
+    kml->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
+    kml->as_id = as_id;
+
+    for (i = 0; i < s->nr_slots; i++) {
+        kml->slots[i].slot = i;
+    }
+
+    kml->listener.region_add = kvm_region_add;
+    kml->listener.region_del = kvm_region_del;
+    kml->listener.log_start = kvm_log_start;
+    kml->listener.log_stop = kvm_log_stop;
+    kml->listener.log_sync = kvm_log_sync;
+    kml->listener.priority = 10;
+
+    memory_listener_register(&kml->listener, as);
+}
 
 static MemoryListener kvm_io_listener = {
     .eventfd_add = kvm_io_ioeventfd_add,
@@ -968,8 +990,7 @@
     event.irq = irq;
     ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
     if (ret < 0) {
-        perror("kvm_set_irq");
-        die("vm ioctl", __func__);
+        qemu_abort("%s: %s", __func__, strerror(errno));
     }
 
     return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
@@ -983,12 +1004,12 @@
 
 static void set_gsi(KVMState *s, unsigned int gsi)
 {
-    s->used_gsi_bitmap[gsi / 32] |= 1U << (gsi % 32);
+    set_bit(gsi, s->used_gsi_bitmap);
 }
 
 static void clear_gsi(KVMState *s, unsigned int gsi)
 {
-    s->used_gsi_bitmap[gsi / 32] &= ~(1U << (gsi % 32));
+    clear_bit(gsi, s->used_gsi_bitmap);
 }
 
 void kvm_init_irq_routing(KVMState *s)
@@ -997,23 +1018,15 @@
 
     gsi_count = kvm_check_extension(s, KVM_CAP_IRQ_ROUTING) - 1;
     if (gsi_count > 0) {
-        unsigned int gsi_bits, i;
-
         /* Round up so we can search ints using ffs */
-        gsi_bits = ALIGN(gsi_count, 32);
-        s->used_gsi_bitmap = g_malloc0(gsi_bits / 8);
+        s->used_gsi_bitmap = bitmap_new(gsi_count);
         s->gsi_count = gsi_count;
-
-        /* Mark any over-allocated bits as already in use */
-        for (i = gsi_count; i < gsi_bits; i++) {
-            set_gsi(s, i);
-        }
     }
 
     s->irq_routes = g_malloc0(sizeof(*s->irq_routes));
     s->nr_allocated_irq_routes = 0;
 
-    if (!s->direct_msi) {
+    if (!kvm_direct_msi_allowed) {
         for (i = 0; i < KVM_MSI_HASHTAB_SIZE; i++) {
             QTAILQ_INIT(&s->msi_hashtab[i]);
         }
@@ -1026,7 +1039,16 @@
 {
     int ret;
 
+    if (kvm_gsi_direct_mapping()) {
+        return;
+    }
+
+    if (!kvm_gsi_routing_enabled()) {
+        return;
+    }
+
     s->irq_routes->flags = 0;
+    trace_kvm_irqchip_commit_routes();
     ret = kvm_vm_ioctl(s, KVM_SET_GSI_ROUTING, s->irq_routes);
     assert(ret == 0);
     (void)ret;
@@ -1074,8 +1096,6 @@
 
         *entry = *new_entry;
 
-        kvm_irqchip_commit_routes(s);
-
         return 0;
     }
 
@@ -1113,6 +1133,7 @@
         }
     }
     clear_gsi(s, virq);
+    kvm_arch_release_virq_post(virq);
 }
 
 static unsigned int kvm_hash_msi(uint32_t data)
@@ -1138,28 +1159,25 @@
 
 static int kvm_irqchip_get_virq(KVMState *s)
 {
-    uint32_t *word = s->used_gsi_bitmap;
-    int max_words = ALIGN(s->gsi_count, 32) / 32;
-    int i, bit;
-    bool retry = true;
+    int next_virq;
 
-again:
-    /* Return the lowest unused GSI in the bitmap */
-    for (i = 0; i < max_words; i++) {
-        bit = ffs(~word[i]);
-        if (!bit) {
-            continue;
-        }
-
-        return bit - 1 + i * 32;
-    }
-    if (!s->direct_msi && retry) {
-        retry = false;
+    /*
+     * PIC and IOAPIC share the first 16 GSI numbers, thus the available
+     * GSI numbers are more than the number of IRQ route. Allocating a GSI
+     * number can succeed even though a new route entry cannot be added.
+     * When this happens, flush dynamic MSI entries to free IRQ route entries.
+     */
+    if (!kvm_direct_msi_allowed && s->irq_routes->nr == s->gsi_count) {
         kvm_flush_dynamic_msi_routes(s);
-        goto again;
     }
-    return -ENOSPC;
 
+    /* Return the lowest unused GSI in the bitmap */
+    next_virq = find_first_zero_bit(s->used_gsi_bitmap, s->gsi_count);
+    if (next_virq >= s->gsi_count) {
+        return -ENOSPC;
+    } else {
+        return next_virq;
+    }
 }
 
 static KVMMSIRoute *kvm_lookup_msi_route(KVMState *s, MSIMessage msg)
@@ -1182,7 +1200,7 @@
     struct kvm_msi msi;
     KVMMSIRoute *route;
 
-    if (s->direct_msi) {
+    if (kvm_direct_msi_allowed) {
         msi.address_lo = (uint32_t)msg.address;
         msi.address_hi = msg.address >> 32;
         msi.data = le32_to_cpu(msg.data);
@@ -1221,13 +1239,18 @@
     return kvm_set_irq(s, route->kroute.gsi, 1);
 }
 
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
 {
     struct kvm_irq_routing_entry kroute = {};
     int virq;
+    MSIMessage msg = {0, 0};
+
+    if (dev) {
+        msg = pci_get_msi_message(dev, vector);
+    }
 
     if (kvm_gsi_direct_mapping()) {
-        return msg.data & 0xffff;
+        return kvm_arch_msi_data_to_gsi(msg.data);
     }
 
     if (!kvm_gsi_routing_enabled()) {
@@ -1245,14 +1268,22 @@
     kroute.u.msi.address_lo = (uint32_t)msg.address;
     kroute.u.msi.address_hi = msg.address >> 32;
     kroute.u.msi.data = le32_to_cpu(msg.data);
+    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
+        kvm_irqchip_release_virq(s, virq);
+        return -EINVAL;
+    }
+
+    trace_kvm_irqchip_add_msi_route(virq);
 
     kvm_add_routing_entry(s, &kroute);
+    kvm_arch_add_msi_route_post(&kroute, vector, dev);
     kvm_irqchip_commit_routes(s);
 
     return virq;
 }
 
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev)
 {
     struct kvm_irq_routing_entry kroute = {};
 
@@ -1270,6 +1301,11 @@
     kroute.u.msi.address_lo = (uint32_t)msg.address;
     kroute.u.msi.address_hi = msg.address >> 32;
     kroute.u.msi.data = le32_to_cpu(msg.data);
+    if (kvm_arch_fixup_msi_route(&kroute, msg.address, msg.data, dev)) {
+        return -EINVAL;
+    }
+
+    trace_kvm_irqchip_update_msi_route(virq);
 
     return kvm_update_routing_entry(s, &kroute);
 }
@@ -1297,7 +1333,7 @@
 
 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
 {
-    struct kvm_irq_routing_entry kroute;
+    struct kvm_irq_routing_entry kroute = {};
     int virq;
 
     if (!kvm_gsi_routing_enabled()) {
@@ -1319,6 +1355,33 @@
     kroute.u.adapter.adapter_id = adapter->adapter_id;
 
     kvm_add_routing_entry(s, &kroute);
+
+    return virq;
+}
+
+int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
+{
+    struct kvm_irq_routing_entry kroute = {};
+    int virq;
+
+    if (!kvm_gsi_routing_enabled()) {
+        return -ENOSYS;
+    }
+    if (!kvm_check_extension(s, KVM_CAP_HYPERV_SYNIC)) {
+        return -ENOSYS;
+    }
+    virq = kvm_irqchip_get_virq(s);
+    if (virq < 0) {
+        return virq;
+    }
+
+    kroute.gsi = virq;
+    kroute.type = KVM_IRQ_ROUTING_HV_SINT;
+    kroute.flags = 0;
+    kroute.u.hv_sint.vcpu = vcpu;
+    kroute.u.hv_sint.sint = sint;
+
+    kvm_add_routing_entry(s, &kroute);
     kvm_irqchip_commit_routes(s);
 
     return virq;
@@ -1336,10 +1399,10 @@
 
 int kvm_irqchip_send_msi(KVMState *s, MSIMessage msg)
 {
-    die("should not be called", __func__);
+    qemu_abort("%s: should not be called\n", __func__);
 }
 
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
 {
     return -ENOSYS;
 }
@@ -1349,9 +1412,14 @@
     return -ENOSYS;
 }
 
+int kvm_irqchip_add_hv_sint_route(KVMState *s, uint32_t vcpu, uint32_t sint)
+{
+    return -ENOSYS;
+}
+
 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
-    die("should not be called", __func__);
+    qemu_abort("%s: should not be called\n", __func__);
 }
 
 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
@@ -1360,41 +1428,80 @@
 }
 #endif /* !KVM_CAP_IRQ_ROUTING */
 
-int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
-                                   EventNotifier *rn, int virq)
+int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
+                                       EventNotifier *rn, int virq)
 {
     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n),
            rn ? event_notifier_get_fd(rn) : -1, virq, true);
 }
 
-int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
+                                          int virq)
 {
     return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), -1, virq,
            false);
 }
 
-static int kvm_irqchip_create(KVMState *s)
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
+                                   EventNotifier *rn, qemu_irq irq)
+{
+    gpointer key, gsi;
+    gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
+
+    if (!found) {
+        return -ENXIO;
+    }
+    return kvm_irqchip_add_irqfd_notifier_gsi(s, n, rn, GPOINTER_TO_INT(gsi));
+}
+
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n,
+                                      qemu_irq irq)
+{
+    gpointer key, gsi;
+    gboolean found = g_hash_table_lookup_extended(s->gsimap, irq, &key, &gsi);
+
+    if (!found) {
+        return -ENXIO;
+    }
+    return kvm_irqchip_remove_irqfd_notifier_gsi(s, n, GPOINTER_TO_INT(gsi));
+}
+
+void kvm_irqchip_set_qemuirq_gsi(KVMState *s, qemu_irq irq, int gsi)
+{
+    g_hash_table_insert(s->gsimap, irq, GINT_TO_POINTER(gsi));
+}
+
+static void kvm_irqchip_create(MachineState *machine, KVMState *s)
 {
     int ret;
 
-    if (!qemu_opt_get_bool(qemu_get_machine_opts(), "kernel_irqchip", true) ||
-        (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
-         (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
-        return 0;
+    if (kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
+        ;
+    } else if (kvm_check_extension(s, KVM_CAP_S390_IRQCHIP)) {
+        ret = kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0);
+        if (ret < 0) {
+            fprintf(stderr, "Enable kernel irqchip failed: %s\n", strerror(-ret));
+            exit(1);
+        }
+    } else {
+        return;
     }
 
     /* First probe and see if there's a arch-specific hook to create the
      * in-kernel irqchip for us */
-    ret = kvm_arch_irqchip_create(s);
-    if (ret < 0) {
-        return ret;
-    } else if (ret == 0) {
-        ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
-        if (ret < 0) {
-            fprintf(stderr, "Create kernel irqchip failed\n");
-            return ret;
+    ret = kvm_arch_irqchip_create(machine, s);
+    if (ret == 0) {
+        if (machine_kernel_irqchip_split(machine)) {
+            perror("Split IRQ chip mode not supported.");
+            exit(1);
+        } else {
+            ret = kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP);
         }
     }
+    if (ret < 0) {
+        fprintf(stderr, "Create kernel irqchip failed: %s\n", strerror(-ret));
+        exit(1);
+    }
 
     kvm_kernel_irqchip = true;
     /* If we have an in-kernel IRQ chip then we must have asynchronous
@@ -1405,7 +1512,7 @@
 
     kvm_init_irq_routing(s);
 
-    return 0;
+    s->gsimap = g_hash_table_new(g_direct_hash, g_direct_equal);
 }
 
 /* Find number of supported CPUs using the recommended
@@ -1424,6 +1531,18 @@
     return (ret) ? ret : kvm_recommended_vcpus(s);
 }
 
+static int kvm_max_vcpu_id(KVMState *s)
+{
+    int ret = kvm_check_extension(s, KVM_CAP_MAX_VCPU_ID);
+    return (ret) ? ret : kvm_max_vcpus(s);
+}
+
+bool kvm_vcpu_id_is_valid(int vcpu_id)
+{
+    KVMState *s = KVM_STATE(current_machine->accelerator);
+    return vcpu_id >= 0 && vcpu_id < kvm_max_vcpu_id(s);
+}
+
 static int kvm_init(MachineState *ms)
 {
     MachineClass *mc = MACHINE_GET_CLASS(ms);
@@ -1442,7 +1561,7 @@
     KVMState *s;
     const KVMCapabilityInfo *missing_cap;
     int ret;
-    int i, type = 0;
+    int type = 0;
     const char *kvm_type;
 
     s = KVM_STATE(ms->accelerator);
@@ -1454,13 +1573,13 @@
      * page size for the system though.
      */
     assert(TARGET_PAGE_SIZE <= getpagesize());
-    page_size_init();
 
     s->sigmask_len = 8;
 
 #ifdef KVM_CAP_SET_GUEST_DEBUG
     QTAILQ_INIT(&s->kvm_sw_breakpoints);
 #endif
+    QLIST_INIT(&s->kvm_parked_vcpus);
     s->vmfd = -1;
     s->fd = qemu_open("/dev/kvm", O_RDWR);
     if (s->fd == -1) {
@@ -1493,12 +1612,6 @@
         s->nr_slots = 32;
     }
 
-    s->slots = g_malloc0(s->nr_slots * sizeof(KVMSlot));
-
-    for (i = 0; i < s->nr_slots; i++) {
-        s->slots[i].slot = i;
-    }
-
     /* check the vcpu limits */
     soft_vcpus_limit = kvm_recommended_vcpus(s);
     hard_vcpus_limit = kvm_max_vcpus(s);
@@ -1538,8 +1651,17 @@
                 strerror(-ret));
 
 #ifdef TARGET_S390X
-        fprintf(stderr, "Please add the 'switch_amode' kernel parameter to "
-                        "your host kernel command line\n");
+        if (ret == -EINVAL) {
+            fprintf(stderr,
+                    "Host kernel setup problem detected. Please verify:\n");
+            fprintf(stderr, "- for kernels supporting the switch_amode or"
+                    " user_mode parameters, whether\n");
+            fprintf(stderr,
+                    "  user space is running in primary address space\n");
+            fprintf(stderr,
+                    "- for kernels supporting the vm.allocate_pgste sysctl, "
+                    "whether it is enabled\n");
+        }
 #endif
         goto err;
     }
@@ -1576,20 +1698,8 @@
     s->debugregs = kvm_check_extension(s, KVM_CAP_DEBUGREGS);
 #endif
 
-#ifdef KVM_CAP_XSAVE
-    s->xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
-#endif
-
-#ifdef KVM_CAP_XCRS
-    s->xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
-#endif
-
-#ifdef KVM_CAP_PIT_STATE2
-    s->pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
-#endif
-
 #ifdef KVM_CAP_IRQ_ROUTING
-    s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
+    kvm_direct_msi_allowed = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
 #endif
 
     s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
@@ -1607,19 +1717,40 @@
     kvm_eventfds_allowed =
         (kvm_check_extension(s, KVM_CAP_IOEVENTFD) > 0);
 
-    ret = kvm_arch_init(s);
+    kvm_irqfds_allowed =
+        (kvm_check_extension(s, KVM_CAP_IRQFD) > 0);
+
+    kvm_resamplefds_allowed =
+        (kvm_check_extension(s, KVM_CAP_IRQFD_RESAMPLE) > 0);
+
+    kvm_vm_attributes_allowed =
+        (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES) > 0);
+
+    kvm_ioeventfd_any_length_allowed =
+        (kvm_check_extension(s, KVM_CAP_IOEVENTFD_ANY_LENGTH) > 0);
+
+    ret = kvm_arch_init(ms, s);
     if (ret < 0) {
         goto err;
     }
 
-    ret = kvm_irqchip_create(s);
-    if (ret < 0) {
-        goto err;
+    if (machine_kernel_irqchip_allowed(ms)) {
+        kvm_irqchip_create(ms, s);
     }
 
     kvm_state = s;
-    memory_listener_register(&kvm_memory_listener, &address_space_memory);
-    memory_listener_register(&kvm_io_listener, &address_space_io);
+
+    if (kvm_eventfds_allowed) {
+        s->memory_listener.listener.eventfd_add = kvm_mem_ioeventfd_add;
+        s->memory_listener.listener.eventfd_del = kvm_mem_ioeventfd_del;
+    }
+    s->memory_listener.listener.coalesced_mmio_add = kvm_coalesce_mmio_region;
+    s->memory_listener.listener.coalesced_mmio_del = kvm_uncoalesce_mmio_region;
+
+    kvm_memory_listener_register(s, &s->memory_listener,
+                                 &address_space_memory, 0);
+    memory_listener_register(&kvm_io_listener,
+                             &address_space_io);
 
     s->many_ioeventfds = kvm_check_many_ioeventfds();
 
@@ -1635,7 +1766,7 @@
     if (s->fd != -1) {
         close(s->fd);
     }
-    g_free(s->slots);
+    g_free(s->memory_listener.slots);
 
     return ret;
 }
@@ -1645,14 +1776,15 @@
     s->sigmask_len = sigmask_len;
 }
 
-static void kvm_handle_io(uint16_t port, void *data, int direction, int size,
-                          uint32_t count)
+static void kvm_handle_io(uint16_t port, MemTxAttrs attrs, void *data, int direction,
+                          int size, uint32_t count)
 {
     int i;
     uint8_t *ptr = data;
 
     for (i = 0; i < count; i++) {
-        address_space_rw(&address_space_io, port, ptr, size,
+        address_space_rw(&address_space_io, port, attrs,
+                         ptr, size,
                          direction == KVM_EXIT_IO_OUT);
         ptr += size;
     }
@@ -1753,11 +1885,6 @@
     run_on_cpu(cpu, do_kvm_cpu_synchronize_post_init, cpu);
 }
 
-void kvm_cpu_clean_state(CPUState *cpu)
-{
-    cpu->kvm_vcpu_dirty = false;
-}
-
 int kvm_cpu_exec(CPUState *cpu)
 {
     struct kvm_run *run = cpu->kvm_run;
@@ -1770,7 +1897,11 @@
         return EXCP_HLT;
     }
 
+    qemu_mutex_unlock_iothread();
+
     do {
+        MemTxAttrs attrs;
+
         if (cpu->kvm_vcpu_dirty) {
             kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE);
             cpu->kvm_vcpu_dirty = false;
@@ -1786,12 +1917,10 @@
              */
             qemu_cpu_kick_self();
         }
-        qemu_mutex_unlock_iothread();
 
         run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0);
 
-        qemu_mutex_lock_iothread();
-        kvm_arch_post_run(cpu, run);
+        attrs = kvm_arch_post_run(cpu, run);
 
         if (run_ret < 0) {
             if (run_ret == -EINTR || run_ret == -EAGAIN) {
@@ -1801,6 +1930,14 @@
             }
             fprintf(stderr, "error: kvm run failed %s\n",
                     strerror(-run_ret));
+#ifdef TARGET_PPC
+            if (run_ret == -EBUSY) {
+                fprintf(stderr,
+                        "This is probably because your SMT is enabled.\n"
+                        "VCPU can only run on primary threads with all "
+                        "secondary threads offline.\n");
+            }
+#endif
             ret = -1;
             break;
         }
@@ -1809,7 +1946,8 @@
         switch (run->exit_reason) {
         case KVM_EXIT_IO:
             DPRINTF("handle_io\n");
-            kvm_handle_io(run->io.port,
+            /* Called outside BQL */
+            kvm_handle_io(run->io.port, attrs,
                           (uint8_t *)run + run->io.data_offset,
                           run->io.direction,
                           run->io.size,
@@ -1818,10 +1956,12 @@
             break;
         case KVM_EXIT_MMIO:
             DPRINTF("handle_mmio\n");
-            cpu_physical_memory_rw(run->mmio.phys_addr,
-                                   run->mmio.data,
-                                   run->mmio.len,
-                                   run->mmio.is_write);
+            /* Called outside BQL */
+            address_space_rw(&address_space_memory,
+                             run->mmio.phys_addr, attrs,
+                             run->mmio.data,
+                             run->mmio.len,
+                             run->mmio.is_write);
             ret = 0;
             break;
         case KVM_EXIT_IRQ_WINDOW_OPEN:
@@ -1851,6 +1991,12 @@
                 qemu_system_reset_request();
                 ret = EXCP_INTERRUPT;
                 break;
+            case KVM_SYSTEM_EVENT_CRASH:
+                qemu_mutex_lock_iothread();
+                qemu_system_guest_panicked();
+                qemu_mutex_unlock_iothread();
+                ret = 0;
+                break;
             default:
                 DPRINTF("kvm_arch_handle_exit\n");
                 ret = kvm_arch_handle_exit(cpu, run);
@@ -1864,6 +2010,8 @@
         }
     } while (ret == 0);
 
+    qemu_mutex_lock_iothread();
+
     if (ret < 0) {
         cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE);
         vm_stop(RUN_STATE_INTERNAL_ERROR);
@@ -1945,6 +2093,55 @@
     return ret;
 }
 
+int kvm_vm_check_attr(KVMState *s, uint32_t group, uint64_t attr)
+{
+    int ret;
+    struct kvm_device_attr attribute = {
+        .group = group,
+        .attr = attr,
+    };
+
+    if (!kvm_vm_attributes_allowed) {
+        return 0;
+    }
+
+    ret = kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attribute);
+    /* kvm returns 0 on success for HAS_DEVICE_ATTR */
+    return ret ? 0 : 1;
+}
+
+int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
+{
+    struct kvm_device_attr attribute = {
+        .group = group,
+        .attr = attr,
+        .flags = 0,
+    };
+
+    return kvm_device_ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute) ? 0 : 1;
+}
+
+void kvm_device_access(int fd, int group, uint64_t attr,
+                       void *val, bool write)
+{
+    struct kvm_device_attr kvmattr;
+    int err;
+
+    kvmattr.flags = 0;
+    kvmattr.group = group;
+    kvmattr.attr = attr;
+    kvmattr.addr = (uintptr_t)val;
+
+    err = kvm_device_ioctl(fd,
+                           write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
+                           &kvmattr);
+    if (err < 0) {
+        error_report("KVM_%s_DEVICE_ATTR failed: %s",
+                     write ? "SET" : "GET", strerror(-err));
+        qemu_abort("Group %d attr 0x%016" PRIx64 "\n", group, attr);
+    }
+}
+
 int kvm_has_sync_mmu(void)
 {
     return kvm_check_extension(kvm_state, KVM_CAP_SYNC_MMU);
@@ -1965,21 +2162,6 @@
     return kvm_state->debugregs;
 }
 
-int kvm_has_xsave(void)
-{
-    return kvm_state->xsave;
-}
-
-int kvm_has_xcrs(void)
-{
-    return kvm_state->xcrs;
-}
-
-int kvm_has_pit_state2(void)
-{
-    return kvm_state->pit_state2;
-}
-
 int kvm_has_many_ioeventfds(void)
 {
     if (!kvm_enabled()) {
@@ -2079,10 +2261,6 @@
         }
 
         bp = g_malloc(sizeof(struct kvm_sw_breakpoint));
-        if (!bp) {
-            return -ENOMEM;
-        }
-
         bp->pc = addr;
         bp->use_count = 1;
         err = kvm_arch_insert_sw_breakpoint(cpu, bp);
@@ -2247,6 +2425,21 @@
     return test ? 0 : create_dev.fd;
 }
 
+bool kvm_device_supported(int vmfd, uint64_t type)
+{
+    struct kvm_create_device create_dev = {
+        .type = type,
+        .fd = -1,
+        .flags = KVM_CREATE_DEVICE_TEST,
+    };
+
+    if (ioctl(vmfd, KVM_CHECK_EXTENSION, KVM_CAP_DEVICE_CTRL) <= 0) {
+        return false;
+    }
+
+    return (ioctl(vmfd, KVM_CREATE_DEVICE, &create_dev) >= 0);
+}
+
 int kvm_set_one_reg(CPUState *cs, uint64_t id, void *source)
 {
     struct kvm_one_reg reg;
@@ -2256,7 +2449,7 @@
     reg.addr = (uintptr_t) source;
     r = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
     if (r) {
-        trace_kvm_failed_reg_set(id, strerror(r));
+        trace_kvm_failed_reg_set(id, strerror(-r));
     }
     return r;
 }
@@ -2270,7 +2463,7 @@
     reg.addr = (uintptr_t) target;
     r = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
     if (r) {
-        trace_kvm_failed_reg_get(id, strerror(r));
+        trace_kvm_failed_reg_get(id, strerror(-r));
     }
     return r;
 }
diff --git a/kvm-stub.c b/kvm-stub.c
index 7ba90c5..64e23f6 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -10,8 +10,8 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "hw/hw.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
 
@@ -24,11 +24,18 @@
 bool kvm_async_interrupts_allowed;
 bool kvm_eventfds_allowed;
 bool kvm_irqfds_allowed;
+bool kvm_resamplefds_allowed;
 bool kvm_msi_via_irqfd_allowed;
 bool kvm_gsi_routing_allowed;
 bool kvm_gsi_direct_mapping;
 bool kvm_allowed;
 bool kvm_readonly_mem_allowed;
+bool kvm_ioeventfd_any_length_allowed;
+
+int kvm_destroy_vcpu(CPUState *cpu)
+{
+    return -ENOSYS;
+}
 
 int kvm_init_vcpu(CPUState *cpu)
 {
@@ -66,11 +73,6 @@
     return 0;
 }
 
-int kvm_has_pit_state2(void)
-{
-    return 0;
-}
-
 void kvm_setup_guest_memory(void *start, size_t size)
 {
 }
@@ -114,7 +116,7 @@
 }
 
 #ifndef CONFIG_USER_ONLY
-int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
+int kvm_irqchip_add_msi_route(KVMState *s, int vector, PCIDevice *dev)
 {
     return -ENOSYS;
 }
@@ -127,23 +129,29 @@
 {
 }
 
-int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg,
+                                 PCIDevice *dev)
 {
     return -ENOSYS;
 }
 
+void kvm_irqchip_commit_routes(KVMState *s)
+{
+}
+
 int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
 {
     return -ENOSYS;
 }
 
-int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
-                                   EventNotifier *rn, int virq)
+int kvm_irqchip_add_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
+                                       EventNotifier *rn, int virq)
 {
     return -ENOSYS;
 }
 
-int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_remove_irqfd_notifier_gsi(KVMState *s, EventNotifier *n,
+                                          int virq)
 {
     return -ENOSYS;
 }
diff --git a/libcacard/Makefile b/libcacard/Makefile
deleted file mode 100644
index 0e7903f..0000000
--- a/libcacard/Makefile
+++ /dev/null
@@ -1,43 +0,0 @@
-libcacard_includedir=$(includedir)/cacard
-
-TOOLS += vscclient$(EXESUF)
-
-# objects linked into a shared library, built with libtool with -fPIC if required
-libcacard-obj-y = $(libcacard-y)
-libcacard-lobj-y=$(patsubst %.o,%.lo,$(libcacard-obj-y))
-
-# libtool will build the .o files, too
-$(libcacard-obj-y): | $(libcacard-lobj-y)
-
-all: libcacard.la libcacard.pc
-
-vscclient$(EXESUF): libcacard/vscclient.o libcacard.la
-	$(call LINK,$^)
-
-#########################################################################
-# Rules for building libcacard standalone library
-
-libcacard.la: LDFLAGS += -rpath $(libdir) -no-undefined \
-	-export-symbols $(SRC_PATH)/libcacard/libcacard.syms
-libcacard.la: $(libcacard-lobj-y)
-	$(call LINK,$^)
-
-libcacard.pc: $(SRC_PATH)/libcacard/libcacard.pc.in
-	$(call quiet-command,sed -e 's|@LIBDIR@|$(libdir)|' \
-		-e 's|@INCLUDEDIR@|$(libcacard_includedir)|' \
-	    -e 's|@VERSION@|$(shell cat $(SRC_PATH)/VERSION)|' \
-		-e 's|@PREFIX@|$(prefix)|' $< > libcacard.pc,\
-	"  GEN   $@")
-
-.PHONY: install-libcacard
-
-install: install-libcacard
-install-libcacard: libcacard.pc libcacard.la
-	$(INSTALL_DIR) "$(DESTDIR)$(libdir)"
-	$(INSTALL_DIR) "$(DESTDIR)$(libdir)/pkgconfig"
-	$(INSTALL_DIR) "$(DESTDIR)$(libcacard_includedir)"
-	$(INSTALL_LIB) libcacard.la "$(DESTDIR)$(libdir)"
-	$(INSTALL_DATA) libcacard.pc "$(DESTDIR)$(libdir)/pkgconfig"
-	for inc in $(SRC_PATH)/libcacard/*.h; do \
-		$(INSTALL_DATA) $$inc "$(DESTDIR)$(libcacard_includedir)"; \
-	done
diff --git a/libcacard/cac.c b/libcacard/cac.c
deleted file mode 100644
index f38fdce..0000000
--- a/libcacard/cac.c
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * implement the applets for the CAC card.
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "qemu-common.h"
-
-#include "cac.h"
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-
-/* private data for PKI applets */
-typedef struct CACPKIAppletDataStruct {
-    unsigned char *cert;
-    int cert_len;
-    unsigned char *cert_buffer;
-    int cert_buffer_len;
-    unsigned char *sign_buffer;
-    int sign_buffer_len;
-    VCardKey *key;
-} CACPKIAppletData;
-
-/*
- * CAC applet private data
- */
-struct VCardAppletPrivateStruct {
-    union {
-        CACPKIAppletData pki_data;
-        void *reserved;
-    } u;
-};
-
-/*
- * handle all the APDU's that are common to all CAC applets
- */
-static VCardStatus
-cac_common_process_apdu(VCard *card, VCardAPDU *apdu, VCardResponse **response)
-{
-    int ef;
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case VCARD7816_INS_SELECT_FILE:
-        if (apdu->a_p1 != 0x02) {
-            /* let the 7816 code handle applet switches */
-            ret = VCARD_NEXT;
-            break;
-        }
-        /* handle file id setting */
-        if (apdu->a_Lc != 2) {
-            *response = vcard_make_response(
-                VCARD7816_STATUS_ERROR_DATA_INVALID);
-            ret = VCARD_DONE;
-            break;
-        }
-        /* CAC 1.0 only supports ef = 0 */
-        ef = apdu->a_body[0] | (apdu->a_body[1] << 8);
-        if (ef != 0) {
-            *response = vcard_make_response(
-                VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-            ret = VCARD_DONE;
-            break;
-        }
-        *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
-        ret = VCARD_DONE;
-        break;
-    case VCARD7816_INS_GET_RESPONSE:
-    case VCARD7816_INS_VERIFY:
-        /* let the 7816 code handle these */
-        ret = VCARD_NEXT;
-        break;
-    case CAC_GET_PROPERTIES:
-    case CAC_GET_ACR:
-        /* skip these for now, this will probably be needed */
-        *response = vcard_make_response(VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-        ret = VCARD_DONE;
-        break;
-    default:
-        *response = vcard_make_response(
-            VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    }
-    return ret;
-}
-
-/*
- *  reset the inter call state between applet selects
- */
-static VCardStatus
-cac_applet_pki_reset(VCard *card, int channel)
-{
-    VCardAppletPrivate *applet_private;
-    CACPKIAppletData *pki_applet;
-    applet_private = vcard_get_current_applet_private(card, channel);
-    assert(applet_private);
-    pki_applet = &(applet_private->u.pki_data);
-
-    pki_applet->cert_buffer = NULL;
-    g_free(pki_applet->sign_buffer);
-    pki_applet->sign_buffer = NULL;
-    pki_applet->cert_buffer_len = 0;
-    pki_applet->sign_buffer_len = 0;
-    return VCARD_DONE;
-}
-
-static VCardStatus
-cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
-                            VCardResponse **response)
-{
-    CACPKIAppletData *pki_applet;
-    VCardAppletPrivate *applet_private;
-    int size, next;
-    unsigned char *sign_buffer;
-    bool retain_sign_buffer = FALSE;
-    vcard_7816_status_t status;
-    VCardStatus ret = VCARD_FAIL;
-
-    applet_private = vcard_get_current_applet_private(card, apdu->a_channel);
-    assert(applet_private);
-    pki_applet = &(applet_private->u.pki_data);
-
-    switch (apdu->a_ins) {
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-            VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-        ret = VCARD_DONE;
-        break;
-    case CAC_GET_CERTIFICATE:
-        if ((apdu->a_p2 != 0) || (apdu->a_p1 != 0)) {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        assert(pki_applet->cert != NULL);
-        size = apdu->a_Le;
-        if (pki_applet->cert_buffer == NULL) {
-            pki_applet->cert_buffer = pki_applet->cert;
-            pki_applet->cert_buffer_len = pki_applet->cert_len;
-        }
-        size = MIN(size, pki_applet->cert_buffer_len);
-        next = MIN(255, pki_applet->cert_buffer_len - size);
-        *response = vcard_response_new_bytes(
-                        card, pki_applet->cert_buffer, size,
-                        apdu->a_Le, next ?
-                        VCARD7816_SW1_WARNING_CHANGE :
-                        VCARD7816_SW1_SUCCESS,
-                        next);
-        pki_applet->cert_buffer += size;
-        pki_applet->cert_buffer_len -= size;
-        if ((*response == NULL) || (next == 0)) {
-            pki_applet->cert_buffer = NULL;
-        }
-        if (*response == NULL) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        ret = VCARD_DONE;
-        break;
-    case CAC_SIGN_DECRYPT:
-        if (apdu->a_p2 != 0) {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        size = apdu->a_Lc;
-
-        sign_buffer = g_realloc(pki_applet->sign_buffer,
-                                pki_applet->sign_buffer_len + size);
-        memcpy(sign_buffer+pki_applet->sign_buffer_len, apdu->a_body, size);
-        size += pki_applet->sign_buffer_len;
-        switch (apdu->a_p1) {
-        case  0x80:
-            /* p1 == 0x80 means we haven't yet sent the whole buffer, wait for
-             * the rest */
-            pki_applet->sign_buffer = sign_buffer;
-            pki_applet->sign_buffer_len = size;
-            *response = vcard_make_response(VCARD7816_STATUS_SUCCESS);
-            retain_sign_buffer = TRUE;
-            break;
-        case 0x00:
-            /* we now have the whole buffer, do the operation, result will be
-             * in the sign_buffer */
-            status = vcard_emul_rsa_op(card, pki_applet->key,
-                                       sign_buffer, size);
-            if (status != VCARD7816_STATUS_SUCCESS) {
-                *response = vcard_make_response(status);
-                break;
-            }
-            *response = vcard_response_new(card, sign_buffer, size, apdu->a_Le,
-                                                     VCARD7816_STATUS_SUCCESS);
-            if (*response == NULL) {
-                *response = vcard_make_response(
-                                VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-            }
-            break;
-        default:
-           *response = vcard_make_response(
-                                VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-            break;
-        }
-        if (!retain_sign_buffer) {
-            g_free(sign_buffer);
-            pki_applet->sign_buffer = NULL;
-            pki_applet->sign_buffer_len = 0;
-        }
-        ret = VCARD_DONE;
-        break;
-    case CAC_READ_BUFFER:
-        /* new CAC call, go ahead and use the old version for now */
-        /* TODO: implement */
-        *response = vcard_make_response(
-                                VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-
-static VCardStatus
-cac_applet_id_process_apdu(VCard *card, VCardAPDU *apdu,
-                           VCardResponse **response)
-{
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-        ret = VCARD_DONE;
-        break;
-    case CAC_READ_BUFFER:
-        /* new CAC call, go ahead and use the old version for now */
-        /* TODO: implement */
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-
-/*
- * TODO: if we ever want to support general CAC middleware, we will need to
- * implement the various containers.
- */
-static VCardStatus
-cac_applet_container_process_apdu(VCard *card, VCardAPDU *apdu,
-                                  VCardResponse **response)
-{
-    VCardStatus ret = VCARD_FAIL;
-
-    switch (apdu->a_ins) {
-    case CAC_READ_BUFFER:
-    case CAC_UPDATE_BUFFER:
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        ret = VCARD_DONE;
-        break;
-    default:
-        ret = cac_common_process_apdu(card, apdu, response);
-        break;
-    }
-    return ret;
-}
-
-/*
- * utilities for creating and destroying the private applet data
- */
-static void
-cac_delete_pki_applet_private(VCardAppletPrivate *applet_private)
-{
-    CACPKIAppletData *pki_applet_data;
-
-    if (applet_private == NULL) {
-        return;
-    }
-    pki_applet_data = &(applet_private->u.pki_data);
-    g_free(pki_applet_data->cert);
-    g_free(pki_applet_data->sign_buffer);
-    if (pki_applet_data->key != NULL) {
-        vcard_emul_delete_key(pki_applet_data->key);
-    }
-    g_free(applet_private);
-}
-
-static VCardAppletPrivate *
-cac_new_pki_applet_private(const unsigned char *cert,
-                           int cert_len, VCardKey *key)
-{
-    CACPKIAppletData *pki_applet_data;
-    VCardAppletPrivate *applet_private;
-
-    applet_private = g_new0(VCardAppletPrivate, 1);
-    pki_applet_data = &(applet_private->u.pki_data);
-    pki_applet_data->cert = (unsigned char *)g_malloc(cert_len+1);
-    /*
-     * if we want to support compression, then we simply change the 0 to a 1
-     * and compress the cert data with libz
-     */
-    pki_applet_data->cert[0] = 0; /* not compressed */
-    memcpy(&pki_applet_data->cert[1], cert, cert_len);
-    pki_applet_data->cert_len = cert_len+1;
-
-    pki_applet_data->key = key;
-    return applet_private;
-}
-
-
-/*
- * create a new cac applet which links to a given cert
- */
-static VCardApplet *
-cac_new_pki_applet(int i, const unsigned char *cert,
-                   int cert_len, VCardKey *key)
-{
-    VCardAppletPrivate *applet_private;
-    VCardApplet *applet;
-    unsigned char pki_aid[] = { 0xa0, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00 };
-    int pki_aid_len = sizeof(pki_aid);
-
-    pki_aid[pki_aid_len-1] = i;
-
-    applet_private = cac_new_pki_applet_private(cert, cert_len, key);
-    if (applet_private == NULL) {
-        goto failure;
-    }
-    applet = vcard_new_applet(cac_applet_pki_process_apdu, cac_applet_pki_reset,
-                              pki_aid, pki_aid_len);
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_set_applet_private(applet, applet_private,
-                             cac_delete_pki_applet_private);
-    applet_private = NULL;
-
-    return applet;
-
-failure:
-    if (applet_private != NULL) {
-        cac_delete_pki_applet_private(applet_private);
-    }
-    return NULL;
-}
-
-
-static unsigned char cac_default_container_aid[] = {
-    0xa0, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00 };
-static unsigned char cac_id_aid[] = {
-    0xa0, 0x00, 0x00, 0x00, 0x79, 0x03, 0x00 };
-/*
- * Initialize the cac card. This is the only public function in this file. All
- * the rest are connected through function pointers.
- */
-VCardStatus
-cac_card_init(VReader *reader, VCard *card,
-              const char *params,
-              unsigned char * const *cert,
-              int cert_len[],
-              VCardKey *key[] /* adopt the keys*/,
-              int cert_count)
-{
-    int i;
-    VCardApplet *applet;
-
-    /* CAC Cards are VM Cards */
-    vcard_set_type(card, VCARD_VM);
-
-    /* create one PKI applet for each cert */
-    for (i = 0; i < cert_count; i++) {
-        applet = cac_new_pki_applet(i, cert[i], cert_len[i], key[i]);
-        if (applet == NULL) {
-            goto failure;
-        }
-        vcard_add_applet(card, applet);
-    }
-
-    /* create a default blank container applet */
-    applet = vcard_new_applet(cac_applet_container_process_apdu,
-                              NULL, cac_default_container_aid,
-                              sizeof(cac_default_container_aid));
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_add_applet(card, applet);
-
-    /* create a default blank container applet */
-    applet = vcard_new_applet(cac_applet_id_process_apdu,
-                              NULL, cac_id_aid,
-                              sizeof(cac_id_aid));
-    if (applet == NULL) {
-        goto failure;
-    }
-    vcard_add_applet(card, applet);
-    return VCARD_DONE;
-
-failure:
-    return VCARD_FAIL;
-}
-
diff --git a/libcacard/cac.h b/libcacard/cac.h
deleted file mode 100644
index d24a2a8..0000000
--- a/libcacard/cac.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * defines the entry point for the cac card. Only used by cac.c anc
- * vcard_emul_type.c
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CAC_H
-#define CAC_H 1
-#include "vcard.h"
-#include "vreader.h"
-
-#define CAC_GET_PROPERTIES  0x56
-#define CAC_GET_ACR         0x4c
-#define CAC_READ_BUFFER     0x52
-#define CAC_UPDATE_BUFFER   0x58
-#define CAC_SIGN_DECRYPT    0x42
-#define CAC_GET_CERTIFICATE 0x36
-
-/*
- * Initialize the cac card. This is the only public function in this file. All
- * the rest are connected through function pointers.
- */
-VCardStatus cac_card_init(VReader *reader, VCard *card, const char *params,
-              unsigned char * const *cert, int cert_len[],
-              VCardKey *key[] /* adopt the keys*/,
-              int cert_count);
-
-/* not yet implemented */
-VCardStatus cac_is_cac_card(VReader *reader);
-#endif
diff --git a/libcacard/card_7816.c b/libcacard/card_7816.c
deleted file mode 100644
index 814fa16..0000000
--- a/libcacard/card_7816.c
+++ /dev/null
@@ -1,755 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "qemu-common.h"
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-
-/*
- * set the status bytes based on the status word
- */
-static void
-vcard_response_set_status(VCardResponse *response, vcard_7816_status_t status)
-{
-    unsigned char sw1, sw2;
-    response->b_status = status; /* make sure the status and swX representations
-                                  * are consistent */
-    sw1 = (status >> 8) & 0xff;
-    sw2 = status & 0xff;
-    response->b_sw1 = sw1;
-    response->b_sw2 = sw2;
-    response->b_data[response->b_len] = sw1;
-    response->b_data[response->b_len+1] = sw2;
-}
-
-/*
- * set the status bytes in a response buffer
- */
-static void
-vcard_response_set_status_bytes(VCardResponse *response,
-                               unsigned char sw1, unsigned char sw2)
-{
-    response->b_status = sw1 << 8 | sw2;
-    response->b_sw1 = sw1;
-    response->b_sw2 = sw2;
-    response->b_data[response->b_len] = sw1;
-    response->b_data[response->b_len+1] = sw2;
-}
-
-/*
- * allocate a VCardResponse structure, plus space for the data buffer, and
- * set up everything but the resonse bytes.
- */
-VCardResponse *
-vcard_response_new_data(unsigned char *buf, int len)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = g_malloc(len + 2);
-    memcpy(new_response->b_data, buf, len);
-    new_response->b_total_len = len+2;
-    new_response->b_len = len;
-    new_response->b_type = VCARD_MALLOC;
-    return new_response;
-}
-
-static VCardResponse *
-vcard_init_buffer_response(VCard *card, unsigned char *buf, int len)
-{
-    VCardResponse *response;
-    VCardBufferResponse *buffer_response;
-
-    buffer_response = vcard_get_buffer_response(card);
-    if (buffer_response) {
-        vcard_set_buffer_response(card, NULL);
-        vcard_buffer_response_delete(buffer_response);
-    }
-    buffer_response = vcard_buffer_response_new(buf, len);
-    if (buffer_response == NULL) {
-        return NULL;
-    }
-    response = vcard_response_new_status_bytes(VCARD7816_SW1_RESPONSE_BYTES,
-                                               len > 255 ? 0 : len);
-    if (response == NULL) {
-        return NULL;
-    }
-    vcard_set_buffer_response(card, buffer_response);
-    return response;
-}
-
-/*
- * general buffer to hold results from APDU calls
- */
-VCardResponse *
-vcard_response_new(VCard *card, unsigned char *buf,
-                   int len, int Le, vcard_7816_status_t status)
-{
-    VCardResponse *new_response;
-
-    if (len > Le) {
-        return vcard_init_buffer_response(card, buf, len);
-    }
-    new_response = vcard_response_new_data(buf, len);
-    if (new_response == NULL) {
-        return NULL;
-    }
-    vcard_response_set_status(new_response, status);
-    return new_response;
-}
-
-/*
- * general buffer to hold results from APDU calls
- */
-VCardResponse *
-vcard_response_new_bytes(VCard *card, unsigned char *buf, int len, int Le,
-                         unsigned char sw1, unsigned char sw2)
-{
-    VCardResponse *new_response;
-
-    if (len > Le) {
-        return vcard_init_buffer_response(card, buf, len);
-    }
-    new_response = vcard_response_new_data(buf, len);
-    if (new_response == NULL) {
-        return NULL;
-    }
-    vcard_response_set_status_bytes(new_response, sw1, sw2);
-    return new_response;
-}
-
-/*
- * get a new Response buffer that only has a status.
- */
-static VCardResponse *
-vcard_response_new_status(vcard_7816_status_t status)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = &new_response->b_sw1;
-    new_response->b_len = 0;
-    new_response->b_total_len = 2;
-    new_response->b_type = VCARD_MALLOC_STRUCT;
-    vcard_response_set_status(new_response, status);
-    return new_response;
-}
-
-/*
- * same as above, but specify the status as separate bytes
- */
-VCardResponse *
-vcard_response_new_status_bytes(unsigned char sw1, unsigned char sw2)
-{
-    VCardResponse *new_response;
-
-    new_response = g_new(VCardResponse, 1);
-    new_response->b_data = &new_response->b_sw1;
-    new_response->b_len = 0;
-    new_response->b_total_len = 2;
-    new_response->b_type = VCARD_MALLOC_STRUCT;
-    vcard_response_set_status_bytes(new_response, sw1, sw2);
-    return new_response;
-}
-
-
-/*
- * free the response buffer. The Buffer has a type to handle the buffer
- * allocated in other ways than through malloc.
- */
-void
-vcard_response_delete(VCardResponse *response)
-{
-    if (response == NULL) {
-        return;
-    }
-    switch (response->b_type) {
-    case VCARD_MALLOC:
-        /* everything was malloc'ed */
-        g_free(response->b_data);
-        g_free(response);
-        break;
-    case VCARD_MALLOC_DATA:
-        /* only the data buffer was malloc'ed */
-        g_free(response->b_data);
-        break;
-    case VCARD_MALLOC_STRUCT:
-        /* only the structure was malloc'ed */
-        g_free(response);
-        break;
-    case VCARD_STATIC:
-        break;
-    }
-}
-
-/*
- * decode the class bit and set our generic type field, channel, and
- * secure messaging values.
- */
-static vcard_7816_status_t
-vcard_apdu_set_class(VCardAPDU *apdu) {
-    apdu->a_channel = 0;
-    apdu->a_secure_messaging = 0;
-    apdu->a_type = apdu->a_cla & 0xf0;
-    apdu->a_gen_type = VCARD_7816_ISO;
-
-    /* parse the class  tables 8 & 9 of the 7816-4 Part 4 spec */
-    switch (apdu->a_type) {
-        /* we only support the basic types */
-    case 0x00:
-    case 0x80:
-    case 0x90:
-    case 0xa0:
-        apdu->a_channel = apdu->a_cla & 3;
-        apdu->a_secure_messaging = apdu->a_cla & 0xe;
-        break;
-    case 0xb0:
-    case 0xc0:
-        break;
-
-    case 0x10:
-    case 0x20:
-    case 0x30:
-    case 0x40:
-    case 0x50:
-    case 0x60:
-    case 0x70:
-        /* Reserved for future use */
-        apdu->a_gen_type = VCARD_7816_RFU;
-        break;
-    case 0xd0:
-    case 0xe0:
-    case 0xf0:
-    default:
-        apdu->a_gen_type =
-            (apdu->a_cla == 0xff) ? VCARD_7816_PTS : VCARD_7816_PROPRIETARY;
-        break;
-    }
-    return VCARD7816_STATUS_SUCCESS;
-}
-
-/*
- * set the Le and Lc fields according to table 5 of the
- * 7816-4 part 4 spec
- */
-static vcard_7816_status_t
-vcard_apdu_set_length(VCardAPDU *apdu)
-{
-    int L, Le;
-
-    /* process according to table 5 of the 7816-4 Part 4 spec.
-     * variable names match the variables in the spec */
-    L = apdu->a_len-4; /* fixed APDU header */
-    apdu->a_Lc = 0;
-    apdu->a_Le = 0;
-    apdu->a_body = NULL;
-    switch (L) {
-    case 0:
-        /* 1 minimal apdu */
-        return VCARD7816_STATUS_SUCCESS;
-    case 1:
-        /* 2S only return values apdu */
-        /*   zero maps to 256 here */
-        apdu->a_Le = apdu->a_header->ah_Le ?
-                         apdu->a_header->ah_Le : 256;
-        return VCARD7816_STATUS_SUCCESS;
-    default:
-        /* if the ah_Le byte is zero and we have more than
-         * 1 byte in the header, then we must be using extended Le and Lc.
-         * process the extended now. */
-        if (apdu->a_header->ah_Le == 0) {
-            if (L < 3) {
-                /* coding error, need at least 3 bytes */
-                return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-            }
-            /* calculate the first extended value. Could be either Le or Lc */
-            Le = (apdu->a_header->ah_body[0] << 8)
-               || apdu->a_header->ah_body[1];
-            if (L == 3) {
-                /* 2E extended, return data only */
-                /*   zero maps to 65536 */
-                apdu->a_Le = Le ? Le : 65536;
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            if (Le == 0) {
-                /* reserved for future use, probably for next time we need
-                 * to extend the lengths */
-                return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-            }
-            /* we know that the first extended value is Lc now */
-            apdu->a_Lc = Le;
-            apdu->a_body = &apdu->a_header->ah_body[2];
-            if (L == Le+3) {
-                /* 3E extended, only body parameters */
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            if (L == Le+5) {
-                /* 4E extended, parameters and return data */
-                Le = (apdu->a_data[apdu->a_len-2] << 8)
-                   || apdu->a_data[apdu->a_len-1];
-                apdu->a_Le = Le ? Le : 65536;
-                return VCARD7816_STATUS_SUCCESS;
-            }
-            return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-        }
-        /* not extended */
-        apdu->a_Lc = apdu->a_header->ah_Le;
-        apdu->a_body = &apdu->a_header->ah_body[0];
-        if (L ==  apdu->a_Lc + 1) {
-            /* 3S only body parameters */
-            return VCARD7816_STATUS_SUCCESS;
-        }
-        if (L ==  apdu->a_Lc + 2) {
-            /* 4S parameters and return data */
-            Le = apdu->a_data[apdu->a_len-1];
-            apdu->a_Le = Le ?  Le : 256;
-            return VCARD7816_STATUS_SUCCESS;
-        }
-        break;
-    }
-    return VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-}
-
-/*
- * create a new APDU from a raw set of bytes. This will decode all the
- * above fields. users of VCARDAPDU's can then depend on the already decoded
- * values.
- */
-VCardAPDU *
-vcard_apdu_new(unsigned char *raw_apdu, int len, vcard_7816_status_t *status)
-{
-    VCardAPDU *new_apdu;
-
-    *status = VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE;
-    if (len < 4) {
-        *status = VCARD7816_STATUS_ERROR_WRONG_LENGTH;
-        return NULL;
-    }
-
-    new_apdu = g_new(VCardAPDU, 1);
-    new_apdu->a_data = g_memdup(raw_apdu, len);
-    new_apdu->a_len = len;
-    *status = vcard_apdu_set_class(new_apdu);
-    if (*status != VCARD7816_STATUS_SUCCESS) {
-        g_free(new_apdu);
-        return NULL;
-    }
-    *status = vcard_apdu_set_length(new_apdu);
-    if (*status != VCARD7816_STATUS_SUCCESS) {
-        g_free(new_apdu);
-        new_apdu = NULL;
-    }
-    return new_apdu;
-}
-
-void
-vcard_apdu_delete(VCardAPDU *apdu)
-{
-    if (apdu == NULL) {
-        return;
-    }
-    g_free(apdu->a_data);
-    g_free(apdu);
-}
-
-
-/*
- * declare response buffers for all the 7816 defined error codes
- */
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_SUCCESS)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_RET_CORUPT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_CHANGE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_WARNING_FILE_FILLED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR_CHANGE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_LENGTH)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(
-                    VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_NO_EF)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_PARAMETERS)
-VCARD_RESPONSE_NEW_STATIC_STATUS(
-                            VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_FILE_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_P1_P2_INCORRECT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_DATA_NOT_FOUND)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_INS_CODE_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_CLA_INVALID)
-VCARD_RESPONSE_NEW_STATIC_STATUS(VCARD7816_STATUS_ERROR_GENERAL)
-
-/*
- * return a single response code. This function cannot fail. It will always
- * return a response.
- */
-VCardResponse *
-vcard_make_response(vcard_7816_status_t status)
-{
-    VCardResponse *response;
-
-    switch (status) {
-    /* known 7816 response codes */
-    case VCARD7816_STATUS_SUCCESS:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_SUCCESS);
-    case VCARD7816_STATUS_WARNING:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING);
-    case VCARD7816_STATUS_WARNING_RET_CORUPT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_RET_CORUPT);
-    case VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE);
-    case VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED);
-    case VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID);
-    case VCARD7816_STATUS_WARNING_CHANGE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_CHANGE);
-    case VCARD7816_STATUS_WARNING_FILE_FILLED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_WARNING_FILE_FILLED);
-    case VCARD7816_STATUS_EXC_ERROR:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR);
-    case VCARD7816_STATUS_EXC_ERROR_CHANGE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR_CHANGE);
-    case VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-    case VCARD7816_STATUS_ERROR_WRONG_LENGTH:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_LENGTH);
-    case VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE);
-    case VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED);
-    case VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED);
-    case VCARD7816_STATUS_ERROR_DATA_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_INVALID);
-    case VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED);
-    case VCARD7816_STATUS_ERROR_DATA_NO_EF:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_NO_EF);
-    case VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING);
-    case VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA);
-    case VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED);
-    case VCARD7816_STATUS_ERROR_FILE_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE);
-    case VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT);
-    case VCARD7816_STATUS_ERROR_P1_P2_INCORRECT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_P1_P2_INCORRECT);
-    case VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT);
-    case VCARD7816_STATUS_ERROR_DATA_NOT_FOUND:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-    case VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2);
-    case VCARD7816_STATUS_ERROR_INS_CODE_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_INS_CODE_INVALID);
-    case VCARD7816_STATUS_ERROR_CLA_INVALID:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_CLA_INVALID);
-    case VCARD7816_STATUS_ERROR_GENERAL:
-        return VCARD_RESPONSE_GET_STATIC(
-                    VCARD7816_STATUS_ERROR_GENERAL);
-    default:
-        /* we don't know this status code, create a response buffer to
-         * hold it */
-        response = vcard_response_new_status(status);
-        if (response == NULL) {
-            /* couldn't allocate the buffer, return memmory error */
-            return VCARD_RESPONSE_GET_STATIC(
-                        VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        return response;
-    }
-}
-
-/*
- * Add File card support here if you need it.
- */
-static VCardStatus
-vcard7816_file_system_process_apdu(VCard *card, VCardAPDU *apdu,
-                                   VCardResponse **response)
-{
-    /* TODO: if we want to support a virtual file system card, we do it here.
-     * It would probably be a pkcs #15 card type */
-    *response = vcard_make_response(
-                    VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    return VCARD_DONE;
-}
-
-/*
- * VM card (including java cards)
- */
-static VCardStatus
-vcard7816_vm_process_apdu(VCard *card, VCardAPDU *apdu,
-                          VCardResponse **response)
-{
-    int bytes_to_copy, next_byte_count, count;
-    VCardApplet *current_applet;
-    VCardBufferResponse *buffer_response;
-    vcard_7816_status_t status;
-
-    /* parse the class first */
-    if (apdu->a_gen_type !=  VCARD_7816_ISO) {
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        return VCARD_DONE;
-    }
-
-    /* use a switch so that if we need to support secure channel stuff later,
-     * we know where to put it */
-    switch (apdu->a_secure_messaging) {
-    case 0x0: /* no SM */
-        break;
-    case 0x4: /* proprietary SM */
-    case 0x8: /* header not authenticated */
-    case 0xc: /* header authenticated */
-    default:
-        /* for now, don't try to support secure channel stuff in the
-         * virtual card. */
-        *response = vcard_make_response(
-                        VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED);
-        return VCARD_DONE;
-    }
-
-    /* now parse the instruction */
-    switch (apdu->a_ins) {
-    case  VCARD7816_INS_MANAGE_CHANNEL: /* secure channel op */
-    case  VCARD7816_INS_EXTERNAL_AUTHENTICATE: /* secure channel op */
-    case  VCARD7816_INS_GET_CHALLENGE: /* secure channel op */
-    case  VCARD7816_INS_INTERNAL_AUTHENTICATE: /* secure channel op */
-    case  VCARD7816_INS_ERASE_BINARY: /* applet control op */
-    case  VCARD7816_INS_READ_BINARY: /* applet control op */
-    case  VCARD7816_INS_WRITE_BINARY: /* applet control op */
-    case  VCARD7816_INS_UPDATE_BINARY: /* applet control op */
-    case  VCARD7816_INS_READ_RECORD: /* file op */
-    case  VCARD7816_INS_WRITE_RECORD: /* file op */
-    case  VCARD7816_INS_UPDATE_RECORD: /* file op */
-    case  VCARD7816_INS_APPEND_RECORD: /* file op */
-    case  VCARD7816_INS_ENVELOPE:
-    case  VCARD7816_INS_PUT_DATA:
-        *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-
-    case  VCARD7816_INS_SELECT_FILE:
-        if (apdu->a_p1 != 0x04) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED);
-            break;
-        }
-
-        /* side effect, deselect the current applet if no applet has been found
-         * */
-        current_applet = vcard_find_applet(card, apdu->a_body, apdu->a_Lc);
-        vcard_select_applet(card, apdu->a_channel, current_applet);
-        if (current_applet) {
-            unsigned char *aid;
-            int aid_len;
-            aid = vcard_applet_get_aid(current_applet, &aid_len);
-            *response = vcard_response_new(card, aid, aid_len, apdu->a_Le,
-                                          VCARD7816_STATUS_SUCCESS);
-        } else {
-            *response = vcard_make_response(
-                             VCARD7816_STATUS_ERROR_FILE_NOT_FOUND);
-        }
-        break;
-
-    case  VCARD7816_INS_VERIFY:
-        if ((apdu->a_p1 != 0x00) || (apdu->a_p2 != 0x00)) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_WRONG_PARAMETERS);
-        } else {
-            if (apdu->a_Lc == 0) {
-                /* handle pin count if possible */
-                count = vcard_emul_get_login_count(card);
-                if (count < 0) {
-                    *response = vcard_make_response(
-                                    VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-                } else {
-                    if (count > 0xf) {
-                        count = 0xf;
-                    }
-                    *response = vcard_response_new_status_bytes(
-                                                VCARD7816_SW1_WARNING_CHANGE,
-                                                                0xc0 | count);
-                    if (*response == NULL) {
-                        *response = vcard_make_response(
-                                    VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-                    }
-                }
-            } else {
-                    status = vcard_emul_login(card, apdu->a_body, apdu->a_Lc);
-                *response = vcard_make_response(status);
-            }
-        }
-        break;
-
-    case VCARD7816_INS_GET_RESPONSE:
-        buffer_response = vcard_get_buffer_response(card);
-        if (!buffer_response) {
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_ERROR_DATA_NOT_FOUND);
-            /* handle error */
-            break;
-        }
-        bytes_to_copy = MIN(buffer_response->len, apdu->a_Le);
-        next_byte_count = MIN(256, buffer_response->len - bytes_to_copy);
-        *response = vcard_response_new_bytes(
-                        card, buffer_response->current, bytes_to_copy,
-                        apdu->a_Le,
-                        next_byte_count ?
-                        VCARD7816_SW1_RESPONSE_BYTES : VCARD7816_SW1_SUCCESS,
-                        next_byte_count);
-        buffer_response->current += bytes_to_copy;
-        buffer_response->len -= bytes_to_copy;
-        if (*response == NULL || (next_byte_count == 0)) {
-            vcard_set_buffer_response(card, NULL);
-            vcard_buffer_response_delete(buffer_response);
-        }
-        if (*response == NULL) {
-            *response =
-                vcard_make_response(VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-        }
-        break;
-
-    case VCARD7816_INS_GET_DATA:
-        *response =
-            vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-
-    default:
-        *response =
-            vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-        break;
-    }
-
-    /* response should have been set somewhere */
-    assert(*response != NULL);
-    return VCARD_DONE;
-}
-
-
-/*
- * APDU processing starts here. This routes the card processing stuff to the
- * right location.
- */
-VCardStatus
-vcard_process_apdu(VCard *card, VCardAPDU *apdu, VCardResponse **response)
-{
-    VCardStatus status;
-    VCardBufferResponse *buffer_response;
-
-    /* first handle any PTS commands, which aren't really APDU's */
-    if (apdu->a_type == VCARD_7816_PTS) {
-        /* the PTS responses aren't really responses either */
-        *response = vcard_response_new_data(apdu->a_data, apdu->a_len);
-        /* PTS responses have no status bytes */
-        (*response)->b_total_len = (*response)->b_len;
-        return VCARD_DONE;
-    }
-    buffer_response = vcard_get_buffer_response(card);
-    if (buffer_response && apdu->a_ins != VCARD7816_INS_GET_RESPONSE) {
-        /* clear out buffer_response, return an error */
-        vcard_set_buffer_response(card, NULL);
-        vcard_buffer_response_delete(buffer_response);
-        *response = vcard_make_response(VCARD7816_STATUS_EXC_ERROR);
-        return VCARD_DONE;
-    }
-
-    status = vcard_process_applet_apdu(card, apdu, response);
-    if (status != VCARD_NEXT) {
-        return status;
-    }
-    switch (vcard_get_type(card)) {
-    case VCARD_FILE_SYSTEM:
-        return vcard7816_file_system_process_apdu(card, apdu, response);
-    case VCARD_VM:
-        return vcard7816_vm_process_apdu(card, apdu, response);
-    case VCARD_DIRECT:
-        /* if we are type direct, then the applet should handle everything */
-        assert(!"VCARD_DIRECT: applet failure");
-        break;
-    }
-    *response =
-        vcard_make_response(VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED);
-    return VCARD_DONE;
-}
diff --git a/libcacard/card_7816.h b/libcacard/card_7816.h
deleted file mode 100644
index 4a01993..0000000
--- a/libcacard/card_7816.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CARD_7816_H
-#define CARD_7816_H  1
-
-#include "card_7816t.h"
-#include "vcardt.h"
-
-/*
- * constructors for VCardResponse's
- */
-/* response from a return buffer and a status */
-VCardResponse *vcard_response_new(VCard *card, unsigned char *buf, int len,
-                                  int Le, vcard_7816_status_t status);
-/* response from a return buffer and status bytes */
-VCardResponse *vcard_response_new_bytes(VCard *card, unsigned char *buf,
-                                        int len, int Le,
-                                        unsigned char sw1, unsigned char sw2);
-/* response from just status bytes */
-VCardResponse *vcard_response_new_status_bytes(unsigned char sw1,
-                                               unsigned char sw2);
-/* response from just status: NOTE this cannot fail, it will always return a
- * valid response, if it can't allocate memory, the response will be
- * VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE */
-VCardResponse *vcard_make_response(vcard_7816_status_t status);
-
-/* create a raw response (status has already been encoded */
-VCardResponse *vcard_response_new_data(unsigned char *buf, int len);
-
-
-
-
-/*
- * destructor for VCardResponse.
- *  Can be called with a NULL response
- */
-void vcard_response_delete(VCardResponse *response);
-
-/*
- * constructor for VCardAPDU
- */
-VCardAPDU *vcard_apdu_new(unsigned char *raw_apdu, int len,
-                          unsigned short *status);
-
-/*
- * destructor for VCardAPDU
- *  Can be called with a NULL apdu
- */
-void vcard_apdu_delete(VCardAPDU *apdu);
-
-/*
- * APDU processing starts here. This routes the card processing stuff to the
- * right location. Always returns a valid response.
- */
-VCardStatus vcard_process_apdu(VCard *card, VCardAPDU *apdu,
-                               VCardResponse **response);
-
-#endif
diff --git a/libcacard/card_7816t.h b/libcacard/card_7816t.h
deleted file mode 100644
index 8eef0ce..0000000
--- a/libcacard/card_7816t.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Implement the 7816 portion of the card spec
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef CARD_7816T_H
-#define CARD_7816T_H 1
-
-typedef unsigned short vcard_7816_status_t;
-
-struct VCardResponseStruct {
-    unsigned char *b_data;
-    vcard_7816_status_t b_status;
-    unsigned char b_sw1;
-    unsigned char b_sw2;
-    int b_len;
-    int b_total_len;
-    enum VCardResponseBufferType {
-        VCARD_MALLOC,
-        VCARD_MALLOC_DATA,
-        VCARD_MALLOC_STRUCT,
-        VCARD_STATIC
-    } b_type;
-};
-
-#define VCARD_RESPONSE_NEW_STATIC_STATUS(stat) \
-static const VCardResponse VCardResponse##stat = \
-        {(unsigned char *)&VCardResponse##stat.b_sw1, (stat), ((stat) >> 8), \
-         ((stat) & 0xff), 0, 2, VCARD_STATIC};
-
-#define VCARD_RESPONSE_NEW_STATIC_STATUS_BYTES(sw1, sw2) \
-static const VCardResponse VCARDResponse##sw1 = \
-        {(unsigned char *)&VCardResponse##name.b_sw1, ((sw1) << 8 | (sw2)), \
-         (sw1), (sw2), 0, 2, VCARD_STATIC};
-
-/* cast away the const, callers need may need to 'free' the
- * result, and const implies that they don't */
-#define VCARD_RESPONSE_GET_STATIC(name) \
-        ((VCardResponse *)(&VCardResponse##name))
-
-typedef enum {
-    VCARD_7816_ISO,
-    VCARD_7816_RFU,
-    VCARD_7816_PTS,
-    VCARD_7816_PROPRIETARY
-} VCardAPDUType;
-
-
-/*
- * 7816 header. All APDU's have this header.
- * They must be laid out in this order.
- */
-struct VCardAPDUHeader {
-    unsigned char ah_cla;
-    unsigned char ah_ins;
-    unsigned char ah_p1;
-    unsigned char ah_p2;
-    unsigned char ah_Le;
-    unsigned char ah_body[1]; /* indefinate length */
-};
-
-/*
- * 7816 APDU structure. The raw bytes are stored in the union and can be
- * accessed directly through u.data (which is aliased as a_data).
- *
- * Names of the fields match the 7816 documentation.
- */
-struct VCardAPDUStruct {
-    int a_len;                /* length of the whole buffer, including header */
-    int a_Lc;                 /* 7816 Lc (parameter length) value */
-    int a_Le;                 /* 7816 Le (expected result length) value */
-    unsigned char *a_body;    /* pointer to the parameter */
-    int a_channel;            /* decoded channel */
-    int a_secure_messaging;   /* decoded secure messaging type */
-    int a_type;               /* decoded type from cla (top nibble of class) */
-    VCardAPDUType a_gen_type; /* generic type (7816, PROPRIETARY, RFU, etc) */
-    union {
-        struct VCardAPDUHeader *header;
-        unsigned char   *data;
-    } u;
-/* give the subfields a unified look */
-#define a_header u.header
-#define a_data u.data
-#define a_cla a_header->ah_cla /* class */
-#define a_ins a_header->ah_ins /* instruction */
-#define a_p1 a_header->ah_p1   /* parameter 1 */
-#define a_p2 a_header->ah_p2   /* parameter 2 */
-};
-
-/* 7816 status codes */
-#define VCARD7816_STATUS_SUCCESS                              0x9000
-#define VCARD7816_STATUS_WARNING                              0x6200
-#define VCARD7816_STATUS_WARNING_RET_CORUPT                   0x6281
-#define VCARD7816_STATUS_WARNING_BUF_END_BEFORE_LE            0x6282
-#define VCARD7816_STATUS_WARNING_INVALID_FILE_SELECTED        0x6283
-#define VCARD7816_STATUS_WARNING_FCI_FORMAT_INVALID           0x6284
-#define VCARD7816_STATUS_WARNING_CHANGE                       0x6300
-#define VCARD7816_STATUS_WARNING_FILE_FILLED                  0x6381
-#define VCARD7816_STATUS_EXC_ERROR                            0x6400
-#define VCARD7816_STATUS_EXC_ERROR_CHANGE                     0x6500
-#define VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE             0x6581
-#define VCARD7816_STATUS_ERROR_WRONG_LENGTH                   0x6700
-#define VCARD7816_STATUS_ERROR_CLA_NOT_SUPPORTED              0x6800
-#define VCARD7816_STATUS_ERROR_CHANNEL_NOT_SUPPORTED          0x6881
-#define VCARD7816_STATUS_ERROR_SECURE_NOT_SUPPORTED           0x6882
-#define VCARD7816_STATUS_ERROR_COMMAND_NOT_SUPPORTED          0x6900
-#define VCARD7816_STATUS_ERROR_COMMAND_INCOMPATIBLE_WITH_FILE 0x6981
-#define VCARD7816_STATUS_ERROR_SECURITY_NOT_SATISFIED         0x6982
-#define VCARD7816_STATUS_ERROR_AUTHENTICATION_BLOCKED         0x6983
-#define VCARD7816_STATUS_ERROR_DATA_INVALID                   0x6984
-#define VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED        0x6985
-#define VCARD7816_STATUS_ERROR_DATA_NO_EF                     0x6986
-#define VCARD7816_STATUS_ERROR_SM_OBJECT_MISSING              0x6987
-#define VCARD7816_STATUS_ERROR_SM_OBJECT_INCORRECT            0x6988
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS               0x6a00
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_IN_DATA       0x6a80
-#define VCARD7816_STATUS_ERROR_FUNCTION_NOT_SUPPORTED         0x6a81
-#define VCARD7816_STATUS_ERROR_FILE_NOT_FOUND                 0x6a82
-#define VCARD7816_STATUS_ERROR_RECORD_NOT_FOUND               0x6a83
-#define VCARD7816_STATUS_ERROR_NO_SPACE_FOR_FILE              0x6a84
-#define VCARD7816_STATUS_ERROR_LC_TLV_INCONSISTENT            0x6a85
-#define VCARD7816_STATUS_ERROR_P1_P2_INCORRECT                0x6a86
-#define VCARD7816_STATUS_ERROR_LC_P1_P2_INCONSISTENT          0x6a87
-#define VCARD7816_STATUS_ERROR_DATA_NOT_FOUND                 0x6a88
-#define VCARD7816_STATUS_ERROR_WRONG_PARAMETERS_2             0x6b00
-#define VCARD7816_STATUS_ERROR_INS_CODE_INVALID               0x6d00
-#define VCARD7816_STATUS_ERROR_CLA_INVALID                    0x6e00
-#define VCARD7816_STATUS_ERROR_GENERAL                        0x6f00
-/* 7816 sw1 codes */
-#define VCARD7816_SW1_SUCCESS               0x90
-#define VCARD7816_SW1_RESPONSE_BYTES        0x61
-#define VCARD7816_SW1_WARNING               0x62
-#define VCARD7816_SW1_WARNING_CHANGE        0x63
-#define VCARD7816_SW1_EXC_ERROR             0x64
-#define VCARD7816_SW1_EXC_ERROR_CHANGE      0x65
-#define VCARD7816_SW1_ERROR_WRONG_LENGTH    0x67
-#define VCARD7816_SW1_CLA_ERROR             0x68
-#define VCARD7816_SW1_COMMAND_ERROR         0x69
-#define VCARD7816_SW1_P1_P2_ERROR           0x6a
-#define VCARD7816_SW1_LE_ERROR              0x6c
-#define VCARD7816_SW1_INS_ERROR             0x6d
-#define VCARD7816_SW1_CLA_NOT_SUPPORTED     0x6e
-
-/* 7816 Instructions */
-#define VCARD7816_INS_MANAGE_CHANNEL        0x70
-#define VCARD7816_INS_EXTERNAL_AUTHENTICATE 0x82
-#define VCARD7816_INS_GET_CHALLENGE         0x84
-#define VCARD7816_INS_INTERNAL_AUTHENTICATE 0x88
-#define VCARD7816_INS_ERASE_BINARY          0x0e
-#define VCARD7816_INS_READ_BINARY           0xb0
-#define VCARD7816_INS_WRITE_BINARY          0xd0
-#define VCARD7816_INS_UPDATE_BINARY         0xd6
-#define VCARD7816_INS_READ_RECORD           0xb2
-#define VCARD7816_INS_WRITE_RECORD          0xd2
-#define VCARD7816_INS_UPDATE_RECORD         0xdc
-#define VCARD7816_INS_APPEND_RECORD         0xe2
-#define VCARD7816_INS_ENVELOPE              0xc2
-#define VCARD7816_INS_PUT_DATA              0xda
-#define VCARD7816_INS_GET_DATA              0xca
-#define VCARD7816_INS_SELECT_FILE           0xa4
-#define VCARD7816_INS_VERIFY                0x20
-#define VCARD7816_INS_GET_RESPONSE          0xc0
-
-#endif
diff --git a/libcacard/event.c b/libcacard/event.c
deleted file mode 100644
index 4c551e4..0000000
--- a/libcacard/event.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * event queue implementation.
- *
- * This code is licensed under the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "qemu-common.h"
-
-#include "vcard.h"
-#include "vreader.h"
-#include "vevent.h"
-
-VEvent *
-vevent_new(VEventType type, VReader *reader, VCard *card)
-{
-    VEvent *new_vevent;
-
-    new_vevent = g_new(VEvent, 1);
-    new_vevent->next = NULL;
-    new_vevent->type = type;
-    new_vevent->reader = vreader_reference(reader);
-    new_vevent->card = vcard_reference(card);
-
-    return new_vevent;
-}
-
-void
-vevent_delete(VEvent *vevent)
-{
-    if (vevent == NULL) {
-        return;
-    }
-    vreader_free(vevent->reader);
-    vcard_free(vevent->card);
-    g_free(vevent);
-}
-
-/*
- * VEvent queue management
- */
-
-static VEvent *vevent_queue_head;
-static VEvent *vevent_queue_tail;
-static CompatGMutex vevent_queue_lock;
-static CompatGCond vevent_queue_condition;
-
-void vevent_queue_init(void)
-{
-    vevent_queue_head = vevent_queue_tail = NULL;
-}
-
-void
-vevent_queue_vevent(VEvent *vevent)
-{
-    vevent->next = NULL;
-    g_mutex_lock(&vevent_queue_lock);
-    if (vevent_queue_head) {
-        assert(vevent_queue_tail);
-        vevent_queue_tail->next = vevent;
-    } else {
-        vevent_queue_head = vevent;
-    }
-    vevent_queue_tail = vevent;
-    g_cond_signal(&vevent_queue_condition);
-    g_mutex_unlock(&vevent_queue_lock);
-}
-
-/* must have lock */
-static VEvent *
-vevent_dequeue_vevent(void)
-{
-    VEvent *vevent = NULL;
-    if (vevent_queue_head) {
-        vevent = vevent_queue_head;
-        vevent_queue_head = vevent->next;
-        vevent->next = NULL;
-    }
-    return vevent;
-}
-
-VEvent *vevent_wait_next_vevent(void)
-{
-    VEvent *vevent;
-
-    g_mutex_lock(&vevent_queue_lock);
-    while ((vevent = vevent_dequeue_vevent()) == NULL) {
-        g_cond_wait(&vevent_queue_condition, &vevent_queue_lock);
-    }
-    g_mutex_unlock(&vevent_queue_lock);
-    return vevent;
-}
-
-VEvent *vevent_get_next_vevent(void)
-{
-    VEvent *vevent;
-
-    g_mutex_lock(&vevent_queue_lock);
-    vevent = vevent_dequeue_vevent();
-    g_mutex_unlock(&vevent_queue_lock);
-    return vevent;
-}
-
diff --git a/libcacard/eventt.h b/libcacard/eventt.h
deleted file mode 100644
index 0dc7bd4..0000000
--- a/libcacard/eventt.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef EVENTT_H
-#define EVENTT_H 1
-#include "vreadert.h"
-#include "vcardt.h"
-
-typedef struct VEventStruct VEvent;
-
-typedef enum {
-    VEVENT_READER_INSERT,
-    VEVENT_READER_REMOVE,
-    VEVENT_CARD_INSERT,
-    VEVENT_CARD_REMOVE,
-    VEVENT_LAST,
-} VEventType;
-
-struct VEventStruct {
-    VEvent *next;
-    VEventType type;
-    VReader *reader;
-    VCard *card;
-};
-#endif
-
-
diff --git a/libcacard/libcacard.pc.in b/libcacard/libcacard.pc.in
deleted file mode 100644
index b6859b0..0000000
--- a/libcacard/libcacard.pc.in
+++ /dev/null
@@ -1,13 +0,0 @@
-prefix=@PREFIX@
-exec_prefix=${prefix}
-libdir=@LIBDIR@
-includedir=@INCLUDEDIR@
-
-Name: cacard
-Description: CA Card library
-Version: @VERSION@
-
-Requires:  nss
-Libs: -L${libdir} -lcacard
-Libs.private:
-Cflags: -I${includedir}
diff --git a/libcacard/libcacard.syms b/libcacard/libcacard.syms
deleted file mode 100644
index 1697515..0000000
--- a/libcacard/libcacard.syms
+++ /dev/null
@@ -1,77 +0,0 @@
-cac_card_init
-cac_is_cac_card
-vcard_add_applet
-vcard_apdu_delete
-vcard_apdu_new
-vcard_applet_get_aid
-vcard_buffer_response_delete
-vcard_buffer_response_new
-vcard_delete_applet
-vcard_emul_delete_key
-vcard_emul_force_card_insert
-vcard_emul_force_card_remove
-vcard_emul_get_atr
-vcard_emul_get_login_count
-vcard_emul_init
-vcard_emul_login
-vcard_emul_options
-vcard_emul_replay_insertion_events
-vcard_emul_reset
-vcard_emul_rsa_op
-vcard_emul_type_from_string
-vcard_emul_type_select
-vcard_emul_usage
-vcard_find_applet
-vcard_free
-vcard_get_atr
-vcard_get_buffer_response
-vcard_get_current_applet_private
-vcard_get_private
-vcard_get_type
-vcard_init
-vcard_make_response
-vcard_new
-vcard_new_applet
-vcard_process_apdu
-vcard_process_applet_apdu
-vcard_reference
-vcard_reset
-vcard_response_delete
-vcard_response_new
-vcard_response_new_bytes
-vcard_response_new_data
-vcard_response_new_status_bytes
-vcard_select_applet
-vcard_set_applet_private
-vcard_set_atr_func
-vcard_set_buffer_response
-vcard_set_type
-vevent_delete
-vevent_get_next_vevent
-vevent_new
-vevent_queue_init
-vevent_queue_vevent
-vevent_wait_next_vevent
-vreader_add_reader
-vreader_card_is_present
-vreader_free
-vreader_get_id
-vreader_get_name
-vreader_get_private
-vreader_get_reader_by_id
-vreader_get_reader_by_name
-vreader_get_reader_list
-vreader_init
-vreader_insert_card
-vreader_list_delete
-vreader_list_get_first
-vreader_list_get_next
-vreader_list_get_reader
-vreader_new
-vreader_power_off
-vreader_power_on
-vreader_queue_card_event
-vreader_reference
-vreader_remove_reader
-vreader_set_id
-vreader_xfr_bytes
diff --git a/libcacard/link_test.c b/libcacard/link_test.c
deleted file mode 100644
index 6f67a23..0000000
--- a/libcacard/link_test.c
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <stdio.h>
-#include "vcard.h"
-
-VCardStatus cac_card_init(const char *flags, VCard *card,
-                const unsigned char *cert[],
-                int cert_len[], VCardKey *key[] /* adopt the keys*/,
-                int cert_count);
-/*
- * this will crash... just test the linkage right now
- */
-
-main(int argc, char **argv)
-{
-    VCard *card; /* no constructor yet */
-    cac_card_init("", card, NULL, 0, NULL, 0);
-}
-
diff --git a/libcacard/vcard.c b/libcacard/vcard.c
deleted file mode 100644
index d140a8e..0000000
--- a/libcacard/vcard.c
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * implement the Java card standard.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "qemu-common.h"
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816t.h"
-
-struct VCardAppletStruct {
-    VCardApplet   *next;
-    VCardProcessAPDU process_apdu;
-    VCardResetApplet reset_applet;
-    unsigned char *aid;
-    int aid_len;
-    void *applet_private;
-    VCardAppletPrivateFree applet_private_free;
-};
-
-struct VCardStruct {
-    int reference_count;
-    VCardApplet *applet_list;
-    VCardApplet *current_applet[MAX_CHANNEL];
-    VCardBufferResponse *vcard_buffer_response;
-    VCardType type;
-    VCardEmul *vcard_private;
-    VCardEmulFree vcard_private_free;
-    VCardGetAtr vcard_get_atr;
-};
-
-VCardBufferResponse *
-vcard_buffer_response_new(unsigned char *buffer, int size)
-{
-    VCardBufferResponse *new_buffer;
-
-    new_buffer = g_new(VCardBufferResponse, 1);
-    new_buffer->buffer = (unsigned char *)g_memdup(buffer, size);
-    new_buffer->buffer_len = size;
-    new_buffer->current = new_buffer->buffer;
-    new_buffer->len = size;
-    return new_buffer;
-}
-
-void
-vcard_buffer_response_delete(VCardBufferResponse *buffer_response)
-{
-    if (buffer_response == NULL) {
-        return;
-    }
-    g_free(buffer_response->buffer);
-    g_free(buffer_response);
-}
-
-
-/*
- * clean up state after a reset
- */
-void
-vcard_reset(VCard *card, VCardPower power)
-{
-    int i;
-    VCardApplet *applet = NULL;
-
-    if (card->type ==  VCARD_DIRECT) {
-        /* select the last applet */
-        VCardApplet *current_applet = NULL;
-        for (current_applet = card->applet_list; current_applet;
-                                       current_applet = current_applet->next) {
-            applet = current_applet;
-        }
-    }
-    for (i = 0; i < MAX_CHANNEL; i++) {
-        card->current_applet[i] = applet;
-    }
-    if (card->vcard_buffer_response) {
-        vcard_buffer_response_delete(card->vcard_buffer_response);
-        card->vcard_buffer_response = NULL;
-    }
-    vcard_emul_reset(card, power);
-    if (applet) {
-        applet->reset_applet(card, 0);
-    }
-}
-
-/* applet utilities */
-
-/*
- * applet utilities
- */
-/* constructor */
-VCardApplet *
-vcard_new_applet(VCardProcessAPDU applet_process_function,
-                 VCardResetApplet applet_reset_function,
-                 unsigned char *aid, int aid_len)
-{
-    VCardApplet *applet;
-
-    applet = g_new0(VCardApplet, 1);
-    applet->process_apdu = applet_process_function;
-    applet->reset_applet = applet_reset_function;
-
-    applet->aid = g_memdup(aid, aid_len);
-    applet->aid_len = aid_len;
-    return applet;
-}
-
-/* destructor */
-void
-vcard_delete_applet(VCardApplet *applet)
-{
-    if (applet == NULL) {
-        return;
-    }
-    if (applet->applet_private_free) {
-        applet->applet_private_free(applet->applet_private);
-    }
-    g_free(applet->aid);
-    g_free(applet);
-}
-
-/* accessor */
-void
-vcard_set_applet_private(VCardApplet *applet, VCardAppletPrivate *private,
-                         VCardAppletPrivateFree private_free)
-{
-    if (applet->applet_private_free) {
-        applet->applet_private_free(applet->applet_private);
-    }
-    applet->applet_private = private;
-    applet->applet_private_free = private_free;
-}
-
-VCard *
-vcard_new(VCardEmul *private, VCardEmulFree private_free)
-{
-    VCard *new_card;
-
-    new_card = g_new0(VCard, 1);
-    new_card->type = VCARD_VM;
-    new_card->vcard_private = private;
-    new_card->vcard_private_free = private_free;
-    new_card->reference_count = 1;
-    return new_card;
-}
-
-VCard *
-vcard_reference(VCard *vcard)
-{
-    if (vcard == NULL) {
-        return NULL;
-    }
-    vcard->reference_count++;
-    return vcard;
-}
-
-void
-vcard_free(VCard *vcard)
-{
-    VCardApplet *current_applet;
-    VCardApplet *next_applet;
-
-    if (vcard == NULL) {
-        return;
-    }
-    vcard->reference_count--;
-    if (vcard->reference_count != 0) {
-        return;
-    }
-    if (vcard->vcard_private_free) {
-        (*vcard->vcard_private_free)(vcard->vcard_private);
-    }
-    for (current_applet = vcard->applet_list; current_applet;
-                                        current_applet = next_applet) {
-        next_applet = current_applet->next;
-        vcard_delete_applet(current_applet);
-    }
-    vcard_buffer_response_delete(vcard->vcard_buffer_response);
-    g_free(vcard);
-}
-
-void
-vcard_get_atr(VCard *vcard, unsigned char *atr, int *atr_len)
-{
-    if (vcard->vcard_get_atr) {
-        (*vcard->vcard_get_atr)(vcard, atr, atr_len);
-        return;
-    }
-    vcard_emul_get_atr(vcard, atr, atr_len);
-}
-
-void
-vcard_set_atr_func(VCard *card, VCardGetAtr vcard_get_atr)
-{
-    card->vcard_get_atr = vcard_get_atr;
-}
-
-
-VCardStatus
-vcard_add_applet(VCard *card, VCardApplet *applet)
-{
-    applet->next = card->applet_list;
-    card->applet_list = applet;
-    /* if our card-type is direct, always call the applet */
-    if (card->type ==  VCARD_DIRECT) {
-        int i;
-
-        for (i = 0; i < MAX_CHANNEL; i++) {
-            card->current_applet[i] = applet;
-        }
-    }
-    return VCARD_DONE;
-}
-
-/*
- * manage applets
- */
-VCardApplet *
-vcard_find_applet(VCard *card, unsigned char *aid, int aid_len)
-{
-    VCardApplet *current_applet;
-
-    for (current_applet = card->applet_list; current_applet;
-                                        current_applet = current_applet->next) {
-        if (current_applet->aid_len != aid_len) {
-            continue;
-        }
-        if (memcmp(current_applet->aid, aid, aid_len) == 0) {
-            break;
-        }
-    }
-    return current_applet;
-}
-
-unsigned char *
-vcard_applet_get_aid(VCardApplet *applet, int *aid_len)
-{
-    if (applet == NULL) {
-        return NULL;
-    }
-    *aid_len = applet->aid_len;
-    return applet->aid;
-}
-
-
-void
-vcard_select_applet(VCard *card, int channel, VCardApplet *applet)
-{
-    assert(channel < MAX_CHANNEL);
-
-    /* If using an emulated card, make sure to log out of any already logged in
-     * session. */
-    vcard_emul_logout(card);
-
-    card->current_applet[channel] = applet;
-    /* reset the applet */
-    if (applet && applet->reset_applet) {
-        applet->reset_applet(card, channel);
-    }
-}
-
-VCardAppletPrivate *
-vcard_get_current_applet_private(VCard *card, int channel)
-{
-    VCardApplet *applet = card->current_applet[channel];
-
-    if (applet == NULL) {
-        return NULL;
-    }
-    return applet->applet_private;
-}
-
-VCardStatus
-vcard_process_applet_apdu(VCard *card, VCardAPDU *apdu,
-                          VCardResponse **response)
-{
-    if (card->current_applet[apdu->a_channel]) {
-        return card->current_applet[apdu->a_channel]->process_apdu(
-                                                        card, apdu, response);
-    }
-    return VCARD_NEXT;
-}
-
-/*
- * Accessor functions
- */
-/* accessor functions for the response buffer */
-VCardBufferResponse *
-vcard_get_buffer_response(VCard *card)
-{
-    return card->vcard_buffer_response;
-}
-
-void
-vcard_set_buffer_response(VCard *card, VCardBufferResponse *buffer)
-{
-    card->vcard_buffer_response = buffer;
-}
-
-
-/* accessor functions for the type */
-VCardType
-vcard_get_type(VCard *card)
-{
-    return card->type;
-}
-
-void
-vcard_set_type(VCard *card, VCardType type)
-{
-    card->type = type;
-}
-
-/* accessor for private data */
-VCardEmul *
-vcard_get_private(VCard *vcard)
-{
-    return vcard->vcard_private;
-}
-
diff --git a/libcacard/vcard.h b/libcacard/vcard.h
deleted file mode 100644
index 47dc703..0000000
--- a/libcacard/vcard.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef VCARD_H
-#define VCARD_H 1
-
-#include "vcardt.h"
-
-/*
- * response buffer constructors and destructors.
- *
- * response buffers are used when we need to return more data than will fit in
- * a normal APDU response (nominally 254 bytes).
- */
-VCardBufferResponse *vcard_buffer_response_new(unsigned char *buffer, int size);
-void vcard_buffer_response_delete(VCardBufferResponse *buffer_response);
-
-
-/*
- * clean up state on reset
- */
-void vcard_reset(VCard *card, VCardPower power);
-
-/*
- * applet utilities
- */
-/*
- * Constructor for a VCardApplet
- */
-VCardApplet *vcard_new_applet(VCardProcessAPDU applet_process_function,
-                              VCardResetApplet applet_reset_function,
-                              unsigned char *aid, int aid_len);
-
-/*
- * destructor for a VCardApplet
- *  Can be called with a NULL applet
- */
-void vcard_delete_applet(VCardApplet *applet);
-
-/* accessor - set the card type specific private data */
-void vcard_set_applet_private(VCardApplet *applet, VCardAppletPrivate *_private,
-                              VCardAppletPrivateFree private_free);
-
-/* set type of vcard */
-void vcard_set_type(VCard *card, VCardType type);
-
-/*
- * utilities interacting with the current applet
- */
-/* add a new applet to a card */
-VCardStatus vcard_add_applet(VCard *card, VCardApplet *applet);
-/* find the applet on the card with the given aid */
-VCardApplet *vcard_find_applet(VCard *card, unsigned char *aid, int aid_len);
-/* set the following applet to be current on the given channel */
-void vcard_select_applet(VCard *card, int channel, VCardApplet *applet);
-/* get the card type specific private data on the given channel */
-VCardAppletPrivate *vcard_get_current_applet_private(VCard *card, int channel);
-/* fetch the applet's id */
-unsigned char *vcard_applet_get_aid(VCardApplet *applet, int *aid_len);
-
-/* process the apdu for the current selected applet/file */
-VCardStatus vcard_process_applet_apdu(VCard *card, VCardAPDU *apdu,
-                                      VCardResponse **response);
-/*
- * VCard utilities
- */
-/* constructor */
-VCard *vcard_new(VCardEmul *_private, VCardEmulFree private_free);
-/* get a reference */
-VCard *vcard_reference(VCard *);
-/* destructor (reference counted) */
-void vcard_free(VCard *);
-/* get the atr from the card */
-void vcard_get_atr(VCard *card, unsigned char *atr, int *atr_len);
-void vcard_set_atr_func(VCard *card, VCardGetAtr vcard_get_atr);
-
-/* accessor functions for the response buffer */
-VCardBufferResponse *vcard_get_buffer_response(VCard *card);
-void vcard_set_buffer_response(VCard *card, VCardBufferResponse *buffer);
-/* accessor functions for the type */
-VCardType vcard_get_type(VCard *card);
-/* get the private data */
-VCardEmul *vcard_get_private(VCard *card);
-
-#endif
diff --git a/libcacard/vcard_emul.h b/libcacard/vcard_emul.h
deleted file mode 100644
index f09ee98..0000000
--- a/libcacard/vcard_emul.h
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * This is the actual card emulator.
- *
- * These functions can be implemented in different ways on different platforms
- * using the underlying system primitives. For Linux it uses NSS, though direct
- * to PKCS #11, openssl+pkcs11, or even gnu crypto libraries+pkcs #11 could be
- * used. On Windows CAPI could be used.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VCARD_EMUL_H
-#define VCARD_EMUL_H 1
-
-#include "card_7816t.h"
-#include "vcard.h"
-#include "vcard_emul_type.h"
-
-/*
- * types
- */
-typedef enum {
-    VCARD_EMUL_OK = 0,
-    VCARD_EMUL_FAIL,
-    /* return values by vcard_emul_init */
-    VCARD_EMUL_INIT_ALREADY_INITED,
-} VCardEmulError;
-
-/* options are emul specific. call card_emul_parse_args to change a string
- * To an options struct */
-typedef struct VCardEmulOptionsStruct VCardEmulOptions;
-
-/*
- * Login functions
- */
-/* return the number of login attempts still possible on the card. if unknown,
- * return -1 */
-int vcard_emul_get_login_count(VCard *card);
-/* login into the card, return the 7816 status word (sw2 || sw1) */
-vcard_7816_status_t vcard_emul_login(VCard *card, unsigned char *pin,
-                                     int pin_len);
-void vcard_emul_logout(VCard *card);
-
-/*
- * key functions
- */
-/* delete a key */
-void vcard_emul_delete_key(VCardKey *key);
-/* RSA sign/decrypt with the key, signature happens 'in place' */
-vcard_7816_status_t vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                                  unsigned char *buffer, int buffer_size);
-
-void vcard_emul_reset(VCard *card, VCardPower power);
-void vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len);
-
-/* Re-insert of a card that has been removed by force removal */
-VCardEmulError vcard_emul_force_card_insert(VReader *vreader);
-/* Force a card removal even if the card is not physically removed */
-VCardEmulError vcard_emul_force_card_remove(VReader *vreader);
-
-VCardEmulOptions *vcard_emul_options(const char *args);
-VCardEmulError vcard_emul_init(const VCardEmulOptions *options);
-void vcard_emul_replay_insertion_events(void);
-void vcard_emul_usage(void);
-#endif
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
deleted file mode 100644
index 950edee..0000000
--- a/libcacard/vcard_emul_nss.c
+++ /dev/null
@@ -1,1274 +0,0 @@
-/*
- * This is the actual card emulator.
- *
- * These functions can be implemented in different ways on different platforms
- * using the underlying system primitives. For Linux it uses NSS, though direct
- * to PKCS #11, openssl+pkcs11, or even gnu crypto libraries+pkcs #11 could be
- * used. On Windows CAPI could be used.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-/*
- * NSS headers
- */
-
-/* avoid including prototypes.h that redefines uint32 */
-#define NO_NSPR_10_SUPPORT
-
-#include <nss.h>
-#include <pk11pub.h>
-#include <cert.h>
-#include <key.h>
-#include <secmod.h>
-#include <prthread.h>
-#include <secerr.h>
-
-#include "qemu-common.h"
-
-#include "vcard.h"
-#include "card_7816t.h"
-#include "vcard_emul.h"
-#include "vreader.h"
-#include "vevent.h"
-
-#include "libcacard/vcardt_internal.h"
-
-
-typedef enum {
-    VCardEmulUnknown = -1,
-    VCardEmulFalse = 0,
-    VCardEmulTrue = 1
-} VCardEmulTriState;
-
-struct VCardKeyStruct {
-    CERTCertificate *cert;
-    PK11SlotInfo *slot;
-    SECKEYPrivateKey *key;
-    VCardEmulTriState failedX509;
-};
-
-
-typedef struct VirtualReaderOptionsStruct VirtualReaderOptions;
-
-struct VReaderEmulStruct {
-    PK11SlotInfo *slot;
-    VCardEmulType default_type;
-    char *type_params;
-    PRBool present;
-    int     series;
-    VCard *saved_vcard;
-};
-
-/*
- *  NSS Specific options
- */
-struct VirtualReaderOptionsStruct {
-    char *name;
-    char *vname;
-    VCardEmulType card_type;
-    char *type_params;
-    char **cert_name;
-    int cert_count;
-};
-
-struct VCardEmulOptionsStruct {
-    void *nss_db;
-    VirtualReaderOptions *vreader;
-    int vreader_count;
-    VCardEmulType hw_card_type;
-    const char *hw_type_params;
-    PRBool use_hw;
-};
-
-static int nss_emul_init;
-
-/* if we have more that just the slot, define
- * VCardEmulStruct here */
-
-/*
- * allocate the set of arrays for certs, cert_len, key
- */
-static void
-vcard_emul_alloc_arrays(unsigned char ***certsp, int **cert_lenp,
-                        VCardKey ***keysp, int cert_count)
-{
-    *certsp = g_new(unsigned char *, cert_count);
-    *cert_lenp = g_new(int, cert_count);
-    *keysp = g_new(VCardKey *, cert_count);
-}
-
-/*
- * Emulator specific card information
- */
-typedef struct CardEmulCardStruct CardEmulPrivate;
-
-static VCardEmul *
-vcard_emul_new_card(PK11SlotInfo *slot)
-{
-    PK11_ReferenceSlot(slot);
-    /* currently we don't need anything other than the slot */
-    return (VCardEmul *)slot;
-}
-
-static void
-vcard_emul_delete_card(VCardEmul *vcard_emul)
-{
-    PK11SlotInfo *slot = (PK11SlotInfo *)vcard_emul;
-    if (slot == NULL) {
-        return;
-    }
-    PK11_FreeSlot(slot);
-}
-
-static PK11SlotInfo *
-vcard_emul_card_get_slot(VCard *card)
-{
-    /* note, the card is holding the reference, no need to get another one */
-    return (PK11SlotInfo *)vcard_get_private(card);
-}
-
-
-/*
- * key functions
- */
-/* private constructure */
-static VCardKey *
-vcard_emul_make_key(PK11SlotInfo *slot, CERTCertificate *cert)
-{
-    VCardKey *key;
-
-    key = g_new(VCardKey, 1);
-    key->slot = PK11_ReferenceSlot(slot);
-    key->cert = CERT_DupCertificate(cert);
-    /* NOTE: if we aren't logged into the token, this could return NULL */
-    /* NOTE: the cert is a temp cert, not necessarily the cert in the token,
-     * use the DER version of this function */
-    key->key = PK11_FindKeyByDERCert(slot, cert, NULL);
-    key->failedX509 = VCardEmulUnknown;
-    return key;
-}
-
-/* destructor */
-void
-vcard_emul_delete_key(VCardKey *key)
-{
-    if (!nss_emul_init || (key == NULL)) {
-        return;
-    }
-    if (key->key) {
-        SECKEY_DestroyPrivateKey(key->key);
-        key->key = NULL;
-    }
-    if (key->cert) {
-        CERT_DestroyCertificate(key->cert);
-    }
-    if (key->slot) {
-        PK11_FreeSlot(key->slot);
-    }
-}
-
-/*
- * grab the nss key from a VCardKey. If it doesn't exist, try to look it up
- */
-static SECKEYPrivateKey *
-vcard_emul_get_nss_key(VCardKey *key)
-{
-    if (key->key) {
-        return key->key;
-    }
-    /* NOTE: if we aren't logged into the token, this could return NULL */
-    key->key = PK11_FindPrivateKeyFromCert(key->slot, key->cert, NULL);
-    return key->key;
-}
-
-/*
- * Map NSS errors to 7816 errors
- */
-static vcard_7816_status_t
-vcard_emul_map_error(int error)
-{
-    switch (error) {
-    case SEC_ERROR_TOKEN_NOT_LOGGED_IN:
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    case SEC_ERROR_BAD_DATA:
-    case SEC_ERROR_OUTPUT_LEN:
-    case SEC_ERROR_INPUT_LEN:
-    case SEC_ERROR_INVALID_ARGS:
-    case SEC_ERROR_INVALID_ALGORITHM:
-    case SEC_ERROR_NO_KEY:
-    case SEC_ERROR_INVALID_KEY:
-    case SEC_ERROR_DECRYPTION_DISALLOWED:
-        return VCARD7816_STATUS_ERROR_DATA_INVALID;
-    case SEC_ERROR_NO_MEMORY:
-        return VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE;
-    }
-    return VCARD7816_STATUS_EXC_ERROR_CHANGE;
-}
-
-/* RSA sign/decrypt with the key, signature happens 'in place' */
-vcard_7816_status_t
-vcard_emul_rsa_op(VCard *card, VCardKey *key,
-                  unsigned char *buffer, int buffer_size)
-{
-    SECKEYPrivateKey *priv_key;
-    unsigned signature_len;
-    PK11SlotInfo *slot;
-    SECStatus rv;
-    unsigned char buf[2048];
-    unsigned char *bp = NULL;
-    int pad_len;
-    vcard_7816_status_t ret = VCARD7816_STATUS_SUCCESS;
-
-    if ((!nss_emul_init) || (key == NULL)) {
-        /* couldn't get the key, indicate that we aren't logged in */
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    priv_key = vcard_emul_get_nss_key(key);
-    if (priv_key == NULL) {
-        /* couldn't get the key, indicate that we aren't logged in */
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    slot = vcard_emul_card_get_slot(card);
-
-    /*
-     * this is only true of the rsa signature
-     */
-    signature_len = PK11_SignatureLen(priv_key);
-    if (buffer_size != signature_len) {
-        return  VCARD7816_STATUS_ERROR_DATA_INVALID;
-    }
-    /* be able to handle larger keys if necessariy */
-    bp = &buf[0];
-    if (sizeof(buf) < signature_len) {
-        bp = g_malloc(signature_len);
-    }
-
-    /*
-     * do the raw operations. Some tokens claim to do CKM_RSA_X_509, but then
-     * choke when they try to do the actual operations. Try to detect
-     * those cases and treat them as if the token didn't claim support for
-     * X_509.
-     */
-    if (key->failedX509 != VCardEmulTrue
-                              && PK11_DoesMechanism(slot, CKM_RSA_X_509)) {
-        rv = PK11_PrivDecryptRaw(priv_key, bp, &signature_len, signature_len,
-                                 buffer, buffer_size);
-        if (rv == SECSuccess) {
-            assert(buffer_size == signature_len);
-            memcpy(buffer, bp, signature_len);
-            key->failedX509 = VCardEmulFalse;
-            goto cleanup;
-        }
-        /*
-         * we've had a successful X509 operation, this failure must be
-         * somethine else
-         */
-        if (key->failedX509 == VCardEmulFalse) {
-            ret = vcard_emul_map_error(PORT_GetError());
-            goto cleanup;
-        }
-        /*
-         * key->failedX509 must be Unknown at this point, try the
-         * non-x_509 case
-         */
-    }
-    /* token does not support CKM_RSA_X509, emulate that with CKM_RSA_PKCS */
-    /* is this a PKCS #1 formatted signature? */
-    if ((buffer[0] == 0) && (buffer[1] == 1)) {
-        int i;
-
-        for (i = 2; i < buffer_size; i++) {
-            /* rsa signature pad */
-            if (buffer[i] != 0xff) {
-                break;
-            }
-        }
-        if ((i < buffer_size) && (buffer[i] == 0)) {
-            /* yes, we have a properly formatted PKCS #1 signature */
-            /*
-             * NOTE: even if we accidentally got an encrypt buffer, which
-             * through sheer luck started with 00, 01, ff, 00, it won't matter
-             * because the resulting Sign operation will effectively decrypt
-             * the real buffer.
-             */
-            SECItem signature;
-            SECItem hash;
-
-            i++;
-            hash.data = &buffer[i];
-            hash.len = buffer_size - i;
-            signature.data = bp;
-            signature.len = signature_len;
-            rv = PK11_Sign(priv_key,  &signature, &hash);
-            if (rv != SECSuccess) {
-                ret = vcard_emul_map_error(PORT_GetError());
-                goto cleanup;
-            }
-            assert(buffer_size == signature.len);
-            memcpy(buffer, bp, signature.len);
-            /*
-             * we got here because either the X509 attempt failed, or the
-             * token couldn't do the X509 operation, in either case stay
-             * with the PKCS version for future operations on this key
-             */
-            key->failedX509 = VCardEmulTrue;
-            goto cleanup;
-        }
-    }
-    pad_len = buffer_size - signature_len;
-    assert(pad_len < 4);
-    /*
-     * OK now we've decrypted the payload, package it up in PKCS #1 for the
-     * upper layer.
-     */
-    buffer[0] = 0;
-    buffer[1] = 2; /* RSA_encrypt  */
-    pad_len -= 3; /* format is 0 || 2 || pad || 0 || data */
-    /*
-     * padding for PKCS #1 encrypted data is a string of random bytes. The
-     * random butes protect against potential decryption attacks against RSA.
-     * Since PrivDecrypt has already stripped those bytes, we can't reconstruct
-     * them. This shouldn't matter to the upper level code which should just
-     * strip this code out anyway, so We'll pad with a constant 3.
-     */
-    memset(&buffer[2], 0x03, pad_len);
-    pad_len += 2; /* index to the end of the pad */
-    buffer[pad_len] = 0;
-    pad_len++; /* index to the start of the data */
-    memcpy(&buffer[pad_len], bp, signature_len);
-    /*
-     * we got here because either the X509 attempt failed, or the
-     * token couldn't do the X509 operation, in either case stay
-     * with the PKCS version for future operations on this key
-     */
-    key->failedX509 = VCardEmulTrue;
-cleanup:
-    if (bp != buf) {
-        g_free(bp);
-    }
-    return ret;
-}
-
-/*
- * Login functions
- */
-/* return the number of login attempts still possible on the card. if unknown,
- * return -1 */
-int
-vcard_emul_get_login_count(VCard *card)
-{
-    return -1;
-}
-
-/* login into the card, return the 7816 status word (sw2 || sw1) */
-vcard_7816_status_t
-vcard_emul_login(VCard *card, unsigned char *pin, int pin_len)
-{
-    PK11SlotInfo *slot;
-    unsigned char *pin_string;
-    int i;
-    SECStatus rv;
-
-    if (!nss_emul_init) {
-        return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-    }
-    slot = vcard_emul_card_get_slot(card);
-     /* We depend on the PKCS #11 module internal login state here because we
-      * create a separate process to handle each guest instance. If we needed
-      * to handle multiple guests from one process, then we would need to keep
-      * a lot of extra state in our card structure
-      * */
-    pin_string = g_malloc(pin_len+1);
-    memcpy(pin_string, pin, pin_len);
-    pin_string[pin_len] = 0;
-
-    /* handle CAC expanded pins correctly */
-    for (i = pin_len-1; i >= 0 && (pin_string[i] == 0xff); i--) {
-        pin_string[i] = 0;
-    }
-
-    rv = PK11_Authenticate(slot, PR_FALSE, pin_string);
-    memset(pin_string, 0, pin_len);  /* don't let the pin hang around in memory
-                                        to be snooped */
-    g_free(pin_string);
-    if (rv == SECSuccess) {
-        return VCARD7816_STATUS_SUCCESS;
-    }
-    /* map the error from port get error */
-    return VCARD7816_STATUS_ERROR_CONDITION_NOT_SATISFIED;
-}
-
-void
-vcard_emul_logout(VCard *card)
-{
-    PK11SlotInfo *slot;
-
-    if (!nss_emul_init) {
-        return;
-    }
-
-    slot = vcard_emul_card_get_slot(card);
-    if (PK11_IsLoggedIn(slot, NULL)) {
-        PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
-    }
-}
-
-void
-vcard_emul_reset(VCard *card, VCardPower power)
-{
-    /*
-     * if we reset the card (either power on or power off), we lose our login
-     * state
-     */
-    vcard_emul_logout(card);
-
-    /* TODO: we may also need to send insertion/removal events? */
-}
-
-static VReader *
-vcard_emul_find_vreader_from_slot(PK11SlotInfo *slot)
-{
-    VReaderList *reader_list = vreader_get_reader_list();
-    VReaderListEntry *current_entry;
-
-    if (reader_list == NULL) {
-        return NULL;
-    }
-    for (current_entry = vreader_list_get_first(reader_list); current_entry;
-                        current_entry = vreader_list_get_next(current_entry)) {
-        VReader *reader = vreader_list_get_reader(current_entry);
-        VReaderEmul *reader_emul = vreader_get_private(reader);
-        if (reader_emul->slot == slot) {
-            vreader_list_delete(reader_list);
-            return reader;
-        }
-        vreader_free(reader);
-    }
-
-    vreader_list_delete(reader_list);
-    return NULL;
-}
-
-/*
- * create a new reader emul
- */
-static VReaderEmul *
-vreader_emul_new(PK11SlotInfo *slot, VCardEmulType type, const char *params)
-{
-    VReaderEmul *new_reader_emul;
-
-    new_reader_emul = g_new(VReaderEmul, 1);
-
-    new_reader_emul->slot = PK11_ReferenceSlot(slot);
-    new_reader_emul->default_type = type;
-    new_reader_emul->type_params = g_strdup(params);
-    new_reader_emul->present = PR_FALSE;
-    new_reader_emul->series = 0;
-    new_reader_emul->saved_vcard = NULL;
-    return new_reader_emul;
-}
-
-static void
-vreader_emul_delete(VReaderEmul *vreader_emul)
-{
-    if (vreader_emul == NULL) {
-        return;
-    }
-    if (vreader_emul->slot) {
-        PK11_FreeSlot(vreader_emul->slot);
-    }
-    g_free(vreader_emul->type_params);
-    g_free(vreader_emul);
-}
-
-/*
- *  TODO: move this to emulater non-specific file
- */
-static VCardEmulType
-vcard_emul_get_type(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-
-    vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul && vreader_emul->default_type != VCARD_EMUL_NONE) {
-        return vreader_emul->default_type;
-    }
-
-    return vcard_emul_type_select(vreader);
-}
-/*
- *  TODO: move this to emulater non-specific file
- */
-static const char *
-vcard_emul_get_type_params(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-
-    vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul && vreader_emul->type_params) {
-        return vreader_emul->type_params;
-    }
-
-    return "";
-}
-
-/* pull the slot out of the reader private data */
-static PK11SlotInfo *
-vcard_emul_reader_get_slot(VReader *vreader)
-{
-    VReaderEmul *vreader_emul = vreader_get_private(vreader);
-    if (vreader_emul == NULL) {
-        return NULL;
-    }
-    return vreader_emul->slot;
-}
-
-/*
- *  Card ATR's map to physical cards. vcard_alloc_atr will set appropriate
- *  historical bytes for any software emulated card. The remaining bytes can be
- *  used to indicate the actual emulator
- */
-static unsigned char *nss_atr;
-static int nss_atr_len;
-
-void
-vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len)
-{
-    int len;
-    assert(atr != NULL);
-
-    if (nss_atr == NULL) {
-        nss_atr = vcard_alloc_atr("NSS", &nss_atr_len);
-    }
-    len = MIN(nss_atr_len, *atr_len);
-    memcpy(atr, nss_atr, len);
-    *atr_len = len;
-}
-
-/*
- * create a new card from certs and keys
- */
-static VCard *
-vcard_emul_make_card(VReader *reader,
-                     unsigned char * const *certs, int *cert_len,
-                     VCardKey *keys[], int cert_count)
-{
-    VCardEmul *vcard_emul;
-    VCard *vcard;
-    PK11SlotInfo *slot;
-    VCardEmulType type;
-    const char *params;
-
-    type = vcard_emul_get_type(reader);
-
-    /* ignore the inserted card */
-    if (type == VCARD_EMUL_NONE) {
-        return NULL;
-    }
-    slot = vcard_emul_reader_get_slot(reader);
-    if (slot == NULL) {
-        return NULL;
-    }
-
-    params = vcard_emul_get_type_params(reader);
-    /* params these can be NULL */
-
-    vcard_emul = vcard_emul_new_card(slot);
-    if (vcard_emul == NULL) {
-        return NULL;
-    }
-    vcard = vcard_new(vcard_emul, vcard_emul_delete_card);
-    if (vcard == NULL) {
-        vcard_emul_delete_card(vcard_emul);
-        return NULL;
-    }
-    vcard_init(reader, vcard, type, params, certs, cert_len, keys, cert_count);
-    return vcard;
-}
-
-
-/*
- * 'clone' a physical card as a virtual card
- */
-static VCard *
-vcard_emul_mirror_card(VReader *vreader)
-{
-    /*
-     * lookup certs using the C_FindObjects. The Stan Cert handle won't give
-     * us the real certs until we log in.
-     */
-    PK11GenericObject *firstObj, *thisObj;
-    int cert_count;
-    unsigned char **certs;
-    int *cert_len;
-    VCardKey **keys;
-    PK11SlotInfo *slot;
-    VCard *card;
-
-    slot = vcard_emul_reader_get_slot(vreader);
-    if (slot == NULL) {
-        return NULL;
-    }
-
-    firstObj = PK11_FindGenericObjects(slot, CKO_CERTIFICATE);
-    if (firstObj == NULL) {
-        return NULL;
-    }
-
-    /* count the certs */
-    cert_count = 0;
-    for (thisObj = firstObj; thisObj;
-                             thisObj = PK11_GetNextGenericObject(thisObj)) {
-        cert_count++;
-    }
-
-    /* allocate the arrays */
-    vcard_emul_alloc_arrays(&certs, &cert_len, &keys, cert_count);
-
-    /* fill in the arrays */
-    cert_count = 0;
-    for (thisObj = firstObj; thisObj;
-                             thisObj = PK11_GetNextGenericObject(thisObj)) {
-        SECItem derCert;
-        CERTCertificate *cert;
-        SECStatus rv;
-
-        rv = PK11_ReadRawAttribute(PK11_TypeGeneric, thisObj,
-                                   CKA_VALUE, &derCert);
-        if (rv != SECSuccess) {
-            continue;
-        }
-        /* create floating temp cert. This gives us a cert structure even if
-         * the token isn't logged in */
-        cert = CERT_NewTempCertificate(CERT_GetDefaultCertDB(), &derCert,
-                                       NULL, PR_FALSE, PR_TRUE);
-        SECITEM_FreeItem(&derCert, PR_FALSE);
-        if (cert == NULL) {
-            continue;
-        }
-
-        certs[cert_count] = cert->derCert.data;
-        cert_len[cert_count] = cert->derCert.len;
-        keys[cert_count] = vcard_emul_make_key(slot, cert);
-        cert_count++;
-        CERT_DestroyCertificate(cert); /* key obj still has a reference */
-    }
-
-    /* now create the card */
-    card = vcard_emul_make_card(vreader, certs, cert_len, keys, cert_count);
-    g_free(certs);
-    g_free(cert_len);
-    g_free(keys);
-
-    return card;
-}
-
-static VCardEmulType default_card_type = VCARD_EMUL_NONE;
-static const char *default_type_params = "";
-
-/*
- * This thread looks for card and reader insertions and puts events on the
- * event queue
- */
-static void
-vcard_emul_event_thread(void *arg)
-{
-    PK11SlotInfo *slot;
-    VReader *vreader;
-    VReaderEmul *vreader_emul;
-    VCard *vcard;
-    SECMODModule *module = (SECMODModule *)arg;
-
-    do {
-        /*
-         * XXX - the latency value doesn't matter one bit. you only get no
-         * blocking (flags |= CKF_DONT_BLOCK) or PKCS11_WAIT_LATENCY (==500),
-         * hard coded in coolkey.  And it isn't coolkey's fault - the timeout
-         * value we pass get's dropped on the floor before C_WaitForSlotEvent
-         * is called.
-         */
-        slot = SECMOD_WaitForAnyTokenEvent(module, 0, 500);
-        if (slot == NULL) {
-            /* this could be just a no event indication */
-            if (PORT_GetError() == SEC_ERROR_NO_EVENT) {
-                continue;
-            }
-            break;
-        }
-        vreader = vcard_emul_find_vreader_from_slot(slot);
-        if (vreader == NULL) {
-            /* new vreader */
-            vreader_emul = vreader_emul_new(slot, default_card_type,
-                                            default_type_params);
-            vreader = vreader_new(PK11_GetSlotName(slot), vreader_emul,
-                                  vreader_emul_delete);
-            PK11_FreeSlot(slot);
-            slot = NULL;
-            vreader_add_reader(vreader);
-            vreader_free(vreader);
-            continue;
-        }
-        /* card remove/insert */
-        vreader_emul = vreader_get_private(vreader);
-        if (PK11_IsPresent(slot)) {
-            int series = PK11_GetSlotSeries(slot);
-            if (series != vreader_emul->series) {
-                if (vreader_emul->present) {
-                    vreader_insert_card(vreader, NULL);
-                }
-                vcard = vcard_emul_mirror_card(vreader);
-                vreader_insert_card(vreader, vcard);
-                vcard_free(vcard);
-            }
-            vreader_emul->series = series;
-            vreader_emul->present = 1;
-            vreader_free(vreader);
-            PK11_FreeSlot(slot);
-            continue;
-        }
-        if (vreader_emul->present) {
-            vreader_insert_card(vreader, NULL);
-        }
-        vreader_emul->series = 0;
-        vreader_emul->present = 0;
-        PK11_FreeSlot(slot);
-        vreader_free(vreader);
-    } while (1);
-}
-
-/* if the card is inserted when we start up, make sure our state is correct */
-static void
-vcard_emul_init_series(VReader *vreader, VCard *vcard)
-{
-    VReaderEmul *vreader_emul = vreader_get_private(vreader);
-    PK11SlotInfo *slot = vreader_emul->slot;
-
-    vreader_emul->present = PK11_IsPresent(slot);
-    vreader_emul->series = PK11_GetSlotSeries(slot);
-    if (vreader_emul->present == 0) {
-        vreader_insert_card(vreader, NULL);
-    }
-}
-
-/*
- * each module has a separate wait call, create a thread for each module that
- * we are using.
- */
-static void
-vcard_emul_new_event_thread(SECMODModule *module)
-{
-    PR_CreateThread(PR_SYSTEM_THREAD, vcard_emul_event_thread,
-                     module, PR_PRIORITY_HIGH, PR_GLOBAL_THREAD,
-                     PR_UNJOINABLE_THREAD, 0);
-}
-
-static const VCardEmulOptions default_options = {
-    .nss_db = NULL,
-    .vreader = NULL,
-    .vreader_count = 0,
-    .hw_card_type = VCARD_EMUL_CAC,
-    .hw_type_params = "",
-    .use_hw = PR_TRUE
-};
-
-
-/*
- *  NSS needs the app to supply a password prompt. In our case the only time
- *  the password is supplied is as part of the Login APDU. The actual password
- *  is passed in the pw_arg in that case. In all other cases pw_arg should be
- *  NULL.
- */
-static char *
-vcard_emul_get_password(PK11SlotInfo *slot, PRBool retries, void *pw_arg)
-{
-    /* if it didn't work the first time, don't keep trying */
-    if (retries) {
-        return NULL;
-    }
-    /* we are looking up a password when we don't have one in hand */
-    if (pw_arg == NULL) {
-        return NULL;
-    }
-    /* TODO: we really should verify that were are using the right slot */
-    return PORT_Strdup(pw_arg);
-}
-
-/* Force a card removal even if the card is not physically removed */
-VCardEmulError
-vcard_emul_force_card_remove(VReader *vreader)
-{
-    if (!nss_emul_init || (vreader_card_is_present(vreader) != VREADER_OK)) {
-        return VCARD_EMUL_FAIL; /* card is already removed */
-    }
-
-    /* OK, remove it */
-    vreader_insert_card(vreader, NULL);
-    return VCARD_EMUL_OK;
-}
-
-/* Re-insert of a card that has been removed by force removal */
-VCardEmulError
-vcard_emul_force_card_insert(VReader *vreader)
-{
-    VReaderEmul *vreader_emul;
-    VCard *vcard;
-
-    if (!nss_emul_init || (vreader_card_is_present(vreader) == VREADER_OK)) {
-        return VCARD_EMUL_FAIL; /* card is already removed */
-    }
-    vreader_emul = vreader_get_private(vreader);
-
-    /* if it's a softcard, get the saved vcard from the reader emul structure */
-    if (vreader_emul->saved_vcard) {
-        vcard = vcard_reference(vreader_emul->saved_vcard);
-    } else {
-        /* it must be a physical card, rebuild it */
-        if (!PK11_IsPresent(vreader_emul->slot)) {
-            /* physical card has been removed, not way to reinsert it */
-            return VCARD_EMUL_FAIL;
-        }
-        vcard = vcard_emul_mirror_card(vreader);
-    }
-    vreader_insert_card(vreader, vcard);
-    vcard_free(vcard);
-
-    return VCARD_EMUL_OK;
-}
-
-
-static PRBool
-module_has_removable_hw_slots(SECMODModule *mod)
-{
-    int i;
-    PRBool ret = PR_FALSE;
-    SECMODListLock *moduleLock = SECMOD_GetDefaultModuleListLock();
-
-    if (!moduleLock) {
-        PORT_SetError(SEC_ERROR_NOT_INITIALIZED);
-        return ret;
-    }
-    SECMOD_GetReadLock(moduleLock);
-    for (i = 0; i < mod->slotCount; i++) {
-        PK11SlotInfo *slot = mod->slots[i];
-        if (PK11_IsRemovable(slot) && PK11_IsHW(slot)) {
-            ret = PR_TRUE;
-            break;
-        }
-    }
-    SECMOD_ReleaseReadLock(moduleLock);
-    return ret;
-}
-
-/* Previously we returned FAIL if no readers found. This makes
- * no sense when using hardware, since there may be no readers connected
- * at the time vcard_emul_init is called, but they will be properly
- * recognized later. So Instead return FAIL only if no_hw==1 and no
- * vcards can be created (indicates error with certificates provided
- * or db), or if any other higher level error (NSS error, missing coolkey). */
-static int vcard_emul_init_called;
-
-VCardEmulError
-vcard_emul_init(const VCardEmulOptions *options)
-{
-    SECStatus rv;
-    PRBool has_readers = PR_FALSE;
-    VReader *vreader;
-    VReaderEmul *vreader_emul;
-    SECMODListLock *module_lock;
-    SECMODModuleList *module_list;
-    SECMODModuleList *mlp;
-    int i;
-
-    if (vcard_emul_init_called) {
-        return VCARD_EMUL_INIT_ALREADY_INITED;
-    }
-    vcard_emul_init_called = 1;
-    vreader_init();
-    vevent_queue_init();
-
-    if (options == NULL) {
-        options = &default_options;
-    }
-
-    /* first initialize NSS */
-    if (options->nss_db) {
-        rv = NSS_Init(options->nss_db);
-    } else {
-        gchar *path;
-#ifndef _WIN32
-        path = g_strdup("/etc/pki/nssdb");
-#else
-        if (g_get_system_config_dirs() == NULL ||
-            g_get_system_config_dirs()[0] == NULL) {
-            return VCARD_EMUL_FAIL;
-        }
-
-        path = g_build_filename(
-            g_get_system_config_dirs()[0], "pki", "nssdb", NULL);
-#endif
-
-        rv = NSS_Init(path);
-        g_free(path);
-    }
-    if (rv != SECSuccess) {
-        return VCARD_EMUL_FAIL;
-    }
-    /* Set password callback function */
-    PK11_SetPasswordFunc(vcard_emul_get_password);
-
-    /* set up soft cards emulated by software certs rather than physical cards
-     * */
-    for (i = 0; i < options->vreader_count; i++) {
-        int j;
-        int cert_count;
-        unsigned char **certs;
-        int *cert_len;
-        VCardKey **keys;
-        PK11SlotInfo *slot;
-
-        slot = PK11_FindSlotByName(options->vreader[i].name);
-        if (slot == NULL) {
-            continue;
-        }
-        vreader_emul = vreader_emul_new(slot, options->vreader[i].card_type,
-                                        options->vreader[i].type_params);
-        vreader = vreader_new(options->vreader[i].vname, vreader_emul,
-                              vreader_emul_delete);
-        vreader_add_reader(vreader);
-
-        vcard_emul_alloc_arrays(&certs, &cert_len, &keys,
-                                options->vreader[i].cert_count);
-
-        cert_count = 0;
-        for (j = 0; j < options->vreader[i].cert_count; j++) {
-            /* we should have a better way of identifying certs than by
-             * nickname here */
-            CERTCertificate *cert = PK11_FindCertFromNickname(
-                                        options->vreader[i].cert_name[j],
-                                        NULL);
-            if (cert == NULL) {
-                continue;
-            }
-            certs[cert_count] = cert->derCert.data;
-            cert_len[cert_count] = cert->derCert.len;
-            keys[cert_count] = vcard_emul_make_key(slot, cert);
-            /* this is safe because the key is still holding a cert reference */
-            CERT_DestroyCertificate(cert);
-            cert_count++;
-        }
-        if (cert_count) {
-            VCard *vcard = vcard_emul_make_card(vreader, certs, cert_len,
-                                                keys, cert_count);
-            vreader_insert_card(vreader, vcard);
-            vcard_emul_init_series(vreader, vcard);
-            /* allow insertion and removal of soft cards */
-            vreader_emul->saved_vcard = vcard_reference(vcard);
-            vcard_free(vcard);
-            vreader_free(vreader);
-            has_readers = PR_TRUE;
-        }
-        g_free(certs);
-        g_free(cert_len);
-        g_free(keys);
-    }
-
-    /* if we aren't suppose to use hw, skip looking up hardware tokens */
-    if (!options->use_hw) {
-        nss_emul_init = has_readers;
-        return has_readers ? VCARD_EMUL_OK : VCARD_EMUL_FAIL;
-    }
-
-    /* make sure we have some PKCS #11 module loaded */
-    module_lock = SECMOD_GetDefaultModuleListLock();
-    module_list = SECMOD_GetDefaultModuleList();
-    SECMOD_GetReadLock(module_lock);
-    for (mlp = module_list; mlp; mlp = mlp->next) {
-        SECMODModule *module = mlp->module;
-        if (module_has_removable_hw_slots(module)) {
-            break;
-        }
-    }
-    SECMOD_ReleaseReadLock(module_lock);
-
-    /* now examine all the slots, finding which should be readers */
-    /* We should control this with options. For now we mirror out any
-     * removable hardware slot */
-    default_card_type = options->hw_card_type;
-    default_type_params = g_strdup(options->hw_type_params);
-
-    SECMOD_GetReadLock(module_lock);
-    for (mlp = module_list; mlp; mlp = mlp->next) {
-        SECMODModule *module = mlp->module;
-
-        /* Ignore the internal module */
-        if (module == NULL || module == SECMOD_GetInternalModule()) {
-            continue;
-        }
-
-        for (i = 0; i < module->slotCount; i++) {
-            PK11SlotInfo *slot = module->slots[i];
-
-            /* only map removable HW slots */
-            if (slot == NULL || !PK11_IsRemovable(slot) || !PK11_IsHW(slot)) {
-                continue;
-            }
-            if (strcmp("E-Gate 0 0", PK11_GetSlotName(slot)) == 0) {
-                /*
-                 * coolkey <= 1.1.0-20 emulates this reader if it can't find
-                 * any hardware readers. This causes problems, warn user of
-                 * problems.
-                 */
-                fprintf(stderr, "known bad coolkey version - see "
-                        "https://bugzilla.redhat.com/show_bug.cgi?id=802435\n");
-                continue;
-            }
-            vreader_emul = vreader_emul_new(slot, options->hw_card_type,
-                                            options->hw_type_params);
-            vreader = vreader_new(PK11_GetSlotName(slot), vreader_emul,
-                                  vreader_emul_delete);
-            vreader_add_reader(vreader);
-
-            if (PK11_IsPresent(slot)) {
-                VCard *vcard;
-                vcard = vcard_emul_mirror_card(vreader);
-                vreader_insert_card(vreader, vcard);
-                vcard_emul_init_series(vreader, vcard);
-                vcard_free(vcard);
-            }
-        }
-        vcard_emul_new_event_thread(module);
-    }
-    SECMOD_ReleaseReadLock(module_lock);
-    nss_emul_init = PR_TRUE;
-
-    return VCARD_EMUL_OK;
-}
-
-/* Recreate card insert events for all readers (user should
- * deduce implied reader insert. perhaps do a reader insert as well?)
- */
-void
-vcard_emul_replay_insertion_events(void)
-{
-    VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry;
-    VReaderList *list = vreader_get_reader_list();
-
-    for (current_entry = vreader_list_get_first(list); current_entry;
-            current_entry = next_entry) {
-        VReader *vreader = vreader_list_get_reader(current_entry);
-        next_entry = vreader_list_get_next(current_entry);
-        vreader_queue_card_event(vreader);
-    }
-
-    vreader_list_delete(list);
-}
-
-/*
- *  Silly little functions to help parsing our argument string
- */
-static int
-count_tokens(const char *str, char token, char token_end)
-{
-    int count = 0;
-
-    for (; *str; str++) {
-        if (*str == token) {
-            count++;
-        }
-        if (*str == token_end) {
-            break;
-        }
-    }
-    return count;
-}
-
-static const char *
-strip(const char *str)
-{
-    for (; *str && isspace(*str); str++) {
-    }
-    return str;
-}
-
-static const char *
-find_blank(const char *str)
-{
-    for (; *str && !isspace(*str); str++) {
-    }
-    return str;
-}
-
-
-/*
- *  We really want to use some existing argument parsing library here. That
- *  would give us a consistent look */
-static VCardEmulOptions options;
-#define READER_STEP 4
-
-/* Expects "args" to be at the beginning of a token (ie right after the ','
- * ending the previous token), and puts the next token start in "token",
- * and its length in "token_length". "token" will not be nul-terminated.
- * After calling the macro, "args" will be advanced to the beginning of
- * the next token.
- * This macro may call continue or break.
- */
-#define NEXT_TOKEN(token) \
-            (token) = args; \
-            args = strpbrk(args, ",)"); \
-            if (*args == 0) { \
-                break; \
-            } \
-            if (*args == ')') { \
-                args++; \
-                continue; \
-            } \
-            (token##_length) = args - (token); \
-            args = strip(args+1);
-
-VCardEmulOptions *
-vcard_emul_options(const char *args)
-{
-    int reader_count = 0;
-    VCardEmulOptions *opts;
-
-    /* Allow the future use of allocating the options structure on the fly */
-    memcpy(&options, &default_options, sizeof(options));
-    opts = &options;
-
-    do {
-        args = strip(args); /* strip off the leading spaces */
-        if (*args == ',') {
-            continue;
-        }
-        /* soft=(slot_name,virt_name,emul_type,emul_flags,cert_1, (no eol)
-         *       cert_2,cert_3...) */
-        if (strncmp(args, "soft=", 5) == 0) {
-            const char *name;
-            size_t name_length;
-            const char *vname;
-            size_t vname_length;
-            const char *type_params;
-            size_t type_params_length;
-            char type_str[100];
-            VCardEmulType type;
-            int count, i;
-            VirtualReaderOptions *vreaderOpt;
-
-            args = strip(args + 5);
-            if (*args != '(') {
-                continue;
-            }
-            args = strip(args+1);
-
-            NEXT_TOKEN(name)
-            NEXT_TOKEN(vname)
-            NEXT_TOKEN(type_params)
-            type_params_length = MIN(type_params_length, sizeof(type_str)-1);
-            memcpy(type_str, type_params, type_params_length);
-            type_str[type_params_length] = '\0';
-            type = vcard_emul_type_from_string(type_str);
-
-            NEXT_TOKEN(type_params)
-
-            if (*args == 0) {
-                break;
-            }
-
-            if (opts->vreader_count >= reader_count) {
-                reader_count += READER_STEP;
-                opts->vreader = g_renew(VirtualReaderOptions, opts->vreader,
-                                        reader_count);
-            }
-            vreaderOpt = &opts->vreader[opts->vreader_count];
-            vreaderOpt->name = g_strndup(name, name_length);
-            vreaderOpt->vname = g_strndup(vname, vname_length);
-            vreaderOpt->card_type = type;
-            vreaderOpt->type_params =
-                g_strndup(type_params, type_params_length);
-            count = count_tokens(args, ',', ')') + 1;
-            vreaderOpt->cert_count = count;
-            vreaderOpt->cert_name = g_new(char *, count);
-            for (i = 0; i < count; i++) {
-                const char *cert = args;
-                args = strpbrk(args, ",)");
-                vreaderOpt->cert_name[i] = g_strndup(cert, args - cert);
-                args = strip(args+1);
-            }
-            if (*args == ')') {
-                args++;
-            }
-            opts->vreader_count++;
-        /* use_hw= */
-        } else if (strncmp(args, "use_hw=", 7) == 0) {
-            args = strip(args+7);
-            if (*args == '0' || *args == 'N' || *args == 'n' || *args == 'F') {
-                opts->use_hw = PR_FALSE;
-            } else {
-                opts->use_hw = PR_TRUE;
-            }
-            args = find_blank(args);
-        /* hw_type= */
-        } else if (strncmp(args, "hw_type=", 8) == 0) {
-            args = strip(args+8);
-            opts->hw_card_type = vcard_emul_type_from_string(args);
-            args = find_blank(args);
-        /* hw_params= */
-        } else if (strncmp(args, "hw_params=", 10) == 0) {
-            const char *params;
-            args = strip(args+10);
-            params = args;
-            args = find_blank(args);
-            opts->hw_type_params = g_strndup(params, args-params);
-        /* db="/data/base/path" */
-        } else if (strncmp(args, "db=", 3) == 0) {
-            const char *db;
-            args = strip(args+3);
-            if (*args != '"') {
-                continue;
-            }
-            args++;
-            db = args;
-            args = strpbrk(args, "\"\n");
-            opts->nss_db = g_strndup(db, args-db);
-            if (*args != 0) {
-                args++;
-            }
-        } else {
-            args = find_blank(args);
-        }
-    } while (*args != 0);
-
-    return opts;
-}
-
-void
-vcard_emul_usage(void)
-{
-   fprintf(stderr,
-"emul args: comma separated list of the following arguments\n"
-" db={nss_database}               (default sql:/etc/pki/nssdb)\n"
-" use_hw=[yes|no]                 (default yes)\n"
-" hw_type={card_type_to_emulate}  (default CAC)\n"
-" hw_param={param_for_card}       (default \"\")\n"
-" soft=({slot_name},{vreader_name},{card_type_to_emulate},{params_for_card},\n"
-"       {cert1},{cert2},{cert3}    (default none)\n"
-"\n"
-"  {nss_database}          The location of the NSS cert & key database\n"
-"  {card_type_to_emulate}  What card interface to present to the guest\n"
-"  {param_for_card}        Card interface specific parameters\n"
-"  {slot_name}             NSS slot that contains the certs\n"
-"  {vreader_name}          Virtual reader name to present to the guest\n"
-"  {certN}                 Nickname of the certificate n on the virtual card\n"
-"\n"
-"These parameters come as a single string separated by blanks or newlines."
-"\n"
-"Unless use_hw is set to no, all tokens that look like removable hardware\n"
-"tokens will be presented to the guest using the emulator specified by\n"
-"hw_type, and parameters of hw_param.\n"
-"\n"
-"If more one or more soft= parameters are specified, these readers will be\n"
-"presented to the guest\n");
-}
diff --git a/libcacard/vcard_emul_type.c b/libcacard/vcard_emul_type.c
deleted file mode 100644
index 59a1458..0000000
--- a/libcacard/vcard_emul_type.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- *  This file contains utility functions which abstract the different card
- *  types.  The goal is that new card types can easily be added by simply
- *  changing this file and vcard_emul_type.h. It is currently not a requirement
- *  to dynamically add new card types.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include <strings.h>
-#include "vcardt.h"
-#include "vcard_emul_type.h"
-#include "cac.h"
-
-VCardStatus vcard_init(VReader *vreader, VCard *vcard,
-                       VCardEmulType type, const char *params,
-                       unsigned char *const *cert, int cert_len[],
-                       VCardKey *key[], int cert_count)
-{
-    switch (type) {
-    case VCARD_EMUL_NONE:
-        break;
-    case VCARD_EMUL_CAC:
-        return cac_card_init(vreader, vcard, params,
-                             cert, cert_len, key,  cert_count);
-    /* add new ones here */
-    default:
-        break;
-    }
-    return VCARD_FAIL;
-}
-
-VCardEmulType vcard_emul_type_select(VReader *vreader)
-{
-#ifdef notdef
-    /* since there is only one emulator no need to call this function */
-    if (cac_is_cac_card(vreader) == VCARD_DONE) {
-        return VCARD_EMUL_CAC;
-    }
-#endif
-    /* return the default */
-    return VCARD_EMUL_CAC;
-}
-
-VCardEmulType vcard_emul_type_from_string(const char *type_string)
-{
-     if (strcasecmp(type_string, "CAC") == 0) {
-        return VCARD_EMUL_CAC;
-     }
-#ifdef USE_PASSTHRU
-     if (strcasecmp(type_string, "PASSTHRU") == 0) {
-        return VCARD_EMUL_PASSTHRU;
-     }
-#endif
-     return VCARD_EMUL_NONE;
-}
diff --git a/libcacard/vcard_emul_type.h b/libcacard/vcard_emul_type.h
deleted file mode 100644
index 0242f40..0000000
--- a/libcacard/vcard_emul_type.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *  This header file abstracts the different card types. The goal is new card
- *  types can easily be added by simply changing this file and
- *  vcard_emul_type.c. It is currently not a requirement to dynamically add new
- *  card types.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VCARD_EMUL_TYPE_H
-#define VCARD_EMUL_TYPE_H 1
-#include "vcardt.h"
-#include "vreadert.h"
-
-/*
- * types
- */
-typedef enum {
-     VCARD_EMUL_NONE = 0,
-     VCARD_EMUL_CAC,
-     VCARD_EMUL_PASSTHRU
-} VCardEmulType;
-
-/* functions used by the rest of the emulator */
-VCardStatus vcard_init(VReader *vreader, VCard *vcard, VCardEmulType type,
-                       const char *params, unsigned char * const *cert,
-                       int cert_len[], VCardKey *key[], int cert_count);
-VCardEmulType vcard_emul_type_select(VReader *vreader);
-VCardEmulType vcard_emul_type_from_string(const char *type_string);
-
-#endif
diff --git a/libcacard/vcardt.c b/libcacard/vcardt.c
deleted file mode 100644
index 9ce4648..0000000
--- a/libcacard/vcardt.c
+++ /dev/null
@@ -1,40 +0,0 @@
-#include <stdlib.h>
-#include <string.h>
-#include <glib.h>
-
-#include "libcacard/vcardt.h"
-
-#include "libcacard/vcardt_internal.h"
-
-/* create an ATR with appropriate historical bytes */
-#define ATR_TS_DIRECT_CONVENTION 0x3b
-#define ATR_TA_PRESENT 0x10
-#define ATR_TB_PRESENT 0x20
-#define ATR_TC_PRESENT 0x40
-#define ATR_TD_PRESENT 0x80
-
-unsigned char *vcard_alloc_atr(const char *postfix, int *atr_len)
-{
-    int postfix_len;
-    const char prefix[] = "VCARD_";
-    const char default_postfix[] = "DEFAULT";
-    const int prefix_len = sizeof(prefix) - 1;
-    int total_len;
-    unsigned char *atr;
-
-    if (postfix == NULL) {
-        postfix = default_postfix;
-    }
-    postfix_len = strlen(postfix);
-    total_len = 3 + prefix_len + postfix_len;
-    atr = g_malloc(total_len);
-    atr[0] = ATR_TS_DIRECT_CONVENTION;
-    atr[1] = ATR_TD_PRESENT + prefix_len + postfix_len;
-    atr[2] = 0x00;
-    memcpy(&atr[3], prefix, prefix_len);
-    memcpy(&atr[3 + prefix_len], postfix, postfix_len);
-    if (atr_len) {
-        *atr_len = total_len;
-    }
-    return atr;
-}
diff --git a/libcacard/vcardt.h b/libcacard/vcardt.h
deleted file mode 100644
index 795e265..0000000
--- a/libcacard/vcardt.h
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef VCARDT_H
-#define VCARDT_H 1
-
-/*
- * these should come from some common spice header file
- */
-#include <assert.h>
-#ifndef MIN
-#define MIN(x, y) ((x) > (y) ? (y) : (x))
-#define MAX(x, y) ((x) > (y) ? (x) : (y))
-#endif
-
-typedef struct VCardStruct VCard;
-typedef struct VCardAPDUStruct VCardAPDU;
-typedef struct VCardResponseStruct VCardResponse;
-typedef struct VCardBufferResponseStruct VCardBufferResponse;
-typedef struct VCardAppletStruct VCardApplet;
-typedef struct VCardAppletPrivateStruct VCardAppletPrivate;
-typedef struct VCardKeyStruct VCardKey;  /* opaque */
-typedef struct VCardEmulStruct VCardEmul;
-
-#define MAX_CHANNEL 4
-
-typedef enum {
-    VCARD_DONE,
-    VCARD_NEXT,
-    VCARD_FAIL
-} VCardStatus;
-
-typedef enum {
-    VCARD_FILE_SYSTEM,
-    VCARD_VM,
-    VCARD_DIRECT
-} VCardType;
-
-typedef enum {
-    VCARD_POWER_ON,
-    VCARD_POWER_OFF
-} VCardPower;
-
-typedef VCardStatus (*VCardProcessAPDU)(VCard *card, VCardAPDU *apdu,
-                                        VCardResponse **response);
-typedef VCardStatus (*VCardResetApplet)(VCard *card, int channel);
-typedef void (*VCardAppletPrivateFree) (VCardAppletPrivate *);
-typedef void (*VCardEmulFree) (VCardEmul *);
-typedef void (*VCardGetAtr) (VCard *, unsigned char *atr, int *atr_len);
-
-struct VCardBufferResponseStruct {
-    unsigned char *buffer;
-    int buffer_len;
-    unsigned char *current;
-    int len;
-};
-
-#endif
diff --git a/libcacard/vcardt_internal.h b/libcacard/vcardt_internal.h
deleted file mode 100644
index e5c8d2d..0000000
--- a/libcacard/vcardt_internal.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef VCARDT_INTERNAL_H
-#define VCARDT_INTERNAL_H
-
-unsigned char *vcard_alloc_atr(const char *postfix, int *atr_len);
-
-#endif
diff --git a/libcacard/vevent.h b/libcacard/vevent.h
deleted file mode 100644
index 38c3482..0000000
--- a/libcacard/vevent.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-#ifndef EVENT_H
-#define EVENT_H 1
-#include "eventt.h"
-#include "vreadert.h"
-#include "vcardt.h"
-
-VEvent *vevent_new(VEventType type, VReader *reader, VCard *card);
-void vevent_delete(VEvent *);
-
-/*
- * VEvent queueing services
- */
-void vevent_queue_vevent(VEvent *);
-void vevent_queue_init(void);
-
-/*
- *  VEvent dequeing services
- */
-VEvent *vevent_wait_next_vevent(void);
-VEvent *vevent_get_next_vevent(void);
-
-
-#endif
diff --git a/libcacard/vreader.c b/libcacard/vreader.c
deleted file mode 100644
index 0315dd8..0000000
--- a/libcacard/vreader.c
+++ /dev/null
@@ -1,576 +0,0 @@
-/*
- * emulate the reader
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifdef G_LOG_DOMAIN
-#undef G_LOG_DOMAIN
-#endif
-#define G_LOG_DOMAIN "libcacard"
-
-#include "qemu-common.h"
-
-#include "vcard.h"
-#include "vcard_emul.h"
-#include "card_7816.h"
-#include "vreader.h"
-#include "vevent.h"
-#include "cac.h" /* just for debugging defines */
-
-#define LIBCACARD_LOG_DOMAIN "libcacard"
-
-struct VReaderStruct {
-    int    reference_count;
-    VCard *card;
-    char *name;
-    vreader_id_t id;
-    CompatGMutex lock;
-    VReaderEmul  *reader_private;
-    VReaderEmulFree reader_private_free;
-};
-
-/*
- * Debug helpers
- */
-
-static const char *
-apdu_ins_to_string(int ins)
-{
-    switch (ins) {
-    case VCARD7816_INS_MANAGE_CHANNEL:
-        return "manage channel";
-    case VCARD7816_INS_EXTERNAL_AUTHENTICATE:
-        return "external authenticate";
-    case VCARD7816_INS_GET_CHALLENGE:
-        return "get challenge";
-    case VCARD7816_INS_INTERNAL_AUTHENTICATE:
-        return "internal authenticate";
-    case VCARD7816_INS_ERASE_BINARY:
-        return "erase binary";
-    case VCARD7816_INS_READ_BINARY:
-        return "read binary";
-    case VCARD7816_INS_WRITE_BINARY:
-        return "write binary";
-    case VCARD7816_INS_UPDATE_BINARY:
-        return "update binary";
-    case VCARD7816_INS_READ_RECORD:
-        return "read record";
-    case VCARD7816_INS_WRITE_RECORD:
-        return "write record";
-    case VCARD7816_INS_UPDATE_RECORD:
-        return "update record";
-    case VCARD7816_INS_APPEND_RECORD:
-        return "append record";
-    case VCARD7816_INS_ENVELOPE:
-        return "envelope";
-    case VCARD7816_INS_PUT_DATA:
-        return "put data";
-    case VCARD7816_INS_GET_DATA:
-        return "get data";
-    case VCARD7816_INS_SELECT_FILE:
-        return "select file";
-    case VCARD7816_INS_VERIFY:
-        return "verify";
-    case VCARD7816_INS_GET_RESPONSE:
-        return "get response";
-    case CAC_GET_PROPERTIES:
-        return "get properties";
-    case CAC_GET_ACR:
-        return "get acr";
-    case CAC_READ_BUFFER:
-        return "read buffer";
-    case CAC_UPDATE_BUFFER:
-        return "update buffer";
-    case CAC_SIGN_DECRYPT:
-        return "sign decrypt";
-    case CAC_GET_CERTIFICATE:
-        return "get certificate";
-    }
-    return "unknown";
-}
-
-/* manage locking */
-static inline void
-vreader_lock(VReader *reader)
-{
-    g_mutex_lock(&reader->lock);
-}
-
-static inline void
-vreader_unlock(VReader *reader)
-{
-    g_mutex_unlock(&reader->lock);
-}
-
-/*
- * vreader constructor
- */
-VReader *
-vreader_new(const char *name, VReaderEmul *private,
-            VReaderEmulFree private_free)
-{
-    VReader *reader;
-
-    reader = g_new(VReader, 1);
-    g_mutex_init(&reader->lock);
-    reader->reference_count = 1;
-    reader->name = g_strdup(name);
-    reader->card = NULL;
-    reader->id = (vreader_id_t)-1;
-    reader->reader_private = private;
-    reader->reader_private_free = private_free;
-    return reader;
-}
-
-/* get a reference */
-VReader*
-vreader_reference(VReader *reader)
-{
-    if (reader == NULL) {
-        return NULL;
-    }
-    vreader_lock(reader);
-    reader->reference_count++;
-    vreader_unlock(reader);
-    return reader;
-}
-
-/* free a reference */
-void
-vreader_free(VReader *reader)
-{
-    if (reader == NULL) {
-        return;
-    }
-    vreader_lock(reader);
-    if (reader->reference_count-- > 1) {
-        vreader_unlock(reader);
-        return;
-    }
-    vreader_unlock(reader);
-    g_mutex_clear(&reader->lock);
-    if (reader->card) {
-        vcard_free(reader->card);
-    }
-    g_free(reader->name);
-    if (reader->reader_private_free) {
-        reader->reader_private_free(reader->reader_private);
-    }
-    g_free(reader);
-}
-
-static VCard *
-vreader_get_card(VReader *reader)
-{
-    VCard *card;
-
-    vreader_lock(reader);
-    card = vcard_reference(reader->card);
-    vreader_unlock(reader);
-    return card;
-}
-
-VReaderStatus
-vreader_card_is_present(VReader *reader)
-{
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-    vcard_free(card);
-    return VREADER_OK;
-}
-
-vreader_id_t
-vreader_get_id(VReader *reader)
-{
-    if (reader == NULL) {
-        return (vreader_id_t)-1;
-    }
-    return reader->id;
-}
-
-VReaderStatus
-vreader_set_id(VReader *reader, vreader_id_t id)
-{
-    if (reader == NULL) {
-        return VREADER_NO_CARD;
-    }
-    reader->id = id;
-    return VREADER_OK;
-}
-
-const char *
-vreader_get_name(VReader *reader)
-{
-    if (reader == NULL) {
-        return NULL;
-    }
-    return reader->name;
-}
-
-VReaderEmul *
-vreader_get_private(VReader *reader)
-{
-    return reader->reader_private;
-}
-
-static VReaderStatus
-vreader_reset(VReader *reader, VCardPower power, unsigned char *atr, int *len)
-{
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-    /*
-     * clean up our state
-     */
-    vcard_reset(card, power);
-    if (atr) {
-        vcard_get_atr(card, atr, len);
-    }
-    vcard_free(card); /* free our reference */
-    return VREADER_OK;
-}
-
-VReaderStatus
-vreader_power_on(VReader *reader, unsigned char *atr, int *len)
-{
-    return vreader_reset(reader, VCARD_POWER_ON, atr, len);
-}
-
-VReaderStatus
-vreader_power_off(VReader *reader)
-{
-    return vreader_reset(reader, VCARD_POWER_OFF, NULL, 0);
-}
-
-
-VReaderStatus
-vreader_xfr_bytes(VReader *reader,
-                  unsigned char *send_buf, int send_buf_len,
-                  unsigned char *receive_buf, int *receive_buf_len)
-{
-    VCardAPDU *apdu;
-    VCardResponse *response = NULL;
-    VCardStatus card_status;
-    unsigned short status;
-    VCard *card = vreader_get_card(reader);
-
-    if (card == NULL) {
-        return VREADER_NO_CARD;
-    }
-
-    apdu = vcard_apdu_new(send_buf, send_buf_len, &status);
-    if (apdu == NULL) {
-        response = vcard_make_response(status);
-        card_status = VCARD_DONE;
-    } else {
-        g_debug("%s: CLS=0x%x,INS=0x%x,P1=0x%x,P2=0x%x,Lc=%d,Le=%d %s",
-              __func__, apdu->a_cla, apdu->a_ins, apdu->a_p1, apdu->a_p2,
-              apdu->a_Lc, apdu->a_Le, apdu_ins_to_string(apdu->a_ins));
-        card_status = vcard_process_apdu(card, apdu, &response);
-        if (response) {
-            g_debug("%s: status=%d sw1=0x%x sw2=0x%x len=%d (total=%d)",
-                  __func__, response->b_status, response->b_sw1,
-                  response->b_sw2, response->b_len, response->b_total_len);
-        }
-    }
-    assert(card_status == VCARD_DONE && response);
-    int size = MIN(*receive_buf_len, response->b_total_len);
-    memcpy(receive_buf, response->b_data, size);
-    *receive_buf_len = size;
-    vcard_response_delete(response);
-    vcard_apdu_delete(apdu);
-    vcard_free(card); /* free our reference */
-    return VREADER_OK;
-}
-
-struct VReaderListStruct {
-    VReaderListEntry *head;
-    VReaderListEntry *tail;
-};
-
-struct VReaderListEntryStruct {
-    VReaderListEntry *next;
-    VReaderListEntry *prev;
-    VReader *reader;
-};
-
-
-static VReaderListEntry *
-vreader_list_entry_new(VReader *reader)
-{
-    VReaderListEntry *new_reader_list_entry;
-
-    new_reader_list_entry = g_new0(VReaderListEntry, 1);
-    new_reader_list_entry->reader = vreader_reference(reader);
-    return new_reader_list_entry;
-}
-
-static void
-vreader_list_entry_delete(VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    vreader_free(entry->reader);
-    g_free(entry);
-}
-
-
-static VReaderList *
-vreader_list_new(void)
-{
-    VReaderList *new_reader_list;
-
-    new_reader_list = g_new0(VReaderList, 1);
-    return new_reader_list;
-}
-
-void
-vreader_list_delete(VReaderList *list)
-{
-    VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry;
-    for (current_entry = vreader_list_get_first(list); current_entry;
-         current_entry = next_entry) {
-        next_entry = vreader_list_get_next(current_entry);
-        vreader_list_entry_delete(current_entry);
-    }
-    g_free(list);
-}
-
-
-VReaderListEntry *
-vreader_list_get_first(VReaderList *list)
-{
-    return list ? list->head : NULL;
-}
-
-VReaderListEntry *
-vreader_list_get_next(VReaderListEntry *current)
-{
-    return current ? current->next : NULL;
-}
-
-VReader *
-vreader_list_get_reader(VReaderListEntry *entry)
-{
-    return entry ? vreader_reference(entry->reader) : NULL;
-}
-
-static void
-vreader_queue(VReaderList *list, VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    entry->next = NULL;
-    entry->prev = list->tail;
-    if (list->head) {
-        list->tail->next = entry;
-    } else {
-        list->head = entry;
-    }
-    list->tail = entry;
-}
-
-static void
-vreader_dequeue(VReaderList *list, VReaderListEntry *entry)
-{
-    if (entry == NULL) {
-        return;
-    }
-    if (entry->next == NULL) {
-        list->tail = entry->prev;
-    } else if (entry->prev == NULL) {
-        list->head = entry->next;
-    } else {
-        entry->prev->next = entry->next;
-        entry->next->prev = entry->prev;
-    }
-    if ((list->tail == NULL) || (list->head == NULL)) {
-        list->head = list->tail = NULL;
-    }
-    entry->next = entry->prev = NULL;
-}
-
-static VReaderList *vreader_list;
-static CompatGMutex vreader_list_mutex;
-
-static void
-vreader_list_init(void)
-{
-    vreader_list = vreader_list_new();
-}
-
-static void
-vreader_list_lock(void)
-{
-    g_mutex_lock(&vreader_list_mutex);
-}
-
-static void
-vreader_list_unlock(void)
-{
-    g_mutex_unlock(&vreader_list_mutex);
-}
-
-static VReaderList *
-vreader_copy_list(VReaderList *list)
-{
-    VReaderList *new_list;
-    VReaderListEntry *current_entry;
-
-    new_list = vreader_list_new();
-    if (new_list == NULL) {
-        return NULL;
-    }
-    for (current_entry = vreader_list_get_first(list); current_entry;
-         current_entry = vreader_list_get_next(current_entry)) {
-        VReader *reader = vreader_list_get_reader(current_entry);
-        VReaderListEntry *new_entry = vreader_list_entry_new(reader);
-
-        vreader_free(reader);
-        vreader_queue(new_list, new_entry);
-    }
-    return new_list;
-}
-
-VReaderList *
-vreader_get_reader_list(void)
-{
-    VReaderList *new_reader_list;
-
-    vreader_list_lock();
-    new_reader_list = vreader_copy_list(vreader_list);
-    vreader_list_unlock();
-    return new_reader_list;
-}
-
-VReader *
-vreader_get_reader_by_id(vreader_id_t id)
-{
-    VReader *reader = NULL;
-    VReaderListEntry *current_entry;
-
-    if (id == (vreader_id_t) -1) {
-        return NULL;
-    }
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-            current_entry = vreader_list_get_next(current_entry)) {
-        VReader *creader = vreader_list_get_reader(current_entry);
-        if (creader->id == id) {
-            reader = creader;
-            break;
-        }
-        vreader_free(creader);
-    }
-    vreader_list_unlock();
-    return reader;
-}
-
-VReader *
-vreader_get_reader_by_name(const char *name)
-{
-    VReader *reader = NULL;
-    VReaderListEntry *current_entry;
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-            current_entry = vreader_list_get_next(current_entry)) {
-        VReader *creader = vreader_list_get_reader(current_entry);
-        if (strcmp(creader->name, name) == 0) {
-            reader = creader;
-            break;
-        }
-        vreader_free(creader);
-    }
-    vreader_list_unlock();
-    return reader;
-}
-
-/* called from card_emul to initialize the readers */
-VReaderStatus
-vreader_add_reader(VReader *reader)
-{
-    VReaderListEntry *reader_entry;
-
-    reader_entry = vreader_list_entry_new(reader);
-    if (reader_entry == NULL) {
-        return VREADER_OUT_OF_MEMORY;
-    }
-    vreader_list_lock();
-    vreader_queue(vreader_list, reader_entry);
-    vreader_list_unlock();
-    vevent_queue_vevent(vevent_new(VEVENT_READER_INSERT, reader, NULL));
-    return VREADER_OK;
-}
-
-
-VReaderStatus
-vreader_remove_reader(VReader *reader)
-{
-    VReaderListEntry *current_entry;
-
-    vreader_list_lock();
-    for (current_entry = vreader_list_get_first(vreader_list); current_entry;
-         current_entry = vreader_list_get_next(current_entry)) {
-        if (current_entry->reader == reader) {
-            break;
-        }
-    }
-    vreader_dequeue(vreader_list, current_entry);
-    vreader_list_unlock();
-    vreader_list_entry_delete(current_entry);
-    vevent_queue_vevent(vevent_new(VEVENT_READER_REMOVE, reader, NULL));
-    return VREADER_OK;
-}
-
-/*
- * Generate VEVENT_CARD_INSERT or VEVENT_CARD_REMOVE based on vreader
- * state. Separated from vreader_insert_card to allow replaying events
- * for a given state.
- */
-void
-vreader_queue_card_event(VReader *reader)
-{
-    vevent_queue_vevent(vevent_new(
-        reader->card ? VEVENT_CARD_INSERT : VEVENT_CARD_REMOVE, reader,
-        reader->card));
-}
-
-/*
- * insert/remove a new card. for removal, card == NULL
- */
-VReaderStatus
-vreader_insert_card(VReader *reader, VCard *card)
-{
-    vreader_lock(reader);
-    if (reader->card) {
-        /* decrement reference count */
-        vcard_free(reader->card);
-        reader->card = NULL;
-    }
-    reader->card = vcard_reference(card);
-    vreader_unlock(reader);
-    vreader_queue_card_event(reader);
-    return VREADER_OK;
-}
-
-/*
- * initialize all the static reader structures
- */
-void
-vreader_init(void)
-{
-    vreader_list_init();
-}
-
diff --git a/libcacard/vreader.h b/libcacard/vreader.h
deleted file mode 100644
index ec20421..0000000
--- a/libcacard/vreader.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VREADER_H
-#define VREADER_H 1
-
-#include "eventt.h"
-#include "vreadert.h"
-#include "vcardt.h"
-
-/*
- * calls for reader front end
- */
-VReaderStatus vreader_power_on(VReader *reader, unsigned char *atr, int *len);
-VReaderStatus vreader_power_off(VReader *reader);
-VReaderStatus vreader_xfr_bytes(VReader *reader, unsigned char *send_buf,
-                                int send_buf_len, unsigned char *receive_buf,
-                                int *receive_buf_len);
-
-/* constructor */
-VReader *vreader_new(const char *readerName, VReaderEmul *emul_private,
-                     VReaderEmulFree private_free);
-/* get a new reference to a reader */
-VReader *vreader_reference(VReader *reader);
-/* "destructor" (readers are reference counted) */
-void vreader_free(VReader *reader);
-
-/* accessors */
-VReaderEmul *vreader_get_private(VReader *);
-VReaderStatus vreader_card_is_present(VReader *reader);
-void vreader_queue_card_event(VReader *reader);
-const char *vreader_get_name(VReader *reader);
-vreader_id_t vreader_get_id(VReader *reader);
-VReaderStatus vreader_set_id(VReader *reader, vreader_id_t id);
-
-/* list operations */
-VReaderList *vreader_get_reader_list(void);
-void vreader_list_delete(VReaderList *list);
-VReader *vreader_list_get_reader(VReaderListEntry *entry);
-VReaderListEntry *vreader_list_get_first(VReaderList *list);
-VReaderListEntry *vreader_list_get_next(VReaderListEntry *list);
-VReader *vreader_get_reader_by_id(vreader_id_t id);
-VReader *vreader_get_reader_by_name(const char *name);
-
-/*
- * list tools for vcard_emul
- */
-void vreader_init(void);
-VReaderStatus vreader_add_reader(VReader *reader);
-VReaderStatus vreader_remove_reader(VReader *reader);
-VReaderStatus vreader_insert_card(VReader *reader, VCard *card);
-
-#endif
diff --git a/libcacard/vreadert.h b/libcacard/vreadert.h
deleted file mode 100644
index f97e0a7..0000000
--- a/libcacard/vreadert.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VREADERT_H
-#define VREADERT_H 1
-
-typedef enum {
-    VREADER_OK = 0,
-    VREADER_NO_CARD,
-    VREADER_OUT_OF_MEMORY
-} VReaderStatus;
-
-typedef unsigned int vreader_id_t;
-typedef struct VReaderStruct VReader;
-typedef struct VReaderListStruct VReaderList;
-typedef struct VReaderListEntryStruct VReaderListEntry;
-
-typedef struct VReaderEmulStruct VReaderEmul;
-typedef void (*VReaderEmulFree)(VReaderEmul *);
-
-#endif
-
diff --git a/libcacard/vscard_common.h b/libcacard/vscard_common.h
deleted file mode 100644
index 08f68e4..0000000
--- a/libcacard/vscard_common.h
+++ /dev/null
@@ -1,178 +0,0 @@
-/* Virtual Smart Card protocol definition
- *
- * This protocol is between a host using virtual smart card readers,
- * and a client providing the smart cards, perhaps by emulating them or by
- * access to real cards.
- *
- * Definitions for this protocol:
- *  Host   - user of the card
- *  Client - owner of the card
- *
- * The current implementation passes the raw APDU's from 7816 and additionally
- * contains messages to setup and teardown readers, handle insertion and
- * removal of cards, negotiate the protocol via capabilities and provide
- * for error responses.
- *
- * Copyright (c) 2011 Red Hat.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef VSCARD_COMMON_H
-#define VSCARD_COMMON_H
-
-#include <stdint.h>
-
-#define VERSION_MAJOR_BITS 11
-#define VERSION_MIDDLE_BITS 11
-#define VERSION_MINOR_BITS 10
-
-#define MAKE_VERSION(major, middle, minor) \
-     ((major  << (VERSION_MINOR_BITS + VERSION_MIDDLE_BITS)) \
-      | (middle <<  VERSION_MINOR_BITS) \
-      | (minor))
-
-/*
- * IMPORTANT NOTE on VERSION
- *
- * The version below MUST be changed whenever a change in this file is made.
- *
- * The last digit, the minor, is for bug fix changes only.
- *
- * The middle digit is for backward / forward compatible changes, updates
- * to the existing messages, addition of fields.
- *
- * The major digit is for a breaking change of protocol, presumably
- * something that cannot be accommodated with the existing protocol.
- */
-
-#define VSCARD_VERSION MAKE_VERSION(0, 0, 2)
-
-typedef enum VSCMsgType {
-    VSC_Init = 1,
-    VSC_Error,
-    VSC_ReaderAdd,
-    VSC_ReaderRemove,
-    VSC_ATR,
-    VSC_CardRemove,
-    VSC_APDU,
-    VSC_Flush,
-    VSC_FlushComplete
-} VSCMsgType;
-
-typedef enum VSCErrorCode {
-    VSC_SUCCESS = 0,
-    VSC_GENERAL_ERROR = 1,
-    VSC_CANNOT_ADD_MORE_READERS,
-    VSC_CARD_ALREAY_INSERTED,
-} VSCErrorCode;
-
-#define VSCARD_UNDEFINED_READER_ID  0xffffffff
-#define VSCARD_MINIMAL_READER_ID    0
-
-#define VSCARD_MAGIC (*(uint32_t *)"VSCD")
-
-/*
- * Header
- * Each message starts with the header.
- * type - message type
- * reader_id - used by messages that are reader specific
- * length - length of payload (not including header, i.e. zero for
- *  messages containing empty payloads)
- */
-typedef struct VSCMsgHeader {
-    uint32_t   type;
-    uint32_t   reader_id;
-    uint32_t   length;
-    uint8_t    data[0];
-} VSCMsgHeader;
-
-/*
- * VSCMsgInit               Client <-> Host
- * Client sends it on connection, with its own capabilities.
- * Host replies with VSCMsgInit filling in its capabilities.
- *
- * It is not meant to be used for negotiation, i.e. sending more then
- * once from any side, but could be used for that in the future.
- */
-typedef struct VSCMsgInit {
-    uint32_t   magic;
-    uint32_t   version;
-    uint32_t   capabilities[1]; /* receiver must check length,
-                                   array may grow in the future*/
-} VSCMsgInit;
-
-/*
- * VSCMsgError              Client <-> Host
- * This message is a response to any of:
- *  Reader Add
- *  Reader Remove
- *  Card Remove
- * If the operation was successful then VSC_SUCCESS
- * is returned, other wise a specific error code.
- */
-typedef struct VSCMsgError {
-    uint32_t   code;
-} VSCMsgError;
-
-/*
- * VSCMsgReaderAdd          Client -> Host
- * Host replies with allocated reader id in VSCMsgError with code==SUCCESS.
- *
- * name - name of the reader on client side, UTF-8 encoded. Only used
- *  for client presentation (may be translated to the device presented to the
- *  guest), protocol wise only reader_id is important.
- */
-typedef struct VSCMsgReaderAdd {
-    uint8_t    name[0];
-} VSCMsgReaderAdd;
-
-/*
- * VSCMsgReaderRemove       Client -> Host
- * The client's reader has been removed.
- */
-typedef struct VSCMsgReaderRemove {
-} VSCMsgReaderRemove;
-
-/*
- * VSCMsgATR                Client -> Host
- * Answer to reset. Sent for card insertion or card reset. The reset/insertion
- * happens on the client side, they do not require any action from the host.
- */
-typedef struct VSCMsgATR {
-    uint8_t     atr[0];
-} VSCMsgATR;
-
-/*
- * VSCMsgCardRemove         Client -> Host
- * The client card has been removed.
- */
-typedef struct VSCMsgCardRemove {
-} VSCMsgCardRemove;
-
-/*
- * VSCMsgAPDU               Client <-> Host
- * Main reason of existence. Transfer a single APDU in either direction.
- */
-typedef struct VSCMsgAPDU {
-    uint8_t    data[0];
-} VSCMsgAPDU;
-
-/*
- * VSCMsgFlush               Host -> Client
- * Request client to send a FlushComplete message when it is done
- * servicing all outstanding APDUs
- */
-typedef struct VSCMsgFlush {
-} VSCMsgFlush;
-
-/*
- * VSCMsgFlush               Client -> Host
- * Client response to Flush after all APDUs have been processed and
- * responses sent.
- */
-typedef struct VSCMsgFlushComplete {
-} VSCMsgFlushComplete;
-
-#endif /* VSCARD_COMMON_H */
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
deleted file mode 100644
index fa6041d..0000000
--- a/libcacard/vscclient.c
+++ /dev/null
@@ -1,779 +0,0 @@
-/*
- * Tester for VSCARD protocol, client side.
- *
- * Can be used with ccid-card-passthru.
- *
- * Copyright (c) 2011 Red Hat.
- * Written by Alon Levy.
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#ifndef _WIN32
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netdb.h>
-#define closesocket(x) close(x)
-#endif
-
-#include "qemu-common.h"
-
-#include "vscard_common.h"
-
-#include "vreader.h"
-#include "vcard_emul.h"
-#include "vevent.h"
-
-static int verbose;
-
-static void
-print_byte_array(
-    uint8_t *arrBytes,
-    unsigned int nSize
-) {
-    int i;
-    for (i = 0; i < nSize; i++) {
-        printf("%02X ", arrBytes[i]);
-    }
-    printf("\n");
-}
-
-static void
-print_usage(void) {
-    printf("vscclient [-c <certname> .. -e <emul_args> -d <level>%s] "
-            "<host> <port>\n",
-#ifdef USE_PASSTHRU
-    " -p");
-    printf(" -p use passthrough mode\n");
-#else
-   "");
-#endif
-    vcard_emul_usage();
-}
-
-static GIOChannel *channel_socket;
-static GByteArray *socket_to_send;
-static CompatGMutex socket_to_send_lock;
-static guint socket_tag;
-
-static void
-update_socket_watch(void);
-
-static gboolean
-do_socket_send(GIOChannel *source,
-               GIOCondition condition,
-               gpointer data)
-{
-    gsize bw;
-    GError *err = NULL;
-
-    g_return_val_if_fail(socket_to_send->len != 0, FALSE);
-    g_return_val_if_fail(condition & G_IO_OUT, FALSE);
-
-    g_io_channel_write_chars(channel_socket,
-        (gchar *)socket_to_send->data, socket_to_send->len, &bw, &err);
-    if (err != NULL) {
-        g_error("Error while sending socket %s", err->message);
-        return FALSE;
-    }
-    g_byte_array_remove_range(socket_to_send, 0, bw);
-
-    if (socket_to_send->len == 0) {
-        update_socket_watch();
-        return FALSE;
-    }
-    return TRUE;
-}
-
-static gboolean
-socket_prepare_sending(gpointer user_data)
-{
-    update_socket_watch();
-
-    return FALSE;
-}
-
-static int
-send_msg(
-    VSCMsgType type,
-    uint32_t reader_id,
-    const void *msg,
-    unsigned int length
-) {
-    VSCMsgHeader mhHeader;
-
-    g_mutex_lock(&socket_to_send_lock);
-
-    if (verbose > 10) {
-        printf("sending type=%d id=%u, len =%u (0x%x)\n",
-               type, reader_id, length, length);
-    }
-
-    mhHeader.type = htonl(type);
-    mhHeader.reader_id = 0;
-    mhHeader.length = htonl(length);
-    g_byte_array_append(socket_to_send, (guint8 *)&mhHeader, sizeof(mhHeader));
-    g_byte_array_append(socket_to_send, (guint8 *)msg, length);
-    g_idle_add(socket_prepare_sending, NULL);
-
-    g_mutex_unlock(&socket_to_send_lock);
-
-    return 0;
-}
-
-static VReader *pending_reader;
-static CompatGMutex pending_reader_lock;
-static CompatGCond pending_reader_condition;
-
-#define MAX_ATR_LEN 40
-static gpointer
-event_thread(gpointer arg)
-{
-    unsigned char atr[MAX_ATR_LEN];
-    int atr_len;
-    VEvent *event;
-    unsigned int reader_id;
-
-
-    while (1) {
-        const char *reader_name;
-
-        event = vevent_wait_next_vevent();
-        if (event == NULL) {
-            break;
-        }
-        reader_id = vreader_get_id(event->reader);
-        if (reader_id == VSCARD_UNDEFINED_READER_ID &&
-            event->type != VEVENT_READER_INSERT) {
-            /* ignore events from readers qemu has rejected */
-            /* if qemu is still deciding on this reader, wait to see if need to
-             * forward this event */
-            g_mutex_lock(&pending_reader_lock);
-            if (!pending_reader || (pending_reader != event->reader)) {
-                /* wasn't for a pending reader, this reader has already been
-                 * rejected by qemu */
-                g_mutex_unlock(&pending_reader_lock);
-                vevent_delete(event);
-                continue;
-            }
-            /* this reader hasn't been told its status from qemu yet, wait for
-             * that status */
-            while (pending_reader != NULL) {
-                g_cond_wait(&pending_reader_condition, &pending_reader_lock);
-            }
-            g_mutex_unlock(&pending_reader_lock);
-            /* now recheck the id */
-            reader_id = vreader_get_id(event->reader);
-            if (reader_id == VSCARD_UNDEFINED_READER_ID) {
-                /* this reader was rejected */
-                vevent_delete(event);
-                continue;
-            }
-            /* reader was accepted, now forward the event */
-        }
-        switch (event->type) {
-        case VEVENT_READER_INSERT:
-            /* tell qemu to insert a new CCID reader */
-            /* wait until qemu has responded to our first reader insert
-             * before we send a second. That way we won't confuse the responses
-             * */
-            g_mutex_lock(&pending_reader_lock);
-            while (pending_reader != NULL) {
-                g_cond_wait(&pending_reader_condition, &pending_reader_lock);
-            }
-            pending_reader = vreader_reference(event->reader);
-            g_mutex_unlock(&pending_reader_lock);
-            reader_name = vreader_get_name(event->reader);
-            if (verbose > 10) {
-                printf(" READER INSERT: %s\n", reader_name);
-            }
-            send_msg(VSC_ReaderAdd,
-                reader_id, /* currerntly VSCARD_UNDEFINED_READER_ID */
-                NULL, 0 /* TODO reader_name, strlen(reader_name) */);
-            break;
-        case VEVENT_READER_REMOVE:
-            /* future, tell qemu that an old CCID reader has been removed */
-            if (verbose > 10) {
-                printf(" READER REMOVE: %u\n", reader_id);
-            }
-            send_msg(VSC_ReaderRemove, reader_id, NULL, 0);
-            break;
-        case VEVENT_CARD_INSERT:
-            /* get the ATR (intended as a response to a power on from the
-             * reader */
-            atr_len = MAX_ATR_LEN;
-            vreader_power_on(event->reader, atr, &atr_len);
-            /* ATR call functions as a Card Insert event */
-            if (verbose > 10) {
-                printf(" CARD INSERT %u: ", reader_id);
-                print_byte_array(atr, atr_len);
-            }
-            send_msg(VSC_ATR, reader_id, atr, atr_len);
-            break;
-        case VEVENT_CARD_REMOVE:
-            /* Card removed */
-            if (verbose > 10) {
-                printf(" CARD REMOVE %u:\n", reader_id);
-            }
-            send_msg(VSC_CardRemove, reader_id, NULL, 0);
-            break;
-        default:
-            break;
-        }
-        vevent_delete(event);
-    }
-    return NULL;
-}
-
-
-static unsigned int
-get_id_from_string(char *string, unsigned int default_id)
-{
-    unsigned int id = atoi(string);
-
-    /* don't accidentally swith to zero because no numbers have been supplied */
-    if ((id == 0) && *string != '0') {
-        return default_id;
-    }
-    return id;
-}
-
-static int
-on_host_init(VSCMsgHeader *mhHeader, VSCMsgInit *incoming)
-{
-    uint32_t *capabilities = (incoming->capabilities);
-    int num_capabilities =
-        1 + ((mhHeader->length - sizeof(VSCMsgInit)) / sizeof(uint32_t));
-    int i;
-
-    incoming->version = ntohl(incoming->version);
-    if (incoming->version != VSCARD_VERSION) {
-        if (verbose > 0) {
-            printf("warning: host has version %d, we have %d\n",
-                verbose, VSCARD_VERSION);
-        }
-    }
-    if (incoming->magic != VSCARD_MAGIC) {
-        printf("unexpected magic: got %d, expected %d\n",
-            incoming->magic, VSCARD_MAGIC);
-        return -1;
-    }
-    for (i = 0 ; i < num_capabilities; ++i) {
-        capabilities[i] = ntohl(capabilities[i]);
-    }
-    /* Future: check capabilities */
-    /* remove whatever reader might be left in qemu,
-     * in case of an unclean previous exit. */
-    send_msg(VSC_ReaderRemove, VSCARD_MINIMAL_READER_ID, NULL, 0);
-    /* launch the event_thread. This will trigger reader adds for all the
-     * existing readers */
-    g_thread_new("vsc/event", event_thread, NULL);
-    return 0;
-}
-
-
-enum {
-    STATE_HEADER,
-    STATE_MESSAGE,
-};
-
-#define APDUBufSize 270
-
-static gboolean
-do_socket_read(GIOChannel *source,
-               GIOCondition condition,
-               gpointer data)
-{
-    int rv;
-    int dwSendLength;
-    int dwRecvLength;
-    uint8_t pbRecvBuffer[APDUBufSize];
-    static uint8_t pbSendBuffer[APDUBufSize];
-    VReaderStatus reader_status;
-    VReader *reader = NULL;
-    static VSCMsgHeader mhHeader;
-    VSCMsgError *error_msg;
-    GError *err = NULL;
-
-    static gchar *buf;
-    static gsize br, to_read;
-    static int state = STATE_HEADER;
-
-    if (state == STATE_HEADER && to_read == 0) {
-        buf = (gchar *)&mhHeader;
-        to_read = sizeof(mhHeader);
-    }
-
-    if (to_read > 0) {
-        g_io_channel_read_chars(source, (gchar *)buf, to_read, &br, &err);
-        if (err != NULL) {
-            g_error("error while reading: %s", err->message);
-        }
-        buf += br;
-        to_read -= br;
-        if (to_read != 0) {
-            return TRUE;
-        }
-    }
-
-    if (state == STATE_HEADER) {
-        mhHeader.type = ntohl(mhHeader.type);
-        mhHeader.reader_id = ntohl(mhHeader.reader_id);
-        mhHeader.length = ntohl(mhHeader.length);
-        if (verbose) {
-            printf("Header: type=%d, reader_id=%u length=%d (0x%x)\n",
-                   mhHeader.type, mhHeader.reader_id, mhHeader.length,
-                   mhHeader.length);
-        }
-        switch (mhHeader.type) {
-        case VSC_APDU:
-        case VSC_Flush:
-        case VSC_Error:
-        case VSC_Init:
-            buf = (gchar *)pbSendBuffer;
-            to_read = mhHeader.length;
-            state = STATE_MESSAGE;
-            return TRUE;
-        default:
-            fprintf(stderr, "Unexpected message of type 0x%X\n", mhHeader.type);
-            return FALSE;
-        }
-    }
-
-    if (state == STATE_MESSAGE) {
-        switch (mhHeader.type) {
-        case VSC_APDU:
-            if (verbose) {
-                printf(" recv APDU: ");
-                print_byte_array(pbSendBuffer, mhHeader.length);
-            }
-            /* Transmit received APDU */
-            dwSendLength = mhHeader.length;
-            dwRecvLength = sizeof(pbRecvBuffer);
-            reader = vreader_get_reader_by_id(mhHeader.reader_id);
-            reader_status = vreader_xfr_bytes(reader,
-                                              pbSendBuffer, dwSendLength,
-                                              pbRecvBuffer, &dwRecvLength);
-            if (reader_status == VREADER_OK) {
-                mhHeader.length = dwRecvLength;
-                if (verbose) {
-                    printf(" send response: ");
-                    print_byte_array(pbRecvBuffer, mhHeader.length);
-                }
-                send_msg(VSC_APDU, mhHeader.reader_id,
-                         pbRecvBuffer, dwRecvLength);
-            } else {
-                rv = reader_status; /* warning: not meaningful */
-                send_msg(VSC_Error, mhHeader.reader_id, &rv, sizeof(uint32_t));
-            }
-            vreader_free(reader);
-            reader = NULL; /* we've freed it, don't use it by accident
-                              again */
-            break;
-        case VSC_Flush:
-            /* TODO: actually flush */
-            send_msg(VSC_FlushComplete, mhHeader.reader_id, NULL, 0);
-            break;
-        case VSC_Error:
-            error_msg = (VSCMsgError *) pbSendBuffer;
-            if (error_msg->code == VSC_SUCCESS) {
-                g_mutex_lock(&pending_reader_lock);
-                if (pending_reader) {
-                    vreader_set_id(pending_reader, mhHeader.reader_id);
-                    vreader_free(pending_reader);
-                    pending_reader = NULL;
-                    g_cond_signal(&pending_reader_condition);
-                }
-                g_mutex_unlock(&pending_reader_lock);
-                break;
-            }
-            printf("warning: qemu refused to add reader\n");
-            if (error_msg->code == VSC_CANNOT_ADD_MORE_READERS) {
-                /* clear pending reader, qemu can't handle any more */
-                g_mutex_lock(&pending_reader_lock);
-                if (pending_reader) {
-                    pending_reader = NULL;
-                    /* make sure the event loop doesn't hang */
-                    g_cond_signal(&pending_reader_condition);
-                }
-                g_mutex_unlock(&pending_reader_lock);
-            }
-            break;
-        case VSC_Init:
-            if (on_host_init(&mhHeader, (VSCMsgInit *)pbSendBuffer) < 0) {
-                return FALSE;
-            }
-            break;
-        default:
-            g_assert_not_reached();
-            return FALSE;
-        }
-
-        state = STATE_HEADER;
-    }
-
-
-    return TRUE;
-}
-
-static gboolean
-do_socket(GIOChannel *source,
-          GIOCondition condition,
-          gpointer data)
-{
-    /* not sure if two watches work well with a single win32 sources */
-    if (condition & G_IO_OUT) {
-        if (!do_socket_send(source, condition, data)) {
-            return FALSE;
-        }
-    }
-
-    if (condition & G_IO_IN) {
-        if (!do_socket_read(source, condition, data)) {
-            return FALSE;
-        }
-    }
-
-    return TRUE;
-}
-
-static void
-update_socket_watch(void)
-{
-    gboolean out = socket_to_send->len > 0;
-
-    if (socket_tag != 0) {
-        g_source_remove(socket_tag);
-    }
-
-    socket_tag = g_io_add_watch(channel_socket,
-        G_IO_IN | (out ? G_IO_OUT : 0), do_socket, NULL);
-}
-
-static gboolean
-do_command(GIOChannel *source,
-           GIOCondition condition,
-           gpointer data)
-{
-    char *string;
-    VCardEmulError error;
-    static unsigned int default_reader_id;
-    unsigned int reader_id;
-    VReader *reader = NULL;
-    GError *err = NULL;
-
-    g_assert(condition & G_IO_IN);
-
-    reader_id = default_reader_id;
-    g_io_channel_read_line(source, &string, NULL, NULL, &err);
-    if (err != NULL) {
-        g_error("Error while reading command: %s", err->message);
-    }
-
-    if (string != NULL) {
-        if (strncmp(string, "exit", 4) == 0) {
-            /* remove all the readers */
-            VReaderList *list = vreader_get_reader_list();
-            VReaderListEntry *reader_entry;
-            printf("Active Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id == -1) {
-                    continue;
-                }
-                /* be nice and signal card removal first (qemu probably should
-                 * do this itself) */
-                if (vreader_card_is_present(reader) == VREADER_OK) {
-                    send_msg(VSC_CardRemove, reader_id, NULL, 0);
-                }
-                send_msg(VSC_ReaderRemove, reader_id, NULL, 0);
-            }
-            exit(0);
-        } else if (strncmp(string, "insert", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7], reader_id);
-            }
-            reader = vreader_get_reader_by_id(reader_id);
-            if (reader != NULL) {
-                error = vcard_emul_force_card_insert(reader);
-                printf("insert %s, returned %d\n",
-                       vreader_get_name(reader), error);
-            } else {
-                printf("no reader by id %u found\n", reader_id);
-            }
-        } else if (strncmp(string, "remove", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7], reader_id);
-            }
-            reader = vreader_get_reader_by_id(reader_id);
-            if (reader != NULL) {
-                error = vcard_emul_force_card_remove(reader);
-                printf("remove %s, returned %d\n",
-                       vreader_get_name(reader), error);
-            } else {
-                printf("no reader by id %u found\n", reader_id);
-            }
-        } else if (strncmp(string, "select", 6) == 0) {
-            if (string[6] == ' ') {
-                reader_id = get_id_from_string(&string[7],
-                                               VSCARD_UNDEFINED_READER_ID);
-            }
-            if (reader_id != VSCARD_UNDEFINED_READER_ID) {
-                reader = vreader_get_reader_by_id(reader_id);
-            }
-            if (reader) {
-                printf("Selecting reader %u, %s\n", reader_id,
-                        vreader_get_name(reader));
-                default_reader_id = reader_id;
-            } else {
-                printf("Reader with id %u not found\n", reader_id);
-            }
-        } else if (strncmp(string, "debug", 5) == 0) {
-            if (string[5] == ' ') {
-                verbose = get_id_from_string(&string[6], 0);
-            }
-            printf("debug level = %d\n", verbose);
-        } else if (strncmp(string, "list", 4) == 0) {
-            VReaderList *list = vreader_get_reader_list();
-            VReaderListEntry *reader_entry;
-            printf("Active Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id == -1) {
-                    continue;
-                }
-                printf("%3u %s %s\n", reader_id,
-                       vreader_card_is_present(reader) == VREADER_OK ?
-                       "CARD_PRESENT" : "            ",
-                       vreader_get_name(reader));
-            }
-            printf("Inactive Readers:\n");
-            for (reader_entry = vreader_list_get_first(list); reader_entry;
-                 reader_entry = vreader_list_get_next(reader_entry)) {
-                VReader *reader = vreader_list_get_reader(reader_entry);
-                vreader_id_t reader_id;
-                reader_id = vreader_get_id(reader);
-                if (reader_id != -1) {
-                    continue;
-                }
-
-                printf("INA %s %s\n",
-                       vreader_card_is_present(reader) == VREADER_OK ?
-                       "CARD_PRESENT" : "            ",
-                       vreader_get_name(reader));
-            }
-            vreader_list_delete(list);
-        } else if (*string != 0) {
-            printf("valid commands:\n");
-            printf("insert [reader_id]\n");
-            printf("remove [reader_id]\n");
-            printf("select reader_id\n");
-            printf("list\n");
-            printf("debug [level]\n");
-            printf("exit\n");
-        }
-    }
-    vreader_free(reader);
-    printf("> ");
-    fflush(stdout);
-
-    return TRUE;
-}
-
-
-/* just for ease of parsing command line arguments. */
-#define MAX_CERTS 100
-
-static int
-connect_to_qemu(
-    const char *host,
-    const char *port
-) {
-    struct addrinfo hints;
-    struct addrinfo *server = NULL;
-    int ret, sock;
-
-    sock = socket(AF_INET, SOCK_STREAM, 0);
-    if (sock < 0) {
-        /* Error */
-        fprintf(stderr, "Error opening socket!\n");
-        return -1;
-    }
-
-    memset(&hints, 0, sizeof(struct addrinfo));
-    hints.ai_family = AF_UNSPEC;
-    hints.ai_socktype = SOCK_STREAM;
-    hints.ai_flags = 0;
-    hints.ai_protocol = 0;          /* Any protocol */
-
-    ret = getaddrinfo(host, port, &hints, &server);
-
-    if (ret != 0) {
-        /* Error */
-        fprintf(stderr, "getaddrinfo failed\n");
-        goto cleanup_socket;
-    }
-
-    if (connect(sock, server->ai_addr, server->ai_addrlen) < 0) {
-        /* Error */
-        fprintf(stderr, "Could not connect\n");
-        goto cleanup_socket;
-    }
-    if (verbose) {
-        printf("Connected (sizeof Header=%zd)!\n", sizeof(VSCMsgHeader));
-    }
-
-    freeaddrinfo(server);
-    return sock;
-
-cleanup_socket:
-    if (server) {
-        freeaddrinfo(server);
-    }
-    closesocket(sock);
-    return -1;
-}
-
-int
-main(
-    int argc,
-    char *argv[]
-) {
-    GMainLoop *loop;
-    GIOChannel *channel_stdin;
-    char *qemu_host;
-    char *qemu_port;
-
-    VCardEmulOptions *command_line_options = NULL;
-
-    char *cert_names[MAX_CERTS];
-    char *emul_args = NULL;
-    int cert_count = 0;
-    int c, sock;
-
-#ifdef _WIN32
-    WSADATA Data;
-
-    if (WSAStartup(MAKEWORD(2, 2), &Data) != 0) {
-        c = WSAGetLastError();
-        fprintf(stderr, "WSAStartup: %d\n", c);
-        return 1;
-    }
-#endif
-#if !GLIB_CHECK_VERSION(2, 31, 0)
-    if (!g_thread_supported()) {
-         g_thread_init(NULL);
-    }
-#endif
-
-    while ((c = getopt(argc, argv, "c:e:pd:")) != -1) {
-        switch (c) {
-        case 'c':
-            if (cert_count >= MAX_CERTS) {
-                printf("too many certificates (max = %d)\n", MAX_CERTS);
-                exit(5);
-            }
-            cert_names[cert_count++] = optarg;
-            break;
-        case 'e':
-            emul_args = optarg;
-            break;
-        case 'p':
-            print_usage();
-            exit(4);
-            break;
-        case 'd':
-            verbose = get_id_from_string(optarg, 1);
-            break;
-        }
-    }
-
-    if (argc - optind != 2) {
-        print_usage();
-        exit(4);
-    }
-
-    if (cert_count > 0) {
-        char *new_args;
-        int len, i;
-        /* if we've given some -c options, we clearly we want do so some
-         * software emulation.  add that emulation now. this is NSS Emulator
-         * specific */
-        if (emul_args == NULL) {
-            emul_args = (char *)"db=\"/etc/pki/nssdb\"";
-        }
-#define SOFT_STRING ",soft=(,Virtual Reader,CAC,,"
-             /* 2 == close paren & null */
-        len = strlen(emul_args) + strlen(SOFT_STRING) + 2;
-        for (i = 0; i < cert_count; i++) {
-            len += strlen(cert_names[i])+1; /* 1 == comma */
-        }
-        new_args = g_malloc(len);
-        strcpy(new_args, emul_args);
-        strcat(new_args, SOFT_STRING);
-        for (i = 0; i < cert_count; i++) {
-            strcat(new_args, cert_names[i]);
-            strcat(new_args, ",");
-        }
-        strcat(new_args, ")");
-        emul_args = new_args;
-    }
-    if (emul_args) {
-        command_line_options = vcard_emul_options(emul_args);
-    }
-
-    qemu_host = g_strdup(argv[argc - 2]);
-    qemu_port = g_strdup(argv[argc - 1]);
-    sock = connect_to_qemu(qemu_host, qemu_port);
-    if (sock == -1) {
-        fprintf(stderr, "error opening socket, exiting.\n");
-        exit(5);
-    }
-
-    socket_to_send = g_byte_array_new();
-    vcard_emul_init(command_line_options);
-    loop = g_main_loop_new(NULL, TRUE);
-
-    printf("> ");
-    fflush(stdout);
-
-#ifdef _WIN32
-    channel_stdin = g_io_channel_win32_new_fd(STDIN_FILENO);
-#else
-    channel_stdin = g_io_channel_unix_new(STDIN_FILENO);
-#endif
-    g_io_add_watch(channel_stdin, G_IO_IN, do_command, NULL);
-#ifdef _WIN32
-    channel_socket = g_io_channel_win32_new_socket(sock);
-#else
-    channel_socket = g_io_channel_unix_new(sock);
-#endif
-    g_io_channel_set_encoding(channel_socket, NULL, NULL);
-    /* we buffer ourself for thread safety reasons */
-    g_io_channel_set_buffered(channel_socket, FALSE);
-
-    /* Send init message, Host responds (and then we send reader attachments) */
-    VSCMsgInit init = {
-        .version = htonl(VSCARD_VERSION),
-        .magic = VSCARD_MAGIC,
-        .capabilities = {0}
-    };
-    send_msg(VSC_Init, 0, &init, sizeof(init));
-
-    g_main_loop_run(loop);
-    g_main_loop_unref(loop);
-
-    g_io_channel_unref(channel_stdin);
-    g_io_channel_unref(channel_socket);
-    g_byte_array_free(socket_to_send, TRUE);
-
-    closesocket(sock);
-    return 0;
-}
diff --git a/libdecnumber/decContext.c b/libdecnumber/decContext.c
index 8b6ae21..7d97a65 100644
--- a/libdecnumber/decContext.c
+++ b/libdecnumber/decContext.c
@@ -35,8 +35,7 @@
 /* context structures.						      */
 /* ------------------------------------------------------------------ */
 
-#include <string.h>	      /* for strcmp */
-#include <stdio.h>	      /* for printf if DECCHECK */
+#include "qemu/osdep.h"
 #include "libdecnumber/dconfig.h"
 #include "libdecnumber/decContext.h"
 #include "libdecnumber/decNumberLocal.h"
@@ -186,7 +185,7 @@
 /*  newstatus is the source for the bits to be restored		      */
 /*  mask indicates the bits to be restored (the status bit that	      */
 /*    corresponds to each 1 bit in the mask is set to the value of    */
-/*    the correspnding bit in newstatus)			      */
+/*    the corresponding bit in newstatus)			      */
 /*  returns context						      */
 /*								      */
 /* No error is possible.					      */
diff --git a/libdecnumber/decNumber.c b/libdecnumber/decNumber.c
index 58211e7..c9e7807 100644
--- a/libdecnumber/decNumber.c
+++ b/libdecnumber/decNumber.c
@@ -166,10 +166,7 @@
 /*	**  -- raise to the power				      */
 /* ------------------------------------------------------------------ */
 
-#include <stdlib.h>		   /* for malloc, free, etc. */
-#include <stdio.h>		   /* for printf [if needed] */
-#include <string.h>		   /* for strcpy */
-#include <ctype.h>		   /* for lower */
+#include "qemu/osdep.h"
 #include "libdecnumber/dconfig.h"
 #include "libdecnumber/decNumber.h"
 #include "libdecnumber/decNumberLocal.h"
@@ -2238,7 +2235,7 @@
       /* if a negative power the constant 1 is needed, and if not subset */
       /* invert the lhs now rather than inverting the result later */
       if (decNumberIsNegative(rhs)) {	/* was a **-n [hence digits>0] */
-	decNumber *inv=invbuff;		/* asssume use fixed buffer */
+	decNumber *inv=invbuff;		/* assume use fixed buffer */
 	decNumberCopy(&dnOne, dac);	/* dnOne=1;  [needed now or later] */
 	#if DECSUBSET
 	if (set->extended) {		/* need to calculate 1/lhs */
@@ -3849,7 +3846,7 @@
 /*								      */
 /* Addition, especially x=x+1, is speed-critical.		      */
 /* The static buffer is larger than might be expected to allow for    */
-/* calls from higher-level funtions (notable exp).		      */
+/* calls from higher-level functions (notably exp).		      */
 /* ------------------------------------------------------------------ */
 static decNumber * decAddOp(decNumber *res, const decNumber *lhs,
 			    const decNumber *rhs, decContext *set,
@@ -4263,7 +4260,7 @@
 /* long subtractions.  These are acc and var1 respectively.	      */
 /* var1 is a copy of the lhs coefficient, var2 is the rhs coefficient.*/
 /* The static buffers may be larger than might be expected to allow   */
-/* for calls from higher-level funtions (notable exp).		      */
+/* for calls from higher-level functions (notably exp).		      */
 /* ------------------------------------------------------------------ */
 static decNumber * decDivideOp(decNumber *res,
 			       const decNumber *lhs, const decNumber *rhs,
@@ -5254,7 +5251,7 @@
 /*    exp(-x) where x can be the tiniest number (Ntiny).	      */
 /*								      */
 /* 2. Normalizing x to be <=0.1 (instead of <=1) reduces loop	      */
-/*    iterations by appoximately a third with additional (although    */
+/*    iterations by approximately a third with additional (although    */
 /*    diminishing) returns as the range is reduced to even smaller    */
 /*    fractions.  However, h (the power of 10 used to correct the     */
 /*    result at the end, see below) must be kept <=8 as otherwise     */
diff --git a/libdecnumber/dpd/decimal128.c b/libdecnumber/dpd/decimal128.c
index 7551b7c..ca4764e 100644
--- a/libdecnumber/dpd/decimal128.c
+++ b/libdecnumber/dpd/decimal128.c
@@ -39,8 +39,7 @@
 /*								      */
 /* Error handling is the same as decNumber (qv.).		      */
 /* ------------------------------------------------------------------ */
-#include <string.h>	      /* [for memset/memcpy] */
-#include <stdio.h>	      /* [for printf] */
+#include "qemu/osdep.h"
 
 #include "libdecnumber/dconfig.h"
 #define	 DECNUMDIGITS 34      /* make decNumbers with space for 34 */
diff --git a/libdecnumber/dpd/decimal128Local.h b/libdecnumber/dpd/decimal128Local.h
deleted file mode 100644
index 1963678..0000000
--- a/libdecnumber/dpd/decimal128Local.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* Local definitions for use with the decNumber C Library.
-   Copyright (C) 2007, 2009 Free Software Foundation, Inc.
-
-   This file is part of GCC.
-
-   GCC is free software; you can redistribute it and/or modify it under
-   the terms of the GNU General Public License as published by the Free
-   Software Foundation; either version 3, or (at your option) any later
-   version.
-
-   GCC is distributed in the hope that it will be useful, but WITHOUT ANY
-   WARRANTY; without even the implied warranty of MERCHANTABILITY or
-   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-   for more details.
-
-Under Section 7 of GPL version 3, you are granted additional
-permissions described in the GCC Runtime Library Exception, version
-3.1, as published by the Free Software Foundation.
-
-You should have received a copy of the GNU General Public License and
-a copy of the GCC Runtime Library Exception along with this program;
-see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
-<http://www.gnu.org/licenses/>.  */
-
-#if !defined(DECIMAL128LOCAL)
-
-/* The compiler needs sign manipulation functions for decimal128 which
-   are not part of the decNumber package.  */
-
-/* Set sign; this assumes the sign was previously zero.  */
-#define decimal128SetSign(d,b) \
-  { (d)->bytes[WORDS_BIGENDIAN ? 0 : 15] |= ((unsigned) (b) << 7); }
-
-/* Clear sign.  */
-#define decimal128ClearSign(d) \
-  { (d)->bytes[WORDS_BIGENDIAN ? 0 : 15] &= ~0x80; }
-
-/* Flip sign.  */
-#define decimal128FlipSign(d) \
-  { (d)->bytes[WORDS_BIGENDIAN ? 0 : 15] ^= 0x80; }
-
-#endif
diff --git a/libdecnumber/dpd/decimal32.c b/libdecnumber/dpd/decimal32.c
index 095ab75..53f2978 100644
--- a/libdecnumber/dpd/decimal32.c
+++ b/libdecnumber/dpd/decimal32.c
@@ -39,8 +39,7 @@
 /*								      */
 /* Error handling is the same as decNumber (qv.).		      */
 /* ------------------------------------------------------------------ */
-#include <string.h>	      /* [for memset/memcpy] */
-#include <stdio.h>	      /* [for printf] */
+#include "qemu/osdep.h"
 
 #include "libdecnumber/dconfig.h"
 #define	 DECNUMDIGITS  7      /* make decNumbers with space for 7 */
diff --git a/libdecnumber/dpd/decimal64.c b/libdecnumber/dpd/decimal64.c
index 8256084..4816176 100644
--- a/libdecnumber/dpd/decimal64.c
+++ b/libdecnumber/dpd/decimal64.c
@@ -39,8 +39,7 @@
 /*								      */
 /* Error handling is the same as decNumber (qv.).		      */
 /* ------------------------------------------------------------------ */
-#include <string.h>	      /* [for memset/memcpy] */
-#include <stdio.h>	      /* [for printf] */
+#include "qemu/osdep.h"
 
 #include "libdecnumber/dconfig.h"
 #define	 DECNUMDIGITS 16      /* make decNumbers with space for 16 */
diff --git a/linux-headers/asm-arm/kvm.h b/linux-headers/asm-arm/kvm.h
index 09ee408..c98e4dc 100644
--- a/linux-headers/asm-arm/kvm.h
+++ b/linux-headers/asm-arm/kvm.h
@@ -175,6 +175,8 @@
 #define   KVM_DEV_ARM_VGIC_OFFSET_SHIFT	0
 #define   KVM_DEV_ARM_VGIC_OFFSET_MASK	(0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
 #define KVM_DEV_ARM_VGIC_GRP_NR_IRQS	3
+#define KVM_DEV_ARM_VGIC_GRP_CTRL       4
+#define   KVM_DEV_ARM_VGIC_CTRL_INIT    0
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
@@ -193,9 +195,16 @@
 #define KVM_ARM_IRQ_CPU_IRQ		0
 #define KVM_ARM_IRQ_CPU_FIQ		1
 
-/* Highest supported SPI, from VGIC_NR_IRQS */
+/*
+ * This used to hold the highest supported SPI, but it is now obsolete
+ * and only here to provide source code level compatibility with older
+ * userland. The highest SPI number can be set via KVM_DEV_ARM_VGIC_GRP_NR_IRQS.
+ */
 #define KVM_ARM_IRQ_GIC_MAX		127
 
+/* One single KVM irqchip, ie. the VGIC */
+#define KVM_NR_IRQCHIPS          1
+
 /* PSCI interface */
 #define KVM_PSCI_FN_BASE		0x95c1ba5e
 #define KVM_PSCI_FN(n)			(KVM_PSCI_FN_BASE + (n))
diff --git a/linux-headers/asm-arm/unistd.h b/linux-headers/asm-arm/unistd.h
new file mode 100644
index 0000000..ceb5450
--- /dev/null
+++ b/linux-headers/asm-arm/unistd.h
@@ -0,0 +1,452 @@
+/*
+ *  arch/arm/include/asm/unistd.h
+ *
+ *  Copyright (C) 2001-2005 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Please forward _all_ changes to this file to rmk@arm.linux.org.uk,
+ * no matter what the change is.  Thanks!
+ */
+#ifndef __ASM_ARM_UNISTD_H
+#define __ASM_ARM_UNISTD_H
+
+#define __NR_OABI_SYSCALL_BASE	0x900000
+
+#if defined(__thumb__) || defined(__ARM_EABI__)
+#define __NR_SYSCALL_BASE	0
+#else
+#define __NR_SYSCALL_BASE	__NR_OABI_SYSCALL_BASE
+#endif
+
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_restart_syscall		(__NR_SYSCALL_BASE+  0)
+#define __NR_exit			(__NR_SYSCALL_BASE+  1)
+#define __NR_fork			(__NR_SYSCALL_BASE+  2)
+#define __NR_read			(__NR_SYSCALL_BASE+  3)
+#define __NR_write			(__NR_SYSCALL_BASE+  4)
+#define __NR_open			(__NR_SYSCALL_BASE+  5)
+#define __NR_close			(__NR_SYSCALL_BASE+  6)
+					/* 7 was sys_waitpid */
+#define __NR_creat			(__NR_SYSCALL_BASE+  8)
+#define __NR_link			(__NR_SYSCALL_BASE+  9)
+#define __NR_unlink			(__NR_SYSCALL_BASE+ 10)
+#define __NR_execve			(__NR_SYSCALL_BASE+ 11)
+#define __NR_chdir			(__NR_SYSCALL_BASE+ 12)
+#define __NR_time			(__NR_SYSCALL_BASE+ 13)
+#define __NR_mknod			(__NR_SYSCALL_BASE+ 14)
+#define __NR_chmod			(__NR_SYSCALL_BASE+ 15)
+#define __NR_lchown			(__NR_SYSCALL_BASE+ 16)
+					/* 17 was sys_break */
+					/* 18 was sys_stat */
+#define __NR_lseek			(__NR_SYSCALL_BASE+ 19)
+#define __NR_getpid			(__NR_SYSCALL_BASE+ 20)
+#define __NR_mount			(__NR_SYSCALL_BASE+ 21)
+#define __NR_umount			(__NR_SYSCALL_BASE+ 22)
+#define __NR_setuid			(__NR_SYSCALL_BASE+ 23)
+#define __NR_getuid			(__NR_SYSCALL_BASE+ 24)
+#define __NR_stime			(__NR_SYSCALL_BASE+ 25)
+#define __NR_ptrace			(__NR_SYSCALL_BASE+ 26)
+#define __NR_alarm			(__NR_SYSCALL_BASE+ 27)
+					/* 28 was sys_fstat */
+#define __NR_pause			(__NR_SYSCALL_BASE+ 29)
+#define __NR_utime			(__NR_SYSCALL_BASE+ 30)
+					/* 31 was sys_stty */
+					/* 32 was sys_gtty */
+#define __NR_access			(__NR_SYSCALL_BASE+ 33)
+#define __NR_nice			(__NR_SYSCALL_BASE+ 34)
+					/* 35 was sys_ftime */
+#define __NR_sync			(__NR_SYSCALL_BASE+ 36)
+#define __NR_kill			(__NR_SYSCALL_BASE+ 37)
+#define __NR_rename			(__NR_SYSCALL_BASE+ 38)
+#define __NR_mkdir			(__NR_SYSCALL_BASE+ 39)
+#define __NR_rmdir			(__NR_SYSCALL_BASE+ 40)
+#define __NR_dup			(__NR_SYSCALL_BASE+ 41)
+#define __NR_pipe			(__NR_SYSCALL_BASE+ 42)
+#define __NR_times			(__NR_SYSCALL_BASE+ 43)
+					/* 44 was sys_prof */
+#define __NR_brk			(__NR_SYSCALL_BASE+ 45)
+#define __NR_setgid			(__NR_SYSCALL_BASE+ 46)
+#define __NR_getgid			(__NR_SYSCALL_BASE+ 47)
+					/* 48 was sys_signal */
+#define __NR_geteuid			(__NR_SYSCALL_BASE+ 49)
+#define __NR_getegid			(__NR_SYSCALL_BASE+ 50)
+#define __NR_acct			(__NR_SYSCALL_BASE+ 51)
+#define __NR_umount2			(__NR_SYSCALL_BASE+ 52)
+					/* 53 was sys_lock */
+#define __NR_ioctl			(__NR_SYSCALL_BASE+ 54)
+#define __NR_fcntl			(__NR_SYSCALL_BASE+ 55)
+					/* 56 was sys_mpx */
+#define __NR_setpgid			(__NR_SYSCALL_BASE+ 57)
+					/* 58 was sys_ulimit */
+					/* 59 was sys_olduname */
+#define __NR_umask			(__NR_SYSCALL_BASE+ 60)
+#define __NR_chroot			(__NR_SYSCALL_BASE+ 61)
+#define __NR_ustat			(__NR_SYSCALL_BASE+ 62)
+#define __NR_dup2			(__NR_SYSCALL_BASE+ 63)
+#define __NR_getppid			(__NR_SYSCALL_BASE+ 64)
+#define __NR_getpgrp			(__NR_SYSCALL_BASE+ 65)
+#define __NR_setsid			(__NR_SYSCALL_BASE+ 66)
+#define __NR_sigaction			(__NR_SYSCALL_BASE+ 67)
+					/* 68 was sys_sgetmask */
+					/* 69 was sys_ssetmask */
+#define __NR_setreuid			(__NR_SYSCALL_BASE+ 70)
+#define __NR_setregid			(__NR_SYSCALL_BASE+ 71)
+#define __NR_sigsuspend			(__NR_SYSCALL_BASE+ 72)
+#define __NR_sigpending			(__NR_SYSCALL_BASE+ 73)
+#define __NR_sethostname		(__NR_SYSCALL_BASE+ 74)
+#define __NR_setrlimit			(__NR_SYSCALL_BASE+ 75)
+#define __NR_getrlimit			(__NR_SYSCALL_BASE+ 76)	/* Back compat 2GB limited rlimit */
+#define __NR_getrusage			(__NR_SYSCALL_BASE+ 77)
+#define __NR_gettimeofday		(__NR_SYSCALL_BASE+ 78)
+#define __NR_settimeofday		(__NR_SYSCALL_BASE+ 79)
+#define __NR_getgroups			(__NR_SYSCALL_BASE+ 80)
+#define __NR_setgroups			(__NR_SYSCALL_BASE+ 81)
+#define __NR_select			(__NR_SYSCALL_BASE+ 82)
+#define __NR_symlink			(__NR_SYSCALL_BASE+ 83)
+					/* 84 was sys_lstat */
+#define __NR_readlink			(__NR_SYSCALL_BASE+ 85)
+#define __NR_uselib			(__NR_SYSCALL_BASE+ 86)
+#define __NR_swapon			(__NR_SYSCALL_BASE+ 87)
+#define __NR_reboot			(__NR_SYSCALL_BASE+ 88)
+#define __NR_readdir			(__NR_SYSCALL_BASE+ 89)
+#define __NR_mmap			(__NR_SYSCALL_BASE+ 90)
+#define __NR_munmap			(__NR_SYSCALL_BASE+ 91)
+#define __NR_truncate			(__NR_SYSCALL_BASE+ 92)
+#define __NR_ftruncate			(__NR_SYSCALL_BASE+ 93)
+#define __NR_fchmod			(__NR_SYSCALL_BASE+ 94)
+#define __NR_fchown			(__NR_SYSCALL_BASE+ 95)
+#define __NR_getpriority		(__NR_SYSCALL_BASE+ 96)
+#define __NR_setpriority		(__NR_SYSCALL_BASE+ 97)
+					/* 98 was sys_profil */
+#define __NR_statfs			(__NR_SYSCALL_BASE+ 99)
+#define __NR_fstatfs			(__NR_SYSCALL_BASE+100)
+					/* 101 was sys_ioperm */
+#define __NR_socketcall			(__NR_SYSCALL_BASE+102)
+#define __NR_syslog			(__NR_SYSCALL_BASE+103)
+#define __NR_setitimer			(__NR_SYSCALL_BASE+104)
+#define __NR_getitimer			(__NR_SYSCALL_BASE+105)
+#define __NR_stat			(__NR_SYSCALL_BASE+106)
+#define __NR_lstat			(__NR_SYSCALL_BASE+107)
+#define __NR_fstat			(__NR_SYSCALL_BASE+108)
+					/* 109 was sys_uname */
+					/* 110 was sys_iopl */
+#define __NR_vhangup			(__NR_SYSCALL_BASE+111)
+					/* 112 was sys_idle */
+#define __NR_syscall			(__NR_SYSCALL_BASE+113) /* syscall to call a syscall! */
+#define __NR_wait4			(__NR_SYSCALL_BASE+114)
+#define __NR_swapoff			(__NR_SYSCALL_BASE+115)
+#define __NR_sysinfo			(__NR_SYSCALL_BASE+116)
+#define __NR_ipc			(__NR_SYSCALL_BASE+117)
+#define __NR_fsync			(__NR_SYSCALL_BASE+118)
+#define __NR_sigreturn			(__NR_SYSCALL_BASE+119)
+#define __NR_clone			(__NR_SYSCALL_BASE+120)
+#define __NR_setdomainname		(__NR_SYSCALL_BASE+121)
+#define __NR_uname			(__NR_SYSCALL_BASE+122)
+					/* 123 was sys_modify_ldt */
+#define __NR_adjtimex			(__NR_SYSCALL_BASE+124)
+#define __NR_mprotect			(__NR_SYSCALL_BASE+125)
+#define __NR_sigprocmask		(__NR_SYSCALL_BASE+126)
+					/* 127 was sys_create_module */
+#define __NR_init_module		(__NR_SYSCALL_BASE+128)
+#define __NR_delete_module		(__NR_SYSCALL_BASE+129)
+					/* 130 was sys_get_kernel_syms */
+#define __NR_quotactl			(__NR_SYSCALL_BASE+131)
+#define __NR_getpgid			(__NR_SYSCALL_BASE+132)
+#define __NR_fchdir			(__NR_SYSCALL_BASE+133)
+#define __NR_bdflush			(__NR_SYSCALL_BASE+134)
+#define __NR_sysfs			(__NR_SYSCALL_BASE+135)
+#define __NR_personality		(__NR_SYSCALL_BASE+136)
+					/* 137 was sys_afs_syscall */
+#define __NR_setfsuid			(__NR_SYSCALL_BASE+138)
+#define __NR_setfsgid			(__NR_SYSCALL_BASE+139)
+#define __NR__llseek			(__NR_SYSCALL_BASE+140)
+#define __NR_getdents			(__NR_SYSCALL_BASE+141)
+#define __NR__newselect			(__NR_SYSCALL_BASE+142)
+#define __NR_flock			(__NR_SYSCALL_BASE+143)
+#define __NR_msync			(__NR_SYSCALL_BASE+144)
+#define __NR_readv			(__NR_SYSCALL_BASE+145)
+#define __NR_writev			(__NR_SYSCALL_BASE+146)
+#define __NR_getsid			(__NR_SYSCALL_BASE+147)
+#define __NR_fdatasync			(__NR_SYSCALL_BASE+148)
+#define __NR__sysctl			(__NR_SYSCALL_BASE+149)
+#define __NR_mlock			(__NR_SYSCALL_BASE+150)
+#define __NR_munlock			(__NR_SYSCALL_BASE+151)
+#define __NR_mlockall			(__NR_SYSCALL_BASE+152)
+#define __NR_munlockall			(__NR_SYSCALL_BASE+153)
+#define __NR_sched_setparam		(__NR_SYSCALL_BASE+154)
+#define __NR_sched_getparam		(__NR_SYSCALL_BASE+155)
+#define __NR_sched_setscheduler		(__NR_SYSCALL_BASE+156)
+#define __NR_sched_getscheduler		(__NR_SYSCALL_BASE+157)
+#define __NR_sched_yield		(__NR_SYSCALL_BASE+158)
+#define __NR_sched_get_priority_max	(__NR_SYSCALL_BASE+159)
+#define __NR_sched_get_priority_min	(__NR_SYSCALL_BASE+160)
+#define __NR_sched_rr_get_interval	(__NR_SYSCALL_BASE+161)
+#define __NR_nanosleep			(__NR_SYSCALL_BASE+162)
+#define __NR_mremap			(__NR_SYSCALL_BASE+163)
+#define __NR_setresuid			(__NR_SYSCALL_BASE+164)
+#define __NR_getresuid			(__NR_SYSCALL_BASE+165)
+					/* 166 was sys_vm86 */
+					/* 167 was sys_query_module */
+#define __NR_poll			(__NR_SYSCALL_BASE+168)
+#define __NR_nfsservctl			(__NR_SYSCALL_BASE+169)
+#define __NR_setresgid			(__NR_SYSCALL_BASE+170)
+#define __NR_getresgid			(__NR_SYSCALL_BASE+171)
+#define __NR_prctl			(__NR_SYSCALL_BASE+172)
+#define __NR_rt_sigreturn		(__NR_SYSCALL_BASE+173)
+#define __NR_rt_sigaction		(__NR_SYSCALL_BASE+174)
+#define __NR_rt_sigprocmask		(__NR_SYSCALL_BASE+175)
+#define __NR_rt_sigpending		(__NR_SYSCALL_BASE+176)
+#define __NR_rt_sigtimedwait		(__NR_SYSCALL_BASE+177)
+#define __NR_rt_sigqueueinfo		(__NR_SYSCALL_BASE+178)
+#define __NR_rt_sigsuspend		(__NR_SYSCALL_BASE+179)
+#define __NR_pread64			(__NR_SYSCALL_BASE+180)
+#define __NR_pwrite64			(__NR_SYSCALL_BASE+181)
+#define __NR_chown			(__NR_SYSCALL_BASE+182)
+#define __NR_getcwd			(__NR_SYSCALL_BASE+183)
+#define __NR_capget			(__NR_SYSCALL_BASE+184)
+#define __NR_capset			(__NR_SYSCALL_BASE+185)
+#define __NR_sigaltstack		(__NR_SYSCALL_BASE+186)
+#define __NR_sendfile			(__NR_SYSCALL_BASE+187)
+					/* 188 reserved */
+					/* 189 reserved */
+#define __NR_vfork			(__NR_SYSCALL_BASE+190)
+#define __NR_ugetrlimit			(__NR_SYSCALL_BASE+191)	/* SuS compliant getrlimit */
+#define __NR_mmap2			(__NR_SYSCALL_BASE+192)
+#define __NR_truncate64			(__NR_SYSCALL_BASE+193)
+#define __NR_ftruncate64		(__NR_SYSCALL_BASE+194)
+#define __NR_stat64			(__NR_SYSCALL_BASE+195)
+#define __NR_lstat64			(__NR_SYSCALL_BASE+196)
+#define __NR_fstat64			(__NR_SYSCALL_BASE+197)
+#define __NR_lchown32			(__NR_SYSCALL_BASE+198)
+#define __NR_getuid32			(__NR_SYSCALL_BASE+199)
+#define __NR_getgid32			(__NR_SYSCALL_BASE+200)
+#define __NR_geteuid32			(__NR_SYSCALL_BASE+201)
+#define __NR_getegid32			(__NR_SYSCALL_BASE+202)
+#define __NR_setreuid32			(__NR_SYSCALL_BASE+203)
+#define __NR_setregid32			(__NR_SYSCALL_BASE+204)
+#define __NR_getgroups32		(__NR_SYSCALL_BASE+205)
+#define __NR_setgroups32		(__NR_SYSCALL_BASE+206)
+#define __NR_fchown32			(__NR_SYSCALL_BASE+207)
+#define __NR_setresuid32		(__NR_SYSCALL_BASE+208)
+#define __NR_getresuid32		(__NR_SYSCALL_BASE+209)
+#define __NR_setresgid32		(__NR_SYSCALL_BASE+210)
+#define __NR_getresgid32		(__NR_SYSCALL_BASE+211)
+#define __NR_chown32			(__NR_SYSCALL_BASE+212)
+#define __NR_setuid32			(__NR_SYSCALL_BASE+213)
+#define __NR_setgid32			(__NR_SYSCALL_BASE+214)
+#define __NR_setfsuid32			(__NR_SYSCALL_BASE+215)
+#define __NR_setfsgid32			(__NR_SYSCALL_BASE+216)
+#define __NR_getdents64			(__NR_SYSCALL_BASE+217)
+#define __NR_pivot_root			(__NR_SYSCALL_BASE+218)
+#define __NR_mincore			(__NR_SYSCALL_BASE+219)
+#define __NR_madvise			(__NR_SYSCALL_BASE+220)
+#define __NR_fcntl64			(__NR_SYSCALL_BASE+221)
+					/* 222 for tux */
+					/* 223 is unused */
+#define __NR_gettid			(__NR_SYSCALL_BASE+224)
+#define __NR_readahead			(__NR_SYSCALL_BASE+225)
+#define __NR_setxattr			(__NR_SYSCALL_BASE+226)
+#define __NR_lsetxattr			(__NR_SYSCALL_BASE+227)
+#define __NR_fsetxattr			(__NR_SYSCALL_BASE+228)
+#define __NR_getxattr			(__NR_SYSCALL_BASE+229)
+#define __NR_lgetxattr			(__NR_SYSCALL_BASE+230)
+#define __NR_fgetxattr			(__NR_SYSCALL_BASE+231)
+#define __NR_listxattr			(__NR_SYSCALL_BASE+232)
+#define __NR_llistxattr			(__NR_SYSCALL_BASE+233)
+#define __NR_flistxattr			(__NR_SYSCALL_BASE+234)
+#define __NR_removexattr		(__NR_SYSCALL_BASE+235)
+#define __NR_lremovexattr		(__NR_SYSCALL_BASE+236)
+#define __NR_fremovexattr		(__NR_SYSCALL_BASE+237)
+#define __NR_tkill			(__NR_SYSCALL_BASE+238)
+#define __NR_sendfile64			(__NR_SYSCALL_BASE+239)
+#define __NR_futex			(__NR_SYSCALL_BASE+240)
+#define __NR_sched_setaffinity		(__NR_SYSCALL_BASE+241)
+#define __NR_sched_getaffinity		(__NR_SYSCALL_BASE+242)
+#define __NR_io_setup			(__NR_SYSCALL_BASE+243)
+#define __NR_io_destroy			(__NR_SYSCALL_BASE+244)
+#define __NR_io_getevents		(__NR_SYSCALL_BASE+245)
+#define __NR_io_submit			(__NR_SYSCALL_BASE+246)
+#define __NR_io_cancel			(__NR_SYSCALL_BASE+247)
+#define __NR_exit_group			(__NR_SYSCALL_BASE+248)
+#define __NR_lookup_dcookie		(__NR_SYSCALL_BASE+249)
+#define __NR_epoll_create		(__NR_SYSCALL_BASE+250)
+#define __NR_epoll_ctl			(__NR_SYSCALL_BASE+251)
+#define __NR_epoll_wait			(__NR_SYSCALL_BASE+252)
+#define __NR_remap_file_pages		(__NR_SYSCALL_BASE+253)
+					/* 254 for set_thread_area */
+					/* 255 for get_thread_area */
+#define __NR_set_tid_address		(__NR_SYSCALL_BASE+256)
+#define __NR_timer_create		(__NR_SYSCALL_BASE+257)
+#define __NR_timer_settime		(__NR_SYSCALL_BASE+258)
+#define __NR_timer_gettime		(__NR_SYSCALL_BASE+259)
+#define __NR_timer_getoverrun		(__NR_SYSCALL_BASE+260)
+#define __NR_timer_delete		(__NR_SYSCALL_BASE+261)
+#define __NR_clock_settime		(__NR_SYSCALL_BASE+262)
+#define __NR_clock_gettime		(__NR_SYSCALL_BASE+263)
+#define __NR_clock_getres		(__NR_SYSCALL_BASE+264)
+#define __NR_clock_nanosleep		(__NR_SYSCALL_BASE+265)
+#define __NR_statfs64			(__NR_SYSCALL_BASE+266)
+#define __NR_fstatfs64			(__NR_SYSCALL_BASE+267)
+#define __NR_tgkill			(__NR_SYSCALL_BASE+268)
+#define __NR_utimes			(__NR_SYSCALL_BASE+269)
+#define __NR_arm_fadvise64_64		(__NR_SYSCALL_BASE+270)
+#define __NR_pciconfig_iobase		(__NR_SYSCALL_BASE+271)
+#define __NR_pciconfig_read		(__NR_SYSCALL_BASE+272)
+#define __NR_pciconfig_write		(__NR_SYSCALL_BASE+273)
+#define __NR_mq_open			(__NR_SYSCALL_BASE+274)
+#define __NR_mq_unlink			(__NR_SYSCALL_BASE+275)
+#define __NR_mq_timedsend		(__NR_SYSCALL_BASE+276)
+#define __NR_mq_timedreceive		(__NR_SYSCALL_BASE+277)
+#define __NR_mq_notify			(__NR_SYSCALL_BASE+278)
+#define __NR_mq_getsetattr		(__NR_SYSCALL_BASE+279)
+#define __NR_waitid			(__NR_SYSCALL_BASE+280)
+#define __NR_socket			(__NR_SYSCALL_BASE+281)
+#define __NR_bind			(__NR_SYSCALL_BASE+282)
+#define __NR_connect			(__NR_SYSCALL_BASE+283)
+#define __NR_listen			(__NR_SYSCALL_BASE+284)
+#define __NR_accept			(__NR_SYSCALL_BASE+285)
+#define __NR_getsockname		(__NR_SYSCALL_BASE+286)
+#define __NR_getpeername		(__NR_SYSCALL_BASE+287)
+#define __NR_socketpair			(__NR_SYSCALL_BASE+288)
+#define __NR_send			(__NR_SYSCALL_BASE+289)
+#define __NR_sendto			(__NR_SYSCALL_BASE+290)
+#define __NR_recv			(__NR_SYSCALL_BASE+291)
+#define __NR_recvfrom			(__NR_SYSCALL_BASE+292)
+#define __NR_shutdown			(__NR_SYSCALL_BASE+293)
+#define __NR_setsockopt			(__NR_SYSCALL_BASE+294)
+#define __NR_getsockopt			(__NR_SYSCALL_BASE+295)
+#define __NR_sendmsg			(__NR_SYSCALL_BASE+296)
+#define __NR_recvmsg			(__NR_SYSCALL_BASE+297)
+#define __NR_semop			(__NR_SYSCALL_BASE+298)
+#define __NR_semget			(__NR_SYSCALL_BASE+299)
+#define __NR_semctl			(__NR_SYSCALL_BASE+300)
+#define __NR_msgsnd			(__NR_SYSCALL_BASE+301)
+#define __NR_msgrcv			(__NR_SYSCALL_BASE+302)
+#define __NR_msgget			(__NR_SYSCALL_BASE+303)
+#define __NR_msgctl			(__NR_SYSCALL_BASE+304)
+#define __NR_shmat			(__NR_SYSCALL_BASE+305)
+#define __NR_shmdt			(__NR_SYSCALL_BASE+306)
+#define __NR_shmget			(__NR_SYSCALL_BASE+307)
+#define __NR_shmctl			(__NR_SYSCALL_BASE+308)
+#define __NR_add_key			(__NR_SYSCALL_BASE+309)
+#define __NR_request_key		(__NR_SYSCALL_BASE+310)
+#define __NR_keyctl			(__NR_SYSCALL_BASE+311)
+#define __NR_semtimedop			(__NR_SYSCALL_BASE+312)
+#define __NR_vserver			(__NR_SYSCALL_BASE+313)
+#define __NR_ioprio_set			(__NR_SYSCALL_BASE+314)
+#define __NR_ioprio_get			(__NR_SYSCALL_BASE+315)
+#define __NR_inotify_init		(__NR_SYSCALL_BASE+316)
+#define __NR_inotify_add_watch		(__NR_SYSCALL_BASE+317)
+#define __NR_inotify_rm_watch		(__NR_SYSCALL_BASE+318)
+#define __NR_mbind			(__NR_SYSCALL_BASE+319)
+#define __NR_get_mempolicy		(__NR_SYSCALL_BASE+320)
+#define __NR_set_mempolicy		(__NR_SYSCALL_BASE+321)
+#define __NR_openat			(__NR_SYSCALL_BASE+322)
+#define __NR_mkdirat			(__NR_SYSCALL_BASE+323)
+#define __NR_mknodat			(__NR_SYSCALL_BASE+324)
+#define __NR_fchownat			(__NR_SYSCALL_BASE+325)
+#define __NR_futimesat			(__NR_SYSCALL_BASE+326)
+#define __NR_fstatat64			(__NR_SYSCALL_BASE+327)
+#define __NR_unlinkat			(__NR_SYSCALL_BASE+328)
+#define __NR_renameat			(__NR_SYSCALL_BASE+329)
+#define __NR_linkat			(__NR_SYSCALL_BASE+330)
+#define __NR_symlinkat			(__NR_SYSCALL_BASE+331)
+#define __NR_readlinkat			(__NR_SYSCALL_BASE+332)
+#define __NR_fchmodat			(__NR_SYSCALL_BASE+333)
+#define __NR_faccessat			(__NR_SYSCALL_BASE+334)
+#define __NR_pselect6			(__NR_SYSCALL_BASE+335)
+#define __NR_ppoll			(__NR_SYSCALL_BASE+336)
+#define __NR_unshare			(__NR_SYSCALL_BASE+337)
+#define __NR_set_robust_list		(__NR_SYSCALL_BASE+338)
+#define __NR_get_robust_list		(__NR_SYSCALL_BASE+339)
+#define __NR_splice			(__NR_SYSCALL_BASE+340)
+#define __NR_arm_sync_file_range	(__NR_SYSCALL_BASE+341)
+#define __NR_sync_file_range2		__NR_arm_sync_file_range
+#define __NR_tee			(__NR_SYSCALL_BASE+342)
+#define __NR_vmsplice			(__NR_SYSCALL_BASE+343)
+#define __NR_move_pages			(__NR_SYSCALL_BASE+344)
+#define __NR_getcpu			(__NR_SYSCALL_BASE+345)
+#define __NR_epoll_pwait		(__NR_SYSCALL_BASE+346)
+#define __NR_kexec_load			(__NR_SYSCALL_BASE+347)
+#define __NR_utimensat			(__NR_SYSCALL_BASE+348)
+#define __NR_signalfd			(__NR_SYSCALL_BASE+349)
+#define __NR_timerfd_create		(__NR_SYSCALL_BASE+350)
+#define __NR_eventfd			(__NR_SYSCALL_BASE+351)
+#define __NR_fallocate			(__NR_SYSCALL_BASE+352)
+#define __NR_timerfd_settime		(__NR_SYSCALL_BASE+353)
+#define __NR_timerfd_gettime		(__NR_SYSCALL_BASE+354)
+#define __NR_signalfd4			(__NR_SYSCALL_BASE+355)
+#define __NR_eventfd2			(__NR_SYSCALL_BASE+356)
+#define __NR_epoll_create1		(__NR_SYSCALL_BASE+357)
+#define __NR_dup3			(__NR_SYSCALL_BASE+358)
+#define __NR_pipe2			(__NR_SYSCALL_BASE+359)
+#define __NR_inotify_init1		(__NR_SYSCALL_BASE+360)
+#define __NR_preadv			(__NR_SYSCALL_BASE+361)
+#define __NR_pwritev			(__NR_SYSCALL_BASE+362)
+#define __NR_rt_tgsigqueueinfo		(__NR_SYSCALL_BASE+363)
+#define __NR_perf_event_open		(__NR_SYSCALL_BASE+364)
+#define __NR_recvmmsg			(__NR_SYSCALL_BASE+365)
+#define __NR_accept4			(__NR_SYSCALL_BASE+366)
+#define __NR_fanotify_init		(__NR_SYSCALL_BASE+367)
+#define __NR_fanotify_mark		(__NR_SYSCALL_BASE+368)
+#define __NR_prlimit64			(__NR_SYSCALL_BASE+369)
+#define __NR_name_to_handle_at		(__NR_SYSCALL_BASE+370)
+#define __NR_open_by_handle_at		(__NR_SYSCALL_BASE+371)
+#define __NR_clock_adjtime		(__NR_SYSCALL_BASE+372)
+#define __NR_syncfs			(__NR_SYSCALL_BASE+373)
+#define __NR_sendmmsg			(__NR_SYSCALL_BASE+374)
+#define __NR_setns			(__NR_SYSCALL_BASE+375)
+#define __NR_process_vm_readv		(__NR_SYSCALL_BASE+376)
+#define __NR_process_vm_writev		(__NR_SYSCALL_BASE+377)
+#define __NR_kcmp			(__NR_SYSCALL_BASE+378)
+#define __NR_finit_module		(__NR_SYSCALL_BASE+379)
+#define __NR_sched_setattr		(__NR_SYSCALL_BASE+380)
+#define __NR_sched_getattr		(__NR_SYSCALL_BASE+381)
+#define __NR_renameat2			(__NR_SYSCALL_BASE+382)
+#define __NR_seccomp			(__NR_SYSCALL_BASE+383)
+#define __NR_getrandom			(__NR_SYSCALL_BASE+384)
+#define __NR_memfd_create		(__NR_SYSCALL_BASE+385)
+#define __NR_bpf			(__NR_SYSCALL_BASE+386)
+#define __NR_execveat			(__NR_SYSCALL_BASE+387)
+#define __NR_userfaultfd		(__NR_SYSCALL_BASE+388)
+#define __NR_membarrier			(__NR_SYSCALL_BASE+389)
+#define __NR_mlock2			(__NR_SYSCALL_BASE+390)
+#define __NR_copy_file_range		(__NR_SYSCALL_BASE+391)
+#define __NR_preadv2			(__NR_SYSCALL_BASE+392)
+#define __NR_pwritev2			(__NR_SYSCALL_BASE+393)
+
+/*
+ * The following SWIs are ARM private.
+ */
+#define __ARM_NR_BASE			(__NR_SYSCALL_BASE+0x0f0000)
+#define __ARM_NR_breakpoint		(__ARM_NR_BASE+1)
+#define __ARM_NR_cacheflush		(__ARM_NR_BASE+2)
+#define __ARM_NR_usr26			(__ARM_NR_BASE+3)
+#define __ARM_NR_usr32			(__ARM_NR_BASE+4)
+#define __ARM_NR_set_tls		(__ARM_NR_BASE+5)
+
+/*
+ * The following syscalls are obsolete and no longer available for EABI.
+ */
+#if defined(__ARM_EABI__)
+#undef __NR_time
+#undef __NR_umount
+#undef __NR_stime
+#undef __NR_alarm
+#undef __NR_utime
+#undef __NR_getrlimit
+#undef __NR_select
+#undef __NR_readdir
+#undef __NR_mmap
+#undef __NR_socketcall
+#undef __NR_syscall
+#undef __NR_ipc
+#endif
+
+#endif /* __ASM_ARM_UNISTD_H */
diff --git a/linux-headers/asm-arm64/kvm.h b/linux-headers/asm-arm64/kvm.h
index 8e38878..7d82d1f 100644
--- a/linux-headers/asm-arm64/kvm.h
+++ b/linux-headers/asm-arm64/kvm.h
@@ -32,7 +32,7 @@
 
 #ifndef __ASSEMBLY__
 #include <linux/psci.h>
-#include <asm/types.h>
+#include <linux/types.h>
 #include <asm/ptrace.h>
 
 #define __KVM_HAVE_GUEST_DEBUG
@@ -53,14 +53,20 @@
 	struct user_fpsimd_state fp_regs;
 };
 
-/* Supported Processor Types */
+/*
+ * Supported CPU Targets - Adding a new target type is not recommended,
+ * unless there are some special registers not supported by the
+ * genericv8 syreg table.
+ */
 #define KVM_ARM_TARGET_AEM_V8		0
 #define KVM_ARM_TARGET_FOUNDATION_V8	1
 #define KVM_ARM_TARGET_CORTEX_A57	2
 #define KVM_ARM_TARGET_XGENE_POTENZA	3
 #define KVM_ARM_TARGET_CORTEX_A53	4
+/* Generic ARM v8 target */
+#define KVM_ARM_TARGET_GENERIC_V8	5
 
-#define KVM_ARM_NUM_TARGETS		5
+#define KVM_ARM_NUM_TARGETS		6
 
 /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */
 #define KVM_ARM_DEVICE_TYPE_SHIFT	0
@@ -78,9 +84,17 @@
 #define KVM_VGIC_V2_DIST_SIZE		0x1000
 #define KVM_VGIC_V2_CPU_SIZE		0x2000
 
+/* Supported VGICv3 address types  */
+#define KVM_VGIC_V3_ADDR_TYPE_DIST	2
+#define KVM_VGIC_V3_ADDR_TYPE_REDIST	3
+
+#define KVM_VGIC_V3_DIST_SIZE		SZ_64K
+#define KVM_VGIC_V3_REDIST_SIZE		(2 * SZ_64K)
+
 #define KVM_ARM_VCPU_POWER_OFF		0 /* CPU is started in OFF state */
 #define KVM_ARM_VCPU_EL1_32BIT		1 /* CPU running a 32bit VM */
 #define KVM_ARM_VCPU_PSCI_0_2		2 /* CPU uses PSCI v0.2 */
+#define KVM_ARM_VCPU_PMU_V3		3 /* Support guest PMUv3 */
 
 struct kvm_vcpu_init {
 	__u32 target;
@@ -93,12 +107,39 @@
 struct kvm_fpu {
 };
 
+/*
+ * See v8 ARM ARM D7.3: Debug Registers
+ *
+ * The architectural limit is 16 debug registers of each type although
+ * in practice there are usually less (see ID_AA64DFR0_EL1).
+ *
+ * Although the control registers are architecturally defined as 32
+ * bits wide we use a 64 bit structure here to keep parity with
+ * KVM_GET/SET_ONE_REG behaviour which treats all system registers as
+ * 64 bit values. It also allows for the possibility of the
+ * architecture expanding the control registers without having to
+ * change the userspace ABI.
+ */
+#define KVM_ARM_MAX_DBG_REGS 16
 struct kvm_guest_debug_arch {
+	__u64 dbg_bcr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_bvr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_wcr[KVM_ARM_MAX_DBG_REGS];
+	__u64 dbg_wvr[KVM_ARM_MAX_DBG_REGS];
 };
 
 struct kvm_debug_exit_arch {
+	__u32 hsr;
+	__u64 far;	/* used for watchpoints */
 };
 
+/*
+ * Architecture specific defines for kvm_guest_debug->control
+ */
+
+#define KVM_GUESTDBG_USE_SW_BP		(1 << 16)
+#define KVM_GUESTDBG_USE_HW		(1 << 17)
+
 struct kvm_sync_regs {
 };
 
@@ -161,6 +202,13 @@
 #define   KVM_DEV_ARM_VGIC_OFFSET_SHIFT	0
 #define   KVM_DEV_ARM_VGIC_OFFSET_MASK	(0xffffffffULL << KVM_DEV_ARM_VGIC_OFFSET_SHIFT)
 #define KVM_DEV_ARM_VGIC_GRP_NR_IRQS	3
+#define KVM_DEV_ARM_VGIC_GRP_CTRL	4
+#define   KVM_DEV_ARM_VGIC_CTRL_INIT	0
+
+/* Device Control API on vcpu fd */
+#define KVM_ARM_VCPU_PMU_V3_CTRL	0
+#define   KVM_ARM_VCPU_PMU_V3_IRQ	0
+#define   KVM_ARM_VCPU_PMU_V3_INIT	1
 
 /* KVM_IRQ_LINE irq field index values */
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
@@ -179,9 +227,16 @@
 #define KVM_ARM_IRQ_CPU_IRQ		0
 #define KVM_ARM_IRQ_CPU_FIQ		1
 
-/* Highest supported SPI, from VGIC_NR_IRQS */
+/*
+ * This used to hold the highest supported SPI, but it is now obsolete
+ * and only here to provide source code level compatibility with older
+ * userland. The highest SPI number can be set via KVM_DEV_ARM_VGIC_GRP_NR_IRQS.
+ */
 #define KVM_ARM_IRQ_GIC_MAX		127
 
+/* One single KVM irqchip, ie. the VGIC */
+#define KVM_NR_IRQCHIPS          1
+
 /* PSCI interface */
 #define KVM_PSCI_FN_BASE		0x95c1ba5e
 #define KVM_PSCI_FN(n)			(KVM_PSCI_FN_BASE + (n))
diff --git a/linux-headers/asm-arm64/unistd.h b/linux-headers/asm-arm64/unistd.h
new file mode 100644
index 0000000..043d17a
--- /dev/null
+++ b/linux-headers/asm-arm64/unistd.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2012 ARM Ltd.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define __ARCH_WANT_RENAMEAT
+
+#include <asm-generic/unistd.h>
diff --git a/linux-headers/asm-mips/kvm.h b/linux-headers/asm-mips/kvm.h
index 2c04b6d..6985eb5 100644
--- a/linux-headers/asm-mips/kvm.h
+++ b/linux-headers/asm-mips/kvm.h
@@ -36,77 +36,85 @@
 
 /*
  * for KVM_GET_FPU and KVM_SET_FPU
- *
- * If Status[FR] is zero (32-bit FPU), the upper 32-bits of the FPRs
- * are zero filled.
  */
 struct kvm_fpu {
-	__u64 fpr[32];
-	__u32 fir;
-	__u32 fccr;
-	__u32 fexr;
-	__u32 fenr;
-	__u32 fcsr;
-	__u32 pad;
 };
 
 
 /*
- * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access CP0
+ * For MIPS, we use KVM_SET_ONE_REG and KVM_GET_ONE_REG to access various
  * registers.  The id field is broken down as follows:
  *
- *  bits[2..0]   - Register 'sel' index.
- *  bits[7..3]   - Register 'rd'  index.
- *  bits[15..8]  - Must be zero.
- *  bits[31..16] - 1 -> CP0 registers.
- *  bits[51..32] - Must be zero.
  *  bits[63..52] - As per linux/kvm.h
+ *  bits[51..32] - Must be zero.
+ *  bits[31..16] - Register set.
+ *
+ * Register set = 0: GP registers from kvm_regs (see definitions below).
+ *
+ * Register set = 1: CP0 registers.
+ *  bits[15..8]  - Must be zero.
+ *  bits[7..3]   - Register 'rd'  index.
+ *  bits[2..0]   - Register 'sel' index.
+ *
+ * Register set = 2: KVM specific registers (see definitions below).
+ *
+ * Register set = 3: FPU / MSA registers (see definitions below).
  *
  * Other sets registers may be added in the future.  Each set would
  * have its own identifier in bits[31..16].
- *
- * The registers defined in struct kvm_regs are also accessible, the
- * id values for these are below.
  */
 
-#define KVM_REG_MIPS_R0 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0)
-#define KVM_REG_MIPS_R1 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 1)
-#define KVM_REG_MIPS_R2 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 2)
-#define KVM_REG_MIPS_R3 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 3)
-#define KVM_REG_MIPS_R4 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 4)
-#define KVM_REG_MIPS_R5 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 5)
-#define KVM_REG_MIPS_R6 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 6)
-#define KVM_REG_MIPS_R7 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 7)
-#define KVM_REG_MIPS_R8 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 8)
-#define KVM_REG_MIPS_R9 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 9)
-#define KVM_REG_MIPS_R10 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 10)
-#define KVM_REG_MIPS_R11 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 11)
-#define KVM_REG_MIPS_R12 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 12)
-#define KVM_REG_MIPS_R13 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 13)
-#define KVM_REG_MIPS_R14 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 14)
-#define KVM_REG_MIPS_R15 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 15)
-#define KVM_REG_MIPS_R16 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 16)
-#define KVM_REG_MIPS_R17 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 17)
-#define KVM_REG_MIPS_R18 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 18)
-#define KVM_REG_MIPS_R19 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 19)
-#define KVM_REG_MIPS_R20 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 20)
-#define KVM_REG_MIPS_R21 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 21)
-#define KVM_REG_MIPS_R22 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 22)
-#define KVM_REG_MIPS_R23 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 23)
-#define KVM_REG_MIPS_R24 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 24)
-#define KVM_REG_MIPS_R25 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 25)
-#define KVM_REG_MIPS_R26 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 26)
-#define KVM_REG_MIPS_R27 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 27)
-#define KVM_REG_MIPS_R28 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 28)
-#define KVM_REG_MIPS_R29 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 29)
-#define KVM_REG_MIPS_R30 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 30)
-#define KVM_REG_MIPS_R31 (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 31)
+#define KVM_REG_MIPS_GP		(KVM_REG_MIPS | 0x0000000000000000ULL)
+#define KVM_REG_MIPS_CP0	(KVM_REG_MIPS | 0x0000000000010000ULL)
+#define KVM_REG_MIPS_KVM	(KVM_REG_MIPS | 0x0000000000020000ULL)
+#define KVM_REG_MIPS_FPU	(KVM_REG_MIPS | 0x0000000000030000ULL)
 
-#define KVM_REG_MIPS_HI (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 32)
-#define KVM_REG_MIPS_LO (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 33)
-#define KVM_REG_MIPS_PC (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 34)
 
-/* KVM specific control registers */
+/*
+ * KVM_REG_MIPS_GP - General purpose registers from kvm_regs.
+ */
+
+#define KVM_REG_MIPS_R0		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  0)
+#define KVM_REG_MIPS_R1		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  1)
+#define KVM_REG_MIPS_R2		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  2)
+#define KVM_REG_MIPS_R3		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  3)
+#define KVM_REG_MIPS_R4		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  4)
+#define KVM_REG_MIPS_R5		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  5)
+#define KVM_REG_MIPS_R6		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  6)
+#define KVM_REG_MIPS_R7		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  7)
+#define KVM_REG_MIPS_R8		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  8)
+#define KVM_REG_MIPS_R9		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 |  9)
+#define KVM_REG_MIPS_R10	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 10)
+#define KVM_REG_MIPS_R11	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 11)
+#define KVM_REG_MIPS_R12	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 12)
+#define KVM_REG_MIPS_R13	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 13)
+#define KVM_REG_MIPS_R14	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 14)
+#define KVM_REG_MIPS_R15	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 15)
+#define KVM_REG_MIPS_R16	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 16)
+#define KVM_REG_MIPS_R17	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 17)
+#define KVM_REG_MIPS_R18	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 18)
+#define KVM_REG_MIPS_R19	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 19)
+#define KVM_REG_MIPS_R20	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 20)
+#define KVM_REG_MIPS_R21	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 21)
+#define KVM_REG_MIPS_R22	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 22)
+#define KVM_REG_MIPS_R23	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 23)
+#define KVM_REG_MIPS_R24	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 24)
+#define KVM_REG_MIPS_R25	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 25)
+#define KVM_REG_MIPS_R26	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 26)
+#define KVM_REG_MIPS_R27	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 27)
+#define KVM_REG_MIPS_R28	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 28)
+#define KVM_REG_MIPS_R29	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 29)
+#define KVM_REG_MIPS_R30	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 30)
+#define KVM_REG_MIPS_R31	(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 31)
+
+#define KVM_REG_MIPS_HI		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 32)
+#define KVM_REG_MIPS_LO		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 33)
+#define KVM_REG_MIPS_PC		(KVM_REG_MIPS_GP | KVM_REG_SIZE_U64 | 34)
+
+
+/*
+ * KVM_REG_MIPS_KVM - KVM specific control registers.
+ */
 
 /*
  * CP0_Count control
@@ -118,8 +126,7 @@
  *        safely without losing time or guest timer interrupts.
  * Other: Reserved, do not change.
  */
-#define KVM_REG_MIPS_COUNT_CTL		(KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-					 0x20000 | 0)
+#define KVM_REG_MIPS_COUNT_CTL	    (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 0)
 #define KVM_REG_MIPS_COUNT_CTL_DC	0x00000001
 
 /*
@@ -131,15 +138,46 @@
  * emulated.
  * Modifications to times in the future are rejected.
  */
-#define KVM_REG_MIPS_COUNT_RESUME	(KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-					 0x20000 | 1)
+#define KVM_REG_MIPS_COUNT_RESUME   (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 1)
 /*
  * CP0_Count rate in Hz
  * Specifies the rate of the CP0_Count timer in Hz. Modifications occur without
  * discontinuities in CP0_Count.
  */
-#define KVM_REG_MIPS_COUNT_HZ		(KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-					 0x20000 | 2)
+#define KVM_REG_MIPS_COUNT_HZ	    (KVM_REG_MIPS_KVM | KVM_REG_SIZE_U64 | 2)
+
+
+/*
+ * KVM_REG_MIPS_FPU - Floating Point and MIPS SIMD Architecture (MSA) registers.
+ *
+ *  bits[15..8]  - Register subset (see definitions below).
+ *  bits[7..5]   - Must be zero.
+ *  bits[4..0]   - Register number within register subset.
+ */
+
+#define KVM_REG_MIPS_FPR	(KVM_REG_MIPS_FPU | 0x0000000000000000ULL)
+#define KVM_REG_MIPS_FCR	(KVM_REG_MIPS_FPU | 0x0000000000000100ULL)
+#define KVM_REG_MIPS_MSACR	(KVM_REG_MIPS_FPU | 0x0000000000000200ULL)
+
+/*
+ * KVM_REG_MIPS_FPR - Floating point / Vector registers.
+ */
+#define KVM_REG_MIPS_FPR_32(n)	(KVM_REG_MIPS_FPR | KVM_REG_SIZE_U32  | (n))
+#define KVM_REG_MIPS_FPR_64(n)	(KVM_REG_MIPS_FPR | KVM_REG_SIZE_U64  | (n))
+#define KVM_REG_MIPS_VEC_128(n)	(KVM_REG_MIPS_FPR | KVM_REG_SIZE_U128 | (n))
+
+/*
+ * KVM_REG_MIPS_FCR - Floating point control registers.
+ */
+#define KVM_REG_MIPS_FCR_IR	(KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 |  0)
+#define KVM_REG_MIPS_FCR_CSR	(KVM_REG_MIPS_FCR | KVM_REG_SIZE_U32 | 31)
+
+/*
+ * KVM_REG_MIPS_MSACR - MIPS SIMD Architecture (MSA) control registers.
+ */
+#define KVM_REG_MIPS_MSA_IR	 (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 |  0)
+#define KVM_REG_MIPS_MSA_CSR	 (KVM_REG_MIPS_MSACR | KVM_REG_SIZE_U32 |  1)
+
 
 /*
  * KVM MIPS specific structures and definitions
diff --git a/linux-headers/asm-mips/unistd.h b/linux-headers/asm-mips/unistd.h
new file mode 100644
index 0000000..2a20209
--- /dev/null
+++ b/linux-headers/asm-mips/unistd.h
@@ -0,0 +1,1063 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * Copyright (C) 1995, 96, 97, 98, 99, 2000 by Ralf Baechle
+ * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
+ *
+ * Changed system calls macros _syscall5 - _syscall7 to push args 5 to 7 onto
+ * the stack. Robin Farine for ACN S.A, Copyright (C) 1996 by ACN S.A
+ */
+#ifndef _ASM_UNISTD_H
+#define _ASM_UNISTD_H
+
+#include <asm/sgidefs.h>
+
+#if _MIPS_SIM == _MIPS_SIM_ABI32
+
+/*
+ * Linux o32 style syscalls are in the range from 4000 to 4999.
+ */
+#define __NR_Linux			4000
+#define __NR_syscall			(__NR_Linux +	0)
+#define __NR_exit			(__NR_Linux +	1)
+#define __NR_fork			(__NR_Linux +	2)
+#define __NR_read			(__NR_Linux +	3)
+#define __NR_write			(__NR_Linux +	4)
+#define __NR_open			(__NR_Linux +	5)
+#define __NR_close			(__NR_Linux +	6)
+#define __NR_waitpid			(__NR_Linux +	7)
+#define __NR_creat			(__NR_Linux +	8)
+#define __NR_link			(__NR_Linux +	9)
+#define __NR_unlink			(__NR_Linux +  10)
+#define __NR_execve			(__NR_Linux +  11)
+#define __NR_chdir			(__NR_Linux +  12)
+#define __NR_time			(__NR_Linux +  13)
+#define __NR_mknod			(__NR_Linux +  14)
+#define __NR_chmod			(__NR_Linux +  15)
+#define __NR_lchown			(__NR_Linux +  16)
+#define __NR_break			(__NR_Linux +  17)
+#define __NR_unused18			(__NR_Linux +  18)
+#define __NR_lseek			(__NR_Linux +  19)
+#define __NR_getpid			(__NR_Linux +  20)
+#define __NR_mount			(__NR_Linux +  21)
+#define __NR_umount			(__NR_Linux +  22)
+#define __NR_setuid			(__NR_Linux +  23)
+#define __NR_getuid			(__NR_Linux +  24)
+#define __NR_stime			(__NR_Linux +  25)
+#define __NR_ptrace			(__NR_Linux +  26)
+#define __NR_alarm			(__NR_Linux +  27)
+#define __NR_unused28			(__NR_Linux +  28)
+#define __NR_pause			(__NR_Linux +  29)
+#define __NR_utime			(__NR_Linux +  30)
+#define __NR_stty			(__NR_Linux +  31)
+#define __NR_gtty			(__NR_Linux +  32)
+#define __NR_access			(__NR_Linux +  33)
+#define __NR_nice			(__NR_Linux +  34)
+#define __NR_ftime			(__NR_Linux +  35)
+#define __NR_sync			(__NR_Linux +  36)
+#define __NR_kill			(__NR_Linux +  37)
+#define __NR_rename			(__NR_Linux +  38)
+#define __NR_mkdir			(__NR_Linux +  39)
+#define __NR_rmdir			(__NR_Linux +  40)
+#define __NR_dup			(__NR_Linux +  41)
+#define __NR_pipe			(__NR_Linux +  42)
+#define __NR_times			(__NR_Linux +  43)
+#define __NR_prof			(__NR_Linux +  44)
+#define __NR_brk			(__NR_Linux +  45)
+#define __NR_setgid			(__NR_Linux +  46)
+#define __NR_getgid			(__NR_Linux +  47)
+#define __NR_signal			(__NR_Linux +  48)
+#define __NR_geteuid			(__NR_Linux +  49)
+#define __NR_getegid			(__NR_Linux +  50)
+#define __NR_acct			(__NR_Linux +  51)
+#define __NR_umount2			(__NR_Linux +  52)
+#define __NR_lock			(__NR_Linux +  53)
+#define __NR_ioctl			(__NR_Linux +  54)
+#define __NR_fcntl			(__NR_Linux +  55)
+#define __NR_mpx			(__NR_Linux +  56)
+#define __NR_setpgid			(__NR_Linux +  57)
+#define __NR_ulimit			(__NR_Linux +  58)
+#define __NR_unused59			(__NR_Linux +  59)
+#define __NR_umask			(__NR_Linux +  60)
+#define __NR_chroot			(__NR_Linux +  61)
+#define __NR_ustat			(__NR_Linux +  62)
+#define __NR_dup2			(__NR_Linux +  63)
+#define __NR_getppid			(__NR_Linux +  64)
+#define __NR_getpgrp			(__NR_Linux +  65)
+#define __NR_setsid			(__NR_Linux +  66)
+#define __NR_sigaction			(__NR_Linux +  67)
+#define __NR_sgetmask			(__NR_Linux +  68)
+#define __NR_ssetmask			(__NR_Linux +  69)
+#define __NR_setreuid			(__NR_Linux +  70)
+#define __NR_setregid			(__NR_Linux +  71)
+#define __NR_sigsuspend			(__NR_Linux +  72)
+#define __NR_sigpending			(__NR_Linux +  73)
+#define __NR_sethostname		(__NR_Linux +  74)
+#define __NR_setrlimit			(__NR_Linux +  75)
+#define __NR_getrlimit			(__NR_Linux +  76)
+#define __NR_getrusage			(__NR_Linux +  77)
+#define __NR_gettimeofday		(__NR_Linux +  78)
+#define __NR_settimeofday		(__NR_Linux +  79)
+#define __NR_getgroups			(__NR_Linux +  80)
+#define __NR_setgroups			(__NR_Linux +  81)
+#define __NR_reserved82			(__NR_Linux +  82)
+#define __NR_symlink			(__NR_Linux +  83)
+#define __NR_unused84			(__NR_Linux +  84)
+#define __NR_readlink			(__NR_Linux +  85)
+#define __NR_uselib			(__NR_Linux +  86)
+#define __NR_swapon			(__NR_Linux +  87)
+#define __NR_reboot			(__NR_Linux +  88)
+#define __NR_readdir			(__NR_Linux +  89)
+#define __NR_mmap			(__NR_Linux +  90)
+#define __NR_munmap			(__NR_Linux +  91)
+#define __NR_truncate			(__NR_Linux +  92)
+#define __NR_ftruncate			(__NR_Linux +  93)
+#define __NR_fchmod			(__NR_Linux +  94)
+#define __NR_fchown			(__NR_Linux +  95)
+#define __NR_getpriority		(__NR_Linux +  96)
+#define __NR_setpriority		(__NR_Linux +  97)
+#define __NR_profil			(__NR_Linux +  98)
+#define __NR_statfs			(__NR_Linux +  99)
+#define __NR_fstatfs			(__NR_Linux + 100)
+#define __NR_ioperm			(__NR_Linux + 101)
+#define __NR_socketcall			(__NR_Linux + 102)
+#define __NR_syslog			(__NR_Linux + 103)
+#define __NR_setitimer			(__NR_Linux + 104)
+#define __NR_getitimer			(__NR_Linux + 105)
+#define __NR_stat			(__NR_Linux + 106)
+#define __NR_lstat			(__NR_Linux + 107)
+#define __NR_fstat			(__NR_Linux + 108)
+#define __NR_unused109			(__NR_Linux + 109)
+#define __NR_iopl			(__NR_Linux + 110)
+#define __NR_vhangup			(__NR_Linux + 111)
+#define __NR_idle			(__NR_Linux + 112)
+#define __NR_vm86			(__NR_Linux + 113)
+#define __NR_wait4			(__NR_Linux + 114)
+#define __NR_swapoff			(__NR_Linux + 115)
+#define __NR_sysinfo			(__NR_Linux + 116)
+#define __NR_ipc			(__NR_Linux + 117)
+#define __NR_fsync			(__NR_Linux + 118)
+#define __NR_sigreturn			(__NR_Linux + 119)
+#define __NR_clone			(__NR_Linux + 120)
+#define __NR_setdomainname		(__NR_Linux + 121)
+#define __NR_uname			(__NR_Linux + 122)
+#define __NR_modify_ldt			(__NR_Linux + 123)
+#define __NR_adjtimex			(__NR_Linux + 124)
+#define __NR_mprotect			(__NR_Linux + 125)
+#define __NR_sigprocmask		(__NR_Linux + 126)
+#define __NR_create_module		(__NR_Linux + 127)
+#define __NR_init_module		(__NR_Linux + 128)
+#define __NR_delete_module		(__NR_Linux + 129)
+#define __NR_get_kernel_syms		(__NR_Linux + 130)
+#define __NR_quotactl			(__NR_Linux + 131)
+#define __NR_getpgid			(__NR_Linux + 132)
+#define __NR_fchdir			(__NR_Linux + 133)
+#define __NR_bdflush			(__NR_Linux + 134)
+#define __NR_sysfs			(__NR_Linux + 135)
+#define __NR_personality		(__NR_Linux + 136)
+#define __NR_afs_syscall		(__NR_Linux + 137) /* Syscall for Andrew File System */
+#define __NR_setfsuid			(__NR_Linux + 138)
+#define __NR_setfsgid			(__NR_Linux + 139)
+#define __NR__llseek			(__NR_Linux + 140)
+#define __NR_getdents			(__NR_Linux + 141)
+#define __NR__newselect			(__NR_Linux + 142)
+#define __NR_flock			(__NR_Linux + 143)
+#define __NR_msync			(__NR_Linux + 144)
+#define __NR_readv			(__NR_Linux + 145)
+#define __NR_writev			(__NR_Linux + 146)
+#define __NR_cacheflush			(__NR_Linux + 147)
+#define __NR_cachectl			(__NR_Linux + 148)
+#define __NR_sysmips			(__NR_Linux + 149)
+#define __NR_unused150			(__NR_Linux + 150)
+#define __NR_getsid			(__NR_Linux + 151)
+#define __NR_fdatasync			(__NR_Linux + 152)
+#define __NR__sysctl			(__NR_Linux + 153)
+#define __NR_mlock			(__NR_Linux + 154)
+#define __NR_munlock			(__NR_Linux + 155)
+#define __NR_mlockall			(__NR_Linux + 156)
+#define __NR_munlockall			(__NR_Linux + 157)
+#define __NR_sched_setparam		(__NR_Linux + 158)
+#define __NR_sched_getparam		(__NR_Linux + 159)
+#define __NR_sched_setscheduler		(__NR_Linux + 160)
+#define __NR_sched_getscheduler		(__NR_Linux + 161)
+#define __NR_sched_yield		(__NR_Linux + 162)
+#define __NR_sched_get_priority_max	(__NR_Linux + 163)
+#define __NR_sched_get_priority_min	(__NR_Linux + 164)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 165)
+#define __NR_nanosleep			(__NR_Linux + 166)
+#define __NR_mremap			(__NR_Linux + 167)
+#define __NR_accept			(__NR_Linux + 168)
+#define __NR_bind			(__NR_Linux + 169)
+#define __NR_connect			(__NR_Linux + 170)
+#define __NR_getpeername		(__NR_Linux + 171)
+#define __NR_getsockname		(__NR_Linux + 172)
+#define __NR_getsockopt			(__NR_Linux + 173)
+#define __NR_listen			(__NR_Linux + 174)
+#define __NR_recv			(__NR_Linux + 175)
+#define __NR_recvfrom			(__NR_Linux + 176)
+#define __NR_recvmsg			(__NR_Linux + 177)
+#define __NR_send			(__NR_Linux + 178)
+#define __NR_sendmsg			(__NR_Linux + 179)
+#define __NR_sendto			(__NR_Linux + 180)
+#define __NR_setsockopt			(__NR_Linux + 181)
+#define __NR_shutdown			(__NR_Linux + 182)
+#define __NR_socket			(__NR_Linux + 183)
+#define __NR_socketpair			(__NR_Linux + 184)
+#define __NR_setresuid			(__NR_Linux + 185)
+#define __NR_getresuid			(__NR_Linux + 186)
+#define __NR_query_module		(__NR_Linux + 187)
+#define __NR_poll			(__NR_Linux + 188)
+#define __NR_nfsservctl			(__NR_Linux + 189)
+#define __NR_setresgid			(__NR_Linux + 190)
+#define __NR_getresgid			(__NR_Linux + 191)
+#define __NR_prctl			(__NR_Linux + 192)
+#define __NR_rt_sigreturn		(__NR_Linux + 193)
+#define __NR_rt_sigaction		(__NR_Linux + 194)
+#define __NR_rt_sigprocmask		(__NR_Linux + 195)
+#define __NR_rt_sigpending		(__NR_Linux + 196)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 197)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 198)
+#define __NR_rt_sigsuspend		(__NR_Linux + 199)
+#define __NR_pread64			(__NR_Linux + 200)
+#define __NR_pwrite64			(__NR_Linux + 201)
+#define __NR_chown			(__NR_Linux + 202)
+#define __NR_getcwd			(__NR_Linux + 203)
+#define __NR_capget			(__NR_Linux + 204)
+#define __NR_capset			(__NR_Linux + 205)
+#define __NR_sigaltstack		(__NR_Linux + 206)
+#define __NR_sendfile			(__NR_Linux + 207)
+#define __NR_getpmsg			(__NR_Linux + 208)
+#define __NR_putpmsg			(__NR_Linux + 209)
+#define __NR_mmap2			(__NR_Linux + 210)
+#define __NR_truncate64			(__NR_Linux + 211)
+#define __NR_ftruncate64		(__NR_Linux + 212)
+#define __NR_stat64			(__NR_Linux + 213)
+#define __NR_lstat64			(__NR_Linux + 214)
+#define __NR_fstat64			(__NR_Linux + 215)
+#define __NR_pivot_root			(__NR_Linux + 216)
+#define __NR_mincore			(__NR_Linux + 217)
+#define __NR_madvise			(__NR_Linux + 218)
+#define __NR_getdents64			(__NR_Linux + 219)
+#define __NR_fcntl64			(__NR_Linux + 220)
+#define __NR_reserved221		(__NR_Linux + 221)
+#define __NR_gettid			(__NR_Linux + 222)
+#define __NR_readahead			(__NR_Linux + 223)
+#define __NR_setxattr			(__NR_Linux + 224)
+#define __NR_lsetxattr			(__NR_Linux + 225)
+#define __NR_fsetxattr			(__NR_Linux + 226)
+#define __NR_getxattr			(__NR_Linux + 227)
+#define __NR_lgetxattr			(__NR_Linux + 228)
+#define __NR_fgetxattr			(__NR_Linux + 229)
+#define __NR_listxattr			(__NR_Linux + 230)
+#define __NR_llistxattr			(__NR_Linux + 231)
+#define __NR_flistxattr			(__NR_Linux + 232)
+#define __NR_removexattr		(__NR_Linux + 233)
+#define __NR_lremovexattr		(__NR_Linux + 234)
+#define __NR_fremovexattr		(__NR_Linux + 235)
+#define __NR_tkill			(__NR_Linux + 236)
+#define __NR_sendfile64			(__NR_Linux + 237)
+#define __NR_futex			(__NR_Linux + 238)
+#define __NR_sched_setaffinity		(__NR_Linux + 239)
+#define __NR_sched_getaffinity		(__NR_Linux + 240)
+#define __NR_io_setup			(__NR_Linux + 241)
+#define __NR_io_destroy			(__NR_Linux + 242)
+#define __NR_io_getevents		(__NR_Linux + 243)
+#define __NR_io_submit			(__NR_Linux + 244)
+#define __NR_io_cancel			(__NR_Linux + 245)
+#define __NR_exit_group			(__NR_Linux + 246)
+#define __NR_lookup_dcookie		(__NR_Linux + 247)
+#define __NR_epoll_create		(__NR_Linux + 248)
+#define __NR_epoll_ctl			(__NR_Linux + 249)
+#define __NR_epoll_wait			(__NR_Linux + 250)
+#define __NR_remap_file_pages		(__NR_Linux + 251)
+#define __NR_set_tid_address		(__NR_Linux + 252)
+#define __NR_restart_syscall		(__NR_Linux + 253)
+#define __NR_fadvise64			(__NR_Linux + 254)
+#define __NR_statfs64			(__NR_Linux + 255)
+#define __NR_fstatfs64			(__NR_Linux + 256)
+#define __NR_timer_create		(__NR_Linux + 257)
+#define __NR_timer_settime		(__NR_Linux + 258)
+#define __NR_timer_gettime		(__NR_Linux + 259)
+#define __NR_timer_getoverrun		(__NR_Linux + 260)
+#define __NR_timer_delete		(__NR_Linux + 261)
+#define __NR_clock_settime		(__NR_Linux + 262)
+#define __NR_clock_gettime		(__NR_Linux + 263)
+#define __NR_clock_getres		(__NR_Linux + 264)
+#define __NR_clock_nanosleep		(__NR_Linux + 265)
+#define __NR_tgkill			(__NR_Linux + 266)
+#define __NR_utimes			(__NR_Linux + 267)
+#define __NR_mbind			(__NR_Linux + 268)
+#define __NR_get_mempolicy		(__NR_Linux + 269)
+#define __NR_set_mempolicy		(__NR_Linux + 270)
+#define __NR_mq_open			(__NR_Linux + 271)
+#define __NR_mq_unlink			(__NR_Linux + 272)
+#define __NR_mq_timedsend		(__NR_Linux + 273)
+#define __NR_mq_timedreceive		(__NR_Linux + 274)
+#define __NR_mq_notify			(__NR_Linux + 275)
+#define __NR_mq_getsetattr		(__NR_Linux + 276)
+#define __NR_vserver			(__NR_Linux + 277)
+#define __NR_waitid			(__NR_Linux + 278)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 279) */
+#define __NR_add_key			(__NR_Linux + 280)
+#define __NR_request_key		(__NR_Linux + 281)
+#define __NR_keyctl			(__NR_Linux + 282)
+#define __NR_set_thread_area		(__NR_Linux + 283)
+#define __NR_inotify_init		(__NR_Linux + 284)
+#define __NR_inotify_add_watch		(__NR_Linux + 285)
+#define __NR_inotify_rm_watch		(__NR_Linux + 286)
+#define __NR_migrate_pages		(__NR_Linux + 287)
+#define __NR_openat			(__NR_Linux + 288)
+#define __NR_mkdirat			(__NR_Linux + 289)
+#define __NR_mknodat			(__NR_Linux + 290)
+#define __NR_fchownat			(__NR_Linux + 291)
+#define __NR_futimesat			(__NR_Linux + 292)
+#define __NR_fstatat64			(__NR_Linux + 293)
+#define __NR_unlinkat			(__NR_Linux + 294)
+#define __NR_renameat			(__NR_Linux + 295)
+#define __NR_linkat			(__NR_Linux + 296)
+#define __NR_symlinkat			(__NR_Linux + 297)
+#define __NR_readlinkat			(__NR_Linux + 298)
+#define __NR_fchmodat			(__NR_Linux + 299)
+#define __NR_faccessat			(__NR_Linux + 300)
+#define __NR_pselect6			(__NR_Linux + 301)
+#define __NR_ppoll			(__NR_Linux + 302)
+#define __NR_unshare			(__NR_Linux + 303)
+#define __NR_splice			(__NR_Linux + 304)
+#define __NR_sync_file_range		(__NR_Linux + 305)
+#define __NR_tee			(__NR_Linux + 306)
+#define __NR_vmsplice			(__NR_Linux + 307)
+#define __NR_move_pages			(__NR_Linux + 308)
+#define __NR_set_robust_list		(__NR_Linux + 309)
+#define __NR_get_robust_list		(__NR_Linux + 310)
+#define __NR_kexec_load			(__NR_Linux + 311)
+#define __NR_getcpu			(__NR_Linux + 312)
+#define __NR_epoll_pwait		(__NR_Linux + 313)
+#define __NR_ioprio_set			(__NR_Linux + 314)
+#define __NR_ioprio_get			(__NR_Linux + 315)
+#define __NR_utimensat			(__NR_Linux + 316)
+#define __NR_signalfd			(__NR_Linux + 317)
+#define __NR_timerfd			(__NR_Linux + 318)
+#define __NR_eventfd			(__NR_Linux + 319)
+#define __NR_fallocate			(__NR_Linux + 320)
+#define __NR_timerfd_create		(__NR_Linux + 321)
+#define __NR_timerfd_gettime		(__NR_Linux + 322)
+#define __NR_timerfd_settime		(__NR_Linux + 323)
+#define __NR_signalfd4			(__NR_Linux + 324)
+#define __NR_eventfd2			(__NR_Linux + 325)
+#define __NR_epoll_create1		(__NR_Linux + 326)
+#define __NR_dup3			(__NR_Linux + 327)
+#define __NR_pipe2			(__NR_Linux + 328)
+#define __NR_inotify_init1		(__NR_Linux + 329)
+#define __NR_preadv			(__NR_Linux + 330)
+#define __NR_pwritev			(__NR_Linux + 331)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 332)
+#define __NR_perf_event_open		(__NR_Linux + 333)
+#define __NR_accept4			(__NR_Linux + 334)
+#define __NR_recvmmsg			(__NR_Linux + 335)
+#define __NR_fanotify_init		(__NR_Linux + 336)
+#define __NR_fanotify_mark		(__NR_Linux + 337)
+#define __NR_prlimit64			(__NR_Linux + 338)
+#define __NR_name_to_handle_at		(__NR_Linux + 339)
+#define __NR_open_by_handle_at		(__NR_Linux + 340)
+#define __NR_clock_adjtime		(__NR_Linux + 341)
+#define __NR_syncfs			(__NR_Linux + 342)
+#define __NR_sendmmsg			(__NR_Linux + 343)
+#define __NR_setns			(__NR_Linux + 344)
+#define __NR_process_vm_readv		(__NR_Linux + 345)
+#define __NR_process_vm_writev		(__NR_Linux + 346)
+#define __NR_kcmp			(__NR_Linux + 347)
+#define __NR_finit_module		(__NR_Linux + 348)
+#define __NR_sched_setattr		(__NR_Linux + 349)
+#define __NR_sched_getattr		(__NR_Linux + 350)
+#define __NR_renameat2			(__NR_Linux + 351)
+#define __NR_seccomp			(__NR_Linux + 352)
+#define __NR_getrandom			(__NR_Linux + 353)
+#define __NR_memfd_create		(__NR_Linux + 354)
+#define __NR_bpf			(__NR_Linux + 355)
+#define __NR_execveat			(__NR_Linux + 356)
+
+/*
+ * Offset of the last Linux o32 flavoured syscall
+ */
+#define __NR_Linux_syscalls		356
+
+#endif /* _MIPS_SIM == _MIPS_SIM_ABI32 */
+
+#define __NR_O32_Linux			4000
+#define __NR_O32_Linux_syscalls		356
+
+#if _MIPS_SIM == _MIPS_SIM_ABI64
+
+/*
+ * Linux 64-bit syscalls are in the range from 5000 to 5999.
+ */
+#define __NR_Linux			5000
+#define __NR_read			(__NR_Linux +	0)
+#define __NR_write			(__NR_Linux +	1)
+#define __NR_open			(__NR_Linux +	2)
+#define __NR_close			(__NR_Linux +	3)
+#define __NR_stat			(__NR_Linux +	4)
+#define __NR_fstat			(__NR_Linux +	5)
+#define __NR_lstat			(__NR_Linux +	6)
+#define __NR_poll			(__NR_Linux +	7)
+#define __NR_lseek			(__NR_Linux +	8)
+#define __NR_mmap			(__NR_Linux +	9)
+#define __NR_mprotect			(__NR_Linux +  10)
+#define __NR_munmap			(__NR_Linux +  11)
+#define __NR_brk			(__NR_Linux +  12)
+#define __NR_rt_sigaction		(__NR_Linux +  13)
+#define __NR_rt_sigprocmask		(__NR_Linux +  14)
+#define __NR_ioctl			(__NR_Linux +  15)
+#define __NR_pread64			(__NR_Linux +  16)
+#define __NR_pwrite64			(__NR_Linux +  17)
+#define __NR_readv			(__NR_Linux +  18)
+#define __NR_writev			(__NR_Linux +  19)
+#define __NR_access			(__NR_Linux +  20)
+#define __NR_pipe			(__NR_Linux +  21)
+#define __NR__newselect			(__NR_Linux +  22)
+#define __NR_sched_yield		(__NR_Linux +  23)
+#define __NR_mremap			(__NR_Linux +  24)
+#define __NR_msync			(__NR_Linux +  25)
+#define __NR_mincore			(__NR_Linux +  26)
+#define __NR_madvise			(__NR_Linux +  27)
+#define __NR_shmget			(__NR_Linux +  28)
+#define __NR_shmat			(__NR_Linux +  29)
+#define __NR_shmctl			(__NR_Linux +  30)
+#define __NR_dup			(__NR_Linux +  31)
+#define __NR_dup2			(__NR_Linux +  32)
+#define __NR_pause			(__NR_Linux +  33)
+#define __NR_nanosleep			(__NR_Linux +  34)
+#define __NR_getitimer			(__NR_Linux +  35)
+#define __NR_setitimer			(__NR_Linux +  36)
+#define __NR_alarm			(__NR_Linux +  37)
+#define __NR_getpid			(__NR_Linux +  38)
+#define __NR_sendfile			(__NR_Linux +  39)
+#define __NR_socket			(__NR_Linux +  40)
+#define __NR_connect			(__NR_Linux +  41)
+#define __NR_accept			(__NR_Linux +  42)
+#define __NR_sendto			(__NR_Linux +  43)
+#define __NR_recvfrom			(__NR_Linux +  44)
+#define __NR_sendmsg			(__NR_Linux +  45)
+#define __NR_recvmsg			(__NR_Linux +  46)
+#define __NR_shutdown			(__NR_Linux +  47)
+#define __NR_bind			(__NR_Linux +  48)
+#define __NR_listen			(__NR_Linux +  49)
+#define __NR_getsockname		(__NR_Linux +  50)
+#define __NR_getpeername		(__NR_Linux +  51)
+#define __NR_socketpair			(__NR_Linux +  52)
+#define __NR_setsockopt			(__NR_Linux +  53)
+#define __NR_getsockopt			(__NR_Linux +  54)
+#define __NR_clone			(__NR_Linux +  55)
+#define __NR_fork			(__NR_Linux +  56)
+#define __NR_execve			(__NR_Linux +  57)
+#define __NR_exit			(__NR_Linux +  58)
+#define __NR_wait4			(__NR_Linux +  59)
+#define __NR_kill			(__NR_Linux +  60)
+#define __NR_uname			(__NR_Linux +  61)
+#define __NR_semget			(__NR_Linux +  62)
+#define __NR_semop			(__NR_Linux +  63)
+#define __NR_semctl			(__NR_Linux +  64)
+#define __NR_shmdt			(__NR_Linux +  65)
+#define __NR_msgget			(__NR_Linux +  66)
+#define __NR_msgsnd			(__NR_Linux +  67)
+#define __NR_msgrcv			(__NR_Linux +  68)
+#define __NR_msgctl			(__NR_Linux +  69)
+#define __NR_fcntl			(__NR_Linux +  70)
+#define __NR_flock			(__NR_Linux +  71)
+#define __NR_fsync			(__NR_Linux +  72)
+#define __NR_fdatasync			(__NR_Linux +  73)
+#define __NR_truncate			(__NR_Linux +  74)
+#define __NR_ftruncate			(__NR_Linux +  75)
+#define __NR_getdents			(__NR_Linux +  76)
+#define __NR_getcwd			(__NR_Linux +  77)
+#define __NR_chdir			(__NR_Linux +  78)
+#define __NR_fchdir			(__NR_Linux +  79)
+#define __NR_rename			(__NR_Linux +  80)
+#define __NR_mkdir			(__NR_Linux +  81)
+#define __NR_rmdir			(__NR_Linux +  82)
+#define __NR_creat			(__NR_Linux +  83)
+#define __NR_link			(__NR_Linux +  84)
+#define __NR_unlink			(__NR_Linux +  85)
+#define __NR_symlink			(__NR_Linux +  86)
+#define __NR_readlink			(__NR_Linux +  87)
+#define __NR_chmod			(__NR_Linux +  88)
+#define __NR_fchmod			(__NR_Linux +  89)
+#define __NR_chown			(__NR_Linux +  90)
+#define __NR_fchown			(__NR_Linux +  91)
+#define __NR_lchown			(__NR_Linux +  92)
+#define __NR_umask			(__NR_Linux +  93)
+#define __NR_gettimeofday		(__NR_Linux +  94)
+#define __NR_getrlimit			(__NR_Linux +  95)
+#define __NR_getrusage			(__NR_Linux +  96)
+#define __NR_sysinfo			(__NR_Linux +  97)
+#define __NR_times			(__NR_Linux +  98)
+#define __NR_ptrace			(__NR_Linux +  99)
+#define __NR_getuid			(__NR_Linux + 100)
+#define __NR_syslog			(__NR_Linux + 101)
+#define __NR_getgid			(__NR_Linux + 102)
+#define __NR_setuid			(__NR_Linux + 103)
+#define __NR_setgid			(__NR_Linux + 104)
+#define __NR_geteuid			(__NR_Linux + 105)
+#define __NR_getegid			(__NR_Linux + 106)
+#define __NR_setpgid			(__NR_Linux + 107)
+#define __NR_getppid			(__NR_Linux + 108)
+#define __NR_getpgrp			(__NR_Linux + 109)
+#define __NR_setsid			(__NR_Linux + 110)
+#define __NR_setreuid			(__NR_Linux + 111)
+#define __NR_setregid			(__NR_Linux + 112)
+#define __NR_getgroups			(__NR_Linux + 113)
+#define __NR_setgroups			(__NR_Linux + 114)
+#define __NR_setresuid			(__NR_Linux + 115)
+#define __NR_getresuid			(__NR_Linux + 116)
+#define __NR_setresgid			(__NR_Linux + 117)
+#define __NR_getresgid			(__NR_Linux + 118)
+#define __NR_getpgid			(__NR_Linux + 119)
+#define __NR_setfsuid			(__NR_Linux + 120)
+#define __NR_setfsgid			(__NR_Linux + 121)
+#define __NR_getsid			(__NR_Linux + 122)
+#define __NR_capget			(__NR_Linux + 123)
+#define __NR_capset			(__NR_Linux + 124)
+#define __NR_rt_sigpending		(__NR_Linux + 125)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 126)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 127)
+#define __NR_rt_sigsuspend		(__NR_Linux + 128)
+#define __NR_sigaltstack		(__NR_Linux + 129)
+#define __NR_utime			(__NR_Linux + 130)
+#define __NR_mknod			(__NR_Linux + 131)
+#define __NR_personality		(__NR_Linux + 132)
+#define __NR_ustat			(__NR_Linux + 133)
+#define __NR_statfs			(__NR_Linux + 134)
+#define __NR_fstatfs			(__NR_Linux + 135)
+#define __NR_sysfs			(__NR_Linux + 136)
+#define __NR_getpriority		(__NR_Linux + 137)
+#define __NR_setpriority		(__NR_Linux + 138)
+#define __NR_sched_setparam		(__NR_Linux + 139)
+#define __NR_sched_getparam		(__NR_Linux + 140)
+#define __NR_sched_setscheduler		(__NR_Linux + 141)
+#define __NR_sched_getscheduler		(__NR_Linux + 142)
+#define __NR_sched_get_priority_max	(__NR_Linux + 143)
+#define __NR_sched_get_priority_min	(__NR_Linux + 144)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 145)
+#define __NR_mlock			(__NR_Linux + 146)
+#define __NR_munlock			(__NR_Linux + 147)
+#define __NR_mlockall			(__NR_Linux + 148)
+#define __NR_munlockall			(__NR_Linux + 149)
+#define __NR_vhangup			(__NR_Linux + 150)
+#define __NR_pivot_root			(__NR_Linux + 151)
+#define __NR__sysctl			(__NR_Linux + 152)
+#define __NR_prctl			(__NR_Linux + 153)
+#define __NR_adjtimex			(__NR_Linux + 154)
+#define __NR_setrlimit			(__NR_Linux + 155)
+#define __NR_chroot			(__NR_Linux + 156)
+#define __NR_sync			(__NR_Linux + 157)
+#define __NR_acct			(__NR_Linux + 158)
+#define __NR_settimeofday		(__NR_Linux + 159)
+#define __NR_mount			(__NR_Linux + 160)
+#define __NR_umount2			(__NR_Linux + 161)
+#define __NR_swapon			(__NR_Linux + 162)
+#define __NR_swapoff			(__NR_Linux + 163)
+#define __NR_reboot			(__NR_Linux + 164)
+#define __NR_sethostname		(__NR_Linux + 165)
+#define __NR_setdomainname		(__NR_Linux + 166)
+#define __NR_create_module		(__NR_Linux + 167)
+#define __NR_init_module		(__NR_Linux + 168)
+#define __NR_delete_module		(__NR_Linux + 169)
+#define __NR_get_kernel_syms		(__NR_Linux + 170)
+#define __NR_query_module		(__NR_Linux + 171)
+#define __NR_quotactl			(__NR_Linux + 172)
+#define __NR_nfsservctl			(__NR_Linux + 173)
+#define __NR_getpmsg			(__NR_Linux + 174)
+#define __NR_putpmsg			(__NR_Linux + 175)
+#define __NR_afs_syscall		(__NR_Linux + 176)
+#define __NR_reserved177		(__NR_Linux + 177)
+#define __NR_gettid			(__NR_Linux + 178)
+#define __NR_readahead			(__NR_Linux + 179)
+#define __NR_setxattr			(__NR_Linux + 180)
+#define __NR_lsetxattr			(__NR_Linux + 181)
+#define __NR_fsetxattr			(__NR_Linux + 182)
+#define __NR_getxattr			(__NR_Linux + 183)
+#define __NR_lgetxattr			(__NR_Linux + 184)
+#define __NR_fgetxattr			(__NR_Linux + 185)
+#define __NR_listxattr			(__NR_Linux + 186)
+#define __NR_llistxattr			(__NR_Linux + 187)
+#define __NR_flistxattr			(__NR_Linux + 188)
+#define __NR_removexattr		(__NR_Linux + 189)
+#define __NR_lremovexattr		(__NR_Linux + 190)
+#define __NR_fremovexattr		(__NR_Linux + 191)
+#define __NR_tkill			(__NR_Linux + 192)
+#define __NR_reserved193		(__NR_Linux + 193)
+#define __NR_futex			(__NR_Linux + 194)
+#define __NR_sched_setaffinity		(__NR_Linux + 195)
+#define __NR_sched_getaffinity		(__NR_Linux + 196)
+#define __NR_cacheflush			(__NR_Linux + 197)
+#define __NR_cachectl			(__NR_Linux + 198)
+#define __NR_sysmips			(__NR_Linux + 199)
+#define __NR_io_setup			(__NR_Linux + 200)
+#define __NR_io_destroy			(__NR_Linux + 201)
+#define __NR_io_getevents		(__NR_Linux + 202)
+#define __NR_io_submit			(__NR_Linux + 203)
+#define __NR_io_cancel			(__NR_Linux + 204)
+#define __NR_exit_group			(__NR_Linux + 205)
+#define __NR_lookup_dcookie		(__NR_Linux + 206)
+#define __NR_epoll_create		(__NR_Linux + 207)
+#define __NR_epoll_ctl			(__NR_Linux + 208)
+#define __NR_epoll_wait			(__NR_Linux + 209)
+#define __NR_remap_file_pages		(__NR_Linux + 210)
+#define __NR_rt_sigreturn		(__NR_Linux + 211)
+#define __NR_set_tid_address		(__NR_Linux + 212)
+#define __NR_restart_syscall		(__NR_Linux + 213)
+#define __NR_semtimedop			(__NR_Linux + 214)
+#define __NR_fadvise64			(__NR_Linux + 215)
+#define __NR_timer_create		(__NR_Linux + 216)
+#define __NR_timer_settime		(__NR_Linux + 217)
+#define __NR_timer_gettime		(__NR_Linux + 218)
+#define __NR_timer_getoverrun		(__NR_Linux + 219)
+#define __NR_timer_delete		(__NR_Linux + 220)
+#define __NR_clock_settime		(__NR_Linux + 221)
+#define __NR_clock_gettime		(__NR_Linux + 222)
+#define __NR_clock_getres		(__NR_Linux + 223)
+#define __NR_clock_nanosleep		(__NR_Linux + 224)
+#define __NR_tgkill			(__NR_Linux + 225)
+#define __NR_utimes			(__NR_Linux + 226)
+#define __NR_mbind			(__NR_Linux + 227)
+#define __NR_get_mempolicy		(__NR_Linux + 228)
+#define __NR_set_mempolicy		(__NR_Linux + 229)
+#define __NR_mq_open			(__NR_Linux + 230)
+#define __NR_mq_unlink			(__NR_Linux + 231)
+#define __NR_mq_timedsend		(__NR_Linux + 232)
+#define __NR_mq_timedreceive		(__NR_Linux + 233)
+#define __NR_mq_notify			(__NR_Linux + 234)
+#define __NR_mq_getsetattr		(__NR_Linux + 235)
+#define __NR_vserver			(__NR_Linux + 236)
+#define __NR_waitid			(__NR_Linux + 237)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 238) */
+#define __NR_add_key			(__NR_Linux + 239)
+#define __NR_request_key		(__NR_Linux + 240)
+#define __NR_keyctl			(__NR_Linux + 241)
+#define __NR_set_thread_area		(__NR_Linux + 242)
+#define __NR_inotify_init		(__NR_Linux + 243)
+#define __NR_inotify_add_watch		(__NR_Linux + 244)
+#define __NR_inotify_rm_watch		(__NR_Linux + 245)
+#define __NR_migrate_pages		(__NR_Linux + 246)
+#define __NR_openat			(__NR_Linux + 247)
+#define __NR_mkdirat			(__NR_Linux + 248)
+#define __NR_mknodat			(__NR_Linux + 249)
+#define __NR_fchownat			(__NR_Linux + 250)
+#define __NR_futimesat			(__NR_Linux + 251)
+#define __NR_newfstatat			(__NR_Linux + 252)
+#define __NR_unlinkat			(__NR_Linux + 253)
+#define __NR_renameat			(__NR_Linux + 254)
+#define __NR_linkat			(__NR_Linux + 255)
+#define __NR_symlinkat			(__NR_Linux + 256)
+#define __NR_readlinkat			(__NR_Linux + 257)
+#define __NR_fchmodat			(__NR_Linux + 258)
+#define __NR_faccessat			(__NR_Linux + 259)
+#define __NR_pselect6			(__NR_Linux + 260)
+#define __NR_ppoll			(__NR_Linux + 261)
+#define __NR_unshare			(__NR_Linux + 262)
+#define __NR_splice			(__NR_Linux + 263)
+#define __NR_sync_file_range		(__NR_Linux + 264)
+#define __NR_tee			(__NR_Linux + 265)
+#define __NR_vmsplice			(__NR_Linux + 266)
+#define __NR_move_pages			(__NR_Linux + 267)
+#define __NR_set_robust_list		(__NR_Linux + 268)
+#define __NR_get_robust_list		(__NR_Linux + 269)
+#define __NR_kexec_load			(__NR_Linux + 270)
+#define __NR_getcpu			(__NR_Linux + 271)
+#define __NR_epoll_pwait		(__NR_Linux + 272)
+#define __NR_ioprio_set			(__NR_Linux + 273)
+#define __NR_ioprio_get			(__NR_Linux + 274)
+#define __NR_utimensat			(__NR_Linux + 275)
+#define __NR_signalfd			(__NR_Linux + 276)
+#define __NR_timerfd			(__NR_Linux + 277)
+#define __NR_eventfd			(__NR_Linux + 278)
+#define __NR_fallocate			(__NR_Linux + 279)
+#define __NR_timerfd_create		(__NR_Linux + 280)
+#define __NR_timerfd_gettime		(__NR_Linux + 281)
+#define __NR_timerfd_settime		(__NR_Linux + 282)
+#define __NR_signalfd4			(__NR_Linux + 283)
+#define __NR_eventfd2			(__NR_Linux + 284)
+#define __NR_epoll_create1		(__NR_Linux + 285)
+#define __NR_dup3			(__NR_Linux + 286)
+#define __NR_pipe2			(__NR_Linux + 287)
+#define __NR_inotify_init1		(__NR_Linux + 288)
+#define __NR_preadv			(__NR_Linux + 289)
+#define __NR_pwritev			(__NR_Linux + 290)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 291)
+#define __NR_perf_event_open		(__NR_Linux + 292)
+#define __NR_accept4			(__NR_Linux + 293)
+#define __NR_recvmmsg			(__NR_Linux + 294)
+#define __NR_fanotify_init		(__NR_Linux + 295)
+#define __NR_fanotify_mark		(__NR_Linux + 296)
+#define __NR_prlimit64			(__NR_Linux + 297)
+#define __NR_name_to_handle_at		(__NR_Linux + 298)
+#define __NR_open_by_handle_at		(__NR_Linux + 299)
+#define __NR_clock_adjtime		(__NR_Linux + 300)
+#define __NR_syncfs			(__NR_Linux + 301)
+#define __NR_sendmmsg			(__NR_Linux + 302)
+#define __NR_setns			(__NR_Linux + 303)
+#define __NR_process_vm_readv		(__NR_Linux + 304)
+#define __NR_process_vm_writev		(__NR_Linux + 305)
+#define __NR_kcmp			(__NR_Linux + 306)
+#define __NR_finit_module		(__NR_Linux + 307)
+#define __NR_getdents64			(__NR_Linux + 308)
+#define __NR_sched_setattr		(__NR_Linux + 309)
+#define __NR_sched_getattr		(__NR_Linux + 310)
+#define __NR_renameat2			(__NR_Linux + 311)
+#define __NR_seccomp			(__NR_Linux + 312)
+#define __NR_getrandom			(__NR_Linux + 313)
+#define __NR_memfd_create		(__NR_Linux + 314)
+#define __NR_bpf			(__NR_Linux + 315)
+#define __NR_execveat			(__NR_Linux + 316)
+
+/*
+ * Offset of the last Linux 64-bit flavoured syscall
+ */
+#define __NR_Linux_syscalls		316
+
+#endif /* _MIPS_SIM == _MIPS_SIM_ABI64 */
+
+#define __NR_64_Linux			5000
+#define __NR_64_Linux_syscalls		316
+
+#if _MIPS_SIM == _MIPS_SIM_NABI32
+
+/*
+ * Linux N32 syscalls are in the range from 6000 to 6999.
+ */
+#define __NR_Linux			6000
+#define __NR_read			(__NR_Linux +	0)
+#define __NR_write			(__NR_Linux +	1)
+#define __NR_open			(__NR_Linux +	2)
+#define __NR_close			(__NR_Linux +	3)
+#define __NR_stat			(__NR_Linux +	4)
+#define __NR_fstat			(__NR_Linux +	5)
+#define __NR_lstat			(__NR_Linux +	6)
+#define __NR_poll			(__NR_Linux +	7)
+#define __NR_lseek			(__NR_Linux +	8)
+#define __NR_mmap			(__NR_Linux +	9)
+#define __NR_mprotect			(__NR_Linux +  10)
+#define __NR_munmap			(__NR_Linux +  11)
+#define __NR_brk			(__NR_Linux +  12)
+#define __NR_rt_sigaction		(__NR_Linux +  13)
+#define __NR_rt_sigprocmask		(__NR_Linux +  14)
+#define __NR_ioctl			(__NR_Linux +  15)
+#define __NR_pread64			(__NR_Linux +  16)
+#define __NR_pwrite64			(__NR_Linux +  17)
+#define __NR_readv			(__NR_Linux +  18)
+#define __NR_writev			(__NR_Linux +  19)
+#define __NR_access			(__NR_Linux +  20)
+#define __NR_pipe			(__NR_Linux +  21)
+#define __NR__newselect			(__NR_Linux +  22)
+#define __NR_sched_yield		(__NR_Linux +  23)
+#define __NR_mremap			(__NR_Linux +  24)
+#define __NR_msync			(__NR_Linux +  25)
+#define __NR_mincore			(__NR_Linux +  26)
+#define __NR_madvise			(__NR_Linux +  27)
+#define __NR_shmget			(__NR_Linux +  28)
+#define __NR_shmat			(__NR_Linux +  29)
+#define __NR_shmctl			(__NR_Linux +  30)
+#define __NR_dup			(__NR_Linux +  31)
+#define __NR_dup2			(__NR_Linux +  32)
+#define __NR_pause			(__NR_Linux +  33)
+#define __NR_nanosleep			(__NR_Linux +  34)
+#define __NR_getitimer			(__NR_Linux +  35)
+#define __NR_setitimer			(__NR_Linux +  36)
+#define __NR_alarm			(__NR_Linux +  37)
+#define __NR_getpid			(__NR_Linux +  38)
+#define __NR_sendfile			(__NR_Linux +  39)
+#define __NR_socket			(__NR_Linux +  40)
+#define __NR_connect			(__NR_Linux +  41)
+#define __NR_accept			(__NR_Linux +  42)
+#define __NR_sendto			(__NR_Linux +  43)
+#define __NR_recvfrom			(__NR_Linux +  44)
+#define __NR_sendmsg			(__NR_Linux +  45)
+#define __NR_recvmsg			(__NR_Linux +  46)
+#define __NR_shutdown			(__NR_Linux +  47)
+#define __NR_bind			(__NR_Linux +  48)
+#define __NR_listen			(__NR_Linux +  49)
+#define __NR_getsockname		(__NR_Linux +  50)
+#define __NR_getpeername		(__NR_Linux +  51)
+#define __NR_socketpair			(__NR_Linux +  52)
+#define __NR_setsockopt			(__NR_Linux +  53)
+#define __NR_getsockopt			(__NR_Linux +  54)
+#define __NR_clone			(__NR_Linux +  55)
+#define __NR_fork			(__NR_Linux +  56)
+#define __NR_execve			(__NR_Linux +  57)
+#define __NR_exit			(__NR_Linux +  58)
+#define __NR_wait4			(__NR_Linux +  59)
+#define __NR_kill			(__NR_Linux +  60)
+#define __NR_uname			(__NR_Linux +  61)
+#define __NR_semget			(__NR_Linux +  62)
+#define __NR_semop			(__NR_Linux +  63)
+#define __NR_semctl			(__NR_Linux +  64)
+#define __NR_shmdt			(__NR_Linux +  65)
+#define __NR_msgget			(__NR_Linux +  66)
+#define __NR_msgsnd			(__NR_Linux +  67)
+#define __NR_msgrcv			(__NR_Linux +  68)
+#define __NR_msgctl			(__NR_Linux +  69)
+#define __NR_fcntl			(__NR_Linux +  70)
+#define __NR_flock			(__NR_Linux +  71)
+#define __NR_fsync			(__NR_Linux +  72)
+#define __NR_fdatasync			(__NR_Linux +  73)
+#define __NR_truncate			(__NR_Linux +  74)
+#define __NR_ftruncate			(__NR_Linux +  75)
+#define __NR_getdents			(__NR_Linux +  76)
+#define __NR_getcwd			(__NR_Linux +  77)
+#define __NR_chdir			(__NR_Linux +  78)
+#define __NR_fchdir			(__NR_Linux +  79)
+#define __NR_rename			(__NR_Linux +  80)
+#define __NR_mkdir			(__NR_Linux +  81)
+#define __NR_rmdir			(__NR_Linux +  82)
+#define __NR_creat			(__NR_Linux +  83)
+#define __NR_link			(__NR_Linux +  84)
+#define __NR_unlink			(__NR_Linux +  85)
+#define __NR_symlink			(__NR_Linux +  86)
+#define __NR_readlink			(__NR_Linux +  87)
+#define __NR_chmod			(__NR_Linux +  88)
+#define __NR_fchmod			(__NR_Linux +  89)
+#define __NR_chown			(__NR_Linux +  90)
+#define __NR_fchown			(__NR_Linux +  91)
+#define __NR_lchown			(__NR_Linux +  92)
+#define __NR_umask			(__NR_Linux +  93)
+#define __NR_gettimeofday		(__NR_Linux +  94)
+#define __NR_getrlimit			(__NR_Linux +  95)
+#define __NR_getrusage			(__NR_Linux +  96)
+#define __NR_sysinfo			(__NR_Linux +  97)
+#define __NR_times			(__NR_Linux +  98)
+#define __NR_ptrace			(__NR_Linux +  99)
+#define __NR_getuid			(__NR_Linux + 100)
+#define __NR_syslog			(__NR_Linux + 101)
+#define __NR_getgid			(__NR_Linux + 102)
+#define __NR_setuid			(__NR_Linux + 103)
+#define __NR_setgid			(__NR_Linux + 104)
+#define __NR_geteuid			(__NR_Linux + 105)
+#define __NR_getegid			(__NR_Linux + 106)
+#define __NR_setpgid			(__NR_Linux + 107)
+#define __NR_getppid			(__NR_Linux + 108)
+#define __NR_getpgrp			(__NR_Linux + 109)
+#define __NR_setsid			(__NR_Linux + 110)
+#define __NR_setreuid			(__NR_Linux + 111)
+#define __NR_setregid			(__NR_Linux + 112)
+#define __NR_getgroups			(__NR_Linux + 113)
+#define __NR_setgroups			(__NR_Linux + 114)
+#define __NR_setresuid			(__NR_Linux + 115)
+#define __NR_getresuid			(__NR_Linux + 116)
+#define __NR_setresgid			(__NR_Linux + 117)
+#define __NR_getresgid			(__NR_Linux + 118)
+#define __NR_getpgid			(__NR_Linux + 119)
+#define __NR_setfsuid			(__NR_Linux + 120)
+#define __NR_setfsgid			(__NR_Linux + 121)
+#define __NR_getsid			(__NR_Linux + 122)
+#define __NR_capget			(__NR_Linux + 123)
+#define __NR_capset			(__NR_Linux + 124)
+#define __NR_rt_sigpending		(__NR_Linux + 125)
+#define __NR_rt_sigtimedwait		(__NR_Linux + 126)
+#define __NR_rt_sigqueueinfo		(__NR_Linux + 127)
+#define __NR_rt_sigsuspend		(__NR_Linux + 128)
+#define __NR_sigaltstack		(__NR_Linux + 129)
+#define __NR_utime			(__NR_Linux + 130)
+#define __NR_mknod			(__NR_Linux + 131)
+#define __NR_personality		(__NR_Linux + 132)
+#define __NR_ustat			(__NR_Linux + 133)
+#define __NR_statfs			(__NR_Linux + 134)
+#define __NR_fstatfs			(__NR_Linux + 135)
+#define __NR_sysfs			(__NR_Linux + 136)
+#define __NR_getpriority		(__NR_Linux + 137)
+#define __NR_setpriority		(__NR_Linux + 138)
+#define __NR_sched_setparam		(__NR_Linux + 139)
+#define __NR_sched_getparam		(__NR_Linux + 140)
+#define __NR_sched_setscheduler		(__NR_Linux + 141)
+#define __NR_sched_getscheduler		(__NR_Linux + 142)
+#define __NR_sched_get_priority_max	(__NR_Linux + 143)
+#define __NR_sched_get_priority_min	(__NR_Linux + 144)
+#define __NR_sched_rr_get_interval	(__NR_Linux + 145)
+#define __NR_mlock			(__NR_Linux + 146)
+#define __NR_munlock			(__NR_Linux + 147)
+#define __NR_mlockall			(__NR_Linux + 148)
+#define __NR_munlockall			(__NR_Linux + 149)
+#define __NR_vhangup			(__NR_Linux + 150)
+#define __NR_pivot_root			(__NR_Linux + 151)
+#define __NR__sysctl			(__NR_Linux + 152)
+#define __NR_prctl			(__NR_Linux + 153)
+#define __NR_adjtimex			(__NR_Linux + 154)
+#define __NR_setrlimit			(__NR_Linux + 155)
+#define __NR_chroot			(__NR_Linux + 156)
+#define __NR_sync			(__NR_Linux + 157)
+#define __NR_acct			(__NR_Linux + 158)
+#define __NR_settimeofday		(__NR_Linux + 159)
+#define __NR_mount			(__NR_Linux + 160)
+#define __NR_umount2			(__NR_Linux + 161)
+#define __NR_swapon			(__NR_Linux + 162)
+#define __NR_swapoff			(__NR_Linux + 163)
+#define __NR_reboot			(__NR_Linux + 164)
+#define __NR_sethostname		(__NR_Linux + 165)
+#define __NR_setdomainname		(__NR_Linux + 166)
+#define __NR_create_module		(__NR_Linux + 167)
+#define __NR_init_module		(__NR_Linux + 168)
+#define __NR_delete_module		(__NR_Linux + 169)
+#define __NR_get_kernel_syms		(__NR_Linux + 170)
+#define __NR_query_module		(__NR_Linux + 171)
+#define __NR_quotactl			(__NR_Linux + 172)
+#define __NR_nfsservctl			(__NR_Linux + 173)
+#define __NR_getpmsg			(__NR_Linux + 174)
+#define __NR_putpmsg			(__NR_Linux + 175)
+#define __NR_afs_syscall		(__NR_Linux + 176)
+#define __NR_reserved177		(__NR_Linux + 177)
+#define __NR_gettid			(__NR_Linux + 178)
+#define __NR_readahead			(__NR_Linux + 179)
+#define __NR_setxattr			(__NR_Linux + 180)
+#define __NR_lsetxattr			(__NR_Linux + 181)
+#define __NR_fsetxattr			(__NR_Linux + 182)
+#define __NR_getxattr			(__NR_Linux + 183)
+#define __NR_lgetxattr			(__NR_Linux + 184)
+#define __NR_fgetxattr			(__NR_Linux + 185)
+#define __NR_listxattr			(__NR_Linux + 186)
+#define __NR_llistxattr			(__NR_Linux + 187)
+#define __NR_flistxattr			(__NR_Linux + 188)
+#define __NR_removexattr		(__NR_Linux + 189)
+#define __NR_lremovexattr		(__NR_Linux + 190)
+#define __NR_fremovexattr		(__NR_Linux + 191)
+#define __NR_tkill			(__NR_Linux + 192)
+#define __NR_reserved193		(__NR_Linux + 193)
+#define __NR_futex			(__NR_Linux + 194)
+#define __NR_sched_setaffinity		(__NR_Linux + 195)
+#define __NR_sched_getaffinity		(__NR_Linux + 196)
+#define __NR_cacheflush			(__NR_Linux + 197)
+#define __NR_cachectl			(__NR_Linux + 198)
+#define __NR_sysmips			(__NR_Linux + 199)
+#define __NR_io_setup			(__NR_Linux + 200)
+#define __NR_io_destroy			(__NR_Linux + 201)
+#define __NR_io_getevents		(__NR_Linux + 202)
+#define __NR_io_submit			(__NR_Linux + 203)
+#define __NR_io_cancel			(__NR_Linux + 204)
+#define __NR_exit_group			(__NR_Linux + 205)
+#define __NR_lookup_dcookie		(__NR_Linux + 206)
+#define __NR_epoll_create		(__NR_Linux + 207)
+#define __NR_epoll_ctl			(__NR_Linux + 208)
+#define __NR_epoll_wait			(__NR_Linux + 209)
+#define __NR_remap_file_pages		(__NR_Linux + 210)
+#define __NR_rt_sigreturn		(__NR_Linux + 211)
+#define __NR_fcntl64			(__NR_Linux + 212)
+#define __NR_set_tid_address		(__NR_Linux + 213)
+#define __NR_restart_syscall		(__NR_Linux + 214)
+#define __NR_semtimedop			(__NR_Linux + 215)
+#define __NR_fadvise64			(__NR_Linux + 216)
+#define __NR_statfs64			(__NR_Linux + 217)
+#define __NR_fstatfs64			(__NR_Linux + 218)
+#define __NR_sendfile64			(__NR_Linux + 219)
+#define __NR_timer_create		(__NR_Linux + 220)
+#define __NR_timer_settime		(__NR_Linux + 221)
+#define __NR_timer_gettime		(__NR_Linux + 222)
+#define __NR_timer_getoverrun		(__NR_Linux + 223)
+#define __NR_timer_delete		(__NR_Linux + 224)
+#define __NR_clock_settime		(__NR_Linux + 225)
+#define __NR_clock_gettime		(__NR_Linux + 226)
+#define __NR_clock_getres		(__NR_Linux + 227)
+#define __NR_clock_nanosleep		(__NR_Linux + 228)
+#define __NR_tgkill			(__NR_Linux + 229)
+#define __NR_utimes			(__NR_Linux + 230)
+#define __NR_mbind			(__NR_Linux + 231)
+#define __NR_get_mempolicy		(__NR_Linux + 232)
+#define __NR_set_mempolicy		(__NR_Linux + 233)
+#define __NR_mq_open			(__NR_Linux + 234)
+#define __NR_mq_unlink			(__NR_Linux + 235)
+#define __NR_mq_timedsend		(__NR_Linux + 236)
+#define __NR_mq_timedreceive		(__NR_Linux + 237)
+#define __NR_mq_notify			(__NR_Linux + 238)
+#define __NR_mq_getsetattr		(__NR_Linux + 239)
+#define __NR_vserver			(__NR_Linux + 240)
+#define __NR_waitid			(__NR_Linux + 241)
+/* #define __NR_sys_setaltroot		(__NR_Linux + 242) */
+#define __NR_add_key			(__NR_Linux + 243)
+#define __NR_request_key		(__NR_Linux + 244)
+#define __NR_keyctl			(__NR_Linux + 245)
+#define __NR_set_thread_area		(__NR_Linux + 246)
+#define __NR_inotify_init		(__NR_Linux + 247)
+#define __NR_inotify_add_watch		(__NR_Linux + 248)
+#define __NR_inotify_rm_watch		(__NR_Linux + 249)
+#define __NR_migrate_pages		(__NR_Linux + 250)
+#define __NR_openat			(__NR_Linux + 251)
+#define __NR_mkdirat			(__NR_Linux + 252)
+#define __NR_mknodat			(__NR_Linux + 253)
+#define __NR_fchownat			(__NR_Linux + 254)
+#define __NR_futimesat			(__NR_Linux + 255)
+#define __NR_newfstatat			(__NR_Linux + 256)
+#define __NR_unlinkat			(__NR_Linux + 257)
+#define __NR_renameat			(__NR_Linux + 258)
+#define __NR_linkat			(__NR_Linux + 259)
+#define __NR_symlinkat			(__NR_Linux + 260)
+#define __NR_readlinkat			(__NR_Linux + 261)
+#define __NR_fchmodat			(__NR_Linux + 262)
+#define __NR_faccessat			(__NR_Linux + 263)
+#define __NR_pselect6			(__NR_Linux + 264)
+#define __NR_ppoll			(__NR_Linux + 265)
+#define __NR_unshare			(__NR_Linux + 266)
+#define __NR_splice			(__NR_Linux + 267)
+#define __NR_sync_file_range		(__NR_Linux + 268)
+#define __NR_tee			(__NR_Linux + 269)
+#define __NR_vmsplice			(__NR_Linux + 270)
+#define __NR_move_pages			(__NR_Linux + 271)
+#define __NR_set_robust_list		(__NR_Linux + 272)
+#define __NR_get_robust_list		(__NR_Linux + 273)
+#define __NR_kexec_load			(__NR_Linux + 274)
+#define __NR_getcpu			(__NR_Linux + 275)
+#define __NR_epoll_pwait		(__NR_Linux + 276)
+#define __NR_ioprio_set			(__NR_Linux + 277)
+#define __NR_ioprio_get			(__NR_Linux + 278)
+#define __NR_utimensat			(__NR_Linux + 279)
+#define __NR_signalfd			(__NR_Linux + 280)
+#define __NR_timerfd			(__NR_Linux + 281)
+#define __NR_eventfd			(__NR_Linux + 282)
+#define __NR_fallocate			(__NR_Linux + 283)
+#define __NR_timerfd_create		(__NR_Linux + 284)
+#define __NR_timerfd_gettime		(__NR_Linux + 285)
+#define __NR_timerfd_settime		(__NR_Linux + 286)
+#define __NR_signalfd4			(__NR_Linux + 287)
+#define __NR_eventfd2			(__NR_Linux + 288)
+#define __NR_epoll_create1		(__NR_Linux + 289)
+#define __NR_dup3			(__NR_Linux + 290)
+#define __NR_pipe2			(__NR_Linux + 291)
+#define __NR_inotify_init1		(__NR_Linux + 292)
+#define __NR_preadv			(__NR_Linux + 293)
+#define __NR_pwritev			(__NR_Linux + 294)
+#define __NR_rt_tgsigqueueinfo		(__NR_Linux + 295)
+#define __NR_perf_event_open		(__NR_Linux + 296)
+#define __NR_accept4			(__NR_Linux + 297)
+#define __NR_recvmmsg			(__NR_Linux + 298)
+#define __NR_getdents64			(__NR_Linux + 299)
+#define __NR_fanotify_init		(__NR_Linux + 300)
+#define __NR_fanotify_mark		(__NR_Linux + 301)
+#define __NR_prlimit64			(__NR_Linux + 302)
+#define __NR_name_to_handle_at		(__NR_Linux + 303)
+#define __NR_open_by_handle_at		(__NR_Linux + 304)
+#define __NR_clock_adjtime		(__NR_Linux + 305)
+#define __NR_syncfs			(__NR_Linux + 306)
+#define __NR_sendmmsg			(__NR_Linux + 307)
+#define __NR_setns			(__NR_Linux + 308)
+#define __NR_process_vm_readv		(__NR_Linux + 309)
+#define __NR_process_vm_writev		(__NR_Linux + 310)
+#define __NR_kcmp			(__NR_Linux + 311)
+#define __NR_finit_module		(__NR_Linux + 312)
+#define __NR_sched_setattr		(__NR_Linux + 313)
+#define __NR_sched_getattr		(__NR_Linux + 314)
+#define __NR_renameat2			(__NR_Linux + 315)
+#define __NR_seccomp			(__NR_Linux + 316)
+#define __NR_getrandom			(__NR_Linux + 317)
+#define __NR_memfd_create		(__NR_Linux + 318)
+#define __NR_bpf			(__NR_Linux + 319)
+#define __NR_execveat			(__NR_Linux + 320)
+
+/*
+ * Offset of the last N32 flavoured syscall
+ */
+#define __NR_Linux_syscalls		320
+
+#endif /* _MIPS_SIM == _MIPS_SIM_NABI32 */
+
+#define __NR_N32_Linux			6000
+#define __NR_N32_Linux_syscalls		320
+
+#endif /* _ASM_UNISTD_H */
diff --git a/linux-headers/asm-powerpc/epapr_hcalls.h b/linux-headers/asm-powerpc/epapr_hcalls.h
index 06f7247..33b3f89 100644
--- a/linux-headers/asm-powerpc/epapr_hcalls.h
+++ b/linux-headers/asm-powerpc/epapr_hcalls.h
@@ -78,7 +78,7 @@
 #define EV_SUCCESS		0
 #define EV_EPERM		1	/* Operation not permitted */
 #define EV_ENOENT		2	/*  Entry Not Found */
-#define EV_EIO			3	/* I/O error occured */
+#define EV_EIO			3	/* I/O error occurred */
 #define EV_EAGAIN		4	/* The operation had insufficient
 					 * resources to complete and should be
 					 * retried
@@ -89,7 +89,7 @@
 #define EV_ENODEV		7	/* No such device */
 #define EV_EINVAL		8	/* An argument supplied to the hcall
 					   was out of range or invalid */
-#define EV_INTERNAL		9	/* An internal error occured */
+#define EV_INTERNAL		9	/* An internal error occurred */
 #define EV_CONFIG		10	/* A configuration error was detected */
 #define EV_INVALID_STATE	11	/* The object is in an invalid state */
 #define EV_UNIMPLEMENTED	12	/* Unimplemented hypercall */
diff --git a/linux-headers/asm-powerpc/kvm.h b/linux-headers/asm-powerpc/kvm.h
index ab4d473..c93cf35 100644
--- a/linux-headers/asm-powerpc/kvm.h
+++ b/linux-headers/asm-powerpc/kvm.h
@@ -333,6 +333,15 @@
 	__u32 window_size;
 };
 
+/* for KVM_CAP_SPAPR_TCE_64 */
+struct kvm_create_spapr_tce_64 {
+	__u64 liobn;
+	__u32 page_shift;
+	__u32 flags;
+	__u64 offset;	/* in pages */
+	__u64 size;	/* in pages */
+};
+
 /* for KVM_ALLOCATE_RMA */
 struct kvm_allocate_rma {
 	__u64 rma_size;
diff --git a/linux-headers/asm-powerpc/unistd.h b/linux-headers/asm-powerpc/unistd.h
new file mode 100644
index 0000000..1e66eba
--- /dev/null
+++ b/linux-headers/asm-powerpc/unistd.h
@@ -0,0 +1,396 @@
+/*
+ * This file contains the system call numbers.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ */
+#ifndef _ASM_POWERPC_UNISTD_H_
+#define _ASM_POWERPC_UNISTD_H_
+
+
+#define __NR_restart_syscall	  0
+#define __NR_exit		  1
+#define __NR_fork		  2
+#define __NR_read		  3
+#define __NR_write		  4
+#define __NR_open		  5
+#define __NR_close		  6
+#define __NR_waitpid		  7
+#define __NR_creat		  8
+#define __NR_link		  9
+#define __NR_unlink		 10
+#define __NR_execve		 11
+#define __NR_chdir		 12
+#define __NR_time		 13
+#define __NR_mknod		 14
+#define __NR_chmod		 15
+#define __NR_lchown		 16
+#define __NR_break		 17
+#define __NR_oldstat		 18
+#define __NR_lseek		 19
+#define __NR_getpid		 20
+#define __NR_mount		 21
+#define __NR_umount		 22
+#define __NR_setuid		 23
+#define __NR_getuid		 24
+#define __NR_stime		 25
+#define __NR_ptrace		 26
+#define __NR_alarm		 27
+#define __NR_oldfstat		 28
+#define __NR_pause		 29
+#define __NR_utime		 30
+#define __NR_stty		 31
+#define __NR_gtty		 32
+#define __NR_access		 33
+#define __NR_nice		 34
+#define __NR_ftime		 35
+#define __NR_sync		 36
+#define __NR_kill		 37
+#define __NR_rename		 38
+#define __NR_mkdir		 39
+#define __NR_rmdir		 40
+#define __NR_dup		 41
+#define __NR_pipe		 42
+#define __NR_times		 43
+#define __NR_prof		 44
+#define __NR_brk		 45
+#define __NR_setgid		 46
+#define __NR_getgid		 47
+#define __NR_signal		 48
+#define __NR_geteuid		 49
+#define __NR_getegid		 50
+#define __NR_acct		 51
+#define __NR_umount2		 52
+#define __NR_lock		 53
+#define __NR_ioctl		 54
+#define __NR_fcntl		 55
+#define __NR_mpx		 56
+#define __NR_setpgid		 57
+#define __NR_ulimit		 58
+#define __NR_oldolduname	 59
+#define __NR_umask		 60
+#define __NR_chroot		 61
+#define __NR_ustat		 62
+#define __NR_dup2		 63
+#define __NR_getppid		 64
+#define __NR_getpgrp		 65
+#define __NR_setsid		 66
+#define __NR_sigaction		 67
+#define __NR_sgetmask		 68
+#define __NR_ssetmask		 69
+#define __NR_setreuid		 70
+#define __NR_setregid		 71
+#define __NR_sigsuspend		 72
+#define __NR_sigpending		 73
+#define __NR_sethostname	 74
+#define __NR_setrlimit		 75
+#define __NR_getrlimit		 76
+#define __NR_getrusage		 77
+#define __NR_gettimeofday	 78
+#define __NR_settimeofday	 79
+#define __NR_getgroups		 80
+#define __NR_setgroups		 81
+#define __NR_select		 82
+#define __NR_symlink		 83
+#define __NR_oldlstat		 84
+#define __NR_readlink		 85
+#define __NR_uselib		 86
+#define __NR_swapon		 87
+#define __NR_reboot		 88
+#define __NR_readdir		 89
+#define __NR_mmap		 90
+#define __NR_munmap		 91
+#define __NR_truncate		 92
+#define __NR_ftruncate		 93
+#define __NR_fchmod		 94
+#define __NR_fchown		 95
+#define __NR_getpriority	 96
+#define __NR_setpriority	 97
+#define __NR_profil		 98
+#define __NR_statfs		 99
+#define __NR_fstatfs		100
+#define __NR_ioperm		101
+#define __NR_socketcall		102
+#define __NR_syslog		103
+#define __NR_setitimer		104
+#define __NR_getitimer		105
+#define __NR_stat		106
+#define __NR_lstat		107
+#define __NR_fstat		108
+#define __NR_olduname		109
+#define __NR_iopl		110
+#define __NR_vhangup		111
+#define __NR_idle		112
+#define __NR_vm86		113
+#define __NR_wait4		114
+#define __NR_swapoff		115
+#define __NR_sysinfo		116
+#define __NR_ipc		117
+#define __NR_fsync		118
+#define __NR_sigreturn		119
+#define __NR_clone		120
+#define __NR_setdomainname	121
+#define __NR_uname		122
+#define __NR_modify_ldt		123
+#define __NR_adjtimex		124
+#define __NR_mprotect		125
+#define __NR_sigprocmask	126
+#define __NR_create_module	127
+#define __NR_init_module	128
+#define __NR_delete_module	129
+#define __NR_get_kernel_syms	130
+#define __NR_quotactl		131
+#define __NR_getpgid		132
+#define __NR_fchdir		133
+#define __NR_bdflush		134
+#define __NR_sysfs		135
+#define __NR_personality	136
+#define __NR_afs_syscall	137 /* Syscall for Andrew File System */
+#define __NR_setfsuid		138
+#define __NR_setfsgid		139
+#define __NR__llseek		140
+#define __NR_getdents		141
+#define __NR__newselect		142
+#define __NR_flock		143
+#define __NR_msync		144
+#define __NR_readv		145
+#define __NR_writev		146
+#define __NR_getsid		147
+#define __NR_fdatasync		148
+#define __NR__sysctl		149
+#define __NR_mlock		150
+#define __NR_munlock		151
+#define __NR_mlockall		152
+#define __NR_munlockall		153
+#define __NR_sched_setparam		154
+#define __NR_sched_getparam		155
+#define __NR_sched_setscheduler		156
+#define __NR_sched_getscheduler		157
+#define __NR_sched_yield		158
+#define __NR_sched_get_priority_max	159
+#define __NR_sched_get_priority_min	160
+#define __NR_sched_rr_get_interval	161
+#define __NR_nanosleep		162
+#define __NR_mremap		163
+#define __NR_setresuid		164
+#define __NR_getresuid		165
+#define __NR_query_module	166
+#define __NR_poll		167
+#define __NR_nfsservctl		168
+#define __NR_setresgid		169
+#define __NR_getresgid		170
+#define __NR_prctl		171
+#define __NR_rt_sigreturn	172
+#define __NR_rt_sigaction	173
+#define __NR_rt_sigprocmask	174
+#define __NR_rt_sigpending	175
+#define __NR_rt_sigtimedwait	176
+#define __NR_rt_sigqueueinfo	177
+#define __NR_rt_sigsuspend	178
+#define __NR_pread64		179
+#define __NR_pwrite64		180
+#define __NR_chown		181
+#define __NR_getcwd		182
+#define __NR_capget		183
+#define __NR_capset		184
+#define __NR_sigaltstack	185
+#define __NR_sendfile		186
+#define __NR_getpmsg		187	/* some people actually want streams */
+#define __NR_putpmsg		188	/* some people actually want streams */
+#define __NR_vfork		189
+#define __NR_ugetrlimit		190	/* SuS compliant getrlimit */
+#define __NR_readahead		191
+#ifndef __powerpc64__			/* these are 32-bit only */
+#define __NR_mmap2		192
+#define __NR_truncate64		193
+#define __NR_ftruncate64	194
+#define __NR_stat64		195
+#define __NR_lstat64		196
+#define __NR_fstat64		197
+#endif
+#define __NR_pciconfig_read	198
+#define __NR_pciconfig_write	199
+#define __NR_pciconfig_iobase	200
+#define __NR_multiplexer	201
+#define __NR_getdents64		202
+#define __NR_pivot_root		203
+#ifndef __powerpc64__
+#define __NR_fcntl64		204
+#endif
+#define __NR_madvise		205
+#define __NR_mincore		206
+#define __NR_gettid		207
+#define __NR_tkill		208
+#define __NR_setxattr		209
+#define __NR_lsetxattr		210
+#define __NR_fsetxattr		211
+#define __NR_getxattr		212
+#define __NR_lgetxattr		213
+#define __NR_fgetxattr		214
+#define __NR_listxattr		215
+#define __NR_llistxattr		216
+#define __NR_flistxattr		217
+#define __NR_removexattr	218
+#define __NR_lremovexattr	219
+#define __NR_fremovexattr	220
+#define __NR_futex		221
+#define __NR_sched_setaffinity	222
+#define __NR_sched_getaffinity	223
+/* 224 currently unused */
+#define __NR_tuxcall		225
+#ifndef __powerpc64__
+#define __NR_sendfile64		226
+#endif
+#define __NR_io_setup		227
+#define __NR_io_destroy		228
+#define __NR_io_getevents	229
+#define __NR_io_submit		230
+#define __NR_io_cancel		231
+#define __NR_set_tid_address	232
+#define __NR_fadvise64		233
+#define __NR_exit_group		234
+#define __NR_lookup_dcookie	235
+#define __NR_epoll_create	236
+#define __NR_epoll_ctl		237
+#define __NR_epoll_wait		238
+#define __NR_remap_file_pages	239
+#define __NR_timer_create	240
+#define __NR_timer_settime	241
+#define __NR_timer_gettime	242
+#define __NR_timer_getoverrun	243
+#define __NR_timer_delete	244
+#define __NR_clock_settime	245
+#define __NR_clock_gettime	246
+#define __NR_clock_getres	247
+#define __NR_clock_nanosleep	248
+#define __NR_swapcontext	249
+#define __NR_tgkill		250
+#define __NR_utimes		251
+#define __NR_statfs64		252
+#define __NR_fstatfs64		253
+#ifndef __powerpc64__
+#define __NR_fadvise64_64	254
+#endif
+#define __NR_rtas		255
+#define __NR_sys_debug_setcontext 256
+/* Number 257 is reserved for vserver */
+#define __NR_migrate_pages	258
+#define __NR_mbind		259
+#define __NR_get_mempolicy	260
+#define __NR_set_mempolicy	261
+#define __NR_mq_open		262
+#define __NR_mq_unlink		263
+#define __NR_mq_timedsend	264
+#define __NR_mq_timedreceive	265
+#define __NR_mq_notify		266
+#define __NR_mq_getsetattr	267
+#define __NR_kexec_load		268
+#define __NR_add_key		269
+#define __NR_request_key	270
+#define __NR_keyctl		271
+#define __NR_waitid		272
+#define __NR_ioprio_set		273
+#define __NR_ioprio_get		274
+#define __NR_inotify_init	275
+#define __NR_inotify_add_watch	276
+#define __NR_inotify_rm_watch	277
+#define __NR_spu_run		278
+#define __NR_spu_create		279
+#define __NR_pselect6		280
+#define __NR_ppoll		281
+#define __NR_unshare		282
+#define __NR_splice		283
+#define __NR_tee		284
+#define __NR_vmsplice		285
+#define __NR_openat		286
+#define __NR_mkdirat		287
+#define __NR_mknodat		288
+#define __NR_fchownat		289
+#define __NR_futimesat		290
+#ifdef __powerpc64__
+#define __NR_newfstatat		291
+#else
+#define __NR_fstatat64		291
+#endif
+#define __NR_unlinkat		292
+#define __NR_renameat		293
+#define __NR_linkat		294
+#define __NR_symlinkat		295
+#define __NR_readlinkat		296
+#define __NR_fchmodat		297
+#define __NR_faccessat		298
+#define __NR_get_robust_list	299
+#define __NR_set_robust_list	300
+#define __NR_move_pages		301
+#define __NR_getcpu		302
+#define __NR_epoll_pwait	303
+#define __NR_utimensat		304
+#define __NR_signalfd		305
+#define __NR_timerfd_create	306
+#define __NR_eventfd		307
+#define __NR_sync_file_range2	308
+#define __NR_fallocate		309
+#define __NR_subpage_prot	310
+#define __NR_timerfd_settime	311
+#define __NR_timerfd_gettime	312
+#define __NR_signalfd4		313
+#define __NR_eventfd2		314
+#define __NR_epoll_create1	315
+#define __NR_dup3		316
+#define __NR_pipe2		317
+#define __NR_inotify_init1	318
+#define __NR_perf_event_open	319
+#define __NR_preadv		320
+#define __NR_pwritev		321
+#define __NR_rt_tgsigqueueinfo	322
+#define __NR_fanotify_init	323
+#define __NR_fanotify_mark	324
+#define __NR_prlimit64		325
+#define __NR_socket		326
+#define __NR_bind		327
+#define __NR_connect		328
+#define __NR_listen		329
+#define __NR_accept		330
+#define __NR_getsockname	331
+#define __NR_getpeername	332
+#define __NR_socketpair		333
+#define __NR_send		334
+#define __NR_sendto		335
+#define __NR_recv		336
+#define __NR_recvfrom		337
+#define __NR_shutdown		338
+#define __NR_setsockopt		339
+#define __NR_getsockopt		340
+#define __NR_sendmsg		341
+#define __NR_recvmsg		342
+#define __NR_recvmmsg		343
+#define __NR_accept4		344
+#define __NR_name_to_handle_at	345
+#define __NR_open_by_handle_at	346
+#define __NR_clock_adjtime	347
+#define __NR_syncfs		348
+#define __NR_sendmmsg		349
+#define __NR_setns		350
+#define __NR_process_vm_readv	351
+#define __NR_process_vm_writev	352
+#define __NR_finit_module	353
+#define __NR_kcmp		354
+#define __NR_sched_setattr	355
+#define __NR_sched_getattr	356
+#define __NR_renameat2		357
+#define __NR_seccomp		358
+#define __NR_getrandom		359
+#define __NR_memfd_create	360
+#define __NR_bpf		361
+#define __NR_execveat		362
+#define __NR_switch_endian	363
+#define __NR_userfaultfd	364
+#define __NR_membarrier		365
+#define __NR_mlock2		378
+#define __NR_copy_file_range	379
+#define __NR_preadv2		380
+#define __NR_pwritev2		381
+
+#endif /* _ASM_POWERPC_UNISTD_H_ */
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index d36b2fa..09ae5dc 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -25,6 +25,7 @@
 #define KVM_DEV_FLIC_APF_DISABLE_WAIT	5
 #define KVM_DEV_FLIC_ADAPTER_REGISTER	6
 #define KVM_DEV_FLIC_ADAPTER_MODIFY	7
+#define KVM_DEV_FLIC_CLEAR_IO_IRQ	8
 /*
  * We can have up to 4*64k pending subchannels + 8 adapter interrupts,
  * as well as up  to ASYNC_PF_PER_VCPU*KVM_MAX_VCPUS pfault done interrupts.
@@ -57,10 +58,46 @@
 
 /* kvm attr_group  on vm fd */
 #define KVM_S390_VM_MEM_CTRL		0
+#define KVM_S390_VM_TOD			1
+#define KVM_S390_VM_CRYPTO		2
+#define KVM_S390_VM_CPU_MODEL		3
 
 /* kvm attributes for mem_ctrl */
 #define KVM_S390_VM_MEM_ENABLE_CMMA	0
 #define KVM_S390_VM_MEM_CLR_CMMA	1
+#define KVM_S390_VM_MEM_LIMIT_SIZE	2
+
+#define KVM_S390_NO_MEM_LIMIT		U64_MAX
+
+/* kvm attributes for KVM_S390_VM_TOD */
+#define KVM_S390_VM_TOD_LOW		0
+#define KVM_S390_VM_TOD_HIGH		1
+
+/* kvm attributes for KVM_S390_VM_CPU_MODEL */
+/* processor related attributes are r/w */
+#define KVM_S390_VM_CPU_PROCESSOR	0
+struct kvm_s390_vm_cpu_processor {
+	__u64 cpuid;
+	__u16 ibc;
+	__u8  pad[6];
+	__u64 fac_list[256];
+};
+
+/* machine related attributes are r/o */
+#define KVM_S390_VM_CPU_MACHINE		1
+struct kvm_s390_vm_cpu_machine {
+	__u64 cpuid;
+	__u32 ibc;
+	__u8  pad[4];
+	__u64 fac_mask[256];
+	__u64 fac_list[256];
+};
+
+/* kvm attributes for crypto */
+#define KVM_S390_VM_CRYPTO_ENABLE_AES_KW	0
+#define KVM_S390_VM_CRYPTO_ENABLE_DEA_KW	1
+#define KVM_S390_VM_CRYPTO_DISABLE_AES_KW	2
+#define KVM_S390_VM_CRYPTO_DISABLE_DEA_KW	3
 
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
@@ -107,12 +144,18 @@
 	struct kvm_hw_breakpoint *hw_bp;
 };
 
+/* for KVM_SYNC_PFAULT and KVM_REG_S390_PFTOKEN */
+#define KVM_S390_PFAULT_TOKEN_INVALID	0xffffffffffffffffULL
+
 #define KVM_SYNC_PREFIX (1UL << 0)
 #define KVM_SYNC_GPRS   (1UL << 1)
 #define KVM_SYNC_ACRS   (1UL << 2)
 #define KVM_SYNC_CRS    (1UL << 3)
 #define KVM_SYNC_ARCH0  (1UL << 4)
 #define KVM_SYNC_PFAULT (1UL << 5)
+#define KVM_SYNC_VRS    (1UL << 6)
+#define KVM_SYNC_RICCB  (1UL << 7)
+#define KVM_SYNC_FPRS   (1UL << 8)
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
 	__u64 prefix;	/* prefix register */
@@ -127,6 +170,14 @@
 	__u64 pft;	/* pfault token [PFAULT] */
 	__u64 pfs;	/* pfault select [PFAULT] */
 	__u64 pfc;	/* pfault compare [PFAULT] */
+	union {
+		__u64 vrs[32][2];	/* vector registers (KVM_SYNC_VRS) */
+		__u64 fprs[16];		/* fp registers (KVM_SYNC_FPRS) */
+	};
+	__u8  reserved[512];	/* for future vector expansion */
+	__u32 fpc;		/* valid on KVM_SYNC_VRS or KVM_SYNC_FPRS */
+	__u8 padding[52];	/* riccb needs to be 64byte aligned */
+	__u8 riccb[64];		/* runtime instrumentation controls block */
 };
 
 #define KVM_REG_S390_TODPR	(KVM_REG_S390 | KVM_REG_SIZE_U32 | 0x1)
diff --git a/linux-headers/asm-s390/unistd.h b/linux-headers/asm-s390/unistd.h
new file mode 100644
index 0000000..8a404fd
--- /dev/null
+++ b/linux-headers/asm-s390/unistd.h
@@ -0,0 +1,408 @@
+/*
+ *  S390 version
+ *
+ *  Derived from "include/asm-i386/unistd.h"
+ */
+
+#ifndef _ASM_S390_UNISTD_H_
+#define _ASM_S390_UNISTD_H_
+
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_exit                 1
+#define __NR_fork                 2
+#define __NR_read                 3
+#define __NR_write                4
+#define __NR_open                 5
+#define __NR_close                6
+#define __NR_restart_syscall	  7
+#define __NR_creat                8
+#define __NR_link                 9
+#define __NR_unlink              10
+#define __NR_execve              11
+#define __NR_chdir               12
+#define __NR_mknod               14
+#define __NR_chmod               15
+#define __NR_lseek               19
+#define __NR_getpid              20
+#define __NR_mount               21
+#define __NR_umount              22
+#define __NR_ptrace              26
+#define __NR_alarm               27
+#define __NR_pause               29
+#define __NR_utime               30
+#define __NR_access              33
+#define __NR_nice                34
+#define __NR_sync                36
+#define __NR_kill                37
+#define __NR_rename              38
+#define __NR_mkdir               39
+#define __NR_rmdir               40
+#define __NR_dup                 41
+#define __NR_pipe                42
+#define __NR_times               43
+#define __NR_brk                 45
+#define __NR_signal              48
+#define __NR_acct                51
+#define __NR_umount2             52
+#define __NR_ioctl               54
+#define __NR_fcntl               55
+#define __NR_setpgid             57
+#define __NR_umask               60
+#define __NR_chroot              61
+#define __NR_ustat               62
+#define __NR_dup2                63
+#define __NR_getppid             64
+#define __NR_getpgrp             65
+#define __NR_setsid              66
+#define __NR_sigaction           67
+#define __NR_sigsuspend          72
+#define __NR_sigpending          73
+#define __NR_sethostname         74
+#define __NR_setrlimit           75
+#define __NR_getrusage           77
+#define __NR_gettimeofday        78
+#define __NR_settimeofday        79
+#define __NR_symlink             83
+#define __NR_readlink            85
+#define __NR_uselib              86
+#define __NR_swapon              87
+#define __NR_reboot              88
+#define __NR_readdir             89
+#define __NR_mmap                90
+#define __NR_munmap              91
+#define __NR_truncate            92
+#define __NR_ftruncate           93
+#define __NR_fchmod              94
+#define __NR_getpriority         96
+#define __NR_setpriority         97
+#define __NR_statfs              99
+#define __NR_fstatfs            100
+#define __NR_socketcall         102
+#define __NR_syslog             103
+#define __NR_setitimer          104
+#define __NR_getitimer          105
+#define __NR_stat               106
+#define __NR_lstat              107
+#define __NR_fstat              108
+#define __NR_lookup_dcookie     110
+#define __NR_vhangup            111
+#define __NR_idle               112
+#define __NR_wait4              114
+#define __NR_swapoff            115
+#define __NR_sysinfo            116
+#define __NR_ipc                117
+#define __NR_fsync              118
+#define __NR_sigreturn          119
+#define __NR_clone              120
+#define __NR_setdomainname      121
+#define __NR_uname              122
+#define __NR_adjtimex           124
+#define __NR_mprotect           125
+#define __NR_sigprocmask        126
+#define __NR_create_module      127
+#define __NR_init_module        128
+#define __NR_delete_module      129
+#define __NR_get_kernel_syms    130
+#define __NR_quotactl           131
+#define __NR_getpgid            132
+#define __NR_fchdir             133
+#define __NR_bdflush            134
+#define __NR_sysfs              135
+#define __NR_personality        136
+#define __NR_afs_syscall        137 /* Syscall for Andrew File System */
+#define __NR_getdents           141
+#define __NR_flock              143
+#define __NR_msync              144
+#define __NR_readv              145
+#define __NR_writev             146
+#define __NR_getsid             147
+#define __NR_fdatasync          148
+#define __NR__sysctl            149
+#define __NR_mlock              150
+#define __NR_munlock            151
+#define __NR_mlockall           152
+#define __NR_munlockall         153
+#define __NR_sched_setparam             154
+#define __NR_sched_getparam             155
+#define __NR_sched_setscheduler         156
+#define __NR_sched_getscheduler         157
+#define __NR_sched_yield                158
+#define __NR_sched_get_priority_max     159
+#define __NR_sched_get_priority_min     160
+#define __NR_sched_rr_get_interval      161
+#define __NR_nanosleep          162
+#define __NR_mremap             163
+#define __NR_query_module       167
+#define __NR_poll               168
+#define __NR_nfsservctl         169
+#define __NR_prctl              172
+#define __NR_rt_sigreturn       173
+#define __NR_rt_sigaction       174
+#define __NR_rt_sigprocmask     175
+#define __NR_rt_sigpending      176
+#define __NR_rt_sigtimedwait    177
+#define __NR_rt_sigqueueinfo    178
+#define __NR_rt_sigsuspend      179
+#define __NR_pread64            180
+#define __NR_pwrite64           181
+#define __NR_getcwd             183
+#define __NR_capget             184
+#define __NR_capset             185
+#define __NR_sigaltstack        186
+#define __NR_sendfile           187
+#define __NR_getpmsg		188
+#define __NR_putpmsg		189
+#define __NR_vfork		190
+#define __NR_pivot_root         217
+#define __NR_mincore            218
+#define __NR_madvise            219
+#define __NR_getdents64		220
+#define __NR_readahead		222
+#define __NR_setxattr		224
+#define __NR_lsetxattr		225
+#define __NR_fsetxattr		226
+#define __NR_getxattr		227
+#define __NR_lgetxattr		228
+#define __NR_fgetxattr		229
+#define __NR_listxattr		230
+#define __NR_llistxattr		231
+#define __NR_flistxattr		232
+#define __NR_removexattr	233
+#define __NR_lremovexattr	234
+#define __NR_fremovexattr	235
+#define __NR_gettid		236
+#define __NR_tkill		237
+#define __NR_futex		238
+#define __NR_sched_setaffinity	239
+#define __NR_sched_getaffinity	240
+#define __NR_tgkill		241
+/* Number 242 is reserved for tux */
+#define __NR_io_setup		243
+#define __NR_io_destroy		244
+#define __NR_io_getevents	245
+#define __NR_io_submit		246
+#define __NR_io_cancel		247
+#define __NR_exit_group		248
+#define __NR_epoll_create	249
+#define __NR_epoll_ctl		250
+#define __NR_epoll_wait		251
+#define __NR_set_tid_address	252
+#define __NR_fadvise64		253
+#define __NR_timer_create	254
+#define __NR_timer_settime	255
+#define __NR_timer_gettime	256
+#define __NR_timer_getoverrun	257
+#define __NR_timer_delete	258
+#define __NR_clock_settime	259
+#define __NR_clock_gettime	260
+#define __NR_clock_getres	261
+#define __NR_clock_nanosleep	262
+/* Number 263 is reserved for vserver */
+#define __NR_statfs64		265
+#define __NR_fstatfs64		266
+#define __NR_remap_file_pages	267
+#define __NR_mbind		268
+#define __NR_get_mempolicy	269
+#define __NR_set_mempolicy	270
+#define __NR_mq_open		271
+#define __NR_mq_unlink		272
+#define __NR_mq_timedsend	273
+#define __NR_mq_timedreceive	274
+#define __NR_mq_notify		275
+#define __NR_mq_getsetattr	276
+#define __NR_kexec_load		277
+#define __NR_add_key		278
+#define __NR_request_key	279
+#define __NR_keyctl		280
+#define __NR_waitid		281
+#define __NR_ioprio_set		282
+#define __NR_ioprio_get		283
+#define __NR_inotify_init	284
+#define __NR_inotify_add_watch	285
+#define __NR_inotify_rm_watch	286
+#define __NR_migrate_pages	287
+#define __NR_openat		288
+#define __NR_mkdirat		289
+#define __NR_mknodat		290
+#define __NR_fchownat		291
+#define __NR_futimesat		292
+#define __NR_unlinkat		294
+#define __NR_renameat		295
+#define __NR_linkat		296
+#define __NR_symlinkat		297
+#define __NR_readlinkat		298
+#define __NR_fchmodat		299
+#define __NR_faccessat		300
+#define __NR_pselect6		301
+#define __NR_ppoll		302
+#define __NR_unshare		303
+#define __NR_set_robust_list	304
+#define __NR_get_robust_list	305
+#define __NR_splice		306
+#define __NR_sync_file_range	307
+#define __NR_tee		308
+#define __NR_vmsplice		309
+#define __NR_move_pages		310
+#define __NR_getcpu		311
+#define __NR_epoll_pwait	312
+#define __NR_utimes		313
+#define __NR_fallocate		314
+#define __NR_utimensat		315
+#define __NR_signalfd		316
+#define __NR_timerfd		317
+#define __NR_eventfd		318
+#define __NR_timerfd_create	319
+#define __NR_timerfd_settime	320
+#define __NR_timerfd_gettime	321
+#define __NR_signalfd4		322
+#define __NR_eventfd2		323
+#define __NR_inotify_init1	324
+#define __NR_pipe2		325
+#define __NR_dup3		326
+#define __NR_epoll_create1	327
+#define	__NR_preadv		328
+#define	__NR_pwritev		329
+#define __NR_rt_tgsigqueueinfo	330
+#define __NR_perf_event_open	331
+#define __NR_fanotify_init	332
+#define __NR_fanotify_mark	333
+#define __NR_prlimit64		334
+#define __NR_name_to_handle_at	335
+#define __NR_open_by_handle_at	336
+#define __NR_clock_adjtime	337
+#define __NR_syncfs		338
+#define __NR_setns		339
+#define __NR_process_vm_readv	340
+#define __NR_process_vm_writev	341
+#define __NR_s390_runtime_instr 342
+#define __NR_kcmp		343
+#define __NR_finit_module	344
+#define __NR_sched_setattr	345
+#define __NR_sched_getattr	346
+#define __NR_renameat2		347
+#define __NR_seccomp		348
+#define __NR_getrandom		349
+#define __NR_memfd_create	350
+#define __NR_bpf		351
+#define __NR_s390_pci_mmio_write	352
+#define __NR_s390_pci_mmio_read		353
+#define __NR_execveat		354
+#define __NR_userfaultfd	355
+#define __NR_membarrier		356
+#define __NR_recvmmsg		357
+#define __NR_sendmmsg		358
+#define __NR_socket		359
+#define __NR_socketpair		360
+#define __NR_bind		361
+#define __NR_connect		362
+#define __NR_listen		363
+#define __NR_accept4		364
+#define __NR_getsockopt		365
+#define __NR_setsockopt		366
+#define __NR_getsockname	367
+#define __NR_getpeername	368
+#define __NR_sendto		369
+#define __NR_sendmsg		370
+#define __NR_recvfrom		371
+#define __NR_recvmsg		372
+#define __NR_shutdown		373
+#define __NR_mlock2		374
+#define __NR_copy_file_range	375
+#define __NR_preadv2		376
+#define __NR_pwritev2		377
+#define NR_syscalls 378
+
+/* 
+ * There are some system calls that are not present on 64 bit, some
+ * have a different name although they do the same (e.g. __NR_chown32
+ * is __NR_chown on 64 bit).
+ */
+#ifndef __s390x__
+
+#define __NR_time		 13
+#define __NR_lchown		 16
+#define __NR_setuid		 23
+#define __NR_getuid		 24
+#define __NR_stime		 25
+#define __NR_setgid		 46
+#define __NR_getgid		 47
+#define __NR_geteuid		 49
+#define __NR_getegid		 50
+#define __NR_setreuid		 70
+#define __NR_setregid		 71
+#define __NR_getrlimit		 76
+#define __NR_getgroups		 80
+#define __NR_setgroups		 81
+#define __NR_fchown		 95
+#define __NR_ioperm		101
+#define __NR_setfsuid		138
+#define __NR_setfsgid		139
+#define __NR__llseek		140
+#define __NR__newselect 	142
+#define __NR_setresuid		164
+#define __NR_getresuid		165
+#define __NR_setresgid		170
+#define __NR_getresgid		171
+#define __NR_chown		182
+#define __NR_ugetrlimit		191	/* SuS compliant getrlimit */
+#define __NR_mmap2		192
+#define __NR_truncate64		193
+#define __NR_ftruncate64	194
+#define __NR_stat64		195
+#define __NR_lstat64		196
+#define __NR_fstat64		197
+#define __NR_lchown32		198
+#define __NR_getuid32		199
+#define __NR_getgid32		200
+#define __NR_geteuid32		201
+#define __NR_getegid32		202
+#define __NR_setreuid32		203
+#define __NR_setregid32		204
+#define __NR_getgroups32	205
+#define __NR_setgroups32	206
+#define __NR_fchown32		207
+#define __NR_setresuid32	208
+#define __NR_getresuid32	209
+#define __NR_setresgid32	210
+#define __NR_getresgid32	211
+#define __NR_chown32		212
+#define __NR_setuid32		213
+#define __NR_setgid32		214
+#define __NR_setfsuid32		215
+#define __NR_setfsgid32		216
+#define __NR_fcntl64		221
+#define __NR_sendfile64		223
+#define __NR_fadvise64_64	264
+#define __NR_fstatat64		293
+
+#else
+
+#define __NR_select		142
+#define __NR_getrlimit		191	/* SuS compliant getrlimit */
+#define __NR_lchown  		198
+#define __NR_getuid  		199
+#define __NR_getgid  		200
+#define __NR_geteuid  		201
+#define __NR_getegid  		202
+#define __NR_setreuid  		203
+#define __NR_setregid  		204
+#define __NR_getgroups  	205
+#define __NR_setgroups  	206
+#define __NR_fchown  		207
+#define __NR_setresuid  	208
+#define __NR_getresuid  	209
+#define __NR_setresgid  	210
+#define __NR_getresgid  	211
+#define __NR_chown  		212
+#define __NR_setuid  		213
+#define __NR_setgid  		214
+#define __NR_setfsuid  		215
+#define __NR_setfsgid  		216
+#define __NR_newfstatat		293
+
+#endif
+
+#endif /* _ASM_S390_UNISTD_H_ */
diff --git a/linux-headers/asm-x86/hyperv.h b/linux-headers/asm-x86/hyperv.h
index 462efe7..01af4d8 100644
--- a/linux-headers/asm-x86/hyperv.h
+++ b/linux-headers/asm-x86/hyperv.h
@@ -1,226 +1 @@
-#ifndef _ASM_X86_HYPERV_H
-#define _ASM_X86_HYPERV_H
-
-#include <linux/types.h>
-
-/*
- * The below CPUID leaves are present if VersionAndFeatures.HypervisorPresent
- * is set by CPUID(HvCpuIdFunctionVersionAndFeatures).
- */
-#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS	0x40000000
-#define HYPERV_CPUID_INTERFACE			0x40000001
-#define HYPERV_CPUID_VERSION			0x40000002
-#define HYPERV_CPUID_FEATURES			0x40000003
-#define HYPERV_CPUID_ENLIGHTMENT_INFO		0x40000004
-#define HYPERV_CPUID_IMPLEMENT_LIMITS		0x40000005
-
-#define HYPERV_HYPERVISOR_PRESENT_BIT		0x80000000
-#define HYPERV_CPUID_MIN			0x40000005
-#define HYPERV_CPUID_MAX			0x4000ffff
-
-/*
- * Feature identification. EAX indicates which features are available
- * to the partition based upon the current partition privileges.
- */
-
-/* VP Runtime (HV_X64_MSR_VP_RUNTIME) available */
-#define HV_X64_MSR_VP_RUNTIME_AVAILABLE		(1 << 0)
-/* Partition Reference Counter (HV_X64_MSR_TIME_REF_COUNT) available*/
-#define HV_X64_MSR_TIME_REF_COUNT_AVAILABLE	(1 << 1)
-
-/* A partition's reference time stamp counter (TSC) page */
-#define HV_X64_MSR_REFERENCE_TSC		0x40000021
-
-/*
- * There is a single feature flag that signifies the presence of the MSR
- * that can be used to retrieve both the local APIC Timer frequency as
- * well as the TSC frequency.
- */
-
-/* Local APIC timer frequency MSR (HV_X64_MSR_APIC_FREQUENCY) is available */
-#define HV_X64_MSR_APIC_FREQUENCY_AVAILABLE (1 << 11)
-
-/* TSC frequency MSR (HV_X64_MSR_TSC_FREQUENCY) is available */
-#define HV_X64_MSR_TSC_FREQUENCY_AVAILABLE (1 << 11)
-
-/*
- * Basic SynIC MSRs (HV_X64_MSR_SCONTROL through HV_X64_MSR_EOM
- * and HV_X64_MSR_SINT0 through HV_X64_MSR_SINT15) available
- */
-#define HV_X64_MSR_SYNIC_AVAILABLE		(1 << 2)
-/*
- * Synthetic Timer MSRs (HV_X64_MSR_STIMER0_CONFIG through
- * HV_X64_MSR_STIMER3_COUNT) available
- */
-#define HV_X64_MSR_SYNTIMER_AVAILABLE		(1 << 3)
-/*
- * APIC access MSRs (HV_X64_MSR_EOI, HV_X64_MSR_ICR and HV_X64_MSR_TPR)
- * are available
- */
-#define HV_X64_MSR_APIC_ACCESS_AVAILABLE	(1 << 4)
-/* Hypercall MSRs (HV_X64_MSR_GUEST_OS_ID and HV_X64_MSR_HYPERCALL) available*/
-#define HV_X64_MSR_HYPERCALL_AVAILABLE		(1 << 5)
-/* Access virtual processor index MSR (HV_X64_MSR_VP_INDEX) available*/
-#define HV_X64_MSR_VP_INDEX_AVAILABLE		(1 << 6)
-/* Virtual system reset MSR (HV_X64_MSR_RESET) is available*/
-#define HV_X64_MSR_RESET_AVAILABLE		(1 << 7)
- /*
-  * Access statistics pages MSRs (HV_X64_MSR_STATS_PARTITION_RETAIL_PAGE,
-  * HV_X64_MSR_STATS_PARTITION_INTERNAL_PAGE, HV_X64_MSR_STATS_VP_RETAIL_PAGE,
-  * HV_X64_MSR_STATS_VP_INTERNAL_PAGE) available
-  */
-#define HV_X64_MSR_STAT_PAGES_AVAILABLE		(1 << 8)
-
-/*
- * Feature identification: EBX indicates which flags were specified at
- * partition creation. The format is the same as the partition creation
- * flag structure defined in section Partition Creation Flags.
- */
-#define HV_X64_CREATE_PARTITIONS		(1 << 0)
-#define HV_X64_ACCESS_PARTITION_ID		(1 << 1)
-#define HV_X64_ACCESS_MEMORY_POOL		(1 << 2)
-#define HV_X64_ADJUST_MESSAGE_BUFFERS		(1 << 3)
-#define HV_X64_POST_MESSAGES			(1 << 4)
-#define HV_X64_SIGNAL_EVENTS			(1 << 5)
-#define HV_X64_CREATE_PORT			(1 << 6)
-#define HV_X64_CONNECT_PORT			(1 << 7)
-#define HV_X64_ACCESS_STATS			(1 << 8)
-#define HV_X64_DEBUGGING			(1 << 11)
-#define HV_X64_CPU_POWER_MANAGEMENT		(1 << 12)
-#define HV_X64_CONFIGURE_PROFILER		(1 << 13)
-
-/*
- * Feature identification. EDX indicates which miscellaneous features
- * are available to the partition.
- */
-/* The MWAIT instruction is available (per section MONITOR / MWAIT) */
-#define HV_X64_MWAIT_AVAILABLE				(1 << 0)
-/* Guest debugging support is available */
-#define HV_X64_GUEST_DEBUGGING_AVAILABLE		(1 << 1)
-/* Performance Monitor support is available*/
-#define HV_X64_PERF_MONITOR_AVAILABLE			(1 << 2)
-/* Support for physical CPU dynamic partitioning events is available*/
-#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE	(1 << 3)
-/*
- * Support for passing hypercall input parameter block via XMM
- * registers is available
- */
-#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE		(1 << 4)
-/* Support for a virtual guest idle state is available */
-#define HV_X64_GUEST_IDLE_STATE_AVAILABLE		(1 << 5)
-
-/*
- * Implementation recommendations. Indicates which behaviors the hypervisor
- * recommends the OS implement for optimal performance.
- */
- /*
-  * Recommend using hypercall for address space switches rather
-  * than MOV to CR3 instruction
-  */
-#define HV_X64_MWAIT_RECOMMENDED		(1 << 0)
-/* Recommend using hypercall for local TLB flushes rather
- * than INVLPG or MOV to CR3 instructions */
-#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED	(1 << 1)
-/*
- * Recommend using hypercall for remote TLB flushes rather
- * than inter-processor interrupts
- */
-#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED	(1 << 2)
-/*
- * Recommend using MSRs for accessing APIC registers
- * EOI, ICR and TPR rather than their memory-mapped counterparts
- */
-#define HV_X64_APIC_ACCESS_RECOMMENDED		(1 << 3)
-/* Recommend using the hypervisor-provided MSR to initiate a system RESET */
-#define HV_X64_SYSTEM_RESET_RECOMMENDED		(1 << 4)
-/*
- * Recommend using relaxed timing for this partition. If used,
- * the VM should disable any watchdog timeouts that rely on the
- * timely delivery of external interrupts
- */
-#define HV_X64_RELAXED_TIMING_RECOMMENDED	(1 << 5)
-
-/* MSR used to identify the guest OS. */
-#define HV_X64_MSR_GUEST_OS_ID			0x40000000
-
-/* MSR used to setup pages used to communicate with the hypervisor. */
-#define HV_X64_MSR_HYPERCALL			0x40000001
-
-/* MSR used to provide vcpu index */
-#define HV_X64_MSR_VP_INDEX			0x40000002
-
-/* MSR used to read the per-partition time reference counter */
-#define HV_X64_MSR_TIME_REF_COUNT		0x40000020
-
-/* MSR used to retrieve the TSC frequency */
-#define HV_X64_MSR_TSC_FREQUENCY		0x40000022
-
-/* MSR used to retrieve the local APIC timer frequency */
-#define HV_X64_MSR_APIC_FREQUENCY		0x40000023
-
-/* Define the virtual APIC registers */
-#define HV_X64_MSR_EOI				0x40000070
-#define HV_X64_MSR_ICR				0x40000071
-#define HV_X64_MSR_TPR				0x40000072
-#define HV_X64_MSR_APIC_ASSIST_PAGE		0x40000073
-
-/* Define synthetic interrupt controller model specific registers. */
-#define HV_X64_MSR_SCONTROL			0x40000080
-#define HV_X64_MSR_SVERSION			0x40000081
-#define HV_X64_MSR_SIEFP			0x40000082
-#define HV_X64_MSR_SIMP				0x40000083
-#define HV_X64_MSR_EOM				0x40000084
-#define HV_X64_MSR_SINT0			0x40000090
-#define HV_X64_MSR_SINT1			0x40000091
-#define HV_X64_MSR_SINT2			0x40000092
-#define HV_X64_MSR_SINT3			0x40000093
-#define HV_X64_MSR_SINT4			0x40000094
-#define HV_X64_MSR_SINT5			0x40000095
-#define HV_X64_MSR_SINT6			0x40000096
-#define HV_X64_MSR_SINT7			0x40000097
-#define HV_X64_MSR_SINT8			0x40000098
-#define HV_X64_MSR_SINT9			0x40000099
-#define HV_X64_MSR_SINT10			0x4000009A
-#define HV_X64_MSR_SINT11			0x4000009B
-#define HV_X64_MSR_SINT12			0x4000009C
-#define HV_X64_MSR_SINT13			0x4000009D
-#define HV_X64_MSR_SINT14			0x4000009E
-#define HV_X64_MSR_SINT15			0x4000009F
-
-
-#define HV_X64_MSR_HYPERCALL_ENABLE		0x00000001
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_HYPERCALL_PAGE_ADDRESS_SHIFT) - 1))
-
-/* Declare the various hypercall operations. */
-#define HV_X64_HV_NOTIFY_LONG_SPIN_WAIT		0x0008
-
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ENABLE		0x00000001
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT	12
-#define HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_MASK	\
-		(~((1ull << HV_X64_MSR_APIC_ASSIST_PAGE_ADDRESS_SHIFT) - 1))
-
-#define HV_X64_MSR_TSC_REFERENCE_ENABLE		0x00000001
-#define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT	12
-
-#define HV_PROCESSOR_POWER_STATE_C0		0
-#define HV_PROCESSOR_POWER_STATE_C1		1
-#define HV_PROCESSOR_POWER_STATE_C2		2
-#define HV_PROCESSOR_POWER_STATE_C3		3
-
-/* hypercall status code */
-#define HV_STATUS_SUCCESS			0
-#define HV_STATUS_INVALID_HYPERCALL_CODE	2
-#define HV_STATUS_INVALID_HYPERCALL_INPUT	3
-#define HV_STATUS_INVALID_ALIGNMENT		4
-#define HV_STATUS_INSUFFICIENT_BUFFERS		19
-
-typedef struct _HV_REFERENCE_TSC_PAGE {
-	__u32 tsc_sequence;
-	__u32 res1;
-	__u64 tsc_scale;
-	__s64 tsc_offset;
-} HV_REFERENCE_TSC_PAGE, *PHV_REFERENCE_TSC_PAGE;
-
-#endif
+#include "standard-headers/asm-x86/hyperv.h"
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index d7dcef5..739c0c5 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -106,6 +106,8 @@
 #define KVM_IRQCHIP_IOAPIC       2
 #define KVM_NR_IRQCHIPS          3
 
+#define KVM_RUN_X86_SMM		 (1 << 0)
+
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 	/* out (KVM_GET_REGS) / in (KVM_SET_REGS) */
@@ -214,9 +216,9 @@
 	__u32 padding[3];
 };
 
-#define KVM_CPUID_FLAG_SIGNIFCANT_INDEX		BIT(0)
-#define KVM_CPUID_FLAG_STATEFUL_FUNC		BIT(1)
-#define KVM_CPUID_FLAG_STATE_READ_NEXT		BIT(2)
+#define KVM_CPUID_FLAG_SIGNIFCANT_INDEX		(1 << 0)
+#define KVM_CPUID_FLAG_STATEFUL_FUNC		(1 << 1)
+#define KVM_CPUID_FLAG_STATE_READ_NEXT		(1 << 2)
 
 /* for KVM_SET_CPUID2 */
 struct kvm_cpuid2 {
@@ -281,6 +283,7 @@
 #define KVM_VCPUEVENT_VALID_NMI_PENDING	0x00000001
 #define KVM_VCPUEVENT_VALID_SIPI_VECTOR	0x00000002
 #define KVM_VCPUEVENT_VALID_SHADOW	0x00000004
+#define KVM_VCPUEVENT_VALID_SMM		0x00000008
 
 /* Interrupt shadow states */
 #define KVM_X86_SHADOW_INT_MOV_SS	0x01
@@ -309,7 +312,13 @@
 	} nmi;
 	__u32 sipi_vector;
 	__u32 flags;
-	__u32 reserved[10];
+	struct {
+		__u8 smm;
+		__u8 pending;
+		__u8 smm_inside_nmi;
+		__u8 latched_init;
+	} smi;
+	__u32 reserved[9];
 };
 
 /* for KVM_GET/SET_DEBUGREGS */
@@ -345,4 +354,7 @@
 struct kvm_sync_regs {
 };
 
+#define KVM_X86_QUIRK_LINT0_REENABLED	(1 << 0)
+#define KVM_X86_QUIRK_CD_NW_CLEARED	(1 << 1)
+
 #endif /* _ASM_X86_KVM_H */
diff --git a/linux-headers/asm-x86/unistd.h b/linux-headers/asm-x86/unistd.h
new file mode 100644
index 0000000..1f99b12
--- /dev/null
+++ b/linux-headers/asm-x86/unistd.h
@@ -0,0 +1,15 @@
+#ifndef _ASM_X86_UNISTD_H
+#define _ASM_X86_UNISTD_H
+
+/* x32 syscall flag bit */
+#define __X32_SYSCALL_BIT	0x40000000
+
+# ifdef __i386__
+#  include <asm/unistd_32.h>
+# elif defined(__ILP32__)
+#  include <asm/unistd_x32.h>
+# else
+#  include <asm/unistd_64.h>
+# endif
+
+#endif /* _ASM_X86_UNISTD_H */
diff --git a/linux-headers/asm-x86/unistd_32.h b/linux-headers/asm-x86/unistd_32.h
new file mode 100644
index 0000000..abeaf40
--- /dev/null
+++ b/linux-headers/asm-x86/unistd_32.h
@@ -0,0 +1,381 @@
+#ifndef _ASM_X86_UNISTD_32_H
+#define _ASM_X86_UNISTD_32_H 1
+
+#define __NR_restart_syscall 0
+#define __NR_exit 1
+#define __NR_fork 2
+#define __NR_read 3
+#define __NR_write 4
+#define __NR_open 5
+#define __NR_close 6
+#define __NR_waitpid 7
+#define __NR_creat 8
+#define __NR_link 9
+#define __NR_unlink 10
+#define __NR_execve 11
+#define __NR_chdir 12
+#define __NR_time 13
+#define __NR_mknod 14
+#define __NR_chmod 15
+#define __NR_lchown 16
+#define __NR_break 17
+#define __NR_oldstat 18
+#define __NR_lseek 19
+#define __NR_getpid 20
+#define __NR_mount 21
+#define __NR_umount 22
+#define __NR_setuid 23
+#define __NR_getuid 24
+#define __NR_stime 25
+#define __NR_ptrace 26
+#define __NR_alarm 27
+#define __NR_oldfstat 28
+#define __NR_pause 29
+#define __NR_utime 30
+#define __NR_stty 31
+#define __NR_gtty 32
+#define __NR_access 33
+#define __NR_nice 34
+#define __NR_ftime 35
+#define __NR_sync 36
+#define __NR_kill 37
+#define __NR_rename 38
+#define __NR_mkdir 39
+#define __NR_rmdir 40
+#define __NR_dup 41
+#define __NR_pipe 42
+#define __NR_times 43
+#define __NR_prof 44
+#define __NR_brk 45
+#define __NR_setgid 46
+#define __NR_getgid 47
+#define __NR_signal 48
+#define __NR_geteuid 49
+#define __NR_getegid 50
+#define __NR_acct 51
+#define __NR_umount2 52
+#define __NR_lock 53
+#define __NR_ioctl 54
+#define __NR_fcntl 55
+#define __NR_mpx 56
+#define __NR_setpgid 57
+#define __NR_ulimit 58
+#define __NR_oldolduname 59
+#define __NR_umask 60
+#define __NR_chroot 61
+#define __NR_ustat 62
+#define __NR_dup2 63
+#define __NR_getppid 64
+#define __NR_getpgrp 65
+#define __NR_setsid 66
+#define __NR_sigaction 67
+#define __NR_sgetmask 68
+#define __NR_ssetmask 69
+#define __NR_setreuid 70
+#define __NR_setregid 71
+#define __NR_sigsuspend 72
+#define __NR_sigpending 73
+#define __NR_sethostname 74
+#define __NR_setrlimit 75
+#define __NR_getrlimit 76
+#define __NR_getrusage 77
+#define __NR_gettimeofday 78
+#define __NR_settimeofday 79
+#define __NR_getgroups 80
+#define __NR_setgroups 81
+#define __NR_select 82
+#define __NR_symlink 83
+#define __NR_oldlstat 84
+#define __NR_readlink 85
+#define __NR_uselib 86
+#define __NR_swapon 87
+#define __NR_reboot 88
+#define __NR_readdir 89
+#define __NR_mmap 90
+#define __NR_munmap 91
+#define __NR_truncate 92
+#define __NR_ftruncate 93
+#define __NR_fchmod 94
+#define __NR_fchown 95
+#define __NR_getpriority 96
+#define __NR_setpriority 97
+#define __NR_profil 98
+#define __NR_statfs 99
+#define __NR_fstatfs 100
+#define __NR_ioperm 101
+#define __NR_socketcall 102
+#define __NR_syslog 103
+#define __NR_setitimer 104
+#define __NR_getitimer 105
+#define __NR_stat 106
+#define __NR_lstat 107
+#define __NR_fstat 108
+#define __NR_olduname 109
+#define __NR_iopl 110
+#define __NR_vhangup 111
+#define __NR_idle 112
+#define __NR_vm86old 113
+#define __NR_wait4 114
+#define __NR_swapoff 115
+#define __NR_sysinfo 116
+#define __NR_ipc 117
+#define __NR_fsync 118
+#define __NR_sigreturn 119
+#define __NR_clone 120
+#define __NR_setdomainname 121
+#define __NR_uname 122
+#define __NR_modify_ldt 123
+#define __NR_adjtimex 124
+#define __NR_mprotect 125
+#define __NR_sigprocmask 126
+#define __NR_create_module 127
+#define __NR_init_module 128
+#define __NR_delete_module 129
+#define __NR_get_kernel_syms 130
+#define __NR_quotactl 131
+#define __NR_getpgid 132
+#define __NR_fchdir 133
+#define __NR_bdflush 134
+#define __NR_sysfs 135
+#define __NR_personality 136
+#define __NR_afs_syscall 137
+#define __NR_setfsuid 138
+#define __NR_setfsgid 139
+#define __NR__llseek 140
+#define __NR_getdents 141
+#define __NR__newselect 142
+#define __NR_flock 143
+#define __NR_msync 144
+#define __NR_readv 145
+#define __NR_writev 146
+#define __NR_getsid 147
+#define __NR_fdatasync 148
+#define __NR__sysctl 149
+#define __NR_mlock 150
+#define __NR_munlock 151
+#define __NR_mlockall 152
+#define __NR_munlockall 153
+#define __NR_sched_setparam 154
+#define __NR_sched_getparam 155
+#define __NR_sched_setscheduler 156
+#define __NR_sched_getscheduler 157
+#define __NR_sched_yield 158
+#define __NR_sched_get_priority_max 159
+#define __NR_sched_get_priority_min 160
+#define __NR_sched_rr_get_interval 161
+#define __NR_nanosleep 162
+#define __NR_mremap 163
+#define __NR_setresuid 164
+#define __NR_getresuid 165
+#define __NR_vm86 166
+#define __NR_query_module 167
+#define __NR_poll 168
+#define __NR_nfsservctl 169
+#define __NR_setresgid 170
+#define __NR_getresgid 171
+#define __NR_prctl 172
+#define __NR_rt_sigreturn 173
+#define __NR_rt_sigaction 174
+#define __NR_rt_sigprocmask 175
+#define __NR_rt_sigpending 176
+#define __NR_rt_sigtimedwait 177
+#define __NR_rt_sigqueueinfo 178
+#define __NR_rt_sigsuspend 179
+#define __NR_pread64 180
+#define __NR_pwrite64 181
+#define __NR_chown 182
+#define __NR_getcwd 183
+#define __NR_capget 184
+#define __NR_capset 185
+#define __NR_sigaltstack 186
+#define __NR_sendfile 187
+#define __NR_getpmsg 188
+#define __NR_putpmsg 189
+#define __NR_vfork 190
+#define __NR_ugetrlimit 191
+#define __NR_mmap2 192
+#define __NR_truncate64 193
+#define __NR_ftruncate64 194
+#define __NR_stat64 195
+#define __NR_lstat64 196
+#define __NR_fstat64 197
+#define __NR_lchown32 198
+#define __NR_getuid32 199
+#define __NR_getgid32 200
+#define __NR_geteuid32 201
+#define __NR_getegid32 202
+#define __NR_setreuid32 203
+#define __NR_setregid32 204
+#define __NR_getgroups32 205
+#define __NR_setgroups32 206
+#define __NR_fchown32 207
+#define __NR_setresuid32 208
+#define __NR_getresuid32 209
+#define __NR_setresgid32 210
+#define __NR_getresgid32 211
+#define __NR_chown32 212
+#define __NR_setuid32 213
+#define __NR_setgid32 214
+#define __NR_setfsuid32 215
+#define __NR_setfsgid32 216
+#define __NR_pivot_root 217
+#define __NR_mincore 218
+#define __NR_madvise 219
+#define __NR_getdents64 220
+#define __NR_fcntl64 221
+#define __NR_gettid 224
+#define __NR_readahead 225
+#define __NR_setxattr 226
+#define __NR_lsetxattr 227
+#define __NR_fsetxattr 228
+#define __NR_getxattr 229
+#define __NR_lgetxattr 230
+#define __NR_fgetxattr 231
+#define __NR_listxattr 232
+#define __NR_llistxattr 233
+#define __NR_flistxattr 234
+#define __NR_removexattr 235
+#define __NR_lremovexattr 236
+#define __NR_fremovexattr 237
+#define __NR_tkill 238
+#define __NR_sendfile64 239
+#define __NR_futex 240
+#define __NR_sched_setaffinity 241
+#define __NR_sched_getaffinity 242
+#define __NR_set_thread_area 243
+#define __NR_get_thread_area 244
+#define __NR_io_setup 245
+#define __NR_io_destroy 246
+#define __NR_io_getevents 247
+#define __NR_io_submit 248
+#define __NR_io_cancel 249
+#define __NR_fadvise64 250
+#define __NR_exit_group 252
+#define __NR_lookup_dcookie 253
+#define __NR_epoll_create 254
+#define __NR_epoll_ctl 255
+#define __NR_epoll_wait 256
+#define __NR_remap_file_pages 257
+#define __NR_set_tid_address 258
+#define __NR_timer_create 259
+#define __NR_timer_settime 260
+#define __NR_timer_gettime 261
+#define __NR_timer_getoverrun 262
+#define __NR_timer_delete 263
+#define __NR_clock_settime 264
+#define __NR_clock_gettime 265
+#define __NR_clock_getres 266
+#define __NR_clock_nanosleep 267
+#define __NR_statfs64 268
+#define __NR_fstatfs64 269
+#define __NR_tgkill 270
+#define __NR_utimes 271
+#define __NR_fadvise64_64 272
+#define __NR_vserver 273
+#define __NR_mbind 274
+#define __NR_get_mempolicy 275
+#define __NR_set_mempolicy 276
+#define __NR_mq_open 277
+#define __NR_mq_unlink 278
+#define __NR_mq_timedsend 279
+#define __NR_mq_timedreceive 280
+#define __NR_mq_notify 281
+#define __NR_mq_getsetattr 282
+#define __NR_kexec_load 283
+#define __NR_waitid 284
+#define __NR_add_key 286
+#define __NR_request_key 287
+#define __NR_keyctl 288
+#define __NR_ioprio_set 289
+#define __NR_ioprio_get 290
+#define __NR_inotify_init 291
+#define __NR_inotify_add_watch 292
+#define __NR_inotify_rm_watch 293
+#define __NR_migrate_pages 294
+#define __NR_openat 295
+#define __NR_mkdirat 296
+#define __NR_mknodat 297
+#define __NR_fchownat 298
+#define __NR_futimesat 299
+#define __NR_fstatat64 300
+#define __NR_unlinkat 301
+#define __NR_renameat 302
+#define __NR_linkat 303
+#define __NR_symlinkat 304
+#define __NR_readlinkat 305
+#define __NR_fchmodat 306
+#define __NR_faccessat 307
+#define __NR_pselect6 308
+#define __NR_ppoll 309
+#define __NR_unshare 310
+#define __NR_set_robust_list 311
+#define __NR_get_robust_list 312
+#define __NR_splice 313
+#define __NR_sync_file_range 314
+#define __NR_tee 315
+#define __NR_vmsplice 316
+#define __NR_move_pages 317
+#define __NR_getcpu 318
+#define __NR_epoll_pwait 319
+#define __NR_utimensat 320
+#define __NR_signalfd 321
+#define __NR_timerfd_create 322
+#define __NR_eventfd 323
+#define __NR_fallocate 324
+#define __NR_timerfd_settime 325
+#define __NR_timerfd_gettime 326
+#define __NR_signalfd4 327
+#define __NR_eventfd2 328
+#define __NR_epoll_create1 329
+#define __NR_dup3 330
+#define __NR_pipe2 331
+#define __NR_inotify_init1 332
+#define __NR_preadv 333
+#define __NR_pwritev 334
+#define __NR_rt_tgsigqueueinfo 335
+#define __NR_perf_event_open 336
+#define __NR_recvmmsg 337
+#define __NR_fanotify_init 338
+#define __NR_fanotify_mark 339
+#define __NR_prlimit64 340
+#define __NR_name_to_handle_at 341
+#define __NR_open_by_handle_at 342
+#define __NR_clock_adjtime 343
+#define __NR_syncfs 344
+#define __NR_sendmmsg 345
+#define __NR_setns 346
+#define __NR_process_vm_readv 347
+#define __NR_process_vm_writev 348
+#define __NR_kcmp 349
+#define __NR_finit_module 350
+#define __NR_sched_setattr 351
+#define __NR_sched_getattr 352
+#define __NR_renameat2 353
+#define __NR_seccomp 354
+#define __NR_getrandom 355
+#define __NR_memfd_create 356
+#define __NR_bpf 357
+#define __NR_execveat 358
+#define __NR_socket 359
+#define __NR_socketpair 360
+#define __NR_bind 361
+#define __NR_connect 362
+#define __NR_listen 363
+#define __NR_accept4 364
+#define __NR_getsockopt 365
+#define __NR_setsockopt 366
+#define __NR_getsockname 367
+#define __NR_getpeername 368
+#define __NR_sendto 369
+#define __NR_sendmsg 370
+#define __NR_recvfrom 371
+#define __NR_recvmsg 372
+#define __NR_shutdown 373
+#define __NR_userfaultfd 374
+#define __NR_membarrier 375
+#define __NR_mlock2 376
+#define __NR_copy_file_range 377
+#define __NR_preadv2 378
+#define __NR_pwritev2 379
+
+#endif /* _ASM_X86_UNISTD_32_H */
diff --git a/linux-headers/asm-x86/unistd_64.h b/linux-headers/asm-x86/unistd_64.h
new file mode 100644
index 0000000..73c3d1f
--- /dev/null
+++ b/linux-headers/asm-x86/unistd_64.h
@@ -0,0 +1,334 @@
+#ifndef _ASM_X86_UNISTD_64_H
+#define _ASM_X86_UNISTD_64_H 1
+
+#define __NR_read 0
+#define __NR_write 1
+#define __NR_open 2
+#define __NR_close 3
+#define __NR_stat 4
+#define __NR_fstat 5
+#define __NR_lstat 6
+#define __NR_poll 7
+#define __NR_lseek 8
+#define __NR_mmap 9
+#define __NR_mprotect 10
+#define __NR_munmap 11
+#define __NR_brk 12
+#define __NR_rt_sigaction 13
+#define __NR_rt_sigprocmask 14
+#define __NR_rt_sigreturn 15
+#define __NR_ioctl 16
+#define __NR_pread64 17
+#define __NR_pwrite64 18
+#define __NR_readv 19
+#define __NR_writev 20
+#define __NR_access 21
+#define __NR_pipe 22
+#define __NR_select 23
+#define __NR_sched_yield 24
+#define __NR_mremap 25
+#define __NR_msync 26
+#define __NR_mincore 27
+#define __NR_madvise 28
+#define __NR_shmget 29
+#define __NR_shmat 30
+#define __NR_shmctl 31
+#define __NR_dup 32
+#define __NR_dup2 33
+#define __NR_pause 34
+#define __NR_nanosleep 35
+#define __NR_getitimer 36
+#define __NR_alarm 37
+#define __NR_setitimer 38
+#define __NR_getpid 39
+#define __NR_sendfile 40
+#define __NR_socket 41
+#define __NR_connect 42
+#define __NR_accept 43
+#define __NR_sendto 44
+#define __NR_recvfrom 45
+#define __NR_sendmsg 46
+#define __NR_recvmsg 47
+#define __NR_shutdown 48
+#define __NR_bind 49
+#define __NR_listen 50
+#define __NR_getsockname 51
+#define __NR_getpeername 52
+#define __NR_socketpair 53
+#define __NR_setsockopt 54
+#define __NR_getsockopt 55
+#define __NR_clone 56
+#define __NR_fork 57
+#define __NR_vfork 58
+#define __NR_execve 59
+#define __NR_exit 60
+#define __NR_wait4 61
+#define __NR_kill 62
+#define __NR_uname 63
+#define __NR_semget 64
+#define __NR_semop 65
+#define __NR_semctl 66
+#define __NR_shmdt 67
+#define __NR_msgget 68
+#define __NR_msgsnd 69
+#define __NR_msgrcv 70
+#define __NR_msgctl 71
+#define __NR_fcntl 72
+#define __NR_flock 73
+#define __NR_fsync 74
+#define __NR_fdatasync 75
+#define __NR_truncate 76
+#define __NR_ftruncate 77
+#define __NR_getdents 78
+#define __NR_getcwd 79
+#define __NR_chdir 80
+#define __NR_fchdir 81
+#define __NR_rename 82
+#define __NR_mkdir 83
+#define __NR_rmdir 84
+#define __NR_creat 85
+#define __NR_link 86
+#define __NR_unlink 87
+#define __NR_symlink 88
+#define __NR_readlink 89
+#define __NR_chmod 90
+#define __NR_fchmod 91
+#define __NR_chown 92
+#define __NR_fchown 93
+#define __NR_lchown 94
+#define __NR_umask 95
+#define __NR_gettimeofday 96
+#define __NR_getrlimit 97
+#define __NR_getrusage 98
+#define __NR_sysinfo 99
+#define __NR_times 100
+#define __NR_ptrace 101
+#define __NR_getuid 102
+#define __NR_syslog 103
+#define __NR_getgid 104
+#define __NR_setuid 105
+#define __NR_setgid 106
+#define __NR_geteuid 107
+#define __NR_getegid 108
+#define __NR_setpgid 109
+#define __NR_getppid 110
+#define __NR_getpgrp 111
+#define __NR_setsid 112
+#define __NR_setreuid 113
+#define __NR_setregid 114
+#define __NR_getgroups 115
+#define __NR_setgroups 116
+#define __NR_setresuid 117
+#define __NR_getresuid 118
+#define __NR_setresgid 119
+#define __NR_getresgid 120
+#define __NR_getpgid 121
+#define __NR_setfsuid 122
+#define __NR_setfsgid 123
+#define __NR_getsid 124
+#define __NR_capget 125
+#define __NR_capset 126
+#define __NR_rt_sigpending 127
+#define __NR_rt_sigtimedwait 128
+#define __NR_rt_sigqueueinfo 129
+#define __NR_rt_sigsuspend 130
+#define __NR_sigaltstack 131
+#define __NR_utime 132
+#define __NR_mknod 133
+#define __NR_uselib 134
+#define __NR_personality 135
+#define __NR_ustat 136
+#define __NR_statfs 137
+#define __NR_fstatfs 138
+#define __NR_sysfs 139
+#define __NR_getpriority 140
+#define __NR_setpriority 141
+#define __NR_sched_setparam 142
+#define __NR_sched_getparam 143
+#define __NR_sched_setscheduler 144
+#define __NR_sched_getscheduler 145
+#define __NR_sched_get_priority_max 146
+#define __NR_sched_get_priority_min 147
+#define __NR_sched_rr_get_interval 148
+#define __NR_mlock 149
+#define __NR_munlock 150
+#define __NR_mlockall 151
+#define __NR_munlockall 152
+#define __NR_vhangup 153
+#define __NR_modify_ldt 154
+#define __NR_pivot_root 155
+#define __NR__sysctl 156
+#define __NR_prctl 157
+#define __NR_arch_prctl 158
+#define __NR_adjtimex 159
+#define __NR_setrlimit 160
+#define __NR_chroot 161
+#define __NR_sync 162
+#define __NR_acct 163
+#define __NR_settimeofday 164
+#define __NR_mount 165
+#define __NR_umount2 166
+#define __NR_swapon 167
+#define __NR_swapoff 168
+#define __NR_reboot 169
+#define __NR_sethostname 170
+#define __NR_setdomainname 171
+#define __NR_iopl 172
+#define __NR_ioperm 173
+#define __NR_create_module 174
+#define __NR_init_module 175
+#define __NR_delete_module 176
+#define __NR_get_kernel_syms 177
+#define __NR_query_module 178
+#define __NR_quotactl 179
+#define __NR_nfsservctl 180
+#define __NR_getpmsg 181
+#define __NR_putpmsg 182
+#define __NR_afs_syscall 183
+#define __NR_tuxcall 184
+#define __NR_security 185
+#define __NR_gettid 186
+#define __NR_readahead 187
+#define __NR_setxattr 188
+#define __NR_lsetxattr 189
+#define __NR_fsetxattr 190
+#define __NR_getxattr 191
+#define __NR_lgetxattr 192
+#define __NR_fgetxattr 193
+#define __NR_listxattr 194
+#define __NR_llistxattr 195
+#define __NR_flistxattr 196
+#define __NR_removexattr 197
+#define __NR_lremovexattr 198
+#define __NR_fremovexattr 199
+#define __NR_tkill 200
+#define __NR_time 201
+#define __NR_futex 202
+#define __NR_sched_setaffinity 203
+#define __NR_sched_getaffinity 204
+#define __NR_set_thread_area 205
+#define __NR_io_setup 206
+#define __NR_io_destroy 207
+#define __NR_io_getevents 208
+#define __NR_io_submit 209
+#define __NR_io_cancel 210
+#define __NR_get_thread_area 211
+#define __NR_lookup_dcookie 212
+#define __NR_epoll_create 213
+#define __NR_epoll_ctl_old 214
+#define __NR_epoll_wait_old 215
+#define __NR_remap_file_pages 216
+#define __NR_getdents64 217
+#define __NR_set_tid_address 218
+#define __NR_restart_syscall 219
+#define __NR_semtimedop 220
+#define __NR_fadvise64 221
+#define __NR_timer_create 222
+#define __NR_timer_settime 223
+#define __NR_timer_gettime 224
+#define __NR_timer_getoverrun 225
+#define __NR_timer_delete 226
+#define __NR_clock_settime 227
+#define __NR_clock_gettime 228
+#define __NR_clock_getres 229
+#define __NR_clock_nanosleep 230
+#define __NR_exit_group 231
+#define __NR_epoll_wait 232
+#define __NR_epoll_ctl 233
+#define __NR_tgkill 234
+#define __NR_utimes 235
+#define __NR_vserver 236
+#define __NR_mbind 237
+#define __NR_set_mempolicy 238
+#define __NR_get_mempolicy 239
+#define __NR_mq_open 240
+#define __NR_mq_unlink 241
+#define __NR_mq_timedsend 242
+#define __NR_mq_timedreceive 243
+#define __NR_mq_notify 244
+#define __NR_mq_getsetattr 245
+#define __NR_kexec_load 246
+#define __NR_waitid 247
+#define __NR_add_key 248
+#define __NR_request_key 249
+#define __NR_keyctl 250
+#define __NR_ioprio_set 251
+#define __NR_ioprio_get 252
+#define __NR_inotify_init 253
+#define __NR_inotify_add_watch 254
+#define __NR_inotify_rm_watch 255
+#define __NR_migrate_pages 256
+#define __NR_openat 257
+#define __NR_mkdirat 258
+#define __NR_mknodat 259
+#define __NR_fchownat 260
+#define __NR_futimesat 261
+#define __NR_newfstatat 262
+#define __NR_unlinkat 263
+#define __NR_renameat 264
+#define __NR_linkat 265
+#define __NR_symlinkat 266
+#define __NR_readlinkat 267
+#define __NR_fchmodat 268
+#define __NR_faccessat 269
+#define __NR_pselect6 270
+#define __NR_ppoll 271
+#define __NR_unshare 272
+#define __NR_set_robust_list 273
+#define __NR_get_robust_list 274
+#define __NR_splice 275
+#define __NR_tee 276
+#define __NR_sync_file_range 277
+#define __NR_vmsplice 278
+#define __NR_move_pages 279
+#define __NR_utimensat 280
+#define __NR_epoll_pwait 281
+#define __NR_signalfd 282
+#define __NR_timerfd_create 283
+#define __NR_eventfd 284
+#define __NR_fallocate 285
+#define __NR_timerfd_settime 286
+#define __NR_timerfd_gettime 287
+#define __NR_accept4 288
+#define __NR_signalfd4 289
+#define __NR_eventfd2 290
+#define __NR_epoll_create1 291
+#define __NR_dup3 292
+#define __NR_pipe2 293
+#define __NR_inotify_init1 294
+#define __NR_preadv 295
+#define __NR_pwritev 296
+#define __NR_rt_tgsigqueueinfo 297
+#define __NR_perf_event_open 298
+#define __NR_recvmmsg 299
+#define __NR_fanotify_init 300
+#define __NR_fanotify_mark 301
+#define __NR_prlimit64 302
+#define __NR_name_to_handle_at 303
+#define __NR_open_by_handle_at 304
+#define __NR_clock_adjtime 305
+#define __NR_syncfs 306
+#define __NR_sendmmsg 307
+#define __NR_setns 308
+#define __NR_getcpu 309
+#define __NR_process_vm_readv 310
+#define __NR_process_vm_writev 311
+#define __NR_kcmp 312
+#define __NR_finit_module 313
+#define __NR_sched_setattr 314
+#define __NR_sched_getattr 315
+#define __NR_renameat2 316
+#define __NR_seccomp 317
+#define __NR_getrandom 318
+#define __NR_memfd_create 319
+#define __NR_kexec_file_load 320
+#define __NR_bpf 321
+#define __NR_execveat 322
+#define __NR_userfaultfd 323
+#define __NR_membarrier 324
+#define __NR_mlock2 325
+#define __NR_copy_file_range 326
+#define __NR_preadv2 327
+#define __NR_pwritev2 328
+
+#endif /* _ASM_X86_UNISTD_64_H */
diff --git a/linux-headers/asm-x86/unistd_x32.h b/linux-headers/asm-x86/unistd_x32.h
new file mode 100644
index 0000000..0230779
--- /dev/null
+++ b/linux-headers/asm-x86/unistd_x32.h
@@ -0,0 +1,323 @@
+#ifndef _ASM_X86_UNISTD_X32_H
+#define _ASM_X86_UNISTD_X32_H 1
+
+#define __NR_read (__X32_SYSCALL_BIT + 0)
+#define __NR_write (__X32_SYSCALL_BIT + 1)
+#define __NR_open (__X32_SYSCALL_BIT + 2)
+#define __NR_close (__X32_SYSCALL_BIT + 3)
+#define __NR_stat (__X32_SYSCALL_BIT + 4)
+#define __NR_fstat (__X32_SYSCALL_BIT + 5)
+#define __NR_lstat (__X32_SYSCALL_BIT + 6)
+#define __NR_poll (__X32_SYSCALL_BIT + 7)
+#define __NR_lseek (__X32_SYSCALL_BIT + 8)
+#define __NR_mmap (__X32_SYSCALL_BIT + 9)
+#define __NR_mprotect (__X32_SYSCALL_BIT + 10)
+#define __NR_munmap (__X32_SYSCALL_BIT + 11)
+#define __NR_brk (__X32_SYSCALL_BIT + 12)
+#define __NR_rt_sigprocmask (__X32_SYSCALL_BIT + 14)
+#define __NR_pread64 (__X32_SYSCALL_BIT + 17)
+#define __NR_pwrite64 (__X32_SYSCALL_BIT + 18)
+#define __NR_access (__X32_SYSCALL_BIT + 21)
+#define __NR_pipe (__X32_SYSCALL_BIT + 22)
+#define __NR_select (__X32_SYSCALL_BIT + 23)
+#define __NR_sched_yield (__X32_SYSCALL_BIT + 24)
+#define __NR_mremap (__X32_SYSCALL_BIT + 25)
+#define __NR_msync (__X32_SYSCALL_BIT + 26)
+#define __NR_mincore (__X32_SYSCALL_BIT + 27)
+#define __NR_madvise (__X32_SYSCALL_BIT + 28)
+#define __NR_shmget (__X32_SYSCALL_BIT + 29)
+#define __NR_shmat (__X32_SYSCALL_BIT + 30)
+#define __NR_shmctl (__X32_SYSCALL_BIT + 31)
+#define __NR_dup (__X32_SYSCALL_BIT + 32)
+#define __NR_dup2 (__X32_SYSCALL_BIT + 33)
+#define __NR_pause (__X32_SYSCALL_BIT + 34)
+#define __NR_nanosleep (__X32_SYSCALL_BIT + 35)
+#define __NR_getitimer (__X32_SYSCALL_BIT + 36)
+#define __NR_alarm (__X32_SYSCALL_BIT + 37)
+#define __NR_setitimer (__X32_SYSCALL_BIT + 38)
+#define __NR_getpid (__X32_SYSCALL_BIT + 39)
+#define __NR_sendfile (__X32_SYSCALL_BIT + 40)
+#define __NR_socket (__X32_SYSCALL_BIT + 41)
+#define __NR_connect (__X32_SYSCALL_BIT + 42)
+#define __NR_accept (__X32_SYSCALL_BIT + 43)
+#define __NR_sendto (__X32_SYSCALL_BIT + 44)
+#define __NR_shutdown (__X32_SYSCALL_BIT + 48)
+#define __NR_bind (__X32_SYSCALL_BIT + 49)
+#define __NR_listen (__X32_SYSCALL_BIT + 50)
+#define __NR_getsockname (__X32_SYSCALL_BIT + 51)
+#define __NR_getpeername (__X32_SYSCALL_BIT + 52)
+#define __NR_socketpair (__X32_SYSCALL_BIT + 53)
+#define __NR_clone (__X32_SYSCALL_BIT + 56)
+#define __NR_fork (__X32_SYSCALL_BIT + 57)
+#define __NR_vfork (__X32_SYSCALL_BIT + 58)
+#define __NR_exit (__X32_SYSCALL_BIT + 60)
+#define __NR_wait4 (__X32_SYSCALL_BIT + 61)
+#define __NR_kill (__X32_SYSCALL_BIT + 62)
+#define __NR_uname (__X32_SYSCALL_BIT + 63)
+#define __NR_semget (__X32_SYSCALL_BIT + 64)
+#define __NR_semop (__X32_SYSCALL_BIT + 65)
+#define __NR_semctl (__X32_SYSCALL_BIT + 66)
+#define __NR_shmdt (__X32_SYSCALL_BIT + 67)
+#define __NR_msgget (__X32_SYSCALL_BIT + 68)
+#define __NR_msgsnd (__X32_SYSCALL_BIT + 69)
+#define __NR_msgrcv (__X32_SYSCALL_BIT + 70)
+#define __NR_msgctl (__X32_SYSCALL_BIT + 71)
+#define __NR_fcntl (__X32_SYSCALL_BIT + 72)
+#define __NR_flock (__X32_SYSCALL_BIT + 73)
+#define __NR_fsync (__X32_SYSCALL_BIT + 74)
+#define __NR_fdatasync (__X32_SYSCALL_BIT + 75)
+#define __NR_truncate (__X32_SYSCALL_BIT + 76)
+#define __NR_ftruncate (__X32_SYSCALL_BIT + 77)
+#define __NR_getdents (__X32_SYSCALL_BIT + 78)
+#define __NR_getcwd (__X32_SYSCALL_BIT + 79)
+#define __NR_chdir (__X32_SYSCALL_BIT + 80)
+#define __NR_fchdir (__X32_SYSCALL_BIT + 81)
+#define __NR_rename (__X32_SYSCALL_BIT + 82)
+#define __NR_mkdir (__X32_SYSCALL_BIT + 83)
+#define __NR_rmdir (__X32_SYSCALL_BIT + 84)
+#define __NR_creat (__X32_SYSCALL_BIT + 85)
+#define __NR_link (__X32_SYSCALL_BIT + 86)
+#define __NR_unlink (__X32_SYSCALL_BIT + 87)
+#define __NR_symlink (__X32_SYSCALL_BIT + 88)
+#define __NR_readlink (__X32_SYSCALL_BIT + 89)
+#define __NR_chmod (__X32_SYSCALL_BIT + 90)
+#define __NR_fchmod (__X32_SYSCALL_BIT + 91)
+#define __NR_chown (__X32_SYSCALL_BIT + 92)
+#define __NR_fchown (__X32_SYSCALL_BIT + 93)
+#define __NR_lchown (__X32_SYSCALL_BIT + 94)
+#define __NR_umask (__X32_SYSCALL_BIT + 95)
+#define __NR_gettimeofday (__X32_SYSCALL_BIT + 96)
+#define __NR_getrlimit (__X32_SYSCALL_BIT + 97)
+#define __NR_getrusage (__X32_SYSCALL_BIT + 98)
+#define __NR_sysinfo (__X32_SYSCALL_BIT + 99)
+#define __NR_times (__X32_SYSCALL_BIT + 100)
+#define __NR_getuid (__X32_SYSCALL_BIT + 102)
+#define __NR_syslog (__X32_SYSCALL_BIT + 103)
+#define __NR_getgid (__X32_SYSCALL_BIT + 104)
+#define __NR_setuid (__X32_SYSCALL_BIT + 105)
+#define __NR_setgid (__X32_SYSCALL_BIT + 106)
+#define __NR_geteuid (__X32_SYSCALL_BIT + 107)
+#define __NR_getegid (__X32_SYSCALL_BIT + 108)
+#define __NR_setpgid (__X32_SYSCALL_BIT + 109)
+#define __NR_getppid (__X32_SYSCALL_BIT + 110)
+#define __NR_getpgrp (__X32_SYSCALL_BIT + 111)
+#define __NR_setsid (__X32_SYSCALL_BIT + 112)
+#define __NR_setreuid (__X32_SYSCALL_BIT + 113)
+#define __NR_setregid (__X32_SYSCALL_BIT + 114)
+#define __NR_getgroups (__X32_SYSCALL_BIT + 115)
+#define __NR_setgroups (__X32_SYSCALL_BIT + 116)
+#define __NR_setresuid (__X32_SYSCALL_BIT + 117)
+#define __NR_getresuid (__X32_SYSCALL_BIT + 118)
+#define __NR_setresgid (__X32_SYSCALL_BIT + 119)
+#define __NR_getresgid (__X32_SYSCALL_BIT + 120)
+#define __NR_getpgid (__X32_SYSCALL_BIT + 121)
+#define __NR_setfsuid (__X32_SYSCALL_BIT + 122)
+#define __NR_setfsgid (__X32_SYSCALL_BIT + 123)
+#define __NR_getsid (__X32_SYSCALL_BIT + 124)
+#define __NR_capget (__X32_SYSCALL_BIT + 125)
+#define __NR_capset (__X32_SYSCALL_BIT + 126)
+#define __NR_rt_sigsuspend (__X32_SYSCALL_BIT + 130)
+#define __NR_utime (__X32_SYSCALL_BIT + 132)
+#define __NR_mknod (__X32_SYSCALL_BIT + 133)
+#define __NR_personality (__X32_SYSCALL_BIT + 135)
+#define __NR_ustat (__X32_SYSCALL_BIT + 136)
+#define __NR_statfs (__X32_SYSCALL_BIT + 137)
+#define __NR_fstatfs (__X32_SYSCALL_BIT + 138)
+#define __NR_sysfs (__X32_SYSCALL_BIT + 139)
+#define __NR_getpriority (__X32_SYSCALL_BIT + 140)
+#define __NR_setpriority (__X32_SYSCALL_BIT + 141)
+#define __NR_sched_setparam (__X32_SYSCALL_BIT + 142)
+#define __NR_sched_getparam (__X32_SYSCALL_BIT + 143)
+#define __NR_sched_setscheduler (__X32_SYSCALL_BIT + 144)
+#define __NR_sched_getscheduler (__X32_SYSCALL_BIT + 145)
+#define __NR_sched_get_priority_max (__X32_SYSCALL_BIT + 146)
+#define __NR_sched_get_priority_min (__X32_SYSCALL_BIT + 147)
+#define __NR_sched_rr_get_interval (__X32_SYSCALL_BIT + 148)
+#define __NR_mlock (__X32_SYSCALL_BIT + 149)
+#define __NR_munlock (__X32_SYSCALL_BIT + 150)
+#define __NR_mlockall (__X32_SYSCALL_BIT + 151)
+#define __NR_munlockall (__X32_SYSCALL_BIT + 152)
+#define __NR_vhangup (__X32_SYSCALL_BIT + 153)
+#define __NR_modify_ldt (__X32_SYSCALL_BIT + 154)
+#define __NR_pivot_root (__X32_SYSCALL_BIT + 155)
+#define __NR_prctl (__X32_SYSCALL_BIT + 157)
+#define __NR_arch_prctl (__X32_SYSCALL_BIT + 158)
+#define __NR_adjtimex (__X32_SYSCALL_BIT + 159)
+#define __NR_setrlimit (__X32_SYSCALL_BIT + 160)
+#define __NR_chroot (__X32_SYSCALL_BIT + 161)
+#define __NR_sync (__X32_SYSCALL_BIT + 162)
+#define __NR_acct (__X32_SYSCALL_BIT + 163)
+#define __NR_settimeofday (__X32_SYSCALL_BIT + 164)
+#define __NR_mount (__X32_SYSCALL_BIT + 165)
+#define __NR_umount2 (__X32_SYSCALL_BIT + 166)
+#define __NR_swapon (__X32_SYSCALL_BIT + 167)
+#define __NR_swapoff (__X32_SYSCALL_BIT + 168)
+#define __NR_reboot (__X32_SYSCALL_BIT + 169)
+#define __NR_sethostname (__X32_SYSCALL_BIT + 170)
+#define __NR_setdomainname (__X32_SYSCALL_BIT + 171)
+#define __NR_iopl (__X32_SYSCALL_BIT + 172)
+#define __NR_ioperm (__X32_SYSCALL_BIT + 173)
+#define __NR_init_module (__X32_SYSCALL_BIT + 175)
+#define __NR_delete_module (__X32_SYSCALL_BIT + 176)
+#define __NR_quotactl (__X32_SYSCALL_BIT + 179)
+#define __NR_getpmsg (__X32_SYSCALL_BIT + 181)
+#define __NR_putpmsg (__X32_SYSCALL_BIT + 182)
+#define __NR_afs_syscall (__X32_SYSCALL_BIT + 183)
+#define __NR_tuxcall (__X32_SYSCALL_BIT + 184)
+#define __NR_security (__X32_SYSCALL_BIT + 185)
+#define __NR_gettid (__X32_SYSCALL_BIT + 186)
+#define __NR_readahead (__X32_SYSCALL_BIT + 187)
+#define __NR_setxattr (__X32_SYSCALL_BIT + 188)
+#define __NR_lsetxattr (__X32_SYSCALL_BIT + 189)
+#define __NR_fsetxattr (__X32_SYSCALL_BIT + 190)
+#define __NR_getxattr (__X32_SYSCALL_BIT + 191)
+#define __NR_lgetxattr (__X32_SYSCALL_BIT + 192)
+#define __NR_fgetxattr (__X32_SYSCALL_BIT + 193)
+#define __NR_listxattr (__X32_SYSCALL_BIT + 194)
+#define __NR_llistxattr (__X32_SYSCALL_BIT + 195)
+#define __NR_flistxattr (__X32_SYSCALL_BIT + 196)
+#define __NR_removexattr (__X32_SYSCALL_BIT + 197)
+#define __NR_lremovexattr (__X32_SYSCALL_BIT + 198)
+#define __NR_fremovexattr (__X32_SYSCALL_BIT + 199)
+#define __NR_tkill (__X32_SYSCALL_BIT + 200)
+#define __NR_time (__X32_SYSCALL_BIT + 201)
+#define __NR_futex (__X32_SYSCALL_BIT + 202)
+#define __NR_sched_setaffinity (__X32_SYSCALL_BIT + 203)
+#define __NR_sched_getaffinity (__X32_SYSCALL_BIT + 204)
+#define __NR_io_destroy (__X32_SYSCALL_BIT + 207)
+#define __NR_io_getevents (__X32_SYSCALL_BIT + 208)
+#define __NR_io_cancel (__X32_SYSCALL_BIT + 210)
+#define __NR_lookup_dcookie (__X32_SYSCALL_BIT + 212)
+#define __NR_epoll_create (__X32_SYSCALL_BIT + 213)
+#define __NR_remap_file_pages (__X32_SYSCALL_BIT + 216)
+#define __NR_getdents64 (__X32_SYSCALL_BIT + 217)
+#define __NR_set_tid_address (__X32_SYSCALL_BIT + 218)
+#define __NR_restart_syscall (__X32_SYSCALL_BIT + 219)
+#define __NR_semtimedop (__X32_SYSCALL_BIT + 220)
+#define __NR_fadvise64 (__X32_SYSCALL_BIT + 221)
+#define __NR_timer_settime (__X32_SYSCALL_BIT + 223)
+#define __NR_timer_gettime (__X32_SYSCALL_BIT + 224)
+#define __NR_timer_getoverrun (__X32_SYSCALL_BIT + 225)
+#define __NR_timer_delete (__X32_SYSCALL_BIT + 226)
+#define __NR_clock_settime (__X32_SYSCALL_BIT + 227)
+#define __NR_clock_gettime (__X32_SYSCALL_BIT + 228)
+#define __NR_clock_getres (__X32_SYSCALL_BIT + 229)
+#define __NR_clock_nanosleep (__X32_SYSCALL_BIT + 230)
+#define __NR_exit_group (__X32_SYSCALL_BIT + 231)
+#define __NR_epoll_wait (__X32_SYSCALL_BIT + 232)
+#define __NR_epoll_ctl (__X32_SYSCALL_BIT + 233)
+#define __NR_tgkill (__X32_SYSCALL_BIT + 234)
+#define __NR_utimes (__X32_SYSCALL_BIT + 235)
+#define __NR_mbind (__X32_SYSCALL_BIT + 237)
+#define __NR_set_mempolicy (__X32_SYSCALL_BIT + 238)
+#define __NR_get_mempolicy (__X32_SYSCALL_BIT + 239)
+#define __NR_mq_open (__X32_SYSCALL_BIT + 240)
+#define __NR_mq_unlink (__X32_SYSCALL_BIT + 241)
+#define __NR_mq_timedsend (__X32_SYSCALL_BIT + 242)
+#define __NR_mq_timedreceive (__X32_SYSCALL_BIT + 243)
+#define __NR_mq_getsetattr (__X32_SYSCALL_BIT + 245)
+#define __NR_add_key (__X32_SYSCALL_BIT + 248)
+#define __NR_request_key (__X32_SYSCALL_BIT + 249)
+#define __NR_keyctl (__X32_SYSCALL_BIT + 250)
+#define __NR_ioprio_set (__X32_SYSCALL_BIT + 251)
+#define __NR_ioprio_get (__X32_SYSCALL_BIT + 252)
+#define __NR_inotify_init (__X32_SYSCALL_BIT + 253)
+#define __NR_inotify_add_watch (__X32_SYSCALL_BIT + 254)
+#define __NR_inotify_rm_watch (__X32_SYSCALL_BIT + 255)
+#define __NR_migrate_pages (__X32_SYSCALL_BIT + 256)
+#define __NR_openat (__X32_SYSCALL_BIT + 257)
+#define __NR_mkdirat (__X32_SYSCALL_BIT + 258)
+#define __NR_mknodat (__X32_SYSCALL_BIT + 259)
+#define __NR_fchownat (__X32_SYSCALL_BIT + 260)
+#define __NR_futimesat (__X32_SYSCALL_BIT + 261)
+#define __NR_newfstatat (__X32_SYSCALL_BIT + 262)
+#define __NR_unlinkat (__X32_SYSCALL_BIT + 263)
+#define __NR_renameat (__X32_SYSCALL_BIT + 264)
+#define __NR_linkat (__X32_SYSCALL_BIT + 265)
+#define __NR_symlinkat (__X32_SYSCALL_BIT + 266)
+#define __NR_readlinkat (__X32_SYSCALL_BIT + 267)
+#define __NR_fchmodat (__X32_SYSCALL_BIT + 268)
+#define __NR_faccessat (__X32_SYSCALL_BIT + 269)
+#define __NR_pselect6 (__X32_SYSCALL_BIT + 270)
+#define __NR_ppoll (__X32_SYSCALL_BIT + 271)
+#define __NR_unshare (__X32_SYSCALL_BIT + 272)
+#define __NR_splice (__X32_SYSCALL_BIT + 275)
+#define __NR_tee (__X32_SYSCALL_BIT + 276)
+#define __NR_sync_file_range (__X32_SYSCALL_BIT + 277)
+#define __NR_utimensat (__X32_SYSCALL_BIT + 280)
+#define __NR_epoll_pwait (__X32_SYSCALL_BIT + 281)
+#define __NR_signalfd (__X32_SYSCALL_BIT + 282)
+#define __NR_timerfd_create (__X32_SYSCALL_BIT + 283)
+#define __NR_eventfd (__X32_SYSCALL_BIT + 284)
+#define __NR_fallocate (__X32_SYSCALL_BIT + 285)
+#define __NR_timerfd_settime (__X32_SYSCALL_BIT + 286)
+#define __NR_timerfd_gettime (__X32_SYSCALL_BIT + 287)
+#define __NR_accept4 (__X32_SYSCALL_BIT + 288)
+#define __NR_signalfd4 (__X32_SYSCALL_BIT + 289)
+#define __NR_eventfd2 (__X32_SYSCALL_BIT + 290)
+#define __NR_epoll_create1 (__X32_SYSCALL_BIT + 291)
+#define __NR_dup3 (__X32_SYSCALL_BIT + 292)
+#define __NR_pipe2 (__X32_SYSCALL_BIT + 293)
+#define __NR_inotify_init1 (__X32_SYSCALL_BIT + 294)
+#define __NR_perf_event_open (__X32_SYSCALL_BIT + 298)
+#define __NR_fanotify_init (__X32_SYSCALL_BIT + 300)
+#define __NR_fanotify_mark (__X32_SYSCALL_BIT + 301)
+#define __NR_prlimit64 (__X32_SYSCALL_BIT + 302)
+#define __NR_name_to_handle_at (__X32_SYSCALL_BIT + 303)
+#define __NR_open_by_handle_at (__X32_SYSCALL_BIT + 304)
+#define __NR_clock_adjtime (__X32_SYSCALL_BIT + 305)
+#define __NR_syncfs (__X32_SYSCALL_BIT + 306)
+#define __NR_setns (__X32_SYSCALL_BIT + 308)
+#define __NR_getcpu (__X32_SYSCALL_BIT + 309)
+#define __NR_kcmp (__X32_SYSCALL_BIT + 312)
+#define __NR_finit_module (__X32_SYSCALL_BIT + 313)
+#define __NR_sched_setattr (__X32_SYSCALL_BIT + 314)
+#define __NR_sched_getattr (__X32_SYSCALL_BIT + 315)
+#define __NR_renameat2 (__X32_SYSCALL_BIT + 316)
+#define __NR_seccomp (__X32_SYSCALL_BIT + 317)
+#define __NR_getrandom (__X32_SYSCALL_BIT + 318)
+#define __NR_memfd_create (__X32_SYSCALL_BIT + 319)
+#define __NR_kexec_file_load (__X32_SYSCALL_BIT + 320)
+#define __NR_bpf (__X32_SYSCALL_BIT + 321)
+#define __NR_userfaultfd (__X32_SYSCALL_BIT + 323)
+#define __NR_membarrier (__X32_SYSCALL_BIT + 324)
+#define __NR_mlock2 (__X32_SYSCALL_BIT + 325)
+#define __NR_copy_file_range (__X32_SYSCALL_BIT + 326)
+#define __NR_rt_sigaction (__X32_SYSCALL_BIT + 512)
+#define __NR_rt_sigreturn (__X32_SYSCALL_BIT + 513)
+#define __NR_ioctl (__X32_SYSCALL_BIT + 514)
+#define __NR_readv (__X32_SYSCALL_BIT + 515)
+#define __NR_writev (__X32_SYSCALL_BIT + 516)
+#define __NR_recvfrom (__X32_SYSCALL_BIT + 517)
+#define __NR_sendmsg (__X32_SYSCALL_BIT + 518)
+#define __NR_recvmsg (__X32_SYSCALL_BIT + 519)
+#define __NR_execve (__X32_SYSCALL_BIT + 520)
+#define __NR_ptrace (__X32_SYSCALL_BIT + 521)
+#define __NR_rt_sigpending (__X32_SYSCALL_BIT + 522)
+#define __NR_rt_sigtimedwait (__X32_SYSCALL_BIT + 523)
+#define __NR_rt_sigqueueinfo (__X32_SYSCALL_BIT + 524)
+#define __NR_sigaltstack (__X32_SYSCALL_BIT + 525)
+#define __NR_timer_create (__X32_SYSCALL_BIT + 526)
+#define __NR_mq_notify (__X32_SYSCALL_BIT + 527)
+#define __NR_kexec_load (__X32_SYSCALL_BIT + 528)
+#define __NR_waitid (__X32_SYSCALL_BIT + 529)
+#define __NR_set_robust_list (__X32_SYSCALL_BIT + 530)
+#define __NR_get_robust_list (__X32_SYSCALL_BIT + 531)
+#define __NR_vmsplice (__X32_SYSCALL_BIT + 532)
+#define __NR_move_pages (__X32_SYSCALL_BIT + 533)
+#define __NR_preadv (__X32_SYSCALL_BIT + 534)
+#define __NR_preadv2 (__X32_SYSCALL_BIT + 534)
+#define __NR_pwritev (__X32_SYSCALL_BIT + 535)
+#define __NR_pwritev2 (__X32_SYSCALL_BIT + 535)
+#define __NR_rt_tgsigqueueinfo (__X32_SYSCALL_BIT + 536)
+#define __NR_recvmmsg (__X32_SYSCALL_BIT + 537)
+#define __NR_sendmmsg (__X32_SYSCALL_BIT + 538)
+#define __NR_process_vm_readv (__X32_SYSCALL_BIT + 539)
+#define __NR_process_vm_writev (__X32_SYSCALL_BIT + 540)
+#define __NR_setsockopt (__X32_SYSCALL_BIT + 541)
+#define __NR_getsockopt (__X32_SYSCALL_BIT + 542)
+#define __NR_io_setup (__X32_SYSCALL_BIT + 543)
+#define __NR_io_submit (__X32_SYSCALL_BIT + 544)
+#define __NR_execveat (__X32_SYSCALL_BIT + 545)
+
+#endif /* _ASM_X86_UNISTD_X32_H */
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 12045a1..e60e21b 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -147,6 +147,36 @@
 
 #define KVM_PIT_SPEAKER_DUMMY     1
 
+struct kvm_s390_skeys {
+	__u64 start_gfn;
+	__u64 count;
+	__u64 skeydata_addr;
+	__u32 flags;
+	__u32 reserved[9];
+};
+
+struct kvm_hyperv_exit {
+#define KVM_EXIT_HYPERV_SYNIC          1
+#define KVM_EXIT_HYPERV_HCALL          2
+	__u32 type;
+	union {
+		struct {
+			__u32 msr;
+			__u64 control;
+			__u64 evt_page;
+			__u64 msg_page;
+		} synic;
+		struct {
+			__u64 input;
+			__u64 result;
+			__u64 params[2];
+		} hcall;
+	} u;
+};
+
+#define KVM_S390_GET_SKEYS_NONE   1
+#define KVM_S390_SKEYS_MAX        1048576
+
 #define KVM_EXIT_UNKNOWN          0
 #define KVM_EXIT_EXCEPTION        1
 #define KVM_EXIT_IO               2
@@ -172,6 +202,9 @@
 #define KVM_EXIT_S390_TSCH        22
 #define KVM_EXIT_EPR              23
 #define KVM_EXIT_SYSTEM_EVENT     24
+#define KVM_EXIT_S390_STSI        25
+#define KVM_EXIT_IOAPIC_EOI       26
+#define KVM_EXIT_HYPERV           27
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -191,7 +224,7 @@
 	__u32 exit_reason;
 	__u8 ready_for_interrupt_injection;
 	__u8 if_flag;
-	__u8 padding2[2];
+	__u16 flags;
 
 	/* in (pre_kvm_run), out (post_kvm_run) */
 	__u64 cr8;
@@ -226,6 +259,7 @@
 			__u32 count;
 			__u64 data_offset; /* relative to kvm_run start */
 		} io;
+		/* KVM_EXIT_DEBUG */
 		struct {
 			struct kvm_debug_exit_arch arch;
 		} debug;
@@ -274,6 +308,7 @@
 			__u32 data;
 			__u8  is_write;
 		} dcr;
+		/* KVM_EXIT_INTERNAL_ERROR */
 		struct {
 			__u32 suberror;
 			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
@@ -284,6 +319,7 @@
 		struct {
 			__u64 gprs[32];
 		} osi;
+		/* KVM_EXIT_PAPR_HCALL */
 		struct {
 			__u64 nr;
 			__u64 ret;
@@ -306,9 +342,25 @@
 		struct {
 #define KVM_SYSTEM_EVENT_SHUTDOWN       1
 #define KVM_SYSTEM_EVENT_RESET          2
+#define KVM_SYSTEM_EVENT_CRASH          3
 			__u32 type;
 			__u64 flags;
 		} system_event;
+		/* KVM_EXIT_S390_STSI */
+		struct {
+			__u64 addr;
+			__u8 ar;
+			__u8 reserved;
+			__u8 fc;
+			__u8 sel1;
+			__u16 sel2;
+		} s390_stsi;
+		/* KVM_EXIT_IOAPIC_EOI */
+		struct {
+			__u8 vector;
+		} eoi;
+		/* KVM_EXIT_HYPERV */
+		struct kvm_hyperv_exit hyperv;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -324,7 +376,7 @@
 	__u64 kvm_dirty_regs;
 	union {
 		struct kvm_sync_regs regs;
-		char padding[1024];
+		char padding[2048];
 	} s;
 };
 
@@ -365,6 +417,24 @@
 	__u8  pad[5];
 };
 
+/* for KVM_S390_MEM_OP */
+struct kvm_s390_mem_op {
+	/* in */
+	__u64 gaddr;		/* the guest address */
+	__u64 flags;		/* flags */
+	__u32 size;		/* amount of bytes */
+	__u32 op;		/* type of operation */
+	__u64 buf;		/* buffer in userspace */
+	__u8 ar;		/* the access register number */
+	__u8 reserved[31];	/* should be set to 0 */
+};
+/* types for kvm_s390_mem_op->op */
+#define KVM_S390_MEMOP_LOGICAL_READ	0
+#define KVM_S390_MEMOP_LOGICAL_WRITE	1
+/* flags for kvm_s390_mem_op->flags */
+#define KVM_S390_MEMOP_F_CHECK_ONLY		(1ULL << 0)
+#define KVM_S390_MEMOP_F_INJECT_EXCEPTION	(1ULL << 1)
+
 /* for KVM_INTERRUPT */
 struct kvm_interrupt {
 	/* in */
@@ -443,6 +513,7 @@
 	 ((ai) << 26))
 #define KVM_S390_INT_IO_MIN		0x00000000u
 #define KVM_S390_INT_IO_MAX		0xfffdffffu
+#define KVM_S390_INT_IO_AI_MASK		0x04000000u
 
 
 struct kvm_s390_interrupt {
@@ -476,7 +547,13 @@
 	__u8 exc_access_id;
 	__u8 per_access_id;
 	__u8 op_access_id;
-	__u8 pad[3];
+#define KVM_S390_PGM_FLAGS_ILC_VALID	0x01
+#define KVM_S390_PGM_FLAGS_ILC_0	0x02
+#define KVM_S390_PGM_FLAGS_ILC_1	0x04
+#define KVM_S390_PGM_FLAGS_ILC_MASK	0x06
+#define KVM_S390_PGM_FLAGS_NO_REWIND	0x08
+	__u8 flags;
+	__u8 pad[2];
 };
 
 struct kvm_s390_prefix_info {
@@ -491,6 +568,11 @@
 	__u16 code;
 };
 
+#define KVM_S390_STOP_FLAG_STORE_STATUS	0x01
+struct kvm_s390_stop_info {
+	__u32 flags;
+};
+
 struct kvm_s390_mchk_info {
 	__u64 cr14;
 	__u64 mcic;
@@ -509,11 +591,19 @@
 		struct kvm_s390_emerg_info emerg;
 		struct kvm_s390_extcall_info extcall;
 		struct kvm_s390_prefix_info prefix;
+		struct kvm_s390_stop_info stop;
 		struct kvm_s390_mchk_info mchk;
 		char reserved[64];
 	} u;
 };
 
+struct kvm_s390_irq_state {
+	__u64 buf;
+	__u32 flags;
+	__u32 len;
+	__u32 reserved[4];
+};
+
 /* for KVM_SET_GUEST_DEBUG */
 
 #define KVM_GUESTDBG_ENABLE		0x00000001
@@ -647,11 +737,7 @@
 #define KVM_CAP_MP_STATE 14
 #define KVM_CAP_COALESCED_MMIO 15
 #define KVM_CAP_SYNC_MMU 16  /* Changes to host mmap are reflected in guest */
-#define KVM_CAP_DEVICE_ASSIGNMENT 17
 #define KVM_CAP_IOMMU 18
-#ifdef __KVM_HAVE_MSI
-#define KVM_CAP_DEVICE_MSI 20
-#endif
 /* Bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_DESTROY_MEMORY_REGION_WORKS 21
 #define KVM_CAP_USER_NMI 22
@@ -663,10 +749,6 @@
 #endif
 #define KVM_CAP_IRQ_ROUTING 25
 #define KVM_CAP_IRQ_INJECT_STATUS 26
-#define KVM_CAP_DEVICE_DEASSIGNMENT 27
-#ifdef __KVM_HAVE_MSIX
-#define KVM_CAP_DEVICE_MSIX 28
-#endif
 #define KVM_CAP_ASSIGN_DEV_IRQ 29
 /* Another bug in KVM_SET_USER_MEMORY_REGION fixed: */
 #define KVM_CAP_JOIN_MEMORY_REGIONS_WORKS 30
@@ -761,6 +843,29 @@
 #define KVM_CAP_PPC_FIXUP_HCALL 103
 #define KVM_CAP_PPC_ENABLE_HCALL 104
 #define KVM_CAP_CHECK_EXTENSION_VM 105
+#define KVM_CAP_S390_USER_SIGP 106
+#define KVM_CAP_S390_VECTOR_REGISTERS 107
+#define KVM_CAP_S390_MEM_OP 108
+#define KVM_CAP_S390_USER_STSI 109
+#define KVM_CAP_S390_SKEYS 110
+#define KVM_CAP_MIPS_FPU 111
+#define KVM_CAP_MIPS_MSA 112
+#define KVM_CAP_S390_INJECT_IRQ 113
+#define KVM_CAP_S390_IRQ_STATE 114
+#define KVM_CAP_PPC_HWRNG 115
+#define KVM_CAP_DISABLE_QUIRKS 116
+#define KVM_CAP_X86_SMM 117
+#define KVM_CAP_MULTI_ADDRESS_SPACE 118
+#define KVM_CAP_GUEST_DEBUG_HW_BPS 119
+#define KVM_CAP_GUEST_DEBUG_HW_WPS 120
+#define KVM_CAP_SPLIT_IRQCHIP 121
+#define KVM_CAP_IOEVENTFD_ANY_LENGTH 122
+#define KVM_CAP_HYPERV_SYNIC 123
+#define KVM_CAP_S390_RI 124
+#define KVM_CAP_SPAPR_TCE_64 125
+#define KVM_CAP_ARM_PMU_V3 126
+#define KVM_CAP_VCPU_ATTRIBUTES 127
+#define KVM_CAP_MAX_VCPU_ID 128
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -784,10 +889,16 @@
 	__u32 adapter_id;
 };
 
+struct kvm_irq_routing_hv_sint {
+	__u32 vcpu;
+	__u32 sint;
+};
+
 /* gsi routing entry types */
 #define KVM_IRQ_ROUTING_IRQCHIP 1
 #define KVM_IRQ_ROUTING_MSI 2
 #define KVM_IRQ_ROUTING_S390_ADAPTER 3
+#define KVM_IRQ_ROUTING_HV_SINT 4
 
 struct kvm_irq_routing_entry {
 	__u32 gsi;
@@ -798,6 +909,7 @@
 		struct kvm_irq_routing_irqchip irqchip;
 		struct kvm_irq_routing_msi msi;
 		struct kvm_irq_routing_s390_adapter adapter;
+		struct kvm_irq_routing_hv_sint hv_sint;
 		__u32 pad[8];
 	} u;
 };
@@ -841,7 +953,7 @@
  *
  * KVM_IRQFD_FLAG_RESAMPLE indicates resamplefd is valid and specifies
  * the irqfd to operate in resampling mode for level triggered interrupt
- * emlation.  See Documentation/virtual/kvm/api.txt.
+ * emulation.  See Documentation/virtual/kvm/api.txt.
  */
 #define KVM_IRQFD_FLAG_RESAMPLE (1 << 1)
 
@@ -960,6 +1072,8 @@
 #define KVM_DEV_TYPE_ARM_VGIC_V2	KVM_DEV_TYPE_ARM_VGIC_V2
 	KVM_DEV_TYPE_FLIC,
 #define KVM_DEV_TYPE_FLIC		KVM_DEV_TYPE_FLIC
+	KVM_DEV_TYPE_ARM_VGIC_V3,
+#define KVM_DEV_TYPE_ARM_VGIC_V3	KVM_DEV_TYPE_ARM_VGIC_V3
 	KVM_DEV_TYPE_MAX,
 };
 
@@ -1044,6 +1158,8 @@
 /* Available with KVM_CAP_PPC_ALLOC_HTAB */
 #define KVM_PPC_ALLOCATE_HTAB	  _IOWR(KVMIO, 0xa7, __u32)
 #define KVM_CREATE_SPAPR_TCE	  _IOW(KVMIO,  0xa8, struct kvm_create_spapr_tce)
+#define KVM_CREATE_SPAPR_TCE_64	  _IOW(KVMIO,  0xa8, \
+				       struct kvm_create_spapr_tce_64)
 /* Available with KVM_CAP_RMA */
 #define KVM_ALLOCATE_RMA	  _IOR(KVMIO,  0xa9, struct kvm_allocate_rma)
 /* Available with KVM_CAP_PPC_HTAB_FD */
@@ -1107,9 +1223,6 @@
 #define KVM_X86_SETUP_MCE         _IOW(KVMIO,  0x9c, __u64)
 #define KVM_X86_GET_MCE_CAP_SUPPORTED _IOR(KVMIO,  0x9d, __u64)
 #define KVM_X86_SET_MCE           _IOW(KVMIO,  0x9e, struct kvm_x86_mce)
-/* IA64 stack access */
-#define KVM_IA64_VCPU_GET_STACK   _IOR(KVMIO,  0x9a, void *)
-#define KVM_IA64_VCPU_SET_STACK   _IOW(KVMIO,  0x9b, void *)
 /* Available with KVM_CAP_VCPU_EVENTS */
 #define KVM_GET_VCPU_EVENTS       _IOR(KVMIO,  0x9f, struct kvm_vcpu_events)
 #define KVM_SET_VCPU_EVENTS       _IOW(KVMIO,  0xa0, struct kvm_vcpu_events)
@@ -1137,6 +1250,18 @@
 #define KVM_ARM_VCPU_INIT	  _IOW(KVMIO,  0xae, struct kvm_vcpu_init)
 #define KVM_ARM_PREFERRED_TARGET  _IOR(KVMIO,  0xaf, struct kvm_vcpu_init)
 #define KVM_GET_REG_LIST	  _IOWR(KVMIO, 0xb0, struct kvm_reg_list)
+/* Available with KVM_CAP_S390_MEM_OP */
+#define KVM_S390_MEM_OP		  _IOW(KVMIO,  0xb1, struct kvm_s390_mem_op)
+/* Available with KVM_CAP_S390_SKEYS */
+#define KVM_S390_GET_SKEYS      _IOW(KVMIO, 0xb2, struct kvm_s390_skeys)
+#define KVM_S390_SET_SKEYS      _IOW(KVMIO, 0xb3, struct kvm_s390_skeys)
+/* Available with KVM_CAP_S390_INJECT_IRQ */
+#define KVM_S390_IRQ              _IOW(KVMIO,  0xb4, struct kvm_s390_irq)
+/* Available with KVM_CAP_S390_IRQ_STATE */
+#define KVM_S390_SET_IRQ_STATE	  _IOW(KVMIO, 0xb5, struct kvm_s390_irq_state)
+#define KVM_S390_GET_IRQ_STATE	  _IOW(KVMIO, 0xb6, struct kvm_s390_irq_state)
+/* Available with KVM_CAP_X86_SMM */
+#define KVM_SMI                   _IO(KVMIO,   0xb7)
 
 #define KVM_DEV_ASSIGN_ENABLE_IOMMU	(1 << 0)
 #define KVM_DEV_ASSIGN_PCI_2_3		(1 << 1)
diff --git a/linux-headers/linux/psci.h b/linux-headers/linux/psci.h
index 5a76763..08d443f 100644
--- a/linux-headers/linux/psci.h
+++ b/linux-headers/linux/psci.h
@@ -46,6 +46,11 @@
 #define PSCI_0_2_FN64_MIGRATE			PSCI_0_2_FN64(5)
 #define PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU	PSCI_0_2_FN64(7)
 
+#define PSCI_1_0_FN_PSCI_FEATURES		PSCI_0_2_FN(10)
+#define PSCI_1_0_FN_SYSTEM_SUSPEND		PSCI_0_2_FN(14)
+
+#define PSCI_1_0_FN64_SYSTEM_SUSPEND		PSCI_0_2_FN64(14)
+
 /* PSCI v0.2 power state encoding for CPU_SUSPEND function */
 #define PSCI_0_2_POWER_STATE_ID_MASK		0xffff
 #define PSCI_0_2_POWER_STATE_ID_SHIFT		0
@@ -56,6 +61,13 @@
 #define PSCI_0_2_POWER_STATE_AFFL_MASK		\
 				(0x3 << PSCI_0_2_POWER_STATE_AFFL_SHIFT)
 
+/* PSCI extended power state encoding for CPU_SUSPEND function */
+#define PSCI_1_0_EXT_POWER_STATE_ID_MASK	0xfffffff
+#define PSCI_1_0_EXT_POWER_STATE_ID_SHIFT	0
+#define PSCI_1_0_EXT_POWER_STATE_TYPE_SHIFT	30
+#define PSCI_1_0_EXT_POWER_STATE_TYPE_MASK	\
+				(0x1 << PSCI_1_0_EXT_POWER_STATE_TYPE_SHIFT)
+
 /* PSCI v0.2 affinity level state returned by AFFINITY_INFO */
 #define PSCI_0_2_AFFINITY_LEVEL_ON		0
 #define PSCI_0_2_AFFINITY_LEVEL_OFF		1
@@ -76,6 +88,11 @@
 #define PSCI_VERSION_MINOR(ver)			\
 		((ver) & PSCI_VERSION_MINOR_MASK)
 
+/* PSCI features decoding (>=1.0) */
+#define PSCI_1_0_FEATURES_CPU_SUSPEND_PF_SHIFT	1
+#define PSCI_1_0_FEATURES_CPU_SUSPEND_PF_MASK	\
+			(0x1 << PSCI_1_0_FEATURES_CPU_SUSPEND_PF_SHIFT)
+
 /* PSCI return values (inclusive of all PSCI versions) */
 #define PSCI_RET_SUCCESS			0
 #define PSCI_RET_NOT_SUPPORTED			-1
@@ -86,5 +103,6 @@
 #define PSCI_RET_INTERNAL_FAILURE		-6
 #define PSCI_RET_NOT_PRESENT			-7
 #define PSCI_RET_DISABLED			-8
+#define PSCI_RET_INVALID_ADDRESS		-9
 
 #endif /* _LINUX_PSCI_H */
diff --git a/linux-headers/linux/userfaultfd.h b/linux-headers/linux/userfaultfd.h
new file mode 100644
index 0000000..19e8453
--- /dev/null
+++ b/linux-headers/linux/userfaultfd.h
@@ -0,0 +1,167 @@
+/*
+ *  include/linux/userfaultfd.h
+ *
+ *  Copyright (C) 2007  Davide Libenzi <davidel@xmailserver.org>
+ *  Copyright (C) 2015  Red Hat, Inc.
+ *
+ */
+
+#ifndef _LINUX_USERFAULTFD_H
+#define _LINUX_USERFAULTFD_H
+
+#include <linux/types.h>
+
+#define UFFD_API ((__u64)0xAA)
+/*
+ * After implementing the respective features it will become:
+ * #define UFFD_API_FEATURES (UFFD_FEATURE_PAGEFAULT_FLAG_WP | \
+ *			      UFFD_FEATURE_EVENT_FORK)
+ */
+#define UFFD_API_FEATURES (0)
+#define UFFD_API_IOCTLS				\
+	((__u64)1 << _UFFDIO_REGISTER |		\
+	 (__u64)1 << _UFFDIO_UNREGISTER |	\
+	 (__u64)1 << _UFFDIO_API)
+#define UFFD_API_RANGE_IOCTLS			\
+	((__u64)1 << _UFFDIO_WAKE |		\
+	 (__u64)1 << _UFFDIO_COPY |		\
+	 (__u64)1 << _UFFDIO_ZEROPAGE)
+
+/*
+ * Valid ioctl command number range with this API is from 0x00 to
+ * 0x3F.  UFFDIO_API is the fixed number, everything else can be
+ * changed by implementing a different UFFD_API. If sticking to the
+ * same UFFD_API more ioctl can be added and userland will be aware of
+ * which ioctl the running kernel implements through the ioctl command
+ * bitmask written by the UFFDIO_API.
+ */
+#define _UFFDIO_REGISTER		(0x00)
+#define _UFFDIO_UNREGISTER		(0x01)
+#define _UFFDIO_WAKE			(0x02)
+#define _UFFDIO_COPY			(0x03)
+#define _UFFDIO_ZEROPAGE		(0x04)
+#define _UFFDIO_API			(0x3F)
+
+/* userfaultfd ioctl ids */
+#define UFFDIO 0xAA
+#define UFFDIO_API		_IOWR(UFFDIO, _UFFDIO_API,	\
+				      struct uffdio_api)
+#define UFFDIO_REGISTER		_IOWR(UFFDIO, _UFFDIO_REGISTER, \
+				      struct uffdio_register)
+#define UFFDIO_UNREGISTER	_IOR(UFFDIO, _UFFDIO_UNREGISTER,	\
+				     struct uffdio_range)
+#define UFFDIO_WAKE		_IOR(UFFDIO, _UFFDIO_WAKE,	\
+				     struct uffdio_range)
+#define UFFDIO_COPY		_IOWR(UFFDIO, _UFFDIO_COPY,	\
+				      struct uffdio_copy)
+#define UFFDIO_ZEROPAGE		_IOWR(UFFDIO, _UFFDIO_ZEROPAGE,	\
+				      struct uffdio_zeropage)
+
+/* read() structure */
+struct uffd_msg {
+	__u8	event;
+
+	__u8	reserved1;
+	__u16	reserved2;
+	__u32	reserved3;
+
+	union {
+		struct {
+			__u64	flags;
+			__u64	address;
+		} pagefault;
+
+		struct {
+			/* unused reserved fields */
+			__u64	reserved1;
+			__u64	reserved2;
+			__u64	reserved3;
+		} reserved;
+	} arg;
+} __attribute__((packed));
+
+/*
+ * Start at 0x12 and not at 0 to be more strict against bugs.
+ */
+#define UFFD_EVENT_PAGEFAULT	0x12
+#if 0 /* not available yet */
+#define UFFD_EVENT_FORK		0x13
+#endif
+
+/* flags for UFFD_EVENT_PAGEFAULT */
+#define UFFD_PAGEFAULT_FLAG_WRITE	(1<<0)	/* If this was a write fault */
+#define UFFD_PAGEFAULT_FLAG_WP		(1<<1)	/* If reason is VM_UFFD_WP */
+
+struct uffdio_api {
+	/* userland asks for an API number and the features to enable */
+	__u64 api;
+	/*
+	 * Kernel answers below with the all available features for
+	 * the API, this notifies userland of which events and/or
+	 * which flags for each event are enabled in the current
+	 * kernel.
+	 *
+	 * Note: UFFD_EVENT_PAGEFAULT and UFFD_PAGEFAULT_FLAG_WRITE
+	 * are to be considered implicitly always enabled in all kernels as
+	 * long as the uffdio_api.api requested matches UFFD_API.
+	 */
+#if 0 /* not available yet */
+#define UFFD_FEATURE_PAGEFAULT_FLAG_WP		(1<<0)
+#define UFFD_FEATURE_EVENT_FORK			(1<<1)
+#endif
+	__u64 features;
+
+	__u64 ioctls;
+};
+
+struct uffdio_range {
+	__u64 start;
+	__u64 len;
+};
+
+struct uffdio_register {
+	struct uffdio_range range;
+#define UFFDIO_REGISTER_MODE_MISSING	((__u64)1<<0)
+#define UFFDIO_REGISTER_MODE_WP		((__u64)1<<1)
+	__u64 mode;
+
+	/*
+	 * kernel answers which ioctl commands are available for the
+	 * range, keep at the end as the last 8 bytes aren't read.
+	 */
+	__u64 ioctls;
+};
+
+struct uffdio_copy {
+	__u64 dst;
+	__u64 src;
+	__u64 len;
+	/*
+	 * There will be a wrprotection flag later that allows to map
+	 * pages wrprotected on the fly. And such a flag will be
+	 * available if the wrprotection ioctl are implemented for the
+	 * range according to the uffdio_register.ioctls.
+	 */
+#define UFFDIO_COPY_MODE_DONTWAKE		((__u64)1<<0)
+	__u64 mode;
+
+	/*
+	 * "copy" is written by the ioctl and must be at the end: the
+	 * copy_from_user will not read the last 8 bytes.
+	 */
+	__s64 copy;
+};
+
+struct uffdio_zeropage {
+	struct uffdio_range range;
+#define UFFDIO_ZEROPAGE_MODE_DONTWAKE		((__u64)1<<0)
+	__u64 mode;
+
+	/*
+	 * "zeropage" is written by the ioctl and must be at the end:
+	 * the copy_from_user will not read the last 8 bytes.
+	 */
+	__s64 zeropage;
+};
+
+#endif /* _LINUX_USERFAULTFD_H */
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 0f21aa6..759b850 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -36,6 +36,15 @@
 /* Two-stage IOMMU */
 #define VFIO_TYPE1_NESTING_IOMMU	6	/* Implies v2 */
 
+#define VFIO_SPAPR_TCE_v2_IOMMU		7
+
+/*
+ * The No-IOMMU IOMMU offers no translation or isolation for devices and
+ * supports no ioctls outside of VFIO_CHECK_EXTENSION.  Use of VFIO's No-IOMMU
+ * code will taint the host kernel and should be used with extreme caution.
+ */
+#define VFIO_NOIOMMU_IOMMU		8
+
 /*
  * The IOCTL interface is designed for extensibility by embedding the
  * structure length (argsz) and flags into structures passed between
@@ -50,6 +59,33 @@
 #define VFIO_TYPE	(';')
 #define VFIO_BASE	100
 
+/*
+ * For extension of INFO ioctls, VFIO makes use of a capability chain
+ * designed after PCI/e capabilities.  A flag bit indicates whether
+ * this capability chain is supported and a field defined in the fixed
+ * structure defines the offset of the first capability in the chain.
+ * This field is only valid when the corresponding bit in the flags
+ * bitmap is set.  This offset field is relative to the start of the
+ * INFO buffer, as is the next field within each capability header.
+ * The id within the header is a shared address space per INFO ioctl,
+ * while the version field is specific to the capability id.  The
+ * contents following the header are specific to the capability id.
+ */
+struct vfio_info_cap_header {
+	__u16	id;		/* Identifies capability */
+	__u16	version;	/* Version specific to the capability ID */
+	__u32	next;		/* Offset of next capability */
+};
+
+/*
+ * Callers of INFO ioctls passing insufficiently sized buffers will see
+ * the capability chain flag bit set, a zero value for the first capability
+ * offset (if available within the provided argsz), and argsz will be
+ * updated to report the necessary buffer size.  For compatibility, the
+ * INFO ioctl will not report error in this case, but the capability chain
+ * will not be available.
+ */
+
 /* -------- IOCTLs for VFIO file descriptor (/dev/vfio/vfio) -------- */
 
 /**
@@ -160,6 +196,8 @@
 	__u32	flags;
 #define VFIO_DEVICE_FLAGS_RESET	(1 << 0)	/* Device supports reset */
 #define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
+#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)	/* vfio-platform device */
+#define VFIO_DEVICE_FLAGS_AMBA  (1 << 3)	/* vfio-amba device */
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 };
@@ -183,13 +221,73 @@
 #define VFIO_REGION_INFO_FLAG_READ	(1 << 0) /* Region supports read */
 #define VFIO_REGION_INFO_FLAG_WRITE	(1 << 1) /* Region supports write */
 #define VFIO_REGION_INFO_FLAG_MMAP	(1 << 2) /* Region supports mmap */
+#define VFIO_REGION_INFO_FLAG_CAPS	(1 << 3) /* Info supports caps */
 	__u32	index;		/* Region index */
-	__u32	resv;		/* Reserved for alignment */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 	__u64	size;		/* Region size (bytes) */
 	__u64	offset;		/* Region offset from start of device fd */
 };
 #define VFIO_DEVICE_GET_REGION_INFO	_IO(VFIO_TYPE, VFIO_BASE + 8)
 
+/*
+ * The sparse mmap capability allows finer granularity of specifying areas
+ * within a region with mmap support.  When specified, the user should only
+ * mmap the offset ranges specified by the areas array.  mmaps outside of the
+ * areas specified may fail (such as the range covering a PCI MSI-X table) or
+ * may result in improper device behavior.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_REGION_INFO_CAP_SPARSE_MMAP	1
+
+struct vfio_region_sparse_mmap_area {
+	__u64	offset;	/* Offset of mmap'able area within region */
+	__u64	size;	/* Size of mmap'able area */
+};
+
+struct vfio_region_info_cap_sparse_mmap {
+	struct vfio_info_cap_header header;
+	__u32	nr_areas;
+	__u32	reserved;
+	struct vfio_region_sparse_mmap_area areas[];
+};
+
+/*
+ * The device specific type capability allows regions unique to a specific
+ * device or class of devices to be exposed.  This helps solve the problem for
+ * vfio bus drivers of defining which region indexes correspond to which region
+ * on the device, without needing to resort to static indexes, as done by
+ * vfio-pci.  For instance, if we were to go back in time, we might remove
+ * VFIO_PCI_VGA_REGION_INDEX and let vfio-pci simply define that all indexes
+ * greater than or equal to VFIO_PCI_NUM_REGIONS are device specific and we'd
+ * make a "VGA" device specific type to describe the VGA access space.  This
+ * means that non-VGA devices wouldn't need to waste this index, and thus the
+ * address space associated with it due to implementation of device file
+ * descriptor offsets in vfio-pci.
+ *
+ * The current implementation is now part of the user ABI, so we can't use this
+ * for VGA, but there are other upcoming use cases, such as opregions for Intel
+ * IGD devices and framebuffers for vGPU devices.  We missed VGA, but we'll
+ * use this for future additions.
+ *
+ * The structure below defines version 1 of this capability.
+ */
+#define VFIO_REGION_INFO_CAP_TYPE	2
+
+struct vfio_region_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;	/* global per bus driver */
+	__u32 subtype;	/* type specific */
+};
+
+#define VFIO_REGION_TYPE_PCI_VENDOR_TYPE	(1 << 31)
+#define VFIO_REGION_TYPE_PCI_VENDOR_MASK	(0xffff)
+
+/* 8086 Vendor sub-types */
+#define VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION	(1)
+#define VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG	(2)
+#define VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG	(3)
+
 /**
  * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9,
  *				    struct vfio_irq_info)
@@ -325,7 +423,8 @@
 	 * between described ranges are unimplemented.
 	 */
 	VFIO_PCI_VGA_REGION_INDEX,
-	VFIO_PCI_NUM_REGIONS
+	VFIO_PCI_NUM_REGIONS = 9 /* Fixed user ABI, region indexes >=9 use */
+				 /* device specific cap to define content. */
 };
 
 enum {
@@ -333,6 +432,7 @@
 	VFIO_PCI_MSI_IRQ_INDEX,
 	VFIO_PCI_MSIX_IRQ_INDEX,
 	VFIO_PCI_ERR_IRQ_INDEX,
+	VFIO_PCI_REQ_IRQ_INDEX,
 	VFIO_PCI_NUM_IRQS
 };
 
@@ -440,6 +540,23 @@
 /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */
 
 /*
+ * The SPAPR TCE DDW info struct provides the information about
+ * the details of Dynamic DMA window capability.
+ *
+ * @pgsizes contains a page size bitmask, 4K/64K/16M are supported.
+ * @max_dynamic_windows_supported tells the maximum number of windows
+ * which the platform can create.
+ * @levels tells the maximum number of levels in multi-level IOMMU tables;
+ * this allows splitting a table into smaller chunks which reduces
+ * the amount of physically contiguous memory required for the table.
+ */
+struct vfio_iommu_spapr_tce_ddw_info {
+	__u64 pgsizes;			/* Bitmap of supported page sizes */
+	__u32 max_dynamic_windows_supported;
+	__u32 levels;
+};
+
+/*
  * The SPAPR TCE info struct provides the information about the PCI bus
  * address ranges available for DMA, these values are programmed into
  * the hardware so the guest has to know that information.
@@ -449,14 +566,17 @@
  * addresses too so the window works as a filter rather than an offset
  * for IOVA addresses.
  *
- * A flag will need to be added if other page sizes are supported,
- * so as defined here, it is always 4k.
+ * Flags supported:
+ * - VFIO_IOMMU_SPAPR_INFO_DDW: informs the userspace that dynamic DMA windows
+ *   (DDW) support is present. @ddw is only supported when DDW is present.
  */
 struct vfio_iommu_spapr_tce_info {
 	__u32 argsz;
-	__u32 flags;			/* reserved for future use */
+	__u32 flags;
+#define VFIO_IOMMU_SPAPR_INFO_DDW	(1 << 0)	/* DDW supported */
 	__u32 dma32_window_start;	/* 32 bit window start (bytes) */
 	__u32 dma32_window_size;	/* 32 bit window size (bytes) */
+	struct vfio_iommu_spapr_tce_ddw_info ddw;
 };
 
 #define VFIO_IOMMU_SPAPR_TCE_GET_INFO	_IO(VFIO_TYPE, VFIO_BASE + 12)
@@ -467,12 +587,23 @@
  * - unfreeze IO/DMA for frozen PE;
  * - read PE state;
  * - reset PE;
- * - configure PE.
+ * - configure PE;
+ * - inject EEH error.
  */
+struct vfio_eeh_pe_err {
+	__u32 type;
+	__u32 func;
+	__u64 addr;
+	__u64 mask;
+};
+
 struct vfio_eeh_pe_op {
 	__u32 argsz;
 	__u32 flags;
 	__u32 op;
+	union {
+		struct vfio_eeh_pe_err err;
+	};
 };
 
 #define VFIO_EEH_PE_DISABLE		0	/* Disable EEH functionality */
@@ -489,9 +620,72 @@
 #define VFIO_EEH_PE_RESET_HOT		6	/* Assert hot reset          */
 #define VFIO_EEH_PE_RESET_FUNDAMENTAL	7	/* Assert fundamental reset  */
 #define VFIO_EEH_PE_CONFIGURE		8	/* PE configuration          */
+#define VFIO_EEH_PE_INJECT_ERR		9	/* Inject EEH error          */
 
 #define VFIO_EEH_PE_OP			_IO(VFIO_TYPE, VFIO_BASE + 21)
 
+/**
+ * VFIO_IOMMU_SPAPR_REGISTER_MEMORY - _IOW(VFIO_TYPE, VFIO_BASE + 17, struct vfio_iommu_spapr_register_memory)
+ *
+ * Registers user space memory where DMA is allowed. It pins
+ * user pages and does the locked memory accounting so
+ * subsequent VFIO_IOMMU_MAP_DMA/VFIO_IOMMU_UNMAP_DMA calls
+ * get faster.
+ */
+struct vfio_iommu_spapr_register_memory {
+	__u32	argsz;
+	__u32	flags;
+	__u64	vaddr;				/* Process virtual address */
+	__u64	size;				/* Size of mapping (bytes) */
+};
+#define VFIO_IOMMU_SPAPR_REGISTER_MEMORY	_IO(VFIO_TYPE, VFIO_BASE + 17)
+
+/**
+ * VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY - _IOW(VFIO_TYPE, VFIO_BASE + 18, struct vfio_iommu_spapr_register_memory)
+ *
+ * Unregisters user space memory registered with
+ * VFIO_IOMMU_SPAPR_REGISTER_MEMORY.
+ * Uses vfio_iommu_spapr_register_memory for parameters.
+ */
+#define VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY	_IO(VFIO_TYPE, VFIO_BASE + 18)
+
+/**
+ * VFIO_IOMMU_SPAPR_TCE_CREATE - _IOWR(VFIO_TYPE, VFIO_BASE + 19, struct vfio_iommu_spapr_tce_create)
+ *
+ * Creates an additional TCE table and programs it (sets a new DMA window)
+ * to every IOMMU group in the container. It receives page shift, window
+ * size and number of levels in the TCE table being created.
+ *
+ * It allocates and returns an offset on a PCI bus of the new DMA window.
+ */
+struct vfio_iommu_spapr_tce_create {
+	__u32 argsz;
+	__u32 flags;
+	/* in */
+	__u32 page_shift;
+	__u32 __resv1;
+	__u64 window_size;
+	__u32 levels;
+	__u32 __resv2;
+	/* out */
+	__u64 start_addr;
+};
+#define VFIO_IOMMU_SPAPR_TCE_CREATE	_IO(VFIO_TYPE, VFIO_BASE + 19)
+
+/**
+ * VFIO_IOMMU_SPAPR_TCE_REMOVE - _IOW(VFIO_TYPE, VFIO_BASE + 20, struct vfio_iommu_spapr_tce_remove)
+ *
+ * Unprograms a TCE table from all groups in the container and destroys it.
+ * It receives a PCI bus offset as a window id.
+ */
+struct vfio_iommu_spapr_tce_remove {
+	__u32 argsz;
+	__u32 flags;
+	/* in */
+	__u64 start_addr;
+};
+#define VFIO_IOMMU_SPAPR_TCE_REMOVE	_IO(VFIO_TYPE, VFIO_BASE + 20)
+
 /* ***************************************************************** */
 
 #endif /* VFIO_H */
diff --git a/linux-headers/linux/vhost.h b/linux-headers/linux/vhost.h
index c656f61..571294c 100644
--- a/linux-headers/linux/vhost.h
+++ b/linux-headers/linux/vhost.h
@@ -103,6 +103,20 @@
 /* Get accessor: reads index, writes value in num */
 #define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
 
+/* Set the vring byte order in num. Valid values are VHOST_VRING_LITTLE_ENDIAN
+ * or VHOST_VRING_BIG_ENDIAN (other values return -EINVAL).
+ * The byte order cannot be changed while the device is active: trying to do so
+ * returns -EBUSY.
+ * This is a legacy only API that is simply ignored when VIRTIO_F_VERSION_1 is
+ * set.
+ * Not all kernel configurations support this ioctl, but all configurations that
+ * support SET also support GET.
+ */
+#define VHOST_VRING_LITTLE_ENDIAN 0
+#define VHOST_VRING_BIG_ENDIAN 1
+#define VHOST_SET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x13, struct vhost_vring_state)
+#define VHOST_GET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x14, struct vhost_vring_state)
+
 /* The following ioctls use eventfd file descriptors to signal and poll
  * for events. */
 
@@ -112,6 +126,12 @@
 #define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file)
 /* Set eventfd to signal an error */
 #define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
+/* Set busy loop timeout (in us) */
+#define VHOST_SET_VRING_BUSYLOOP_TIMEOUT _IOW(VHOST_VIRTIO, 0x23,	\
+					 struct vhost_vring_state)
+/* Get busy loop timeout (in us) */
+#define VHOST_GET_VRING_BUSYLOOP_TIMEOUT _IOW(VHOST_VIRTIO, 0x24,	\
+					 struct vhost_vring_state)
 
 /* VHOST_NET specific defines */
 
diff --git a/linux-headers/linux/virtio_config.h b/linux-headers/linux/virtio_config.h
index 75dc20b..5590f7d 100644
--- a/linux-headers/linux/virtio_config.h
+++ b/linux-headers/linux/virtio_config.h
@@ -1,57 +1 @@
-#ifndef _LINUX_VIRTIO_CONFIG_H
-#define _LINUX_VIRTIO_CONFIG_H
-/* This header, excluding the #ifdef __KERNEL__ part, is BSD licensed so
- * anyone can use the definitions to implement compatible drivers/servers.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of IBM nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE. */
-
-/* Virtio devices use a standardized configuration space to define their
- * features and pass configuration information, but each implementation can
- * store and access that space differently. */
-#include <linux/types.h>
-
-/* Status byte for guest to report progress, and synchronize features. */
-/* We have seen device and processed generic fields (VIRTIO_CONFIG_F_VIRTIO) */
-#define VIRTIO_CONFIG_S_ACKNOWLEDGE	1
-/* We have found a driver for the device. */
-#define VIRTIO_CONFIG_S_DRIVER		2
-/* Driver has used its parts of the config, and is happy */
-#define VIRTIO_CONFIG_S_DRIVER_OK	4
-/* We've given up on this device. */
-#define VIRTIO_CONFIG_S_FAILED		0x80
-
-/* Some virtio feature bits (currently bits 28 through 31) are reserved for the
- * transport being used (eg. virtio_ring), the rest are per-device feature
- * bits. */
-#define VIRTIO_TRANSPORT_F_START	28
-#define VIRTIO_TRANSPORT_F_END		32
-
-/* Do we get callbacks when the ring is completely used, even if we've
- * suppressed them? */
-#define VIRTIO_F_NOTIFY_ON_EMPTY	24
-
-/* Can the device handle any descriptor layout? */
-#define VIRTIO_F_ANY_LAYOUT		27
-
-#endif /* _LINUX_VIRTIO_CONFIG_H */
+#include "standard-headers/linux/virtio_config.h"
diff --git a/linux-headers/linux/virtio_ring.h b/linux-headers/linux/virtio_ring.h
index 1b333e2..c6f0fb6 100644
--- a/linux-headers/linux/virtio_ring.h
+++ b/linux-headers/linux/virtio_ring.h
@@ -1,163 +1 @@
-#ifndef _LINUX_VIRTIO_RING_H
-#define _LINUX_VIRTIO_RING_H
-/* An interface for efficient virtio implementation, currently for use by KVM
- * and lguest, but hopefully others soon.  Do NOT change this since it will
- * break existing servers and clients.
- *
- * This header is BSD licensed so anyone can use the definitions to implement
- * compatible drivers/servers.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of IBM nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * Copyright Rusty Russell IBM Corporation 2007. */
-#include <linux/types.h>
-
-/* This marks a buffer as continuing via the next field. */
-#define VRING_DESC_F_NEXT	1
-/* This marks a buffer as write-only (otherwise read-only). */
-#define VRING_DESC_F_WRITE	2
-/* This means the buffer contains a list of buffer descriptors. */
-#define VRING_DESC_F_INDIRECT	4
-
-/* The Host uses this in used->flags to advise the Guest: don't kick me when
- * you add a buffer.  It's unreliable, so it's simply an optimization.  Guest
- * will still kick if it's out of buffers. */
-#define VRING_USED_F_NO_NOTIFY	1
-/* The Guest uses this in avail->flags to advise the Host: don't interrupt me
- * when you consume a buffer.  It's unreliable, so it's simply an
- * optimization.  */
-#define VRING_AVAIL_F_NO_INTERRUPT	1
-
-/* We support indirect buffer descriptors */
-#define VIRTIO_RING_F_INDIRECT_DESC	28
-
-/* The Guest publishes the used index for which it expects an interrupt
- * at the end of the avail ring. Host should ignore the avail->flags field. */
-/* The Host publishes the avail index for which it expects a kick
- * at the end of the used ring. Guest should ignore the used->flags field. */
-#define VIRTIO_RING_F_EVENT_IDX		29
-
-/* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
-struct vring_desc {
-	/* Address (guest-physical). */
-	__u64 addr;
-	/* Length. */
-	__u32 len;
-	/* The flags as indicated above. */
-	__u16 flags;
-	/* We chain unused descriptors via this, too */
-	__u16 next;
-};
-
-struct vring_avail {
-	__u16 flags;
-	__u16 idx;
-	__u16 ring[];
-};
-
-/* u32 is used here for ids for padding reasons. */
-struct vring_used_elem {
-	/* Index of start of used descriptor chain. */
-	__u32 id;
-	/* Total length of the descriptor chain which was used (written to) */
-	__u32 len;
-};
-
-struct vring_used {
-	__u16 flags;
-	__u16 idx;
-	struct vring_used_elem ring[];
-};
-
-struct vring {
-	unsigned int num;
-
-	struct vring_desc *desc;
-
-	struct vring_avail *avail;
-
-	struct vring_used *used;
-};
-
-/* The standard layout for the ring is a continuous chunk of memory which looks
- * like this.  We assume num is a power of 2.
- *
- * struct vring
- * {
- *	// The actual descriptors (16 bytes each)
- *	struct vring_desc desc[num];
- *
- *	// A ring of available descriptor heads with free-running index.
- *	__u16 avail_flags;
- *	__u16 avail_idx;
- *	__u16 available[num];
- *	__u16 used_event_idx;
- *
- *	// Padding to the next align boundary.
- *	char pad[];
- *
- *	// A ring of used descriptor heads with free-running index.
- *	__u16 used_flags;
- *	__u16 used_idx;
- *	struct vring_used_elem used[num];
- *	__u16 avail_event_idx;
- * };
- */
-/* We publish the used event index at the end of the available ring, and vice
- * versa. They are at the end for backwards compatibility. */
-#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num])
-#define vring_avail_event(vr) (*(__u16 *)&(vr)->used->ring[(vr)->num])
-
-static __inline__ void vring_init(struct vring *vr, unsigned int num, void *p,
-			      unsigned long align)
-{
-	vr->num = num;
-	vr->desc = p;
-	vr->avail = p + num*sizeof(struct vring_desc);
-	vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + sizeof(__u16)
-		+ align-1) & ~(align - 1));
-}
-
-static __inline__ unsigned vring_size(unsigned int num, unsigned long align)
-{
-	return ((sizeof(struct vring_desc) * num + sizeof(__u16) * (3 + num)
-		 + align - 1) & ~(align - 1))
-		+ sizeof(__u16) * 3 + sizeof(struct vring_used_elem) * num;
-}
-
-/* The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX */
-/* Assuming a given event_idx value from the other size, if
- * we have just incremented index from old to new_idx,
- * should we trigger an event? */
-static __inline__ int vring_need_event(__u16 event_idx, __u16 new_idx, __u16 old)
-{
-	/* Note: Xen has similar logic for notification hold-off
-	 * in include/xen/interface/io/ring.h with req_event and req_prod
-	 * corresponding to event_idx + 1 and new_idx respectively.
-	 * Note also that req_event and req_prod in Xen start at 1,
-	 * event indexes in virtio start at 0. */
-	return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old);
-}
-
-#endif /* _LINUX_VIRTIO_RING_H */
+#include "standard-headers/linux/virtio_ring.h"
diff --git a/linux-user/Makefile.objs b/linux-user/Makefile.objs
index fd50217..8c93058 100644
--- a/linux-user/Makefile.objs
+++ b/linux-user/Makefile.objs
@@ -1,5 +1,6 @@
 obj-y = main.o syscall.o strace.o mmap.o signal.o \
-	elfload.o linuxload.o uaccess.o uname.o
+	elfload.o linuxload.o uaccess.o uname.o \
+	safe-syscall.o
 
 obj-$(TARGET_HAS_BFLT) += flatload.o
 obj-$(TARGET_I386) += vm86.o
diff --git a/linux-user/aarch64/syscall.h b/linux-user/aarch64/syscall.h
deleted file mode 100644
index dc72a15..0000000
--- a/linux-user/aarch64/syscall.h
+++ /dev/null
@@ -1,13 +0,0 @@
-struct target_pt_regs {
-    uint64_t        regs[31];
-    uint64_t        sp;
-    uint64_t        pc;
-    uint64_t        pstate;
-};
-
-#define UNAME_MACHINE "aarch64"
-#define UNAME_MINIMUM_RELEASE "3.8.0"
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ       2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/aarch64/syscall_nr.h b/linux-user/aarch64/syscall_nr.h
index 743255d..a3c9a3b 100644
--- a/linux-user/aarch64/syscall_nr.h
+++ b/linux-user/aarch64/syscall_nr.h
@@ -86,8 +86,7 @@
 #define TARGET_NR_sync 81
 #define TARGET_NR_fsync 82
 #define TARGET_NR_fdatasync 83
-#define TARGET_NR_sync_file_range2 84
-/* #define TARGET_NR_sync_file_range 84 */
+#define TARGET_NR_sync_file_range 84
 #define TARGET_NR_timerfd_create 85
 #define TARGET_NR_timerfd_settime 86
 #define TARGET_NR_timerfd_gettime 87
@@ -262,62 +261,16 @@
 #define TARGET_NR_process_vm_writev 271
 #define TARGET_NR_kcmp 272
 #define TARGET_NR_finit_module 273
-#define TARGET_NR_open 1024
-#define TARGET_NR_link 1025
-#define TARGET_NR_unlink 1026
-#define TARGET_NR_mknod 1027
-#define TARGET_NR_chmod 1028
-#define TARGET_NR_chown 1029
-#define TARGET_NR_mkdir 1030
-#define TARGET_NR_rmdir 1031
-#define TARGET_NR_lchown 1032
-#define TARGET_NR_access 1033
-#define TARGET_NR_rename 1034
-#define TARGET_NR_readlink 1035
-#define TARGET_NR_symlink 1036
-#define TARGET_NR_utimes 1037
-#define TARGET_NR_stat 1038
-#define TARGET_NR_lstat 1039
-#define TARGET_NR_pipe 1040
-#define TARGET_NR_dup2 1041
-#define TARGET_NR_epoll_create 1042
-#define TARGET_NR_inotify_init 1043
-#define TARGET_NR_eventfd 1044
-#define TARGET_NR_signalfd 1045
-#define TARGET_NR_sendfile64 1046
-#define TARGET_NR_ftruncate64 1047
-#define TARGET_NR_truncate64 1048
-#define TARGET_NR_stat64 1049
-#define TARGET_NR_lstat64 1050
-#define TARGET_NR_fstat64 1051
-#define TARGET_NR_fcntl64 1052
-/* #define TARGET_NR_fadvise64 1053 */
-#define TARGET_NR_newfstatat 1054
-#define TARGET_NR_fstatfs64 1055
-#define TARGET_NR_statfs64 1056
-#define TARGET_NR_lseek64 1057
-#define TARGET_NR_mmap64 1058
-#define TARGET_NR_alarm 1059
-#define TARGET_NR_getpgrp 1060
-#define TARGET_NR_pause 1061
-#define TARGET_NR_time 1062
-#define TARGET_NR_utime 1063
-#define TARGET_NR_creat 1064
-#define TARGET_NR_getdents 1065
-#define TARGET_NR_futimesat 1066
-#define TARGET_NR_select 1067
-#define TARGET_NR_poll 1068
-#define TARGET_NR_epoll_wait 1069
-#define TARGET_NR_ustat 1070
-#define TARGET_NR_vfork 1071
-#define TARGET_NR_oldwait4 1072
-#define TARGET_NR_recv 1073
-#define TARGET_NR_send 1074
-#define TARGET_NR_bdflush 1075
-#define TARGET_NR_umount 1076
-#define TARGET_NR_uselib 1077
-#define TARGET_NR__sysctl 1078
-#define TARGET_NR_fork 1079
-#define TARGET_NR_syscalls (__NR_fork+1)
+#define TARGET_NR_sched_setattr 274
+#define TARGET_NR_sched_getattr 275
+#define TARGET_NR_renameat2 276
+#define TARGET_NR_seccomp 277
+#define TARGET_NR_getrandom 278
+#define TARGET_NR_memfd_create 279
+#define TARGET_NR_bpf 280
+#define TARGET_NR_execveat 281
+#define TARGET_NR_userfaultfd 282
+#define TARGET_NR_membarrier 283
+#define TARGET_NR_mlock2 284
+#define TARGET_NR_copy_file_range 285
 
-#define TARGET_NR_sigreturn 1999
diff --git a/linux-user/aarch64/target_cpu.h b/linux-user/aarch64/target_cpu.h
index 21560ef..777ce29 100644
--- a/linux-user/aarch64/target_cpu.h
+++ b/linux-user/aarch64/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef AARCH64_TARGET_CPU_H
+#define AARCH64_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUARMState *env, target_ulong newsp)
 {
@@ -32,7 +32,7 @@
     /* Note that AArch64 Linux keeps the TLS pointer in TPIDR; this is
      * different from AArch32 Linux, which uses TPIDRRO.
      */
-    env->cp15.tpidr_el0 = newtls;
+    env->cp15.tpidr_el[0] = newtls;
 }
 
 #endif
diff --git a/linux-user/aarch64/target_signal.h b/linux-user/aarch64/target_signal.h
index e8c677d..e66367c 100644
--- a/linux-user/aarch64/target_signal.h
+++ b/linux-user/aarch64/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef AARCH64_TARGET_SIGNAL_H
+#define AARCH64_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,4 @@
    return state->xregs[31];
 }
 
-#endif /* TARGET_SIGNAL_H */
+#endif /* AARCH64_TARGET_SIGNAL_H */
diff --git a/linux-user/aarch64/target_structs.h b/linux-user/aarch64/target_structs.h
index 21c1f2c..a4998a7 100644
--- a/linux-user/aarch64/target_structs.h
+++ b/linux-user/aarch64/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef AARCH64_TARGET_STRUCTS_H
+#define AARCH64_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/aarch64/target_syscall.h b/linux-user/aarch64/target_syscall.h
new file mode 100644
index 0000000..1b62953
--- /dev/null
+++ b/linux-user/aarch64/target_syscall.h
@@ -0,0 +1,18 @@
+#ifndef AARCH64_TARGET_SYSCALL_H
+#define AARCH64_TARGET_SYSCALL_H
+
+struct target_pt_regs {
+    uint64_t        regs[31];
+    uint64_t        sp;
+    uint64_t        pc;
+    uint64_t        pstate;
+};
+
+#define UNAME_MACHINE "aarch64"
+#define UNAME_MINIMUM_RELEASE "3.8.0"
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ       2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* AARCH64_TARGET_SYSCALL_H */
diff --git a/linux-user/alpha/syscall.h b/linux-user/alpha/syscall.h
deleted file mode 100644
index 245cff2..0000000
--- a/linux-user/alpha/syscall.h
+++ /dev/null
@@ -1,257 +0,0 @@
-/* default linux values for the selectors */
-#define __USER_DS	(1)
-
-struct target_pt_regs {
-	abi_ulong r0;
-	abi_ulong r1;
-	abi_ulong r2;
-	abi_ulong r3;
-	abi_ulong r4;
-	abi_ulong r5;
-	abi_ulong r6;
-	abi_ulong r7;
-	abi_ulong r8;
-	abi_ulong r19;
-	abi_ulong r20;
-	abi_ulong r21;
-	abi_ulong r22;
-	abi_ulong r23;
-	abi_ulong r24;
-	abi_ulong r25;
-	abi_ulong r26;
-	abi_ulong r27;
-	abi_ulong r28;
-	abi_ulong hae;
-/* JRP - These are the values provided to a0-a2 by PALcode */
-	abi_ulong trap_a0;
-	abi_ulong trap_a1;
-	abi_ulong trap_a2;
-/* These are saved by PAL-code: */
-	abi_ulong ps;
-	abi_ulong pc;
-	abi_ulong gp;
-	abi_ulong r16;
-	abi_ulong r17;
-	abi_ulong r18;
-/* Those is needed by qemu to temporary store the user stack pointer */
-        abi_ulong usp;
-        abi_ulong unique;
-};
-
-#define UNAME_MACHINE "alpha"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#undef TARGET_EDEADLK
-#define TARGET_EDEADLK		11
-#undef TARGET_EAGAIN
-#define TARGET_EAGAIN		35
-#undef TARGET_EINPROGRESS
-#define TARGET_EINPROGRESS	36
-#undef TARGET_EALREADY
-#define TARGET_EALREADY		37
-#undef TARGET_ENOTSOCK
-#define TARGET_ENOTSOCK		38
-#undef TARGET_EDESTADDRREQ
-#define TARGET_EDESTADDRREQ	39
-#undef TARGET_EMSGSIZE
-#define TARGET_EMSGSIZE		40
-#undef TARGET_EPROTOTYPE
-#define TARGET_EPROTOTYPE	41
-#undef TARGET_ENOPROTOOPT
-#define TARGET_ENOPROTOOPT	42
-#undef TARGET_EPROTONOSUPPORT
-#define TARGET_EPROTONOSUPPORT	43
-#undef TARGET_ESOCKTNOSUPPORT
-#define TARGET_ESOCKTNOSUPPORT	44
-#undef TARGET_EOPNOTSUPP
-#define TARGET_EOPNOTSUPP	45
-#undef TARGET_EPFNOSUPPORT
-#define TARGET_EPFNOSUPPORT	46
-#undef TARGET_EAFNOSUPPORT
-#define TARGET_EAFNOSUPPORT	47
-#undef TARGET_EADDRINUSE
-#define TARGET_EADDRINUSE	48
-#undef TARGET_EADDRNOTAVAIL
-#define TARGET_EADDRNOTAVAIL	49
-#undef TARGET_ENETDOWN
-#define TARGET_ENETDOWN		50
-#undef TARGET_ENETUNREACH
-#define TARGET_ENETUNREACH	51
-#undef TARGET_ENETRESET
-#define TARGET_ENETRESET	52
-#undef TARGET_ECONNABORTED
-#define TARGET_ECONNABORTED	53
-#undef TARGET_ECONNRESET
-#define TARGET_ECONNRESET	54
-#undef TARGET_ENOBUFS
-#define TARGET_ENOBUFS		55
-#undef TARGET_EISCONN
-#define TARGET_EISCONN		56
-#undef TARGET_ENOTCONN
-#define TARGET_ENOTCONN		57
-#undef TARGET_ESHUTDOWN
-#define TARGET_ESHUTDOWN	58
-#undef TARGET_ETOOMANYREFS
-#define TARGET_ETOOMANYREFS	59
-#undef TARGET_ETIMEDOUT
-#define TARGET_ETIMEDOUT	60
-#undef TARGET_ECONNREFUSED
-#define TARGET_ECONNREFUSED	61
-#undef TARGET_ELOOP
-#define TARGET_ELOOP		62
-#undef TARGET_ENAMETOOLONG
-#define TARGET_ENAMETOOLONG	63
-#undef TARGET_EHOSTDOWN
-#define TARGET_EHOSTDOWN	64
-#undef TARGET_EHOSTUNREACH
-#define TARGET_EHOSTUNREACH	65
-#undef TARGET_ENOTEMPTY
-#define TARGET_ENOTEMPTY	66
-// Unused			67
-#undef TARGET_EUSERS
-#define TARGET_EUSERS		68
-#undef TARGET_EDQUOT
-#define TARGET_EDQUOT		69
-#undef TARGET_ESTALE
-#define TARGET_ESTALE		70
-#undef TARGET_EREMOTE
-#define TARGET_EREMOTE		71
-// Unused			72-76
-#undef TARGET_ENOLCK
-#define TARGET_ENOLCK		77
-#undef TARGET_ENOSYS
-#define TARGET_ENOSYS		78
-// Unused			79
-#undef TARGET_ENOMSG
-#define TARGET_ENOMSG		80
-#undef TARGET_EIDRM
-#define TARGET_EIDRM		81
-#undef TARGET_ENOSR
-#define TARGET_ENOSR		82
-#undef TARGET_ETIME
-#define TARGET_ETIME		83
-#undef TARGET_EBADMSG
-#define TARGET_EBADMSG		84
-#undef TARGET_EPROTO
-#define TARGET_EPROTO		85
-#undef TARGET_ENODATA
-#define TARGET_ENODATA		86
-#undef TARGET_ENOSTR
-#define TARGET_ENOSTR		87
-#undef TARGET_ECHRNG
-#define TARGET_ECHRNG		88
-#undef TARGET_EL2NSYNC
-#define TARGET_EL2NSYNC		89
-#undef TARGET_EL3HLT
-#define TARGET_EL3HLT		90
-#undef TARGET_EL3RST
-#define TARGET_EL3RST		91
-#undef TARGET_ENOPKG
-#define TARGET_ENOPKG		92
-#undef TARGET_ELNRNG
-#define TARGET_ELNRNG		93
-#undef TARGET_EUNATCH
-#define TARGET_EUNATCH		94
-#undef TARGET_ENOCSI
-#define TARGET_ENOCSI		95
-#undef TARGET_EL2HLT
-#define TARGET_EL2HLT		96
-#undef TARGET_EBADE
-#define TARGET_EBADE		97
-#undef TARGET_EBADR
-#define TARGET_EBADR		98
-#undef TARGET_EXFULL
-#define TARGET_EXFULL		99
-#undef TARGET_ENOANO
-#define TARGET_ENOANO		100
-#undef TARGET_EBADRQC
-#define TARGET_EBADRQC		101
-#undef TARGET_EBADSLT
-#define TARGET_EBADSLT		102
-// Unused			103
-#undef TARGET_EBFONT
-#define TARGET_EBFONT		104
-#undef TARGET_ENONET
-#define TARGET_ENONET		105
-#undef TARGET_ENOLINK
-#define TARGET_ENOLINK		106
-#undef TARGET_EADV
-#define TARGET_EADV		107
-#undef TARGET_ESRMNT
-#define TARGET_ESRMNT		108
-#undef TARGET_ECOMM
-#define TARGET_ECOMM		109
-#undef TARGET_EMULTIHOP
-#define TARGET_EMULTIHOP	110
-#undef TARGET_EDOTDOT
-#define TARGET_EDOTDOT		111
-#undef TARGET_EOVERFLOW
-#define TARGET_EOVERFLOW	112
-#undef TARGET_ENOTUNIQ
-#define TARGET_ENOTUNIQ		113
-#undef TARGET_EBADFD
-#define TARGET_EBADFD		114
-#undef TARGET_EREMCHG
-#define TARGET_EREMCHG		115
-#undef TARGET_EILSEQ
-#define TARGET_EILSEQ		116
-
-// Same as default		117-121
-
-#undef TARGET_ELIBACC
-#define TARGET_ELIBACC		122
-#undef TARGET_ELIBBAD
-#define TARGET_ELIBBAD		123
-#undef TARGET_ELIBSCN
-#define TARGET_ELIBSCN		124
-#undef TARGET_ELIBMAX
-#define TARGET_ELIBMAX		125
-#undef TARGET_ELIBEXEC
-#define TARGET_ELIBEXEC		126
-#undef TARGET_ERESTART
-#define TARGET_ERESTART		127
-#undef TARGET_ESTRPIPE
-#define TARGET_ESTRPIPE		128
-#undef TARGET_ENOMEDIUM
-#define TARGET_ENOMEDIUM	129
-#undef TARGET_EMEDIUMTYPE
-#define TARGET_EMEDIUMTYPE	130
-#undef TARGET_ECANCELED
-#define TARGET_ECANCELED	131
-#undef TARGET_ENOKEY
-#define TARGET_ENOKEY		132
-#undef TARGET_EKEYEXPIRED
-#define TARGET_EKEYEXPIRED	133
-#undef TARGET_EKEYREVOKED
-#define TARGET_EKEYREVOKED	134
-#undef TARGET_EKEYREJECTED
-#define TARGET_EKEYREJECTED	135
-#undef TARGET_EOWNERDEAD
-#define TARGET_EOWNERDEAD	136
-#undef TARGET_ENOTRECOVERABLE
-#define TARGET_ENOTRECOVERABLE	137
-#undef TARGET_ERFKILL
-#define TARGET_ERFKILL		138
-
-// For sys_osf_getsysinfo
-#define TARGET_GSI_UACPROC		8
-#define TARGET_GSI_IEEE_FP_CONTROL	45
-#define TARGET_GSI_IEEE_STATE_AT_SIGNAL	46
-#define TARGET_GSI_PROC_TYPE		60
-#define TARGET_GSI_GET_HWRPB		101
-
-// For sys_ofs_setsysinfo
-#define TARGET_SSI_NVPAIRS		1
-#define TARGET_SSI_IEEE_FP_CONTROL	14
-#define TARGET_SSI_IEEE_STATE_AT_SIGNAL	15
-#define TARGET_SSI_IEEE_IGNORE_STATE_AT_SIGNAL 16
-#define TARGET_SSI_IEEE_RAISE_EXCEPTION	1001
-
-#define TARGET_SSIN_UACPROC		6
-
-#define TARGET_UAC_NOPRINT		1
-#define TARGET_UAC_NOFIX		2
-#define TARGET_UAC_SIGBUS		4
-#define TARGET_MINSIGSTKSZ              4096
-#define TARGET_MLOCKALL_MCL_CURRENT     0x2000
-#define TARGET_MLOCKALL_MCL_FUTURE      0x4000
diff --git a/linux-user/alpha/syscall_nr.h b/linux-user/alpha/syscall_nr.h
index 625f301..00e14bb 100644
--- a/linux-user/alpha/syscall_nr.h
+++ b/linux-user/alpha/syscall_nr.h
@@ -185,6 +185,10 @@
 #define TARGET_NR_osf_utsname	207
 #define TARGET_NR_lchown		208
 #define TARGET_NR_osf_shmat		209
+/* this has the usual shmat semantics so give it the name syscall.c expects
+ * so that our support for it is enabled.
+ */
+#define TARGET_NR_shmat TARGET_NR_osf_shmat
 #define TARGET_NR_shmctl		210
 #define TARGET_NR_shmdt		211
 #define TARGET_NR_shmget		212
@@ -440,3 +444,9 @@
 #define TARGET_NR_process_vm_writev             505
 #define TARGET_NR_kcmp                          506
 #define TARGET_NR_finit_module                  507
+#define TARGET_NR_sched_setattr                 508
+#define TARGET_NR_sched_getattr                 509
+#define TARGET_NR_renameat2                     510
+#define TARGET_NR_getrandom                     511
+#define TARGET_NR_memfd_create                  512
+#define TARGET_NR_execveat                      513
diff --git a/linux-user/alpha/target_cpu.h b/linux-user/alpha/target_cpu.h
index 4256245..ad124da 100644
--- a/linux-user/alpha/target_cpu.h
+++ b/linux-user/alpha/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef ALPHA_TARGET_CPU_H
+#define ALPHA_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUAlphaState *env, target_ulong newsp)
 {
diff --git a/linux-user/alpha/target_signal.h b/linux-user/alpha/target_signal.h
index d3822da..f1ed00d 100644
--- a/linux-user/alpha/target_signal.h
+++ b/linux-user/alpha/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef ALPHA_TARGET_SIGNAL_H
+#define ALPHA_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -27,6 +27,7 @@
     return state->ir[IR_SP];
 }
 
+
 /* From <asm/gentrap.h>.  */
 #define TARGET_GEN_INTOVF      -1      /* integer overflow */
 #define TARGET_GEN_INTDIV      -2      /* integer division by zero */
@@ -54,4 +55,4 @@
 #define TARGET_GEN_SUBRNG6     -24
 #define TARGET_GEN_SUBRNG7     -25
 
-#endif /* TARGET_SIGNAL_H */
+#endif /* ALPHA_TARGET_SIGNAL_H */
diff --git a/linux-user/alpha/target_structs.h b/linux-user/alpha/target_structs.h
index 50e7708..db2bfe2 100644
--- a/linux-user/alpha/target_structs.h
+++ b/linux-user/alpha/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef ALPHA_TARGET_STRUCTS_H
+#define ALPHA_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/alpha/target_syscall.h b/linux-user/alpha/target_syscall.h
new file mode 100644
index 0000000..b580fc5
--- /dev/null
+++ b/linux-user/alpha/target_syscall.h
@@ -0,0 +1,262 @@
+#ifndef ALPHA_TARGET_SYSCALL_H
+#define ALPHA_TARGET_SYSCALL_H
+
+/* default linux values for the selectors */
+#define __USER_DS	(1)
+
+struct target_pt_regs {
+	abi_ulong r0;
+	abi_ulong r1;
+	abi_ulong r2;
+	abi_ulong r3;
+	abi_ulong r4;
+	abi_ulong r5;
+	abi_ulong r6;
+	abi_ulong r7;
+	abi_ulong r8;
+	abi_ulong r19;
+	abi_ulong r20;
+	abi_ulong r21;
+	abi_ulong r22;
+	abi_ulong r23;
+	abi_ulong r24;
+	abi_ulong r25;
+	abi_ulong r26;
+	abi_ulong r27;
+	abi_ulong r28;
+	abi_ulong hae;
+/* JRP - These are the values provided to a0-a2 by PALcode */
+	abi_ulong trap_a0;
+	abi_ulong trap_a1;
+	abi_ulong trap_a2;
+/* These are saved by PAL-code: */
+	abi_ulong ps;
+	abi_ulong pc;
+	abi_ulong gp;
+	abi_ulong r16;
+	abi_ulong r17;
+	abi_ulong r18;
+/* Those is needed by qemu to temporary store the user stack pointer */
+        abi_ulong usp;
+        abi_ulong unique;
+};
+
+#define UNAME_MACHINE "alpha"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#undef TARGET_EDEADLK
+#define TARGET_EDEADLK		11
+#undef TARGET_EAGAIN
+#define TARGET_EAGAIN		35
+#undef TARGET_EINPROGRESS
+#define TARGET_EINPROGRESS	36
+#undef TARGET_EALREADY
+#define TARGET_EALREADY		37
+#undef TARGET_ENOTSOCK
+#define TARGET_ENOTSOCK		38
+#undef TARGET_EDESTADDRREQ
+#define TARGET_EDESTADDRREQ	39
+#undef TARGET_EMSGSIZE
+#define TARGET_EMSGSIZE		40
+#undef TARGET_EPROTOTYPE
+#define TARGET_EPROTOTYPE	41
+#undef TARGET_ENOPROTOOPT
+#define TARGET_ENOPROTOOPT	42
+#undef TARGET_EPROTONOSUPPORT
+#define TARGET_EPROTONOSUPPORT	43
+#undef TARGET_ESOCKTNOSUPPORT
+#define TARGET_ESOCKTNOSUPPORT	44
+#undef TARGET_EOPNOTSUPP
+#define TARGET_EOPNOTSUPP	45
+#undef TARGET_EPFNOSUPPORT
+#define TARGET_EPFNOSUPPORT	46
+#undef TARGET_EAFNOSUPPORT
+#define TARGET_EAFNOSUPPORT	47
+#undef TARGET_EADDRINUSE
+#define TARGET_EADDRINUSE	48
+#undef TARGET_EADDRNOTAVAIL
+#define TARGET_EADDRNOTAVAIL	49
+#undef TARGET_ENETDOWN
+#define TARGET_ENETDOWN		50
+#undef TARGET_ENETUNREACH
+#define TARGET_ENETUNREACH	51
+#undef TARGET_ENETRESET
+#define TARGET_ENETRESET	52
+#undef TARGET_ECONNABORTED
+#define TARGET_ECONNABORTED	53
+#undef TARGET_ECONNRESET
+#define TARGET_ECONNRESET	54
+#undef TARGET_ENOBUFS
+#define TARGET_ENOBUFS		55
+#undef TARGET_EISCONN
+#define TARGET_EISCONN		56
+#undef TARGET_ENOTCONN
+#define TARGET_ENOTCONN		57
+#undef TARGET_ESHUTDOWN
+#define TARGET_ESHUTDOWN	58
+#undef TARGET_ETOOMANYREFS
+#define TARGET_ETOOMANYREFS	59
+#undef TARGET_ETIMEDOUT
+#define TARGET_ETIMEDOUT	60
+#undef TARGET_ECONNREFUSED
+#define TARGET_ECONNREFUSED	61
+#undef TARGET_ELOOP
+#define TARGET_ELOOP		62
+#undef TARGET_ENAMETOOLONG
+#define TARGET_ENAMETOOLONG	63
+#undef TARGET_EHOSTDOWN
+#define TARGET_EHOSTDOWN	64
+#undef TARGET_EHOSTUNREACH
+#define TARGET_EHOSTUNREACH	65
+#undef TARGET_ENOTEMPTY
+#define TARGET_ENOTEMPTY	66
+// Unused			67
+#undef TARGET_EUSERS
+#define TARGET_EUSERS		68
+#undef TARGET_EDQUOT
+#define TARGET_EDQUOT		69
+#undef TARGET_ESTALE
+#define TARGET_ESTALE		70
+#undef TARGET_EREMOTE
+#define TARGET_EREMOTE		71
+// Unused			72-76
+#undef TARGET_ENOLCK
+#define TARGET_ENOLCK		77
+#undef TARGET_ENOSYS
+#define TARGET_ENOSYS		78
+// Unused			79
+#undef TARGET_ENOMSG
+#define TARGET_ENOMSG		80
+#undef TARGET_EIDRM
+#define TARGET_EIDRM		81
+#undef TARGET_ENOSR
+#define TARGET_ENOSR		82
+#undef TARGET_ETIME
+#define TARGET_ETIME		83
+#undef TARGET_EBADMSG
+#define TARGET_EBADMSG		84
+#undef TARGET_EPROTO
+#define TARGET_EPROTO		85
+#undef TARGET_ENODATA
+#define TARGET_ENODATA		86
+#undef TARGET_ENOSTR
+#define TARGET_ENOSTR		87
+#undef TARGET_ECHRNG
+#define TARGET_ECHRNG		88
+#undef TARGET_EL2NSYNC
+#define TARGET_EL2NSYNC		89
+#undef TARGET_EL3HLT
+#define TARGET_EL3HLT		90
+#undef TARGET_EL3RST
+#define TARGET_EL3RST		91
+#undef TARGET_ENOPKG
+#define TARGET_ENOPKG		92
+#undef TARGET_ELNRNG
+#define TARGET_ELNRNG		93
+#undef TARGET_EUNATCH
+#define TARGET_EUNATCH		94
+#undef TARGET_ENOCSI
+#define TARGET_ENOCSI		95
+#undef TARGET_EL2HLT
+#define TARGET_EL2HLT		96
+#undef TARGET_EBADE
+#define TARGET_EBADE		97
+#undef TARGET_EBADR
+#define TARGET_EBADR		98
+#undef TARGET_EXFULL
+#define TARGET_EXFULL		99
+#undef TARGET_ENOANO
+#define TARGET_ENOANO		100
+#undef TARGET_EBADRQC
+#define TARGET_EBADRQC		101
+#undef TARGET_EBADSLT
+#define TARGET_EBADSLT		102
+// Unused			103
+#undef TARGET_EBFONT
+#define TARGET_EBFONT		104
+#undef TARGET_ENONET
+#define TARGET_ENONET		105
+#undef TARGET_ENOLINK
+#define TARGET_ENOLINK		106
+#undef TARGET_EADV
+#define TARGET_EADV		107
+#undef TARGET_ESRMNT
+#define TARGET_ESRMNT		108
+#undef TARGET_ECOMM
+#define TARGET_ECOMM		109
+#undef TARGET_EMULTIHOP
+#define TARGET_EMULTIHOP	110
+#undef TARGET_EDOTDOT
+#define TARGET_EDOTDOT		111
+#undef TARGET_EOVERFLOW
+#define TARGET_EOVERFLOW	112
+#undef TARGET_ENOTUNIQ
+#define TARGET_ENOTUNIQ		113
+#undef TARGET_EBADFD
+#define TARGET_EBADFD		114
+#undef TARGET_EREMCHG
+#define TARGET_EREMCHG		115
+#undef TARGET_EILSEQ
+#define TARGET_EILSEQ		116
+
+// Same as default		117-121
+
+#undef TARGET_ELIBACC
+#define TARGET_ELIBACC		122
+#undef TARGET_ELIBBAD
+#define TARGET_ELIBBAD		123
+#undef TARGET_ELIBSCN
+#define TARGET_ELIBSCN		124
+#undef TARGET_ELIBMAX
+#define TARGET_ELIBMAX		125
+#undef TARGET_ELIBEXEC
+#define TARGET_ELIBEXEC		126
+#undef TARGET_ERESTART
+#define TARGET_ERESTART		127
+#undef TARGET_ESTRPIPE
+#define TARGET_ESTRPIPE		128
+#undef TARGET_ENOMEDIUM
+#define TARGET_ENOMEDIUM	129
+#undef TARGET_EMEDIUMTYPE
+#define TARGET_EMEDIUMTYPE	130
+#undef TARGET_ECANCELED
+#define TARGET_ECANCELED	131
+#undef TARGET_ENOKEY
+#define TARGET_ENOKEY		132
+#undef TARGET_EKEYEXPIRED
+#define TARGET_EKEYEXPIRED	133
+#undef TARGET_EKEYREVOKED
+#define TARGET_EKEYREVOKED	134
+#undef TARGET_EKEYREJECTED
+#define TARGET_EKEYREJECTED	135
+#undef TARGET_EOWNERDEAD
+#define TARGET_EOWNERDEAD	136
+#undef TARGET_ENOTRECOVERABLE
+#define TARGET_ENOTRECOVERABLE	137
+#undef TARGET_ERFKILL
+#define TARGET_ERFKILL		138
+
+// For sys_osf_getsysinfo
+#define TARGET_GSI_UACPROC		8
+#define TARGET_GSI_IEEE_FP_CONTROL	45
+#define TARGET_GSI_IEEE_STATE_AT_SIGNAL	46
+#define TARGET_GSI_PROC_TYPE		60
+#define TARGET_GSI_GET_HWRPB		101
+
+// For sys_ofs_setsysinfo
+#define TARGET_SSI_NVPAIRS		1
+#define TARGET_SSI_IEEE_FP_CONTROL	14
+#define TARGET_SSI_IEEE_STATE_AT_SIGNAL	15
+#define TARGET_SSI_IEEE_IGNORE_STATE_AT_SIGNAL 16
+#define TARGET_SSI_IEEE_RAISE_EXCEPTION	1001
+
+#define TARGET_SSIN_UACPROC		6
+
+#define TARGET_UAC_NOPRINT		1
+#define TARGET_UAC_NOFIX		2
+#define TARGET_UAC_SIGBUS		4
+#define TARGET_MINSIGSTKSZ              4096
+#define TARGET_MLOCKALL_MCL_CURRENT     0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE      0x4000
+
+#endif /* ALPHA_TARGET_SYSCALL_H */
diff --git a/linux-user/arm/nwfpe/double_cpdo.c b/linux-user/arm/nwfpe/double_cpdo.c
index 41c28f3..1cef380 100644
--- a/linux-user/arm/nwfpe/double_cpdo.c
+++ b/linux-user/arm/nwfpe/double_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/linux-user/arm/nwfpe/extended_cpdo.c b/linux-user/arm/nwfpe/extended_cpdo.c
index 48eca3b..1c8a412 100644
--- a/linux-user/arm/nwfpe/extended_cpdo.c
+++ b/linux-user/arm/nwfpe/extended_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/linux-user/arm/nwfpe/fpa11.c b/linux-user/arm/nwfpe/fpa11.c
index eebd93f..441e3b1 100644
--- a/linux-user/arm/nwfpe/fpa11.c
+++ b/linux-user/arm/nwfpe/fpa11.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 
 #include "fpopcode.h"
@@ -27,7 +28,6 @@
 
 //#include <asm/system.h>
 
-#include <stdio.h>
 
 FPA11* qemufpa = NULL;
 CPUARMState* user_registers;
diff --git a/linux-user/arm/nwfpe/fpa11.h b/linux-user/arm/nwfpe/fpa11.h
index bb9ac65..d459c5d 100644
--- a/linux-user/arm/nwfpe/fpa11.h
+++ b/linux-user/arm/nwfpe/fpa11.h
@@ -18,14 +18,10 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
-#ifndef __FPA11_H__
-#define __FPA11_H__
+#ifndef FPA11_H
+#define FPA11_H
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-
-#include <cpu.h>
+#include "cpu.h"
 
 #define GET_FPA11() (qemufpa)
 
@@ -108,7 +104,7 @@
 
 static inline void writeConditionCodes(unsigned int x)
 {
-        cpsr_write(user_registers,x,CPSR_NZCV);
+    cpsr_write(user_registers, x, CPSR_NZCV, CPSRWriteByInstr);
 }
 
 #define ARM_REG_PC 15
diff --git a/linux-user/arm/nwfpe/fpa11_cpdo.c b/linux-user/arm/nwfpe/fpa11_cpdo.c
index 5f4a6a4..94ac98a 100644
--- a/linux-user/arm/nwfpe/fpa11_cpdo.c
+++ b/linux-user/arm/nwfpe/fpa11_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpopcode.h"
 
diff --git a/linux-user/arm/nwfpe/fpa11_cpdt.c b/linux-user/arm/nwfpe/fpa11_cpdt.c
index 007a3d6..c32b0c2 100644
--- a/linux-user/arm/nwfpe/fpa11_cpdt.c
+++ b/linux-user/arm/nwfpe/fpa11_cpdt.c
@@ -19,6 +19,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/linux-user/arm/nwfpe/fpa11_cprt.c b/linux-user/arm/nwfpe/fpa11_cprt.c
index 7be93fa..04dc2eb 100644
--- a/linux-user/arm/nwfpe/fpa11_cprt.c
+++ b/linux-user/arm/nwfpe/fpa11_cprt.c
@@ -19,6 +19,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/linux-user/arm/nwfpe/fpopcode.c b/linux-user/arm/nwfpe/fpopcode.c
index 0dc5c9c..6784256 100644
--- a/linux-user/arm/nwfpe/fpopcode.c
+++ b/linux-user/arm/nwfpe/fpopcode.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
@@ -88,25 +89,3 @@
 
   return(nRc);
 }
-
-/* condition code lookup table
- index into the table is test code: EQ, NE, ... LT, GT, AL, NV
- bit position in short is condition code: NZCV */
-static const unsigned short aCC[16] = {
-    0xF0F0, // EQ == Z set
-    0x0F0F, // NE
-    0xCCCC, // CS == C set
-    0x3333, // CC
-    0xFF00, // MI == N set
-    0x00FF, // PL
-    0xAAAA, // VS == V set
-    0x5555, // VC
-    0x0C0C, // HI == C set && Z clear
-    0xF3F3, // LS == C clear || Z set
-    0xAA55, // GE == (N==V)
-    0x55AA, // LT == (N!=V)
-    0x0A05, // GT == (!Z && (N==V))
-    0xF5FA, // LE == (Z || (N!=V))
-    0xFFFF, // AL always
-    0 // NV
-};
diff --git a/linux-user/arm/nwfpe/fpopcode.h b/linux-user/arm/nwfpe/fpopcode.h
index 1b1137f..06cd909 100644
--- a/linux-user/arm/nwfpe/fpopcode.h
+++ b/linux-user/arm/nwfpe/fpopcode.h
@@ -18,8 +18,8 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
-#ifndef __FPOPCODE_H__
-#define __FPOPCODE_H__
+#ifndef FPOPCODE_H
+#define FPOPCODE_H
 
 /*
 ARM Floating Point Instruction Classes
diff --git a/linux-user/arm/nwfpe/fpsr.h b/linux-user/arm/nwfpe/fpsr.h
index 859dcd5..8c978f0 100644
--- a/linux-user/arm/nwfpe/fpsr.h
+++ b/linux-user/arm/nwfpe/fpsr.h
@@ -18,8 +18,8 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
-#ifndef __FPSR_H__
-#define __FPSR_H__
+#ifndef FPSR_H
+#define FPSR_H
 
 /*
 The FPSR is a 32 bit register consisting of 4 parts, each exactly
diff --git a/linux-user/arm/nwfpe/single_cpdo.c b/linux-user/arm/nwfpe/single_cpdo.c
index 2bfb359..21e177b 100644
--- a/linux-user/arm/nwfpe/single_cpdo.c
+++ b/linux-user/arm/nwfpe/single_cpdo.c
@@ -18,6 +18,7 @@
     along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#include "qemu/osdep.h"
 #include "fpa11.h"
 #include "fpu/softfloat.h"
 #include "fpopcode.h"
diff --git a/linux-user/arm/syscall.h b/linux-user/arm/syscall.h
deleted file mode 100644
index 3844a96..0000000
--- a/linux-user/arm/syscall.h
+++ /dev/null
@@ -1,50 +0,0 @@
-
-/* this struct defines the way the registers are stored on the
-   stack during a system call. */
-
-struct target_pt_regs {
-    abi_long uregs[18];
-};
-
-#define ARM_cpsr	uregs[16]
-#define ARM_pc		uregs[15]
-#define ARM_lr		uregs[14]
-#define ARM_sp		uregs[13]
-#define ARM_ip		uregs[12]
-#define ARM_fp		uregs[11]
-#define ARM_r10		uregs[10]
-#define ARM_r9		uregs[9]
-#define ARM_r8		uregs[8]
-#define ARM_r7		uregs[7]
-#define ARM_r6		uregs[6]
-#define ARM_r5		uregs[5]
-#define ARM_r4		uregs[4]
-#define ARM_r3		uregs[3]
-#define ARM_r2		uregs[2]
-#define ARM_r1		uregs[1]
-#define ARM_r0		uregs[0]
-#define ARM_ORIG_r0	uregs[17]
-
-#define ARM_SYSCALL_BASE	0x900000
-#define ARM_THUMB_SYSCALL	0
-
-#define ARM_NR_BASE	  0xf0000
-#define ARM_NR_breakpoint (ARM_NR_BASE + 1)
-#define ARM_NR_cacheflush (ARM_NR_BASE + 2)
-#define ARM_NR_set_tls	  (ARM_NR_BASE + 5)
-
-#define ARM_NR_semihosting	  0x123456
-#define ARM_NR_thumb_semihosting  0xAB
-
-#if defined(TARGET_WORDS_BIGENDIAN)
-#define UNAME_MACHINE "armv5teb"
-#else
-#define UNAME_MACHINE "armv5tel"
-#endif
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_CLONE_BACKWARDS
-
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/arm/syscall_nr.h b/linux-user/arm/syscall_nr.h
index 7d7be7c..cc9089c 100644
--- a/linux-user/arm/syscall_nr.h
+++ b/linux-user/arm/syscall_nr.h
@@ -354,7 +354,7 @@
 #define TARGET_NR_kexec_load			(347)
 #define TARGET_NR_utimensat			(348)
 #define TARGET_NR_signalfd			(349)
-#define TARGET_NR_timerfd			(350)
+#define TARGET_NR_timerfd_create		(350)
 #define TARGET_NR_eventfd			(351)
 #define TARGET_NR_fallocate			(352)
 #define TARGET_NR_timerfd_settime		(353)
@@ -384,3 +384,15 @@
 #define TARGET_NR_process_vm_writev            (377)
 #define TARGET_NR_kcmp                         (378)
 #define TARGET_NR_finit_module                 (379)
+
+#define TARGET_NR_sched_setattr                (380)
+#define TARGET_NR_sched_getattr                (381)
+#define TARGET_NR_renameat2                    (382)
+#define TARGET_NR_seccomp                      (383)
+#define TARGET_NR_getrandom                    (384)
+#define TARGET_NR_memfd_create                 (385)
+#define TARGET_NR_bpf                          (386)
+#define TARGET_NR_execveat                     (387)
+#define TARGET_NR_userfaultfd                  (388)
+#define TARGET_NR_membarrier                   (389)
+#define TARGET_NR_mlock2                       (390)
diff --git a/linux-user/arm/target_cpu.h b/linux-user/arm/target_cpu.h
index 39d65b6..d888219 100644
--- a/linux-user/arm/target_cpu.h
+++ b/linux-user/arm/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef ARM_TARGET_CPU_H
+#define ARM_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUARMState *env, target_ulong newsp)
 {
@@ -29,7 +29,20 @@
 
 static inline void cpu_set_tls(CPUARMState *env, target_ulong newtls)
 {
-    env->cp15.tpidrro_el0 = newtls;
+    if (access_secure_reg(env)) {
+        env->cp15.tpidruro_s = newtls;
+    } else {
+        env->cp15.tpidrro_el[0] = newtls;
+    }
+}
+
+static inline target_ulong cpu_get_tls(CPUARMState *env)
+{
+    if (access_secure_reg(env)) {
+        return env->cp15.tpidruro_s;
+    } else {
+        return env->cp15.tpidrro_el[0];
+    }
 }
 
 #endif
diff --git a/linux-user/arm/target_signal.h b/linux-user/arm/target_signal.h
index 2b32813..cbbeb09 100644
--- a/linux-user/arm/target_signal.h
+++ b/linux-user/arm/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef ARM_TARGET_SIGNAL_H
+#define ARM_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
    return state->regs[13];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* ARM_TARGET_SIGNAL_H */
diff --git a/linux-user/arm/target_structs.h b/linux-user/arm/target_structs.h
index f3c85d4..0bf034c 100644
--- a/linux-user/arm/target_structs.h
+++ b/linux-user/arm/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef ARM_TARGET_STRUCTS_H
+#define ARM_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/arm/target_syscall.h b/linux-user/arm/target_syscall.h
new file mode 100644
index 0000000..cd021ff
--- /dev/null
+++ b/linux-user/arm/target_syscall.h
@@ -0,0 +1,36 @@
+#ifndef ARM_TARGET_SYSCALL_H
+#define ARM_TARGET_SYSCALL_H
+
+/* this struct defines the way the registers are stored on the
+   stack during a system call. */
+
+/* uregs[0..15] are r0 to r15; uregs[16] is CPSR; uregs[17] is ORIG_r0 */
+struct target_pt_regs {
+    abi_long uregs[18];
+};
+
+#define ARM_SYSCALL_BASE	0x900000
+#define ARM_THUMB_SYSCALL	0
+
+#define ARM_NR_BASE	  0xf0000
+#define ARM_NR_breakpoint (ARM_NR_BASE + 1)
+#define ARM_NR_cacheflush (ARM_NR_BASE + 2)
+#define ARM_NR_set_tls	  (ARM_NR_BASE + 5)
+
+#define ARM_NR_semihosting	  0x123456
+#define ARM_NR_thumb_semihosting  0xAB
+
+#if defined(TARGET_WORDS_BIGENDIAN)
+#define UNAME_MACHINE "armv5teb"
+#else
+#define UNAME_MACHINE "armv5tel"
+#endif
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_CLONE_BACKWARDS
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* ARM_TARGET_SYSCALL_H */
diff --git a/linux-user/cris/syscall.h b/linux-user/cris/syscall.h
deleted file mode 100644
index 2957b0d..0000000
--- a/linux-user/cris/syscall.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#ifndef CRIS_SYSCALL_H
-#define CRIS_SYSCALL_H 1
-
-#define UNAME_MACHINE "cris"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-/* pt_regs not only specifices the format in the user-struct during
- * ptrace but is also the frame format used in the kernel prologue/epilogues
- * themselves
- */
-
-struct target_pt_regs {
-        unsigned long orig_r10;
-        /* pushed by movem r13, [sp] in SAVE_ALL. */
-        unsigned long r0;
-        unsigned long r1;
-        unsigned long r2;
-        unsigned long r3;
-        unsigned long r4;
-        unsigned long r5;
-        unsigned long r6;
-        unsigned long r7;
-        unsigned long r8;
-        unsigned long r9;
-        unsigned long r10;
-        unsigned long r11;
-        unsigned long r12;
-        unsigned long r13;
-        unsigned long acr;
-        unsigned long srs;
-        unsigned long mof;
-        unsigned long spc;
-        unsigned long ccs;
-        unsigned long srp;
-        unsigned long erp; /* This is actually the debugged process' PC */
-        /* For debugging purposes; saved only when needed. */
-        unsigned long exs;
-        unsigned long eda;
-};
-
-#define TARGET_CLONE_BACKWARDS2
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
-
-#endif
diff --git a/linux-user/cris/syscall_nr.h b/linux-user/cris/syscall_nr.h
index 694bd02..44f0b64 100644
--- a/linux-user/cris/syscall_nr.h
+++ b/linux-user/cris/syscall_nr.h
@@ -336,3 +336,27 @@
 #define TARGET_NR_preadv             333
 #define TARGET_NR_pwritev            334
 #define TARGET_NR_setns              335
+#define TARGET_NR_name_to_handle_at  336
+#define TARGET_NR_open_by_handle_at  337
+#define TARGET_NR_rt_tgsigqueueinfo  338
+#define TARGET_NR_perf_event_open    339
+#define TARGET_NR_recvmmsg           340
+#define TARGET_NR_accept4            341
+#define TARGET_NR_fanotify_init      342
+#define TARGET_NR_fanotify_mark      343
+#define TARGET_NR_prlimit64          344
+#define TARGET_NR_clock_adjtime      345
+#define TARGET_NR_syncfs             346
+#define TARGET_NR_sendmmsg           347
+#define TARGET_NR_process_vm_readv   348
+#define TARGET_NR_process_vm_writev  349
+#define TARGET_NR_kcmp               350
+#define TARGET_NR_finit_module       351
+#define TARGET_NR_sched_setattr      352
+#define TARGET_NR_sched_getattr      353
+#define TARGET_NR_renameat2          354
+#define TARGET_NR_seccomp            355
+#define TARGET_NR_getrandom          356
+#define TARGET_NR_memfd_create       357
+#define TARGET_NR_bpf                358
+#define TARGET_NR_execveat           359
diff --git a/linux-user/cris/target_cpu.h b/linux-user/cris/target_cpu.h
index 4d787e5..c43aac6 100644
--- a/linux-user/cris/target_cpu.h
+++ b/linux-user/cris/target_cpu.h
@@ -17,8 +17,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef CRIS_TARGET_CPU_H
+#define CRIS_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUCRISState *env, target_ulong newsp)
 {
diff --git a/linux-user/cris/target_signal.h b/linux-user/cris/target_signal.h
index 5611840..664621b 100644
--- a/linux-user/cris/target_signal.h
+++ b/linux-user/cris/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef CRIS_TARGET_SIGNAL_H
+#define CRIS_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
     return state->regs[14];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* CRIS_TARGET_SIGNAL_H */
diff --git a/linux-user/cris/target_structs.h b/linux-user/cris/target_structs.h
index e4a1ffb..76f9653 100644
--- a/linux-user/cris/target_structs.h
+++ b/linux-user/cris/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef CRIS_TARGET_STRUCTS_H
+#define CRIS_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/cris/target_syscall.h b/linux-user/cris/target_syscall.h
new file mode 100644
index 0000000..29d6900
--- /dev/null
+++ b/linux-user/cris/target_syscall.h
@@ -0,0 +1,46 @@
+#ifndef CRIS_TARGET_SYSCALL_H
+#define CRIS_TARGET_SYSCALL_H
+
+#define UNAME_MACHINE "cris"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+/* pt_regs not only specifices the format in the user-struct during
+ * ptrace but is also the frame format used in the kernel prologue/epilogues
+ * themselves
+ */
+
+struct target_pt_regs {
+        unsigned long orig_r10;
+        /* pushed by movem r13, [sp] in SAVE_ALL. */
+        unsigned long r0;
+        unsigned long r1;
+        unsigned long r2;
+        unsigned long r3;
+        unsigned long r4;
+        unsigned long r5;
+        unsigned long r6;
+        unsigned long r7;
+        unsigned long r8;
+        unsigned long r9;
+        unsigned long r10;
+        unsigned long r11;
+        unsigned long r12;
+        unsigned long r13;
+        unsigned long acr;
+        unsigned long srs;
+        unsigned long mof;
+        unsigned long spc;
+        unsigned long ccs;
+        unsigned long srp;
+        unsigned long erp; /* This is actually the debugged process' PC */
+        /* For debugging purposes; saved only when needed. */
+        unsigned long exs;
+        unsigned long eda;
+};
+
+#define TARGET_CLONE_BACKWARDS2
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index e2596a4..f807baf 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -1,20 +1,12 @@
 /* This is the Linux kernel elf-loading code, ported into user space */
-#include <sys/time.h>
+#include "qemu/osdep.h"
 #include <sys/param.h>
 
-#include <stdio.h>
-#include <sys/types.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-#include <sys/mman.h>
 #include <sys/resource.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
 
 #include "qemu.h"
 #include "disas/disas.h"
+#include "qemu/path.h"
 
 #ifdef _ARCH_PPC64
 #undef ARCH_DLINFO
@@ -145,7 +137,6 @@
 
 #ifdef TARGET_X86_64
 #define ELF_START_MMAP 0x2aaaaab000ULL
-#define elf_check_arch(x) ( ((x) == ELF_ARCH) )
 
 #define ELF_CLASS      ELFCLASS64
 #define ELF_ARCH       EM_X86_64
@@ -273,9 +264,7 @@
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ((x) == ELF_MACHINE)
-
-#define ELF_ARCH        ELF_MACHINE
+#define ELF_ARCH        EM_ARM
 #define ELF_CLASS       ELFCLASS32
 
 static inline void init_thread(struct target_pt_regs *regs,
@@ -284,19 +273,20 @@
     abi_long stack = infop->start_stack;
     memset(regs, 0, sizeof(*regs));
 
-    regs->ARM_cpsr = 0x10;
-    if (infop->entry & 1)
-        regs->ARM_cpsr |= CPSR_T;
-    regs->ARM_pc = infop->entry & 0xfffffffe;
-    regs->ARM_sp = infop->start_stack;
+    regs->uregs[16] = ARM_CPU_MODE_USR;
+    if (infop->entry & 1) {
+        regs->uregs[16] |= CPSR_T;
+    }
+    regs->uregs[15] = infop->entry & 0xfffffffe;
+    regs->uregs[13] = infop->start_stack;
     /* FIXME - what to for failure of get_user()? */
-    get_user_ual(regs->ARM_r2, stack + 8); /* envp */
-    get_user_ual(regs->ARM_r1, stack + 4); /* envp */
+    get_user_ual(regs->uregs[2], stack + 8); /* envp */
+    get_user_ual(regs->uregs[1], stack + 4); /* envp */
     /* XXX: it seems that r0 is zeroed after ! */
-    regs->ARM_r0 = 0;
+    regs->uregs[0] = 0;
     /* For uClinux PIC binaries.  */
     /* XXX: Linux does this only on ARM with no MMU (do we care ?) */
-    regs->ARM_r10 = infop->start_data;
+    regs->uregs[10] = infop->start_data;
 }
 
 #define ELF_NREG    18
@@ -481,9 +471,7 @@
 /* 64 bit ARM definitions */
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ((x) == ELF_MACHINE)
-
-#define ELF_ARCH        ELF_MACHINE
+#define ELF_ARCH        EM_AARCH64
 #define ELF_CLASS       ELFCLASS64
 #define ELF_PLATFORM    "aarch64"
 
@@ -556,8 +544,6 @@
 
 #define ELF_START_MMAP          0x80000000
 
-#define elf_check_arch(x)       ((x) == EM_UNICORE32)
-
 #define ELF_CLASS               ELFCLASS32
 #define ELF_DATA                ELFDATA2LSB
 #define ELF_ARCH                EM_UNICORE32
@@ -666,7 +652,6 @@
 #define ELF_START_MMAP 0x80000000
 #define ELF_HWCAP  (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | HWCAP_SPARC_SWAP \
                     | HWCAP_SPARC_MULDIV)
-#define elf_check_arch(x) ( (x) == EM_SPARC )
 
 #define ELF_CLASS   ELFCLASS32
 #define ELF_ARCH    EM_SPARC
@@ -686,6 +671,7 @@
 
 #ifdef TARGET_PPC
 
+#define ELF_MACHINE    PPC_ELF_MACHINE
 #define ELF_START_MMAP 0x80000000
 
 #if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
@@ -696,8 +682,6 @@
 
 #else
 
-#define elf_check_arch(x) ( (x) == EM_PPC )
-
 #define ELF_CLASS       ELFCLASS32
 
 #endif
@@ -829,8 +813,11 @@
     _regs->gpr[1] = infop->start_stack;
 #if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
     if (get_ppc64_abi(infop) < 2) {
-        _regs->gpr[2] = ldq_raw(infop->entry + 8) + infop->load_bias;
-        infop->entry = ldq_raw(infop->entry) + infop->load_bias;
+        uint64_t val;
+        get_user_u64(val, infop->entry + 8);
+        _regs->gpr[2] = val + infop->load_bias;
+        get_user_u64(val, infop->entry);
+        infop->entry = val + infop->load_bias;
     } else {
         _regs->gpr[12] = infop->entry;  /* r12 set to global entry address */
     }
@@ -872,8 +859,6 @@
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_MIPS )
-
 #ifdef TARGET_MIPS64
 #define ELF_CLASS   ELFCLASS64
 #else
@@ -982,8 +967,6 @@
 
 #define ELF_START_MMAP 0x08000000
 
-#define elf_check_arch(x) ((x) == EM_OPENRISC)
-
 #define ELF_ARCH EM_OPENRISC
 #define ELF_CLASS ELFCLASS32
 #define ELF_DATA  ELFDATA2MSB
@@ -1023,8 +1006,6 @@
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_SH )
-
 #define ELF_CLASS ELFCLASS32
 #define ELF_ARCH  EM_SH
 
@@ -1072,14 +1053,41 @@
 #define USE_ELF_CORE_DUMP
 #define ELF_EXEC_PAGESIZE        4096
 
+enum {
+    SH_CPU_HAS_FPU            = 0x0001, /* Hardware FPU support */
+    SH_CPU_HAS_P2_FLUSH_BUG   = 0x0002, /* Need to flush the cache in P2 area */
+    SH_CPU_HAS_MMU_PAGE_ASSOC = 0x0004, /* SH3: TLB way selection bit support */
+    SH_CPU_HAS_DSP            = 0x0008, /* SH-DSP: DSP support */
+    SH_CPU_HAS_PERF_COUNTER   = 0x0010, /* Hardware performance counters */
+    SH_CPU_HAS_PTEA           = 0x0020, /* PTEA register */
+    SH_CPU_HAS_LLSC           = 0x0040, /* movli.l/movco.l */
+    SH_CPU_HAS_L2_CACHE       = 0x0080, /* Secondary cache / URAM */
+    SH_CPU_HAS_OP32           = 0x0100, /* 32-bit instruction support */
+    SH_CPU_HAS_PTEAEX         = 0x0200, /* PTE ASID Extension support */
+};
+
+#define ELF_HWCAP get_elf_hwcap()
+
+static uint32_t get_elf_hwcap(void)
+{
+    SuperHCPU *cpu = SUPERH_CPU(thread_cpu);
+    uint32_t hwcap = 0;
+
+    hwcap |= SH_CPU_HAS_FPU;
+
+    if (cpu->env.features & SH_FEATURE_SH4A) {
+        hwcap |= SH_CPU_HAS_LLSC;
+    }
+
+    return hwcap;
+}
+
 #endif
 
 #ifdef TARGET_CRIS
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_CRIS )
-
 #define ELF_CLASS ELFCLASS32
 #define ELF_ARCH  EM_CRIS
 
@@ -1097,8 +1105,6 @@
 
 #define ELF_START_MMAP 0x80000000
 
-#define elf_check_arch(x) ( (x) == EM_68K )
-
 #define ELF_CLASS       ELFCLASS32
 #define ELF_ARCH        EM_68K
 
@@ -1150,8 +1156,6 @@
 
 #define ELF_START_MMAP (0x30000000000ULL)
 
-#define elf_check_arch(x) ( (x) == ELF_ARCH )
-
 #define ELF_CLASS      ELFCLASS64
 #define ELF_ARCH       EM_ALPHA
 
@@ -1171,8 +1175,6 @@
 
 #define ELF_START_MMAP (0x20000000000ULL)
 
-#define elf_check_arch(x) ( (x) == ELF_ARCH )
-
 #define ELF_CLASS	ELFCLASS64
 #define ELF_DATA	ELFDATA2MSB
 #define ELF_ARCH	EM_S390
@@ -1186,10 +1188,41 @@
 
 #endif /* TARGET_S390X */
 
+#ifdef TARGET_TILEGX
+
+/* 42 bits real used address, a half for user mode */
+#define ELF_START_MMAP (0x00000020000000000ULL)
+
+#define elf_check_arch(x) ((x) == EM_TILEGX)
+
+#define ELF_CLASS   ELFCLASS64
+#define ELF_DATA    ELFDATA2LSB
+#define ELF_ARCH    EM_TILEGX
+
+static inline void init_thread(struct target_pt_regs *regs,
+                               struct image_info *infop)
+{
+    regs->pc = infop->entry;
+    regs->sp = infop->start_stack;
+
+}
+
+#define ELF_EXEC_PAGESIZE        65536 /* TILE-Gx page size is 64KB */
+
+#endif /* TARGET_TILEGX */
+
 #ifndef ELF_PLATFORM
 #define ELF_PLATFORM (NULL)
 #endif
 
+#ifndef ELF_MACHINE
+#define ELF_MACHINE ELF_ARCH
+#endif
+
+#ifndef elf_check_arch
+#define elf_check_arch(x) ((x) == ELF_ARCH)
+#endif
+
 #ifndef ELF_HWCAP
 #define ELF_HWCAP 0
 #endif
@@ -1224,7 +1257,8 @@
 
 /* Necessary parameters */
 #define TARGET_ELF_EXEC_PAGESIZE TARGET_PAGE_SIZE
-#define TARGET_ELF_PAGESTART(_v) ((_v) & ~(unsigned long)(TARGET_ELF_EXEC_PAGESIZE-1))
+#define TARGET_ELF_PAGESTART(_v) ((_v) & \
+                                 ~(abi_ulong)(TARGET_ELF_EXEC_PAGESIZE-1))
 #define TARGET_ELF_PAGEOFFSET(_v) ((_v) & (TARGET_ELF_EXEC_PAGESIZE-1))
 
 #define DLINFO_ITEMS 14
@@ -1332,66 +1366,69 @@
  * to be put directly into the top of new user memory.
  *
  */
-static abi_ulong copy_elf_strings(int argc,char ** argv, void **page,
-                                  abi_ulong p)
+static abi_ulong copy_elf_strings(int argc, char **argv, char *scratch,
+                                  abi_ulong p, abi_ulong stack_limit)
 {
-    char *tmp, *tmp1, *pag = NULL;
-    int len, offset = 0;
+    char *tmp;
+    int len, offset;
+    abi_ulong top = p;
 
     if (!p) {
         return 0;       /* bullet-proofing */
     }
+
+    offset = ((p - 1) % TARGET_PAGE_SIZE) + 1;
+
     while (argc-- > 0) {
         tmp = argv[argc];
         if (!tmp) {
             fprintf(stderr, "VFS: argc is wrong");
             exit(-1);
         }
-        tmp1 = tmp;
-        while (*tmp++);
-        len = tmp - tmp1;
-        if (p < len) {  /* this shouldn't happen - 128kB */
+        len = strlen(tmp) + 1;
+        tmp += len;
+
+        if (len > (p - stack_limit)) {
             return 0;
         }
         while (len) {
-            --p; --tmp; --len;
-            if (--offset < 0) {
-                offset = p % TARGET_PAGE_SIZE;
-                pag = (char *)page[p/TARGET_PAGE_SIZE];
-                if (!pag) {
-                    pag = g_try_malloc0(TARGET_PAGE_SIZE);
-                    page[p/TARGET_PAGE_SIZE] = pag;
-                    if (!pag)
-                        return 0;
-                }
-            }
-            if (len == 0 || offset == 0) {
-                *(pag + offset) = *tmp;
-            }
-            else {
-                int bytes_to_copy = (len > offset) ? offset : len;
-                tmp -= bytes_to_copy;
-                p -= bytes_to_copy;
-                offset -= bytes_to_copy;
-                len -= bytes_to_copy;
-                memcpy_fromfs(pag + offset, tmp, bytes_to_copy + 1);
+            int bytes_to_copy = (len > offset) ? offset : len;
+            tmp -= bytes_to_copy;
+            p -= bytes_to_copy;
+            offset -= bytes_to_copy;
+            len -= bytes_to_copy;
+
+            memcpy_fromfs(scratch + offset, tmp, bytes_to_copy);
+
+            if (offset == 0) {
+                memcpy_to_target(p, scratch, top - p);
+                top = p;
+                offset = TARGET_PAGE_SIZE;
             }
         }
     }
+    if (offset) {
+        memcpy_to_target(p, scratch + offset, top - p);
+    }
+
     return p;
 }
 
-static abi_ulong setup_arg_pages(abi_ulong p, struct linux_binprm *bprm,
+/* Older linux kernels provide up to MAX_ARG_PAGES (default: 32) of
+ * argument/environment space. Newer kernels (>2.6.33) allow more,
+ * dependent on stack size, but guarantee at least 32 pages for
+ * backwards compatibility.
+ */
+#define STACK_LOWER_LIMIT (32 * TARGET_PAGE_SIZE)
+
+static abi_ulong setup_arg_pages(struct linux_binprm *bprm,
                                  struct image_info *info)
 {
-    abi_ulong stack_base, size, error, guard;
-    int i;
+    abi_ulong size, error, guard;
 
-    /* Create enough stack to hold everything.  If we don't use
-       it for args, we'll use it for something else.  */
     size = guest_stack_size;
-    if (size < MAX_ARG_PAGES*TARGET_PAGE_SIZE) {
-        size = MAX_ARG_PAGES*TARGET_PAGE_SIZE;
+    if (size < STACK_LOWER_LIMIT) {
+        size = STACK_LOWER_LIMIT;
     }
     guard = TARGET_PAGE_SIZE;
     if (guard < qemu_real_host_page_size) {
@@ -1409,19 +1446,8 @@
     target_mprotect(error, guard, PROT_NONE);
 
     info->stack_limit = error + guard;
-    stack_base = info->stack_limit + size - MAX_ARG_PAGES*TARGET_PAGE_SIZE;
-    p += stack_base;
 
-    for (i = 0 ; i < MAX_ARG_PAGES ; i++) {
-        if (bprm->page[i]) {
-            info->rss++;
-            /* FIXME - check return value of memcpy_to_target() for failure */
-            memcpy_to_target(stack_base, bprm->page[i], TARGET_PAGE_SIZE);
-            g_free(bprm->page[i]);
-        }
-        stack_base += TARGET_PAGE_SIZE;
-    }
-    return p;
+    return info->stack_limit + size - sizeof(void *);
 }
 
 /* Map and zero the bss.  We need to explicitly zero any fractional pages
@@ -1445,8 +1471,7 @@
 
     host_start = (uintptr_t) g2h(elf_bss);
     host_end = (uintptr_t) g2h(last_bss);
-    host_map_start = (host_start + qemu_real_host_page_size - 1);
-    host_map_start &= -qemu_real_host_page_size;
+    host_map_start = REAL_HOST_PAGE_ALIGN(host_start);
 
     if (host_map_start < host_end) {
         void *p = mmap((void *)host_map_start, host_end - host_map_start,
@@ -1711,7 +1736,7 @@
         }
     }
 
-    qemu_log("Reserved 0x%lx bytes of guest address space\n", host_size);
+    qemu_log_mask(CPU_LOG_PAGE, "Reserved 0x%lx bytes of guest address space\n", host_size);
 
     return real_start;
 }
@@ -1723,7 +1748,6 @@
      * it explicitly, and set guest_base appropriately.
      * In case of error we will print a suitable message and exit.
      */
-#if defined(CONFIG_USE_GUEST_BASE)
     const char *errmsg;
     if (!have_guest_base && !reserved_va) {
         unsigned long host_start, real_start, host_size;
@@ -1753,16 +1777,15 @@
         }
         guest_base = real_start - loaddr;
 
-        qemu_log("Relocating guest address space from 0x"
-                 TARGET_ABI_FMT_lx " to 0x%lx\n",
-                 loaddr, real_start);
+        qemu_log_mask(CPU_LOG_PAGE, "Relocating guest address space from 0x"
+                      TARGET_ABI_FMT_lx " to 0x%lx\n",
+                      loaddr, real_start);
     }
     return;
 
 exit_errmsg:
     fprintf(stderr, "%s: %s\n", image_name, errmsg);
     exit(-1);
-#endif
 }
 
 
@@ -2165,10 +2188,9 @@
     struct image_info interp_info;
     struct elfhdr elf_ex;
     char *elf_interpreter = NULL;
+    char *scratch;
 
     info->start_mmap = (abi_ulong)ELF_START_MMAP;
-    info->mmap = 0;
-    info->rss = 0;
 
     load_elf_image(bprm->filename, bprm->fd, info,
                    &elf_interpreter, bprm->buf);
@@ -2178,18 +2200,24 @@
        when we load the interpreter.  */
     elf_ex = *(struct elfhdr *)bprm->buf;
 
-    bprm->p = copy_elf_strings(1, &bprm->filename, bprm->page, bprm->p);
-    bprm->p = copy_elf_strings(bprm->envc,bprm->envp,bprm->page,bprm->p);
-    bprm->p = copy_elf_strings(bprm->argc,bprm->argv,bprm->page,bprm->p);
+    /* Do this so that we can load the interpreter, if need be.  We will
+       change some of these later */
+    bprm->p = setup_arg_pages(bprm, info);
+
+    scratch = g_new0(char, TARGET_PAGE_SIZE);
+    bprm->p = copy_elf_strings(1, &bprm->filename, scratch,
+                               bprm->p, info->stack_limit);
+    bprm->p = copy_elf_strings(bprm->envc, bprm->envp, scratch,
+                               bprm->p, info->stack_limit);
+    bprm->p = copy_elf_strings(bprm->argc, bprm->argv, scratch,
+                               bprm->p, info->stack_limit);
+    g_free(scratch);
+
     if (!bprm->p) {
         fprintf(stderr, "%s: %s\n", bprm->filename, strerror(E2BIG));
         exit(-1);
     }
 
-    /* Do this so that we can load the interpreter, if need be.  We will
-       change some of these later */
-    bprm->p = setup_arg_pages(bprm->p, bprm, info);
-
     if (elf_interpreter) {
         load_elf_interp(elf_interpreter, &interp_info, bprm->buf);
 
@@ -2817,7 +2845,7 @@
     TaskState *ts = (TaskState *)cpu->opaque;
     int i;
 
-    info->notes = g_malloc0(NUMNOTES * sizeof (struct memelfnote));
+    info->notes = g_new0(struct memelfnote, NUMNOTES);
     if (info->notes == NULL)
         return (-ENOMEM);
     info->prstatus = g_malloc0(sizeof (*info->prstatus));
@@ -2884,8 +2912,7 @@
             return (error);
 
     /* write prstatus for each thread */
-    for (ets = info->thread_list.tqh_first; ets != NULL;
-         ets = ets->ets_link.tqe_next) {
+    QTAILQ_FOREACH(ets, &info->thread_list, ets_link) {
         if ((error = write_note(&ets->notes[0], fd)) != 0)
             return (error);
     }
@@ -2984,7 +3011,7 @@
     if (dump_write(fd, &elf, sizeof (elf)) != 0)
         goto out;
 
-    /* fill in in-memory version of notes */
+    /* fill in the in-memory version of notes */
     if (fill_note_info(&info, signr, env) < 0)
         goto out;
 
diff --git a/linux-user/errno_defs.h b/linux-user/errno_defs.h
index 8a1cf76..65522c4 100644
--- a/linux-user/errno_defs.h
+++ b/linux-user/errno_defs.h
@@ -139,3 +139,20 @@
 /* for robust mutexes */
 #define TARGET_EOWNERDEAD      130     /* Owner died */
 #define TARGET_ENOTRECOVERABLE 131     /* State not recoverable */
+
+/* QEMU internal, not visible to the guest. This is returned when a
+ * system call should be restarted, to tell the main loop that it
+ * should wind the guest PC backwards so it will re-execute the syscall
+ * after handling any pending signals. They match with the ones the guest
+ * kernel uses for the same purpose.
+ */
+#define TARGET_ERESTARTSYS     512     /* Restart system call (if SA_RESTART) */
+
+/* QEMU internal, not visible to the guest. This is returned by the
+ * do_sigreturn() code after a successful sigreturn syscall, to indicate
+ * that it has correctly set the guest registers and so the main loop
+ * should not touch them. We use the value the guest would use for
+ * ERESTART_NOINTR (which is kernel internal) to guarantee that we won't
+ * clash with a valid guest errno now or in the future.
+ */
+#define TARGET_QEMU_ESIGRETURN 513     /* Return from signal */
diff --git a/linux-user/flatload.c b/linux-user/flatload.c
index 566a7a8..42d1079 100644
--- a/linux-user/flatload.c
+++ b/linux-user/flatload.c
@@ -33,16 +33,11 @@
 
 /****************************************************************************/
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <errno.h>
-#include <sys/mman.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "flat.h"
-#define ntohl(x) be32_to_cpu(x)
-#include <target_flat.h>
+#include "target_flat.h"
 
 //#define DEBUG
 
@@ -707,7 +702,7 @@
 int load_flt_binary(struct linux_binprm *bprm, struct image_info *info)
 {
     struct lib_info libinfo[MAX_SHARED_LIBS];
-    abi_ulong p = bprm->p;
+    abi_ulong p;
     abi_ulong stack_len;
     abi_ulong start_addr;
     abi_ulong sp;
diff --git a/linux-user/host/aarch64/hostdep.h b/linux-user/host/aarch64/hostdep.h
new file mode 100644
index 0000000..64f75ce
--- /dev/null
+++ b/linux-user/host/aarch64/hostdep.h
@@ -0,0 +1,38 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef AARCH64_HOSTDEP_H
+#define AARCH64_HOSTDEP_H
+
+/* We have a safe-syscall.inc.S */
+#define HAVE_SAFE_SYSCALL
+
+#ifndef __ASSEMBLER__
+
+/* These are defined by the safe-syscall.inc.S file */
+extern char safe_syscall_start[];
+extern char safe_syscall_end[];
+
+/* Adjust the signal context to rewind out of safe-syscall if we're in it */
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    struct ucontext *uc = puc;
+    __u64 *pcreg = &uc->uc_mcontext.pc;
+
+    if (*pcreg > (uintptr_t)safe_syscall_start
+        && *pcreg < (uintptr_t)safe_syscall_end) {
+        *pcreg = (uintptr_t)safe_syscall_start;
+    }
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif
diff --git a/linux-user/host/aarch64/safe-syscall.inc.S b/linux-user/host/aarch64/safe-syscall.inc.S
new file mode 100644
index 0000000..58a2329
--- /dev/null
+++ b/linux-user/host/aarch64/safe-syscall.inc.S
@@ -0,0 +1,75 @@
+/*
+ * safe-syscall.inc.S : host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ * This is intended to be included by linux-user/safe-syscall.S
+ *
+ * Written by Richard Henderson <rth@twiddle.net>
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+	.global safe_syscall_base
+	.global safe_syscall_start
+	.global safe_syscall_end
+	.type	safe_syscall_base, #function
+	.type	safe_syscall_start, #function
+	.type	safe_syscall_end, #function
+
+	/* This is the entry point for making a system call. The calling
+	 * convention here is that of a C varargs function with the
+	 * first argument an 'int *' to the signal_pending flag, the
+	 * second one the system call number (as a 'long'), and all further
+	 * arguments being syscall arguments (also 'long').
+	 * We return a long which is the syscall's return value, which
+	 * may be negative-errno on failure. Conversion to the
+	 * -1-and-errno-set convention is done by the calling wrapper.
+	 */
+safe_syscall_base:
+	.cfi_startproc
+	/* The syscall calling convention isn't the same as the
+	 * C one:
+	 * we enter with x0 == *signal_pending
+	 *               x1 == syscall number
+	 *               x2 ... x7, (stack) == syscall arguments
+	 *               and return the result in x0
+	 * and the syscall instruction needs
+	 *               x8 == syscall number
+	 *               x0 ... x7 == syscall arguments
+	 *               and returns the result in x0
+	 * Shuffle everything around appropriately.
+	 */
+	mov	x9, x0		/* signal_pending pointer */
+	mov	x8, x1		/* syscall number */
+	mov	x0, x2		/* syscall arguments */
+	mov	x1, x3
+	mov	x2, x4
+	mov	x3, x5
+	mov	x4, x6
+	mov	x6, x7
+	ldr	x7, [sp]
+
+	/* This next sequence of code works in conjunction with the
+	 * rewind_if_safe_syscall_function(). If a signal is taken
+	 * and the interrupted PC is anywhere between 'safe_syscall_start'
+	 * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'.
+	 * The code sequence must therefore be able to cope with this, and
+	 * the syscall instruction must be the final one in the sequence.
+	 */
+safe_syscall_start:
+	/* if signal_pending is non-zero, don't do the call */
+	ldr	w10, [x9]
+	cbnz	w10, 0f 
+	svc	0x0
+safe_syscall_end:
+	/* code path for having successfully executed the syscall */
+	ret
+
+0:
+	/* code path when we didn't execute the syscall */
+	mov	x0, #-TARGET_ERESTARTSYS
+	ret
+	.cfi_endproc
+
+	.size	safe_syscall_base, .-safe_syscall_base
diff --git a/linux-user/host/arm/hostdep.h b/linux-user/host/arm/hostdep.h
new file mode 100644
index 0000000..5c1ae60
--- /dev/null
+++ b/linux-user/host/arm/hostdep.h
@@ -0,0 +1,38 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef ARM_HOSTDEP_H
+#define ARM_HOSTDEP_H
+
+/* We have a safe-syscall.inc.S */
+#define HAVE_SAFE_SYSCALL
+
+#ifndef __ASSEMBLER__
+
+/* These are defined by the safe-syscall.inc.S file */
+extern char safe_syscall_start[];
+extern char safe_syscall_end[];
+
+/* Adjust the signal context to rewind out of safe-syscall if we're in it */
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    struct ucontext *uc = puc;
+    unsigned long *pcreg = &uc->uc_mcontext.arm_pc;
+
+    if (*pcreg > (uintptr_t)safe_syscall_start
+        && *pcreg < (uintptr_t)safe_syscall_end) {
+        *pcreg = (uintptr_t)safe_syscall_start;
+    }
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif
diff --git a/linux-user/host/arm/safe-syscall.inc.S b/linux-user/host/arm/safe-syscall.inc.S
new file mode 100644
index 0000000..88c4958
--- /dev/null
+++ b/linux-user/host/arm/safe-syscall.inc.S
@@ -0,0 +1,90 @@
+/*
+ * safe-syscall.inc.S : host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ * This is intended to be included by linux-user/safe-syscall.S
+ *
+ * Written by Richard Henderson <rth@twiddle.net>
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+	.global safe_syscall_base
+	.global safe_syscall_start
+	.global safe_syscall_end
+	.type	safe_syscall_base, %function
+
+	.cfi_sections	.debug_frame
+
+	.text
+	.syntax unified
+	.arm
+	.align 2
+
+	/* This is the entry point for making a system call. The calling
+	 * convention here is that of a C varargs function with the
+	 * first argument an 'int *' to the signal_pending flag, the
+	 * second one the system call number (as a 'long'), and all further
+	 * arguments being syscall arguments (also 'long').
+	 * We return a long which is the syscall's return value, which
+	 * may be negative-errno on failure. Conversion to the
+	 * -1-and-errno-set convention is done by the calling wrapper.
+	 */
+safe_syscall_base:
+	.fnstart
+	.cfi_startproc
+	mov	r12, sp			/* save entry stack */
+	push	{ r4, r5, r6, r7, r8, lr }
+	.save	{ r4, r5, r6, r7, r8, lr }
+	.cfi_adjust_cfa_offset 24
+	.cfi_rel_offset r4, 0
+	.cfi_rel_offset r5, 4
+	.cfi_rel_offset r6, 8
+	.cfi_rel_offset r7, 12
+	.cfi_rel_offset r8, 16
+	.cfi_rel_offset lr, 20
+
+	/* The syscall calling convention isn't the same as the C one:
+	 * we enter with r0 == *signal_pending
+	 *               r1 == syscall number
+	 *               r2, r3, [sp+0] ... [sp+12] == syscall arguments
+	 *               and return the result in r0
+	 * and the syscall instruction needs
+	 *               r7 == syscall number
+	 *               r0 ... r6 == syscall arguments
+	 *               and returns the result in r0
+	 * Shuffle everything around appropriately.
+	 * Note the 16 bytes that we pushed to save registers.
+	 */
+	mov	r8, r0			/* copy signal_pending */
+	mov	r7, r1			/* syscall number */
+	mov	r0, r2			/* syscall args */
+	mov	r1, r3
+	ldm	r12, { r2, r3, r4, r5, r6 }
+
+	/* This next sequence of code works in conjunction with the
+	 * rewind_if_safe_syscall_function(). If a signal is taken
+	 * and the interrupted PC is anywhere between 'safe_syscall_start'
+	 * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'.
+	 * The code sequence must therefore be able to cope with this, and
+	 * the syscall instruction must be the final one in the sequence.
+	 */
+safe_syscall_start:
+	/* if signal_pending is non-zero, don't do the call */
+	ldr	r12, [r8]		/* signal_pending */
+	tst	r12, r12
+	bne	1f
+	swi	0
+safe_syscall_end:
+	/* code path for having successfully executed the syscall */
+	pop	{ r4, r5, r6, r7, r8, pc }
+
+1:
+	/* code path when we didn't execute the syscall */
+	ldr	r0, =-TARGET_ERESTARTSYS
+	pop	{ r4, r5, r6, r7, r8, pc }
+	.fnend
+	.cfi_endproc
+
+	.size	safe_syscall_base, .-safe_syscall_base
diff --git a/linux-user/host/i386/hostdep.h b/linux-user/host/i386/hostdep.h
new file mode 100644
index 0000000..d834bd8
--- /dev/null
+++ b/linux-user/host/i386/hostdep.h
@@ -0,0 +1,38 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef I386_HOSTDEP_H
+#define I386_HOSTDEP_H
+
+/* We have a safe-syscall.inc.S */
+#define HAVE_SAFE_SYSCALL
+
+#ifndef __ASSEMBLER__
+
+/* These are defined by the safe-syscall.inc.S file */
+extern char safe_syscall_start[];
+extern char safe_syscall_end[];
+
+/* Adjust the signal context to rewind out of safe-syscall if we're in it */
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    struct ucontext *uc = puc;
+    greg_t *pcreg = &uc->uc_mcontext.gregs[REG_EIP];
+
+    if (*pcreg > (uintptr_t)safe_syscall_start
+        && *pcreg < (uintptr_t)safe_syscall_end) {
+        *pcreg = (uintptr_t)safe_syscall_start;
+    }
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif
diff --git a/linux-user/host/i386/safe-syscall.inc.S b/linux-user/host/i386/safe-syscall.inc.S
new file mode 100644
index 0000000..9e58fc6
--- /dev/null
+++ b/linux-user/host/i386/safe-syscall.inc.S
@@ -0,0 +1,100 @@
+/*
+ * safe-syscall.inc.S : host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ * This is intended to be included by linux-user/safe-syscall.S
+ *
+ * Written by Richard Henderson <rth@twiddle.net>
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+	.global safe_syscall_base
+	.global safe_syscall_start
+	.global safe_syscall_end
+	.type	safe_syscall_base, @function
+
+	/* This is the entry point for making a system call. The calling
+	 * convention here is that of a C varargs function with the
+	 * first argument an 'int *' to the signal_pending flag, the
+	 * second one the system call number (as a 'long'), and all further
+	 * arguments being syscall arguments (also 'long').
+	 * We return a long which is the syscall's return value, which
+	 * may be negative-errno on failure. Conversion to the
+	 * -1-and-errno-set convention is done by the calling wrapper.
+	 */
+safe_syscall_base:
+	.cfi_startproc
+	push	%ebp
+	.cfi_adjust_cfa_offset 4
+	.cfi_rel_offset ebp, 0
+	push	%esi
+	.cfi_adjust_cfa_offset 4
+	.cfi_rel_offset esi, 0
+	push	%edi
+	.cfi_adjust_cfa_offset 4
+	.cfi_rel_offset edi, 0
+	push	%ebx
+	.cfi_adjust_cfa_offset 4
+	.cfi_rel_offset ebx, 0
+
+	/* The syscall calling convention isn't the same as the C one:
+	 * we enter with 0(%esp) == return address
+	 *               4(%esp) == *signal_pending
+	 *               8(%esp) == syscall number
+	 *               12(%esp) ... 32(%esp) == syscall arguments
+	 *               and return the result in eax
+	 * and the syscall instruction needs
+	 *               eax == syscall number
+	 *               ebx, ecx, edx, esi, edi, ebp == syscall arguments
+	 *               and returns the result in eax
+	 * Shuffle everything around appropriately.
+	 * Note the 16 bytes that we pushed to save registers.
+	 */
+	mov	12+16(%esp), %ebx	/* the syscall arguments */
+	mov	16+16(%esp), %ecx
+	mov	20+16(%esp), %edx
+	mov	24+16(%esp), %esi
+	mov	28+16(%esp), %edi
+	mov	32+16(%esp), %ebp
+
+	/* This next sequence of code works in conjunction with the
+	 * rewind_if_safe_syscall_function(). If a signal is taken
+	 * and the interrupted PC is anywhere between 'safe_syscall_start'
+	 * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'.
+	 * The code sequence must therefore be able to cope with this, and
+	 * the syscall instruction must be the final one in the sequence.
+	 */
+safe_syscall_start:
+	/* if signal_pending is non-zero, don't do the call */
+	mov	4+16(%esp), %eax	/* signal_pending */
+	cmpl	$0, (%eax)
+	jnz	1f
+	mov	8+16(%esp), %eax	/* syscall number */
+	int	$0x80
+safe_syscall_end:
+	/* code path for having successfully executed the syscall */
+	pop	%ebx
+	.cfi_remember_state
+	.cfi_adjust_cfa_offset -4
+	.cfi_restore ebx
+	pop	%edi
+	.cfi_adjust_cfa_offset -4
+	.cfi_restore edi
+	pop	%esi
+	.cfi_adjust_cfa_offset -4
+	.cfi_restore esi
+	pop	%ebp
+	.cfi_adjust_cfa_offset -4
+	.cfi_restore ebp
+	ret
+
+1:
+	/* code path when we didn't execute the syscall */
+	.cfi_restore_state
+	mov	$-TARGET_ERESTARTSYS, %eax
+	jmp	safe_syscall_end
+	.cfi_endproc
+
+	.size	safe_syscall_base, .-safe_syscall_base
diff --git a/linux-user/host/ia64/hostdep.h b/linux-user/host/ia64/hostdep.h
new file mode 100644
index 0000000..263bf76
--- /dev/null
+++ b/linux-user/host/ia64/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef IA64_HOSTDEP_H
+#define IA64_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/mips/hostdep.h b/linux-user/host/mips/hostdep.h
new file mode 100644
index 0000000..ba111d7
--- /dev/null
+++ b/linux-user/host/mips/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef MIPS_HOSTDEP_H
+#define MIPS_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/ppc/hostdep.h b/linux-user/host/ppc/hostdep.h
new file mode 100644
index 0000000..23d8bd9
--- /dev/null
+++ b/linux-user/host/ppc/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef PPC_HOSTDEP_H
+#define PPC_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/ppc64/hostdep.h b/linux-user/host/ppc64/hostdep.h
new file mode 100644
index 0000000..0b0f5f7
--- /dev/null
+++ b/linux-user/host/ppc64/hostdep.h
@@ -0,0 +1,38 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef PPC64_HOSTDEP_H
+#define PPC64_HOSTDEP_H
+
+/* We have a safe-syscall.inc.S */
+#define HAVE_SAFE_SYSCALL
+
+#ifndef __ASSEMBLER__
+
+/* These are defined by the safe-syscall.inc.S file */
+extern char safe_syscall_start[];
+extern char safe_syscall_end[];
+
+/* Adjust the signal context to rewind out of safe-syscall if we're in it */
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    struct ucontext *uc = puc;
+    unsigned long *pcreg = &uc->uc_mcontext.gp_regs[PT_NIP];
+
+    if (*pcreg > (uintptr_t)safe_syscall_start
+        && *pcreg < (uintptr_t)safe_syscall_end) {
+        *pcreg = (uintptr_t)safe_syscall_start;
+    }
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif
diff --git a/linux-user/host/ppc64/safe-syscall.inc.S b/linux-user/host/ppc64/safe-syscall.inc.S
new file mode 100644
index 0000000..d30050a
--- /dev/null
+++ b/linux-user/host/ppc64/safe-syscall.inc.S
@@ -0,0 +1,92 @@
+/*
+ * safe-syscall.inc.S : host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ * This is intended to be included by linux-user/safe-syscall.S
+ *
+ * Written by Richard Henderson <rth@twiddle.net>
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+	.global safe_syscall_base
+	.global safe_syscall_start
+	.global safe_syscall_end
+	.type	safe_syscall_base, @function
+
+	.text
+
+	/* This is the entry point for making a system call. The calling
+	 * convention here is that of a C varargs function with the
+	 * first argument an 'int *' to the signal_pending flag, the
+	 * second one the system call number (as a 'long'), and all further
+	 * arguments being syscall arguments (also 'long').
+	 * We return a long which is the syscall's return value, which
+	 * may be negative-errno on failure. Conversion to the
+	 * -1-and-errno-set convention is done by the calling wrapper.
+	 */
+#if _CALL_ELF == 2
+safe_syscall_base:
+	.cfi_startproc
+	.localentry safe_syscall_base,0
+#else
+	.section ".opd","aw"
+	.align	3
+safe_syscall_base:
+	.quad	.L.safe_syscall_base,.TOC.@tocbase,0
+	.previous
+.L.safe_syscall_base:
+	.cfi_startproc
+#endif
+	/* We enter with r3 == *signal_pending
+	 *               r4 == syscall number
+	 *               r5 ... r10 == syscall arguments
+	 *               and return the result in r3
+	 * and the syscall instruction needs
+	 *               r0 == syscall number
+	 *               r3 ... r8 == syscall arguments
+	 *               and returns the result in r3
+	 * Shuffle everything around appropriately.
+	 */
+	mr	11, 3	/* signal_pending */
+	mr	0, 4	/* syscall number */
+	mr	3, 5	/* syscall arguments */
+	mr	4, 6
+	mr	5, 7
+	mr	6, 8
+	mr	7, 9
+	mr	8, 10
+
+	/* This next sequence of code works in conjunction with the
+	 * rewind_if_safe_syscall_function(). If a signal is taken
+	 * and the interrupted PC is anywhere between 'safe_syscall_start'
+	 * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'.
+	 * The code sequence must therefore be able to cope with this, and
+	 * the syscall instruction must be the final one in the sequence.
+	 */
+safe_syscall_start:
+	/* if signal_pending is non-zero, don't do the call */
+	lwz	12, 0(11)
+	cmpwi	0, 12, 0
+	bne-	0f
+	sc
+safe_syscall_end:
+	/* code path when we did execute the syscall */
+	bnslr+
+
+	/* syscall failed; return negative errno */
+	neg	3, 3
+	blr
+
+	/* code path when we didn't execute the syscall */
+0:	addi	3, 0, -TARGET_ERESTARTSYS
+	blr
+	.cfi_endproc
+
+#if _CALL_ELF == 2
+	.size	safe_syscall_base, .-safe_syscall_base
+#else
+	.size	safe_syscall_base, .-.L.safe_syscall_base
+	.size	.L.safe_syscall_base, .-.L.safe_syscall_base
+#endif
diff --git a/linux-user/host/s390/hostdep.h b/linux-user/host/s390/hostdep.h
new file mode 100644
index 0000000..afcba5a
--- /dev/null
+++ b/linux-user/host/s390/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef S390_HOSTDEP_H
+#define S390_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/s390x/hostdep.h b/linux-user/host/s390x/hostdep.h
new file mode 100644
index 0000000..6f9da9c
--- /dev/null
+++ b/linux-user/host/s390x/hostdep.h
@@ -0,0 +1,38 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef S390X_HOSTDEP_H
+#define S390X_HOSTDEP_H
+
+/* We have a safe-syscall.inc.S */
+#define HAVE_SAFE_SYSCALL
+
+#ifndef __ASSEMBLER__
+
+/* These are defined by the safe-syscall.inc.S file */
+extern char safe_syscall_start[];
+extern char safe_syscall_end[];
+
+/* Adjust the signal context to rewind out of safe-syscall if we're in it */
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    struct ucontext *uc = puc;
+    unsigned long *pcreg = &uc->uc_mcontext.psw.addr;
+
+    if (*pcreg > (uintptr_t)safe_syscall_start
+        && *pcreg < (uintptr_t)safe_syscall_end) {
+        *pcreg = (uintptr_t)safe_syscall_start;
+    }
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif
diff --git a/linux-user/host/s390x/safe-syscall.inc.S b/linux-user/host/s390x/safe-syscall.inc.S
new file mode 100644
index 0000000..f1b446a
--- /dev/null
+++ b/linux-user/host/s390x/safe-syscall.inc.S
@@ -0,0 +1,90 @@
+/*
+ * safe-syscall.inc.S : host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ * This is intended to be included by linux-user/safe-syscall.S
+ *
+ * Written by Richard Henderson <rth@twiddle.net>
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+	.global safe_syscall_base
+	.global safe_syscall_start
+	.global safe_syscall_end
+	.type	safe_syscall_base, @function
+
+	/* This is the entry point for making a system call. The calling
+	 * convention here is that of a C varargs function with the
+	 * first argument an 'int *' to the signal_pending flag, the
+	 * second one the system call number (as a 'long'), and all further
+	 * arguments being syscall arguments (also 'long').
+	 * We return a long which is the syscall's return value, which
+	 * may be negative-errno on failure. Conversion to the
+	 * -1-and-errno-set convention is done by the calling wrapper.
+	 */
+safe_syscall_base:
+	.cfi_startproc
+	stmg	%r6,%r15,48(%r15)	/* save all call-saved registers */
+	.cfi_offset %r15,-40
+	.cfi_offset %r14,-48
+	.cfi_offset %r13,-56
+	.cfi_offset %r12,-64
+	.cfi_offset %r11,-72
+	.cfi_offset %r10,-80
+	.cfi_offset %r9,-88
+	.cfi_offset %r8,-96
+	.cfi_offset %r7,-104
+	.cfi_offset %r6,-112
+	lgr	%r1,%r15
+	lg	%r0,8(%r15)		/* load eos */
+	aghi	%r15,-160
+	.cfi_adjust_cfa_offset 160
+	stg	%r1,0(%r15)		/* store back chain */
+	stg	%r0,8(%r15)		/* store eos */
+
+	/* The syscall calling convention isn't the same as the
+	 * C one:
+	 * we enter with r2 == *signal_pending
+	 *               r3 == syscall number
+	 *               r4, r5, r6, (stack) == syscall arguments
+	 *               and return the result in r2
+	 * and the syscall instruction needs
+	 *               r1 == syscall number
+	 *               r2 ... r7 == syscall arguments
+	 *               and returns the result in r2
+	 * Shuffle everything around appropriately.
+	 */
+	lgr	%r8,%r2			/* signal_pending pointer */
+	lgr	%r1,%r3			/* syscall number */
+	lgr	%r2,%r4			/* syscall args */
+	lgr	%r3,%r5
+	lgr	%r4,%r6
+	lmg	%r5,%r7,320(%r15)
+
+	/* This next sequence of code works in conjunction with the
+	 * rewind_if_safe_syscall_function(). If a signal is taken
+	 * and the interrupted PC is anywhere between 'safe_syscall_start'
+	 * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'.
+	 * The code sequence must therefore be able to cope with this, and
+	 * the syscall instruction must be the final one in the sequence.
+	 */
+safe_syscall_start:
+	/* if signal_pending is non-zero, don't do the call */
+	lt	%r0,0(%r8)
+	jne	2f
+	svc	0
+safe_syscall_end:
+
+1:	lg	%r15,0(%r15)		/* load back chain */
+	.cfi_remember_state
+	.cfi_adjust_cfa_offset -160
+	lmg	%r6,%r15,48(%r15)	/* load saved registers */
+	br	%r14
+	.cfi_restore_state
+2:	lghi	%r2, -TARGET_ERESTARTSYS
+	j	1b
+	.cfi_endproc
+
+	.size	safe_syscall_base, .-safe_syscall_base
diff --git a/linux-user/host/sparc/hostdep.h b/linux-user/host/sparc/hostdep.h
new file mode 100644
index 0000000..391ad92
--- /dev/null
+++ b/linux-user/host/sparc/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef SPARC_HOSTDEP_H
+#define SPARC_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/sparc64/hostdep.h b/linux-user/host/sparc64/hostdep.h
new file mode 100644
index 0000000..ce3968f
--- /dev/null
+++ b/linux-user/host/sparc64/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef SPARC64_HOSTDEP_H
+#define SPARC64_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/x32/hostdep.h b/linux-user/host/x32/hostdep.h
new file mode 100644
index 0000000..2c2d6d3
--- /dev/null
+++ b/linux-user/host/x32/hostdep.h
@@ -0,0 +1,15 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef X32_HOSTDEP_H
+#define X32_HOSTDEP_H
+
+#endif
diff --git a/linux-user/host/x86_64/hostdep.h b/linux-user/host/x86_64/hostdep.h
new file mode 100644
index 0000000..3b42596
--- /dev/null
+++ b/linux-user/host/x86_64/hostdep.h
@@ -0,0 +1,38 @@
+/*
+ * hostdep.h : things which are dependent on the host architecture
+ *
+ *  * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef X86_64_HOSTDEP_H
+#define X86_64_HOSTDEP_H
+
+/* We have a safe-syscall.inc.S */
+#define HAVE_SAFE_SYSCALL
+
+#ifndef __ASSEMBLER__
+
+/* These are defined by the safe-syscall.inc.S file */
+extern char safe_syscall_start[];
+extern char safe_syscall_end[];
+
+/* Adjust the signal context to rewind out of safe-syscall if we're in it */
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    struct ucontext *uc = puc;
+    greg_t *pcreg = &uc->uc_mcontext.gregs[REG_RIP];
+
+    if (*pcreg > (uintptr_t)safe_syscall_start
+        && *pcreg < (uintptr_t)safe_syscall_end) {
+        *pcreg = (uintptr_t)safe_syscall_start;
+    }
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif
diff --git a/linux-user/host/x86_64/safe-syscall.inc.S b/linux-user/host/x86_64/safe-syscall.inc.S
new file mode 100644
index 0000000..f36992d
--- /dev/null
+++ b/linux-user/host/x86_64/safe-syscall.inc.S
@@ -0,0 +1,91 @@
+/*
+ * safe-syscall.inc.S : host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ * This is intended to be included by linux-user/safe-syscall.S
+ *
+ * Copyright (C) 2015 Timothy Edward Baldwin <T.E.Baldwin99@members.leeds.ac.uk>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+        .global safe_syscall_base
+        .global safe_syscall_start
+        .global safe_syscall_end
+        .type   safe_syscall_base, @function
+
+        /* This is the entry point for making a system call. The calling
+         * convention here is that of a C varargs function with the
+         * first argument an 'int *' to the signal_pending flag, the
+         * second one the system call number (as a 'long'), and all further
+         * arguments being syscall arguments (also 'long').
+         * We return a long which is the syscall's return value, which
+         * may be negative-errno on failure. Conversion to the
+         * -1-and-errno-set convention is done by the calling wrapper.
+         */
+safe_syscall_base:
+        .cfi_startproc
+        /* This saves a frame pointer and aligns the stack for the syscall.
+         * (It's unclear if the syscall ABI has the same stack alignment
+         * requirements as the userspace function call ABI, but better safe than
+         * sorry. Appendix A2 of http://www.x86-64.org/documentation/abi.pdf
+         * does not list any ABI differences regarding stack alignment.)
+         */
+        push    %rbp
+        .cfi_adjust_cfa_offset 8
+        .cfi_rel_offset rbp, 0
+
+        /* The syscall calling convention isn't the same as the
+         * C one:
+         * we enter with rdi == *signal_pending
+         *               rsi == syscall number
+         *               rdx, rcx, r8, r9, (stack), (stack) == syscall arguments
+         *               and return the result in rax
+         * and the syscall instruction needs
+         *               rax == syscall number
+         *               rdi, rsi, rdx, r10, r8, r9 == syscall arguments
+         *               and returns the result in rax
+         * Shuffle everything around appropriately.
+         * Note that syscall will trash rcx and r11.
+         */
+        mov     %rsi, %rax /* syscall number */
+        mov     %rdi, %rbp /* signal_pending pointer */
+        /* and the syscall arguments */
+        mov     %rdx, %rdi
+        mov     %rcx, %rsi
+        mov     %r8,  %rdx
+        mov     %r9,  %r10
+        mov     16(%rsp), %r8
+        mov     24(%rsp), %r9
+
+        /* This next sequence of code works in conjunction with the
+         * rewind_if_safe_syscall_function(). If a signal is taken
+         * and the interrupted PC is anywhere between 'safe_syscall_start'
+         * and 'safe_syscall_end' then we rewind it to 'safe_syscall_start'.
+         * The code sequence must therefore be able to cope with this, and
+         * the syscall instruction must be the final one in the sequence.
+         */
+safe_syscall_start:
+        /* if signal_pending is non-zero, don't do the call */
+        cmpl	$0, (%rbp)
+        jnz     1f
+        syscall
+safe_syscall_end:
+        /* code path for having successfully executed the syscall */
+        pop     %rbp
+        .cfi_remember_state
+        .cfi_def_cfa_offset 8
+        .cfi_restore rbp
+        ret
+
+1:
+        /* code path when we didn't execute the syscall */
+        .cfi_restore_state
+        mov     $-TARGET_ERESTARTSYS, %rax
+        pop     %rbp
+        .cfi_def_cfa_offset 8
+        .cfi_restore rbp
+        ret
+        .cfi_endproc
+
+        .size   safe_syscall_base, .-safe_syscall_base
diff --git a/linux-user/i386/syscall.h b/linux-user/i386/syscall.h
deleted file mode 100644
index 906aaac..0000000
--- a/linux-user/i386/syscall.h
+++ /dev/null
@@ -1,152 +0,0 @@
-/* default linux values for the selectors */
-#define __USER_CS	(0x23)
-#define __USER_DS	(0x2B)
-
-struct target_pt_regs {
-	long ebx;
-	long ecx;
-	long edx;
-	long esi;
-	long edi;
-	long ebp;
-	long eax;
-	int  xds;
-	int  xes;
-	long orig_eax;
-	long eip;
-	int  xcs;
-	long eflags;
-	long esp;
-	int  xss;
-};
-
-/* ioctls */
-
-#define TARGET_LDT_ENTRIES      8192
-#define TARGET_LDT_ENTRY_SIZE	8
-
-#define TARGET_GDT_ENTRIES             9
-#define TARGET_GDT_ENTRY_TLS_ENTRIES   3
-#define TARGET_GDT_ENTRY_TLS_MIN       6
-#define TARGET_GDT_ENTRY_TLS_MAX       (TARGET_GDT_ENTRY_TLS_MIN + TARGET_GDT_ENTRY_TLS_ENTRIES - 1)
-
-struct target_modify_ldt_ldt_s {
-    unsigned int  entry_number;
-    abi_ulong base_addr;
-    unsigned int limit;
-    unsigned int flags;
-};
-
-/* vm86 defines */
-
-#define TARGET_BIOSSEG		0x0f000
-
-#define TARGET_CPU_086		0
-#define TARGET_CPU_186		1
-#define TARGET_CPU_286		2
-#define TARGET_CPU_386		3
-#define TARGET_CPU_486		4
-#define TARGET_CPU_586		5
-
-#define TARGET_VM86_SIGNAL	0	/* return due to signal */
-#define TARGET_VM86_UNKNOWN	1	/* unhandled GP fault - IO-instruction or similar */
-#define TARGET_VM86_INTx	2	/* int3/int x instruction (ARG = x) */
-#define TARGET_VM86_STI	3	/* sti/popf/iret instruction enabled virtual interrupts */
-
-/*
- * Additional return values when invoking new vm86()
- */
-#define TARGET_VM86_PICRETURN	4	/* return due to pending PIC request */
-#define TARGET_VM86_TRAP	6	/* return due to DOS-debugger request */
-
-/*
- * function codes when invoking new vm86()
- */
-#define TARGET_VM86_PLUS_INSTALL_CHECK	0
-#define TARGET_VM86_ENTER		1
-#define TARGET_VM86_ENTER_NO_BYPASS	2
-#define	TARGET_VM86_REQUEST_IRQ	3
-#define TARGET_VM86_FREE_IRQ		4
-#define TARGET_VM86_GET_IRQ_BITS	5
-#define TARGET_VM86_GET_AND_RESET_IRQ	6
-
-/*
- * This is the stack-layout seen by the user space program when we have
- * done a translation of "SAVE_ALL" from vm86 mode. The real kernel layout
- * is 'kernel_vm86_regs' (see below).
- */
-
-struct target_vm86_regs {
-/*
- * normal regs, with special meaning for the segment descriptors..
- */
-	abi_long ebx;
-	abi_long ecx;
-	abi_long edx;
-	abi_long esi;
-	abi_long edi;
-	abi_long ebp;
-	abi_long eax;
-	abi_long __null_ds;
-	abi_long __null_es;
-	abi_long __null_fs;
-	abi_long __null_gs;
-	abi_long orig_eax;
-	abi_long eip;
-	unsigned short cs, __csh;
-	abi_long eflags;
-	abi_long esp;
-	unsigned short ss, __ssh;
-/*
- * these are specific to v86 mode:
- */
-	unsigned short es, __esh;
-	unsigned short ds, __dsh;
-	unsigned short fs, __fsh;
-	unsigned short gs, __gsh;
-};
-
-struct target_revectored_struct {
-	abi_ulong __map[8];			/* 256 bits */
-};
-
-struct target_vm86_struct {
-	struct target_vm86_regs regs;
-	abi_ulong flags;
-	abi_ulong screen_bitmap;
-	abi_ulong cpu_type;
-	struct target_revectored_struct int_revectored;
-	struct target_revectored_struct int21_revectored;
-};
-
-/*
- * flags masks
- */
-#define TARGET_VM86_SCREEN_BITMAP	0x0001
-
-struct target_vm86plus_info_struct {
-        abi_ulong flags;
-#define TARGET_force_return_for_pic (1 << 0)
-#define TARGET_vm86dbg_active       (1 << 1)  /* for debugger */
-#define TARGET_vm86dbg_TFpendig     (1 << 2)  /* for debugger */
-#define TARGET_is_vm86pus           (1 << 31) /* for vm86 internal use */
-	unsigned char vm86dbg_intxxtab[32];   /* for debugger */
-};
-
-struct target_vm86plus_struct {
-	struct target_vm86_regs regs;
-	abi_ulong flags;
-	abi_ulong screen_bitmap;
-	abi_ulong cpu_type;
-	struct target_revectored_struct int_revectored;
-	struct target_revectored_struct int21_revectored;
-	struct target_vm86plus_info_struct vm86plus;
-};
-
-#define UNAME_MACHINE "i686"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/i386/syscall_nr.h b/linux-user/i386/syscall_nr.h
index c8f7302..bc1bc23 100644
--- a/linux-user/i386/syscall_nr.h
+++ b/linux-user/i386/syscall_nr.h
@@ -324,7 +324,7 @@
 #define TARGET_NR_epoll_pwait	319
 #define TARGET_NR_utimensat		320
 #define TARGET_NR_signalfd		321
-#define TARGET_NR_timerfd		322
+#define TARGET_NR_timerfd_create	322
 #define TARGET_NR_eventfd		323
 #define TARGET_NR_fallocate		324
 #define TARGET_NR_timerfd_settime	325
@@ -353,3 +353,30 @@
 #define TARGET_NR_process_vm_writev     348
 #define TARGET_NR_kcmp                  349
 #define TARGET_NR_finit_module          350
+#define TARGET_NR_sched_setattr         351
+#define TARGET_NR_sched_getattr         352
+#define TARGET_NR_renameat2             353
+#define TARGET_NR_seccomp               354
+#define TARGET_NR_getrandom             355
+#define TARGET_NR_memfd_create          356
+#define TARGET_NR_bpf                   357
+#define TARGET_NR_execveat              358
+#define TARGET_NR_socket                359
+#define TARGET_NR_socketpair            360
+#define TARGET_NR_bind                  361
+#define TARGET_NR_connect               362
+#define TARGET_NR_listen                363
+#define TARGET_NR_accept4               364
+#define TARGET_NR_getsockopt            365
+#define TARGET_NR_setsockopt            366
+#define TARGET_NR_getsockname           367
+#define TARGET_NR_getpeername           368
+#define TARGET_NR_sendto                369
+#define TARGET_NR_sendmsg               370
+#define TARGET_NR_recvfrom              371
+#define TARGET_NR_recvmsg               372
+#define TARGET_NR_shutdown              373
+#define TARGET_NR_userfaultfd           374
+#define TARGET_NR_membarrier            375
+#define TARGET_NR_mlock2                376
+#define TARGET_NR_copy_file_range       377
diff --git a/linux-user/i386/target_cpu.h b/linux-user/i386/target_cpu.h
index 58f8645..7fbcf9b 100644
--- a/linux-user/i386/target_cpu.h
+++ b/linux-user/i386/target_cpu.h
@@ -17,8 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef I386_TARGET_CPU_H
+#define I386_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUX86State *env, target_ulong newsp)
 {
@@ -45,4 +45,4 @@
 }
 #endif /* defined(TARGET_ABI32) */
 
-#endif /* !defined(TARGET_CPU_H) */
+#endif /* I386_TARGET_CPU_H */
diff --git a/linux-user/i386/target_signal.h b/linux-user/i386/target_signal.h
index 9baf7fb..837e90f 100644
--- a/linux-user/i386/target_signal.h
+++ b/linux-user/i386/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef I386_TARGET_SIGNAL_H
+#define I386_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,4 @@
     return state->regs[R_ESP];
 }
 
-#endif /* TARGET_SIGNAL_H */
+#endif /* I386_TARGET_SIGNAL_H */
diff --git a/linux-user/i386/target_structs.h b/linux-user/i386/target_structs.h
index 65f535e..25388a7 100644
--- a/linux-user/i386/target_structs.h
+++ b/linux-user/i386/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef I386_TARGET_STRUCTS_H
+#define I386_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/i386/target_syscall.h b/linux-user/i386/target_syscall.h
new file mode 100644
index 0000000..b4e895f
--- /dev/null
+++ b/linux-user/i386/target_syscall.h
@@ -0,0 +1,157 @@
+#ifndef I386_TARGET_SYSCALL_H
+#define I386_TARGET_SYSCALL_H
+
+/* default linux values for the selectors */
+#define __USER_CS	(0x23)
+#define __USER_DS	(0x2B)
+
+struct target_pt_regs {
+	long ebx;
+	long ecx;
+	long edx;
+	long esi;
+	long edi;
+	long ebp;
+	long eax;
+	int  xds;
+	int  xes;
+	long orig_eax;
+	long eip;
+	int  xcs;
+	long eflags;
+	long esp;
+	int  xss;
+};
+
+/* ioctls */
+
+#define TARGET_LDT_ENTRIES      8192
+#define TARGET_LDT_ENTRY_SIZE	8
+
+#define TARGET_GDT_ENTRIES             9
+#define TARGET_GDT_ENTRY_TLS_ENTRIES   3
+#define TARGET_GDT_ENTRY_TLS_MIN       6
+#define TARGET_GDT_ENTRY_TLS_MAX       (TARGET_GDT_ENTRY_TLS_MIN + TARGET_GDT_ENTRY_TLS_ENTRIES - 1)
+
+struct target_modify_ldt_ldt_s {
+    unsigned int  entry_number;
+    abi_ulong base_addr;
+    unsigned int limit;
+    unsigned int flags;
+};
+
+/* vm86 defines */
+
+#define TARGET_BIOSSEG		0x0f000
+
+#define TARGET_CPU_086		0
+#define TARGET_CPU_186		1
+#define TARGET_CPU_286		2
+#define TARGET_CPU_386		3
+#define TARGET_CPU_486		4
+#define TARGET_CPU_586		5
+
+#define TARGET_VM86_SIGNAL	0	/* return due to signal */
+#define TARGET_VM86_UNKNOWN	1	/* unhandled GP fault - IO-instruction or similar */
+#define TARGET_VM86_INTx	2	/* int3/int x instruction (ARG = x) */
+#define TARGET_VM86_STI	3	/* sti/popf/iret instruction enabled virtual interrupts */
+
+/*
+ * Additional return values when invoking new vm86()
+ */
+#define TARGET_VM86_PICRETURN	4	/* return due to pending PIC request */
+#define TARGET_VM86_TRAP	6	/* return due to DOS-debugger request */
+
+/*
+ * function codes when invoking new vm86()
+ */
+#define TARGET_VM86_PLUS_INSTALL_CHECK	0
+#define TARGET_VM86_ENTER		1
+#define TARGET_VM86_ENTER_NO_BYPASS	2
+#define	TARGET_VM86_REQUEST_IRQ	3
+#define TARGET_VM86_FREE_IRQ		4
+#define TARGET_VM86_GET_IRQ_BITS	5
+#define TARGET_VM86_GET_AND_RESET_IRQ	6
+
+/*
+ * This is the stack-layout seen by the user space program when we have
+ * done a translation of "SAVE_ALL" from vm86 mode. The real kernel layout
+ * is 'kernel_vm86_regs' (see below).
+ */
+
+struct target_vm86_regs {
+/*
+ * normal regs, with special meaning for the segment descriptors..
+ */
+	abi_long ebx;
+	abi_long ecx;
+	abi_long edx;
+	abi_long esi;
+	abi_long edi;
+	abi_long ebp;
+	abi_long eax;
+	abi_long __null_ds;
+	abi_long __null_es;
+	abi_long __null_fs;
+	abi_long __null_gs;
+	abi_long orig_eax;
+	abi_long eip;
+	unsigned short cs, __csh;
+	abi_long eflags;
+	abi_long esp;
+	unsigned short ss, __ssh;
+/*
+ * these are specific to v86 mode:
+ */
+	unsigned short es, __esh;
+	unsigned short ds, __dsh;
+	unsigned short fs, __fsh;
+	unsigned short gs, __gsh;
+};
+
+struct target_revectored_struct {
+	abi_ulong __map[8];			/* 256 bits */
+};
+
+struct target_vm86_struct {
+	struct target_vm86_regs regs;
+	abi_ulong flags;
+	abi_ulong screen_bitmap;
+	abi_ulong cpu_type;
+	struct target_revectored_struct int_revectored;
+	struct target_revectored_struct int21_revectored;
+};
+
+/*
+ * flags masks
+ */
+#define TARGET_VM86_SCREEN_BITMAP	0x0001
+
+struct target_vm86plus_info_struct {
+        abi_ulong flags;
+#define TARGET_force_return_for_pic (1 << 0)
+#define TARGET_vm86dbg_active       (1 << 1)  /* for debugger */
+#define TARGET_vm86dbg_TFpendig     (1 << 2)  /* for debugger */
+#define TARGET_is_vm86pus           (1 << 31) /* for vm86 internal use */
+	unsigned char vm86dbg_intxxtab[32];   /* for debugger */
+};
+
+struct target_vm86plus_struct {
+	struct target_vm86_regs regs;
+	abi_ulong flags;
+	abi_ulong screen_bitmap;
+	abi_ulong cpu_type;
+	struct target_revectored_struct int_revectored;
+	struct target_revectored_struct int21_revectored;
+	struct target_vm86plus_info_struct vm86plus;
+};
+
+#define UNAME_MACHINE "i686"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* I386_TARGET_SYSCALL_H */
diff --git a/linux-user/ioctls.h b/linux-user/ioctls.h
index e672655..7e2c133 100644
--- a/linux-user/ioctls.h
+++ b/linux-user/ioctls.h
@@ -76,10 +76,39 @@
      IOCTL(BLKFLSBUF, 0, TYPE_NULL)
      IOCTL(BLKRASET, 0, TYPE_INT)
      IOCTL(BLKRAGET, IOC_R, MK_PTR(TYPE_LONG))
-     IOCTL(BLKSSZGET, IOC_R, MK_PTR(TYPE_LONG))
+     IOCTL(BLKSSZGET, IOC_R, MK_PTR(TYPE_INT))
      IOCTL(BLKBSZGET, IOC_R, MK_PTR(TYPE_INT))
      IOCTL_SPECIAL(BLKPG, IOC_W, do_ioctl_blkpg,
                    MK_PTR(MK_STRUCT(STRUCT_blkpg_ioctl_arg)))
+
+#ifdef BLKDISCARD
+     IOCTL(BLKDISCARD, IOC_W, MK_PTR(MK_ARRAY(TYPE_ULONGLONG, 2)))
+#endif
+#ifdef BLKIOMIN
+     IOCTL(BLKIOMIN, IOC_R, MK_PTR(TYPE_INT))
+#endif
+#ifdef BLKIOOPT
+     IOCTL(BLKIOOPT, IOC_R, MK_PTR(TYPE_INT))
+#endif
+#ifdef BLKALIGNOFF
+     IOCTL(BLKALIGNOFF, IOC_R, MK_PTR(TYPE_INT))
+#endif
+#ifdef BLKPBSZGET
+     IOCTL(BLKPBSZGET, IOC_R, MK_PTR(TYPE_INT))
+#endif
+#ifdef BLKDISCARDZEROES
+     IOCTL(BLKDISCARDZEROES, IOC_R, MK_PTR(TYPE_INT))
+#endif
+#ifdef BLKSECDISCARD
+     IOCTL(BLKSECDISCARD, IOC_W, MK_PTR(MK_ARRAY(TYPE_ULONGLONG, 2)))
+#endif
+#ifdef BLKROTATIONAL
+     IOCTL(BLKROTATIONAL, IOC_R, MK_PTR(TYPE_SHORT))
+#endif
+#ifdef BLKZEROOUT
+     IOCTL(BLKZEROOUT, IOC_W, MK_PTR(MK_ARRAY(TYPE_ULONGLONG, 2)))
+#endif
+
 #ifdef FIBMAP
      IOCTL(FIBMAP, IOC_W | IOC_R, MK_PTR(TYPE_LONG))
 #endif
@@ -91,7 +120,7 @@
                    MK_PTR(MK_STRUCT(STRUCT_fiemap)))
 #endif
 
-  IOCTL(SIOCATMARK, 0, TYPE_NULL)
+  IOCTL(SIOCATMARK, IOC_R, MK_PTR(TYPE_INT))
   IOCTL(SIOCGIFNAME, IOC_RW, MK_PTR(TYPE_INT))
   IOCTL(SIOCGIFFLAGS, IOC_W | IOC_R, MK_PTR(MK_STRUCT(STRUCT_short_ifreq)))
   IOCTL(SIOCSIFFLAGS, IOC_W, MK_PTR(MK_STRUCT(STRUCT_short_ifreq)))
@@ -322,11 +351,15 @@
   IOCTL(LOOP_SET_FD, 0, TYPE_INT)
   IOCTL(LOOP_CLR_FD, 0, TYPE_INT)
   IOCTL(LOOP_SET_STATUS, IOC_W, MK_PTR(MK_STRUCT(STRUCT_loop_info)))
-  IOCTL(LOOP_GET_STATUS, IOC_W, MK_PTR(MK_STRUCT(STRUCT_loop_info)))
+  IOCTL(LOOP_GET_STATUS, IOC_R, MK_PTR(MK_STRUCT(STRUCT_loop_info)))
   IOCTL(LOOP_SET_STATUS64, IOC_W, MK_PTR(MK_STRUCT(STRUCT_loop_info64)))
-  IOCTL(LOOP_GET_STATUS64, IOC_W, MK_PTR(MK_STRUCT(STRUCT_loop_info64)))
+  IOCTL(LOOP_GET_STATUS64, IOC_R, MK_PTR(MK_STRUCT(STRUCT_loop_info64)))
   IOCTL(LOOP_CHANGE_FD, 0, TYPE_INT)
 
+  IOCTL(LOOP_CTL_ADD, 0, TYPE_INT)
+  IOCTL(LOOP_CTL_REMOVE, 0, TYPE_INT)
+  IOCTL(LOOP_CTL_GET_FREE, 0, TYPE_NULL)
+
   IOCTL(MTIOCTOP, IOC_W, MK_PTR(MK_STRUCT(STRUCT_mtop)))
   IOCTL(MTIOCGET, IOC_R, MK_PTR(MK_STRUCT(STRUCT_mtget)))
   IOCTL(MTIOCPOS, IOC_R, MK_PTR(MK_STRUCT(STRUCT_mtpos)))
diff --git a/linux-user/linux_loop.h b/linux-user/linux_loop.h
index 8974caa..c69fea1 100644
--- a/linux-user/linux_loop.h
+++ b/linux-user/linux_loop.h
@@ -1,6 +1,9 @@
-/* Copied from 2.6.25 kernel headers to avoid problems on older hosts.  */
-#ifndef _LINUX_LOOP_H
-#define _LINUX_LOOP_H
+/* Copied from 2.6.25 kernel headers to avoid problems on older hosts,
+ * and subsequently updated to match newer additions to the API.
+ */
+
+#ifndef LINUX_LOOP_H
+#define LINUX_LOOP_H
 
 /*
  * include/linux/loop.h
@@ -91,5 +94,12 @@
 #define LOOP_SET_STATUS64	0x4C04
 #define LOOP_GET_STATUS64	0x4C05
 #define LOOP_CHANGE_FD		0x4C06
+#define LOOP_SET_CAPACITY       0x4C07
+#define LOOP_SET_DIRECT_IO      0x4C08
+
+/* /dev/loop-control interface */
+#define LOOP_CTL_ADD            0x4C80
+#define LOOP_CTL_REMOVE         0x4C81
+#define LOOP_CTL_GET_FREE       0x4C82
 
 #endif
diff --git a/linux-user/linuxload.c b/linux-user/linuxload.c
index 506e837..6f0d605 100644
--- a/linux-user/linuxload.c
+++ b/linux-user/linuxload.c
@@ -1,12 +1,6 @@
 /* Code for loading Linux executables.  Mostly linux kernel code.  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <errno.h>
-#include <unistd.h>
-#include <stdio.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
@@ -135,10 +129,7 @@
              struct linux_binprm *bprm)
 {
     int retval;
-    int i;
 
-    bprm->p = TARGET_PAGE_SIZE*MAX_ARG_PAGES-sizeof(unsigned int);
-    memset(bprm->page, 0, sizeof(bprm->page));
     bprm->fd = fdexec;
     bprm->filename = (char *)filename;
     bprm->argc = count(argv);
@@ -172,9 +163,5 @@
         return retval;
     }
 
-    /* Something went wrong, return the inode and free the argument pages*/
-    for (i=0 ; i<MAX_ARG_PAGES ; i++) {
-        g_free(bprm->page[i]);
-    }
     return(retval);
 }
diff --git a/linux-user/m68k-sim.c b/linux-user/m68k-sim.c
index 1994e40..34d332d 100644
--- a/linux-user/m68k-sim.c
+++ b/linux-user/m68k-sim.c
@@ -17,14 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
diff --git a/linux-user/m68k/syscall.h b/linux-user/m68k/syscall.h
deleted file mode 100644
index 9218493..0000000
--- a/linux-user/m68k/syscall.h
+++ /dev/null
@@ -1,25 +0,0 @@
-
-/* this struct defines the way the registers are stored on the
-   stack during a system call. */
-
-struct target_pt_regs {
-    abi_long d1, d2, d3, d4, d5, d6, d7;
-    abi_long a0, a1, a2, a3, a4, a5, a6;
-    abi_ulong d0;
-    abi_ulong usp;
-    abi_ulong orig_d0;
-    int16_t stkadj;
-    uint16_t sr;
-    abi_ulong pc;
-    uint16_t fntvex;
-    uint16_t __fill;
-};
-
-#define UNAME_MACHINE "m68k"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
-
-void do_m68k_simcall(CPUM68KState *, int);
diff --git a/linux-user/m68k/syscall_nr.h b/linux-user/m68k/syscall_nr.h
index 25f8521..4b50fb2 100644
--- a/linux-user/m68k/syscall_nr.h
+++ b/linux-user/m68k/syscall_nr.h
@@ -317,7 +317,7 @@
 #define TARGET_NR_epoll_pwait	315
 #define TARGET_NR_utimensat		316
 #define TARGET_NR_signalfd		317
-#define TARGET_NR_timerfd		318
+#define TARGET_NR_timerfd_create	318
 #define TARGET_NR_eventfd		319
 #define TARGET_NR_fallocate		320
 #define TARGET_NR_timerfd_settime	321
@@ -349,3 +349,30 @@
 #define TARGET_NR_process_vm_writev     346
 #define TARGET_NR_kcmp                  347
 #define TARGET_NR_finit_module          348
+#define TARGET_NR_sched_setattr         349
+#define TARGET_NR_sched_getattr         350
+#define TARGET_NR_renameat2             351
+#define TARGET_NR_getrandom             352
+#define TARGET_NR_memfd_create          353
+#define TARGET_NR_bpf                   354
+#define TARGET_NR_execveat              355
+#define TARGET_NR_socket                356
+#define TARGET_NR_socketpair            357
+#define TARGET_NR_bind                  358
+#define TARGET_NR_connect               359
+#define TARGET_NR_listen                360
+#define TARGET_NR_accept4               361
+#define TARGET_NR_getsockopt            362
+#define TARGET_NR_setsockopt            363
+#define TARGET_NR_getsockname           364
+#define TARGET_NR_getpeername           365
+#define TARGET_NR_sendto                366
+#define TARGET_NR_sendmsg               367
+#define TARGET_NR_recvfrom              368
+#define TARGET_NR_recvmsg               369
+#define TARGET_NR_shutdown              370
+#define TARGET_NR_recvmmsg              371
+#define TARGET_NR_sendmmsg              372
+#define TARGET_NR_userfaultfd           373
+#define TARGET_NR_membarrier            374
+#define TARGET_NR_mlock2                375
diff --git a/linux-user/m68k/target_cpu.h b/linux-user/m68k/target_cpu.h
index bb4d3fa..cc0bfc2 100644
--- a/linux-user/m68k/target_cpu.h
+++ b/linux-user/m68k/target_cpu.h
@@ -18,8 +18,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef M68K_TARGET_CPU_H
+#define M68K_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUM68KState *env, target_ulong newsp)
 {
diff --git a/linux-user/m68k/target_signal.h b/linux-user/m68k/target_signal.h
index 479758a..9d2d734 100644
--- a/linux-user/m68k/target_signal.h
+++ b/linux-user/m68k/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef M68K_TARGET_SIGNAL_H
+#define M68K_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
     return state->aregs[7];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* M68K_TARGET_SIGNAL_H */
diff --git a/linux-user/m68k/target_structs.h b/linux-user/m68k/target_structs.h
index de257c9..a003676 100644
--- a/linux-user/m68k/target_structs.h
+++ b/linux-user/m68k/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef M68K_TARGET_STRUCTS_H
+#define M68K_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/m68k/target_syscall.h b/linux-user/m68k/target_syscall.h
new file mode 100644
index 0000000..db2be4f
--- /dev/null
+++ b/linux-user/m68k/target_syscall.h
@@ -0,0 +1,29 @@
+#ifndef M68K_TARGET_SYSCALL_H
+#define M68K_TARGET_SYSCALL_H
+
+/* this struct defines the way the registers are stored on the
+   stack during a system call. */
+
+struct target_pt_regs {
+    abi_long d1, d2, d3, d4, d5, d6, d7;
+    abi_long a0, a1, a2, a3, a4, a5, a6;
+    abi_ulong d0;
+    abi_ulong usp;
+    abi_ulong orig_d0;
+    int16_t stkadj;
+    uint16_t sr;
+    abi_ulong pc;
+    uint16_t fntvex;
+    uint16_t __fill;
+};
+
+#define UNAME_MACHINE "m68k"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+void do_m68k_simcall(CPUM68KState *, int);
+
+#endif /* M68K_TARGET_SYSCALL_H */
diff --git a/linux-user/main.c b/linux-user/main.c
index 5c14c1e..f2f4d2f 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -16,36 +16,50 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <sys/mman.h>
+#include "qemu/osdep.h"
+#include "qemu-version.h"
 #include <sys/syscall.h>
 #include <sys/resource.h>
 
+#include "qapi/error.h"
 #include "qemu.h"
-#include "qemu-common.h"
+#include "qemu/path.h"
+#include "qemu/config-file.h"
+#include "qemu/cutils.h"
+#include "qemu/help_option.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
 #include "qemu/timer.h"
 #include "qemu/envlist.h"
 #include "elf.h"
+#include "exec/log.h"
+#include "trace/control.h"
+#include "glib-compat.h"
 
 char *exec_path;
 
 int singlestep;
-const char *filename;
-const char *argv0;
-int gdbstub_port;
-envlist_t *envlist;
+static const char *filename;
+static const char *argv0;
+static int gdbstub_port;
+static envlist_t *envlist;
 static const char *cpu_model;
 unsigned long mmap_min_addr;
-#if defined(CONFIG_USE_GUEST_BASE)
 unsigned long guest_base;
 int have_guest_base;
+
+#define EXCP_DUMP(env, fmt, ...)                                        \
+do {                                                                    \
+    CPUState *cs = ENV_GET_CPU(env);                                    \
+    fprintf(stderr, fmt , ## __VA_ARGS__);                              \
+    cpu_dump_state(cs, stderr, fprintf, 0);                             \
+    if (qemu_log_separate()) {                                          \
+        qemu_log(fmt, ## __VA_ARGS__);                                  \
+        log_cpu_state(cs, 0);                                           \
+    }                                                                   \
+} while (0)
+
 #if (TARGET_LONG_BITS == 32) && (HOST_LONG_BITS == 64)
 /*
  * When running 32-on-64 we should make sure we can fit all of the possible
@@ -63,9 +77,8 @@
 #else
 unsigned long reserved_va;
 #endif
-#endif
 
-static void usage(void);
+static void usage(int exitcode);
 
 static const char *interp_prefix = CONFIG_QEMU_INTERP_PREFIX;
 const char *qemu_uname_release;
@@ -107,7 +120,7 @@
 /* Make sure everything is in a consistent state for calling fork().  */
 void fork_start(void)
 {
-    pthread_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+    qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
     pthread_mutex_lock(&exclusive_lock);
     mmap_fork_start();
 }
@@ -121,7 +134,7 @@
            Discard information about the parent threads.  */
         CPU_FOREACH_SAFE(cpu, next_cpu) {
             if (cpu != thread_cpu) {
-                QTAILQ_REMOVE(&cpus, thread_cpu, node);
+                QTAILQ_REMOVE(&cpus, cpu, node);
             }
         }
         pending_cpus = 0;
@@ -129,11 +142,11 @@
         pthread_mutex_init(&cpu_list_mutex, NULL);
         pthread_cond_init(&exclusive_cond, NULL);
         pthread_cond_init(&exclusive_resume, NULL);
-        pthread_mutex_init(&tcg_ctx.tb_ctx.tb_lock, NULL);
-        gdbserver_fork((CPUArchState *)thread_cpu->env_ptr);
+        qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
+        gdbserver_fork(thread_cpu);
     } else {
         pthread_mutex_unlock(&exclusive_lock);
-        pthread_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+        qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
     }
 }
 
@@ -147,7 +160,7 @@
 }
 
 /* Start an exclusive operation.
-   Must only be called from outside cpu_arm_exec.   */
+   Must only be called from outside cpu_exec.   */
 static inline void start_exclusive(void)
 {
     CPUState *other_cpu;
@@ -169,7 +182,7 @@
 }
 
 /* Finish an exclusive operation.  */
-static inline void end_exclusive(void)
+static inline void __attribute__((unused)) end_exclusive(void)
 {
     pending_cpus = 0;
     pthread_cond_broadcast(&exclusive_resume);
@@ -215,13 +228,9 @@
 /***********************************************************/
 /* CPUX86 core interface */
 
-void cpu_smm_update(CPUX86State *env)
-{
-}
-
 uint64_t cpu_get_tsc(CPUX86State *env)
 {
-    return cpu_get_real_ticks();
+    return cpu_get_host_ticks();
 }
 
 static void write_dt(void *ptr, unsigned long addr, unsigned long limit,
@@ -280,40 +289,53 @@
     CPUState *cs = CPU(x86_env_get_cpu(env));
     int trapnr;
     abi_ulong pc;
+    abi_ulong ret;
     target_siginfo_t info;
 
     for(;;) {
-        trapnr = cpu_x86_exec(env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
         switch(trapnr) {
         case 0x80:
             /* linux syscall from int $0x80 */
-            env->regs[R_EAX] = do_syscall(env,
-                                          env->regs[R_EAX],
-                                          env->regs[R_EBX],
-                                          env->regs[R_ECX],
-                                          env->regs[R_EDX],
-                                          env->regs[R_ESI],
-                                          env->regs[R_EDI],
-                                          env->regs[R_EBP],
-                                          0, 0);
+            ret = do_syscall(env,
+                             env->regs[R_EAX],
+                             env->regs[R_EBX],
+                             env->regs[R_ECX],
+                             env->regs[R_EDX],
+                             env->regs[R_ESI],
+                             env->regs[R_EDI],
+                             env->regs[R_EBP],
+                             0, 0);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->eip -= 2;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->regs[R_EAX] = ret;
+            }
             break;
 #ifndef TARGET_ABI32
         case EXCP_SYSCALL:
             /* linux syscall from syscall instruction */
-            env->regs[R_EAX] = do_syscall(env,
-                                          env->regs[R_EAX],
-                                          env->regs[R_EDI],
-                                          env->regs[R_ESI],
-                                          env->regs[R_EDX],
-                                          env->regs[10],
-                                          env->regs[8],
-                                          env->regs[9],
-                                          0, 0);
+            ret = do_syscall(env,
+                             env->regs[R_EAX],
+                             env->regs[R_EDI],
+                             env->regs[R_ESI],
+                             env->regs[R_EDX],
+                             env->regs[10],
+                             env->regs[8],
+                             env->regs[9],
+                             0, 0);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->eip -= 2;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->regs[R_EAX] = ret;
+            }
             break;
 #endif
         case EXCP0B_NOSEG:
         case EXCP0C_STACK:
-            info.si_signo = SIGBUS;
+            info.si_signo = TARGET_SIGBUS;
             info.si_errno = 0;
             info.si_code = TARGET_SI_KERNEL;
             info._sifields._sigfault._addr = 0;
@@ -327,7 +349,7 @@
             } else
 #endif
             {
-                info.si_signo = SIGSEGV;
+                info.si_signo = TARGET_SIGSEGV;
                 info.si_errno = 0;
                 info.si_code = TARGET_SI_KERNEL;
                 info._sifields._sigfault._addr = 0;
@@ -335,7 +357,7 @@
             }
             break;
         case EXCP0E_PAGE:
-            info.si_signo = SIGSEGV;
+            info.si_signo = TARGET_SIGSEGV;
             info.si_errno = 0;
             if (!(env->error_code & 1))
                 info.si_code = TARGET_SEGV_MAPERR;
@@ -352,7 +374,7 @@
 #endif
             {
                 /* division by zero */
-                info.si_signo = SIGFPE;
+                info.si_signo = TARGET_SIGFPE;
                 info.si_errno = 0;
                 info.si_code = TARGET_FPE_INTDIV;
                 info._sifields._sigfault._addr = env->eip;
@@ -367,7 +389,7 @@
             } else
 #endif
             {
-                info.si_signo = SIGTRAP;
+                info.si_signo = TARGET_SIGTRAP;
                 info.si_errno = 0;
                 if (trapnr == EXCP01_DB) {
                     info.si_code = TARGET_TRAP_BRKPT;
@@ -387,7 +409,7 @@
             } else
 #endif
             {
-                info.si_signo = SIGSEGV;
+                info.si_signo = TARGET_SIGSEGV;
                 info.si_errno = 0;
                 info.si_code = TARGET_SI_KERNEL;
                 info._sifields._sigfault._addr = 0;
@@ -395,7 +417,7 @@
             }
             break;
         case EXCP06_ILLOP:
-            info.si_signo = SIGILL;
+            info.si_signo = TARGET_SIGILL;
             info.si_errno = 0;
             info.si_code = TARGET_ILL_ILLOPN;
             info._sifields._sigfault._addr = env->eip;
@@ -420,8 +442,8 @@
             break;
         default:
             pc = env->segs[R_CS].base + env->eip;
-            fprintf(stderr, "qemu: 0x%08lx: unhandled CPU exception 0x%x - aborting\n",
-                    (long)pc, trapnr);
+            EXCP_DUMP(env, "qemu: 0x%08lx: unhandled CPU exception 0x%x - aborting\n",
+                      (long)pc, trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -431,22 +453,54 @@
 
 #ifdef TARGET_ARM
 
-#define get_user_code_u32(x, gaddr, doswap)             \
+#define get_user_code_u32(x, gaddr, env)                \
     ({ abi_long __r = get_user_u32((x), (gaddr));       \
-        if (!__r && (doswap)) {                         \
+        if (!__r && bswap_code(arm_sctlr_b(env))) {     \
             (x) = bswap32(x);                           \
         }                                               \
         __r;                                            \
     })
 
-#define get_user_code_u16(x, gaddr, doswap)             \
+#define get_user_code_u16(x, gaddr, env)                \
     ({ abi_long __r = get_user_u16((x), (gaddr));       \
-        if (!__r && (doswap)) {                         \
+        if (!__r && bswap_code(arm_sctlr_b(env))) {     \
             (x) = bswap16(x);                           \
         }                                               \
         __r;                                            \
     })
 
+#define get_user_data_u32(x, gaddr, env)                \
+    ({ abi_long __r = get_user_u32((x), (gaddr));       \
+        if (!__r && arm_cpu_bswap_data(env)) {          \
+            (x) = bswap32(x);                           \
+        }                                               \
+        __r;                                            \
+    })
+
+#define get_user_data_u16(x, gaddr, env)                \
+    ({ abi_long __r = get_user_u16((x), (gaddr));       \
+        if (!__r && arm_cpu_bswap_data(env)) {          \
+            (x) = bswap16(x);                           \
+        }                                               \
+        __r;                                            \
+    })
+
+#define put_user_data_u32(x, gaddr, env)                \
+    ({ typeof(x) __x = (x);                             \
+        if (arm_cpu_bswap_data(env)) {                  \
+            __x = bswap32(__x);                         \
+        }                                               \
+        put_user_u32(__x, (gaddr));                     \
+    })
+
+#define put_user_data_u16(x, gaddr, env)                \
+    ({ typeof(x) __x = (x);                             \
+        if (arm_cpu_bswap_data(env)) {                  \
+            __x = bswap16(__x);                         \
+        }                                               \
+        put_user_u16(__x, (gaddr));                     \
+    })
+
 #ifdef TARGET_ABI32
 /* Commpage handling -- there is no commpage for AArch64 */
 
@@ -509,7 +563,7 @@
         env->regs[0] = -1;
         cpsr &= ~CPSR_C;
     }
-    cpsr_write(env, cpsr, CPSR_C);
+    cpsr_write(env, cpsr, CPSR_C, CPSRWriteByInstr);
     end_exclusive();
     return;
 
@@ -517,14 +571,12 @@
     end_exclusive();
     /* We get the PC of the entry address - which is as good as anything,
        on a real kernel what you get depends on which mode it uses. */
-    info.si_signo = SIGSEGV;
+    info.si_signo = TARGET_SIGSEGV;
     info.si_errno = 0;
     /* XXX: check env->error_code */
     info.si_code = TARGET_SEGV_MAPERR;
     info._sifields._sigfault._addr = env->exception.vaddress;
     queue_signal(env, info.si_signo, &info);
-
-    end_exclusive();
 }
 
 /* Handle a jump to the kernel code page.  */
@@ -560,11 +612,11 @@
             env->regs[0] = -1;
             cpsr &= ~CPSR_C;
         }
-        cpsr_write(env, cpsr, CPSR_C);
+        cpsr_write(env, cpsr, CPSR_C, CPSRWriteByInstr);
         end_exclusive();
         break;
     case 0xffff0fe0: /* __kernel_get_tls */
-        env->regs[0] = env->cp15.tpidrro_el0;
+        env->regs[0] = cpu_get_tls(env);
         break;
     case 0xffff0f60: /* __kernel_cmpxchg64 */
         arm_kernel_cmpxchg64_helper(env);
@@ -608,11 +660,11 @@
         segv = get_user_u8(val, addr);
         break;
     case 1:
-        segv = get_user_u16(val, addr);
+        segv = get_user_data_u16(val, addr, env);
         break;
     case 2:
     case 3:
-        segv = get_user_u32(val, addr);
+        segv = get_user_data_u32(val, addr, env);
         break;
     default:
         abort();
@@ -623,12 +675,16 @@
     }
     if (size == 3) {
         uint32_t valhi;
-        segv = get_user_u32(valhi, addr + 4);
+        segv = get_user_data_u32(valhi, addr + 4, env);
         if (segv) {
             env->exception.vaddress = addr + 4;
             goto done;
         }
-        val = deposit64(val, 32, 32, valhi);
+        if (arm_cpu_bswap_data(env)) {
+            val = deposit64((uint64_t)valhi, 32, 32, val);
+        } else {
+            val = deposit64(val, 32, 32, valhi);
+        }
     }
     if (val != env->exclusive_val) {
         goto fail;
@@ -640,11 +696,11 @@
         segv = put_user_u8(val, addr);
         break;
     case 1:
-        segv = put_user_u16(val, addr);
+        segv = put_user_data_u16(val, addr, env);
         break;
     case 2:
     case 3:
-        segv = put_user_u32(val, addr);
+        segv = put_user_data_u32(val, addr, env);
         break;
     }
     if (segv) {
@@ -653,7 +709,7 @@
     }
     if (size == 3) {
         val = env->regs[(env->exclusive_info >> 12) & 0xf];
-        segv = put_user_u32(val, addr + 4);
+        segv = put_user_data_u32(val, addr + 4, env);
         if (segv) {
             env->exception.vaddress = addr + 4;
             goto done;
@@ -675,10 +731,11 @@
     unsigned int n, insn;
     target_siginfo_t info;
     uint32_t addr;
+    abi_ulong ret;
 
     for(;;) {
         cpu_exec_start(cs);
-        trapnr = cpu_arm_exec(env);
+        trapnr = cpu_exec(cs);
         cpu_exec_end(cs);
         switch(trapnr) {
         case EXCP_UDEF:
@@ -690,11 +747,11 @@
                 /* we handle the FPU emulation here, as Linux */
                 /* we get the opcode */
                 /* FIXME - what to do if get_user() fails? */
-                get_user_code_u32(opcode, env->regs[15], env->bswap_code);
+                get_user_code_u32(opcode, env->regs[15], env);
 
                 rc = EmulateAll(opcode, &ts->fpa, env);
                 if (rc == 0) { /* illegal instruction */
-                    info.si_signo = SIGILL;
+                    info.si_signo = TARGET_SIGILL;
                     info.si_errno = 0;
                     info.si_code = TARGET_ILL_ILLOPN;
                     info._sifields._sigfault._addr = env->regs[15];
@@ -718,7 +775,7 @@
                     //printf("fpsr 0x%x, arm_fpe 0x%x\n",fpsr,arm_fpe);
 
                     if (fpsr & (arm_fpe << 16)) { /* exception enabled? */
-                      info.si_signo = SIGFPE;
+                      info.si_signo = TARGET_SIGFPE;
                       info.si_errno = 0;
 
                       /* ordered by priority, least first */
@@ -760,25 +817,23 @@
                 if (trapnr == EXCP_BKPT) {
                     if (env->thumb) {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u16(insn, env->regs[15], env->bswap_code);
+                        get_user_code_u16(insn, env->regs[15], env);
                         n = insn & 0xff;
                         env->regs[15] += 2;
                     } else {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u32(insn, env->regs[15], env->bswap_code);
+                        get_user_code_u32(insn, env->regs[15], env);
                         n = (insn & 0xf) | ((insn >> 4) & 0xff0);
                         env->regs[15] += 4;
                     }
                 } else {
                     if (env->thumb) {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u16(insn, env->regs[15] - 2,
-                                          env->bswap_code);
+                        get_user_code_u16(insn, env->regs[15] - 2, env);
                         n = insn & 0xff;
                     } else {
                         /* FIXME - what to do if get_user() fails? */
-                        get_user_code_u32(insn, env->regs[15] - 4,
-                                          env->bswap_code);
+                        get_user_code_u32(insn, env->regs[15] - 4, env);
                         n = insn & 0xffffff;
                     }
                 }
@@ -815,15 +870,20 @@
                             break;
                         }
                     } else {
-                        env->regs[0] = do_syscall(env,
-                                                  n,
-                                                  env->regs[0],
-                                                  env->regs[1],
-                                                  env->regs[2],
-                                                  env->regs[3],
-                                                  env->regs[4],
-                                                  env->regs[5],
-                                                  0, 0);
+                        ret = do_syscall(env,
+                                         n,
+                                         env->regs[0],
+                                         env->regs[1],
+                                         env->regs[2],
+                                         env->regs[3],
+                                         env->regs[4],
+                                         env->regs[5],
+                                         0, 0);
+                        if (ret == -TARGET_ERESTARTSYS) {
+                            env->regs[15] -= env->thumb ? 2 : 4;
+                        } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                            env->regs[0] = ret;
+                        }
                     }
                 } else {
                     goto error;
@@ -842,7 +902,7 @@
         case EXCP_DATA_ABORT:
             addr = env->exception.vaddress;
             {
-                info.si_signo = SIGSEGV;
+                info.si_signo = TARGET_SIGSEGV;
                 info.si_errno = 0;
                 /* XXX: check env->error_code */
                 info.si_code = TARGET_SEGV_MAPERR;
@@ -869,11 +929,12 @@
             if (do_kernel_trap(env))
               goto error;
             break;
+        case EXCP_YIELD:
+            /* nothing to do here for user-mode, just resume guest code */
+            break;
         default:
         error:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -1005,30 +1066,36 @@
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int trapnr, sig;
+    abi_long ret;
     target_siginfo_t info;
 
     for (;;) {
         cpu_exec_start(cs);
-        trapnr = cpu_arm_exec(env);
+        trapnr = cpu_exec(cs);
         cpu_exec_end(cs);
 
         switch (trapnr) {
         case EXCP_SWI:
-            env->xregs[0] = do_syscall(env,
-                                       env->xregs[8],
-                                       env->xregs[0],
-                                       env->xregs[1],
-                                       env->xregs[2],
-                                       env->xregs[3],
-                                       env->xregs[4],
-                                       env->xregs[5],
-                                       0, 0);
+            ret = do_syscall(env,
+                             env->xregs[8],
+                             env->xregs[0],
+                             env->xregs[1],
+                             env->xregs[2],
+                             env->xregs[3],
+                             env->xregs[4],
+                             env->xregs[5],
+                             0, 0);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->pc -= 4;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->xregs[0] = ret;
+            }
             break;
         case EXCP_INTERRUPT:
             /* just indicate that signals should be handled asap */
             break;
         case EXCP_UDEF:
-            info.si_signo = SIGILL;
+            info.si_signo = TARGET_SIGILL;
             info.si_errno = 0;
             info.si_code = TARGET_ILL_ILLOPN;
             info._sifields._sigfault._addr = env->pc;
@@ -1041,7 +1108,7 @@
             /* fall through for segv */
         case EXCP_PREFETCH_ABORT:
         case EXCP_DATA_ABORT:
-            info.si_signo = SIGSEGV;
+            info.si_signo = TARGET_SIGSEGV;
             info.si_errno = 0;
             /* XXX: check env->error_code */
             info.si_code = TARGET_SEGV_MAPERR;
@@ -1058,10 +1125,14 @@
                 queue_signal(env, info.si_signo, &info);
             }
             break;
+        case EXCP_SEMIHOST:
+            env->xregs[0] = do_arm_semihosting(env);
+            break;
+        case EXCP_YIELD:
+            /* nothing to do here for user-mode, just resume guest code */
+            break;
         default:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -1088,7 +1159,7 @@
 
     for (;;) {
         cpu_exec_start(cs);
-        trapnr = uc32_cpu_exec(env);
+        trapnr = cpu_exec(cs);
         cpu_exec_end(cs);
         switch (trapnr) {
         case UC32_EXCP_PRIV:
@@ -1104,7 +1175,7 @@
                             cpu_set_tls(env, env->regs[0]);
                             env->regs[0] = 0;
                     } else {
-                        env->regs[0] = do_syscall(env,
+                        abi_long ret = do_syscall(env,
                                                   n,
                                                   env->regs[0],
                                                   env->regs[1],
@@ -1113,6 +1184,11 @@
                                                   env->regs[4],
                                                   env->regs[5],
                                                   0, 0);
+                        if (ret == -TARGET_ERESTARTSYS) {
+                            env->regs[31] -= 4;
+                        } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                            env->regs[0] = ret;
+                        }
                     }
                 } else {
                     goto error;
@@ -1121,7 +1197,7 @@
             break;
         case UC32_EXCP_DTRAP:
         case UC32_EXCP_ITRAP:
-            info.si_signo = SIGSEGV;
+            info.si_signo = TARGET_SIGSEGV;
             info.si_errno = 0;
             /* XXX: check env->error_code */
             info.si_code = TARGET_SEGV_MAPERR;
@@ -1151,8 +1227,7 @@
     }
 
 error:
-    fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
-    cpu_dump_state(cs, stderr, fprintf, 0);
+    EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
     abort();
 }
 #endif
@@ -1288,7 +1363,9 @@
     target_siginfo_t info;
 
     while (1) {
-        trapnr = cpu_sparc_exec (env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
 
         /* Compute PSR before exposing state.  */
         if (env->cc_op != CC_OP_FLAGS) {
@@ -1308,6 +1385,9 @@
                               env->regwptr[2], env->regwptr[3],
                               env->regwptr[4], env->regwptr[5],
                               0, 0);
+            if (ret == -TARGET_ERESTARTSYS || ret == -TARGET_QEMU_ESIGRETURN) {
+                break;
+            }
             if ((abi_ulong)ret >= (abi_ulong)(-515)) {
 #if defined(TARGET_SPARC64) && !defined(TARGET_ABI32)
                 env->xcc |= PSR_CARRY;
@@ -1415,7 +1495,7 @@
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -1426,8 +1506,7 @@
 #ifdef TARGET_PPC
 static inline uint64_t cpu_ppc_get_tb(CPUPPCState *env)
 {
-    /* TO FIX */
-    return 0;
+    return cpu_get_host_ticks();
 }
 
 uint64_t cpu_ppc_load_tbl(CPUPPCState *env)
@@ -1469,17 +1548,6 @@
     return -1;
 }
 
-#define EXCP_DUMP(env, fmt, ...)                                        \
-do {                                                                    \
-    CPUState *cs = ENV_GET_CPU(env);                                    \
-    fprintf(stderr, fmt , ## __VA_ARGS__);                              \
-    cpu_dump_state(cs, stderr, fprintf, 0);                             \
-    qemu_log(fmt, ## __VA_ARGS__);                                      \
-    if (qemu_log_enabled()) {                                           \
-        log_cpu_state(cs, 0);                                           \
-    }                                                                   \
-} while (0)
-
 static int do_store_exclusive(CPUPPCState *env)
 {
     target_ulong addr;
@@ -1568,7 +1636,7 @@
 
     for(;;) {
         cpu_exec_start(cs);
-        trapnr = cpu_ppc_exec(env);
+        trapnr = cpu_exec(cs);
         cpu_exec_end(cs);
         switch(trapnr) {
         case POWERPC_EXCP_NONE:
@@ -1652,10 +1720,11 @@
             info.si_signo = TARGET_SIGBUS;
             info.si_errno = 0;
             info.si_code = TARGET_BUS_ADRALN;
-            info._sifields._sigfault._addr = env->nip - 4;
+            info._sifields._sigfault._addr = env->nip;
             queue_signal(env, info.si_signo, &info);
             break;
         case POWERPC_EXCP_PROGRAM:  /* Program exception                     */
+        case POWERPC_EXCP_HV_EMU:   /* HV emulation                          */
             /* XXX: check this */
             switch (env->error_code & ~0xF) {
             case POWERPC_EXCP_FP:
@@ -1931,6 +2000,10 @@
             ret = do_syscall(env, env->gpr[0], env->gpr[3], env->gpr[4],
                              env->gpr[5], env->gpr[6], env->gpr[7],
                              env->gpr[8], 0, 0);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->nip -= 4;
+                break;
+            }
             if (ret == (target_ulong)(-TARGET_QEMU_ESIGRETURN)) {
                 /* Returning from a successful sigreturn syscall.
                    Avoid corrupting register state.  */
@@ -2420,7 +2493,7 @@
 
     for(;;) {
         cpu_exec_start(cs);
-        trapnr = cpu_mips_exec(env);
+        trapnr = cpu_exec(cs);
         cpu_exec_end(cs);
         switch(trapnr) {
         case EXCP_SYSCALL:
@@ -2472,6 +2545,10 @@
                              env->active_tc.gpr[8], env->active_tc.gpr[9],
                              env->active_tc.gpr[10], env->active_tc.gpr[11]);
 # endif /* O32 */
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->active_tc.PC -= 4;
+                break;
+            }
             if (ret == -TARGET_QEMU_ESIGRETURN) {
                 /* Returning from a successful sigreturn syscall.
                    Avoid clobbering register state.  */
@@ -2580,7 +2657,7 @@
                         code = (trap_instr >> 6) & 0x3f;
                     }
                 } else {
-                    ret = get_user_ual(trap_instr, env->active_tc.PC);
+                    ret = get_user_u32(trap_instr, env->active_tc.PC);
                     if (ret != 0) {
                         goto error;
                     }
@@ -2614,7 +2691,7 @@
 
                     trap_instr = (instr[0] << 16) | instr[1];
                 } else {
-                    ret = get_user_ual(trap_instr, env->active_tc.PC);
+                    ret = get_user_u32(trap_instr, env->active_tc.PC);
                 }
 
                 if (ret != 0) {
@@ -2638,9 +2715,7 @@
             break;
         default:
 error:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -2654,19 +2729,22 @@
 {
     CPUState *cs = CPU(openrisc_env_get_cpu(env));
     int trapnr, gdbsig;
+    abi_long ret;
 
     for (;;) {
-        trapnr = cpu_exec(env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
         gdbsig = 0;
 
         switch (trapnr) {
         case EXCP_RESET:
-            qemu_log("\nReset request, exit, pc is %#x\n", env->pc);
-            exit(1);
+            qemu_log_mask(CPU_LOG_INT, "\nReset request, exit, pc is %#x\n", env->pc);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_BUSERR:
-            qemu_log("\nBus error, exit, pc is %#x\n", env->pc);
-            gdbsig = SIGBUS;
+            qemu_log_mask(CPU_LOG_INT, "\nBus error, exit, pc is %#x\n", env->pc);
+            gdbsig = TARGET_SIGBUS;
             break;
         case EXCP_DPF:
         case EXCP_IPF:
@@ -2674,59 +2752,63 @@
             gdbsig = TARGET_SIGSEGV;
             break;
         case EXCP_TICK:
-            qemu_log("\nTick time interrupt pc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nTick time interrupt pc is %#x\n", env->pc);
             break;
         case EXCP_ALIGN:
-            qemu_log("\nAlignment pc is %#x\n", env->pc);
-            gdbsig = SIGBUS;
+            qemu_log_mask(CPU_LOG_INT, "\nAlignment pc is %#x\n", env->pc);
+            gdbsig = TARGET_SIGBUS;
             break;
         case EXCP_ILLEGAL:
-            qemu_log("\nIllegal instructionpc is %#x\n", env->pc);
-            gdbsig = SIGILL;
+            qemu_log_mask(CPU_LOG_INT, "\nIllegal instructionpc is %#x\n", env->pc);
+            gdbsig = TARGET_SIGILL;
             break;
         case EXCP_INT:
-            qemu_log("\nExternal interruptpc is %#x\n", env->pc);
+            qemu_log_mask(CPU_LOG_INT, "\nExternal interruptpc is %#x\n", env->pc);
             break;
         case EXCP_DTLBMISS:
         case EXCP_ITLBMISS:
-            qemu_log("\nTLB miss\n");
+            qemu_log_mask(CPU_LOG_INT, "\nTLB miss\n");
             break;
         case EXCP_RANGE:
-            qemu_log("\nRange\n");
-            gdbsig = SIGSEGV;
+            qemu_log_mask(CPU_LOG_INT, "\nRange\n");
+            gdbsig = TARGET_SIGSEGV;
             break;
         case EXCP_SYSCALL:
             env->pc += 4;   /* 0xc00; */
-            env->gpr[11] = do_syscall(env,
-                                      env->gpr[11], /* return value       */
-                                      env->gpr[3],  /* r3 - r7 are params */
-                                      env->gpr[4],
-                                      env->gpr[5],
-                                      env->gpr[6],
-                                      env->gpr[7],
-                                      env->gpr[8], 0, 0);
+            ret = do_syscall(env,
+                             env->gpr[11], /* return value       */
+                             env->gpr[3],  /* r3 - r7 are params */
+                             env->gpr[4],
+                             env->gpr[5],
+                             env->gpr[6],
+                             env->gpr[7],
+                             env->gpr[8], 0, 0);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->pc -= 4;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->gpr[11] = ret;
+            }
             break;
         case EXCP_FPE:
-            qemu_log("\nFloating point error\n");
+            qemu_log_mask(CPU_LOG_INT, "\nFloating point error\n");
             break;
         case EXCP_TRAP:
-            qemu_log("\nTrap\n");
-            gdbsig = SIGTRAP;
+            qemu_log_mask(CPU_LOG_INT, "\nTrap\n");
+            gdbsig = TARGET_SIGTRAP;
             break;
         case EXCP_NR:
-            qemu_log("\nNR\n");
+            qemu_log_mask(CPU_LOG_INT, "\nNR\n");
             break;
         default:
-            qemu_log("\nqemu: unhandled CPU exception %#x - aborting\n",
+            EXCP_DUMP(env, "\nqemu: unhandled CPU exception %#x - aborting\n",
                      trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
             gdbsig = TARGET_SIGILL;
             break;
         }
         if (gdbsig) {
             gdb_handlesig(cs, gdbsig);
             if (gdbsig != TARGET_SIGTRAP) {
-                exit(1);
+                exit(EXIT_FAILURE);
             }
         }
 
@@ -2744,7 +2826,9 @@
     target_siginfo_t info;
 
     while (1) {
-        trapnr = cpu_sh4_exec (env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
 
         switch (trapnr) {
         case 0x160:
@@ -2758,7 +2842,11 @@
                              env->gregs[0],
                              env->gregs[1],
                              0, 0);
-            env->gregs[0] = ret;
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->pc -= 2;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->gregs[0] = ret;
+            }
             break;
         case EXCP_INTERRUPT:
             /* just indicate that signals should be handled asap */
@@ -2779,7 +2867,7 @@
             break;
 	case 0xa0:
 	case 0xc0:
-            info.si_signo = SIGSEGV;
+            info.si_signo = TARGET_SIGSEGV;
             info.si_errno = 0;
             info.si_code = TARGET_SEGV_MAPERR;
             info._sifields._sigfault._addr = env->tea;
@@ -2789,7 +2877,7 @@
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2804,11 +2892,13 @@
     target_siginfo_t info;
     
     while (1) {
-        trapnr = cpu_cris_exec (env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
         switch (trapnr) {
         case 0xaa:
             {
-                info.si_signo = SIGSEGV;
+                info.si_signo = TARGET_SIGSEGV;
                 info.si_errno = 0;
                 /* XXX: check env->error_code */
                 info.si_code = TARGET_SEGV_MAPERR;
@@ -2829,7 +2919,11 @@
                              env->pregs[7], 
                              env->pregs[11],
                              0, 0);
-            env->regs[10] = ret;
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->pc -= 2;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->regs[10] = ret;
+            }
             break;
         case EXCP_DEBUG:
             {
@@ -2848,7 +2942,7 @@
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2863,11 +2957,13 @@
     target_siginfo_t info;
     
     while (1) {
-        trapnr = cpu_mb_exec (env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
         switch (trapnr) {
         case 0xaa:
             {
-                info.si_signo = SIGSEGV;
+                info.si_signo = TARGET_SIGSEGV;
                 info.si_errno = 0;
                 /* XXX: check env->error_code */
                 info.si_code = TARGET_SEGV_MAPERR;
@@ -2891,7 +2987,19 @@
                              env->regs[9], 
                              env->regs[10],
                              0, 0);
-            env->regs[3] = ret;
+            if (ret == -TARGET_ERESTARTSYS) {
+                /* Wind back to before the syscall. */
+                env->sregs[SR_PC] -= 4;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->regs[3] = ret;
+            }
+            /* All syscall exits result in guest r14 being equal to the
+             * PC we return to, because the kernel syscall exit "rtbd" does
+             * this. (This is true even for sigreturn(); note that r14 is
+             * not a userspace-usable register, as the kernel may clobber it
+             * at any point.)
+             */
+            env->regs[14] = env->sregs[SR_PC];
             break;
         case EXCP_HW_EXCP:
             env->regs[17] = env->sregs[SR_PC] + 4;
@@ -2905,14 +3013,14 @@
 
             switch (env->sregs[SR_ESR] & 31) {
                 case ESR_EC_DIVZERO:
-                    info.si_signo = SIGFPE;
+                    info.si_signo = TARGET_SIGFPE;
                     info.si_errno = 0;
                     info.si_code = TARGET_FPE_FLTDIV;
                     info._sifields._sigfault._addr = 0;
                     queue_signal(env, info.si_signo, &info);
                     break;
                 case ESR_EC_FPU:
-                    info.si_signo = SIGFPE;
+                    info.si_signo = TARGET_SIGFPE;
                     info.si_errno = 0;
                     if (env->sregs[SR_FSR] & FSR_IO) {
                         info.si_code = TARGET_FPE_FLTINV;
@@ -2927,7 +3035,7 @@
                     printf ("Unhandled hw-exception: 0x%x\n",
                             env->sregs[SR_ESR] & ESR_EC_MASK);
                     cpu_dump_state(cs, stderr, fprintf, 0);
-                    exit (1);
+                    exit(EXIT_FAILURE);
                     break;
             }
             break;
@@ -2948,7 +3056,7 @@
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -2966,13 +3074,15 @@
     TaskState *ts = cs->opaque;
 
     for(;;) {
-        trapnr = cpu_m68k_exec(env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
         switch(trapnr) {
         case EXCP_ILLEGAL:
             {
                 if (ts->sim_syscalls) {
                     uint16_t nr;
-                    nr = lduw(env->pc + 2);
+                    get_user_u16(nr, env->pc + 2);
                     env->pc += 4;
                     do_m68k_simcall(env, nr);
                 } else {
@@ -2989,7 +3099,7 @@
         case EXCP_LINEF:
         case EXCP_UNSUPPORTED:
         do_sigill:
-            info.si_signo = SIGILL;
+            info.si_signo = TARGET_SIGILL;
             info.si_errno = 0;
             info.si_code = TARGET_ILL_ILLOPN;
             info._sifields._sigfault._addr = env->pc;
@@ -2997,18 +3107,24 @@
             break;
         case EXCP_TRAP0:
             {
+                abi_long ret;
                 ts->sim_syscalls = 0;
                 n = env->dregs[0];
                 env->pc += 2;
-                env->dregs[0] = do_syscall(env,
-                                          n,
-                                          env->dregs[1],
-                                          env->dregs[2],
-                                          env->dregs[3],
-                                          env->dregs[4],
-                                          env->dregs[5],
-                                          env->aregs[0],
-                                          0, 0);
+                ret = do_syscall(env,
+                                 n,
+                                 env->dregs[1],
+                                 env->dregs[2],
+                                 env->dregs[3],
+                                 env->dregs[4],
+                                 env->dregs[5],
+                                 env->aregs[0],
+                                 0, 0);
+                if (ret == -TARGET_ERESTARTSYS) {
+                    env->pc -= 2;
+                } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                    env->dregs[0] = ret;
+                }
             }
             break;
         case EXCP_INTERRUPT:
@@ -3016,7 +3132,7 @@
             break;
         case EXCP_ACCESS:
             {
-                info.si_signo = SIGSEGV;
+                info.si_signo = TARGET_SIGSEGV;
                 info.si_errno = 0;
                 /* XXX: check env->error_code */
                 info.si_code = TARGET_SEGV_MAPERR;
@@ -3039,9 +3155,7 @@
             }
             break;
         default:
-            fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
-                    trapnr);
-            cpu_dump_state(cs, stderr, fprintf, 0);
+            EXCP_DUMP(env, "qemu: unhandled CPU exception 0x%x - aborting\n", trapnr);
             abort();
         }
         process_pending_signals(env);
@@ -3103,7 +3217,9 @@
     abi_long sysret;
 
     while (1) {
-        trapnr = cpu_alpha_exec (env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
 
         /* All of the traps imply a transition through PALcode, which
            implies an REI instruction has been executed.  Which means
@@ -3113,17 +3229,17 @@
         switch (trapnr) {
         case EXCP_RESET:
             fprintf(stderr, "Reset requested. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_MCHK:
             fprintf(stderr, "Machine check exception. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_SMP_INTERRUPT:
         case EXCP_CLK_INTERRUPT:
         case EXCP_DEV_INTERRUPT:
             fprintf(stderr, "External interrupt. Exit\n");
-            exit(1);
+            exit(EXIT_FAILURE);
             break;
         case EXCP_MMFAULT:
             env->lock_addr = -1;
@@ -3189,8 +3305,11 @@
                                     env->ir[IR_A2], env->ir[IR_A3],
                                     env->ir[IR_A4], env->ir[IR_A5],
                                     0, 0);
-                if (trapnr == TARGET_NR_sigreturn
-                    || trapnr == TARGET_NR_rt_sigreturn) {
+                if (sysret == -TARGET_ERESTARTSYS) {
+                    env->pc -= 4;
+                    break;
+                }
+                if (sysret == -TARGET_QEMU_ESIGRETURN) {
                     break;
                 }
                 /* Syscall writes 0 to V0 to bypass error check, similar
@@ -3273,7 +3392,7 @@
         default:
             printf ("Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit (1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3287,9 +3406,12 @@
     int trapnr, n, sig;
     target_siginfo_t info;
     target_ulong addr;
+    abi_long ret;
 
     while (1) {
-        trapnr = cpu_s390x_exec(env);
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
         switch (trapnr) {
         case EXCP_INTERRUPT:
             /* Just indicate that signals should be handled asap.  */
@@ -3302,9 +3424,14 @@
                 n = env->regs[1];
             }
             env->psw.addr += env->int_svc_ilen;
-            env->regs[2] = do_syscall(env, n, env->regs[2], env->regs[3],
-                                      env->regs[4], env->regs[5],
-                                      env->regs[6], env->regs[7], 0, 0);
+            ret = do_syscall(env, n, env->regs[2], env->regs[3],
+                             env->regs[4], env->regs[5],
+                             env->regs[6], env->regs[7], 0, 0);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->psw.addr -= env->int_svc_ilen;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->regs[2] = ret;
+            }
             break;
 
         case EXCP_DEBUG:
@@ -3319,12 +3446,12 @@
             switch (n) {
             case PGM_OPERATION:
             case PGM_PRIVILEGED:
-                sig = SIGILL;
+                sig = TARGET_SIGILL;
                 n = TARGET_ILL_ILLOPC;
                 goto do_signal_pc;
             case PGM_PROTECTION:
             case PGM_ADDRESSING:
-                sig = SIGSEGV;
+                sig = TARGET_SIGSEGV;
                 /* XXX: check env->error_code */
                 n = TARGET_SEGV_MAPERR;
                 addr = env->__excp_addr;
@@ -3334,16 +3461,16 @@
             case PGM_SPECIAL_OP:
             case PGM_OPERAND:
             do_sigill_opn:
-                sig = SIGILL;
+                sig = TARGET_SIGILL;
                 n = TARGET_ILL_ILLOPN;
                 goto do_signal_pc;
 
             case PGM_FIXPT_OVERFLOW:
-                sig = SIGFPE;
+                sig = TARGET_SIGFPE;
                 n = TARGET_FPE_INTOVF;
                 goto do_signal_pc;
             case PGM_FIXPT_DIVIDE:
-                sig = SIGFPE;
+                sig = TARGET_SIGFPE;
                 n = TARGET_FPE_INTDIV;
                 goto do_signal_pc;
 
@@ -3368,14 +3495,14 @@
                         /* ??? Quantum exception; BFP, DFP error.  */
                         goto do_sigill_opn;
                     }
-                    sig = SIGFPE;
+                    sig = TARGET_SIGFPE;
                     goto do_signal_pc;
                 }
 
             default:
                 fprintf(stderr, "Unhandled program exception: %#x\n", n);
                 cpu_dump_state(cs, stderr, fprintf, 0);
-                exit(1);
+                exit(EXIT_FAILURE);
             }
             break;
 
@@ -3392,7 +3519,7 @@
         default:
             fprintf(stderr, "Unhandled trap: 0x%x\n", trapnr);
             cpu_dump_state(cs, stderr, fprintf, 0);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
         process_pending_signals (env);
     }
@@ -3400,6 +3527,257 @@
 
 #endif /* TARGET_S390X */
 
+#ifdef TARGET_TILEGX
+
+static void gen_sigill_reg(CPUTLGState *env)
+{
+    target_siginfo_t info;
+
+    info.si_signo = TARGET_SIGILL;
+    info.si_errno = 0;
+    info.si_code = TARGET_ILL_PRVREG;
+    info._sifields._sigfault._addr = env->pc;
+    queue_signal(env, info.si_signo, &info);
+}
+
+static void do_signal(CPUTLGState *env, int signo, int sigcode)
+{
+    target_siginfo_t info;
+
+    info.si_signo = signo;
+    info.si_errno = 0;
+    info._sifields._sigfault._addr = env->pc;
+
+    if (signo == TARGET_SIGSEGV) {
+        /* The passed in sigcode is a dummy; check for a page mapping
+           and pass either MAPERR or ACCERR.  */
+        target_ulong addr = env->excaddr;
+        info._sifields._sigfault._addr = addr;
+        if (page_check_range(addr, 1, PAGE_VALID) < 0) {
+            sigcode = TARGET_SEGV_MAPERR;
+        } else {
+            sigcode = TARGET_SEGV_ACCERR;
+        }
+    }
+    info.si_code = sigcode;
+
+    queue_signal(env, info.si_signo, &info);
+}
+
+static void gen_sigsegv_maperr(CPUTLGState *env, target_ulong addr)
+{
+    env->excaddr = addr;
+    do_signal(env, TARGET_SIGSEGV, 0);
+}
+
+static void set_regval(CPUTLGState *env, uint8_t reg, uint64_t val)
+{
+    if (unlikely(reg >= TILEGX_R_COUNT)) {
+        switch (reg) {
+        case TILEGX_R_SN:
+        case TILEGX_R_ZERO:
+            return;
+        case TILEGX_R_IDN0:
+        case TILEGX_R_IDN1:
+        case TILEGX_R_UDN0:
+        case TILEGX_R_UDN1:
+        case TILEGX_R_UDN2:
+        case TILEGX_R_UDN3:
+            gen_sigill_reg(env);
+            return;
+        default:
+            g_assert_not_reached();
+        }
+    }
+    env->regs[reg] = val;
+}
+
+/*
+ * Compare the 8-byte contents of the CmpValue SPR with the 8-byte value in
+ * memory at the address held in the first source register. If the values are
+ * not equal, then no memory operation is performed. If the values are equal,
+ * the 8-byte quantity from the second source register is written into memory
+ * at the address held in the first source register. In either case, the result
+ * of the instruction is the value read from memory. The compare and write to
+ * memory are atomic and thus can be used for synchronization purposes. This
+ * instruction only operates for addresses aligned to a 8-byte boundary.
+ * Unaligned memory access causes an Unaligned Data Reference interrupt.
+ *
+ * Functional Description (64-bit)
+ *       uint64_t memVal = memoryReadDoubleWord (rf[SrcA]);
+ *       rf[Dest] = memVal;
+ *       if (memVal == SPR[CmpValueSPR])
+ *           memoryWriteDoubleWord (rf[SrcA], rf[SrcB]);
+ *
+ * Functional Description (32-bit)
+ *       uint64_t memVal = signExtend32 (memoryReadWord (rf[SrcA]));
+ *       rf[Dest] = memVal;
+ *       if (memVal == signExtend32 (SPR[CmpValueSPR]))
+ *           memoryWriteWord (rf[SrcA], rf[SrcB]);
+ *
+ *
+ * This function also processes exch and exch4 which need not process SPR.
+ */
+static void do_exch(CPUTLGState *env, bool quad, bool cmp)
+{
+    target_ulong addr;
+    target_long val, sprval;
+
+    start_exclusive();
+
+    addr = env->atomic_srca;
+    if (quad ? get_user_s64(val, addr) : get_user_s32(val, addr)) {
+        goto sigsegv_maperr;
+    }
+
+    if (cmp) {
+        if (quad) {
+            sprval = env->spregs[TILEGX_SPR_CMPEXCH];
+        } else {
+            sprval = sextract64(env->spregs[TILEGX_SPR_CMPEXCH], 0, 32);
+        }
+    }
+
+    if (!cmp || val == sprval) {
+        target_long valb = env->atomic_srcb;
+        if (quad ? put_user_u64(valb, addr) : put_user_u32(valb, addr)) {
+            goto sigsegv_maperr;
+        }
+    }
+
+    set_regval(env, env->atomic_dstr, val);
+    end_exclusive();
+    return;
+
+ sigsegv_maperr:
+    end_exclusive();
+    gen_sigsegv_maperr(env, addr);
+}
+
+static void do_fetch(CPUTLGState *env, int trapnr, bool quad)
+{
+    int8_t write = 1;
+    target_ulong addr;
+    target_long val, valb;
+
+    start_exclusive();
+
+    addr = env->atomic_srca;
+    valb = env->atomic_srcb;
+    if (quad ? get_user_s64(val, addr) : get_user_s32(val, addr)) {
+        goto sigsegv_maperr;
+    }
+
+    switch (trapnr) {
+    case TILEGX_EXCP_OPCODE_FETCHADD:
+    case TILEGX_EXCP_OPCODE_FETCHADD4:
+        valb += val;
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHADDGEZ:
+        valb += val;
+        if (valb < 0) {
+            write = 0;
+        }
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHADDGEZ4:
+        valb += val;
+        if ((int32_t)valb < 0) {
+            write = 0;
+        }
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHAND:
+    case TILEGX_EXCP_OPCODE_FETCHAND4:
+        valb &= val;
+        break;
+    case TILEGX_EXCP_OPCODE_FETCHOR:
+    case TILEGX_EXCP_OPCODE_FETCHOR4:
+        valb |= val;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if (write) {
+        if (quad ? put_user_u64(valb, addr) : put_user_u32(valb, addr)) {
+            goto sigsegv_maperr;
+        }
+    }
+
+    set_regval(env, env->atomic_dstr, val);
+    end_exclusive();
+    return;
+
+ sigsegv_maperr:
+    end_exclusive();
+    gen_sigsegv_maperr(env, addr);
+}
+
+void cpu_loop(CPUTLGState *env)
+{
+    CPUState *cs = CPU(tilegx_env_get_cpu(env));
+    int trapnr;
+
+    while (1) {
+        cpu_exec_start(cs);
+        trapnr = cpu_exec(cs);
+        cpu_exec_end(cs);
+        switch (trapnr) {
+        case TILEGX_EXCP_SYSCALL:
+        {
+            abi_ulong ret = do_syscall(env, env->regs[TILEGX_R_NR],
+                                       env->regs[0], env->regs[1],
+                                       env->regs[2], env->regs[3],
+                                       env->regs[4], env->regs[5],
+                                       env->regs[6], env->regs[7]);
+            if (ret == -TARGET_ERESTARTSYS) {
+                env->pc -= 8;
+            } else if (ret != -TARGET_QEMU_ESIGRETURN) {
+                env->regs[TILEGX_R_RE] = ret;
+                env->regs[TILEGX_R_ERR] = TILEGX_IS_ERRNO(ret) ? -ret : 0;
+            }
+            break;
+        }
+        case TILEGX_EXCP_OPCODE_EXCH:
+            do_exch(env, true, false);
+            break;
+        case TILEGX_EXCP_OPCODE_EXCH4:
+            do_exch(env, false, false);
+            break;
+        case TILEGX_EXCP_OPCODE_CMPEXCH:
+            do_exch(env, true, true);
+            break;
+        case TILEGX_EXCP_OPCODE_CMPEXCH4:
+            do_exch(env, false, true);
+            break;
+        case TILEGX_EXCP_OPCODE_FETCHADD:
+        case TILEGX_EXCP_OPCODE_FETCHADDGEZ:
+        case TILEGX_EXCP_OPCODE_FETCHAND:
+        case TILEGX_EXCP_OPCODE_FETCHOR:
+            do_fetch(env, trapnr, true);
+            break;
+        case TILEGX_EXCP_OPCODE_FETCHADD4:
+        case TILEGX_EXCP_OPCODE_FETCHADDGEZ4:
+        case TILEGX_EXCP_OPCODE_FETCHAND4:
+        case TILEGX_EXCP_OPCODE_FETCHOR4:
+            do_fetch(env, trapnr, false);
+            break;
+        case TILEGX_EXCP_SIGNAL:
+            do_signal(env, env->signo, env->sigcode);
+            break;
+        case TILEGX_EXCP_REG_IDN_ACCESS:
+        case TILEGX_EXCP_REG_UDN_ACCESS:
+            gen_sigill_reg(env);
+            break;
+        default:
+            fprintf(stderr, "trapnr is %d[0x%x].\n", trapnr, trapnr);
+            g_assert_not_reached();
+        }
+        process_pending_signals(env);
+    }
+}
+
+#endif
+
 THREAD CPUState *thread_cpu;
 
 void task_settid(TaskState *ts)
@@ -3421,25 +3799,16 @@
 /* Assumes contents are already zeroed.  */
 void init_task_state(TaskState *ts)
 {
-    int i;
- 
     ts->used = 1;
-    ts->first_free = ts->sigqueue_table;
-    for (i = 0; i < MAX_SIGQUEUE_SIZE - 1; i++) {
-        ts->sigqueue_table[i].next = &ts->sigqueue_table[i + 1];
-    }
-    ts->sigqueue_table[i].next = NULL;
 }
 
 CPUArchState *cpu_copy(CPUArchState *env)
 {
     CPUState *cpu = ENV_GET_CPU(env);
-    CPUArchState *new_env = cpu_init(cpu_model);
-    CPUState *new_cpu = ENV_GET_CPU(new_env);
-#if defined(TARGET_HAS_ICE)
+    CPUState *new_cpu = cpu_init(cpu_model);
+    CPUArchState *new_env = new_cpu->env_ptr;
     CPUBreakpoint *bp;
     CPUWatchpoint *wp;
-#endif
 
     /* Reset non arch specific state */
     cpu_reset(new_cpu);
@@ -3449,23 +3818,21 @@
     /* Clone all break/watchpoints.
        Note: Once we support ptrace with hw-debug register access, make sure
        BP_CPU break/watchpoints are handled correctly on clone. */
-    QTAILQ_INIT(&cpu->breakpoints);
-    QTAILQ_INIT(&cpu->watchpoints);
-#if defined(TARGET_HAS_ICE)
+    QTAILQ_INIT(&new_cpu->breakpoints);
+    QTAILQ_INIT(&new_cpu->watchpoints);
     QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
         cpu_breakpoint_insert(new_cpu, bp->pc, bp->flags, NULL);
     }
     QTAILQ_FOREACH(wp, &cpu->watchpoints, entry) {
         cpu_watchpoint_insert(new_cpu, wp->vaddr, wp->len, wp->flags, NULL);
     }
-#endif
 
     return new_env;
 }
 
 static void handle_arg_help(const char *arg)
 {
-    usage();
+    usage(EXIT_SUCCESS);
 }
 
 static void handle_arg_log(const char *arg)
@@ -3475,14 +3842,15 @@
     mask = qemu_str_to_log_mask(arg);
     if (!mask) {
         qemu_print_log_usage(stdout);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
+    qemu_log_needs_buffers();
     qemu_set_log(mask);
 }
 
 static void handle_arg_log_filename(const char *arg)
 {
-    qemu_set_log_filename(arg);
+    qemu_set_log_filename(arg, &error_fatal);
 }
 
 static void handle_arg_set_env(const char *arg)
@@ -3491,7 +3859,7 @@
     r = p = strdup(arg);
     while ((token = strsep(&p, ",")) != NULL) {
         if (envlist_setenv(envlist, token) != 0) {
-            usage();
+            usage(EXIT_FAILURE);
         }
     }
     free(r);
@@ -3503,7 +3871,7 @@
     r = p = strdup(arg);
     while ((token = strsep(&p, ",")) != NULL) {
         if (envlist_unsetenv(envlist, token) != 0) {
-            usage();
+            usage(EXIT_FAILURE);
         }
     }
     free(r);
@@ -3519,7 +3887,7 @@
     char *p;
     guest_stack_size = strtoul(arg, &p, 0);
     if (guest_stack_size == 0) {
-        usage();
+        usage(EXIT_FAILURE);
     }
 
     if (*p == 'M') {
@@ -3540,7 +3908,7 @@
     if (qemu_host_page_size == 0 ||
         (qemu_host_page_size & (qemu_host_page_size - 1)) != 0) {
         fprintf(stderr, "page size must be a power of two\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
 
@@ -3550,7 +3918,7 @@
 
     if (parse_uint_full(arg, &seed, 0) != 0 || seed > UINT_MAX) {
         fprintf(stderr, "Invalid seed number: %s\n", arg);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     srand(seed);
 }
@@ -3573,11 +3941,10 @@
 #if defined(cpu_list)
         cpu_list(stdout, &fprintf);
 #endif
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
 
-#if defined(CONFIG_USE_GUEST_BASE)
 static void handle_arg_guest_base(const char *arg)
 {
     guest_base = strtol(arg, NULL, 0);
@@ -3611,15 +3978,14 @@
 #endif
             ) {
             fprintf(stderr, "Reserved virtual address too big\n");
-            exit(1);
+            exit(EXIT_FAILURE);
         }
     }
     if (*p) {
         fprintf(stderr, "Unrecognised -R size suffix '%s'\n", p);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 }
-#endif
 
 static void handle_arg_singlestep(const char *arg)
 {
@@ -3634,8 +4000,15 @@
 static void handle_arg_version(const char *arg)
 {
     printf("qemu-" TARGET_NAME " version " QEMU_VERSION QEMU_PKGVERSION
-           ", Copyright (c) 2003-2008 Fabrice Bellard\n");
-    exit(0);
+           ", " QEMU_COPYRIGHT "\n");
+    exit(EXIT_SUCCESS);
+}
+
+static char *trace_file;
+static void handle_arg_trace(const char *arg)
+{
+    g_free(trace_file);
+    trace_file = trace_opt_parse(arg);
 }
 
 struct qemu_argument {
@@ -3650,6 +4023,8 @@
 static const struct qemu_argument arg_table[] = {
     {"h",          "",                 false, handle_arg_help,
      "",           "print this help"},
+    {"help",       "",                 false, handle_arg_help,
+     "",           ""},
     {"g",          "QEMU_GDB",         true,  handle_arg_gdb,
      "port",       "wait gdb connection to 'port'"},
     {"L",          "QEMU_LD_PREFIX",   true,  handle_arg_ld_prefix,
@@ -3666,12 +4041,10 @@
      "argv0",      "forces target process argv[0] to be 'argv0'"},
     {"r",          "QEMU_UNAME",       true,  handle_arg_uname,
      "uname",      "set qemu uname release string to 'uname'"},
-#if defined(CONFIG_USE_GUEST_BASE)
     {"B",          "QEMU_GUEST_BASE",  true,  handle_arg_guest_base,
      "address",    "set guest_base address to 'address'"},
     {"R",          "QEMU_RESERVED_VA", true,  handle_arg_reserved_va,
      "size",       "reserve 'size' bytes for guest virtual address space"},
-#endif
     {"d",          "QEMU_LOG",         true,  handle_arg_log,
      "item[,...]", "enable logging of specified items "
      "(use '-d help' for a list of items)"},
@@ -3685,12 +4058,14 @@
      "",           "log system calls"},
     {"seed",       "QEMU_RAND_SEED",   true,  handle_arg_randseed,
      "",           "Seed for pseudo-random number generator"},
+    {"trace",      "QEMU_TRACE",       true,  handle_arg_trace,
+     "",           "[[enable=]<pattern>][,events=<file>][,file=<file>]"},
     {"version",    "QEMU_VERSION",     false, handle_arg_version,
      "",           "display version information and exit"},
     {NULL, NULL, false, NULL, NULL, NULL}
 };
 
-static void usage(void)
+static void usage(int exitcode)
 {
     const struct qemu_argument *arginfo;
     int maxarglen;
@@ -3757,7 +4132,7 @@
            "Note that if you provide several changes to a single variable\n"
            "the last change will stay in effect.\n");
 
-    exit(1);
+    exit(exitcode);
 }
 
 static int parse_args(int argc, char **argv)
@@ -3791,12 +4166,18 @@
         if (!strcmp(r, "-")) {
             break;
         }
+        /* Treat --foo the same as -foo.  */
+        if (r[0] == '-') {
+            r++;
+        }
 
         for (arginfo = arg_table; arginfo->handle_opt != NULL; arginfo++) {
             if (!strcmp(r, arginfo->argv)) {
                 if (arginfo->has_arg) {
                     if (optind >= argc) {
-                        usage();
+                        (void) fprintf(stderr,
+                            "qemu: missing argument for option '%s'\n", r);
+                        exit(EXIT_FAILURE);
                     }
                     arginfo->handle_opt(argv[optind]);
                     optind++;
@@ -3809,12 +4190,14 @@
 
         /* no option matched the current argv */
         if (arginfo->handle_opt == NULL) {
-            usage();
+            (void) fprintf(stderr, "qemu: unknown option '%s'\n", r);
+            exit(EXIT_FAILURE);
         }
     }
 
     if (optind >= argc) {
-        usage();
+        (void) fprintf(stderr, "qemu: no user program specified\n");
+        exit(EXIT_FAILURE);
     }
 
     filename = argv[optind];
@@ -3843,7 +4226,7 @@
 
     if ((envlist = envlist_create()) == NULL) {
         (void) fprintf(stderr, "Unable to allocate envlist\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 
     /* add current environment into the list */
@@ -3863,14 +4246,18 @@
     }
 
     cpu_model = NULL;
-#if defined(cpudef_setup)
-    cpudef_setup(); /* parse cpu definitions in target config file (TBD) */
-#endif
 
     srand(time(NULL));
 
+    qemu_add_opts(&qemu_trace_opts);
+
     optind = parse_args(argc, argv);
 
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+
     /* Zero out regs */
     memset(regs, 0, sizeof(struct target_pt_regs));
 
@@ -3905,7 +4292,7 @@
 #endif
 #elif defined(TARGET_MIPS)
 #if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64)
-        cpu_model = "20Kc";
+        cpu_model = "5KEf";
 #else
         cpu_model = "24Kf";
 #endif
@@ -3913,24 +4300,25 @@
         cpu_model = "or1200";
 #elif defined(TARGET_PPC)
 # ifdef TARGET_PPC64
-        cpu_model = "POWER7";
+        cpu_model = "POWER8";
 # else
         cpu_model = "750";
 # endif
+#elif defined TARGET_SH4
+        cpu_model = TYPE_SH7785_CPU;
 #else
         cpu_model = "any";
 #endif
     }
     tcg_exec_init(0);
-    cpu_exec_init_all();
     /* NOTE: we need to init the CPU at this stage to get
        qemu_host_page_size */
-    env = cpu_init(cpu_model);
-    if (!env) {
+    cpu = cpu_init(cpu_model);
+    if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
-    cpu = ENV_GET_CPU(env);
+    env = cpu->env_ptr;
     cpu_reset(cpu);
 
     thread_cpu = cpu;
@@ -3946,7 +4334,6 @@
     target_environ = envlist_to_environ(envlist, NULL);
     envlist_free(envlist);
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /*
      * Now that page sizes are configured in cpu_init() we can do
      * proper page alignment for guest_base.
@@ -3961,14 +4348,13 @@
                     "space for use as guest address space (check your virtual "
                     "memory ulimit setting or reserve less using -R option)\n",
                     reserved_va);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
 
         if (reserved_va) {
             mmap_next_start = reserved_va;
         }
     }
-#endif /* CONFIG_USE_GUEST_BASE */
 
     /*
      * Read in mmap_min_addr kernel parameter.  This value is used
@@ -3982,7 +4368,7 @@
             unsigned long tmp;
             if (fscanf(fp, "%lu", &tmp) == 1) {
                 mmap_min_addr = tmp;
-                qemu_log("host mmap_min_addr=0x%lx\n", mmap_min_addr);
+                qemu_log_mask(CPU_LOG_PAGE, "host mmap_min_addr=0x%lx\n", mmap_min_addr);
             }
             fclose(fp);
         }
@@ -3995,7 +4381,7 @@
     target_argv = calloc(target_argc + 1, sizeof (char *));
     if (target_argv == NULL) {
 	(void) fprintf(stderr, "Unable to allocate memory for target_argv\n");
-	exit(1);
+	exit(EXIT_FAILURE);
     }
 
     /*
@@ -4011,7 +4397,7 @@
     }
     target_argv[target_argc] = NULL;
 
-    ts = g_malloc0 (sizeof(TaskState));
+    ts = g_new0(TaskState, 1);
     init_task_state(ts);
     /* build Task State */
     ts->info = info;
@@ -4024,7 +4410,7 @@
         execfd = open(filename, O_RDONLY);
         if (execfd < 0) {
             printf("Error while loading %s: %s\n", filename, strerror(errno));
-            _exit(1);
+            _exit(EXIT_FAILURE);
         }
     }
 
@@ -4032,7 +4418,7 @@
         info, &bprm);
     if (ret != 0) {
         printf("Error while loading %s: %s\n", filename, strerror(-ret));
-        _exit(1);
+        _exit(EXIT_FAILURE);
     }
 
     for (wrk = target_environ; *wrk; wrk++) {
@@ -4041,10 +4427,8 @@
 
     free(target_environ);
 
-    if (qemu_log_enabled()) {
-#if defined(CONFIG_USE_GUEST_BASE)
+    if (qemu_loglevel_mask(CPU_LOG_PAGE)) {
         qemu_log("guest_base  0x%lx\n", guest_base);
-#endif
         log_page_dump();
 
         qemu_log("start_brk   0x" TARGET_ABI_FMT_lx "\n", info->start_brk);
@@ -4064,12 +4448,10 @@
     syscall_init();
     signal_init();
 
-#if defined(CONFIG_USE_GUEST_BASE)
     /* Now that we've loaded the binary, GUEST_BASE is fixed.  Delay
        generating the prologue until now so that the prologue can take
        the real value of GUEST_BASE into account.  */
     tcg_prologue_init(&tcg_ctx);
-#endif
 
 #if defined(TARGET_I386)
     env->cr[0] = CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK;
@@ -4082,7 +4464,7 @@
     /* enable 64 bit mode if possible */
     if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_LM)) {
         fprintf(stderr, "The selected x86 CPU does not support 64 bit mode\n");
-        exit(1);
+        exit(EXIT_FAILURE);
     }
     env->cr[4] |= CR4_PAE_MASK;
     env->efer |= MSR_EFER_LMA | MSR_EFER_LME;
@@ -4192,7 +4574,7 @@
         if (!(arm_feature(env, ARM_FEATURE_AARCH64))) {
             fprintf(stderr,
                     "The selected ARM CPU does not support 64 bit mode\n");
-            exit(1);
+            exit(EXIT_FAILURE);
         }
 
         for (i = 0; i < 31; i++) {
@@ -4204,15 +4586,21 @@
 #elif defined(TARGET_ARM)
     {
         int i;
-        cpsr_write(env, regs->uregs[16], 0xffffffff);
+        cpsr_write(env, regs->uregs[16], CPSR_USER | CPSR_EXEC,
+                   CPSRWriteByInstr);
         for(i = 0; i < 16; i++) {
             env->regs[i] = regs->uregs[i];
         }
+#ifdef TARGET_WORDS_BIGENDIAN
         /* Enable BE8.  */
         if (EF_ARM_EABI_VERSION(info->elf_flags) >= EF_ARM_EABI_VER4
             && (info->elf_flags & EF_ARM_BE8)) {
-            env->bswap_code = 1;
+            env->uncached_cpsr |= CPSR_E;
+            env->cp15.sctlr_el[1] |= SCTLR_E0E;
+        } else {
+            env->cp15.sctlr_el[1] |= SCTLR_B;
         }
+#endif
     }
 #elif defined(TARGET_UNICORE32)
     {
@@ -4318,6 +4706,20 @@
         if (regs->cp0_epc & 1) {
             env->hflags |= MIPS_HFLAG_M16;
         }
+        if (((info->elf_flags & EF_MIPS_NAN2008) != 0) !=
+            ((env->active_fpu.fcr31 & (1 << FCR31_NAN2008)) != 0)) {
+            if ((env->active_fpu.fcr31_rw_bitmask &
+                  (1 << FCR31_NAN2008)) == 0) {
+                fprintf(stderr, "ELF binary's NaN mode not supported by CPU\n");
+                exit(1);
+            }
+            if ((info->elf_flags & EF_MIPS_NAN2008) != 0) {
+                env->active_fpu.fcr31 |= (1 << FCR31_NAN2008);
+            } else {
+                env->active_fpu.fcr31 &= ~(1 << FCR31_NAN2008);
+            }
+            restore_snan_bit_mode(env);
+        }
     }
 #elif defined(TARGET_OPENRISC)
     {
@@ -4378,6 +4780,17 @@
             env->psw.mask = regs->psw.mask;
             env->psw.addr = regs->psw.addr;
     }
+#elif defined(TARGET_TILEGX)
+    {
+        int i;
+        for (i = 0; i < TILEGX_R_COUNT; i++) {
+            env->regs[i] = regs->regs[i];
+        }
+        for (i = 0; i < TILEGX_SPR_COUNT; i++) {
+            env->spregs[i] = 0;
+        }
+        env->pc = regs->pc;
+    }
 #else
 #error unsupported target CPU
 #endif
@@ -4393,10 +4806,11 @@
         if (gdbserver_start(gdbstub_port) < 0) {
             fprintf(stderr, "qemu: could not open gdbserver on port %d\n",
                     gdbstub_port);
-            exit(1);
+            exit(EXIT_FAILURE);
         }
         gdb_handlesig(cpu, 0);
     }
+    trace_init_vcpu_events();
     cpu_loop(env);
     /* never exits */
     return 0;
diff --git a/linux-user/microblaze/syscall.h b/linux-user/microblaze/syscall.h
deleted file mode 100644
index 3c1ed27..0000000
--- a/linux-user/microblaze/syscall.h
+++ /dev/null
@@ -1,56 +0,0 @@
-#ifndef MICROBLAZE_SYSCALLS_H
-#define MICROBLAZE_SYSCALLS_H 1
-
-#define UNAME_MACHINE "microblaze"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-/* We use microblaze_reg_t to keep things similar to the kernel sources.  */
-typedef uint32_t microblaze_reg_t;
-
-struct target_pt_regs {
-        microblaze_reg_t r0;
-        microblaze_reg_t r1;
-        microblaze_reg_t r2;
-        microblaze_reg_t r3;
-        microblaze_reg_t r4;
-        microblaze_reg_t r5;
-        microblaze_reg_t r6;
-        microblaze_reg_t r7;
-        microblaze_reg_t r8;
-        microblaze_reg_t r9;
-        microblaze_reg_t r10;
-        microblaze_reg_t r11;
-        microblaze_reg_t r12;
-        microblaze_reg_t r13;
-        microblaze_reg_t r14;
-        microblaze_reg_t r15;
-        microblaze_reg_t r16;
-        microblaze_reg_t r17;
-        microblaze_reg_t r18;
-        microblaze_reg_t r19;
-        microblaze_reg_t r20;
-        microblaze_reg_t r21;
-        microblaze_reg_t r22;
-        microblaze_reg_t r23;
-        microblaze_reg_t r24;
-        microblaze_reg_t r25;
-        microblaze_reg_t r26;
-        microblaze_reg_t r27;
-        microblaze_reg_t r28;
-        microblaze_reg_t r29;
-        microblaze_reg_t r30;
-        microblaze_reg_t r31;
-        microblaze_reg_t pc;
-        microblaze_reg_t msr;
-        microblaze_reg_t ear;
-        microblaze_reg_t esr;
-        microblaze_reg_t fsr;
-        uint32_t kernel_mode;
-};
-
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ      2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
-
-#endif
diff --git a/linux-user/microblaze/syscall_nr.h b/linux-user/microblaze/syscall_nr.h
index 6f530f9..0704449 100644
--- a/linux-user/microblaze/syscall_nr.h
+++ b/linux-user/microblaze/syscall_nr.h
@@ -382,3 +382,11 @@
 #define TARGET_NR_process_vm_writev     378
 #define TARGET_NR_kcmp                  379
 #define TARGET_NR_finit_module          380
+#define TARGET_NR_sched_setattr         381
+#define TARGET_NR_sched_getattr         382
+#define TARGET_NR_renameat2             383
+#define TARGET_NR_seccomp               384
+#define TARGET_NR_getrandom             385
+#define TARGET_NR_memfd_create          386
+#define TARGET_NR_bpf                   387
+#define TARGET_NR_execveat              388
diff --git a/linux-user/microblaze/target_cpu.h b/linux-user/microblaze/target_cpu.h
index c6386ea..7dd979f 100644
--- a/linux-user/microblaze/target_cpu.h
+++ b/linux-user/microblaze/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef MICROBLAZE_TARGET_CPU_H
+#define MICROBLAZE_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUMBState *env, target_ulong newsp)
 {
diff --git a/linux-user/microblaze/target_signal.h b/linux-user/microblaze/target_signal.h
index 3d1f7a7..de2b0f4 100644
--- a/linux-user/microblaze/target_signal.h
+++ b/linux-user/microblaze/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef MICROBLAZE_TARGET_SIGNAL_H
+#define MICROBLAZE_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
     return state->regs[14];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* MICROBLAZE_TARGET_SIGNAL_H */
diff --git a/linux-user/microblaze/target_structs.h b/linux-user/microblaze/target_structs.h
index 325e2f6..70dbdb6 100644
--- a/linux-user/microblaze/target_structs.h
+++ b/linux-user/microblaze/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef MICROBLAZE_TARGET_STRUCTS_H
+#define MICROBLAZE_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/microblaze/target_syscall.h b/linux-user/microblaze/target_syscall.h
new file mode 100644
index 0000000..0b6980c
--- /dev/null
+++ b/linux-user/microblaze/target_syscall.h
@@ -0,0 +1,56 @@
+#ifndef MICROBLAZE_TARGET_SYSCALL_H
+#define MICROBLAZE_TARGET_SYSCALL_H
+
+#define UNAME_MACHINE "microblaze"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+/* We use microblaze_reg_t to keep things similar to the kernel sources.  */
+typedef uint32_t microblaze_reg_t;
+
+struct target_pt_regs {
+        microblaze_reg_t r0;
+        microblaze_reg_t r1;
+        microblaze_reg_t r2;
+        microblaze_reg_t r3;
+        microblaze_reg_t r4;
+        microblaze_reg_t r5;
+        microblaze_reg_t r6;
+        microblaze_reg_t r7;
+        microblaze_reg_t r8;
+        microblaze_reg_t r9;
+        microblaze_reg_t r10;
+        microblaze_reg_t r11;
+        microblaze_reg_t r12;
+        microblaze_reg_t r13;
+        microblaze_reg_t r14;
+        microblaze_reg_t r15;
+        microblaze_reg_t r16;
+        microblaze_reg_t r17;
+        microblaze_reg_t r18;
+        microblaze_reg_t r19;
+        microblaze_reg_t r20;
+        microblaze_reg_t r21;
+        microblaze_reg_t r22;
+        microblaze_reg_t r23;
+        microblaze_reg_t r24;
+        microblaze_reg_t r25;
+        microblaze_reg_t r26;
+        microblaze_reg_t r27;
+        microblaze_reg_t r28;
+        microblaze_reg_t r29;
+        microblaze_reg_t r30;
+        microblaze_reg_t r31;
+        microblaze_reg_t pc;
+        microblaze_reg_t msr;
+        microblaze_reg_t ear;
+        microblaze_reg_t esr;
+        microblaze_reg_t fsr;
+        uint32_t kernel_mode;
+};
+
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif
diff --git a/linux-user/mips/syscall.h b/linux-user/mips/syscall.h
deleted file mode 100644
index 35ca23b..0000000
--- a/linux-user/mips/syscall.h
+++ /dev/null
@@ -1,233 +0,0 @@
-
-/* this struct defines the way the registers are stored on the
-   stack during a system call. */
-
-struct target_pt_regs {
-	/* Pad bytes for argument save space on the stack. */
-	abi_ulong pad0[6];
-
-	/* Saved main processor registers. */
-	abi_ulong regs[32];
-
-	/* Saved special registers. */
-	abi_ulong cp0_status;
-	abi_ulong lo;
-	abi_ulong hi;
-	abi_ulong cp0_badvaddr;
-	abi_ulong cp0_cause;
-	abi_ulong cp0_epc;
-};
-
-/* Target errno definitions taken from asm-mips/errno.h */
-#undef TARGET_ENOMSG
-#define TARGET_ENOMSG          35      /* Identifier removed */
-#undef TARGET_EIDRM
-#define TARGET_EIDRM           36      /* Identifier removed */
-#undef TARGET_ECHRNG
-#define TARGET_ECHRNG          37      /* Channel number out of range */
-#undef TARGET_EL2NSYNC
-#define TARGET_EL2NSYNC        38      /* Level 2 not synchronized */
-#undef TARGET_EL3HLT
-#define TARGET_EL3HLT          39      /* Level 3 halted */
-#undef TARGET_EL3RST
-#define TARGET_EL3RST          40      /* Level 3 reset */
-#undef TARGET_ELNRNG
-#define TARGET_ELNRNG          41      /* Link number out of range */
-#undef TARGET_EUNATCH
-#define TARGET_EUNATCH         42      /* Protocol driver not attached */
-#undef TARGET_ENOCSI
-#define TARGET_ENOCSI          43      /* No CSI structure available */
-#undef TARGET_EL2HLT
-#define TARGET_EL2HLT          44      /* Level 2 halted */
-#undef TARGET_EDEADLK
-#define TARGET_EDEADLK         45      /* Resource deadlock would occur */
-#undef TARGET_ENOLCK
-#define TARGET_ENOLCK          46      /* No record locks available */
-#undef TARGET_EBADE
-#define TARGET_EBADE           50      /* Invalid exchange */
-#undef TARGET_EBADR
-#define TARGET_EBADR           51      /* Invalid request descriptor */
-#undef TARGET_EXFULL
-#define TARGET_EXFULL          52      /* TARGET_Exchange full */
-#undef TARGET_ENOANO
-#define TARGET_ENOANO          53      /* No anode */
-#undef TARGET_EBADRQC
-#define TARGET_EBADRQC         54      /* Invalid request code */
-#undef TARGET_EBADSLT
-#define TARGET_EBADSLT         55      /* Invalid slot */
-#undef TARGET_EDEADLOCK
-#define TARGET_EDEADLOCK       56      /* File locking deadlock error */
-#undef TARGET_EBFONT
-#define TARGET_EBFONT          59      /* Bad font file format */
-#undef TARGET_ENOSTR
-#define TARGET_ENOSTR          60      /* Device not a stream */
-#undef TARGET_ENODATA
-#define TARGET_ENODATA         61      /* No data available */
-#undef TARGET_ETIME
-#define TARGET_ETIME           62      /* Timer expired */
-#undef TARGET_ENOSR
-#define TARGET_ENOSR           63      /* Out of streams resources */
-#undef TARGET_ENONET
-#define TARGET_ENONET          64      /* Machine is not on the network */
-#undef TARGET_ENOPKG
-#define TARGET_ENOPKG          65      /* Package not installed */
-#undef TARGET_EREMOTE
-#define TARGET_EREMOTE         66      /* Object is remote */
-#undef TARGET_ENOLINK
-#define TARGET_ENOLINK         67      /* Link has been severed */
-#undef TARGET_EADV
-#define TARGET_EADV            68      /* Advertise error */
-#undef TARGET_ESRMNT
-#define TARGET_ESRMNT          69      /* Srmount error */
-#undef TARGET_ECOMM
-#define TARGET_ECOMM           70      /* Communication error on send */
-#undef TARGET_EPROTO
-#define TARGET_EPROTO          71      /* Protocol error */
-#undef TARGET_EDOTDOT
-#define TARGET_EDOTDOT         73      /* RFS specific error */
-#undef TARGET_EMULTIHOP
-#define TARGET_EMULTIHOP       74      /* Multihop attempted */
-#undef TARGET_EBADMSG
-#define TARGET_EBADMSG         77      /* Not a data message */
-#undef TARGET_ENAMETOOLONG
-#define TARGET_ENAMETOOLONG    78      /* File name too long */
-#undef TARGET_EOVERFLOW
-#define TARGET_EOVERFLOW       79      /* Value too large for defined data type */
-#undef TARGET_ENOTUNIQ
-#define TARGET_ENOTUNIQ        80      /* Name not unique on network */
-#undef TARGET_EBADFD
-#define TARGET_EBADFD          81      /* File descriptor in bad state */
-#undef TARGET_EREMCHG
-#define TARGET_EREMCHG         82      /* Remote address changed */
-#undef TARGET_ELIBACC
-#define TARGET_ELIBACC         83      /* Can not access a needed shared library */
-#undef TARGET_ELIBBAD
-#define TARGET_ELIBBAD         84      /* Accessing a corrupted shared library */
-#undef TARGET_ELIBSCN
-#define TARGET_ELIBSCN         85      /* .lib section in a.out corrupted */
-#undef TARGET_ELIBMAX
-#define TARGET_ELIBMAX         86      /* Attempting to link in too many shared libraries */
-#undef TARGET_ELIBEXEC
-#define TARGET_ELIBEXEC        87      /* Cannot exec a shared library directly */
-#undef TARGET_EILSEQ
-#define TARGET_EILSEQ          88      /* Illegal byte sequence */
-#undef TARGET_ENOSYS
-#define TARGET_ENOSYS          89      /* Function not implemented */
-#undef TARGET_ELOOP
-#define TARGET_ELOOP           90      /* Too many symbolic links encountered */
-#undef TARGET_ERESTART
-#define TARGET_ERESTART        91      /* Interrupted system call should be restarted */
-#undef TARGET_ESTRPIPE
-#define TARGET_ESTRPIPE        92      /* Streams pipe error */
-#undef TARGET_ENOTEMPTY
-#define TARGET_ENOTEMPTY       93      /* Directory not empty */
-#undef TARGET_EUSERS
-#define TARGET_EUSERS          94      /* Too many users */
-#undef TARGET_ENOTSOCK
-#define TARGET_ENOTSOCK        95      /* Socket operation on non-socket */
-#undef TARGET_EDESTADDRREQ
-#define TARGET_EDESTADDRREQ    96      /* Destination address required */
-#undef TARGET_EMSGSIZE
-#define TARGET_EMSGSIZE        97      /* Message too long */
-#undef TARGET_EPROTOTYPE
-#define TARGET_EPROTOTYPE      98      /* Protocol wrong type for socket */
-#undef TARGET_ENOPROTOOPT
-#define TARGET_ENOPROTOOPT     99      /* Protocol not available */
-#undef TARGET_EPROTONOSUPPORT
-#define TARGET_EPROTONOSUPPORT 120     /* Protocol not supported */
-#undef TARGET_ESOCKTNOSUPPORT
-#define TARGET_ESOCKTNOSUPPORT 121     /* Socket type not supported */
-#undef TARGET_EOPNOTSUPP
-#define TARGET_EOPNOTSUPP      122     /* Operation not supported on transport endpoint */
-#undef TARGET_EPFNOSUPPORT
-#define TARGET_EPFNOSUPPORT    123     /* Protocol family not supported */
-#undef TARGET_EAFNOSUPPORT
-#define TARGET_EAFNOSUPPORT    124     /* Address family not supported by protocol */
-#undef TARGET_EADDRINUSE
-#define TARGET_EADDRINUSE      125     /* Address already in use */
-#undef TARGET_EADDRNOTAVAIL
-#define TARGET_EADDRNOTAVAIL   126     /* Cannot assign requested address */
-#undef TARGET_ENETDOWN
-#define TARGET_ENETDOWN        127     /* Network is down */
-#undef TARGET_ENETUNREACH
-#define TARGET_ENETUNREACH     128     /* Network is unreachable */
-#undef TARGET_ENETRESET
-#define TARGET_ENETRESET       129     /* Network dropped connection because of reset */
-#undef TARGET_ECONNABORTED
-#define TARGET_ECONNABORTED    130     /* Software caused connection abort */
-#undef TARGET_ECONNRESET
-#define TARGET_ECONNRESET      131     /* Connection reset by peer */
-#undef TARGET_ENOBUFS
-#define TARGET_ENOBUFS         132     /* No buffer space available */
-#undef TARGET_EISCONN
-#define TARGET_EISCONN         133     /* Transport endpoint is already connected */
-#undef TARGET_ENOTCONN
-#define TARGET_ENOTCONN        134     /* Transport endpoint is not connected */
-#undef TARGET_EUCLEAN
-#define TARGET_EUCLEAN         135     /* Structure needs cleaning */
-#undef TARGET_ENOTNAM
-#define TARGET_ENOTNAM         137     /* Not a XENIX named type file */
-#undef TARGET_ENAVAIL
-#define TARGET_ENAVAIL         138     /* No XENIX semaphores available */
-#undef TARGET_EISNAM
-#define TARGET_EISNAM          139     /* Is a named type file */
-#undef TARGET_EREMOTEIO
-#define TARGET_EREMOTEIO       140     /* Remote I/O error */
-#undef TARGET_EINIT
-#define TARGET_EINIT           141     /* Reserved */
-#undef TARGET_EREMDEV
-#define TARGET_EREMDEV         142     /* TARGET_Error 142 */
-#undef TARGET_ESHUTDOWN
-#define TARGET_ESHUTDOWN       143     /* Cannot send after transport endpoint shutdown */
-#undef TARGET_ETOOMANYREFS
-#define TARGET_ETOOMANYREFS    144     /* Too many references: cannot splice */
-#undef TARGET_ETIMEDOUT
-#define TARGET_ETIMEDOUT       145     /* Connection timed out */
-#undef TARGET_ECONNREFUSED
-#define TARGET_ECONNREFUSED    146     /* Connection refused */
-#undef TARGET_EHOSTDOWN
-#define TARGET_EHOSTDOWN       147     /* Host is down */
-#undef TARGET_EHOSTUNREACH
-#define TARGET_EHOSTUNREACH    148     /* No route to host */
-#undef TARGET_EALREADY
-#define TARGET_EALREADY        149     /* Operation already in progress */
-#undef TARGET_EINPROGRESS
-#define TARGET_EINPROGRESS     150     /* Operation now in progress */
-#undef TARGET_ESTALE
-#define TARGET_ESTALE          151     /* Stale NFS file handle */
-#undef TARGET_ECANCELED
-#define TARGET_ECANCELED       158     /* AIO operation canceled */
-/*
- * These error are Linux extensions.
- */
-#undef TARGET_ENOMEDIUM
-#define TARGET_ENOMEDIUM       159     /* No medium found */
-#undef TARGET_EMEDIUMTYPE
-#define TARGET_EMEDIUMTYPE     160     /* Wrong medium type */
-#undef TARGET_ENOKEY
-#define TARGET_ENOKEY          161     /* Required key not available */
-#undef TARGET_EKEYEXPIRED
-#define TARGET_EKEYEXPIRED     162     /* Key has expired */
-#undef TARGET_EKEYREVOKED
-#define TARGET_EKEYREVOKED     163     /* Key has been revoked */
-#undef TARGET_EKEYREJECTED
-#define TARGET_EKEYREJECTED    164     /* Key was rejected by service */
-
-/* for robust mutexes */
-#undef TARGET_EOWNERDEAD
-#define TARGET_EOWNERDEAD      165     /* Owner died */
-#undef TARGET_ENOTRECOVERABLE
-#define TARGET_ENOTRECOVERABLE 166     /* State not recoverable */
-
-
-
-/* Nasty hack: define a fake errno value for use by sigreturn.  */
-#define TARGET_QEMU_ESIGRETURN 255
-
-#define UNAME_MACHINE "mips"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/mips/syscall_nr.h b/linux-user/mips/syscall_nr.h
index 2d1a13e..6819f86 100644
--- a/linux-user/mips/syscall_nr.h
+++ b/linux-user/mips/syscall_nr.h
@@ -351,3 +351,15 @@
 #define TARGET_NR_process_vm_writev     (TARGET_NR_Linux + 346)
 #define TARGET_NR_kcmp                  (TARGET_NR_Linux + 347)
 #define TARGET_NR_finit_module          (TARGET_NR_Linux + 348)
+
+#define TARGET_NR_sched_setattr         (TARGET_NR_Linux + 349)
+#define TARGET_NR_sched_getattr         (TARGET_NR_Linux + 350)
+#define TARGET_NR_renameat2             (TARGET_NR_Linux + 351)
+#define TARGET_NR_seccomp               (TARGET_NR_Linux + 352)
+#define TARGET_NR_getrandom             (TARGET_NR_Linux + 353)
+#define TARGET_NR_memfd_create          (TARGET_NR_Linux + 354)
+#define TARGET_NR_bpf                   (TARGET_NR_Linux + 355)
+#define TARGET_NR_execveat              (TARGET_NR_Linux + 356)
+#define TARGET_NR_userfaultfd           (TARGET_NR_Linux + 357)
+#define TARGET_NR_membarrier            (TARGET_NR_Linux + 358)
+#define TARGET_NR_mlock2                (TARGET_NR_Linux + 359)
diff --git a/linux-user/mips/target_cpu.h b/linux-user/mips/target_cpu.h
index 19b8855..2002920 100644
--- a/linux-user/mips/target_cpu.h
+++ b/linux-user/mips/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef MIPS_TARGET_CPU_H
+#define MIPS_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUMIPSState *env, target_ulong newsp)
 {
diff --git a/linux-user/mips/target_signal.h b/linux-user/mips/target_signal.h
index 6e1dc8b..8dd27ce 100644
--- a/linux-user/mips/target_signal.h
+++ b/linux-user/mips/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef MIPS_TARGET_SIGNAL_H
+#define MIPS_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
     return state->active_tc.gpr[29];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* MIPS_TARGET_SIGNAL_H */
diff --git a/linux-user/mips/target_structs.h b/linux-user/mips/target_structs.h
index 16021e8..fbd9955 100644
--- a/linux-user/mips/target_structs.h
+++ b/linux-user/mips/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef MIPS_TARGET_STRUCTS_H
+#define MIPS_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/mips/target_syscall.h b/linux-user/mips/target_syscall.h
new file mode 100644
index 0000000..2b4f390
--- /dev/null
+++ b/linux-user/mips/target_syscall.h
@@ -0,0 +1,233 @@
+#ifndef MIPS_TARGET_SYSCALL_H
+#define MIPS_TARGET_SYSCALL_H
+
+/* this struct defines the way the registers are stored on the
+   stack during a system call. */
+
+struct target_pt_regs {
+	/* Pad bytes for argument save space on the stack. */
+	abi_ulong pad0[6];
+
+	/* Saved main processor registers. */
+	abi_ulong regs[32];
+
+	/* Saved special registers. */
+	abi_ulong cp0_status;
+	abi_ulong lo;
+	abi_ulong hi;
+	abi_ulong cp0_badvaddr;
+	abi_ulong cp0_cause;
+	abi_ulong cp0_epc;
+};
+
+/* Target errno definitions taken from asm-mips/errno.h */
+#undef TARGET_ENOMSG
+#define TARGET_ENOMSG          35      /* Identifier removed */
+#undef TARGET_EIDRM
+#define TARGET_EIDRM           36      /* Identifier removed */
+#undef TARGET_ECHRNG
+#define TARGET_ECHRNG          37      /* Channel number out of range */
+#undef TARGET_EL2NSYNC
+#define TARGET_EL2NSYNC        38      /* Level 2 not synchronized */
+#undef TARGET_EL3HLT
+#define TARGET_EL3HLT          39      /* Level 3 halted */
+#undef TARGET_EL3RST
+#define TARGET_EL3RST          40      /* Level 3 reset */
+#undef TARGET_ELNRNG
+#define TARGET_ELNRNG          41      /* Link number out of range */
+#undef TARGET_EUNATCH
+#define TARGET_EUNATCH         42      /* Protocol driver not attached */
+#undef TARGET_ENOCSI
+#define TARGET_ENOCSI          43      /* No CSI structure available */
+#undef TARGET_EL2HLT
+#define TARGET_EL2HLT          44      /* Level 2 halted */
+#undef TARGET_EDEADLK
+#define TARGET_EDEADLK         45      /* Resource deadlock would occur */
+#undef TARGET_ENOLCK
+#define TARGET_ENOLCK          46      /* No record locks available */
+#undef TARGET_EBADE
+#define TARGET_EBADE           50      /* Invalid exchange */
+#undef TARGET_EBADR
+#define TARGET_EBADR           51      /* Invalid request descriptor */
+#undef TARGET_EXFULL
+#define TARGET_EXFULL          52      /* TARGET_Exchange full */
+#undef TARGET_ENOANO
+#define TARGET_ENOANO          53      /* No anode */
+#undef TARGET_EBADRQC
+#define TARGET_EBADRQC         54      /* Invalid request code */
+#undef TARGET_EBADSLT
+#define TARGET_EBADSLT         55      /* Invalid slot */
+#undef TARGET_EDEADLOCK
+#define TARGET_EDEADLOCK       56      /* File locking deadlock error */
+#undef TARGET_EBFONT
+#define TARGET_EBFONT          59      /* Bad font file format */
+#undef TARGET_ENOSTR
+#define TARGET_ENOSTR          60      /* Device not a stream */
+#undef TARGET_ENODATA
+#define TARGET_ENODATA         61      /* No data available */
+#undef TARGET_ETIME
+#define TARGET_ETIME           62      /* Timer expired */
+#undef TARGET_ENOSR
+#define TARGET_ENOSR           63      /* Out of streams resources */
+#undef TARGET_ENONET
+#define TARGET_ENONET          64      /* Machine is not on the network */
+#undef TARGET_ENOPKG
+#define TARGET_ENOPKG          65      /* Package not installed */
+#undef TARGET_EREMOTE
+#define TARGET_EREMOTE         66      /* Object is remote */
+#undef TARGET_ENOLINK
+#define TARGET_ENOLINK         67      /* Link has been severed */
+#undef TARGET_EADV
+#define TARGET_EADV            68      /* Advertise error */
+#undef TARGET_ESRMNT
+#define TARGET_ESRMNT          69      /* Srmount error */
+#undef TARGET_ECOMM
+#define TARGET_ECOMM           70      /* Communication error on send */
+#undef TARGET_EPROTO
+#define TARGET_EPROTO          71      /* Protocol error */
+#undef TARGET_EDOTDOT
+#define TARGET_EDOTDOT         73      /* RFS specific error */
+#undef TARGET_EMULTIHOP
+#define TARGET_EMULTIHOP       74      /* Multihop attempted */
+#undef TARGET_EBADMSG
+#define TARGET_EBADMSG         77      /* Not a data message */
+#undef TARGET_ENAMETOOLONG
+#define TARGET_ENAMETOOLONG    78      /* File name too long */
+#undef TARGET_EOVERFLOW
+#define TARGET_EOVERFLOW       79      /* Value too large for defined data type */
+#undef TARGET_ENOTUNIQ
+#define TARGET_ENOTUNIQ        80      /* Name not unique on network */
+#undef TARGET_EBADFD
+#define TARGET_EBADFD          81      /* File descriptor in bad state */
+#undef TARGET_EREMCHG
+#define TARGET_EREMCHG         82      /* Remote address changed */
+#undef TARGET_ELIBACC
+#define TARGET_ELIBACC         83      /* Can not access a needed shared library */
+#undef TARGET_ELIBBAD
+#define TARGET_ELIBBAD         84      /* Accessing a corrupted shared library */
+#undef TARGET_ELIBSCN
+#define TARGET_ELIBSCN         85      /* .lib section in a.out corrupted */
+#undef TARGET_ELIBMAX
+#define TARGET_ELIBMAX         86      /* Attempting to link in too many shared libraries */
+#undef TARGET_ELIBEXEC
+#define TARGET_ELIBEXEC        87      /* Cannot exec a shared library directly */
+#undef TARGET_EILSEQ
+#define TARGET_EILSEQ          88      /* Illegal byte sequence */
+#undef TARGET_ENOSYS
+#define TARGET_ENOSYS          89      /* Function not implemented */
+#undef TARGET_ELOOP
+#define TARGET_ELOOP           90      /* Too many symbolic links encountered */
+#undef TARGET_ERESTART
+#define TARGET_ERESTART        91      /* Interrupted system call should be restarted */
+#undef TARGET_ESTRPIPE
+#define TARGET_ESTRPIPE        92      /* Streams pipe error */
+#undef TARGET_ENOTEMPTY
+#define TARGET_ENOTEMPTY       93      /* Directory not empty */
+#undef TARGET_EUSERS
+#define TARGET_EUSERS          94      /* Too many users */
+#undef TARGET_ENOTSOCK
+#define TARGET_ENOTSOCK        95      /* Socket operation on non-socket */
+#undef TARGET_EDESTADDRREQ
+#define TARGET_EDESTADDRREQ    96      /* Destination address required */
+#undef TARGET_EMSGSIZE
+#define TARGET_EMSGSIZE        97      /* Message too long */
+#undef TARGET_EPROTOTYPE
+#define TARGET_EPROTOTYPE      98      /* Protocol wrong type for socket */
+#undef TARGET_ENOPROTOOPT
+#define TARGET_ENOPROTOOPT     99      /* Protocol not available */
+#undef TARGET_EPROTONOSUPPORT
+#define TARGET_EPROTONOSUPPORT 120     /* Protocol not supported */
+#undef TARGET_ESOCKTNOSUPPORT
+#define TARGET_ESOCKTNOSUPPORT 121     /* Socket type not supported */
+#undef TARGET_EOPNOTSUPP
+#define TARGET_EOPNOTSUPP      122     /* Operation not supported on transport endpoint */
+#undef TARGET_EPFNOSUPPORT
+#define TARGET_EPFNOSUPPORT    123     /* Protocol family not supported */
+#undef TARGET_EAFNOSUPPORT
+#define TARGET_EAFNOSUPPORT    124     /* Address family not supported by protocol */
+#undef TARGET_EADDRINUSE
+#define TARGET_EADDRINUSE      125     /* Address already in use */
+#undef TARGET_EADDRNOTAVAIL
+#define TARGET_EADDRNOTAVAIL   126     /* Cannot assign requested address */
+#undef TARGET_ENETDOWN
+#define TARGET_ENETDOWN        127     /* Network is down */
+#undef TARGET_ENETUNREACH
+#define TARGET_ENETUNREACH     128     /* Network is unreachable */
+#undef TARGET_ENETRESET
+#define TARGET_ENETRESET       129     /* Network dropped connection because of reset */
+#undef TARGET_ECONNABORTED
+#define TARGET_ECONNABORTED    130     /* Software caused connection abort */
+#undef TARGET_ECONNRESET
+#define TARGET_ECONNRESET      131     /* Connection reset by peer */
+#undef TARGET_ENOBUFS
+#define TARGET_ENOBUFS         132     /* No buffer space available */
+#undef TARGET_EISCONN
+#define TARGET_EISCONN         133     /* Transport endpoint is already connected */
+#undef TARGET_ENOTCONN
+#define TARGET_ENOTCONN        134     /* Transport endpoint is not connected */
+#undef TARGET_EUCLEAN
+#define TARGET_EUCLEAN         135     /* Structure needs cleaning */
+#undef TARGET_ENOTNAM
+#define TARGET_ENOTNAM         137     /* Not a XENIX named type file */
+#undef TARGET_ENAVAIL
+#define TARGET_ENAVAIL         138     /* No XENIX semaphores available */
+#undef TARGET_EISNAM
+#define TARGET_EISNAM          139     /* Is a named type file */
+#undef TARGET_EREMOTEIO
+#define TARGET_EREMOTEIO       140     /* Remote I/O error */
+#undef TARGET_EINIT
+#define TARGET_EINIT           141     /* Reserved */
+#undef TARGET_EREMDEV
+#define TARGET_EREMDEV         142     /* TARGET_Error 142 */
+#undef TARGET_ESHUTDOWN
+#define TARGET_ESHUTDOWN       143     /* Cannot send after transport endpoint shutdown */
+#undef TARGET_ETOOMANYREFS
+#define TARGET_ETOOMANYREFS    144     /* Too many references: cannot splice */
+#undef TARGET_ETIMEDOUT
+#define TARGET_ETIMEDOUT       145     /* Connection timed out */
+#undef TARGET_ECONNREFUSED
+#define TARGET_ECONNREFUSED    146     /* Connection refused */
+#undef TARGET_EHOSTDOWN
+#define TARGET_EHOSTDOWN       147     /* Host is down */
+#undef TARGET_EHOSTUNREACH
+#define TARGET_EHOSTUNREACH    148     /* No route to host */
+#undef TARGET_EALREADY
+#define TARGET_EALREADY        149     /* Operation already in progress */
+#undef TARGET_EINPROGRESS
+#define TARGET_EINPROGRESS     150     /* Operation now in progress */
+#undef TARGET_ESTALE
+#define TARGET_ESTALE          151     /* Stale NFS file handle */
+#undef TARGET_ECANCELED
+#define TARGET_ECANCELED       158     /* AIO operation canceled */
+/*
+ * These error are Linux extensions.
+ */
+#undef TARGET_ENOMEDIUM
+#define TARGET_ENOMEDIUM       159     /* No medium found */
+#undef TARGET_EMEDIUMTYPE
+#define TARGET_EMEDIUMTYPE     160     /* Wrong medium type */
+#undef TARGET_ENOKEY
+#define TARGET_ENOKEY          161     /* Required key not available */
+#undef TARGET_EKEYEXPIRED
+#define TARGET_EKEYEXPIRED     162     /* Key has expired */
+#undef TARGET_EKEYREVOKED
+#define TARGET_EKEYREVOKED     163     /* Key has been revoked */
+#undef TARGET_EKEYREJECTED
+#define TARGET_EKEYREJECTED    164     /* Key was rejected by service */
+
+/* for robust mutexes */
+#undef TARGET_EOWNERDEAD
+#define TARGET_EOWNERDEAD      165     /* Owner died */
+#undef TARGET_ENOTRECOVERABLE
+#define TARGET_ENOTRECOVERABLE 166     /* State not recoverable */
+
+
+#define UNAME_MACHINE "mips"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* MIPS_TARGET_SYSCALL_H */
diff --git a/linux-user/mips64/syscall.h b/linux-user/mips64/syscall.h
deleted file mode 100644
index 6733107..0000000
--- a/linux-user/mips64/syscall.h
+++ /dev/null
@@ -1,230 +0,0 @@
-
-/* this struct defines the way the registers are stored on the
-   stack during a system call. */
-
-struct target_pt_regs {
-        /* Saved main processor registers. */
-        target_ulong regs[32];
-
-        /* Saved special registers. */
-        target_ulong cp0_status;
-        target_ulong lo;
-        target_ulong hi;
-        target_ulong cp0_badvaddr;
-        target_ulong cp0_cause;
-        target_ulong cp0_epc;
-};
-
-/* Target errno definitions taken from asm-mips/errno.h */
-#undef TARGET_ENOMSG
-#define TARGET_ENOMSG          35      /* Identifier removed */
-#undef TARGET_EIDRM
-#define TARGET_EIDRM           36      /* Identifier removed */
-#undef TARGET_ECHRNG
-#define TARGET_ECHRNG          37      /* Channel number out of range */
-#undef TARGET_EL2NSYNC
-#define TARGET_EL2NSYNC        38      /* Level 2 not synchronized */
-#undef TARGET_EL3HLT
-#define TARGET_EL3HLT          39      /* Level 3 halted */
-#undef TARGET_EL3RST
-#define TARGET_EL3RST          40      /* Level 3 reset */
-#undef TARGET_ELNRNG
-#define TARGET_ELNRNG          41      /* Link number out of range */
-#undef TARGET_EUNATCH
-#define TARGET_EUNATCH         42      /* Protocol driver not attached */
-#undef TARGET_ENOCSI
-#define TARGET_ENOCSI          43      /* No CSI structure available */
-#undef TARGET_EL2HLT
-#define TARGET_EL2HLT          44      /* Level 2 halted */
-#undef TARGET_EDEADLK
-#define TARGET_EDEADLK         45      /* Resource deadlock would occur */
-#undef TARGET_ENOLCK
-#define TARGET_ENOLCK          46      /* No record locks available */
-#undef TARGET_EBADE
-#define TARGET_EBADE           50      /* Invalid exchange */
-#undef TARGET_EBADR
-#define TARGET_EBADR           51      /* Invalid request descriptor */
-#undef TARGET_EXFULL
-#define TARGET_EXFULL          52      /* TARGET_Exchange full */
-#undef TARGET_ENOANO
-#define TARGET_ENOANO          53      /* No anode */
-#undef TARGET_EBADRQC
-#define TARGET_EBADRQC         54      /* Invalid request code */
-#undef TARGET_EBADSLT
-#define TARGET_EBADSLT         55      /* Invalid slot */
-#undef TARGET_EDEADLOCK
-#define TARGET_EDEADLOCK       56      /* File locking deadlock error */
-#undef TARGET_EBFONT
-#define TARGET_EBFONT          59      /* Bad font file format */
-#undef TARGET_ENOSTR
-#define TARGET_ENOSTR          60      /* Device not a stream */
-#undef TARGET_ENODATA
-#define TARGET_ENODATA         61      /* No data available */
-#undef TARGET_ETIME
-#define TARGET_ETIME           62      /* Timer expired */
-#undef TARGET_ENOSR
-#define TARGET_ENOSR           63      /* Out of streams resources */
-#undef TARGET_ENONET
-#define TARGET_ENONET          64      /* Machine is not on the network */
-#undef TARGET_ENOPKG
-#define TARGET_ENOPKG          65      /* Package not installed */
-#undef TARGET_EREMOTE
-#define TARGET_EREMOTE         66      /* Object is remote */
-#undef TARGET_ENOLINK
-#define TARGET_ENOLINK         67      /* Link has been severed */
-#undef TARGET_EADV
-#define TARGET_EADV            68      /* Advertise error */
-#undef TARGET_ESRMNT
-#define TARGET_ESRMNT          69      /* Srmount error */
-#undef TARGET_ECOMM
-#define TARGET_ECOMM           70      /* Communication error on send */
-#undef TARGET_EPROTO
-#define TARGET_EPROTO          71      /* Protocol error */
-#undef TARGET_EDOTDOT
-#define TARGET_EDOTDOT         73      /* RFS specific error */
-#undef TARGET_EMULTIHOP
-#define TARGET_EMULTIHOP       74      /* Multihop attempted */
-#undef TARGET_EBADMSG
-#define TARGET_EBADMSG         77      /* Not a data message */
-#undef TARGET_ENAMETOOLONG
-#define TARGET_ENAMETOOLONG    78      /* File name too long */
-#undef TARGET_EOVERFLOW
-#define TARGET_EOVERFLOW       79      /* Value too large for defined data type */
-#undef TARGET_ENOTUNIQ
-#define TARGET_ENOTUNIQ        80      /* Name not unique on network */
-#undef TARGET_EBADFD
-#define TARGET_EBADFD          81      /* File descriptor in bad state */
-#undef TARGET_EREMCHG
-#define TARGET_EREMCHG         82      /* Remote address changed */
-#undef TARGET_ELIBACC
-#define TARGET_ELIBACC         83      /* Can not access a needed shared library */
-#undef TARGET_ELIBBAD
-#define TARGET_ELIBBAD         84      /* Accessing a corrupted shared library */
-#undef TARGET_ELIBSCN
-#define TARGET_ELIBSCN         85      /* .lib section in a.out corrupted */
-#undef TARGET_ELIBMAX
-#define TARGET_ELIBMAX         86      /* Attempting to link in too many shared libraries */
-#undef TARGET_ELIBEXEC
-#define TARGET_ELIBEXEC        87      /* Cannot exec a shared library directly */
-#undef TARGET_EILSEQ
-#define TARGET_EILSEQ          88      /* Illegal byte sequence */
-#undef TARGET_ENOSYS
-#define TARGET_ENOSYS          89      /* Function not implemented */
-#undef TARGET_ELOOP
-#define TARGET_ELOOP           90      /* Too many symbolic links encountered */
-#undef TARGET_ERESTART
-#define TARGET_ERESTART        91      /* Interrupted system call should be restarted */
-#undef TARGET_ESTRPIPE
-#define TARGET_ESTRPIPE        92      /* Streams pipe error */
-#undef TARGET_ENOTEMPTY
-#define TARGET_ENOTEMPTY       93      /* Directory not empty */
-#undef TARGET_EUSERS
-#define TARGET_EUSERS          94      /* Too many users */
-#undef TARGET_ENOTSOCK
-#define TARGET_ENOTSOCK        95      /* Socket operation on non-socket */
-#undef TARGET_EDESTADDRREQ
-#define TARGET_EDESTADDRREQ    96      /* Destination address required */
-#undef TARGET_EMSGSIZE
-#define TARGET_EMSGSIZE        97      /* Message too long */
-#undef TARGET_EPROTOTYPE
-#define TARGET_EPROTOTYPE      98      /* Protocol wrong type for socket */
-#undef TARGET_ENOPROTOOPT
-#define TARGET_ENOPROTOOPT     99      /* Protocol not available */
-#undef TARGET_EPROTONOSUPPORT
-#define TARGET_EPROTONOSUPPORT 120     /* Protocol not supported */
-#undef TARGET_ESOCKTNOSUPPORT
-#define TARGET_ESOCKTNOSUPPORT 121     /* Socket type not supported */
-#undef TARGET_EOPNOTSUPP
-#define TARGET_EOPNOTSUPP      122     /* Operation not supported on transport endpoint */
-#undef TARGET_EPFNOSUPPORT
-#define TARGET_EPFNOSUPPORT    123     /* Protocol family not supported */
-#undef TARGET_EAFNOSUPPORT
-#define TARGET_EAFNOSUPPORT    124     /* Address family not supported by protocol */
-#undef TARGET_EADDRINUSE
-#define TARGET_EADDRINUSE      125     /* Address already in use */
-#undef TARGET_EADDRNOTAVAIL
-#define TARGET_EADDRNOTAVAIL   126     /* Cannot assign requested address */
-#undef TARGET_ENETDOWN
-#define TARGET_ENETDOWN        127     /* Network is down */
-#undef TARGET_ENETUNREACH
-#define TARGET_ENETUNREACH     128     /* Network is unreachable */
-#undef TARGET_ENETRESET
-#define TARGET_ENETRESET       129     /* Network dropped connection because of reset */
-#undef TARGET_ECONNABORTED
-#define TARGET_ECONNABORTED    130     /* Software caused connection abort */
-#undef TARGET_ECONNRESET
-#define TARGET_ECONNRESET      131     /* Connection reset by peer */
-#undef TARGET_ENOBUFS
-#define TARGET_ENOBUFS         132     /* No buffer space available */
-#undef TARGET_EISCONN
-#define TARGET_EISCONN         133     /* Transport endpoint is already connected */
-#undef TARGET_ENOTCONN
-#define TARGET_ENOTCONN        134     /* Transport endpoint is not connected */
-#undef TARGET_EUCLEAN
-#define TARGET_EUCLEAN         135     /* Structure needs cleaning */
-#undef TARGET_ENOTNAM
-#define TARGET_ENOTNAM         137     /* Not a XENIX named type file */
-#undef TARGET_ENAVAIL
-#define TARGET_ENAVAIL         138     /* No XENIX semaphores available */
-#undef TARGET_EISNAM
-#define TARGET_EISNAM          139     /* Is a named type file */
-#undef TARGET_EREMOTEIO
-#define TARGET_EREMOTEIO       140     /* Remote I/O error */
-#undef TARGET_EINIT
-#define TARGET_EINIT           141     /* Reserved */
-#undef TARGET_EREMDEV
-#define TARGET_EREMDEV         142     /* TARGET_Error 142 */
-#undef TARGET_ESHUTDOWN
-#define TARGET_ESHUTDOWN       143     /* Cannot send after transport endpoint shutdown */
-#undef TARGET_ETOOMANYREFS
-#define TARGET_ETOOMANYREFS    144     /* Too many references: cannot splice */
-#undef TARGET_ETIMEDOUT
-#define TARGET_ETIMEDOUT       145     /* Connection timed out */
-#undef TARGET_ECONNREFUSED
-#define TARGET_ECONNREFUSED    146     /* Connection refused */
-#undef TARGET_EHOSTDOWN
-#define TARGET_EHOSTDOWN       147     /* Host is down */
-#undef TARGET_EHOSTUNREACH
-#define TARGET_EHOSTUNREACH    148     /* No route to host */
-#undef TARGET_EALREADY
-#define TARGET_EALREADY        149     /* Operation already in progress */
-#undef TARGET_EINPROGRESS
-#define TARGET_EINPROGRESS     150     /* Operation now in progress */
-#undef TARGET_ESTALE
-#define TARGET_ESTALE          151     /* Stale NFS file handle */
-#undef TARGET_ECANCELED
-#define TARGET_ECANCELED       158     /* AIO operation canceled */
-/*
- * These error are Linux extensions.
- */
-#undef TARGET_ENOMEDIUM
-#define TARGET_ENOMEDIUM       159     /* No medium found */
-#undef TARGET_EMEDIUMTYPE
-#define TARGET_EMEDIUMTYPE     160     /* Wrong medium type */
-#undef TARGET_ENOKEY
-#define TARGET_ENOKEY          161     /* Required key not available */
-#undef TARGET_EKEYEXPIRED
-#define TARGET_EKEYEXPIRED     162     /* Key has expired */
-#undef TARGET_EKEYREVOKED
-#define TARGET_EKEYREVOKED     163     /* Key has been revoked */
-#undef TARGET_EKEYREJECTED
-#define TARGET_EKEYREJECTED    164     /* Key was rejected by service */
-
-/* for robust mutexes */
-#undef TARGET_EOWNERDEAD
-#define TARGET_EOWNERDEAD      165     /* Owner died */
-#undef TARGET_ENOTRECOVERABLE
-#define TARGET_ENOTRECOVERABLE 166     /* State not recoverable */
-
-
-
-/* Nasty hack: define a fake errno value for use by sigreturn. */
-#define TARGET_QEMU_ESIGRETURN 255
-
-#define UNAME_MACHINE "mips64"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ      2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/mips64/syscall_nr.h b/linux-user/mips64/syscall_nr.h
index 004232a..746cc26 100644
--- a/linux-user/mips64/syscall_nr.h
+++ b/linux-user/mips64/syscall_nr.h
@@ -316,6 +316,18 @@
 #define TARGET_NR_process_vm_writev     (TARGET_NR_Linux + 310)
 #define TARGET_NR_kcmp                  (TARGET_NR_Linux + 311)
 #define TARGET_NR_finit_module          (TARGET_NR_Linux + 312)
+#define TARGET_NR_sched_setattr         (TARGET_NR_Linux + 313)
+#define TARGET_NR_sched_getattr         (TARGET_NR_Linux + 314)
+#define TARGET_NR_renameat2             (TARGET_NR_Linux + 315)
+#define TARGET_NR_seccomp               (TARGET_NR_Linux + 316)
+#define TARGET_NR_getrandom             (TARGET_NR_Linux + 317)
+#define TARGET_NR_memfd_create          (TARGET_NR_Linux + 318)
+#define TARGET_NR_bpf                   (TARGET_NR_Linux + 319)
+#define TARGET_NR_execveat              (TARGET_NR_Linux + 320)
+#define TARGET_NR_userfaultfd           (TARGET_NR_Linux + 321)
+#define TARGET_NR_membarrier            (TARGET_NR_Linux + 322)
+#define TARGET_NR_mlock2                (TARGET_NR_Linux + 323)
+
 #else
 /*
  * Linux 64-bit syscalls are in the range from 5000 to 5999.
@@ -630,4 +642,15 @@
 #define TARGET_NR_kcmp                  (TARGET_NR_Linux + 306)
 #define TARGET_NR_finit_module          (TARGET_NR_Linux + 307)
 #define TARGET_NR_getdents64            (TARGET_NR_Linux + 308)
+#define TARGET_NR_sched_setattr         (TARGET_NR_Linux + 309)
+#define TARGET_NR_sched_getattr         (TARGET_NR_Linux + 310)
+#define TARGET_NR_renameat2             (TARGET_NR_Linux + 311)
+#define TARGET_NR_seccomp               (TARGET_NR_Linux + 312)
+#define TARGET_NR_getrandom             (TARGET_NR_Linux + 313)
+#define TARGET_NR_memfd_create          (TARGET_NR_Linux + 314)
+#define TARGET_NR_bpf                   (TARGET_NR_Linux + 315)
+#define TARGET_NR_execveat              (TARGET_NR_Linux + 316)
+#define TARGET_NR_userfaultfd           (TARGET_NR_Linux + 317)
+#define TARGET_NR_membarrier            (TARGET_NR_Linux + 318)
+#define TARGET_NR_mlock2                (TARGET_NR_Linux + 319)
 #endif
diff --git a/linux-user/mips64/target_signal.h b/linux-user/mips64/target_signal.h
index 6e1dc8b..67ef5a1 100644
--- a/linux-user/mips64/target_signal.h
+++ b/linux-user/mips64/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef MIPS64_TARGET_SIGNAL_H
+#define MIPS64_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -8,7 +8,7 @@
 typedef struct target_sigaltstack {
 	abi_long ss_sp;
 	abi_ulong ss_size;
-	abi_long ss_flags;
+	abi_int ss_flags;
 } target_stack_t;
 
 
@@ -26,4 +26,5 @@
     return state->active_tc.gpr[29];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* MIPS64_TARGET_SIGNAL_H */
diff --git a/linux-user/mips64/target_syscall.h b/linux-user/mips64/target_syscall.h
new file mode 100644
index 0000000..8da9c1f
--- /dev/null
+++ b/linux-user/mips64/target_syscall.h
@@ -0,0 +1,230 @@
+#ifndef MIPS64_TARGET_SYSCALL_H
+#define MIPS64_TARGET_SYSCALL_H
+
+/* this struct defines the way the registers are stored on the
+   stack during a system call. */
+
+struct target_pt_regs {
+        /* Saved main processor registers. */
+        target_ulong regs[32];
+
+        /* Saved special registers. */
+        target_ulong cp0_status;
+        target_ulong lo;
+        target_ulong hi;
+        target_ulong cp0_badvaddr;
+        target_ulong cp0_cause;
+        target_ulong cp0_epc;
+};
+
+/* Target errno definitions taken from asm-mips/errno.h */
+#undef TARGET_ENOMSG
+#define TARGET_ENOMSG          35      /* Identifier removed */
+#undef TARGET_EIDRM
+#define TARGET_EIDRM           36      /* Identifier removed */
+#undef TARGET_ECHRNG
+#define TARGET_ECHRNG          37      /* Channel number out of range */
+#undef TARGET_EL2NSYNC
+#define TARGET_EL2NSYNC        38      /* Level 2 not synchronized */
+#undef TARGET_EL3HLT
+#define TARGET_EL3HLT          39      /* Level 3 halted */
+#undef TARGET_EL3RST
+#define TARGET_EL3RST          40      /* Level 3 reset */
+#undef TARGET_ELNRNG
+#define TARGET_ELNRNG          41      /* Link number out of range */
+#undef TARGET_EUNATCH
+#define TARGET_EUNATCH         42      /* Protocol driver not attached */
+#undef TARGET_ENOCSI
+#define TARGET_ENOCSI          43      /* No CSI structure available */
+#undef TARGET_EL2HLT
+#define TARGET_EL2HLT          44      /* Level 2 halted */
+#undef TARGET_EDEADLK
+#define TARGET_EDEADLK         45      /* Resource deadlock would occur */
+#undef TARGET_ENOLCK
+#define TARGET_ENOLCK          46      /* No record locks available */
+#undef TARGET_EBADE
+#define TARGET_EBADE           50      /* Invalid exchange */
+#undef TARGET_EBADR
+#define TARGET_EBADR           51      /* Invalid request descriptor */
+#undef TARGET_EXFULL
+#define TARGET_EXFULL          52      /* TARGET_Exchange full */
+#undef TARGET_ENOANO
+#define TARGET_ENOANO          53      /* No anode */
+#undef TARGET_EBADRQC
+#define TARGET_EBADRQC         54      /* Invalid request code */
+#undef TARGET_EBADSLT
+#define TARGET_EBADSLT         55      /* Invalid slot */
+#undef TARGET_EDEADLOCK
+#define TARGET_EDEADLOCK       56      /* File locking deadlock error */
+#undef TARGET_EBFONT
+#define TARGET_EBFONT          59      /* Bad font file format */
+#undef TARGET_ENOSTR
+#define TARGET_ENOSTR          60      /* Device not a stream */
+#undef TARGET_ENODATA
+#define TARGET_ENODATA         61      /* No data available */
+#undef TARGET_ETIME
+#define TARGET_ETIME           62      /* Timer expired */
+#undef TARGET_ENOSR
+#define TARGET_ENOSR           63      /* Out of streams resources */
+#undef TARGET_ENONET
+#define TARGET_ENONET          64      /* Machine is not on the network */
+#undef TARGET_ENOPKG
+#define TARGET_ENOPKG          65      /* Package not installed */
+#undef TARGET_EREMOTE
+#define TARGET_EREMOTE         66      /* Object is remote */
+#undef TARGET_ENOLINK
+#define TARGET_ENOLINK         67      /* Link has been severed */
+#undef TARGET_EADV
+#define TARGET_EADV            68      /* Advertise error */
+#undef TARGET_ESRMNT
+#define TARGET_ESRMNT          69      /* Srmount error */
+#undef TARGET_ECOMM
+#define TARGET_ECOMM           70      /* Communication error on send */
+#undef TARGET_EPROTO
+#define TARGET_EPROTO          71      /* Protocol error */
+#undef TARGET_EDOTDOT
+#define TARGET_EDOTDOT         73      /* RFS specific error */
+#undef TARGET_EMULTIHOP
+#define TARGET_EMULTIHOP       74      /* Multihop attempted */
+#undef TARGET_EBADMSG
+#define TARGET_EBADMSG         77      /* Not a data message */
+#undef TARGET_ENAMETOOLONG
+#define TARGET_ENAMETOOLONG    78      /* File name too long */
+#undef TARGET_EOVERFLOW
+#define TARGET_EOVERFLOW       79      /* Value too large for defined data type */
+#undef TARGET_ENOTUNIQ
+#define TARGET_ENOTUNIQ        80      /* Name not unique on network */
+#undef TARGET_EBADFD
+#define TARGET_EBADFD          81      /* File descriptor in bad state */
+#undef TARGET_EREMCHG
+#define TARGET_EREMCHG         82      /* Remote address changed */
+#undef TARGET_ELIBACC
+#define TARGET_ELIBACC         83      /* Can not access a needed shared library */
+#undef TARGET_ELIBBAD
+#define TARGET_ELIBBAD         84      /* Accessing a corrupted shared library */
+#undef TARGET_ELIBSCN
+#define TARGET_ELIBSCN         85      /* .lib section in a.out corrupted */
+#undef TARGET_ELIBMAX
+#define TARGET_ELIBMAX         86      /* Attempting to link in too many shared libraries */
+#undef TARGET_ELIBEXEC
+#define TARGET_ELIBEXEC        87      /* Cannot exec a shared library directly */
+#undef TARGET_EILSEQ
+#define TARGET_EILSEQ          88      /* Illegal byte sequence */
+#undef TARGET_ENOSYS
+#define TARGET_ENOSYS          89      /* Function not implemented */
+#undef TARGET_ELOOP
+#define TARGET_ELOOP           90      /* Too many symbolic links encountered */
+#undef TARGET_ERESTART
+#define TARGET_ERESTART        91      /* Interrupted system call should be restarted */
+#undef TARGET_ESTRPIPE
+#define TARGET_ESTRPIPE        92      /* Streams pipe error */
+#undef TARGET_ENOTEMPTY
+#define TARGET_ENOTEMPTY       93      /* Directory not empty */
+#undef TARGET_EUSERS
+#define TARGET_EUSERS          94      /* Too many users */
+#undef TARGET_ENOTSOCK
+#define TARGET_ENOTSOCK        95      /* Socket operation on non-socket */
+#undef TARGET_EDESTADDRREQ
+#define TARGET_EDESTADDRREQ    96      /* Destination address required */
+#undef TARGET_EMSGSIZE
+#define TARGET_EMSGSIZE        97      /* Message too long */
+#undef TARGET_EPROTOTYPE
+#define TARGET_EPROTOTYPE      98      /* Protocol wrong type for socket */
+#undef TARGET_ENOPROTOOPT
+#define TARGET_ENOPROTOOPT     99      /* Protocol not available */
+#undef TARGET_EPROTONOSUPPORT
+#define TARGET_EPROTONOSUPPORT 120     /* Protocol not supported */
+#undef TARGET_ESOCKTNOSUPPORT
+#define TARGET_ESOCKTNOSUPPORT 121     /* Socket type not supported */
+#undef TARGET_EOPNOTSUPP
+#define TARGET_EOPNOTSUPP      122     /* Operation not supported on transport endpoint */
+#undef TARGET_EPFNOSUPPORT
+#define TARGET_EPFNOSUPPORT    123     /* Protocol family not supported */
+#undef TARGET_EAFNOSUPPORT
+#define TARGET_EAFNOSUPPORT    124     /* Address family not supported by protocol */
+#undef TARGET_EADDRINUSE
+#define TARGET_EADDRINUSE      125     /* Address already in use */
+#undef TARGET_EADDRNOTAVAIL
+#define TARGET_EADDRNOTAVAIL   126     /* Cannot assign requested address */
+#undef TARGET_ENETDOWN
+#define TARGET_ENETDOWN        127     /* Network is down */
+#undef TARGET_ENETUNREACH
+#define TARGET_ENETUNREACH     128     /* Network is unreachable */
+#undef TARGET_ENETRESET
+#define TARGET_ENETRESET       129     /* Network dropped connection because of reset */
+#undef TARGET_ECONNABORTED
+#define TARGET_ECONNABORTED    130     /* Software caused connection abort */
+#undef TARGET_ECONNRESET
+#define TARGET_ECONNRESET      131     /* Connection reset by peer */
+#undef TARGET_ENOBUFS
+#define TARGET_ENOBUFS         132     /* No buffer space available */
+#undef TARGET_EISCONN
+#define TARGET_EISCONN         133     /* Transport endpoint is already connected */
+#undef TARGET_ENOTCONN
+#define TARGET_ENOTCONN        134     /* Transport endpoint is not connected */
+#undef TARGET_EUCLEAN
+#define TARGET_EUCLEAN         135     /* Structure needs cleaning */
+#undef TARGET_ENOTNAM
+#define TARGET_ENOTNAM         137     /* Not a XENIX named type file */
+#undef TARGET_ENAVAIL
+#define TARGET_ENAVAIL         138     /* No XENIX semaphores available */
+#undef TARGET_EISNAM
+#define TARGET_EISNAM          139     /* Is a named type file */
+#undef TARGET_EREMOTEIO
+#define TARGET_EREMOTEIO       140     /* Remote I/O error */
+#undef TARGET_EINIT
+#define TARGET_EINIT           141     /* Reserved */
+#undef TARGET_EREMDEV
+#define TARGET_EREMDEV         142     /* TARGET_Error 142 */
+#undef TARGET_ESHUTDOWN
+#define TARGET_ESHUTDOWN       143     /* Cannot send after transport endpoint shutdown */
+#undef TARGET_ETOOMANYREFS
+#define TARGET_ETOOMANYREFS    144     /* Too many references: cannot splice */
+#undef TARGET_ETIMEDOUT
+#define TARGET_ETIMEDOUT       145     /* Connection timed out */
+#undef TARGET_ECONNREFUSED
+#define TARGET_ECONNREFUSED    146     /* Connection refused */
+#undef TARGET_EHOSTDOWN
+#define TARGET_EHOSTDOWN       147     /* Host is down */
+#undef TARGET_EHOSTUNREACH
+#define TARGET_EHOSTUNREACH    148     /* No route to host */
+#undef TARGET_EALREADY
+#define TARGET_EALREADY        149     /* Operation already in progress */
+#undef TARGET_EINPROGRESS
+#define TARGET_EINPROGRESS     150     /* Operation now in progress */
+#undef TARGET_ESTALE
+#define TARGET_ESTALE          151     /* Stale NFS file handle */
+#undef TARGET_ECANCELED
+#define TARGET_ECANCELED       158     /* AIO operation canceled */
+/*
+ * These error are Linux extensions.
+ */
+#undef TARGET_ENOMEDIUM
+#define TARGET_ENOMEDIUM       159     /* No medium found */
+#undef TARGET_EMEDIUMTYPE
+#define TARGET_EMEDIUMTYPE     160     /* Wrong medium type */
+#undef TARGET_ENOKEY
+#define TARGET_ENOKEY          161     /* Required key not available */
+#undef TARGET_EKEYEXPIRED
+#define TARGET_EKEYEXPIRED     162     /* Key has expired */
+#undef TARGET_EKEYREVOKED
+#define TARGET_EKEYREVOKED     163     /* Key has been revoked */
+#undef TARGET_EKEYREJECTED
+#define TARGET_EKEYREJECTED    164     /* Key was rejected by service */
+
+/* for robust mutexes */
+#undef TARGET_EOWNERDEAD
+#define TARGET_EOWNERDEAD      165     /* Owner died */
+#undef TARGET_ENOTRECOVERABLE
+#define TARGET_ENOTRECOVERABLE 166     /* State not recoverable */
+
+
+#define UNAME_MACHINE "mips64"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* MIPS64_TARGET_SYSCALL_H */
diff --git a/linux-user/mmap.c b/linux-user/mmap.c
index a249f0c..c4371d9 100644
--- a/linux-user/mmap.c
+++ b/linux-user/mmap.c
@@ -16,20 +16,13 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/mman.h>
+#include "qemu/osdep.h"
 #include <linux/mman.h>
 #include <linux/unistd.h>
 
 #include "qemu.h"
 #include "qemu-common.h"
+#include "translate-all.h"
 
 //#define DEBUG_MMAP
 
@@ -185,10 +178,12 @@
         if (prot_new != (prot1 | PROT_WRITE))
             mprotect(host_start, qemu_host_page_size, prot_new);
     } else {
-        /* just update the protection */
         if (prot_new != prot1) {
             mprotect(host_start, qemu_host_page_size, prot_new);
         }
+        if (prot_new & PROT_WRITE) {
+            memset(g2h(start), 0, end - start);
+        }
     }
     return 0;
 }
@@ -205,7 +200,6 @@
 
 unsigned long last_brk;
 
-#ifdef CONFIG_USE_GUEST_BASE
 /* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk
    of guest address space.  */
 static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size)
@@ -215,14 +209,14 @@
     int prot;
     int looped = 0;
 
-    if (size > RESERVED_VA) {
+    if (size > reserved_va) {
         return (abi_ulong)-1;
     }
 
     size = HOST_PAGE_ALIGN(size);
     end_addr = start + size;
-    if (end_addr > RESERVED_VA) {
-        end_addr = RESERVED_VA;
+    if (end_addr > reserved_va) {
+        end_addr = reserved_va;
     }
     addr = end_addr - qemu_host_page_size;
 
@@ -231,7 +225,7 @@
             if (looped) {
                 return (abi_ulong)-1;
             }
-            end_addr = RESERVED_VA;
+            end_addr = reserved_va;
             addr = end_addr - qemu_host_page_size;
             looped = 1;
             continue;
@@ -252,7 +246,6 @@
 
     return addr;
 }
-#endif
 
 /*
  * Find and reserve a free memory area of size 'size'. The search
@@ -275,11 +268,9 @@
 
     size = HOST_PAGE_ALIGN(size);
 
-#ifdef CONFIG_USE_GUEST_BASE
-    if (RESERVED_VA) {
+    if (reserved_va) {
         return mmap_find_vma_reserved(start, size);
     }
-#endif
 
     addr = start;
     wrapped = repeat = 0;
@@ -447,9 +438,7 @@
            /* If so, truncate the file map at eof aligned with 
               the hosts real pagesize. Additional anonymous maps
               will be created beyond EOF.  */
-           len = (sb.st_size - offset);
-           len += qemu_real_host_page_size - 1;
-           len &= ~(qemu_real_host_page_size - 1);
+           len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset);
        }
     }
 
@@ -517,10 +506,7 @@
                 goto fail;
             if (!(prot & PROT_WRITE)) {
                 ret = target_mprotect(start, len, prot);
-                if (ret != 0) {
-                    start = ret;
-                    goto the_end;
-                }
+                assert(ret == 0);
             }
             goto the_end;
         }
@@ -544,7 +530,7 @@
         /* handle the end of the mapping */
         if (end < real_end) {
             ret = mmap_frag(real_end - qemu_host_page_size,
-                            real_end - qemu_host_page_size, real_end,
+                            real_end - qemu_host_page_size, end,
                             prot, flags, fd,
                             offset + real_end - qemu_host_page_size - start);
             if (ret == -1)
@@ -574,7 +560,7 @@
     page_dump(stdout);
     printf("\n");
 #endif
-    tb_invalidate_phys_range(start, start + len, 0);
+    tb_invalidate_phys_range(start, start + len);
     mmap_unlock();
     return start;
 fail:
@@ -670,7 +656,7 @@
     ret = 0;
     /* unmap what we can */
     if (real_start < real_end) {
-        if (RESERVED_VA) {
+        if (reserved_va) {
             mmap_reserve(real_start, real_end - real_start);
         } else {
             ret = munmap(g2h(real_start), real_end - real_start);
@@ -679,7 +665,7 @@
 
     if (ret == 0) {
         page_set_flags(start, start + len, 0);
-        tb_invalidate_phys_range(start, start + len, 0);
+        tb_invalidate_phys_range(start, start + len);
     }
     mmap_unlock();
     return ret;
@@ -700,7 +686,7 @@
                                      flags,
                                      g2h(new_addr));
 
-        if (RESERVED_VA && host_addr != MAP_FAILED) {
+        if (reserved_va && host_addr != MAP_FAILED) {
             /* If new and old addresses overlap then the above mremap will
                already have failed with EINVAL.  */
             mmap_reserve(old_addr, old_size);
@@ -718,13 +704,13 @@
                                          old_size, new_size,
                                          flags | MREMAP_FIXED,
                                          g2h(mmap_start));
-            if ( RESERVED_VA ) {
+            if (reserved_va) {
                 mmap_reserve(old_addr, old_size);
             }
         }
     } else {
         int prot = 0;
-        if (RESERVED_VA && old_size < new_size) {
+        if (reserved_va && old_size < new_size) {
             abi_ulong addr;
             for (addr = old_addr + old_size;
                  addr < old_addr + new_size;
@@ -734,7 +720,7 @@
         }
         if (prot == 0) {
             host_addr = mremap(g2h(old_addr), old_size, new_size, flags);
-            if (host_addr != MAP_FAILED && RESERVED_VA && old_size > new_size) {
+            if (host_addr != MAP_FAILED && reserved_va && old_size > new_size) {
                 mmap_reserve(old_addr + old_size, new_size - old_size);
             }
         } else {
@@ -758,7 +744,7 @@
         page_set_flags(old_addr, old_addr + old_size, 0);
         page_set_flags(new_addr, new_addr + new_size, prot | PAGE_VALID);
     }
-    tb_invalidate_phys_range(new_addr, new_addr + new_size, 0);
+    tb_invalidate_phys_range(new_addr, new_addr + new_size);
     mmap_unlock();
     return new_addr;
 }
diff --git a/linux-user/openrisc/syscall.h b/linux-user/openrisc/syscall.h
deleted file mode 100644
index 8ac0365..0000000
--- a/linux-user/openrisc/syscall.h
+++ /dev/null
@@ -1,29 +0,0 @@
-struct target_pt_regs {
-    union {
-        struct {
-            /* Named registers */
-            uint32_t sr;       /* Stored in place of r0 */
-            target_ulong sp;   /* r1 */
-        };
-        struct {
-            /* Old style */
-            target_ulong offset[2];
-            target_ulong gprs[30];
-        };
-        struct {
-            /* New style */
-            target_ulong gpr[32];
-        };
-    };
-    target_ulong pc;
-    target_ulong orig_gpr11;   /* For restarting system calls */
-    uint32_t syscallno;        /* Syscall number (used by strace) */
-    target_ulong dummy;     /* Cheap alignment fix */
-};
-
-#define UNAME_MACHINE "openrisc"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/openrisc/syscall_nr.h b/linux-user/openrisc/syscall_nr.h
index 4c386ea..6b1c7d2 100644
--- a/linux-user/openrisc/syscall_nr.h
+++ b/linux-user/openrisc/syscall_nr.h
@@ -382,9 +382,18 @@
 #define TARGET_NR_process_vm_writev 271
 #define TARGET_NR_kcmp 272
 #define TARGET_NR_finit_module 273
-
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls 274
+#define TARGET_NR_sched_setattr 274
+#define TARGET_NR_sched_getattr 275
+#define TARGET_NR_renameat2 276
+#define TARGET_NR_seccomp 277
+#define TARGET_NR_getrandom 278
+#define TARGET_NR_memfd_create 279
+#define TARGET_NR_bpf 280
+#define TARGET_NR_execveat 281
+#define TARGET_NR_userfaultfd 282
+#define TARGET_NR_membarrier 283
+#define TARGET_NR_mlock2 284
+#define TARGET_NR_copy_file_range 285
 
 /*
  * All syscalls below here should go away really,
@@ -411,9 +420,6 @@
 #define TARGET_NR_3264_stat 1038
 #define TARGET_NR_3264_lstat 1039
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_3264_lstat+1)
-
 #define TARGET_NR_pipe 1040
 #define TARGET_NR_dup2 1041
 #define TARGET_NR_epoll_create 1042
@@ -421,10 +427,6 @@
 #define TARGET_NR_eventfd 1044
 #define TARGET_NR_signalfd 1045
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_signalfd+1)
-
-
 #define TARGET_NR_sendfile 1046
 #define TARGET_NR_ftruncate 1047
 #define TARGET_NR_truncate 1048
@@ -441,9 +443,6 @@
 #define TARGET_NR_lseek 1057
 #define TARGET_NR_mmap 1058
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_mmap+1)
-
 #define TARGET_NR_alarm 1059
 #define __ARCH_WANT_SYS_ALARM
 #define TARGET_NR_getpgrp 1060
@@ -477,9 +476,6 @@
 
 #define TARGET_NR_fork 1079
 
-#undef TARGET_NR_syscalls
-#define TARGET_NR_syscalls (TARGET_NR_fork+1)
-
 
 /*
  * 32 bit systems traditionally used different
diff --git a/linux-user/openrisc/target_cpu.h b/linux-user/openrisc/target_cpu.h
index 32a46ac..a21ed1a 100644
--- a/linux-user/openrisc/target_cpu.h
+++ b/linux-user/openrisc/target_cpu.h
@@ -17,8 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef OPENRISC_TARGET_CPU_H
+#define OPENRISC_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUOpenRISCState *env, target_ulong newsp)
 {
diff --git a/linux-user/openrisc/target_signal.h b/linux-user/openrisc/target_signal.h
index 964aed6..9f2c493 100644
--- a/linux-user/openrisc/target_signal.h
+++ b/linux-user/openrisc/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef OPENRISC_TARGET_SIGNAL_H
+#define OPENRISC_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -23,4 +23,5 @@
     return state->gpr[1];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* OPENRISC_TARGET_SIGNAL_H */
diff --git a/linux-user/openrisc/target_structs.h b/linux-user/openrisc/target_structs.h
index f4d560f..afbb7ad 100644
--- a/linux-user/openrisc/target_structs.h
+++ b/linux-user/openrisc/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef OPENRISC_TARGET_STRUCTS_H
+#define OPENRISC_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/openrisc/target_syscall.h b/linux-user/openrisc/target_syscall.h
new file mode 100644
index 0000000..9d3380f
--- /dev/null
+++ b/linux-user/openrisc/target_syscall.h
@@ -0,0 +1,34 @@
+#ifndef OPENRISC_TARGET_SYSCALL_H
+#define OPENRISC_TARGET_SYSCALL_H
+
+struct target_pt_regs {
+    union {
+        struct {
+            /* Named registers */
+            uint32_t sr;       /* Stored in place of r0 */
+            target_ulong sp;   /* r1 */
+        };
+        struct {
+            /* Old style */
+            target_ulong offset[2];
+            target_ulong gprs[30];
+        };
+        struct {
+            /* New style */
+            target_ulong gpr[32];
+        };
+    };
+    target_ulong pc;
+    target_ulong orig_gpr11;   /* For restarting system calls */
+    uint32_t syscallno;        /* Syscall number (used by strace) */
+    target_ulong dummy;     /* Cheap alignment fix */
+};
+
+#define UNAME_MACHINE "openrisc"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* OPENRISC_TARGET_SYSCALL_H */
diff --git a/linux-user/ppc/syscall.h b/linux-user/ppc/syscall.h
deleted file mode 100644
index 0daf5cd..0000000
--- a/linux-user/ppc/syscall.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- *  PPC emulation for qemu: syscall definitions.
- *
- *  Copyright (c) 2003 Jocelyn Mayer
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/* XXX: ABSOLUTELY BUGGY:
- * for now, this is quite just a cut-and-paste from i386 target...
- */
-
-/* default linux values for the selectors */
-#define __USER_DS	(1)
-
-struct target_pt_regs {
-	abi_ulong gpr[32];
-	abi_ulong nip;
-	abi_ulong msr;
-	abi_ulong orig_gpr3;	/* Used for restarting system calls */
-	abi_ulong ctr;
-	abi_ulong link;
-	abi_ulong xer;
-	abi_ulong ccr;
-#if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
-        abi_ulong softe;
-#else
-	abi_ulong mq;		/* 601 only (not used at present) */
-#endif
-					/* Used on APUS to hold IPL value. */
-	abi_ulong trap;		/* Reason for being here */
-	abi_ulong dar;		/* Fault registers */
-	abi_ulong dsisr;
-	abi_ulong result; 		/* Result of a system call */
-};
-
-/* ioctls */
-struct target_revectored_struct {
-	abi_ulong __map[8];			/* 256 bits */
-};
-
-/* Nasty hack: define a fake errno value for use by sigreturn.  */
-#define TARGET_QEMU_ESIGRETURN 255
-
-/*
- * flags masks
- */
-
-#if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
-#ifdef TARGET_WORDS_BIGENDIAN
-#define UNAME_MACHINE "ppc64"
-#else
-#define UNAME_MACHINE "ppc64le"
-#endif
-#else
-#define UNAME_MACHINE "ppc"
-#endif
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_CLONE_BACKWARDS
-
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
-#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
diff --git a/linux-user/ppc/syscall_nr.h b/linux-user/ppc/syscall_nr.h
index 1e1736e..46ed8a6 100644
--- a/linux-user/ppc/syscall_nr.h
+++ b/linux-user/ppc/syscall_nr.h
@@ -319,7 +319,7 @@
 #define TARGET_NR_epoll_pwait	303
 #define TARGET_NR_utimensat		304
 #define TARGET_NR_signalfd		305
-#define TARGET_NR_timerfd		306
+#define TARGET_NR_timerfd_create	306
 #define TARGET_NR_eventfd		307
 #define TARGET_NR_sync_file_range2	308
 #define TARGET_NR_fallocate		309
@@ -368,3 +368,27 @@
 #define TARGET_NR_process_vm_writev     352
 #define TARGET_NR_finit_module          353
 #define TARGET_NR_kcmp                  354
+#define TARGET_NR_sched_setattr         355
+#define TARGET_NR_sched_getattr         356
+#define TARGET_NR_renameat2             357
+#define TARGET_NR_seccomp               358
+#define TARGET_NR_getrandom             359
+#define TARGET_NR_memfd_create          360
+#define TARGET_NR_bpf                   361
+#define TARGET_NR_execveat              362
+#define TARGET_NR_switch_endian         363
+#define TARGET_NR_userfaultfd           364
+#define TARGET_NR_membarrier            365
+#define TARGET_NR_semop                 366
+#define TARGET_NR_semget                367
+#define TARGET_NR_semctl                368
+#define TARGET_NR_semtimedop            369
+#define TARGET_NR_msgsnd                370
+#define TARGET_NR_msgrcv                371
+#define TARGET_NR_msgget                372
+#define TARGET_NR_msgctl                373
+#define TARGET_NR_shmat                 374
+#define TARGET_NR_shmdt                 375
+#define TARGET_NR_shmget                376
+#define TARGET_NR_shmctl                377
+#define TARGET_NR_mlock2                378
diff --git a/linux-user/ppc/target_cpu.h b/linux-user/ppc/target_cpu.h
index 26f4ba2..3aab3d1 100644
--- a/linux-user/ppc/target_cpu.h
+++ b/linux-user/ppc/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef PPC_TARGET_CPU_H
+#define PPC_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUPPCState *env, target_ulong newsp)
 {
diff --git a/linux-user/ppc/target_signal.h b/linux-user/ppc/target_signal.h
index a93b5cf..865c52f 100644
--- a/linux-user/ppc/target_signal.h
+++ b/linux-user/ppc/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef PPC_TARGET_SIGNAL_H
+#define PPC_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
     return state->gpr[1];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* PPC_TARGET_SIGNAL_H */
diff --git a/linux-user/ppc/target_structs.h b/linux-user/ppc/target_structs.h
index 2b87613..6b1f579 100644
--- a/linux-user/ppc/target_structs.h
+++ b/linux-user/ppc/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef PPC_TARGET_STRUCTS_H
+#define PPC_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/ppc/target_syscall.h b/linux-user/ppc/target_syscall.h
new file mode 100644
index 0000000..a8662f4
--- /dev/null
+++ b/linux-user/ppc/target_syscall.h
@@ -0,0 +1,78 @@
+/*
+ *  PPC emulation for qemu: syscall definitions.
+ *
+ *  Copyright (c) 2003 Jocelyn Mayer
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef PPC_TARGET_SYSCALL_H
+#define PPC_TARGET_SYSCALL_H
+
+/* XXX: ABSOLUTELY BUGGY:
+ * for now, this is quite just a cut-and-paste from i386 target...
+ */
+
+/* default linux values for the selectors */
+#define __USER_DS	(1)
+
+struct target_pt_regs {
+	abi_ulong gpr[32];
+	abi_ulong nip;
+	abi_ulong msr;
+	abi_ulong orig_gpr3;	/* Used for restarting system calls */
+	abi_ulong ctr;
+	abi_ulong link;
+	abi_ulong xer;
+	abi_ulong ccr;
+#if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
+        abi_ulong softe;
+#else
+	abi_ulong mq;		/* 601 only (not used at present) */
+#endif
+					/* Used on APUS to hold IPL value. */
+	abi_ulong trap;		/* Reason for being here */
+	abi_ulong dar;		/* Fault registers */
+	abi_ulong dsisr;
+	abi_ulong result; 		/* Result of a system call */
+};
+
+/* ioctls */
+struct target_revectored_struct {
+	abi_ulong __map[8];			/* 256 bits */
+};
+
+
+/*
+ * flags masks
+ */
+
+#if defined(TARGET_PPC64) && !defined(TARGET_ABI32)
+#ifdef TARGET_WORDS_BIGENDIAN
+#define UNAME_MACHINE "ppc64"
+#else
+#define UNAME_MACHINE "ppc64le"
+#endif
+#else
+#define UNAME_MACHINE "ppc"
+#endif
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_CLONE_BACKWARDS
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
+
+#endif /* PPC_TARGET_SYSCALL_H */
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 8012cc2..bef465d 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -1,27 +1,30 @@
 #ifndef QEMU_H
 #define QEMU_H
 
-#include <signal.h>
-#include <string.h>
-
+#include "hostdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #undef DEBUG_REMAP
 #ifdef DEBUG_REMAP
-#include <stdlib.h>
 #endif /* DEBUG_REMAP */
 
 #include "exec/user/abitypes.h"
 
 #include "exec/user/thunk.h"
 #include "syscall_defs.h"
-#include "syscall.h"
+#include "target_syscall.h"
 #include "exec/gdbstub.h"
 #include "qemu/queue.h"
 
 #define THREAD __thread
 
+/* This is the size of the host kernel's sigset_t, needed where we make
+ * direct system calls that take a sigset_t pointer and a size.
+ */
+#define SIGSET_T_SIZE (_NSIG / 8)
+
 /* This struct is used to hold certain information about the image.
  * Basically, it replicates in user space what would be certain
  * task_struct fields in the kernel
@@ -36,8 +39,6 @@
         abi_ulong       start_brk;
         abi_ulong       brk;
         abi_ulong       start_mmap;
-        abi_ulong       mmap;
-        abi_ulong       rss;
         abi_ulong       start_stack;
         abi_ulong       stack_limit;
         abi_ulong       entry;
@@ -82,16 +83,9 @@
 
 #define MAX_SIGQUEUE_SIZE 1024
 
-struct sigqueue {
-    struct sigqueue *next;
-    target_siginfo_t info;
-};
-
 struct emulated_sigtable {
     int pending; /* true if signal is pending */
-    struct sigqueue *first;
-    struct sigqueue info; /* in order to always have memory for the
-                             first signal, we put it here */
+    target_siginfo_t info;
 };
 
 /* NOTE: we force a big alignment so that the stack stored after is
@@ -122,19 +116,37 @@
 #endif
 #if defined(TARGET_ARM) || defined(TARGET_M68K) || defined(TARGET_UNICORE32)
     /* Extra fields for semihosted binaries.  */
-    uint32_t heap_base;
-    uint32_t heap_limit;
+    abi_ulong heap_base;
+    abi_ulong heap_limit;
 #endif
-    uint32_t stack_base;
+    abi_ulong stack_base;
     int used; /* non zero if used */
-    bool sigsegv_blocked; /* SIGSEGV blocked by guest */
     struct image_info *info;
     struct linux_binprm *bprm;
 
+    struct emulated_sigtable sync_signal;
     struct emulated_sigtable sigtab[TARGET_NSIG];
-    struct sigqueue sigqueue_table[MAX_SIGQUEUE_SIZE]; /* siginfo queue */
-    struct sigqueue *first_free; /* first free siginfo queue entry */
-    int signal_pending; /* non zero if a signal may be pending */
+    /* This thread's signal mask, as requested by the guest program.
+     * The actual signal mask of this thread may differ:
+     *  + we don't let SIGSEGV and SIGBUS be blocked while running guest code
+     *  + sometimes we block all signals to avoid races
+     */
+    sigset_t signal_mask;
+    /* The signal mask imposed by a guest sigsuspend syscall, if we are
+     * currently in the middle of such a syscall
+     */
+    sigset_t sigsuspend_mask;
+    /* Nonzero if we're leaving a sigsuspend and sigsuspend_mask is valid. */
+    int in_sigsuspend;
+
+    /* Nonzero if process_pending_signals() needs to do something (either
+     * handle a pending signal or unblock signals).
+     * This flag is written from a signal handler so should be accessed via
+     * the atomic_read() and atomic_write() functions. (It is not accessed
+     * from multiple threads.)
+     */
+    int signal_pending;
+
 } __attribute__((aligned(16))) TaskState;
 
 extern char *exec_path;
@@ -145,12 +157,6 @@
 extern unsigned long mmap_min_addr;
 
 /* ??? See if we can avoid exposing so much of the loader internals.  */
-/*
- * MAX_ARG_PAGES defines the number of pages allocated for arguments
- * and envelope for the new program. 32 should suffice, this gives
- * a maximum env+arg of 128kB w/4KB pages!
- */
-#define MAX_ARG_PAGES 33
 
 /* Read a good amount of data initially, to hopefully get all the
    program headers loaded.  */
@@ -162,7 +168,6 @@
  */
 struct linux_binprm {
         char buf[BPRM_BUF_SIZE] __attribute__((aligned));
-        void *page[MAX_ARG_PAGES];
         abi_ulong p;
 	int fd;
         int e_uid, e_gid;
@@ -195,7 +200,7 @@
 void gemu_log(const char *fmt, ...) GCC_FMT_ATTR(1, 2);
 extern THREAD CPUState *thread_cpu;
 void cpu_loop(CPUArchState *env);
-char *target_strerror(int err);
+const char *target_strerror(int err);
 int get_osversion(void);
 void init_qemu_uname_release(void);
 void fork_start(void);
@@ -216,6 +221,139 @@
 
 #include "qemu/log.h"
 
+/* safe_syscall.S */
+
+/**
+ * safe_syscall:
+ * @int number: number of system call to make
+ * ...: arguments to the system call
+ *
+ * Call a system call if guest signal not pending.
+ * This has the same API as the libc syscall() function, except that it
+ * may return -1 with errno == TARGET_ERESTARTSYS if a signal was pending.
+ *
+ * Returns: the system call result, or -1 with an error code in errno
+ * (Errnos are host errnos; we rely on TARGET_ERESTARTSYS not clashing
+ * with any of the host errno values.)
+ */
+
+/* A guide to using safe_syscall() to handle interactions between guest
+ * syscalls and guest signals:
+ *
+ * Guest syscalls come in two flavours:
+ *
+ * (1) Non-interruptible syscalls
+ *
+ * These are guest syscalls that never get interrupted by signals and
+ * so never return EINTR. They can be implemented straightforwardly in
+ * QEMU: just make sure that if the implementation code has to make any
+ * blocking calls that those calls are retried if they return EINTR.
+ * It's also OK to implement these with safe_syscall, though it will be
+ * a little less efficient if a signal is delivered at the 'wrong' moment.
+ *
+ * Some non-interruptible syscalls need to be handled using block_signals()
+ * to block signals for the duration of the syscall. This mainly applies
+ * to code which needs to modify the data structures used by the
+ * host_signal_handler() function and the functions it calls, including
+ * all syscalls which change the thread's signal mask.
+ *
+ * (2) Interruptible syscalls
+ *
+ * These are guest syscalls that can be interrupted by signals and
+ * for which we need to either return EINTR or arrange for the guest
+ * syscall to be restarted. This category includes both syscalls which
+ * always restart (and in the kernel return -ERESTARTNOINTR), ones
+ * which only restart if there is no handler (kernel returns -ERESTARTNOHAND
+ * or -ERESTART_RESTARTBLOCK), and the most common kind which restart
+ * if the handler was registered with SA_RESTART (kernel returns
+ * -ERESTARTSYS). System calls which are only interruptible in some
+ * situations (like 'open') also need to be handled this way.
+ *
+ * Here it is important that the host syscall is made
+ * via this safe_syscall() function, and *not* via the host libc.
+ * If the host libc is used then the implementation will appear to work
+ * most of the time, but there will be a race condition where a
+ * signal could arrive just before we make the host syscall inside libc,
+ * and then then guest syscall will not correctly be interrupted.
+ * Instead the implementation of the guest syscall can use the safe_syscall
+ * function but otherwise just return the result or errno in the usual
+ * way; the main loop code will take care of restarting the syscall
+ * if appropriate.
+ *
+ * (If the implementation needs to make multiple host syscalls this is
+ * OK; any which might really block must be via safe_syscall(); for those
+ * which are only technically blocking (ie which we know in practice won't
+ * stay in the host kernel indefinitely) it's OK to use libc if necessary.
+ * You must be able to cope with backing out correctly if some safe_syscall
+ * you make in the implementation returns either -TARGET_ERESTARTSYS or
+ * EINTR though.)
+ *
+ * block_signals() cannot be used for interruptible syscalls.
+ *
+ *
+ * How and why the safe_syscall implementation works:
+ *
+ * The basic setup is that we make the host syscall via a known
+ * section of host native assembly. If a signal occurs, our signal
+ * handler checks the interrupted host PC against the addresse of that
+ * known section. If the PC is before or at the address of the syscall
+ * instruction then we change the PC to point at a "return
+ * -TARGET_ERESTARTSYS" code path instead, and then exit the signal handler
+ * (causing the safe_syscall() call to immediately return that value).
+ * Then in the main.c loop if we see this magic return value we adjust
+ * the guest PC to wind it back to before the system call, and invoke
+ * the guest signal handler as usual.
+ *
+ * This winding-back will happen in two cases:
+ * (1) signal came in just before we took the host syscall (a race);
+ *   in this case we'll take the guest signal and have another go
+ *   at the syscall afterwards, and this is indistinguishable for the
+ *   guest from the timing having been different such that the guest
+ *   signal really did win the race
+ * (2) signal came in while the host syscall was blocking, and the
+ *   host kernel decided the syscall should be restarted;
+ *   in this case we want to restart the guest syscall also, and so
+ *   rewinding is the right thing. (Note that "restart" semantics mean
+ *   "first call the signal handler, then reattempt the syscall".)
+ * The other situation to consider is when a signal came in while the
+ * host syscall was blocking, and the host kernel decided that the syscall
+ * should not be restarted; in this case QEMU's host signal handler will
+ * be invoked with the PC pointing just after the syscall instruction,
+ * with registers indicating an EINTR return; the special code in the
+ * handler will not kick in, and we will return EINTR to the guest as
+ * we should.
+ *
+ * Notice that we can leave the host kernel to make the decision for
+ * us about whether to do a restart of the syscall or not; we do not
+ * need to check SA_RESTART flags in QEMU or distinguish the various
+ * kinds of restartability.
+ */
+#ifdef HAVE_SAFE_SYSCALL
+/* The core part of this function is implemented in assembly */
+extern long safe_syscall_base(int *pending, long number, ...);
+
+#define safe_syscall(...)                                               \
+    ({                                                                  \
+        long ret_;                                                      \
+        int *psp_ = &((TaskState *)thread_cpu->opaque)->signal_pending; \
+        ret_ = safe_syscall_base(psp_, __VA_ARGS__);                    \
+        if (is_error(ret_)) {                                           \
+            errno = -ret_;                                              \
+            ret_ = -1;                                                  \
+        }                                                               \
+        ret_;                                                           \
+    })
+
+#else
+
+/* Fallback for architectures which don't yet provide a safe-syscall assembly
+ * fragment; note that this is racy!
+ * This should go away when all host architectures have been updated.
+ */
+#define safe_syscall syscall
+
+#endif
+
 /* syscall.c */
 int host_to_target_waitstatus(int status);
 
@@ -238,6 +376,25 @@
 long do_rt_sigreturn(CPUArchState *env);
 abi_long do_sigaltstack(abi_ulong uss_addr, abi_ulong uoss_addr, abi_ulong sp);
 int do_sigprocmask(int how, const sigset_t *set, sigset_t *oldset);
+/**
+ * block_signals: block all signals while handling this guest syscall
+ *
+ * Block all signals, and arrange that the signal mask is returned to
+ * its correct value for the guest before we resume execution of guest code.
+ * If this function returns non-zero, then the caller should immediately
+ * return -TARGET_ERESTARTSYS to the main loop, which will take the pending
+ * signal and restart execution of the syscall.
+ * If block_signals() returns zero, then the caller can continue with
+ * emulation of the system call knowing that no signals can be taken
+ * (and therefore that no race conditions will result).
+ * This should only be called once, because if it is called a second time
+ * it will always return non-zero. (Think of it like a mutex that can't
+ * be recursively locked.)
+ * Signals will be unblocked again by process_pending_signals().
+ *
+ * Return value: non-zero if there was a pending signal, zero if not.
+ */
+int block_signals(void); /* Returns non zero if signal pending */
 
 #ifdef TARGET_I386
 /* vm86.c */
@@ -261,11 +418,7 @@
 int target_msync(abi_ulong start, abi_ulong len, int flags);
 extern unsigned long last_brk;
 extern abi_ulong mmap_next_start;
-void mmap_lock(void);
-void mmap_unlock(void);
 abi_ulong mmap_find_vma(abi_ulong, abi_ulong);
-void cpu_list_lock(void);
-void cpu_list_unlock(void);
 void mmap_fork_start(void);
 void mmap_fork_end(int child);
 
diff --git a/linux-user/s390x/syscall.h b/linux-user/s390x/syscall.h
deleted file mode 100644
index 35f170a..0000000
--- a/linux-user/s390x/syscall.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* this typedef defines how a Program Status Word looks like */
-typedef struct {
-    abi_ulong mask;
-    abi_ulong addr;
-} __attribute__ ((aligned(8))) target_psw_t;
-
-/*
- * The pt_regs struct defines the way the registers are stored on
- * the stack during a system call.
- */
-
-#define TARGET_NUM_GPRS        16
-
-struct target_pt_regs {
-    abi_ulong args[1];
-    target_psw_t psw;
-    abi_ulong gprs[TARGET_NUM_GPRS];
-    abi_ulong orig_gpr2;
-    unsigned short ilen;
-    unsigned short trap;
-};
-
-#define UNAME_MACHINE "s390x"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_CLONE_BACKWARDS2
-#define TARGET_MINSIGSTKSZ        2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/s390x/syscall_nr.h b/linux-user/s390x/syscall_nr.h
index 7c0b8b2..1a66c55 100644
--- a/linux-user/s390x/syscall_nr.h
+++ b/linux-user/s390x/syscall_nr.h
@@ -271,6 +271,36 @@
 #define TARGET_NR_s390_runtime_instr    342
 #define TARGET_NR_kcmp                  343
 #define TARGET_NR_finit_module          344
+#define TARGET_NR_sched_setattr         345
+#define TARGET_NR_sched_getattr         346
+#define TARGET_NR_renameat2             347
+#define TARGET_NR_seccomp               348
+#define TARGET_NR_getrandom             349
+#define TARGET_NR_memfd_create          350
+#define TARGET_NR_bpf                   351
+#define TARGET_NR_s390_pci_mmio_write   352
+#define TARGET_NR_s390_pci_mmio_read    353
+#define TARGET_NR_execveat              354
+#define TARGET_NR_userfaultfd           355
+#define TARGET_NR_membarrier            356
+#define TARGET_NR_recvmmsg              357
+#define TARGET_NR_sendmmsg              358
+#define TARGET_NR_socket                359
+#define TARGET_NR_socketpair            360
+#define TARGET_NR_bind                  361
+#define TARGET_NR_connect               362
+#define TARGET_NR_listen                363
+#define TARGET_NR_accept4               364
+#define TARGET_NR_getsockopt            365
+#define TARGET_NR_setsockopt            366
+#define TARGET_NR_getsockname           367
+#define TARGET_NR_getpeername           368
+#define TARGET_NR_sendto                369
+#define TARGET_NR_sendmsg               370
+#define TARGET_NR_recvfrom              371
+#define TARGET_NR_recvmsg               372
+#define TARGET_NR_shutdown              373
+#define TARGET_NR_mlock2                374
 
 /*
  * There are some system calls that are not present on 64 bit, some
diff --git a/linux-user/s390x/target_cpu.h b/linux-user/s390x/target_cpu.h
index f10abe8..87ea4d2 100644
--- a/linux-user/s390x/target_cpu.h
+++ b/linux-user/s390x/target_cpu.h
@@ -19,8 +19,8 @@
  * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef S390X_TARGET_CPU_H
+#define S390X_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUS390XState *env, target_ulong newsp)
 {
diff --git a/linux-user/s390x/target_signal.h b/linux-user/s390x/target_signal.h
index b4816b0..6f7b6ab 100644
--- a/linux-user/s390x/target_signal.h
+++ b/linux-user/s390x/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef S390X_TARGET_SIGNAL_H
+#define S390X_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -23,4 +23,5 @@
    return state->regs[15];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* S390X_TARGET_SIGNAL_H */
diff --git a/linux-user/s390x/target_structs.h b/linux-user/s390x/target_structs.h
index 6b6f5b5..cadff6d 100644
--- a/linux-user/s390x/target_structs.h
+++ b/linux-user/s390x/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef S390X_TARGET_STRUCTS_H
+#define S390X_TARGET_STRUCTS_H
 
 
 struct target_ipc_perm {
diff --git a/linux-user/s390x/target_syscall.h b/linux-user/s390x/target_syscall.h
new file mode 100644
index 0000000..8d4f609
--- /dev/null
+++ b/linux-user/s390x/target_syscall.h
@@ -0,0 +1,34 @@
+#ifndef S390X_TARGET_SYSCALL_H
+#define S390X_TARGET_SYSCALL_H
+
+/* this typedef defines how a Program Status Word looks like */
+typedef struct {
+    abi_ulong mask;
+    abi_ulong addr;
+} __attribute__ ((aligned(8))) target_psw_t;
+
+/*
+ * The pt_regs struct defines the way the registers are stored on
+ * the stack during a system call.
+ */
+
+#define TARGET_NUM_GPRS        16
+
+struct target_pt_regs {
+    abi_ulong args[1];
+    target_psw_t psw;
+    abi_ulong gprs[TARGET_NUM_GPRS];
+    abi_ulong orig_gpr2;
+    unsigned short ilen;
+    unsigned short trap;
+};
+
+#define UNAME_MACHINE "s390x"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_CLONE_BACKWARDS2
+#define TARGET_MINSIGSTKSZ        2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* S390X_TARGET_SYSCALL_H */
diff --git a/linux-user/safe-syscall.S b/linux-user/safe-syscall.S
new file mode 100644
index 0000000..b5df625
--- /dev/null
+++ b/linux-user/safe-syscall.S
@@ -0,0 +1,30 @@
+/*
+ * safe-syscall.S : include the host-specific assembly fragment
+ * to handle signals occurring at the same time as system calls.
+ *
+ * Written by Peter Maydell <peter.maydell@linaro.org>
+ *
+ * Copyright (C) 2016 Linaro Limited
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "hostdep.h"
+#include "errno_defs.h"
+
+/* We have the correct host directory on our include path
+ * so that this will pull in the right fragment for the architecture.
+ */
+#ifdef HAVE_SAFE_SYSCALL
+#include "safe-syscall.inc.S"
+#endif
+
+/* We must specifically say that we're happy for the stack to not be
+ * executable, otherwise the toolchain will default to assuming our
+ * assembly needs an executable stack and the whole QEMU binary will
+ * needlessly end up with one. This should be the last thing in this file.
+ */
+#if defined(__linux__) && defined(__ELF__)
+.section        .note.GNU-stack, "", %progbits
+#endif
diff --git a/linux-user/sh4/syscall.h b/linux-user/sh4/syscall.h
deleted file mode 100644
index 7aa4f23..0000000
--- a/linux-user/sh4/syscall.h
+++ /dev/null
@@ -1,17 +0,0 @@
-struct target_pt_regs {
-        unsigned long regs[16];
-        unsigned long pc;
-        unsigned long pr;
-        unsigned long sr;
-        unsigned long gbr;
-        unsigned long mach;
-        unsigned long macl;
-        long tra;
-};
-
-#define UNAME_MACHINE "sh4"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/sh4/syscall_nr.h b/linux-user/sh4/syscall_nr.h
index bdf8742..5009984 100644
--- a/linux-user/sh4/syscall_nr.h
+++ b/linux-user/sh4/syscall_nr.h
@@ -323,7 +323,7 @@
 #define TARGET_NR_epoll_pwait	319
 #define TARGET_NR_utimensat		320
 #define TARGET_NR_signalfd		321
-#define TARGET_NR_timerfd		322
+#define TARGET_NR_timerfd_create	322
 #define TARGET_NR_eventfd		323
 #define TARGET_NR_fallocate		324
 #define TARGET_NR_timerfd_settime	325
diff --git a/linux-user/sh4/target_cpu.h b/linux-user/sh4/target_cpu.h
index 141856f..9d305d2 100644
--- a/linux-user/sh4/target_cpu.h
+++ b/linux-user/sh4/target_cpu.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef SH4_TARGET_CPU_H
+#define SH4_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUSH4State *env, target_ulong newsp)
 {
diff --git a/linux-user/sh4/target_signal.h b/linux-user/sh4/target_signal.h
index e148da0..cbf23b6 100644
--- a/linux-user/sh4/target_signal.h
+++ b/linux-user/sh4/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef SH4_TARGET_SIGNAL_H
+#define SH4_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,5 @@
     return state->gregs[15];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* SH4_TARGET_SIGNAL_H */
diff --git a/linux-user/sh4/target_structs.h b/linux-user/sh4/target_structs.h
index 32b235e..3e832bf 100644
--- a/linux-user/sh4/target_structs.h
+++ b/linux-user/sh4/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef SH4_TARGET_STRUCTS_H
+#define SH4_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/sh4/target_syscall.h b/linux-user/sh4/target_syscall.h
new file mode 100644
index 0000000..78d5557
--- /dev/null
+++ b/linux-user/sh4/target_syscall.h
@@ -0,0 +1,22 @@
+#ifndef SH4_TARGET_SYSCALL_H
+#define SH4_TARGET_SYSCALL_H
+
+struct target_pt_regs {
+        unsigned long regs[16];
+        unsigned long pc;
+        unsigned long pr;
+        unsigned long sr;
+        unsigned long gbr;
+        unsigned long mach;
+        unsigned long macl;
+        long tra;
+};
+
+#define UNAME_MACHINE "sh4"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* SH4_TARGET_SYSCALL_H */
diff --git a/linux-user/signal.c b/linux-user/signal.c
index e11b208..9a4d894 100644
--- a/linux-user/signal.c
+++ b/linux-user/signal.c
@@ -16,21 +16,15 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdarg.h>
-#include <unistd.h>
-#include <errno.h>
-#include <assert.h>
+#include "qemu/osdep.h"
+#include "qemu/bitops.h"
 #include <sys/ucontext.h>
 #include <sys/resource.h>
 
 #include "qemu.h"
 #include "qemu-common.h"
 #include "target_signal.h"
-
-//#define DEBUG_SIGNAL
+#include "trace.h"
 
 static struct target_sigaltstack target_sigaltstack_used = {
     .ss_sp = 0,
@@ -164,7 +158,7 @@
         if (target_sigismember(s, i)) {
             sigaddset(d, target_to_host_signal(i));
         }
-     }
+    }
 }
 
 void target_to_host_sigset(sigset_t *d, const target_sigset_t *s)
@@ -197,142 +191,241 @@
     target_to_host_sigset(sigset, &d);
 }
 
+int block_signals(void)
+{
+    TaskState *ts = (TaskState *)thread_cpu->opaque;
+    sigset_t set;
+
+    /* It's OK to block everything including SIGSEGV, because we won't
+     * run any further guest code before unblocking signals in
+     * process_pending_signals().
+     */
+    sigfillset(&set);
+    sigprocmask(SIG_SETMASK, &set, 0);
+
+    return atomic_xchg(&ts->signal_pending, 1);
+}
+
 /* Wrapper for sigprocmask function
  * Emulates a sigprocmask in a safe way for the guest. Note that set and oldset
- * are host signal set, not guest ones. This wraps the sigprocmask host calls
- * that should be protected (calls originated from guest)
+ * are host signal set, not guest ones. Returns -TARGET_ERESTARTSYS if
+ * a signal was already pending and the syscall must be restarted, or
+ * 0 on success.
+ * If set is NULL, this is guaranteed not to fail.
  */
 int do_sigprocmask(int how, const sigset_t *set, sigset_t *oldset)
 {
-    int ret;
-    sigset_t val;
-    sigset_t *temp = NULL;
-    CPUState *cpu = thread_cpu;
-    TaskState *ts = (TaskState *)cpu->opaque;
-    bool segv_was_blocked = ts->sigsegv_blocked;
+    TaskState *ts = (TaskState *)thread_cpu->opaque;
+
+    if (oldset) {
+        *oldset = ts->signal_mask;
+    }
 
     if (set) {
-        bool has_sigsegv = sigismember(set, SIGSEGV);
-        val = *set;
-        temp = &val;
+        int i;
 
-        sigdelset(temp, SIGSEGV);
+        if (block_signals()) {
+            return -TARGET_ERESTARTSYS;
+        }
 
         switch (how) {
         case SIG_BLOCK:
-            if (has_sigsegv) {
-                ts->sigsegv_blocked = true;
-            }
+            sigorset(&ts->signal_mask, &ts->signal_mask, set);
             break;
         case SIG_UNBLOCK:
-            if (has_sigsegv) {
-                ts->sigsegv_blocked = false;
+            for (i = 1; i <= NSIG; ++i) {
+                if (sigismember(set, i)) {
+                    sigdelset(&ts->signal_mask, i);
+                }
             }
             break;
         case SIG_SETMASK:
-            ts->sigsegv_blocked = has_sigsegv;
+            ts->signal_mask = *set;
             break;
         default:
             g_assert_not_reached();
         }
+
+        /* Silently ignore attempts to change blocking status of KILL or STOP */
+        sigdelset(&ts->signal_mask, SIGKILL);
+        sigdelset(&ts->signal_mask, SIGSTOP);
     }
-
-    ret = sigprocmask(how, temp, oldset);
-
-    if (oldset && segv_was_blocked) {
-        sigaddset(oldset, SIGSEGV);
-    }
-
-    return ret;
+    return 0;
 }
 
+#if !defined(TARGET_OPENRISC) && !defined(TARGET_UNICORE32) && \
+    !defined(TARGET_X86_64)
+/* Just set the guest's signal mask to the specified value; the
+ * caller is assumed to have called block_signals() already.
+ */
+static void set_sigmask(const sigset_t *set)
+{
+    TaskState *ts = (TaskState *)thread_cpu->opaque;
+
+    ts->signal_mask = *set;
+}
+#endif
+
 /* siginfo conversion */
 
 static inline void host_to_target_siginfo_noswap(target_siginfo_t *tinfo,
                                                  const siginfo_t *info)
 {
     int sig = host_to_target_signal(info->si_signo);
+    int si_code = info->si_code;
+    int si_type;
     tinfo->si_signo = sig;
     tinfo->si_errno = 0;
     tinfo->si_code = info->si_code;
 
-    if (sig == TARGET_SIGILL || sig == TARGET_SIGFPE || sig == TARGET_SIGSEGV
-        || sig == TARGET_SIGBUS || sig == TARGET_SIGTRAP) {
-        /* Should never come here, but who knows. The information for
-           the target is irrelevant.  */
-        tinfo->_sifields._sigfault._addr = 0;
-    } else if (sig == TARGET_SIGIO) {
-        tinfo->_sifields._sigpoll._band = info->si_band;
-	tinfo->_sifields._sigpoll._fd = info->si_fd;
-    } else if (sig == TARGET_SIGCHLD) {
-        tinfo->_sifields._sigchld._pid = info->si_pid;
-        tinfo->_sifields._sigchld._uid = info->si_uid;
-        tinfo->_sifields._sigchld._status
-            = host_to_target_waitstatus(info->si_status);
-        tinfo->_sifields._sigchld._utime = info->si_utime;
-        tinfo->_sifields._sigchld._stime = info->si_stime;
-    } else if (sig >= TARGET_SIGRTMIN) {
-        tinfo->_sifields._rt._pid = info->si_pid;
-        tinfo->_sifields._rt._uid = info->si_uid;
-        /* XXX: potential problem if 64 bit */
-        tinfo->_sifields._rt._sigval.sival_ptr
-            = (abi_ulong)(unsigned long)info->si_value.sival_ptr;
+    /* This memset serves two purposes:
+     * (1) ensure we don't leak random junk to the guest later
+     * (2) placate false positives from gcc about fields
+     *     being used uninitialized if it chooses to inline both this
+     *     function and tswap_siginfo() into host_to_target_siginfo().
+     */
+    memset(tinfo->_sifields._pad, 0, sizeof(tinfo->_sifields._pad));
+
+    /* This is awkward, because we have to use a combination of
+     * the si_code and si_signo to figure out which of the union's
+     * members are valid. (Within the host kernel it is always possible
+     * to tell, but the kernel carefully avoids giving userspace the
+     * high 16 bits of si_code, so we don't have the information to
+     * do this the easy way...) We therefore make our best guess,
+     * bearing in mind that a guest can spoof most of the si_codes
+     * via rt_sigqueueinfo() if it likes.
+     *
+     * Once we have made our guess, we record it in the top 16 bits of
+     * the si_code, so that tswap_siginfo() later can use it.
+     * tswap_siginfo() will strip these top bits out before writing
+     * si_code to the guest (sign-extending the lower bits).
+     */
+
+    switch (si_code) {
+    case SI_USER:
+    case SI_TKILL:
+    case SI_KERNEL:
+        /* Sent via kill(), tkill() or tgkill(), or direct from the kernel.
+         * These are the only unspoofable si_code values.
+         */
+        tinfo->_sifields._kill._pid = info->si_pid;
+        tinfo->_sifields._kill._uid = info->si_uid;
+        si_type = QEMU_SI_KILL;
+        break;
+    default:
+        /* Everything else is spoofable. Make best guess based on signal */
+        switch (sig) {
+        case TARGET_SIGCHLD:
+            tinfo->_sifields._sigchld._pid = info->si_pid;
+            tinfo->_sifields._sigchld._uid = info->si_uid;
+            tinfo->_sifields._sigchld._status
+                = host_to_target_waitstatus(info->si_status);
+            tinfo->_sifields._sigchld._utime = info->si_utime;
+            tinfo->_sifields._sigchld._stime = info->si_stime;
+            si_type = QEMU_SI_CHLD;
+            break;
+        case TARGET_SIGIO:
+            tinfo->_sifields._sigpoll._band = info->si_band;
+            tinfo->_sifields._sigpoll._fd = info->si_fd;
+            si_type = QEMU_SI_POLL;
+            break;
+        default:
+            /* Assume a sigqueue()/mq_notify()/rt_sigqueueinfo() source. */
+            tinfo->_sifields._rt._pid = info->si_pid;
+            tinfo->_sifields._rt._uid = info->si_uid;
+            /* XXX: potential problem if 64 bit */
+            tinfo->_sifields._rt._sigval.sival_ptr
+                = (abi_ulong)(unsigned long)info->si_value.sival_ptr;
+            si_type = QEMU_SI_RT;
+            break;
+        }
+        break;
     }
+
+    tinfo->si_code = deposit32(si_code, 16, 16, si_type);
 }
 
 static void tswap_siginfo(target_siginfo_t *tinfo,
                           const target_siginfo_t *info)
 {
-    int sig = info->si_signo;
-    tinfo->si_signo = tswap32(sig);
-    tinfo->si_errno = tswap32(info->si_errno);
-    tinfo->si_code = tswap32(info->si_code);
+    int si_type = extract32(info->si_code, 16, 16);
+    int si_code = sextract32(info->si_code, 0, 16);
 
-    if (sig == TARGET_SIGILL || sig == TARGET_SIGFPE || sig == TARGET_SIGSEGV
-        || sig == TARGET_SIGBUS || sig == TARGET_SIGTRAP) {
-        tinfo->_sifields._sigfault._addr
-            = tswapal(info->_sifields._sigfault._addr);
-    } else if (sig == TARGET_SIGIO) {
-        tinfo->_sifields._sigpoll._band
-            = tswap32(info->_sifields._sigpoll._band);
-        tinfo->_sifields._sigpoll._fd = tswap32(info->_sifields._sigpoll._fd);
-    } else if (sig == TARGET_SIGCHLD) {
-        tinfo->_sifields._sigchld._pid
-            = tswap32(info->_sifields._sigchld._pid);
-        tinfo->_sifields._sigchld._uid
-            = tswap32(info->_sifields._sigchld._uid);
-        tinfo->_sifields._sigchld._status
-            = tswap32(info->_sifields._sigchld._status);
-        tinfo->_sifields._sigchld._utime
-            = tswapal(info->_sifields._sigchld._utime);
-        tinfo->_sifields._sigchld._stime
-            = tswapal(info->_sifields._sigchld._stime);
-    } else if (sig >= TARGET_SIGRTMIN) {
-        tinfo->_sifields._rt._pid = tswap32(info->_sifields._rt._pid);
-        tinfo->_sifields._rt._uid = tswap32(info->_sifields._rt._uid);
-        tinfo->_sifields._rt._sigval.sival_ptr
-            = tswapal(info->_sifields._rt._sigval.sival_ptr);
+    __put_user(info->si_signo, &tinfo->si_signo);
+    __put_user(info->si_errno, &tinfo->si_errno);
+    __put_user(si_code, &tinfo->si_code);
+
+    /* We can use our internal marker of which fields in the structure
+     * are valid, rather than duplicating the guesswork of
+     * host_to_target_siginfo_noswap() here.
+     */
+    switch (si_type) {
+    case QEMU_SI_KILL:
+        __put_user(info->_sifields._kill._pid, &tinfo->_sifields._kill._pid);
+        __put_user(info->_sifields._kill._uid, &tinfo->_sifields._kill._uid);
+        break;
+    case QEMU_SI_TIMER:
+        __put_user(info->_sifields._timer._timer1,
+                   &tinfo->_sifields._timer._timer1);
+        __put_user(info->_sifields._timer._timer2,
+                   &tinfo->_sifields._timer._timer2);
+        break;
+    case QEMU_SI_POLL:
+        __put_user(info->_sifields._sigpoll._band,
+                   &tinfo->_sifields._sigpoll._band);
+        __put_user(info->_sifields._sigpoll._fd,
+                   &tinfo->_sifields._sigpoll._fd);
+        break;
+    case QEMU_SI_FAULT:
+        __put_user(info->_sifields._sigfault._addr,
+                   &tinfo->_sifields._sigfault._addr);
+        break;
+    case QEMU_SI_CHLD:
+        __put_user(info->_sifields._sigchld._pid,
+                   &tinfo->_sifields._sigchld._pid);
+        __put_user(info->_sifields._sigchld._uid,
+                   &tinfo->_sifields._sigchld._uid);
+        __put_user(info->_sifields._sigchld._status,
+                   &tinfo->_sifields._sigchld._status);
+        __put_user(info->_sifields._sigchld._utime,
+                   &tinfo->_sifields._sigchld._utime);
+        __put_user(info->_sifields._sigchld._stime,
+                   &tinfo->_sifields._sigchld._stime);
+        break;
+    case QEMU_SI_RT:
+        __put_user(info->_sifields._rt._pid, &tinfo->_sifields._rt._pid);
+        __put_user(info->_sifields._rt._uid, &tinfo->_sifields._rt._uid);
+        __put_user(info->_sifields._rt._sigval.sival_ptr,
+                   &tinfo->_sifields._rt._sigval.sival_ptr);
+        break;
+    default:
+        g_assert_not_reached();
     }
 }
 
-
 void host_to_target_siginfo(target_siginfo_t *tinfo, const siginfo_t *info)
 {
-    host_to_target_siginfo_noswap(tinfo, info);
-    tswap_siginfo(tinfo, tinfo);
+    target_siginfo_t tgt_tmp;
+    host_to_target_siginfo_noswap(&tgt_tmp, info);
+    tswap_siginfo(tinfo, &tgt_tmp);
 }
 
 /* XXX: we support only POSIX RT signals are used. */
 /* XXX: find a solution for 64 bit (additional malloced data is needed) */
 void target_to_host_siginfo(siginfo_t *info, const target_siginfo_t *tinfo)
 {
-    info->si_signo = tswap32(tinfo->si_signo);
-    info->si_errno = tswap32(tinfo->si_errno);
-    info->si_code = tswap32(tinfo->si_code);
-    info->si_pid = tswap32(tinfo->_sifields._rt._pid);
-    info->si_uid = tswap32(tinfo->_sifields._rt._uid);
-    info->si_value.sival_ptr =
-            (void *)(long)tswapal(tinfo->_sifields._rt._sigval.sival_ptr);
+    /* This conversion is used only for the rt_sigqueueinfo syscall,
+     * and so we know that the _rt fields are the valid ones.
+     */
+    abi_ulong sival_ptr;
+
+    __get_user(info->si_signo, &tinfo->si_signo);
+    __get_user(info->si_errno, &tinfo->si_errno);
+    __get_user(info->si_code, &tinfo->si_code);
+    __get_user(info->si_pid, &tinfo->_sifields._rt._pid);
+    __get_user(info->si_uid, &tinfo->_sifields._rt._uid);
+    __get_user(sival_ptr, &tinfo->_sifields._rt._sigval.sival_ptr);
+    info->si_value.sival_ptr = (void *)(long)sival_ptr;
 }
 
 static int fatal_signal (int sig)
@@ -374,6 +467,7 @@
 
 void signal_init(void)
 {
+    TaskState *ts = (TaskState *)thread_cpu->opaque;
     struct sigaction act;
     struct sigaction oact;
     int i, j;
@@ -389,6 +483,9 @@
         target_to_host_signal_table[j] = i;
     }
 
+    /* Set the signal mask from the host mask. */
+    sigprocmask(0, 0, &ts->signal_mask);
+
     /* set all host signal handlers. ALL signals are blocked during
        the handlers to serialize them. */
     memset(sigact_table, 0, sizeof(sigact_table));
@@ -415,27 +512,6 @@
     }
 }
 
-/* signal queue handling */
-
-static inline struct sigqueue *alloc_sigqueue(CPUArchState *env)
-{
-    CPUState *cpu = ENV_GET_CPU(env);
-    TaskState *ts = cpu->opaque;
-    struct sigqueue *q = ts->first_free;
-    if (!q)
-        return NULL;
-    ts->first_free = q->next;
-    return q;
-}
-
-static inline void free_sigqueue(CPUArchState *env, struct sigqueue *q)
-{
-    CPUState *cpu = ENV_GET_CPU(env);
-    TaskState *ts = cpu->opaque;
-
-    q->next = ts->first_free;
-    ts->first_free = q;
-}
 
 /* abort execution with signal */
 static void QEMU_NORETURN force_sig(int target_sig)
@@ -445,7 +521,9 @@
     TaskState *ts = (TaskState *)cpu->opaque;
     int host_sig, core_dumped = 0;
     struct sigaction act;
+
     host_sig = target_to_host_signal(target_sig);
+    trace_user_force_sig(env, target_sig, host_sig);
     gdb_signalled(env, target_sig);
 
     /* dump core if supported by target binary format */
@@ -495,86 +573,41 @@
 {
     CPUState *cpu = ENV_GET_CPU(env);
     TaskState *ts = cpu->opaque;
-    struct emulated_sigtable *k;
-    struct sigqueue *q, **pq;
-    abi_ulong handler;
-    int queue;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "queue_signal: sig=%d\n",
-            sig);
-#endif
-    k = &ts->sigtab[sig - 1];
-    queue = gdb_queuesig ();
-    handler = sigact_table[sig - 1]._sa_handler;
+    trace_user_queue_signal(env, sig);
 
-    if (ts->sigsegv_blocked && sig == TARGET_SIGSEGV) {
-        /* Guest has blocked SIGSEGV but we got one anyway. Assume this
-         * is a forced SIGSEGV (ie one the kernel handles via force_sig_info
-         * because it got a real MMU fault). A blocked SIGSEGV in that
-         * situation is treated as if using the default handler. This is
-         * not correct if some other process has randomly sent us a SIGSEGV
-         * via kill(), but that is not easy to distinguish at this point,
-         * so we assume it doesn't happen.
-         */
-        handler = TARGET_SIG_DFL;
-    }
+    /* Currently all callers define siginfo structures which
+     * use the _sifields._sigfault union member, so we can
+     * set the type here. If that changes we should push this
+     * out so the si_type is passed in by callers.
+     */
+    info->si_code = deposit32(info->si_code, 16, 16, QEMU_SI_FAULT);
 
-    if (!queue && handler == TARGET_SIG_DFL) {
-        if (sig == TARGET_SIGTSTP || sig == TARGET_SIGTTIN || sig == TARGET_SIGTTOU) {
-            kill(getpid(),SIGSTOP);
-            return 0;
-        } else
-        /* default handler : ignore some signal. The other are fatal */
-        if (sig != TARGET_SIGCHLD &&
-            sig != TARGET_SIGURG &&
-            sig != TARGET_SIGWINCH &&
-            sig != TARGET_SIGCONT) {
-            force_sig(sig);
-        } else {
-            return 0; /* indicate ignored */
-        }
-    } else if (!queue && handler == TARGET_SIG_IGN) {
-        /* ignore signal */
-        return 0;
-    } else if (!queue && handler == TARGET_SIG_ERR) {
-        force_sig(sig);
-    } else {
-        pq = &k->first;
-        if (sig < TARGET_SIGRTMIN) {
-            /* if non real time signal, we queue exactly one signal */
-            if (!k->pending)
-                q = &k->info;
-            else
-                return 0;
-        } else {
-            if (!k->pending) {
-                /* first signal */
-                q = &k->info;
-            } else {
-                q = alloc_sigqueue(env);
-                if (!q)
-                    return -EAGAIN;
-                while (*pq != NULL)
-                    pq = &(*pq)->next;
-            }
-        }
-        *pq = q;
-        q->info = *info;
-        q->next = NULL;
-        k->pending = 1;
-        /* signal that a new signal is pending */
-        ts->signal_pending = 1;
-        return 1; /* indicates that the signal was queued */
-    }
+    ts->sync_signal.info = *info;
+    ts->sync_signal.pending = sig;
+    /* signal that a new signal is pending */
+    atomic_set(&ts->signal_pending, 1);
+    return 1; /* indicates that the signal was queued */
 }
 
+#ifndef HAVE_SAFE_SYSCALL
+static inline void rewind_if_in_safe_syscall(void *puc)
+{
+    /* Default version: never rewind */
+}
+#endif
+
 static void host_signal_handler(int host_signum, siginfo_t *info,
                                 void *puc)
 {
     CPUArchState *env = thread_cpu->env_ptr;
+    CPUState *cpu = ENV_GET_CPU(env);
+    TaskState *ts = cpu->opaque;
+
     int sig;
     target_siginfo_t tinfo;
+    ucontext_t *uc = puc;
+    struct emulated_sigtable *k;
 
     /* the CPU emulator uses some host signals to detect exceptions,
        we forward to it some signals */
@@ -588,14 +621,36 @@
     sig = host_to_target_signal(host_signum);
     if (sig < 1 || sig > TARGET_NSIG)
         return;
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "qemu: got signal %d\n", sig);
-#endif
+    trace_user_host_signal(env, host_signum, sig);
+
+    rewind_if_in_safe_syscall(puc);
+
     host_to_target_siginfo_noswap(&tinfo, info);
-    if (queue_signal(env, sig, &tinfo) == 1) {
-        /* interrupt the virtual CPU as soon as possible */
-        cpu_exit(thread_cpu);
-    }
+    k = &ts->sigtab[sig - 1];
+    k->info = tinfo;
+    k->pending = sig;
+    ts->signal_pending = 1;
+
+    /* Block host signals until target signal handler entered. We
+     * can't block SIGSEGV or SIGBUS while we're executing guest
+     * code in case the guest code provokes one in the window between
+     * now and it getting out to the main loop. Signals will be
+     * unblocked again in process_pending_signals().
+     *
+     * WARNING: we cannot use sigfillset() here because the uc_sigmask
+     * field is a kernel sigset_t, which is much smaller than the
+     * libc sigset_t which sigfillset() operates on. Using sigfillset()
+     * would write 0xff bytes off the end of the structure and trash
+     * data on the struct.
+     * We can't use sizeof(uc->uc_sigmask) either, because the libc
+     * headers define the struct field with the wrong (too large) type.
+     */
+    memset(&uc->uc_sigmask, 0xff, SIGSET_T_SIZE);
+    sigdelset(&uc->uc_sigmask, SIGSEGV);
+    sigdelset(&uc->uc_sigmask, SIGBUS);
+
+    /* interrupt the virtual CPU as soon as possible */
+    cpu_exit(thread_cpu);
 }
 
 /* do_sigaltstack() returns target values and errnos. */
@@ -671,7 +726,7 @@
     return ret;
 }
 
-/* do_sigaction() return host values and errnos */
+/* do_sigaction() return target values and host errnos */
 int do_sigaction(int sig, const struct target_sigaction *act,
                  struct target_sigaction *oact)
 {
@@ -680,13 +735,15 @@
     int host_sig;
     int ret = 0;
 
-    if (sig < 1 || sig > TARGET_NSIG || sig == TARGET_SIGKILL || sig == TARGET_SIGSTOP)
-        return -EINVAL;
+    if (sig < 1 || sig > TARGET_NSIG || sig == TARGET_SIGKILL || sig == TARGET_SIGSTOP) {
+        return -TARGET_EINVAL;
+    }
+
+    if (block_signals()) {
+        return -TARGET_ERESTARTSYS;
+    }
+
     k = &sigact_table[sig - 1];
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "sigaction sig=%d act=0x%p, oact=0x%p\n",
-            sig, act, oact);
-#endif
     if (oact) {
         __put_user(k->_sa_handler, &oact->_sa_handler);
         __put_user(k->sa_flags, &oact->sa_flags);
@@ -732,92 +789,80 @@
     return ret;
 }
 
-static inline void copy_siginfo_to_user(target_siginfo_t *tinfo,
-                                       const target_siginfo_t *info)
-{
-    tswap_siginfo(tinfo, info);
-}
-
-static inline int current_exec_domain_sig(int sig)
-{
-    return /* current->exec_domain && current->exec_domain->signal_invmap
-	      && sig < 32 ? current->exec_domain->signal_invmap[sig] : */ sig;
-}
-
 #if defined(TARGET_I386) && TARGET_ABI_BITS == 32
 
 /* from the Linux kernel */
 
 struct target_fpreg {
-	uint16_t significand[4];
-	uint16_t exponent;
+    uint16_t significand[4];
+    uint16_t exponent;
 };
 
 struct target_fpxreg {
-	uint16_t significand[4];
-	uint16_t exponent;
-	uint16_t padding[3];
+    uint16_t significand[4];
+    uint16_t exponent;
+    uint16_t padding[3];
 };
 
 struct target_xmmreg {
-	abi_ulong element[4];
+    abi_ulong element[4];
 };
 
 struct target_fpstate {
-	/* Regular FPU environment */
-        abi_ulong       cw;
-        abi_ulong       sw;
-        abi_ulong       tag;
-        abi_ulong       ipoff;
-        abi_ulong       cssel;
-        abi_ulong       dataoff;
-        abi_ulong       datasel;
-	struct target_fpreg	_st[8];
-	uint16_t	status;
-	uint16_t	magic;		/* 0xffff = regular FPU data only */
+    /* Regular FPU environment */
+    abi_ulong cw;
+    abi_ulong sw;
+    abi_ulong tag;
+    abi_ulong ipoff;
+    abi_ulong cssel;
+    abi_ulong dataoff;
+    abi_ulong datasel;
+    struct target_fpreg _st[8];
+    uint16_t  status;
+    uint16_t  magic;          /* 0xffff = regular FPU data only */
 
-	/* FXSR FPU environment */
-        abi_ulong       _fxsr_env[6];   /* FXSR FPU env is ignored */
-        abi_ulong       mxcsr;
-        abi_ulong       reserved;
-	struct target_fpxreg	_fxsr_st[8];	/* FXSR FPU reg data is ignored */
-	struct target_xmmreg	_xmm[8];
-        abi_ulong       padding[56];
+    /* FXSR FPU environment */
+    abi_ulong _fxsr_env[6];   /* FXSR FPU env is ignored */
+    abi_ulong mxcsr;
+    abi_ulong reserved;
+    struct target_fpxreg _fxsr_st[8]; /* FXSR FPU reg data is ignored */
+    struct target_xmmreg _xmm[8];
+    abi_ulong padding[56];
 };
 
 #define X86_FXSR_MAGIC		0x0000
 
 struct target_sigcontext {
-	uint16_t gs, __gsh;
-	uint16_t fs, __fsh;
-	uint16_t es, __esh;
-	uint16_t ds, __dsh;
-        abi_ulong edi;
-        abi_ulong esi;
-        abi_ulong ebp;
-        abi_ulong esp;
-        abi_ulong ebx;
-        abi_ulong edx;
-        abi_ulong ecx;
-        abi_ulong eax;
-        abi_ulong trapno;
-        abi_ulong err;
-        abi_ulong eip;
-	uint16_t cs, __csh;
-        abi_ulong eflags;
-        abi_ulong esp_at_signal;
-	uint16_t ss, __ssh;
-        abi_ulong fpstate; /* pointer */
-        abi_ulong oldmask;
-        abi_ulong cr2;
+    uint16_t gs, __gsh;
+    uint16_t fs, __fsh;
+    uint16_t es, __esh;
+    uint16_t ds, __dsh;
+    abi_ulong edi;
+    abi_ulong esi;
+    abi_ulong ebp;
+    abi_ulong esp;
+    abi_ulong ebx;
+    abi_ulong edx;
+    abi_ulong ecx;
+    abi_ulong eax;
+    abi_ulong trapno;
+    abi_ulong err;
+    abi_ulong eip;
+    uint16_t cs, __csh;
+    abi_ulong eflags;
+    abi_ulong esp_at_signal;
+    uint16_t ss, __ssh;
+    abi_ulong fpstate; /* pointer */
+    abi_ulong oldmask;
+    abi_ulong cr2;
 };
 
 struct target_ucontext {
-        abi_ulong         tuc_flags;
-        abi_ulong         tuc_link;
-	target_stack_t	  tuc_stack;
-	struct target_sigcontext tuc_mcontext;
-	target_sigset_t	  tuc_sigmask;	/* mask last for extensibility */
+    abi_ulong         tuc_flags;
+    abi_ulong         tuc_link;
+    target_stack_t    tuc_stack;
+    struct target_sigcontext tuc_mcontext;
+    target_sigset_t   tuc_sigmask;  /* mask last for extensibility */
 };
 
 struct sigframe
@@ -854,7 +899,7 @@
     CPUState *cs = CPU(x86_env_get_cpu(env));
     uint16_t magic;
 
-	/* already locked in setup_frame() */
+    /* already locked in setup_frame() */
     __put_user(env->segs[R_GS].selector, (unsigned int *)&sc->gs);
     __put_user(env->segs[R_FS].selector, (unsigned int *)&sc->fs);
     __put_user(env->segs[R_ES].selector, (unsigned int *)&sc->es);
@@ -875,13 +920,13 @@
     __put_user(env->regs[R_ESP], &sc->esp_at_signal);
     __put_user(env->segs[R_SS].selector, (unsigned int *)&sc->ss);
 
-        cpu_x86_fsave(env, fpstate_addr, 1);
-        fpstate->status = fpstate->sw;
-        magic = 0xffff;
+    cpu_x86_fsave(env, fpstate_addr, 1);
+    fpstate->status = fpstate->sw;
+    magic = 0xffff;
     __put_user(magic, &fpstate->magic);
     __put_user(fpstate_addr, &sc->fpstate);
 
-	/* non-iBCS2 extensions.. */
+    /* non-iBCS2 extensions.. */
     __put_user(mask, &sc->oldmask);
     __put_user(env->cr[2], &sc->cr2);
 }
@@ -893,109 +938,112 @@
 static inline abi_ulong
 get_sigframe(struct target_sigaction *ka, CPUX86State *env, size_t frame_size)
 {
-	unsigned long esp;
+    unsigned long esp;
 
-	/* Default to using normal stack */
-	esp = env->regs[R_ESP];
-	/* This is the X/Open sanctioned signal stack switching.  */
-        if (ka->sa_flags & TARGET_SA_ONSTACK) {
-            if (sas_ss_flags(esp) == 0)
-                esp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+    /* Default to using normal stack */
+    esp = env->regs[R_ESP];
+    /* This is the X/Open sanctioned signal stack switching.  */
+    if (ka->sa_flags & TARGET_SA_ONSTACK) {
+        if (sas_ss_flags(esp) == 0) {
+            esp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
         }
+    } else {
 
-	/* This is the legacy signal stack switching. */
-	else
+        /* This is the legacy signal stack switching. */
         if ((env->segs[R_SS].selector & 0xffff) != __USER_DS &&
-            !(ka->sa_flags & TARGET_SA_RESTORER) &&
-            ka->sa_restorer) {
+                !(ka->sa_flags & TARGET_SA_RESTORER) &&
+                ka->sa_restorer) {
             esp = (unsigned long) ka->sa_restorer;
-	}
-        return (esp - frame_size) & -8ul;
+        }
+    }
+    return (esp - frame_size) & -8ul;
 }
 
 /* compare linux/arch/i386/kernel/signal.c:setup_frame() */
 static void setup_frame(int sig, struct target_sigaction *ka,
-			target_sigset_t *set, CPUX86State *env)
+                        target_sigset_t *set, CPUX86State *env)
 {
-	abi_ulong frame_addr;
-	struct sigframe *frame;
-	int i;
+    abi_ulong frame_addr;
+    struct sigframe *frame;
+    int i;
 
-	frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_frame(env, frame_addr);
 
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-		goto give_sigsegv;
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
+        goto give_sigsegv;
 
-    __put_user(current_exec_domain_sig(sig),
-               &frame->sig);
+    __put_user(sig, &frame->sig);
 
-	setup_sigcontext(&frame->sc, &frame->fpstate, env, set->sig[0],
-                         frame_addr + offsetof(struct sigframe, fpstate));
+    setup_sigcontext(&frame->sc, &frame->fpstate, env, set->sig[0],
+            frame_addr + offsetof(struct sigframe, fpstate));
 
     for(i = 1; i < TARGET_NSIG_WORDS; i++) {
         __put_user(set->sig[i], &frame->extramask[i - 1]);
     }
 
-	/* Set up to return from userspace.  If provided, use a stub
-	   already in userspace.  */
-	if (ka->sa_flags & TARGET_SA_RESTORER) {
+    /* Set up to return from userspace.  If provided, use a stub
+       already in userspace.  */
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
         __put_user(ka->sa_restorer, &frame->pretcode);
-	} else {
-                uint16_t val16;
-                abi_ulong retcode_addr;
-                retcode_addr = frame_addr + offsetof(struct sigframe, retcode);
+    } else {
+        uint16_t val16;
+        abi_ulong retcode_addr;
+        retcode_addr = frame_addr + offsetof(struct sigframe, retcode);
         __put_user(retcode_addr, &frame->pretcode);
-		/* This is popl %eax ; movl $,%eax ; int $0x80 */
-                val16 = 0xb858;
+        /* This is popl %eax ; movl $,%eax ; int $0x80 */
+        val16 = 0xb858;
         __put_user(val16, (uint16_t *)(frame->retcode+0));
         __put_user(TARGET_NR_sigreturn, (int *)(frame->retcode+2));
-                val16 = 0x80cd;
+        val16 = 0x80cd;
         __put_user(val16, (uint16_t *)(frame->retcode+6));
-	}
+    }
 
 
-	/* Set up registers for signal handler */
-	env->regs[R_ESP] = frame_addr;
-	env->eip = ka->_sa_handler;
+    /* Set up registers for signal handler */
+    env->regs[R_ESP] = frame_addr;
+    env->eip = ka->_sa_handler;
 
-        cpu_x86_load_seg(env, R_DS, __USER_DS);
-        cpu_x86_load_seg(env, R_ES, __USER_DS);
-        cpu_x86_load_seg(env, R_SS, __USER_DS);
-        cpu_x86_load_seg(env, R_CS, __USER_CS);
-	env->eflags &= ~TF_MASK;
+    cpu_x86_load_seg(env, R_DS, __USER_DS);
+    cpu_x86_load_seg(env, R_ES, __USER_DS);
+    cpu_x86_load_seg(env, R_SS, __USER_DS);
+    cpu_x86_load_seg(env, R_CS, __USER_CS);
+    env->eflags &= ~TF_MASK;
 
-	unlock_user_struct(frame, frame_addr, 1);
+    unlock_user_struct(frame, frame_addr, 1);
 
-	return;
+    return;
 
 give_sigsegv:
-	if (sig == TARGET_SIGSEGV)
-		ka->_sa_handler = TARGET_SIG_DFL;
-	force_sig(TARGET_SIGSEGV /* , current */);
+    if (sig == TARGET_SIGSEGV) {
+        ka->_sa_handler = TARGET_SIG_DFL;
+    }
+    force_sig(TARGET_SIGSEGV /* , current */);
 }
 
 /* compare linux/arch/i386/kernel/signal.c:setup_rt_frame() */
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
                            target_siginfo_t *info,
-			   target_sigset_t *set, CPUX86State *env)
+                           target_sigset_t *set, CPUX86State *env)
 {
-        abi_ulong frame_addr, addr;
-	struct rt_sigframe *frame;
-	int i;
+    abi_ulong frame_addr, addr;
+    struct rt_sigframe *frame;
+    int i;
 
-	frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
 
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-		goto give_sigsegv;
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
+        goto give_sigsegv;
 
-    __put_user(current_exec_domain_sig(sig), &frame->sig);
-        addr = frame_addr + offsetof(struct rt_sigframe, info);
+    __put_user(sig, &frame->sig);
+    addr = frame_addr + offsetof(struct rt_sigframe, info);
     __put_user(addr, &frame->pinfo);
-        addr = frame_addr + offsetof(struct rt_sigframe, uc);
+    addr = frame_addr + offsetof(struct rt_sigframe, uc);
     __put_user(addr, &frame->puc);
-	copy_siginfo_to_user(&frame->info, info);
+    tswap_siginfo(&frame->info, info);
 
-	/* Create the ucontext.  */
+    /* Create the ucontext.  */
     __put_user(0, &frame->uc.tuc_flags);
     __put_user(0, &frame->uc.tuc_link);
     __put_user(target_sigaltstack_used.ss_sp, &frame->uc.tuc_stack.ss_sp);
@@ -1010,81 +1058,82 @@
         __put_user(set->sig[i], &frame->uc.tuc_sigmask.sig[i]);
     }
 
-	/* Set up to return from userspace.  If provided, use a stub
-	   already in userspace.  */
-	if (ka->sa_flags & TARGET_SA_RESTORER) {
+    /* Set up to return from userspace.  If provided, use a stub
+       already in userspace.  */
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
         __put_user(ka->sa_restorer, &frame->pretcode);
-	} else {
-                uint16_t val16;
-                addr = frame_addr + offsetof(struct rt_sigframe, retcode);
+    } else {
+        uint16_t val16;
+        addr = frame_addr + offsetof(struct rt_sigframe, retcode);
         __put_user(addr, &frame->pretcode);
-		/* This is movl $,%eax ; int $0x80 */
+        /* This is movl $,%eax ; int $0x80 */
         __put_user(0xb8, (char *)(frame->retcode+0));
         __put_user(TARGET_NR_rt_sigreturn, (int *)(frame->retcode+1));
-                val16 = 0x80cd;
+        val16 = 0x80cd;
         __put_user(val16, (uint16_t *)(frame->retcode+5));
-	}
+    }
 
-	/* Set up registers for signal handler */
-	env->regs[R_ESP] = frame_addr;
-	env->eip = ka->_sa_handler;
+    /* Set up registers for signal handler */
+    env->regs[R_ESP] = frame_addr;
+    env->eip = ka->_sa_handler;
 
-        cpu_x86_load_seg(env, R_DS, __USER_DS);
-        cpu_x86_load_seg(env, R_ES, __USER_DS);
-        cpu_x86_load_seg(env, R_SS, __USER_DS);
-        cpu_x86_load_seg(env, R_CS, __USER_CS);
-	env->eflags &= ~TF_MASK;
+    cpu_x86_load_seg(env, R_DS, __USER_DS);
+    cpu_x86_load_seg(env, R_ES, __USER_DS);
+    cpu_x86_load_seg(env, R_SS, __USER_DS);
+    cpu_x86_load_seg(env, R_CS, __USER_CS);
+    env->eflags &= ~TF_MASK;
 
-	unlock_user_struct(frame, frame_addr, 1);
+    unlock_user_struct(frame, frame_addr, 1);
 
-	return;
+    return;
 
 give_sigsegv:
-	if (sig == TARGET_SIGSEGV)
-		ka->_sa_handler = TARGET_SIG_DFL;
-	force_sig(TARGET_SIGSEGV /* , current */);
+    if (sig == TARGET_SIGSEGV) {
+        ka->_sa_handler = TARGET_SIG_DFL;
+    }
+    force_sig(TARGET_SIGSEGV /* , current */);
 }
 
 static int
-restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc, int *peax)
+restore_sigcontext(CPUX86State *env, struct target_sigcontext *sc)
 {
-	unsigned int err = 0;
-        abi_ulong fpstate_addr;
-        unsigned int tmpflags;
+    unsigned int err = 0;
+    abi_ulong fpstate_addr;
+    unsigned int tmpflags;
 
-        cpu_x86_load_seg(env, R_GS, tswap16(sc->gs));
-        cpu_x86_load_seg(env, R_FS, tswap16(sc->fs));
-        cpu_x86_load_seg(env, R_ES, tswap16(sc->es));
-        cpu_x86_load_seg(env, R_DS, tswap16(sc->ds));
+    cpu_x86_load_seg(env, R_GS, tswap16(sc->gs));
+    cpu_x86_load_seg(env, R_FS, tswap16(sc->fs));
+    cpu_x86_load_seg(env, R_ES, tswap16(sc->es));
+    cpu_x86_load_seg(env, R_DS, tswap16(sc->ds));
 
-        env->regs[R_EDI] = tswapl(sc->edi);
-        env->regs[R_ESI] = tswapl(sc->esi);
-        env->regs[R_EBP] = tswapl(sc->ebp);
-        env->regs[R_ESP] = tswapl(sc->esp);
-        env->regs[R_EBX] = tswapl(sc->ebx);
-        env->regs[R_EDX] = tswapl(sc->edx);
-        env->regs[R_ECX] = tswapl(sc->ecx);
-        env->eip = tswapl(sc->eip);
+    env->regs[R_EDI] = tswapl(sc->edi);
+    env->regs[R_ESI] = tswapl(sc->esi);
+    env->regs[R_EBP] = tswapl(sc->ebp);
+    env->regs[R_ESP] = tswapl(sc->esp);
+    env->regs[R_EBX] = tswapl(sc->ebx);
+    env->regs[R_EDX] = tswapl(sc->edx);
+    env->regs[R_ECX] = tswapl(sc->ecx);
+    env->regs[R_EAX] = tswapl(sc->eax);
+    env->eip = tswapl(sc->eip);
 
-        cpu_x86_load_seg(env, R_CS, lduw_p(&sc->cs) | 3);
-        cpu_x86_load_seg(env, R_SS, lduw_p(&sc->ss) | 3);
+    cpu_x86_load_seg(env, R_CS, lduw_p(&sc->cs) | 3);
+    cpu_x86_load_seg(env, R_SS, lduw_p(&sc->ss) | 3);
 
-        tmpflags = tswapl(sc->eflags);
-        env->eflags = (env->eflags & ~0x40DD5) | (tmpflags & 0x40DD5);
-        //		regs->orig_eax = -1;		/* disable syscall checks */
+    tmpflags = tswapl(sc->eflags);
+    env->eflags = (env->eflags & ~0x40DD5) | (tmpflags & 0x40DD5);
+    //		regs->orig_eax = -1;		/* disable syscall checks */
 
-        fpstate_addr = tswapl(sc->fpstate);
-	if (fpstate_addr != 0) {
-                if (!access_ok(VERIFY_READ, fpstate_addr, 
-                               sizeof(struct target_fpstate)))
-                        goto badframe;
-                cpu_x86_frstor(env, fpstate_addr, 1);
-	}
+    fpstate_addr = tswapl(sc->fpstate);
+    if (fpstate_addr != 0) {
+        if (!access_ok(VERIFY_READ, fpstate_addr,
+                       sizeof(struct target_fpstate)))
+            goto badframe;
+        cpu_x86_frstor(env, fpstate_addr, 1);
+    }
 
-        *peax = tswapl(sc->eax);
-	return err;
+    return err;
 badframe:
-	return 1;
+    return 1;
 }
 
 long do_sigreturn(CPUX86State *env)
@@ -1093,11 +1142,9 @@
     abi_ulong frame_addr = env->regs[R_ESP] - 8;
     target_sigset_t target_set;
     sigset_t set;
-    int eax, i;
+    int i;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_sigreturn\n");
-#endif
+    trace_user_do_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
         goto badframe;
     /* set blocked signals */
@@ -1107,13 +1154,13 @@
     }
 
     target_to_host_sigset_internal(&set, &target_set);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     /* restore registers */
-    if (restore_sigcontext(env, &frame->sc, &eax))
+    if (restore_sigcontext(env, &frame->sc))
         goto badframe;
     unlock_user_struct(frame, frame_addr, 0);
-    return eax;
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     unlock_user_struct(frame, frame_addr, 0);
@@ -1123,31 +1170,33 @@
 
 long do_rt_sigreturn(CPUX86State *env)
 {
-        abi_ulong frame_addr;
-	struct rt_sigframe *frame;
-        sigset_t set;
-	int eax;
+    abi_ulong frame_addr;
+    struct rt_sigframe *frame;
+    sigset_t set;
 
-        frame_addr = env->regs[R_ESP] - 4;
-        if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-                goto badframe;
-        target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
-        do_sigprocmask(SIG_SETMASK, &set, NULL);
+    frame_addr = env->regs[R_ESP] - 4;
+    trace_user_do_rt_sigreturn(env, frame_addr);
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
+        goto badframe;
+    target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
+    set_sigmask(&set);
 
-	if (restore_sigcontext(env, &frame->uc.tuc_mcontext, &eax))
-		goto badframe;
+    if (restore_sigcontext(env, &frame->uc.tuc_mcontext)) {
+        goto badframe;
+    }
 
-	if (do_sigaltstack(frame_addr + offsetof(struct rt_sigframe, uc.tuc_stack), 0, 
-                           get_sp_from_cpustate(env)) == -EFAULT)
-		goto badframe;
+    if (do_sigaltstack(frame_addr + offsetof(struct rt_sigframe, uc.tuc_stack), 0,
+                       get_sp_from_cpustate(env)) == -EFAULT) {
+        goto badframe;
+    }
 
-        unlock_user_struct(frame, frame_addr, 0);
-	return eax;
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
-        unlock_user_struct(frame, frame_addr, 0);
-        force_sig(TARGET_SIGSEGV);
-	return 0;
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV);
+    return 0;
 }
 
 #elif defined(TARGET_AARCH64)
@@ -1270,7 +1319,7 @@
     uint64_t pstate;
 
     target_to_host_sigset(&set, &sf->uc.tuc_sigmask);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     for (i = 0; i < 31; i++) {
         __get_user(env->xregs[i], &sf->uc.tuc_mcontext.regs[i]);
@@ -1332,6 +1381,7 @@
     abi_ulong frame_addr, return_addr;
 
     frame_addr = get_sigframe(ka, env);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -1360,7 +1410,7 @@
     env->pc = ka->_sa_handler;
     env->xregs[30] = return_addr;
     if (info) {
-        copy_siginfo_to_user(&frame->info, info);
+        tswap_siginfo(&frame->info, info);
         env->xregs[1] = frame_addr + offsetof(struct target_rt_sigframe, info);
         env->xregs[2] = frame_addr + offsetof(struct target_rt_sigframe, uc);
     }
@@ -1391,6 +1441,7 @@
     struct target_rt_sigframe *frame = NULL;
     abi_ulong frame_addr = env->xregs[31];
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (frame_addr & 15) {
         goto badframe;
     }
@@ -1410,7 +1461,7 @@
     }
 
     unlock_user_struct(frame, frame_addr, 0);
-    return env->xregs[0];
+    return -TARGET_QEMU_ESIGRETURN;
 
  badframe:
     unlock_user_struct(frame, frame_addr, 0);
@@ -1426,27 +1477,27 @@
 #elif defined(TARGET_ARM)
 
 struct target_sigcontext {
-	abi_ulong trap_no;
-	abi_ulong error_code;
-	abi_ulong oldmask;
-	abi_ulong arm_r0;
-	abi_ulong arm_r1;
-	abi_ulong arm_r2;
-	abi_ulong arm_r3;
-	abi_ulong arm_r4;
-	abi_ulong arm_r5;
-	abi_ulong arm_r6;
-	abi_ulong arm_r7;
-	abi_ulong arm_r8;
-	abi_ulong arm_r9;
-	abi_ulong arm_r10;
-	abi_ulong arm_fp;
-	abi_ulong arm_ip;
-	abi_ulong arm_sp;
-	abi_ulong arm_lr;
-	abi_ulong arm_pc;
-	abi_ulong arm_cpsr;
-	abi_ulong fault_address;
+    abi_ulong trap_no;
+    abi_ulong error_code;
+    abi_ulong oldmask;
+    abi_ulong arm_r0;
+    abi_ulong arm_r1;
+    abi_ulong arm_r2;
+    abi_ulong arm_r3;
+    abi_ulong arm_r4;
+    abi_ulong arm_r5;
+    abi_ulong arm_r6;
+    abi_ulong arm_r7;
+    abi_ulong arm_r8;
+    abi_ulong arm_r9;
+    abi_ulong arm_r10;
+    abi_ulong arm_fp;
+    abi_ulong arm_ip;
+    abi_ulong arm_sp;
+    abi_ulong arm_lr;
+    abi_ulong arm_pc;
+    abi_ulong arm_cpsr;
+    abi_ulong fault_address;
 };
 
 struct target_ucontext_v1 {
@@ -1560,82 +1611,84 @@
 setup_sigcontext(struct target_sigcontext *sc, /*struct _fpstate *fpstate,*/
                  CPUARMState *env, abi_ulong mask)
 {
-	__put_user(env->regs[0], &sc->arm_r0);
-	__put_user(env->regs[1], &sc->arm_r1);
-	__put_user(env->regs[2], &sc->arm_r2);
-	__put_user(env->regs[3], &sc->arm_r3);
-	__put_user(env->regs[4], &sc->arm_r4);
-	__put_user(env->regs[5], &sc->arm_r5);
-	__put_user(env->regs[6], &sc->arm_r6);
-	__put_user(env->regs[7], &sc->arm_r7);
-	__put_user(env->regs[8], &sc->arm_r8);
-	__put_user(env->regs[9], &sc->arm_r9);
-	__put_user(env->regs[10], &sc->arm_r10);
-	__put_user(env->regs[11], &sc->arm_fp);
-	__put_user(env->regs[12], &sc->arm_ip);
-	__put_user(env->regs[13], &sc->arm_sp);
-	__put_user(env->regs[14], &sc->arm_lr);
-	__put_user(env->regs[15], &sc->arm_pc);
+    __put_user(env->regs[0], &sc->arm_r0);
+    __put_user(env->regs[1], &sc->arm_r1);
+    __put_user(env->regs[2], &sc->arm_r2);
+    __put_user(env->regs[3], &sc->arm_r3);
+    __put_user(env->regs[4], &sc->arm_r4);
+    __put_user(env->regs[5], &sc->arm_r5);
+    __put_user(env->regs[6], &sc->arm_r6);
+    __put_user(env->regs[7], &sc->arm_r7);
+    __put_user(env->regs[8], &sc->arm_r8);
+    __put_user(env->regs[9], &sc->arm_r9);
+    __put_user(env->regs[10], &sc->arm_r10);
+    __put_user(env->regs[11], &sc->arm_fp);
+    __put_user(env->regs[12], &sc->arm_ip);
+    __put_user(env->regs[13], &sc->arm_sp);
+    __put_user(env->regs[14], &sc->arm_lr);
+    __put_user(env->regs[15], &sc->arm_pc);
 #ifdef TARGET_CONFIG_CPU_32
-	__put_user(cpsr_read(env), &sc->arm_cpsr);
+    __put_user(cpsr_read(env), &sc->arm_cpsr);
 #endif
 
-	__put_user(/* current->thread.trap_no */ 0, &sc->trap_no);
-	__put_user(/* current->thread.error_code */ 0, &sc->error_code);
-	__put_user(/* current->thread.address */ 0, &sc->fault_address);
-	__put_user(mask, &sc->oldmask);
+    __put_user(/* current->thread.trap_no */ 0, &sc->trap_no);
+    __put_user(/* current->thread.error_code */ 0, &sc->error_code);
+    __put_user(/* current->thread.address */ 0, &sc->fault_address);
+    __put_user(mask, &sc->oldmask);
 }
 
 static inline abi_ulong
 get_sigframe(struct target_sigaction *ka, CPUARMState *regs, int framesize)
 {
-	unsigned long sp = regs->regs[13];
+    unsigned long sp = regs->regs[13];
 
-	/*
-	 * This is the X/Open sanctioned signal stack switching.
-	 */
-	if ((ka->sa_flags & TARGET_SA_ONSTACK) && !sas_ss_flags(sp))
-            sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
-	/*
-	 * ATPCS B01 mandates 8-byte alignment
-	 */
-	return (sp - framesize) & ~7;
+    /*
+     * This is the X/Open sanctioned signal stack switching.
+     */
+    if ((ka->sa_flags & TARGET_SA_ONSTACK) && !sas_ss_flags(sp)) {
+        sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+    }
+    /*
+     * ATPCS B01 mandates 8-byte alignment
+     */
+    return (sp - framesize) & ~7;
 }
 
 static void
 setup_return(CPUARMState *env, struct target_sigaction *ka,
-	     abi_ulong *rc, abi_ulong frame_addr, int usig, abi_ulong rc_addr)
+             abi_ulong *rc, abi_ulong frame_addr, int usig, abi_ulong rc_addr)
 {
-	abi_ulong handler = ka->_sa_handler;
-	abi_ulong retcode;
-	int thumb = handler & 1;
-	uint32_t cpsr = cpsr_read(env);
+    abi_ulong handler = ka->_sa_handler;
+    abi_ulong retcode;
+    int thumb = handler & 1;
+    uint32_t cpsr = cpsr_read(env);
 
-	cpsr &= ~CPSR_IT;
-	if (thumb) {
-		cpsr |= CPSR_T;
-	} else {
-		cpsr &= ~CPSR_T;
-	}
+    cpsr &= ~CPSR_IT;
+    if (thumb) {
+        cpsr |= CPSR_T;
+    } else {
+        cpsr &= ~CPSR_T;
+    }
 
-	if (ka->sa_flags & TARGET_SA_RESTORER) {
-		retcode = ka->sa_restorer;
-	} else {
-		unsigned int idx = thumb;
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
+        retcode = ka->sa_restorer;
+    } else {
+        unsigned int idx = thumb;
 
-		if (ka->sa_flags & TARGET_SA_SIGINFO)
-			idx += 2;
+        if (ka->sa_flags & TARGET_SA_SIGINFO) {
+            idx += 2;
+        }
 
         __put_user(retcodes[idx], rc);
 
-		retcode = rc_addr + thumb;
-	}
+        retcode = rc_addr + thumb;
+    }
 
-	env->regs[0] = usig;
-	env->regs[13] = frame_addr;
-	env->regs[14] = retcode;
-	env->regs[15] = handler & (thumb ? ~1 : ~3);
-	cpsr_write(env, cpsr, 0xffffffff);
+    env->regs[0] = usig;
+    env->regs[13] = frame_addr;
+    env->regs[14] = retcode;
+    env->regs[15] = handler & (thumb ? ~1 : ~3);
+    cpsr_write(env, cpsr, CPSR_IT | CPSR_T, CPSRWriteByInstr);
 }
 
 static abi_ulong *setup_sigframe_v2_vfp(abi_ulong *regspace, CPUARMState *env)
@@ -1713,40 +1766,44 @@
 static void setup_frame_v1(int usig, struct target_sigaction *ka,
                            target_sigset_t *set, CPUARMState *regs)
 {
-	struct sigframe_v1 *frame;
-	abi_ulong frame_addr = get_sigframe(ka, regs, sizeof(*frame));
-	int i;
+    struct sigframe_v1 *frame;
+    abi_ulong frame_addr = get_sigframe(ka, regs, sizeof(*frame));
+    int i;
 
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-		return;
+    trace_user_setup_frame(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        return;
+    }
 
-	setup_sigcontext(&frame->sc, regs, set->sig[0]);
+    setup_sigcontext(&frame->sc, regs, set->sig[0]);
 
     for(i = 1; i < TARGET_NSIG_WORDS; i++) {
         __put_user(set->sig[i], &frame->extramask[i - 1]);
     }
 
-        setup_return(regs, ka, &frame->retcode, frame_addr, usig,
-                     frame_addr + offsetof(struct sigframe_v1, retcode));
+    setup_return(regs, ka, &frame->retcode, frame_addr, usig,
+                 frame_addr + offsetof(struct sigframe_v1, retcode));
 
-	unlock_user_struct(frame, frame_addr, 1);
+    unlock_user_struct(frame, frame_addr, 1);
 }
 
 static void setup_frame_v2(int usig, struct target_sigaction *ka,
                            target_sigset_t *set, CPUARMState *regs)
 {
-	struct sigframe_v2 *frame;
-	abi_ulong frame_addr = get_sigframe(ka, regs, sizeof(*frame));
+    struct sigframe_v2 *frame;
+    abi_ulong frame_addr = get_sigframe(ka, regs, sizeof(*frame));
 
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-		return;
+    trace_user_setup_frame(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        return;
+    }
 
-        setup_sigframe_v2(&frame->uc, set, regs);
+    setup_sigframe_v2(&frame->uc, set, regs);
 
-        setup_return(regs, ka, &frame->retcode, frame_addr, usig,
-                     frame_addr + offsetof(struct sigframe_v2, retcode));
+    setup_return(regs, ka, &frame->retcode, frame_addr, usig,
+                 frame_addr + offsetof(struct sigframe_v2, retcode));
 
-	unlock_user_struct(frame, frame_addr, 1);
+    unlock_user_struct(frame, frame_addr, 1);
 }
 
 static void setup_frame(int usig, struct target_sigaction *ka,
@@ -1764,68 +1821,72 @@
                               target_siginfo_t *info,
                               target_sigset_t *set, CPUARMState *env)
 {
-	struct rt_sigframe_v1 *frame;
-	abi_ulong frame_addr = get_sigframe(ka, env, sizeof(*frame));
-	struct target_sigaltstack stack;
-	int i;
-        abi_ulong info_addr, uc_addr;
+    struct rt_sigframe_v1 *frame;
+    abi_ulong frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    struct target_sigaltstack stack;
+    int i;
+    abi_ulong info_addr, uc_addr;
 
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-            return /* 1 */;
+    trace_user_setup_rt_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        return /* 1 */;
+    }
 
-        info_addr = frame_addr + offsetof(struct rt_sigframe_v1, info);
-	__put_user(info_addr, &frame->pinfo);
-        uc_addr = frame_addr + offsetof(struct rt_sigframe_v1, uc);
-	__put_user(uc_addr, &frame->puc);
-	copy_siginfo_to_user(&frame->info, info);
+    info_addr = frame_addr + offsetof(struct rt_sigframe_v1, info);
+    __put_user(info_addr, &frame->pinfo);
+    uc_addr = frame_addr + offsetof(struct rt_sigframe_v1, uc);
+    __put_user(uc_addr, &frame->puc);
+    tswap_siginfo(&frame->info, info);
 
-	/* Clear all the bits of the ucontext we don't use.  */
-	memset(&frame->uc, 0, offsetof(struct target_ucontext_v1, tuc_mcontext));
+    /* Clear all the bits of the ucontext we don't use.  */
+    memset(&frame->uc, 0, offsetof(struct target_ucontext_v1, tuc_mcontext));
 
-        memset(&stack, 0, sizeof(stack));
-        __put_user(target_sigaltstack_used.ss_sp, &stack.ss_sp);
-        __put_user(target_sigaltstack_used.ss_size, &stack.ss_size);
-        __put_user(sas_ss_flags(get_sp_from_cpustate(env)), &stack.ss_flags);
-        memcpy(&frame->uc.tuc_stack, &stack, sizeof(stack));
+    memset(&stack, 0, sizeof(stack));
+    __put_user(target_sigaltstack_used.ss_sp, &stack.ss_sp);
+    __put_user(target_sigaltstack_used.ss_size, &stack.ss_size);
+    __put_user(sas_ss_flags(get_sp_from_cpustate(env)), &stack.ss_flags);
+    memcpy(&frame->uc.tuc_stack, &stack, sizeof(stack));
 
-	setup_sigcontext(&frame->uc.tuc_mcontext, env, set->sig[0]);
-        for(i = 0; i < TARGET_NSIG_WORDS; i++) {
-            __put_user(set->sig[i], &frame->uc.tuc_sigmask.sig[i]);
-        }
+    setup_sigcontext(&frame->uc.tuc_mcontext, env, set->sig[0]);
+    for(i = 0; i < TARGET_NSIG_WORDS; i++) {
+        __put_user(set->sig[i], &frame->uc.tuc_sigmask.sig[i]);
+    }
 
-        setup_return(env, ka, &frame->retcode, frame_addr, usig,
-                     frame_addr + offsetof(struct rt_sigframe_v1, retcode));
+    setup_return(env, ka, &frame->retcode, frame_addr, usig,
+                 frame_addr + offsetof(struct rt_sigframe_v1, retcode));
 
-        env->regs[1] = info_addr;
-        env->regs[2] = uc_addr;
+    env->regs[1] = info_addr;
+    env->regs[2] = uc_addr;
 
-	unlock_user_struct(frame, frame_addr, 1);
+    unlock_user_struct(frame, frame_addr, 1);
 }
 
 static void setup_rt_frame_v2(int usig, struct target_sigaction *ka,
                               target_siginfo_t *info,
                               target_sigset_t *set, CPUARMState *env)
 {
-	struct rt_sigframe_v2 *frame;
-	abi_ulong frame_addr = get_sigframe(ka, env, sizeof(*frame));
-        abi_ulong info_addr, uc_addr;
+    struct rt_sigframe_v2 *frame;
+    abi_ulong frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    abi_ulong info_addr, uc_addr;
 
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-            return /* 1 */;
+    trace_user_setup_rt_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        return /* 1 */;
+    }
 
-        info_addr = frame_addr + offsetof(struct rt_sigframe_v2, info);
-        uc_addr = frame_addr + offsetof(struct rt_sigframe_v2, uc);
-	copy_siginfo_to_user(&frame->info, info);
+    info_addr = frame_addr + offsetof(struct rt_sigframe_v2, info);
+    uc_addr = frame_addr + offsetof(struct rt_sigframe_v2, uc);
+    tswap_siginfo(&frame->info, info);
 
-        setup_sigframe_v2(&frame->uc, set, env);
+    setup_sigframe_v2(&frame->uc, set, env);
 
-        setup_return(env, ka, &frame->retcode, frame_addr, usig,
-                     frame_addr + offsetof(struct rt_sigframe_v2, retcode));
+    setup_return(env, ka, &frame->retcode, frame_addr, usig,
+                 frame_addr + offsetof(struct rt_sigframe_v2, retcode));
 
-        env->regs[1] = info_addr;
-        env->regs[2] = uc_addr;
+    env->regs[1] = info_addr;
+    env->regs[2] = uc_addr;
 
-	unlock_user_struct(frame, frame_addr, 1);
+    unlock_user_struct(frame, frame_addr, 1);
 }
 
 static void setup_rt_frame(int usig, struct target_sigaction *ka,
@@ -1842,8 +1903,8 @@
 static int
 restore_sigcontext(CPUARMState *env, struct target_sigcontext *sc)
 {
-	int err = 0;
-        uint32_t cpsr;
+    int err = 0;
+    uint32_t cpsr;
 
     __get_user(env->regs[0], &sc->arm_r0);
     __get_user(env->regs[1], &sc->arm_r1);
@@ -1863,57 +1924,60 @@
     __get_user(env->regs[15], &sc->arm_pc);
 #ifdef TARGET_CONFIG_CPU_32
     __get_user(cpsr, &sc->arm_cpsr);
-        cpsr_write(env, cpsr, CPSR_USER | CPSR_EXEC);
+    cpsr_write(env, cpsr, CPSR_USER | CPSR_EXEC, CPSRWriteByInstr);
 #endif
 
-	err |= !valid_user_regs(env);
+    err |= !valid_user_regs(env);
 
-	return err;
+    return err;
 }
 
 static long do_sigreturn_v1(CPUARMState *env)
 {
-        abi_ulong frame_addr;
-        struct sigframe_v1 *frame = NULL;
-	target_sigset_t set;
-        sigset_t host_set;
-        int i;
+    abi_ulong frame_addr;
+    struct sigframe_v1 *frame = NULL;
+    target_sigset_t set;
+    sigset_t host_set;
+    int i;
 
-	/*
-	 * Since we stacked the signal on a 64-bit boundary,
-	 * then 'sp' should be word aligned here.  If it's
-	 * not, then the user is trying to mess with us.
-	 */
-        frame_addr = env->regs[13];
-        if (frame_addr & 7) {
-            goto badframe;
-        }
+    /*
+     * Since we stacked the signal on a 64-bit boundary,
+     * then 'sp' should be word aligned here.  If it's
+     * not, then the user is trying to mess with us.
+     */
+    frame_addr = env->regs[13];
+    trace_user_do_sigreturn(env, frame_addr);
+    if (frame_addr & 7) {
+        goto badframe;
+    }
 
-	if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-                goto badframe;
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
     __get_user(set.sig[0], &frame->sc.oldmask);
     for(i = 1; i < TARGET_NSIG_WORDS; i++) {
         __get_user(set.sig[i], &frame->extramask[i - 1]);
     }
 
-        target_to_host_sigset_internal(&host_set, &set);
-        do_sigprocmask(SIG_SETMASK, &host_set, NULL);
+    target_to_host_sigset_internal(&host_set, &set);
+    set_sigmask(&host_set);
 
-	if (restore_sigcontext(env, &frame->sc))
-		goto badframe;
+    if (restore_sigcontext(env, &frame->sc)) {
+        goto badframe;
+    }
 
 #if 0
-	/* Send SIGTRAP if we're single-stepping */
-	if (ptrace_cancel_bpt(current))
-		send_sig(SIGTRAP, current, 1);
+    /* Send SIGTRAP if we're single-stepping */
+    if (ptrace_cancel_bpt(current))
+        send_sig(SIGTRAP, current, 1);
 #endif
-	unlock_user_struct(frame, frame_addr, 0);
-        return env->regs[0];
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
-        force_sig(TARGET_SIGSEGV /* , current */);
-	return 0;
+    force_sig(TARGET_SIGSEGV /* , current */);
+    return 0;
 }
 
 static abi_ulong *restore_sigframe_v2_vfp(CPUARMState *env, abi_ulong *regspace)
@@ -1978,7 +2042,7 @@
     abi_ulong *regspace;
 
     target_to_host_sigset(&host_set, &uc->tuc_sigmask);
-    do_sigprocmask(SIG_SETMASK, &host_set, NULL);
+    set_sigmask(&host_set);
 
     if (restore_sigcontext(env, &uc->tuc_mcontext))
         return 1;
@@ -2004,7 +2068,7 @@
 #if 0
     /* Send SIGTRAP if we're single-stepping */
     if (ptrace_cancel_bpt(current))
-            send_sig(SIGTRAP, current, 1);
+        send_sig(SIGTRAP, current, 1);
 #endif
 
     return 0;
@@ -2012,32 +2076,35 @@
 
 static long do_sigreturn_v2(CPUARMState *env)
 {
-        abi_ulong frame_addr;
-        struct sigframe_v2 *frame = NULL;
+    abi_ulong frame_addr;
+    struct sigframe_v2 *frame = NULL;
 
-	/*
-	 * Since we stacked the signal on a 64-bit boundary,
-	 * then 'sp' should be word aligned here.  If it's
-	 * not, then the user is trying to mess with us.
-	 */
-        frame_addr = env->regs[13];
-        if (frame_addr & 7) {
-            goto badframe;
-        }
+    /*
+     * Since we stacked the signal on a 64-bit boundary,
+     * then 'sp' should be word aligned here.  If it's
+     * not, then the user is trying to mess with us.
+     */
+    frame_addr = env->regs[13];
+    trace_user_do_sigreturn(env, frame_addr);
+    if (frame_addr & 7) {
+        goto badframe;
+    }
 
-	if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-                goto badframe;
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
-        if (do_sigframe_return_v2(env, frame_addr, &frame->uc))
-                goto badframe;
+    if (do_sigframe_return_v2(env, frame_addr, &frame->uc)) {
+        goto badframe;
+    }
 
-	unlock_user_struct(frame, frame_addr, 0);
-	return env->regs[0];
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
-	unlock_user_struct(frame, frame_addr, 0);
-        force_sig(TARGET_SIGSEGV /* , current */);
-	return 0;
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV /* , current */);
+    return 0;
 }
 
 long do_sigreturn(CPUARMState *env)
@@ -2051,74 +2118,80 @@
 
 static long do_rt_sigreturn_v1(CPUARMState *env)
 {
-        abi_ulong frame_addr;
-        struct rt_sigframe_v1 *frame = NULL;
-        sigset_t host_set;
+    abi_ulong frame_addr;
+    struct rt_sigframe_v1 *frame = NULL;
+    sigset_t host_set;
 
-	/*
-	 * Since we stacked the signal on a 64-bit boundary,
-	 * then 'sp' should be word aligned here.  If it's
-	 * not, then the user is trying to mess with us.
-	 */
-        frame_addr = env->regs[13];
-        if (frame_addr & 7) {
-            goto badframe;
-        }
+    /*
+     * Since we stacked the signal on a 64-bit boundary,
+     * then 'sp' should be word aligned here.  If it's
+     * not, then the user is trying to mess with us.
+     */
+    frame_addr = env->regs[13];
+    trace_user_do_rt_sigreturn(env, frame_addr);
+    if (frame_addr & 7) {
+        goto badframe;
+    }
 
-	if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-                goto badframe;
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
-        target_to_host_sigset(&host_set, &frame->uc.tuc_sigmask);
-        do_sigprocmask(SIG_SETMASK, &host_set, NULL);
+    target_to_host_sigset(&host_set, &frame->uc.tuc_sigmask);
+    set_sigmask(&host_set);
 
-	if (restore_sigcontext(env, &frame->uc.tuc_mcontext))
-		goto badframe;
+    if (restore_sigcontext(env, &frame->uc.tuc_mcontext)) {
+        goto badframe;
+    }
 
-	if (do_sigaltstack(frame_addr + offsetof(struct rt_sigframe_v1, uc.tuc_stack), 0, get_sp_from_cpustate(env)) == -EFAULT)
-		goto badframe;
+    if (do_sigaltstack(frame_addr + offsetof(struct rt_sigframe_v1, uc.tuc_stack), 0, get_sp_from_cpustate(env)) == -EFAULT)
+        goto badframe;
 
 #if 0
-	/* Send SIGTRAP if we're single-stepping */
-	if (ptrace_cancel_bpt(current))
-		send_sig(SIGTRAP, current, 1);
+    /* Send SIGTRAP if we're single-stepping */
+    if (ptrace_cancel_bpt(current))
+        send_sig(SIGTRAP, current, 1);
 #endif
-	unlock_user_struct(frame, frame_addr, 0);
-	return env->regs[0];
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
-	unlock_user_struct(frame, frame_addr, 0);
-        force_sig(TARGET_SIGSEGV /* , current */);
-	return 0;
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV /* , current */);
+    return 0;
 }
 
 static long do_rt_sigreturn_v2(CPUARMState *env)
 {
-        abi_ulong frame_addr;
-        struct rt_sigframe_v2 *frame = NULL;
+    abi_ulong frame_addr;
+    struct rt_sigframe_v2 *frame = NULL;
 
-	/*
-	 * Since we stacked the signal on a 64-bit boundary,
-	 * then 'sp' should be word aligned here.  If it's
-	 * not, then the user is trying to mess with us.
-	 */
-        frame_addr = env->regs[13];
-        if (frame_addr & 7) {
-            goto badframe;
-        }
+    /*
+     * Since we stacked the signal on a 64-bit boundary,
+     * then 'sp' should be word aligned here.  If it's
+     * not, then the user is trying to mess with us.
+     */
+    frame_addr = env->regs[13];
+    trace_user_do_rt_sigreturn(env, frame_addr);
+    if (frame_addr & 7) {
+        goto badframe;
+    }
 
-	if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-                goto badframe;
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
-        if (do_sigframe_return_v2(env, frame_addr, &frame->uc))
-                goto badframe;
+    if (do_sigframe_return_v2(env, frame_addr, &frame->uc)) {
+        goto badframe;
+    }
 
-	unlock_user_struct(frame, frame_addr, 0);
-	return env->regs[0];
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
-	unlock_user_struct(frame, frame_addr, 0);
-        force_sig(TARGET_SIGSEGV /* , current */);
-	return 0;
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV /* , current */);
+    return 0;
 }
 
 long do_rt_sigreturn(CPUARMState *env)
@@ -2136,83 +2209,83 @@
 
 /* This is what SunOS does, so shall I. */
 struct target_sigcontext {
-        abi_ulong sigc_onstack;      /* state to restore */
+    abi_ulong sigc_onstack;      /* state to restore */
 
-        abi_ulong sigc_mask;         /* sigmask to restore */
-        abi_ulong sigc_sp;           /* stack pointer */
-        abi_ulong sigc_pc;           /* program counter */
-        abi_ulong sigc_npc;          /* next program counter */
-        abi_ulong sigc_psr;          /* for condition codes etc */
-        abi_ulong sigc_g1;           /* User uses these two registers */
-        abi_ulong sigc_o0;           /* within the trampoline code. */
+    abi_ulong sigc_mask;         /* sigmask to restore */
+    abi_ulong sigc_sp;           /* stack pointer */
+    abi_ulong sigc_pc;           /* program counter */
+    abi_ulong sigc_npc;          /* next program counter */
+    abi_ulong sigc_psr;          /* for condition codes etc */
+    abi_ulong sigc_g1;           /* User uses these two registers */
+    abi_ulong sigc_o0;           /* within the trampoline code. */
 
-        /* Now comes information regarding the users window set
+    /* Now comes information regarding the users window set
          * at the time of the signal.
          */
-        abi_ulong sigc_oswins;       /* outstanding windows */
+    abi_ulong sigc_oswins;       /* outstanding windows */
 
-        /* stack ptrs for each regwin buf */
-        char *sigc_spbuf[__SUNOS_MAXWIN];
+    /* stack ptrs for each regwin buf */
+    char *sigc_spbuf[__SUNOS_MAXWIN];
 
-        /* Windows to restore after signal */
-        struct {
-                abi_ulong locals[8];
-                abi_ulong ins[8];
-        } sigc_wbuf[__SUNOS_MAXWIN];
+    /* Windows to restore after signal */
+    struct {
+        abi_ulong locals[8];
+        abi_ulong ins[8];
+    } sigc_wbuf[__SUNOS_MAXWIN];
 };
 /* A Sparc stack frame */
 struct sparc_stackf {
-        abi_ulong locals[8];
-        abi_ulong ins[8];
-        /* It's simpler to treat fp and callers_pc as elements of ins[]
+    abi_ulong locals[8];
+    abi_ulong ins[8];
+    /* It's simpler to treat fp and callers_pc as elements of ins[]
          * since we never need to access them ourselves.
          */
-        char *structptr;
-        abi_ulong xargs[6];
-        abi_ulong xxargs[1];
+    char *structptr;
+    abi_ulong xargs[6];
+    abi_ulong xxargs[1];
 };
 
 typedef struct {
-        struct {
-                abi_ulong psr;
-                abi_ulong pc;
-                abi_ulong npc;
-                abi_ulong y;
-                abi_ulong u_regs[16]; /* globals and ins */
-        }               si_regs;
-        int             si_mask;
+    struct {
+        abi_ulong psr;
+        abi_ulong pc;
+        abi_ulong npc;
+        abi_ulong y;
+        abi_ulong u_regs[16]; /* globals and ins */
+    }               si_regs;
+    int             si_mask;
 } __siginfo_t;
 
 typedef struct {
-        abi_ulong       si_float_regs[32];
-        unsigned   long si_fsr;
-        unsigned   long si_fpqdepth;
-        struct {
-                unsigned long *insn_addr;
-                unsigned long insn;
-        } si_fpqueue [16];
+    abi_ulong  si_float_regs[32];
+    unsigned   long si_fsr;
+    unsigned   long si_fpqdepth;
+    struct {
+        unsigned long *insn_addr;
+        unsigned long insn;
+    } si_fpqueue [16];
 } qemu_siginfo_fpu_t;
 
 
 struct target_signal_frame {
-	struct sparc_stackf	ss;
-	__siginfo_t		info;
-	abi_ulong               fpu_save;
-	abi_ulong		insns[2] __attribute__ ((aligned (8)));
-	abi_ulong		extramask[TARGET_NSIG_WORDS - 1];
-	abi_ulong		extra_size; /* Should be 0 */
-	qemu_siginfo_fpu_t	fpu_state;
+    struct sparc_stackf ss;
+    __siginfo_t         info;
+    abi_ulong           fpu_save;
+    abi_ulong           insns[2] __attribute__ ((aligned (8)));
+    abi_ulong           extramask[TARGET_NSIG_WORDS - 1];
+    abi_ulong           extra_size; /* Should be 0 */
+    qemu_siginfo_fpu_t fpu_state;
 };
 struct target_rt_signal_frame {
-	struct sparc_stackf	ss;
-	siginfo_t		info;
-	abi_ulong		regs[20];
-	sigset_t		mask;
-	abi_ulong               fpu_save;
-	unsigned int		insns[2];
-	stack_t			stack;
-	unsigned int		extra_size; /* Should be 0 */
-	qemu_siginfo_fpu_t	fpu_state;
+    struct sparc_stackf ss;
+    siginfo_t           info;
+    abi_ulong           regs[20];
+    sigset_t            mask;
+    abi_ulong           fpu_save;
+    unsigned int        insns[2];
+    stack_t             stack;
+    unsigned int        extra_size; /* Should be 0 */
+    qemu_siginfo_fpu_t  fpu_state;
 };
 
 #define UREG_O0        16
@@ -2233,36 +2306,37 @@
                                      CPUSPARCState *env,
                                      unsigned long framesize)
 {
-	abi_ulong sp;
+    abi_ulong sp;
 
-	sp = env->regwptr[UREG_FP];
+    sp = env->regwptr[UREG_FP];
 
-	/* This is the X/Open sanctioned signal stack switching.  */
-	if (sa->sa_flags & TARGET_SA_ONSTACK) {
-            if (!on_sig_stack(sp)
-                && !((target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size) & 7))
-                sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
-	}
-	return sp - framesize;
+    /* This is the X/Open sanctioned signal stack switching.  */
+    if (sa->sa_flags & TARGET_SA_ONSTACK) {
+        if (!on_sig_stack(sp)
+                && !((target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size) & 7)) {
+            sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+        }
+    }
+    return sp - framesize;
 }
 
 static int
 setup___siginfo(__siginfo_t *si, CPUSPARCState *env, abi_ulong mask)
 {
-	int err = 0, i;
+    int err = 0, i;
 
     __put_user(env->psr, &si->si_regs.psr);
     __put_user(env->pc, &si->si_regs.pc);
     __put_user(env->npc, &si->si_regs.npc);
     __put_user(env->y, &si->si_regs.y);
-	for (i=0; i < 8; i++) {
+    for (i=0; i < 8; i++) {
         __put_user(env->gregs[i], &si->si_regs.u_regs[i]);
-	}
-	for (i=0; i < 8; i++) {
+    }
+    for (i=0; i < 8; i++) {
         __put_user(env->regwptr[UREG_I0 + i], &si->si_regs.u_regs[i+8]);
-	}
+    }
     __put_user(mask, &si->si_mask);
-	return err;
+    return err;
 }
 
 #if 0
@@ -2270,7 +2344,7 @@
 setup_sigcontext(struct target_sigcontext *sc, /*struct _fpstate *fpstate,*/
                  CPUSPARCState *env, unsigned long mask)
 {
-	int err = 0;
+    int err = 0;
 
     __put_user(mask, &sc->sigc_mask);
     __put_user(env->regwptr[UREG_SP], &sc->sigc_sp);
@@ -2280,7 +2354,7 @@
     __put_user(env->gregs[1], &sc->sigc_g1);
     __put_user(env->regwptr[UREG_O0], &sc->sigc_o0);
 
-	return err;
+    return err;
 }
 #endif
 #define NF_ALIGNEDSZ  (((sizeof(struct target_signal_frame) + 7) & (~7)))
@@ -2288,91 +2362,90 @@
 static void setup_frame(int sig, struct target_sigaction *ka,
                         target_sigset_t *set, CPUSPARCState *env)
 {
-        abi_ulong sf_addr;
-	struct target_signal_frame *sf;
-	int sigframe_size, err, i;
+    abi_ulong sf_addr;
+    struct target_signal_frame *sf;
+    int sigframe_size, err, i;
 
-	/* 1. Make sure everything is clean */
-	//synchronize_user_stack();
+    /* 1. Make sure everything is clean */
+    //synchronize_user_stack();
 
-        sigframe_size = NF_ALIGNEDSZ;
-	sf_addr = get_sigframe(ka, env, sigframe_size);
+    sigframe_size = NF_ALIGNEDSZ;
+    sf_addr = get_sigframe(ka, env, sigframe_size);
+    trace_user_setup_frame(env, sf_addr);
 
-        sf = lock_user(VERIFY_WRITE, sf_addr, 
-                       sizeof(struct target_signal_frame), 0);
-        if (!sf)
-		goto sigsegv;
-                
-	//fprintf(stderr, "sf: %x pc %x fp %x sp %x\n", sf, env->pc, env->regwptr[UREG_FP], env->regwptr[UREG_SP]);
+    sf = lock_user(VERIFY_WRITE, sf_addr,
+                   sizeof(struct target_signal_frame), 0);
+    if (!sf) {
+        goto sigsegv;
+    }
 #if 0
-	if (invalid_frame_pointer(sf, sigframe_size))
-		goto sigill_and_return;
+    if (invalid_frame_pointer(sf, sigframe_size))
+        goto sigill_and_return;
 #endif
-	/* 2. Save the current process state */
-	err = setup___siginfo(&sf->info, env, set->sig[0]);
+    /* 2. Save the current process state */
+    err = setup___siginfo(&sf->info, env, set->sig[0]);
     __put_user(0, &sf->extra_size);
 
-	//save_fpu_state(regs, &sf->fpu_state);
-	//__put_user(&sf->fpu_state, &sf->fpu_save);
+    //save_fpu_state(regs, &sf->fpu_state);
+    //__put_user(&sf->fpu_state, &sf->fpu_save);
 
     __put_user(set->sig[0], &sf->info.si_mask);
-	for (i = 0; i < TARGET_NSIG_WORDS - 1; i++) {
+    for (i = 0; i < TARGET_NSIG_WORDS - 1; i++) {
         __put_user(set->sig[i + 1], &sf->extramask[i]);
-	}
+    }
 
-	for (i = 0; i < 8; i++) {
+    for (i = 0; i < 8; i++) {
         __put_user(env->regwptr[i + UREG_L0], &sf->ss.locals[i]);
-	}
-	for (i = 0; i < 8; i++) {
+    }
+    for (i = 0; i < 8; i++) {
         __put_user(env->regwptr[i + UREG_I0], &sf->ss.ins[i]);
-	}
-	if (err)
-		goto sigsegv;
+    }
+    if (err)
+        goto sigsegv;
 
-	/* 3. signal handler back-trampoline and parameters */
-	env->regwptr[UREG_FP] = sf_addr;
-	env->regwptr[UREG_I0] = sig;
-	env->regwptr[UREG_I1] = sf_addr + 
-                offsetof(struct target_signal_frame, info);
-	env->regwptr[UREG_I2] = sf_addr + 
-                offsetof(struct target_signal_frame, info);
+    /* 3. signal handler back-trampoline and parameters */
+    env->regwptr[UREG_FP] = sf_addr;
+    env->regwptr[UREG_I0] = sig;
+    env->regwptr[UREG_I1] = sf_addr +
+            offsetof(struct target_signal_frame, info);
+    env->regwptr[UREG_I2] = sf_addr +
+            offsetof(struct target_signal_frame, info);
 
-	/* 4. signal handler */
-	env->pc = ka->_sa_handler;
-	env->npc = (env->pc + 4);
-	/* 5. return to kernel instructions */
-	if (ka->sa_restorer)
-		env->regwptr[UREG_I7] = ka->sa_restorer;
-	else {
-                uint32_t val32;
+    /* 4. signal handler */
+    env->pc = ka->_sa_handler;
+    env->npc = (env->pc + 4);
+    /* 5. return to kernel instructions */
+    if (ka->sa_restorer) {
+        env->regwptr[UREG_I7] = ka->sa_restorer;
+    } else {
+        uint32_t val32;
 
-		env->regwptr[UREG_I7] = sf_addr + 
-                        offsetof(struct target_signal_frame, insns) - 2 * 4;
+        env->regwptr[UREG_I7] = sf_addr +
+                offsetof(struct target_signal_frame, insns) - 2 * 4;
 
-		/* mov __NR_sigreturn, %g1 */
-                val32 = 0x821020d8;
+        /* mov __NR_sigreturn, %g1 */
+        val32 = 0x821020d8;
         __put_user(val32, &sf->insns[0]);
 
-		/* t 0x10 */
-                val32 = 0x91d02010;
+        /* t 0x10 */
+        val32 = 0x91d02010;
         __put_user(val32, &sf->insns[1]);
-		if (err)
-			goto sigsegv;
+        if (err)
+            goto sigsegv;
 
-		/* Flush instruction space. */
-		//flush_sig_insns(current->mm, (unsigned long) &(sf->insns[0]));
-                //		tb_flush(env);
-	}
-        unlock_user(sf, sf_addr, sizeof(struct target_signal_frame));
-	return;
+        /* Flush instruction space. */
+        // flush_sig_insns(current->mm, (unsigned long) &(sf->insns[0]));
+        // tb_flush(env);
+    }
+    unlock_user(sf, sf_addr, sizeof(struct target_signal_frame));
+    return;
 #if 0
 sigill_and_return:
-	force_sig(TARGET_SIGILL);
+    force_sig(TARGET_SIGILL);
 #endif
 sigsegv:
-	//fprintf(stderr, "force_sig\n");
-        unlock_user(sf, sf_addr, sizeof(struct target_signal_frame));
-	force_sig(TARGET_SIGSEGV);
+    unlock_user(sf, sf_addr, sizeof(struct target_signal_frame));
+    force_sig(TARGET_SIGSEGV);
 }
 
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
@@ -2384,79 +2457,79 @@
 
 long do_sigreturn(CPUSPARCState *env)
 {
-        abi_ulong sf_addr;
-        struct target_signal_frame *sf;
-        uint32_t up_psr, pc, npc;
-        target_sigset_t set;
-        sigset_t host_set;
-        int err=0, i;
+    abi_ulong sf_addr;
+    struct target_signal_frame *sf;
+    uint32_t up_psr, pc, npc;
+    target_sigset_t set;
+    sigset_t host_set;
+    int err=0, i;
 
-        sf_addr = env->regwptr[UREG_FP];
-        if (!lock_user_struct(VERIFY_READ, sf, sf_addr, 1))
-                goto segv_and_exit;
-#if 0
-	fprintf(stderr, "sigreturn\n");
-	fprintf(stderr, "sf: %x pc %x fp %x sp %x\n", sf, env->pc, env->regwptr[UREG_FP], env->regwptr[UREG_SP]);
-#endif
-	//cpu_dump_state(env, stderr, fprintf, 0);
+    sf_addr = env->regwptr[UREG_FP];
+    trace_user_do_sigreturn(env, sf_addr);
+    if (!lock_user_struct(VERIFY_READ, sf, sf_addr, 1)) {
+        goto segv_and_exit;
+    }
 
-        /* 1. Make sure we are not getting garbage from the user */
+    /* 1. Make sure we are not getting garbage from the user */
 
-        if (sf_addr & 3)
-                goto segv_and_exit;
+    if (sf_addr & 3)
+        goto segv_and_exit;
 
-        __get_user(pc,  &sf->info.si_regs.pc);
-        __get_user(npc, &sf->info.si_regs.npc);
+    __get_user(pc,  &sf->info.si_regs.pc);
+    __get_user(npc, &sf->info.si_regs.npc);
 
-        if ((pc | npc) & 3)
-                goto segv_and_exit;
+    if ((pc | npc) & 3) {
+        goto segv_and_exit;
+    }
 
-        /* 2. Restore the state */
-        __get_user(up_psr, &sf->info.si_regs.psr);
+    /* 2. Restore the state */
+    __get_user(up_psr, &sf->info.si_regs.psr);
 
-        /* User can only change condition codes and FPU enabling in %psr. */
-        env->psr = (up_psr & (PSR_ICC /* | PSR_EF */))
-                  | (env->psr & ~(PSR_ICC /* | PSR_EF */));
+    /* User can only change condition codes and FPU enabling in %psr. */
+    env->psr = (up_psr & (PSR_ICC /* | PSR_EF */))
+            | (env->psr & ~(PSR_ICC /* | PSR_EF */));
 
-	env->pc = pc;
-	env->npc = npc;
-        __get_user(env->y, &sf->info.si_regs.y);
-	for (i=0; i < 8; i++) {
-		__get_user(env->gregs[i], &sf->info.si_regs.u_regs[i]);
-	}
-	for (i=0; i < 8; i++) {
-		__get_user(env->regwptr[i + UREG_I0], &sf->info.si_regs.u_regs[i+8]);
-	}
+    env->pc = pc;
+    env->npc = npc;
+    __get_user(env->y, &sf->info.si_regs.y);
+    for (i=0; i < 8; i++) {
+        __get_user(env->gregs[i], &sf->info.si_regs.u_regs[i]);
+    }
+    for (i=0; i < 8; i++) {
+        __get_user(env->regwptr[i + UREG_I0], &sf->info.si_regs.u_regs[i+8]);
+    }
 
-        /* FIXME: implement FPU save/restore:
+    /* FIXME: implement FPU save/restore:
          * __get_user(fpu_save, &sf->fpu_save);
          * if (fpu_save)
          *        err |= restore_fpu_state(env, fpu_save);
          */
 
-        /* This is pretty much atomic, no amount locking would prevent
+    /* This is pretty much atomic, no amount locking would prevent
          * the races which exist anyways.
          */
-        __get_user(set.sig[0], &sf->info.si_mask);
-        for(i = 1; i < TARGET_NSIG_WORDS; i++) {
-            __get_user(set.sig[i], &sf->extramask[i - 1]);
-        }
+    __get_user(set.sig[0], &sf->info.si_mask);
+    for(i = 1; i < TARGET_NSIG_WORDS; i++) {
+        __get_user(set.sig[i], &sf->extramask[i - 1]);
+    }
 
-        target_to_host_sigset_internal(&host_set, &set);
-        do_sigprocmask(SIG_SETMASK, &host_set, NULL);
+    target_to_host_sigset_internal(&host_set, &set);
+    set_sigmask(&host_set);
 
-        if (err)
-                goto segv_and_exit;
-        unlock_user_struct(sf, sf_addr, 0);
-        return env->regwptr[0];
+    if (err) {
+        goto segv_and_exit;
+    }
+    unlock_user_struct(sf, sf_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 segv_and_exit:
-        unlock_user_struct(sf, sf_addr, 0);
-	force_sig(TARGET_SIGSEGV);
+    unlock_user_struct(sf, sf_addr, 0);
+    force_sig(TARGET_SIGSEGV);
 }
 
 long do_rt_sigreturn(CPUSPARCState *env)
 {
+    trace_user_do_rt_sigreturn(env, 0);
     fprintf(stderr, "do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
@@ -2540,13 +2613,15 @@
     unsigned int i;
 
     ucp_addr = env->regwptr[UREG_I0];
-    if (!lock_user_struct(VERIFY_READ, ucp, ucp_addr, 1))
+    if (!lock_user_struct(VERIFY_READ, ucp, ucp_addr, 1)) {
         goto do_sigsegv;
+    }
     grp  = &ucp->tuc_mcontext.mc_gregs;
     __get_user(pc, &((*grp)[MC_PC]));
     __get_user(npc, &((*grp)[MC_NPC]));
-    if ((pc | npc) & 3)
+    if ((pc | npc) & 3) {
         goto do_sigsegv;
+    }
     if (env->regwptr[UREG_I1]) {
         target_sigset_t target_set;
         sigset_t set;
@@ -2562,7 +2637,7 @@
             }
         }
         target_to_host_sigset_internal(&set, &target_set);
-        do_sigprocmask(SIG_SETMASK, &set, NULL);
+        set_sigmask(&set);
     }
     env->pc = pc;
     env->npc = npc;
@@ -2591,12 +2666,14 @@
     __get_user(i7, &(ucp->tuc_mcontext.mc_i7));
 
     w_addr = TARGET_STACK_BIAS+env->regwptr[UREG_I6];
-    if (put_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]), 
-                 abi_ulong) != 0)
+    if (put_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]),
+                 abi_ulong) != 0) {
         goto do_sigsegv;
-    if (put_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]), 
-                 abi_ulong) != 0)
+    }
+    if (put_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]),
+                 abi_ulong) != 0) {
         goto do_sigsegv;
+    }
     /* FIXME this does not match how the kernel handles the FPU in
      * its sparc64_set_context implementation. In particular the FPU
      * is only restored if fenab is non-zero in:
@@ -2619,7 +2696,7 @@
                &(ucp->tuc_mcontext.mc_fpregs.mcfpu_gsr));
     unlock_user_struct(ucp, ucp_addr, 0);
     return;
- do_sigsegv:
+do_sigsegv:
     unlock_user_struct(ucp, ucp_addr, 0);
     force_sig(TARGET_SIGSEGV);
 }
@@ -2637,8 +2714,9 @@
     sigset_t set;
 
     ucp_addr = env->regwptr[UREG_I0];
-    if (!lock_user_struct(VERIFY_WRITE, ucp, ucp_addr, 0))
+    if (!lock_user_struct(VERIFY_WRITE, ucp, ucp_addr, 0)) {
         goto do_sigsegv;
+    }
     
     mcp = &ucp->tuc_mcontext;
     grp = &mcp->mc_gregs;
@@ -2647,9 +2725,13 @@
     env->pc = env->npc;
     env->npc += 4;
 
-    err = 0;
-
-    do_sigprocmask(0, NULL, &set);
+    /* If we're only reading the signal mask then do_sigprocmask()
+     * is guaranteed not to fail, which is important because we don't
+     * have any way to signal a failure or restart this operation since
+     * this is not a normal syscall.
+     */
+    err = do_sigprocmask(0, NULL, &set);
+    assert(err == 0);
     host_to_target_sigset_internal(&target_set, &set);
     if (TARGET_NSIG_WORDS == 1) {
         __put_user(target_set.sig[0],
@@ -2688,12 +2770,14 @@
 
     w_addr = TARGET_STACK_BIAS+env->regwptr[UREG_I6];
     fp = i7 = 0;
-    if (get_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]), 
-                 abi_ulong) != 0)
+    if (get_user(fp, w_addr + offsetof(struct target_reg_window, ins[6]),
+                 abi_ulong) != 0) {
         goto do_sigsegv;
-    if (get_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]), 
-                 abi_ulong) != 0)
+    }
+    if (get_user(i7, w_addr + offsetof(struct target_reg_window, ins[7]),
+                 abi_ulong) != 0) {
         goto do_sigsegv;
+    }
     __put_user(fp, &(mcp->mc_fp));
     __put_user(i7, &(mcp->mc_i7));
 
@@ -2715,7 +2799,7 @@
         goto do_sigsegv;
     unlock_user_struct(ucp, ucp_addr, 1);
     return;
- do_sigsegv:
+do_sigsegv:
     unlock_user_struct(ucp, ucp_addr, 1);
     force_sig(TARGET_SIGSEGV);
 }
@@ -2805,7 +2889,7 @@
 }
 
 static inline void setup_sigcontext(CPUMIPSState *regs,
-        struct target_sigcontext *sc)
+                                    struct target_sigcontext *sc)
 {
     int i;
 
@@ -2916,8 +3000,10 @@
     int i;
 
     frame_addr = get_sigframe(ka, regs, sizeof(*frame));
-    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-	goto give_sigsegv;
+    trace_user_setup_frame(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
 
     install_sigtramp(frame->sf_code, TARGET_NR_sigreturn);
 
@@ -2962,19 +3048,17 @@
     target_sigset_t target_set;
     int i;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_sigreturn\n");
-#endif
     frame_addr = regs->active_tc.gpr[29];
+    trace_user_do_sigreturn(regs, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-   	goto badframe;
+        goto badframe;
 
     for(i = 0; i < TARGET_NSIG_WORDS; i++) {
         __get_user(target_set.sig[i], &frame->sf_mask.sig[i]);
     }
 
     target_to_host_sigset_internal(&blocked, &target_set);
-    do_sigprocmask(SIG_SETMASK, &blocked, NULL);
+    set_sigmask(&blocked);
 
     restore_sigcontext(regs, &frame->sf_sc);
 
@@ -3012,12 +3096,14 @@
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
-    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-	goto give_sigsegv;
+    trace_user_setup_rt_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
 
     install_sigtramp(frame->rs_code, TARGET_NR_rt_sigreturn);
 
-    copy_siginfo_to_user(&frame->rs_info, info);
+    tswap_siginfo(&frame->rs_info, info);
 
     __put_user(0, &frame->rs_uc.tuc_flags);
     __put_user(0, &frame->rs_uc.tuc_link);
@@ -3069,21 +3155,20 @@
     abi_ulong frame_addr;
     sigset_t blocked;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_rt_sigreturn\n");
-#endif
     frame_addr = env->active_tc.gpr[29];
-    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-   	goto badframe;
+    trace_user_do_rt_sigreturn(env, frame_addr);
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
     target_to_host_sigset(&blocked, &frame->rs_uc.tuc_sigmask);
-    do_sigprocmask(SIG_SETMASK, &blocked, NULL);
+    set_sigmask(&blocked);
 
     restore_sigcontext(env, &frame->rs_uc.tuc_mcontext);
 
     if (do_sigaltstack(frame_addr +
-		       offsetof(struct target_rt_sigframe, rs_uc.tuc_stack),
-		       0, get_sp_from_cpustate(env)) == -EFAULT)
+                       offsetof(struct target_rt_sigframe, rs_uc.tuc_stack),
+                       0, get_sp_from_cpustate(env)) == -EFAULT)
         goto badframe;
 
     env->active_tc.PC = env->CP0_EPC;
@@ -3154,7 +3239,7 @@
 #define TRAP_NOARG 0xc310         /* Syscall w/no args (NR in R3) SH3/4 */
 
 static abi_ulong get_sigframe(struct target_sigaction *ka,
-                         unsigned long sp, size_t frame_size)
+                              unsigned long sp, size_t frame_size)
 {
     if ((ka->sa_flags & TARGET_SA_ONSTACK) && (sas_ss_flags(sp) == 0)) {
         sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
@@ -3164,7 +3249,7 @@
 }
 
 static void setup_sigcontext(struct target_sigcontext *sc,
-                            CPUSH4State *regs, unsigned long mask)
+                             CPUSH4State *regs, unsigned long mask)
 {
     int i;
 
@@ -3192,13 +3277,12 @@
     __put_user(mask, &sc->oldmask);
 }
 
-static void restore_sigcontext(CPUSH4State *regs, struct target_sigcontext *sc,
-                              target_ulong *r0_p)
+static void restore_sigcontext(CPUSH4State *regs, struct target_sigcontext *sc)
 {
     int i;
 
 #define COPY(x)         __get_user(regs->x, &sc->sc_##x)
-    COPY(gregs[1]);
+    COPY(gregs[0]); COPY(gregs[1]);
     COPY(gregs[2]); COPY(gregs[3]);
     COPY(gregs[4]); COPY(gregs[5]);
     COPY(gregs[6]); COPY(gregs[7]);
@@ -3218,7 +3302,6 @@
     __get_user(regs->fpul, &sc->sc_fpul);
 
     regs->tra = -1;         /* disable syscall checks */
-    __get_user(*r0_p, &sc->sc_gregs[0]);
 }
 
 static void setup_frame(int sig, struct target_sigaction *ka,
@@ -3227,14 +3310,12 @@
     struct target_sigframe *frame;
     abi_ulong frame_addr;
     int i;
-    int err = 0;
-    int signal;
 
     frame_addr = get_sigframe(ka, regs->gregs[15], sizeof(*frame));
-    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-	goto give_sigsegv;
-
-    signal = current_exec_domain_sig(sig);
+    trace_user_setup_frame(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
 
     setup_sigcontext(&frame->sc, regs, set->sig[0]);
 
@@ -3248,18 +3329,17 @@
         regs->pr = (unsigned long) ka->sa_restorer;
     } else {
         /* Generate return code (system call to sigreturn) */
+        abi_ulong retcode_addr = frame_addr +
+                                 offsetof(struct target_sigframe, retcode);
         __put_user(MOVW(2), &frame->retcode[0]);
         __put_user(TRAP_NOARG, &frame->retcode[1]);
         __put_user((TARGET_NR_sigreturn), &frame->retcode[2]);
-        regs->pr = (unsigned long) frame->retcode;
+        regs->pr = (unsigned long) retcode_addr;
     }
 
-    if (err)
-        goto give_sigsegv;
-
     /* Set up registers for signal handler */
     regs->gregs[15] = frame_addr;
-    regs->gregs[4] = signal; /* Arg for signal handler */
+    regs->gregs[4] = sig; /* Arg for signal handler */
     regs->gregs[5] = 0;
     regs->gregs[6] = frame_addr += offsetof(typeof(*frame), sc);
     regs->pc = (unsigned long) ka->_sa_handler;
@@ -3279,16 +3359,14 @@
     struct target_rt_sigframe *frame;
     abi_ulong frame_addr;
     int i;
-    int err = 0;
-    int signal;
 
     frame_addr = get_sigframe(ka, regs->gregs[15], sizeof(*frame));
-    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-	goto give_sigsegv;
+    trace_user_setup_rt_frame(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
 
-    signal = current_exec_domain_sig(sig);
-
-    copy_siginfo_to_user(&frame->info, info);
+    tswap_siginfo(&frame->info, info);
 
     /* Create the ucontext.  */
     __put_user(0, &frame->uc.tuc_flags);
@@ -3300,7 +3378,7 @@
     __put_user(target_sigaltstack_used.ss_size,
                &frame->uc.tuc_stack.ss_size);
     setup_sigcontext(&frame->uc.tuc_mcontext,
-			    regs, set->sig[0]);
+                     regs, set->sig[0]);
     for(i = 0; i < TARGET_NSIG_WORDS; i++) {
         __put_user(set->sig[i], &frame->uc.tuc_sigmask.sig[i]);
     }
@@ -3311,18 +3389,17 @@
         regs->pr = (unsigned long) ka->sa_restorer;
     } else {
         /* Generate return code (system call to sigreturn) */
+        abi_ulong retcode_addr = frame_addr +
+                                 offsetof(struct target_rt_sigframe, retcode);
         __put_user(MOVW(2), &frame->retcode[0]);
         __put_user(TRAP_NOARG, &frame->retcode[1]);
         __put_user((TARGET_NR_rt_sigreturn), &frame->retcode[2]);
-        regs->pr = (unsigned long) frame->retcode;
+        regs->pr = (unsigned long) retcode_addr;
     }
 
-    if (err)
-        goto give_sigsegv;
-
     /* Set up registers for signal handler */
     regs->gregs[15] = frame_addr;
-    regs->gregs[4] = signal; /* Arg for signal handler */
+    regs->gregs[4] = sig; /* Arg for signal handler */
     regs->gregs[5] = frame_addr + offsetof(typeof(*frame), info);
     regs->gregs[6] = frame_addr + offsetof(typeof(*frame), uc);
     regs->pc = (unsigned long) ka->_sa_handler;
@@ -3341,16 +3418,14 @@
     abi_ulong frame_addr;
     sigset_t blocked;
     target_sigset_t target_set;
-    target_ulong r0;
     int i;
     int err = 0;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_sigreturn\n");
-#endif
     frame_addr = regs->gregs[15];
-    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-   	goto badframe;
+    trace_user_do_sigreturn(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
     __get_user(target_set.sig[0], &frame->sc.oldmask);
     for(i = 1; i < TARGET_NSIG_WORDS; i++) {
@@ -3361,12 +3436,12 @@
         goto badframe;
 
     target_to_host_sigset_internal(&blocked, &target_set);
-    do_sigprocmask(SIG_SETMASK, &blocked, NULL);
+    set_sigmask(&blocked);
 
-    restore_sigcontext(regs, &frame->sc, &r0);
+    restore_sigcontext(regs, &frame->sc);
 
     unlock_user_struct(frame, frame_addr, 0);
-    return r0;
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     unlock_user_struct(frame, frame_addr, 0);
@@ -3379,27 +3454,26 @@
     struct target_rt_sigframe *frame;
     abi_ulong frame_addr;
     sigset_t blocked;
-    target_ulong r0;
 
-#if defined(DEBUG_SIGNAL)
-    fprintf(stderr, "do_rt_sigreturn\n");
-#endif
     frame_addr = regs->gregs[15];
-    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
-   	goto badframe;
+    trace_user_do_rt_sigreturn(regs, frame_addr);
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
     target_to_host_sigset(&blocked, &frame->uc.tuc_sigmask);
-    do_sigprocmask(SIG_SETMASK, &blocked, NULL);
+    set_sigmask(&blocked);
 
-    restore_sigcontext(regs, &frame->uc.tuc_mcontext, &r0);
+    restore_sigcontext(regs, &frame->uc.tuc_mcontext);
 
     if (do_sigaltstack(frame_addr +
-		       offsetof(struct target_rt_sigframe, uc.tuc_stack),
-		       0, get_sp_from_cpustate(regs)) == -EFAULT)
+                       offsetof(struct target_rt_sigframe, uc.tuc_stack),
+                       0, get_sp_from_cpustate(regs)) == -EFAULT) {
         goto badframe;
+    }
 
     unlock_user_struct(frame, frame_addr, 0);
-    return r0;
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     unlock_user_struct(frame, frame_addr, 0);
@@ -3534,6 +3608,7 @@
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
         goto badframe;
 
@@ -3563,7 +3638,8 @@
 
         /* Return from sighandler will jump to the tramp.
            Negative 8 offset because return is rtsd r15, 8 */
-        env->regs[15] = ((unsigned long)frame->tramp) - 8;
+        env->regs[15] = frame_addr + offsetof(struct target_signal_frame, tramp)
+                                   - 8;
     }
 
     /* Set up registers for signal handler */
@@ -3579,7 +3655,7 @@
 
     unlock_user_struct(frame, frame_addr, 1);
     return;
-  badframe:
+badframe:
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -3599,6 +3675,7 @@
     int i;
 
     frame_addr = env->regs[R_SP];
+    trace_user_do_sigreturn(env, frame_addr);
     /* Make sure the guest isn't playing games.  */
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
         goto badframe;
@@ -3606,24 +3683,25 @@
     /* Restore blocked signals */
     __get_user(target_set.sig[0], &frame->uc.tuc_mcontext.oldmask);
     for(i = 1; i < TARGET_NSIG_WORDS; i++) {
-       __get_user(target_set.sig[i], &frame->extramask[i - 1]);
+        __get_user(target_set.sig[i], &frame->extramask[i - 1]);
     }
     target_to_host_sigset_internal(&set, &target_set);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     restore_sigcontext(&frame->uc.tuc_mcontext, env);
     /* We got here through a sigreturn syscall, our path back is via an
        rtb insn so setup r14 for that.  */
     env->regs[14] = env->sregs[SR_PC];
- 
+
     unlock_user_struct(frame, frame_addr, 0);
-    return env->regs[10];
-  badframe:
+    return -TARGET_QEMU_ESIGRETURN;
+badframe:
     force_sig(TARGET_SIGSEGV);
 }
 
 long do_rt_sigreturn(CPUMBState *env)
 {
+    trace_user_do_rt_sigreturn(env, 0);
     fprintf(stderr, "Microblaze do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
@@ -3631,123 +3709,124 @@
 #elif defined(TARGET_CRIS)
 
 struct target_sigcontext {
-        struct target_pt_regs regs;  /* needs to be first */
-        uint32_t oldmask;
-        uint32_t usp;    /* usp before stacking this gunk on it */
+    struct target_pt_regs regs;  /* needs to be first */
+    uint32_t oldmask;
+    uint32_t usp;    /* usp before stacking this gunk on it */
 };
 
 /* Signal frames. */
 struct target_signal_frame {
-        struct target_sigcontext sc;
-        uint32_t extramask[TARGET_NSIG_WORDS - 1];
-        uint16_t retcode[4];      /* Trampoline code. */
+    struct target_sigcontext sc;
+    uint32_t extramask[TARGET_NSIG_WORDS - 1];
+    uint16_t retcode[4];      /* Trampoline code. */
 };
 
 struct rt_signal_frame {
-        siginfo_t *pinfo;
-        void *puc;
-        siginfo_t info;
-        struct ucontext uc;
-        uint16_t retcode[4];      /* Trampoline code. */
+    siginfo_t *pinfo;
+    void *puc;
+    siginfo_t info;
+    struct ucontext uc;
+    uint16_t retcode[4];      /* Trampoline code. */
 };
 
 static void setup_sigcontext(struct target_sigcontext *sc, CPUCRISState *env)
 {
-	__put_user(env->regs[0], &sc->regs.r0);
-	__put_user(env->regs[1], &sc->regs.r1);
-	__put_user(env->regs[2], &sc->regs.r2);
-	__put_user(env->regs[3], &sc->regs.r3);
-	__put_user(env->regs[4], &sc->regs.r4);
-	__put_user(env->regs[5], &sc->regs.r5);
-	__put_user(env->regs[6], &sc->regs.r6);
-	__put_user(env->regs[7], &sc->regs.r7);
-	__put_user(env->regs[8], &sc->regs.r8);
-	__put_user(env->regs[9], &sc->regs.r9);
-	__put_user(env->regs[10], &sc->regs.r10);
-	__put_user(env->regs[11], &sc->regs.r11);
-	__put_user(env->regs[12], &sc->regs.r12);
-	__put_user(env->regs[13], &sc->regs.r13);
-	__put_user(env->regs[14], &sc->usp);
-	__put_user(env->regs[15], &sc->regs.acr);
-	__put_user(env->pregs[PR_MOF], &sc->regs.mof);
-	__put_user(env->pregs[PR_SRP], &sc->regs.srp);
-	__put_user(env->pc, &sc->regs.erp);
+    __put_user(env->regs[0], &sc->regs.r0);
+    __put_user(env->regs[1], &sc->regs.r1);
+    __put_user(env->regs[2], &sc->regs.r2);
+    __put_user(env->regs[3], &sc->regs.r3);
+    __put_user(env->regs[4], &sc->regs.r4);
+    __put_user(env->regs[5], &sc->regs.r5);
+    __put_user(env->regs[6], &sc->regs.r6);
+    __put_user(env->regs[7], &sc->regs.r7);
+    __put_user(env->regs[8], &sc->regs.r8);
+    __put_user(env->regs[9], &sc->regs.r9);
+    __put_user(env->regs[10], &sc->regs.r10);
+    __put_user(env->regs[11], &sc->regs.r11);
+    __put_user(env->regs[12], &sc->regs.r12);
+    __put_user(env->regs[13], &sc->regs.r13);
+    __put_user(env->regs[14], &sc->usp);
+    __put_user(env->regs[15], &sc->regs.acr);
+    __put_user(env->pregs[PR_MOF], &sc->regs.mof);
+    __put_user(env->pregs[PR_SRP], &sc->regs.srp);
+    __put_user(env->pc, &sc->regs.erp);
 }
 
 static void restore_sigcontext(struct target_sigcontext *sc, CPUCRISState *env)
 {
-	__get_user(env->regs[0], &sc->regs.r0);
-	__get_user(env->regs[1], &sc->regs.r1);
-	__get_user(env->regs[2], &sc->regs.r2);
-	__get_user(env->regs[3], &sc->regs.r3);
-	__get_user(env->regs[4], &sc->regs.r4);
-	__get_user(env->regs[5], &sc->regs.r5);
-	__get_user(env->regs[6], &sc->regs.r6);
-	__get_user(env->regs[7], &sc->regs.r7);
-	__get_user(env->regs[8], &sc->regs.r8);
-	__get_user(env->regs[9], &sc->regs.r9);
-	__get_user(env->regs[10], &sc->regs.r10);
-	__get_user(env->regs[11], &sc->regs.r11);
-	__get_user(env->regs[12], &sc->regs.r12);
-	__get_user(env->regs[13], &sc->regs.r13);
-	__get_user(env->regs[14], &sc->usp);
-	__get_user(env->regs[15], &sc->regs.acr);
-	__get_user(env->pregs[PR_MOF], &sc->regs.mof);
-	__get_user(env->pregs[PR_SRP], &sc->regs.srp);
-	__get_user(env->pc, &sc->regs.erp);
+    __get_user(env->regs[0], &sc->regs.r0);
+    __get_user(env->regs[1], &sc->regs.r1);
+    __get_user(env->regs[2], &sc->regs.r2);
+    __get_user(env->regs[3], &sc->regs.r3);
+    __get_user(env->regs[4], &sc->regs.r4);
+    __get_user(env->regs[5], &sc->regs.r5);
+    __get_user(env->regs[6], &sc->regs.r6);
+    __get_user(env->regs[7], &sc->regs.r7);
+    __get_user(env->regs[8], &sc->regs.r8);
+    __get_user(env->regs[9], &sc->regs.r9);
+    __get_user(env->regs[10], &sc->regs.r10);
+    __get_user(env->regs[11], &sc->regs.r11);
+    __get_user(env->regs[12], &sc->regs.r12);
+    __get_user(env->regs[13], &sc->regs.r13);
+    __get_user(env->regs[14], &sc->usp);
+    __get_user(env->regs[15], &sc->regs.acr);
+    __get_user(env->pregs[PR_MOF], &sc->regs.mof);
+    __get_user(env->pregs[PR_SRP], &sc->regs.srp);
+    __get_user(env->pc, &sc->regs.erp);
 }
 
 static abi_ulong get_sigframe(CPUCRISState *env, int framesize)
 {
-	abi_ulong sp;
-	/* Align the stack downwards to 4.  */
-	sp = (env->regs[R_SP] & ~3);
-	return sp - framesize;
+    abi_ulong sp;
+    /* Align the stack downwards to 4.  */
+    sp = (env->regs[R_SP] & ~3);
+    return sp - framesize;
 }
 
 static void setup_frame(int sig, struct target_sigaction *ka,
                         target_sigset_t *set, CPUCRISState *env)
 {
-	struct target_signal_frame *frame;
-	abi_ulong frame_addr;
-	int i;
+    struct target_signal_frame *frame;
+    abi_ulong frame_addr;
+    int i;
 
-	frame_addr = get_sigframe(env, sizeof *frame);
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-		goto badframe;
+    frame_addr = get_sigframe(env, sizeof *frame);
+    trace_user_setup_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
+        goto badframe;
 
-	/*
-	 * The CRIS signal return trampoline. A real linux/CRIS kernel doesn't
-	 * use this trampoline anymore but it sets it up for GDB.
-	 * In QEMU, using the trampoline simplifies things a bit so we use it.
-	 *
-	 * This is movu.w __NR_sigreturn, r9; break 13;
-	 */
+    /*
+     * The CRIS signal return trampoline. A real linux/CRIS kernel doesn't
+     * use this trampoline anymore but it sets it up for GDB.
+     * In QEMU, using the trampoline simplifies things a bit so we use it.
+     *
+     * This is movu.w __NR_sigreturn, r9; break 13;
+     */
     __put_user(0x9c5f, frame->retcode+0);
     __put_user(TARGET_NR_sigreturn,
                frame->retcode + 1);
     __put_user(0xe93d, frame->retcode + 2);
 
-	/* Save the mask.  */
+    /* Save the mask.  */
     __put_user(set->sig[0], &frame->sc.oldmask);
 
     for(i = 1; i < TARGET_NSIG_WORDS; i++) {
         __put_user(set->sig[i], &frame->extramask[i - 1]);
     }
 
-	setup_sigcontext(&frame->sc, env);
+    setup_sigcontext(&frame->sc, env);
 
-	/* Move the stack and setup the arguments for the handler.  */
-	env->regs[R_SP] = frame_addr;
-	env->regs[10] = sig;
-	env->pc = (unsigned long) ka->_sa_handler;
-	/* Link SRP so the guest returns through the trampoline.  */
-	env->pregs[PR_SRP] = frame_addr + offsetof(typeof(*frame), retcode);
+    /* Move the stack and setup the arguments for the handler.  */
+    env->regs[R_SP] = frame_addr;
+    env->regs[10] = sig;
+    env->pc = (unsigned long) ka->_sa_handler;
+    /* Link SRP so the guest returns through the trampoline.  */
+    env->pregs[PR_SRP] = frame_addr + offsetof(typeof(*frame), retcode);
 
-	unlock_user_struct(frame, frame_addr, 1);
-	return;
-  badframe:
-	force_sig(TARGET_SIGSEGV);
+    unlock_user_struct(frame, frame_addr, 1);
+    return;
+badframe:
+    force_sig(TARGET_SIGSEGV);
 }
 
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
@@ -3759,34 +3838,37 @@
 
 long do_sigreturn(CPUCRISState *env)
 {
-	struct target_signal_frame *frame;
-	abi_ulong frame_addr;
-	target_sigset_t target_set;
-	sigset_t set;
-	int i;
+    struct target_signal_frame *frame;
+    abi_ulong frame_addr;
+    target_sigset_t target_set;
+    sigset_t set;
+    int i;
 
-	frame_addr = env->regs[R_SP];
-	/* Make sure the guest isn't playing games.  */
-	if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
-		goto badframe;
+    frame_addr = env->regs[R_SP];
+    trace_user_do_sigreturn(env, frame_addr);
+    /* Make sure the guest isn't playing games.  */
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1)) {
+        goto badframe;
+    }
 
-	/* Restore blocked signals */
+    /* Restore blocked signals */
     __get_user(target_set.sig[0], &frame->sc.oldmask);
-	for(i = 1; i < TARGET_NSIG_WORDS; i++) {
+    for(i = 1; i < TARGET_NSIG_WORDS; i++) {
         __get_user(target_set.sig[i], &frame->extramask[i - 1]);
-	}
-	target_to_host_sigset_internal(&set, &target_set);
-        do_sigprocmask(SIG_SETMASK, &set, NULL);
+    }
+    target_to_host_sigset_internal(&set, &target_set);
+    set_sigmask(&set);
 
-	restore_sigcontext(&frame->sc, env);
-	unlock_user_struct(frame, frame_addr, 0);
-	return env->regs[10];
-  badframe:
-	force_sig(TARGET_SIGSEGV);
+    restore_sigcontext(&frame->sc, env);
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
+badframe:
+    force_sig(TARGET_SIGSEGV);
 }
 
 long do_rt_sigreturn(CPUCRISState *env)
 {
+    trace_user_do_rt_sigreturn(env, 0);
     fprintf(stderr, "CRIS do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
@@ -3867,8 +3949,8 @@
 /* Set up a signal frame.  */
 
 static void setup_sigcontext(struct target_sigcontext *sc,
-                            CPUOpenRISCState *regs,
-                            unsigned long mask)
+                             CPUOpenRISCState *regs,
+                             unsigned long mask)
 {
     unsigned long usp = regs->gpr[1];
 
@@ -3888,9 +3970,7 @@
 
 static inline unsigned long align_sigframe(unsigned long sp)
 {
-    unsigned long i;
-    i = sp & ~3UL;
-    return i;
+    return sp & ~3UL;
 }
 
 static inline abi_ulong get_sigframe(struct target_sigaction *ka,
@@ -3920,12 +4000,6 @@
     return sp;
 }
 
-static void setup_frame(int sig, struct target_sigaction *ka,
-                        target_sigset_t *set, CPUOpenRISCState *env)
-{
-    qemu_log("Not implement.\n");
-}
-
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
                            target_siginfo_t *info,
                            target_sigset_t *set, CPUOpenRISCState *env)
@@ -3937,6 +4011,7 @@
     abi_ulong info_addr, uc_addr;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -3947,7 +4022,7 @@
     __put_user(uc_addr, &frame->puc);
 
     if (ka->sa_flags & SA_SIGINFO) {
-        copy_siginfo_to_user(&frame->info, info);
+        tswap_siginfo(&frame->info, info);
     }
 
     /*err |= __clear_user(&frame->uc, offsetof(struct ucontext, uc_mcontext));*/
@@ -3998,14 +4073,15 @@
 
 long do_sigreturn(CPUOpenRISCState *env)
 {
-
-    qemu_log("do_sigreturn: not implemented\n");
+    trace_user_do_sigreturn(env, 0);
+    fprintf(stderr, "do_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
 
 long do_rt_sigreturn(CPUOpenRISCState *env)
 {
-    qemu_log("do_rt_sigreturn: not implemented\n");
+    trace_user_do_rt_sigreturn(env, 0);
+    fprintf(stderr, "do_rt_sigreturn: not implemented\n");
     return -TARGET_ENOSYS;
 }
 /* TARGET_OPENRISC */
@@ -4117,7 +4193,7 @@
      */
     //save_fp_regs(&current->thread.fp_regs); FIXME
     for (i = 0; i < 16; i++) {
-        __put_user(env->fregs[i].ll, &sregs->fpregs.fprs[i]);
+        __put_user(get_freg(env, i)->ll, &sregs->fpregs.fprs[i]);
     }
 }
 
@@ -4128,13 +4204,11 @@
     abi_ulong frame_addr;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
-            goto give_sigsegv;
+        goto give_sigsegv;
     }
 
-    qemu_log("%s: 1\n", __FUNCTION__);
     __put_user(set->sig[0], &frame->sc.oldmask[0]);
 
     save_sigregs(env, &frame->sregs);
@@ -4145,13 +4219,13 @@
     /* Set up to return from userspace.  If provided, use a stub
        already in userspace.  */
     if (ka->sa_flags & TARGET_SA_RESTORER) {
-            env->regs[14] = (unsigned long)
-                    ka->sa_restorer | PSW_ADDR_AMODE;
+        env->regs[14] = (unsigned long)
+                ka->sa_restorer | PSW_ADDR_AMODE;
     } else {
-            env->regs[14] = (unsigned long)
-                    frame->retcode | PSW_ADDR_AMODE;
-            __put_user(S390_SYSCALL_OPCODE | TARGET_NR_sigreturn,
-                       (uint16_t *)(frame->retcode));
+        env->regs[14] = (frame_addr + offsetof(sigframe, retcode))
+                        | PSW_ADDR_AMODE;
+        __put_user(S390_SYSCALL_OPCODE | TARGET_NR_sigreturn,
+                   (uint16_t *)(frame->retcode));
     }
 
     /* Set up backchain. */
@@ -4175,7 +4249,6 @@
     return;
 
 give_sigsegv:
-    qemu_log("%s: give_sigsegv\n", __FUNCTION__);
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -4188,26 +4261,24 @@
     abi_ulong frame_addr;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
 
-    qemu_log("%s: 1\n", __FUNCTION__);
-    copy_siginfo_to_user(&frame->info, info);
+    tswap_siginfo(&frame->info, info);
 
     /* Create the ucontext.  */
     __put_user(0, &frame->uc.tuc_flags);
     __put_user((abi_ulong)0, (abi_ulong *)&frame->uc.tuc_link);
     __put_user(target_sigaltstack_used.ss_sp, &frame->uc.tuc_stack.ss_sp);
     __put_user(sas_ss_flags(get_sp_from_cpustate(env)),
-                      &frame->uc.tuc_stack.ss_flags);
+               &frame->uc.tuc_stack.ss_flags);
     __put_user(target_sigaltstack_used.ss_size, &frame->uc.tuc_stack.ss_size);
     save_sigregs(env, &frame->uc.tuc_mcontext);
     for (i = 0; i < TARGET_NSIG_WORDS; i++) {
         __put_user((abi_ulong)set->sig[i],
-        (abi_ulong *)&frame->uc.tuc_sigmask.sig[i]);
+                   (abi_ulong *)&frame->uc.tuc_sigmask.sig[i]);
     }
 
     /* Set up to return from userspace.  If provided, use a stub
@@ -4233,7 +4304,6 @@
     return;
 
 give_sigsegv:
-    qemu_log("%s: give_sigsegv\n", __FUNCTION__);
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -4248,9 +4318,8 @@
     }
 
     __get_user(env->psw.mask, &sc->regs.psw.mask);
-    qemu_log("%s: sc->regs.psw.addr 0x%llx env->psw.addr 0x%llx\n",
-             __FUNCTION__, (unsigned long long)sc->regs.psw.addr,
-             (unsigned long long)env->psw.addr);
+    trace_user_s390x_restore_sigregs(env, (unsigned long long)sc->regs.psw.addr,
+                                     (unsigned long long)env->psw.addr);
     __get_user(env->psw.addr, &sc->regs.psw.addr);
     /* FIXME: 31-bit -> | PSW_ADDR_AMODE */
 
@@ -4258,7 +4327,7 @@
         __get_user(env->aregs[i], &sc->regs.acrs[i]);
     }
     for (i = 0; i < 16; i++) {
-        __get_user(env->fregs[i].ll, &sc->fpregs.fprs[i]);
+        __get_user(get_freg(env, i)->ll, &sc->fpregs.fprs[i]);
     }
 
     return err;
@@ -4268,25 +4337,24 @@
 {
     sigframe *frame;
     abi_ulong frame_addr = env->regs[15];
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
     target_sigset_t target_set;
     sigset_t set;
 
+    trace_user_do_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
         goto badframe;
     }
     __get_user(target_set.sig[0], &frame->sc.oldmask[0]);
 
     target_to_host_sigset_internal(&set, &target_set);
-    do_sigprocmask(SIG_SETMASK, &set, NULL); /* ~_BLOCKABLE? */
+    set_sigmask(&set); /* ~_BLOCKABLE? */
 
     if (restore_sigregs(env, &frame->sregs)) {
         goto badframe;
     }
 
     unlock_user_struct(frame, frame_addr, 0);
-    return env->regs[2];
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     force_sig(TARGET_SIGSEGV);
@@ -4297,16 +4365,15 @@
 {
     rt_sigframe *frame;
     abi_ulong frame_addr = env->regs[15];
-    qemu_log("%s: frame_addr 0x%llx\n", __FUNCTION__,
-             (unsigned long long)frame_addr);
     sigset_t set;
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
         goto badframe;
     }
     target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
 
-    do_sigprocmask(SIG_SETMASK, &set, NULL); /* ~_BLOCKABLE? */
+    set_sigmask(&set); /* ~_BLOCKABLE? */
 
     if (restore_sigregs(env, &frame->uc.tuc_mcontext)) {
         goto badframe;
@@ -4317,7 +4384,7 @@
         goto badframe;
     }
     unlock_user_struct(frame, frame_addr, 0);
-    return env->regs[2];
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     unlock_user_struct(frame, frame_addr, 0);
@@ -4462,15 +4529,15 @@
 #define TARGET_TRAMP_SIZE 6
 
 struct target_rt_sigframe {
-        /* sys_rt_sigreturn requires the ucontext be the first field */
-        struct target_ucontext uc;
-        target_ulong  _unused[2];
-        uint32_t trampoline[TARGET_TRAMP_SIZE];
-        target_ulong pinfo; /* struct siginfo __user * */
-        target_ulong puc; /* void __user * */
-        struct target_siginfo info;
-        /* 64 bit ABI allows for 288 bytes below sp before decrementing it. */
-        char abigap[288];
+    /* sys_rt_sigreturn requires the ucontext be the first field */
+    struct target_ucontext uc;
+    target_ulong  _unused[2];
+    uint32_t trampoline[TARGET_TRAMP_SIZE];
+    target_ulong pinfo; /* struct siginfo __user * */
+    target_ulong puc; /* void __user * */
+    struct target_siginfo info;
+    /* 64 bit ABI allows for 288 bytes below sp before decrementing it. */
+    char abigap[288];
 } __attribute__((aligned(16)));
 
 #else
@@ -4500,19 +4567,17 @@
                                  CPUPPCState *env,
                                  int frame_size)
 {
-    target_ulong oldsp, newsp;
+    target_ulong oldsp;
 
     oldsp = env->gpr[1];
 
     if ((ka->sa_flags & TARGET_SA_ONSTACK) &&
-        (sas_ss_flags(oldsp) == 0)) {
+            (sas_ss_flags(oldsp) == 0)) {
         oldsp = (target_sigaltstack_used.ss_sp
                  + target_sigaltstack_used.ss_size);
     }
 
-    newsp = (oldsp - frame_size) & ~0xFUL;
-
-    return newsp;
+    return (oldsp - frame_size) & ~0xFUL;
 }
 
 static void save_user_regs(CPUPPCState *env, struct target_mcontext *frame)
@@ -4527,7 +4592,7 @@
 
     /* Save general registers.  */
     for (i = 0; i < ARRAY_SIZE(env->gpr); i++) {
-       __put_user(env->gpr[i], &frame->mc_gregs[i]);
+        __put_user(env->gpr[i], &frame->mc_gregs[i]);
     }
     __put_user(env->nip, &frame->mc_gregs[TARGET_PT_NIP]);
     __put_user(env->ctr, &frame->mc_gregs[TARGET_PT_CTR]);
@@ -4628,7 +4693,7 @@
 
     /* If doing signal return, restore the previous little-endian mode.  */
     if (sig)
-        env->msr = (env->msr & ~MSR_LE) | (msr & MSR_LE);
+        env->msr = (env->msr & ~(1ull << MSR_LE)) | (msr & (1ull << MSR_LE));
 
     /* Restore Altivec registers if necessary.  */
     if (env->insns_flags & PPC_ALTIVEC) {
@@ -4680,18 +4745,16 @@
     struct target_sigcontext *sc;
     target_ulong frame_addr, newsp;
     int err = 0;
-    int signal;
 #if defined(TARGET_PPC64)
     struct image_info *image = ((TaskState *)thread_cpu->opaque)->info;
 #endif
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 1))
         goto sigsegv;
     sc = &frame->sctx;
 
-    signal = current_exec_domain_sig(sig);
-
     __put_user(ka->_sa_handler, &sc->handler);
     __put_user(set->sig[0], &sc->oldmask);
 #if TARGET_ABI_BITS == 64
@@ -4724,7 +4787,7 @@
 
     /* Set up registers for signal handler.  */
     env->gpr[1] = newsp;
-    env->gpr[3] = signal;
+    env->gpr[3] = sig;
     env->gpr[4] = frame_addr + offsetof(struct target_sigframe, sctx);
 
 #if defined(TARGET_PPC64)
@@ -4745,14 +4808,13 @@
 #endif
 
     /* Signal handlers are entered in big-endian mode.  */
-    env->msr &= ~MSR_LE;
+    env->msr &= ~(1ull << MSR_LE);
 
     unlock_user_struct(frame, frame_addr, 1);
     return;
 
 sigsegv:
     unlock_user_struct(frame, frame_addr, 1);
-    qemu_log("segfaulting from setup_frame\n");
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -4765,7 +4827,6 @@
     struct target_mcontext *mctx = 0;
     target_ulong rt_sf_addr, newsp = 0;
     int i, err = 0;
-    int signal;
 #if defined(TARGET_PPC64)
     struct image_info *image = ((TaskState *)thread_cpu->opaque)->info;
 #endif
@@ -4774,9 +4835,7 @@
     if (!lock_user_struct(VERIFY_WRITE, rt_sf, rt_sf_addr, 1))
         goto sigsegv;
 
-    signal = current_exec_domain_sig(sig);
-
-    copy_siginfo_to_user(&rt_sf->info, info);
+    tswap_siginfo(&rt_sf->info, info);
 
     __put_user(0, &rt_sf->uc.tuc_flags);
     __put_user(0, &rt_sf->uc.tuc_link);
@@ -4821,7 +4880,7 @@
 
     /* Set up registers for signal handler.  */
     env->gpr[1] = newsp;
-    env->gpr[3] = (target_ulong) signal;
+    env->gpr[3] = (target_ulong) sig;
     env->gpr[4] = (target_ulong) h2g(&rt_sf->info);
     env->gpr[5] = (target_ulong) h2g(&rt_sf->uc);
     env->gpr[6] = (target_ulong) h2g(rt_sf);
@@ -4844,14 +4903,13 @@
 #endif
 
     /* Signal handlers are entered in big-endian mode.  */
-    env->msr &= ~MSR_LE;
+    env->msr &= ~(1ull << MSR_LE);
 
     unlock_user_struct(rt_sf, rt_sf_addr, 1);
     return;
 
 sigsegv:
     unlock_user_struct(rt_sf, rt_sf_addr, 1);
-    qemu_log("segfaulting from setup_rt_frame\n");
     force_sig(TARGET_SIGSEGV);
 
 }
@@ -4875,7 +4933,7 @@
     __get_user(set.sig[1], &sc->_unused[3]);
 #endif
     target_to_host_sigset_internal(&blocked, &set);
-    do_sigprocmask(SIG_SETMASK, &blocked, NULL);
+    set_sigmask(&blocked);
 
     __get_user(sr_addr, &sc->regs);
     if (!lock_user_struct(VERIFY_READ, sr, sr_addr, 1))
@@ -4889,7 +4947,6 @@
 sigsegv:
     unlock_user_struct(sr, sr_addr, 1);
     unlock_user_struct(sc, sc_addr, 1);
-    qemu_log("segfaulting from do_sigreturn\n");
     force_sig(TARGET_SIGSEGV);
     return 0;
 }
@@ -4917,7 +4974,7 @@
         return 1;
 
     target_to_host_sigset_internal(&blocked, &set);
-    do_sigprocmask(SIG_SETMASK, &blocked, NULL);
+    set_sigmask(&blocked);
     restore_user_regs(env, mcp, sig);
 
     unlock_user_struct(mcp, mcp_addr, 1);
@@ -4945,7 +5002,6 @@
 
 sigsegv:
     unlock_user_struct(rt_sf, rt_sf_addr, 1);
-    qemu_log("segfaulting from do_rt_sigreturn\n");
     force_sig(TARGET_SIGSEGV);
     return 0;
 }
@@ -4973,7 +5029,7 @@
     abi_ulong extramask[TARGET_NSIG_WORDS-1];
     struct target_sigcontext sc;
 };
- 
+
 typedef int target_greg_t;
 #define TARGET_NGREG 18
 typedef target_greg_t target_gregset_t[TARGET_NGREG];
@@ -5012,7 +5068,7 @@
 };
 
 static void setup_sigcontext(struct target_sigcontext *sc, CPUM68KState *env,
-        abi_ulong mask)
+                             abi_ulong mask)
 {
     __put_user(mask, &sc->sc_mask);
     __put_user(env->aregs[7], &sc->sc_usp);
@@ -5025,19 +5081,18 @@
 }
 
 static void
-restore_sigcontext(CPUM68KState *env, struct target_sigcontext *sc, int *pd0)
+restore_sigcontext(CPUM68KState *env, struct target_sigcontext *sc)
 {
     int temp;
 
     __get_user(env->aregs[7], &sc->sc_usp);
+    __get_user(env->dregs[0], &sc->sc_d0);
     __get_user(env->dregs[1], &sc->sc_d1);
     __get_user(env->aregs[0], &sc->sc_a0);
     __get_user(env->aregs[1], &sc->sc_a1);
     __get_user(env->pc, &sc->sc_pc);
     __get_user(temp, &sc->sc_sr);
     env->sr = (env->sr & 0xff00) | (temp & 0xff);
-
-    *pd0 = tswapl(sc->sc_d0);
 }
 
 /*
@@ -5069,8 +5124,10 @@
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
-    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-	goto give_sigsegv;
+    trace_user_setup_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
 
     __put_user(sig, &frame->sig);
 
@@ -5091,7 +5148,7 @@
     /* moveq #,d0; trap #0 */
 
     __put_user(0x70004e40 + (TARGET_NR_sigreturn << 16),
-                      (long *)(frame->retcode));
+               (uint32_t *)(frame->retcode));
 
     /* Set up to return from userspace */
 
@@ -5132,10 +5189,9 @@
 
     return 0;
 }
- 
+
 static inline int target_rt_restore_ucontext(CPUM68KState *env,
-                                             struct target_ucontext *uc,
-                                             int *pd0)
+                                             struct target_ucontext *uc)
 {
     int temp;
     target_greg_t *gregs = uc->tuc_mcontext.gregs;
@@ -5165,7 +5221,6 @@
     __get_user(temp, &gregs[17]);
     env->sr = (env->sr & 0xff00) | (temp & 0xff);
 
-    *pd0 = env->dregs[0];
     return 0;
 
 badframe:
@@ -5185,8 +5240,10 @@
     int i;
 
     frame_addr = get_sigframe(ka, env, sizeof *frame);
-    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0))
-	goto give_sigsegv;
+    trace_user_setup_rt_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
 
     __put_user(sig, &frame->sig);
 
@@ -5196,7 +5253,7 @@
     uc_addr = frame_addr + offsetof(struct target_rt_sigframe, uc);
     __put_user(uc_addr, &frame->puc);
 
-    copy_siginfo_to_user(&frame->info, info);
+    tswap_siginfo(&frame->info, info);
 
     /* Create the ucontext */
 
@@ -5205,13 +5262,13 @@
     __put_user(target_sigaltstack_used.ss_sp,
                &frame->uc.tuc_stack.ss_sp);
     __put_user(sas_ss_flags(env->aregs[7]),
-               &frame->uc.tuc_stack.ss_flags);
+            &frame->uc.tuc_stack.ss_flags);
     __put_user(target_sigaltstack_used.ss_size,
                &frame->uc.tuc_stack.ss_size);
     err |= target_rt_setup_ucontext(&frame->uc, env);
 
     if (err)
-            goto give_sigsegv;
+        goto give_sigsegv;
 
     for(i = 0; i < TARGET_NSIG_WORDS; i++) {
         __put_user(set->sig[i], &frame->uc.tuc_sigmask.sig[i]);
@@ -5225,8 +5282,8 @@
     /* moveq #,d0; notb d0; trap #0 */
 
     __put_user(0x70004600 + ((TARGET_NR_rt_sigreturn ^ 0xff) << 16),
-               (long *)(frame->retcode + 0));
-    __put_user(0x4e40, (short *)(frame->retcode + 4));
+               (uint32_t *)(frame->retcode + 0));
+    __put_user(0x4e40, (uint16_t *)(frame->retcode + 4));
 
     if (err)
         goto give_sigsegv;
@@ -5250,8 +5307,9 @@
     abi_ulong frame_addr = env->aregs[7] - 4;
     target_sigset_t target_set;
     sigset_t set;
-    int d0, i;
+    int i;
 
+    trace_user_do_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
         goto badframe;
 
@@ -5264,14 +5322,14 @@
     }
 
     target_to_host_sigset_internal(&set, &target_set);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     /* restore registers */
 
-    restore_sigcontext(env, &frame->sc, &d0);
+    restore_sigcontext(env, &frame->sc);
 
     unlock_user_struct(frame, frame_addr, 0);
-    return d0;
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     force_sig(TARGET_SIGSEGV);
@@ -5284,17 +5342,17 @@
     abi_ulong frame_addr = env->aregs[7] - 4;
     target_sigset_t target_set;
     sigset_t set;
-    int d0;
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1))
         goto badframe;
 
     target_to_host_sigset_internal(&set, &target_set);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     /* restore registers */
 
-    if (target_rt_restore_ucontext(env, &frame->uc, &d0))
+    if (target_rt_restore_ucontext(env, &frame->uc))
         goto badframe;
 
     if (do_sigaltstack(frame_addr +
@@ -5303,7 +5361,7 @@
         goto badframe;
 
     unlock_user_struct(frame, frame_addr, 0);
-    return d0;
+    return -TARGET_QEMU_ESIGRETURN;
 
 badframe:
     unlock_user_struct(frame, frame_addr, 0);
@@ -5360,7 +5418,7 @@
 #define INSN_CALLSYS            0x00000083
 
 static void setup_sigcontext(struct target_sigcontext *sc, CPUAlphaState *env,
-                            abi_ulong frame_addr, target_sigset_t *set)
+                             abi_ulong frame_addr, target_sigset_t *set)
 {
     int i;
 
@@ -5386,7 +5444,7 @@
 }
 
 static void restore_sigcontext(CPUAlphaState *env,
-                              struct target_sigcontext *sc)
+                               struct target_sigcontext *sc)
 {
     uint64_t fpcr;
     int i;
@@ -5425,6 +5483,7 @@
     int err = 0;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
@@ -5445,7 +5504,7 @@
     unlock_user_struct(frame, frame_addr, 1);
 
     if (err) {
-    give_sigsegv:
+give_sigsegv:
         if (sig == TARGET_SIGSEGV) {
             ka->_sa_handler = TARGET_SIG_DFL;
         }
@@ -5469,11 +5528,12 @@
     int i, err = 0;
 
     frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
     if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
         goto give_sigsegv;
     }
 
-    copy_siginfo_to_user(&frame->info, info);
+    tswap_siginfo(&frame->info, info);
 
     __put_user(0, &frame->uc.tuc_flags);
     __put_user(0, &frame->uc.tuc_link);
@@ -5501,8 +5561,8 @@
     }
 
     if (err) {
-    give_sigsegv:
-       if (sig == TARGET_SIGSEGV) {
+give_sigsegv:
+        if (sig == TARGET_SIGSEGV) {
             ka->_sa_handler = TARGET_SIG_DFL;
         }
         force_sig(TARGET_SIGSEGV);
@@ -5531,13 +5591,13 @@
     __get_user(target_set.sig[0], &sc->sc_mask);
 
     target_to_host_sigset_internal(&set, &target_set);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     restore_sigcontext(env, sc);
     unlock_user_struct(sc, sc_addr, 0);
-    return env->ir[IR_V0];
+    return -TARGET_QEMU_ESIGRETURN;
 
- badframe:
+badframe:
     force_sig(TARGET_SIGSEGV);
 }
 
@@ -5547,11 +5607,12 @@
     struct target_rt_sigframe *frame;
     sigset_t set;
 
+    trace_user_do_rt_sigreturn(env, frame_addr);
     if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
         goto badframe;
     }
     target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
-    do_sigprocmask(SIG_SETMASK, &set, NULL);
+    set_sigmask(&set);
 
     restore_sigcontext(env, &frame->uc.tuc_mcontext);
     if (do_sigaltstack(frame_addr + offsetof(struct target_rt_sigframe,
@@ -5561,7 +5622,174 @@
     }
 
     unlock_user_struct(frame, frame_addr, 0);
-    return env->ir[IR_V0];
+    return -TARGET_QEMU_ESIGRETURN;
+
+
+badframe:
+    unlock_user_struct(frame, frame_addr, 0);
+    force_sig(TARGET_SIGSEGV);
+}
+
+#elif defined(TARGET_TILEGX)
+
+struct target_sigcontext {
+    union {
+        /* General-purpose registers.  */
+        abi_ulong gregs[56];
+        struct {
+            abi_ulong __gregs[53];
+            abi_ulong tp;        /* Aliases gregs[TREG_TP].  */
+            abi_ulong sp;        /* Aliases gregs[TREG_SP].  */
+            abi_ulong lr;        /* Aliases gregs[TREG_LR].  */
+        };
+    };
+    abi_ulong pc;        /* Program counter.  */
+    abi_ulong ics;       /* In Interrupt Critical Section?  */
+    abi_ulong faultnum;  /* Fault number.  */
+    abi_ulong pad[5];
+};
+
+struct target_ucontext {
+    abi_ulong tuc_flags;
+    abi_ulong tuc_link;
+    target_stack_t tuc_stack;
+    struct target_sigcontext tuc_mcontext;
+    target_sigset_t tuc_sigmask;   /* mask last for extensibility */
+};
+
+struct target_rt_sigframe {
+    unsigned char save_area[16]; /* caller save area */
+    struct target_siginfo info;
+    struct target_ucontext uc;
+    abi_ulong retcode[2];
+};
+
+#define INSN_MOVELI_R10_139  0x00045fe551483000ULL /* { moveli r10, 139 } */
+#define INSN_SWINT1          0x286b180051485000ULL /* { swint1 } */
+
+
+static void setup_sigcontext(struct target_sigcontext *sc,
+                             CPUArchState *env, int signo)
+{
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; ++i) {
+        __put_user(env->regs[i], &sc->gregs[i]);
+    }
+
+    __put_user(env->pc, &sc->pc);
+    __put_user(0, &sc->ics);
+    __put_user(signo, &sc->faultnum);
+}
+
+static void restore_sigcontext(CPUTLGState *env, struct target_sigcontext *sc)
+{
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; ++i) {
+        __get_user(env->regs[i], &sc->gregs[i]);
+    }
+
+    __get_user(env->pc, &sc->pc);
+}
+
+static abi_ulong get_sigframe(struct target_sigaction *ka, CPUArchState *env,
+                              size_t frame_size)
+{
+    unsigned long sp = env->regs[TILEGX_R_SP];
+
+    if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size))) {
+        return -1UL;
+    }
+
+    if ((ka->sa_flags & SA_ONSTACK) && !sas_ss_flags(sp)) {
+        sp = target_sigaltstack_used.ss_sp + target_sigaltstack_used.ss_size;
+    }
+
+    sp -= frame_size;
+    sp &= -16UL;
+    return sp;
+}
+
+static void setup_rt_frame(int sig, struct target_sigaction *ka,
+                           target_siginfo_t *info,
+                           target_sigset_t *set, CPUArchState *env)
+{
+    abi_ulong frame_addr;
+    struct target_rt_sigframe *frame;
+    unsigned long restorer;
+
+    frame_addr = get_sigframe(ka, env, sizeof(*frame));
+    trace_user_setup_rt_frame(env, frame_addr);
+    if (!lock_user_struct(VERIFY_WRITE, frame, frame_addr, 0)) {
+        goto give_sigsegv;
+    }
+
+    /* Always write at least the signal number for the stack backtracer. */
+    if (ka->sa_flags & TARGET_SA_SIGINFO) {
+        /* At sigreturn time, restore the callee-save registers too. */
+        tswap_siginfo(&frame->info, info);
+        /* regs->flags |= PT_FLAGS_RESTORE_REGS; FIXME: we can skip it? */
+    } else {
+        __put_user(info->si_signo, &frame->info.si_signo);
+    }
+
+    /* Create the ucontext.  */
+    __put_user(0, &frame->uc.tuc_flags);
+    __put_user(0, &frame->uc.tuc_link);
+    __put_user(target_sigaltstack_used.ss_sp, &frame->uc.tuc_stack.ss_sp);
+    __put_user(sas_ss_flags(env->regs[TILEGX_R_SP]),
+               &frame->uc.tuc_stack.ss_flags);
+    __put_user(target_sigaltstack_used.ss_size, &frame->uc.tuc_stack.ss_size);
+    setup_sigcontext(&frame->uc.tuc_mcontext, env, info->si_signo);
+
+    if (ka->sa_flags & TARGET_SA_RESTORER) {
+        restorer = (unsigned long) ka->sa_restorer;
+    } else {
+        __put_user(INSN_MOVELI_R10_139, &frame->retcode[0]);
+        __put_user(INSN_SWINT1, &frame->retcode[1]);
+        restorer = frame_addr + offsetof(struct target_rt_sigframe, retcode);
+    }
+    env->pc = (unsigned long) ka->_sa_handler;
+    env->regs[TILEGX_R_SP] = (unsigned long) frame;
+    env->regs[TILEGX_R_LR] = restorer;
+    env->regs[0] = (unsigned long) sig;
+    env->regs[1] = (unsigned long) &frame->info;
+    env->regs[2] = (unsigned long) &frame->uc;
+    /* regs->flags |= PT_FLAGS_CALLER_SAVES; FIXME: we can skip it? */
+
+    unlock_user_struct(frame, frame_addr, 1);
+    return;
+
+give_sigsegv:
+    if (sig == TARGET_SIGSEGV) {
+        ka->_sa_handler = TARGET_SIG_DFL;
+    }
+    force_sig(TARGET_SIGSEGV /* , current */);
+}
+
+long do_rt_sigreturn(CPUTLGState *env)
+{
+    abi_ulong frame_addr = env->regs[TILEGX_R_SP];
+    struct target_rt_sigframe *frame;
+    sigset_t set;
+
+    trace_user_do_rt_sigreturn(env, frame_addr);
+    if (!lock_user_struct(VERIFY_READ, frame, frame_addr, 1)) {
+        goto badframe;
+    }
+    target_to_host_sigset(&set, &frame->uc.tuc_sigmask);
+    set_sigmask(&set);
+
+    restore_sigcontext(env, &frame->uc.tuc_mcontext);
+    if (do_sigaltstack(frame_addr + offsetof(struct target_rt_sigframe,
+                                             uc.tuc_stack),
+                       0, env->regs[TILEGX_R_SP]) == -EFAULT) {
+        goto badframe;
+    }
+
+    unlock_user_struct(frame, frame_addr, 0);
+    return -TARGET_QEMU_ESIGRETURN;
 
 
  badframe:
@@ -5572,14 +5800,14 @@
 #else
 
 static void setup_frame(int sig, struct target_sigaction *ka,
-			target_sigset_t *set, CPUArchState *env)
+                        target_sigset_t *set, CPUArchState *env)
 {
     fprintf(stderr, "setup_frame: not implemented\n");
 }
 
 static void setup_rt_frame(int sig, struct target_sigaction *ka,
                            target_siginfo_t *info,
-			   target_sigset_t *set, CPUArchState *env)
+                           target_sigset_t *set, CPUArchState *env)
 {
     fprintf(stderr, "setup_rt_frame: not implemented\n");
 }
@@ -5598,41 +5826,19 @@
 
 #endif
 
-void process_pending_signals(CPUArchState *cpu_env)
+static void handle_pending_signal(CPUArchState *cpu_env, int sig,
+                                  struct emulated_sigtable *k)
 {
     CPUState *cpu = ENV_GET_CPU(cpu_env);
-    int sig;
     abi_ulong handler;
-    sigset_t set, old_set;
+    sigset_t set;
     target_sigset_t target_old_set;
-    struct emulated_sigtable *k;
     struct target_sigaction *sa;
-    struct sigqueue *q;
     TaskState *ts = cpu->opaque;
 
-    if (!ts->signal_pending)
-        return;
-
-    /* FIXME: This is not threadsafe.  */
-    k = ts->sigtab;
-    for(sig = 1; sig <= TARGET_NSIG; sig++) {
-        if (k->pending)
-            goto handle_signal;
-        k++;
-    }
-    /* if no signal is pending, just return */
-    ts->signal_pending = 0;
-    return;
-
- handle_signal:
-#ifdef DEBUG_SIGNAL
-    fprintf(stderr, "qemu: process signal %d\n", sig);
-#endif
+    trace_user_handle_signal(cpu_env, sig);
     /* dequeue signal */
-    q = k->first;
-    k->first = q->next;
-    if (!k->first)
-        k->pending = 0;
+    k->pending = 0;
 
     sig = gdb_handlesig(cpu, sig);
     if (!sig) {
@@ -5643,14 +5849,6 @@
         handler = sa->_sa_handler;
     }
 
-    if (ts->sigsegv_blocked && sig == TARGET_SIGSEGV) {
-        /* Guest has blocked SIGSEGV but we got one anyway. Assume this
-         * is a forced SIGSEGV (ie one the kernel handles via force_sig_info
-         * because it got a real MMU fault), and treat as if default handler.
-         */
-        handler = TARGET_SIG_DFL;
-    }
-
     if (handler == TARGET_SIG_DFL) {
         /* default handler : ignore some signal. The other are job control or fatal */
         if (sig == TARGET_SIGTSTP || sig == TARGET_SIGTTIN || sig == TARGET_SIGTTOU) {
@@ -5667,17 +5865,23 @@
         force_sig(sig);
     } else {
         /* compute the blocked signals during the handler execution */
+        sigset_t *blocked_set;
+
         target_to_host_sigset(&set, &sa->sa_mask);
         /* SA_NODEFER indicates that the current signal should not be
            blocked during the handler */
         if (!(sa->sa_flags & TARGET_SA_NODEFER))
             sigaddset(&set, target_to_host_signal(sig));
 
-        /* block signals in the handler using Linux */
-        do_sigprocmask(SIG_BLOCK, &set, &old_set);
         /* save the previous blocked signal state to restore it at the
            end of the signal execution (see do_sigreturn) */
-        host_to_target_sigset_internal(&target_old_set, &old_set);
+        host_to_target_sigset_internal(&target_old_set, &ts->signal_mask);
+
+        /* block signals in the handler */
+        blocked_set = ts->in_sigsuspend ?
+            &ts->sigsuspend_mask : &ts->signal_mask;
+        sigorset(&ts->signal_mask, blocked_set, &set);
+        ts->in_sigsuspend = 0;
 
         /* if the CPU is in VM86 mode, we restore the 32 bit values */
 #if defined(TARGET_I386) && !defined(TARGET_X86_64)
@@ -5688,18 +5892,77 @@
         }
 #endif
         /* prepare the stack frame of the virtual CPU */
-#if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64)
+#if defined(TARGET_ABI_MIPSN32) || defined(TARGET_ABI_MIPSN64) \
+    || defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
         /* These targets do not have traditional signals.  */
-        setup_rt_frame(sig, sa, &q->info, &target_old_set, cpu_env);
+        setup_rt_frame(sig, sa, &k->info, &target_old_set, cpu_env);
 #else
         if (sa->sa_flags & TARGET_SA_SIGINFO)
-            setup_rt_frame(sig, sa, &q->info, &target_old_set, cpu_env);
+            setup_rt_frame(sig, sa, &k->info, &target_old_set, cpu_env);
         else
             setup_frame(sig, sa, &target_old_set, cpu_env);
 #endif
-	if (sa->sa_flags & TARGET_SA_RESETHAND)
+        if (sa->sa_flags & TARGET_SA_RESETHAND) {
             sa->_sa_handler = TARGET_SIG_DFL;
+        }
     }
-    if (q != &k->info)
-        free_sigqueue(cpu_env, q);
+}
+
+void process_pending_signals(CPUArchState *cpu_env)
+{
+    CPUState *cpu = ENV_GET_CPU(cpu_env);
+    int sig;
+    TaskState *ts = cpu->opaque;
+    sigset_t set;
+    sigset_t *blocked_set;
+
+    while (atomic_read(&ts->signal_pending)) {
+        /* FIXME: This is not threadsafe.  */
+        sigfillset(&set);
+        sigprocmask(SIG_SETMASK, &set, 0);
+
+        sig = ts->sync_signal.pending;
+        if (sig) {
+            /* Synchronous signals are forced,
+             * see force_sig_info() and callers in Linux
+             * Note that not all of our queue_signal() calls in QEMU correspond
+             * to force_sig_info() calls in Linux (some are send_sig_info()).
+             * However it seems like a kernel bug to me to allow the process
+             * to block a synchronous signal since it could then just end up
+             * looping round and round indefinitely.
+             */
+            if (sigismember(&ts->signal_mask, target_to_host_signal_table[sig])
+                || sigact_table[sig - 1]._sa_handler == TARGET_SIG_IGN) {
+                sigdelset(&ts->signal_mask, target_to_host_signal_table[sig]);
+                sigact_table[sig - 1]._sa_handler = TARGET_SIG_DFL;
+            }
+
+            handle_pending_signal(cpu_env, sig, &ts->sync_signal);
+        }
+
+        for (sig = 1; sig <= TARGET_NSIG; sig++) {
+            blocked_set = ts->in_sigsuspend ?
+                &ts->sigsuspend_mask : &ts->signal_mask;
+
+            if (ts->sigtab[sig - 1].pending &&
+                (!sigismember(blocked_set,
+                              target_to_host_signal_table[sig]))) {
+                handle_pending_signal(cpu_env, sig, &ts->sigtab[sig - 1]);
+                /* Restart scan from the beginning */
+                sig = 1;
+            }
+        }
+
+        /* if no signal is pending, unblock signals and recheck (the act
+         * of unblocking might cause us to take another host signal which
+         * will set signal_pending again).
+         */
+        atomic_set(&ts->signal_pending, 0);
+        ts->in_sigsuspend = 0;
+        set = ts->signal_mask;
+        sigdelset(&set, SIGSEGV);
+        sigdelset(&set, SIGBUS);
+        sigprocmask(SIG_SETMASK, &set, 0);
+    }
+    ts->in_sigsuspend = 0;
 }
diff --git a/linux-user/sparc/syscall.h b/linux-user/sparc/syscall.h
deleted file mode 100644
index 58573b9..0000000
--- a/linux-user/sparc/syscall.h
+++ /dev/null
@@ -1,20 +0,0 @@
-struct target_pt_regs {
-	abi_ulong psr;
-	abi_ulong pc;
-	abi_ulong npc;
-	abi_ulong y;
-	abi_ulong u_regs[16];
-};
-
-#define UNAME_MACHINE "sun4"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-/* SPARC kernels don't define this in their Kconfig, but they have the
- * same ABI as if they did, implemented by sparc-specific code which fishes
- * directly in the u_regs() struct for half the parameters in sparc_do_fork()
- * and copy_thread().
- */
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ      4096
-#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
-#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
diff --git a/linux-user/sparc/syscall_nr.h b/linux-user/sparc/syscall_nr.h
index 181cd32..e713c9d 100644
--- a/linux-user/sparc/syscall_nr.h
+++ b/linux-user/sparc/syscall_nr.h
@@ -179,6 +179,9 @@
 #define TARGET_NR_readahead          205 /* Linux Specific                              */
 #define TARGET_NR_socketcall         206 /* Linux Specific                              */
 #define TARGET_NR_syslog             207 /* Linux Specific                              */
+#define TARGET_NR_lookup_dcookie     208 /* Linux Specific                              */
+#define TARGET_NR_fadvise64          209 /* Linux Specific                              */
+#define TARGET_NR_fadvise64_64       210 /* Linux Specific                              */
 #define TARGET_NR_tgkill             211 /* Linux Specific                              */
 #define TARGET_NR_waitpid            212 /* Linux Specific                              */
 #define TARGET_NR_swapoff            213 /* Linux Specific                              */
@@ -278,7 +281,7 @@
 #define TARGET_NR_epoll_pwait	309
 #define TARGET_NR_utimensat		310
 #define TARGET_NR_signalfd		311
-#define TARGET_NR_timerfd		312
+#define TARGET_NR_timerfd_create	312
 #define TARGET_NR_eventfd		313
 #define TARGET_NR_fallocate		314
 #define TARGET_NR_timerfd_settime	315
@@ -309,3 +312,17 @@
 #define TARGET_NR_kern_features         340
 #define TARGET_NR_kcmp                  341
 #define TARGET_NR_finit_module          342
+#define TARGET_NR_sched_setattr         343
+#define TARGET_NR_sched_getattr         344
+#define TARGET_NR_renameat2             345
+#define TARGET_NR_seccomp               346
+#define TARGET_NR_getrandom             347
+#define TARGET_NR_memfd_create          348
+#define TARGET_NR_bpf                   349
+#define TARGET_NR_execveat              350
+#define TARGET_NR_membarrier            351
+#define TARGET_NR_userfaultfd           352
+#define TARGET_NR_bind                  353
+#define TARGET_NR_listen                354
+#define TARGET_NR_setsockopt            355
+#define TARGET_NR_mlock2                356
diff --git a/linux-user/sparc/target_cpu.h b/linux-user/sparc/target_cpu.h
index 4944d46..f2fe526 100644
--- a/linux-user/sparc/target_cpu.h
+++ b/linux-user/sparc/target_cpu.h
@@ -17,8 +17,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef SPARC_TARGET_CPU_H
+#define SPARC_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUSPARCState *env, target_ulong newsp)
 {
diff --git a/linux-user/sparc/target_signal.h b/linux-user/sparc/target_signal.h
index c7de300..e445e2b 100644
--- a/linux-user/sparc/target_signal.h
+++ b/linux-user/sparc/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef SPARC_TARGET_SIGNAL_H
+#define SPARC_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -33,4 +33,5 @@
     return state->regwptr[UREG_FP];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* SPARC_TARGET_SIGNAL_H */
diff --git a/linux-user/sparc/target_structs.h b/linux-user/sparc/target_structs.h
index c139e09..ee24c3b 100644
--- a/linux-user/sparc/target_structs.h
+++ b/linux-user/sparc/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef SPARC_TARGET_STRUCTS_H
+#define SPARC_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/sparc/target_syscall.h b/linux-user/sparc/target_syscall.h
new file mode 100644
index 0000000..326f674
--- /dev/null
+++ b/linux-user/sparc/target_syscall.h
@@ -0,0 +1,25 @@
+#ifndef SPARC_TARGET_SYSCALL_H
+#define SPARC_TARGET_SYSCALL_H
+
+struct target_pt_regs {
+	abi_ulong psr;
+	abi_ulong pc;
+	abi_ulong npc;
+	abi_ulong y;
+	abi_ulong u_regs[16];
+};
+
+#define UNAME_MACHINE "sun4"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+/* SPARC kernels don't define this in their Kconfig, but they have the
+ * same ABI as if they did, implemented by sparc-specific code which fishes
+ * directly in the u_regs() struct for half the parameters in sparc_do_fork()
+ * and copy_thread().
+ */
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      4096
+#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
+
+#endif /* SPARC_TARGET_SYSCALL_H */
diff --git a/linux-user/sparc64/syscall.h b/linux-user/sparc64/syscall.h
deleted file mode 100644
index 8398d3f..0000000
--- a/linux-user/sparc64/syscall.h
+++ /dev/null
@@ -1,21 +0,0 @@
-struct target_pt_regs {
-	abi_ulong u_regs[16];
-	abi_ulong tstate;
-	abi_ulong pc;
-	abi_ulong npc;
-	abi_ulong y;
-	abi_ulong fprs;
-};
-
-#define UNAME_MACHINE "sun4u"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-/* SPARC kernels don't define this in their Kconfig, but they have the
- * same ABI as if they did, implemented by sparc-specific code which fishes
- * directly in the u_regs() struct for half the parameters in sparc_do_fork()
- * and copy_thread().
- */
-#define TARGET_CLONE_BACKWARDS
-#define TARGET_MINSIGSTKSZ      4096
-#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
-#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
diff --git a/linux-user/sparc64/syscall_nr.h b/linux-user/sparc64/syscall_nr.h
index 34a984c..2b49ead 100644
--- a/linux-user/sparc64/syscall_nr.h
+++ b/linux-user/sparc64/syscall_nr.h
@@ -341,3 +341,17 @@
 #define TARGET_NR_kern_features         340
 #define TARGET_NR_kcmp                  341
 #define TARGET_NR_finit_module          342
+#define TARGET_NR_sched_setattr         343
+#define TARGET_NR_sched_getattr         344
+#define TARGET_NR_renameat2             345
+#define TARGET_NR_seccomp               346
+#define TARGET_NR_getrandom             347
+#define TARGET_NR_memfd_create          348
+#define TARGET_NR_bpf                   349
+#define TARGET_NR_execveat              350
+#define TARGET_NR_membarrier            351
+#define TARGET_NR_userfaultfd           352
+#define TARGET_NR_bind                  353
+#define TARGET_NR_listen                354
+#define TARGET_NR_setsockopt            355
+#define TARGET_NR_mlock2                356
diff --git a/linux-user/sparc64/target_signal.h b/linux-user/sparc64/target_signal.h
index c7de300..4449457 100644
--- a/linux-user/sparc64/target_signal.h
+++ b/linux-user/sparc64/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef SPARC64_TARGET_SIGNAL_H
+#define SPARC64_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -33,4 +33,5 @@
     return state->regwptr[UREG_FP];
 }
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* SPARC64_TARGET_SIGNAL_H */
diff --git a/linux-user/sparc64/target_structs.h b/linux-user/sparc64/target_structs.h
index fc17290..1808132 100644
--- a/linux-user/sparc64/target_structs.h
+++ b/linux-user/sparc64/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef SPARC64_TARGET_STRUCTS_H
+#define SPARC64_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/sparc64/target_syscall.h b/linux-user/sparc64/target_syscall.h
new file mode 100644
index 0000000..b7e3bf8
--- /dev/null
+++ b/linux-user/sparc64/target_syscall.h
@@ -0,0 +1,26 @@
+#ifndef SPARC64_TARGET_SYSCALL_H
+#define SPARC64_TARGET_SYSCALL_H
+
+struct target_pt_regs {
+	abi_ulong u_regs[16];
+	abi_ulong tstate;
+	abi_ulong pc;
+	abi_ulong npc;
+	abi_ulong y;
+	abi_ulong fprs;
+};
+
+#define UNAME_MACHINE "sun4u"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+/* SPARC kernels don't define this in their Kconfig, but they have the
+ * same ABI as if they did, implemented by sparc-specific code which fishes
+ * directly in the u_regs() struct for half the parameters in sparc_do_fork()
+ * and copy_thread().
+ */
+#define TARGET_CLONE_BACKWARDS
+#define TARGET_MINSIGSTKSZ      4096
+#define TARGET_MLOCKALL_MCL_CURRENT 0x2000
+#define TARGET_MLOCKALL_MCL_FUTURE  0x4000
+
+#endif /* SPARC64_TARGET_SYSCALL_H */
diff --git a/linux-user/strace.c b/linux-user/strace.c
index ea6c1d2..cc10dc4 100644
--- a/linux-user/strace.c
+++ b/linux-user/strace.c
@@ -1,13 +1,13 @@
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <sys/ipc.h>
 #include <sys/msg.h>
 #include <sys/sem.h>
 #include <sys/shm.h>
 #include <sys/select.h>
-#include <sys/types.h>
 #include <sys/mount.h>
-#include <sys/mman.h>
-#include <unistd.h>
+#include <arpa/inet.h>
+#include <netinet/tcp.h>
+#include <linux/if_packet.h>
 #include <sched.h>
 #include "qemu.h"
 
@@ -60,10 +60,15 @@
 UNUSED static void print_syscall_prologue(const struct syscallname *);
 UNUSED static void print_syscall_epilogue(const struct syscallname *);
 UNUSED static void print_string(abi_long, int);
+UNUSED static void print_buf(abi_long addr, abi_long len, int last);
 UNUSED static void print_raw_param(const char *, abi_long, int);
 UNUSED static void print_timeval(abi_ulong, int);
 UNUSED static void print_number(abi_long, int);
 UNUSED static void print_signal(abi_ulong, int);
+UNUSED static void print_sockaddr(abi_ulong addr, abi_long addrlen);
+UNUSED static void print_socket_domain(int domain);
+UNUSED static void print_socket_type(int type);
+UNUSED static void print_socket_protocol(int domain, int type, int protocol);
 
 /*
  * Utility functions
@@ -149,6 +154,165 @@
     gemu_log("%s%s", signal_name, get_comma(last));
 }
 
+static void
+print_sockaddr(abi_ulong addr, abi_long addrlen)
+{
+    struct target_sockaddr *sa;
+    int i;
+    int sa_family;
+
+    sa = lock_user(VERIFY_READ, addr, addrlen, 1);
+    if (sa) {
+        sa_family = tswap16(sa->sa_family);
+        switch (sa_family) {
+        case AF_UNIX: {
+            struct target_sockaddr_un *un = (struct target_sockaddr_un *)sa;
+            int i;
+            gemu_log("{sun_family=AF_UNIX,sun_path=\"");
+            for (i = 0; i < addrlen -
+                            offsetof(struct target_sockaddr_un, sun_path) &&
+                 un->sun_path[i]; i++) {
+                gemu_log("%c", un->sun_path[i]);
+            }
+            gemu_log("\"}");
+            break;
+        }
+        case AF_INET: {
+            struct target_sockaddr_in *in = (struct target_sockaddr_in *)sa;
+            uint8_t *c = (uint8_t *)&in->sin_addr.s_addr;
+            gemu_log("{sin_family=AF_INET,sin_port=htons(%d),",
+                     ntohs(in->sin_port));
+            gemu_log("sin_addr=inet_addr(\"%d.%d.%d.%d\")",
+                     c[0], c[1], c[2], c[3]);
+            gemu_log("}");
+            break;
+        }
+        case AF_PACKET: {
+            struct target_sockaddr_ll *ll = (struct target_sockaddr_ll *)sa;
+            uint8_t *c = (uint8_t *)&ll->sll_addr;
+            gemu_log("{sll_family=AF_PACKET,"
+                     "sll_protocol=htons(0x%04x),if%d,pkttype=",
+                     ntohs(ll->sll_protocol), ll->sll_ifindex);
+            switch (ll->sll_pkttype) {
+            case PACKET_HOST:
+                gemu_log("PACKET_HOST");
+                break;
+            case PACKET_BROADCAST:
+                gemu_log("PACKET_BROADCAST");
+                break;
+            case PACKET_MULTICAST:
+                gemu_log("PACKET_MULTICAST");
+                break;
+            case PACKET_OTHERHOST:
+                gemu_log("PACKET_OTHERHOST");
+                break;
+            case PACKET_OUTGOING:
+                gemu_log("PACKET_OUTGOING");
+                break;
+            default:
+                gemu_log("%d", ll->sll_pkttype);
+                break;
+            }
+            gemu_log(",sll_addr=%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+                     c[0], c[1], c[2], c[3], c[4], c[5], c[6], c[7]);
+            gemu_log("}");
+            break;
+        }
+        default:
+            gemu_log("{sa_family=%d, sa_data={", sa->sa_family);
+            for (i = 0; i < 13; i++) {
+                gemu_log("%02x, ", sa->sa_data[i]);
+            }
+            gemu_log("%02x}", sa->sa_data[i]);
+            gemu_log("}");
+            break;
+        }
+        unlock_user(sa, addr, 0);
+    } else {
+        print_raw_param("0x"TARGET_ABI_FMT_lx, addr, 0);
+    }
+    gemu_log(", "TARGET_ABI_FMT_ld, addrlen);
+}
+
+static void
+print_socket_domain(int domain)
+{
+    switch (domain) {
+    case PF_UNIX:
+        gemu_log("PF_UNIX");
+        break;
+    case PF_INET:
+        gemu_log("PF_INET");
+        break;
+    case PF_PACKET:
+        gemu_log("PF_PACKET");
+        break;
+    default:
+        gemu_log("%d", domain);
+        break;
+    }
+}
+
+static void
+print_socket_type(int type)
+{
+    switch (type) {
+    case TARGET_SOCK_DGRAM:
+        gemu_log("SOCK_DGRAM");
+        break;
+    case TARGET_SOCK_STREAM:
+        gemu_log("SOCK_STREAM");
+        break;
+    case TARGET_SOCK_RAW:
+        gemu_log("SOCK_RAW");
+        break;
+    case TARGET_SOCK_RDM:
+        gemu_log("SOCK_RDM");
+        break;
+    case TARGET_SOCK_SEQPACKET:
+        gemu_log("SOCK_SEQPACKET");
+        break;
+    case TARGET_SOCK_PACKET:
+        gemu_log("SOCK_PACKET");
+        break;
+    }
+}
+
+static void
+print_socket_protocol(int domain, int type, int protocol)
+{
+    if (domain == AF_PACKET ||
+        (domain == AF_INET && type == TARGET_SOCK_PACKET)) {
+        switch (protocol) {
+        case 0x0003:
+            gemu_log("ETH_P_ALL");
+            break;
+        default:
+            gemu_log("%d", protocol);
+        }
+        return;
+    }
+
+    switch (protocol) {
+    case IPPROTO_IP:
+        gemu_log("IPPROTO_IP");
+        break;
+    case IPPROTO_TCP:
+        gemu_log("IPPROTO_TCP");
+        break;
+    case IPPROTO_UDP:
+        gemu_log("IPPROTO_UDP");
+        break;
+    case IPPROTO_RAW:
+        gemu_log("IPPROTO_RAW");
+        break;
+    default:
+        gemu_log("%d", protocol);
+        break;
+    }
+}
+
+
 #ifdef TARGET_NR__newselect
 static void
 print_fdset(int n, abi_ulong target_fds_addr)
@@ -283,7 +447,7 @@
 static void
 print_syscall_ret_addr(const struct syscallname *name, abi_long ret)
 {
-    char *errstr = NULL;
+    const char *errstr = NULL;
 
     if (ret < 0) {
         errstr = target_strerror(-ret);
@@ -500,6 +664,26 @@
     FLAG_END,
 };
 
+UNUSED static struct flags msg_flags[] = {
+    /* send */
+    FLAG_GENERIC(MSG_CONFIRM),
+    FLAG_GENERIC(MSG_DONTROUTE),
+    FLAG_GENERIC(MSG_DONTWAIT),
+    FLAG_GENERIC(MSG_EOR),
+    FLAG_GENERIC(MSG_MORE),
+    FLAG_GENERIC(MSG_NOSIGNAL),
+    FLAG_GENERIC(MSG_OOB),
+    /* recv */
+    FLAG_GENERIC(MSG_CMSG_CLOEXEC),
+    FLAG_GENERIC(MSG_ERRQUEUE),
+    FLAG_GENERIC(MSG_PEEK),
+    FLAG_GENERIC(MSG_TRUNC),
+    FLAG_GENERIC(MSG_WAITALL),
+    /* recvmsg */
+    FLAG_GENERIC(MSG_CTRUNC),
+    FLAG_END,
+};
+
 /*
  * print_xxx utility functions.  These are used to print syscall
  * parameters in certain format.  All of these have parameter
@@ -621,6 +805,36 @@
     }
 }
 
+#define MAX_PRINT_BUF 40
+static void
+print_buf(abi_long addr, abi_long len, int last)
+{
+    uint8_t *s;
+    int i;
+
+    s = lock_user(VERIFY_READ, addr, len, 1);
+    if (s) {
+        gemu_log("\"");
+        for (i = 0; i < MAX_PRINT_BUF && i < len; i++) {
+            if (isprint(s[i])) {
+                gemu_log("%c", s[i]);
+            } else {
+                gemu_log("\\%o", s[i]);
+            }
+        }
+        gemu_log("\"");
+        if (i != len) {
+            gemu_log("...");
+        }
+        if (!last) {
+            gemu_log(",");
+        }
+        unlock_user(s, addr, 0);
+    } else {
+        print_pointer(addr, last);
+    }
+}
+
 /*
  * Prints out raw parameter using given format.  Caller needs
  * to do byte swapping if needed.
@@ -743,33 +957,31 @@
 #endif
 
 #ifdef TARGET_NR_clone
+static void do_print_clone(unsigned int flags, abi_ulong newsp,
+                           abi_ulong parent_tidptr, target_ulong newtls,
+                           abi_ulong child_tidptr)
+{
+    print_flags(clone_flags, flags, 0);
+    print_raw_param("child_stack=0x" TARGET_ABI_FMT_lx, newsp, 0);
+    print_raw_param("parent_tidptr=0x" TARGET_ABI_FMT_lx, parent_tidptr, 0);
+    print_raw_param("tls=0x" TARGET_ABI_FMT_lx, newtls, 0);
+    print_raw_param("child_tidptr=0x" TARGET_ABI_FMT_lx, child_tidptr, 1);
+}
+
 static void
 print_clone(const struct syscallname *name,
-    abi_long arg0, abi_long arg1, abi_long arg2,
-    abi_long arg3, abi_long arg4, abi_long arg5)
+    abi_long arg1, abi_long arg2, abi_long arg3,
+    abi_long arg4, abi_long arg5, abi_long arg6)
 {
     print_syscall_prologue(name);
-#if defined(TARGET_M68K)
-    print_flags(clone_flags, arg0, 0);
-    print_raw_param("newsp=0x" TARGET_ABI_FMT_lx, arg1, 1);
-#elif defined(TARGET_SH4) || defined(TARGET_ALPHA)
-    print_flags(clone_flags, arg0, 0);
-    print_raw_param("child_stack=0x" TARGET_ABI_FMT_lx, arg1, 0);
-    print_raw_param("parent_tidptr=0x" TARGET_ABI_FMT_lx, arg2, 0);
-    print_raw_param("child_tidptr=0x" TARGET_ABI_FMT_lx, arg3, 0);
-    print_raw_param("tls=0x" TARGET_ABI_FMT_lx, arg4, 1);
-#elif defined(TARGET_CRIS)
-    print_raw_param("child_stack=0x" TARGET_ABI_FMT_lx, arg0, 0);
-    print_flags(clone_flags, arg1, 0);
-    print_raw_param("parent_tidptr=0x" TARGET_ABI_FMT_lx, arg2, 0);
-    print_raw_param("tls=0x" TARGET_ABI_FMT_lx, arg3, 0);
-    print_raw_param("child_tidptr=0x" TARGET_ABI_FMT_lx, arg4, 1);
+#if defined(TARGET_MICROBLAZE)
+    do_print_clone(arg1, arg2, arg4, arg6, arg5);
+#elif defined(TARGET_CLONE_BACKWARDS)
+    do_print_clone(arg1, arg2, arg3, arg4, arg5);
+#elif defined(TARGET_CLONE_BACKWARDS2)
+    do_print_clone(arg2, arg1, arg3, arg5, arg4);
 #else
-    print_flags(clone_flags, arg0, 0);
-    print_raw_param("child_stack=0x" TARGET_ABI_FMT_lx, arg1, 0);
-    print_raw_param("parent_tidptr=0x" TARGET_ABI_FMT_lx, arg2, 0);
-    print_raw_param("tls=0x" TARGET_ABI_FMT_lx, arg3, 0);
-    print_raw_param("child_tidptr=0x" TARGET_ABI_FMT_lx, arg4, 1);
+    do_print_clone(arg1, arg2, arg3, arg5, arg4);
 #endif
     print_syscall_epilogue(name);
 }
@@ -921,6 +1133,13 @@
     case TARGET_F_GETLEASE:
         gemu_log("F_GETLEASE");
         break;
+    case TARGET_F_SETPIPE_SZ:
+        gemu_log("F_SETPIPE_SZ,");
+        print_raw_param(TARGET_ABI_FMT_ld, arg2, 1);
+        break;
+    case TARGET_F_GETPIPE_SZ:
+        gemu_log("F_GETPIPE_SZ");
+        break;
     case TARGET_F_DUPFD_CLOEXEC:
         gemu_log("F_DUPFD_CLOEXEC,");
         print_raw_param(TARGET_ABI_FMT_ld, arg2, 1);
@@ -1006,6 +1225,361 @@
 }
 #endif
 
+#if defined(TARGET_NR_socket)
+static void
+print_socket(const struct syscallname *name,
+             abi_long arg0, abi_long arg1, abi_long arg2,
+             abi_long arg3, abi_long arg4, abi_long arg5)
+{
+    abi_ulong domain = arg0, type = arg1, protocol = arg2;
+
+    print_syscall_prologue(name);
+    print_socket_domain(domain);
+    gemu_log(",");
+    print_socket_type(type);
+    gemu_log(",");
+    if (domain == AF_PACKET ||
+        (domain == AF_INET && type == TARGET_SOCK_PACKET)) {
+        protocol = tswap16(protocol);
+    }
+    print_socket_protocol(domain, type, protocol);
+    print_syscall_epilogue(name);
+}
+
+#endif
+
+#if defined(TARGET_NR_socketcall)
+
+#define get_user_ualx(x, gaddr, idx) \
+        get_user_ual(x, (gaddr) + (idx) * sizeof(abi_long))
+
+static void do_print_socket(const char *name, abi_long arg1)
+{
+    abi_ulong domain, type, protocol;
+
+    get_user_ualx(domain, arg1, 0);
+    get_user_ualx(type, arg1, 1);
+    get_user_ualx(protocol, arg1, 2);
+    gemu_log("%s(", name);
+    print_socket_domain(domain);
+    gemu_log(",");
+    print_socket_type(type);
+    gemu_log(",");
+    if (domain == AF_PACKET ||
+        (domain == AF_INET && type == TARGET_SOCK_PACKET)) {
+        protocol = tswap16(protocol);
+    }
+    print_socket_protocol(domain, type, protocol);
+    gemu_log(")");
+}
+
+static void do_print_sockaddr(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, addr, addrlen;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(addr, arg1, 1);
+    get_user_ualx(addrlen, arg1, 2);
+
+    gemu_log("%s(", name);
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    print_sockaddr(addr, addrlen);
+    gemu_log(")");
+}
+
+static void do_print_listen(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, backlog;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(backlog, arg1, 1);
+
+    gemu_log("%s(", name);
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, backlog, 1);
+    gemu_log(")");
+}
+
+static void do_print_socketpair(const char *name, abi_long arg1)
+{
+    abi_ulong domain, type, protocol, tab;
+
+    get_user_ualx(domain, arg1, 0);
+    get_user_ualx(type, arg1, 1);
+    get_user_ualx(protocol, arg1, 2);
+    get_user_ualx(tab, arg1, 3);
+
+    gemu_log("%s(", name);
+    print_socket_domain(domain);
+    gemu_log(",");
+    print_socket_type(type);
+    gemu_log(",");
+    print_socket_protocol(domain, type, protocol);
+    gemu_log(",");
+    print_raw_param(TARGET_ABI_FMT_lx, tab, 1);
+    gemu_log(")");
+}
+
+static void do_print_sendrecv(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, msg, len, flags;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(msg, arg1, 1);
+    get_user_ualx(len, arg1, 2);
+    get_user_ualx(flags, arg1, 3);
+
+    gemu_log("%s(", name);
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    print_buf(msg, len, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, len, 0);
+    print_flags(msg_flags, flags, 1);
+    gemu_log(")");
+}
+
+static void do_print_msgaddr(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, msg, len, flags, addr, addrlen;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(msg, arg1, 1);
+    get_user_ualx(len, arg1, 2);
+    get_user_ualx(flags, arg1, 3);
+    get_user_ualx(addr, arg1, 4);
+    get_user_ualx(addrlen, arg1, 5);
+
+    gemu_log("%s(", name);
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    print_buf(msg, len, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, len, 0);
+    print_flags(msg_flags, flags, 0);
+    print_sockaddr(addr, addrlen);
+    gemu_log(")");
+}
+
+static void do_print_shutdown(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, how;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(how, arg1, 1);
+
+    gemu_log("shutdown(");
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    switch (how) {
+    case SHUT_RD:
+        gemu_log("SHUT_RD");
+        break;
+    case SHUT_WR:
+        gemu_log("SHUT_WR");
+        break;
+    case SHUT_RDWR:
+        gemu_log("SHUT_RDWR");
+        break;
+    default:
+        print_raw_param(TARGET_ABI_FMT_ld, how, 1);
+        break;
+    }
+    gemu_log(")");
+}
+
+static void do_print_msg(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, msg, flags;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(msg, arg1, 1);
+    get_user_ualx(flags, arg1, 2);
+
+    gemu_log("%s(", name);
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    print_pointer(msg, 0);
+    print_flags(msg_flags, flags, 1);
+    gemu_log(")");
+}
+
+static void do_print_sockopt(const char *name, abi_long arg1)
+{
+    abi_ulong sockfd, level, optname, optval, optlen;
+
+    get_user_ualx(sockfd, arg1, 0);
+    get_user_ualx(level, arg1, 1);
+    get_user_ualx(optname, arg1, 2);
+    get_user_ualx(optval, arg1, 3);
+    get_user_ualx(optlen, arg1, 4);
+
+    gemu_log("%s(", name);
+    print_raw_param(TARGET_ABI_FMT_ld, sockfd, 0);
+    switch (level) {
+    case SOL_TCP:
+        gemu_log("SOL_TCP,");
+        print_raw_param(TARGET_ABI_FMT_ld, optname, 0);
+        print_pointer(optval, 0);
+        break;
+    case SOL_IP:
+        gemu_log("SOL_IP,");
+        print_raw_param(TARGET_ABI_FMT_ld, optname, 0);
+        print_pointer(optval, 0);
+        break;
+    case SOL_RAW:
+        gemu_log("SOL_RAW,");
+        print_raw_param(TARGET_ABI_FMT_ld, optname, 0);
+        print_pointer(optval, 0);
+        break;
+    case TARGET_SOL_SOCKET:
+        gemu_log("SOL_SOCKET,");
+        switch (optname) {
+        case TARGET_SO_DEBUG:
+            gemu_log("SO_DEBUG,");
+print_optint:
+            print_number(optval, 0);
+            break;
+        case TARGET_SO_REUSEADDR:
+            gemu_log("SO_REUSEADDR,");
+            goto print_optint;
+        case TARGET_SO_TYPE:
+            gemu_log("SO_TYPE,");
+            goto print_optint;
+        case TARGET_SO_ERROR:
+            gemu_log("SO_ERROR,");
+            goto print_optint;
+        case TARGET_SO_DONTROUTE:
+            gemu_log("SO_DONTROUTE,");
+            goto print_optint;
+        case TARGET_SO_BROADCAST:
+            gemu_log("SO_BROADCAST,");
+            goto print_optint;
+        case TARGET_SO_SNDBUF:
+            gemu_log("SO_SNDBUF,");
+            goto print_optint;
+        case TARGET_SO_RCVBUF:
+            gemu_log("SO_RCVBUF,");
+            goto print_optint;
+        case TARGET_SO_KEEPALIVE:
+            gemu_log("SO_KEEPALIVE,");
+            goto print_optint;
+        case TARGET_SO_OOBINLINE:
+            gemu_log("SO_OOBINLINE,");
+            goto print_optint;
+        case TARGET_SO_NO_CHECK:
+            gemu_log("SO_NO_CHECK,");
+            goto print_optint;
+        case TARGET_SO_PRIORITY:
+            gemu_log("SO_PRIORITY,");
+            goto print_optint;
+        case TARGET_SO_BSDCOMPAT:
+            gemu_log("SO_BSDCOMPAT,");
+            goto print_optint;
+        case TARGET_SO_PASSCRED:
+            gemu_log("SO_PASSCRED,");
+            goto print_optint;
+        case TARGET_SO_TIMESTAMP:
+            gemu_log("SO_TIMESTAMP,");
+            goto print_optint;
+        case TARGET_SO_RCVLOWAT:
+            gemu_log("SO_RCVLOWAT,");
+            goto print_optint;
+        case TARGET_SO_RCVTIMEO:
+            gemu_log("SO_RCVTIMEO,");
+            print_timeval(optval, 0);
+            break;
+        case TARGET_SO_SNDTIMEO:
+            gemu_log("SO_SNDTIMEO,");
+            print_timeval(optval, 0);
+            break;
+        case TARGET_SO_ATTACH_FILTER: {
+            struct target_sock_fprog *fprog;
+
+            gemu_log("SO_ATTACH_FILTER,");
+
+            if (lock_user_struct(VERIFY_READ, fprog, optval,  0)) {
+                struct target_sock_filter *filter;
+                gemu_log("{");
+                if (lock_user_struct(VERIFY_READ, filter,
+                                     tswapal(fprog->filter),  0)) {
+                    int i;
+                    for (i = 0; i < tswap16(fprog->len) - 1; i++) {
+                        gemu_log("[%d]{0x%x,%d,%d,0x%x},",
+                                 i, tswap16(filter[i].code),
+                                 filter[i].jt, filter[i].jf,
+                                 tswap32(filter[i].k));
+                    }
+                    gemu_log("[%d]{0x%x,%d,%d,0x%x}",
+                             i, tswap16(filter[i].code),
+                             filter[i].jt, filter[i].jf,
+                             tswap32(filter[i].k));
+                } else {
+                    gemu_log(TARGET_ABI_FMT_lx, tswapal(fprog->filter));
+                }
+                gemu_log(",%d},", tswap16(fprog->len));
+                unlock_user(fprog, optval, 0);
+            } else {
+                print_pointer(optval, 0);
+            }
+            break;
+        }
+        default:
+            print_raw_param(TARGET_ABI_FMT_ld, optname, 0);
+            print_pointer(optval, 0);
+            break;
+        }
+        break;
+    default:
+        print_raw_param(TARGET_ABI_FMT_ld, level, 0);
+        print_raw_param(TARGET_ABI_FMT_ld, optname, 0);
+        print_pointer(optval, 0);
+        break;
+    }
+    print_raw_param(TARGET_ABI_FMT_ld, optlen, 1);
+    gemu_log(")");
+}
+
+#define PRINT_SOCKOP(name, func) \
+    [SOCKOP_##name] = { #name, func }
+
+static struct {
+    const char *name;
+    void (*print)(const char *, abi_long);
+} scall[] = {
+    PRINT_SOCKOP(socket, do_print_socket),
+    PRINT_SOCKOP(bind, do_print_sockaddr),
+    PRINT_SOCKOP(connect, do_print_sockaddr),
+    PRINT_SOCKOP(listen, do_print_listen),
+    PRINT_SOCKOP(accept, do_print_sockaddr),
+    PRINT_SOCKOP(getsockname, do_print_sockaddr),
+    PRINT_SOCKOP(getpeername, do_print_sockaddr),
+    PRINT_SOCKOP(socketpair, do_print_socketpair),
+    PRINT_SOCKOP(send, do_print_sendrecv),
+    PRINT_SOCKOP(recv, do_print_sendrecv),
+    PRINT_SOCKOP(sendto, do_print_msgaddr),
+    PRINT_SOCKOP(recvfrom, do_print_msgaddr),
+    PRINT_SOCKOP(shutdown, do_print_shutdown),
+    PRINT_SOCKOP(sendmsg, do_print_msg),
+    PRINT_SOCKOP(recvmsg, do_print_msg),
+    PRINT_SOCKOP(setsockopt, do_print_sockopt),
+    PRINT_SOCKOP(getsockopt, do_print_sockopt),
+};
+
+static void
+print_socketcall(const struct syscallname *name,
+                 abi_long arg0, abi_long arg1, abi_long arg2,
+                 abi_long arg3, abi_long arg4, abi_long arg5)
+{
+    if (arg0 >= 0 && arg0 < ARRAY_SIZE(scall) && scall[arg0].print) {
+        scall[arg0].print(scall[arg0].name, arg1);
+        return;
+    }
+    print_syscall_prologue(name);
+    print_raw_param(TARGET_ABI_FMT_ld, arg0, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, arg1, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, arg2, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, arg3, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, arg4, 0);
+    print_raw_param(TARGET_ABI_FMT_ld, arg5, 0);
+    print_syscall_epilogue(name);
+}
+#endif
+
 #if defined(TARGET_NR_stat) || defined(TARGET_NR_stat64) || \
     defined(TARGET_NR_lstat) || defined(TARGET_NR_lstat64)
 static void
@@ -1596,7 +2170,7 @@
 print_syscall_ret(int num, abi_long ret)
 {
     int i;
-    char *errstr = NULL;
+    const char *errstr = NULL;
 
     for(i=0;i<nsyscalls;i++)
         if( scnames[i].nr == num ) {
diff --git a/linux-user/strace.list b/linux-user/strace.list
index aa0cd73..aa967a2 100644
--- a/linux-user/strace.list
+++ b/linux-user/strace.list
@@ -337,7 +337,8 @@
 { TARGET_NR_getsockopt, "getsockopt" , NULL, NULL, NULL },
 #endif
 #ifdef TARGET_NR_get_thread_area
-{ TARGET_NR_get_thread_area, "get_thread_area" , NULL, NULL, NULL },
+{ TARGET_NR_get_thread_area, "get_thread_area", "%s(0x"TARGET_ABI_FMT_lx")",
+  NULL, NULL },
 #endif
 #ifdef TARGET_NR_gettid
 { TARGET_NR_gettid, "gettid" , NULL, NULL, NULL },
@@ -1234,7 +1235,8 @@
 { TARGET_NR_setsockopt, "setsockopt" , NULL, NULL, NULL },
 #endif
 #ifdef TARGET_NR_set_thread_area
-{ TARGET_NR_set_thread_area, "set_thread_area" , NULL, NULL, NULL },
+{ TARGET_NR_set_thread_area, "set_thread_area", "%s(0x"TARGET_ABI_FMT_lx")",
+  NULL, NULL },
 #endif
 #ifdef TARGET_NR_set_tid_address
 { TARGET_NR_set_tid_address, "set_tid_address" , NULL, NULL, NULL },
@@ -1291,10 +1293,10 @@
 { TARGET_NR_sigsuspend, "sigsuspend" , NULL, NULL, NULL },
 #endif
 #ifdef TARGET_NR_socket
-{ TARGET_NR_socket, "socket" , NULL, NULL, NULL },
+{ TARGET_NR_socket, "socket" , NULL, print_socket, NULL },
 #endif
 #ifdef TARGET_NR_socketcall
-{ TARGET_NR_socketcall, "socketcall" , NULL, NULL, NULL },
+{ TARGET_NR_socketcall, "socketcall" , NULL, print_socketcall, NULL },
 #endif
 #ifdef TARGET_NR_socketpair
 { TARGET_NR_socketpair, "socketpair" , NULL, NULL, NULL },
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index aaac6a2..378f0f5 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -17,34 +17,23 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #define _ATFILE_SOURCE
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/path.h"
 #include <elf.h>
 #include <endian.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <limits.h>
 #include <grp.h>
-#include <sys/types.h>
 #include <sys/ipc.h>
 #include <sys/msg.h>
 #include <sys/wait.h>
-#include <sys/time.h>
-#include <sys/stat.h>
 #include <sys/mount.h>
 #include <sys/file.h>
 #include <sys/fsuid.h>
 #include <sys/personality.h>
 #include <sys/prctl.h>
 #include <sys/resource.h>
-#include <sys/mman.h>
 #include <sys/swap.h>
 #include <linux/capability.h>
-#include <signal.h>
 #include <sched.h>
 #ifdef __ia64__
 int __clone2(int (*fn)(void *), void *child_stack_base,
@@ -60,6 +49,7 @@
 #include <sys/statfs.h>
 #include <utime.h>
 #include <sys/sysinfo.h>
+#include <sys/signalfd.h>
 //#include <sys/user.h>
 #include <netinet/ip.h>
 #include <netinet/tcp.h>
@@ -110,6 +100,13 @@
 #include <linux/route.h>
 #include <linux/filter.h>
 #include <linux/blkpg.h>
+#include <netpacket/packet.h>
+#include <linux/netlink.h>
+#ifdef CONFIG_RTNETLINK
+#include <linux/rtnetlink.h>
+#include <linux/if_bridge.h>
+#endif
+#include <linux/audit.h>
 #include "linux_loop.h"
 #include "uname.h"
 
@@ -119,12 +116,15 @@
     CLONE_PARENT_SETTID | CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID)
 
 //#define DEBUG
+/* Define DEBUG_ERESTARTSYS to force every syscall to be restarted
+ * once. This exercises the codepaths for restart.
+ */
+//#define DEBUG_ERESTARTSYS
 
 //#include <linux/msdos_fs.h>
 #define	VFAT_IOCTL_READDIR_BOTH		_IOR('r', 1, struct linux_dirent [2])
 #define	VFAT_IOCTL_READDIR_SHORT	_IOR('r', 2, struct linux_dirent [2])
 
-
 #undef _syscall0
 #undef _syscall1
 #undef _syscall2
@@ -187,8 +187,6 @@
 #define __NR_sys_getpriority __NR_getpriority
 #define __NR_sys_rt_sigqueueinfo __NR_rt_sigqueueinfo
 #define __NR_sys_syslog __NR_syslog
-#define __NR_sys_tgkill __NR_tgkill
-#define __NR_sys_tkill __NR_tkill
 #define __NR_sys_futex __NR_futex
 #define __NR_sys_inotify_init __NR_inotify_init
 #define __NR_sys_inotify_add_watch __NR_inotify_add_watch
@@ -213,7 +211,7 @@
     return -ENOSYS;
 }
 #endif
-#ifdef __NR_getdents
+#if defined(TARGET_NR_getdents) && defined(__NR_getdents)
 _syscall3(int, sys_getdents, uint, fd, struct linux_dirent *, dirp, uint, count);
 #endif
 #if !defined(__NR_getdents) || \
@@ -226,12 +224,6 @@
 #endif
 _syscall3(int,sys_rt_sigqueueinfo,int,pid,int,sig,siginfo_t *,uinfo)
 _syscall3(int,sys_syslog,int,type,char*,bufp,int,len)
-#if defined(TARGET_NR_tgkill) && defined(__NR_tgkill)
-_syscall3(int,sys_tgkill,int,tgid,int,pid,int,sig)
-#endif
-#if defined(TARGET_NR_tkill) && defined(__NR_tkill)
-_syscall2(int,sys_tkill,int,tid,int,sig)
-#endif
 #ifdef __NR_exit_group
 _syscall1(int,exit_group,int,error_code)
 #endif
@@ -260,6 +252,9 @@
 #if defined(TARGET_NR_ioprio_set) && defined(__NR_ioprio_set)
 _syscall3(int, ioprio_set, int, which, int, who, int, ioprio)
 #endif
+#if defined(TARGET_NR_getrandom) && defined(__NR_getrandom)
+_syscall3(int, getrandom, void *, buf, size_t, buflen, unsigned int, flags)
+#endif
 
 static bitmask_transtbl fcntl_flags_tbl[] = {
   { TARGET_O_ACCMODE,   TARGET_O_WRONLY,    O_ACCMODE,   O_WRONLY,    },
@@ -294,6 +289,227 @@
   { 0, 0, 0, 0 }
 };
 
+enum {
+    QEMU_IFLA_BR_UNSPEC,
+    QEMU_IFLA_BR_FORWARD_DELAY,
+    QEMU_IFLA_BR_HELLO_TIME,
+    QEMU_IFLA_BR_MAX_AGE,
+    QEMU_IFLA_BR_AGEING_TIME,
+    QEMU_IFLA_BR_STP_STATE,
+    QEMU_IFLA_BR_PRIORITY,
+    QEMU_IFLA_BR_VLAN_FILTERING,
+    QEMU_IFLA_BR_VLAN_PROTOCOL,
+    QEMU_IFLA_BR_GROUP_FWD_MASK,
+    QEMU_IFLA_BR_ROOT_ID,
+    QEMU_IFLA_BR_BRIDGE_ID,
+    QEMU_IFLA_BR_ROOT_PORT,
+    QEMU_IFLA_BR_ROOT_PATH_COST,
+    QEMU_IFLA_BR_TOPOLOGY_CHANGE,
+    QEMU_IFLA_BR_TOPOLOGY_CHANGE_DETECTED,
+    QEMU_IFLA_BR_HELLO_TIMER,
+    QEMU_IFLA_BR_TCN_TIMER,
+    QEMU_IFLA_BR_TOPOLOGY_CHANGE_TIMER,
+    QEMU_IFLA_BR_GC_TIMER,
+    QEMU_IFLA_BR_GROUP_ADDR,
+    QEMU_IFLA_BR_FDB_FLUSH,
+    QEMU_IFLA_BR_MCAST_ROUTER,
+    QEMU_IFLA_BR_MCAST_SNOOPING,
+    QEMU_IFLA_BR_MCAST_QUERY_USE_IFADDR,
+    QEMU_IFLA_BR_MCAST_QUERIER,
+    QEMU_IFLA_BR_MCAST_HASH_ELASTICITY,
+    QEMU_IFLA_BR_MCAST_HASH_MAX,
+    QEMU_IFLA_BR_MCAST_LAST_MEMBER_CNT,
+    QEMU_IFLA_BR_MCAST_STARTUP_QUERY_CNT,
+    QEMU_IFLA_BR_MCAST_LAST_MEMBER_INTVL,
+    QEMU_IFLA_BR_MCAST_MEMBERSHIP_INTVL,
+    QEMU_IFLA_BR_MCAST_QUERIER_INTVL,
+    QEMU_IFLA_BR_MCAST_QUERY_INTVL,
+    QEMU_IFLA_BR_MCAST_QUERY_RESPONSE_INTVL,
+    QEMU_IFLA_BR_MCAST_STARTUP_QUERY_INTVL,
+    QEMU_IFLA_BR_NF_CALL_IPTABLES,
+    QEMU_IFLA_BR_NF_CALL_IP6TABLES,
+    QEMU_IFLA_BR_NF_CALL_ARPTABLES,
+    QEMU_IFLA_BR_VLAN_DEFAULT_PVID,
+    QEMU_IFLA_BR_PAD,
+    QEMU_IFLA_BR_VLAN_STATS_ENABLED,
+    QEMU_IFLA_BR_MCAST_STATS_ENABLED,
+    QEMU___IFLA_BR_MAX,
+};
+
+enum {
+    QEMU_IFLA_UNSPEC,
+    QEMU_IFLA_ADDRESS,
+    QEMU_IFLA_BROADCAST,
+    QEMU_IFLA_IFNAME,
+    QEMU_IFLA_MTU,
+    QEMU_IFLA_LINK,
+    QEMU_IFLA_QDISC,
+    QEMU_IFLA_STATS,
+    QEMU_IFLA_COST,
+    QEMU_IFLA_PRIORITY,
+    QEMU_IFLA_MASTER,
+    QEMU_IFLA_WIRELESS,
+    QEMU_IFLA_PROTINFO,
+    QEMU_IFLA_TXQLEN,
+    QEMU_IFLA_MAP,
+    QEMU_IFLA_WEIGHT,
+    QEMU_IFLA_OPERSTATE,
+    QEMU_IFLA_LINKMODE,
+    QEMU_IFLA_LINKINFO,
+    QEMU_IFLA_NET_NS_PID,
+    QEMU_IFLA_IFALIAS,
+    QEMU_IFLA_NUM_VF,
+    QEMU_IFLA_VFINFO_LIST,
+    QEMU_IFLA_STATS64,
+    QEMU_IFLA_VF_PORTS,
+    QEMU_IFLA_PORT_SELF,
+    QEMU_IFLA_AF_SPEC,
+    QEMU_IFLA_GROUP,
+    QEMU_IFLA_NET_NS_FD,
+    QEMU_IFLA_EXT_MASK,
+    QEMU_IFLA_PROMISCUITY,
+    QEMU_IFLA_NUM_TX_QUEUES,
+    QEMU_IFLA_NUM_RX_QUEUES,
+    QEMU_IFLA_CARRIER,
+    QEMU_IFLA_PHYS_PORT_ID,
+    QEMU_IFLA_CARRIER_CHANGES,
+    QEMU_IFLA_PHYS_SWITCH_ID,
+    QEMU_IFLA_LINK_NETNSID,
+    QEMU_IFLA_PHYS_PORT_NAME,
+    QEMU_IFLA_PROTO_DOWN,
+    QEMU_IFLA_GSO_MAX_SEGS,
+    QEMU_IFLA_GSO_MAX_SIZE,
+    QEMU_IFLA_PAD,
+    QEMU_IFLA_XDP,
+    QEMU___IFLA_MAX
+};
+
+enum {
+    QEMU_IFLA_BRPORT_UNSPEC,
+    QEMU_IFLA_BRPORT_STATE,
+    QEMU_IFLA_BRPORT_PRIORITY,
+    QEMU_IFLA_BRPORT_COST,
+    QEMU_IFLA_BRPORT_MODE,
+    QEMU_IFLA_BRPORT_GUARD,
+    QEMU_IFLA_BRPORT_PROTECT,
+    QEMU_IFLA_BRPORT_FAST_LEAVE,
+    QEMU_IFLA_BRPORT_LEARNING,
+    QEMU_IFLA_BRPORT_UNICAST_FLOOD,
+    QEMU_IFLA_BRPORT_PROXYARP,
+    QEMU_IFLA_BRPORT_LEARNING_SYNC,
+    QEMU_IFLA_BRPORT_PROXYARP_WIFI,
+    QEMU_IFLA_BRPORT_ROOT_ID,
+    QEMU_IFLA_BRPORT_BRIDGE_ID,
+    QEMU_IFLA_BRPORT_DESIGNATED_PORT,
+    QEMU_IFLA_BRPORT_DESIGNATED_COST,
+    QEMU_IFLA_BRPORT_ID,
+    QEMU_IFLA_BRPORT_NO,
+    QEMU_IFLA_BRPORT_TOPOLOGY_CHANGE_ACK,
+    QEMU_IFLA_BRPORT_CONFIG_PENDING,
+    QEMU_IFLA_BRPORT_MESSAGE_AGE_TIMER,
+    QEMU_IFLA_BRPORT_FORWARD_DELAY_TIMER,
+    QEMU_IFLA_BRPORT_HOLD_TIMER,
+    QEMU_IFLA_BRPORT_FLUSH,
+    QEMU_IFLA_BRPORT_MULTICAST_ROUTER,
+    QEMU_IFLA_BRPORT_PAD,
+    QEMU___IFLA_BRPORT_MAX
+};
+
+enum {
+    QEMU_IFLA_INFO_UNSPEC,
+    QEMU_IFLA_INFO_KIND,
+    QEMU_IFLA_INFO_DATA,
+    QEMU_IFLA_INFO_XSTATS,
+    QEMU_IFLA_INFO_SLAVE_KIND,
+    QEMU_IFLA_INFO_SLAVE_DATA,
+    QEMU___IFLA_INFO_MAX,
+};
+
+enum {
+    QEMU_IFLA_INET_UNSPEC,
+    QEMU_IFLA_INET_CONF,
+    QEMU___IFLA_INET_MAX,
+};
+
+enum {
+    QEMU_IFLA_INET6_UNSPEC,
+    QEMU_IFLA_INET6_FLAGS,
+    QEMU_IFLA_INET6_CONF,
+    QEMU_IFLA_INET6_STATS,
+    QEMU_IFLA_INET6_MCAST,
+    QEMU_IFLA_INET6_CACHEINFO,
+    QEMU_IFLA_INET6_ICMP6STATS,
+    QEMU_IFLA_INET6_TOKEN,
+    QEMU_IFLA_INET6_ADDR_GEN_MODE,
+    QEMU___IFLA_INET6_MAX
+};
+
+typedef abi_long (*TargetFdDataFunc)(void *, size_t);
+typedef abi_long (*TargetFdAddrFunc)(void *, abi_ulong, socklen_t);
+typedef struct TargetFdTrans {
+    TargetFdDataFunc host_to_target_data;
+    TargetFdDataFunc target_to_host_data;
+    TargetFdAddrFunc target_to_host_addr;
+} TargetFdTrans;
+
+static TargetFdTrans **target_fd_trans;
+
+static unsigned int target_fd_max;
+
+static TargetFdDataFunc fd_trans_target_to_host_data(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max && target_fd_trans[fd]) {
+        return target_fd_trans[fd]->target_to_host_data;
+    }
+    return NULL;
+}
+
+static TargetFdDataFunc fd_trans_host_to_target_data(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max && target_fd_trans[fd]) {
+        return target_fd_trans[fd]->host_to_target_data;
+    }
+    return NULL;
+}
+
+static TargetFdAddrFunc fd_trans_target_to_host_addr(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max && target_fd_trans[fd]) {
+        return target_fd_trans[fd]->target_to_host_addr;
+    }
+    return NULL;
+}
+
+static void fd_trans_register(int fd, TargetFdTrans *trans)
+{
+    unsigned int oldmax;
+
+    if (fd >= target_fd_max) {
+        oldmax = target_fd_max;
+        target_fd_max = ((fd >> 6) + 1) << 6; /* by slice of 64 entries */
+        target_fd_trans = g_renew(TargetFdTrans *,
+                                  target_fd_trans, target_fd_max);
+        memset((void *)(target_fd_trans + oldmax), 0,
+               (target_fd_max - oldmax) * sizeof(TargetFdTrans *));
+    }
+    target_fd_trans[fd] = trans;
+}
+
+static void fd_trans_unregister(int fd)
+{
+    if (fd >= 0 && fd < target_fd_max) {
+        target_fd_trans[fd] = NULL;
+    }
+}
+
+static void fd_trans_dup(int oldfd, int newfd)
+{
+    fd_trans_unregister(newfd);
+    if (oldfd < target_fd_max && target_fd_trans[oldfd]) {
+        fd_trans_register(newfd, target_fd_trans[oldfd]);
+    }
+}
+
 static int sys_getcwd1(char *buf, size_t size)
 {
   if (getcwd(buf, size) == NULL) {
@@ -303,18 +519,6 @@
   return strlen(buf)+1;
 }
 
-static int sys_openat(int dirfd, const char *pathname, int flags, mode_t mode)
-{
-  /*
-   * open(2) has extra parameter 'mode' when called with
-   * flag O_CREAT.
-   */
-  if ((flags & O_CREAT) != 0) {
-      return (openat(dirfd, pathname, flags, mode));
-  }
-  return (openat(dirfd, pathname, flags));
-}
-
 #ifdef TARGET_NR_utimensat
 #ifdef CONFIG_UTIMENSAT
 static int sys_utimensat(int dirfd, const char *pathname,
@@ -376,25 +580,6 @@
 #undef TARGET_NR_inotify_rm_watch
 #endif /* CONFIG_INOTIFY  */
 
-#if defined(TARGET_NR_ppoll)
-#ifndef __NR_ppoll
-# define __NR_ppoll -1
-#endif
-#define __NR_sys_ppoll __NR_ppoll
-_syscall5(int, sys_ppoll, struct pollfd *, fds, nfds_t, nfds,
-          struct timespec *, timeout, const sigset_t *, sigmask,
-          size_t, sigsetsize)
-#endif
-
-#if defined(TARGET_NR_pselect6)
-#ifndef __NR_pselect6
-# define __NR_pselect6 -1
-#endif
-#define __NR_sys_pselect6 __NR_pselect6
-_syscall6(int, sys_pselect6, int, nfds, fd_set *, readfds, fd_set *, writefds,
-          fd_set *, exceptfds, struct timespec *, timeout, void *, sig);
-#endif
-
 #if defined(TARGET_NR_prlimit64)
 #ifndef __NR_prlimit64
 # define __NR_prlimit64 -1
@@ -457,6 +642,7 @@
  * minus the errnos that are not actually generic to all archs.
  */
 static uint16_t host_to_target_errno_table[ERRNO_TABLE_SIZE] = {
+    [EAGAIN]		= TARGET_EAGAIN,
     [EIDRM]		= TARGET_EIDRM,
     [ECHRNG]		= TARGET_ECHRNG,
     [EL2NSYNC]		= TARGET_EL2NSYNC,
@@ -566,15 +752,19 @@
 
 static inline int host_to_target_errno(int err)
 {
-    if(host_to_target_errno_table[err])
+    if (err >= 0 && err < ERRNO_TABLE_SIZE &&
+        host_to_target_errno_table[err]) {
         return host_to_target_errno_table[err];
+    }
     return err;
 }
 
 static inline int target_to_host_errno(int err)
 {
-    if (target_to_host_errno_table[err])
+    if (err >= 0 && err < ERRNO_TABLE_SIZE &&
+        target_to_host_errno_table[err]) {
         return target_to_host_errno_table[err];
+    }
     return err;
 }
 
@@ -591,14 +781,171 @@
     return (abi_ulong)ret >= (abi_ulong)(-4096);
 }
 
-char *target_strerror(int err)
+const char *target_strerror(int err)
 {
+    if (err == TARGET_ERESTARTSYS) {
+        return "To be restarted";
+    }
+    if (err == TARGET_QEMU_ESIGRETURN) {
+        return "Successful exit from sigreturn";
+    }
+
     if ((err >= ERRNO_TABLE_SIZE) || (err < 0)) {
         return NULL;
     }
     return strerror(target_to_host_errno(err));
 }
 
+#define safe_syscall0(type, name) \
+static type safe_##name(void) \
+{ \
+    return safe_syscall(__NR_##name); \
+}
+
+#define safe_syscall1(type, name, type1, arg1) \
+static type safe_##name(type1 arg1) \
+{ \
+    return safe_syscall(__NR_##name, arg1); \
+}
+
+#define safe_syscall2(type, name, type1, arg1, type2, arg2) \
+static type safe_##name(type1 arg1, type2 arg2) \
+{ \
+    return safe_syscall(__NR_##name, arg1, arg2); \
+}
+
+#define safe_syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \
+static type safe_##name(type1 arg1, type2 arg2, type3 arg3) \
+{ \
+    return safe_syscall(__NR_##name, arg1, arg2, arg3); \
+}
+
+#define safe_syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \
+    type4, arg4) \
+static type safe_##name(type1 arg1, type2 arg2, type3 arg3, type4 arg4) \
+{ \
+    return safe_syscall(__NR_##name, arg1, arg2, arg3, arg4); \
+}
+
+#define safe_syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \
+    type4, arg4, type5, arg5) \
+static type safe_##name(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \
+    type5 arg5) \
+{ \
+    return safe_syscall(__NR_##name, arg1, arg2, arg3, arg4, arg5); \
+}
+
+#define safe_syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \
+    type4, arg4, type5, arg5, type6, arg6) \
+static type safe_##name(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \
+    type5 arg5, type6 arg6) \
+{ \
+    return safe_syscall(__NR_##name, arg1, arg2, arg3, arg4, arg5, arg6); \
+}
+
+safe_syscall3(ssize_t, read, int, fd, void *, buff, size_t, count)
+safe_syscall3(ssize_t, write, int, fd, const void *, buff, size_t, count)
+safe_syscall4(int, openat, int, dirfd, const char *, pathname, \
+              int, flags, mode_t, mode)
+safe_syscall4(pid_t, wait4, pid_t, pid, int *, status, int, options, \
+              struct rusage *, rusage)
+safe_syscall5(int, waitid, idtype_t, idtype, id_t, id, siginfo_t *, infop, \
+              int, options, struct rusage *, rusage)
+safe_syscall3(int, execve, const char *, filename, char **, argv, char **, envp)
+safe_syscall6(int, pselect6, int, nfds, fd_set *, readfds, fd_set *, writefds, \
+              fd_set *, exceptfds, struct timespec *, timeout, void *, sig)
+safe_syscall5(int, ppoll, struct pollfd *, ufds, unsigned int, nfds,
+              struct timespec *, tsp, const sigset_t *, sigmask,
+              size_t, sigsetsize)
+safe_syscall6(int, epoll_pwait, int, epfd, struct epoll_event *, events,
+              int, maxevents, int, timeout, const sigset_t *, sigmask,
+              size_t, sigsetsize)
+safe_syscall6(int,futex,int *,uaddr,int,op,int,val, \
+              const struct timespec *,timeout,int *,uaddr2,int,val3)
+safe_syscall2(int, rt_sigsuspend, sigset_t *, newset, size_t, sigsetsize)
+safe_syscall2(int, kill, pid_t, pid, int, sig)
+safe_syscall2(int, tkill, int, tid, int, sig)
+safe_syscall3(int, tgkill, int, tgid, int, pid, int, sig)
+safe_syscall3(ssize_t, readv, int, fd, const struct iovec *, iov, int, iovcnt)
+safe_syscall3(ssize_t, writev, int, fd, const struct iovec *, iov, int, iovcnt)
+safe_syscall3(int, connect, int, fd, const struct sockaddr *, addr,
+              socklen_t, addrlen)
+safe_syscall6(ssize_t, sendto, int, fd, const void *, buf, size_t, len,
+              int, flags, const struct sockaddr *, addr, socklen_t, addrlen)
+safe_syscall6(ssize_t, recvfrom, int, fd, void *, buf, size_t, len,
+              int, flags, struct sockaddr *, addr, socklen_t *, addrlen)
+safe_syscall3(ssize_t, sendmsg, int, fd, const struct msghdr *, msg, int, flags)
+safe_syscall3(ssize_t, recvmsg, int, fd, struct msghdr *, msg, int, flags)
+safe_syscall2(int, flock, int, fd, int, operation)
+safe_syscall4(int, rt_sigtimedwait, const sigset_t *, these, siginfo_t *, uinfo,
+              const struct timespec *, uts, size_t, sigsetsize)
+safe_syscall4(int, accept4, int, fd, struct sockaddr *, addr, socklen_t *, len,
+              int, flags)
+safe_syscall2(int, nanosleep, const struct timespec *, req,
+              struct timespec *, rem)
+#ifdef TARGET_NR_clock_nanosleep
+safe_syscall4(int, clock_nanosleep, const clockid_t, clock, int, flags,
+              const struct timespec *, req, struct timespec *, rem)
+#endif
+#ifdef __NR_msgsnd
+safe_syscall4(int, msgsnd, int, msgid, const void *, msgp, size_t, sz,
+              int, flags)
+safe_syscall5(int, msgrcv, int, msgid, void *, msgp, size_t, sz,
+              long, msgtype, int, flags)
+safe_syscall4(int, semtimedop, int, semid, struct sembuf *, tsops,
+              unsigned, nsops, const struct timespec *, timeout)
+#else
+/* This host kernel architecture uses a single ipc syscall; fake up
+ * wrappers for the sub-operations to hide this implementation detail.
+ * Annoyingly we can't include linux/ipc.h to get the constant definitions
+ * for the call parameter because some structs in there conflict with the
+ * sys/ipc.h ones. So we just define them here, and rely on them being
+ * the same for all host architectures.
+ */
+#define Q_SEMTIMEDOP 4
+#define Q_MSGSND 11
+#define Q_MSGRCV 12
+#define Q_IPCCALL(VERSION, OP) ((VERSION) << 16 | (OP))
+
+safe_syscall6(int, ipc, int, call, long, first, long, second, long, third,
+              void *, ptr, long, fifth)
+static int safe_msgsnd(int msgid, const void *msgp, size_t sz, int flags)
+{
+    return safe_ipc(Q_IPCCALL(0, Q_MSGSND), msgid, sz, flags, (void *)msgp, 0);
+}
+static int safe_msgrcv(int msgid, void *msgp, size_t sz, long type, int flags)
+{
+    return safe_ipc(Q_IPCCALL(1, Q_MSGRCV), msgid, sz, flags, msgp, type);
+}
+static int safe_semtimedop(int semid, struct sembuf *tsops, unsigned nsops,
+                           const struct timespec *timeout)
+{
+    return safe_ipc(Q_IPCCALL(0, Q_SEMTIMEDOP), semid, nsops, 0, tsops,
+                    (long)timeout);
+}
+#endif
+#if defined(TARGET_NR_mq_open) && defined(__NR_mq_open)
+safe_syscall5(int, mq_timedsend, int, mqdes, const char *, msg_ptr,
+              size_t, len, unsigned, prio, const struct timespec *, timeout)
+safe_syscall5(int, mq_timedreceive, int, mqdes, char *, msg_ptr,
+              size_t, len, unsigned *, prio, const struct timespec *, timeout)
+#endif
+/* We do ioctl like this rather than via safe_syscall3 to preserve the
+ * "third argument might be integer or pointer or not present" behaviour of
+ * the libc function.
+ */
+#define safe_ioctl(...) safe_syscall(__NR_ioctl, __VA_ARGS__)
+/* Similarly for fcntl. Note that callers must always:
+ *  pass the F_GETLK64 etc constants rather than the unsuffixed F_GETLK
+ *  use the flock64 struct rather than unsuffixed flock
+ * This will then work and use a 64-bit offset for both 32-bit and 64-bit hosts.
+ */
+#ifdef __NR_fcntl64
+#define safe_fcntl(...) safe_syscall(__NR_fcntl64, __VA_ARGS__)
+#else
+#define safe_fcntl(...) safe_syscall(__NR_fcntl, __VA_ARGS__)
+#endif
+
 static inline int host_to_target_sock_type(int host_type)
 {
     int target_type;
@@ -647,7 +994,7 @@
 abi_long do_brk(abi_ulong new_brk)
 {
     abi_long mapped_addr;
-    int	new_alloc_size;
+    abi_ulong new_alloc_size;
 
     DEBUGF_BRK("do_brk(" TARGET_ABI_FMT_lx ") -> ", new_brk);
 
@@ -728,7 +1075,7 @@
     int i, nw, j, k;
     abi_ulong b, *target_fds;
 
-    nw = (n + TARGET_ABI_BITS - 1) / TARGET_ABI_BITS;
+    nw = DIV_ROUND_UP(n, TARGET_ABI_BITS);
     if (!(target_fds = lock_user(VERIFY_READ,
                                  target_fds_addr,
                                  sizeof(abi_ulong) * nw,
@@ -775,7 +1122,7 @@
     abi_long v;
     abi_ulong *target_fds;
 
-    nw = (n + TARGET_ABI_BITS - 1) / TARGET_ABI_BITS;
+    nw = DIV_ROUND_UP(n, TARGET_ABI_BITS);
     if (!(target_fds = lock_user(VERIFY_WRITE,
                                  target_fds_addr,
                                  sizeof(abi_ulong) * nw,
@@ -1009,7 +1356,8 @@
 {
     fd_set rfds, wfds, efds;
     fd_set *rfds_ptr, *wfds_ptr, *efds_ptr;
-    struct timeval tv, *tv_ptr;
+    struct timeval tv;
+    struct timespec ts, *ts_ptr;
     abi_long ret;
 
     ret = copy_from_user_fdset_ptr(&rfds, &rfds_ptr, rfd_addr, n);
@@ -1028,12 +1376,15 @@
     if (target_tv_addr) {
         if (copy_from_user_timeval(&tv, target_tv_addr))
             return -TARGET_EFAULT;
-        tv_ptr = &tv;
+        ts.tv_sec = tv.tv_sec;
+        ts.tv_nsec = tv.tv_usec * 1000;
+        ts_ptr = &ts;
     } else {
-        tv_ptr = NULL;
+        ts_ptr = NULL;
     }
 
-    ret = get_errno(select(n, rfds_ptr, wfds_ptr, efds_ptr, tv_ptr));
+    ret = get_errno(safe_pselect6(n, rfds_ptr, wfds_ptr, efds_ptr,
+                                  ts_ptr, NULL));
 
     if (!is_error(ret)) {
         if (rfd_addr && copy_to_user_fdset(rfd_addr, &rfds, n))
@@ -1043,8 +1394,13 @@
         if (efd_addr && copy_to_user_fdset(efd_addr, &efds, n))
             return -TARGET_EFAULT;
 
-        if (target_tv_addr && copy_to_user_timeval(target_tv_addr, &tv))
-            return -TARGET_EFAULT;
+        if (target_tv_addr) {
+            tv.tv_sec = ts.tv_sec;
+            tv.tv_usec = ts.tv_nsec / 1000;
+            if (copy_to_user_timeval(target_tv_addr, &tv)) {
+                return -TARGET_EFAULT;
+            }
+        }
     }
 
     return ret;
@@ -1112,7 +1468,7 @@
     return 0;
 }
 
-static inline abi_long target_to_host_sockaddr(struct sockaddr *addr,
+static inline abi_long target_to_host_sockaddr(int fd, struct sockaddr *addr,
                                                abi_ulong target_addr,
                                                socklen_t len)
 {
@@ -1120,6 +1476,10 @@
     sa_family_t sa_family;
     struct target_sockaddr *target_saddr;
 
+    if (fd_trans_target_to_host_addr(fd)) {
+        return fd_trans_target_to_host_addr(fd)(addr, target_addr, len);
+    }
+
     target_saddr = lock_user(VERIFY_READ, target_addr, len, 1);
     if (!target_saddr)
         return -TARGET_EFAULT;
@@ -1147,7 +1507,13 @@
 
     memcpy(addr, target_saddr, len);
     addr->sa_family = sa_family;
-    if (sa_family == AF_PACKET) {
+    if (sa_family == AF_NETLINK) {
+        struct sockaddr_nl *nladdr;
+
+        nladdr = (struct sockaddr_nl *)addr;
+        nladdr->nl_pid = tswap32(nladdr->nl_pid);
+        nladdr->nl_groups = tswap32(nladdr->nl_groups);
+    } else if (sa_family == AF_PACKET) {
 	struct target_sockaddr_ll *lladdr;
 
 	lladdr = (struct target_sockaddr_ll *)addr;
@@ -1165,11 +1531,27 @@
 {
     struct target_sockaddr *target_saddr;
 
+    if (len == 0) {
+        return 0;
+    }
+
     target_saddr = lock_user(VERIFY_WRITE, target_addr, len, 0);
     if (!target_saddr)
         return -TARGET_EFAULT;
     memcpy(target_saddr, addr, len);
-    target_saddr->sa_family = tswap16(addr->sa_family);
+    if (len >= offsetof(struct target_sockaddr, sa_family) +
+        sizeof(target_saddr->sa_family)) {
+        target_saddr->sa_family = tswap16(addr->sa_family);
+    }
+    if (addr->sa_family == AF_NETLINK && len >= sizeof(struct sockaddr_nl)) {
+        struct sockaddr_nl *target_nl = (struct sockaddr_nl *)target_saddr;
+        target_nl->nl_pid = tswap32(target_nl->nl_pid);
+        target_nl->nl_groups = tswap32(target_nl->nl_groups);
+    } else if (addr->sa_family == AF_PACKET) {
+        struct sockaddr_ll *target_ll = (struct sockaddr_ll *)target_saddr;
+        target_ll->sll_ifindex = tswap32(target_ll->sll_ifindex);
+        target_ll->sll_hatype = tswap16(target_ll->sll_hatype);
+    }
     unlock_user(target_saddr, target_addr, len);
 
     return 0;
@@ -1181,7 +1563,7 @@
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(msgh);
     abi_long msg_controllen;
     abi_ulong target_cmsg_addr;
-    struct target_cmsghdr *target_cmsg;
+    struct target_cmsghdr *target_cmsg, *target_cmsg_start;
     socklen_t space = 0;
     
     msg_controllen = tswapal(target_msgh->msg_controllen);
@@ -1189,6 +1571,7 @@
         goto the_end;
     target_cmsg_addr = tswapal(target_msgh->msg_control);
     target_cmsg = lock_user(VERIFY_READ, target_cmsg_addr, msg_controllen, 1);
+    target_cmsg_start = target_cmsg;
     if (!target_cmsg)
         return -TARGET_EFAULT;
 
@@ -1202,6 +1585,15 @@
         space += CMSG_SPACE(len);
         if (space > msgh->msg_controllen) {
             space -= CMSG_SPACE(len);
+            /* This is a QEMU bug, since we allocated the payload
+             * area ourselves (unlike overflow in host-to-target
+             * conversion, which is just the guest giving us a buffer
+             * that's too small). It can't happen for the payload types
+             * we currently support; if it becomes an issue in future
+             * we would need to improve our allocation strategy to
+             * something more intelligent than "twice the size of the
+             * target buffer we're reading from".
+             */
             gemu_log("Host cmsg overflow\n");
             break;
         }
@@ -1214,20 +1606,32 @@
         cmsg->cmsg_type = tswap32(target_cmsg->cmsg_type);
         cmsg->cmsg_len = CMSG_LEN(len);
 
-        if (cmsg->cmsg_level != SOL_SOCKET || cmsg->cmsg_type != SCM_RIGHTS) {
-            gemu_log("Unsupported ancillary data: %d/%d\n", cmsg->cmsg_level, cmsg->cmsg_type);
-            memcpy(data, target_data, len);
-        } else {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
             int *fd = (int *)data;
             int *target_fd = (int *)target_data;
             int i, numfds = len / sizeof(int);
 
-            for (i = 0; i < numfds; i++)
-                fd[i] = tswap32(target_fd[i]);
+            for (i = 0; i < numfds; i++) {
+                __get_user(fd[i], target_fd + i);
+            }
+        } else if (cmsg->cmsg_level == SOL_SOCKET
+               &&  cmsg->cmsg_type == SCM_CREDENTIALS) {
+            struct ucred *cred = (struct ucred *)data;
+            struct target_ucred *target_cred =
+                (struct target_ucred *)target_data;
+
+            __get_user(cred->pid, &target_cred->pid);
+            __get_user(cred->uid, &target_cred->uid);
+            __get_user(cred->gid, &target_cred->gid);
+        } else {
+            gemu_log("Unsupported ancillary data: %d/%d\n",
+                                        cmsg->cmsg_level, cmsg->cmsg_type);
+            memcpy(data, target_data, len);
         }
 
         cmsg = CMSG_NXTHDR(msgh, cmsg);
-        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg);
+        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg,
+                                         target_cmsg_start);
     }
     unlock_user(target_cmsg, target_cmsg_addr, 0);
  the_end:
@@ -1241,7 +1645,7 @@
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(msgh);
     abi_long msg_controllen;
     abi_ulong target_cmsg_addr;
-    struct target_cmsghdr *target_cmsg;
+    struct target_cmsghdr *target_cmsg, *target_cmsg_start;
     socklen_t space = 0;
 
     msg_controllen = tswapal(target_msgh->msg_controllen);
@@ -1249,6 +1653,7 @@
         goto the_end;
     target_cmsg_addr = tswapal(target_msgh->msg_control);
     target_cmsg = lock_user(VERIFY_WRITE, target_cmsg_addr, msg_controllen, 0);
+    target_cmsg_start = target_cmsg;
     if (!target_cmsg)
         return -TARGET_EFAULT;
 
@@ -1257,11 +1662,16 @@
         void *target_data = TARGET_CMSG_DATA(target_cmsg);
 
         int len = cmsg->cmsg_len - CMSG_ALIGN(sizeof (struct cmsghdr));
+        int tgt_len, tgt_space;
 
-        space += TARGET_CMSG_SPACE(len);
-        if (space > msg_controllen) {
-            space -= TARGET_CMSG_SPACE(len);
-            gemu_log("Target cmsg overflow\n");
+        /* We never copy a half-header but may copy half-data;
+         * this is Linux's behaviour in put_cmsg(). Note that
+         * truncation here is a guest problem (which we report
+         * to the guest via the CTRUNC bit), unlike truncation
+         * in target_to_host_cmsg, which is a QEMU bug.
+         */
+        if (msg_controllen < sizeof(struct cmsghdr)) {
+            target_msgh->msg_flags |= tswap32(MSG_CTRUNC);
             break;
         }
 
@@ -1271,8 +1681,35 @@
             target_cmsg->cmsg_level = tswap32(cmsg->cmsg_level);
         }
         target_cmsg->cmsg_type = tswap32(cmsg->cmsg_type);
-        target_cmsg->cmsg_len = tswapal(TARGET_CMSG_LEN(len));
 
+        tgt_len = TARGET_CMSG_LEN(len);
+
+        /* Payload types which need a different size of payload on
+         * the target must adjust tgt_len here.
+         */
+        switch (cmsg->cmsg_level) {
+        case SOL_SOCKET:
+            switch (cmsg->cmsg_type) {
+            case SO_TIMESTAMP:
+                tgt_len = sizeof(struct target_timeval);
+                break;
+            default:
+                break;
+            }
+        default:
+            break;
+        }
+
+        if (msg_controllen < tgt_len) {
+            target_msgh->msg_flags |= tswap32(MSG_CTRUNC);
+            tgt_len = msg_controllen;
+        }
+
+        /* We must now copy-and-convert len bytes of payload
+         * into tgt_len bytes of destination space. Bear in mind
+         * that in both source and destination we may be dealing
+         * with a truncated value!
+         */
         switch (cmsg->cmsg_level) {
         case SOL_SOCKET:
             switch (cmsg->cmsg_type) {
@@ -1280,10 +1717,11 @@
             {
                 int *fd = (int *)data;
                 int *target_fd = (int *)target_data;
-                int i, numfds = len / sizeof(int);
+                int i, numfds = tgt_len / sizeof(int);
 
-                for (i = 0; i < numfds; i++)
-                    target_fd[i] = tswap32(fd[i]);
+                for (i = 0; i < numfds; i++) {
+                    __put_user(fd[i], target_fd + i);
+                }
                 break;
             }
             case SO_TIMESTAMP:
@@ -1292,12 +1730,14 @@
                 struct target_timeval *target_tv =
                     (struct target_timeval *)target_data;
 
-                if (len != sizeof(struct timeval))
+                if (len != sizeof(struct timeval) ||
+                    tgt_len != sizeof(struct target_timeval)) {
                     goto unimplemented;
+                }
 
                 /* copy struct timeval to target */
-                target_tv->tv_sec = tswapal(tv->tv_sec);
-                target_tv->tv_usec = tswapal(tv->tv_usec);
+                __put_user(tv->tv_sec, &target_tv->tv_sec);
+                __put_user(tv->tv_usec, &target_tv->tv_usec);
                 break;
             }
             case SCM_CREDENTIALS:
@@ -1320,11 +1760,22 @@
         unimplemented:
             gemu_log("Unsupported ancillary data: %d/%d\n",
                                         cmsg->cmsg_level, cmsg->cmsg_type);
-            memcpy(target_data, data, len);
+            memcpy(target_data, data, MIN(len, tgt_len));
+            if (tgt_len > len) {
+                memset(target_data + len, 0, tgt_len - len);
+            }
         }
 
+        target_cmsg->cmsg_len = tswapal(tgt_len);
+        tgt_space = TARGET_CMSG_SPACE(len);
+        if (msg_controllen < tgt_space) {
+            tgt_space = msg_controllen;
+        }
+        msg_controllen -= tgt_space;
+        space += tgt_space;
         cmsg = CMSG_NXTHDR(msgh, cmsg);
-        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg);
+        target_cmsg = TARGET_CMSG_NXTHDR(target_msgh, target_cmsg,
+                                         target_cmsg_start);
     }
     unlock_user(target_cmsg, target_cmsg_addr, space);
  the_end:
@@ -1332,6 +1783,875 @@
     return 0;
 }
 
+static void tswap_nlmsghdr(struct nlmsghdr *nlh)
+{
+    nlh->nlmsg_len = tswap32(nlh->nlmsg_len);
+    nlh->nlmsg_type = tswap16(nlh->nlmsg_type);
+    nlh->nlmsg_flags = tswap16(nlh->nlmsg_flags);
+    nlh->nlmsg_seq = tswap32(nlh->nlmsg_seq);
+    nlh->nlmsg_pid = tswap32(nlh->nlmsg_pid);
+}
+
+static abi_long host_to_target_for_each_nlmsg(struct nlmsghdr *nlh,
+                                              size_t len,
+                                              abi_long (*host_to_target_nlmsg)
+                                                       (struct nlmsghdr *))
+{
+    uint32_t nlmsg_len;
+    abi_long ret;
+
+    while (len > sizeof(struct nlmsghdr)) {
+
+        nlmsg_len = nlh->nlmsg_len;
+        if (nlmsg_len < sizeof(struct nlmsghdr) ||
+            nlmsg_len > len) {
+            break;
+        }
+
+        switch (nlh->nlmsg_type) {
+        case NLMSG_DONE:
+            tswap_nlmsghdr(nlh);
+            return 0;
+        case NLMSG_NOOP:
+            break;
+        case NLMSG_ERROR:
+        {
+            struct nlmsgerr *e = NLMSG_DATA(nlh);
+            e->error = tswap32(e->error);
+            tswap_nlmsghdr(&e->msg);
+            tswap_nlmsghdr(nlh);
+            return 0;
+        }
+        default:
+            ret = host_to_target_nlmsg(nlh);
+            if (ret < 0) {
+                tswap_nlmsghdr(nlh);
+                return ret;
+            }
+            break;
+        }
+        tswap_nlmsghdr(nlh);
+        len -= NLMSG_ALIGN(nlmsg_len);
+        nlh = (struct nlmsghdr *)(((char*)nlh) + NLMSG_ALIGN(nlmsg_len));
+    }
+    return 0;
+}
+
+static abi_long target_to_host_for_each_nlmsg(struct nlmsghdr *nlh,
+                                              size_t len,
+                                              abi_long (*target_to_host_nlmsg)
+                                                       (struct nlmsghdr *))
+{
+    int ret;
+
+    while (len > sizeof(struct nlmsghdr)) {
+        if (tswap32(nlh->nlmsg_len) < sizeof(struct nlmsghdr) ||
+            tswap32(nlh->nlmsg_len) > len) {
+            break;
+        }
+        tswap_nlmsghdr(nlh);
+        switch (nlh->nlmsg_type) {
+        case NLMSG_DONE:
+            return 0;
+        case NLMSG_NOOP:
+            break;
+        case NLMSG_ERROR:
+        {
+            struct nlmsgerr *e = NLMSG_DATA(nlh);
+            e->error = tswap32(e->error);
+            tswap_nlmsghdr(&e->msg);
+            return 0;
+        }
+        default:
+            ret = target_to_host_nlmsg(nlh);
+            if (ret < 0) {
+                return ret;
+            }
+        }
+        len -= NLMSG_ALIGN(nlh->nlmsg_len);
+        nlh = (struct nlmsghdr *)(((char *)nlh) + NLMSG_ALIGN(nlh->nlmsg_len));
+    }
+    return 0;
+}
+
+#ifdef CONFIG_RTNETLINK
+static abi_long host_to_target_for_each_nlattr(struct nlattr *nlattr,
+                                               size_t len, void *context,
+                                               abi_long (*host_to_target_nlattr)
+                                                        (struct nlattr *,
+                                                         void *context))
+{
+    unsigned short nla_len;
+    abi_long ret;
+
+    while (len > sizeof(struct nlattr)) {
+        nla_len = nlattr->nla_len;
+        if (nla_len < sizeof(struct nlattr) ||
+            nla_len > len) {
+            break;
+        }
+        ret = host_to_target_nlattr(nlattr, context);
+        nlattr->nla_len = tswap16(nlattr->nla_len);
+        nlattr->nla_type = tswap16(nlattr->nla_type);
+        if (ret < 0) {
+            return ret;
+        }
+        len -= NLA_ALIGN(nla_len);
+        nlattr = (struct nlattr *)(((char *)nlattr) + NLA_ALIGN(nla_len));
+    }
+    return 0;
+}
+
+static abi_long host_to_target_for_each_rtattr(struct rtattr *rtattr,
+                                               size_t len,
+                                               abi_long (*host_to_target_rtattr)
+                                                        (struct rtattr *))
+{
+    unsigned short rta_len;
+    abi_long ret;
+
+    while (len > sizeof(struct rtattr)) {
+        rta_len = rtattr->rta_len;
+        if (rta_len < sizeof(struct rtattr) ||
+            rta_len > len) {
+            break;
+        }
+        ret = host_to_target_rtattr(rtattr);
+        rtattr->rta_len = tswap16(rtattr->rta_len);
+        rtattr->rta_type = tswap16(rtattr->rta_type);
+        if (ret < 0) {
+            return ret;
+        }
+        len -= RTA_ALIGN(rta_len);
+        rtattr = (struct rtattr *)(((char *)rtattr) + RTA_ALIGN(rta_len));
+    }
+    return 0;
+}
+
+#define NLA_DATA(nla) ((void *)((char *)(nla)) + NLA_HDRLEN)
+
+static abi_long host_to_target_data_bridge_nlattr(struct nlattr *nlattr,
+                                                  void *context)
+{
+    uint16_t *u16;
+    uint32_t *u32;
+    uint64_t *u64;
+
+    switch (nlattr->nla_type) {
+    /* no data */
+    case QEMU_IFLA_BR_FDB_FLUSH:
+        break;
+    /* binary */
+    case QEMU_IFLA_BR_GROUP_ADDR:
+        break;
+    /* uint8_t */
+    case QEMU_IFLA_BR_VLAN_FILTERING:
+    case QEMU_IFLA_BR_TOPOLOGY_CHANGE:
+    case QEMU_IFLA_BR_TOPOLOGY_CHANGE_DETECTED:
+    case QEMU_IFLA_BR_MCAST_ROUTER:
+    case QEMU_IFLA_BR_MCAST_SNOOPING:
+    case QEMU_IFLA_BR_MCAST_QUERY_USE_IFADDR:
+    case QEMU_IFLA_BR_MCAST_QUERIER:
+    case QEMU_IFLA_BR_NF_CALL_IPTABLES:
+    case QEMU_IFLA_BR_NF_CALL_IP6TABLES:
+    case QEMU_IFLA_BR_NF_CALL_ARPTABLES:
+        break;
+    /* uint16_t */
+    case QEMU_IFLA_BR_PRIORITY:
+    case QEMU_IFLA_BR_VLAN_PROTOCOL:
+    case QEMU_IFLA_BR_GROUP_FWD_MASK:
+    case QEMU_IFLA_BR_ROOT_PORT:
+    case QEMU_IFLA_BR_VLAN_DEFAULT_PVID:
+        u16 = NLA_DATA(nlattr);
+        *u16 = tswap16(*u16);
+        break;
+    /* uint32_t */
+    case QEMU_IFLA_BR_FORWARD_DELAY:
+    case QEMU_IFLA_BR_HELLO_TIME:
+    case QEMU_IFLA_BR_MAX_AGE:
+    case QEMU_IFLA_BR_AGEING_TIME:
+    case QEMU_IFLA_BR_STP_STATE:
+    case QEMU_IFLA_BR_ROOT_PATH_COST:
+    case QEMU_IFLA_BR_MCAST_HASH_ELASTICITY:
+    case QEMU_IFLA_BR_MCAST_HASH_MAX:
+    case QEMU_IFLA_BR_MCAST_LAST_MEMBER_CNT:
+    case QEMU_IFLA_BR_MCAST_STARTUP_QUERY_CNT:
+        u32 = NLA_DATA(nlattr);
+        *u32 = tswap32(*u32);
+        break;
+    /* uint64_t */
+    case QEMU_IFLA_BR_HELLO_TIMER:
+    case QEMU_IFLA_BR_TCN_TIMER:
+    case QEMU_IFLA_BR_GC_TIMER:
+    case QEMU_IFLA_BR_TOPOLOGY_CHANGE_TIMER:
+    case QEMU_IFLA_BR_MCAST_LAST_MEMBER_INTVL:
+    case QEMU_IFLA_BR_MCAST_MEMBERSHIP_INTVL:
+    case QEMU_IFLA_BR_MCAST_QUERIER_INTVL:
+    case QEMU_IFLA_BR_MCAST_QUERY_INTVL:
+    case QEMU_IFLA_BR_MCAST_QUERY_RESPONSE_INTVL:
+    case QEMU_IFLA_BR_MCAST_STARTUP_QUERY_INTVL:
+        u64 = NLA_DATA(nlattr);
+        *u64 = tswap64(*u64);
+        break;
+    /* ifla_bridge_id: uin8_t[] */
+    case QEMU_IFLA_BR_ROOT_ID:
+    case QEMU_IFLA_BR_BRIDGE_ID:
+        break;
+    default:
+        gemu_log("Unknown QEMU_IFLA_BR type %d\n", nlattr->nla_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long host_to_target_slave_data_bridge_nlattr(struct nlattr *nlattr,
+                                                        void *context)
+{
+    uint16_t *u16;
+    uint32_t *u32;
+    uint64_t *u64;
+
+    switch (nlattr->nla_type) {
+    /* uint8_t */
+    case QEMU_IFLA_BRPORT_STATE:
+    case QEMU_IFLA_BRPORT_MODE:
+    case QEMU_IFLA_BRPORT_GUARD:
+    case QEMU_IFLA_BRPORT_PROTECT:
+    case QEMU_IFLA_BRPORT_FAST_LEAVE:
+    case QEMU_IFLA_BRPORT_LEARNING:
+    case QEMU_IFLA_BRPORT_UNICAST_FLOOD:
+    case QEMU_IFLA_BRPORT_PROXYARP:
+    case QEMU_IFLA_BRPORT_LEARNING_SYNC:
+    case QEMU_IFLA_BRPORT_PROXYARP_WIFI:
+    case QEMU_IFLA_BRPORT_TOPOLOGY_CHANGE_ACK:
+    case QEMU_IFLA_BRPORT_CONFIG_PENDING:
+    case QEMU_IFLA_BRPORT_MULTICAST_ROUTER:
+        break;
+    /* uint16_t */
+    case QEMU_IFLA_BRPORT_PRIORITY:
+    case QEMU_IFLA_BRPORT_DESIGNATED_PORT:
+    case QEMU_IFLA_BRPORT_DESIGNATED_COST:
+    case QEMU_IFLA_BRPORT_ID:
+    case QEMU_IFLA_BRPORT_NO:
+        u16 = NLA_DATA(nlattr);
+        *u16 = tswap16(*u16);
+        break;
+    /* uin32_t */
+    case QEMU_IFLA_BRPORT_COST:
+        u32 = NLA_DATA(nlattr);
+        *u32 = tswap32(*u32);
+        break;
+    /* uint64_t */
+    case QEMU_IFLA_BRPORT_MESSAGE_AGE_TIMER:
+    case QEMU_IFLA_BRPORT_FORWARD_DELAY_TIMER:
+    case QEMU_IFLA_BRPORT_HOLD_TIMER:
+        u64 = NLA_DATA(nlattr);
+        *u64 = tswap64(*u64);
+        break;
+    /* ifla_bridge_id: uint8_t[] */
+    case QEMU_IFLA_BRPORT_ROOT_ID:
+    case QEMU_IFLA_BRPORT_BRIDGE_ID:
+        break;
+    default:
+        gemu_log("Unknown QEMU_IFLA_BRPORT type %d\n", nlattr->nla_type);
+        break;
+    }
+    return 0;
+}
+
+struct linkinfo_context {
+    int len;
+    char *name;
+    int slave_len;
+    char *slave_name;
+};
+
+static abi_long host_to_target_data_linkinfo_nlattr(struct nlattr *nlattr,
+                                                    void *context)
+{
+    struct linkinfo_context *li_context = context;
+
+    switch (nlattr->nla_type) {
+    /* string */
+    case QEMU_IFLA_INFO_KIND:
+        li_context->name = NLA_DATA(nlattr);
+        li_context->len = nlattr->nla_len - NLA_HDRLEN;
+        break;
+    case QEMU_IFLA_INFO_SLAVE_KIND:
+        li_context->slave_name = NLA_DATA(nlattr);
+        li_context->slave_len = nlattr->nla_len - NLA_HDRLEN;
+        break;
+    /* stats */
+    case QEMU_IFLA_INFO_XSTATS:
+        /* FIXME: only used by CAN */
+        break;
+    /* nested */
+    case QEMU_IFLA_INFO_DATA:
+        if (strncmp(li_context->name, "bridge",
+                    li_context->len) == 0) {
+            return host_to_target_for_each_nlattr(NLA_DATA(nlattr),
+                                                  nlattr->nla_len,
+                                                  NULL,
+                                             host_to_target_data_bridge_nlattr);
+        } else {
+            gemu_log("Unknown QEMU_IFLA_INFO_KIND %s\n", li_context->name);
+        }
+        break;
+    case QEMU_IFLA_INFO_SLAVE_DATA:
+        if (strncmp(li_context->slave_name, "bridge",
+                    li_context->slave_len) == 0) {
+            return host_to_target_for_each_nlattr(NLA_DATA(nlattr),
+                                                  nlattr->nla_len,
+                                                  NULL,
+                                       host_to_target_slave_data_bridge_nlattr);
+        } else {
+            gemu_log("Unknown QEMU_IFLA_INFO_SLAVE_KIND %s\n",
+                     li_context->slave_name);
+        }
+        break;
+    default:
+        gemu_log("Unknown host QEMU_IFLA_INFO type: %d\n", nlattr->nla_type);
+        break;
+    }
+
+    return 0;
+}
+
+static abi_long host_to_target_data_inet_nlattr(struct nlattr *nlattr,
+                                                void *context)
+{
+    uint32_t *u32;
+    int i;
+
+    switch (nlattr->nla_type) {
+    case QEMU_IFLA_INET_CONF:
+        u32 = NLA_DATA(nlattr);
+        for (i = 0; i < (nlattr->nla_len - NLA_HDRLEN) / sizeof(*u32);
+             i++) {
+            u32[i] = tswap32(u32[i]);
+        }
+        break;
+    default:
+        gemu_log("Unknown host AF_INET type: %d\n", nlattr->nla_type);
+    }
+    return 0;
+}
+
+static abi_long host_to_target_data_inet6_nlattr(struct nlattr *nlattr,
+                                                void *context)
+{
+    uint32_t *u32;
+    uint64_t *u64;
+    struct ifla_cacheinfo *ci;
+    int i;
+
+    switch (nlattr->nla_type) {
+    /* binaries */
+    case QEMU_IFLA_INET6_TOKEN:
+        break;
+    /* uint8_t */
+    case QEMU_IFLA_INET6_ADDR_GEN_MODE:
+        break;
+    /* uint32_t */
+    case QEMU_IFLA_INET6_FLAGS:
+        u32 = NLA_DATA(nlattr);
+        *u32 = tswap32(*u32);
+        break;
+    /* uint32_t[] */
+    case QEMU_IFLA_INET6_CONF:
+        u32 = NLA_DATA(nlattr);
+        for (i = 0; i < (nlattr->nla_len - NLA_HDRLEN) / sizeof(*u32);
+             i++) {
+            u32[i] = tswap32(u32[i]);
+        }
+        break;
+    /* ifla_cacheinfo */
+    case QEMU_IFLA_INET6_CACHEINFO:
+        ci = NLA_DATA(nlattr);
+        ci->max_reasm_len = tswap32(ci->max_reasm_len);
+        ci->tstamp = tswap32(ci->tstamp);
+        ci->reachable_time = tswap32(ci->reachable_time);
+        ci->retrans_time = tswap32(ci->retrans_time);
+        break;
+    /* uint64_t[] */
+    case QEMU_IFLA_INET6_STATS:
+    case QEMU_IFLA_INET6_ICMP6STATS:
+        u64 = NLA_DATA(nlattr);
+        for (i = 0; i < (nlattr->nla_len - NLA_HDRLEN) / sizeof(*u64);
+             i++) {
+            u64[i] = tswap64(u64[i]);
+        }
+        break;
+    default:
+        gemu_log("Unknown host AF_INET6 type: %d\n", nlattr->nla_type);
+    }
+    return 0;
+}
+
+static abi_long host_to_target_data_spec_nlattr(struct nlattr *nlattr,
+                                                    void *context)
+{
+    switch (nlattr->nla_type) {
+    case AF_INET:
+        return host_to_target_for_each_nlattr(NLA_DATA(nlattr), nlattr->nla_len,
+                                              NULL,
+                                             host_to_target_data_inet_nlattr);
+    case AF_INET6:
+        return host_to_target_for_each_nlattr(NLA_DATA(nlattr), nlattr->nla_len,
+                                              NULL,
+                                             host_to_target_data_inet6_nlattr);
+    default:
+        gemu_log("Unknown host AF_SPEC type: %d\n", nlattr->nla_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long host_to_target_data_link_rtattr(struct rtattr *rtattr)
+{
+    uint32_t *u32;
+    struct rtnl_link_stats *st;
+    struct rtnl_link_stats64 *st64;
+    struct rtnl_link_ifmap *map;
+    struct linkinfo_context li_context;
+
+    switch (rtattr->rta_type) {
+    /* binary stream */
+    case QEMU_IFLA_ADDRESS:
+    case QEMU_IFLA_BROADCAST:
+    /* string */
+    case QEMU_IFLA_IFNAME:
+    case QEMU_IFLA_QDISC:
+        break;
+    /* uin8_t */
+    case QEMU_IFLA_OPERSTATE:
+    case QEMU_IFLA_LINKMODE:
+    case QEMU_IFLA_CARRIER:
+    case QEMU_IFLA_PROTO_DOWN:
+        break;
+    /* uint32_t */
+    case QEMU_IFLA_MTU:
+    case QEMU_IFLA_LINK:
+    case QEMU_IFLA_WEIGHT:
+    case QEMU_IFLA_TXQLEN:
+    case QEMU_IFLA_CARRIER_CHANGES:
+    case QEMU_IFLA_NUM_RX_QUEUES:
+    case QEMU_IFLA_NUM_TX_QUEUES:
+    case QEMU_IFLA_PROMISCUITY:
+    case QEMU_IFLA_EXT_MASK:
+    case QEMU_IFLA_LINK_NETNSID:
+    case QEMU_IFLA_GROUP:
+    case QEMU_IFLA_MASTER:
+    case QEMU_IFLA_NUM_VF:
+        u32 = RTA_DATA(rtattr);
+        *u32 = tswap32(*u32);
+        break;
+    /* struct rtnl_link_stats */
+    case QEMU_IFLA_STATS:
+        st = RTA_DATA(rtattr);
+        st->rx_packets = tswap32(st->rx_packets);
+        st->tx_packets = tswap32(st->tx_packets);
+        st->rx_bytes = tswap32(st->rx_bytes);
+        st->tx_bytes = tswap32(st->tx_bytes);
+        st->rx_errors = tswap32(st->rx_errors);
+        st->tx_errors = tswap32(st->tx_errors);
+        st->rx_dropped = tswap32(st->rx_dropped);
+        st->tx_dropped = tswap32(st->tx_dropped);
+        st->multicast = tswap32(st->multicast);
+        st->collisions = tswap32(st->collisions);
+
+        /* detailed rx_errors: */
+        st->rx_length_errors = tswap32(st->rx_length_errors);
+        st->rx_over_errors = tswap32(st->rx_over_errors);
+        st->rx_crc_errors = tswap32(st->rx_crc_errors);
+        st->rx_frame_errors = tswap32(st->rx_frame_errors);
+        st->rx_fifo_errors = tswap32(st->rx_fifo_errors);
+        st->rx_missed_errors = tswap32(st->rx_missed_errors);
+
+        /* detailed tx_errors */
+        st->tx_aborted_errors = tswap32(st->tx_aborted_errors);
+        st->tx_carrier_errors = tswap32(st->tx_carrier_errors);
+        st->tx_fifo_errors = tswap32(st->tx_fifo_errors);
+        st->tx_heartbeat_errors = tswap32(st->tx_heartbeat_errors);
+        st->tx_window_errors = tswap32(st->tx_window_errors);
+
+        /* for cslip etc */
+        st->rx_compressed = tswap32(st->rx_compressed);
+        st->tx_compressed = tswap32(st->tx_compressed);
+        break;
+    /* struct rtnl_link_stats64 */
+    case QEMU_IFLA_STATS64:
+        st64 = RTA_DATA(rtattr);
+        st64->rx_packets = tswap64(st64->rx_packets);
+        st64->tx_packets = tswap64(st64->tx_packets);
+        st64->rx_bytes = tswap64(st64->rx_bytes);
+        st64->tx_bytes = tswap64(st64->tx_bytes);
+        st64->rx_errors = tswap64(st64->rx_errors);
+        st64->tx_errors = tswap64(st64->tx_errors);
+        st64->rx_dropped = tswap64(st64->rx_dropped);
+        st64->tx_dropped = tswap64(st64->tx_dropped);
+        st64->multicast = tswap64(st64->multicast);
+        st64->collisions = tswap64(st64->collisions);
+
+        /* detailed rx_errors: */
+        st64->rx_length_errors = tswap64(st64->rx_length_errors);
+        st64->rx_over_errors = tswap64(st64->rx_over_errors);
+        st64->rx_crc_errors = tswap64(st64->rx_crc_errors);
+        st64->rx_frame_errors = tswap64(st64->rx_frame_errors);
+        st64->rx_fifo_errors = tswap64(st64->rx_fifo_errors);
+        st64->rx_missed_errors = tswap64(st64->rx_missed_errors);
+
+        /* detailed tx_errors */
+        st64->tx_aborted_errors = tswap64(st64->tx_aborted_errors);
+        st64->tx_carrier_errors = tswap64(st64->tx_carrier_errors);
+        st64->tx_fifo_errors = tswap64(st64->tx_fifo_errors);
+        st64->tx_heartbeat_errors = tswap64(st64->tx_heartbeat_errors);
+        st64->tx_window_errors = tswap64(st64->tx_window_errors);
+
+        /* for cslip etc */
+        st64->rx_compressed = tswap64(st64->rx_compressed);
+        st64->tx_compressed = tswap64(st64->tx_compressed);
+        break;
+    /* struct rtnl_link_ifmap */
+    case QEMU_IFLA_MAP:
+        map = RTA_DATA(rtattr);
+        map->mem_start = tswap64(map->mem_start);
+        map->mem_end = tswap64(map->mem_end);
+        map->base_addr = tswap64(map->base_addr);
+        map->irq = tswap16(map->irq);
+        break;
+    /* nested */
+    case QEMU_IFLA_LINKINFO:
+        memset(&li_context, 0, sizeof(li_context));
+        return host_to_target_for_each_nlattr(RTA_DATA(rtattr), rtattr->rta_len,
+                                              &li_context,
+                                           host_to_target_data_linkinfo_nlattr);
+    case QEMU_IFLA_AF_SPEC:
+        return host_to_target_for_each_nlattr(RTA_DATA(rtattr), rtattr->rta_len,
+                                              NULL,
+                                             host_to_target_data_spec_nlattr);
+    default:
+        gemu_log("Unknown host QEMU_IFLA type: %d\n", rtattr->rta_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long host_to_target_data_addr_rtattr(struct rtattr *rtattr)
+{
+    uint32_t *u32;
+    struct ifa_cacheinfo *ci;
+
+    switch (rtattr->rta_type) {
+    /* binary: depends on family type */
+    case IFA_ADDRESS:
+    case IFA_LOCAL:
+        break;
+    /* string */
+    case IFA_LABEL:
+        break;
+    /* u32 */
+    case IFA_FLAGS:
+    case IFA_BROADCAST:
+        u32 = RTA_DATA(rtattr);
+        *u32 = tswap32(*u32);
+        break;
+    /* struct ifa_cacheinfo */
+    case IFA_CACHEINFO:
+        ci = RTA_DATA(rtattr);
+        ci->ifa_prefered = tswap32(ci->ifa_prefered);
+        ci->ifa_valid = tswap32(ci->ifa_valid);
+        ci->cstamp = tswap32(ci->cstamp);
+        ci->tstamp = tswap32(ci->tstamp);
+        break;
+    default:
+        gemu_log("Unknown host IFA type: %d\n", rtattr->rta_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long host_to_target_data_route_rtattr(struct rtattr *rtattr)
+{
+    uint32_t *u32;
+    switch (rtattr->rta_type) {
+    /* binary: depends on family type */
+    case RTA_GATEWAY:
+    case RTA_DST:
+    case RTA_PREFSRC:
+        break;
+    /* u32 */
+    case RTA_PRIORITY:
+    case RTA_TABLE:
+    case RTA_OIF:
+        u32 = RTA_DATA(rtattr);
+        *u32 = tswap32(*u32);
+        break;
+    default:
+        gemu_log("Unknown host RTA type: %d\n", rtattr->rta_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long host_to_target_link_rtattr(struct rtattr *rtattr,
+                                         uint32_t rtattr_len)
+{
+    return host_to_target_for_each_rtattr(rtattr, rtattr_len,
+                                          host_to_target_data_link_rtattr);
+}
+
+static abi_long host_to_target_addr_rtattr(struct rtattr *rtattr,
+                                         uint32_t rtattr_len)
+{
+    return host_to_target_for_each_rtattr(rtattr, rtattr_len,
+                                          host_to_target_data_addr_rtattr);
+}
+
+static abi_long host_to_target_route_rtattr(struct rtattr *rtattr,
+                                         uint32_t rtattr_len)
+{
+    return host_to_target_for_each_rtattr(rtattr, rtattr_len,
+                                          host_to_target_data_route_rtattr);
+}
+
+static abi_long host_to_target_data_route(struct nlmsghdr *nlh)
+{
+    uint32_t nlmsg_len;
+    struct ifinfomsg *ifi;
+    struct ifaddrmsg *ifa;
+    struct rtmsg *rtm;
+
+    nlmsg_len = nlh->nlmsg_len;
+    switch (nlh->nlmsg_type) {
+    case RTM_NEWLINK:
+    case RTM_DELLINK:
+    case RTM_GETLINK:
+        if (nlh->nlmsg_len >= NLMSG_LENGTH(sizeof(*ifi))) {
+            ifi = NLMSG_DATA(nlh);
+            ifi->ifi_type = tswap16(ifi->ifi_type);
+            ifi->ifi_index = tswap32(ifi->ifi_index);
+            ifi->ifi_flags = tswap32(ifi->ifi_flags);
+            ifi->ifi_change = tswap32(ifi->ifi_change);
+            host_to_target_link_rtattr(IFLA_RTA(ifi),
+                                       nlmsg_len - NLMSG_LENGTH(sizeof(*ifi)));
+        }
+        break;
+    case RTM_NEWADDR:
+    case RTM_DELADDR:
+    case RTM_GETADDR:
+        if (nlh->nlmsg_len >= NLMSG_LENGTH(sizeof(*ifa))) {
+            ifa = NLMSG_DATA(nlh);
+            ifa->ifa_index = tswap32(ifa->ifa_index);
+            host_to_target_addr_rtattr(IFA_RTA(ifa),
+                                       nlmsg_len - NLMSG_LENGTH(sizeof(*ifa)));
+        }
+        break;
+    case RTM_NEWROUTE:
+    case RTM_DELROUTE:
+    case RTM_GETROUTE:
+        if (nlh->nlmsg_len >= NLMSG_LENGTH(sizeof(*rtm))) {
+            rtm = NLMSG_DATA(nlh);
+            rtm->rtm_flags = tswap32(rtm->rtm_flags);
+            host_to_target_route_rtattr(RTM_RTA(rtm),
+                                        nlmsg_len - NLMSG_LENGTH(sizeof(*rtm)));
+        }
+        break;
+    default:
+        return -TARGET_EINVAL;
+    }
+    return 0;
+}
+
+static inline abi_long host_to_target_nlmsg_route(struct nlmsghdr *nlh,
+                                                  size_t len)
+{
+    return host_to_target_for_each_nlmsg(nlh, len, host_to_target_data_route);
+}
+
+static abi_long target_to_host_for_each_rtattr(struct rtattr *rtattr,
+                                               size_t len,
+                                               abi_long (*target_to_host_rtattr)
+                                                        (struct rtattr *))
+{
+    abi_long ret;
+
+    while (len >= sizeof(struct rtattr)) {
+        if (tswap16(rtattr->rta_len) < sizeof(struct rtattr) ||
+            tswap16(rtattr->rta_len) > len) {
+            break;
+        }
+        rtattr->rta_len = tswap16(rtattr->rta_len);
+        rtattr->rta_type = tswap16(rtattr->rta_type);
+        ret = target_to_host_rtattr(rtattr);
+        if (ret < 0) {
+            return ret;
+        }
+        len -= RTA_ALIGN(rtattr->rta_len);
+        rtattr = (struct rtattr *)(((char *)rtattr) +
+                 RTA_ALIGN(rtattr->rta_len));
+    }
+    return 0;
+}
+
+static abi_long target_to_host_data_link_rtattr(struct rtattr *rtattr)
+{
+    switch (rtattr->rta_type) {
+    default:
+        gemu_log("Unknown target QEMU_IFLA type: %d\n", rtattr->rta_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long target_to_host_data_addr_rtattr(struct rtattr *rtattr)
+{
+    switch (rtattr->rta_type) {
+    /* binary: depends on family type */
+    case IFA_LOCAL:
+    case IFA_ADDRESS:
+        break;
+    default:
+        gemu_log("Unknown target IFA type: %d\n", rtattr->rta_type);
+        break;
+    }
+    return 0;
+}
+
+static abi_long target_to_host_data_route_rtattr(struct rtattr *rtattr)
+{
+    uint32_t *u32;
+    switch (rtattr->rta_type) {
+    /* binary: depends on family type */
+    case RTA_DST:
+    case RTA_SRC:
+    case RTA_GATEWAY:
+        break;
+    /* u32 */
+    case RTA_OIF:
+        u32 = RTA_DATA(rtattr);
+        *u32 = tswap32(*u32);
+        break;
+    default:
+        gemu_log("Unknown target RTA type: %d\n", rtattr->rta_type);
+        break;
+    }
+    return 0;
+}
+
+static void target_to_host_link_rtattr(struct rtattr *rtattr,
+                                       uint32_t rtattr_len)
+{
+    target_to_host_for_each_rtattr(rtattr, rtattr_len,
+                                   target_to_host_data_link_rtattr);
+}
+
+static void target_to_host_addr_rtattr(struct rtattr *rtattr,
+                                     uint32_t rtattr_len)
+{
+    target_to_host_for_each_rtattr(rtattr, rtattr_len,
+                                   target_to_host_data_addr_rtattr);
+}
+
+static void target_to_host_route_rtattr(struct rtattr *rtattr,
+                                     uint32_t rtattr_len)
+{
+    target_to_host_for_each_rtattr(rtattr, rtattr_len,
+                                   target_to_host_data_route_rtattr);
+}
+
+static abi_long target_to_host_data_route(struct nlmsghdr *nlh)
+{
+    struct ifinfomsg *ifi;
+    struct ifaddrmsg *ifa;
+    struct rtmsg *rtm;
+
+    switch (nlh->nlmsg_type) {
+    case RTM_GETLINK:
+        break;
+    case RTM_NEWLINK:
+    case RTM_DELLINK:
+        if (nlh->nlmsg_len >= NLMSG_LENGTH(sizeof(*ifi))) {
+            ifi = NLMSG_DATA(nlh);
+            ifi->ifi_type = tswap16(ifi->ifi_type);
+            ifi->ifi_index = tswap32(ifi->ifi_index);
+            ifi->ifi_flags = tswap32(ifi->ifi_flags);
+            ifi->ifi_change = tswap32(ifi->ifi_change);
+            target_to_host_link_rtattr(IFLA_RTA(ifi), nlh->nlmsg_len -
+                                       NLMSG_LENGTH(sizeof(*ifi)));
+        }
+        break;
+    case RTM_GETADDR:
+    case RTM_NEWADDR:
+    case RTM_DELADDR:
+        if (nlh->nlmsg_len >= NLMSG_LENGTH(sizeof(*ifa))) {
+            ifa = NLMSG_DATA(nlh);
+            ifa->ifa_index = tswap32(ifa->ifa_index);
+            target_to_host_addr_rtattr(IFA_RTA(ifa), nlh->nlmsg_len -
+                                       NLMSG_LENGTH(sizeof(*ifa)));
+        }
+        break;
+    case RTM_GETROUTE:
+        break;
+    case RTM_NEWROUTE:
+    case RTM_DELROUTE:
+        if (nlh->nlmsg_len >= NLMSG_LENGTH(sizeof(*rtm))) {
+            rtm = NLMSG_DATA(nlh);
+            rtm->rtm_flags = tswap32(rtm->rtm_flags);
+            target_to_host_route_rtattr(RTM_RTA(rtm), nlh->nlmsg_len -
+                                        NLMSG_LENGTH(sizeof(*rtm)));
+        }
+        break;
+    default:
+        return -TARGET_EOPNOTSUPP;
+    }
+    return 0;
+}
+
+static abi_long target_to_host_nlmsg_route(struct nlmsghdr *nlh, size_t len)
+{
+    return target_to_host_for_each_nlmsg(nlh, len, target_to_host_data_route);
+}
+#endif /* CONFIG_RTNETLINK */
+
+static abi_long host_to_target_data_audit(struct nlmsghdr *nlh)
+{
+    switch (nlh->nlmsg_type) {
+    default:
+        gemu_log("Unknown host audit message type %d\n",
+                 nlh->nlmsg_type);
+        return -TARGET_EINVAL;
+    }
+    return 0;
+}
+
+static inline abi_long host_to_target_nlmsg_audit(struct nlmsghdr *nlh,
+                                                  size_t len)
+{
+    return host_to_target_for_each_nlmsg(nlh, len, host_to_target_data_audit);
+}
+
+static abi_long target_to_host_data_audit(struct nlmsghdr *nlh)
+{
+    switch (nlh->nlmsg_type) {
+    case AUDIT_USER:
+    case AUDIT_FIRST_USER_MSG ... AUDIT_LAST_USER_MSG:
+    case AUDIT_FIRST_USER_MSG2 ... AUDIT_LAST_USER_MSG2:
+        break;
+    default:
+        gemu_log("Unknown target audit message type %d\n",
+                 nlh->nlmsg_type);
+        return -TARGET_EINVAL;
+    }
+
+    return 0;
+}
+
+static abi_long target_to_host_nlmsg_audit(struct nlmsghdr *nlh, size_t len)
+{
+    return target_to_host_for_each_nlmsg(nlh, len, target_to_host_data_audit);
+}
+
 /* do_setsockopt() Must return target values and target errnos. */
 static abi_long do_setsockopt(int sockfd, int level, int optname,
                               abi_ulong optval_addr, socklen_t optlen)
@@ -1489,7 +2809,7 @@
                 }
 
                 fprog.len = tswap16(tfprog->len);
-                filter = malloc(fprog.len * sizeof(*filter));
+                filter = g_try_new(struct sock_filter, fprog.len);
                 if (filter == NULL) {
                     unlock_user_struct(tfilter, tfprog->filter, 1);
                     unlock_user_struct(tfprog, optval_addr, 1);
@@ -1505,7 +2825,7 @@
 
                 ret = get_errno(setsockopt(sockfd, SOL_SOCKET,
                                 SO_ATTACH_FILTER, &fprog, sizeof(fprog)));
-                free(filter);
+                g_free(filter);
 
                 unlock_user_struct(tfilter, tfprog->filter, 1);
                 unlock_user_struct(tfprog, optval_addr, 1);
@@ -1526,7 +2846,8 @@
 		addr_ifname = alloca(IFNAMSIZ);
 		memcpy(addr_ifname, dev_ifname, optlen);
 		addr_ifname[optlen] = 0;
-		ret = get_errno(setsockopt(sockfd, level, optname, addr_ifname, optlen));
+		ret = get_errno(setsockopt(sockfd, SOL_SOCKET, optname,
+                                           addr_ifname, optlen));
 		unlock_user (dev_ifname, optval_addr, 0);
 		return ret;
 	}
@@ -1816,7 +3137,7 @@
         return NULL;
     }
 
-    vec = calloc(count, sizeof(struct iovec));
+    vec = g_try_new0(struct iovec, count);
     if (vec == NULL) {
         errno = ENOMEM;
         return NULL;
@@ -1873,9 +3194,14 @@
     return vec;
 
  fail:
+    while (--i >= 0) {
+        if (tswapal(target_vec[i].iov_len) > 0) {
+            unlock_user(vec[i].iov_base, tswapal(target_vec[i].iov_base), 0);
+        }
+    }
     unlock_user(target_vec, target_addr, 0);
  fail2:
-    free(vec);
+    g_free(vec);
     errno = err;
     return NULL;
 }
@@ -1891,7 +3217,7 @@
     if (target_vec) {
         for (i = 0; i < count; i++) {
             abi_ulong base = tswapal(target_vec[i].iov_base);
-            abi_long len = tswapal(target_vec[i].iov_base);
+            abi_long len = tswapal(target_vec[i].iov_len);
             if (len < 0) {
                 break;
             }
@@ -1900,7 +3226,7 @@
         unlock_user(target_vec, target_addr, 0);
     }
 
-    free(vec);
+    g_free(vec);
 }
 
 static inline int target_to_host_sock_type(int *type)
@@ -1952,6 +3278,90 @@
     return fd;
 }
 
+static abi_long packet_target_to_host_sockaddr(void *host_addr,
+                                               abi_ulong target_addr,
+                                               socklen_t len)
+{
+    struct sockaddr *addr = host_addr;
+    struct target_sockaddr *target_saddr;
+
+    target_saddr = lock_user(VERIFY_READ, target_addr, len, 1);
+    if (!target_saddr) {
+        return -TARGET_EFAULT;
+    }
+
+    memcpy(addr, target_saddr, len);
+    addr->sa_family = tswap16(target_saddr->sa_family);
+    /* spkt_protocol is big-endian */
+
+    unlock_user(target_saddr, target_addr, 0);
+    return 0;
+}
+
+static TargetFdTrans target_packet_trans = {
+    .target_to_host_addr = packet_target_to_host_sockaddr,
+};
+
+#ifdef CONFIG_RTNETLINK
+static abi_long netlink_route_target_to_host(void *buf, size_t len)
+{
+    abi_long ret;
+
+    ret = target_to_host_nlmsg_route(buf, len);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return len;
+}
+
+static abi_long netlink_route_host_to_target(void *buf, size_t len)
+{
+    abi_long ret;
+
+    ret = host_to_target_nlmsg_route(buf, len);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return len;
+}
+
+static TargetFdTrans target_netlink_route_trans = {
+    .target_to_host_data = netlink_route_target_to_host,
+    .host_to_target_data = netlink_route_host_to_target,
+};
+#endif /* CONFIG_RTNETLINK */
+
+static abi_long netlink_audit_target_to_host(void *buf, size_t len)
+{
+    abi_long ret;
+
+    ret = target_to_host_nlmsg_audit(buf, len);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return len;
+}
+
+static abi_long netlink_audit_host_to_target(void *buf, size_t len)
+{
+    abi_long ret;
+
+    ret = host_to_target_nlmsg_audit(buf, len);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return len;
+}
+
+static TargetFdTrans target_netlink_audit_trans = {
+    .target_to_host_data = netlink_audit_target_to_host,
+    .host_to_target_data = netlink_audit_host_to_target,
+};
+
 /* do_socket() Must return target values and target errnos. */
 static abi_long do_socket(int domain, int type, int protocol)
 {
@@ -1963,11 +3373,45 @@
         return ret;
     }
 
-    if (domain == PF_NETLINK)
-        return -TARGET_EAFNOSUPPORT;
+    if (domain == PF_NETLINK && !(
+#ifdef CONFIG_RTNETLINK
+         protocol == NETLINK_ROUTE ||
+#endif
+         protocol == NETLINK_KOBJECT_UEVENT ||
+         protocol == NETLINK_AUDIT)) {
+        return -EPFNOSUPPORT;
+    }
+
+    if (domain == AF_PACKET ||
+        (domain == AF_INET && type == SOCK_PACKET)) {
+        protocol = tswap16(protocol);
+    }
+
     ret = get_errno(socket(domain, type, protocol));
     if (ret >= 0) {
         ret = sock_flags_fixup(ret, target_type);
+        if (type == SOCK_PACKET) {
+            /* Manage an obsolete case :
+             * if socket type is SOCK_PACKET, bind by name
+             */
+            fd_trans_register(ret, &target_packet_trans);
+        } else if (domain == PF_NETLINK) {
+            switch (protocol) {
+#ifdef CONFIG_RTNETLINK
+            case NETLINK_ROUTE:
+                fd_trans_register(ret, &target_netlink_route_trans);
+                break;
+#endif
+            case NETLINK_KOBJECT_UEVENT:
+                /* nothing to do: messages are strings */
+                break;
+            case NETLINK_AUDIT:
+                fd_trans_register(ret, &target_netlink_audit_trans);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+        }
     }
     return ret;
 }
@@ -1985,7 +3429,7 @@
 
     addr = alloca(addrlen+1);
 
-    ret = target_to_host_sockaddr(addr, target_addr, addrlen);
+    ret = target_to_host_sockaddr(sockfd, addr, target_addr, addrlen);
     if (ret)
         return ret;
 
@@ -2005,11 +3449,11 @@
 
     addr = alloca(addrlen+1);
 
-    ret = target_to_host_sockaddr(addr, target_addr, addrlen);
+    ret = target_to_host_sockaddr(sockfd, addr, target_addr, addrlen);
     if (ret)
         return ret;
 
-    return get_errno(connect(sockfd, addr, addrlen));
+    return get_errno(safe_connect(sockfd, addr, addrlen));
 }
 
 /* do_sendrecvmsg_locked() Must return target values and target errnos. */
@@ -2025,8 +3469,9 @@
     if (msgp->msg_name) {
         msg.msg_namelen = tswap32(msgp->msg_namelen);
         msg.msg_name = alloca(msg.msg_namelen+1);
-        ret = target_to_host_sockaddr(msg.msg_name, tswapal(msgp->msg_name),
-                                msg.msg_namelen);
+        ret = target_to_host_sockaddr(fd, msg.msg_name,
+                                      tswapal(msgp->msg_name),
+                                      msg.msg_namelen);
         if (ret) {
             goto out2;
         }
@@ -2050,14 +3495,34 @@
     msg.msg_iov = vec;
 
     if (send) {
-        ret = target_to_host_cmsg(&msg, msgp);
-        if (ret == 0)
-            ret = get_errno(sendmsg(fd, &msg, flags));
+        if (fd_trans_target_to_host_data(fd)) {
+            void *host_msg;
+
+            host_msg = g_malloc(msg.msg_iov->iov_len);
+            memcpy(host_msg, msg.msg_iov->iov_base, msg.msg_iov->iov_len);
+            ret = fd_trans_target_to_host_data(fd)(host_msg,
+                                                   msg.msg_iov->iov_len);
+            if (ret >= 0) {
+                msg.msg_iov->iov_base = host_msg;
+                ret = get_errno(safe_sendmsg(fd, &msg, flags));
+            }
+            g_free(host_msg);
+        } else {
+            ret = target_to_host_cmsg(&msg, msgp);
+            if (ret == 0) {
+                ret = get_errno(safe_sendmsg(fd, &msg, flags));
+            }
+        }
     } else {
-        ret = get_errno(recvmsg(fd, &msg, flags));
+        ret = get_errno(safe_recvmsg(fd, &msg, flags));
         if (!is_error(ret)) {
             len = ret;
-            ret = host_to_target_cmsg(msgp, &msg);
+            if (fd_trans_host_to_target_data(fd)) {
+                ret = fd_trans_host_to_target_data(fd)(msg.msg_iov->iov_base,
+                                                       len);
+            } else {
+                ret = host_to_target_cmsg(msgp, &msg);
+            }
             if (!is_error(ret)) {
                 msgp->msg_namelen = tswap32(msg.msg_namelen);
                 if (msg.msg_name != NULL) {
@@ -2096,7 +3561,6 @@
     return ret;
 }
 
-#ifdef TARGET_NR_sendmmsg
 /* We don't rely on the C library to have sendmmsg/recvmmsg support,
  * so it might not have this *mmsg-specific flag either.
  */
@@ -2143,20 +3607,6 @@
     }
     return ret;
 }
-#endif
-
-/* If we don't have a system accept4() then just call accept.
- * The callsites to do_accept4() will ensure that they don't
- * pass a non-zero flags argument in this config.
- */
-#ifndef CONFIG_ACCEPT4
-static inline int accept4(int sockfd, struct sockaddr *addr,
-                          socklen_t *addrlen, int flags)
-{
-    assert(flags == 0);
-    return accept(sockfd, addr, addrlen);
-}
-#endif
 
 /* do_accept4() Must return target values and target errnos. */
 static abi_long do_accept4(int fd, abi_ulong target_addr,
@@ -2170,7 +3620,7 @@
     host_flags = target_to_host_bitmask(flags, fcntl_flags_tbl);
 
     if (target_addr == 0) {
-        return get_errno(accept4(fd, NULL, NULL, host_flags));
+        return get_errno(safe_accept4(fd, NULL, NULL, host_flags));
     }
 
     /* linux returns EINVAL if addrlen pointer is invalid */
@@ -2186,7 +3636,7 @@
 
     addr = alloca(addrlen);
 
-    ret = get_errno(accept4(fd, addr, &addrlen, host_flags));
+    ret = get_errno(safe_accept4(fd, addr, &addrlen, host_flags));
     if (!is_error(ret)) {
         host_to_target_sockaddr(target_addr, addr, addrlen);
         if (put_user_u32(addrlen, target_addrlen_addr))
@@ -2277,6 +3727,7 @@
 {
     void *addr;
     void *host_msg;
+    void *copy_msg = NULL;
     abi_long ret;
 
     if ((int)addrlen < 0) {
@@ -2286,16 +3737,29 @@
     host_msg = lock_user(VERIFY_READ, msg, len, 1);
     if (!host_msg)
         return -TARGET_EFAULT;
+    if (fd_trans_target_to_host_data(fd)) {
+        copy_msg = host_msg;
+        host_msg = g_malloc(len);
+        memcpy(host_msg, copy_msg, len);
+        ret = fd_trans_target_to_host_data(fd)(host_msg, len);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
     if (target_addr) {
         addr = alloca(addrlen+1);
-        ret = target_to_host_sockaddr(addr, target_addr, addrlen);
+        ret = target_to_host_sockaddr(fd, addr, target_addr, addrlen);
         if (ret) {
-            unlock_user(host_msg, msg, 0);
-            return ret;
+            goto fail;
         }
-        ret = get_errno(sendto(fd, host_msg, len, flags, addr, addrlen));
+        ret = get_errno(safe_sendto(fd, host_msg, len, flags, addr, addrlen));
     } else {
-        ret = get_errno(send(fd, host_msg, len, flags));
+        ret = get_errno(safe_sendto(fd, host_msg, len, flags, NULL, 0));
+    }
+fail:
+    if (copy_msg) {
+        g_free(host_msg);
+        host_msg = copy_msg;
     }
     unlock_user(host_msg, msg, 0);
     return ret;
@@ -2324,12 +3788,16 @@
             goto fail;
         }
         addr = alloca(addrlen);
-        ret = get_errno(recvfrom(fd, host_msg, len, flags, addr, &addrlen));
+        ret = get_errno(safe_recvfrom(fd, host_msg, len, flags,
+                                      addr, &addrlen));
     } else {
         addr = NULL; /* To keep compiler quiet.  */
-        ret = get_errno(qemu_recv(fd, host_msg, len, flags));
+        ret = get_errno(safe_recvfrom(fd, host_msg, len, flags, NULL, 0));
     }
     if (!is_error(ret)) {
+        if (fd_trans_host_to_target_data(fd)) {
+            ret = fd_trans_host_to_target_data(fd)(host_msg, ret);
+        }
         if (target_addr) {
             host_to_target_sockaddr(target_addr, addr, addrlen);
             if (put_user_u32(addrlen, target_addrlen)) {
@@ -2366,6 +3834,8 @@
         [SOCKOP_shutdown] = 2,    /* sockfd, how */
         [SOCKOP_sendmsg] = 3,     /* sockfd, msg, flags */
         [SOCKOP_recvmsg] = 3,     /* sockfd, msg, flags */
+        [SOCKOP_sendmmsg] = 4,    /* sockfd, msgvec, vlen, flags */
+        [SOCKOP_recvmmsg] = 4,    /* sockfd, msgvec, vlen, flags */
         [SOCKOP_setsockopt] = 5,  /* sockfd, level, optname, optval, optlen */
         [SOCKOP_getsockopt] = 5,  /* sockfd, level, optname, optval, optlen */
     };
@@ -2416,6 +3886,10 @@
         return do_sendrecvmsg(a[0], a[1], a[2], 1);
     case SOCKOP_recvmsg: /* sockfd, msg, flags */
         return do_sendrecvmsg(a[0], a[1], a[2], 0);
+    case SOCKOP_sendmmsg: /* sockfd, msgvec, vlen, flags */
+        return do_sendrecvmmsg(a[0], a[1], a[2], a[3], 1);
+    case SOCKOP_recvmmsg: /* sockfd, msgvec, vlen, flags */
+        return do_sendrecvmmsg(a[0], a[1], a[2], a[3], 0);
     case SOCKOP_setsockopt: /* sockfd, level, optname, optval, optlen */
         return do_setsockopt(a[0], a[1], a[2], a[3], a[4]);
     case SOCKOP_getsockopt: /* sockfd, level, optname, optval, optlen */
@@ -2430,31 +3904,35 @@
 #define N_SHM_REGIONS	32
 
 static struct shm_region {
-    abi_ulong	start;
-    abi_ulong	size;
+    abi_ulong start;
+    abi_ulong size;
+    bool in_use;
 } shm_regions[N_SHM_REGIONS];
 
-struct target_semid_ds
+#ifndef TARGET_SEMID64_DS
+/* asm-generic version of this struct */
+struct target_semid64_ds
 {
   struct target_ipc_perm sem_perm;
   abi_ulong sem_otime;
-#if !defined(TARGET_PPC64)
+#if TARGET_ABI_BITS == 32
   abi_ulong __unused1;
 #endif
   abi_ulong sem_ctime;
-#if !defined(TARGET_PPC64)
+#if TARGET_ABI_BITS == 32
   abi_ulong __unused2;
 #endif
   abi_ulong sem_nsems;
   abi_ulong __unused3;
   abi_ulong __unused4;
 };
+#endif
 
 static inline abi_long target_to_host_ipc_perm(struct ipc_perm *host_ip,
                                                abi_ulong target_addr)
 {
     struct target_ipc_perm *target_ip;
-    struct target_semid_ds *target_sd;
+    struct target_semid64_ds *target_sd;
 
     if (!lock_user_struct(VERIFY_READ, target_sd, target_addr, 1))
         return -TARGET_EFAULT;
@@ -2482,7 +3960,7 @@
                                                struct ipc_perm *host_ip)
 {
     struct target_ipc_perm *target_ip;
-    struct target_semid_ds *target_sd;
+    struct target_semid64_ds *target_sd;
 
     if (!lock_user_struct(VERIFY_WRITE, target_sd, target_addr, 0))
         return -TARGET_EFAULT;
@@ -2509,7 +3987,7 @@
 static inline abi_long target_to_host_semid_ds(struct semid_ds *host_sd,
                                                abi_ulong target_addr)
 {
-    struct target_semid_ds *target_sd;
+    struct target_semid64_ds *target_sd;
 
     if (!lock_user_struct(VERIFY_READ, target_sd, target_addr, 1))
         return -TARGET_EFAULT;
@@ -2525,7 +4003,7 @@
 static inline abi_long host_to_target_semid_ds(abi_ulong target_addr,
                                                struct semid_ds *host_sd)
 {
-    struct target_semid_ds *target_sd;
+    struct target_semid64_ds *target_sd;
 
     if (!lock_user_struct(VERIFY_WRITE, target_sd, target_addr, 0))
         return -TARGET_EFAULT;
@@ -2602,14 +4080,14 @@
 
     nsems = semid_ds.sem_nsems;
 
-    *host_array = malloc(nsems*sizeof(unsigned short));
+    *host_array = g_try_new(unsigned short, nsems);
     if (!*host_array) {
         return -TARGET_ENOMEM;
     }
     array = lock_user(VERIFY_READ, target_addr,
                       nsems*sizeof(unsigned short), 1);
     if (!array) {
-        free(*host_array);
+        g_free(*host_array);
         return -TARGET_EFAULT;
     }
 
@@ -2646,15 +4124,16 @@
     for(i=0; i<nsems; i++) {
         __put_user((*host_array)[i], &array[i]);
     }
-    free(*host_array);
+    g_free(*host_array);
     unlock_user(array, target_addr, 1);
 
     return 0;
 }
 
 static inline abi_long do_semctl(int semid, int semnum, int cmd,
-                                 union target_semun target_su)
+                                 abi_ulong target_arg)
 {
+    union target_semun target_su = { .buf = target_arg };
     union semun arg;
     struct semid_ds dsarg;
     unsigned short *array = NULL;
@@ -2757,7 +4236,7 @@
     if (target_to_host_sembuf(sops, ptr, nsops))
         return -TARGET_EFAULT;
 
-    return get_errno(semop(semid, sops, nsops));
+    return get_errno(safe_semtimedop(semid, sops, nsops, NULL));
 }
 
 struct target_msqid_ds
@@ -2905,22 +4384,22 @@
 
     if (!lock_user_struct(VERIFY_READ, target_mb, msgp, 0))
         return -TARGET_EFAULT;
-    host_mb = malloc(msgsz+sizeof(long));
+    host_mb = g_try_malloc(msgsz + sizeof(long));
     if (!host_mb) {
         unlock_user_struct(target_mb, msgp, 0);
         return -TARGET_ENOMEM;
     }
     host_mb->mtype = (abi_long) tswapal(target_mb->mtype);
     memcpy(host_mb->mtext, target_mb->mtext, msgsz);
-    ret = get_errno(msgsnd(msqid, host_mb, msgsz, msgflg));
-    free(host_mb);
+    ret = get_errno(safe_msgsnd(msqid, host_mb, msgsz, msgflg));
+    g_free(host_mb);
     unlock_user_struct(target_mb, msgp, 0);
 
     return ret;
 }
 
 static inline abi_long do_msgrcv(int msqid, abi_long msgp,
-                                 unsigned int msgsz, abi_long msgtyp,
+                                 ssize_t msgsz, abi_long msgtyp,
                                  int msgflg)
 {
     struct target_msgbuf *target_mb;
@@ -2928,11 +4407,19 @@
     struct msgbuf *host_mb;
     abi_long ret = 0;
 
+    if (msgsz < 0) {
+        return -TARGET_EINVAL;
+    }
+
     if (!lock_user_struct(VERIFY_WRITE, target_mb, msgp, 0))
         return -TARGET_EFAULT;
 
-    host_mb = g_malloc(msgsz+sizeof(long));
-    ret = get_errno(msgrcv(msqid, host_mb, msgsz, msgtyp, msgflg));
+    host_mb = g_try_malloc(msgsz + sizeof(long));
+    if (!host_mb) {
+        ret = -TARGET_ENOMEM;
+        goto end;
+    }
+    ret = get_errno(safe_msgrcv(msqid, host_mb, msgsz, msgtyp, msgflg));
 
     if (ret > 0) {
         abi_ulong target_mtext_addr = msgp + sizeof(abi_ulong);
@@ -3122,7 +4609,8 @@
                    ((shmflg & SHM_RDONLY)? 0 : PAGE_WRITE));
 
     for (i = 0; i < N_SHM_REGIONS; i++) {
-        if (shm_regions[i].start == 0) {
+        if (!shm_regions[i].in_use) {
+            shm_regions[i].in_use = true;
             shm_regions[i].start = raddr;
             shm_regions[i].size = shm_info.shm_segsz;
             break;
@@ -3139,8 +4627,8 @@
     int i;
 
     for (i = 0; i < N_SHM_REGIONS; ++i) {
-        if (shm_regions[i].start == shmaddr) {
-            shm_regions[i].start = 0;
+        if (shm_regions[i].in_use && shm_regions[i].start == shmaddr) {
+            shm_regions[i].in_use = false;
             page_set_flags(shmaddr, shmaddr + shm_regions[i].size, 0);
             break;
         }
@@ -3176,8 +4664,7 @@
          * ptr argument. */
         abi_ulong atptr;
         get_user_ual(atptr, ptr);
-        ret = do_semctl(first, second, third,
-                (union target_semun) atptr);
+        ret = do_semctl(first, second, third, atptr);
         break;
     }
 
@@ -3262,6 +4749,7 @@
 #define STRUCT_SPECIAL(name) STRUCT_ ## name,
 enum {
 #include "syscall_types.h"
+STRUCT_MAX
 };
 #undef STRUCT
 #undef STRUCT_SPECIAL
@@ -3275,10 +4763,10 @@
 typedef struct IOCTLEntry IOCTLEntry;
 
 typedef abi_long do_ioctl_fn(const IOCTLEntry *ie, uint8_t *buf_temp,
-                             int fd, abi_long cmd, abi_long arg);
+                             int fd, int cmd, abi_long arg);
 
 struct IOCTLEntry {
-    unsigned int target_cmd;
+    int target_cmd;
     unsigned int host_cmd;
     const char *name;
     int access;
@@ -3301,7 +4789,7 @@
                             / sizeof(struct fiemap_extent))
 
 static abi_long do_ioctl_fs_ioc_fiemap(const IOCTLEntry *ie, uint8_t *buf_temp,
-                                       int fd, abi_long cmd, abi_long arg)
+                                       int fd, int cmd, abi_long arg)
 {
     /* The parameter for this ioctl is a struct fiemap followed
      * by an array of struct fiemap_extent whose size is set
@@ -3340,14 +4828,14 @@
         /* We can't fit all the extents into the fixed size buffer.
          * Allocate one that is large enough and use it instead.
          */
-        fm = malloc(outbufsz);
+        fm = g_try_malloc(outbufsz);
         if (!fm) {
             return -TARGET_ENOMEM;
         }
         memcpy(fm, buf_temp, sizeof(struct fiemap));
         free_fm = 1;
     }
-    ret = get_errno(ioctl(fd, ie->host_cmd, fm));
+    ret = get_errno(safe_ioctl(fd, ie->host_cmd, fm));
     if (!is_error(ret)) {
         target_size_out = target_size_in;
         /* An extent_count of 0 means we were only counting the extents
@@ -3375,14 +4863,14 @@
         }
     }
     if (free_fm) {
-        free(fm);
+        g_free(fm);
     }
     return ret;
 }
 #endif
 
 static abi_long do_ioctl_ifconf(const IOCTLEntry *ie, uint8_t *buf_temp,
-                                int fd, abi_long cmd, abi_long arg)
+                                int fd, int cmd, abi_long arg)
 {
     const argtype *arg_type = ie->arg_type;
     int target_size;
@@ -3437,7 +4925,7 @@
     host_ifconf->ifc_len = host_ifc_len;
     host_ifconf->ifc_buf = host_ifc_buf;
 
-    ret = get_errno(ioctl(fd, ie->host_cmd, host_ifconf));
+    ret = get_errno(safe_ioctl(fd, ie->host_cmd, host_ifconf));
     if (!is_error(ret)) {
 	/* convert host ifc_len to target ifc_len */
 
@@ -3476,7 +4964,7 @@
 }
 
 static abi_long do_ioctl_dm(const IOCTLEntry *ie, uint8_t *buf_temp, int fd,
-                            abi_long cmd, abi_long arg)
+                            int cmd, abi_long arg)
 {
     void *argptr;
     struct dm_ioctl *host_dm;
@@ -3561,11 +5049,12 @@
     }
     default:
         ret = -TARGET_EINVAL;
+        unlock_user(argptr, guest_data, 0);
         goto out;
     }
     unlock_user(argptr, guest_data, 0);
 
-    ret = get_errno(ioctl(fd, ie->host_cmd, buf_temp));
+    ret = get_errno(safe_ioctl(fd, ie->host_cmd, buf_temp));
     if (!is_error(ret)) {
         guest_data = arg + host_dm->data_start;
         guest_data_size = host_dm->data_size - host_dm->data_start;
@@ -3680,6 +5169,7 @@
             break;
         }
         default:
+            unlock_user(argptr, guest_data, 0);
             ret = -TARGET_EINVAL;
             goto out;
         }
@@ -3699,7 +5189,7 @@
 }
 
 static abi_long do_ioctl_blkpg(const IOCTLEntry *ie, uint8_t *buf_temp, int fd,
-                               abi_long cmd, abi_long arg)
+                               int cmd, abi_long arg)
 {
     void *argptr;
     int target_size;
@@ -3745,14 +5235,14 @@
 
     /* Swizzle the data pointer to our local copy and call! */
     host_blkpg->data = &host_part;
-    ret = get_errno(ioctl(fd, ie->host_cmd, host_blkpg));
+    ret = get_errno(safe_ioctl(fd, ie->host_cmd, host_blkpg));
 
 out:
     return ret;
 }
 
 static abi_long do_ioctl_rt(const IOCTLEntry *ie, uint8_t *buf_temp,
-                                int fd, abi_long cmd, abi_long arg)
+                                int fd, int cmd, abi_long arg)
 {
     const argtype *arg_type = ie->arg_type;
     const StructEntry *se;
@@ -3806,7 +5296,7 @@
     }
     unlock_user(argptr, arg, 0);
 
-    ret = get_errno(ioctl(fd, ie->host_cmd, buf_temp));
+    ret = get_errno(safe_ioctl(fd, ie->host_cmd, buf_temp));
     if (*host_rt_dev_ptr != 0) {
         unlock_user((void *)*host_rt_dev_ptr,
                     *target_rt_dev_ptr, 0);
@@ -3815,10 +5305,10 @@
 }
 
 static abi_long do_ioctl_kdsigaccept(const IOCTLEntry *ie, uint8_t *buf_temp,
-                                     int fd, abi_long cmd, abi_long arg)
+                                     int fd, int cmd, abi_long arg)
 {
     int sig = target_to_host_signal(arg);
-    return get_errno(ioctl(fd, ie->host_cmd, sig));
+    return get_errno(safe_ioctl(fd, ie->host_cmd, sig));
 }
 
 static IOCTLEntry ioctl_entries[] = {
@@ -3832,7 +5322,7 @@
 
 /* ??? Implement proper locking for ioctls.  */
 /* do_ioctl() Must return target values and target errnos. */
-static abi_long do_ioctl(int fd, abi_long cmd, abi_long arg)
+static abi_long do_ioctl(int fd, int cmd, abi_long arg)
 {
     const IOCTLEntry *ie;
     const argtype *arg_type;
@@ -3862,19 +5352,18 @@
     switch(arg_type[0]) {
     case TYPE_NULL:
         /* no argument */
-        ret = get_errno(ioctl(fd, ie->host_cmd));
+        ret = get_errno(safe_ioctl(fd, ie->host_cmd));
         break;
     case TYPE_PTRVOID:
     case TYPE_INT:
-        /* int argment */
-        ret = get_errno(ioctl(fd, ie->host_cmd, arg));
+        ret = get_errno(safe_ioctl(fd, ie->host_cmd, arg));
         break;
     case TYPE_PTR:
         arg_type++;
         target_size = thunk_type_size(arg_type, 0);
         switch(ie->access) {
         case IOC_R:
-            ret = get_errno(ioctl(fd, ie->host_cmd, buf_temp));
+            ret = get_errno(safe_ioctl(fd, ie->host_cmd, buf_temp));
             if (!is_error(ret)) {
                 argptr = lock_user(VERIFY_WRITE, arg, target_size, 0);
                 if (!argptr)
@@ -3889,7 +5378,7 @@
                 return -TARGET_EFAULT;
             thunk_convert(buf_temp, argptr, arg_type, THUNK_HOST);
             unlock_user(argptr, arg, 0);
-            ret = get_errno(ioctl(fd, ie->host_cmd, buf_temp));
+            ret = get_errno(safe_ioctl(fd, ie->host_cmd, buf_temp));
             break;
         default:
         case IOC_RW:
@@ -3898,7 +5387,7 @@
                 return -TARGET_EFAULT;
             thunk_convert(buf_temp, argptr, arg_type, THUNK_HOST);
             unlock_user(argptr, arg, 0);
-            ret = get_errno(ioctl(fd, ie->host_cmd, buf_temp));
+            ret = get_errno(safe_ioctl(fd, ie->host_cmd, buf_temp));
             if (!is_error(ret)) {
                 argptr = lock_user(VERIFY_WRITE, arg, target_size, 0);
                 if (!argptr)
@@ -4440,6 +5929,7 @@
     CPUState *cpu;
     TaskState *ts;
 
+    rcu_register_thread();
     env = info->env;
     cpu = ENV_GET_CPU(env);
     thread_cpu = cpu;
@@ -4488,7 +5978,7 @@
         new_thread_info info;
         pthread_attr_t attr;
 
-        ts = g_malloc0(sizeof(TaskState));
+        ts = g_new0(TaskState, 1);
         init_task_state(ts);
         /* we create a new CPU instance. */
         new_env = cpu_copy(env);
@@ -4498,6 +5988,7 @@
         new_cpu->opaque = ts;
         ts->bprm = parent_ts->bprm;
         ts->info = parent_ts->info;
+        ts->signal_mask = parent_ts->signal_mask;
         nptl_flags = flags;
         flags &= ~CLONE_NPTL_FLAGS2;
 
@@ -4549,12 +6040,19 @@
         pthread_mutex_unlock(&clone_lock);
     } else {
         /* if no CLONE_VM, we consider it is a fork */
-        if ((flags & ~(CSIGNAL | CLONE_NPTL_FLAGS2)) != 0)
-            return -EINVAL;
+        if ((flags & ~(CSIGNAL | CLONE_NPTL_FLAGS2)) != 0) {
+            return -TARGET_EINVAL;
+        }
+
+        if (block_signals()) {
+            return -TARGET_ERESTARTSYS;
+        }
+
         fork_start();
         ret = fork();
         if (ret == 0) {
             /* Child Process.  */
+            rcu_after_fork();
             cpu_clone_regs(env, newsp);
             fork_end(1);
             /* There is a race condition here.  The parent process could
@@ -4590,11 +6088,11 @@
 	case TARGET_F_SETFL:
             return cmd;
         case TARGET_F_GETLK:
-	    return F_GETLK;
-	case TARGET_F_SETLK:
-	    return F_SETLK;
-	case TARGET_F_SETLKW:
-	    return F_SETLKW;
+            return F_GETLK64;
+        case TARGET_F_SETLK:
+            return F_SETLK64;
+        case TARGET_F_SETLKW:
+            return F_SETLKW64;
 	case TARGET_F_GETOWN:
 	    return F_GETOWN;
 	case TARGET_F_SETOWN:
@@ -4629,6 +6127,12 @@
 	case TARGET_F_SETOWN_EX:
 	    return F_SETOWN_EX;
 #endif
+#ifdef F_SETPIPE_SZ
+        case TARGET_F_SETPIPE_SZ:
+            return F_SETPIPE_SZ;
+        case TARGET_F_GETPIPE_SZ:
+            return F_GETPIPE_SZ;
+#endif
 	default:
             return -TARGET_EINVAL;
     }
@@ -4645,12 +6149,134 @@
     { 0, 0, 0, 0 }
 };
 
+static inline abi_long copy_from_user_flock(struct flock64 *fl,
+                                            abi_ulong target_flock_addr)
+{
+    struct target_flock *target_fl;
+    short l_type;
+
+    if (!lock_user_struct(VERIFY_READ, target_fl, target_flock_addr, 1)) {
+        return -TARGET_EFAULT;
+    }
+
+    __get_user(l_type, &target_fl->l_type);
+    fl->l_type = target_to_host_bitmask(l_type, flock_tbl);
+    __get_user(fl->l_whence, &target_fl->l_whence);
+    __get_user(fl->l_start, &target_fl->l_start);
+    __get_user(fl->l_len, &target_fl->l_len);
+    __get_user(fl->l_pid, &target_fl->l_pid);
+    unlock_user_struct(target_fl, target_flock_addr, 0);
+    return 0;
+}
+
+static inline abi_long copy_to_user_flock(abi_ulong target_flock_addr,
+                                          const struct flock64 *fl)
+{
+    struct target_flock *target_fl;
+    short l_type;
+
+    if (!lock_user_struct(VERIFY_WRITE, target_fl, target_flock_addr, 0)) {
+        return -TARGET_EFAULT;
+    }
+
+    l_type = host_to_target_bitmask(fl->l_type, flock_tbl);
+    __put_user(l_type, &target_fl->l_type);
+    __put_user(fl->l_whence, &target_fl->l_whence);
+    __put_user(fl->l_start, &target_fl->l_start);
+    __put_user(fl->l_len, &target_fl->l_len);
+    __put_user(fl->l_pid, &target_fl->l_pid);
+    unlock_user_struct(target_fl, target_flock_addr, 1);
+    return 0;
+}
+
+typedef abi_long from_flock64_fn(struct flock64 *fl, abi_ulong target_addr);
+typedef abi_long to_flock64_fn(abi_ulong target_addr, const struct flock64 *fl);
+
+#if defined(TARGET_ARM) && TARGET_ABI_BITS == 32
+static inline abi_long copy_from_user_eabi_flock64(struct flock64 *fl,
+                                                   abi_ulong target_flock_addr)
+{
+    struct target_eabi_flock64 *target_fl;
+    short l_type;
+
+    if (!lock_user_struct(VERIFY_READ, target_fl, target_flock_addr, 1)) {
+        return -TARGET_EFAULT;
+    }
+
+    __get_user(l_type, &target_fl->l_type);
+    fl->l_type = target_to_host_bitmask(l_type, flock_tbl);
+    __get_user(fl->l_whence, &target_fl->l_whence);
+    __get_user(fl->l_start, &target_fl->l_start);
+    __get_user(fl->l_len, &target_fl->l_len);
+    __get_user(fl->l_pid, &target_fl->l_pid);
+    unlock_user_struct(target_fl, target_flock_addr, 0);
+    return 0;
+}
+
+static inline abi_long copy_to_user_eabi_flock64(abi_ulong target_flock_addr,
+                                                 const struct flock64 *fl)
+{
+    struct target_eabi_flock64 *target_fl;
+    short l_type;
+
+    if (!lock_user_struct(VERIFY_WRITE, target_fl, target_flock_addr, 0)) {
+        return -TARGET_EFAULT;
+    }
+
+    l_type = host_to_target_bitmask(fl->l_type, flock_tbl);
+    __put_user(l_type, &target_fl->l_type);
+    __put_user(fl->l_whence, &target_fl->l_whence);
+    __put_user(fl->l_start, &target_fl->l_start);
+    __put_user(fl->l_len, &target_fl->l_len);
+    __put_user(fl->l_pid, &target_fl->l_pid);
+    unlock_user_struct(target_fl, target_flock_addr, 1);
+    return 0;
+}
+#endif
+
+static inline abi_long copy_from_user_flock64(struct flock64 *fl,
+                                              abi_ulong target_flock_addr)
+{
+    struct target_flock64 *target_fl;
+    short l_type;
+
+    if (!lock_user_struct(VERIFY_READ, target_fl, target_flock_addr, 1)) {
+        return -TARGET_EFAULT;
+    }
+
+    __get_user(l_type, &target_fl->l_type);
+    fl->l_type = target_to_host_bitmask(l_type, flock_tbl);
+    __get_user(fl->l_whence, &target_fl->l_whence);
+    __get_user(fl->l_start, &target_fl->l_start);
+    __get_user(fl->l_len, &target_fl->l_len);
+    __get_user(fl->l_pid, &target_fl->l_pid);
+    unlock_user_struct(target_fl, target_flock_addr, 0);
+    return 0;
+}
+
+static inline abi_long copy_to_user_flock64(abi_ulong target_flock_addr,
+                                            const struct flock64 *fl)
+{
+    struct target_flock64 *target_fl;
+    short l_type;
+
+    if (!lock_user_struct(VERIFY_WRITE, target_fl, target_flock_addr, 0)) {
+        return -TARGET_EFAULT;
+    }
+
+    l_type = host_to_target_bitmask(fl->l_type, flock_tbl);
+    __put_user(l_type, &target_fl->l_type);
+    __put_user(fl->l_whence, &target_fl->l_whence);
+    __put_user(fl->l_start, &target_fl->l_start);
+    __put_user(fl->l_len, &target_fl->l_len);
+    __put_user(fl->l_pid, &target_fl->l_pid);
+    unlock_user_struct(target_fl, target_flock_addr, 1);
+    return 0;
+}
+
 static abi_long do_fcntl(int fd, int cmd, abi_ulong arg)
 {
-    struct flock fl;
-    struct target_flock *target_fl;
     struct flock64 fl64;
-    struct target_flock64 *target_fl64;
 #ifdef F_GETOWN_EX
     struct f_owner_ex fox;
     struct target_f_owner_ex *target_fox;
@@ -4663,94 +6289,60 @@
 
     switch(cmd) {
     case TARGET_F_GETLK:
-        if (!lock_user_struct(VERIFY_READ, target_fl, arg, 1))
-            return -TARGET_EFAULT;
-        fl.l_type =
-                  target_to_host_bitmask(tswap16(target_fl->l_type), flock_tbl);
-        fl.l_whence = tswap16(target_fl->l_whence);
-        fl.l_start = tswapal(target_fl->l_start);
-        fl.l_len = tswapal(target_fl->l_len);
-        fl.l_pid = tswap32(target_fl->l_pid);
-        unlock_user_struct(target_fl, arg, 0);
-        ret = get_errno(fcntl(fd, host_cmd, &fl));
+        ret = copy_from_user_flock(&fl64, arg);
+        if (ret) {
+            return ret;
+        }
+        ret = get_errno(safe_fcntl(fd, host_cmd, &fl64));
         if (ret == 0) {
-            if (!lock_user_struct(VERIFY_WRITE, target_fl, arg, 0))
-                return -TARGET_EFAULT;
-            target_fl->l_type =
-                          host_to_target_bitmask(tswap16(fl.l_type), flock_tbl);
-            target_fl->l_whence = tswap16(fl.l_whence);
-            target_fl->l_start = tswapal(fl.l_start);
-            target_fl->l_len = tswapal(fl.l_len);
-            target_fl->l_pid = tswap32(fl.l_pid);
-            unlock_user_struct(target_fl, arg, 1);
+            ret = copy_to_user_flock(arg, &fl64);
         }
         break;
 
     case TARGET_F_SETLK:
     case TARGET_F_SETLKW:
-        if (!lock_user_struct(VERIFY_READ, target_fl, arg, 1))
-            return -TARGET_EFAULT;
-        fl.l_type =
-                  target_to_host_bitmask(tswap16(target_fl->l_type), flock_tbl);
-        fl.l_whence = tswap16(target_fl->l_whence);
-        fl.l_start = tswapal(target_fl->l_start);
-        fl.l_len = tswapal(target_fl->l_len);
-        fl.l_pid = tswap32(target_fl->l_pid);
-        unlock_user_struct(target_fl, arg, 0);
-        ret = get_errno(fcntl(fd, host_cmd, &fl));
+        ret = copy_from_user_flock(&fl64, arg);
+        if (ret) {
+            return ret;
+        }
+        ret = get_errno(safe_fcntl(fd, host_cmd, &fl64));
         break;
 
     case TARGET_F_GETLK64:
-        if (!lock_user_struct(VERIFY_READ, target_fl64, arg, 1))
-            return -TARGET_EFAULT;
-        fl64.l_type =
-           target_to_host_bitmask(tswap16(target_fl64->l_type), flock_tbl) >> 1;
-        fl64.l_whence = tswap16(target_fl64->l_whence);
-        fl64.l_start = tswap64(target_fl64->l_start);
-        fl64.l_len = tswap64(target_fl64->l_len);
-        fl64.l_pid = tswap32(target_fl64->l_pid);
-        unlock_user_struct(target_fl64, arg, 0);
-        ret = get_errno(fcntl(fd, host_cmd, &fl64));
+        ret = copy_from_user_flock64(&fl64, arg);
+        if (ret) {
+            return ret;
+        }
+        ret = get_errno(safe_fcntl(fd, host_cmd, &fl64));
         if (ret == 0) {
-            if (!lock_user_struct(VERIFY_WRITE, target_fl64, arg, 0))
-                return -TARGET_EFAULT;
-            target_fl64->l_type =
-                   host_to_target_bitmask(tswap16(fl64.l_type), flock_tbl) >> 1;
-            target_fl64->l_whence = tswap16(fl64.l_whence);
-            target_fl64->l_start = tswap64(fl64.l_start);
-            target_fl64->l_len = tswap64(fl64.l_len);
-            target_fl64->l_pid = tswap32(fl64.l_pid);
-            unlock_user_struct(target_fl64, arg, 1);
+            ret = copy_to_user_flock64(arg, &fl64);
         }
         break;
     case TARGET_F_SETLK64:
     case TARGET_F_SETLKW64:
-        if (!lock_user_struct(VERIFY_READ, target_fl64, arg, 1))
-            return -TARGET_EFAULT;
-        fl64.l_type =
-           target_to_host_bitmask(tswap16(target_fl64->l_type), flock_tbl) >> 1;
-        fl64.l_whence = tswap16(target_fl64->l_whence);
-        fl64.l_start = tswap64(target_fl64->l_start);
-        fl64.l_len = tswap64(target_fl64->l_len);
-        fl64.l_pid = tswap32(target_fl64->l_pid);
-        unlock_user_struct(target_fl64, arg, 0);
-        ret = get_errno(fcntl(fd, host_cmd, &fl64));
+        ret = copy_from_user_flock64(&fl64, arg);
+        if (ret) {
+            return ret;
+        }
+        ret = get_errno(safe_fcntl(fd, host_cmd, &fl64));
         break;
 
     case TARGET_F_GETFL:
-        ret = get_errno(fcntl(fd, host_cmd, arg));
+        ret = get_errno(safe_fcntl(fd, host_cmd, arg));
         if (ret >= 0) {
             ret = host_to_target_bitmask(ret, fcntl_flags_tbl);
         }
         break;
 
     case TARGET_F_SETFL:
-        ret = get_errno(fcntl(fd, host_cmd, target_to_host_bitmask(arg, fcntl_flags_tbl)));
+        ret = get_errno(safe_fcntl(fd, host_cmd,
+                                   target_to_host_bitmask(arg,
+                                                          fcntl_flags_tbl)));
         break;
 
 #ifdef F_GETOWN_EX
     case TARGET_F_GETOWN_EX:
-        ret = get_errno(fcntl(fd, host_cmd, &fox));
+        ret = get_errno(safe_fcntl(fd, host_cmd, &fox));
         if (ret >= 0) {
             if (!lock_user_struct(VERIFY_WRITE, target_fox, arg, 0))
                 return -TARGET_EFAULT;
@@ -4768,7 +6360,7 @@
         fox.type = tswap32(target_fox->type);
         fox.pid = tswap32(target_fox->pid);
         unlock_user_struct(target_fox, arg, 0);
-        ret = get_errno(fcntl(fd, host_cmd, &fox));
+        ret = get_errno(safe_fcntl(fd, host_cmd, &fox));
         break;
 #endif
 
@@ -4778,11 +6370,13 @@
     case TARGET_F_GETSIG:
     case TARGET_F_SETLEASE:
     case TARGET_F_GETLEASE:
-        ret = get_errno(fcntl(fd, host_cmd, arg));
+    case TARGET_F_SETPIPE_SZ:
+    case TARGET_F_GETPIPE_SZ:
+        ret = get_errno(safe_fcntl(fd, host_cmd, arg));
         break;
 
     default:
-        ret = get_errno(fcntl(fd, cmd, arg));
+        ret = get_errno(safe_fcntl(fd, cmd, arg));
         break;
     }
     return ret;
@@ -4854,6 +6448,40 @@
 
 #endif /* USE_UID16 */
 
+/* We must do direct syscalls for setting UID/GID, because we want to
+ * implement the Linux system call semantics of "change only for this thread",
+ * not the libc/POSIX semantics of "change for all threads in process".
+ * (See http://ewontfix.com/17/ for more details.)
+ * We use the 32-bit version of the syscalls if present; if it is not
+ * then either the host architecture supports 32-bit UIDs natively with
+ * the standard syscall, or the 16-bit UID is the best we can do.
+ */
+#ifdef __NR_setuid32
+#define __NR_sys_setuid __NR_setuid32
+#else
+#define __NR_sys_setuid __NR_setuid
+#endif
+#ifdef __NR_setgid32
+#define __NR_sys_setgid __NR_setgid32
+#else
+#define __NR_sys_setgid __NR_setgid
+#endif
+#ifdef __NR_setresuid32
+#define __NR_sys_setresuid __NR_setresuid32
+#else
+#define __NR_sys_setresuid __NR_setresuid
+#endif
+#ifdef __NR_setresgid32
+#define __NR_sys_setresgid __NR_setresgid32
+#else
+#define __NR_sys_setresgid __NR_setresgid
+#endif
+
+_syscall1(int, sys_setuid, uid_t, uid)
+_syscall1(int, sys_setgid, gid_t, gid)
+_syscall3(int, sys_setresuid, uid_t, ruid, uid_t, euid, uid_t, suid)
+_syscall3(int, sys_setresgid, gid_t, rgid, gid_t, egid, gid_t, sgid)
+
 void syscall_init(void)
 {
     IOCTLEntry *ie;
@@ -4861,6 +6489,8 @@
     int size;
     int i;
 
+    thunk_init(STRUCT_MAX);
+
 #define STRUCT(name, ...) thunk_register_struct(STRUCT_ ## name, #name, struct_ ## name ## _def);
 #define STRUCT_SPECIAL(name) thunk_register_struct_direct(STRUCT_ ## name, #name, &struct_ ## name ## _def);
 #include "syscall_types.h"
@@ -4955,8 +6585,8 @@
 
     if (!lock_user_struct(VERIFY_READ, target_ts, target_addr, 1))
         return -TARGET_EFAULT;
-    host_ts->tv_sec = tswapal(target_ts->tv_sec);
-    host_ts->tv_nsec = tswapal(target_ts->tv_nsec);
+    __get_user(host_ts->tv_sec, &target_ts->tv_sec);
+    __get_user(host_ts->tv_nsec, &target_ts->tv_nsec);
     unlock_user_struct(target_ts, target_addr, 0);
     return 0;
 }
@@ -4968,8 +6598,8 @@
 
     if (!lock_user_struct(VERIFY_WRITE, target_ts, target_addr, 0))
         return -TARGET_EFAULT;
-    target_ts->tv_sec = tswapal(host_ts->tv_sec);
-    target_ts->tv_nsec = tswapal(host_ts->tv_nsec);
+    __put_user(host_ts->tv_sec, &target_ts->tv_sec);
+    __put_user(host_ts->tv_nsec, &target_ts->tv_nsec);
     unlock_user_struct(target_ts, target_addr, 1);
     return 0;
 }
@@ -5054,7 +6684,6 @@
 }
 #endif
 
-#if defined(TARGET_NR_stat64) || defined(TARGET_NR_newfstatat)
 static inline abi_long host_to_target_stat64(void *cpu_env,
                                              abi_ulong target_addr,
                                              struct stat *host_st)
@@ -5117,7 +6746,6 @@
 
     return 0;
 }
-#endif
 
 /* ??? Using host futex calls even when target atomic operations
    are not really atomic probably breaks things.  However implementing
@@ -5146,12 +6774,12 @@
         } else {
             pts = NULL;
         }
-        return get_errno(sys_futex(g2h(uaddr), op, tswap32(val),
+        return get_errno(safe_futex(g2h(uaddr), op, tswap32(val),
                          pts, NULL, val3));
     case FUTEX_WAKE:
-        return get_errno(sys_futex(g2h(uaddr), op, val, NULL, NULL, 0));
+        return get_errno(safe_futex(g2h(uaddr), op, val, NULL, NULL, 0));
     case FUTEX_FD:
-        return get_errno(sys_futex(g2h(uaddr), op, val, NULL, NULL, 0));
+        return get_errno(safe_futex(g2h(uaddr), op, val, NULL, NULL, 0));
     case FUTEX_REQUEUE:
     case FUTEX_CMP_REQUEUE:
     case FUTEX_WAKE_OP:
@@ -5161,15 +6789,188 @@
            to satisfy the compiler.  We do not need to tswap TIMEOUT
            since it's not compared to guest memory.  */
         pts = (struct timespec *)(uintptr_t) timeout;
-        return get_errno(sys_futex(g2h(uaddr), op, val, pts,
-                                   g2h(uaddr2),
-                                   (base_op == FUTEX_CMP_REQUEUE
-                                    ? tswap32(val3)
-                                    : val3)));
+        return get_errno(safe_futex(g2h(uaddr), op, val, pts,
+                                    g2h(uaddr2),
+                                    (base_op == FUTEX_CMP_REQUEUE
+                                     ? tswap32(val3)
+                                     : val3)));
     default:
         return -TARGET_ENOSYS;
     }
 }
+#if defined(TARGET_NR_name_to_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+static abi_long do_name_to_handle_at(abi_long dirfd, abi_long pathname,
+                                     abi_long handle, abi_long mount_id,
+                                     abi_long flags)
+{
+    struct file_handle *target_fh;
+    struct file_handle *fh;
+    int mid = 0;
+    abi_long ret;
+    char *name;
+    unsigned int size, total_size;
+
+    if (get_user_s32(size, handle)) {
+        return -TARGET_EFAULT;
+    }
+
+    name = lock_user_string(pathname);
+    if (!name) {
+        return -TARGET_EFAULT;
+    }
+
+    total_size = sizeof(struct file_handle) + size;
+    target_fh = lock_user(VERIFY_WRITE, handle, total_size, 0);
+    if (!target_fh) {
+        unlock_user(name, pathname, 0);
+        return -TARGET_EFAULT;
+    }
+
+    fh = g_malloc0(total_size);
+    fh->handle_bytes = size;
+
+    ret = get_errno(name_to_handle_at(dirfd, path(name), fh, &mid, flags));
+    unlock_user(name, pathname, 0);
+
+    /* man name_to_handle_at(2):
+     * Other than the use of the handle_bytes field, the caller should treat
+     * the file_handle structure as an opaque data type
+     */
+
+    memcpy(target_fh, fh, total_size);
+    target_fh->handle_bytes = tswap32(fh->handle_bytes);
+    target_fh->handle_type = tswap32(fh->handle_type);
+    g_free(fh);
+    unlock_user(target_fh, handle, total_size);
+
+    if (put_user_s32(mid, mount_id)) {
+        return -TARGET_EFAULT;
+    }
+
+    return ret;
+
+}
+#endif
+
+#if defined(TARGET_NR_open_by_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+static abi_long do_open_by_handle_at(abi_long mount_fd, abi_long handle,
+                                     abi_long flags)
+{
+    struct file_handle *target_fh;
+    struct file_handle *fh;
+    unsigned int size, total_size;
+    abi_long ret;
+
+    if (get_user_s32(size, handle)) {
+        return -TARGET_EFAULT;
+    }
+
+    total_size = sizeof(struct file_handle) + size;
+    target_fh = lock_user(VERIFY_READ, handle, total_size, 1);
+    if (!target_fh) {
+        return -TARGET_EFAULT;
+    }
+
+    fh = g_memdup(target_fh, total_size);
+    fh->handle_bytes = size;
+    fh->handle_type = tswap32(target_fh->handle_type);
+
+    ret = get_errno(open_by_handle_at(mount_fd, fh,
+                    target_to_host_bitmask(flags, fcntl_flags_tbl)));
+
+    g_free(fh);
+
+    unlock_user(target_fh, handle, total_size);
+
+    return ret;
+}
+#endif
+
+#if defined(TARGET_NR_signalfd) || defined(TARGET_NR_signalfd4)
+
+/* signalfd siginfo conversion */
+
+static void
+host_to_target_signalfd_siginfo(struct signalfd_siginfo *tinfo,
+                                const struct signalfd_siginfo *info)
+{
+    int sig = host_to_target_signal(info->ssi_signo);
+
+    /* linux/signalfd.h defines a ssi_addr_lsb
+     * not defined in sys/signalfd.h but used by some kernels
+     */
+
+#ifdef BUS_MCEERR_AO
+    if (tinfo->ssi_signo == SIGBUS &&
+        (tinfo->ssi_code == BUS_MCEERR_AR ||
+         tinfo->ssi_code == BUS_MCEERR_AO)) {
+        uint16_t *ssi_addr_lsb = (uint16_t *)(&info->ssi_addr + 1);
+        uint16_t *tssi_addr_lsb = (uint16_t *)(&tinfo->ssi_addr + 1);
+        *tssi_addr_lsb = tswap16(*ssi_addr_lsb);
+    }
+#endif
+
+    tinfo->ssi_signo = tswap32(sig);
+    tinfo->ssi_errno = tswap32(tinfo->ssi_errno);
+    tinfo->ssi_code = tswap32(info->ssi_code);
+    tinfo->ssi_pid = tswap32(info->ssi_pid);
+    tinfo->ssi_uid = tswap32(info->ssi_uid);
+    tinfo->ssi_fd = tswap32(info->ssi_fd);
+    tinfo->ssi_tid = tswap32(info->ssi_tid);
+    tinfo->ssi_band = tswap32(info->ssi_band);
+    tinfo->ssi_overrun = tswap32(info->ssi_overrun);
+    tinfo->ssi_trapno = tswap32(info->ssi_trapno);
+    tinfo->ssi_status = tswap32(info->ssi_status);
+    tinfo->ssi_int = tswap32(info->ssi_int);
+    tinfo->ssi_ptr = tswap64(info->ssi_ptr);
+    tinfo->ssi_utime = tswap64(info->ssi_utime);
+    tinfo->ssi_stime = tswap64(info->ssi_stime);
+    tinfo->ssi_addr = tswap64(info->ssi_addr);
+}
+
+static abi_long host_to_target_data_signalfd(void *buf, size_t len)
+{
+    int i;
+
+    for (i = 0; i < len; i += sizeof(struct signalfd_siginfo)) {
+        host_to_target_signalfd_siginfo(buf + i, buf + i);
+    }
+
+    return len;
+}
+
+static TargetFdTrans target_signalfd_trans = {
+    .host_to_target_data = host_to_target_data_signalfd,
+};
+
+static abi_long do_signalfd4(int fd, abi_long mask, int flags)
+{
+    int host_flags;
+    target_sigset_t *target_mask;
+    sigset_t host_mask;
+    abi_long ret;
+
+    if (flags & ~(TARGET_O_NONBLOCK | TARGET_O_CLOEXEC)) {
+        return -TARGET_EINVAL;
+    }
+    if (!lock_user_struct(VERIFY_READ, target_mask, mask, 1)) {
+        return -TARGET_EFAULT;
+    }
+
+    target_to_host_sigset(&host_mask, target_mask);
+
+    host_flags = target_to_host_bitmask(flags, fcntl_flags_tbl);
+
+    ret = get_errno(signalfd(fd, &host_mask, host_flags));
+    if (ret >= 0) {
+        fd_trans_register(ret, &target_signalfd_trans);
+    }
+
+    unlock_user_struct(target_mask, mask, 0);
+
+    return ret;
+}
+#endif
 
 /* Map host to target signal numbers for the wait family of syscalls.
    Assume all other status bits are the same.  */
@@ -5202,7 +7003,9 @@
 
         nb_read = read(fd_orig, buf, sizeof(buf));
         if (nb_read < 0) {
+            int e = errno;
             fd_orig = close(fd_orig);
+            errno = e;
             return -1;
         } else if (nb_read == 0) {
             break;
@@ -5211,7 +7014,7 @@
         if (!word_skipped) {
             /* Skip the first string, which is the path to qemu-*-static
                instead of the actual command. */
-            cp_buf = memchr(buf, 0, sizeof(buf));
+            cp_buf = memchr(buf, 0, nb_read);
             if (cp_buf) {
                 /* Null byte found, skip one string */
                 cp_buf++;
@@ -5222,7 +7025,9 @@
 
         if (word_skipped) {
             if (write(fd, cp_buf, nb_read) != nb_read) {
+                int e = errno;
                 close(fd_orig);
+                errno = e;
                 return -1;
             }
         }
@@ -5242,7 +7047,7 @@
 
     fp = fopen("/proc/self/maps", "r");
     if (fp == NULL) {
-        return -EACCES;
+        return -1;
     }
 
     while ((read = getline(&line, &len, fp)) != -1) {
@@ -5386,7 +7191,7 @@
 
     fp = fopen("/proc/net/route", "r");
     if (fp == NULL) {
-        return -EACCES;
+        return -1;
     }
 
     /* read header */
@@ -5436,7 +7241,7 @@
 
     if (is_proc_myself(pathname, "exe")) {
         int execfd = qemu_getauxval(AT_EXECFD);
-        return execfd ? execfd : get_errno(sys_openat(dirfd, exec_path, flags, mode));
+        return execfd ? execfd : safe_openat(dirfd, exec_path, flags, mode);
     }
 
     for (fake_open = fakes; fake_open->filename; fake_open++) {
@@ -5462,7 +7267,9 @@
         unlink(filename);
 
         if ((r = fake_open->fill(cpu_env, fd))) {
+            int e = errno;
             close(fd);
+            errno = e;
             return r;
         }
         lseek(fd, 0, SEEK_SET);
@@ -5470,7 +7277,7 @@
         return fd;
     }
 
-    return get_errno(sys_openat(dirfd, path(pathname), flags, mode));
+    return safe_openat(dirfd, path(pathname), flags, mode);
 }
 
 #define TIMER_MAGIC 0x0caf0000
@@ -5508,9 +7315,25 @@
     struct statfs stfs;
     void *p;
 
+#if defined(DEBUG_ERESTARTSYS)
+    /* Debug-only code for exercising the syscall-restart code paths
+     * in the per-architecture cpu main loops: restart every syscall
+     * the guest makes once before letting it through.
+     */
+    {
+        static int flag;
+
+        flag = !flag;
+        if (flag) {
+            return -TARGET_ERESTARTSYS;
+        }
+    }
+#endif
+
 #ifdef DEBUG
     gemu_log("syscall %d", num);
 #endif
+    trace_guest_user_syscall(cpu, num, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8);
     if(do_strace)
         print_syscall(num, arg1, arg2, arg3, arg4, arg5, arg6);
 
@@ -5520,8 +7343,12 @@
            However in threaded applictions it is used for thread termination,
            and _exit_group is used for application termination.
            Do thread termination if we have more then one thread.  */
-        /* FIXME: This probably breaks if a signal arrives.  We should probably
-           be disabling signals.  */
+
+        if (block_signals()) {
+            ret = -TARGET_ERESTARTSYS;
+            break;
+        }
+
         if (CPU_NEXT(first_cpu)) {
             TaskState *ts;
 
@@ -5538,6 +7365,7 @@
             thread_cpu = NULL;
             object_unref(OBJECT(cpu));
             g_free(ts);
+            rcu_unregister_thread();
             pthread_exit(NULL);
         }
 #ifdef TARGET_GPROF
@@ -5553,46 +7381,68 @@
         else {
             if (!(p = lock_user(VERIFY_WRITE, arg2, arg3, 0)))
                 goto efault;
-            ret = get_errno(read(arg1, p, arg3));
+            ret = get_errno(safe_read(arg1, p, arg3));
+            if (ret >= 0 &&
+                fd_trans_host_to_target_data(arg1)) {
+                ret = fd_trans_host_to_target_data(arg1)(p, ret);
+            }
             unlock_user(p, arg2, ret);
         }
         break;
     case TARGET_NR_write:
         if (!(p = lock_user(VERIFY_READ, arg2, arg3, 1)))
             goto efault;
-        ret = get_errno(write(arg1, p, arg3));
+        ret = get_errno(safe_write(arg1, p, arg3));
         unlock_user(p, arg2, 0);
         break;
+#ifdef TARGET_NR_open
     case TARGET_NR_open:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(do_openat(cpu_env, AT_FDCWD, p,
                                   target_to_host_bitmask(arg2, fcntl_flags_tbl),
                                   arg3));
+        fd_trans_unregister(ret);
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_openat:
         if (!(p = lock_user_string(arg2)))
             goto efault;
         ret = get_errno(do_openat(cpu_env, arg1, p,
                                   target_to_host_bitmask(arg3, fcntl_flags_tbl),
                                   arg4));
+        fd_trans_unregister(ret);
         unlock_user(p, arg2, 0);
         break;
+#if defined(TARGET_NR_name_to_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+    case TARGET_NR_name_to_handle_at:
+        ret = do_name_to_handle_at(arg1, arg2, arg3, arg4, arg5);
+        break;
+#endif
+#if defined(TARGET_NR_open_by_handle_at) && defined(CONFIG_OPEN_BY_HANDLE)
+    case TARGET_NR_open_by_handle_at:
+        ret = do_open_by_handle_at(arg1, arg2, arg3);
+        fd_trans_unregister(ret);
+        break;
+#endif
     case TARGET_NR_close:
+        fd_trans_unregister(arg1);
         ret = get_errno(close(arg1));
         break;
     case TARGET_NR_brk:
         ret = do_brk(arg1);
         break;
+#ifdef TARGET_NR_fork
     case TARGET_NR_fork:
         ret = get_errno(do_fork(cpu_env, SIGCHLD, 0, 0, 0, 0));
         break;
+#endif
 #ifdef TARGET_NR_waitpid
     case TARGET_NR_waitpid:
         {
             int status;
-            ret = get_errno(waitpid(arg1, &status, arg3));
+            ret = get_errno(safe_wait4(arg1, &status, arg3, 0));
             if (!is_error(ret) && arg2 && ret
                 && put_user_s32(host_to_target_waitstatus(status), arg2))
                 goto efault;
@@ -5604,7 +7454,7 @@
         {
             siginfo_t info;
             info.si_pid = 0;
-            ret = get_errno(waitid(arg1, arg2, &info, arg4));
+            ret = get_errno(safe_waitid(arg1, arg2, &info, arg4, NULL));
             if (!is_error(ret) && arg3 && info.si_pid != 0) {
                 if (!(p = lock_user(VERIFY_WRITE, arg3, sizeof(target_siginfo_t), 0)))
                     goto efault;
@@ -5619,9 +7469,11 @@
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(creat(p, arg2));
+        fd_trans_unregister(ret);
         unlock_user(p, arg1, 0);
         break;
 #endif
+#ifdef TARGET_NR_link
     case TARGET_NR_link:
         {
             void * p2;
@@ -5635,6 +7487,7 @@
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_linkat)
     case TARGET_NR_linkat:
         {
@@ -5652,12 +7505,14 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_unlink
     case TARGET_NR_unlink:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(unlink(p));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_unlinkat)
     case TARGET_NR_unlinkat:
         if (!(p = lock_user_string(arg2)))
@@ -5723,15 +7578,19 @@
             }
             *q = NULL;
 
-            /* This case will not be caught by the host's execve() if its
-               page size is bigger than the target's. */
-            if (total_size > MAX_ARG_PAGES * TARGET_PAGE_SIZE) {
-                ret = -TARGET_E2BIG;
-                goto execve_end;
-            }
             if (!(p = lock_user_string(arg1)))
                 goto execve_efault;
-            ret = get_errno(execve(p, argp, envp));
+            /* Although execve() is not an interruptible syscall it is
+             * a special case where we must use the safe_syscall wrapper:
+             * if we allow a signal to happen before we make the host
+             * syscall then we will 'lose' it, because at the point of
+             * execve the process leaves QEMU's control. So we use the
+             * safe syscall wrapper to ensure that we either take the
+             * signal as a guest signal, or else it does not happen
+             * before the execve completes and makes it the other
+             * program's problem.
+             */
+            ret = get_errno(safe_execve(p, argp, envp));
             unlock_user(p, arg1, 0);
 
             goto execve_end;
@@ -5774,12 +7633,14 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_mknod
     case TARGET_NR_mknod:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(mknod(p, arg2, arg3));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_mknodat)
     case TARGET_NR_mknodat:
         if (!(p = lock_user_string(arg2)))
@@ -5788,12 +7649,14 @@
         unlock_user(p, arg2, 0);
         break;
 #endif
+#ifdef TARGET_NR_chmod
     case TARGET_NR_chmod:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(chmod(p, arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #ifdef TARGET_NR_break
     case TARGET_NR_break:
         goto unimplemented;
@@ -5903,7 +7766,10 @@
 #endif
 #ifdef TARGET_NR_pause /* not on alpha */
     case TARGET_NR_pause:
-        ret = get_errno(pause());
+        if (!block_signals()) {
+            sigsuspend(&((TaskState *)cpu->opaque)->signal_mask);
+        }
+        ret = -TARGET_EINTR;
         break;
 #endif
 #ifdef TARGET_NR_utime
@@ -5928,6 +7794,7 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_utimes
     case TARGET_NR_utimes:
         {
             struct timeval *tvp, tv[2];
@@ -5946,6 +7813,7 @@
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_futimesat)
     case TARGET_NR_futimesat:
         {
@@ -5974,12 +7842,14 @@
     case TARGET_NR_gtty:
         goto unimplemented;
 #endif
+#ifdef TARGET_NR_access
     case TARGET_NR_access:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(access(path(p), arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_faccessat) && defined(__NR_faccessat)
     case TARGET_NR_faccessat:
         if (!(p = lock_user_string(arg2)))
@@ -6002,8 +7872,9 @@
         ret = 0;
         break;
     case TARGET_NR_kill:
-        ret = get_errno(kill(arg1, target_to_host_signal(arg2)));
+        ret = get_errno(safe_kill(arg1, target_to_host_signal(arg2)));
         break;
+#ifdef TARGET_NR_rename
     case TARGET_NR_rename:
         {
             void *p2;
@@ -6017,6 +7888,7 @@
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_renameat)
     case TARGET_NR_renameat:
         {
@@ -6032,12 +7904,14 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_mkdir
     case TARGET_NR_mkdir:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(mkdir(p, arg2));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #if defined(TARGET_NR_mkdirat)
     case TARGET_NR_mkdirat:
         if (!(p = lock_user_string(arg2)))
@@ -6046,18 +7920,25 @@
         unlock_user(p, arg2, 0);
         break;
 #endif
+#ifdef TARGET_NR_rmdir
     case TARGET_NR_rmdir:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(rmdir(p));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_dup:
         ret = get_errno(dup(arg1));
+        if (ret >= 0) {
+            fd_trans_dup(arg1, ret);
+        }
         break;
+#ifdef TARGET_NR_pipe
     case TARGET_NR_pipe:
         ret = do_pipe(cpu_env, arg1, 0, 0);
         break;
+#endif
 #ifdef TARGET_NR_pipe2
     case TARGET_NR_pipe2:
         ret = do_pipe(cpu_env, arg1,
@@ -6142,14 +8023,24 @@
         ret = get_errno(chroot(p));
         unlock_user(p, arg1, 0);
         break;
+#ifdef TARGET_NR_ustat
     case TARGET_NR_ustat:
         goto unimplemented;
+#endif
+#ifdef TARGET_NR_dup2
     case TARGET_NR_dup2:
         ret = get_errno(dup2(arg1, arg2));
+        if (ret >= 0) {
+            fd_trans_dup(arg1, arg2);
+        }
         break;
+#endif
 #if defined(CONFIG_DUP3) && defined(TARGET_NR_dup3)
     case TARGET_NR_dup3:
         ret = get_errno(dup3(arg1, arg2, arg3));
+        if (ret >= 0) {
+            fd_trans_dup(arg1, arg2);
+        }
         break;
 #endif
 #ifdef TARGET_NR_getppid /* not on alpha */
@@ -6157,9 +8048,11 @@
         ret = get_errno(getppid());
         break;
 #endif
+#ifdef TARGET_NR_getpgrp
     case TARGET_NR_getpgrp:
         ret = get_errno(getpgrp());
         break;
+#endif
     case TARGET_NR_setsid:
         ret = get_errno(setsid());
         break;
@@ -6250,7 +8143,11 @@
 #if defined(TARGET_ALPHA)
             struct target_sigaction act, oact, *pact = 0;
             struct target_rt_sigaction *rt_act;
-            /* ??? arg4 == sizeof(sigset_t).  */
+
+            if (arg4 != sizeof(target_sigset_t)) {
+                ret = -TARGET_EINVAL;
+                break;
+            }
             if (arg2) {
                 if (!lock_user_struct(VERIFY_READ, rt_act, arg2, 1))
                     goto efault;
@@ -6274,6 +8171,10 @@
             struct target_sigaction *act;
             struct target_sigaction *oact;
 
+            if (arg4 != sizeof(target_sigset_t)) {
+                ret = -TARGET_EINVAL;
+                break;
+            }
             if (arg2) {
                 if (!lock_user_struct(VERIFY_READ, act, arg2, 1))
                     goto efault;
@@ -6300,9 +8201,11 @@
         {
             sigset_t cur_set;
             abi_ulong target_set;
-            do_sigprocmask(0, NULL, &cur_set);
-            host_to_target_old_sigset(&target_set, &cur_set);
-            ret = target_set;
+            ret = do_sigprocmask(0, NULL, &cur_set);
+            if (!ret) {
+                host_to_target_old_sigset(&target_set, &cur_set);
+                ret = target_set;
+            }
         }
         break;
 #endif
@@ -6311,12 +8214,20 @@
         {
             sigset_t set, oset, cur_set;
             abi_ulong target_set = arg1;
-            do_sigprocmask(0, NULL, &cur_set);
+            /* We only have one word of the new mask so we must read
+             * the rest of it with do_sigprocmask() and OR in this word.
+             * We are guaranteed that a do_sigprocmask() that only queries
+             * the signal mask will not fail.
+             */
+            ret = do_sigprocmask(0, NULL, &cur_set);
+            assert(!ret);
             target_to_host_old_sigset(&set, &target_set);
             sigorset(&set, &set, &cur_set);
-            do_sigprocmask(SIG_SETMASK, &set, &oset);
-            host_to_target_old_sigset(&target_set, &oset);
-            ret = target_set;
+            ret = do_sigprocmask(SIG_SETMASK, &set, &oset);
+            if (!ret) {
+                host_to_target_old_sigset(&target_set, &oset);
+                ret = target_set;
+            }
         }
         break;
 #endif
@@ -6345,7 +8256,7 @@
             mask = arg2;
             target_to_host_old_sigset(&set, &mask);
 
-            ret = get_errno(do_sigprocmask(how, &set, &oldset));
+            ret = do_sigprocmask(how, &set, &oldset);
             if (!is_error(ret)) {
                 host_to_target_old_sigset(&mask, &oldset);
                 ret = mask;
@@ -6379,7 +8290,7 @@
                 how = 0;
                 set_ptr = NULL;
             }
-            ret = get_errno(do_sigprocmask(how, set_ptr, &oldset));
+            ret = do_sigprocmask(how, set_ptr, &oldset);
             if (!is_error(ret) && arg3) {
                 if (!(p = lock_user(VERIFY_WRITE, arg3, sizeof(target_sigset_t), 0)))
                     goto efault;
@@ -6395,6 +8306,11 @@
             int how = arg1;
             sigset_t set, oldset, *set_ptr;
 
+            if (arg4 != sizeof(target_sigset_t)) {
+                ret = -TARGET_EINVAL;
+                break;
+            }
+
             if (arg2) {
                 switch(how) {
                 case TARGET_SIG_BLOCK:
@@ -6419,7 +8335,7 @@
                 how = 0;
                 set_ptr = NULL;
             }
-            ret = get_errno(do_sigprocmask(how, set_ptr, &oldset));
+            ret = do_sigprocmask(how, set_ptr, &oldset);
             if (!is_error(ret) && arg3) {
                 if (!(p = lock_user(VERIFY_WRITE, arg3, sizeof(target_sigset_t), 0)))
                     goto efault;
@@ -6445,6 +8361,17 @@
     case TARGET_NR_rt_sigpending:
         {
             sigset_t set;
+
+            /* Yes, this check is >, not != like most. We follow the kernel's
+             * logic and it does it like this because it implements
+             * NR_sigpending through the same code path, and in that case
+             * the old_sigset_t is smaller in size.
+             */
+            if (arg2 > sizeof(target_sigset_t)) {
+                ret = -TARGET_EINVAL;
+                break;
+            }
+
             ret = get_errno(sigpending(&set));
             if (!is_error(ret)) {
                 if (!(p = lock_user(VERIFY_WRITE, arg1, sizeof(target_sigset_t), 0)))
@@ -6457,28 +8384,41 @@
 #ifdef TARGET_NR_sigsuspend
     case TARGET_NR_sigsuspend:
         {
-            sigset_t set;
+            TaskState *ts = cpu->opaque;
 #if defined(TARGET_ALPHA)
             abi_ulong mask = arg1;
-            target_to_host_old_sigset(&set, &mask);
+            target_to_host_old_sigset(&ts->sigsuspend_mask, &mask);
 #else
             if (!(p = lock_user(VERIFY_READ, arg1, sizeof(target_sigset_t), 1)))
                 goto efault;
-            target_to_host_old_sigset(&set, p);
+            target_to_host_old_sigset(&ts->sigsuspend_mask, p);
             unlock_user(p, arg1, 0);
 #endif
-            ret = get_errno(sigsuspend(&set));
+            ret = get_errno(safe_rt_sigsuspend(&ts->sigsuspend_mask,
+                                               SIGSET_T_SIZE));
+            if (ret != -TARGET_ERESTARTSYS) {
+                ts->in_sigsuspend = 1;
+            }
         }
         break;
 #endif
     case TARGET_NR_rt_sigsuspend:
         {
-            sigset_t set;
+            TaskState *ts = cpu->opaque;
+
+            if (arg2 != sizeof(target_sigset_t)) {
+                ret = -TARGET_EINVAL;
+                break;
+            }
             if (!(p = lock_user(VERIFY_READ, arg1, sizeof(target_sigset_t), 1)))
                 goto efault;
-            target_to_host_sigset(&set, p);
+            target_to_host_sigset(&ts->sigsuspend_mask, p);
             unlock_user(p, arg1, 0);
-            ret = get_errno(sigsuspend(&set));
+            ret = get_errno(safe_rt_sigsuspend(&ts->sigsuspend_mask,
+                                               SIGSET_T_SIZE));
+            if (ret != -TARGET_ERESTARTSYS) {
+                ts->in_sigsuspend = 1;
+            }
         }
         break;
     case TARGET_NR_rt_sigtimedwait:
@@ -6487,6 +8427,11 @@
             struct timespec uts, *puts;
             siginfo_t uinfo;
 
+            if (arg4 != sizeof(target_sigset_t)) {
+                ret = -TARGET_EINVAL;
+                break;
+            }
+
             if (!(p = lock_user(VERIFY_READ, arg1, sizeof(target_sigset_t), 1)))
                 goto efault;
             target_to_host_sigset(&set, p);
@@ -6497,7 +8442,8 @@
             } else {
                 puts = NULL;
             }
-            ret = get_errno(sigtimedwait(&set, &uinfo, puts));
+            ret = get_errno(safe_rt_sigtimedwait(&set, &uinfo, puts,
+                                                 SIGSET_T_SIZE));
             if (!is_error(ret)) {
                 if (arg2) {
                     p = lock_user(VERIFY_WRITE, arg2, sizeof(target_siginfo_t),
@@ -6515,8 +8461,11 @@
     case TARGET_NR_rt_sigqueueinfo:
         {
             siginfo_t uinfo;
-            if (!(p = lock_user(VERIFY_READ, arg3, sizeof(target_sigset_t), 1)))
+
+            p = lock_user(VERIFY_READ, arg3, sizeof(target_siginfo_t), 1);
+            if (!p) {
                 goto efault;
+            }
             target_to_host_siginfo(&uinfo, p);
             unlock_user(p, arg1, 0);
             ret = get_errno(sys_rt_sigqueueinfo(arg1, arg2, &uinfo));
@@ -6524,13 +8473,19 @@
         break;
 #ifdef TARGET_NR_sigreturn
     case TARGET_NR_sigreturn:
-        /* NOTE: ret is eax, so not transcoding must be done */
-        ret = do_sigreturn(cpu_env);
+        if (block_signals()) {
+            ret = -TARGET_ERESTARTSYS;
+        } else {
+            ret = do_sigreturn(cpu_env);
+        }
         break;
 #endif
     case TARGET_NR_rt_sigreturn:
-        /* NOTE: ret is eax, so not transcoding must be done */
-        ret = do_rt_sigreturn(cpu_env);
+        if (block_signals()) {
+            ret = -TARGET_ERESTARTSYS;
+        } else {
+            ret = do_rt_sigreturn(cpu_env);
+        }
         break;
     case TARGET_NR_sethostname:
         if (!(p = lock_user_string(arg1)))
@@ -6687,7 +8642,7 @@
             /* Extract the two packed args for the sigset */
             if (arg6) {
                 sig_ptr = &sig;
-                sig.size = _NSIG / 8;
+                sig.size = SIGSET_T_SIZE;
 
                 arg7 = lock_user(VERIFY_READ, arg6, sizeof(*arg7) * 2, 1);
                 if (!arg7) {
@@ -6718,8 +8673,8 @@
                 sig_ptr = NULL;
             }
 
-            ret = get_errno(sys_pselect6(n, rfds_ptr, wfds_ptr, efds_ptr,
-                                         ts_ptr, sig_ptr));
+            ret = get_errno(safe_pselect6(n, rfds_ptr, wfds_ptr, efds_ptr,
+                                          ts_ptr, sig_ptr));
 
             if (!is_error(ret)) {
                 if (rfd_addr && copy_to_user_fdset(rfd_addr, &rfds, n))
@@ -6735,6 +8690,7 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_symlink
     case TARGET_NR_symlink:
         {
             void *p2;
@@ -6748,6 +8704,7 @@
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_symlinkat)
     case TARGET_NR_symlinkat:
         {
@@ -6767,6 +8724,7 @@
     case TARGET_NR_oldlstat:
         goto unimplemented;
 #endif
+#ifdef TARGET_NR_readlink
     case TARGET_NR_readlink:
         {
             void *p2;
@@ -6797,6 +8755,7 @@
             unlock_user(p, arg1, 0);
         }
         break;
+#endif
 #if defined(TARGET_NR_readlinkat)
     case TARGET_NR_readlinkat:
         {
@@ -7056,11 +9015,7 @@
 #endif
 #ifdef TARGET_NR_accept4
     case TARGET_NR_accept4:
-#ifdef CONFIG_ACCEPT4
         ret = do_accept4(arg1, arg2, arg3, arg4);
-#else
-        goto unimplemented;
-#endif
         break;
 #endif
 #ifdef TARGET_NR_bind
@@ -7136,9 +9091,20 @@
         ret = get_errno(shutdown(arg1, arg2));
         break;
 #endif
+#if defined(TARGET_NR_getrandom) && defined(__NR_getrandom)
+    case TARGET_NR_getrandom:
+        p = lock_user(VERIFY_WRITE, arg1, arg2, 0);
+        if (!p) {
+            goto efault;
+        }
+        ret = get_errno(getrandom(p, arg2, arg3));
+        unlock_user(p, arg1, ret);
+        break;
+#endif
 #ifdef TARGET_NR_socket
     case TARGET_NR_socket:
         ret = do_socket(arg1, arg2, arg3);
+        fd_trans_unregister(ret);
         break;
 #endif
 #ifdef TARGET_NR_socketpair
@@ -7196,22 +9162,28 @@
             }
         }
         break;
+#ifdef TARGET_NR_stat
     case TARGET_NR_stat:
         if (!(p = lock_user_string(arg1)))
             goto efault;
-        ret = get_errno(stat(path(p), &st));
+        ret = get_errno(qemu_stat(path(p), &st));
         unlock_user(p, arg1, 0);
         goto do_stat;
+#endif
+#ifdef TARGET_NR_lstat
     case TARGET_NR_lstat:
         if (!(p = lock_user_string(arg1)))
             goto efault;
-        ret = get_errno(lstat(path(p), &st));
+        ret = get_errno(qemu_lstat(path(p), &st));
         unlock_user(p, arg1, 0);
         goto do_stat;
+#endif
     case TARGET_NR_fstat:
         {
             ret = get_errno(fstat(arg1, &st));
+#if defined(TARGET_NR_stat) || defined(TARGET_NR_lstat)
         do_stat:
+#endif
             if (!is_error(ret)) {
                 struct target_stat *target_st;
 
@@ -7267,7 +9239,7 @@
                 rusage_ptr = &rusage;
             else
                 rusage_ptr = NULL;
-            ret = get_errno(wait4(arg1, &status, arg3, rusage_ptr));
+            ret = get_errno(safe_wait4(arg1, &status, arg3, rusage_ptr));
             if (!is_error(ret)) {
                 if (status_ptr && ret) {
                     status = host_to_target_waitstatus(status);
@@ -7335,7 +9307,7 @@
 #endif
 #ifdef TARGET_NR_semctl
     case TARGET_NR_semctl:
-        ret = do_semctl(arg1, arg2, arg3, (union target_semun)(abi_ulong)arg4);
+        ret = do_semctl(arg1, arg2, arg3, arg4);
         break;
 #endif
 #ifdef TARGET_NR_msgctl
@@ -7423,12 +9395,14 @@
                 goto efault;
             ret = get_errno(sys_uname(buf));
             if (!is_error(ret)) {
-                /* Overrite the native machine name with whatever is being
+                /* Overwrite the native machine name with whatever is being
                    emulated. */
                 strcpy (buf->machine, cpu_to_uname_machine(cpu_env));
                 /* Allow the user to override the reported release.  */
-                if (qemu_uname_release && *qemu_uname_release)
-                  strcpy (buf->release, qemu_uname_release);
+                if (qemu_uname_release && *qemu_uname_release) {
+                    g_strlcpy(buf->release, qemu_uname_release,
+                              sizeof(buf->release));
+                }
             }
             unlock_user_struct(buf, arg1, 1);
         }
@@ -7484,7 +9458,7 @@
         {
             int64_t res;
 #if !defined(__NR_llseek)
-            res = lseek(arg1, ((uint64_t)arg2 << 32) | arg3, arg5);
+            res = lseek(arg1, ((uint64_t)arg2 << 32) | (abi_ulong)arg3, arg5);
             if (res == -1) {
                 ret = get_errno(res);
             } else {
@@ -7499,6 +9473,7 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_getdents
     case TARGET_NR_getdents:
 #ifdef __NR_getdents
 #if TARGET_ABI_BITS == 32 && HOST_LONG_BITS == 64
@@ -7507,8 +9482,8 @@
             struct linux_dirent *dirp;
             abi_long count = arg3;
 
-	    dirp = malloc(count);
-	    if (!dirp) {
+            dirp = g_try_malloc(count);
+            if (!dirp) {
                 ret = -TARGET_ENOMEM;
                 goto fail;
             }
@@ -7544,7 +9519,7 @@
 		ret = count1;
                 unlock_user(target_dirp, arg2, ret);
             }
-	    free(dirp);
+            g_free(dirp);
         }
 #else
         {
@@ -7629,6 +9604,7 @@
         }
 #endif
         break;
+#endif /* TARGET_NR_getdents */
 #if defined(TARGET_NR_getdents64) && defined(__NR_getdents64)
     case TARGET_NR_getdents64:
         {
@@ -7672,22 +9648,29 @@
         {
             struct target_pollfd *target_pfd;
             unsigned int nfds = arg2;
-            int timeout = arg3;
             struct pollfd *pfd;
             unsigned int i;
 
-            target_pfd = lock_user(VERIFY_WRITE, arg1, sizeof(struct target_pollfd) * nfds, 1);
-            if (!target_pfd)
-                goto efault;
+            pfd = NULL;
+            target_pfd = NULL;
+            if (nfds) {
+                target_pfd = lock_user(VERIFY_WRITE, arg1,
+                                       sizeof(struct target_pollfd) * nfds, 1);
+                if (!target_pfd) {
+                    goto efault;
+                }
 
-            pfd = alloca(sizeof(struct pollfd) * nfds);
-            for(i = 0; i < nfds; i++) {
-                pfd[i].fd = tswap32(target_pfd[i].fd);
-                pfd[i].events = tswap16(target_pfd[i].events);
+                pfd = alloca(sizeof(struct pollfd) * nfds);
+                for (i = 0; i < nfds; i++) {
+                    pfd[i].fd = tswap32(target_pfd[i].fd);
+                    pfd[i].events = tswap16(target_pfd[i].events);
+                }
             }
 
+            switch (num) {
 # ifdef TARGET_NR_ppoll
-            if (num == TARGET_NR_ppoll) {
+            case TARGET_NR_ppoll:
+            {
                 struct timespec _timeout_ts, *timeout_ts = &_timeout_ts;
                 target_sigset_t *target_set;
                 sigset_t _set, *set = &_set;
@@ -7702,6 +9685,12 @@
                 }
 
                 if (arg4) {
+                    if (arg5 != sizeof(target_sigset_t)) {
+                        unlock_user(target_pfd, arg1, 0);
+                        ret = -TARGET_EINVAL;
+                        break;
+                    }
+
                     target_set = lock_user(VERIFY_READ, arg4, sizeof(target_sigset_t), 1);
                     if (!target_set) {
                         unlock_user(target_pfd, arg1, 0);
@@ -7712,7 +9701,8 @@
                     set = NULL;
                 }
 
-                ret = get_errno(sys_ppoll(pfd, nfds, timeout_ts, set, _NSIG/8));
+                ret = get_errno(safe_ppoll(pfd, nfds, timeout_ts,
+                                           set, SIGSET_T_SIZE));
 
                 if (!is_error(ret) && arg3) {
                     host_to_target_timespec(arg3, timeout_ts);
@@ -7720,9 +9710,30 @@
                 if (arg4) {
                     unlock_user(target_set, arg4, 0);
                 }
-            } else
+                break;
+            }
 # endif
-                ret = get_errno(poll(pfd, nfds, timeout));
+# ifdef TARGET_NR_poll
+            case TARGET_NR_poll:
+            {
+                struct timespec ts, *pts;
+
+                if (arg3 >= 0) {
+                    /* Convert ms to secs, ns */
+                    ts.tv_sec = arg3 / 1000;
+                    ts.tv_nsec = (arg3 % 1000) * 1000000LL;
+                    pts = &ts;
+                } else {
+                    /* -ve poll() timeout means "infinite" */
+                    pts = NULL;
+                }
+                ret = get_errno(safe_ppoll(pfd, nfds, pts, NULL, 0));
+                break;
+            }
+# endif
+            default:
+                g_assert_not_reached();
+            }
 
             if (!is_error(ret)) {
                 for(i = 0; i < nfds; i++) {
@@ -7736,13 +9747,13 @@
     case TARGET_NR_flock:
         /* NOTE: the flock constant seems to be the same for every
            Linux platform */
-        ret = get_errno(flock(arg1, arg2));
+        ret = get_errno(safe_flock(arg1, arg2));
         break;
     case TARGET_NR_readv:
         {
             struct iovec *vec = lock_iovec(VERIFY_WRITE, arg2, arg3, 0);
             if (vec != NULL) {
-                ret = get_errno(readv(arg1, vec, arg3));
+                ret = get_errno(safe_readv(arg1, vec, arg3));
                 unlock_iovec(vec, arg2, arg3, 1);
             } else {
                 ret = -host_to_target_errno(errno);
@@ -7753,7 +9764,7 @@
         {
             struct iovec *vec = lock_iovec(VERIFY_READ, arg2, arg3, 1);
             if (vec != NULL) {
-                ret = get_errno(writev(arg1, vec, arg3));
+                ret = get_errno(safe_writev(arg1, vec, arg3));
                 unlock_iovec(vec, arg2, arg3, 0);
             } else {
                 ret = -host_to_target_errno(errno);
@@ -7768,11 +9779,13 @@
         ret = get_errno(fdatasync(arg1));
         break;
 #endif
+#ifdef TARGET_NR__sysctl
     case TARGET_NR__sysctl:
         /* We don't implement this, but ENOTDIR is always a safe
            return value. */
         ret = -TARGET_ENOTDIR;
         break;
+#endif
     case TARGET_NR_sched_getaffinity:
         {
             unsigned int mask_size;
@@ -7910,7 +9923,7 @@
         {
             struct timespec req, rem;
             target_to_host_timespec(&req, arg1);
-            ret = get_errno(nanosleep(&req, &rem));
+            ret = get_errno(safe_nanosleep(&req, &rem));
             if (is_error(ret) && arg2) {
                 host_to_target_timespec(arg2, &rem);
             }
@@ -8074,14 +10087,8 @@
         break;
     }
     case TARGET_NR_sigaltstack:
-#if defined(TARGET_I386) || defined(TARGET_ARM) || defined(TARGET_MIPS) || \
-    defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_ALPHA) || \
-    defined(TARGET_M68K) || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
         ret = do_sigaltstack(arg1, arg2, get_sp_from_cpustate((CPUArchState *)cpu_env));
         break;
-#else
-        goto unimplemented;
-#endif
 
 #ifdef CONFIG_SENDFILE
     case TARGET_NR_sendfile:
@@ -8182,7 +10189,7 @@
     case TARGET_NR_stat64:
         if (!(p = lock_user_string(arg1)))
             goto efault;
-        ret = get_errno(stat(path(p), &st));
+        ret = get_errno(qemu_stat(path(p), &st));
         unlock_user(p, arg1, 0);
         if (!is_error(ret))
             ret = host_to_target_stat64(cpu_env, arg2, &st);
@@ -8192,7 +10199,7 @@
     case TARGET_NR_lstat64:
         if (!(p = lock_user_string(arg1)))
             goto efault;
-        ret = get_errno(lstat(path(p), &st));
+        ret = get_errno(qemu_lstat(path(p), &st));
         unlock_user(p, arg1, 0);
         if (!is_error(ret))
             ret = host_to_target_stat64(cpu_env, arg2, &st);
@@ -8219,12 +10226,14 @@
             ret = host_to_target_stat64(cpu_env, arg3, &st);
         break;
 #endif
+#ifdef TARGET_NR_lchown
     case TARGET_NR_lchown:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(lchown(p, low2highuid(arg2), low2highgid(arg3)));
         unlock_user(p, arg1, 0);
         break;
+#endif
 #ifdef TARGET_NR_getuid
     case TARGET_NR_getuid:
         ret = get_errno(high2lowuid(getuid()));
@@ -8307,9 +10316,9 @@
 #endif
 #ifdef TARGET_NR_setresuid
     case TARGET_NR_setresuid:
-        ret = get_errno(setresuid(low2highuid(arg1),
-                                  low2highuid(arg2),
-                                  low2highuid(arg3)));
+        ret = get_errno(sys_setresuid(low2highuid(arg1),
+                                      low2highuid(arg2),
+                                      low2highuid(arg3)));
         break;
 #endif
 #ifdef TARGET_NR_getresuid
@@ -8328,9 +10337,9 @@
 #endif
 #ifdef TARGET_NR_getresgid
     case TARGET_NR_setresgid:
-        ret = get_errno(setresgid(low2highgid(arg1),
-                                  low2highgid(arg2),
-                                  low2highgid(arg3)));
+        ret = get_errno(sys_setresgid(low2highgid(arg1),
+                                      low2highgid(arg2),
+                                      low2highgid(arg3)));
         break;
 #endif
 #ifdef TARGET_NR_getresgid
@@ -8347,17 +10356,19 @@
         }
         break;
 #endif
+#ifdef TARGET_NR_chown
     case TARGET_NR_chown:
         if (!(p = lock_user_string(arg1)))
             goto efault;
         ret = get_errno(chown(p, low2highuid(arg2), low2highgid(arg3)));
         unlock_user(p, arg1, 0);
         break;
+#endif
     case TARGET_NR_setuid:
-        ret = get_errno(setuid(low2highuid(arg1)));
+        ret = get_errno(sys_setuid(low2highuid(arg1)));
         break;
     case TARGET_NR_setgid:
-        ret = get_errno(setgid(low2highgid(arg1)));
+        ret = get_errno(sys_setgid(low2highgid(arg1)));
         break;
     case TARGET_NR_setfsuid:
         ret = get_errno(setfsuid(arg1));
@@ -8554,9 +10565,11 @@
             }
             mask = arg2;
             target_to_host_old_sigset(&set, &mask);
-            do_sigprocmask(how, &set, &oldset);
-            host_to_target_old_sigset(&mask, &oldset);
-            ret = mask;
+            ret = do_sigprocmask(how, &set, &oldset);
+            if (!ret) {
+                host_to_target_old_sigset(&mask, &oldset);
+                ret = mask;
+            }
         }
         break;
 #endif
@@ -8639,7 +10652,7 @@
 #endif
 #ifdef TARGET_NR_setresuid32
     case TARGET_NR_setresuid32:
-        ret = get_errno(setresuid(arg1, arg2, arg3));
+        ret = get_errno(sys_setresuid(arg1, arg2, arg3));
         break;
 #endif
 #ifdef TARGET_NR_getresuid32
@@ -8658,7 +10671,7 @@
 #endif
 #ifdef TARGET_NR_setresgid32
     case TARGET_NR_setresgid32:
-        ret = get_errno(setresgid(arg1, arg2, arg3));
+        ret = get_errno(sys_setresgid(arg1, arg2, arg3));
         break;
 #endif
 #ifdef TARGET_NR_getresgid32
@@ -8685,12 +10698,12 @@
 #endif
 #ifdef TARGET_NR_setuid32
     case TARGET_NR_setuid32:
-        ret = get_errno(setuid(arg1));
+        ret = get_errno(sys_setuid(arg1));
         break;
 #endif
 #ifdef TARGET_NR_setgid32
     case TARGET_NR_setgid32:
-        ret = get_errno(setgid(arg1));
+        ret = get_errno(sys_setgid(arg1));
         break;
 #endif
 #ifdef TARGET_NR_setfsuid32
@@ -8724,18 +10737,56 @@
 #endif
 #ifdef TARGET_NR_arm_fadvise64_64
     case TARGET_NR_arm_fadvise64_64:
-	{
-		/*
-		 * arm_fadvise64_64 looks like fadvise64_64 but
-		 * with different argument order
-		 */
-		abi_long temp;
-		temp = arg3;
-		arg3 = arg4;
-		arg4 = temp;
-	}
+        /* arm_fadvise64_64 looks like fadvise64_64 but
+         * with different argument order: fd, advice, offset, len
+         * rather than the usual fd, offset, len, advice.
+         * Note that offset and len are both 64-bit so appear as
+         * pairs of 32-bit registers.
+         */
+        ret = posix_fadvise(arg1, target_offset64(arg3, arg4),
+                            target_offset64(arg5, arg6), arg2);
+        ret = -host_to_target_errno(ret);
+        break;
 #endif
-#if defined(TARGET_NR_fadvise64_64) || defined(TARGET_NR_arm_fadvise64_64) || defined(TARGET_NR_fadvise64)
+
+#if TARGET_ABI_BITS == 32
+
+#ifdef TARGET_NR_fadvise64_64
+    case TARGET_NR_fadvise64_64:
+        /* 6 args: fd, offset (high, low), len (high, low), advice */
+        if (regpairs_aligned(cpu_env)) {
+            /* offset is in (3,4), len in (5,6) and advice in 7 */
+            arg2 = arg3;
+            arg3 = arg4;
+            arg4 = arg5;
+            arg5 = arg6;
+            arg6 = arg7;
+        }
+        ret = -host_to_target_errno(posix_fadvise(arg1,
+                                                  target_offset64(arg2, arg3),
+                                                  target_offset64(arg4, arg5),
+                                                  arg6));
+        break;
+#endif
+
+#ifdef TARGET_NR_fadvise64
+    case TARGET_NR_fadvise64:
+        /* 5 args: fd, offset (high, low), len, advice */
+        if (regpairs_aligned(cpu_env)) {
+            /* offset is in (3,4), len in 5 and advice in 6 */
+            arg2 = arg3;
+            arg3 = arg4;
+            arg4 = arg5;
+            arg5 = arg6;
+        }
+        ret = -host_to_target_errno(posix_fadvise(arg1,
+                                                  target_offset64(arg2, arg3),
+                                                  arg4, arg5));
+        break;
+#endif
+
+#else /* not a 32-bit ABI */
+#if defined(TARGET_NR_fadvise64_64) || defined(TARGET_NR_fadvise64)
 #ifdef TARGET_NR_fadvise64_64
     case TARGET_NR_fadvise64_64:
 #endif
@@ -8751,9 +10802,11 @@
         default: break;
         }
 #endif
-        ret = -posix_fadvise(arg1, arg2, arg3, arg4);
-	break;
+        ret = -host_to_target_errno(posix_fadvise(arg1, arg2, arg3, arg4));
+        break;
 #endif
+#endif /* end of 64-bit ABI fadvise handling */
+
 #ifdef TARGET_NR_madvise
     case TARGET_NR_madvise:
         /* A straight passthrough may not be safe because qemu sometimes
@@ -8768,9 +10821,14 @@
     {
 	int cmd;
 	struct flock64 fl;
-	struct target_flock64 *target_fl;
+        from_flock64_fn *copyfrom = copy_from_user_flock64;
+        to_flock64_fn *copyto = copy_to_user_flock64;
+
 #ifdef TARGET_ARM
-	struct target_eabi_flock64 *target_efl;
+        if (((CPUARMState *)cpu_env)->eabi) {
+            copyfrom = copy_from_user_eabi_flock64;
+            copyto = copy_to_user_eabi_flock64;
+        }
 #endif
 
 	cmd = target_to_host_fcntl_cmd(arg2);
@@ -8781,80 +10839,23 @@
 
         switch(arg2) {
         case TARGET_F_GETLK64:
-#ifdef TARGET_ARM
-            if (((CPUARMState *)cpu_env)->eabi) {
-                if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) 
-                    goto efault;
-                fl.l_type = tswap16(target_efl->l_type);
-                fl.l_whence = tswap16(target_efl->l_whence);
-                fl.l_start = tswap64(target_efl->l_start);
-                fl.l_len = tswap64(target_efl->l_len);
-                fl.l_pid = tswap32(target_efl->l_pid);
-                unlock_user_struct(target_efl, arg3, 0);
-            } else
-#endif
-            {
-                if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) 
-                    goto efault;
-                fl.l_type = tswap16(target_fl->l_type);
-                fl.l_whence = tswap16(target_fl->l_whence);
-                fl.l_start = tswap64(target_fl->l_start);
-                fl.l_len = tswap64(target_fl->l_len);
-                fl.l_pid = tswap32(target_fl->l_pid);
-                unlock_user_struct(target_fl, arg3, 0);
+            ret = copyfrom(&fl, arg3);
+            if (ret) {
+                break;
             }
             ret = get_errno(fcntl(arg1, cmd, &fl));
-	    if (ret == 0) {
-#ifdef TARGET_ARM
-                if (((CPUARMState *)cpu_env)->eabi) {
-                    if (!lock_user_struct(VERIFY_WRITE, target_efl, arg3, 0)) 
-                        goto efault;
-                    target_efl->l_type = tswap16(fl.l_type);
-                    target_efl->l_whence = tswap16(fl.l_whence);
-                    target_efl->l_start = tswap64(fl.l_start);
-                    target_efl->l_len = tswap64(fl.l_len);
-                    target_efl->l_pid = tswap32(fl.l_pid);
-                    unlock_user_struct(target_efl, arg3, 1);
-                } else
-#endif
-                {
-                    if (!lock_user_struct(VERIFY_WRITE, target_fl, arg3, 0)) 
-                        goto efault;
-                    target_fl->l_type = tswap16(fl.l_type);
-                    target_fl->l_whence = tswap16(fl.l_whence);
-                    target_fl->l_start = tswap64(fl.l_start);
-                    target_fl->l_len = tswap64(fl.l_len);
-                    target_fl->l_pid = tswap32(fl.l_pid);
-                    unlock_user_struct(target_fl, arg3, 1);
-                }
-	    }
+            if (ret == 0) {
+                ret = copyto(arg3, &fl);
+            }
 	    break;
 
         case TARGET_F_SETLK64:
         case TARGET_F_SETLKW64:
-#ifdef TARGET_ARM
-            if (((CPUARMState *)cpu_env)->eabi) {
-                if (!lock_user_struct(VERIFY_READ, target_efl, arg3, 1)) 
-                    goto efault;
-                fl.l_type = tswap16(target_efl->l_type);
-                fl.l_whence = tswap16(target_efl->l_whence);
-                fl.l_start = tswap64(target_efl->l_start);
-                fl.l_len = tswap64(target_efl->l_len);
-                fl.l_pid = tswap32(target_efl->l_pid);
-                unlock_user_struct(target_efl, arg3, 0);
-            } else
-#endif
-            {
-                if (!lock_user_struct(VERIFY_READ, target_fl, arg3, 1)) 
-                    goto efault;
-                fl.l_type = tswap16(target_fl->l_type);
-                fl.l_whence = tswap16(target_fl->l_whence);
-                fl.l_start = tswap64(target_fl->l_start);
-                fl.l_len = tswap64(target_fl->l_len);
-                fl.l_pid = tswap32(target_fl->l_pid);
-                unlock_user_struct(target_fl, arg3, 0);
+            ret = copyfrom(&fl, arg3);
+            if (ret) {
+                break;
             }
-            ret = get_errno(fcntl(arg1, cmd, &fl));
+            ret = get_errno(safe_fcntl(arg1, cmd, &fl));
 	    break;
         default:
             ret = do_fcntl(arg1, arg2, arg3);
@@ -9138,14 +11139,15 @@
     {
         struct timespec ts;
         target_to_host_timespec(&ts, arg3);
-        ret = get_errno(clock_nanosleep(arg1, arg2, &ts, arg4 ? &ts : NULL));
+        ret = get_errno(safe_clock_nanosleep(arg1, arg2,
+                                             &ts, arg4 ? &ts : NULL));
         if (arg4)
             host_to_target_timespec(arg4, &ts);
 
 #if defined(TARGET_PPC)
         /* clock_nanosleep is odd in that it returns positive errno values.
          * On PPC, CR0 bit 3 should be set in such a situation. */
-        if (ret) {
+        if (ret && ret != -TARGET_ERESTARTSYS) {
             ((CPUPPCState *)cpu_env)->crf[0] |= 1;
         }
 #endif
@@ -9159,18 +11161,14 @@
         break;
 #endif
 
-#if defined(TARGET_NR_tkill) && defined(__NR_tkill)
     case TARGET_NR_tkill:
-        ret = get_errno(sys_tkill((int)arg1, target_to_host_signal(arg2)));
+        ret = get_errno(safe_tkill((int)arg1, target_to_host_signal(arg2)));
         break;
-#endif
 
-#if defined(TARGET_NR_tgkill) && defined(__NR_tgkill)
     case TARGET_NR_tgkill:
-	ret = get_errno(sys_tgkill((int)arg1, (int)arg2,
+        ret = get_errno(safe_tgkill((int)arg1, (int)arg2,
                         target_to_host_signal(arg3)));
-	break;
-#endif
+        break;
 
 #ifdef TARGET_NR_set_robust_list
     case TARGET_NR_set_robust_list:
@@ -9272,11 +11270,11 @@
             p = lock_user (VERIFY_READ, arg2, arg3, 1);
             if (arg5 != 0) {
                 target_to_host_timespec(&ts, arg5);
-                ret = get_errno(mq_timedsend(arg1, p, arg3, arg4, &ts));
+                ret = get_errno(safe_mq_timedsend(arg1, p, arg3, arg4, &ts));
                 host_to_target_timespec(arg5, &ts);
+            } else {
+                ret = get_errno(safe_mq_timedsend(arg1, p, arg3, arg4, NULL));
             }
-            else
-                ret = get_errno(mq_send(arg1, p, arg3, arg4));
             unlock_user (p, arg2, arg3);
         }
         break;
@@ -9289,11 +11287,13 @@
             p = lock_user (VERIFY_READ, arg2, arg3, 1);
             if (arg5 != 0) {
                 target_to_host_timespec(&ts, arg5);
-                ret = get_errno(mq_timedreceive(arg1, p, arg3, &prio, &ts));
+                ret = get_errno(safe_mq_timedreceive(arg1, p, arg3,
+                                                     &prio, &ts));
                 host_to_target_timespec(arg5, &ts);
+            } else {
+                ret = get_errno(safe_mq_timedreceive(arg1, p, arg3,
+                                                     &prio, NULL));
             }
-            else
-                ret = get_errno(mq_receive(arg1, p, arg3, &prio));
             unlock_user (p, arg2, arg3);
             if (arg4 != 0)
                 put_user_u32(prio, arg4);
@@ -9334,15 +11334,29 @@
         {
             loff_t loff_in, loff_out;
             loff_t *ploff_in = NULL, *ploff_out = NULL;
-            if(arg2) {
-                get_user_u64(loff_in, arg2);
+            if (arg2) {
+                if (get_user_u64(loff_in, arg2)) {
+                    goto efault;
+                }
                 ploff_in = &loff_in;
             }
-            if(arg4) {
-                get_user_u64(loff_out, arg2);
+            if (arg4) {
+                if (get_user_u64(loff_out, arg4)) {
+                    goto efault;
+                }
                 ploff_out = &loff_out;
             }
             ret = get_errno(splice(arg1, ploff_in, arg3, ploff_out, arg5, arg6));
+            if (arg2) {
+                if (put_user_u64(loff_in, arg2)) {
+                    goto efault;
+                }
+            }
+            if (arg4) {
+                if (put_user_u64(loff_out, arg4)) {
+                    goto efault;
+                }
+            }
         }
         break;
 #endif
@@ -9364,6 +11378,7 @@
 #if defined(TARGET_NR_eventfd)
     case TARGET_NR_eventfd:
         ret = get_errno(eventfd(arg1, 0));
+        fd_trans_unregister(ret);
         break;
 #endif
 #if defined(TARGET_NR_eventfd2)
@@ -9377,6 +11392,7 @@
             host_flags |= O_CLOEXEC;
         }
         ret = get_errno(eventfd(arg1, host_flags));
+        fd_trans_unregister(ret);
         break;
     }
 #endif
@@ -9419,6 +11435,16 @@
         break;
 #endif
 #endif
+#if defined(TARGET_NR_signalfd4)
+    case TARGET_NR_signalfd4:
+        ret = do_signalfd4(arg1, arg2, arg4);
+        break;
+#endif
+#if defined(TARGET_NR_signalfd)
+    case TARGET_NR_signalfd:
+        ret = do_signalfd4(arg1, arg2, 0);
+        break;
+#endif
 #if defined(CONFIG_EPOLL)
 #if defined(TARGET_NR_epoll_create)
     case TARGET_NR_epoll_create:
@@ -9454,14 +11480,11 @@
     }
 #endif
 
-#if defined(TARGET_NR_epoll_pwait) && defined(CONFIG_EPOLL_PWAIT)
-#define IMPLEMENT_EPOLL_PWAIT
-#endif
-#if defined(TARGET_NR_epoll_wait) || defined(IMPLEMENT_EPOLL_PWAIT)
+#if defined(TARGET_NR_epoll_wait) || defined(TARGET_NR_epoll_pwait)
 #if defined(TARGET_NR_epoll_wait)
     case TARGET_NR_epoll_wait:
 #endif
-#if defined(IMPLEMENT_EPOLL_PWAIT)
+#if defined(TARGET_NR_epoll_pwait)
     case TARGET_NR_epoll_pwait:
 #endif
     {
@@ -9480,13 +11503,18 @@
         ep = alloca(maxevents * sizeof(struct epoll_event));
 
         switch (num) {
-#if defined(IMPLEMENT_EPOLL_PWAIT)
+#if defined(TARGET_NR_epoll_pwait)
         case TARGET_NR_epoll_pwait:
         {
             target_sigset_t *target_set;
             sigset_t _set, *set = &_set;
 
             if (arg5) {
+                if (arg6 != sizeof(target_sigset_t)) {
+                    ret = -TARGET_EINVAL;
+                    break;
+                }
+
                 target_set = lock_user(VERIFY_READ, arg5,
                                        sizeof(target_sigset_t), 1);
                 if (!target_set) {
@@ -9499,13 +11527,15 @@
                 set = NULL;
             }
 
-            ret = get_errno(epoll_pwait(epfd, ep, maxevents, timeout, set));
+            ret = get_errno(safe_epoll_pwait(epfd, ep, maxevents, timeout,
+                                             set, SIGSET_T_SIZE));
             break;
         }
 #endif
 #if defined(TARGET_NR_epoll_wait)
         case TARGET_NR_epoll_wait:
-            ret = get_errno(epoll_wait(epfd, ep, maxevents, timeout));
+            ret = get_errno(safe_epoll_pwait(epfd, ep, maxevents, timeout,
+                                             NULL, 0));
             break;
 #endif
         default:
@@ -9529,6 +11559,7 @@
         /* args: pid, resource number, ptr to new rlimit, ptr to old rlimit */
         struct target_rlimit64 *target_rnew, *target_rold;
         struct host_rlimit64 rnew, rold, *rnewp = 0;
+        int resource = target_to_host_resource(arg2);
         if (arg3) {
             if (!lock_user_struct(VERIFY_READ, target_rnew, arg3, 1)) {
                 goto efault;
@@ -9539,7 +11570,7 @@
             rnewp = &rnew;
         }
 
-        ret = get_errno(sys_prlimit64(arg1, arg2, rnewp, arg4 ? &rold : 0));
+        ret = get_errno(sys_prlimit64(arg1, resource, rnewp, arg4 ? &rold : 0));
         if (!is_error(ret) && arg4) {
             if (!lock_user_struct(VERIFY_WRITE, target_rold, arg4, 1)) {
                 goto efault;
@@ -9689,6 +11720,7 @@
             timer_t htimer = g_posix_timers[timerid];
             ret = get_errno(timer_getoverrun(htimer));
         }
+        fd_trans_unregister(ret);
         break;
     }
 #endif
@@ -9792,6 +11824,7 @@
 #endif
     if(do_strace)
         print_syscall_ret(num, ret);
+    trace_guest_user_syscall_ret(cpu, num, ret);
     return ret;
 efault:
     ret = -TARGET_EFAULT;
diff --git a/linux-user/syscall_defs.h b/linux-user/syscall_defs.h
index ebb3be1..7835654 100644
--- a/linux-user/syscall_defs.h
+++ b/linux-user/syscall_defs.h
@@ -5,8 +5,7 @@
    necessary */
 
 #ifndef SYSCALL_DEFS_H
-#define SYSCALL_DEFS_H 1
-
+#define SYSCALL_DEFS_H
 
 #include "syscall_nr.h"
 
@@ -28,6 +27,8 @@
 #define SOCKOP_sendmsg          16
 #define SOCKOP_recvmsg          17
 #define SOCKOP_accept4          18
+#define SOCKOP_recvmmsg         19
+#define SOCKOP_sendmmsg         20
 
 #define IPCOP_semop		1
 #define IPCOP_semget		2
@@ -53,7 +54,8 @@
 #define TARGET_IOC_NRBITS	8
 #define TARGET_IOC_TYPEBITS	8
 
-#if defined(TARGET_I386) || (defined(TARGET_ARM) && defined(TARGET_ABI32)) \
+#if (defined(TARGET_I386) && defined(TARGET_ABI32)) \
+    || (defined(TARGET_ARM) && defined(TARGET_ABI32)) \
     || defined(TARGET_SPARC) \
     || defined(TARGET_M68K) || defined(TARGET_SH4) || defined(TARGET_CRIS)
     /* 16 bit uid wrappers emulation */
@@ -64,8 +66,9 @@
 #endif
 
 #if defined(TARGET_I386) || defined(TARGET_ARM) || defined(TARGET_SH4) \
-    || defined(TARGET_M68K) || defined(TARGET_CRIS) || defined(TARGET_UNICORE32) \
-    || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
+    || defined(TARGET_M68K) || defined(TARGET_CRIS) \
+    || defined(TARGET_UNICORE32) || defined(TARGET_S390X) \
+    || defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
 
 #define TARGET_IOC_SIZEBITS	14
 #define TARGET_IOC_DIRBITS	2
@@ -131,6 +134,24 @@
     uint8_t  sll_addr[8];  /* Physical layer address */
 };
 
+struct target_sockaddr_un {
+    uint16_t su_family;
+    uint8_t sun_path[108];
+};
+
+struct target_in_addr {
+    uint32_t s_addr; /* big endian */
+};
+
+struct target_sockaddr_in {
+  uint16_t sin_family;
+  int16_t sin_port; /* big endian */
+  struct target_in_addr sin_addr;
+  uint8_t __pad[sizeof(struct target_sockaddr) -
+                sizeof(uint16_t) - sizeof(int16_t) -
+                sizeof(struct target_in_addr)];
+};
+
 struct target_sock_filter {
     abi_ushort code;
     uint8_t jt;
@@ -143,10 +164,6 @@
     abi_ulong filter;
 };
 
-struct target_in_addr {
-    uint32_t s_addr; /* big endian */
-};
-
 struct target_ip_mreq {
     struct target_in_addr imr_multiaddr;
     struct target_in_addr imr_address;
@@ -234,7 +251,8 @@
 };
 
 #define TARGET_CMSG_DATA(cmsg) ((unsigned char *) ((struct target_cmsghdr *) (cmsg) + 1))
-#define TARGET_CMSG_NXTHDR(mhdr, cmsg) __target_cmsg_nxthdr (mhdr, cmsg)
+#define TARGET_CMSG_NXTHDR(mhdr, cmsg, cmsg_start) \
+                               __target_cmsg_nxthdr(mhdr, cmsg, cmsg_start)
 #define TARGET_CMSG_ALIGN(len) (((len) + sizeof (abi_long) - 1) \
                                & (size_t) ~(sizeof (abi_long) - 1))
 #define TARGET_CMSG_SPACE(len) (TARGET_CMSG_ALIGN (len) \
@@ -242,17 +260,20 @@
 #define TARGET_CMSG_LEN(len)   (TARGET_CMSG_ALIGN (sizeof (struct target_cmsghdr)) + (len))
 
 static __inline__ struct target_cmsghdr *
-__target_cmsg_nxthdr (struct target_msghdr *__mhdr, struct target_cmsghdr *__cmsg)
+__target_cmsg_nxthdr(struct target_msghdr *__mhdr,
+                     struct target_cmsghdr *__cmsg,
+                     struct target_cmsghdr *__cmsg_start)
 {
   struct target_cmsghdr *__ptr;
 
   __ptr = (struct target_cmsghdr *)((unsigned char *) __cmsg
                                     + TARGET_CMSG_ALIGN (tswapal(__cmsg->cmsg_len)));
-  if ((unsigned long)((char *)(__ptr+1) - (char *)(size_t)tswapal(__mhdr->msg_control))
-      > tswapal(__mhdr->msg_controllen))
+  if ((unsigned long)((char *)(__ptr+1) - (char *)__cmsg_start)
+      > tswapal(__mhdr->msg_controllen)) {
     /* No more entries.  */
     return (struct target_cmsghdr *)0;
-  return __cmsg;
+  }
+  return __ptr;
 }
 
 struct target_mmsghdr {
@@ -365,7 +386,8 @@
     || defined(TARGET_PPC) || defined(TARGET_MIPS) || defined(TARGET_SH4) \
     || defined(TARGET_M68K) || defined(TARGET_ALPHA) || defined(TARGET_CRIS) \
     || defined(TARGET_MICROBLAZE) || defined(TARGET_UNICORE32) \
-    || defined(TARGET_S390X) || defined(TARGET_OPENRISC)
+    || defined(TARGET_S390X) || defined(TARGET_OPENRISC) \
+    || defined(TARGET_TILEGX)
 
 #if defined(TARGET_SPARC)
 #define TARGET_SA_NOCLDSTOP    8u
@@ -655,7 +677,29 @@
 #endif
 
 #define TARGET_SI_MAX_SIZE	128
-#define TARGET_SI_PAD_SIZE	((TARGET_SI_MAX_SIZE/sizeof(int)) - 3)
+
+#if TARGET_ABI_BITS == 32
+#define TARGET_SI_PREAMBLE_SIZE (3 * sizeof(int))
+#else
+#define TARGET_SI_PREAMBLE_SIZE (4 * sizeof(int))
+#endif
+
+#define TARGET_SI_PAD_SIZE ((TARGET_SI_MAX_SIZE - TARGET_SI_PREAMBLE_SIZE) / sizeof(int))
+
+/* Within QEMU the top 16 bits of si_code indicate which of the parts of
+ * the union in target_siginfo is valid. This only applies between
+ * host_to_target_siginfo_noswap() and tswap_siginfo(); it does not
+ * appear either within host siginfo_t or in target_siginfo structures
+ * which we get from the guest userspace program. (The Linux kernel
+ * does a similar thing with using the top bits for its own internal
+ * purposes but not letting them be visible to userspace.)
+ */
+#define QEMU_SI_KILL 0
+#define QEMU_SI_TIMER 1
+#define QEMU_SI_POLL 2
+#define QEMU_SI_FAULT 3
+#define QEMU_SI_CHLD 4
+#define QEMU_SI_RT 5
 
 typedef struct target_siginfo {
 #ifdef TARGET_MIPS
@@ -735,6 +779,10 @@
 #define TARGET_ILL_PRVREG	(6)	/* privileged register */
 #define TARGET_ILL_COPROC	(7)	/* coprocessor error */
 #define TARGET_ILL_BADSTK	(8)	/* internal stack error */
+#ifdef TARGET_TILEGX
+#define TARGET_ILL_DBLFLT       (9)     /* double fault */
+#define TARGET_ILL_HARDWALL     (10)    /* user networks hardwall violation */
+#endif
 
 /*
  * SIGFPE si_codes
@@ -754,6 +802,7 @@
  */
 #define TARGET_SEGV_MAPERR     (1)  /* address not mapped to object */
 #define TARGET_SEGV_ACCERR     (2)  /* invalid permissions for mapped object */
+#define TARGET_SEGV_BNDERR     (3)  /* failed address bound checks */
 
 /*
  * SIGBUS si_codes
@@ -761,12 +810,18 @@
 #define TARGET_BUS_ADRALN       (1)	/* invalid address alignment */
 #define TARGET_BUS_ADRERR       (2)	/* non-existent physical address */
 #define TARGET_BUS_OBJERR       (3)	/* object specific hardware error */
+/* hardware memory error consumed on a machine check: action required */
+#define TARGET_BUS_MCEERR_AR    (4)
+/* hardware memory error detected in process but not consumed: action optional*/
+#define TARGET_BUS_MCEERR_AO    (5)
 
 /*
  * SIGTRAP si_codes
  */
 #define TARGET_TRAP_BRKPT	(1)	/* process breakpoint */
 #define TARGET_TRAP_TRACE	(2)	/* process trace trap */
+#define TARGET_TRAP_BRANCH      (3)     /* process taken branch trap */
+#define TARGET_TRAP_HWBKPT      (4)     /* hardware breakpoint/watchpoint */
 
 #endif /* defined(TARGET_I386) || defined(TARGET_ARM) */
 
@@ -930,6 +985,17 @@
 #define TARGET_BLKGETSIZE64 TARGET_IOR(0x12,114,abi_ulong)
                                              /* return device size in bytes
                                                 (u64 *arg) */
+
+#define TARGET_BLKDISCARD TARGET_IO(0x12, 119)
+#define TARGET_BLKIOMIN TARGET_IO(0x12, 120)
+#define TARGET_BLKIOOPT TARGET_IO(0x12, 121)
+#define TARGET_BLKALIGNOFF TARGET_IO(0x12, 122)
+#define TARGET_BLKPBSZGET TARGET_IO(0x12, 123)
+#define TARGET_BLKDISCARDZEROES TARGET_IO(0x12, 124)
+#define TARGET_BLKSECDISCARD TARGET_IO(0x12, 125)
+#define TARGET_BLKROTATIONAL TARGET_IO(0x12, 126)
+#define TARGET_BLKZEROOUT TARGET_IO(0x12, 127)
+
 #define TARGET_FIBMAP     TARGET_IO(0x00,1)  /* bmap access */
 #define TARGET_FIGETBSZ   TARGET_IO(0x00,2)  /* get the block size used for bmap */
 #define TARGET_FS_IOC_FIEMAP TARGET_IOWR('f',11,struct fiemap)
@@ -962,7 +1028,7 @@
                                            (struct cdrom_multisession) */
 #define TARGET_CDROM_GET_MCN		0x5311 /* Obtain the "Universal Product Code"
                                            if available (struct cdrom_mcn) */
-#define TARGET_CDROM_GET_UPC		TARGET_CDROM_GET_MCN  /* This one is depricated,
+#define TARGET_CDROM_GET_UPC		TARGET_CDROM_GET_MCN  /* This one is deprecated,
                                           but here anyway for compatibility */
 #define TARGET_CDROMRESET		0x5312 /* hard-reset the drive */
 #define TARGET_CDROMVOLREAD		0x5313 /* Get the drive's volume setting
@@ -1062,6 +1128,10 @@
 #define TARGET_LOOP_GET_STATUS64      0x4C05
 #define TARGET_LOOP_CHANGE_FD         0x4C06
 
+#define TARGET_LOOP_CTL_ADD           0x4C80
+#define TARGET_LOOP_CTL_REMOVE        0x4C81
+#define TARGET_LOOP_CTL_GET_FREE      0x4C82
+
 /* fb ioctls */
 #define TARGET_FBIOGET_VSCREENINFO    0x4600
 #define TARGET_FBIOPUT_VSCREENINFO    0x4601
@@ -1600,73 +1670,25 @@
 #elif defined(TARGET_ABI_MIPSN32)
 
 struct target_stat {
-	unsigned	st_dev;
-	int		st_pad1[3];		/* Reserved for network id */
-	unsigned int	st_ino;
-	unsigned int	st_mode;
-	unsigned int	st_nlink;
-	int		st_uid;
-	int		st_gid;
-	unsigned 	st_rdev;
-	unsigned int	st_pad2[2];
-	unsigned int	st_size;
-	unsigned int	st_pad3;
-	/*
-	 * Actually this should be timestruc_t st_atime, st_mtime and st_ctime
-	 * but we don't have it under Linux.
-	 */
-	unsigned int		target_st_atime;
-	unsigned int		target_st_atime_nsec;
-	unsigned int		target_st_mtime;
-	unsigned int		target_st_mtime_nsec;
-	unsigned int		target_st_ctime;
-	unsigned int		target_st_ctime_nsec;
-	unsigned int		st_blksize;
-	unsigned int		st_blocks;
-	unsigned int		st_pad4[14];
-};
-
-/*
- * This matches struct stat64 in glibc2.1, hence the absolutely insane
- * amounts of padding around dev_t's.  The memory layout is the same as of
- * struct stat of the 64-bit kernel.
- */
-
-#define TARGET_HAS_STRUCT_STAT64
-struct target_stat64 {
-	unsigned int	st_dev;
-	unsigned int	st_pad0[3];	/* Reserved for st_dev expansion  */
-
-	target_ulong	st_ino;
-
-        unsigned int	st_mode;
-        unsigned int	st_nlink;
-
-	int		st_uid;
-	int		st_gid;
-
-	unsigned int	st_rdev;
-	unsigned int	st_pad1[3];	/* Reserved for st_rdev expansion  */
-
-	int		st_size;
-
-	/*
-	 * Actually this should be timestruc_t st_atime, st_mtime and st_ctime
-	 * but we don't have it under Linux.
-	 */
-	int		target_st_atime;
-	unsigned int	target_st_atime_nsec;	/* Reserved for st_atime expansion  */
-
-	int		target_st_mtime;
-	unsigned int	target_st_mtime_nsec;	/* Reserved for st_mtime expansion  */
-
-	int		target_st_ctime;
-	unsigned int	target_st_ctime_nsec;	/* Reserved for st_ctime expansion  */
-
-	unsigned int	st_blksize;
-	unsigned int	st_pad2;
-
-	int		st_blocks;
+        abi_ulong    st_dev;
+        abi_ulong    st_pad0[3]; /* Reserved for st_dev expansion */
+        uint64_t     st_ino;
+        unsigned int st_mode;
+        unsigned int st_nlink;
+        int          st_uid;
+        int          st_gid;
+        abi_ulong    st_rdev;
+        abi_ulong    st_pad1[3]; /* Reserved for st_rdev expansion */
+        int64_t      st_size;
+        abi_long     target_st_atime;
+        abi_ulong    target_st_atime_nsec; /* Reserved for st_atime expansion */
+        abi_long     target_st_mtime;
+        abi_ulong    target_st_mtime_nsec; /* Reserved for st_mtime expansion */
+        abi_long     target_st_ctime;
+        abi_ulong    target_st_ctime_nsec; /* Reserved for st_ctime expansion */
+        abi_ulong    st_blksize;
+        abi_ulong    st_pad2;
+        int64_t      st_blocks;
 };
 
 #elif defined(TARGET_ABI_MIPSO32)
@@ -1912,7 +1934,7 @@
     abi_ulong  target_st_ctime_nsec;
     unsigned int __unused[2];
 };
-#elif defined(TARGET_OPENRISC)
+#elif defined(TARGET_OPENRISC) || defined(TARGET_TILEGX)
 
 /* These are the asm-generic versions of the stat and stat64 structures */
 
@@ -2172,6 +2194,8 @@
 #define TARGET_F_SETLEASE (TARGET_F_LINUX_SPECIFIC_BASE + 0)
 #define TARGET_F_GETLEASE (TARGET_F_LINUX_SPECIFIC_BASE + 1)
 #define TARGET_F_DUPFD_CLOEXEC (TARGET_F_LINUX_SPECIFIC_BASE + 6)
+#define TARGET_F_SETPIPE_SZ (TARGET_F_LINUX_SPECIFIC_BASE + 7)
+#define TARGET_F_GETPIPE_SZ (TARGET_F_LINUX_SPECIFIC_BASE + 8)
 #define TARGET_F_NOTIFY  (TARGET_F_LINUX_SPECIFIC_BASE+2)
 
 #if defined(TARGET_ALPHA)
@@ -2295,32 +2319,34 @@
 #endif
 
 struct target_flock {
-	short l_type;
-	short l_whence;
-	abi_ulong l_start;
-	abi_ulong l_len;
-	int l_pid;
+    short l_type;
+    short l_whence;
+    abi_long l_start;
+    abi_long l_len;
+    int l_pid;
 };
 
 struct target_flock64 {
-	short  l_type;
-	short  l_whence;
-#if defined(TARGET_PPC) || defined(TARGET_X86_64) || defined(TARGET_MIPS) || defined(TARGET_SPARC) || defined(TARGET_HPPA) || defined (TARGET_MICROBLAZE)
-        int __pad;
+    short  l_type;
+    short  l_whence;
+#if defined(TARGET_PPC) || defined(TARGET_X86_64) || defined(TARGET_MIPS) \
+    || defined(TARGET_SPARC) || defined(TARGET_HPPA) \
+    || defined(TARGET_MICROBLAZE) || defined(TARGET_TILEGX)
+    int __pad;
 #endif
-	unsigned long long l_start;
-	unsigned long long l_len;
-	int  l_pid;
+    abi_llong l_start;
+    abi_llong l_len;
+    int  l_pid;
 } QEMU_PACKED;
 
 #ifdef TARGET_ARM
 struct target_eabi_flock64 {
-	short  l_type;
-	short  l_whence;
-        int __pad;
-	unsigned long long l_start;
-	unsigned long long l_len;
-	int  l_pid;
+    short  l_type;
+    short  l_whence;
+    int __pad;
+    abi_llong l_start;
+    abi_llong l_len;
+    int  l_pid;
 } QEMU_PACKED;
 #endif
 
@@ -2330,7 +2356,7 @@
 };
 
 /* soundcard defines */
-/* XXX: convert them all to arch indepedent entries */
+/* XXX: convert them all to arch independent entries */
 #define TARGET_SNDCTL_COPR_HALT           TARGET_IOWR('C',  7, int);
 #define TARGET_SNDCTL_COPR_LOAD           0xcfb04301
 #define TARGET_SNDCTL_COPR_RCODE          0xc0144303
@@ -2536,20 +2562,23 @@
 #define FUTEX_CMD_MASK          ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME)
 
 #ifdef CONFIG_EPOLL
+#if defined(TARGET_X86_64)
+#define TARGET_EPOLL_PACKED QEMU_PACKED
+#else
+#define TARGET_EPOLL_PACKED
+#endif
+
 typedef union target_epoll_data {
     abi_ulong ptr;
-    abi_ulong fd;
-    uint32_t u32;
-    uint64_t u64;
+    abi_int fd;
+    abi_uint u32;
+    abi_ullong u64;
 } target_epoll_data_t;
 
 struct target_epoll_event {
-    uint32_t events;
-#if defined(TARGET_ARM) || defined(TARGET_MIPS) || defined(TARGET_MIPS64)
-    uint32_t __pad;
-#endif
+    abi_uint events;
     target_epoll_data_t data;
-} QEMU_PACKED;
+} TARGET_EPOLL_PACKED;
 #endif
 struct target_rlimit64 {
     uint64_t rlim_cur;
@@ -2562,8 +2591,6 @@
     uint32_t gid;
 };
 
-#endif
-
 typedef int32_t target_timer_t;
 
 #define TARGET_SIGEV_MAX_SIZE 64
@@ -2605,3 +2632,5 @@
     uint32_t permitted;
     uint32_t inheritable;
 };
+
+#endif
diff --git a/linux-user/syscall_types.h b/linux-user/syscall_types.h
index 1fd4ee0..af79fbf 100644
--- a/linux-user/syscall_types.h
+++ b/linux-user/syscall_types.h
@@ -103,10 +103,11 @@
        TYPE_ULONGLONG,           /* lo_inode */
        TYPE_ULONGLONG,           /* lo_rdevice */
        TYPE_ULONGLONG,           /* lo_offset */
-       TYPE_ULONG,               /* lo_number */
-       TYPE_ULONG,               /* lo_encrypt_type */
-       TYPE_ULONG,               /* lo_encrypt_key_size */
-       TYPE_ULONG,               /* lo_flags */
+       TYPE_ULONGLONG,           /* lo_sizelimit */
+       TYPE_INT,                 /* lo_number */
+       TYPE_INT,                 /* lo_encrypt_type */
+       TYPE_INT,                 /* lo_encrypt_key_size */
+       TYPE_INT,                 /* lo_flags */
        MK_ARRAY(TYPE_CHAR, 64),  /* lo_name */
        MK_ARRAY(TYPE_CHAR, 64),  /* lo_crypt_name */
        MK_ARRAY(TYPE_CHAR, 32),  /* lo_encrypt_key */
diff --git a/linux-user/tilegx/syscall_nr.h b/linux-user/tilegx/syscall_nr.h
new file mode 100644
index 0000000..8e30cd1
--- /dev/null
+++ b/linux-user/tilegx/syscall_nr.h
@@ -0,0 +1,328 @@
+#ifndef TILEGX_SYSCALL_NR_H
+#define TILEGX_SYSCALL_NR_H
+
+/*
+ * Copy from linux kernel asm-generic/unistd.h, which tilegx uses.
+ */
+#define TARGET_NR_io_setup                      0
+#define TARGET_NR_io_destroy                    1
+#define TARGET_NR_io_submit                     2
+#define TARGET_NR_io_cancel                     3
+#define TARGET_NR_io_getevents                  4
+#define TARGET_NR_setxattr                      5
+#define TARGET_NR_lsetxattr                     6
+#define TARGET_NR_fsetxattr                     7
+#define TARGET_NR_getxattr                      8
+#define TARGET_NR_lgetxattr                     9
+#define TARGET_NR_fgetxattr                     10
+#define TARGET_NR_listxattr                     11
+#define TARGET_NR_llistxattr                    12
+#define TARGET_NR_flistxattr                    13
+#define TARGET_NR_removexattr                   14
+#define TARGET_NR_lremovexattr                  15
+#define TARGET_NR_fremovexattr                  16
+#define TARGET_NR_getcwd                        17
+#define TARGET_NR_lookup_dcookie                18
+#define TARGET_NR_eventfd2                      19
+#define TARGET_NR_epoll_create1                 20
+#define TARGET_NR_epoll_ctl                     21
+#define TARGET_NR_epoll_pwait                   22
+#define TARGET_NR_dup                           23
+#define TARGET_NR_dup3                          24
+#define TARGET_NR_fcntl                         25
+#define TARGET_NR_inotify_init1                 26
+#define TARGET_NR_inotify_add_watch             27
+#define TARGET_NR_inotify_rm_watch              28
+#define TARGET_NR_ioctl                         29
+#define TARGET_NR_ioprio_set                    30
+#define TARGET_NR_ioprio_get                    31
+#define TARGET_NR_flock                         32
+#define TARGET_NR_mknodat                       33
+#define TARGET_NR_mkdirat                       34
+#define TARGET_NR_unlinkat                      35
+#define TARGET_NR_symlinkat                     36
+#define TARGET_NR_linkat                        37
+#define TARGET_NR_renameat                      38
+#define TARGET_NR_umount2                       39
+#define TARGET_NR_mount                         40
+#define TARGET_NR_pivot_root                    41
+#define TARGET_NR_nfsservctl                    42
+#define TARGET_NR_statfs                        43
+#define TARGET_NR_fstatfs                       44
+#define TARGET_NR_truncate                      45
+#define TARGET_NR_ftruncate                     46
+#define TARGET_NR_fallocate                     47
+#define TARGET_NR_faccessat                     48
+#define TARGET_NR_chdir                         49
+#define TARGET_NR_fchdir                        50
+#define TARGET_NR_chroot                        51
+#define TARGET_NR_fchmod                        52
+#define TARGET_NR_fchmodat                      53
+#define TARGET_NR_fchownat                      54
+#define TARGET_NR_fchown                        55
+#define TARGET_NR_openat                        56
+#define TARGET_NR_close                         57
+#define TARGET_NR_vhangup                       58
+#define TARGET_NR_pipe2                         59
+#define TARGET_NR_quotactl                      60
+#define TARGET_NR_getdents64                    61
+#define TARGET_NR_lseek                         62
+#define TARGET_NR_read                          63
+#define TARGET_NR_write                         64
+#define TARGET_NR_readv                         65
+#define TARGET_NR_writev                        66
+#define TARGET_NR_pread64                       67
+#define TARGET_NR_pwrite64                      68
+#define TARGET_NR_preadv                        69
+#define TARGET_NR_pwritev                       70
+#define TARGET_NR_sendfile                      71
+#define TARGET_NR_pselect6                      72
+#define TARGET_NR_ppoll                         73
+#define TARGET_NR_signalfd4                     74
+#define TARGET_NR_vmsplice                      75
+#define TARGET_NR_splice                        76
+#define TARGET_NR_tee                           77
+#define TARGET_NR_readlinkat                    78
+#define TARGET_NR_fstatat64                     79 /* let syscall.c known */
+#define TARGET_NR_fstat                         80
+#define TARGET_NR_sync                          81
+#define TARGET_NR_fsync                         82
+#define TARGET_NR_fdatasync                     83
+#define TARGET_NR_sync_file_range               84 /* For tilegx, no range2 */
+#define TARGET_NR_timerfd_create                85
+#define TARGET_NR_timerfd_settime               86
+#define TARGET_NR_timerfd_gettime               87
+#define TARGET_NR_utimensat                     88
+#define TARGET_NR_acct                          89
+#define TARGET_NR_capget                        90
+#define TARGET_NR_capset                        91
+#define TARGET_NR_personality                   92
+#define TARGET_NR_exit                          93
+#define TARGET_NR_exit_group                    94
+#define TARGET_NR_waitid                        95
+#define TARGET_NR_set_tid_address               96
+#define TARGET_NR_unshare                       97
+#define TARGET_NR_futex                         98
+#define TARGET_NR_set_robust_list               99
+#define TARGET_NR_get_robust_list               100
+#define TARGET_NR_nanosleep                     101
+#define TARGET_NR_getitimer                     102
+#define TARGET_NR_setitimer                     103
+#define TARGET_NR_kexec_load                    104
+#define TARGET_NR_init_module                   105
+#define TARGET_NR_delete_module                 106
+#define TARGET_NR_timer_create                  107
+#define TARGET_NR_timer_gettime                 108
+#define TARGET_NR_timer_getoverrun              109
+#define TARGET_NR_timer_settime                 110
+#define TARGET_NR_timer_delete                  111
+#define TARGET_NR_clock_settime                 112
+#define TARGET_NR_clock_gettime                 113
+#define TARGET_NR_clock_getres                  114
+#define TARGET_NR_clock_nanosleep               115
+#define TARGET_NR_syslog                        116
+#define TARGET_NR_ptrace                        117
+#define TARGET_NR_sched_setparam                118
+#define TARGET_NR_sched_setscheduler            119
+#define TARGET_NR_sched_getscheduler            120
+#define TARGET_NR_sched_getparam                121
+#define TARGET_NR_sched_setaffinity             122
+#define TARGET_NR_sched_getaffinity             123
+#define TARGET_NR_sched_yield                   124
+#define TARGET_NR_sched_get_priority_max        125
+#define TARGET_NR_sched_get_priority_min        126
+#define TARGET_NR_sched_rr_get_interval         127
+#define TARGET_NR_restart_syscall               128
+#define TARGET_NR_kill                          129
+#define TARGET_NR_tkill                         130
+#define TARGET_NR_tgkill                        131
+#define TARGET_NR_sigaltstack                   132
+#define TARGET_NR_rt_sigsuspend                 133
+#define TARGET_NR_rt_sigaction                  134
+#define TARGET_NR_rt_sigprocmask                135
+#define TARGET_NR_rt_sigpending                 136
+#define TARGET_NR_rt_sigtimedwait               137
+#define TARGET_NR_rt_sigqueueinfo               138
+#define TARGET_NR_rt_sigreturn                  139
+#define TARGET_NR_setpriority                   140
+#define TARGET_NR_getpriority                   141
+#define TARGET_NR_reboot                        142
+#define TARGET_NR_setregid                      143
+#define TARGET_NR_setgid                        144
+#define TARGET_NR_setreuid                      145
+#define TARGET_NR_setuid                        146
+#define TARGET_NR_setresuid                     147
+#define TARGET_NR_getresuid                     148
+#define TARGET_NR_setresgid                     149
+#define TARGET_NR_getresgid                     150
+#define TARGET_NR_setfsuid                      151
+#define TARGET_NR_setfsgid                      152
+#define TARGET_NR_times                         153
+#define TARGET_NR_setpgid                       154
+#define TARGET_NR_getpgid                       155
+#define TARGET_NR_getsid                        156
+#define TARGET_NR_setsid                        157
+#define TARGET_NR_getgroups                     158
+#define TARGET_NR_setgroups                     159
+#define TARGET_NR_uname                         160
+#define TARGET_NR_sethostname                   161
+#define TARGET_NR_setdomainname                 162
+#define TARGET_NR_getrlimit                     163
+#define TARGET_NR_setrlimit                     164
+#define TARGET_NR_getrusage                     165
+#define TARGET_NR_umask                         166
+#define TARGET_NR_prctl                         167
+#define TARGET_NR_getcpu                        168
+#define TARGET_NR_gettimeofday                  169
+#define TARGET_NR_settimeofday                  170
+#define TARGET_NR_adjtimex                      171
+#define TARGET_NR_getpid                        172
+#define TARGET_NR_getppid                       173
+#define TARGET_NR_getuid                        174
+#define TARGET_NR_geteuid                       175
+#define TARGET_NR_getgid                        176
+#define TARGET_NR_getegid                       177
+#define TARGET_NR_gettid                        178
+#define TARGET_NR_sysinfo                       179
+#define TARGET_NR_mq_open                       180
+#define TARGET_NR_mq_unlink                     181
+#define TARGET_NR_mq_timedsend                  182
+#define TARGET_NR_mq_timedreceive               183
+#define TARGET_NR_mq_notify                     184
+#define TARGET_NR_mq_getsetattr                 185
+#define TARGET_NR_msgget                        186
+#define TARGET_NR_msgctl                        187
+#define TARGET_NR_msgrcv                        188
+#define TARGET_NR_msgsnd                        189
+#define TARGET_NR_semget                        190
+#define TARGET_NR_semctl                        191
+#define TARGET_NR_semtimedop                    192
+#define TARGET_NR_semop                         193
+#define TARGET_NR_shmget                        194
+#define TARGET_NR_shmctl                        195
+#define TARGET_NR_shmat                         196
+#define TARGET_NR_shmdt                         197
+#define TARGET_NR_socket                        198
+#define TARGET_NR_socketpair                    199
+#define TARGET_NR_bind                          200
+#define TARGET_NR_listen                        201
+#define TARGET_NR_accept                        202
+#define TARGET_NR_connect                       203
+#define TARGET_NR_getsockname                   204
+#define TARGET_NR_getpeername                   205
+#define TARGET_NR_sendto                        206
+#define TARGET_NR_recvfrom                      207
+#define TARGET_NR_setsockopt                    208
+#define TARGET_NR_getsockopt                    209
+#define TARGET_NR_shutdown                      210
+#define TARGET_NR_sendmsg                       211
+#define TARGET_NR_recvmsg                       212
+#define TARGET_NR_readahead                     213
+#define TARGET_NR_brk                           214
+#define TARGET_NR_munmap                        215
+#define TARGET_NR_mremap                        216
+#define TARGET_NR_add_key                       217
+#define TARGET_NR_request_key                   218
+#define TARGET_NR_keyctl                        219
+#define TARGET_NR_clone                         220
+#define TARGET_NR_execve                        221
+#define TARGET_NR_mmap                          222
+#define TARGET_NR_fadvise64                     223
+#define TARGET_NR_swapon                        224
+#define TARGET_NR_swapoff                       225
+#define TARGET_NR_mprotect                      226
+#define TARGET_NR_msync                         227
+#define TARGET_NR_mlock                         228
+#define TARGET_NR_munlock                       229
+#define TARGET_NR_mlockall                      230
+#define TARGET_NR_munlockall                    231
+#define TARGET_NR_mincore                       232
+#define TARGET_NR_madvise                       233
+#define TARGET_NR_remap_file_pages              234
+#define TARGET_NR_mbind                         235
+#define TARGET_NR_get_mempolicy                 236
+#define TARGET_NR_set_mempolicy                 237
+#define TARGET_NR_migrate_pages                 238
+#define TARGET_NR_move_pages                    239
+#define TARGET_NR_rt_tgsigqueueinfo             240
+#define TARGET_NR_perf_event_open               241
+#define TARGET_NR_accept4                       242
+#define TARGET_NR_recvmmsg                      243
+
+#define TARGET_NR_arch_specific_syscall         244
+#define TARGET_NR_cacheflush                    245  /* tilegx own syscall */
+
+#define TARGET_NR_wait4                         260
+#define TARGET_NR_prlimit64                     261
+#define TARGET_NR_fanotify_init                 262
+#define TARGET_NR_fanotify_mark                 263
+#define TARGET_NR_name_to_handle_at             264
+#define TARGET_NR_open_by_handle_at             265
+#define TARGET_NR_clock_adjtime                 266
+#define TARGET_NR_syncfs                        267
+#define TARGET_NR_setns                         268
+#define TARGET_NR_sendmmsg                      269
+#define TARGET_NR_process_vm_readv              270
+#define TARGET_NR_process_vm_writev             271
+#define TARGET_NR_kcmp                          272
+#define TARGET_NR_finit_module                  273
+#define TARGET_NR_sched_setattr                 274
+#define TARGET_NR_sched_getattr                 275
+#define TARGET_NR_renameat2                     276
+#define TARGET_NR_seccomp                       277
+#define TARGET_NR_getrandom                     278
+#define TARGET_NR_memfd_create                  279
+#define TARGET_NR_bpf                           280
+#define TARGET_NR_execveat                      281
+#define TARGET_NR_userfaultfd                   282
+#define TARGET_NR_membarrier                    283
+#define TARGET_NR_mlock2                        284
+#define TARGET_NR_copy_file_range               285
+
+#define TARGET_NR_open                          1024
+#define TARGET_NR_link                          1025
+#define TARGET_NR_unlink                        1026
+#define TARGET_NR_mknod                         1027
+#define TARGET_NR_chmod                         1028
+#define TARGET_NR_chown                         1029
+#define TARGET_NR_mkdir                         1030
+#define TARGET_NR_rmdir                         1031
+#define TARGET_NR_lchown                        1032
+#define TARGET_NR_access                        1033
+#define TARGET_NR_rename                        1034
+#define TARGET_NR_readlink                      1035
+#define TARGET_NR_symlink                       1036
+#define TARGET_NR_utimes                        1037
+#define TARGET_NR_stat64                        1038 /* let syscall.c known */
+#define TARGET_NR_lstat                         1039
+
+#define TARGET_NR_pipe                          1040
+#define TARGET_NR_dup2                          1041
+#define TARGET_NR_epoll_create                  1042
+#define TARGET_NR_inotify_init                  1043
+#define TARGET_NR_eventfd                       1044
+#define TARGET_NR_signalfd                      1045
+
+#define TARGET_NR_alarm                         1059
+#define TARGET_NR_getpgrp                       1060
+#define TARGET_NR_pause                         1061
+#define TARGET_NR_time                          1062
+#define TARGET_NR_utime                         1063
+#define TARGET_NR_creat                         1064
+#define TARGET_NR_getdents                      1065
+#define TARGET_NR_futimesat                     1066
+#define TARGET_NR_select                        1067
+#define TARGET_NR_poll                          1068
+#define TARGET_NR_epoll_wait                    1069
+#define TARGET_NR_ustat                         1070
+#define TARGET_NR_vfork                         1071
+#define TARGET_NR_oldwait4                      1072
+#define TARGET_NR_recv                          1073
+#define TARGET_NR_send                          1074
+#define TARGET_NR_bdflush                       1075
+#define TARGET_NR_umount                        1076
+#define TARGET_NR_uselib                        1077
+#define TARGET_NR__sysctl                       1078
+#define TARGET_NR_fork                          1079
+
+#endif
diff --git a/linux-user/tilegx/target_cpu.h b/linux-user/tilegx/target_cpu.h
new file mode 100644
index 0000000..4878e01
--- /dev/null
+++ b/linux-user/tilegx/target_cpu.h
@@ -0,0 +1,35 @@
+/*
+ * TILE-Gx specific CPU ABI and functions for linux-user
+ *
+ * Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TILEGX_TARGET_CPU_H
+#define TILEGX_TARGET_CPU_H
+
+static inline void cpu_clone_regs(CPUTLGState *env, target_ulong newsp)
+{
+    if (newsp) {
+        env->regs[TILEGX_R_SP] = newsp;
+    }
+    env->regs[TILEGX_R_RE] = 0;
+}
+
+static inline void cpu_set_tls(CPUTLGState *env, target_ulong newtls)
+{
+    env->regs[TILEGX_R_TP] = newtls;
+}
+
+#endif
diff --git a/linux-user/tilegx/target_signal.h b/linux-user/tilegx/target_signal.h
new file mode 100644
index 0000000..f64551a
--- /dev/null
+++ b/linux-user/tilegx/target_signal.h
@@ -0,0 +1,29 @@
+#ifndef TILEGX_TARGET_SIGNAL_H
+#define TILEGX_TARGET_SIGNAL_H
+
+#include "cpu.h"
+
+/* this struct defines a stack used during syscall handling */
+
+typedef struct target_sigaltstack {
+    abi_ulong ss_sp;
+    abi_int ss_flags;
+    abi_ulong ss_size;
+} target_stack_t;
+
+/*
+ * sigaltstack controls
+ */
+#define TARGET_SS_ONSTACK     1
+#define TARGET_SS_DISABLE     2
+
+#define TARGET_MINSIGSTKSZ    2048
+#define TARGET_SIGSTKSZ       8192
+
+static inline abi_ulong get_sp_from_cpustate(CPUTLGState *state)
+{
+    return state->regs[TILEGX_R_SP];
+}
+
+
+#endif /* TILEGX_TARGET_SIGNAL_H */
diff --git a/linux-user/tilegx/target_structs.h b/linux-user/tilegx/target_structs.h
new file mode 100644
index 0000000..de8b1f2
--- /dev/null
+++ b/linux-user/tilegx/target_structs.h
@@ -0,0 +1,46 @@
+/*
+ * TILE-Gx specific structures for linux-user
+ *
+ * Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef TILEGX_TARGET_STRUCTS_H
+#define TILEGX_TARGET_STRUCTS_H
+
+struct target_ipc_perm {
+    abi_int __key;                      /* Key.  */
+    abi_uint uid;                       /* Owner's user ID.  */
+    abi_uint gid;                       /* Owner's group ID.  */
+    abi_uint cuid;                      /* Creator's user ID.  */
+    abi_uint cgid;                      /* Creator's group ID.  */
+    abi_uint mode;                      /* Read/write permission.  */
+    abi_ushort __seq;                   /* Sequence number.  */
+};
+
+struct target_shmid_ds {
+    struct target_ipc_perm shm_perm;    /* operation permission struct */
+    abi_long shm_segsz;                 /* size of segment in bytes */
+    abi_ulong shm_atime;                /* time of last shmat() */
+    abi_ulong shm_dtime;                /* time of last shmdt() */
+    abi_ulong shm_ctime;                /* time of last change by shmctl() */
+    abi_int shm_cpid;                   /* pid of creator */
+    abi_int shm_lpid;                   /* pid of last shmop */
+    abi_ushort shm_nattch;              /* number of current attaches */
+    abi_ushort shm_unused;              /* compatibility */
+    abi_ulong __unused4;
+    abi_ulong __unused5;
+};
+
+#endif
diff --git a/linux-user/tilegx/target_syscall.h b/linux-user/tilegx/target_syscall.h
new file mode 100644
index 0000000..d731acd
--- /dev/null
+++ b/linux-user/tilegx/target_syscall.h
@@ -0,0 +1,43 @@
+#ifndef TILEGX_TARGET_SYSCALL_H
+#define TILEGX_TARGET_SYSCALL_H
+
+#define UNAME_MACHINE "tilegx"
+#define UNAME_MINIMUM_RELEASE "3.19"
+
+#define MMAP_SHIFT TARGET_PAGE_BITS
+
+#define TILEGX_IS_ERRNO(ret) \
+                       ((ret) > 0xfffffffffffff000ULL) /* errno is 0 -- 4096 */
+
+typedef uint64_t tilegx_reg_t;
+
+struct target_pt_regs {
+
+    union {
+        /* Saved main processor registers; 56..63 are special. */
+        tilegx_reg_t regs[56];
+        struct {
+            tilegx_reg_t __regs[53];
+            tilegx_reg_t tp;    /* aliases regs[TREG_TP] */
+            tilegx_reg_t sp;    /* aliases regs[TREG_SP] */
+            tilegx_reg_t lr;    /* aliases regs[TREG_LR] */
+        };
+    };
+
+    /* Saved special registers. */
+    tilegx_reg_t pc;            /* stored in EX_CONTEXT_K_0 */
+    tilegx_reg_t ex1;           /* stored in EX_CONTEXT_K_1 (PL and ICS bit) */
+    tilegx_reg_t faultnum;      /* fault number (INT_SWINT_1 for syscall) */
+    tilegx_reg_t orig_r0;       /* r0 at syscall entry, else zero */
+    tilegx_reg_t flags;         /* flags (see below) */
+    tilegx_reg_t cmpexch;       /* value of CMPEXCH_VALUE SPR at interrupt */
+    tilegx_reg_t pad[2];
+};
+
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+/* For faultnum */
+#define TARGET_INT_SWINT_1            14
+
+#endif
diff --git a/linux-user/tilegx/termbits.h b/linux-user/tilegx/termbits.h
new file mode 100644
index 0000000..91ec236
--- /dev/null
+++ b/linux-user/tilegx/termbits.h
@@ -0,0 +1,274 @@
+#ifndef TILEGX_TERMBITS_H
+#define TILEGX_TERMBITS_H
+
+/* From asm-generic/termbits.h, which is used by tilegx */
+
+#define TARGET_NCCS 19
+struct target_termios {
+    unsigned int c_iflag;             /* input mode flags */
+    unsigned int c_oflag;             /* output mode flags */
+    unsigned int c_cflag;             /* control mode flags */
+    unsigned int c_lflag;             /* local mode flags */
+    unsigned char c_line;             /* line discipline */
+    unsigned char c_cc[TARGET_NCCS];  /* control characters */
+};
+
+struct target_termios2 {
+    unsigned int c_iflag;             /* input mode flags */
+    unsigned int c_oflag;             /* output mode flags */
+    unsigned int c_cflag;             /* control mode flags */
+    unsigned int c_lflag;             /* local mode flags */
+    unsigned char c_line;             /* line discipline */
+    unsigned char c_cc[TARGET_NCCS];  /* control characters */
+    unsigned int c_ispeed;            /* input speed */
+    unsigned int c_ospeed;            /* output speed */
+};
+
+/* c_cc characters */
+#define TARGET_VINTR     0
+#define TARGET_VQUIT     1
+#define TARGET_VERASE    2
+#define TARGET_VKILL     3
+#define TARGET_VEOF      4
+#define TARGET_VTIME     5
+#define TARGET_VMIN      6
+#define TARGET_VSWTC     7
+#define TARGET_VSTART    8
+#define TARGET_VSTOP     9
+#define TARGET_VSUSP     10
+#define TARGET_VEOL      11
+#define TARGET_VREPRINT  12
+#define TARGET_VDISCARD  13
+#define TARGET_VWERASE   14
+#define TARGET_VLNEXT    15
+#define TARGET_VEOL2     16
+
+/* c_iflag bits */
+#define TARGET_IGNBRK    0000001
+#define TARGET_BRKINT    0000002
+#define TARGET_IGNPAR    0000004
+#define TARGET_PARMRK    0000010
+#define TARGET_INPCK     0000020
+#define TARGET_ISTRIP    0000040
+#define TARGET_INLCR     0000100
+#define TARGET_IGNCR     0000200
+#define TARGET_ICRNL     0000400
+#define TARGET_IUCLC     0001000
+#define TARGET_IXON      0002000
+#define TARGET_IXANY     0004000
+#define TARGET_IXOFF     0010000
+#define TARGET_IMAXBEL   0020000
+#define TARGET_IUTF8     0040000
+
+/* c_oflag bits */
+#define TARGET_OPOST     0000001
+#define TARGET_OLCUC     0000002
+#define TARGET_ONLCR     0000004
+#define TARGET_OCRNL     0000010
+#define TARGET_ONOCR     0000020
+#define TARGET_ONLRET    0000040
+#define TARGET_OFILL     0000100
+#define TARGET_OFDEL     0000200
+#define TARGET_NLDLY     0000400
+#define   TARGET_NL0     0000000
+#define   TARGET_NL1     0000400
+#define TARGET_CRDLY     0003000
+#define   TARGET_CR0     0000000
+#define   TARGET_CR1     0001000
+#define   TARGET_CR2     0002000
+#define   TARGET_CR3     0003000
+#define TARGET_TABDLY    0014000
+#define   TARGET_TAB0    0000000
+#define   TARGET_TAB1    0004000
+#define   TARGET_TAB2    0010000
+#define   TARGET_TAB3    0014000
+#define   TARGET_XTABS   0014000
+#define TARGET_BSDLY     0020000
+#define   TARGET_BS0     0000000
+#define   TARGET_BS1     0020000
+#define TARGET_VTDLY     0040000
+#define   TARGET_VT0     0000000
+#define   TARGET_VT1     0040000
+#define TARGET_FFDLY     0100000
+#define   TARGET_FF0     0000000
+#define   TARGET_FF1     0100000
+
+/* c_cflag bit meaning */
+#define TARGET_CBAUD     0010017
+#define  TARGET_B0       0000000        /* hang up */
+#define  TARGET_B50      0000001
+#define  TARGET_B75      0000002
+#define  TARGET_B110     0000003
+#define  TARGET_B134     0000004
+#define  TARGET_B150     0000005
+#define  TARGET_B200     0000006
+#define  TARGET_B300     0000007
+#define  TARGET_B600     0000010
+#define  TARGET_B1200    0000011
+#define  TARGET_B1800    0000012
+#define  TARGET_B2400    0000013
+#define  TARGET_B4800    0000014
+#define  TARGET_B9600    0000015
+#define  TARGET_B19200   0000016
+#define  TARGET_B38400   0000017
+#define TARGET_EXTA      TARGET_B19200
+#define TARGET_EXTB      TARGET_B38400
+#define TARGET_CSIZE     0000060
+#define   TARGET_CS5     0000000
+#define   TARGET_CS6     0000020
+#define   TARGET_CS7     0000040
+#define   TARGET_CS8     0000060
+#define TARGET_CSTOPB    0000100
+#define TARGET_CREAD     0000200
+#define TARGET_PARENB    0000400
+#define TARGET_PARODD    0001000
+#define TARGET_HUPCL     0002000
+#define TARGET_CLOCAL    0004000
+#define TARGET_CBAUDEX   0010000
+#define    TARGET_BOTHER 0010000
+#define    TARGET_B57600 0010001
+#define   TARGET_B115200 0010002
+#define   TARGET_B230400 0010003
+#define   TARGET_B460800 0010004
+#define   TARGET_B500000 0010005
+#define   TARGET_B576000 0010006
+#define   TARGET_B921600 0010007
+#define  TARGET_B1000000 0010010
+#define  TARGET_B1152000 0010011
+#define  TARGET_B1500000 0010012
+#define  TARGET_B2000000 0010013
+#define  TARGET_B2500000 0010014
+#define  TARGET_B3000000 0010015
+#define  TARGET_B3500000 0010016
+#define  TARGET_B4000000 0010017
+#define TARGET_CIBAUD    002003600000    /* input baud rate */
+#define TARGET_CMSPAR    010000000000    /* mark or space (stick) parity */
+#define TARGET_CRTSCTS   020000000000    /* flow control */
+
+#define TARGET_IBSHIFT   16        /* Shift from CBAUD to CIBAUD */
+
+/* c_lflag bits */
+#define TARGET_ISIG      0000001
+#define TARGET_ICANON    0000002
+#define TARGET_XCASE     0000004
+#define TARGET_ECHO      0000010
+#define TARGET_ECHOE     0000020
+#define TARGET_ECHOK     0000040
+#define TARGET_ECHONL    0000100
+#define TARGET_NOFLSH    0000200
+#define TARGET_TOSTOP    0000400
+#define TARGET_ECHOCTL   0001000
+#define TARGET_ECHOPRT   0002000
+#define TARGET_ECHOKE    0004000
+#define TARGET_FLUSHO    0010000
+#define TARGET_PENDIN    0040000
+#define TARGET_IEXTEN    0100000
+#define TARGET_EXTPROC   0200000
+
+/* tcflow() and TCXONC use these */
+#define TARGET_TCOOFF    0
+#define TARGET_TCOON     1
+#define TARGET_TCIOFF    2
+#define TARGET_TCION     3
+
+/* tcflush() and TCFLSH use these */
+#define TARGET_TCIFLUSH  0
+#define TARGET_TCOFLUSH  1
+#define TARGET_TCIOFLUSH 2
+
+/* tcsetattr uses these */
+#define TARGET_TCSANOW   0
+#define TARGET_TCSADRAIN 1
+#define TARGET_TCSAFLUSH 2
+
+/* From asm-generic/ioctls.h, which is used by tilegx */
+
+#define TARGET_TCGETS                   0x5401
+#define TARGET_TCSETS                   0x5402
+#define TARGET_TCSETSW                  0x5403
+#define TARGET_TCSETSF                  0x5404
+#define TARGET_TCGETA                   0x5405
+#define TARGET_TCSETA                   0x5406
+#define TARGET_TCSETAW                  0x5407
+#define TARGET_TCSETAF                  0x5408
+#define TARGET_TCSBRK                   0x5409
+#define TARGET_TCXONC                   0x540A
+#define TARGET_TCFLSH                   0x540B
+#define TARGET_TIOCEXCL                 0x540C
+#define TARGET_TIOCNXCL                 0x540D
+#define TARGET_TIOCSCTTY                0x540E
+#define TARGET_TIOCGPGRP                0x540F
+#define TARGET_TIOCSPGRP                0x5410
+#define TARGET_TIOCOUTQ                 0x5411
+#define TARGET_TIOCSTI                  0x5412
+#define TARGET_TIOCGWINSZ               0x5413
+#define TARGET_TIOCSWINSZ               0x5414
+#define TARGET_TIOCMGET                 0x5415
+#define TARGET_TIOCMBIS                 0x5416
+#define TARGET_TIOCMBIC                 0x5417
+#define TARGET_TIOCMSET                 0x5418
+#define TARGET_TIOCGSOFTCAR             0x5419
+#define TARGET_TIOCSSOFTCAR             0x541A
+#define TARGET_FIONREAD                 0x541B
+#define TARGET_TIOCINQ                  TARGET_FIONREAD
+#define TARGET_TIOCLINUX                0x541C
+#define TARGET_TIOCCONS                 0x541D
+#define TARGET_TIOCGSERIAL              0x541E
+#define TARGET_TIOCSSERIAL              0x541F
+#define TARGET_TIOCPKT                  0x5420
+#define TARGET_FIONBIO                  0x5421
+#define TARGET_TIOCNOTTY                0x5422
+#define TARGET_TIOCSETD                 0x5423
+#define TARGET_TIOCGETD                 0x5424
+#define TARGET_TCSBRKP                  0x5425
+#define TARGET_TIOCSBRK                 0x5427
+#define TARGET_TIOCCBRK                 0x5428
+#define TARGET_TIOCGSID                 0x5429
+#define TARGET_TCGETS2                  TARGET_IOR('T', 0x2A, struct termios2)
+#define TARGET_TCSETS2                  TARGET_IOW('T', 0x2B, struct termios2)
+#define TARGET_TCSETSW2                 TARGET_IOW('T', 0x2C, struct termios2)
+#define TARGET_TCSETSF2                 TARGET_IOW('T', 0x2D, struct termios2)
+#define TARGET_TIOCGRS485               0x542E
+#define TARGET_TIOCSRS485               0x542F
+#define TARGET_TIOCGPTN                 TARGET_IOR('T', 0x30, unsigned int)
+#define TARGET_TIOCSPTLCK               TARGET_IOW('T', 0x31, int)
+#define TARGET_TIOCGDEV                 TARGET_IOR('T', 0x32, unsigned int)
+#define TARGET_TCGETX                   0x5432
+#define TARGET_TCSETX                   0x5433
+#define TARGET_TCSETXF                  0x5434
+#define TARGET_TCSETXW                  0x5435
+#define TARGET_TIOCSIG                  TARGET_IOW('T', 0x36, int)
+#define TARGET_TIOCVHANGUP              0x5437
+#define TARGET_TIOCGPKT                 TARGET_IOR('T', 0x38, int)
+#define TARGET_TIOCGPTLCK               TARGET_IOR('T', 0x39, int)
+#define TARGET_TIOCGEXCL                TARGET_IOR('T', 0x40, int)
+
+#define TARGET_FIONCLEX                 0x5450
+#define TARGET_FIOCLEX                  0x5451
+#define TARGET_FIOASYNC                 0x5452
+#define TARGET_TIOCSERCONFIG            0x5453
+#define TARGET_TIOCSERGWILD             0x5454
+#define TARGET_TIOCSERSWILD             0x5455
+#define TARGET_TIOCGLCKTRMIOS           0x5456
+#define TARGET_TIOCSLCKTRMIOS           0x5457
+#define TARGET_TIOCSERGSTRUCT           0x5458
+#define TARGET_TIOCSERGETLSR            0x5459
+#define TARGET_TIOCSERGETMULTI          0x545A
+#define TARGET_TIOCSERSETMULTI          0x545B
+
+#define TARGET_TIOCMIWAIT               0x545C
+#define TARGET_TIOCGICOUNT              0x545D
+#define TARGET_FIOQSIZE                 0x5460
+
+#define TARGET_TIOCPKT_DATA             0
+#define TARGET_TIOCPKT_FLUSHREAD        1
+#define TARGET_TIOCPKT_FLUSHWRITE       2
+#define TARGET_TIOCPKT_STOP             4
+#define TARGET_TIOCPKT_START            8
+#define TARGET_TIOCPKT_NOSTOP           16
+#define TARGET_TIOCPKT_DOSTOP           32
+#define TARGET_TIOCPKT_IOCTL            64
+
+#define TARGET_TIOCSER_TEMT             0x01
+
+#endif
diff --git a/linux-user/trace-events b/linux-user/trace-events
new file mode 100644
index 0000000..fc71f91
--- /dev/null
+++ b/linux-user/trace-events
@@ -0,0 +1,12 @@
+# See docs/tracing.txt for syntax documentation.
+
+# linux-user/signal.c
+user_setup_frame(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_setup_rt_frame(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_do_rt_sigreturn(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_do_sigreturn(void *env, uint64_t frame_addr) "env=%p frame_addr=%"PRIx64
+user_force_sig(void *env, int target_sig, int host_sig) "env=%p signal %d (host %d)"
+user_handle_signal(void *env, int target_sig) "env=%p signal %d"
+user_host_signal(void *env, int host_sig, int target_sig) "env=%p signal %d (target %d("
+user_queue_signal(void *env, int target_sig) "env=%p signal %d"
+user_s390x_restore_sigregs(void *env, uint64_t sc_psw_addr, uint64_t env_psw_addr) "env=%p frame psw.addr %"PRIx64 " current psw.addr %"PRIx64
diff --git a/linux-user/uaccess.c b/linux-user/uaccess.c
index a4d108c..0a5c0b0 100644
--- a/linux-user/uaccess.c
+++ b/linux-user/uaccess.c
@@ -1,6 +1,6 @@
 /* User memory access */
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 
 #include "qemu.h"
 
diff --git a/linux-user/uname.c b/linux-user/uname.c
index 1e6560d..313b79d 100644
--- a/linux-user/uname.c
+++ b/linux-user/uname.c
@@ -17,7 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 //#include "qemu-common.h"
diff --git a/linux-user/uname.h b/linux-user/uname.h
index cc62e76..4503094 100644
--- a/linux-user/uname.h
+++ b/linux-user/uname.h
@@ -1,5 +1,5 @@
 #ifndef UNAME_H
-#define UNAME_H 1
+#define UNAME_H
 
 #include <sys/utsname.h>
 #include <linux/utsname.h>
@@ -7,4 +7,4 @@
 const char *cpu_to_uname_machine(void *cpu_env);
 int sys_uname(struct new_utsname *buf);
 
-#endif /* UNAME _H */
+#endif /* UNAME_H */
diff --git a/linux-user/unicore32/syscall.h b/linux-user/unicore32/syscall.h
deleted file mode 100644
index 385a975..0000000
--- a/linux-user/unicore32/syscall.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (C) 2010-2011 GUAN Xue-tao
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#ifndef __UC32_SYSCALL_H__
-#define __UC32_SYSCALL_H__
-struct target_pt_regs {
-    abi_ulong uregs[34];
-};
-
-#define UC32_REG_pc             uregs[31]
-#define UC32_REG_lr             uregs[30]
-#define UC32_REG_sp             uregs[29]
-#define UC32_REG_ip             uregs[28]
-#define UC32_REG_fp             uregs[27]
-#define UC32_REG_26             uregs[26]
-#define UC32_REG_25             uregs[25]
-#define UC32_REG_24             uregs[24]
-#define UC32_REG_23             uregs[23]
-#define UC32_REG_22             uregs[22]
-#define UC32_REG_21             uregs[21]
-#define UC32_REG_20             uregs[20]
-#define UC32_REG_19             uregs[19]
-#define UC32_REG_18             uregs[18]
-#define UC32_REG_17             uregs[17]
-#define UC32_REG_16             uregs[16]
-#define UC32_REG_15             uregs[15]
-#define UC32_REG_14             uregs[14]
-#define UC32_REG_13             uregs[13]
-#define UC32_REG_12             uregs[12]
-#define UC32_REG_11             uregs[11]
-#define UC32_REG_10             uregs[10]
-#define UC32_REG_09             uregs[9]
-#define UC32_REG_08             uregs[8]
-#define UC32_REG_07             uregs[7]
-#define UC32_REG_06             uregs[6]
-#define UC32_REG_05             uregs[5]
-#define UC32_REG_04             uregs[4]
-#define UC32_REG_03             uregs[3]
-#define UC32_REG_02             uregs[2]
-#define UC32_REG_01             uregs[1]
-#define UC32_REG_00             uregs[0]
-#define UC32_REG_asr            uregs[32]
-#define UC32_REG_ORIG_00        uregs[33]
-
-#define UC32_SYSCALL_BASE               0x900000
-#define UC32_SYSCALL_ARCH_BASE          0xf0000
-#define UC32_SYSCALL_NR_set_tls         (UC32_SYSCALL_ARCH_BASE + 5)
-
-#define UNAME_MACHINE "UniCore-II"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
-
-#endif /* __UC32_SYSCALL_H__ */
diff --git a/linux-user/unicore32/target_cpu.h b/linux-user/unicore32/target_cpu.h
index fb79087..d7d2e7b 100644
--- a/linux-user/unicore32/target_cpu.h
+++ b/linux-user/unicore32/target_cpu.h
@@ -8,8 +8,8 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#ifndef TARGET_CPU_H
-#define TARGET_CPU_H
+#ifndef UNICORE32_TARGET_CPU_H
+#define UNICORE32_TARGET_CPU_H
 
 static inline void cpu_clone_regs(CPUUniCore32State *env, target_ulong newsp)
 {
diff --git a/linux-user/unicore32/target_signal.h b/linux-user/unicore32/target_signal.h
index 8b255c4..c6496fb 100644
--- a/linux-user/unicore32/target_signal.h
+++ b/linux-user/unicore32/target_signal.h
@@ -5,8 +5,8 @@
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef UNICORE32_TARGET_SIGNAL_H
+#define UNICORE32_TARGET_SIGNAL_H
 
 /* this struct defines a stack used during syscall handling */
 typedef struct target_sigaltstack {
@@ -21,6 +21,10 @@
 #define TARGET_SS_ONSTACK               1
 #define TARGET_SS_DISABLE               2
 
-#define get_sp_from_cpustate(cpustate)  (cpustate->regs[29])
+static inline abi_ulong get_sp_from_cpustate(CPUUniCore32State *state)
+{
+    return state->regs[29];
+}
 
-#endif /* TARGET_SIGNAL_H */
+
+#endif /* UNICORE32_TARGET_SIGNAL_H */
diff --git a/linux-user/unicore32/target_structs.h b/linux-user/unicore32/target_structs.h
index 7893695..fbd4fa3 100644
--- a/linux-user/unicore32/target_structs.h
+++ b/linux-user/unicore32/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef UNICORE32_TARGET_STRUCTS_H
+#define UNICORE32_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
diff --git a/linux-user/unicore32/target_syscall.h b/linux-user/unicore32/target_syscall.h
new file mode 100644
index 0000000..346b207
--- /dev/null
+++ b/linux-user/unicore32/target_syscall.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2010-2011 GUAN Xue-tao
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef UNICORE32_TARGET_SYSCALL_H
+#define UNICORE32_TARGET_SYSCALL_H
+
+struct target_pt_regs {
+    abi_ulong uregs[34];
+};
+
+#define UC32_REG_pc             uregs[31]
+#define UC32_REG_lr             uregs[30]
+#define UC32_REG_sp             uregs[29]
+#define UC32_REG_ip             uregs[28]
+#define UC32_REG_fp             uregs[27]
+#define UC32_REG_26             uregs[26]
+#define UC32_REG_25             uregs[25]
+#define UC32_REG_24             uregs[24]
+#define UC32_REG_23             uregs[23]
+#define UC32_REG_22             uregs[22]
+#define UC32_REG_21             uregs[21]
+#define UC32_REG_20             uregs[20]
+#define UC32_REG_19             uregs[19]
+#define UC32_REG_18             uregs[18]
+#define UC32_REG_17             uregs[17]
+#define UC32_REG_16             uregs[16]
+#define UC32_REG_15             uregs[15]
+#define UC32_REG_14             uregs[14]
+#define UC32_REG_13             uregs[13]
+#define UC32_REG_12             uregs[12]
+#define UC32_REG_11             uregs[11]
+#define UC32_REG_10             uregs[10]
+#define UC32_REG_09             uregs[9]
+#define UC32_REG_08             uregs[8]
+#define UC32_REG_07             uregs[7]
+#define UC32_REG_06             uregs[6]
+#define UC32_REG_05             uregs[5]
+#define UC32_REG_04             uregs[4]
+#define UC32_REG_03             uregs[3]
+#define UC32_REG_02             uregs[2]
+#define UC32_REG_01             uregs[1]
+#define UC32_REG_00             uregs[0]
+#define UC32_REG_asr            uregs[32]
+#define UC32_REG_ORIG_00        uregs[33]
+
+#define UC32_SYSCALL_BASE               0x900000
+#define UC32_SYSCALL_ARCH_BASE          0xf0000
+#define UC32_SYSCALL_NR_set_tls         (UC32_SYSCALL_ARCH_BASE + 5)
+
+#define UNAME_MACHINE "UniCore-II"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* UNICORE32_TARGET_SYSCALL_H */
diff --git a/linux-user/vm86.c b/linux-user/vm86.c
index 45ef559..3829b9a 100644
--- a/linux-user/vm86.c
+++ b/linux-user/vm86.c
@@ -16,12 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 
@@ -45,29 +40,34 @@
     return (((uint8_t *)bitmap)[nr >> 3] >> (nr & 7)) & 1;
 }
 
-static inline void vm_putw(uint32_t segptr, unsigned int reg16, unsigned int val)
+static inline void vm_putw(CPUX86State *env, uint32_t segptr,
+                           unsigned int reg16, unsigned int val)
 {
-    stw(segptr + (reg16 & 0xffff), val);
+    cpu_stw_data(env, segptr + (reg16 & 0xffff), val);
 }
 
-static inline void vm_putl(uint32_t segptr, unsigned int reg16, unsigned int val)
+static inline void vm_putl(CPUX86State *env, uint32_t segptr,
+                           unsigned int reg16, unsigned int val)
 {
-    stl(segptr + (reg16 & 0xffff), val);
+    cpu_stl_data(env, segptr + (reg16 & 0xffff), val);
 }
 
-static inline unsigned int vm_getb(uint32_t segptr, unsigned int reg16)
+static inline unsigned int vm_getb(CPUX86State *env,
+                                   uint32_t segptr, unsigned int reg16)
 {
-    return ldub(segptr + (reg16 & 0xffff));
+    return cpu_ldub_data(env, segptr + (reg16 & 0xffff));
 }
 
-static inline unsigned int vm_getw(uint32_t segptr, unsigned int reg16)
+static inline unsigned int vm_getw(CPUX86State *env,
+                                   uint32_t segptr, unsigned int reg16)
 {
-    return lduw(segptr + (reg16 & 0xffff));
+    return cpu_lduw_data(env, segptr + (reg16 & 0xffff));
 }
 
-static inline unsigned int vm_getl(uint32_t segptr, unsigned int reg16)
+static inline unsigned int vm_getl(CPUX86State *env,
+                                   uint32_t segptr, unsigned int reg16)
 {
-    return ldl(segptr + (reg16 & 0xffff));
+    return cpu_ldl_data(env, segptr + (reg16 & 0xffff));
 }
 
 void save_v86_state(CPUX86State *env)
@@ -221,7 +221,7 @@
                                        &ts->vm86plus.int21_revectored))
         goto cannot_handle;
     int_addr = (intno << 2);
-    segoffs = ldl(int_addr);
+    segoffs = cpu_ldl_data(env, int_addr);
     if ((segoffs >> 16) == TARGET_BIOSSEG)
         goto cannot_handle;
     LOG_VM86("VM86: emulating int 0x%x. CS:IP=%04x:%04x\n",
@@ -229,9 +229,9 @@
     /* save old state */
     ssp = env->segs[R_SS].selector << 4;
     sp = env->regs[R_ESP] & 0xffff;
-    vm_putw(ssp, sp - 2, get_vflags(env));
-    vm_putw(ssp, sp - 4, env->segs[R_CS].selector);
-    vm_putw(ssp, sp - 6, env->eip);
+    vm_putw(env, ssp, sp - 2, get_vflags(env));
+    vm_putw(env, ssp, sp - 4, env->segs[R_CS].selector);
+    vm_putw(env, ssp, sp - 6, env->eip);
     ADD16(env->regs[R_ESP], -6);
     /* goto interrupt handler */
     env->eip = segoffs & 0xffff;
@@ -285,7 +285,7 @@
     data32 = 0;
     pref_done = 0;
     do {
-        opcode = vm_getb(csp, ip);
+        opcode = vm_getb(env, csp, ip);
         ADD16(ip, 1);
         switch (opcode) {
         case 0x66:      /* 32-bit data */     data32=1; break;
@@ -306,10 +306,10 @@
     switch(opcode) {
     case 0x9c: /* pushf */
         if (data32) {
-            vm_putl(ssp, sp - 4, get_vflags(env));
+            vm_putl(env, ssp, sp - 4, get_vflags(env));
             ADD16(env->regs[R_ESP], -4);
         } else {
-            vm_putw(ssp, sp - 2, get_vflags(env));
+            vm_putw(env, ssp, sp - 2, get_vflags(env));
             ADD16(env->regs[R_ESP], -2);
         }
         env->eip = ip;
@@ -317,10 +317,10 @@
 
     case 0x9d: /* popf */
         if (data32) {
-            newflags = vm_getl(ssp, sp);
+            newflags = vm_getl(env, ssp, sp);
             ADD16(env->regs[R_ESP], 4);
         } else {
-            newflags = vm_getw(ssp, sp);
+            newflags = vm_getw(env, ssp, sp);
             ADD16(env->regs[R_ESP], 2);
         }
         env->eip = ip;
@@ -335,7 +335,7 @@
         VM86_FAULT_RETURN;
 
     case 0xcd: /* int */
-        intno = vm_getb(csp, ip);
+        intno = vm_getb(env, csp, ip);
         ADD16(ip, 1);
         env->eip = ip;
         if (ts->vm86plus.vm86plus.flags & TARGET_vm86dbg_active) {
@@ -350,14 +350,14 @@
 
     case 0xcf: /* iret */
         if (data32) {
-            newip = vm_getl(ssp, sp) & 0xffff;
-            newcs = vm_getl(ssp, sp + 4) & 0xffff;
-            newflags = vm_getl(ssp, sp + 8);
+            newip = vm_getl(env, ssp, sp) & 0xffff;
+            newcs = vm_getl(env, ssp, sp + 4) & 0xffff;
+            newflags = vm_getl(env, ssp, sp + 8);
             ADD16(env->regs[R_ESP], 12);
         } else {
-            newip = vm_getw(ssp, sp);
-            newcs = vm_getw(ssp, sp + 2);
-            newflags = vm_getw(ssp, sp + 4);
+            newip = vm_getw(env, ssp, sp);
+            newcs = vm_getw(env, ssp, sp + 2);
+            newflags = vm_getw(env, ssp, sp + 4);
             ADD16(env->regs[R_ESP], 6);
         }
         env->eip = newip;
diff --git a/linux-user/x86_64/syscall.h b/linux-user/x86_64/syscall.h
deleted file mode 100644
index 88b3c3f..0000000
--- a/linux-user/x86_64/syscall.h
+++ /dev/null
@@ -1,102 +0,0 @@
-#define __USER_CS	(0x33)
-#define __USER_DS	(0x2B)
-
-struct target_pt_regs {
-	abi_ulong r15;
-	abi_ulong r14;
-	abi_ulong r13;
-	abi_ulong r12;
-	abi_ulong rbp;
-	abi_ulong rbx;
-/* arguments: non interrupts/non tracing syscalls only save up to here */
- 	abi_ulong r11;
-	abi_ulong r10;
-	abi_ulong r9;
-	abi_ulong r8;
-	abi_ulong rax;
-	abi_ulong rcx;
-	abi_ulong rdx;
-	abi_ulong rsi;
-	abi_ulong rdi;
-	abi_ulong orig_rax;
-/* end of arguments */
-/* cpu exception frame or undefined */
-	abi_ulong rip;
-	abi_ulong cs;
-	abi_ulong eflags;
-	abi_ulong rsp;
-	abi_ulong ss;
-/* top of stack page */
-};
-
-/* Maximum number of LDT entries supported. */
-#define TARGET_LDT_ENTRIES	8192
-/* The size of each LDT entry. */
-#define TARGET_LDT_ENTRY_SIZE	8
-
-#define TARGET_GDT_ENTRIES 16
-#define TARGET_GDT_ENTRY_TLS_ENTRIES 3
-#define TARGET_GDT_ENTRY_TLS_MIN 12
-#define TARGET_GDT_ENTRY_TLS_MAX 14
-
-#if 0 // Redefine this
-struct target_modify_ldt_ldt_s {
-	unsigned int  entry_number;
-        abi_ulong     base_addr;
-	unsigned int  limit;
-	unsigned int  seg_32bit:1;
-	unsigned int  contents:2;
-	unsigned int  read_exec_only:1;
-	unsigned int  limit_in_pages:1;
-	unsigned int  seg_not_present:1;
-	unsigned int  useable:1;
-	unsigned int  lm:1;
-};
-#else
-struct target_modify_ldt_ldt_s {
-	unsigned int  entry_number;
-        abi_ulong     base_addr;
-	unsigned int  limit;
-        unsigned int flags;
-};
-#endif
-
-struct target_ipc64_perm
-{
-	int		key;
-	uint32_t	uid;
-	uint32_t	gid;
-	uint32_t	cuid;
-	uint32_t	cgid;
-	unsigned short		mode;
-	unsigned short		__pad1;
-	unsigned short		seq;
-	unsigned short		__pad2;
-	abi_ulong		__unused1;
-	abi_ulong		__unused2;
-};
-
-struct target_msqid64_ds {
-	struct target_ipc64_perm msg_perm;
-	unsigned int msg_stime;	/* last msgsnd time */
-	unsigned int msg_rtime;	/* last msgrcv time */
-	unsigned int msg_ctime;	/* last change time */
-	abi_ulong  msg_cbytes;	/* current number of bytes on queue */
-	abi_ulong  msg_qnum;	/* number of messages in queue */
-	abi_ulong  msg_qbytes;	/* max number of bytes on queue */
-	unsigned int msg_lspid;	/* pid of last msgsnd */
-	unsigned int msg_lrpid;	/* last receive pid */
-	abi_ulong  __unused4;
-	abi_ulong  __unused5;
-};
-
-#define UNAME_MACHINE "x86_64"
-#define UNAME_MINIMUM_RELEASE "2.6.32"
-
-#define TARGET_ARCH_SET_GS 0x1001
-#define TARGET_ARCH_SET_FS 0x1002
-#define TARGET_ARCH_GET_FS 0x1003
-#define TARGET_ARCH_GET_GS 0x1004
-#define TARGET_MINSIGSTKSZ 2048
-#define TARGET_MLOCKALL_MCL_CURRENT 1
-#define TARGET_MLOCKALL_MCL_FUTURE  2
diff --git a/linux-user/x86_64/syscall_nr.h b/linux-user/x86_64/syscall_nr.h
index 7c59e3a..16397b3 100644
--- a/linux-user/x86_64/syscall_nr.h
+++ b/linux-user/x86_64/syscall_nr.h
@@ -281,7 +281,7 @@
 #define TARGET_NR_utimensat		280
 #define TARGET_NR_epoll_pwait	281
 #define TARGET_NR_signalfd		282
-#define TARGET_NR_timerfd		283
+#define TARGET_NR_timerfd_create	283
 #define TARGET_NR_eventfd		284
 #define TARGET_NR_fallocate		285
 #define TARGET_NR_timerfd_settime	286
@@ -312,3 +312,16 @@
 #define TARGET_NR_process_vm_writev     311
 #define TARGET_NR_kcmp                  312
 #define TARGET_NR_finit_module          313
+#define TARGET_NR_sched_setattr         314
+#define TARGET_NR_sched_getattr         315
+#define TARGET_NR_renameat2             316
+#define TARGET_NR_seccomp               317
+#define TARGET_NR_getrandom             318
+#define TARGET_NR_memfd_create          319
+#define TARGET_NR_kexec_file_load       320
+#define TARGET_NR_bpf                   321
+#define TARGET_NR_execveat              322
+#define TARGET_NR_userfaultfd           323
+#define TARGET_NR_membarrier            324
+#define TARGET_NR_mlock2                325
+#define TARGET_NR_copy_file_range       326
diff --git a/linux-user/x86_64/target_signal.h b/linux-user/x86_64/target_signal.h
index 9baf7fb..1e95f4a 100644
--- a/linux-user/x86_64/target_signal.h
+++ b/linux-user/x86_64/target_signal.h
@@ -1,5 +1,5 @@
-#ifndef TARGET_SIGNAL_H
-#define TARGET_SIGNAL_H
+#ifndef X86_64_TARGET_SIGNAL_H
+#define X86_64_TARGET_SIGNAL_H
 
 #include "cpu.h"
 
@@ -26,4 +26,4 @@
     return state->regs[R_ESP];
 }
 
-#endif /* TARGET_SIGNAL_H */
+#endif /* X86_64_TARGET_SIGNAL_H */
diff --git a/linux-user/x86_64/target_structs.h b/linux-user/x86_64/target_structs.h
index d934056..b6e82a8 100644
--- a/linux-user/x86_64/target_structs.h
+++ b/linux-user/x86_64/target_structs.h
@@ -16,8 +16,8 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef TARGET_STRUCTS_H
-#define TARGET_STRUCTS_H
+#ifndef X86_64_TARGET_STRUCTS_H
+#define X86_64_TARGET_STRUCTS_H
 
 struct target_ipc_perm {
     abi_int __key;                      /* Key.  */
@@ -55,4 +55,19 @@
     abi_ulong __unused5;
 };
 
+/* The x86 definition differs from the generic one in that the
+ * two padding fields exist whether the ABI is 32 bits or 64 bits.
+ */
+#define TARGET_SEMID64_DS
+struct target_semid64_ds {
+    struct target_ipc_perm sem_perm;
+    abi_ulong sem_otime;
+    abi_ulong __unused1;
+    abi_ulong sem_ctime;
+    abi_ulong __unused2;
+    abi_ulong sem_nsems;
+    abi_ulong __unused3;
+    abi_ulong __unused4;
+};
+
 #endif
diff --git a/linux-user/x86_64/target_syscall.h b/linux-user/x86_64/target_syscall.h
new file mode 100644
index 0000000..983fb23
--- /dev/null
+++ b/linux-user/x86_64/target_syscall.h
@@ -0,0 +1,107 @@
+#ifndef X86_64_TARGET_SYSCALL_H
+#define X86_64_TARGET_SYSCALL_H
+
+#define __USER_CS	(0x33)
+#define __USER_DS	(0x2B)
+
+struct target_pt_regs {
+	abi_ulong r15;
+	abi_ulong r14;
+	abi_ulong r13;
+	abi_ulong r12;
+	abi_ulong rbp;
+	abi_ulong rbx;
+/* arguments: non interrupts/non tracing syscalls only save up to here */
+ 	abi_ulong r11;
+	abi_ulong r10;
+	abi_ulong r9;
+	abi_ulong r8;
+	abi_ulong rax;
+	abi_ulong rcx;
+	abi_ulong rdx;
+	abi_ulong rsi;
+	abi_ulong rdi;
+	abi_ulong orig_rax;
+/* end of arguments */
+/* cpu exception frame or undefined */
+	abi_ulong rip;
+	abi_ulong cs;
+	abi_ulong eflags;
+	abi_ulong rsp;
+	abi_ulong ss;
+/* top of stack page */
+};
+
+/* Maximum number of LDT entries supported. */
+#define TARGET_LDT_ENTRIES	8192
+/* The size of each LDT entry. */
+#define TARGET_LDT_ENTRY_SIZE	8
+
+#define TARGET_GDT_ENTRIES 16
+#define TARGET_GDT_ENTRY_TLS_ENTRIES 3
+#define TARGET_GDT_ENTRY_TLS_MIN 12
+#define TARGET_GDT_ENTRY_TLS_MAX 14
+
+#if 0 // Redefine this
+struct target_modify_ldt_ldt_s {
+	unsigned int  entry_number;
+        abi_ulong     base_addr;
+	unsigned int  limit;
+	unsigned int  seg_32bit:1;
+	unsigned int  contents:2;
+	unsigned int  read_exec_only:1;
+	unsigned int  limit_in_pages:1;
+	unsigned int  seg_not_present:1;
+	unsigned int  useable:1;
+	unsigned int  lm:1;
+};
+#else
+struct target_modify_ldt_ldt_s {
+	unsigned int  entry_number;
+        abi_ulong     base_addr;
+	unsigned int  limit;
+        unsigned int flags;
+};
+#endif
+
+struct target_ipc64_perm
+{
+	int		key;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	cuid;
+	uint32_t	cgid;
+	unsigned short		mode;
+	unsigned short		__pad1;
+	unsigned short		seq;
+	unsigned short		__pad2;
+	abi_ulong		__unused1;
+	abi_ulong		__unused2;
+};
+
+struct target_msqid64_ds {
+	struct target_ipc64_perm msg_perm;
+	unsigned int msg_stime;	/* last msgsnd time */
+	unsigned int msg_rtime;	/* last msgrcv time */
+	unsigned int msg_ctime;	/* last change time */
+	abi_ulong  msg_cbytes;	/* current number of bytes on queue */
+	abi_ulong  msg_qnum;	/* number of messages in queue */
+	abi_ulong  msg_qbytes;	/* max number of bytes on queue */
+	unsigned int msg_lspid;	/* pid of last msgsnd */
+	unsigned int msg_lrpid;	/* last receive pid */
+	abi_ulong  __unused4;
+	abi_ulong  __unused5;
+};
+
+#define UNAME_MACHINE "x86_64"
+#define UNAME_MINIMUM_RELEASE "2.6.32"
+
+#define TARGET_ARCH_SET_GS 0x1001
+#define TARGET_ARCH_SET_FS 0x1002
+#define TARGET_ARCH_GET_FS 0x1003
+#define TARGET_ARCH_GET_GS 0x1004
+#define TARGET_MINSIGSTKSZ 2048
+#define TARGET_MLOCKALL_MCL_CURRENT 1
+#define TARGET_MLOCKALL_MCL_FUTURE  2
+
+#endif /* X86_64_TARGET_SYSCALL_H */
diff --git a/linux-user/x86_64/termbits.h b/linux-user/x86_64/termbits.h
index 1c3445c..387e742 100644
--- a/linux-user/x86_64/termbits.h
+++ b/linux-user/x86_64/termbits.h
@@ -209,12 +209,12 @@
 #define TARGET_TIOCSBRK	0x5427  /* BSD compatibility */
 #define TARGET_TIOCCBRK	0x5428  /* BSD compatibility */
 #define TARGET_TIOCGSID	0x5429  /* Return the session ID of FD */
-#define TARGET_TCGETS2		_IOR('T',0x2A, struct termios2)
-#define TARGET_TCSETS2		_IOW('T',0x2B, struct termios2)
-#define TARGET_TCSETSW2	_IOW('T',0x2C, struct termios2)
-#define TARGET_TCSETSF2	_IOW('T',0x2D, struct termios2)
-#define TARGET_TIOCGPTN	_IOR('T',0x30, unsigned int) /* Get Pty Number (of pty-mux device) */
-#define TARGET_TIOCSPTLCK	_IOW('T',0x31, int)  /* Lock/unlock Pty */
+#define TARGET_TCGETS2          TARGET_IOR('T',0x2A, struct termios2)
+#define TARGET_TCSETS2          TARGET_IOW('T',0x2B, struct termios2)
+#define TARGET_TCSETSW2         TARGET_IOW('T',0x2C, struct termios2)
+#define TARGET_TCSETSF2         TARGET_IOW('T',0x2D, struct termios2)
+#define TARGET_TIOCGPTN         TARGET_IOR('T',0x30, unsigned int) /* Get Pty Number (of pty-mux device) */
+#define TARGET_TIOCSPTLCK       TARGET_IOW('T',0x31, int)  /* Lock/unlock Pty */
 
 #define TARGET_FIONCLEX	0x5450  /* these numbers need to be adjusted. */
 #define TARGET_FIOCLEX		0x5451
diff --git a/main-loop.c b/main-loop.c
index bb3cdf4..5359dca 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -22,7 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qemu/timer.h"
 #include "qemu/sockets.h"	// struct in_addr needed for libslirp.h
 #include "sysemu/qtest.h"
@@ -30,9 +32,12 @@
 #include "qemu/main-loop.h"
 #include "block/aio.h"
 
-#if defined(CONFIG_ANDROID)
-#include "android-qemu2-glue/emulation/charpipe.h"
-#endif
+static MainLoopPollCallback main_loop_poll_callback;
+
+void main_loop_register_poll_callback(MainLoopPollCallback poll_func)
+{
+    main_loop_poll_callback = poll_func;
+}
 
 #ifndef _WIN32
 
@@ -104,8 +109,7 @@
 
     fcntl_setfl(sigfd, O_NONBLOCK);
 
-    qemu_set_fd_handler2(sigfd, NULL, sigfd_handler, NULL,
-                         (void *)(intptr_t)sigfd);
+    qemu_set_fd_handler(sigfd, sigfd_handler, NULL, (void *)(intptr_t)sigfd);
 
     return 0;
 }
@@ -119,6 +123,14 @@
 #endif
 
 static AioContext *qemu_aio_context;
+static QEMUBH *qemu_notify_bh;
+
+static void notify_event_cb(void *opaque)
+{
+    /* No need to do anything; this bottom half is only used to
+     * kick the kernel out of ppoll/poll/WaitForMultipleObjects.
+     */
+}
 
 AioContext *qemu_get_aio_context(void)
 {
@@ -130,7 +142,7 @@
     if (!qemu_aio_context) {
         return;
     }
-    aio_notify(qemu_aio_context);
+    qemu_bh_schedule(qemu_notify_bh);
 }
 
 static GArray *gpollfds;
@@ -153,10 +165,14 @@
         error_propagate(errp, local_error);
         return -EMFILE;
     }
+    qemu_notify_bh = qemu_bh_new(notify_event_cb, NULL);
     gpollfds = g_array_new(FALSE, FALSE, sizeof(GPollFD));
     src = aio_get_g_source(qemu_aio_context);
     g_source_attach(src, NULL);
     g_source_unref(src);
+    src = iohandler_get_g_source();
+    g_source_attach(src, NULL);
+    g_source_unref(src);
     return 0;
 }
 
@@ -223,7 +239,7 @@
     if (!timeout && (spin_counter > MAX_MAIN_LOOP_SPIN)) {
         static bool notified;
 
-        if (!notified && !qtest_enabled()) {
+        if (!notified && !qtest_driver()) {
             fprintf(stderr,
                     "main-loop: WARNING: I/O thread spun for %d iterations\n",
                     MAX_MAIN_LOOP_SPIN);
@@ -483,7 +499,6 @@
 #ifdef CONFIG_SLIRP
     slirp_pollfds_fill(gpollfds, &timeout);
 #endif
-    qemu_iohandler_fill(gpollfds);
 
     if (timeout == UINT32_MAX) {
         timeout_ns = -1;
@@ -496,15 +511,16 @@
                                           &main_loop_tlg));
 
     ret = os_host_main_loop_wait(timeout_ns);
-    qemu_iohandler_poll(gpollfds, ret);
 #ifdef CONFIG_SLIRP
     slirp_pollfds_poll(gpollfds, (ret < 0));
 #endif
 
-#if defined(CONFIG_ANDROID)
-    charpipe_poll();
-#endif
+    if (main_loop_poll_callback)
+        (*main_loop_poll_callback)();
 
+    /* CPU thread can infinitely wait for event after
+       missing the warp */
+    qemu_start_warp_timer();
     qemu_clock_run_all_timers();
 
     return ret;
diff --git a/memory.c b/memory.c
index 15cf9eb..0eb6895 100644
--- a/memory.c
+++ b/memory.c
@@ -13,17 +13,22 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
 #include "exec/ioport.h"
 #include "qapi/visitor.h"
 #include "qemu/bitops.h"
+#include "qemu/error-report.h"
 #include "qom/object.h"
 #include "trace.h"
-#include <assert.h>
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
+#include "sysemu/kvm.h"
 #include "sysemu/sysemu.h"
 
 //#define DEBUG_UNASSIGNED
@@ -33,26 +38,12 @@
 static bool ioeventfd_update_pending;
 static bool global_dirty_log = false;
 
-/* flat_view_mutex is taken around reading as->current_map; the critical
- * section is extremely short, so I'm using a single mutex for every AS.
- * We could also RCU for the read-side.
- *
- * The BQL is taken around transaction commits, hence both locks are taken
- * while writing to as->current_map (with the BQL taken outside).
- */
-static QemuMutex flat_view_mutex;
-
 static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
     = QTAILQ_HEAD_INITIALIZER(memory_listeners);
 
 static QTAILQ_HEAD(, AddressSpace) address_spaces
     = QTAILQ_HEAD_INITIALIZER(address_spaces);
 
-static void memory_init(void)
-{
-    qemu_mutex_init(&flat_view_mutex);
-}
-
 typedef struct AddrRange AddrRange;
 
 /*
@@ -166,7 +157,7 @@
     } while (0)
 
 /* No need to ref/unref .mr, the FlatRange keeps it alive.  */
-#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback)            \
+#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback, _args...)  \
     MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) {       \
         .mr = (fr)->mr,                                                 \
         .address_space = (as),                                          \
@@ -174,7 +165,7 @@
         .size = (fr)->addr.size,                                        \
         .offset_within_address_space = int128_get64((fr)->addr.start),  \
         .readonly = (fr)->readonly,                                     \
-              }))
+              }), ##_args)
 
 struct CoalescedMemoryRange {
     AddrRange addr;
@@ -242,6 +233,7 @@
  * order.
  */
 struct FlatView {
+    struct rcu_head rcu;
     unsigned ref;
     FlatRange *ranges;
     unsigned nr;
@@ -381,84 +373,203 @@
     }
 }
 
-static void memory_region_oldmmio_read_accessor(MemoryRegion *mr,
-                                                hwaddr addr,
-                                                uint64_t *value,
-                                                unsigned size,
-                                                unsigned shift,
-                                                uint64_t mask)
+static hwaddr memory_region_to_absolute_addr(MemoryRegion *mr, hwaddr offset)
+{
+    MemoryRegion *root;
+    hwaddr abs_addr = offset;
+
+    abs_addr += mr->addr;
+    for (root = mr; root->container; ) {
+        root = root->container;
+        abs_addr += root->addr;
+    }
+
+    return abs_addr;
+}
+
+static int get_cpu_index(void)
+{
+    if (current_cpu) {
+        return current_cpu->cpu_index;
+    }
+    return -1;
+}
+
+static MemTxResult memory_region_oldmmio_read_accessor(MemoryRegion *mr,
+                                                       hwaddr addr,
+                                                       uint64_t *value,
+                                                       unsigned size,
+                                                       unsigned shift,
+                                                       uint64_t mask,
+                                                       MemTxAttrs attrs)
 {
     uint64_t tmp;
 
     tmp = mr->ops->old_mmio.read[ctz32(size)](mr->opaque, addr);
-    trace_memory_region_ops_read(mr, addr, tmp, size);
+    if (mr->subpage) {
+        trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
     *value |= (tmp & mask) << shift;
+    return MEMTX_OK;
 }
 
-static void memory_region_read_accessor(MemoryRegion *mr,
-                                        hwaddr addr,
-                                        uint64_t *value,
-                                        unsigned size,
-                                        unsigned shift,
-                                        uint64_t mask)
+static MemTxResult  memory_region_read_accessor(MemoryRegion *mr,
+                                                hwaddr addr,
+                                                uint64_t *value,
+                                                unsigned size,
+                                                unsigned shift,
+                                                uint64_t mask,
+                                                MemTxAttrs attrs)
 {
     uint64_t tmp;
 
-    if (mr->flush_coalesced_mmio) {
-        qemu_flush_coalesced_mmio_buffer();
-    }
     tmp = mr->ops->read(mr->opaque, addr, size);
-    trace_memory_region_ops_read(mr, addr, tmp, size);
-    *value |= (tmp & mask) << shift;
-}
-
-static void memory_region_oldmmio_write_accessor(MemoryRegion *mr,
-                                                 hwaddr addr,
-                                                 uint64_t *value,
-                                                 unsigned size,
-                                                 unsigned shift,
-                                                 uint64_t mask)
-{
-    uint64_t tmp;
-
-    tmp = (*value >> shift) & mask;
-    trace_memory_region_ops_write(mr, addr, tmp, size);
-    mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp);
-}
-
-static void memory_region_write_accessor(MemoryRegion *mr,
-                                         hwaddr addr,
-                                         uint64_t *value,
-                                         unsigned size,
-                                         unsigned shift,
-                                         uint64_t mask)
-{
-    uint64_t tmp;
-
-    if (mr->flush_coalesced_mmio) {
-        qemu_flush_coalesced_mmio_buffer();
+    if (mr->subpage) {
+        trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
     }
-    tmp = (*value >> shift) & mask;
-    trace_memory_region_ops_write(mr, addr, tmp, size);
-    mr->ops->write(mr->opaque, addr, tmp, size);
+    *value |= (tmp & mask) << shift;
+    return MEMTX_OK;
 }
 
-static void access_with_adjusted_size(hwaddr addr,
+static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr,
+                                                          hwaddr addr,
+                                                          uint64_t *value,
+                                                          unsigned size,
+                                                          unsigned shift,
+                                                          uint64_t mask,
+                                                          MemTxAttrs attrs)
+{
+    uint64_t tmp = 0;
+    MemTxResult r;
+
+    r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs);
+    if (mr->subpage) {
+        trace_memory_region_subpage_read(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_read(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_READ_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_read(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
+    *value |= (tmp & mask) << shift;
+    return r;
+}
+
+static MemTxResult memory_region_oldmmio_write_accessor(MemoryRegion *mr,
+                                                        hwaddr addr,
+                                                        uint64_t *value,
+                                                        unsigned size,
+                                                        unsigned shift,
+                                                        uint64_t mask,
+                                                        MemTxAttrs attrs)
+{
+    uint64_t tmp;
+
+    tmp = (*value >> shift) & mask;
+    if (mr->subpage) {
+        trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
+    mr->ops->old_mmio.write[ctz32(size)](mr->opaque, addr, tmp);
+    return MEMTX_OK;
+}
+
+static MemTxResult memory_region_write_accessor(MemoryRegion *mr,
+                                                hwaddr addr,
+                                                uint64_t *value,
+                                                unsigned size,
+                                                unsigned shift,
+                                                uint64_t mask,
+                                                MemTxAttrs attrs)
+{
+    uint64_t tmp;
+
+    tmp = (*value >> shift) & mask;
+    if (mr->subpage) {
+        trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
+    mr->ops->write(mr->opaque, addr, tmp, size);
+    return MEMTX_OK;
+}
+
+static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr,
+                                                           hwaddr addr,
+                                                           uint64_t *value,
+                                                           unsigned size,
+                                                           unsigned shift,
+                                                           uint64_t mask,
+                                                           MemTxAttrs attrs)
+{
+    uint64_t tmp;
+
+    tmp = (*value >> shift) & mask;
+    if (mr->subpage) {
+        trace_memory_region_subpage_write(get_cpu_index(), mr, addr, tmp, size);
+    } else if (mr == &io_mem_notdirty) {
+        /* Accesses to code which has previously been translated into a TB show
+         * up in the MMIO path, as accesses to the io_mem_notdirty
+         * MemoryRegion. */
+        trace_memory_region_tb_write(get_cpu_index(), addr, tmp, size);
+    } else if (TRACE_MEMORY_REGION_OPS_WRITE_ENABLED) {
+        hwaddr abs_addr = memory_region_to_absolute_addr(mr, addr);
+        trace_memory_region_ops_write(get_cpu_index(), mr, abs_addr, tmp, size);
+    }
+    return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs);
+}
+
+static MemTxResult access_with_adjusted_size(hwaddr addr,
                                       uint64_t *value,
                                       unsigned size,
                                       unsigned access_size_min,
                                       unsigned access_size_max,
-                                      void (*access)(MemoryRegion *mr,
-                                                     hwaddr addr,
-                                                     uint64_t *value,
-                                                     unsigned size,
-                                                     unsigned shift,
-                                                     uint64_t mask),
-                                      MemoryRegion *mr)
+                                      MemTxResult (*access)(MemoryRegion *mr,
+                                                            hwaddr addr,
+                                                            uint64_t *value,
+                                                            unsigned size,
+                                                            unsigned shift,
+                                                            uint64_t mask,
+                                                            MemTxAttrs attrs),
+                                      MemoryRegion *mr,
+                                      MemTxAttrs attrs)
 {
     uint64_t access_mask;
     unsigned access_size;
     unsigned i;
+    MemTxResult r = MEMTX_OK;
 
     if (!access_size_min) {
         access_size_min = 1;
@@ -472,14 +583,16 @@
     access_mask = -1ULL >> (64 - access_size * 8);
     if (memory_region_big_endian(mr)) {
         for (i = 0; i < size; i += access_size) {
-            access(mr, addr + i, value, access_size,
-                   (size - access_size - i) * 8, access_mask);
+            r |= access(mr, addr + i, value, access_size,
+                        (size - access_size - i) * 8, access_mask, attrs);
         }
     } else {
         for (i = 0; i < size; i += access_size) {
-            access(mr, addr + i, value, access_size, i * 8, access_mask);
+            r |= access(mr, addr + i, value, access_size, i * 8,
+                        access_mask, attrs);
         }
     }
+    return r;
 }
 
 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
@@ -550,7 +663,7 @@
     remain = clip.size;
 
     fr.mr = mr;
-    fr.dirty_log_mask = mr->dirty_log_mask;
+    fr.dirty_log_mask = memory_region_get_dirty_log_mask(mr);
     fr.romd_mode = mr->romd_mode;
     fr.readonly = readonly;
 
@@ -654,10 +767,10 @@
 {
     FlatView *view;
 
-    qemu_mutex_lock(&flat_view_mutex);
-    view = as->current_map;
+    rcu_read_lock();
+    view = atomic_rcu_read(&as->current_map);
     flatview_ref(view);
-    qemu_mutex_unlock(&flat_view_mutex);
+    rcu_read_unlock();
     return view;
 }
 
@@ -736,10 +849,15 @@
 
             if (adding) {
                 MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
-                if (frold->dirty_log_mask && !frnew->dirty_log_mask) {
-                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop);
-                } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) {
-                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start);
+                if (frnew->dirty_log_mask & ~frold->dirty_log_mask) {
+                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start,
+                                                  frold->dirty_log_mask,
+                                                  frnew->dirty_log_mask);
+                }
+                if (frold->dirty_log_mask & ~frnew->dirty_log_mask) {
+                    MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop,
+                                                  frold->dirty_log_mask,
+                                                  frnew->dirty_log_mask);
                 }
             }
 
@@ -766,10 +884,9 @@
     address_space_update_topology_pass(as, old_view, new_view, false);
     address_space_update_topology_pass(as, old_view, new_view, true);
 
-    qemu_mutex_lock(&flat_view_mutex);
-    flatview_unref(as->current_map);
-    as->current_map = new_view;
-    qemu_mutex_unlock(&flat_view_mutex);
+    /* Writes are protected by the BQL.  */
+    atomic_rcu_set(&as->current_map, new_view);
+    call_rcu(old_view, flatview_unref, rcu);
 
     /* Note that all the old MemoryRegions are still alive up to this
      * point.  This relieves most MemoryListeners from the need to
@@ -824,22 +941,12 @@
 
 static void memory_region_destructor_ram(MemoryRegion *mr)
 {
-    qemu_ram_free(mr->ram_addr);
-}
-
-static void memory_region_destructor_alias(MemoryRegion *mr)
-{
-    memory_region_unref(mr->alias);
-}
-
-static void memory_region_destructor_ram_from_ptr(MemoryRegion *mr)
-{
-    qemu_ram_free_from_ptr(mr->ram_addr);
+    qemu_ram_free(mr->ram_block);
 }
 
 static void memory_region_destructor_rom_device(MemoryRegion *mr)
 {
-    qemu_ram_free(mr->ram_addr & TARGET_PAGE_MASK);
+    qemu_ram_free(mr->ram_block);
 }
 
 static bool memory_region_need_escape(char c)
@@ -881,20 +988,23 @@
                         const char *name,
                         uint64_t size)
 {
-    if (!owner) {
-        owner = qdev_get_machine();
-    }
-
     object_initialize(mr, sizeof(*mr), TYPE_MEMORY_REGION);
     mr->size = int128_make64(size);
     if (size == UINT64_MAX) {
         mr->size = int128_2_64();
     }
     mr->name = g_strdup(name);
+    mr->owner = owner;
+    mr->ram_block = NULL;
 
     if (name) {
         char *escaped_name = memory_region_escape_name(name);
         char *name_array = g_strdup_printf("%s[*]", escaped_name);
+
+        if (!owner) {
+            owner = container_get(qdev_get_machine(), "/unattached");
+        }
+
         object_property_add_child(owner, name_array, OBJECT(mr), &error_abort);
         object_unref(OBJECT(mr));
         g_free(name_array);
@@ -902,17 +1012,18 @@
     }
 }
 
-static void memory_region_get_addr(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void memory_region_get_addr(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = mr->addr;
 
-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);
 }
 
-static void memory_region_get_container(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void memory_region_get_container(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     gchar *path = (gchar *)"";
@@ -920,7 +1031,7 @@
     if (mr->container) {
         path = object_get_canonical_path(OBJECT(mr->container));
     }
-    visit_type_str(v, &path, name, errp);
+    visit_type_str(v, name, &path, errp);
     if (mr->container) {
         g_free(path);
     }
@@ -934,29 +1045,23 @@
     return OBJECT(mr->container);
 }
 
-static void memory_region_get_priority(Object *obj, Visitor *v, void *opaque,
-                                       const char *name, Error **errp)
+static void memory_region_get_priority(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     int32_t value = mr->priority;
 
-    visit_type_int32(v, &value, name, errp);
+    visit_type_int32(v, name, &value, errp);
 }
 
-static bool memory_region_get_may_overlap(Object *obj, Error **errp)
-{
-    MemoryRegion *mr = MEMORY_REGION(obj);
-
-    return mr->may_overlap;
-}
-
-static void memory_region_get_size(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void memory_region_get_size(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
     uint64_t value = memory_region_size(mr);
 
-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);
 }
 
 static void memory_region_initfn(Object *obj)
@@ -967,6 +1072,7 @@
     mr->ops = &unassigned_mem_ops;
     mr->enabled = true;
     mr->romd_mode = true;
+    mr->global_locking = true;
     mr->destructor = memory_region_destructor_none;
     QTAILQ_INIT(&mr->subregions);
     QTAILQ_INIT(&mr->coalesced);
@@ -986,10 +1092,6 @@
                         memory_region_get_priority,
                         NULL, /* memory_region_set_priority */
                         NULL, NULL, &error_abort);
-    object_property_add_bool(OBJECT(mr), "may-overlap",
-                             memory_region_get_may_overlap,
-                             NULL, /* memory_region_set_may_overlap */
-                             &error_abort);
     object_property_add(OBJECT(mr), "size", "uint64",
                         memory_region_get_size,
                         NULL, /* memory_region_set_size, */
@@ -1067,62 +1169,113 @@
     return true;
 }
 
-static uint64_t memory_region_dispatch_read1(MemoryRegion *mr,
-                                             hwaddr addr,
-                                             unsigned size)
+static MemTxResult memory_region_dispatch_read1(MemoryRegion *mr,
+                                                hwaddr addr,
+                                                uint64_t *pval,
+                                                unsigned size,
+                                                MemTxAttrs attrs)
 {
-    uint64_t data = 0;
+    *pval = 0;
 
     if (mr->ops->read) {
-        access_with_adjusted_size(addr, &data, size,
-                                  mr->ops->impl.min_access_size,
-                                  mr->ops->impl.max_access_size,
-                                  memory_region_read_accessor, mr);
+        return access_with_adjusted_size(addr, pval, size,
+                                         mr->ops->impl.min_access_size,
+                                         mr->ops->impl.max_access_size,
+                                         memory_region_read_accessor,
+                                         mr, attrs);
+    } else if (mr->ops->read_with_attrs) {
+        return access_with_adjusted_size(addr, pval, size,
+                                         mr->ops->impl.min_access_size,
+                                         mr->ops->impl.max_access_size,
+                                         memory_region_read_with_attrs_accessor,
+                                         mr, attrs);
     } else {
-        access_with_adjusted_size(addr, &data, size, 1, 4,
-                                  memory_region_oldmmio_read_accessor, mr);
+        return access_with_adjusted_size(addr, pval, size, 1, 4,
+                                         memory_region_oldmmio_read_accessor,
+                                         mr, attrs);
     }
-
-    return data;
 }
 
-static bool memory_region_dispatch_read(MemoryRegion *mr,
+MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
                                         hwaddr addr,
                                         uint64_t *pval,
-                                        unsigned size)
+                                        unsigned size,
+                                        MemTxAttrs attrs)
 {
+    MemTxResult r;
+
     if (!memory_region_access_valid(mr, addr, size, false)) {
         *pval = unassigned_mem_read(mr, addr, size);
-        return true;
+        return MEMTX_DECODE_ERROR;
     }
 
-    *pval = memory_region_dispatch_read1(mr, addr, size);
+    r = memory_region_dispatch_read1(mr, addr, pval, size, attrs);
     adjust_endianness(mr, pval, size);
+    return r;
+}
+
+/* Return true if an eventfd was signalled */
+static bool memory_region_dispatch_write_eventfds(MemoryRegion *mr,
+                                                    hwaddr addr,
+                                                    uint64_t data,
+                                                    unsigned size,
+                                                    MemTxAttrs attrs)
+{
+    MemoryRegionIoeventfd ioeventfd = {
+        .addr = addrrange_make(int128_make64(addr), int128_make64(size)),
+        .data = data,
+    };
+    unsigned i;
+
+    for (i = 0; i < mr->ioeventfd_nb; i++) {
+        ioeventfd.match_data = mr->ioeventfds[i].match_data;
+        ioeventfd.e = mr->ioeventfds[i].e;
+
+        if (memory_region_ioeventfd_equal(ioeventfd, mr->ioeventfds[i])) {
+            event_notifier_set(ioeventfd.e);
+            return true;
+        }
+    }
+
     return false;
 }
 
-static bool memory_region_dispatch_write(MemoryRegion *mr,
+MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
                                          hwaddr addr,
                                          uint64_t data,
-                                         unsigned size)
+                                         unsigned size,
+                                         MemTxAttrs attrs)
 {
     if (!memory_region_access_valid(mr, addr, size, true)) {
         unassigned_mem_write(mr, addr, data, size);
-        return true;
+        return MEMTX_DECODE_ERROR;
     }
 
     adjust_endianness(mr, &data, size);
 
-    if (mr->ops->write) {
-        access_with_adjusted_size(addr, &data, size,
-                                  mr->ops->impl.min_access_size,
-                                  mr->ops->impl.max_access_size,
-                                  memory_region_write_accessor, mr);
-    } else {
-        access_with_adjusted_size(addr, &data, size, 1, 4,
-                                  memory_region_oldmmio_write_accessor, mr);
+    if ((!kvm_eventfds_enabled()) &&
+        memory_region_dispatch_write_eventfds(mr, addr, data, size, attrs)) {
+        return MEMTX_OK;
     }
-    return false;
+
+    if (mr->ops->write) {
+        return access_with_adjusted_size(addr, &data, size,
+                                         mr->ops->impl.min_access_size,
+                                         mr->ops->impl.max_access_size,
+                                         memory_region_write_accessor, mr,
+                                         attrs);
+    } else if (mr->ops->write_with_attrs) {
+        return
+            access_with_adjusted_size(addr, &data, size,
+                                      mr->ops->impl.min_access_size,
+                                      mr->ops->impl.max_access_size,
+                                      memory_region_write_with_attrs_accessor,
+                                      mr, attrs);
+    } else {
+        return access_with_adjusted_size(addr, &data, size, 1, 4,
+                                         memory_region_oldmmio_write_accessor,
+                                         mr, attrs);
+    }
 }
 
 void memory_region_init_io(MemoryRegion *mr,
@@ -1133,10 +1286,9 @@
                            uint64_t size)
 {
     memory_region_init(mr, owner, name, size);
-    mr->ops = ops;
+    mr->ops = ops ? ops : &unassigned_mem_ops;
     mr->opaque = opaque;
     mr->terminates = true;
-    mr->ram_addr = ~(ram_addr_t)0;
 }
 
 void memory_region_init_ram(MemoryRegion *mr,
@@ -1149,7 +1301,27 @@
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
-    mr->ram_addr = qemu_ram_alloc(size, mr, errp);
+    mr->ram_block = qemu_ram_alloc(size, mr, errp);
+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
+}
+
+void memory_region_init_resizeable_ram(MemoryRegion *mr,
+                                       Object *owner,
+                                       const char *name,
+                                       uint64_t size,
+                                       uint64_t max_size,
+                                       void (*resized)(const char*,
+                                                       uint64_t length,
+                                                       void *host),
+                                       Error **errp)
+{
+    memory_region_init(mr, owner, name, size);
+    mr->ram = true;
+    mr->terminates = true;
+    mr->destructor = memory_region_destructor_ram;
+    mr->ram_block = qemu_ram_alloc_resizeable(size, max_size, resized,
+                                              mr, errp);
+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 }
 
 #ifdef __linux__
@@ -1165,7 +1337,8 @@
     mr->ram = true;
     mr->terminates = true;
     mr->destructor = memory_region_destructor_ram;
-    mr->ram_addr = qemu_ram_alloc_from_file(size, mr, share, path, errp);
+    mr->ram_block = qemu_ram_alloc_from_file(size, mr, share, path, errp);
+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 }
 #endif
 
@@ -1178,11 +1351,12 @@
     memory_region_init(mr, owner, name, size);
     mr->ram = true;
     mr->terminates = true;
-    mr->destructor = memory_region_destructor_ram_from_ptr;
+    mr->destructor = memory_region_destructor_ram;
+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
 
     /* qemu_ram_alloc_from_ptr cannot fail with ptr != NULL.  */
     assert(ptr != NULL);
-    mr->ram_addr = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_abort);
+    mr->ram_block = qemu_ram_alloc_from_ptr(size, ptr, mr, &error_fatal);
 }
 
 void memory_region_set_skip_dump(MemoryRegion *mr)
@@ -1198,12 +1372,25 @@
                               uint64_t size)
 {
     memory_region_init(mr, owner, name, size);
-    memory_region_ref(orig);
-    mr->destructor = memory_region_destructor_alias;
     mr->alias = orig;
     mr->alias_offset = offset;
 }
 
+void memory_region_init_rom(MemoryRegion *mr,
+                            struct Object *owner,
+                            const char *name,
+                            uint64_t size,
+                            Error **errp)
+{
+    memory_region_init(mr, owner, name, size);
+    mr->ram = true;
+    mr->readonly = true;
+    mr->terminates = true;
+    mr->destructor = memory_region_destructor_ram;
+    mr->ram_block = qemu_ram_alloc(size, mr, errp);
+    mr->dirty_log_mask = tcg_enabled() ? (1 << DIRTY_MEMORY_CODE) : 0;
+}
+
 void memory_region_init_rom_device(MemoryRegion *mr,
                                    Object *owner,
                                    const MemoryRegionOps *ops,
@@ -1212,13 +1399,14 @@
                                    uint64_t size,
                                    Error **errp)
 {
+    assert(ops);
     memory_region_init(mr, owner, name, size);
     mr->ops = ops;
     mr->opaque = opaque;
     mr->terminates = true;
     mr->rom_device = true;
     mr->destructor = memory_region_destructor_rom_device;
-    mr->ram_addr = qemu_ram_alloc(size, mr, errp);
+    mr->ram_block = qemu_ram_alloc(size, mr, errp);
 }
 
 void memory_region_init_iommu(MemoryRegion *mr,
@@ -1233,20 +1421,26 @@
     notifier_list_init(&mr->iommu_notify);
 }
 
-void memory_region_init_reservation(MemoryRegion *mr,
-                                    Object *owner,
-                                    const char *name,
-                                    uint64_t size)
-{
-    memory_region_init_io(mr, owner, &unassigned_mem_ops, mr, name, size);
-}
-
 static void memory_region_finalize(Object *obj)
 {
     MemoryRegion *mr = MEMORY_REGION(obj);
 
-    assert(QTAILQ_EMPTY(&mr->subregions));
-    assert(memory_region_transaction_depth == 0);
+    assert(!mr->container);
+
+    /* We know the region is not visible in any address space (it
+     * does not have a container and cannot be a root either because
+     * it has no references, so we can blindly clear mr->enabled.
+     * memory_region_set_enabled instead could trigger a transaction
+     * and cause an infinite loop.
+     */
+    mr->enabled = false;
+    memory_region_transaction_begin();
+    while (!QTAILQ_EMPTY(&mr->subregions)) {
+        MemoryRegion *subregion = QTAILQ_FIRST(&mr->subregions);
+        memory_region_del_subregion(mr, subregion);
+    }
+    memory_region_transaction_commit();
+
     mr->destructor(mr);
     memory_region_clear_coalescing(mr);
     g_free((char *)mr->name);
@@ -1268,24 +1462,18 @@
      * The memory region is a child of its owner.  As long as the
      * owner doesn't call unparent itself on the memory region,
      * ref-ing the owner will also keep the memory region alive.
-     * Memory regions without an owner are supposed to never go away,
-     * but we still ref/unref them for debugging purposes.
+     * Memory regions without an owner are supposed to never go away;
+     * we do not ref/unref them because it slows down DMA sensibly.
      */
-    Object *obj = OBJECT(mr);
-    if (obj && obj->parent) {
-        object_ref(obj->parent);
-    } else {
-        object_ref(obj);
+    if (mr && mr->owner) {
+        object_ref(mr->owner);
     }
 }
 
 void memory_region_unref(MemoryRegion *mr)
 {
-    Object *obj = OBJECT(mr);
-    if (obj && obj->parent) {
-        object_unref(obj->parent);
-    } else {
-        object_unref(obj);
+    if (mr && mr->owner) {
+        object_unref(mr->owner);
     }
 }
 
@@ -1306,39 +1494,71 @@
     return mr->name;
 }
 
-bool memory_region_is_ram(MemoryRegion *mr)
-{
-    return mr->ram;
-}
-
 bool memory_region_is_skip_dump(MemoryRegion *mr)
 {
     return mr->skip_dump;
 }
 
-bool memory_region_is_logging(MemoryRegion *mr)
+uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr)
 {
-    return mr->dirty_log_mask;
+    uint8_t mask = mr->dirty_log_mask;
+    if (global_dirty_log) {
+        mask |= (1 << DIRTY_MEMORY_MIGRATION);
+    }
+    return mask;
 }
 
-bool memory_region_is_rom(MemoryRegion *mr)
+bool memory_region_is_logging(MemoryRegion *mr, uint8_t client)
 {
-    return mr->ram && mr->readonly;
-}
-
-bool memory_region_is_iommu(MemoryRegion *mr)
-{
-    return mr->iommu_ops;
+    return memory_region_get_dirty_log_mask(mr) & (1 << client);
 }
 
 void memory_region_register_iommu_notifier(MemoryRegion *mr, Notifier *n)
 {
+    if (mr->iommu_ops->notify_started &&
+        QLIST_EMPTY(&mr->iommu_notify.notifiers)) {
+        mr->iommu_ops->notify_started(mr);
+    }
     notifier_list_add(&mr->iommu_notify, n);
 }
 
-void memory_region_unregister_iommu_notifier(Notifier *n)
+uint64_t memory_region_iommu_get_min_page_size(MemoryRegion *mr)
+{
+    assert(memory_region_is_iommu(mr));
+    if (mr->iommu_ops && mr->iommu_ops->get_min_page_size) {
+        return mr->iommu_ops->get_min_page_size(mr);
+    }
+    return TARGET_PAGE_SIZE;
+}
+
+void memory_region_iommu_replay(MemoryRegion *mr, Notifier *n, bool is_write)
+{
+    hwaddr addr, granularity;
+    IOMMUTLBEntry iotlb;
+
+    granularity = memory_region_iommu_get_min_page_size(mr);
+
+    for (addr = 0; addr < memory_region_size(mr); addr += granularity) {
+        iotlb = mr->iommu_ops->translate(mr, addr, is_write);
+        if (iotlb.perm != IOMMU_NONE) {
+            n->notify(n, &iotlb);
+        }
+
+        /* if (2^64 - MR size) < granularity, it's possible to get an
+         * infinite loop here.  This should catch such a wraparound */
+        if ((addr + granularity) < addr) {
+            break;
+        }
+    }
+}
+
+void memory_region_unregister_iommu_notifier(MemoryRegion *mr, Notifier *n)
 {
     notifier_remove(n);
+    if (mr->iommu_ops->notify_stopped &&
+        QLIST_EMPTY(&mr->iommu_notify.notifiers)) {
+        mr->iommu_ops->notify_stopped(mr);
+    }
 }
 
 void memory_region_notify_iommu(MemoryRegion *mr,
@@ -1351,6 +1571,14 @@
 void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 {
     uint8_t mask = 1 << client;
+    uint8_t old_logging;
+
+    assert(client == DIRTY_MEMORY_VGA);
+    old_logging = mr->vga_logging_count;
+    mr->vga_logging_count += log ? 1 : -1;
+    if (!!old_logging == !!mr->vga_logging_count) {
+        return;
+    }
 
     memory_region_transaction_begin();
     mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
@@ -1361,27 +1589,26 @@
 bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size, unsigned client)
 {
-    assert(mr->terminates);
-    return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client);
+    assert(mr->ram_block);
+    return cpu_physical_memory_get_dirty(memory_region_get_ram_addr(mr) + addr,
+                                         size, client);
 }
 
 void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
                              hwaddr size)
 {
-    assert(mr->terminates);
-    cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size);
+    assert(mr->ram_block);
+    cpu_physical_memory_set_dirty_range(memory_region_get_ram_addr(mr) + addr,
+                                        size,
+                                        memory_region_get_dirty_log_mask(mr));
 }
 
 bool memory_region_test_and_clear_dirty(MemoryRegion *mr, hwaddr addr,
                                         hwaddr size, unsigned client)
 {
-    bool ret;
-    assert(mr->terminates);
-    ret = cpu_physical_memory_get_dirty(mr->ram_addr + addr, size, client);
-    if (ret) {
-        cpu_physical_memory_reset_dirty(mr->ram_addr + addr, size, client);
-    }
-    return ret;
+    assert(mr->ram_block);
+    return cpu_physical_memory_test_and_clear_dirty(
+                memory_region_get_ram_addr(mr) + addr, size, client);
 }
 
 
@@ -1424,30 +1651,74 @@
 void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
                                hwaddr size, unsigned client)
 {
-    assert(mr->terminates);
-    cpu_physical_memory_reset_dirty(mr->ram_addr + addr, size, client);
+    assert(mr->ram_block);
+    cpu_physical_memory_test_and_clear_dirty(
+        memory_region_get_ram_addr(mr) + addr, size, client);
 }
 
 int memory_region_get_fd(MemoryRegion *mr)
 {
-    if (mr->alias) {
-        return memory_region_get_fd(mr->alias);
+    int fd;
+
+    rcu_read_lock();
+    while (mr->alias) {
+        mr = mr->alias;
     }
+    fd = mr->ram_block->fd;
+    rcu_read_unlock();
 
-    assert(mr->terminates);
+    return fd;
+}
 
-    return qemu_get_ram_fd(mr->ram_addr & TARGET_PAGE_MASK);
+void memory_region_set_fd(MemoryRegion *mr, int fd)
+{
+    rcu_read_lock();
+    while (mr->alias) {
+        mr = mr->alias;
+    }
+    mr->ram_block->fd = fd;
+    rcu_read_unlock();
 }
 
 void *memory_region_get_ram_ptr(MemoryRegion *mr)
 {
-    if (mr->alias) {
-        return memory_region_get_ram_ptr(mr->alias) + mr->alias_offset;
+    void *ptr;
+    uint64_t offset = 0;
+
+    rcu_read_lock();
+    while (mr->alias) {
+        offset += mr->alias_offset;
+        mr = mr->alias;
+    }
+    assert(mr->ram_block);
+    ptr = qemu_map_ram_ptr(mr->ram_block, offset);
+    rcu_read_unlock();
+
+    return ptr;
+}
+
+MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset)
+{
+    RAMBlock *block;
+
+    block = qemu_ram_block_from_host(ptr, false, offset);
+    if (!block) {
+        return NULL;
     }
 
-    assert(mr->terminates);
+    return block->mr;
+}
 
-    return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK);
+ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
+{
+    return mr->ram_block ? mr->ram_block->offset : RAM_ADDR_INVALID;
+}
+
+void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize, Error **errp)
+{
+    assert(mr->ram_block);
+
+    qemu_ram_resize(mr->ram_block, newsize, errp);
 }
 
 static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
@@ -1547,6 +1818,18 @@
     }
 }
 
+void memory_region_set_global_locking(MemoryRegion *mr)
+{
+    mr->global_locking = true;
+}
+
+void memory_region_clear_global_locking(MemoryRegion *mr)
+{
+    mr->global_locking = false;
+}
+
+static bool userspace_eventfd_warning;
+
 void memory_region_add_eventfd(MemoryRegion *mr,
                                hwaddr addr,
                                unsigned size,
@@ -1563,7 +1846,16 @@
     };
     unsigned i;
 
-    adjust_endianness(mr, &mrfd.data, size);
+    if (kvm_enabled() && (!(kvm_eventfds_enabled() ||
+                            userspace_eventfd_warning))) {
+        userspace_eventfd_warning = true;
+        error_report("Using eventfd without MMIO binding in KVM. "
+                     "Suboptimal performance expected");
+    }
+
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
@@ -1596,7 +1888,9 @@
     };
     unsigned i;
 
-    adjust_endianness(mr, &mrfd.data, size);
+    if (size) {
+        adjust_endianness(mr, &mrfd.data, size);
+    }
     memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
@@ -1615,7 +1909,6 @@
 
 static void memory_region_update_container_subregions(MemoryRegion *subregion)
 {
-    hwaddr offset = subregion->addr;
     MemoryRegion *mr = subregion->container;
     MemoryRegion *other;
 
@@ -1623,27 +1916,6 @@
 
     memory_region_ref(subregion);
     QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
-        if (subregion->may_overlap || other->may_overlap) {
-            continue;
-        }
-        if (int128_ge(int128_make64(offset),
-                      int128_add(int128_make64(other->addr), other->size))
-            || int128_le(int128_add(int128_make64(offset), subregion->size),
-                         int128_make64(other->addr))) {
-            continue;
-        }
-#if 0
-        printf("warning: subregion collision %llx/%llx (%s) "
-               "vs %llx/%llx (%s)\n",
-               (unsigned long long)offset,
-               (unsigned long long)int128_get64(subregion->size),
-               subregion->name,
-               (unsigned long long)other->addr,
-               (unsigned long long)int128_get64(other->size),
-               other->name);
-#endif
-    }
-    QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
         if (subregion->priority >= other->priority) {
             QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
             goto done;
@@ -1669,7 +1941,6 @@
                                  hwaddr offset,
                                  MemoryRegion *subregion)
 {
-    subregion->may_overlap = false;
     subregion->priority = 0;
     memory_region_add_subregion_common(mr, offset, subregion);
 }
@@ -1679,7 +1950,6 @@
                                          MemoryRegion *subregion,
                                          int priority)
 {
-    subregion->may_overlap = true;
     subregion->priority = priority;
     memory_region_add_subregion_common(mr, offset, subregion);
 }
@@ -1707,6 +1977,22 @@
     memory_region_transaction_commit();
 }
 
+void memory_region_set_size(MemoryRegion *mr, uint64_t size)
+{
+    Int128 s = int128_make64(size);
+
+    if (size == UINT64_MAX) {
+        s = int128_2_64();
+    }
+    if (int128_eq(s, mr->size)) {
+        return;
+    }
+    memory_region_transaction_begin();
+    mr->size = s;
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();
+}
+
 static void memory_region_readd_subregion(MemoryRegion *mr)
 {
     MemoryRegion *container = mr->container;
@@ -1744,11 +2030,6 @@
     memory_region_transaction_commit();
 }
 
-ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
-{
-    return mr->ram_addr;
-}
-
 uint64_t memory_region_get_alignment(const MemoryRegion *mr)
 {
     return mr->align;
@@ -1773,23 +2054,16 @@
                    sizeof(FlatRange), cmp_flatrange_addr);
 }
 
-bool memory_region_present(MemoryRegion *container, hwaddr addr)
-{
-    MemoryRegion *mr = memory_region_find(container, addr, 1).mr;
-    if (!mr || (mr == container)) {
-        return false;
-    }
-    memory_region_unref(mr);
-    return true;
-}
-
 bool memory_region_is_mapped(MemoryRegion *mr)
 {
     return mr->container ? true : false;
 }
 
-MemoryRegionSection memory_region_find(MemoryRegion *mr,
-                                       hwaddr addr, uint64_t size)
+/* Same as memory_region_find, but it does not add a reference to the
+ * returned region.  It must be called from an RCU critical section.
+ */
+static MemoryRegionSection memory_region_find_rcu(MemoryRegion *mr,
+                                                  hwaddr addr, uint64_t size)
 {
     MemoryRegionSection ret = { .mr = NULL };
     MemoryRegion *root;
@@ -1810,10 +2084,9 @@
     }
     range = addrrange_make(int128_make64(addr), int128_make64(size));
 
-    view = address_space_get_flatview(as);
+    view = atomic_rcu_read(&as->current_map);
     fr = flatview_lookup(view, range);
     if (!fr) {
-        flatview_unref(view);
         return ret;
     }
 
@@ -1830,12 +2103,32 @@
     ret.size = range.size;
     ret.offset_within_address_space = int128_get64(range.start);
     ret.readonly = fr->readonly;
-    memory_region_ref(ret.mr);
-
-    flatview_unref(view);
     return ret;
 }
 
+MemoryRegionSection memory_region_find(MemoryRegion *mr,
+                                       hwaddr addr, uint64_t size)
+{
+    MemoryRegionSection ret;
+    rcu_read_lock();
+    ret = memory_region_find_rcu(mr, addr, size);
+    if (ret.mr) {
+        memory_region_ref(ret.mr);
+    }
+    rcu_read_unlock();
+    return ret;
+}
+
+bool memory_region_present(MemoryRegion *container, hwaddr addr)
+{
+    MemoryRegion *mr;
+
+    rcu_read_lock();
+    mr = memory_region_find_rcu(container, addr, 1).mr;
+    rcu_read_unlock();
+    return mr && mr != container;
+}
+
 void address_space_sync_dirty_bitmap(AddressSpace *as)
 {
     FlatView *view;
@@ -1851,12 +2144,24 @@
 void memory_global_dirty_log_start(void)
 {
     global_dirty_log = true;
+
     MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
+
+    /* Refresh DIRTY_LOG_MIGRATION bit.  */
+    memory_region_transaction_begin();
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();
 }
 
 void memory_global_dirty_log_stop(void)
 {
     global_dirty_log = false;
+
+    /* Refresh DIRTY_LOG_MIGRATION bit.  */
+    memory_region_transaction_begin();
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();
+
     MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
 }
 
@@ -1871,6 +2176,9 @@
         return;
     }
 
+    if (listener->begin) {
+        listener->begin(listener);
+    }
     if (global_dirty_log) {
         if (listener->log_global_start) {
             listener->log_global_start(listener);
@@ -1887,10 +2195,16 @@
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };
+        if (fr->dirty_log_mask && listener->log_start) {
+            listener->log_start(listener, &section, 0, fr->dirty_log_mask);
+        }
         if (listener->region_add) {
             listener->region_add(listener, &section);
         }
     }
+    if (listener->commit) {
+        listener->commit(listener);
+    }
     flatview_unref(view);
 }
 
@@ -1925,12 +2239,11 @@
 
 void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name)
 {
-    if (QTAILQ_EMPTY(&address_spaces)) {
-        memory_init();
-    }
-
+    memory_region_ref(root);
     memory_region_transaction_begin();
+    as->ref_count = 1;
     as->root = root;
+    as->malloced = false;
     as->current_map = g_new(FlatView, 1);
     flatview_init(as->current_map);
     as->ioeventfd_nb = 0;
@@ -1942,15 +2255,11 @@
     memory_region_transaction_commit();
 }
 
-void address_space_destroy(AddressSpace *as)
+static void do_address_space_destroy(AddressSpace *as)
 {
     MemoryListener *listener;
+    bool do_free = as->malloced;
 
-    /* Flush out anything from MemoryListeners listening in on this */
-    memory_region_transaction_begin();
-    as->root = NULL;
-    memory_region_transaction_commit();
-    QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
     address_space_destroy_dispatch(as);
 
     QTAILQ_FOREACH(listener, &memory_listeners, link) {
@@ -1960,17 +2269,50 @@
     flatview_unref(as->current_map);
     g_free(as->name);
     g_free(as->ioeventfds);
+    memory_region_unref(as->root);
+    if (do_free) {
+        g_free(as);
+    }
 }
 
-bool io_mem_read(MemoryRegion *mr, hwaddr addr, uint64_t *pval, unsigned size)
+AddressSpace *address_space_init_shareable(MemoryRegion *root, const char *name)
 {
-    return memory_region_dispatch_read(mr, addr, pval, size);
+    AddressSpace *as;
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (root == as->root && as->malloced) {
+            as->ref_count++;
+            return as;
+        }
+    }
+
+    as = g_malloc0(sizeof *as);
+    address_space_init(as, root, name);
+    as->malloced = true;
+    return as;
 }
 
-bool io_mem_write(MemoryRegion *mr, hwaddr addr,
-                  uint64_t val, unsigned size)
+void address_space_destroy(AddressSpace *as)
 {
-    return memory_region_dispatch_write(mr, addr, val, size);
+    MemoryRegion *root = as->root;
+
+    as->ref_count--;
+    if (as->ref_count) {
+        return;
+    }
+    /* Flush out anything from MemoryListeners listening in on this */
+    memory_region_transaction_begin();
+    as->root = NULL;
+    memory_region_transaction_commit();
+    QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
+    address_space_unregister(as);
+
+    /* At this point, as->dispatch and as->current_map are dummy
+     * entries that the guest should never use.  Wait for the old
+     * values to expire before freeing the data.
+     */
+    as->root = root;
+    call_rcu(as, do_address_space_destroy, rcu);
 }
 
 typedef struct MemoryRegionList MemoryRegionList;
@@ -1992,7 +2334,7 @@
     const MemoryRegion *submr;
     unsigned int i;
 
-    if (!mr || !mr->enabled) {
+    if (!mr) {
         return;
     }
 
@@ -2018,7 +2360,7 @@
         }
         mon_printf(f, TARGET_FMT_plx "-" TARGET_FMT_plx
                    " (prio %d, %c%c): alias %s @%s " TARGET_FMT_plx
-                   "-" TARGET_FMT_plx "\n",
+                   "-" TARGET_FMT_plx "%s\n",
                    base + mr->addr,
                    base + mr->addr
                    + (int128_nz(mr->size) ?
@@ -2034,10 +2376,11 @@
                    mr->alias_offset
                    + (int128_nz(mr->size) ?
                       (hwaddr)int128_get64(int128_sub(mr->size,
-                                                      int128_one())) : 0));
+                                                      int128_one())) : 0),
+                   mr->enabled ? "" : " [disabled]");
     } else {
         mon_printf(f,
-                   TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n",
+                   TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s%s\n",
                    base + mr->addr,
                    base + mr->addr
                    + (int128_nz(mr->size) ?
@@ -2047,7 +2390,8 @@
                    mr->romd_mode ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->romd_mode) ? 'W'
                                                                        : '-',
-                   memory_region_name(mr));
+                   memory_region_name(mr),
+                   mr->enabled ? "" : " [disabled]");
     }
 
     QTAILQ_INIT(&submr_print_queue);
@@ -2088,15 +2432,16 @@
     QTAILQ_INIT(&ml_head);
 
     QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
-        mon_printf(f, "%s\n", as->name);
-        mtree_print_mr(mon_printf, f, as->root, 0, 0, &ml_head);
+        mon_printf(f, "address-space: %s\n", as->name);
+        mtree_print_mr(mon_printf, f, as->root, 1, 0, &ml_head);
+        mon_printf(f, "\n");
     }
 
-    mon_printf(f, "aliases\n");
     /* print aliased regions */
     QTAILQ_FOREACH(ml, &ml_head, queue) {
-        mon_printf(f, "%s\n", memory_region_name(ml->mr));
-        mtree_print_mr(mon_printf, f, ml->mr, 0, 0, &ml_head);
+        mon_printf(f, "memory-region: %s\n", memory_region_name(ml->mr));
+        mtree_print_mr(mon_printf, f, ml->mr, 1, 0, &ml_head);
+        mon_printf(f, "\n");
     }
 
     QTAILQ_FOREACH_SAFE(ml, &ml_head, queue, ml2) {
diff --git a/memory_mapping.c b/memory_mapping.c
index 7b69801..e3e0d95 100644
--- a/memory_mapping.c
+++ b/memory_mapping.c
@@ -11,10 +11,11 @@
  *
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 
+#include "qemu-common.h"
 #include "cpu.h"
-#include "exec/cpu-all.h"
 #include "sysemu/memory_mapping.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
@@ -177,6 +178,7 @@
 
     QTAILQ_FOREACH_SAFE(p, &list->head, next, q) {
         QTAILQ_REMOVE(&list->head, p, next);
+        memory_region_unref(p->mr);
         g_free(p);
     }
     list->num = 0;
@@ -240,6 +242,8 @@
         block->target_start = target_start;
         block->target_end   = target_end;
         block->host_addr    = host_addr;
+        block->mr           = section->mr;
+        memory_region_ref(section->mr);
 
         QTAILQ_INSERT_TAIL(&g->list->head, block, next);
         ++g->list->num;
diff --git a/migration-exec.c b/migration-exec.c
deleted file mode 100644
index 4790247..0000000
--- a/migration-exec.c
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * QEMU live migration
- *
- * Copyright IBM, Corp. 2008
- * Copyright Dell MessageOne 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *  Charles Duffy     <charles_duffy@messageone.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "qemu/sockets.h"
-#include "qemu/main-loop.h"
-#include "migration/migration.h"
-#include "migration/qemu-file.h"
-#include "block/block.h"
-#include <sys/types.h>
-#include <sys/wait.h>
-
-//#define DEBUG_MIGRATION_EXEC
-
-#ifdef DEBUG_MIGRATION_EXEC
-#define DPRINTF(fmt, ...) \
-    do { printf("migration-exec: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp)
-{
-    s->file = qemu_popen_cmd(command, "w");
-    if (s->file == NULL) {
-        error_setg_errno(errp, errno, "failed to popen the migration target");
-        return;
-    }
-
-    migrate_fd_connect(s);
-}
-
-static void exec_accept_incoming_migration(void *opaque)
-{
-    QEMUFile *f = opaque;
-
-    qemu_set_fd_handler2(qemu_get_fd(f), NULL, NULL, NULL, NULL);
-    process_incoming_migration(f);
-}
-
-void exec_start_incoming_migration(const char *command, Error **errp)
-{
-    QEMUFile *f;
-
-    DPRINTF("Attempting to start an incoming migration\n");
-    f = qemu_popen_cmd(command, "r");
-    if(f == NULL) {
-        error_setg_errno(errp, errno, "failed to popen the migration source");
-        return;
-    }
-
-    qemu_set_fd_handler2(qemu_get_fd(f), NULL,
-			 exec_accept_incoming_migration, NULL, f);
-}
diff --git a/migration-fd.c b/migration-fd.c
deleted file mode 100644
index d2e523a..0000000
--- a/migration-fd.c
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * QEMU live migration via generic fd
- *
- * Copyright Red Hat, Inc. 2009
- *
- * Authors:
- *  Chris Lalancette <clalance@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "qemu/main-loop.h"
-#include "qemu/sockets.h"
-#include "migration/migration.h"
-#include "monitor/monitor.h"
-#include "migration/qemu-file.h"
-#include "block/block.h"
-
-//#define DEBUG_MIGRATION_FD
-
-#ifdef DEBUG_MIGRATION_FD
-#define DPRINTF(fmt, ...) \
-    do { printf("migration-fd: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp)
-{
-    int fd = monitor_get_fd(cur_mon, fdname, errp);
-    if (fd == -1) {
-        return;
-    }
-    s->file = qemu_fdopen(fd, "wb");
-
-    migrate_fd_connect(s);
-}
-
-static void fd_accept_incoming_migration(void *opaque)
-{
-    QEMUFile *f = opaque;
-
-    qemu_set_fd_handler2(qemu_get_fd(f), NULL, NULL, NULL, NULL);
-    process_incoming_migration(f);
-}
-
-void fd_start_incoming_migration(const char *infd, Error **errp)
-{
-    int fd;
-    QEMUFile *f;
-
-    DPRINTF("Attempting to start an incoming migration via fd\n");
-
-    fd = strtol(infd, NULL, 0);
-    f = qemu_fdopen(fd, "rb");
-    if(f == NULL) {
-        error_setg_errno(errp, errno, "failed to open the source descriptor");
-        return;
-    }
-
-    qemu_set_fd_handler2(fd, NULL, fd_accept_incoming_migration, NULL, f);
-}
diff --git a/migration-rdma.c b/migration-rdma.c
deleted file mode 100644
index b32dbdf..0000000
--- a/migration-rdma.c
+++ /dev/null
@@ -1,3438 +0,0 @@
-/*
- * RDMA protocol and interfaces
- *
- * Copyright IBM, Corp. 2010-2013
- *
- * Authors:
- *  Michael R. Hines <mrhines@us.ibm.com>
- *  Jiuxing Liu <jl@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or
- * later.  See the COPYING file in the top-level directory.
- *
- */
-#include "qemu-common.h"
-#include "migration/migration.h"
-#include "migration/qemu-file.h"
-#include "exec/cpu-common.h"
-#include "qemu/main-loop.h"
-#include "qemu/sockets.h"
-#include "qemu/bitmap.h"
-#include "block/coroutine.h"
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <string.h>
-#include <rdma/rdma_cma.h>
-
-//#define DEBUG_RDMA
-//#define DEBUG_RDMA_VERBOSE
-//#define DEBUG_RDMA_REALLY_VERBOSE
-
-#ifdef DEBUG_RDMA
-#define DPRINTF(fmt, ...) \
-    do { printf("rdma: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-#ifdef DEBUG_RDMA_VERBOSE
-#define DDPRINTF(fmt, ...) \
-    do { printf("rdma: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DDPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-#ifdef DEBUG_RDMA_REALLY_VERBOSE
-#define DDDPRINTF(fmt, ...) \
-    do { printf("rdma: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DDDPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-/*
- * Print and error on both the Monitor and the Log file.
- */
-#define ERROR(errp, fmt, ...) \
-    do { \
-        fprintf(stderr, "RDMA ERROR: " fmt "\n", ## __VA_ARGS__); \
-        if (errp && (*(errp) == NULL)) { \
-            error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \
-        } \
-    } while (0)
-
-#define RDMA_RESOLVE_TIMEOUT_MS 10000
-
-/* Do not merge data if larger than this. */
-#define RDMA_MERGE_MAX (2 * 1024 * 1024)
-#define RDMA_SIGNALED_SEND_MAX (RDMA_MERGE_MAX / 4096)
-
-#define RDMA_REG_CHUNK_SHIFT 20 /* 1 MB */
-
-/*
- * This is only for non-live state being migrated.
- * Instead of RDMA_WRITE messages, we use RDMA_SEND
- * messages for that state, which requires a different
- * delivery design than main memory.
- */
-#define RDMA_SEND_INCREMENT 32768
-
-/*
- * Maximum size infiniband SEND message
- */
-#define RDMA_CONTROL_MAX_BUFFER (512 * 1024)
-#define RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE 4096
-
-#define RDMA_CONTROL_VERSION_CURRENT 1
-/*
- * Capabilities for negotiation.
- */
-#define RDMA_CAPABILITY_PIN_ALL 0x01
-
-/*
- * Add the other flags above to this list of known capabilities
- * as they are introduced.
- */
-static uint32_t known_capabilities = RDMA_CAPABILITY_PIN_ALL;
-
-#define CHECK_ERROR_STATE() \
-    do { \
-        if (rdma->error_state) { \
-            if (!rdma->error_reported) { \
-                fprintf(stderr, "RDMA is in an error state waiting migration" \
-                                " to abort!\n"); \
-                rdma->error_reported = 1; \
-            } \
-            return rdma->error_state; \
-        } \
-    } while (0);
-
-/*
- * A work request ID is 64-bits and we split up these bits
- * into 3 parts:
- *
- * bits 0-15 : type of control message, 2^16
- * bits 16-29: ram block index, 2^14
- * bits 30-63: ram block chunk number, 2^34
- *
- * The last two bit ranges are only used for RDMA writes,
- * in order to track their completion and potentially
- * also track unregistration status of the message.
- */
-#define RDMA_WRID_TYPE_SHIFT  0UL
-#define RDMA_WRID_BLOCK_SHIFT 16UL
-#define RDMA_WRID_CHUNK_SHIFT 30UL
-
-#define RDMA_WRID_TYPE_MASK \
-    ((1UL << RDMA_WRID_BLOCK_SHIFT) - 1UL)
-
-#define RDMA_WRID_BLOCK_MASK \
-    (~RDMA_WRID_TYPE_MASK & ((1UL << RDMA_WRID_CHUNK_SHIFT) - 1UL))
-
-#define RDMA_WRID_CHUNK_MASK (~RDMA_WRID_BLOCK_MASK & ~RDMA_WRID_TYPE_MASK)
-
-/*
- * RDMA migration protocol:
- * 1. RDMA Writes (data messages, i.e. RAM)
- * 2. IB Send/Recv (control channel messages)
- */
-enum {
-    RDMA_WRID_NONE = 0,
-    RDMA_WRID_RDMA_WRITE = 1,
-    RDMA_WRID_SEND_CONTROL = 2000,
-    RDMA_WRID_RECV_CONTROL = 4000,
-};
-
-const char *wrid_desc[] = {
-    [RDMA_WRID_NONE] = "NONE",
-    [RDMA_WRID_RDMA_WRITE] = "WRITE RDMA",
-    [RDMA_WRID_SEND_CONTROL] = "CONTROL SEND",
-    [RDMA_WRID_RECV_CONTROL] = "CONTROL RECV",
-};
-
-/*
- * Work request IDs for IB SEND messages only (not RDMA writes).
- * This is used by the migration protocol to transmit
- * control messages (such as device state and registration commands)
- *
- * We could use more WRs, but we have enough for now.
- */
-enum {
-    RDMA_WRID_READY = 0,
-    RDMA_WRID_DATA,
-    RDMA_WRID_CONTROL,
-    RDMA_WRID_MAX,
-};
-
-/*
- * SEND/RECV IB Control Messages.
- */
-enum {
-    RDMA_CONTROL_NONE = 0,
-    RDMA_CONTROL_ERROR,
-    RDMA_CONTROL_READY,               /* ready to receive */
-    RDMA_CONTROL_QEMU_FILE,           /* QEMUFile-transmitted bytes */
-    RDMA_CONTROL_RAM_BLOCKS_REQUEST,  /* RAMBlock synchronization */
-    RDMA_CONTROL_RAM_BLOCKS_RESULT,   /* RAMBlock synchronization */
-    RDMA_CONTROL_COMPRESS,            /* page contains repeat values */
-    RDMA_CONTROL_REGISTER_REQUEST,    /* dynamic page registration */
-    RDMA_CONTROL_REGISTER_RESULT,     /* key to use after registration */
-    RDMA_CONTROL_REGISTER_FINISHED,   /* current iteration finished */
-    RDMA_CONTROL_UNREGISTER_REQUEST,  /* dynamic UN-registration */
-    RDMA_CONTROL_UNREGISTER_FINISHED, /* unpinning finished */
-};
-
-const char *control_desc[] = {
-    [RDMA_CONTROL_NONE] = "NONE",
-    [RDMA_CONTROL_ERROR] = "ERROR",
-    [RDMA_CONTROL_READY] = "READY",
-    [RDMA_CONTROL_QEMU_FILE] = "QEMU FILE",
-    [RDMA_CONTROL_RAM_BLOCKS_REQUEST] = "RAM BLOCKS REQUEST",
-    [RDMA_CONTROL_RAM_BLOCKS_RESULT] = "RAM BLOCKS RESULT",
-    [RDMA_CONTROL_COMPRESS] = "COMPRESS",
-    [RDMA_CONTROL_REGISTER_REQUEST] = "REGISTER REQUEST",
-    [RDMA_CONTROL_REGISTER_RESULT] = "REGISTER RESULT",
-    [RDMA_CONTROL_REGISTER_FINISHED] = "REGISTER FINISHED",
-    [RDMA_CONTROL_UNREGISTER_REQUEST] = "UNREGISTER REQUEST",
-    [RDMA_CONTROL_UNREGISTER_FINISHED] = "UNREGISTER FINISHED",
-};
-
-/*
- * Memory and MR structures used to represent an IB Send/Recv work request.
- * This is *not* used for RDMA writes, only IB Send/Recv.
- */
-typedef struct {
-    uint8_t  control[RDMA_CONTROL_MAX_BUFFER]; /* actual buffer to register */
-    struct   ibv_mr *control_mr;               /* registration metadata */
-    size_t   control_len;                      /* length of the message */
-    uint8_t *control_curr;                     /* start of unconsumed bytes */
-} RDMAWorkRequestData;
-
-/*
- * Negotiate RDMA capabilities during connection-setup time.
- */
-typedef struct {
-    uint32_t version;
-    uint32_t flags;
-} RDMACapabilities;
-
-static void caps_to_network(RDMACapabilities *cap)
-{
-    cap->version = htonl(cap->version);
-    cap->flags = htonl(cap->flags);
-}
-
-static void network_to_caps(RDMACapabilities *cap)
-{
-    cap->version = ntohl(cap->version);
-    cap->flags = ntohl(cap->flags);
-}
-
-/*
- * Representation of a RAMBlock from an RDMA perspective.
- * This is not transmitted, only local.
- * This and subsequent structures cannot be linked lists
- * because we're using a single IB message to transmit
- * the information. It's small anyway, so a list is overkill.
- */
-typedef struct RDMALocalBlock {
-    uint8_t  *local_host_addr; /* local virtual address */
-    uint64_t remote_host_addr; /* remote virtual address */
-    uint64_t offset;
-    uint64_t length;
-    struct   ibv_mr **pmr;     /* MRs for chunk-level registration */
-    struct   ibv_mr *mr;       /* MR for non-chunk-level registration */
-    uint32_t *remote_keys;     /* rkeys for chunk-level registration */
-    uint32_t remote_rkey;      /* rkeys for non-chunk-level registration */
-    int      index;            /* which block are we */
-    bool     is_ram_block;
-    int      nb_chunks;
-    unsigned long *transit_bitmap;
-    unsigned long *unregister_bitmap;
-} RDMALocalBlock;
-
-/*
- * Also represents a RAMblock, but only on the dest.
- * This gets transmitted by the dest during connection-time
- * to the source VM and then is used to populate the
- * corresponding RDMALocalBlock with
- * the information needed to perform the actual RDMA.
- */
-typedef struct QEMU_PACKED RDMARemoteBlock {
-    uint64_t remote_host_addr;
-    uint64_t offset;
-    uint64_t length;
-    uint32_t remote_rkey;
-    uint32_t padding;
-} RDMARemoteBlock;
-
-static uint64_t htonll(uint64_t v)
-{
-    union { uint32_t lv[2]; uint64_t llv; } u;
-    u.lv[0] = htonl(v >> 32);
-    u.lv[1] = htonl(v & 0xFFFFFFFFULL);
-    return u.llv;
-}
-
-static uint64_t ntohll(uint64_t v) {
-    union { uint32_t lv[2]; uint64_t llv; } u;
-    u.llv = v;
-    return ((uint64_t)ntohl(u.lv[0]) << 32) | (uint64_t) ntohl(u.lv[1]);
-}
-
-static void remote_block_to_network(RDMARemoteBlock *rb)
-{
-    rb->remote_host_addr = htonll(rb->remote_host_addr);
-    rb->offset = htonll(rb->offset);
-    rb->length = htonll(rb->length);
-    rb->remote_rkey = htonl(rb->remote_rkey);
-}
-
-static void network_to_remote_block(RDMARemoteBlock *rb)
-{
-    rb->remote_host_addr = ntohll(rb->remote_host_addr);
-    rb->offset = ntohll(rb->offset);
-    rb->length = ntohll(rb->length);
-    rb->remote_rkey = ntohl(rb->remote_rkey);
-}
-
-/*
- * Virtual address of the above structures used for transmitting
- * the RAMBlock descriptions at connection-time.
- * This structure is *not* transmitted.
- */
-typedef struct RDMALocalBlocks {
-    int nb_blocks;
-    bool     init;             /* main memory init complete */
-    RDMALocalBlock *block;
-} RDMALocalBlocks;
-
-/*
- * Main data structure for RDMA state.
- * While there is only one copy of this structure being allocated right now,
- * this is the place where one would start if you wanted to consider
- * having more than one RDMA connection open at the same time.
- */
-typedef struct RDMAContext {
-    char *host;
-    int port;
-
-    RDMAWorkRequestData wr_data[RDMA_WRID_MAX];
-
-    /*
-     * This is used by *_exchange_send() to figure out whether or not
-     * the initial "READY" message has already been received or not.
-     * This is because other functions may potentially poll() and detect
-     * the READY message before send() does, in which case we need to
-     * know if it completed.
-     */
-    int control_ready_expected;
-
-    /* number of outstanding writes */
-    int nb_sent;
-
-    /* store info about current buffer so that we can
-       merge it with future sends */
-    uint64_t current_addr;
-    uint64_t current_length;
-    /* index of ram block the current buffer belongs to */
-    int current_index;
-    /* index of the chunk in the current ram block */
-    int current_chunk;
-
-    bool pin_all;
-
-    /*
-     * infiniband-specific variables for opening the device
-     * and maintaining connection state and so forth.
-     *
-     * cm_id also has ibv_context, rdma_event_channel, and ibv_qp in
-     * cm_id->verbs, cm_id->channel, and cm_id->qp.
-     */
-    struct rdma_cm_id *cm_id;               /* connection manager ID */
-    struct rdma_cm_id *listen_id;
-    bool connected;
-
-    struct ibv_context          *verbs;
-    struct rdma_event_channel   *channel;
-    struct ibv_qp *qp;                      /* queue pair */
-    struct ibv_comp_channel *comp_channel;  /* completion channel */
-    struct ibv_pd *pd;                      /* protection domain */
-    struct ibv_cq *cq;                      /* completion queue */
-
-    /*
-     * If a previous write failed (perhaps because of a failed
-     * memory registration, then do not attempt any future work
-     * and remember the error state.
-     */
-    int error_state;
-    int error_reported;
-
-    /*
-     * Description of ram blocks used throughout the code.
-     */
-    RDMALocalBlocks local_ram_blocks;
-    RDMARemoteBlock *block;
-
-    /*
-     * Migration on *destination* started.
-     * Then use coroutine yield function.
-     * Source runs in a thread, so we don't care.
-     */
-    int migration_started_on_destination;
-
-    int total_registrations;
-    int total_writes;
-
-    int unregister_current, unregister_next;
-    uint64_t unregistrations[RDMA_SIGNALED_SEND_MAX];
-
-    GHashTable *blockmap;
-} RDMAContext;
-
-/*
- * Interface to the rest of the migration call stack.
- */
-typedef struct QEMUFileRDMA {
-    RDMAContext *rdma;
-    size_t len;
-    void *file;
-} QEMUFileRDMA;
-
-/*
- * Main structure for IB Send/Recv control messages.
- * This gets prepended at the beginning of every Send/Recv.
- */
-typedef struct QEMU_PACKED {
-    uint32_t len;     /* Total length of data portion */
-    uint32_t type;    /* which control command to perform */
-    uint32_t repeat;  /* number of commands in data portion of same type */
-    uint32_t padding;
-} RDMAControlHeader;
-
-static void control_to_network(RDMAControlHeader *control)
-{
-    control->type = htonl(control->type);
-    control->len = htonl(control->len);
-    control->repeat = htonl(control->repeat);
-}
-
-static void network_to_control(RDMAControlHeader *control)
-{
-    control->type = ntohl(control->type);
-    control->len = ntohl(control->len);
-    control->repeat = ntohl(control->repeat);
-}
-
-/*
- * Register a single Chunk.
- * Information sent by the source VM to inform the dest
- * to register an single chunk of memory before we can perform
- * the actual RDMA operation.
- */
-typedef struct QEMU_PACKED {
-    union QEMU_PACKED {
-        uint64_t current_addr;  /* offset into the ramblock of the chunk */
-        uint64_t chunk;         /* chunk to lookup if unregistering */
-    } key;
-    uint32_t current_index; /* which ramblock the chunk belongs to */
-    uint32_t padding;
-    uint64_t chunks;            /* how many sequential chunks to register */
-} RDMARegister;
-
-static void register_to_network(RDMARegister *reg)
-{
-    reg->key.current_addr = htonll(reg->key.current_addr);
-    reg->current_index = htonl(reg->current_index);
-    reg->chunks = htonll(reg->chunks);
-}
-
-static void network_to_register(RDMARegister *reg)
-{
-    reg->key.current_addr = ntohll(reg->key.current_addr);
-    reg->current_index = ntohl(reg->current_index);
-    reg->chunks = ntohll(reg->chunks);
-}
-
-typedef struct QEMU_PACKED {
-    uint32_t value;     /* if zero, we will madvise() */
-    uint32_t block_idx; /* which ram block index */
-    uint64_t offset;    /* where in the remote ramblock this chunk */
-    uint64_t length;    /* length of the chunk */
-} RDMACompress;
-
-static void compress_to_network(RDMACompress *comp)
-{
-    comp->value = htonl(comp->value);
-    comp->block_idx = htonl(comp->block_idx);
-    comp->offset = htonll(comp->offset);
-    comp->length = htonll(comp->length);
-}
-
-static void network_to_compress(RDMACompress *comp)
-{
-    comp->value = ntohl(comp->value);
-    comp->block_idx = ntohl(comp->block_idx);
-    comp->offset = ntohll(comp->offset);
-    comp->length = ntohll(comp->length);
-}
-
-/*
- * The result of the dest's memory registration produces an "rkey"
- * which the source VM must reference in order to perform
- * the RDMA operation.
- */
-typedef struct QEMU_PACKED {
-    uint32_t rkey;
-    uint32_t padding;
-    uint64_t host_addr;
-} RDMARegisterResult;
-
-static void result_to_network(RDMARegisterResult *result)
-{
-    result->rkey = htonl(result->rkey);
-    result->host_addr = htonll(result->host_addr);
-};
-
-static void network_to_result(RDMARegisterResult *result)
-{
-    result->rkey = ntohl(result->rkey);
-    result->host_addr = ntohll(result->host_addr);
-};
-
-const char *print_wrid(int wrid);
-static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
-                                   uint8_t *data, RDMAControlHeader *resp,
-                                   int *resp_idx,
-                                   int (*callback)(RDMAContext *rdma));
-
-static inline uint64_t ram_chunk_index(const uint8_t *start,
-                                       const uint8_t *host)
-{
-    return ((uintptr_t) host - (uintptr_t) start) >> RDMA_REG_CHUNK_SHIFT;
-}
-
-static inline uint8_t *ram_chunk_start(const RDMALocalBlock *rdma_ram_block,
-                                       uint64_t i)
-{
-    return (uint8_t *) (((uintptr_t) rdma_ram_block->local_host_addr)
-                                    + (i << RDMA_REG_CHUNK_SHIFT));
-}
-
-static inline uint8_t *ram_chunk_end(const RDMALocalBlock *rdma_ram_block,
-                                     uint64_t i)
-{
-    uint8_t *result = ram_chunk_start(rdma_ram_block, i) +
-                                         (1UL << RDMA_REG_CHUNK_SHIFT);
-
-    if (result > (rdma_ram_block->local_host_addr + rdma_ram_block->length)) {
-        result = rdma_ram_block->local_host_addr + rdma_ram_block->length;
-    }
-
-    return result;
-}
-
-static int __qemu_rdma_add_block(RDMAContext *rdma, void *host_addr,
-                         ram_addr_t block_offset, uint64_t length)
-{
-    RDMALocalBlocks *local = &rdma->local_ram_blocks;
-    RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap,
-        (void *) block_offset);
-    RDMALocalBlock *old = local->block;
-
-    assert(block == NULL);
-
-    local->block = g_malloc0(sizeof(RDMALocalBlock) * (local->nb_blocks + 1));
-
-    if (local->nb_blocks) {
-        int x;
-
-        for (x = 0; x < local->nb_blocks; x++) {
-            g_hash_table_remove(rdma->blockmap, (void *)old[x].offset);
-            g_hash_table_insert(rdma->blockmap, (void *)old[x].offset,
-                                                &local->block[x]);
-        }
-        memcpy(local->block, old, sizeof(RDMALocalBlock) * local->nb_blocks);
-        g_free(old);
-    }
-
-    block = &local->block[local->nb_blocks];
-
-    block->local_host_addr = host_addr;
-    block->offset = block_offset;
-    block->length = length;
-    block->index = local->nb_blocks;
-    block->nb_chunks = ram_chunk_index(host_addr, host_addr + length) + 1UL;
-    block->transit_bitmap = bitmap_new(block->nb_chunks);
-    bitmap_clear(block->transit_bitmap, 0, block->nb_chunks);
-    block->unregister_bitmap = bitmap_new(block->nb_chunks);
-    bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks);
-    block->remote_keys = g_malloc0(block->nb_chunks * sizeof(uint32_t));
-
-    block->is_ram_block = local->init ? false : true;
-
-    g_hash_table_insert(rdma->blockmap, (void *) block_offset, block);
-
-    DDPRINTF("Added Block: %d, addr: %" PRIu64 ", offset: %" PRIu64
-           " length: %" PRIu64 " end: %" PRIu64 " bits %" PRIu64 " chunks %d\n",
-            local->nb_blocks, (uint64_t) block->local_host_addr, block->offset,
-            block->length, (uint64_t) (block->local_host_addr + block->length),
-                BITS_TO_LONGS(block->nb_chunks) *
-                    sizeof(unsigned long) * 8, block->nb_chunks);
-
-    local->nb_blocks++;
-
-    return 0;
-}
-
-/*
- * Memory regions need to be registered with the device and queue pairs setup
- * in advanced before the migration starts. This tells us where the RAM blocks
- * are so that we can register them individually.
- */
-static void qemu_rdma_init_one_block(void *host_addr,
-    ram_addr_t block_offset, ram_addr_t length, void *opaque)
-{
-    __qemu_rdma_add_block(opaque, host_addr, block_offset, length);
-}
-
-/*
- * Identify the RAMBlocks and their quantity. They will be references to
- * identify chunk boundaries inside each RAMBlock and also be referenced
- * during dynamic page registration.
- */
-static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
-{
-    RDMALocalBlocks *local = &rdma->local_ram_blocks;
-
-    assert(rdma->blockmap == NULL);
-    rdma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal);
-    memset(local, 0, sizeof *local);
-    qemu_ram_foreach_block(qemu_rdma_init_one_block, rdma);
-    DPRINTF("Allocated %d local ram block structures\n", local->nb_blocks);
-    rdma->block = (RDMARemoteBlock *) g_malloc0(sizeof(RDMARemoteBlock) *
-                        rdma->local_ram_blocks.nb_blocks);
-    local->init = true;
-    return 0;
-}
-
-static int __qemu_rdma_delete_block(RDMAContext *rdma, ram_addr_t block_offset)
-{
-    RDMALocalBlocks *local = &rdma->local_ram_blocks;
-    RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap,
-        (void *) block_offset);
-    RDMALocalBlock *old = local->block;
-    int x;
-
-    assert(block);
-
-    if (block->pmr) {
-        int j;
-
-        for (j = 0; j < block->nb_chunks; j++) {
-            if (!block->pmr[j]) {
-                continue;
-            }
-            ibv_dereg_mr(block->pmr[j]);
-            rdma->total_registrations--;
-        }
-        g_free(block->pmr);
-        block->pmr = NULL;
-    }
-
-    if (block->mr) {
-        ibv_dereg_mr(block->mr);
-        rdma->total_registrations--;
-        block->mr = NULL;
-    }
-
-    g_free(block->transit_bitmap);
-    block->transit_bitmap = NULL;
-
-    g_free(block->unregister_bitmap);
-    block->unregister_bitmap = NULL;
-
-    g_free(block->remote_keys);
-    block->remote_keys = NULL;
-
-    for (x = 0; x < local->nb_blocks; x++) {
-        g_hash_table_remove(rdma->blockmap, (void *)old[x].offset);
-    }
-
-    if (local->nb_blocks > 1) {
-
-        local->block = g_malloc0(sizeof(RDMALocalBlock) *
-                                    (local->nb_blocks - 1));
-
-        if (block->index) {
-            memcpy(local->block, old, sizeof(RDMALocalBlock) * block->index);
-        }
-
-        if (block->index < (local->nb_blocks - 1)) {
-            memcpy(local->block + block->index, old + (block->index + 1),
-                sizeof(RDMALocalBlock) *
-                    (local->nb_blocks - (block->index + 1)));
-        }
-    } else {
-        assert(block == local->block);
-        local->block = NULL;
-    }
-
-    DDPRINTF("Deleted Block: %d, addr: %" PRIu64 ", offset: %" PRIu64
-           " length: %" PRIu64 " end: %" PRIu64 " bits %" PRIu64 " chunks %d\n",
-            local->nb_blocks, (uint64_t) block->local_host_addr, block->offset,
-            block->length, (uint64_t) (block->local_host_addr + block->length),
-                BITS_TO_LONGS(block->nb_chunks) *
-                    sizeof(unsigned long) * 8, block->nb_chunks);
-
-    g_free(old);
-
-    local->nb_blocks--;
-
-    if (local->nb_blocks) {
-        for (x = 0; x < local->nb_blocks; x++) {
-            g_hash_table_insert(rdma->blockmap, (void *)local->block[x].offset,
-                                                &local->block[x]);
-        }
-    }
-
-    return 0;
-}
-
-/*
- * Put in the log file which RDMA device was opened and the details
- * associated with that device.
- */
-static void qemu_rdma_dump_id(const char *who, struct ibv_context *verbs)
-{
-    struct ibv_port_attr port;
-
-    if (ibv_query_port(verbs, 1, &port)) {
-        fprintf(stderr, "FAILED TO QUERY PORT INFORMATION!\n");
-        return;
-    }
-
-    printf("%s RDMA Device opened: kernel name %s "
-           "uverbs device name %s, "
-           "infiniband_verbs class device path %s, "
-           "infiniband class device path %s, "
-           "transport: (%d) %s\n",
-                who,
-                verbs->device->name,
-                verbs->device->dev_name,
-                verbs->device->dev_path,
-                verbs->device->ibdev_path,
-                port.link_layer,
-                (port.link_layer == IBV_LINK_LAYER_INFINIBAND) ? "Infiniband" :
-                 ((port.link_layer == IBV_LINK_LAYER_ETHERNET) 
-                    ? "Ethernet" : "Unknown"));
-}
-
-/*
- * Put in the log file the RDMA gid addressing information,
- * useful for folks who have trouble understanding the
- * RDMA device hierarchy in the kernel.
- */
-static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id)
-{
-    char sgid[33];
-    char dgid[33];
-    inet_ntop(AF_INET6, &id->route.addr.addr.ibaddr.sgid, sgid, sizeof sgid);
-    inet_ntop(AF_INET6, &id->route.addr.addr.ibaddr.dgid, dgid, sizeof dgid);
-    DPRINTF("%s Source GID: %s, Dest GID: %s\n", who, sgid, dgid);
-}
-
-/*
- * As of now, IPv6 over RoCE / iWARP is not supported by linux.
- * We will try the next addrinfo struct, and fail if there are
- * no other valid addresses to bind against.
- *
- * If user is listening on '[::]', then we will not have a opened a device
- * yet and have no way of verifying if the device is RoCE or not.
- *
- * In this case, the source VM will throw an error for ALL types of
- * connections (both IPv4 and IPv6) if the destination machine does not have
- * a regular infiniband network available for use.
- *
- * The only way to guarantee that an error is thrown for broken kernels is
- * for the management software to choose a *specific* interface at bind time
- * and validate what time of hardware it is.
- *
- * Unfortunately, this puts the user in a fix:
- * 
- *  If the source VM connects with an IPv4 address without knowing that the
- *  destination has bound to '[::]' the migration will unconditionally fail
- *  unless the management software is explicitly listening on the the IPv4
- *  address while using a RoCE-based device.
- *
- *  If the source VM connects with an IPv6 address, then we're OK because we can
- *  throw an error on the source (and similarly on the destination).
- * 
- *  But in mixed environments, this will be broken for a while until it is fixed
- *  inside linux.
- *
- * We do provide a *tiny* bit of help in this function: We can list all of the
- * devices in the system and check to see if all the devices are RoCE or
- * Infiniband. 
- *
- * If we detect that we have a *pure* RoCE environment, then we can safely
- * thrown an error even if the management software has specified '[::]' as the
- * bind address.
- *
- * However, if there is are multiple hetergeneous devices, then we cannot make
- * this assumption and the user just has to be sure they know what they are
- * doing.
- *
- * Patches are being reviewed on linux-rdma.
- */
-static int qemu_rdma_broken_ipv6_kernel(Error **errp, struct ibv_context *verbs)
-{
-    struct ibv_port_attr port_attr;
-
-    /* This bug only exists in linux, to our knowledge. */
-#ifdef CONFIG_LINUX
-
-    /* 
-     * Verbs are only NULL if management has bound to '[::]'.
-     * 
-     * Let's iterate through all the devices and see if there any pure IB
-     * devices (non-ethernet).
-     * 
-     * If not, then we can safely proceed with the migration.
-     * Otherwise, there are no guarantees until the bug is fixed in linux.
-     */
-    if (!verbs) {
-	    int num_devices, x;
-        struct ibv_device ** dev_list = ibv_get_device_list(&num_devices);
-        bool roce_found = false;
-        bool ib_found = false;
-
-        for (x = 0; x < num_devices; x++) {
-            verbs = ibv_open_device(dev_list[x]);
-
-            if (ibv_query_port(verbs, 1, &port_attr)) {
-                ibv_close_device(verbs);
-                ERROR(errp, "Could not query initial IB port");
-                return -EINVAL;
-            }
-
-            if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
-                ib_found = true;
-            } else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
-                roce_found = true;
-            }
-
-            ibv_close_device(verbs);
-
-        }
-
-        if (roce_found) {
-            if (ib_found) {
-                fprintf(stderr, "WARN: migrations may fail:"
-                                " IPv6 over RoCE / iWARP in linux"
-                                " is broken. But since you appear to have a"
-                                " mixed RoCE / IB environment, be sure to only"
-                                " migrate over the IB fabric until the kernel "
-                                " fixes the bug.\n");
-            } else {
-                ERROR(errp, "You only have RoCE / iWARP devices in your systems"
-                            " and your management software has specified '[::]'"
-                            ", but IPv6 over RoCE / iWARP is not supported in Linux.");
-                return -ENONET;
-            }
-        }
-
-        return 0;
-    }
-
-    /*
-     * If we have a verbs context, that means that some other than '[::]' was
-     * used by the management software for binding. In which case we can actually 
-     * warn the user about a potential broken kernel;
-     */
-
-    /* IB ports start with 1, not 0 */
-    if (ibv_query_port(verbs, 1, &port_attr)) {
-        ERROR(errp, "Could not query initial IB port");
-        return -EINVAL;
-    }
-
-    if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
-        ERROR(errp, "Linux kernel's RoCE / iWARP does not support IPv6 "
-                    "(but patches on linux-rdma in progress)");
-        return -ENONET;
-    }
-
-#endif
-
-    return 0;
-}
-
-/*
- * Figure out which RDMA device corresponds to the requested IP hostname
- * Also create the initial connection manager identifiers for opening
- * the connection.
- */
-static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
-{
-    int ret;
-    struct rdma_addrinfo *res;
-    char port_str[16];
-    struct rdma_cm_event *cm_event;
-    char ip[40] = "unknown";
-    struct rdma_addrinfo *e;
-
-    if (rdma->host == NULL || !strcmp(rdma->host, "")) {
-        ERROR(errp, "RDMA hostname has not been set");
-        return -EINVAL;
-    }
-
-    /* create CM channel */
-    rdma->channel = rdma_create_event_channel();
-    if (!rdma->channel) {
-        ERROR(errp, "could not create CM channel");
-        return -EINVAL;
-    }
-
-    /* create CM id */
-    ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP);
-    if (ret) {
-        ERROR(errp, "could not create channel id");
-        goto err_resolve_create_id;
-    }
-
-    snprintf(port_str, 16, "%d", rdma->port);
-    port_str[15] = '\0';
-
-    ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
-    if (ret < 0) {
-        ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
-        goto err_resolve_get_addr;
-    }
-
-    for (e = res; e != NULL; e = e->ai_next) {
-        inet_ntop(e->ai_family,
-            &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip);
-        DPRINTF("Trying %s => %s\n", rdma->host, ip);
-
-        ret = rdma_resolve_addr(rdma->cm_id, NULL, e->ai_dst_addr,
-                RDMA_RESOLVE_TIMEOUT_MS);
-        if (!ret) {
-            if (e->ai_family == AF_INET6) {
-                ret = qemu_rdma_broken_ipv6_kernel(errp, rdma->cm_id->verbs);
-                if (ret) {
-                    continue;
-                }
-            }
-            goto route;
-        }
-    }
-
-    ERROR(errp, "could not resolve address %s", rdma->host);
-    goto err_resolve_get_addr;
-
-route:
-    qemu_rdma_dump_gid("source_resolve_addr", rdma->cm_id);
-
-    ret = rdma_get_cm_event(rdma->channel, &cm_event);
-    if (ret) {
-        ERROR(errp, "could not perform event_addr_resolved");
-        goto err_resolve_get_addr;
-    }
-
-    if (cm_event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
-        ERROR(errp, "result not equal to event_addr_resolved %s",
-                rdma_event_str(cm_event->event));
-        perror("rdma_resolve_addr");
-        rdma_ack_cm_event(cm_event);
-        ret = -EINVAL;
-        goto err_resolve_get_addr;
-    }
-    rdma_ack_cm_event(cm_event);
-
-    /* resolve route */
-    ret = rdma_resolve_route(rdma->cm_id, RDMA_RESOLVE_TIMEOUT_MS);
-    if (ret) {
-        ERROR(errp, "could not resolve rdma route");
-        goto err_resolve_get_addr;
-    }
-
-    ret = rdma_get_cm_event(rdma->channel, &cm_event);
-    if (ret) {
-        ERROR(errp, "could not perform event_route_resolved");
-        goto err_resolve_get_addr;
-    }
-    if (cm_event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
-        ERROR(errp, "result not equal to event_route_resolved: %s",
-                        rdma_event_str(cm_event->event));
-        rdma_ack_cm_event(cm_event);
-        ret = -EINVAL;
-        goto err_resolve_get_addr;
-    }
-    rdma_ack_cm_event(cm_event);
-    rdma->verbs = rdma->cm_id->verbs;
-    qemu_rdma_dump_id("source_resolve_host", rdma->cm_id->verbs);
-    qemu_rdma_dump_gid("source_resolve_host", rdma->cm_id);
-    return 0;
-
-err_resolve_get_addr:
-    rdma_destroy_id(rdma->cm_id);
-    rdma->cm_id = NULL;
-err_resolve_create_id:
-    rdma_destroy_event_channel(rdma->channel);
-    rdma->channel = NULL;
-    return ret;
-}
-
-/*
- * Create protection domain and completion queues
- */
-static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma)
-{
-    /* allocate pd */
-    rdma->pd = ibv_alloc_pd(rdma->verbs);
-    if (!rdma->pd) {
-        fprintf(stderr, "failed to allocate protection domain\n");
-        return -1;
-    }
-
-    /* create completion channel */
-    rdma->comp_channel = ibv_create_comp_channel(rdma->verbs);
-    if (!rdma->comp_channel) {
-        fprintf(stderr, "failed to allocate completion channel\n");
-        goto err_alloc_pd_cq;
-    }
-
-    /*
-     * Completion queue can be filled by both read and write work requests,
-     * so must reflect the sum of both possible queue sizes.
-     */
-    rdma->cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
-            NULL, rdma->comp_channel, 0);
-    if (!rdma->cq) {
-        fprintf(stderr, "failed to allocate completion queue\n");
-        goto err_alloc_pd_cq;
-    }
-
-    return 0;
-
-err_alloc_pd_cq:
-    if (rdma->pd) {
-        ibv_dealloc_pd(rdma->pd);
-    }
-    if (rdma->comp_channel) {
-        ibv_destroy_comp_channel(rdma->comp_channel);
-    }
-    rdma->pd = NULL;
-    rdma->comp_channel = NULL;
-    return -1;
-
-}
-
-/*
- * Create queue pairs.
- */
-static int qemu_rdma_alloc_qp(RDMAContext *rdma)
-{
-    struct ibv_qp_init_attr attr = { 0 };
-    int ret;
-
-    attr.cap.max_send_wr = RDMA_SIGNALED_SEND_MAX;
-    attr.cap.max_recv_wr = 3;
-    attr.cap.max_send_sge = 1;
-    attr.cap.max_recv_sge = 1;
-    attr.send_cq = rdma->cq;
-    attr.recv_cq = rdma->cq;
-    attr.qp_type = IBV_QPT_RC;
-
-    ret = rdma_create_qp(rdma->cm_id, rdma->pd, &attr);
-    if (ret) {
-        return -1;
-    }
-
-    rdma->qp = rdma->cm_id->qp;
-    return 0;
-}
-
-static int qemu_rdma_reg_whole_ram_blocks(RDMAContext *rdma)
-{
-    int i;
-    RDMALocalBlocks *local = &rdma->local_ram_blocks;
-
-    for (i = 0; i < local->nb_blocks; i++) {
-        local->block[i].mr =
-            ibv_reg_mr(rdma->pd,
-                    local->block[i].local_host_addr,
-                    local->block[i].length,
-                    IBV_ACCESS_LOCAL_WRITE |
-                    IBV_ACCESS_REMOTE_WRITE
-                    );
-        if (!local->block[i].mr) {
-            perror("Failed to register local dest ram block!\n");
-            break;
-        }
-        rdma->total_registrations++;
-    }
-
-    if (i >= local->nb_blocks) {
-        return 0;
-    }
-
-    for (i--; i >= 0; i--) {
-        ibv_dereg_mr(local->block[i].mr);
-        rdma->total_registrations--;
-    }
-
-    return -1;
-
-}
-
-/*
- * Find the ram block that corresponds to the page requested to be
- * transmitted by QEMU.
- *
- * Once the block is found, also identify which 'chunk' within that
- * block that the page belongs to.
- *
- * This search cannot fail or the migration will fail.
- */
-static int qemu_rdma_search_ram_block(RDMAContext *rdma,
-                                      uint64_t block_offset,
-                                      uint64_t offset,
-                                      uint64_t length,
-                                      uint64_t *block_index,
-                                      uint64_t *chunk_index)
-{
-    uint64_t current_addr = block_offset + offset;
-    RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap,
-                                                (void *) block_offset);
-    assert(block);
-    assert(current_addr >= block->offset);
-    assert((current_addr + length) <= (block->offset + block->length));
-
-    *block_index = block->index;
-    *chunk_index = ram_chunk_index(block->local_host_addr,
-                block->local_host_addr + (current_addr - block->offset));
-
-    return 0;
-}
-
-/*
- * Register a chunk with IB. If the chunk was already registered
- * previously, then skip.
- *
- * Also return the keys associated with the registration needed
- * to perform the actual RDMA operation.
- */
-static int qemu_rdma_register_and_get_keys(RDMAContext *rdma,
-        RDMALocalBlock *block, uint8_t *host_addr,
-        uint32_t *lkey, uint32_t *rkey, int chunk,
-        uint8_t *chunk_start, uint8_t *chunk_end)
-{
-    if (block->mr) {
-        if (lkey) {
-            *lkey = block->mr->lkey;
-        }
-        if (rkey) {
-            *rkey = block->mr->rkey;
-        }
-        return 0;
-    }
-
-    /* allocate memory to store chunk MRs */
-    if (!block->pmr) {
-        block->pmr = g_malloc0(block->nb_chunks * sizeof(struct ibv_mr *));
-        if (!block->pmr) {
-            return -1;
-        }
-    }
-
-    /*
-     * If 'rkey', then we're the destination, so grant access to the source.
-     *
-     * If 'lkey', then we're the source VM, so grant access only to ourselves.
-     */
-    if (!block->pmr[chunk]) {
-        uint64_t len = chunk_end - chunk_start;
-
-        DDPRINTF("Registering %" PRIu64 " bytes @ %p\n",
-                 len, chunk_start);
-
-        block->pmr[chunk] = ibv_reg_mr(rdma->pd,
-                chunk_start, len,
-                (rkey ? (IBV_ACCESS_LOCAL_WRITE |
-                        IBV_ACCESS_REMOTE_WRITE) : 0));
-
-        if (!block->pmr[chunk]) {
-            perror("Failed to register chunk!");
-            fprintf(stderr, "Chunk details: block: %d chunk index %d"
-                            " start %" PRIu64 " end %" PRIu64 " host %" PRIu64
-                            " local %" PRIu64 " registrations: %d\n",
-                            block->index, chunk, (uint64_t) chunk_start,
-                            (uint64_t) chunk_end, (uint64_t) host_addr,
-                            (uint64_t) block->local_host_addr,
-                            rdma->total_registrations);
-            return -1;
-        }
-        rdma->total_registrations++;
-    }
-
-    if (lkey) {
-        *lkey = block->pmr[chunk]->lkey;
-    }
-    if (rkey) {
-        *rkey = block->pmr[chunk]->rkey;
-    }
-    return 0;
-}
-
-/*
- * Register (at connection time) the memory used for control
- * channel messages.
- */
-static int qemu_rdma_reg_control(RDMAContext *rdma, int idx)
-{
-    rdma->wr_data[idx].control_mr = ibv_reg_mr(rdma->pd,
-            rdma->wr_data[idx].control, RDMA_CONTROL_MAX_BUFFER,
-            IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
-    if (rdma->wr_data[idx].control_mr) {
-        rdma->total_registrations++;
-        return 0;
-    }
-    fprintf(stderr, "qemu_rdma_reg_control failed!\n");
-    return -1;
-}
-
-const char *print_wrid(int wrid)
-{
-    if (wrid >= RDMA_WRID_RECV_CONTROL) {
-        return wrid_desc[RDMA_WRID_RECV_CONTROL];
-    }
-    return wrid_desc[wrid];
-}
-
-/*
- * RDMA requires memory registration (mlock/pinning), but this is not good for
- * overcommitment.
- *
- * In preparation for the future where LRU information or workload-specific
- * writable writable working set memory access behavior is available to QEMU
- * it would be nice to have in place the ability to UN-register/UN-pin
- * particular memory regions from the RDMA hardware when it is determine that
- * those regions of memory will likely not be accessed again in the near future.
- *
- * While we do not yet have such information right now, the following
- * compile-time option allows us to perform a non-optimized version of this
- * behavior.
- *
- * By uncommenting this option, you will cause *all* RDMA transfers to be
- * unregistered immediately after the transfer completes on both sides of the
- * connection. This has no effect in 'rdma-pin-all' mode, only regular mode.
- *
- * This will have a terrible impact on migration performance, so until future
- * workload information or LRU information is available, do not attempt to use
- * this feature except for basic testing.
- */
-//#define RDMA_UNREGISTRATION_EXAMPLE
-
-/*
- * Perform a non-optimized memory unregistration after every transfer
- * for demonsration purposes, only if pin-all is not requested.
- *
- * Potential optimizations:
- * 1. Start a new thread to run this function continuously
-        - for bit clearing
-        - and for receipt of unregister messages
- * 2. Use an LRU.
- * 3. Use workload hints.
- */
-static int qemu_rdma_unregister_waiting(RDMAContext *rdma)
-{
-    while (rdma->unregistrations[rdma->unregister_current]) {
-        int ret;
-        uint64_t wr_id = rdma->unregistrations[rdma->unregister_current];
-        uint64_t chunk =
-            (wr_id & RDMA_WRID_CHUNK_MASK) >> RDMA_WRID_CHUNK_SHIFT;
-        uint64_t index =
-            (wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT;
-        RDMALocalBlock *block =
-            &(rdma->local_ram_blocks.block[index]);
-        RDMARegister reg = { .current_index = index };
-        RDMAControlHeader resp = { .type = RDMA_CONTROL_UNREGISTER_FINISHED,
-                                 };
-        RDMAControlHeader head = { .len = sizeof(RDMARegister),
-                                   .type = RDMA_CONTROL_UNREGISTER_REQUEST,
-                                   .repeat = 1,
-                                 };
-
-        DDPRINTF("Processing unregister for chunk: %" PRIu64
-                 " at position %d\n", chunk, rdma->unregister_current);
-
-        rdma->unregistrations[rdma->unregister_current] = 0;
-        rdma->unregister_current++;
-
-        if (rdma->unregister_current == RDMA_SIGNALED_SEND_MAX) {
-            rdma->unregister_current = 0;
-        }
-
-
-        /*
-         * Unregistration is speculative (because migration is single-threaded
-         * and we cannot break the protocol's inifinband message ordering).
-         * Thus, if the memory is currently being used for transmission,
-         * then abort the attempt to unregister and try again
-         * later the next time a completion is received for this memory.
-         */
-        clear_bit(chunk, block->unregister_bitmap);
-
-        if (test_bit(chunk, block->transit_bitmap)) {
-            DDPRINTF("Cannot unregister inflight chunk: %" PRIu64 "\n", chunk);
-            continue;
-        }
-
-        DDPRINTF("Sending unregister for chunk: %" PRIu64 "\n", chunk);
-
-        ret = ibv_dereg_mr(block->pmr[chunk]);
-        block->pmr[chunk] = NULL;
-        block->remote_keys[chunk] = 0;
-
-        if (ret != 0) {
-            perror("unregistration chunk failed");
-            return -ret;
-        }
-        rdma->total_registrations--;
-
-        reg.key.chunk = chunk;
-        register_to_network(&reg);
-        ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) &reg,
-                                &resp, NULL, NULL);
-        if (ret < 0) {
-            return ret;
-        }
-
-        DDPRINTF("Unregister for chunk: %" PRIu64 " complete.\n", chunk);
-    }
-
-    return 0;
-}
-
-static uint64_t qemu_rdma_make_wrid(uint64_t wr_id, uint64_t index,
-                                         uint64_t chunk)
-{
-    uint64_t result = wr_id & RDMA_WRID_TYPE_MASK;
-
-    result |= (index << RDMA_WRID_BLOCK_SHIFT);
-    result |= (chunk << RDMA_WRID_CHUNK_SHIFT);
-
-    return result;
-}
-
-/*
- * Set bit for unregistration in the next iteration.
- * We cannot transmit right here, but will unpin later.
- */
-static void qemu_rdma_signal_unregister(RDMAContext *rdma, uint64_t index,
-                                        uint64_t chunk, uint64_t wr_id)
-{
-    if (rdma->unregistrations[rdma->unregister_next] != 0) {
-        fprintf(stderr, "rdma migration: queue is full!\n");
-    } else {
-        RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]);
-
-        if (!test_and_set_bit(chunk, block->unregister_bitmap)) {
-            DDPRINTF("Appending unregister chunk %" PRIu64
-                    " at position %d\n", chunk, rdma->unregister_next);
-
-            rdma->unregistrations[rdma->unregister_next++] =
-                    qemu_rdma_make_wrid(wr_id, index, chunk);
-
-            if (rdma->unregister_next == RDMA_SIGNALED_SEND_MAX) {
-                rdma->unregister_next = 0;
-            }
-        } else {
-            DDPRINTF("Unregister chunk %" PRIu64 " already in queue.\n",
-                    chunk);
-        }
-    }
-}
-
-/*
- * Consult the connection manager to see a work request
- * (of any kind) has completed.
- * Return the work request ID that completed.
- */
-static uint64_t qemu_rdma_poll(RDMAContext *rdma, uint64_t *wr_id_out,
-                               uint32_t *byte_len)
-{
-    int ret;
-    struct ibv_wc wc;
-    uint64_t wr_id;
-
-    ret = ibv_poll_cq(rdma->cq, 1, &wc);
-
-    if (!ret) {
-        *wr_id_out = RDMA_WRID_NONE;
-        return 0;
-    }
-
-    if (ret < 0) {
-        fprintf(stderr, "ibv_poll_cq return %d!\n", ret);
-        return ret;
-    }
-
-    wr_id = wc.wr_id & RDMA_WRID_TYPE_MASK;
-
-    if (wc.status != IBV_WC_SUCCESS) {
-        fprintf(stderr, "ibv_poll_cq wc.status=%d %s!\n",
-                        wc.status, ibv_wc_status_str(wc.status));
-        fprintf(stderr, "ibv_poll_cq wrid=%s!\n", wrid_desc[wr_id]);
-
-        return -1;
-    }
-
-    if (rdma->control_ready_expected &&
-        (wr_id >= RDMA_WRID_RECV_CONTROL)) {
-        DDDPRINTF("completion %s #%" PRId64 " received (%" PRId64 ")"
-                  " left %d\n", wrid_desc[RDMA_WRID_RECV_CONTROL],
-                  wr_id - RDMA_WRID_RECV_CONTROL, wr_id, rdma->nb_sent);
-        rdma->control_ready_expected = 0;
-    }
-
-    if (wr_id == RDMA_WRID_RDMA_WRITE) {
-        uint64_t chunk =
-            (wc.wr_id & RDMA_WRID_CHUNK_MASK) >> RDMA_WRID_CHUNK_SHIFT;
-        uint64_t index =
-            (wc.wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT;
-        RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]);
-
-        DDDPRINTF("completions %s (%" PRId64 ") left %d, "
-                 "block %" PRIu64 ", chunk: %" PRIu64 " %p %p\n",
-                 print_wrid(wr_id), wr_id, rdma->nb_sent, index, chunk,
-                 block->local_host_addr, (void *)block->remote_host_addr);
-
-        clear_bit(chunk, block->transit_bitmap);
-
-        if (rdma->nb_sent > 0) {
-            rdma->nb_sent--;
-        }
-
-        if (!rdma->pin_all) {
-            /*
-             * FYI: If one wanted to signal a specific chunk to be unregistered
-             * using LRU or workload-specific information, this is the function
-             * you would call to do so. That chunk would then get asynchronously
-             * unregistered later.
-             */
-#ifdef RDMA_UNREGISTRATION_EXAMPLE
-            qemu_rdma_signal_unregister(rdma, index, chunk, wc.wr_id);
-#endif
-        }
-    } else {
-        DDDPRINTF("other completion %s (%" PRId64 ") received left %d\n",
-            print_wrid(wr_id), wr_id, rdma->nb_sent);
-    }
-
-    *wr_id_out = wc.wr_id;
-    if (byte_len) {
-        *byte_len = wc.byte_len;
-    }
-
-    return  0;
-}
-
-/*
- * Block until the next work request has completed.
- *
- * First poll to see if a work request has already completed,
- * otherwise block.
- *
- * If we encounter completed work requests for IDs other than
- * the one we're interested in, then that's generally an error.
- *
- * The only exception is actual RDMA Write completions. These
- * completions only need to be recorded, but do not actually
- * need further processing.
- */
-static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested,
-                                    uint32_t *byte_len)
-{
-    int num_cq_events = 0, ret = 0;
-    struct ibv_cq *cq;
-    void *cq_ctx;
-    uint64_t wr_id = RDMA_WRID_NONE, wr_id_in;
-
-    if (ibv_req_notify_cq(rdma->cq, 0)) {
-        return -1;
-    }
-    /* poll cq first */
-    while (wr_id != wrid_requested) {
-        ret = qemu_rdma_poll(rdma, &wr_id_in, byte_len);
-        if (ret < 0) {
-            return ret;
-        }
-
-        wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
-
-        if (wr_id == RDMA_WRID_NONE) {
-            break;
-        }
-        if (wr_id != wrid_requested) {
-            DDDPRINTF("A Wanted wrid %s (%d) but got %s (%" PRIu64 ")\n",
-                print_wrid(wrid_requested),
-                wrid_requested, print_wrid(wr_id), wr_id);
-        }
-    }
-
-    if (wr_id == wrid_requested) {
-        return 0;
-    }
-
-    while (1) {
-        /*
-         * Coroutine doesn't start until process_incoming_migration()
-         * so don't yield unless we know we're running inside of a coroutine.
-         */
-        if (rdma->migration_started_on_destination) {
-            yield_until_fd_readable(rdma->comp_channel->fd);
-        }
-
-        if (ibv_get_cq_event(rdma->comp_channel, &cq, &cq_ctx)) {
-            perror("ibv_get_cq_event");
-            goto err_block_for_wrid;
-        }
-
-        num_cq_events++;
-
-        if (ibv_req_notify_cq(cq, 0)) {
-            goto err_block_for_wrid;
-        }
-
-        while (wr_id != wrid_requested) {
-            ret = qemu_rdma_poll(rdma, &wr_id_in, byte_len);
-            if (ret < 0) {
-                goto err_block_for_wrid;
-            }
-
-            wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
-
-            if (wr_id == RDMA_WRID_NONE) {
-                break;
-            }
-            if (wr_id != wrid_requested) {
-                DDDPRINTF("B Wanted wrid %s (%d) but got %s (%" PRIu64 ")\n",
-                    print_wrid(wrid_requested), wrid_requested,
-                    print_wrid(wr_id), wr_id);
-            }
-        }
-
-        if (wr_id == wrid_requested) {
-            goto success_block_for_wrid;
-        }
-    }
-
-success_block_for_wrid:
-    if (num_cq_events) {
-        ibv_ack_cq_events(cq, num_cq_events);
-    }
-    return 0;
-
-err_block_for_wrid:
-    if (num_cq_events) {
-        ibv_ack_cq_events(cq, num_cq_events);
-    }
-    return ret;
-}
-
-/*
- * Post a SEND message work request for the control channel
- * containing some data and block until the post completes.
- */
-static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf,
-                                       RDMAControlHeader *head)
-{
-    int ret = 0;
-    RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL];
-    struct ibv_send_wr *bad_wr;
-    struct ibv_sge sge = {
-                           .addr = (uint64_t)(wr->control),
-                           .length = head->len + sizeof(RDMAControlHeader),
-                           .lkey = wr->control_mr->lkey,
-                         };
-    struct ibv_send_wr send_wr = {
-                                   .wr_id = RDMA_WRID_SEND_CONTROL,
-                                   .opcode = IBV_WR_SEND,
-                                   .send_flags = IBV_SEND_SIGNALED,
-                                   .sg_list = &sge,
-                                   .num_sge = 1,
-                                };
-
-    DDDPRINTF("CONTROL: sending %s..\n", control_desc[head->type]);
-
-    /*
-     * We don't actually need to do a memcpy() in here if we used
-     * the "sge" properly, but since we're only sending control messages
-     * (not RAM in a performance-critical path), then its OK for now.
-     *
-     * The copy makes the RDMAControlHeader simpler to manipulate
-     * for the time being.
-     */
-    assert(head->len <= RDMA_CONTROL_MAX_BUFFER - sizeof(*head));
-    memcpy(wr->control, head, sizeof(RDMAControlHeader));
-    control_to_network((void *) wr->control);
-
-    if (buf) {
-        memcpy(wr->control + sizeof(RDMAControlHeader), buf, head->len);
-    }
-
-
-    ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr);
-
-    if (ret > 0) {
-        fprintf(stderr, "Failed to use post IB SEND for control!\n");
-        return -ret;
-    }
-
-    ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL);
-    if (ret < 0) {
-        fprintf(stderr, "rdma migration: send polling control error!\n");
-    }
-
-    return ret;
-}
-
-/*
- * Post a RECV work request in anticipation of some future receipt
- * of data on the control channel.
- */
-static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx)
-{
-    struct ibv_recv_wr *bad_wr;
-    struct ibv_sge sge = {
-                            .addr = (uint64_t)(rdma->wr_data[idx].control),
-                            .length = RDMA_CONTROL_MAX_BUFFER,
-                            .lkey = rdma->wr_data[idx].control_mr->lkey,
-                         };
-
-    struct ibv_recv_wr recv_wr = {
-                                    .wr_id = RDMA_WRID_RECV_CONTROL + idx,
-                                    .sg_list = &sge,
-                                    .num_sge = 1,
-                                 };
-
-
-    if (ibv_post_recv(rdma->qp, &recv_wr, &bad_wr)) {
-        return -1;
-    }
-
-    return 0;
-}
-
-/*
- * Block and wait for a RECV control channel message to arrive.
- */
-static int qemu_rdma_exchange_get_response(RDMAContext *rdma,
-                RDMAControlHeader *head, int expecting, int idx)
-{
-    uint32_t byte_len;
-    int ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RECV_CONTROL + idx,
-                                       &byte_len);
-
-    if (ret < 0) {
-        fprintf(stderr, "rdma migration: recv polling control error!\n");
-        return ret;
-    }
-
-    network_to_control((void *) rdma->wr_data[idx].control);
-    memcpy(head, rdma->wr_data[idx].control, sizeof(RDMAControlHeader));
-
-    DDDPRINTF("CONTROL: %s receiving...\n", control_desc[expecting]);
-
-    if (expecting == RDMA_CONTROL_NONE) {
-        DDDPRINTF("Surprise: got %s (%d)\n",
-                  control_desc[head->type], head->type);
-    } else if (head->type != expecting || head->type == RDMA_CONTROL_ERROR) {
-        fprintf(stderr, "Was expecting a %s (%d) control message"
-                ", but got: %s (%d), length: %d\n",
-                control_desc[expecting], expecting,
-                control_desc[head->type], head->type, head->len);
-        return -EIO;
-    }
-    if (head->len > RDMA_CONTROL_MAX_BUFFER - sizeof(*head)) {
-        fprintf(stderr, "too long length: %d\n", head->len);
-        return -EINVAL;
-    }
-    if (sizeof(*head) + head->len != byte_len) {
-        fprintf(stderr, "Malformed length: %d byte_len %d\n",
-                head->len, byte_len);
-        return -EINVAL;
-    }
-
-    return 0;
-}
-
-/*
- * When a RECV work request has completed, the work request's
- * buffer is pointed at the header.
- *
- * This will advance the pointer to the data portion
- * of the control message of the work request's buffer that
- * was populated after the work request finished.
- */
-static void qemu_rdma_move_header(RDMAContext *rdma, int idx,
-                                  RDMAControlHeader *head)
-{
-    rdma->wr_data[idx].control_len = head->len;
-    rdma->wr_data[idx].control_curr =
-        rdma->wr_data[idx].control + sizeof(RDMAControlHeader);
-}
-
-/*
- * This is an 'atomic' high-level operation to deliver a single, unified
- * control-channel message.
- *
- * Additionally, if the user is expecting some kind of reply to this message,
- * they can request a 'resp' response message be filled in by posting an
- * additional work request on behalf of the user and waiting for an additional
- * completion.
- *
- * The extra (optional) response is used during registration to us from having
- * to perform an *additional* exchange of message just to provide a response by
- * instead piggy-backing on the acknowledgement.
- */
-static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
-                                   uint8_t *data, RDMAControlHeader *resp,
-                                   int *resp_idx,
-                                   int (*callback)(RDMAContext *rdma))
-{
-    int ret = 0;
-
-    /*
-     * Wait until the dest is ready before attempting to deliver the message
-     * by waiting for a READY message.
-     */
-    if (rdma->control_ready_expected) {
-        RDMAControlHeader resp;
-        ret = qemu_rdma_exchange_get_response(rdma,
-                                    &resp, RDMA_CONTROL_READY, RDMA_WRID_READY);
-        if (ret < 0) {
-            return ret;
-        }
-    }
-
-    /*
-     * If the user is expecting a response, post a WR in anticipation of it.
-     */
-    if (resp) {
-        ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA);
-        if (ret) {
-            fprintf(stderr, "rdma migration: error posting"
-                    " extra control recv for anticipated result!");
-            return ret;
-        }
-    }
-
-    /*
-     * Post a WR to replace the one we just consumed for the READY message.
-     */
-    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
-    if (ret) {
-        fprintf(stderr, "rdma migration: error posting first control recv!");
-        return ret;
-    }
-
-    /*
-     * Deliver the control message that was requested.
-     */
-    ret = qemu_rdma_post_send_control(rdma, data, head);
-
-    if (ret < 0) {
-        fprintf(stderr, "Failed to send control buffer!\n");
-        return ret;
-    }
-
-    /*
-     * If we're expecting a response, block and wait for it.
-     */
-    if (resp) {
-        if (callback) {
-            DDPRINTF("Issuing callback before receiving response...\n");
-            ret = callback(rdma);
-            if (ret < 0) {
-                return ret;
-            }
-        }
-
-        DDPRINTF("Waiting for response %s\n", control_desc[resp->type]);
-        ret = qemu_rdma_exchange_get_response(rdma, resp,
-                                              resp->type, RDMA_WRID_DATA);
-
-        if (ret < 0) {
-            return ret;
-        }
-
-        qemu_rdma_move_header(rdma, RDMA_WRID_DATA, resp);
-        if (resp_idx) {
-            *resp_idx = RDMA_WRID_DATA;
-        }
-        DDPRINTF("Response %s received.\n", control_desc[resp->type]);
-    }
-
-    rdma->control_ready_expected = 1;
-
-    return 0;
-}
-
-/*
- * This is an 'atomic' high-level operation to receive a single, unified
- * control-channel message.
- */
-static int qemu_rdma_exchange_recv(RDMAContext *rdma, RDMAControlHeader *head,
-                                int expecting)
-{
-    RDMAControlHeader ready = {
-                                .len = 0,
-                                .type = RDMA_CONTROL_READY,
-                                .repeat = 1,
-                              };
-    int ret;
-
-    /*
-     * Inform the source that we're ready to receive a message.
-     */
-    ret = qemu_rdma_post_send_control(rdma, NULL, &ready);
-
-    if (ret < 0) {
-        fprintf(stderr, "Failed to send control buffer!\n");
-        return ret;
-    }
-
-    /*
-     * Block and wait for the message.
-     */
-    ret = qemu_rdma_exchange_get_response(rdma, head,
-                                          expecting, RDMA_WRID_READY);
-
-    if (ret < 0) {
-        return ret;
-    }
-
-    qemu_rdma_move_header(rdma, RDMA_WRID_READY, head);
-
-    /*
-     * Post a new RECV work request to replace the one we just consumed.
-     */
-    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
-    if (ret) {
-        fprintf(stderr, "rdma migration: error posting second control recv!");
-        return ret;
-    }
-
-    return 0;
-}
-
-/*
- * Write an actual chunk of memory using RDMA.
- *
- * If we're using dynamic registration on the dest-side, we have to
- * send a registration command first.
- */
-static int qemu_rdma_write_one(QEMUFile *f, RDMAContext *rdma,
-                               int current_index, uint64_t current_addr,
-                               uint64_t length)
-{
-    struct ibv_sge sge;
-    struct ibv_send_wr send_wr = { 0 };
-    struct ibv_send_wr *bad_wr;
-    int reg_result_idx, ret, count = 0;
-    uint64_t chunk, chunks;
-    uint8_t *chunk_start, *chunk_end;
-    RDMALocalBlock *block = &(rdma->local_ram_blocks.block[current_index]);
-    RDMARegister reg;
-    RDMARegisterResult *reg_result;
-    RDMAControlHeader resp = { .type = RDMA_CONTROL_REGISTER_RESULT };
-    RDMAControlHeader head = { .len = sizeof(RDMARegister),
-                               .type = RDMA_CONTROL_REGISTER_REQUEST,
-                               .repeat = 1,
-                             };
-
-retry:
-    sge.addr = (uint64_t)(block->local_host_addr +
-                            (current_addr - block->offset));
-    sge.length = length;
-
-    chunk = ram_chunk_index(block->local_host_addr, (uint8_t *) sge.addr);
-    chunk_start = ram_chunk_start(block, chunk);
-
-    if (block->is_ram_block) {
-        chunks = length / (1UL << RDMA_REG_CHUNK_SHIFT);
-
-        if (chunks && ((length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) {
-            chunks--;
-        }
-    } else {
-        chunks = block->length / (1UL << RDMA_REG_CHUNK_SHIFT);
-
-        if (chunks && ((block->length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) {
-            chunks--;
-        }
-    }
-
-    DDPRINTF("Writing %" PRIu64 " chunks, (%" PRIu64 " MB)\n",
-        chunks + 1, (chunks + 1) * (1UL << RDMA_REG_CHUNK_SHIFT) / 1024 / 1024);
-
-    chunk_end = ram_chunk_end(block, chunk + chunks);
-
-    if (!rdma->pin_all) {
-#ifdef RDMA_UNREGISTRATION_EXAMPLE
-        qemu_rdma_unregister_waiting(rdma);
-#endif
-    }
-
-    while (test_bit(chunk, block->transit_bitmap)) {
-        (void)count;
-        DDPRINTF("(%d) Not clobbering: block: %d chunk %" PRIu64
-                " current %" PRIu64 " len %" PRIu64 " %d %d\n",
-                count++, current_index, chunk,
-                sge.addr, length, rdma->nb_sent, block->nb_chunks);
-
-        ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
-
-        if (ret < 0) {
-            fprintf(stderr, "Failed to Wait for previous write to complete "
-                    "block %d chunk %" PRIu64
-                    " current %" PRIu64 " len %" PRIu64 " %d\n",
-                    current_index, chunk, sge.addr, length, rdma->nb_sent);
-            return ret;
-        }
-    }
-
-    if (!rdma->pin_all || !block->is_ram_block) {
-        if (!block->remote_keys[chunk]) {
-            /*
-             * This chunk has not yet been registered, so first check to see
-             * if the entire chunk is zero. If so, tell the other size to
-             * memset() + madvise() the entire chunk without RDMA.
-             */
-
-            if (can_use_buffer_find_nonzero_offset((void *)sge.addr, length)
-                   && buffer_find_nonzero_offset((void *)sge.addr,
-                                                    length) == length) {
-                RDMACompress comp = {
-                                        .offset = current_addr,
-                                        .value = 0,
-                                        .block_idx = current_index,
-                                        .length = length,
-                                    };
-
-                head.len = sizeof(comp);
-                head.type = RDMA_CONTROL_COMPRESS;
-
-                DDPRINTF("Entire chunk is zero, sending compress: %"
-                    PRIu64 " for %d "
-                    "bytes, index: %d, offset: %" PRId64 "...\n",
-                    chunk, sge.length, current_index, current_addr);
-
-                compress_to_network(&comp);
-                ret = qemu_rdma_exchange_send(rdma, &head,
-                                (uint8_t *) &comp, NULL, NULL, NULL);
-
-                if (ret < 0) {
-                    return -EIO;
-                }
-
-                acct_update_position(f, sge.length, true);
-
-                return 1;
-            }
-
-            /*
-             * Otherwise, tell other side to register.
-             */
-            reg.current_index = current_index;
-            if (block->is_ram_block) {
-                reg.key.current_addr = current_addr;
-            } else {
-                reg.key.chunk = chunk;
-            }
-            reg.chunks = chunks;
-
-            DDPRINTF("Sending registration request chunk %" PRIu64 " for %d "
-                    "bytes, index: %d, offset: %" PRId64 "...\n",
-                    chunk, sge.length, current_index, current_addr);
-
-            register_to_network(&reg);
-            ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) &reg,
-                                    &resp, &reg_result_idx, NULL);
-            if (ret < 0) {
-                return ret;
-            }
-
-            /* try to overlap this single registration with the one we sent. */
-            if (qemu_rdma_register_and_get_keys(rdma, block,
-                                                (uint8_t *) sge.addr,
-                                                &sge.lkey, NULL, chunk,
-                                                chunk_start, chunk_end)) {
-                fprintf(stderr, "cannot get lkey!\n");
-                return -EINVAL;
-            }
-
-            reg_result = (RDMARegisterResult *)
-                    rdma->wr_data[reg_result_idx].control_curr;
-
-            network_to_result(reg_result);
-
-            DDPRINTF("Received registration result:"
-                    " my key: %x their key %x, chunk %" PRIu64 "\n",
-                    block->remote_keys[chunk], reg_result->rkey, chunk);
-
-            block->remote_keys[chunk] = reg_result->rkey;
-            block->remote_host_addr = reg_result->host_addr;
-        } else {
-            /* already registered before */
-            if (qemu_rdma_register_and_get_keys(rdma, block,
-                                                (uint8_t *)sge.addr,
-                                                &sge.lkey, NULL, chunk,
-                                                chunk_start, chunk_end)) {
-                fprintf(stderr, "cannot get lkey!\n");
-                return -EINVAL;
-            }
-        }
-
-        send_wr.wr.rdma.rkey = block->remote_keys[chunk];
-    } else {
-        send_wr.wr.rdma.rkey = block->remote_rkey;
-
-        if (qemu_rdma_register_and_get_keys(rdma, block, (uint8_t *)sge.addr,
-                                                     &sge.lkey, NULL, chunk,
-                                                     chunk_start, chunk_end)) {
-            fprintf(stderr, "cannot get lkey!\n");
-            return -EINVAL;
-        }
-    }
-
-    /*
-     * Encode the ram block index and chunk within this wrid.
-     * We will use this information at the time of completion
-     * to figure out which bitmap to check against and then which
-     * chunk in the bitmap to look for.
-     */
-    send_wr.wr_id = qemu_rdma_make_wrid(RDMA_WRID_RDMA_WRITE,
-                                        current_index, chunk);
-
-    send_wr.opcode = IBV_WR_RDMA_WRITE;
-    send_wr.send_flags = IBV_SEND_SIGNALED;
-    send_wr.sg_list = &sge;
-    send_wr.num_sge = 1;
-    send_wr.wr.rdma.remote_addr = block->remote_host_addr +
-                                (current_addr - block->offset);
-
-    DDDPRINTF("Posting chunk: %" PRIu64 ", addr: %lx"
-              " remote: %lx, bytes %" PRIu32 "\n",
-              chunk, sge.addr, send_wr.wr.rdma.remote_addr,
-              sge.length);
-
-    /*
-     * ibv_post_send() does not return negative error numbers,
-     * per the specification they are positive - no idea why.
-     */
-    ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr);
-
-    if (ret == ENOMEM) {
-        DDPRINTF("send queue is full. wait a little....\n");
-        ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
-        if (ret < 0) {
-            fprintf(stderr, "rdma migration: failed to make "
-                            "room in full send queue! %d\n", ret);
-            return ret;
-        }
-
-        goto retry;
-
-    } else if (ret > 0) {
-        perror("rdma migration: post rdma write failed");
-        return -ret;
-    }
-
-    set_bit(chunk, block->transit_bitmap);
-    acct_update_position(f, sge.length, false);
-    rdma->total_writes++;
-
-    return 0;
-}
-
-/*
- * Push out any unwritten RDMA operations.
- *
- * We support sending out multiple chunks at the same time.
- * Not all of them need to get signaled in the completion queue.
- */
-static int qemu_rdma_write_flush(QEMUFile *f, RDMAContext *rdma)
-{
-    int ret;
-
-    if (!rdma->current_length) {
-        return 0;
-    }
-
-    ret = qemu_rdma_write_one(f, rdma,
-            rdma->current_index, rdma->current_addr, rdma->current_length);
-
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret == 0) {
-        rdma->nb_sent++;
-        DDDPRINTF("sent total: %d\n", rdma->nb_sent);
-    }
-
-    rdma->current_length = 0;
-    rdma->current_addr = 0;
-
-    return 0;
-}
-
-static inline int qemu_rdma_buffer_mergable(RDMAContext *rdma,
-                    uint64_t offset, uint64_t len)
-{
-    RDMALocalBlock *block;
-    uint8_t *host_addr;
-    uint8_t *chunk_end;
-
-    if (rdma->current_index < 0) {
-        return 0;
-    }
-
-    if (rdma->current_chunk < 0) {
-        return 0;
-    }
-
-    block = &(rdma->local_ram_blocks.block[rdma->current_index]);
-    host_addr = block->local_host_addr + (offset - block->offset);
-    chunk_end = ram_chunk_end(block, rdma->current_chunk);
-
-    if (rdma->current_length == 0) {
-        return 0;
-    }
-
-    /*
-     * Only merge into chunk sequentially.
-     */
-    if (offset != (rdma->current_addr + rdma->current_length)) {
-        return 0;
-    }
-
-    if (offset < block->offset) {
-        return 0;
-    }
-
-    if ((offset + len) > (block->offset + block->length)) {
-        return 0;
-    }
-
-    if ((host_addr + len) > chunk_end) {
-        return 0;
-    }
-
-    return 1;
-}
-
-/*
- * We're not actually writing here, but doing three things:
- *
- * 1. Identify the chunk the buffer belongs to.
- * 2. If the chunk is full or the buffer doesn't belong to the current
- *    chunk, then start a new chunk and flush() the old chunk.
- * 3. To keep the hardware busy, we also group chunks into batches
- *    and only require that a batch gets acknowledged in the completion
- *    qeueue instead of each individual chunk.
- */
-static int qemu_rdma_write(QEMUFile *f, RDMAContext *rdma,
-                           uint64_t block_offset, uint64_t offset,
-                           uint64_t len)
-{
-    uint64_t current_addr = block_offset + offset;
-    uint64_t index = rdma->current_index;
-    uint64_t chunk = rdma->current_chunk;
-    int ret;
-
-    /* If we cannot merge it, we flush the current buffer first. */
-    if (!qemu_rdma_buffer_mergable(rdma, current_addr, len)) {
-        ret = qemu_rdma_write_flush(f, rdma);
-        if (ret) {
-            return ret;
-        }
-        rdma->current_length = 0;
-        rdma->current_addr = current_addr;
-
-        ret = qemu_rdma_search_ram_block(rdma, block_offset,
-                                         offset, len, &index, &chunk);
-        if (ret) {
-            fprintf(stderr, "ram block search failed\n");
-            return ret;
-        }
-        rdma->current_index = index;
-        rdma->current_chunk = chunk;
-    }
-
-    /* merge it */
-    rdma->current_length += len;
-
-    /* flush it if buffer is too large */
-    if (rdma->current_length >= RDMA_MERGE_MAX) {
-        return qemu_rdma_write_flush(f, rdma);
-    }
-
-    return 0;
-}
-
-static void qemu_rdma_cleanup(RDMAContext *rdma)
-{
-    struct rdma_cm_event *cm_event;
-    int ret, idx;
-
-    if (rdma->cm_id && rdma->connected) {
-        if (rdma->error_state) {
-            RDMAControlHeader head = { .len = 0,
-                                       .type = RDMA_CONTROL_ERROR,
-                                       .repeat = 1,
-                                     };
-            fprintf(stderr, "Early error. Sending error.\n");
-            qemu_rdma_post_send_control(rdma, NULL, &head);
-        }
-
-        ret = rdma_disconnect(rdma->cm_id);
-        if (!ret) {
-            DDPRINTF("waiting for disconnect\n");
-            ret = rdma_get_cm_event(rdma->channel, &cm_event);
-            if (!ret) {
-                rdma_ack_cm_event(cm_event);
-            }
-        }
-        DDPRINTF("Disconnected.\n");
-        rdma->connected = false;
-    }
-
-    g_free(rdma->block);
-    rdma->block = NULL;
-
-    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
-        if (rdma->wr_data[idx].control_mr) {
-            rdma->total_registrations--;
-            ibv_dereg_mr(rdma->wr_data[idx].control_mr);
-        }
-        rdma->wr_data[idx].control_mr = NULL;
-    }
-
-    if (rdma->local_ram_blocks.block) {
-        while (rdma->local_ram_blocks.nb_blocks) {
-            __qemu_rdma_delete_block(rdma,
-                    rdma->local_ram_blocks.block->offset);
-        }
-    }
-
-    if (rdma->cq) {
-        ibv_destroy_cq(rdma->cq);
-        rdma->cq = NULL;
-    }
-    if (rdma->comp_channel) {
-        ibv_destroy_comp_channel(rdma->comp_channel);
-        rdma->comp_channel = NULL;
-    }
-    if (rdma->pd) {
-        ibv_dealloc_pd(rdma->pd);
-        rdma->pd = NULL;
-    }
-    if (rdma->listen_id) {
-        rdma_destroy_id(rdma->listen_id);
-        rdma->listen_id = NULL;
-    }
-    if (rdma->cm_id) {
-        if (rdma->qp) {
-            rdma_destroy_qp(rdma->cm_id);
-            rdma->qp = NULL;
-        }
-        rdma_destroy_id(rdma->cm_id);
-        rdma->cm_id = NULL;
-    }
-    if (rdma->channel) {
-        rdma_destroy_event_channel(rdma->channel);
-        rdma->channel = NULL;
-    }
-    g_free(rdma->host);
-    rdma->host = NULL;
-}
-
-
-static int qemu_rdma_source_init(RDMAContext *rdma, Error **errp, bool pin_all)
-{
-    int ret, idx;
-    Error *local_err = NULL, **temp = &local_err;
-
-    /*
-     * Will be validated against destination's actual capabilities
-     * after the connect() completes.
-     */
-    rdma->pin_all = pin_all;
-
-    ret = qemu_rdma_resolve_host(rdma, temp);
-    if (ret) {
-        goto err_rdma_source_init;
-    }
-
-    ret = qemu_rdma_alloc_pd_cq(rdma);
-    if (ret) {
-        ERROR(temp, "rdma migration: error allocating pd and cq! Your mlock()"
-                    " limits may be too low. Please check $ ulimit -a # and "
-                    "search for 'ulimit -l' in the output");
-        goto err_rdma_source_init;
-    }
-
-    ret = qemu_rdma_alloc_qp(rdma);
-    if (ret) {
-        ERROR(temp, "rdma migration: error allocating qp!");
-        goto err_rdma_source_init;
-    }
-
-    ret = qemu_rdma_init_ram_blocks(rdma);
-    if (ret) {
-        ERROR(temp, "rdma migration: error initializing ram blocks!");
-        goto err_rdma_source_init;
-    }
-
-    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
-        ret = qemu_rdma_reg_control(rdma, idx);
-        if (ret) {
-            ERROR(temp, "rdma migration: error registering %d control!",
-                                                            idx);
-            goto err_rdma_source_init;
-        }
-    }
-
-    return 0;
-
-err_rdma_source_init:
-    error_propagate(errp, local_err);
-    qemu_rdma_cleanup(rdma);
-    return -1;
-}
-
-static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
-{
-    RDMACapabilities cap = {
-                                .version = RDMA_CONTROL_VERSION_CURRENT,
-                                .flags = 0,
-                           };
-    struct rdma_conn_param conn_param = { .initiator_depth = 2,
-                                          .retry_count = 5,
-                                          .private_data = &cap,
-                                          .private_data_len = sizeof(cap),
-                                        };
-    struct rdma_cm_event *cm_event;
-    int ret;
-
-    /*
-     * Only negotiate the capability with destination if the user
-     * on the source first requested the capability.
-     */
-    if (rdma->pin_all) {
-        DPRINTF("Server pin-all memory requested.\n");
-        cap.flags |= RDMA_CAPABILITY_PIN_ALL;
-    }
-
-    caps_to_network(&cap);
-
-    ret = rdma_connect(rdma->cm_id, &conn_param);
-    if (ret) {
-        perror("rdma_connect");
-        ERROR(errp, "connecting to destination!");
-        rdma_destroy_id(rdma->cm_id);
-        rdma->cm_id = NULL;
-        goto err_rdma_source_connect;
-    }
-
-    ret = rdma_get_cm_event(rdma->channel, &cm_event);
-    if (ret) {
-        perror("rdma_get_cm_event after rdma_connect");
-        ERROR(errp, "connecting to destination!");
-        rdma_ack_cm_event(cm_event);
-        rdma_destroy_id(rdma->cm_id);
-        rdma->cm_id = NULL;
-        goto err_rdma_source_connect;
-    }
-
-    if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
-        perror("rdma_get_cm_event != EVENT_ESTABLISHED after rdma_connect");
-        ERROR(errp, "connecting to destination!");
-        rdma_ack_cm_event(cm_event);
-        rdma_destroy_id(rdma->cm_id);
-        rdma->cm_id = NULL;
-        goto err_rdma_source_connect;
-    }
-    rdma->connected = true;
-
-    memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap));
-    network_to_caps(&cap);
-
-    /*
-     * Verify that the *requested* capabilities are supported by the destination
-     * and disable them otherwise.
-     */
-    if (rdma->pin_all && !(cap.flags & RDMA_CAPABILITY_PIN_ALL)) {
-        ERROR(errp, "Server cannot support pinning all memory. "
-                        "Will register memory dynamically.");
-        rdma->pin_all = false;
-    }
-
-    DPRINTF("Pin all memory: %s\n", rdma->pin_all ? "enabled" : "disabled");
-
-    rdma_ack_cm_event(cm_event);
-
-    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
-    if (ret) {
-        ERROR(errp, "posting second control recv!");
-        goto err_rdma_source_connect;
-    }
-
-    rdma->control_ready_expected = 1;
-    rdma->nb_sent = 0;
-    return 0;
-
-err_rdma_source_connect:
-    qemu_rdma_cleanup(rdma);
-    return -1;
-}
-
-static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
-{
-    int ret = -EINVAL, idx;
-    struct rdma_cm_id *listen_id;
-    char ip[40] = "unknown";
-    struct rdma_addrinfo *res;
-    char port_str[16];
-
-    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
-        rdma->wr_data[idx].control_len = 0;
-        rdma->wr_data[idx].control_curr = NULL;
-    }
-
-    if (rdma->host == NULL) {
-        ERROR(errp, "RDMA host is not set!");
-        rdma->error_state = -EINVAL;
-        return -1;
-    }
-    /* create CM channel */
-    rdma->channel = rdma_create_event_channel();
-    if (!rdma->channel) {
-        ERROR(errp, "could not create rdma event channel");
-        rdma->error_state = -EINVAL;
-        return -1;
-    }
-
-    /* create CM id */
-    ret = rdma_create_id(rdma->channel, &listen_id, NULL, RDMA_PS_TCP);
-    if (ret) {
-        ERROR(errp, "could not create cm_id!");
-        goto err_dest_init_create_listen_id;
-    }
-
-    snprintf(port_str, 16, "%d", rdma->port);
-    port_str[15] = '\0';
-
-    if (rdma->host && strcmp("", rdma->host)) {
-        struct rdma_addrinfo *e;
-
-        ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
-        if (ret < 0) {
-            ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
-            goto err_dest_init_bind_addr;
-        }
-
-        for (e = res; e != NULL; e = e->ai_next) {
-            inet_ntop(e->ai_family,
-                &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip);
-            DPRINTF("Trying %s => %s\n", rdma->host, ip);
-            ret = rdma_bind_addr(listen_id, e->ai_dst_addr);
-            if (!ret) {
-                if (e->ai_family == AF_INET6) {
-                    ret = qemu_rdma_broken_ipv6_kernel(errp, listen_id->verbs);
-                    if (ret) {
-                        continue;
-                    }
-                }
-                    
-                goto listen;
-            }
-        }
-
-        ERROR(errp, "Error: could not rdma_bind_addr!");
-        goto err_dest_init_bind_addr;
-    } else {
-        ERROR(errp, "migration host and port not specified!");
-        ret = -EINVAL;
-        goto err_dest_init_bind_addr;
-    }
-listen:
-
-    rdma->listen_id = listen_id;
-    qemu_rdma_dump_gid("dest_init", listen_id);
-    return 0;
-
-err_dest_init_bind_addr:
-    rdma_destroy_id(listen_id);
-err_dest_init_create_listen_id:
-    rdma_destroy_event_channel(rdma->channel);
-    rdma->channel = NULL;
-    rdma->error_state = ret;
-    return ret;
-
-}
-
-static void *qemu_rdma_data_init(const char *host_port, Error **errp)
-{
-    RDMAContext *rdma = NULL;
-    InetSocketAddress *addr;
-
-    if (host_port) {
-        rdma = g_malloc0(sizeof(RDMAContext));
-        memset(rdma, 0, sizeof(RDMAContext));
-        rdma->current_index = -1;
-        rdma->current_chunk = -1;
-
-        addr = inet_parse(host_port, NULL);
-        if (addr != NULL) {
-            rdma->port = atoi(addr->port);
-            rdma->host = g_strdup(addr->host);
-        } else {
-            ERROR(errp, "bad RDMA migration address '%s'", host_port);
-            g_free(rdma);
-            rdma = NULL;
-        }
-
-        qapi_free_InetSocketAddress(addr);
-    }
-
-    return rdma;
-}
-
-/*
- * QEMUFile interface to the control channel.
- * SEND messages for control only.
- * VM's ram is handled with regular RDMA messages.
- */
-static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf,
-                                int64_t pos, int size)
-{
-    QEMUFileRDMA *r = opaque;
-    QEMUFile *f = r->file;
-    RDMAContext *rdma = r->rdma;
-    size_t remaining = size;
-    uint8_t * data = (void *) buf;
-    int ret;
-
-    CHECK_ERROR_STATE();
-
-    /*
-     * Push out any writes that
-     * we're queued up for VM's ram.
-     */
-    ret = qemu_rdma_write_flush(f, rdma);
-    if (ret < 0) {
-        rdma->error_state = ret;
-        return ret;
-    }
-
-    while (remaining) {
-        RDMAControlHeader head;
-
-        r->len = MIN(remaining, RDMA_SEND_INCREMENT);
-        remaining -= r->len;
-
-        head.len = r->len;
-        head.type = RDMA_CONTROL_QEMU_FILE;
-
-        ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL);
-
-        if (ret < 0) {
-            rdma->error_state = ret;
-            return ret;
-        }
-
-        data += r->len;
-    }
-
-    return size;
-}
-
-static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf,
-                             int size, int idx)
-{
-    size_t len = 0;
-
-    if (rdma->wr_data[idx].control_len) {
-        DDDPRINTF("RDMA %" PRId64 " of %d bytes already in buffer\n",
-                    rdma->wr_data[idx].control_len, size);
-
-        len = MIN(size, rdma->wr_data[idx].control_len);
-        memcpy(buf, rdma->wr_data[idx].control_curr, len);
-        rdma->wr_data[idx].control_curr += len;
-        rdma->wr_data[idx].control_len -= len;
-    }
-
-    return len;
-}
-
-/*
- * QEMUFile interface to the control channel.
- * RDMA links don't use bytestreams, so we have to
- * return bytes to QEMUFile opportunistically.
- */
-static int qemu_rdma_get_buffer(void *opaque, uint8_t *buf,
-                                int64_t pos, int size)
-{
-    QEMUFileRDMA *r = opaque;
-    RDMAContext *rdma = r->rdma;
-    RDMAControlHeader head;
-    int ret = 0;
-
-    CHECK_ERROR_STATE();
-
-    /*
-     * First, we hold on to the last SEND message we
-     * were given and dish out the bytes until we run
-     * out of bytes.
-     */
-    r->len = qemu_rdma_fill(r->rdma, buf, size, 0);
-    if (r->len) {
-        return r->len;
-    }
-
-    /*
-     * Once we run out, we block and wait for another
-     * SEND message to arrive.
-     */
-    ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE);
-
-    if (ret < 0) {
-        rdma->error_state = ret;
-        return ret;
-    }
-
-    /*
-     * SEND was received with new bytes, now try again.
-     */
-    return qemu_rdma_fill(r->rdma, buf, size, 0);
-}
-
-/*
- * Block until all the outstanding chunks have been delivered by the hardware.
- */
-static int qemu_rdma_drain_cq(QEMUFile *f, RDMAContext *rdma)
-{
-    int ret;
-
-    if (qemu_rdma_write_flush(f, rdma) < 0) {
-        return -EIO;
-    }
-
-    while (rdma->nb_sent) {
-        ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
-        if (ret < 0) {
-            fprintf(stderr, "rdma migration: complete polling error!\n");
-            return -EIO;
-        }
-    }
-
-    qemu_rdma_unregister_waiting(rdma);
-
-    return 0;
-}
-
-static int qemu_rdma_close(void *opaque)
-{
-    DPRINTF("Shutting down connection.\n");
-    QEMUFileRDMA *r = opaque;
-    if (r->rdma) {
-        qemu_rdma_cleanup(r->rdma);
-        g_free(r->rdma);
-    }
-    g_free(r);
-    return 0;
-}
-
-/*
- * Parameters:
- *    @offset == 0 :
- *        This means that 'block_offset' is a full virtual address that does not
- *        belong to a RAMBlock of the virtual machine and instead
- *        represents a private malloc'd memory area that the caller wishes to
- *        transfer.
- *
- *    @offset != 0 :
- *        Offset is an offset to be added to block_offset and used
- *        to also lookup the corresponding RAMBlock.
- *
- *    @size > 0 :
- *        Initiate an transfer this size.
- *
- *    @size == 0 :
- *        A 'hint' or 'advice' that means that we wish to speculatively
- *        and asynchronously unregister this memory. In this case, there is no
- *        guarantee that the unregister will actually happen, for example,
- *        if the memory is being actively transmitted. Additionally, the memory
- *        may be re-registered at any future time if a write within the same
- *        chunk was requested again, even if you attempted to unregister it
- *        here.
- *
- *    @size < 0 : TODO, not yet supported
- *        Unregister the memory NOW. This means that the caller does not
- *        expect there to be any future RDMA transfers and we just want to clean
- *        things up. This is used in case the upper layer owns the memory and
- *        cannot wait for qemu_fclose() to occur.
- *
- *    @bytes_sent : User-specificed pointer to indicate how many bytes were
- *                  sent. Usually, this will not be more than a few bytes of
- *                  the protocol because most transfers are sent asynchronously.
- */
-static size_t qemu_rdma_save_page(QEMUFile *f, void *opaque,
-                                  ram_addr_t block_offset, ram_addr_t offset,
-                                  size_t size, int *bytes_sent)
-{
-    QEMUFileRDMA *rfile = opaque;
-    RDMAContext *rdma = rfile->rdma;
-    int ret;
-
-    CHECK_ERROR_STATE();
-
-    qemu_fflush(f);
-
-    if (size > 0) {
-        /*
-         * Add this page to the current 'chunk'. If the chunk
-         * is full, or the page doen't belong to the current chunk,
-         * an actual RDMA write will occur and a new chunk will be formed.
-         */
-        ret = qemu_rdma_write(f, rdma, block_offset, offset, size);
-        if (ret < 0) {
-            fprintf(stderr, "rdma migration: write error! %d\n", ret);
-            goto err;
-        }
-
-        /*
-         * We always return 1 bytes because the RDMA
-         * protocol is completely asynchronous. We do not yet know
-         * whether an  identified chunk is zero or not because we're
-         * waiting for other pages to potentially be merged with
-         * the current chunk. So, we have to call qemu_update_position()
-         * later on when the actual write occurs.
-         */
-        if (bytes_sent) {
-            *bytes_sent = 1;
-        }
-    } else {
-        uint64_t index, chunk;
-
-        /* TODO: Change QEMUFileOps prototype to be signed: size_t => long
-        if (size < 0) {
-            ret = qemu_rdma_drain_cq(f, rdma);
-            if (ret < 0) {
-                fprintf(stderr, "rdma: failed to synchronously drain"
-                                " completion queue before unregistration.\n");
-                goto err;
-            }
-        }
-        */
-
-        ret = qemu_rdma_search_ram_block(rdma, block_offset,
-                                         offset, size, &index, &chunk);
-
-        if (ret) {
-            fprintf(stderr, "ram block search failed\n");
-            goto err;
-        }
-
-        qemu_rdma_signal_unregister(rdma, index, chunk, 0);
-
-        /*
-         * TODO: Synchronous, guaranteed unregistration (should not occur during
-         * fast-path). Otherwise, unregisters will process on the next call to
-         * qemu_rdma_drain_cq()
-        if (size < 0) {
-            qemu_rdma_unregister_waiting(rdma);
-        }
-        */
-    }
-
-    /*
-     * Drain the Completion Queue if possible, but do not block,
-     * just poll.
-     *
-     * If nothing to poll, the end of the iteration will do this
-     * again to make sure we don't overflow the request queue.
-     */
-    while (1) {
-        uint64_t wr_id, wr_id_in;
-        int ret = qemu_rdma_poll(rdma, &wr_id_in, NULL);
-        if (ret < 0) {
-            fprintf(stderr, "rdma migration: polling error! %d\n", ret);
-            goto err;
-        }
-
-        wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
-
-        if (wr_id == RDMA_WRID_NONE) {
-            break;
-        }
-    }
-
-    return RAM_SAVE_CONTROL_DELAYED;
-err:
-    rdma->error_state = ret;
-    return ret;
-}
-
-static int qemu_rdma_accept(RDMAContext *rdma)
-{
-    RDMACapabilities cap;
-    struct rdma_conn_param conn_param = {
-                                            .responder_resources = 2,
-                                            .private_data = &cap,
-                                            .private_data_len = sizeof(cap),
-                                         };
-    struct rdma_cm_event *cm_event;
-    struct ibv_context *verbs;
-    int ret = -EINVAL;
-    int idx;
-
-    ret = rdma_get_cm_event(rdma->channel, &cm_event);
-    if (ret) {
-        goto err_rdma_dest_wait;
-    }
-
-    if (cm_event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
-        rdma_ack_cm_event(cm_event);
-        goto err_rdma_dest_wait;
-    }
-
-    memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap));
-
-    network_to_caps(&cap);
-
-    if (cap.version < 1 || cap.version > RDMA_CONTROL_VERSION_CURRENT) {
-            fprintf(stderr, "Unknown source RDMA version: %d, bailing...\n",
-                            cap.version);
-            rdma_ack_cm_event(cm_event);
-            goto err_rdma_dest_wait;
-    }
-
-    /*
-     * Respond with only the capabilities this version of QEMU knows about.
-     */
-    cap.flags &= known_capabilities;
-
-    /*
-     * Enable the ones that we do know about.
-     * Add other checks here as new ones are introduced.
-     */
-    if (cap.flags & RDMA_CAPABILITY_PIN_ALL) {
-        rdma->pin_all = true;
-    }
-
-    rdma->cm_id = cm_event->id;
-    verbs = cm_event->id->verbs;
-
-    rdma_ack_cm_event(cm_event);
-
-    DPRINTF("Memory pin all: %s\n", rdma->pin_all ? "enabled" : "disabled");
-
-    caps_to_network(&cap);
-
-    DPRINTF("verbs context after listen: %p\n", verbs);
-
-    if (!rdma->verbs) {
-        rdma->verbs = verbs;
-    } else if (rdma->verbs != verbs) {
-            fprintf(stderr, "ibv context not matching %p, %p!\n",
-                    rdma->verbs, verbs);
-            goto err_rdma_dest_wait;
-    }
-
-    qemu_rdma_dump_id("dest_init", verbs);
-
-    ret = qemu_rdma_alloc_pd_cq(rdma);
-    if (ret) {
-        fprintf(stderr, "rdma migration: error allocating pd and cq!\n");
-        goto err_rdma_dest_wait;
-    }
-
-    ret = qemu_rdma_alloc_qp(rdma);
-    if (ret) {
-        fprintf(stderr, "rdma migration: error allocating qp!\n");
-        goto err_rdma_dest_wait;
-    }
-
-    ret = qemu_rdma_init_ram_blocks(rdma);
-    if (ret) {
-        fprintf(stderr, "rdma migration: error initializing ram blocks!\n");
-        goto err_rdma_dest_wait;
-    }
-
-    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
-        ret = qemu_rdma_reg_control(rdma, idx);
-        if (ret) {
-            fprintf(stderr, "rdma: error registering %d control!\n", idx);
-            goto err_rdma_dest_wait;
-        }
-    }
-
-    qemu_set_fd_handler2(rdma->channel->fd, NULL, NULL, NULL, NULL);
-
-    ret = rdma_accept(rdma->cm_id, &conn_param);
-    if (ret) {
-        fprintf(stderr, "rdma_accept returns %d!\n", ret);
-        goto err_rdma_dest_wait;
-    }
-
-    ret = rdma_get_cm_event(rdma->channel, &cm_event);
-    if (ret) {
-        fprintf(stderr, "rdma_accept get_cm_event failed %d!\n", ret);
-        goto err_rdma_dest_wait;
-    }
-
-    if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
-        fprintf(stderr, "rdma_accept not event established!\n");
-        rdma_ack_cm_event(cm_event);
-        goto err_rdma_dest_wait;
-    }
-
-    rdma_ack_cm_event(cm_event);
-    rdma->connected = true;
-
-    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
-    if (ret) {
-        fprintf(stderr, "rdma migration: error posting second control recv!\n");
-        goto err_rdma_dest_wait;
-    }
-
-    qemu_rdma_dump_gid("dest_connect", rdma->cm_id);
-
-    return 0;
-
-err_rdma_dest_wait:
-    rdma->error_state = ret;
-    qemu_rdma_cleanup(rdma);
-    return ret;
-}
-
-/*
- * During each iteration of the migration, we listen for instructions
- * by the source VM to perform dynamic page registrations before they
- * can perform RDMA operations.
- *
- * We respond with the 'rkey'.
- *
- * Keep doing this until the source tells us to stop.
- */
-static int qemu_rdma_registration_handle(QEMUFile *f, void *opaque,
-                                         uint64_t flags)
-{
-    RDMAControlHeader reg_resp = { .len = sizeof(RDMARegisterResult),
-                               .type = RDMA_CONTROL_REGISTER_RESULT,
-                               .repeat = 0,
-                             };
-    RDMAControlHeader unreg_resp = { .len = 0,
-                               .type = RDMA_CONTROL_UNREGISTER_FINISHED,
-                               .repeat = 0,
-                             };
-    RDMAControlHeader blocks = { .type = RDMA_CONTROL_RAM_BLOCKS_RESULT,
-                                 .repeat = 1 };
-    QEMUFileRDMA *rfile = opaque;
-    RDMAContext *rdma = rfile->rdma;
-    RDMALocalBlocks *local = &rdma->local_ram_blocks;
-    RDMAControlHeader head;
-    RDMARegister *reg, *registers;
-    RDMACompress *comp;
-    RDMARegisterResult *reg_result;
-    static RDMARegisterResult results[RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE];
-    RDMALocalBlock *block;
-    void *host_addr;
-    int ret = 0;
-    int idx = 0;
-    int count = 0;
-    int i = 0;
-
-    CHECK_ERROR_STATE();
-
-    do {
-        DDDPRINTF("Waiting for next request %" PRIu64 "...\n", flags);
-
-        ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE);
-
-        if (ret < 0) {
-            break;
-        }
-
-        if (head.repeat > RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE) {
-            fprintf(stderr, "rdma: Too many requests in this message (%d)."
-                            "Bailing.\n", head.repeat);
-            ret = -EIO;
-            break;
-        }
-
-        switch (head.type) {
-        case RDMA_CONTROL_COMPRESS:
-            comp = (RDMACompress *) rdma->wr_data[idx].control_curr;
-            network_to_compress(comp);
-
-            DDPRINTF("Zapping zero chunk: %" PRId64
-                    " bytes, index %d, offset %" PRId64 "\n",
-                    comp->length, comp->block_idx, comp->offset);
-            block = &(rdma->local_ram_blocks.block[comp->block_idx]);
-
-            host_addr = block->local_host_addr +
-                            (comp->offset - block->offset);
-
-            ram_handle_compressed(host_addr, comp->value, comp->length);
-            break;
-
-        case RDMA_CONTROL_REGISTER_FINISHED:
-            DDDPRINTF("Current registrations complete.\n");
-            goto out;
-
-        case RDMA_CONTROL_RAM_BLOCKS_REQUEST:
-            DPRINTF("Initial setup info requested.\n");
-
-            if (rdma->pin_all) {
-                ret = qemu_rdma_reg_whole_ram_blocks(rdma);
-                if (ret) {
-                    fprintf(stderr, "rdma migration: error dest "
-                                    "registering ram blocks!\n");
-                    goto out;
-                }
-            }
-
-            /*
-             * Dest uses this to prepare to transmit the RAMBlock descriptions
-             * to the source VM after connection setup.
-             * Both sides use the "remote" structure to communicate and update
-             * their "local" descriptions with what was sent.
-             */
-            for (i = 0; i < local->nb_blocks; i++) {
-                rdma->block[i].remote_host_addr =
-                    (uint64_t)(local->block[i].local_host_addr);
-
-                if (rdma->pin_all) {
-                    rdma->block[i].remote_rkey = local->block[i].mr->rkey;
-                }
-
-                rdma->block[i].offset = local->block[i].offset;
-                rdma->block[i].length = local->block[i].length;
-
-                remote_block_to_network(&rdma->block[i]);
-            }
-
-            blocks.len = rdma->local_ram_blocks.nb_blocks
-                                                * sizeof(RDMARemoteBlock);
-
-
-            ret = qemu_rdma_post_send_control(rdma,
-                                        (uint8_t *) rdma->block, &blocks);
-
-            if (ret < 0) {
-                fprintf(stderr, "rdma migration: error sending remote info!\n");
-                goto out;
-            }
-
-            break;
-        case RDMA_CONTROL_REGISTER_REQUEST:
-            DDPRINTF("There are %d registration requests\n", head.repeat);
-
-            reg_resp.repeat = head.repeat;
-            registers = (RDMARegister *) rdma->wr_data[idx].control_curr;
-
-            for (count = 0; count < head.repeat; count++) {
-                uint64_t chunk;
-                uint8_t *chunk_start, *chunk_end;
-
-                reg = &registers[count];
-                network_to_register(reg);
-
-                reg_result = &results[count];
-
-                DDPRINTF("Registration request (%d): index %d, current_addr %"
-                         PRIu64 " chunks: %" PRIu64 "\n", count,
-                         reg->current_index, reg->key.current_addr, reg->chunks);
-
-                block = &(rdma->local_ram_blocks.block[reg->current_index]);
-                if (block->is_ram_block) {
-                    host_addr = (block->local_host_addr +
-                                (reg->key.current_addr - block->offset));
-                    chunk = ram_chunk_index(block->local_host_addr,
-                                            (uint8_t *) host_addr);
-                } else {
-                    chunk = reg->key.chunk;
-                    host_addr = block->local_host_addr +
-                        (reg->key.chunk * (1UL << RDMA_REG_CHUNK_SHIFT));
-                }
-                chunk_start = ram_chunk_start(block, chunk);
-                chunk_end = ram_chunk_end(block, chunk + reg->chunks);
-                if (qemu_rdma_register_and_get_keys(rdma, block,
-                            (uint8_t *)host_addr, NULL, &reg_result->rkey,
-                            chunk, chunk_start, chunk_end)) {
-                    fprintf(stderr, "cannot get rkey!\n");
-                    ret = -EINVAL;
-                    goto out;
-                }
-
-                reg_result->host_addr = (uint64_t) block->local_host_addr;
-
-                DDPRINTF("Registered rkey for this request: %x\n",
-                                reg_result->rkey);
-
-                result_to_network(reg_result);
-            }
-
-            ret = qemu_rdma_post_send_control(rdma,
-                            (uint8_t *) results, &reg_resp);
-
-            if (ret < 0) {
-                fprintf(stderr, "Failed to send control buffer!\n");
-                goto out;
-            }
-            break;
-        case RDMA_CONTROL_UNREGISTER_REQUEST:
-            DDPRINTF("There are %d unregistration requests\n", head.repeat);
-            unreg_resp.repeat = head.repeat;
-            registers = (RDMARegister *) rdma->wr_data[idx].control_curr;
-
-            for (count = 0; count < head.repeat; count++) {
-                reg = &registers[count];
-                network_to_register(reg);
-
-                DDPRINTF("Unregistration request (%d): "
-                         " index %d, chunk %" PRIu64 "\n",
-                         count, reg->current_index, reg->key.chunk);
-
-                block = &(rdma->local_ram_blocks.block[reg->current_index]);
-
-                ret = ibv_dereg_mr(block->pmr[reg->key.chunk]);
-                block->pmr[reg->key.chunk] = NULL;
-
-                if (ret != 0) {
-                    perror("rdma unregistration chunk failed");
-                    ret = -ret;
-                    goto out;
-                }
-
-                rdma->total_registrations--;
-
-                DDPRINTF("Unregistered chunk %" PRIu64 " successfully.\n",
-                            reg->key.chunk);
-            }
-
-            ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp);
-
-            if (ret < 0) {
-                fprintf(stderr, "Failed to send control buffer!\n");
-                goto out;
-            }
-            break;
-        case RDMA_CONTROL_REGISTER_RESULT:
-            fprintf(stderr, "Invalid RESULT message at dest.\n");
-            ret = -EIO;
-            goto out;
-        default:
-            fprintf(stderr, "Unknown control message %s\n",
-                                control_desc[head.type]);
-            ret = -EIO;
-            goto out;
-        }
-    } while (1);
-out:
-    if (ret < 0) {
-        rdma->error_state = ret;
-    }
-    return ret;
-}
-
-static int qemu_rdma_registration_start(QEMUFile *f, void *opaque,
-                                        uint64_t flags)
-{
-    QEMUFileRDMA *rfile = opaque;
-    RDMAContext *rdma = rfile->rdma;
-
-    CHECK_ERROR_STATE();
-
-    DDDPRINTF("start section: %" PRIu64 "\n", flags);
-    qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
-    qemu_fflush(f);
-
-    return 0;
-}
-
-/*
- * Inform dest that dynamic registrations are done for now.
- * First, flush writes, if any.
- */
-static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque,
-                                       uint64_t flags)
-{
-    Error *local_err = NULL, **errp = &local_err;
-    QEMUFileRDMA *rfile = opaque;
-    RDMAContext *rdma = rfile->rdma;
-    RDMAControlHeader head = { .len = 0, .repeat = 1 };
-    int ret = 0;
-
-    CHECK_ERROR_STATE();
-
-    qemu_fflush(f);
-    ret = qemu_rdma_drain_cq(f, rdma);
-
-    if (ret < 0) {
-        goto err;
-    }
-
-    if (flags == RAM_CONTROL_SETUP) {
-        RDMAControlHeader resp = {.type = RDMA_CONTROL_RAM_BLOCKS_RESULT };
-        RDMALocalBlocks *local = &rdma->local_ram_blocks;
-        int reg_result_idx, i, j, nb_remote_blocks;
-
-        head.type = RDMA_CONTROL_RAM_BLOCKS_REQUEST;
-        DPRINTF("Sending registration setup for ram blocks...\n");
-
-        /*
-         * Make sure that we parallelize the pinning on both sides.
-         * For very large guests, doing this serially takes a really
-         * long time, so we have to 'interleave' the pinning locally
-         * with the control messages by performing the pinning on this
-         * side before we receive the control response from the other
-         * side that the pinning has completed.
-         */
-        ret = qemu_rdma_exchange_send(rdma, &head, NULL, &resp,
-                    &reg_result_idx, rdma->pin_all ?
-                    qemu_rdma_reg_whole_ram_blocks : NULL);
-        if (ret < 0) {
-            ERROR(errp, "receiving remote info!");
-            return ret;
-        }
-
-        nb_remote_blocks = resp.len / sizeof(RDMARemoteBlock);
-
-        /*
-         * The protocol uses two different sets of rkeys (mutually exclusive):
-         * 1. One key to represent the virtual address of the entire ram block.
-         *    (dynamic chunk registration disabled - pin everything with one rkey.)
-         * 2. One to represent individual chunks within a ram block.
-         *    (dynamic chunk registration enabled - pin individual chunks.)
-         *
-         * Once the capability is successfully negotiated, the destination transmits
-         * the keys to use (or sends them later) including the virtual addresses
-         * and then propagates the remote ram block descriptions to his local copy.
-         */
-
-        if (local->nb_blocks != nb_remote_blocks) {
-            ERROR(errp, "ram blocks mismatch #1! "
-                        "Your QEMU command line parameters are probably "
-                        "not identical on both the source and destination.");
-            return -EINVAL;
-        }
-
-        qemu_rdma_move_header(rdma, reg_result_idx, &resp);
-        memcpy(rdma->block,
-            rdma->wr_data[reg_result_idx].control_curr, resp.len);
-        for (i = 0; i < nb_remote_blocks; i++) {
-            network_to_remote_block(&rdma->block[i]);
-
-            /* search local ram blocks */
-            for (j = 0; j < local->nb_blocks; j++) {
-                if (rdma->block[i].offset != local->block[j].offset) {
-                    continue;
-                }
-
-                if (rdma->block[i].length != local->block[j].length) {
-                    ERROR(errp, "ram blocks mismatch #2! "
-                        "Your QEMU command line parameters are probably "
-                        "not identical on both the source and destination.");
-                    return -EINVAL;
-                }
-                local->block[j].remote_host_addr =
-                        rdma->block[i].remote_host_addr;
-                local->block[j].remote_rkey = rdma->block[i].remote_rkey;
-                break;
-            }
-
-            if (j >= local->nb_blocks) {
-                ERROR(errp, "ram blocks mismatch #3! "
-                        "Your QEMU command line parameters are probably "
-                        "not identical on both the source and destination.");
-                return -EINVAL;
-            }
-        }
-    }
-
-    DDDPRINTF("Sending registration finish %" PRIu64 "...\n", flags);
-
-    head.type = RDMA_CONTROL_REGISTER_FINISHED;
-    ret = qemu_rdma_exchange_send(rdma, &head, NULL, NULL, NULL, NULL);
-
-    if (ret < 0) {
-        goto err;
-    }
-
-    return 0;
-err:
-    rdma->error_state = ret;
-    return ret;
-}
-
-static int qemu_rdma_get_fd(void *opaque)
-{
-    QEMUFileRDMA *rfile = opaque;
-    RDMAContext *rdma = rfile->rdma;
-
-    return rdma->comp_channel->fd;
-}
-
-const QEMUFileOps rdma_read_ops = {
-    .get_buffer    = qemu_rdma_get_buffer,
-    .get_fd        = qemu_rdma_get_fd,
-    .close         = qemu_rdma_close,
-    .hook_ram_load = qemu_rdma_registration_handle,
-};
-
-const QEMUFileOps rdma_write_ops = {
-    .put_buffer         = qemu_rdma_put_buffer,
-    .close              = qemu_rdma_close,
-    .before_ram_iterate = qemu_rdma_registration_start,
-    .after_ram_iterate  = qemu_rdma_registration_stop,
-    .save_page          = qemu_rdma_save_page,
-};
-
-static void *qemu_fopen_rdma(RDMAContext *rdma, const char *mode)
-{
-    QEMUFileRDMA *r = g_malloc0(sizeof(QEMUFileRDMA));
-
-    if (qemu_file_mode_is_not_valid(mode)) {
-        return NULL;
-    }
-
-    r->rdma = rdma;
-
-    if (mode[0] == 'w') {
-        r->file = qemu_fopen_ops(r, &rdma_write_ops);
-    } else {
-        r->file = qemu_fopen_ops(r, &rdma_read_ops);
-    }
-
-    return r->file;
-}
-
-static void rdma_accept_incoming_migration(void *opaque)
-{
-    RDMAContext *rdma = opaque;
-    int ret;
-    QEMUFile *f;
-    Error *local_err = NULL, **errp = &local_err;
-
-    DPRINTF("Accepting rdma connection...\n");
-    ret = qemu_rdma_accept(rdma);
-
-    if (ret) {
-        ERROR(errp, "RDMA Migration initialization failed!");
-        return;
-    }
-
-    DPRINTF("Accepted migration\n");
-
-    f = qemu_fopen_rdma(rdma, "rb");
-    if (f == NULL) {
-        ERROR(errp, "could not qemu_fopen_rdma!");
-        qemu_rdma_cleanup(rdma);
-        return;
-    }
-
-    rdma->migration_started_on_destination = 1;
-    process_incoming_migration(f);
-}
-
-void rdma_start_incoming_migration(const char *host_port, Error **errp)
-{
-    int ret;
-    RDMAContext *rdma;
-    Error *local_err = NULL;
-
-    DPRINTF("Starting RDMA-based incoming migration\n");
-    rdma = qemu_rdma_data_init(host_port, &local_err);
-
-    if (rdma == NULL) {
-        goto err;
-    }
-
-    ret = qemu_rdma_dest_init(rdma, &local_err);
-
-    if (ret) {
-        goto err;
-    }
-
-    DPRINTF("qemu_rdma_dest_init success\n");
-
-    ret = rdma_listen(rdma->listen_id, 5);
-
-    if (ret) {
-        ERROR(errp, "listening on socket!");
-        goto err;
-    }
-
-    DPRINTF("rdma_listen success\n");
-
-    qemu_set_fd_handler2(rdma->channel->fd, NULL,
-                         rdma_accept_incoming_migration, NULL,
-                            (void *)(intptr_t) rdma);
-    return;
-err:
-    error_propagate(errp, local_err);
-    g_free(rdma);
-}
-
-void rdma_start_outgoing_migration(void *opaque,
-                            const char *host_port, Error **errp)
-{
-    MigrationState *s = opaque;
-    Error *local_err = NULL, **temp = &local_err;
-    RDMAContext *rdma = qemu_rdma_data_init(host_port, &local_err);
-    int ret = 0;
-
-    if (rdma == NULL) {
-        ERROR(temp, "Failed to initialize RDMA data structures! %d", ret);
-        goto err;
-    }
-
-    ret = qemu_rdma_source_init(rdma, &local_err,
-        s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL]);
-
-    if (ret) {
-        goto err;
-    }
-
-    DPRINTF("qemu_rdma_source_init success\n");
-    ret = qemu_rdma_connect(rdma, &local_err);
-
-    if (ret) {
-        goto err;
-    }
-
-    DPRINTF("qemu_rdma_source_connect success\n");
-
-    s->file = qemu_fopen_rdma(rdma, "wb");
-    migrate_fd_connect(s);
-    return;
-err:
-    error_propagate(errp, local_err);
-    g_free(rdma);
-    migrate_fd_error(s);
-}
diff --git a/migration-tcp.c b/migration-tcp.c
deleted file mode 100644
index 91c9cf3..0000000
--- a/migration-tcp.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * QEMU live migration
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include <string.h>
-
-#include "qemu-common.h"
-#include "qemu/error-report.h"
-#include "qemu/sockets.h"
-#include "migration/migration.h"
-#include "migration/qemu-file.h"
-#include "block/block.h"
-#include "qemu/main-loop.h"
-
-//#define DEBUG_MIGRATION_TCP
-
-#ifdef DEBUG_MIGRATION_TCP
-#define DPRINTF(fmt, ...) \
-    do { printf("migration-tcp: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-static void tcp_wait_for_connect(int fd, Error *err, void *opaque)
-{
-    MigrationState *s = opaque;
-
-    if (fd < 0) {
-        DPRINTF("migrate connect error: %s\n", error_get_pretty(err));
-        s->file = NULL;
-        migrate_fd_error(s);
-    } else {
-        DPRINTF("migrate connect success\n");
-        s->file = qemu_fopen_socket(fd, "wb");
-        migrate_fd_connect(s);
-    }
-}
-
-void tcp_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp)
-{
-    inet_nonblocking_connect(host_port, tcp_wait_for_connect, s, errp);
-}
-
-static void tcp_accept_incoming_migration(void *opaque)
-{
-    struct sockaddr_in addr;
-    socklen_t addrlen = sizeof(addr);
-    int s = (intptr_t)opaque;
-    QEMUFile *f;
-    int c, err;
-
-    do {
-        c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
-        err = socket_error();
-    } while (c < 0 && err == EINTR);
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    closesocket(s);
-
-    DPRINTF("accepted migration\n");
-
-    if (c < 0) {
-        error_report("could not accept migration connection (%s)",
-                     strerror(err));
-        return;
-    }
-
-    f = qemu_fopen_socket(c, "rb");
-    if (f == NULL) {
-        error_report("could not qemu_fopen socket");
-        goto out;
-    }
-
-    process_incoming_migration(f);
-    return;
-
-out:
-    closesocket(c);
-}
-
-void tcp_start_incoming_migration(const char *host_port, Error **errp)
-{
-    int s;
-
-    s = inet_listen(host_port, NULL, 256, SOCK_STREAM, 0, errp);
-    if (s < 0) {
-        return;
-    }
-
-    qemu_set_fd_handler2(s, NULL, tcp_accept_incoming_migration, NULL,
-                         (void *)(intptr_t)s);
-}
diff --git a/migration-unix.c b/migration-unix.c
deleted file mode 100644
index 1cdadfb..0000000
--- a/migration-unix.c
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * QEMU live migration via Unix Domain Sockets
- *
- * Copyright Red Hat, Inc. 2009
- *
- * Authors:
- *  Chris Lalancette <clalance@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include <string.h>
-
-#include "qemu-common.h"
-#include "qemu/error-report.h"
-#include "qemu/sockets.h"
-#include "qemu/main-loop.h"
-#include "migration/migration.h"
-#include "migration/qemu-file.h"
-#include "block/block.h"
-
-//#define DEBUG_MIGRATION_UNIX
-
-#ifdef DEBUG_MIGRATION_UNIX
-#define DPRINTF(fmt, ...) \
-    do { printf("migration-unix: " fmt, ## __VA_ARGS__); } while (0)
-#else
-#define DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
-
-static void unix_wait_for_connect(int fd, Error *err, void *opaque)
-{
-    MigrationState *s = opaque;
-
-    if (fd < 0) {
-        DPRINTF("migrate connect error: %s\n", error_get_pretty(err));
-        s->file = NULL;
-        migrate_fd_error(s);
-    } else {
-        DPRINTF("migrate connect success\n");
-        s->file = qemu_fopen_socket(fd, "wb");
-        migrate_fd_connect(s);
-    }
-}
-
-void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp)
-{
-    unix_nonblocking_connect(path, unix_wait_for_connect, s, errp);
-}
-
-static void unix_accept_incoming_migration(void *opaque)
-{
-    struct sockaddr_un addr;
-    socklen_t addrlen = sizeof(addr);
-    int s = (intptr_t)opaque;
-    QEMUFile *f;
-    int c, err;
-
-    do {
-        c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
-        err = errno;
-    } while (c < 0 && err == EINTR);
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
-
-    DPRINTF("accepted migration\n");
-
-    if (c < 0) {
-        error_report("could not accept migration connection (%s)",
-                     strerror(err));
-        return;
-    }
-
-    f = qemu_fopen_socket(c, "rb");
-    if (f == NULL) {
-        error_report("could not qemu_fopen socket");
-        goto out;
-    }
-
-    process_incoming_migration(f);
-    return;
-
-out:
-    close(c);
-}
-
-void unix_start_incoming_migration(const char *path, Error **errp)
-{
-    int s;
-
-    s = unix_listen(path, NULL, 0, errp);
-    if (s < 0) {
-        return;
-    }
-
-    qemu_set_fd_handler2(s, NULL, unix_accept_incoming_migration, NULL,
-                         (void *)(intptr_t)s);
-}
diff --git a/migration.c b/migration.c
deleted file mode 100644
index c49a05a..0000000
--- a/migration.c
+++ /dev/null
@@ -1,700 +0,0 @@
-/*
- * QEMU live migration
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "qemu/main-loop.h"
-#include "migration/migration.h"
-#include "monitor/monitor.h"
-#include "migration/qemu-file.h"
-#include "sysemu/sysemu.h"
-#include "block/block.h"
-#include "qemu/sockets.h"
-#include "migration/block.h"
-#include "qemu/thread.h"
-#include "qmp-commands.h"
-#include "trace.h"
-
-enum {
-    MIG_STATE_ERROR = -1,
-    MIG_STATE_NONE,
-    MIG_STATE_SETUP,
-    MIG_STATE_CANCELLING,
-    MIG_STATE_CANCELLED,
-    MIG_STATE_ACTIVE,
-    MIG_STATE_COMPLETED,
-};
-
-#define MAX_THROTTLE  (32 << 20)      /* Migration speed throttling */
-
-/* Amount of time to allocate to each "chunk" of bandwidth-throttled
- * data. */
-#define BUFFER_DELAY     100
-#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
-
-/* Migration XBZRLE default cache size */
-#define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
-
-static NotifierList migration_state_notifiers =
-    NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
-
-/* When we add fault tolerance, we could have several
-   migrations at once.  For now we don't need to add
-   dynamic creation of migration */
-
-MigrationState *migrate_get_current(void)
-{
-    static MigrationState current_migration = {
-        .state = MIG_STATE_NONE,
-        .bandwidth_limit = MAX_THROTTLE,
-        .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE,
-        .mbps = -1,
-    };
-
-    return &current_migration;
-}
-
-void qemu_start_incoming_migration(const char *uri, Error **errp)
-{
-    const char *p;
-
-    if (strstart(uri, "tcp:", &p))
-        tcp_start_incoming_migration(p, errp);
-#ifdef CONFIG_RDMA
-    else if (strstart(uri, "rdma:", &p))
-        rdma_start_incoming_migration(p, errp);
-#endif
-#if !defined(WIN32)
-    else if (strstart(uri, "exec:", &p))
-        exec_start_incoming_migration(p, errp);
-    else if (strstart(uri, "unix:", &p))
-        unix_start_incoming_migration(p, errp);
-    else if (strstart(uri, "fd:", &p))
-        fd_start_incoming_migration(p, errp);
-#endif
-    else {
-        error_setg(errp, "unknown migration protocol: %s", uri);
-    }
-}
-
-static void process_incoming_migration_co(void *opaque)
-{
-    QEMUFile *f = opaque;
-    Error *local_err = NULL;
-    int ret;
-
-    ret = qemu_loadvm_state(f);
-    qemu_fclose(f);
-    free_xbzrle_decoded_buf();
-    if (ret < 0) {
-        error_report("load of migration failed: %s", strerror(-ret));
-        exit(EXIT_FAILURE);
-    }
-    qemu_announce_self();
-
-    /* Make sure all file formats flush their mutable metadata */
-    bdrv_invalidate_cache_all(&local_err);
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        exit(EXIT_FAILURE);
-    }
-
-    if (autostart) {
-        vm_start();
-    } else {
-        runstate_set(RUN_STATE_PAUSED);
-    }
-}
-
-void process_incoming_migration(QEMUFile *f)
-{
-    Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
-    int fd = qemu_get_fd(f);
-
-    assert(fd != -1);
-    qemu_set_nonblock(fd);
-    qemu_coroutine_enter(co, f);
-}
-
-/* amount of nanoseconds we are willing to wait for migration to be down.
- * the choice of nanoseconds is because it is the maximum resolution that
- * get_clock() can achieve. It is an internal measure. All user-visible
- * units must be in seconds */
-static uint64_t max_downtime = 300000000;
-
-uint64_t migrate_max_downtime(void)
-{
-    return max_downtime;
-}
-
-MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
-{
-    MigrationCapabilityStatusList *head = NULL;
-    MigrationCapabilityStatusList *caps;
-    MigrationState *s = migrate_get_current();
-    int i;
-
-    caps = NULL; /* silence compiler warning */
-    for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
-        if (head == NULL) {
-            head = g_malloc0(sizeof(*caps));
-            caps = head;
-        } else {
-            caps->next = g_malloc0(sizeof(*caps));
-            caps = caps->next;
-        }
-        caps->value =
-            g_malloc(sizeof(*caps->value));
-        caps->value->capability = i;
-        caps->value->state = s->enabled_capabilities[i];
-    }
-
-    return head;
-}
-
-static void get_xbzrle_cache_stats(MigrationInfo *info)
-{
-    if (migrate_use_xbzrle()) {
-        info->has_xbzrle_cache = true;
-        info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
-        info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
-        info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred();
-        info->xbzrle_cache->pages = xbzrle_mig_pages_transferred();
-        info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss();
-        info->xbzrle_cache->cache_miss_rate = xbzrle_mig_cache_miss_rate();
-        info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow();
-    }
-}
-
-MigrationInfo *qmp_query_migrate(Error **errp)
-{
-    MigrationInfo *info = g_malloc0(sizeof(*info));
-    MigrationState *s = migrate_get_current();
-
-    switch (s->state) {
-    case MIG_STATE_NONE:
-        /* no migration has happened ever */
-        break;
-    case MIG_STATE_SETUP:
-        info->has_status = true;
-        info->status = g_strdup("setup");
-        info->has_total_time = false;
-        break;
-    case MIG_STATE_ACTIVE:
-    case MIG_STATE_CANCELLING:
-        info->has_status = true;
-        info->status = g_strdup("active");
-        info->has_total_time = true;
-        info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
-            - s->total_time;
-        info->has_expected_downtime = true;
-        info->expected_downtime = s->expected_downtime;
-        info->has_setup_time = true;
-        info->setup_time = s->setup_time;
-
-        info->has_ram = true;
-        info->ram = g_malloc0(sizeof(*info->ram));
-        info->ram->transferred = ram_bytes_transferred();
-        info->ram->remaining = ram_bytes_remaining();
-        info->ram->total = ram_bytes_total();
-        info->ram->duplicate = dup_mig_pages_transferred();
-        info->ram->skipped = skipped_mig_pages_transferred();
-        info->ram->normal = norm_mig_pages_transferred();
-        info->ram->normal_bytes = norm_mig_bytes_transferred();
-        info->ram->dirty_pages_rate = s->dirty_pages_rate;
-        info->ram->mbps = s->mbps;
-        info->ram->dirty_sync_count = s->dirty_sync_count;
-
-        if (blk_mig_active()) {
-            info->has_disk = true;
-            info->disk = g_malloc0(sizeof(*info->disk));
-            info->disk->transferred = blk_mig_bytes_transferred();
-            info->disk->remaining = blk_mig_bytes_remaining();
-            info->disk->total = blk_mig_bytes_total();
-        }
-
-        get_xbzrle_cache_stats(info);
-        break;
-    case MIG_STATE_COMPLETED:
-        get_xbzrle_cache_stats(info);
-
-        info->has_status = true;
-        info->status = g_strdup("completed");
-        info->has_total_time = true;
-        info->total_time = s->total_time;
-        info->has_downtime = true;
-        info->downtime = s->downtime;
-        info->has_setup_time = true;
-        info->setup_time = s->setup_time;
-
-        info->has_ram = true;
-        info->ram = g_malloc0(sizeof(*info->ram));
-        info->ram->transferred = ram_bytes_transferred();
-        info->ram->remaining = 0;
-        info->ram->total = ram_bytes_total();
-        info->ram->duplicate = dup_mig_pages_transferred();
-        info->ram->skipped = skipped_mig_pages_transferred();
-        info->ram->normal = norm_mig_pages_transferred();
-        info->ram->normal_bytes = norm_mig_bytes_transferred();
-        info->ram->mbps = s->mbps;
-        info->ram->dirty_sync_count = s->dirty_sync_count;
-        break;
-    case MIG_STATE_ERROR:
-        info->has_status = true;
-        info->status = g_strdup("failed");
-        break;
-    case MIG_STATE_CANCELLED:
-        info->has_status = true;
-        info->status = g_strdup("cancelled");
-        break;
-    }
-
-    return info;
-}
-
-void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
-                                  Error **errp)
-{
-    MigrationState *s = migrate_get_current();
-    MigrationCapabilityStatusList *cap;
-
-    if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP) {
-        error_set(errp, QERR_MIGRATION_ACTIVE);
-        return;
-    }
-
-    for (cap = params; cap; cap = cap->next) {
-        s->enabled_capabilities[cap->value->capability] = cap->value->state;
-    }
-}
-
-/* shared migration helpers */
-
-static void migrate_set_state(MigrationState *s, int old_state, int new_state)
-{
-    if (atomic_cmpxchg(&s->state, old_state, new_state) == new_state) {
-        trace_migrate_set_state(new_state);
-    }
-}
-
-static void migrate_fd_cleanup(void *opaque)
-{
-    MigrationState *s = opaque;
-
-    qemu_bh_delete(s->cleanup_bh);
-    s->cleanup_bh = NULL;
-
-    if (s->file) {
-        trace_migrate_fd_cleanup();
-        qemu_mutex_unlock_iothread();
-        qemu_thread_join(&s->thread);
-        qemu_mutex_lock_iothread();
-
-        qemu_fclose(s->file);
-        s->file = NULL;
-    }
-
-    assert(s->state != MIG_STATE_ACTIVE);
-
-    if (s->state != MIG_STATE_COMPLETED) {
-        qemu_savevm_state_cancel();
-        if (s->state == MIG_STATE_CANCELLING) {
-            migrate_set_state(s, MIG_STATE_CANCELLING, MIG_STATE_CANCELLED);
-        }
-    }
-
-    notifier_list_notify(&migration_state_notifiers, s);
-}
-
-void migrate_fd_error(MigrationState *s)
-{
-    trace_migrate_fd_error();
-    assert(s->file == NULL);
-    s->state = MIG_STATE_ERROR;
-    trace_migrate_set_state(MIG_STATE_ERROR);
-    notifier_list_notify(&migration_state_notifiers, s);
-}
-
-static void migrate_fd_cancel(MigrationState *s)
-{
-    int old_state ;
-    trace_migrate_fd_cancel();
-
-    do {
-        old_state = s->state;
-        if (old_state != MIG_STATE_SETUP && old_state != MIG_STATE_ACTIVE) {
-            break;
-        }
-        migrate_set_state(s, old_state, MIG_STATE_CANCELLING);
-    } while (s->state != MIG_STATE_CANCELLING);
-}
-
-void add_migration_state_change_notifier(Notifier *notify)
-{
-    notifier_list_add(&migration_state_notifiers, notify);
-}
-
-void remove_migration_state_change_notifier(Notifier *notify)
-{
-    notifier_remove(notify);
-}
-
-bool migration_in_setup(MigrationState *s)
-{
-    return s->state == MIG_STATE_SETUP;
-}
-
-bool migration_has_finished(MigrationState *s)
-{
-    return s->state == MIG_STATE_COMPLETED;
-}
-
-bool migration_has_failed(MigrationState *s)
-{
-    return (s->state == MIG_STATE_CANCELLED ||
-            s->state == MIG_STATE_ERROR);
-}
-
-static MigrationState *migrate_init(const MigrationParams *params)
-{
-    MigrationState *s = migrate_get_current();
-    int64_t bandwidth_limit = s->bandwidth_limit;
-    bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
-    int64_t xbzrle_cache_size = s->xbzrle_cache_size;
-
-    memcpy(enabled_capabilities, s->enabled_capabilities,
-           sizeof(enabled_capabilities));
-
-    memset(s, 0, sizeof(*s));
-    s->params = *params;
-    memcpy(s->enabled_capabilities, enabled_capabilities,
-           sizeof(enabled_capabilities));
-    s->xbzrle_cache_size = xbzrle_cache_size;
-
-    s->bandwidth_limit = bandwidth_limit;
-    s->state = MIG_STATE_SETUP;
-    trace_migrate_set_state(MIG_STATE_SETUP);
-
-    s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-    return s;
-}
-
-static GSList *migration_blockers;
-
-void migrate_add_blocker(Error *reason)
-{
-    migration_blockers = g_slist_prepend(migration_blockers, reason);
-}
-
-void migrate_del_blocker(Error *reason)
-{
-    migration_blockers = g_slist_remove(migration_blockers, reason);
-}
-
-void qmp_migrate(const char *uri, bool has_blk, bool blk,
-                 bool has_inc, bool inc, bool has_detach, bool detach,
-                 Error **errp)
-{
-    Error *local_err = NULL;
-    MigrationState *s = migrate_get_current();
-    MigrationParams params;
-    const char *p;
-
-    params.blk = has_blk && blk;
-    params.shared = has_inc && inc;
-
-    if (s->state == MIG_STATE_ACTIVE || s->state == MIG_STATE_SETUP ||
-        s->state == MIG_STATE_CANCELLING) {
-        error_set(errp, QERR_MIGRATION_ACTIVE);
-        return;
-    }
-
-    if (runstate_check(RUN_STATE_INMIGRATE)) {
-        error_setg(errp, "Guest is waiting for an incoming migration");
-        return;
-    }
-
-    if (qemu_savevm_state_blocked(errp)) {
-        return;
-    }
-
-    if (migration_blockers) {
-        *errp = error_copy(migration_blockers->data);
-        return;
-    }
-
-    s = migrate_init(&params);
-
-    if (strstart(uri, "tcp:", &p)) {
-        tcp_start_outgoing_migration(s, p, &local_err);
-#ifdef CONFIG_RDMA
-    } else if (strstart(uri, "rdma:", &p)) {
-        rdma_start_outgoing_migration(s, p, &local_err);
-#endif
-#if !defined(WIN32)
-    } else if (strstart(uri, "exec:", &p)) {
-        exec_start_outgoing_migration(s, p, &local_err);
-    } else if (strstart(uri, "unix:", &p)) {
-        unix_start_outgoing_migration(s, p, &local_err);
-    } else if (strstart(uri, "fd:", &p)) {
-        fd_start_outgoing_migration(s, p, &local_err);
-#endif
-    } else {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "uri", "a valid migration protocol");
-        s->state = MIG_STATE_ERROR;
-        return;
-    }
-
-    if (local_err) {
-        migrate_fd_error(s);
-        error_propagate(errp, local_err);
-        return;
-    }
-}
-
-void qmp_migrate_cancel(Error **errp)
-{
-    migrate_fd_cancel(migrate_get_current());
-}
-
-void qmp_migrate_set_cache_size(int64_t value, Error **errp)
-{
-    MigrationState *s = migrate_get_current();
-    int64_t new_size;
-
-    /* Check for truncation */
-    if (value != (size_t)value) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
-                  "exceeding address space");
-        return;
-    }
-
-    /* Cache should not be larger than guest ram size */
-    if (value > ram_bytes_total()) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
-                  "exceeds guest ram size ");
-        return;
-    }
-
-    new_size = xbzrle_cache_resize(value);
-    if (new_size < 0) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
-                  "is smaller than page size");
-        return;
-    }
-
-    s->xbzrle_cache_size = new_size;
-}
-
-int64_t qmp_query_migrate_cache_size(Error **errp)
-{
-    return migrate_xbzrle_cache_size();
-}
-
-void qmp_migrate_set_speed(int64_t value, Error **errp)
-{
-    MigrationState *s;
-
-    if (value < 0) {
-        value = 0;
-    }
-    if (value > SIZE_MAX) {
-        value = SIZE_MAX;
-    }
-
-    s = migrate_get_current();
-    s->bandwidth_limit = value;
-    if (s->file) {
-        qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO);
-    }
-}
-
-void qmp_migrate_set_downtime(double value, Error **errp)
-{
-    value *= 1e9;
-    value = MAX(0, MIN(UINT64_MAX, value));
-    max_downtime = (uint64_t)value;
-}
-
-bool migrate_rdma_pin_all(void)
-{
-    MigrationState *s;
-
-    s = migrate_get_current();
-
-    return s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL];
-}
-
-bool migrate_auto_converge(void)
-{
-    MigrationState *s;
-
-    s = migrate_get_current();
-
-    return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
-}
-
-bool migrate_zero_blocks(void)
-{
-    MigrationState *s;
-
-    s = migrate_get_current();
-
-    return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
-}
-
-int migrate_use_xbzrle(void)
-{
-    MigrationState *s;
-
-    s = migrate_get_current();
-
-    return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
-}
-
-int64_t migrate_xbzrle_cache_size(void)
-{
-    MigrationState *s;
-
-    s = migrate_get_current();
-
-    return s->xbzrle_cache_size;
-}
-
-/* migration thread support */
-
-static void *migration_thread(void *opaque)
-{
-    MigrationState *s = opaque;
-    int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-    int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
-    int64_t initial_bytes = 0;
-    int64_t max_size = 0;
-    int64_t start_time = initial_time;
-    bool old_vm_running = false;
-
-    qemu_savevm_state_begin(s->file, &s->params);
-
-    s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
-    migrate_set_state(s, MIG_STATE_SETUP, MIG_STATE_ACTIVE);
-
-    while (s->state == MIG_STATE_ACTIVE) {
-        int64_t current_time;
-        uint64_t pending_size;
-
-        if (!qemu_file_rate_limit(s->file)) {
-            pending_size = qemu_savevm_state_pending(s->file, max_size);
-            trace_migrate_pending(pending_size, max_size);
-            if (pending_size && pending_size >= max_size) {
-                qemu_savevm_state_iterate(s->file);
-            } else {
-                int ret;
-
-                qemu_mutex_lock_iothread();
-                start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-                qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
-                old_vm_running = runstate_is_running();
-
-                ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
-                if (ret >= 0) {
-                    qemu_file_set_rate_limit(s->file, INT64_MAX);
-                    qemu_savevm_state_complete(s->file);
-                }
-                qemu_mutex_unlock_iothread();
-
-                if (ret < 0) {
-                    migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_ERROR);
-                    break;
-                }
-
-                if (!qemu_file_get_error(s->file)) {
-                    migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_COMPLETED);
-                    break;
-                }
-            }
-        }
-
-        if (qemu_file_get_error(s->file)) {
-            migrate_set_state(s, MIG_STATE_ACTIVE, MIG_STATE_ERROR);
-            break;
-        }
-        current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-        if (current_time >= initial_time + BUFFER_DELAY) {
-            uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes;
-            uint64_t time_spent = current_time - initial_time;
-            double bandwidth = transferred_bytes / time_spent;
-            max_size = bandwidth * migrate_max_downtime() / 1000000;
-
-            s->mbps = time_spent ? (((double) transferred_bytes * 8.0) /
-                    ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1;
-
-            trace_migrate_transferred(transferred_bytes, time_spent,
-                                      bandwidth, max_size);
-            /* if we haven't sent anything, we don't want to recalculate
-               10000 is a small enough number for our purposes */
-            if (s->dirty_bytes_rate && transferred_bytes > 10000) {
-                s->expected_downtime = s->dirty_bytes_rate / bandwidth;
-            }
-
-            qemu_file_reset_rate_limit(s->file);
-            initial_time = current_time;
-            initial_bytes = qemu_ftell(s->file);
-        }
-        if (qemu_file_rate_limit(s->file)) {
-            /* usleep expects microseconds */
-            g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
-        }
-    }
-
-    qemu_mutex_lock_iothread();
-    if (s->state == MIG_STATE_COMPLETED) {
-        int64_t end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
-        uint64_t transferred_bytes = qemu_ftell(s->file);
-        s->total_time = end_time - s->total_time;
-        s->downtime = end_time - start_time;
-        if (s->total_time) {
-            s->mbps = (((double) transferred_bytes * 8.0) /
-                       ((double) s->total_time)) / 1000;
-        }
-        runstate_set(RUN_STATE_POSTMIGRATE);
-    } else {
-        if (old_vm_running) {
-            vm_start();
-        }
-    }
-    qemu_bh_schedule(s->cleanup_bh);
-    qemu_mutex_unlock_iothread();
-
-    return NULL;
-}
-
-void migrate_fd_connect(MigrationState *s)
-{
-    s->state = MIG_STATE_SETUP;
-    trace_migrate_set_state(MIG_STATE_SETUP);
-
-    /* This is a best 1st approximation. ns to ms */
-    s->expected_downtime = max_downtime/1000000;
-    s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s);
-
-    qemu_file_set_rate_limit(s->file,
-                             s->bandwidth_limit / XFER_LIMIT_RATIO);
-
-    /* Notify before starting migration thread */
-    notifier_list_notify(&migration_state_notifiers, s);
-
-    qemu_thread_create(&s->thread, "migration", migration_thread, s,
-                       QEMU_THREAD_JOINABLE);
-}
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
new file mode 100644
index 0000000..30ad945
--- /dev/null
+++ b/migration/Makefile.objs
@@ -0,0 +1,12 @@
+common-obj-y += migration.o socket.o fd.o exec.o
+common-obj-y += tls.o
+common-obj-y += vmstate.o
+common-obj-y += qemu-file.o
+common-obj-y += qemu-file-channel.o
+common-obj-y += xbzrle.o postcopy-ram.o
+common-obj-y += qjson.o
+
+common-obj-$(CONFIG_RDMA) += rdma.o
+
+common-obj-y += block.o
+
diff --git a/migration/block.c b/migration/block.c
new file mode 100644
index 0000000..ebc10e6
--- /dev/null
+++ b/migration/block.c
@@ -0,0 +1,981 @@
+/*
+ * QEMU live block migration
+ *
+ * Copyright IBM, Corp. 2009
+ *
+ * Authors:
+ *  Liran Schour   <lirans@il.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "block/block.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+#include "hw/hw.h"
+#include "qemu/cutils.h"
+#include "qemu/queue.h"
+#include "qemu/timer.h"
+#include "migration/block.h"
+#include "migration/migration.h"
+#include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
+
+#define BLOCK_SIZE                       (1 << 20)
+#define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
+
+#define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
+#define BLK_MIG_FLAG_EOS                0x02
+#define BLK_MIG_FLAG_PROGRESS           0x04
+#define BLK_MIG_FLAG_ZERO_BLOCK         0x08
+
+#define MAX_IS_ALLOCATED_SEARCH 65536
+
+#define MAX_INFLIGHT_IO 512
+
+//#define DEBUG_BLK_MIGRATION
+
+#ifdef DEBUG_BLK_MIGRATION
+#define DPRINTF(fmt, ...) \
+    do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+typedef struct BlkMigDevState {
+    /* Written during setup phase.  Can be read without a lock.  */
+    BlockBackend *blk;
+    char *blk_name;
+    int shared_base;
+    int64_t total_sectors;
+    QSIMPLEQ_ENTRY(BlkMigDevState) entry;
+    Error *blocker;
+
+    /* Only used by migration thread.  Does not need a lock.  */
+    int bulk_completed;
+    int64_t cur_sector;
+    int64_t cur_dirty;
+
+    /* Data in the aio_bitmap is protected by block migration lock.
+     * Allocation and free happen during setup and cleanup respectively.
+     */
+    unsigned long *aio_bitmap;
+
+    /* Protected by block migration lock.  */
+    int64_t completed_sectors;
+
+    /* During migration this is protected by iothread lock / AioContext.
+     * Allocation and free happen during setup and cleanup respectively.
+     */
+    BdrvDirtyBitmap *dirty_bitmap;
+} BlkMigDevState;
+
+typedef struct BlkMigBlock {
+    /* Only used by migration thread.  */
+    uint8_t *buf;
+    BlkMigDevState *bmds;
+    int64_t sector;
+    int nr_sectors;
+    struct iovec iov;
+    QEMUIOVector qiov;
+    BlockAIOCB *aiocb;
+
+    /* Protected by block migration lock.  */
+    int ret;
+    QSIMPLEQ_ENTRY(BlkMigBlock) entry;
+} BlkMigBlock;
+
+typedef struct BlkMigState {
+    /* Written during setup phase.  Can be read without a lock.  */
+    int blk_enable;
+    int shared_base;
+    QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
+    int64_t total_sector_sum;
+    bool zero_blocks;
+
+    /* Protected by lock.  */
+    QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
+    int submitted;
+    int read_done;
+
+    /* Only used by migration thread.  Does not need a lock.  */
+    int transferred;
+    int prev_progress;
+    int bulk_completed;
+
+    /* Lock must be taken _inside_ the iothread lock and any AioContexts.  */
+    QemuMutex lock;
+} BlkMigState;
+
+static BlkMigState block_mig_state;
+
+static void blk_mig_lock(void)
+{
+    qemu_mutex_lock(&block_mig_state.lock);
+}
+
+static void blk_mig_unlock(void)
+{
+    qemu_mutex_unlock(&block_mig_state.lock);
+}
+
+/* Must run outside of the iothread lock during the bulk phase,
+ * or the VM will stall.
+ */
+
+static void blk_send(QEMUFile *f, BlkMigBlock * blk)
+{
+    int len;
+    uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;
+
+    if (block_mig_state.zero_blocks &&
+        buffer_is_zero(blk->buf, BLOCK_SIZE)) {
+        flags |= BLK_MIG_FLAG_ZERO_BLOCK;
+    }
+
+    /* sector number and flags */
+    qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
+                     | flags);
+
+    /* device name */
+    len = strlen(blk->bmds->blk_name);
+    qemu_put_byte(f, len);
+    qemu_put_buffer(f, (uint8_t *) blk->bmds->blk_name, len);
+
+    /* if a block is zero we need to flush here since the network
+     * bandwidth is now a lot higher than the storage device bandwidth.
+     * thus if we queue zero blocks we slow down the migration */
+    if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
+        qemu_fflush(f);
+        return;
+    }
+
+    qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
+}
+
+int blk_mig_active(void)
+{
+    return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
+}
+
+uint64_t blk_mig_bytes_transferred(void)
+{
+    BlkMigDevState *bmds;
+    uint64_t sum = 0;
+
+    blk_mig_lock();
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        sum += bmds->completed_sectors;
+    }
+    blk_mig_unlock();
+    return sum << BDRV_SECTOR_BITS;
+}
+
+uint64_t blk_mig_bytes_remaining(void)
+{
+    return blk_mig_bytes_total() - blk_mig_bytes_transferred();
+}
+
+uint64_t blk_mig_bytes_total(void)
+{
+    BlkMigDevState *bmds;
+    uint64_t sum = 0;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        sum += bmds->total_sectors;
+    }
+    return sum << BDRV_SECTOR_BITS;
+}
+
+
+/* Called with migration lock held.  */
+
+static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
+{
+    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    if (sector < blk_nb_sectors(bmds->blk)) {
+        return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
+            (1UL << (chunk % (sizeof(unsigned long) * 8))));
+    } else {
+        return 0;
+    }
+}
+
+/* Called with migration lock held.  */
+
+static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
+                             int nb_sectors, int set)
+{
+    int64_t start, end;
+    unsigned long val, idx, bit;
+
+    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
+    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    for (; start <= end; start++) {
+        idx = start / (sizeof(unsigned long) * 8);
+        bit = start % (sizeof(unsigned long) * 8);
+        val = bmds->aio_bitmap[idx];
+        if (set) {
+            val |= 1UL << bit;
+        } else {
+            val &= ~(1UL << bit);
+        }
+        bmds->aio_bitmap[idx] = val;
+    }
+}
+
+static void alloc_aio_bitmap(BlkMigDevState *bmds)
+{
+    BlockBackend *bb = bmds->blk;
+    int64_t bitmap_size;
+
+    bitmap_size = blk_nb_sectors(bb) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
+    bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;
+
+    bmds->aio_bitmap = g_malloc0(bitmap_size);
+}
+
+/* Never hold migration lock when yielding to the main loop!  */
+
+static void blk_mig_read_cb(void *opaque, int ret)
+{
+    BlkMigBlock *blk = opaque;
+
+    blk_mig_lock();
+    blk->ret = ret;
+
+    QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
+    bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
+
+    block_mig_state.submitted--;
+    block_mig_state.read_done++;
+    assert(block_mig_state.submitted >= 0);
+    blk_mig_unlock();
+}
+
+/* Called with no lock taken.  */
+
+static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
+{
+    int64_t total_sectors = bmds->total_sectors;
+    int64_t cur_sector = bmds->cur_sector;
+    BlockBackend *bb = bmds->blk;
+    BlkMigBlock *blk;
+    int nr_sectors;
+
+    if (bmds->shared_base) {
+        qemu_mutex_lock_iothread();
+        aio_context_acquire(blk_get_aio_context(bb));
+        while (cur_sector < total_sectors &&
+               !bdrv_is_allocated(blk_bs(bb), cur_sector,
+                                  MAX_IS_ALLOCATED_SEARCH, &nr_sectors)) {
+            cur_sector += nr_sectors;
+        }
+        aio_context_release(blk_get_aio_context(bb));
+        qemu_mutex_unlock_iothread();
+    }
+
+    if (cur_sector >= total_sectors) {
+        bmds->cur_sector = bmds->completed_sectors = total_sectors;
+        return 1;
+    }
+
+    bmds->completed_sectors = cur_sector;
+
+    cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);
+
+    /* we are going to transfer a full block even if it is not allocated */
+    nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
+        nr_sectors = total_sectors - cur_sector;
+    }
+
+    blk = g_new(BlkMigBlock, 1);
+    blk->buf = g_malloc(BLOCK_SIZE);
+    blk->bmds = bmds;
+    blk->sector = cur_sector;
+    blk->nr_sectors = nr_sectors;
+
+    blk->iov.iov_base = blk->buf;
+    blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
+    qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
+
+    blk_mig_lock();
+    block_mig_state.submitted++;
+    blk_mig_unlock();
+
+    /* We do not know if bs is under the main thread (and thus does
+     * not acquire the AioContext when doing AIO) or rather under
+     * dataplane.  Thus acquire both the iothread mutex and the
+     * AioContext.
+     *
+     * This is ugly and will disappear when we make bdrv_* thread-safe,
+     * without the need to acquire the AioContext.
+     */
+    qemu_mutex_lock_iothread();
+    aio_context_acquire(blk_get_aio_context(bmds->blk));
+    blk->aiocb = blk_aio_preadv(bb, cur_sector * BDRV_SECTOR_SIZE, &blk->qiov,
+                                0, blk_mig_read_cb, blk);
+
+    bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector, nr_sectors);
+    aio_context_release(blk_get_aio_context(bmds->blk));
+    qemu_mutex_unlock_iothread();
+
+    bmds->cur_sector = cur_sector + nr_sectors;
+    return (bmds->cur_sector >= total_sectors);
+}
+
+/* Called with iothread lock taken.  */
+
+static int set_dirty_tracking(void)
+{
+    BlkMigDevState *bmds;
+    int ret;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(blk_get_aio_context(bmds->blk));
+        bmds->dirty_bitmap = bdrv_create_dirty_bitmap(blk_bs(bmds->blk),
+                                                      BLOCK_SIZE, NULL, NULL);
+        aio_context_release(blk_get_aio_context(bmds->blk));
+        if (!bmds->dirty_bitmap) {
+            ret = -errno;
+            goto fail;
+        }
+    }
+    return 0;
+
+fail:
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        if (bmds->dirty_bitmap) {
+            aio_context_acquire(blk_get_aio_context(bmds->blk));
+            bdrv_release_dirty_bitmap(blk_bs(bmds->blk), bmds->dirty_bitmap);
+            aio_context_release(blk_get_aio_context(bmds->blk));
+        }
+    }
+    return ret;
+}
+
+/* Called with iothread lock taken.  */
+
+static void unset_dirty_tracking(void)
+{
+    BlkMigDevState *bmds;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(blk_get_aio_context(bmds->blk));
+        bdrv_release_dirty_bitmap(blk_bs(bmds->blk), bmds->dirty_bitmap);
+        aio_context_release(blk_get_aio_context(bmds->blk));
+    }
+}
+
+static void init_blk_migration(QEMUFile *f)
+{
+    BlockDriverState *bs;
+    BlkMigDevState *bmds;
+    int64_t sectors;
+    BdrvNextIterator it;
+    int i, num_bs = 0;
+    struct {
+        BlkMigDevState *bmds;
+        BlockDriverState *bs;
+    } *bmds_bs;
+
+    block_mig_state.submitted = 0;
+    block_mig_state.read_done = 0;
+    block_mig_state.transferred = 0;
+    block_mig_state.total_sector_sum = 0;
+    block_mig_state.prev_progress = -1;
+    block_mig_state.bulk_completed = 0;
+    block_mig_state.zero_blocks = migrate_zero_blocks();
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        num_bs++;
+    }
+    bmds_bs = g_malloc0(num_bs * sizeof(*bmds_bs));
+
+    for (i = 0, bs = bdrv_first(&it); bs; bs = bdrv_next(&it), i++) {
+        if (bdrv_is_read_only(bs)) {
+            continue;
+        }
+
+        sectors = bdrv_nb_sectors(bs);
+        if (sectors <= 0) {
+            goto out;
+        }
+
+        bmds = g_new0(BlkMigDevState, 1);
+        bmds->blk = blk_new();
+        bmds->blk_name = g_strdup(bdrv_get_device_name(bs));
+        bmds->bulk_completed = 0;
+        bmds->total_sectors = sectors;
+        bmds->completed_sectors = 0;
+        bmds->shared_base = block_mig_state.shared_base;
+
+        assert(i < num_bs);
+        bmds_bs[i].bmds = bmds;
+        bmds_bs[i].bs = bs;
+
+        block_mig_state.total_sector_sum += sectors;
+
+        if (bmds->shared_base) {
+            DPRINTF("Start migration for %s with shared base image\n",
+                    bdrv_get_device_name(bs));
+        } else {
+            DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
+        }
+
+        QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
+    }
+
+    /* Can only insert new BDSes now because doing so while iterating block
+     * devices may end up in a deadlock (iterating the new BDSes, too). */
+    for (i = 0; i < num_bs; i++) {
+        BlkMigDevState *bmds = bmds_bs[i].bmds;
+        BlockDriverState *bs = bmds_bs[i].bs;
+
+        if (bmds) {
+            blk_insert_bs(bmds->blk, bs);
+
+            alloc_aio_bitmap(bmds);
+            error_setg(&bmds->blocker, "block device is in use by migration");
+            bdrv_op_block_all(bs, bmds->blocker);
+        }
+    }
+
+out:
+    g_free(bmds_bs);
+}
+
+/* Called with no lock taken.  */
+
+static int blk_mig_save_bulked_block(QEMUFile *f)
+{
+    int64_t completed_sector_sum = 0;
+    BlkMigDevState *bmds;
+    int progress;
+    int ret = 0;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        if (bmds->bulk_completed == 0) {
+            if (mig_save_device_bulk(f, bmds) == 1) {
+                /* completed bulk section for this device */
+                bmds->bulk_completed = 1;
+            }
+            completed_sector_sum += bmds->completed_sectors;
+            ret = 1;
+            break;
+        } else {
+            completed_sector_sum += bmds->completed_sectors;
+        }
+    }
+
+    if (block_mig_state.total_sector_sum != 0) {
+        progress = completed_sector_sum * 100 /
+                   block_mig_state.total_sector_sum;
+    } else {
+        progress = 100;
+    }
+    if (progress != block_mig_state.prev_progress) {
+        block_mig_state.prev_progress = progress;
+        qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
+                         | BLK_MIG_FLAG_PROGRESS);
+        DPRINTF("Completed %d %%\r", progress);
+    }
+
+    return ret;
+}
+
+static void blk_mig_reset_dirty_cursor(void)
+{
+    BlkMigDevState *bmds;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        bmds->cur_dirty = 0;
+    }
+}
+
+/* Called with iothread lock and AioContext taken.  */
+
+static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
+                                 int is_async)
+{
+    BlkMigBlock *blk;
+    BlockDriverState *bs = blk_bs(bmds->blk);
+    int64_t total_sectors = bmds->total_sectors;
+    int64_t sector;
+    int nr_sectors;
+    int ret = -EIO;
+
+    for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
+        blk_mig_lock();
+        if (bmds_aio_inflight(bmds, sector)) {
+            blk_mig_unlock();
+            blk_drain(bmds->blk);
+        } else {
+            blk_mig_unlock();
+        }
+        if (bdrv_get_dirty(bs, bmds->dirty_bitmap, sector)) {
+
+            if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
+                nr_sectors = total_sectors - sector;
+            } else {
+                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
+            }
+            blk = g_new(BlkMigBlock, 1);
+            blk->buf = g_malloc(BLOCK_SIZE);
+            blk->bmds = bmds;
+            blk->sector = sector;
+            blk->nr_sectors = nr_sectors;
+
+            if (is_async) {
+                blk->iov.iov_base = blk->buf;
+                blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
+                qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
+
+                blk->aiocb = blk_aio_preadv(bmds->blk,
+                                            sector * BDRV_SECTOR_SIZE,
+                                            &blk->qiov, 0, blk_mig_read_cb,
+                                            blk);
+
+                blk_mig_lock();
+                block_mig_state.submitted++;
+                bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
+                blk_mig_unlock();
+            } else {
+                ret = blk_pread(bmds->blk, sector * BDRV_SECTOR_SIZE, blk->buf,
+                                nr_sectors * BDRV_SECTOR_SIZE);
+                if (ret < 0) {
+                    goto error;
+                }
+                blk_send(f, blk);
+
+                g_free(blk->buf);
+                g_free(blk);
+            }
+
+            bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, sector, nr_sectors);
+            break;
+        }
+        sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
+        bmds->cur_dirty = sector;
+    }
+
+    return (bmds->cur_dirty >= bmds->total_sectors);
+
+error:
+    DPRINTF("Error reading sector %" PRId64 "\n", sector);
+    g_free(blk->buf);
+    g_free(blk);
+    return ret;
+}
+
+/* Called with iothread lock taken.
+ *
+ * return value:
+ * 0: too much data for max_downtime
+ * 1: few enough data for max_downtime
+*/
+static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
+{
+    BlkMigDevState *bmds;
+    int ret = 1;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(blk_get_aio_context(bmds->blk));
+        ret = mig_save_device_dirty(f, bmds, is_async);
+        aio_context_release(blk_get_aio_context(bmds->blk));
+        if (ret <= 0) {
+            break;
+        }
+    }
+
+    return ret;
+}
+
+/* Called with no locks taken.  */
+
+static int flush_blks(QEMUFile *f)
+{
+    BlkMigBlock *blk;
+    int ret = 0;
+
+    DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
+            __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
+            block_mig_state.transferred);
+
+    blk_mig_lock();
+    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
+        if (qemu_file_rate_limit(f)) {
+            break;
+        }
+        if (blk->ret < 0) {
+            ret = blk->ret;
+            break;
+        }
+
+        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
+        blk_mig_unlock();
+        blk_send(f, blk);
+        blk_mig_lock();
+
+        g_free(blk->buf);
+        g_free(blk);
+
+        block_mig_state.read_done--;
+        block_mig_state.transferred++;
+        assert(block_mig_state.read_done >= 0);
+    }
+    blk_mig_unlock();
+
+    DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
+            block_mig_state.submitted, block_mig_state.read_done,
+            block_mig_state.transferred);
+    return ret;
+}
+
+/* Called with iothread lock taken.  */
+
+static int64_t get_remaining_dirty(void)
+{
+    BlkMigDevState *bmds;
+    int64_t dirty = 0;
+
+    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
+        aio_context_acquire(blk_get_aio_context(bmds->blk));
+        dirty += bdrv_get_dirty_count(bmds->dirty_bitmap);
+        aio_context_release(blk_get_aio_context(bmds->blk));
+    }
+
+    return dirty << BDRV_SECTOR_BITS;
+}
+
+/* Called with iothread lock taken.  */
+
+static void block_migration_cleanup(void *opaque)
+{
+    BlkMigDevState *bmds;
+    BlkMigBlock *blk;
+    AioContext *ctx;
+
+    bdrv_drain_all();
+
+    unset_dirty_tracking();
+
+    while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
+        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
+        bdrv_op_unblock_all(blk_bs(bmds->blk), bmds->blocker);
+        error_free(bmds->blocker);
+
+        /* Save ctx, because bmds->blk can disappear during blk_unref.  */
+        ctx = blk_get_aio_context(bmds->blk);
+        aio_context_acquire(ctx);
+        blk_unref(bmds->blk);
+        aio_context_release(ctx);
+
+        g_free(bmds->blk_name);
+        g_free(bmds->aio_bitmap);
+        g_free(bmds);
+    }
+
+    blk_mig_lock();
+    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
+        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
+        g_free(blk->buf);
+        g_free(blk);
+    }
+    blk_mig_unlock();
+}
+
+static int block_save_setup(QEMUFile *f, void *opaque)
+{
+    int ret;
+
+    DPRINTF("Enter save live setup submitted %d transferred %d\n",
+            block_mig_state.submitted, block_mig_state.transferred);
+
+    qemu_mutex_lock_iothread();
+    init_blk_migration(f);
+
+    /* start track dirty blocks */
+    ret = set_dirty_tracking();
+
+    qemu_mutex_unlock_iothread();
+
+    if (ret) {
+        return ret;
+    }
+
+    ret = flush_blks(f);
+    blk_mig_reset_dirty_cursor();
+    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
+
+    return ret;
+}
+
+static int block_save_iterate(QEMUFile *f, void *opaque)
+{
+    int ret;
+    int64_t last_ftell = qemu_ftell(f);
+    int64_t delta_ftell;
+
+    DPRINTF("Enter save live iterate submitted %d transferred %d\n",
+            block_mig_state.submitted, block_mig_state.transferred);
+
+    ret = flush_blks(f);
+    if (ret) {
+        return ret;
+    }
+
+    blk_mig_reset_dirty_cursor();
+
+    /* control the rate of transfer */
+    blk_mig_lock();
+    while ((block_mig_state.submitted +
+            block_mig_state.read_done) * BLOCK_SIZE <
+           qemu_file_get_rate_limit(f) &&
+           (block_mig_state.submitted +
+            block_mig_state.read_done) <
+           MAX_INFLIGHT_IO) {
+        blk_mig_unlock();
+        if (block_mig_state.bulk_completed == 0) {
+            /* first finish the bulk phase */
+            if (blk_mig_save_bulked_block(f) == 0) {
+                /* finished saving bulk on all devices */
+                block_mig_state.bulk_completed = 1;
+            }
+            ret = 0;
+        } else {
+            /* Always called with iothread lock taken for
+             * simplicity, block_save_complete also calls it.
+             */
+            qemu_mutex_lock_iothread();
+            ret = blk_mig_save_dirty_block(f, 1);
+            qemu_mutex_unlock_iothread();
+        }
+        if (ret < 0) {
+            return ret;
+        }
+        blk_mig_lock();
+        if (ret != 0) {
+            /* no more dirty blocks */
+            break;
+        }
+    }
+    blk_mig_unlock();
+
+    ret = flush_blks(f);
+    if (ret) {
+        return ret;
+    }
+
+    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
+    delta_ftell = qemu_ftell(f) - last_ftell;
+    if (delta_ftell > 0) {
+        return 1;
+    } else if (delta_ftell < 0) {
+        return -1;
+    } else {
+        return 0;
+    }
+}
+
+/* Called with iothread lock taken.  */
+
+static int block_save_complete(QEMUFile *f, void *opaque)
+{
+    int ret;
+
+    DPRINTF("Enter save live complete submitted %d transferred %d\n",
+            block_mig_state.submitted, block_mig_state.transferred);
+
+    ret = flush_blks(f);
+    if (ret) {
+        return ret;
+    }
+
+    blk_mig_reset_dirty_cursor();
+
+    /* we know for sure that save bulk is completed and
+       all async read completed */
+    blk_mig_lock();
+    assert(block_mig_state.submitted == 0);
+    blk_mig_unlock();
+
+    do {
+        ret = blk_mig_save_dirty_block(f, 0);
+        if (ret < 0) {
+            return ret;
+        }
+    } while (ret == 0);
+
+    /* report completion */
+    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
+
+    DPRINTF("Block migration completed\n");
+
+    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
+
+    return 0;
+}
+
+static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                               uint64_t *non_postcopiable_pending,
+                               uint64_t *postcopiable_pending)
+{
+    /* Estimate pending number of bytes to send */
+    uint64_t pending;
+
+    qemu_mutex_lock_iothread();
+    pending = get_remaining_dirty();
+    qemu_mutex_unlock_iothread();
+
+    blk_mig_lock();
+    pending += block_mig_state.submitted * BLOCK_SIZE +
+               block_mig_state.read_done * BLOCK_SIZE;
+    blk_mig_unlock();
+
+    /* Report at least one block pending during bulk phase */
+    if (pending <= max_size && !block_mig_state.bulk_completed) {
+        pending = max_size + BLOCK_SIZE;
+    }
+
+    DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
+    /* We don't do postcopy */
+    *non_postcopiable_pending += pending;
+}
+
+static int block_load(QEMUFile *f, void *opaque, int version_id)
+{
+    static int banner_printed;
+    int len, flags;
+    char device_name[256];
+    int64_t addr;
+    BlockBackend *blk, *blk_prev = NULL;;
+    Error *local_err = NULL;
+    uint8_t *buf;
+    int64_t total_sectors = 0;
+    int nr_sectors;
+    int ret;
+
+    do {
+        addr = qemu_get_be64(f);
+
+        flags = addr & ~BDRV_SECTOR_MASK;
+        addr >>= BDRV_SECTOR_BITS;
+
+        if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
+            /* get device name */
+            len = qemu_get_byte(f);
+            qemu_get_buffer(f, (uint8_t *)device_name, len);
+            device_name[len] = '\0';
+
+            blk = blk_by_name(device_name);
+            if (!blk) {
+                fprintf(stderr, "Error unknown block device %s\n",
+                        device_name);
+                return -EINVAL;
+            }
+
+            if (blk != blk_prev) {
+                blk_prev = blk;
+                total_sectors = blk_nb_sectors(blk);
+                if (total_sectors <= 0) {
+                    error_report("Error getting length of block device %s",
+                                 device_name);
+                    return -EINVAL;
+                }
+
+                blk_invalidate_cache(blk, &local_err);
+                if (local_err) {
+                    error_report_err(local_err);
+                    return -EINVAL;
+                }
+            }
+
+            if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
+                nr_sectors = total_sectors - addr;
+            } else {
+                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
+            }
+
+            if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
+                ret = blk_pwrite_zeroes(blk, addr * BDRV_SECTOR_SIZE,
+                                        nr_sectors * BDRV_SECTOR_SIZE,
+                                        BDRV_REQ_MAY_UNMAP);
+            } else {
+                buf = g_malloc(BLOCK_SIZE);
+                qemu_get_buffer(f, buf, BLOCK_SIZE);
+                ret = blk_pwrite(blk, addr * BDRV_SECTOR_SIZE, buf,
+                                 nr_sectors * BDRV_SECTOR_SIZE, 0);
+                g_free(buf);
+            }
+
+            if (ret < 0) {
+                return ret;
+            }
+        } else if (flags & BLK_MIG_FLAG_PROGRESS) {
+            if (!banner_printed) {
+                printf("Receiving block device images\n");
+                banner_printed = 1;
+            }
+            printf("Completed %d %%%c", (int)addr,
+                   (addr == 100) ? '\n' : '\r');
+            fflush(stdout);
+        } else if (!(flags & BLK_MIG_FLAG_EOS)) {
+            fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
+            return -EINVAL;
+        }
+        ret = qemu_file_get_error(f);
+        if (ret != 0) {
+            return ret;
+        }
+    } while (!(flags & BLK_MIG_FLAG_EOS));
+
+    return 0;
+}
+
+static void block_set_params(const MigrationParams *params, void *opaque)
+{
+    block_mig_state.blk_enable = params->blk;
+    block_mig_state.shared_base = params->shared;
+
+    /* shared base means that blk_enable = 1 */
+    block_mig_state.blk_enable |= params->shared;
+}
+
+static bool block_is_active(void *opaque)
+{
+    return block_mig_state.blk_enable == 1;
+}
+
+static SaveVMHandlers savevm_block_handlers = {
+    .set_params = block_set_params,
+    .save_live_setup = block_save_setup,
+    .save_live_iterate = block_save_iterate,
+    .save_live_complete_precopy = block_save_complete,
+    .save_live_pending = block_save_pending,
+    .load_state = block_load,
+    .cleanup = block_migration_cleanup,
+    .is_active = block_is_active,
+};
+
+void blk_mig_init(void)
+{
+    QSIMPLEQ_INIT(&block_mig_state.bmds_list);
+    QSIMPLEQ_INIT(&block_mig_state.blk_list);
+    qemu_mutex_init(&block_mig_state.lock);
+
+    register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
+                         &block_mig_state);
+}
diff --git a/migration/exec.c b/migration/exec.c
new file mode 100644
index 0000000..2af63cc
--- /dev/null
+++ b/migration/exec.c
@@ -0,0 +1,72 @@
+/*
+ * QEMU live migration
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Dell MessageOne 2008
+ * Copyright Red Hat, Inc. 2015-2016
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Charles Duffy     <charles_duffy@messageone.com>
+ *  Daniel P. Berrange <berrange@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "io/channel-command.h"
+#include "trace.h"
+
+
+void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp)
+{
+    QIOChannel *ioc;
+    const char *argv[] = { "/bin/sh", "-c", command, NULL };
+
+    trace_migration_exec_outgoing(command);
+    ioc = QIO_CHANNEL(qio_channel_command_new_spawn(argv,
+                                                    O_WRONLY,
+                                                    errp));
+    if (!ioc) {
+        return;
+    }
+
+    migration_channel_connect(s, ioc, NULL);
+    object_unref(OBJECT(ioc));
+}
+
+static gboolean exec_accept_incoming_migration(QIOChannel *ioc,
+                                               GIOCondition condition,
+                                               gpointer opaque)
+{
+    migration_channel_process_incoming(migrate_get_current(), ioc);
+    object_unref(OBJECT(ioc));
+    return FALSE; /* unregister */
+}
+
+void exec_start_incoming_migration(const char *command, Error **errp)
+{
+    QIOChannel *ioc;
+    const char *argv[] = { "/bin/sh", "-c", command, NULL };
+
+    trace_migration_exec_incoming(command);
+    ioc = QIO_CHANNEL(qio_channel_command_new_spawn(argv,
+                                                    O_RDONLY,
+                                                    errp));
+    if (!ioc) {
+        return;
+    }
+
+    qio_channel_add_watch(ioc,
+                          G_IO_IN,
+                          exec_accept_incoming_migration,
+                          NULL,
+                          NULL);
+}
diff --git a/migration/fd.c b/migration/fd.c
new file mode 100644
index 0000000..84a10fd
--- /dev/null
+++ b/migration/fd.c
@@ -0,0 +1,73 @@
+/*
+ * QEMU live migration via generic fd
+ *
+ * Copyright Red Hat, Inc. 2009-2016
+ *
+ * Authors:
+ *  Chris Lalancette <clalance@redhat.com>
+ *  Daniel P. Berrange <berrange@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "monitor/monitor.h"
+#include "io/channel-util.h"
+#include "trace.h"
+
+
+void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp)
+{
+    QIOChannel *ioc;
+    int fd = monitor_get_fd(cur_mon, fdname, errp);
+    if (fd == -1) {
+        return;
+    }
+
+    trace_migration_fd_outgoing(fd);
+    ioc = qio_channel_new_fd(fd, errp);
+    if (!ioc) {
+        close(fd);
+        return;
+    }
+
+    migration_channel_connect(s, ioc, NULL);
+    object_unref(OBJECT(ioc));
+}
+
+static gboolean fd_accept_incoming_migration(QIOChannel *ioc,
+                                             GIOCondition condition,
+                                             gpointer opaque)
+{
+    migration_channel_process_incoming(migrate_get_current(), ioc);
+    object_unref(OBJECT(ioc));
+    return FALSE; /* unregister */
+}
+
+void fd_start_incoming_migration(const char *infd, Error **errp)
+{
+    QIOChannel *ioc;
+    int fd;
+
+    fd = strtol(infd, NULL, 0);
+    trace_migration_fd_incoming(fd);
+
+    ioc = qio_channel_new_fd(fd, errp);
+    if (!ioc) {
+        close(fd);
+        return;
+    }
+
+    qio_channel_add_watch(ioc,
+                          G_IO_IN,
+                          fd_accept_incoming_migration,
+                          NULL,
+                          NULL);
+}
diff --git a/migration/migration.c b/migration/migration.c
new file mode 100644
index 0000000..955d5ee
--- /dev/null
+++ b/migration/migration.c
@@ -0,0 +1,1896 @@
+/*
+ * QEMU live migration
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "sysemu/sysemu.h"
+#include "block/block.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/util.h"
+#include "qemu/sockets.h"
+#include "qemu/rcu.h"
+#include "migration/block.h"
+#include "migration/postcopy-ram.h"
+#include "qemu/thread.h"
+#include "qmp-commands.h"
+#include "trace.h"
+#include "qapi-event.h"
+#include "qom/cpu.h"
+#include "exec/memory.h"
+#include "exec/address-spaces.h"
+#include "io/channel-buffer.h"
+#include "io/channel-tls.h"
+
+#define MAX_THROTTLE  (32 << 20)      /* Migration transfer speed throttling */
+
+/* Amount of time to allocate to each "chunk" of bandwidth-throttled
+ * data. */
+#define BUFFER_DELAY     100
+#define XFER_LIMIT_RATIO (1000 / BUFFER_DELAY)
+
+/* Default compression thread count */
+#define DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT 8
+/* Default decompression thread count, usually decompression is at
+ * least 4 times as fast as compression.*/
+#define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2
+/*0: means nocompress, 1: best speed, ... 9: best compress ratio */
+#define DEFAULT_MIGRATE_COMPRESS_LEVEL 1
+/* Define default autoconverge cpu throttle migration parameters */
+#define DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL 20
+#define DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT 10
+
+/* Migration XBZRLE default cache size */
+#define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024)
+
+static NotifierList migration_state_notifiers =
+    NOTIFIER_LIST_INITIALIZER(migration_state_notifiers);
+
+static bool deferred_incoming;
+
+/*
+ * Current state of incoming postcopy; note this is not part of
+ * MigrationIncomingState since it's state is used during cleanup
+ * at the end as MIS is being freed.
+ */
+static PostcopyState incoming_postcopy_state;
+
+/* When we add fault tolerance, we could have several
+   migrations at once.  For now we don't need to add
+   dynamic creation of migration */
+
+/* For outgoing */
+MigrationState *migrate_get_current(void)
+{
+    static bool once;
+    static MigrationState current_migration = {
+        .state = MIGRATION_STATUS_NONE,
+        .bandwidth_limit = MAX_THROTTLE,
+        .xbzrle_cache_size = DEFAULT_MIGRATE_CACHE_SIZE,
+        .mbps = -1,
+        .parameters = {
+            .compress_level = DEFAULT_MIGRATE_COMPRESS_LEVEL,
+            .compress_threads = DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT,
+            .decompress_threads = DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT,
+            .cpu_throttle_initial = DEFAULT_MIGRATE_CPU_THROTTLE_INITIAL,
+            .cpu_throttle_increment = DEFAULT_MIGRATE_CPU_THROTTLE_INCREMENT,
+        },
+    };
+
+    if (!once) {
+        qemu_mutex_init(&current_migration.src_page_req_mutex);
+        once = true;
+    }
+    return &current_migration;
+}
+
+/* For incoming */
+static MigrationIncomingState *mis_current;
+
+MigrationIncomingState *migration_incoming_get_current(void)
+{
+    return mis_current;
+}
+
+MigrationIncomingState *migration_incoming_state_new(QEMUFile* f)
+{
+    mis_current = g_new0(MigrationIncomingState, 1);
+    mis_current->from_src_file = f;
+    mis_current->state = MIGRATION_STATUS_NONE;
+    QLIST_INIT(&mis_current->loadvm_handlers);
+    qemu_mutex_init(&mis_current->rp_mutex);
+    qemu_event_init(&mis_current->main_thread_load_event, false);
+
+    return mis_current;
+}
+
+void migration_incoming_state_destroy(void)
+{
+    qemu_event_destroy(&mis_current->main_thread_load_event);
+    loadvm_free_handlers(mis_current);
+    g_free(mis_current);
+    mis_current = NULL;
+}
+
+
+typedef struct {
+    bool optional;
+    uint32_t size;
+    uint8_t runstate[100];
+    RunState state;
+    bool received;
+} GlobalState;
+
+static GlobalState global_state;
+
+int global_state_store(void)
+{
+    if (!runstate_store((char *)global_state.runstate,
+                        sizeof(global_state.runstate))) {
+        error_report("runstate name too big: %s", global_state.runstate);
+        trace_migrate_state_too_big();
+        return -EINVAL;
+    }
+    return 0;
+}
+
+void global_state_store_running(void)
+{
+    const char *state = RunState_lookup[RUN_STATE_RUNNING];
+    strncpy((char *)global_state.runstate,
+           state, sizeof(global_state.runstate));
+}
+
+static bool global_state_received(void)
+{
+    return global_state.received;
+}
+
+static RunState global_state_get_runstate(void)
+{
+    return global_state.state;
+}
+
+void global_state_set_optional(void)
+{
+    global_state.optional = true;
+}
+
+static bool global_state_needed(void *opaque)
+{
+    GlobalState *s = opaque;
+    char *runstate = (char *)s->runstate;
+
+    /* If it is not optional, it is mandatory */
+
+    if (s->optional == false) {
+        return true;
+    }
+
+    /* If state is running or paused, it is not needed */
+
+    if (strcmp(runstate, "running") == 0 ||
+        strcmp(runstate, "paused") == 0) {
+        return false;
+    }
+
+    /* for any other state it is needed */
+    return true;
+}
+
+static int global_state_post_load(void *opaque, int version_id)
+{
+    GlobalState *s = opaque;
+    Error *local_err = NULL;
+    int r;
+    char *runstate = (char *)s->runstate;
+
+    s->received = true;
+    trace_migrate_global_state_post_load(runstate);
+
+    r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE__MAX,
+                                -1, &local_err);
+
+    if (r == -1) {
+        if (local_err) {
+            error_report_err(local_err);
+        }
+        return -EINVAL;
+    }
+    s->state = r;
+
+    return 0;
+}
+
+static void global_state_pre_save(void *opaque)
+{
+    GlobalState *s = opaque;
+
+    trace_migrate_global_state_pre_save((char *)s->runstate);
+    s->size = strlen((char *)s->runstate) + 1;
+}
+
+static const VMStateDescription vmstate_globalstate = {
+    .name = "globalstate",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .post_load = global_state_post_load,
+    .pre_save = global_state_pre_save,
+    .needed = global_state_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(size, GlobalState),
+        VMSTATE_BUFFER(runstate, GlobalState),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+void register_global_state(void)
+{
+    /* We would use it independently that we receive it */
+    strcpy((char *)&global_state.runstate, "");
+    global_state.received = false;
+    vmstate_register(NULL, 0, &vmstate_globalstate, &global_state);
+}
+
+static void migrate_generate_event(int new_state)
+{
+    if (migrate_use_events()) {
+        qapi_event_send_migration(new_state, &error_abort);
+    }
+}
+
+/*
+ * Called on -incoming with a defer: uri.
+ * The migration can be started later after any parameters have been
+ * changed.
+ */
+static void deferred_incoming_migration(Error **errp)
+{
+    if (deferred_incoming) {
+        error_setg(errp, "Incoming migration already deferred");
+    }
+    deferred_incoming = true;
+}
+
+/* Request a range of pages from the source VM at the given
+ * start address.
+ *   rbname: Name of the RAMBlock to request the page in, if NULL it's the same
+ *           as the last request (a name must have been given previously)
+ *   Start: Address offset within the RB
+ *   Len: Length in bytes required - must be a multiple of pagesize
+ */
+void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
+                               ram_addr_t start, size_t len)
+{
+    uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname up to 256 */
+    size_t msglen = 12; /* start + len */
+
+    *(uint64_t *)bufc = cpu_to_be64((uint64_t)start);
+    *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len);
+
+    if (rbname) {
+        int rbname_len = strlen(rbname);
+        assert(rbname_len < 256);
+
+        bufc[msglen++] = rbname_len;
+        memcpy(bufc + msglen, rbname, rbname_len);
+        msglen += rbname_len;
+        migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES_ID, msglen, bufc);
+    } else {
+        migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES, msglen, bufc);
+    }
+}
+
+void qemu_start_incoming_migration(const char *uri, Error **errp)
+{
+    const char *p;
+
+    qapi_event_send_migration(MIGRATION_STATUS_SETUP, &error_abort);
+    if (!strcmp(uri, "defer")) {
+        deferred_incoming_migration(errp);
+    } else if (strstart(uri, "tcp:", &p)) {
+        tcp_start_incoming_migration(p, errp);
+#ifdef CONFIG_RDMA
+    } else if (strstart(uri, "rdma:", &p)) {
+        rdma_start_incoming_migration(p, errp);
+#endif
+    } else if (strstart(uri, "exec:", &p)) {
+        exec_start_incoming_migration(p, errp);
+    } else if (strstart(uri, "unix:", &p)) {
+        unix_start_incoming_migration(p, errp);
+    } else if (strstart(uri, "fd:", &p)) {
+        fd_start_incoming_migration(p, errp);
+    } else {
+        error_setg(errp, "unknown migration protocol: %s", uri);
+    }
+}
+
+static void process_incoming_migration_bh(void *opaque)
+{
+    Error *local_err = NULL;
+    MigrationIncomingState *mis = opaque;
+
+    /* Make sure all file formats flush their mutable metadata */
+    bdrv_invalidate_cache_all(&local_err);
+    if (local_err) {
+        migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
+        error_report_err(local_err);
+        migrate_decompress_threads_join();
+        exit(EXIT_FAILURE);
+    }
+
+    /*
+     * This must happen after all error conditions are dealt with and
+     * we're sure the VM is going to be running on this host.
+     */
+    qemu_announce_self();
+
+    /* If global state section was not received or we are in running
+       state, we need to obey autostart. Any other state is set with
+       runstate_set. */
+
+    if (!global_state_received() ||
+        global_state_get_runstate() == RUN_STATE_RUNNING) {
+        if (autostart) {
+            vm_start();
+        } else {
+            runstate_set(RUN_STATE_PAUSED);
+        }
+    } else {
+        runstate_set(global_state_get_runstate());
+    }
+    migrate_decompress_threads_join();
+    /*
+     * This must happen after any state changes since as soon as an external
+     * observer sees this event they might start to prod at the VM assuming
+     * it's ready to use.
+     */
+    migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                      MIGRATION_STATUS_COMPLETED);
+    qemu_bh_delete(mis->bh);
+    migration_incoming_state_destroy();
+}
+
+static void process_incoming_migration_co(void *opaque)
+{
+    QEMUFile *f = opaque;
+    MigrationIncomingState *mis;
+    PostcopyState ps;
+    int ret;
+
+    mis = migration_incoming_state_new(f);
+    postcopy_state_set(POSTCOPY_INCOMING_NONE);
+    migrate_set_state(&mis->state, MIGRATION_STATUS_NONE,
+                      MIGRATION_STATUS_ACTIVE);
+    ret = qemu_loadvm_state(f);
+
+    ps = postcopy_state_get();
+    trace_process_incoming_migration_co_end(ret, ps);
+    if (ps != POSTCOPY_INCOMING_NONE) {
+        if (ps == POSTCOPY_INCOMING_ADVISE) {
+            /*
+             * Where a migration had postcopy enabled (and thus went to advise)
+             * but managed to complete within the precopy period, we can use
+             * the normal exit.
+             */
+            postcopy_ram_incoming_cleanup(mis);
+        } else if (ret >= 0) {
+            /*
+             * Postcopy was started, cleanup should happen at the end of the
+             * postcopy thread.
+             */
+            trace_process_incoming_migration_co_postcopy_end_main();
+            return;
+        }
+        /* Else if something went wrong then just fall out of the normal exit */
+    }
+
+    qemu_fclose(f);
+    free_xbzrle_decoded_buf();
+
+    if (ret < 0) {
+        migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
+        error_report("load of migration failed: %s", strerror(-ret));
+        migrate_decompress_threads_join();
+        exit(EXIT_FAILURE);
+    }
+
+    mis->bh = qemu_bh_new(process_incoming_migration_bh, mis);
+    qemu_bh_schedule(mis->bh);
+}
+
+void migration_fd_process_incoming(QEMUFile *f)
+{
+    Coroutine *co = qemu_coroutine_create(process_incoming_migration_co, f);
+
+    migrate_decompress_threads_create();
+    qemu_file_set_blocking(f, false);
+    qemu_coroutine_enter(co);
+}
+
+
+void migration_channel_process_incoming(MigrationState *s,
+                                        QIOChannel *ioc)
+{
+    trace_migration_set_incoming_channel(
+        ioc, object_get_typename(OBJECT(ioc)));
+
+    if (s->parameters.tls_creds &&
+        !object_dynamic_cast(OBJECT(ioc),
+                             TYPE_QIO_CHANNEL_TLS)) {
+        Error *local_err = NULL;
+        migration_tls_channel_process_incoming(s, ioc, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
+        }
+    } else {
+        QEMUFile *f = qemu_fopen_channel_input(ioc);
+        migration_fd_process_incoming(f);
+    }
+}
+
+
+void migration_channel_connect(MigrationState *s,
+                               QIOChannel *ioc,
+                               const char *hostname)
+{
+    trace_migration_set_outgoing_channel(
+        ioc, object_get_typename(OBJECT(ioc)), hostname);
+
+    if (s->parameters.tls_creds &&
+        !object_dynamic_cast(OBJECT(ioc),
+                             TYPE_QIO_CHANNEL_TLS)) {
+        Error *local_err = NULL;
+        migration_tls_channel_connect(s, ioc, hostname, &local_err);
+        if (local_err) {
+            migrate_fd_error(s, local_err);
+            error_free(local_err);
+        }
+    } else {
+        QEMUFile *f = qemu_fopen_channel_output(ioc);
+
+        s->to_dst_file = f;
+
+        migrate_fd_connect(s);
+    }
+}
+
+
+/*
+ * Send a message on the return channel back to the source
+ * of the migration.
+ */
+void migrate_send_rp_message(MigrationIncomingState *mis,
+                             enum mig_rp_message_type message_type,
+                             uint16_t len, void *data)
+{
+    trace_migrate_send_rp_message((int)message_type, len);
+    qemu_mutex_lock(&mis->rp_mutex);
+    qemu_put_be16(mis->to_src_file, (unsigned int)message_type);
+    qemu_put_be16(mis->to_src_file, len);
+    qemu_put_buffer(mis->to_src_file, data, len);
+    qemu_fflush(mis->to_src_file);
+    qemu_mutex_unlock(&mis->rp_mutex);
+}
+
+/*
+ * Send a 'SHUT' message on the return channel with the given value
+ * to indicate that we've finished with the RP.  Non-0 value indicates
+ * error.
+ */
+void migrate_send_rp_shut(MigrationIncomingState *mis,
+                          uint32_t value)
+{
+    uint32_t buf;
+
+    buf = cpu_to_be32(value);
+    migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf);
+}
+
+/*
+ * Send a 'PONG' message on the return channel with the given value
+ * (normally in response to a 'PING')
+ */
+void migrate_send_rp_pong(MigrationIncomingState *mis,
+                          uint32_t value)
+{
+    uint32_t buf;
+
+    buf = cpu_to_be32(value);
+    migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf);
+}
+
+/* amount of nanoseconds we are willing to wait for migration to be down.
+ * the choice of nanoseconds is because it is the maximum resolution that
+ * get_clock() can achieve. It is an internal measure. All user-visible
+ * units must be in seconds */
+static uint64_t max_downtime = 300000000;
+
+uint64_t migrate_max_downtime(void)
+{
+    return max_downtime;
+}
+
+MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp)
+{
+    MigrationCapabilityStatusList *head = NULL;
+    MigrationCapabilityStatusList *caps;
+    MigrationState *s = migrate_get_current();
+    int i;
+
+    caps = NULL; /* silence compiler warning */
+    for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
+        if (head == NULL) {
+            head = g_malloc0(sizeof(*caps));
+            caps = head;
+        } else {
+            caps->next = g_malloc0(sizeof(*caps));
+            caps = caps->next;
+        }
+        caps->value =
+            g_malloc(sizeof(*caps->value));
+        caps->value->capability = i;
+        caps->value->state = s->enabled_capabilities[i];
+    }
+
+    return head;
+}
+
+MigrationParameters *qmp_query_migrate_parameters(Error **errp)
+{
+    MigrationParameters *params;
+    MigrationState *s = migrate_get_current();
+
+    params = g_malloc0(sizeof(*params));
+    params->compress_level = s->parameters.compress_level;
+    params->compress_threads = s->parameters.compress_threads;
+    params->decompress_threads = s->parameters.decompress_threads;
+    params->cpu_throttle_initial = s->parameters.cpu_throttle_initial;
+    params->cpu_throttle_increment = s->parameters.cpu_throttle_increment;
+    params->tls_creds = g_strdup(s->parameters.tls_creds);
+    params->tls_hostname = g_strdup(s->parameters.tls_hostname);
+
+    return params;
+}
+
+/*
+ * Return true if we're already in the middle of a migration
+ * (i.e. any of the active or setup states)
+ */
+static bool migration_is_setup_or_active(int state)
+{
+    switch (state) {
+    case MIGRATION_STATUS_ACTIVE:
+    case MIGRATION_STATUS_POSTCOPY_ACTIVE:
+    case MIGRATION_STATUS_SETUP:
+        return true;
+
+    default:
+        return false;
+
+    }
+}
+
+static void get_xbzrle_cache_stats(MigrationInfo *info)
+{
+    if (migrate_use_xbzrle()) {
+        info->has_xbzrle_cache = true;
+        info->xbzrle_cache = g_malloc0(sizeof(*info->xbzrle_cache));
+        info->xbzrle_cache->cache_size = migrate_xbzrle_cache_size();
+        info->xbzrle_cache->bytes = xbzrle_mig_bytes_transferred();
+        info->xbzrle_cache->pages = xbzrle_mig_pages_transferred();
+        info->xbzrle_cache->cache_miss = xbzrle_mig_pages_cache_miss();
+        info->xbzrle_cache->cache_miss_rate = xbzrle_mig_cache_miss_rate();
+        info->xbzrle_cache->overflow = xbzrle_mig_pages_overflow();
+    }
+}
+
+static void populate_ram_info(MigrationInfo *info, MigrationState *s)
+{
+    info->has_ram = true;
+    info->ram = g_malloc0(sizeof(*info->ram));
+    info->ram->transferred = ram_bytes_transferred();
+    info->ram->total = ram_bytes_total();
+    info->ram->duplicate = dup_mig_pages_transferred();
+    info->ram->skipped = skipped_mig_pages_transferred();
+    info->ram->normal = norm_mig_pages_transferred();
+    info->ram->normal_bytes = norm_mig_bytes_transferred();
+    info->ram->mbps = s->mbps;
+    info->ram->dirty_sync_count = s->dirty_sync_count;
+    info->ram->postcopy_requests = s->postcopy_requests;
+
+    if (s->state != MIGRATION_STATUS_COMPLETED) {
+        info->ram->remaining = ram_bytes_remaining();
+        info->ram->dirty_pages_rate = s->dirty_pages_rate;
+    }
+}
+
+MigrationInfo *qmp_query_migrate(Error **errp)
+{
+    MigrationInfo *info = g_malloc0(sizeof(*info));
+    MigrationState *s = migrate_get_current();
+
+    switch (s->state) {
+    case MIGRATION_STATUS_NONE:
+        /* no migration has happened ever */
+        break;
+    case MIGRATION_STATUS_SETUP:
+        info->has_status = true;
+        info->has_total_time = false;
+        break;
+    case MIGRATION_STATUS_ACTIVE:
+    case MIGRATION_STATUS_CANCELLING:
+        info->has_status = true;
+        info->has_total_time = true;
+        info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
+            - s->total_time;
+        info->has_expected_downtime = true;
+        info->expected_downtime = s->expected_downtime;
+        info->has_setup_time = true;
+        info->setup_time = s->setup_time;
+
+        populate_ram_info(info, s);
+
+        if (blk_mig_active()) {
+            info->has_disk = true;
+            info->disk = g_malloc0(sizeof(*info->disk));
+            info->disk->transferred = blk_mig_bytes_transferred();
+            info->disk->remaining = blk_mig_bytes_remaining();
+            info->disk->total = blk_mig_bytes_total();
+        }
+
+        if (cpu_throttle_active()) {
+            info->has_cpu_throttle_percentage = true;
+            info->cpu_throttle_percentage = cpu_throttle_get_percentage();
+        }
+
+        get_xbzrle_cache_stats(info);
+        break;
+    case MIGRATION_STATUS_POSTCOPY_ACTIVE:
+        /* Mostly the same as active; TODO add some postcopy stats */
+        info->has_status = true;
+        info->has_total_time = true;
+        info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
+            - s->total_time;
+        info->has_expected_downtime = true;
+        info->expected_downtime = s->expected_downtime;
+        info->has_setup_time = true;
+        info->setup_time = s->setup_time;
+
+        populate_ram_info(info, s);
+
+        if (blk_mig_active()) {
+            info->has_disk = true;
+            info->disk = g_malloc0(sizeof(*info->disk));
+            info->disk->transferred = blk_mig_bytes_transferred();
+            info->disk->remaining = blk_mig_bytes_remaining();
+            info->disk->total = blk_mig_bytes_total();
+        }
+
+        get_xbzrle_cache_stats(info);
+        break;
+    case MIGRATION_STATUS_COMPLETED:
+        get_xbzrle_cache_stats(info);
+
+        info->has_status = true;
+        info->has_total_time = true;
+        info->total_time = s->total_time;
+        info->has_downtime = true;
+        info->downtime = s->downtime;
+        info->has_setup_time = true;
+        info->setup_time = s->setup_time;
+
+        populate_ram_info(info, s);
+        break;
+    case MIGRATION_STATUS_FAILED:
+        info->has_status = true;
+        if (s->error) {
+            info->has_error_desc = true;
+            info->error_desc = g_strdup(error_get_pretty(s->error));
+        }
+        break;
+    case MIGRATION_STATUS_CANCELLED:
+        info->has_status = true;
+        break;
+    }
+    info->status = s->state;
+
+    return info;
+}
+
+void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params,
+                                  Error **errp)
+{
+    MigrationState *s = migrate_get_current();
+    MigrationCapabilityStatusList *cap;
+    bool old_postcopy_cap = migrate_postcopy_ram();
+
+    if (migration_is_setup_or_active(s->state)) {
+        error_setg(errp, QERR_MIGRATION_ACTIVE);
+        return;
+    }
+
+    for (cap = params; cap; cap = cap->next) {
+        s->enabled_capabilities[cap->value->capability] = cap->value->state;
+    }
+
+    if (migrate_postcopy_ram()) {
+        if (migrate_use_compression()) {
+            /* The decompression threads asynchronously write into RAM
+             * rather than use the atomic copies needed to avoid
+             * userfaulting.  It should be possible to fix the decompression
+             * threads for compatibility in future.
+             */
+            error_report("Postcopy is not currently compatible with "
+                         "compression");
+            s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM] =
+                false;
+        }
+        /* This check is reasonably expensive, so only when it's being
+         * set the first time, also it's only the destination that needs
+         * special support.
+         */
+        if (!old_postcopy_cap && runstate_check(RUN_STATE_INMIGRATE) &&
+            !postcopy_ram_supported_by_host()) {
+            /* postcopy_ram_supported_by_host will have emitted a more
+             * detailed message
+             */
+            error_report("Postcopy is not supported");
+            s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM] =
+                false;
+        }
+    }
+}
+
+void qmp_migrate_set_parameters(bool has_compress_level,
+                                int64_t compress_level,
+                                bool has_compress_threads,
+                                int64_t compress_threads,
+                                bool has_decompress_threads,
+                                int64_t decompress_threads,
+                                bool has_cpu_throttle_initial,
+                                int64_t cpu_throttle_initial,
+                                bool has_cpu_throttle_increment,
+                                int64_t cpu_throttle_increment,
+                                bool has_tls_creds,
+                                const char *tls_creds,
+                                bool has_tls_hostname,
+                                const char *tls_hostname,
+                                Error **errp)
+{
+    MigrationState *s = migrate_get_current();
+
+    if (has_compress_level && (compress_level < 0 || compress_level > 9)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "compress_level",
+                   "is invalid, it should be in the range of 0 to 9");
+        return;
+    }
+    if (has_compress_threads &&
+            (compress_threads < 1 || compress_threads > 255)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "compress_threads",
+                   "is invalid, it should be in the range of 1 to 255");
+        return;
+    }
+    if (has_decompress_threads &&
+            (decompress_threads < 1 || decompress_threads > 255)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "decompress_threads",
+                   "is invalid, it should be in the range of 1 to 255");
+        return;
+    }
+    if (has_cpu_throttle_initial &&
+            (cpu_throttle_initial < 1 || cpu_throttle_initial > 99)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "cpu_throttle_initial",
+                   "an integer in the range of 1 to 99");
+    }
+    if (has_cpu_throttle_increment &&
+            (cpu_throttle_increment < 1 || cpu_throttle_increment > 99)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   "cpu_throttle_increment",
+                   "an integer in the range of 1 to 99");
+    }
+
+    if (has_compress_level) {
+        s->parameters.compress_level = compress_level;
+    }
+    if (has_compress_threads) {
+        s->parameters.compress_threads = compress_threads;
+    }
+    if (has_decompress_threads) {
+        s->parameters.decompress_threads = decompress_threads;
+    }
+    if (has_cpu_throttle_initial) {
+        s->parameters.cpu_throttle_initial = cpu_throttle_initial;
+    }
+    if (has_cpu_throttle_increment) {
+        s->parameters.cpu_throttle_increment = cpu_throttle_increment;
+    }
+    if (has_tls_creds) {
+        g_free(s->parameters.tls_creds);
+        s->parameters.tls_creds = g_strdup(tls_creds);
+    }
+    if (has_tls_hostname) {
+        g_free(s->parameters.tls_hostname);
+        s->parameters.tls_hostname = g_strdup(tls_hostname);
+    }
+}
+
+
+void qmp_migrate_start_postcopy(Error **errp)
+{
+    MigrationState *s = migrate_get_current();
+
+    if (!migrate_postcopy_ram()) {
+        error_setg(errp, "Enable postcopy with migrate_set_capability before"
+                         " the start of migration");
+        return;
+    }
+
+    if (s->state == MIGRATION_STATUS_NONE) {
+        error_setg(errp, "Postcopy must be started after migration has been"
+                         " started");
+        return;
+    }
+    /*
+     * we don't error if migration has finished since that would be racy
+     * with issuing this command.
+     */
+    atomic_set(&s->start_postcopy, true);
+}
+
+/* shared migration helpers */
+
+void migrate_set_state(int *state, int old_state, int new_state)
+{
+    if (atomic_cmpxchg(state, old_state, new_state) == old_state) {
+        trace_migrate_set_state(new_state);
+        migrate_generate_event(new_state);
+    }
+}
+
+static void migrate_fd_cleanup(void *opaque)
+{
+    MigrationState *s = opaque;
+
+    qemu_bh_delete(s->cleanup_bh);
+    s->cleanup_bh = NULL;
+
+    flush_page_queue(s);
+
+    if (s->to_dst_file) {
+        trace_migrate_fd_cleanup();
+        qemu_mutex_unlock_iothread();
+        if (s->migration_thread_running) {
+            qemu_thread_join(&s->thread);
+            s->migration_thread_running = false;
+        }
+        qemu_mutex_lock_iothread();
+
+        migrate_compress_threads_join();
+        qemu_fclose(s->to_dst_file);
+        s->to_dst_file = NULL;
+    }
+
+    assert((s->state != MIGRATION_STATUS_ACTIVE) &&
+           (s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE));
+
+    if (s->state == MIGRATION_STATUS_CANCELLING) {
+        migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING,
+                          MIGRATION_STATUS_CANCELLED);
+    }
+
+    notifier_list_notify(&migration_state_notifiers, s);
+}
+
+void migrate_fd_error(MigrationState *s, const Error *error)
+{
+    trace_migrate_fd_error(error ? error_get_pretty(error) : "");
+    assert(s->to_dst_file == NULL);
+    migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                      MIGRATION_STATUS_FAILED);
+    if (!s->error) {
+        s->error = error_copy(error);
+    }
+    notifier_list_notify(&migration_state_notifiers, s);
+}
+
+static void migrate_fd_cancel(MigrationState *s)
+{
+    int old_state ;
+    QEMUFile *f = migrate_get_current()->to_dst_file;
+    trace_migrate_fd_cancel();
+
+    if (s->rp_state.from_dst_file) {
+        /* shutdown the rp socket, so causing the rp thread to shutdown */
+        qemu_file_shutdown(s->rp_state.from_dst_file);
+    }
+
+    do {
+        old_state = s->state;
+        if (!migration_is_setup_or_active(old_state)) {
+            break;
+        }
+        migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING);
+    } while (s->state != MIGRATION_STATUS_CANCELLING);
+
+    /*
+     * If we're unlucky the migration code might be stuck somewhere in a
+     * send/write while the network has failed and is waiting to timeout;
+     * if we've got shutdown(2) available then we can force it to quit.
+     * The outgoing qemu file gets closed in migrate_fd_cleanup that is
+     * called in a bh, so there is no race against this cancel.
+     */
+    if (s->state == MIGRATION_STATUS_CANCELLING && f) {
+        qemu_file_shutdown(f);
+    }
+}
+
+void add_migration_state_change_notifier(Notifier *notify)
+{
+    notifier_list_add(&migration_state_notifiers, notify);
+}
+
+void remove_migration_state_change_notifier(Notifier *notify)
+{
+    notifier_remove(notify);
+}
+
+bool migration_in_setup(MigrationState *s)
+{
+    return s->state == MIGRATION_STATUS_SETUP;
+}
+
+bool migration_has_finished(MigrationState *s)
+{
+    return s->state == MIGRATION_STATUS_COMPLETED;
+}
+
+bool migration_has_failed(MigrationState *s)
+{
+    return (s->state == MIGRATION_STATUS_CANCELLED ||
+            s->state == MIGRATION_STATUS_FAILED);
+}
+
+bool migration_in_postcopy(MigrationState *s)
+{
+    return (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE);
+}
+
+bool migration_in_postcopy_after_devices(MigrationState *s)
+{
+    return migration_in_postcopy(s) && s->postcopy_after_devices;
+}
+
+MigrationState *migrate_init(const MigrationParams *params)
+{
+    MigrationState *s = migrate_get_current();
+
+    /*
+     * Reinitialise all migration state, except
+     * parameters/capabilities that the user set, and
+     * locks.
+     */
+    s->bytes_xfer = 0;
+    s->xfer_limit = 0;
+    s->cleanup_bh = 0;
+    s->to_dst_file = NULL;
+    s->state = MIGRATION_STATUS_NONE;
+    s->params = *params;
+    s->rp_state.from_dst_file = NULL;
+    s->rp_state.error = false;
+    s->mbps = 0.0;
+    s->downtime = 0;
+    s->expected_downtime = 0;
+    s->dirty_pages_rate = 0;
+    s->dirty_bytes_rate = 0;
+    s->setup_time = 0;
+    s->dirty_sync_count = 0;
+    s->start_postcopy = false;
+    s->postcopy_after_devices = false;
+    s->postcopy_requests = 0;
+    s->migration_thread_running = false;
+    s->last_req_rb = NULL;
+    error_free(s->error);
+    s->error = NULL;
+
+    migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP);
+
+    QSIMPLEQ_INIT(&s->src_page_requests);
+
+    s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    return s;
+}
+
+static GSList *migration_blockers;
+
+void migrate_add_blocker(Error *reason)
+{
+    migration_blockers = g_slist_prepend(migration_blockers, reason);
+}
+
+void migrate_del_blocker(Error *reason)
+{
+    migration_blockers = g_slist_remove(migration_blockers, reason);
+}
+
+void qmp_migrate_incoming(const char *uri, Error **errp)
+{
+    Error *local_err = NULL;
+    static bool once = true;
+
+    if (!deferred_incoming) {
+        error_setg(errp, "For use with '-incoming defer'");
+        return;
+    }
+    if (!once) {
+        error_setg(errp, "The incoming migration has already been started");
+    }
+
+    qemu_start_incoming_migration(uri, &local_err);
+
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    once = false;
+}
+
+bool migration_is_blocked(Error **errp)
+{
+    if (qemu_savevm_state_blocked(errp)) {
+        return true;
+    }
+
+    if (migration_blockers) {
+        *errp = error_copy(migration_blockers->data);
+        return true;
+    }
+
+    return false;
+}
+
+void qmp_migrate(const char *uri, bool has_blk, bool blk,
+                 bool has_inc, bool inc, bool has_detach, bool detach,
+                 Error **errp)
+{
+    Error *local_err = NULL;
+    MigrationState *s = migrate_get_current();
+    MigrationParams params;
+    const char *p;
+
+    params.blk = has_blk && blk;
+    params.shared = has_inc && inc;
+
+    if (migration_is_setup_or_active(s->state) ||
+        s->state == MIGRATION_STATUS_CANCELLING) {
+        error_setg(errp, QERR_MIGRATION_ACTIVE);
+        return;
+    }
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        error_setg(errp, "Guest is waiting for an incoming migration");
+        return;
+    }
+
+    if (migration_is_blocked(errp)) {
+        return;
+    }
+
+    s = migrate_init(&params);
+
+    if (strstart(uri, "tcp:", &p)) {
+        tcp_start_outgoing_migration(s, p, &local_err);
+#ifdef CONFIG_RDMA
+    } else if (strstart(uri, "rdma:", &p)) {
+        rdma_start_outgoing_migration(s, p, &local_err);
+#endif
+    } else if (strstart(uri, "exec:", &p)) {
+        exec_start_outgoing_migration(s, p, &local_err);
+    } else if (strstart(uri, "unix:", &p)) {
+        unix_start_outgoing_migration(s, p, &local_err);
+    } else if (strstart(uri, "fd:", &p)) {
+        fd_start_outgoing_migration(s, p, &local_err);
+    } else {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri",
+                   "a valid migration protocol");
+        migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                          MIGRATION_STATUS_FAILED);
+        return;
+    }
+
+    if (local_err) {
+        migrate_fd_error(s, local_err);
+        error_propagate(errp, local_err);
+        return;
+    }
+}
+
+void qmp_migrate_cancel(Error **errp)
+{
+    migrate_fd_cancel(migrate_get_current());
+}
+
+void qmp_migrate_set_cache_size(int64_t value, Error **errp)
+{
+    MigrationState *s = migrate_get_current();
+    int64_t new_size;
+
+    /* Check for truncation */
+    if (value != (size_t)value) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
+                   "exceeding address space");
+        return;
+    }
+
+    /* Cache should not be larger than guest ram size */
+    if (value > ram_bytes_total()) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
+                   "exceeds guest ram size ");
+        return;
+    }
+
+    new_size = xbzrle_cache_resize(value);
+    if (new_size < 0) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cache size",
+                   "is smaller than page size");
+        return;
+    }
+
+    s->xbzrle_cache_size = new_size;
+}
+
+int64_t qmp_query_migrate_cache_size(Error **errp)
+{
+    return migrate_xbzrle_cache_size();
+}
+
+void qmp_migrate_set_speed(int64_t value, Error **errp)
+{
+    MigrationState *s;
+
+    if (value < 0) {
+        value = 0;
+    }
+    if (value > SIZE_MAX) {
+        value = SIZE_MAX;
+    }
+
+    s = migrate_get_current();
+    s->bandwidth_limit = value;
+    if (s->to_dst_file) {
+        qemu_file_set_rate_limit(s->to_dst_file,
+                                 s->bandwidth_limit / XFER_LIMIT_RATIO);
+    }
+}
+
+void qmp_migrate_set_downtime(double value, Error **errp)
+{
+    value *= 1e9;
+    value = MAX(0, MIN(UINT64_MAX, value));
+    max_downtime = (uint64_t)value;
+}
+
+bool migrate_postcopy_ram(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM];
+}
+
+bool migrate_auto_converge(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_AUTO_CONVERGE];
+}
+
+bool migrate_zero_blocks(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_ZERO_BLOCKS];
+}
+
+bool migrate_use_compression(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_COMPRESS];
+}
+
+int migrate_compress_level(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->parameters.compress_level;
+}
+
+int migrate_compress_threads(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->parameters.compress_threads;
+}
+
+int migrate_decompress_threads(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->parameters.decompress_threads;
+}
+
+bool migrate_use_events(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_EVENTS];
+}
+
+int migrate_use_xbzrle(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->enabled_capabilities[MIGRATION_CAPABILITY_XBZRLE];
+}
+
+int64_t migrate_xbzrle_cache_size(void)
+{
+    MigrationState *s;
+
+    s = migrate_get_current();
+
+    return s->xbzrle_cache_size;
+}
+
+/* migration thread support */
+/*
+ * Something bad happened to the RP stream, mark an error
+ * The caller shall print or trace something to indicate why
+ */
+static void mark_source_rp_bad(MigrationState *s)
+{
+    s->rp_state.error = true;
+}
+
+static struct rp_cmd_args {
+    ssize_t     len; /* -1 = variable */
+    const char *name;
+} rp_cmd_args[] = {
+    [MIG_RP_MSG_INVALID]        = { .len = -1, .name = "INVALID" },
+    [MIG_RP_MSG_SHUT]           = { .len =  4, .name = "SHUT" },
+    [MIG_RP_MSG_PONG]           = { .len =  4, .name = "PONG" },
+    [MIG_RP_MSG_REQ_PAGES]      = { .len = 12, .name = "REQ_PAGES" },
+    [MIG_RP_MSG_REQ_PAGES_ID]   = { .len = -1, .name = "REQ_PAGES_ID" },
+    [MIG_RP_MSG_MAX]            = { .len = -1, .name = "MAX" },
+};
+
+/*
+ * Process a request for pages received on the return path,
+ * We're allowed to send more than requested (e.g. to round to our page size)
+ * and we don't need to send pages that have already been sent.
+ */
+static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname,
+                                       ram_addr_t start, size_t len)
+{
+    long our_host_ps = getpagesize();
+
+    trace_migrate_handle_rp_req_pages(rbname, start, len);
+
+    /*
+     * Since we currently insist on matching page sizes, just sanity check
+     * we're being asked for whole host pages.
+     */
+    if (start & (our_host_ps-1) ||
+       (len & (our_host_ps-1))) {
+        error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT
+                     " len: %zd", __func__, start, len);
+        mark_source_rp_bad(ms);
+        return;
+    }
+
+    if (ram_save_queue_pages(ms, rbname, start, len)) {
+        mark_source_rp_bad(ms);
+    }
+}
+
+/*
+ * Handles messages sent on the return path towards the source VM
+ *
+ */
+static void *source_return_path_thread(void *opaque)
+{
+    MigrationState *ms = opaque;
+    QEMUFile *rp = ms->rp_state.from_dst_file;
+    uint16_t header_len, header_type;
+    uint8_t buf[512];
+    uint32_t tmp32, sibling_error;
+    ram_addr_t start = 0; /* =0 to silence warning */
+    size_t  len = 0, expected_len;
+    int res;
+
+    trace_source_return_path_thread_entry();
+    while (!ms->rp_state.error && !qemu_file_get_error(rp) &&
+           migration_is_setup_or_active(ms->state)) {
+        trace_source_return_path_thread_loop_top();
+        header_type = qemu_get_be16(rp);
+        header_len = qemu_get_be16(rp);
+
+        if (header_type >= MIG_RP_MSG_MAX ||
+            header_type == MIG_RP_MSG_INVALID) {
+            error_report("RP: Received invalid message 0x%04x length 0x%04x",
+                    header_type, header_len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        if ((rp_cmd_args[header_type].len != -1 &&
+            header_len != rp_cmd_args[header_type].len) ||
+            header_len > sizeof(buf)) {
+            error_report("RP: Received '%s' message (0x%04x) with"
+                    "incorrect length %d expecting %zu",
+                    rp_cmd_args[header_type].name, header_type, header_len,
+                    (size_t)rp_cmd_args[header_type].len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        /* We know we've got a valid header by this point */
+        res = qemu_get_buffer(rp, buf, header_len);
+        if (res != header_len) {
+            error_report("RP: Failed reading data for message 0x%04x"
+                         " read %d expected %d",
+                         header_type, res, header_len);
+            mark_source_rp_bad(ms);
+            goto out;
+        }
+
+        /* OK, we have the message and the data */
+        switch (header_type) {
+        case MIG_RP_MSG_SHUT:
+            sibling_error = ldl_be_p(buf);
+            trace_source_return_path_thread_shut(sibling_error);
+            if (sibling_error) {
+                error_report("RP: Sibling indicated error %d", sibling_error);
+                mark_source_rp_bad(ms);
+            }
+            /*
+             * We'll let the main thread deal with closing the RP
+             * we could do a shutdown(2) on it, but we're the only user
+             * anyway, so there's nothing gained.
+             */
+            goto out;
+
+        case MIG_RP_MSG_PONG:
+            tmp32 = ldl_be_p(buf);
+            trace_source_return_path_thread_pong(tmp32);
+            break;
+
+        case MIG_RP_MSG_REQ_PAGES:
+            start = ldq_be_p(buf);
+            len = ldl_be_p(buf + 8);
+            migrate_handle_rp_req_pages(ms, NULL, start, len);
+            break;
+
+        case MIG_RP_MSG_REQ_PAGES_ID:
+            expected_len = 12 + 1; /* header + termination */
+
+            if (header_len >= expected_len) {
+                start = ldq_be_p(buf);
+                len = ldl_be_p(buf + 8);
+                /* Now we expect an idstr */
+                tmp32 = buf[12]; /* Length of the following idstr */
+                buf[13 + tmp32] = '\0';
+                expected_len += tmp32;
+            }
+            if (header_len != expected_len) {
+                error_report("RP: Req_Page_id with length %d expecting %zd",
+                        header_len, expected_len);
+                mark_source_rp_bad(ms);
+                goto out;
+            }
+            migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len);
+            break;
+
+        default:
+            break;
+        }
+    }
+    if (qemu_file_get_error(rp)) {
+        trace_source_return_path_thread_bad_end();
+        mark_source_rp_bad(ms);
+    }
+
+    trace_source_return_path_thread_end();
+out:
+    ms->rp_state.from_dst_file = NULL;
+    qemu_fclose(rp);
+    return NULL;
+}
+
+static int open_return_path_on_source(MigrationState *ms)
+{
+
+    ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file);
+    if (!ms->rp_state.from_dst_file) {
+        return -1;
+    }
+
+    trace_open_return_path_on_source();
+    qemu_thread_create(&ms->rp_state.rp_thread, "return path",
+                       source_return_path_thread, ms, QEMU_THREAD_JOINABLE);
+
+    trace_open_return_path_on_source_continue();
+
+    return 0;
+}
+
+/* Returns 0 if the RP was ok, otherwise there was an error on the RP */
+static int await_return_path_close_on_source(MigrationState *ms)
+{
+    /*
+     * If this is a normal exit then the destination will send a SHUT and the
+     * rp_thread will exit, however if there's an error we need to cause
+     * it to exit.
+     */
+    if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) {
+        /*
+         * shutdown(2), if we have it, will cause it to unblock if it's stuck
+         * waiting for the destination.
+         */
+        qemu_file_shutdown(ms->rp_state.from_dst_file);
+        mark_source_rp_bad(ms);
+    }
+    trace_await_return_path_close_on_source_joining();
+    qemu_thread_join(&ms->rp_state.rp_thread);
+    trace_await_return_path_close_on_source_close();
+    return ms->rp_state.error;
+}
+
+/*
+ * Switch from normal iteration to postcopy
+ * Returns non-0 on error
+ */
+static int postcopy_start(MigrationState *ms, bool *old_vm_running)
+{
+    int ret;
+    QIOChannelBuffer *bioc;
+    QEMUFile *fb;
+    int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE,
+                      MIGRATION_STATUS_POSTCOPY_ACTIVE);
+
+    trace_postcopy_start();
+    qemu_mutex_lock_iothread();
+    trace_postcopy_start_set_run();
+
+    qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+    *old_vm_running = runstate_is_running();
+    global_state_store();
+    ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+    if (ret < 0) {
+        goto fail;
+    }
+
+    ret = bdrv_inactivate_all();
+    if (ret < 0) {
+        goto fail;
+    }
+
+    /*
+     * Cause any non-postcopiable, but iterative devices to
+     * send out their final data.
+     */
+    qemu_savevm_state_complete_precopy(ms->to_dst_file, true);
+
+    /*
+     * in Finish migrate and with the io-lock held everything should
+     * be quiet, but we've potentially still got dirty pages and we
+     * need to tell the destination to throw any pages it's already received
+     * that are dirty
+     */
+    if (ram_postcopy_send_discard_bitmap(ms)) {
+        error_report("postcopy send discard bitmap failed");
+        goto fail;
+    }
+
+    /*
+     * send rest of state - note things that are doing postcopy
+     * will notice we're in POSTCOPY_ACTIVE and not actually
+     * wrap their state up here
+     */
+    qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX);
+    /* Ping just for debugging, helps line traces up */
+    qemu_savevm_send_ping(ms->to_dst_file, 2);
+
+    /*
+     * While loading the device state we may trigger page transfer
+     * requests and the fd must be free to process those, and thus
+     * the destination must read the whole device state off the fd before
+     * it starts processing it.  Unfortunately the ad-hoc migration format
+     * doesn't allow the destination to know the size to read without fully
+     * parsing it through each devices load-state code (especially the open
+     * coded devices that use get/put).
+     * So we wrap the device state up in a package with a length at the start;
+     * to do this we use a qemu_buf to hold the whole of the device state.
+     */
+    bioc = qio_channel_buffer_new(4096);
+    fb = qemu_fopen_channel_output(QIO_CHANNEL(bioc));
+    object_unref(OBJECT(bioc));
+
+    /*
+     * Make sure the receiver can get incoming pages before we send the rest
+     * of the state
+     */
+    qemu_savevm_send_postcopy_listen(fb);
+
+    qemu_savevm_state_complete_precopy(fb, false);
+    qemu_savevm_send_ping(fb, 3);
+
+    qemu_savevm_send_postcopy_run(fb);
+
+    /* <><> end of stuff going into the package */
+
+    /* Now send that blob */
+    if (qemu_savevm_send_packaged(ms->to_dst_file, bioc->data, bioc->usage)) {
+        goto fail_closefb;
+    }
+    qemu_fclose(fb);
+
+    /* Send a notify to give a chance for anything that needs to happen
+     * at the transition to postcopy and after the device state; in particular
+     * spice needs to trigger a transition now
+     */
+    ms->postcopy_after_devices = true;
+    notifier_list_notify(&migration_state_notifiers, ms);
+
+    ms->downtime =  qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop;
+
+    qemu_mutex_unlock_iothread();
+
+    /*
+     * Although this ping is just for debug, it could potentially be
+     * used for getting a better measurement of downtime at the source.
+     */
+    qemu_savevm_send_ping(ms->to_dst_file, 4);
+
+    ret = qemu_file_get_error(ms->to_dst_file);
+    if (ret) {
+        error_report("postcopy_start: Migration stream errored");
+        migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                              MIGRATION_STATUS_FAILED);
+    }
+
+    return ret;
+
+fail_closefb:
+    qemu_fclose(fb);
+fail:
+    migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                          MIGRATION_STATUS_FAILED);
+    qemu_mutex_unlock_iothread();
+    return -1;
+}
+
+/**
+ * migration_completion: Used by migration_thread when there's not much left.
+ *   The caller 'breaks' the loop when this returns.
+ *
+ * @s: Current migration state
+ * @current_active_state: The migration state we expect to be in
+ * @*old_vm_running: Pointer to old_vm_running flag
+ * @*start_time: Pointer to time to update
+ */
+static void migration_completion(MigrationState *s, int current_active_state,
+                                 bool *old_vm_running,
+                                 int64_t *start_time)
+{
+    int ret;
+
+    if (s->state == MIGRATION_STATUS_ACTIVE) {
+        qemu_mutex_lock_iothread();
+        *start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+        qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
+        *old_vm_running = runstate_is_running();
+        ret = global_state_store();
+
+        if (!ret) {
+            ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
+            if (ret >= 0) {
+                ret = bdrv_inactivate_all();
+            }
+            if (ret >= 0) {
+                qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX);
+                qemu_savevm_state_complete_precopy(s->to_dst_file, false);
+            }
+        }
+        qemu_mutex_unlock_iothread();
+
+        if (ret < 0) {
+            goto fail;
+        }
+    } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
+        trace_migration_completion_postcopy_end();
+
+        qemu_savevm_state_complete_postcopy(s->to_dst_file);
+        trace_migration_completion_postcopy_end_after_complete();
+    }
+
+    /*
+     * If rp was opened we must clean up the thread before
+     * cleaning everything else up (since if there are no failures
+     * it will wait for the destination to send it's status in
+     * a SHUT command).
+     * Postcopy opens rp if enabled (even if it's not avtivated)
+     */
+    if (migrate_postcopy_ram()) {
+        int rp_error;
+        trace_migration_completion_postcopy_end_before_rp();
+        rp_error = await_return_path_close_on_source(s);
+        trace_migration_completion_postcopy_end_after_rp(rp_error);
+        if (rp_error) {
+            goto fail_invalidate;
+        }
+    }
+
+    if (qemu_file_get_error(s->to_dst_file)) {
+        trace_migration_completion_file_err();
+        goto fail_invalidate;
+    }
+
+    migrate_set_state(&s->state, current_active_state,
+                      MIGRATION_STATUS_COMPLETED);
+    return;
+
+fail_invalidate:
+    /* If not doing postcopy, vm_start() will be called: let's regain
+     * control on images.
+     */
+    if (s->state == MIGRATION_STATUS_ACTIVE) {
+        Error *local_err = NULL;
+
+        bdrv_invalidate_cache_all(&local_err);
+        if (local_err) {
+            error_report_err(local_err);
+        }
+    }
+
+fail:
+    migrate_set_state(&s->state, current_active_state,
+                      MIGRATION_STATUS_FAILED);
+}
+
+/*
+ * Master migration thread on the source VM.
+ * It drives the migration and pumps the data down the outgoing channel.
+ */
+static void *migration_thread(void *opaque)
+{
+    MigrationState *s = opaque;
+    /* Used by the bandwidth calcs, updated later */
+    int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST);
+    int64_t initial_bytes = 0;
+    int64_t max_size = 0;
+    int64_t start_time = initial_time;
+    int64_t end_time;
+    bool old_vm_running = false;
+    bool entered_postcopy = false;
+    /* The active state we expect to be in; ACTIVE or POSTCOPY_ACTIVE */
+    enum MigrationStatus current_active_state = MIGRATION_STATUS_ACTIVE;
+
+    rcu_register_thread();
+
+    qemu_savevm_state_header(s->to_dst_file);
+
+    if (migrate_postcopy_ram()) {
+        /* Now tell the dest that it should open its end so it can reply */
+        qemu_savevm_send_open_return_path(s->to_dst_file);
+
+        /* And do a ping that will make stuff easier to debug */
+        qemu_savevm_send_ping(s->to_dst_file, 1);
+
+        /*
+         * Tell the destination that we *might* want to do postcopy later;
+         * if the other end can't do postcopy it should fail now, nice and
+         * early.
+         */
+        qemu_savevm_send_postcopy_advise(s->to_dst_file);
+    }
+
+    qemu_savevm_state_begin(s->to_dst_file, &s->params);
+
+    s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
+    current_active_state = MIGRATION_STATUS_ACTIVE;
+    migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                      MIGRATION_STATUS_ACTIVE);
+
+    trace_migration_thread_setup_complete();
+
+    while (s->state == MIGRATION_STATUS_ACTIVE ||
+           s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) {
+        int64_t current_time;
+        uint64_t pending_size;
+
+        if (!qemu_file_rate_limit(s->to_dst_file)) {
+            uint64_t pend_post, pend_nonpost;
+
+            qemu_savevm_state_pending(s->to_dst_file, max_size, &pend_nonpost,
+                                      &pend_post);
+            pending_size = pend_nonpost + pend_post;
+            trace_migrate_pending(pending_size, max_size,
+                                  pend_post, pend_nonpost);
+            if (pending_size && pending_size >= max_size) {
+                /* Still a significant amount to transfer */
+
+                if (migrate_postcopy_ram() &&
+                    s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE &&
+                    pend_nonpost <= max_size &&
+                    atomic_read(&s->start_postcopy)) {
+
+                    if (!postcopy_start(s, &old_vm_running)) {
+                        current_active_state = MIGRATION_STATUS_POSTCOPY_ACTIVE;
+                        entered_postcopy = true;
+                    }
+
+                    continue;
+                }
+                /* Just another iteration step */
+                qemu_savevm_state_iterate(s->to_dst_file, entered_postcopy);
+            } else {
+                trace_migration_thread_low_pending(pending_size);
+                migration_completion(s, current_active_state,
+                                     &old_vm_running, &start_time);
+                break;
+            }
+        }
+
+        if (qemu_file_get_error(s->to_dst_file)) {
+            migrate_set_state(&s->state, current_active_state,
+                              MIGRATION_STATUS_FAILED);
+            trace_migration_thread_file_err();
+            break;
+        }
+        current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+        if (current_time >= initial_time + BUFFER_DELAY) {
+            uint64_t transferred_bytes = qemu_ftell(s->to_dst_file) -
+                                         initial_bytes;
+            uint64_t time_spent = current_time - initial_time;
+            double bandwidth = (double)transferred_bytes / time_spent;
+            max_size = bandwidth * migrate_max_downtime() / 1000000;
+
+            s->mbps = (((double) transferred_bytes * 8.0) /
+                    ((double) time_spent / 1000.0)) / 1000.0 / 1000.0;
+
+            trace_migrate_transferred(transferred_bytes, time_spent,
+                                      bandwidth, max_size);
+            /* if we haven't sent anything, we don't want to recalculate
+               10000 is a small enough number for our purposes */
+            if (s->dirty_bytes_rate && transferred_bytes > 10000) {
+                s->expected_downtime = s->dirty_bytes_rate / bandwidth;
+            }
+
+            qemu_file_reset_rate_limit(s->to_dst_file);
+            initial_time = current_time;
+            initial_bytes = qemu_ftell(s->to_dst_file);
+        }
+        if (qemu_file_rate_limit(s->to_dst_file)) {
+            /* usleep expects microseconds */
+            g_usleep((initial_time + BUFFER_DELAY - current_time)*1000);
+        }
+    }
+
+    trace_migration_thread_after_loop();
+    /* If we enabled cpu throttling for auto-converge, turn it off. */
+    cpu_throttle_stop();
+    end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+
+    qemu_mutex_lock_iothread();
+    qemu_savevm_state_cleanup();
+    if (s->state == MIGRATION_STATUS_COMPLETED) {
+        uint64_t transferred_bytes = qemu_ftell(s->to_dst_file);
+        s->total_time = end_time - s->total_time;
+        if (!entered_postcopy) {
+            s->downtime = end_time - start_time;
+        }
+        if (s->total_time) {
+            s->mbps = (((double) transferred_bytes * 8.0) /
+                       ((double) s->total_time)) / 1000;
+        }
+        runstate_set(RUN_STATE_POSTMIGRATE);
+    } else {
+        if (old_vm_running && !entered_postcopy) {
+            vm_start();
+        } else {
+            if (runstate_check(RUN_STATE_FINISH_MIGRATE)) {
+                runstate_set(RUN_STATE_POSTMIGRATE);
+            }
+        }
+    }
+    qemu_bh_schedule(s->cleanup_bh);
+    qemu_mutex_unlock_iothread();
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+void migrate_fd_connect(MigrationState *s)
+{
+    /* This is a best 1st approximation. ns to ms */
+    s->expected_downtime = max_downtime/1000000;
+    s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s);
+
+    qemu_file_set_blocking(s->to_dst_file, true);
+    qemu_file_set_rate_limit(s->to_dst_file,
+                             s->bandwidth_limit / XFER_LIMIT_RATIO);
+
+    /* Notify before starting migration thread */
+    notifier_list_notify(&migration_state_notifiers, s);
+
+    /*
+     * Open the return path; currently for postcopy but other things might
+     * also want it.
+     */
+    if (migrate_postcopy_ram()) {
+        if (open_return_path_on_source(s)) {
+            error_report("Unable to open return-path for postcopy");
+            migrate_set_state(&s->state, MIGRATION_STATUS_SETUP,
+                              MIGRATION_STATUS_FAILED);
+            migrate_fd_cleanup(s);
+            return;
+        }
+    }
+
+    migrate_compress_threads_create();
+    qemu_thread_create(&s->thread, "migration", migration_thread, s,
+                       QEMU_THREAD_JOINABLE);
+    s->migration_thread_running = true;
+}
+
+PostcopyState  postcopy_state_get(void)
+{
+    return atomic_mb_read(&incoming_postcopy_state);
+}
+
+/* Set the state and return the old state */
+PostcopyState postcopy_state_set(PostcopyState new_state)
+{
+    return atomic_xchg(&incoming_postcopy_state, new_state);
+}
+
diff --git a/migration/postcopy-ram.c b/migration/postcopy-ram.c
new file mode 100644
index 0000000..9b04778
--- /dev/null
+++ b/migration/postcopy-ram.c
@@ -0,0 +1,759 @@
+/*
+ * Postcopy migration for RAM
+ *
+ * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Dave Gilbert  <dgilbert@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+/*
+ * Postcopy is a migration technique where the execution flips from the
+ * source to the destination before all the data has been copied.
+ */
+
+#include "qemu/osdep.h"
+
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "migration/postcopy-ram.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/balloon.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+
+/* Arbitrary limit on size of each discard command,
+ * keeps them around ~200 bytes
+ */
+#define MAX_DISCARDS_PER_COMMAND 12
+
+struct PostcopyDiscardState {
+    const char *ramblock_name;
+    uint64_t offset; /* Bitmap entry for the 1st bit of this RAMBlock */
+    uint16_t cur_entry;
+    /*
+     * Start and length of a discard range (bytes)
+     */
+    uint64_t start_list[MAX_DISCARDS_PER_COMMAND];
+    uint64_t length_list[MAX_DISCARDS_PER_COMMAND];
+    unsigned int nsentwords;
+    unsigned int nsentcmds;
+};
+
+/* Postcopy needs to detect accesses to pages that haven't yet been copied
+ * across, and efficiently map new pages in, the techniques for doing this
+ * are target OS specific.
+ */
+#if defined(__linux__)
+
+#include <poll.h>
+#include <sys/ioctl.h>
+#include <sys/syscall.h>
+#include <asm/types.h> /* for __u64 */
+#endif
+
+#if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD)
+#include <sys/eventfd.h>
+#include <linux/userfaultfd.h>
+
+static bool ufd_version_check(int ufd)
+{
+    struct uffdio_api api_struct;
+    uint64_t ioctl_mask;
+
+    api_struct.api = UFFD_API;
+    api_struct.features = 0;
+    if (ioctl(ufd, UFFDIO_API, &api_struct)) {
+        error_report("postcopy_ram_supported_by_host: UFFDIO_API failed: %s",
+                     strerror(errno));
+        return false;
+    }
+
+    ioctl_mask = (__u64)1 << _UFFDIO_REGISTER |
+                 (__u64)1 << _UFFDIO_UNREGISTER;
+    if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
+        error_report("Missing userfault features: %" PRIx64,
+                     (uint64_t)(~api_struct.ioctls & ioctl_mask));
+        return false;
+    }
+
+    return true;
+}
+
+/*
+ * Note: This has the side effect of munlock'ing all of RAM, that's
+ * normally fine since if the postcopy succeeds it gets turned back on at the
+ * end.
+ */
+bool postcopy_ram_supported_by_host(void)
+{
+    long pagesize = getpagesize();
+    int ufd = -1;
+    bool ret = false; /* Error unless we change it */
+    void *testarea = NULL;
+    struct uffdio_register reg_struct;
+    struct uffdio_range range_struct;
+    uint64_t feature_mask;
+
+    if ((1ul << qemu_target_page_bits()) > pagesize) {
+        error_report("Target page size bigger than host page size");
+        goto out;
+    }
+
+    ufd = syscall(__NR_userfaultfd, O_CLOEXEC);
+    if (ufd == -1) {
+        error_report("%s: userfaultfd not available: %s", __func__,
+                     strerror(errno));
+        goto out;
+    }
+
+    /* Version and features check */
+    if (!ufd_version_check(ufd)) {
+        goto out;
+    }
+
+    /*
+     * userfault and mlock don't go together; we'll put it back later if
+     * it was enabled.
+     */
+    if (munlockall()) {
+        error_report("%s: munlockall: %s", __func__,  strerror(errno));
+        return -1;
+    }
+
+    /*
+     *  We need to check that the ops we need are supported on anon memory
+     *  To do that we need to register a chunk and see the flags that
+     *  are returned.
+     */
+    testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE |
+                                    MAP_ANONYMOUS, -1, 0);
+    if (testarea == MAP_FAILED) {
+        error_report("%s: Failed to map test area: %s", __func__,
+                     strerror(errno));
+        goto out;
+    }
+    g_assert(((size_t)testarea & (pagesize-1)) == 0);
+
+    reg_struct.range.start = (uintptr_t)testarea;
+    reg_struct.range.len = pagesize;
+    reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+    if (ioctl(ufd, UFFDIO_REGISTER, &reg_struct)) {
+        error_report("%s userfault register: %s", __func__, strerror(errno));
+        goto out;
+    }
+
+    range_struct.start = (uintptr_t)testarea;
+    range_struct.len = pagesize;
+    if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) {
+        error_report("%s userfault unregister: %s", __func__, strerror(errno));
+        goto out;
+    }
+
+    feature_mask = (__u64)1 << _UFFDIO_WAKE |
+                   (__u64)1 << _UFFDIO_COPY |
+                   (__u64)1 << _UFFDIO_ZEROPAGE;
+    if ((reg_struct.ioctls & feature_mask) != feature_mask) {
+        error_report("Missing userfault map features: %" PRIx64,
+                     (uint64_t)(~reg_struct.ioctls & feature_mask));
+        goto out;
+    }
+
+    /* Success! */
+    ret = true;
+out:
+    if (testarea) {
+        munmap(testarea, pagesize);
+    }
+    if (ufd != -1) {
+        close(ufd);
+    }
+    return ret;
+}
+
+/**
+ * postcopy_ram_discard_range: Discard a range of memory.
+ * We can assume that if we've been called postcopy_ram_hosttest returned true.
+ *
+ * @mis: Current incoming migration state.
+ * @start, @length: range of memory to discard.
+ *
+ * returns: 0 on success.
+ */
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length)
+{
+    trace_postcopy_ram_discard_range(start, length);
+    if (madvise(start, length, MADV_DONTNEED)) {
+        error_report("%s MADV_DONTNEED: %s", __func__, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Setup an area of RAM so that it *can* be used for postcopy later; this
+ * must be done right at the start prior to pre-copy.
+ * opaque should be the MIS.
+ */
+static int init_range(const char *block_name, void *host_addr,
+                      ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+
+    trace_postcopy_init_range(block_name, host_addr, offset, length);
+
+    /*
+     * We need the whole of RAM to be truly empty for postcopy, so things
+     * like ROMs and any data tables built during init must be zero'd
+     * - we're going to get the copy from the source anyway.
+     * (Precopy will just overwrite this data, so doesn't need the discard)
+     */
+    if (postcopy_ram_discard_range(mis, host_addr, length)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * At the end of migration, undo the effects of init_range
+ * opaque should be the MIS.
+ */
+static int cleanup_range(const char *block_name, void *host_addr,
+                        ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffdio_range range_struct;
+    trace_postcopy_cleanup_range(block_name, host_addr, offset, length);
+
+    /*
+     * We turned off hugepage for the precopy stage with postcopy enabled
+     * we can turn it back on now.
+     */
+    qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE);
+
+    /*
+     * We can also turn off userfault now since we should have all the
+     * pages.   It can be useful to leave it on to debug postcopy
+     * if you're not sure it's always getting every page.
+     */
+    range_struct.start = (uintptr_t)host_addr;
+    range_struct.len = length;
+
+    if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) {
+        error_report("%s: userfault unregister %s", __func__, strerror(errno));
+
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Initialise postcopy-ram, setting the RAM to a state where we can go into
+ * postcopy later; must be called prior to any precopy.
+ * called from arch_init's similarly named ram_postcopy_incoming_init
+ */
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages)
+{
+    if (qemu_ram_foreach_block(init_range, mis)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * At the end of a migration where postcopy_ram_incoming_init was called.
+ */
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
+{
+    trace_postcopy_ram_incoming_cleanup_entry();
+
+    if (mis->have_fault_thread) {
+        uint64_t tmp64;
+
+        if (qemu_ram_foreach_block(cleanup_range, mis)) {
+            return -1;
+        }
+        /*
+         * Tell the fault_thread to exit, it's an eventfd that should
+         * currently be at 0, we're going to increment it to 1
+         */
+        tmp64 = 1;
+        if (write(mis->userfault_quit_fd, &tmp64, 8) == 8) {
+            trace_postcopy_ram_incoming_cleanup_join();
+            qemu_thread_join(&mis->fault_thread);
+        } else {
+            /* Not much we can do here, but may as well report it */
+            error_report("%s: incrementing userfault_quit_fd: %s", __func__,
+                         strerror(errno));
+        }
+        trace_postcopy_ram_incoming_cleanup_closeuf();
+        close(mis->userfault_fd);
+        close(mis->userfault_quit_fd);
+        mis->have_fault_thread = false;
+    }
+
+    qemu_balloon_inhibit(false);
+
+    if (enable_mlock) {
+        if (os_mlock() < 0) {
+            error_report("mlock: %s", strerror(errno));
+            /*
+             * It doesn't feel right to fail at this point, we have a valid
+             * VM state.
+             */
+        }
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_END);
+    migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0);
+
+    if (mis->postcopy_tmp_page) {
+        munmap(mis->postcopy_tmp_page, getpagesize());
+        mis->postcopy_tmp_page = NULL;
+    }
+    trace_postcopy_ram_incoming_cleanup_exit();
+    return 0;
+}
+
+/*
+ * Disable huge pages on an area
+ */
+static int nhp_range(const char *block_name, void *host_addr,
+                    ram_addr_t offset, ram_addr_t length, void *opaque)
+{
+    trace_postcopy_nhp_range(block_name, host_addr, offset, length);
+
+    /*
+     * Before we do discards we need to ensure those discards really
+     * do delete areas of the page, even if THP thinks a hugepage would
+     * be a good idea, so force hugepages off.
+     */
+    qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE);
+
+    return 0;
+}
+
+/*
+ * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard
+ * however leaving it until after precopy means that most of the precopy
+ * data is still THPd
+ */
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
+{
+    if (qemu_ram_foreach_block(nhp_range, mis)) {
+        return -1;
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_DISCARD);
+
+    return 0;
+}
+
+/*
+ * Mark the given area of RAM as requiring notification to unwritten areas
+ * Used as a  callback on qemu_ram_foreach_block.
+ *   host_addr: Base of area to mark
+ *   offset: Offset in the whole ram arena
+ *   length: Length of the section
+ *   opaque: MigrationIncomingState pointer
+ * Returns 0 on success
+ */
+static int ram_block_enable_notify(const char *block_name, void *host_addr,
+                                   ram_addr_t offset, ram_addr_t length,
+                                   void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffdio_register reg_struct;
+
+    reg_struct.range.start = (uintptr_t)host_addr;
+    reg_struct.range.len = length;
+    reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING;
+
+    /* Now tell our userfault_fd that it's responsible for this area */
+    if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, &reg_struct)) {
+        error_report("%s userfault register: %s", __func__, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Handle faults detected by the USERFAULT markings
+ */
+static void *postcopy_ram_fault_thread(void *opaque)
+{
+    MigrationIncomingState *mis = opaque;
+    struct uffd_msg msg;
+    int ret;
+    size_t hostpagesize = getpagesize();
+    RAMBlock *rb = NULL;
+    RAMBlock *last_rb = NULL; /* last RAMBlock we sent part of */
+
+    trace_postcopy_ram_fault_thread_entry();
+    qemu_sem_post(&mis->fault_thread_sem);
+
+    while (true) {
+        ram_addr_t rb_offset;
+        struct pollfd pfd[2];
+
+        /*
+         * We're mainly waiting for the kernel to give us a faulting HVA,
+         * however we can be told to quit via userfault_quit_fd which is
+         * an eventfd
+         */
+        pfd[0].fd = mis->userfault_fd;
+        pfd[0].events = POLLIN;
+        pfd[0].revents = 0;
+        pfd[1].fd = mis->userfault_quit_fd;
+        pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */
+        pfd[1].revents = 0;
+
+        if (poll(pfd, 2, -1 /* Wait forever */) == -1) {
+            error_report("%s: userfault poll: %s", __func__, strerror(errno));
+            break;
+        }
+
+        if (pfd[1].revents) {
+            trace_postcopy_ram_fault_thread_quit();
+            break;
+        }
+
+        ret = read(mis->userfault_fd, &msg, sizeof(msg));
+        if (ret != sizeof(msg)) {
+            if (errno == EAGAIN) {
+                /*
+                 * if a wake up happens on the other thread just after
+                 * the poll, there is nothing to read.
+                 */
+                continue;
+            }
+            if (ret < 0) {
+                error_report("%s: Failed to read full userfault message: %s",
+                             __func__, strerror(errno));
+                break;
+            } else {
+                error_report("%s: Read %d bytes from userfaultfd expected %zd",
+                             __func__, ret, sizeof(msg));
+                break; /* Lost alignment, don't know what we'd read next */
+            }
+        }
+        if (msg.event != UFFD_EVENT_PAGEFAULT) {
+            error_report("%s: Read unexpected event %ud from userfaultfd",
+                         __func__, msg.event);
+            continue; /* It's not a page fault, shouldn't happen */
+        }
+
+        rb = qemu_ram_block_from_host(
+                 (void *)(uintptr_t)msg.arg.pagefault.address,
+                 true, &rb_offset);
+        if (!rb) {
+            error_report("postcopy_ram_fault_thread: Fault outside guest: %"
+                         PRIx64, (uint64_t)msg.arg.pagefault.address);
+            break;
+        }
+
+        rb_offset &= ~(hostpagesize - 1);
+        trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address,
+                                                qemu_ram_get_idstr(rb),
+                                                rb_offset);
+
+        /*
+         * Send the request to the source - we want to request one
+         * of our host page sizes (which is >= TPS)
+         */
+        if (rb != last_rb) {
+            last_rb = rb;
+            migrate_send_rp_req_pages(mis, qemu_ram_get_idstr(rb),
+                                     rb_offset, hostpagesize);
+        } else {
+            /* Save some space */
+            migrate_send_rp_req_pages(mis, NULL,
+                                     rb_offset, hostpagesize);
+        }
+    }
+    trace_postcopy_ram_fault_thread_exit();
+    return NULL;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+    /* Open the fd for the kernel to give us userfaults */
+    mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
+    if (mis->userfault_fd == -1) {
+        error_report("%s: Failed to open userfault fd: %s", __func__,
+                     strerror(errno));
+        return -1;
+    }
+
+    /*
+     * Although the host check already tested the API, we need to
+     * do the check again as an ABI handshake on the new fd.
+     */
+    if (!ufd_version_check(mis->userfault_fd)) {
+        return -1;
+    }
+
+    /* Now an eventfd we use to tell the fault-thread to quit */
+    mis->userfault_quit_fd = eventfd(0, EFD_CLOEXEC);
+    if (mis->userfault_quit_fd == -1) {
+        error_report("%s: Opening userfault_quit_fd: %s", __func__,
+                     strerror(errno));
+        close(mis->userfault_fd);
+        return -1;
+    }
+
+    qemu_sem_init(&mis->fault_thread_sem, 0);
+    qemu_thread_create(&mis->fault_thread, "postcopy/fault",
+                       postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE);
+    qemu_sem_wait(&mis->fault_thread_sem);
+    qemu_sem_destroy(&mis->fault_thread_sem);
+    mis->have_fault_thread = true;
+
+    /* Mark so that we get notified of accesses to unwritten areas */
+    if (qemu_ram_foreach_block(ram_block_enable_notify, mis)) {
+        return -1;
+    }
+
+    /*
+     * Ballooning can mark pages as absent while we're postcopying
+     * that would cause false userfaults.
+     */
+    qemu_balloon_inhibit(true);
+
+    trace_postcopy_ram_enable_notify();
+
+    return 0;
+}
+
+/*
+ * Place a host page (from) at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from)
+{
+    struct uffdio_copy copy_struct;
+
+    copy_struct.dst = (uint64_t)(uintptr_t)host;
+    copy_struct.src = (uint64_t)(uintptr_t)from;
+    copy_struct.len = getpagesize();
+    copy_struct.mode = 0;
+
+    /* copy also acks to the kernel waking the stalled thread up
+     * TODO: We can inhibit that ack and only do it if it was requested
+     * which would be slightly cheaper, but we'd have to be careful
+     * of the order of updating our page state.
+     */
+    if (ioctl(mis->userfault_fd, UFFDIO_COPY, &copy_struct)) {
+        int e = errno;
+        error_report("%s: %s copy host: %p from: %p",
+                     __func__, strerror(e), host, from);
+
+        return -e;
+    }
+
+    trace_postcopy_place_page(host);
+    return 0;
+}
+
+/*
+ * Place a zero page at (host) atomically
+ * returns 0 on success
+ */
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host)
+{
+    struct uffdio_zeropage zero_struct;
+
+    zero_struct.range.start = (uint64_t)(uintptr_t)host;
+    zero_struct.range.len = getpagesize();
+    zero_struct.mode = 0;
+
+    if (ioctl(mis->userfault_fd, UFFDIO_ZEROPAGE, &zero_struct)) {
+        int e = errno;
+        error_report("%s: %s zero host: %p",
+                     __func__, strerror(e), host);
+
+        return -e;
+    }
+
+    trace_postcopy_place_page_zero(host);
+    return 0;
+}
+
+/*
+ * Returns a target page of memory that can be mapped at a later point in time
+ * using postcopy_place_page
+ * The same address is used repeatedly, postcopy_place_page just takes the
+ * backing page away.
+ * Returns: Pointer to allocated page
+ *
+ */
+void *postcopy_get_tmp_page(MigrationIncomingState *mis)
+{
+    if (!mis->postcopy_tmp_page) {
+        mis->postcopy_tmp_page = mmap(NULL, getpagesize(),
+                             PROT_READ | PROT_WRITE, MAP_PRIVATE |
+                             MAP_ANONYMOUS, -1, 0);
+        if (mis->postcopy_tmp_page == MAP_FAILED) {
+            mis->postcopy_tmp_page = NULL;
+            error_report("%s: %s", __func__, strerror(errno));
+            return NULL;
+        }
+    }
+
+    return mis->postcopy_tmp_page;
+}
+
+#else
+/* No target OS support, stubs just fail */
+bool postcopy_ram_supported_by_host(void)
+{
+    error_report("%s: No OS support", __func__);
+    return false;
+}
+
+int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages)
+{
+    error_report("postcopy_ram_incoming_init: No OS support");
+    return -1;
+}
+
+int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start,
+                               size_t length)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_prepare_discard(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_ram_enable_notify(MigrationIncomingState *mis)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from)
+{
+    assert(0);
+    return -1;
+}
+
+int postcopy_place_page_zero(MigrationIncomingState *mis, void *host)
+{
+    assert(0);
+    return -1;
+}
+
+void *postcopy_get_tmp_page(MigrationIncomingState *mis)
+{
+    assert(0);
+    return NULL;
+}
+
+#endif
+
+/* ------------------------------------------------------------------------- */
+
+/**
+ * postcopy_discard_send_init: Called at the start of each RAMBlock before
+ *   asking to discard individual ranges.
+ *
+ * @ms: The current migration state.
+ * @offset: the bitmap offset of the named RAMBlock in the migration
+ *   bitmap.
+ * @name: RAMBlock that discards will operate on.
+ *
+ * returns: a new PDS.
+ */
+PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms,
+                                                 unsigned long offset,
+                                                 const char *name)
+{
+    PostcopyDiscardState *res = g_malloc0(sizeof(PostcopyDiscardState));
+
+    if (res) {
+        res->ramblock_name = name;
+        res->offset = offset;
+    }
+
+    return res;
+}
+
+/**
+ * postcopy_discard_send_range: Called by the bitmap code for each chunk to
+ *   discard. May send a discard message, may just leave it queued to
+ *   be sent later.
+ *
+ * @ms: Current migration state.
+ * @pds: Structure initialised by postcopy_discard_send_init().
+ * @start,@length: a range of pages in the migration bitmap in the
+ *   RAM block passed to postcopy_discard_send_init() (length=1 is one page)
+ */
+void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds,
+                                unsigned long start, unsigned long length)
+{
+    size_t tp_bits = qemu_target_page_bits();
+    /* Convert to byte offsets within the RAM block */
+    pds->start_list[pds->cur_entry] = (start - pds->offset) << tp_bits;
+    pds->length_list[pds->cur_entry] = length << tp_bits;
+    trace_postcopy_discard_send_range(pds->ramblock_name, start, length);
+    pds->cur_entry++;
+    pds->nsentwords++;
+
+    if (pds->cur_entry == MAX_DISCARDS_PER_COMMAND) {
+        /* Full set, ship it! */
+        qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file,
+                                              pds->ramblock_name,
+                                              pds->cur_entry,
+                                              pds->start_list,
+                                              pds->length_list);
+        pds->nsentcmds++;
+        pds->cur_entry = 0;
+    }
+}
+
+/**
+ * postcopy_discard_send_finish: Called at the end of each RAMBlock by the
+ * bitmap code. Sends any outstanding discard messages, frees the PDS
+ *
+ * @ms: Current migration state.
+ * @pds: Structure initialised by postcopy_discard_send_init().
+ */
+void postcopy_discard_send_finish(MigrationState *ms, PostcopyDiscardState *pds)
+{
+    /* Anything unsent? */
+    if (pds->cur_entry) {
+        qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file,
+                                              pds->ramblock_name,
+                                              pds->cur_entry,
+                                              pds->start_list,
+                                              pds->length_list);
+        pds->nsentcmds++;
+    }
+
+    trace_postcopy_discard_send_finish(pds->ramblock_name, pds->nsentwords,
+                                       pds->nsentcmds);
+
+    g_free(pds);
+}
diff --git a/migration/qemu-file-channel.c b/migration/qemu-file-channel.c
new file mode 100644
index 0000000..45c13f1
--- /dev/null
+++ b/migration/qemu-file-channel.c
@@ -0,0 +1,180 @@
+/*
+ * QEMUFile backend for QIOChannel objects
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "migration/qemu-file.h"
+#include "io/channel-socket.h"
+#include "qemu/iov.h"
+
+
+static ssize_t channel_writev_buffer(void *opaque,
+                                     struct iovec *iov,
+                                     int iovcnt,
+                                     int64_t pos)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+    ssize_t done = 0;
+    struct iovec *local_iov = g_new(struct iovec, iovcnt);
+    struct iovec *local_iov_head = local_iov;
+    unsigned int nlocal_iov = iovcnt;
+
+    nlocal_iov = iov_copy(local_iov, nlocal_iov,
+                          iov, iovcnt,
+                          0, iov_size(iov, iovcnt));
+
+    while (nlocal_iov > 0) {
+        ssize_t len;
+        len = qio_channel_writev(ioc, local_iov, nlocal_iov, NULL);
+        if (len == QIO_CHANNEL_ERR_BLOCK) {
+            qio_channel_wait(ioc, G_IO_OUT);
+            continue;
+        }
+        if (len < 0) {
+            /* XXX handle Error objects */
+            done = -EIO;
+            goto cleanup;
+        }
+
+        iov_discard_front(&local_iov, &nlocal_iov, len);
+        done += len;
+    }
+
+ cleanup:
+    g_free(local_iov_head);
+    return done;
+}
+
+
+static ssize_t channel_get_buffer(void *opaque,
+                                  uint8_t *buf,
+                                  int64_t pos,
+                                  size_t size)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+    ssize_t ret;
+
+    do {
+        ret = qio_channel_read(ioc, (char *)buf, size, NULL);
+        if (ret < 0) {
+            if (ret == QIO_CHANNEL_ERR_BLOCK) {
+                qio_channel_yield(ioc, G_IO_IN);
+            } else {
+                /* XXX handle Error * object */
+                return -EIO;
+            }
+        }
+    } while (ret == QIO_CHANNEL_ERR_BLOCK);
+
+    return ret;
+}
+
+
+static int channel_close(void *opaque)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+    qio_channel_close(ioc, NULL);
+    object_unref(OBJECT(ioc));
+    return 0;
+}
+
+
+static int channel_shutdown(void *opaque,
+                            bool rd,
+                            bool wr)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+
+    if (qio_channel_has_feature(ioc,
+                                QIO_CHANNEL_FEATURE_SHUTDOWN)) {
+        QIOChannelShutdown mode;
+        if (rd && wr) {
+            mode = QIO_CHANNEL_SHUTDOWN_BOTH;
+        } else if (rd) {
+            mode = QIO_CHANNEL_SHUTDOWN_READ;
+        } else {
+            mode = QIO_CHANNEL_SHUTDOWN_WRITE;
+        }
+        if (qio_channel_shutdown(ioc, mode, NULL) < 0) {
+            /* XXX handler Error * object */
+            return -EIO;
+        }
+    }
+    return 0;
+}
+
+
+static int channel_set_blocking(void *opaque,
+                                bool enabled)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+
+    if (qio_channel_set_blocking(ioc, enabled, NULL) < 0) {
+        return -1;
+    }
+    return 0;
+}
+
+static QEMUFile *channel_get_input_return_path(void *opaque)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+
+    return qemu_fopen_channel_output(ioc);
+}
+
+static QEMUFile *channel_get_output_return_path(void *opaque)
+{
+    QIOChannel *ioc = QIO_CHANNEL(opaque);
+
+    return qemu_fopen_channel_input(ioc);
+}
+
+static const QEMUFileOps channel_input_ops = {
+    .get_buffer = channel_get_buffer,
+    .close = channel_close,
+    .shut_down = channel_shutdown,
+    .set_blocking = channel_set_blocking,
+    .get_return_path = channel_get_input_return_path,
+};
+
+
+static const QEMUFileOps channel_output_ops = {
+    .writev_buffer = channel_writev_buffer,
+    .close = channel_close,
+    .shut_down = channel_shutdown,
+    .set_blocking = channel_set_blocking,
+    .get_return_path = channel_get_output_return_path,
+};
+
+
+QEMUFile *qemu_fopen_channel_input(QIOChannel *ioc)
+{
+    object_ref(OBJECT(ioc));
+    return qemu_fopen_ops(ioc, &channel_input_ops);
+}
+
+QEMUFile *qemu_fopen_channel_output(QIOChannel *ioc)
+{
+    object_ref(OBJECT(ioc));
+    return qemu_fopen_ops(ioc, &channel_output_ops);
+}
diff --git a/migration/qemu-file.c b/migration/qemu-file.c
new file mode 100644
index 0000000..e9fae31
--- /dev/null
+++ b/migration/qemu-file.c
@@ -0,0 +1,704 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include <zlib.h>
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "qemu/coroutine.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "trace.h"
+
+#define IO_BUF_SIZE 32768
+#define MAX_IOV_SIZE MIN(IOV_MAX, 64)
+
+struct QEMUFile {
+    const QEMUFileOps *ops;
+    const QEMUFileHooks *hooks;
+    void *opaque;
+
+    int64_t bytes_xfer;
+    int64_t xfer_limit;
+
+    int64_t pos; /* start of buffer when writing, end of buffer
+                    when reading */
+    int buf_index;
+    int buf_size; /* 0 when writing */
+    uint8_t buf[IO_BUF_SIZE];
+
+    struct iovec iov[MAX_IOV_SIZE];
+    unsigned int iovcnt;
+
+    int last_error;
+};
+
+/*
+ * Stop a file from being read/written - not all backing files can do this
+ * typically only sockets can.
+ */
+int qemu_file_shutdown(QEMUFile *f)
+{
+    if (!f->ops->shut_down) {
+        return -ENOSYS;
+    }
+    return f->ops->shut_down(f->opaque, true, true);
+}
+
+/*
+ * Result: QEMUFile* for a 'return path' for comms in the opposite direction
+ *         NULL if not available
+ */
+QEMUFile *qemu_file_get_return_path(QEMUFile *f)
+{
+    if (!f->ops->get_return_path) {
+        return NULL;
+    }
+    return f->ops->get_return_path(f->opaque);
+}
+
+bool qemu_file_mode_is_not_valid(const char *mode)
+{
+    if (mode == NULL ||
+        (mode[0] != 'r' && mode[0] != 'w') ||
+        mode[1] != 'b' || mode[2] != 0) {
+        fprintf(stderr, "qemu_fopen: Argument validity check failed\n");
+        return true;
+    }
+
+    return false;
+}
+
+QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
+{
+    QEMUFile *f;
+
+    f = g_new0(QEMUFile, 1);
+
+    f->opaque = opaque;
+    f->ops = ops;
+    return f;
+}
+
+
+void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks)
+{
+    f->hooks = hooks;
+}
+
+/*
+ * Get last error for stream f
+ *
+ * Return negative error value if there has been an error on previous
+ * operations, return 0 if no error happened.
+ *
+ */
+int qemu_file_get_error(QEMUFile *f)
+{
+    return f->last_error;
+}
+
+void qemu_file_set_error(QEMUFile *f, int ret)
+{
+    if (f->last_error == 0) {
+        f->last_error = ret;
+    }
+}
+
+bool qemu_file_is_writable(QEMUFile *f)
+{
+    return f->ops->writev_buffer;
+}
+
+/**
+ * Flushes QEMUFile buffer
+ *
+ * If there is writev_buffer QEMUFileOps it uses it otherwise uses
+ * put_buffer ops. This will flush all pending data. If data was
+ * only partially flushed, it will set an error state.
+ */
+void qemu_fflush(QEMUFile *f)
+{
+    ssize_t ret = 0;
+    ssize_t expect = 0;
+
+    if (!qemu_file_is_writable(f)) {
+        return;
+    }
+
+    if (f->iovcnt > 0) {
+        expect = iov_size(f->iov, f->iovcnt);
+        ret = f->ops->writev_buffer(f->opaque, f->iov, f->iovcnt, f->pos);
+    }
+
+    if (ret >= 0) {
+        f->pos += ret;
+    }
+    /* We expect the QEMUFile write impl to send the full
+     * data set we requested, so sanity check that.
+     */
+    if (ret != expect) {
+        qemu_file_set_error(f, ret < 0 ? ret : -EIO);
+    }
+    f->buf_index = 0;
+    f->iovcnt = 0;
+}
+
+void ram_control_before_iterate(QEMUFile *f, uint64_t flags)
+{
+    int ret = 0;
+
+    if (f->hooks && f->hooks->before_ram_iterate) {
+        ret = f->hooks->before_ram_iterate(f, f->opaque, flags, NULL);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+        }
+    }
+}
+
+void ram_control_after_iterate(QEMUFile *f, uint64_t flags)
+{
+    int ret = 0;
+
+    if (f->hooks && f->hooks->after_ram_iterate) {
+        ret = f->hooks->after_ram_iterate(f, f->opaque, flags, NULL);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+        }
+    }
+}
+
+void ram_control_load_hook(QEMUFile *f, uint64_t flags, void *data)
+{
+    int ret = -EINVAL;
+
+    if (f->hooks && f->hooks->hook_ram_load) {
+        ret = f->hooks->hook_ram_load(f, f->opaque, flags, data);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+        }
+    } else {
+        /*
+         * Hook is a hook specifically requested by the source sending a flag
+         * that expects there to be a hook on the destination.
+         */
+        if (flags == RAM_CONTROL_HOOK) {
+            qemu_file_set_error(f, ret);
+        }
+    }
+}
+
+size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset,
+                             ram_addr_t offset, size_t size,
+                             uint64_t *bytes_sent)
+{
+    if (f->hooks && f->hooks->save_page) {
+        int ret = f->hooks->save_page(f, f->opaque, block_offset,
+                                      offset, size, bytes_sent);
+
+        if (ret != RAM_SAVE_CONTROL_DELAYED) {
+            if (bytes_sent && *bytes_sent > 0) {
+                qemu_update_position(f, *bytes_sent);
+            } else if (ret < 0) {
+                qemu_file_set_error(f, ret);
+            }
+        }
+
+        return ret;
+    }
+
+    return RAM_SAVE_CONTROL_NOT_SUPP;
+}
+
+/*
+ * Attempt to fill the buffer from the underlying file
+ * Returns the number of bytes read, or negative value for an error.
+ *
+ * Note that it can return a partially full buffer even in a not error/not EOF
+ * case if the underlying file descriptor gives a short read, and that can
+ * happen even on a blocking fd.
+ */
+static ssize_t qemu_fill_buffer(QEMUFile *f)
+{
+    int len;
+    int pending;
+
+    assert(!qemu_file_is_writable(f));
+
+    pending = f->buf_size - f->buf_index;
+    if (pending > 0) {
+        memmove(f->buf, f->buf + f->buf_index, pending);
+    }
+    f->buf_index = 0;
+    f->buf_size = pending;
+
+    len = f->ops->get_buffer(f->opaque, f->buf + pending, f->pos,
+                        IO_BUF_SIZE - pending);
+    if (len > 0) {
+        f->buf_size += len;
+        f->pos += len;
+    } else if (len == 0) {
+        qemu_file_set_error(f, -EIO);
+    } else if (len != -EAGAIN) {
+        qemu_file_set_error(f, len);
+    }
+
+    return len;
+}
+
+void qemu_update_position(QEMUFile *f, size_t size)
+{
+    f->pos += size;
+}
+
+/** Closes the file
+ *
+ * Returns negative error value if any error happened on previous operations or
+ * while closing the file. Returns 0 or positive number on success.
+ *
+ * The meaning of return value on success depends on the specific backend
+ * being used.
+ */
+int qemu_fclose(QEMUFile *f)
+{
+    int ret;
+    qemu_fflush(f);
+    ret = qemu_file_get_error(f);
+
+    if (f->ops->close) {
+        int ret2 = f->ops->close(f->opaque);
+        if (ret >= 0) {
+            ret = ret2;
+        }
+    }
+    /* If any error was spotted before closing, we should report it
+     * instead of the close() return value.
+     */
+    if (f->last_error) {
+        ret = f->last_error;
+    }
+    g_free(f);
+    trace_qemu_file_fclose();
+    return ret;
+}
+
+static void add_to_iovec(QEMUFile *f, const uint8_t *buf, size_t size)
+{
+    /* check for adjacent buffer and coalesce them */
+    if (f->iovcnt > 0 && buf == f->iov[f->iovcnt - 1].iov_base +
+        f->iov[f->iovcnt - 1].iov_len) {
+        f->iov[f->iovcnt - 1].iov_len += size;
+    } else {
+        f->iov[f->iovcnt].iov_base = (uint8_t *)buf;
+        f->iov[f->iovcnt++].iov_len = size;
+    }
+
+    if (f->iovcnt >= MAX_IOV_SIZE) {
+        qemu_fflush(f);
+    }
+}
+
+void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size)
+{
+    if (f->last_error) {
+        return;
+    }
+
+    f->bytes_xfer += size;
+    add_to_iovec(f, buf, size);
+}
+
+void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size)
+{
+    size_t l;
+
+    if (f->last_error) {
+        return;
+    }
+
+    while (size > 0) {
+        l = IO_BUF_SIZE - f->buf_index;
+        if (l > size) {
+            l = size;
+        }
+        memcpy(f->buf + f->buf_index, buf, l);
+        f->bytes_xfer += l;
+        add_to_iovec(f, f->buf + f->buf_index, l);
+        f->buf_index += l;
+        if (f->buf_index == IO_BUF_SIZE) {
+            qemu_fflush(f);
+        }
+        if (qemu_file_get_error(f)) {
+            break;
+        }
+        buf += l;
+        size -= l;
+    }
+}
+
+void qemu_put_byte(QEMUFile *f, int v)
+{
+    if (f->last_error) {
+        return;
+    }
+
+    f->buf[f->buf_index] = v;
+    f->bytes_xfer++;
+    add_to_iovec(f, f->buf + f->buf_index, 1);
+    f->buf_index++;
+    if (f->buf_index == IO_BUF_SIZE) {
+        qemu_fflush(f);
+    }
+}
+
+void qemu_file_skip(QEMUFile *f, int size)
+{
+    if (f->buf_index + size <= f->buf_size) {
+        f->buf_index += size;
+    }
+}
+
+/*
+ * Read 'size' bytes from file (at 'offset') without moving the
+ * pointer and set 'buf' to point to that data.
+ *
+ * It will return size bytes unless there was an error, in which case it will
+ * return as many as it managed to read (assuming blocking fd's which
+ * all current QEMUFile are)
+ */
+size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset)
+{
+    ssize_t pending;
+    size_t index;
+
+    assert(!qemu_file_is_writable(f));
+    assert(offset < IO_BUF_SIZE);
+    assert(size <= IO_BUF_SIZE - offset);
+
+    /* The 1st byte to read from */
+    index = f->buf_index + offset;
+    /* The number of available bytes starting at index */
+    pending = f->buf_size - index;
+
+    /*
+     * qemu_fill_buffer might return just a few bytes, even when there isn't
+     * an error, so loop collecting them until we get enough.
+     */
+    while (pending < size) {
+        int received = qemu_fill_buffer(f);
+
+        if (received <= 0) {
+            break;
+        }
+
+        index = f->buf_index + offset;
+        pending = f->buf_size - index;
+    }
+
+    if (pending <= 0) {
+        return 0;
+    }
+    if (size > pending) {
+        size = pending;
+    }
+
+    *buf = f->buf + index;
+    return size;
+}
+
+/*
+ * Read 'size' bytes of data from the file into buf.
+ * 'size' can be larger than the internal buffer.
+ *
+ * It will return size bytes unless there was an error, in which case it will
+ * return as many as it managed to read (assuming blocking fd's which
+ * all current QEMUFile are)
+ */
+size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size)
+{
+    size_t pending = size;
+    size_t done = 0;
+
+    while (pending > 0) {
+        size_t res;
+        uint8_t *src;
+
+        res = qemu_peek_buffer(f, &src, MIN(pending, IO_BUF_SIZE), 0);
+        if (res == 0) {
+            return done;
+        }
+        memcpy(buf, src, res);
+        qemu_file_skip(f, res);
+        buf += res;
+        pending -= res;
+        done += res;
+    }
+    return done;
+}
+
+/*
+ * Read 'size' bytes of data from the file.
+ * 'size' can be larger than the internal buffer.
+ *
+ * The data:
+ *   may be held on an internal buffer (in which case *buf is updated
+ *     to point to it) that is valid until the next qemu_file operation.
+ * OR
+ *   will be copied to the *buf that was passed in.
+ *
+ * The code tries to avoid the copy if possible.
+ *
+ * It will return size bytes unless there was an error, in which case it will
+ * return as many as it managed to read (assuming blocking fd's which
+ * all current QEMUFile are)
+ *
+ * Note: Since **buf may get changed, the caller should take care to
+ *       keep a pointer to the original buffer if it needs to deallocate it.
+ */
+size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size)
+{
+    if (size < IO_BUF_SIZE) {
+        size_t res;
+        uint8_t *src;
+
+        res = qemu_peek_buffer(f, &src, size, 0);
+
+        if (res == size) {
+            qemu_file_skip(f, res);
+            *buf = src;
+            return res;
+        }
+    }
+
+    return qemu_get_buffer(f, *buf, size);
+}
+
+/*
+ * Peeks a single byte from the buffer; this isn't guaranteed to work if
+ * offset leaves a gap after the previous read/peeked data.
+ */
+int qemu_peek_byte(QEMUFile *f, int offset)
+{
+    int index = f->buf_index + offset;
+
+    assert(!qemu_file_is_writable(f));
+    assert(offset < IO_BUF_SIZE);
+
+    if (index >= f->buf_size) {
+        qemu_fill_buffer(f);
+        index = f->buf_index + offset;
+        if (index >= f->buf_size) {
+            return 0;
+        }
+    }
+    return f->buf[index];
+}
+
+int qemu_get_byte(QEMUFile *f)
+{
+    int result;
+
+    result = qemu_peek_byte(f, 0);
+    qemu_file_skip(f, 1);
+    return result;
+}
+
+int64_t qemu_ftell_fast(QEMUFile *f)
+{
+    int64_t ret = f->pos;
+    int i;
+
+    for (i = 0; i < f->iovcnt; i++) {
+        ret += f->iov[i].iov_len;
+    }
+
+    return ret;
+}
+
+int64_t qemu_ftell(QEMUFile *f)
+{
+    qemu_fflush(f);
+    return f->pos;
+}
+
+int qemu_file_rate_limit(QEMUFile *f)
+{
+    if (qemu_file_get_error(f)) {
+        return 1;
+    }
+    if (f->xfer_limit > 0 && f->bytes_xfer > f->xfer_limit) {
+        return 1;
+    }
+    return 0;
+}
+
+int64_t qemu_file_get_rate_limit(QEMUFile *f)
+{
+    return f->xfer_limit;
+}
+
+void qemu_file_set_rate_limit(QEMUFile *f, int64_t limit)
+{
+    f->xfer_limit = limit;
+}
+
+void qemu_file_reset_rate_limit(QEMUFile *f)
+{
+    f->bytes_xfer = 0;
+}
+
+void qemu_put_be16(QEMUFile *f, unsigned int v)
+{
+    qemu_put_byte(f, v >> 8);
+    qemu_put_byte(f, v);
+}
+
+void qemu_put_be32(QEMUFile *f, unsigned int v)
+{
+    qemu_put_byte(f, v >> 24);
+    qemu_put_byte(f, v >> 16);
+    qemu_put_byte(f, v >> 8);
+    qemu_put_byte(f, v);
+}
+
+void qemu_put_be64(QEMUFile *f, uint64_t v)
+{
+    qemu_put_be32(f, v >> 32);
+    qemu_put_be32(f, v);
+}
+
+unsigned int qemu_get_be16(QEMUFile *f)
+{
+    unsigned int v;
+    v = qemu_get_byte(f) << 8;
+    v |= qemu_get_byte(f);
+    return v;
+}
+
+unsigned int qemu_get_be32(QEMUFile *f)
+{
+    unsigned int v;
+    v = (unsigned int)qemu_get_byte(f) << 24;
+    v |= qemu_get_byte(f) << 16;
+    v |= qemu_get_byte(f) << 8;
+    v |= qemu_get_byte(f);
+    return v;
+}
+
+uint64_t qemu_get_be64(QEMUFile *f)
+{
+    uint64_t v;
+    v = (uint64_t)qemu_get_be32(f) << 32;
+    v |= qemu_get_be32(f);
+    return v;
+}
+
+/* Compress size bytes of data start at p with specific compression
+ * level and store the compressed data to the buffer of f.
+ *
+ * When f is not writable, return -1 if f has no space to save the
+ * compressed data.
+ * When f is wirtable and it has no space to save the compressed data,
+ * do fflush first, if f still has no space to save the compressed
+ * data, return -1.
+ */
+
+ssize_t qemu_put_compression_data(QEMUFile *f, const uint8_t *p, size_t size,
+                                  int level)
+{
+    ssize_t blen = IO_BUF_SIZE - f->buf_index - sizeof(int32_t);
+
+    if (blen < compressBound(size)) {
+        if (!qemu_file_is_writable(f)) {
+            return -1;
+        }
+        qemu_fflush(f);
+        blen = IO_BUF_SIZE - sizeof(int32_t);
+        if (blen < compressBound(size)) {
+            return -1;
+        }
+    }
+    if (compress2(f->buf + f->buf_index + sizeof(int32_t), (uLongf *)&blen,
+                  (Bytef *)p, size, level) != Z_OK) {
+        error_report("Compress Failed!");
+        return 0;
+    }
+    qemu_put_be32(f, blen);
+    if (f->ops->writev_buffer) {
+        add_to_iovec(f, f->buf + f->buf_index, blen);
+    }
+    f->buf_index += blen;
+    if (f->buf_index == IO_BUF_SIZE) {
+        qemu_fflush(f);
+    }
+    return blen + sizeof(int32_t);
+}
+
+/* Put the data in the buffer of f_src to the buffer of f_des, and
+ * then reset the buf_index of f_src to 0.
+ */
+
+int qemu_put_qemu_file(QEMUFile *f_des, QEMUFile *f_src)
+{
+    int len = 0;
+
+    if (f_src->buf_index > 0) {
+        len = f_src->buf_index;
+        qemu_put_buffer(f_des, f_src->buf, f_src->buf_index);
+        f_src->buf_index = 0;
+        f_src->iovcnt = 0;
+    }
+    return len;
+}
+
+/*
+ * Get a string whose length is determined by a single preceding byte
+ * A preallocated 256 byte buffer must be passed in.
+ * Returns: len on success and a 0 terminated string in the buffer
+ *          else 0
+ *          (Note a 0 length string will return 0 either way)
+ */
+size_t qemu_get_counted_string(QEMUFile *f, char buf[256])
+{
+    size_t len = qemu_get_byte(f);
+    size_t res = qemu_get_buffer(f, (uint8_t *)buf, len);
+
+    buf[res] = 0;
+
+    return res == len ? res : 0;
+}
+
+/*
+ * Set the blocking state of the QEMUFile.
+ * Note: On some transports the OS only keeps a single blocking state for
+ *       both directions, and thus changing the blocking on the main
+ *       QEMUFile can also affect the return path.
+ */
+void qemu_file_set_blocking(QEMUFile *f, bool block)
+{
+    if (f->ops->set_blocking) {
+        f->ops->set_blocking(f->opaque, block);
+    }
+}
diff --git a/migration/qjson.c b/migration/qjson.c
new file mode 100644
index 0000000..f345904
--- /dev/null
+++ b/migration/qjson.c
@@ -0,0 +1,114 @@
+/*
+ * A simple JSON writer
+ *
+ * Copyright Alexander Graf
+ *
+ * Authors:
+ *  Alexander Graf <agraf@suse.de>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+/*
+ * Type QJSON lets you build JSON text.  Its interface mirrors (a
+ * subset of) abstract JSON syntax.
+ *
+ * It does *not* detect incorrect use.  It happily produces invalid
+ * JSON then.  This is what migration wants.
+ *
+ * QAPI output visitors also produce JSON text.  However, they do
+ * assert their preconditions and invariants, and therefore abort on
+ * incorrect use.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/qmp/qstring.h"
+#include "migration/qjson.h"
+
+struct QJSON {
+    QString *str;
+    bool omit_comma;
+};
+
+static void json_emit_element(QJSON *json, const char *name)
+{
+    /* Check whether we need to print a , before an element */
+    if (json->omit_comma) {
+        json->omit_comma = false;
+    } else {
+        qstring_append(json->str, ", ");
+    }
+
+    if (name) {
+        qstring_append(json->str, "\"");
+        qstring_append(json->str, name);
+        qstring_append(json->str, "\" : ");
+    }
+}
+
+void json_start_object(QJSON *json, const char *name)
+{
+    json_emit_element(json, name);
+    qstring_append(json->str, "{ ");
+    json->omit_comma = true;
+}
+
+void json_end_object(QJSON *json)
+{
+    qstring_append(json->str, " }");
+    json->omit_comma = false;
+}
+
+void json_start_array(QJSON *json, const char *name)
+{
+    json_emit_element(json, name);
+    qstring_append(json->str, "[ ");
+    json->omit_comma = true;
+}
+
+void json_end_array(QJSON *json)
+{
+    qstring_append(json->str, " ]");
+    json->omit_comma = false;
+}
+
+void json_prop_int(QJSON *json, const char *name, int64_t val)
+{
+    json_emit_element(json, name);
+    qstring_append_int(json->str, val);
+}
+
+void json_prop_str(QJSON *json, const char *name, const char *str)
+{
+    json_emit_element(json, name);
+    qstring_append_chr(json->str, '"');
+    qstring_append(json->str, str);
+    qstring_append_chr(json->str, '"');
+}
+
+const char *qjson_get_str(QJSON *json)
+{
+    return qstring_get_str(json->str);
+}
+
+QJSON *qjson_new(void)
+{
+    QJSON *json = g_new0(QJSON, 1);
+
+    json->str = qstring_from_str("{ ");
+    json->omit_comma = true;
+    return json;
+}
+
+void qjson_finish(QJSON *json)
+{
+    json_end_object(json);
+}
+
+void qjson_destroy(QJSON *json)
+{
+    QDECREF(json->str);
+    g_free(json);
+}
diff --git a/migration/ram.c b/migration/ram.c
new file mode 100644
index 0000000..a3d70c4
--- /dev/null
+++ b/migration/ram.c
@@ -0,0 +1,2595 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2011-2015 Red Hat Inc
+ *
+ * Authors:
+ *  Juan Quintela <quintela@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include <zlib.h>
+#include "qapi-event.h"
+#include "qemu/cutils.h"
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
+#include "qemu/timer.h"
+#include "qemu/main-loop.h"
+#include "migration/migration.h"
+#include "migration/postcopy-ram.h"
+#include "exec/address-spaces.h"
+#include "migration/page_cache.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+#include "exec/ram_addr.h"
+#include "qemu/rcu_queue.h"
+
+#ifdef DEBUG_MIGRATION_RAM
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stdout, "migration_ram: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+static int dirty_rate_high_cnt;
+
+static uint64_t bitmap_sync_count;
+
+/***********************************************************/
+/* ram save/restore */
+
+#define RAM_SAVE_FLAG_FULL     0x01 /* Obsolete, not used anymore */
+#define RAM_SAVE_FLAG_COMPRESS 0x02
+#define RAM_SAVE_FLAG_MEM_SIZE 0x04
+#define RAM_SAVE_FLAG_PAGE     0x08
+#define RAM_SAVE_FLAG_EOS      0x10
+#define RAM_SAVE_FLAG_CONTINUE 0x20
+#define RAM_SAVE_FLAG_XBZRLE   0x40
+/* 0x80 is reserved in migration.h start with 0x100 next */
+#define RAM_SAVE_FLAG_COMPRESS_PAGE    0x100
+
+static const uint8_t ZERO_TARGET_PAGE[TARGET_PAGE_SIZE];
+
+static inline bool is_zero_range(uint8_t *p, uint64_t size)
+{
+    return buffer_find_nonzero_offset(p, size) == size;
+}
+
+/* struct contains XBZRLE cache and a static page
+   used by the compression */
+static struct {
+    /* buffer used for XBZRLE encoding */
+    uint8_t *encoded_buf;
+    /* buffer for storing page content */
+    uint8_t *current_buf;
+    /* Cache for XBZRLE, Protected by lock. */
+    PageCache *cache;
+    QemuMutex lock;
+} XBZRLE;
+
+/* buffer used for XBZRLE decoding */
+static uint8_t *xbzrle_decoded_buf;
+
+static void XBZRLE_cache_lock(void)
+{
+    if (migrate_use_xbzrle())
+        qemu_mutex_lock(&XBZRLE.lock);
+}
+
+static void XBZRLE_cache_unlock(void)
+{
+    if (migrate_use_xbzrle())
+        qemu_mutex_unlock(&XBZRLE.lock);
+}
+
+/*
+ * called from qmp_migrate_set_cache_size in main thread, possibly while
+ * a migration is in progress.
+ * A running migration maybe using the cache and might finish during this
+ * call, hence changes to the cache are protected by XBZRLE.lock().
+ */
+int64_t xbzrle_cache_resize(int64_t new_size)
+{
+    PageCache *new_cache;
+    int64_t ret;
+
+    if (new_size < TARGET_PAGE_SIZE) {
+        return -1;
+    }
+
+    XBZRLE_cache_lock();
+
+    if (XBZRLE.cache != NULL) {
+        if (pow2floor(new_size) == migrate_xbzrle_cache_size()) {
+            goto out_new_size;
+        }
+        new_cache = cache_init(new_size / TARGET_PAGE_SIZE,
+                                        TARGET_PAGE_SIZE);
+        if (!new_cache) {
+            error_report("Error creating cache");
+            ret = -1;
+            goto out;
+        }
+
+        cache_fini(XBZRLE.cache);
+        XBZRLE.cache = new_cache;
+    }
+
+out_new_size:
+    ret = pow2floor(new_size);
+out:
+    XBZRLE_cache_unlock();
+    return ret;
+}
+
+/* accounting for migration statistics */
+typedef struct AccountingInfo {
+    uint64_t dup_pages;
+    uint64_t skipped_pages;
+    uint64_t norm_pages;
+    uint64_t iterations;
+    uint64_t xbzrle_bytes;
+    uint64_t xbzrle_pages;
+    uint64_t xbzrle_cache_miss;
+    double xbzrle_cache_miss_rate;
+    uint64_t xbzrle_overflows;
+} AccountingInfo;
+
+static AccountingInfo acct_info;
+
+static void acct_clear(void)
+{
+    memset(&acct_info, 0, sizeof(acct_info));
+}
+
+uint64_t dup_mig_bytes_transferred(void)
+{
+    return acct_info.dup_pages * TARGET_PAGE_SIZE;
+}
+
+uint64_t dup_mig_pages_transferred(void)
+{
+    return acct_info.dup_pages;
+}
+
+uint64_t skipped_mig_bytes_transferred(void)
+{
+    return acct_info.skipped_pages * TARGET_PAGE_SIZE;
+}
+
+uint64_t skipped_mig_pages_transferred(void)
+{
+    return acct_info.skipped_pages;
+}
+
+uint64_t norm_mig_bytes_transferred(void)
+{
+    return acct_info.norm_pages * TARGET_PAGE_SIZE;
+}
+
+uint64_t norm_mig_pages_transferred(void)
+{
+    return acct_info.norm_pages;
+}
+
+uint64_t xbzrle_mig_bytes_transferred(void)
+{
+    return acct_info.xbzrle_bytes;
+}
+
+uint64_t xbzrle_mig_pages_transferred(void)
+{
+    return acct_info.xbzrle_pages;
+}
+
+uint64_t xbzrle_mig_pages_cache_miss(void)
+{
+    return acct_info.xbzrle_cache_miss;
+}
+
+double xbzrle_mig_cache_miss_rate(void)
+{
+    return acct_info.xbzrle_cache_miss_rate;
+}
+
+uint64_t xbzrle_mig_pages_overflow(void)
+{
+    return acct_info.xbzrle_overflows;
+}
+
+/* This is the last block that we have visited serching for dirty pages
+ */
+static RAMBlock *last_seen_block;
+/* This is the last block from where we have sent data */
+static RAMBlock *last_sent_block;
+static ram_addr_t last_offset;
+static QemuMutex migration_bitmap_mutex;
+static uint64_t migration_dirty_pages;
+static uint32_t last_version;
+static bool ram_bulk_stage;
+
+/* used by the search for pages to send */
+struct PageSearchStatus {
+    /* Current block being searched */
+    RAMBlock    *block;
+    /* Current offset to search from */
+    ram_addr_t   offset;
+    /* Set once we wrap around */
+    bool         complete_round;
+};
+typedef struct PageSearchStatus PageSearchStatus;
+
+static struct BitmapRcu {
+    struct rcu_head rcu;
+    /* Main migration bitmap */
+    unsigned long *bmap;
+    /* bitmap of pages that haven't been sent even once
+     * only maintained and used in postcopy at the moment
+     * where it's used to send the dirtymap at the start
+     * of the postcopy phase
+     */
+    unsigned long *unsentmap;
+} *migration_bitmap_rcu;
+
+struct CompressParam {
+    bool done;
+    bool quit;
+    QEMUFile *file;
+    QemuMutex mutex;
+    QemuCond cond;
+    RAMBlock *block;
+    ram_addr_t offset;
+};
+typedef struct CompressParam CompressParam;
+
+struct DecompressParam {
+    bool done;
+    bool quit;
+    QemuMutex mutex;
+    QemuCond cond;
+    void *des;
+    uint8_t *compbuf;
+    int len;
+};
+typedef struct DecompressParam DecompressParam;
+
+static CompressParam *comp_param;
+static QemuThread *compress_threads;
+/* comp_done_cond is used to wake up the migration thread when
+ * one of the compression threads has finished the compression.
+ * comp_done_lock is used to co-work with comp_done_cond.
+ */
+static QemuMutex comp_done_lock;
+static QemuCond comp_done_cond;
+/* The empty QEMUFileOps will be used by file in CompressParam */
+static const QEMUFileOps empty_ops = { };
+
+static bool compression_switch;
+static DecompressParam *decomp_param;
+static QemuThread *decompress_threads;
+static QemuMutex decomp_done_lock;
+static QemuCond decomp_done_cond;
+
+static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
+                                ram_addr_t offset);
+
+static void *do_data_compress(void *opaque)
+{
+    CompressParam *param = opaque;
+    RAMBlock *block;
+    ram_addr_t offset;
+
+    qemu_mutex_lock(&param->mutex);
+    while (!param->quit) {
+        if (param->block) {
+            block = param->block;
+            offset = param->offset;
+            param->block = NULL;
+            qemu_mutex_unlock(&param->mutex);
+
+            do_compress_ram_page(param->file, block, offset);
+
+            qemu_mutex_lock(&comp_done_lock);
+            param->done = true;
+            qemu_cond_signal(&comp_done_cond);
+            qemu_mutex_unlock(&comp_done_lock);
+
+            qemu_mutex_lock(&param->mutex);
+        } else {
+            qemu_cond_wait(&param->cond, &param->mutex);
+        }
+    }
+    qemu_mutex_unlock(&param->mutex);
+
+    return NULL;
+}
+
+static inline void terminate_compression_threads(void)
+{
+    int idx, thread_count;
+
+    thread_count = migrate_compress_threads();
+    for (idx = 0; idx < thread_count; idx++) {
+        qemu_mutex_lock(&comp_param[idx].mutex);
+        comp_param[idx].quit = true;
+        qemu_cond_signal(&comp_param[idx].cond);
+        qemu_mutex_unlock(&comp_param[idx].mutex);
+    }
+}
+
+void migrate_compress_threads_join(void)
+{
+    int i, thread_count;
+
+    if (!migrate_use_compression()) {
+        return;
+    }
+    terminate_compression_threads();
+    thread_count = migrate_compress_threads();
+    for (i = 0; i < thread_count; i++) {
+        qemu_thread_join(compress_threads + i);
+        qemu_fclose(comp_param[i].file);
+        qemu_mutex_destroy(&comp_param[i].mutex);
+        qemu_cond_destroy(&comp_param[i].cond);
+    }
+    qemu_mutex_destroy(&comp_done_lock);
+    qemu_cond_destroy(&comp_done_cond);
+    g_free(compress_threads);
+    g_free(comp_param);
+    compress_threads = NULL;
+    comp_param = NULL;
+}
+
+void migrate_compress_threads_create(void)
+{
+    int i, thread_count;
+
+    if (!migrate_use_compression()) {
+        return;
+    }
+    compression_switch = true;
+    thread_count = migrate_compress_threads();
+    compress_threads = g_new0(QemuThread, thread_count);
+    comp_param = g_new0(CompressParam, thread_count);
+    qemu_cond_init(&comp_done_cond);
+    qemu_mutex_init(&comp_done_lock);
+    for (i = 0; i < thread_count; i++) {
+        /* comp_param[i].file is just used as a dummy buffer to save data,
+         * set its ops to empty.
+         */
+        comp_param[i].file = qemu_fopen_ops(NULL, &empty_ops);
+        comp_param[i].done = true;
+        comp_param[i].quit = false;
+        qemu_mutex_init(&comp_param[i].mutex);
+        qemu_cond_init(&comp_param[i].cond);
+        qemu_thread_create(compress_threads + i, "compress",
+                           do_data_compress, comp_param + i,
+                           QEMU_THREAD_JOINABLE);
+    }
+}
+
+/**
+ * save_page_header: Write page header to wire
+ *
+ * If this is the 1st block, it also writes the block identification
+ *
+ * Returns: Number of bytes written
+ *
+ * @f: QEMUFile where to send the data
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ *          in the lower bits, it contains flags
+ */
+static size_t save_page_header(QEMUFile *f, RAMBlock *block, ram_addr_t offset)
+{
+    size_t size, len;
+
+    qemu_put_be64(f, offset);
+    size = 8;
+
+    if (!(offset & RAM_SAVE_FLAG_CONTINUE)) {
+        len = strlen(block->idstr);
+        qemu_put_byte(f, len);
+        qemu_put_buffer(f, (uint8_t *)block->idstr, len);
+        size += 1 + len;
+    }
+    return size;
+}
+
+/* Reduce amount of guest cpu execution to hopefully slow down memory writes.
+ * If guest dirty memory rate is reduced below the rate at which we can
+ * transfer pages to the destination then we should be able to complete
+ * migration. Some workloads dirty memory way too fast and will not effectively
+ * converge, even with auto-converge.
+ */
+static void mig_throttle_guest_down(void)
+{
+    MigrationState *s = migrate_get_current();
+    uint64_t pct_initial = s->parameters.cpu_throttle_initial;
+    uint64_t pct_icrement = s->parameters.cpu_throttle_increment;
+
+    /* We have not started throttling yet. Let's start it. */
+    if (!cpu_throttle_active()) {
+        cpu_throttle_set(pct_initial);
+    } else {
+        /* Throttling already on, just increase the rate */
+        cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement);
+    }
+}
+
+/* Update the xbzrle cache to reflect a page that's been sent as all 0.
+ * The important thing is that a stale (not-yet-0'd) page be replaced
+ * by the new data.
+ * As a bonus, if the page wasn't in the cache it gets added so that
+ * when a small write is made into the 0'd page it gets XBZRLE sent
+ */
+static void xbzrle_cache_zero_page(ram_addr_t current_addr)
+{
+    if (ram_bulk_stage || !migrate_use_xbzrle()) {
+        return;
+    }
+
+    /* We don't care if this fails to allocate a new cache page
+     * as long as it updated an old one */
+    cache_insert(XBZRLE.cache, current_addr, ZERO_TARGET_PAGE,
+                 bitmap_sync_count);
+}
+
+#define ENCODING_FLAG_XBZRLE 0x1
+
+/**
+ * save_xbzrle_page: compress and send current page
+ *
+ * Returns: 1 means that we wrote the page
+ *          0 means that page is identical to the one already sent
+ *          -1 means that xbzrle would be longer than normal
+ *
+ * @f: QEMUFile where to send the data
+ * @current_data:
+ * @current_addr:
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ */
+static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data,
+                            ram_addr_t current_addr, RAMBlock *block,
+                            ram_addr_t offset, bool last_stage,
+                            uint64_t *bytes_transferred)
+{
+    int encoded_len = 0, bytes_xbzrle;
+    uint8_t *prev_cached_page;
+
+    if (!cache_is_cached(XBZRLE.cache, current_addr, bitmap_sync_count)) {
+        acct_info.xbzrle_cache_miss++;
+        if (!last_stage) {
+            if (cache_insert(XBZRLE.cache, current_addr, *current_data,
+                             bitmap_sync_count) == -1) {
+                return -1;
+            } else {
+                /* update *current_data when the page has been
+                   inserted into cache */
+                *current_data = get_cached_data(XBZRLE.cache, current_addr);
+            }
+        }
+        return -1;
+    }
+
+    prev_cached_page = get_cached_data(XBZRLE.cache, current_addr);
+
+    /* save current buffer into memory */
+    memcpy(XBZRLE.current_buf, *current_data, TARGET_PAGE_SIZE);
+
+    /* XBZRLE encoding (if there is no overflow) */
+    encoded_len = xbzrle_encode_buffer(prev_cached_page, XBZRLE.current_buf,
+                                       TARGET_PAGE_SIZE, XBZRLE.encoded_buf,
+                                       TARGET_PAGE_SIZE);
+    if (encoded_len == 0) {
+        DPRINTF("Skipping unmodified page\n");
+        return 0;
+    } else if (encoded_len == -1) {
+        DPRINTF("Overflow\n");
+        acct_info.xbzrle_overflows++;
+        /* update data in the cache */
+        if (!last_stage) {
+            memcpy(prev_cached_page, *current_data, TARGET_PAGE_SIZE);
+            *current_data = prev_cached_page;
+        }
+        return -1;
+    }
+
+    /* we need to update the data in the cache, in order to get the same data */
+    if (!last_stage) {
+        memcpy(prev_cached_page, XBZRLE.current_buf, TARGET_PAGE_SIZE);
+    }
+
+    /* Send XBZRLE based compressed page */
+    bytes_xbzrle = save_page_header(f, block, offset | RAM_SAVE_FLAG_XBZRLE);
+    qemu_put_byte(f, ENCODING_FLAG_XBZRLE);
+    qemu_put_be16(f, encoded_len);
+    qemu_put_buffer(f, XBZRLE.encoded_buf, encoded_len);
+    bytes_xbzrle += encoded_len + 1 + 2;
+    acct_info.xbzrle_pages++;
+    acct_info.xbzrle_bytes += bytes_xbzrle;
+    *bytes_transferred += bytes_xbzrle;
+
+    return 1;
+}
+
+/* Called with rcu_read_lock() to protect migration_bitmap
+ * rb: The RAMBlock  to search for dirty pages in
+ * start: Start address (typically so we can continue from previous page)
+ * ram_addr_abs: Pointer into which to store the address of the dirty page
+ *               within the global ram_addr space
+ *
+ * Returns: byte offset within memory region of the start of a dirty page
+ */
+static inline
+ram_addr_t migration_bitmap_find_dirty(RAMBlock *rb,
+                                       ram_addr_t start,
+                                       ram_addr_t *ram_addr_abs)
+{
+    unsigned long base = rb->offset >> TARGET_PAGE_BITS;
+    unsigned long nr = base + (start >> TARGET_PAGE_BITS);
+    uint64_t rb_size = rb->used_length;
+    unsigned long size = base + (rb_size >> TARGET_PAGE_BITS);
+    unsigned long *bitmap;
+
+    unsigned long next;
+
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    if (ram_bulk_stage && nr > base) {
+        next = nr + 1;
+    } else {
+        next = find_next_bit(bitmap, size, nr);
+    }
+
+    *ram_addr_abs = next << TARGET_PAGE_BITS;
+    return (next - base) << TARGET_PAGE_BITS;
+}
+
+static inline bool migration_bitmap_clear_dirty(ram_addr_t addr)
+{
+    bool ret;
+    int nr = addr >> TARGET_PAGE_BITS;
+    unsigned long *bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+
+    ret = test_and_clear_bit(nr, bitmap);
+
+    if (ret) {
+        migration_dirty_pages--;
+    }
+    return ret;
+}
+
+static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length)
+{
+    unsigned long *bitmap;
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    migration_dirty_pages +=
+        cpu_physical_memory_sync_dirty_bitmap(bitmap, start, length);
+}
+
+/* Fix me: there are too many global variables used in migration process. */
+static int64_t start_time;
+static int64_t bytes_xfer_prev;
+static int64_t num_dirty_pages_period;
+static uint64_t xbzrle_cache_miss_prev;
+static uint64_t iterations_prev;
+
+static void migration_bitmap_sync_init(void)
+{
+    start_time = 0;
+    bytes_xfer_prev = 0;
+    num_dirty_pages_period = 0;
+    xbzrle_cache_miss_prev = 0;
+    iterations_prev = 0;
+}
+
+static void migration_bitmap_sync(void)
+{
+    RAMBlock *block;
+    uint64_t num_dirty_pages_init = migration_dirty_pages;
+    MigrationState *s = migrate_get_current();
+    int64_t end_time;
+    int64_t bytes_xfer_now;
+
+    bitmap_sync_count++;
+
+    if (!bytes_xfer_prev) {
+        bytes_xfer_prev = ram_bytes_transferred();
+    }
+
+    if (!start_time) {
+        start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    }
+
+    trace_migration_bitmap_sync_start();
+    address_space_sync_dirty_bitmap(&address_space_memory);
+
+    qemu_mutex_lock(&migration_bitmap_mutex);
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        migration_bitmap_sync_range(block->offset, block->used_length);
+    }
+    rcu_read_unlock();
+    qemu_mutex_unlock(&migration_bitmap_mutex);
+
+    trace_migration_bitmap_sync_end(migration_dirty_pages
+                                    - num_dirty_pages_init);
+    num_dirty_pages_period += migration_dirty_pages - num_dirty_pages_init;
+    end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+
+    /* more than 1 second = 1000 millisecons */
+    if (end_time > start_time + 1000) {
+        if (migrate_auto_converge()) {
+            /* The following detection logic can be refined later. For now:
+               Check to see if the dirtied bytes is 50% more than the approx.
+               amount of bytes that just got transferred since the last time we
+               were in this routine. If that happens twice, start or increase
+               throttling */
+            bytes_xfer_now = ram_bytes_transferred();
+
+            if (s->dirty_pages_rate &&
+               (num_dirty_pages_period * TARGET_PAGE_SIZE >
+                   (bytes_xfer_now - bytes_xfer_prev)/2) &&
+               (dirty_rate_high_cnt++ >= 2)) {
+                    trace_migration_throttle();
+                    dirty_rate_high_cnt = 0;
+                    mig_throttle_guest_down();
+             }
+             bytes_xfer_prev = bytes_xfer_now;
+        }
+
+        if (migrate_use_xbzrle()) {
+            if (iterations_prev != acct_info.iterations) {
+                acct_info.xbzrle_cache_miss_rate =
+                   (double)(acct_info.xbzrle_cache_miss -
+                            xbzrle_cache_miss_prev) /
+                   (acct_info.iterations - iterations_prev);
+            }
+            iterations_prev = acct_info.iterations;
+            xbzrle_cache_miss_prev = acct_info.xbzrle_cache_miss;
+        }
+        s->dirty_pages_rate = num_dirty_pages_period * 1000
+            / (end_time - start_time);
+        s->dirty_bytes_rate = s->dirty_pages_rate * TARGET_PAGE_SIZE;
+        start_time = end_time;
+        num_dirty_pages_period = 0;
+    }
+    s->dirty_sync_count = bitmap_sync_count;
+    if (migrate_use_events()) {
+        qapi_event_send_migration_pass(bitmap_sync_count, NULL);
+    }
+}
+
+/**
+ * save_zero_page: Send the zero page to the stream
+ *
+ * Returns: Number of pages written.
+ *
+ * @f: QEMUFile where to send the data
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ * @p: pointer to the page
+ * @bytes_transferred: increase it with the number of transferred bytes
+ */
+static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
+                          uint8_t *p, uint64_t *bytes_transferred)
+{
+    int pages = -1;
+
+    if (is_zero_range(p, TARGET_PAGE_SIZE)) {
+        acct_info.dup_pages++;
+        *bytes_transferred += save_page_header(f, block,
+                                               offset | RAM_SAVE_FLAG_COMPRESS);
+        qemu_put_byte(f, 0);
+        *bytes_transferred += 1;
+        pages = 1;
+    }
+
+    return pages;
+}
+
+/**
+ * ram_save_page: Send the given page to the stream
+ *
+ * Returns: Number of pages written.
+ *          < 0 - error
+ *          >=0 - Number of pages written - this might legally be 0
+ *                if xbzrle noticed the page was the same.
+ *
+ * @f: QEMUFile where to send the data
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ */
+static int ram_save_page(QEMUFile *f, PageSearchStatus *pss,
+                         bool last_stage, uint64_t *bytes_transferred)
+{
+    int pages = -1;
+    uint64_t bytes_xmit;
+    ram_addr_t current_addr;
+    uint8_t *p;
+    int ret;
+    bool send_async = true;
+    RAMBlock *block = pss->block;
+    ram_addr_t offset = pss->offset;
+
+    p = block->host + offset;
+
+    /* In doubt sent page as normal */
+    bytes_xmit = 0;
+    ret = ram_control_save_page(f, block->offset,
+                           offset, TARGET_PAGE_SIZE, &bytes_xmit);
+    if (bytes_xmit) {
+        *bytes_transferred += bytes_xmit;
+        pages = 1;
+    }
+
+    XBZRLE_cache_lock();
+
+    current_addr = block->offset + offset;
+
+    if (block == last_sent_block) {
+        offset |= RAM_SAVE_FLAG_CONTINUE;
+    }
+    if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
+        if (ret != RAM_SAVE_CONTROL_DELAYED) {
+            if (bytes_xmit > 0) {
+                acct_info.norm_pages++;
+            } else if (bytes_xmit == 0) {
+                acct_info.dup_pages++;
+            }
+        }
+    } else {
+        pages = save_zero_page(f, block, offset, p, bytes_transferred);
+        if (pages > 0) {
+            /* Must let xbzrle know, otherwise a previous (now 0'd) cached
+             * page would be stale
+             */
+            xbzrle_cache_zero_page(current_addr);
+        } else if (!ram_bulk_stage && migrate_use_xbzrle()) {
+            pages = save_xbzrle_page(f, &p, current_addr, block,
+                                     offset, last_stage, bytes_transferred);
+            if (!last_stage) {
+                /* Can't send this cached data async, since the cache page
+                 * might get updated before it gets to the wire
+                 */
+                send_async = false;
+            }
+        }
+    }
+
+    /* XBZRLE overflow or normal page */
+    if (pages == -1) {
+        *bytes_transferred += save_page_header(f, block,
+                                               offset | RAM_SAVE_FLAG_PAGE);
+        if (send_async) {
+            qemu_put_buffer_async(f, p, TARGET_PAGE_SIZE);
+        } else {
+            qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
+        }
+        *bytes_transferred += TARGET_PAGE_SIZE;
+        pages = 1;
+        acct_info.norm_pages++;
+    }
+
+    XBZRLE_cache_unlock();
+
+    return pages;
+}
+
+static int do_compress_ram_page(QEMUFile *f, RAMBlock *block,
+                                ram_addr_t offset)
+{
+    int bytes_sent, blen;
+    uint8_t *p = block->host + (offset & TARGET_PAGE_MASK);
+
+    bytes_sent = save_page_header(f, block, offset |
+                                  RAM_SAVE_FLAG_COMPRESS_PAGE);
+    blen = qemu_put_compression_data(f, p, TARGET_PAGE_SIZE,
+                                     migrate_compress_level());
+    if (blen < 0) {
+        bytes_sent = 0;
+        qemu_file_set_error(migrate_get_current()->to_dst_file, blen);
+        error_report("compressed data failed!");
+    } else {
+        bytes_sent += blen;
+    }
+
+    return bytes_sent;
+}
+
+static uint64_t bytes_transferred;
+
+static void flush_compressed_data(QEMUFile *f)
+{
+    int idx, len, thread_count;
+
+    if (!migrate_use_compression()) {
+        return;
+    }
+    thread_count = migrate_compress_threads();
+
+    qemu_mutex_lock(&comp_done_lock);
+    for (idx = 0; idx < thread_count; idx++) {
+        while (!comp_param[idx].done) {
+            qemu_cond_wait(&comp_done_cond, &comp_done_lock);
+        }
+    }
+    qemu_mutex_unlock(&comp_done_lock);
+
+    for (idx = 0; idx < thread_count; idx++) {
+        qemu_mutex_lock(&comp_param[idx].mutex);
+        if (!comp_param[idx].quit) {
+            len = qemu_put_qemu_file(f, comp_param[idx].file);
+            bytes_transferred += len;
+        }
+        qemu_mutex_unlock(&comp_param[idx].mutex);
+    }
+}
+
+static inline void set_compress_params(CompressParam *param, RAMBlock *block,
+                                       ram_addr_t offset)
+{
+    param->block = block;
+    param->offset = offset;
+}
+
+static int compress_page_with_multi_thread(QEMUFile *f, RAMBlock *block,
+                                           ram_addr_t offset,
+                                           uint64_t *bytes_transferred)
+{
+    int idx, thread_count, bytes_xmit = -1, pages = -1;
+
+    thread_count = migrate_compress_threads();
+    qemu_mutex_lock(&comp_done_lock);
+    while (true) {
+        for (idx = 0; idx < thread_count; idx++) {
+            if (comp_param[idx].done) {
+                comp_param[idx].done = false;
+                bytes_xmit = qemu_put_qemu_file(f, comp_param[idx].file);
+                qemu_mutex_lock(&comp_param[idx].mutex);
+                set_compress_params(&comp_param[idx], block, offset);
+                qemu_cond_signal(&comp_param[idx].cond);
+                qemu_mutex_unlock(&comp_param[idx].mutex);
+                pages = 1;
+                acct_info.norm_pages++;
+                *bytes_transferred += bytes_xmit;
+                break;
+            }
+        }
+        if (pages > 0) {
+            break;
+        } else {
+            qemu_cond_wait(&comp_done_cond, &comp_done_lock);
+        }
+    }
+    qemu_mutex_unlock(&comp_done_lock);
+
+    return pages;
+}
+
+/**
+ * ram_save_compressed_page: compress the given page and send it to the stream
+ *
+ * Returns: Number of pages written.
+ *
+ * @f: QEMUFile where to send the data
+ * @block: block that contains the page we want to send
+ * @offset: offset inside the block for the page
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ */
+static int ram_save_compressed_page(QEMUFile *f, PageSearchStatus *pss,
+                                    bool last_stage,
+                                    uint64_t *bytes_transferred)
+{
+    int pages = -1;
+    uint64_t bytes_xmit = 0;
+    uint8_t *p;
+    int ret, blen;
+    RAMBlock *block = pss->block;
+    ram_addr_t offset = pss->offset;
+
+    p = block->host + offset;
+
+    ret = ram_control_save_page(f, block->offset,
+                                offset, TARGET_PAGE_SIZE, &bytes_xmit);
+    if (bytes_xmit) {
+        *bytes_transferred += bytes_xmit;
+        pages = 1;
+    }
+    if (ret != RAM_SAVE_CONTROL_NOT_SUPP) {
+        if (ret != RAM_SAVE_CONTROL_DELAYED) {
+            if (bytes_xmit > 0) {
+                acct_info.norm_pages++;
+            } else if (bytes_xmit == 0) {
+                acct_info.dup_pages++;
+            }
+        }
+    } else {
+        /* When starting the process of a new block, the first page of
+         * the block should be sent out before other pages in the same
+         * block, and all the pages in last block should have been sent
+         * out, keeping this order is important, because the 'cont' flag
+         * is used to avoid resending the block name.
+         */
+        if (block != last_sent_block) {
+            flush_compressed_data(f);
+            pages = save_zero_page(f, block, offset, p, bytes_transferred);
+            if (pages == -1) {
+                /* Make sure the first page is sent out before other pages */
+                bytes_xmit = save_page_header(f, block, offset |
+                                              RAM_SAVE_FLAG_COMPRESS_PAGE);
+                blen = qemu_put_compression_data(f, p, TARGET_PAGE_SIZE,
+                                                 migrate_compress_level());
+                if (blen > 0) {
+                    *bytes_transferred += bytes_xmit + blen;
+                    acct_info.norm_pages++;
+                    pages = 1;
+                } else {
+                    qemu_file_set_error(f, blen);
+                    error_report("compressed data failed!");
+                }
+            }
+        } else {
+            offset |= RAM_SAVE_FLAG_CONTINUE;
+            pages = save_zero_page(f, block, offset, p, bytes_transferred);
+            if (pages == -1) {
+                pages = compress_page_with_multi_thread(f, block, offset,
+                                                        bytes_transferred);
+            }
+        }
+    }
+
+    return pages;
+}
+
+/*
+ * Find the next dirty page and update any state associated with
+ * the search process.
+ *
+ * Returns: True if a page is found
+ *
+ * @f: Current migration stream.
+ * @pss: Data about the state of the current dirty page scan.
+ * @*again: Set to false if the search has scanned the whole of RAM
+ * *ram_addr_abs: Pointer into which to store the address of the dirty page
+ *               within the global ram_addr space
+ */
+static bool find_dirty_block(QEMUFile *f, PageSearchStatus *pss,
+                             bool *again, ram_addr_t *ram_addr_abs)
+{
+    pss->offset = migration_bitmap_find_dirty(pss->block, pss->offset,
+                                              ram_addr_abs);
+    if (pss->complete_round && pss->block == last_seen_block &&
+        pss->offset >= last_offset) {
+        /*
+         * We've been once around the RAM and haven't found anything.
+         * Give up.
+         */
+        *again = false;
+        return false;
+    }
+    if (pss->offset >= pss->block->used_length) {
+        /* Didn't find anything in this RAM Block */
+        pss->offset = 0;
+        pss->block = QLIST_NEXT_RCU(pss->block, next);
+        if (!pss->block) {
+            /* Hit the end of the list */
+            pss->block = QLIST_FIRST_RCU(&ram_list.blocks);
+            /* Flag that we've looped */
+            pss->complete_round = true;
+            ram_bulk_stage = false;
+            if (migrate_use_xbzrle()) {
+                /* If xbzrle is on, stop using the data compression at this
+                 * point. In theory, xbzrle can do better than compression.
+                 */
+                flush_compressed_data(f);
+                compression_switch = false;
+            }
+        }
+        /* Didn't find anything this time, but try again on the new block */
+        *again = true;
+        return false;
+    } else {
+        /* Can go around again, but... */
+        *again = true;
+        /* We've found something so probably don't need to */
+        return true;
+    }
+}
+
+/*
+ * Helper for 'get_queued_page' - gets a page off the queue
+ *      ms:      MigrationState in
+ * *offset:      Used to return the offset within the RAMBlock
+ * ram_addr_abs: global offset in the dirty/sent bitmaps
+ *
+ * Returns:      block (or NULL if none available)
+ */
+static RAMBlock *unqueue_page(MigrationState *ms, ram_addr_t *offset,
+                              ram_addr_t *ram_addr_abs)
+{
+    RAMBlock *block = NULL;
+
+    qemu_mutex_lock(&ms->src_page_req_mutex);
+    if (!QSIMPLEQ_EMPTY(&ms->src_page_requests)) {
+        struct MigrationSrcPageRequest *entry =
+                                QSIMPLEQ_FIRST(&ms->src_page_requests);
+        block = entry->rb;
+        *offset = entry->offset;
+        *ram_addr_abs = (entry->offset + entry->rb->offset) &
+                        TARGET_PAGE_MASK;
+
+        if (entry->len > TARGET_PAGE_SIZE) {
+            entry->len -= TARGET_PAGE_SIZE;
+            entry->offset += TARGET_PAGE_SIZE;
+        } else {
+            memory_region_unref(block->mr);
+            QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
+            g_free(entry);
+        }
+    }
+    qemu_mutex_unlock(&ms->src_page_req_mutex);
+
+    return block;
+}
+
+/*
+ * Unqueue a page from the queue fed by postcopy page requests; skips pages
+ * that are already sent (!dirty)
+ *
+ *      ms:      MigrationState in
+ *     pss:      PageSearchStatus structure updated with found block/offset
+ * ram_addr_abs: global offset in the dirty/sent bitmaps
+ *
+ * Returns:      true if a queued page is found
+ */
+static bool get_queued_page(MigrationState *ms, PageSearchStatus *pss,
+                            ram_addr_t *ram_addr_abs)
+{
+    RAMBlock  *block;
+    ram_addr_t offset;
+    bool dirty;
+
+    do {
+        block = unqueue_page(ms, &offset, ram_addr_abs);
+        /*
+         * We're sending this page, and since it's postcopy nothing else
+         * will dirty it, and we must make sure it doesn't get sent again
+         * even if this queue request was received after the background
+         * search already sent it.
+         */
+        if (block) {
+            unsigned long *bitmap;
+            bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+            dirty = test_bit(*ram_addr_abs >> TARGET_PAGE_BITS, bitmap);
+            if (!dirty) {
+                trace_get_queued_page_not_dirty(
+                    block->idstr, (uint64_t)offset,
+                    (uint64_t)*ram_addr_abs,
+                    test_bit(*ram_addr_abs >> TARGET_PAGE_BITS,
+                         atomic_rcu_read(&migration_bitmap_rcu)->unsentmap));
+            } else {
+                trace_get_queued_page(block->idstr,
+                                      (uint64_t)offset,
+                                      (uint64_t)*ram_addr_abs);
+            }
+        }
+
+    } while (block && !dirty);
+
+    if (block) {
+        /*
+         * As soon as we start servicing pages out of order, then we have
+         * to kill the bulk stage, since the bulk stage assumes
+         * in (migration_bitmap_find_and_reset_dirty) that every page is
+         * dirty, that's no longer true.
+         */
+        ram_bulk_stage = false;
+
+        /*
+         * We want the background search to continue from the queued page
+         * since the guest is likely to want other pages near to the page
+         * it just requested.
+         */
+        pss->block = block;
+        pss->offset = offset;
+    }
+
+    return !!block;
+}
+
+/**
+ * flush_page_queue: Flush any remaining pages in the ram request queue
+ *    it should be empty at the end anyway, but in error cases there may be
+ *    some left.
+ *
+ * ms: MigrationState
+ */
+void flush_page_queue(MigrationState *ms)
+{
+    struct MigrationSrcPageRequest *mspr, *next_mspr;
+    /* This queue generally should be empty - but in the case of a failed
+     * migration might have some droppings in.
+     */
+    rcu_read_lock();
+    QSIMPLEQ_FOREACH_SAFE(mspr, &ms->src_page_requests, next_req, next_mspr) {
+        memory_region_unref(mspr->rb->mr);
+        QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req);
+        g_free(mspr);
+    }
+    rcu_read_unlock();
+}
+
+/**
+ * Queue the pages for transmission, e.g. a request from postcopy destination
+ *   ms: MigrationStatus in which the queue is held
+ *   rbname: The RAMBlock the request is for - may be NULL (to mean reuse last)
+ *   start: Offset from the start of the RAMBlock
+ *   len: Length (in bytes) to send
+ *   Return: 0 on success
+ */
+int ram_save_queue_pages(MigrationState *ms, const char *rbname,
+                         ram_addr_t start, ram_addr_t len)
+{
+    RAMBlock *ramblock;
+
+    ms->postcopy_requests++;
+    rcu_read_lock();
+    if (!rbname) {
+        /* Reuse last RAMBlock */
+        ramblock = ms->last_req_rb;
+
+        if (!ramblock) {
+            /*
+             * Shouldn't happen, we can't reuse the last RAMBlock if
+             * it's the 1st request.
+             */
+            error_report("ram_save_queue_pages no previous block");
+            goto err;
+        }
+    } else {
+        ramblock = qemu_ram_block_by_name(rbname);
+
+        if (!ramblock) {
+            /* We shouldn't be asked for a non-existent RAMBlock */
+            error_report("ram_save_queue_pages no block '%s'", rbname);
+            goto err;
+        }
+        ms->last_req_rb = ramblock;
+    }
+    trace_ram_save_queue_pages(ramblock->idstr, start, len);
+    if (start+len > ramblock->used_length) {
+        error_report("%s request overrun start=" RAM_ADDR_FMT " len="
+                     RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT,
+                     __func__, start, len, ramblock->used_length);
+        goto err;
+    }
+
+    struct MigrationSrcPageRequest *new_entry =
+        g_malloc0(sizeof(struct MigrationSrcPageRequest));
+    new_entry->rb = ramblock;
+    new_entry->offset = start;
+    new_entry->len = len;
+
+    memory_region_ref(ramblock->mr);
+    qemu_mutex_lock(&ms->src_page_req_mutex);
+    QSIMPLEQ_INSERT_TAIL(&ms->src_page_requests, new_entry, next_req);
+    qemu_mutex_unlock(&ms->src_page_req_mutex);
+    rcu_read_unlock();
+
+    return 0;
+
+err:
+    rcu_read_unlock();
+    return -1;
+}
+
+/**
+ * ram_save_target_page: Save one target page
+ *
+ *
+ * @f: QEMUFile where to send the data
+ * @block: pointer to block that contains the page we want to send
+ * @offset: offset inside the block for the page;
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
+ *
+ * Returns: Number of pages written.
+ */
+static int ram_save_target_page(MigrationState *ms, QEMUFile *f,
+                                PageSearchStatus *pss,
+                                bool last_stage,
+                                uint64_t *bytes_transferred,
+                                ram_addr_t dirty_ram_abs)
+{
+    int res = 0;
+
+    /* Check the pages is dirty and if it is send it */
+    if (migration_bitmap_clear_dirty(dirty_ram_abs)) {
+        unsigned long *unsentmap;
+        if (compression_switch && migrate_use_compression()) {
+            res = ram_save_compressed_page(f, pss,
+                                           last_stage,
+                                           bytes_transferred);
+        } else {
+            res = ram_save_page(f, pss, last_stage,
+                                bytes_transferred);
+        }
+
+        if (res < 0) {
+            return res;
+        }
+        unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+        if (unsentmap) {
+            clear_bit(dirty_ram_abs >> TARGET_PAGE_BITS, unsentmap);
+        }
+        /* Only update last_sent_block if a block was actually sent; xbzrle
+         * might have decided the page was identical so didn't bother writing
+         * to the stream.
+         */
+        if (res > 0) {
+            last_sent_block = pss->block;
+        }
+    }
+
+    return res;
+}
+
+/**
+ * ram_save_host_page: Starting at *offset send pages up to the end
+ *                     of the current host page.  It's valid for the initial
+ *                     offset to point into the middle of a host page
+ *                     in which case the remainder of the hostpage is sent.
+ *                     Only dirty target pages are sent.
+ *
+ * Returns: Number of pages written.
+ *
+ * @f: QEMUFile where to send the data
+ * @block: pointer to block that contains the page we want to send
+ * @offset: offset inside the block for the page; updated to last target page
+ *          sent
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space
+ */
+static int ram_save_host_page(MigrationState *ms, QEMUFile *f,
+                              PageSearchStatus *pss,
+                              bool last_stage,
+                              uint64_t *bytes_transferred,
+                              ram_addr_t dirty_ram_abs)
+{
+    int tmppages, pages = 0;
+    do {
+        tmppages = ram_save_target_page(ms, f, pss, last_stage,
+                                        bytes_transferred, dirty_ram_abs);
+        if (tmppages < 0) {
+            return tmppages;
+        }
+
+        pages += tmppages;
+        pss->offset += TARGET_PAGE_SIZE;
+        dirty_ram_abs += TARGET_PAGE_SIZE;
+    } while (pss->offset & (qemu_host_page_size - 1));
+
+    /* The offset we leave with is the last one we looked at */
+    pss->offset -= TARGET_PAGE_SIZE;
+    return pages;
+}
+
+/**
+ * ram_find_and_save_block: Finds a dirty page and sends it to f
+ *
+ * Called within an RCU critical section.
+ *
+ * Returns:  The number of pages written
+ *           0 means no dirty pages
+ *
+ * @f: QEMUFile where to send the data
+ * @last_stage: if we are at the completion stage
+ * @bytes_transferred: increase it with the number of transferred bytes
+ *
+ * On systems where host-page-size > target-page-size it will send all the
+ * pages in a host page that are dirty.
+ */
+
+static int ram_find_and_save_block(QEMUFile *f, bool last_stage,
+                                   uint64_t *bytes_transferred)
+{
+    PageSearchStatus pss;
+    MigrationState *ms = migrate_get_current();
+    int pages = 0;
+    bool again, found;
+    ram_addr_t dirty_ram_abs; /* Address of the start of the dirty page in
+                                 ram_addr_t space */
+
+    pss.block = last_seen_block;
+    pss.offset = last_offset;
+    pss.complete_round = false;
+
+    if (!pss.block) {
+        pss.block = QLIST_FIRST_RCU(&ram_list.blocks);
+    }
+
+    do {
+        again = true;
+        found = get_queued_page(ms, &pss, &dirty_ram_abs);
+
+        if (!found) {
+            /* priority queue empty, so just search for something dirty */
+            found = find_dirty_block(f, &pss, &again, &dirty_ram_abs);
+        }
+
+        if (found) {
+            pages = ram_save_host_page(ms, f, &pss,
+                                       last_stage, bytes_transferred,
+                                       dirty_ram_abs);
+        }
+    } while (!pages && again);
+
+    last_seen_block = pss.block;
+    last_offset = pss.offset;
+
+    return pages;
+}
+
+void acct_update_position(QEMUFile *f, size_t size, bool zero)
+{
+    uint64_t pages = size / TARGET_PAGE_SIZE;
+    if (zero) {
+        acct_info.dup_pages += pages;
+    } else {
+        acct_info.norm_pages += pages;
+        bytes_transferred += size;
+        qemu_update_position(f, size);
+    }
+}
+
+static ram_addr_t ram_save_remaining(void)
+{
+    return migration_dirty_pages;
+}
+
+uint64_t ram_bytes_remaining(void)
+{
+    return ram_save_remaining() * TARGET_PAGE_SIZE;
+}
+
+uint64_t ram_bytes_transferred(void)
+{
+    return bytes_transferred;
+}
+
+uint64_t ram_bytes_total(void)
+{
+    RAMBlock *block;
+    uint64_t total = 0;
+
+    rcu_read_lock();
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next)
+        total += block->used_length;
+    rcu_read_unlock();
+    return total;
+}
+
+void free_xbzrle_decoded_buf(void)
+{
+    g_free(xbzrle_decoded_buf);
+    xbzrle_decoded_buf = NULL;
+}
+
+static void migration_bitmap_free(struct BitmapRcu *bmap)
+{
+    g_free(bmap->bmap);
+    g_free(bmap->unsentmap);
+    g_free(bmap);
+}
+
+static void ram_migration_cleanup(void *opaque)
+{
+    /* caller have hold iothread lock or is in a bh, so there is
+     * no writing race against this migration_bitmap
+     */
+    struct BitmapRcu *bitmap = migration_bitmap_rcu;
+    atomic_rcu_set(&migration_bitmap_rcu, NULL);
+    if (bitmap) {
+        memory_global_dirty_log_stop();
+        call_rcu(bitmap, migration_bitmap_free, rcu);
+    }
+
+    XBZRLE_cache_lock();
+    if (XBZRLE.cache) {
+        cache_fini(XBZRLE.cache);
+        g_free(XBZRLE.encoded_buf);
+        g_free(XBZRLE.current_buf);
+        XBZRLE.cache = NULL;
+        XBZRLE.encoded_buf = NULL;
+        XBZRLE.current_buf = NULL;
+    }
+    XBZRLE_cache_unlock();
+}
+
+static void reset_ram_globals(void)
+{
+    last_seen_block = NULL;
+    last_sent_block = NULL;
+    last_offset = 0;
+    last_version = ram_list.version;
+    ram_bulk_stage = true;
+}
+
+#define MAX_WAIT 50 /* ms, half buffered_file limit */
+
+void migration_bitmap_extend(ram_addr_t old, ram_addr_t new)
+{
+    /* called in qemu main thread, so there is
+     * no writing race against this migration_bitmap
+     */
+    if (migration_bitmap_rcu) {
+        struct BitmapRcu *old_bitmap = migration_bitmap_rcu, *bitmap;
+        bitmap = g_new(struct BitmapRcu, 1);
+        bitmap->bmap = bitmap_new(new);
+
+        /* prevent migration_bitmap content from being set bit
+         * by migration_bitmap_sync_range() at the same time.
+         * it is safe to migration if migration_bitmap is cleared bit
+         * at the same time.
+         */
+        qemu_mutex_lock(&migration_bitmap_mutex);
+        bitmap_copy(bitmap->bmap, old_bitmap->bmap, old);
+        bitmap_set(bitmap->bmap, old, new - old);
+
+        /* We don't have a way to safely extend the sentmap
+         * with RCU; so mark it as missing, entry to postcopy
+         * will fail.
+         */
+        bitmap->unsentmap = NULL;
+
+        atomic_rcu_set(&migration_bitmap_rcu, bitmap);
+        qemu_mutex_unlock(&migration_bitmap_mutex);
+        migration_dirty_pages += new - old;
+        call_rcu(old_bitmap, migration_bitmap_free, rcu);
+    }
+}
+
+/*
+ * 'expected' is the value you expect the bitmap mostly to be full
+ * of; it won't bother printing lines that are all this value.
+ * If 'todump' is null the migration bitmap is dumped.
+ */
+void ram_debug_dump_bitmap(unsigned long *todump, bool expected)
+{
+    int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    int64_t cur;
+    int64_t linelen = 128;
+    char linebuf[129];
+
+    if (!todump) {
+        todump = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    }
+
+    for (cur = 0; cur < ram_pages; cur += linelen) {
+        int64_t curb;
+        bool found = false;
+        /*
+         * Last line; catch the case where the line length
+         * is longer than remaining ram
+         */
+        if (cur + linelen > ram_pages) {
+            linelen = ram_pages - cur;
+        }
+        for (curb = 0; curb < linelen; curb++) {
+            bool thisbit = test_bit(cur + curb, todump);
+            linebuf[curb] = thisbit ? '1' : '.';
+            found = found || (thisbit != expected);
+        }
+        if (found) {
+            linebuf[curb] = '\0';
+            fprintf(stderr,  "0x%08" PRIx64 " : %s\n", cur, linebuf);
+        }
+    }
+}
+
+/* **** functions for postcopy ***** */
+
+/*
+ * Callback from postcopy_each_ram_send_discard for each RAMBlock
+ * Note: At this point the 'unsentmap' is the processed bitmap combined
+ *       with the dirtymap; so a '1' means it's either dirty or unsent.
+ * start,length: Indexes into the bitmap for the first bit
+ *            representing the named block and length in target-pages
+ */
+static int postcopy_send_discard_bm_ram(MigrationState *ms,
+                                        PostcopyDiscardState *pds,
+                                        unsigned long start,
+                                        unsigned long length)
+{
+    unsigned long end = start + length; /* one after the end */
+    unsigned long current;
+    unsigned long *unsentmap;
+
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+    for (current = start; current < end; ) {
+        unsigned long one = find_next_bit(unsentmap, end, current);
+
+        if (one <= end) {
+            unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1);
+            unsigned long discard_length;
+
+            if (zero >= end) {
+                discard_length = end - one;
+            } else {
+                discard_length = zero - one;
+            }
+            if (discard_length) {
+                postcopy_discard_send_range(ms, pds, one, discard_length);
+            }
+            current = one + discard_length;
+        } else {
+            current = one;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Utility for the outgoing postcopy code.
+ *   Calls postcopy_send_discard_bm_ram for each RAMBlock
+ *   passing it bitmap indexes and name.
+ * Returns: 0 on success
+ * (qemu_ram_foreach_block ends up passing unscaled lengths
+ *  which would mean postcopy code would have to deal with target page)
+ */
+static int postcopy_each_ram_send_discard(MigrationState *ms)
+{
+    struct RAMBlock *block;
+    int ret;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        unsigned long first = block->offset >> TARGET_PAGE_BITS;
+        PostcopyDiscardState *pds = postcopy_discard_send_init(ms,
+                                                               first,
+                                                               block->idstr);
+
+        /*
+         * Postcopy sends chunks of bitmap over the wire, but it
+         * just needs indexes at this point, avoids it having
+         * target page specific code.
+         */
+        ret = postcopy_send_discard_bm_ram(ms, pds, first,
+                                    block->used_length >> TARGET_PAGE_BITS);
+        postcopy_discard_send_finish(ms, pds);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Helper for postcopy_chunk_hostpages; it's called twice to cleanup
+ *   the two bitmaps, that are similar, but one is inverted.
+ *
+ * We search for runs of target-pages that don't start or end on a
+ * host page boundary;
+ * unsent_pass=true: Cleans up partially unsent host pages by searching
+ *                 the unsentmap
+ * unsent_pass=false: Cleans up partially dirty host pages by searching
+ *                 the main migration bitmap
+ *
+ */
+static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass,
+                                          RAMBlock *block,
+                                          PostcopyDiscardState *pds)
+{
+    unsigned long *bitmap;
+    unsigned long *unsentmap;
+    unsigned int host_ratio = qemu_host_page_size / TARGET_PAGE_SIZE;
+    unsigned long first = block->offset >> TARGET_PAGE_BITS;
+    unsigned long len = block->used_length >> TARGET_PAGE_BITS;
+    unsigned long last = first + (len - 1);
+    unsigned long run_start;
+
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+
+    if (unsent_pass) {
+        /* Find a sent page */
+        run_start = find_next_zero_bit(unsentmap, last + 1, first);
+    } else {
+        /* Find a dirty page */
+        run_start = find_next_bit(bitmap, last + 1, first);
+    }
+
+    while (run_start <= last) {
+        bool do_fixup = false;
+        unsigned long fixup_start_addr;
+        unsigned long host_offset;
+
+        /*
+         * If the start of this run of pages is in the middle of a host
+         * page, then we need to fixup this host page.
+         */
+        host_offset = run_start % host_ratio;
+        if (host_offset) {
+            do_fixup = true;
+            run_start -= host_offset;
+            fixup_start_addr = run_start;
+            /* For the next pass */
+            run_start = run_start + host_ratio;
+        } else {
+            /* Find the end of this run */
+            unsigned long run_end;
+            if (unsent_pass) {
+                run_end = find_next_bit(unsentmap, last + 1, run_start + 1);
+            } else {
+                run_end = find_next_zero_bit(bitmap, last + 1, run_start + 1);
+            }
+            /*
+             * If the end isn't at the start of a host page, then the
+             * run doesn't finish at the end of a host page
+             * and we need to discard.
+             */
+            host_offset = run_end % host_ratio;
+            if (host_offset) {
+                do_fixup = true;
+                fixup_start_addr = run_end - host_offset;
+                /*
+                 * This host page has gone, the next loop iteration starts
+                 * from after the fixup
+                 */
+                run_start = fixup_start_addr + host_ratio;
+            } else {
+                /*
+                 * No discards on this iteration, next loop starts from
+                 * next sent/dirty page
+                 */
+                run_start = run_end + 1;
+            }
+        }
+
+        if (do_fixup) {
+            unsigned long page;
+
+            /* Tell the destination to discard this page */
+            if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) {
+                /* For the unsent_pass we:
+                 *     discard partially sent pages
+                 * For the !unsent_pass (dirty) we:
+                 *     discard partially dirty pages that were sent
+                 *     (any partially sent pages were already discarded
+                 *     by the previous unsent_pass)
+                 */
+                postcopy_discard_send_range(ms, pds, fixup_start_addr,
+                                            host_ratio);
+            }
+
+            /* Clean up the bitmap */
+            for (page = fixup_start_addr;
+                 page < fixup_start_addr + host_ratio; page++) {
+                /* All pages in this host page are now not sent */
+                set_bit(page, unsentmap);
+
+                /*
+                 * Remark them as dirty, updating the count for any pages
+                 * that weren't previously dirty.
+                 */
+                migration_dirty_pages += !test_and_set_bit(page, bitmap);
+            }
+        }
+
+        if (unsent_pass) {
+            /* Find the next sent page for the next iteration */
+            run_start = find_next_zero_bit(unsentmap, last + 1,
+                                           run_start);
+        } else {
+            /* Find the next dirty page for the next iteration */
+            run_start = find_next_bit(bitmap, last + 1, run_start);
+        }
+    }
+}
+
+/*
+ * Utility for the outgoing postcopy code.
+ *
+ * Discard any partially sent host-page size chunks, mark any partially
+ * dirty host-page size chunks as all dirty.
+ *
+ * Returns: 0 on success
+ */
+static int postcopy_chunk_hostpages(MigrationState *ms)
+{
+    struct RAMBlock *block;
+
+    if (qemu_host_page_size == TARGET_PAGE_SIZE) {
+        /* Easy case - TPS==HPS - nothing to be done */
+        return 0;
+    }
+
+    /* Easiest way to make sure we don't resume in the middle of a host-page */
+    last_seen_block = NULL;
+    last_sent_block = NULL;
+    last_offset     = 0;
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        unsigned long first = block->offset >> TARGET_PAGE_BITS;
+
+        PostcopyDiscardState *pds =
+                         postcopy_discard_send_init(ms, first, block->idstr);
+
+        /* First pass: Discard all partially sent host pages */
+        postcopy_chunk_hostpages_pass(ms, true, block, pds);
+        /*
+         * Second pass: Ensure that all partially dirty host pages are made
+         * fully dirty.
+         */
+        postcopy_chunk_hostpages_pass(ms, false, block, pds);
+
+        postcopy_discard_send_finish(ms, pds);
+    } /* ram_list loop */
+
+    return 0;
+}
+
+/*
+ * Transmit the set of pages to be discarded after precopy to the target
+ * these are pages that:
+ *     a) Have been previously transmitted but are now dirty again
+ *     b) Pages that have never been transmitted, this ensures that
+ *        any pages on the destination that have been mapped by background
+ *        tasks get discarded (transparent huge pages is the specific concern)
+ * Hopefully this is pretty sparse
+ */
+int ram_postcopy_send_discard_bitmap(MigrationState *ms)
+{
+    int ret;
+    unsigned long *bitmap, *unsentmap;
+
+    rcu_read_lock();
+
+    /* This should be our last sync, the src is now paused */
+    migration_bitmap_sync();
+
+    unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap;
+    if (!unsentmap) {
+        /* We don't have a safe way to resize the sentmap, so
+         * if the bitmap was resized it will be NULL at this
+         * point.
+         */
+        error_report("migration ram resized during precopy phase");
+        rcu_read_unlock();
+        return -EINVAL;
+    }
+
+    /* Deal with TPS != HPS */
+    ret = postcopy_chunk_hostpages(ms);
+    if (ret) {
+        rcu_read_unlock();
+        return ret;
+    }
+
+    /*
+     * Update the unsentmap to be unsentmap = unsentmap | dirty
+     */
+    bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap;
+    bitmap_or(unsentmap, unsentmap, bitmap,
+               last_ram_offset() >> TARGET_PAGE_BITS);
+
+
+    trace_ram_postcopy_send_discard_bitmap();
+#ifdef DEBUG_POSTCOPY
+    ram_debug_dump_bitmap(unsentmap, true);
+#endif
+
+    ret = postcopy_each_ram_send_discard(ms);
+    rcu_read_unlock();
+
+    return ret;
+}
+
+/*
+ * At the start of the postcopy phase of migration, any now-dirty
+ * precopied pages are discarded.
+ *
+ * start, length describe a byte address range within the RAMBlock
+ *
+ * Returns 0 on success.
+ */
+int ram_discard_range(MigrationIncomingState *mis,
+                      const char *block_name,
+                      uint64_t start, size_t length)
+{
+    int ret = -1;
+
+    rcu_read_lock();
+    RAMBlock *rb = qemu_ram_block_by_name(block_name);
+
+    if (!rb) {
+        error_report("ram_discard_range: Failed to find block '%s'",
+                     block_name);
+        goto err;
+    }
+
+    uint8_t *host_startaddr = rb->host + start;
+
+    if ((uintptr_t)host_startaddr & (qemu_host_page_size - 1)) {
+        error_report("ram_discard_range: Unaligned start address: %p",
+                     host_startaddr);
+        goto err;
+    }
+
+    if ((start + length) <= rb->used_length) {
+        uint8_t *host_endaddr = host_startaddr + length;
+        if ((uintptr_t)host_endaddr & (qemu_host_page_size - 1)) {
+            error_report("ram_discard_range: Unaligned end address: %p",
+                         host_endaddr);
+            goto err;
+        }
+        ret = postcopy_ram_discard_range(mis, host_startaddr, length);
+    } else {
+        error_report("ram_discard_range: Overrun block '%s' (%" PRIu64
+                     "/%zx/" RAM_ADDR_FMT")",
+                     block_name, start, length, rb->used_length);
+    }
+
+err:
+    rcu_read_unlock();
+
+    return ret;
+}
+
+
+/* Each of ram_save_setup, ram_save_iterate and ram_save_complete has
+ * long-running RCU critical section.  When rcu-reclaims in the code
+ * start to become numerous it will be necessary to reduce the
+ * granularity of these critical sections.
+ */
+
+static int ram_save_setup(QEMUFile *f, void *opaque)
+{
+    RAMBlock *block;
+    int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
+
+    dirty_rate_high_cnt = 0;
+    bitmap_sync_count = 0;
+    migration_bitmap_sync_init();
+    qemu_mutex_init(&migration_bitmap_mutex);
+
+    if (migrate_use_xbzrle()) {
+        XBZRLE_cache_lock();
+        XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
+                                  TARGET_PAGE_SIZE,
+                                  TARGET_PAGE_SIZE);
+        if (!XBZRLE.cache) {
+            XBZRLE_cache_unlock();
+            error_report("Error creating cache");
+            return -1;
+        }
+        XBZRLE_cache_unlock();
+
+        /* We prefer not to abort if there is no memory */
+        XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
+        if (!XBZRLE.encoded_buf) {
+            error_report("Error allocating encoded_buf");
+            return -1;
+        }
+
+        XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
+        if (!XBZRLE.current_buf) {
+            error_report("Error allocating current_buf");
+            g_free(XBZRLE.encoded_buf);
+            XBZRLE.encoded_buf = NULL;
+            return -1;
+        }
+
+        acct_clear();
+    }
+
+    /* For memory_global_dirty_log_start below.  */
+    qemu_mutex_lock_iothread();
+
+    qemu_mutex_lock_ramlist();
+    rcu_read_lock();
+    bytes_transferred = 0;
+    reset_ram_globals();
+
+    ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+    migration_bitmap_rcu = g_new0(struct BitmapRcu, 1);
+    migration_bitmap_rcu->bmap = bitmap_new(ram_bitmap_pages);
+    bitmap_set(migration_bitmap_rcu->bmap, 0, ram_bitmap_pages);
+
+    if (migrate_postcopy_ram()) {
+        migration_bitmap_rcu->unsentmap = bitmap_new(ram_bitmap_pages);
+        bitmap_set(migration_bitmap_rcu->unsentmap, 0, ram_bitmap_pages);
+    }
+
+    /*
+     * Count the total number of pages used by ram blocks not including any
+     * gaps due to alignment or unplugs.
+     */
+    migration_dirty_pages = ram_bytes_total() >> TARGET_PAGE_BITS;
+
+    memory_global_dirty_log_start();
+    migration_bitmap_sync();
+    qemu_mutex_unlock_ramlist();
+    qemu_mutex_unlock_iothread();
+
+    qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
+
+    QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+        qemu_put_byte(f, strlen(block->idstr));
+        qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));
+        qemu_put_be64(f, block->used_length);
+    }
+
+    rcu_read_unlock();
+
+    ram_control_before_iterate(f, RAM_CONTROL_SETUP);
+    ram_control_after_iterate(f, RAM_CONTROL_SETUP);
+
+    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+
+    return 0;
+}
+
+static int ram_save_iterate(QEMUFile *f, void *opaque)
+{
+    int ret;
+    int i;
+    int64_t t0;
+    int pages_sent = 0;
+
+    rcu_read_lock();
+    if (ram_list.version != last_version) {
+        reset_ram_globals();
+    }
+
+    /* Read version before ram_list.blocks */
+    smp_rmb();
+
+    ram_control_before_iterate(f, RAM_CONTROL_ROUND);
+
+    t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    i = 0;
+    while ((ret = qemu_file_rate_limit(f)) == 0) {
+        int pages;
+
+        pages = ram_find_and_save_block(f, false, &bytes_transferred);
+        /* no more pages to sent */
+        if (pages == 0) {
+            break;
+        }
+        pages_sent += pages;
+        acct_info.iterations++;
+
+        /* we want to check in the 1st loop, just in case it was the 1st time
+           and we had to sync the dirty bitmap.
+           qemu_get_clock_ns() is a bit expensive, so we only check each some
+           iterations
+        */
+        if ((i & 63) == 0) {
+            uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
+            if (t1 > MAX_WAIT) {
+                DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
+                        t1, i);
+                break;
+            }
+        }
+        i++;
+    }
+    flush_compressed_data(f);
+    rcu_read_unlock();
+
+    /*
+     * Must occur before EOS (or any QEMUFile operation)
+     * because of RDMA protocol.
+     */
+    ram_control_after_iterate(f, RAM_CONTROL_ROUND);
+
+    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+    bytes_transferred += 8;
+
+    ret = qemu_file_get_error(f);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return pages_sent;
+}
+
+/* Called with iothread lock */
+static int ram_save_complete(QEMUFile *f, void *opaque)
+{
+    rcu_read_lock();
+
+    if (!migration_in_postcopy(migrate_get_current())) {
+        migration_bitmap_sync();
+    }
+
+    ram_control_before_iterate(f, RAM_CONTROL_FINISH);
+
+    /* try transferring iterative blocks of memory */
+
+    /* flush all remaining blocks regardless of rate limiting */
+    while (true) {
+        int pages;
+
+        pages = ram_find_and_save_block(f, true, &bytes_transferred);
+        /* no more blocks to sent */
+        if (pages == 0) {
+            break;
+        }
+    }
+
+    flush_compressed_data(f);
+    ram_control_after_iterate(f, RAM_CONTROL_FINISH);
+
+    rcu_read_unlock();
+
+    qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
+
+    return 0;
+}
+
+static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
+                             uint64_t *non_postcopiable_pending,
+                             uint64_t *postcopiable_pending)
+{
+    uint64_t remaining_size;
+
+    remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
+
+    if (!migration_in_postcopy(migrate_get_current()) &&
+        remaining_size < max_size) {
+        qemu_mutex_lock_iothread();
+        rcu_read_lock();
+        migration_bitmap_sync();
+        rcu_read_unlock();
+        qemu_mutex_unlock_iothread();
+        remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE;
+    }
+
+    /* We can do postcopy, and all the data is postcopiable */
+    *postcopiable_pending += remaining_size;
+}
+
+static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host)
+{
+    unsigned int xh_len;
+    int xh_flags;
+    uint8_t *loaded_data;
+
+    if (!xbzrle_decoded_buf) {
+        xbzrle_decoded_buf = g_malloc(TARGET_PAGE_SIZE);
+    }
+    loaded_data = xbzrle_decoded_buf;
+
+    /* extract RLE header */
+    xh_flags = qemu_get_byte(f);
+    xh_len = qemu_get_be16(f);
+
+    if (xh_flags != ENCODING_FLAG_XBZRLE) {
+        error_report("Failed to load XBZRLE page - wrong compression!");
+        return -1;
+    }
+
+    if (xh_len > TARGET_PAGE_SIZE) {
+        error_report("Failed to load XBZRLE page - len overflow!");
+        return -1;
+    }
+    /* load data and decode */
+    qemu_get_buffer_in_place(f, &loaded_data, xh_len);
+
+    /* decode RLE */
+    if (xbzrle_decode_buffer(loaded_data, xh_len, host,
+                             TARGET_PAGE_SIZE) == -1) {
+        error_report("Failed to load XBZRLE page - decode error!");
+        return -1;
+    }
+
+    return 0;
+}
+
+/* Must be called from within a rcu critical section.
+ * Returns a pointer from within the RCU-protected ram_list.
+ */
+/*
+ * Read a RAMBlock ID from the stream f.
+ *
+ * f: Stream to read from
+ * flags: Page flags (mostly to see if it's a continuation of previous block)
+ */
+static inline RAMBlock *ram_block_from_stream(QEMUFile *f,
+                                              int flags)
+{
+    static RAMBlock *block = NULL;
+    char id[256];
+    uint8_t len;
+
+    if (flags & RAM_SAVE_FLAG_CONTINUE) {
+        if (!block) {
+            error_report("Ack, bad migration stream!");
+            return NULL;
+        }
+        return block;
+    }
+
+    len = qemu_get_byte(f);
+    qemu_get_buffer(f, (uint8_t *)id, len);
+    id[len] = 0;
+
+    block = qemu_ram_block_by_name(id);
+    if (!block) {
+        error_report("Can't find block %s", id);
+        return NULL;
+    }
+
+    return block;
+}
+
+static inline void *host_from_ram_block_offset(RAMBlock *block,
+                                               ram_addr_t offset)
+{
+    if (!offset_in_ramblock(block, offset)) {
+        return NULL;
+    }
+
+    return block->host + offset;
+}
+
+/*
+ * If a page (or a whole RDMA chunk) has been
+ * determined to be zero, then zap it.
+ */
+void ram_handle_compressed(void *host, uint8_t ch, uint64_t size)
+{
+    if (ch != 0 || !is_zero_range(host, size)) {
+        memset(host, ch, size);
+    }
+}
+
+static void *do_data_decompress(void *opaque)
+{
+    DecompressParam *param = opaque;
+    unsigned long pagesize;
+    uint8_t *des;
+    int len;
+
+    qemu_mutex_lock(&param->mutex);
+    while (!param->quit) {
+        if (param->des) {
+            des = param->des;
+            len = param->len;
+            param->des = 0;
+            qemu_mutex_unlock(&param->mutex);
+
+            pagesize = TARGET_PAGE_SIZE;
+            /* uncompress() will return failed in some case, especially
+             * when the page is dirted when doing the compression, it's
+             * not a problem because the dirty page will be retransferred
+             * and uncompress() won't break the data in other pages.
+             */
+            uncompress((Bytef *)des, &pagesize,
+                       (const Bytef *)param->compbuf, len);
+
+            qemu_mutex_lock(&decomp_done_lock);
+            param->done = true;
+            qemu_cond_signal(&decomp_done_cond);
+            qemu_mutex_unlock(&decomp_done_lock);
+
+            qemu_mutex_lock(&param->mutex);
+        } else {
+            qemu_cond_wait(&param->cond, &param->mutex);
+        }
+    }
+    qemu_mutex_unlock(&param->mutex);
+
+    return NULL;
+}
+
+static void wait_for_decompress_done(void)
+{
+    int idx, thread_count;
+
+    if (!migrate_use_compression()) {
+        return;
+    }
+
+    thread_count = migrate_decompress_threads();
+    qemu_mutex_lock(&decomp_done_lock);
+    for (idx = 0; idx < thread_count; idx++) {
+        while (!decomp_param[idx].done) {
+            qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
+        }
+    }
+    qemu_mutex_unlock(&decomp_done_lock);
+}
+
+void migrate_decompress_threads_create(void)
+{
+    int i, thread_count;
+
+    thread_count = migrate_decompress_threads();
+    decompress_threads = g_new0(QemuThread, thread_count);
+    decomp_param = g_new0(DecompressParam, thread_count);
+    qemu_mutex_init(&decomp_done_lock);
+    qemu_cond_init(&decomp_done_cond);
+    for (i = 0; i < thread_count; i++) {
+        qemu_mutex_init(&decomp_param[i].mutex);
+        qemu_cond_init(&decomp_param[i].cond);
+        decomp_param[i].compbuf = g_malloc0(compressBound(TARGET_PAGE_SIZE));
+        decomp_param[i].done = true;
+        decomp_param[i].quit = false;
+        qemu_thread_create(decompress_threads + i, "decompress",
+                           do_data_decompress, decomp_param + i,
+                           QEMU_THREAD_JOINABLE);
+    }
+}
+
+void migrate_decompress_threads_join(void)
+{
+    int i, thread_count;
+
+    thread_count = migrate_decompress_threads();
+    for (i = 0; i < thread_count; i++) {
+        qemu_mutex_lock(&decomp_param[i].mutex);
+        decomp_param[i].quit = true;
+        qemu_cond_signal(&decomp_param[i].cond);
+        qemu_mutex_unlock(&decomp_param[i].mutex);
+    }
+    for (i = 0; i < thread_count; i++) {
+        qemu_thread_join(decompress_threads + i);
+        qemu_mutex_destroy(&decomp_param[i].mutex);
+        qemu_cond_destroy(&decomp_param[i].cond);
+        g_free(decomp_param[i].compbuf);
+    }
+    g_free(decompress_threads);
+    g_free(decomp_param);
+    decompress_threads = NULL;
+    decomp_param = NULL;
+}
+
+static void decompress_data_with_multi_threads(QEMUFile *f,
+                                               void *host, int len)
+{
+    int idx, thread_count;
+
+    thread_count = migrate_decompress_threads();
+    qemu_mutex_lock(&decomp_done_lock);
+    while (true) {
+        for (idx = 0; idx < thread_count; idx++) {
+            if (decomp_param[idx].done) {
+                decomp_param[idx].done = false;
+                qemu_mutex_lock(&decomp_param[idx].mutex);
+                qemu_get_buffer(f, decomp_param[idx].compbuf, len);
+                decomp_param[idx].des = host;
+                decomp_param[idx].len = len;
+                qemu_cond_signal(&decomp_param[idx].cond);
+                qemu_mutex_unlock(&decomp_param[idx].mutex);
+                break;
+            }
+        }
+        if (idx < thread_count) {
+            break;
+        } else {
+            qemu_cond_wait(&decomp_done_cond, &decomp_done_lock);
+        }
+    }
+    qemu_mutex_unlock(&decomp_done_lock);
+}
+
+/*
+ * Allocate data structures etc needed by incoming migration with postcopy-ram
+ * postcopy-ram's similarly names postcopy_ram_incoming_init does the work
+ */
+int ram_postcopy_incoming_init(MigrationIncomingState *mis)
+{
+    size_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    return postcopy_ram_incoming_init(mis, ram_pages);
+}
+
+/*
+ * Called in postcopy mode by ram_load().
+ * rcu_read_lock is taken prior to this being called.
+ */
+static int ram_load_postcopy(QEMUFile *f)
+{
+    int flags = 0, ret = 0;
+    bool place_needed = false;
+    bool matching_page_sizes = qemu_host_page_size == TARGET_PAGE_SIZE;
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    /* Temporary page that is later 'placed' */
+    void *postcopy_host_page = postcopy_get_tmp_page(mis);
+    void *last_host = NULL;
+    bool all_zero = false;
+
+    while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+        ram_addr_t addr;
+        void *host = NULL;
+        void *page_buffer = NULL;
+        void *place_source = NULL;
+        uint8_t ch;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        trace_ram_load_postcopy_loop((uint64_t)addr, flags);
+        place_needed = false;
+        if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE)) {
+            RAMBlock *block = ram_block_from_stream(f, flags);
+
+            host = host_from_ram_block_offset(block, addr);
+            if (!host) {
+                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+            /*
+             * Postcopy requires that we place whole host pages atomically.
+             * To make it atomic, the data is read into a temporary page
+             * that's moved into place later.
+             * The migration protocol uses,  possibly smaller, target-pages
+             * however the source ensures it always sends all the components
+             * of a host page in order.
+             */
+            page_buffer = postcopy_host_page +
+                          ((uintptr_t)host & ~qemu_host_page_mask);
+            /* If all TP are zero then we can optimise the place */
+            if (!((uintptr_t)host & ~qemu_host_page_mask)) {
+                all_zero = true;
+            } else {
+                /* not the 1st TP within the HP */
+                if (host != (last_host + TARGET_PAGE_SIZE)) {
+                    error_report("Non-sequential target page %p/%p",
+                                  host, last_host);
+                    ret = -EINVAL;
+                    break;
+                }
+            }
+
+
+            /*
+             * If it's the last part of a host page then we place the host
+             * page
+             */
+            place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) &
+                                     ~qemu_host_page_mask) == 0;
+            place_source = postcopy_host_page;
+        }
+        last_host = host;
+
+        switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
+        case RAM_SAVE_FLAG_COMPRESS:
+            ch = qemu_get_byte(f);
+            memset(page_buffer, ch, TARGET_PAGE_SIZE);
+            if (ch) {
+                all_zero = false;
+            }
+            break;
+
+        case RAM_SAVE_FLAG_PAGE:
+            all_zero = false;
+            if (!place_needed || !matching_page_sizes) {
+                qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE);
+            } else {
+                /* Avoids the qemu_file copy during postcopy, which is
+                 * going to do a copy later; can only do it when we
+                 * do this read in one go (matching page sizes)
+                 */
+                qemu_get_buffer_in_place(f, (uint8_t **)&place_source,
+                                         TARGET_PAGE_SIZE);
+            }
+            break;
+        case RAM_SAVE_FLAG_EOS:
+            /* normal exit */
+            break;
+        default:
+            error_report("Unknown combination of migration flags: %#x"
+                         " (postcopy mode)", flags);
+            ret = -EINVAL;
+        }
+
+        if (place_needed) {
+            /* This gets called at the last target page in the host page */
+            if (all_zero) {
+                ret = postcopy_place_page_zero(mis,
+                                               host + TARGET_PAGE_SIZE -
+                                               qemu_host_page_size);
+            } else {
+                ret = postcopy_place_page(mis, host + TARGET_PAGE_SIZE -
+                                               qemu_host_page_size,
+                                               place_source);
+            }
+        }
+        if (!ret) {
+            ret = qemu_file_get_error(f);
+        }
+    }
+
+    return ret;
+}
+
+static int ram_load(QEMUFile *f, void *opaque, int version_id)
+{
+    int flags = 0, ret = 0;
+    static uint64_t seq_iter;
+    int len = 0;
+    /*
+     * If system is running in postcopy mode, page inserts to host memory must
+     * be atomic
+     */
+    bool postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING;
+
+    seq_iter++;
+
+    if (version_id != 4) {
+        ret = -EINVAL;
+    }
+
+    /* This RCU critical section can be very long running.
+     * When RCU reclaims in the code start to become numerous,
+     * it will be necessary to reduce the granularity of this
+     * critical section.
+     */
+    rcu_read_lock();
+
+    if (postcopy_running) {
+        ret = ram_load_postcopy(f);
+    }
+
+    while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) {
+        ram_addr_t addr, total_ram_bytes;
+        void *host = NULL;
+        uint8_t ch;
+
+        addr = qemu_get_be64(f);
+        flags = addr & ~TARGET_PAGE_MASK;
+        addr &= TARGET_PAGE_MASK;
+
+        if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE |
+                     RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
+            RAMBlock *block = ram_block_from_stream(f, flags);
+
+            host = host_from_ram_block_offset(block, addr);
+            if (!host) {
+                error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+        }
+
+        switch (flags & ~RAM_SAVE_FLAG_CONTINUE) {
+        case RAM_SAVE_FLAG_MEM_SIZE:
+            /* Synchronize RAM block list */
+            total_ram_bytes = addr;
+            while (!ret && total_ram_bytes) {
+                RAMBlock *block;
+                char id[256];
+                ram_addr_t length;
+
+                len = qemu_get_byte(f);
+                qemu_get_buffer(f, (uint8_t *)id, len);
+                id[len] = 0;
+                length = qemu_get_be64(f);
+
+                block = qemu_ram_block_by_name(id);
+                if (block) {
+                    if (length != block->used_length) {
+                        Error *local_err = NULL;
+
+                        ret = qemu_ram_resize(block, length,
+                                              &local_err);
+                        if (local_err) {
+                            error_report_err(local_err);
+                        }
+                    }
+                    ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG,
+                                          block->idstr);
+                } else {
+                    error_report("Unknown ramblock \"%s\", cannot "
+                                 "accept migration", id);
+                    ret = -EINVAL;
+                }
+
+                total_ram_bytes -= length;
+            }
+            break;
+
+        case RAM_SAVE_FLAG_COMPRESS:
+            ch = qemu_get_byte(f);
+            ram_handle_compressed(host, ch, TARGET_PAGE_SIZE);
+            break;
+
+        case RAM_SAVE_FLAG_PAGE:
+            qemu_get_buffer(f, host, TARGET_PAGE_SIZE);
+            break;
+
+        case RAM_SAVE_FLAG_COMPRESS_PAGE:
+            len = qemu_get_be32(f);
+            if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) {
+                error_report("Invalid compressed data length: %d", len);
+                ret = -EINVAL;
+                break;
+            }
+            decompress_data_with_multi_threads(f, host, len);
+            break;
+
+        case RAM_SAVE_FLAG_XBZRLE:
+            if (load_xbzrle(f, addr, host) < 0) {
+                error_report("Failed to decompress XBZRLE page at "
+                             RAM_ADDR_FMT, addr);
+                ret = -EINVAL;
+                break;
+            }
+            break;
+        case RAM_SAVE_FLAG_EOS:
+            /* normal exit */
+            break;
+        default:
+            if (flags & RAM_SAVE_FLAG_HOOK) {
+                ram_control_load_hook(f, RAM_CONTROL_HOOK, NULL);
+            } else {
+                error_report("Unknown combination of migration flags: %#x",
+                             flags);
+                ret = -EINVAL;
+            }
+        }
+        if (!ret) {
+            ret = qemu_file_get_error(f);
+        }
+    }
+
+    wait_for_decompress_done();
+    rcu_read_unlock();
+    DPRINTF("Completed load of VM with exit code %d seq iteration "
+            "%" PRIu64 "\n", ret, seq_iter);
+    return ret;
+}
+
+static SaveVMHandlers savevm_ram_handlers = {
+    .save_live_setup = ram_save_setup,
+    .save_live_iterate = ram_save_iterate,
+    .save_live_complete_postcopy = ram_save_complete,
+    .save_live_complete_precopy = ram_save_complete,
+    .save_live_pending = ram_save_pending,
+    .load_state = ram_load,
+    .cleanup = ram_migration_cleanup,
+};
+
+void ram_mig_init(void)
+{
+    qemu_mutex_init(&XBZRLE.lock);
+    register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL);
+}
diff --git a/migration/rdma.c b/migration/rdma.c
new file mode 100644
index 0000000..5110ec8
--- /dev/null
+++ b/migration/rdma.c
@@ -0,0 +1,3696 @@
+/*
+ * RDMA protocol and interfaces
+ *
+ * Copyright IBM, Corp. 2010-2013
+ * Copyright Red Hat, Inc. 2015-2016
+ *
+ * Authors:
+ *  Michael R. Hines <mrhines@us.ibm.com>
+ *  Jiuxing Liu <jl@us.ibm.com>
+ *  Daniel P. Berrange <berrange@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ *
+ */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/cutils.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "exec/cpu-common.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+#include "qemu/sockets.h"
+#include "qemu/bitmap.h"
+#include "qemu/coroutine.h"
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <rdma/rdma_cma.h>
+#include "trace.h"
+
+/*
+ * Print and error on both the Monitor and the Log file.
+ */
+#define ERROR(errp, fmt, ...) \
+    do { \
+        fprintf(stderr, "RDMA ERROR: " fmt "\n", ## __VA_ARGS__); \
+        if (errp && (*(errp) == NULL)) { \
+            error_setg(errp, "RDMA ERROR: " fmt, ## __VA_ARGS__); \
+        } \
+    } while (0)
+
+#define RDMA_RESOLVE_TIMEOUT_MS 10000
+
+/* Do not merge data if larger than this. */
+#define RDMA_MERGE_MAX (2 * 1024 * 1024)
+#define RDMA_SIGNALED_SEND_MAX (RDMA_MERGE_MAX / 4096)
+
+#define RDMA_REG_CHUNK_SHIFT 20 /* 1 MB */
+
+/*
+ * This is only for non-live state being migrated.
+ * Instead of RDMA_WRITE messages, we use RDMA_SEND
+ * messages for that state, which requires a different
+ * delivery design than main memory.
+ */
+#define RDMA_SEND_INCREMENT 32768
+
+/*
+ * Maximum size infiniband SEND message
+ */
+#define RDMA_CONTROL_MAX_BUFFER (512 * 1024)
+#define RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE 4096
+
+#define RDMA_CONTROL_VERSION_CURRENT 1
+/*
+ * Capabilities for negotiation.
+ */
+#define RDMA_CAPABILITY_PIN_ALL 0x01
+
+/*
+ * Add the other flags above to this list of known capabilities
+ * as they are introduced.
+ */
+static uint32_t known_capabilities = RDMA_CAPABILITY_PIN_ALL;
+
+#define CHECK_ERROR_STATE() \
+    do { \
+        if (rdma->error_state) { \
+            if (!rdma->error_reported) { \
+                error_report("RDMA is in an error state waiting migration" \
+                                " to abort!"); \
+                rdma->error_reported = 1; \
+            } \
+            return rdma->error_state; \
+        } \
+    } while (0);
+
+/*
+ * A work request ID is 64-bits and we split up these bits
+ * into 3 parts:
+ *
+ * bits 0-15 : type of control message, 2^16
+ * bits 16-29: ram block index, 2^14
+ * bits 30-63: ram block chunk number, 2^34
+ *
+ * The last two bit ranges are only used for RDMA writes,
+ * in order to track their completion and potentially
+ * also track unregistration status of the message.
+ */
+#define RDMA_WRID_TYPE_SHIFT  0UL
+#define RDMA_WRID_BLOCK_SHIFT 16UL
+#define RDMA_WRID_CHUNK_SHIFT 30UL
+
+#define RDMA_WRID_TYPE_MASK \
+    ((1UL << RDMA_WRID_BLOCK_SHIFT) - 1UL)
+
+#define RDMA_WRID_BLOCK_MASK \
+    (~RDMA_WRID_TYPE_MASK & ((1UL << RDMA_WRID_CHUNK_SHIFT) - 1UL))
+
+#define RDMA_WRID_CHUNK_MASK (~RDMA_WRID_BLOCK_MASK & ~RDMA_WRID_TYPE_MASK)
+
+/*
+ * RDMA migration protocol:
+ * 1. RDMA Writes (data messages, i.e. RAM)
+ * 2. IB Send/Recv (control channel messages)
+ */
+enum {
+    RDMA_WRID_NONE = 0,
+    RDMA_WRID_RDMA_WRITE = 1,
+    RDMA_WRID_SEND_CONTROL = 2000,
+    RDMA_WRID_RECV_CONTROL = 4000,
+};
+
+static const char *wrid_desc[] = {
+    [RDMA_WRID_NONE] = "NONE",
+    [RDMA_WRID_RDMA_WRITE] = "WRITE RDMA",
+    [RDMA_WRID_SEND_CONTROL] = "CONTROL SEND",
+    [RDMA_WRID_RECV_CONTROL] = "CONTROL RECV",
+};
+
+/*
+ * Work request IDs for IB SEND messages only (not RDMA writes).
+ * This is used by the migration protocol to transmit
+ * control messages (such as device state and registration commands)
+ *
+ * We could use more WRs, but we have enough for now.
+ */
+enum {
+    RDMA_WRID_READY = 0,
+    RDMA_WRID_DATA,
+    RDMA_WRID_CONTROL,
+    RDMA_WRID_MAX,
+};
+
+/*
+ * SEND/RECV IB Control Messages.
+ */
+enum {
+    RDMA_CONTROL_NONE = 0,
+    RDMA_CONTROL_ERROR,
+    RDMA_CONTROL_READY,               /* ready to receive */
+    RDMA_CONTROL_QEMU_FILE,           /* QEMUFile-transmitted bytes */
+    RDMA_CONTROL_RAM_BLOCKS_REQUEST,  /* RAMBlock synchronization */
+    RDMA_CONTROL_RAM_BLOCKS_RESULT,   /* RAMBlock synchronization */
+    RDMA_CONTROL_COMPRESS,            /* page contains repeat values */
+    RDMA_CONTROL_REGISTER_REQUEST,    /* dynamic page registration */
+    RDMA_CONTROL_REGISTER_RESULT,     /* key to use after registration */
+    RDMA_CONTROL_REGISTER_FINISHED,   /* current iteration finished */
+    RDMA_CONTROL_UNREGISTER_REQUEST,  /* dynamic UN-registration */
+    RDMA_CONTROL_UNREGISTER_FINISHED, /* unpinning finished */
+};
+
+static const char *control_desc[] = {
+    [RDMA_CONTROL_NONE] = "NONE",
+    [RDMA_CONTROL_ERROR] = "ERROR",
+    [RDMA_CONTROL_READY] = "READY",
+    [RDMA_CONTROL_QEMU_FILE] = "QEMU FILE",
+    [RDMA_CONTROL_RAM_BLOCKS_REQUEST] = "RAM BLOCKS REQUEST",
+    [RDMA_CONTROL_RAM_BLOCKS_RESULT] = "RAM BLOCKS RESULT",
+    [RDMA_CONTROL_COMPRESS] = "COMPRESS",
+    [RDMA_CONTROL_REGISTER_REQUEST] = "REGISTER REQUEST",
+    [RDMA_CONTROL_REGISTER_RESULT] = "REGISTER RESULT",
+    [RDMA_CONTROL_REGISTER_FINISHED] = "REGISTER FINISHED",
+    [RDMA_CONTROL_UNREGISTER_REQUEST] = "UNREGISTER REQUEST",
+    [RDMA_CONTROL_UNREGISTER_FINISHED] = "UNREGISTER FINISHED",
+};
+
+/*
+ * Memory and MR structures used to represent an IB Send/Recv work request.
+ * This is *not* used for RDMA writes, only IB Send/Recv.
+ */
+typedef struct {
+    uint8_t  control[RDMA_CONTROL_MAX_BUFFER]; /* actual buffer to register */
+    struct   ibv_mr *control_mr;               /* registration metadata */
+    size_t   control_len;                      /* length of the message */
+    uint8_t *control_curr;                     /* start of unconsumed bytes */
+} RDMAWorkRequestData;
+
+/*
+ * Negotiate RDMA capabilities during connection-setup time.
+ */
+typedef struct {
+    uint32_t version;
+    uint32_t flags;
+} RDMACapabilities;
+
+static void caps_to_network(RDMACapabilities *cap)
+{
+    cap->version = htonl(cap->version);
+    cap->flags = htonl(cap->flags);
+}
+
+static void network_to_caps(RDMACapabilities *cap)
+{
+    cap->version = ntohl(cap->version);
+    cap->flags = ntohl(cap->flags);
+}
+
+/*
+ * Representation of a RAMBlock from an RDMA perspective.
+ * This is not transmitted, only local.
+ * This and subsequent structures cannot be linked lists
+ * because we're using a single IB message to transmit
+ * the information. It's small anyway, so a list is overkill.
+ */
+typedef struct RDMALocalBlock {
+    char          *block_name;
+    uint8_t       *local_host_addr; /* local virtual address */
+    uint64_t       remote_host_addr; /* remote virtual address */
+    uint64_t       offset;
+    uint64_t       length;
+    struct         ibv_mr **pmr;    /* MRs for chunk-level registration */
+    struct         ibv_mr *mr;      /* MR for non-chunk-level registration */
+    uint32_t      *remote_keys;     /* rkeys for chunk-level registration */
+    uint32_t       remote_rkey;     /* rkeys for non-chunk-level registration */
+    int            index;           /* which block are we */
+    unsigned int   src_index;       /* (Only used on dest) */
+    bool           is_ram_block;
+    int            nb_chunks;
+    unsigned long *transit_bitmap;
+    unsigned long *unregister_bitmap;
+} RDMALocalBlock;
+
+/*
+ * Also represents a RAMblock, but only on the dest.
+ * This gets transmitted by the dest during connection-time
+ * to the source VM and then is used to populate the
+ * corresponding RDMALocalBlock with
+ * the information needed to perform the actual RDMA.
+ */
+typedef struct QEMU_PACKED RDMADestBlock {
+    uint64_t remote_host_addr;
+    uint64_t offset;
+    uint64_t length;
+    uint32_t remote_rkey;
+    uint32_t padding;
+} RDMADestBlock;
+
+static uint64_t htonll(uint64_t v)
+{
+    union { uint32_t lv[2]; uint64_t llv; } u;
+    u.lv[0] = htonl(v >> 32);
+    u.lv[1] = htonl(v & 0xFFFFFFFFULL);
+    return u.llv;
+}
+
+static uint64_t ntohll(uint64_t v) {
+    union { uint32_t lv[2]; uint64_t llv; } u;
+    u.llv = v;
+    return ((uint64_t)ntohl(u.lv[0]) << 32) | (uint64_t) ntohl(u.lv[1]);
+}
+
+static void dest_block_to_network(RDMADestBlock *db)
+{
+    db->remote_host_addr = htonll(db->remote_host_addr);
+    db->offset = htonll(db->offset);
+    db->length = htonll(db->length);
+    db->remote_rkey = htonl(db->remote_rkey);
+}
+
+static void network_to_dest_block(RDMADestBlock *db)
+{
+    db->remote_host_addr = ntohll(db->remote_host_addr);
+    db->offset = ntohll(db->offset);
+    db->length = ntohll(db->length);
+    db->remote_rkey = ntohl(db->remote_rkey);
+}
+
+/*
+ * Virtual address of the above structures used for transmitting
+ * the RAMBlock descriptions at connection-time.
+ * This structure is *not* transmitted.
+ */
+typedef struct RDMALocalBlocks {
+    int nb_blocks;
+    bool     init;             /* main memory init complete */
+    RDMALocalBlock *block;
+} RDMALocalBlocks;
+
+/*
+ * Main data structure for RDMA state.
+ * While there is only one copy of this structure being allocated right now,
+ * this is the place where one would start if you wanted to consider
+ * having more than one RDMA connection open at the same time.
+ */
+typedef struct RDMAContext {
+    char *host;
+    int port;
+
+    RDMAWorkRequestData wr_data[RDMA_WRID_MAX];
+
+    /*
+     * This is used by *_exchange_send() to figure out whether or not
+     * the initial "READY" message has already been received or not.
+     * This is because other functions may potentially poll() and detect
+     * the READY message before send() does, in which case we need to
+     * know if it completed.
+     */
+    int control_ready_expected;
+
+    /* number of outstanding writes */
+    int nb_sent;
+
+    /* store info about current buffer so that we can
+       merge it with future sends */
+    uint64_t current_addr;
+    uint64_t current_length;
+    /* index of ram block the current buffer belongs to */
+    int current_index;
+    /* index of the chunk in the current ram block */
+    int current_chunk;
+
+    bool pin_all;
+
+    /*
+     * infiniband-specific variables for opening the device
+     * and maintaining connection state and so forth.
+     *
+     * cm_id also has ibv_context, rdma_event_channel, and ibv_qp in
+     * cm_id->verbs, cm_id->channel, and cm_id->qp.
+     */
+    struct rdma_cm_id *cm_id;               /* connection manager ID */
+    struct rdma_cm_id *listen_id;
+    bool connected;
+
+    struct ibv_context          *verbs;
+    struct rdma_event_channel   *channel;
+    struct ibv_qp *qp;                      /* queue pair */
+    struct ibv_comp_channel *comp_channel;  /* completion channel */
+    struct ibv_pd *pd;                      /* protection domain */
+    struct ibv_cq *cq;                      /* completion queue */
+
+    /*
+     * If a previous write failed (perhaps because of a failed
+     * memory registration, then do not attempt any future work
+     * and remember the error state.
+     */
+    int error_state;
+    int error_reported;
+
+    /*
+     * Description of ram blocks used throughout the code.
+     */
+    RDMALocalBlocks local_ram_blocks;
+    RDMADestBlock  *dest_blocks;
+
+    /* Index of the next RAMBlock received during block registration */
+    unsigned int    next_src_index;
+
+    /*
+     * Migration on *destination* started.
+     * Then use coroutine yield function.
+     * Source runs in a thread, so we don't care.
+     */
+    int migration_started_on_destination;
+
+    int total_registrations;
+    int total_writes;
+
+    int unregister_current, unregister_next;
+    uint64_t unregistrations[RDMA_SIGNALED_SEND_MAX];
+
+    GHashTable *blockmap;
+} RDMAContext;
+
+#define TYPE_QIO_CHANNEL_RDMA "qio-channel-rdma"
+#define QIO_CHANNEL_RDMA(obj)                                     \
+    OBJECT_CHECK(QIOChannelRDMA, (obj), TYPE_QIO_CHANNEL_RDMA)
+
+typedef struct QIOChannelRDMA QIOChannelRDMA;
+
+
+struct QIOChannelRDMA {
+    QIOChannel parent;
+    RDMAContext *rdma;
+    QEMUFile *file;
+    size_t len;
+    bool blocking; /* XXX we don't actually honour this yet */
+};
+
+/*
+ * Main structure for IB Send/Recv control messages.
+ * This gets prepended at the beginning of every Send/Recv.
+ */
+typedef struct QEMU_PACKED {
+    uint32_t len;     /* Total length of data portion */
+    uint32_t type;    /* which control command to perform */
+    uint32_t repeat;  /* number of commands in data portion of same type */
+    uint32_t padding;
+} RDMAControlHeader;
+
+static void control_to_network(RDMAControlHeader *control)
+{
+    control->type = htonl(control->type);
+    control->len = htonl(control->len);
+    control->repeat = htonl(control->repeat);
+}
+
+static void network_to_control(RDMAControlHeader *control)
+{
+    control->type = ntohl(control->type);
+    control->len = ntohl(control->len);
+    control->repeat = ntohl(control->repeat);
+}
+
+/*
+ * Register a single Chunk.
+ * Information sent by the source VM to inform the dest
+ * to register an single chunk of memory before we can perform
+ * the actual RDMA operation.
+ */
+typedef struct QEMU_PACKED {
+    union QEMU_PACKED {
+        uint64_t current_addr;  /* offset into the ram_addr_t space */
+        uint64_t chunk;         /* chunk to lookup if unregistering */
+    } key;
+    uint32_t current_index; /* which ramblock the chunk belongs to */
+    uint32_t padding;
+    uint64_t chunks;            /* how many sequential chunks to register */
+} RDMARegister;
+
+static void register_to_network(RDMAContext *rdma, RDMARegister *reg)
+{
+    RDMALocalBlock *local_block;
+    local_block  = &rdma->local_ram_blocks.block[reg->current_index];
+
+    if (local_block->is_ram_block) {
+        /*
+         * current_addr as passed in is an address in the local ram_addr_t
+         * space, we need to translate this for the destination
+         */
+        reg->key.current_addr -= local_block->offset;
+        reg->key.current_addr += rdma->dest_blocks[reg->current_index].offset;
+    }
+    reg->key.current_addr = htonll(reg->key.current_addr);
+    reg->current_index = htonl(reg->current_index);
+    reg->chunks = htonll(reg->chunks);
+}
+
+static void network_to_register(RDMARegister *reg)
+{
+    reg->key.current_addr = ntohll(reg->key.current_addr);
+    reg->current_index = ntohl(reg->current_index);
+    reg->chunks = ntohll(reg->chunks);
+}
+
+typedef struct QEMU_PACKED {
+    uint32_t value;     /* if zero, we will madvise() */
+    uint32_t block_idx; /* which ram block index */
+    uint64_t offset;    /* Address in remote ram_addr_t space */
+    uint64_t length;    /* length of the chunk */
+} RDMACompress;
+
+static void compress_to_network(RDMAContext *rdma, RDMACompress *comp)
+{
+    comp->value = htonl(comp->value);
+    /*
+     * comp->offset as passed in is an address in the local ram_addr_t
+     * space, we need to translate this for the destination
+     */
+    comp->offset -= rdma->local_ram_blocks.block[comp->block_idx].offset;
+    comp->offset += rdma->dest_blocks[comp->block_idx].offset;
+    comp->block_idx = htonl(comp->block_idx);
+    comp->offset = htonll(comp->offset);
+    comp->length = htonll(comp->length);
+}
+
+static void network_to_compress(RDMACompress *comp)
+{
+    comp->value = ntohl(comp->value);
+    comp->block_idx = ntohl(comp->block_idx);
+    comp->offset = ntohll(comp->offset);
+    comp->length = ntohll(comp->length);
+}
+
+/*
+ * The result of the dest's memory registration produces an "rkey"
+ * which the source VM must reference in order to perform
+ * the RDMA operation.
+ */
+typedef struct QEMU_PACKED {
+    uint32_t rkey;
+    uint32_t padding;
+    uint64_t host_addr;
+} RDMARegisterResult;
+
+static void result_to_network(RDMARegisterResult *result)
+{
+    result->rkey = htonl(result->rkey);
+    result->host_addr = htonll(result->host_addr);
+};
+
+static void network_to_result(RDMARegisterResult *result)
+{
+    result->rkey = ntohl(result->rkey);
+    result->host_addr = ntohll(result->host_addr);
+};
+
+const char *print_wrid(int wrid);
+static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
+                                   uint8_t *data, RDMAControlHeader *resp,
+                                   int *resp_idx,
+                                   int (*callback)(RDMAContext *rdma));
+
+static inline uint64_t ram_chunk_index(const uint8_t *start,
+                                       const uint8_t *host)
+{
+    return ((uintptr_t) host - (uintptr_t) start) >> RDMA_REG_CHUNK_SHIFT;
+}
+
+static inline uint8_t *ram_chunk_start(const RDMALocalBlock *rdma_ram_block,
+                                       uint64_t i)
+{
+    return (uint8_t *)(uintptr_t)(rdma_ram_block->local_host_addr +
+                                  (i << RDMA_REG_CHUNK_SHIFT));
+}
+
+static inline uint8_t *ram_chunk_end(const RDMALocalBlock *rdma_ram_block,
+                                     uint64_t i)
+{
+    uint8_t *result = ram_chunk_start(rdma_ram_block, i) +
+                                         (1UL << RDMA_REG_CHUNK_SHIFT);
+
+    if (result > (rdma_ram_block->local_host_addr + rdma_ram_block->length)) {
+        result = rdma_ram_block->local_host_addr + rdma_ram_block->length;
+    }
+
+    return result;
+}
+
+static int rdma_add_block(RDMAContext *rdma, const char *block_name,
+                         void *host_addr,
+                         ram_addr_t block_offset, uint64_t length)
+{
+    RDMALocalBlocks *local = &rdma->local_ram_blocks;
+    RDMALocalBlock *block;
+    RDMALocalBlock *old = local->block;
+
+    local->block = g_new0(RDMALocalBlock, local->nb_blocks + 1);
+
+    if (local->nb_blocks) {
+        int x;
+
+        if (rdma->blockmap) {
+            for (x = 0; x < local->nb_blocks; x++) {
+                g_hash_table_remove(rdma->blockmap,
+                                    (void *)(uintptr_t)old[x].offset);
+                g_hash_table_insert(rdma->blockmap,
+                                    (void *)(uintptr_t)old[x].offset,
+                                    &local->block[x]);
+            }
+        }
+        memcpy(local->block, old, sizeof(RDMALocalBlock) * local->nb_blocks);
+        g_free(old);
+    }
+
+    block = &local->block[local->nb_blocks];
+
+    block->block_name = g_strdup(block_name);
+    block->local_host_addr = host_addr;
+    block->offset = block_offset;
+    block->length = length;
+    block->index = local->nb_blocks;
+    block->src_index = ~0U; /* Filled in by the receipt of the block list */
+    block->nb_chunks = ram_chunk_index(host_addr, host_addr + length) + 1UL;
+    block->transit_bitmap = bitmap_new(block->nb_chunks);
+    bitmap_clear(block->transit_bitmap, 0, block->nb_chunks);
+    block->unregister_bitmap = bitmap_new(block->nb_chunks);
+    bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks);
+    block->remote_keys = g_new0(uint32_t, block->nb_chunks);
+
+    block->is_ram_block = local->init ? false : true;
+
+    if (rdma->blockmap) {
+        g_hash_table_insert(rdma->blockmap, (void *)(uintptr_t)block_offset, block);
+    }
+
+    trace_rdma_add_block(block_name, local->nb_blocks,
+                         (uintptr_t) block->local_host_addr,
+                         block->offset, block->length,
+                         (uintptr_t) (block->local_host_addr + block->length),
+                         BITS_TO_LONGS(block->nb_chunks) *
+                             sizeof(unsigned long) * 8,
+                         block->nb_chunks);
+
+    local->nb_blocks++;
+
+    return 0;
+}
+
+/*
+ * Memory regions need to be registered with the device and queue pairs setup
+ * in advanced before the migration starts. This tells us where the RAM blocks
+ * are so that we can register them individually.
+ */
+static int qemu_rdma_init_one_block(const char *block_name, void *host_addr,
+    ram_addr_t block_offset, ram_addr_t length, void *opaque)
+{
+    return rdma_add_block(opaque, block_name, host_addr, block_offset, length);
+}
+
+/*
+ * Identify the RAMBlocks and their quantity. They will be references to
+ * identify chunk boundaries inside each RAMBlock and also be referenced
+ * during dynamic page registration.
+ */
+static int qemu_rdma_init_ram_blocks(RDMAContext *rdma)
+{
+    RDMALocalBlocks *local = &rdma->local_ram_blocks;
+
+    assert(rdma->blockmap == NULL);
+    memset(local, 0, sizeof *local);
+    qemu_ram_foreach_block(qemu_rdma_init_one_block, rdma);
+    trace_qemu_rdma_init_ram_blocks(local->nb_blocks);
+    rdma->dest_blocks = g_new0(RDMADestBlock,
+                               rdma->local_ram_blocks.nb_blocks);
+    local->init = true;
+    return 0;
+}
+
+/*
+ * Note: If used outside of cleanup, the caller must ensure that the destination
+ * block structures are also updated
+ */
+static int rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block)
+{
+    RDMALocalBlocks *local = &rdma->local_ram_blocks;
+    RDMALocalBlock *old = local->block;
+    int x;
+
+    if (rdma->blockmap) {
+        g_hash_table_remove(rdma->blockmap, (void *)(uintptr_t)block->offset);
+    }
+    if (block->pmr) {
+        int j;
+
+        for (j = 0; j < block->nb_chunks; j++) {
+            if (!block->pmr[j]) {
+                continue;
+            }
+            ibv_dereg_mr(block->pmr[j]);
+            rdma->total_registrations--;
+        }
+        g_free(block->pmr);
+        block->pmr = NULL;
+    }
+
+    if (block->mr) {
+        ibv_dereg_mr(block->mr);
+        rdma->total_registrations--;
+        block->mr = NULL;
+    }
+
+    g_free(block->transit_bitmap);
+    block->transit_bitmap = NULL;
+
+    g_free(block->unregister_bitmap);
+    block->unregister_bitmap = NULL;
+
+    g_free(block->remote_keys);
+    block->remote_keys = NULL;
+
+    g_free(block->block_name);
+    block->block_name = NULL;
+
+    if (rdma->blockmap) {
+        for (x = 0; x < local->nb_blocks; x++) {
+            g_hash_table_remove(rdma->blockmap,
+                                (void *)(uintptr_t)old[x].offset);
+        }
+    }
+
+    if (local->nb_blocks > 1) {
+
+        local->block = g_new0(RDMALocalBlock, local->nb_blocks - 1);
+
+        if (block->index) {
+            memcpy(local->block, old, sizeof(RDMALocalBlock) * block->index);
+        }
+
+        if (block->index < (local->nb_blocks - 1)) {
+            memcpy(local->block + block->index, old + (block->index + 1),
+                sizeof(RDMALocalBlock) *
+                    (local->nb_blocks - (block->index + 1)));
+        }
+    } else {
+        assert(block == local->block);
+        local->block = NULL;
+    }
+
+    trace_rdma_delete_block(block, (uintptr_t)block->local_host_addr,
+                           block->offset, block->length,
+                            (uintptr_t)(block->local_host_addr + block->length),
+                           BITS_TO_LONGS(block->nb_chunks) *
+                               sizeof(unsigned long) * 8, block->nb_chunks);
+
+    g_free(old);
+
+    local->nb_blocks--;
+
+    if (local->nb_blocks && rdma->blockmap) {
+        for (x = 0; x < local->nb_blocks; x++) {
+            g_hash_table_insert(rdma->blockmap,
+                                (void *)(uintptr_t)local->block[x].offset,
+                                &local->block[x]);
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Put in the log file which RDMA device was opened and the details
+ * associated with that device.
+ */
+static void qemu_rdma_dump_id(const char *who, struct ibv_context *verbs)
+{
+    struct ibv_port_attr port;
+
+    if (ibv_query_port(verbs, 1, &port)) {
+        error_report("Failed to query port information");
+        return;
+    }
+
+    printf("%s RDMA Device opened: kernel name %s "
+           "uverbs device name %s, "
+           "infiniband_verbs class device path %s, "
+           "infiniband class device path %s, "
+           "transport: (%d) %s\n",
+                who,
+                verbs->device->name,
+                verbs->device->dev_name,
+                verbs->device->dev_path,
+                verbs->device->ibdev_path,
+                port.link_layer,
+                (port.link_layer == IBV_LINK_LAYER_INFINIBAND) ? "Infiniband" :
+                 ((port.link_layer == IBV_LINK_LAYER_ETHERNET)
+                    ? "Ethernet" : "Unknown"));
+}
+
+/*
+ * Put in the log file the RDMA gid addressing information,
+ * useful for folks who have trouble understanding the
+ * RDMA device hierarchy in the kernel.
+ */
+static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id)
+{
+    char sgid[33];
+    char dgid[33];
+    inet_ntop(AF_INET6, &id->route.addr.addr.ibaddr.sgid, sgid, sizeof sgid);
+    inet_ntop(AF_INET6, &id->route.addr.addr.ibaddr.dgid, dgid, sizeof dgid);
+    trace_qemu_rdma_dump_gid(who, sgid, dgid);
+}
+
+/*
+ * As of now, IPv6 over RoCE / iWARP is not supported by linux.
+ * We will try the next addrinfo struct, and fail if there are
+ * no other valid addresses to bind against.
+ *
+ * If user is listening on '[::]', then we will not have a opened a device
+ * yet and have no way of verifying if the device is RoCE or not.
+ *
+ * In this case, the source VM will throw an error for ALL types of
+ * connections (both IPv4 and IPv6) if the destination machine does not have
+ * a regular infiniband network available for use.
+ *
+ * The only way to guarantee that an error is thrown for broken kernels is
+ * for the management software to choose a *specific* interface at bind time
+ * and validate what time of hardware it is.
+ *
+ * Unfortunately, this puts the user in a fix:
+ *
+ *  If the source VM connects with an IPv4 address without knowing that the
+ *  destination has bound to '[::]' the migration will unconditionally fail
+ *  unless the management software is explicitly listening on the IPv4
+ *  address while using a RoCE-based device.
+ *
+ *  If the source VM connects with an IPv6 address, then we're OK because we can
+ *  throw an error on the source (and similarly on the destination).
+ *
+ *  But in mixed environments, this will be broken for a while until it is fixed
+ *  inside linux.
+ *
+ * We do provide a *tiny* bit of help in this function: We can list all of the
+ * devices in the system and check to see if all the devices are RoCE or
+ * Infiniband.
+ *
+ * If we detect that we have a *pure* RoCE environment, then we can safely
+ * thrown an error even if the management software has specified '[::]' as the
+ * bind address.
+ *
+ * However, if there is are multiple hetergeneous devices, then we cannot make
+ * this assumption and the user just has to be sure they know what they are
+ * doing.
+ *
+ * Patches are being reviewed on linux-rdma.
+ */
+static int qemu_rdma_broken_ipv6_kernel(Error **errp, struct ibv_context *verbs)
+{
+    struct ibv_port_attr port_attr;
+
+    /* This bug only exists in linux, to our knowledge. */
+#ifdef CONFIG_LINUX
+
+    /*
+     * Verbs are only NULL if management has bound to '[::]'.
+     *
+     * Let's iterate through all the devices and see if there any pure IB
+     * devices (non-ethernet).
+     *
+     * If not, then we can safely proceed with the migration.
+     * Otherwise, there are no guarantees until the bug is fixed in linux.
+     */
+    if (!verbs) {
+        int num_devices, x;
+        struct ibv_device ** dev_list = ibv_get_device_list(&num_devices);
+        bool roce_found = false;
+        bool ib_found = false;
+
+        for (x = 0; x < num_devices; x++) {
+            verbs = ibv_open_device(dev_list[x]);
+            if (!verbs) {
+                if (errno == EPERM) {
+                    continue;
+                } else {
+                    return -EINVAL;
+                }
+            }
+
+            if (ibv_query_port(verbs, 1, &port_attr)) {
+                ibv_close_device(verbs);
+                ERROR(errp, "Could not query initial IB port");
+                return -EINVAL;
+            }
+
+            if (port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) {
+                ib_found = true;
+            } else if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
+                roce_found = true;
+            }
+
+            ibv_close_device(verbs);
+
+        }
+
+        if (roce_found) {
+            if (ib_found) {
+                fprintf(stderr, "WARN: migrations may fail:"
+                                " IPv6 over RoCE / iWARP in linux"
+                                " is broken. But since you appear to have a"
+                                " mixed RoCE / IB environment, be sure to only"
+                                " migrate over the IB fabric until the kernel "
+                                " fixes the bug.\n");
+            } else {
+                ERROR(errp, "You only have RoCE / iWARP devices in your systems"
+                            " and your management software has specified '[::]'"
+                            ", but IPv6 over RoCE / iWARP is not supported in Linux.");
+                return -ENONET;
+            }
+        }
+
+        return 0;
+    }
+
+    /*
+     * If we have a verbs context, that means that some other than '[::]' was
+     * used by the management software for binding. In which case we can
+     * actually warn the user about a potentially broken kernel.
+     */
+
+    /* IB ports start with 1, not 0 */
+    if (ibv_query_port(verbs, 1, &port_attr)) {
+        ERROR(errp, "Could not query initial IB port");
+        return -EINVAL;
+    }
+
+    if (port_attr.link_layer == IBV_LINK_LAYER_ETHERNET) {
+        ERROR(errp, "Linux kernel's RoCE / iWARP does not support IPv6 "
+                    "(but patches on linux-rdma in progress)");
+        return -ENONET;
+    }
+
+#endif
+
+    return 0;
+}
+
+/*
+ * Figure out which RDMA device corresponds to the requested IP hostname
+ * Also create the initial connection manager identifiers for opening
+ * the connection.
+ */
+static int qemu_rdma_resolve_host(RDMAContext *rdma, Error **errp)
+{
+    int ret;
+    struct rdma_addrinfo *res;
+    char port_str[16];
+    struct rdma_cm_event *cm_event;
+    char ip[40] = "unknown";
+    struct rdma_addrinfo *e;
+
+    if (rdma->host == NULL || !strcmp(rdma->host, "")) {
+        ERROR(errp, "RDMA hostname has not been set");
+        return -EINVAL;
+    }
+
+    /* create CM channel */
+    rdma->channel = rdma_create_event_channel();
+    if (!rdma->channel) {
+        ERROR(errp, "could not create CM channel");
+        return -EINVAL;
+    }
+
+    /* create CM id */
+    ret = rdma_create_id(rdma->channel, &rdma->cm_id, NULL, RDMA_PS_TCP);
+    if (ret) {
+        ERROR(errp, "could not create channel id");
+        goto err_resolve_create_id;
+    }
+
+    snprintf(port_str, 16, "%d", rdma->port);
+    port_str[15] = '\0';
+
+    ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
+    if (ret < 0) {
+        ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
+        goto err_resolve_get_addr;
+    }
+
+    for (e = res; e != NULL; e = e->ai_next) {
+        inet_ntop(e->ai_family,
+            &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip);
+        trace_qemu_rdma_resolve_host_trying(rdma->host, ip);
+
+        ret = rdma_resolve_addr(rdma->cm_id, NULL, e->ai_dst_addr,
+                RDMA_RESOLVE_TIMEOUT_MS);
+        if (!ret) {
+            if (e->ai_family == AF_INET6) {
+                ret = qemu_rdma_broken_ipv6_kernel(errp, rdma->cm_id->verbs);
+                if (ret) {
+                    continue;
+                }
+            }
+            goto route;
+        }
+    }
+
+    ERROR(errp, "could not resolve address %s", rdma->host);
+    goto err_resolve_get_addr;
+
+route:
+    qemu_rdma_dump_gid("source_resolve_addr", rdma->cm_id);
+
+    ret = rdma_get_cm_event(rdma->channel, &cm_event);
+    if (ret) {
+        ERROR(errp, "could not perform event_addr_resolved");
+        goto err_resolve_get_addr;
+    }
+
+    if (cm_event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
+        ERROR(errp, "result not equal to event_addr_resolved %s",
+                rdma_event_str(cm_event->event));
+        perror("rdma_resolve_addr");
+        rdma_ack_cm_event(cm_event);
+        ret = -EINVAL;
+        goto err_resolve_get_addr;
+    }
+    rdma_ack_cm_event(cm_event);
+
+    /* resolve route */
+    ret = rdma_resolve_route(rdma->cm_id, RDMA_RESOLVE_TIMEOUT_MS);
+    if (ret) {
+        ERROR(errp, "could not resolve rdma route");
+        goto err_resolve_get_addr;
+    }
+
+    ret = rdma_get_cm_event(rdma->channel, &cm_event);
+    if (ret) {
+        ERROR(errp, "could not perform event_route_resolved");
+        goto err_resolve_get_addr;
+    }
+    if (cm_event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
+        ERROR(errp, "result not equal to event_route_resolved: %s",
+                        rdma_event_str(cm_event->event));
+        rdma_ack_cm_event(cm_event);
+        ret = -EINVAL;
+        goto err_resolve_get_addr;
+    }
+    rdma_ack_cm_event(cm_event);
+    rdma->verbs = rdma->cm_id->verbs;
+    qemu_rdma_dump_id("source_resolve_host", rdma->cm_id->verbs);
+    qemu_rdma_dump_gid("source_resolve_host", rdma->cm_id);
+    return 0;
+
+err_resolve_get_addr:
+    rdma_destroy_id(rdma->cm_id);
+    rdma->cm_id = NULL;
+err_resolve_create_id:
+    rdma_destroy_event_channel(rdma->channel);
+    rdma->channel = NULL;
+    return ret;
+}
+
+/*
+ * Create protection domain and completion queues
+ */
+static int qemu_rdma_alloc_pd_cq(RDMAContext *rdma)
+{
+    /* allocate pd */
+    rdma->pd = ibv_alloc_pd(rdma->verbs);
+    if (!rdma->pd) {
+        error_report("failed to allocate protection domain");
+        return -1;
+    }
+
+    /* create completion channel */
+    rdma->comp_channel = ibv_create_comp_channel(rdma->verbs);
+    if (!rdma->comp_channel) {
+        error_report("failed to allocate completion channel");
+        goto err_alloc_pd_cq;
+    }
+
+    /*
+     * Completion queue can be filled by both read and write work requests,
+     * so must reflect the sum of both possible queue sizes.
+     */
+    rdma->cq = ibv_create_cq(rdma->verbs, (RDMA_SIGNALED_SEND_MAX * 3),
+            NULL, rdma->comp_channel, 0);
+    if (!rdma->cq) {
+        error_report("failed to allocate completion queue");
+        goto err_alloc_pd_cq;
+    }
+
+    return 0;
+
+err_alloc_pd_cq:
+    if (rdma->pd) {
+        ibv_dealloc_pd(rdma->pd);
+    }
+    if (rdma->comp_channel) {
+        ibv_destroy_comp_channel(rdma->comp_channel);
+    }
+    rdma->pd = NULL;
+    rdma->comp_channel = NULL;
+    return -1;
+
+}
+
+/*
+ * Create queue pairs.
+ */
+static int qemu_rdma_alloc_qp(RDMAContext *rdma)
+{
+    struct ibv_qp_init_attr attr = { 0 };
+    int ret;
+
+    attr.cap.max_send_wr = RDMA_SIGNALED_SEND_MAX;
+    attr.cap.max_recv_wr = 3;
+    attr.cap.max_send_sge = 1;
+    attr.cap.max_recv_sge = 1;
+    attr.send_cq = rdma->cq;
+    attr.recv_cq = rdma->cq;
+    attr.qp_type = IBV_QPT_RC;
+
+    ret = rdma_create_qp(rdma->cm_id, rdma->pd, &attr);
+    if (ret) {
+        return -1;
+    }
+
+    rdma->qp = rdma->cm_id->qp;
+    return 0;
+}
+
+static int qemu_rdma_reg_whole_ram_blocks(RDMAContext *rdma)
+{
+    int i;
+    RDMALocalBlocks *local = &rdma->local_ram_blocks;
+
+    for (i = 0; i < local->nb_blocks; i++) {
+        local->block[i].mr =
+            ibv_reg_mr(rdma->pd,
+                    local->block[i].local_host_addr,
+                    local->block[i].length,
+                    IBV_ACCESS_LOCAL_WRITE |
+                    IBV_ACCESS_REMOTE_WRITE
+                    );
+        if (!local->block[i].mr) {
+            perror("Failed to register local dest ram block!\n");
+            break;
+        }
+        rdma->total_registrations++;
+    }
+
+    if (i >= local->nb_blocks) {
+        return 0;
+    }
+
+    for (i--; i >= 0; i--) {
+        ibv_dereg_mr(local->block[i].mr);
+        rdma->total_registrations--;
+    }
+
+    return -1;
+
+}
+
+/*
+ * Find the ram block that corresponds to the page requested to be
+ * transmitted by QEMU.
+ *
+ * Once the block is found, also identify which 'chunk' within that
+ * block that the page belongs to.
+ *
+ * This search cannot fail or the migration will fail.
+ */
+static int qemu_rdma_search_ram_block(RDMAContext *rdma,
+                                      uintptr_t block_offset,
+                                      uint64_t offset,
+                                      uint64_t length,
+                                      uint64_t *block_index,
+                                      uint64_t *chunk_index)
+{
+    uint64_t current_addr = block_offset + offset;
+    RDMALocalBlock *block = g_hash_table_lookup(rdma->blockmap,
+                                                (void *) block_offset);
+    assert(block);
+    assert(current_addr >= block->offset);
+    assert((current_addr + length) <= (block->offset + block->length));
+
+    *block_index = block->index;
+    *chunk_index = ram_chunk_index(block->local_host_addr,
+                block->local_host_addr + (current_addr - block->offset));
+
+    return 0;
+}
+
+/*
+ * Register a chunk with IB. If the chunk was already registered
+ * previously, then skip.
+ *
+ * Also return the keys associated with the registration needed
+ * to perform the actual RDMA operation.
+ */
+static int qemu_rdma_register_and_get_keys(RDMAContext *rdma,
+        RDMALocalBlock *block, uintptr_t host_addr,
+        uint32_t *lkey, uint32_t *rkey, int chunk,
+        uint8_t *chunk_start, uint8_t *chunk_end)
+{
+    if (block->mr) {
+        if (lkey) {
+            *lkey = block->mr->lkey;
+        }
+        if (rkey) {
+            *rkey = block->mr->rkey;
+        }
+        return 0;
+    }
+
+    /* allocate memory to store chunk MRs */
+    if (!block->pmr) {
+        block->pmr = g_new0(struct ibv_mr *, block->nb_chunks);
+    }
+
+    /*
+     * If 'rkey', then we're the destination, so grant access to the source.
+     *
+     * If 'lkey', then we're the source VM, so grant access only to ourselves.
+     */
+    if (!block->pmr[chunk]) {
+        uint64_t len = chunk_end - chunk_start;
+
+        trace_qemu_rdma_register_and_get_keys(len, chunk_start);
+
+        block->pmr[chunk] = ibv_reg_mr(rdma->pd,
+                chunk_start, len,
+                (rkey ? (IBV_ACCESS_LOCAL_WRITE |
+                        IBV_ACCESS_REMOTE_WRITE) : 0));
+
+        if (!block->pmr[chunk]) {
+            perror("Failed to register chunk!");
+            fprintf(stderr, "Chunk details: block: %d chunk index %d"
+                            " start %" PRIuPTR " end %" PRIuPTR
+                            " host %" PRIuPTR
+                            " local %" PRIuPTR " registrations: %d\n",
+                            block->index, chunk, (uintptr_t)chunk_start,
+                            (uintptr_t)chunk_end, host_addr,
+                            (uintptr_t)block->local_host_addr,
+                            rdma->total_registrations);
+            return -1;
+        }
+        rdma->total_registrations++;
+    }
+
+    if (lkey) {
+        *lkey = block->pmr[chunk]->lkey;
+    }
+    if (rkey) {
+        *rkey = block->pmr[chunk]->rkey;
+    }
+    return 0;
+}
+
+/*
+ * Register (at connection time) the memory used for control
+ * channel messages.
+ */
+static int qemu_rdma_reg_control(RDMAContext *rdma, int idx)
+{
+    rdma->wr_data[idx].control_mr = ibv_reg_mr(rdma->pd,
+            rdma->wr_data[idx].control, RDMA_CONTROL_MAX_BUFFER,
+            IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
+    if (rdma->wr_data[idx].control_mr) {
+        rdma->total_registrations++;
+        return 0;
+    }
+    error_report("qemu_rdma_reg_control failed");
+    return -1;
+}
+
+const char *print_wrid(int wrid)
+{
+    if (wrid >= RDMA_WRID_RECV_CONTROL) {
+        return wrid_desc[RDMA_WRID_RECV_CONTROL];
+    }
+    return wrid_desc[wrid];
+}
+
+/*
+ * RDMA requires memory registration (mlock/pinning), but this is not good for
+ * overcommitment.
+ *
+ * In preparation for the future where LRU information or workload-specific
+ * writable writable working set memory access behavior is available to QEMU
+ * it would be nice to have in place the ability to UN-register/UN-pin
+ * particular memory regions from the RDMA hardware when it is determine that
+ * those regions of memory will likely not be accessed again in the near future.
+ *
+ * While we do not yet have such information right now, the following
+ * compile-time option allows us to perform a non-optimized version of this
+ * behavior.
+ *
+ * By uncommenting this option, you will cause *all* RDMA transfers to be
+ * unregistered immediately after the transfer completes on both sides of the
+ * connection. This has no effect in 'rdma-pin-all' mode, only regular mode.
+ *
+ * This will have a terrible impact on migration performance, so until future
+ * workload information or LRU information is available, do not attempt to use
+ * this feature except for basic testing.
+ */
+//#define RDMA_UNREGISTRATION_EXAMPLE
+
+/*
+ * Perform a non-optimized memory unregistration after every transfer
+ * for demonstration purposes, only if pin-all is not requested.
+ *
+ * Potential optimizations:
+ * 1. Start a new thread to run this function continuously
+        - for bit clearing
+        - and for receipt of unregister messages
+ * 2. Use an LRU.
+ * 3. Use workload hints.
+ */
+static int qemu_rdma_unregister_waiting(RDMAContext *rdma)
+{
+    while (rdma->unregistrations[rdma->unregister_current]) {
+        int ret;
+        uint64_t wr_id = rdma->unregistrations[rdma->unregister_current];
+        uint64_t chunk =
+            (wr_id & RDMA_WRID_CHUNK_MASK) >> RDMA_WRID_CHUNK_SHIFT;
+        uint64_t index =
+            (wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT;
+        RDMALocalBlock *block =
+            &(rdma->local_ram_blocks.block[index]);
+        RDMARegister reg = { .current_index = index };
+        RDMAControlHeader resp = { .type = RDMA_CONTROL_UNREGISTER_FINISHED,
+                                 };
+        RDMAControlHeader head = { .len = sizeof(RDMARegister),
+                                   .type = RDMA_CONTROL_UNREGISTER_REQUEST,
+                                   .repeat = 1,
+                                 };
+
+        trace_qemu_rdma_unregister_waiting_proc(chunk,
+                                                rdma->unregister_current);
+
+        rdma->unregistrations[rdma->unregister_current] = 0;
+        rdma->unregister_current++;
+
+        if (rdma->unregister_current == RDMA_SIGNALED_SEND_MAX) {
+            rdma->unregister_current = 0;
+        }
+
+
+        /*
+         * Unregistration is speculative (because migration is single-threaded
+         * and we cannot break the protocol's inifinband message ordering).
+         * Thus, if the memory is currently being used for transmission,
+         * then abort the attempt to unregister and try again
+         * later the next time a completion is received for this memory.
+         */
+        clear_bit(chunk, block->unregister_bitmap);
+
+        if (test_bit(chunk, block->transit_bitmap)) {
+            trace_qemu_rdma_unregister_waiting_inflight(chunk);
+            continue;
+        }
+
+        trace_qemu_rdma_unregister_waiting_send(chunk);
+
+        ret = ibv_dereg_mr(block->pmr[chunk]);
+        block->pmr[chunk] = NULL;
+        block->remote_keys[chunk] = 0;
+
+        if (ret != 0) {
+            perror("unregistration chunk failed");
+            return -ret;
+        }
+        rdma->total_registrations--;
+
+        reg.key.chunk = chunk;
+        register_to_network(rdma, &reg);
+        ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) &reg,
+                                &resp, NULL, NULL);
+        if (ret < 0) {
+            return ret;
+        }
+
+        trace_qemu_rdma_unregister_waiting_complete(chunk);
+    }
+
+    return 0;
+}
+
+static uint64_t qemu_rdma_make_wrid(uint64_t wr_id, uint64_t index,
+                                         uint64_t chunk)
+{
+    uint64_t result = wr_id & RDMA_WRID_TYPE_MASK;
+
+    result |= (index << RDMA_WRID_BLOCK_SHIFT);
+    result |= (chunk << RDMA_WRID_CHUNK_SHIFT);
+
+    return result;
+}
+
+/*
+ * Set bit for unregistration in the next iteration.
+ * We cannot transmit right here, but will unpin later.
+ */
+static void qemu_rdma_signal_unregister(RDMAContext *rdma, uint64_t index,
+                                        uint64_t chunk, uint64_t wr_id)
+{
+    if (rdma->unregistrations[rdma->unregister_next] != 0) {
+        error_report("rdma migration: queue is full");
+    } else {
+        RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]);
+
+        if (!test_and_set_bit(chunk, block->unregister_bitmap)) {
+            trace_qemu_rdma_signal_unregister_append(chunk,
+                                                     rdma->unregister_next);
+
+            rdma->unregistrations[rdma->unregister_next++] =
+                    qemu_rdma_make_wrid(wr_id, index, chunk);
+
+            if (rdma->unregister_next == RDMA_SIGNALED_SEND_MAX) {
+                rdma->unregister_next = 0;
+            }
+        } else {
+            trace_qemu_rdma_signal_unregister_already(chunk);
+        }
+    }
+}
+
+/*
+ * Consult the connection manager to see a work request
+ * (of any kind) has completed.
+ * Return the work request ID that completed.
+ */
+static uint64_t qemu_rdma_poll(RDMAContext *rdma, uint64_t *wr_id_out,
+                               uint32_t *byte_len)
+{
+    int ret;
+    struct ibv_wc wc;
+    uint64_t wr_id;
+
+    ret = ibv_poll_cq(rdma->cq, 1, &wc);
+
+    if (!ret) {
+        *wr_id_out = RDMA_WRID_NONE;
+        return 0;
+    }
+
+    if (ret < 0) {
+        error_report("ibv_poll_cq return %d", ret);
+        return ret;
+    }
+
+    wr_id = wc.wr_id & RDMA_WRID_TYPE_MASK;
+
+    if (wc.status != IBV_WC_SUCCESS) {
+        fprintf(stderr, "ibv_poll_cq wc.status=%d %s!\n",
+                        wc.status, ibv_wc_status_str(wc.status));
+        fprintf(stderr, "ibv_poll_cq wrid=%s!\n", wrid_desc[wr_id]);
+
+        return -1;
+    }
+
+    if (rdma->control_ready_expected &&
+        (wr_id >= RDMA_WRID_RECV_CONTROL)) {
+        trace_qemu_rdma_poll_recv(wrid_desc[RDMA_WRID_RECV_CONTROL],
+                  wr_id - RDMA_WRID_RECV_CONTROL, wr_id, rdma->nb_sent);
+        rdma->control_ready_expected = 0;
+    }
+
+    if (wr_id == RDMA_WRID_RDMA_WRITE) {
+        uint64_t chunk =
+            (wc.wr_id & RDMA_WRID_CHUNK_MASK) >> RDMA_WRID_CHUNK_SHIFT;
+        uint64_t index =
+            (wc.wr_id & RDMA_WRID_BLOCK_MASK) >> RDMA_WRID_BLOCK_SHIFT;
+        RDMALocalBlock *block = &(rdma->local_ram_blocks.block[index]);
+
+        trace_qemu_rdma_poll_write(print_wrid(wr_id), wr_id, rdma->nb_sent,
+                                   index, chunk, block->local_host_addr,
+                                   (void *)(uintptr_t)block->remote_host_addr);
+
+        clear_bit(chunk, block->transit_bitmap);
+
+        if (rdma->nb_sent > 0) {
+            rdma->nb_sent--;
+        }
+
+        if (!rdma->pin_all) {
+            /*
+             * FYI: If one wanted to signal a specific chunk to be unregistered
+             * using LRU or workload-specific information, this is the function
+             * you would call to do so. That chunk would then get asynchronously
+             * unregistered later.
+             */
+#ifdef RDMA_UNREGISTRATION_EXAMPLE
+            qemu_rdma_signal_unregister(rdma, index, chunk, wc.wr_id);
+#endif
+        }
+    } else {
+        trace_qemu_rdma_poll_other(print_wrid(wr_id), wr_id, rdma->nb_sent);
+    }
+
+    *wr_id_out = wc.wr_id;
+    if (byte_len) {
+        *byte_len = wc.byte_len;
+    }
+
+    return  0;
+}
+
+/*
+ * Block until the next work request has completed.
+ *
+ * First poll to see if a work request has already completed,
+ * otherwise block.
+ *
+ * If we encounter completed work requests for IDs other than
+ * the one we're interested in, then that's generally an error.
+ *
+ * The only exception is actual RDMA Write completions. These
+ * completions only need to be recorded, but do not actually
+ * need further processing.
+ */
+static int qemu_rdma_block_for_wrid(RDMAContext *rdma, int wrid_requested,
+                                    uint32_t *byte_len)
+{
+    int num_cq_events = 0, ret = 0;
+    struct ibv_cq *cq;
+    void *cq_ctx;
+    uint64_t wr_id = RDMA_WRID_NONE, wr_id_in;
+
+    if (ibv_req_notify_cq(rdma->cq, 0)) {
+        return -1;
+    }
+    /* poll cq first */
+    while (wr_id != wrid_requested) {
+        ret = qemu_rdma_poll(rdma, &wr_id_in, byte_len);
+        if (ret < 0) {
+            return ret;
+        }
+
+        wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
+
+        if (wr_id == RDMA_WRID_NONE) {
+            break;
+        }
+        if (wr_id != wrid_requested) {
+            trace_qemu_rdma_block_for_wrid_miss(print_wrid(wrid_requested),
+                       wrid_requested, print_wrid(wr_id), wr_id);
+        }
+    }
+
+    if (wr_id == wrid_requested) {
+        return 0;
+    }
+
+    while (1) {
+        /*
+         * Coroutine doesn't start until migration_fd_process_incoming()
+         * so don't yield unless we know we're running inside of a coroutine.
+         */
+        if (rdma->migration_started_on_destination) {
+            yield_until_fd_readable(rdma->comp_channel->fd);
+        }
+
+        if (ibv_get_cq_event(rdma->comp_channel, &cq, &cq_ctx)) {
+            perror("ibv_get_cq_event");
+            goto err_block_for_wrid;
+        }
+
+        num_cq_events++;
+
+        if (ibv_req_notify_cq(cq, 0)) {
+            goto err_block_for_wrid;
+        }
+
+        while (wr_id != wrid_requested) {
+            ret = qemu_rdma_poll(rdma, &wr_id_in, byte_len);
+            if (ret < 0) {
+                goto err_block_for_wrid;
+            }
+
+            wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
+
+            if (wr_id == RDMA_WRID_NONE) {
+                break;
+            }
+            if (wr_id != wrid_requested) {
+                trace_qemu_rdma_block_for_wrid_miss(print_wrid(wrid_requested),
+                                   wrid_requested, print_wrid(wr_id), wr_id);
+            }
+        }
+
+        if (wr_id == wrid_requested) {
+            goto success_block_for_wrid;
+        }
+    }
+
+success_block_for_wrid:
+    if (num_cq_events) {
+        ibv_ack_cq_events(cq, num_cq_events);
+    }
+    return 0;
+
+err_block_for_wrid:
+    if (num_cq_events) {
+        ibv_ack_cq_events(cq, num_cq_events);
+    }
+    return ret;
+}
+
+/*
+ * Post a SEND message work request for the control channel
+ * containing some data and block until the post completes.
+ */
+static int qemu_rdma_post_send_control(RDMAContext *rdma, uint8_t *buf,
+                                       RDMAControlHeader *head)
+{
+    int ret = 0;
+    RDMAWorkRequestData *wr = &rdma->wr_data[RDMA_WRID_CONTROL];
+    struct ibv_send_wr *bad_wr;
+    struct ibv_sge sge = {
+                           .addr = (uintptr_t)(wr->control),
+                           .length = head->len + sizeof(RDMAControlHeader),
+                           .lkey = wr->control_mr->lkey,
+                         };
+    struct ibv_send_wr send_wr = {
+                                   .wr_id = RDMA_WRID_SEND_CONTROL,
+                                   .opcode = IBV_WR_SEND,
+                                   .send_flags = IBV_SEND_SIGNALED,
+                                   .sg_list = &sge,
+                                   .num_sge = 1,
+                                };
+
+    trace_qemu_rdma_post_send_control(control_desc[head->type]);
+
+    /*
+     * We don't actually need to do a memcpy() in here if we used
+     * the "sge" properly, but since we're only sending control messages
+     * (not RAM in a performance-critical path), then its OK for now.
+     *
+     * The copy makes the RDMAControlHeader simpler to manipulate
+     * for the time being.
+     */
+    assert(head->len <= RDMA_CONTROL_MAX_BUFFER - sizeof(*head));
+    memcpy(wr->control, head, sizeof(RDMAControlHeader));
+    control_to_network((void *) wr->control);
+
+    if (buf) {
+        memcpy(wr->control + sizeof(RDMAControlHeader), buf, head->len);
+    }
+
+
+    ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr);
+
+    if (ret > 0) {
+        error_report("Failed to use post IB SEND for control");
+        return -ret;
+    }
+
+    ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_SEND_CONTROL, NULL);
+    if (ret < 0) {
+        error_report("rdma migration: send polling control error");
+    }
+
+    return ret;
+}
+
+/*
+ * Post a RECV work request in anticipation of some future receipt
+ * of data on the control channel.
+ */
+static int qemu_rdma_post_recv_control(RDMAContext *rdma, int idx)
+{
+    struct ibv_recv_wr *bad_wr;
+    struct ibv_sge sge = {
+                            .addr = (uintptr_t)(rdma->wr_data[idx].control),
+                            .length = RDMA_CONTROL_MAX_BUFFER,
+                            .lkey = rdma->wr_data[idx].control_mr->lkey,
+                         };
+
+    struct ibv_recv_wr recv_wr = {
+                                    .wr_id = RDMA_WRID_RECV_CONTROL + idx,
+                                    .sg_list = &sge,
+                                    .num_sge = 1,
+                                 };
+
+
+    if (ibv_post_recv(rdma->qp, &recv_wr, &bad_wr)) {
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * Block and wait for a RECV control channel message to arrive.
+ */
+static int qemu_rdma_exchange_get_response(RDMAContext *rdma,
+                RDMAControlHeader *head, int expecting, int idx)
+{
+    uint32_t byte_len;
+    int ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RECV_CONTROL + idx,
+                                       &byte_len);
+
+    if (ret < 0) {
+        error_report("rdma migration: recv polling control error!");
+        return ret;
+    }
+
+    network_to_control((void *) rdma->wr_data[idx].control);
+    memcpy(head, rdma->wr_data[idx].control, sizeof(RDMAControlHeader));
+
+    trace_qemu_rdma_exchange_get_response_start(control_desc[expecting]);
+
+    if (expecting == RDMA_CONTROL_NONE) {
+        trace_qemu_rdma_exchange_get_response_none(control_desc[head->type],
+                                             head->type);
+    } else if (head->type != expecting || head->type == RDMA_CONTROL_ERROR) {
+        error_report("Was expecting a %s (%d) control message"
+                ", but got: %s (%d), length: %d",
+                control_desc[expecting], expecting,
+                control_desc[head->type], head->type, head->len);
+        return -EIO;
+    }
+    if (head->len > RDMA_CONTROL_MAX_BUFFER - sizeof(*head)) {
+        error_report("too long length: %d", head->len);
+        return -EINVAL;
+    }
+    if (sizeof(*head) + head->len != byte_len) {
+        error_report("Malformed length: %d byte_len %d", head->len, byte_len);
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+/*
+ * When a RECV work request has completed, the work request's
+ * buffer is pointed at the header.
+ *
+ * This will advance the pointer to the data portion
+ * of the control message of the work request's buffer that
+ * was populated after the work request finished.
+ */
+static void qemu_rdma_move_header(RDMAContext *rdma, int idx,
+                                  RDMAControlHeader *head)
+{
+    rdma->wr_data[idx].control_len = head->len;
+    rdma->wr_data[idx].control_curr =
+        rdma->wr_data[idx].control + sizeof(RDMAControlHeader);
+}
+
+/*
+ * This is an 'atomic' high-level operation to deliver a single, unified
+ * control-channel message.
+ *
+ * Additionally, if the user is expecting some kind of reply to this message,
+ * they can request a 'resp' response message be filled in by posting an
+ * additional work request on behalf of the user and waiting for an additional
+ * completion.
+ *
+ * The extra (optional) response is used during registration to us from having
+ * to perform an *additional* exchange of message just to provide a response by
+ * instead piggy-backing on the acknowledgement.
+ */
+static int qemu_rdma_exchange_send(RDMAContext *rdma, RDMAControlHeader *head,
+                                   uint8_t *data, RDMAControlHeader *resp,
+                                   int *resp_idx,
+                                   int (*callback)(RDMAContext *rdma))
+{
+    int ret = 0;
+
+    /*
+     * Wait until the dest is ready before attempting to deliver the message
+     * by waiting for a READY message.
+     */
+    if (rdma->control_ready_expected) {
+        RDMAControlHeader resp;
+        ret = qemu_rdma_exchange_get_response(rdma,
+                                    &resp, RDMA_CONTROL_READY, RDMA_WRID_READY);
+        if (ret < 0) {
+            return ret;
+        }
+    }
+
+    /*
+     * If the user is expecting a response, post a WR in anticipation of it.
+     */
+    if (resp) {
+        ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_DATA);
+        if (ret) {
+            error_report("rdma migration: error posting"
+                    " extra control recv for anticipated result!");
+            return ret;
+        }
+    }
+
+    /*
+     * Post a WR to replace the one we just consumed for the READY message.
+     */
+    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+    if (ret) {
+        error_report("rdma migration: error posting first control recv!");
+        return ret;
+    }
+
+    /*
+     * Deliver the control message that was requested.
+     */
+    ret = qemu_rdma_post_send_control(rdma, data, head);
+
+    if (ret < 0) {
+        error_report("Failed to send control buffer!");
+        return ret;
+    }
+
+    /*
+     * If we're expecting a response, block and wait for it.
+     */
+    if (resp) {
+        if (callback) {
+            trace_qemu_rdma_exchange_send_issue_callback();
+            ret = callback(rdma);
+            if (ret < 0) {
+                return ret;
+            }
+        }
+
+        trace_qemu_rdma_exchange_send_waiting(control_desc[resp->type]);
+        ret = qemu_rdma_exchange_get_response(rdma, resp,
+                                              resp->type, RDMA_WRID_DATA);
+
+        if (ret < 0) {
+            return ret;
+        }
+
+        qemu_rdma_move_header(rdma, RDMA_WRID_DATA, resp);
+        if (resp_idx) {
+            *resp_idx = RDMA_WRID_DATA;
+        }
+        trace_qemu_rdma_exchange_send_received(control_desc[resp->type]);
+    }
+
+    rdma->control_ready_expected = 1;
+
+    return 0;
+}
+
+/*
+ * This is an 'atomic' high-level operation to receive a single, unified
+ * control-channel message.
+ */
+static int qemu_rdma_exchange_recv(RDMAContext *rdma, RDMAControlHeader *head,
+                                int expecting)
+{
+    RDMAControlHeader ready = {
+                                .len = 0,
+                                .type = RDMA_CONTROL_READY,
+                                .repeat = 1,
+                              };
+    int ret;
+
+    /*
+     * Inform the source that we're ready to receive a message.
+     */
+    ret = qemu_rdma_post_send_control(rdma, NULL, &ready);
+
+    if (ret < 0) {
+        error_report("Failed to send control buffer!");
+        return ret;
+    }
+
+    /*
+     * Block and wait for the message.
+     */
+    ret = qemu_rdma_exchange_get_response(rdma, head,
+                                          expecting, RDMA_WRID_READY);
+
+    if (ret < 0) {
+        return ret;
+    }
+
+    qemu_rdma_move_header(rdma, RDMA_WRID_READY, head);
+
+    /*
+     * Post a new RECV work request to replace the one we just consumed.
+     */
+    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+    if (ret) {
+        error_report("rdma migration: error posting second control recv!");
+        return ret;
+    }
+
+    return 0;
+}
+
+/*
+ * Write an actual chunk of memory using RDMA.
+ *
+ * If we're using dynamic registration on the dest-side, we have to
+ * send a registration command first.
+ */
+static int qemu_rdma_write_one(QEMUFile *f, RDMAContext *rdma,
+                               int current_index, uint64_t current_addr,
+                               uint64_t length)
+{
+    struct ibv_sge sge;
+    struct ibv_send_wr send_wr = { 0 };
+    struct ibv_send_wr *bad_wr;
+    int reg_result_idx, ret, count = 0;
+    uint64_t chunk, chunks;
+    uint8_t *chunk_start, *chunk_end;
+    RDMALocalBlock *block = &(rdma->local_ram_blocks.block[current_index]);
+    RDMARegister reg;
+    RDMARegisterResult *reg_result;
+    RDMAControlHeader resp = { .type = RDMA_CONTROL_REGISTER_RESULT };
+    RDMAControlHeader head = { .len = sizeof(RDMARegister),
+                               .type = RDMA_CONTROL_REGISTER_REQUEST,
+                               .repeat = 1,
+                             };
+
+retry:
+    sge.addr = (uintptr_t)(block->local_host_addr +
+                            (current_addr - block->offset));
+    sge.length = length;
+
+    chunk = ram_chunk_index(block->local_host_addr,
+                            (uint8_t *)(uintptr_t)sge.addr);
+    chunk_start = ram_chunk_start(block, chunk);
+
+    if (block->is_ram_block) {
+        chunks = length / (1UL << RDMA_REG_CHUNK_SHIFT);
+
+        if (chunks && ((length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) {
+            chunks--;
+        }
+    } else {
+        chunks = block->length / (1UL << RDMA_REG_CHUNK_SHIFT);
+
+        if (chunks && ((block->length % (1UL << RDMA_REG_CHUNK_SHIFT)) == 0)) {
+            chunks--;
+        }
+    }
+
+    trace_qemu_rdma_write_one_top(chunks + 1,
+                                  (chunks + 1) *
+                                  (1UL << RDMA_REG_CHUNK_SHIFT) / 1024 / 1024);
+
+    chunk_end = ram_chunk_end(block, chunk + chunks);
+
+    if (!rdma->pin_all) {
+#ifdef RDMA_UNREGISTRATION_EXAMPLE
+        qemu_rdma_unregister_waiting(rdma);
+#endif
+    }
+
+    while (test_bit(chunk, block->transit_bitmap)) {
+        (void)count;
+        trace_qemu_rdma_write_one_block(count++, current_index, chunk,
+                sge.addr, length, rdma->nb_sent, block->nb_chunks);
+
+        ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
+
+        if (ret < 0) {
+            error_report("Failed to Wait for previous write to complete "
+                    "block %d chunk %" PRIu64
+                    " current %" PRIu64 " len %" PRIu64 " %d",
+                    current_index, chunk, sge.addr, length, rdma->nb_sent);
+            return ret;
+        }
+    }
+
+    if (!rdma->pin_all || !block->is_ram_block) {
+        if (!block->remote_keys[chunk]) {
+            /*
+             * This chunk has not yet been registered, so first check to see
+             * if the entire chunk is zero. If so, tell the other size to
+             * memset() + madvise() the entire chunk without RDMA.
+             */
+
+            if (can_use_buffer_find_nonzero_offset((void *)(uintptr_t)sge.addr,
+                                                   length)
+                   && buffer_find_nonzero_offset((void *)(uintptr_t)sge.addr,
+                                                    length) == length) {
+                RDMACompress comp = {
+                                        .offset = current_addr,
+                                        .value = 0,
+                                        .block_idx = current_index,
+                                        .length = length,
+                                    };
+
+                head.len = sizeof(comp);
+                head.type = RDMA_CONTROL_COMPRESS;
+
+                trace_qemu_rdma_write_one_zero(chunk, sge.length,
+                                               current_index, current_addr);
+
+                compress_to_network(rdma, &comp);
+                ret = qemu_rdma_exchange_send(rdma, &head,
+                                (uint8_t *) &comp, NULL, NULL, NULL);
+
+                if (ret < 0) {
+                    return -EIO;
+                }
+
+                acct_update_position(f, sge.length, true);
+
+                return 1;
+            }
+
+            /*
+             * Otherwise, tell other side to register.
+             */
+            reg.current_index = current_index;
+            if (block->is_ram_block) {
+                reg.key.current_addr = current_addr;
+            } else {
+                reg.key.chunk = chunk;
+            }
+            reg.chunks = chunks;
+
+            trace_qemu_rdma_write_one_sendreg(chunk, sge.length, current_index,
+                                              current_addr);
+
+            register_to_network(rdma, &reg);
+            ret = qemu_rdma_exchange_send(rdma, &head, (uint8_t *) &reg,
+                                    &resp, &reg_result_idx, NULL);
+            if (ret < 0) {
+                return ret;
+            }
+
+            /* try to overlap this single registration with the one we sent. */
+            if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr,
+                                                &sge.lkey, NULL, chunk,
+                                                chunk_start, chunk_end)) {
+                error_report("cannot get lkey");
+                return -EINVAL;
+            }
+
+            reg_result = (RDMARegisterResult *)
+                    rdma->wr_data[reg_result_idx].control_curr;
+
+            network_to_result(reg_result);
+
+            trace_qemu_rdma_write_one_recvregres(block->remote_keys[chunk],
+                                                 reg_result->rkey, chunk);
+
+            block->remote_keys[chunk] = reg_result->rkey;
+            block->remote_host_addr = reg_result->host_addr;
+        } else {
+            /* already registered before */
+            if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr,
+                                                &sge.lkey, NULL, chunk,
+                                                chunk_start, chunk_end)) {
+                error_report("cannot get lkey!");
+                return -EINVAL;
+            }
+        }
+
+        send_wr.wr.rdma.rkey = block->remote_keys[chunk];
+    } else {
+        send_wr.wr.rdma.rkey = block->remote_rkey;
+
+        if (qemu_rdma_register_and_get_keys(rdma, block, sge.addr,
+                                                     &sge.lkey, NULL, chunk,
+                                                     chunk_start, chunk_end)) {
+            error_report("cannot get lkey!");
+            return -EINVAL;
+        }
+    }
+
+    /*
+     * Encode the ram block index and chunk within this wrid.
+     * We will use this information at the time of completion
+     * to figure out which bitmap to check against and then which
+     * chunk in the bitmap to look for.
+     */
+    send_wr.wr_id = qemu_rdma_make_wrid(RDMA_WRID_RDMA_WRITE,
+                                        current_index, chunk);
+
+    send_wr.opcode = IBV_WR_RDMA_WRITE;
+    send_wr.send_flags = IBV_SEND_SIGNALED;
+    send_wr.sg_list = &sge;
+    send_wr.num_sge = 1;
+    send_wr.wr.rdma.remote_addr = block->remote_host_addr +
+                                (current_addr - block->offset);
+
+    trace_qemu_rdma_write_one_post(chunk, sge.addr, send_wr.wr.rdma.remote_addr,
+                                   sge.length);
+
+    /*
+     * ibv_post_send() does not return negative error numbers,
+     * per the specification they are positive - no idea why.
+     */
+    ret = ibv_post_send(rdma->qp, &send_wr, &bad_wr);
+
+    if (ret == ENOMEM) {
+        trace_qemu_rdma_write_one_queue_full();
+        ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
+        if (ret < 0) {
+            error_report("rdma migration: failed to make "
+                         "room in full send queue! %d", ret);
+            return ret;
+        }
+
+        goto retry;
+
+    } else if (ret > 0) {
+        perror("rdma migration: post rdma write failed");
+        return -ret;
+    }
+
+    set_bit(chunk, block->transit_bitmap);
+    acct_update_position(f, sge.length, false);
+    rdma->total_writes++;
+
+    return 0;
+}
+
+/*
+ * Push out any unwritten RDMA operations.
+ *
+ * We support sending out multiple chunks at the same time.
+ * Not all of them need to get signaled in the completion queue.
+ */
+static int qemu_rdma_write_flush(QEMUFile *f, RDMAContext *rdma)
+{
+    int ret;
+
+    if (!rdma->current_length) {
+        return 0;
+    }
+
+    ret = qemu_rdma_write_one(f, rdma,
+            rdma->current_index, rdma->current_addr, rdma->current_length);
+
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret == 0) {
+        rdma->nb_sent++;
+        trace_qemu_rdma_write_flush(rdma->nb_sent);
+    }
+
+    rdma->current_length = 0;
+    rdma->current_addr = 0;
+
+    return 0;
+}
+
+static inline int qemu_rdma_buffer_mergable(RDMAContext *rdma,
+                    uint64_t offset, uint64_t len)
+{
+    RDMALocalBlock *block;
+    uint8_t *host_addr;
+    uint8_t *chunk_end;
+
+    if (rdma->current_index < 0) {
+        return 0;
+    }
+
+    if (rdma->current_chunk < 0) {
+        return 0;
+    }
+
+    block = &(rdma->local_ram_blocks.block[rdma->current_index]);
+    host_addr = block->local_host_addr + (offset - block->offset);
+    chunk_end = ram_chunk_end(block, rdma->current_chunk);
+
+    if (rdma->current_length == 0) {
+        return 0;
+    }
+
+    /*
+     * Only merge into chunk sequentially.
+     */
+    if (offset != (rdma->current_addr + rdma->current_length)) {
+        return 0;
+    }
+
+    if (offset < block->offset) {
+        return 0;
+    }
+
+    if ((offset + len) > (block->offset + block->length)) {
+        return 0;
+    }
+
+    if ((host_addr + len) > chunk_end) {
+        return 0;
+    }
+
+    return 1;
+}
+
+/*
+ * We're not actually writing here, but doing three things:
+ *
+ * 1. Identify the chunk the buffer belongs to.
+ * 2. If the chunk is full or the buffer doesn't belong to the current
+ *    chunk, then start a new chunk and flush() the old chunk.
+ * 3. To keep the hardware busy, we also group chunks into batches
+ *    and only require that a batch gets acknowledged in the completion
+ *    qeueue instead of each individual chunk.
+ */
+static int qemu_rdma_write(QEMUFile *f, RDMAContext *rdma,
+                           uint64_t block_offset, uint64_t offset,
+                           uint64_t len)
+{
+    uint64_t current_addr = block_offset + offset;
+    uint64_t index = rdma->current_index;
+    uint64_t chunk = rdma->current_chunk;
+    int ret;
+
+    /* If we cannot merge it, we flush the current buffer first. */
+    if (!qemu_rdma_buffer_mergable(rdma, current_addr, len)) {
+        ret = qemu_rdma_write_flush(f, rdma);
+        if (ret) {
+            return ret;
+        }
+        rdma->current_length = 0;
+        rdma->current_addr = current_addr;
+
+        ret = qemu_rdma_search_ram_block(rdma, block_offset,
+                                         offset, len, &index, &chunk);
+        if (ret) {
+            error_report("ram block search failed");
+            return ret;
+        }
+        rdma->current_index = index;
+        rdma->current_chunk = chunk;
+    }
+
+    /* merge it */
+    rdma->current_length += len;
+
+    /* flush it if buffer is too large */
+    if (rdma->current_length >= RDMA_MERGE_MAX) {
+        return qemu_rdma_write_flush(f, rdma);
+    }
+
+    return 0;
+}
+
+static void qemu_rdma_cleanup(RDMAContext *rdma)
+{
+    struct rdma_cm_event *cm_event;
+    int ret, idx;
+
+    if (rdma->cm_id && rdma->connected) {
+        if (rdma->error_state) {
+            RDMAControlHeader head = { .len = 0,
+                                       .type = RDMA_CONTROL_ERROR,
+                                       .repeat = 1,
+                                     };
+            error_report("Early error. Sending error.");
+            qemu_rdma_post_send_control(rdma, NULL, &head);
+        }
+
+        ret = rdma_disconnect(rdma->cm_id);
+        if (!ret) {
+            trace_qemu_rdma_cleanup_waiting_for_disconnect();
+            ret = rdma_get_cm_event(rdma->channel, &cm_event);
+            if (!ret) {
+                rdma_ack_cm_event(cm_event);
+            }
+        }
+        trace_qemu_rdma_cleanup_disconnect();
+        rdma->connected = false;
+    }
+
+    g_free(rdma->dest_blocks);
+    rdma->dest_blocks = NULL;
+
+    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
+        if (rdma->wr_data[idx].control_mr) {
+            rdma->total_registrations--;
+            ibv_dereg_mr(rdma->wr_data[idx].control_mr);
+        }
+        rdma->wr_data[idx].control_mr = NULL;
+    }
+
+    if (rdma->local_ram_blocks.block) {
+        while (rdma->local_ram_blocks.nb_blocks) {
+            rdma_delete_block(rdma, &rdma->local_ram_blocks.block[0]);
+        }
+    }
+
+    if (rdma->qp) {
+        rdma_destroy_qp(rdma->cm_id);
+        rdma->qp = NULL;
+    }
+    if (rdma->cq) {
+        ibv_destroy_cq(rdma->cq);
+        rdma->cq = NULL;
+    }
+    if (rdma->comp_channel) {
+        ibv_destroy_comp_channel(rdma->comp_channel);
+        rdma->comp_channel = NULL;
+    }
+    if (rdma->pd) {
+        ibv_dealloc_pd(rdma->pd);
+        rdma->pd = NULL;
+    }
+    if (rdma->cm_id) {
+        rdma_destroy_id(rdma->cm_id);
+        rdma->cm_id = NULL;
+    }
+    if (rdma->listen_id) {
+        rdma_destroy_id(rdma->listen_id);
+        rdma->listen_id = NULL;
+    }
+    if (rdma->channel) {
+        rdma_destroy_event_channel(rdma->channel);
+        rdma->channel = NULL;
+    }
+    g_free(rdma->host);
+    rdma->host = NULL;
+}
+
+
+static int qemu_rdma_source_init(RDMAContext *rdma, Error **errp, bool pin_all)
+{
+    int ret, idx;
+    Error *local_err = NULL, **temp = &local_err;
+
+    /*
+     * Will be validated against destination's actual capabilities
+     * after the connect() completes.
+     */
+    rdma->pin_all = pin_all;
+
+    ret = qemu_rdma_resolve_host(rdma, temp);
+    if (ret) {
+        goto err_rdma_source_init;
+    }
+
+    ret = qemu_rdma_alloc_pd_cq(rdma);
+    if (ret) {
+        ERROR(temp, "rdma migration: error allocating pd and cq! Your mlock()"
+                    " limits may be too low. Please check $ ulimit -a # and "
+                    "search for 'ulimit -l' in the output");
+        goto err_rdma_source_init;
+    }
+
+    ret = qemu_rdma_alloc_qp(rdma);
+    if (ret) {
+        ERROR(temp, "rdma migration: error allocating qp!");
+        goto err_rdma_source_init;
+    }
+
+    ret = qemu_rdma_init_ram_blocks(rdma);
+    if (ret) {
+        ERROR(temp, "rdma migration: error initializing ram blocks!");
+        goto err_rdma_source_init;
+    }
+
+    /* Build the hash that maps from offset to RAMBlock */
+    rdma->blockmap = g_hash_table_new(g_direct_hash, g_direct_equal);
+    for (idx = 0; idx < rdma->local_ram_blocks.nb_blocks; idx++) {
+        g_hash_table_insert(rdma->blockmap,
+                (void *)(uintptr_t)rdma->local_ram_blocks.block[idx].offset,
+                &rdma->local_ram_blocks.block[idx]);
+    }
+
+    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
+        ret = qemu_rdma_reg_control(rdma, idx);
+        if (ret) {
+            ERROR(temp, "rdma migration: error registering %d control!",
+                                                            idx);
+            goto err_rdma_source_init;
+        }
+    }
+
+    return 0;
+
+err_rdma_source_init:
+    error_propagate(errp, local_err);
+    qemu_rdma_cleanup(rdma);
+    return -1;
+}
+
+static int qemu_rdma_connect(RDMAContext *rdma, Error **errp)
+{
+    RDMACapabilities cap = {
+                                .version = RDMA_CONTROL_VERSION_CURRENT,
+                                .flags = 0,
+                           };
+    struct rdma_conn_param conn_param = { .initiator_depth = 2,
+                                          .retry_count = 5,
+                                          .private_data = &cap,
+                                          .private_data_len = sizeof(cap),
+                                        };
+    struct rdma_cm_event *cm_event;
+    int ret;
+
+    /*
+     * Only negotiate the capability with destination if the user
+     * on the source first requested the capability.
+     */
+    if (rdma->pin_all) {
+        trace_qemu_rdma_connect_pin_all_requested();
+        cap.flags |= RDMA_CAPABILITY_PIN_ALL;
+    }
+
+    caps_to_network(&cap);
+
+    ret = rdma_connect(rdma->cm_id, &conn_param);
+    if (ret) {
+        perror("rdma_connect");
+        ERROR(errp, "connecting to destination!");
+        goto err_rdma_source_connect;
+    }
+
+    ret = rdma_get_cm_event(rdma->channel, &cm_event);
+    if (ret) {
+        perror("rdma_get_cm_event after rdma_connect");
+        ERROR(errp, "connecting to destination!");
+        rdma_ack_cm_event(cm_event);
+        goto err_rdma_source_connect;
+    }
+
+    if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
+        perror("rdma_get_cm_event != EVENT_ESTABLISHED after rdma_connect");
+        ERROR(errp, "connecting to destination!");
+        rdma_ack_cm_event(cm_event);
+        goto err_rdma_source_connect;
+    }
+    rdma->connected = true;
+
+    memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap));
+    network_to_caps(&cap);
+
+    /*
+     * Verify that the *requested* capabilities are supported by the destination
+     * and disable them otherwise.
+     */
+    if (rdma->pin_all && !(cap.flags & RDMA_CAPABILITY_PIN_ALL)) {
+        ERROR(errp, "Server cannot support pinning all memory. "
+                        "Will register memory dynamically.");
+        rdma->pin_all = false;
+    }
+
+    trace_qemu_rdma_connect_pin_all_outcome(rdma->pin_all);
+
+    rdma_ack_cm_event(cm_event);
+
+    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+    if (ret) {
+        ERROR(errp, "posting second control recv!");
+        goto err_rdma_source_connect;
+    }
+
+    rdma->control_ready_expected = 1;
+    rdma->nb_sent = 0;
+    return 0;
+
+err_rdma_source_connect:
+    qemu_rdma_cleanup(rdma);
+    return -1;
+}
+
+static int qemu_rdma_dest_init(RDMAContext *rdma, Error **errp)
+{
+    int ret, idx;
+    struct rdma_cm_id *listen_id;
+    char ip[40] = "unknown";
+    struct rdma_addrinfo *res, *e;
+    char port_str[16];
+
+    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
+        rdma->wr_data[idx].control_len = 0;
+        rdma->wr_data[idx].control_curr = NULL;
+    }
+
+    if (!rdma->host || !rdma->host[0]) {
+        ERROR(errp, "RDMA host is not set!");
+        rdma->error_state = -EINVAL;
+        return -1;
+    }
+    /* create CM channel */
+    rdma->channel = rdma_create_event_channel();
+    if (!rdma->channel) {
+        ERROR(errp, "could not create rdma event channel");
+        rdma->error_state = -EINVAL;
+        return -1;
+    }
+
+    /* create CM id */
+    ret = rdma_create_id(rdma->channel, &listen_id, NULL, RDMA_PS_TCP);
+    if (ret) {
+        ERROR(errp, "could not create cm_id!");
+        goto err_dest_init_create_listen_id;
+    }
+
+    snprintf(port_str, 16, "%d", rdma->port);
+    port_str[15] = '\0';
+
+    ret = rdma_getaddrinfo(rdma->host, port_str, NULL, &res);
+    if (ret < 0) {
+        ERROR(errp, "could not rdma_getaddrinfo address %s", rdma->host);
+        goto err_dest_init_bind_addr;
+    }
+
+    for (e = res; e != NULL; e = e->ai_next) {
+        inet_ntop(e->ai_family,
+            &((struct sockaddr_in *) e->ai_dst_addr)->sin_addr, ip, sizeof ip);
+        trace_qemu_rdma_dest_init_trying(rdma->host, ip);
+        ret = rdma_bind_addr(listen_id, e->ai_dst_addr);
+        if (ret) {
+            continue;
+        }
+        if (e->ai_family == AF_INET6) {
+            ret = qemu_rdma_broken_ipv6_kernel(errp, listen_id->verbs);
+            if (ret) {
+                continue;
+            }
+        }
+        break;
+    }
+
+    if (!e) {
+        ERROR(errp, "Error: could not rdma_bind_addr!");
+        goto err_dest_init_bind_addr;
+    }
+
+    rdma->listen_id = listen_id;
+    qemu_rdma_dump_gid("dest_init", listen_id);
+    return 0;
+
+err_dest_init_bind_addr:
+    rdma_destroy_id(listen_id);
+err_dest_init_create_listen_id:
+    rdma_destroy_event_channel(rdma->channel);
+    rdma->channel = NULL;
+    rdma->error_state = ret;
+    return ret;
+
+}
+
+static void *qemu_rdma_data_init(const char *host_port, Error **errp)
+{
+    RDMAContext *rdma = NULL;
+    InetSocketAddress *addr;
+
+    if (host_port) {
+        rdma = g_new0(RDMAContext, 1);
+        rdma->current_index = -1;
+        rdma->current_chunk = -1;
+
+        addr = inet_parse(host_port, NULL);
+        if (addr != NULL) {
+            rdma->port = atoi(addr->port);
+            rdma->host = g_strdup(addr->host);
+        } else {
+            ERROR(errp, "bad RDMA migration address '%s'", host_port);
+            g_free(rdma);
+            rdma = NULL;
+        }
+
+        qapi_free_InetSocketAddress(addr);
+    }
+
+    return rdma;
+}
+
+/*
+ * QEMUFile interface to the control channel.
+ * SEND messages for control only.
+ * VM's ram is handled with regular RDMA messages.
+ */
+static ssize_t qio_channel_rdma_writev(QIOChannel *ioc,
+                                       const struct iovec *iov,
+                                       size_t niov,
+                                       int *fds,
+                                       size_t nfds,
+                                       Error **errp)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
+    QEMUFile *f = rioc->file;
+    RDMAContext *rdma = rioc->rdma;
+    int ret;
+    ssize_t done = 0;
+    size_t i;
+
+    CHECK_ERROR_STATE();
+
+    /*
+     * Push out any writes that
+     * we're queued up for VM's ram.
+     */
+    ret = qemu_rdma_write_flush(f, rdma);
+    if (ret < 0) {
+        rdma->error_state = ret;
+        return ret;
+    }
+
+    for (i = 0; i < niov; i++) {
+        size_t remaining = iov[i].iov_len;
+        uint8_t * data = (void *)iov[i].iov_base;
+        while (remaining) {
+            RDMAControlHeader head;
+
+            rioc->len = MIN(remaining, RDMA_SEND_INCREMENT);
+            remaining -= rioc->len;
+
+            head.len = rioc->len;
+            head.type = RDMA_CONTROL_QEMU_FILE;
+
+            ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL);
+
+            if (ret < 0) {
+                rdma->error_state = ret;
+                return ret;
+            }
+
+            data += rioc->len;
+            done += rioc->len;
+        }
+    }
+
+    return done;
+}
+
+static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf,
+                             size_t size, int idx)
+{
+    size_t len = 0;
+
+    if (rdma->wr_data[idx].control_len) {
+        trace_qemu_rdma_fill(rdma->wr_data[idx].control_len, size);
+
+        len = MIN(size, rdma->wr_data[idx].control_len);
+        memcpy(buf, rdma->wr_data[idx].control_curr, len);
+        rdma->wr_data[idx].control_curr += len;
+        rdma->wr_data[idx].control_len -= len;
+    }
+
+    return len;
+}
+
+/*
+ * QEMUFile interface to the control channel.
+ * RDMA links don't use bytestreams, so we have to
+ * return bytes to QEMUFile opportunistically.
+ */
+static ssize_t qio_channel_rdma_readv(QIOChannel *ioc,
+                                      const struct iovec *iov,
+                                      size_t niov,
+                                      int **fds,
+                                      size_t *nfds,
+                                      Error **errp)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
+    RDMAContext *rdma = rioc->rdma;
+    RDMAControlHeader head;
+    int ret = 0;
+    ssize_t i;
+    size_t done = 0;
+
+    CHECK_ERROR_STATE();
+
+    for (i = 0; i < niov; i++) {
+        size_t want = iov[i].iov_len;
+        uint8_t *data = (void *)iov[i].iov_base;
+
+        /*
+         * First, we hold on to the last SEND message we
+         * were given and dish out the bytes until we run
+         * out of bytes.
+         */
+        ret = qemu_rdma_fill(rioc->rdma, data, want, 0);
+        done += ret;
+        want -= ret;
+        /* Got what we needed, so go to next iovec */
+        if (want == 0) {
+            continue;
+        }
+
+        /* If we got any data so far, then don't wait
+         * for more, just return what we have */
+        if (done > 0) {
+            break;
+        }
+
+
+        /* We've got nothing at all, so lets wait for
+         * more to arrive
+         */
+        ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_QEMU_FILE);
+
+        if (ret < 0) {
+            rdma->error_state = ret;
+            return ret;
+        }
+
+        /*
+         * SEND was received with new bytes, now try again.
+         */
+        ret = qemu_rdma_fill(rioc->rdma, data, want, 0);
+        done += ret;
+        want -= ret;
+
+        /* Still didn't get enough, so lets just return */
+        if (want) {
+            if (done == 0) {
+                return QIO_CHANNEL_ERR_BLOCK;
+            } else {
+                break;
+            }
+        }
+    }
+    rioc->len = done;
+    return rioc->len;
+}
+
+/*
+ * Block until all the outstanding chunks have been delivered by the hardware.
+ */
+static int qemu_rdma_drain_cq(QEMUFile *f, RDMAContext *rdma)
+{
+    int ret;
+
+    if (qemu_rdma_write_flush(f, rdma) < 0) {
+        return -EIO;
+    }
+
+    while (rdma->nb_sent) {
+        ret = qemu_rdma_block_for_wrid(rdma, RDMA_WRID_RDMA_WRITE, NULL);
+        if (ret < 0) {
+            error_report("rdma migration: complete polling error!");
+            return -EIO;
+        }
+    }
+
+    qemu_rdma_unregister_waiting(rdma);
+
+    return 0;
+}
+
+
+static int qio_channel_rdma_set_blocking(QIOChannel *ioc,
+                                         bool blocking,
+                                         Error **errp)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
+    /* XXX we should make readv/writev actually honour this :-) */
+    rioc->blocking = blocking;
+    return 0;
+}
+
+
+typedef struct QIOChannelRDMASource QIOChannelRDMASource;
+struct QIOChannelRDMASource {
+    GSource parent;
+    QIOChannelRDMA *rioc;
+    GIOCondition condition;
+};
+
+static gboolean
+qio_channel_rdma_source_prepare(GSource *source,
+                                gint *timeout)
+{
+    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
+    RDMAContext *rdma = rsource->rioc->rdma;
+    GIOCondition cond = 0;
+    *timeout = -1;
+
+    if (rdma->wr_data[0].control_len) {
+        cond |= G_IO_IN;
+    }
+    cond |= G_IO_OUT;
+
+    return cond & rsource->condition;
+}
+
+static gboolean
+qio_channel_rdma_source_check(GSource *source)
+{
+    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
+    RDMAContext *rdma = rsource->rioc->rdma;
+    GIOCondition cond = 0;
+
+    if (rdma->wr_data[0].control_len) {
+        cond |= G_IO_IN;
+    }
+    cond |= G_IO_OUT;
+
+    return cond & rsource->condition;
+}
+
+static gboolean
+qio_channel_rdma_source_dispatch(GSource *source,
+                                 GSourceFunc callback,
+                                 gpointer user_data)
+{
+    QIOChannelFunc func = (QIOChannelFunc)callback;
+    QIOChannelRDMASource *rsource = (QIOChannelRDMASource *)source;
+    RDMAContext *rdma = rsource->rioc->rdma;
+    GIOCondition cond = 0;
+
+    if (rdma->wr_data[0].control_len) {
+        cond |= G_IO_IN;
+    }
+    cond |= G_IO_OUT;
+
+    return (*func)(QIO_CHANNEL(rsource->rioc),
+                   (cond & rsource->condition),
+                   user_data);
+}
+
+static void
+qio_channel_rdma_source_finalize(GSource *source)
+{
+    QIOChannelRDMASource *ssource = (QIOChannelRDMASource *)source;
+
+    object_unref(OBJECT(ssource->rioc));
+}
+
+GSourceFuncs qio_channel_rdma_source_funcs = {
+    qio_channel_rdma_source_prepare,
+    qio_channel_rdma_source_check,
+    qio_channel_rdma_source_dispatch,
+    qio_channel_rdma_source_finalize
+};
+
+static GSource *qio_channel_rdma_create_watch(QIOChannel *ioc,
+                                              GIOCondition condition)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
+    QIOChannelRDMASource *ssource;
+    GSource *source;
+
+    source = g_source_new(&qio_channel_rdma_source_funcs,
+                          sizeof(QIOChannelRDMASource));
+    ssource = (QIOChannelRDMASource *)source;
+
+    ssource->rioc = rioc;
+    object_ref(OBJECT(rioc));
+
+    ssource->condition = condition;
+
+    return source;
+}
+
+
+static int qio_channel_rdma_close(QIOChannel *ioc,
+                                  Error **errp)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(ioc);
+    trace_qemu_rdma_close();
+    if (rioc->rdma) {
+        qemu_rdma_cleanup(rioc->rdma);
+        g_free(rioc->rdma);
+        rioc->rdma = NULL;
+    }
+    return 0;
+}
+
+/*
+ * Parameters:
+ *    @offset == 0 :
+ *        This means that 'block_offset' is a full virtual address that does not
+ *        belong to a RAMBlock of the virtual machine and instead
+ *        represents a private malloc'd memory area that the caller wishes to
+ *        transfer.
+ *
+ *    @offset != 0 :
+ *        Offset is an offset to be added to block_offset and used
+ *        to also lookup the corresponding RAMBlock.
+ *
+ *    @size > 0 :
+ *        Initiate an transfer this size.
+ *
+ *    @size == 0 :
+ *        A 'hint' or 'advice' that means that we wish to speculatively
+ *        and asynchronously unregister this memory. In this case, there is no
+ *        guarantee that the unregister will actually happen, for example,
+ *        if the memory is being actively transmitted. Additionally, the memory
+ *        may be re-registered at any future time if a write within the same
+ *        chunk was requested again, even if you attempted to unregister it
+ *        here.
+ *
+ *    @size < 0 : TODO, not yet supported
+ *        Unregister the memory NOW. This means that the caller does not
+ *        expect there to be any future RDMA transfers and we just want to clean
+ *        things up. This is used in case the upper layer owns the memory and
+ *        cannot wait for qemu_fclose() to occur.
+ *
+ *    @bytes_sent : User-specificed pointer to indicate how many bytes were
+ *                  sent. Usually, this will not be more than a few bytes of
+ *                  the protocol because most transfers are sent asynchronously.
+ */
+static size_t qemu_rdma_save_page(QEMUFile *f, void *opaque,
+                                  ram_addr_t block_offset, ram_addr_t offset,
+                                  size_t size, uint64_t *bytes_sent)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque);
+    RDMAContext *rdma = rioc->rdma;
+    int ret;
+
+    CHECK_ERROR_STATE();
+
+    qemu_fflush(f);
+
+    if (size > 0) {
+        /*
+         * Add this page to the current 'chunk'. If the chunk
+         * is full, or the page doen't belong to the current chunk,
+         * an actual RDMA write will occur and a new chunk will be formed.
+         */
+        ret = qemu_rdma_write(f, rdma, block_offset, offset, size);
+        if (ret < 0) {
+            error_report("rdma migration: write error! %d", ret);
+            goto err;
+        }
+
+        /*
+         * We always return 1 bytes because the RDMA
+         * protocol is completely asynchronous. We do not yet know
+         * whether an  identified chunk is zero or not because we're
+         * waiting for other pages to potentially be merged with
+         * the current chunk. So, we have to call qemu_update_position()
+         * later on when the actual write occurs.
+         */
+        if (bytes_sent) {
+            *bytes_sent = 1;
+        }
+    } else {
+        uint64_t index, chunk;
+
+        /* TODO: Change QEMUFileOps prototype to be signed: size_t => long
+        if (size < 0) {
+            ret = qemu_rdma_drain_cq(f, rdma);
+            if (ret < 0) {
+                fprintf(stderr, "rdma: failed to synchronously drain"
+                                " completion queue before unregistration.\n");
+                goto err;
+            }
+        }
+        */
+
+        ret = qemu_rdma_search_ram_block(rdma, block_offset,
+                                         offset, size, &index, &chunk);
+
+        if (ret) {
+            error_report("ram block search failed");
+            goto err;
+        }
+
+        qemu_rdma_signal_unregister(rdma, index, chunk, 0);
+
+        /*
+         * TODO: Synchronous, guaranteed unregistration (should not occur during
+         * fast-path). Otherwise, unregisters will process on the next call to
+         * qemu_rdma_drain_cq()
+        if (size < 0) {
+            qemu_rdma_unregister_waiting(rdma);
+        }
+        */
+    }
+
+    /*
+     * Drain the Completion Queue if possible, but do not block,
+     * just poll.
+     *
+     * If nothing to poll, the end of the iteration will do this
+     * again to make sure we don't overflow the request queue.
+     */
+    while (1) {
+        uint64_t wr_id, wr_id_in;
+        int ret = qemu_rdma_poll(rdma, &wr_id_in, NULL);
+        if (ret < 0) {
+            error_report("rdma migration: polling error! %d", ret);
+            goto err;
+        }
+
+        wr_id = wr_id_in & RDMA_WRID_TYPE_MASK;
+
+        if (wr_id == RDMA_WRID_NONE) {
+            break;
+        }
+    }
+
+    return RAM_SAVE_CONTROL_DELAYED;
+err:
+    rdma->error_state = ret;
+    return ret;
+}
+
+static int qemu_rdma_accept(RDMAContext *rdma)
+{
+    RDMACapabilities cap;
+    struct rdma_conn_param conn_param = {
+                                            .responder_resources = 2,
+                                            .private_data = &cap,
+                                            .private_data_len = sizeof(cap),
+                                         };
+    struct rdma_cm_event *cm_event;
+    struct ibv_context *verbs;
+    int ret = -EINVAL;
+    int idx;
+
+    ret = rdma_get_cm_event(rdma->channel, &cm_event);
+    if (ret) {
+        goto err_rdma_dest_wait;
+    }
+
+    if (cm_event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
+        rdma_ack_cm_event(cm_event);
+        goto err_rdma_dest_wait;
+    }
+
+    memcpy(&cap, cm_event->param.conn.private_data, sizeof(cap));
+
+    network_to_caps(&cap);
+
+    if (cap.version < 1 || cap.version > RDMA_CONTROL_VERSION_CURRENT) {
+            error_report("Unknown source RDMA version: %d, bailing...",
+                            cap.version);
+            rdma_ack_cm_event(cm_event);
+            goto err_rdma_dest_wait;
+    }
+
+    /*
+     * Respond with only the capabilities this version of QEMU knows about.
+     */
+    cap.flags &= known_capabilities;
+
+    /*
+     * Enable the ones that we do know about.
+     * Add other checks here as new ones are introduced.
+     */
+    if (cap.flags & RDMA_CAPABILITY_PIN_ALL) {
+        rdma->pin_all = true;
+    }
+
+    rdma->cm_id = cm_event->id;
+    verbs = cm_event->id->verbs;
+
+    rdma_ack_cm_event(cm_event);
+
+    trace_qemu_rdma_accept_pin_state(rdma->pin_all);
+
+    caps_to_network(&cap);
+
+    trace_qemu_rdma_accept_pin_verbsc(verbs);
+
+    if (!rdma->verbs) {
+        rdma->verbs = verbs;
+    } else if (rdma->verbs != verbs) {
+            error_report("ibv context not matching %p, %p!", rdma->verbs,
+                         verbs);
+            goto err_rdma_dest_wait;
+    }
+
+    qemu_rdma_dump_id("dest_init", verbs);
+
+    ret = qemu_rdma_alloc_pd_cq(rdma);
+    if (ret) {
+        error_report("rdma migration: error allocating pd and cq!");
+        goto err_rdma_dest_wait;
+    }
+
+    ret = qemu_rdma_alloc_qp(rdma);
+    if (ret) {
+        error_report("rdma migration: error allocating qp!");
+        goto err_rdma_dest_wait;
+    }
+
+    ret = qemu_rdma_init_ram_blocks(rdma);
+    if (ret) {
+        error_report("rdma migration: error initializing ram blocks!");
+        goto err_rdma_dest_wait;
+    }
+
+    for (idx = 0; idx < RDMA_WRID_MAX; idx++) {
+        ret = qemu_rdma_reg_control(rdma, idx);
+        if (ret) {
+            error_report("rdma: error registering %d control", idx);
+            goto err_rdma_dest_wait;
+        }
+    }
+
+    qemu_set_fd_handler(rdma->channel->fd, NULL, NULL, NULL);
+
+    ret = rdma_accept(rdma->cm_id, &conn_param);
+    if (ret) {
+        error_report("rdma_accept returns %d", ret);
+        goto err_rdma_dest_wait;
+    }
+
+    ret = rdma_get_cm_event(rdma->channel, &cm_event);
+    if (ret) {
+        error_report("rdma_accept get_cm_event failed %d", ret);
+        goto err_rdma_dest_wait;
+    }
+
+    if (cm_event->event != RDMA_CM_EVENT_ESTABLISHED) {
+        error_report("rdma_accept not event established");
+        rdma_ack_cm_event(cm_event);
+        goto err_rdma_dest_wait;
+    }
+
+    rdma_ack_cm_event(cm_event);
+    rdma->connected = true;
+
+    ret = qemu_rdma_post_recv_control(rdma, RDMA_WRID_READY);
+    if (ret) {
+        error_report("rdma migration: error posting second control recv");
+        goto err_rdma_dest_wait;
+    }
+
+    qemu_rdma_dump_gid("dest_connect", rdma->cm_id);
+
+    return 0;
+
+err_rdma_dest_wait:
+    rdma->error_state = ret;
+    qemu_rdma_cleanup(rdma);
+    return ret;
+}
+
+static int dest_ram_sort_func(const void *a, const void *b)
+{
+    unsigned int a_index = ((const RDMALocalBlock *)a)->src_index;
+    unsigned int b_index = ((const RDMALocalBlock *)b)->src_index;
+
+    return (a_index < b_index) ? -1 : (a_index != b_index);
+}
+
+/*
+ * During each iteration of the migration, we listen for instructions
+ * by the source VM to perform dynamic page registrations before they
+ * can perform RDMA operations.
+ *
+ * We respond with the 'rkey'.
+ *
+ * Keep doing this until the source tells us to stop.
+ */
+static int qemu_rdma_registration_handle(QEMUFile *f, void *opaque)
+{
+    RDMAControlHeader reg_resp = { .len = sizeof(RDMARegisterResult),
+                               .type = RDMA_CONTROL_REGISTER_RESULT,
+                               .repeat = 0,
+                             };
+    RDMAControlHeader unreg_resp = { .len = 0,
+                               .type = RDMA_CONTROL_UNREGISTER_FINISHED,
+                               .repeat = 0,
+                             };
+    RDMAControlHeader blocks = { .type = RDMA_CONTROL_RAM_BLOCKS_RESULT,
+                                 .repeat = 1 };
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque);
+    RDMAContext *rdma = rioc->rdma;
+    RDMALocalBlocks *local = &rdma->local_ram_blocks;
+    RDMAControlHeader head;
+    RDMARegister *reg, *registers;
+    RDMACompress *comp;
+    RDMARegisterResult *reg_result;
+    static RDMARegisterResult results[RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE];
+    RDMALocalBlock *block;
+    void *host_addr;
+    int ret = 0;
+    int idx = 0;
+    int count = 0;
+    int i = 0;
+
+    CHECK_ERROR_STATE();
+
+    do {
+        trace_qemu_rdma_registration_handle_wait();
+
+        ret = qemu_rdma_exchange_recv(rdma, &head, RDMA_CONTROL_NONE);
+
+        if (ret < 0) {
+            break;
+        }
+
+        if (head.repeat > RDMA_CONTROL_MAX_COMMANDS_PER_MESSAGE) {
+            error_report("rdma: Too many requests in this message (%d)."
+                            "Bailing.", head.repeat);
+            ret = -EIO;
+            break;
+        }
+
+        switch (head.type) {
+        case RDMA_CONTROL_COMPRESS:
+            comp = (RDMACompress *) rdma->wr_data[idx].control_curr;
+            network_to_compress(comp);
+
+            trace_qemu_rdma_registration_handle_compress(comp->length,
+                                                         comp->block_idx,
+                                                         comp->offset);
+            if (comp->block_idx >= rdma->local_ram_blocks.nb_blocks) {
+                error_report("rdma: 'compress' bad block index %u (vs %d)",
+                             (unsigned int)comp->block_idx,
+                             rdma->local_ram_blocks.nb_blocks);
+                ret = -EIO;
+                goto out;
+            }
+            block = &(rdma->local_ram_blocks.block[comp->block_idx]);
+
+            host_addr = block->local_host_addr +
+                            (comp->offset - block->offset);
+
+            ram_handle_compressed(host_addr, comp->value, comp->length);
+            break;
+
+        case RDMA_CONTROL_REGISTER_FINISHED:
+            trace_qemu_rdma_registration_handle_finished();
+            goto out;
+
+        case RDMA_CONTROL_RAM_BLOCKS_REQUEST:
+            trace_qemu_rdma_registration_handle_ram_blocks();
+
+            /* Sort our local RAM Block list so it's the same as the source,
+             * we can do this since we've filled in a src_index in the list
+             * as we received the RAMBlock list earlier.
+             */
+            qsort(rdma->local_ram_blocks.block,
+                  rdma->local_ram_blocks.nb_blocks,
+                  sizeof(RDMALocalBlock), dest_ram_sort_func);
+            if (rdma->pin_all) {
+                ret = qemu_rdma_reg_whole_ram_blocks(rdma);
+                if (ret) {
+                    error_report("rdma migration: error dest "
+                                    "registering ram blocks");
+                    goto out;
+                }
+            }
+
+            /*
+             * Dest uses this to prepare to transmit the RAMBlock descriptions
+             * to the source VM after connection setup.
+             * Both sides use the "remote" structure to communicate and update
+             * their "local" descriptions with what was sent.
+             */
+            for (i = 0; i < local->nb_blocks; i++) {
+                rdma->dest_blocks[i].remote_host_addr =
+                    (uintptr_t)(local->block[i].local_host_addr);
+
+                if (rdma->pin_all) {
+                    rdma->dest_blocks[i].remote_rkey = local->block[i].mr->rkey;
+                }
+
+                rdma->dest_blocks[i].offset = local->block[i].offset;
+                rdma->dest_blocks[i].length = local->block[i].length;
+
+                dest_block_to_network(&rdma->dest_blocks[i]);
+                trace_qemu_rdma_registration_handle_ram_blocks_loop(
+                    local->block[i].block_name,
+                    local->block[i].offset,
+                    local->block[i].length,
+                    local->block[i].local_host_addr,
+                    local->block[i].src_index);
+            }
+
+            blocks.len = rdma->local_ram_blocks.nb_blocks
+                                                * sizeof(RDMADestBlock);
+
+
+            ret = qemu_rdma_post_send_control(rdma,
+                                        (uint8_t *) rdma->dest_blocks, &blocks);
+
+            if (ret < 0) {
+                error_report("rdma migration: error sending remote info");
+                goto out;
+            }
+
+            break;
+        case RDMA_CONTROL_REGISTER_REQUEST:
+            trace_qemu_rdma_registration_handle_register(head.repeat);
+
+            reg_resp.repeat = head.repeat;
+            registers = (RDMARegister *) rdma->wr_data[idx].control_curr;
+
+            for (count = 0; count < head.repeat; count++) {
+                uint64_t chunk;
+                uint8_t *chunk_start, *chunk_end;
+
+                reg = &registers[count];
+                network_to_register(reg);
+
+                reg_result = &results[count];
+
+                trace_qemu_rdma_registration_handle_register_loop(count,
+                         reg->current_index, reg->key.current_addr, reg->chunks);
+
+                if (reg->current_index >= rdma->local_ram_blocks.nb_blocks) {
+                    error_report("rdma: 'register' bad block index %u (vs %d)",
+                                 (unsigned int)reg->current_index,
+                                 rdma->local_ram_blocks.nb_blocks);
+                    ret = -ENOENT;
+                    goto out;
+                }
+                block = &(rdma->local_ram_blocks.block[reg->current_index]);
+                if (block->is_ram_block) {
+                    if (block->offset > reg->key.current_addr) {
+                        error_report("rdma: bad register address for block %s"
+                            " offset: %" PRIx64 " current_addr: %" PRIx64,
+                            block->block_name, block->offset,
+                            reg->key.current_addr);
+                        ret = -ERANGE;
+                        goto out;
+                    }
+                    host_addr = (block->local_host_addr +
+                                (reg->key.current_addr - block->offset));
+                    chunk = ram_chunk_index(block->local_host_addr,
+                                            (uint8_t *) host_addr);
+                } else {
+                    chunk = reg->key.chunk;
+                    host_addr = block->local_host_addr +
+                        (reg->key.chunk * (1UL << RDMA_REG_CHUNK_SHIFT));
+                    /* Check for particularly bad chunk value */
+                    if (host_addr < (void *)block->local_host_addr) {
+                        error_report("rdma: bad chunk for block %s"
+                            " chunk: %" PRIx64,
+                            block->block_name, reg->key.chunk);
+                        ret = -ERANGE;
+                        goto out;
+                    }
+                }
+                chunk_start = ram_chunk_start(block, chunk);
+                chunk_end = ram_chunk_end(block, chunk + reg->chunks);
+                if (qemu_rdma_register_and_get_keys(rdma, block,
+                            (uintptr_t)host_addr, NULL, &reg_result->rkey,
+                            chunk, chunk_start, chunk_end)) {
+                    error_report("cannot get rkey");
+                    ret = -EINVAL;
+                    goto out;
+                }
+
+                reg_result->host_addr = (uintptr_t)block->local_host_addr;
+
+                trace_qemu_rdma_registration_handle_register_rkey(
+                                                           reg_result->rkey);
+
+                result_to_network(reg_result);
+            }
+
+            ret = qemu_rdma_post_send_control(rdma,
+                            (uint8_t *) results, &reg_resp);
+
+            if (ret < 0) {
+                error_report("Failed to send control buffer");
+                goto out;
+            }
+            break;
+        case RDMA_CONTROL_UNREGISTER_REQUEST:
+            trace_qemu_rdma_registration_handle_unregister(head.repeat);
+            unreg_resp.repeat = head.repeat;
+            registers = (RDMARegister *) rdma->wr_data[idx].control_curr;
+
+            for (count = 0; count < head.repeat; count++) {
+                reg = &registers[count];
+                network_to_register(reg);
+
+                trace_qemu_rdma_registration_handle_unregister_loop(count,
+                           reg->current_index, reg->key.chunk);
+
+                block = &(rdma->local_ram_blocks.block[reg->current_index]);
+
+                ret = ibv_dereg_mr(block->pmr[reg->key.chunk]);
+                block->pmr[reg->key.chunk] = NULL;
+
+                if (ret != 0) {
+                    perror("rdma unregistration chunk failed");
+                    ret = -ret;
+                    goto out;
+                }
+
+                rdma->total_registrations--;
+
+                trace_qemu_rdma_registration_handle_unregister_success(
+                                                       reg->key.chunk);
+            }
+
+            ret = qemu_rdma_post_send_control(rdma, NULL, &unreg_resp);
+
+            if (ret < 0) {
+                error_report("Failed to send control buffer");
+                goto out;
+            }
+            break;
+        case RDMA_CONTROL_REGISTER_RESULT:
+            error_report("Invalid RESULT message at dest.");
+            ret = -EIO;
+            goto out;
+        default:
+            error_report("Unknown control message %s", control_desc[head.type]);
+            ret = -EIO;
+            goto out;
+        }
+    } while (1);
+out:
+    if (ret < 0) {
+        rdma->error_state = ret;
+    }
+    return ret;
+}
+
+/* Destination:
+ * Called via a ram_control_load_hook during the initial RAM load section which
+ * lists the RAMBlocks by name.  This lets us know the order of the RAMBlocks
+ * on the source.
+ * We've already built our local RAMBlock list, but not yet sent the list to
+ * the source.
+ */
+static int
+rdma_block_notification_handle(QIOChannelRDMA *rioc, const char *name)
+{
+    RDMAContext *rdma = rioc->rdma;
+    int curr;
+    int found = -1;
+
+    /* Find the matching RAMBlock in our local list */
+    for (curr = 0; curr < rdma->local_ram_blocks.nb_blocks; curr++) {
+        if (!strcmp(rdma->local_ram_blocks.block[curr].block_name, name)) {
+            found = curr;
+            break;
+        }
+    }
+
+    if (found == -1) {
+        error_report("RAMBlock '%s' not found on destination", name);
+        return -ENOENT;
+    }
+
+    rdma->local_ram_blocks.block[curr].src_index = rdma->next_src_index;
+    trace_rdma_block_notification_handle(name, rdma->next_src_index);
+    rdma->next_src_index++;
+
+    return 0;
+}
+
+static int rdma_load_hook(QEMUFile *f, void *opaque, uint64_t flags, void *data)
+{
+    switch (flags) {
+    case RAM_CONTROL_BLOCK_REG:
+        return rdma_block_notification_handle(opaque, data);
+
+    case RAM_CONTROL_HOOK:
+        return qemu_rdma_registration_handle(f, opaque);
+
+    default:
+        /* Shouldn't be called with any other values */
+        abort();
+    }
+}
+
+static int qemu_rdma_registration_start(QEMUFile *f, void *opaque,
+                                        uint64_t flags, void *data)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque);
+    RDMAContext *rdma = rioc->rdma;
+
+    CHECK_ERROR_STATE();
+
+    trace_qemu_rdma_registration_start(flags);
+    qemu_put_be64(f, RAM_SAVE_FLAG_HOOK);
+    qemu_fflush(f);
+
+    return 0;
+}
+
+/*
+ * Inform dest that dynamic registrations are done for now.
+ * First, flush writes, if any.
+ */
+static int qemu_rdma_registration_stop(QEMUFile *f, void *opaque,
+                                       uint64_t flags, void *data)
+{
+    Error *local_err = NULL, **errp = &local_err;
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(opaque);
+    RDMAContext *rdma = rioc->rdma;
+    RDMAControlHeader head = { .len = 0, .repeat = 1 };
+    int ret = 0;
+
+    CHECK_ERROR_STATE();
+
+    qemu_fflush(f);
+    ret = qemu_rdma_drain_cq(f, rdma);
+
+    if (ret < 0) {
+        goto err;
+    }
+
+    if (flags == RAM_CONTROL_SETUP) {
+        RDMAControlHeader resp = {.type = RDMA_CONTROL_RAM_BLOCKS_RESULT };
+        RDMALocalBlocks *local = &rdma->local_ram_blocks;
+        int reg_result_idx, i, nb_dest_blocks;
+
+        head.type = RDMA_CONTROL_RAM_BLOCKS_REQUEST;
+        trace_qemu_rdma_registration_stop_ram();
+
+        /*
+         * Make sure that we parallelize the pinning on both sides.
+         * For very large guests, doing this serially takes a really
+         * long time, so we have to 'interleave' the pinning locally
+         * with the control messages by performing the pinning on this
+         * side before we receive the control response from the other
+         * side that the pinning has completed.
+         */
+        ret = qemu_rdma_exchange_send(rdma, &head, NULL, &resp,
+                    &reg_result_idx, rdma->pin_all ?
+                    qemu_rdma_reg_whole_ram_blocks : NULL);
+        if (ret < 0) {
+            ERROR(errp, "receiving remote info!");
+            return ret;
+        }
+
+        nb_dest_blocks = resp.len / sizeof(RDMADestBlock);
+
+        /*
+         * The protocol uses two different sets of rkeys (mutually exclusive):
+         * 1. One key to represent the virtual address of the entire ram block.
+         *    (dynamic chunk registration disabled - pin everything with one rkey.)
+         * 2. One to represent individual chunks within a ram block.
+         *    (dynamic chunk registration enabled - pin individual chunks.)
+         *
+         * Once the capability is successfully negotiated, the destination transmits
+         * the keys to use (or sends them later) including the virtual addresses
+         * and then propagates the remote ram block descriptions to his local copy.
+         */
+
+        if (local->nb_blocks != nb_dest_blocks) {
+            ERROR(errp, "ram blocks mismatch (Number of blocks %d vs %d) "
+                        "Your QEMU command line parameters are probably "
+                        "not identical on both the source and destination.",
+                        local->nb_blocks, nb_dest_blocks);
+            rdma->error_state = -EINVAL;
+            return -EINVAL;
+        }
+
+        qemu_rdma_move_header(rdma, reg_result_idx, &resp);
+        memcpy(rdma->dest_blocks,
+            rdma->wr_data[reg_result_idx].control_curr, resp.len);
+        for (i = 0; i < nb_dest_blocks; i++) {
+            network_to_dest_block(&rdma->dest_blocks[i]);
+
+            /* We require that the blocks are in the same order */
+            if (rdma->dest_blocks[i].length != local->block[i].length) {
+                ERROR(errp, "Block %s/%d has a different length %" PRIu64
+                            "vs %" PRIu64, local->block[i].block_name, i,
+                            local->block[i].length,
+                            rdma->dest_blocks[i].length);
+                rdma->error_state = -EINVAL;
+                return -EINVAL;
+            }
+            local->block[i].remote_host_addr =
+                    rdma->dest_blocks[i].remote_host_addr;
+            local->block[i].remote_rkey = rdma->dest_blocks[i].remote_rkey;
+        }
+    }
+
+    trace_qemu_rdma_registration_stop(flags);
+
+    head.type = RDMA_CONTROL_REGISTER_FINISHED;
+    ret = qemu_rdma_exchange_send(rdma, &head, NULL, NULL, NULL, NULL);
+
+    if (ret < 0) {
+        goto err;
+    }
+
+    return 0;
+err:
+    rdma->error_state = ret;
+    return ret;
+}
+
+static const QEMUFileHooks rdma_read_hooks = {
+    .hook_ram_load = rdma_load_hook,
+};
+
+static const QEMUFileHooks rdma_write_hooks = {
+    .before_ram_iterate = qemu_rdma_registration_start,
+    .after_ram_iterate  = qemu_rdma_registration_stop,
+    .save_page          = qemu_rdma_save_page,
+};
+
+
+static void qio_channel_rdma_finalize(Object *obj)
+{
+    QIOChannelRDMA *rioc = QIO_CHANNEL_RDMA(obj);
+    if (rioc->rdma) {
+        qemu_rdma_cleanup(rioc->rdma);
+        g_free(rioc->rdma);
+        rioc->rdma = NULL;
+    }
+}
+
+static void qio_channel_rdma_class_init(ObjectClass *klass,
+                                        void *class_data G_GNUC_UNUSED)
+{
+    QIOChannelClass *ioc_klass = QIO_CHANNEL_CLASS(klass);
+
+    ioc_klass->io_writev = qio_channel_rdma_writev;
+    ioc_klass->io_readv = qio_channel_rdma_readv;
+    ioc_klass->io_set_blocking = qio_channel_rdma_set_blocking;
+    ioc_klass->io_close = qio_channel_rdma_close;
+    ioc_klass->io_create_watch = qio_channel_rdma_create_watch;
+}
+
+static const TypeInfo qio_channel_rdma_info = {
+    .parent = TYPE_QIO_CHANNEL,
+    .name = TYPE_QIO_CHANNEL_RDMA,
+    .instance_size = sizeof(QIOChannelRDMA),
+    .instance_finalize = qio_channel_rdma_finalize,
+    .class_init = qio_channel_rdma_class_init,
+};
+
+static void qio_channel_rdma_register_types(void)
+{
+    type_register_static(&qio_channel_rdma_info);
+}
+
+type_init(qio_channel_rdma_register_types);
+
+static QEMUFile *qemu_fopen_rdma(RDMAContext *rdma, const char *mode)
+{
+    QIOChannelRDMA *rioc;
+
+    if (qemu_file_mode_is_not_valid(mode)) {
+        return NULL;
+    }
+
+    rioc = QIO_CHANNEL_RDMA(object_new(TYPE_QIO_CHANNEL_RDMA));
+    rioc->rdma = rdma;
+
+    if (mode[0] == 'w') {
+        rioc->file = qemu_fopen_channel_output(QIO_CHANNEL(rioc));
+        qemu_file_set_hooks(rioc->file, &rdma_write_hooks);
+    } else {
+        rioc->file = qemu_fopen_channel_input(QIO_CHANNEL(rioc));
+        qemu_file_set_hooks(rioc->file, &rdma_read_hooks);
+    }
+
+    return rioc->file;
+}
+
+static void rdma_accept_incoming_migration(void *opaque)
+{
+    RDMAContext *rdma = opaque;
+    int ret;
+    QEMUFile *f;
+    Error *local_err = NULL, **errp = &local_err;
+
+    trace_qemu_rdma_accept_incoming_migration();
+    ret = qemu_rdma_accept(rdma);
+
+    if (ret) {
+        ERROR(errp, "RDMA Migration initialization failed!");
+        return;
+    }
+
+    trace_qemu_rdma_accept_incoming_migration_accepted();
+
+    f = qemu_fopen_rdma(rdma, "rb");
+    if (f == NULL) {
+        ERROR(errp, "could not qemu_fopen_rdma!");
+        qemu_rdma_cleanup(rdma);
+        return;
+    }
+
+    rdma->migration_started_on_destination = 1;
+    migration_fd_process_incoming(f);
+}
+
+void rdma_start_incoming_migration(const char *host_port, Error **errp)
+{
+    int ret;
+    RDMAContext *rdma;
+    Error *local_err = NULL;
+
+    trace_rdma_start_incoming_migration();
+    rdma = qemu_rdma_data_init(host_port, &local_err);
+
+    if (rdma == NULL) {
+        goto err;
+    }
+
+    ret = qemu_rdma_dest_init(rdma, &local_err);
+
+    if (ret) {
+        goto err;
+    }
+
+    trace_rdma_start_incoming_migration_after_dest_init();
+
+    ret = rdma_listen(rdma->listen_id, 5);
+
+    if (ret) {
+        ERROR(errp, "listening on socket!");
+        goto err;
+    }
+
+    trace_rdma_start_incoming_migration_after_rdma_listen();
+
+    qemu_set_fd_handler(rdma->channel->fd, rdma_accept_incoming_migration,
+                        NULL, (void *)(intptr_t)rdma);
+    return;
+err:
+    error_propagate(errp, local_err);
+    g_free(rdma);
+}
+
+void rdma_start_outgoing_migration(void *opaque,
+                            const char *host_port, Error **errp)
+{
+    MigrationState *s = opaque;
+    RDMAContext *rdma = qemu_rdma_data_init(host_port, errp);
+    int ret = 0;
+
+    if (rdma == NULL) {
+        goto err;
+    }
+
+    ret = qemu_rdma_source_init(rdma, errp,
+        s->enabled_capabilities[MIGRATION_CAPABILITY_RDMA_PIN_ALL]);
+
+    if (ret) {
+        goto err;
+    }
+
+    trace_rdma_start_outgoing_migration_after_rdma_source_init();
+    ret = qemu_rdma_connect(rdma, errp);
+
+    if (ret) {
+        goto err;
+    }
+
+    trace_rdma_start_outgoing_migration_after_rdma_connect();
+
+    s->to_dst_file = qemu_fopen_rdma(rdma, "wb");
+    migrate_fd_connect(s);
+    return;
+err:
+    g_free(rdma);
+}
diff --git a/migration/savevm.c b/migration/savevm.c
new file mode 100644
index 0000000..ca9d29e
--- /dev/null
+++ b/migration/savevm.c
@@ -0,0 +1,2385 @@
+/*
+ * QEMU System Emulator
+ *
+ * Copyright (c) 2003-2008 Fabrice Bellard
+ * Copyright (c) 2009-2015 Red Hat Inc
+ *
+ * Authors:
+ *  Juan Quintela <quintela@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "hw/boards.h"
+#include "hw/hw.h"
+#include "hw/qdev.h"
+#include "hw/xen/xen.h"
+#include "net/net.h"
+#include "monitor/monitor.h"
+#include "sysemu/sysemu.h"
+#include "qemu/timer.h"
+#include "audio/audio.h"
+#include "migration/migration.h"
+#include "migration/postcopy-ram.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+#include "qemu/sockets.h"
+#include "qemu/queue.h"
+#include "sysemu/cpus.h"
+#include "exec/memory.h"
+#include "qmp-commands.h"
+#include "trace.h"
+#include "qemu/bitops.h"
+#include "qemu/iov.h"
+#include "block/snapshot.h"
+#include "block/qapi.h"
+#include "qemu/cutils.h"
+#include "io/channel-buffer.h"
+#include "io/channel-file.h"
+
+#ifndef ETH_P_RARP
+#define ETH_P_RARP 0x8035
+#endif
+#define ARP_HTYPE_ETH 0x0001
+#define ARP_PTYPE_IP 0x0800
+#define ARP_OP_REQUEST_REV 0x3
+
+const unsigned int postcopy_ram_discard_version = 0;
+
+static bool skip_section_footers;
+
+static struct mig_cmd_args {
+    ssize_t     len; /* -1 = variable */
+    const char *name;
+} mig_cmd_args[] = {
+    [MIG_CMD_INVALID]          = { .len = -1, .name = "INVALID" },
+    [MIG_CMD_OPEN_RETURN_PATH] = { .len =  0, .name = "OPEN_RETURN_PATH" },
+    [MIG_CMD_PING]             = { .len = sizeof(uint32_t), .name = "PING" },
+    [MIG_CMD_POSTCOPY_ADVISE]  = { .len = 16, .name = "POSTCOPY_ADVISE" },
+    [MIG_CMD_POSTCOPY_LISTEN]  = { .len =  0, .name = "POSTCOPY_LISTEN" },
+    [MIG_CMD_POSTCOPY_RUN]     = { .len =  0, .name = "POSTCOPY_RUN" },
+    [MIG_CMD_POSTCOPY_RAM_DISCARD] = {
+                                   .len = -1, .name = "POSTCOPY_RAM_DISCARD" },
+    [MIG_CMD_PACKAGED]         = { .len =  4, .name = "PACKAGED" },
+    [MIG_CMD_MAX]              = { .len = -1, .name = "MAX" },
+};
+
+static int announce_self_create(uint8_t *buf,
+                                uint8_t *mac_addr)
+{
+    /* Ethernet header. */
+    memset(buf, 0xff, 6);         /* destination MAC addr */
+    memcpy(buf + 6, mac_addr, 6); /* source MAC addr */
+    *(uint16_t *)(buf + 12) = htons(ETH_P_RARP); /* ethertype */
+
+    /* RARP header. */
+    *(uint16_t *)(buf + 14) = htons(ARP_HTYPE_ETH); /* hardware addr space */
+    *(uint16_t *)(buf + 16) = htons(ARP_PTYPE_IP); /* protocol addr space */
+    *(buf + 18) = 6; /* hardware addr length (ethernet) */
+    *(buf + 19) = 4; /* protocol addr length (IPv4) */
+    *(uint16_t *)(buf + 20) = htons(ARP_OP_REQUEST_REV); /* opcode */
+    memcpy(buf + 22, mac_addr, 6); /* source hw addr */
+    memset(buf + 28, 0x00, 4);     /* source protocol addr */
+    memcpy(buf + 32, mac_addr, 6); /* target hw addr */
+    memset(buf + 38, 0x00, 4);     /* target protocol addr */
+
+    /* Padding to get up to 60 bytes (ethernet min packet size, minus FCS). */
+    memset(buf + 42, 0x00, 18);
+
+    return 60; /* len (FCS will be added by hardware) */
+}
+
+static void qemu_announce_self_iter(NICState *nic, void *opaque)
+{
+    uint8_t buf[60];
+    int len;
+
+    trace_qemu_announce_self_iter(qemu_ether_ntoa(&nic->conf->macaddr));
+    len = announce_self_create(buf, nic->conf->macaddr.a);
+
+    qemu_send_packet_raw(qemu_get_queue(nic), buf, len);
+}
+
+
+static void qemu_announce_self_once(void *opaque)
+{
+    static int count = SELF_ANNOUNCE_ROUNDS;
+    QEMUTimer *timer = *(QEMUTimer **)opaque;
+
+    qemu_foreach_nic(qemu_announce_self_iter, NULL);
+
+    if (--count) {
+        /* delay 50ms, 150ms, 250ms, ... */
+        timer_mod(timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME) +
+                  self_announce_delay(count));
+    } else {
+            timer_del(timer);
+            timer_free(timer);
+    }
+}
+
+void qemu_announce_self(void)
+{
+    static QEMUTimer *timer;
+    timer = timer_new_ms(QEMU_CLOCK_REALTIME, qemu_announce_self_once, &timer);
+    qemu_announce_self_once(&timer);
+}
+
+/***********************************************************/
+/* savevm/loadvm support */
+
+static ssize_t block_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
+                                   int64_t pos)
+{
+    int ret;
+    QEMUIOVector qiov;
+
+    qemu_iovec_init_external(&qiov, iov, iovcnt);
+    ret = bdrv_writev_vmstate(opaque, &qiov, pos);
+    if (ret < 0) {
+        return ret;
+    }
+
+    return qiov.size;
+}
+
+static ssize_t block_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
+                                size_t size)
+{
+    return bdrv_load_vmstate(opaque, buf, pos, size);
+}
+
+static int bdrv_fclose(void *opaque)
+{
+    return bdrv_flush(opaque);
+}
+
+static const QEMUFileOps bdrv_read_ops = {
+    .get_buffer = block_get_buffer,
+    .close =      bdrv_fclose
+};
+
+static const QEMUFileOps bdrv_write_ops = {
+    .writev_buffer  = block_writev_buffer,
+    .close          = bdrv_fclose
+};
+
+static QEMUFile *qemu_fopen_bdrv(BlockDriverState *bs, int is_writable)
+{
+    if (is_writable) {
+        return qemu_fopen_ops(bs, &bdrv_write_ops);
+    }
+    return qemu_fopen_ops(bs, &bdrv_read_ops);
+}
+
+
+/* QEMUFile timer support.
+ * Not in qemu-file.c to not add qemu-timer.c as dependency to qemu-file.c
+ */
+
+void timer_put(QEMUFile *f, QEMUTimer *ts)
+{
+    uint64_t expire_time;
+
+    expire_time = timer_expire_time_ns(ts);
+    qemu_put_be64(f, expire_time);
+}
+
+void timer_get(QEMUFile *f, QEMUTimer *ts)
+{
+    uint64_t expire_time;
+
+    expire_time = qemu_get_be64(f);
+    if (expire_time != -1) {
+        timer_mod_ns(ts, expire_time);
+    } else {
+        timer_del(ts);
+    }
+}
+
+
+/* VMState timer support.
+ * Not in vmstate.c to not add qemu-timer.c as dependency to vmstate.c
+ */
+
+static int get_timer(QEMUFile *f, void *pv, size_t size)
+{
+    QEMUTimer *v = pv;
+    timer_get(f, v);
+    return 0;
+}
+
+static void put_timer(QEMUFile *f, void *pv, size_t size)
+{
+    QEMUTimer *v = pv;
+    timer_put(f, v);
+}
+
+const VMStateInfo vmstate_info_timer = {
+    .name = "timer",
+    .get  = get_timer,
+    .put  = put_timer,
+};
+
+
+typedef struct CompatEntry {
+    char idstr[256];
+    int instance_id;
+} CompatEntry;
+
+typedef struct SaveStateEntry {
+    QTAILQ_ENTRY(SaveStateEntry) entry;
+    char idstr[256];
+    int instance_id;
+    int alias_id;
+    int version_id;
+    int section_id;
+    SaveVMHandlers *ops;
+    const VMStateDescription *vmsd;
+    void *opaque;
+    CompatEntry *compat;
+    int is_ram;
+} SaveStateEntry;
+
+typedef struct SaveState {
+    QTAILQ_HEAD(, SaveStateEntry) handlers;
+    int global_section_id;
+    bool skip_configuration;
+    uint32_t len;
+    const char *name;
+} SaveState;
+
+static SaveState savevm_state = {
+    .handlers = QTAILQ_HEAD_INITIALIZER(savevm_state.handlers),
+    .global_section_id = 0,
+    .skip_configuration = false,
+};
+
+void savevm_skip_configuration(void)
+{
+    savevm_state.skip_configuration = true;
+}
+
+
+static void configuration_pre_save(void *opaque)
+{
+    SaveState *state = opaque;
+    const char *current_name = MACHINE_GET_CLASS(current_machine)->name;
+
+    state->len = strlen(current_name);
+    state->name = current_name;
+}
+
+static int configuration_post_load(void *opaque, int version_id)
+{
+    SaveState *state = opaque;
+    const char *current_name = MACHINE_GET_CLASS(current_machine)->name;
+
+    if (strncmp(state->name, current_name, state->len) != 0) {
+        error_report("Machine type received is '%.*s' and local is '%s'",
+                     (int) state->len, state->name, current_name);
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static const VMStateDescription vmstate_configuration = {
+    .name = "configuration",
+    .version_id = 1,
+    .post_load = configuration_post_load,
+    .pre_save = configuration_pre_save,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(len, SaveState),
+        VMSTATE_VBUFFER_ALLOC_UINT32(name, SaveState, 0, NULL, 0, len),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
+static void dump_vmstate_vmsd(FILE *out_file,
+                              const VMStateDescription *vmsd, int indent,
+                              bool is_subsection);
+
+static void dump_vmstate_vmsf(FILE *out_file, const VMStateField *field,
+                              int indent)
+{
+    fprintf(out_file, "%*s{\n", indent, "");
+    indent += 2;
+    fprintf(out_file, "%*s\"field\": \"%s\",\n", indent, "", field->name);
+    fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "",
+            field->version_id);
+    fprintf(out_file, "%*s\"field_exists\": %s,\n", indent, "",
+            field->field_exists ? "true" : "false");
+    fprintf(out_file, "%*s\"size\": %zu", indent, "", field->size);
+    if (field->vmsd != NULL) {
+        fprintf(out_file, ",\n");
+        dump_vmstate_vmsd(out_file, field->vmsd, indent, false);
+    }
+    fprintf(out_file, "\n%*s}", indent - 2, "");
+}
+
+static void dump_vmstate_vmss(FILE *out_file,
+                              const VMStateDescription **subsection,
+                              int indent)
+{
+    if (*subsection != NULL) {
+        dump_vmstate_vmsd(out_file, *subsection, indent, true);
+    }
+}
+
+static void dump_vmstate_vmsd(FILE *out_file,
+                              const VMStateDescription *vmsd, int indent,
+                              bool is_subsection)
+{
+    if (is_subsection) {
+        fprintf(out_file, "%*s{\n", indent, "");
+    } else {
+        fprintf(out_file, "%*s\"%s\": {\n", indent, "", "Description");
+    }
+    indent += 2;
+    fprintf(out_file, "%*s\"name\": \"%s\",\n", indent, "", vmsd->name);
+    fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "",
+            vmsd->version_id);
+    fprintf(out_file, "%*s\"minimum_version_id\": %d", indent, "",
+            vmsd->minimum_version_id);
+    if (vmsd->fields != NULL) {
+        const VMStateField *field = vmsd->fields;
+        bool first;
+
+        fprintf(out_file, ",\n%*s\"Fields\": [\n", indent, "");
+        first = true;
+        while (field->name != NULL) {
+            if (field->flags & VMS_MUST_EXIST) {
+                /* Ignore VMSTATE_VALIDATE bits; these don't get migrated */
+                field++;
+                continue;
+            }
+            if (!first) {
+                fprintf(out_file, ",\n");
+            }
+            dump_vmstate_vmsf(out_file, field, indent + 2);
+            field++;
+            first = false;
+        }
+        fprintf(out_file, "\n%*s]", indent, "");
+    }
+    if (vmsd->subsections != NULL) {
+        const VMStateDescription **subsection = vmsd->subsections;
+        bool first;
+
+        fprintf(out_file, ",\n%*s\"Subsections\": [\n", indent, "");
+        first = true;
+        while (*subsection != NULL) {
+            if (!first) {
+                fprintf(out_file, ",\n");
+            }
+            dump_vmstate_vmss(out_file, subsection, indent + 2);
+            subsection++;
+            first = false;
+        }
+        fprintf(out_file, "\n%*s]", indent, "");
+    }
+    fprintf(out_file, "\n%*s}", indent - 2, "");
+}
+
+static void dump_machine_type(FILE *out_file)
+{
+    MachineClass *mc;
+
+    mc = MACHINE_GET_CLASS(current_machine);
+
+    fprintf(out_file, "  \"vmschkmachine\": {\n");
+    fprintf(out_file, "    \"Name\": \"%s\"\n", mc->name);
+    fprintf(out_file, "  },\n");
+}
+
+void dump_vmstate_json_to_file(FILE *out_file)
+{
+    GSList *list, *elt;
+    bool first;
+
+    fprintf(out_file, "{\n");
+    dump_machine_type(out_file);
+
+    first = true;
+    list = object_class_get_list(TYPE_DEVICE, true);
+    for (elt = list; elt; elt = elt->next) {
+        DeviceClass *dc = OBJECT_CLASS_CHECK(DeviceClass, elt->data,
+                                             TYPE_DEVICE);
+        const char *name;
+        int indent = 2;
+
+        if (!dc->vmsd) {
+            continue;
+        }
+
+        if (!first) {
+            fprintf(out_file, ",\n");
+        }
+        name = object_class_get_name(OBJECT_CLASS(dc));
+        fprintf(out_file, "%*s\"%s\": {\n", indent, "", name);
+        indent += 2;
+        fprintf(out_file, "%*s\"Name\": \"%s\",\n", indent, "", name);
+        fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "",
+                dc->vmsd->version_id);
+        fprintf(out_file, "%*s\"minimum_version_id\": %d,\n", indent, "",
+                dc->vmsd->minimum_version_id);
+
+        dump_vmstate_vmsd(out_file, dc->vmsd, indent, false);
+
+        fprintf(out_file, "\n%*s}", indent - 2, "");
+        first = false;
+    }
+    fprintf(out_file, "\n}\n");
+    fclose(out_file);
+}
+
+static int calculate_new_instance_id(const char *idstr)
+{
+    SaveStateEntry *se;
+    int instance_id = 0;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (strcmp(idstr, se->idstr) == 0
+            && instance_id <= se->instance_id) {
+            instance_id = se->instance_id + 1;
+        }
+    }
+    return instance_id;
+}
+
+static int calculate_compat_instance_id(const char *idstr)
+{
+    SaveStateEntry *se;
+    int instance_id = 0;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->compat) {
+            continue;
+        }
+
+        if (strcmp(idstr, se->compat->idstr) == 0
+            && instance_id <= se->compat->instance_id) {
+            instance_id = se->compat->instance_id + 1;
+        }
+    }
+    return instance_id;
+}
+
+/* TODO: Individual devices generally have very little idea about the rest
+   of the system, so instance_id should be removed/replaced.
+   Meanwhile pass -1 as instance_id if you do not already have a clearly
+   distinguishing id for all instances of your device class. */
+int register_savevm_live(DeviceState *dev,
+                         const char *idstr,
+                         int instance_id,
+                         int version_id,
+                         SaveVMHandlers *ops,
+                         void *opaque)
+{
+    SaveStateEntry *se;
+
+    se = g_new0(SaveStateEntry, 1);
+    se->version_id = version_id;
+    se->section_id = savevm_state.global_section_id++;
+    se->ops = ops;
+    se->opaque = opaque;
+    se->vmsd = NULL;
+    /* if this is a live_savem then set is_ram */
+    if (ops->save_live_setup != NULL) {
+        se->is_ram = 1;
+    }
+
+    if (dev) {
+        char *id = qdev_get_dev_path(dev);
+        if (id) {
+            pstrcpy(se->idstr, sizeof(se->idstr), id);
+            pstrcat(se->idstr, sizeof(se->idstr), "/");
+            g_free(id);
+
+            se->compat = g_new0(CompatEntry, 1);
+            pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), idstr);
+            se->compat->instance_id = instance_id == -1 ?
+                         calculate_compat_instance_id(idstr) : instance_id;
+            instance_id = -1;
+        }
+    }
+    pstrcat(se->idstr, sizeof(se->idstr), idstr);
+
+    if (instance_id == -1) {
+        se->instance_id = calculate_new_instance_id(se->idstr);
+    } else {
+        se->instance_id = instance_id;
+    }
+    assert(!se->compat || se->instance_id == 0);
+    /* add at the end of list */
+    QTAILQ_INSERT_TAIL(&savevm_state.handlers, se, entry);
+    return 0;
+}
+
+int register_savevm(DeviceState *dev,
+                    const char *idstr,
+                    int instance_id,
+                    int version_id,
+                    SaveStateHandler *save_state,
+                    LoadStateHandler *load_state,
+                    void *opaque)
+{
+    SaveVMHandlers *ops = g_new0(SaveVMHandlers, 1);
+    ops->save_state = save_state;
+    ops->load_state = load_state;
+    return register_savevm_live(dev, idstr, instance_id, version_id,
+                                ops, opaque);
+}
+
+int register_savevm_with_post_load(DeviceState *dev,
+                                   const char *idstr,
+                                   int instance_id,
+                                   int version_id,
+                                   SaveStateHandler *save_state,
+                                   LoadStateHandler *load_state,
+                                   PostLoadHandler *post_load,
+                                   void *opaque)
+{
+    SaveVMHandlers *ops = g_new0(SaveVMHandlers, 1);
+    ops->save_state = save_state;
+    ops->load_state = load_state;
+    ops->post_load = post_load;
+    return register_savevm_live(dev, idstr, instance_id, version_id,
+                                ops, opaque);
+}
+
+void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque)
+{
+    SaveStateEntry *se, *new_se;
+    char id[256] = "";
+
+    if (dev) {
+        char *path = qdev_get_dev_path(dev);
+        if (path) {
+            pstrcpy(id, sizeof(id), path);
+            pstrcat(id, sizeof(id), "/");
+            g_free(path);
+        }
+    }
+    pstrcat(id, sizeof(id), idstr);
+
+    QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) {
+        if (strcmp(se->idstr, id) == 0 && se->opaque == opaque) {
+            QTAILQ_REMOVE(&savevm_state.handlers, se, entry);
+            g_free(se->compat);
+            g_free(se->ops);
+            g_free(se);
+        }
+    }
+}
+
+int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
+                                   const VMStateDescription *vmsd,
+                                   void *opaque, int alias_id,
+                                   int required_for_version)
+{
+    SaveStateEntry *se;
+
+    /* If this triggers, alias support can be dropped for the vmsd. */
+    assert(alias_id == -1 || required_for_version >= vmsd->minimum_version_id);
+
+    se = g_new0(SaveStateEntry, 1);
+    se->version_id = vmsd->version_id;
+    se->section_id = savevm_state.global_section_id++;
+    se->opaque = opaque;
+    se->vmsd = vmsd;
+    se->alias_id = alias_id;
+
+    if (dev) {
+        char *id = qdev_get_dev_path(dev);
+        if (id) {
+            pstrcpy(se->idstr, sizeof(se->idstr), id);
+            pstrcat(se->idstr, sizeof(se->idstr), "/");
+            g_free(id);
+
+            se->compat = g_new0(CompatEntry, 1);
+            pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), vmsd->name);
+            se->compat->instance_id = instance_id == -1 ?
+                         calculate_compat_instance_id(vmsd->name) : instance_id;
+            instance_id = -1;
+        }
+    }
+    pstrcat(se->idstr, sizeof(se->idstr), vmsd->name);
+
+    if (instance_id == -1) {
+        se->instance_id = calculate_new_instance_id(se->idstr);
+    } else {
+        se->instance_id = instance_id;
+    }
+    assert(!se->compat || se->instance_id == 0);
+    /* add at the end of list */
+    QTAILQ_INSERT_TAIL(&savevm_state.handlers, se, entry);
+    return 0;
+}
+
+void vmstate_unregister(DeviceState *dev, const VMStateDescription *vmsd,
+                        void *opaque)
+{
+    SaveStateEntry *se, *new_se;
+
+    QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) {
+        if (se->vmsd == vmsd && se->opaque == opaque) {
+            QTAILQ_REMOVE(&savevm_state.handlers, se, entry);
+            g_free(se->compat);
+            g_free(se);
+        }
+    }
+}
+
+static int vmstate_load(QEMUFile *f, SaveStateEntry *se, int version_id)
+{
+    trace_vmstate_load(se->idstr, se->vmsd ? se->vmsd->name : "(old)");
+    if (!se->vmsd) {         /* Old style */
+        return se->ops->load_state(f, se->opaque, version_id);
+    }
+    return vmstate_load_state(f, se->vmsd, se->opaque, version_id);
+}
+
+static void vmstate_save_old_style(QEMUFile *f, SaveStateEntry *se, QJSON *vmdesc)
+{
+    int64_t old_offset, size;
+
+    old_offset = qemu_ftell_fast(f);
+    se->ops->save_state(f, se->opaque);
+    size = qemu_ftell_fast(f) - old_offset;
+
+    if (vmdesc) {
+        json_prop_int(vmdesc, "size", size);
+        json_start_array(vmdesc, "fields");
+        json_start_object(vmdesc, NULL);
+        json_prop_str(vmdesc, "name", "data");
+        json_prop_int(vmdesc, "size", size);
+        json_prop_str(vmdesc, "type", "buffer");
+        json_end_object(vmdesc);
+        json_end_array(vmdesc);
+    }
+}
+
+static void vmstate_save(QEMUFile *f, SaveStateEntry *se, QJSON *vmdesc)
+{
+    trace_vmstate_save(se->idstr, se->vmsd ? se->vmsd->name : "(old)");
+    if (!se->vmsd) {
+        vmstate_save_old_style(f, se, vmdesc);
+        return;
+    }
+    vmstate_save_state(f, se->vmsd, se->opaque, vmdesc);
+}
+
+void savevm_skip_section_footers(void)
+{
+    skip_section_footers = true;
+}
+
+/*
+ * Write the header for device section (QEMU_VM_SECTION START/END/PART/FULL)
+ */
+static void save_section_header(QEMUFile *f, SaveStateEntry *se,
+                                uint8_t section_type)
+{
+    qemu_put_byte(f, section_type);
+    qemu_put_be32(f, se->section_id);
+
+    if (section_type == QEMU_VM_SECTION_FULL ||
+        section_type == QEMU_VM_SECTION_START) {
+        /* ID string */
+        size_t len = strlen(se->idstr);
+        qemu_put_byte(f, len);
+        qemu_put_buffer(f, (uint8_t *)se->idstr, len);
+
+        qemu_put_be32(f, se->instance_id);
+        qemu_put_be32(f, se->version_id);
+    }
+}
+
+/*
+ * Write a footer onto device sections that catches cases misformatted device
+ * sections.
+ */
+static void save_section_footer(QEMUFile *f, SaveStateEntry *se)
+{
+    if (!skip_section_footers) {
+        qemu_put_byte(f, QEMU_VM_SECTION_FOOTER);
+        qemu_put_be32(f, se->section_id);
+    }
+}
+
+/**
+ * qemu_savevm_command_send: Send a 'QEMU_VM_COMMAND' type element with the
+ *                           command and associated data.
+ *
+ * @f: File to send command on
+ * @command: Command type to send
+ * @len: Length of associated data
+ * @data: Data associated with command.
+ */
+void qemu_savevm_command_send(QEMUFile *f,
+                              enum qemu_vm_cmd command,
+                              uint16_t len,
+                              uint8_t *data)
+{
+    trace_savevm_command_send(command, len);
+    qemu_put_byte(f, QEMU_VM_COMMAND);
+    qemu_put_be16(f, (uint16_t)command);
+    qemu_put_be16(f, len);
+    qemu_put_buffer(f, data, len);
+    qemu_fflush(f);
+}
+
+void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
+{
+    uint32_t buf;
+
+    trace_savevm_send_ping(value);
+    buf = cpu_to_be32(value);
+    qemu_savevm_command_send(f, MIG_CMD_PING, sizeof(value), (uint8_t *)&buf);
+}
+
+void qemu_savevm_send_open_return_path(QEMUFile *f)
+{
+    trace_savevm_send_open_return_path();
+    qemu_savevm_command_send(f, MIG_CMD_OPEN_RETURN_PATH, 0, NULL);
+}
+
+/* We have a buffer of data to send; we don't want that all to be loaded
+ * by the command itself, so the command contains just the length of the
+ * extra buffer that we then send straight after it.
+ * TODO: Must be a better way to organise that
+ *
+ * Returns:
+ *    0 on success
+ *    -ve on error
+ */
+int qemu_savevm_send_packaged(QEMUFile *f, const uint8_t *buf, size_t len)
+{
+    uint32_t tmp;
+
+    if (len > MAX_VM_CMD_PACKAGED_SIZE) {
+        error_report("%s: Unreasonably large packaged state: %zu",
+                     __func__, len);
+        return -1;
+    }
+
+    tmp = cpu_to_be32(len);
+
+    trace_qemu_savevm_send_packaged();
+    qemu_savevm_command_send(f, MIG_CMD_PACKAGED, 4, (uint8_t *)&tmp);
+
+    qemu_put_buffer(f, buf, len);
+
+    return 0;
+}
+
+/* Send prior to any postcopy transfer */
+void qemu_savevm_send_postcopy_advise(QEMUFile *f)
+{
+    uint64_t tmp[2];
+    tmp[0] = cpu_to_be64(getpagesize());
+    tmp[1] = cpu_to_be64(1ul << qemu_target_page_bits());
+
+    trace_qemu_savevm_send_postcopy_advise();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_ADVISE, 16, (uint8_t *)tmp);
+}
+
+/* Sent prior to starting the destination running in postcopy, discard pages
+ * that have already been sent but redirtied on the source.
+ * CMD_POSTCOPY_RAM_DISCARD consist of:
+ *      byte   version (0)
+ *      byte   Length of name field (not including 0)
+ *  n x byte   RAM block name
+ *      byte   0 terminator (just for safety)
+ *  n x        Byte ranges within the named RAMBlock
+ *      be64   Start of the range
+ *      be64   Length
+ *
+ *  name:  RAMBlock name that these entries are part of
+ *  len: Number of page entries
+ *  start_list: 'len' addresses
+ *  length_list: 'len' addresses
+ *
+ */
+void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
+                                           uint16_t len,
+                                           uint64_t *start_list,
+                                           uint64_t *length_list)
+{
+    uint8_t *buf;
+    uint16_t tmplen;
+    uint16_t t;
+    size_t name_len = strlen(name);
+
+    trace_qemu_savevm_send_postcopy_ram_discard(name, len);
+    assert(name_len < 256);
+    buf = g_malloc0(1 + 1 + name_len + 1 + (8 + 8) * len);
+    buf[0] = postcopy_ram_discard_version;
+    buf[1] = name_len;
+    memcpy(buf + 2, name, name_len);
+    tmplen = 2 + name_len;
+    buf[tmplen++] = '\0';
+
+    for (t = 0; t < len; t++) {
+        stq_be_p(buf + tmplen, start_list[t]);
+        tmplen += 8;
+        stq_be_p(buf + tmplen, length_list[t]);
+        tmplen += 8;
+    }
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RAM_DISCARD, tmplen, buf);
+    g_free(buf);
+}
+
+/* Get the destination into a state where it can receive postcopy data. */
+void qemu_savevm_send_postcopy_listen(QEMUFile *f)
+{
+    trace_savevm_send_postcopy_listen();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_LISTEN, 0, NULL);
+}
+
+/* Kick the destination into running */
+void qemu_savevm_send_postcopy_run(QEMUFile *f)
+{
+    trace_savevm_send_postcopy_run();
+    qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RUN, 0, NULL);
+}
+
+bool qemu_savevm_state_blocked(Error **errp)
+{
+    SaveStateEntry *se;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (se->vmsd && se->vmsd->unmigratable) {
+            error_setg(errp, "State blocked by non-migratable device '%s'",
+                       se->idstr);
+            return true;
+        }
+    }
+    return false;
+}
+
+static bool enforce_config_section(void)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    return machine->enforce_config_section;
+}
+
+void qemu_savevm_state_header(QEMUFile *f)
+{
+    trace_savevm_state_header();
+    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+    qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+
+    if (!savevm_state.skip_configuration || enforce_config_section()) {
+        qemu_put_byte(f, QEMU_VM_CONFIGURATION);
+        vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0);
+    }
+
+}
+
+void qemu_savevm_state_begin(QEMUFile *f,
+                             const MigrationParams *params)
+{
+    SaveStateEntry *se;
+    int ret;
+
+    trace_savevm_state_begin();
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->set_params) {
+            continue;
+        }
+        se->ops->set_params(params, se->opaque);
+    }
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->save_live_setup) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        save_section_header(f, se, QEMU_VM_SECTION_START);
+
+        ret = se->ops->save_live_setup(f, se->opaque);
+        save_section_footer(f, se);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+            break;
+        }
+    }
+}
+
+/*
+ * this function has three return values:
+ *   negative: there was one error, and we have -errno.
+ *   0 : We haven't finished, caller have to go again
+ *   1 : We have finished, we can go to complete phase
+ */
+int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy)
+{
+    SaveStateEntry *se;
+    int ret = 1;
+
+    trace_savevm_state_iterate();
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->save_live_iterate) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        /*
+         * In the postcopy phase, any device that doesn't know how to
+         * do postcopy should have saved it's state in the _complete
+         * call that's already run, it might get confused if we call
+         * iterate afterwards.
+         */
+        if (postcopy && !se->ops->save_live_complete_postcopy) {
+            continue;
+        }
+        if (qemu_file_rate_limit(f)) {
+            return 0;
+        }
+        trace_savevm_section_start(se->idstr, se->section_id);
+
+        save_section_header(f, se, QEMU_VM_SECTION_PART);
+
+        ret = se->ops->save_live_iterate(f, se->opaque);
+        trace_savevm_section_end(se->idstr, se->section_id, ret);
+        save_section_footer(f, se);
+
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+        }
+        if (ret <= 0) {
+            /* Do not proceed to the next vmstate before this one reported
+               completion of the current stage. This serializes the migration
+               and reduces the probability that a faster changing state is
+               synchronized over and over again. */
+            break;
+        }
+    }
+    return ret;
+}
+
+static bool should_send_vmdesc(void)
+{
+    MachineState *machine = MACHINE(qdev_get_machine());
+    bool in_postcopy = migration_in_postcopy(migrate_get_current());
+    return !machine->suppress_vmdesc && !in_postcopy;
+}
+
+/*
+ * Calls the save_live_complete_postcopy methods
+ * causing the last few pages to be sent immediately and doing any associated
+ * cleanup.
+ * Note postcopy also calls qemu_savevm_state_complete_precopy to complete
+ * all the other devices, but that happens at the point we switch to postcopy.
+ */
+void qemu_savevm_state_complete_postcopy(QEMUFile *f)
+{
+    SaveStateEntry *se;
+    int ret;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->save_live_complete_postcopy) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        trace_savevm_section_start(se->idstr, se->section_id);
+        /* Section type */
+        qemu_put_byte(f, QEMU_VM_SECTION_END);
+        qemu_put_be32(f, se->section_id);
+
+        ret = se->ops->save_live_complete_postcopy(f, se->opaque);
+        trace_savevm_section_end(se->idstr, se->section_id, ret);
+        save_section_footer(f, se);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+            return;
+        }
+    }
+
+    qemu_put_byte(f, QEMU_VM_EOF);
+    qemu_fflush(f);
+}
+
+void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only)
+{
+    QJSON *vmdesc;
+    int vmdesc_len;
+    SaveStateEntry *se;
+    int ret;
+    bool in_postcopy = migration_in_postcopy(migrate_get_current());
+
+    trace_savevm_state_complete_precopy();
+
+    cpu_synchronize_all_states();
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops ||
+            (in_postcopy && se->ops->save_live_complete_postcopy) ||
+            (in_postcopy && !iterable_only) ||
+            !se->ops->save_live_complete_precopy) {
+            continue;
+        }
+
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        trace_savevm_section_start(se->idstr, se->section_id);
+
+        save_section_header(f, se, QEMU_VM_SECTION_END);
+
+        ret = se->ops->save_live_complete_precopy(f, se->opaque);
+        trace_savevm_section_end(se->idstr, se->section_id, ret);
+        save_section_footer(f, se);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+            return;
+        }
+    }
+
+    if (iterable_only) {
+        return;
+    }
+
+    vmdesc = qjson_new();
+    json_prop_int(vmdesc, "page_size", TARGET_PAGE_SIZE);
+    json_start_array(vmdesc, "devices");
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+
+        if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
+            continue;
+        }
+        if (se->vmsd && !vmstate_save_needed(se->vmsd, se->opaque)) {
+            trace_savevm_section_skip(se->idstr, se->section_id);
+            continue;
+        }
+
+        trace_savevm_section_start(se->idstr, se->section_id);
+
+        json_start_object(vmdesc, NULL);
+        json_prop_str(vmdesc, "name", se->idstr);
+        json_prop_int(vmdesc, "instance_id", se->instance_id);
+
+        save_section_header(f, se, QEMU_VM_SECTION_FULL);
+        vmstate_save(f, se, vmdesc);
+        trace_savevm_section_end(se->idstr, se->section_id, 0);
+        save_section_footer(f, se);
+
+        json_end_object(vmdesc);
+    }
+
+    if (!in_postcopy) {
+        /* Postcopy stream will still be going */
+        qemu_put_byte(f, QEMU_VM_EOF);
+    }
+
+    json_end_array(vmdesc);
+    qjson_finish(vmdesc);
+    vmdesc_len = strlen(qjson_get_str(vmdesc));
+
+    if (should_send_vmdesc()) {
+        qemu_put_byte(f, QEMU_VM_VMDESCRIPTION);
+        qemu_put_be32(f, vmdesc_len);
+        qemu_put_buffer(f, (uint8_t *)qjson_get_str(vmdesc), vmdesc_len);
+    }
+    qjson_destroy(vmdesc);
+
+    qemu_fflush(f);
+}
+
+/* Give an estimate of the amount left to be transferred,
+ * the result is split into the amount for units that can and
+ * for units that can't do postcopy.
+ */
+void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size,
+                               uint64_t *res_non_postcopiable,
+                               uint64_t *res_postcopiable)
+{
+    SaveStateEntry *se;
+
+    *res_non_postcopiable = 0;
+    *res_postcopiable = 0;
+
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!se->ops || !se->ops->save_live_pending) {
+            continue;
+        }
+        if (se->ops && se->ops->is_active) {
+            if (!se->ops->is_active(se->opaque)) {
+                continue;
+            }
+        }
+        se->ops->save_live_pending(f, se->opaque, max_size,
+                                   res_non_postcopiable, res_postcopiable);
+    }
+}
+
+void qemu_savevm_state_cleanup(void)
+{
+    SaveStateEntry *se;
+
+    trace_savevm_state_cleanup();
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (se->ops && se->ops->cleanup) {
+            se->ops->cleanup(se->opaque);
+        }
+    }
+}
+
+static int qemu_savevm_state(QEMUFile *f, Error **errp)
+{
+    int ret;
+    MigrationParams params = {
+        .blk = 0,
+        .shared = 0
+    };
+    MigrationState *ms = migrate_init(&params);
+    MigrationStatus status;
+    ms->to_dst_file = f;
+
+    if (migration_is_blocked(errp)) {
+        ret = -EINVAL;
+        goto done;
+    }
+
+    qemu_mutex_unlock_iothread();
+    qemu_savevm_state_header(f);
+    qemu_savevm_state_begin(f, &params);
+    qemu_mutex_lock_iothread();
+
+    while (qemu_file_get_error(f) == 0) {
+        if (qemu_savevm_state_iterate(f, false) > 0) {
+            break;
+        }
+    }
+
+    ret = qemu_file_get_error(f);
+    if (ret == 0) {
+        qemu_savevm_state_complete_precopy(f, false);
+        ret = qemu_file_get_error(f);
+    }
+    qemu_savevm_state_cleanup();
+    if (ret != 0) {
+        error_setg_errno(errp, -ret, "Error while writing VM state");
+    }
+
+done:
+    if (ret != 0) {
+        status = MIGRATION_STATUS_FAILED;
+    } else {
+        status = MIGRATION_STATUS_COMPLETED;
+    }
+    migrate_set_state(&ms->state, MIGRATION_STATUS_SETUP, status);
+    return ret;
+}
+
+static int qemu_save_device_state(QEMUFile *f)
+{
+    SaveStateEntry *se;
+
+    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+    qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+
+    cpu_synchronize_all_states();
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (se->is_ram) {
+            continue;
+        }
+        if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
+            continue;
+        }
+        if (se->vmsd && !vmstate_save_needed(se->vmsd, se->opaque)) {
+            continue;
+        }
+
+        save_section_header(f, se, QEMU_VM_SECTION_FULL);
+
+        vmstate_save(f, se, NULL);
+
+        save_section_footer(f, se);
+    }
+
+    qemu_put_byte(f, QEMU_VM_EOF);
+
+    return qemu_file_get_error(f);
+}
+
+static SaveStateEntry *find_se(const char *idstr, int instance_id)
+{
+    SaveStateEntry *se;
+
+    QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
+        if (!strcmp(se->idstr, idstr) &&
+            (instance_id == se->instance_id ||
+             instance_id == se->alias_id))
+            return se;
+        /* Migrating from an older version? */
+        if (strstr(se->idstr, idstr) && se->compat) {
+            if (!strcmp(se->compat->idstr, idstr) &&
+                (instance_id == se->compat->instance_id ||
+                 instance_id == se->alias_id))
+                return se;
+        }
+    }
+    return NULL;
+}
+
+enum LoadVMExitCodes {
+    /* Allow a command to quit all layers of nested loadvm loops */
+    LOADVM_QUIT     =  1,
+};
+
+static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
+
+/* ------ incoming postcopy messages ------ */
+/* 'advise' arrives before any transfers just to tell us that a postcopy
+ * *might* happen - it might be skipped if precopy transferred everything
+ * quickly.
+ */
+static int loadvm_postcopy_handle_advise(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_ADVISE);
+    uint64_t remote_hps, remote_tps;
+
+    trace_loadvm_postcopy_handle_advise();
+    if (ps != POSTCOPY_INCOMING_NONE) {
+        error_report("CMD_POSTCOPY_ADVISE in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+
+    if (!postcopy_ram_supported_by_host()) {
+        return -1;
+    }
+
+    remote_hps = qemu_get_be64(mis->from_src_file);
+    if (remote_hps != getpagesize())  {
+        /*
+         * Some combinations of mismatch are probably possible but it gets
+         * a bit more complicated.  In particular we need to place whole
+         * host pages on the dest at once, and we need to ensure that we
+         * handle dirtying to make sure we never end up sending part of
+         * a hostpage on it's own.
+         */
+        error_report("Postcopy needs matching host page sizes (s=%d d=%d)",
+                     (int)remote_hps, getpagesize());
+        return -1;
+    }
+
+    remote_tps = qemu_get_be64(mis->from_src_file);
+    if (remote_tps != (1ul << qemu_target_page_bits())) {
+        /*
+         * Again, some differences could be dealt with, but for now keep it
+         * simple.
+         */
+        error_report("Postcopy needs matching target page sizes (s=%d d=%d)",
+                     (int)remote_tps, 1 << qemu_target_page_bits());
+        return -1;
+    }
+
+    if (ram_postcopy_incoming_init(mis)) {
+        return -1;
+    }
+
+    postcopy_state_set(POSTCOPY_INCOMING_ADVISE);
+
+    return 0;
+}
+
+/* After postcopy we will be told to throw some pages away since they're
+ * dirty and will have to be demand fetched.  Must happen before CPU is
+ * started.
+ * There can be 0..many of these messages, each encoding multiple pages.
+ */
+static int loadvm_postcopy_ram_handle_discard(MigrationIncomingState *mis,
+                                              uint16_t len)
+{
+    int tmp;
+    char ramid[256];
+    PostcopyState ps = postcopy_state_get();
+
+    trace_loadvm_postcopy_ram_handle_discard();
+
+    switch (ps) {
+    case POSTCOPY_INCOMING_ADVISE:
+        /* 1st discard */
+        tmp = postcopy_ram_prepare_discard(mis);
+        if (tmp) {
+            return tmp;
+        }
+        break;
+
+    case POSTCOPY_INCOMING_DISCARD:
+        /* Expected state */
+        break;
+
+    default:
+        error_report("CMD_POSTCOPY_RAM_DISCARD in wrong postcopy state (%d)",
+                     ps);
+        return -1;
+    }
+    /* We're expecting a
+     *    Version (0)
+     *    a RAM ID string (length byte, name, 0 term)
+     *    then at least 1 16 byte chunk
+    */
+    if (len < (1 + 1 + 1 + 1 + 2 * 8)) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len);
+        return -1;
+    }
+
+    tmp = qemu_get_byte(mis->from_src_file);
+    if (tmp != postcopy_ram_discard_version) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid version (%d)", tmp);
+        return -1;
+    }
+
+    if (!qemu_get_counted_string(mis->from_src_file, ramid)) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD Failed to read RAMBlock ID");
+        return -1;
+    }
+    tmp = qemu_get_byte(mis->from_src_file);
+    if (tmp != 0) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD missing nil (%d)", tmp);
+        return -1;
+    }
+
+    len -= 3 + strlen(ramid);
+    if (len % 16) {
+        error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len);
+        return -1;
+    }
+    trace_loadvm_postcopy_ram_handle_discard_header(ramid, len);
+    while (len) {
+        uint64_t start_addr, block_length;
+        start_addr = qemu_get_be64(mis->from_src_file);
+        block_length = qemu_get_be64(mis->from_src_file);
+
+        len -= 16;
+        int ret = ram_discard_range(mis, ramid, start_addr,
+                                    block_length);
+        if (ret) {
+            return ret;
+        }
+    }
+    trace_loadvm_postcopy_ram_handle_discard_end();
+
+    return 0;
+}
+
+/*
+ * Triggered by a postcopy_listen command; this thread takes over reading
+ * the input stream, leaving the main thread free to carry on loading the rest
+ * of the device state (from RAM).
+ * (TODO:This could do with being in a postcopy file - but there again it's
+ * just another input loop, not that postcopy specific)
+ */
+static void *postcopy_ram_listen_thread(void *opaque)
+{
+    QEMUFile *f = opaque;
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    int load_res;
+
+    migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+                                   MIGRATION_STATUS_POSTCOPY_ACTIVE);
+    qemu_sem_post(&mis->listen_thread_sem);
+    trace_postcopy_ram_listen_thread_start();
+
+    /*
+     * Because we're a thread and not a coroutine we can't yield
+     * in qemu_file, and thus we must be blocking now.
+     */
+    qemu_file_set_blocking(f, true);
+    load_res = qemu_loadvm_state_main(f, mis);
+    /* And non-blocking again so we don't block in any cleanup */
+    qemu_file_set_blocking(f, false);
+
+    trace_postcopy_ram_listen_thread_exit();
+    if (load_res < 0) {
+        error_report("%s: loadvm failed: %d", __func__, load_res);
+        qemu_file_set_error(f, load_res);
+        migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                                       MIGRATION_STATUS_FAILED);
+    } else {
+        /*
+         * This looks good, but it's possible that the device loading in the
+         * main thread hasn't finished yet, and so we might not be in 'RUN'
+         * state yet; wait for the end of the main thread.
+         */
+        qemu_event_wait(&mis->main_thread_load_event);
+    }
+    postcopy_ram_incoming_cleanup(mis);
+
+    if (load_res < 0) {
+        /*
+         * If something went wrong then we have a bad state so exit;
+         * depending how far we got it might be possible at this point
+         * to leave the guest running and fire MCEs for pages that never
+         * arrived as a desperate recovery step.
+         */
+        exit(EXIT_FAILURE);
+    }
+
+    migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE,
+                                   MIGRATION_STATUS_COMPLETED);
+    /*
+     * If everything has worked fine, then the main thread has waited
+     * for us to start, and we're the last use of the mis.
+     * (If something broke then qemu will have to exit anyway since it's
+     * got a bad migration state).
+     */
+    migration_incoming_state_destroy();
+
+
+    return NULL;
+}
+
+/* After this message we must be able to immediately receive postcopy data */
+static int loadvm_postcopy_handle_listen(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_LISTENING);
+    trace_loadvm_postcopy_handle_listen();
+    if (ps != POSTCOPY_INCOMING_ADVISE && ps != POSTCOPY_INCOMING_DISCARD) {
+        error_report("CMD_POSTCOPY_LISTEN in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+    if (ps == POSTCOPY_INCOMING_ADVISE) {
+        /*
+         * A rare case, we entered listen without having to do any discards,
+         * so do the setup that's normally done at the time of the 1st discard.
+         */
+        postcopy_ram_prepare_discard(mis);
+    }
+
+    /*
+     * Sensitise RAM - can now generate requests for blocks that don't exist
+     * However, at this point the CPU shouldn't be running, and the IO
+     * shouldn't be doing anything yet so don't actually expect requests
+     */
+    if (postcopy_ram_enable_notify(mis)) {
+        return -1;
+    }
+
+    if (mis->have_listen_thread) {
+        error_report("CMD_POSTCOPY_RAM_LISTEN already has a listen thread");
+        return -1;
+    }
+
+    mis->have_listen_thread = true;
+    /* Start up the listening thread and wait for it to signal ready */
+    qemu_sem_init(&mis->listen_thread_sem, 0);
+    qemu_thread_create(&mis->listen_thread, "postcopy/listen",
+                       postcopy_ram_listen_thread, mis->from_src_file,
+                       QEMU_THREAD_DETACHED);
+    qemu_sem_wait(&mis->listen_thread_sem);
+    qemu_sem_destroy(&mis->listen_thread_sem);
+
+    return 0;
+}
+
+
+typedef struct {
+    QEMUBH *bh;
+} HandleRunBhData;
+
+static void loadvm_postcopy_handle_run_bh(void *opaque)
+{
+    Error *local_err = NULL;
+    HandleRunBhData *data = opaque;
+
+    /* TODO we should move all of this lot into postcopy_ram.c or a shared code
+     * in migration.c
+     */
+    cpu_synchronize_all_post_init();
+
+    qemu_announce_self();
+
+    /* Make sure all file formats flush their mutable metadata */
+    bdrv_invalidate_cache_all(&local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    }
+
+    trace_loadvm_postcopy_handle_run_cpu_sync();
+    cpu_synchronize_all_post_init();
+
+    trace_loadvm_postcopy_handle_run_vmstart();
+
+    if (autostart) {
+        /* Hold onto your hats, starting the CPU */
+        vm_start();
+    } else {
+        /* leave it paused and let management decide when to start the CPU */
+        runstate_set(RUN_STATE_PAUSED);
+    }
+
+    qemu_bh_delete(data->bh);
+    g_free(data);
+}
+
+/* After all discards we can start running and asking for pages */
+static int loadvm_postcopy_handle_run(MigrationIncomingState *mis)
+{
+    PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_RUNNING);
+    HandleRunBhData *data;
+
+    trace_loadvm_postcopy_handle_run();
+    if (ps != POSTCOPY_INCOMING_LISTENING) {
+        error_report("CMD_POSTCOPY_RUN in wrong postcopy state (%d)", ps);
+        return -1;
+    }
+
+    data = g_new(HandleRunBhData, 1);
+    data->bh = qemu_bh_new(loadvm_postcopy_handle_run_bh, data);
+    qemu_bh_schedule(data->bh);
+
+    /* We need to finish reading the stream from the package
+     * and also stop reading anything more from the stream that loaded the
+     * package (since it's now being read by the listener thread).
+     * LOADVM_QUIT will quit all the layers of nested loadvm loops.
+     */
+    return LOADVM_QUIT;
+}
+
+/**
+ * Immediately following this command is a blob of data containing an embedded
+ * chunk of migration stream; read it and load it.
+ *
+ * @mis: Incoming state
+ * @length: Length of packaged data to read
+ *
+ * Returns: Negative values on error
+ *
+ */
+static int loadvm_handle_cmd_packaged(MigrationIncomingState *mis)
+{
+    int ret;
+    size_t length;
+    QIOChannelBuffer *bioc;
+
+    length = qemu_get_be32(mis->from_src_file);
+    trace_loadvm_handle_cmd_packaged(length);
+
+    if (length > MAX_VM_CMD_PACKAGED_SIZE) {
+        error_report("Unreasonably large packaged state: %zu", length);
+        return -1;
+    }
+
+    bioc = qio_channel_buffer_new(length);
+    ret = qemu_get_buffer(mis->from_src_file,
+                          bioc->data,
+                          length);
+    if (ret != length) {
+        object_unref(OBJECT(bioc));
+        error_report("CMD_PACKAGED: Buffer receive fail ret=%d length=%zu",
+                     ret, length);
+        return (ret < 0) ? ret : -EAGAIN;
+    }
+    bioc->usage += length;
+    trace_loadvm_handle_cmd_packaged_received(ret);
+
+    QEMUFile *packf = qemu_fopen_channel_input(QIO_CHANNEL(bioc));
+
+    ret = qemu_loadvm_state_main(packf, mis);
+    trace_loadvm_handle_cmd_packaged_main(ret);
+    qemu_fclose(packf);
+    object_unref(OBJECT(bioc));
+
+    return ret;
+}
+
+/*
+ * Process an incoming 'QEMU_VM_COMMAND'
+ * 0           just a normal return
+ * LOADVM_QUIT All good, but exit the loop
+ * <0          Error
+ */
+static int loadvm_process_command(QEMUFile *f)
+{
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    uint16_t cmd;
+    uint16_t len;
+    uint32_t tmp32;
+
+    cmd = qemu_get_be16(f);
+    len = qemu_get_be16(f);
+
+    trace_loadvm_process_command(cmd, len);
+    if (cmd >= MIG_CMD_MAX || cmd == MIG_CMD_INVALID) {
+        error_report("MIG_CMD 0x%x unknown (len 0x%x)", cmd, len);
+        return -EINVAL;
+    }
+
+    if (mig_cmd_args[cmd].len != -1 && mig_cmd_args[cmd].len != len) {
+        error_report("%s received with bad length - expecting %zu, got %d",
+                     mig_cmd_args[cmd].name,
+                     (size_t)mig_cmd_args[cmd].len, len);
+        return -ERANGE;
+    }
+
+    switch (cmd) {
+    case MIG_CMD_OPEN_RETURN_PATH:
+        if (mis->to_src_file) {
+            error_report("CMD_OPEN_RETURN_PATH called when RP already open");
+            /* Not really a problem, so don't give up */
+            return 0;
+        }
+        mis->to_src_file = qemu_file_get_return_path(f);
+        if (!mis->to_src_file) {
+            error_report("CMD_OPEN_RETURN_PATH failed");
+            return -1;
+        }
+        break;
+
+    case MIG_CMD_PING:
+        tmp32 = qemu_get_be32(f);
+        trace_loadvm_process_command_ping(tmp32);
+        if (!mis->to_src_file) {
+            error_report("CMD_PING (0x%x) received with no return path",
+                         tmp32);
+            return -1;
+        }
+        migrate_send_rp_pong(mis, tmp32);
+        break;
+
+    case MIG_CMD_PACKAGED:
+        return loadvm_handle_cmd_packaged(mis);
+
+    case MIG_CMD_POSTCOPY_ADVISE:
+        return loadvm_postcopy_handle_advise(mis);
+
+    case MIG_CMD_POSTCOPY_LISTEN:
+        return loadvm_postcopy_handle_listen(mis);
+
+    case MIG_CMD_POSTCOPY_RUN:
+        return loadvm_postcopy_handle_run(mis);
+
+    case MIG_CMD_POSTCOPY_RAM_DISCARD:
+        return loadvm_postcopy_ram_handle_discard(mis, len);
+    }
+
+    return 0;
+}
+
+struct LoadStateEntry {
+    QLIST_ENTRY(LoadStateEntry) entry;
+    SaveStateEntry *se;
+    int section_id;
+    int version_id;
+};
+
+/*
+ * Read a footer off the wire and check that it matches the expected section
+ *
+ * Returns: true if the footer was good
+ *          false if there is a problem (and calls error_report to say why)
+ */
+static bool check_section_footer(QEMUFile *f, LoadStateEntry *le)
+{
+    uint8_t read_mark;
+    uint32_t read_section_id;
+
+    if (skip_section_footers) {
+        /* No footer to check */
+        return true;
+    }
+
+    read_mark = qemu_get_byte(f);
+
+    if (read_mark != QEMU_VM_SECTION_FOOTER) {
+        error_report("Missing section footer for %s", le->se->idstr);
+        return false;
+    }
+
+    read_section_id = qemu_get_be32(f);
+    if (read_section_id != le->section_id) {
+        error_report("Mismatched section id in footer for %s -"
+                     " read 0x%x expected 0x%x",
+                     le->se->idstr, read_section_id, le->section_id);
+        return false;
+    }
+
+    /* All good */
+    return true;
+}
+
+void loadvm_free_handlers(MigrationIncomingState *mis)
+{
+    LoadStateEntry *le, *new_le;
+
+    QLIST_FOREACH_SAFE(le, &mis->loadvm_handlers, entry, new_le) {
+        QLIST_REMOVE(le, entry);
+        g_free(le);
+    }
+}
+
+static int
+qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis)
+{
+    uint32_t instance_id, version_id, section_id;
+    SaveStateEntry *se;
+    LoadStateEntry *le;
+    char idstr[256];
+    int ret;
+
+    /* Read section start */
+    section_id = qemu_get_be32(f);
+    if (!qemu_get_counted_string(f, idstr)) {
+        error_report("Unable to read ID string for section %u",
+                     section_id);
+        return -EINVAL;
+    }
+    instance_id = qemu_get_be32(f);
+    version_id = qemu_get_be32(f);
+
+    trace_qemu_loadvm_state_section_startfull(section_id, idstr,
+            instance_id, version_id);
+    /* Find savevm section */
+    se = find_se(idstr, instance_id);
+    if (se == NULL) {
+        error_report("Unknown savevm section or instance '%s' %d",
+                     idstr, instance_id);
+        return -EINVAL;
+    }
+
+    /* Validate version */
+    if (version_id > se->version_id) {
+        error_report("savevm: unsupported version %d for '%s' v%d",
+                     version_id, idstr, se->version_id);
+        return -EINVAL;
+    }
+
+    /* Validate if it is a device's state */
+    if (xen_enabled() && se->is_ram) {
+        error_report("loadvm: %s RAM loading not allowed on Xen", idstr);
+        return -EINVAL;
+    }
+
+    /* Add entry */
+    le = g_malloc0(sizeof(*le));
+
+    le->se = se;
+    le->section_id = section_id;
+    le->version_id = version_id;
+    QLIST_INSERT_HEAD(&mis->loadvm_handlers, le, entry);
+
+    ret = vmstate_load(f, le->se, le->version_id);
+    if (ret < 0) {
+        error_report("error while loading state for instance 0x%x of"
+                     " device '%s'", instance_id, idstr);
+        return ret;
+    }
+    if (!check_section_footer(f, le)) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int
+qemu_loadvm_section_part_end(QEMUFile *f, MigrationIncomingState *mis)
+{
+    uint32_t section_id;
+    LoadStateEntry *le;
+    int ret;
+
+    section_id = qemu_get_be32(f);
+
+    trace_qemu_loadvm_state_section_partend(section_id);
+    QLIST_FOREACH(le, &mis->loadvm_handlers, entry) {
+        if (le->section_id == section_id) {
+            break;
+        }
+    }
+    if (le == NULL) {
+        error_report("Unknown savevm section %d", section_id);
+        return -EINVAL;
+    }
+
+    ret = vmstate_load(f, le->se, le->version_id);
+    if (ret < 0) {
+        error_report("error while loading state section id %d(%s)",
+                     section_id, le->se->idstr);
+        return ret;
+    }
+    if (!check_section_footer(f, le)) {
+        return -EINVAL;
+    }
+
+    return 0;
+}
+
+static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
+{
+    uint8_t section_type;
+    int ret;
+
+    while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) {
+
+        trace_qemu_loadvm_state_section(section_type);
+        switch (section_type) {
+        case QEMU_VM_SECTION_START:
+        case QEMU_VM_SECTION_FULL:
+            ret = qemu_loadvm_section_start_full(f, mis);
+            if (ret < 0) {
+                return ret;
+            }
+            break;
+        case QEMU_VM_SECTION_PART:
+        case QEMU_VM_SECTION_END:
+            ret = qemu_loadvm_section_part_end(f, mis);
+            if (ret < 0) {
+                return ret;
+            }
+            break;
+        case QEMU_VM_COMMAND:
+            ret = loadvm_process_command(f);
+            trace_qemu_loadvm_state_section_command(ret);
+            if ((ret < 0) || (ret & LOADVM_QUIT)) {
+                return ret;
+            }
+            break;
+        default:
+            error_report("Unknown savevm section type %d", section_type);
+            return -EINVAL;
+        }
+    }
+
+    return 0;
+}
+
+int qemu_loadvm_state(QEMUFile *f)
+{
+    MigrationIncomingState *mis = migration_incoming_get_current();
+    Error *local_err = NULL;
+    unsigned int v;
+    int ret;
+
+    if (qemu_savevm_state_blocked(&local_err)) {
+        error_report_err(local_err);
+        return -EINVAL;
+    }
+
+    v = qemu_get_be32(f);
+    if (v != QEMU_VM_FILE_MAGIC) {
+        error_report("Not a migration stream");
+        return -EINVAL;
+    }
+
+    v = qemu_get_be32(f);
+    if (v == QEMU_VM_FILE_VERSION_COMPAT) {
+        error_report("SaveVM v2 format is obsolete and don't work anymore");
+        return -ENOTSUP;
+    }
+    if (v != QEMU_VM_FILE_VERSION) {
+        error_report("Unsupported migration stream version");
+        return -ENOTSUP;
+    }
+
+    if (!savevm_state.skip_configuration || enforce_config_section()) {
+        if (qemu_get_byte(f) != QEMU_VM_CONFIGURATION) {
+            error_report("Configuration section missing");
+            return -EINVAL;
+        }
+        ret = vmstate_load_state(f, &vmstate_configuration, &savevm_state, 0);
+
+        if (ret) {
+            return ret;
+        }
+    }
+
+    ret = qemu_loadvm_state_main(f, mis);
+    qemu_event_set(&mis->main_thread_load_event);
+
+    trace_qemu_loadvm_state_post_main(ret);
+
+    if (mis->have_listen_thread) {
+        /* Listen thread still going, can't clean up yet */
+        return ret;
+    }
+
+    if (ret == 0) {
+        ret = qemu_file_get_error(f);
+    }
+
+    /*
+     * Try to read in the VMDESC section as well, so that dumping tools that
+     * intercept our migration stream have the chance to see it.
+     */
+
+    /* We've got to be careful; if we don't read the data and just shut the fd
+     * then the sender can error if we close while it's still sending.
+     * We also mustn't read data that isn't there; some transports (RDMA)
+     * will stall waiting for that data when the source has already closed.
+     */
+    if (ret == 0 && should_send_vmdesc()) {
+        uint8_t *buf;
+        uint32_t size;
+        uint8_t  section_type = qemu_get_byte(f);
+
+        if (section_type != QEMU_VM_VMDESCRIPTION) {
+            error_report("Expected vmdescription section, but got %d",
+                         section_type);
+            /*
+             * It doesn't seem worth failing at this point since
+             * we apparently have an otherwise valid VM state
+             */
+        } else {
+            buf = g_malloc(0x1000);
+            size = qemu_get_be32(f);
+
+            while (size > 0) {
+                uint32_t read_chunk = MIN(size, 0x1000);
+                qemu_get_buffer(f, buf, read_chunk);
+                size -= read_chunk;
+            }
+            g_free(buf);
+        }
+    }
+
+    cpu_synchronize_all_post_init();
+
+    {
+        LoadStateEntry *le;
+
+        QLIST_FOREACH(le, &mis->loadvm_handlers, entry) {
+            SaveStateEntry *se = le->se;
+            if (!se->vmsd && se->ops->post_load) {
+                se->ops->post_load(se->opaque);
+            }
+        }
+    }
+
+    return ret;
+}
+
+void hmp_savevm(Monitor *mon, const QDict *qdict)
+{
+    BlockDriverState *bs, *bs1;
+    QEMUSnapshotInfo sn1, *sn = &sn1, old_sn1, *old_sn = &old_sn1;
+    int ret;
+    QEMUFile *f;
+    int saved_vm_running;
+    uint64_t vm_state_size;
+    qemu_timeval tv;
+    struct tm tm;
+    const char *name = qdict_get_try_str(qdict, "name");
+    Error *local_err = NULL;
+    AioContext *aio_context;
+
+    if (!bdrv_all_can_snapshot(&bs)) {
+        monitor_printf(mon, "Device '%s' is writable but does not "
+                       "support snapshots.\n", bdrv_get_device_name(bs));
+        return;
+    }
+
+    /* Delete old snapshots of the same name */
+    if (name && bdrv_all_delete_snapshot(name, &bs1, &local_err) < 0) {
+        error_reportf_err(local_err,
+                          "Error while deleting snapshot on device '%s': ",
+                          bdrv_get_device_name(bs1));
+        return;
+    }
+
+    bs = bdrv_all_find_vmstate_bs();
+    if (bs == NULL) {
+        monitor_printf(mon, "No block device can accept snapshots\n");
+        return;
+    }
+    aio_context = bdrv_get_aio_context(bs);
+
+    saved_vm_running = runstate_is_running();
+
+    ret = global_state_store();
+    if (ret) {
+        monitor_printf(mon, "Error saving global state\n");
+        return;
+    }
+    vm_stop(RUN_STATE_SAVE_VM);
+
+    aio_context_acquire(aio_context);
+
+    memset(sn, 0, sizeof(*sn));
+
+    /* fill auxiliary fields */
+    qemu_gettimeofday(&tv);
+    sn->date_sec = tv.tv_sec;
+    sn->date_nsec = tv.tv_usec * 1000;
+    sn->vm_clock_nsec = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
+
+    if (name) {
+        ret = bdrv_snapshot_find(bs, old_sn, name);
+        if (ret >= 0) {
+            pstrcpy(sn->name, sizeof(sn->name), old_sn->name);
+            pstrcpy(sn->id_str, sizeof(sn->id_str), old_sn->id_str);
+        } else {
+            pstrcpy(sn->name, sizeof(sn->name), name);
+        }
+    } else {
+        /* cast below needed for OpenBSD where tv_sec is still 'long' */
+        localtime_r((const time_t *)&tv.tv_sec, &tm);
+        strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm);
+    }
+
+    /* save the VM state */
+    f = qemu_fopen_bdrv(bs, 1);
+    if (!f) {
+        monitor_printf(mon, "Could not open VM state file\n");
+        goto the_end;
+    }
+    ret = qemu_savevm_state(f, &local_err);
+    vm_state_size = qemu_ftell(f);
+    qemu_fclose(f);
+    if (ret < 0) {
+        error_report_err(local_err);
+        goto the_end;
+    }
+
+    ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, &bs);
+    if (ret < 0) {
+        monitor_printf(mon, "Error while creating snapshot on '%s'\n",
+                       bdrv_get_device_name(bs));
+    }
+
+ the_end:
+    aio_context_release(aio_context);
+    if (saved_vm_running) {
+        vm_start();
+    }
+}
+
+void qmp_xen_save_devices_state(const char *filename, Error **errp)
+{
+    QEMUFile *f;
+    QIOChannelFile *ioc;
+    int saved_vm_running;
+    int ret;
+
+    saved_vm_running = runstate_is_running();
+    vm_stop(RUN_STATE_SAVE_VM);
+    global_state_store_running();
+
+    ioc = qio_channel_file_new_path(filename, O_WRONLY | O_CREAT, 0660, errp);
+    if (!ioc) {
+        goto the_end;
+    }
+    f = qemu_fopen_channel_output(QIO_CHANNEL(ioc));
+    ret = qemu_save_device_state(f);
+    qemu_fclose(f);
+    if (ret < 0) {
+        error_setg(errp, QERR_IO_ERROR);
+    }
+
+ the_end:
+    if (saved_vm_running) {
+        vm_start();
+    }
+}
+
+void qmp_xen_load_devices_state(const char *filename, Error **errp)
+{
+    QEMUFile *f;
+    QIOChannelFile *ioc;
+    int ret;
+
+    /* Guest must be paused before loading the device state; the RAM state
+     * will already have been loaded by xc
+     */
+    if (runstate_is_running()) {
+        error_setg(errp, "Cannot update device state while vm is running");
+        return;
+    }
+    vm_stop(RUN_STATE_RESTORE_VM);
+
+    ioc = qio_channel_file_new_path(filename, O_RDONLY | O_BINARY, 0, errp);
+    if (!ioc) {
+        return;
+    }
+    f = qemu_fopen_channel_input(QIO_CHANNEL(ioc));
+
+    migration_incoming_state_new(f);
+    ret = qemu_loadvm_state(f);
+    qemu_fclose(f);
+    if (ret < 0) {
+        error_setg(errp, QERR_IO_ERROR);
+    }
+    migration_incoming_state_destroy();
+}
+
+int load_vmstate(const char *name)
+{
+    BlockDriverState *bs, *bs_vm_state;
+    QEMUSnapshotInfo sn;
+    QEMUFile *f;
+    int ret;
+    AioContext *aio_context;
+
+    if (!bdrv_all_can_snapshot(&bs)) {
+        error_report("Device '%s' is writable but does not support snapshots.",
+                     bdrv_get_device_name(bs));
+        return -ENOTSUP;
+    }
+    ret = bdrv_all_find_snapshot(name, &bs);
+    if (ret < 0) {
+        error_report("Device '%s' does not have the requested snapshot '%s'",
+                     bdrv_get_device_name(bs), name);
+        return ret;
+    }
+
+    bs_vm_state = bdrv_all_find_vmstate_bs();
+    if (!bs_vm_state) {
+        error_report("No block device supports snapshots");
+        return -ENOTSUP;
+    }
+    aio_context = bdrv_get_aio_context(bs_vm_state);
+
+    /* Don't even try to load empty VM states */
+    aio_context_acquire(aio_context);
+    ret = bdrv_snapshot_find(bs_vm_state, &sn, name);
+    aio_context_release(aio_context);
+    if (ret < 0) {
+        return ret;
+    } else if (sn.vm_state_size == 0) {
+        error_report("This is a disk-only snapshot. Revert to it offline "
+            "using qemu-img.");
+        return -EINVAL;
+    }
+
+    /* Flush all IO requests so they don't interfere with the new state.  */
+    bdrv_drain_all();
+
+    ret = bdrv_all_goto_snapshot(name, &bs);
+    if (ret < 0) {
+        error_report("Error %d while activating snapshot '%s' on '%s'",
+                     ret, name, bdrv_get_device_name(bs));
+        return ret;
+    }
+
+    /* restore the VM state */
+    f = qemu_fopen_bdrv(bs_vm_state, 0);
+    if (!f) {
+        error_report("Could not open VM state file");
+        return -EINVAL;
+    }
+
+    qemu_system_reset(VMRESET_SILENT);
+    migration_incoming_state_new(f);
+
+    aio_context_acquire(aio_context);
+    ret = qemu_loadvm_state(f);
+    qemu_fclose(f);
+    aio_context_release(aio_context);
+
+    migration_incoming_state_destroy();
+    if (ret < 0) {
+        error_report("Error %d while loading VM state", ret);
+        return ret;
+    }
+
+    return 0;
+}
+
+void hmp_delvm(Monitor *mon, const QDict *qdict)
+{
+    BlockDriverState *bs;
+    Error *err;
+    const char *name = qdict_get_str(qdict, "name");
+
+    if (bdrv_all_delete_snapshot(name, &bs, &err) < 0) {
+        error_reportf_err(err,
+                          "Error while deleting snapshot on device '%s': ",
+                          bdrv_get_device_name(bs));
+    }
+}
+
+void hmp_info_snapshots(Monitor *mon, const QDict *qdict)
+{
+    BlockDriverState *bs, *bs1;
+    BdrvNextIterator it1;
+    QEMUSnapshotInfo *sn_tab, *sn;
+    bool no_snapshot = true;
+    int nb_sns, i;
+    int total;
+    int *global_snapshots;
+    AioContext *aio_context;
+
+    typedef struct SnapshotEntry {
+        QEMUSnapshotInfo sn;
+        QTAILQ_ENTRY(SnapshotEntry) next;
+    } SnapshotEntry;
+
+    typedef struct ImageEntry {
+        const char *imagename;
+        QTAILQ_ENTRY(ImageEntry) next;
+        QTAILQ_HEAD(, SnapshotEntry) snapshots;
+    } ImageEntry;
+
+    QTAILQ_HEAD(, ImageEntry) image_list =
+        QTAILQ_HEAD_INITIALIZER(image_list);
+
+    ImageEntry *image_entry, *next_ie;
+    SnapshotEntry *snapshot_entry;
+
+    bs = bdrv_all_find_vmstate_bs();
+    if (!bs) {
+        monitor_printf(mon, "No available block device supports snapshots\n");
+        return;
+    }
+    aio_context = bdrv_get_aio_context(bs);
+
+    aio_context_acquire(aio_context);
+    nb_sns = bdrv_snapshot_list(bs, &sn_tab);
+    aio_context_release(aio_context);
+
+    if (nb_sns < 0) {
+        monitor_printf(mon, "bdrv_snapshot_list: error %d\n", nb_sns);
+        return;
+    }
+
+    for (bs1 = bdrv_first(&it1); bs1; bs1 = bdrv_next(&it1)) {
+        int bs1_nb_sns = 0;
+        ImageEntry *ie;
+        SnapshotEntry *se;
+        AioContext *ctx = bdrv_get_aio_context(bs1);
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs1)) {
+            sn = NULL;
+            bs1_nb_sns = bdrv_snapshot_list(bs1, &sn);
+            if (bs1_nb_sns > 0) {
+                no_snapshot = false;
+                ie = g_new0(ImageEntry, 1);
+                ie->imagename = bdrv_get_device_name(bs1);
+                QTAILQ_INIT(&ie->snapshots);
+                QTAILQ_INSERT_TAIL(&image_list, ie, next);
+                for (i = 0; i < bs1_nb_sns; i++) {
+                    se = g_new0(SnapshotEntry, 1);
+                    se->sn = sn[i];
+                    QTAILQ_INSERT_TAIL(&ie->snapshots, se, next);
+                }
+            }
+            g_free(sn);
+        }
+        aio_context_release(ctx);
+    }
+
+    if (no_snapshot) {
+        monitor_printf(mon, "There is no snapshot available.\n");
+        return;
+    }
+
+    global_snapshots = g_new0(int, nb_sns);
+    total = 0;
+    for (i = 0; i < nb_sns; i++) {
+        SnapshotEntry *next_sn;
+        if (bdrv_all_find_snapshot(sn_tab[i].name, &bs1) == 0) {
+            global_snapshots[total] = i;
+            total++;
+            QTAILQ_FOREACH(image_entry, &image_list, next) {
+                QTAILQ_FOREACH_SAFE(snapshot_entry, &image_entry->snapshots,
+                                    next, next_sn) {
+                    if (!strcmp(sn_tab[i].name, snapshot_entry->sn.name)) {
+                        QTAILQ_REMOVE(&image_entry->snapshots, snapshot_entry,
+                                      next);
+                        g_free(snapshot_entry);
+                    }
+                }
+            }
+        }
+    }
+
+    monitor_printf(mon, "List of snapshots present on all disks:\n");
+
+    if (total > 0) {
+        bdrv_snapshot_dump((fprintf_function)monitor_printf, mon, NULL);
+        monitor_printf(mon, "\n");
+        for (i = 0; i < total; i++) {
+            sn = &sn_tab[global_snapshots[i]];
+            /* The ID is not guaranteed to be the same on all images, so
+             * overwrite it.
+             */
+            pstrcpy(sn->id_str, sizeof(sn->id_str), "--");
+            bdrv_snapshot_dump((fprintf_function)monitor_printf, mon, sn);
+            monitor_printf(mon, "\n");
+        }
+    } else {
+        monitor_printf(mon, "None\n");
+    }
+
+    QTAILQ_FOREACH(image_entry, &image_list, next) {
+        if (QTAILQ_EMPTY(&image_entry->snapshots)) {
+            continue;
+        }
+        monitor_printf(mon,
+                       "\nList of partial (non-loadable) snapshots on '%s':\n",
+                       image_entry->imagename);
+        bdrv_snapshot_dump((fprintf_function)monitor_printf, mon, NULL);
+        monitor_printf(mon, "\n");
+        QTAILQ_FOREACH(snapshot_entry, &image_entry->snapshots, next) {
+            bdrv_snapshot_dump((fprintf_function)monitor_printf, mon,
+                               &snapshot_entry->sn);
+            monitor_printf(mon, "\n");
+        }
+    }
+
+    QTAILQ_FOREACH_SAFE(image_entry, &image_list, next, next_ie) {
+        SnapshotEntry *next_sn;
+        QTAILQ_FOREACH_SAFE(snapshot_entry, &image_entry->snapshots, next,
+                            next_sn) {
+            g_free(snapshot_entry);
+        }
+        g_free(image_entry);
+    }
+    g_free(sn_tab);
+    g_free(global_snapshots);
+
+}
+
+void vmstate_register_ram(MemoryRegion *mr, DeviceState *dev)
+{
+    qemu_ram_set_idstr(mr->ram_block,
+                       memory_region_name(mr), dev);
+}
+
+void vmstate_unregister_ram(MemoryRegion *mr, DeviceState *dev)
+{
+    qemu_ram_unset_idstr(mr->ram_block);
+}
+
+void vmstate_register_ram_global(MemoryRegion *mr)
+{
+    vmstate_register_ram(mr, NULL);
+}
diff --git a/migration/socket.c b/migration/socket.c
new file mode 100644
index 0000000..00de1fe
--- /dev/null
+++ b/migration/socket.c
@@ -0,0 +1,185 @@
+/*
+ * QEMU live migration via socket
+ *
+ * Copyright Red Hat, Inc. 2009-2016
+ *
+ * Authors:
+ *  Chris Lalancette <clalance@redhat.com>
+ *  Daniel P. Berrange <berrange@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu/osdep.h"
+
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "io/channel-socket.h"
+#include "trace.h"
+
+
+static SocketAddress *tcp_build_address(const char *host_port, Error **errp)
+{
+    InetSocketAddress *iaddr = inet_parse(host_port, errp);
+    SocketAddress *saddr;
+
+    if (!iaddr) {
+        return NULL;
+    }
+
+    saddr = g_new0(SocketAddress, 1);
+    saddr->type = SOCKET_ADDRESS_KIND_INET;
+    saddr->u.inet.data = iaddr;
+
+    return saddr;
+}
+
+
+static SocketAddress *unix_build_address(const char *path)
+{
+    SocketAddress *saddr;
+
+    saddr = g_new0(SocketAddress, 1);
+    saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+    saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    saddr->u.q_unix.data->path = g_strdup(path);
+
+    return saddr;
+}
+
+
+struct SocketConnectData {
+    MigrationState *s;
+    char *hostname;
+};
+
+static void socket_connect_data_free(void *opaque)
+{
+    struct SocketConnectData *data = opaque;
+    if (!data) {
+        return;
+    }
+    g_free(data->hostname);
+    g_free(data);
+}
+
+static void socket_outgoing_migration(Object *src,
+                                      Error *err,
+                                      gpointer opaque)
+{
+    struct SocketConnectData *data = opaque;
+    QIOChannel *sioc = QIO_CHANNEL(src);
+
+    if (err) {
+        trace_migration_socket_outgoing_error(error_get_pretty(err));
+        data->s->to_dst_file = NULL;
+        migrate_fd_error(data->s, err);
+    } else {
+        trace_migration_socket_outgoing_connected(data->hostname);
+        migration_channel_connect(data->s, sioc, data->hostname);
+    }
+    object_unref(src);
+}
+
+static void socket_start_outgoing_migration(MigrationState *s,
+                                            SocketAddress *saddr,
+                                            Error **errp)
+{
+    QIOChannelSocket *sioc = qio_channel_socket_new();
+    struct SocketConnectData *data = g_new0(struct SocketConnectData, 1);
+
+    data->s = s;
+    if (saddr->type == SOCKET_ADDRESS_KIND_INET) {
+        data->hostname = g_strdup(saddr->u.inet.data->host);
+    }
+
+    qio_channel_socket_connect_async(sioc,
+                                     saddr,
+                                     socket_outgoing_migration,
+                                     data,
+                                     socket_connect_data_free);
+    qapi_free_SocketAddress(saddr);
+}
+
+void tcp_start_outgoing_migration(MigrationState *s,
+                                  const char *host_port,
+                                  Error **errp)
+{
+    SocketAddress *saddr = tcp_build_address(host_port, errp);
+    socket_start_outgoing_migration(s, saddr, errp);
+}
+
+void unix_start_outgoing_migration(MigrationState *s,
+                                   const char *path,
+                                   Error **errp)
+{
+    SocketAddress *saddr = unix_build_address(path);
+    socket_start_outgoing_migration(s, saddr, errp);
+}
+
+
+static gboolean socket_accept_incoming_migration(QIOChannel *ioc,
+                                                 GIOCondition condition,
+                                                 gpointer opaque)
+{
+    QIOChannelSocket *sioc;
+    Error *err = NULL;
+
+    sioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc),
+                                     &err);
+    if (!sioc) {
+        error_report("could not accept migration connection (%s)",
+                     error_get_pretty(err));
+        goto out;
+    }
+
+    trace_migration_socket_incoming_accepted();
+
+    migration_channel_process_incoming(migrate_get_current(),
+                                       QIO_CHANNEL(sioc));
+    object_unref(OBJECT(sioc));
+
+out:
+    /* Close listening socket as its no longer needed */
+    qio_channel_close(ioc, NULL);
+    return FALSE; /* unregister */
+}
+
+
+static void socket_start_incoming_migration(SocketAddress *saddr,
+                                            Error **errp)
+{
+    QIOChannelSocket *listen_ioc = qio_channel_socket_new();
+
+    if (qio_channel_socket_listen_sync(listen_ioc, saddr, errp) < 0) {
+        object_unref(OBJECT(listen_ioc));
+        qapi_free_SocketAddress(saddr);
+        return;
+    }
+
+    qio_channel_add_watch(QIO_CHANNEL(listen_ioc),
+                          G_IO_IN,
+                          socket_accept_incoming_migration,
+                          listen_ioc,
+                          (GDestroyNotify)object_unref);
+    qapi_free_SocketAddress(saddr);
+}
+
+void tcp_start_incoming_migration(const char *host_port, Error **errp)
+{
+    SocketAddress *saddr = tcp_build_address(host_port, errp);
+    socket_start_incoming_migration(saddr, errp);
+}
+
+void unix_start_incoming_migration(const char *path, Error **errp)
+{
+    SocketAddress *saddr = unix_build_address(path);
+    socket_start_incoming_migration(saddr, errp);
+}
diff --git a/migration/tls.c b/migration/tls.c
new file mode 100644
index 0000000..12c053d
--- /dev/null
+++ b/migration/tls.c
@@ -0,0 +1,161 @@
+/*
+ * QEMU migration TLS support
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "migration/migration.h"
+#include "io/channel-tls.h"
+#include "crypto/tlscreds.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+#include "trace.h"
+
+static QCryptoTLSCreds *
+migration_tls_get_creds(MigrationState *s,
+                        QCryptoTLSCredsEndpoint endpoint,
+                        Error **errp)
+{
+    Object *creds;
+    QCryptoTLSCreds *ret;
+
+    creds = object_resolve_path_component(
+        object_get_objects_root(), s->parameters.tls_creds);
+    if (!creds) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   s->parameters.tls_creds);
+        return NULL;
+    }
+    ret = (QCryptoTLSCreds *)object_dynamic_cast(
+        creds, TYPE_QCRYPTO_TLS_CREDS);
+    if (!ret) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   s->parameters.tls_creds);
+        return NULL;
+    }
+    if (ret->endpoint != endpoint) {
+        error_setg(errp,
+                   "Expected TLS credentials for a %s endpoint",
+                   endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT ?
+                   "client" : "server");
+        return NULL;
+    }
+
+    object_ref(OBJECT(ret));
+    return ret;
+}
+
+
+static void migration_tls_incoming_handshake(Object *src,
+                                             Error *err,
+                                             gpointer opaque)
+{
+    QIOChannel *ioc = QIO_CHANNEL(src);
+
+    if (err) {
+        trace_migration_tls_incoming_handshake_error(error_get_pretty(err));
+        error_report("%s", error_get_pretty(err));
+    } else {
+        trace_migration_tls_incoming_handshake_complete();
+        migration_channel_process_incoming(migrate_get_current(), ioc);
+    }
+    object_unref(OBJECT(ioc));
+}
+
+void migration_tls_channel_process_incoming(MigrationState *s,
+                                            QIOChannel *ioc,
+                                            Error **errp)
+{
+    QCryptoTLSCreds *creds;
+    QIOChannelTLS *tioc;
+
+    creds = migration_tls_get_creds(
+        s, QCRYPTO_TLS_CREDS_ENDPOINT_SERVER, errp);
+    if (!creds) {
+        return;
+    }
+
+    tioc = qio_channel_tls_new_server(
+        ioc, creds,
+        NULL, /* XXX pass ACL name */
+        errp);
+    if (!tioc) {
+        return;
+    }
+
+    trace_migration_tls_incoming_handshake_start();
+    qio_channel_tls_handshake(tioc,
+                              migration_tls_incoming_handshake,
+                              NULL,
+                              NULL);
+}
+
+
+static void migration_tls_outgoing_handshake(Object *src,
+                                             Error *err,
+                                             gpointer opaque)
+{
+    MigrationState *s = opaque;
+    QIOChannel *ioc = QIO_CHANNEL(src);
+
+    if (err) {
+        trace_migration_tls_outgoing_handshake_error(error_get_pretty(err));
+        s->to_dst_file = NULL;
+        migrate_fd_error(s, err);
+    } else {
+        trace_migration_tls_outgoing_handshake_complete();
+        migration_channel_connect(s, ioc, NULL);
+    }
+    object_unref(OBJECT(ioc));
+}
+
+
+void migration_tls_channel_connect(MigrationState *s,
+                                   QIOChannel *ioc,
+                                   const char *hostname,
+                                   Error **errp)
+{
+    QCryptoTLSCreds *creds;
+    QIOChannelTLS *tioc;
+
+    creds = migration_tls_get_creds(
+        s, QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT, errp);
+    if (!creds) {
+        return;
+    }
+
+    if (s->parameters.tls_hostname) {
+        hostname = s->parameters.tls_hostname;
+    }
+    if (!hostname) {
+        error_setg(errp, "No hostname available for TLS");
+        return;
+    }
+
+    tioc = qio_channel_tls_new_client(
+        ioc, creds, hostname, errp);
+    if (!tioc) {
+        return;
+    }
+
+    trace_migration_tls_outgoing_handshake_start(hostname);
+    qio_channel_tls_handshake(tioc,
+                              migration_tls_outgoing_handshake,
+                              s,
+                              NULL);
+}
diff --git a/migration/trace-events b/migration/trace-events
new file mode 100644
index 0000000..dfee75a
--- /dev/null
+++ b/migration/trace-events
@@ -0,0 +1,209 @@
+# See docs/tracing.txt for syntax documentation.
+
+# migration/savevm.c
+qemu_loadvm_state_section(unsigned int section_type) "%d"
+qemu_loadvm_state_section_command(int ret) "%d"
+qemu_loadvm_state_section_partend(uint32_t section_id) "%u"
+qemu_loadvm_state_main(void) ""
+qemu_loadvm_state_main_quit_parent(void) ""
+qemu_loadvm_state_post_main(int ret) "%d"
+qemu_loadvm_state_section_startfull(uint32_t section_id, const char *idstr, uint32_t instance_id, uint32_t version_id) "%u(%s) %u %u"
+qemu_savevm_send_packaged(void) ""
+loadvm_handle_cmd_packaged(unsigned int length) "%u"
+loadvm_handle_cmd_packaged_main(int ret) "%d"
+loadvm_handle_cmd_packaged_received(int ret) "%d"
+loadvm_postcopy_handle_advise(void) ""
+loadvm_postcopy_handle_listen(void) ""
+loadvm_postcopy_handle_run(void) ""
+loadvm_postcopy_handle_run_cpu_sync(void) ""
+loadvm_postcopy_handle_run_vmstart(void) ""
+loadvm_postcopy_ram_handle_discard(void) ""
+loadvm_postcopy_ram_handle_discard_end(void) ""
+loadvm_postcopy_ram_handle_discard_header(const char *ramid, uint16_t len) "%s: %ud"
+loadvm_process_command(uint16_t com, uint16_t len) "com=0x%x len=%d"
+loadvm_process_command_ping(uint32_t val) "%x"
+postcopy_ram_listen_thread_exit(void) ""
+postcopy_ram_listen_thread_start(void) ""
+qemu_savevm_send_postcopy_advise(void) ""
+qemu_savevm_send_postcopy_ram_discard(const char *id, uint16_t len) "%s: %ud"
+savevm_command_send(uint16_t command, uint16_t len) "com=0x%x len=%d"
+savevm_section_start(const char *id, unsigned int section_id) "%s, section_id %u"
+savevm_section_end(const char *id, unsigned int section_id, int ret) "%s, section_id %u -> %d"
+savevm_section_skip(const char *id, unsigned int section_id) "%s, section_id %u"
+savevm_send_open_return_path(void) ""
+savevm_send_ping(uint32_t val) "%x"
+savevm_send_postcopy_listen(void) ""
+savevm_send_postcopy_run(void) ""
+savevm_state_begin(void) ""
+savevm_state_header(void) ""
+savevm_state_iterate(void) ""
+savevm_state_cleanup(void) ""
+savevm_state_complete_precopy(void) ""
+vmstate_save(const char *idstr, const char *vmsd_name) "%s, %s"
+vmstate_load(const char *idstr, const char *vmsd_name) "%s, %s"
+qemu_announce_self_iter(const char *mac) "%s"
+
+# migration/vmstate.c
+vmstate_load_field_error(const char *field, int ret) "field \"%s\" load failed, ret = %d"
+vmstate_load_state(const char *name, int version_id) "%s v%d"
+vmstate_load_state_end(const char *name, const char *reason, int val) "%s %s/%d"
+vmstate_load_state_field(const char *name, const char *field) "%s:%s"
+vmstate_n_elems(const char *name, int n_elems) "%s: %d"
+vmstate_subsection_load(const char *parent) "%s"
+vmstate_subsection_load_bad(const char *parent,  const char *sub, const char *sub2) "%s: %s/%s"
+vmstate_subsection_load_good(const char *parent) "%s"
+
+# migration/qemu-file.c
+qemu_file_fclose(void) ""
+
+# migration/ram.c
+get_queued_page(const char *block_name, uint64_t tmp_offset, uint64_t ram_addr) "%s/%" PRIx64 " ram_addr=%" PRIx64
+get_queued_page_not_dirty(const char *block_name, uint64_t tmp_offset, uint64_t ram_addr, int sent) "%s/%" PRIx64 " ram_addr=%" PRIx64 " (sent=%d)"
+migration_bitmap_sync_start(void) ""
+migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64
+migration_throttle(void) ""
+ram_load_postcopy_loop(uint64_t addr, int flags) "@%" PRIx64 " %x"
+ram_postcopy_send_discard_bitmap(void) ""
+ram_save_queue_pages(const char *rbname, size_t start, size_t len) "%s: start: %zx len: %zx"
+
+# migration/migration.c
+await_return_path_close_on_source_close(void) ""
+await_return_path_close_on_source_joining(void) ""
+migrate_set_state(int new_state) "new state %d"
+migrate_fd_cleanup(void) ""
+migrate_fd_error(const char *error_desc) "error=%s"
+migrate_fd_cancel(void) ""
+migrate_handle_rp_req_pages(const char *rbname, size_t start, size_t len) "in %s at %zx len %zx"
+migrate_pending(uint64_t size, uint64_t max, uint64_t post, uint64_t nonpost) "pending size %" PRIu64 " max %" PRIu64 " (post=%" PRIu64 " nonpost=%" PRIu64 ")"
+migrate_send_rp_message(int msg_type, uint16_t len) "%d: len %d"
+migration_completion_file_err(void) ""
+migration_completion_postcopy_end(void) ""
+migration_completion_postcopy_end_after_complete(void) ""
+migration_completion_postcopy_end_before_rp(void) ""
+migration_completion_postcopy_end_after_rp(int rp_error) "%d"
+migration_thread_after_loop(void) ""
+migration_thread_file_err(void) ""
+migration_thread_setup_complete(void) ""
+open_return_path_on_source(void) ""
+open_return_path_on_source_continue(void) ""
+postcopy_start(void) ""
+postcopy_start_set_run(void) ""
+source_return_path_thread_bad_end(void) ""
+source_return_path_thread_end(void) ""
+source_return_path_thread_entry(void) ""
+source_return_path_thread_loop_top(void) ""
+source_return_path_thread_pong(uint32_t val) "%x"
+source_return_path_thread_shut(uint32_t val) "%x"
+migrate_global_state_post_load(const char *state) "loaded state: %s"
+migrate_global_state_pre_save(const char *state) "saved state: %s"
+migration_thread_low_pending(uint64_t pending) "%" PRIu64
+migrate_state_too_big(void) ""
+migrate_transferred(uint64_t tranferred, uint64_t time_spent, double bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %g max_size %" PRId64
+process_incoming_migration_co_end(int ret, int ps) "ret=%d postcopy-state=%d"
+process_incoming_migration_co_postcopy_end_main(void) ""
+migration_set_incoming_channel(void *ioc, const char *ioctype) "ioc=%p ioctype=%s"
+migration_set_outgoing_channel(void *ioc, const char *ioctype, const char *hostname)  "ioc=%p ioctype=%s hostname=%s"
+
+# migration/rdma.c
+qemu_rdma_accept_incoming_migration(void) ""
+qemu_rdma_accept_incoming_migration_accepted(void) ""
+qemu_rdma_accept_pin_state(bool pin) "%d"
+qemu_rdma_accept_pin_verbsc(void *verbs) "Verbs context after listen: %p"
+qemu_rdma_block_for_wrid_miss(const char *wcompstr, int wcomp, const char *gcompstr, uint64_t req) "A Wanted wrid %s (%d) but got %s (%" PRIu64 ")"
+qemu_rdma_block_for_wrid_miss_b(const char *wcompstr, int wcomp, const char *gcompstr, uint64_t req) "B Wanted wrid %s (%d) but got %s (%" PRIu64 ")"
+qemu_rdma_cleanup_disconnect(void) ""
+qemu_rdma_cleanup_waiting_for_disconnect(void) ""
+qemu_rdma_close(void) ""
+qemu_rdma_connect_pin_all_requested(void) ""
+qemu_rdma_connect_pin_all_outcome(bool pin) "%d"
+qemu_rdma_dest_init_trying(const char *host, const char *ip) "%s => %s"
+qemu_rdma_dump_gid(const char *who, const char *src, const char *dst) "%s Source GID: %s, Dest GID: %s"
+qemu_rdma_exchange_get_response_start(const char *desc) "CONTROL: %s receiving..."
+qemu_rdma_exchange_get_response_none(const char *desc, int type) "Surprise: got %s (%d)"
+qemu_rdma_exchange_send_issue_callback(void) ""
+qemu_rdma_exchange_send_waiting(const char *desc) "Waiting for response %s"
+qemu_rdma_exchange_send_received(const char *desc) "Response %s received."
+qemu_rdma_fill(size_t control_len, size_t size) "RDMA %zd of %zd bytes already in buffer"
+qemu_rdma_init_ram_blocks(int blocks) "Allocated %d local ram block structures"
+qemu_rdma_poll_recv(const char *compstr, int64_t comp, int64_t id, int sent) "completion %s #%" PRId64 " received (%" PRId64 ") left %d"
+qemu_rdma_poll_write(const char *compstr, int64_t comp, int left, uint64_t block, uint64_t chunk, void *local, void *remote) "completions %s (%" PRId64 ") left %d, block %" PRIu64 ", chunk: %" PRIu64 " %p %p"
+qemu_rdma_poll_other(const char *compstr, int64_t comp, int left) "other completion %s (%" PRId64 ") received left %d"
+qemu_rdma_post_send_control(const char *desc) "CONTROL: sending %s.."
+qemu_rdma_register_and_get_keys(uint64_t len, void *start) "Registering %" PRIu64 " bytes @ %p"
+qemu_rdma_registration_handle_compress(int64_t length, int index, int64_t offset) "Zapping zero chunk: %" PRId64 " bytes, index %d, offset %" PRId64
+qemu_rdma_registration_handle_finished(void) ""
+qemu_rdma_registration_handle_ram_blocks(void) ""
+qemu_rdma_registration_handle_ram_blocks_loop(const char *name, uint64_t offset, uint64_t length, void *local_host_addr, unsigned int src_index) "%s: @%" PRIx64 "/%" PRIu64 " host:@%p src_index: %u"
+qemu_rdma_registration_handle_register(int requests) "%d requests"
+qemu_rdma_registration_handle_register_loop(int req, int index, uint64_t addr, uint64_t chunks) "Registration request (%d): index %d, current_addr %" PRIu64 " chunks: %" PRIu64
+qemu_rdma_registration_handle_register_rkey(int rkey) "%x"
+qemu_rdma_registration_handle_unregister(int requests) "%d requests"
+qemu_rdma_registration_handle_unregister_loop(int count, int index, uint64_t chunk) "Unregistration request (%d): index %d, chunk %" PRIu64
+qemu_rdma_registration_handle_unregister_success(uint64_t chunk) "%" PRIu64
+qemu_rdma_registration_handle_wait(void) ""
+qemu_rdma_registration_start(uint64_t flags) "%" PRIu64
+qemu_rdma_registration_stop(uint64_t flags) "%" PRIu64
+qemu_rdma_registration_stop_ram(void) ""
+qemu_rdma_resolve_host_trying(const char *host, const char *ip) "Trying %s => %s"
+qemu_rdma_signal_unregister_append(uint64_t chunk, int pos) "Appending unregister chunk %" PRIu64 " at position %d"
+qemu_rdma_signal_unregister_already(uint64_t chunk) "Unregister chunk %" PRIu64 " already in queue"
+qemu_rdma_unregister_waiting_inflight(uint64_t chunk) "Cannot unregister inflight chunk: %" PRIu64
+qemu_rdma_unregister_waiting_proc(uint64_t chunk, int pos) "Processing unregister for chunk: %" PRIu64 " at position %d"
+qemu_rdma_unregister_waiting_send(uint64_t chunk) "Sending unregister for chunk: %" PRIu64
+qemu_rdma_unregister_waiting_complete(uint64_t chunk) "Unregister for chunk: %" PRIu64 " complete."
+qemu_rdma_write_flush(int sent) "sent total: %d"
+qemu_rdma_write_one_block(int count, int block, uint64_t chunk, uint64_t current, uint64_t len, int nb_sent, int nb_chunks) "(%d) Not clobbering: block: %d chunk %" PRIu64 " current %" PRIu64 " len %" PRIu64 " %d %d"
+qemu_rdma_write_one_post(uint64_t chunk, long addr, long remote, uint32_t len) "Posting chunk: %" PRIu64 ", addr: %lx remote: %lx, bytes %" PRIu32
+qemu_rdma_write_one_queue_full(void) ""
+qemu_rdma_write_one_recvregres(int mykey, int theirkey, uint64_t chunk) "Received registration result: my key: %x their key %x, chunk %" PRIu64
+qemu_rdma_write_one_sendreg(uint64_t chunk, int len, int index, int64_t offset) "Sending registration request chunk %" PRIu64 " for %d bytes, index: %d, offset: %" PRId64
+qemu_rdma_write_one_top(uint64_t chunks, uint64_t size) "Writing %" PRIu64 " chunks, (%" PRIu64 " MB)"
+qemu_rdma_write_one_zero(uint64_t chunk, int len, int index, int64_t offset) "Entire chunk is zero, sending compress: %" PRIu64 " for %d bytes, index: %d, offset: %" PRId64
+rdma_add_block(const char *block_name, int block, uint64_t addr, uint64_t offset, uint64_t len, uint64_t end, uint64_t bits, int chunks) "Added Block: '%s':%d, addr: %" PRIu64 ", offset: %" PRIu64 " length: %" PRIu64 " end: %" PRIu64 " bits %" PRIu64 " chunks %d"
+rdma_block_notification_handle(const char *name, int index) "%s at %d"
+rdma_delete_block(void *block, uint64_t addr, uint64_t offset, uint64_t len, uint64_t end, uint64_t bits, int chunks) "Deleted Block: %p, addr: %" PRIu64 ", offset: %" PRIu64 " length: %" PRIu64 " end: %" PRIu64 " bits %" PRIu64 " chunks %d"
+rdma_start_incoming_migration(void) ""
+rdma_start_incoming_migration_after_dest_init(void) ""
+rdma_start_incoming_migration_after_rdma_listen(void) ""
+rdma_start_outgoing_migration_after_rdma_connect(void) ""
+rdma_start_outgoing_migration_after_rdma_source_init(void) ""
+
+# migration/postcopy-ram.c
+postcopy_discard_send_finish(const char *ramblock, int nwords, int ncmds) "%s mask words sent=%d in %d commands"
+postcopy_discard_send_range(const char *ramblock, unsigned long start, unsigned long length) "%s:%lx/%lx"
+postcopy_ram_discard_range(void *start, size_t length) "%p,+%zx"
+postcopy_cleanup_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_init_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_nhp_range(const char *ramblock, void *host_addr, size_t offset, size_t length) "%s: %p offset=%zx length=%zx"
+postcopy_place_page(void *host_addr) "host=%p"
+postcopy_place_page_zero(void *host_addr) "host=%p"
+postcopy_ram_enable_notify(void) ""
+postcopy_ram_fault_thread_entry(void) ""
+postcopy_ram_fault_thread_exit(void) ""
+postcopy_ram_fault_thread_quit(void) ""
+postcopy_ram_fault_thread_request(uint64_t hostaddr, const char *ramblock, size_t offset) "Request for HVA=%" PRIx64 " rb=%s offset=%zx"
+postcopy_ram_incoming_cleanup_closeuf(void) ""
+postcopy_ram_incoming_cleanup_entry(void) ""
+postcopy_ram_incoming_cleanup_exit(void) ""
+postcopy_ram_incoming_cleanup_join(void) ""
+
+# migration/exec.c
+migration_exec_outgoing(const char *cmd) "cmd=%s"
+migration_exec_incoming(const char *cmd) "cmd=%s"
+
+# migration/fd.c
+migration_fd_outgoing(int fd) "fd=%d"
+migration_fd_incoming(int fd) "fd=%d"
+
+# migration/socket.c
+migration_socket_incoming_accepted(void) ""
+migration_socket_outgoing_connected(const char *hostname) "hostname=%s"
+migration_socket_outgoing_error(const char *err) "error=%s"
+
+# migration/tls.c
+migration_tls_outgoing_handshake_start(const char *hostname) "hostname=%s"
+migration_tls_outgoing_handshake_error(const char *err) "err=%s"
+migration_tls_outgoing_handshake_complete(void) ""
+migration_tls_incoming_handshake_start(void) ""
+migration_tls_incoming_handshake_error(const char *err) "err=%s"
+migration_tls_incoming_handshake_complete(void) ""
diff --git a/migration/vmstate.c b/migration/vmstate.c
new file mode 100644
index 0000000..fc29acf
--- /dev/null
+++ b/migration/vmstate.c
@@ -0,0 +1,918 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "migration/migration.h"
+#include "migration/qemu-file.h"
+#include "migration/vmstate.h"
+#include "qemu/bitops.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+
+static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
+                                    void *opaque, QJSON *vmdesc);
+static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd,
+                                   void *opaque);
+
+static int vmstate_n_elems(void *opaque, VMStateField *field)
+{
+    int n_elems = 1;
+
+    if (field->flags & VMS_ARRAY) {
+        n_elems = field->num;
+    } else if (field->flags & VMS_VARRAY_INT32) {
+        n_elems = *(int32_t *)(opaque+field->num_offset);
+    } else if (field->flags & VMS_VARRAY_UINT32) {
+        n_elems = *(uint32_t *)(opaque+field->num_offset);
+    } else if (field->flags & VMS_VARRAY_UINT16) {
+        n_elems = *(uint16_t *)(opaque+field->num_offset);
+    } else if (field->flags & VMS_VARRAY_UINT8) {
+        n_elems = *(uint8_t *)(opaque+field->num_offset);
+    }
+
+    if (field->flags & VMS_MULTIPLY_ELEMENTS) {
+        n_elems *= field->num;
+    }
+
+    trace_vmstate_n_elems(field->name, n_elems);
+    return n_elems;
+}
+
+static int vmstate_size(void *opaque, VMStateField *field)
+{
+    int size = field->size;
+
+    if (field->flags & VMS_VBUFFER) {
+        size = *(int32_t *)(opaque+field->size_offset);
+        if (field->flags & VMS_MULTIPLY) {
+            size *= field->size;
+        }
+    }
+
+    return size;
+}
+
+static void *vmstate_base_addr(void *opaque, VMStateField *field, bool alloc)
+{
+    void *base_addr = opaque + field->offset;
+
+    if (field->flags & VMS_POINTER) {
+        if (alloc && (field->flags & VMS_ALLOC)) {
+            gsize size = 0;
+            if (field->flags & VMS_VBUFFER) {
+                size = vmstate_size(opaque, field);
+            } else {
+                int n_elems = vmstate_n_elems(opaque, field);
+                if (n_elems) {
+                    size = n_elems * field->size;
+                }
+            }
+            if (size) {
+                *((void **)base_addr + field->start) = g_malloc(size);
+            }
+        }
+        base_addr = *(void **)base_addr + field->start;
+    }
+
+    return base_addr;
+}
+
+int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
+                       void *opaque, int version_id)
+{
+    VMStateField *field = vmsd->fields;
+    int ret = 0;
+
+    trace_vmstate_load_state(vmsd->name, version_id);
+    if (version_id > vmsd->version_id) {
+        trace_vmstate_load_state_end(vmsd->name, "too new", -EINVAL);
+        return -EINVAL;
+    }
+    if  (version_id < vmsd->minimum_version_id) {
+        if (vmsd->load_state_old &&
+            version_id >= vmsd->minimum_version_id_old) {
+            ret = vmsd->load_state_old(f, opaque, version_id);
+            trace_vmstate_load_state_end(vmsd->name, "old path", ret);
+            return ret;
+        }
+        trace_vmstate_load_state_end(vmsd->name, "too old", -EINVAL);
+        return -EINVAL;
+    }
+    if (vmsd->pre_load) {
+        int ret = vmsd->pre_load(opaque);
+        if (ret) {
+            return ret;
+        }
+    }
+    while (field->name) {
+        trace_vmstate_load_state_field(vmsd->name, field->name);
+        if ((field->field_exists &&
+             field->field_exists(opaque, version_id)) ||
+            (!field->field_exists &&
+             field->version_id <= version_id)) {
+            void *base_addr = vmstate_base_addr(opaque, field, true);
+            int i, n_elems = vmstate_n_elems(opaque, field);
+            int size = vmstate_size(opaque, field);
+
+            for (i = 0; i < n_elems; i++) {
+                void *addr = base_addr + size * i;
+
+                if (field->flags & VMS_ARRAY_OF_POINTER) {
+                    addr = *(void **)addr;
+                }
+                if (field->flags & VMS_STRUCT) {
+                    ret = vmstate_load_state(f, field->vmsd, addr,
+                                             field->vmsd->version_id);
+                } else {
+                    ret = field->info->get(f, addr, size);
+
+                }
+                if (ret >= 0) {
+                    ret = qemu_file_get_error(f);
+                }
+                if (ret < 0) {
+                    qemu_file_set_error(f, ret);
+                    trace_vmstate_load_field_error(field->name, ret);
+                    return ret;
+                }
+            }
+        } else if (field->flags & VMS_MUST_EXIST) {
+            error_report("Input validation failed: %s/%s",
+                         vmsd->name, field->name);
+            return -1;
+        }
+        field++;
+    }
+    ret = vmstate_subsection_load(f, vmsd, opaque);
+    if (ret != 0) {
+        return ret;
+    }
+    if (vmsd->post_load) {
+        ret = vmsd->post_load(opaque, version_id);
+    }
+    trace_vmstate_load_state_end(vmsd->name, "end", ret);
+    return ret;
+}
+
+static int vmfield_name_num(VMStateField *start, VMStateField *search)
+{
+    VMStateField *field;
+    int found = 0;
+
+    for (field = start; field->name; field++) {
+        if (!strcmp(field->name, search->name)) {
+            if (field == search) {
+                return found;
+            }
+            found++;
+        }
+    }
+
+    return -1;
+}
+
+static bool vmfield_name_is_unique(VMStateField *start, VMStateField *search)
+{
+    VMStateField *field;
+    int found = 0;
+
+    for (field = start; field->name; field++) {
+        if (!strcmp(field->name, search->name)) {
+            found++;
+            /* name found more than once, so it's not unique */
+            if (found > 1) {
+                return false;
+            }
+        }
+    }
+
+    return true;
+}
+
+static const char *vmfield_get_type_name(VMStateField *field)
+{
+    const char *type = "unknown";
+
+    if (field->flags & VMS_STRUCT) {
+        type = "struct";
+    } else if (field->info->name) {
+        type = field->info->name;
+    }
+
+    return type;
+}
+
+static bool vmsd_can_compress(VMStateField *field)
+{
+    if (field->field_exists) {
+        /* Dynamically existing fields mess up compression */
+        return false;
+    }
+
+    if (field->flags & VMS_STRUCT) {
+        VMStateField *sfield = field->vmsd->fields;
+        while (sfield->name) {
+            if (!vmsd_can_compress(sfield)) {
+                /* Child elements can't compress, so can't we */
+                return false;
+            }
+            sfield++;
+        }
+
+        if (field->vmsd->subsections) {
+            /* Subsections may come and go, better don't compress */
+            return false;
+        }
+    }
+
+    return true;
+}
+
+static void vmsd_desc_field_start(const VMStateDescription *vmsd, QJSON *vmdesc,
+                                  VMStateField *field, int i, int max)
+{
+    char *name, *old_name;
+    bool is_array = max > 1;
+    bool can_compress = vmsd_can_compress(field);
+
+    if (!vmdesc) {
+        return;
+    }
+
+    name = g_strdup(field->name);
+
+    /* Field name is not unique, need to make it unique */
+    if (!vmfield_name_is_unique(vmsd->fields, field)) {
+        int num = vmfield_name_num(vmsd->fields, field);
+        old_name = name;
+        name = g_strdup_printf("%s[%d]", name, num);
+        g_free(old_name);
+    }
+
+    json_start_object(vmdesc, NULL);
+    json_prop_str(vmdesc, "name", name);
+    if (is_array) {
+        if (can_compress) {
+            json_prop_int(vmdesc, "array_len", max);
+        } else {
+            json_prop_int(vmdesc, "index", i);
+        }
+    }
+    json_prop_str(vmdesc, "type", vmfield_get_type_name(field));
+
+    if (field->flags & VMS_STRUCT) {
+        json_start_object(vmdesc, "struct");
+    }
+
+    g_free(name);
+}
+
+static void vmsd_desc_field_end(const VMStateDescription *vmsd, QJSON *vmdesc,
+                                VMStateField *field, size_t size, int i)
+{
+    if (!vmdesc) {
+        return;
+    }
+
+    if (field->flags & VMS_STRUCT) {
+        /* We printed a struct in between, close its child object */
+        json_end_object(vmdesc);
+    }
+
+    json_prop_int(vmdesc, "size", size);
+    json_end_object(vmdesc);
+}
+
+
+bool vmstate_save_needed(const VMStateDescription *vmsd, void *opaque)
+{
+    if (vmsd->needed && !vmsd->needed(opaque)) {
+        /* optional section not needed */
+        return false;
+    }
+    return true;
+}
+
+
+void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
+                        void *opaque, QJSON *vmdesc)
+{
+    VMStateField *field = vmsd->fields;
+
+    if (vmsd->pre_save) {
+        vmsd->pre_save(opaque);
+    }
+
+    if (vmdesc) {
+        json_prop_str(vmdesc, "vmsd_name", vmsd->name);
+        json_prop_int(vmdesc, "version", vmsd->version_id);
+        json_start_array(vmdesc, "fields");
+    }
+
+    while (field->name) {
+        if (!field->field_exists ||
+            field->field_exists(opaque, vmsd->version_id)) {
+            void *base_addr = vmstate_base_addr(opaque, field, false);
+            int i, n_elems = vmstate_n_elems(opaque, field);
+            int size = vmstate_size(opaque, field);
+            int64_t old_offset, written_bytes;
+            QJSON *vmdesc_loop = vmdesc;
+
+            for (i = 0; i < n_elems; i++) {
+                void *addr = base_addr + size * i;
+
+                vmsd_desc_field_start(vmsd, vmdesc_loop, field, i, n_elems);
+                old_offset = qemu_ftell_fast(f);
+
+                if (field->flags & VMS_ARRAY_OF_POINTER) {
+                    addr = *(void **)addr;
+                }
+                if (field->flags & VMS_STRUCT) {
+                    vmstate_save_state(f, field->vmsd, addr, vmdesc_loop);
+                } else {
+                    field->info->put(f, addr, size);
+                }
+
+                written_bytes = qemu_ftell_fast(f) - old_offset;
+                vmsd_desc_field_end(vmsd, vmdesc_loop, field, written_bytes, i);
+
+                /* Compressed arrays only care about the first element */
+                if (vmdesc_loop && vmsd_can_compress(field)) {
+                    vmdesc_loop = NULL;
+                }
+            }
+        } else {
+            if (field->flags & VMS_MUST_EXIST) {
+                error_report("Output state validation failed: %s/%s",
+                        vmsd->name, field->name);
+                assert(!(field->flags & VMS_MUST_EXIST));
+            }
+        }
+        field++;
+    }
+
+    if (vmdesc) {
+        json_end_array(vmdesc);
+    }
+
+    vmstate_subsection_save(f, vmsd, opaque, vmdesc);
+}
+
+static const VMStateDescription *
+vmstate_get_subsection(const VMStateDescription **sub, char *idstr)
+{
+    while (sub && *sub && (*sub)->needed) {
+        if (strcmp(idstr, (*sub)->name) == 0) {
+            return *sub;
+        }
+        sub++;
+    }
+    return NULL;
+}
+
+static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd,
+                                   void *opaque)
+{
+    trace_vmstate_subsection_load(vmsd->name);
+
+    while (qemu_peek_byte(f, 0) == QEMU_VM_SUBSECTION) {
+        char idstr[256], *idstr_ret;
+        int ret;
+        uint8_t version_id, len, size;
+        const VMStateDescription *sub_vmsd;
+
+        len = qemu_peek_byte(f, 1);
+        if (len < strlen(vmsd->name) + 1) {
+            /* subsection name has be be "section_name/a" */
+            trace_vmstate_subsection_load_bad(vmsd->name, "(short)", "");
+            return 0;
+        }
+        size = qemu_peek_buffer(f, (uint8_t **)&idstr_ret, len, 2);
+        if (size != len) {
+            trace_vmstate_subsection_load_bad(vmsd->name, "(peek fail)", "");
+            return 0;
+        }
+        memcpy(idstr, idstr_ret, size);
+        idstr[size] = 0;
+
+        if (strncmp(vmsd->name, idstr, strlen(vmsd->name)) != 0) {
+            trace_vmstate_subsection_load_bad(vmsd->name, idstr, "(prefix)");
+            /* it doesn't have a valid subsection name */
+            return 0;
+        }
+        sub_vmsd = vmstate_get_subsection(vmsd->subsections, idstr);
+        if (sub_vmsd == NULL) {
+            trace_vmstate_subsection_load_bad(vmsd->name, idstr, "(lookup)");
+            return -ENOENT;
+        }
+        qemu_file_skip(f, 1); /* subsection */
+        qemu_file_skip(f, 1); /* len */
+        qemu_file_skip(f, len); /* idstr */
+        version_id = qemu_get_be32(f);
+
+        ret = vmstate_load_state(f, sub_vmsd, opaque, version_id);
+        if (ret) {
+            trace_vmstate_subsection_load_bad(vmsd->name, idstr, "(child)");
+            return ret;
+        }
+    }
+
+    trace_vmstate_subsection_load_good(vmsd->name);
+    return 0;
+}
+
+static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
+                                    void *opaque, QJSON *vmdesc)
+{
+    const VMStateDescription **sub = vmsd->subsections;
+    bool subsection_found = false;
+
+    while (sub && *sub && (*sub)->needed) {
+        if ((*sub)->needed(opaque)) {
+            const VMStateDescription *vmsd = *sub;
+            uint8_t len;
+
+            if (vmdesc) {
+                /* Only create subsection array when we have any */
+                if (!subsection_found) {
+                    json_start_array(vmdesc, "subsections");
+                    subsection_found = true;
+                }
+
+                json_start_object(vmdesc, NULL);
+            }
+
+            qemu_put_byte(f, QEMU_VM_SUBSECTION);
+            len = strlen(vmsd->name);
+            qemu_put_byte(f, len);
+            qemu_put_buffer(f, (uint8_t *)vmsd->name, len);
+            qemu_put_be32(f, vmsd->version_id);
+            vmstate_save_state(f, vmsd, opaque, vmdesc);
+
+            if (vmdesc) {
+                json_end_object(vmdesc);
+            }
+        }
+        sub++;
+    }
+
+    if (vmdesc && subsection_found) {
+        json_end_array(vmdesc);
+    }
+}
+
+/* bool */
+
+static int get_bool(QEMUFile *f, void *pv, size_t size)
+{
+    bool *v = pv;
+    *v = qemu_get_byte(f);
+    return 0;
+}
+
+static void put_bool(QEMUFile *f, void *pv, size_t size)
+{
+    bool *v = pv;
+    qemu_put_byte(f, *v);
+}
+
+const VMStateInfo vmstate_info_bool = {
+    .name = "bool",
+    .get  = get_bool,
+    .put  = put_bool,
+};
+
+/* 8 bit int */
+
+static int get_int8(QEMUFile *f, void *pv, size_t size)
+{
+    int8_t *v = pv;
+    qemu_get_s8s(f, v);
+    return 0;
+}
+
+static void put_int8(QEMUFile *f, void *pv, size_t size)
+{
+    int8_t *v = pv;
+    qemu_put_s8s(f, v);
+}
+
+const VMStateInfo vmstate_info_int8 = {
+    .name = "int8",
+    .get  = get_int8,
+    .put  = put_int8,
+};
+
+/* 16 bit int */
+
+static int get_int16(QEMUFile *f, void *pv, size_t size)
+{
+    int16_t *v = pv;
+    qemu_get_sbe16s(f, v);
+    return 0;
+}
+
+static void put_int16(QEMUFile *f, void *pv, size_t size)
+{
+    int16_t *v = pv;
+    qemu_put_sbe16s(f, v);
+}
+
+const VMStateInfo vmstate_info_int16 = {
+    .name = "int16",
+    .get  = get_int16,
+    .put  = put_int16,
+};
+
+/* 32 bit int */
+
+static int get_int32(QEMUFile *f, void *pv, size_t size)
+{
+    int32_t *v = pv;
+    qemu_get_sbe32s(f, v);
+    return 0;
+}
+
+static void put_int32(QEMUFile *f, void *pv, size_t size)
+{
+    int32_t *v = pv;
+    qemu_put_sbe32s(f, v);
+}
+
+const VMStateInfo vmstate_info_int32 = {
+    .name = "int32",
+    .get  = get_int32,
+    .put  = put_int32,
+};
+
+/* 32 bit int. See that the received value is the same than the one
+   in the field */
+
+static int get_int32_equal(QEMUFile *f, void *pv, size_t size)
+{
+    int32_t *v = pv;
+    int32_t v2;
+    qemu_get_sbe32s(f, &v2);
+
+    if (*v == v2) {
+        return 0;
+    }
+    return -EINVAL;
+}
+
+const VMStateInfo vmstate_info_int32_equal = {
+    .name = "int32 equal",
+    .get  = get_int32_equal,
+    .put  = put_int32,
+};
+
+/* 32 bit int. Check that the received value is non-negative
+ * and less than or equal to the one in the field.
+ */
+
+static int get_int32_le(QEMUFile *f, void *pv, size_t size)
+{
+    int32_t *cur = pv;
+    int32_t loaded;
+    qemu_get_sbe32s(f, &loaded);
+
+    if (loaded >= 0 && loaded <= *cur) {
+        *cur = loaded;
+        return 0;
+    }
+    return -EINVAL;
+}
+
+const VMStateInfo vmstate_info_int32_le = {
+    .name = "int32 le",
+    .get  = get_int32_le,
+    .put  = put_int32,
+};
+
+/* 64 bit int */
+
+static int get_int64(QEMUFile *f, void *pv, size_t size)
+{
+    int64_t *v = pv;
+    qemu_get_sbe64s(f, v);
+    return 0;
+}
+
+static void put_int64(QEMUFile *f, void *pv, size_t size)
+{
+    int64_t *v = pv;
+    qemu_put_sbe64s(f, v);
+}
+
+const VMStateInfo vmstate_info_int64 = {
+    .name = "int64",
+    .get  = get_int64,
+    .put  = put_int64,
+};
+
+/* 8 bit unsigned int */
+
+static int get_uint8(QEMUFile *f, void *pv, size_t size)
+{
+    uint8_t *v = pv;
+    qemu_get_8s(f, v);
+    return 0;
+}
+
+static void put_uint8(QEMUFile *f, void *pv, size_t size)
+{
+    uint8_t *v = pv;
+    qemu_put_8s(f, v);
+}
+
+const VMStateInfo vmstate_info_uint8 = {
+    .name = "uint8",
+    .get  = get_uint8,
+    .put  = put_uint8,
+};
+
+/* 16 bit unsigned int */
+
+static int get_uint16(QEMUFile *f, void *pv, size_t size)
+{
+    uint16_t *v = pv;
+    qemu_get_be16s(f, v);
+    return 0;
+}
+
+static void put_uint16(QEMUFile *f, void *pv, size_t size)
+{
+    uint16_t *v = pv;
+    qemu_put_be16s(f, v);
+}
+
+const VMStateInfo vmstate_info_uint16 = {
+    .name = "uint16",
+    .get  = get_uint16,
+    .put  = put_uint16,
+};
+
+/* 32 bit unsigned int */
+
+static int get_uint32(QEMUFile *f, void *pv, size_t size)
+{
+    uint32_t *v = pv;
+    qemu_get_be32s(f, v);
+    return 0;
+}
+
+static void put_uint32(QEMUFile *f, void *pv, size_t size)
+{
+    uint32_t *v = pv;
+    qemu_put_be32s(f, v);
+}
+
+const VMStateInfo vmstate_info_uint32 = {
+    .name = "uint32",
+    .get  = get_uint32,
+    .put  = put_uint32,
+};
+
+/* 32 bit uint. See that the received value is the same than the one
+   in the field */
+
+static int get_uint32_equal(QEMUFile *f, void *pv, size_t size)
+{
+    uint32_t *v = pv;
+    uint32_t v2;
+    qemu_get_be32s(f, &v2);
+
+    if (*v == v2) {
+        return 0;
+    }
+    return -EINVAL;
+}
+
+const VMStateInfo vmstate_info_uint32_equal = {
+    .name = "uint32 equal",
+    .get  = get_uint32_equal,
+    .put  = put_uint32,
+};
+
+/* 64 bit unsigned int */
+
+static int get_uint64(QEMUFile *f, void *pv, size_t size)
+{
+    uint64_t *v = pv;
+    qemu_get_be64s(f, v);
+    return 0;
+}
+
+static void put_uint64(QEMUFile *f, void *pv, size_t size)
+{
+    uint64_t *v = pv;
+    qemu_put_be64s(f, v);
+}
+
+const VMStateInfo vmstate_info_uint64 = {
+    .name = "uint64",
+    .get  = get_uint64,
+    .put  = put_uint64,
+};
+
+/* 64 bit unsigned int. See that the received value is the same than the one
+   in the field */
+
+static int get_uint64_equal(QEMUFile *f, void *pv, size_t size)
+{
+    uint64_t *v = pv;
+    uint64_t v2;
+    qemu_get_be64s(f, &v2);
+
+    if (*v == v2) {
+        return 0;
+    }
+    return -EINVAL;
+}
+
+const VMStateInfo vmstate_info_uint64_equal = {
+    .name = "int64 equal",
+    .get  = get_uint64_equal,
+    .put  = put_uint64,
+};
+
+/* 8 bit int. See that the received value is the same than the one
+   in the field */
+
+static int get_uint8_equal(QEMUFile *f, void *pv, size_t size)
+{
+    uint8_t *v = pv;
+    uint8_t v2;
+    qemu_get_8s(f, &v2);
+
+    if (*v == v2) {
+        return 0;
+    }
+    return -EINVAL;
+}
+
+const VMStateInfo vmstate_info_uint8_equal = {
+    .name = "uint8 equal",
+    .get  = get_uint8_equal,
+    .put  = put_uint8,
+};
+
+/* 16 bit unsigned int int. See that the received value is the same than the one
+   in the field */
+
+static int get_uint16_equal(QEMUFile *f, void *pv, size_t size)
+{
+    uint16_t *v = pv;
+    uint16_t v2;
+    qemu_get_be16s(f, &v2);
+
+    if (*v == v2) {
+        return 0;
+    }
+    return -EINVAL;
+}
+
+const VMStateInfo vmstate_info_uint16_equal = {
+    .name = "uint16 equal",
+    .get  = get_uint16_equal,
+    .put  = put_uint16,
+};
+
+/* floating point */
+
+static int get_float64(QEMUFile *f, void *pv, size_t size)
+{
+    float64 *v = pv;
+
+    *v = make_float64(qemu_get_be64(f));
+    return 0;
+}
+
+static void put_float64(QEMUFile *f, void *pv, size_t size)
+{
+    uint64_t *v = pv;
+
+    qemu_put_be64(f, float64_val(*v));
+}
+
+const VMStateInfo vmstate_info_float64 = {
+    .name = "float64",
+    .get  = get_float64,
+    .put  = put_float64,
+};
+
+/* CPU_DoubleU type */
+
+static int get_cpudouble(QEMUFile *f, void *pv, size_t size)
+{
+    CPU_DoubleU *v = pv;
+    qemu_get_be32s(f, &v->l.upper);
+    qemu_get_be32s(f, &v->l.lower);
+    return 0;
+}
+
+static void put_cpudouble(QEMUFile *f, void *pv, size_t size)
+{
+    CPU_DoubleU *v = pv;
+    qemu_put_be32s(f, &v->l.upper);
+    qemu_put_be32s(f, &v->l.lower);
+}
+
+const VMStateInfo vmstate_info_cpudouble = {
+    .name = "CPU_Double_U",
+    .get  = get_cpudouble,
+    .put  = put_cpudouble,
+};
+
+/* uint8_t buffers */
+
+static int get_buffer(QEMUFile *f, void *pv, size_t size)
+{
+    uint8_t *v = pv;
+    qemu_get_buffer(f, v, size);
+    return 0;
+}
+
+static void put_buffer(QEMUFile *f, void *pv, size_t size)
+{
+    uint8_t *v = pv;
+    qemu_put_buffer(f, v, size);
+}
+
+const VMStateInfo vmstate_info_buffer = {
+    .name = "buffer",
+    .get  = get_buffer,
+    .put  = put_buffer,
+};
+
+/* unused buffers: space that was used for some fields that are
+   not useful anymore */
+
+static int get_unused_buffer(QEMUFile *f, void *pv, size_t size)
+{
+    uint8_t buf[1024];
+    int block_len;
+
+    while (size > 0) {
+        block_len = MIN(sizeof(buf), size);
+        size -= block_len;
+        qemu_get_buffer(f, buf, block_len);
+    }
+   return 0;
+}
+
+static void put_unused_buffer(QEMUFile *f, void *pv, size_t size)
+{
+    static const uint8_t buf[1024];
+    int block_len;
+
+    while (size > 0) {
+        block_len = MIN(sizeof(buf), size);
+        size -= block_len;
+        qemu_put_buffer(f, buf, block_len);
+    }
+}
+
+const VMStateInfo vmstate_info_unused_buffer = {
+    .name = "unused_buffer",
+    .get  = get_unused_buffer,
+    .put  = put_unused_buffer,
+};
+
+/* bitmaps (as defined by bitmap.h). Note that size here is the size
+ * of the bitmap in bits. The on-the-wire format of a bitmap is 64
+ * bit words with the bits in big endian order. The in-memory format
+ * is an array of 'unsigned long', which may be either 32 or 64 bits.
+ */
+/* This is the number of 64 bit words sent over the wire */
+#define BITS_TO_U64S(nr) DIV_ROUND_UP(nr, 64)
+static int get_bitmap(QEMUFile *f, void *pv, size_t size)
+{
+    unsigned long *bmp = pv;
+    int i, idx = 0;
+    for (i = 0; i < BITS_TO_U64S(size); i++) {
+        uint64_t w = qemu_get_be64(f);
+        bmp[idx++] = w;
+        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
+            bmp[idx++] = w >> 32;
+        }
+    }
+    return 0;
+}
+
+static void put_bitmap(QEMUFile *f, void *pv, size_t size)
+{
+    unsigned long *bmp = pv;
+    int i, idx = 0;
+    for (i = 0; i < BITS_TO_U64S(size); i++) {
+        uint64_t w = bmp[idx++];
+        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
+            w |= ((uint64_t)bmp[idx++]) << 32;
+        }
+        qemu_put_be64(f, w);
+    }
+}
+
+const VMStateInfo vmstate_info_bitmap = {
+    .name = "bitmap",
+    .get = get_bitmap,
+    .put = put_bitmap,
+};
diff --git a/migration/xbzrle.c b/migration/xbzrle.c
new file mode 100644
index 0000000..c858339
--- /dev/null
+++ b/migration/xbzrle.c
@@ -0,0 +1,176 @@
+/*
+ * Xor Based Zero Run Length Encoding
+ *
+ * Copyright 2013 Red Hat, Inc. and/or its affiliates
+ *
+ * Authors:
+ *  Orit Wasserman  <owasserm@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
+#include "include/migration/migration.h"
+
+/*
+  page = zrun nzrun
+       | zrun nzrun page
+
+  zrun = length
+
+  nzrun = length byte...
+
+  length = uleb128 encoded integer
+ */
+int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
+                         uint8_t *dst, int dlen)
+{
+    uint32_t zrun_len = 0, nzrun_len = 0;
+    int d = 0, i = 0;
+    long res;
+    uint8_t *nzrun_start = NULL;
+
+    g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) %
+               sizeof(long)));
+
+    while (i < slen) {
+        /* overflow */
+        if (d + 2 > dlen) {
+            return -1;
+        }
+
+        /* not aligned to sizeof(long) */
+        res = (slen - i) % sizeof(long);
+        while (res && old_buf[i] == new_buf[i]) {
+            zrun_len++;
+            i++;
+            res--;
+        }
+
+        /* word at a time for speed */
+        if (!res) {
+            while (i < slen &&
+                   (*(long *)(old_buf + i)) == (*(long *)(new_buf + i))) {
+                i += sizeof(long);
+                zrun_len += sizeof(long);
+            }
+
+            /* go over the rest */
+            while (i < slen && old_buf[i] == new_buf[i]) {
+                zrun_len++;
+                i++;
+            }
+        }
+
+        /* buffer unchanged */
+        if (zrun_len == slen) {
+            return 0;
+        }
+
+        /* skip last zero run */
+        if (i == slen) {
+            return d;
+        }
+
+        d += uleb128_encode_small(dst + d, zrun_len);
+
+        zrun_len = 0;
+        nzrun_start = new_buf + i;
+
+        /* overflow */
+        if (d + 2 > dlen) {
+            return -1;
+        }
+        /* not aligned to sizeof(long) */
+        res = (slen - i) % sizeof(long);
+        while (res && old_buf[i] != new_buf[i]) {
+            i++;
+            nzrun_len++;
+            res--;
+        }
+
+        /* word at a time for speed, use of 32-bit long okay */
+        if (!res) {
+            /* truncation to 32-bit long okay */
+            unsigned long mask = (unsigned long)0x0101010101010101ULL;
+            while (i < slen) {
+                unsigned long xor;
+                xor = *(unsigned long *)(old_buf + i)
+                    ^ *(unsigned long *)(new_buf + i);
+                if ((xor - mask) & ~xor & (mask << 7)) {
+                    /* found the end of an nzrun within the current long */
+                    while (old_buf[i] != new_buf[i]) {
+                        nzrun_len++;
+                        i++;
+                    }
+                    break;
+                } else {
+                    i += sizeof(long);
+                    nzrun_len += sizeof(long);
+                }
+            }
+        }
+
+        d += uleb128_encode_small(dst + d, nzrun_len);
+        /* overflow */
+        if (d + nzrun_len > dlen) {
+            return -1;
+        }
+        memcpy(dst + d, nzrun_start, nzrun_len);
+        d += nzrun_len;
+        nzrun_len = 0;
+    }
+
+    return d;
+}
+
+int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen)
+{
+    int i = 0, d = 0;
+    int ret;
+    uint32_t count = 0;
+
+    while (i < slen) {
+
+        /* zrun */
+        if ((slen - i) < 2) {
+            return -1;
+        }
+
+        ret = uleb128_decode_small(src + i, &count);
+        if (ret < 0 || (i && !count)) {
+            return -1;
+        }
+        i += ret;
+        d += count;
+
+        /* overflow */
+        if (d > dlen) {
+            return -1;
+        }
+
+        /* nzrun */
+        if ((slen - i) < 2) {
+            return -1;
+        }
+
+        ret = uleb128_decode_small(src + i, &count);
+        if (ret < 0 || !count) {
+            return -1;
+        }
+        i += ret;
+
+        /* overflow */
+        if (d + count > dlen || i + count > slen) {
+            return -1;
+        }
+
+        memcpy(dst + d, src + i, count);
+        d += count;
+        i += count;
+    }
+
+    return d;
+}
diff --git a/module-common.c b/module-common.c
index 50c6750..0a3cb8a 100644
--- a/module-common.c
+++ b/module-common.c
@@ -1,4 +1,4 @@
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu/module.h"
 
 void qemu_module_dummy(void)
diff --git a/monitor.c b/monitor.c
index 43102a4..30f5ebd 100644
--- a/monitor.c
+++ b/monitor.c
@@ -21,8 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <dirent.h>
-#include <glib.h>
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "monitor/qdev.h"
 #include "hw/usb.h"
@@ -36,11 +38,13 @@
 #include "sysemu/char.h"
 #include "ui/qemu-spice.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/numa.h"
 #include "monitor/monitor.h"
 #include "qemu/readline.h"
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "audio/audio.h"
 #include "disas/disas.h"
 #include "sysemu/balloon.h"
@@ -49,39 +53,42 @@
 #include "sysemu/kvm.h"
 #include "qemu/acl.h"
 #include "sysemu/tpm.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qfloat.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qbool.h"
-#include "qapi/qmp/qstring.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/types.h"
 #include "qapi/qmp/qjson.h"
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/json-parser.h"
-#include <qom/object_interfaces.h>
-#include "qemu/osdep.h"
+#include "qom/object_interfaces.h"
 #include "cpu.h"
 #include "trace.h"
 #include "trace/control.h"
+#include "monitor/hmp-target.h"
 #ifdef CONFIG_TRACE_SIMPLE
 #include "trace/simple.h"
 #endif
 #include "exec/memory.h"
-#include "exec/cpu_ldst.h"
+#include "exec/exec-all.h"
+#include "qemu/log.h"
 #include "qmp-commands.h"
 #include "hmp.h"
 #include "qemu/thread.h"
 #include "block/qapi.h"
 #include "qapi/qmp-event.h"
 #include "qapi-event.h"
+#include "qmp-introspect.h"
+#include "sysemu/block-backend.h"
+#include "sysemu/qtest.h"
+#include "qemu/cutils.h"
 
-/* for pic/irq_info */
+/* for hmp_info_irq/pic */
 #if defined(TARGET_SPARC)
 #include "hw/sparc/sun4m.h"
 #endif
 #include "hw/lm32/lm32_pic.h"
 
-//#define DEBUG
-//#define DEBUG_COMPLETION
+#if defined(TARGET_S390X)
+#include "hw/s390x/storage-keys.h"
+#endif
 
 /*
  * Supported types:
@@ -117,11 +124,22 @@
  *
  */
 
-typedef struct MonitorCompletionData MonitorCompletionData;
-struct MonitorCompletionData {
-    Monitor *mon;
-    void (*user_print)(Monitor *mon, const QObject *data);
-};
+typedef struct mon_cmd_t {
+    const char *name;
+    const char *args_type;
+    const char *params;
+    const char *help;
+    union {
+        void (*cmd)(Monitor *mon, const QDict *qdict);
+        void (*cmd_new)(QDict *params, QObject **ret_data, Error **errp);
+    } mhandler;
+    /* @sub_table is a list of 2nd level of commands. If it do not exist,
+     * mhandler should be used. If it exist, sub_table[?].mhandler should be
+     * used, and mhandler of 1st level plays the role of help function.
+     */
+    struct mon_cmd_t *sub_table;
+    void (*command_completion)(ReadLineState *rs, int nb_args, const char *str);
+} mon_cmd_t;
 
 /* file descriptors passed via SCM_RIGHTS */
 typedef struct mon_fd_t mon_fd_t;
@@ -149,11 +167,16 @@
     QLIST_ENTRY(MonFdset) next;
 };
 
-typedef struct MonitorControl {
+typedef struct {
     QObject *id;
     JSONMessageParser parser;
-    int command_mode;
-} MonitorControl;
+    /*
+     * When a client connects, we're in capabilities negotiation mode.
+     * When command qmp_capabilities succeeds, we go into command
+     * mode.
+     */
+    bool in_command_mode;       /* are we in command mode? */
+} MonitorQMP;
 
 /*
  * To prevent flooding clients, events can be throttled. The
@@ -161,13 +184,16 @@
  * instance.
  */
 typedef struct MonitorQAPIEventState {
-    QAPIEvent event;    /* Event being tracked */
-    int64_t rate;       /* Minimum time (in ns) between two events */
-    int64_t last;       /* QEMU_CLOCK_REALTIME value at last emission */
+    QAPIEvent event;    /* Throttling state for this event type and... */
+    QDict *data;        /* ... data, see qapi_event_throttle_equal() */
     QEMUTimer *timer;   /* Timer for handling delayed events */
-    QObject *data;      /* Event pending delayed dispatch */
+    QDict *qdict;       /* Delayed event (if any) */
 } MonitorQAPIEventState;
 
+typedef struct {
+    int64_t rate;       /* Minimum time (in ns) between two events */
+} MonitorQAPIEventConf;
+
 struct Monitor {
     CharDriverState *chr;
     int reset_seen;
@@ -183,18 +209,11 @@
     int mux_out;
 
     ReadLineState *rs;
-    MonitorControl *mc;
+    MonitorQMP qmp;
     CPUState *mon_cpu;
     BlockCompletionFunc *password_completion_cb;
     void *password_opaque;
-
-    /* Monitors created with the MONITOR_DYNAMIC_CMDS flag
-     * will index the dynamic table. This table can be modified on the
-     * fly to add new target specific commands.
-     */
-    cmd_table_t cmds;
-
-    QError *error;
+    mon_cmd_t *cmd_table;
     QLIST_HEAD(,mon_fd_t) fds;
     QLIST_ENTRY(Monitor) entry;
 };
@@ -215,26 +234,26 @@
 static const mon_cmd_t qmp_cmds[];
 
 Monitor *cur_mon;
-Monitor *default_mon;
+
+static QEMUClockType event_clock_type = QEMU_CLOCK_REALTIME;
 
 static void monitor_command_cb(void *opaque, const char *cmdline,
                                void *readline_opaque);
 
-static inline int qmp_cmd_mode(const Monitor *mon)
-{
-    return (mon->mc ? mon->mc->command_mode : 0);
-}
-
-/* Return true if in control mode, false otherwise */
-static inline int monitor_ctrl_mode(const Monitor *mon)
+/**
+ * Is @mon a QMP monitor?
+ */
+static inline bool monitor_is_qmp(const Monitor *mon)
 {
     return (mon->flags & MONITOR_USE_CONTROL);
 }
 
-/* Return non-zero iff we have a current monitor, and it is in QMP mode.  */
-int monitor_cur_is_qmp(void)
+/**
+ * Is the current monitor, if any, a QMP monitor?
+ */
+bool monitor_cur_is_qmp(void)
 {
-    return cur_mon && monitor_ctrl_mode(cur_mon);
+    return cur_mon && monitor_is_qmp(cur_mon);
 }
 
 void monitor_read_command(Monitor *mon, int show_prompt)
@@ -250,10 +269,7 @@
 int monitor_read_password(Monitor *mon, ReadLineFunc *readline_func,
                           void *opaque)
 {
-    if (monitor_ctrl_mode(mon)) {
-        qerror_report(QERR_MISSING_PARAMETER, "password");
-        return -EINVAL;
-    } else if (mon->rs) {
+    if (mon->rs) {
         readline_start(mon->rs, "Password: ", 1, readline_func, opaque);
         /* prompt is printed on return from the command handler */
         return 0;
@@ -300,7 +316,7 @@
             return;
         }
         if (rc > 0) {
-            /* partinal write */
+            /* partial write */
             QString *tmp = qstring_from_str(buf + rc);
             QDECREF(mon->outbuf);
             mon->outbuf = tmp;
@@ -347,7 +363,7 @@
     if (!mon)
         return;
 
-    if (monitor_ctrl_mode(mon)) {
+    if (monitor_is_qmp(mon)) {
         return;
     }
 
@@ -364,8 +380,7 @@
     va_end(ap);
 }
 
-static int GCC_FMT_ATTR(2, 3) monitor_fprintf(FILE *stream,
-                                              const char *fmt, ...)
+int monitor_fprintf(FILE *stream, const char *fmt, ...)
 {
     va_list ap;
     va_start(ap, fmt);
@@ -374,23 +389,6 @@
     return 0;
 }
 
-static void monitor_user_noop(Monitor *mon, const QObject *data) { }
-
-static inline int handler_is_qobject(const mon_cmd_t *cmd)
-{
-    return cmd->user_print != NULL;
-}
-
-static inline bool handler_is_async(const mon_cmd_t *cmd)
-{
-    return cmd->flags & MONITOR_CMD_ASYNC;
-}
-
-static inline int monitor_has_error(const Monitor *mon)
-{
-    return mon->error != NULL;
-}
-
 static void monitor_json_emitter(Monitor *mon, const QObject *data)
 {
     QString *json;
@@ -405,24 +403,25 @@
     QDECREF(json);
 }
 
-static QDict *build_qmp_error_dict(const QError *err)
+static QDict *build_qmp_error_dict(Error *err)
 {
     QObject *obj;
 
-    obj = qobject_from_jsonf("{ 'error': { 'class': %s, 'desc': %p } }",
-                             ErrorClass_lookup[err->err_class],
-                             qerror_human(err));
+    obj = qobject_from_jsonf("{ 'error': { 'class': %s, 'desc': %s } }",
+                             QapiErrorClass_lookup[error_get_class(err)],
+                             error_get_pretty(err));
 
     return qobject_to_qdict(obj);
 }
 
-static void monitor_protocol_emitter(Monitor *mon, QObject *data)
+static void monitor_protocol_emitter(Monitor *mon, QObject *data,
+                                     Error *err)
 {
     QDict *qmp;
 
     trace_monitor_protocol_emitter(mon);
 
-    if (!monitor_has_error(mon)) {
+    if (!err) {
         /* success response */
         qmp = qdict_new();
         if (data) {
@@ -434,14 +433,12 @@
         }
     } else {
         /* error response */
-        qmp = build_qmp_error_dict(mon->error);
-        QDECREF(mon->error);
-        mon->error = NULL;
+        qmp = build_qmp_error_dict(err);
     }
 
-    if (mon->mc->id) {
-        qdict_put_obj(qmp, "id", mon->mc->id);
-        mon->mc->id = NULL;
+    if (mon->qmp.id) {
+        qdict_put_obj(qmp, "id", mon->qmp.id);
+        mon->qmp.id = NULL;
     }
 
     monitor_json_emitter(mon, QOBJECT(qmp));
@@ -449,147 +446,183 @@
 }
 
 
-static MonitorQAPIEventState monitor_qapi_event_state[QAPI_EVENT_MAX];
+static MonitorQAPIEventConf monitor_qapi_event_conf[QAPI_EVENT__MAX] = {
+    /* Limit guest-triggerable events to 1 per second */
+    [QAPI_EVENT_RTC_CHANGE]        = { 1000 * SCALE_MS },
+    [QAPI_EVENT_WATCHDOG]          = { 1000 * SCALE_MS },
+    [QAPI_EVENT_BALLOON_CHANGE]    = { 1000 * SCALE_MS },
+    [QAPI_EVENT_QUORUM_REPORT_BAD] = { 1000 * SCALE_MS },
+    [QAPI_EVENT_QUORUM_FAILURE]    = { 1000 * SCALE_MS },
+    [QAPI_EVENT_VSERPORT_CHANGE]   = { 1000 * SCALE_MS },
+};
+
+GHashTable *monitor_qapi_event_state;
 
 /*
  * Emits the event to every monitor instance, @event is only used for trace
  * Called with monitor_lock held.
  */
-static void monitor_qapi_event_emit(QAPIEvent event, QObject *data)
+static void monitor_qapi_event_emit(QAPIEvent event, QDict *qdict)
 {
     Monitor *mon;
 
-    trace_monitor_protocol_event_emit(event, data);
+    trace_monitor_protocol_event_emit(event, qdict);
     QLIST_FOREACH(mon, &mon_list, entry) {
-        if (monitor_ctrl_mode(mon) && qmp_cmd_mode(mon)) {
-            monitor_json_emitter(mon, data);
+        if (monitor_is_qmp(mon) && mon->qmp.in_command_mode) {
+            monitor_json_emitter(mon, QOBJECT(qdict));
         }
     }
 }
 
+static void monitor_qapi_event_handler(void *opaque);
+
 /*
  * Queue a new event for emission to Monitor instances,
  * applying any rate limiting if required.
  */
 static void
-monitor_qapi_event_queue(QAPIEvent event, QDict *data, Error **errp)
+monitor_qapi_event_queue(QAPIEvent event, QDict *qdict, Error **errp)
 {
+    MonitorQAPIEventConf *evconf;
     MonitorQAPIEventState *evstate;
-    assert(event < QAPI_EVENT_MAX);
-    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
 
-    evstate = &(monitor_qapi_event_state[event]);
-    trace_monitor_protocol_event_queue(event,
-                                       data,
-                                       evstate->rate,
-                                       evstate->last,
-                                       now);
+    assert(event < QAPI_EVENT__MAX);
+    evconf = &monitor_qapi_event_conf[event];
+    trace_monitor_protocol_event_queue(event, qdict, evconf->rate);
 
-    /* Rate limit of 0 indicates no throttling */
     qemu_mutex_lock(&monitor_lock);
-    if (!evstate->rate) {
-        monitor_qapi_event_emit(event, QOBJECT(data));
-        evstate->last = now;
+
+    if (!evconf->rate) {
+        /* Unthrottled event */
+        monitor_qapi_event_emit(event, qdict);
     } else {
-        int64_t delta = now - evstate->last;
-        if (evstate->data ||
-            delta < evstate->rate) {
-            /* If there's an existing event pending, replace
-             * it with the new event, otherwise schedule a
-             * timer for delayed emission
+        QDict *data = qobject_to_qdict(qdict_get(qdict, "data"));
+        MonitorQAPIEventState key = { .event = event, .data = data };
+
+        evstate = g_hash_table_lookup(monitor_qapi_event_state, &key);
+        assert(!evstate || timer_pending(evstate->timer));
+
+        if (evstate) {
+            /*
+             * Timer is pending for (at least) evconf->rate ns after
+             * last send.  Store event for sending when timer fires,
+             * replacing a prior stored event if any.
              */
-            if (evstate->data) {
-                qobject_decref(evstate->data);
-            } else {
-                int64_t then = evstate->last + evstate->rate;
-                timer_mod_ns(evstate->timer, then);
-            }
-            evstate->data = QOBJECT(data);
-            qobject_incref(evstate->data);
+            QDECREF(evstate->qdict);
+            evstate->qdict = qdict;
+            QINCREF(evstate->qdict);
         } else {
-            monitor_qapi_event_emit(event, QOBJECT(data));
-            evstate->last = now;
+            /*
+             * Last send was (at least) evconf->rate ns ago.
+             * Send immediately, and arm the timer to call
+             * monitor_qapi_event_handler() in evconf->rate ns.  Any
+             * events arriving before then will be delayed until then.
+             */
+            int64_t now = qemu_clock_get_ns(event_clock_type);
+
+            monitor_qapi_event_emit(event, qdict);
+
+            evstate = g_new(MonitorQAPIEventState, 1);
+            evstate->event = event;
+            evstate->data = data;
+            QINCREF(evstate->data);
+            evstate->qdict = NULL;
+            evstate->timer = timer_new_ns(event_clock_type,
+                                          monitor_qapi_event_handler,
+                                          evstate);
+            g_hash_table_add(monitor_qapi_event_state, evstate);
+            timer_mod_ns(evstate->timer, now + evconf->rate);
         }
     }
+
     qemu_mutex_unlock(&monitor_lock);
 }
 
 /*
- * The callback invoked by QemuTimer when a delayed
- * event is ready to be emitted
+ * This function runs evconf->rate ns after sending a throttled
+ * event.
+ * If another event has since been stored, send it.
  */
 static void monitor_qapi_event_handler(void *opaque)
 {
     MonitorQAPIEventState *evstate = opaque;
-    int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+    MonitorQAPIEventConf *evconf = &monitor_qapi_event_conf[evstate->event];
 
-    trace_monitor_protocol_event_handler(evstate->event,
-                                         evstate->data,
-                                         evstate->last,
-                                         now);
+    trace_monitor_protocol_event_handler(evstate->event, evstate->qdict);
     qemu_mutex_lock(&monitor_lock);
-    if (evstate->data) {
-        monitor_qapi_event_emit(evstate->event, evstate->data);
-        qobject_decref(evstate->data);
-        evstate->data = NULL;
+
+    if (evstate->qdict) {
+        int64_t now = qemu_clock_get_ns(event_clock_type);
+
+        monitor_qapi_event_emit(evstate->event, evstate->qdict);
+        QDECREF(evstate->qdict);
+        evstate->qdict = NULL;
+        timer_mod_ns(evstate->timer, now + evconf->rate);
+    } else {
+        g_hash_table_remove(monitor_qapi_event_state, evstate);
+        QDECREF(evstate->data);
+        timer_free(evstate->timer);
+        g_free(evstate);
     }
-    evstate->last = now;
+
     qemu_mutex_unlock(&monitor_lock);
 }
 
-/*
- * @event: the event ID to be limited
- * @rate: the rate limit in milliseconds
- *
- * Sets a rate limit on a particular event, so no
- * more than 1 event will be emitted within @rate
- * milliseconds
- */
-static void
-monitor_qapi_event_throttle(QAPIEvent event, int64_t rate)
+static unsigned int qapi_event_throttle_hash(const void *key)
 {
-    MonitorQAPIEventState *evstate;
-    assert(event < QAPI_EVENT_MAX);
+    const MonitorQAPIEventState *evstate = key;
+    unsigned int hash = evstate->event * 255;
 
-    evstate = &(monitor_qapi_event_state[event]);
+    if (evstate->event == QAPI_EVENT_VSERPORT_CHANGE) {
+        hash += g_str_hash(qdict_get_str(evstate->data, "id"));
+    }
 
-    trace_monitor_protocol_event_throttle(event, rate);
-    evstate->event = event;
-    assert(rate * SCALE_MS <= INT64_MAX);
-    evstate->rate = rate * SCALE_MS;
-    evstate->last = 0;
-    evstate->data = NULL;
-    evstate->timer = timer_new(QEMU_CLOCK_REALTIME,
-                               SCALE_MS,
-                               monitor_qapi_event_handler,
-                               evstate);
+    if (evstate->event == QAPI_EVENT_QUORUM_REPORT_BAD) {
+        hash += g_str_hash(qdict_get_str(evstate->data, "node-name"));
+    }
+
+    return hash;
+}
+
+static gboolean qapi_event_throttle_equal(const void *a, const void *b)
+{
+    const MonitorQAPIEventState *eva = a;
+    const MonitorQAPIEventState *evb = b;
+
+    if (eva->event != evb->event) {
+        return FALSE;
+    }
+
+    if (eva->event == QAPI_EVENT_VSERPORT_CHANGE) {
+        return !strcmp(qdict_get_str(eva->data, "id"),
+                       qdict_get_str(evb->data, "id"));
+    }
+
+    if (eva->event == QAPI_EVENT_QUORUM_REPORT_BAD) {
+        return !strcmp(qdict_get_str(eva->data, "node-name"),
+                       qdict_get_str(evb->data, "node-name"));
+    }
+
+    return TRUE;
 }
 
 static void monitor_qapi_event_init(void)
 {
-    /* Limit guest-triggerable events to 1 per second */
-    monitor_qapi_event_throttle(QAPI_EVENT_RTC_CHANGE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_WATCHDOG, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_BALLOON_CHANGE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_QUORUM_REPORT_BAD, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_QUORUM_FAILURE, 1000);
-    monitor_qapi_event_throttle(QAPI_EVENT_VSERPORT_CHANGE, 1000);
+    if (qtest_enabled()) {
+        event_clock_type = QEMU_CLOCK_VIRTUAL;
+    }
 
+    monitor_qapi_event_state = g_hash_table_new(qapi_event_throttle_hash,
+                                                qapi_event_throttle_equal);
     qmp_event_set_func_emit(monitor_qapi_event_queue);
 }
 
-static int do_qmp_capabilities(Monitor *mon, const QDict *params,
-                               QObject **ret_data)
+static void qmp_capabilities(QDict *params, QObject **ret_data, Error **errp)
 {
-    /* Will setup QMP capabilities in the future */
-    if (monitor_ctrl_mode(mon)) {
-        mon->mc->command_mode = 1;
-    }
-
-    return 0;
+    cur_mon->qmp.in_command_mode = true;
 }
 
-static void handle_user_command(Monitor *mon, const char *cmdline);
+static void handle_hmp_command(Monitor *mon, const char *cmdline);
 
 static void monitor_data_init(Monitor *mon)
 {
@@ -597,11 +630,18 @@
     qemu_mutex_init(&mon->out_lock);
     mon->outbuf = qstring_new();
     /* Use *mon_cmds by default. */
-    mon->cmds.static_table = mon_cmds;
+    mon->cmd_table = mon_cmds;
 }
 
 static void monitor_data_destroy(Monitor *mon)
 {
+    if (mon->chr) {
+        qemu_chr_add_handlers(mon->chr, NULL, NULL, NULL, NULL);
+    }
+    if (monitor_is_qmp(mon)) {
+        json_message_parser_destroy(&mon->qmp.parser);
+    }
+    g_free(mon->rs);
     QDECREF(mon->outbuf);
     qemu_mutex_destroy(&mon->out_lock);
 }
@@ -622,13 +662,13 @@
         int ret = monitor_set_cpu(cpu_index);
         if (ret < 0) {
             cur_mon = old_mon;
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, "cpu-index",
-                      "a CPU number");
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "cpu-index",
+                       "a CPU number");
             goto out;
         }
     }
 
-    handle_user_command(&hmp, command_line);
+    handle_hmp_command(&hmp, command_line);
     cur_mon = old_mon;
 
     qemu_mutex_lock(&hmp.out_lock);
@@ -699,7 +739,7 @@
                 case '\"':
                     break;
                 default:
-                    qemu_printf("unsupported escape code: '\\%c'\n", c);
+                    printf("unsupported escape code: '\\%c'\n", c);
                     goto fail;
                 }
                 if ((q - buf) < buf_size - 1) {
@@ -713,7 +753,7 @@
             }
         }
         if (*p != '\"') {
-            qemu_printf("unterminated string\n");
+            printf("unterminated string\n");
             goto fail;
         }
         p++;
@@ -804,12 +844,9 @@
     monitor_printf(mon, "%s %s -- %s\n", cmd->name, cmd->params, cmd->help);
 }
 
-static void help_cmd_dump_dynamic(Monitor *mon, GArray *cmd_array,
-                                  char **args, int nb_args, int arg_index);
-
 /* @args[@arg_index] is the valid command need to find in @cmds */
-static void help_cmd_dump_static(Monitor *mon, const mon_cmd_t *cmds,
-                                 char **args, int nb_args, int arg_index)
+static void help_cmd_dump(Monitor *mon, const mon_cmd_t *cmds,
+                          char **args, int nb_args, int arg_index)
 {
     const mon_cmd_t *cmd;
 
@@ -824,40 +861,18 @@
     /* Find one entry to dump */
     for (cmd = cmds; cmd->name != NULL; cmd++) {
         if (compare_cmd(args[arg_index], cmd->name)) {
-            if (mon->flags & MONITOR_DYNAMIC_CMDS) {
-                if (cmd->sub_cmds.dynamic_table) {
-                    /* continue with next arg */
-                    help_cmd_dump_dynamic(mon, cmd->sub_cmds.dynamic_table,
-                                          args, nb_args, arg_index + 1);
-
-                } else {
-                    help_cmd_dump_one(mon, cmd, args, arg_index);
-                }
+            if (cmd->sub_table) {
+                /* continue with next arg */
+                help_cmd_dump(mon, cmd->sub_table,
+                              args, nb_args, arg_index + 1);
             } else {
-                if (cmd->sub_cmds.static_table) {
-                    /* continue with next arg */
-                    help_cmd_dump_static(mon, cmd->sub_cmds.static_table,
-                                         args, nb_args, arg_index + 1);
-                } else {
-                    help_cmd_dump_one(mon, cmd, args, arg_index);
-                }
+                help_cmd_dump_one(mon, cmd, args, arg_index);
             }
             break;
         }
     }
 }
 
-/* @args[@arg_index] is the valid command need to find in @cmds */
-static void help_cmd_dump_dynamic(Monitor *mon, GArray *cmd_array,
-                                  char **args, int nb_args, int arg_index)
-{
-    mon_cmd_t *cmds = (mon_cmd_t *) cmd_array->data;
-
-    g_assert(mon->flags & MONITOR_DYNAMIC_CMDS);
-
-    help_cmd_dump_static(mon, cmds, args, nb_args, arg_index);
-}
-
 static void help_cmd(Monitor *mon, const char *name)
 {
     char *args[MAX_ARGS];
@@ -882,11 +897,8 @@
     }
 
     /* 2. dump the contents according to parsed args */
-    if (mon->flags & MONITOR_DYNAMIC_CMDS) {
-        help_cmd_dump_dynamic(mon, mon->cmds.dynamic_table, args, nb_args, 0);
-    } else {
-        help_cmd_dump_static(mon, mon->cmds.static_table, args, nb_args, 0);
-    }
+    help_cmd_dump(mon, mon->cmd_table, args, nb_args, 0);
+
     free_cmdline_args(args, nb_args);
 }
 
@@ -895,30 +907,27 @@
     help_cmd(mon, qdict_get_try_str(qdict, "name"));
 }
 
-static mon_cmd_t * get_command_table(Monitor *mon, cmd_table_t cmds)
-{
-    if (mon->flags & MONITOR_DYNAMIC_CMDS) {
-        return (mon_cmd_t *) (cmds.dynamic_table?cmds.dynamic_table->data:NULL);
-    } else {
-        return cmds.static_table;
-    }
-}
-
-static void do_trace_event_set_state(Monitor *mon, const QDict *qdict)
+static void hmp_trace_event(Monitor *mon, const QDict *qdict)
 {
     const char *tp_name = qdict_get_str(qdict, "name");
     bool new_state = qdict_get_bool(qdict, "option");
+    bool has_vcpu = qdict_haskey(qdict, "vcpu");
+    int vcpu = qdict_get_try_int(qdict, "vcpu", 0);
     Error *local_err = NULL;
 
-    qmp_trace_event_set_state(tp_name, new_state, true, true, &local_err);
+    if (vcpu < 0) {
+        monitor_printf(mon, "argument vcpu must be positive");
+        return;
+    }
+
+    qmp_trace_event_set_state(tp_name, new_state, true, true, has_vcpu, vcpu, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
     }
 }
 
 #ifdef CONFIG_TRACE_SIMPLE
-static void do_trace_file(Monitor *mon, const QDict *qdict)
+static void hmp_trace_file(Monitor *mon, const QDict *qdict)
 {
     const char *op = qdict_get_try_str(qdict, "op");
     const char *arg = qdict_get_try_str(qdict, "arg");
@@ -942,46 +951,7 @@
 }
 #endif
 
-static void user_monitor_complete(void *opaque, QObject *ret_data)
-{
-    MonitorCompletionData *data = (MonitorCompletionData *)opaque; 
-
-    if (ret_data) {
-        data->user_print(data->mon, ret_data);
-    }
-    monitor_resume(data->mon);
-    g_free(data);
-}
-
-static void qmp_monitor_complete(void *opaque, QObject *ret_data)
-{
-    monitor_protocol_emitter(opaque, ret_data);
-}
-
-static int qmp_async_cmd_handler(Monitor *mon, const mon_cmd_t *cmd,
-                                 const QDict *params)
-{
-    return cmd->mhandler.cmd_async(mon, params, qmp_monitor_complete, mon);
-}
-
-static void user_async_cmd_handler(Monitor *mon, const mon_cmd_t *cmd,
-                                   const QDict *params)
-{
-    int ret;
-
-    MonitorCompletionData *cb_data = g_malloc(sizeof(*cb_data));
-    cb_data->mon = mon;
-    cb_data->user_print = cmd->user_print;
-    monitor_suspend(mon);
-    ret = cmd->mhandler.cmd_async(mon, params,
-                                  user_monitor_complete, cb_data);
-    if (ret < 0) {
-        monitor_resume(mon);
-        g_free(cb_data);
-    }
-}
-
-static void do_info_help(Monitor *mon, const QDict *qdict)
+static void hmp_info_help(Monitor *mon, const QDict *qdict)
 {
     help_cmd(mon, "info");
 }
@@ -1008,7 +978,7 @@
     EventInfoList *info, *ev_list = NULL;
     QAPIEvent e;
 
-    for (e = 0 ; e < QAPI_EVENT_MAX ; e++) {
+    for (e = 0 ; e < QAPI_EVENT__MAX ; e++) {
         const char *event_name = QAPIEvent_lookup[e];
         assert(event_name != NULL);
         info = g_malloc0(sizeof(*info));
@@ -1022,6 +992,21 @@
     return ev_list;
 }
 
+/*
+ * Minor hack: generated marshalling suppressed for this command
+ * ('gen': false in the schema) so we can parse the JSON string
+ * directly into QObject instead of first parsing it with
+ * visit_type_SchemaInfoList() into a SchemaInfoList, then marshal it
+ * to QObject with generated output marshallers, every time.  Instead,
+ * we do it in test-qmp-input-visitor.c, just to make sure
+ * qapi-introspect.py's output actually conforms to the schema.
+ */
+static void qmp_query_qmp_schema(QDict *qdict, QObject **ret_data,
+                                 Error **errp)
+{
+    *ret_data = qobject_from_json(qmp_schema_json);
+}
+
 /* set the current CPU defined by the user */
 int monitor_set_cpu(int cpu_index)
 {
@@ -1035,37 +1020,42 @@
     return 0;
 }
 
-static CPUArchState *mon_get_cpu(void)
+CPUState *mon_get_cpu(void)
 {
     if (!cur_mon->mon_cpu) {
         monitor_set_cpu(0);
     }
     cpu_synchronize_state(cur_mon->mon_cpu);
-    return cur_mon->mon_cpu->env_ptr;
+    return cur_mon->mon_cpu;
+}
+
+CPUArchState *mon_get_cpu_env(void)
+{
+    return mon_get_cpu()->env_ptr;
 }
 
 int monitor_get_cpu_index(void)
 {
-    CPUState *cpu = ENV_GET_CPU(mon_get_cpu());
-    return cpu->cpu_index;
+    return mon_get_cpu()->cpu_index;
 }
 
-static void do_info_registers(Monitor *mon, const QDict *qdict)
+static void hmp_info_registers(Monitor *mon, const QDict *qdict)
 {
-    CPUState *cpu;
-    CPUArchState *env;
-    env = mon_get_cpu();
-    cpu = ENV_GET_CPU(env);
-    cpu_dump_state(cpu, (FILE *)mon, monitor_fprintf, CPU_DUMP_FPU);
+    cpu_dump_state(mon_get_cpu(), (FILE *)mon, monitor_fprintf, CPU_DUMP_FPU);
 }
 
-static void do_info_jit(Monitor *mon, const QDict *qdict)
+static void hmp_info_jit(Monitor *mon, const QDict *qdict)
 {
     dump_exec_info((FILE *)mon, monitor_fprintf);
     dump_drift_info((FILE *)mon, monitor_fprintf);
 }
 
-static void do_info_history(Monitor *mon, const QDict *qdict)
+static void hmp_info_opcount(Monitor *mon, const QDict *qdict)
+{
+    dump_opcount_info((FILE *)mon, monitor_fprintf);
+}
+
+static void hmp_info_history(Monitor *mon, const QDict *qdict)
 {
     int i;
     const char *str;
@@ -1082,20 +1072,33 @@
     }
 }
 
-static void do_info_cpu_stats(Monitor *mon, const QDict *qdict)
+static void hmp_info_cpustats(Monitor *mon, const QDict *qdict)
 {
-    CPUState *cpu;
-    CPUArchState *env;
-
-    env = mon_get_cpu();
-    cpu = ENV_GET_CPU(env);
-    cpu_dump_statistics(cpu, (FILE *)mon, &monitor_fprintf, 0);
+    cpu_dump_statistics(mon_get_cpu(), (FILE *)mon, &monitor_fprintf, 0);
 }
 
-static void do_trace_print_events(Monitor *mon, const QDict *qdict)
+static void hmp_info_trace_events(Monitor *mon, const QDict *qdict)
 {
-    TraceEventInfoList *events = qmp_trace_event_get_state("*", NULL);
+    const char *name = qdict_get_try_str(qdict, "name");
+    bool has_vcpu = qdict_haskey(qdict, "vcpu");
+    int vcpu = qdict_get_try_int(qdict, "vcpu", 0);
+    TraceEventInfoList *events;
     TraceEventInfoList *elem;
+    Error *local_err = NULL;
+
+    if (name == NULL) {
+        name = "*";
+    }
+    if (vcpu < 0) {
+        monitor_printf(mon, "argument vcpu must be positive");
+        return;
+    }
+
+    events = qmp_trace_event_get_state(name, has_vcpu, vcpu, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+        return;
+    }
 
     for (elem = events; elem != NULL; elem = elem->next) {
         monitor_printf(mon, "%s : state %u\n",
@@ -1105,46 +1108,46 @@
     qapi_free_TraceEventInfoList(events);
 }
 
-static int client_migrate_info(Monitor *mon, const QDict *qdict,
-                               MonitorCompletion cb, void *opaque)
+void qmp_client_migrate_info(const char *protocol, const char *hostname,
+                             bool has_port, int64_t port,
+                             bool has_tls_port, int64_t tls_port,
+                             bool has_cert_subject, const char *cert_subject,
+                             Error **errp)
 {
-    const char *protocol = qdict_get_str(qdict, "protocol");
-    const char *hostname = qdict_get_str(qdict, "hostname");
-    const char *subject  = qdict_get_try_str(qdict, "cert-subject");
-    int port             = qdict_get_try_int(qdict, "port", -1);
-    int tls_port         = qdict_get_try_int(qdict, "tls-port", -1);
-    int ret;
-
     if (strcmp(protocol, "spice") == 0) {
-        if (!using_spice) {
-            qerror_report(QERR_DEVICE_NOT_ACTIVE, "spice");
-            return -1;
+        if (!qemu_using_spice(errp)) {
+            return;
         }
 
-        if (port == -1 && tls_port == -1) {
-            qerror_report(QERR_MISSING_PARAMETER, "port/tls-port");
-            return -1;
+        if (!has_port && !has_tls_port) {
+            error_setg(errp, QERR_MISSING_PARAMETER, "port/tls-port");
+            return;
         }
 
-        ret = qemu_spice_migrate_info(hostname, port, tls_port, subject,
-                                      cb, opaque);
-        if (ret != 0) {
-            qerror_report(QERR_UNDEFINED_ERROR);
-            return -1;
+        if (qemu_spice_migrate_info(hostname,
+                                    has_port ? port : -1,
+                                    has_tls_port ? tls_port : -1,
+                                    cert_subject)) {
+            error_setg(errp, QERR_UNDEFINED_ERROR);
+            return;
         }
-        return 0;
+        return;
     }
 
-    qerror_report(QERR_INVALID_PARAMETER, "protocol");
-    return -1;
+    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "protocol", "spice");
 }
 
-static void do_logfile(Monitor *mon, const QDict *qdict)
+static void hmp_logfile(Monitor *mon, const QDict *qdict)
 {
-    qemu_set_log_filename(qdict_get_str(qdict, "filename"));
+    Error *err = NULL;
+
+    qemu_set_log_filename(qdict_get_str(qdict, "filename"), &err);
+    if (err) {
+        error_report_err(err);
+    }
 }
 
-static void do_log(Monitor *mon, const QDict *qdict)
+static void hmp_log(Monitor *mon, const QDict *qdict)
 {
     int mask;
     const char *items = qdict_get_str(qdict, "items");
@@ -1161,7 +1164,7 @@
     qemu_set_log(mask);
 }
 
-static void do_singlestep(Monitor *mon, const QDict *qdict)
+static void hmp_singlestep(Monitor *mon, const QDict *qdict)
 {
     const char *option = qdict_get_try_str(qdict, "option");
     if (!option || !strcmp(option, "on")) {
@@ -1173,7 +1176,7 @@
     }
 }
 
-static void do_gdbserver(Monitor *mon, const QDict *qdict)
+static void hmp_gdbserver(Monitor *mon, const QDict *qdict)
 {
     const char *device = qdict_get_try_str(qdict, "device");
     if (!device)
@@ -1189,7 +1192,7 @@
     }
 }
 
-static void do_watchdog_action(Monitor *mon, const QDict *qdict)
+static void hmp_watchdog_action(Monitor *mon, const QDict *qdict)
 {
     const char *action = qdict_get_str(qdict, "action");
     if (select_watchdog_action(action) == -1) {
@@ -1227,16 +1230,14 @@
 static void memory_dump(Monitor *mon, int count, int format, int wsize,
                         hwaddr addr, int is_physical)
 {
-    CPUArchState *env;
     int l, line_size, i, max_digits, len;
     uint8_t buf[16];
     uint64_t v;
 
     if (format == 'i') {
-        int flags;
-        flags = 0;
-        env = mon_get_cpu();
+        int flags = 0;
 #ifdef TARGET_I386
+        CPUArchState *env = mon_get_cpu_env();
         if (wsize == 2) {
             flags = 1;
         } else if (wsize == 4) {
@@ -1257,10 +1258,11 @@
         }
 #endif
 #ifdef TARGET_PPC
+        CPUArchState *env = mon_get_cpu_env();
         flags = msr_le << 16;
         flags |= env->bfd_mach;
 #endif
-        monitor_disas(mon, env, addr, count, is_physical, flags);
+        monitor_disas(mon, mon_get_cpu(), addr, count, is_physical, flags);
         return;
     }
 
@@ -1299,8 +1301,7 @@
         if (is_physical) {
             cpu_physical_memory_read(addr, buf, l);
         } else {
-            env = mon_get_cpu();
-            if (cpu_memory_rw_debug(ENV_GET_CPU(env), addr, buf, l, 0) < 0) {
+            if (cpu_memory_rw_debug(mon_get_cpu(), addr, buf, l, 0) < 0) {
                 monitor_printf(mon, " Cannot access memory\n");
                 break;
             }
@@ -1310,16 +1311,16 @@
             switch(wsize) {
             default:
             case 1:
-                v = ldub_raw(buf + i);
+                v = ldub_p(buf + i);
                 break;
             case 2:
-                v = lduw_raw(buf + i);
+                v = lduw_p(buf + i);
                 break;
             case 4:
-                v = (uint32_t)ldl_raw(buf + i);
+                v = (uint32_t)ldl_p(buf + i);
                 break;
             case 8:
-                v = ldq_raw(buf + i);
+                v = ldq_p(buf + i);
                 break;
             }
             monitor_printf(mon, " ");
@@ -1348,7 +1349,7 @@
     }
 }
 
-static void do_memory_dump(Monitor *mon, const QDict *qdict)
+static void hmp_memory_dump(Monitor *mon, const QDict *qdict)
 {
     int count = qdict_get_int(qdict, "count");
     int format = qdict_get_int(qdict, "format");
@@ -1358,7 +1359,7 @@
     memory_dump(mon, count, format, size, addr, 0);
 }
 
-static void do_physical_memory_dump(Monitor *mon, const QDict *qdict)
+static void hmp_physical_memory_dump(Monitor *mon, const QDict *qdict)
 {
     int count = qdict_get_int(qdict, "count");
     int format = qdict_get_int(qdict, "format");
@@ -1394,7 +1395,7 @@
     monitor_printf(mon, "\n");
 }
 
-static void do_sum(Monitor *mon, const QDict *qdict)
+static void hmp_sum(Monitor *mon, const QDict *qdict)
 {
     uint32_t addr;
     uint16_t sum;
@@ -1403,7 +1404,8 @@
 
     sum = 0;
     for(addr = start; addr < (start + size); addr++) {
-        uint8_t val = ldub_phys(&address_space_memory, addr);
+        uint8_t val = address_space_ldub(&address_space_memory, addr,
+                                         MEMTXATTRS_UNSPECIFIED, NULL);
         /* BSD sum algorithm ('sum' Unix command) */
         sum = (sum >> 1) | (sum << 15);
         sum += val;
@@ -1413,7 +1415,7 @@
 
 static int mouse_button_state;
 
-static void do_mouse_move(Monitor *mon, const QDict *qdict)
+static void hmp_mouse_move(Monitor *mon, const QDict *qdict)
 {
     int dx, dy, dz, button;
     const char *dx_str = qdict_get_str(qdict, "dx_str");
@@ -1437,9 +1439,9 @@
     qemu_input_event_sync();
 }
 
-static void do_mouse_button(Monitor *mon, const QDict *qdict)
+static void hmp_mouse_button(Monitor *mon, const QDict *qdict)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE]     = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]      = MOUSE_EVENT_RBUTTON,
@@ -1454,7 +1456,7 @@
     mouse_button_state = button_state;
 }
 
-static void do_ioport_read(Monitor *mon, const QDict *qdict)
+static void hmp_ioport_read(Monitor *mon, const QDict *qdict)
 {
     int size = qdict_get_int(qdict, "size");
     int addr = qdict_get_int(qdict, "addr");
@@ -1488,7 +1490,7 @@
                    suffix, addr, size * 2, val);
 }
 
-static void do_ioport_write(Monitor *mon, const QDict *qdict)
+static void hmp_ioport_write(Monitor *mon, const QDict *qdict)
 {
     int size = qdict_get_int(qdict, "size");
     int addr = qdict_get_int(qdict, "addr");
@@ -1510,464 +1512,25 @@
     }
 }
 
-static void do_boot_set(Monitor *mon, const QDict *qdict)
+static void hmp_boot_set(Monitor *mon, const QDict *qdict)
 {
-    int res;
+    Error *local_err = NULL;
     const char *bootdevice = qdict_get_str(qdict, "bootdevice");
 
-    res = qemu_boot_set(bootdevice);
-    if (res == 0) {
+    qemu_boot_set(bootdevice, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    } else {
         monitor_printf(mon, "boot device list now set to %s\n", bootdevice);
-    } else if (res > 0) {
-        monitor_printf(mon, "setting boot device list failed\n");
-    } else {
-        monitor_printf(mon, "no function defined to set boot device list for "
-                       "this architecture\n");
     }
 }
 
-#if defined(TARGET_I386)
-static void print_pte(Monitor *mon, hwaddr addr,
-                      hwaddr pte,
-                      hwaddr mask)
-{
-#ifdef TARGET_X86_64
-    if (addr & (1ULL << 47)) {
-        addr |= -1LL << 48;
-    }
-#endif
-    monitor_printf(mon, TARGET_FMT_plx ": " TARGET_FMT_plx
-                   " %c%c%c%c%c%c%c%c%c\n",
-                   addr,
-                   pte & mask,
-                   pte & PG_NX_MASK ? 'X' : '-',
-                   pte & PG_GLOBAL_MASK ? 'G' : '-',
-                   pte & PG_PSE_MASK ? 'P' : '-',
-                   pte & PG_DIRTY_MASK ? 'D' : '-',
-                   pte & PG_ACCESSED_MASK ? 'A' : '-',
-                   pte & PG_PCD_MASK ? 'C' : '-',
-                   pte & PG_PWT_MASK ? 'T' : '-',
-                   pte & PG_USER_MASK ? 'U' : '-',
-                   pte & PG_RW_MASK ? 'W' : '-');
-}
-
-static void tlb_info_32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2;
-    uint32_t pgd, pde, pte;
-
-    pgd = env->cr[3] & ~0xfff;
-    for(l1 = 0; l1 < 1024; l1++) {
-        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
-        pde = le32_to_cpu(pde);
-        if (pde & PG_PRESENT_MASK) {
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                /* 4M pages */
-                print_pte(mon, (l1 << 22), pde, ~((1 << 21) - 1));
-            } else {
-                for(l2 = 0; l2 < 1024; l2++) {
-                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
-                    pte = le32_to_cpu(pte);
-                    if (pte & PG_PRESENT_MASK) {
-                        print_pte(mon, (l1 << 22) + (l2 << 12),
-                                  pte & ~PG_PSE_MASK,
-                                  ~0xfff);
-                    }
-                }
-            }
-        }
-    }
-}
-
-static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2, l3;
-    uint64_t pdpe, pde, pte;
-    uint64_t pdp_addr, pd_addr, pt_addr;
-
-    pdp_addr = env->cr[3] & ~0x1f;
-    for (l1 = 0; l1 < 4; l1++) {
-        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
-        pdpe = le64_to_cpu(pdpe);
-        if (pdpe & PG_PRESENT_MASK) {
-            pd_addr = pdpe & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
-                pde = le64_to_cpu(pde);
-                if (pde & PG_PRESENT_MASK) {
-                    if (pde & PG_PSE_MASK) {
-                        /* 2M pages with PAE, CR4.PSE is ignored */
-                        print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
-                                  ~((hwaddr)(1 << 20) - 1));
-                    } else {
-                        pt_addr = pde & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
-                            pte = le64_to_cpu(pte);
-                            if (pte & PG_PRESENT_MASK) {
-                                print_pte(mon, (l1 << 30 ) + (l2 << 21)
-                                          + (l3 << 12),
-                                          pte & ~PG_PSE_MASK,
-                                          ~(hwaddr)0xfff);
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-#ifdef TARGET_X86_64
-static void tlb_info_64(Monitor *mon, CPUArchState *env)
-{
-    uint64_t l1, l2, l3, l4;
-    uint64_t pml4e, pdpe, pde, pte;
-    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr;
-
-    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
-    for (l1 = 0; l1 < 512; l1++) {
-        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
-        pml4e = le64_to_cpu(pml4e);
-        if (pml4e & PG_PRESENT_MASK) {
-            pdp_addr = pml4e & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
-                pdpe = le64_to_cpu(pdpe);
-                if (pdpe & PG_PRESENT_MASK) {
-                    if (pdpe & PG_PSE_MASK) {
-                        /* 1G pages, CR4.PSE is ignored */
-                        print_pte(mon, (l1 << 39) + (l2 << 30), pdpe,
-                                  0x3ffffc0000000ULL);
-                    } else {
-                        pd_addr = pdpe & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
-                            pde = le64_to_cpu(pde);
-                            if (pde & PG_PRESENT_MASK) {
-                                if (pde & PG_PSE_MASK) {
-                                    /* 2M pages, CR4.PSE is ignored */
-                                    print_pte(mon, (l1 << 39) + (l2 << 30) +
-                                              (l3 << 21), pde,
-                                              0x3ffffffe00000ULL);
-                                } else {
-                                    pt_addr = pde & 0x3fffffffff000ULL;
-                                    for (l4 = 0; l4 < 512; l4++) {
-                                        cpu_physical_memory_read(pt_addr
-                                                                 + l4 * 8,
-                                                                 &pte, 8);
-                                        pte = le64_to_cpu(pte);
-                                        if (pte & PG_PRESENT_MASK) {
-                                            print_pte(mon, (l1 << 39) +
-                                                      (l2 << 30) +
-                                                      (l3 << 21) + (l4 << 12),
-                                                      pte & ~PG_PSE_MASK,
-                                                      0x3fffffffff000ULL);
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-#endif
-
-static void tlb_info(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env;
-
-    env = mon_get_cpu();
-
-    if (!(env->cr[0] & CR0_PG_MASK)) {
-        monitor_printf(mon, "PG disabled\n");
-        return;
-    }
-    if (env->cr[4] & CR4_PAE_MASK) {
-#ifdef TARGET_X86_64
-        if (env->hflags & HF_LMA_MASK) {
-            tlb_info_64(mon, env);
-        } else
-#endif
-        {
-            tlb_info_pae32(mon, env);
-        }
-    } else {
-        tlb_info_32(mon, env);
-    }
-}
-
-static void mem_print(Monitor *mon, hwaddr *pstart,
-                      int *plast_prot,
-                      hwaddr end, int prot)
-{
-    int prot1;
-    prot1 = *plast_prot;
-    if (prot != prot1) {
-        if (*pstart != -1) {
-            monitor_printf(mon, TARGET_FMT_plx "-" TARGET_FMT_plx " "
-                           TARGET_FMT_plx " %c%c%c\n",
-                           *pstart, end, end - *pstart,
-                           prot1 & PG_USER_MASK ? 'u' : '-',
-                           'r',
-                           prot1 & PG_RW_MASK ? 'w' : '-');
-        }
-        if (prot != 0)
-            *pstart = end;
-        else
-            *pstart = -1;
-        *plast_prot = prot;
-    }
-}
-
-static void mem_info_32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2;
-    int prot, last_prot;
-    uint32_t pgd, pde, pte;
-    hwaddr start, end;
-
-    pgd = env->cr[3] & ~0xfff;
-    last_prot = 0;
-    start = -1;
-    for(l1 = 0; l1 < 1024; l1++) {
-        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
-        pde = le32_to_cpu(pde);
-        end = l1 << 22;
-        if (pde & PG_PRESENT_MASK) {
-            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-                prot = pde & (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
-                mem_print(mon, &start, &last_prot, end, prot);
-            } else {
-                for(l2 = 0; l2 < 1024; l2++) {
-                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
-                    pte = le32_to_cpu(pte);
-                    end = (l1 << 22) + (l2 << 12);
-                    if (pte & PG_PRESENT_MASK) {
-                        prot = pte & pde &
-                            (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
-                    } else {
-                        prot = 0;
-                    }
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
-}
-
-static void mem_info_pae32(Monitor *mon, CPUArchState *env)
-{
-    unsigned int l1, l2, l3;
-    int prot, last_prot;
-    uint64_t pdpe, pde, pte;
-    uint64_t pdp_addr, pd_addr, pt_addr;
-    hwaddr start, end;
-
-    pdp_addr = env->cr[3] & ~0x1f;
-    last_prot = 0;
-    start = -1;
-    for (l1 = 0; l1 < 4; l1++) {
-        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
-        pdpe = le64_to_cpu(pdpe);
-        end = l1 << 30;
-        if (pdpe & PG_PRESENT_MASK) {
-            pd_addr = pdpe & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
-                pde = le64_to_cpu(pde);
-                end = (l1 << 30) + (l2 << 21);
-                if (pde & PG_PRESENT_MASK) {
-                    if (pde & PG_PSE_MASK) {
-                        prot = pde & (PG_USER_MASK | PG_RW_MASK |
-                                      PG_PRESENT_MASK);
-                        mem_print(mon, &start, &last_prot, end, prot);
-                    } else {
-                        pt_addr = pde & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
-                            pte = le64_to_cpu(pte);
-                            end = (l1 << 30) + (l2 << 21) + (l3 << 12);
-                            if (pte & PG_PRESENT_MASK) {
-                                prot = pte & pde & (PG_USER_MASK | PG_RW_MASK |
-                                                    PG_PRESENT_MASK);
-                            } else {
-                                prot = 0;
-                            }
-                            mem_print(mon, &start, &last_prot, end, prot);
-                        }
-                    }
-                } else {
-                    prot = 0;
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
-}
-
-
-#ifdef TARGET_X86_64
-static void mem_info_64(Monitor *mon, CPUArchState *env)
-{
-    int prot, last_prot;
-    uint64_t l1, l2, l3, l4;
-    uint64_t pml4e, pdpe, pde, pte;
-    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr, start, end;
-
-    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
-    last_prot = 0;
-    start = -1;
-    for (l1 = 0; l1 < 512; l1++) {
-        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
-        pml4e = le64_to_cpu(pml4e);
-        end = l1 << 39;
-        if (pml4e & PG_PRESENT_MASK) {
-            pdp_addr = pml4e & 0x3fffffffff000ULL;
-            for (l2 = 0; l2 < 512; l2++) {
-                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
-                pdpe = le64_to_cpu(pdpe);
-                end = (l1 << 39) + (l2 << 30);
-                if (pdpe & PG_PRESENT_MASK) {
-                    if (pdpe & PG_PSE_MASK) {
-                        prot = pdpe & (PG_USER_MASK | PG_RW_MASK |
-                                       PG_PRESENT_MASK);
-                        prot &= pml4e;
-                        mem_print(mon, &start, &last_prot, end, prot);
-                    } else {
-                        pd_addr = pdpe & 0x3fffffffff000ULL;
-                        for (l3 = 0; l3 < 512; l3++) {
-                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
-                            pde = le64_to_cpu(pde);
-                            end = (l1 << 39) + (l2 << 30) + (l3 << 21);
-                            if (pde & PG_PRESENT_MASK) {
-                                if (pde & PG_PSE_MASK) {
-                                    prot = pde & (PG_USER_MASK | PG_RW_MASK |
-                                                  PG_PRESENT_MASK);
-                                    prot &= pml4e & pdpe;
-                                    mem_print(mon, &start, &last_prot, end, prot);
-                                } else {
-                                    pt_addr = pde & 0x3fffffffff000ULL;
-                                    for (l4 = 0; l4 < 512; l4++) {
-                                        cpu_physical_memory_read(pt_addr
-                                                                 + l4 * 8,
-                                                                 &pte, 8);
-                                        pte = le64_to_cpu(pte);
-                                        end = (l1 << 39) + (l2 << 30) +
-                                            (l3 << 21) + (l4 << 12);
-                                        if (pte & PG_PRESENT_MASK) {
-                                            prot = pte & (PG_USER_MASK | PG_RW_MASK |
-                                                          PG_PRESENT_MASK);
-                                            prot &= pml4e & pdpe & pde;
-                                        } else {
-                                            prot = 0;
-                                        }
-                                        mem_print(mon, &start, &last_prot, end, prot);
-                                    }
-                                }
-                            } else {
-                                prot = 0;
-                                mem_print(mon, &start, &last_prot, end, prot);
-                            }
-                        }
-                    }
-                } else {
-                    prot = 0;
-                    mem_print(mon, &start, &last_prot, end, prot);
-                }
-            }
-        } else {
-            prot = 0;
-            mem_print(mon, &start, &last_prot, end, prot);
-        }
-    }
-    /* Flush last range */
-    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
-}
-#endif
-
-static void mem_info(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env;
-
-    env = mon_get_cpu();
-
-    if (!(env->cr[0] & CR0_PG_MASK)) {
-        monitor_printf(mon, "PG disabled\n");
-        return;
-    }
-    if (env->cr[4] & CR4_PAE_MASK) {
-#ifdef TARGET_X86_64
-        if (env->hflags & HF_LMA_MASK) {
-            mem_info_64(mon, env);
-        } else
-#endif
-        {
-            mem_info_pae32(mon, env);
-        }
-    } else {
-        mem_info_32(mon, env);
-    }
-}
-#endif
-
-#if defined(TARGET_SH4)
-
-static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
-{
-    monitor_printf(mon, " tlb%i:\t"
-                   "asid=%hhu vpn=%x\tppn=%x\tsz=%hhu size=%u\t"
-                   "v=%hhu shared=%hhu cached=%hhu prot=%hhu "
-                   "dirty=%hhu writethrough=%hhu\n",
-                   idx,
-                   tlb->asid, tlb->vpn, tlb->ppn, tlb->sz, tlb->size,
-                   tlb->v, tlb->sh, tlb->c, tlb->pr,
-                   tlb->d, tlb->wt);
-}
-
-static void tlb_info(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env = mon_get_cpu();
-    int i;
-
-    monitor_printf (mon, "ITLB:\n");
-    for (i = 0 ; i < ITLB_SIZE ; i++)
-        print_tlb (mon, i, &env->itlb[i]);
-    monitor_printf (mon, "UTLB:\n");
-    for (i = 0 ; i < UTLB_SIZE ; i++)
-        print_tlb (mon, i, &env->utlb[i]);
-}
-
-#endif
-
-#if defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_XTENSA)
-static void tlb_info(Monitor *mon, const QDict *qdict)
-{
-    CPUArchState *env1 = mon_get_cpu();
-
-    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
-}
-#endif
-
-static void do_info_mtree(Monitor *mon, const QDict *qdict)
+static void hmp_info_mtree(Monitor *mon, const QDict *qdict)
 {
     mtree_info((fprintf_function)monitor_printf, mon);
 }
 
-static void do_info_numa(Monitor *mon, const QDict *qdict)
+static void hmp_info_numa(Monitor *mon, const QDict *qdict)
 {
     int i;
     CPUState *cpu;
@@ -1992,20 +1555,20 @@
 
 #ifdef CONFIG_PROFILER
 
-int64_t qemu_time;
+int64_t tcg_time;
 int64_t dev_time;
 
-static void do_info_profile(Monitor *mon, const QDict *qdict)
+static void hmp_info_profile(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "async time  %" PRId64 " (%0.3f)\n",
-                   dev_time, dev_time / (double)get_ticks_per_sec());
+                   dev_time, dev_time / (double)NANOSECONDS_PER_SECOND);
     monitor_printf(mon, "qemu time   %" PRId64 " (%0.3f)\n",
-                   qemu_time, qemu_time / (double)get_ticks_per_sec());
-    qemu_time = 0;
+                   tcg_time, tcg_time / (double)NANOSECONDS_PER_SECOND);
+    tcg_time = 0;
     dev_time = 0;
 }
 #else
-static void do_info_profile(Monitor *mon, const QDict *qdict)
+static void hmp_info_profile(Monitor *mon, const QDict *qdict)
 {
     monitor_printf(mon, "Internal profiler not compiled\n");
 }
@@ -2014,7 +1577,7 @@
 /* Capture support */
 static QLIST_HEAD (capture_list_head, CaptureState) capture_head;
 
-static void do_info_capture(Monitor *mon, const QDict *qdict)
+static void hmp_info_capture(Monitor *mon, const QDict *qdict)
 {
     int i;
     CaptureState *s;
@@ -2025,7 +1588,7 @@
     }
 }
 
-static void do_stop_capture(Monitor *mon, const QDict *qdict)
+static void hmp_stopcapture(Monitor *mon, const QDict *qdict)
 {
     int i;
     int n = qdict_get_int(qdict, "n");
@@ -2041,7 +1604,7 @@
     }
 }
 
-static void do_wav_capture(Monitor *mon, const QDict *qdict)
+static void hmp_wavcapture(Monitor *mon, const QDict *qdict)
 {
     const char *path = qdict_get_str(qdict, "path");
     int has_freq = qdict_haskey(qdict, "freq");
@@ -2076,7 +1639,7 @@
     return acl;
 }
 
-static void do_acl_show(Monitor *mon, const QDict *qdict)
+static void hmp_acl_show(Monitor *mon, const QDict *qdict)
 {
     const char *aclname = qdict_get_str(qdict, "aclname");
     qemu_acl *acl = find_acl(mon, aclname);
@@ -2094,7 +1657,7 @@
     }
 }
 
-static void do_acl_reset(Monitor *mon, const QDict *qdict)
+static void hmp_acl_reset(Monitor *mon, const QDict *qdict)
 {
     const char *aclname = qdict_get_str(qdict, "aclname");
     qemu_acl *acl = find_acl(mon, aclname);
@@ -2105,7 +1668,7 @@
     }
 }
 
-static void do_acl_policy(Monitor *mon, const QDict *qdict)
+static void hmp_acl_policy(Monitor *mon, const QDict *qdict)
 {
     const char *aclname = qdict_get_str(qdict, "aclname");
     const char *policy = qdict_get_str(qdict, "policy");
@@ -2125,7 +1688,7 @@
     }
 }
 
-static void do_acl_add(Monitor *mon, const QDict *qdict)
+static void hmp_acl_add(Monitor *mon, const QDict *qdict)
 {
     const char *aclname = qdict_get_str(qdict, "aclname");
     const char *match = qdict_get_str(qdict, "match");
@@ -2156,7 +1719,7 @@
     }
 }
 
-static void do_acl_remove(Monitor *mon, const QDict *qdict)
+static void hmp_acl_remove(Monitor *mon, const QDict *qdict)
 {
     const char *aclname = qdict_get_str(qdict, "aclname");
     const char *match = qdict_get_str(qdict, "match");
@@ -2172,31 +1735,6 @@
     }
 }
 
-#if defined(TARGET_I386)
-static void do_inject_mce(Monitor *mon, const QDict *qdict)
-{
-    X86CPU *cpu;
-    CPUState *cs;
-    int cpu_index = qdict_get_int(qdict, "cpu_index");
-    int bank = qdict_get_int(qdict, "bank");
-    uint64_t status = qdict_get_int(qdict, "status");
-    uint64_t mcg_status = qdict_get_int(qdict, "mcg_status");
-    uint64_t addr = qdict_get_int(qdict, "addr");
-    uint64_t misc = qdict_get_int(qdict, "misc");
-    int flags = MCE_INJECT_UNCOND_AO;
-
-    if (qdict_get_try_bool(qdict, "broadcast", 0)) {
-        flags |= MCE_INJECT_BROADCAST;
-    }
-    cs = qemu_get_cpu(cpu_index);
-    if (cs != NULL) {
-        cpu = X86_CPU(cs);
-        cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
-                           flags);
-    }
-}
-#endif
-
 void qmp_getfd(const char *fdname, Error **errp)
 {
     mon_fd_t *monfd;
@@ -2204,14 +1742,14 @@
 
     fd = qemu_chr_fe_get_msgfd(cur_mon->chr);
     if (fd == -1) {
-        error_set(errp, QERR_FD_NOT_SUPPLIED);
+        error_setg(errp, QERR_FD_NOT_SUPPLIED);
         return;
     }
 
     if (qemu_isdigit(fdname[0])) {
         close(fd);
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "fdname",
-                  "a name not starting with a digit");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "fdname",
+                   "a name not starting with a digit");
         return;
     }
 
@@ -2248,10 +1786,10 @@
         return;
     }
 
-    error_set(errp, QERR_FD_NOT_FOUND, fdname);
+    error_setg(errp, QERR_FD_NOT_FOUND, fdname);
 }
 
-static void do_loadvm(Monitor *mon, const QDict *qdict)
+static void hmp_loadvm(Monitor *mon, const QDict *qdict)
 {
     int saved_vm_running  = runstate_is_running();
     const char *name = qdict_get_str(qdict, "name");
@@ -2329,7 +1867,7 @@
 
     fd = qemu_chr_fe_get_msgfd(mon->chr);
     if (fd == -1) {
-        error_set(errp, QERR_FD_NOT_SUPPLIED);
+        error_setg(errp, QERR_FD_NOT_SUPPLIED);
         goto error;
     }
 
@@ -2381,7 +1919,7 @@
     } else {
         snprintf(fd_str, sizeof(fd_str), "fdset-id:%" PRId64, fdset_id);
     }
-    error_set(errp, QERR_FD_NOT_FOUND, fd_str);
+    error_setg(errp, QERR_FD_NOT_FOUND, fd_str);
 }
 
 FdsetInfoList *qmp_query_fdsets(Error **errp)
@@ -2449,8 +1987,8 @@
 
         if (has_fdset_id) {
             if (fdset_id < 0) {
-                error_set(errp, QERR_INVALID_PARAMETER_VALUE, "fdset-id",
-                          "a non-negative value");
+                error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "fdset-id",
+                           "a non-negative value");
                 return NULL;
             }
             /* Use specified fdset ID */
@@ -2589,20 +2127,7 @@
     monitor_fdset_dup_fd_find_remove(dup_fd, true);
 }
 
-int monitor_handle_fd_param(Monitor *mon, const char *fdname)
-{
-    int fd;
-    Error *local_err = NULL;
-
-    fd = monitor_handle_fd_param2(mon, fdname, &local_err);
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-    }
-    return fd;
-}
-
-int monitor_handle_fd_param2(Monitor *mon, const char *fdname, Error **errp)
+int monitor_fd_param(Monitor *mon, const char *fdname, Error **errp)
 {
     int fd;
     Error *local_err = NULL;
@@ -2628,328 +2153,8 @@
 
 /* Please update hmp-commands.hx when adding or changing commands */
 static mon_cmd_t info_cmds[] = {
-    {
-        .name       = "version",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the version of QEMU",
-        .mhandler.cmd = hmp_info_version,
-    },
-    {
-        .name       = "network",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the network state",
-        .mhandler.cmd = do_info_network,
-    },
-    {
-        .name       = "chardev",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the character devices",
-        .mhandler.cmd = hmp_info_chardev,
-    },
-    {
-        .name       = "block",
-        .args_type  = "verbose:-v,device:B?",
-        .params     = "[-v] [device]",
-        .help       = "show info of one block device or all block devices "
-                      "(and details of images with -v option)",
-        .mhandler.cmd = hmp_info_block,
-    },
-    {
-        .name       = "blockstats",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show block device statistics",
-        .mhandler.cmd = hmp_info_blockstats,
-    },
-    {
-        .name       = "block-jobs",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show progress of ongoing block device operations",
-        .mhandler.cmd = hmp_info_block_jobs,
-    },
-    {
-        .name       = "registers",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the cpu registers",
-        .mhandler.cmd = do_info_registers,
-    },
-    {
-        .name       = "cpus",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show infos for each CPU",
-        .mhandler.cmd = hmp_info_cpus,
-    },
-    {
-        .name       = "history",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the command line history",
-        .mhandler.cmd = do_info_history,
-    },
-#if defined(TARGET_I386) || defined(TARGET_PPC) || defined(TARGET_MIPS) || \
-    defined(TARGET_LM32) || (defined(TARGET_SPARC) && !defined(TARGET_SPARC64))
-    {
-        .name       = "irq",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the interrupts statistics (if available)",
-#ifdef TARGET_SPARC
-        .mhandler.cmd = sun4m_irq_info,
-#elif defined(TARGET_LM32)
-        .mhandler.cmd = lm32_irq_info,
-#else
-        .mhandler.cmd = irq_info,
-#endif
-    },
-    {
-        .name       = "pic",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show i8259 (PIC) state",
-#ifdef TARGET_SPARC
-        .mhandler.cmd = sun4m_pic_info,
-#elif defined(TARGET_LM32)
-        .mhandler.cmd = lm32_do_pic_info,
-#else
-        .mhandler.cmd = pic_info,
-#endif
-    },
-#endif
-    {
-        .name       = "pci",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show PCI info",
-        .mhandler.cmd = hmp_info_pci,
-    },
-#if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \
-    defined(TARGET_PPC) || defined(TARGET_XTENSA)
-    {
-        .name       = "tlb",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show virtual to physical memory mappings",
-        .mhandler.cmd = tlb_info,
-    },
-#endif
-#if defined(TARGET_I386)
-    {
-        .name       = "mem",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the active virtual memory mappings",
-        .mhandler.cmd = mem_info,
-    },
-#endif
-    {
-        .name       = "mtree",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory tree",
-        .mhandler.cmd = do_info_mtree,
-    },
-    {
-        .name       = "jit",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show dynamic compiler info",
-        .mhandler.cmd = do_info_jit,
-    },
-    {
-        .name       = "kvm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show KVM information",
-        .mhandler.cmd = hmp_info_kvm,
-    },
-    {
-        .name       = "numa",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show NUMA information",
-        .mhandler.cmd = do_info_numa,
-    },
-    {
-        .name       = "usb",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show guest USB devices",
-        .mhandler.cmd = usb_info,
-    },
-    {
-        .name       = "usbhost",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show host USB devices",
-        .mhandler.cmd = usb_host_info,
-    },
-    {
-        .name       = "profile",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show profiling information",
-        .mhandler.cmd = do_info_profile,
-    },
-    {
-        .name       = "capture",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show capture information",
-        .mhandler.cmd = do_info_capture,
-    },
-    {
-        .name       = "snapshots",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the currently saved VM snapshots",
-        .mhandler.cmd = do_info_snapshots,
-    },
-    {
-        .name       = "status",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM status (running|paused)",
-        .mhandler.cmd = hmp_info_status,
-    },
-    {
-        .name       = "mice",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show which guest mouse is receiving events",
-        .mhandler.cmd = hmp_info_mice,
-    },
-    {
-        .name       = "vnc",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the vnc server status",
-        .mhandler.cmd = hmp_info_vnc,
-    },
-#if defined(CONFIG_SPICE)
-    {
-        .name       = "spice",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the spice server status",
-        .mhandler.cmd = hmp_info_spice,
-    },
-#endif
-    {
-        .name       = "name",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM name",
-        .mhandler.cmd = hmp_info_name,
-    },
-    {
-        .name       = "uuid",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the current VM UUID",
-        .mhandler.cmd = hmp_info_uuid,
-    },
-    {
-        .name       = "cpustats",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show CPU statistics",
-        .mhandler.cmd = do_info_cpu_stats,
-    },
-#if defined(CONFIG_SLIRP)
-    {
-        .name       = "usernet",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show user network stack connection states",
-        .mhandler.cmd = do_info_usernet,
-    },
-#endif
-    {
-        .name       = "migrate",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show migration status",
-        .mhandler.cmd = hmp_info_migrate,
-    },
-    {
-        .name       = "migrate_capabilities",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration capabilities",
-        .mhandler.cmd = hmp_info_migrate_capabilities,
-    },
-    {
-        .name       = "migrate_cache_size",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show current migration xbzrle cache size",
-        .mhandler.cmd = hmp_info_migrate_cache_size,
-    },
-    {
-        .name       = "balloon",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show balloon information",
-        .mhandler.cmd = hmp_info_balloon,
-    },
-    {
-        .name       = "qtree",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show device tree",
-        .mhandler.cmd = do_info_qtree,
-    },
-    {
-        .name       = "qdm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show qdev device model list",
-        .mhandler.cmd = do_info_qdm,
-    },
-    {
-        .name       = "roms",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show roms",
-        .mhandler.cmd = do_info_roms,
-    },
-    {
-        .name       = "trace-events",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show available trace-events & their state",
-        .mhandler.cmd = do_trace_print_events,
-    },
-    {
-        .name       = "tpm",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show the TPM device",
-        .mhandler.cmd = hmp_info_tpm,
-    },
-    {
-        .name       = "memdev",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory backends",
-        .mhandler.cmd = hmp_info_memdev,
-    },
-    {
-        .name       = "memory-devices",
-        .args_type  = "",
-        .params     = "",
-        .help       = "show memory devices",
-        .mhandler.cmd = hmp_info_memory_devices,
-    },
-    {
-        .name       = NULL,
-    },
+#include "hmp-commands-info.h"
+    { NULL, NULL, },
 };
 
 /* mon_cmds and info_cmds would be sorted at runtime */
@@ -2968,394 +2173,6 @@
 static const char *pch;
 static sigjmp_buf expr_env;
 
-#define MD_TLONG 0
-#define MD_I32   1
-
-typedef struct MonitorDef {
-    const char *name;
-    int offset;
-    target_long (*get_value)(const struct MonitorDef *md, int val);
-    int type;
-} MonitorDef;
-
-#if defined(TARGET_I386)
-static target_long monitor_get_pc (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return env->eip + env->segs[R_CS].base;
-}
-#endif
-
-#if defined(TARGET_PPC)
-static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    unsigned int u;
-    int i;
-
-    u = 0;
-    for (i = 0; i < 8; i++)
-        u |= env->crf[i] << (32 - (4 * (i + 1)));
-
-    return u;
-}
-
-static target_long monitor_get_msr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return env->msr;
-}
-
-static target_long monitor_get_xer (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return env->xer;
-}
-
-static target_long monitor_get_decr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return cpu_ppc_load_decr(env);
-}
-
-static target_long monitor_get_tbu (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return cpu_ppc_load_tbu(env);
-}
-
-static target_long monitor_get_tbl (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return cpu_ppc_load_tbl(env);
-}
-#endif
-
-#if defined(TARGET_SPARC)
-#ifndef TARGET_SPARC64
-static target_long monitor_get_psr (const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-
-    return cpu_get_psr(env);
-}
-#endif
-
-static target_long monitor_get_reg(const struct MonitorDef *md, int val)
-{
-    CPUArchState *env = mon_get_cpu();
-    return env->regwptr[val];
-}
-#endif
-
-static const MonitorDef monitor_defs[] = {
-#ifdef TARGET_I386
-
-#define SEG(name, seg) \
-    { name, offsetof(CPUX86State, segs[seg].selector), NULL, MD_I32 },\
-    { name ".base", offsetof(CPUX86State, segs[seg].base) },\
-    { name ".limit", offsetof(CPUX86State, segs[seg].limit), NULL, MD_I32 },
-
-    { "eax", offsetof(CPUX86State, regs[0]) },
-    { "ecx", offsetof(CPUX86State, regs[1]) },
-    { "edx", offsetof(CPUX86State, regs[2]) },
-    { "ebx", offsetof(CPUX86State, regs[3]) },
-    { "esp|sp", offsetof(CPUX86State, regs[4]) },
-    { "ebp|fp", offsetof(CPUX86State, regs[5]) },
-    { "esi", offsetof(CPUX86State, regs[6]) },
-    { "edi", offsetof(CPUX86State, regs[7]) },
-#ifdef TARGET_X86_64
-    { "r8", offsetof(CPUX86State, regs[8]) },
-    { "r9", offsetof(CPUX86State, regs[9]) },
-    { "r10", offsetof(CPUX86State, regs[10]) },
-    { "r11", offsetof(CPUX86State, regs[11]) },
-    { "r12", offsetof(CPUX86State, regs[12]) },
-    { "r13", offsetof(CPUX86State, regs[13]) },
-    { "r14", offsetof(CPUX86State, regs[14]) },
-    { "r15", offsetof(CPUX86State, regs[15]) },
-#endif
-    { "eflags", offsetof(CPUX86State, eflags) },
-    { "eip", offsetof(CPUX86State, eip) },
-    SEG("cs", R_CS)
-    SEG("ds", R_DS)
-    SEG("es", R_ES)
-    SEG("ss", R_SS)
-    SEG("fs", R_FS)
-    SEG("gs", R_GS)
-    { "pc", 0, monitor_get_pc, },
-#elif defined(TARGET_PPC)
-    /* General purpose registers */
-    { "r0", offsetof(CPUPPCState, gpr[0]) },
-    { "r1", offsetof(CPUPPCState, gpr[1]) },
-    { "r2", offsetof(CPUPPCState, gpr[2]) },
-    { "r3", offsetof(CPUPPCState, gpr[3]) },
-    { "r4", offsetof(CPUPPCState, gpr[4]) },
-    { "r5", offsetof(CPUPPCState, gpr[5]) },
-    { "r6", offsetof(CPUPPCState, gpr[6]) },
-    { "r7", offsetof(CPUPPCState, gpr[7]) },
-    { "r8", offsetof(CPUPPCState, gpr[8]) },
-    { "r9", offsetof(CPUPPCState, gpr[9]) },
-    { "r10", offsetof(CPUPPCState, gpr[10]) },
-    { "r11", offsetof(CPUPPCState, gpr[11]) },
-    { "r12", offsetof(CPUPPCState, gpr[12]) },
-    { "r13", offsetof(CPUPPCState, gpr[13]) },
-    { "r14", offsetof(CPUPPCState, gpr[14]) },
-    { "r15", offsetof(CPUPPCState, gpr[15]) },
-    { "r16", offsetof(CPUPPCState, gpr[16]) },
-    { "r17", offsetof(CPUPPCState, gpr[17]) },
-    { "r18", offsetof(CPUPPCState, gpr[18]) },
-    { "r19", offsetof(CPUPPCState, gpr[19]) },
-    { "r20", offsetof(CPUPPCState, gpr[20]) },
-    { "r21", offsetof(CPUPPCState, gpr[21]) },
-    { "r22", offsetof(CPUPPCState, gpr[22]) },
-    { "r23", offsetof(CPUPPCState, gpr[23]) },
-    { "r24", offsetof(CPUPPCState, gpr[24]) },
-    { "r25", offsetof(CPUPPCState, gpr[25]) },
-    { "r26", offsetof(CPUPPCState, gpr[26]) },
-    { "r27", offsetof(CPUPPCState, gpr[27]) },
-    { "r28", offsetof(CPUPPCState, gpr[28]) },
-    { "r29", offsetof(CPUPPCState, gpr[29]) },
-    { "r30", offsetof(CPUPPCState, gpr[30]) },
-    { "r31", offsetof(CPUPPCState, gpr[31]) },
-    /* Floating point registers */
-    { "f0", offsetof(CPUPPCState, fpr[0]) },
-    { "f1", offsetof(CPUPPCState, fpr[1]) },
-    { "f2", offsetof(CPUPPCState, fpr[2]) },
-    { "f3", offsetof(CPUPPCState, fpr[3]) },
-    { "f4", offsetof(CPUPPCState, fpr[4]) },
-    { "f5", offsetof(CPUPPCState, fpr[5]) },
-    { "f6", offsetof(CPUPPCState, fpr[6]) },
-    { "f7", offsetof(CPUPPCState, fpr[7]) },
-    { "f8", offsetof(CPUPPCState, fpr[8]) },
-    { "f9", offsetof(CPUPPCState, fpr[9]) },
-    { "f10", offsetof(CPUPPCState, fpr[10]) },
-    { "f11", offsetof(CPUPPCState, fpr[11]) },
-    { "f12", offsetof(CPUPPCState, fpr[12]) },
-    { "f13", offsetof(CPUPPCState, fpr[13]) },
-    { "f14", offsetof(CPUPPCState, fpr[14]) },
-    { "f15", offsetof(CPUPPCState, fpr[15]) },
-    { "f16", offsetof(CPUPPCState, fpr[16]) },
-    { "f17", offsetof(CPUPPCState, fpr[17]) },
-    { "f18", offsetof(CPUPPCState, fpr[18]) },
-    { "f19", offsetof(CPUPPCState, fpr[19]) },
-    { "f20", offsetof(CPUPPCState, fpr[20]) },
-    { "f21", offsetof(CPUPPCState, fpr[21]) },
-    { "f22", offsetof(CPUPPCState, fpr[22]) },
-    { "f23", offsetof(CPUPPCState, fpr[23]) },
-    { "f24", offsetof(CPUPPCState, fpr[24]) },
-    { "f25", offsetof(CPUPPCState, fpr[25]) },
-    { "f26", offsetof(CPUPPCState, fpr[26]) },
-    { "f27", offsetof(CPUPPCState, fpr[27]) },
-    { "f28", offsetof(CPUPPCState, fpr[28]) },
-    { "f29", offsetof(CPUPPCState, fpr[29]) },
-    { "f30", offsetof(CPUPPCState, fpr[30]) },
-    { "f31", offsetof(CPUPPCState, fpr[31]) },
-    { "fpscr", offsetof(CPUPPCState, fpscr) },
-    /* Next instruction pointer */
-    { "nip|pc", offsetof(CPUPPCState, nip) },
-    { "lr", offsetof(CPUPPCState, lr) },
-    { "ctr", offsetof(CPUPPCState, ctr) },
-    { "decr", 0, &monitor_get_decr, },
-    { "ccr", 0, &monitor_get_ccr, },
-    /* Machine state register */
-    { "msr", 0, &monitor_get_msr, },
-    { "xer", 0, &monitor_get_xer, },
-    { "tbu", 0, &monitor_get_tbu, },
-    { "tbl", 0, &monitor_get_tbl, },
-    /* Segment registers */
-    { "sdr1", offsetof(CPUPPCState, spr[SPR_SDR1]) },
-    { "sr0", offsetof(CPUPPCState, sr[0]) },
-    { "sr1", offsetof(CPUPPCState, sr[1]) },
-    { "sr2", offsetof(CPUPPCState, sr[2]) },
-    { "sr3", offsetof(CPUPPCState, sr[3]) },
-    { "sr4", offsetof(CPUPPCState, sr[4]) },
-    { "sr5", offsetof(CPUPPCState, sr[5]) },
-    { "sr6", offsetof(CPUPPCState, sr[6]) },
-    { "sr7", offsetof(CPUPPCState, sr[7]) },
-    { "sr8", offsetof(CPUPPCState, sr[8]) },
-    { "sr9", offsetof(CPUPPCState, sr[9]) },
-    { "sr10", offsetof(CPUPPCState, sr[10]) },
-    { "sr11", offsetof(CPUPPCState, sr[11]) },
-    { "sr12", offsetof(CPUPPCState, sr[12]) },
-    { "sr13", offsetof(CPUPPCState, sr[13]) },
-    { "sr14", offsetof(CPUPPCState, sr[14]) },
-    { "sr15", offsetof(CPUPPCState, sr[15]) },
-    /* Too lazy to put BATs... */
-    { "pvr", offsetof(CPUPPCState, spr[SPR_PVR]) },
-
-    { "srr0", offsetof(CPUPPCState, spr[SPR_SRR0]) },
-    { "srr1", offsetof(CPUPPCState, spr[SPR_SRR1]) },
-    { "dar", offsetof(CPUPPCState, spr[SPR_DAR]) },
-    { "dsisr", offsetof(CPUPPCState, spr[SPR_DSISR]) },
-    { "cfar", offsetof(CPUPPCState, spr[SPR_CFAR]) },
-    { "sprg0", offsetof(CPUPPCState, spr[SPR_SPRG0]) },
-    { "sprg1", offsetof(CPUPPCState, spr[SPR_SPRG1]) },
-    { "sprg2", offsetof(CPUPPCState, spr[SPR_SPRG2]) },
-    { "sprg3", offsetof(CPUPPCState, spr[SPR_SPRG3]) },
-    { "sprg4", offsetof(CPUPPCState, spr[SPR_SPRG4]) },
-    { "sprg5", offsetof(CPUPPCState, spr[SPR_SPRG5]) },
-    { "sprg6", offsetof(CPUPPCState, spr[SPR_SPRG6]) },
-    { "sprg7", offsetof(CPUPPCState, spr[SPR_SPRG7]) },
-    { "pid", offsetof(CPUPPCState, spr[SPR_BOOKE_PID]) },
-    { "csrr0", offsetof(CPUPPCState, spr[SPR_BOOKE_CSRR0]) },
-    { "csrr1", offsetof(CPUPPCState, spr[SPR_BOOKE_CSRR1]) },
-    { "esr", offsetof(CPUPPCState, spr[SPR_BOOKE_ESR]) },
-    { "dear", offsetof(CPUPPCState, spr[SPR_BOOKE_DEAR]) },
-    { "mcsr", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSR]) },
-    { "tsr", offsetof(CPUPPCState, spr[SPR_BOOKE_TSR]) },
-    { "tcr", offsetof(CPUPPCState, spr[SPR_BOOKE_TCR]) },
-    { "vrsave", offsetof(CPUPPCState, spr[SPR_VRSAVE]) },
-    { "pir", offsetof(CPUPPCState, spr[SPR_BOOKE_PIR]) },
-    { "mcsrr0", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSRR0]) },
-    { "mcsrr1", offsetof(CPUPPCState, spr[SPR_BOOKE_MCSRR1]) },
-    { "decar", offsetof(CPUPPCState, spr[SPR_BOOKE_DECAR]) },
-    { "ivpr", offsetof(CPUPPCState, spr[SPR_BOOKE_IVPR]) },
-    { "epcr", offsetof(CPUPPCState, spr[SPR_BOOKE_EPCR]) },
-    { "sprg8", offsetof(CPUPPCState, spr[SPR_BOOKE_SPRG8]) },
-    { "ivor0", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR0]) },
-    { "ivor1", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR1]) },
-    { "ivor2", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR2]) },
-    { "ivor3", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR3]) },
-    { "ivor4", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR4]) },
-    { "ivor5", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR5]) },
-    { "ivor6", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR6]) },
-    { "ivor7", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR7]) },
-    { "ivor8", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR8]) },
-    { "ivor9", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR9]) },
-    { "ivor10", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR10]) },
-    { "ivor11", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR11]) },
-    { "ivor12", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR12]) },
-    { "ivor13", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR13]) },
-    { "ivor14", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR14]) },
-    { "ivor15", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR15]) },
-    { "ivor32", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR32]) },
-    { "ivor33", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR33]) },
-    { "ivor34", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR34]) },
-    { "ivor35", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR35]) },
-    { "ivor36", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR36]) },
-    { "ivor37", offsetof(CPUPPCState, spr[SPR_BOOKE_IVOR37]) },
-    { "mas0", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS0]) },
-    { "mas1", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS1]) },
-    { "mas2", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS2]) },
-    { "mas3", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS3]) },
-    { "mas4", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS4]) },
-    { "mas6", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS6]) },
-    { "mas7", offsetof(CPUPPCState, spr[SPR_BOOKE_MAS7]) },
-    { "mmucfg", offsetof(CPUPPCState, spr[SPR_MMUCFG]) },
-    { "tlb0cfg", offsetof(CPUPPCState, spr[SPR_BOOKE_TLB0CFG]) },
-    { "tlb1cfg", offsetof(CPUPPCState, spr[SPR_BOOKE_TLB1CFG]) },
-    { "epr", offsetof(CPUPPCState, spr[SPR_BOOKE_EPR]) },
-    { "eplc", offsetof(CPUPPCState, spr[SPR_BOOKE_EPLC]) },
-    { "epsc", offsetof(CPUPPCState, spr[SPR_BOOKE_EPSC]) },
-    { "svr", offsetof(CPUPPCState, spr[SPR_E500_SVR]) },
-    { "mcar", offsetof(CPUPPCState, spr[SPR_Exxx_MCAR]) },
-    { "pid1", offsetof(CPUPPCState, spr[SPR_BOOKE_PID1]) },
-    { "pid2", offsetof(CPUPPCState, spr[SPR_BOOKE_PID2]) },
-    { "hid0", offsetof(CPUPPCState, spr[SPR_HID0]) },
-
-#elif defined(TARGET_SPARC)
-    { "g0", offsetof(CPUSPARCState, gregs[0]) },
-    { "g1", offsetof(CPUSPARCState, gregs[1]) },
-    { "g2", offsetof(CPUSPARCState, gregs[2]) },
-    { "g3", offsetof(CPUSPARCState, gregs[3]) },
-    { "g4", offsetof(CPUSPARCState, gregs[4]) },
-    { "g5", offsetof(CPUSPARCState, gregs[5]) },
-    { "g6", offsetof(CPUSPARCState, gregs[6]) },
-    { "g7", offsetof(CPUSPARCState, gregs[7]) },
-    { "o0", 0, monitor_get_reg },
-    { "o1", 1, monitor_get_reg },
-    { "o2", 2, monitor_get_reg },
-    { "o3", 3, monitor_get_reg },
-    { "o4", 4, monitor_get_reg },
-    { "o5", 5, monitor_get_reg },
-    { "o6", 6, monitor_get_reg },
-    { "o7", 7, monitor_get_reg },
-    { "l0", 8, monitor_get_reg },
-    { "l1", 9, monitor_get_reg },
-    { "l2", 10, monitor_get_reg },
-    { "l3", 11, monitor_get_reg },
-    { "l4", 12, monitor_get_reg },
-    { "l5", 13, monitor_get_reg },
-    { "l6", 14, monitor_get_reg },
-    { "l7", 15, monitor_get_reg },
-    { "i0", 16, monitor_get_reg },
-    { "i1", 17, monitor_get_reg },
-    { "i2", 18, monitor_get_reg },
-    { "i3", 19, monitor_get_reg },
-    { "i4", 20, monitor_get_reg },
-    { "i5", 21, monitor_get_reg },
-    { "i6", 22, monitor_get_reg },
-    { "i7", 23, monitor_get_reg },
-    { "pc", offsetof(CPUSPARCState, pc) },
-    { "npc", offsetof(CPUSPARCState, npc) },
-    { "y", offsetof(CPUSPARCState, y) },
-#ifndef TARGET_SPARC64
-    { "psr", 0, &monitor_get_psr, },
-    { "wim", offsetof(CPUSPARCState, wim) },
-#endif
-    { "tbr", offsetof(CPUSPARCState, tbr) },
-    { "fsr", offsetof(CPUSPARCState, fsr) },
-    { "f0", offsetof(CPUSPARCState, fpr[0].l.upper) },
-    { "f1", offsetof(CPUSPARCState, fpr[0].l.lower) },
-    { "f2", offsetof(CPUSPARCState, fpr[1].l.upper) },
-    { "f3", offsetof(CPUSPARCState, fpr[1].l.lower) },
-    { "f4", offsetof(CPUSPARCState, fpr[2].l.upper) },
-    { "f5", offsetof(CPUSPARCState, fpr[2].l.lower) },
-    { "f6", offsetof(CPUSPARCState, fpr[3].l.upper) },
-    { "f7", offsetof(CPUSPARCState, fpr[3].l.lower) },
-    { "f8", offsetof(CPUSPARCState, fpr[4].l.upper) },
-    { "f9", offsetof(CPUSPARCState, fpr[4].l.lower) },
-    { "f10", offsetof(CPUSPARCState, fpr[5].l.upper) },
-    { "f11", offsetof(CPUSPARCState, fpr[5].l.lower) },
-    { "f12", offsetof(CPUSPARCState, fpr[6].l.upper) },
-    { "f13", offsetof(CPUSPARCState, fpr[6].l.lower) },
-    { "f14", offsetof(CPUSPARCState, fpr[7].l.upper) },
-    { "f15", offsetof(CPUSPARCState, fpr[7].l.lower) },
-    { "f16", offsetof(CPUSPARCState, fpr[8].l.upper) },
-    { "f17", offsetof(CPUSPARCState, fpr[8].l.lower) },
-    { "f18", offsetof(CPUSPARCState, fpr[9].l.upper) },
-    { "f19", offsetof(CPUSPARCState, fpr[9].l.lower) },
-    { "f20", offsetof(CPUSPARCState, fpr[10].l.upper) },
-    { "f21", offsetof(CPUSPARCState, fpr[10].l.lower) },
-    { "f22", offsetof(CPUSPARCState, fpr[11].l.upper) },
-    { "f23", offsetof(CPUSPARCState, fpr[11].l.lower) },
-    { "f24", offsetof(CPUSPARCState, fpr[12].l.upper) },
-    { "f25", offsetof(CPUSPARCState, fpr[12].l.lower) },
-    { "f26", offsetof(CPUSPARCState, fpr[13].l.upper) },
-    { "f27", offsetof(CPUSPARCState, fpr[13].l.lower) },
-    { "f28", offsetof(CPUSPARCState, fpr[14].l.upper) },
-    { "f29", offsetof(CPUSPARCState, fpr[14].l.lower) },
-    { "f30", offsetof(CPUSPARCState, fpr[15].l.upper) },
-    { "f31", offsetof(CPUSPARCState, fpr[15].l.lower) },
-#ifdef TARGET_SPARC64
-    { "f32", offsetof(CPUSPARCState, fpr[16]) },
-    { "f34", offsetof(CPUSPARCState, fpr[17]) },
-    { "f36", offsetof(CPUSPARCState, fpr[18]) },
-    { "f38", offsetof(CPUSPARCState, fpr[19]) },
-    { "f40", offsetof(CPUSPARCState, fpr[20]) },
-    { "f42", offsetof(CPUSPARCState, fpr[21]) },
-    { "f44", offsetof(CPUSPARCState, fpr[22]) },
-    { "f46", offsetof(CPUSPARCState, fpr[23]) },
-    { "f48", offsetof(CPUSPARCState, fpr[24]) },
-    { "f50", offsetof(CPUSPARCState, fpr[25]) },
-    { "f52", offsetof(CPUSPARCState, fpr[26]) },
-    { "f54", offsetof(CPUSPARCState, fpr[27]) },
-    { "f56", offsetof(CPUSPARCState, fpr[28]) },
-    { "f58", offsetof(CPUSPARCState, fpr[29]) },
-    { "f60", offsetof(CPUSPARCState, fpr[30]) },
-    { "f62", offsetof(CPUSPARCState, fpr[31]) },
-    { "asi", offsetof(CPUSPARCState, asi) },
-    { "pstate", offsetof(CPUSPARCState, pstate) },
-    { "cansave", offsetof(CPUSPARCState, cansave) },
-    { "canrestore", offsetof(CPUSPARCState, canrestore) },
-    { "otherwin", offsetof(CPUSPARCState, otherwin) },
-    { "wstate", offsetof(CPUSPARCState, wstate) },
-    { "cleanwin", offsetof(CPUSPARCState, cleanwin) },
-    { "fprs", offsetof(CPUSPARCState, fprs) },
-#endif
-#endif
-    { NULL },
-};
 
 static void GCC_FMT_ATTR(2, 3) QEMU_NORETURN
 expr_error(Monitor *mon, const char *fmt, ...)
@@ -3371,15 +2188,21 @@
 /* return 0 if OK, -1 if not found */
 static int get_monitor_def(target_long *pval, const char *name)
 {
-    const MonitorDef *md;
+    const MonitorDef *md = target_monitor_defs();
     void *ptr;
+    uint64_t tmp = 0;
+    int ret;
 
-    for(md = monitor_defs; md->name != NULL; md++) {
+    if (md == NULL) {
+        return -1;
+    }
+
+    for(; md->name != NULL; md++) {
         if (compare_cmd(name, md->name)) {
             if (md->get_value) {
                 *pval = md->get_value(md, md->offset);
             } else {
-                CPUArchState *env = mon_get_cpu();
+                CPUArchState *env = mon_get_cpu_env();
                 ptr = (uint8_t *)env + md->offset;
                 switch(md->type) {
                 case MD_I32:
@@ -3396,7 +2219,13 @@
             return 0;
         }
     }
-    return -1;
+
+    ret = target_get_monitor_def(mon_get_cpu(), name, &tmp);
+    if (!ret) {
+        *pval = (target_long) tmp;
+    }
+
+    return ret;
 }
 
 static void next(void)
@@ -3695,44 +2524,32 @@
 }
 
 /*
- * Parse @cmdline according to command table @table.
- * If @cmdline is blank, return NULL.
- * If it can't be parsed, report to @mon, and return NULL.
- * Else, insert command arguments into @qdict, and return the command.
- * If a sub-command table exists, and if @cmdline contains an additional string
- * for a sub-command, this function will try to search the sub-command table.
- * If no additional string for a sub-command is present, this function will
- * return the command found in @table.
- * Do not assume the returned command points into @table!  It doesn't
- * when the command is a sub-command.
+ * Parse command name from @cmdp according to command table @table.
+ * If blank, return NULL.
+ * Else, if no valid command can be found, report to @mon, and return
+ * NULL.
+ * Else, change @cmdp to point right behind the name, and return its
+ * command table entry.
+ * Do not assume the return value points into @table!  It doesn't when
+ * the command is found in a sub-command table.
  */
 static const mon_cmd_t *monitor_parse_command(Monitor *mon,
-                                              const char *cmdline,
-                                              int start,
-                                              const mon_cmd_t *table,
-                                              QDict *qdict)
+                                              const char **cmdp,
+                                              mon_cmd_t *table)
 {
-    const char *p, *typestr;
-    int c;
+    const char *p;
     const mon_cmd_t *cmd;
-    const mon_cmd_t *sub_cmds;
     char cmdname[256];
-    char buf[1024];
-    char *key;
-
-#ifdef DEBUG
-    monitor_printf(mon, "command='%s', start='%d'\n", cmdline, start);
-#endif
 
     /* extract the command name */
-    p = get_command_name(cmdline + start, cmdname, sizeof(cmdname));
+    p = get_command_name(*cmdp, cmdname, sizeof(cmdname));
     if (!p)
         return NULL;
 
     cmd = search_dispatch_table(table, cmdname);
     if (!cmd) {
         monitor_printf(mon, "unknown command: '%.*s'\n",
-                       (int)(p - cmdline), cmdline);
+                       (int)(p - *cmdp), *cmdp);
         return NULL;
     }
 
@@ -3740,17 +2557,34 @@
     while (qemu_isspace(*p)) {
         p++;
     }
+
+    *cmdp = p;
     /* search sub command */
-    sub_cmds = get_command_table(mon, cmd->sub_cmds);
-    if (sub_cmds) {
-        /* check if user set additional command */
-        if (*p == '\0') {
-            return cmd;
-        }
-        return monitor_parse_command(mon, cmdline, p - cmdline,
-                                     sub_cmds, qdict);
+    if (cmd->sub_table != NULL && *p != '\0') {
+        return monitor_parse_command(mon, cmdp, cmd->sub_table);
     }
 
+    return cmd;
+}
+
+/*
+ * Parse arguments for @cmd.
+ * If it can't be parsed, report to @mon, and return NULL.
+ * Else, insert command arguments into a QDict, and return it.
+ * Note: On success, caller has to free the QDict structure.
+ */
+
+static QDict *monitor_parse_arguments(Monitor *mon,
+                                      const char **endp,
+                                      const mon_cmd_t *cmd)
+{
+    const char *typestr;
+    char *key;
+    int c;
+    const char *p = *endp;
+    char buf[1024];
+    QDict *qdict = qdict_new();
+
     /* parse the parameters */
     typestr = cmd->args_type;
     for(;;) {
@@ -3780,14 +2614,14 @@
                     switch(c) {
                     case 'F':
                         monitor_printf(mon, "%s: filename expected\n",
-                                       cmdname);
+                                       cmd->name);
                         break;
                     case 'B':
                         monitor_printf(mon, "%s: block device name expected\n",
-                                       cmdname);
+                                       cmd->name);
                         break;
                     default:
-                        monitor_printf(mon, "%s: string expected\n", cmdname);
+                        monitor_printf(mon, "%s: string expected\n", cmd->name);
                         break;
                     }
                     goto fail;
@@ -3812,7 +2646,7 @@
                 if (get_str(buf, sizeof(buf), &p) < 0) {
                     goto fail;
                 }
-                opts = qemu_opts_parse(opts_list, buf, 1);
+                opts = qemu_opts_parse_noisily(opts_list, buf, true);
                 if (!opts) {
                     goto fail;
                 }
@@ -3929,7 +2763,7 @@
                     goto fail;
                 /* Check if 'i' is greater than 32-bit */
                 if ((c == 'i') && ((val >> 32) & 0xffffffff)) {
-                    monitor_printf(mon, "\'%s\' has failed: ", cmdname);
+                    monitor_printf(mon, "\'%s\' has failed: ", cmd->name);
                     monitor_printf(mon, "integer is for 32-bit values\n");
                     goto fail;
                 } else if (c == 'M') {
@@ -3956,7 +2790,7 @@
                         break;
                     }
                 }
-                val = strtosz(p, &end);
+                val = qemu_strtosz(p, &end);
                 if (val < 0) {
                     monitor_printf(mon, "invalid size\n");
                     goto fail;
@@ -4000,7 +2834,7 @@
         case 'b':
             {
                 const char *beg;
-                int val;
+                bool val;
 
                 while (qemu_isspace(*p)) {
                     p++;
@@ -4010,14 +2844,14 @@
                     p++;
                 }
                 if (p - beg == 2 && !memcmp(beg, "on", p - beg)) {
-                    val = 1;
+                    val = true;
                 } else if (p - beg == 3 && !memcmp(beg, "off", p - beg)) {
-                    val = 0;
+                    val = false;
                 } else {
                     monitor_printf(mon, "Expected 'on' or 'off'\n");
                     goto fail;
                 }
-                qdict_put(qdict, key, qbool_from_int(val));
+                qdict_put(qdict, key, qbool_from_bool(val));
             }
             break;
         case '-':
@@ -4037,7 +2871,7 @@
                         if(!is_valid_option(p, typestr)) {
                   
                             monitor_printf(mon, "%s: unsupported option -%c\n",
-                                           cmdname, *p);
+                                           cmd->name, *p);
                             goto fail;
                         } else {
                             skip_key = 1;
@@ -4048,7 +2882,7 @@
                     } else {
                         /* has option */
                         p++;
-                        qdict_put(qdict, key, qbool_from_int(1));
+                        qdict_put(qdict, key, qbool_from_bool(true));
                     }
                 }
             }
@@ -4071,8 +2905,8 @@
                 len = strlen(p);
                 if (len <= 0) {
                     monitor_printf(mon, "%s: string expected\n",
-                                   cmdname);
-                    break;
+                                   cmd->name);
+                    goto fail;
                 }
                 qdict_put(qdict, key, qstring_from_str(p));
                 p += len;
@@ -4080,7 +2914,7 @@
             break;
         default:
         bad_type:
-            monitor_printf(mon, "%s: unknown type '%c'\n", cmdname, c);
+            monitor_printf(mon, "%s: unknown type '%c'\n", cmd->name, c);
             goto fail;
         }
         g_free(key);
@@ -4091,69 +2925,36 @@
         p++;
     if (*p != '\0') {
         monitor_printf(mon, "%s: extraneous characters at the end of line\n",
-                       cmdname);
+                       cmd->name);
         goto fail;
     }
 
-    return cmd;
+    return qdict;
 
 fail:
+    QDECREF(qdict);
     g_free(key);
     return NULL;
 }
 
-void monitor_set_error(Monitor *mon, QError *qerror)
-{
-    /* report only the first error */
-    if (!mon->error) {
-        mon->error = qerror;
-    } else {
-        QDECREF(qerror);
-    }
-}
-
-static void handler_audit(Monitor *mon, const mon_cmd_t *cmd, int ret)
-{
-    if (ret && !monitor_has_error(mon)) {
-        /*
-         * If it returns failure, it must have passed on error.
-         *
-         * Action: Report an internal error to the client if in QMP.
-         */
-        qerror_report(QERR_UNDEFINED_ERROR);
-    }
-}
-
-static void handle_user_command(Monitor *mon, const char *cmdline)
+static void handle_hmp_command(Monitor *mon, const char *cmdline)
 {
     QDict *qdict;
     const mon_cmd_t *cmd;
 
-    qdict = qdict_new();
-
-    cmd = monitor_parse_command(mon, cmdline, 0,
-                                get_command_table(mon, mon->cmds),
-                                qdict);
-    if (!cmd)
-        goto out;
-
-    if (handler_is_async(cmd)) {
-        user_async_cmd_handler(mon, cmd, qdict);
-    } else if (handler_is_qobject(cmd)) {
-        QObject *data = NULL;
-
-        /* XXX: ignores the error code */
-        cmd->mhandler.cmd_new(mon, qdict, &data);
-        assert(!monitor_has_error(mon));
-        if (data) {
-            cmd->user_print(mon, data);
-            qobject_decref(data);
-        }
-    } else {
-        cmd->mhandler.cmd(mon, qdict);
+    cmd = monitor_parse_command(mon, &cmdline, mon->cmd_table);
+    if (!cmd) {
+        return;
     }
 
-out:
+    qdict = monitor_parse_arguments(mon, &cmdline, cmd);
+    if (!qdict) {
+        monitor_printf(mon, "Try \"help %s\" for more information\n",
+                       cmd->name);
+        return;
+    }
+
+    cmd->mhandler.cmd(mon, qdict);
     QDECREF(qdict);
 }
 
@@ -4205,10 +3006,7 @@
         path[input_path_len] = '\0';
         pstrcpy(file_prefix, sizeof(file_prefix), p + 1);
     }
-#ifdef DEBUG_COMPLETION
-    monitor_printf(mon, "input='%s' path='%s' prefix='%s'\n",
-                   input, path, file_prefix);
-#endif
+
     ffs = opendir(path);
     if (!ffs)
         return;
@@ -4230,7 +3028,7 @@
             /* stat the file to find out if it's a directory.
              * In that case add a slash to speed up typing long paths
              */
-            if (stat(file, &sb) == 0 && S_ISDIR(sb.st_mode)) {
+            if (qemu_stat(file, &sb) == 0 && S_ISDIR(sb.st_mode)) {
                 pstrcat(file, sizeof(file), "/");
             }
             readline_add_completion(mon->rs, file);
@@ -4289,8 +3087,8 @@
     }
     len = strlen(str);
     readline_set_completion_index(rs, len);
-    for (i = 0; NetClientOptionsKind_lookup[i]; i++) {
-        add_completion_option(rs, str, NetClientOptionsKind_lookup[i]);
+    for (i = 0; NetClientDriver_lookup[i]; i++) {
+        add_completion_option(rs, str, NetClientDriver_lookup[i]);
     }
 }
 
@@ -4413,14 +3211,6 @@
     qapi_free_ChardevInfoList(start);
 }
 
-void ringbuf_read_completion(ReadLineState *rs, int nb_args, const char *str)
-{
-    if (nb_args != 2) {
-        return;
-    }
-    ringbuf_completion(rs, str);
-}
-
 void ringbuf_write_completion(ReadLineState *rs, int nb_args, const char *str)
 {
     if (nb_args != 2) {
@@ -4481,7 +3271,7 @@
     }
     len = strlen(str);
     readline_set_completion_index(rs, len);
-    for (i = 0; i < Q_KEY_CODE_MAX; i++) {
+    for (i = 0; i < Q_KEY_CODE__MAX; i++) {
         if (!strncmp(str, QKeyCode_lookup[i], len)) {
             readline_add_completion(rs, QKeyCode_lookup[i]);
         }
@@ -4495,11 +3285,12 @@
     len = strlen(str);
     readline_set_completion_index(rs, len);
     if (nb_args == 2) {
-        NetClientState *ncs[255];
+        NetClientState *ncs[MAX_QUEUE_NUM];
         int count, i;
         count = qemu_find_net_clients_except(NULL, ncs,
-                                             NET_CLIENT_OPTIONS_KIND_NONE, 255);
-        for (i = 0; i < count; i++) {
+                                             NET_CLIENT_DRIVER_NONE,
+                                             MAX_QUEUE_NUM);
+        for (i = 0; i < MIN(count, MAX_QUEUE_NUM); i++) {
             const char *name = ncs[i]->name;
             if (!strncmp(str, name, len)) {
                 readline_add_completion(rs, name);
@@ -4514,7 +3305,7 @@
 void netdev_del_completion(ReadLineState *rs, int nb_args, const char *str)
 {
     int len, count, i;
-    NetClientState *ncs[255];
+    NetClientState *ncs[MAX_QUEUE_NUM];
 
     if (nb_args != 2) {
         return;
@@ -4522,9 +3313,9 @@
 
     len = strlen(str);
     readline_set_completion_index(rs, len);
-    count = qemu_find_net_clients_except(NULL, ncs, NET_CLIENT_OPTIONS_KIND_NIC,
-                                         255);
-    for (i = 0; i < count; i++) {
+    count = qemu_find_net_clients_except(NULL, ncs, NET_CLIENT_DRIVER_NIC,
+                                         MAX_QUEUE_NUM);
+    for (i = 0; i < MIN(count, MAX_QUEUE_NUM); i++) {
         QemuOpts *opts;
         const char *name = ncs[i]->name;
         if (strncmp(str, name, len)) {
@@ -4537,6 +3328,43 @@
     }
 }
 
+void info_trace_events_completion(ReadLineState *rs, int nb_args, const char *str)
+{
+    size_t len;
+
+    len = strlen(str);
+    readline_set_completion_index(rs, len);
+    if (nb_args == 2) {
+        TraceEventID id;
+        for (id = 0; id < trace_event_count(); id++) {
+            const char *event_name = trace_event_get_name(trace_event_id(id));
+            if (!strncmp(str, event_name, len)) {
+                readline_add_completion(rs, event_name);
+            }
+        }
+    }
+}
+
+void trace_event_completion(ReadLineState *rs, int nb_args, const char *str)
+{
+    size_t len;
+
+    len = strlen(str);
+    readline_set_completion_index(rs, len);
+    if (nb_args == 2) {
+        TraceEventID id;
+        for (id = 0; id < trace_event_count(); id++) {
+            const char *event_name = trace_event_get_name(trace_event_id(id));
+            if (!strncmp(str, event_name, len)) {
+                readline_add_completion(rs, event_name);
+            }
+        }
+    } else if (nb_args == 3) {
+        add_completion_option(rs, str, "on");
+        add_completion_option(rs, str, "off");
+    }
+}
+
 void watchdog_action_completion(ReadLineState *rs, int nb_args, const char *str)
 {
     int i;
@@ -4559,7 +3387,7 @@
     readline_set_completion_index(rs, len);
     if (nb_args == 2) {
         int i;
-        for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) {
+        for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) {
             const char *name = MigrationCapability_lookup[i];
             if (!strncmp(str, name, len)) {
                 readline_add_completion(rs, name);
@@ -4571,6 +3399,24 @@
     }
 }
 
+void migrate_set_parameter_completion(ReadLineState *rs, int nb_args,
+                                      const char *str)
+{
+    size_t len;
+
+    len = strlen(str);
+    readline_set_completion_index(rs, len);
+    if (nb_args == 2) {
+        int i;
+        for (i = 0; i < MIGRATION_PARAMETER__MAX; i++) {
+            const char *name = MigrationParameter_lookup[i];
+            if (!strncmp(str, name, len)) {
+                readline_add_completion(rs, name);
+            }
+        }
+    }
+}
+
 void host_net_add_completion(ReadLineState *rs, int nb_args, const char *str)
 {
     int i;
@@ -4589,15 +3435,16 @@
 
 void host_net_remove_completion(ReadLineState *rs, int nb_args, const char *str)
 {
-    NetClientState *ncs[255];
+    NetClientState *ncs[MAX_QUEUE_NUM];
     int count, i, len;
 
     len = strlen(str);
     readline_set_completion_index(rs, len);
     if (nb_args == 2) {
         count = qemu_find_net_clients_except(NULL, ncs,
-                                             NET_CLIENT_OPTIONS_KIND_NONE, 255);
-        for (i = 0; i < count; i++) {
+                                             NET_CLIENT_DRIVER_NONE,
+                                             MAX_QUEUE_NUM);
+        for (i = 0; i < MIN(count, MAX_QUEUE_NUM); i++) {
             int id;
             char name[16];
 
@@ -4612,10 +3459,16 @@
         return;
     } else if (nb_args == 3) {
         count = qemu_find_net_clients_except(NULL, ncs,
-                                             NET_CLIENT_OPTIONS_KIND_NIC, 255);
-        for (i = 0; i < count; i++) {
+                                             NET_CLIENT_DRIVER_NIC,
+                                             MAX_QUEUE_NUM);
+        for (i = 0; i < MIN(count, MAX_QUEUE_NUM); i++) {
+            int id;
             const char *name;
 
+            if (ncs[i]->info->type == NET_CLIENT_DRIVER_HUBPORT ||
+                net_hub_id_for_client(ncs[i], &id)) {
+                continue;
+            }
             name = ncs[i]->name;
             if (!strncmp(str, name, len)) {
                 readline_add_completion(rs, name);
@@ -4628,19 +3481,26 @@
 static void vm_completion(ReadLineState *rs, const char *str)
 {
     size_t len;
-    BlockDriverState *bs = NULL;
+    BlockDriverState *bs;
+    BdrvNextIterator it;
 
     len = strlen(str);
     readline_set_completion_index(rs, len);
-    while ((bs = bdrv_next(bs))) {
-        SnapshotInfoList *snapshots, *snapshot;
 
-        if (!bdrv_can_snapshot(bs)) {
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        SnapshotInfoList *snapshots, *snapshot;
+        AioContext *ctx = bdrv_get_aio_context(bs);
+        bool ok = false;
+
+        aio_context_acquire(ctx);
+        if (bdrv_can_snapshot(bs)) {
+            ok = bdrv_query_snapshot_info_list(bs, &snapshots, NULL) == 0;
+        }
+        aio_context_release(ctx);
+        if (!ok) {
             continue;
         }
-        if (bdrv_query_snapshot_info_list(bs, &snapshots, NULL)) {
-            continue;
-        }
+
         snapshot = snapshots;
         while (snapshot) {
             char *completion = snapshot->value->name;
@@ -4681,7 +3541,7 @@
     int i;
     const char *ptype, *str, *name;
     const mon_cmd_t *cmd;
-    BlockDriverState *bs;
+    BlockBackend *blk = NULL;
 
     if (nb_args <= 1) {
         /* command completion */
@@ -4694,7 +3554,6 @@
             cmd_completion(mon, cmdname, cmd->name);
         }
     } else {
-        mon_cmd_t *sub_cmds;
         /* find the command */
         for (cmd = cmd_table; cmd->name != NULL; cmd++) {
             if (compare_cmd(args[0], cmd->name)) {
@@ -4705,14 +3564,15 @@
             return;
         }
 
-        sub_cmds = get_command_table(mon, cmd->sub_cmds);
-        if (sub_cmds) {
+        if (cmd->sub_table) {
             /* do the job again */
-            return monitor_find_completion_by_table(mon, sub_cmds,
-                                                    &args[1], nb_args - 1);
+            monitor_find_completion_by_table(mon, cmd->sub_table,
+                                             &args[1], nb_args - 1);
+            return;
         }
         if (cmd->command_completion) {
-            return cmd->command_completion(mon->rs, nb_args, args[nb_args - 1]);
+            cmd->command_completion(mon->rs, nb_args, args[nb_args - 1]);
+            return;
         }
 
         ptype = next_arg_type(cmd->args_type);
@@ -4724,7 +3584,7 @@
             }
         }
         str = args[nb_args - 1];
-        if (*ptype == '-' && ptype[1] != '\0') {
+        while (*ptype == '-' && ptype[1] != '\0') {
             ptype = next_arg_type(ptype);
         }
         switch(*ptype) {
@@ -4736,8 +3596,8 @@
         case 'B':
             /* block device name completion */
             readline_set_completion_index(mon->rs, strlen(str));
-            for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-                name = bdrv_get_device_name(bs);
+            while ((blk = blk_next(blk)) != NULL) {
+                name = blk_name(blk);
                 if (str[0] == '\0' ||
                     !strncmp(name, str, strlen(str))) {
                     readline_add_completion(mon->rs, name);
@@ -4768,14 +3628,6 @@
     if (parse_cmdline(cmdline, &nb_args, args) < 0) {
         return;
     }
-#ifdef DEBUG_COMPLETION
-    {
-        int i;
-        for (i = 0; i < nb_args; i++) {
-            monitor_printf(mon, "arg%d = '%s'\n", i, args[i]);
-        }
-    }
-#endif
 
     /* if the line ends with a space, it means we want to complete the
        next arg */
@@ -4788,9 +3640,7 @@
     }
 
     /* 2. auto complete according to args */
-    monitor_find_completion_by_table(mon,
-                                     get_command_table(mon, mon->cmds),
-                                     args, nb_args);
+    monitor_find_completion_by_table(mon, mon->cmd_table, args, nb_args);
 
 cleanup:
     free_cmdline_args(args, nb_args);
@@ -4803,10 +3653,24 @@
     return (mon->suspend_cnt == 0) ? 1 : 0;
 }
 
-static int invalid_qmp_mode(const Monitor *mon, const char *cmd_name)
+static bool invalid_qmp_mode(const Monitor *mon, const mon_cmd_t *cmd,
+                             Error **errp)
 {
-    int is_cap = compare_cmd(cmd_name, "qmp_capabilities");
-    return (qmp_cmd_mode(mon) ? is_cap : !is_cap);
+    bool is_cap = cmd->mhandler.cmd_new == qmp_capabilities;
+
+    if (is_cap && mon->qmp.in_command_mode) {
+        error_set(errp, ERROR_CLASS_COMMAND_NOT_FOUND,
+                  "Capabilities negotiation is already complete, command "
+                  "'%s' ignored", cmd->name);
+        return true;
+    }
+    if (!is_cap && !mon->qmp.in_command_mode) {
+        error_set(errp, ERROR_CLASS_COMMAND_NOT_FOUND,
+                  "Expecting capabilities negotiation with "
+                  "'qmp_capabilities' before command '%s'", cmd->name);
+        return true;
+    }
+    return false;
 }
 
 /*
@@ -4819,8 +3683,9 @@
  *               the QMP_ACCEPT_UNKNOWNS flag is set, then the
  *               checking is skipped for it.
  */
-static int check_client_args_type(const QDict *client_args,
-                                  const QDict *cmd_args, int flags)
+static void check_client_args_type(const QDict *client_args,
+                                   const QDict *cmd_args, int flags,
+                                   Error **errp)
 {
     const QDictEntry *ent;
 
@@ -4837,8 +3702,8 @@
                 continue;
             }
             /* client arg doesn't exist */
-            qerror_report(QERR_INVALID_PARAMETER, client_arg_name);
-            return -1;
+            error_setg(errp, QERR_INVALID_PARAMETER, client_arg_name);
+            return;
         }
 
         arg_type = qobject_to_qstring(obj);
@@ -4850,9 +3715,9 @@
         case 'B':
         case 's':
             if (qobject_type(client_arg) != QTYPE_QSTRING) {
-                qerror_report(QERR_INVALID_PARAMETER_TYPE, client_arg_name,
-                              "string");
-                return -1;
+                error_setg(errp, QERR_INVALID_PARAMETER_TYPE,
+                           client_arg_name, "string");
+                return;
             }
         break;
         case 'i':
@@ -4860,25 +3725,25 @@
         case 'M':
         case 'o':
             if (qobject_type(client_arg) != QTYPE_QINT) {
-                qerror_report(QERR_INVALID_PARAMETER_TYPE, client_arg_name,
-                              "int");
-                return -1; 
+                error_setg(errp, QERR_INVALID_PARAMETER_TYPE,
+                           client_arg_name, "int");
+                return;
             }
             break;
         case 'T':
             if (qobject_type(client_arg) != QTYPE_QINT &&
                 qobject_type(client_arg) != QTYPE_QFLOAT) {
-                qerror_report(QERR_INVALID_PARAMETER_TYPE, client_arg_name,
-                              "number");
-               return -1; 
+                error_setg(errp, QERR_INVALID_PARAMETER_TYPE,
+                           client_arg_name, "number");
+                return;
             }
             break;
         case 'b':
         case '-':
             if (qobject_type(client_arg) != QTYPE_QBOOL) {
-                qerror_report(QERR_INVALID_PARAMETER_TYPE, client_arg_name,
-                              "bool");
-               return -1; 
+                error_setg(errp, QERR_INVALID_PARAMETER_TYPE,
+                           client_arg_name, "bool");
+                return;
             }
             break;
         case 'O':
@@ -4897,16 +3762,15 @@
             abort();
         }
     }
-
-    return 0;
 }
 
 /*
  * - Check if the client has passed all mandatory args
  * - Set special flags for argument validation
  */
-static int check_mandatory_args(const QDict *cmd_args,
-                                const QDict *client_args, int *flags)
+static void check_mandatory_args(const QDict *cmd_args,
+                                 const QDict *client_args, int *flags,
+                                 Error **errp)
 {
     const QDictEntry *ent;
 
@@ -4921,12 +3785,10 @@
         } else if (qstring_get_str(type)[0] != '-' &&
                    qstring_get_str(type)[1] != '?' &&
                    !qdict_haskey(client_args, cmd_arg_name)) {
-            qerror_report(QERR_MISSING_PARAMETER, cmd_arg_name);
-            return -1;
+            error_setg(errp, QERR_MISSING_PARAMETER, cmd_arg_name);
+            return;
         }
     }
-
-    return 0;
 }
 
 static QDict *qdict_from_args_type(const char *args_type)
@@ -4982,24 +3844,26 @@
  * 3. Each argument provided by the client must have the type expected
  *    by the command
  */
-static int qmp_check_client_args(const mon_cmd_t *cmd, QDict *client_args)
+static void qmp_check_client_args(const mon_cmd_t *cmd, QDict *client_args,
+                                  Error **errp)
 {
-    int flags, err;
+    Error *err = NULL;
+    int flags;
     QDict *cmd_args;
 
     cmd_args = qdict_from_args_type(cmd->args_type);
 
     flags = 0;
-    err = check_mandatory_args(cmd_args, client_args, &flags);
+    check_mandatory_args(cmd_args, client_args, &flags, &err);
     if (err) {
         goto out;
     }
 
-    err = check_client_args_type(client_args, cmd_args, flags);
+    check_client_args_type(client_args, cmd_args, flags, &err);
 
 out:
+    error_propagate(errp, err);
     QDECREF(cmd_args);
-    return err;
 }
 
 /*
@@ -5012,14 +3876,14 @@
  * 5. If the "id" key exists, it can be anything (ie. json-value)
  * 6. Any argument not listed above is considered invalid
  */
-static QDict *qmp_check_input_obj(QObject *input_obj)
+static QDict *qmp_check_input_obj(QObject *input_obj, Error **errp)
 {
     const QDictEntry *ent;
     int has_exec_key = 0;
     QDict *input_dict;
 
     if (qobject_type(input_obj) != QTYPE_QDICT) {
-        qerror_report(QERR_QMP_BAD_INPUT_OBJECT, "object");
+        error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT, "object");
         return NULL;
     }
 
@@ -5031,82 +3895,70 @@
 
         if (!strcmp(arg_name, "execute")) {
             if (qobject_type(arg_obj) != QTYPE_QSTRING) {
-                qerror_report(QERR_QMP_BAD_INPUT_OBJECT_MEMBER, "execute",
-                              "string");
+                error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT_MEMBER,
+                           "execute", "string");
                 return NULL;
             }
             has_exec_key = 1;
         } else if (!strcmp(arg_name, "arguments")) {
             if (qobject_type(arg_obj) != QTYPE_QDICT) {
-                qerror_report(QERR_QMP_BAD_INPUT_OBJECT_MEMBER, "arguments",
-                              "object");
+                error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT_MEMBER,
+                           "arguments", "object");
                 return NULL;
             }
         } else if (!strcmp(arg_name, "id")) {
-            /* FIXME: check duplicated IDs for async commands */
+            /* Any string is acceptable as "id", so nothing to check */
         } else {
-            qerror_report(QERR_QMP_EXTRA_MEMBER, arg_name);
+            error_setg(errp, QERR_QMP_EXTRA_MEMBER, arg_name);
             return NULL;
         }
     }
 
     if (!has_exec_key) {
-        qerror_report(QERR_QMP_BAD_INPUT_OBJECT, "execute");
+        error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT, "execute");
         return NULL;
     }
 
     return input_dict;
 }
 
-static void qmp_call_cmd(Monitor *mon, const mon_cmd_t *cmd,
-                         const QDict *params)
+static void handle_qmp_command(JSONMessageParser *parser, GQueue *tokens)
 {
-    int ret;
-    QObject *data = NULL;
-
-    ret = cmd->mhandler.cmd_new(mon, params, &data);
-    handler_audit(mon, cmd, ret);
-    monitor_protocol_emitter(mon, data);
-    qobject_decref(data);
-}
-
-static void handle_qmp_command(JSONMessageParser *parser, QList *tokens)
-{
-    int err;
-    QObject *obj;
+    Error *local_err = NULL;
+    QObject *obj, *data;
     QDict *input, *args;
     const mon_cmd_t *cmd;
     const char *cmd_name;
     Monitor *mon = cur_mon;
 
     args = input = NULL;
+    data = NULL;
 
     obj = json_parser_parse(tokens, NULL);
     if (!obj) {
         // FIXME: should be triggered in json_parser_parse()
-        qerror_report(QERR_JSON_PARSING);
+        error_setg(&local_err, QERR_JSON_PARSING);
         goto err_out;
     }
 
-    input = qmp_check_input_obj(obj);
+    input = qmp_check_input_obj(obj, &local_err);
     if (!input) {
         qobject_decref(obj);
         goto err_out;
     }
 
-    mon->mc->id = qdict_get(input, "id");
-    qobject_incref(mon->mc->id);
+    mon->qmp.id = qdict_get(input, "id");
+    qobject_incref(mon->qmp.id);
 
     cmd_name = qdict_get_str(input, "execute");
     trace_handle_qmp_command(mon, cmd_name);
-    if (invalid_qmp_mode(mon, cmd_name)) {
-        qerror_report(QERR_COMMAND_NOT_FOUND, cmd_name);
-        goto err_out;
-    }
-
     cmd = qmp_find_cmd(cmd_name);
     if (!cmd) {
-        qerror_report(QERR_COMMAND_NOT_FOUND, cmd_name);
+        error_set(&local_err, ERROR_CLASS_COMMAND_NOT_FOUND,
+                  "The command %s has not been found", cmd_name);
+        goto err_out;
+    }
+    if (invalid_qmp_mode(mon, cmd, &local_err)) {
         goto err_out;
     }
 
@@ -5118,40 +3970,28 @@
         QINCREF(args);
     }
 
-    err = qmp_check_client_args(cmd, args);
-    if (err < 0) {
+    qmp_check_client_args(cmd, args, &local_err);
+    if (local_err) {
         goto err_out;
     }
 
-    if (handler_is_async(cmd)) {
-        err = qmp_async_cmd_handler(mon, cmd, args);
-        if (err) {
-            /* emit the error response */
-            goto err_out;
-        }
-    } else {
-        qmp_call_cmd(mon, cmd, args);
-    }
-
-    goto out;
+    cmd->mhandler.cmd_new(args, &data, &local_err);
 
 err_out:
-    monitor_protocol_emitter(mon, NULL);
-out:
+    monitor_protocol_emitter(mon, data, local_err);
+    qobject_decref(data);
+    error_free(local_err);
     QDECREF(input);
     QDECREF(args);
 }
 
-/**
- * monitor_control_read(): Read and handle QMP input
- */
-static void monitor_control_read(void *opaque, const uint8_t *buf, int size)
+static void monitor_qmp_read(void *opaque, const uint8_t *buf, int size)
 {
     Monitor *old_mon = cur_mon;
 
     cur_mon = opaque;
 
-    json_message_parser_feed(&cur_mon->mc->parser, (const char *) buf, size);
+    json_message_parser_feed(&cur_mon->qmp.parser, (const char *) buf, size);
 
     cur_mon = old_mon;
 }
@@ -5170,7 +4010,7 @@
         if (size == 0 || buf[size - 1] != 0)
             monitor_printf(cur_mon, "corrupted command\n");
         else
-            handle_user_command(cur_mon, (char *)buf);
+            handle_hmp_command(cur_mon, (char *)buf);
     }
 
     cur_mon = old_mon;
@@ -5182,7 +4022,7 @@
     Monitor *mon = opaque;
 
     monitor_suspend(mon);
-    handle_user_command(mon, cmdline);
+    handle_hmp_command(mon, cmdline);
     monitor_resume(mon);
 }
 
@@ -5206,29 +4046,26 @@
 {
     QObject *ver = NULL;
 
-    qmp_marshal_input_query_version(NULL, NULL, &ver);
+    qmp_marshal_query_version(NULL, &ver, NULL);
     return qobject_from_jsonf("{'QMP':{'version': %p,'capabilities': []}}",ver);
 }
 
-/**
- * monitor_control_event(): Print QMP gretting
- */
-static void monitor_control_event(void *opaque, int event)
+static void monitor_qmp_event(void *opaque, int event)
 {
     QObject *data;
     Monitor *mon = opaque;
 
     switch (event) {
     case CHR_EVENT_OPENED:
-        mon->mc->command_mode = 0;
+        mon->qmp.in_command_mode = false;
         data = get_qmp_greeting();
         monitor_json_emitter(mon, data);
         qobject_decref(data);
         mon_refcount++;
         break;
     case CHR_EVENT_CLOSED:
-        json_message_parser_destroy(&mon->mc->parser);
-        json_message_parser_init(&mon->mc->parser, handle_qmp_command);
+        json_message_parser_destroy(&mon->qmp.parser);
+        json_message_parser_init(&mon->qmp.parser, handle_qmp_command);
         mon_refcount--;
         monitor_fdsets_cleanup();
         break;
@@ -5305,15 +4142,6 @@
     qsort((void *)info_cmds, array_num, elem_size, compare_mon_cmd);
 }
 
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- *  tab-width: 8
- * End:
- */
-
 /* These functions just adapt the readline interface in a typesafe way.  We
  * could cast function pointers but that discards compiler checks.
  */
@@ -5336,48 +4164,7 @@
     qemu_mutex_init(&monitor_lock);
 }
 
-void monitor_add_command(Monitor *mon, mon_cmd_t *cmd)
-{
-    if (mon->flags & MONITOR_DYNAMIC_CMDS) {
-        qemu_mutex_lock(&monitor_lock);
-        g_array_append_val(mon->cmds.dynamic_table, *cmd);
-        qemu_mutex_unlock(&monitor_lock);
-    } else {
-        monitor_printf(mon,
-                       "Unable to add %s to non-dynamic monitor\n",
-                       cmd->name);
-    }
-}
-
-/* Create a dynamic copy of a static command table */
-static GArray * make_dynamic_table(mon_cmd_t *cmds)
-{
-    int i = 0;
-    GArray *cmd_array;
-
-    while (cmds[i].name) {
-        i++;
-    }
-
-    cmd_array = g_array_sized_new(true, true,
-                                  sizeof(mon_cmd_t),
-                                  i);
-    g_array_append_vals(cmd_array, cmds, i);
-
-    /* Now go through the array and convert any static sub_cmd tables
-     * into dynamic ones */
-    for (i=0; i<cmd_array->len; i++) {
-        mon_cmd_t *cmd = &g_array_index(cmd_array, mon_cmd_t, i);
-        if (cmd->sub_cmds.static_table) {
-            cmd->sub_cmds.dynamic_table =
-                make_dynamic_table(cmd->sub_cmds.static_table);
-        }
-    }
-
-    return cmd_array;
-}
-
-Monitor * monitor_init(CharDriverState *chr, int flags)
+void monitor_init(CharDriverState *chr, int flags)
 {
     static int is_first_init = 1;
     Monitor *mon;
@@ -5391,10 +4178,6 @@
     mon = g_malloc(sizeof(*mon));
     monitor_data_init(mon);
 
-    if (flags & MONITOR_DYNAMIC_CMDS) {
-        mon->cmds.dynamic_table = make_dynamic_table(mon->cmds.static_table);
-    }
-
     mon->chr = chr;
     mon->flags = flags;
     if (flags & MONITOR_USE_READLINE) {
@@ -5405,14 +4188,11 @@
         monitor_read_command(mon, 0);
     }
 
-    if (monitor_ctrl_mode(mon)) {
-        mon->mc = g_malloc0(sizeof(MonitorControl));
-        /* Control mode requires special handlers */
-        qemu_chr_add_handlers(chr, monitor_can_read, monitor_control_read,
-                              monitor_control_event, mon);
+    if (monitor_is_qmp(mon)) {
+        qemu_chr_add_handlers(chr, monitor_can_read, monitor_qmp_read,
+                              monitor_qmp_event, mon);
         qemu_chr_fe_set_echo(chr, true);
-
-        json_message_parser_init(&mon->mc->parser, handle_qmp_command);
+        json_message_parser_init(&mon->qmp.parser, handle_qmp_command);
     } else {
         qemu_chr_add_handlers(chr, monitor_can_read, monitor_read,
                               monitor_event, mon);
@@ -5421,11 +4201,19 @@
     qemu_mutex_lock(&monitor_lock);
     QLIST_INSERT_HEAD(&mon_list, mon, entry);
     qemu_mutex_unlock(&monitor_lock);
+}
 
-    if (!default_mon || (flags & MONITOR_IS_DEFAULT))
-        default_mon = mon;
+void monitor_cleanup(void)
+{
+    Monitor *mon, *next;
 
-    return mon;
+    qemu_mutex_lock(&monitor_lock);
+    QLIST_FOREACH_SAFE(mon, &mon_list, entry, next) {
+        QLIST_REMOVE(mon, entry);
+        monitor_data_destroy(mon);
+        g_free(mon);
+    }
+    qemu_mutex_unlock(&monitor_lock);
 }
 
 static void bdrv_password_cb(void *opaque, const char *password,
@@ -5434,9 +4222,11 @@
     Monitor *mon = opaque;
     BlockDriverState *bs = readline_opaque;
     int ret = 0;
+    Error *local_err = NULL;
 
-    if (bdrv_set_key(bs, password) != 0) {
-        monitor_printf(mon, "invalid password\n");
+    bdrv_add_key(bs, password, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
         ret = -EPERM;
     }
     if (mon->password_completion_cb)
@@ -5445,29 +4235,12 @@
     monitor_read_command(mon, 1);
 }
 
-ReadLineState *monitor_get_rs(Monitor *mon)
-{
-    return mon->rs;
-}
-
 int monitor_read_bdrv_key_start(Monitor *mon, BlockDriverState *bs,
                                 BlockCompletionFunc *completion_cb,
                                 void *opaque)
 {
     int err;
 
-    if (!bdrv_key_required(bs)) {
-        if (completion_cb)
-            completion_cb(opaque, 0);
-        return 0;
-    }
-
-    if (monitor_ctrl_mode(mon)) {
-        qerror_report(QERR_DEVICE_ENCRYPTED, bdrv_get_device_name(bs),
-                      bdrv_get_encrypted_filename(bs));
-        return -1;
-    }
-
     monitor_printf(mon, "%s (%s) is encrypted.\n", bdrv_get_device_name(bs),
                    bdrv_get_encrypted_filename(bs));
 
@@ -5486,15 +4259,29 @@
                                   BlockCompletionFunc *completion_cb,
                                   void *opaque)
 {
-    BlockDriverState *bs;
+    Error *err = NULL;
+    BlockBackend *blk;
 
-    bs = bdrv_find(device);
-    if (!bs) {
+    blk = blk_by_name(device);
+    if (!blk) {
         monitor_printf(mon, "Device not found %s\n", device);
         return -1;
     }
+    if (!blk_bs(blk)) {
+        monitor_printf(mon, "Device '%s' has no medium\n", device);
+        return -1;
+    }
 
-    return monitor_read_bdrv_key_start(mon, bs, completion_cb, opaque);
+    bdrv_add_key(blk_bs(blk), NULL, &err);
+    if (err) {
+        error_free(err);
+        return monitor_read_bdrv_key_start(mon, blk_bs(blk), completion_cb, opaque);
+    }
+
+    if (completion_cb) {
+        completion_cb(opaque, 0);
+    }
+    return 0;
 }
 
 QemuOptsList qemu_mon_opts = {
@@ -5522,6 +4309,34 @@
 #ifndef TARGET_I386
 void qmp_rtc_reset_reinjection(Error **errp)
 {
-    error_set(errp, QERR_FEATURE_DISABLED, "rtc-reset-reinjection");
+    error_setg(errp, QERR_FEATURE_DISABLED, "rtc-reset-reinjection");
 }
 #endif
+
+#ifndef TARGET_S390X
+void qmp_dump_skeys(const char *filename, Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "dump-skeys");
+}
+#endif
+
+#ifndef TARGET_ARM
+GICCapabilityList *qmp_query_gic_capabilities(Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "query-gic-capabilities");
+    return NULL;
+}
+#endif
+
+HotpluggableCPUList *qmp_query_hotpluggable_cpus(Error **errp)
+{
+    MachineState *ms = MACHINE(qdev_get_machine());
+    MachineClass *mc = MACHINE_GET_CLASS(ms);
+
+    if (!mc->query_hotpluggable_cpus) {
+        error_setg(errp, QERR_FEATURE_DISABLED, "query-hotpluggable-cpus");
+        return NULL;
+    }
+
+    return mc->query_hotpluggable_cpus(ms);
+}
diff --git a/nbd.c b/nbd.c
deleted file mode 100644
index a7bce45..0000000
--- a/nbd.c
+++ /dev/null
@@ -1,1379 +0,0 @@
-/*
- *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
- *
- *  Network Block Device
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; under version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "block/nbd.h"
-#include "block/block.h"
-#include "block/block_int.h"
-
-#include "block/coroutine.h"
-
-#include <errno.h>
-#include <string.h>
-#ifndef _WIN32
-#include <sys/ioctl.h>
-#endif
-#if defined(__sun__) || defined(__HAIKU__)
-#include <sys/ioccom.h>
-#endif
-#include <ctype.h>
-#include <inttypes.h>
-
-#ifdef __linux__
-#include <linux/fs.h>
-#endif
-
-#include "qemu/sockets.h"
-#include "qemu/queue.h"
-#include "qemu/main-loop.h"
-
-//#define DEBUG_NBD
-
-#ifdef DEBUG_NBD
-#define TRACE(msg, ...) do { \
-    LOG(msg, ## __VA_ARGS__); \
-} while(0)
-#else
-#define TRACE(msg, ...) \
-    do { } while (0)
-#endif
-
-#define LOG(msg, ...) do { \
-    fprintf(stderr, "%s:%s():L%d: " msg "\n", \
-            __FILE__, __FUNCTION__, __LINE__, ## __VA_ARGS__); \
-} while(0)
-
-/* This is all part of the "official" NBD API.
- *
- * The most up-to-date documentation is available at:
- * https://github.com/yoe/nbd/blob/master/doc/proto.txt
- */
-
-#define NBD_REQUEST_SIZE        (4 + 4 + 8 + 8 + 4)
-#define NBD_REPLY_SIZE          (4 + 4 + 8)
-#define NBD_REQUEST_MAGIC       0x25609513
-#define NBD_REPLY_MAGIC         0x67446698
-#define NBD_OPTS_MAGIC          0x49484156454F5054LL
-#define NBD_CLIENT_MAGIC        0x0000420281861253LL
-#define NBD_REP_MAGIC           0x3e889045565a9LL
-
-#define NBD_SET_SOCK            _IO(0xab, 0)
-#define NBD_SET_BLKSIZE         _IO(0xab, 1)
-#define NBD_SET_SIZE            _IO(0xab, 2)
-#define NBD_DO_IT               _IO(0xab, 3)
-#define NBD_CLEAR_SOCK          _IO(0xab, 4)
-#define NBD_CLEAR_QUE           _IO(0xab, 5)
-#define NBD_PRINT_DEBUG         _IO(0xab, 6)
-#define NBD_SET_SIZE_BLOCKS     _IO(0xab, 7)
-#define NBD_DISCONNECT          _IO(0xab, 8)
-#define NBD_SET_TIMEOUT         _IO(0xab, 9)
-#define NBD_SET_FLAGS           _IO(0xab, 10)
-
-#define NBD_OPT_EXPORT_NAME     (1)
-#define NBD_OPT_ABORT           (2)
-#define NBD_OPT_LIST            (3)
-
-/* Definitions for opaque data types */
-
-typedef struct NBDRequest NBDRequest;
-
-struct NBDRequest {
-    QSIMPLEQ_ENTRY(NBDRequest) entry;
-    NBDClient *client;
-    uint8_t *data;
-};
-
-struct NBDExport {
-    int refcount;
-    void (*close)(NBDExport *exp);
-
-    BlockDriverState *bs;
-    char *name;
-    off_t dev_offset;
-    off_t size;
-    uint32_t nbdflags;
-    QTAILQ_HEAD(, NBDClient) clients;
-    QTAILQ_ENTRY(NBDExport) next;
-
-    AioContext *ctx;
-};
-
-static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
-
-struct NBDClient {
-    int refcount;
-    void (*close)(NBDClient *client);
-
-    NBDExport *exp;
-    int sock;
-
-    Coroutine *recv_coroutine;
-
-    CoMutex send_lock;
-    Coroutine *send_coroutine;
-
-    bool can_read;
-
-    QTAILQ_ENTRY(NBDClient) next;
-    int nb_requests;
-    bool closing;
-};
-
-/* That's all folks */
-
-static void nbd_set_handlers(NBDClient *client);
-static void nbd_unset_handlers(NBDClient *client);
-static void nbd_update_can_read(NBDClient *client);
-
-ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
-{
-    size_t offset = 0;
-    int err;
-
-    if (qemu_in_coroutine()) {
-        if (do_read) {
-            return qemu_co_recv(fd, buffer, size);
-        } else {
-            return qemu_co_send(fd, buffer, size);
-        }
-    }
-
-    while (offset < size) {
-        ssize_t len;
-
-        if (do_read) {
-            len = qemu_recv(fd, buffer + offset, size - offset, 0);
-        } else {
-            len = send(fd, buffer + offset, size - offset, 0);
-        }
-
-        if (len < 0) {
-            err = socket_error();
-
-            /* recoverable error */
-            if (err == EINTR || (offset > 0 && (err == EAGAIN || err == EWOULDBLOCK))) {
-                continue;
-            }
-
-            /* unrecoverable error */
-            return -err;
-        }
-
-        /* eof */
-        if (len == 0) {
-            break;
-        }
-
-        offset += len;
-    }
-
-    return offset;
-}
-
-static ssize_t read_sync(int fd, void *buffer, size_t size)
-{
-    /* Sockets are kept in blocking mode in the negotiation phase.  After
-     * that, a non-readable socket simply means that another thread stole
-     * our request/reply.  Synchronization is done with recv_coroutine, so
-     * that this is coroutine-safe.
-     */
-    return nbd_wr_sync(fd, buffer, size, true);
-}
-
-static ssize_t write_sync(int fd, void *buffer, size_t size)
-{
-    int ret;
-    do {
-        /* For writes, we do expect the socket to be writable.  */
-        ret = nbd_wr_sync(fd, buffer, size, false);
-    } while (ret == -EAGAIN);
-    return ret;
-}
-
-/* Basic flow for negotiation
-
-   Server         Client
-   Negotiate
-
-   or
-
-   Server         Client
-   Negotiate #1
-                  Option
-   Negotiate #2
-
-   ----
-
-   followed by
-
-   Server         Client
-                  Request
-   Response
-                  Request
-   Response
-                  ...
-   ...
-                  Request (type == 2)
-
-*/
-
-static int nbd_send_rep(int csock, uint32_t type, uint32_t opt)
-{
-    uint64_t magic;
-    uint32_t len;
-
-    magic = cpu_to_be64(NBD_REP_MAGIC);
-    if (write_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-        LOG("write failed (rep magic)");
-        return -EINVAL;
-    }
-    opt = cpu_to_be32(opt);
-    if (write_sync(csock, &opt, sizeof(opt)) != sizeof(opt)) {
-        LOG("write failed (rep opt)");
-        return -EINVAL;
-    }
-    type = cpu_to_be32(type);
-    if (write_sync(csock, &type, sizeof(type)) != sizeof(type)) {
-        LOG("write failed (rep type)");
-        return -EINVAL;
-    }
-    len = cpu_to_be32(0);
-    if (write_sync(csock, &len, sizeof(len)) != sizeof(len)) {
-        LOG("write failed (rep data length)");
-        return -EINVAL;
-    }
-    return 0;
-}
-
-static int nbd_send_rep_list(int csock, NBDExport *exp)
-{
-    uint64_t magic, name_len;
-    uint32_t opt, type, len;
-
-    name_len = strlen(exp->name);
-    magic = cpu_to_be64(NBD_REP_MAGIC);
-    if (write_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-        LOG("write failed (magic)");
-        return -EINVAL;
-     }
-    opt = cpu_to_be32(NBD_OPT_LIST);
-    if (write_sync(csock, &opt, sizeof(opt)) != sizeof(opt)) {
-        LOG("write failed (opt)");
-        return -EINVAL;
-    }
-    type = cpu_to_be32(NBD_REP_SERVER);
-    if (write_sync(csock, &type, sizeof(type)) != sizeof(type)) {
-        LOG("write failed (reply type)");
-        return -EINVAL;
-    }
-    len = cpu_to_be32(name_len + sizeof(len));
-    if (write_sync(csock, &len, sizeof(len)) != sizeof(len)) {
-        LOG("write failed (length)");
-        return -EINVAL;
-    }
-    len = cpu_to_be32(name_len);
-    if (write_sync(csock, &len, sizeof(len)) != sizeof(len)) {
-        LOG("write failed (length)");
-        return -EINVAL;
-    }
-    if (write_sync(csock, exp->name, name_len) != name_len) {
-        LOG("write failed (buffer)");
-        return -EINVAL;
-    }
-    return 0;
-}
-
-static int nbd_handle_list(NBDClient *client, uint32_t length)
-{
-    int csock;
-    NBDExport *exp;
-
-    csock = client->sock;
-    if (length) {
-        return nbd_send_rep(csock, NBD_REP_ERR_INVALID, NBD_OPT_LIST);
-    }
-
-    /* For each export, send a NBD_REP_SERVER reply. */
-    QTAILQ_FOREACH(exp, &exports, next) {
-        if (nbd_send_rep_list(csock, exp)) {
-            return -EINVAL;
-        }
-    }
-    /* Finish with a NBD_REP_ACK. */
-    return nbd_send_rep(csock, NBD_REP_ACK, NBD_OPT_LIST);
-}
-
-static int nbd_handle_export_name(NBDClient *client, uint32_t length)
-{
-    int rc = -EINVAL, csock = client->sock;
-    char name[256];
-
-    /* Client sends:
-        [20 ..  xx]   export name (length bytes)
-     */
-    TRACE("Checking length");
-    if (length > 255) {
-        LOG("Bad length received");
-        goto fail;
-    }
-    if (read_sync(csock, name, length) != length) {
-        LOG("read failed");
-        goto fail;
-    }
-    name[length] = '\0';
-
-    client->exp = nbd_export_find(name);
-    if (!client->exp) {
-        LOG("export not found");
-        goto fail;
-    }
-
-    QTAILQ_INSERT_TAIL(&client->exp->clients, client, next);
-    nbd_export_get(client->exp);
-    rc = 0;
-fail:
-    return rc;
-}
-
-static int nbd_receive_options(NBDClient *client)
-{
-    while (1) {
-        int csock = client->sock;
-        uint32_t tmp, length;
-        uint64_t magic;
-
-        /* Client sends:
-            [ 0 ..   3]   client flags
-            [ 4 ..  11]   NBD_OPTS_MAGIC
-            [12 ..  15]   NBD option
-            [16 ..  19]   length
-            ...           Rest of request
-        */
-
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-            LOG("read failed");
-            return -EINVAL;
-        }
-        TRACE("Checking client flags");
-        tmp = be32_to_cpu(tmp);
-        if (tmp != 0 && tmp != NBD_FLAG_C_FIXED_NEWSTYLE) {
-            LOG("Bad client flags received");
-            return -EINVAL;
-        }
-
-        if (read_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-            LOG("read failed");
-            return -EINVAL;
-        }
-        TRACE("Checking opts magic");
-        if (magic != be64_to_cpu(NBD_OPTS_MAGIC)) {
-            LOG("Bad magic received");
-            return -EINVAL;
-        }
-
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-            LOG("read failed");
-            return -EINVAL;
-        }
-
-        if (read_sync(csock, &length, sizeof(length)) != sizeof(length)) {
-            LOG("read failed");
-            return -EINVAL;
-        }
-        length = be32_to_cpu(length);
-
-        TRACE("Checking option");
-        switch (be32_to_cpu(tmp)) {
-        case NBD_OPT_LIST:
-            if (nbd_handle_list(client, length) < 0) {
-                return 1;
-            }
-            break;
-
-        case NBD_OPT_ABORT:
-            return -EINVAL;
-
-        case NBD_OPT_EXPORT_NAME:
-            return nbd_handle_export_name(client, length);
-
-        default:
-            tmp = be32_to_cpu(tmp);
-            LOG("Unsupported option 0x%x", tmp);
-            nbd_send_rep(client->sock, NBD_REP_ERR_UNSUP, tmp);
-            return -EINVAL;
-        }
-    }
-}
-
-static int nbd_send_negotiate(NBDClient *client)
-{
-    int csock = client->sock;
-    char buf[8 + 8 + 8 + 128];
-    int rc;
-    const int myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
-                         NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
-
-    /* Negotiation header without options:
-        [ 0 ..   7]   passwd       ("NBDMAGIC")
-        [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
-        [16 ..  23]   size
-        [24 ..  25]   server flags (0)
-        [26 ..  27]   export flags
-        [28 .. 151]   reserved     (0)
-
-       Negotiation header with options, part 1:
-        [ 0 ..   7]   passwd       ("NBDMAGIC")
-        [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
-        [16 ..  17]   server flags (0)
-
-       part 2 (after options are sent):
-        [18 ..  25]   size
-        [26 ..  27]   export flags
-        [28 .. 151]   reserved     (0)
-     */
-
-    qemu_set_block(csock);
-    rc = -EINVAL;
-
-    TRACE("Beginning negotiation.");
-    memset(buf, 0, sizeof(buf));
-    memcpy(buf, "NBDMAGIC", 8);
-    if (client->exp) {
-        assert ((client->exp->nbdflags & ~65535) == 0);
-        cpu_to_be64w((uint64_t*)(buf + 8), NBD_CLIENT_MAGIC);
-        cpu_to_be64w((uint64_t*)(buf + 16), client->exp->size);
-        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
-    } else {
-        cpu_to_be64w((uint64_t*)(buf + 8), NBD_OPTS_MAGIC);
-        cpu_to_be16w((uint16_t *)(buf + 16), NBD_FLAG_FIXED_NEWSTYLE);
-    }
-
-    if (client->exp) {
-        if (write_sync(csock, buf, sizeof(buf)) != sizeof(buf)) {
-            LOG("write failed");
-            goto fail;
-        }
-    } else {
-        if (write_sync(csock, buf, 18) != 18) {
-            LOG("write failed");
-            goto fail;
-        }
-        rc = nbd_receive_options(client);
-        if (rc != 0) {
-            LOG("option negotiation failed");
-            goto fail;
-        }
-
-        assert ((client->exp->nbdflags & ~65535) == 0);
-        cpu_to_be64w((uint64_t*)(buf + 18), client->exp->size);
-        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
-        if (write_sync(csock, buf + 18, sizeof(buf) - 18) != sizeof(buf) - 18) {
-            LOG("write failed");
-            goto fail;
-        }
-    }
-
-    TRACE("Negotiation succeeded.");
-    rc = 0;
-fail:
-    qemu_set_nonblock(csock);
-    return rc;
-}
-
-int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
-                          off_t *size, size_t *blocksize)
-{
-    char buf[256];
-    uint64_t magic, s;
-    uint16_t tmp;
-    int rc;
-
-    TRACE("Receiving negotiation.");
-
-    rc = -EINVAL;
-
-    if (read_sync(csock, buf, 8) != 8) {
-        LOG("read failed");
-        goto fail;
-    }
-
-    buf[8] = '\0';
-    if (strlen(buf) == 0) {
-        LOG("server connection closed");
-        goto fail;
-    }
-
-    TRACE("Magic is %c%c%c%c%c%c%c%c",
-          qemu_isprint(buf[0]) ? buf[0] : '.',
-          qemu_isprint(buf[1]) ? buf[1] : '.',
-          qemu_isprint(buf[2]) ? buf[2] : '.',
-          qemu_isprint(buf[3]) ? buf[3] : '.',
-          qemu_isprint(buf[4]) ? buf[4] : '.',
-          qemu_isprint(buf[5]) ? buf[5] : '.',
-          qemu_isprint(buf[6]) ? buf[6] : '.',
-          qemu_isprint(buf[7]) ? buf[7] : '.');
-
-    if (memcmp(buf, "NBDMAGIC", 8) != 0) {
-        LOG("Invalid magic received");
-        goto fail;
-    }
-
-    if (read_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-        LOG("read failed");
-        goto fail;
-    }
-    magic = be64_to_cpu(magic);
-    TRACE("Magic is 0x%" PRIx64, magic);
-
-    if (name) {
-        uint32_t reserved = 0;
-        uint32_t opt;
-        uint32_t namesize;
-
-        TRACE("Checking magic (opts_magic)");
-        if (magic != NBD_OPTS_MAGIC) {
-            LOG("Bad magic received");
-            goto fail;
-        }
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-            LOG("flags read failed");
-            goto fail;
-        }
-        *flags = be16_to_cpu(tmp) << 16;
-        /* reserved for future use */
-        if (write_sync(csock, &reserved, sizeof(reserved)) !=
-            sizeof(reserved)) {
-            LOG("write failed (reserved)");
-            goto fail;
-        }
-        /* write the export name */
-        magic = cpu_to_be64(magic);
-        if (write_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
-            LOG("write failed (magic)");
-            goto fail;
-        }
-        opt = cpu_to_be32(NBD_OPT_EXPORT_NAME);
-        if (write_sync(csock, &opt, sizeof(opt)) != sizeof(opt)) {
-            LOG("write failed (opt)");
-            goto fail;
-        }
-        namesize = cpu_to_be32(strlen(name));
-        if (write_sync(csock, &namesize, sizeof(namesize)) !=
-            sizeof(namesize)) {
-            LOG("write failed (namesize)");
-            goto fail;
-        }
-        if (write_sync(csock, (char*)name, strlen(name)) != strlen(name)) {
-            LOG("write failed (name)");
-            goto fail;
-        }
-    } else {
-        TRACE("Checking magic (cli_magic)");
-
-        if (magic != NBD_CLIENT_MAGIC) {
-            LOG("Bad magic received");
-            goto fail;
-        }
-    }
-
-    if (read_sync(csock, &s, sizeof(s)) != sizeof(s)) {
-        LOG("read failed");
-        goto fail;
-    }
-    *size = be64_to_cpu(s);
-    *blocksize = 1024;
-    TRACE("Size is %" PRIu64, *size);
-
-    if (!name) {
-        if (read_sync(csock, flags, sizeof(*flags)) != sizeof(*flags)) {
-            LOG("read failed (flags)");
-            goto fail;
-        }
-        *flags = be32_to_cpup(flags);
-    } else {
-        if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
-            LOG("read failed (tmp)");
-            goto fail;
-        }
-        *flags |= be32_to_cpu(tmp);
-    }
-    if (read_sync(csock, &buf, 124) != 124) {
-        LOG("read failed (buf)");
-        goto fail;
-    }
-    rc = 0;
-
-fail:
-    return rc;
-}
-
-#ifdef __linux__
-int nbd_init(int fd, int csock, uint32_t flags, off_t size, size_t blocksize)
-{
-    TRACE("Setting NBD socket");
-
-    if (ioctl(fd, NBD_SET_SOCK, csock) < 0) {
-        int serrno = errno;
-        LOG("Failed to set NBD socket");
-        return -serrno;
-    }
-
-    TRACE("Setting block size to %lu", (unsigned long)blocksize);
-
-    if (ioctl(fd, NBD_SET_BLKSIZE, blocksize) < 0) {
-        int serrno = errno;
-        LOG("Failed setting NBD block size");
-        return -serrno;
-    }
-
-        TRACE("Setting size to %zd block(s)", (size_t)(size / blocksize));
-
-    if (ioctl(fd, NBD_SET_SIZE_BLOCKS, size / blocksize) < 0) {
-        int serrno = errno;
-        LOG("Failed setting size (in blocks)");
-        return -serrno;
-    }
-
-    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0) {
-        if (errno == ENOTTY) {
-            int read_only = (flags & NBD_FLAG_READ_ONLY) != 0;
-            TRACE("Setting readonly attribute");
-
-            if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
-                int serrno = errno;
-                LOG("Failed setting read-only attribute");
-                return -serrno;
-            }
-        } else {
-            int serrno = errno;
-            LOG("Failed setting flags");
-            return -serrno;
-        }
-    }
-
-    TRACE("Negotiation ended");
-
-    return 0;
-}
-
-int nbd_disconnect(int fd)
-{
-    ioctl(fd, NBD_CLEAR_QUE);
-    ioctl(fd, NBD_DISCONNECT);
-    ioctl(fd, NBD_CLEAR_SOCK);
-    return 0;
-}
-
-int nbd_client(int fd)
-{
-    int ret;
-    int serrno;
-
-    TRACE("Doing NBD loop");
-
-    ret = ioctl(fd, NBD_DO_IT);
-    if (ret < 0 && errno == EPIPE) {
-        /* NBD_DO_IT normally returns EPIPE when someone has disconnected
-         * the socket via NBD_DISCONNECT.  We do not want to return 1 in
-         * that case.
-         */
-        ret = 0;
-    }
-    serrno = errno;
-
-    TRACE("NBD loop returned %d: %s", ret, strerror(serrno));
-
-    TRACE("Clearing NBD queue");
-    ioctl(fd, NBD_CLEAR_QUE);
-
-    TRACE("Clearing NBD socket");
-    ioctl(fd, NBD_CLEAR_SOCK);
-
-    errno = serrno;
-    return ret;
-}
-#else
-int nbd_init(int fd, int csock, uint32_t flags, off_t size, size_t blocksize)
-{
-    return -ENOTSUP;
-}
-
-int nbd_disconnect(int fd)
-{
-    return -ENOTSUP;
-}
-
-int nbd_client(int fd)
-{
-    return -ENOTSUP;
-}
-#endif
-
-ssize_t nbd_send_request(int csock, struct nbd_request *request)
-{
-    uint8_t buf[NBD_REQUEST_SIZE];
-    ssize_t ret;
-
-    cpu_to_be32w((uint32_t*)buf, NBD_REQUEST_MAGIC);
-    cpu_to_be32w((uint32_t*)(buf + 4), request->type);
-    cpu_to_be64w((uint64_t*)(buf + 8), request->handle);
-    cpu_to_be64w((uint64_t*)(buf + 16), request->from);
-    cpu_to_be32w((uint32_t*)(buf + 24), request->len);
-
-    TRACE("Sending request to client: "
-          "{ .from = %" PRIu64", .len = %u, .handle = %" PRIu64", .type=%i}",
-          request->from, request->len, request->handle, request->type);
-
-    ret = write_sync(csock, buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret != sizeof(buf)) {
-        LOG("writing to socket failed");
-        return -EINVAL;
-    }
-    return 0;
-}
-
-static ssize_t nbd_receive_request(int csock, struct nbd_request *request)
-{
-    uint8_t buf[NBD_REQUEST_SIZE];
-    uint32_t magic;
-    ssize_t ret;
-
-    ret = read_sync(csock, buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret != sizeof(buf)) {
-        LOG("read failed");
-        return -EINVAL;
-    }
-
-    /* Request
-       [ 0 ..  3]   magic   (NBD_REQUEST_MAGIC)
-       [ 4 ..  7]   type    (0 == READ, 1 == WRITE)
-       [ 8 .. 15]   handle
-       [16 .. 23]   from
-       [24 .. 27]   len
-     */
-
-    magic = be32_to_cpup((uint32_t*)buf);
-    request->type  = be32_to_cpup((uint32_t*)(buf + 4));
-    request->handle = be64_to_cpup((uint64_t*)(buf + 8));
-    request->from  = be64_to_cpup((uint64_t*)(buf + 16));
-    request->len   = be32_to_cpup((uint32_t*)(buf + 24));
-
-    TRACE("Got request: "
-          "{ magic = 0x%x, .type = %d, from = %" PRIu64" , len = %u }",
-          magic, request->type, request->from, request->len);
-
-    if (magic != NBD_REQUEST_MAGIC) {
-        LOG("invalid magic (got 0x%x)", magic);
-        return -EINVAL;
-    }
-    return 0;
-}
-
-ssize_t nbd_receive_reply(int csock, struct nbd_reply *reply)
-{
-    uint8_t buf[NBD_REPLY_SIZE];
-    uint32_t magic;
-    ssize_t ret;
-
-    ret = read_sync(csock, buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret != sizeof(buf)) {
-        LOG("read failed");
-        return -EINVAL;
-    }
-
-    /* Reply
-       [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
-       [ 4 ..  7]    error   (0 == no error)
-       [ 7 .. 15]    handle
-     */
-
-    magic = be32_to_cpup((uint32_t*)buf);
-    reply->error  = be32_to_cpup((uint32_t*)(buf + 4));
-    reply->handle = be64_to_cpup((uint64_t*)(buf + 8));
-
-    TRACE("Got reply: "
-          "{ magic = 0x%x, .error = %d, handle = %" PRIu64" }",
-          magic, reply->error, reply->handle);
-
-    if (magic != NBD_REPLY_MAGIC) {
-        LOG("invalid magic (got 0x%x)", magic);
-        return -EINVAL;
-    }
-    return 0;
-}
-
-static ssize_t nbd_send_reply(int csock, struct nbd_reply *reply)
-{
-    uint8_t buf[NBD_REPLY_SIZE];
-    ssize_t ret;
-
-    /* Reply
-       [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
-       [ 4 ..  7]    error   (0 == no error)
-       [ 7 .. 15]    handle
-     */
-    cpu_to_be32w((uint32_t*)buf, NBD_REPLY_MAGIC);
-    cpu_to_be32w((uint32_t*)(buf + 4), reply->error);
-    cpu_to_be64w((uint64_t*)(buf + 8), reply->handle);
-
-    TRACE("Sending response to client");
-
-    ret = write_sync(csock, buf, sizeof(buf));
-    if (ret < 0) {
-        return ret;
-    }
-
-    if (ret != sizeof(buf)) {
-        LOG("writing to socket failed");
-        return -EINVAL;
-    }
-    return 0;
-}
-
-#define MAX_NBD_REQUESTS 16
-
-void nbd_client_get(NBDClient *client)
-{
-    client->refcount++;
-}
-
-void nbd_client_put(NBDClient *client)
-{
-    if (--client->refcount == 0) {
-        /* The last reference should be dropped by client->close,
-         * which is called by nbd_client_close.
-         */
-        assert(client->closing);
-
-        nbd_unset_handlers(client);
-        close(client->sock);
-        client->sock = -1;
-        if (client->exp) {
-            QTAILQ_REMOVE(&client->exp->clients, client, next);
-            nbd_export_put(client->exp);
-        }
-        g_free(client);
-    }
-}
-
-void nbd_client_close(NBDClient *client)
-{
-    if (client->closing) {
-        return;
-    }
-
-    client->closing = true;
-
-    /* Force requests to finish.  They will drop their own references,
-     * then we'll close the socket and free the NBDClient.
-     */
-    shutdown(client->sock, 2);
-
-    /* Also tell the client, so that they release their reference.  */
-    if (client->close) {
-        client->close(client);
-    }
-}
-
-static NBDRequest *nbd_request_get(NBDClient *client)
-{
-    NBDRequest *req;
-
-    assert(client->nb_requests <= MAX_NBD_REQUESTS - 1);
-    client->nb_requests++;
-    nbd_update_can_read(client);
-
-    req = g_slice_new0(NBDRequest);
-    nbd_client_get(client);
-    req->client = client;
-    return req;
-}
-
-static void nbd_request_put(NBDRequest *req)
-{
-    NBDClient *client = req->client;
-
-    if (req->data) {
-        qemu_vfree(req->data);
-    }
-    g_slice_free(NBDRequest, req);
-
-    client->nb_requests--;
-    nbd_update_can_read(client);
-    nbd_client_put(client);
-}
-
-static void bs_aio_attached(AioContext *ctx, void *opaque)
-{
-    NBDExport *exp = opaque;
-    NBDClient *client;
-
-    TRACE("Export %s: Attaching clients to AIO context %p\n", exp->name, ctx);
-
-    exp->ctx = ctx;
-
-    QTAILQ_FOREACH(client, &exp->clients, next) {
-        nbd_set_handlers(client);
-    }
-}
-
-static void bs_aio_detach(void *opaque)
-{
-    NBDExport *exp = opaque;
-    NBDClient *client;
-
-    TRACE("Export %s: Detaching clients from AIO context %p\n", exp->name, exp->ctx);
-
-    QTAILQ_FOREACH(client, &exp->clients, next) {
-        nbd_unset_handlers(client);
-    }
-
-    exp->ctx = NULL;
-}
-
-NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
-                          off_t size, uint32_t nbdflags,
-                          void (*close)(NBDExport *))
-{
-    NBDExport *exp = g_malloc0(sizeof(NBDExport));
-    exp->refcount = 1;
-    QTAILQ_INIT(&exp->clients);
-    exp->bs = bs;
-    exp->dev_offset = dev_offset;
-    exp->nbdflags = nbdflags;
-    exp->size = size == -1 ? bdrv_getlength(bs) : size;
-    exp->close = close;
-    exp->ctx = bdrv_get_aio_context(bs);
-    bdrv_ref(bs);
-    bdrv_add_aio_context_notifier(bs, bs_aio_attached, bs_aio_detach, exp);
-    /*
-     * NBD exports are used for non-shared storage migration.  Make sure
-     * that BDRV_O_INCOMING is cleared and the image is ready for write
-     * access since the export could be available before migration handover.
-     */
-    bdrv_invalidate_cache(bs, NULL);
-    return exp;
-}
-
-NBDExport *nbd_export_find(const char *name)
-{
-    NBDExport *exp;
-    QTAILQ_FOREACH(exp, &exports, next) {
-        if (strcmp(name, exp->name) == 0) {
-            return exp;
-        }
-    }
-
-    return NULL;
-}
-
-void nbd_export_set_name(NBDExport *exp, const char *name)
-{
-    if (exp->name == name) {
-        return;
-    }
-
-    nbd_export_get(exp);
-    if (exp->name != NULL) {
-        g_free(exp->name);
-        exp->name = NULL;
-        QTAILQ_REMOVE(&exports, exp, next);
-        nbd_export_put(exp);
-    }
-    if (name != NULL) {
-        nbd_export_get(exp);
-        exp->name = g_strdup(name);
-        QTAILQ_INSERT_TAIL(&exports, exp, next);
-    }
-    nbd_export_put(exp);
-}
-
-void nbd_export_close(NBDExport *exp)
-{
-    NBDClient *client, *next;
-
-    nbd_export_get(exp);
-    QTAILQ_FOREACH_SAFE(client, &exp->clients, next, next) {
-        nbd_client_close(client);
-    }
-    nbd_export_set_name(exp, NULL);
-    nbd_export_put(exp);
-    if (exp->bs) {
-        bdrv_remove_aio_context_notifier(exp->bs, bs_aio_attached,
-                                         bs_aio_detach, exp);
-        bdrv_unref(exp->bs);
-        exp->bs = NULL;
-    }
-}
-
-void nbd_export_get(NBDExport *exp)
-{
-    assert(exp->refcount > 0);
-    exp->refcount++;
-}
-
-void nbd_export_put(NBDExport *exp)
-{
-    assert(exp->refcount > 0);
-    if (exp->refcount == 1) {
-        nbd_export_close(exp);
-    }
-
-    if (--exp->refcount == 0) {
-        assert(exp->name == NULL);
-
-        if (exp->close) {
-            exp->close(exp);
-        }
-
-        g_free(exp);
-    }
-}
-
-BlockDriverState *nbd_export_get_blockdev(NBDExport *exp)
-{
-    return exp->bs;
-}
-
-void nbd_export_close_all(void)
-{
-    NBDExport *exp, *next;
-
-    QTAILQ_FOREACH_SAFE(exp, &exports, next, next) {
-        nbd_export_close(exp);
-    }
-}
-
-static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
-                                 int len)
-{
-    NBDClient *client = req->client;
-    int csock = client->sock;
-    ssize_t rc, ret;
-
-    qemu_co_mutex_lock(&client->send_lock);
-    client->send_coroutine = qemu_coroutine_self();
-    nbd_set_handlers(client);
-
-    if (!len) {
-        rc = nbd_send_reply(csock, reply);
-    } else {
-        socket_set_cork(csock, 1);
-        rc = nbd_send_reply(csock, reply);
-        if (rc >= 0) {
-            ret = qemu_co_send(csock, req->data, len);
-            if (ret != len) {
-                rc = -EIO;
-            }
-        }
-        socket_set_cork(csock, 0);
-    }
-
-    client->send_coroutine = NULL;
-    nbd_set_handlers(client);
-    qemu_co_mutex_unlock(&client->send_lock);
-    return rc;
-}
-
-static ssize_t nbd_co_receive_request(NBDRequest *req, struct nbd_request *request)
-{
-    NBDClient *client = req->client;
-    int csock = client->sock;
-    uint32_t command;
-    ssize_t rc;
-
-    client->recv_coroutine = qemu_coroutine_self();
-    nbd_update_can_read(client);
-
-    rc = nbd_receive_request(csock, request);
-    if (rc < 0) {
-        if (rc != -EAGAIN) {
-            rc = -EIO;
-        }
-        goto out;
-    }
-
-    if (request->len > NBD_MAX_BUFFER_SIZE) {
-        LOG("len (%u) is larger than max len (%u)",
-            request->len, NBD_MAX_BUFFER_SIZE);
-        rc = -EINVAL;
-        goto out;
-    }
-
-    if ((request->from + request->len) < request->from) {
-        LOG("integer overflow detected! "
-            "you're probably being attacked");
-        rc = -EINVAL;
-        goto out;
-    }
-
-    TRACE("Decoding type");
-
-    command = request->type & NBD_CMD_MASK_COMMAND;
-    if (command == NBD_CMD_READ || command == NBD_CMD_WRITE) {
-        req->data = qemu_blockalign(client->exp->bs, request->len);
-    }
-    if (command == NBD_CMD_WRITE) {
-        TRACE("Reading %u byte(s)", request->len);
-
-        if (qemu_co_recv(csock, req->data, request->len) != request->len) {
-            LOG("reading from socket failed");
-            rc = -EIO;
-            goto out;
-        }
-    }
-    rc = 0;
-
-out:
-    client->recv_coroutine = NULL;
-    nbd_update_can_read(client);
-
-    return rc;
-}
-
-static void nbd_trip(void *opaque)
-{
-    NBDClient *client = opaque;
-    NBDExport *exp = client->exp;
-    NBDRequest *req;
-    struct nbd_request request;
-    struct nbd_reply reply;
-    ssize_t ret;
-    uint32_t command;
-
-    TRACE("Reading request.");
-    if (client->closing) {
-        return;
-    }
-
-    req = nbd_request_get(client);
-    ret = nbd_co_receive_request(req, &request);
-    if (ret == -EAGAIN) {
-        goto done;
-    }
-    if (ret == -EIO) {
-        goto out;
-    }
-
-    reply.handle = request.handle;
-    reply.error = 0;
-
-    if (ret < 0) {
-        reply.error = -ret;
-        goto error_reply;
-    }
-    command = request.type & NBD_CMD_MASK_COMMAND;
-    if (command != NBD_CMD_DISC && (request.from + request.len) > exp->size) {
-            LOG("From: %" PRIu64 ", Len: %u, Size: %" PRIu64
-            ", Offset: %" PRIu64 "\n",
-                    request.from, request.len,
-                    (uint64_t)exp->size, (uint64_t)exp->dev_offset);
-        LOG("requested operation past EOF--bad client?");
-        goto invalid_request;
-    }
-
-    switch (command) {
-    case NBD_CMD_READ:
-        TRACE("Request type is READ");
-
-        if (request.type & NBD_CMD_FLAG_FUA) {
-            ret = bdrv_co_flush(exp->bs);
-            if (ret < 0) {
-                LOG("flush failed");
-                reply.error = -ret;
-                goto error_reply;
-            }
-        }
-
-        ret = bdrv_read(exp->bs, (request.from + exp->dev_offset) / 512,
-                        req->data, request.len / 512);
-        if (ret < 0) {
-            LOG("reading from file failed");
-            reply.error = -ret;
-            goto error_reply;
-        }
-
-        TRACE("Read %u byte(s)", request.len);
-        if (nbd_co_send_reply(req, &reply, request.len) < 0)
-            goto out;
-        break;
-    case NBD_CMD_WRITE:
-        TRACE("Request type is WRITE");
-
-        if (exp->nbdflags & NBD_FLAG_READ_ONLY) {
-            TRACE("Server is read-only, return error");
-            reply.error = EROFS;
-            goto error_reply;
-        }
-
-        TRACE("Writing to device");
-
-        ret = bdrv_write(exp->bs, (request.from + exp->dev_offset) / 512,
-                         req->data, request.len / 512);
-        if (ret < 0) {
-            LOG("writing to file failed");
-            reply.error = -ret;
-            goto error_reply;
-        }
-
-        if (request.type & NBD_CMD_FLAG_FUA) {
-            ret = bdrv_co_flush(exp->bs);
-            if (ret < 0) {
-                LOG("flush failed");
-                reply.error = -ret;
-                goto error_reply;
-            }
-        }
-
-        if (nbd_co_send_reply(req, &reply, 0) < 0) {
-            goto out;
-        }
-        break;
-    case NBD_CMD_DISC:
-        TRACE("Request type is DISCONNECT");
-        errno = 0;
-        goto out;
-    case NBD_CMD_FLUSH:
-        TRACE("Request type is FLUSH");
-
-        ret = bdrv_co_flush(exp->bs);
-        if (ret < 0) {
-            LOG("flush failed");
-            reply.error = -ret;
-        }
-        if (nbd_co_send_reply(req, &reply, 0) < 0) {
-            goto out;
-        }
-        break;
-    case NBD_CMD_TRIM:
-        TRACE("Request type is TRIM");
-        ret = bdrv_co_discard(exp->bs, (request.from + exp->dev_offset) / 512,
-                              request.len / 512);
-        if (ret < 0) {
-            LOG("discard failed");
-            reply.error = -ret;
-        }
-        if (nbd_co_send_reply(req, &reply, 0) < 0) {
-            goto out;
-        }
-        break;
-    default:
-        LOG("invalid request type (%u) received", request.type);
-    invalid_request:
-        reply.error = -EINVAL;
-    error_reply:
-        if (nbd_co_send_reply(req, &reply, 0) < 0) {
-            goto out;
-        }
-        break;
-    }
-
-    TRACE("Request/Reply complete");
-
-done:
-    nbd_request_put(req);
-    return;
-
-out:
-    nbd_request_put(req);
-    nbd_client_close(client);
-}
-
-static void nbd_read(void *opaque)
-{
-    NBDClient *client = opaque;
-
-    if (client->recv_coroutine) {
-        qemu_coroutine_enter(client->recv_coroutine, NULL);
-    } else {
-        qemu_coroutine_enter(qemu_coroutine_create(nbd_trip), client);
-    }
-}
-
-static void nbd_restart_write(void *opaque)
-{
-    NBDClient *client = opaque;
-
-    qemu_coroutine_enter(client->send_coroutine, NULL);
-}
-
-static void nbd_set_handlers(NBDClient *client)
-{
-    if (client->exp && client->exp->ctx) {
-        aio_set_fd_handler(client->exp->ctx, client->sock,
-                           client->can_read ? nbd_read : NULL,
-                           client->send_coroutine ? nbd_restart_write : NULL,
-                           client);
-    }
-}
-
-static void nbd_unset_handlers(NBDClient *client)
-{
-    if (client->exp && client->exp->ctx) {
-        aio_set_fd_handler(client->exp->ctx, client->sock, NULL, NULL, NULL);
-    }
-}
-
-static void nbd_update_can_read(NBDClient *client)
-{
-    bool can_read = client->recv_coroutine ||
-                    client->nb_requests < MAX_NBD_REQUESTS;
-
-    if (can_read != client->can_read) {
-        client->can_read = can_read;
-        nbd_set_handlers(client);
-
-        /* There is no need to invoke aio_notify(), since aio_set_fd_handler()
-         * in nbd_set_handlers() will have taken care of that */
-    }
-}
-
-NBDClient *nbd_client_new(NBDExport *exp, int csock,
-                          void (*close)(NBDClient *))
-{
-    NBDClient *client;
-    client = g_malloc0(sizeof(NBDClient));
-    client->refcount = 1;
-    client->exp = exp;
-    client->sock = csock;
-    client->can_read = true;
-    if (nbd_send_negotiate(client)) {
-        g_free(client);
-        return NULL;
-    }
-    client->close = close;
-    qemu_co_mutex_init(&client->send_lock);
-    nbd_set_handlers(client);
-
-    if (exp) {
-        QTAILQ_INSERT_TAIL(&exp->clients, client, next);
-        nbd_export_get(exp);
-    }
-    return client;
-}
diff --git a/nbd/Makefile.objs b/nbd/Makefile.objs
new file mode 100644
index 0000000..eb3dd44
--- /dev/null
+++ b/nbd/Makefile.objs
@@ -0,0 +1 @@
+block-obj-y += server.o client.o common.o
diff --git a/nbd/client.c b/nbd/client.c
new file mode 100644
index 0000000..a92f1e2
--- /dev/null
+++ b/nbd/client.c
@@ -0,0 +1,775 @@
+/*
+ *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
+ *
+ *  Network Block Device Client Side
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "nbd-internal.h"
+
+static int nbd_errno_to_system_errno(int err)
+{
+    switch (err) {
+    case NBD_SUCCESS:
+        return 0;
+    case NBD_EPERM:
+        return EPERM;
+    case NBD_EIO:
+        return EIO;
+    case NBD_ENOMEM:
+        return ENOMEM;
+    case NBD_ENOSPC:
+        return ENOSPC;
+    default:
+        TRACE("Squashing unexpected error %d to EINVAL", err);
+        /* fallthrough */
+    case NBD_EINVAL:
+        return EINVAL;
+    }
+}
+
+/* Definitions for opaque data types */
+
+static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
+
+/* That's all folks */
+
+/* Basic flow for negotiation
+
+   Server         Client
+   Negotiate
+
+   or
+
+   Server         Client
+   Negotiate #1
+                  Option
+   Negotiate #2
+
+   ----
+
+   followed by
+
+   Server         Client
+                  Request
+   Response
+                  Request
+   Response
+                  ...
+   ...
+                  Request (type == 2)
+
+*/
+
+
+/* If type represents success, return 1 without further action.
+ * If type represents an error reply, consume the rest of the packet on ioc.
+ * Then return 0 for unsupported (so the client can fall back to
+ * other approaches), or -1 with errp set for other errors.
+ */
+static int nbd_handle_reply_err(QIOChannel *ioc, uint32_t opt, uint32_t type,
+                                Error **errp)
+{
+    uint32_t len;
+    char *msg = NULL;
+    int result = -1;
+
+    if (!(type & (1 << 31))) {
+        return 1;
+    }
+
+    if (read_sync(ioc, &len, sizeof(len)) != sizeof(len)) {
+        error_setg(errp, "failed to read option length");
+        return -1;
+    }
+    len = be32_to_cpu(len);
+    if (len) {
+        if (len > NBD_MAX_BUFFER_SIZE) {
+            error_setg(errp, "server's error message is too long");
+            goto cleanup;
+        }
+        msg = g_malloc(len + 1);
+        if (read_sync(ioc, msg, len) != len) {
+            error_setg(errp, "failed to read option error message");
+            goto cleanup;
+        }
+        msg[len] = '\0';
+    }
+
+    switch (type) {
+    case NBD_REP_ERR_UNSUP:
+        TRACE("server doesn't understand request %" PRIx32
+              ", attempting fallback", opt);
+        result = 0;
+        goto cleanup;
+
+    case NBD_REP_ERR_POLICY:
+        error_setg(errp, "Denied by server for option %" PRIx32, opt);
+        break;
+
+    case NBD_REP_ERR_INVALID:
+        error_setg(errp, "Invalid data length for option %" PRIx32, opt);
+        break;
+
+    case NBD_REP_ERR_TLS_REQD:
+        error_setg(errp, "TLS negotiation required before option %" PRIx32,
+                   opt);
+        break;
+
+    default:
+        error_setg(errp, "Unknown error code when asking for option %" PRIx32,
+                   opt);
+        break;
+    }
+
+    if (msg) {
+        error_append_hint(errp, "%s\n", msg);
+    }
+
+ cleanup:
+    g_free(msg);
+    return result;
+}
+
+static int nbd_receive_list(QIOChannel *ioc, char **name, Error **errp)
+{
+    uint64_t magic;
+    uint32_t opt;
+    uint32_t type;
+    uint32_t len;
+    uint32_t namelen;
+    int error;
+
+    *name = NULL;
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "failed to read list option magic");
+        return -1;
+    }
+    magic = be64_to_cpu(magic);
+    if (magic != NBD_REP_MAGIC) {
+        error_setg(errp, "Unexpected option list magic");
+        return -1;
+    }
+    if (read_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "failed to read list option");
+        return -1;
+    }
+    opt = be32_to_cpu(opt);
+    if (opt != NBD_OPT_LIST) {
+        error_setg(errp, "Unexpected option type %" PRIx32 " expected %x",
+                   opt, NBD_OPT_LIST);
+        return -1;
+    }
+
+    if (read_sync(ioc, &type, sizeof(type)) != sizeof(type)) {
+        error_setg(errp, "failed to read list option type");
+        return -1;
+    }
+    type = be32_to_cpu(type);
+    error = nbd_handle_reply_err(ioc, opt, type, errp);
+    if (error <= 0) {
+        return error;
+    }
+
+    if (read_sync(ioc, &len, sizeof(len)) != sizeof(len)) {
+        error_setg(errp, "failed to read option length");
+        return -1;
+    }
+    len = be32_to_cpu(len);
+
+    if (type == NBD_REP_ACK) {
+        if (len != 0) {
+            error_setg(errp, "length too long for option end");
+            return -1;
+        }
+    } else if (type == NBD_REP_SERVER) {
+        if (len < sizeof(namelen) || len > NBD_MAX_BUFFER_SIZE) {
+            error_setg(errp, "incorrect option length");
+            return -1;
+        }
+        if (read_sync(ioc, &namelen, sizeof(namelen)) != sizeof(namelen)) {
+            error_setg(errp, "failed to read option name length");
+            return -1;
+        }
+        namelen = be32_to_cpu(namelen);
+        len -= sizeof(namelen);
+        if (len < namelen) {
+            error_setg(errp, "incorrect option name length");
+            return -1;
+        }
+        if (namelen > NBD_MAX_NAME_SIZE) {
+            error_setg(errp, "export name length too long %" PRIu32, namelen);
+            return -1;
+        }
+
+        *name = g_new0(char, namelen + 1);
+        if (read_sync(ioc, *name, namelen) != namelen) {
+            error_setg(errp, "failed to read export name");
+            g_free(*name);
+            *name = NULL;
+            return -1;
+        }
+        (*name)[namelen] = '\0';
+        len -= namelen;
+        if (len) {
+            char *buf = g_malloc(len + 1);
+            if (read_sync(ioc, buf, len) != len) {
+                error_setg(errp, "failed to read export description");
+                g_free(*name);
+                g_free(buf);
+                *name = NULL;
+                return -1;
+            }
+            buf[len] = '\0';
+            TRACE("Ignoring export description: %s", buf);
+            g_free(buf);
+        }
+    } else {
+        error_setg(errp, "Unexpected reply type %" PRIx32 " expected %x",
+                   type, NBD_REP_SERVER);
+        return -1;
+    }
+    return 1;
+}
+
+
+static int nbd_receive_query_exports(QIOChannel *ioc,
+                                     const char *wantname,
+                                     Error **errp)
+{
+    uint64_t magic = cpu_to_be64(NBD_OPTS_MAGIC);
+    uint32_t opt = cpu_to_be32(NBD_OPT_LIST);
+    uint32_t length = 0;
+    bool foundExport = false;
+
+    TRACE("Querying export list");
+    if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to send list option magic");
+        return -1;
+    }
+
+    if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "Failed to send list option number");
+        return -1;
+    }
+
+    if (write_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "Failed to send list option length");
+        return -1;
+    }
+
+    TRACE("Reading available export names");
+    while (1) {
+        char *name = NULL;
+        int ret = nbd_receive_list(ioc, &name, errp);
+
+        if (ret < 0) {
+            g_free(name);
+            name = NULL;
+            return -1;
+        }
+        if (ret == 0) {
+            /* Server doesn't support export listing, so
+             * we will just assume an export with our
+             * wanted name exists */
+            foundExport = true;
+            break;
+        }
+        if (name == NULL) {
+            TRACE("End of export name list");
+            break;
+        }
+        if (g_str_equal(name, wantname)) {
+            foundExport = true;
+            TRACE("Found desired export name '%s'", name);
+        } else {
+            TRACE("Ignored export name '%s'", name);
+        }
+        g_free(name);
+    }
+
+    if (!foundExport) {
+        error_setg(errp, "No export with name '%s' available", wantname);
+        return -1;
+    }
+
+    return 0;
+}
+
+static QIOChannel *nbd_receive_starttls(QIOChannel *ioc,
+                                        QCryptoTLSCreds *tlscreds,
+                                        const char *hostname, Error **errp)
+{
+    uint64_t magic = cpu_to_be64(NBD_OPTS_MAGIC);
+    uint32_t opt = cpu_to_be32(NBD_OPT_STARTTLS);
+    uint32_t length = 0;
+    uint32_t type;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Requesting TLS from server");
+    if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to send option magic");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "Failed to send option number");
+        return NULL;
+    }
+
+    if (write_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "Failed to send option length");
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server1");
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "failed to read option magic");
+        return NULL;
+    }
+    magic = be64_to_cpu(magic);
+    if (magic != NBD_REP_MAGIC) {
+        error_setg(errp, "Unexpected option magic");
+        return NULL;
+    }
+    TRACE("Getting TLS reply from server2");
+    if (read_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        error_setg(errp, "failed to read option");
+        return NULL;
+    }
+    opt = be32_to_cpu(opt);
+    if (opt != NBD_OPT_STARTTLS) {
+        error_setg(errp, "Unexpected option type %" PRIx32 " expected %x",
+                   opt, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &type, sizeof(type)) != sizeof(type)) {
+        error_setg(errp, "failed to read option type");
+        return NULL;
+    }
+    type = be32_to_cpu(type);
+    if (type != NBD_REP_ACK) {
+        error_setg(errp, "Server rejected request to start TLS %" PRIx32,
+                   type);
+        return NULL;
+    }
+
+    TRACE("Getting TLS reply from server");
+    if (read_sync(ioc, &length, sizeof(length)) != sizeof(length)) {
+        error_setg(errp, "failed to read option length");
+        return NULL;
+    }
+    length = be32_to_cpu(length);
+    if (length != 0) {
+        error_setg(errp, "Start TLS response was not zero %" PRIu32,
+                   length);
+        return NULL;
+    }
+
+    TRACE("TLS request approved, setting up TLS");
+    tioc = qio_channel_tls_new_client(ioc, tlscreds, hostname, errp);
+    if (!tioc) {
+        return NULL;
+    }
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    TRACE("Starting TLS handshake");
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        error_propagate(errp, data.error);
+        object_unref(OBJECT(tioc));
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
+int nbd_receive_negotiate(QIOChannel *ioc, const char *name, uint16_t *flags,
+                          QCryptoTLSCreds *tlscreds, const char *hostname,
+                          QIOChannel **outioc,
+                          off_t *size, Error **errp)
+{
+    char buf[256];
+    uint64_t magic, s;
+    int rc;
+
+    TRACE("Receiving negotiation tlscreds=%p hostname=%s.",
+          tlscreds, hostname ? hostname : "<null>");
+
+    rc = -EINVAL;
+
+    if (outioc) {
+        *outioc = NULL;
+    }
+    if (tlscreds && !outioc) {
+        error_setg(errp, "Output I/O channel required for TLS");
+        goto fail;
+    }
+
+    if (read_sync(ioc, buf, 8) != 8) {
+        error_setg(errp, "Failed to read data");
+        goto fail;
+    }
+
+    buf[8] = '\0';
+    if (strlen(buf) == 0) {
+        error_setg(errp, "Server connection closed unexpectedly");
+        goto fail;
+    }
+
+    TRACE("Magic is %c%c%c%c%c%c%c%c",
+          qemu_isprint(buf[0]) ? buf[0] : '.',
+          qemu_isprint(buf[1]) ? buf[1] : '.',
+          qemu_isprint(buf[2]) ? buf[2] : '.',
+          qemu_isprint(buf[3]) ? buf[3] : '.',
+          qemu_isprint(buf[4]) ? buf[4] : '.',
+          qemu_isprint(buf[5]) ? buf[5] : '.',
+          qemu_isprint(buf[6]) ? buf[6] : '.',
+          qemu_isprint(buf[7]) ? buf[7] : '.');
+
+    if (memcmp(buf, "NBDMAGIC", 8) != 0) {
+        error_setg(errp, "Invalid magic received");
+        goto fail;
+    }
+
+    if (read_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        error_setg(errp, "Failed to read magic");
+        goto fail;
+    }
+    magic = be64_to_cpu(magic);
+    TRACE("Magic is 0x%" PRIx64, magic);
+
+    if (magic == NBD_OPTS_MAGIC) {
+        uint32_t clientflags = 0;
+        uint32_t opt;
+        uint32_t namesize;
+        uint16_t globalflags;
+        bool fixedNewStyle = false;
+
+        if (read_sync(ioc, &globalflags, sizeof(globalflags)) !=
+            sizeof(globalflags)) {
+            error_setg(errp, "Failed to read server flags");
+            goto fail;
+        }
+        globalflags = be16_to_cpu(globalflags);
+        TRACE("Global flags are %" PRIx32, globalflags);
+        if (globalflags & NBD_FLAG_FIXED_NEWSTYLE) {
+            fixedNewStyle = true;
+            TRACE("Server supports fixed new style");
+            clientflags |= NBD_FLAG_C_FIXED_NEWSTYLE;
+        }
+        /* client requested flags */
+        clientflags = cpu_to_be32(clientflags);
+        if (write_sync(ioc, &clientflags, sizeof(clientflags)) !=
+            sizeof(clientflags)) {
+            error_setg(errp, "Failed to send clientflags field");
+            goto fail;
+        }
+        if (tlscreds) {
+            if (fixedNewStyle) {
+                *outioc = nbd_receive_starttls(ioc, tlscreds, hostname, errp);
+                if (!*outioc) {
+                    goto fail;
+                }
+                ioc = *outioc;
+            } else {
+                error_setg(errp, "Server does not support STARTTLS");
+                goto fail;
+            }
+        }
+        if (!name) {
+            TRACE("Using default NBD export name \"\"");
+            name = "";
+        }
+        if (fixedNewStyle) {
+            /* Check our desired export is present in the
+             * server export list. Since NBD_OPT_EXPORT_NAME
+             * cannot return an error message, running this
+             * query gives us good error reporting if the
+             * server required TLS
+             */
+            if (nbd_receive_query_exports(ioc, name, errp) < 0) {
+                goto fail;
+            }
+        }
+        /* write the export name */
+        magic = cpu_to_be64(magic);
+        if (write_sync(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+            error_setg(errp, "Failed to send export name magic");
+            goto fail;
+        }
+        opt = cpu_to_be32(NBD_OPT_EXPORT_NAME);
+        if (write_sync(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+            error_setg(errp, "Failed to send export name option number");
+            goto fail;
+        }
+        namesize = cpu_to_be32(strlen(name));
+        if (write_sync(ioc, &namesize, sizeof(namesize)) !=
+            sizeof(namesize)) {
+            error_setg(errp, "Failed to send export name length");
+            goto fail;
+        }
+        if (write_sync(ioc, (char *)name, strlen(name)) != strlen(name)) {
+            error_setg(errp, "Failed to send export name");
+            goto fail;
+        }
+
+        if (read_sync(ioc, &s, sizeof(s)) != sizeof(s)) {
+            error_setg(errp, "Failed to read export length");
+            goto fail;
+        }
+        *size = be64_to_cpu(s);
+
+        if (read_sync(ioc, flags, sizeof(*flags)) != sizeof(*flags)) {
+            error_setg(errp, "Failed to read export flags");
+            goto fail;
+        }
+        be16_to_cpus(flags);
+    } else if (magic == NBD_CLIENT_MAGIC) {
+        uint32_t oldflags;
+
+        if (name) {
+            error_setg(errp, "Server does not support export names");
+            goto fail;
+        }
+        if (tlscreds) {
+            error_setg(errp, "Server does not support STARTTLS");
+            goto fail;
+        }
+
+        if (read_sync(ioc, &s, sizeof(s)) != sizeof(s)) {
+            error_setg(errp, "Failed to read export length");
+            goto fail;
+        }
+        *size = be64_to_cpu(s);
+        TRACE("Size is %" PRIu64, *size);
+
+        if (read_sync(ioc, &oldflags, sizeof(oldflags)) != sizeof(oldflags)) {
+            error_setg(errp, "Failed to read export flags");
+            goto fail;
+        }
+        be32_to_cpus(&oldflags);
+        if (oldflags & ~0xffff) {
+            error_setg(errp, "Unexpected export flags %0x" PRIx32, oldflags);
+            goto fail;
+        }
+        *flags = oldflags;
+    } else {
+        error_setg(errp, "Bad magic received");
+        goto fail;
+    }
+
+    TRACE("Size is %" PRIu64 ", export flags %" PRIx16, *size, *flags);
+    if (read_sync(ioc, &buf, 124) != 124) {
+        error_setg(errp, "Failed to read reserved block");
+        goto fail;
+    }
+    rc = 0;
+
+fail:
+    return rc;
+}
+
+#ifdef __linux__
+int nbd_init(int fd, QIOChannelSocket *sioc, uint16_t flags, off_t size)
+{
+    unsigned long sectors = size / BDRV_SECTOR_SIZE;
+    if (size / BDRV_SECTOR_SIZE != sectors) {
+        LOG("Export size %lld too large for 32-bit kernel", (long long) size);
+        return -E2BIG;
+    }
+
+    TRACE("Setting NBD socket");
+
+    if (ioctl(fd, NBD_SET_SOCK, (unsigned long) sioc->fd) < 0) {
+        int serrno = errno;
+        LOG("Failed to set NBD socket");
+        return -serrno;
+    }
+
+    TRACE("Setting block size to %lu", (unsigned long)BDRV_SECTOR_SIZE);
+
+    if (ioctl(fd, NBD_SET_BLKSIZE, (unsigned long)BDRV_SECTOR_SIZE) < 0) {
+        int serrno = errno;
+        LOG("Failed setting NBD block size");
+        return -serrno;
+    }
+
+    TRACE("Setting size to %lu block(s)", sectors);
+    if (size % BDRV_SECTOR_SIZE) {
+        TRACE("Ignoring trailing %d bytes of export",
+              (int) (size % BDRV_SECTOR_SIZE));
+    }
+
+    if (ioctl(fd, NBD_SET_SIZE_BLOCKS, sectors) < 0) {
+        int serrno = errno;
+        LOG("Failed setting size (in blocks)");
+        return -serrno;
+    }
+
+    if (ioctl(fd, NBD_SET_FLAGS, (unsigned long) flags) < 0) {
+        if (errno == ENOTTY) {
+            int read_only = (flags & NBD_FLAG_READ_ONLY) != 0;
+            TRACE("Setting readonly attribute");
+
+            if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
+                int serrno = errno;
+                LOG("Failed setting read-only attribute");
+                return -serrno;
+            }
+        } else {
+            int serrno = errno;
+            LOG("Failed setting flags");
+            return -serrno;
+        }
+    }
+
+    TRACE("Negotiation ended");
+
+    return 0;
+}
+
+int nbd_client(int fd)
+{
+    int ret;
+    int serrno;
+
+    TRACE("Doing NBD loop");
+
+    ret = ioctl(fd, NBD_DO_IT);
+    if (ret < 0 && errno == EPIPE) {
+        /* NBD_DO_IT normally returns EPIPE when someone has disconnected
+         * the socket via NBD_DISCONNECT.  We do not want to return 1 in
+         * that case.
+         */
+        ret = 0;
+    }
+    serrno = errno;
+
+    TRACE("NBD loop returned %d: %s", ret, strerror(serrno));
+
+    TRACE("Clearing NBD queue");
+    ioctl(fd, NBD_CLEAR_QUE);
+
+    TRACE("Clearing NBD socket");
+    ioctl(fd, NBD_CLEAR_SOCK);
+
+    errno = serrno;
+    return ret;
+}
+
+int nbd_disconnect(int fd)
+{
+    ioctl(fd, NBD_CLEAR_QUE);
+    ioctl(fd, NBD_DISCONNECT);
+    ioctl(fd, NBD_CLEAR_SOCK);
+    return 0;
+}
+
+#else
+int nbd_init(int fd, QIOChannelSocket *ioc, uint16_t flags, off_t size)
+{
+    return -ENOTSUP;
+}
+
+int nbd_client(int fd)
+{
+    return -ENOTSUP;
+}
+int nbd_disconnect(int fd)
+{
+    return -ENOTSUP;
+}
+#endif
+
+ssize_t nbd_send_request(QIOChannel *ioc, struct nbd_request *request)
+{
+    uint8_t buf[NBD_REQUEST_SIZE];
+    ssize_t ret;
+
+    TRACE("Sending request to server: "
+          "{ .from = %" PRIu64", .len = %" PRIu32 ", .handle = %" PRIu64
+          ", .type=%" PRIu32 " }",
+          request->from, request->len, request->handle, request->type);
+
+    stl_be_p(buf, NBD_REQUEST_MAGIC);
+    stl_be_p(buf + 4, request->type);
+    stq_be_p(buf + 8, request->handle);
+    stq_be_p(buf + 16, request->from);
+    stl_be_p(buf + 24, request->len);
+
+    ret = write_sync(ioc, buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret != sizeof(buf)) {
+        LOG("writing to socket failed");
+        return -EINVAL;
+    }
+    return 0;
+}
+
+ssize_t nbd_receive_reply(QIOChannel *ioc, struct nbd_reply *reply)
+{
+    uint8_t buf[NBD_REPLY_SIZE];
+    uint32_t magic;
+    ssize_t ret;
+
+    ret = read_sync(ioc, buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret != sizeof(buf)) {
+        LOG("read failed");
+        return -EINVAL;
+    }
+
+    /* Reply
+       [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
+       [ 4 ..  7]    error   (0 == no error)
+       [ 7 .. 15]    handle
+     */
+
+    magic = ldl_be_p(buf);
+    reply->error  = ldl_be_p(buf + 4);
+    reply->handle = ldq_be_p(buf + 8);
+
+    reply->error = nbd_errno_to_system_errno(reply->error);
+
+    TRACE("Got reply: { magic = 0x%" PRIx32 ", .error = % " PRId32
+          ", handle = %" PRIu64" }",
+          magic, reply->error, reply->handle);
+
+    if (magic != NBD_REPLY_MAGIC) {
+        LOG("invalid magic (got 0x%" PRIx32 ")", magic);
+        return -EINVAL;
+    }
+    return 0;
+}
+
diff --git a/nbd/common.c b/nbd/common.c
new file mode 100644
index 0000000..b583a4f
--- /dev/null
+++ b/nbd/common.c
@@ -0,0 +1,93 @@
+/*
+ *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
+ *
+ *  Network Block Device Common Code
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "nbd-internal.h"
+
+ssize_t nbd_wr_syncv(QIOChannel *ioc,
+                     struct iovec *iov,
+                     size_t niov,
+                     size_t length,
+                     bool do_read)
+{
+    ssize_t done = 0;
+    Error *local_err = NULL;
+    struct iovec *local_iov = g_new(struct iovec, niov);
+    struct iovec *local_iov_head = local_iov;
+    unsigned int nlocal_iov = niov;
+
+    nlocal_iov = iov_copy(local_iov, nlocal_iov, iov, niov, 0, length);
+
+    while (nlocal_iov > 0) {
+        ssize_t len;
+        if (do_read) {
+            len = qio_channel_readv(ioc, local_iov, nlocal_iov, &local_err);
+        } else {
+            len = qio_channel_writev(ioc, local_iov, nlocal_iov, &local_err);
+        }
+        if (len == QIO_CHANNEL_ERR_BLOCK) {
+            if (qemu_in_coroutine()) {
+                /* XXX figure out if we can create a variant on
+                 * qio_channel_yield() that works with AIO contexts
+                 * and consider using that in this branch */
+                qemu_coroutine_yield();
+            } else if (done) {
+                /* XXX this is needed by nbd_reply_ready.  */
+                qio_channel_wait(ioc,
+                                 do_read ? G_IO_IN : G_IO_OUT);
+            } else {
+                return -EAGAIN;
+            }
+            continue;
+        }
+        if (len < 0) {
+            TRACE("I/O error: %s", error_get_pretty(local_err));
+            error_free(local_err);
+            /* XXX handle Error objects */
+            done = -EIO;
+            goto cleanup;
+        }
+
+        if (do_read && len == 0) {
+            break;
+        }
+
+        iov_discard_front(&local_iov, &nlocal_iov, len);
+        done += len;
+    }
+
+ cleanup:
+    g_free(local_iov_head);
+    return done;
+}
+
+
+void nbd_tls_handshake(Object *src,
+                       Error *err,
+                       void *opaque)
+{
+    struct NBDTLSHandshakeData *data = opaque;
+
+    if (err) {
+        TRACE("TLS failed %s", error_get_pretty(err));
+        data->error = error_copy(err);
+    }
+    data->complete = true;
+    g_main_loop_quit(data->loop);
+}
diff --git a/nbd/nbd-internal.h b/nbd/nbd-internal.h
new file mode 100644
index 0000000..93a6ca8
--- /dev/null
+++ b/nbd/nbd-internal.h
@@ -0,0 +1,125 @@
+/*
+ * NBD Internal Declarations
+ *
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef NBD_INTERNAL_H
+#define NBD_INTERNAL_H
+#include "block/nbd.h"
+#include "sysemu/block-backend.h"
+#include "io/channel-tls.h"
+
+#include "qemu/coroutine.h"
+#include "qemu/iov.h"
+
+#ifndef _WIN32
+#include <sys/ioctl.h>
+#endif
+#if defined(__sun__) || defined(__HAIKU__)
+#include <sys/ioccom.h>
+#endif
+
+#ifdef __linux__
+#include <linux/fs.h>
+#endif
+
+#include "qemu/bswap.h"
+#include "qemu/queue.h"
+#include "qemu/main-loop.h"
+
+/* #define DEBUG_NBD */
+
+#ifdef DEBUG_NBD
+#define DEBUG_NBD_PRINT 1
+#else
+#define DEBUG_NBD_PRINT 0
+#endif
+
+#define TRACE(msg, ...) do { \
+    if (DEBUG_NBD_PRINT) { \
+        LOG(msg, ## __VA_ARGS__); \
+    } \
+} while (0)
+
+#define LOG(msg, ...) do { \
+    fprintf(stderr, "%s:%s():L%d: " msg "\n", \
+            __FILE__, __FUNCTION__, __LINE__, ## __VA_ARGS__); \
+} while (0)
+
+/* This is all part of the "official" NBD API.
+ *
+ * The most up-to-date documentation is available at:
+ * https://github.com/yoe/nbd/blob/master/doc/proto.txt
+ */
+
+#define NBD_REQUEST_SIZE        (4 + 4 + 8 + 8 + 4)
+#define NBD_REPLY_SIZE          (4 + 4 + 8)
+#define NBD_REQUEST_MAGIC       0x25609513
+#define NBD_REPLY_MAGIC         0x67446698
+#define NBD_OPTS_MAGIC          0x49484156454F5054LL
+#define NBD_CLIENT_MAGIC        0x0000420281861253LL
+#define NBD_REP_MAGIC           0x3e889045565a9LL
+
+#define NBD_SET_SOCK            _IO(0xab, 0)
+#define NBD_SET_BLKSIZE         _IO(0xab, 1)
+#define NBD_SET_SIZE            _IO(0xab, 2)
+#define NBD_DO_IT               _IO(0xab, 3)
+#define NBD_CLEAR_SOCK          _IO(0xab, 4)
+#define NBD_CLEAR_QUE           _IO(0xab, 5)
+#define NBD_PRINT_DEBUG         _IO(0xab, 6)
+#define NBD_SET_SIZE_BLOCKS     _IO(0xab, 7)
+#define NBD_DISCONNECT          _IO(0xab, 8)
+#define NBD_SET_TIMEOUT         _IO(0xab, 9)
+#define NBD_SET_FLAGS           _IO(0xab, 10)
+
+#define NBD_OPT_EXPORT_NAME     (1)
+#define NBD_OPT_ABORT           (2)
+#define NBD_OPT_LIST            (3)
+#define NBD_OPT_PEEK_EXPORT     (4)
+#define NBD_OPT_STARTTLS        (5)
+
+/* NBD errors are based on errno numbers, so there is a 1:1 mapping,
+ * but only a limited set of errno values is specified in the protocol.
+ * Everything else is squashed to EINVAL.
+ */
+#define NBD_SUCCESS    0
+#define NBD_EPERM      1
+#define NBD_EIO        5
+#define NBD_ENOMEM     12
+#define NBD_EINVAL     22
+#define NBD_ENOSPC     28
+
+static inline ssize_t read_sync(QIOChannel *ioc, void *buffer, size_t size)
+{
+    struct iovec iov = { .iov_base = buffer, .iov_len = size };
+    /* Sockets are kept in blocking mode in the negotiation phase.  After
+     * that, a non-readable socket simply means that another thread stole
+     * our request/reply.  Synchronization is done with recv_coroutine, so
+     * that this is coroutine-safe.
+     */
+    return nbd_wr_syncv(ioc, &iov, 1, size, true);
+}
+
+static inline ssize_t write_sync(QIOChannel *ioc, void *buffer, size_t size)
+{
+    struct iovec iov = { .iov_base = buffer, .iov_len = size };
+
+    return nbd_wr_syncv(ioc, &iov, 1, size, false);
+}
+
+struct NBDTLSHandshakeData {
+    GMainLoop *loop;
+    bool complete;
+    Error *error;
+};
+
+
+void nbd_tls_handshake(Object *src,
+                       Error *err,
+                       void *opaque);
+
+#endif
diff --git a/nbd/server.c b/nbd/server.c
new file mode 100644
index 0000000..80fbb4d
--- /dev/null
+++ b/nbd/server.c
@@ -0,0 +1,1319 @@
+/*
+ *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
+ *
+ *  Network Block Device Server Side
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; under version 2 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "nbd-internal.h"
+
+static int system_errno_to_nbd_errno(int err)
+{
+    switch (err) {
+    case 0:
+        return NBD_SUCCESS;
+    case EPERM:
+    case EROFS:
+        return NBD_EPERM;
+    case EIO:
+        return NBD_EIO;
+    case ENOMEM:
+        return NBD_ENOMEM;
+#ifdef EDQUOT
+    case EDQUOT:
+#endif
+    case EFBIG:
+    case ENOSPC:
+        return NBD_ENOSPC;
+    case EINVAL:
+    default:
+        return NBD_EINVAL;
+    }
+}
+
+/* Definitions for opaque data types */
+
+typedef struct NBDRequest NBDRequest;
+
+struct NBDRequest {
+    QSIMPLEQ_ENTRY(NBDRequest) entry;
+    NBDClient *client;
+    uint8_t *data;
+    bool complete;
+};
+
+struct NBDExport {
+    int refcount;
+    void (*close)(NBDExport *exp);
+
+    BlockBackend *blk;
+    char *name;
+    off_t dev_offset;
+    off_t size;
+    uint16_t nbdflags;
+    QTAILQ_HEAD(, NBDClient) clients;
+    QTAILQ_ENTRY(NBDExport) next;
+
+    AioContext *ctx;
+
+    Notifier eject_notifier;
+};
+
+static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
+
+struct NBDClient {
+    int refcount;
+    void (*close)(NBDClient *client);
+
+    NBDExport *exp;
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
+    QIOChannelSocket *sioc; /* The underlying data channel */
+    QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
+
+    Coroutine *recv_coroutine;
+
+    CoMutex send_lock;
+    Coroutine *send_coroutine;
+
+    bool can_read;
+
+    QTAILQ_ENTRY(NBDClient) next;
+    int nb_requests;
+    bool closing;
+};
+
+/* That's all folks */
+
+static void nbd_set_handlers(NBDClient *client);
+static void nbd_unset_handlers(NBDClient *client);
+static void nbd_update_can_read(NBDClient *client);
+
+static gboolean nbd_negotiate_continue(QIOChannel *ioc,
+                                       GIOCondition condition,
+                                       void *opaque)
+{
+    qemu_coroutine_enter(opaque);
+    return TRUE;
+}
+
+static ssize_t nbd_negotiate_read(QIOChannel *ioc, void *buffer, size_t size)
+{
+    ssize_t ret;
+    guint watch;
+
+    assert(qemu_in_coroutine());
+    /* Negotiation are always in main loop. */
+    watch = qio_channel_add_watch(ioc,
+                                  G_IO_IN,
+                                  nbd_negotiate_continue,
+                                  qemu_coroutine_self(),
+                                  NULL);
+    ret = read_sync(ioc, buffer, size);
+    g_source_remove(watch);
+    return ret;
+
+}
+
+static ssize_t nbd_negotiate_write(QIOChannel *ioc, void *buffer, size_t size)
+{
+    ssize_t ret;
+    guint watch;
+
+    assert(qemu_in_coroutine());
+    /* Negotiation are always in main loop. */
+    watch = qio_channel_add_watch(ioc,
+                                  G_IO_OUT,
+                                  nbd_negotiate_continue,
+                                  qemu_coroutine_self(),
+                                  NULL);
+    ret = write_sync(ioc, buffer, size);
+    g_source_remove(watch);
+    return ret;
+}
+
+static ssize_t nbd_negotiate_drop_sync(QIOChannel *ioc, size_t size)
+{
+    ssize_t ret, dropped = size;
+    uint8_t *buffer = g_malloc(MIN(65536, size));
+
+    while (size > 0) {
+        ret = nbd_negotiate_read(ioc, buffer, MIN(65536, size));
+        if (ret < 0) {
+            g_free(buffer);
+            return ret;
+        }
+
+        assert(ret <= size);
+        size -= ret;
+    }
+
+    g_free(buffer);
+    return dropped;
+}
+
+/* Basic flow for negotiation
+
+   Server         Client
+   Negotiate
+
+   or
+
+   Server         Client
+   Negotiate #1
+                  Option
+   Negotiate #2
+
+   ----
+
+   followed by
+
+   Server         Client
+                  Request
+   Response
+                  Request
+   Response
+                  ...
+   ...
+                  Request (type == 2)
+
+*/
+
+static int nbd_negotiate_send_rep(QIOChannel *ioc, uint32_t type, uint32_t opt)
+{
+    uint64_t magic;
+    uint32_t len;
+
+    TRACE("Reply opt=%" PRIx32 " type=%" PRIx32, type, opt);
+
+    magic = cpu_to_be64(NBD_REP_MAGIC);
+    if (nbd_negotiate_write(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        LOG("write failed (rep magic)");
+        return -EINVAL;
+    }
+    opt = cpu_to_be32(opt);
+    if (nbd_negotiate_write(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        LOG("write failed (rep opt)");
+        return -EINVAL;
+    }
+    type = cpu_to_be32(type);
+    if (nbd_negotiate_write(ioc, &type, sizeof(type)) != sizeof(type)) {
+        LOG("write failed (rep type)");
+        return -EINVAL;
+    }
+    len = cpu_to_be32(0);
+    if (nbd_negotiate_write(ioc, &len, sizeof(len)) != sizeof(len)) {
+        LOG("write failed (rep data length)");
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static int nbd_negotiate_send_rep_list(QIOChannel *ioc, NBDExport *exp)
+{
+    uint64_t magic, name_len;
+    uint32_t opt, type, len;
+
+    TRACE("Advertising export name '%s'", exp->name ? exp->name : "");
+    name_len = strlen(exp->name);
+    magic = cpu_to_be64(NBD_REP_MAGIC);
+    if (nbd_negotiate_write(ioc, &magic, sizeof(magic)) != sizeof(magic)) {
+        LOG("write failed (magic)");
+        return -EINVAL;
+     }
+    opt = cpu_to_be32(NBD_OPT_LIST);
+    if (nbd_negotiate_write(ioc, &opt, sizeof(opt)) != sizeof(opt)) {
+        LOG("write failed (opt)");
+        return -EINVAL;
+    }
+    type = cpu_to_be32(NBD_REP_SERVER);
+    if (nbd_negotiate_write(ioc, &type, sizeof(type)) != sizeof(type)) {
+        LOG("write failed (reply type)");
+        return -EINVAL;
+    }
+    len = cpu_to_be32(name_len + sizeof(len));
+    if (nbd_negotiate_write(ioc, &len, sizeof(len)) != sizeof(len)) {
+        LOG("write failed (length)");
+        return -EINVAL;
+    }
+    len = cpu_to_be32(name_len);
+    if (nbd_negotiate_write(ioc, &len, sizeof(len)) != sizeof(len)) {
+        LOG("write failed (length)");
+        return -EINVAL;
+    }
+    if (nbd_negotiate_write(ioc, exp->name, name_len) != name_len) {
+        LOG("write failed (buffer)");
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static int nbd_negotiate_handle_list(NBDClient *client, uint32_t length)
+{
+    NBDExport *exp;
+
+    if (length) {
+        if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+            return -EIO;
+        }
+        return nbd_negotiate_send_rep(client->ioc,
+                                      NBD_REP_ERR_INVALID, NBD_OPT_LIST);
+    }
+
+    /* For each export, send a NBD_REP_SERVER reply. */
+    QTAILQ_FOREACH(exp, &exports, next) {
+        if (nbd_negotiate_send_rep_list(client->ioc, exp)) {
+            return -EINVAL;
+        }
+    }
+    /* Finish with a NBD_REP_ACK. */
+    return nbd_negotiate_send_rep(client->ioc, NBD_REP_ACK, NBD_OPT_LIST);
+}
+
+static int nbd_negotiate_handle_export_name(NBDClient *client, uint32_t length)
+{
+    int rc = -EINVAL;
+    char name[NBD_MAX_NAME_SIZE + 1];
+
+    /* Client sends:
+        [20 ..  xx]   export name (length bytes)
+     */
+    TRACE("Checking length");
+    if (length >= sizeof(name)) {
+        LOG("Bad length received");
+        goto fail;
+    }
+    if (nbd_negotiate_read(client->ioc, name, length) != length) {
+        LOG("read failed");
+        goto fail;
+    }
+    name[length] = '\0';
+
+    TRACE("Client requested export '%s'", name);
+
+    client->exp = nbd_export_find(name);
+    if (!client->exp) {
+        LOG("export not found");
+        goto fail;
+    }
+
+    QTAILQ_INSERT_TAIL(&client->exp->clients, client, next);
+    nbd_export_get(client->exp);
+    rc = 0;
+fail:
+    return rc;
+}
+
+
+static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
+                                                 uint32_t length)
+{
+    QIOChannel *ioc;
+    QIOChannelTLS *tioc;
+    struct NBDTLSHandshakeData data = { 0 };
+
+    TRACE("Setting up TLS");
+    ioc = client->ioc;
+    if (length) {
+        if (nbd_negotiate_drop_sync(ioc, length) != length) {
+            return NULL;
+        }
+        nbd_negotiate_send_rep(ioc, NBD_REP_ERR_INVALID, NBD_OPT_STARTTLS);
+        return NULL;
+    }
+
+    if (nbd_negotiate_send_rep(client->ioc, NBD_REP_ACK,
+                               NBD_OPT_STARTTLS) < 0) {
+        return NULL;
+    }
+
+    tioc = qio_channel_tls_new_server(ioc,
+                                      client->tlscreds,
+                                      client->tlsaclname,
+                                      NULL);
+    if (!tioc) {
+        return NULL;
+    }
+
+    TRACE("Starting TLS handshake");
+    data.loop = g_main_loop_new(g_main_context_default(), FALSE);
+    qio_channel_tls_handshake(tioc,
+                              nbd_tls_handshake,
+                              &data,
+                              NULL);
+
+    if (!data.complete) {
+        g_main_loop_run(data.loop);
+    }
+    g_main_loop_unref(data.loop);
+    if (data.error) {
+        object_unref(OBJECT(tioc));
+        error_free(data.error);
+        return NULL;
+    }
+
+    return QIO_CHANNEL(tioc);
+}
+
+
+static int nbd_negotiate_options(NBDClient *client)
+{
+    uint32_t flags;
+    bool fixedNewstyle = false;
+
+    /* Client sends:
+        [ 0 ..   3]   client flags
+
+        [ 0 ..   7]   NBD_OPTS_MAGIC
+        [ 8 ..  11]   NBD option
+        [12 ..  15]   Data length
+        ...           Rest of request
+
+        [ 0 ..   7]   NBD_OPTS_MAGIC
+        [ 8 ..  11]   Second NBD option
+        [12 ..  15]   Data length
+        ...           Rest of request
+    */
+
+    if (nbd_negotiate_read(client->ioc, &flags, sizeof(flags)) !=
+        sizeof(flags)) {
+        LOG("read failed");
+        return -EIO;
+    }
+    TRACE("Checking client flags");
+    be32_to_cpus(&flags);
+    if (flags & NBD_FLAG_C_FIXED_NEWSTYLE) {
+        TRACE("Client supports fixed newstyle handshake");
+        fixedNewstyle = true;
+        flags &= ~NBD_FLAG_C_FIXED_NEWSTYLE;
+    }
+    if (flags != 0) {
+        TRACE("Unknown client flags 0x%" PRIx32 " received", flags);
+        return -EIO;
+    }
+
+    while (1) {
+        int ret;
+        uint32_t clientflags, length;
+        uint64_t magic;
+
+        if (nbd_negotiate_read(client->ioc, &magic, sizeof(magic)) !=
+            sizeof(magic)) {
+            LOG("read failed");
+            return -EINVAL;
+        }
+        TRACE("Checking opts magic");
+        if (magic != be64_to_cpu(NBD_OPTS_MAGIC)) {
+            LOG("Bad magic received");
+            return -EINVAL;
+        }
+
+        if (nbd_negotiate_read(client->ioc, &clientflags,
+                               sizeof(clientflags)) != sizeof(clientflags)) {
+            LOG("read failed");
+            return -EINVAL;
+        }
+        clientflags = be32_to_cpu(clientflags);
+
+        if (nbd_negotiate_read(client->ioc, &length, sizeof(length)) !=
+            sizeof(length)) {
+            LOG("read failed");
+            return -EINVAL;
+        }
+        length = be32_to_cpu(length);
+
+        TRACE("Checking option 0x%" PRIx32, clientflags);
+        if (client->tlscreds &&
+            client->ioc == (QIOChannel *)client->sioc) {
+            QIOChannel *tioc;
+            if (!fixedNewstyle) {
+                TRACE("Unsupported option 0x%" PRIx32, clientflags);
+                return -EINVAL;
+            }
+            switch (clientflags) {
+            case NBD_OPT_STARTTLS:
+                tioc = nbd_negotiate_handle_starttls(client, length);
+                if (!tioc) {
+                    return -EIO;
+                }
+                object_unref(OBJECT(client->ioc));
+                client->ioc = QIO_CHANNEL(tioc);
+                break;
+
+            case NBD_OPT_EXPORT_NAME:
+                /* No way to return an error to client, so drop connection */
+                TRACE("Option 0x%x not permitted before TLS", clientflags);
+                return -EINVAL;
+
+            default:
+                TRACE("Option 0x%" PRIx32 " not permitted before TLS",
+                      clientflags);
+                if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+                    return -EIO;
+                }
+                ret = nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_TLS_REQD,
+                                             clientflags);
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+            }
+        } else if (fixedNewstyle) {
+            switch (clientflags) {
+            case NBD_OPT_LIST:
+                ret = nbd_negotiate_handle_list(client, length);
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+
+            case NBD_OPT_ABORT:
+                return -EINVAL;
+
+            case NBD_OPT_EXPORT_NAME:
+                return nbd_negotiate_handle_export_name(client, length);
+
+            case NBD_OPT_STARTTLS:
+                if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+                    return -EIO;
+                }
+                if (client->tlscreds) {
+                    TRACE("TLS already enabled");
+                    ret = nbd_negotiate_send_rep(client->ioc,
+                                                 NBD_REP_ERR_INVALID,
+                                                 clientflags);
+                } else {
+                    TRACE("TLS not configured");
+                    ret = nbd_negotiate_send_rep(client->ioc,
+                                                 NBD_REP_ERR_POLICY,
+                                                 clientflags);
+                }
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+            default:
+                TRACE("Unsupported option 0x%" PRIx32, clientflags);
+                if (nbd_negotiate_drop_sync(client->ioc, length) != length) {
+                    return -EIO;
+                }
+                ret = nbd_negotiate_send_rep(client->ioc, NBD_REP_ERR_UNSUP,
+                                             clientflags);
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+            }
+        } else {
+            /*
+             * If broken new-style we should drop the connection
+             * for anything except NBD_OPT_EXPORT_NAME
+             */
+            switch (clientflags) {
+            case NBD_OPT_EXPORT_NAME:
+                return nbd_negotiate_handle_export_name(client, length);
+
+            default:
+                TRACE("Unsupported option 0x%" PRIx32, clientflags);
+                return -EINVAL;
+            }
+        }
+    }
+}
+
+typedef struct {
+    NBDClient *client;
+    Coroutine *co;
+} NBDClientNewData;
+
+static coroutine_fn int nbd_negotiate(NBDClientNewData *data)
+{
+    NBDClient *client = data->client;
+    char buf[8 + 8 + 8 + 128];
+    int rc;
+    const uint16_t myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
+                              NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
+    bool oldStyle;
+
+    /* Old style negotiation header without options
+        [ 0 ..   7]   passwd       ("NBDMAGIC")
+        [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
+        [16 ..  23]   size
+        [24 ..  25]   server flags (0)
+        [26 ..  27]   export flags
+        [28 .. 151]   reserved     (0)
+
+       New style negotiation header with options
+        [ 0 ..   7]   passwd       ("NBDMAGIC")
+        [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
+        [16 ..  17]   server flags (0)
+        ....options sent....
+        [18 ..  25]   size
+        [26 ..  27]   export flags
+        [28 .. 151]   reserved     (0)
+     */
+
+    qio_channel_set_blocking(client->ioc, false, NULL);
+    rc = -EINVAL;
+
+    TRACE("Beginning negotiation.");
+    memset(buf, 0, sizeof(buf));
+    memcpy(buf, "NBDMAGIC", 8);
+
+    oldStyle = client->exp != NULL && !client->tlscreds;
+    if (oldStyle) {
+        TRACE("advertising size %" PRIu64 " and flags %x",
+              client->exp->size, client->exp->nbdflags | myflags);
+        stq_be_p(buf + 8, NBD_CLIENT_MAGIC);
+        stq_be_p(buf + 16, client->exp->size);
+        stw_be_p(buf + 26, client->exp->nbdflags | myflags);
+    } else {
+        stq_be_p(buf + 8, NBD_OPTS_MAGIC);
+        stw_be_p(buf + 16, NBD_FLAG_FIXED_NEWSTYLE);
+    }
+
+    if (oldStyle) {
+        if (client->tlscreds) {
+            TRACE("TLS cannot be enabled with oldstyle protocol");
+            goto fail;
+        }
+        if (nbd_negotiate_write(client->ioc, buf, sizeof(buf)) != sizeof(buf)) {
+            LOG("write failed");
+            goto fail;
+        }
+    } else {
+        if (nbd_negotiate_write(client->ioc, buf, 18) != 18) {
+            LOG("write failed");
+            goto fail;
+        }
+        rc = nbd_negotiate_options(client);
+        if (rc != 0) {
+            LOG("option negotiation failed");
+            goto fail;
+        }
+
+        TRACE("advertising size %" PRIu64 " and flags %x",
+              client->exp->size, client->exp->nbdflags | myflags);
+        stq_be_p(buf + 18, client->exp->size);
+        stw_be_p(buf + 26, client->exp->nbdflags | myflags);
+        if (nbd_negotiate_write(client->ioc, buf + 18, sizeof(buf) - 18) !=
+            sizeof(buf) - 18) {
+            LOG("write failed");
+            goto fail;
+        }
+    }
+
+    TRACE("Negotiation succeeded.");
+    rc = 0;
+fail:
+    return rc;
+}
+
+static ssize_t nbd_receive_request(QIOChannel *ioc, struct nbd_request *request)
+{
+    uint8_t buf[NBD_REQUEST_SIZE];
+    uint32_t magic;
+    ssize_t ret;
+
+    ret = read_sync(ioc, buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret != sizeof(buf)) {
+        LOG("read failed");
+        return -EINVAL;
+    }
+
+    /* Request
+       [ 0 ..  3]   magic   (NBD_REQUEST_MAGIC)
+       [ 4 ..  7]   type    (0 == READ, 1 == WRITE)
+       [ 8 .. 15]   handle
+       [16 .. 23]   from
+       [24 .. 27]   len
+     */
+
+    magic = ldl_be_p(buf);
+    request->type   = ldl_be_p(buf + 4);
+    request->handle = ldq_be_p(buf + 8);
+    request->from   = ldq_be_p(buf + 16);
+    request->len    = ldl_be_p(buf + 24);
+
+    TRACE("Got request: { magic = 0x%" PRIx32 ", .type = %" PRIx32
+          ", from = %" PRIu64 " , len = %" PRIu32 " }",
+          magic, request->type, request->from, request->len);
+
+    if (magic != NBD_REQUEST_MAGIC) {
+        LOG("invalid magic (got 0x%" PRIx32 ")", magic);
+        return -EINVAL;
+    }
+    return 0;
+}
+
+static ssize_t nbd_send_reply(QIOChannel *ioc, struct nbd_reply *reply)
+{
+    uint8_t buf[NBD_REPLY_SIZE];
+    ssize_t ret;
+
+    reply->error = system_errno_to_nbd_errno(reply->error);
+
+    TRACE("Sending response to client: { .error = %" PRId32
+          ", handle = %" PRIu64 " }",
+          reply->error, reply->handle);
+
+    /* Reply
+       [ 0 ..  3]    magic   (NBD_REPLY_MAGIC)
+       [ 4 ..  7]    error   (0 == no error)
+       [ 7 .. 15]    handle
+     */
+    stl_be_p(buf, NBD_REPLY_MAGIC);
+    stl_be_p(buf + 4, reply->error);
+    stq_be_p(buf + 8, reply->handle);
+
+    ret = write_sync(ioc, buf, sizeof(buf));
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (ret != sizeof(buf)) {
+        LOG("writing to socket failed");
+        return -EINVAL;
+    }
+    return 0;
+}
+
+#define MAX_NBD_REQUESTS 16
+
+void nbd_client_get(NBDClient *client)
+{
+    client->refcount++;
+}
+
+void nbd_client_put(NBDClient *client)
+{
+    if (--client->refcount == 0) {
+        /* The last reference should be dropped by client->close,
+         * which is called by client_close.
+         */
+        assert(client->closing);
+
+        nbd_unset_handlers(client);
+        object_unref(OBJECT(client->sioc));
+        object_unref(OBJECT(client->ioc));
+        if (client->tlscreds) {
+            object_unref(OBJECT(client->tlscreds));
+        }
+        g_free(client->tlsaclname);
+        if (client->exp) {
+            QTAILQ_REMOVE(&client->exp->clients, client, next);
+            nbd_export_put(client->exp);
+        }
+        g_free(client);
+    }
+}
+
+static void client_close(NBDClient *client)
+{
+    if (client->closing) {
+        return;
+    }
+
+    client->closing = true;
+
+    /* Force requests to finish.  They will drop their own references,
+     * then we'll close the socket and free the NBDClient.
+     */
+    qio_channel_shutdown(client->ioc, QIO_CHANNEL_SHUTDOWN_BOTH,
+                         NULL);
+
+    /* Also tell the client, so that they release their reference.  */
+    if (client->close) {
+        client->close(client);
+    }
+}
+
+static NBDRequest *nbd_request_get(NBDClient *client)
+{
+    NBDRequest *req;
+
+    assert(client->nb_requests <= MAX_NBD_REQUESTS - 1);
+    client->nb_requests++;
+    nbd_update_can_read(client);
+
+    req = g_new0(NBDRequest, 1);
+    nbd_client_get(client);
+    req->client = client;
+    return req;
+}
+
+static void nbd_request_put(NBDRequest *req)
+{
+    NBDClient *client = req->client;
+
+    if (req->data) {
+        qemu_vfree(req->data);
+    }
+    g_free(req);
+
+    client->nb_requests--;
+    nbd_update_can_read(client);
+    nbd_client_put(client);
+}
+
+static void blk_aio_attached(AioContext *ctx, void *opaque)
+{
+    NBDExport *exp = opaque;
+    NBDClient *client;
+
+    TRACE("Export %s: Attaching clients to AIO context %p\n", exp->name, ctx);
+
+    exp->ctx = ctx;
+
+    QTAILQ_FOREACH(client, &exp->clients, next) {
+        nbd_set_handlers(client);
+    }
+}
+
+static void blk_aio_detach(void *opaque)
+{
+    NBDExport *exp = opaque;
+    NBDClient *client;
+
+    TRACE("Export %s: Detaching clients from AIO context %p\n", exp->name, exp->ctx);
+
+    QTAILQ_FOREACH(client, &exp->clients, next) {
+        nbd_unset_handlers(client);
+    }
+
+    exp->ctx = NULL;
+}
+
+static void nbd_eject_notifier(Notifier *n, void *data)
+{
+    NBDExport *exp = container_of(n, NBDExport, eject_notifier);
+    nbd_export_close(exp);
+}
+
+NBDExport *nbd_export_new(BlockBackend *blk, off_t dev_offset, off_t size,
+                          uint16_t nbdflags, void (*close)(NBDExport *),
+                          Error **errp)
+{
+    NBDExport *exp = g_malloc0(sizeof(NBDExport));
+    exp->refcount = 1;
+    QTAILQ_INIT(&exp->clients);
+    exp->blk = blk;
+    exp->dev_offset = dev_offset;
+    exp->nbdflags = nbdflags;
+    exp->size = size < 0 ? blk_getlength(blk) : size;
+    if (exp->size < 0) {
+        error_setg_errno(errp, -exp->size,
+                         "Failed to determine the NBD export's length");
+        goto fail;
+    }
+    exp->size -= exp->size % BDRV_SECTOR_SIZE;
+
+    exp->close = close;
+    exp->ctx = blk_get_aio_context(blk);
+    blk_ref(blk);
+    blk_add_aio_context_notifier(blk, blk_aio_attached, blk_aio_detach, exp);
+
+    exp->eject_notifier.notify = nbd_eject_notifier;
+    blk_add_remove_bs_notifier(blk, &exp->eject_notifier);
+
+    /*
+     * NBD exports are used for non-shared storage migration.  Make sure
+     * that BDRV_O_INACTIVE is cleared and the image is ready for write
+     * access since the export could be available before migration handover.
+     */
+    aio_context_acquire(exp->ctx);
+    blk_invalidate_cache(blk, NULL);
+    aio_context_release(exp->ctx);
+    return exp;
+
+fail:
+    g_free(exp);
+    return NULL;
+}
+
+NBDExport *nbd_export_find(const char *name)
+{
+    NBDExport *exp;
+    QTAILQ_FOREACH(exp, &exports, next) {
+        if (strcmp(name, exp->name) == 0) {
+            return exp;
+        }
+    }
+
+    return NULL;
+}
+
+void nbd_export_set_name(NBDExport *exp, const char *name)
+{
+    if (exp->name == name) {
+        return;
+    }
+
+    nbd_export_get(exp);
+    if (exp->name != NULL) {
+        g_free(exp->name);
+        exp->name = NULL;
+        QTAILQ_REMOVE(&exports, exp, next);
+        nbd_export_put(exp);
+    }
+    if (name != NULL) {
+        nbd_export_get(exp);
+        exp->name = g_strdup(name);
+        QTAILQ_INSERT_TAIL(&exports, exp, next);
+    }
+    nbd_export_put(exp);
+}
+
+void nbd_export_close(NBDExport *exp)
+{
+    NBDClient *client, *next;
+
+    nbd_export_get(exp);
+    QTAILQ_FOREACH_SAFE(client, &exp->clients, next, next) {
+        client_close(client);
+    }
+    nbd_export_set_name(exp, NULL);
+    nbd_export_put(exp);
+}
+
+void nbd_export_get(NBDExport *exp)
+{
+    assert(exp->refcount > 0);
+    exp->refcount++;
+}
+
+void nbd_export_put(NBDExport *exp)
+{
+    assert(exp->refcount > 0);
+    if (exp->refcount == 1) {
+        nbd_export_close(exp);
+    }
+
+    if (--exp->refcount == 0) {
+        assert(exp->name == NULL);
+
+        if (exp->close) {
+            exp->close(exp);
+        }
+
+        if (exp->blk) {
+            notifier_remove(&exp->eject_notifier);
+            blk_remove_aio_context_notifier(exp->blk, blk_aio_attached,
+                                            blk_aio_detach, exp);
+            blk_unref(exp->blk);
+            exp->blk = NULL;
+        }
+
+        g_free(exp);
+    }
+}
+
+BlockBackend *nbd_export_get_blockdev(NBDExport *exp)
+{
+    return exp->blk;
+}
+
+void nbd_export_close_all(void)
+{
+    NBDExport *exp, *next;
+
+    QTAILQ_FOREACH_SAFE(exp, &exports, next, next) {
+        nbd_export_close(exp);
+    }
+}
+
+static ssize_t nbd_co_send_reply(NBDRequest *req, struct nbd_reply *reply,
+                                 int len)
+{
+    NBDClient *client = req->client;
+    ssize_t rc, ret;
+
+    g_assert(qemu_in_coroutine());
+    qemu_co_mutex_lock(&client->send_lock);
+    client->send_coroutine = qemu_coroutine_self();
+    nbd_set_handlers(client);
+
+    if (!len) {
+        rc = nbd_send_reply(client->ioc, reply);
+    } else {
+        qio_channel_set_cork(client->ioc, true);
+        rc = nbd_send_reply(client->ioc, reply);
+        if (rc >= 0) {
+            ret = write_sync(client->ioc, req->data, len);
+            if (ret != len) {
+                rc = -EIO;
+            }
+        }
+        qio_channel_set_cork(client->ioc, false);
+    }
+
+    client->send_coroutine = NULL;
+    nbd_set_handlers(client);
+    qemu_co_mutex_unlock(&client->send_lock);
+    return rc;
+}
+
+/* Collect a client request.  Return 0 if request looks valid, -EAGAIN
+ * to keep trying the collection, -EIO to drop connection right away,
+ * and any other negative value to report an error to the client
+ * (although the caller may still need to disconnect after reporting
+ * the error).  */
+static ssize_t nbd_co_receive_request(NBDRequest *req,
+                                      struct nbd_request *request)
+{
+    NBDClient *client = req->client;
+    uint32_t command;
+    ssize_t rc;
+
+    g_assert(qemu_in_coroutine());
+    client->recv_coroutine = qemu_coroutine_self();
+    nbd_update_can_read(client);
+
+    rc = nbd_receive_request(client->ioc, request);
+    if (rc < 0) {
+        if (rc != -EAGAIN) {
+            rc = -EIO;
+        }
+        goto out;
+    }
+
+    TRACE("Decoding type");
+
+    command = request->type & NBD_CMD_MASK_COMMAND;
+    if (command != NBD_CMD_WRITE) {
+        /* No payload, we are ready to read the next request.  */
+        req->complete = true;
+    }
+
+    if (command == NBD_CMD_DISC) {
+        /* Special case: we're going to disconnect without a reply,
+         * whether or not flags, from, or len are bogus */
+        TRACE("Request type is DISCONNECT");
+        rc = -EIO;
+        goto out;
+    }
+
+    /* Check for sanity in the parameters, part 1.  Defer as many
+     * checks as possible until after reading any NBD_CMD_WRITE
+     * payload, so we can try and keep the connection alive.  */
+    if ((request->from + request->len) < request->from) {
+        LOG("integer overflow detected, you're probably being attacked");
+        rc = -EINVAL;
+        goto out;
+    }
+
+    if (command == NBD_CMD_READ || command == NBD_CMD_WRITE) {
+        if (request->len > NBD_MAX_BUFFER_SIZE) {
+            LOG("len (%" PRIu32" ) is larger than max len (%u)",
+                request->len, NBD_MAX_BUFFER_SIZE);
+            rc = -EINVAL;
+            goto out;
+        }
+
+        req->data = blk_try_blockalign(client->exp->blk, request->len);
+        if (req->data == NULL) {
+            rc = -ENOMEM;
+            goto out;
+        }
+    }
+    if (command == NBD_CMD_WRITE) {
+        TRACE("Reading %" PRIu32 " byte(s)", request->len);
+
+        if (read_sync(client->ioc, req->data, request->len) != request->len) {
+            LOG("reading from socket failed");
+            rc = -EIO;
+            goto out;
+        }
+        req->complete = true;
+    }
+
+    /* Sanity checks, part 2. */
+    if (request->from + request->len > client->exp->size) {
+        LOG("operation past EOF; From: %" PRIu64 ", Len: %" PRIu32
+            ", Size: %" PRIu64, request->from, request->len,
+            (uint64_t)client->exp->size);
+        rc = command == NBD_CMD_WRITE ? -ENOSPC : -EINVAL;
+        goto out;
+    }
+    if (request->type & ~NBD_CMD_MASK_COMMAND & ~NBD_CMD_FLAG_FUA) {
+        LOG("unsupported flags (got 0x%x)",
+            request->type & ~NBD_CMD_MASK_COMMAND);
+        rc = -EINVAL;
+        goto out;
+    }
+
+    rc = 0;
+
+out:
+    client->recv_coroutine = NULL;
+    nbd_update_can_read(client);
+
+    return rc;
+}
+
+static void nbd_trip(void *opaque)
+{
+    NBDClient *client = opaque;
+    NBDExport *exp = client->exp;
+    NBDRequest *req;
+    struct nbd_request request;
+    struct nbd_reply reply;
+    ssize_t ret;
+    uint32_t command;
+    int flags;
+
+    TRACE("Reading request.");
+    if (client->closing) {
+        return;
+    }
+
+    req = nbd_request_get(client);
+    ret = nbd_co_receive_request(req, &request);
+    if (ret == -EAGAIN) {
+        goto done;
+    }
+    if (ret == -EIO) {
+        goto out;
+    }
+
+    reply.handle = request.handle;
+    reply.error = 0;
+
+    if (ret < 0) {
+        reply.error = -ret;
+        goto error_reply;
+    }
+    command = request.type & NBD_CMD_MASK_COMMAND;
+
+    if (client->closing) {
+        /*
+         * The client may be closed when we are blocked in
+         * nbd_co_receive_request()
+         */
+        goto done;
+    }
+
+    switch (command) {
+    case NBD_CMD_READ:
+        TRACE("Request type is READ");
+
+        if (request.type & NBD_CMD_FLAG_FUA) {
+            ret = blk_co_flush(exp->blk);
+            if (ret < 0) {
+                LOG("flush failed");
+                reply.error = -ret;
+                goto error_reply;
+            }
+        }
+
+        ret = blk_pread(exp->blk, request.from + exp->dev_offset,
+                        req->data, request.len);
+        if (ret < 0) {
+            LOG("reading from file failed");
+            reply.error = -ret;
+            goto error_reply;
+        }
+
+        TRACE("Read %" PRIu32" byte(s)", request.len);
+        if (nbd_co_send_reply(req, &reply, request.len) < 0)
+            goto out;
+        break;
+    case NBD_CMD_WRITE:
+        TRACE("Request type is WRITE");
+
+        if (exp->nbdflags & NBD_FLAG_READ_ONLY) {
+            TRACE("Server is read-only, return error");
+            reply.error = EROFS;
+            goto error_reply;
+        }
+
+        TRACE("Writing to device");
+
+        flags = 0;
+        if (request.type & NBD_CMD_FLAG_FUA) {
+            flags |= BDRV_REQ_FUA;
+        }
+        ret = blk_pwrite(exp->blk, request.from + exp->dev_offset,
+                         req->data, request.len, flags);
+        if (ret < 0) {
+            LOG("writing to file failed");
+            reply.error = -ret;
+            goto error_reply;
+        }
+
+        if (nbd_co_send_reply(req, &reply, 0) < 0) {
+            goto out;
+        }
+        break;
+
+    case NBD_CMD_DISC:
+        /* unreachable, thanks to special case in nbd_co_receive_request() */
+        abort();
+
+    case NBD_CMD_FLUSH:
+        TRACE("Request type is FLUSH");
+
+        ret = blk_co_flush(exp->blk);
+        if (ret < 0) {
+            LOG("flush failed");
+            reply.error = -ret;
+        }
+        if (nbd_co_send_reply(req, &reply, 0) < 0) {
+            goto out;
+        }
+        break;
+    case NBD_CMD_TRIM:
+        TRACE("Request type is TRIM");
+        ret = blk_co_pdiscard(exp->blk, request.from + exp->dev_offset,
+                              request.len);
+        if (ret < 0) {
+            LOG("discard failed");
+            reply.error = -ret;
+        }
+        if (nbd_co_send_reply(req, &reply, 0) < 0) {
+            goto out;
+        }
+        break;
+    default:
+        LOG("invalid request type (%" PRIu32 ") received", request.type);
+        reply.error = EINVAL;
+    error_reply:
+        /* We must disconnect after NBD_CMD_WRITE if we did not
+         * read the payload.
+         */
+        if (nbd_co_send_reply(req, &reply, 0) < 0 || !req->complete) {
+            goto out;
+        }
+        break;
+    }
+
+    TRACE("Request/Reply complete");
+
+done:
+    nbd_request_put(req);
+    return;
+
+out:
+    nbd_request_put(req);
+    client_close(client);
+}
+
+static void nbd_read(void *opaque)
+{
+    NBDClient *client = opaque;
+
+    if (client->recv_coroutine) {
+        qemu_coroutine_enter(client->recv_coroutine);
+    } else {
+        qemu_coroutine_enter(qemu_coroutine_create(nbd_trip, client));
+    }
+}
+
+static void nbd_restart_write(void *opaque)
+{
+    NBDClient *client = opaque;
+
+    qemu_coroutine_enter(client->send_coroutine);
+}
+
+static void nbd_set_handlers(NBDClient *client)
+{
+    if (client->exp && client->exp->ctx) {
+        aio_set_fd_handler(client->exp->ctx, client->sioc->fd,
+                           true,
+                           client->can_read ? nbd_read : NULL,
+                           client->send_coroutine ? nbd_restart_write : NULL,
+                           client);
+    }
+}
+
+static void nbd_unset_handlers(NBDClient *client)
+{
+    if (client->exp && client->exp->ctx) {
+        aio_set_fd_handler(client->exp->ctx, client->sioc->fd,
+                           true, NULL, NULL, NULL);
+    }
+}
+
+static void nbd_update_can_read(NBDClient *client)
+{
+    bool can_read = client->recv_coroutine ||
+                    client->nb_requests < MAX_NBD_REQUESTS;
+
+    if (can_read != client->can_read) {
+        client->can_read = can_read;
+        nbd_set_handlers(client);
+
+        /* There is no need to invoke aio_notify(), since aio_set_fd_handler()
+         * in nbd_set_handlers() will have taken care of that */
+    }
+}
+
+static coroutine_fn void nbd_co_client_start(void *opaque)
+{
+    NBDClientNewData *data = opaque;
+    NBDClient *client = data->client;
+    NBDExport *exp = client->exp;
+
+    if (exp) {
+        nbd_export_get(exp);
+    }
+    if (nbd_negotiate(data)) {
+        client_close(client);
+        goto out;
+    }
+    qemu_co_mutex_init(&client->send_lock);
+    nbd_set_handlers(client);
+
+    if (exp) {
+        QTAILQ_INSERT_TAIL(&exp->clients, client, next);
+    }
+out:
+    g_free(data);
+}
+
+void nbd_client_new(NBDExport *exp,
+                    QIOChannelSocket *sioc,
+                    QCryptoTLSCreds *tlscreds,
+                    const char *tlsaclname,
+                    void (*close_fn)(NBDClient *))
+{
+    NBDClient *client;
+    NBDClientNewData *data = g_new(NBDClientNewData, 1);
+
+    client = g_malloc0(sizeof(NBDClient));
+    client->refcount = 1;
+    client->exp = exp;
+    client->tlscreds = tlscreds;
+    if (tlscreds) {
+        object_ref(OBJECT(client->tlscreds));
+    }
+    client->tlsaclname = g_strdup(tlsaclname);
+    client->sioc = sioc;
+    object_ref(OBJECT(client->sioc));
+    client->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(client->ioc));
+    client->can_read = true;
+    client->close = close_fn;
+
+    data->client = client;
+    data->co = qemu_coroutine_create(nbd_co_client_start, data);
+    qemu_coroutine_enter(data->co);
+}
diff --git a/net/Makefile.objs b/net/Makefile.objs
index ec19cb3..b7c22fd 100644
--- a/net/Makefile.objs
+++ b/net/Makefile.objs
@@ -13,3 +13,6 @@
 common-obj-$(CONFIG_SLIRP) += slirp.o
 common-obj-$(CONFIG_VDE) += vde.o
 common-obj-$(CONFIG_NETMAP) += netmap.o
+common-obj-y += filter.o
+common-obj-y += filter-buffer.o
+common-obj-y += filter-mirror.o
diff --git a/net/checksum.c b/net/checksum.c
index 14c0855..23323b0 100644
--- a/net/checksum.c
+++ b/net/checksum.c
@@ -15,11 +15,10 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "net/checksum.h"
-
-#define PROTO_TCP  6
-#define PROTO_UDP 17
+#include "net/eth.h"
 
 uint32_t net_checksum_add_cont(int len, uint8_t *buf, int seq)
 {
@@ -56,44 +55,118 @@
 
 void net_checksum_calculate(uint8_t *data, int length)
 {
-    int hlen, plen, proto, csum_offset;
-    uint16_t csum;
+    int mac_hdr_len, ip_len;
+    struct ip_header *ip;
 
-    if ((data[14] & 0xf0) != 0x40)
-	return; /* not IPv4 */
-    hlen  = (data[14] & 0x0f) * 4;
-    plen  = (data[16] << 8 | data[17]) - hlen;
-    proto = data[23];
+    /*
+     * Note: We cannot assume "data" is aligned, so the all code uses
+     * some macros that take care of possible unaligned access for
+     * struct members (just in case).
+     */
 
-    switch (proto) {
-    case PROTO_TCP:
-	csum_offset = 16;
-	break;
-    case PROTO_UDP:
-	csum_offset = 6;
-	break;
-    default:
-	return;
+    /* Ensure we have at least an Eth header */
+    if (length < sizeof(struct eth_header)) {
+        return;
     }
 
-    if (plen < csum_offset+2)
-	return;
+    /* Handle the optionnal VLAN headers */
+    switch (lduw_be_p(&PKT_GET_ETH_HDR(data)->h_proto)) {
+    case ETH_P_VLAN:
+        mac_hdr_len = sizeof(struct eth_header) +
+                     sizeof(struct vlan_header);
+        break;
+    case ETH_P_DVLAN:
+        if (lduw_be_p(&PKT_GET_VLAN_HDR(data)->h_proto) == ETH_P_VLAN) {
+            mac_hdr_len = sizeof(struct eth_header) +
+                         2 * sizeof(struct vlan_header);
+        } else {
+            mac_hdr_len = sizeof(struct eth_header) +
+                         sizeof(struct vlan_header);
+        }
+        break;
+    default:
+        mac_hdr_len = sizeof(struct eth_header);
+        break;
+    }
 
-    data[14+hlen+csum_offset]   = 0;
-    data[14+hlen+csum_offset+1] = 0;
-    csum = net_checksum_tcpudp(plen, proto, data+14+12, data+14+hlen);
-    data[14+hlen+csum_offset]   = csum >> 8;
-    data[14+hlen+csum_offset+1] = csum & 0xff;
+    length -= mac_hdr_len;
+
+    /* Now check we have an IP header (with an optionnal VLAN header) */
+    if (length < sizeof(struct ip_header)) {
+        return;
+    }
+
+    ip = (struct ip_header *)(data + mac_hdr_len);
+
+    if (IP_HEADER_VERSION(ip) != IP_HEADER_VERSION_4) {
+        return; /* not IPv4 */
+    }
+
+    ip_len = lduw_be_p(&ip->ip_len);
+
+    /* Last, check that we have enough data for the all IP frame */
+    if (length < ip_len) {
+        return;
+    }
+
+    ip_len -= IP_HDR_GET_LEN(ip);
+
+    switch (ip->ip_p) {
+    case IP_PROTO_TCP:
+    {
+        uint16_t csum;
+        tcp_header *tcp = (tcp_header *)(ip + 1);
+
+        if (ip_len < sizeof(tcp_header)) {
+            return;
+        }
+
+        /* Set csum to 0 */
+        stw_he_p(&tcp->th_sum, 0);
+
+        csum = net_checksum_tcpudp(ip_len, ip->ip_p,
+                                   (uint8_t *)&ip->ip_src,
+                                   (uint8_t *)tcp);
+
+        /* Store computed csum */
+        stw_be_p(&tcp->th_sum, csum);
+
+        break;
+    }
+    case IP_PROTO_UDP:
+    {
+        uint16_t csum;
+        udp_header *udp = (udp_header *)(ip + 1);
+
+        if (ip_len < sizeof(udp_header)) {
+            return;
+        }
+
+        /* Set csum to 0 */
+        stw_he_p(&udp->uh_sum, 0);
+
+        csum = net_checksum_tcpudp(ip_len, ip->ip_p,
+                                   (uint8_t *)&ip->ip_src,
+                                   (uint8_t *)udp);
+
+        /* Store computed csum */
+        stw_be_p(&udp->uh_sum, csum);
+
+        break;
+    }
+    default:
+        /* Can't handle any other protocol */
+        break;
+    }
 }
 
 uint32_t
 net_checksum_add_iov(const struct iovec *iov, const unsigned int iov_cnt,
-                     uint32_t iov_off, uint32_t size)
+                     uint32_t iov_off, uint32_t size, uint32_t csum_offset)
 {
     size_t iovec_off, buf_off;
     unsigned int i;
     uint32_t res = 0;
-    uint32_t seq = 0;
 
     iovec_off = 0;
     buf_off = 0;
@@ -102,8 +175,8 @@
             size_t len = MIN((iovec_off + iov[i].iov_len) - iov_off , size);
             void *chunk_buf = iov[i].iov_base + (iov_off - iovec_off);
 
-            res += net_checksum_add_cont(len, chunk_buf, seq);
-            seq += len;
+            res += net_checksum_add_cont(len, chunk_buf, csum_offset);
+            csum_offset += len;
 
             buf_off += len;
             iov_off += len;
diff --git a/net/clients.h b/net/clients.h
index 2e8feda..5cae479 100644
--- a/net/clients.h
+++ b/net/clients.h
@@ -27,39 +27,39 @@
 #include "net/net.h"
 #include "qapi-types.h"
 
-int net_init_dump(const NetClientOptions *opts, const char *name,
-                  NetClientState *peer);
+int net_init_dump(const Netdev *netdev, const char *name,
+                  NetClientState *peer, Error **errp);
 
 #ifdef CONFIG_SLIRP
-int net_init_slirp(const NetClientOptions *opts, const char *name,
-                   NetClientState *peer);
+int net_init_slirp(const Netdev *netdev, const char *name,
+                   NetClientState *peer, Error **errp);
 #endif
 
-int net_init_hubport(const NetClientOptions *opts, const char *name,
-                     NetClientState *peer);
+int net_init_hubport(const Netdev *netdev, const char *name,
+                     NetClientState *peer, Error **errp);
 
-int net_init_socket(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
+int net_init_socket(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp);
 
-int net_init_tap(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer);
+int net_init_tap(const Netdev *netdev, const char *name,
+                 NetClientState *peer, Error **errp);
 
-int net_init_bridge(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
+int net_init_bridge(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp);
 
-int net_init_l2tpv3(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
+int net_init_l2tpv3(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp);
 #ifdef CONFIG_VDE
-int net_init_vde(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer);
+int net_init_vde(const Netdev *netdev, const char *name,
+                 NetClientState *peer, Error **errp);
 #endif
 
 #ifdef CONFIG_NETMAP
-int net_init_netmap(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
+int net_init_netmap(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp);
 #endif
 
-int net_init_vhost_user(const NetClientOptions *opts, const char *name,
-                        NetClientState *peer);
+int net_init_vhost_user(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp);
 
 #endif /* QEMU_NET_CLIENTS_H */
diff --git a/net/dump.c b/net/dump.c
index 90fa3ab..219d8d0 100644
--- a/net/dump.c
+++ b/net/dump.c
@@ -22,15 +22,18 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "clients.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
+#include "qemu/iov.h"
 #include "qemu/log.h"
 #include "qemu/timer.h"
-#include "hub.h"
+#include "qapi/visitor.h"
+#include "net/filter.h"
 
 typedef struct DumpState {
-    NetClientState nc;
     int64_t start_ts;
     int fd;
     int pcap_caplen;
@@ -57,28 +60,33 @@
     uint32_t len;
 };
 
-static ssize_t dump_receive(NetClientState *nc, const uint8_t *buf, size_t size)
+static ssize_t dump_receive_iov(DumpState *s, const struct iovec *iov, int cnt)
 {
-    DumpState *s = DO_UPCAST(DumpState, nc, nc);
     struct pcap_sf_pkthdr hdr;
     int64_t ts;
     int caplen;
+    size_t size = iov_size(iov, cnt);
+    struct iovec dumpiov[cnt + 1];
 
     /* Early return in case of previous error. */
     if (s->fd < 0) {
         return size;
     }
 
-    ts = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL), 1000000, get_ticks_per_sec());
+    ts = qemu_clock_get_us(QEMU_CLOCK_VIRTUAL);
     caplen = size > s->pcap_caplen ? s->pcap_caplen : size;
 
     hdr.ts.tv_sec = ts / 1000000 + s->start_ts;
     hdr.ts.tv_usec = ts % 1000000;
     hdr.caplen = caplen;
     hdr.len = size;
-    if (write(s->fd, &hdr, sizeof(hdr)) != sizeof(hdr) ||
-        write(s->fd, buf, caplen) != caplen) {
-        qemu_log("-net dump write error - stop dump\n");
+
+    dumpiov[0].iov_base = &hdr;
+    dumpiov[0].iov_len = sizeof(hdr);
+    cnt = iov_copy(&dumpiov[1], cnt, iov, cnt, 0, caplen);
+
+    if (writev(s->fd, dumpiov, cnt + 1) != sizeof(hdr) + caplen) {
+        error_report("network dump write error - stopping dump");
         close(s->fd);
         s->fd = -1;
     }
@@ -86,32 +94,22 @@
     return size;
 }
 
-static void dump_cleanup(NetClientState *nc)
+static void dump_cleanup(DumpState *s)
 {
-    DumpState *s = DO_UPCAST(DumpState, nc, nc);
-
     close(s->fd);
+    s->fd = -1;
 }
 
-static NetClientInfo net_dump_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_DUMP,
-    .size = sizeof(DumpState),
-    .receive = dump_receive,
-    .cleanup = dump_cleanup,
-};
-
-static int net_dump_init(NetClientState *peer, const char *device,
-                         const char *name, const char *filename, int len)
+static int net_dump_state_init(DumpState *s, const char *filename,
+                               int len, Error **errp)
 {
     struct pcap_file_hdr hdr;
-    NetClientState *nc;
-    DumpState *s;
     struct tm tm;
     int fd;
 
     fd = open(filename, O_CREAT | O_TRUNC | O_WRONLY | O_BINARY, 0644);
     if (fd < 0) {
-        error_report("-net dump: can't open %s", filename);
+        error_setg_errno(errp, errno, "-net dump: can't open %s", filename);
         return -1;
     }
 
@@ -124,18 +122,11 @@
     hdr.linktype = 1;
 
     if (write(fd, &hdr, sizeof(hdr)) < sizeof(hdr)) {
-        error_report("-net dump write error: %s", strerror(errno));
+        error_setg_errno(errp, errno, "-net dump write error");
         close(fd);
         return -1;
     }
 
-    nc = qemu_new_net_client(&net_dump_info, peer, device, name);
-
-    snprintf(nc->info_str, sizeof(nc->info_str),
-             "dump to %s (len=%d)", filename, len);
-
-    s = DO_UPCAST(DumpState, nc, nc);
-
     s->fd = fd;
     s->pcap_caplen = len;
 
@@ -145,16 +136,61 @@
     return 0;
 }
 
-int net_init_dump(const NetClientOptions *opts, const char *name,
-                  NetClientState *peer)
+/* Dumping via VLAN netclient */
+
+struct DumpNetClient {
+    NetClientState nc;
+    DumpState ds;
+};
+typedef struct DumpNetClient DumpNetClient;
+
+static ssize_t dumpclient_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
 {
-    int len;
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = size
+    };
+
+    return dump_receive_iov(&dc->ds, &iov, 1);
+}
+
+static ssize_t dumpclient_receive_iov(NetClientState *nc,
+                                      const struct iovec *iov, int cnt)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+
+    return dump_receive_iov(&dc->ds, iov, cnt);
+}
+
+static void dumpclient_cleanup(NetClientState *nc)
+{
+    DumpNetClient *dc = DO_UPCAST(DumpNetClient, nc, nc);
+
+    dump_cleanup(&dc->ds);
+}
+
+static NetClientInfo net_dump_info = {
+    .type = NET_CLIENT_DRIVER_DUMP,
+    .size = sizeof(DumpNetClient),
+    .receive = dumpclient_receive,
+    .receive_iov = dumpclient_receive_iov,
+    .cleanup = dumpclient_cleanup,
+};
+
+int net_init_dump(const Netdev *netdev, const char *name,
+                  NetClientState *peer, Error **errp)
+{
+    int len, rc;
     const char *file;
     char def_file[128];
     const NetdevDumpOptions *dump;
+    NetClientState *nc;
+    DumpNetClient *dnc;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_DUMP);
-    dump = opts->dump;
+    assert(netdev->type == NET_CLIENT_DRIVER_DUMP);
+    dump = &netdev->u.dump;
 
     assert(peer);
 
@@ -174,7 +210,7 @@
 
     if (dump->has_len) {
         if (dump->len > INT_MAX) {
-            error_report("invalid length: %"PRIu64, dump->len);
+            error_setg(errp, "invalid length: %"PRIu64, dump->len);
             return -1;
         }
         len = dump->len;
@@ -182,5 +218,148 @@
         len = 65536;
     }
 
-    return net_dump_init(peer, "dump", name, file, len);
+    nc = qemu_new_net_client(&net_dump_info, peer, "dump", name);
+    snprintf(nc->info_str, sizeof(nc->info_str),
+             "dump to %s (len=%d)", file, len);
+
+    dnc = DO_UPCAST(DumpNetClient, nc, nc);
+    rc = net_dump_state_init(&dnc->ds, file, len, errp);
+    if (rc) {
+        qemu_del_net_client(nc);
+    }
+    return rc;
 }
+
+/* Dumping via filter */
+
+#define TYPE_FILTER_DUMP "filter-dump"
+
+#define FILTER_DUMP(obj) \
+    OBJECT_CHECK(NetFilterDumpState, (obj), TYPE_FILTER_DUMP)
+
+struct NetFilterDumpState {
+    NetFilterState nfs;
+    DumpState ds;
+    char *filename;
+    uint32_t maxlen;
+};
+typedef struct NetFilterDumpState NetFilterDumpState;
+
+static ssize_t filter_dump_receive_iov(NetFilterState *nf, NetClientState *sndr,
+                                       unsigned flags, const struct iovec *iov,
+                                       int iovcnt, NetPacketSent *sent_cb)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    dump_receive_iov(&nfds->ds, iov, iovcnt);
+    return 0;
+}
+
+static void filter_dump_cleanup(NetFilterState *nf)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    dump_cleanup(&nfds->ds);
+}
+
+static void filter_dump_setup(NetFilterState *nf, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(nf);
+
+    if (!nfds->filename) {
+        error_setg(errp, "dump filter needs 'file' property set!");
+        return;
+    }
+
+    net_dump_state_init(&nfds->ds, nfds->filename, nfds->maxlen, errp);
+}
+
+static void filter_dump_get_maxlen(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+    uint32_t value = nfds->maxlen;
+
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void filter_dump_set_maxlen(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (value == 0) {
+        error_setg(&local_err, "Property '%s.%s' doesn't take value '%u'",
+                   object_get_typename(obj), name, value);
+        goto out;
+    }
+    nfds->maxlen = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static char *file_dump_get_filename(Object *obj, Error **errp)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    return g_strdup(nfds->filename);
+}
+
+static void file_dump_set_filename(Object *obj, const char *value, Error **errp)
+{
+   NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    g_free(nfds->filename);
+    nfds->filename = g_strdup(value);
+}
+
+static void filter_dump_instance_init(Object *obj)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    nfds->maxlen = 65536;
+
+    object_property_add(obj, "maxlen", "int", filter_dump_get_maxlen,
+                        filter_dump_set_maxlen, NULL, NULL, NULL);
+    object_property_add_str(obj, "file", file_dump_get_filename,
+                            file_dump_set_filename, NULL);
+}
+
+static void filter_dump_instance_finalize(Object *obj)
+{
+    NetFilterDumpState *nfds = FILTER_DUMP(obj);
+
+    g_free(nfds->filename);
+}
+
+static void filter_dump_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_dump_setup;
+    nfc->cleanup = filter_dump_cleanup;
+    nfc->receive_iov = filter_dump_receive_iov;
+}
+
+static const TypeInfo filter_dump_info = {
+    .name = TYPE_FILTER_DUMP,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_dump_class_init,
+    .instance_init = filter_dump_instance_init,
+    .instance_finalize = filter_dump_instance_finalize,
+    .instance_size = sizeof(NetFilterDumpState),
+};
+
+static void filter_dump_register_types(void)
+{
+    type_register_static(&filter_dump_info);
+}
+
+type_init(filter_dump_register_types);
diff --git a/net/eth.c b/net/eth.c
index 7c61132..df81efb 100644
--- a/net/eth.c
+++ b/net/eth.c
@@ -15,13 +15,14 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "net/eth.h"
 #include "net/checksum.h"
 #include "qemu-common.h"
 #include "net/tap.h"
 
-void eth_setup_vlan_headers(struct eth_header *ehdr, uint16_t vlan_tag,
-    bool *is_new)
+void eth_setup_vlan_headers_ex(struct eth_header *ehdr, uint16_t vlan_tag,
+    uint16_t vlan_ethtype, bool *is_new)
 {
     struct vlan_header *vhdr = PKT_GET_VLAN_HDR(ehdr);
 
@@ -35,7 +36,7 @@
     default:
         /* No VLAN header, put a new one */
         vhdr->h_proto = ehdr->h_proto;
-        ehdr->h_proto = cpu_to_be16(ETH_P_VLAN);
+        ehdr->h_proto = cpu_to_be16(vlan_ethtype);
         *is_new = true;
         break;
     }
@@ -78,26 +79,100 @@
     return VIRTIO_NET_HDR_GSO_NONE | ecn_state;
 }
 
-void eth_get_protocols(const uint8_t *headers,
-                       uint32_t hdr_length,
+uint16_t
+eth_get_l3_proto(const struct iovec *l2hdr_iov, int iovcnt, size_t l2hdr_len)
+{
+    uint16_t proto;
+    size_t copied;
+    size_t size = iov_size(l2hdr_iov, iovcnt);
+    size_t proto_offset = l2hdr_len - sizeof(proto);
+
+    if (size < proto_offset) {
+        return ETH_P_UNKNOWN;
+    }
+
+    copied = iov_to_buf(l2hdr_iov, iovcnt, proto_offset,
+                        &proto, sizeof(proto));
+
+    return (copied == sizeof(proto)) ? be16_to_cpu(proto) : ETH_P_UNKNOWN;
+}
+
+static bool
+_eth_copy_chunk(size_t input_size,
+                const struct iovec *iov, int iovcnt,
+                size_t offset, size_t length,
+                void *buffer)
+{
+    size_t copied;
+
+    if (input_size < offset) {
+        return false;
+    }
+
+    copied = iov_to_buf(iov, iovcnt, offset, buffer, length);
+
+    if (copied < length) {
+        return false;
+    }
+
+    return true;
+}
+
+static bool
+_eth_tcp_has_data(bool is_ip4,
+                  const struct ip_header  *ip4_hdr,
+                  const struct ip6_header *ip6_hdr,
+                  size_t full_ip6hdr_len,
+                  const struct tcp_header *tcp)
+{
+    uint32_t l4len;
+
+    if (is_ip4) {
+        l4len = be16_to_cpu(ip4_hdr->ip_len) - IP_HDR_GET_LEN(ip4_hdr);
+    } else {
+        size_t opts_len = full_ip6hdr_len - sizeof(struct ip6_header);
+        l4len = be16_to_cpu(ip6_hdr->ip6_ctlun.ip6_un1.ip6_un1_plen) - opts_len;
+    }
+
+    return l4len > TCP_HEADER_DATA_OFFSET(tcp);
+}
+
+void eth_get_protocols(const struct iovec *iov, int iovcnt,
                        bool *isip4, bool *isip6,
-                       bool *isudp, bool *istcp)
+                       bool *isudp, bool *istcp,
+                       size_t *l3hdr_off,
+                       size_t *l4hdr_off,
+                       size_t *l5hdr_off,
+                       eth_ip6_hdr_info *ip6hdr_info,
+                       eth_ip4_hdr_info *ip4hdr_info,
+                       eth_l4_hdr_info  *l4hdr_info)
 {
     int proto;
-    size_t l2hdr_len = eth_get_l2_hdr_length(headers);
-    assert(hdr_length >= eth_get_l2_hdr_length(headers));
+    bool fragment = false;
+    size_t l2hdr_len = eth_get_l2_hdr_length_iov(iov, iovcnt);
+    size_t input_size = iov_size(iov, iovcnt);
+    size_t copied;
+
     *isip4 = *isip6 = *isudp = *istcp = false;
 
-    proto = eth_get_l3_proto(headers, l2hdr_len);
+    proto = eth_get_l3_proto(iov, iovcnt, l2hdr_len);
+
+    *l3hdr_off = l2hdr_len;
+
     if (proto == ETH_P_IP) {
+        struct ip_header *iphdr = &ip4hdr_info->ip4_hdr;
+
+        if (input_size < l2hdr_len) {
+            return;
+        }
+
+        copied = iov_to_buf(iov, iovcnt, l2hdr_len, iphdr, sizeof(*iphdr));
+
         *isip4 = true;
 
-        struct ip_header *iphdr;
-
-        assert(hdr_length >=
-            eth_get_l2_hdr_length(headers) + sizeof(struct ip_header));
-
-        iphdr = PKT_GET_IP_HDR(headers);
+        if (copied < sizeof(*iphdr)) {
+            return;
+        }
 
         if (IP_HEADER_VERSION(iphdr) == IP_HEADER_VERSION_4) {
             if (iphdr->ip_p == IP_PROTO_TCP) {
@@ -106,33 +181,149 @@
                 *isudp = true;
             }
         }
-    } else if (proto == ETH_P_IPV6) {
-        uint8_t l4proto;
-        size_t full_ip6hdr_len;
 
-        struct iovec hdr_vec;
-        hdr_vec.iov_base = (void *) headers;
-        hdr_vec.iov_len = hdr_length;
+        ip4hdr_info->fragment = IP4_IS_FRAGMENT(iphdr);
+        *l4hdr_off = l2hdr_len + IP_HDR_GET_LEN(iphdr);
+
+        fragment = ip4hdr_info->fragment;
+    } else if (proto == ETH_P_IPV6) {
 
         *isip6 = true;
-        if (eth_parse_ipv6_hdr(&hdr_vec, 1, l2hdr_len,
-                              &l4proto, &full_ip6hdr_len)) {
-            if (l4proto == IP_PROTO_TCP) {
+        if (eth_parse_ipv6_hdr(iov, iovcnt, l2hdr_len,
+                               ip6hdr_info)) {
+            if (ip6hdr_info->l4proto == IP_PROTO_TCP) {
                 *istcp = true;
-            } else if (l4proto == IP_PROTO_UDP) {
+            } else if (ip6hdr_info->l4proto == IP_PROTO_UDP) {
                 *isudp = true;
             }
+        } else {
+            return;
+        }
+
+        *l4hdr_off = l2hdr_len + ip6hdr_info->full_hdr_len;
+        fragment = ip6hdr_info->fragment;
+    }
+
+    if (!fragment) {
+        if (*istcp) {
+            *istcp = _eth_copy_chunk(input_size,
+                                     iov, iovcnt,
+                                     *l4hdr_off, sizeof(l4hdr_info->hdr.tcp),
+                                     &l4hdr_info->hdr.tcp);
+
+            if (*istcp) {
+                *l5hdr_off = *l4hdr_off +
+                    TCP_HEADER_DATA_OFFSET(&l4hdr_info->hdr.tcp);
+
+                l4hdr_info->has_tcp_data =
+                    _eth_tcp_has_data(proto == ETH_P_IP,
+                                      &ip4hdr_info->ip4_hdr,
+                                      &ip6hdr_info->ip6_hdr,
+                                      *l4hdr_off - *l3hdr_off,
+                                      &l4hdr_info->hdr.tcp);
+            }
+        } else if (*isudp) {
+            *isudp = _eth_copy_chunk(input_size,
+                                     iov, iovcnt,
+                                     *l4hdr_off, sizeof(l4hdr_info->hdr.udp),
+                                     &l4hdr_info->hdr.udp);
+            *l5hdr_off = *l4hdr_off + sizeof(l4hdr_info->hdr.udp);
         }
     }
 }
 
+bool
+eth_strip_vlan(const struct iovec *iov, int iovcnt, size_t iovoff,
+               uint8_t *new_ehdr_buf,
+               uint16_t *payload_offset, uint16_t *tci)
+{
+    struct vlan_header vlan_hdr;
+    struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf;
+
+    size_t copied = iov_to_buf(iov, iovcnt, iovoff,
+                               new_ehdr, sizeof(*new_ehdr));
+
+    if (copied < sizeof(*new_ehdr)) {
+        return false;
+    }
+
+    switch (be16_to_cpu(new_ehdr->h_proto)) {
+    case ETH_P_VLAN:
+    case ETH_P_DVLAN:
+        copied = iov_to_buf(iov, iovcnt, iovoff + sizeof(*new_ehdr),
+                            &vlan_hdr, sizeof(vlan_hdr));
+
+        if (copied < sizeof(vlan_hdr)) {
+            return false;
+        }
+
+        new_ehdr->h_proto = vlan_hdr.h_proto;
+
+        *tci = be16_to_cpu(vlan_hdr.h_tci);
+        *payload_offset = iovoff + sizeof(*new_ehdr) + sizeof(vlan_hdr);
+
+        if (be16_to_cpu(new_ehdr->h_proto) == ETH_P_VLAN) {
+
+            copied = iov_to_buf(iov, iovcnt, *payload_offset,
+                                PKT_GET_VLAN_HDR(new_ehdr), sizeof(vlan_hdr));
+
+            if (copied < sizeof(vlan_hdr)) {
+                return false;
+            }
+
+            *payload_offset += sizeof(vlan_hdr);
+        }
+        return true;
+    default:
+        return false;
+    }
+}
+
+bool
+eth_strip_vlan_ex(const struct iovec *iov, int iovcnt, size_t iovoff,
+                  uint16_t vet, uint8_t *new_ehdr_buf,
+                  uint16_t *payload_offset, uint16_t *tci)
+{
+    struct vlan_header vlan_hdr;
+    struct eth_header *new_ehdr = (struct eth_header *) new_ehdr_buf;
+
+    size_t copied = iov_to_buf(iov, iovcnt, iovoff,
+                               new_ehdr, sizeof(*new_ehdr));
+
+    if (copied < sizeof(*new_ehdr)) {
+        return false;
+    }
+
+    if (be16_to_cpu(new_ehdr->h_proto) == vet) {
+        copied = iov_to_buf(iov, iovcnt, iovoff + sizeof(*new_ehdr),
+                            &vlan_hdr, sizeof(vlan_hdr));
+
+        if (copied < sizeof(vlan_hdr)) {
+            return false;
+        }
+
+        new_ehdr->h_proto = vlan_hdr.h_proto;
+
+        *tci = be16_to_cpu(vlan_hdr.h_tci);
+        *payload_offset = iovoff + sizeof(*new_ehdr) + sizeof(vlan_hdr);
+        return true;
+    }
+
+    return false;
+}
+
 void
 eth_setup_ip4_fragmentation(const void *l2hdr, size_t l2hdr_len,
                             void *l3hdr, size_t l3hdr_len,
                             size_t l3payload_len,
                             size_t frag_offset, bool more_frags)
 {
-    if (eth_get_l3_proto(l2hdr, l2hdr_len) == ETH_P_IP) {
+    const struct iovec l2vec = {
+        .iov_base = (void *) l2hdr,
+        .iov_len = l2hdr_len
+    };
+
+    if (eth_get_l3_proto(&l2vec, 1, l2hdr_len) == ETH_P_IP) {
         uint16_t orig_flags;
         struct ip_header *iphdr = (struct ip_header *) l3hdr;
         uint16_t frag_off_units = frag_offset / IP_FRAG_UNIT_SIZE;
@@ -157,7 +348,9 @@
 }
 
 uint32_t
-eth_calc_pseudo_hdr_csum(struct ip_header *iphdr, uint16_t csl)
+eth_calc_ip4_pseudo_hdr_csum(struct ip_header *iphdr,
+                             uint16_t csl,
+                             uint32_t *cso)
 {
     struct ip_pseudo_header ipph;
     ipph.ip_src = iphdr->ip_src;
@@ -165,7 +358,26 @@
     ipph.ip_payload = cpu_to_be16(csl);
     ipph.ip_proto = iphdr->ip_p;
     ipph.zeros = 0;
-    return net_checksum_add(sizeof(ipph), (uint8_t *) &ipph);
+    *cso = sizeof(ipph);
+    return net_checksum_add(*cso, (uint8_t *) &ipph);
+}
+
+uint32_t
+eth_calc_ip6_pseudo_hdr_csum(struct ip6_header *iphdr,
+                             uint16_t csl,
+                             uint8_t l4_proto,
+                             uint32_t *cso)
+{
+    struct ip6_pseudo_header ipph;
+    ipph.ip6_src = iphdr->ip6_src;
+    ipph.ip6_dst = iphdr->ip6_dst;
+    ipph.len = cpu_to_be16(csl);
+    ipph.zero[0] = 0;
+    ipph.zero[1] = 0;
+    ipph.zero[2] = 0;
+    ipph.next_hdr = l4_proto;
+    *cso = sizeof(ipph);
+    return net_checksum_add(*cso, (uint8_t *)&ipph);
 }
 
 static bool
@@ -185,33 +397,152 @@
     }
 }
 
-bool eth_parse_ipv6_hdr(struct iovec *pkt, int pkt_frags,
-                        size_t ip6hdr_off, uint8_t *l4proto,
-                        size_t *full_hdr_len)
+static bool
+_eth_get_rss_ex_dst_addr(const struct iovec *pkt, int pkt_frags,
+                        size_t rthdr_offset,
+                        struct ip6_ext_hdr *ext_hdr,
+                        struct in6_address *dst_addr)
 {
-    struct ip6_header ip6_hdr;
+    struct ip6_ext_hdr_routing *rthdr = (struct ip6_ext_hdr_routing *) ext_hdr;
+
+    if ((rthdr->rtype == 2) &&
+        (rthdr->len == sizeof(struct in6_address) / 8) &&
+        (rthdr->segleft == 1)) {
+
+        size_t input_size = iov_size(pkt, pkt_frags);
+        size_t bytes_read;
+
+        if (input_size < rthdr_offset + sizeof(*ext_hdr)) {
+            return false;
+        }
+
+        bytes_read = iov_to_buf(pkt, pkt_frags,
+                                rthdr_offset + sizeof(*ext_hdr),
+                                dst_addr, sizeof(*dst_addr));
+
+        return bytes_read == sizeof(dst_addr);
+    }
+
+    return false;
+}
+
+static bool
+_eth_get_rss_ex_src_addr(const struct iovec *pkt, int pkt_frags,
+                        size_t dsthdr_offset,
+                        struct ip6_ext_hdr *ext_hdr,
+                        struct in6_address *src_addr)
+{
+    size_t bytes_left = (ext_hdr->ip6r_len + 1) * 8 - sizeof(*ext_hdr);
+    struct ip6_option_hdr opthdr;
+    size_t opt_offset = dsthdr_offset + sizeof(*ext_hdr);
+
+    while (bytes_left > sizeof(opthdr)) {
+        size_t input_size = iov_size(pkt, pkt_frags);
+        size_t bytes_read, optlen;
+
+        if (input_size < opt_offset) {
+            return false;
+        }
+
+        bytes_read = iov_to_buf(pkt, pkt_frags, opt_offset,
+                                &opthdr, sizeof(opthdr));
+
+        if (bytes_read != sizeof(opthdr)) {
+            return false;
+        }
+
+        optlen = (opthdr.type == IP6_OPT_PAD1) ? 1
+                                               : (opthdr.len + sizeof(opthdr));
+
+        if (optlen > bytes_left) {
+            return false;
+        }
+
+        if (opthdr.type == IP6_OPT_HOME) {
+            size_t input_size = iov_size(pkt, pkt_frags);
+
+            if (input_size < opt_offset + sizeof(opthdr)) {
+                return false;
+            }
+
+            bytes_read = iov_to_buf(pkt, pkt_frags,
+                                    opt_offset + sizeof(opthdr),
+                                    src_addr, sizeof(*src_addr));
+
+            return bytes_read == sizeof(src_addr);
+        }
+
+        opt_offset += optlen;
+        bytes_left -= optlen;
+    }
+
+    return false;
+}
+
+bool eth_parse_ipv6_hdr(const struct iovec *pkt, int pkt_frags,
+                        size_t ip6hdr_off, eth_ip6_hdr_info *info)
+{
     struct ip6_ext_hdr ext_hdr;
     size_t bytes_read;
+    uint8_t curr_ext_hdr_type;
+    size_t input_size = iov_size(pkt, pkt_frags);
 
-    bytes_read = iov_to_buf(pkt, pkt_frags, ip6hdr_off,
-                            &ip6_hdr, sizeof(ip6_hdr));
-    if (bytes_read < sizeof(ip6_hdr)) {
+    info->rss_ex_dst_valid = false;
+    info->rss_ex_src_valid = false;
+    info->fragment = false;
+
+    if (input_size < ip6hdr_off) {
         return false;
     }
 
-    *full_hdr_len = sizeof(struct ip6_header);
+    bytes_read = iov_to_buf(pkt, pkt_frags, ip6hdr_off,
+                            &info->ip6_hdr, sizeof(info->ip6_hdr));
+    if (bytes_read < sizeof(info->ip6_hdr)) {
+        return false;
+    }
 
-    if (!eth_is_ip6_extension_header_type(ip6_hdr.ip6_nxt)) {
-        *l4proto = ip6_hdr.ip6_nxt;
+    info->full_hdr_len = sizeof(struct ip6_header);
+
+    curr_ext_hdr_type = info->ip6_hdr.ip6_nxt;
+
+    if (!eth_is_ip6_extension_header_type(curr_ext_hdr_type)) {
+        info->l4proto = info->ip6_hdr.ip6_nxt;
+        info->has_ext_hdrs = false;
         return true;
     }
 
-    do {
-        bytes_read = iov_to_buf(pkt, pkt_frags, ip6hdr_off + *full_hdr_len,
-                                &ext_hdr, sizeof(ext_hdr));
-        *full_hdr_len += (ext_hdr.ip6r_len + 1) * IP6_EXT_GRANULARITY;
-    } while (eth_is_ip6_extension_header_type(ext_hdr.ip6r_nxt));
+    info->has_ext_hdrs = true;
 
-    *l4proto = ext_hdr.ip6r_nxt;
+    do {
+        if (input_size < ip6hdr_off + info->full_hdr_len) {
+            return false;
+        }
+
+        bytes_read = iov_to_buf(pkt, pkt_frags, ip6hdr_off + info->full_hdr_len,
+                                &ext_hdr, sizeof(ext_hdr));
+
+        if (bytes_read < sizeof(ext_hdr)) {
+            return false;
+        }
+
+        if (curr_ext_hdr_type == IP6_ROUTING) {
+            info->rss_ex_dst_valid =
+                _eth_get_rss_ex_dst_addr(pkt, pkt_frags,
+                                         ip6hdr_off + info->full_hdr_len,
+                                         &ext_hdr, &info->rss_ex_dst);
+        } else if (curr_ext_hdr_type == IP6_DESTINATON) {
+            info->rss_ex_src_valid =
+                _eth_get_rss_ex_src_addr(pkt, pkt_frags,
+                                         ip6hdr_off + info->full_hdr_len,
+                                         &ext_hdr, &info->rss_ex_src);
+        } else if (curr_ext_hdr_type == IP6_FRAGMENT) {
+            info->fragment = true;
+        }
+
+        info->full_hdr_len += (ext_hdr.ip6r_len + 1) * IP6_EXT_GRANULARITY;
+        curr_ext_hdr_type = ext_hdr.ip6r_nxt;
+    } while (eth_is_ip6_extension_header_type(curr_ext_hdr_type));
+
+    info->l4proto = ext_hdr.ip6r_nxt;
     return true;
 }
diff --git a/net/filter-buffer.c b/net/filter-buffer.c
new file mode 100644
index 0000000..cc6bd94
--- /dev/null
+++ b/net/filter-buffer.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "net/filter.h"
+#include "net/queue.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/timer.h"
+#include "qemu/iov.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi-visit.h"
+#include "qom/object.h"
+
+#define TYPE_FILTER_BUFFER "filter-buffer"
+
+#define FILTER_BUFFER(obj) \
+    OBJECT_CHECK(FilterBufferState, (obj), TYPE_FILTER_BUFFER)
+
+typedef struct FilterBufferState {
+    NetFilterState parent_obj;
+
+    NetQueue *incoming_queue;
+    uint32_t interval;
+    QEMUTimer release_timer;
+} FilterBufferState;
+
+static void filter_buffer_flush(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (!qemu_net_queue_flush(s->incoming_queue)) {
+        /* Unable to empty the queue, purge remaining packets */
+        qemu_net_queue_purge(s->incoming_queue, nf->netdev);
+    }
+}
+
+static void filter_buffer_release_timer(void *opaque)
+{
+    NetFilterState *nf = opaque;
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * Note: filter_buffer_flush() drops packets that can't be sent
+     * TODO: We should leave them queued.  But currently there's no way
+     * for the next filter or receiver to notify us that it can receive
+     * more packets.
+     */
+    filter_buffer_flush(nf);
+    /* Timer rearmed to fire again in s->interval microseconds. */
+    timer_mod(&s->release_timer,
+              qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + s->interval);
+}
+
+/* filter APIs */
+static ssize_t filter_buffer_receive_iov(NetFilterState *nf,
+                                         NetClientState *sender,
+                                         unsigned flags,
+                                         const struct iovec *iov,
+                                         int iovcnt,
+                                         NetPacketSent *sent_cb)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * We return size when buffer a packet, the sender will take it as
+     * a already sent packet, so sent_cb should not be called later.
+     *
+     * FIXME: Even if the guest can't receive packets for some reasons,
+     * the filter can still accept packets until its internal queue is full.
+     * For example:
+     *   For some reason, receiver could not receive more packets
+     * (.can_receive() returns zero). Without a filter, at most one packet
+     * will be queued in incoming queue and sender's poll will be disabled
+     * unit its sent_cb() was called. With a filter, it will keep receiving
+     * the packets without caring about the receiver. This is suboptimal.
+     * May need more thoughts (e.g keeping sent_cb).
+     */
+    qemu_net_queue_append_iov(s->incoming_queue, sender, flags,
+                              iov, iovcnt, NULL);
+    return iov_size(iov, iovcnt);
+}
+
+static void filter_buffer_cleanup(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (s->interval) {
+        timer_del(&s->release_timer);
+    }
+
+    /* flush packets */
+    if (s->incoming_queue) {
+        filter_buffer_flush(nf);
+        g_free(s->incoming_queue);
+    }
+}
+
+static void filter_buffer_setup_timer(NetFilterState *nf)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (s->interval) {
+        timer_init_us(&s->release_timer, QEMU_CLOCK_VIRTUAL,
+                      filter_buffer_release_timer, nf);
+        /* Timer armed to fire in s->interval microseconds. */
+        timer_mod(&s->release_timer,
+                  qemu_clock_get_us(QEMU_CLOCK_VIRTUAL) + s->interval);
+    }
+}
+
+static void filter_buffer_setup(NetFilterState *nf, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    /*
+     * We may want to accept zero interval when VM FT solutions like MC
+     * or COLO use this filter to release packets on demand.
+     */
+    if (!s->interval) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "interval",
+                   "a non-zero interval");
+        return;
+    }
+
+    s->incoming_queue = qemu_new_net_queue(qemu_netfilter_pass_to_next, nf);
+    filter_buffer_setup_timer(nf);
+}
+
+static void filter_buffer_status_changed(NetFilterState *nf, Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(nf);
+
+    if (!nf->on) {
+        if (s->interval) {
+            timer_del(&s->release_timer);
+        }
+        filter_buffer_flush(nf);
+    } else {
+        filter_buffer_setup_timer(nf);
+    }
+}
+
+static void filter_buffer_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_buffer_setup;
+    nfc->cleanup = filter_buffer_cleanup;
+    nfc->receive_iov = filter_buffer_receive_iov;
+    nfc->status_changed = filter_buffer_status_changed;
+}
+
+static void filter_buffer_get_interval(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(obj);
+    uint32_t value = s->interval;
+
+    visit_type_uint32(v, name, &value, errp);
+}
+
+static void filter_buffer_set_interval(Object *obj, Visitor *v,
+                                       const char *name, void *opaque,
+                                       Error **errp)
+{
+    FilterBufferState *s = FILTER_BUFFER(obj);
+    Error *local_err = NULL;
+    uint32_t value;
+
+    visit_type_uint32(v, name, &value, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    if (!value) {
+        error_setg(&local_err, "Property '%s.%s' requires a positive value",
+                   object_get_typename(obj), name);
+        goto out;
+    }
+    s->interval = value;
+
+out:
+    error_propagate(errp, local_err);
+}
+
+static void filter_buffer_init(Object *obj)
+{
+    object_property_add(obj, "interval", "int",
+                        filter_buffer_get_interval,
+                        filter_buffer_set_interval, NULL, NULL, NULL);
+}
+
+static const TypeInfo filter_buffer_info = {
+    .name = TYPE_FILTER_BUFFER,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_buffer_class_init,
+    .instance_init = filter_buffer_init,
+    .instance_size = sizeof(FilterBufferState),
+};
+
+static void register_types(void)
+{
+    type_register_static(&filter_buffer_info);
+}
+
+type_init(register_types);
diff --git a/net/filter-mirror.c b/net/filter-mirror.c
new file mode 100644
index 0000000..35df374
--- /dev/null
+++ b/net/filter-mirror.c
@@ -0,0 +1,393 @@
+/*
+ * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2016 FUJITSU LIMITED
+ * Copyright (c) 2016 Intel Corporation
+ *
+ * Author: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "net/filter.h"
+#include "net/net.h"
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qapi/qmp/qerror.h"
+#include "qapi-visit.h"
+#include "qom/object.h"
+#include "qemu/main-loop.h"
+#include "qemu/error-report.h"
+#include "trace.h"
+#include "sysemu/char.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+
+#define FILTER_MIRROR(obj) \
+    OBJECT_CHECK(MirrorState, (obj), TYPE_FILTER_MIRROR)
+
+#define FILTER_REDIRECTOR(obj) \
+    OBJECT_CHECK(MirrorState, (obj), TYPE_FILTER_REDIRECTOR)
+
+#define TYPE_FILTER_MIRROR "filter-mirror"
+#define TYPE_FILTER_REDIRECTOR "filter-redirector"
+#define REDIRECTOR_MAX_LEN NET_BUFSIZE
+
+typedef struct MirrorState {
+    NetFilterState parent_obj;
+    char *indev;
+    char *outdev;
+    CharDriverState *chr_in;
+    CharDriverState *chr_out;
+    SocketReadState rs;
+} MirrorState;
+
+static int filter_mirror_send(CharDriverState *chr_out,
+                              const struct iovec *iov,
+                              int iovcnt)
+{
+    int ret = 0;
+    ssize_t size = 0;
+    uint32_t len =  0;
+    char *buf;
+
+    size = iov_size(iov, iovcnt);
+    if (!size) {
+        return 0;
+    }
+
+    len = htonl(size);
+    ret = qemu_chr_fe_write_all(chr_out, (uint8_t *)&len, sizeof(len));
+    if (ret != sizeof(len)) {
+        goto err;
+    }
+
+    buf = g_malloc(size);
+    iov_to_buf(iov, iovcnt, 0, buf, size);
+    ret = qemu_chr_fe_write_all(chr_out, (uint8_t *)buf, size);
+    g_free(buf);
+    if (ret != size) {
+        goto err;
+    }
+
+    return 0;
+
+err:
+    return ret < 0 ? ret : -EIO;
+}
+
+static void
+redirector_to_filter(NetFilterState *nf, const uint8_t *buf, int len)
+{
+    struct iovec iov = {
+        .iov_base = (void *)buf,
+        .iov_len = len,
+    };
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL ||
+        nf->direction == NET_FILTER_DIRECTION_TX) {
+        qemu_netfilter_pass_to_next(nf->netdev, 0, &iov, 1, nf);
+    }
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL ||
+        nf->direction == NET_FILTER_DIRECTION_RX) {
+        qemu_netfilter_pass_to_next(nf->netdev->peer, 0, &iov, 1, nf);
+     }
+}
+
+static int redirector_chr_can_read(void *opaque)
+{
+    return REDIRECTOR_MAX_LEN;
+}
+
+static void redirector_chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    NetFilterState *nf = opaque;
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+    int ret;
+
+    ret = net_fill_rstate(&s->rs, buf, size);
+
+    if (ret == -1) {
+        qemu_chr_add_handlers(s->chr_in, NULL, NULL, NULL, NULL);
+    }
+}
+
+static void redirector_chr_event(void *opaque, int event)
+{
+    NetFilterState *nf = opaque;
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+
+    switch (event) {
+    case CHR_EVENT_CLOSED:
+        qemu_chr_add_handlers(s->chr_in, NULL, NULL, NULL, NULL);
+        break;
+    default:
+        break;
+    }
+}
+
+static ssize_t filter_mirror_receive_iov(NetFilterState *nf,
+                                         NetClientState *sender,
+                                         unsigned flags,
+                                         const struct iovec *iov,
+                                         int iovcnt,
+                                         NetPacketSent *sent_cb)
+{
+    MirrorState *s = FILTER_MIRROR(nf);
+    int ret;
+
+    ret = filter_mirror_send(s->chr_out, iov, iovcnt);
+    if (ret) {
+        error_report("filter_mirror_send failed(%s)", strerror(-ret));
+    }
+
+    /*
+     * we don't hope this error interrupt the normal
+     * path of net packet, so we always return zero.
+     */
+    return 0;
+}
+
+static ssize_t filter_redirector_receive_iov(NetFilterState *nf,
+                                             NetClientState *sender,
+                                             unsigned flags,
+                                             const struct iovec *iov,
+                                             int iovcnt,
+                                             NetPacketSent *sent_cb)
+{
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+    int ret;
+
+    if (s->chr_out) {
+        ret = filter_mirror_send(s->chr_out, iov, iovcnt);
+        if (ret) {
+            error_report("filter_mirror_send failed(%s)", strerror(-ret));
+        }
+        return iov_size(iov, iovcnt);
+    } else {
+        return 0;
+    }
+}
+
+static void filter_mirror_cleanup(NetFilterState *nf)
+{
+    MirrorState *s = FILTER_MIRROR(nf);
+
+    if (s->chr_out) {
+        qemu_chr_fe_release(s->chr_out);
+    }
+}
+
+static void filter_redirector_cleanup(NetFilterState *nf)
+{
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+
+    if (s->chr_in) {
+        qemu_chr_add_handlers(s->chr_in, NULL, NULL, NULL, NULL);
+        qemu_chr_fe_release(s->chr_in);
+    }
+    if (s->chr_out) {
+        qemu_chr_fe_release(s->chr_out);
+    }
+}
+
+static void filter_mirror_setup(NetFilterState *nf, Error **errp)
+{
+    MirrorState *s = FILTER_MIRROR(nf);
+
+    if (!s->outdev) {
+        error_setg(errp, "filter filter mirror needs 'outdev' "
+                   "property set");
+        return;
+    }
+
+    s->chr_out = qemu_chr_find(s->outdev);
+    if (s->chr_out == NULL) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", s->outdev);
+        return;
+    }
+
+    if (qemu_chr_fe_claim(s->chr_out) != 0) {
+        error_setg(errp, QERR_DEVICE_IN_USE, s->outdev);
+        return;
+    }
+}
+
+static void redirector_rs_finalize(SocketReadState *rs)
+{
+    MirrorState *s = container_of(rs, MirrorState, rs);
+    NetFilterState *nf = NETFILTER(s);
+
+    redirector_to_filter(nf, rs->buf, rs->packet_len);
+}
+
+static void filter_redirector_setup(NetFilterState *nf, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(nf);
+
+    if (!s->indev && !s->outdev) {
+        error_setg(errp, "filter redirector needs 'indev' or "
+                   "'outdev' at least one property set");
+        return;
+    } else if (s->indev && s->outdev) {
+        if (!strcmp(s->indev, s->outdev)) {
+            error_setg(errp, "'indev' and 'outdev' could not be same "
+                       "for filter redirector");
+            return;
+        }
+    }
+
+    net_socket_rs_init(&s->rs, redirector_rs_finalize);
+
+    if (s->indev) {
+        s->chr_in = qemu_chr_find(s->indev);
+        if (s->chr_in == NULL) {
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "IN Device '%s' not found", s->indev);
+            return;
+        }
+
+        qemu_chr_fe_claim_no_fail(s->chr_in);
+        qemu_chr_add_handlers(s->chr_in, redirector_chr_can_read,
+                              redirector_chr_read, redirector_chr_event, nf);
+    }
+
+    if (s->outdev) {
+        s->chr_out = qemu_chr_find(s->outdev);
+        if (s->chr_out == NULL) {
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "OUT Device '%s' not found", s->outdev);
+            return;
+        }
+        qemu_chr_fe_claim_no_fail(s->chr_out);
+    }
+}
+
+static void filter_mirror_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_mirror_setup;
+    nfc->cleanup = filter_mirror_cleanup;
+    nfc->receive_iov = filter_mirror_receive_iov;
+}
+
+static void filter_redirector_class_init(ObjectClass *oc, void *data)
+{
+    NetFilterClass *nfc = NETFILTER_CLASS(oc);
+
+    nfc->setup = filter_redirector_setup;
+    nfc->cleanup = filter_redirector_cleanup;
+    nfc->receive_iov = filter_redirector_receive_iov;
+}
+
+static char *filter_redirector_get_indev(Object *obj, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    return g_strdup(s->indev);
+}
+
+static void
+filter_redirector_set_indev(Object *obj, const char *value, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    g_free(s->indev);
+    s->indev = g_strdup(value);
+}
+
+static char *filter_mirror_get_outdev(Object *obj, Error **errp)
+{
+    MirrorState *s = FILTER_MIRROR(obj);
+
+    return g_strdup(s->outdev);
+}
+
+static void
+filter_mirror_set_outdev(Object *obj, const char *value, Error **errp)
+{
+    MirrorState *s = FILTER_MIRROR(obj);
+
+    g_free(s->outdev);
+    s->outdev = g_strdup(value);
+    if (!s->outdev) {
+        error_setg(errp, "filter filter mirror needs 'outdev' "
+                   "property set");
+        return;
+    }
+}
+
+static char *filter_redirector_get_outdev(Object *obj, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    return g_strdup(s->outdev);
+}
+
+static void
+filter_redirector_set_outdev(Object *obj, const char *value, Error **errp)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    g_free(s->outdev);
+    s->outdev = g_strdup(value);
+}
+
+static void filter_mirror_init(Object *obj)
+{
+    object_property_add_str(obj, "outdev", filter_mirror_get_outdev,
+                            filter_mirror_set_outdev, NULL);
+}
+
+static void filter_redirector_init(Object *obj)
+{
+    object_property_add_str(obj, "indev", filter_redirector_get_indev,
+                            filter_redirector_set_indev, NULL);
+    object_property_add_str(obj, "outdev", filter_redirector_get_outdev,
+                            filter_redirector_set_outdev, NULL);
+}
+
+static void filter_mirror_fini(Object *obj)
+{
+    MirrorState *s = FILTER_MIRROR(obj);
+
+    g_free(s->outdev);
+}
+
+static void filter_redirector_fini(Object *obj)
+{
+    MirrorState *s = FILTER_REDIRECTOR(obj);
+
+    g_free(s->indev);
+    g_free(s->outdev);
+}
+
+static const TypeInfo filter_redirector_info = {
+    .name = TYPE_FILTER_REDIRECTOR,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_redirector_class_init,
+    .instance_init = filter_redirector_init,
+    .instance_finalize = filter_redirector_fini,
+    .instance_size = sizeof(MirrorState),
+};
+
+static const TypeInfo filter_mirror_info = {
+    .name = TYPE_FILTER_MIRROR,
+    .parent = TYPE_NETFILTER,
+    .class_init = filter_mirror_class_init,
+    .instance_init = filter_mirror_init,
+    .instance_finalize = filter_mirror_fini,
+    .instance_size = sizeof(MirrorState),
+};
+
+static void register_types(void)
+{
+    type_register_static(&filter_mirror_info);
+    type_register_static(&filter_redirector_info);
+}
+
+type_init(register_types);
diff --git a/net/filter.c b/net/filter.c
new file mode 100644
index 0000000..888fe6d
--- /dev/null
+++ b/net/filter.c
@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
+
+#include "net/filter.h"
+#include "net/net.h"
+#include "net/vhost_net.h"
+#include "qom/object_interfaces.h"
+#include "qemu/iov.h"
+
+static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
+{
+    return !nf->on;
+}
+
+ssize_t qemu_netfilter_receive(NetFilterState *nf,
+                               NetFilterDirection direction,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb)
+{
+    if (qemu_can_skip_netfilter(nf)) {
+        return 0;
+    }
+    if (nf->direction == direction ||
+        nf->direction == NET_FILTER_DIRECTION_ALL) {
+        return NETFILTER_GET_CLASS(OBJECT(nf))->receive_iov(
+                                   nf, sender, flags, iov, iovcnt, sent_cb);
+    }
+
+    return 0;
+}
+
+static NetFilterState *netfilter_next(NetFilterState *nf,
+                                      NetFilterDirection dir)
+{
+    NetFilterState *next;
+
+    if (dir == NET_FILTER_DIRECTION_TX) {
+        /* forward walk through filters */
+        next = QTAILQ_NEXT(nf, next);
+    } else {
+        /* reverse order */
+        next = QTAILQ_PREV(nf, NetFilterHead, next);
+    }
+
+    return next;
+}
+
+ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
+                                    unsigned flags,
+                                    const struct iovec *iov,
+                                    int iovcnt,
+                                    void *opaque)
+{
+    int ret = 0;
+    int direction;
+    NetFilterState *nf = opaque;
+    NetFilterState *next = NULL;
+
+    if (!sender || !sender->peer) {
+        /* no receiver, or sender been deleted, no need to pass it further */
+        goto out;
+    }
+
+    if (nf->direction == NET_FILTER_DIRECTION_ALL) {
+        if (sender == nf->netdev) {
+            /* This packet is sent by netdev itself */
+            direction = NET_FILTER_DIRECTION_TX;
+        } else {
+            direction = NET_FILTER_DIRECTION_RX;
+        }
+    } else {
+        direction = nf->direction;
+    }
+
+    next = netfilter_next(nf, direction);
+    while (next) {
+        /*
+         * if qemu_netfilter_pass_to_next been called, means that
+         * the packet has been hold by filter and has already retured size
+         * to the sender, so sent_cb shouldn't be called later, just
+         * pass NULL to next.
+         */
+        ret = qemu_netfilter_receive(next, direction, sender, flags, iov,
+                                     iovcnt, NULL);
+        if (ret) {
+            return ret;
+        }
+        next = netfilter_next(next, direction);
+    }
+
+    /*
+     * We have gone through all filters, pass it to receiver.
+     * Do the valid check again incase sender or receiver been
+     * deleted while we go through filters.
+     */
+    if (sender && sender->peer) {
+        qemu_net_queue_send_iov(sender->peer->incoming_queue,
+                                sender, flags, iov, iovcnt, NULL);
+    }
+
+out:
+    /* no receiver, or sender been deleted */
+    return iov_size(iov, iovcnt);
+}
+
+static char *netfilter_get_netdev_id(Object *obj, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    return g_strdup(nf->netdev_id);
+}
+
+static void netfilter_set_netdev_id(Object *obj, const char *str, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    nf->netdev_id = g_strdup(str);
+}
+
+static int netfilter_get_direction(Object *obj, Error **errp G_GNUC_UNUSED)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    return nf->direction;
+}
+
+static void netfilter_set_direction(Object *obj, int direction, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    nf->direction = direction;
+}
+
+static char *netfilter_get_status(Object *obj, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    return nf->on ? g_strdup("on") : g_strdup("off");
+}
+
+static void netfilter_set_status(Object *obj, const char *str, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(obj);
+
+    if (strcmp(str, "on") && strcmp(str, "off")) {
+        error_setg(errp, "Invalid value for netfilter status, "
+                         "should be 'on' or 'off'");
+        return;
+    }
+    if (nf->on == !strcmp(str, "on")) {
+        return;
+    }
+    nf->on = !nf->on;
+    if (nf->netdev && nfc->status_changed) {
+        nfc->status_changed(nf, errp);
+    }
+}
+
+static void netfilter_init(Object *obj)
+{
+    NetFilterState *nf = NETFILTER(obj);
+
+    nf->on = true;
+
+    object_property_add_str(obj, "netdev",
+                            netfilter_get_netdev_id, netfilter_set_netdev_id,
+                            NULL);
+    object_property_add_enum(obj, "queue", "NetFilterDirection",
+                             NetFilterDirection_lookup,
+                             netfilter_get_direction, netfilter_set_direction,
+                             NULL);
+    object_property_add_str(obj, "status",
+                            netfilter_get_status, netfilter_set_status,
+                            NULL);
+}
+
+static void netfilter_complete(UserCreatable *uc, Error **errp)
+{
+    NetFilterState *nf = NETFILTER(uc);
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(uc);
+    int queues;
+    Error *local_err = NULL;
+
+    if (!nf->netdev_id) {
+        error_setg(errp, "Parameter 'netdev' is required");
+        return;
+    }
+
+    queues = qemu_find_net_clients_except(nf->netdev_id, ncs,
+                                          NET_CLIENT_DRIVER_NIC,
+                                          MAX_QUEUE_NUM);
+    if (queues < 1) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "netdev",
+                   "a network backend id");
+        return;
+    } else if (queues > 1) {
+        error_setg(errp, "multiqueue is not supported");
+        return;
+    }
+
+    if (get_vhost_net(ncs[0])) {
+        error_setg(errp, "Vhost is not supported");
+        return;
+    }
+
+    nf->netdev = ncs[0];
+
+    if (nfc->setup) {
+        nfc->setup(nf, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+    QTAILQ_INSERT_TAIL(&nf->netdev->filters, nf, next);
+}
+
+static void netfilter_finalize(Object *obj)
+{
+    NetFilterState *nf = NETFILTER(obj);
+    NetFilterClass *nfc = NETFILTER_GET_CLASS(obj);
+
+    if (nfc->cleanup) {
+        nfc->cleanup(nf);
+    }
+
+    if (nf->netdev && !QTAILQ_EMPTY(&nf->netdev->filters) &&
+        nf->next.tqe_prev) {
+        QTAILQ_REMOVE(&nf->netdev->filters, nf, next);
+    }
+    g_free(nf->netdev_id);
+}
+
+static void netfilter_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = netfilter_complete;
+}
+
+static const TypeInfo netfilter_info = {
+    .name = TYPE_NETFILTER,
+    .parent = TYPE_OBJECT,
+    .abstract = true,
+    .class_size = sizeof(NetFilterClass),
+    .class_init = netfilter_class_init,
+    .instance_size = sizeof(NetFilterState),
+    .instance_init = netfilter_init,
+    .instance_finalize = netfilter_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+static void register_types(void)
+{
+    type_register_static(&netfilter_info);
+}
+
+type_init(register_types);
diff --git a/net/hub.c b/net/hub.c
index 7e0f2d6..32d8cf5 100644
--- a/net/hub.c
+++ b/net/hub.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "monitor/monitor.h"
 #include "net/net.h"
 #include "clients.h"
@@ -130,7 +131,7 @@
 }
 
 static NetClientInfo net_hub_port_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_HUBPORT,
+    .type = NET_CLIENT_DRIVER_HUBPORT,
     .size = sizeof(NetHubPort),
     .can_receive = net_hub_port_can_receive,
     .receive = net_hub_port_receive,
@@ -245,9 +246,12 @@
     QLIST_FOREACH(hub, &hubs, next) {
         monitor_printf(mon, "hub %d\n", hub->id);
         QLIST_FOREACH(port, &hub->ports, next) {
+            monitor_printf(mon, " \\ %s", port->nc.name);
             if (port->nc.peer) {
-                monitor_printf(mon, " \\ ");
+                monitor_printf(mon, ": ");
                 print_net_client(mon, port->nc.peer);
+            } else {
+                monitor_printf(mon, "\n");
             }
         }
     }
@@ -262,10 +266,10 @@
 {
     NetHubPort *port;
 
-    if (nc->info->type == NET_CLIENT_OPTIONS_KIND_HUBPORT) {
+    if (nc->info->type == NET_CLIENT_DRIVER_HUBPORT) {
         port = DO_UPCAST(NetHubPort, nc, nc);
     } else if (nc->peer != NULL && nc->peer->info->type ==
-            NET_CLIENT_OPTIONS_KIND_HUBPORT) {
+            NET_CLIENT_DRIVER_HUBPORT) {
         port = DO_UPCAST(NetHubPort, nc, nc->peer);
     } else {
         return -ENOENT;
@@ -277,18 +281,14 @@
     return 0;
 }
 
-int net_init_hubport(const NetClientOptions *opts, const char *name,
-                     NetClientState *peer)
+int net_init_hubport(const Netdev *netdev, const char *name,
+                     NetClientState *peer, Error **errp)
 {
     const NetdevHubPortOptions *hubport;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_HUBPORT);
-    hubport = opts->hubport;
-
-    /* Treat hub port like a backend, NIC must be the one to peer */
-    if (peer) {
-        return -EINVAL;
-    }
+    assert(netdev->type == NET_CLIENT_DRIVER_HUBPORT);
+    assert(!peer);
+    hubport = &netdev->u.hubport;
 
     net_hub_add_port(hubport->hubid, name);
     return 0;
@@ -315,14 +315,14 @@
             }
 
             switch (peer->info->type) {
-            case NET_CLIENT_OPTIONS_KIND_NIC:
+            case NET_CLIENT_DRIVER_NIC:
                 has_nic = 1;
                 break;
-            case NET_CLIENT_OPTIONS_KIND_USER:
-            case NET_CLIENT_OPTIONS_KIND_TAP:
-            case NET_CLIENT_OPTIONS_KIND_SOCKET:
-            case NET_CLIENT_OPTIONS_KIND_VDE:
-            case NET_CLIENT_OPTIONS_KIND_VHOST_USER:
+            case NET_CLIENT_DRIVER_USER:
+            case NET_CLIENT_DRIVER_TAP:
+            case NET_CLIENT_DRIVER_SOCKET:
+            case NET_CLIENT_DRIVER_VDE:
+            case NET_CLIENT_DRIVER_VHOST_USER:
                 has_host_dev = 1;
                 break;
             default:
diff --git a/net/l2tpv3.c b/net/l2tpv3.c
index 3b805a7..6745b78 100644
--- a/net/l2tpv3.c
+++ b/net/l2tpv3.c
@@ -23,12 +23,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include <linux/ip.h>
 #include <netdb.h>
-#include "config-host.h"
 #include "net/net.h"
 #include "clients.h"
-#include "monitor/monitor.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
@@ -133,17 +132,15 @@
 
 } NetL2TPV3State;
 
-static int l2tpv3_can_send(void *opaque);
 static void net_l2tpv3_send(void *opaque);
 static void l2tpv3_writable(void *opaque);
 
 static void l2tpv3_update_fd_handler(NetL2TPV3State *s)
 {
-    qemu_set_fd_handler2(s->fd,
-                         s->read_poll ? l2tpv3_can_send : NULL,
-                         s->read_poll ? net_l2tpv3_send     : NULL,
-                         s->write_poll ? l2tpv3_writable : NULL,
-                         s);
+    qemu_set_fd_handler(s->fd,
+                        s->read_poll ? net_l2tpv3_send : NULL,
+                        s->write_poll ? l2tpv3_writable : NULL,
+                        s);
 }
 
 static void l2tpv3_read_poll(NetL2TPV3State *s, bool enable)
@@ -169,13 +166,6 @@
     qemu_flush_queued_packets(&s->nc);
 }
 
-static int l2tpv3_can_send(void *opaque)
-{
-    NetL2TPV3State *s = opaque;
-
-    return qemu_can_send_packet(&s->nc);
-}
-
 static void l2tpv3_send_completed(NetClientState *nc, ssize_t len)
 {
     NetL2TPV3State *s = DO_UPCAST(NetL2TPV3State, nc, nc);
@@ -335,7 +325,7 @@
         if (s->cookie_is_64) {
             cookie = ldq_be_p(buf + s->cookie_offset);
         } else {
-            cookie = ldl_be_p(buf + s->cookie_offset);
+            cookie = ldl_be_p(buf + s->cookie_offset) & 0xffffffffULL;
         }
         if (cookie != s->rx_cookie) {
             if (!s->header_mismatch) {
@@ -489,12 +479,12 @@
     struct iovec *iov;
     struct mmsghdr *msgvec, *result;
 
-    msgvec = g_malloc(sizeof(struct mmsghdr) * count);
+    msgvec = g_new(struct mmsghdr, count);
     result = msgvec;
     for (i = 0; i < count ; i++) {
         msgvec->msg_hdr.msg_name = NULL;
         msgvec->msg_hdr.msg_namelen = 0;
-        iov =  g_malloc(sizeof(struct iovec) * IOVSIZE);
+        iov =  g_new(struct iovec, IOVSIZE);
         msgvec->msg_hdr.msg_iov = iov;
         iov->iov_base = g_malloc(s->header_size);
         iov->iov_len = s->header_size;
@@ -526,7 +516,7 @@
 }
 
 static NetClientInfo net_l2tpv3_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_L2TPV3,
+    .type = NET_CLIENT_DRIVER_L2TPV3,
     .size = sizeof(NetL2TPV3State),
     .receive = net_l2tpv3_receive_dgram,
     .receive_iov = net_l2tpv3_receive_dgram_iov,
@@ -534,12 +524,11 @@
     .cleanup = net_l2tpv3_cleanup,
 };
 
-int net_init_l2tpv3(const NetClientOptions *opts,
+int net_init_l2tpv3(const Netdev *netdev,
                     const char *name,
-                    NetClientState *peer)
+                    NetClientState *peer, Error **errp)
 {
-
-
+    /* FIXME error_setg(errp, ...) on failure */
     const NetdevL2TPv3Options *l2tpv3;
     NetL2TPV3State *s;
     NetClientState *nc;
@@ -556,8 +545,8 @@
     s->queue_tail = 0;
     s->header_mismatch = false;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_L2TPV3);
-    l2tpv3 = opts->l2tpv3;
+    assert(netdev->type == NET_CLIENT_DRIVER_L2TPV3);
+    l2tpv3 = &netdev->u.l2tpv3;
 
     if (l2tpv3->has_ipv6 && l2tpv3->ipv6) {
         s->ipv6 = l2tpv3->ipv6;
@@ -695,8 +684,7 @@
         goto outerr;
     }
 
-    s->dgram_dst = g_malloc(sizeof(struct sockaddr_storage));
-    memset(s->dgram_dst, '\0' , sizeof(struct sockaddr_storage));
+    s->dgram_dst = g_new0(struct sockaddr_storage, 1);
     memcpy(s->dgram_dst, result->ai_addr, result->ai_addrlen);
     s->dst_size = result->ai_addrlen;
 
@@ -730,7 +718,7 @@
     }
 
     s->msgvec = build_l2tpv3_vector(s, MAX_L2TPV3_MSGCNT);
-    s->vec = g_malloc(sizeof(struct iovec) * MAX_L2TPV3_IOVCNT);
+    s->vec = g_new(struct iovec, MAX_L2TPV3_IOVCNT);
     s->header_buf = g_malloc(s->header_size);
 
     qemu_set_nonblock(fd);
diff --git a/net/net.c b/net/net.c
index 7acc162..d51cb29 100644
--- a/net/net.c
+++ b/net/net.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include "net/net.h"
 #include "clients.h"
@@ -32,7 +32,11 @@
 
 #include "monitor/monitor.h"
 #include "qemu-common.h"
+#include "qemu/help_option.h"
+#include "qapi/qmp/qerror.h"
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
+#include "qemu/cutils.h"
 #include "qemu/config-file.h"
 #include "qmp-commands.h"
 #include "hw/qdev.h"
@@ -40,8 +44,9 @@
 #include "qemu/main-loop.h"
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/dealloc-visitor.h"
 #include "sysemu/sysemu.h"
+#include "net/filter.h"
+#include "qapi/string-output-visitor.h"
 
 /* Net bridge is currently not supported for W32. */
 #if !defined(_WIN32)
@@ -58,6 +63,9 @@
 #ifdef CONFIG_NET_BRIDGE
     "bridge",
 #endif
+#ifdef CONFIG_NETMAP
+    "netmap",
+#endif
 #ifdef CONFIG_SLIRP
     "user",
 #endif
@@ -68,39 +76,9 @@
     NULL,
 };
 
-int default_net = 1;
-
 /***********************************************************/
 /* network device redirectors */
 
-#if defined(DEBUG_NET)
-static void hex_dump(FILE *f, const uint8_t *buf, int size)
-{
-    int len, i, j, c;
-
-    for(i=0;i<size;i+=16) {
-        len = size - i;
-        if (len > 16)
-            len = 16;
-        fprintf(f, "%08x ", i);
-        for(j=0;j<16;j++) {
-            if (j < len)
-                fprintf(f, " %02x", buf[i+j]);
-            else
-                fprintf(f, "   ");
-        }
-        fprintf(f, " ");
-        for(j=0;j<len;j++) {
-            c = buf[i+j];
-            if (c < ' ' || c > '~')
-                c = '.';
-            fprintf(f, "%c", c);
-        }
-        fprintf(f, "\n");
-    }
-}
-#endif
-
 static int get_str_sep(char *buf, int buf_size, const char **pp, int sep)
 {
     const char *p, *p1;
@@ -151,6 +129,13 @@
     return 0;
 }
 
+char *qemu_mac_strdup_printf(const uint8_t *macaddr)
+{
+    return g_strdup_printf("%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",
+                           macaddr[0], macaddr[1], macaddr[2],
+                           macaddr[3], macaddr[4], macaddr[5]);
+}
+
 void qemu_format_nic_info_str(NetClientState *nc, uint8_t macaddr[6])
 {
     snprintf(nc->info_str, sizeof(nc->info_str),
@@ -160,19 +145,68 @@
              macaddr[3], macaddr[4], macaddr[5]);
 }
 
+static int mac_table[256] = {0};
+
+static void qemu_macaddr_set_used(MACAddr *macaddr)
+{
+    int index;
+
+    for (index = 0x56; index < 0xFF; index++) {
+        if (macaddr->a[5] == index) {
+            mac_table[index]++;
+        }
+    }
+}
+
+static void qemu_macaddr_set_free(MACAddr *macaddr)
+{
+    int index;
+    static const MACAddr base = { .a = { 0x52, 0x54, 0x00, 0x12, 0x34, 0 } };
+
+    if (memcmp(macaddr->a, &base.a, (sizeof(base.a) - 1)) != 0) {
+        return;
+    }
+    for (index = 0x56; index < 0xFF; index++) {
+        if (macaddr->a[5] == index) {
+            mac_table[index]--;
+        }
+    }
+}
+
+static int qemu_macaddr_get_free(void)
+{
+    int index;
+
+    for (index = 0x56; index < 0xFF; index++) {
+        if (mac_table[index] == 0) {
+            return index;
+        }
+    }
+
+    return -1;
+}
+
 void qemu_macaddr_default_if_unset(MACAddr *macaddr)
 {
-    static int index = 0;
     static const MACAddr zero = { .a = { 0,0,0,0,0,0 } };
+    static const MACAddr base = { .a = { 0x52, 0x54, 0x00, 0x12, 0x34, 0 } };
 
-    if (memcmp(macaddr, &zero, sizeof(zero)) != 0)
-        return;
+    if (memcmp(macaddr, &zero, sizeof(zero)) != 0) {
+        if (memcmp(macaddr->a, &base.a, (sizeof(base.a) - 1)) != 0) {
+            return;
+        } else {
+            qemu_macaddr_set_used(macaddr);
+            return;
+        }
+    }
+
     macaddr->a[0] = 0x52;
     macaddr->a[1] = 0x54;
     macaddr->a[2] = 0x00;
     macaddr->a[3] = 0x12;
     macaddr->a[4] = 0x34;
-    macaddr->a[5] = 0x56 + index++;
+    macaddr->a[5] = qemu_macaddr_get_free();
+    qemu_macaddr_set_used(macaddr);
 }
 
 /**
@@ -224,8 +258,9 @@
     }
     QTAILQ_INSERT_TAIL(&net_clients, nc, next);
 
-    nc->incoming_queue = qemu_new_net_queue(nc);
+    nc->incoming_queue = qemu_new_net_queue(qemu_deliver_packet_iov, nc);
     nc->destructor = destructor;
+    QTAILQ_INIT(&nc->filters);
 }
 
 NetClientState *qemu_new_net_client(NetClientInfo *info,
@@ -254,7 +289,7 @@
     NICState *nic;
     int i, queues = MAX(1, conf->peers.queues);
 
-    assert(info->type == NET_CLIENT_OPTIONS_KIND_NIC);
+    assert(info->type == NET_CLIENT_DRIVER_NIC);
     assert(info->size >= sizeof(NICState));
 
     nic = g_malloc0(info->size + sizeof(NetClientState) * queues);
@@ -323,17 +358,24 @@
 {
     NetClientState *ncs[MAX_QUEUE_NUM];
     int queues, i;
+    NetFilterState *nf, *next;
+
+    assert(nc->info->type != NET_CLIENT_DRIVER_NIC);
 
     /* If the NetClientState belongs to a multiqueue backend, we will change all
      * other NetClientStates also.
      */
     queues = qemu_find_net_clients_except(nc->name, ncs,
-                                          NET_CLIENT_OPTIONS_KIND_NIC,
+                                          NET_CLIENT_DRIVER_NIC,
                                           MAX_QUEUE_NUM);
     assert(queues != 0);
 
+    QTAILQ_FOREACH_SAFE(nf, &nc->filters, next, next) {
+        object_unparent(OBJECT(nf));
+    }
+
     /* If there is a peer NIC, delete and cleanup client, but do not free. */
-    if (nc->peer && nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
+    if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
         NICState *nic = qemu_get_nic(nc->peer);
         if (nic->peer_deleted) {
             return;
@@ -355,8 +397,6 @@
         return;
     }
 
-    assert(nc->info->type != NET_CLIENT_OPTIONS_KIND_NIC);
-
     for (i = 0; i < queues; i++) {
         qemu_cleanup_net_client(ncs[i]);
         qemu_free_net_client(ncs[i]);
@@ -367,6 +407,8 @@
 {
     int i, queues = MAX(nic->conf->peers.queues, 1);
 
+    qemu_macaddr_set_free(&nic->conf->macaddr);
+
     /* If this is a peer NIC and peer has already been deleted, free it now. */
     if (nic->peer_deleted) {
         for (i = 0; i < queues; i++) {
@@ -389,7 +431,7 @@
     NetClientState *nc;
 
     QTAILQ_FOREACH(nc, &net_clients, next) {
-        if (nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
+        if (nc->info->type == NET_CLIENT_DRIVER_NIC) {
             if (nc->queue_index == 0) {
                 func(qemu_get_nic(nc), opaque);
             }
@@ -452,6 +494,32 @@
     nc->info->set_vnet_hdr_len(nc, len);
 }
 
+int qemu_set_vnet_le(NetClientState *nc, bool is_le)
+{
+#ifdef HOST_WORDS_BIGENDIAN
+    if (!nc || !nc->info->set_vnet_le) {
+        return -ENOSYS;
+    }
+
+    return nc->info->set_vnet_le(nc, is_le);
+#else
+    return 0;
+#endif
+}
+
+int qemu_set_vnet_be(NetClientState *nc, bool is_be)
+{
+#ifdef HOST_WORDS_BIGENDIAN
+    return 0;
+#else
+    if (!nc || !nc->info->set_vnet_be) {
+        return -ENOSYS;
+    }
+
+    return nc->info->set_vnet_be(nc, is_be);
+#endif
+}
+
 int qemu_can_send_packet(NetClientState *sender)
 {
     int vm_running = runstate_is_running();
@@ -473,36 +541,54 @@
     return 1;
 }
 
-ssize_t qemu_deliver_packet(NetClientState *sender,
-                            unsigned flags,
-                            const uint8_t *data,
-                            size_t size,
-                            void *opaque)
+static ssize_t filter_receive_iov(NetClientState *nc,
+                                  NetFilterDirection direction,
+                                  NetClientState *sender,
+                                  unsigned flags,
+                                  const struct iovec *iov,
+                                  int iovcnt,
+                                  NetPacketSent *sent_cb)
 {
-    NetClientState *nc = opaque;
-    ssize_t ret;
+    ssize_t ret = 0;
+    NetFilterState *nf = NULL;
 
-    if (nc->link_down) {
-        return size;
-    }
-
-    if (nc->receive_disabled) {
-        return 0;
-    }
-
-    if (flags & QEMU_NET_PACKET_FLAG_RAW && nc->info->receive_raw) {
-        ret = nc->info->receive_raw(nc, data, size);
+    if (direction == NET_FILTER_DIRECTION_TX) {
+        QTAILQ_FOREACH(nf, &nc->filters, next) {
+            ret = qemu_netfilter_receive(nf, direction, sender, flags, iov,
+                                         iovcnt, sent_cb);
+            if (ret) {
+                return ret;
+            }
+        }
     } else {
-        ret = nc->info->receive(nc, data, size);
-    }
-
-    if (ret == 0) {
-        nc->receive_disabled = 1;
+        QTAILQ_FOREACH_REVERSE(nf, &nc->filters, NetFilterHead, next) {
+            ret = qemu_netfilter_receive(nf, direction, sender, flags, iov,
+                                         iovcnt, sent_cb);
+            if (ret) {
+                return ret;
+            }
+        }
     }
 
     return ret;
 }
 
+static ssize_t filter_receive(NetClientState *nc,
+                              NetFilterDirection direction,
+                              NetClientState *sender,
+                              unsigned flags,
+                              const uint8_t *data,
+                              size_t size,
+                              NetPacketSent *sent_cb)
+{
+    struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = size
+    };
+
+    return filter_receive_iov(nc, direction, sender, flags, &iov, 1, sent_cb);
+}
+
 void qemu_purge_queued_packets(NetClientState *nc)
 {
     if (!nc->peer) {
@@ -517,7 +603,7 @@
 {
     nc->receive_disabled = 0;
 
-    if (nc->peer && nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_HUBPORT) {
+    if (nc->peer && nc->peer->info->type == NET_CLIENT_DRIVER_HUBPORT) {
         if (net_hub_flush(nc->peer)) {
             qemu_notify_event();
         }
@@ -544,16 +630,30 @@
                                                  NetPacketSent *sent_cb)
 {
     NetQueue *queue;
+    int ret;
 
 #ifdef DEBUG_NET
     printf("qemu_send_packet_async:\n");
-    hex_dump(stdout, buf, size);
+    qemu_hexdump((const char *)buf, stdout, "net", size);
 #endif
 
     if (sender->link_down || !sender->peer) {
         return size;
     }
 
+    /* Let filters handle the packet first */
+    ret = filter_receive(sender, NET_FILTER_DIRECTION_TX,
+                         sender, flags, buf, size, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
+    ret = filter_receive(sender->peer, NET_FILTER_DIRECTION_RX,
+                         sender, flags, buf, size, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
     queue = sender->peer->incoming_queue;
 
     return qemu_net_queue_send(queue, sender, flags, buf, size, sent_cb);
@@ -579,14 +679,30 @@
 }
 
 static ssize_t nc_sendv_compat(NetClientState *nc, const struct iovec *iov,
-                               int iovcnt)
+                               int iovcnt, unsigned flags)
 {
-    uint8_t buffer[NET_BUFSIZE];
+    uint8_t *buf = NULL;
+    uint8_t *buffer;
     size_t offset;
+    ssize_t ret;
 
-    offset = iov_to_buf(iov, iovcnt, 0, buffer, sizeof(buffer));
+    if (iovcnt == 1) {
+        buffer = iov[0].iov_base;
+        offset = iov[0].iov_len;
+    } else {
+        buf = g_new(uint8_t, NET_BUFSIZE);
+        buffer = buf;
+        offset = iov_to_buf(iov, iovcnt, 0, buf, NET_BUFSIZE);
+    }
 
-    return nc->info->receive(nc, buffer, offset);
+    if (flags & QEMU_NET_PACKET_FLAG_RAW && nc->info->receive_raw) {
+        ret = nc->info->receive_raw(nc, buffer, offset);
+    } else {
+        ret = nc->info->receive(nc, buffer, offset);
+    }
+
+    g_free(buf);
+    return ret;
 }
 
 ssize_t qemu_deliver_packet_iov(NetClientState *sender,
@@ -606,10 +722,10 @@
         return 0;
     }
 
-    if (nc->info->receive_iov) {
+    if (nc->info->receive_iov && !(flags & QEMU_NET_PACKET_FLAG_RAW)) {
         ret = nc->info->receive_iov(nc, iov, iovcnt);
     } else {
-        ret = nc_sendv_compat(nc, iov, iovcnt);
+        ret = nc_sendv_compat(nc, iov, iovcnt, flags);
     }
 
     if (ret == 0) {
@@ -624,11 +740,25 @@
                                 NetPacketSent *sent_cb)
 {
     NetQueue *queue;
+    int ret;
 
     if (sender->link_down || !sender->peer) {
         return iov_size(iov, iovcnt);
     }
 
+    /* Let filters handle the packet first */
+    ret = filter_receive_iov(sender, NET_FILTER_DIRECTION_TX, sender,
+                             QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
+    ret = filter_receive_iov(sender->peer, NET_FILTER_DIRECTION_RX, sender,
+                             QEMU_NET_PACKET_FLAG_NONE, iov, iovcnt, sent_cb);
+    if (ret) {
+        return ret;
+    }
+
     queue = sender->peer->incoming_queue;
 
     return qemu_net_queue_send_iov(queue, sender,
@@ -647,7 +777,7 @@
     NetClientState *nc;
 
     QTAILQ_FOREACH(nc, &net_clients, next) {
-        if (nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC)
+        if (nc->info->type == NET_CLIENT_DRIVER_NIC)
             continue;
         if (!strcmp(nc->name, id)) {
             return nc;
@@ -658,7 +788,7 @@
 }
 
 int qemu_find_net_clients_except(const char *id, NetClientState **ncs,
-                                 NetClientOptionsKind type, int max)
+                                 NetClientDriver type, int max)
 {
     NetClientState *nc;
     int ret = 0;
@@ -732,19 +862,19 @@
     return -1;
 }
 
-static int net_init_nic(const NetClientOptions *opts, const char *name,
-                        NetClientState *peer)
+static int net_init_nic(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp)
 {
     int idx;
     NICInfo *nd;
     const NetLegacyNicOptions *nic;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_NIC);
-    nic = opts->nic;
+    assert(netdev->type == NET_CLIENT_DRIVER_NIC);
+    nic = &netdev->u.nic;
 
     idx = nic_get_free_idx();
     if (idx == -1 || nb_nics >= MAX_NICS) {
-        error_report("Too Many NICs");
+        error_setg(errp, "too many NICs");
         return -1;
     }
 
@@ -755,7 +885,7 @@
     if (nic->has_netdev) {
         nd->netdev = qemu_find_netdev(nic->netdev);
         if (!nd->netdev) {
-            error_report("netdev '%s' not found", nic->netdev);
+            error_setg(errp, "netdev '%s' not found", nic->netdev);
             return -1;
         }
     } else {
@@ -772,19 +902,20 @@
 
     if (nic->has_macaddr &&
         net_parse_macaddr(nd->macaddr.a, nic->macaddr) < 0) {
-        error_report("invalid syntax for ethernet address");
+        error_setg(errp, "invalid syntax for ethernet address");
         return -1;
     }
     if (nic->has_macaddr &&
         is_multicast_ether_addr(nd->macaddr.a)) {
-        error_report("NIC cannot have multicast MAC address (odd 1st byte)");
+        error_setg(errp,
+                   "NIC cannot have multicast MAC address (odd 1st byte)");
         return -1;
     }
     qemu_macaddr_default_if_unset(&nd->macaddr);
 
     if (nic->has_vectors) {
         if (nic->vectors > 0x7ffffff) {
-            error_report("invalid # of vectors: %"PRIu32, nic->vectors);
+            error_setg(errp, "invalid # of vectors: %"PRIu32, nic->vectors);
             return -1;
         }
         nd->nvectors = nic->vectors;
@@ -799,143 +930,192 @@
 }
 
 
-static int (* const net_client_init_fun[NET_CLIENT_OPTIONS_KIND_MAX])(
-    const NetClientOptions *opts,
+static int (* const net_client_init_fun[NET_CLIENT_DRIVER__MAX])(
+    const Netdev *netdev,
     const char *name,
-    NetClientState *peer) = {
-        [NET_CLIENT_OPTIONS_KIND_NIC]       = net_init_nic,
+    NetClientState *peer, Error **errp) = {
+        [NET_CLIENT_DRIVER_NIC]       = net_init_nic,
 #ifdef CONFIG_SLIRP
-        [NET_CLIENT_OPTIONS_KIND_USER]      = net_init_slirp,
+        [NET_CLIENT_DRIVER_USER]      = net_init_slirp,
 #endif
-        [NET_CLIENT_OPTIONS_KIND_TAP]       = net_init_tap,
-        [NET_CLIENT_OPTIONS_KIND_SOCKET]    = net_init_socket,
+        [NET_CLIENT_DRIVER_TAP]       = net_init_tap,
+        [NET_CLIENT_DRIVER_SOCKET]    = net_init_socket,
 #ifdef CONFIG_VDE
-        [NET_CLIENT_OPTIONS_KIND_VDE]       = net_init_vde,
+        [NET_CLIENT_DRIVER_VDE]       = net_init_vde,
 #endif
 #ifdef CONFIG_NETMAP
-        [NET_CLIENT_OPTIONS_KIND_NETMAP]    = net_init_netmap,
+        [NET_CLIENT_DRIVER_NETMAP]    = net_init_netmap,
 #endif
-        [NET_CLIENT_OPTIONS_KIND_DUMP]      = net_init_dump,
+        [NET_CLIENT_DRIVER_DUMP]      = net_init_dump,
 #ifdef CONFIG_NET_BRIDGE
-        [NET_CLIENT_OPTIONS_KIND_BRIDGE]    = net_init_bridge,
+        [NET_CLIENT_DRIVER_BRIDGE]    = net_init_bridge,
 #endif
-        [NET_CLIENT_OPTIONS_KIND_HUBPORT]   = net_init_hubport,
+        [NET_CLIENT_DRIVER_HUBPORT]   = net_init_hubport,
 #ifdef CONFIG_VHOST_NET_USED
-        [NET_CLIENT_OPTIONS_KIND_VHOST_USER] = net_init_vhost_user,
+        [NET_CLIENT_DRIVER_VHOST_USER] = net_init_vhost_user,
 #endif
 #ifdef CONFIG_L2TPV3
-        [NET_CLIENT_OPTIONS_KIND_L2TPV3]    = net_init_l2tpv3,
+        [NET_CLIENT_DRIVER_L2TPV3]    = net_init_l2tpv3,
 #endif
 };
 
 
-static int net_client_init1(const void *object, int is_netdev, Error **errp)
+static int net_client_init1(const void *object, bool is_netdev, Error **errp)
 {
-    union {
-        const Netdev    *netdev;
-        const NetLegacy *net;
-    } u;
-    const NetClientOptions *opts;
+    Netdev legacy = {0};
+    const Netdev *netdev;
     const char *name;
+    NetClientState *peer = NULL;
 
     if (is_netdev) {
-        u.netdev = object;
-        opts = u.netdev->opts;
-        name = u.netdev->id;
+        netdev = object;
+        name = netdev->id;
 
-        switch (opts->kind) {
-#ifdef CONFIG_SLIRP
-        case NET_CLIENT_OPTIONS_KIND_USER:
-#endif
-        case NET_CLIENT_OPTIONS_KIND_TAP:
-        case NET_CLIENT_OPTIONS_KIND_SOCKET:
-#ifdef CONFIG_VDE
-        case NET_CLIENT_OPTIONS_KIND_VDE:
-#endif
-#ifdef CONFIG_NETMAP
-        case NET_CLIENT_OPTIONS_KIND_NETMAP:
-#endif
-#ifdef CONFIG_NET_BRIDGE
-        case NET_CLIENT_OPTIONS_KIND_BRIDGE:
-#endif
-        case NET_CLIENT_OPTIONS_KIND_HUBPORT:
-#ifdef CONFIG_VHOST_NET_USED
-        case NET_CLIENT_OPTIONS_KIND_VHOST_USER:
-#endif
-#ifdef CONFIG_L2TPV3
-        case NET_CLIENT_OPTIONS_KIND_L2TPV3:
-#endif
-            break;
-
-        default:
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, "type",
-                      "a netdev backend type");
+        if (netdev->type == NET_CLIENT_DRIVER_DUMP ||
+            netdev->type == NET_CLIENT_DRIVER_NIC ||
+            !net_client_init_fun[netdev->type]) {
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
+                       "a netdev backend type");
             return -1;
         }
     } else {
-        u.net = object;
-        opts = u.net->opts;
+        const NetLegacy *net = object;
+        const NetLegacyOptions *opts = net->opts;
+        legacy.id = net->id;
+        netdev = &legacy;
         /* missing optional values have been initialized to "all bits zero" */
-        name = u.net->has_id ? u.net->id : u.net->name;
-    }
+        name = net->has_id ? net->id : net->name;
 
-    if (net_client_init_fun[opts->kind]) {
-        NetClientState *peer = NULL;
-
-        /* Do not add to a vlan if it's a -netdev or a nic with a netdev=
-         * parameter. */
-        if (!is_netdev &&
-            (opts->kind != NET_CLIENT_OPTIONS_KIND_NIC ||
-             !opts->nic->has_netdev)) {
-            peer = net_hub_add_port(u.net->has_vlan ? u.net->vlan : 0, NULL);
+        /* Map the old options to the new flat type */
+        switch (opts->type) {
+        case NET_LEGACY_OPTIONS_KIND_NONE:
+            return 0; /* nothing to do */
+        case NET_LEGACY_OPTIONS_KIND_NIC:
+            legacy.type = NET_CLIENT_DRIVER_NIC;
+            legacy.u.nic = *opts->u.nic.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_USER:
+            legacy.type = NET_CLIENT_DRIVER_USER;
+            legacy.u.user = *opts->u.user.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_TAP:
+            legacy.type = NET_CLIENT_DRIVER_TAP;
+            legacy.u.tap = *opts->u.tap.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_L2TPV3:
+            legacy.type = NET_CLIENT_DRIVER_L2TPV3;
+            legacy.u.l2tpv3 = *opts->u.l2tpv3.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_SOCKET:
+            legacy.type = NET_CLIENT_DRIVER_SOCKET;
+            legacy.u.socket = *opts->u.socket.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_VDE:
+            legacy.type = NET_CLIENT_DRIVER_VDE;
+            legacy.u.vde = *opts->u.vde.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_DUMP:
+            legacy.type = NET_CLIENT_DRIVER_DUMP;
+            legacy.u.dump = *opts->u.dump.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_BRIDGE:
+            legacy.type = NET_CLIENT_DRIVER_BRIDGE;
+            legacy.u.bridge = *opts->u.bridge.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_NETMAP:
+            legacy.type = NET_CLIENT_DRIVER_NETMAP;
+            legacy.u.netmap = *opts->u.netmap.data;
+            break;
+        case NET_LEGACY_OPTIONS_KIND_VHOST_USER:
+            legacy.type = NET_CLIENT_DRIVER_VHOST_USER;
+            legacy.u.vhost_user = *opts->u.vhost_user.data;
+            break;
+        default:
+            abort();
         }
 
-        if (net_client_init_fun[opts->kind](opts, name, peer) < 0) {
-            /* TODO push error reporting into init() methods */
-            error_set(errp, QERR_DEVICE_INIT_FAILED,
-                      NetClientOptionsKind_lookup[opts->kind]);
+        if (!net_client_init_fun[netdev->type]) {
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "type",
+                       "a net backend type (maybe it is not compiled "
+                       "into this binary)");
             return -1;
         }
+
+        /* Do not add to a vlan if it's a nic with a netdev= parameter. */
+        if (netdev->type != NET_CLIENT_DRIVER_NIC ||
+            !opts->u.nic.data->has_netdev) {
+            peer = net_hub_add_port(net->has_vlan ? net->vlan : 0, NULL);
+        }
+    }
+
+    if (net_client_init_fun[netdev->type](netdev, name, peer, errp) < 0) {
+        /* FIXME drop when all init functions store an Error */
+        if (errp && !*errp) {
+            error_setg(errp, QERR_DEVICE_INIT_FAILED,
+                       NetClientDriver_lookup[netdev->type]);
+        }
+        return -1;
     }
     return 0;
 }
 
 
-static void net_visit(Visitor *v, int is_netdev, void **object, Error **errp)
-{
-    if (is_netdev) {
-        visit_type_Netdev(v, (Netdev **)object, NULL, errp);
-    } else {
-        visit_type_NetLegacy(v, (NetLegacy **)object, NULL, errp);
-    }
-}
-
-
-int net_client_init(QemuOpts *opts, int is_netdev, Error **errp)
+int net_client_init(QemuOpts *opts, bool is_netdev, Error **errp)
 {
     void *object = NULL;
     Error *err = NULL;
     int ret = -1;
+    Visitor *v = opts_visitor_new(opts);
 
     {
-        OptsVisitor *ov = opts_visitor_new(opts);
+        /* Parse convenience option format ip6-net=fec0::0[/64] */
+        const char *ip6_net = qemu_opt_get(opts, "ipv6-net");
 
-        net_visit(opts_get_visitor(ov), is_netdev, &object, &err);
-        opts_visitor_cleanup(ov);
+        if (ip6_net) {
+            char buf[strlen(ip6_net) + 1];
+
+            if (get_str_sep(buf, sizeof(buf), &ip6_net, '/') < 0) {
+                /* Default 64bit prefix length.  */
+                qemu_opt_set(opts, "ipv6-prefix", ip6_net, &error_abort);
+                qemu_opt_set_number(opts, "ipv6-prefixlen", 64, &error_abort);
+            } else {
+                /* User-specified prefix length.  */
+                unsigned long len;
+                int err;
+
+                qemu_opt_set(opts, "ipv6-prefix", buf, &error_abort);
+                err = qemu_strtoul(ip6_net, NULL, 10, &len);
+
+                if (err) {
+                    error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                              "ipv6-prefix", "a number");
+                } else {
+                    qemu_opt_set_number(opts, "ipv6-prefixlen", len,
+                                        &error_abort);
+                }
+            }
+            qemu_opt_unset(opts, "ipv6-net");
+        }
+    }
+
+    if (is_netdev) {
+        visit_type_Netdev(v, NULL, (Netdev **)&object, &err);
+    } else {
+        visit_type_NetLegacy(v, NULL, (NetLegacy **)&object, &err);
     }
 
     if (!err) {
         ret = net_client_init1(object, is_netdev, &err);
     }
 
-    if (object) {
-        QapiDeallocVisitor *dv = qapi_dealloc_visitor_new();
-
-        net_visit(qapi_dealloc_get_visitor(dv), is_netdev, &object, NULL);
-        qapi_dealloc_visitor_cleanup(dv);
+    if (is_netdev) {
+        qapi_free_Netdev(object);
+    } else {
+        qapi_free_NetLegacy(object);
     }
 
     error_propagate(errp, err);
+    visit_free(v);
     return ret;
 }
 
@@ -953,7 +1133,7 @@
     return 0;
 }
 
-void net_host_device_add(Monitor *mon, const QDict *qdict)
+void hmp_host_net_add(Monitor *mon, const QDict *qdict)
 {
     const char *device = qdict_get_str(qdict, "device");
     const char *opts_str = qdict_get_try_str(qdict, "opts");
@@ -965,22 +1145,22 @@
         return;
     }
 
-    opts = qemu_opts_parse(qemu_find_opts("net"), opts_str ? opts_str : "", 0);
+    opts = qemu_opts_parse_noisily(qemu_find_opts("net"),
+                                   opts_str ? opts_str : "", false);
     if (!opts) {
         return;
     }
 
-    qemu_opt_set(opts, "type", device);
+    qemu_opt_set(opts, "type", device, &error_abort);
 
-    net_client_init(opts, 0, &local_err);
+    net_client_init(opts, false, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         monitor_printf(mon, "adding host network device %s failed\n", device);
     }
 }
 
-void net_host_device_remove(Monitor *mon, const QDict *qdict)
+void hmp_host_net_remove(Monitor *mon, const QDict *qdict)
 {
     NetClientState *nc;
     int vlan_id = qdict_get_int(qdict, "vlan_id");
@@ -992,19 +1172,21 @@
                      device, vlan_id);
         return;
     }
-    if (!net_host_check_device(nc->model)) {
+    if (nc->info->type == NET_CLIENT_DRIVER_NIC) {
         error_report("invalid host network device '%s'", device);
         return;
     }
+
+    qemu_del_net_client(nc->peer);
     qemu_del_net_client(nc);
 }
 
 void netdev_add(QemuOpts *opts, Error **errp)
 {
-    net_client_init(opts, 1, errp);
+    net_client_init(opts, true, errp);
 }
 
-int qmp_netdev_add(Monitor *mon, const QDict *qdict, QObject **ret)
+void qmp_netdev_add(QDict *qdict, QObject **ret, Error **errp)
 {
     Error *local_err = NULL;
     QemuOptsList *opts_list;
@@ -1012,26 +1194,22 @@
 
     opts_list = qemu_find_opts_err("netdev", &local_err);
     if (local_err) {
-        goto exit_err;
+        goto out;
     }
 
     opts = qemu_opts_from_qdict(opts_list, qdict, &local_err);
     if (local_err) {
-        goto exit_err;
+        goto out;
     }
 
     netdev_add(opts, &local_err);
     if (local_err) {
         qemu_opts_del(opts);
-        goto exit_err;
+        goto out;
     }
 
-    return 0;
-
-exit_err:
-    qerror_report_err(local_err);
-    error_free(local_err);
-    return -1;
+out:
+    error_propagate(errp, local_err);
 }
 
 void qmp_netdev_del(const char *id, Error **errp)
@@ -1041,7 +1219,8 @@
 
     nc = qemu_find_netdev(id);
     if (!nc) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, id);
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", id);
         return;
     }
 
@@ -1055,12 +1234,48 @@
     qemu_opts_del(opts);
 }
 
+static void netfilter_print_info(Monitor *mon, NetFilterState *nf)
+{
+    char *str;
+    ObjectProperty *prop;
+    ObjectPropertyIterator iter;
+    Visitor *v;
+
+    /* generate info str */
+    object_property_iter_init(&iter, OBJECT(nf));
+    while ((prop = object_property_iter_next(&iter))) {
+        if (!strcmp(prop->name, "type")) {
+            continue;
+        }
+        v = string_output_visitor_new(false, &str);
+        object_property_get(OBJECT(nf), v, prop->name, NULL);
+        visit_complete(v, &str);
+        visit_free(v);
+        monitor_printf(mon, ",%s=%s", prop->name, str);
+        g_free(str);
+    }
+    monitor_printf(mon, "\n");
+}
+
 void print_net_client(Monitor *mon, NetClientState *nc)
 {
+    NetFilterState *nf;
+
     monitor_printf(mon, "%s: index=%d,type=%s,%s\n", nc->name,
                    nc->queue_index,
-                   NetClientOptionsKind_lookup[nc->info->type],
+                   NetClientDriver_lookup[nc->info->type],
                    nc->info_str);
+    if (!QTAILQ_EMPTY(&nc->filters)) {
+        monitor_printf(mon, "filters:\n");
+    }
+    QTAILQ_FOREACH(nf, &nc->filters, next) {
+        char *path = object_get_canonical_path_component(OBJECT(nf));
+
+        monitor_printf(mon, "  - %s: type=%s", path,
+                       object_get_typename(OBJECT(nf)));
+        netfilter_print_info(mon, nf);
+        g_free(path);
+    }
 }
 
 RxFilterInfoList *qmp_query_rx_filter(bool has_name, const char *name,
@@ -1078,7 +1293,7 @@
         }
 
         /* only query rx-filter information of NIC */
-        if (nc->info->type != NET_CLIENT_OPTIONS_KIND_NIC) {
+        if (nc->info->type != NET_CLIENT_DRIVER_NIC) {
             if (has_name) {
                 error_setg(errp, "net client(%s) isn't a NIC", name);
                 return NULL;
@@ -1086,6 +1301,12 @@
             continue;
         }
 
+        /* only query information on queue 0 since the info is per nic,
+         * not per queue
+         */
+        if (nc->queue_index != 0)
+            continue;
+
         if (nc->info->query_rx_filter) {
             info = nc->info->query_rx_filter(nc);
             entry = g_malloc0(sizeof(*entry));
@@ -1115,10 +1336,10 @@
     return filter_list;
 }
 
-void do_info_network(Monitor *mon, const QDict *qdict)
+void hmp_info_network(Monitor *mon, const QDict *qdict)
 {
     NetClientState *nc, *peer;
-    NetClientOptionsKind type;
+    NetClientDriver type;
 
     net_hub_info(mon);
 
@@ -1131,10 +1352,10 @@
             continue;
         }
 
-        if (!peer || type == NET_CLIENT_OPTIONS_KIND_NIC) {
+        if (!peer || type == NET_CLIENT_DRIVER_NIC) {
             print_net_client(mon, nc);
         } /* else it's a netdev connected to a NIC, printed with the NIC */
-        if (peer && type == NET_CLIENT_OPTIONS_KIND_NIC) {
+        if (peer && type == NET_CLIENT_DRIVER_NIC) {
             monitor_printf(mon, " \\ ");
             print_net_client(mon, peer);
         }
@@ -1148,11 +1369,12 @@
     int queues, i;
 
     queues = qemu_find_net_clients_except(name, ncs,
-                                          NET_CLIENT_OPTIONS_KIND_MAX,
+                                          NET_CLIENT_DRIVER__MAX,
                                           MAX_QUEUE_NUM);
 
     if (queues == 0) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, name);
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", name);
         return;
     }
     nc = ncs[0];
@@ -1174,7 +1396,7 @@
          * multiple clients that can still communicate with each other in
          * disconnected mode. For now maintain this compatibility.
          */
-        if (nc->peer->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
+        if (nc->peer->info->type == NET_CLIENT_DRIVER_NIC) {
             for (i = 0; i < queues; i++) {
                 ncs[i]->peer->link_down = !up;
             }
@@ -1188,14 +1410,19 @@
 static void net_vm_change_state_handler(void *opaque, int running,
                                         RunState state)
 {
-    /* Complete all queued packets, to guarantee we don't modify
-     * state later when VM is not running.
-     */
-    if (!running) {
-        NetClientState *nc;
-        NetClientState *tmp;
+    NetClientState *nc;
+    NetClientState *tmp;
 
-        QTAILQ_FOREACH_SAFE(nc, &net_clients, next, tmp) {
+    QTAILQ_FOREACH_SAFE(nc, &net_clients, next, tmp) {
+        if (running) {
+            /* Flush queued packets and wake up backends. */
+            if (nc->peer && qemu_can_send_packet(nc)) {
+                qemu_flush_queued_packets(nc->peer);
+            }
+        } else {
+            /* Complete all queued packets, to guarantee we don't modify
+             * state later when VM is not running.
+             */
             qemu_flush_or_purge_queued_packets(nc, true);
         }
     }
@@ -1210,7 +1437,7 @@
      */
     while (!QTAILQ_EMPTY(&net_clients)) {
         nc = QTAILQ_FIRST(&net_clients);
-        if (nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC) {
+        if (nc->info->type == NET_CLIENT_DRIVER_NIC) {
             qemu_del_nic(qemu_get_nic(nc));
         } else {
             qemu_del_net_client(nc);
@@ -1225,24 +1452,12 @@
     NetClientState *nc;
     int i;
 
-    /* Don't warn about the default network setup that you get if
-     * no command line -net or -netdev options are specified. There
-     * are two cases that we would otherwise complain about:
-     * (1) board doesn't support a NIC but the implicit "-net nic"
-     * requested one
-     * (2) CONFIG_SLIRP not set, in which case the implicit "-net nic"
-     * sets up a nic that isn't connected to anything.
-     */
-    if (default_net) {
-        return;
-    }
-
     net_hub_check_clients();
 
     QTAILQ_FOREACH(nc, &net_clients, next) {
         if (!nc->peer) {
             fprintf(stderr, "Warning: %s %s has no peer\n",
-                    nc->info->type == NET_CLIENT_OPTIONS_KIND_NIC ?
+                    nc->info->type == NET_CLIENT_DRIVER_NIC ?
                     "nic" : "netdev", nc->name);
         }
     }
@@ -1262,29 +1477,27 @@
     }
 }
 
-static int net_init_client(QemuOpts *opts, void *dummy)
+static int net_init_client(void *dummy, QemuOpts *opts, Error **errp)
 {
     Error *local_err = NULL;
 
-    net_client_init(opts, 0, &local_err);
+    net_client_init(opts, false, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return -1;
     }
 
     return 0;
 }
 
-static int net_init_netdev(QemuOpts *opts, void *dummy)
+static int net_init_netdev(void *dummy, QemuOpts *opts, Error **errp)
 {
     Error *local_err = NULL;
     int ret;
 
-    ret = net_client_init(opts, 1, &local_err);
+    ret = net_client_init(opts, true, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return -1;
     }
 
@@ -1295,23 +1508,17 @@
 {
     QemuOptsList *net = qemu_find_opts("net");
 
-    if (default_net) {
-        /* if no clients, we use a default config */
-        qemu_opts_set(net, NULL, "type", "nic");
-#ifdef CONFIG_SLIRP
-        qemu_opts_set(net, NULL, "type", "user");
-#endif
-    }
-
     net_change_state_entry =
         qemu_add_vm_change_state_handler(net_vm_change_state_handler, NULL);
 
     QTAILQ_INIT(&net_clients);
 
-    if (qemu_opts_foreach(qemu_find_opts("netdev"), net_init_netdev, NULL, 1) == -1)
+    if (qemu_opts_foreach(qemu_find_opts("netdev"),
+                          net_init_netdev, NULL, NULL)) {
         return -1;
+    }
 
-    if (qemu_opts_foreach(net, net_init_client, NULL, 1) == -1) {
+    if (qemu_opts_foreach(net, net_init_client, NULL, NULL)) {
         return -1;
     }
 
@@ -1327,11 +1534,10 @@
     }
 #endif
 
-    if (!qemu_opts_parse(opts_list, optarg, 1)) {
+    if (!qemu_opts_parse_noisily(opts_list, optarg, true)) {
         return -1;
     }
 
-    default_net = 0;
     return 0;
 }
 
@@ -1383,3 +1589,73 @@
         { /* end of list */ }
     },
 };
+
+void net_socket_rs_init(SocketReadState *rs,
+                        SocketReadStateFinalize *finalize)
+{
+    rs->state = 0;
+    rs->index = 0;
+    rs->packet_len = 0;
+    memset(rs->buf, 0, sizeof(rs->buf));
+    rs->finalize = finalize;
+}
+
+/*
+ * Returns
+ * 0: success
+ * -1: error occurs
+ */
+int net_fill_rstate(SocketReadState *rs, const uint8_t *buf, int size)
+{
+    unsigned int l;
+
+    while (size > 0) {
+        /* reassemble a packet from the network */
+        switch (rs->state) { /* 0 = getting length, 1 = getting data */
+        case 0:
+            l = 4 - rs->index;
+            if (l > size) {
+                l = size;
+            }
+            memcpy(rs->buf + rs->index, buf, l);
+            buf += l;
+            size -= l;
+            rs->index += l;
+            if (rs->index == 4) {
+                /* got length */
+                rs->packet_len = ntohl(*(uint32_t *)rs->buf);
+                rs->index = 0;
+                rs->state = 1;
+            }
+            break;
+        case 1:
+            l = rs->packet_len - rs->index;
+            if (l > size) {
+                l = size;
+            }
+            if (rs->index + l <= sizeof(rs->buf)) {
+                memcpy(rs->buf + rs->index, buf, l);
+            } else {
+                fprintf(stderr, "serious error: oversized packet received,"
+                    "connection terminated.\n");
+                rs->index = rs->state = 0;
+                return -1;
+            }
+
+            rs->index += l;
+            buf += l;
+            size -= l;
+            if (rs->index >= rs->packet_len) {
+                rs->index = 0;
+                rs->state = 0;
+                if (rs->finalize) {
+                    rs->finalize(rs);
+                }
+            }
+            break;
+        }
+    }
+
+    assert(size == 0);
+    return 0;
+}
diff --git a/net/netmap.c b/net/netmap.c
index 0c1772b..2d11a8f 100644
--- a/net/netmap.c
+++ b/net/netmap.c
@@ -23,11 +23,9 @@
  */
 
 
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <net/if.h>
-#include <sys/mman.h>
-#include <stdint.h>
-#include <stdio.h>
 #define NETMAP_WITH_LIBS
 #include <net/netmap.h>
 #include <net/netmap_user.h>
@@ -37,23 +35,16 @@
 #include "clients.h"
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
+#include "qapi/error.h"
 #include "qemu/iov.h"
-
-/* Private netmap device info. */
-typedef struct NetmapPriv {
-    int                 fd;
-    size_t              memsize;
-    void                *mem;
-    struct netmap_if    *nifp;
-    struct netmap_ring  *rx;
-    struct netmap_ring  *tx;
-    char                fdname[PATH_MAX];        /* Normally "/dev/netmap". */
-    char                ifname[IFNAMSIZ];
-} NetmapPriv;
+#include "qemu/cutils.h"
 
 typedef struct NetmapState {
     NetClientState      nc;
-    NetmapPriv          me;
+    struct nm_desc      *nmd;
+    char                ifname[IFNAMSIZ];
+    struct netmap_ring  *tx;
+    struct netmap_ring  *rx;
     bool                read_poll;
     bool                write_poll;
     struct iovec        iov[IOV_MAX];
@@ -90,55 +81,23 @@
  * Open a netmap device. We assume there is only one queue
  * (which is the case for the VALE bridge).
  */
-static int netmap_open(NetmapPriv *me)
+static struct nm_desc *netmap_open(const NetdevNetmapOptions *nm_opts,
+                                   Error **errp)
 {
-    int fd;
-    int err;
-    size_t l;
+    struct nm_desc *nmd;
     struct nmreq req;
 
-    me->fd = fd = open(me->fdname, O_RDWR);
-    if (fd < 0) {
-        error_report("Unable to open netmap device '%s' (%s)",
-                        me->fdname, strerror(errno));
-        return -1;
-    }
     memset(&req, 0, sizeof(req));
-    pstrcpy(req.nr_name, sizeof(req.nr_name), me->ifname);
-    req.nr_ringid = NETMAP_NO_TX_POLL;
-    req.nr_version = NETMAP_API;
-    err = ioctl(fd, NIOCREGIF, &req);
-    if (err) {
-        error_report("Unable to register %s: %s", me->ifname, strerror(errno));
-        goto error;
-    }
-    l = me->memsize = req.nr_memsize;
 
-    me->mem = mmap(0, l, PROT_WRITE | PROT_READ, MAP_SHARED, fd, 0);
-    if (me->mem == MAP_FAILED) {
-        error_report("Unable to mmap netmap shared memory: %s",
-                        strerror(errno));
-        me->mem = NULL;
-        goto error;
+    nmd = nm_open(nm_opts->ifname, &req, NETMAP_NO_TX_POLL,
+                  NULL);
+    if (nmd == NULL) {
+        error_setg_errno(errp, errno, "Failed to nm_open() %s",
+                         nm_opts->ifname);
+        return NULL;
     }
 
-    me->nifp = NETMAP_IF(me->mem, req.nr_offset);
-    me->tx = NETMAP_TXRING(me->nifp, 0);
-    me->rx = NETMAP_RXRING(me->nifp, 0);
-    return 0;
-
-error:
-    close(me->fd);
-    return -1;
-}
-
-/* Tell the event-loop if the netmap backend can send packets
-   to the frontend. */
-static int netmap_can_send(void *opaque)
-{
-    NetmapState *s = opaque;
-
-    return qemu_can_send_packet(&s->nc);
+    return nmd;
 }
 
 static void netmap_send(void *opaque);
@@ -147,11 +106,10 @@
 /* Set the event-loop handlers for the netmap backend. */
 static void netmap_update_fd_handler(NetmapState *s)
 {
-    qemu_set_fd_handler2(s->me.fd,
-                         s->read_poll  ? netmap_can_send : NULL,
-                         s->read_poll  ? netmap_send     : NULL,
-                         s->write_poll ? netmap_writable : NULL,
-                         s);
+    qemu_set_fd_handler(s->nmd->fd,
+                        s->read_poll ? netmap_send : NULL,
+                        s->write_poll ? netmap_writable : NULL,
+                        s);
 }
 
 /* Update the read handler. */
@@ -200,7 +158,7 @@
       const uint8_t *buf, size_t size)
 {
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
-    struct netmap_ring *ring = s->me.tx;
+    struct netmap_ring *ring = s->tx;
     uint32_t i;
     uint32_t idx;
     uint8_t *dst;
@@ -230,7 +188,7 @@
     ring->slot[i].flags = 0;
     pkt_copy(buf, dst, size);
     ring->cur = ring->head = nm_ring_next(ring, i);
-    ioctl(s->me.fd, NIOCTXSYNC, NULL);
+    ioctl(s->nmd->fd, NIOCTXSYNC, NULL);
 
     return size;
 }
@@ -239,7 +197,7 @@
                     const struct iovec *iov, int iovcnt)
 {
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
-    struct netmap_ring *ring = s->me.tx;
+    struct netmap_ring *ring = s->tx;
     uint32_t last;
     uint32_t idx;
     uint8_t *dst;
@@ -296,7 +254,7 @@
     /* Now update ring->cur and ring->head. */
     ring->cur = ring->head = i;
 
-    ioctl(s->me.fd, NIOCTXSYNC, NULL);
+    ioctl(s->nmd->fd, NIOCTXSYNC, NULL);
 
     return iov_size(iov, iovcnt);
 }
@@ -313,11 +271,11 @@
 static void netmap_send(void *opaque)
 {
     NetmapState *s = opaque;
-    struct netmap_ring *ring = s->me.rx;
+    struct netmap_ring *ring = s->rx;
 
     /* Keep sending while there are available packets into the netmap
        RX ring and the forwarding path towards the peer is open. */
-    while (!nm_ring_empty(ring) && qemu_can_send_packet(&s->nc)) {
+    while (!nm_ring_empty(ring)) {
         uint32_t i;
         uint32_t idx;
         bool morefrag;
@@ -361,27 +319,52 @@
     qemu_purge_queued_packets(nc);
 
     netmap_poll(nc, false);
-    munmap(s->me.mem, s->me.memsize);
-    close(s->me.fd);
-
-    s->me.fd = -1;
+    nm_close(s->nmd);
+    s->nmd = NULL;
 }
 
 /* Offloading manipulation support callbacks. */
-static bool netmap_has_ufo(NetClientState *nc)
+static int netmap_fd_set_vnet_hdr_len(NetmapState *s, int len)
 {
-    return true;
-}
+    struct nmreq req;
 
-static bool netmap_has_vnet_hdr(NetClientState *nc)
-{
-    return true;
+    /* Issue a NETMAP_BDG_VNET_HDR command to change the virtio-net header
+     * length for the netmap adapter associated to 's->ifname'.
+     */
+    memset(&req, 0, sizeof(req));
+    pstrcpy(req.nr_name, sizeof(req.nr_name), s->ifname);
+    req.nr_version = NETMAP_API;
+    req.nr_cmd = NETMAP_BDG_VNET_HDR;
+    req.nr_arg1 = len;
+
+    return ioctl(s->nmd->fd, NIOCREGIF, &req);
 }
 
 static bool netmap_has_vnet_hdr_len(NetClientState *nc, int len)
 {
-    return len == 0 || len == sizeof(struct virtio_net_hdr) ||
-                len == sizeof(struct virtio_net_hdr_mrg_rxbuf);
+    NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
+    int prev_len = s->vnet_hdr_len;
+
+    /* Check that we can set the new length. */
+    if (netmap_fd_set_vnet_hdr_len(s, len)) {
+        return false;
+    }
+
+    /* Restore the previous length. */
+    if (netmap_fd_set_vnet_hdr_len(s, prev_len)) {
+        error_report("Failed to restore vnet-hdr length %d on %s: %s",
+                     prev_len, s->ifname, strerror(errno));
+        abort();
+    }
+
+    return true;
+}
+
+/* A netmap interface that supports virtio-net headers always
+ * supports UFO, so we use this callback also for the has_ufo hook. */
+static bool netmap_has_vnet_hdr(NetClientState *nc)
+{
+    return netmap_has_vnet_hdr_len(nc, sizeof(struct virtio_net_hdr));
 }
 
 static void netmap_using_vnet_hdr(NetClientState *nc, bool enable)
@@ -392,20 +375,11 @@
 {
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
     int err;
-    struct nmreq req;
 
-    /* Issue a NETMAP_BDG_VNET_HDR command to change the virtio-net header
-     * length for the netmap adapter associated to 'me->ifname'.
-     */
-    memset(&req, 0, sizeof(req));
-    pstrcpy(req.nr_name, sizeof(req.nr_name), s->me.ifname);
-    req.nr_version = NETMAP_API;
-    req.nr_cmd = NETMAP_BDG_VNET_HDR;
-    req.nr_arg1 = len;
-    err = ioctl(s->me.fd, NIOCREGIF, &req);
+    err = netmap_fd_set_vnet_hdr_len(s, len);
     if (err) {
-        error_report("Unable to execute NETMAP_BDG_VNET_HDR on %s: %s",
-                     s->me.ifname, strerror(errno));
+        error_report("Unable to set vnet-hdr length %d on %s: %s",
+                     len, s->ifname, strerror(errno));
     } else {
         /* Keep track of the current length. */
         s->vnet_hdr_len = len;
@@ -418,8 +392,7 @@
     NetmapState *s = DO_UPCAST(NetmapState, nc, nc);
 
     /* Setting a virtio-net header length greater than zero automatically
-     * enables the offloadings.
-     */
+     * enables the offloadings. */
     if (!s->vnet_hdr_len) {
         netmap_set_vnet_hdr_len(nc, sizeof(struct virtio_net_hdr));
     }
@@ -427,13 +400,13 @@
 
 /* NetClientInfo methods */
 static NetClientInfo net_netmap_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_NETMAP,
+    .type = NET_CLIENT_DRIVER_NETMAP,
     .size = sizeof(NetmapState),
     .receive = netmap_receive,
     .receive_iov = netmap_receive_iov,
     .poll = netmap_poll,
     .cleanup = netmap_cleanup,
-    .has_ufo = netmap_has_ufo,
+    .has_ufo = netmap_has_vnet_hdr,
     .has_vnet_hdr = netmap_has_vnet_hdr,
     .has_vnet_hdr_len = netmap_has_vnet_hdr_len,
     .using_vnet_hdr = netmap_using_vnet_hdr,
@@ -445,26 +418,28 @@
  *
  * ... -net netmap,ifname="..."
  */
-int net_init_netmap(const NetClientOptions *opts,
-        const char *name, NetClientState *peer)
+int net_init_netmap(const Netdev *netdev,
+                    const char *name, NetClientState *peer, Error **errp)
 {
-    const NetdevNetmapOptions *netmap_opts = opts->netmap;
+    const NetdevNetmapOptions *netmap_opts = &netdev->u.netmap;
+    struct nm_desc *nmd;
     NetClientState *nc;
-    NetmapPriv me;
+    Error *err = NULL;
     NetmapState *s;
 
-    pstrcpy(me.fdname, sizeof(me.fdname),
-        netmap_opts->has_devname ? netmap_opts->devname : "/dev/netmap");
-    /* Set default name for the port if not supplied. */
-    pstrcpy(me.ifname, sizeof(me.ifname), netmap_opts->ifname);
-    if (netmap_open(&me)) {
+    nmd = netmap_open(netmap_opts, &err);
+    if (err) {
+        error_propagate(errp, err);
         return -1;
     }
     /* Create the object. */
     nc = qemu_new_net_client(&net_netmap_info, peer, "netmap", name);
     s = DO_UPCAST(NetmapState, nc, nc);
-    s->me = me;
+    s->nmd = nmd;
+    s->tx = NETMAP_TXRING(nmd->nifp, 0);
+    s->rx = NETMAP_RXRING(nmd->nifp, 0);
     s->vnet_hdr_len = 0;
+    pstrcpy(s->ifname, sizeof(s->ifname), netmap_opts->ifname);
     netmap_read_poll(s, true); /* Initially only poll for reads. */
 
     return 0;
diff --git a/net/queue.c b/net/queue.c
index f948318..9c32abd 100644
--- a/net/queue.c
+++ b/net/queue.c
@@ -21,6 +21,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "net/queue.h"
 #include "qemu/queue.h"
 #include "net/net.h"
@@ -52,21 +53,23 @@
     void *opaque;
     uint32_t nq_maxlen;
     uint32_t nq_count;
+    NetQueueDeliverFunc *deliver;
 
     QTAILQ_HEAD(packets, NetPacket) packets;
 
     unsigned delivering : 1;
 };
 
-NetQueue *qemu_new_net_queue(void *opaque)
+NetQueue *qemu_new_net_queue(NetQueueDeliverFunc *deliver, void *opaque)
 {
     NetQueue *queue;
 
-    queue = g_malloc0(sizeof(NetQueue));
+    queue = g_new0(NetQueue, 1);
 
     queue->opaque = opaque;
     queue->nq_maxlen = 10000;
     queue->nq_count = 0;
+    queue->deliver = deliver;
 
     QTAILQ_INIT(&queue->packets);
 
@@ -110,12 +113,12 @@
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
 }
 
-static void qemu_net_queue_append_iov(NetQueue *queue,
-                                      NetClientState *sender,
-                                      unsigned flags,
-                                      const struct iovec *iov,
-                                      int iovcnt,
-                                      NetPacketSent *sent_cb)
+void qemu_net_queue_append_iov(NetQueue *queue,
+                               NetClientState *sender,
+                               unsigned flags,
+                               const struct iovec *iov,
+                               int iovcnt,
+                               NetPacketSent *sent_cb)
 {
     NetPacket *packet;
     size_t max_len = 0;
@@ -152,9 +155,13 @@
                                       size_t size)
 {
     ssize_t ret = -1;
+    struct iovec iov = {
+        .iov_base = (void *)data,
+        .iov_len = size
+    };
 
     queue->delivering = 1;
-    ret = qemu_deliver_packet(sender, flags, data, size, queue->opaque);
+    ret = queue->deliver(sender, flags, &iov, 1, queue->opaque);
     queue->delivering = 0;
 
     return ret;
@@ -169,7 +176,7 @@
     ssize_t ret = -1;
 
     queue->delivering = 1;
-    ret = qemu_deliver_packet_iov(sender, flags, iov, iovcnt, queue->opaque);
+    ret = queue->deliver(sender, flags, iov, iovcnt, queue->opaque);
     queue->delivering = 0;
 
     return ret;
diff --git a/net/slirp.c b/net/slirp.c
index 7919d1f..b99ac14 100644
--- a/net/slirp.c
+++ b/net/slirp.c
@@ -21,9 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "net/slirp.h"
 
-#include "config-host.h"
 
 #ifndef _WIN32
 #include <pwd.h>
@@ -33,14 +33,14 @@
 #include "clients.h"
 #include "hub.h"
 #include "monitor/monitor.h"
+#include "qemu/abort.h"
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "slirp/libslirp.h"
+#include "slirp/ip6.h"
 #include "sysemu/char.h"
-
-#if defined(CONFIG_ANDROID)
-#include "android/shaper.h"
-#include "android-qemu2-glue/net-android.h"
-#endif
+#include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
 static int get_str_sep(char *buf, int buf_size, const char **pp, int sep)
 {
@@ -74,10 +74,18 @@
     int legacy_format;
 };
 
+typedef struct SlirpShaper {
+    void* peer;
+    void (*send)(void *peer, const void *packet, int packet_len);
+} SlirpShaper;
+
 typedef struct SlirpState {
     NetClientState nc;
     QTAILQ_ENTRY(SlirpState) entry;
     Slirp *slirp;
+    Notifier exit_notifier;
+    SlirpShaper shaper_out;
+    SlirpShaper shaper_in;
 #ifndef _WIN32
     char smb_dir[128];
 #endif
@@ -89,7 +97,7 @@
 static QTAILQ_HEAD(slirp_stacks, SlirpState) slirp_stacks =
     QTAILQ_HEAD_INITIALIZER(slirp_stacks);
 
-static int slirp_hostfwd(Slirp *s, const char *redir_str,
+static int slirp_hostfwd(SlirpState *s, const char *redir_str,
                          int legacy_format);
 static int slirp_guestfwd(SlirpState *s, const char *config_str,
                           int legacy_format);
@@ -104,41 +112,63 @@
 static inline void slirp_smb_cleanup(SlirpState *s) { }
 #endif
 
+void net_slirp_output_raw(void *opaque, const uint8_t *pkt, int pkt_len)
+{
+    SlirpState *s = opaque;
+    qemu_send_packet(&s->nc, pkt, pkt_len);
+}
+
 void slirp_output(void *opaque, const uint8_t *pkt, int pkt_len)
 {
-#if defined(CONFIG_ANDROID)
-    netshaper_send(android_net_shaper_out, (void*)pkt, pkt_len);
-#else
     SlirpState *s = opaque;
+    SlirpShaper* shaper = &s->shaper_out;
 
-    qemu_send_packet(&s->nc, pkt, pkt_len);
-#endif
+    if (shaper->send) {
+        shaper->send(shaper->peer, pkt, pkt_len);
+    } else {
+        net_slirp_output_raw(opaque, pkt, pkt_len);
+    }
+}
+
+void net_slirp_receive_raw(void* opaque, const uint8_t *buf, size_t size)
+{
+    SlirpState *s = opaque;
+    slirp_input(s->slirp, buf, size);
 }
 
 static ssize_t net_slirp_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
-#if defined(CONFIG_ANDROID)
-    netshaper_send(android_net_shaper_in, (char*)buf, size);
-#else
     SlirpState *s = DO_UPCAST(SlirpState, nc, nc);
+    SlirpShaper *shaper = &s->shaper_in;
 
-    slirp_input(s->slirp, buf, size);
-#endif
-
+    if (shaper->send) {
+        shaper->send(shaper->peer, buf, size);
+    } else {
+        net_slirp_receive_raw(s, buf, size);
+    }
     return size;
 }
 
+static void slirp_smb_exit(Notifier *n, void *data)
+{
+    SlirpState *s = container_of(n, SlirpState, exit_notifier);
+    slirp_smb_cleanup(s);
+}
+
 static void net_slirp_cleanup(NetClientState *nc)
 {
     SlirpState *s = DO_UPCAST(SlirpState, nc, nc);
 
     slirp_cleanup(s->slirp);
+    if (s->exit_notifier.notify) {
+        qemu_remove_exit_notifier(&s->exit_notifier);
+    }
     slirp_smb_cleanup(s);
     QTAILQ_REMOVE(&slirp_stacks, s, entry);
 }
 
 static NetClientInfo net_slirp_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_USER,
+    .type = NET_CLIENT_DRIVER_USER,
     .size = sizeof(SlirpState),
     .receive = net_slirp_receive,
     .cleanup = net_slirp_cleanup,
@@ -146,11 +176,14 @@
 
 static int net_slirp_init(NetClientState *peer, const char *model,
                           const char *name, int restricted,
-                          const char *vnetwork, const char *vhost,
+                          bool ipv4, const char *vnetwork, const char *vhost,
+                          bool ipv6, const char *vprefix6, int vprefix6_len,
+                          const char *vhost6,
                           const char *vhostname, const char *tftp_export,
                           const char *bootfile, const char *vdhcp_start,
-                          const char *vnameserver, const char *smb_export,
-                          const char *vsmbserver, const char **dnssearch)
+                          const char *vnameserver, const char *vnameserver6,
+                          const char *smb_export, const char *vsmbserver,
+                          const char **dnssearch)
 {
     /* default settings according to historic slirp */
     struct in_addr net  = { .s_addr = htonl(0x0a000200) }; /* 10.0.2.0 */
@@ -158,6 +191,9 @@
     struct in_addr host = { .s_addr = htonl(0x0a000202) }; /* 10.0.2.2 */
     struct in_addr dhcp = { .s_addr = htonl(0x0a00020f) }; /* 10.0.2.15 */
     struct in_addr dns  = { .s_addr = htonl(0x0a000203) }; /* 10.0.2.3 */
+    struct in6_addr ip6_prefix;
+    struct in6_addr ip6_host;
+    struct in6_addr ip6_dns;
 #ifndef _WIN32
     struct in_addr smbsrv = { .s_addr = 0 };
 #endif
@@ -169,6 +205,19 @@
     char *end;
     struct slirp_config_str *config;
 
+    if (!ipv4 && (vnetwork || vhost || vnameserver)) {
+        return -1;
+    }
+
+    if (!ipv6 && (vprefix6 || vhost6 || vnameserver6)) {
+        return -1;
+    }
+
+    if (!ipv4 && !ipv6) {
+        /* It doesn't make sense to disable both */
+        return -1;
+    }
+
     if (!tftp_export) {
         tftp_export = legacy_tftp_prefix;
     }
@@ -247,6 +296,64 @@
     }
 #endif
 
+#if defined(_WIN32) && (_WIN32_WINNT < 0x0600)
+    /* No inet_pton helper before Vista... */
+    if (vprefix6) {
+        /* Unsupported */
+        return -1;
+    }
+    memset(&ip6_prefix, 0, sizeof(ip6_prefix));
+    ip6_prefix.s6_addr[0] = 0xfe;
+    ip6_prefix.s6_addr[1] = 0xc0;
+#else
+    if (!vprefix6) {
+        vprefix6 = "fec0::";
+    }
+    if (!inet_pton(AF_INET6, vprefix6, &ip6_prefix)) {
+        return -1;
+    }
+#endif
+
+    if (!vprefix6_len) {
+        vprefix6_len = 64;
+    }
+    if (vprefix6_len < 0 || vprefix6_len > 126) {
+        return -1;
+    }
+
+    if (vhost6) {
+#if defined(_WIN32) && (_WIN32_WINNT < 0x0600)
+        return -1;
+#else
+        if (!inet_pton(AF_INET6, vhost6, &ip6_host)) {
+            return -1;
+        }
+        if (!in6_equal_net(&ip6_prefix, &ip6_host, vprefix6_len)) {
+            return -1;
+        }
+#endif
+    } else {
+        ip6_host = ip6_prefix;
+        ip6_host.s6_addr[15] |= 2;
+    }
+
+    if (vnameserver6) {
+#if defined(_WIN32) && (_WIN32_WINNT < 0x0600)
+        return -1;
+#else
+        if (!inet_pton(AF_INET6, vnameserver6, &ip6_dns)) {
+            return -1;
+        }
+        if (!in6_equal_net(&ip6_prefix, &ip6_dns, vprefix6_len)) {
+            return -1;
+        }
+#endif
+    } else {
+        ip6_dns = ip6_prefix;
+        ip6_dns.s6_addr[15] |= 3;
+    }
+
+
     nc = qemu_new_net_client(&net_slirp_info, peer, model, name);
 
     snprintf(nc->info_str, sizeof(nc->info_str),
@@ -255,13 +362,15 @@
 
     s = DO_UPCAST(SlirpState, nc, nc);
 
-    s->slirp = slirp_init(restricted, net, mask, host, vhostname,
-                          tftp_export, bootfile, dhcp, dns, dnssearch, s);
+    s->slirp = slirp_init(restricted, ipv4, net, mask, host,
+                          ipv6, ip6_prefix, vprefix6_len, ip6_host,
+                          vhostname, tftp_export, bootfile, dhcp,
+                          dns, ip6_dns, dnssearch, s);
     QTAILQ_INSERT_TAIL(&slirp_stacks, s, entry);
 
     for (config = slirp_configs; config; config = config->next) {
         if (config->flags & SLIRP_CFG_HOSTFWD) {
-            if (slirp_hostfwd(s->slirp, config->str,
+            if (slirp_hostfwd(s, config->str,
                               config->flags & SLIRP_CFG_LEGACY) < 0)
                 goto error;
         } else {
@@ -280,10 +389,11 @@
     }
 #endif
 
-#if defined(CONFIG_ANDROID)
-    slirp_init_shapers(s, &s->nc, s->slirp);
-#endif
+    s->exit_notifier.notify = slirp_smb_exit;
+    qemu_add_exit_notifier(&s->exit_notifier);
 
+    s->shaper_out.send = NULL;
+    s->shaper_in.send = NULL;
     return 0;
 
 error:
@@ -291,56 +401,58 @@
     return -1;
 }
 
-Slirp *net_slirp_lookup(const char *vlan, const char *stack, Error **errp)
+static SlirpState *slirp_lookup(Monitor *mon, const char *vlan,
+                                const char *stack)
 {
-    SlirpState *ss;
 
     if (vlan) {
         NetClientState *nc;
         nc = net_hub_find_client_by_name(strtol(vlan, NULL, 0), stack);
         if (!nc) {
-            error_setg(errp, "unrecognized (vlan-id, stackname) pair\n");
+            monitor_printf(mon, "unrecognized (vlan-id, stackname) pair\n");
             return NULL;
         }
         if (strcmp(nc->model, "user")) {
-            error_setg(errp, "invalid device specified");
+            monitor_printf(mon, "invalid device specified\n");
             return NULL;
         }
-        ss = DO_UPCAST(SlirpState, nc, nc);
+        return DO_UPCAST(SlirpState, nc, nc);
     } else {
         if (QTAILQ_EMPTY(&slirp_stacks)) {
-            error_setg(errp, "user mode network stack not in use");
+            monitor_printf(mon, "user mode network stack not in use\n");
             return NULL;
         }
-        ss = QTAILQ_FIRST(&slirp_stacks);
+        return QTAILQ_FIRST(&slirp_stacks);
     }
-    return ss->slirp;
 }
 
-void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict)
+void *net_slirp_state(void)
+{
+    SlirpState *ss = slirp_lookup(NULL, NULL, NULL);
+    return ss ? ss->slirp : NULL;
+}
+
+void hmp_hostfwd_remove(Monitor *mon, const QDict *qdict)
 {
     struct in_addr host_addr = { .s_addr = INADDR_ANY };
     int host_port;
     char buf[256];
     const char *src_str, *p;
-    Slirp *s;
+    SlirpState *s;
     int is_udp = 0;
     int err;
-    Error *local_err = NULL;
     const char *arg1 = qdict_get_str(qdict, "arg1");
     const char *arg2 = qdict_get_try_str(qdict, "arg2");
     const char *arg3 = qdict_get_try_str(qdict, "arg3");
 
     if (arg2) {
-        s = net_slirp_lookup(arg1, arg2, &local_err);
+        s = slirp_lookup(mon, arg1, arg2);
         src_str = arg3;
     } else {
-        s = net_slirp_lookup(NULL, NULL, &local_err);
+        s = slirp_lookup(mon, NULL, NULL);
         src_str = arg1;
     }
-    if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+    if (!s) {
         return;
     }
 
@@ -366,7 +478,7 @@
 
     host_port = atoi(p);
 
-    err = slirp_remove_hostfwd(s, is_udp, host_addr, host_port);
+    err = slirp_remove_hostfwd(s->slirp, is_udp, host_addr, host_port);
 
     monitor_printf(mon, "host forwarding rule for %s %s\n", src_str,
                    err ? "not found" : "removed");
@@ -376,7 +488,7 @@
     monitor_printf(mon, "invalid format\n");
 }
 
-static int slirp_hostfwd(Slirp *s, const char *redir_str,
+static int slirp_hostfwd(SlirpState *s, const char *redir_str,
                          int legacy_format)
 {
     struct in_addr host_addr = { .s_addr = INADDR_ANY };
@@ -428,7 +540,7 @@
         goto fail_syntax;
     }
 
-    if (slirp_add_hostfwd(s, is_udp, host_addr, host_port, guest_addr,
+    if (slirp_add_hostfwd(s->slirp, is_udp, host_addr, host_port, guest_addr,
                           guest_port) < 0) {
         error_report("could not set up host forwarding rule '%s'",
                      redir_str);
@@ -441,29 +553,25 @@
     return -1;
 }
 
-void net_slirp_hostfwd_add(Monitor *mon, const QDict *qdict)
+void hmp_hostfwd_add(Monitor *mon, const QDict *qdict)
 {
     const char *redir_str;
-    Slirp *s;
-    Error *err = NULL;
+    SlirpState *s;
     const char *arg1 = qdict_get_str(qdict, "arg1");
     const char *arg2 = qdict_get_try_str(qdict, "arg2");
     const char *arg3 = qdict_get_try_str(qdict, "arg3");
 
     if (arg2) {
-        s = net_slirp_lookup(arg1, arg2, &err);
+        s = slirp_lookup(mon, arg1, arg2);
         redir_str = arg3;
     } else {
-        s = net_slirp_lookup(NULL, NULL, &err);
+        s = slirp_lookup(mon, NULL, NULL);
         redir_str = arg1;
     }
-    if (err) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
-        return;
+    if (s) {
+        slirp_hostfwd(s, redir_str, 0);
     }
 
-    slirp_hostfwd(s, redir_str, 0);
 }
 
 int net_slirp_redir(const char *redir_str)
@@ -479,7 +587,7 @@
         return 0;
     }
 
-    return slirp_hostfwd(QTAILQ_FIRST(&slirp_stacks)->slirp, redir_str, 1);
+    return slirp_hostfwd(QTAILQ_FIRST(&slirp_stacks), redir_str, 1);
 }
 
 #ifndef _WIN32
@@ -506,7 +614,6 @@
 static int slirp_smb(SlirpState* s, const char *exported_dir,
                      struct in_addr vserver_addr)
 {
-    static int instance;
     char smb_conf[128];
     char smb_cmdline[128];
     struct passwd *passwd;
@@ -530,10 +637,10 @@
         return -1;
     }
 
-    snprintf(s->smb_dir, sizeof(s->smb_dir), "/tmp/qemu-smb.%ld-%d",
-             (long)getpid(), instance++);
-    if (mkdir(s->smb_dir, 0700) < 0) {
+    snprintf(s->smb_dir, sizeof(s->smb_dir), "/tmp/qemu-smb.XXXXXX");
+    if (!mkdtemp(s->smb_dir)) {
         error_report("could not create samba server dir '%s'", s->smb_dir);
+        s->smb_dir[0] = 0;
         return -1;
     }
     snprintf(smb_conf, sizeof(smb_conf), "%s/%s", s->smb_dir, "smb.conf");
@@ -677,7 +784,7 @@
             return -1;
         }
     } else {
-        fwd = g_malloc(sizeof(struct GuestFwd));
+        fwd = g_new(struct GuestFwd, 1);
         fwd->hd = qemu_chr_new(buf, p, NULL);
         if (!fwd->hd) {
             error_report("could not open guest forwarding device '%s'", buf);
@@ -706,7 +813,7 @@
     return -1;
 }
 
-void do_info_usernet(Monitor *mon, const QDict *qdict)
+void hmp_info_usernet(Monitor *mon, const QDict *qdict)
 {
     SlirpState *s;
 
@@ -761,17 +868,28 @@
     return ret;
 }
 
-int net_init_slirp(const NetClientOptions *opts, const char *name,
-                   NetClientState *peer)
+int net_init_slirp(const Netdev *netdev, const char *name,
+                   NetClientState *peer, Error **errp)
 {
+    /* FIXME error_setg(errp, ...) on failure */
     struct slirp_config_str *config;
     char *vnet;
     int ret;
     const NetdevUserOptions *user;
     const char **dnssearch;
+    bool ipv4 = true, ipv6 = true;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_USER);
-    user = opts->user;
+    assert(netdev->type == NET_CLIENT_DRIVER_USER);
+    user = &netdev->u.user;
+
+    if ((user->has_ipv6 && user->ipv6 && !user->has_ipv4) ||
+        (user->has_ipv4 && !user->ipv4)) {
+        ipv4 = 0;
+    }
+    if ((user->has_ipv4 && user->ipv4 && !user->has_ipv6) ||
+        (user->has_ipv6 && !user->ipv6)) {
+        ipv6 = 0;
+    }
 
     vnet = user->has_net ? g_strdup(user->net) :
            user->has_ip  ? g_strdup_printf("%s/24", user->ip) :
@@ -784,9 +902,12 @@
     net_init_slirp_configs(user->hostfwd, SLIRP_CFG_HOSTFWD);
     net_init_slirp_configs(user->guestfwd, 0);
 
-    ret = net_slirp_init(peer, "user", name, user->q_restrict, vnet,
-                         user->host, user->hostname, user->tftp,
-                         user->bootfile, user->dhcpstart, user->dns, user->smb,
+    ret = net_slirp_init(peer, "user", name, user->q_restrict,
+                         ipv4, vnet, user->host,
+                         ipv6, user->ipv6_prefix, user->ipv6_prefixlen,
+                         user->ipv6_host, user->hostname, user->tftp,
+                         user->bootfile, user->dhcpstart,
+                         user->dns, user->ipv6_dns, user->smb,
                          user->smbserver, dnssearch);
 
     while (slirp_configs) {
@@ -808,6 +929,9 @@
         return 0;
     }
 
+    error_report("The '-net channel' option is deprecated. "
+                 "Please use '-netdev user,guestfwd=...' instead.");
+
     /* handle legacy -net channel,port:chr */
     optarg += strlen("channel,");
 
@@ -827,6 +951,20 @@
     return 1;
 }
 
-int net_slirp_is_inited(void) {
-    return !QTAILQ_EMPTY(&slirp_stacks);
+void* net_slirp_set_shapers(void* out_opaque,
+                            SlirpShaperSendFunc out_send,
+                            void* in_opaque,
+                            SlirpShaperSendFunc in_send)
+{
+    SlirpState *s = QTAILQ_FIRST(&slirp_stacks);
+    if (!s) {
+        qemu_abort("Trying to set network shapers before slirp "
+                   "initialization\n");
+    }
+    s->shaper_out.peer = out_opaque;
+    s->shaper_out.send = out_send;
+    s->shaper_in.peer = in_opaque;
+    s->shaper_in.send = in_send;
+
+    return s;
 }
diff --git a/net/socket.c b/net/socket.c
index 68a93cd..3f98eef 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -21,11 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include "net/net.h"
 #include "clients.h"
 #include "monitor/monitor.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
@@ -37,11 +38,8 @@
     NetClientState nc;
     int listen_fd;
     int fd;
-    int state; /* 0 = getting length, 1 = getting data */
-    unsigned int index;
-    unsigned int packet_len;
+    SocketReadState rs;
     unsigned int send_index;      /* number of bytes sent (only SOCK_STREAM) */
-    uint8_t buf[NET_BUFSIZE];
     struct sockaddr_in dgram_dst; /* contains inet host and port destination iff connectionless (SOCK_DGRAM) */
     IOHandler *send_fn;           /* differs between SOCK_STREAM/SOCK_DGRAM */
     bool read_poll;               /* waiting to receive data? */
@@ -51,21 +49,12 @@
 static void net_socket_accept(void *opaque);
 static void net_socket_writable(void *opaque);
 
-/* Only read packets from socket when peer can receive them */
-static int net_socket_can_send(void *opaque)
-{
-    NetSocketState *s = opaque;
-
-    return qemu_can_send_packet(&s->nc);
-}
-
 static void net_socket_update_fd_handler(NetSocketState *s)
 {
-    qemu_set_fd_handler2(s->fd,
-                         s->read_poll  ? net_socket_can_send : NULL,
-                         s->read_poll  ? s->send_fn : NULL,
-                         s->write_poll ? net_socket_writable : NULL,
-                         s);
+    qemu_set_fd_handler(s->fd,
+                        s->read_poll ? s->send_fn : NULL,
+                        s->write_poll ? net_socket_writable : NULL,
+                        s);
 }
 
 static void net_socket_read_poll(NetSocketState *s, bool enable)
@@ -142,18 +131,37 @@
     return ret;
 }
 
+static void net_socket_send_completed(NetClientState *nc, ssize_t len)
+{
+    NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
+
+    if (!s->read_poll) {
+        net_socket_read_poll(s, true);
+    }
+}
+
+static void net_socket_rs_finalize(SocketReadState *rs)
+{
+    NetSocketState *s = container_of(rs, NetSocketState, rs);
+
+    if (qemu_send_packet_async(&s->nc, rs->buf,
+                               rs->packet_len,
+                               net_socket_send_completed) == 0) {
+        net_socket_read_poll(s, false);
+    }
+}
+
 static void net_socket_send(void *opaque)
 {
     NetSocketState *s = opaque;
-    int size, err;
-    unsigned l;
+    int size;
+    int ret;
     uint8_t buf1[NET_BUFSIZE];
     const uint8_t *buf;
 
     size = qemu_recv(s->fd, buf1, sizeof(buf1), 0);
     if (size < 0) {
-        err = socket_error();
-        if (err != EWOULDBLOCK)
+        if (errno != EWOULDBLOCK)
             goto eoc;
     } else if (size == 0) {
         /* end of connection */
@@ -166,57 +174,18 @@
         closesocket(s->fd);
 
         s->fd = -1;
-        s->state = 0;
-        s->index = 0;
-        s->packet_len = 0;
+        net_socket_rs_init(&s->rs, net_socket_rs_finalize);
         s->nc.link_down = true;
-        memset(s->buf, 0, sizeof(s->buf));
         memset(s->nc.info_str, 0, sizeof(s->nc.info_str));
 
         return;
     }
     buf = buf1;
-    while (size > 0) {
-        /* reassemble a packet from the network */
-        switch(s->state) {
-        case 0:
-            l = 4 - s->index;
-            if (l > size)
-                l = size;
-            memcpy(s->buf + s->index, buf, l);
-            buf += l;
-            size -= l;
-            s->index += l;
-            if (s->index == 4) {
-                /* got length */
-                s->packet_len = ntohl(*(uint32_t *)s->buf);
-                s->index = 0;
-                s->state = 1;
-            }
-            break;
-        case 1:
-            l = s->packet_len - s->index;
-            if (l > size)
-                l = size;
-            if (s->index + l <= sizeof(s->buf)) {
-                memcpy(s->buf + s->index, buf, l);
-            } else {
-                fprintf(stderr, "serious error: oversized packet received,"
-                    "connection terminated.\n");
-                s->state = 0;
-                goto eoc;
-            }
 
-            s->index += l;
-            buf += l;
-            size -= l;
-            if (s->index >= s->packet_len) {
-                qemu_send_packet(&s->nc, s->buf, s->packet_len);
-                s->index = 0;
-                s->state = 0;
-            }
-            break;
-        }
+    ret = net_fill_rstate(&s->rs, buf, size);
+
+    if (ret == -1) {
+        goto eoc;
     }
 }
 
@@ -225,7 +194,7 @@
     NetSocketState *s = opaque;
     int size;
 
-    size = qemu_recv(s->fd, s->buf, sizeof(s->buf), 0);
+    size = qemu_recv(s->fd, s->rs.buf, sizeof(s->rs.buf), 0);
     if (size < 0)
         return;
     if (size == 0) {
@@ -234,7 +203,10 @@
         net_socket_write_poll(s, false);
         return;
     }
-    qemu_send_packet(&s->nc, s->buf, size);
+    if (qemu_send_packet_async(&s->nc, s->rs.buf, size,
+                               net_socket_send_completed) == 0) {
+        net_socket_read_poll(s, false);
+    }
 }
 
 static int net_socket_mcast_create(struct sockaddr_in *mcastaddr, struct in_addr *localaddr)
@@ -339,7 +311,7 @@
 }
 
 static NetClientInfo net_dgram_socket_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_SOCKET,
+    .type = NET_CLIENT_DRIVER_SOCKET,
     .size = sizeof(NetSocketState),
     .receive = net_socket_receive_dgram,
     .cleanup = net_socket_cleanup,
@@ -394,6 +366,7 @@
     s->fd = fd;
     s->listen_fd = -1;
     s->send_fn = net_socket_send_dgram;
+    net_socket_rs_init(&s->rs, net_socket_rs_finalize);
     net_socket_read_poll(s, true);
 
     /* mcast: save bound address as dst */
@@ -422,7 +395,7 @@
 }
 
 static NetClientInfo net_socket_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_SOCKET,
+    .type = NET_CLIENT_DRIVER_SOCKET,
     .size = sizeof(NetSocketState),
     .receive = net_socket_receive,
     .cleanup = net_socket_cleanup,
@@ -444,6 +417,7 @@
 
     s->fd = fd;
     s->listen_fd = -1;
+    net_socket_rs_init(&s->rs, net_socket_rs_finalize);
 
     /* Disable Nagle algorithm on TCP sockets to reduce latency */
     socket_set_nodelay(fd);
@@ -559,7 +533,7 @@
                                    const char *host_str)
 {
     NetSocketState *s;
-    int fd, connected, ret, err;
+    int fd, connected, ret;
     struct sockaddr_in saddr;
 
     if (parse_host_port(&saddr, host_str) < 0)
@@ -576,14 +550,12 @@
     for(;;) {
         ret = connect(fd, (struct sockaddr *)&saddr, sizeof(saddr));
         if (ret < 0) {
-            err = socket_error();
-            if (err == EINTR || err == EWOULDBLOCK) {
-            } else if (err == EINPROGRESS) {
+            if (errno == EINTR || errno == EWOULDBLOCK) {
+                /* continue */
+            } else if (errno == EINPROGRESS ||
+                       errno == EALREADY ||
+                       errno == EINVAL) {
                 break;
-#ifdef _WIN32
-            } else if (err == WSAEALREADY || err == WSAEINVAL) {
-                break;
-#endif
             } else {
                 perror("connect");
                 closesocket(fd);
@@ -692,13 +664,15 @@
     return 0;
 }
 
-int net_init_socket(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer)
+int net_init_socket(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp)
 {
+    /* FIXME error_setg(errp, ...) on failure */
+    Error *err = NULL;
     const NetdevSocketOptions *sock;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_SOCKET);
-    sock = opts->socket;
+    assert(netdev->type == NET_CLIENT_DRIVER_SOCKET);
+    sock = &netdev->u.socket;
 
     if (sock->has_fd + sock->has_listen + sock->has_connect + sock->has_mcast +
         sock->has_udp != 1) {
@@ -715,8 +689,9 @@
     if (sock->has_fd) {
         int fd;
 
-        fd = monitor_handle_fd_param(cur_mon, sock->fd);
+        fd = monitor_fd_param(cur_mon, sock->fd, &err);
         if (fd == -1) {
+            error_report_err(err);
             return -1;
         }
         qemu_set_nonblock(fd);
diff --git a/net/tap-aix.c b/net/tap-aix.c
index 804d164..0e6da63 100644
--- a/net/tap-aix.c
+++ b/net/tap-aix.c
@@ -22,19 +22,19 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
-#include <stdio.h>
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required)
+             int vnet_hdr_required, int mq_required, Error **errp)
 {
-    fprintf(stderr, "no tap on AIX\n");
+    error_setg(errp, "no tap on AIX");
     return -1;
 }
 
-int tap_set_sndbuf(int fd, const NetdevTapOptions *tap)
+void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
 {
-    return 0;
 }
 
 int tap_probe_vnet_hdr(int fd)
@@ -56,6 +56,16 @@
 {
 }
 
+int tap_fd_set_vnet_le(int fd, int is_le)
+{
+    return -EINVAL;
+}
+
+int tap_fd_set_vnet_be(int fd, int is_be)
+{
+    return -EINVAL;
+}
+
 void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
diff --git a/net/tap-bsd.c b/net/tap-bsd.c
index bf91bd0..c506ac3 100644
--- a/net/tap-bsd.c
+++ b/net/tap-bsd.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
 
@@ -35,7 +37,7 @@
 
 #ifndef __FreeBSD__
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required)
+             int vnet_hdr_required, int mq_required, Error **errp)
 {
     int fd;
 #ifdef TAPGIFNAME
@@ -71,23 +73,19 @@
         }
     }
     if (fd < 0) {
-        error_report("warning: could not open %s (%s): no virtual network emulation",
-                   dname, strerror(errno));
+        error_setg_errno(errp, errno, "could not open %s", dname);
         return -1;
     }
 
 #ifdef TAPGIFNAME
     if (ioctl(fd, TAPGIFNAME, (void *)&ifr) < 0) {
-        fprintf(stderr, "warning: could not get tap name: %s\n",
-            strerror(errno));
+        error_setg_errno(errp, errno, "could not get tap name");
         return -1;
     }
     pstrcpy(ifname, ifname_size, ifr.ifr_name);
 #else
     if (fstat(fd, &s) < 0) {
-        fprintf(stderr,
-            "warning: could not stat /dev/tap: no virtual network emulation: %s\n",
-            strerror(errno));
+        error_setg_errno(errp, errno, "could not stat %s", dname);
         return -1;
     }
     dev = devname(s.st_rdev, S_IFCHR);
@@ -99,8 +97,8 @@
         *vnet_hdr = 0;
 
         if (vnet_hdr_required && !*vnet_hdr) {
-            error_report("vnet_hdr=1 requested, but no kernel "
-                         "support for IFF_VNET_HDR available");
+            error_setg(errp, "vnet_hdr=1 requested, but no kernel "
+                       "support for IFF_VNET_HDR available");
             close(fd);
             return -1;
         }
@@ -113,15 +111,14 @@
 
 #define PATH_NET_TAP "/dev/tap"
 
-int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required)
+static int tap_open_clone(char *ifname, int ifname_size, Error **errp)
 {
     int fd, s, ret;
     struct ifreq ifr;
 
     TFR(fd = open(PATH_NET_TAP, O_RDWR));
     if (fd < 0) {
-        error_report("could not open %s: %s", PATH_NET_TAP, strerror(errno));
+        error_setg_errno(errp, errno, "could not open %s", PATH_NET_TAP);
         return -1;
     }
 
@@ -129,41 +126,71 @@
 
     ret = ioctl(fd, TAPGIFNAME, (void *)&ifr);
     if (ret < 0) {
-        error_report("could not get tap interface name");
-        goto error;
+        error_setg_errno(errp, errno, "could not get tap interface name");
+        close(fd);
+        return -1;
     }
 
     if (ifname[0] != '\0') {
         /* User requested the interface to have a specific name */
         s = socket(AF_LOCAL, SOCK_DGRAM, 0);
         if (s < 0) {
-            error_report("could not open socket to set interface name");
-            goto error;
+            error_setg_errno(errp, errno,
+                             "could not open socket to set interface name");
+            close(fd);
+            return -1;
         }
         ifr.ifr_data = ifname;
         ret = ioctl(s, SIOCSIFNAME, (void *)&ifr);
         close(s);
         if (ret < 0) {
-            error_report("could not set tap interface name");
-            goto error;
+            error_setg(errp, "could not set tap interface name");
+            close(fd);
+            return -1;
         }
     } else {
         pstrcpy(ifname, ifname_size, ifr.ifr_name);
     }
 
+    return fd;
+}
+
+int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
+             int vnet_hdr_required, int mq_required, Error **errp)
+{
+    int fd = -1;
+
+    /* If the specified tap device already exists just use it. */
+    if (ifname[0] != '\0') {
+        char dname[100];
+        snprintf(dname, sizeof dname, "/dev/%s", ifname);
+        TFR(fd = open(dname, O_RDWR));
+        if (fd < 0 && errno != ENOENT) {
+            error_setg_errno(errp, errno, "could not open %s", dname);
+            return -1;
+        }
+    }
+
+    if (fd < 0) {
+        /* Tap device not specified or does not exist. */
+        if ((fd = tap_open_clone(ifname, ifname_size, errp)) < 0) {
+            return -1;
+        }
+    }
+
     if (*vnet_hdr) {
         /* BSD doesn't have IFF_VNET_HDR */
         *vnet_hdr = 0;
 
         if (vnet_hdr_required && !*vnet_hdr) {
-            error_report("vnet_hdr=1 requested, but no kernel "
-                         "support for IFF_VNET_HDR available");
+            error_setg(errp, "vnet_hdr=1 requested, but no kernel "
+                       "support for IFF_VNET_HDR available");
             goto error;
         }
     }
     if (mq_required) {
-        error_report("mq_required requested, but not kernel support"
-                     "for IFF_MULTI_QUEUE available");
+        error_setg(errp, "mq_required requested, but no kernel support"
+                   " for IFF_MULTI_QUEUE available");
         goto error;
     }
 
@@ -176,9 +203,8 @@
 }
 #endif /* __FreeBSD__ */
 
-int tap_set_sndbuf(int fd, const NetdevTapOptions *tap)
+void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
 {
-    return 0;
 }
 
 int tap_probe_vnet_hdr(int fd)
@@ -200,6 +226,16 @@
 {
 }
 
+int tap_fd_set_vnet_le(int fd, int is_le)
+{
+    return -EINVAL;
+}
+
+int tap_fd_set_vnet_be(int fd, int is_be)
+{
+    return -EINVAL;
+}
+
 void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
diff --git a/net/tap-haiku.c b/net/tap-haiku.c
index e5ce436..b27e57e 100644
--- a/net/tap-haiku.c
+++ b/net/tap-haiku.c
@@ -22,19 +22,19 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
-#include <stdio.h>
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required)
+             int vnet_hdr_required, int mq_required, Error **errp)
 {
-    fprintf(stderr, "no tap on Haiku\n");
+    error_setg(errp, "no tap on Haiku");
     return -1;
 }
 
-int tap_set_sndbuf(int fd, const NetdevTapOptions *tap)
+void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
 {
-    return 0;
 }
 
 int tap_probe_vnet_hdr(int fd)
@@ -56,6 +56,16 @@
 {
 }
 
+int tap_fd_set_vnet_le(int fd, int is_le)
+{
+    return -EINVAL;
+}
+
+int tap_fd_set_vnet_be(int fd, int is_be)
+{
+    return -EINVAL;
+}
+
 void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
diff --git a/net/tap-linux.c b/net/tap-linux.c
index 812bf2d..a503fa9 100644
--- a/net/tap-linux.c
+++ b/net/tap-linux.c
@@ -23,6 +23,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "tap_int.h"
 #include "tap-linux.h"
 #include "net/tap.h"
@@ -31,13 +32,14 @@
 #include <sys/ioctl.h>
 
 #include "sysemu/sysemu.h"
-#include "qemu-common.h"
+#include "qapi/error.h"
 #include "qemu/error-report.h"
+#include "qemu/cutils.h"
 
 #define PATH_NET_TUN "/dev/net/tun"
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required)
+             int vnet_hdr_required, int mq_required, Error **errp)
 {
     struct ifreq ifr;
     int fd, ret;
@@ -46,7 +48,7 @@
 
     TFR(fd = open(PATH_NET_TUN, O_RDWR));
     if (fd < 0) {
-        error_report("could not open %s: %m", PATH_NET_TUN);
+        error_setg_errno(errp, errno, "could not open %s", PATH_NET_TUN);
         return -1;
     }
     memset(&ifr, 0, sizeof(ifr));
@@ -70,8 +72,8 @@
         }
 
         if (vnet_hdr_required && !*vnet_hdr) {
-            error_report("vnet_hdr=1 requested, but no kernel "
-                         "support for IFF_VNET_HDR available");
+            error_setg(errp, "vnet_hdr=1 requested, but no kernel "
+                       "support for IFF_VNET_HDR available");
             close(fd);
             return -1;
         }
@@ -86,8 +88,8 @@
 
     if (mq_required) {
         if (!(features & IFF_MULTI_QUEUE)) {
-            error_report("multiqueue required, but no kernel "
-                         "support for IFF_MULTI_QUEUE available");
+            error_setg(errp, "multiqueue required, but no kernel "
+                       "support for IFF_MULTI_QUEUE available");
             close(fd);
             return -1;
         } else {
@@ -102,9 +104,11 @@
     ret = ioctl(fd, TUNSETIFF, (void *) &ifr);
     if (ret != 0) {
         if (ifname[0] != '\0') {
-            error_report("could not configure %s (%s): %m", PATH_NET_TUN, ifr.ifr_name);
+            error_setg_errno(errp, errno, "could not configure %s (%s)",
+                             PATH_NET_TUN, ifr.ifr_name);
         } else {
-            error_report("could not configure %s: %m", PATH_NET_TUN);
+            error_setg_errno(errp, errno, "could not configure %s",
+                             PATH_NET_TUN);
         }
         close(fd);
         return -1;
@@ -126,7 +130,7 @@
  */
 #define TAP_DEFAULT_SNDBUF 0
 
-int tap_set_sndbuf(int fd, const NetdevTapOptions *tap)
+void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
 {
     int sndbuf;
 
@@ -139,10 +143,8 @@
     }
 
     if (ioctl(fd, TUNSETSNDBUF, &sndbuf) == -1 && tap->has_sndbuf) {
-        error_report("TUNSETSNDBUF ioctl failed: %s", strerror(errno));
-        return -1;
+        error_setg_errno(errp, errno, "TUNSETSNDBUF ioctl failed");
     }
-    return 0;
 }
 
 int tap_probe_vnet_hdr(int fd)
@@ -198,6 +200,40 @@
     }
 }
 
+int tap_fd_set_vnet_le(int fd, int is_le)
+{
+    int arg = is_le ? 1 : 0;
+
+    if (!ioctl(fd, TUNSETVNETLE, &arg)) {
+        return 0;
+    }
+
+    /* Check if our kernel supports TUNSETVNETLE */
+    if (errno == EINVAL) {
+        return -errno;
+    }
+
+    error_report("TUNSETVNETLE ioctl() failed: %s.", strerror(errno));
+    abort();
+}
+
+int tap_fd_set_vnet_be(int fd, int is_be)
+{
+    int arg = is_be ? 1 : 0;
+
+    if (!ioctl(fd, TUNSETVNETBE, &arg)) {
+        return 0;
+    }
+
+    /* Check if our kernel supports TUNSETVNETBE */
+    if (errno == EINVAL) {
+        return -errno;
+    }
+
+    error_report("TUNSETVNETBE ioctl() failed: %s.", strerror(errno));
+    abort();
+}
+
 void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
diff --git a/net/tap-linux.h b/net/tap-linux.h
index 1cf35d4..2f36d10 100644
--- a/net/tap-linux.h
+++ b/net/tap-linux.h
@@ -16,7 +16,6 @@
 #ifndef QEMU_TAP_LINUX_H
 #define QEMU_TAP_LINUX_H
 
-#include <stdint.h>
 #ifdef __linux__
 
 #include <linux/ioctl.h>
@@ -30,6 +29,8 @@
 #define TUNGETVNETHDRSZ _IOR('T', 215, int)
 #define TUNSETVNETHDRSZ _IOW('T', 216, int)
 #define TUNSETQUEUE  _IOW('T', 217, int)
+#define TUNSETVNETLE _IOW('T', 220, int)
+#define TUNSETVNETBE _IOW('T', 222, int)
 
 #endif
 
@@ -49,4 +50,4 @@
 #define TUN_F_TSO_ECN	0x08	/* I can handle TSO with ECN bits. */
 #define TUN_F_UFO	0x10	/* I can handle UFO packets */
 
-#endif /* QEMU_TAP_H */
+#endif /* QEMU_TAP_LINUX_H */
diff --git a/net/tap-solaris.c b/net/tap-solaris.c
index 9c7278f..a2a9235 100644
--- a/net/tap-solaris.c
+++ b/net/tap-solaris.c
@@ -22,10 +22,12 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "tap_int.h"
 #include "sysemu/sysemu.h"
+#include "qemu/cutils.h"
 
-#include <sys/stat.h>
 #include <sys/ethernet.h>
 #include <sys/sockio.h>
 #include <netinet/arp.h>
@@ -36,7 +38,6 @@
 #include <netinet/udp.h>
 #include <netinet/tcp.h>
 #include <net/if.h>
-#include <syslog.h>
 #include <stropts.h>
 #include "qemu/error-report.h"
 
@@ -56,8 +57,10 @@
  * Allocate TAP device, returns opened fd.
  * Stores dev name in the first arg(must be large enough).
  */
-static int tap_alloc(char *dev, size_t dev_size)
+static int tap_alloc(char *dev, size_t dev_size, Error **errp)
 {
+    /* FIXME leaks like a sieve on error paths */
+    /* FIXME suspicious: many errors are reported, then ignored */
     int tap_fd, if_fd, ppa = -1;
     static int ip_fd = 0;
     char *ptr;
@@ -83,14 +86,14 @@
 
     TFR(ip_fd = open("/dev/udp", O_RDWR, 0));
     if (ip_fd < 0) {
-       syslog(LOG_ERR, "Can't open /dev/ip (actually /dev/udp)");
-       return -1;
+        error_setg(errp, "Can't open /dev/ip (actually /dev/udp)");
+        return -1;
     }
 
     TFR(tap_fd = open("/dev/tap", O_RDWR, 0));
     if (tap_fd < 0) {
-       syslog(LOG_ERR, "Can't open /dev/tap");
-       return -1;
+        error_setg(errp, "Can't open /dev/tap");
+        return -1;
     }
 
     /* Assign a new PPA and get its unit number. */
@@ -99,20 +102,20 @@
     strioc_ppa.ic_len = sizeof(ppa);
     strioc_ppa.ic_dp = (char *)&ppa;
     if ((ppa = ioctl (tap_fd, I_STR, &strioc_ppa)) < 0)
-       syslog (LOG_ERR, "Can't assign new interface");
+        error_report("Can't assign new interface");
 
     TFR(if_fd = open("/dev/tap", O_RDWR, 0));
     if (if_fd < 0) {
-       syslog(LOG_ERR, "Can't open /dev/tap (2)");
-       return -1;
+        error_setg(errp, "Can't open /dev/tap (2)");
+        return -1;
     }
     if(ioctl(if_fd, I_PUSH, "ip") < 0){
-       syslog(LOG_ERR, "Can't push IP module");
-       return -1;
+        error_setg(errp, "Can't push IP module");
+        return -1;
     }
 
     if (ioctl(if_fd, SIOCGLIFFLAGS, &ifr) < 0)
-	syslog(LOG_ERR, "Can't get flags\n");
+        error_report("Can't get flags");
 
     snprintf (actual_name, 32, "tap%d", ppa);
     pstrcpy(ifr.lifr_name, sizeof(ifr.lifr_name), actual_name);
@@ -121,22 +124,22 @@
     /* Assign ppa according to the unit number returned by tun device */
 
     if (ioctl (if_fd, SIOCSLIFNAME, &ifr) < 0)
-        syslog (LOG_ERR, "Can't set PPA %d", ppa);
+        error_report("Can't set PPA %d", ppa);
     if (ioctl(if_fd, SIOCGLIFFLAGS, &ifr) <0)
-        syslog (LOG_ERR, "Can't get flags\n");
+        error_report("Can't get flags");
     /* Push arp module to if_fd */
     if (ioctl (if_fd, I_PUSH, "arp") < 0)
-        syslog (LOG_ERR, "Can't push ARP module (2)");
+        error_report("Can't push ARP module (2)");
 
     /* Push arp module to ip_fd */
     if (ioctl (ip_fd, I_POP, NULL) < 0)
-        syslog (LOG_ERR, "I_POP failed\n");
+        error_report("I_POP failed");
     if (ioctl (ip_fd, I_PUSH, "arp") < 0)
-        syslog (LOG_ERR, "Can't push ARP module (3)\n");
+        error_report("Can't push ARP module (3)");
     /* Open arp_fd */
     TFR(arp_fd = open ("/dev/tap", O_RDWR, 0));
     if (arp_fd < 0)
-       syslog (LOG_ERR, "Can't open %s\n", "/dev/tap");
+        error_report("Can't open %s", "/dev/tap");
 
     /* Set ifname to arp */
     strioc_if.ic_cmd = SIOCSLIFNAME;
@@ -144,16 +147,16 @@
     strioc_if.ic_len = sizeof(ifr);
     strioc_if.ic_dp = (char *)&ifr;
     if (ioctl(arp_fd, I_STR, &strioc_if) < 0){
-        syslog (LOG_ERR, "Can't set ifname to arp\n");
+        error_report("Can't set ifname to arp");
     }
 
     if((ip_muxid = ioctl(ip_fd, I_LINK, if_fd)) < 0){
-       syslog(LOG_ERR, "Can't link TAP device to IP");
-       return -1;
+        error_setg(errp, "Can't link TAP device to IP");
+        return -1;
     }
 
     if ((arp_muxid = ioctl (ip_fd, link_type, arp_fd)) < 0)
-        syslog (LOG_ERR, "Can't link TAP device to ARP");
+        error_report("Can't link TAP device to ARP");
 
     close (if_fd);
 
@@ -166,7 +169,7 @@
     {
       ioctl (ip_fd, I_PUNLINK , arp_muxid);
       ioctl (ip_fd, I_PUNLINK, ip_muxid);
-      syslog (LOG_ERR, "Can't set multiplexor id");
+      error_report("Can't set multiplexor id");
     }
 
     snprintf(dev, dev_size, "tap%d", ppa);
@@ -174,13 +177,14 @@
 }
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required)
+             int vnet_hdr_required, int mq_required, Error **errp)
 {
     char  dev[10]="";
     int fd;
-    if( (fd = tap_alloc(dev, sizeof(dev))) < 0 ){
-       fprintf(stderr, "Cannot allocate TAP device\n");
-       return -1;
+
+    fd = tap_alloc(dev, sizeof(dev), errp);
+    if (fd < 0) {
+        return -1;
     }
     pstrcpy(ifname, ifname_size, dev);
     if (*vnet_hdr) {
@@ -188,8 +192,8 @@
         *vnet_hdr = 0;
 
         if (vnet_hdr_required && !*vnet_hdr) {
-            error_report("vnet_hdr=1 requested, but no kernel "
-                         "support for IFF_VNET_HDR available");
+            error_setg(errp, "vnet_hdr=1 requested, but no kernel "
+                       "support for IFF_VNET_HDR available");
             close(fd);
             return -1;
         }
@@ -198,9 +202,8 @@
     return fd;
 }
 
-int tap_set_sndbuf(int fd, const NetdevTapOptions *tap)
+void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp)
 {
-    return 0;
 }
 
 int tap_probe_vnet_hdr(int fd)
@@ -222,6 +225,16 @@
 {
 }
 
+int tap_fd_set_vnet_le(int fd, int is_le)
+{
+    return -EINVAL;
+}
+
+int tap_fd_set_vnet_be(int fd, int is_be)
+{
+    return -EINVAL;
+}
+
 void tap_fd_set_offload(int fd, int csum, int tso4,
                         int tso6, int ecn, int ufo)
 {
diff --git a/net/tap-win32.c b/net/tap-win32.c
index 8aee611..8f21ac3 100644
--- a/net/tap-win32.c
+++ b/net/tap-win32.c
@@ -26,6 +26,7 @@
  *  distribution); if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "tap_int.h"
 
 #include "qemu-common.h"
@@ -34,7 +35,6 @@
 #include "net/tap.h"            /* tap_has_ufo, ... */
 #include "sysemu/sysemu.h"
 #include "qemu/error-report.h"
-#include <stdio.h>
 #include <windows.h>
 #include <winioctl.h>
 
@@ -77,7 +77,12 @@
 
 //#define DEBUG_TAP_WIN32
 
-#define TUN_ASYNCHRONOUS_WRITES 1
+/* FIXME: The asynch write path appears to be broken at
+ * present. WriteFile() ignores the lpNumberOfBytesWritten parameter
+ * for overlapped writes, with the result we return zero bytes sent,
+ * and after handling a single packet, receive is disabled for this
+ * interface. */
+/* #define TUN_ASYNCHRONOUS_WRITES 1 */
 
 #define TUN_BUFFER_SIZE 1560
 #define TUN_MAX_BUFFER_COUNT 32
@@ -356,7 +361,8 @@
                 &len);
 
             if (status != ERROR_SUCCESS || name_type != REG_SZ) {
-                    return -1;
+                ++i;
+                continue;
             }
             else {
                 if (is_tap_win32_dev(enum_name)) {
@@ -460,27 +466,48 @@
     BOOL result;
     DWORD error;
 
+#ifdef TUN_ASYNCHRONOUS_WRITES
     result = GetOverlappedResult( overlapped->handle, &overlapped->write_overlapped,
                                   &write_size, FALSE);
 
     if (!result && GetLastError() == ERROR_IO_INCOMPLETE)
         WaitForSingleObject(overlapped->write_event, INFINITE);
+#endif
 
     result = WriteFile(overlapped->handle, buffer, size,
                        &write_size, &overlapped->write_overlapped);
 
+#ifdef TUN_ASYNCHRONOUS_WRITES
+    /* FIXME: we can't sensibly set write_size here, without waiting
+     * for the IO to complete! Moreover, we can't return zero,
+     * because that will disable receive on this interface, and we
+     * also can't assume it will succeed and return the full size,
+     * because that will result in the buffer being reclaimed while
+     * the IO is in progress. */
+#error Async writes are broken. Please disable TUN_ASYNCHRONOUS_WRITES.
+#else /* !TUN_ASYNCHRONOUS_WRITES */
     if (!result) {
-        switch (error = GetLastError())
-        {
-        case ERROR_IO_PENDING:
-#ifndef TUN_ASYNCHRONOUS_WRITES
-            WaitForSingleObject(overlapped->write_event, INFINITE);
-#endif
-            break;
-        default:
-            return -1;
+        error = GetLastError();
+        if (error == ERROR_IO_PENDING) {
+            result = GetOverlappedResult(overlapped->handle,
+                                         &overlapped->write_overlapped,
+                                         &write_size, TRUE);
         }
     }
+#endif
+
+    if (!result) {
+#ifdef DEBUG_TAP_WIN32
+        LPTSTR msgbuf;
+        error = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM,
+                      NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                      &msgbuf, 0, NULL);
+        fprintf(stderr, "Tap-Win32: Error WriteFile %d - %s\n", error, msgbuf);
+        LocalFree(msgbuf);
+#endif
+        return 0;
+    }
 
     return write_size;
 }
@@ -595,7 +622,7 @@
               device_guid,
               TAPSUFFIX);
 
-    handle = CreateFile (
+    handle = win32CreateFile (
         device_path,
         GENERIC_READ | GENERIC_WRITE,
         0,
@@ -688,6 +715,16 @@
 {
 }
 
+int tap_fd_set_vnet_le(int fd, int is_le)
+{
+    return -EINVAL;
+}
+
+int tap_fd_set_vnet_be(int fd, int is_be)
+{
+    return -EINVAL;
+}
+
 static void tap_using_vnet_hdr(NetClientState *nc, bool using_vnet_hdr)
 {
 }
@@ -713,7 +750,7 @@
 }
 
 static NetClientInfo net_tap_win32_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_TAP,
+    .type = NET_CLIENT_DRIVER_TAP,
     .size = sizeof(TAPState),
     .receive = tap_receive,
     .cleanup = tap_cleanup,
@@ -751,13 +788,14 @@
     return 0;
 }
 
-int net_init_tap(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer)
+int net_init_tap(const Netdev *netdev, const char *name,
+                 NetClientState *peer, Error **errp)
 {
+    /* FIXME error_setg(errp, ...) on failure */
     const NetdevTapOptions *tap;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
-    tap = opts->tap;
+    assert(netdev->type == NET_CLIENT_DRIVER_TAP);
+    tap = &netdev->u.tap;
 
     if (!tap->has_ifname) {
         error_report("tap: no interface name");
diff --git a/net/tap.c b/net/tap.c
index bde6b58..6abb962 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -23,12 +23,11 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "tap_int.h"
 
-#include "config-host.h"
 
 #include <sys/ioctl.h>
-#include <sys/stat.h>
 #include <sys/wait.h>
 #include <sys/socket.h>
 #include <net/if.h>
@@ -37,7 +36,9 @@
 #include "clients.h"
 #include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/error-report.h"
 
 #include "net/tap.h"
@@ -57,21 +58,21 @@
     bool enabled;
     VHostNetState *vhost_net;
     unsigned host_vnet_hdr_len;
+    Notifier exit;
 } TAPState;
 
-static int launch_script(const char *setup_script, const char *ifname, int fd);
+static void launch_script(const char *setup_script, const char *ifname,
+                          int fd, Error **errp);
 
-static int tap_can_send(void *opaque);
 static void tap_send(void *opaque);
 static void tap_writable(void *opaque);
 
 static void tap_update_fd_handler(TAPState *s)
 {
-    qemu_set_fd_handler2(s->fd,
-                         s->read_poll && s->enabled ? tap_can_send : NULL,
-                         s->read_poll && s->enabled ? tap_send     : NULL,
-                         s->write_poll && s->enabled ? tap_writable : NULL,
-                         s);
+    qemu_set_fd_handler(s->fd,
+                        s->read_poll && s->enabled ? tap_send : NULL,
+                        s->write_poll && s->enabled ? tap_writable : NULL,
+                        s);
 }
 
 static void tap_read_poll(TAPState *s, bool enable)
@@ -165,13 +166,6 @@
     return tap_write_packet(s, iov, 1);
 }
 
-static int tap_can_send(void *opaque)
-{
-    TAPState *s = opaque;
-
-    return qemu_can_send_packet(&s->nc);
-}
-
 #ifndef __sun__
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen)
 {
@@ -189,8 +183,9 @@
 {
     TAPState *s = opaque;
     int size;
+    int packets = 0;
 
-    while (qemu_can_send_packet(&s->nc)) {
+    while (true) {
         uint8_t *buf = s->buf;
 
         size = tap_read_packet(s->fd, s->buf, sizeof(s->buf));
@@ -210,6 +205,17 @@
         } else if (size < 0) {
             break;
         }
+
+        /*
+         * When the host keeps receiving more packets while tap_send() is
+         * running we can hog the QEMU global mutex.  Limit the number of
+         * packets that are processed per tap_send() callback to prevent
+         * stalling the guest.
+         */
+        packets++;
+        if (packets >= 50) {
+            break;
+        }
     }
 }
 
@@ -217,7 +223,7 @@
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
 
     return s->has_ufo;
 }
@@ -226,7 +232,7 @@
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
 
     return !!s->host_vnet_hdr_len;
 }
@@ -235,7 +241,7 @@
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
 
     return !!tap_probe_vnet_hdr_len(s->fd, len);
 }
@@ -244,7 +250,7 @@
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
     assert(len == sizeof(struct virtio_net_hdr_mrg_rxbuf) ||
            len == sizeof(struct virtio_net_hdr));
 
@@ -256,12 +262,26 @@
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
     assert(!!s->host_vnet_hdr_len == using_vnet_hdr);
 
     s->using_vnet_hdr = using_vnet_hdr;
 }
 
+static int tap_set_vnet_le(NetClientState *nc, bool is_le)
+{
+    TAPState *s = DO_UPCAST(TAPState, nc, nc);
+
+    return tap_fd_set_vnet_le(s->fd, is_le);
+}
+
+static int tap_set_vnet_be(NetClientState *nc, bool is_be)
+{
+    TAPState *s = DO_UPCAST(TAPState, nc, nc);
+
+    return tap_fd_set_vnet_be(s->fd, is_be);
+}
+
 static void tap_set_offload(NetClientState *nc, int csum, int tso4,
                      int tso6, int ecn, int ufo)
 {
@@ -273,19 +293,33 @@
     tap_fd_set_offload(s->fd, csum, tso4, tso6, ecn, ufo);
 }
 
+static void tap_exit_notify(Notifier *notifier, void *data)
+{
+    TAPState *s = container_of(notifier, TAPState, exit);
+    Error *err = NULL;
+
+    if (s->down_script[0]) {
+        launch_script(s->down_script, s->down_script_arg, s->fd, &err);
+        if (err) {
+            error_report_err(err);
+        }
+    }
+}
+
 static void tap_cleanup(NetClientState *nc)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
 
     if (s->vhost_net) {
         vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
         s->vhost_net = NULL;
     }
 
     qemu_purge_queued_packets(nc);
 
-    if (s->down_script[0])
-        launch_script(s->down_script, s->down_script_arg, s->fd);
+    tap_exit_notify(&s->exit, NULL);
+    qemu_remove_exit_notifier(&s->exit);
 
     tap_read_poll(s, false);
     tap_write_poll(s, false);
@@ -303,14 +337,14 @@
 int tap_get_fd(NetClientState *nc)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
     return s->fd;
 }
 
 /* fd support */
 
 static NetClientInfo net_tap_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_TAP,
+    .type = NET_CLIENT_DRIVER_TAP,
     .size = sizeof(TAPState),
     .receive = tap_receive,
     .receive_raw = tap_receive_raw,
@@ -323,6 +357,8 @@
     .using_vnet_hdr = tap_using_vnet_hdr,
     .set_offload = tap_set_offload,
     .set_vnet_hdr_len = tap_set_vnet_hdr_len,
+    .set_vnet_le = tap_set_vnet_le,
+    .set_vnet_be = tap_set_vnet_be,
 };
 
 static TAPState *net_tap_fd_init(NetClientState *peer,
@@ -353,10 +389,15 @@
     }
     tap_read_poll(s, true);
     s->vhost_net = NULL;
+
+    s->exit.notify = tap_exit_notify;
+    qemu_add_exit_notifier(&s->exit);
+
     return s;
 }
 
-static int launch_script(const char *setup_script, const char *ifname, int fd)
+static void launch_script(const char *setup_script, const char *ifname,
+                          int fd, Error **errp)
 {
     int pid, status;
     char *args[3];
@@ -364,6 +405,11 @@
 
     /* try to launch network script */
     pid = fork();
+    if (pid < 0) {
+        error_setg_errno(errp, errno, "could not launch network script %s",
+                         setup_script);
+        return;
+    }
     if (pid == 0) {
         int open_max = sysconf(_SC_OPEN_MAX), i;
 
@@ -378,17 +424,17 @@
         *parg = NULL;
         execv(setup_script, args);
         _exit(1);
-    } else if (pid > 0) {
+    } else {
         while (waitpid(pid, &status, 0) != pid) {
             /* loop */
         }
 
         if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
-            return 0;
+            return;
         }
+        error_setg(errp, "network script %s failed with status %d",
+                   setup_script, status);
     }
-    fprintf(stderr, "%s: could not launch network script\n", setup_script);
-    return -1;
 }
 
 static int recv_fd(int c)
@@ -425,7 +471,8 @@
     return len;
 }
 
-static int net_bridge_run_helper(const char *helper, const char *bridge)
+static int net_bridge_run_helper(const char *helper, const char *bridge,
+                                 Error **errp)
 {
     sigset_t oldmask, mask;
     int pid, status;
@@ -438,11 +485,16 @@
     sigprocmask(SIG_BLOCK, &mask, &oldmask);
 
     if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
+        error_setg_errno(errp, errno, "socketpair() failed");
         return -1;
     }
 
     /* try to launch bridge helper */
     pid = fork();
+    if (pid < 0) {
+        error_setg_errno(errp, errno, "Can't fork bridge helper");
+        return -1;
+    }
     if (pid == 0) {
         int open_max = sysconf(_SC_OPEN_MAX), i;
         char fd_buf[6+10];
@@ -490,14 +542,16 @@
         }
         _exit(1);
 
-    } else if (pid > 0) {
+    } else {
         int fd;
+        int saved_errno;
 
         close(sv[1]);
 
         do {
             fd = recv_fd(sv[0]);
         } while (fd == -1 && errno == EINTR);
+        saved_errno = errno;
 
         close(sv[0]);
 
@@ -506,47 +560,40 @@
         }
         sigprocmask(SIG_SETMASK, &oldmask, NULL);
         if (fd < 0) {
-            fprintf(stderr, "failed to recv file descriptor\n");
+            error_setg_errno(errp, saved_errno,
+                             "failed to recv file descriptor");
             return -1;
         }
-
-        if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
-            return fd;
+        if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+            error_setg(errp, "bridge helper failed");
+            return -1;
         }
+        return fd;
     }
-    fprintf(stderr, "failed to launch bridge helper\n");
-    return -1;
 }
 
-int net_init_bridge(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer)
+int net_init_bridge(const Netdev *netdev, const char *name,
+                    NetClientState *peer, Error **errp)
 {
     const NetdevBridgeOptions *bridge;
     const char *helper, *br;
-
     TAPState *s;
     int fd, vnet_hdr;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_BRIDGE);
-    bridge = opts->bridge;
+    assert(netdev->type == NET_CLIENT_DRIVER_BRIDGE);
+    bridge = &netdev->u.bridge;
 
     helper = bridge->has_helper ? bridge->helper : DEFAULT_BRIDGE_HELPER;
     br     = bridge->has_br     ? bridge->br     : DEFAULT_BRIDGE_INTERFACE;
 
-    fd = net_bridge_run_helper(helper, br);
+    fd = net_bridge_run_helper(helper, br, errp);
     if (fd == -1) {
         return -1;
     }
 
     fcntl(fd, F_SETFL, O_NONBLOCK);
-
     vnet_hdr = tap_probe_vnet_hdr(fd);
-
     s = net_tap_fd_init(peer, "bridge", name, fd, vnet_hdr);
-    if (!s) {
-        close(fd);
-        return -1;
-    }
 
     snprintf(s->nc.info_str, sizeof(s->nc.info_str), "helper=%s,br=%s", helper,
              br);
@@ -556,8 +603,9 @@
 
 static int net_tap_init(const NetdevTapOptions *tap, int *vnet_hdr,
                         const char *setup_script, char *ifname,
-                        size_t ifname_sz, int mq_required)
+                        size_t ifname_sz, int mq_required, Error **errp)
 {
+    Error *err = NULL;
     int fd, vnet_hdr_required;
 
     if (tap->has_vnet_hdr) {
@@ -569,17 +617,20 @@
     }
 
     TFR(fd = tap_open(ifname, ifname_sz, vnet_hdr, vnet_hdr_required,
-                      mq_required));
+                      mq_required, errp));
     if (fd < 0) {
         return -1;
     }
 
     if (setup_script &&
         setup_script[0] != '\0' &&
-        strcmp(setup_script, "no") != 0 &&
-        launch_script(setup_script, ifname, fd)) {
-        close(fd);
-        return -1;
+        strcmp(setup_script, "no") != 0) {
+        launch_script(setup_script, ifname, fd, &err);
+        if (err) {
+            error_propagate(errp, err);
+            close(fd);
+            return -1;
+        }
     }
 
     return fd;
@@ -587,22 +638,20 @@
 
 #define MAX_TAP_QUEUES 1024
 
-static int net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
-                            const char *model, const char *name,
-                            const char *ifname, const char *script,
-                            const char *downscript, const char *vhostfdname,
-                            int vnet_hdr, int fd)
+static void net_init_tap_one(const NetdevTapOptions *tap, NetClientState *peer,
+                             const char *model, const char *name,
+                             const char *ifname, const char *script,
+                             const char *downscript, const char *vhostfdname,
+                             int vnet_hdr, int fd, Error **errp)
 {
-    TAPState *s;
+    Error *err = NULL;
+    TAPState *s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
     int vhostfd;
 
-    s = net_tap_fd_init(peer, model, name, fd, vnet_hdr);
-    if (!s) {
-        return -1;
-    }
-
-    if (tap_set_sndbuf(s->fd, tap) < 0) {
-        return -1;
+    tap_set_sndbuf(s->fd, tap, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
 
     if (tap->has_fd || tap->has_fds) {
@@ -628,34 +677,37 @@
 
         options.backend_type = VHOST_BACKEND_TYPE_KERNEL;
         options.net_backend = &s->nc;
-        options.force = tap->has_vhostforce && tap->vhostforce;
+        if (tap->has_poll_us) {
+            options.busyloop_timeout = tap->poll_us;
+        } else {
+            options.busyloop_timeout = 0;
+        }
 
-        if (tap->has_vhostfd || tap->has_vhostfds) {
-            vhostfd = monitor_handle_fd_param(cur_mon, vhostfdname);
+        if (vhostfdname) {
+            vhostfd = monitor_fd_param(cur_mon, vhostfdname, &err);
             if (vhostfd == -1) {
-                return -1;
+                error_propagate(errp, err);
+                return;
             }
         } else {
             vhostfd = open("/dev/vhost-net", O_RDWR);
             if (vhostfd < 0) {
-                error_report("tap: open vhost char device failed: %s",
-                           strerror(errno));
-                return -1;
+                error_setg_errno(errp, errno,
+                                 "tap: open vhost char device failed");
+                return;
             }
         }
         options.opaque = (void *)(uintptr_t)vhostfd;
 
         s->vhost_net = vhost_net_init(&options);
         if (!s->vhost_net) {
-            error_report("vhost-net requested but could not be initialized");
-            return -1;
+            error_setg(errp,
+                       "vhost-net requested but could not be initialized");
+            return;
         }
-    } else if (tap->has_vhostfd || tap->has_vhostfds) {
-        error_report("vhostfd= is not valid without vhost");
-        return -1;
+    } else if (vhostfdname) {
+        error_setg(errp, "vhostfd(s)= is not valid without vhost");
     }
-
-    return 0;
 }
 
 static int get_fds(char *str, char *fds[], int max)
@@ -684,26 +736,27 @@
     return i;
 }
 
-int net_init_tap(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer)
+int net_init_tap(const Netdev *netdev, const char *name,
+                 NetClientState *peer, Error **errp)
 {
     const NetdevTapOptions *tap;
     int fd, vnet_hdr = 0, i = 0, queues;
     /* for the no-fd, no-helper case */
     const char *script = NULL; /* suppress wrong "uninit'd use" gcc warning */
     const char *downscript = NULL;
+    Error *err = NULL;
     const char *vhostfdname;
     char ifname[128];
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_TAP);
-    tap = opts->tap;
+    assert(netdev->type == NET_CLIENT_DRIVER_TAP);
+    tap = &netdev->u.tap;
     queues = tap->has_queues ? tap->queues : 1;
     vhostfdname = tap->has_vhostfd ? tap->vhostfd : NULL;
 
     /* QEMU vlans does not support multiqueue tap, in this case peer is set.
      * For -netdev, peer is always NULL. */
     if (peer && (tap->has_queues || tap->has_fds || tap->has_vhostfds)) {
-        error_report("Multiqueue tap cannot be used with QEMU vlans");
+        error_setg(errp, "Multiqueue tap cannot be used with QEMU vlans");
         return -1;
     }
 
@@ -711,14 +764,15 @@
         if (tap->has_ifname || tap->has_script || tap->has_downscript ||
             tap->has_vnet_hdr || tap->has_helper || tap->has_queues ||
             tap->has_fds || tap->has_vhostfds) {
-            error_report("ifname=, script=, downscript=, vnet_hdr=, "
-                         "helper=, queues=, fds=, and vhostfds= "
-                         "are invalid with fd=");
+            error_setg(errp, "ifname=, script=, downscript=, vnet_hdr=, "
+                       "helper=, queues=, fds=, and vhostfds= "
+                       "are invalid with fd=");
             return -1;
         }
 
-        fd = monitor_handle_fd_param(cur_mon, tap->fd);
+        fd = monitor_fd_param(cur_mon, tap->fd, &err);
         if (fd == -1) {
+            error_propagate(errp, err);
             return -1;
         }
 
@@ -726,22 +780,24 @@
 
         vnet_hdr = tap_probe_vnet_hdr(fd);
 
-        if (net_init_tap_one(tap, peer, "tap", name, NULL,
-                             script, downscript,
-                             vhostfdname, vnet_hdr, fd)) {
+        net_init_tap_one(tap, peer, "tap", name, NULL,
+                         script, downscript,
+                         vhostfdname, vnet_hdr, fd, &err);
+        if (err) {
+            error_propagate(errp, err);
             return -1;
         }
     } else if (tap->has_fds) {
-        char *fds[MAX_TAP_QUEUES];
-        char *vhost_fds[MAX_TAP_QUEUES];
+        char **fds = g_new0(char *, MAX_TAP_QUEUES);
+        char **vhost_fds = g_new0(char *, MAX_TAP_QUEUES);
         int nfds, nvhosts;
 
         if (tap->has_ifname || tap->has_script || tap->has_downscript ||
             tap->has_vnet_hdr || tap->has_helper || tap->has_queues ||
             tap->has_vhostfd) {
-            error_report("ifname=, script=, downscript=, vnet_hdr=, "
-                         "helper=, queues=, and vhostfd= "
-                         "are invalid with fds=");
+            error_setg(errp, "ifname=, script=, downscript=, vnet_hdr=, "
+                       "helper=, queues=, and vhostfd= "
+                       "are invalid with fds=");
             return -1;
         }
 
@@ -749,16 +805,17 @@
         if (tap->has_vhostfds) {
             nvhosts = get_fds(tap->vhostfds, vhost_fds, MAX_TAP_QUEUES);
             if (nfds != nvhosts) {
-                error_report("The number of fds passed does not match the "
-                             "number of vhostfds passed");
-                return -1;
+                error_setg(errp, "The number of fds passed does not match "
+                           "the number of vhostfds passed");
+                goto free_fail;
             }
         }
 
         for (i = 0; i < nfds; i++) {
-            fd = monitor_handle_fd_param(cur_mon, fds[i]);
+            fd = monitor_fd_param(cur_mon, fds[i], &err);
             if (fd == -1) {
-                return -1;
+                error_propagate(errp, err);
+                goto free_fail;
             }
 
             fcntl(fd, F_SETFL, O_NONBLOCK);
@@ -766,26 +823,42 @@
             if (i == 0) {
                 vnet_hdr = tap_probe_vnet_hdr(fd);
             } else if (vnet_hdr != tap_probe_vnet_hdr(fd)) {
-                error_report("vnet_hdr not consistent across given tap fds");
-                return -1;
+                error_setg(errp,
+                           "vnet_hdr not consistent across given tap fds");
+                goto free_fail;
             }
 
-            if (net_init_tap_one(tap, peer, "tap", name, ifname,
-                                 script, downscript,
-                                 tap->has_vhostfds ? vhost_fds[i] : NULL,
-                                 vnet_hdr, fd)) {
-                return -1;
+            net_init_tap_one(tap, peer, "tap", name, ifname,
+                             script, downscript,
+                             tap->has_vhostfds ? vhost_fds[i] : NULL,
+                             vnet_hdr, fd, &err);
+            if (err) {
+                error_propagate(errp, err);
+                goto free_fail;
             }
         }
+        g_free(fds);
+        g_free(vhost_fds);
+        return 0;
+
+free_fail:
+        for (i = 0; i < nfds; i++) {
+            g_free(fds[i]);
+            g_free(vhost_fds[i]);
+        }
+        g_free(fds);
+        g_free(vhost_fds);
+        return -1;
     } else if (tap->has_helper) {
         if (tap->has_ifname || tap->has_script || tap->has_downscript ||
             tap->has_vnet_hdr || tap->has_queues || tap->has_vhostfds) {
-            error_report("ifname=, script=, downscript=, and vnet_hdr= "
-                         "queues=, and vhostfds= are invalid with helper=");
+            error_setg(errp, "ifname=, script=, downscript=, vnet_hdr=, "
+                       "queues=, and vhostfds= are invalid with helper=");
             return -1;
         }
 
-        fd = net_bridge_run_helper(tap->helper, DEFAULT_BRIDGE_INTERFACE);
+        fd = net_bridge_run_helper(tap->helper, DEFAULT_BRIDGE_INTERFACE,
+                                   errp);
         if (fd == -1) {
             return -1;
         }
@@ -793,15 +866,17 @@
         fcntl(fd, F_SETFL, O_NONBLOCK);
         vnet_hdr = tap_probe_vnet_hdr(fd);
 
-        if (net_init_tap_one(tap, peer, "bridge", name, ifname,
-                             script, downscript, vhostfdname,
-                             vnet_hdr, fd)) {
+        net_init_tap_one(tap, peer, "bridge", name, ifname,
+                         script, downscript, vhostfdname,
+                         vnet_hdr, fd, &err);
+        if (err) {
+            error_propagate(errp, err);
             close(fd);
             return -1;
         }
     } else {
         if (tap->has_vhostfds) {
-            error_report("vhostfds= is invalid if fds= wasn't specified");
+            error_setg(errp, "vhostfds= is invalid if fds= wasn't specified");
             return -1;
         }
         script = tap->has_script ? tap->script : DEFAULT_NETWORK_SCRIPT;
@@ -816,23 +891,25 @@
 
         for (i = 0; i < queues; i++) {
             fd = net_tap_init(tap, &vnet_hdr, i >= 1 ? "no" : script,
-                              ifname, sizeof ifname, queues > 1);
+                              ifname, sizeof ifname, queues > 1, errp);
             if (fd == -1) {
                 return -1;
             }
 
             if (queues > 1 && i == 0 && !tap->has_ifname) {
                 if (tap_fd_get_ifname(fd, ifname)) {
-                    error_report("Fail to get ifname");
+                    error_setg(errp, "Fail to get ifname");
                     close(fd);
                     return -1;
                 }
             }
 
-            if (net_init_tap_one(tap, peer, "tap", name, ifname,
-                                 i >= 1 ? "no" : script,
-                                 i >= 1 ? "no" : downscript,
-                                 vhostfdname, vnet_hdr, fd)) {
+            net_init_tap_one(tap, peer, "tap", name, ifname,
+                             i >= 1 ? "no" : script,
+                             i >= 1 ? "no" : downscript,
+                             vhostfdname, vnet_hdr, fd, &err);
+            if (err) {
+                error_propagate(errp, err);
                 close(fd);
                 return -1;
             }
@@ -845,7 +922,7 @@
 VHostNetState *tap_get_vhost_net(NetClientState *nc)
 {
     TAPState *s = DO_UPCAST(TAPState, nc, nc);
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_TAP);
+    assert(nc->info->type == NET_CLIENT_DRIVER_TAP);
     return s->vhost_net;
 }
 
diff --git a/net/tap_int.h b/net/tap_int.h
index 79afdf2..ae6888f 100644
--- a/net/tap_int.h
+++ b/net/tap_int.h
@@ -23,25 +23,27 @@
  * THE SOFTWARE.
  */
 
-#ifndef QEMU_TAP_H
-#define QEMU_TAP_H
+#ifndef NET_TAP_INT_H
+#define NET_TAP_INT_H
 
 #include "qemu-common.h"
 #include "qapi-types.h"
 
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr,
-             int vnet_hdr_required, int mq_required);
+             int vnet_hdr_required, int mq_required, Error **errp);
 
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen);
 
-int tap_set_sndbuf(int fd, const NetdevTapOptions *tap);
+void tap_set_sndbuf(int fd, const NetdevTapOptions *tap, Error **errp);
 int tap_probe_vnet_hdr(int fd);
 int tap_probe_vnet_hdr_len(int fd, int len);
 int tap_probe_has_ufo(int fd);
 void tap_fd_set_offload(int fd, int csum, int tso4, int tso6, int ecn, int ufo);
 void tap_fd_set_vnet_hdr_len(int fd, int len);
+int tap_fd_set_vnet_le(int fd, int vnet_is_le);
+int tap_fd_set_vnet_be(int fd, int vnet_is_be);
 int tap_fd_enable(int fd);
 int tap_fd_disable(int fd);
 int tap_fd_get_ifname(int fd, char *ifname);
 
-#endif /* QEMU_TAP_H */
+#endif /* NET_TAP_INT_H */
diff --git a/net/trace-events b/net/trace-events
new file mode 100644
index 0000000..65c46a4
--- /dev/null
+++ b/net/trace-events
@@ -0,0 +1,4 @@
+# See docs/tracing.txt for syntax documentation.
+
+# net/vhost-user.c
+vhost_user_event(const char *chr, int event) "chr: %s got event: %d"
diff --git a/net/util.c b/net/util.c
index 7e95076..0b3dbfe 100644
--- a/net/util.c
+++ b/net/util.c
@@ -22,9 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "util.h"
-#include <errno.h>
-#include <stdlib.h>
 
 int net_parse_macaddr(uint8_t *macaddr, const char *p)
 {
diff --git a/net/util.h b/net/util.h
index 10c7da9..60b73d3 100644
--- a/net/util.h
+++ b/net/util.h
@@ -25,7 +25,6 @@
 #ifndef QEMU_NET_UTIL_H
 #define QEMU_NET_UTIL_H
 
-#include <stdint.h>
 
 int net_parse_macaddr(uint8_t *macaddr, const char *p);
 
diff --git a/net/vde.c b/net/vde.c
index 2a619fb..e50e5d6 100644
--- a/net/vde.c
+++ b/net/vde.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 #include <libvdeplug.h>
 
@@ -68,7 +68,7 @@
 }
 
 static NetClientInfo net_vde_info = {
-    .type = NET_CLIENT_OPTIONS_KIND_VDE,
+    .type = NET_CLIENT_DRIVER_VDE,
     .size = sizeof(VDEState),
     .receive = vde_receive,
     .cleanup = vde_cleanup,
@@ -109,13 +109,14 @@
     return 0;
 }
 
-int net_init_vde(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer)
+int net_init_vde(const Netdev *netdev, const char *name,
+                 NetClientState *peer, Error **errp)
 {
+    /* FIXME error_setg(errp, ...) on failure */
     const NetdevVdeOptions *vde;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_VDE);
-    vde = opts->vde;
+    assert(netdev->type == NET_CLIENT_DRIVER_VDE);
+    vde = &netdev->u.vde;
 
     /* missing optional values have been initialized to "all bits zero" */
     if (net_vde_init(peer, "vde", name, vde->sock, vde->port, vde->group,
diff --git a/net/vhost-user.c b/net/vhost-user.c
index 24e050c..b0595f8 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -8,156 +8,279 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "clients.h"
 #include "net/vhost_net.h"
 #include "net/vhost-user.h"
 #include "sysemu/char.h"
 #include "qemu/config-file.h"
 #include "qemu/error-report.h"
+#include "qmp-commands.h"
+#include "trace.h"
 
 typedef struct VhostUserState {
     NetClientState nc;
     CharDriverState *chr;
-    bool vhostforce;
     VHostNetState *vhost_net;
+    guint watch;
+    uint64_t acked_features;
+    bool started;
 } VhostUserState;
 
 typedef struct VhostUserChardevProps {
     bool is_socket;
     bool is_unix;
-    bool is_server;
 } VhostUserChardevProps;
 
 VHostNetState *vhost_user_get_vhost_net(NetClientState *nc)
 {
     VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_USER);
     return s->vhost_net;
 }
 
-static int vhost_user_running(VhostUserState *s)
+uint64_t vhost_user_get_acked_features(NetClientState *nc)
 {
-    return (s->vhost_net) ? 1 : 0;
+    VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_USER);
+    return s->acked_features;
 }
 
-static int vhost_user_start(VhostUserState *s)
+static void vhost_user_stop(int queues, NetClientState *ncs[])
+{
+    VhostUserState *s;
+    int i;
+
+    for (i = 0; i < queues; i++) {
+        assert(ncs[i]->info->type == NET_CLIENT_DRIVER_VHOST_USER);
+
+        s = DO_UPCAST(VhostUserState, nc, ncs[i]);
+
+        if (s->vhost_net) {
+            /* save acked features */
+            uint64_t features = vhost_net_get_acked_features(s->vhost_net);
+            if (features) {
+                s->acked_features = features;
+            }
+            vhost_net_cleanup(s->vhost_net);
+        }
+    }
+}
+
+static int vhost_user_start(int queues, NetClientState *ncs[])
 {
     VhostNetOptions options;
-
-    if (vhost_user_running(s)) {
-        return 0;
-    }
+    struct vhost_net *net = NULL;
+    VhostUserState *s;
+    int max_queues;
+    int i;
 
     options.backend_type = VHOST_BACKEND_TYPE_USER;
-    options.net_backend = &s->nc;
-    options.opaque = s->chr;
-    options.force = s->vhostforce;
 
-    s->vhost_net = vhost_net_init(&options);
+    for (i = 0; i < queues; i++) {
+        assert(ncs[i]->info->type == NET_CLIENT_DRIVER_VHOST_USER);
 
-    return vhost_user_running(s) ? 0 : -1;
-}
+        s = DO_UPCAST(VhostUserState, nc, ncs[i]);
 
-static void vhost_user_stop(VhostUserState *s)
-{
-    if (vhost_user_running(s)) {
-        vhost_net_cleanup(s->vhost_net);
+        options.net_backend = ncs[i];
+        options.opaque      = s->chr;
+        options.busyloop_timeout = 0;
+        net = vhost_net_init(&options);
+        if (!net) {
+            error_report("failed to init vhost_net for queue %d", i);
+            goto err;
+        }
+
+        if (i == 0) {
+            max_queues = vhost_net_get_max_queues(net);
+            if (queues > max_queues) {
+                error_report("you are asking more queues than supported: %d",
+                             max_queues);
+                goto err;
+            }
+        }
+
+        if (s->vhost_net) {
+            vhost_net_cleanup(s->vhost_net);
+            g_free(s->vhost_net);
+        }
+        s->vhost_net = net;
     }
 
-    s->vhost_net = 0;
+    return 0;
+
+err:
+    if (net) {
+        vhost_net_cleanup(net);
+    }
+    vhost_user_stop(i, ncs);
+    return -1;
+}
+
+static ssize_t vhost_user_receive(NetClientState *nc, const uint8_t *buf,
+                                  size_t size)
+{
+    /* In case of RARP (message size is 60) notify backup to send a fake RARP.
+       This fake RARP will be sent by backend only for guest
+       without GUEST_ANNOUNCE capability.
+     */
+    if (size == 60) {
+        VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
+        int r;
+        static int display_rarp_failure = 1;
+        char mac_addr[6];
+
+        /* extract guest mac address from the RARP message */
+        memcpy(mac_addr, &buf[6], 6);
+
+        r = vhost_net_notify_migration_done(s->vhost_net, mac_addr);
+
+        if ((r != 0) && (display_rarp_failure)) {
+            fprintf(stderr,
+                    "Vhost user backend fails to broadcast fake RARP\n");
+            fflush(stderr);
+            display_rarp_failure = 0;
+        }
+    }
+
+    return size;
 }
 
 static void vhost_user_cleanup(NetClientState *nc)
 {
     VhostUserState *s = DO_UPCAST(VhostUserState, nc, nc);
 
-    vhost_user_stop(s);
+    if (s->vhost_net) {
+        vhost_net_cleanup(s->vhost_net);
+        g_free(s->vhost_net);
+        s->vhost_net = NULL;
+    }
+    if (s->chr) {
+        qemu_chr_add_handlers(s->chr, NULL, NULL, NULL, NULL);
+        qemu_chr_fe_release(s->chr);
+        s->chr = NULL;
+    }
+
     qemu_purge_queued_packets(nc);
 }
 
 static bool vhost_user_has_vnet_hdr(NetClientState *nc)
 {
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_USER);
 
     return true;
 }
 
 static bool vhost_user_has_ufo(NetClientState *nc)
 {
-    assert(nc->info->type == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
+    assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_USER);
 
     return true;
 }
 
 static NetClientInfo net_vhost_user_info = {
-        .type = NET_CLIENT_OPTIONS_KIND_VHOST_USER,
+        .type = NET_CLIENT_DRIVER_VHOST_USER,
         .size = sizeof(VhostUserState),
+        .receive = vhost_user_receive,
         .cleanup = vhost_user_cleanup,
         .has_vnet_hdr = vhost_user_has_vnet_hdr,
         .has_ufo = vhost_user_has_ufo,
 };
 
-static void net_vhost_link_down(VhostUserState *s, bool link_down)
+static gboolean net_vhost_user_watch(GIOChannel *chan, GIOCondition cond,
+                                           void *opaque)
 {
-    s->nc.link_down = link_down;
+    VhostUserState *s = opaque;
 
-    if (s->nc.peer) {
-        s->nc.peer->link_down = link_down;
-    }
+    qemu_chr_disconnect(s->chr);
 
-    if (s->nc.info->link_status_changed) {
-        s->nc.info->link_status_changed(&s->nc);
-    }
-
-    if (s->nc.peer && s->nc.peer->info->link_status_changed) {
-        s->nc.peer->info->link_status_changed(s->nc.peer);
-    }
+    return FALSE;
 }
 
 static void net_vhost_user_event(void *opaque, int event)
 {
-    VhostUserState *s = opaque;
+    const char *name = opaque;
+    NetClientState *ncs[MAX_QUEUE_NUM];
+    VhostUserState *s;
+    Error *err = NULL;
+    int queues;
 
+    queues = qemu_find_net_clients_except(name, ncs,
+                                          NET_CLIENT_DRIVER_NIC,
+                                          MAX_QUEUE_NUM);
+    assert(queues < MAX_QUEUE_NUM);
+
+    s = DO_UPCAST(VhostUserState, nc, ncs[0]);
+    trace_vhost_user_event(s->chr->label, event);
     switch (event) {
     case CHR_EVENT_OPENED:
-        vhost_user_start(s);
-        net_vhost_link_down(s, false);
-        error_report("chardev \"%s\" went up\n", s->chr->label);
+        s->watch = qemu_chr_fe_add_watch(s->chr, G_IO_HUP,
+                                         net_vhost_user_watch, s);
+        if (vhost_user_start(queues, ncs) < 0) {
+            qemu_chr_disconnect(s->chr);
+            return;
+        }
+        qmp_set_link(name, true, &err);
+        s->started = true;
         break;
     case CHR_EVENT_CLOSED:
-        net_vhost_link_down(s, true);
-        vhost_user_stop(s);
-        error_report("chardev \"%s\" went down\n", s->chr->label);
+        qmp_set_link(name, false, &err);
+        vhost_user_stop(queues, ncs);
+        g_source_remove(s->watch);
+        s->watch = 0;
         break;
     }
+
+    if (err) {
+        error_report_err(err);
+    }
 }
 
 static int net_vhost_user_init(NetClientState *peer, const char *device,
                                const char *name, CharDriverState *chr,
-                               bool vhostforce)
+                               int queues)
 {
-    NetClientState *nc;
+    NetClientState *nc, *nc0 = NULL;
     VhostUserState *s;
+    int i;
 
-    nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
+    assert(name);
+    assert(queues > 0);
 
-    snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user to %s",
-             chr->label);
+    for (i = 0; i < queues; i++) {
+        nc = qemu_new_net_client(&net_vhost_user_info, peer, device, name);
+        if (!nc0) {
+            nc0 = nc;
+        }
 
-    s = DO_UPCAST(VhostUserState, nc, nc);
+        snprintf(nc->info_str, sizeof(nc->info_str), "vhost-user%d to %s",
+                 i, chr->label);
 
-    /* We don't provide a receive callback */
-    s->nc.receive_disabled = 1;
-    s->chr = chr;
-    s->vhostforce = vhostforce;
+        nc->queue_index = i;
 
-    qemu_chr_add_handlers(s->chr, NULL, NULL, net_vhost_user_event, s);
+        s = DO_UPCAST(VhostUserState, nc, nc);
+        s->chr = chr;
+    }
+
+    s = DO_UPCAST(VhostUserState, nc, nc0);
+    do {
+        Error *err = NULL;
+        if (qemu_chr_wait_connected(chr, &err) < 0) {
+            error_report_err(err);
+            return -1;
+        }
+        qemu_chr_add_handlers(chr, NULL, NULL,
+                              net_vhost_user_event, nc0->name);
+    } while (!s->started);
+
+    assert(s->vhost_net);
 
     return 0;
 }
 
-static int net_vhost_chardev_opts(const char *name, const char *value,
-                                  void *opaque)
+static int net_vhost_chardev_opts(void *opaque,
+                                  const char *name, const char *value,
+                                  Error **errp)
 {
     VhostUserChardevProps *props = opaque;
 
@@ -166,35 +289,35 @@
     } else if (strcmp(name, "path") == 0) {
         props->is_unix = true;
     } else if (strcmp(name, "server") == 0) {
-        props->is_server = true;
     } else {
-        error_report("vhost-user does not support a chardev"
-                     " with the following option:\n %s = %s",
-                     name, value);
+        error_setg(errp,
+                   "vhost-user does not support a chardev with option %s=%s",
+                   name, value);
         return -1;
     }
     return 0;
 }
 
-static CharDriverState *net_vhost_parse_chardev(const NetdevVhostUserOptions *opts)
+static CharDriverState *net_vhost_parse_chardev(
+    const NetdevVhostUserOptions *opts, Error **errp)
 {
     CharDriverState *chr = qemu_chr_find(opts->chardev);
     VhostUserChardevProps props;
 
     if (chr == NULL) {
-        error_report("chardev \"%s\" not found", opts->chardev);
+        error_setg(errp, "chardev \"%s\" not found", opts->chardev);
         return NULL;
     }
 
     /* inspect chardev opts */
     memset(&props, 0, sizeof(props));
-    if (qemu_opt_foreach(chr->opts, net_vhost_chardev_opts, &props, true) != 0) {
+    if (qemu_opt_foreach(chr->opts, net_vhost_chardev_opts, &props, errp)) {
         return NULL;
     }
 
     if (!props.is_socket || !props.is_unix) {
-        error_report("chardev \"%s\" is not a unix socket",
-                     opts->chardev);
+        error_setg(errp, "chardev \"%s\" is not a unix socket",
+                   opts->chardev);
         return NULL;
     }
 
@@ -203,11 +326,10 @@
     return chr;
 }
 
-static int net_vhost_check_net(QemuOpts *opts, void *opaque)
+static int net_vhost_check_net(void *opaque, QemuOpts *opts, Error **errp)
 {
     const char *name = opaque;
     const char *driver, *netdev;
-    const char virtio_name[] = "virtio-net-";
 
     driver = qemu_opt_get(opts, "driver");
     netdev = qemu_opt_get(opts, "netdev");
@@ -217,42 +339,42 @@
     }
 
     if (strcmp(netdev, name) == 0 &&
-        strncmp(driver, virtio_name, strlen(virtio_name)) != 0) {
-        error_report("vhost-user requires frontend driver virtio-net-*");
+        !g_str_has_prefix(driver, "virtio-net-")) {
+        error_setg(errp, "vhost-user requires frontend driver virtio-net-*");
         return -1;
     }
 
     return 0;
 }
 
-int net_init_vhost_user(const NetClientOptions *opts, const char *name,
-                        NetClientState *peer)
+int net_init_vhost_user(const Netdev *netdev, const char *name,
+                        NetClientState *peer, Error **errp)
 {
+    int queues;
     const NetdevVhostUserOptions *vhost_user_opts;
     CharDriverState *chr;
-    bool vhostforce;
 
-    assert(opts->kind == NET_CLIENT_OPTIONS_KIND_VHOST_USER);
-    vhost_user_opts = opts->vhost_user;
+    assert(netdev->type == NET_CLIENT_DRIVER_VHOST_USER);
+    vhost_user_opts = &netdev->u.vhost_user;
 
-    chr = net_vhost_parse_chardev(vhost_user_opts);
+    chr = net_vhost_parse_chardev(vhost_user_opts, errp);
     if (!chr) {
-        error_report("No suitable chardev found");
         return -1;
     }
 
     /* verify net frontend */
     if (qemu_opts_foreach(qemu_find_opts("device"), net_vhost_check_net,
-                          (char *)name, true) == -1) {
+                          (char *)name, errp)) {
         return -1;
     }
 
-    /* vhostforce for non-MSIX */
-    if (vhost_user_opts->has_vhostforce) {
-        vhostforce = vhost_user_opts->vhostforce;
-    } else {
-        vhostforce = false;
+    queues = vhost_user_opts->has_queues ? vhost_user_opts->queues : 1;
+    if (queues < 1 || queues > MAX_QUEUE_NUM) {
+        error_setg(errp,
+                   "vhost-user number of queues must be in range [1, %d]",
+                   MAX_QUEUE_NUM);
+        return -1;
     }
 
-    return net_vhost_user_init(peer, "vhost_user", name, chr, vhostforce);
+    return net_vhost_user_init(peer, "vhost_user", name, chr, queues);
 }
diff --git a/numa.c b/numa.c
index 18391a9..cb3ee84 100644
--- a/numa.c
+++ b/numa.c
@@ -22,7 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu/sysemu.h"
+#include "qemu/osdep.h"
+#include "sysemu/numa.h"
 #include "exec/cpu-common.h"
 #include "qemu/bitmap.h"
 #include "qom/cpu.h"
@@ -30,16 +31,12 @@
 #include "include/exec/cpu-common.h" /* for RAM_ADDR_FMT */
 #include "qapi-visit.h"
 #include "qapi/opts-visitor.h"
-#include "qapi/dealloc-visitor.h"
-#include "qapi/qmp/qerror.h"
 #include "hw/boards.h"
 #include "sysemu/hostmem.h"
 #include "qmp-commands.h"
 #include "hw/mem/pc-dimm.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/error-messages.h"
-#endif  // CONFIG_ANDROID
+#include "qemu/option.h"
+#include "qemu/config-file.h"
 
 QemuOptsList qemu_numa_opts = {
     .name = "numa",
@@ -49,6 +46,98 @@
 };
 
 static int have_memdevs = -1;
+static int max_numa_nodeid; /* Highest specified NUMA node ID, plus one.
+                             * For all nodes, nodeid < max_numa_nodeid
+                             */
+int nb_numa_nodes;
+NodeInfo numa_info[MAX_NODES];
+
+void numa_set_mem_node_id(ram_addr_t addr, uint64_t size, uint32_t node)
+{
+    struct numa_addr_range *range;
+
+    /*
+     * Memory-less nodes can come here with 0 size in which case,
+     * there is nothing to do.
+     */
+    if (!size) {
+        return;
+    }
+
+    range = g_malloc0(sizeof(*range));
+    range->mem_start = addr;
+    range->mem_end = addr + size - 1;
+    QLIST_INSERT_HEAD(&numa_info[node].addr, range, entry);
+}
+
+void numa_unset_mem_node_id(ram_addr_t addr, uint64_t size, uint32_t node)
+{
+    struct numa_addr_range *range, *next;
+
+    QLIST_FOREACH_SAFE(range, &numa_info[node].addr, entry, next) {
+        if (addr == range->mem_start && (addr + size - 1) == range->mem_end) {
+            QLIST_REMOVE(range, entry);
+            g_free(range);
+            return;
+        }
+    }
+}
+
+static void numa_set_mem_ranges(void)
+{
+    int i;
+    ram_addr_t mem_start = 0;
+
+    /*
+     * Deduce start address of each node and use it to store
+     * the address range info in numa_info address range list
+     */
+    for (i = 0; i < nb_numa_nodes; i++) {
+        numa_set_mem_node_id(mem_start, numa_info[i].node_mem, i);
+        mem_start += numa_info[i].node_mem;
+    }
+}
+
+/*
+ * Check if @addr falls under NUMA @node.
+ */
+static bool numa_addr_belongs_to_node(ram_addr_t addr, uint32_t node)
+{
+    struct numa_addr_range *range;
+
+    QLIST_FOREACH(range, &numa_info[node].addr, entry) {
+        if (addr >= range->mem_start && addr <= range->mem_end) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/*
+ * Given an address, return the index of the NUMA node to which the
+ * address belongs to.
+ */
+uint32_t numa_get_node(ram_addr_t addr, Error **errp)
+{
+    uint32_t i;
+
+    /* For non NUMA configurations, check if the addr falls under node 0 */
+    if (!nb_numa_nodes) {
+        if (numa_addr_belongs_to_node(addr, 0)) {
+            return 0;
+        }
+    }
+
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (numa_addr_belongs_to_node(addr, i)) {
+            return i;
+        }
+    }
+
+    error_setg(errp, "Address 0x" RAM_ADDR_FMT " doesn't belong to any "
+                "NUMA node", addr);
+    return -1;
+}
 
 static void numa_node_parse(NumaNodeOptions *node, QemuOpts *opts, Error **errp)
 {
@@ -63,7 +152,7 @@
 
     if (nodenr >= MAX_NODES) {
         error_setg(errp, "Max number of NUMA nodes reached: %"
-                   PRIu16 "\n", nodenr);
+                   PRIu16 "", nodenr);
         return;
     }
 
@@ -73,16 +162,18 @@
     }
 
     for (cpus = node->cpus; cpus; cpus = cpus->next) {
-        if (cpus->value > MAX_CPUMASK_BITS) {
-            error_setg(errp, "CPU number %" PRIu16 " is bigger than %d",
-                       cpus->value, MAX_CPUMASK_BITS);
+        if (cpus->value >= max_cpus) {
+            error_setg(errp,
+                       "CPU index (%" PRIu16 ")"
+                       " should be smaller than maxcpus (%d)",
+                       cpus->value, max_cpus);
             return;
         }
         bitmap_set(numa_info[nodenr].node_cpu, cpus->value, 1);
     }
 
     if (node->has_mem && node->has_memdev) {
-        error_setg(errp, "qemu: cannot specify both mem= and memdev=\n");
+        error_setg(errp, "qemu: cannot specify both mem= and memdev=");
         return;
     }
 
@@ -91,7 +182,7 @@
     }
     if (node->has_memdev != have_memdevs) {
         error_setg(errp, "qemu: memdev option must be specified for either "
-                   "all or no nodes\n");
+                   "all or no nodes");
         return;
     }
 
@@ -120,26 +211,26 @@
     max_numa_nodeid = MAX(max_numa_nodeid, nodenr + 1);
 }
 
-int numa_init_func(QemuOpts *opts, void *opaque)
+static int parse_numa(void *opaque, QemuOpts *opts, Error **errp)
 {
     NumaOptions *object = NULL;
     Error *err = NULL;
 
     {
-        OptsVisitor *ov = opts_visitor_new(opts);
-        visit_type_NumaOptions(opts_get_visitor(ov), &object, NULL, &err);
-        opts_visitor_cleanup(ov);
+        Visitor *v = opts_visitor_new(opts);
+        visit_type_NumaOptions(v, NULL, &object, &err);
+        visit_free(v);
     }
 
     if (err) {
-        goto error;
+        goto end;
     }
 
-    switch (object->kind) {
+    switch (object->type) {
     case NUMA_OPTIONS_KIND_NODE:
-        numa_node_parse(object->node, opts, &err);
+        numa_node_parse(object->u.node.data, opts, &err);
         if (err) {
-            goto error;
+            goto end;
         }
         nb_numa_nodes++;
         break;
@@ -147,25 +238,68 @@
         abort();
     }
 
-    return 0;
-
-error:
-    qerror_report_err(err);
-    error_free(err);
-
-    if (object) {
-        QapiDeallocVisitor *dv = qapi_dealloc_visitor_new();
-        visit_type_NumaOptions(qapi_dealloc_get_visitor(dv),
-                               &object, NULL, NULL);
-        qapi_dealloc_visitor_cleanup(dv);
+end:
+    qapi_free_NumaOptions(object);
+    if (err) {
+        error_report_err(err);
+        return -1;
     }
 
-    return -1;
+    return 0;
 }
 
-void set_numa_nodes(void)
+static char *enumerate_cpus(unsigned long *cpus, int max_cpus)
+{
+    int cpu;
+    bool first = true;
+    GString *s = g_string_new(NULL);
+
+    for (cpu = find_first_bit(cpus, max_cpus);
+        cpu < max_cpus;
+        cpu = find_next_bit(cpus, max_cpus, cpu + 1)) {
+        g_string_append_printf(s, "%s%d", first ? "" : " ", cpu);
+        first = false;
+    }
+    return g_string_free(s, FALSE);
+}
+
+static void validate_numa_cpus(void)
 {
     int i;
+    DECLARE_BITMAP(seen_cpus, MAX_CPUMASK_BITS);
+
+    bitmap_zero(seen_cpus, MAX_CPUMASK_BITS);
+    for (i = 0; i < nb_numa_nodes; i++) {
+        if (bitmap_intersects(seen_cpus, numa_info[i].node_cpu,
+                              MAX_CPUMASK_BITS)) {
+            bitmap_and(seen_cpus, seen_cpus,
+                       numa_info[i].node_cpu, MAX_CPUMASK_BITS);
+            error_report("CPU(s) present in multiple NUMA nodes: %s",
+                         enumerate_cpus(seen_cpus, max_cpus));
+            exit(EXIT_FAILURE);
+        }
+        bitmap_or(seen_cpus, seen_cpus,
+                  numa_info[i].node_cpu, MAX_CPUMASK_BITS);
+    }
+
+    if (!bitmap_full(seen_cpus, max_cpus)) {
+        char *msg;
+        bitmap_complement(seen_cpus, seen_cpus, max_cpus);
+        msg = enumerate_cpus(seen_cpus, max_cpus);
+        error_report("warning: CPU(s) not present in any NUMA nodes: %s", msg);
+        error_report("warning: All CPU(s) up to maxcpus should be described "
+                     "in NUMA config");
+        g_free(msg);
+    }
+}
+
+void parse_numa_opts(MachineClass *mc)
+{
+    int i;
+
+    if (qemu_opts_foreach(qemu_find_opts("numa"), parse_numa, NULL, NULL)) {
+        exit(1);
+    }
 
     assert(max_numa_nodeid <= MAX_NODES);
 
@@ -222,23 +356,41 @@
         }
 
         for (i = 0; i < nb_numa_nodes; i++) {
+            QLIST_INIT(&numa_info[i].addr);
+        }
+
+        numa_set_mem_ranges();
+
+        for (i = 0; i < nb_numa_nodes; i++) {
             if (!bitmap_empty(numa_info[i].node_cpu, MAX_CPUMASK_BITS)) {
                 break;
             }
         }
-        /* assigning the VCPUs round-robin is easier to implement, guest OSes
-         * must cope with this anyway, because there are BIOSes out there in
-         * real machines which also use this scheme.
+        /* Historically VCPUs were assigned in round-robin order to NUMA
+         * nodes. However it causes issues with guest not handling it nice
+         * in case where cores/threads from a multicore CPU appear on
+         * different nodes. So allow boards to override default distribution
+         * rule grouping VCPUs by socket so that VCPUs from the same socket
+         * would be on the same node.
          */
         if (i == nb_numa_nodes) {
             for (i = 0; i < max_cpus; i++) {
-                set_bit(i, numa_info[i % nb_numa_nodes].node_cpu);
+                unsigned node_id = i % nb_numa_nodes;
+                if (mc->cpu_index_to_socket_id) {
+                    node_id = mc->cpu_index_to_socket_id(i) % nb_numa_nodes;
+                }
+
+                set_bit(i, numa_info[node_id].node_cpu);
             }
         }
+
+        validate_numa_cpus();
+    } else {
+        numa_set_mem_node_id(0, ram_size, 0);
     }
 }
 
-void set_numa_modes(void)
+void numa_post_machine_init(void)
 {
     CPUState *cpu;
     int i;
@@ -261,29 +413,25 @@
         Error *err = NULL;
         memory_region_init_ram_from_file(mr, owner, name, ram_size, false,
                                          mem_path, &err);
-
-        /* Legacy behavior: if allocation failed, fall back to
-         * regular RAM allocation.
-         */
         if (err) {
-            qerror_report_err(err);
-            error_free(err);
-            memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
+            error_report_err(err);
+            if (mem_prealloc) {
+                exit(1);
+            }
+
+            /* Legacy behavior: if allocation failed, fall back to
+             * regular RAM allocation.
+             */
+            memory_region_init_ram(mr, owner, name, ram_size, &error_fatal);
         }
 #else
         fprintf(stderr, "-mem-path not supported on this host\n");
         exit(1);
 #endif
     } else {
-#ifdef CONFIG_ANDROID
-        memory_region_init_ram(mr, owner, name, ram_size,
-                               &android_init_error_with_message);
-        if (android_init_error_occurred()) {
+        memory_region_init_ram(mr, owner, name, ram_size, &error_during_init);
+        if (error_during_init != NULL)
             return;
-        }
-#else  // !CONFIG_ANDROID
-        memory_region_init_ram(mr, owner, name, ram_size, &error_abort);
-#endif  // CONFIG_ANDROID
     }
     vmstate_register_ram_global(mr);
 }
@@ -302,17 +450,13 @@
 
     memory_region_init(mr, owner, name, ram_size);
     for (i = 0; i < MAX_NODES; i++) {
-        Error *local_err = NULL;
         uint64_t size = numa_info[i].node_mem;
         HostMemoryBackend *backend = numa_info[i].node_memdev;
         if (!backend) {
             continue;
         }
-        MemoryRegion *seg = host_memory_backend_get_memory(backend, &local_err);
-        if (local_err) {
-            qerror_report_err(local_err);
-            exit(1);
-        }
+        MemoryRegion *seg = host_memory_backend_get_memory(backend,
+                                                           &error_fatal);
 
         if (memory_region_is_mapped(seg)) {
             char *path = object_get_canonical_path_component(OBJECT(backend));
@@ -322,6 +466,7 @@
             exit(1);
         }
 
+        host_memory_backend_set_mapped(backend, true);
         memory_region_add_subregion(mr, addr, seg);
         vmstate_register_ram_global(seg);
         addr += size;
@@ -339,9 +484,9 @@
         MemoryDeviceInfo *value = info->value;
 
         if (value) {
-            switch (value->kind) {
+            switch (value->type) {
             case MEMORY_DEVICE_INFO_KIND_DIMM:
-                node_mem[value->dimm->node] += value->dimm->size;
+                node_mem[value->u.dimm.data->node] += value->u.dimm.data->size;
                 break;
             default:
                 break;
@@ -369,7 +514,6 @@
 {
     MemdevList **list = opaque;
     MemdevList *m = NULL;
-    Error *err = NULL;
 
     if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
         m = g_malloc0(sizeof(*m));
@@ -377,72 +521,34 @@
         m->value = g_malloc0(sizeof(*m->value));
 
         m->value->size = object_property_get_int(obj, "size",
-                                                 &err);
-        if (err) {
-            goto error;
-        }
-
+                                                 &error_abort);
         m->value->merge = object_property_get_bool(obj, "merge",
-                                                   &err);
-        if (err) {
-            goto error;
-        }
-
+                                                   &error_abort);
         m->value->dump = object_property_get_bool(obj, "dump",
-                                                  &err);
-        if (err) {
-            goto error;
-        }
-
+                                                  &error_abort);
         m->value->prealloc = object_property_get_bool(obj,
-                                                      "prealloc", &err);
-        if (err) {
-            goto error;
-        }
-
+                                                      "prealloc",
+                                                      &error_abort);
         m->value->policy = object_property_get_enum(obj,
                                                     "policy",
-                                                    HostMemPolicy_lookup,
-                                                    &err);
-        if (err) {
-            goto error;
-        }
-
+                                                    "HostMemPolicy",
+                                                    &error_abort);
         object_property_get_uint16List(obj, "host-nodes",
-                                       &m->value->host_nodes, &err);
-        if (err) {
-            goto error;
-        }
+                                       &m->value->host_nodes,
+                                       &error_abort);
 
         m->next = *list;
         *list = m;
     }
 
     return 0;
-error:
-    g_free(m->value);
-    g_free(m);
-
-    return -1;
 }
 
 MemdevList *qmp_query_memdev(Error **errp)
 {
-    Object *obj;
+    Object *obj = object_get_objects_root();
     MemdevList *list = NULL;
 
-    obj = object_resolve_path("/objects", NULL);
-    if (obj == NULL) {
-        return NULL;
-    }
-
-    if (object_child_foreach(obj, query_memdev, &list) != 0) {
-        goto error;
-    }
-
+    object_child_foreach(obj, query_memdev, &list);
     return list;
-
-error:
-    qapi_free_MemdevList(list);
-    return NULL;
 }
diff --git a/os-posix.c b/os-posix.c
index 1e9a690..a2a8e31 100644
--- a/os-posix.c
+++ b/os-posix.c
@@ -23,26 +23,24 @@
  * THE SOFTWARE.
  */
 
-#include <unistd.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <sys/types.h>
+/* Special case to include "qemu-options.h" here without issues */
+#undef POISON_CONFIG_ANDROID
+
+#include "qemu/osdep.h"
 #include <sys/wait.h>
 /*needed for MAP_POPULATE before including qemu-options.h */
-#include <sys/mman.h>
 #include <pwd.h>
 #include <grp.h>
 #include <libgen.h>
 
 /* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "net/slirp.h"
 #include "qemu-options.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/skin/winsys.h"
-#endif
+#include "qemu/rcu.h"
+#include "qemu/error-report.h"
+#include "qemu/log.h"
+#include "qemu/cutils.h"
 
 #ifdef CONFIG_LINUX
 #include <sys/prctl.h>
@@ -62,16 +60,19 @@
     sigaction(SIGPIPE, &act, NULL);
 }
 
+static void (*qemu_ctrlc_handler)(void) = NULL;
+
+void qemu_set_ctrlc_handler(void(*handler)(void)) {
+    qemu_ctrlc_handler = handler;
+}
+
 static void termsig_handler(int signal, siginfo_t *info, void *c)
 {
-#ifdef CONFIG_ANDROID
-    // In android, request closing the UI, instead of short-circuting down to
-    // qemu. This will eventually call qemu_system_shutdown_request via a skin
-    // event.
-    skin_winsys_quit_request();
-#else
-    qemu_system_killed(info->si_signo, info->si_pid);
-#endif  // !CONFIG_ANDROID
+    if (qemu_ctrlc_handler) {
+        qemu_ctrlc_handler();
+    } else {
+        qemu_system_killed(info->si_signo, info->si_pid);
+    }
 }
 
 void os_setup_signal_handling(void)
@@ -101,7 +102,7 @@
     if (exec_dir == NULL) {
         return NULL;
     }
-    dir = dirname(exec_dir);
+    dir = g_path_get_dirname(exec_dir);
 
     max_len = strlen(dir) +
         MAX(strlen(SHARE_SUFFIX), strlen(BUILD_SUFFIX)) + 1;
@@ -115,6 +116,7 @@
         }
     }
 
+    g_free(dir);
     g_free(exec_dir);
     return res;
 }
@@ -149,6 +151,8 @@
     switch (index) {
 #ifdef CONFIG_SLIRP
     case QEMU_OPTION_smb:
+        error_report("The -smb option is deprecated. "
+                     "Please use '-netdev user,smb=...' instead.");
         if (net_slirp_smb(optarg) < 0)
             exit(1);
         break;
@@ -258,6 +262,7 @@
         signal(SIGTSTP, SIG_IGN);
         signal(SIGTTOU, SIG_IGN);
         signal(SIGTTIN, SIG_IGN);
+        rcu_after_fork();
     }
 }
 
@@ -285,7 +290,10 @@
 
         dup2(fd, 0);
         dup2(fd, 1);
-        dup2(fd, 2);
+        /* In case -D is given do not redirect stderr to /dev/null */
+        if (!qemu_logfile) {
+            dup2(fd, 2);
+        }
 
         close(fd);
 
diff --git a/os-win32.c b/os-win32.c
index bdbdee1..41c41e6 100644
--- a/os-win32.c
+++ b/os-win32.c
@@ -22,22 +22,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+/* Special case to include "qemu-options.h" here without issues */
+#undef POISON_CONFIG_ANDROID
+
+#include "qemu/osdep.h"
 #include <windows.h>
 #include <mmsystem.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
-#include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "qemu-options.h"
 
-#ifdef CONFIG_ANDROID
-#include "android/skin/winsys.h"
-#endif
-
 /***********************************************************/
 /* Functions missing in mingw */
 
@@ -60,27 +53,30 @@
     return result;
 }
 
+static void (*ctrlc_handler)(void) = NULL;
+
+void qemu_set_ctrlc_handler(void(*handler)(void)) {
+    ctrlc_handler = handler;
+}
+
 static BOOL WINAPI qemu_ctrl_handler(DWORD type)
 {
-#ifdef CONFIG_ANDROID
-    // In android, request closing the UI, instead of short-circuting down to
-    // qemu. This will eventually call qemu_system_shutdown_request via a skin
-    // event.
-    skin_winsys_quit_request();
-#else
-    qemu_system_shutdown_request();
-#endif  // !CONFIG_ANDROID
+    if (ctrlc_handler) {
+        (*ctrlc_handler)();
+    } else {
+        qemu_system_shutdown_request();
+    }
     fflush(stdout);
     fflush(stderr);
+
     /* Windows 7 kills application when the function returns.
        Sleep here to give QEMU a try for closing.
        Sleep period is 10000ms because Windows kills the program
        after 10 seconds anyway. */
     Sleep(10000);
 
-    // Sometimes (?) it happens that Windows doesn't actually kill the emulator
-    // after 10 seconds. Let's return FALSE so the default handler finishes the
-    // process anyway.
+    /* Sometimes Windows doesn't actually kill QEMU after 10 seconds.
+     * Return FALSE so the default handler finished the process anyway */
     return FALSE;
 }
 
@@ -129,7 +125,7 @@
     BOOL ret;
     memset(&overlap, 0, sizeof(overlap));
 
-    file = CreateFile(filename, GENERIC_WRITE, FILE_SHARE_READ, NULL,
+    file = win32CreateFile(filename, GENERIC_WRITE, FILE_SHARE_READ, NULL,
                       OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
 
     if (file == INVALID_HANDLE_VALUE) {
@@ -144,28 +140,3 @@
     }
     return 0;
 }
-
-char* strtok_r(char *str,
-               const char *delim,
-               char **nextp)
-{
-    char *ret;
-
-    if (str == NULL) {
-        str = *nextp;
-    }
-
-    str += strspn(str, delim);
-    if (*str == '\0') {
-        return NULL;
-    }
-
-    ret = str;
-    str += strcspn(str, delim);
-    if (*str) {
-        *str++ = '\0';
-    }
-
-    *nextp = str;
-    return ret;
-}
diff --git a/page_cache.c b/page_cache.c
index 89bb1ec..5f85787 100644
--- a/page_cache.c
+++ b/page_cache.c
@@ -12,17 +12,10 @@
  *
  */
 
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <stdbool.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qemu/host-utils.h"
 #include "migration/page_cache.h"
 
 #ifdef DEBUG_CACHE
@@ -33,6 +26,9 @@
     do { } while (0)
 #endif
 
+/* the page in cache will not be replaced in two cycles */
+#define CACHED_PAGE_LIFETIME 2
+
 typedef struct CacheItem CacheItem;
 
 struct CacheItem {
@@ -115,23 +111,8 @@
 static size_t cache_get_cache_pos(const PageCache *cache,
                                   uint64_t address)
 {
-    size_t pos;
-
     g_assert(cache->max_num_items);
-    pos = (address / cache->page_size) & (cache->max_num_items - 1);
-    return pos;
-}
-
-bool cache_is_cached(const PageCache *cache, uint64_t addr)
-{
-    size_t pos;
-
-    g_assert(cache);
-    g_assert(cache->page_cache);
-
-    pos = cache_get_cache_pos(cache, addr);
-
-    return (cache->page_cache[pos].it_addr == addr);
+    return (address / cache->page_size) & (cache->max_num_items - 1);
 }
 
 static CacheItem *cache_get_by_addr(const PageCache *cache, uint64_t addr)
@@ -151,17 +132,35 @@
     return cache_get_by_addr(cache, addr)->it_data;
 }
 
-int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata)
+bool cache_is_cached(const PageCache *cache, uint64_t addr,
+                     uint64_t current_age)
+{
+    CacheItem *it;
+
+    it = cache_get_by_addr(cache, addr);
+
+    if (it->it_addr == addr) {
+        /* update the it_age when the cache hit */
+        it->it_age = current_age;
+        return true;
+    }
+    return false;
+}
+
+int cache_insert(PageCache *cache, uint64_t addr, const uint8_t *pdata,
+                 uint64_t current_age)
 {
 
-    CacheItem *it = NULL;
-
-    g_assert(cache);
-    g_assert(cache->page_cache);
+    CacheItem *it;
 
     /* actual update of entry */
     it = cache_get_by_addr(cache, addr);
 
+    if (it->it_data && it->it_addr != addr &&
+        it->it_age + CACHED_PAGE_LIFETIME > current_age) {
+        /* the cache page is fresh, don't replace it */
+        return -1;
+    }
     /* allocate page */
     if (!it->it_data) {
         it->it_data = g_try_malloc(cache->page_size);
@@ -174,7 +173,7 @@
 
     memcpy(it->it_data, pdata, cache->page_size);
 
-    it->it_age = ++cache->max_item_age;
+    it->it_age = current_age;
     it->it_addr = addr;
 
     return 0;
diff --git a/pc-bios/README b/pc-bios/README
index edfadd7..5a8a931 100644
--- a/pc-bios/README
+++ b/pc-bios/README
@@ -17,7 +17,7 @@
 - SLOF (Slimline Open Firmware) is a free IEEE 1275 Open Firmware
   implementation for certain IBM POWER hardware.  The sources are at
   https://github.com/aik/SLOF, and the image currently in qemu is
-  built from git tag qemu-slof-20140630.
+  built from git tag qemu-slof-20160223.
 
 - sgabios (the Serial Graphics Adapter option ROM) provides a means for
   legacy x86 software to communicate with an attached serial console as
@@ -35,10 +35,6 @@
 	10ec:8139 -> pxe-rtl8139.rom
 	1af4:1000 -> pxe-virtio.rom
 
-- The S390 zipl loader is an addition to the official IBM s390-tools
-  package. That fork is maintained in its own git repository at:
-  git://repo.or.cz/s390-tools.git
-
 - The sources for the Alpha palcode image is available from:
   git://github.com/rth7680/qemu-palcode.git
 
diff --git a/pc-bios/bios-256k.bin b/pc-bios/bios-256k.bin
index fab9da2..57fb4d8 100644
--- a/pc-bios/bios-256k.bin
+++ b/pc-bios/bios-256k.bin
Binary files differ
diff --git a/pc-bios/bios.bin b/pc-bios/bios.bin
index 8c718e1..8a6869f 100644
--- a/pc-bios/bios.bin
+++ b/pc-bios/bios.bin
Binary files differ
diff --git a/pc-bios/efi-e1000.rom b/pc-bios/efi-e1000.rom
index 776e217..4e61f9b 100644
--- a/pc-bios/efi-e1000.rom
+++ b/pc-bios/efi-e1000.rom
Binary files differ
diff --git a/pc-bios/efi-e1000e.rom b/pc-bios/efi-e1000e.rom
new file mode 100644
index 0000000..192a437
--- /dev/null
+++ b/pc-bios/efi-e1000e.rom
Binary files differ
diff --git a/pc-bios/efi-eepro100.rom b/pc-bios/efi-eepro100.rom
index 677a8c3..66c5226 100644
--- a/pc-bios/efi-eepro100.rom
+++ b/pc-bios/efi-eepro100.rom
Binary files differ
diff --git a/pc-bios/efi-ne2k_pci.rom b/pc-bios/efi-ne2k_pci.rom
index 9dd6d91..8c3e5fd 100644
--- a/pc-bios/efi-ne2k_pci.rom
+++ b/pc-bios/efi-ne2k_pci.rom
Binary files differ
diff --git a/pc-bios/efi-pcnet.rom b/pc-bios/efi-pcnet.rom
index cae3a85..802e225 100644
--- a/pc-bios/efi-pcnet.rom
+++ b/pc-bios/efi-pcnet.rom
Binary files differ
diff --git a/pc-bios/efi-rtl8139.rom b/pc-bios/efi-rtl8139.rom
index 477f9b9..8827181 100644
--- a/pc-bios/efi-rtl8139.rom
+++ b/pc-bios/efi-rtl8139.rom
Binary files differ
diff --git a/pc-bios/efi-virtio.rom b/pc-bios/efi-virtio.rom
index 935c927..2fc0497 100644
--- a/pc-bios/efi-virtio.rom
+++ b/pc-bios/efi-virtio.rom
Binary files differ
diff --git a/pc-bios/efi-vmxnet3.rom b/pc-bios/efi-vmxnet3.rom
new file mode 100644
index 0000000..3d42635
--- /dev/null
+++ b/pc-bios/efi-vmxnet3.rom
Binary files differ
diff --git a/pc-bios/keymaps/ru b/pc-bios/keymaps/ru
index b3e7d24..8f652d5 100644
--- a/pc-bios/keymaps/ru
+++ b/pc-bios/keymaps/ru
@@ -4,7 +4,7 @@
 exclam 0x02 shift
 at 0x03 shift
 quotedbl 0x03 shift altgr
-numbersign 0x04 shift
+numerosign 0x04 shift
 dollar 0x05 shift
 asterisk 0x05 shift altgr
 percent 0x06 shift
diff --git a/pc-bios/linuxboot.bin b/pc-bios/linuxboot.bin
index 130103f..923d179 100644
--- a/pc-bios/linuxboot.bin
+++ b/pc-bios/linuxboot.bin
Binary files differ
diff --git a/pc-bios/linuxboot_dma.bin b/pc-bios/linuxboot_dma.bin
new file mode 100644
index 0000000..238a195
--- /dev/null
+++ b/pc-bios/linuxboot_dma.bin
Binary files differ
diff --git a/pc-bios/openbios-ppc b/pc-bios/openbios-ppc
index 994052f..d913fd0 100644
--- a/pc-bios/openbios-ppc
+++ b/pc-bios/openbios-ppc
Binary files differ
diff --git a/pc-bios/openbios-sparc32 b/pc-bios/openbios-sparc32
index 6d5a381..c8c6fcb 100644
--- a/pc-bios/openbios-sparc32
+++ b/pc-bios/openbios-sparc32
Binary files differ
diff --git a/pc-bios/openbios-sparc64 b/pc-bios/openbios-sparc64
index 61bd46b..73f03a0 100644
--- a/pc-bios/openbios-sparc64
+++ b/pc-bios/openbios-sparc64
Binary files differ
diff --git a/pc-bios/optionrom/Makefile b/pc-bios/optionrom/Makefile
index ce4852a..afa48f1 100644
--- a/pc-bios/optionrom/Makefile
+++ b/pc-bios/optionrom/Makefile
@@ -9,19 +9,44 @@
 
 .PHONY : all clean build-all
 
-CFLAGS := -Wall -Wstrict-prototypes -Werror -fomit-frame-pointer -fno-builtin
-CFLAGS += -I$(SRC_PATH)
-CFLAGS += $(call cc-option, $(CFLAGS), -fno-stack-protector)
-CFLAGS += $(CFLAGS_NOPIE)
-QEMU_CFLAGS = $(CFLAGS)
+# Compiling with no optimization creates ROMs that are too large
+ifeq ($(lastword $(filter -O%, -O0 $(CFLAGS))),-O0)
+override CFLAGS += -O2
+endif
 
-build-all: multiboot.bin linuxboot.bin kvmvapic.bin
+# Drop -fstack-protector and the like
+QEMU_CFLAGS := $(filter -W%, $(QEMU_CFLAGS)) $(CFLAGS_NOPIE) -ffreestanding
+QEMU_CFLAGS += $(call cc-option, $(QEMU_CFLAGS), -fno-stack-protector)
+QEMU_CFLAGS += $(call cc-option, $(QEMU_CFLAGS), -m16)
+ifeq ($(filter -m16, $(QEMU_CFLAGS)),)
+# Attempt to work around compilers that lack -m16 (GCC <= 4.8, clang <= ??)
+# On GCC we add -fno-toplevel-reorder to keep the order of asm blocks with
+# respect to the rest of the code.  clang does not have -fno-toplevel-reorder,
+# but it places all asm blocks at the beginning and we're relying on it for
+# the option ROM header.  So just force clang not to use the integrated
+# assembler, which doesn't support .code16gcc.
+QEMU_CFLAGS += $(call cc-option, $(QEMU_CFLAGS), -fno-toplevel-reorder)
+QEMU_CFLAGS += $(call cc-option, $(QEMU_CFLAGS), -no-integrated-as)
+QEMU_CFLAGS += -m32 -include $(SRC_PATH)/pc-bios/optionrom/code16gcc.h
+endif
+
+QEMU_INCLUDES += -I$(SRC_PATH)
+
+Wa = -Wa,
+ASFLAGS += -32
+QEMU_CFLAGS += $(call cc-c-option, $(QEMU_CFLAGS), $(Wa)-32)
+
+build-all: multiboot.bin linuxboot.bin linuxboot_dma.bin kvmvapic.bin
 
 # suppress auto-removal of intermediate files
 .SECONDARY:
 
+
+%.o: %.S
+	$(call quiet-command,$(CPP) $(QEMU_INCLUDES) $(QEMU_DGFLAGS) -c -o - $< | $(AS) $(ASFLAGS) -o $@,"  AS    $(TARGET_DIR)$@")
+
 %.img: %.o
-	$(call quiet-command,$(LD) $(LDFLAGS_NOPIE) -Ttext 0 -e _start -s -o $@ $<,"  Building $(TARGET_DIR)$@")
+	$(call quiet-command,$(LD) $(LDFLAGS_NOPIE) -m $(LD_I386_EMULATION) -T $(SRC_PATH)/pc-bios/optionrom/flat.lds -s -o $@ $<,"  Building $(TARGET_DIR)$@")
 
 %.raw: %.img
 	$(call quiet-command,$(OBJCOPY) -O binary -j .text $< $@,"  Building $(TARGET_DIR)$@")
diff --git a/pc-bios/optionrom/code16gcc.h b/pc-bios/optionrom/code16gcc.h
new file mode 100644
index 0000000..9c8d25d
--- /dev/null
+++ b/pc-bios/optionrom/code16gcc.h
@@ -0,0 +1,3 @@
+asm(
+".code16gcc\n"
+);
diff --git a/pc-bios/optionrom/flat.lds b/pc-bios/optionrom/flat.lds
new file mode 100644
index 0000000..cee2eda
--- /dev/null
+++ b/pc-bios/optionrom/flat.lds
@@ -0,0 +1,6 @@
+SECTIONS
+{
+  . = 0;
+  .text : { *(.text) *(.text.$) }
+}
+ENTRY(_start)
diff --git a/pc-bios/optionrom/linuxboot.S b/pc-bios/optionrom/linuxboot.S
index 5bc0af0..ba821ab 100644
--- a/pc-bios/optionrom/linuxboot.S
+++ b/pc-bios/optionrom/linuxboot.S
@@ -76,7 +76,31 @@
 
 
 copy_kernel:
-	/* Compute initrd address */
+	/* Read info block in low memory (0x10000 or 0x90000) */
+	read_fw		FW_CFG_SETUP_ADDR
+	shr		$4, %eax
+	mov		%eax, %es
+	xor		%edi, %edi
+	read_fw_blob_addr32_edi(FW_CFG_SETUP)
+
+	cmpw            $0x203, %es:0x206      // if protocol >= 0x203
+	jae             1f                     // have initrd_max
+	movl            $0x37ffffff, %es:0x22c // else assume 0x37ffffff
+1:
+
+	/* Check if using kernel-specified initrd address */
+	read_fw		FW_CFG_INITRD_ADDR
+	mov		%eax, %edi             // (load_kernel wants it in %edi)
+	read_fw		FW_CFG_INITRD_SIZE     // find end of initrd
+	add		%edi, %eax
+	xor		%es:0x22c, %eax        // if it matches es:0x22c
+	and		$-4096, %eax           // (apart from padding for page)
+	jz		load_kernel            // then initrd is not at top
+					       // of memory
+
+	/* pc.c placed the initrd at end of memory.  Compute a better
+	 * initrd address based on e801 data.
+	 */
 	mov		$0xe801, %ax
 	xor		%cx, %cx
 	xor		%dx, %dx
@@ -107,7 +131,9 @@
 	read_fw         FW_CFG_INITRD_SIZE
 	subl            %eax, %edi
 	andl            $-4096, %edi          /* EDI = start of initrd */
+	movl		%edi, %es:0x218       /* put it in the header */
 
+load_kernel:
 	/* We need to load the kernel into memory we can't access in 16 bit
 	   mode, so let's get into 32 bit mode, write the kernel and jump
 	   back again. */
@@ -139,19 +165,10 @@
 	/* We're now running in 16-bit CS, but 32-bit ES! */
 
 	/* Load kernel and initrd */
-	pushl		%edi
 	read_fw_blob_addr32_edi(FW_CFG_INITRD)
 	read_fw_blob_addr32(FW_CFG_KERNEL)
 	read_fw_blob_addr32(FW_CFG_CMDLINE)
 
-	read_fw		FW_CFG_SETUP_ADDR
-	mov		%eax, %edi
-	mov		%eax, %ebx
-	read_fw_blob_addr32_edi(FW_CFG_SETUP)
-
-	/* Update the header with the initrd address we chose above */
-	popl		%es:0x218(%ebx)
-
 	/* And now jump into Linux! */
 	mov		$0, %eax
 	mov		%eax, %cr0
diff --git a/pc-bios/optionrom/linuxboot_dma.c b/pc-bios/optionrom/linuxboot_dma.c
new file mode 100644
index 0000000..7549797
--- /dev/null
+++ b/pc-bios/optionrom/linuxboot_dma.c
@@ -0,0 +1,298 @@
+/*
+ * Linux Boot Option ROM for fw_cfg DMA
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * Copyright (c) 2015-2016 Red Hat Inc.
+ *   Authors:
+ *     Marc Marí <marc.mari.barcelo@gmail.com>
+ *     Richard W.M. Jones <rjones@redhat.com>
+ */
+
+asm(
+".text\n"
+".global _start\n"
+"_start:\n"
+"   .short 0xaa55\n"
+"   .byte 3\n" /* desired size in 512 units; signrom.py adds padding */
+"   .byte 0xcb\n" /* far return without prefix */
+"   .org 0x18\n"
+"   .short 0\n"
+"   .short _pnph\n"
+"_pnph:\n"
+"   .ascii \"$PnP\"\n"
+"   .byte 0x01\n"
+"   .byte (_pnph_len / 16)\n"
+"   .short 0x0000\n"
+"   .byte 0x00\n"
+"   .byte 0x00\n"
+"   .long 0x00000000\n"
+"   .short _manufacturer\n"
+"   .short _product\n"
+"   .long 0x00000000\n"
+"   .short 0x0000\n"
+"   .short 0x0000\n"
+"   .short _bev\n"
+"   .short 0x0000\n"
+"   .short 0x0000\n"
+"   .equ _pnph_len, . - _pnph\n"
+"_manufacturer:\n"
+"   .asciz \"QEMU\"\n"
+"_product:\n"
+"   .asciz \"Linux loader DMA\"\n"
+"   .align 4, 0\n"
+"_bev:\n"
+"   cli\n"
+"   cld\n"
+"   jmp load_kernel\n"
+);
+
+#include "../../include/hw/nvram/fw_cfg_keys.h"
+
+/* QEMU_CFG_DMA_CONTROL bits */
+#define BIOS_CFG_DMA_CTL_ERROR   0x01
+#define BIOS_CFG_DMA_CTL_READ    0x02
+#define BIOS_CFG_DMA_CTL_SKIP    0x04
+#define BIOS_CFG_DMA_CTL_SELECT  0x08
+
+#define BIOS_CFG_DMA_ADDR_HIGH 0x514
+#define BIOS_CFG_DMA_ADDR_LOW  0x518
+
+#define uint64_t unsigned long long
+#define uint32_t unsigned int
+#define uint16_t unsigned short
+
+#define barrier() asm("" : : : "memory")
+
+typedef struct FWCfgDmaAccess {
+    uint32_t control;
+    uint32_t length;
+    uint64_t address;
+} __attribute__((packed)) FWCfgDmaAccess;
+
+static inline void outl(uint32_t value, uint16_t port)
+{
+    asm("outl %0, %w1" : : "a"(value), "Nd"(port));
+}
+
+static inline void set_es(void *addr)
+{
+    uint32_t seg = (uint32_t)addr >> 4;
+    asm("movl %0, %%es" : : "r"(seg));
+}
+
+#ifdef __clang__
+#define ADDR32
+#else
+#define ADDR32 "addr32 "
+#endif
+
+static inline uint16_t readw_es(uint16_t offset)
+{
+    uint16_t val;
+    asm(ADDR32 "movw %%es:(%1), %0" : "=r"(val) : "r"((uint32_t)offset));
+    barrier();
+    return val;
+}
+
+static inline uint32_t readl_es(uint16_t offset)
+{
+    uint32_t val;
+    asm(ADDR32 "movl %%es:(%1), %0" : "=r"(val) : "r"((uint32_t)offset));
+    barrier();
+    return val;
+}
+
+static inline void writel_es(uint16_t offset, uint32_t val)
+{
+    barrier();
+    asm(ADDR32 "movl %0, %%es:(%1)" : : "r"(val), "r"((uint32_t)offset));
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return
+        ((x & 0x000000ffU) << 24) |
+        ((x & 0x0000ff00U) <<  8) |
+        ((x & 0x00ff0000U) >>  8) |
+        ((x & 0xff000000U) >> 24);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return
+        ((x & 0x00000000000000ffULL) << 56) |
+        ((x & 0x000000000000ff00ULL) << 40) |
+        ((x & 0x0000000000ff0000ULL) << 24) |
+        ((x & 0x00000000ff000000ULL) <<  8) |
+        ((x & 0x000000ff00000000ULL) >>  8) |
+        ((x & 0x0000ff0000000000ULL) >> 24) |
+        ((x & 0x00ff000000000000ULL) >> 40) |
+        ((x & 0xff00000000000000ULL) >> 56);
+}
+
+static inline uint64_t cpu_to_be64(uint64_t x)
+{
+    return bswap64(x);
+}
+
+static inline uint32_t cpu_to_be32(uint32_t x)
+{
+    return bswap32(x);
+}
+
+static inline uint32_t be32_to_cpu(uint32_t x)
+{
+    return bswap32(x);
+}
+
+/* clang is happy to inline this function, and bloats the
+ * ROM.
+ */
+static __attribute__((__noinline__))
+void bios_cfg_read_entry(void *buf, uint16_t entry, uint32_t len)
+{
+    FWCfgDmaAccess access;
+    uint32_t control = (entry << 16) | BIOS_CFG_DMA_CTL_SELECT
+                        | BIOS_CFG_DMA_CTL_READ;
+
+    access.address = cpu_to_be64((uint64_t)(uint32_t)buf);
+    access.length = cpu_to_be32(len);
+    access.control = cpu_to_be32(control);
+
+    barrier();
+
+    outl(cpu_to_be32((uint32_t)&access), BIOS_CFG_DMA_ADDR_LOW);
+
+    while (be32_to_cpu(access.control) & ~BIOS_CFG_DMA_CTL_ERROR) {
+        barrier();
+    }
+}
+
+/* Return top of memory using BIOS function E801. */
+static uint32_t get_e801_addr(void)
+{
+    uint16_t ax, bx, cx, dx;
+    uint32_t ret;
+
+    asm("int $0x15\n"
+        : "=a"(ax), "=b"(bx), "=c"(cx), "=d"(dx)
+        : "a"(0xe801), "b"(0), "c"(0), "d"(0));
+
+    /* Not SeaBIOS, but in theory a BIOS could return CX=DX=0 in which
+     * case we need to use the result from AX & BX instead.
+     */
+    if (cx == 0 && dx == 0) {
+        cx = ax;
+        dx = bx;
+    }
+
+    if (dx) {
+        /* DX = extended memory above 16M, in 64K units.
+         * Convert it to bytes and return.
+         */
+        ret = ((uint32_t)dx + 256 /* 16M in 64K units */) << 16;
+    } else {
+        /* This is a fallback path for machines with <= 16MB of RAM,
+         * which probably would never be the case, but deal with it
+         * anyway.
+         *
+         * CX = extended memory between 1M and 16M, in kilobytes
+         * Convert it to bytes and return.
+         */
+        ret = ((uint32_t)cx + 1024 /* 1M in K */) << 10;
+    }
+
+    return ret;
+}
+
+/* Force the asm name without leading underscore, even on Win32. */
+extern void load_kernel(void) asm("load_kernel");
+
+void load_kernel(void)
+{
+    void *setup_addr;
+    void *initrd_addr;
+    void *kernel_addr;
+    void *cmdline_addr;
+    uint32_t setup_size;
+    uint32_t initrd_size;
+    uint32_t kernel_size;
+    uint32_t cmdline_size;
+    uint32_t initrd_end_page, max_allowed_page;
+    uint32_t segment_addr, stack_addr;
+
+    bios_cfg_read_entry(&setup_addr, FW_CFG_SETUP_ADDR, 4);
+    bios_cfg_read_entry(&setup_size, FW_CFG_SETUP_SIZE, 4);
+    bios_cfg_read_entry(setup_addr, FW_CFG_SETUP_DATA, setup_size);
+
+    set_es(setup_addr);
+
+    /* For protocol < 0x203 we don't have initrd_max ... */
+    if (readw_es(0x206) < 0x203) {
+        /* ... so we assume initrd_max = 0x37ffffff. */
+        writel_es(0x22c, 0x37ffffff);
+    }
+
+    bios_cfg_read_entry(&initrd_addr, FW_CFG_INITRD_ADDR, 4);
+    bios_cfg_read_entry(&initrd_size, FW_CFG_INITRD_SIZE, 4);
+
+    initrd_end_page = ((uint32_t)(initrd_addr + initrd_size) & -4096);
+    max_allowed_page = (readl_es(0x22c) & -4096);
+
+    if (initrd_end_page != 0 && max_allowed_page != 0 &&
+        initrd_end_page != max_allowed_page) {
+        /* Initrd at the end of memory. Compute better initrd address
+         * based on e801 data
+         */
+        initrd_addr = (void *)((get_e801_addr() - initrd_size) & -4096);
+        writel_es(0x218, (uint32_t)initrd_addr);
+
+    }
+
+    bios_cfg_read_entry(initrd_addr, FW_CFG_INITRD_DATA, initrd_size);
+
+    bios_cfg_read_entry(&kernel_addr, FW_CFG_KERNEL_ADDR, 4);
+    bios_cfg_read_entry(&kernel_size, FW_CFG_KERNEL_SIZE, 4);
+    bios_cfg_read_entry(kernel_addr, FW_CFG_KERNEL_DATA, kernel_size);
+
+    bios_cfg_read_entry(&cmdline_addr, FW_CFG_CMDLINE_ADDR, 4);
+    bios_cfg_read_entry(&cmdline_size, FW_CFG_CMDLINE_SIZE, 4);
+    bios_cfg_read_entry(cmdline_addr, FW_CFG_CMDLINE_DATA, cmdline_size);
+
+    /* Boot linux */
+    segment_addr = ((uint32_t)setup_addr >> 4);
+    stack_addr = (uint32_t)(cmdline_addr - setup_addr - 16);
+
+    /* As we are changing critical registers, we cannot leave freedom to the
+     * compiler.
+     */
+    asm("movw %%ax, %%ds\n"
+        "movw %%ax, %%es\n"
+        "movw %%ax, %%fs\n"
+        "movw %%ax, %%gs\n"
+        "movw %%ax, %%ss\n"
+        "movl %%ebx, %%esp\n"
+        "addw $0x20, %%ax\n"
+        "pushw %%ax\n" /* CS */
+        "pushw $0\n" /* IP */
+        /* Clear registers and jump to Linux */
+        "xor %%ebx, %%ebx\n"
+        "xor %%ecx, %%ecx\n"
+        "xor %%edx, %%edx\n"
+        "xor %%edi, %%edi\n"
+        "xor %%ebp, %%ebp\n"
+        "lretw\n"
+        : : "a"(segment_addr), "b"(stack_addr));
+}
diff --git a/pc-bios/optionrom/optionrom.h b/pc-bios/optionrom/optionrom.h
index f1a9021..6c4c2c8 100644
--- a/pc-bios/optionrom/optionrom.h
+++ b/pc-bios/optionrom/optionrom.h
@@ -19,8 +19,7 @@
  */
 
 
-#define NO_QEMU_PROTOS
-#include "../../include/hw/nvram/fw_cfg.h"
+#include "../../include/hw/nvram/fw_cfg_keys.h"
 
 #define BIOS_CFG_IOPORT_CFG	0x510
 #define BIOS_CFG_IOPORT_DATA	0x511
diff --git a/pc-bios/q35-acpi-dsdt.aml b/pc-bios/q35-acpi-dsdt.aml
deleted file mode 100644
index d71b3a3..0000000
--- a/pc-bios/q35-acpi-dsdt.aml
+++ /dev/null
Binary files differ
diff --git a/pc-bios/qemu.rsrc b/pc-bios/qemu.rsrc
new file mode 100644
index 0000000..e3d205b
--- /dev/null
+++ b/pc-bios/qemu.rsrc
@@ -0,0 +1,1504 @@
+data 'icns' (-16455) {
+	$"6963 6E73 0000 5DC3 6974 3332 0000 1DB3"            /* icns..]√it32...≥ */
+	$"0000 0000 FF00 FF00 B000 0F08 1222 3645"            /* ....ˇ.ˇ.∞...."6E */
+	$"4D56 6063 5A51 473C 2816 0CE8 0019 0416"            /* MV`cZQG<(..Ë.... */
+	$"3764 94B5 CCDD E7E9 EBEC EDEE ECEB EAE8"            /* 7dîµÃ›ÁÈÎÏÌÓÏÎÍË */
+	$"E2D4 BEA3 7746 200B E000 0608 2763 A5D0"            /* ‚‘æ£wF .‡...'c•– */
+	$"E6EE 80F5 8AF6 08F5 F4F0 EAD9 B97C 3B0F"            /* ÊÓÄıäˆ.ıÙÍŸπ|;. */
+	$"DB00 061C 64AF DAED F0F2 81F1 8BF0 83F1"            /* €...dØ⁄ÌÚÅÒãÉÒ */
+	$"05EE E3C1 8432 07D5 0007 0635 87CD E8ED"            /* .Ó„¡Ñ2.’...5áÕËÌ */
+	$"EDEE 97ED 07EE EEED EBDC A551 11D1 0007"            /* ÌÓóÌ.ÓÓÌÎ‹•Q.—.. */
+	$"0333 96D9 E6E9 EAE9 9CE8 06E9 E9E8 E1B8"            /* .3ñŸÊÈÍÈúË.ÈÈË·∏ */
+	$"560E CE00 0628 8CD3 E3E6 E5E4 93E3 80E4"            /* V.Œ..(å”„ÊÂ‰ì„Ä‰ */
+	$"87E3 06E4 E5E4 DEAE 4A07 CA00 050D 63C8"            /* á„.‰Â‰ﬁÆJ. ...c» */
+	$"DFE1 E196 E084 DF86 E004 E1E0 D791 23C8"            /* ﬂ··ñ‡ÑﬂÜ‡.·‡◊ë#» */
+	$"0004 2FA0 D8DC DC84 DA06 D9DA D9D9 DAD9"            /* ../†ÿ‹‹Ñ⁄.Ÿ⁄ŸŸ⁄Ÿ */
+	$"D987 DA00 D990 DA05 DBDC DBC0 5708 C400"            /* Ÿá⁄.Ÿê⁄.€‹€¿W.ƒ. */
+	$"1804 54C3 D7D7 D6D6 D7D6 D5D5 D6DF F1DF"            /* ..T√◊◊÷÷◊÷’’÷ﬂÒﬂ */
+	$"DADF E5F1 EEDD D5D6 D5D4 81D5 81D6 82D8"            /* ⁄ﬂÂÒÓ›’÷’‘Å’Å÷Çÿ */
+	$"83D6 80D8 80D6 80D7 04D6 D7D2 8615 C200"            /* É÷ÄÿÄ÷Ä◊.÷◊“Ü.¬. */
+	$"060E 75CB D4D3 D2D2 80D0 11E0 EAF0 F8FD"            /* ..uÀ‘”““Ä–.‡Í¯˝ */
+	$"F9F7 FAFB FDFD F7DB CED6 DDD7 D380 D08C"            /* ˘˜˙˚˝˝˜€Œ÷›◊”Ä–å */
+	$"D286 D104 D2D3 D2A3 29C0 0003 1591 CDCF"            /* “Ü—.“”“£)¿...ëÕœ */
+	$"80CE 06CC D7DB DEFB FCFD 86FE 0BF5 DBEB"            /* ÄŒ.Ã◊€ﬁ˚¸˝Ü˛.ı€Î */
+	$"FAF7 F3EB E0D7 D0CD CD86 CE84 CD84 CE03"            /* ˙˜ÛÎ‡◊–ÕÕÜŒÑÕÑŒ. */
+	$"CFD0 B739 BE00 0318 96CB CA81 C803 C7DB"            /* œ–∑9æ...ñÀ Å».«€ */
+	$"FAF9 8AFE 0BFA FCFE FEFD FCFB FAF2 E6D8"            /* ˙˘ä˛.˙¸˛˛˝¸˚˙ÚÊÿ */
+	$"CD8C C883 CA05 C8C9 CACC BA3F BC00 041B"            /* Õå»É .»… Ã∫?º... */
+	$"98C7 C5C3 81C4 02C3 C8F1 93FE 09FD FCFA"            /* ò«≈√Åƒ.√»Òì˛∆˝¸˙ */
+	$"F3E6 D5CB C3C2 C383 C584 C683 C502 C7B8"            /* ÛÊ’À√¬√É≈Ñ∆É≈.«∏ */
+	$"42BA 0003 1291 C4C1 84C0 02BC CFFB 96FE"            /* B∫...ëƒ¡Ñ¿.ºœ˚ñ˛ */
+	$"06FC FAF5 E9D5 C2BD 87C0 86BF 02C1 B536"            /* .¸˙ıÈ’¬Ωá¿Üø.¡µ6 */
+	$"B800 030A 7DC1 BC84 BB03 B9CD D7EE 99FE"            /* ∏..¬}¡ºÑª.πÕ◊Óô˛ */
+	$"07FC F9EF D4BB BBBE BD82 BC87 BB03 BCC0"            /* .¸˘Ô‘ªªæΩÇºáª.º¿ */
+	$"A825 B600 0304 64BB B981 B880 B704 B5C6"            /* ®%∂...dªπÅ∏Ä∑.µ∆ */
+	$"F5FA FB9B FE04 FDF9 E6C3 B58F B802 BC97"            /* ı˙˚õ˛.˝˘Ê√µè∏.ºó */
+	$"15B5 0002 39B2 B582 B480 B201 B5EA A0FE"            /* .µ..9≤µÇ¥Ä≤.µÍ†˛ */
+	$"04FC F1C9 B1B3 87B2 01B3 B382 B402 B873"            /* .¸Ò…±≥á≤.≥≥Ç¥.∏s */
+	$"06B3 0002 1899 B385 B002 ADCC FDA2 FE02"            /* .≥...ô≥Ö∞.≠Ã˝¢˛. */
+	$"F7CB AD8E B002 B1B1 43B2 0002 0671 AE86"            /* ˜À≠é∞.±±C≤...qÆÜ */
+	$"AC01 ADEC A4FE 02F7 C3A8 81AB 80AA 01AC"            /* ¨.≠Ï§˛.˜√®Å´Ä™.¨ */
+	$"AB84 AC03 AAAD 9C1C B100 0231 A6A7 80A6"            /* ´Ñ¨.™≠ú.±..1¶ßÄ¶ */
+	$"82A7 02A4 BEFD 97FE 05DC 9C83 8DAE E085"            /* Çß.§æ˝ó˛.‹úÉçÆ‡Ö */
+	$"FE06 EEB0 A6A7 A8A6 A685 A800 A781 A602"            /* ˛.Ó∞¶ß®¶¶Ö®.ßÅ¶. */
+	$"AD69 04AF 0002 0A7E A681 A281 A302 A2A0"            /* ≠i.Ø..¬~¶Å¢Å£.¢† */
+	$"E097 FE08 A228 110E 0F14 295A BC83 FE03"            /* ‡ó˛.¢(....)ZºÉ˛. */
+	$"FDDF A3A1 81A2 80A3 86A2 02A4 A128 AF00"            /* ˝ﬂ£°Å¢Ä£Ü¢.§°(Ø. */
+	$"013E A286 9E02 9DAE F996 FE02 F228 0181"            /* .>¢Üû.ùÆ˘ñ˛.Ú(.Å */
+	$"0004 0108 1B72 F582 FE04 FBC3 9A9E 9E82"            /* .....rıÇ˛.˚√öûûÇ */
+	$"A086 9E03 A0A3 7407 AD00 0209 7C9D 8599"            /* †Üû.†£t.≠..∆|ùÖô */
+	$"029A 96D4 97FE 01F2 1B85 0002 0B60 F982"            /* .öñ‘ó˛.Ú.Ö...`˘Ç */
+	$"FE03 EB9E 9898 8899 819A 029B 9C28 AD00"            /* ˛.ÎûòòàôÅö.õú(≠. */
+	$"012C 9986 9802 93AC F897 FE01 FD72 8600"            /* .,ôÜò.ì¨¯ó˛.˝rÜ. */
+	$"0109 A882 FE04 FCA6 9497 9780 9885 9781"            /* .∆®Ç˛.¸¶îóóÄòÖóÅ */
+	$"9801 9D60 AC00 0304 6897 9184 9202 9199"            /* ò.ù`¨...hóëÑí.ëô */
+	$"E699 FE01 F35B 8600 005D 83FE 01C2 8E8F"            /* Êô˛.Û[Ü..]É˛.¬éè */
+	$"9201 8E18 AB00 0317 8B8F 8F83 9003 8F90"            /* í.é.´...ãèèÉê.èê */
+	$"DBFD 9AFE 02F5 8E1F 8400 0095 83FE 02EE"            /* €˝ö˛.ıé.Ñ..ïÉ˛.Ó */
+	$"978D 8590 808E 8090 048E 8E8F 9642 AB00"            /* óçÖêÄéÄê.ééèñB´. */
+	$"0338 9089 8983 8A02 87AA FC9C FE09 F8E3"            /* .8êââÉä.á™¸ú˛∆¯„ */
+	$"934A 2315 1830 8DF8 84FE 02D9 8F86 828B"            /* ìJ#..0ç¯Ñ˛.ŸèÜÇã */
+	$"828A 808B 808A 028F 6904 AA00 015D 8C82"            /* ÇäÄãÄä.èi.™..]åÇ */
+	$"8480 8601 83D3 9EFE 07FD F8F3 EEDF E1F2"            /* ÑÄÜ.É”û˛.˝¯ÛÓﬂ·Ú */
+	$"F885 FE04 FCE1 A284 8380 8683 8582 8602"            /* ¯Ö˛.¸·¢ÑÉÄÜÉÖÇÜ. */
+	$"8781 14A9 0002 0E79 8282 8104 8383 818F"            /* áÅ.©...yÇÇÅ.ÉÉÅè */
+	$"F498 FE03 FDDE E2F6 83FE 01FD FD88 FE07"            /* Ùò˛.˝ﬁ‚ˆÉ˛.˝˝à˛. */
+	$"FDF8 D3A3 887E 7F81 8282 8383 0189 2EA9"            /* ˝¯”£à~.ÅÇÇÉÉ.â.© */
+	$"0002 1E82 7F82 7E03 8080 7BAA 99FE 0BFD"            /* ...Ç.Ç~.ÄÄ{™ô˛.˝ */
+	$"B46C 5B58 6278 96B7 D4ED FD8A FE08 FCF9"            /* ¥l[Xbxñ∑‘Ì˝ä˛.¸˘ */
+	$"E5C1 9D84 7A7B 7F83 8003 7F7F 834A A900"            /* Â¡ùÑz{.ÉÄ...ÉJ©. */
+	$"0432 827A 7B7B 8079 037A 7A75 CB9A FE11"            /* .2Çz{{Äy.zzuÀö˛. */
+	$"FAF5 E8C7 9E72 4B30 2422 2D41 5D7E 9FB6"            /* ˙ıË«ûrK0$"-A]~ü∂ */
+	$"D2F3 85FE 08FD FBF9 E6C2 967A 7579 817A"            /* “ÛÖ˛.˝˚˘Ê¬ñzuyÅz */
+	$"047B 797F 5E03 A800 0143 7E80 7781 7602"            /* .{y.^.®..C~ÄwÅv. */
+	$"757B E99D FE13 FCF9 F6F3 EFD2 A877 4925"            /* u{Èù˛.¸˘ˆÛÔ“®wI% */
+	$"1109 0B0D 2747 709E C9EB 83FE 07FD FBF7"            /* .∆..'Gpû…ÎÉ˛.˝˚˜ */
+	$"D8A7 8072 7481 7602 776B 0BA8 0001 4F77"            /* ÿßÄrtÅv.wk.®..Ow */
+	$"8073 8172 0270 8DFC A2FE 13FC FAF6 F3F1"            /* ÄsÅr.pç¸¢˛.¸˙ˆÛÒ */
+	$"D5AE 7F51 270D 0609 0C20 3C62 8FBB E181"            /* ’Æ.Q'..∆. <bèª·Å */
+	$"FE0B FCF9 E9BB 856F 7273 7374 6F0F A800"            /* ˛.¸˘ÈªÖorssto.®. */
+	$"0156 7281 7080 6F01 69AA A8FE 12FD FAF7"            /* .VrÅpÄo.i™®˛.˝˙˜ */
+	$"F4F1 D0A0 6937 1809 0508 0B19 3979 C3F1"            /* ÙÒ–†i7.∆....9y√Ò */
+	$"80FE 08FB EFB2 746D 706F 7017 A700 0206"            /* Ä˛.˚Ô≤tmpop.ß... */
+	$"596E 816B 806A 0167 CBAD FE19 FCF9 F6F2"            /* YnÅkÄj.gÀ≠˛.¸˘ˆÚ */
+	$"E3CC AD7B 4622 0D07 0B21 3F5C 92E1 FEFA"            /* „Ã≠{F"...!?\í·˛˙ */
+	$"D87D 686B 6D1B A700 0306 586A 6880 6704"            /* ÿ}hkm.ß...XjhÄg. */
+	$"6666 656E E8B2 FE14 FCFA F7F3 EFD2 AE81"            /* ffenË≤˛.¸˙˜ÛÔ“ÆÅ */
+	$"5429 0D08 1C81 F8FC E179 646B 1CA7 0003"            /* T)...Å¯¸·ydk.ß.. */
+	$"0655 6765 8264 0261 7EFB B7FE 0FFC FAF7"            /* .UgeÇd.a~˚∑˛.¸˙˜ */
+	$"F4F1 D088 3207 3FDA FDD2 6767 1BA7 0002"            /* ÙÒ–à2.?⁄˝“gg.ß.. */
+	$"0450 6381 6103 6060 5B9C BDFE 0AFC F8F2"            /* .PcÅa.``[úΩ˛¬¸¯Ú */
+	$"A527 24B9 FC94 5F18 A800 0148 6082 5E02"            /* •'$π¸î_.®..H`Ç^. */
+	$"5D58 C4AD FE0D FCE9 CDBA ADA2 A3A7 AFC0"            /* ]Xƒ≠˛.¸ÈÕ∫≠¢£ßØ¿ */
+	$"D2E6 F4FD 82FE 07FA DD30 31CB D75C 12A8"            /* “ÊÙ˝Ç˛.˙›01À◊\.® */
+	$"0003 3E5E 5959 8058 0257 62EA ABFE 1CDD"            /* ..>^YYÄX.WbÍ´˛.› */
+	$"A57D 695C 5958 5757 5857 595F 6873 7E8C"            /* •}i\YXWWXWY_hs~å */
+	$"98A5 AFAE A07C 5B56 639E 5F0E A800 0330"            /* ò•ØÆ†|[Vcû_.®..0 */
+	$"5955 5580 5402 507C FDA9 FE08 DA8F 5F54"            /* YUUÄT.P|˝©˛.⁄è_T */
+	$"5354 5455 5580 5681 5581 5480 5507 5454"            /* STTUUÄVÅUÅTÄU.TT */
+	$"5556 5557 4E07 A800 0122 5680 5203 5151"            /* UVUWN.®.."VÄR.QQ */
+	$"4C9C A8FE 05E1 955D 5051 5080 5183 5282"            /* Lú®˛.·ï]PQPÄQÉRÇ */
+	$"5180 5208 5151 5051 5152 5254 3FA9 0000"            /* QÄR.QQPQQRRT?©.. */
+	$"1581 5003 4F4F 49B4 A6FE 03D4 905D 4D85"            /* .ÅP.OOI¥¶˛.‘ê]MÖ */
+	$"4F82 5083 4F01 5050 844F 0152 2EA9 0002"            /* OÇPÉO.PPÑO.R.©.. */
+	$"0A45 4B81 4A01 49CE A3FE 06F2 BB7B 5349"            /* ¬EKÅJ.IŒ£˛.Úª{SI */
+	$"4A4A 804B 8D4A 824B 814A 014E 1BAA 0001"            /* JJÄKçJÇKÅJ.N.™.. */
+	$"324A 8047 0246 4EE4 A1FE 06E3 9E65 4944"            /* 2JÄG.FN‰°˛.„ûeID */
+	$"4647 8248 8547 8348 0147 4783 4880 4701"            /* FGÇHÖGÉH.GGÉHÄG. */
+	$"440C AA00 011C 4880 4502 4154 F19F FE05"            /* D.™...HÄE.ATÒü˛. */
+	$"DE8C 5644 4244 8043 8245 8344 8245 8644"            /* ﬁåVDBDÄCÇEÉDÇEÜD */
+	$"8145 0147 33AB 0001 0C3F 8041 023F 5EFB"            /* ÅE.G3´...?ÄA.?^˚ */
+	$"9DFE 02EE 9552 8040 8041 8540 8041 0140"            /* ù˛.ÓïRÄ@ÄAÖ@ÄA.@ */
+	$"4080 4180 4080 4182 4081 4101 4420 AC00"            /* @ÄAÄ@ÄAÇ@ÅA.D ¨. */
+	$"052C 403E 3E38 679D FE04 C35D 3C3D 3E82"            /* .,@>>8gù˛.√]<=>Ç */
+	$"4080 3E80 3F82 4080 3F80 3E8B 3F01 3D0C"            /* @Ä>Ä?Ç@Ä?Ä>ã?.=. */
+	$"AC00 0513 3D3B 3A34 749C FE02 A743 3880"            /* ¨...=;:4tú˛.ßC8Ä */
+	$"3A80 3B82 3A80 3982 3B83 3A80 3B87 3A01"            /* :Ä;Ç:Ä9Ç;É:Ä;á:. */
+	$"3E26 AD00 0504 2E38 3731 789B FE03 9C32"            /* >&≠....871xõ˛.ú2 */
+	$"3536 8037 8038 8337 0136 3688 3783 3680"            /* 56Ä7Ä8É7.66à7É6Ä */
+	$"3781 3601 380F AE00 0415 3634 2D77 9AFE"            /* 7Å6.8.Æ...64-wö˛ */
+	$"02BF 3432 8033 8034 0135 3583 3480 3301"            /* .ø42Ä3Ä4.55É4Ä3. */
+	$"3434 8335 8B34 0136 25AF 0004 0327 322B"            /* 44É5ã4.6%Ø...'2+ */
+	$"6F99 FE02 F24B 2C81 3080 3180 329B 3101"            /* oô˛.ÚK,Å0Ä1Ä2õ1. */
+	$"300D B000 030E 2F28 6099 FE01 9C27 882D"            /* 0.∞.../(`ô˛.ú'à- */
+	$"802E 882D 822E 872D 0130 1DB2 0003 1B29"            /* Ä.à-Ç.á-.0.≤...) */
+	$"4AFC 97FE 02F9 4B26 812A 842B 802C 822B"            /* J¸ó˛.˘K&Å*Ñ+Ä,Ç+ */
+	$"802A 802B 802C 852B 802C 022B 2809 B200"            /* Ä*Ä+Ä,Ö+Ä,.+(∆≤. */
+	$"0306 2137 E997 FE02 CA28 2681 2784 2880"            /* ..!7Èó˛. (&Å'Ñ(Ä */
+	$"2997 2800 10B4 0002 0D25 C897 FE02 931D"            /* )ó(..¥...%»ó˛.ì. */
+	$"2584 248D 2582 2483 2581 2401 2718 B600"            /* %Ñ$ç%Ç$É%Å$.'.∂. */
+	$"0110 B597 FE01 681B 8521 8422 8023 8B22"            /* ..µó˛.h.Ö!Ñ"Ä#ã" */
+	$"8023 8022 0223 1C04 B700 00A2 97FE 014A"            /* Ä#Ä".#..∑..¢ó˛.J */
+	$"1A85 1E84 1F80 2086 1F80 1E82 1F80 2001"            /* .Ö.Ñ.Ä Ü.Ä.Ç.Ä . */
+	$"1C08 B800 0094 96FE 02FB 3C19 841C 881D"            /* ..∏..îñ˛.˚<.Ñ.à. */
+	$"801E 861D 801C 811D 011C 09B9 0000 8C96"            /* Ä.Ü.Ä.Å...∆π..åñ */
+	$"FE02 F434 1688 1994 1A03 191A 190A BA00"            /* ˛.Ù4.à.î.....¬∫. */
+	$"0085 96FE 02F4 3114 8816 8717 8018 8717"            /* .Öñ˛.Ù1.à.á.Ä.á. */
+	$"0218 1609 BB00 007C 96FE 02F7 3111 8B14"            /* ...∆ª..|ñ˛.˜1.ã. */
+	$"8715 8016 8415 0113 07BC 0000 6F97 FE01"            /* á.Ä.Ñ....º..oó˛. */
+	$"3A0E 9D12 0213 0E04 BD00 004C 97FE 0148"            /* :.ù.....Ω..Ló˛.H */
+	$"0A8B 0F8F 1000 0BBF 0001 22F8 96FE 0157"            /* ¬ã.è...ø.."¯ñ˛.W */
+	$"0795 0D83 0E01 0D07 C000 0104 BB96 FE01"            /* .ï.É....¿...ªñ˛. */
+	$"7504 9A0B 0109 03C2 0000 3A96 FE01 9D03"            /* u.ö..∆.¬..:ñ˛.ù. */
+	$"9809 0107 04C5 0000 7295 FE01 A500 9706"            /* ò∆...≈..rï˛.•.ó. */
+	$"0003 C800 0160 F993 FE02 C606 0094 03CC"            /* ..»..`˘ì˛.∆..î.Ã */
+	$"0001 47EE 92FE 01EC 15E5 0001 37E9 92FE"            /* ..GÓí˛.Ï.Â..7Èí˛ */
+	$"0037 E600 0131 E691 FE00 6BE7 0001 32EC"            /* .7Ê..1Êë˛.kÁ..2Ï */
+	$"90FE 00A6 E800 0139 E68F FE01 E010 E800"            /* ê˛.¶Ë..9Êè˛.‡.Ë. */
+	$"0141 EC8F FE00 37E9 0001 4BF3 8EFE 007A"            /* .AÏè˛.7È..KÛé˛.z */
+	$"EA00 0164 F78D FE01 C305 EA00 0181 F98C"            /* Í..d˜ç˛.√.Í..Å˘å */
+	$"FE01 F423 EA00 0209 9EFB 8CFE 0066 EB00"            /* ˛.Ù#Í..∆û˚å˛.fÎ. */
+	$"020F B9FD 8BFE 01BA 04EB 0001 1CD5 8BFE"            /* ..π˝ã˛.∫.Î...’ã˛ */
+	$"01F4 23EC 0001 30E8 8BFE 0067 ED00 014C"            /* .Ù#Ï..0Ëã˛.gÌ..L */
+	$"F58A FE01 BB04 ED00 016F F989 FE01 F424"            /* ıä˛.ª.Ì..o˘â˛.Ù$ */
+	$"ED00 0206 A1FC 89FE 0067 EE00 0113 CD89"            /* Ì...°¸â˛.gÓ...Õâ */
+	$"FE01 BE06 EE00 012E EA88 FE01 F624 EF00"            /* ˛.æ.Ó...Íà˛.ˆ$Ô. */
+	$"015A F888 FE00 6CEF 0002 038B FB87 FE01"            /* .Z¯à˛.lÔ...ã˚á˛. */
+	$"C206 EF00 010F BC87 FE01 F828 F000 0125"            /* ¬.Ô...ºá˛.¯(..% */
+	$"E687 FE00 6DF1 0001 59F9 86FE 01BD 04F0"            /* Êá˛.mÒ..Y˘Ü˛.Ω. */
+	$"0002 049D FD85 FE01 F320 F100 0117 D786"            /* ...ù˝Ö˛.Û Ò...◊Ü */
+	$"FE00 5EF2 0001 42F6 85FE 00A9 F300 0182"            /* ˛.^Ú..BˆÖ˛.©Û..Ç */
+	$"FB84 FE01 E311 F200 010F C585 FE00 35F3"            /* ˚Ñ˛.„.Ú...≈Ö˛.5Û */
+	$"0001 31F1 84FE 0074 F400 0176 F983 FE01"            /* ..1ÒÑ˛.tÙ..v˘É˛. */
+	$"B601 F300 010E C383 FE01 E310 F400 0138"            /* ∂.Û...√É˛.„.Ù..8 */
+	$"F082 FE01 FB2A F500 0184 FB82 FE00 4FF5"            /* Ç˛.˚*ı..Ñ˚Ç˛.Oı */
+	$"0001 13CD 82FE 007D F600 0145 F481 FE00"            /* ...ÕÇ˛.}ˆ..EÙÅ˛. */
+	$"A1F6 0002 049A FD80 FE01 BA01 F600 0121"            /* °ˆ...ö˝Ä˛.∫.ˆ..! */
+	$"E480 FE01 D209 F700 056F FAFE FEE5 11F7"            /* ‰Ä˛.“∆˜..o˙˛˛Â.˜ */
+	$"0005 0ECA FEFE F41D F800 0447 F5FE FE2C"            /* ... ˛˛Ù.¯..Gı˛˛, */
+	$"F800 0406 A5FD FE40 F900 0329 E0FE 5BFA"            /* ¯...•˝˛@˘..)‡˛[˙ */
+	$"0002 3BF7 7EFB 0001 C2A8 FB00 026A CB01"            /* ..;˜~˚..¬®˚..jÀ. */
+	$"FA00 0223 D210 FA00 0204 9920 FB00 0129"            /* ˙..#“.˙...ô ˚..) */
+	$"1EFF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00A8 00FF 00FF 00B0 000F 0812 2236 454D"            /* .®.ˇ.ˇ.∞...."6EM */
+	$"5660 635A 5147 3C28 160C E800 1904 1637"            /* V`cZQG<(..Ë....7 */
+	$"6494 B5CC DDE7 E9EB ECED EEEC EBEA E8E2"            /* dîµÃ›ÁÈÎÏÌÓÏÎÍË‚ */
+	$"D4BE A377 4620 0BE0 0006 0827 63A5 D0E6"            /* ‘æ£wF .‡...'c•–Ê */
+	$"EE80 F58A F608 F5F4 F0EA D9B9 7C3B 0FDB"            /* ÓÄıäˆ.ıÙÍŸπ|;.€ */
+	$"0006 1C64 AFDA EDF0 F281 F18B F083 F105"            /* ...dØ⁄ÌÚÅÒãÉÒ. */
+	$"EEE3 C184 3207 D500 0706 3587 CDE8 EDED"            /* Ó„¡Ñ2.’...5áÕËÌÌ */
+	$"EE97 ED07 EEEE EDEB DCA5 5111 D100 0703"            /* ÓóÌ.ÓÓÌÎ‹•Q.—... */
+	$"3396 D9E6 E9EA E99C E806 E9E9 E8E1 B856"            /* 3ñŸÊÈÍÈúË.ÈÈË·∏V */
+	$"0ECE 0006 288C D3E3 E6E5 E493 E380 E487"            /* .Œ..(å”„ÊÂ‰ì„Ä‰á */
+	$"E306 E4E5 E4DE AE4A 07CA 0005 0D63 C8DF"            /* „.‰Â‰ﬁÆJ. ...c»ﬂ */
+	$"E1E1 96E0 84DF 86E0 04E1 E0D7 9123 C800"            /* ··ñ‡ÑﬂÜ‡.·‡◊ë#». */
+	$"042F A0D8 DCDC 82DA 08DB D9DE DCDE DFDC"            /* ./†ÿ‹‹Ç⁄.€Ÿﬁ‹ﬁﬂ‹ */
+	$"DEDE 87DA 00D9 90DA 05DB DCDB C057 08C4"            /* ﬁﬁá⁄.Ÿê⁄.€‹€¿W.ƒ */
+	$"0004 0454 C3D7 D780 D614 D7DA DBD9 BC82"            /* ...T√◊◊Ä÷.◊⁄€ŸºÇ */
+	$"BDCA BCA7 8185 C2DA D6D8 D9D8 D6D6 D581"            /* Ω ºßÅÖ¬⁄÷ÿŸÿ÷÷’Å */
+	$"D682 D883 D680 D880 D680 D704 D6D7 D286"            /* ÷ÇÿÉ÷ÄÿÄ÷Ä◊.÷◊“Ü */
+	$"15C2 001F 0E75 CBD4 D3D2 D2D4 D6D3 AC8C"            /* .¬...uÀ‘”““‘÷”¨å */
+	$"7E65 5362 685D 5753 5267 B7D8 C5B1 C2CD"            /* ~eSbh]WSRg∑ÿ≈±¬Õ */
+	$"D4D7 D5D3 8BD2 86D1 04D2 D3D2 A329 C000"            /* ‘◊’”ã“Ü—.“”“£)¿. */
+	$"0315 91CD CF80 CE0B D0B5 ABA5 5A54 5352"            /* ..ëÕœÄŒ.–µ´•ZTSR */
+	$"5151 5252 8151 0C66 AA81 5964 6F83 9EB5"            /* QQRRÅQ.f™ÅYdoÉûµ */
+	$"C9D2 D2CF 85CE 84CD 84CE 03CF D0B7 39BE"            /* …““œÖŒÑÕÑŒ.œ–∑9æ */
+	$"0003 1896 CBCA 81C8 06CD 9E57 5D50 5151"            /* ...ñÀ Å».ÕûW]PQQ */
+	$"8552 1151 515A 5551 5152 5456 5E6D 89A6"            /* ÖR.QQZUQQRTV^mâ¶ */
+	$"C0CD CDCB C988 C883 CA05 C8C9 CACC BA3F"            /* ¿ÕÕÀ…à»É .»… Ã∫? */
+	$"BC00 041B 98C7 C5C3 82C4 02BD 684F 8A52"            /* º...ò«≈√Çƒ.ΩhOäR */
+	$"0051 8152 0C51 5152 5254 5A67 82A2 BAC6"            /* .QÅR.QQRRTZgÇ¢∫∆ */
+	$"C8C6 83C5 84C6 83C5 02C7 B842 BA00 0312"            /* »∆É≈Ñ∆É≈.«∏B∫... */
+	$"91C4 C184 C003 C6A2 5551 9252 0951 5152"            /* ëƒ¡Ñ¿.∆¢UQíR∆QQR */
+	$"5359 6379 9ABA C387 C086 BF02 C1B5 36B8"            /* SYcyö∫√á¿Üø.¡µ6∏ */
+	$"0003 0A7D C1BC 84BB 04BF 9E8E 6A50 9552"            /* ..¬}¡ºÑª.øûéjPïR */
+	$"0A51 5152 5357 6B95 BBBF BEBD 82BC 87BB"            /* ¬QQRSWkïªøæΩÇºáª */
+	$"03BC C0A8 25B6 0003 0464 BBB9 81B8 80B7"            /* .º¿®%∂...dªπÅ∏Ä∑ */
+	$"05BA A35E 5656 5198 5280 5103 5672 A8BB"            /* .∫£^VVQòRÄQ.Vr®ª */
+	$"8FB8 02BC 9715 B500 0239 B2B5 82B4 06B2"            /* è∏.ºó.µ..9≤µÇ¥.≤ */
+	$"B2B3 B16B 5051 9D52 0551 5361 95B8 B387"            /* ≤≥±kPQùR.QSaï∏≥á */
+	$"B201 B3B3 82B4 02B8 7306 B300 0218 99B3"            /* ≤.≥≥Ç¥.∏s.≥...ô≥ */
+	$"85B0 03B3 8E51 519F 5204 5150 5A8D B28E"            /* Ö∞.≥éQQüR.QPZç≤é */
+	$"B002 B1B1 43B2 0002 0671 AE85 AC03 ADAA"            /* ∞.±±C≤...qÆÖ¨.≠™ */
+	$"654F 9652 0054 8056 0155 5383 5203 5158"            /* eOñR.TÄV.USÉR.QX */
+	$"92AE 81AB 80AA 01AC AB84 AC03 AAAD 9C1C"            /* íÆÅ´Ä™.¨´Ñ¨.™≠ú. */
+	$"B100 0231 A6A7 80A6 82A7 01AA 8F97 5208"            /* ±..1¶ßÄ¶Çß.™èóR. */
+	$"5548 342D 313B 4954 5582 5207 5060 9EA9"            /* UH4-1;ITUÇR.P`û© */
+	$"A7A8 A6A6 85A8 00A7 81A6 02AD 6904 AF00"            /* ß®¶¶Ö®.ßÅ¶.≠i.Ø. */
+	$"020A 7EA6 81A2 82A3 02A4 6C50 9552 0B55"            /* .¬~¶Å¢Ç£.§lPïR.U */
+	$"3610 0605 0506 1021 3F55 5381 5201 516D"            /* 6......!?USÅR.Qm */
+	$"83A2 80A3 86A2 02A4 A128 AF00 013E A286"            /* É¢Ä£Ü¢.§°(Ø..>¢Ü */
+	$"9E02 A191 5595 5203 534F 1101 8100 0501"            /* û.°ëUïR.SO..Å... */
+	$"030B 2750 5380 5205 5152 81A1 9E9E 82A0"            /* ..'PSÄR.QRÅ°ûûÇ† */
+	$"869E 03A0 A374 07AD 0002 097C 9D85 9903"            /* Üû.†£t.≠..∆|ùÖô. */
+	$"9A9E 6F50 9552 0253 4F0D 8500 0104 2282"            /* öûoPïR.SO.Ö..."Ç */
+	$"5204 505F 9899 9888 9981 9A02 9B9C 28AD"            /* R.P_òôòàôÅö.õú(≠ */
+	$"0001 2C99 8798 0287 5651 9652 0154 2886"            /* ..,ôáò.áVQñR.T(Ü */
+	$"0002 0339 5581 5201 538A 8097 8098 8597"            /* ...9UÅR.SäÄóÄòÖó */
+	$"8198 019D 60AC 0003 0468 9791 8492 0393"            /* Åò.ù`¨...hóëÑí.ì */
+	$"8E60 5098 5201 5021 8600 0121 5681 5202"            /* é`PòR.P!Ü..!VÅR. */
+	$"5074 968F 9201 8E18 AB00 0317 8B8F 8F84"            /* Ptñèí.é.´...ãèèÑ */
+	$"9002 8F64 5099 5203 5351 310F 8400 0133"            /* ê.èdPôR.SQ1.Ñ..3 */
+	$"5581 5202 5159 8786 9080 8E80 9004 8E8E"            /* UÅR.QYáÜêÄéÄê.éé */
+	$"8F96 42AB 0003 3890 8989 838A 038D 7B51"            /* èñB´..8êââÉä.ç{Q */
+	$"519B 5209 544A 331C 0F0B 0C14 3051 8352"            /* QõR∆TJ3.....0QÉR */
+	$"0350 6288 8D82 8B82 8A80 8B80 8A02 8F69"            /* .PbàçÇãÇäÄãÄä.èi */
+	$"04AA 0001 5D8C 8284 8086 0288 6350 9852"            /* .™..]åÇÑÄÜ.àcPòR */
+	$"0153 5381 5206 5452 4C49 494F 5385 5207"            /* .SSÅR.TRLIIOSÖR. */
+	$"515E 7A87 8887 8686 8385 8286 0287 8114"            /* Q^záàáÜÜÉÖÇÜ.áÅ. */
+	$"A900 020E 7982 8281 8083 017F 5699 5203"            /* ©...yÇÇÅÄÉ..VôR. */
+	$"494A 5054 8056 0255 5453 8952 0651 5464"            /* IJPTÄV.UTSâR.QTd */
+	$"7581 8383 8382 8383 0189 2EA9 0002 1E82"            /* uÅÉÉÉÇÉÉ.â.©...Ç */
+	$"7F82 7E04 8080 816F 5098 520C 533D 2621"            /* .Ç~.ÄÄÅoPòR.S=&! */
+	$"2123 2A33 3E46 4E51 5580 5602 5554 5383"            /* !#*3>FNQUÄV.UTSÉ */
+	$"5208 5153 5A66 737D 8080 7F83 8003 7F7F"            /* R.QSZfs}ÄÄ.ÉÄ... */
+	$"834A A900 0432 827A 7B7B 8079 047A 7A7C"            /* ÉJ©..2Çz{{Äy.zz| */
+	$"6050 9A52 1551 4B41 3628 1D13 100E 1319"            /* `PöR.QKA6(...... */
+	$"222C 363E 4550 5556 5654 5380 5208 5151"            /* ",6>EPUVVTSÄR.QQ */
+	$"5358 6471 797B 7B81 7A04 7B79 7F5E 03A8"            /* SXdqy{{Åz.{y.^.® */
+	$"0001 437E 8077 8276 0274 5851 9B52 2251"            /* ..C~ÄwÇv.tXQõR"Q */
+	$"5253 5352 4D44 392A 1B10 0903 0304 101B"            /* RSSRMD9*..∆..... */
+	$"2836 434E 5356 5655 5352 5151 545D 6874"            /* (6CNSVVUSRQQT]ht */
+	$"7877 8176 0277 6B0B A800 014F 7780 7381"            /* xwÅv.wk.®..OwÄsÅ */
+	$"7201 736B A452 1C53 5352 4D47 3A2C 1E10"            /* r.sk§R.SSRMG:,.. */
+	$"0602 0304 0E17 2332 3F4A 5256 5553 5152"            /* ......#2?JRVUSQR */
+	$"5662 6F74 8073 0274 6F0F A800 0156 7281"            /* VbotÄs.to.®..VrÅ */
+	$"7080 6F02 7062 51A8 521D 5353 524E 4536"            /* pÄo.pbQ®R.SSRNE6 */
+	$"2616 0C05 0202 030B 172B 414F 5456 5553"            /* &........+AOTVUS */
+	$"5362 6F70 706F 7017 A700 0206 596E 816B"            /* Sboppop.ß...YnÅk */
+	$"806A 026C 5B51 AD52 1853 5350 4A43 3A2B"            /* Äj.l[Q≠R.SSPJC:+ */
+	$"1A0F 0802 040E 1922 324B 5652 5969 6C6B"            /* ......."2KVRYilk */
+	$"6D1B A700 0306 586A 6880 6780 6601 6754"            /* m.ß...XjhÄgÄf.gT */
+	$"B252 8053 1151 4C46 3A2D 1E11 0603 0D2E"            /* ≤RÄS.QLF:-...... */
+	$"5153 5564 686B 1CA7 0003 0655 6765 8364"            /* QSUdhk.ß...UgeÉd */
+	$"0060 B952 8053 0B4F 452F 1402 1849 5558"            /* .`πRÄS.OE/...IUX */
+	$"6468 1BA7 0002 0450 6381 6103 6060 615C"            /* dh.ß...PcÅa.``a\ */
+	$"BE52 0954 5039 100E 3D53 5D63 18A8 0001"            /* æR∆TP9..=S]c.®.. */
+	$"4860 825E 025D 5D56 AF52 0255 5756 8058"            /* H`Ç^.]]VØR.UWVÄX */
+	$"0357 5555 5384 5207 534B 1F23 5456 5E12"            /* .WUUSÑR.SK.#TV^. */
+	$"A800 033E 5E59 5980 5801 5958 AC52 0353"            /* ®..>^YYÄX.YX¨R.S */
+	$"5558 5888 5900 5880 5609 5452 5557 5A59"            /* UXXàY.XÄV∆TRUWZY */
+	$"5956 560E A800 0330 5955 5580 5401 5554"            /* YVV.®..0YUUÄT.UT */
+	$"AB52 0254 5555 8054 0155 5580 5680 5585"            /* ´R.TUUÄT.UUÄVÄUÖ */
+	$"5680 5504 5656 584D 07A8 0001 2256 8052"            /* VÄU.VVXM.®.."VÄR */
+	$"8051 AD52 8251 8352 8251 8052 8251 0352"            /* ÄQ≠RÇQÉRÇQÄRÇQ.R */
+	$"5254 3FA9 0000 1581 5080 4F00 51A6 5202"            /* RT?©...ÅPÄO.Q¶R. */
+	$"5150 4F80 5083 4F82 5083 4F01 5050 844F"            /* QPOÄPÉOÇPÉO.PPÑO */
+	$"0152 2EA9 0002 0A45 4B82 4A00 50A2 5207"            /* .R.©..¬EKÇJ.P¢R. */
+	$"5351 4F4E 4A4B 4A4A 804B 8D4A 824B 814A"            /* SQONJKJJÄKçJÇKÅJ */
+	$"014E 1BAA 0001 324A 8147 0148 51A0 5204"            /* .N.™..2JÅG.HQ†R. */
+	$"5351 4E4A 4780 4682 4885 4783 4801 4747"            /* SQNJGÄFÇHÖGÉH.GG */
+	$"8348 8047 0144 0CAA 0001 1C48 8045 0244"            /* ÉHÄG.D.™...HÄE.D */
+	$"4551 9E52 0653 504B 4543 4344 8043 8245"            /* EQûR.SPKECCDÄCÇE */
+	$"8344 8245 8644 8145 0147 33AB 0001 0C3F"            /* ÉDÇEÜDÅE.G3´...? */
+	$"8041 0140 449D 5204 5352 4942 4082 4185"            /* ÄA.@DùR.SRIB@ÇAÖ */
+	$"4080 4101 4040 8041 8040 8041 8240 8141"            /* @ÄA.@@ÄAÄ@ÄAÇ@ÅA */
+	$"0144 20AC 0001 2C40 803E 0042 9C52 0553"            /* .D ¨..,@Ä>.BúR.S */
+	$"4D41 3E3F 3F82 4080 3E80 3F82 4080 3F80"            /* MA>??Ç@Ä>Ä?Ç@Ä?Ä */
+	$"3E8B 3F01 3D0C AC00 0613 3D3B 3A39 4153"            /* >ã?.=.¨...=;:9AS */
+	$"9A52 0353 493A 3980 3A80 3B82 3A80 3982"            /* öR.SI:9Ä:Ä;Ç:Ä9Ç */
+	$"3B83 3A80 3B87 3A01 3E26 AD00 0604 2E38"            /* ;É:Ä;á:.>&≠....8 */
+	$"3736 4153 9952 0153 4680 3680 3780 3883"            /* 76ASôR.SFÄ6Ä7Ä8É */
+	$"3701 3636 8837 8336 8037 8136 0138 0FAE"            /* 7.66à7É6Ä7Å6.8.Æ */
+	$"0005 1536 3433 3F53 9852 0353 4933 3280"            /* ...643?SòR.SI32Ä */
+	$"3380 3401 3535 8334 8033 0134 3483 358B"            /* 3Ä4.55É4Ä3.44É5ã */
+	$"3401 3625 AF00 0503 2732 303B 5398 5202"            /* 4.6%Ø...'20;SòR. */
+	$"5135 2F81 3080 3180 329B 3101 300D B000"            /* Q5/Å0Ä1Ä2õ1.0.∞. */
+	$"040E 2F2C 3853 9752 0254 412C 882D 802E"            /* ../,8SóR.TA,à-Ä. */
+	$"882D 822E 872D 0130 1DB2 0003 1B2C 3251"            /* à-Ç.á-.0.≤...,2Q */
+	$"9752 0251 3129 812A 842B 802C 822B 802A"            /* óR.Q1)Å*Ñ+Ä,Ç+Ä* */
+	$"802B 802C 852B 802C 022B 2809 B200 0406"            /* Ä+Ä,Ö+Ä,.+(∆≤... */
+	$"242D 4D53 9552 0153 4983 2784 2880 2997"            /* $-MSïR.SIÉ'Ñ(Ä)ó */
+	$"2800 10B4 0003 0D26 4853 9552 0354 3D22"            /* (..¥...&HSïR.T=" */
+	$"2584 248D 2582 2483 2581 2401 2718 B600"            /* %Ñ$ç%Ç$É%Å$.'.∂. */
+	$"0212 4454 9552 0254 3220 8521 8422 8023"            /* ..DTïR.T2 Ö!Ñ"Ä# */
+	$"8B22 8023 8022 0223 1C04 B700 0138 5595"            /* ã"Ä#Ä".#..∑..8Uï */
+	$"5202 532A 1D85 1E84 1F80 2086 1F80 1E82"            /* R.S*.Ö.Ñ.Ä Ü.Ä.Ç */
+	$"1F80 2001 1C08 B800 0132 5695 5202 5125"            /* .Ä ...∏..2VïR.Q% */
+	$"1B84 1C88 1D80 1E86 1D80 1C81 1D01 1C09"            /* .Ñ.à.Ä.Ü.Ä.Å...∆ */
+	$"B900 012F 5695 5201 5021 8919 941A 0319"            /* π../VïR.P!â.î... */
+	$"1A19 0ABA 0001 2D56 9552 014F 1E89 1687"            /* ..¬∫..-VïR.O.â.á */
+	$"1780 1887 1702 1816 09BB 0001 2A56 9552"            /* .Ä.á....∆ª..*VïR */
+	$"0250 1C13 8B14 8715 8016 8415 0113 07BC"            /* .P..ã.á.Ä.Ñ....º */
+	$"0001 2756 9652 011E 109D 1202 130E 04BD"            /* ..'VñR...ù.....Ω */
+	$"0001 1C56 9552 0254 210E 8B0F 8F10 000B"            /* ...VïR.T!.ã.è... */
+	$"BF00 010E 5095 5202 5524 0B95 0D83 0E01"            /* ø...PïR.U$.ï.É.. */
+	$"0D07 C000 0203 3E55 9452 0256 2B09 9A0B"            /* ..¿...>UîR.V+∆ö. */
+	$"0109 03C2 0002 1654 5393 5202 5535 0698"            /* .∆.¬...TSìR.U5.ò */
+	$"0901 0704 C500 0227 5553 9252 0255 3704"            /* ∆...≈..'USíR.U7. */
+	$"9706 0003 C800 0222 5253 9152 0254 4104"            /* ó...».."RSëR.TA. */
+	$"9503 CC00 021A 4D54 9052 0253 4D09 E500"            /* ï.Ã...MTêR.SM∆Â. */
+	$"0216 4D54 9052 0154 15E6 0002 134B 548F"            /* ..MTêR.T.Ê...KTè */
+	$"5201 5625 E700 0214 4E54 8E52 0155 36E8"            /* R.V%Á...NTéR.U6Ë */
+	$"0002 154C 538D 5202 5348 08E8 0002 184D"            /* ...LSçR.SH.Ë...M */
+	$"538D 5201 5415 E900 021C 5053 8C52 0156"            /* SçR.T.È...PSåR.V */
+	$"2AEA 0001 2353 8C52 0154 3FEB 0001 2D54"            /* *Í..#SåR.T?Î..-T */
+	$"8C52 0150 0FEA 0002 0434 558B 5201 5623"            /* åR.P.Í...4UãR.V# */
+	$"EB00 0208 3C54 8A52 0155 3CEC 0002 0D46"            /* Î...<TäR.U<Ï...F */
+	$"5389 5202 5350 0FEC 0002 134C 5389 5201"            /* SâR.SP.Ï...LSâR. */
+	$"5624 ED00 011C 5189 5201 553D EE00 0127"            /* V$Ì...QâR.U=Ó..' */
+	$"5488 5202 5350 0FED 0002 0435 5588 5201"            /* TàR.SP.Ì...5UàR. */
+	$"5624 EE00 0209 4454 8752 0155 3EEF 0002"            /* V$Ó..∆DTáR.U>Ô.. */
+	$"134D 5387 5201 5010 EF00 011F 5387 5201"            /* .MSáR.P.Ô...SáR. */
+	$"5625 F000 012F 5586 5201 553F F000 0207"            /* V%../UÜR.U?... */
+	$"3E54 8652 0151 10F0 0002 0F4D 5385 5201"            /* >TÜR.Q....MSÖR. */
+	$"5625 F100 011E 5485 5201 553E F100 0203"            /* V%Ò...TÖR.U>Ò... */
+	$"3455 8452 0253 500E F100 020B 4654 8452"            /* 4UÑR.SP.Ò...FTÑR */
+	$"0156 21F2 0001 1951 8452 0155 38F3 0001"            /* .V!Ú...QÑR.U8Û.. */
+	$"2C55 8352 0253 4908 F200 0208 4054 8352"            /* ,UÉR.SI.Ú...@TÉR */
+	$"0153 15F3 0001 134F 8352 0156 27F4 0001"            /* .S.Û...OÉR.V'Ù.. */
+	$"2854 8252 0155 3DF4 0002 0741 5481 5202"            /* (TÇR.U=Ù...ATÅR. */
+	$"5349 08F4 0001 1550 8252 0151 11F5 0001"            /* SI.Ù...PÇR.Q.ı.. */
+	$"2C55 8152 0156 1BF5 0002 0944 5480 5201"            /* ,UÅR.V.ı..∆DTÄR. */
+	$"562A F600 0019 8152 0156 35F7 0005 3456"            /* V*ˆ...ÅR.V5˜..4V */
+	$"5252 553E F700 060E 4B53 5254 4405 F700"            /* RRU>˜...KSRTD.˜. */
+	$"0526 5552 5349 08F7 0005 0842 5453 4E0D"            /* .&URSI.˜...BTSN. */
+	$"F800 0019 8052 0011 F800 0403 3755 5418"            /* ¯...ÄR..¯...7UT. */
+	$"F900 0310 4A57 20FA 0002 1552 2AFB 0001"            /* ˘...JW ˙...R*˚.. */
+	$"3E37 FB00 0125 43FB 0002 0E44 07FA 0002"            /* >7˚..%C˚...D.˙.. */
+	$"0333 0EFB 0001 0F0D FF00 FF00 FF00 FF00"            /* .3.˚....ˇ.ˇ.ˇ.ˇ. */
+	$"FF00 FF00 FF00 FF00 A800 FF00 FF00 B000"            /* ˇ.ˇ.ˇ.ˇ.®.ˇ.ˇ.∞. */
+	$"0F08 1222 3645 4D56 6063 5A51 473C 2816"            /* ..."6EMV`cZQG<(. */
+	$"0CE8 0019 0416 3764 94B5 CCDD E7E9 EBEC"            /* .Ë....7dîµÃ›ÁÈÎÏ */
+	$"EDEE ECEB EAE8 E2D4 BEA3 7746 200B E000"            /* ÌÓÏÎÍË‚‘æ£wF .‡. */
+	$"0608 2763 A5D0 E6EE 80F5 8AF6 08F5 F4F0"            /* ..'c•–ÊÓÄıäˆ.ıÙ */
+	$"EAD9 B97C 3B0F DB00 061C 64AF DAED F0F2"            /* ÍŸπ|;.€...dØ⁄ÌÚ */
+	$"81F1 8BF0 83F1 05EE E3C1 8432 07D5 0007"            /* ÅÒãÉÒ.Ó„¡Ñ2.’.. */
+	$"0635 87CD E8ED EDEE 97ED 07EE EEED EBDC"            /* .5áÕËÌÌÓóÌ.ÓÓÌÎ‹ */
+	$"A551 11D1 0007 0333 96D9 E6E9 EAE9 9CE8"            /* •Q.—...3ñŸÊÈÍÈúË */
+	$"06E9 E9E8 E1B8 560E CE00 0628 8CD3 E3E6"            /* .ÈÈË·∏V.Œ..(å”„Ê */
+	$"E5E4 93E3 80E4 87E3 06E4 E5E4 DEAE 4A07"            /* Â‰ì„Ä‰á„.‰Â‰ﬁÆJ. */
+	$"CA00 050D 63C8 DFE1 E196 E084 DF86 E004"            /*  ...c»ﬂ··ñ‡ÑﬂÜ‡. */
+	$"E1E0 D791 23C8 0004 2FA0 D8DC DC81 DA06"            /* ·‡◊ë#»../†ÿ‹‹Å⁄. */
+	$"DBDC D9E0 DEE0 E480 DF87 DA00 D990 DA05"            /* €‹Ÿ‡ﬁ‡‰Äﬂá⁄.Ÿê⁄. */
+	$"DBDC DBC0 5708 C400 0404 54C3 D7D7 80D6"            /* €‹€¿W.ƒ...T√◊◊Ä÷ */
+	$"14D7 DFDF DDA5 45A8 C1A4 7E43 4DB0 DED6"            /* .◊ﬂﬂ›•E®¡§~CM∞ﬁ÷ */
+	$"D9DD D9D8 D6D5 81D6 82D8 83D6 80D8 80D6"            /* Ÿ›Ÿÿ÷’Å÷ÇÿÉ÷ÄÿÄ÷ */
+	$"80D7 04D6 D7D2 8615 C200 1F0E 75CB D4D3"            /* Ä◊.÷◊“Ü.¬...uÀ‘” */
+	$"D2D2 D6DA D489 563E 1305 131C 130D 0506"            /* ““÷⁄‘âV>........ */
+	$"1E9F E0BA 94B4 C9D7 DBD7 D38B D286 D104"            /* .ü‡∫î¥…◊€◊”ã“Ü—. */
+	$"D2D3 D2A3 29C0 0003 1591 CDCF 80CE 06D2"            /* “”“£)¿...ëÕœÄŒ.“ */
+	$"9F8E 800F 0805 8500 0D01 1B8B 4311 152B"            /* üéÄ...Ö....ãC..+ */
+	$"4872 A0C4 D7D7 D185 CE84 CD84 CE03 CFD0"            /* Hr†ƒ◊◊—ÖŒÑÕÑŒ.œ– */
+	$"B739 BE00 0318 96CB CA81 C803 D176 100E"            /* ∑9æ...ñÀ Å».—v.. */
+	$"8900 1001 0F06 0000 0206 0C12 2951 86B8"            /* â...........)QÜ∏ */
+	$"D0D1 CCC9 88C8 83CA 05C8 C9CA CCBA 3FBC"            /* –—Ã…à»É .»… Ã∫?º */
+	$"0004 1B98 C7C5 C381 C402 C7B6 1D93 0009"            /* ...ò«≈√Åƒ.«∂.ì.∆ */
+	$"0207 0F1F 4982 ADC8 CDC7 83C5 84C6 83C5"            /* ....IÇ≠»Õ«É≈Ñ∆É≈ */
+	$"02C7 B842 BA00 0312 91C4 C184 C001 CB87"            /* .«∏B∫...ëƒ¡Ñ¿.Àá */
+	$"9600 0801 060E 193A 75B5 C7C1 86C0 86BF"            /* ñ......:uµ«¡Ü¿Üø */
+	$"02C1 B536 B800 030A 7DC1 BC83 BB04 BAC1"            /* .¡µ6∏..¬}¡ºÉª.∫¡ */
+	$"8160 1F99 0007 040D 266E B9C2 BCBD 82BC"            /* Å`.ô....&nπ¬ºΩÇº */
+	$"87BB 03BC C0A8 25B6 0003 0464 BBB9 81B8"            /* áª.º¿®%∂...dªπÅ∏ */
+	$"80B7 04BE 8B0E 0B04 9B00 0402 0C33 96BF"            /* Ä∑.æã...õ....3ñø */
+	$"8FB8 02BC 9715 B500 0239 B2B5 82B4 04B2"            /* è∏.ºó.µ..9≤µÇ¥.≤ */
+	$"B2B4 AD25 A000 0404 1677 BAB4 87B2 01B3"            /* ≤¥≠%†....w∫¥á≤.≥ */
+	$"B382 B402 B873 06B3 0002 1899 B385 B002"            /* ≥Ç¥.∏s.≥...ô≥Ö∞. */
+	$"B764 02A2 0002 0D68 B78E B002 B1B1 43B2"            /* ∑d.¢...h∑é∞.±±C≤ */
+	$"0002 0671 AE85 AC02 ADA8 1CA4 0002 0B72"            /* ...qÆÖ¨.≠®.§...r */
+	$"B381 AB80 AA01 ACAB 84AC 03AA AD9C 1CB1"            /* ≥Å´Ä™.¨´Ñ¨.™≠ú.± */
+	$"0002 31A6 A780 A682 A702 AE6D 02A5 0006"            /* ..1¶ßÄ¶Çß.Æm.•.. */
+	$"1692 ACA7 A8A6 A685 A800 A781 A602 AD69"            /* .í¨ß®¶¶Ö®.ßÅ¶.≠i */
+	$"04AF 0002 0A7E A681 A282 A301 A629 A600"            /* .Ø..¬~¶Å¢Ç£.¶)¶. */
+	$"0301 2DA1 A381 A280 A386 A202 A4A1 28AF"            /* ..-°£Å¢Ä£Ü¢.§°(Ø */
+	$"0001 3EA2 869E 02A3 7E04 A700 0403 55A6"            /* ..>¢Üû.£~.ß...U¶ */
+	$"9E9E 82A0 869E 03A0 A374 07AD 0002 097C"            /* ûûÇ†Üû.†£t.≠..∆| */
+	$"9D85 9902 9AA1 35A9 0003 1594 9B98 8899"            /* ùÖô.ö°5©...îõòàô */
+	$"819A 029B 9C28 AD00 012C 9986 9802 9D71"            /* Åö.õú(≠..,ôÜò.ùq */
+	$"05A9 0004 0178 9997 9780 9885 9781 9801"            /* .©...xôóóÄòÖóÅò. */
+	$"9D60 AC00 0304 6897 9184 9202 9586 18AB"            /* ù`¨...hóëÑí.ïÜ.´ */
+	$"0001 4599 8F92 018E 18AB 0003 178B 8F8F"            /* ..Eôèí.é.´...ãèè */
+	$"8490 028E 2501 AB00 020C 7E94 8590 808E"            /* Ñê.é%.´...~îÖêÄé */
+	$"8090 048E 8E8F 9642 AB00 0338 9089 8983"            /* Äê.ééèñB´..8êââÉ */
+	$"8A02 8F5D 01AD 0002 2082 9082 8B82 8A80"            /* ä.è].≠.. ÇêÇãÇäÄ */
+	$"8B80 8A02 8F69 04AA 0001 5D8C 8284 8086"            /* ãÄä.èi.™..]åÇÑÄÜ */
+	$"018C 24AE 0007 0119 618A 8D87 8686 8385"            /* .å$Æ....aäçáÜÜÉÖ */
+	$"8286 0287 8114 A900 020E 7982 8281 0483"            /* ÇÜ.áÅ.©...yÇÇÅ.É */
+	$"8385 7202 AF00 0701 0525 597C 8986 8382"            /* ÉÖr.Ø....%Y|âÜÉÇ */
+	$"8283 8301 892E A900 021E 827F 827E 0380"            /* ÇÉÉ.â.©...Ç.Ç~.Ä */
+	$"8085 4AB2 0007 0105 1232 5876 8382 8480"            /* ÄÖJ≤.....2XvÉÇÑÄ */
+	$"037F 7F83 4AA9 0004 3282 7A7B 7B80 7903"            /* ...ÉJ©..2Çz{{Äy. */
+	$"7A79 7F24 B500 0702 050F 2E5A 797F 7D81"            /* zy.$µ......Zy.}Å */
+	$"7A04 7B79 7F5E 03A8 0001 437E 8077 8176"            /* z.{y.^.®..C~ÄwÅv */
+	$"0277 700B B800 0602 0519 446B 7D79 8176"            /* .wp.∏.....Dk}yÅv */
+	$"0277 6B0B A800 014F 7780 7381 7201 7656"            /* .wk.®..OwÄsÅr.vV */
+	$"BB00 0B01 0309 2F60 7975 7373 746F 0FA8"            /* ª....∆/`yussto.® */
+	$"0001 5672 8170 806F 0174 36BE 0008 0205"            /* ..VrÅpÄo.t6æ.... */
+	$"336C 7370 6F70 17A7 0002 0659 6E81 6B80"            /* 3lspop.ß...YnÅkÄ */
+	$"6A01 6F1C C000 0602 155B 6F6B 6D1B A700"            /* j.o.¿....[okm.ß. */
+	$"0306 586A 6880 6704 6666 6762 07C2 0004"            /* ..XjhÄg.ffgb.¬.. */
+	$"0C57 6A6B 1CA7 0003 0655 6765 8264 0165"            /* .Wjk.ß...UgeÇd.e */
+	$"50C4 0003 1562 681B A700 0204 5063 8161"            /* Pƒ...bh.ß...PcÅa */
+	$"0360 6064 33C4 0003 0139 6718 A800 0148"            /* .``d3ƒ...9g.®..H */
+	$"6082 5E02 5D60 18AE 000A 0414 1E25 2D2C"            /* `Ç^.]`.Æ.¬...%-, */
+	$"2A25 1B10 0786 0005 161E 1310 5E12 A800"            /* *%...Ü......^.®. */
+	$"033E 5E59 5980 5802 5A51 01AB 0006 0B27"            /* .>^YYÄX.ZQ.´...' */
+	$"404E 585A 5C80 5A12 5959 5751 4940 372E"            /* @NXZ\ÄZ.YYWQI@7. */
+	$"2820 202A 4158 5B54 2C50 0EA8 0003 3059"            /* (  *AX[T,P.®..0Y */
+	$"5555 8054 0158 3CAA 0008 0B31 4F57 5655"            /* UUÄT.X<™...1OWVU */
+	$"5455 5580 5680 5502 5656 5780 5809 5757"            /* TUUÄVÄU.VVWÄX∆WW */
+	$"5656 5556 5757 4D07 A800 0122 5680 5203"            /* VVUVWWM.®.."VÄR. */
+	$"5151 5526 A800 0408 2B4B 5452 8151 8352"            /* QQU&®...+KTRÅQÉR */
+	$"8251 8052 8251 0352 5254 3FA9 0000 1581"            /* ÇQÄRÇQ.RRT?©...Å */
+	$"5003 4F4F 5118 A600 020B 2A45 8050 834F"            /* P.OOQ.¶...*EÄPÉO */
+	$"8250 834F 0150 5084 4F01 522E A900 020A"            /* ÇPÉO.PPÑO.R.©..¬ */
+	$"454B 814A 014C 0DA4 0003 1431 454C 824B"            /* EKÅJ.L.§...1ELÇK */
+	$"8D4A 824B 814A 014E 1BAA 0001 324A 8047"            /* çJÇKÅJ.N.™..2JÄG */
+	$"0148 43A2 0006 051E 3947 4847 4682 4885"            /* .HC¢....9GHGFÇHÖ */
+	$"4783 4801 4747 8348 8047 0144 0CAA 0001"            /* GÉH.GGÉHÄG.D.™.. */
+	$"1C48 8145 003C A000 0506 233C 4644 4480"            /* .HÅE.<†...#<FDDÄ */
+	$"4382 4583 4482 4586 4481 4501 4733 AB00"            /* CÇEÉDÇEÜDÅE.G3´. */
+	$"010C 3F80 4101 4232 9F00 021E 3A43 8241"            /* ..?ÄA.B2ü...:CÇA */
+	$"8540 8041 0140 4080 4180 4080 4182 4081"            /* Ö@ÄA.@@ÄAÄ@ÄAÇ@Å */
+	$"4101 4420 AC00 052C 403E 3E40 2C9D 0004"            /* A.D ¨..,@>>@,ù.. */
+	$"0E2F 3F3E 3F82 4080 3E80 3F82 4080 3F80"            /* ./?>?Ç@Ä>Ä?Ç@Ä?Ä */
+	$"3E8B 3F01 3D0C AC00 0513 3D3B 3A3D 249C"            /* >ã?.=.¨...=;:=$ú */
+	$"0005 1436 3B3B 3A3A 803B 823A 8039 823B"            /* ...6;;::Ä;Ç:Ä9Ç; */
+	$"833A 803B 873A 013E 26AD 0005 042E 3837"            /* É:Ä;á:.>&≠....87 */
+	$"3920 9B00 0316 3937 3680 3780 3883 3701"            /* 9 õ...976Ä7Ä8É7. */
+	$"3636 8837 8336 8037 8136 0138 0FAE 0004"            /* 66à7É6Ä7Å6.8.Æ.. */
+	$"1536 3437 1D9A 0001 0A32 8133 8034 0135"            /* .647.ö..¬2Å3Ä4.5 */
+	$"3583 3480 3301 3434 8335 8B34 0136 25AF"            /* 5É4Ä3.44É5ã4.6%Ø */
+	$"0004 0327 3232 1D9A 0001 2732 8130 8031"            /* ...'22.ö..'2Å0Ä1 */
+	$"8032 9B31 0130 0DB0 0003 0E2F 2F20 9900"            /* Ä2õ1.0.∞...// ô. */
+	$"0111 2F88 2D80 2E88 2D82 2E87 2D01 301D"            /* ../à-Ä.à-Ç.á-.0. */
+	$"B200 021B 2D22 9900 0121 2B81 2A84 2B80"            /* ≤...-"ô..!+Å*Ñ+Ä */
+	$"2C82 2B80 2A80 2B80 2C85 2B80 2C02 2B28"            /* ,Ç+Ä*Ä+Ä,Ö+Ä,.+( */
+	$"09B2 0002 0624 2498 0000 0583 2784 2880"            /* ∆≤...$$ò...É'Ñ(Ä */
+	$"2997 2800 10B4 0002 0D26 0497 0001 0E26"            /* )ó(..¥...&.ó...& */
+	$"8524 8D25 8224 8325 8124 0127 18B6 0001"            /* Ö$ç%Ç$É%Å$.'.∂.. */
+	$"1408 9700 0114 2385 2184 2280 238B 2280"            /* ..ó...#Ö!Ñ"Ä#ã"Ä */
+	$"2380 2202 231C 04D2 0001 161F 851E 841F"            /* #Ä".#..“....Ö.Ñ. */
+	$"8020 861F 801E 821F 8020 011C 08D3 0001"            /* Ä Ü.Ä.Ç.Ä ...”.. */
+	$"171D 841C 881D 801E 861D 801C 811D 011C"            /* ..Ñ.à.Ä.Ü.Ä.Å... */
+	$"09D4 0001 151A 8819 941A 0319 1A19 0AD5"            /* ∆‘....à.î.....¬’ */
+	$"0001 1317 8816 8717 8018 8717 0218 1609"            /* ....à.á.Ä.á....∆ */
+	$"D600 0111 158B 1487 1580 1684 1501 1307"            /* ÷....ã.á.Ä.Ñ.... */
+	$"D700 000E 9E12 0213 0E04 D800 010A 108B"            /* ◊...û.....ÿ..¬.ã */
+	$"0F8F 1000 0BDA 0001 080E 950D 830E 010D"            /* .è...⁄....ï.É... */
+	$"07DB 0000 049B 0B01 0903 DD00 9909 0107"            /* .€...õ..∆.›.ô∆.. */
+	$"04DF 0098 0600 03E1 0096 03FF 00FF 00FF"            /* .ﬂ.ò...·.ñ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00FF 00FF 00FF 00FF"            /* .ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ.ˇ */
+	$"00FF 00FF 00FF 00FF 00BD 0074 386D 6B00"            /* .ˇ.ˇ.ˇ.ˇ.Ω.t8mk. */
+	$"0040 0800 0000 0000 0000 0000 0000 0000"            /* .@.............. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0006 111F 303C 5376"            /* ............0<Sv */
+	$"8893 9E9F 9388 7653 3C2F 1F11 0600 0000"            /* àìûüìàvS</...... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0923 4564 819C AEBE C9CF D8E2"            /* ....∆#EdÅúÆæ…œÿ‚ */
+	$"E7EA EDEE E9E6 E1D7 CEC8 BBAC 9A81 6344"            /* ÁÍÌÓÈÊ·◊Œ»ª¨öÅcD */
+	$"2409 0000 0000 0000 0000 0000 0000 0000"            /* $∆.............. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0627 5C92 B9D7 E7ED EFF1 F2F2 F4F6 F7F6"            /* .'\íπ◊ÁÌÔÒÚÚÙˆ˜ˆ */
+	$"F7F7 F6F6 F6F5 F4F2 F0F0 EFED ECEB E9E2"            /* ˜˜ˆˆˆıÙÚÔÌÏÎÈ‚ */
+	$"D2B6 8F5B 2807 0000 0000 0000 0000 0000"            /* “∂è[(........... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0006 3275"            /* ..............2u */
+	$"B0D7 EBF0 F2F2 F5F7 F9FB FCFD FEFE FEFE"            /* ∞◊ÎÚÚı˜˘˚¸˝˛˛˛˛ */
+	$"FEFF FFFE FEFE FEFE FDFB F9F6 F4F1 EFED"            /* ˛ˇˇ˛˛˛˛˛˝˚˘ˆÙÒÔÌ */
+	$"ECEB EAE4 D0AC 7332 0700 0000 0000 0000"            /* ÏÎÍ‰–¨s2........ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0022 6AB0 DDEE"            /* ..........."j∞›Ó */
+	$"F2F3 F5F7 F9FC FDFE FFFF FFFF FFFF FFFF"            /* ÚÛı˜˘¸˝˛ˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FEFE FFFF FFFF FEFD FAF6"            /* ˇˇˇˇˇˇ˛˛ˇˇˇˇ˛˝˙ˆ */
+	$"F1EF EDEC ECEA E7D6 AB69 2200 0000 0000"            /* ÒÔÌÏÏÍÁ÷´i"..... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 043E 94D2 EEF3 F4F5"            /* .........>î“ÓÛÙı */
+	$"F6F8 FBFD FFFF FEFE FEFE FEFE FEFE FEFE"            /* ˆ¯˚˝ˇˇ˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFF"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ˇ */
+	$"FEFB F6F1 ECEB ECEC EAE4 CA90 3D04 0000"            /* ˛˚ˆÒÏÎÏÏÍ‰ ê=... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 074F A8E1 F3F5 F5F5 F7FA"            /* .......O®·Ûııı˜˙ */
+	$"FDFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˝˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FFFD F9F3 EDEA EBEB EBE8 D8A2 4E08"            /* ˛˛ˇ˝˘ÛÌÍÎÎÎËÿ¢N. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 054D B0E8 F4F5 F5F4 F7FB FDFE"            /* .....M∞ËÙııÙ˜˚˝˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FAF3 ECEA EAEB EAE9 DEA9"            /* ˛˛˛˛˛˛˙ÛÏÍÍÎÍÈﬁ© */
+	$"4C06 0000 0000 0000 0000 0000 0000 0000"            /* L............... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 003B A8E7 F3F6 F6F4 F7FB FDFE FEFE"            /* ...;®ÁÛˆˆÙ˜˚˝˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FDFD"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˝˝ */
+	$"FDFD FDFD FDFD FDFD F9F2 EBE9 EAEA EAEA"            /* ˝˝˝˝˝˝˝˝˘ÚÎÈÍÍÍÍ */
+	$"DCA1 3B00 0000 0000 0000 0000 0000 0000"            /* ‹°;............. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0020 8DE0 F4F6 F6F5 F6F9 FDFE FEFE FEFE"            /* . ç‡Ùˆˆıˆ˘˝˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFD FDFD FDFD FDFD FDFD"            /* ˛˛˛˛˛˛˛˝˝˝˝˝˝˝˝˝ */
+	$"FDFD FDFD FDFD FDFD FDFD F7EE E9EA EAE9"            /* ˝˝˝˝˝˝˝˝˝˝˜ÓÈÍÍÈ */
+	$"EAE9 D589 2000 0000 0000 0000 0000 0000"            /* ÍÈ’â ........... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0004"            /* ................ */
+	$"5CCB F4F6 F6F5 F5F7 FCFE FEFE FEFE FEFE"            /* \ÀÙˆˆıı˜¸˛˛˛˛˛˛˛ */
+	$"FEFE FEFE FEFE FEFE FEFE FEFE FEFE FEFE"            /* ˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛˛ */
+	$"FEFD FDFD FDFD FDFD FDFD FDFD FDFD FDFD"            /* ˛˝˝˝˝˝˝˝˝˝˝˝˝˝˝˝ */
+	$"FDFD FDFD FDFD FDFD FDFD FDFA F3EA EAE9"            /* ˝˝˝˝˝˝˝˝˝˝˝˙ÛÍÍÈ */
+	$"E9E9 E9E6 C35B 0500 0000 0000 0000 0000"            /* ÈÈÈÊ√[.......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 239F"            /* ..............#ü */
+	$"EBF5 F6F5 F5F5 F9FE FEFE FEFE FEFE FEFE"            /* Îıˆııı˘˛˛˛˛˛˛˛˛˛ */
+	$"FEFE FFFE FEFE FEFF FFFE FEFE FDFD FDFD"            /* ˛˛ˇ˛˛˛˛ˇˇ˛˛˛˝˝˝˝ */
+	$"FDFD FDFD FDFD FDFD FDFD FDFD FDFD FDFD"            /* ˝˝˝˝˝˝˝˝˝˝˝˝˝˝˝˝ */
+	$"FDFD FDFD FDFD FDFC FCFC FCFC FCF6 ECE9"            /* ˝˝˝˝˝˝˝¸¸¸¸¸¸ˆÏÈ */
+	$"E9E9 E9E9 E9DE 9923 0000 0000 0000 0000"            /* ÈÈÈÈÈﬁô#........ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0052 CBF5"            /* .............RÀı */
+	$"F6F6 F5F5 F6FA FEFE FEFE FEFE FEFE FEFF"            /* ˆˆııˆ˙˛˛˛˛˛˛˛˛˛ˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FDFD FDFE FDFD"            /* ˇˇˇˇˇˇˇˇˇˇ˝˝˝˛˝˝ */
+	$"FDFD FDFD FDFD FDFD FDFD FDFD FDFD FDFD"            /* ˝˝˝˝˝˝˝˝˝˝˝˝˝˝˝˝ */
+	$"FDFD FDFD FCFC FCFC FCFC FCFC FCFD F8EF"            /* ˝˝˝˝¸¸¸¸¸¸¸¸¸˝¯Ô */
+	$"E9E9 E9E9 E8E9 E7C3 5100 0000 0000 0000"            /* ÈÈÈÈËÈÁ√Q....... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 000B 81E6 F5F6"            /* ............ÅÊıˆ */
+	$"F5F5 F5F6 FBFE FEFE FEFE FEFE FEFE FFFF"            /* ıııˆ˚˛˛˛˛˛˛˛˛˛ˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFE FEFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇ˛˛ˇˇˇ */
+	$"FEFE FDFD FDFD FDFD FDFD FDFD FDFD FDFC"            /* ˛˛˝˝˝˝˝˝˝˝˝˝˝˝˝¸ */
+	$"FCFC FCFC FCFC FCFC FCFC FCFC FCFC FCFA"            /* ¸¸¸¸¸¸¸¸¸¸¸¸¸¸¸˙ */
+	$"F0E9 E9E9 E8E8 E9E9 DC7E 0C00 0000 0000"            /* ÈÈÈËËÈÈ‹~...... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 20A7 F1F6 F6F5"            /* .......... ßÒˆˆı */
+	$"F5F5 F6FB FDFD FDFD FDFD FDFE FFFF FFFF"            /* ııˆ˚˝˝˝˝˝˝˝˛ˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FEFE FDFD FDFD FDFD FCFC FCFC"            /* ˇˇˇˇ˛˛˝˝˝˝˝˝¸¸¸¸ */
+	$"FCFC FCFC FCFC FCFC FCFC FCFC FCFB FBFB"            /* ¸¸¸¸¸¸¸¸¸¸¸¸¸˚˚˚ */
+	$"FAF1 E9E9 E9E8 E8E8 E8E4 A120 0000 0000"            /* ˙ÒÈÈÈËËËË‰° .... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0034 C4F5 F5F5 F5F5"            /* .........4ƒııııı */
+	$"F5F7 FBFD FDFD FDFD FDFD FDFD FFFF FFFF"            /* ı˜˚˝˝˝˝˝˝˝˝˝ˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFE FEFD FCFC FCFC FCFC"            /* ˇˇˇˇˇˇˇ˛˛˝¸¸¸¸¸¸ */
+	$"FCFC FCFC FCFC FCFC FCFC FBFB FBFB FBFB"            /* ¸¸¸¸¸¸¸¸¸¸˚˚˚˚˚˚ */
+	$"FBFA F1E8 E9E9 E8E8 E8E9 E8BB 3300 0000"            /* ˚˙ÒËÈÈËËËÈËª3... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 49D4 F6F5 F4F5 F5F5"            /* ........I‘ˆıÙııı */
+	$"F6FB FDFD FDFD FDFD FDFD FDFD FDFF FFFF"            /* ˆ˚˝˝˝˝˝˝˝˝˝˝˝ˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FEFD FCFC FCFC"            /* ˇˇˇˇˇˇˇˇˇˇ˛˝¸¸¸¸ */
+	$"FCFC FCFC FCFC FCFB FBFB FBFB FBFB FBFB"            /* ¸¸¸¸¸¸¸˚˚˚˚˚˚˚˚˚ */
+	$"FBFB FBEF E8E9 E9E8 E8E7 E8E9 CB47 0000"            /* ˚˚˚ÔËÈÈËËÁËÈÀG.. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0055 E0F6 F5F4 F5F5 F5F5"            /* .......U‡ˆıÙıııı */
+	$"FAFD FDFD FDFD FDFD FDFD FDFE FEFE FFFF"            /* ˙˝˝˝˝˝˝˝˝˝˝˛˛˛ˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFD FCFC"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇ˝¸¸ */
+	$"FCFC FCFB FBFB FBFB FBFB FBFB FBFB FBFB"            /* ¸¸¸˚˚˚˚˚˚˚˚˚˚˚˚˚ */
+	$"FBFB FAF9 EEE8 E9E8 E8E7 E7E8 E9D5 5400"            /* ˚˚˙˘ÓËÈËËÁÁËÈ’T. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 5EE4 F7F5 F4F4 F5F5 F5F9"            /* ......^‰˜ıÙÙııı˘ */
+	$"FDFD FDFD FDFD FDFD FDFD FDFF FFFF FFFF"            /* ˝˝˝˝˝˝˝˝˝˝˝ˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FEFD"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˛˝ */
+	$"FCFC FCFB FBFB FBFB FBFB FBFB FBFB FBFA"            /* ¸¸¸˚˚˚˚˚˚˚˚˚˚˚˚˙ */
+	$"FAFA FAFA F7EC E9E9 E8E8 E7E7 E8E8 D85C"            /* ˙˙˙˙˜ÏÈÈËËÁÁËËÿ\ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 005E E6F5 F4F4 F4F4 F5F5 F7FD"            /* .....^ÊıÙÙÙÙıı˜˝ */
+	$"FDFD FDFD FDFD FDFD FDFD FFFF FFFF FFFF"            /* ˝˝˝˝˝˝˝˝˝˝ˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FDFB FBFB FBFB FBFB FBFB FBFB FBFA FAFA"            /* ˝˚˚˚˚˚˚˚˚˚˚˚˚˙˙˙ */
+	$"FAFA FAFA FBF5 E9E9 E9E8 E8E7 E7E8 E8DC"            /* ˙˙˙˙˚ıÈÈÈËËÁÁËË‹ */
+	$"5B00 0000 0000 0000 0000 0000 0000 0000"            /* [............... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 56E5 F6F3 F4F4 F4F5 F5F6 FCFD"            /* ....VÂˆÛÙÙÙııˆ¸˝ */
+	$"FDFD FDFD FDFD FDFD FDFE FFFF FFFF FFFF"            /* ˝˝˝˝˝˝˝˝˝˛ˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFD FBFB FBFB FBFB FBFB FBFA FAFA FAFA"            /* ˇ˝˚˚˚˚˚˚˚˚˚˙˙˙˙˙ */
+	$"FAFA FAFA FAF9 F0E8 E9E8 E8E7 E7E7 E8E8"            /* ˙˙˙˙˙˘ËÈËËÁÁÁËË */
+	$"DA54 0000 0000 0000 0000 0000 0000 0000"            /* ⁄T.............. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0048 E1F6 F3F4 F4F4 F4F5 F5F9 FCFD"            /* ...H·ˆÛÙÙÙÙıı˘¸˝ */
+	$"FDFD FDFD FDFD FDFD FDFF FFFF FFFF FFFF"            /* ˝˝˝˝˝˝˝˝˝ˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FCFB FBFB FBFB FAFA FAFA FAFA FAFA"            /* ˇˇ¸˚˚˚˚˚˙˙˙˙˙˙˙˙ */
+	$"FAFA FAFA F9F9 F8ED E9E9 E8E8 E7E7 E7E8"            /* ˙˙˙˙˘˘¯ÌÈÈËËÁÁÁË */
+	$"E8D6 4600 0000 0000 0000 0000 0000 0000"            /* Ë÷F............. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 33D6 F7F3 F3F4 F4F4 F4F5 F7FC FCFC"            /* ..3÷˜ÛÛÙÙÙÙı˜¸¸¸ */
+	$"FCFC FCFD FDFD FDFD FEFF FFFF FFFF FFFF"            /* ¸¸¸˝˝˝˝˝˛ˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFB FBFB FAFA FAFA FAFA FAFA FAFA"            /* ˇˇˇ˚˚˚˙˙˙˙˙˙˙˙˙˙ */
+	$"FAF9 F9F9 F9F9 F9F4 E9E9 E8E8 E8E7 E7E7"            /* ˙˘˘˘˘˘˘ÙÈÈËËËÁÁÁ */
+	$"E7E8 CD32 0000 0000 0000 0000 0000 0000"            /* ÁËÕ2............ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"001F C5F6 F3F3 F3F4 F4F4 F4F5 FBFC FCFC"            /* ..≈ˆÛÛÛÙÙÙÙı˚¸¸¸ */
+	$"FCFC FDFD FDFD FDFD FEFF FFFF FFFF FFFF"            /* ¸¸˝˝˝˝˝˝˛ˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFE FAFA FAFA FAFA FAFA FAFA FAF9"            /* ˇˇˇ˛˙˙˙˙˙˙˙˙˙˙˙˘ */
+	$"F9F9 F9F9 F9F9 F9F8 EEE8 E9E8 E8E7 E7E7"            /* ˘˘˘˘˘˘˘¯ÓËÈËËÁÁÁ */
+	$"E7E7 EABD 1F00 0000 0000 0000 0000 0000"            /* ÁÁÍΩ............ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0BA9 F7F3 F3F3 F3F4 F4F4 F4F8 FCFC FCFC"            /* .©˜ÛÛÛÛÙÙÙÙ¯¸¸¸¸ */
+	$"FCFC FCFC FCFC FDFD FFFF FFFF FFFF FFFF"            /* ¸¸¸¸¸¸˝˝ˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FCFA FAFA FAFA FAFA FAF9 F9F9"            /* ˇˇˇˇ¸˙˙˙˙˙˙˙˙˘˘˘ */
+	$"F9F9 F9F9 F9F9 F8F8 F4EA E9E9 E8E8 E7E7"            /* ˘˘˘˘˘˘¯¯ÙÍÈÈËËÁÁ */
+	$"E7E6 E7EB A20B 0000 0000 0000 0000 0000"            /* ÁÊÁÎ¢........... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"81F4 F3F2 F3F3 F3F4 F4F4 F5FB FCFC FCFC"            /* ÅÙÛÚÛÛÛÙÙÙı˚¸¸¸¸ */
+	$"FCFC FCFC FCFC FDFE FFFF FFFF FFFF FFFF"            /* ¸¸¸¸¸¸˝˛ˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FEFA FAFA FAFA FAF9 F9F9 F9F9"            /* ˇˇˇˇ˛˙˙˙˙˙˙˘˘˘˘˘ */
+	$"F9F9 F9F9 F8F8 F8F8 F8EE E8E9 E8E8 E7E7"            /* ˘˘˘˘¯¯¯¯¯ÓËÈËËÁÁ */
+	$"E7E7 E6E7 E87C 0100 0000 0000 0000 0000"            /* ÁÁÊÁË|.......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0050"            /* ...............P */
+	$"EBF4 F2F2 F3F3 F3F4 F4F4 F7FC FCFC FCFC"            /* ÎÙÚÚÛÛÛÙÙÙ˜¸¸¸¸¸ */
+	$"FCFC FCFC FCFD FDFF FFFF FFFF FFFF FFFF"            /* ¸¸¸¸¸˝˝ˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFB FAFA FAF9 F9F9 F9F9 F9F9"            /* ˇˇˇˇˇ˚˙˙˙˘˘˘˘˘˘˘ */
+	$"F9F9 F8F8 F8F8 F8F8 F8F3 E9E9 E9E8 E8E7"            /* ˘˘¯¯¯¯¯¯¯ÛÈÈÈËËÁ */
+	$"E7E7 E6E6 E9E0 4D00 0000 0000 0000 0000"            /* ÁÁÊÊÈ‡M......... */
+	$"0000 0000 0000 0000 0000 0000 0000 24D8"            /* ..............$ÿ */
+	$"F5F2 F2F2 F3F3 F3F4 F4F4 FAFB FBFC FCFC"            /* ıÚÚÚÛÛÛÙÙÙ˙˚˚¸¸¸ */
+	$"FCFC FCFC FCFD FFFF FFFF FFFF FFFF FFFF"            /* ¸¸¸¸¸˝ˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFC FAFA FAF9 F9F9 F9F9 F9F9"            /* ˇˇˇˇˇ¸˙˙˙˘˘˘˘˘˘˘ */
+	$"F9F9 F8F8 F8F8 F8F8 F7F7 EDE9 E9E8 E8E8"            /* ˘˘¯¯¯¯¯¯˜˜ÌÈÈËËË */
+	$"E7E7 E7E6 E6E9 CF23 0000 0000 0000 0000"            /* ÁÁÁÊÊÈœ#........ */
+	$"0000 0000 0000 0000 0000 0000 0005 AAF9"            /* ..............™˘ */
+	$"F1F2 F2F2 F3F3 F3F4 F4F5 FBFB FBFC FCFC"            /* ÒÚÚÚÛÛÛÙÙı˚˚˚¸¸¸ */
+	$"FCFC FCFC FCFE FFFF FFFF FFFF FFFF FFFF"            /* ¸¸¸¸¸˛ˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFE FBF9 F9F9 F9F9 F9F9 F9F9"            /* ˇˇˇˇˇ˛˚˘˘˘˘˘˘˘˘˘ */
+	$"F8F8 F8F8 F8F8 F8F8 F7F8 F0E9 E9E9 E8E8"            /* ¯¯¯¯¯¯¯¯˜¯ÈÈÈËË */
+	$"E8E7 E7E7 E6E6 EDA3 0500 0000 0000 0000"            /* ËÁÁÁÊÊÌ£........ */
+	$"0000 0000 0000 0000 0000 0000 0060 F6F1"            /* .............`ˆÒ */
+	$"F1F2 F2F2 F3F3 F3F4 F4F7 FBFB FBFC FCFC"            /* ÒÚÚÚÛÛÛÙÙ˜˚˚˚¸¸¸ */
+	$"FCFC FCFC FDFF FFFF FFFF FFFF FFFF FFFF"            /* ¸¸¸¸˝ˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FEFA F9F9 F9F9 F9F9 F8F8"            /* ˇˇˇˇˇˇ˛˙˘˘˘˘˘˘¯¯ */
+	$"F8F8 F8F8 F8F8 F7F7 F7F7 F5EA E9E9 E8E8"            /* ¯¯¯¯¯¯˜˜˜˜ıÍÈÈËË */
+	$"E8E7 E7E7 E6E6 E6EA 5D00 0000 0000 0000"            /* ËÁÁÁÊÊÊÍ]....... */
+	$"0000 0000 0000 0000 0000 0000 1FD8 F4F1"            /* .............ÿÙÒ */
+	$"F1F2 F2F2 F3F3 F3F4 F4F9 FBFB FBFB FBFB"            /* ÒÚÚÚÛÛÛÙÙ˘˚˚˚˚˚˚ */
+	$"FCFC FCFC FEFF FFFF FFFF FFFF FFFF FFFF"            /* ¸¸¸¸˛ˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFE FBF9 F9F9 F9F9 F8F8"            /* ˇˇˇˇˇˇˇ˛˚˘˘˘˘˘¯¯ */
+	$"F8F8 F8F8 F7F7 F7F7 F7F7 F6EC E9E9 E9E8"            /* ¯¯¯¯˜˜˜˜˜˜ˆÏÈÈÈË */
+	$"E8E8 E7E7 E7E6 E6E9 D01F 0000 0000 0000"            /* ËËÁÁÁÊÊÈ–....... */
+	$"0000 0000 0000 0000 0000 0000 96F9 F0F1"            /* ............ñ˘Ò */
+	$"F1F2 F2F2 F3F3 F3F4 F5FA FBFB FBFB FBFB"            /* ÒÚÚÚÛÛÛÙı˙˚˚˚˚˚˚ */
+	$"FCFC FCFC FFFF FFFF FFFF FFFF FFFF FFFF"            /* ¸¸¸¸ˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFD FBF9 F8F8 F8F8"            /* ˇˇˇˇˇˇˇˇˇ˝˚˘¯¯¯¯ */
+	$"F8F8 F7F8 F7F7 F7F7 F7F7 F7EF E9E9 E9E9"            /* ¯¯˜¯˜˜˜˜˜˜˜ÔÈÈÈÈ */
+	$"E8E8 E7E7 E7E6 E6E6 ED91 0000 0000 0000"            /* ËËÁÁÁÊÊÊÌë...... */
+	$"0000 0000 0000 0000 0000 003C ECF1 F0F1"            /* ...........<ÏÒÒ */
+	$"F1F2 F2F2 F3F3 F3F3 F6FA FBFB FBFB FBFB"            /* ÒÚÚÚÛÛÛÛˆ˙˚˚˚˚˚˚ */
+	$"FCFC FCFD FFFF FFFF FFFF FFFF FFFF FFFF"            /* ¸¸¸˝ˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFE FCFA F8F8"            /* ˇˇˇˇˇˇˇˇˇˇˇ˛¸˙¯¯ */
+	$"F8F8 F7F7 F7F7 F7F7 F6F6 F6F1 EAE9 E9E9"            /* ¯¯˜˜˜˜˜˜ˆˆˆÒÍÈÈÈ */
+	$"E8E8 E8E7 E7E7 E6E6 E6E2 3C00 0000 0000"            /* ËËËÁÁÁÊÊÊ‚<..... */
+	$"0000 0000 0000 0000 0000 06B4 F7F0 F0F1"            /* ...........¥˜Ò */
+	$"F1F1 F2F2 F2F3 F3F3 F7FA FBFB FBFB FBFB"            /* ÒÒÚÚÚÛÛÛ˜˙˚˚˚˚˚˚ */
+	$"FBFB FBFE FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚˚˚˛ˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FEFC"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˛¸ */
+	$"F9F7 F7F7 F7F7 F7F7 F6F6 F6F4 EAEA E9E9"            /* ˘˜˜˜˜˜˜˜ˆˆˆÙÍÍÈÈ */
+	$"E9E8 E8E8 E7E7 E6E6 E6EB AD07 0000 0000"            /* ÈËËËÁÁÊÊÊÎ≠..... */
+	$"0000 0000 0000 0000 0000 50F2 F0F0 F0F1"            /* ..........PÚÒ */
+	$"F1F1 F2F2 F2F3 F3F3 F8FA FAFA FAFB FBFB"            /* ÒÒÚÚÚÛÛÛ¯˙˙˙˙˚˚˚ */
+	$"FBFB FBFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚˚˚ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFD FAF8 F7F7 F6F6 F6F6 F6F5 EBEA E9E9"            /* ˇ˝˙¯˜˜ˆˆˆˆˆıÎÍÈÈ */
+	$"E9E8 E8E8 E7E7 E7E6 E6E6 E94E 0000 0000"            /* ÈËËËÁÁÁÊÊÊÈN.... */
+	$"0000 0000 0000 0000 0009 BCF5 F0F0 F0F1"            /* .........∆ºıÒ */
+	$"F1F1 F2F2 F2F3 F3F3 F8FA FAFA FAFB FBFB"            /* ÒÒÚÚÚÛÛÛ¯˙˙˙˙˚˚˚ */
+	$"FBFB FCFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚˚¸ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFE FBF8 F6F6 F6F6 F6F6 ECEA EAE9"            /* ˇˇˇ˛˚¯ˆˆˆˆˆˆÏÍÍÈ */
+	$"E9E9 E8E8 E7E7 E7E6 E6E6 EBB6 0900 0000"            /* ÈÈËËÁÁÁÊÊÊÎ∂∆... */
+	$"0000 0000 0000 0000 0052 F4F0 EFF0 F0F1"            /* .........RÙÔÒ */
+	$"F1F1 F2F2 F2F3 F3F3 F9FA FAFA FAFA FBFB"            /* ÒÒÚÚÚÛÛÛ˘˙˙˙˙˙˚˚ */
+	$"FBFB FDFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚˚˝ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFE FAF6 F6F6 F5F5 EDEA EAE9"            /* ˇˇˇˇˇ˛˙ˆˆˆııÌÍÍÈ */
+	$"E9E9 E8E8 E8E7 E7E7 E6E6 E6EA 5100 0000"            /* ÈÈËËËÁÁÁÊÊÊÍQ... */
+	$"0000 0000 0000 0000 05B6 F5EF EFF0 F0F0"            /* .........∂ıÔÔ */
+	$"F1F1 F2F2 F2F3 F3F3 F8F9 F9FA FAFA FAFA"            /* ÒÒÚÚÚÛÛÛ¯˘˘˙˙˙˙˙ */
+	$"FAFB FEFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙˚˛ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFD F7F5 F5F5 EEEA EAEA"            /* ˇˇˇˇˇˇˇ˝˜ıııÓÍÍÍ */
+	$"E9E9 E9E8 E8E7 E7E7 E6E6 E6EB B005 0000"            /* ÈÈÈËËÁÁÁÊÊÊÎ∞... */
+	$"0000 0000 0000 0000 40EE F0EF EFF0 F0F0"            /* ........@ÓÔÔ */
+	$"F1F1 F1F2 F2F2 F3F3 F8F9 F9FA FAFA FAFA"            /* ÒÒÒÚÚÚÛÛ¯˘˘˙˙˙˙˙ */
+	$"FAFB FEFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙˚˛ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FEF6 F5F5 EEEA EAEA"            /* ˇˇˇˇˇˇˇˇ˛ˆııÓÍÍÍ */
+	$"E9E9 E9E8 E8E8 E7E7 E7E6 E6E6 E63E 0000"            /* ÈÈÈËËËÁÁÁÊÊÊÊ>.. */
+	$"0000 0000 0000 0000 9FF6 EEEF EFF0 F0F0"            /* ........üˆÓÔÔ */
+	$"F1F1 F1F2 F2F2 F3F3 F8F9 F9FA FAFA FAFA"            /* ÒÒÒÚÚÚÛÛ¯˘˘˙˙˙˙˙ */
+	$"FAFC FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙¸ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFD F5F5 EEEB EAEA"            /* ˇˇˇˇˇˇˇˇˇ˝ııÓÎÍÍ */
+	$"EAE9 E9E9 E8E8 E7E7 E7E6 E6E6 ED9A 0000"            /* ÍÈÈÈËËÁÁÁÊÊÊÌö.. */
+	$"0000 0000 0000 0021 E1F0 EEEF EFEF F0F0"            /* .......!·ÓÔÔÔ */
+	$"F0F1 F1F2 F2F2 F3F3 F8F9 F9F9 FAFA FAFA"            /* ÒÒÚÚÚÛÛ¯˘˘˘˙˙˙˙ */
+	$"FAFD FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙˝ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF F9F5 EEEB EBEA"            /* ˇˇˇˇˇˇˇˇˇˇ˘ıÓÎÎÍ */
+	$"EAE9 E9E9 E8E8 E8E7 E7E6 E6E6 E7DA 2100"            /* ÍÈÈÈËËËÁÁÊÊÊÁ⁄!. */
+	$"0000 0000 0000 0071 F6EF EEEF EFEF F0F0"            /* .......qˆÔÓÔÔÔ */
+	$"F0F1 F1F1 F2F2 F3F3 F7F9 F9F9 F9F9 F9FA"            /* ÒÒÒÚÚÛÛ˜˘˘˘˘˘˘˙ */
+	$"FAFD FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙˝ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFE FDFD FCFB FBFB FBFC FDFE FFFF"            /* ˇˇˇ˛˝˝¸˚˚˚˚¸˝˛ˇˇ */
+	$"FFFF FFFF FFFF FFFC FBFC FDF4 EDEB EBEA"            /* ˇˇˇˇˇˇˇ¸˚¸˝ÙÌÎÎÍ */
+	$"EAEA E9E9 E9E8 E8E7 E7E7 E6E6 E6EE 6E00"            /* ÍÍÈÈÈËËÁÁÁÊÊÊÓn. */
+	$"0000 0000 0000 07C0 F2EE EEEE EFEF F0F0"            /* .......¿ÚÓÓÓÔÔ */
+	$"F0F1 F1F1 F2F2 F2F3 F7F9 F9F9 F9F9 F9FA"            /* ÒÒÒÚÚÚÛ˜˘˘˘˘˘˘˙ */
+	$"FAFE FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙˛ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FEFC FAF9 F9F9 F9F8 F8F8 F7F8 F7F7 F8F8"            /* ˛¸˙˘˘˘˘¯¯¯˜¯˜˜¯¯ */
+	$"F9F9 FAFB FBF9 F7F5 F5F5 F9F5 EDEB EBEB"            /* ˘˘˙˚˚˘˜ııı˘ıÌÎÎÎ */
+	$"EAEA E9E9 E9E8 E8E8 E7E7 E7E6 E6EA BB07"            /* ÍÍÈÈÈËËËÁÁÁÊÊÍª. */
+	$"0000 0000 0000 34EC EFEE EEEE EFEF F0F0"            /* ......4ÏÔÓÓÓÔÔ */
+	$"F0F1 F1F1 F2F2 F2F3 F6F9 F9F9 F9F9 F9FA"            /* ÒÒÒÚÚÚÛˆ˘˘˘˘˘˘˙ */
+	$"FBFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FEFC"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˛¸ */
+	$"FAFA F9F9 F9F9 F9F8 F8F8 F7F7 F7F7 F7F7"            /* ˙˙˘˘˘˘˘¯¯¯˜˜˜˜˜˜ */
+	$"F6F6 F6F5 F5F5 F5F5 F4F4 F4F3 EDEB EBEB"            /* ˆˆˆıııııÙÙÙÛÌÎÎÎ */
+	$"EAEA EAE9 E9E8 E8E8 E7E7 E7E6 E6E6 E333"            /* ÍÍÍÈÈËËËÁÁÁÊÊÊ„3 */
+	$"0000 0000 0000 7CF6 EDED EEEE EFEF EFF0"            /* ......|ˆÌÌÓÓÔÔÔ */
+	$"F0F0 F1F1 F2F2 F2F2 F5F8 F8F8 F8F9 F9F9"            /* ÒÒÚÚÚÚı¯¯¯¯˘˘˘ */
+	$"FCFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ¸ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FEFC FAFA"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˛¸˙˙ */
+	$"FAFA F9F9 F9F8 F8F8 F8F8 F8F7 F7F7 F7F7"            /* ˙˙˘˘˘¯¯¯¯¯¯˜˜˜˜˜ */
+	$"F6F6 F6F5 F5F5 F5F5 F4F4 F4F2 ECEC EBEB"            /* ˆˆˆıııııÙÙÙÚÏÏÎÎ */
+	$"EAEA EAE9 E9E9 E8E8 E8E7 E7E6 E6E6 EF7B"            /* ÍÍÍÈÈÈËËËÁÁÊÊÊÔ{ */
+	$"0000 0000 0006 BFF2 EDED EEEE EEEF EFF0"            /* ......øÚÌÌÓÓÓÔÔ */
+	$"F0F0 F1F1 F1F2 F2F2 F4F8 F8F8 F8F9 F9F9"            /* ÒÒÒÚÚÚÙ¯¯¯¯˘˘˘ */
+	$"FDFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˝ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FEFC FBFA FAFA"            /* ˇˇˇˇˇˇˇˇˇˇ˛¸˚˙˙˙ */
+	$"F9F9 F9F9 F9F9 F8F8 F8F8 F8F7 F7F7 F6F6"            /* ˘˘˘˘˘˘¯¯¯¯¯˜˜˜ˆˆ */
+	$"F6F6 F6F5 F5F5 F5F5 F5F4 F4F1 ECEC EBEB"            /* ˆˆˆııııııÙÙÒÏÏÎÎ */
+	$"EBEA EAEA E9E9 E8E8 E8E7 E7E7 E6E6 EABA"            /* ÎÍÍÍÈÈËËËÁÁÁÊÊÍ∫ */
+	$"0600 0000 0029 E6EE EDED EEEE EEEF EFF0"            /* .....)ÊÓÌÌÓÓÓÔÔ */
+	$"F0F0 F1F1 F1F2 F2F2 F3F7 F7F8 F8F9 F9F9"            /* ÒÒÒÚÚÚÛ˜˜¯¯˘˘˘ */
+	$"FEFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˛ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FEFC FBFB FBFB FAFA"            /* ˇˇˇˇˇˇˇˇ˛¸˚˚˚˚˙˙ */
+	$"FAF9 F9F9 F9F9 F8F8 F8F7 F7F7 F7F7 F6F6"            /* ˙˘˘˘˘˘¯¯¯˜˜˜˜˜ˆˆ */
+	$"F6F6 F6F6 F5F5 F4F4 F4F4 F4F0 ECEC ECEB"            /* ˆˆˆˆııÙÙÙÙÙÏÏÏÎ */
+	$"EBEA EAEA E9E9 E9E8 E8E7 E7E7 E6E6 E7DF"            /* ÎÍÍÍÈÈÈËËÁÁÁÊÊÁﬂ */
+	$"2800 0000 0063 F5EC EDED EDEE EEEE EFEF"            /* (....cıÏÌÌÌÓÓÓÔÔ */
+	$"F0F0 F0F1 F1F1 F2F2 F3F6 F7F8 F8F8 F9F9"            /* ÒÒÒÚÚÛˆ˜¯¯¯˘˘ */
+	$"FEFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˛ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FDFC FCFB FBFB FAFA FAFA"            /* ˇˇˇˇˇˇ˝¸¸˚˚˚˙˙˙˙ */
+	$"FAF9 F9F9 F9F9 F9F8 F8F7 F7F7 F7F7 F7F6"            /* ˙˘˘˘˘˘˘¯¯˜˜˜˜˜˜ˆ */
+	$"F6F6 F5F5 F5F5 F5F4 F4F4 F3EF ECEC ECEB"            /* ˆˆıııııÙÙÙÛÔÏÏÏÎ */
+	$"EBEB EAEA E9E9 E9E8 E8E8 E7E7 E7E6 E6EF"            /* ÎÎÍÍÈÈÈËËËÁÁÁÊÊÔ */
+	$"6000 0000 009E F4EC EDED EDEE EEEE EFEF"            /* `....ûÙÏÌÌÌÓÓÓÔÔ */
+	$"F0F0 F0F1 F1F1 F2F2 F2F5 F7F8 F8F8 F8F9"            /* ÒÒÒÚÚÚı˜¯¯¯¯˘ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FDFC FCFC FCFB FBFB FAFA FAFA"            /* ˇˇˇˇ˝¸¸¸¸˚˚˚˙˙˙˙ */
+	$"FAFA F9F9 F8F8 F8F8 F8F8 F7F7 F6F6 F6F6"            /* ˙˙˘˘¯¯¯¯¯¯˜˜ˆˆˆˆ */
+	$"F6F6 F5F5 F5F4 F4F4 F4F4 F2ED EDEC ECEB"            /* ˆˆıııÙÙÙÙÙÚÌÌÏÏÎ */
+	$"EBEB EAEA EAE9 E9E9 E8E8 E7E7 E7E6 E6EE"            /* ÎÎÍÍÍÈÈÈËËÁÁÁÊÊÓ */
+	$"9B00 0000 0BCB F0EC ECED EDEE EEEE EFEF"            /* õ....ÀÏÏÌÌÓÓÓÔÔ */
+	$"EFF0 F0F0 F1F1 F2F2 F2F4 F7F7 F8F8 F8F9"            /* ÔÒÒÚÚÚÙ˜˜¯¯¯˘ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FEFD FCFC FCFC FCFC FBFB FBFA FAFA"            /* ˇˇ˛˝¸¸¸¸¸¸˚˚˚˙˙˙ */
+	$"FAFA F9F9 F9F8 F8F8 F8F8 F7F7 F7F6 F6F6"            /* ˙˙˘˘˘¯¯¯¯¯˜˜˜ˆˆˆ */
+	$"F6F6 F5F5 F5F4 F4F4 F4F4 F0ED EDEC ECEC"            /* ˆˆıııÙÙÙÙÙÌÌÏÏÏ */
+	$"EBEB EAEA EAE9 E9E9 E8E8 E8E7 E7E6 E6EA"            /* ÎÎÍÍÍÈÈÈËËËÁÁÊÊÍ */
+	$"C50B 0000 2BE6 EDEC ECED EDED EEEE EEEF"            /* ≈...+ÊÌÏÏÌÌÌÓÓÓÔ */
+	$"EFF0 F0F0 F1F1 F1F2 F2F2 F6F7 F7F7 F7FA"            /* ÔÒÒÒÚÚÚˆ˜˜˜˜˙ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFE FDFD FDFC FCFC FCFC FBFB FBFA FAFA"            /* ˇ˛˝˝˝¸¸¸¸¸˚˚˚˙˙˙ */
+	$"F9F9 F9F9 F9F8 F8F8 F7F7 F7F7 F7F6 F6F6"            /* ˘˘˘˘˘¯¯¯˜˜˜˜˜ˆˆˆ */
+	$"F5F5 F5F5 F5F4 F4F4 F3F3 F0ED EDED ECEC"            /* ıııııÙÙÙÛÛÌÌÌÏÏ */
+	$"EBEB EBEA EAEA E9E9 E8E8 E8E7 E7E7 E6E7"            /* ÎÎÎÍÍÍÈÈËËËÁÁÁÊÁ */
+	$"E129 0000 55F3 EBEC ECEC EDED EEEE EEEF"            /* ·)..UÛÎÏÏÏÌÌÓÓÓÔ */
+	$"EFF0 F0F0 F1F1 F1F2 F2F2 F4F7 F7F7 F7FA"            /* ÔÒÒÒÚÚÚÙ˜˜˜˜˙ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FEFE FDFD FDFC FCFC FBFB FBFB FBFA FAFA"            /* ˛˛˝˝˝¸¸¸˚˚˚˚˚˙˙˙ */
+	$"F9F9 F9F9 F9F8 F8F8 F7F7 F7F7 F7F7 F6F6"            /* ˘˘˘˘˘¯¯¯˜˜˜˜˜˜ˆˆ */
+	$"F5F5 F5F4 F4F4 F4F4 F4F1 EEED EDED ECEC"            /* ıııÙÙÙÙÙÙÒÓÌÌÌÏÏ */
+	$"ECEB EBEA EAEA E9E9 E9E8 E8E7 E7E7 E6E6"            /* ÏÎÎÍÍÍÈÈÈËËÁÁÁÊÊ */
+	$"EE52 0000 81F5 EBEC ECEC EDED EDEE EEEF"            /* ÓR..ÅıÎÏÏÏÌÌÌÓÓÔ */
+	$"EFEF F0F0 F0F1 F1F1 F2F2 F3F6 F6F7 F7FA"            /* ÔÔÒÒÒÚÚÛˆˆ˜˜˙ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFE"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˛ */
+	$"FEFE FEFD FDFC FCFC FBFB FBFB FBFB FAFA"            /* ˛˛˛˝˝¸¸¸˚˚˚˚˚˚˙˙ */
+	$"F9F9 F9F9 F9F9 F8F8 F7F7 F7F6 F6F6 F6F6"            /* ˘˘˘˘˘˘¯¯˜˜˜ˆˆˆˆˆ */
+	$"F6F5 F5F4 F4F4 F4F4 F4EF EEED EDED ECEC"            /* ˆııÙÙÙÙÙÙÔÓÌÌÌÏÏ */
+	$"ECEB EBEB EAEA E9E9 E9E8 E8E8 E7E7 E7E6"            /* ÏÎÎÎÍÍÈÈÈËËËÁÁÁÊ */
+	$"F07E 0000 A7F3 EBEB ECEC EDED EDEE EEEE"            /* ~..ßÛÎÎÏÏÌÌÌÓÓÓ */
+	$"EFEF EFF0 F0F1 F1F1 F2F2 F2F4 F6F7 F7FA"            /* ÔÔÔÒÒÒÚÚÚÙˆ˜˜˙ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FEFE"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˛˛ */
+	$"FEFE FEFD FDFD FCFC FCFB FBFB FBFB FAFA"            /* ˛˛˛˝˝˝¸¸¸˚˚˚˚˚˙˙ */
+	$"FAF9 F9F8 F8F8 F8F8 F8F7 F7F7 F6F6 F5F5"            /* ˙˘˘¯¯¯¯¯¯˜˜˜ˆˆıı */
+	$"F5F5 F5F5 F4F4 F3F3 F2EE EEEE EDED EDEC"            /* ııııÙÙÛÛÚÓÓÓÌÌÌÏ */
+	$"ECEB EBEB EAEA EAE9 E9E8 E8E8 E7E7 E7E6"            /* ÏÎÎÎÍÍÍÈÈËËËÁÁÁÊ */
+	$"EDA4 0007 C6EF EBEB ECEC ECED EDED EEEE"            /* Ì§..∆ÔÎÎÏÏÏÌÌÌÓÓ */
+	$"EFEF EFF0 F0F0 F1F1 F2F2 F2F3 F5F6 F7FA"            /* ÔÔÔÒÒÚÚÚÛıˆ˜˙ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFE FEFE"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇ˛˛˛ */
+	$"FEFE FEFD FDFD FCFC FCFB FBFB FBFB FAFA"            /* ˛˛˛˝˝˝¸¸¸˚˚˚˚˚˙˙ */
+	$"FAF9 F9F9 F8F8 F8F8 F8F7 F7F7 F6F6 F6F5"            /* ˙˘˘˘¯¯¯¯¯˜˜˜ˆˆˆı */
+	$"F5F5 F5F5 F4F4 F4F3 F0EE EEEE EDED EDEC"            /* ııııÙÙÙÛÓÓÓÌÌÌÏ */
+	$"ECEC EBEB EAEA EAE9 E9E9 E8E8 E7E7 E7E6"            /* ÏÏÎÎÍÍÍÈÈÈËËÁÁÁÊ */
+	$"EBC3 0716 D9ED EBEB EBEC ECED EDED EEEE"            /* Î√..ŸÌÎÎÎÏÏÌÌÌÓÓ */
+	$"EEEF EFEF F0F0 F1F1 F1F2 F2F2 F4F6 F6F9"            /* ÓÔÔÔÒÒÒÚÚÚÙˆˆ˘ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FEFD FDFD"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˛˝˝˝ */
+	$"FDFD FDFD FDFD FCFC FCFB FBFB FAFA FAFA"            /* ˝˝˝˝˝˝¸¸¸˚˚˚˙˙˙˙ */
+	$"FAF9 F9F9 F8F8 F8F7 F7F7 F7F7 F6F6 F6F5"            /* ˙˘˘˘¯¯¯˜˜˜˜˜ˆˆˆı */
+	$"F5F5 F4F4 F4F4 F4F2 EFEF EEEE EEED EDEC"            /* ııÙÙÙÙÙÚÔÔÓÓÓÌÌÏ */
+	$"ECEC EBEB EBEA EAE9 E9E9 E8E8 E8E7 E7E7"            /* ÏÏÎÎÎÍÍÈÈÈËËËÁÁÁ */
+	$"E8D6 162B E7EB EBEB EBEC ECEC EDED EDEE"            /* Ë÷.+ÁÎÎÎÎÏÏÏÌÌÌÓ */
+	$"EEEF EFEF F0F0 F0F1 F1F2 F2F2 F3F5 F6F8"            /* ÓÔÔÔÒÒÚÚÚÛıˆ¯ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FDFD FDFD"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˝˝˝˝ */
+	$"FDFD FDFD FDFD FCFC FCFB FBFB FAFA FAFA"            /* ˝˝˝˝˝˝¸¸¸˚˚˚˙˙˙˙ */
+	$"FAF9 F9F9 F8F8 F8F7 F7F7 F7F7 F6F6 F6F5"            /* ˙˘˘˘¯¯¯˜˜˜˜˜ˆˆˆı */
+	$"F5F5 F4F4 F4F3 F3F0 EFEF EEEE EEED EDED"            /* ııÙÙÙÛÛÔÔÓÓÓÌÌÌ */
+	$"ECEC EBEB EBEA EAEA E9E9 E9E8 E8E7 E7E7"            /* ÏÏÎÎÎÍÍÍÈÈÈËËÁÁÁ */
+	$"E7E3 2A42 EEEA EAEB EBEB ECEC EDED EDEE"            /* Á„*BÓÍÍÎÎÎÏÏÌÌÌÓ */
+	$"EEEE EFEF F0F0 F0F1 F1F1 F2F2 F2F3 F5F7"            /* ÓÓÔÔÒÒÒÚÚÚÛı˜ */
+	$"FEFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˛ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FDFD FDFD"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˝˝˝˝ */
+	$"FDFD FDFD FDFD FCFC FCFB FBFB FAFA FAF9"            /* ˝˝˝˝˝˝¸¸¸˚˚˚˙˙˙˘ */
+	$"F9F9 F9F9 F8F8 F8F7 F7F7 F6F6 F6F6 F6F5"            /* ˘˘˘˘¯¯¯˜˜˜ˆˆˆˆˆı */
+	$"F5F5 F4F4 F4F3 F1F0 EFEF EEEE EEED EDED"            /* ııÙÙÙÛÒÔÔÓÓÓÌÌÌ */
+	$"ECEC ECEB EBEA EAEA E9E9 E9E8 E8E8 E7E7"            /* ÏÏÏÎÎÍÍÍÈÈÈËËËÁÁ */
+	$"E7EA 4153 F2EA EAEB EBEB ECEC ECED EDEE"            /* ÁÍASÚÍÍÎÎÎÏÏÏÌÌÓ */
+	$"EEEE EFEF EFF0 F0F0 F1F1 F1F2 F2F3 F4F5"            /* ÓÓÔÔÔÒÒÒÚÚÛÙı */
+	$"FDFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˝ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFE FDFD FDFD"            /* ˇˇˇˇˇˇˇˇˇˇˇ˛˝˝˝˝ */
+	$"FDFD FDFD FDFD FCFC FCFB FBFB FAFA FAF9"            /* ˝˝˝˝˝˝¸¸¸˚˚˚˙˙˙˘ */
+	$"F9F9 F9F9 F9F8 F8F7 F7F7 F6F6 F6F5 F5F5"            /* ˘˘˘˘˘¯¯˜˜˜ˆˆˆııı */
+	$"F5F5 F5F4 F4F2 F0F0 EFEF EFEE EEEE EDED"            /* ıııÙÙÚÔÔÔÓÓÓÌÌ */
+	$"ECEC ECEB EBEB EAEA E9E9 E9E8 E8E8 E7E7"            /* ÏÏÏÎÎÎÍÍÈÈÈËËËÁÁ */
+	$"E7EE 5274 F1EA EAEA EBEB EBEC ECED EDED"            /* ÁÓRtÒÍÍÍÎÎÎÏÏÌÌÌ */
+	$"EEEE EEEF EFF0 F0F0 F1F1 F1F2 F2F2 F3F4"            /* ÓÓÓÔÔÒÒÒÚÚÚÛÙ */
+	$"FCFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ¸ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFE FDFD FDFD"            /* ˇˇˇˇˇˇˇˇˇˇˇ˛˝˝˝˝ */
+	$"FDFD FDFD FDFC FCFC FCFB FBFB FAFA FAF9"            /* ˝˝˝˝˝¸¸¸¸˚˚˚˙˙˙˘ */
+	$"F9F9 F8F8 F8F8 F8F8 F7F7 F7F6 F6F5 F5F5"            /* ˘˘¯¯¯¯¯¯˜˜˜ˆˆııı */
+	$"F4F4 F4F4 F3F1 F0F0 EFEF EFEE EEEE EDED"            /* ÙÙÙÙÛÒÔÔÔÓÓÓÌÌ */
+	$"EDEC ECEB EBEB EAEA EAE9 E9E9 E8E8 E7E7"            /* ÌÏÏÎÎÎÍÍÍÈÈÈËËÁÁ */
+	$"E7EE 73A7 EEE9 EAEA EBEB EBEC ECEC EDED"            /* ÁÓsßÓÈÍÍÎÎÎÏÏÏÌÌ */
+	$"EDEE EEEF EFEF F0F0 F0F1 F1F1 F2F2 F3F3"            /* ÌÓÓÔÔÔÒÒÒÚÚÛÛ */
+	$"FBFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFE FDFD FDFD"            /* ˇˇˇˇˇˇˇˇˇˇˇ˛˝˝˝˝ */
+	$"FDFD FDFD FDFC FCFC FCFB FBFB FAFA FAF9"            /* ˝˝˝˝˝¸¸¸¸˚˚˚˙˙˙˘ */
+	$"F9F9 F8F8 F8F8 F8F8 F7F7 F7F6 F6F6 F5F5"            /* ˘˘¯¯¯¯¯¯˜˜˜ˆˆˆıı */
+	$"F4F4 F4F3 F1F1 F0F0 F0EF EFEE EEEE EDED"            /* ÙÙÙÛÒÒÔÔÓÓÓÌÌ */
+	$"EDEC ECEC EBEB EAEA EAE9 E9E9 E8E8 E8E7"            /* ÌÏÏÏÎÎÍÍÍÈÈÈËËËÁ */
+	$"E7EB A5C2 ECE9 EAEA EAEB EBEC ECEC EDED"            /* ÁÎ•¬ÏÈÍÍÍÎÎÏÏÏÌÌ */
+	$"EDEE EEEE EFEF F0F0 F0F1 F1F1 F2F2 F2F3"            /* ÌÓÓÓÔÔÒÒÒÚÚÚÛ */
+	$"FBFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFD FCFC FCFC"            /* ˇˇˇˇˇˇˇˇˇˇˇ˝¸¸¸¸ */
+	$"FCFC FCFC FCFC FCFC FCFB FBFB FAFA FAF9"            /* ¸¸¸¸¸¸¸¸¸˚˚˚˙˙˙˘ */
+	$"F9F9 F8F8 F8F7 F7F7 F7F7 F7F6 F6F6 F5F5"            /* ˘˘¯¯¯˜˜˜˜˜˜ˆˆˆıı */
+	$"F4F4 F4F2 F1F1 F0F0 F0EF EFEF EEEE EDED"            /* ÙÙÙÚÒÒÔÔÔÓÓÌÌ */
+	$"EDEC ECEC EBEB EBEA EAE9 E9E9 E8E8 E8E7"            /* ÌÏÏÏÎÎÎÍÍÈÈÈËËËÁ */
+	$"E7E9 BFD1 EBE9 E9EA EAEB EBEB ECEC ECED"            /* ÁÈø—ÎÈÈÍÍÎÎÎÏÏÏÌ */
+	$"EDED EEEE EFEF EFF0 F0F0 F1F1 F1F2 F2F2"            /* ÌÌÓÓÔÔÔÒÒÒÚÚÚ */
+	$"FBFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˚ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFC FCFC FCFC"            /* ˇˇˇˇˇˇˇˇˇˇˇ¸¸¸¸¸ */
+	$"FCFC FCFC FCFC FCFC FBFB FBFB FAFA FAF9"            /* ¸¸¸¸¸¸¸¸˚˚˚˚˙˙˙˘ */
+	$"F9F9 F8F8 F8F7 F7F7 F6F6 F6F6 F6F6 F5F5"            /* ˘˘¯¯¯˜˜˜ˆˆˆˆˆˆıı */
+	$"F5F4 F3F2 F1F1 F0F0 F0EF EFEF EEEE EEED"            /* ıÙÛÚÒÒÔÔÔÓÓÓÌ */
+	$"EDEC ECEC EBEB EBEA EAEA E9E9 E8E8 E8E7"            /* ÌÏÏÏÎÎÎÍÍÍÈÈËËËÁ */
+	$"E7E9 CEE1 E9E9 E9EA EAEA EBEB EBEC ECED"            /* ÁÈŒ·ÈÈÈÍÍÍÎÎÎÏÏÌ */
+	$"EDED EEEE EEEF EFEF F0F0 F0F1 F1F2 F2F2"            /* ÌÌÓÓÓÔÔÔÒÒÚÚÚ */
+	$"FAFF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˙ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFC FCFC FCFC"            /* ˇˇˇˇˇˇˇˇˇˇˇ¸¸¸¸¸ */
+	$"FCFC FCFC FCFC FCFC FBFB FBFB FAFA FAF9"            /* ¸¸¸¸¸¸¸¸˚˚˚˚˙˙˙˘ */
+	$"F9F9 F8F8 F8F7 F7F7 F6F6 F6F5 F5F5 F5F5"            /* ˘˘¯¯¯˜˜˜ˆˆˆııııı */
+	$"F5F3 F2F2 F1F1 F1F0 F0EF EFEF EEEE EEED"            /* ıÛÚÚÒÒÒÔÔÔÓÓÓÌ */
+	$"EDED ECEC EBEB EBEA EAEA E9E9 E9E8 E8E7"            /* ÌÌÏÏÎÎÎÍÍÍÈÈÈËËÁ */
+	$"E7E7 DFE1 E9E9 E9E9 EAEA EBEB EBEC ECEC"            /* ÁÁﬂ·ÈÈÈÈÍÍÎÎÎÏÏÏ */
+	$"EDED EDEE EEEF EFEF F0F0 F0F1 F1F1 F2F1"            /* ÌÌÌÓÓÔÔÔÒÒÒÚÒ */
+	$"F9FF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˘ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFC FBFB FBFB"            /* ˇˇˇˇˇˇˇˇˇˇˇ¸˚˚˚˚ */
+	$"FBFB FBFB FBFB FBFB FBFB FBFA FAFA FAF9"            /* ˚˚˚˚˚˚˚˚˚˚˚˙˙˙˙˘ */
+	$"F9F9 F8F8 F8F7 F7F7 F6F6 F6F5 F5F5 F4F4"            /* ˘˘¯¯¯˜˜˜ˆˆˆıııÙÙ */
+	$"F3F2 F2F2 F1F1 F1F0 F0F0 EFEF EEEE EEED"            /* ÛÚÚÚÒÒÒÔÔÓÓÓÌ */
+	$"EDED ECEC ECEB EBEA EAEA E9E9 E9E8 E8E8"            /* ÌÌÏÏÏÎÎÍÍÍÈÈÈËËË */
+	$"E7E7 E0D0 EAE9 E9E9 EAEA EAEB EBEB ECEC"            /* ÁÁ‡–ÍÈÈÈÍÍÍÎÎÎÏÏ */
+	$"EDED EDEE EEEE EFEF EFF0 F0F0 F1F1 F2F1"            /* ÌÌÌÓÓÓÔÔÔÒÒÚÒ */
+	$"F9FF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˘ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFB FAFB FBFB"            /* ˇˇˇˇˇˇˇˇˇˇˇ˚˙˚˚˚ */
+	$"FBFB FBFB FBFB FAFA FAFA FAFA FAFA FAF9"            /* ˚˚˚˚˚˚˙˙˙˙˙˙˙˙˙˘ */
+	$"F9F9 F8F8 F8F7 F7F7 F6F6 F6F5 F5F5 F4F3"            /* ˘˘¯¯¯˜˜˜ˆˆˆıııÙÛ */
+	$"F3F2 F2F2 F1F1 F1F0 F0F0 EFEF EFEE EEED"            /* ÛÚÚÚÒÒÒÔÔÔÓÓÌ */
+	$"EDED ECEC ECEB EBEB EAEA E9E9 E9E8 E8E8"            /* ÌÌÏÏÏÎÎÎÍÍÈÈÈËËË */
+	$"E7E8 D2C1 EBE8 E9E9 E9EA EAEB EBEB ECEC"            /* ÁË“¡ÎËÈÈÈÍÍÎÎÎÏÏ */
+	$"ECED EDED EEEE EFEF EFF0 F0F0 F1F1 F1F1"            /* ÏÌÌÌÓÓÔÔÔÒÒÒÒ */
+	$"F7FF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ˜ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFB FAFA FAFB"            /* ˇˇˇˇˇˇˇˇˇˇˇ˚˙˙˙˚ */
+	$"FBFB FBFB FAFA FAFA FAFA FAF9 F9F9 F9F9"            /* ˚˚˚˚˙˙˙˙˙˙˙˘˘˘˘˘ */
+	$"F9F9 F8F8 F8F7 F7F7 F6F6 F6F6 F5F5 F4F3"            /* ˘˘¯¯¯˜˜˜ˆˆˆˆııÙÛ */
+	$"F3F2 F2F2 F1F1 F1F0 F0F0 EFEF EFEE EEEE"            /* ÛÚÚÚÒÒÒÔÔÔÓÓÓ */
+	$"EDED ECEC ECEB EBEB EAEA EAE9 E9E9 E8E8"            /* ÌÌÏÏÏÎÎÎÍÍÍÈÈÈËË */
+	$"E7EA C2A6 EDE8 E9E9 E9EA EAEA EBEB EBEC"            /* ÁÍ¬¶ÌËÈÈÈÍÍÍÎÎÎÏ */
+	$"ECED EDED EEEE EEEF EFEF F0F0 F0F1 F1F1"            /* ÏÌÌÌÓÓÓÔÔÔÒÒÒ */
+	$"F4FF FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* Ùˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFC FAFA FAFA"            /* ˇˇˇˇˇˇˇˇˇˇˇ¸˙˙˙˙ */
+	$"FAFA FAFA FAFA FAFA FAFA F9F9 F9F9 F8F8"            /* ˙˙˙˙˙˙˙˙˙˙˘˘˘˘¯¯ */
+	$"F8F8 F8F8 F8F7 F7F7 F6F6 F6F6 F4F4 F4F3"            /* ¯¯¯¯¯˜˜˜ˆˆˆˆÙÙÙÛ */
+	$"F3F3 F2F2 F1F1 F1F0 F0F0 EFEF EFEE EEEE"            /* ÛÛÚÚÒÒÒÔÔÔÓÓÓ */
+	$"EDED ECEC ECEB EBEB EAEA EAE9 E9E9 E8E8"            /* ÌÌÏÏÏÎÎÎÍÍÍÈÈÈËË */
+	$"E7EC A073 F0E8 E8E9 E9E9 EAEA EAEB EBEC"            /* ÁÏ†sËËÈÈÈÍÍÍÎÎÏ */
+	$"ECEC EDED EDEE EEEE EFEF EFF0 F0F1 F1F1"            /* ÏÏÌÌÌÓÓÓÔÔÔÒÒÒ */
+	$"F2FC FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* Ú¸ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFC FAFA FAFA"            /* ˇˇˇˇˇˇˇˇˇˇˇ¸˙˙˙˙ */
+	$"FAFA FAFA FAFA FAFA FAF9 F9F9 F9F9 F8F8"            /* ˙˙˙˙˙˙˙˙˙˘˘˘˘˘¯¯ */
+	$"F8F8 F7F7 F7F6 F6F6 F6F6 F6F5 F4F4 F4F3"            /* ¯¯˜˜˜ˆˆˆˆˆˆıÙÙÙÛ */
+	$"F3F3 F2F2 F1F1 F1F0 F0F0 EFEF EFEE EEEE"            /* ÛÛÚÚÒÒÒÔÔÔÓÓÓ */
+	$"EDED EDEC ECEC EBEB EAEA EAE9 E9E9 E8E8"            /* ÌÌÌÏÏÏÎÎÍÍÍÈÈÈËË */
+	$"E8EF 7752 EFE8 E8E9 E9E9 EAEA EAEB EBEB"            /* ËÔwRÔËËÈÈÈÍÍÍÎÎÎ */
+	$"ECEC ECED EDED EEEE EFEF EFF0 F0F0 F1F1"            /* ÏÏÏÌÌÌÓÓÔÔÔÒÒ */
+	$"F1F6 FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* Òˆˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFD F9F9 FAFA"            /* ˇˇˇˇˇˇˇˇˇˇˇ˝˘˘˙˙ */
+	$"FAFA FAFA FAFA FAFA F9F9 F9F9 F9F9 F8F8"            /* ˙˙˙˙˙˙˙˙˘˘˘˘˘˘¯¯ */
+	$"F8F8 F7F7 F7F6 F6F6 F5F5 F5F5 F4F4 F4F3"            /* ¯¯˜˜˜ˆˆˆııııÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 EFEF EFEE EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓÓ */
+	$"EDED EDEC ECEC EBEB EBEA EAE9 E9E9 E8E8"            /* ÌÌÌÏÏÏÎÎÎÍÍÈÈÈËË */
+	$"E8F0 5641 EBE8 E8E8 E9E9 E9EA EAEA EBEB"            /* ËVAÎËËËÈÈÈÍÍÍÎÎ */
+	$"EBEC ECED EDED EEEE EEEF EFEF F0F0 F0F1"            /* ÎÏÏÌÌÌÓÓÓÔÔÔÒ */
+	$"F1F1 F9FF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ÒÒ˘ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFD F9F9 F9F9"            /* ˇˇˇˇˇˇˇˇˇˇˇ˝˘˘˘˘ */
+	$"F9FA FAFA FAFA F9F9 F9F9 F9F9 F9F8 F8F8"            /* ˘˙˙˙˙˙˘˘˘˘˘˘˘¯¯¯ */
+	$"F8F7 F7F7 F7F6 F6F6 F5F5 F5F5 F4F4 F4F3"            /* ¯˜˜˜˜ˆˆˆııııÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EDED EDEC ECEC EBEB EBEA EAEA E9E9 E9E8"            /* ÌÌÌÏÏÏÎÎÎÍÍÍÈÈÈË */
+	$"E8EC 402A E4E8 E8E8 E9E9 E9EA EAEA EBEB"            /* ËÏ@*‰ËËËÈÈÈÍÍÍÎÎ */
+	$"EBEC ECEC EDED EDEE EEEE EFEF F0F0 F0F1"            /* ÎÏÏÏÌÌÌÓÓÓÔÔÒ */
+	$"F1F1 F1F8 FFFF FFFF FFFF FFFF FFFF FFFF"            /* ÒÒÒ¯ˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFE F9F9 F9F9"            /* ˇˇˇˇˇˇˇˇˇˇˇ˛˘˘˘˘ */
+	$"F9F9 F9F9 F9F9 F9F9 F9F9 F9F9 F8F8 F8F8"            /* ˘˘˘˘˘˘˘˘˘˘˘˘¯¯¯¯ */
+	$"F8F7 F7F7 F7F6 F6F6 F5F5 F5F5 F4F4 F4F3"            /* ¯˜˜˜˜ˆˆˆııııÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDEC ECEC EBEB EBEA EAEA E9E9 E9E8"            /* ÓÌÌÏÏÏÎÎÎÍÍÍÈÈÈË */
+	$"E9E5 2C16 D7E9 E8E8 E8E9 E9E9 EAEA EAEB"            /* ÈÂ,.◊ÈËËËÈÈÈÍÍÍÎ */
+	$"EBEB ECEC ECED EDEE EEEE EFEF EFF0 F0F0"            /* ÎÎÏÏÏÌÌÓÓÓÔÔÔ */
+	$"F1F1 F1F1 F6FE FFFF FFFF FFFF FFFF FFFF"            /* ÒÒÒÒˆ˛ˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF F9F9 F9F9"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˘˘˘˘ */
+	$"F9F9 F9F9 F9F9 F9F9 F9F9 F9F8 F8F8 F8F8"            /* ˘˘˘˘˘˘˘˘˘˘˘¯¯¯¯¯ */
+	$"F8F7 F7F7 F7F6 F6F6 F5F5 F5F5 F4F4 F4F3"            /* ¯˜˜˜˜ˆˆˆııııÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC EBEB EBEA EAEA E9E9 E9E8"            /* ÓÌÌÌÏÏÎÎÎÍÍÍÈÈÈË */
+	$"EAD9 1907 C4EC E7E8 E8E8 E9E9 E9EA EAEB"            /* ÍŸ..ƒÏÁËËËÈÈÈÍÍÎ */
+	$"EBEB ECEC ECED EDED EEEE EEEF EFEF F0F0"            /* ÎÎÏÏÏÌÌÌÓÓÓÔÔÔ */
+	$"F0F1 F1F1 F2F5 FEFF FFFF FFFF FFFF FFFF"            /* ÒÒÒÚı˛ˇˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FAF8 F8F9"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˙¯¯˘ */
+	$"F9F9 F9F9 F9F9 F9F9 F9F9 F8F8 F8F8 F8F8"            /* ˘˘˘˘˘˘˘˘˘˘¯¯¯¯¯¯ */
+	$"F7F7 F7F7 F6F6 F6F6 F5F5 F5F4 F4F4 F4F3"            /* ˜˜˜˜ˆˆˆˆıııÙÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEA EAEA E9E9 E9E8"            /* ÓÌÌÌÏÏÏÎÎÍÍÍÈÈÈË */
+	$"EDC4 0700 A5EE E7E8 E8E8 E9E9 E9EA EAEA"            /* Ìƒ..•ÓÁËËËÈÈÈÍÍÍ */
+	$"EBEB EBEC ECEC EDED EDEE EEEE EFEF EFF0"            /* ÎÎÎÏÏÏÌÌÌÓÓÓÔÔÔ */
+	$"F0F0 F1F1 F2F2 F5FE FFFF FFFF FFFF FFFF"            /* ÒÒÚÚı˛ˇˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FBF8 F8F8"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˚¯¯¯ */
+	$"F8F8 F9F9 F9F9 F9F9 F8F8 F8F8 F8F8 F8F7"            /* ¯¯˘˘˘˘˘˘¯¯¯¯¯¯¯˜ */
+	$"F7F7 F7F7 F6F6 F6F6 F5F5 F5F4 F4F4 F4F3"            /* ˜˜˜˜ˆˆˆˆıııÙÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEA EAEA E9E9 E9E8"            /* ÓÌÌÌÏÏÏÎÎÍÍÍÈÈÈË */
+	$"EFA9 0000 7EF0 E7E7 E8E8 E8E9 E9E9 EAEA"            /* Ô©..~ÁÁËËËÈÈÈÍÍ */
+	$"EAEB EBEB ECEC EDED EDEE EEEE EFEF EFF0"            /* ÍÎÎÎÏÏÌÌÌÓÓÓÔÔÔ */
+	$"F0F0 F1F1 F1F2 F2F5 FEFF FFFF FFFF FFFF"            /* ÒÒÒÚÚı˛ˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FDF8 F8F8"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˝¯¯¯ */
+	$"F8F8 F8F8 F8F8 F8F8 F8F8 F8F8 F8F8 F7F7"            /* ¯¯¯¯¯¯¯¯¯¯¯¯¯¯˜˜ */
+	$"F7F7 F7F6 F6F6 F6F6 F5F5 F5F4 F4F4 F4F3"            /* ˜˜˜ˆˆˆˆˆıııÙÙÙÙÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEB EAEA EAE9 E9E8"            /* ÓÌÌÌÏÏÏÎÎÎÍÍÍÈÈË */
+	$"F282 0000 53EE E7E7 E7E8 E8E8 E9E9 EAEA"            /* ÚÇ..SÓÁÁÁËËËÈÈÍÍ */
+	$"EAEB EBEB ECEC ECED EDED EEEE EEEF EFEF"            /* ÍÎÎÎÏÏÏÌÌÌÓÓÓÔÔÔ */
+	$"EFF0 F0F1 F1F1 F2F2 F6FF FFFF FFFF FFFF"            /* ÔÒÒÒÚÚˆˇˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FEF8 F8F8"            /* ˇˇˇˇˇˇˇˇˇˇˇˇ˛¯¯¯ */
+	$"F8F8 F8F8 F8F8 F8F8 F8F8 F8F8 F8F7 F7F7"            /* ¯¯¯¯¯¯¯¯¯¯¯¯¯˜˜˜ */
+	$"F7F7 F7F6 F6F6 F6F5 F5F5 F5F4 F4F4 F3F3"            /* ˜˜˜ˆˆˆˆııııÙÙÙÛÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEB EAEA EAE9 E9E9"            /* ÓÌÌÌÏÏÏÎÎÎÍÍÍÈÈÈ */
+	$"EF53 0000 2AE1 E7E7 E7E8 E8E8 E9E9 E9EA"            /* ÔS..*·ÁÁÁËËËÈÈÈÍ */
+	$"EAEA EBEB EBEC ECEC EDED EDEE EEEE EFEF"            /* ÍÍÎÎÎÏÏÏÌÌÌÓÓÓÔÔ */
+	$"EFEF F0F0 F1F1 F1F2 F1F6 FFFF FFFF FFFF"            /* ÔÔÒÒÒÚÒˆˇˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFF9 F7F7"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇ˘˜˜ */
+	$"F8F8 F8F8 F8F8 F8F8 F8F8 F8F8 F7F7 F7F7"            /* ¯¯¯¯¯¯¯¯¯¯¯¯˜˜˜˜ */
+	$"F7F7 F6F6 F6F6 F6F5 F5F5 F5F4 F4F4 F3F3"            /* ˜˜ˆˆˆˆˆııııÙÙÙÛÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEB EAEA EAE9 E9EA"            /* ÓÌÌÌÏÏÏÎÎÎÍÍÍÈÈÍ */
+	$"E42B 0000 0BC6 EAE7 E7E7 E8E8 E8E9 E9E9"            /* ‰+...∆ÍÁÁÁËËËÈÈÈ */
+	$"EAEA EAEB EBEB ECEC ECED EDED EEEE EEEF"            /* ÍÍÍÎÎÎÏÏÏÌÌÌÓÓÓÔ */
+	$"EFEF F0F0 F0F1 F1F1 F2F2 F7FF FFFF FFFF"            /* ÔÔÒÒÒÚÚ˜ˇˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFB F7F7"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇ˚˜˜ */
+	$"F7F7 F7F7 F8F8 F8F8 F8F7 F7F7 F7F7 F7F7"            /* ˜˜˜˜¯¯¯¯¯˜˜˜˜˜˜˜ */
+	$"F7F6 F6F6 F6F6 F5F5 F5F5 F5F4 F4F4 F3F3"            /* ˜ˆˆˆˆˆıııııÙÙÙÛÛ */
+	$"F3F3 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÛÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEB EAEA EAE9 E9EE"            /* ÓÌÌÌÏÏÏÎÎÎÍÍÍÈÈÓ */
+	$"C90B 0000 009B EEE6 E7E7 E7E8 E8E8 E9E9"            /* …....õÓÊÁÁÁËËËÈÈ */
+	$"E9EA EAEB EBEB ECEC ECED EDED EEEE EEEF"            /* ÈÍÍÎÎÎÏÏÏÌÌÌÓÓÓÔ */
+	$"EFEF EFF0 F0F0 F1F1 F1F2 F2F9 FFFF FFFF"            /* ÔÔÔÒÒÒÚÚ˘ˇˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFD F7F7"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇ˝˜˜ */
+	$"F7F7 F7F7 F7F7 F7F7 F7F7 F7F7 F7F7 F7F7"            /* ˜˜˜˜˜˜˜˜˜˜˜˜˜˜˜˜ */
+	$"F6F6 F6F6 F6F6 F5F5 F5F5 F4F4 F4F4 F3F3"            /* ˆˆˆˆˆˆııııÙÙÙÙÛÛ */
+	$"F3F2 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÚÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEB EAEA EAE9 E9F1"            /* ÓÌÌÌÏÏÏÎÎÎÍÍÍÈÈÒ */
+	$"9D00 0000 0060 EFE6 E7E7 E7E8 E8E8 E9E9"            /* ù....`ÔÊÁÁÁËËËÈÈ */
+	$"E9EA EAEA EBEB EBEC ECEC EDED EDEE EEEE"            /* ÈÍÍÍÎÎÎÏÏÏÌÌÌÓÓÓ */
+	$"EFEF EFEF F0F0 F1F1 F1F1 F2F2 FAFF FFFF"            /* ÔÔÔÔÒÒÒÒÚÚ˙ˇˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF F8F7"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ¯˜ */
+	$"F7F7 F7F7 F7F7 F7F7 F7F7 F7F7 F7F7 F7F6"            /* ˜˜˜˜˜˜˜˜˜˜˜˜˜˜˜ˆ */
+	$"F6F6 F6F6 F6F5 F5F5 F5F5 F4F4 F4F4 F3F3"            /* ˆˆˆˆˆıııııÙÙÙÙÛÛ */
+	$"F3F2 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÚÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEB EBEB EAEA EAE9 E9F1"            /* ÓÌÌÌÏÏÏÎÎÎÍÍÍÈÈÒ */
+	$"6200 0000 0027 DFE7 E6E7 E7E7 E8E8 E8E9"            /* b....'ﬂÁÊÁÁÁËËËÈ */
+	$"E9E9 EAEA EAEB EBEB ECEC ECED EDED EEEE"            /* ÈÈÍÍÍÎÎÎÏÏÏÌÌÌÓÓ */
+	$"EEEF EFEF EFF0 F0F1 F1F1 F2F2 F2FB FFFF"            /* ÓÔÔÔÔÒÒÒÚÚÚ˚ˇˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FAF6"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˙ˆ */
+	$"F6F6 F7F7 F7F7 F7F7 F7F7 F7F7 F7F6 F6F6"            /* ˆˆ˜˜˜˜˜˜˜˜˜˜˜ˆˆˆ */
+	$"F6F6 F6F6 F6F5 F5F5 F5F4 F4F4 F4F3 F3F3"            /* ˆˆˆˆˆııııÙÙÙÙÛÛÛ */
+	$"F3F2 F2F2 F2F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÚÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EAEA EAE5"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÍÍÍÂ */
+	$"2B00 0000 0005 B9EB E6E6 E7E7 E7E8 E8E9"            /* +.....πÎÊÊÁÁÁËËÈ */
+	$"E9E9 EAEA EAEB EBEB ECEC ECED EDED EDEE"            /* ÈÈÍÍÍÎÎÎÏÏÏÌÌÌÌÓ */
+	$"EEEE EFEF EFEF F0F0 F1F1 F1F2 F2F3 FDFF"            /* ÓÓÔÔÔÔÒÒÒÚÚÛ˝ˇ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FDF6"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˝ˆ */
+	$"F6F6 F6F6 F6F6 F6F7 F7F6 F6F6 F6F6 F6F6"            /* ˆˆˆˆˆˆˆ˜˜ˆˆˆˆˆˆˆ */
+	$"F6F6 F6F6 F5F5 F5F5 F5F4 F4F4 F4F3 F3F3"            /* ˆˆˆˆıııııÙÙÙÙÛÛÛ */
+	$"F3F2 F2F2 F1F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÛÚÚÚÒÒÒÒÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EAEA EFBE"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÍÍÔæ */
+	$"0700 0000 0000 79EF E6E6 E7E7 E7E8 E8E8"            /* ......yÔÊÊÁÁÁËËË */
+	$"E9E9 E9EA EAEA EBEB EBEC ECEC EDED EDEE"            /* ÈÈÈÍÍÍÎÎÎÏÏÏÌÌÌÓ */
+	$"EEEE EEEF EFEF F0F0 F0F1 F1F1 F2F2 F4FE"            /* ÓÓÓÔÔÔÒÒÒÚÚÙ˛ */
+	$"FFFF FFFF FFFF FFFF FFFF FFFF FFFF FFF7"            /* ˇˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˜ */
+	$"F6F6 F6F6 F6F6 F6F6 F6F6 F6F6 F6F6 F6F6"            /* ˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆ */
+	$"F6F6 F5F5 F5F5 F5F5 F4F4 F4F4 F4F3 F3F3"            /* ˆˆııııııÙÙÙÙÙÛÛÛ */
+	$"F2F2 F2F2 F1F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÚÚÚÚÒÒÒÒÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EAEA F27B"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÍÍÚ{ */
+	$"0000 0000 0000 32E3 E6E6 E6E7 E7E7 E8E8"            /* ......2„ÊÊÊÁÁÁËË */
+	$"E8E9 E9E9 EAEA EAEB EBEB ECEC ECED EDED"            /* ËÈÈÈÍÍÍÎÎÎÏÏÏÌÌÌ */
+	$"EEEE EEEF EFEF EFF0 F0F0 F1F1 F1F2 F2F5"            /* ÓÓÓÔÔÔÔÒÒÒÚÚı */
+	$"FEFF FFFF FFFF FFFF FFFF FFFF FFFF FFFA"            /* ˛ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˙ */
+	$"F5F6 F6F6 F6F6 F6F6 F6F6 F6F6 F6F6 F6F6"            /* ıˆˆˆˆˆˆˆˆˆˆˆˆˆˆˆ */
+	$"F6F5 F5F5 F5F5 F5F5 F4F4 F4F4 F3F3 F3F3"            /* ˆıııııııÙÙÙÙÛÛÛÛ */
+	$"F2F2 F2F2 F1F1 F1F1 F0F0 F0EF EFEF EEEE"            /* ÚÚÚÚÒÒÒÒÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EAEA E836"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÍÍË6 */
+	$"0000 0000 0000 07BA EAE6 E6E6 E7E7 E7E8"            /* .......∫ÍÊÊÊÁÁÁË */
+	$"E8E8 E9E9 E9EA EAEA EBEB EBEC ECEC EDED"            /* ËËÈÈÈÍÍÍÎÎÎÏÏÏÌÌ */
+	$"EDEE EEEE EFEF EFEF F0F0 F0F1 F1F1 F2F2"            /* ÌÓÓÓÔÔÔÔÒÒÒÚÚ */
+	$"F7FF FFFF FFFF FFFF FFFF FFFF FFFF FFFD"            /* ˜ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ˝ */
+	$"F5F5 F5F6 F6F6 F6F6 F6F6 F6F6 F6F6 F6F5"            /* ıııˆˆˆˆˆˆˆˆˆˆˆˆı */
+	$"F5F5 F5F5 F5F5 F5F4 F4F4 F4F4 F3F3 F3F3"            /* ıııııııÙÙÙÙÙÛÛÛÛ */
+	$"F2F2 F2F2 F1F1 F1F0 F0F0 F0EF EFEF EEEE"            /* ÚÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EAF0 B907"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÍπ. */
+	$"0000 0000 0000 006D EEE6 E6E6 E7E7 E7E8"            /* .......mÓÊÊÊÁÁÁË */
+	$"E8E8 E9E9 E9EA EAEA EBEB EBEB ECEC ECED"            /* ËËÈÈÈÍÍÍÎÎÎÎÏÏÏÌ */
+	$"EDED EEEE EEEF EFEF EFF0 F0F0 F1F1 F1F1"            /* ÌÌÓÓÓÔÔÔÔÒÒÒÒ */
+	$"F2F9 FFFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* Ú˘ˇˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"F7F5 F5F5 F5F5 F5F5 F5F5 F5F5 F5F5 F5F5"            /* ˜ııııııııııııııı */
+	$"F5F5 F5F5 F5F5 F4F4 F4F4 F4F3 F3F3 F3F2"            /* ııııııÙÙÙÙÙÛÛÛÛÚ */
+	$"F2F2 F2F2 F1F1 F1F0 F0F0 F0EF EFEF EEEE"            /* ÚÚÚÚÒÒÒÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EAF2 6700"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÍÚg. */
+	$"0000 0000 0000 0020 D9E7 E6E6 E6E7 E7E7"            /* ....... ŸÁÊÊÊÁÁÁ */
+	$"E8E8 E8E9 E9E9 EAEA EAEB EBEB ECEC ECED"            /* ËËËÈÈÈÍÍÍÎÎÎÏÏÏÌ */
+	$"EDED EDEE EEEE EFEF EFEF F0F0 F0F1 F1F1"            /* ÌÌÌÓÓÓÔÔÔÔÒÒÒ */
+	$"F1F2 FBFF FFFF FFFF FFFF FFFF FFFF FFFF"            /* ÒÚ˚ˇˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"F9F5 F5F5 F5F5 F5F5 F5F5 F5F5 F5F5 F5F5"            /* ˘ııııııııııııııı */
+	$"F5F5 F5F5 F5F4 F4F4 F4F4 F4F3 F3F3 F3F2"            /* ıııııÙÙÙÙÙÙÛÛÛÛÚ */
+	$"F2F2 F2F1 F1F1 F1F0 F0F0 EFEF EFEF EEEE"            /* ÚÚÚÒÒÒÒÔÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA EDDB 1E00"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÌ€.. */
+	$"0000 0000 0000 0000 9AED E5E6 E6E6 E7E7"            /* ........öÌÂÊÊÊÁÁ */
+	$"E7E8 E8E8 E9E9 E9EA EAEA EBEB EBEC ECEC"            /* ÁËËËÈÈÈÍÍÍÎÎÎÏÏÏ */
+	$"EDED EDEE EEEE EEEF EFEF EFF0 F0F0 F1F1"            /* ÌÌÌÓÓÓÓÔÔÔÔÒÒ */
+	$"F1F1 F3FD FFFF FFFF FFFF FFFF FFFF FFFF"            /* ÒÒÛ˝ˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FDF4 F5F5 F5F5 F5F5 F5F5 F5F5 F5F5 F5F5"            /* ˝Ùıııııııııııııı */
+	$"F5F5 F5F4 F4F4 F4F4 F4F4 F3F3 F3F3 F3F2"            /* ıııÙÙÙÙÙÙÙÛÛÛÛÛÚ */
+	$"F2F2 F2F1 F1F1 F1F0 F0F0 EFEF EFEF EEEE"            /* ÚÚÚÒÒÒÒÔÔÔÔÓÓ */
+	$"EEED EDED EDEC ECEC EBEB EBEA F299 0000"            /* ÓÌÌÌÌÏÏÏÎÎÎÍÚô.. */
+	$"0000 0000 0000 0000 3DE6 E5E6 E6E6 E7E7"            /* ........=ÊÂÊÊÊÁÁ */
+	$"E7E8 E8E8 E8E9 E9E9 EAEA EAEB EBEB ECEC"            /* ÁËËËËÈÈÈÍÍÍÎÎÎÏÏ */
+	$"ECED EDED EEEE EEEE EFEF EFEF F0F0 F0F1"            /* ÏÌÌÌÓÓÓÓÔÔÔÔÒ */
+	$"F1F1 F1F5 FFFF FFFF FFFF FFFF FFFF FFFF"            /* ÒÒÒıˇˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFF6 F4F4 F4F5 F5F5 F5F5 F5F5 F5F5 F5F5"            /* ˇˆÙÙÙııııııııııı */
+	$"F5F4 F4F4 F4F4 F4F4 F4F3 F3F3 F3F3 F2F2"            /* ıÙÙÙÙÙÙÙÙÛÛÛÛÛÚÚ */
+	$"F2F2 F2F1 F1F1 F1F0 F0F0 EFEF EFEF EEEE"            /* ÚÚÚÒÒÒÒÔÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEC EBEB EBEB EC40 0000"            /* ÓÌÌÌÏÏÏÏÎÎÎÎÏ@.. */
+	$"0000 0000 0000 0000 04AF EBE5 E6E6 E6E7"            /* .........ØÎÂÊÊÊÁ */
+	$"E7E7 E8E8 E8E9 E9E9 EAEA EAEB EBEB EBEC"            /* ÁÁËËËÈÈÈÍÍÍÎÎÎÎÏ */
+	$"ECEC EDED EDEE EEEE EFEF EFEF EFF0 F0F0"            /* ÏÏÌÌÌÓÓÓÔÔÔÔÔ */
+	$"F1F1 F1F1 F7FF FFFF FFFF FFFF FFFF FFFF"            /* ÒÒÒÒ˜ˇˇˇˇˇˇˇˇˇˇˇ */
+	$"FFF9 F4F4 F4F4 F4F4 F4F4 F4F4 F4F4 F4F4"            /* ˇ˘ÙÙÙÙÙÙÙÙÙÙÙÙÙÙ */
+	$"F4F4 F4F4 F4F4 F4F4 F3F3 F3F3 F3F3 F2F2"            /* ÙÙÙÙÙÙÙÙÛÛÛÛÛÛÚÚ */
+	$"F2F2 F1F1 F1F1 F0F0 F0F0 EFEF EFEF EEEE"            /* ÚÚÒÒÒÒÔÔÔÔÓÓ */
+	$"EEED EDED ECEC ECEC EBEB EBF1 AF05 0000"            /* ÓÌÌÌÏÏÏÏÎÎÎÒØ... */
+	$"0000 0000 0000 0000 004F EAE5 E5E6 E6E6"            /* .........OÍÂÂÊÊÊ */
+	$"E7E7 E7E8 E8E8 E9E9 E9EA EAEA EBEB EBEC"            /* ÁÁÁËËËÈÈÈÍÍÍÎÎÎÏ */
+	$"ECEC ECED EDED EEEE EEEE EFEF EFEF F0F0"            /* ÏÏÏÌÌÌÓÓÓÓÔÔÔÔ */
+	$"F0F1 F1F1 F1FA FFFF FFFF FFFF FFFF FFFF"            /* ÒÒÒÒ˙ˇˇˇˇˇˇˇˇˇˇ */
+	$"FFFD F4F4 F4F4 F4F4 F4F4 F4F4 F4F4 F4F4"            /* ˇ˝ÙÙÙÙÙÙÙÙÙÙÙÙÙÙ */
+	$"F4F4 F4F4 F4F4 F3F3 F3F3 F3F3 F3F2 F2F2"            /* ÙÙÙÙÙÙÛÛÛÛÛÛÛÚÚÚ */
+	$"F2F2 F1F1 F1F1 F0F0 F0EF EFEF EFEE EEEE"            /* ÚÚÒÒÒÒÔÔÔÔÓÓÓ */
+	$"EEED EDED ECEC ECEC EBEB ECEC 4900 0000"            /* ÓÌÌÌÏÏÏÏÎÎÏÏI... */
+	$"0000 0000 0000 0000 0008 B5EB E5E5 E6E6"            /* ..........µÎÂÂÊÊ */
+	$"E6E7 E7E7 E8E8 E8E9 E9E9 EAEA EAEB EBEB"            /* ÊÁÁÁËËËÈÈÈÍÍÍÎÎÎ */
+	$"ECEC ECEC EDED EDEE EEEE EEEF EFEF EFF0"            /* ÏÏÏÏÌÌÌÓÓÓÓÔÔÔÔ */
+	$"F0F0 F1F1 F1F2 FDFF FFFF FFFF FFFF FFFF"            /* ÒÒÒÚ˝ˇˇˇˇˇˇˇˇˇ */
+	$"FFFF F5F3 F4F4 F4F4 F4F4 F4F4 F4F4 F4F4"            /* ˇˇıÛÙÙÙÙÙÙÙÙÙÙÙÙ */
+	$"F4F4 F4F4 F4F3 F3F3 F3F3 F3F3 F2F2 F2F2"            /* ÙÙÙÙÙÛÛÛÛÛÛÛÚÚÚÚ */
+	$"F2F1 F1F1 F1F1 F0F0 F0EF EFEF EFEE EEEE"            /* ÚÒÒÒÒÒÔÔÔÔÓÓÓ */
+	$"EEED EDED ECEC ECEC EBEB F1B5 0600 0000"            /* ÓÌÌÌÏÏÏÏÎÎÒµ.... */
+	$"0000 0000 0000 0000 0000 4CE9 E5E5 E6E6"            /* ..........LÈÂÂÊÊ */
+	$"E6E7 E7E7 E8E8 E8E8 E9E9 E9EA EAEA EBEB"            /* ÊÁÁÁËËËËÈÈÈÍÍÍÎÎ */
+	$"EBEC ECEC EDED EDED EEEE EEEE EFEF EFEF"            /* ÎÏÏÏÌÌÌÌÓÓÓÓÔÔÔÔ */
+	$"EFF0 F0F0 F1F1 F4FE FFFF FFFF FFFF FFFF"            /* ÔÒÒÙ˛ˇˇˇˇˇˇˇˇ */
+	$"FFFF F9F3 F3F3 F4F4 F4F4 F4F4 F4F4 F4F4"            /* ˇˇ˘ÛÛÛÙÙÙÙÙÙÙÙÙÙ */
+	$"F4F4 F3F3 F3F3 F3F3 F3F3 F3F2 F2F2 F2F2"            /* ÙÙÛÛÛÛÛÛÛÛÛÚÚÚÚÚ */
+	$"F1F1 F1F1 F1F0 F0F0 F0EF EFEF EFEE EEEE"            /* ÒÒÒÒÒÔÔÔÔÓÓÓ */
+	$"EEED EDED ECEC ECEB EBEC EE4D 0000 0000"            /* ÓÌÌÌÏÏÏÎÎÏÓM.... */
+	$"0000 0000 0000 0000 0000 06AB ECE5 E5E6"            /* ...........´ÏÂÂÊ */
+	$"E6E6 E7E7 E7E8 E8E8 E9E9 E9E9 EAEA EAEB"            /* ÊÊÁÁÁËËËÈÈÈÈÍÍÍÎ */
+	$"EBEB ECEC ECEC EDED EDEE EEEE EEEF EFEF"            /* ÎÎÏÏÏÏÌÌÌÓÓÓÓÔÔÔ */
+	$"EFEF F0F0 F0F1 F0F7 FFFF FFFF FFFF FFFF"            /* ÔÔÒ˜ˇˇˇˇˇˇˇˇ */
+	$"FFFF FCF3 F3F3 F3F3 F3F3 F3F3 F3F3 F3F3"            /* ˇˇ¸ÛÛÛÛÛÛÛÛÛÛÛÛÛ */
+	$"F3F3 F3F3 F3F3 F3F3 F3F3 F2F2 F2F2 F2F2"            /* ÛÛÛÛÛÛÛÛÛÛÚÚÚÚÚÚ */
+	$"F1F1 F1F1 F0F0 F0F0 EFEF EFEF EFEE EEEE"            /* ÒÒÒÒÔÔÔÔÔÓÓÓ */
+	$"EDED EDED ECEC ECEB EBF1 B007 0000 0000"            /* ÌÌÌÌÏÏÏÎÎÒ∞..... */
+	$"0000 0000 0000 0000 0000 003A E2E6 E5E5"            /* ...........:‚ÊÂÂ */
+	$"E6E6 E6E7 E7E7 E8E8 E8E9 E9E9 EAEA EAEA"            /* ÊÊÊÁÁÁËËËÈÈÈÍÍÍÍ */
+	$"EBEB EBEC ECEC ECED EDED EEEE EEEE EFEF"            /* ÎÎÎÏÏÏÏÌÌÌÓÓÓÓÔÔ */
+	$"EFEF EFF0 F0F0 F1F1 FBFF FFFF FFFF FFFF"            /* ÔÔÔÒÒ˚ˇˇˇˇˇˇˇ */
+	$"FFFF FFF5 F3F3 F3F3 F3F3 F3F3 F3F3 F3F3"            /* ˇˇˇıÛÛÛÛÛÛÛÛÛÛÛÛ */
+	$"F3F3 F3F3 F3F3 F3F3 F2F2 F2F2 F2F2 F2F1"            /* ÛÛÛÛÛÛÛÛÚÚÚÚÚÚÚÒ */
+	$"F1F1 F1F1 F0F0 F0F0 EFEF EFEF EEEE EEEE"            /* ÒÒÒÒÔÔÔÔÓÓÓÓ */
+	$"EDED EDED ECEC ECEB EDE6 3900 0000 0000"            /* ÌÌÌÌÏÏÏÎÌÊ9..... */
+	$"0000 0000 0000 0000 0000 0000 8EED E5E5"            /* ............éÌÂÂ */
+	$"E5E6 E6E6 E7E7 E7E8 E8E8 E9E9 E9EA EAEA"            /* ÂÊÊÊÁÁÁËËËÈÈÈÍÍÍ */
+	$"EAEB EBEB ECEC ECEC EDED EDEE EEEE EEEF"            /* ÍÎÎÎÏÏÏÏÌÌÌÓÓÓÓÔ */
+	$"EFEF EFEF F0F0 F0F0 F3FE FFFF FFFF FFFF"            /* ÔÔÔÔÛ˛ˇˇˇˇˇˇ */
+	$"FFFF FFF8 F2F3 F3F3 F3F3 F3F3 F3F3 F3F3"            /* ˇˇˇ¯ÚÛÛÛÛÛÛÛÛÛÛÛ */
+	$"F3F3 F3F3 F3F3 F2F2 F2F2 F2F2 F2F2 F1F1"            /* ÛÛÛÛÛÛÚÚÚÚÚÚÚÚÒÒ */
+	$"F1F1 F1F0 F0F0 F0EF EFEF EFEF EEEE EEEE"            /* ÒÒÒÔÔÔÔÔÓÓÓÓ */
+	$"EDED EDED ECEC ECEB F38B 0000 0000 0000"            /* ÌÌÌÌÏÏÏÎÛã...... */
+	$"0000 0000 0000 0000 0000 0000 1ECF E8E5"            /* .............œËÂ */
+	$"E5E6 E6E6 E7E7 E7E7 E8E8 E8E9 E9E9 EAEA"            /* ÂÊÊÊÁÁÁÁËËËÈÈÈÍÍ */
+	$"EAEA EBEB EBEC ECEC EDED EDED EEEE EEEE"            /* ÍÍÎÎÎÏÏÏÌÌÌÌÓÓÓÓ */
+	$"EFEF EFEF EFEF F0F0 F0F6 FFFF FFFF FFFF"            /* ÔÔÔÔÔÔˆˇˇˇˇˇˇ */
+	$"FFFF FFFB F2F2 F2F2 F3F3 F3F3 F3F3 F3F3"            /* ˇˇˇ˚ÚÚÚÚÛÛÛÛÛÛÛÛ */
+	$"F3F3 F3F3 F2F2 F2F2 F2F2 F2F2 F2F1 F1F1"            /* ÛÛÛÛÚÚÚÚÚÚÚÚÚÒÒÒ */
+	$"F1F1 F1F0 F0F0 F0EF EFEF EFEF EEEE EEEE"            /* ÒÒÒÔÔÔÔÔÓÓÓÓ */
+	$"EDED EDEC ECEC ECF0 D31C 0000 0000 0000"            /* ÌÌÌÏÏÏÏ”....... */
+	$"0000 0000 0000 0000 0000 0000 005B E9E5"            /* .............[ÈÂ */
+	$"E5E5 E6E6 E6E7 E7E7 E8E8 E8E8 E9E9 E9EA"            /* ÂÂÊÊÊÁÁÁËËËËÈÈÈÍ */
+	$"EAEA EBEB EBEB ECEC ECEC EDED EDEE EEEE"            /* ÍÍÎÎÎÎÏÏÏÏÌÌÌÓÓÓ */
+	$"EEEF EFEF EFEF EFF0 F0F0 F9FF FFFF FFFF"            /* ÓÔÔÔÔÔÔ˘ˇˇˇˇˇ */
+	$"FFFF FFFE F3F2 F2F2 F2F2 F2F2 F2F2 F2F2"            /* ˇˇˇ˛ÛÚÚÚÚÚÚÚÚÚÚÚ */
+	$"F2F2 F2F2 F2F2 F2F2 F2F2 F2F2 F1F1 F1F1"            /* ÚÚÚÚÚÚÚÚÚÚÚÚÒÒÒÒ */
+	$"F1F1 F0F0 F0F0 EFEF EFEF EFEE EEEE EEED"            /* ÒÒÔÔÔÔÔÓÓÓÓÌ */
+	$"EDED EDEC ECEC EDEF 6200 0000 0000 0000"            /* ÌÌÌÏÏÏÌÔb....... */
+	$"0000 0000 0000 0000 0000 0000 0005 A2EC"            /* ..............¢Ï */
+	$"E5E5 E5E6 E6E6 E7E7 E7E8 E8E8 E8E9 E9E9"            /* ÂÂÂÊÊÊÁÁÁËËËËÈÈÈ */
+	$"EAEA EAEB EBEB EBEC ECEC ECED EDED EEEE"            /* ÍÍÍÎÎÎÎÏÏÏÏÌÌÌÓÓ */
+	$"EEEE EEEF EFEF EFEF F0F0 F1FC FFFF FFFF"            /* ÓÓÓÔÔÔÔÔÒ¸ˇˇˇˇ */
+	$"FFFF FFFF F5F2 F2F2 F2F2 F2F2 F2F2 F2F2"            /* ˇˇˇˇıÚÚÚÚÚÚÚÚÚÚÚ */
+	$"F2F2 F2F2 F2F2 F2F2 F2F2 F1F1 F1F1 F1F1"            /* ÚÚÚÚÚÚÚÚÚÚÒÒÒÒÒÒ */
+	$"F1F0 F0F0 F0F0 EFEF EFEF EFEE EEEE EEED"            /* ÒÔÔÔÔÔÓÓÓÓÌ */
+	$"EDED EDEC ECED F2A1 0500 0000 0000 0000"            /* ÌÌÌÏÏÌÚ°........ */
+	$"0000 0000 0000 0000 0000 0000 0000 22CE"            /* .............."Œ */
+	$"E8E5 E5E5 E6E6 E6E7 E7E7 E8E8 E8E9 E9E9"            /* ËÂÂÂÊÊÊÁÁÁËËËÈÈÈ */
+	$"E9EA EAEA EBEB EBEB ECEC ECED EDED EDEE"            /* ÈÍÍÍÎÎÎÎÏÏÏÌÌÌÌÓ */
+	$"EEEE EEEE EFEF EFEF EFEF F0F3 FFFF FFFF"            /* ÓÓÓÓÔÔÔÔÔÔÛˇˇˇˇ */
+	$"FFFF FFFF F8F1 F2F2 F2F2 F2F2 F2F2 F2F2"            /* ˇˇˇˇ¯ÒÚÚÚÚÚÚÚÚÚÚ */
+	$"F2F2 F2F2 F2F2 F2F2 F1F1 F1F1 F1F1 F1F1"            /* ÚÚÚÚÚÚÚÚÒÒÒÒÒÒÒÒ */
+	$"F0F0 F0F0 F0EF EFEF EFEF EEEE EEEE EEED"            /* ÔÔÔÔÔÓÓÓÓÓÌ */
+	$"EDED ECEC ECEF D01E 0000 0000 0000 0000"            /* ÌÌÏÏÏÔ–......... */
+	$"0000 0000 0000 0000 0000 0000 0000 004C"            /* ...............L */
+	$"DFE7 E5E5 E5E6 E6E6 E7E7 E7E7 E8E8 E9E9"            /* ﬂÁÂÂÂÊÊÊÁÁÁÁËËÈÈ */
+	$"E9E9 EAEA EAEB EBEB EBEC ECEC ECED EDED"            /* ÈÈÍÍÍÎÎÎÎÏÏÏÏÌÌÌ */
+	$"EDEE EEEE EEEF EFEF EFEF EFEF F8FF FFFF"            /* ÌÓÓÓÓÔÔÔÔÔÔÔ¯ˇˇˇ */
+	$"FFFF FFFF FCF1 F1F1 F1F1 F2F2 F2F2 F2F2"            /* ˇˇˇˇ¸ÒÒÒÒÒÚÚÚÚÚÚ */
+	$"F2F2 F2F2 F2F1 F1F1 F1F1 F1F1 F1F1 F1F0"            /* ÚÚÚÚÚÒÒÒÒÒÒÒÒÒÒ */
+	$"F0F0 F0F0 EFEF EFEF EFEF EEEE EEEE EDED"            /* ÔÔÔÔÔÔÓÓÓÓÌÌ */
+	$"EDED ECEC EEE6 4900 0000 0000 0000 0000"            /* ÌÌÏÏÓÊI......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"7AE8 E6E5 E5E6 E6E6 E6E7 E7E7 E7E8 E8E9"            /* zËÊÂÂÊÊÊÊÁÁÁÁËËÈ */
+	$"E9E9 E9EA EAEA EBEB EBEB ECEC ECEC EDED"            /* ÈÈÈÍÍÍÎÎÎÎÏÏÏÏÌÌ */
+	$"EDED EEEE EEEE EFEF EFEF EFEF F0FC FFFF"            /* ÌÌÓÓÓÓÔÔÔÔÔÔ¸ˇˇ */
+	$"FFFF FFFF FEF2 F1F1 F1F1 F1F1 F1F1 F1F1"            /* ˇˇˇˇ˛ÚÒÒÒÒÒÒÒÒÒÒ */
+	$"F1F1 F1F1 F1F1 F1F1 F1F1 F1F1 F1F1 F0F0"            /* ÒÒÒÒÒÒÒÒÒÒÒÒÒÒ */
+	$"F0F0 F0EF EFEF EFEF EFEE EEEE EEEE EDED"            /* ÔÔÔÔÔÔÓÓÓÓÓÌÌ */
+	$"EDED ECEC F4A4 0000 0000 0000 0000 0000"            /* ÌÌÏÏÙ§.......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0AA0 E9E6 E5E5 E6E6 E6E7 E7E7 E7E7 E8E8"            /* ¬†ÈÊÂÂÊÊÊÁÁÁÁÁËË */
+	$"E9E9 E9E9 EAEA EAEB EBEB EBEC ECEC ECED"            /* ÈÈÈÈÍÍÍÎÎÎÎÏÏÏÏÌ */
+	$"EDED EDEE EEEE EEEE EFEF EFEF EFF3 FFFF"            /* ÌÌÌÓÓÓÓÓÔÔÔÔÔÛˇˇ */
+	$"FFFF FFFF FFF4 F0F1 F1F1 F1F1 F1F1 F1F1"            /* ˇˇˇˇˇÙÒÒÒÒÒÒÒÒÒ */
+	$"F1F1 F1F1 F1F1 F1F1 F1F1 F1F1 F0F0 F0F0"            /* ÒÒÒÒÒÒÒÒÒÒÒÒ */
+	$"F0F0 EFEF EFEF EFEF EFEE EEEE EEED EDED"            /* ÔÔÔÔÔÔÔÓÓÓÓÌÌÌ */
+	$"EDED ECEC F1C0 0D00 0000 0000 0000 0000"            /* ÌÌÏÏÒ¿.......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"001D BBE9 E6E5 E5E6 E6E6 E7E7 E7E7 E7E8"            /* ..ªÈÊÂÂÊÊÊÁÁÁÁÁË */
+	$"E8E9 E9E9 E9EA EAEA EBEB EBEB ECEC ECEC"            /* ËÈÈÈÈÍÍÍÎÎÎÎÏÏÏÏ */
+	$"EDED EDED EDEE EEEE EEEF EFEF EFEF F8FF"            /* ÌÌÌÌÌÓÓÓÓÔÔÔÔÔ¯ˇ */
+	$"FFFF FFFF FFF6 EFF1 F1F1 F1F1 F1F1 F1F1"            /* ˇˇˇˇˇˆÔÒÒÒÒÒÒÒÒÒ */
+	$"F1F1 F1F1 F1F1 F1F1 F1F1 F0F0 F0F0 F0F0"            /* ÒÒÒÒÒÒÒÒÒÒ */
+	$"F0EF EFEF EFEF EFEF EEEE EEEE EEED EDED"            /* ÔÔÔÔÔÔÔÓÓÓÓÓÌÌÌ */
+	$"EDEC ECEC ECEF 4A00 0000 0000 0000 0000"            /* ÌÏÏÏÏÔJ......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 31CA E8E6 E5E5 E6E6 E6E7 E7E7 E7E8"            /* ..1 ËÊÂÂÊÊÊÁÁÁÁË */
+	$"E8E8 E9E9 E9E9 EAEA EAEA EBEB EBEC ECEC"            /* ËËÈÈÈÈÍÍÍÍÎÎÎÏÏÏ */
+	$"ECED EDED EDED EEEE EEEE EEEF EFEF F0FC"            /* ÏÌÌÌÌÌÓÓÓÓÓÔÔÔ¸ */
+	$"FFFF FFFF FFF8 F0F0 F0F0 F0F1 F1F1 F1F1"            /* ˇˇˇˇˇ¯ÒÒÒÒÒ */
+	$"F1F1 F1F1 F1F1 F1F0 F0F0 F0F0 F0F0 F0F0"            /* ÒÒÒÒÒÒÒ */
+	$"EFEF EFEF EFEF EFEE EEEE EEEE EDED EDED"            /* ÔÔÔÔÔÔÔÓÓÓÓÓÌÌÌÌ */
+	$"EDEC ECEC ECF3 A003 0000 0000 0000 0000"            /* ÌÏÏÏÏÛ†......... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0044 D4E8 E5E5 E5E6 E6E6 E7E7 E7E7"            /* ...D‘ËÂÂÂÊÊÊÁÁÁÁ */
+	$"E8E8 E8E9 E9E9 E9EA EAEA EBEB EBEB ECEC"            /* ËËËÈÈÈÈÍÍÍÎÎÎÎÏÏ */
+	$"ECEC EDED EDED EDEE EEEE EEEE EFEF EEF4"            /* ÏÏÌÌÌÌÌÓÓÓÓÓÔÔÓÙ */
+	$"FFFF FFFF FFFA F0F0 F0F0 F0F0 F0F0 F0F0"            /* ˇˇˇˇˇ˙ */
+	$"F0F0 F0F0 F0F0 F0F0 F0F0 F0F0 F0F0 EFEF"            /* ÔÔ */
+	$"EFEF EFEF EFEF EEEE EEEE EEEE EDED EDED"            /* ÔÔÔÔÔÔÓÓÓÓÓÓÌÌÌÌ */
+	$"ECEC ECEC ECED E026 0000 0000 0000 0000"            /* ÏÏÏÏÏÌ‡&........ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 51D7 E8E5 E5E5 E6E6 E6E7 E7E7"            /* ....Q◊ËÂÂÂÊÊÊÁÁÁ */
+	$"E7E7 E8E8 E9E9 E9E9 EAEA EAEB EBEB EBEB"            /* ÁÁËËÈÈÈÈÍÍÍÎÎÎÎÎ */
+	$"ECEC ECEC EDED EDED EDEE EEEE EEEE EFEF"            /* ÏÏÏÏÌÌÌÌÌÓÓÓÓÓÔÔ */
+	$"FAFF FFFF FFFB F0EF EFF0 F0F0 F0F0 F0F0"            /* ˙ˇˇˇˇ˚ÔÔ */
+	$"F0F0 F0F0 F0F0 F0F0 F0F0 F0F0 EFEF EFEF"            /* ÔÔÔÔ */
+	$"EFEF EFEF EFEE EEEE EEEE EEED EDED EDED"            /* ÔÔÔÔÔÓÓÓÓÓÓÌÌÌÌÌ */
+	$"ECEC ECEC EBEB F577 0000 0000 0000 0000"            /* ÏÏÏÏÎÎıw........ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0059 D9E8 E6E5 E5E6 E6E6 E7E7"            /* .....YŸËÊÂÂÊÊÊÁÁ */
+	$"E7E7 E7E8 E8E9 E9E9 E9EA EAEA EAEB EBEB"            /* ÁÁÁËËÈÈÈÈÍÍÍÍÎÎÎ */
+	$"EBEC ECEC ECED EDED EDED EEEE EEEE EEEE"            /* ÎÏÏÏÏÌÌÌÌÌÓÓÓÓÓÓ */
+	$"F1FF FFFF FFFD F0EF EFEF EFEF EFEF F0F0"            /* Òˇˇˇˇ˝ÔÔÔÔÔÔÔ */
+	$"F0F0 F0F0 F0F0 F0F0 EFEF EFEF EFEF EFEF"            /* ÔÔÔÔÔÔÔÔ */
+	$"EFEF EFEF EEEE EEEE EEEE EDED EDED EDEC"            /* ÔÔÔÔÓÓÓÓÓÓÌÌÌÌÌÏ */
+	$"ECEC ECEC EBEB EFC6 0F00 0000 0000 0000"            /* ÏÏÏÏÎÎÔ∆........ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 59D6 E7E6 E5E5 E6E6 E6E7"            /* ......Y÷ÁÊÂÂÊÊÊÁ */
+	$"E7E7 E7E8 E8E8 E9E9 E9E9 EAEA EAEA EBEB"            /* ÁÁÁËËËÈÈÈÈÍÍÍÍÎÎ */
+	$"EBEB ECEC ECEC ECED EDED EDED EEEE EEEE"            /* ÎÎÏÏÏÏÏÌÌÌÌÌÓÓÓÓ */
+	$"EDF6 FFFF FFFE F0EF EFEF EFEF EFEF EFEF"            /* Ìˆˇˇˇ˛ÔÔÔÔÔÔÔÔÔ */
+	$"EFEF EFEF EFEF EFEF EFEF EFEF EFEF EFEF"            /* ÔÔÔÔÔÔÔÔÔÔÔÔÔÔÔÔ */
+	$"EFEF EFEE EEEE EEEE EEED EDED EDED ECEC"            /* ÔÔÔÓÓÓÓÓÓÌÌÌÌÌÏÏ */
+	$"ECEC ECEB EBEB EBEF 4C00 0000 0000 0000"            /* ÏÏÏÎÎÎÎÔL....... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0051 D2E7 E6E5 E6E6 E6E6"            /* .......Q“ÁÊÂÊÊÊÊ */
+	$"E7E7 E7E7 E8E8 E8E9 E9E9 E9EA EAEA EAEB"            /* ÁÁÁÁËËËÈÈÈÈÍÍÍÍÎ */
+	$"EBEB EBEC ECEC ECEC EDED EDED EDEE EEEE"            /* ÎÎÎÏÏÏÏÏÌÌÌÌÌÓÓÓ */
+	$"EEEF FCFF FFFE F1EF EFEF EFEF EFEF EFEF"            /* ÓÔ¸ˇˇ˛ÒÔÔÔÔÔÔÔÔÔ */
+	$"EFEF EFEF EFEF EFEF EFEF EFEF EFEF EFEF"            /* ÔÔÔÔÔÔÔÔÔÔÔÔÔÔÔÔ */
+	$"EFEF EEEE EEEE EEEE EEED EDED EDED ECEC"            /* ÔÔÓÓÓÓÓÓÓÌÌÌÌÌÏÏ */
+	$"ECEC ECEB EBEB EBF2 A203 0000 0000 0000"            /* ÏÏÏÎÎÎÎÚ¢....... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 45C7 E7E6 E5E6 E6E6"            /* ........E«ÁÊÂÊÊÊ */
+	$"E6E7 E7E7 E7E7 E8E8 E9E9 E9E9 EAEA EAEA"            /* ÊÁÁÁÁÁËËÈÈÈÈÍÍÍÍ */
+	$"EBEB EBEB EBEC ECEC ECEC EDED EDED EDED"            /* ÎÎÎÎÎÏÏÏÏÏÌÌÌÌÌÌ */
+	$"EEED F3FF FFFF F2EE EFEF EFEF EFEF EFEF"            /* ÓÌÛˇˇˇÚÓÔÔÔÔÔÔÔÔ */
+	$"EFEF EFEF EFEF EFEF EFEF EFEF EFEF EFEF"            /* ÔÔÔÔÔÔÔÔÔÔÔÔÔÔÔÔ */
+	$"EEEE EEEE EEEE EEED EDED EDED EDEC ECEC"            /* ÓÓÓÓÓÓÓÌÌÌÌÌÌÏÏÏ */
+	$"ECEC EBEB EBEB EBEC E127 0000 0000 0000"            /* ÏÏÎÎÎÎÎÏ·'...... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0031 B9E6 E6E5 E6E6"            /* .........1πÊÊÂÊÊ */
+	$"E6E6 E7E7 E7E7 E8E8 E8E9 E9E9 E9EA EAEA"            /* ÊÊÁÁÁÁËËËÈÈÈÈÍÍÍ */
+	$"EAEA EBEB EBEB ECEC ECEC ECED EDED EDED"            /* ÍÍÎÎÎÎÏÏÏÏÏÌÌÌÌÌ */
+	$"EDED EEFA FFFF F3ED EEEE EEEF EFEF EFEF"            /* ÌÌÓ˙ˇˇÛÌÓÓÓÔÔÔÔÔ */
+	$"EFEF EFEF EFEF EFEF EFEF EFEF EFEE EEEE"            /* ÔÔÔÔÔÔÔÔÔÔÔÔÔÓÓÓ */
+	$"EEEE EEEE EEEE EDED EDED EDED ECEC ECEC"            /* ÓÓÓÓÓÓÌÌÌÌÌÌÏÏÏÏ */
+	$"ECEB EBEB EBEB EAEA F379 0000 0000 0000"            /* ÏÎÎÎÎÎÍÍÛy...... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 1E9C E3E7 E6E6"            /* ...........ú„ÁÊÊ */
+	$"E6E6 E6E7 E7E7 E7E7 E8E8 E9E9 E9E9 EAEA"            /* ÊÊÊÁÁÁÁÁËËÈÈÈÈÍÍ */
+	$"EAEA EAEB EBEB EBEB ECEC ECEC ECED EDED"            /* ÍÍÍÎÎÎÎÎÏÏÏÏÏÌÌÌ */
+	$"EDED EDF1 FEFF F5ED EEEE EEEE EEEE EEEF"            /* ÌÌÌÒ˛ˇıÌÓÓÓÓÓÓÓÔ */
+	$"EFEF EFEF EFEF EFEF EFEF EEEE EEEE EEEE"            /* ÔÔÔÔÔÔÔÔÔÔÓÓÓÓÓÓ */
+	$"EEEE EEEE EEED EDED EDED EDEC ECEC ECEC"            /* ÓÓÓÓÓÌÌÌÌÌÌÏÏÏÏÏ */
+	$"ECEB EBEB EBEB EAEA EFC7 0F00 0000 0000"            /* ÏÎÎÎÎÎÍÍÔ«...... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 000A 79D8 E6E6"            /* ...........¬yÿÊÊ */
+	$"E6E6 E6E6 E7E7 E7E7 E7E8 E8E9 E9E9 E9E9"            /* ÊÊÊÊÁÁÁÁÁËËÈÈÈÈÈ */
+	$"EAEA EAEA EBEB EBEB EBEC ECEC ECEC ECED"            /* ÍÍÍÍÎÎÎÎÎÏÏÏÏÏÏÌ */
+	$"EDED EDED F2FF F8ED EEEE EEEE EEEE EEEE"            /* ÌÌÌÌÚˇ¯ÌÓÓÓÓÓÓÓÓ */
+	$"EEEE EEEE EEEE EEEE EEEE EEEE EEEE EEEE"            /* ÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓ */
+	$"EEEE EEED EDED EDED EDED ECEC ECEC ECEC"            /* ÓÓÓÌÌÌÌÌÌÌÏÏÏÏÏÏ */
+	$"EBEB EBEB EBEA EAEA EAEF 4E00 0000 0000"            /* ÎÎÎÎÎÍÍÍÍÔN..... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 004D BFE5"            /* .............MøÂ */
+	$"E6E7 E6E6 E6E7 E7E7 E7E7 E8E8 E8E9 E9E9"            /* ÊÁÊÊÊÁÁÁÁÁËËËÈÈÈ */
+	$"E9EA EAEA EAEA EBEB EBEB EBEC ECEC ECEC"            /* ÈÍÍÍÍÍÎÎÎÎÎÏÏÏÏÏ */
+	$"ECED EDED EDFB F9ED EDEE EEEE EEEE EEEE"            /* ÏÌÌÌÌ˚˘ÌÌÓÓÓÓÓÓÓ */
+	$"EEEE EEEE EEEE EEEE EEEE EEEE EEEE EEEE"            /* ÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓ */
+	$"EEED EDED EDED EDED EDEC ECEC ECEC ECEB"            /* ÓÌÌÌÌÌÌÌÌÏÏÏÏÏÏÎ */
+	$"EBEB EBEB EAEA EAEA EAF1 A303 0000 0000"            /* ÎÎÎÎÍÍÍÍÍÒ£..... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 2195"            /* ..............!ï */
+	$"DCE5 E7E6 E6E6 E7E7 E7E7 E7E8 E8E8 E9E9"            /* ‹ÂÁÊÊÊÁÁÁÁÁËËËÈÈ */
+	$"E9E9 EAEA EAEA EAEB EBEB EBEB ECEC ECEC"            /* ÈÈÍÍÍÍÍÎÎÎÎÎÏÏÏÏ */
+	$"ECEC ECED ECF7 FCED EDED EDED EEEE EEEE"            /* ÏÏÏÌÏ˜¸ÌÌÌÌÌÓÓÓÓ */
+	$"EEEE EEEE EEEE EEEE EEEE EEEE EEEE EEED"            /* ÓÓÓÓÓÓÓÓÓÓÓÓÓÓÓÌ */
+	$"EDED EDED EDED EDED ECEC ECEC ECEC ECEB"            /* ÌÌÌÌÌÌÌÌÏÏÏÏÏÏÏÎ */
+	$"EBEB EBEB EAEA EAEA E9EB E12A 0000 0000"            /* ÎÎÎÎÍÍÍÍÈÎ·*.... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0004"            /* ................ */
+	$"57BF E3E7 E7E6 E6E7 E7E7 E7E7 E8E8 E8E9"            /* Wø„ÁÁÊÊÁÁÁÁÁËËËÈ */
+	$"E9E9 E9E9 EAEA EAEA EBEB EBEB EBEB ECEC"            /* ÈÈÈÈÍÍÍÍÎÎÎÎÎÎÏÏ */
+	$"ECEC ECEC ECEF FDEF EDED EDED EDED EDED"            /* ÏÏÏÏÏÔ˝ÔÌÌÌÌÌÌÌÌ */
+	$"EDEE EEEE EEEE EEEE EEEE EEED EDED EDED"            /* ÌÓÓÓÓÓÓÓÓÓÓÌÌÌÌÌ */
+	$"EDEE EEED EDED EDEC ECEC ECEC ECEC EBEB"            /* ÌÓÓÌÌÌÌÏÏÏÏÏÏÏÎÎ */
+	$"EBEB EBEA EAEA EAEA E9E9 F27A 0100 0000"            /* ÎÎÎÍÍÍÍÍÈÈÚz.... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"001E 84D1 E6E7 E7E6 E7E7 E7E7 E7E8 E8E8"            /* ..Ñ—ÊÁÁÊÁÁÁÁÁËËË */
+	$"E9E9 E9E9 E9EA EAEA EAEA EBEB EBEB EBEB"            /* ÈÈÈÈÈÍÍÍÍÍÎÎÎÎÎÎ */
+	$"ECEC ECEC ECEC F9EF EDED EDED EDED EDED"            /* ÏÏÏÏÏÏ˘ÔÌÌÌÌÌÌÌÌ */
+	$"EDED EDED EDED EDED EDED EDED EDED EDEE"            /* ÌÌÌÌÌÌÌÌÌÌÌÌÌÌÌÓ */
+	$"EEEB E6ED EDEC ECEC ECEC ECEC ECEB EBEB"            /* ÓÎÊÌÌÏÏÏÏÏÏÏÏÎÎÎ */
+	$"EBEB EAEA EAEA EAE9 E9E9 EEC8 0F00 0000"            /* ÎÎÍÍÍÍÍÈÈÈÓ».... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0038 9DD9 E6E7 E7E7 E7E7 E7E7 E8E8"            /* ...8ùŸÊÁÁÁÁÁÁÁËË */
+	$"E8E8 E9E9 E9E9 EAEA EAEA EAEA EBEB EBEB"            /* ËËÈÈÈÈÍÍÍÍÍÍÎÎÎÎ */
+	$"EBEB ECEC ECEB F0EF ECEC EDED EDED EDED"            /* ÎÎÏÏÏÎÔÏÏÌÌÌÌÌÌ */
+	$"EDED EDED EDED EDED EDED EDED EDEE EEED"            /* ÌÌÌÌÌÌÌÌÌÌÌÌÌÓÓÌ */
+	$"E09C 71EA EDEC ECEC ECEC ECEB EBEB EBEB"            /* ‡úqÍÌÏÏÏÏÏÏÎÎÎÎÎ */
+	$"EBEB EAEA EAEA EAE9 E9E9 E9EE 4F00 0000"            /* ÎÎÍÍÍÍÍÈÈÈÈÓO... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0548 A5DA E5E7 E8E8 E7E7 E7E7"            /* .....H•⁄ÂÁËËÁÁÁÁ */
+	$"E8E8 E8E9 E9E9 E9E9 EAEA EAEA EAEB EBEB"            /* ËËËÈÈÈÈÈÍÍÍÍÍÎÎÎ */
+	$"EBEB EBEB ECEC ECEC ECEC ECEC ECEC EDED"            /* ÎÎÎÎÏÏÏÏÏÏÏÏÏÏÌÌ */
+	$"EDED EDED EDED EDED EDED EDEE EFEC E2AD"            /* ÌÌÌÌÌÌÌÌÌÌÌÓÔÏ‚≠ */
+	$"4B04 06C2 F0EC ECEC ECEC EBEB EBEB EBEB"            /* K..¬ÏÏÏÏÏÎÎÎÎÎÎ */
+	$"EBEA EAEA EAEA E9E9 E9E9 E9F0 A504 0000"            /* ÎÍÍÍÍÍÈÈÈÈÈ•... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 074B 9FD5 E4E6 E9E8 E8E7"            /* .......Kü’‰ÊÈËËÁ */
+	$"E7E8 E8E8 E9E9 E9E9 E9EA EAEA EAEA EAEB"            /* ÁËËËÈÈÈÈÈÍÍÍÍÍÍÎ */
+	$"EBEB EBEB EBEB ECEC ECEC ECEC ECEC ECEC"            /* ÎÎÎÎÎÎÏÏÏÏÏÏÏÏÏÏ */
+	$"ECEC ECEC EDED EDED EEEE EDEB DAA3 4F09"            /* ÏÏÏÏÌÌÌÌÓÓÌÎ⁄£O∆ */
+	$"0000 0072 F4EC ECEC ECEB EBEB EBEB EBEB"            /* ...rÙÏÏÏÏÎÎÎÎÎÎÎ */
+	$"EAEA EAEA EAE9 E9E9 E9E9 E8EA E12B 0000"            /* ÍÍÍÍÍÈÈÈÈÈËÍ·+.. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 033A 8CC6 E0E6 E7E9"            /* .........:å∆‡ÊÁÈ */
+	$"E8E8 E8E8 E8E8 E9E9 E9E9 E9EA EAEA EAEA"            /* ËËËËËËÈÈÈÈÈÍÍÍÍÍ */
+	$"EAEB EBEB EBEB EBEB EBEC ECEC ECEC ECEC"            /* ÍÎÎÎÎÎÎÎÎÏÏÏÏÏÏÏ */
+	$"ECEC ECEC ECED EDED EBE6 CD92 3C04 0000"            /* ÏÏÏÏÏÌÌÌÎÊÕí<... */
+	$"0000 0023 DFEF ECEB EBEB EBEB EBEB EAEA"            /* ...#ﬂÔÏÎÎÎÎÎÎÎÍÍ */
+	$"EAEA EAEA E9E9 E9E9 E9E9 E8E8 F17C 0100"            /* ÍÍÍÍÈÈÈÈÈÈËËÒ|.. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0020 64A7 D1E2"            /* ........... dß—‚ */
+	$"E6E7 E9E8 E8E8 E8E9 E9E9 E9E9 E9EA EAEA"            /* ÊÁÈËËËËÈÈÈÈÈÈÍÍÍ */
+	$"EAEA EAEB EBEB EBEB EBEB EBEC ECEC ECEC"            /* ÍÍÍÎÎÎÎÎÎÎÎÏÏÏÏÏ */
+	$"ECED EDED EDEB E8D8 AB67 2300 0000 0000"            /* ÏÌÌÌÌÎËÿ´g#..... */
+	$"0000 0001 9FF3 EBEB EBEB EBEB EBEA EAEA"            /* ....üÛÎÎÎÎÎÎÎÍÍÍ */
+	$"EAEA EAEA E9E9 E9E9 E9E8 E8E8 EDC9 1000"            /* ÍÍÍÍÈÈÈÈÈËËËÌ….. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0006 3070"            /* ..............0p */
+	$"A8CC DEE5 E8E8 E9E9 E8E9 E9E9 E9E9 EAEA"            /* ®ÃﬁÂËËÈÈËÈÈÈÈÈÍÍ */
+	$"EAEA EAEA EAEB EBEB EBEB EBEB ECEC ECEC"            /* ÍÍÍÍÍÎÎÎÎÎÎÎÏÏÏÏ */
+	$"EDEB EAE5 D1A9 7136 0700 0000 0000 0000"            /* ÌÎÍÂ—©q6........ */
+	$"0000 0000 48ED EBEB EBEB EBEB EAEA EAEA"            /* ....HÌÎÎÎÎÎÎÍÍÍÍ */
+	$"EAEA EAE9 E9E9 E9E9 E8E8 E8E8 E8ED 5200"            /* ÍÍÍÈÈÈÈÈËËËËËÌR. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0625 588B B1CE DCE3 E6E8 E9E9 E9EA EAE9"            /* .%Xã±Œ‹„ÊËÈÈÈÍÍÈ */
+	$"EAEA EAEA EAEA EAEB ECEB EBEB EAE9 E7E1"            /* ÍÍÍÍÍÍÍÎÏÎÎÎÍÈÁ· */
+	$"D0B4 8C5C 2805 0000 0000 0000 0000 0000"            /* –¥å\(........... */
+	$"0000 0000 0DC4 F0EB EBEB EAEA EAEA EAEA"            /* .....ƒÎÎÎÍÍÍÍÍÍ */
+	$"EAEA E9E9 E9E9 E9E9 E8E8 E8E8 E7EF A704"            /* ÍÍÈÈÈÈÈÈËËËËÁÔß. */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0921 4260 7D96 A7B6 C3CA D2DC"            /* ....∆!B`}ñß∂√ “‹ */
+	$"E1E5 E8E9 E3E0 DCD5 CAC2 B8AB 957C 6043"            /* ·ÂËÈ„‡‹’ ¬∏´ï|`C */
+	$"2209 0000 0000 0000 0000 0000 0000 0000"            /* "∆.............. */
+	$"0000 0000 006D F1EB EBEA EAEA EAEA EAEA"            /* .....mÒÎÎÍÍÍÍÍÍÍ */
+	$"EAE9 E9E9 E9E9 E9E8 E8E8 E8E7 E7E8 E435"            /* ÍÈÈÈÈÈÈËËËËÁÁË‰5 */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0000 0000 0006 101E 2E3A 5072"            /* .............:Pr */
+	$"848F 9B99 8A7B 6E56 392A 1D12 0500 0000"            /* Ñèõôä{nV9*...... */
+	$"0000 0000 0000 0000 0000 0000 0000 0000"            /* ................ */
+	$"0000 0000 0024 A1A1 A0A0 A0A0 A0A0 A09F"            /* .....$°°†††††††ü */
+	$"9F9F 9F9F 9F9F 9F9F 9F9F 9F9E 9E9E A548"            /* üüüüüüüüüüüûûû•H */
+	$"0000 00"                                            /* ... */
+};
+
diff --git a/pc-bios/s390-ccw.img b/pc-bios/s390-ccw.img
index 44873ad..089f6ba 100644
--- a/pc-bios/s390-ccw.img
+++ b/pc-bios/s390-ccw.img
Binary files differ
diff --git a/pc-bios/s390-ccw/Makefile b/pc-bios/s390-ccw/Makefile
index ad55a14..0ab2538 100644
--- a/pc-bios/s390-ccw/Makefile
+++ b/pc-bios/s390-ccw/Makefile
@@ -9,10 +9,12 @@
 
 .PHONY : all clean build-all
 
-OBJECTS=main.o bootmap.o sclp-ascii.o virtio.o start.o
-CFLAGS += -fno-stack-protector
-# XXX find a more clever to locate the bootloader
-LDFLAGS += -Wl,-Ttext,0x7e00000,-Tbss,0x7f00000 -nostdlib
+OBJECTS = start.o main.o bootmap.o sclp-ascii.o virtio.o virtio-scsi.o
+QEMU_CFLAGS := $(filter -W%, $(QEMU_CFLAGS))
+QEMU_CFLAGS += -ffreestanding -fno-delete-null-pointer-checks -msoft-float
+QEMU_CFLAGS += -march=z900 -fPIE -fno-strict-aliasing
+QEMU_CFLAGS += $(call cc-option, $(QEMU_CFLAGS), -fno-stack-protector)
+LDFLAGS += -Wl,-pie -nostdlib
 
 build-all: s390-ccw.img
 
@@ -20,7 +22,9 @@
 	$(call quiet-command,$(CC) $(LDFLAGS) -o $@ $(OBJECTS),"  Building $(TARGET_DIR)$@")
 
 s390-ccw.img: s390-ccw.elf
-	$(call quiet-command,strip $< -o $@,"  Stripping $(TARGET_DIR)$@")
+	$(call quiet-command,strip --strip-unneeded $< -o $@,"  Stripping $(TARGET_DIR)$@")
+
+$(OBJECTS): Makefile
 
 clean:
 	rm -f *.o *.d *.img *.elf *~
diff --git a/pc-bios/s390-ccw/bootmap.c b/pc-bios/s390-ccw/bootmap.c
index 115d8bb..611102e 100644
--- a/pc-bios/s390-ccw/bootmap.c
+++ b/pc-bios/s390-ccw/bootmap.c
@@ -33,7 +33,7 @@
     uint32_t ipl_continue;
 } ResetInfo;
 
-ResetInfo save;
+static ResetInfo save;
 
 static void jump_to_IPL_2(void)
 {
@@ -72,7 +72,7 @@
     asm volatile("lghi 1,1\n\t"
                  "diag 1,1,0x308\n\t"
                  : : : "1", "memory");
-    virtio_panic("\n! IPL returns !\n");
+    panic("\n! IPL returns !\n");
 }
 
 /***********************************************************************
@@ -80,11 +80,11 @@
  */
 
 static unsigned char _bprs[8*1024]; /* guessed "max" ECKD sector size */
-const int max_bprs_entries = sizeof(_bprs) / sizeof(ExtEckdBlockPtr);
+static const int max_bprs_entries = sizeof(_bprs) / sizeof(ExtEckdBlockPtr);
 
 static inline void verify_boot_info(BootInfo *bip)
 {
-    IPL_assert(magic_match(bip->magic, ZIPL_MAGIC), "No zIPL magic");
+    IPL_assert(magic_match(bip->magic, ZIPL_MAGIC), "No zIPL sig in BootInfo");
     IPL_assert(bip->version == BOOT_INFO_VERSION, "Wrong zIPL version");
     IPL_assert(bip->bp_type == BOOT_INFO_BP_TYPE_IPL, "DASD is not for IPL");
     IPL_assert(bip->dev_type == BOOT_INFO_DEV_TYPE_ECKD, "DASD is not ECKD");
@@ -315,6 +315,40 @@
     sclp_print(msg);
 }
 
+static void ipl_eckd(void)
+{
+    ScsiMbr *mbr = (void *)sec;
+    LDL_VTOC *vlbl = (void *)sec;
+
+    print_eckd_msg();
+
+    /* Grab the MBR again */
+    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
+    read_block(0, mbr, "Cannot read block 0 on DASD");
+
+    if (magic_match(mbr->magic, IPL1_MAGIC)) {
+        ipl_eckd_cdl(); /* no return */
+    }
+
+    /* LDL/CMS? */
+    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
+    read_block(2, vlbl, "Cannot read block 2");
+
+    if (magic_match(vlbl->magic, CMS1_MAGIC)) {
+        ipl_eckd_ldl(ECKD_CMS); /* no return */
+    }
+    if (magic_match(vlbl->magic, LNX1_MAGIC)) {
+        ipl_eckd_ldl(ECKD_LDL); /* no return */
+    }
+
+    ipl_eckd_ldl(ECKD_LDL_UNLABELED); /* it still may return */
+    /*
+     * Ok, it is not a LDL by any means.
+     * It still might be a CDL with zero record keys for IPL1 and IPL2
+     */
+    ipl_eckd_cdl();
+}
+
 /***********************************************************************
  * IPL a SCSI disk
  */
@@ -382,7 +416,7 @@
     read_block(pte->blockno, tmp_sec, "Cannot read header");
     header = (ComponentHeader *)tmp_sec;
 
-    IPL_assert(magic_match(tmp_sec, ZIPL_MAGIC), "No zIPL magic");
+    IPL_assert(magic_match(tmp_sec, ZIPL_MAGIC), "No zIPL magic in header");
     IPL_assert(header->type == ZIPL_COMP_HEADER_IPL, "Bad header type");
 
     dputs("start loading images\n");
@@ -412,19 +446,29 @@
     const int pte_len = sizeof(ScsiBlockPtr);
     ScsiBlockPtr *prog_table_entry;
 
-    /* The 0-th block (MBR) was already read into sec[] */
+    /* Grab the MBR */
+    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
+    read_block(0, mbr, "Cannot read block 0");
+
+    if (!magic_match(mbr->magic, ZIPL_MAGIC)) {
+        return;
+    }
 
     sclp_print("Using SCSI scheme.\n");
+    debug_print_int("MBR Version", mbr->version_id);
+    IPL_check(mbr->version_id == 1,
+              "Unknown MBR layout version, assuming version 1");
     debug_print_int("program table", mbr->blockptr.blockno);
+    IPL_assert(mbr->blockptr.blockno, "No Program Table");
 
     /* Parse the program table */
     read_block(mbr->blockptr.blockno, sec,
                "Error reading Program Table");
 
-    IPL_assert(magic_match(sec, ZIPL_MAGIC), "No zIPL magic");
+    IPL_assert(magic_match(sec, ZIPL_MAGIC), "No zIPL magic in PT");
 
     ns_end = sec + virtio_get_block_size();
-    for (ns = (sec + pte_len); (ns + pte_len) < ns_end; ns++) {
+    for (ns = (sec + pte_len); (ns + pte_len) < ns_end; ns += pte_len) {
         prog_table_entry = (ScsiBlockPtr *)ns;
         if (!prog_table_entry->blockno) {
             break;
@@ -445,51 +489,258 @@
 }
 
 /***********************************************************************
+ * IPL El Torito ISO9660 image or DVD
+ */
+
+static bool is_iso_bc_entry_compatible(IsoBcSection *s)
+{
+    uint8_t *magic_sec = (uint8_t *)(sec + ISO_SECTOR_SIZE);
+
+    if (s->unused || !s->sector_count) {
+        return false;
+    }
+    read_iso_sector(bswap32(s->load_rba), magic_sec,
+                    "Failed to read image sector 0");
+
+    /* Checking bytes 8 - 32 for S390 Linux magic */
+    return !_memcmp(magic_sec + 8, linux_s390_magic, 24);
+}
+
+/* Location of the current sector of the directory */
+static uint32_t sec_loc[ISO9660_MAX_DIR_DEPTH];
+/* Offset in the current sector of the directory */
+static uint32_t sec_offset[ISO9660_MAX_DIR_DEPTH];
+/* Remained directory space in bytes */
+static uint32_t dir_rem[ISO9660_MAX_DIR_DEPTH];
+
+static inline uint32_t iso_get_file_size(uint32_t load_rba)
+{
+    IsoVolDesc *vd = (IsoVolDesc *)sec;
+    IsoDirHdr *cur_record = &vd->vd.primary.rootdir;
+    uint8_t *temp = sec + ISO_SECTOR_SIZE;
+    int level = 0;
+
+    read_iso_sector(ISO_PRIMARY_VD_SECTOR, sec,
+                    "Failed to read ISO primary descriptor");
+    sec_loc[0] = iso_733_to_u32(cur_record->ext_loc);
+    dir_rem[0] = 0;
+    sec_offset[0] = 0;
+
+    while (level >= 0) {
+        IPL_assert(sec_offset[level] <= ISO_SECTOR_SIZE,
+                   "Directory tree structure violation");
+
+        cur_record = (IsoDirHdr *)(temp + sec_offset[level]);
+
+        if (sec_offset[level] == 0) {
+            read_iso_sector(sec_loc[level], temp,
+                            "Failed to read ISO directory");
+            if (dir_rem[level] == 0) {
+                /* Skip self and parent records */
+                dir_rem[level] = iso_733_to_u32(cur_record->data_len) -
+                                 cur_record->dr_len;
+                sec_offset[level] += cur_record->dr_len;
+
+                cur_record = (IsoDirHdr *)(temp + sec_offset[level]);
+                dir_rem[level] -= cur_record->dr_len;
+                sec_offset[level] += cur_record->dr_len;
+                continue;
+            }
+        }
+
+        if (!cur_record->dr_len || sec_offset[level] == ISO_SECTOR_SIZE) {
+            /* Zero-padding and/or the end of current sector */
+            dir_rem[level] -= ISO_SECTOR_SIZE - sec_offset[level];
+            sec_offset[level] = 0;
+            sec_loc[level]++;
+        } else {
+            /* The directory record is valid */
+            if (load_rba == iso_733_to_u32(cur_record->ext_loc)) {
+                return iso_733_to_u32(cur_record->data_len);
+            }
+
+            dir_rem[level] -= cur_record->dr_len;
+            sec_offset[level] += cur_record->dr_len;
+
+            if (cur_record->file_flags & 0x2) {
+                /* Subdirectory */
+                if (level == ISO9660_MAX_DIR_DEPTH - 1) {
+                    sclp_print("ISO-9660 directory depth limit exceeded\n");
+                } else {
+                    level++;
+                    sec_loc[level] = iso_733_to_u32(cur_record->ext_loc);
+                    sec_offset[level] = 0;
+                    dir_rem[level] = 0;
+                    continue;
+                }
+            }
+        }
+
+        if (dir_rem[level] == 0) {
+            /* Nothing remaining */
+            level--;
+            read_iso_sector(sec_loc[level], temp,
+                            "Failed to read ISO directory");
+        }
+    }
+
+    return 0;
+}
+
+static void load_iso_bc_entry(IsoBcSection *load)
+{
+    IsoBcSection s = *load;
+    /*
+     * According to spec, extent for each file
+     * is padded and ISO_SECTOR_SIZE bytes aligned
+     */
+    uint32_t blks_to_load = bswap16(s.sector_count) >> ET_SECTOR_SHIFT;
+    uint32_t real_size = iso_get_file_size(bswap32(s.load_rba));
+
+    if (real_size) {
+        /* Round up blocks to load */
+        blks_to_load = (real_size + ISO_SECTOR_SIZE - 1) / ISO_SECTOR_SIZE;
+        sclp_print("ISO boot image size verified\n");
+    } else {
+        sclp_print("ISO boot image size could not be verified\n");
+    }
+
+    read_iso_boot_image(bswap32(s.load_rba),
+                        (void *)((uint64_t)bswap16(s.load_segment)),
+                        blks_to_load);
+
+    /* Trying to get PSW at zero address */
+    if (*((uint64_t *)0) & IPL_PSW_MASK) {
+        jump_to_IPL_code((*((uint64_t *)0)) & 0x7fffffff);
+    }
+
+    /* Try default linux start address */
+    jump_to_IPL_code(KERN_IMAGE_START);
+}
+
+static uint32_t find_iso_bc(void)
+{
+    IsoVolDesc *vd = (IsoVolDesc *)sec;
+    uint32_t block_num = ISO_PRIMARY_VD_SECTOR;
+
+    if (virtio_read_many(block_num++, sec, 1)) {
+        /* If primary vd cannot be read, there is no boot catalog */
+        return 0;
+    }
+
+    while (is_iso_vd_valid(vd) && vd->type != VOL_DESC_TERMINATOR) {
+        if (vd->type == VOL_DESC_TYPE_BOOT) {
+            IsoVdElTorito *et = &vd->vd.boot;
+
+            if (!_memcmp(&et->el_torito[0], el_torito_magic, 32)) {
+                return bswap32(et->bc_offset);
+            }
+        }
+        read_iso_sector(block_num++, sec,
+                        "Failed to read ISO volume descriptor");
+    }
+
+    return 0;
+}
+
+static IsoBcSection *find_iso_bc_entry(void)
+{
+    IsoBcEntry *e = (IsoBcEntry *)sec;
+    uint32_t offset = find_iso_bc();
+    int i;
+
+    if (!offset) {
+        return NULL;
+    }
+
+    read_iso_sector(offset, sec, "Failed to read El Torito boot catalog");
+
+    if (!is_iso_bc_valid(e)) {
+        /* The validation entry is mandatory */
+        panic("No valid boot catalog found!\n");
+        return NULL;
+    }
+
+    /*
+     * Each entry has 32 bytes size, so one sector cannot contain > 64 entries.
+     * We consider only boot catalogs with no more than 64 entries.
+     */
+    for (i = 1; i < ISO_BC_ENTRY_PER_SECTOR; i++) {
+        if (e[i].id == ISO_BC_BOOTABLE_SECTION) {
+            if (is_iso_bc_entry_compatible(&e[i].body.sect)) {
+                return &e[i].body.sect;
+            }
+        }
+    }
+
+    panic("No suitable boot entry found on ISO-9660 media!\n");
+
+    return NULL;
+}
+
+static void ipl_iso_el_torito(void)
+{
+    IsoBcSection *s = find_iso_bc_entry();
+
+    if (s) {
+        load_iso_bc_entry(s);
+        /* no return */
+    }
+}
+
+/***********************************************************************
+ * Bus specific IPL sequences
+ */
+
+static void zipl_load_vblk(void)
+{
+    if (virtio_guessed_disk_nature()) {
+        virtio_assume_iso9660();
+    }
+    ipl_iso_el_torito();
+
+    if (virtio_guessed_disk_nature()) {
+        sclp_print("Using guessed DASD geometry.\n");
+        virtio_assume_eckd();
+    }
+    ipl_eckd();
+}
+
+static void zipl_load_vscsi(void)
+{
+    if (virtio_get_block_size() == VIRTIO_ISO_BLOCK_SIZE) {
+        /* Is it an ISO image in non-CD drive? */
+        ipl_iso_el_torito();
+    }
+
+    sclp_print("Using guessed DASD geometry.\n");
+    virtio_assume_eckd();
+    ipl_eckd();
+}
+
+/***********************************************************************
  * IPL starts here
  */
 
 void zipl_load(void)
 {
-    ScsiMbr *mbr = (void *)sec;
-    LDL_VTOC *vlbl = (void *)sec;
-
-    /* Grab the MBR */
-    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
-    read_block(0, mbr, "Cannot read block 0");
-
-    dputs("checking magic\n");
-
-    if (magic_match(mbr->magic, ZIPL_MAGIC)) {
-        ipl_scsi(); /* no return */
+    if (virtio_get_device()->is_cdrom) {
+        ipl_iso_el_torito();
+        panic("\n! Cannot IPL this ISO image !\n");
     }
 
-    /* We have failed to follow the SCSI scheme, so */
-    if (virtio_guessed_disk_nature()) {
-        sclp_print("Using guessed DASD geometry.\n");
-        virtio_assume_eckd();
-    }
-    print_eckd_msg();
-    if (magic_match(mbr->magic, IPL1_MAGIC)) {
-        ipl_eckd_cdl(); /* no return */
+    ipl_scsi();
+
+    switch (virtio_get_device_type()) {
+    case VIRTIO_ID_BLOCK:
+        zipl_load_vblk();
+        break;
+    case VIRTIO_ID_SCSI:
+        zipl_load_vscsi();
+        break;
+    default:
+        panic("\n! Unknown IPL device type !\n");
     }
 
-    /* LDL/CMS? */
-    memset(sec, FREE_SPACE_FILLER, sizeof(sec));
-    read_block(2, vlbl, "Cannot read block 2");
-
-    if (magic_match(vlbl->magic, CMS1_MAGIC)) {
-        ipl_eckd_ldl(ECKD_CMS); /* no return */
-    }
-    if (magic_match(vlbl->magic, LNX1_MAGIC)) {
-        ipl_eckd_ldl(ECKD_LDL); /* no return */
-    }
-
-    ipl_eckd_ldl(ECKD_LDL_UNLABELED); /* it still may return */
-    /*
-     * Ok, it is not a LDL by any means.
-     * It still might be a CDL with zero record keys for IPL1 and IPL2
-     */
-    ipl_eckd_cdl();
-
-    virtio_panic("\n* this can never happen *\n");
+    panic("\n* this can never happen *\n");
 }
diff --git a/pc-bios/s390-ccw/bootmap.h b/pc-bios/s390-ccw/bootmap.h
index 6a4823d..bea1687 100644
--- a/pc-bios/s390-ccw/bootmap.h
+++ b/pc-bios/s390-ccw/bootmap.h
@@ -15,7 +15,7 @@
 #include "virtio.h"
 
 typedef uint64_t block_number_t;
-#define NULL_BLOCK_NR 0xffffffffffffffff
+#define NULL_BLOCK_NR 0xffffffffffffffffULL
 
 #define FREE_SPACE_FILLER '\xAA'
 
@@ -264,15 +264,6 @@
 
 /* utility code below */
 
-static inline void IPL_assert(bool term, const char *message)
-{
-    if (!term) {
-        sclp_print("\n! ");
-        sclp_print(message);
-        virtio_panic(" !\n"); /* no return */
-    }
-}
-
 static const unsigned char ebc2asc[256] =
       /* 0123456789abcdef0123456789abcdef */
         "................................" /* 1F */
@@ -341,4 +332,210 @@
     return *((uint32_t *)data) == *((uint32_t *)magic);
 }
 
+static inline int _memcmp(const void *s1, const void *s2, size_t n)
+{
+    int i;
+    const uint8_t *p1 = s1, *p2 = s2;
+
+    for (i = 0; i < n; i++) {
+        if (p1[i] != p2[i]) {
+            return p1[i] > p2[i] ? 1 : -1;
+        }
+    }
+
+    return 0;
+}
+
+/* from include/qemu/bswap.h */
+
+/* El Torito is always little-endian */
+static inline uint16_t bswap16(uint16_t x)
+{
+    return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8);
+}
+
+static inline uint32_t bswap32(uint32_t x)
+{
+    return ((x & 0x000000ffU) << 24) | ((x & 0x0000ff00U) <<  8) |
+           ((x & 0x00ff0000U) >>  8) | ((x & 0xff000000U) >> 24);
+}
+
+static inline uint64_t bswap64(uint64_t x)
+{
+    return ((x & 0x00000000000000ffULL) << 56) |
+           ((x & 0x000000000000ff00ULL) << 40) |
+           ((x & 0x0000000000ff0000ULL) << 24) |
+           ((x & 0x00000000ff000000ULL) <<  8) |
+           ((x & 0x000000ff00000000ULL) >>  8) |
+           ((x & 0x0000ff0000000000ULL) >> 24) |
+           ((x & 0x00ff000000000000ULL) >> 40) |
+           ((x & 0xff00000000000000ULL) >> 56);
+}
+
+static inline uint32_t iso_733_to_u32(uint64_t x)
+{
+    return (uint32_t)x;
+}
+
+#define ISO_SECTOR_SIZE 2048
+/* El Torito specifies boot image size in 512 byte blocks */
+#define ET_SECTOR_SHIFT 2
+#define KERN_IMAGE_START 0x010000UL
+#define PSW_MASK_64 0x0000000100000000ULL
+#define PSW_MASK_32 0x0000000080000000ULL
+#define IPL_PSW_MASK (PSW_MASK_32 | PSW_MASK_64)
+
+#define ISO_PRIMARY_VD_SECTOR 16
+
+static inline void read_iso_sector(uint32_t block_offset, void *buf,
+                                   const char *errmsg)
+{
+    IPL_assert(virtio_read_many(block_offset, buf, 1) == 0, errmsg);
+}
+
+static inline void read_iso_boot_image(uint32_t block_offset, void *load_addr,
+                                       uint32_t blks_to_load)
+{
+    IPL_assert(virtio_read_many(block_offset, load_addr, blks_to_load) == 0,
+               "Failed to read boot image!");
+}
+
+const uint8_t el_torito_magic[] = "EL TORITO SPECIFICATION"
+                                  "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
+
+#define ISO9660_MAX_DIR_DEPTH 8
+
+typedef struct IsoDirHdr {
+    uint8_t dr_len;
+    uint8_t ear_len;
+    uint64_t ext_loc;
+    uint64_t data_len;
+    uint8_t recording_datetime[7];
+    uint8_t file_flags;
+    uint8_t file_unit_size;
+    uint8_t gap_size;
+    uint32_t vol_seqnum;
+    uint8_t fileid_len;
+} __attribute__((packed)) IsoDirHdr;
+
+typedef struct IsoVdElTorito {
+    uint8_t el_torito[32]; /* must contain el_torito_magic value */
+    uint8_t unused0[32];
+    uint32_t bc_offset;
+    uint8_t unused1[1974];
+} __attribute__((packed)) IsoVdElTorito;
+
+typedef struct IsoVdPrimary {
+    uint8_t unused1;
+    uint8_t sys_id[32];
+    uint8_t vol_id[32];
+    uint8_t unused2[8];
+    uint64_t vol_space_size;
+    uint8_t unused3[32];
+    uint32_t vol_set_size;
+    uint32_t vol_seqnum;
+    uint32_t log_block_size;
+    uint64_t path_table_size;
+    uint32_t l_path_table;
+    uint32_t opt_l_path_table;
+    uint32_t m_path_table;
+    uint32_t opt_m_path_table;
+    IsoDirHdr rootdir;
+    uint8_t root_null;
+    uint8_t reserved2[1858];
+} __attribute__((packed)) IsoVdPrimary;
+
+typedef struct IsoVolDesc {
+    uint8_t type;
+    uint8_t ident[5];
+    uint8_t version;
+    union {
+        IsoVdElTorito boot;
+        IsoVdPrimary primary;
+    } vd;
+} __attribute__((packed)) IsoVolDesc;
+
+const uint8_t vol_desc_magic[] = "CD001";
+#define VOL_DESC_TYPE_BOOT 0
+#define VOL_DESC_TYPE_PRIMARY 1
+#define VOL_DESC_TYPE_SUPPLEMENT 2
+#define VOL_DESC_TYPE_PARTITION 3
+#define VOL_DESC_TERMINATOR 255
+
+static inline bool is_iso_vd_valid(IsoVolDesc *vd)
+{
+    return !_memcmp(&vd->ident[0], vol_desc_magic, 5) &&
+           vd->version == 0x1 &&
+           vd->type <= VOL_DESC_TYPE_PARTITION;
+}
+
+typedef struct IsoBcValid {
+    uint8_t platform_id;
+    uint16_t reserved;
+    uint8_t id[24];
+    uint16_t checksum;
+    uint8_t key[2];
+} __attribute__((packed)) IsoBcValid;
+
+typedef struct IsoBcSection {
+    uint8_t boot_type;
+    uint16_t load_segment;
+    uint8_t sys_type;
+    uint8_t unused;
+    uint16_t sector_count;
+    uint32_t load_rba;
+    uint8_t selection[20];
+} __attribute__((packed)) IsoBcSection;
+
+typedef struct IsoBcHdr {
+    uint8_t platform_id;
+    uint16_t sect_num;
+    uint8_t id[28];
+} __attribute__((packed)) IsoBcHdr;
+
+/*
+ * Match two CCWs located after PSW and eight filler bytes.
+ * From libmagic and arch/s390/kernel/head.S.
+ */
+const uint8_t linux_s390_magic[] = "\x02\x00\x00\x18\x60\x00\x00\x50\x02\x00"
+                                   "\x00\x68\x60\x00\x00\x50\x40\x40\x40\x40"
+                                   "\x40\x40\x40\x40";
+
+typedef struct IsoBcEntry {
+    uint8_t id;
+    union {
+        IsoBcValid valid; /* id == 0x01 */
+        IsoBcSection sect; /* id == 0x88 || id == 0x0 */
+        IsoBcHdr hdr; /* id == 0x90 || id == 0x91 */
+    } body;
+} __attribute__((packed)) IsoBcEntry;
+
+#define ISO_BC_ENTRY_PER_SECTOR (ISO_SECTOR_SIZE / sizeof(IsoBcEntry))
+#define ISO_BC_HDR_VALIDATION 0x01
+#define ISO_BC_BOOTABLE_SECTION 0x88
+#define ISO_BC_MAGIC_55 0x55
+#define ISO_BC_MAGIC_AA 0xaa
+#define ISO_BC_PLATFORM_X86 0x0
+#define ISO_BC_PLATFORM_PPC 0x1
+#define ISO_BC_PLATFORM_MAC 0x2
+
+static inline bool is_iso_bc_valid(IsoBcEntry *e)
+{
+    IsoBcValid *v = &e->body.valid;
+
+    if (e->id != ISO_BC_HDR_VALIDATION) {
+        return false;
+    }
+
+    if (v->platform_id != ISO_BC_PLATFORM_X86 &&
+        v->platform_id != ISO_BC_PLATFORM_PPC &&
+        v->platform_id != ISO_BC_PLATFORM_MAC) {
+        return false;
+    }
+
+    return v->key[0] == ISO_BC_MAGIC_55 &&
+           v->key[1] == ISO_BC_MAGIC_AA &&
+           v->reserved == 0x0;
+}
+
 #endif /* _PC_BIOS_S390_CCW_BOOTMAP_H */
diff --git a/pc-bios/s390-ccw/iplb.h b/pc-bios/s390-ccw/iplb.h
new file mode 100644
index 0000000..86abc56
--- /dev/null
+++ b/pc-bios/s390-ccw/iplb.h
@@ -0,0 +1,91 @@
+/*
+ * QEMU S390 IPL Block
+ *
+ * Copyright 2015 IBM Corp.
+ * Author(s): Alexander Yarygin <yarygin@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef IPLB_H
+#define IPLB_H
+
+struct IplBlockCcw {
+    uint8_t  reserved0[85];
+    uint8_t  ssid;
+    uint16_t devno;
+    uint8_t  vm_flags;
+    uint8_t  reserved3[3];
+    uint32_t vm_parm_len;
+    uint8_t  nss_name[8];
+    uint8_t  vm_parm[64];
+    uint8_t  reserved4[8];
+} __attribute__ ((packed));
+typedef struct IplBlockCcw IplBlockCcw;
+
+struct IplBlockFcp {
+    uint8_t  reserved1[305 - 1];
+    uint8_t  opt;
+    uint8_t  reserved2[3];
+    uint16_t reserved3;
+    uint16_t devno;
+    uint8_t  reserved4[4];
+    uint64_t wwpn;
+    uint64_t lun;
+    uint32_t bootprog;
+    uint8_t  reserved5[12];
+    uint64_t br_lba;
+    uint32_t scp_data_len;
+    uint8_t  reserved6[260];
+    uint8_t  scp_data[];
+} __attribute__ ((packed));
+typedef struct IplBlockFcp IplBlockFcp;
+
+struct IplBlockQemuScsi {
+    uint32_t lun;
+    uint16_t target;
+    uint16_t channel;
+    uint8_t  reserved0[77];
+    uint8_t  ssid;
+    uint16_t devno;
+} __attribute__ ((packed));
+typedef struct IplBlockQemuScsi IplBlockQemuScsi;
+
+struct IplParameterBlock {
+    uint32_t len;
+    uint8_t  reserved0[3];
+    uint8_t  version;
+    uint32_t blk0_len;
+    uint8_t  pbt;
+    uint8_t  flags;
+    uint16_t reserved01;
+    uint8_t  loadparm[8];
+    union {
+        IplBlockCcw ccw;
+        IplBlockFcp fcp;
+        IplBlockQemuScsi scsi;
+    };
+} __attribute__ ((packed));
+typedef struct IplParameterBlock IplParameterBlock;
+
+extern IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE)));
+
+#define S390_IPL_TYPE_FCP 0x00
+#define S390_IPL_TYPE_CCW 0x02
+#define S390_IPL_TYPE_QEMU_SCSI 0xff
+
+static inline bool store_iplb(IplParameterBlock *iplb)
+{
+    register unsigned long addr asm("0") = (unsigned long) iplb;
+    register unsigned long rc asm("1") = 0;
+
+    asm volatile ("diag %0,%2,0x308\n"
+                  : "+d" (addr), "+d" (rc)
+                  : "d" (6)
+                  : "memory", "cc");
+    return rc == 0x01;
+}
+
+#endif /* IPLB_H */
diff --git a/pc-bios/s390-ccw/main.c b/pc-bios/s390-ccw/main.c
index f9ec215..345b848 100644
--- a/pc-bios/s390-ccw/main.c
+++ b/pc-bios/s390-ccw/main.c
@@ -12,8 +12,8 @@
 #include "virtio.h"
 
 char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE)));
-uint64_t boot_value;
-struct subchannel_id blk_schid = { .one = 1 };
+static SubChannelId blk_schid = { .one = 1 };
+IplParameterBlock iplb __attribute__((__aligned__(PAGE_SIZE)));
 
 /*
  * Priniciples of Operations (SA22-7832-09) chapter 17 requires that
@@ -22,7 +22,7 @@
  */
 void write_subsystem_identification(void)
 {
-    struct subchannel_id *schid = (struct subchannel_id *) 184;
+    SubChannelId *schid = (SubChannelId *) 184;
     uint32_t *zeroes = (uint32_t *) 188;
 
     *schid = blk_schid;
@@ -30,70 +30,99 @@
 }
 
 
-void virtio_panic(const char *string)
+void panic(const char *string)
 {
     sclp_print(string);
     disabled_wait();
     while (1) { }
 }
 
-static void virtio_setup(uint64_t dev_info)
+static bool find_dev(Schib *schib, int dev_no)
 {
-    struct schib schib;
-    int i;
-    int r;
-    bool found = false;
-    bool check_devno = false;
-    uint16_t dev_no = -1;
-
-    if (dev_info != -1) {
-        check_devno = true;
-        dev_no = dev_info & 0xffff;
-        debug_print_int("device no. ", dev_no);
-        blk_schid.ssid = (dev_info >> 16) & 0x3;
-        if (blk_schid.ssid != 0) {
-            debug_print_int("ssid ", blk_schid.ssid);
-            if (enable_mss_facility() != 0) {
-                virtio_panic("Failed to enable mss facility\n");
-            }
-        }
-    }
+    int i, r;
 
     for (i = 0; i < 0x10000; i++) {
         blk_schid.sch_no = i;
-        r = stsch_err(blk_schid, &schib);
-        if (r == 3) {
+        r = stsch_err(blk_schid, schib);
+        if ((r == 3) || (r == -EIO)) {
             break;
         }
-        if (schib.pmcw.dnv) {
-            if (!check_devno || (schib.pmcw.dev == dev_no)) {
-                if (virtio_is_blk(blk_schid)) {
-                    found = true;
-                    break;
-                }
+        if (!schib->pmcw.dnv) {
+            continue;
+        }
+        if (!virtio_is_supported(blk_schid)) {
+            continue;
+        }
+        if ((dev_no < 0) || (schib->pmcw.dev == dev_no)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void virtio_setup(void)
+{
+    Schib schib;
+    int ssid;
+    bool found = false;
+    uint16_t dev_no;
+
+    /*
+     * We unconditionally enable mss support. In every sane configuration,
+     * this will succeed; and even if it doesn't, stsch_err() can deal
+     * with the consequences.
+     */
+    enable_mss_facility();
+
+    if (store_iplb(&iplb)) {
+        switch (iplb.pbt) {
+        case S390_IPL_TYPE_CCW:
+            dev_no = iplb.ccw.devno;
+            debug_print_int("device no. ", dev_no);
+            blk_schid.ssid = iplb.ccw.ssid & 0x3;
+            debug_print_int("ssid ", blk_schid.ssid);
+            found = find_dev(&schib, dev_no);
+            break;
+        case S390_IPL_TYPE_QEMU_SCSI:
+        {
+            VDev *vdev = virtio_get_device();
+
+            vdev->scsi_device_selected = true;
+            vdev->selected_scsi_device.channel = iplb.scsi.channel;
+            vdev->selected_scsi_device.target = iplb.scsi.target;
+            vdev->selected_scsi_device.lun = iplb.scsi.lun;
+            blk_schid.ssid = iplb.scsi.ssid & 0x3;
+            found = find_dev(&schib, iplb.scsi.devno);
+            break;
+        }
+        default:
+            panic("List-directed IPL not supported yet!\n");
+        }
+    } else {
+        for (ssid = 0; ssid < 0x3; ssid++) {
+            blk_schid.ssid = ssid;
+            found = find_dev(&schib, -1);
+            if (found) {
+                break;
             }
         }
     }
 
-    if (!found) {
-        virtio_panic("No virtio-blk device found!\n");
-    }
+    IPL_assert(found, "No virtio device found");
 
-    virtio_setup_block(blk_schid);
+    virtio_setup_device(blk_schid);
 
-    if (!virtio_ipl_disk_is_valid()) {
-        virtio_panic("No valid hard disk detected.\n");
-    }
+    IPL_assert(virtio_ipl_disk_is_valid(), "No valid IPL device detected");
 }
 
 int main(void)
 {
     sclp_setup();
-    debug_print_int("boot reg[7] ", boot_value);
-    virtio_setup(boot_value);
+    virtio_setup();
 
     zipl_load(); /* no return */
 
-    virtio_panic("Failed to load OS from hard disk\n");
+    panic("Failed to load OS from hard disk\n");
     return 0; /* make compiler happy */
 }
diff --git a/pc-bios/s390-ccw/s390-ccw.h b/pc-bios/s390-ccw/s390-ccw.h
index 2b773de..ded67bc 100644
--- a/pc-bios/s390-ccw/s390-ccw.h
+++ b/pc-bios/s390-ccw/s390-ccw.h
@@ -44,13 +44,24 @@
 #endif
 
 #include "cio.h"
+#include "iplb.h"
+
+typedef struct irb Irb;
+typedef struct ccw1 Ccw1;
+typedef struct cmd_orb CmdOrb;
+typedef struct schib Schib;
+typedef struct chsc_area_sda ChscAreaSda;
+typedef struct senseid SenseId;
+typedef struct subchannel_id SubChannelId;
 
 /* start.s */
 void disabled_wait(void);
+void consume_sclp_int(void);
 
 /* main.c */
-void virtio_panic(const char *string);
+void panic(const char *string);
 void write_subsystem_identification(void);
+extern char stack[PAGE_SIZE * 8] __attribute__((__aligned__(PAGE_SIZE)));
 
 /* sclp-ascii.c */
 void sclp_print(const char *string);
@@ -59,10 +70,11 @@
 /* virtio.c */
 unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
                                  ulong subchan_id, void *load_addr);
-bool virtio_is_blk(struct subchannel_id schid);
-void virtio_setup_block(struct subchannel_id schid);
+bool virtio_is_supported(SubChannelId schid);
+void virtio_setup_device(SubChannelId schid);
 int virtio_read(ulong sector, void *load_addr);
 int enable_mss_facility(void);
+ulong get_second(void);
 
 /* bootmap.c */
 void zipl_load(void);
@@ -139,4 +151,42 @@
 
 #define MAX_SECTOR_SIZE 4096
 
+static inline void sleep(unsigned int seconds)
+{
+    ulong target = get_second() + seconds;
+
+    while (get_second() < target) {
+        yield();
+    }
+}
+
+static inline void *memcpy(void *s1, const void *s2, size_t n)
+{
+    uint8_t *p1 = s1;
+    const uint8_t *p2 = s2;
+
+    while (n--) {
+        p1[n] = p2[n];
+    }
+    return s1;
+}
+
+static inline void IPL_assert(bool term, const char *message)
+{
+    if (!term) {
+        sclp_print("\n! ");
+        sclp_print(message);
+        panic(" !\n"); /* no return */
+    }
+}
+
+static inline void IPL_check(bool term, const char *message)
+{
+    if (!term) {
+        sclp_print("\n! WARNING: ");
+        sclp_print(message);
+        sclp_print(" !\n");
+    }
+}
+
 #endif /* S390_CCW_H */
diff --git a/pc-bios/s390-ccw/sclp-ascii.c b/pc-bios/s390-ccw/sclp-ascii.c
index 761fb44..dc1c3e4 100644
--- a/pc-bios/s390-ccw/sclp-ascii.c
+++ b/pc-bios/s390-ccw/sclp-ascii.c
@@ -24,6 +24,7 @@
                 "       srl     %0,28"
                 : "=&d" (cc) : "d" (command), "a" (__pa(sccb))
                 : "cc", "memory");
+        consume_sclp_int();
         if (cc == 3)
                 return -EIO;
         if (cc == 2)
diff --git a/pc-bios/s390-ccw/scsi.h b/pc-bios/s390-ccw/scsi.h
new file mode 100644
index 0000000..fc830f7
--- /dev/null
+++ b/pc-bios/s390-ccw/scsi.h
@@ -0,0 +1,184 @@
+/*
+ * SCSI definitions for s390 machine loader for qemu
+ *
+ * Copyright 2015 IBM Corp.
+ * Author: Eugene "jno" Dvurechenski <jno@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef SCSI_H
+#define SCSI_H
+
+#include "s390-ccw.h"
+
+#define SCSI_DEFAULT_CDB_SIZE                   32
+#define SCSI_DEFAULT_SENSE_SIZE                 96
+
+#define CDB_STATUS_GOOD                         0
+#define CDB_STATUS_CHECK_CONDITION              0x02U
+#define CDB_STATUS_VALID(status)    (((status) & ~0x3eU) == 0)
+
+#define SCSI_SENSE_CODE_MASK                    0x7fU
+#define SCSI_SENSE_KEY_MASK                     0x0fU
+#define SCSI_SENSE_KEY_NO_SENSE                 0
+#define SCSI_SENSE_KEY_UNIT_ATTENTION           6
+
+union ScsiLun {
+    uint64_t v64;        /* numeric shortcut                             */
+    uint8_t  v8[8];      /* generic 8 bytes representation               */
+    uint16_t v16[4];     /* 4-level big-endian LUN as specified by SAM-2 */
+};
+typedef union ScsiLun ScsiLun;
+
+struct ScsiSense70 {
+    uint8_t b0;         /* b0 & 7f = resp code (0x70 or 0x71)   */
+    uint8_t b1, b2;     /* b2 & 0f = sense key                  */
+    uint8_t u1[1 * 4 + 1 + 1 * 4];   /* b7 = N - 7                  */
+    uint8_t additional_sense_code;              /* b12          */
+    uint8_t additional_sense_code_qualifier;    /* b13          */
+    uint8_t u2[1 + 3 + 0];           /* up to N (<=252) bytes       */
+} __attribute__((packed));
+typedef struct ScsiSense70 ScsiSense70;
+
+/* don't confuse with virtio-scsi response/status fields! */
+
+static inline uint8_t scsi_sense_response(const void *p)
+{
+    return ((const ScsiSense70 *)p)->b0 & SCSI_SENSE_CODE_MASK;
+}
+
+static inline uint8_t scsi_sense_key(const void *p)
+{
+    return ((const ScsiSense70 *)p)->b2 & SCSI_SENSE_KEY_MASK;
+}
+
+#define SCSI_INQ_RDT_CDROM                      0x05
+
+struct ScsiInquiryStd {
+    uint8_t peripheral_qdt; /* b0, use (b0 & 0x1f) to get SCSI_INQ_RDT  */
+    uint8_t b1;             /* Removable Media Bit = b1 & 0x80          */
+    uint8_t spc_version;    /* b2                                       */
+    uint8_t b3;             /* b3 & 0x0f == resp_data_fmt == 2, must!   */
+    uint8_t u1[1 + 1 + 1 + 1 + 8];  /* b4..b15 unused, b4 = (N - 1)     */
+    char prod_id[16];       /* "QEMU CD-ROM" is here                    */
+    uint8_t u2[4            /* b32..b35 unused, mandatory               */
+              + 8 + 12 + 1 + 1 + 8 * 2 + 22  /* b36..95 unused, optional*/
+              + 0];          /* b96..bN unused, vendor specific          */
+    /* byte N                                                           */
+}  __attribute__((packed));
+typedef struct ScsiInquiryStd ScsiInquiryStd;
+
+struct ScsiCdbInquiry {
+    uint8_t command;     /* b0, == 0x12         */
+    uint8_t b1;          /* b1, |= 0x01 (evpd)  */
+    uint8_t b2;          /* b2; if evpd==1      */
+    uint16_t alloc_len;  /* b3, b4              */
+    uint8_t control;     /* b5                  */
+}  __attribute__((packed));
+typedef struct ScsiCdbInquiry ScsiCdbInquiry;
+
+struct ScsiCdbRead10 {
+    uint8_t command;    /* =0x28    */
+    uint8_t b1;
+    uint32_t lba;
+    uint8_t b6;
+    uint16_t xfer_length;
+    uint8_t control;
+}  __attribute__((packed));
+typedef struct ScsiCdbRead10 ScsiCdbRead10;
+
+struct ScsiCdbTestUnitReady {
+    uint8_t command;    /* =0x00    */
+    uint8_t b1_b4[4];
+    uint8_t control;
+} __attribute__((packed));
+typedef struct ScsiCdbTestUnitReady ScsiCdbTestUnitReady;
+
+struct ScsiCdbReportLuns {
+    uint8_t command;        /* =0xa0        */
+    uint8_t b1;
+    uint8_t select_report;  /* =0x02, "all" */
+    uint8_t b3_b5[3];
+    uint32_t alloc_len;
+    uint8_t b10;
+    uint8_t control;
+} __attribute__((packed));
+typedef struct ScsiCdbReportLuns ScsiCdbReportLuns;
+
+struct ScsiLunReport {
+    uint32_t lun_list_len;
+    uint32_t b4_b7;
+    ScsiLun lun[1];   /* space for at least 1 lun must be allocated */
+} __attribute__((packed));
+typedef struct ScsiLunReport ScsiLunReport;
+
+struct ScsiCdbReadCapacity16 {
+    uint8_t command;        /* =0x9e = "service action in 16"       */
+    uint8_t service_action; /* 5 bits, =0x10 = "read capacity 16"   */
+    uint64_t b2_b9;
+    uint32_t alloc_len;
+    uint8_t b14;
+    uint8_t control;
+} __attribute__((packed));
+typedef struct ScsiCdbReadCapacity16 ScsiCdbReadCapacity16;
+
+struct ScsiReadCapacity16Data {
+    uint64_t ret_lba;             /* get it, 0..7     */
+    uint32_t lb_len;              /* bytes, 8..11     */
+    uint8_t u1[2 + 1 * 2 + 16];   /* b12..b31, unused */
+} __attribute__((packed));
+typedef struct ScsiReadCapacity16Data ScsiReadCapacity16Data;
+
+static inline ScsiLun make_lun(uint16_t channel, uint16_t target, uint32_t lun)
+{
+    ScsiLun r = { .v64 = 0 };
+
+    /* See QEMU code to choose the way to handle LUNs.
+     *
+     * So, a valid LUN must have (always channel #0):
+     *  lun[0] == 1
+     *  lun[1] - target, any value
+     *  lun[2] == 0 or (LUN, MSB, 0x40 set, 0x80 clear)
+     *  lun[3] - LUN, LSB, any value
+     */
+    r.v8[0] = 1;
+    r.v8[1] = target & 0xffU;
+    r.v8[2] = (lun >> 8) & 0x3fU;
+    if (r.v8[2]) {
+        r.v8[2] |= 0x40;
+    }
+    r.v8[3] = lun & 0xffU;
+
+    return r;
+}
+
+static inline const char *scsi_cdb_status_msg(uint8_t status)
+{
+    static char err_msg[] = "STATUS=XX";
+    uint8_t v = status & 0x3eU;
+
+    fill_hex_val(err_msg + 7, &v, 1);
+    return err_msg;
+}
+
+static inline const char *scsi_cdb_asc_msg(const void *s)
+{
+    static char err_msg[] = "RSPN=XX KEY=XX CODE=XX QLFR=XX";
+    const ScsiSense70 *p = s;
+    uint8_t sr = scsi_sense_response(s);
+    uint8_t sk = scsi_sense_key(s);
+    uint8_t ac = p->additional_sense_code;
+    uint8_t cq = p->additional_sense_code_qualifier;
+
+    fill_hex_val(err_msg + 5, &sr, 1);
+    fill_hex_val(err_msg + 12, &sk, 1);
+    fill_hex_val(err_msg + 20, &ac, 1);
+    fill_hex_val(err_msg + 28, &cq, 1);
+
+    return err_msg;
+}
+
+#endif /* SCSI_H */
diff --git a/pc-bios/s390-ccw/start.S b/pc-bios/s390-ccw/start.S
index 5d5df0d..43f9bd2 100644
--- a/pc-bios/s390-ccw/start.S
+++ b/pc-bios/s390-ccw/start.S
@@ -14,8 +14,6 @@
 _start:
 
 larl	%r15, stack + 0x8000    /* Set up stack */
-larl    %r6, boot_value
-stg     %r7, 0(%r6)     /* save the boot_value before any function calls */
 j	main                    /* And call C */
 
 /*
@@ -28,6 +26,38 @@
         larl %r1,disabled_wait_psw
         lpswe   0(%r1)
 
+
+/*
+ * void consume_sclp_int(void)
+ *
+ * eats one sclp interrupt
+ */
+        .globl consume_sclp_int
+consume_sclp_int:
+        /* enable service interrupts in cr0 */
+        stctg 0,0,0(15)
+        oi 6(15), 0x2
+        lctlg 0,0,0(15)
+        /* prepare external call handler */
+        larl %r1, external_new_code
+        stg %r1, 0x1b8
+        larl %r1, external_new_mask
+        mvc 0x1b0(8),0(%r1)
+        /* load enabled wait PSW */
+        larl %r1, enabled_wait_psw
+        lpswe 0(%r1)
+
+external_new_code:
+        /* disable service interrupts in cr0 */
+        stctg 0,0,0(15)
+        ni 6(15), 0xfd
+        lctlg 0,0,0(15)
+        br 14
+
         .align  8
 disabled_wait_psw:
         .quad   0x0002000180000000,0x0000000000000000
+enabled_wait_psw:
+        .quad   0x0302000180000000,0x0000000000000000
+external_new_mask:
+        .quad   0x0000000180000000
diff --git a/pc-bios/s390-ccw/virtio-scsi.c b/pc-bios/s390-ccw/virtio-scsi.c
new file mode 100644
index 0000000..d850a8d
--- /dev/null
+++ b/pc-bios/s390-ccw/virtio-scsi.c
@@ -0,0 +1,353 @@
+/*
+ * Virtio-SCSI implementation for s390 machine loader for qemu
+ *
+ * Copyright 2015 IBM Corp.
+ * Author: Eugene "jno" Dvurechenski <jno@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include "s390-ccw.h"
+#include "virtio.h"
+#include "scsi.h"
+#include "virtio-scsi.h"
+
+static ScsiDevice default_scsi_device;
+static VirtioScsiCmdReq req;
+static VirtioScsiCmdResp resp;
+
+static uint8_t scsi_inquiry_std_response[256];
+
+static inline void vs_assert(bool term, const char **msgs)
+{
+    if (!term) {
+        int i = 0;
+
+        sclp_print("\n! ");
+        while (msgs[i]) {
+            sclp_print(msgs[i++]);
+        }
+        panic(" !\n");
+    }
+}
+
+static void virtio_scsi_verify_response(VirtioScsiCmdResp *resp,
+                                        const char *title)
+{
+    const char *mr[] = {
+        title, ": response ", virtio_scsi_response_msg(resp), 0
+    };
+    const char *ms[] = {
+        title,
+        CDB_STATUS_VALID(resp->status) ? ": " : ": invalid ",
+        scsi_cdb_status_msg(resp->status),
+        resp->status == CDB_STATUS_CHECK_CONDITION ? " " : 0,
+        resp->sense_len ? scsi_cdb_asc_msg(resp->sense)
+                        : "no sense data",
+        scsi_sense_response(resp->sense)  == 0x70 ? ", sure" : "?",
+        0
+    };
+
+    vs_assert(resp->response == VIRTIO_SCSI_S_OK, mr);
+    vs_assert(resp->status == CDB_STATUS_GOOD, ms);
+}
+
+static void prepare_request(VDev *vdev, const void *cdb, int cdb_size,
+                            void *data, uint32_t data_size)
+{
+    const ScsiDevice *sdev = vdev->scsi_device;
+
+    memset(&req, 0, sizeof(req));
+    req.lun = make_lun(sdev->channel, sdev->target, sdev->lun);
+    memcpy(&req.cdb, cdb, cdb_size);
+
+    memset(&resp, 0, sizeof(resp));
+    resp.status = 0xff;     /* set invalid  */
+    resp.response = 0xff;   /*              */
+
+    if (data && data_size) {
+        memset(data, 0, data_size);
+    }
+}
+
+static inline void vs_io_assert(bool term, const char *msg)
+{
+    if (!term) {
+        virtio_scsi_verify_response(&resp, msg);
+    }
+}
+
+static void vs_run(const char *title, VirtioCmd *cmd, VDev *vdev,
+                   const void *cdb, int cdb_size,
+                   void *data, uint32_t data_size)
+{
+    prepare_request(vdev, cdb, cdb_size, data, data_size);
+    vs_io_assert(virtio_run(vdev, VR_REQUEST, cmd) == 0, title);
+}
+
+/* SCSI protocol implementation routines */
+
+static bool scsi_inquiry(VDev *vdev, void *data, uint32_t data_size)
+{
+    ScsiCdbInquiry cdb = {
+        .command = 0x12,
+        .alloc_len = data_size < 65535 ? data_size : 65535,
+    };
+    VirtioCmd inquiry[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size, VRING_DESC_F_WRITE },
+    };
+
+    vs_run("inquiry", inquiry, vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_test_unit_ready(VDev *vdev)
+{
+    ScsiCdbTestUnitReady cdb = {
+        .command = 0x00,
+    };
+    VirtioCmd test_unit_ready[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE },
+    };
+
+    prepare_request(vdev, &cdb, sizeof(cdb), 0, 0);
+    virtio_run(vdev, VR_REQUEST, test_unit_ready); /* ignore errors here */
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_report_luns(VDev *vdev, void *data, uint32_t data_size)
+{
+    ScsiCdbReportLuns cdb = {
+        .command = 0xa0,
+        .select_report = 0x02, /* REPORT ALL */
+        .alloc_len = data_size,
+    };
+    VirtioCmd report_luns[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size, VRING_DESC_F_WRITE },
+    };
+
+    vs_run("report luns", report_luns,
+           vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_read_10(VDev *vdev,
+                         ulong sector, int sectors, void *data)
+{
+    int f = vdev->blk_factor;
+    unsigned int data_size = sectors * virtio_get_block_size() * f;
+    ScsiCdbRead10 cdb = {
+        .command = 0x28,
+        .lba = sector * f,
+        .xfer_length = sectors * f,
+    };
+    VirtioCmd read_10[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size * f, VRING_DESC_F_WRITE },
+    };
+
+    debug_print_int("read_10  sector", sector);
+    debug_print_int("read_10 sectors", sectors);
+
+    vs_run("read(10)", read_10, vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+static bool scsi_read_capacity(VDev *vdev,
+                               void *data, uint32_t data_size)
+{
+    ScsiCdbReadCapacity16 cdb = {
+        .command = 0x9e, /* SERVICE_ACTION_IN_16 */
+        .service_action = 0x10, /* SA_READ_CAPACITY */
+        .alloc_len = data_size,
+    };
+    VirtioCmd read_capacity_16[] = {
+        { &req, sizeof(req), VRING_DESC_F_NEXT },
+        { &resp, sizeof(resp), VRING_DESC_F_WRITE | VRING_DESC_F_NEXT },
+        { data, data_size, VRING_DESC_F_WRITE },
+    };
+
+    vs_run("read capacity", read_capacity_16,
+           vdev, &cdb, sizeof(cdb), data, data_size);
+
+    return virtio_scsi_response_ok(&resp);
+}
+
+/* virtio-scsi routines */
+
+static void virtio_scsi_locate_device(VDev *vdev)
+{
+    const uint16_t channel = 0; /* again, it's what QEMU does */
+    uint16_t target;
+    static uint8_t data[16 + 8 * 63];
+    ScsiLunReport *r = (void *) data;
+    ScsiDevice *sdev = vdev->scsi_device;
+    int i, luns;
+
+    /* QEMU has hardcoded channel #0 in many places.
+     * If this hardcoded value is ever changed, we'll need to add code for
+     * vdev->config.scsi.max_channel != 0 here.
+     */
+    debug_print_int("config.scsi.max_channel", vdev->config.scsi.max_channel);
+    debug_print_int("config.scsi.max_target ", vdev->config.scsi.max_target);
+    debug_print_int("config.scsi.max_lun    ", vdev->config.scsi.max_lun);
+
+    if (vdev->scsi_device_selected) {
+        sdev->channel = vdev->selected_scsi_device.channel;
+        sdev->target = vdev->selected_scsi_device.target;
+        sdev->lun = vdev->selected_scsi_device.lun;
+
+        IPL_check(sdev->channel == 0, "non-zero channel requested");
+        IPL_check(sdev->target <= vdev->config.scsi.max_target, "target# high");
+        IPL_check(sdev->lun <= vdev->config.scsi.max_lun, "LUN# high");
+        return;
+    }
+
+    for (target = 0; target <= vdev->config.scsi.max_target; target++) {
+        sdev->channel = channel;
+        sdev->target = target; /* sdev->lun will be 0 here */
+        if (!scsi_report_luns(vdev, data, sizeof(data))) {
+            if (resp.response == VIRTIO_SCSI_S_BAD_TARGET) {
+                continue;
+            }
+            print_int("target", target);
+            virtio_scsi_verify_response(&resp, "SCSI cannot report LUNs");
+        }
+        if (r->lun_list_len == 0) {
+            print_int("no LUNs for target", target);
+            continue;
+        }
+        luns = r->lun_list_len / 8;
+        debug_print_int("LUNs reported", luns);
+        if (luns == 1) {
+            /* There is no ",lun=#" arg for -device or ",lun=0" given.
+             * Hence, the only LUN reported.
+             * Usually, it's 0.
+             */
+            sdev->lun = r->lun[0].v16[0]; /* it's returned this way */
+            debug_print_int("Have to use LUN", sdev->lun);
+            return; /* we have to use this device */
+        }
+        for (i = 0; i < luns; i++) {
+            if (r->lun[i].v64) {
+                /* Look for non-zero LUN - we have where to choose from */
+                sdev->lun = r->lun[i].v16[0];
+                debug_print_int("Will use LUN", sdev->lun);
+                return; /* we have found a device */
+            }
+        }
+    }
+    panic("\n! Cannot locate virtio-scsi device !\n");
+}
+
+int virtio_scsi_read_many(VDev *vdev,
+                          ulong sector, void *load_addr, int sec_num)
+{
+    if (!scsi_read_10(vdev, sector, sec_num, load_addr)) {
+        virtio_scsi_verify_response(&resp, "virtio-scsi:read_many");
+    }
+
+    return 0;
+}
+
+static bool virtio_scsi_inquiry_response_is_cdrom(void *data)
+{
+    const ScsiInquiryStd *response = data;
+    const int resp_data_fmt = response->b3 & 0x0f;
+    int i;
+
+    IPL_check(resp_data_fmt == 2, "Wrong INQUIRY response format");
+    if (resp_data_fmt != 2) {
+        return false; /* cannot decode */
+    }
+
+    if ((response->peripheral_qdt & 0x1f) == SCSI_INQ_RDT_CDROM) {
+        return true;
+    }
+
+    for (i = 0; i < sizeof(response->prod_id); i++) {
+        if (response->prod_id[i] != QEMU_CDROM_SIGNATURE[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static void scsi_parse_capacity_report(void *data,
+                                       uint64_t *last_lba, uint32_t *lb_len)
+{
+    ScsiReadCapacity16Data *p = data;
+
+    if (last_lba) {
+        *last_lba = p->ret_lba;
+    }
+
+    if (lb_len) {
+        *lb_len = p->lb_len;
+    }
+}
+
+void virtio_scsi_setup(VDev *vdev)
+{
+    int retry_test_unit_ready = 3;
+    uint8_t data[256];
+    uint32_t data_size = sizeof(data);
+
+    vdev->scsi_device = &default_scsi_device;
+    virtio_scsi_locate_device(vdev);
+
+    /* We have to "ping" the device before it becomes readable */
+    while (!scsi_test_unit_ready(vdev)) {
+
+        if (!virtio_scsi_response_ok(&resp)) {
+            uint8_t code = resp.sense[0] & SCSI_SENSE_CODE_MASK;
+            uint8_t sense_key = resp.sense[2] & SCSI_SENSE_KEY_MASK;
+
+            IPL_assert(resp.sense_len != 0, "virtio-scsi:setup: no SENSE data");
+
+            IPL_assert(retry_test_unit_ready && code == 0x70 &&
+                       sense_key == SCSI_SENSE_KEY_UNIT_ATTENTION,
+                       "virtio-scsi:setup: cannot retry");
+
+            /* retry on CHECK_CONDITION/UNIT_ATTENTION as it
+             * may not designate a real error, but it may be
+             * a result of device reset, etc.
+             */
+            retry_test_unit_ready--;
+            sleep(1);
+            continue;
+        }
+
+        virtio_scsi_verify_response(&resp, "virtio-scsi:setup");
+    }
+
+    /* read and cache SCSI INQUIRY response */
+    if (!scsi_inquiry(vdev, scsi_inquiry_std_response,
+                      sizeof(scsi_inquiry_std_response))) {
+        virtio_scsi_verify_response(&resp, "virtio-scsi:setup:inquiry");
+    }
+
+    if (virtio_scsi_inquiry_response_is_cdrom(scsi_inquiry_std_response)) {
+        sclp_print("SCSI CD-ROM detected.\n");
+        vdev->is_cdrom = true;
+        vdev->scsi_block_size = VIRTIO_ISO_BLOCK_SIZE;
+    }
+
+    if (!scsi_read_capacity(vdev, data, data_size)) {
+        virtio_scsi_verify_response(&resp, "virtio-scsi:setup:read_capacity");
+    }
+    scsi_parse_capacity_report(data, &vdev->scsi_last_block,
+                               (uint32_t *) &vdev->scsi_block_size);
+}
diff --git a/pc-bios/s390-ccw/virtio-scsi.h b/pc-bios/s390-ccw/virtio-scsi.h
new file mode 100644
index 0000000..f50b38b
--- /dev/null
+++ b/pc-bios/s390-ccw/virtio-scsi.h
@@ -0,0 +1,72 @@
+/*
+ * Virtio-SCSI definitions for s390 machine loader for qemu
+ *
+ * Copyright 2015 IBM Corp.
+ * Author: Eugene "jno" Dvurechenski <jno@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef VIRTIO_SCSI_H
+#define VIRTIO_SCSI_H
+
+#include "s390-ccw.h"
+#include "virtio.h"
+#include "scsi.h"
+
+#define VIRTIO_SCSI_CDB_SIZE   SCSI_DEFAULT_CDB_SIZE
+#define VIRTIO_SCSI_SENSE_SIZE SCSI_DEFAULT_SENSE_SIZE
+
+/* command-specific response values */
+#define VIRTIO_SCSI_S_OK                     0x00
+#define VIRTIO_SCSI_S_BAD_TARGET             0x03
+
+#define QEMU_CDROM_SIGNATURE "QEMU CD-ROM     "
+
+enum virtio_scsi_vq_id {
+    VR_CONTROL  = 0,
+    VR_EVENT    = 1,
+    VR_REQUEST  = 2,
+};
+
+struct VirtioScsiCmdReq {
+    ScsiLun lun;
+    uint64_t id;
+    uint8_t task_attr;   /* = 0 = VIRTIO_SCSI_S_SIMPLE */
+    uint8_t prio;
+    uint8_t crn;         /* = 0 */
+    uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
+} __attribute__((packed));
+typedef struct VirtioScsiCmdReq VirtioScsiCmdReq;
+
+struct VirtioScsiCmdResp {
+        uint32_t sense_len;
+        uint32_t residual;
+        uint16_t status_qualifier;
+        uint8_t status;      /* first check for .response    */
+        uint8_t response;    /* then for .status             */
+        uint8_t sense[VIRTIO_SCSI_SENSE_SIZE];
+} __attribute__((packed));
+typedef struct VirtioScsiCmdResp VirtioScsiCmdResp;
+
+static inline const char *virtio_scsi_response_msg(const VirtioScsiCmdResp *r)
+{
+    static char err_msg[] = "VS RESP=XX";
+    uint8_t v = r->response;
+
+    fill_hex_val(err_msg + 8, &v, 1);
+    return err_msg;
+}
+
+static inline bool virtio_scsi_response_ok(const VirtioScsiCmdResp *r)
+{
+        return r->response == VIRTIO_SCSI_S_OK && r->status == CDB_STATUS_GOOD;
+}
+
+void virtio_scsi_setup(VDev *vdev);
+int virtio_scsi_read_many(VDev *vdev,
+                          ulong sector, void *load_addr, int sec_num);
+
+#endif /* VIRTIO_SCSI_H */
diff --git a/pc-bios/s390-ccw/virtio.c b/pc-bios/s390-ccw/virtio.c
index c0540d1..1d34e8c 100644
--- a/pc-bios/s390-ccw/virtio.c
+++ b/pc-bios/s390-ccw/virtio.c
@@ -10,39 +10,68 @@
 
 #include "s390-ccw.h"
 #include "virtio.h"
+#include "virtio-scsi.h"
 
-struct vring block;
+#define VRING_WAIT_REPLY_TIMEOUT 3
+
+static VRing block[VIRTIO_MAX_VQS];
+static char ring_area[VIRTIO_RING_SIZE * VIRTIO_MAX_VQS]
+                     __attribute__((__aligned__(PAGE_SIZE)));
 
 static char chsc_page[PAGE_SIZE] __attribute__((__aligned__(PAGE_SIZE)));
 
+static VDev vdev = {
+    .nr_vqs = 1,
+    .vrings = block,
+    .cmd_vr_idx = 0,
+    .ring_area = ring_area,
+    .wait_reply_timeout = VRING_WAIT_REPLY_TIMEOUT,
+    .schid = { .one = 1 },
+    .scsi_block_size = VIRTIO_SCSI_BLOCK_SIZE,
+    .blk_factor = 1,
+};
+
+VDev *virtio_get_device(void)
+{
+    return &vdev;
+}
+
+VirtioDevType virtio_get_device_type(void)
+{
+    return vdev.senseid.cu_model;
+}
+
+/* virtio spec v1.0 para 4.3.3.2 */
 static long kvm_hypercall(unsigned long nr, unsigned long param1,
-                          unsigned long param2)
+                          unsigned long param2, unsigned long param3)
 {
     register ulong r_nr asm("1") = nr;
     register ulong r_param1 asm("2") = param1;
     register ulong r_param2 asm("3") = param2;
+    register ulong r_param3 asm("4") = param3;
     register long retval asm("2");
 
     asm volatile ("diag 2,4,0x500"
                   : "=d" (retval)
-                  : "d" (r_nr), "0" (r_param1), "r"(r_param2)
+                  : "d" (r_nr), "0" (r_param1), "r"(r_param2), "d"(r_param3)
                   : "memory", "cc");
 
     return retval;
 }
 
-static void virtio_notify(struct subchannel_id schid)
+static long virtio_notify(SubChannelId schid, int vq_idx, long cookie)
 {
-    kvm_hypercall(KVM_S390_VIRTIO_CCW_NOTIFY, *(u32 *)&schid, 0);
+    return kvm_hypercall(KVM_S390_VIRTIO_CCW_NOTIFY, *(u32 *)&schid,
+                         vq_idx, cookie);
 }
 
 /***********************************************
  *             Virtio functions                *
  ***********************************************/
 
-static int drain_irqs(struct subchannel_id schid)
+static int drain_irqs(SubChannelId schid)
 {
-    struct irb irb = {};
+    Irb irb = {};
     int r = 0;
 
     while (1) {
@@ -59,17 +88,17 @@
     }
 }
 
-static int run_ccw(struct subchannel_id schid, int cmd, void *ptr, int len)
+static int run_ccw(VDev *vdev, int cmd, void *ptr, int len)
 {
-    struct ccw1 ccw = {};
-    struct cmd_orb orb = {};
-    struct schib schib;
+    Ccw1 ccw = {};
+    CmdOrb orb = {};
+    Schib schib;
     int r;
 
     /* start command processing */
-    stsch_err(schid, &schib);
+    stsch_err(vdev->schid, &schib);
     schib.scsw.ctrl = SCSW_FCTL_START_FUNC;
-    msch(schid, &schib);
+    msch(vdev->schid, &schib);
 
     /* start subchannel command */
     orb.fmt = 1;
@@ -80,41 +109,29 @@
     ccw.cda = (long)ptr;
     ccw.count = len;
 
-    r = ssch(schid, &orb);
+    r = ssch(vdev->schid, &orb);
     /*
      * XXX Wait until device is done processing the CCW. For now we can
      *     assume that a simple tsch will have finished the CCW processing,
      *     but the architecture allows for asynchronous operation
      */
     if (!r) {
-        r = drain_irqs(schid);
+        r = drain_irqs(vdev->schid);
     }
     return r;
 }
 
-static void virtio_set_status(struct subchannel_id schid,
-                              unsigned long dev_addr)
+static void vring_init(VRing *vr, VqInfo *info)
 {
-    unsigned char status = dev_addr;
-    if (run_ccw(schid, CCW_CMD_WRITE_STATUS, &status, sizeof(status))) {
-        virtio_panic("Could not write status to host!\n");
-    }
-}
+    void *p = (void *) info->queue;
 
-static void virtio_reset(struct subchannel_id schid)
-{
-    run_ccw(schid, CCW_CMD_VDEV_RESET, NULL, 0);
-}
-
-static void vring_init(struct vring *vr, unsigned int num, void *p,
-                       unsigned long align)
-{
     debug_print_addr("init p", p);
-    vr->num = num;
+    vr->id = info->index;
+    vr->num = info->num;
     vr->desc = p;
-    vr->avail = p + num*sizeof(struct vring_desc);
-    vr->used = (void *)(((unsigned long)&vr->avail->ring[num] + align-1)
-                & ~(align - 1));
+    vr->avail = p + info->num * sizeof(VRingDesc);
+    vr->used = (void *)(((unsigned long)&vr->avail->ring[info->num]
+               + info->align - 1) & ~(info->align - 1));
 
     /* Zero out all relevant field */
     vr->avail->flags = 0;
@@ -125,16 +142,18 @@
     vr->used->idx = 0;
     vr->used_idx = 0;
     vr->next_idx = 0;
+    vr->cookie = 0;
 
     debug_print_addr("init vr", vr);
 }
 
-static void vring_notify(struct subchannel_id schid)
+static bool vring_notify(VRing *vr)
 {
-    virtio_notify(schid);
+    vr->cookie = virtio_notify(vr->schid, vr->id, vr->cookie);
+    return vr->cookie >= 0;
 }
 
-static void vring_send_buf(struct vring *vr, void *p, int len, int flags)
+static void vring_send_buf(VRing *vr, void *p, int len, int flags)
 {
     /* For follow-up chains we need to keep the first entry point */
     if (!(flags & VRING_HIDDEN_IS_CHAIN)) {
@@ -162,11 +181,26 @@
     return r;
 }
 
-static ulong get_second(void)
+ulong get_second(void)
 {
     return (get_clock() >> 12) / 1000000;
 }
 
+static int vr_poll(VRing *vr)
+{
+    if (vr->used->idx == vr->used_idx) {
+        vring_notify(vr);
+        yield();
+        return 0;
+    }
+
+    vr->used_idx = vr->used->idx;
+    vr->next_idx = 0;
+    vr->desc[0].len = 0;
+    vr->desc[0].flags = 0;
+    return 1; /* vr has been updated */
+}
+
 /*
  * Wait for the host to reply.
  *
@@ -174,67 +208,92 @@
  *
  * Returns 0 on success, 1 on timeout.
  */
-static int vring_wait_reply(struct vring *vr, int timeout)
+static int vring_wait_reply(void)
 {
-    ulong target_second = get_second() + timeout;
-    struct subchannel_id schid = vr->schid;
-    int r = 0;
+    ulong target_second = get_second() + vdev.wait_reply_timeout;
 
-    /* Wait until the used index has moved. */
-    while (vr->used->idx == vr->used_idx) {
-        vring_notify(schid);
-        if (timeout && (get_second() >= target_second)) {
-            r = 1;
-            break;
+    /* Wait for any queue to be updated by the host */
+    do {
+        int i, r = 0;
+
+        for (i = 0; i < vdev.nr_vqs; i++) {
+            r += vr_poll(&vdev.vrings[i]);
         }
         yield();
+        if (r) {
+            return 0;
+        }
+    } while (!vdev.wait_reply_timeout || (get_second() < target_second));
+
+    return 1;
+}
+
+int virtio_run(VDev *vdev, int vqid, VirtioCmd *cmd)
+{
+    VRing *vr = &vdev->vrings[vqid];
+    int i = 0;
+
+    do {
+        vring_send_buf(vr, cmd[i].data, cmd[i].size,
+                       cmd[i].flags | (i ? VRING_HIDDEN_IS_CHAIN : 0));
+    } while (cmd[i++].flags & VRING_DESC_F_NEXT);
+
+    vring_wait_reply();
+    if (drain_irqs(vr->schid)) {
+        return -1;
     }
-
-    vr->used_idx = vr->used->idx;
-    vr->next_idx = 0;
-    vr->desc[0].len = 0;
-    vr->desc[0].flags = 0;
-
-    return r;
+    return 0;
 }
 
 /***********************************************
  *               Virtio block                  *
  ***********************************************/
 
-int virtio_read_many(ulong sector, void *load_addr, int sec_num)
+static int virtio_blk_read_many(VDev *vdev,
+                                ulong sector, void *load_addr, int sec_num)
 {
-    struct virtio_blk_outhdr out_hdr;
+    VirtioBlkOuthdr out_hdr;
     u8 status;
-    int r;
+    VRing *vr = &vdev->vrings[vdev->cmd_vr_idx];
 
     /* Tell the host we want to read */
     out_hdr.type = VIRTIO_BLK_T_IN;
     out_hdr.ioprio = 99;
     out_hdr.sector = virtio_sector_adjust(sector);
 
-    vring_send_buf(&block, &out_hdr, sizeof(out_hdr), VRING_DESC_F_NEXT);
+    vring_send_buf(vr, &out_hdr, sizeof(out_hdr), VRING_DESC_F_NEXT);
 
     /* This is where we want to receive data */
-    vring_send_buf(&block, load_addr, virtio_get_block_size() * sec_num,
+    vring_send_buf(vr, load_addr, virtio_get_block_size() * sec_num,
                    VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN |
                    VRING_DESC_F_NEXT);
 
     /* status field */
-    vring_send_buf(&block, &status, sizeof(u8), VRING_DESC_F_WRITE |
-                   VRING_HIDDEN_IS_CHAIN);
+    vring_send_buf(vr, &status, sizeof(u8),
+                   VRING_DESC_F_WRITE | VRING_HIDDEN_IS_CHAIN);
 
     /* Now we can tell the host to read */
-    vring_wait_reply(&block, 0);
+    vring_wait_reply();
 
-    r = drain_irqs(block.schid);
-    if (r) {
+    if (drain_irqs(vr->schid)) {
         /* Well, whatever status is supposed to contain... */
         status = 1;
     }
     return status;
 }
 
+int virtio_read_many(ulong sector, void *load_addr, int sec_num)
+{
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return virtio_blk_read_many(&vdev, sector, load_addr, sec_num);
+    case VIRTIO_ID_SCSI:
+        return virtio_scsi_read_many(&vdev, sector, load_addr, sec_num);
+    }
+    panic("\n! No readable IPL device !\n");
+    return -1;
+}
+
 unsigned long virtio_load_direct(ulong rec_list1, ulong rec_list2,
                                  ulong subchan_id, void *load_addr)
 {
@@ -251,7 +310,7 @@
     sclp_print(".");
     status = virtio_read_many(sec, (void *)addr, sec_num);
     if (status) {
-        virtio_panic("I/O Error");
+        panic("I/O Error");
     }
     addr += sec_num * virtio_get_block_size();
 
@@ -263,42 +322,6 @@
     return virtio_read_many(sector, load_addr, 1);
 }
 
-static VirtioBlkConfig blk_cfg = {};
-static bool guessed_disk_nature;
-
-bool virtio_guessed_disk_nature(void)
-{
-    return guessed_disk_nature;
-}
-
-void virtio_assume_scsi(void)
-{
-    guessed_disk_nature = true;
-    blk_cfg.blk_size = 512;
-    blk_cfg.physical_block_exp = 0;
-}
-
-void virtio_assume_eckd(void)
-{
-    guessed_disk_nature = true;
-    blk_cfg.blk_size = 4096;
-    blk_cfg.physical_block_exp = 0;
-
-    /* this must be here to calculate code segment position */
-    blk_cfg.geometry.heads = 15;
-    blk_cfg.geometry.sectors = 12;
-}
-
-bool virtio_disk_is_scsi(void)
-{
-    if (guessed_disk_nature) {
-        return (virtio_get_block_size()  == 512);
-    }
-    return (blk_cfg.geometry.heads == 255)
-        && (blk_cfg.geometry.sectors == 63)
-        && (virtio_get_block_size()  == 512);
-}
-
 /*
  * Other supported value pairs, if any, would need to be added here.
  * Note: head count is always 15.
@@ -318,16 +341,91 @@
     return 0;
 }
 
+VirtioGDN virtio_guessed_disk_nature(void)
+{
+    return vdev.guessed_disk_nature;
+}
+
+void virtio_assume_scsi(void)
+{
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev.guessed_disk_nature = VIRTIO_GDN_SCSI;
+        vdev.config.blk.blk_size = VIRTIO_SCSI_BLOCK_SIZE;
+        vdev.config.blk.physical_block_exp = 0;
+        vdev.blk_factor = 1;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev.scsi_block_size = VIRTIO_SCSI_BLOCK_SIZE;
+        break;
+    }
+}
+
+void virtio_assume_iso9660(void)
+{
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev.guessed_disk_nature = VIRTIO_GDN_SCSI;
+        vdev.config.blk.blk_size = VIRTIO_ISO_BLOCK_SIZE;
+        vdev.config.blk.physical_block_exp = 0;
+        vdev.blk_factor = VIRTIO_ISO_BLOCK_SIZE / VIRTIO_SECTOR_SIZE;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev.scsi_block_size = VIRTIO_ISO_BLOCK_SIZE;
+        break;
+    }
+}
+
+void virtio_assume_eckd(void)
+{
+    vdev.guessed_disk_nature = VIRTIO_GDN_DASD;
+    vdev.blk_factor = 1;
+    vdev.config.blk.physical_block_exp = 0;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev.config.blk.blk_size = 4096;
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev.config.blk.blk_size = vdev.scsi_block_size;
+        break;
+    }
+    vdev.config.blk.geometry.heads = 15;
+    vdev.config.blk.geometry.sectors =
+        virtio_eckd_sectors_for_block_size(vdev.config.blk.blk_size);
+}
+
+bool virtio_disk_is_scsi(void)
+{
+    if (vdev.guessed_disk_nature == VIRTIO_GDN_SCSI) {
+        return true;
+    }
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return (vdev.config.blk.geometry.heads == 255)
+            && (vdev.config.blk.geometry.sectors == 63)
+            && (virtio_get_block_size()  == VIRTIO_SCSI_BLOCK_SIZE);
+    case VIRTIO_ID_SCSI:
+        return true;
+    }
+    return false;
+}
+
 bool virtio_disk_is_eckd(void)
 {
     const int block_size = virtio_get_block_size();
 
-    if (guessed_disk_nature) {
-        return (block_size  == 4096);
+    if (vdev.guessed_disk_nature == VIRTIO_GDN_DASD) {
+        return true;
     }
-    return (blk_cfg.geometry.heads == 15)
-        && (blk_cfg.geometry.sectors ==
-            virtio_eckd_sectors_for_block_size(block_size));
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return (vdev.config.blk.geometry.heads == 15)
+            && (vdev.config.blk.geometry.sectors ==
+                virtio_eckd_sectors_for_block_size(block_size));
+    case VIRTIO_ID_SCSI:
+        return false;
+    }
+    return false;
 }
 
 bool virtio_ipl_disk_is_valid(void)
@@ -337,33 +435,80 @@
 
 int virtio_get_block_size(void)
 {
-    return blk_cfg.blk_size << blk_cfg.physical_block_exp;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.blk_size << vdev.config.blk.physical_block_exp;
+    case VIRTIO_ID_SCSI:
+        return vdev.scsi_block_size;
+    }
+    return 0;
 }
 
 uint8_t virtio_get_heads(void)
 {
-    return blk_cfg.geometry.heads;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.geometry.heads;
+    case VIRTIO_ID_SCSI:
+        return vdev.guessed_disk_nature == VIRTIO_GDN_DASD
+               ? vdev.config.blk.geometry.heads : 255;
+    }
+    return 0;
 }
 
 uint8_t virtio_get_sectors(void)
 {
-    return blk_cfg.geometry.sectors;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.geometry.sectors;
+    case VIRTIO_ID_SCSI:
+        return vdev.guessed_disk_nature == VIRTIO_GDN_DASD
+               ? vdev.config.blk.geometry.sectors : 63;
+    }
+    return 0;
 }
 
 uint64_t virtio_get_blocks(void)
 {
-    return blk_cfg.capacity /
-           (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
+    const uint64_t factor = virtio_get_block_size() / VIRTIO_SECTOR_SIZE;
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        return vdev.config.blk.capacity / factor;
+    case VIRTIO_ID_SCSI:
+        return vdev.scsi_last_block / factor;
+    }
+    return 0;
 }
 
-void virtio_setup_block(struct subchannel_id schid)
+static void virtio_setup_ccw(VDev *vdev)
 {
-    struct vq_info_block info;
-    struct vq_config_block config = {};
+    int i, cfg_size = 0;
+    unsigned char status = VIRTIO_CONFIG_S_DRIVER_OK;
 
-    blk_cfg.blk_size = 0; /* mark "illegal" - setup started... */
+    IPL_assert(virtio_is_supported(vdev->schid), "PE");
+    /* device ID has been established now */
 
-    virtio_reset(schid);
+    vdev->config.blk.blk_size = 0; /* mark "illegal" - setup started... */
+    vdev->guessed_disk_nature = VIRTIO_GDN_NONE;
+
+    run_ccw(vdev, CCW_CMD_VDEV_RESET, NULL, 0);
+
+    switch (vdev->senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        vdev->nr_vqs = 1;
+        vdev->cmd_vr_idx = 0;
+        cfg_size = sizeof(vdev->config.blk);
+        break;
+    case VIRTIO_ID_SCSI:
+        vdev->nr_vqs = 3;
+        vdev->cmd_vr_idx = VR_REQUEST;
+        cfg_size = sizeof(vdev->config.scsi);
+        break;
+    default:
+        panic("Unsupported virtio device\n");
+    }
+    IPL_assert(run_ccw(vdev, CCW_CMD_READ_CONF, &vdev->config, cfg_size) == 0,
+               "Could not get block device configuration");
 
     /*
      * Skipping CCW_CMD_READ_FEAT. We're not doing anything fancy, and
@@ -371,54 +516,84 @@
      * expect it.
      */
 
-    config.index = 0;
-    if (run_ccw(schid, CCW_CMD_READ_VQ_CONF, &config, sizeof(config))) {
-        virtio_panic("Could not get block device VQ configuration\n");
-    }
-    if (run_ccw(schid, CCW_CMD_READ_CONF, &blk_cfg, sizeof(blk_cfg))) {
-        virtio_panic("Could not get block device configuration\n");
-    }
-    vring_init(&block, config.num, (void *)(100 * 1024 * 1024),
-               KVM_S390_VIRTIO_RING_ALIGN);
+    for (i = 0; i < vdev->nr_vqs; i++) {
+        VqInfo info = {
+            .queue = (unsigned long long) ring_area + (i * VIRTIO_RING_SIZE),
+            .align = KVM_S390_VIRTIO_RING_ALIGN,
+            .index = i,
+            .num = 0,
+        };
+        VqConfig config = {
+            .index = i,
+            .num = 0,
+        };
 
-    info.queue = (100ULL * 1024ULL* 1024ULL);
-    info.align = KVM_S390_VIRTIO_RING_ALIGN;
-    info.index = 0;
-    info.num = config.num;
-    block.schid = schid;
-
-    if (!run_ccw(schid, CCW_CMD_SET_VQ, &info, sizeof(info))) {
-        virtio_set_status(schid, VIRTIO_CONFIG_S_DRIVER_OK);
+        IPL_assert(
+            run_ccw(vdev, CCW_CMD_READ_VQ_CONF, &config, sizeof(config)) == 0,
+            "Could not get block device VQ configuration");
+        info.num = config.num;
+        vring_init(&vdev->vrings[i], &info);
+        vdev->vrings[i].schid = vdev->schid;
+        IPL_assert(run_ccw(vdev, CCW_CMD_SET_VQ, &info, sizeof(info)) == 0,
+                   "Cannot set VQ info");
     }
+    IPL_assert(
+        run_ccw(vdev, CCW_CMD_WRITE_STATUS, &status, sizeof(status)) == 0,
+        "Could not write status to host");
+}
 
-    if (!virtio_ipl_disk_is_valid()) {
-        /* make sure all getters but blocksize return 0 for invalid IPL disk */
-        memset(&blk_cfg, 0, sizeof(blk_cfg));
-        virtio_assume_scsi();
+void virtio_setup_device(SubChannelId schid)
+{
+    vdev.schid = schid;
+    virtio_setup_ccw(&vdev);
+
+    switch (vdev.senseid.cu_model) {
+    case VIRTIO_ID_BLOCK:
+        sclp_print("Using virtio-blk.\n");
+        if (!virtio_ipl_disk_is_valid()) {
+            /* make sure all getters but blocksize return 0 for
+             * invalid IPL disk
+             */
+            memset(&vdev.config.blk, 0, sizeof(vdev.config.blk));
+            virtio_assume_scsi();
+        }
+        break;
+    case VIRTIO_ID_SCSI:
+        IPL_assert(vdev.config.scsi.sense_size == VIRTIO_SCSI_SENSE_SIZE,
+            "Config: sense size mismatch");
+        IPL_assert(vdev.config.scsi.cdb_size == VIRTIO_SCSI_CDB_SIZE,
+            "Config: CDB size mismatch");
+
+        sclp_print("Using virtio-scsi.\n");
+        virtio_scsi_setup(&vdev);
+        break;
+    default:
+        panic("\n! No IPL device available !\n");
     }
 }
 
-bool virtio_is_blk(struct subchannel_id schid)
+bool virtio_is_supported(SubChannelId schid)
 {
-    int r;
-    struct senseid senseid = {};
-
+    vdev.schid = schid;
+    memset(&vdev.senseid, 0, sizeof(vdev.senseid));
     /* run sense id command */
-    r = run_ccw(schid, CCW_CMD_SENSE_ID, &senseid, sizeof(senseid));
-    if (r) {
+    if (run_ccw(&vdev, CCW_CMD_SENSE_ID, &vdev.senseid, sizeof(vdev.senseid))) {
         return false;
     }
-    if ((senseid.cu_type != 0x3832) || (senseid.cu_model != VIRTIO_ID_BLOCK)) {
-        return false;
+    if (vdev.senseid.cu_type == 0x3832) {
+        switch (vdev.senseid.cu_model) {
+        case VIRTIO_ID_BLOCK:
+        case VIRTIO_ID_SCSI:
+            return true;
+        }
     }
-
-    return true;
+    return false;
 }
 
 int enable_mss_facility(void)
 {
     int ret;
-    struct chsc_area_sda *sda_area = (struct chsc_area_sda *) chsc_page;
+    ChscAreaSda *sda_area = (ChscAreaSda *) chsc_page;
 
     memset(sda_area, 0, PAGE_SIZE);
     sda_area->request.length = 0x0400;
diff --git a/pc-bios/s390-ccw/virtio.h b/pc-bios/s390-ccw/virtio.h
index c23466b..eb35ea5 100644
--- a/pc-bios/s390-ccw/virtio.h
+++ b/pc-bios/s390-ccw/virtio.h
@@ -23,49 +23,58 @@
 /* We've given up on this device. */
 #define VIRTIO_CONFIG_S_FAILED          0x80
 
-enum virtio_dev_type {
+enum VirtioDevType {
     VIRTIO_ID_NET = 1,
     VIRTIO_ID_BLOCK = 2,
     VIRTIO_ID_CONSOLE = 3,
     VIRTIO_ID_BALLOON = 5,
+    VIRTIO_ID_SCSI = 8,
 };
+typedef enum VirtioDevType VirtioDevType;
 
-struct virtio_dev_header {
-    enum virtio_dev_type type : 8;
-    u8  num_vq;
-    u8  feature_len;
-    u8  config_len;
-    u8  status;
-    u8  vqconfig[];
+struct VirtioDevHeader {
+    VirtioDevType type:8;
+    uint8_t num_vq;
+    uint8_t feature_len;
+    uint8_t config_len;
+    uint8_t status;
+    uint8_t vqconfig[];
 } __attribute__((packed));
+typedef struct VirtioDevHeader VirtioDevHeader;
 
-struct virtio_vqconfig {
-    u64 token;
-    u64 address;
-    u16 num;
-    u8  pad[6];
+struct VirtioVqConfig {
+    uint64_t token;
+    uint64_t address;
+    uint16_t num;
+    uint8_t pad[6];
 } __attribute__((packed));
+typedef struct VirtioVqConfig VirtioVqConfig;
 
-struct vq_info_block {
-    u64 queue;
-    u32 align;
-    u16 index;
-    u16 num;
+struct VqInfo {
+    uint64_t queue;
+    uint32_t align;
+    uint16_t index;
+    uint16_t num;
 } __attribute__((packed));
+typedef struct VqInfo VqInfo;
 
-struct vq_config_block {
-    u16 index;
-    u16 num;
+struct VqConfig {
+    uint16_t index;
+    uint16_t num;
 } __attribute__((packed));
+typedef struct VqConfig VqConfig;
 
-struct virtio_dev {
-    struct virtio_dev_header *header;
-    struct virtio_vqconfig *vqconfig;
+struct VirtioDev {
+    VirtioDevHeader *header;
+    VirtioVqConfig *vqconfig;
     char *host_features;
     char *guest_features;
     char *config;
 };
+typedef struct VirtioDev VirtioDev;
 
+#define VIRTIO_RING_SIZE            (PAGE_SIZE * 8)
+#define VIRTIO_MAX_VQS              3
 #define KVM_S390_VIRTIO_RING_ALIGN  4096
 
 #define VRING_USED_F_NO_NOTIFY  1
@@ -81,46 +90,53 @@
 #define VRING_HIDDEN_IS_CHAIN   256
 
 /* Virtio ring descriptors: 16 bytes.  These can chain together via "next". */
-struct vring_desc {
+struct VRingDesc {
     /* Address (guest-physical). */
-    u64 addr;
+    uint64_t addr;
     /* Length. */
-    u32 len;
+    uint32_t len;
     /* The flags as indicated above. */
-    u16 flags;
+    uint16_t flags;
     /* We chain unused descriptors via this, too */
-    u16 next;
+    uint16_t next;
 } __attribute__((packed));
+typedef struct VRingDesc VRingDesc;
 
-struct vring_avail {
-    u16 flags;
-    u16 idx;
-    u16 ring[];
+struct VRingAvail {
+    uint16_t flags;
+    uint16_t idx;
+    uint16_t ring[];
 } __attribute__((packed));
+typedef struct VRingAvail VRingAvail;
 
-/* u32 is used here for ids for padding reasons. */
-struct vring_used_elem {
+/* uint32_t is used here for ids for padding reasons. */
+struct VRingUsedElem {
     /* Index of start of used descriptor chain. */
-    u32 id;
+    uint32_t id;
     /* Total length of the descriptor chain which was used (written to) */
-    u32 len;
+    uint32_t len;
 } __attribute__((packed));
+typedef struct VRingUsedElem VRingUsedElem;
 
-struct vring_used {
-    u16 flags;
-    u16 idx;
-    struct vring_used_elem ring[];
+struct VRingUsed {
+    uint16_t flags;
+    uint16_t idx;
+    VRingUsedElem ring[];
 } __attribute__((packed));
+typedef struct VRingUsed VRingUsed;
 
-struct vring {
+struct VRing {
     unsigned int num;
     int next_idx;
     int used_idx;
-    struct vring_desc *desc;
-    struct vring_avail *avail;
-    struct vring_used *used;
-    struct subchannel_id schid;
+    VRingDesc *desc;
+    VRingAvail *avail;
+    VRingUsed *used;
+    SubChannelId schid;
+    long cookie;
+    int id;
 };
+typedef struct VRing VRing;
 
 
 /***********************************************
@@ -152,41 +168,52 @@
 #define VIRTIO_BLK_T_BARRIER    0x80000000
 
 /* This is the first element of the read scatter-gather list. */
-struct virtio_blk_outhdr {
+struct VirtioBlkOuthdr {
         /* VIRTIO_BLK_T* */
-        u32 type;
+        uint32_t type;
         /* io priority. */
-        u32 ioprio;
+        uint32_t ioprio;
         /* Sector (ie. 512 byte offset) */
-        u64 sector;
+        uint64_t sector;
 };
+typedef struct VirtioBlkOuthdr VirtioBlkOuthdr;
 
-typedef struct VirtioBlkConfig {
-    u64 capacity; /* in 512-byte sectors */
-    u32 size_max; /* max segment size (if VIRTIO_BLK_F_SIZE_MAX) */
-    u32 seg_max;  /* max number of segments (if VIRTIO_BLK_F_SEG_MAX) */
+struct VirtioBlkConfig {
+    uint64_t capacity; /* in 512-byte sectors */
+    uint32_t size_max; /* max segment size (if VIRTIO_BLK_F_SIZE_MAX) */
+    uint32_t seg_max;  /* max number of segments (if VIRTIO_BLK_F_SEG_MAX) */
 
-    struct virtio_blk_geometry {
-        u16 cylinders;
-        u8 heads;
-        u8 sectors;
+    struct VirtioBlkGeometry {
+        uint16_t cylinders;
+        uint8_t heads;
+        uint8_t sectors;
     } geometry; /* (if VIRTIO_BLK_F_GEOMETRY) */
 
-    u32 blk_size; /* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
+    uint32_t blk_size; /* block size of device (if VIRTIO_BLK_F_BLK_SIZE) */
 
     /* the next 4 entries are guarded by VIRTIO_BLK_F_TOPOLOGY  */
-    u8 physical_block_exp; /* exponent for physical block per logical block */
-    u8 alignment_offset;   /* alignment offset in logical blocks */
-    u16 min_io_size;       /* min I/O size without performance penalty
+    uint8_t physical_block_exp; /* exponent for physical blk per logical blk */
+    uint8_t alignment_offset;   /* alignment offset in logical blocks */
+    uint16_t min_io_size;       /* min I/O size without performance penalty
                               in logical blocks */
-    u32 opt_io_size;       /* optimal sustained I/O size in logical blocks */
+    uint32_t opt_io_size;       /* optimal sustained I/O size in logical blks */
 
-    u8 wce; /* writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
-} __attribute__((packed)) VirtioBlkConfig;
+    uint8_t wce; /* writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
+} __attribute__((packed));
+typedef struct VirtioBlkConfig VirtioBlkConfig;
 
-bool virtio_guessed_disk_nature(void);
+enum guessed_disk_nature_type {
+    VIRTIO_GDN_NONE     = 0,
+    VIRTIO_GDN_DASD     = 1,
+    VIRTIO_GDN_CDROM    = 2,
+    VIRTIO_GDN_SCSI     = 3,
+};
+typedef enum guessed_disk_nature_type VirtioGDN;
+
+VirtioGDN virtio_guessed_disk_nature(void);
 void virtio_assume_scsi(void);
 void virtio_assume_eckd(void);
+void virtio_assume_iso9660(void);
 
 extern bool virtio_disk_is_scsi(void);
 extern bool virtio_disk_is_eckd(void);
@@ -198,15 +225,70 @@
 extern int virtio_read_many(ulong sector, void *load_addr, int sec_num);
 
 #define VIRTIO_SECTOR_SIZE 512
-
-static inline ulong virtio_eckd_sector_adjust(ulong sector)
-{
-     return sector * (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
-}
+#define VIRTIO_ISO_BLOCK_SIZE 2048
+#define VIRTIO_SCSI_BLOCK_SIZE 512
 
 static inline ulong virtio_sector_adjust(ulong sector)
 {
-    return virtio_disk_is_eckd() ? virtio_eckd_sector_adjust(sector) : sector;
+    return sector * (virtio_get_block_size() / VIRTIO_SECTOR_SIZE);
 }
 
+struct VirtioScsiConfig {
+    uint32_t num_queues;
+    uint32_t seg_max;
+    uint32_t max_sectors;
+    uint32_t cmd_per_lun;
+    uint32_t event_info_size;
+    uint32_t sense_size;
+    uint32_t cdb_size;
+    uint16_t max_channel;
+    uint16_t max_target;
+    uint32_t max_lun;
+} __attribute__((packed));
+typedef struct VirtioScsiConfig VirtioScsiConfig;
+
+struct ScsiDevice {
+    uint16_t channel;   /* Always 0 in QEMU     */
+    uint16_t target;    /* will be scanned over */
+    uint32_t lun;       /* will be reported     */
+};
+typedef struct ScsiDevice ScsiDevice;
+
+struct VDev {
+    int nr_vqs;
+    VRing *vrings;
+    int cmd_vr_idx;
+    void *ring_area;
+    long wait_reply_timeout;
+    VirtioGDN guessed_disk_nature;
+    SubChannelId schid;
+    SenseId senseid;
+    union {
+        VirtioBlkConfig blk;
+        VirtioScsiConfig scsi;
+    } config;
+    ScsiDevice *scsi_device;
+    bool is_cdrom;
+    int scsi_block_size;
+    int blk_factor;
+    uint64_t scsi_last_block;
+    uint32_t scsi_dev_cyls;
+    uint8_t scsi_dev_heads;
+    bool scsi_device_selected;
+    ScsiDevice selected_scsi_device;
+};
+typedef struct VDev VDev;
+
+VDev *virtio_get_device(void);
+VirtioDevType virtio_get_device_type(void);
+
+struct VirtioCmd {
+    void *data;
+    int size;
+    int flags;
+};
+typedef struct VirtioCmd VirtioCmd;
+
+int virtio_run(VDev *vdev, int vqid, VirtioCmd *cmd);
+
 #endif /* VIRTIO_H */
diff --git a/pc-bios/s390-zipl.rom b/pc-bios/s390-zipl.rom
deleted file mode 100644
index 3115128..0000000
--- a/pc-bios/s390-zipl.rom
+++ /dev/null
Binary files differ
diff --git a/pc-bios/slof.bin b/pc-bios/slof.bin
index 69b0a5d..f238493 100644
--- a/pc-bios/slof.bin
+++ b/pc-bios/slof.bin
Binary files differ
diff --git a/pc-bios/u-boot.e500 b/pc-bios/u-boot.e500
index bdb2a5e..6e547de 100755
--- a/pc-bios/u-boot.e500
+++ b/pc-bios/u-boot.e500
Binary files differ
diff --git a/pc-bios/vgabios-cirrus.bin b/pc-bios/vgabios-cirrus.bin
index 0c4d253..3f4bb30 100644
--- a/pc-bios/vgabios-cirrus.bin
+++ b/pc-bios/vgabios-cirrus.bin
Binary files differ
diff --git a/pc-bios/vgabios-qxl.bin b/pc-bios/vgabios-qxl.bin
index 4e08e13..38d31b6 100644
--- a/pc-bios/vgabios-qxl.bin
+++ b/pc-bios/vgabios-qxl.bin
Binary files differ
diff --git a/pc-bios/vgabios-stdvga.bin b/pc-bios/vgabios-stdvga.bin
index e5e5b14..e469c10 100644
--- a/pc-bios/vgabios-stdvga.bin
+++ b/pc-bios/vgabios-stdvga.bin
Binary files differ
diff --git a/pc-bios/vgabios-virtio.bin b/pc-bios/vgabios-virtio.bin
new file mode 100644
index 0000000..d42b028
--- /dev/null
+++ b/pc-bios/vgabios-virtio.bin
Binary files differ
diff --git a/pc-bios/vgabios-vmware.bin b/pc-bios/vgabios-vmware.bin
index cf2576d..26bc0b7 100644
--- a/pc-bios/vgabios-vmware.bin
+++ b/pc-bios/vgabios-vmware.bin
Binary files differ
diff --git a/pc-bios/vgabios.bin b/pc-bios/vgabios.bin
index bad187d..2d1a7c6 100644
--- a/pc-bios/vgabios.bin
+++ b/pc-bios/vgabios.bin
Binary files differ
diff --git a/po/Makefile b/po/Makefile
index b271f79..7bab09d 100644
--- a/po/Makefile
+++ b/po/Makefile
@@ -32,7 +32,7 @@
 build: $(OBJS)
 
 clean:
-	$(RM) $(OBJS)
+	rm -f $(OBJS)
 
 install: $(OBJS)
 	for obj in $(OBJS); do \
diff --git a/po/bg.po b/po/bg.po
new file mode 100644
index 0000000..5047861
--- /dev/null
+++ b/po/bg.po
@@ -0,0 +1,90 @@
+# Bulgarian translation of qemu po-file.
+# Copyright (C) 2016 Alexander Shopov <ash@kambanaria.org>
+# This file is distributed under the same license as the qemu package.
+# Alexander Shopov <ash@kambanaria.org>, 2016.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: QEMU 2.6.50\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2016-06-26 10:16+0300\n"
+"PO-Revision-Date: 2016-06-09 15:54+0300\n"
+"Last-Translator: Alexander Shopov <ash@kambanaria.org>\n"
+"Language-Team: Bulgarian <dict@ludost.net>\n"
+"Language: bg\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+
+#: ui/gtk.c:274
+msgid " - Press Ctrl+Alt+G to release grab"
+msgstr " — натиснете Ctrl+Alt+G, за да освободите фокуса"
+
+#: ui/gtk.c:278
+msgid " [Paused]"
+msgstr " [пауза]"
+
+#: ui/gtk.c:1906
+msgid "_Pause"
+msgstr "_Пауза"
+
+#: ui/gtk.c:1912
+msgid "_Reset"
+msgstr "_Рестартиране"
+
+#: ui/gtk.c:1915
+msgid "Power _Down"
+msgstr "_Изключване"
+
+#: ui/gtk.c:1921
+msgid "_Quit"
+msgstr "_Спиране на програмата"
+
+#: ui/gtk.c:2013
+msgid "_Fullscreen"
+msgstr "На _цял екран"
+
+#: ui/gtk.c:2016
+msgid "_Copy"
+msgstr "_Копиране"
+
+#: ui/gtk.c:2032
+msgid "Zoom _In"
+msgstr "_Увеличаване"
+
+#: ui/gtk.c:2039
+msgid "Zoom _Out"
+msgstr "_Намаляване"
+
+#: ui/gtk.c:2046
+msgid "Best _Fit"
+msgstr "По_местване"
+
+#: ui/gtk.c:2053
+msgid "Zoom To _Fit"
+msgstr "Напас_ване"
+
+#: ui/gtk.c:2059
+msgid "Grab On _Hover"
+msgstr "Прихващане при посо_чване"
+
+#: ui/gtk.c:2062
+msgid "_Grab Input"
+msgstr "Прихващане на _фокуса"
+
+#: ui/gtk.c:2091
+msgid "Show _Tabs"
+msgstr "Подпро_зорци"
+
+#: ui/gtk.c:2094
+msgid "Detach Tab"
+msgstr "Към самостоятелен подпрозорец"
+
+#: ui/gtk.c:2106
+msgid "_Machine"
+msgstr "_Машина"
+
+#: ui/gtk.c:2111
+msgid "_View"
+msgstr "_Изглед"
diff --git a/po/de_DE.po b/po/de_DE.po
index dcdcf22..97ae655 100644
--- a/po/de_DE.po
+++ b/po/de_DE.po
@@ -6,7 +6,7 @@
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-17 20:39+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2012-02-28 16:00+0100\n"
 "Last-Translator: Kevin Wolf <kwolf@redhat.com>\n"
 "Language-Team: Deutsch <de@li.org>\n"
@@ -16,70 +16,70 @@
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n!=1);\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Strg+Alt+G drücken, um Eingabegeräte freizugeben"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Angehalten]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Angehalten"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Reset"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Herunterfahren"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "_Beenden"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1940
 msgid "_Fullscreen"
 msgstr "_Vollbild"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "_Heranzoomen"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "_Wegzoomen"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "_Einpassen"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Auf _Fenstergröße skalieren"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Tastatur _automatisch einfangen"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Eingabegeräte einfangen"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Reiter anzeigen"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "Reiter abtrennen"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Maschine"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Ansicht"
diff --git a/po/fr_FR.po b/po/fr_FR.po
index bbb5ef8..e9fb040 100644
--- a/po/fr_FR.po
+++ b/po/fr_FR.po
@@ -6,7 +6,7 @@
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-28 23:14+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2014-07-28 23:25+0200\n"
 "Last-Translator: Aurelien Jarno <aurelien@aurel32.net>\n"
 "Language-Team: French <FR@li.org>\n"
@@ -17,70 +17,70 @@
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 "X-Generator: Lokalize 1.4\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr "- Appuyer sur Ctrl+Alt+G pour arrêter la capture"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [En pause]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Pause"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Réinitialiser"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Éteindre"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "_Quitter"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1940
 msgid "_Fullscreen"
 msgstr "Mode _plein écran"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "Zoom _avant"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "_Zoom arrière"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "Zoom _idéal"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Zoomer pour a_juster"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Capturer en _survolant"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Capturer les entrées"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Montrer les _onglets"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "_Détacher l'onglet"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Machine"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Vue"
diff --git a/po/hu.po b/po/hu.po
index 401ed21..1149adf 100644
--- a/po/hu.po
+++ b/po/hu.po
@@ -6,7 +6,7 @@
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2013-05-06 20:42+0200\n"
 "Last-Translator: Ákos Kovács <akoskovacs@gmx.com>\n"
 "Language-Team: Hungarian <hu@li.org>\n"
@@ -15,47 +15,73 @@
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Nyomj Ctrl+Alt+G-t a bemeneti eszközök elengedéséhez"
 
-#: ui/gtk.c:218
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Megállítva]"
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Megállítás"
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "Új_raindítás"
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Leállítás"
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1848
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
+#, fuzzy
+msgid "Zoom _In"
+msgstr "Ablakmérethez _igazítás"
+
+#: ui/gtk.c:1961
+#, fuzzy
+msgid "Zoom _Out"
+msgstr "Ablakmérethez _igazítás"
+
+#: ui/gtk.c:1968
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Ablakmérethez _igazítás"
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Automatikus _elfogás"
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Bemeneti eszközök megragadása"
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "_Fülek megjelenítése"
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:2016
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Gép"
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Nézet"
 
diff --git a/po/it.po b/po/it.po
index e46fb3a..a2c8e55 100644
--- a/po/it.po
+++ b/po/it.po
@@ -6,7 +6,7 @@
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-29 08:14+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2014-07-29 08:25+0200\n"
 "Last-Translator: Paolo Bonzini <pbonzini@redhat.com>\n"
 "Language-Team: Italian <it@li.org>\n"
@@ -16,66 +16,70 @@
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Premere Ctrl+Alt+G per rilasciare l'input"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Pausa]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Pausa"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Reset"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Spegni"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "_Esci"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "_Aumenta zoom"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "_Riduci zoom"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "A_nnulla zoom"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Adatta alla _finestra"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Cattura _automatica input"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "_Cattura input"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Mostra _tab"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "_Sposta in una nuova finestra"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Macchina virtuale"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Visualizza"
diff --git a/po/messages.po b/po/messages.po
index 26c76bc..e8f9ccd 100644
--- a/po/messages.po
+++ b/po/messages.po
@@ -5,9 +5,9 @@
 #, fuzzy
 msgid ""
 msgstr ""
-"Project-Id-Version: QEMU 1.5.50\n"
+"Project-Id-Version: QEMU 2.5.92\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -16,46 +16,70 @@
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr ""
 
-#: ui/gtk.c:218
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr ""
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr ""
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr ""
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr ""
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1848
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
+msgid "Zoom _In"
+msgstr ""
+
+#: ui/gtk.c:1961
+msgid "Zoom _Out"
+msgstr ""
+
+#: ui/gtk.c:1968
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr ""
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr ""
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr ""
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr ""
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:2016
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr ""
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr ""
diff --git a/po/tr.po b/po/tr.po
index d712ced..ec99127 100644
--- a/po/tr.po
+++ b/po/tr.po
@@ -6,7 +6,7 @@
 msgstr ""
 "Project-Id-Version: QEMU 1.4.50\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2013-07-05 22:36+0200\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2013-04-22 18:35+0300\n"
 "Last-Translator: Ozan Çağlayan <ozancag@gmail.com>\n"
 "Language-Team: Türkçe <>\n"
@@ -17,46 +17,72 @@
 "Plural-Forms: nplurals=1; plural=0;\n"
 "X-Generator: Gtranslator 2.91.6\n"
 
-#: ui/gtk.c:214
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - Yakalamayı durdurmak için Ctrl+Alt+G tuşlarına basın"
 
-#: ui/gtk.c:218
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [Duraklatıldı]"
 
-#: ui/gtk.c:1318
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "_Duraklat"
 
-#: ui/gtk.c:1324
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "_Sıfırla"
 
-#: ui/gtk.c:1327
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "_Kapat"
 
-#: ui/gtk.c:1381
+#: ui/gtk.c:1848
+msgid "_Quit"
+msgstr ""
+
+#: ui/gtk.c:1940
+msgid "_Fullscreen"
+msgstr ""
+
+#: ui/gtk.c:1954
+#, fuzzy
+msgid "Zoom _In"
+msgstr "Yakınlaş ve Sığ_dır"
+
+#: ui/gtk.c:1961
+#, fuzzy
+msgid "Zoom _Out"
+msgstr "Yakınlaş ve Sığ_dır"
+
+#: ui/gtk.c:1968
+msgid "Best _Fit"
+msgstr ""
+
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "Yakınlaş ve Sığ_dır"
 
-#: ui/gtk.c:1387
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "Ü_zerindeyken Yakala"
 
-#: ui/gtk.c:1390
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "Girdiyi _Yakala"
 
-#: ui/gtk.c:1416
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "Se_kmeleri Göster"
 
-#: ui/gtk.c:1430
+#: ui/gtk.c:2016
+msgid "Detach Tab"
+msgstr ""
+
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "_Makine"
 
-#: ui/gtk.c:1435
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "_Görüntüle"
diff --git a/po/zh_CN.po b/po/zh_CN.po
index 2b1d42e..b55b6f3 100644
--- a/po/zh_CN.po
+++ b/po/zh_CN.po
@@ -6,7 +6,7 @@
 msgstr ""
 "Project-Id-Version: QEMU 2.2\n"
 "Report-Msgid-Bugs-To: qemu-devel@nongnu.org\n"
-"POT-Creation-Date: 2014-07-31 10:03+0800\n"
+"POT-Creation-Date: 2016-04-18 14:50+0100\n"
 "PO-Revision-Date: 2014-07-31 10:00+0800\n"
 "Last-Translator: Fam Zheng <famz@redhat.com>\n"
 "Language-Team: Chinese <zh@li.org>\n"
@@ -17,70 +17,70 @@
 "Plural-Forms: nplurals=2; plural=n != 1;\n"
 "X-Generator: Lokalize 1.4\n"
 
-#: ui/gtk.c:321
+#: ui/gtk.c:273
 msgid " - Press Ctrl+Alt+G to release grab"
 msgstr " - 按下 Ctrl+Alt+G 取消捕获"
 
-#: ui/gtk.c:325
+#: ui/gtk.c:277
 msgid " [Paused]"
 msgstr " [已暂停]"
 
-#: ui/gtk.c:1601
+#: ui/gtk.c:1833
 msgid "_Pause"
 msgstr "暂停(_P)"
 
-#: ui/gtk.c:1607
+#: ui/gtk.c:1839
 msgid "_Reset"
 msgstr "重置(_R)"
 
-#: ui/gtk.c:1610
+#: ui/gtk.c:1842
 msgid "Power _Down"
 msgstr "关闭电源(_D)"
 
-#: ui/gtk.c:1616
+#: ui/gtk.c:1848
 msgid "_Quit"
 msgstr "退出(_Q)"
 
-#: ui/gtk.c:1692
+#: ui/gtk.c:1940
 msgid "_Fullscreen"
 msgstr "全屏(_F)"
 
-#: ui/gtk.c:1702
+#: ui/gtk.c:1954
 msgid "Zoom _In"
 msgstr "放大(_I)"
 
-#: ui/gtk.c:1709
+#: ui/gtk.c:1961
 msgid "Zoom _Out"
 msgstr "缩小(_O)"
 
-#: ui/gtk.c:1716
+#: ui/gtk.c:1968
 msgid "Best _Fit"
 msgstr "最合适大小(_F)"
 
-#: ui/gtk.c:1723
+#: ui/gtk.c:1975
 msgid "Zoom To _Fit"
 msgstr "缩放以适应大小(_F)"
 
-#: ui/gtk.c:1729
+#: ui/gtk.c:1981
 msgid "Grab On _Hover"
 msgstr "鼠标经过时捕获(_H)"
 
-#: ui/gtk.c:1732
+#: ui/gtk.c:1984
 msgid "_Grab Input"
 msgstr "捕获输入(_G)"
 
-#: ui/gtk.c:1761
+#: ui/gtk.c:2013
 msgid "Show _Tabs"
 msgstr "显示标签页(_T)"
 
-#: ui/gtk.c:1764
+#: ui/gtk.c:2016
 msgid "Detach Tab"
 msgstr "分离标签页"
 
-#: ui/gtk.c:1778
+#: ui/gtk.c:2028
 msgid "_Machine"
 msgstr "虚拟机(_M)"
 
-#: ui/gtk.c:1783
+#: ui/gtk.c:2033
 msgid "_View"
 msgstr "视图(_V)"
diff --git a/qapi-schema.json b/qapi-schema.json
index 9ffdcf8..5658723 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -5,6 +5,9 @@
 # QAPI common definitions
 { 'include': 'qapi/common.json' }
 
+# QAPI crypto definitions
+{ 'include': 'qapi/crypto.json' }
+
 # QAPI block definitions
 { 'include': 'qapi/block.json' }
 
@@ -14,8 +17,11 @@
 # Tracing commands
 { 'include': 'qapi/trace.json' }
 
+# QAPI introspection
+{ 'include': 'qapi/introspect.json' }
+
 ##
-# LostTickPolicy:
+# @LostTickPolicy:
 #
 # Policy for handling lost ticks in timer devices.
 #
@@ -71,7 +77,7 @@
 #
 # Since 0.14.0
 ##
-{ 'type': 'NameInfo', 'data': {'*name': 'str'} }
+{ 'struct': 'NameInfo', 'data': {'*name': 'str'} }
 
 ##
 # @query-name:
@@ -95,7 +101,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'KvmInfo', 'data': {'enabled': 'bool', 'present': 'bool'} }
+{ 'struct': 'KvmInfo', 'data': {'enabled': 'bool', 'present': 'bool'} }
 
 ##
 # @query-kvm:
@@ -170,7 +176,7 @@
 #
 # Notes: @singlestep is enabled through the GDB stub
 ##
-{ 'type': 'StatusInfo',
+{ 'struct': 'StatusInfo',
   'data': {'running': 'bool', 'singlestep': 'bool', 'status': 'RunState'} }
 
 ##
@@ -195,7 +201,7 @@
 #
 # Notes: If no UUID was specified for the guest, a null UUID is returned.
 ##
-{ 'type': 'UuidInfo', 'data': {'UUID': 'str'} }
+{ 'struct': 'UuidInfo', 'data': {'UUID': 'str'} }
 
 ##
 # @query-uuid:
@@ -226,7 +232,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'ChardevInfo', 'data': {'label': 'str',
+{ 'struct': 'ChardevInfo', 'data': {'label': 'str',
                                   'filename': 'str',
                                   'frontend-open': 'bool'} }
 
@@ -250,7 +256,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'ChardevBackendInfo', 'data': {'name': 'str'} }
+{ 'struct': 'ChardevBackendInfo', 'data': {'name': 'str'} }
 
 ##
 # @query-chardev-backends:
@@ -289,8 +295,6 @@
 # @format: #optional data encoding (default 'utf8').
 #          - base64: data must be base64 encoded text.  Its binary
 #            decoding gets written.
-#            Bug: invalid base64 is currently not rejected.
-#            Whitespace *is* invalid.
 #          - utf8: data's UTF-8 encoding is written
 #          - data itself is always Unicode regardless of format, like
 #            any other string.
@@ -339,7 +343,7 @@
 #
 # Since: 1.2.0
 ##
-{ 'type': 'EventInfo', 'data': {'name': 'str'} }
+{ 'struct': 'EventInfo', 'data': {'name': 'str'} }
 
 ##
 # @query-events:
@@ -378,13 +382,17 @@
 #
 # @dirty-sync-count: number of times that dirty ram was synchronized (since 2.1)
 #
+# @postcopy-requests: The number of page requests received from the destination
+#        (since 2.7)
+#
 # Since: 0.14.0
 ##
-{ 'type': 'MigrationStats',
+{ 'struct': 'MigrationStats',
   'data': {'transferred': 'int', 'remaining': 'int', 'total': 'int' ,
            'duplicate': 'int', 'skipped': 'int', 'normal': 'int',
            'normal-bytes': 'int', 'dirty-pages-rate' : 'int',
-           'mbps' : 'number', 'dirty-sync-count' : 'int' } }
+           'mbps' : 'number', 'dirty-sync-count' : 'int',
+           'postcopy-requests' : 'int' } }
 
 ##
 # @XBZRLECacheStats
@@ -405,24 +413,50 @@
 #
 # Since: 1.2
 ##
-{ 'type': 'XBZRLECacheStats',
+{ 'struct': 'XBZRLECacheStats',
   'data': {'cache-size': 'int', 'bytes': 'int', 'pages': 'int',
            'cache-miss': 'int', 'cache-miss-rate': 'number',
            'overflow': 'int' } }
 
+# @MigrationStatus:
+#
+# An enumeration of migration status.
+#
+# @none: no migration has ever happened.
+#
+# @setup: migration process has been initiated.
+#
+# @cancelling: in the process of cancelling migration.
+#
+# @cancelled: cancelling migration is finished.
+#
+# @active: in the process of doing migration.
+#
+# @postcopy-active: like active, but now in postcopy mode. (since 2.5)
+#
+# @completed: migration is finished.
+#
+# @failed: some error occurred during migration process.
+#
+# Since: 2.3
+#
+##
+{ 'enum': 'MigrationStatus',
+  'data': [ 'none', 'setup', 'cancelling', 'cancelled',
+            'active', 'postcopy-active', 'completed', 'failed' ] }
+
 ##
 # @MigrationInfo
 #
 # Information about current migration process.
 #
-# @status: #optional string describing the current migration status.
-#          As of 0.14.0 this can be 'setup', 'active', 'completed', 'failed' or
-#          'cancelled'. If this field is not returned, no migration process
+# @status: #optional @MigrationStatus describing the current migration status.
+#          If this field is not returned, no migration process
 #          has been initiated
 #
 # @ram: #optional @MigrationStats containing detailed migration
 #       status, only returned if status is 'active' or
-#       'completed'. 'comppleted' (since 1.2)
+#       'completed'(since 1.2)
 #
 # @disk: #optional @MigrationStats containing detailed disk migration
 #        status, only returned if status is 'active' and it is a block
@@ -450,16 +484,26 @@
 #        may be expensive, but do not actually occur during the iterative
 #        migration rounds themselves. (since 1.6)
 #
+# @cpu-throttle-percentage: #optional percentage of time guest cpus are being
+#        throttled during auto-converge. This is only present when auto-converge
+#        has started throttling guest cpus. (Since 2.7)
+#
+# @error-desc: #optional the human readable error description string, when
+#              @status is 'failed'. Clients should not attempt to parse the
+#              error strings. (Since 2.7)
+#
 # Since: 0.14.0
 ##
-{ 'type': 'MigrationInfo',
-  'data': {'*status': 'str', '*ram': 'MigrationStats',
+{ 'struct': 'MigrationInfo',
+  'data': {'*status': 'MigrationStatus', '*ram': 'MigrationStats',
            '*disk': 'MigrationStats',
            '*xbzrle-cache': 'XBZRLECacheStats',
            '*total-time': 'int',
            '*expected-downtime': 'int',
            '*downtime': 'int',
-           '*setup-time': 'int'} }
+           '*setup-time': 'int',
+           '*cpu-throttle-percentage': 'int',
+           '*error-desc': 'str'} }
 
 ##
 # @query-migrate
@@ -491,13 +535,29 @@
 #          to enable the capability on the source VM. The feature is disabled by
 #          default. (since 1.6)
 #
+# @compress: Use multiple compression threads to accelerate live migration.
+#          This feature can help to reduce the migration traffic, by sending
+#          compressed pages. Please note that if compress and xbzrle are both
+#          on, compress only takes effect in the ram bulk stage, after that,
+#          it will be disabled and only xbzrle takes effect, this can help to
+#          minimize migration traffic. The feature is disabled by default.
+#          (since 2.4 )
+#
+# @events: generate events for each migration state change
+#          (since 2.4 )
+#
 # @auto-converge: If enabled, QEMU will automatically throttle down the guest
 #          to speed up convergence of RAM migration. (since 1.6)
 #
+# @postcopy-ram: Start executing on the migration target before all of RAM has
+#          been migrated, pulling the remaining pages along as needed. NOTE: If
+#          the migration fails during postcopy the VM will fail.  (since 2.6)
+#
 # Since: 1.2
 ##
 { 'enum': 'MigrationCapability',
-  'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks'] }
+  'data': ['xbzrle', 'rdma-pin-all', 'auto-converge', 'zero-blocks',
+           'compress', 'events', 'postcopy-ram'] }
 
 ##
 # @MigrationCapabilityStatus
@@ -510,7 +570,7 @@
 #
 # Since: 1.2
 ##
-{ 'type': 'MigrationCapabilityStatus',
+{ 'struct': 'MigrationCapabilityStatus',
   'data': { 'capability' : 'MigrationCapability', 'state' : 'bool' } }
 
 ##
@@ -536,6 +596,182 @@
 ##
 { 'command': 'query-migrate-capabilities', 'returns':   ['MigrationCapabilityStatus']}
 
+# @MigrationParameter
+#
+# Migration parameters enumeration
+#
+# @compress-level: Set the compression level to be used in live migration,
+#          the compression level is an integer between 0 and 9, where 0 means
+#          no compression, 1 means the best compression speed, and 9 means best
+#          compression ratio which will consume more CPU.
+#
+# @compress-threads: Set compression thread count to be used in live migration,
+#          the compression thread count is an integer between 1 and 255.
+#
+# @decompress-threads: Set decompression thread count to be used in live
+#          migration, the decompression thread count is an integer between 1
+#          and 255. Usually, decompression is at least 4 times as fast as
+#          compression, so set the decompress-threads to the number about 1/4
+#          of compress-threads is adequate.
+#
+# @cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                        when migration auto-converge is activated. The
+#                        default value is 20. (Since 2.7)
+#
+# @cpu-throttle-increment: throttle percentage increase each time
+#                          auto-converge detects that migration is not making
+#                          progress. The default value is 10. (Since 2.7)
+#
+# @tls-creds: ID of the 'tls-creds' object that provides credentials for
+#             establishing a TLS connection over the migration data channel.
+#             On the outgoing side of the migration, the credentials must
+#             be for a 'client' endpoint, while for the incoming side the
+#             credentials must be for a 'server' endpoint. Setting this
+#             will enable TLS for all migrations. The default is unset,
+#             resulting in unsecured migration at the QEMU level. (Since 2.7)
+#
+# @tls-hostname: hostname of the target host for the migration. This is
+#                required when using x509 based TLS credentials and the
+#                migration URI does not already include a hostname. For
+#                example if using fd: or exec: based migration, the
+#                hostname must be provided so that the server's x509
+#                certificate identity can be validated. (Since 2.7)
+#
+# Since: 2.4
+##
+{ 'enum': 'MigrationParameter',
+  'data': ['compress-level', 'compress-threads', 'decompress-threads',
+           'cpu-throttle-initial', 'cpu-throttle-increment',
+           'tls-creds', 'tls-hostname'] }
+
+#
+# @migrate-set-parameters
+#
+# Set the following migration parameters
+#
+# @compress-level: compression level
+#
+# @compress-threads: compression thread count
+#
+# @decompress-threads: decompression thread count
+#
+# @cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                        when migration auto-converge is activated. The
+#                        default value is 20. (Since 2.7)
+#
+# @cpu-throttle-increment: throttle percentage increase each time
+#                          auto-converge detects that migration is not making
+#                          progress. The default value is 10. (Since 2.7)
+#
+# @tls-creds: ID of the 'tls-creds' object that provides credentials for
+#             establishing a TLS connection over the migration data channel.
+#             On the outgoing side of the migration, the credentials must
+#             be for a 'client' endpoint, while for the incoming side the
+#             credentials must be for a 'server' endpoint. Setting this
+#             will enable TLS for all migrations. The default is unset,
+#             resulting in unsecured migration at the QEMU level. (Since 2.7)
+#
+# @tls-hostname: hostname of the target host for the migration. This is
+#                required when using x509 based TLS credentials and the
+#                migration URI does not already include a hostname. For
+#                example if using fd: or exec: based migration, the
+#                hostname must be provided so that the server's x509
+#                certificate identity can be validated. (Since 2.7)
+#
+# Since: 2.4
+##
+{ 'command': 'migrate-set-parameters',
+  'data': { '*compress-level': 'int',
+            '*compress-threads': 'int',
+            '*decompress-threads': 'int',
+            '*cpu-throttle-initial': 'int',
+            '*cpu-throttle-increment': 'int',
+            '*tls-creds': 'str',
+            '*tls-hostname': 'str'} }
+
+#
+# @MigrationParameters
+#
+# @compress-level: compression level
+#
+# @compress-threads: compression thread count
+#
+# @decompress-threads: decompression thread count
+#
+# @cpu-throttle-initial: Initial percentage of time guest cpus are throttled
+#                        when migration auto-converge is activated. The
+#                        default value is 20. (Since 2.7)
+#
+# @cpu-throttle-increment: throttle percentage increase each time
+#                          auto-converge detects that migration is not making
+#                          progress. The default value is 10. (Since 2.7)
+#
+# @tls-creds: ID of the 'tls-creds' object that provides credentials for
+#             establishing a TLS connection over the migration data channel.
+#             On the outgoing side of the migration, the credentials must
+#             be for a 'client' endpoint, while for the incoming side the
+#             credentials must be for a 'server' endpoint. Setting this
+#             will enable TLS for all migrations. The default is unset,
+#             resulting in unsecured migration at the QEMU level. (Since 2.7)
+#
+# @tls-hostname: hostname of the target host for the migration. This is
+#                required when using x509 based TLS credentials and the
+#                migration URI does not already include a hostname. For
+#                example if using fd: or exec: based migration, the
+#                hostname must be provided so that the server's x509
+#                certificate identity can be validated. (Since 2.7)
+#
+# Since: 2.4
+##
+{ 'struct': 'MigrationParameters',
+  'data': { 'compress-level': 'int',
+            'compress-threads': 'int',
+            'decompress-threads': 'int',
+            'cpu-throttle-initial': 'int',
+            'cpu-throttle-increment': 'int',
+            'tls-creds': 'str',
+            'tls-hostname': 'str'} }
+##
+# @query-migrate-parameters
+#
+# Returns information about the current migration parameters
+#
+# Returns: @MigrationParameters
+#
+# Since: 2.4
+##
+{ 'command': 'query-migrate-parameters',
+  'returns': 'MigrationParameters' }
+
+##
+# @client_migrate_info
+#
+# Set migration information for remote display.  This makes the server
+# ask the client to automatically reconnect using the new parameters
+# once migration finished successfully.  Only implemented for SPICE.
+#
+# @protocol:     must be "spice"
+# @hostname:     migration target hostname
+# @port:         #optional spice tcp port for plaintext channels
+# @tls-port:     #optional spice tcp port for tls-secured channels
+# @cert-subject: #optional server certificate subject
+#
+# Since: 0.14.0
+##
+{ 'command': 'client_migrate_info',
+  'data': { 'protocol': 'str', 'hostname': 'str', '*port': 'int',
+            '*tls-port': 'int', '*cert-subject': 'str' } }
+
+##
+# @migrate-start-postcopy
+#
+# Followup to a migration command to switch the migration to postcopy mode.
+# The postcopy-ram capability must be set before the original migration
+# command.
+#
+# Since: 2.5
+{ 'command': 'migrate-start-postcopy' }
+
 ##
 # @MouseInfo:
 #
@@ -551,7 +787,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'MouseInfo',
+{ 'struct': 'MouseInfo',
   'data': {'name': 'str', 'index': 'int', 'current': 'bool',
            'absolute': 'bool'} }
 
@@ -567,39 +803,117 @@
 { 'command': 'query-mice', 'returns': ['MouseInfo'] }
 
 ##
+# @CpuInfoArch:
+#
+# An enumeration of cpu types that enable additional information during
+# @query-cpus.
+#
+# Since: 2.6
+##
+{ 'enum': 'CpuInfoArch',
+  'data': ['x86', 'sparc', 'ppc', 'mips', 'tricore', 'other' ] }
+
+##
 # @CpuInfo:
 #
 # Information about a virtual CPU
 #
 # @CPU: the index of the virtual CPU
 #
-# @current: this only exists for backwards compatible and should be ignored
+# @current: this only exists for backwards compatibility and should be ignored
 #
 # @halted: true if the virtual CPU is in the halt state.  Halt usually refers
 #          to a processor specific low power mode.
 #
-# @pc: #optional If the target is i386 or x86_64, this is the 64-bit instruction
-#                pointer.
-#                If the target is Sparc, this is the PC component of the
-#                instruction pointer.
-#
-# @nip: #optional If the target is PPC, the instruction pointer
-#
-# @npc: #optional If the target is Sparc, the NPC component of the instruction
-#                 pointer
-#
-# @PC: #optional If the target is MIPS, the instruction pointer
+# @qom_path: path to the CPU object in the QOM tree (since 2.4)
 #
 # @thread_id: ID of the underlying host thread
 #
+# @arch: architecture of the cpu, which determines which additional fields
+#        will be listed (since 2.6)
+#
 # Since: 0.14.0
 #
 # Notes: @halted is a transient state that changes frequently.  By the time the
 #        data is sent to the client, the guest may no longer be halted.
 ##
-{ 'type': 'CpuInfo',
-  'data': {'CPU': 'int', 'current': 'bool', 'halted': 'bool', '*pc': 'int',
-           '*nip': 'int', '*npc': 'int', '*PC': 'int', 'thread_id': 'int'} }
+{ 'union': 'CpuInfo',
+  'base': {'CPU': 'int', 'current': 'bool', 'halted': 'bool',
+           'qom_path': 'str', 'thread_id': 'int', 'arch': 'CpuInfoArch' },
+  'discriminator': 'arch',
+  'data': { 'x86': 'CpuInfoX86',
+            'sparc': 'CpuInfoSPARC',
+            'ppc': 'CpuInfoPPC',
+            'mips': 'CpuInfoMIPS',
+            'tricore': 'CpuInfoTricore',
+            'other': 'CpuInfoOther' } }
+
+##
+# @CpuInfoX86:
+#
+# Additional information about a virtual i386 or x86_64 CPU
+#
+# @pc: the 64-bit instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoX86', 'data': { 'pc': 'int' } }
+
+##
+# @CpuInfoSPARC:
+#
+# Additional information about a virtual SPARC CPU
+#
+# @pc: the PC component of the instruction pointer
+#
+# @npc: the NPC component of the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoSPARC', 'data': { 'pc': 'int', 'npc': 'int' } }
+
+##
+# @CpuInfoPPC:
+#
+# Additional information about a virtual PPC CPU
+#
+# @nip: the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoPPC', 'data': { 'nip': 'int' } }
+
+##
+# @CpuInfoMIPS:
+#
+# Additional information about a virtual MIPS CPU
+#
+# @PC: the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoMIPS', 'data': { 'PC': 'int' } }
+
+##
+# @CpuInfoTricore:
+#
+# Additional information about a virtual Tricore CPU
+#
+# @PC: the instruction pointer
+#
+# Since 2.6
+##
+{ 'struct': 'CpuInfoTricore', 'data': { 'PC': 'int' } }
+
+##
+# @CpuInfoOther:
+#
+# No additional information is available about the virtual CPU
+#
+# Since 2.6
+#
+##
+{ 'struct': 'CpuInfoOther', 'data': { } }
 
 ##
 # @query-cpus:
@@ -623,7 +937,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'IOThreadInfo',
+{ 'struct': 'IOThreadInfo',
   'data': {'id': 'str', 'thread-id': 'int'} }
 
 ##
@@ -672,12 +986,15 @@
 #
 # @family: address family
 #
+# @websocket: true in case the socket is a websocket (since 2.3).
+#
 # Since: 2.1
 ##
-{ 'type': 'VncBasicInfo',
+{ 'struct': 'VncBasicInfo',
   'data': { 'host': 'str',
             'service': 'str',
-            'family': 'NetworkAddressFamily' } }
+            'family': 'NetworkAddressFamily',
+            'websocket': 'bool' } }
 
 ##
 # @VncServerInfo
@@ -688,7 +1005,7 @@
 #
 # Since: 2.1
 ##
-{ 'type': 'VncServerInfo',
+{ 'struct': 'VncServerInfo',
   'base': 'VncBasicInfo',
   'data': { '*auth': 'str' } }
 
@@ -705,7 +1022,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'VncClientInfo',
+{ 'struct': 'VncClientInfo',
   'base': 'VncBasicInfo',
   'data': { '*x509_dname': 'str', '*sasl_username': 'str' } }
 
@@ -745,12 +1062,69 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'VncInfo',
+{ 'struct': 'VncInfo',
   'data': {'enabled': 'bool', '*host': 'str',
            '*family': 'NetworkAddressFamily',
            '*service': 'str', '*auth': 'str', '*clients': ['VncClientInfo']} }
 
 ##
+# @VncPriAuth:
+#
+# vnc primary authentication method.
+#
+# Since: 2.3
+##
+{ 'enum': 'VncPrimaryAuth',
+  'data': [ 'none', 'vnc', 'ra2', 'ra2ne', 'tight', 'ultra',
+            'tls', 'vencrypt', 'sasl' ] }
+
+##
+# @VncVencryptSubAuth:
+#
+# vnc sub authentication method with vencrypt.
+#
+# Since: 2.3
+##
+{ 'enum': 'VncVencryptSubAuth',
+  'data': [ 'plain',
+            'tls-none',  'x509-none',
+            'tls-vnc',   'x509-vnc',
+            'tls-plain', 'x509-plain',
+            'tls-sasl',  'x509-sasl' ] }
+
+##
+# @VncInfo2:
+#
+# Information about a vnc server
+#
+# @id: vnc server name.
+#
+# @server: A list of @VncBasincInfo describing all listening sockets.
+#          The list can be empty (in case the vnc server is disabled).
+#          It also may have multiple entries: normal + websocket,
+#          possibly also ipv4 + ipv6 in the future.
+#
+# @clients: A list of @VncClientInfo of all currently connected clients.
+#           The list can be empty, for obvious reasons.
+#
+# @auth: The current authentication type used by the server
+#
+# @vencrypt: #optional The vencrypt sub authentication type used by the server,
+#            only specified in case auth == vencrypt.
+#
+# @display: #optional The display device the vnc server is linked to.
+#
+# Since: 2.3
+##
+{ 'struct': 'VncInfo2',
+  'data': { 'id'        : 'str',
+            'server'    : ['VncBasicInfo'],
+            'clients'   : ['VncClientInfo'],
+            'auth'      : 'VncPrimaryAuth',
+            '*vencrypt' : 'VncVencryptSubAuth',
+            '*display'  : 'str' } }
+
+##
 # @query-vnc:
 #
 # Returns information about the current VNC server
@@ -762,6 +1136,17 @@
 { 'command': 'query-vnc', 'returns': 'VncInfo' }
 
 ##
+# @query-vnc-servers:
+#
+# Returns a list of vnc servers.  The list can be empty.
+#
+# Returns: a list of @VncInfo2
+#
+# Since: 2.3
+##
+{ 'command': 'query-vnc-servers', 'returns': ['VncInfo2'] }
+
+##
 # @SpiceBasicInfo
 #
 # The basic information for SPICE network connection
@@ -774,7 +1159,7 @@
 #
 # Since: 2.1
 ##
-{ 'type': 'SpiceBasicInfo',
+{ 'struct': 'SpiceBasicInfo',
   'data': { 'host': 'str',
             'port': 'str',
             'family': 'NetworkAddressFamily' } }
@@ -788,7 +1173,7 @@
 #
 # Since: 2.1
 ##
-{ 'type': 'SpiceServerInfo',
+{ 'struct': 'SpiceServerInfo',
   'base': 'SpiceBasicInfo',
   'data': { '*auth': 'str' } }
 
@@ -800,9 +1185,9 @@
 # @connection-id: SPICE connection id number.  All channels with the same id
 #                 belong to the same SPICE session.
 #
-# @connection-type: SPICE channel type number.  "1" is the main control
-#                   channel, filter for this one if you want to track spice
-#                   sessions only
+# @channel-type: SPICE channel type number.  "1" is the main control
+#                channel, filter for this one if you want to track spice
+#                sessions only
 #
 # @channel-id: SPICE channel ID number.  Usually "0", might be different when
 #              multiple channels of the same type exist, such as multiple
@@ -812,7 +1197,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'SpiceChannel',
+{ 'struct': 'SpiceChannel',
   'base': 'SpiceBasicInfo',
   'data': {'connection-id': 'int', 'channel-type': 'int', 'channel-id': 'int',
            'tls': 'bool'} }
@@ -870,7 +1255,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'SpiceInfo',
+{ 'struct': 'SpiceInfo',
   'data': {'enabled': 'bool', 'migrated': 'bool', '*host': 'str', '*port': 'int',
            '*tls-port': 'int', '*auth': 'str', '*compiled-version': 'str',
            'mouse-mode': 'SpiceQueryMouseMode', '*channels': ['SpiceChannel']} }
@@ -896,7 +1281,7 @@
 # Since: 0.14.0
 #
 ##
-{ 'type': 'BalloonInfo', 'data': {'actual': 'int' } }
+{ 'struct': 'BalloonInfo', 'data': {'actual': 'int' } }
 
 ##
 # @query-balloon:
@@ -923,7 +1308,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'PciMemoryRange', 'data': {'base': 'int', 'limit': 'int'} }
+{ 'struct': 'PciMemoryRange', 'data': {'base': 'int', 'limit': 'int'} }
 
 ##
 # @PciMemoryRegion
@@ -941,41 +1326,80 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'PciMemoryRegion',
+{ 'struct': 'PciMemoryRegion',
   'data': {'bar': 'int', 'type': 'str', 'address': 'int', 'size': 'int',
            '*prefetch': 'bool', '*mem_type_64': 'bool' } }
 
 ##
+# @PciBusInfo:
+#
+# Information about a bus of a PCI Bridge device
+#
+# @number: primary bus interface number.  This should be the number of the
+#          bus the device resides on.
+#
+# @secondary: secondary bus interface number.  This is the number of the
+#             main bus for the bridge
+#
+# @subordinate: This is the highest number bus that resides below the
+#               bridge.
+#
+# @io_range: The PIO range for all devices on this bridge
+#
+# @memory_range: The MMIO range for all devices on this bridge
+#
+# @prefetchable_range: The range of prefetchable MMIO for all devices on
+#                      this bridge
+#
+# Since: 2.4
+##
+{ 'struct': 'PciBusInfo',
+  'data': {'number': 'int', 'secondary': 'int', 'subordinate': 'int',
+           'io_range': 'PciMemoryRange',
+           'memory_range': 'PciMemoryRange',
+           'prefetchable_range': 'PciMemoryRange' } }
+
+##
 # @PciBridgeInfo:
 #
 # Information about a PCI Bridge device
 #
-# @bus.number: primary bus interface number.  This should be the number of the
-#              bus the device resides on.
-#
-# @bus.secondary: secondary bus interface number.  This is the number of the
-#                 main bus for the bridge
-#
-# @bus.subordinate: This is the highest number bus that resides below the
-#                   bridge.
-#
-# @bus.io_range: The PIO range for all devices on this bridge
-#
-# @bus.memory_range: The MMIO range for all devices on this bridge
-#
-# @bus.prefetchable_range: The range of prefetchable MMIO for all devices on
-#                          this bridge
+# @bus: information about the bus the device resides on
 #
 # @devices: a list of @PciDeviceInfo for each device on this bridge
 #
 # Since: 0.14.0
 ##
-{ 'type': 'PciBridgeInfo',
-  'data': {'bus': { 'number': 'int', 'secondary': 'int', 'subordinate': 'int',
-                    'io_range': 'PciMemoryRange',
-                    'memory_range': 'PciMemoryRange',
-                    'prefetchable_range': 'PciMemoryRange' },
-           '*devices': ['PciDeviceInfo']} }
+{ 'struct': 'PciBridgeInfo',
+  'data': {'bus': 'PciBusInfo', '*devices': ['PciDeviceInfo']} }
+
+##
+# @PciDeviceClass:
+#
+# Information about the Class of a PCI device
+#
+# @desc: #optional a string description of the device's class
+#
+# @class: the class code of the device
+#
+# Since: 2.4
+##
+{ 'struct': 'PciDeviceClass',
+  'data': {'*desc': 'str', 'class': 'int'} }
+
+##
+# @PciDeviceId:
+#
+# Information about the Id of a PCI device
+#
+# @device: the PCI device id
+#
+# @vendor: the PCI vendor id
+#
+# Since: 2.4
+##
+{ 'struct': 'PciDeviceId',
+  'data': {'device': 'int', 'vendor': 'int'} }
 
 ##
 # @PciDeviceInfo:
@@ -988,13 +1412,9 @@
 #
 # @function: the function of the slot used by the device
 #
-# @class_info.desc: #optional a string description of the device's class
+# @class_info: the class of the device
 #
-# @class_info.class: the class code of the device
-#
-# @id.device: the PCI device id
-#
-# @id.vendor: the PCI vendor id
+# @id: the PCI device id
 #
 # @irq: #optional if an IRQ is assigned to the device, the IRQ number
 #
@@ -1009,10 +1429,9 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'PciDeviceInfo',
+{ 'struct': 'PciDeviceInfo',
   'data': {'bus': 'int', 'slot': 'int', 'function': 'int',
-           'class_info': {'*desc': 'str', 'class': 'int'},
-           'id': {'device': 'int', 'vendor': 'int'},
+           'class_info': 'PciDeviceClass', 'id': 'PciDeviceId',
            '*irq': 'int', 'qdev_id': 'str', '*pci_bridge': 'PciBridgeInfo',
            'regions': ['PciMemoryRegion']} }
 
@@ -1027,7 +1446,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'PciInfo', 'data': {'bus': 'int', 'devices': ['PciDeviceInfo']} }
+{ 'struct': 'PciInfo', 'data': {'bus': 'int', 'devices': ['PciDeviceInfo']} }
 
 ##
 # @query-pci:
@@ -1246,32 +1665,87 @@
 #
 # Since: 1.6
 ###
-{ 'type': 'Abort',
+{ 'struct': 'Abort',
   'data': { } }
 
 ##
+# @ActionCompletionMode
+#
+# An enumeration of Transactional completion modes.
+#
+# @individual: Do not attempt to cancel any other Actions if any Actions fail
+#              after the Transaction request succeeds. All Actions that
+#              can complete successfully will do so without waiting on others.
+#              This is the default.
+#
+# @grouped: If any Action fails after the Transaction succeeds, cancel all
+#           Actions. Actions do not complete until all Actions are ready to
+#           complete. May be rejected by Actions that do not support this
+#           completion mode.
+#
+# Since: 2.5
+##
+{ 'enum': 'ActionCompletionMode',
+  'data': [ 'individual', 'grouped' ] }
+
+##
 # @TransactionAction
 #
 # A discriminated record of operations that can be performed with
 # @transaction.
+#
+# Since 1.1
+#
+# drive-backup since 1.6
+# abort since 1.6
+# blockdev-snapshot-internal-sync since 1.7
+# blockdev-backup since 2.3
+# blockdev-snapshot since 2.5
+# block-dirty-bitmap-add since 2.5
+# block-dirty-bitmap-clear since 2.5
 ##
 { 'union': 'TransactionAction',
   'data': {
-       'blockdev-snapshot-sync': 'BlockdevSnapshot',
+       'blockdev-snapshot': 'BlockdevSnapshot',
+       'blockdev-snapshot-sync': 'BlockdevSnapshotSync',
        'drive-backup': 'DriveBackup',
+       'blockdev-backup': 'BlockdevBackup',
        'abort': 'Abort',
-       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal'
+       'blockdev-snapshot-internal-sync': 'BlockdevSnapshotInternal',
+       'block-dirty-bitmap-add': 'BlockDirtyBitmapAdd',
+       'block-dirty-bitmap-clear': 'BlockDirtyBitmap'
    } }
 
 ##
+# @TransactionProperties
+#
+# Optional arguments to modify the behavior of a Transaction.
+#
+# @completion-mode: #optional Controls how jobs launched asynchronously by
+#                   Actions will complete or fail as a group.
+#                   See @ActionCompletionMode for details.
+#
+# Since: 2.5
+##
+{ 'struct': 'TransactionProperties',
+  'data': {
+       '*completion-mode': 'ActionCompletionMode'
+  }
+}
+
+##
 # @transaction
 #
 # Executes a number of transactionable QMP commands atomically. If any
 # operation fails, then the entire set of actions will be abandoned and the
 # appropriate error returned.
 #
-#  List of:
-#  @TransactionAction: information needed for the respective operation
+# @actions: List of @TransactionAction;
+#           information needed for the respective operations.
+#
+# @properties: #optional structure of additional options to control the
+#              execution of the transaction. See @TransactionProperties
+#              for additional detail.
 #
 # Returns: nothing on success
 #          Errors depend on the operations of the transaction
@@ -1283,7 +1757,10 @@
 # Since 1.1
 ##
 { 'command': 'transaction',
-  'data': { 'actions': [ 'TransactionAction' ] } }
+  'data': { 'actions': [ 'TransactionAction' ],
+            '*properties': 'TransactionProperties'
+          }
+}
 
 ##
 # @human-monitor-command:
@@ -1392,18 +1869,15 @@
 #        1) A primitive type such as 'u8', 'u16', 'bool', 'str', or 'double'.
 #           These types are mapped to the appropriate JSON type.
 #
-#        2) A legacy type in the form 'legacy<subtype>' where subtype is the
-#           legacy qdev typename.  These types are always treated as strings.
-#
-#        3) A child type in the form 'child<subtype>' where subtype is a qdev
+#        2) A child type in the form 'child<subtype>' where subtype is a qdev
 #           device type name.  Child properties create the composition tree.
 #
-#        4) A link type in the form 'link<subtype>' where subtype is a qdev
+#        3) A link type in the form 'link<subtype>' where subtype is a qdev
 #           device type name.  Link properties form the device model graph.
 #
 # Since: 1.2
 ##
-{ 'type': 'ObjectPropertyInfo',
+{ 'struct': 'ObjectPropertyInfo',
   'data': { 'name': 'str', 'type': 'str' } }
 
 ##
@@ -1449,17 +1923,16 @@
 #
 # @property: The property name to read
 #
-# Returns: The property value.  The type depends on the property type.  legacy<>
-#          properties are returned as #str.  child<> and link<> properties are
-#          returns as #str pathnames.  All integer property types (u8, u16, etc)
-#          are returned as #int.
+# Returns: The property value.  The type depends on the property
+#          type. child<> and link<> properties are returned as #str
+#          pathnames.  All integer property types (u8, u16, etc) are
+#          returned as #int.
 #
 # Since: 1.2
 ##
 { 'command': 'qom-get',
   'data': { 'path': 'str', 'property': 'str' },
-  'returns': 'visitor',
-  'gen': 'no' }
+  'returns': 'any' }
 
 ##
 # @qom-set:
@@ -1476,8 +1949,7 @@
 # Since: 1.2
 ##
 { 'command': 'qom-set',
-  'data': { 'path': 'str', 'property': 'str', 'value': 'visitor' },
-  'gen': 'no' }
+  'data': { 'path': 'str', 'property': 'str', 'value': 'any' } }
 
 ##
 # @set_password:
@@ -1533,7 +2005,7 @@
 #
 # Change the VNC server password.
 #
-# @target:  the new password to use with VNC authentication
+# @password:  the new password to use with VNC authentication
 #
 # Since: 1.1
 #
@@ -1569,8 +2041,10 @@
 #          device's password.  The behavior of reads and writes to the block
 #          device between when these calls are executed is undefined.
 #
-# Notes:  It is strongly recommended that this interface is not used especially
-#         for changing block devices.
+# Notes:  This interface is deprecated, and it is strongly recommended that you
+#         avoid using it.  For changing block devices, use
+#         blockdev-change-medium; for changing VNC parameters, use
+#         change-vnc-password.
 #
 # Since: 0.14.0
 ##
@@ -1588,7 +2062,7 @@
 #
 # Notes: This command is experimental and may change syntax in future releases.
 ##
-{ 'type': 'ObjectTypeInfo',
+{ 'struct': 'ObjectTypeInfo',
   'data': { 'name': 'str' } }
 
 ##
@@ -1620,7 +2094,7 @@
 #
 # Since: 1.2
 ##
-{ 'type': 'DevicePropertyInfo',
+{ 'struct': 'DevicePropertyInfo',
   'data': { 'name': 'str', 'type': 'str', '*description': 'str' } }
 
 ##
@@ -1659,6 +2133,24 @@
 { 'command': 'migrate',
   'data': {'uri': 'str', '*blk': 'bool', '*inc': 'bool', '*detach': 'bool' } }
 
+##
+# @migrate-incoming
+#
+# Start an incoming migration, the qemu must have been started
+# with -incoming defer
+#
+# @uri: The Uniform Resource Identifier identifying the source or
+#       address to listen on
+#
+# Returns: nothing on success
+#
+# Since: 2.3
+# Note: It's a bad idea to use a string for the uri, but it needs to stay
+# compatible with -incoming and the format of the uri is already exposed
+# above libvirt
+##
+{ 'command': 'migrate-incoming', 'data': {'uri': 'str' } }
+
 # @xen-save-devices-state:
 #
 # Save the state of all devices to file. The RAM and the block devices
@@ -1692,7 +2184,7 @@
 #
 # Remove a device from a guest
 #
-# @id: the name of the device
+# @id: the name or QOM path of the device
 #
 # Returns: Nothing on success
 #          If @id is not a valid device, DeviceNotFound
@@ -1730,8 +2222,7 @@
 # @dump-guest-memory
 #
 # Dump guest's memory to vmcore. It is a synchronous operation that can take
-# very long depending on the amount of guest memory. This command is only
-# supported on i386 and x86_64.
+# very long depending on the amount of guest memory.
 #
 # @paging: if true, do paging to get guest's memory mapping. This allows
 #          using gdb to process the core file.
@@ -1747,6 +2238,7 @@
 #             2. The guest can be in real-mode even if paging is enabled. For
 #                example, the guest uses ACPI to sleep, and ACPI sleep state
 #                goes in real-mode
+#             3. Currently only supported on i386 and x86_64.
 #
 # @protocol: the filename or file descriptor of the vmcore. The supported
 #            protocols are:
@@ -1756,6 +2248,10 @@
 #            2. fd: the protocol starts with "fd:", and the following string
 #               is the fd's name.
 #
+# @detach: #optional if true, QMP will return immediately rather than
+#          waiting for the dump to finish. The user can track progress
+#          using "query-dump". (since 2.6).
+#
 # @begin: #optional if specified, the starting physical address.
 #
 # @length: #optional if specified, the memory size, in bytes. If you don't
@@ -1772,8 +2268,56 @@
 # Since: 1.2
 ##
 { 'command': 'dump-guest-memory',
-  'data': { 'paging': 'bool', 'protocol': 'str', '*begin': 'int',
-            '*length': 'int', '*format': 'DumpGuestMemoryFormat' } }
+  'data': { 'paging': 'bool', 'protocol': 'str', '*detach': 'bool',
+            '*begin': 'int', '*length': 'int',
+            '*format': 'DumpGuestMemoryFormat'} }
+
+##
+# @DumpStatus
+#
+# Describe the status of a long-running background guest memory dump.
+#
+# @none: no dump-guest-memory has started yet.
+#
+# @active: there is one dump running in background.
+#
+# @completed: the last dump has finished successfully.
+#
+# @failed: the last dump has failed.
+#
+# Since 2.6
+##
+{ 'enum': 'DumpStatus',
+  'data': [ 'none', 'active', 'completed', 'failed' ] }
+
+##
+# @DumpQueryResult
+#
+# The result format for 'query-dump'.
+#
+# @status: enum of @DumpStatus, which shows current dump status
+#
+# @completed: bytes written in latest dump (uncompressed)
+#
+# @total: total bytes to be written in latest dump (uncompressed)
+#
+# Since 2.6
+##
+{ 'struct': 'DumpQueryResult',
+  'data': { 'status': 'DumpStatus',
+            'completed': 'int',
+            'total': 'int' } }
+
+##
+# @query-dump
+#
+# Query latest dump status.
+#
+# Returns: A @DumpStatus object showing the dump status.
+#
+# Since: 2.6
+##
+{ 'command': 'query-dump', 'returns': 'DumpQueryResult' }
 
 ##
 # @DumpGuestMemoryCapability:
@@ -1782,7 +2326,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'DumpGuestMemoryCapability',
+{ 'struct': 'DumpGuestMemoryCapability',
   'data': {
       'formats': ['DumpGuestMemoryFormat'] } }
 
@@ -1800,6 +2344,20 @@
   'returns': 'DumpGuestMemoryCapability' }
 
 ##
+# @dump-skeys
+#
+# Dump guest's storage keys
+#
+# @filename: the path to the file to dump to
+#
+# This command is only supported on s390 architecture.
+#
+# Since: 2.5
+##
+{ 'command': 'dump-skeys',
+  'data': { 'filename': 'str' } }
+
+##
 # @netdev_add:
 #
 # Add a network backend.
@@ -1809,11 +2367,12 @@
 #
 # @id: the name of the new network backend
 #
-# @props: #optional a list of properties to be passed to the backend in
-#         the format 'name=value', like 'ifname=tap0,script=no'
+# Additional arguments depend on the type.
 #
-# Notes: The semantics of @props is not well defined.  Future commands will be
-#        introduced that provide stronger typing for backend creation.
+# TODO This command effectively bypasses QAPI completely due to its
+# "additional arguments" business.  It shouldn't have been added to
+# the schema in this form.  It should be qapified properly, or
+# replaced by a properly qapified command.
 #
 # Since: 0.14.0
 #
@@ -1821,8 +2380,8 @@
 #          If @type is not a valid network backend, DeviceNotFound
 ##
 { 'command': 'netdev_add',
-  'data': {'type': 'str', 'id': 'str', '*props': '**'},
-  'gen': 'no' }
+  'data': {'type': 'str', 'id': 'str'},
+  'gen': false }                # so we can get the additional arguments
 
 ##
 # @netdev_del:
@@ -1855,8 +2414,7 @@
 # Since: 2.0
 ##
 { 'command': 'object-add',
-  'data': {'qom-type': 'str', 'id': 'str', '*props': 'dict'},
-  'gen': 'no' }
+  'data': {'qom-type': 'str', 'id': 'str', '*props': 'any'} }
 
 ##
 # @object-del:
@@ -1879,7 +2437,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevNoneOptions',
+{ 'struct': 'NetdevNoneOptions',
   'data': { } }
 
 ##
@@ -1899,7 +2457,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetLegacyNicOptions',
+{ 'struct': 'NetLegacyNicOptions',
   'data': {
     '*netdev':  'str',
     '*macaddr': 'str',
@@ -1914,7 +2472,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'String',
+{ 'struct': 'String',
   'data': {
     'str': 'str' } }
 
@@ -1928,9 +2486,18 @@
 #
 # @restrict: #optional isolate the guest from the host
 #
+# @ipv4: #optional whether to support IPv4, default true for enabled
+#        (since 2.6)
+#
+# @ipv6: #optional whether to support IPv6, default true for enabled
+#        (since 2.6)
+#
 # @ip: #optional legacy parameter, use net= instead
 #
-# @net: #optional IP address and optional netmask
+# @net: #optional IP network address that the guest will see, in the
+#       form addr[/netmask] The netmask is optional, and can be
+#       either in the form a.b.c.d or as a number of valid top-most
+#       bits. Default is 10.0.2.0/24.
 #
 # @host: #optional guest-visible address of the host
 #
@@ -1946,6 +2513,18 @@
 # @dnssearch: #optional list of DNS suffixes to search, passed as DHCP option
 #             to the guest
 #
+# @ipv6-prefix: #optional IPv6 network prefix (default is fec0::) (since
+#               2.6). The network prefix is given in the usual
+#               hexadecimal IPv6 address notation.
+#
+# @ipv6-prefixlen: #optional IPv6 network prefix length (default is 64)
+#                  (since 2.6)
+#
+# @ipv6-host: #optional guest-visible IPv6 address of the host (since 2.6)
+#
+# @ipv6-dns: #optional guest-visible IPv6 address of the virtual
+#            nameserver (since 2.6)
+#
 # @smb: #optional root directory of the built-in SMB server
 #
 # @smbserver: #optional IP address of the built-in SMB server
@@ -1957,10 +2536,12 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevUserOptions',
+{ 'struct': 'NetdevUserOptions',
   'data': {
     '*hostname':  'str',
     '*restrict':  'bool',
+    '*ipv4':      'bool',
+    '*ipv6':      'bool',
     '*ip':        'str',
     '*net':       'str',
     '*host':      'str',
@@ -1969,6 +2550,10 @@
     '*dhcpstart': 'str',
     '*dns':       'str',
     '*dnssearch': ['String'],
+    '*ipv6-prefix':      'str',
+    '*ipv6-prefixlen':   'int',
+    '*ipv6-host':        'str',
+    '*ipv6-dns':         'str',
     '*smb':       'str',
     '*smbserver': 'str',
     '*hostfwd':   ['String'],
@@ -2007,9 +2592,12 @@
 #
 # @queues: #optional number of queues to be created for multiqueue capable tap
 #
+# @poll-us: #optional maximum number of microseconds that could
+# be spent on busy polling for tap (since 2.7)
+#
 # Since 1.2
 ##
-{ 'type': 'NetdevTapOptions',
+{ 'struct': 'NetdevTapOptions',
   'data': {
     '*ifname':     'str',
     '*fd':         'str',
@@ -2023,7 +2611,8 @@
     '*vhostfd':    'str',
     '*vhostfds':   'str',
     '*vhostforce': 'bool',
-    '*queues':     'uint32'} }
+    '*queues':     'uint32',
+    '*poll-us':    'uint32'} }
 
 ##
 # @NetdevSocketOptions
@@ -2045,7 +2634,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevSocketOptions',
+{ 'struct': 'NetdevSocketOptions',
   'data': {
     '*fd':        'str',
     '*listen':    'str',
@@ -2093,7 +2682,7 @@
 #
 # Since 2.1
 ##
-{ 'type': 'NetdevL2TPv3Options',
+{ 'struct': 'NetdevL2TPv3Options',
   'data': {
     'src':          'str',
     'dst':          'str',
@@ -2125,7 +2714,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevVdeOptions',
+{ 'struct': 'NetdevVdeOptions',
   'data': {
     '*sock':  'str',
     '*port':  'uint16',
@@ -2144,7 +2733,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevDumpOptions',
+{ 'struct': 'NetdevDumpOptions',
   'data': {
     '*len':  'size',
     '*file': 'str' } }
@@ -2160,7 +2749,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevBridgeOptions',
+{ 'struct': 'NetdevBridgeOptions',
   'data': {
     '*br':     'str',
     '*helper': 'str' } }
@@ -2174,7 +2763,7 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetdevHubPortOptions',
+{ 'struct': 'NetdevHubPortOptions',
   'data': {
     'hubid':     'int32' } }
 
@@ -2194,7 +2783,7 @@
 #
 # Since 2.0
 ##
-{ 'type': 'NetdevNetmapOptions',
+{ 'struct': 'NetdevNetmapOptions',
   'data': {
     'ifname':     'str',
     '*devname':    'str' } }
@@ -2208,24 +2797,44 @@
 #
 # @vhostforce: #optional vhost on for non-MSIX virtio guests (default: false).
 #
+# @queues: #optional number of queues to be created for multiqueue vhost-user
+#          (default: 1) (Since 2.5)
+#
 # Since 2.1
 ##
-{ 'type': 'NetdevVhostUserOptions',
+{ 'struct': 'NetdevVhostUserOptions',
   'data': {
     'chardev':        'str',
-    '*vhostforce':    'bool' } }
+    '*vhostforce':    'bool',
+    '*queues':        'int' } }
 
 ##
-# @NetClientOptions
+# @NetClientDriver
 #
-# A discriminated record of network device traits.
+# Available netdev drivers.
+#
+# Since 2.7
+##
+{ 'enum': 'NetClientDriver',
+  'data': [ 'none', 'nic', 'user', 'tap', 'l2tpv3', 'socket', 'vde', 'dump',
+            'bridge', 'hubport', 'netmap', 'vhost-user' ] }
+
+##
+# @Netdev
+#
+# Captures the configuration of a network device.
+#
+# @id: identifier for monitor commands.
+#
+# @type: Specify the driver used for interpreting remaining arguments.
 #
 # Since 1.2
 #
 # 'l2tpv3' - since 2.1
-#
 ##
-{ 'union': 'NetClientOptions',
+{ 'union': 'Netdev',
+  'base': { 'id': 'str', 'type': 'NetClientDriver' },
+  'discriminator': 'type',
   'data': {
     'none':     'NetdevNoneOptions',
     'nic':      'NetLegacyNicOptions',
@@ -2255,28 +2864,53 @@
 #
 # Since 1.2
 ##
-{ 'type': 'NetLegacy',
+{ 'struct': 'NetLegacy',
   'data': {
     '*vlan': 'int32',
     '*id':   'str',
     '*name': 'str',
-    'opts':  'NetClientOptions' } }
+    'opts':  'NetLegacyOptions' } }
 
 ##
-# @Netdev
+# @NetLegacyOptions
 #
-# Captures the configuration of a network device.
-#
-# @id: identifier for monitor commands.
-#
-# @opts: device type specific properties
+# Like Netdev, but for use only by the legacy command line options
 #
 # Since 1.2
 ##
-{ 'type': 'Netdev',
+{ 'union': 'NetLegacyOptions',
   'data': {
-    'id':   'str',
-    'opts': 'NetClientOptions' } }
+    'none':     'NetdevNoneOptions',
+    'nic':      'NetLegacyNicOptions',
+    'user':     'NetdevUserOptions',
+    'tap':      'NetdevTapOptions',
+    'l2tpv3':   'NetdevL2TPv3Options',
+    'socket':   'NetdevSocketOptions',
+    'vde':      'NetdevVdeOptions',
+    'dump':     'NetdevDumpOptions',
+    'bridge':   'NetdevBridgeOptions',
+    'netmap':   'NetdevNetmapOptions',
+    'vhost-user': 'NetdevVhostUserOptions' } }
+
+##
+# @NetFilterDirection
+#
+# Indicates whether a netfilter is attached to a netdev's transmit queue or
+# receive queue or both.
+#
+# @all: the filter is attached both to the receive and the transmit
+#       queue of the netdev (default).
+#
+# @rx: the filter is attached to the receive queue of the netdev,
+#      where it will receive packets sent to the netdev.
+#
+# @tx: the filter is attached to the transmit queue of the netdev,
+#      where it will receive packets sent by the netdev.
+#
+# Since 2.5
+##
+{ 'enum': 'NetFilterDirection',
+  'data': [ 'all', 'rx', 'tx' ] }
 
 ##
 # @InetSocketAddress
@@ -2297,7 +2931,7 @@
 #
 # Since 1.3
 ##
-{ 'type': 'InetSocketAddress',
+{ 'struct': 'InetSocketAddress',
   'data': {
     'host': 'str',
     'port': 'str',
@@ -2314,7 +2948,7 @@
 #
 # Since 1.3
 ##
-{ 'type': 'UnixSocketAddress',
+{ 'struct': 'UnixSocketAddress',
   'data': {
     'path': 'str' } }
 
@@ -2377,11 +3011,14 @@
 # @cpu-max: maximum number of CPUs supported by the machine type
 #           (since 1.5.0)
 #
+# @hotpluggable-cpus: cpu hotplug via -device is supported (since 2.7.0)
+#
 # Since: 1.2.0
 ##
-{ 'type': 'MachineInfo',
+{ 'struct': 'MachineInfo',
   'data': { 'name': 'str', '*alias': 'str',
-            '*is-default': 'bool', 'cpu-max': 'int' } }
+            '*is-default': 'bool', 'cpu-max': 'int',
+            'hotpluggable-cpus': 'bool'} }
 
 ##
 # @query-machines:
@@ -2403,7 +3040,7 @@
 #
 # Since: 1.2.0
 ##
-{ 'type': 'CpuDefinitionInfo',
+{ 'struct': 'CpuDefinitionInfo',
   'data': { 'name': 'str' } }
 
 ##
@@ -2428,7 +3065,7 @@
 #
 # Since: 1.2.0
 ##
-{ 'type': 'AddfdInfo', 'data': {'fdset-id': 'int', 'fd': 'int'} }
+{ 'struct': 'AddfdInfo', 'data': {'fdset-id': 'int', 'fd': 'int'} }
 
 ##
 # @add-fd:
@@ -2484,7 +3121,7 @@
 #
 # Since: 1.2.0
 ##
-{ 'type': 'FdsetFdInfo',
+{ 'struct': 'FdsetFdInfo',
   'data': {'fd': 'int', '*opaque': 'str'} }
 
 ##
@@ -2498,7 +3135,7 @@
 #
 # Since: 1.2.0
 ##
-{ 'type': 'FdsetInfo',
+{ 'struct': 'FdsetInfo',
   'data': {'fdset-id': 'int', 'fds': ['FdsetFdInfo']} }
 
 ##
@@ -2524,7 +3161,7 @@
 #
 # Since: 1.2.0
 ##
-{ 'type': 'TargetInfo',
+{ 'struct': 'TargetInfo',
   'data': { 'arch': 'str' } }
 
 ##
@@ -2548,6 +3185,8 @@
 # Since: 1.3.0
 #
 # 'unmapped' and 'pause' since 2.0
+# 'ro' and 'kp_comma' since 2.4
+# 'kp_equals' and 'power' since 2.6
 ##
 { 'enum': 'QKeyCode',
   'data': [ 'unmapped',
@@ -2565,7 +3204,8 @@
             'kp_9', 'less', 'f11', 'f12', 'print', 'home', 'pgup', 'pgdn', 'end',
             'left', 'up', 'down', 'right', 'insert', 'delete', 'stop', 'again',
             'props', 'undo', 'front', 'copy', 'open', 'paste', 'find', 'cut',
-             'lf', 'help', 'meta_l', 'meta_r', 'compose', 'pause' ] }
+            'lf', 'help', 'meta_l', 'meta_r', 'compose', 'pause', 'ro',
+            'kp_comma', 'kp_equals', 'power' ] }
 
 ##
 # @KeyValue
@@ -2614,6 +3254,21 @@
 ##
 { 'command': 'screendump', 'data': {'filename': 'str'} }
 
+
+##
+# @ChardevCommon:
+#
+# Configuration shared across all chardev backends
+#
+# @logfile: #optional The name of a logfile to save output
+# @logappend: #optional true to append instead of truncate
+#             (default to false to truncate)
+#
+# Since: 2.6
+##
+{ 'struct': 'ChardevCommon', 'data': { '*logfile': 'str',
+                                       '*logappend': 'bool' } }
+
 ##
 # @ChardevFile:
 #
@@ -2621,11 +3276,15 @@
 #
 # @in:  #optional The name of the input file
 # @out: The name of the output file
+# @append: #optional Open the file in append mode (default false to
+#          truncate) (Since 2.6)
 #
 # Since: 1.4
 ##
-{ 'type': 'ChardevFile', 'data': { '*in' : 'str',
-                                   'out' : 'str' } }
+{ 'struct': 'ChardevFile', 'data': { '*in' : 'str',
+                                   'out' : 'str',
+                                   '*append': 'bool' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevHostdev:
@@ -2638,7 +3297,8 @@
 #
 # Since: 1.4
 ##
-{ 'type': 'ChardevHostdev', 'data': { 'device' : 'str' } }
+{ 'struct': 'ChardevHostdev', 'data': { 'device' : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevSocket:
@@ -2647,6 +3307,7 @@
 #
 # @addr: socket address to listen on (server=true)
 #        or connect to (server=false)
+# @tls-creds: #optional the ID of the TLS credentials object (since 2.6)
 # @server: #optional create server socket (default: true)
 # @wait: #optional wait for incoming connection on server
 #        sockets (default: false).
@@ -2660,12 +3321,14 @@
 #
 # Since: 1.4
 ##
-{ 'type': 'ChardevSocket', 'data': { 'addr'       : 'SocketAddress',
+{ 'struct': 'ChardevSocket', 'data': { 'addr'       : 'SocketAddress',
+                                     '*tls-creds'  : 'str',
                                      '*server'    : 'bool',
                                      '*wait'      : 'bool',
                                      '*nodelay'   : 'bool',
                                      '*telnet'    : 'bool',
-                                     '*reconnect' : 'int' } }
+                                     '*reconnect' : 'int' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevUdp:
@@ -2677,8 +3340,9 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevUdp', 'data': { 'remote' : 'SocketAddress',
-                                  '*local' : 'SocketAddress' } }
+{ 'struct': 'ChardevUdp', 'data': { 'remote' : 'SocketAddress',
+                                  '*local' : 'SocketAddress' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevMux:
@@ -2689,7 +3353,8 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevMux', 'data': { 'chardev' : 'str' } }
+{ 'struct': 'ChardevMux', 'data': { 'chardev' : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevStdio:
@@ -2702,7 +3367,9 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevStdio', 'data': { '*signal' : 'bool' } }
+{ 'struct': 'ChardevStdio', 'data': { '*signal' : 'bool' },
+  'base': 'ChardevCommon' }
+
 
 ##
 # @ChardevSpiceChannel:
@@ -2713,7 +3380,8 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevSpiceChannel', 'data': { 'type'  : 'str' } }
+{ 'struct': 'ChardevSpiceChannel', 'data': { 'type'  : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevSpicePort:
@@ -2724,7 +3392,8 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevSpicePort', 'data': { 'fqdn'  : 'str' } }
+{ 'struct': 'ChardevSpicePort', 'data': { 'fqdn'  : 'str' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevVC:
@@ -2738,10 +3407,11 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevVC', 'data': { '*width'  : 'int',
+{ 'struct': 'ChardevVC', 'data': { '*width'  : 'int',
                                  '*height' : 'int',
                                  '*cols'   : 'int',
-                                 '*rows'   : 'int' } }
+                                 '*rows'   : 'int' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevRingbuf:
@@ -2752,7 +3422,8 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'ChardevRingbuf', 'data': { '*size'  : 'int' } }
+{ 'struct': 'ChardevRingbuf', 'data': { '*size'  : 'int' },
+  'base': 'ChardevCommon' }
 
 ##
 # @ChardevBackend:
@@ -2761,22 +3432,20 @@
 #
 # Since: 1.4 (testdev since 2.2)
 ##
-{ 'type': 'ChardevDummy', 'data': { } }
-
 { 'union': 'ChardevBackend', 'data': { 'file'   : 'ChardevFile',
                                        'serial' : 'ChardevHostdev',
                                        'parallel': 'ChardevHostdev',
                                        'pipe'   : 'ChardevHostdev',
                                        'socket' : 'ChardevSocket',
                                        'udp'    : 'ChardevUdp',
-                                       'pty'    : 'ChardevDummy',
-                                       'null'   : 'ChardevDummy',
+                                       'pty'    : 'ChardevCommon',
+                                       'null'   : 'ChardevCommon',
                                        'mux'    : 'ChardevMux',
-                                       'msmouse': 'ChardevDummy',
-                                       'braille': 'ChardevDummy',
-                                       'testdev': 'ChardevDummy',
+                                       'msmouse': 'ChardevCommon',
+                                       'braille': 'ChardevCommon',
+                                       'testdev': 'ChardevCommon',
                                        'stdio'  : 'ChardevStdio',
-                                       'console': 'ChardevDummy',
+                                       'console': 'ChardevCommon',
                                        'spicevmc' : 'ChardevSpiceChannel',
                                        'spiceport' : 'ChardevSpicePort',
                                        'vc'     : 'ChardevVC',
@@ -2794,7 +3463,7 @@
 #
 # Since: 1.4
 ##
-{ 'type' : 'ChardevReturn', 'data': { '*pty' : 'str' } }
+{ 'struct' : 'ChardevReturn', 'data': { '*pty' : 'str' } }
 
 ##
 # @chardev-add:
@@ -2881,7 +3550,7 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'TPMPassthroughOptions', 'data': { '*path' : 'str',
+{ 'struct': 'TPMPassthroughOptions', 'data': { '*path' : 'str',
                                              '*cancel-path' : 'str'} }
 
 ##
@@ -2909,7 +3578,7 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'TPMInfo',
+{ 'struct': 'TPMInfo',
   'data': {'id': 'str',
            'model': 'TpmModel',
            'options': 'TpmTypeOptions' } }
@@ -2971,7 +3640,7 @@
 #
 # Since 1.5
 ##
-{ 'type': 'AcpiTableOptions',
+{ 'struct': 'AcpiTableOptions',
   'data': {
     '*sig':               'str',
     '*rev':               'uint8',
@@ -3017,7 +3686,7 @@
 #
 # Since 1.5
 ##
-{ 'type': 'CommandLineParameterInfo',
+{ 'struct': 'CommandLineParameterInfo',
   'data': { 'name': 'str',
             'type': 'CommandLineParameterType',
             '*help': 'str',
@@ -3034,7 +3703,7 @@
 #
 # Since 1.5
 ##
-{ 'type': 'CommandLineOptionInfo',
+{ 'struct': 'CommandLineOptionInfo',
   'data': { 'option': 'str', 'parameters': ['CommandLineParameterInfo'] } }
 
 ##
@@ -3078,13 +3747,24 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'X86CPUFeatureWordInfo',
+{ 'struct': 'X86CPUFeatureWordInfo',
   'data': { 'cpuid-input-eax': 'int',
             '*cpuid-input-ecx': 'int',
             'cpuid-register': 'X86CPURegister32',
             'features': 'int' } }
 
 ##
+# @DummyForceArrays
+#
+# Not used by QMP; hack to let us use X86CPUFeatureWordInfoList internally
+#
+# Since 2.5
+##
+{ 'struct': 'DummyForceArrays',
+  'data': { 'unused': ['X86CPUFeatureWordInfo'] } }
+
+
+##
 # @RxState:
 #
 # Packets receiving state
@@ -3131,7 +3811,7 @@
 # Since 1.6
 ##
 
-{ 'type': 'RxFilterInfo',
+{ 'struct': 'RxFilterInfo',
   'data': {
     'name':               'str',
     'promiscuous':        'bool',
@@ -3171,17 +3851,17 @@
 # Since: 2.0
 ##
 { 'enum'  : 'InputButton',
-  'data'  : [ 'Left', 'Middle', 'Right', 'WheelUp', 'WheelDown' ] }
+  'data'  : [ 'left', 'middle', 'right', 'wheel-up', 'wheel-down' ] }
 
 ##
-# @InputButton
+# @InputAxis
 #
 # Position axis of a pointer input device (mouse, tablet).
 #
 # Since: 2.0
 ##
 { 'enum'  : 'InputAxis',
-  'data'  : [ 'X', 'Y' ] }
+  'data'  : [ 'x', 'y' ] }
 
 ##
 # @InputKeyEvent
@@ -3193,7 +3873,7 @@
 #
 # Since: 2.0
 ##
-{ 'type'  : 'InputKeyEvent',
+{ 'struct'  : 'InputKeyEvent',
   'data'  : { 'key'     : 'KeyValue',
               'down'    : 'bool' } }
 
@@ -3207,7 +3887,7 @@
 #
 # Since: 2.0
 ##
-{ 'type'  : 'InputBtnEvent',
+{ 'struct'  : 'InputBtnEvent',
   'data'  : { 'button'  : 'InputButton',
               'down'    : 'bool' } }
 
@@ -3222,7 +3902,7 @@
 #
 # Since: 2.0
 ##
-{ 'type'  : 'InputMoveEvent',
+{ 'struct'  : 'InputMoveEvent',
   'data'  : { 'axis'    : 'InputAxis',
               'value'   : 'int' } }
 
@@ -3245,23 +3925,34 @@
               'abs'     : 'InputMoveEvent' } }
 
 ##
-# @x-input-send-event
+# @input-send-event
 #
 # Send input event(s) to guest.
 #
-# @console: #optional console to send event(s) to.
-#
+# @device: #optional display device to send event(s) to.
+# @head: #optional head to send event(s) to, in case the
+#        display device supports multiple scanouts.
 # @events: List of InputEvent union.
 #
 # Returns: Nothing on success.
 #
-# Since: 2.2
+# The @display and @head parameters can be used to send the input
+# event to specific input devices in case (a) multiple input devices
+# of the same kind are added to the virtual machine and (b) you have
+# configured input routing (see docs/multiseat.txt) for those input
+# devices.  The parameters work exactly like the device and head
+# properties of input devices.  If @device is missing, only devices
+# that have no input routing config are admissible.  If @device is
+# specified, both input devices with and without input routing config
+# are admissible, but devices with input routing config take
+# precedence.
 #
-# Note: this command is experimental, and not a stable API.
-#
+# Since: 2.6
 ##
-{ 'command': 'x-input-send-event',
-  'data': { '*console':'int', 'events': [ 'InputEvent' ] } }
+{ 'command': 'input-send-event',
+  'data': { '*device': 'str',
+            '*head'  : 'int',
+            'events' : [ 'InputEvent' ] } }
 
 ##
 # @NumaOptions
@@ -3293,7 +3984,7 @@
 #
 # Since: 2.1
 ##
-{ 'type': 'NumaNodeOptions',
+{ 'struct': 'NumaNodeOptions',
   'data': {
    '*nodeid': 'uint16',
    '*cpus':   ['uint16'],
@@ -3340,7 +4031,7 @@
 # Since: 2.1
 ##
 
-{ 'type': 'Memdev',
+{ 'struct': 'Memdev',
   'data': {
     'size':       'size',
     'merge':      'bool',
@@ -3383,7 +4074,7 @@
 #
 # Since: 2.1
 ##
-{ 'type': 'PCDIMMDeviceInfo',
+{ 'struct': 'PCDIMMDeviceInfo',
   'data': { '*id': 'str',
             'addr': 'int',
             'size': 'int',
@@ -3416,8 +4107,9 @@
 ## @ACPISlotType
 #
 # @DIMM: memory slot
+# @CPU: logical CPU slot (since 2.7)
 #
-{ 'enum': 'ACPISlotType', 'data': [ 'DIMM' ] }
+{ 'enum': 'ACPISlotType', 'data': [ 'DIMM', 'CPU' ] }
 
 ## @ACPIOSTInfo
 #
@@ -3437,7 +4129,7 @@
 #
 # Since: 2.1
 ##
-{ 'type': 'ACPIOSTInfo',
+{ 'struct': 'ACPIOSTInfo',
   'data'  : { '*device': 'str',
               'slot': 'str',
               'slot-type': 'ACPISlotType',
@@ -3473,10 +4165,14 @@
 #
 # @none: nothing is done
 #
+# @inject-nmi: a non-maskable interrupt is injected into the first VCPU (all
+#              VCPUS on x86) (since 2.4)
+#
 # Since: 2.1
 ##
 { 'enum': 'WatchdogExpirationAction',
-  'data': [ 'reset', 'shutdown', 'poweroff', 'pause', 'debug', 'none' ] }
+  'data': [ 'reset', 'shutdown', 'poweroff', 'pause', 'debug', 'none',
+            'inject-nmi' ] }
 
 ##
 # @IoOperationType
@@ -3515,3 +4211,130 @@
 # Since: 2.1
 ##
 { 'command': 'rtc-reset-reinjection' }
+
+# Rocker ethernet network switch
+{ 'include': 'qapi/rocker.json' }
+
+##
+# ReplayMode:
+#
+# Mode of the replay subsystem.
+#
+# @none: normal execution mode. Replay or record are not enabled.
+#
+# @record: record mode. All non-deterministic data is written into the
+#          replay log.
+#
+# @play: replay mode. Non-deterministic data required for system execution
+#        is read from the log.
+#
+# Since: 2.5
+##
+{ 'enum': 'ReplayMode',
+  'data': [ 'none', 'record', 'play' ] }
+
+##
+# @xen-load-devices-state:
+#
+# Load the state of all devices from file. The RAM and the block devices
+# of the VM are not loaded by this command.
+#
+# @filename: the file to load the state of the devices from as binary
+# data. See xen-save-devices-state.txt for a description of the binary
+# format.
+#
+# Since: 2.7
+##
+{ 'command': 'xen-load-devices-state', 'data': {'filename': 'str'} }
+
+##
+# @GICCapability:
+#
+# The struct describes capability for a specific GIC (Generic
+# Interrupt Controller) version. These bits are not only decided by
+# QEMU/KVM software version, but also decided by the hardware that
+# the program is running upon.
+#
+# @version:  version of GIC to be described. Currently, only 2 and 3
+#            are supported.
+#
+# @emulated: whether current QEMU/hardware supports emulated GIC
+#            device in user space.
+#
+# @kernel:   whether current QEMU/hardware supports hardware
+#            accelerated GIC device in kernel.
+#
+# Since: 2.6
+##
+{ 'struct': 'GICCapability',
+  'data': { 'version': 'int',
+            'emulated': 'bool',
+            'kernel': 'bool' } }
+
+##
+# @query-gic-capabilities:
+#
+# This command is ARM-only. It will return a list of GICCapability
+# objects that describe its capability bits.
+#
+# Returns: a list of GICCapability objects.
+#
+# Since: 2.6
+##
+{ 'command': 'query-gic-capabilities', 'returns': ['GICCapability'] }
+
+##
+# CpuInstanceProperties
+#
+# List of properties to be used for hotplugging a CPU instance,
+# it should be passed by management with device_add command when
+# a CPU is being hotplugged.
+#
+# Note: currently there are 4 properties that could be present
+# but management should be prepared to pass through other
+# properties with device_add command to allow for future
+# interface extension. This also requires the filed names to be kept in
+# sync with the properties passed to -device/device_add.
+#
+# @node-id: #optional NUMA node ID the CPU belongs to
+# @socket-id: #optional socket number within node/board the CPU belongs to
+# @core-id: #optional core number within socket the CPU belongs to
+# @thread-id: #optional thread number within core the CPU belongs to
+#
+# Since: 2.7
+##
+{ 'struct': 'CpuInstanceProperties',
+  'data': { '*node-id': 'int',
+            '*socket-id': 'int',
+            '*core-id': 'int',
+            '*thread-id': 'int'
+  }
+}
+
+##
+# @HotpluggableCPU
+#
+# @type: CPU object type for usage with device_add command
+# @props: list of properties to be used for hotplugging CPU
+# @vcpus-count: number of logical VCPU threads @HotpluggableCPU provides
+# @qom-path: #optional link to existing CPU object if CPU is present or
+#            omitted if CPU is not present.
+#
+# Since: 2.7
+##
+{ 'struct': 'HotpluggableCPU',
+  'data': { 'type': 'str',
+            'vcpus-count': 'int',
+            'props': 'CpuInstanceProperties',
+            '*qom-path': 'str'
+          }
+}
+
+##
+# @query-hotpluggable-cpus
+#
+# Returns: a list of HotpluggableCPU objects.
+#
+# Since: 2.7
+##
+{ 'command': 'query-hotpluggable-cpus', 'returns': ['HotpluggableCPU'] }
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index 2278970..7ea4aeb 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,6 +1,6 @@
 util-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
 util-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
 util-obj-y += string-input-visitor.o string-output-visitor.o
-util-obj-y += opts-visitor.o
+util-obj-y += opts-visitor.o qapi-clone-visitor.o
 util-obj-y += qmp-event.o
 util-obj-y += qapi-util.o
diff --git a/qapi/block-core.json b/qapi/block-core.json
index a14e6ab..5e2d7d7 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -26,7 +26,7 @@
 #
 ##
 
-{ 'type': 'SnapshotInfo',
+{ 'struct': 'SnapshotInfo',
   'data': { 'id': 'str', 'name': 'str', 'vm-state-size': 'int',
             'date-sec': 'int', 'date-nsec': 'int',
             'vm-clock-sec': 'int', 'vm-clock-nsec': 'int' } }
@@ -41,13 +41,16 @@
 # @corrupt: #optional true if the image has been marked corrupt; only valid for
 #           compat >= 1.1 (since 2.2)
 #
+# @refcount-bits: width of a refcount entry in bits (since 2.3)
+#
 # Since: 1.7
 ##
-{ 'type': 'ImageInfoSpecificQCow2',
+{ 'struct': 'ImageInfoSpecificQCow2',
   'data': {
       'compat': 'str',
       '*lazy-refcounts': 'bool',
-      '*corrupt': 'bool'
+      '*corrupt': 'bool',
+      'refcount-bits': 'int'
   } }
 
 ##
@@ -63,7 +66,7 @@
 #
 # Since: 1.7
 ##
-{ 'type': 'ImageInfoSpecificVmdk',
+{ 'struct': 'ImageInfoSpecificVmdk',
   'data': {
       'create-type': 'str',
       'cid': 'int',
@@ -82,7 +85,11 @@
 { 'union': 'ImageInfoSpecific',
   'data': {
       'qcow2': 'ImageInfoSpecificQCow2',
-      'vmdk': 'ImageInfoSpecificVmdk'
+      'vmdk': 'ImageInfoSpecificVmdk',
+      # If we need to add block driver specific parameters for
+      # LUKS in future, then we'll subclass QCryptoBlockInfoLUKS
+      # to define a ImageInfoSpecificLUKS
+      'luks': 'QCryptoBlockInfoLUKS'
   } }
 
 ##
@@ -123,7 +130,7 @@
 #
 ##
 
-{ 'type': 'ImageInfo',
+{ 'struct': 'ImageInfo',
   'data': {'filename': 'str', 'format': 'str', '*dirty-flag': 'bool',
            '*actual-size': 'int', 'virtual-size': 'int',
            '*cluster-size': 'int', '*encrypted': 'bool', '*compressed': 'bool',
@@ -175,7 +182,7 @@
 #
 ##
 
-{ 'type': 'ImageCheck',
+{ 'struct': 'ImageCheck',
   'data': {'filename': 'str', 'format': 'str', 'check-errors': 'int',
            '*image-end-offset': 'int', '*corruptions': 'int', '*leaks': 'int',
            '*corruptions-fixed': 'int', '*leaks-fixed': 'int',
@@ -183,6 +190,49 @@
            '*fragmented-clusters': 'int', '*compressed-clusters': 'int' } }
 
 ##
+# @MapEntry:
+#
+# Mapping information from a virtual block range to a host file range
+#
+# @start: the start byte of the mapped virtual range
+#
+# @length: the number of bytes of the mapped virtual range
+#
+# @data: whether the mapped range has data
+#
+# @zero: whether the virtual blocks are zeroed
+#
+# @depth: the depth of the mapping
+#
+# @offset: #optional the offset in file that the virtual sectors are mapped to
+#
+# @filename: #optional filename that is referred to by @offset
+#
+# Since: 2.6
+#
+##
+{ 'struct': 'MapEntry',
+  'data': {'start': 'int', 'length': 'int', 'data': 'bool',
+           'zero': 'bool', 'depth': 'int', '*offset': 'int',
+           '*filename': 'str' } }
+
+##
+# @BlockdevCacheInfo
+#
+# Cache mode information for a block device
+#
+# @writeback:   true if writeback mode is enabled
+# @direct:      true if the host page cache is bypassed (O_DIRECT)
+# @no-flush:    true if flush requests are ignored for the device
+#
+# Since: 2.3
+##
+{ 'struct': 'BlockdevCacheInfo',
+  'data': { 'writeback': 'bool',
+            'direct': 'bool',
+            'no-flush': 'bool' } }
+
+##
 # @BlockDeviceInfo:
 #
 # Information about the backing device for a block device.
@@ -196,9 +246,12 @@
 # @drv: the name of the block format used to open the backing device. As of
 #       0.14.0 this can be: 'blkdebug', 'bochs', 'cloop', 'cow', 'dmg',
 #       'file', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
-#       'host_floppy', 'http', 'https', 'nbd', 'parallels', 'qcow',
+#       'http', 'https', 'luks', 'nbd', 'parallels', 'qcow',
 #       'qcow2', 'raw', 'tftp', 'vdi', 'vmdk', 'vpc', 'vvfat'
 #       2.2: 'archipelago' added, 'cow' dropped
+#       2.3: 'host_floppy' deprecated
+#       2.5: 'host_floppy' dropped
+#       2.6: 'luks' added
 #
 # @backing_file: #optional the name of the backing file (for copy-on-write)
 #
@@ -225,24 +278,55 @@
 #
 # @image: the info of image used (since: 1.6)
 #
-# @bps_max: #optional total max in bytes (Since 1.7)
+# @bps_max: #optional total throughput limit during bursts,
+#                     in bytes (Since 1.7)
 #
-# @bps_rd_max: #optional read max in bytes (Since 1.7)
+# @bps_rd_max: #optional read throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @bps_wr_max: #optional write max in bytes (Since 1.7)
+# @bps_wr_max: #optional write throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @iops_max: #optional total I/O operations max (Since 1.7)
+# @iops_max: #optional total I/O operations per second during bursts,
+#                      in bytes (Since 1.7)
 #
-# @iops_rd_max: #optional read I/O operations max (Since 1.7)
+# @iops_rd_max: #optional read I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
 #
-# @iops_wr_max: #optional write I/O operations max (Since 1.7)
+# @iops_wr_max: #optional write I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
+#
+# @bps_max_length: #optional maximum length of the @bps_max burst
+#                            period, in seconds. (Since 2.6)
+#
+# @bps_rd_max_length: #optional maximum length of the @bps_rd_max
+#                               burst period, in seconds. (Since 2.6)
+#
+# @bps_wr_max_length: #optional maximum length of the @bps_wr_max
+#                               burst period, in seconds. (Since 2.6)
+#
+# @iops_max_length: #optional maximum length of the @iops burst
+#                             period, in seconds. (Since 2.6)
+#
+# @iops_rd_max_length: #optional maximum length of the @iops_rd_max
+#                                burst period, in seconds. (Since 2.6)
+#
+# @iops_wr_max_length: #optional maximum length of the @iops_wr_max
+#                                burst period, in seconds. (Since 2.6)
 #
 # @iops_size: #optional an I/O size in bytes (Since 1.7)
 #
+# @group: #optional throttle group name (Since 2.4)
+#
+# @cache: the cache mode used for the block device (since: 2.3)
+#
+# @write_threshold: configured write threshold for the device.
+#                   0 if disabled. (Since 2.3)
+#
 # Since: 0.14.0
 #
 ##
-{ 'type': 'BlockDeviceInfo',
+{ 'struct': 'BlockDeviceInfo',
   'data': { 'file': 'str', '*node-name': 'str', 'ro': 'bool', 'drv': 'str',
             '*backing_file': 'str', 'backing_file_depth': 'int',
             'encrypted': 'bool', 'encryption_key_missing': 'bool',
@@ -253,7 +337,11 @@
             '*bps_max': 'int', '*bps_rd_max': 'int',
             '*bps_wr_max': 'int', '*iops_max': 'int',
             '*iops_rd_max': 'int', '*iops_wr_max': 'int',
-            '*iops_size': 'int' } }
+            '*bps_max_length': 'int', '*bps_rd_max_length': 'int',
+            '*bps_wr_max_length': 'int', '*iops_max_length': 'int',
+            '*iops_rd_max_length': 'int', '*iops_wr_max_length': 'int',
+            '*iops_size': 'int', '*group': 'str', 'cache': 'BlockdevCacheInfo',
+            'write_threshold': 'int' } }
 
 ##
 # @BlockDeviceIoStatus:
@@ -295,23 +383,47 @@
 #
 # Since 1.7
 ##
-{ 'type': 'BlockDeviceMapEntry',
+{ 'struct': 'BlockDeviceMapEntry',
   'data': { 'start': 'int', 'length': 'int', 'depth': 'int', 'zero': 'bool',
             'data': 'bool', '*offset': 'int' } }
 
 ##
+# @DirtyBitmapStatus:
+#
+# An enumeration of possible states that a dirty bitmap can report to the user.
+#
+# @frozen: The bitmap is currently in-use by a backup operation or block job,
+#          and is immutable.
+#
+# @disabled: The bitmap is currently in-use by an internal operation and is
+#            read-only. It can still be deleted.
+#
+# @active: The bitmap is actively monitoring for new writes, and can be cleared,
+#          deleted, or used for backup operations.
+#
+# Since: 2.4
+##
+{ 'enum': 'DirtyBitmapStatus',
+  'data': ['active', 'disabled', 'frozen'] }
+
+##
 # @BlockDirtyInfo:
 #
 # Block dirty bitmap information.
 #
+# @name: #optional the name of the dirty bitmap (Since 2.4)
+#
 # @count: number of dirty bytes according to the dirty bitmap
 #
 # @granularity: granularity of the dirty bitmap in bytes (since 1.4)
 #
+# @status: current status of the dirty bitmap (since 2.4)
+#
 # Since: 1.3
 ##
-{ 'type': 'BlockDirtyInfo',
-  'data': {'count': 'int', 'granularity': 'int'} }
+{ 'struct': 'BlockDirtyInfo',
+  'data': {'*name': 'str', 'count': 'int', 'granularity': 'uint32',
+           'status': 'DirtyBitmapStatus'} }
 
 ##
 # @BlockInfo:
@@ -329,8 +441,8 @@
 # @locked: True if the guest has locked this device from having its media
 #          removed
 #
-# @tray_open: #optional True if the device has a tray and it is open
-#             (only present if removable is true)
+# @tray_open: #optional True if the device's tray is open
+#             (only present if it has a tray)
 #
 # @dirty-bitmaps: #optional dirty bitmaps information (only present if the
 #                 driver has one or more dirty bitmaps) (Since 2.0)
@@ -344,7 +456,7 @@
 #
 # Since:  0.14.0
 ##
-{ 'type': 'BlockInfo',
+{ 'struct': 'BlockInfo',
   'data': {'device': 'str', 'type': 'str', 'removable': 'bool',
            'locked': 'bool', '*inserted': 'BlockDeviceInfo',
            '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus',
@@ -361,6 +473,59 @@
 ##
 { 'command': 'query-block', 'returns': ['BlockInfo'] }
 
+
+##
+# @BlockDeviceTimedStats:
+#
+# Statistics of a block device during a given interval of time.
+#
+# @interval_length: Interval used for calculating the statistics,
+#                   in seconds.
+#
+# @min_rd_latency_ns: Minimum latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @min_wr_latency_ns: Minimum latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @min_flush_latency_ns: Minimum latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @max_rd_latency_ns: Maximum latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @max_wr_latency_ns: Maximum latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @max_flush_latency_ns: Maximum latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @avg_rd_latency_ns: Average latency of read operations in the
+#                     defined interval, in nanoseconds.
+#
+# @avg_wr_latency_ns: Average latency of write operations in the
+#                     defined interval, in nanoseconds.
+#
+# @avg_flush_latency_ns: Average latency of flush operations in the
+#                        defined interval, in nanoseconds.
+#
+# @avg_rd_queue_depth: Average number of pending read operations
+#                      in the defined interval.
+#
+# @avg_wr_queue_depth: Average number of pending write operations
+#                      in the defined interval.
+#
+# Since: 2.5
+##
+
+{ 'struct': 'BlockDeviceTimedStats',
+  'data': { 'interval_length': 'int', 'min_rd_latency_ns': 'int',
+            'max_rd_latency_ns': 'int', 'avg_rd_latency_ns': 'int',
+            'min_wr_latency_ns': 'int', 'max_wr_latency_ns': 'int',
+            'avg_wr_latency_ns': 'int', 'min_flush_latency_ns': 'int',
+            'max_flush_latency_ns': 'int', 'avg_flush_latency_ns': 'int',
+            'avg_rd_queue_depth': 'number', 'avg_wr_queue_depth': 'number' } }
+
 ##
 # @BlockDeviceStats:
 #
@@ -389,13 +554,56 @@
 #                     growable sparse files (like qcow2) that are used on top
 #                     of a physical device.
 #
+# @rd_merged: Number of read requests that have been merged into another
+#             request (Since 2.3).
+#
+# @wr_merged: Number of write requests that have been merged into another
+#             request (Since 2.3).
+#
+# @idle_time_ns: #optional Time since the last I/O operation, in
+#                nanoseconds. If the field is absent it means that
+#                there haven't been any operations yet (Since 2.5).
+#
+# @failed_rd_operations: The number of failed read operations
+#                        performed by the device (Since 2.5)
+#
+# @failed_wr_operations: The number of failed write operations
+#                        performed by the device (Since 2.5)
+#
+# @failed_flush_operations: The number of failed flush operations
+#                           performed by the device (Since 2.5)
+#
+# @invalid_rd_operations: The number of invalid read operations
+#                          performed by the device (Since 2.5)
+#
+# @invalid_wr_operations: The number of invalid write operations
+#                         performed by the device (Since 2.5)
+#
+# @invalid_flush_operations: The number of invalid flush operations
+#                            performed by the device (Since 2.5)
+#
+# @account_invalid: Whether invalid operations are included in the
+#                   last access statistics (Since 2.5)
+#
+# @account_failed: Whether failed operations are included in the
+#                  latency and last access statistics (Since 2.5)
+#
+# @timed_stats: Statistics specific to the set of previously defined
+#               intervals of time (Since 2.5)
+#
 # Since: 0.14.0
 ##
-{ 'type': 'BlockDeviceStats',
+{ 'struct': 'BlockDeviceStats',
   'data': {'rd_bytes': 'int', 'wr_bytes': 'int', 'rd_operations': 'int',
            'wr_operations': 'int', 'flush_operations': 'int',
            'flush_total_time_ns': 'int', 'wr_total_time_ns': 'int',
-           'rd_total_time_ns': 'int', 'wr_highest_offset': 'int' } }
+           'rd_total_time_ns': 'int', 'wr_highest_offset': 'int',
+           'rd_merged': 'int', 'wr_merged': 'int', '*idle_time_ns': 'int',
+           'failed_rd_operations': 'int', 'failed_wr_operations': 'int',
+           'failed_flush_operations': 'int', 'invalid_rd_operations': 'int',
+           'invalid_wr_operations': 'int', 'invalid_flush_operations': 'int',
+           'account_invalid': 'bool', 'account_failed': 'bool',
+           'timed_stats': ['BlockDeviceTimedStats'] } }
 
 ##
 # @BlockStats:
@@ -405,6 +613,8 @@
 # @device: #optional If the stats are for a virtual block device, the name
 #          corresponding to the virtual block device.
 #
+# @node-name: #optional The node name of the device. (Since 2.3)
+#
 # @stats:  A @BlockDeviceStats for the device.
 #
 # @parent: #optional This describes the file block device if it has one.
@@ -414,8 +624,9 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'BlockStats',
-  'data': {'*device': 'str', 'stats': 'BlockDeviceStats',
+{ 'struct': 'BlockStats',
+  'data': {'*device': 'str', '*node-name': 'str',
+           'stats': 'BlockDeviceStats',
            '*parent': 'BlockStats',
            '*backing': 'BlockStats'} }
 
@@ -424,11 +635,20 @@
 #
 # Query the @BlockStats for all virtual block devices.
 #
+# @query-nodes: #optional If true, the command will query all the block nodes
+#               that have a node name, in a list which will include "parent"
+#               information, but not "backing".
+#               If false or omitted, the behavior is as before - query all the
+#               device backends, recursively including their "parent" and
+#               "backing". (Since 2.3)
+#
 # Returns: A list of @BlockStats for each virtual block devices.
 #
 # Since: 0.14.0
 ##
-{ 'command': 'query-blockstats', 'returns': ['BlockStats'] }
+{ 'command': 'query-blockstats',
+  'data': { '*query-nodes': 'bool' },
+  'returns': ['BlockStats'] }
 
 ##
 # @BlockdevOnError:
@@ -448,10 +668,12 @@
 # @stop: for guest operations, stop the virtual machine;
 #        for jobs, pause the job
 #
+# @auto: inherit the error handling policy of the backend (since: 2.7)
+#
 # Since: 1.3
 ##
 { 'enum': 'BlockdevOnError',
-  'data': ['report', 'ignore', 'enospc', 'stop'] }
+  'data': ['report', 'ignore', 'enospc', 'stop', 'auto'] }
 
 ##
 # @MirrorSyncMode:
@@ -465,10 +687,12 @@
 #
 # @none: only copy data written from now on
 #
+# @incremental: only copy data described by the dirty bitmap. Since: 2.4
+#
 # Since: 1.3
 ##
 { 'enum': 'MirrorSyncMode',
-  'data': ['top', 'full', 'none'] }
+  'data': ['top', 'full', 'none', 'incremental'] }
 
 ##
 # @BlockJobType:
@@ -495,7 +719,8 @@
 #
 # @type: the job type ('stream' for image streaming)
 #
-# @device: the block device name
+# @device: The job identifier. Originally the device name but other
+#          values are allowed since QEMU 2.7
 #
 # @len: the maximum progress value
 #
@@ -515,7 +740,7 @@
 #
 # Since: 1.1
 ##
-{ 'type': 'BlockJobInfo',
+{ 'struct': 'BlockJobInfo',
   'data': {'type': 'str', 'device': 'str', 'len': 'int',
            'offset': 'int', 'busy': 'bool', 'paused': 'bool', 'speed': 'int',
            'io-status': 'BlockDeviceIoStatus', 'ready': 'bool'} }
@@ -608,7 +833,7 @@
   'data': [ 'existing', 'absolute-paths' ] }
 
 ##
-# @BlockdevSnapshot
+# @BlockdevSnapshotSync
 #
 # Either @device or @node-name must be set but not both.
 #
@@ -625,14 +850,32 @@
 # @mode: #optional whether and how QEMU should create a new image, default is
 #        'absolute-paths'.
 ##
-{ 'type': 'BlockdevSnapshot',
+{ 'struct': 'BlockdevSnapshotSync',
   'data': { '*device': 'str', '*node-name': 'str',
             'snapshot-file': 'str', '*snapshot-node-name': 'str',
             '*format': 'str', '*mode': 'NewImageMode' } }
 
 ##
+# @BlockdevSnapshot
+#
+# @node: device or node name that will have a snapshot created.
+#
+# @overlay: reference to the existing block device that will become
+#           the overlay of @node, as part of creating the snapshot.
+#           It must not have a current backing file (this can be
+#           achieved by passing "backing": "" to blockdev-add).
+#
+# Since 2.5
+##
+{ 'struct': 'BlockdevSnapshot',
+  'data': { 'node': 'str', 'overlay': 'str' } }
+
+##
 # @DriveBackup
 #
+# @job-id: #optional identifier for the newly-created block job. If
+#          omitted, the device name will be used. (Since 2.7)
+#
 # @device: the name of the device which should be copied.
 #
 # @target: the target of the new image. If the file exists, or if it
@@ -643,14 +886,18 @@
 #          probe if @mode is 'existing', else the format of the source
 #
 # @sync: what parts of the disk image should be copied to the destination
-#        (all the disk, only the sectors allocated in the topmost image, or
-#        only new I/O).
+#        (all the disk, only the sectors allocated in the topmost image, from a
+#        dirty bitmap, or only new I/O).
 #
 # @mode: #optional whether and how QEMU should create a new image, default is
 #        'absolute-paths'.
 #
 # @speed: #optional the maximum speed, in bytes per second
 #
+# @bitmap: #optional the name of dirty bitmap if sync is "incremental".
+#          Must be present if sync is "incremental", must NOT be present
+#          otherwise. (Since 2.4)
+#
 # @on-source-error: #optional the action to take on an error on the source,
 #                   default 'report'.  'stop' and 'enospc' can only be used
 #                   if the block device supports io-status (see BlockInfo).
@@ -665,9 +912,47 @@
 #
 # Since: 1.6
 ##
-{ 'type': 'DriveBackup',
-  'data': { 'device': 'str', 'target': 'str', '*format': 'str',
-            'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
+{ 'struct': 'DriveBackup',
+  'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
+            '*format': 'str', 'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
+            '*speed': 'int', '*bitmap': 'str',
+            '*on-source-error': 'BlockdevOnError',
+            '*on-target-error': 'BlockdevOnError' } }
+
+##
+# @BlockdevBackup
+#
+# @job-id: #optional identifier for the newly-created block job. If
+#          omitted, the device name will be used. (Since 2.7)
+#
+# @device: the name of the device which should be copied.
+#
+# @target: the device name or node-name of the backup target node.
+#
+# @sync: what parts of the disk image should be copied to the destination
+#        (all the disk, only the sectors allocated in the topmost image, or
+#        only new I/O).
+#
+# @speed: #optional the maximum speed, in bytes per second. The default is 0,
+#         for unlimited.
+#
+# @on-source-error: #optional the action to take on an error on the source,
+#                   default 'report'.  'stop' and 'enospc' can only be used
+#                   if the block device supports io-status (see BlockInfo).
+#
+# @on-target-error: #optional the action to take on an error on the target,
+#                   default 'report' (no limitations, since this applies to
+#                   a different block device than @device).
+#
+# Note that @on-source-error and @on-target-error only affect background I/O.
+# If an error occurs during a guest write request, the device's rerror/werror
+# actions will be used.
+#
+# Since: 2.3
+##
+{ 'struct': 'BlockdevBackup',
+  'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
+            'sync': 'MirrorSyncMode',
             '*speed': 'int',
             '*on-source-error': 'BlockdevOnError',
             '*on-target-error': 'BlockdevOnError' } }
@@ -677,7 +962,7 @@
 #
 # Generates a synchronous snapshot of a block device.
 #
-# For the arguments, see the documentation of BlockdevSnapshot.
+# For the arguments, see the documentation of BlockdevSnapshotSync.
 #
 # Returns: nothing on success
 #          If @device is not a valid block device, DeviceNotFound
@@ -685,6 +970,19 @@
 # Since 0.14.0
 ##
 { 'command': 'blockdev-snapshot-sync',
+  'data': 'BlockdevSnapshotSync' }
+
+
+##
+# @blockdev-snapshot
+#
+# Generates a snapshot of a block device.
+#
+# For the arguments, see the documentation of BlockdevSnapshot.
+#
+# Since 2.5
+##
+{ 'command': 'blockdev-snapshot',
   'data': 'BlockdevSnapshot' }
 
 ##
@@ -719,6 +1017,9 @@
 # Live commit of data from overlay image nodes into backing nodes - i.e.,
 # writes data between 'top' and 'base' into 'base'.
 #
+# @job-id: #optional identifier for the newly-created block job. If
+#          omitted, the device name will be used. (Since 2.7)
+#
 # @device:  the name of the device
 #
 # @base:   #optional The file name of the backing image to write data into.
@@ -770,7 +1071,7 @@
 #
 ##
 { 'command': 'block-commit',
-  'data': { 'device': 'str', '*base': 'str', '*top': 'str',
+  'data': { '*job-id': 'str', 'device': 'str', '*base': 'str', '*top': 'str',
             '*backing-file': 'str', '*speed': 'int' } }
 
 ##
@@ -792,6 +1093,22 @@
 { 'command': 'drive-backup', 'data': 'DriveBackup' }
 
 ##
+# @blockdev-backup
+#
+# Start a point-in-time copy of a block device to a new destination.  The
+# status of ongoing blockdev-backup operations can be checked with
+# query-block-jobs where the BlockJobInfo.type field has the value 'backup'.
+# The operation can be stopped before it has completed using the
+# block-job-cancel command.
+#
+# For the arguments, see the documentation of BlockdevBackup.
+#
+# Since 2.3
+##
+{ 'command': 'blockdev-backup', 'data': 'BlockdevBackup' }
+
+
+##
 # @query-named-block-nodes
 #
 # Get the named block driver list
@@ -807,6 +1124,24 @@
 #
 # Start mirroring a block device's writes to a new destination.
 #
+# See DriveMirror for parameter descriptions
+#
+# Returns: nothing on success
+#          If @device is not a valid block device, DeviceNotFound
+#
+# Since 1.3
+##
+{ 'command': 'drive-mirror', 'boxed': true,
+  'data': 'DriveMirror' }
+
+##
+# DriveMirror
+#
+# A set of parameters describing drive mirror setup.
+#
+# @job-id: #optional identifier for the newly-created block job. If
+#          omitted, the device name will be used. (Since 2.7)
+#
 # @device:  the name of the device whose writes should be mirrored.
 #
 # @target: the target of the new image. If the file exists, or if it
@@ -847,18 +1182,142 @@
 # @on-target-error: #optional the action to take on an error on the target,
 #                   default 'report' (no limitations, since this applies to
 #                   a different block device than @device).
-#
-# Returns: nothing on success
-#          If @device is not a valid block device, DeviceNotFound
+# @unmap: #optional Whether to try to unmap target sectors where source has
+#         only zero. If true, and target unallocated sectors will read as zero,
+#         target image sectors will be unmapped; otherwise, zeroes will be
+#         written. Both will result in identical contents.
+#         Default is true. (Since 2.4)
 #
 # Since 1.3
 ##
-{ 'command': 'drive-mirror',
-  'data': { 'device': 'str', 'target': 'str', '*format': 'str',
-            '*node-name': 'str', '*replaces': 'str',
+{ 'struct': 'DriveMirror',
+  'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
+            '*format': 'str', '*node-name': 'str', '*replaces': 'str',
             'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
             '*speed': 'int', '*granularity': 'uint32',
             '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
+            '*on-target-error': 'BlockdevOnError',
+            '*unmap': 'bool' } }
+
+##
+# @BlockDirtyBitmap
+#
+# @node: name of device/node which the bitmap is tracking
+#
+# @name: name of the dirty bitmap
+#
+# Since 2.4
+##
+{ 'struct': 'BlockDirtyBitmap',
+  'data': { 'node': 'str', 'name': 'str' } }
+
+##
+# @BlockDirtyBitmapAdd
+#
+# @node: name of device/node which the bitmap is tracking
+#
+# @name: name of the dirty bitmap
+#
+# @granularity: #optional the bitmap granularity, default is 64k for
+#               block-dirty-bitmap-add
+#
+# Since 2.4
+##
+{ 'struct': 'BlockDirtyBitmapAdd',
+  'data': { 'node': 'str', 'name': 'str', '*granularity': 'uint32' } }
+
+##
+# @block-dirty-bitmap-add
+#
+# Create a dirty bitmap with a name on the node
+#
+# Returns: nothing on success
+#          If @node is not a valid block device or node, DeviceNotFound
+#          If @name is already taken, GenericError with an explanation
+#
+# Since 2.4
+##
+{ 'command': 'block-dirty-bitmap-add',
+  'data': 'BlockDirtyBitmapAdd' }
+
+##
+# @block-dirty-bitmap-remove
+#
+# Remove a dirty bitmap on the node
+#
+# Returns: nothing on success
+#          If @node is not a valid block device or node, DeviceNotFound
+#          If @name is not found, GenericError with an explanation
+#          if @name is frozen by an operation, GenericError
+#
+# Since 2.4
+##
+{ 'command': 'block-dirty-bitmap-remove',
+  'data': 'BlockDirtyBitmap' }
+
+##
+# @block-dirty-bitmap-clear
+#
+# Clear (reset) a dirty bitmap on the device
+#
+# Returns: nothing on success
+#          If @node is not a valid block device, DeviceNotFound
+#          If @name is not found, GenericError with an explanation
+#
+# Since 2.4
+##
+{ 'command': 'block-dirty-bitmap-clear',
+  'data': 'BlockDirtyBitmap' }
+
+##
+# @blockdev-mirror
+#
+# Start mirroring a block device's writes to a new destination.
+#
+# @job-id: #optional identifier for the newly-created block job. If
+#          omitted, the device name will be used. (Since 2.7)
+#
+# @device: the name of the device whose writes should be mirrored.
+#
+# @target: the id or node-name of the block device to mirror to. This mustn't be
+#          attached to guest.
+#
+# @replaces: #optional with sync=full graph node name to be replaced by the new
+#            image when a whole image copy is done. This can be used to repair
+#            broken Quorum files.
+#
+# @speed:  #optional the maximum speed, in bytes per second
+#
+# @sync: what parts of the disk image should be copied to the destination
+#        (all the disk, only the sectors allocated in the topmost image, or
+#        only new I/O).
+#
+# @granularity: #optional granularity of the dirty bitmap, default is 64K
+#               if the image format doesn't have clusters, 4K if the clusters
+#               are smaller than that, else the cluster size.  Must be a
+#               power of 2 between 512 and 64M
+#
+# @buf-size: #optional maximum amount of data in flight from source to
+#            target
+#
+# @on-source-error: #optional the action to take on an error on the source,
+#                   default 'report'.  'stop' and 'enospc' can only be used
+#                   if the block device supports io-status (see BlockInfo).
+#
+# @on-target-error: #optional the action to take on an error on the target,
+#                   default 'report' (no limitations, since this applies to
+#                   a different block device than @device).
+#
+# Returns: nothing on success.
+#
+# Since 2.6
+##
+{ 'command': 'blockdev-mirror',
+  'data': { '*job-id': 'str', 'device': 'str', 'target': 'str',
+            '*replaces': 'str',
+            'sync': 'MirrorSyncMode',
+            '*speed': 'int', '*granularity': 'uint32',
+            '*buf-size': 'int', '*on-source-error': 'BlockdevOnError',
             '*on-target-error': 'BlockdevOnError' } }
 
 ##
@@ -866,6 +1325,42 @@
 #
 # Change I/O throttle limits for a block drive.
 #
+# Since QEMU 2.4, each device with I/O limits is member of a throttle
+# group.
+#
+# If two or more devices are members of the same group, the limits
+# will apply to the combined I/O of the whole group in a round-robin
+# fashion. Therefore, setting new I/O limits to a device will affect
+# the whole group.
+#
+# The name of the group can be specified using the 'group' parameter.
+# If the parameter is unset, it is assumed to be the current group of
+# that device. If it's not in any group yet, the name of the device
+# will be used as the name for its group.
+#
+# The 'group' parameter can also be used to move a device to a
+# different group. In this case the limits specified in the parameters
+# will be applied to the new group only.
+#
+# I/O limits can be disabled by setting all of them to 0. In this case
+# the device will be removed from its group and the rest of its
+# members will not be affected. The 'group' parameter is ignored.
+#
+# See BlockIOThrottle for parameter descriptions.
+#
+# Returns: Nothing on success
+#          If @device is not a valid block device, DeviceNotFound
+#
+# Since: 1.1
+##
+{ 'command': 'block_set_io_throttle', 'boxed': true,
+  'data': 'BlockIOThrottle' }
+
+##
+# BlockIOThrottle
+#
+# A set of parameters describing block throttling.
+#
 # @device: The name of the device
 #
 # @bps: total throughput limit in bytes per second
@@ -876,36 +1371,74 @@
 #
 # @iops: total I/O operations per second
 #
-# @ops_rd: read I/O operations per second
+# @iops_rd: read I/O operations per second
 #
 # @iops_wr: write I/O operations per second
 #
-# @bps_max: #optional total max in bytes (Since 1.7)
+# @bps_max: #optional total throughput limit during bursts,
+#                     in bytes (Since 1.7)
 #
-# @bps_rd_max: #optional read max in bytes (Since 1.7)
+# @bps_rd_max: #optional read throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @bps_wr_max: #optional write max in bytes (Since 1.7)
+# @bps_wr_max: #optional write throughput limit during bursts,
+#                        in bytes (Since 1.7)
 #
-# @iops_max: #optional total I/O operations max (Since 1.7)
+# @iops_max: #optional total I/O operations per second during bursts,
+#                      in bytes (Since 1.7)
 #
-# @iops_rd_max: #optional read I/O operations max (Since 1.7)
+# @iops_rd_max: #optional read I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
 #
-# @iops_wr_max: #optional write I/O operations max (Since 1.7)
+# @iops_wr_max: #optional write I/O operations per second during bursts,
+#                         in bytes (Since 1.7)
+#
+# @bps_max_length: #optional maximum length of the @bps_max burst
+#                            period, in seconds. It must only
+#                            be set if @bps_max is set as well.
+#                            Defaults to 1. (Since 2.6)
+#
+# @bps_rd_max_length: #optional maximum length of the @bps_rd_max
+#                               burst period, in seconds. It must only
+#                               be set if @bps_rd_max is set as well.
+#                               Defaults to 1. (Since 2.6)
+#
+# @bps_wr_max_length: #optional maximum length of the @bps_wr_max
+#                               burst period, in seconds. It must only
+#                               be set if @bps_wr_max is set as well.
+#                               Defaults to 1. (Since 2.6)
+#
+# @iops_max_length: #optional maximum length of the @iops burst
+#                             period, in seconds. It must only
+#                             be set if @iops_max is set as well.
+#                             Defaults to 1. (Since 2.6)
+#
+# @iops_rd_max_length: #optional maximum length of the @iops_rd_max
+#                                burst period, in seconds. It must only
+#                                be set if @iops_rd_max is set as well.
+#                                Defaults to 1. (Since 2.6)
+#
+# @iops_wr_max_length: #optional maximum length of the @iops_wr_max
+#                                burst period, in seconds. It must only
+#                                be set if @iops_wr_max is set as well.
+#                                Defaults to 1. (Since 2.6)
 #
 # @iops_size: #optional an I/O size in bytes (Since 1.7)
 #
-# Returns: Nothing on success
-#          If @device is not a valid block device, DeviceNotFound
+# @group: #optional throttle group name (Since 2.4)
 #
 # Since: 1.1
 ##
-{ 'command': 'block_set_io_throttle',
+{ 'struct': 'BlockIOThrottle',
   'data': { 'device': 'str', 'bps': 'int', 'bps_rd': 'int', 'bps_wr': 'int',
             'iops': 'int', 'iops_rd': 'int', 'iops_wr': 'int',
             '*bps_max': 'int', '*bps_rd_max': 'int',
             '*bps_wr_max': 'int', '*iops_max': 'int',
             '*iops_rd_max': 'int', '*iops_wr_max': 'int',
-            '*iops_size': 'int' } }
+            '*bps_max_length': 'int', '*bps_rd_max_length': 'int',
+            '*bps_wr_max_length': 'int', '*iops_max_length': 'int',
+            '*iops_rd_max_length': 'int', '*iops_wr_max_length': 'int',
+            '*iops_size': 'int', '*group': 'str' } }
 
 ##
 # @block-stream:
@@ -926,6 +1459,9 @@
 # On successful completion the image file is updated to drop the backing file
 # and the BLOCK_JOB_COMPLETED event is emitted.
 #
+# @job-id: #optional identifier for the newly-created block job. If
+#          omitted, the device name will be used. (Since 2.7)
+#
 # @device: the device name
 #
 # @base:   #optional the common backing file name
@@ -957,8 +1493,9 @@
 # Since: 1.1
 ##
 { 'command': 'block-stream',
-  'data': { 'device': 'str', '*base': 'str', '*backing-file': 'str',
-            '*speed': 'int', '*on-error': 'BlockdevOnError' } }
+  'data': { '*job-id': 'str', 'device': 'str', '*base': 'str',
+            '*backing-file': 'str', '*speed': 'int',
+            '*on-error': 'BlockdevOnError' } }
 
 ##
 # @block-job-set-speed:
@@ -969,7 +1506,9 @@
 #
 # Throttling can be disabled by setting the speed to 0.
 #
-# @device: the device name
+# @device: The job identifier. This used to be a device name (hence
+#          the name of the parameter), but since QEMU 2.7 it can have
+#          other values.
 #
 # @speed:  the maximum speed, in bytes per second, or 0 for unlimited.
 #          Defaults to 0.
@@ -1000,7 +1539,9 @@
 # operation can be started at a later time to finish copying all data from the
 # backing file.
 #
-# @device: the device name
+# @device: The job identifier. This used to be a device name (hence
+#          the name of the parameter), but since QEMU 2.7 it can have
+#          other values.
 #
 # @force: #optional whether to allow cancellation of a paused job (default
 #         false).  Since 1.3.
@@ -1026,7 +1567,9 @@
 # the operation is actually paused.  Cancelling a paused job automatically
 # resumes it.
 #
-# @device: the device name
+# @device: The job identifier. This used to be a device name (hence
+#          the name of the parameter), but since QEMU 2.7 it can have
+#          other values.
 #
 # Returns: Nothing on success
 #          If no background operation is active on this device, DeviceNotActive
@@ -1046,7 +1589,9 @@
 #
 # This command also clears the error status of the job.
 #
-# @device: the device name
+# @device: The job identifier. This used to be a device name (hence
+#          the name of the parameter), but since QEMU 2.7 it can have
+#          other values.
 #
 # Returns: Nothing on success
 #          If no background operation is active on this device, DeviceNotActive
@@ -1072,7 +1617,9 @@
 #
 # A cancelled or paused job cannot be completed.
 #
-# @device: the device name
+# @device: The job identifier. This used to be a device name (hence
+#          the name of the parameter), but since QEMU 2.7 it can have
+#          other values.
 #
 # Returns: Nothing on success
 #          If no background operation is active on this device, DeviceNotActive
@@ -1128,7 +1675,6 @@
 #
 # Includes cache-related options for block devices
 #
-# @writeback:   #optional enables writeback mode for any caches (default: true)
 # @direct:      #optional enables use of O_DIRECT (bypass the host page cache;
 #               default: false)
 # @no-flush:    #optional ignore any flush requests for the device (default:
@@ -1136,9 +1682,8 @@
 #
 # Since: 1.7
 ##
-{ 'type': 'BlockdevCacheOptions',
-  'data': { '*writeback': 'bool',
-            '*direct': 'bool',
+{ 'struct': 'BlockdevCacheOptions',
+  'data': { '*direct': 'bool',
             '*no-flush': 'bool' } }
 
 ##
@@ -1146,53 +1691,17 @@
 #
 # Drivers that are supported in block device operations.
 #
-# @host_device, @host_cdrom, @host_floppy: Since 2.1
+# @host_device, @host_cdrom: Since 2.1
+# @gluster: Since 2.7
 #
 # Since: 2.0
 ##
 { 'enum': 'BlockdevDriver',
   'data': [ 'archipelago', 'blkdebug', 'blkverify', 'bochs', 'cloop',
-            'dmg', 'file', 'ftp', 'ftps', 'host_cdrom', 'host_device',
-            'host_floppy', 'http', 'https', 'null-aio', 'null-co', 'parallels',
-            'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'tftp', 'vdi', 'vhdx',
-            'vmdk', 'vpc', 'vvfat' ] }
-
-##
-# @BlockdevOptionsBase
-#
-# Options that are available for all block devices, independent of the block
-# driver.
-#
-# @driver:        block driver name
-# @id:            #optional id by which the new block device can be referred to.
-#                 This is a required option on the top level of blockdev-add, and
-#                 currently not allowed on any other level.
-# @node-name:     #optional the name of a block driver state node (Since 2.0)
-# @discard:       #optional discard-related options (default: ignore)
-# @cache:         #optional cache-related options
-# @aio:           #optional AIO backend (default: threads)
-# @rerror:        #optional how to handle read errors on the device
-#                 (default: report)
-# @werror:        #optional how to handle write errors on the device
-#                 (default: enospc)
-# @read-only:     #optional whether the block device should be read-only
-#                 (default: false)
-# @detect-zeroes: #optional detect and optimize zero writes (Since 2.1)
-#                 (default: off)
-#
-# Since: 1.7
-##
-{ 'type': 'BlockdevOptionsBase',
-  'data': { 'driver': 'BlockdevDriver',
-            '*id': 'str',
-            '*node-name': 'str',
-            '*discard': 'BlockdevDiscardOptions',
-            '*cache': 'BlockdevCacheOptions',
-            '*aio': 'BlockdevAioOptions',
-            '*rerror': 'BlockdevOnError',
-            '*werror': 'BlockdevOnError',
-            '*read-only': 'bool',
-            '*detect-zeroes': 'BlockdevDetectZeroesOptions' } }
+            'dmg', 'file', 'ftp', 'ftps', 'gluster', 'host_cdrom',
+            'host_device', 'http', 'https', 'luks', 'null-aio', 'null-co',
+            'parallels', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'tftp',
+            'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
 
 ##
 # @BlockdevOptionsFile
@@ -1204,7 +1713,7 @@
 #
 # Since: 1.7
 ##
-{ 'type': 'BlockdevOptionsFile',
+{ 'struct': 'BlockdevOptionsFile',
   'data': { 'filename': 'str' } }
 
 ##
@@ -1213,11 +1722,14 @@
 # Driver specific block device options for the null backend.
 #
 # @size:    #optional size of the device in bytes.
+# @latency-ns: #optional emulated latency (in nanoseconds) in processing
+#              requests. Default to zero which completes requests immediately.
+#              (Since 2.4)
 #
 # Since: 2.2
 ##
-{ 'type': 'BlockdevOptionsNull',
-  'data': { '*size': 'int' } }
+{ 'struct': 'BlockdevOptionsNull',
+  'data': { '*size': 'int', '*latency-ns': 'uint64' } }
 
 ##
 # @BlockdevOptionsVVFAT
@@ -1228,13 +1740,17 @@
 # @fat-type:    #optional FAT type: 12, 16 or 32
 # @floppy:      #optional whether to export a floppy image (true) or
 #               partitioned hard disk (false; default)
+# @label:       #optional set the volume label, limited to 11 bytes. FAT16 and
+#               FAT32 traditionally have some restrictions on labels, which are
+#               ignored by most operating systems. Defaults to "QEMU VVFAT".
+#               (since 2.4)
 # @rw:          #optional whether to allow write operations (default: false)
 #
 # Since: 1.7
 ##
-{ 'type': 'BlockdevOptionsVVFAT',
+{ 'struct': 'BlockdevOptionsVVFAT',
   'data': { 'dir': 'str', '*fat-type': 'int', '*floppy': 'bool',
-            '*rw': 'bool' } }
+            '*label': 'str', '*rw': 'bool' } }
 
 ##
 # @BlockdevOptionsGenericFormat
@@ -1246,10 +1762,26 @@
 #
 # Since: 1.7
 ##
-{ 'type': 'BlockdevOptionsGenericFormat',
+{ 'struct': 'BlockdevOptionsGenericFormat',
   'data': { 'file': 'BlockdevRef' } }
 
 ##
+# @BlockdevOptionsLUKS
+#
+# Driver specific block device options for LUKS.
+#
+# @key-secret: #optional the ID of a QCryptoSecret object providing
+#              the decryption key (since 2.6). Mandatory except when
+#              doing a metadata-only probe of the image.
+#
+# Since: 2.6
+##
+{ 'struct': 'BlockdevOptionsLUKS',
+  'base': 'BlockdevOptionsGenericFormat',
+  'data': { '*key-secret': 'str' } }
+
+
+##
 # @BlockdevOptionsGenericCOWFormat
 #
 # Driver specific block device options for image format that have no option
@@ -1262,7 +1794,7 @@
 #
 # Since: 1.7
 ##
-{ 'type': 'BlockdevOptionsGenericCOWFormat',
+{ 'struct': 'BlockdevOptionsGenericCOWFormat',
   'base': 'BlockdevOptionsGenericFormat',
   'data': { '*backing': 'BlockdevRef' } }
 
@@ -1298,7 +1830,7 @@
 #
 # Since: 2.2
 ##
-{ 'type': 'Qcow2OverlapCheckFlags',
+{ 'struct': 'Qcow2OverlapCheckFlags',
   'data': { '*template':       'Qcow2OverlapCheckMode',
             '*main-header':    'bool',
             '*active-l1':      'bool',
@@ -1322,8 +1854,7 @@
 #
 # Since: 2.2
 ##
-{ 'union': 'Qcow2OverlapChecks',
-  'discriminator': {},
+{ 'alternate': 'Qcow2OverlapChecks',
   'data': { 'flags': 'Qcow2OverlapCheckFlags',
             'mode':  'Qcow2OverlapCheckMode' } }
 
@@ -1358,9 +1889,13 @@
 # @refcount-cache-size:   #optional the maximum size of the refcount block cache
 #                         in bytes (since 2.2)
 #
+# @cache-clean-interval:  #optional clean unused entries in the L2 and refcount
+#                         caches. The interval is in seconds. The default value
+#                         is 0 and it disables this feature (since 2.5)
+#
 # Since: 1.7
 ##
-{ 'type': 'BlockdevOptionsQcow2',
+{ 'struct': 'BlockdevOptionsQcow2',
   'base': 'BlockdevOptionsGenericCOWFormat',
   'data': { '*lazy-refcounts': 'bool',
             '*pass-discard-request': 'bool',
@@ -1369,7 +1904,8 @@
             '*overlap-check': 'Qcow2OverlapChecks',
             '*cache-size': 'int',
             '*l2-cache-size': 'int',
-            '*refcount-cache-size': 'int' } }
+            '*refcount-cache-size': 'int',
+            '*cache-clean-interval': 'int' } }
 
 
 ##
@@ -1395,7 +1931,7 @@
 #                       use the default value, 'archipelago'.
 # Since: 2.2
 ##
-{ 'type': 'BlockdevOptionsArchipelago',
+{ 'struct': 'BlockdevOptionsArchipelago',
   'data': { 'volume': 'str',
             '*mport': 'int',
             '*vport': 'int',
@@ -1406,21 +1942,23 @@
 # @BlkdebugEvent
 #
 # Trigger events supported by blkdebug.
+#
+# Since: 2.0
 ##
-{ 'enum': 'BlkdebugEvent',
-  'data': [ 'l1_update', 'l1_grow.alloc_table', 'l1_grow.write_table',
-            'l1_grow.activate_table', 'l2_load', 'l2_update',
-            'l2_update_compressed', 'l2_alloc.cow_read', 'l2_alloc.write',
+{ 'enum': 'BlkdebugEvent', 'prefix': 'BLKDBG',
+  'data': [ 'l1_update', 'l1_grow_alloc_table', 'l1_grow_write_table',
+            'l1_grow_activate_table', 'l2_load', 'l2_update',
+            'l2_update_compressed', 'l2_alloc_cow_read', 'l2_alloc_write',
             'read_aio', 'read_backing_aio', 'read_compressed', 'write_aio',
             'write_compressed', 'vmstate_load', 'vmstate_save', 'cow_read',
             'cow_write', 'reftable_load', 'reftable_grow', 'reftable_update',
             'refblock_load', 'refblock_update', 'refblock_update_part',
-            'refblock_alloc', 'refblock_alloc.hookup', 'refblock_alloc.write',
-            'refblock_alloc.write_blocks', 'refblock_alloc.write_table',
-            'refblock_alloc.switch_table', 'cluster_alloc',
+            'refblock_alloc', 'refblock_alloc_hookup', 'refblock_alloc_write',
+            'refblock_alloc_write_blocks', 'refblock_alloc_write_table',
+            'refblock_alloc_switch_table', 'cluster_alloc',
             'cluster_alloc_bytes', 'cluster_free', 'flush_to_os',
-            'flush_to_disk', 'pwritev_rmw.head', 'pwritev_rmw.after_head',
-            'pwritev_rmw.tail', 'pwritev_rmw.after_tail', 'pwritev',
+            'flush_to_disk', 'pwritev_rmw_head', 'pwritev_rmw_after_head',
+            'pwritev_rmw_tail', 'pwritev_rmw_after_tail', 'pwritev',
             'pwritev_zero', 'pwritev_done', 'empty_image_prepare' ] }
 
 ##
@@ -1447,7 +1985,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'BlkdebugInjectErrorOptions',
+{ 'struct': 'BlkdebugInjectErrorOptions',
   'data': { 'event': 'BlkdebugEvent',
             '*state': 'int',
             '*errno': 'int',
@@ -1470,7 +2008,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'BlkdebugSetStateOptions',
+{ 'struct': 'BlkdebugSetStateOptions',
   'data': { 'event': 'BlkdebugEvent',
             '*state': 'int',
             'new_state': 'int' } }
@@ -1484,7 +2022,8 @@
 #
 # @config:          #optional filename of the configuration file
 #
-# @align:           #optional required alignment for requests in bytes
+# @align:           #optional required alignment for requests in bytes,
+#                   must be power of 2, or 0 for default
 #
 # @inject-error:    #optional array of error injection descriptions
 #
@@ -1492,7 +2031,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'BlockdevOptionsBlkdebug',
+{ 'struct': 'BlockdevOptionsBlkdebug',
   'data': { 'image': 'BlockdevRef',
             '*config': 'str',
             '*align': 'int',
@@ -1510,7 +2049,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'BlockdevOptionsBlkverify',
+{ 'struct': 'BlockdevOptionsBlkverify',
   'data': { 'test': 'BlockdevRef',
             'raw': 'BlockdevRef' } }
 
@@ -1547,7 +2086,7 @@
 #
 # Since: 2.0
 ##
-{ 'type': 'BlockdevOptionsQuorum',
+{ 'struct': 'BlockdevOptionsQuorum',
   'data': { '*blkverify': 'bool',
             'children': [ 'BlockdevRef' ],
             'vote-threshold': 'int',
@@ -1555,14 +2094,98 @@
             '*read-pattern': 'QuorumReadPattern' } }
 
 ##
+# @GlusterTransport
+#
+# An enumeration of Gluster transport types
+#
+# @tcp:   TCP   - Transmission Control Protocol
+#
+# @unix:  UNIX  - Unix domain socket
+#
+# Since: 2.7
+##
+{ 'enum': 'GlusterTransport',
+  'data': [ 'unix', 'tcp' ] }
+
+
+##
+# @GlusterServer
+#
+# Captures the address of a socket
+#
+# Details for connecting to a gluster server
+#
+# @type:       Transport type used for gluster connection
+#
+# @unix:       socket file
+#
+# @tcp:        host address and port number
+#
+# Since: 2.7
+##
+{ 'union': 'GlusterServer',
+  'base': { 'type': 'GlusterTransport' },
+  'discriminator': 'type',
+  'data': { 'unix': 'UnixSocketAddress',
+            'tcp': 'InetSocketAddress' } }
+
+##
+# @BlockdevOptionsGluster
+#
+# Driver specific block device options for Gluster
+#
+# @volume:      name of gluster volume where VM image resides
+#
+# @path:        absolute path to image file in gluster volume
+#
+# @server:      gluster servers description
+#
+# @debug-level: #optional libgfapi log level (default '4' which is Error)
+#
+# Since: 2.7
+##
+{ 'struct': 'BlockdevOptionsGluster',
+  'data': { 'volume': 'str',
+            'path': 'str',
+            'server': ['GlusterServer'],
+            '*debug-level': 'int' } }
+
+##
 # @BlockdevOptions
 #
-# Options for creating a block device.
+# Options for creating a block device.  Many options are available for all
+# block devices, independent of the block driver:
+#
+# @driver:        block driver name
+# @id:            #optional id by which the new block device can be referred to.
+#                 This option is only allowed on the top level of blockdev-add.
+#                 A BlockBackend will be created by blockdev-add if and only if
+#                 this option is given.
+# @node-name:     #optional the name of a block driver state node (Since 2.0).
+#                 This option is required on the top level of blockdev-add if
+#                 the @id option is not given there.
+# @discard:       #optional discard-related options (default: ignore)
+# @cache:         #optional cache-related options
+# @aio:           #optional AIO backend (default: threads)
+# @read-only:     #optional whether the block device should be read-only
+#                 (default: false)
+# @detect-zeroes: #optional detect and optimize zero writes (Since 2.1)
+#                 (default: off)
+#
+# Remaining options are determined by the block driver.
 #
 # Since: 1.7
 ##
 { 'union': 'BlockdevOptions',
-  'base': 'BlockdevOptionsBase',
+  'base': { 'driver': 'BlockdevDriver',
+# TODO 'id' is a BB-level option, remove it
+            '*id': 'str',
+            '*node-name': 'str',
+            '*discard': 'BlockdevDiscardOptions',
+            '*cache': 'BlockdevCacheOptions',
+            '*aio': 'BlockdevAioOptions',
+            '*read-only': 'bool',
+            '*detect-zeroes': 'BlockdevDetectZeroesOptions' },
   'discriminator': 'driver',
   'data': {
       'archipelago':'BlockdevOptionsArchipelago',
@@ -1574,13 +2197,13 @@
       'file':       'BlockdevOptionsFile',
       'ftp':        'BlockdevOptionsFile',
       'ftps':       'BlockdevOptionsFile',
-# TODO gluster: Wait for structured options
+      'gluster':    'BlockdevOptionsGluster',
       'host_cdrom': 'BlockdevOptionsFile',
       'host_device':'BlockdevOptionsFile',
-      'host_floppy':'BlockdevOptionsFile',
       'http':       'BlockdevOptionsFile',
       'https':      'BlockdevOptionsFile',
 # TODO iscsi: Wait for structured options
+      'luks':       'BlockdevOptionsLUKS',
 # TODO nbd: Should take InetSocketAddress for 'host'?
 # TODO nfs: Wait for structured options
       'null-aio':   'BlockdevOptionsNull',
@@ -1614,15 +2237,20 @@
 #
 # Since: 1.7
 ##
-{ 'union': 'BlockdevRef',
-  'discriminator': {},
+{ 'alternate': 'BlockdevRef',
   'data': { 'definition': 'BlockdevOptions',
             'reference': 'str' } }
 
 ##
 # @blockdev-add:
 #
-# Creates a new block device.
+# Creates a new block device. If the @id option is given at the top level, a
+# BlockBackend will be created; otherwise, @node-name is mandatory at the top
+# level and no BlockBackend will be created.
+#
+# This command is still a work in progress.  It doesn't support all
+# block drivers among other things.  Stay away from it unless you want
+# to help with its development.
 #
 # @options: block device options for the new device
 #
@@ -1630,6 +2258,165 @@
 ##
 { 'command': 'blockdev-add', 'data': { 'options': 'BlockdevOptions' } }
 
+##
+# @x-blockdev-del:
+#
+# Deletes a block device that has been added using blockdev-add.
+# The selected device can be either a block backend or a graph node.
+#
+# In the former case the backend will be destroyed, along with its
+# inserted medium if there's any. The command will fail if the backend
+# or its medium are in use.
+#
+# In the latter case the node will be destroyed. The command will fail
+# if the node is attached to a block backend or is otherwise being
+# used.
+#
+# One of @id or @node-name must be specified, but not both.
+#
+# This command is still a work in progress and is considered
+# experimental. Stay away from it unless you want to help with its
+# development.
+#
+# @id: #optional Name of the block backend device to delete.
+#
+# @node-name: #optional Name of the graph node to delete.
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-del', 'data': { '*id': 'str', '*node-name': 'str' } }
+
+##
+# @blockdev-open-tray:
+#
+# Opens a block device's tray. If there is a block driver state tree inserted as
+# a medium, it will become inaccessible to the guest (but it will remain
+# associated to the block device, so closing the tray will make it accessible
+# again).
+#
+# If the tray was already open before, this will be a no-op.
+#
+# Once the tray opens, a DEVICE_TRAY_MOVED event is emitted. There are cases in
+# which no such event will be generated, these include:
+# - if the guest has locked the tray, @force is false and the guest does not
+#   respond to the eject request
+# - if the BlockBackend denoted by @device does not have a guest device attached
+#   to it
+# - if the guest device does not have an actual tray
+#
+# @device: block device name
+#
+# @force:  #optional if false (the default), an eject request will be sent to
+#          the guest if it has locked the tray (and the tray will not be opened
+#          immediately); if true, the tray will be opened regardless of whether
+#          it is locked
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-open-tray',
+  'data': { 'device': 'str',
+            '*force': 'bool' } }
+
+##
+# @blockdev-close-tray:
+#
+# Closes a block device's tray. If there is a block driver state tree associated
+# with the block device (which is currently ejected), that tree will be loaded
+# as the medium.
+#
+# If the tray was already closed before, this will be a no-op.
+#
+# @device: block device name
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-close-tray',
+  'data': { 'device': 'str' } }
+
+##
+# @x-blockdev-remove-medium:
+#
+# Removes a medium (a block driver state tree) from a block device. That block
+# device's tray must currently be open (unless there is no attached guest
+# device).
+#
+# If the tray is open and there is no medium inserted, this will be a no-op.
+#
+# This command is still a work in progress and is considered experimental.
+# Stay away from it unless you want to help with its development.
+#
+# @device: block device name
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-remove-medium',
+  'data': { 'device': 'str' } }
+
+##
+# @x-blockdev-insert-medium:
+#
+# Inserts a medium (a block driver state tree) into a block device. That block
+# device's tray must currently be open (unless there is no attached guest
+# device) and there must be no medium inserted already.
+#
+# This command is still a work in progress and is considered experimental.
+# Stay away from it unless you want to help with its development.
+#
+# @device:    block device name
+#
+# @node-name: name of a node in the block driver state graph
+#
+# Since: 2.5
+##
+{ 'command': 'x-blockdev-insert-medium',
+  'data': { 'device': 'str',
+            'node-name': 'str'} }
+
+
+##
+# @BlockdevChangeReadOnlyMode:
+#
+# Specifies the new read-only mode of a block device subject to the
+# @blockdev-change-medium command.
+#
+# @retain:      Retains the current read-only mode
+#
+# @read-only:   Makes the device read-only
+#
+# @read-write:  Makes the device writable
+#
+# Since: 2.3
+##
+{ 'enum': 'BlockdevChangeReadOnlyMode',
+  'data': ['retain', 'read-only', 'read-write'] }
+
+
+##
+# @blockdev-change-medium:
+#
+# Changes the medium inserted into a block device by ejecting the current medium
+# and loading a new image file which is inserted as the new medium (this command
+# combines blockdev-open-tray, x-blockdev-remove-medium,
+# x-blockdev-insert-medium and blockdev-close-tray).
+#
+# @device:          block device name
+#
+# @filename:        filename of the new image to be loaded
+#
+# @format:          #optional, format to open the new image with (defaults to
+#                   the probed format)
+#
+# @read-only-mode:  #optional, change the read-only mode of the device; defaults
+#                   to 'retain'
+#
+# Since: 2.5
+##
+{ 'command': 'blockdev-change-medium',
+  'data': { 'device': 'str',
+            'filename': 'str',
+            '*format': 'str',
+            '*read-only-mode': 'BlockdevChangeReadOnlyMode' } }
+
 
 ##
 # @BlockErrorAction
@@ -1653,14 +2440,18 @@
 #
 # Emitted when a corruption has been detected in a disk image
 #
-# @device: device name
+# @device: device name. This is always present for compatibility
+#          reasons, but it can be empty ("") if the image does not
+#          have a device name associated.
+#
+# @node-name: #optional node name (Since: 2.4)
 #
 # @msg: informative message for human consumption, such as the kind of
 #       corruption being detected. It should not be parsed by machine as it is
 #       not guaranteed to be stable
 #
 # @offset: #optional, if the corruption resulted from an image access, this is
-#          the access offset into the image
+#          the host's access offset into the image
 #
 # @size: #optional, if the corruption resulted from an image access, this is
 #        the access size
@@ -1672,11 +2463,12 @@
 # Since: 1.7
 ##
 { 'event': 'BLOCK_IMAGE_CORRUPTED',
-  'data': { 'device' : 'str',
-            'msg'    : 'str',
-            '*offset': 'int',
-            '*size'  : 'int',
-            'fatal'  : 'bool' } }
+  'data': { 'device'     : 'str',
+            '*node-name' : 'str',
+            'msg'        : 'str',
+            '*offset'    : 'int',
+            '*size'      : 'int',
+            'fatal'      : 'bool' } }
 
 ##
 # @BLOCK_IO_ERROR
@@ -1715,7 +2507,8 @@
 #
 # @type: job type
 #
-# @device: device name
+# @device: The job identifier. Originally the device name but other
+#          values are allowed since QEMU 2.7
 #
 # @len: maximum progress value
 #
@@ -1746,7 +2539,8 @@
 #
 # @type: job type
 #
-# @device: device name
+# @device: The job identifier. Originally the device name but other
+#          values are allowed since QEMU 2.7
 #
 # @len: maximum progress value
 #
@@ -1769,7 +2563,8 @@
 #
 # Emitted when a block job encounters an error
 #
-# @device: device name
+# @device: The job identifier. Originally the device name but other
+#          values are allowed since QEMU 2.7
 #
 # @operation: I/O operation
 #
@@ -1789,7 +2584,8 @@
 #
 # @type: job type
 #
-# @device: device name
+# @device: The job identifier. Originally the device name but other
+#          values are allowed since QEMU 2.7
 #
 # @len: maximum progress value
 #
@@ -1826,3 +2622,76 @@
 ##
 { 'enum': 'PreallocMode',
   'data': [ 'off', 'metadata', 'falloc', 'full' ] }
+
+##
+# @BLOCK_WRITE_THRESHOLD
+#
+# Emitted when writes on block device reaches or exceeds the
+# configured write threshold. For thin-provisioned devices, this
+# means the device should be extended to avoid pausing for
+# disk exhaustion.
+# The event is one shot. Once triggered, it needs to be
+# re-registered with another block-set-threshold command.
+#
+# @node-name: graph node name on which the threshold was exceeded.
+#
+# @amount-exceeded: amount of data which exceeded the threshold, in bytes.
+#
+# @write-threshold: last configured threshold, in bytes.
+#
+# Since: 2.3
+##
+{ 'event': 'BLOCK_WRITE_THRESHOLD',
+  'data': { 'node-name': 'str',
+            'amount-exceeded': 'uint64',
+            'write-threshold': 'uint64' } }
+
+##
+# @block-set-write-threshold
+#
+# Change the write threshold for a block drive. An event will be delivered
+# if a write to this block drive crosses the configured threshold.
+# This is useful to transparently resize thin-provisioned drives without
+# the guest OS noticing.
+#
+# @node-name: graph node name on which the threshold must be set.
+#
+# @write-threshold: configured threshold for the block device, bytes.
+#                   Use 0 to disable the threshold.
+#
+# Since: 2.3
+##
+{ 'command': 'block-set-write-threshold',
+  'data': { 'node-name': 'str', 'write-threshold': 'uint64' } }
+
+##
+# @x-blockdev-change
+#
+# Dynamically reconfigure the block driver state graph. It can be used
+# to add, remove, insert or replace a graph node. Currently only the
+# Quorum driver implements this feature to add or remove its child. This
+# is useful to fix a broken quorum child.
+#
+# If @node is specified, it will be inserted under @parent. @child
+# may not be specified in this case. If both @parent and @child are
+# specified but @node is not, @child will be detached from @parent.
+#
+# @parent: the id or name of the parent node.
+#
+# @child: #optional the name of a child under the given parent node.
+#
+# @node: #optional the name of the node that will be added.
+#
+# Note: this command is experimental, and its API is not stable. It
+# does not support all kinds of operations, all kinds of children, nor
+# all block drivers.
+#
+# Warning: The data in a new quorum child MUST be consistent with that of
+# the rest of the array.
+#
+# Since: 2.7
+##
+{ 'command': 'x-blockdev-change',
+  'data' : { 'parent': 'str',
+             '*child': 'str',
+             '*node': 'str' } }
diff --git a/qapi/block.json b/qapi/block.json
index e313465..937337d 100644
--- a/qapi/block.json
+++ b/qapi/block.json
@@ -6,7 +6,7 @@
 { 'include': 'block-core.json' }
 
 ##
-# BiosAtaTranslation:
+# @BiosAtaTranslation:
 #
 # Policy that BIOS should use to interpret cylinder/head/sector
 # addresses.  Note that Bochs BIOS and SeaBIOS will not actually
@@ -40,6 +40,22 @@
   'data': ['auto', 'none', 'lba', 'large', 'rechs']}
 
 ##
+# @FloppyDriveType
+#
+# Type of Floppy drive to be emulated by the Floppy Disk Controller.
+#
+# @144:  1.44MB 3.5" drive
+# @288:  2.88MB 3.5" drive
+# @120:  1.2MB 5.25" drive
+# @none: No drive connected
+# @auto: Automatically determined by inserted media at boot
+#
+# Since: 2.6
+##
+{ 'enum': 'FloppyDriveType',
+  'data': ['144', '288', '120', 'none', 'auto']}
+
+##
 # @BlockdevSnapshotInternal
 #
 # @device: the name of the device to generate the snapshot from
@@ -52,7 +68,7 @@
 #
 # Since: 1.7
 ##
-{ 'type': 'BlockdevSnapshotInternal',
+{ 'struct': 'BlockdevSnapshotInternal',
   'data': { 'device': 'str', 'name': 'str' } }
 
 ##
@@ -130,13 +146,15 @@
 # QEMU instance could refer to them as "nbd:HOST:PORT:exportname=NAME".
 #
 # @addr: Address on which to listen.
+# @tls-creds: (optional) ID of the TLS credentials object. Since 2.6
 #
 # Returns: error if the server is already running.
 #
 # Since: 1.3.0
 ##
 { 'command': 'nbd-server-start',
-  'data': { 'addr': 'SocketAddress' } }
+  'data': { 'addr': 'SocketAddress',
+            '*tls-creds': 'str'} }
 
 ##
 # @nbd-server-add:
@@ -178,3 +196,19 @@
 ##
 { 'event': 'DEVICE_TRAY_MOVED',
   'data': { 'device': 'str', 'tray-open': 'bool' } }
+
+##
+# @QuorumOpType
+#
+# An enumeration of the quorum operation types
+#
+# @read: read operation
+#
+# @write: write operation
+#
+# @flush: flush operation
+#
+# Since: 2.6
+##
+{ 'enum': 'QuorumOpType',
+  'data': [ 'read', 'write', 'flush' ] }
diff --git a/qapi/common.json b/qapi/common.json
index 63ef3b4..9353a7b 100644
--- a/qapi/common.json
+++ b/qapi/common.json
@@ -3,7 +3,7 @@
 # QAPI common definitions
 
 ##
-# @ErrorClass
+# @QapiErrorClass
 #
 # QEMU error classes
 #
@@ -24,20 +24,34 @@
 #
 # Since: 1.2
 ##
-{ 'enum': 'ErrorClass',
+{ 'enum': 'QapiErrorClass',
+  # Keep this in sync with ErrorClass in error.h
   'data': [ 'GenericError', 'CommandNotFound', 'DeviceEncrypted',
             'DeviceNotActive', 'DeviceNotFound', 'KVMMissingCap' ] }
 
 ##
+# @VersionTriple
+#
+# A three-part version number.
+#
+# @qemu.major:  The major version number.
+#
+# @qemu.minor:  The minor version number.
+#
+# @qemu.micro:  The micro version number.
+#
+# Since: 2.4
+##
+{ 'struct': 'VersionTriple',
+  'data': {'major': 'int', 'minor': 'int', 'micro': 'int'} }
+
+
+##
 # @VersionInfo:
 #
 # A description of QEMU's version.
 #
-# @qemu.major:  The major version of QEMU
-#
-# @qemu.minor:  The minor version of QEMU
-#
-# @qemu.micro:  The micro version of QEMU.  By current convention, a micro
+# @qemu:        The version of QEMU.  By current convention, a micro
 #               version of 50 signifies a development branch.  A micro version
 #               greater than or equal to 90 signifies a release candidate for
 #               the next minor version.  A micro version of less than 50
@@ -50,9 +64,8 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'VersionInfo',
-  'data': {'qemu': {'major': 'int', 'minor': 'int', 'micro': 'int'},
-           'package': 'str'} }
+{ 'struct': 'VersionInfo',
+  'data': {'qemu': 'VersionTriple', 'package': 'str'} }
 
 ##
 # @query-version:
@@ -74,7 +87,7 @@
 #
 # Since: 0.14.0
 ##
-{ 'type': 'CommandInfo', 'data': {'name': 'str'} }
+{ 'struct': 'CommandInfo', 'data': {'name': 'str'} }
 
 ##
 # @query-commands:
@@ -102,3 +115,19 @@
 ##
 { 'enum': 'OnOffAuto',
   'data': [ 'auto', 'on', 'off' ] }
+
+##
+# @OnOffSplit
+#
+# An enumeration of three values: on, off, and split
+#
+# @on: Enabled
+#
+# @off: Disabled
+#
+# @split: Mixed
+#
+# Since: 2.6
+##
+{ 'enum': 'OnOffSplit',
+  'data': [ 'on', 'off', 'split' ] }
diff --git a/qapi/crypto.json b/qapi/crypto.json
new file mode 100644
index 0000000..34d2583
--- /dev/null
+++ b/qapi/crypto.json
@@ -0,0 +1,313 @@
+# -*- Mode: Python -*-
+#
+# QAPI crypto definitions
+
+##
+# QCryptoTLSCredsEndpoint:
+#
+# The type of network endpoint that will be using the credentials.
+# Most types of credential require different setup / structures
+# depending on whether they will be used in a server versus a
+# client.
+#
+# @client: the network endpoint is acting as the client
+#
+# @server: the network endpoint is acting as the server
+#
+# Since: 2.5
+##
+{ 'enum': 'QCryptoTLSCredsEndpoint',
+  'prefix': 'QCRYPTO_TLS_CREDS_ENDPOINT',
+  'data': ['client', 'server']}
+
+
+##
+# QCryptoSecretFormat:
+#
+# The data format that the secret is provided in
+#
+# @raw: raw bytes. When encoded in JSON only valid UTF-8 sequences can be used
+# @base64: arbitrary base64 encoded binary data
+# Since: 2.6
+##
+{ 'enum': 'QCryptoSecretFormat',
+  'prefix': 'QCRYPTO_SECRET_FORMAT',
+  'data': ['raw', 'base64']}
+
+
+##
+# QCryptoHashAlgorithm:
+#
+# The supported algorithms for computing content digests
+#
+# @md5: MD5. Should not be used in any new code, legacy compat only
+# @sha1: SHA-1. Should not be used in any new code, legacy compat only
+# @sha224: SHA-224. (since 2.7)
+# @sha256: SHA-256. Current recommended strong hash.
+# @sha384: SHA-384. (since 2.7)
+# @sha512: SHA-512. (since 2.7)
+# @ripemd160: RIPEMD-160. (since 2.7)
+# Since: 2.6
+##
+{ 'enum': 'QCryptoHashAlgorithm',
+  'prefix': 'QCRYPTO_HASH_ALG',
+  'data': ['md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'ripemd160']}
+
+
+##
+# QCryptoCipherAlgorithm:
+#
+# The supported algorithms for content encryption ciphers
+#
+# @aes-128: AES with 128 bit / 16 byte keys
+# @aes-192: AES with 192 bit / 24 byte keys
+# @aes-256: AES with 256 bit / 32 byte keys
+# @des-rfb: RFB specific variant of single DES. Do not use except in VNC.
+# @cast5-128: Cast5 with 128 bit / 16 byte keys
+# @serpent-128: Serpent with 128 bit / 16 byte keys
+# @serpent-192: Serpent with 192 bit / 24 byte keys
+# @serpent-256: Serpent with 256 bit / 32 byte keys
+# @twofish-128: Twofish with 128 bit / 16 byte keys
+# @twofish-192: Twofish with 192 bit / 24 byte keys
+# @twofish-256: Twofish with 256 bit / 32 byte keys
+# Since: 2.6
+##
+{ 'enum': 'QCryptoCipherAlgorithm',
+  'prefix': 'QCRYPTO_CIPHER_ALG',
+  'data': ['aes-128', 'aes-192', 'aes-256',
+           'des-rfb',
+           'cast5-128',
+           'serpent-128', 'serpent-192', 'serpent-256',
+           'twofish-128', 'twofish-192', 'twofish-256']}
+
+
+##
+# QCryptoCipherMode:
+#
+# The supported modes for content encryption ciphers
+#
+# @ecb: Electronic Code Book
+# @cbc: Cipher Block Chaining
+# @xts: XEX with tweaked code book and ciphertext stealing
+# Since: 2.6
+##
+{ 'enum': 'QCryptoCipherMode',
+  'prefix': 'QCRYPTO_CIPHER_MODE',
+  'data': ['ecb', 'cbc', 'xts']}
+
+
+##
+# QCryptoIVGenAlgorithm:
+#
+# The supported algorithms for generating initialization
+# vectors for full disk encryption. The 'plain' generator
+# should not be used for disks with sector numbers larger
+# than 2^32, except where compatibility with pre-existing
+# Linux dm-crypt volumes is required.
+#
+# @plain: 64-bit sector number truncated to 32-bits
+# @plain64: 64-bit sector number
+# @essiv: 64-bit sector number encrypted with a hash of the encryption key
+# Since: 2.6
+##
+{ 'enum': 'QCryptoIVGenAlgorithm',
+  'prefix': 'QCRYPTO_IVGEN_ALG',
+  'data': ['plain', 'plain64', 'essiv']}
+
+##
+# QCryptoBlockFormat:
+#
+# The supported full disk encryption formats
+#
+# @qcow: QCow/QCow2 built-in AES-CBC encryption. Use only
+#        for liberating data from old images.
+# @luks: LUKS encryption format. Recommended for new images
+#
+# Since: 2.6
+##
+{ 'enum': 'QCryptoBlockFormat',
+#  'prefix': 'QCRYPTO_BLOCK_FORMAT',
+  'data': ['qcow', 'luks']}
+
+##
+# QCryptoBlockOptionsBase:
+#
+# The common options that apply to all full disk
+# encryption formats
+#
+# @format: the encryption format
+#
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockOptionsBase',
+  'data': { 'format': 'QCryptoBlockFormat' }}
+
+##
+# QCryptoBlockOptionsQCow:
+#
+# The options that apply to QCow/QCow2 AES-CBC encryption format
+#
+# @key-secret: #optional the ID of a QCryptoSecret object providing the
+#              decryption key. Mandatory except when probing image for
+#              metadata only.
+#
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockOptionsQCow',
+  'data': { '*key-secret': 'str' }}
+
+##
+# QCryptoBlockOptionsLUKS:
+#
+# The options that apply to LUKS encryption format
+#
+# @key-secret: #optional the ID of a QCryptoSecret object providing the
+#              decryption key. Mandatory except when probing image for
+#              metadata only.
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockOptionsLUKS',
+  'data': { '*key-secret': 'str' }}
+
+
+##
+# QCryptoBlockCreateOptionsLUKS:
+#
+# The options that apply to LUKS encryption format initialization
+#
+# @cipher-alg: #optional the cipher algorithm for data encryption
+#              Currently defaults to 'aes'.
+# @cipher-mode: #optional the cipher mode for data encryption
+#               Currently defaults to 'cbc'
+# @ivgen-alg: #optional the initialization vector generator
+#             Currently defaults to 'essiv'
+# @ivgen-hash-alg: #optional the initialization vector generator hash
+#                  Currently defaults to 'sha256'
+# @hash-alg: #optional the master key hash algorithm
+#            Currently defaults to 'sha256'
+# Since: 2.6
+##
+{ 'struct': 'QCryptoBlockCreateOptionsLUKS',
+  'base': 'QCryptoBlockOptionsLUKS',
+  'data': { '*cipher-alg': 'QCryptoCipherAlgorithm',
+            '*cipher-mode': 'QCryptoCipherMode',
+            '*ivgen-alg': 'QCryptoIVGenAlgorithm',
+            '*ivgen-hash-alg': 'QCryptoHashAlgorithm',
+            '*hash-alg': 'QCryptoHashAlgorithm'}}
+
+
+##
+# QCryptoBlockOpenOptions:
+#
+# The options that are available for all encryption formats
+# when opening an existing volume
+#
+# Since: 2.6
+##
+{ 'union': 'QCryptoBlockOpenOptions',
+  'base': 'QCryptoBlockOptionsBase',
+  'discriminator': 'format',
+  'data': { 'qcow': 'QCryptoBlockOptionsQCow',
+            'luks': 'QCryptoBlockOptionsLUKS' } }
+
+
+##
+# QCryptoBlockCreateOptions:
+#
+# The options that are available for all encryption formats
+# when initializing a new volume
+#
+# Since: 2.6
+##
+{ 'union': 'QCryptoBlockCreateOptions',
+  'base': 'QCryptoBlockOptionsBase',
+  'discriminator': 'format',
+  'data': { 'qcow': 'QCryptoBlockOptionsQCow',
+            'luks': 'QCryptoBlockCreateOptionsLUKS' } }
+
+
+##
+# QCryptoBlockInfoBase:
+#
+# The common information that applies to all full disk
+# encryption formats
+#
+# @format: the encryption format
+#
+# Since: 2.7
+##
+{ 'struct': 'QCryptoBlockInfoBase',
+  'data': { 'format': 'QCryptoBlockFormat' }}
+
+
+##
+# QCryptoBlockInfoLUKSSlot:
+#
+# Information about the LUKS block encryption key
+# slot options
+#
+# @active: whether the key slot is currently in use
+# @key-offset: offset to the key material in bytes
+# @iters: #optional number of PBKDF2 iterations for key material
+# @stripes: #optional number of stripes for splitting key material
+#
+# Since: 2.7
+##
+{ 'struct': 'QCryptoBlockInfoLUKSSlot',
+  'data': {'active': 'bool',
+           '*iters': 'int',
+           '*stripes': 'int',
+           'key-offset': 'int' } }
+
+
+##
+# QCryptoBlockInfoLUKS:
+#
+# Information about the LUKS block encryption options
+#
+# @cipher-alg: the cipher algorithm for data encryption
+# @cipher-mode: the cipher mode for data encryption
+# @ivgen-alg: the initialization vector generator
+# @ivgen-hash-alg: #optional the initialization vector generator hash
+# @hash-alg: the master key hash algorithm
+# @payload-offset: offset to the payload data in bytes
+# @master-key-iters: number of PBKDF2 iterations for key material
+# @uuid: unique identifier for the volume
+# @slots: information about each key slot
+#
+# Since: 2.7
+##
+{ 'struct': 'QCryptoBlockInfoLUKS',
+  'data': {'cipher-alg': 'QCryptoCipherAlgorithm',
+           'cipher-mode': 'QCryptoCipherMode',
+           'ivgen-alg': 'QCryptoIVGenAlgorithm',
+           '*ivgen-hash-alg': 'QCryptoHashAlgorithm',
+           'hash-alg': 'QCryptoHashAlgorithm',
+           'payload-offset': 'int',
+           'master-key-iters': 'int',
+           'uuid': 'str',
+           'slots': [ 'QCryptoBlockInfoLUKSSlot' ] }}
+
+##
+# QCryptoBlockInfoQCow:
+#
+# Information about the QCow block encryption options
+#
+# Since: 2.7
+##
+{ 'struct': 'QCryptoBlockInfoQCow',
+  'data': { }}
+
+
+##
+# QCryptoBlockInfo:
+#
+# Information about the block encryption options
+#
+# Since: 2.7
+##
+{ 'union': 'QCryptoBlockInfo',
+  'base': 'QCryptoBlockInfoBase',
+  'discriminator': 'format',
+  'data': { 'qcow': 'QCryptoBlockInfoQCow',
+            'luks': 'QCryptoBlockInfoLUKS' } }
diff --git a/qapi/event.json b/qapi/event.json
index c51dc49..8642052 100644
--- a/qapi/event.json
+++ b/qapi/event.json
@@ -243,6 +243,31 @@
 { 'event': 'SPICE_MIGRATE_COMPLETED' }
 
 ##
+# @MIGRATION
+#
+# Emitted when a migration event happens
+#
+# @status: @MigrationStatus describing the current migration status.
+#
+# Since: 2.4
+##
+{ 'event': 'MIGRATION',
+  'data': {'status': 'MigrationStatus'}}
+
+##
+# @MIGRATION_PASS
+#
+# Emitted from the source side of a migration at the start of each pass
+# (when it syncs the dirty bitmap)
+#
+# @pass: An incrementing count (starting at 1 on the first pass)
+#
+# Since: 2.6
+##
+{ 'event': 'MIGRATION_PASS',
+  'data': { 'pass': 'int' } }
+
+##
 # @ACPI_DEVICE_OST
 #
 # Emitted when guest executes ACPI _OST method.
@@ -300,6 +325,8 @@
 #
 # Emitted to report a corruption of a Quorum file
 #
+# @type: quorum operation type (Since 2.6)
+#
 # @error: #optional, error message. Only present on failure. This field
 #         contains a human-readable error message. There are no semantics other
 #         than that the block layer reported an error and clients should not
@@ -314,7 +341,7 @@
 # Since: 2.0
 ##
 { 'event': 'QUORUM_REPORT_BAD',
-  'data': { '*error': 'str', 'node-name': 'str',
+  'data': { 'type': 'QuorumOpType', '*error': 'str', 'node-name': 'str',
             'sector-num': 'int', 'sectors-count': 'int' } }
 
 ##
@@ -330,3 +357,33 @@
 ##
 { 'event': 'VSERPORT_CHANGE',
   'data': { 'id': 'str', 'open': 'bool' } }
+
+##
+# @MEM_UNPLUG_ERROR
+#
+# Emitted when memory hot unplug error occurs.
+#
+# @device: device name
+#
+# @msg: Informative message
+#
+# Since: 2.4
+##
+{ 'event': 'MEM_UNPLUG_ERROR',
+  'data': { 'device': 'str', 'msg': 'str' } }
+
+##
+# @DUMP_COMPLETED
+#
+# Emitted when background dump has completed
+#
+# @result: DumpQueryResult type described in qapi-schema.json.
+#
+# @error: #optional human-readable error string that provides
+#         hint on why dump failed. Only presents on failure. The
+#         user should not try to interpret the error string.
+#
+# Since: 2.6
+##
+{ 'event': 'DUMP_COMPLETED' ,
+  'data': { 'result': 'DumpQueryResult', '*error': 'str' } }
diff --git a/qapi/introspect.json b/qapi/introspect.json
new file mode 100644
index 0000000..3fd81fb
--- /dev/null
+++ b/qapi/introspect.json
@@ -0,0 +1,279 @@
+# -*- Mode: Python -*-
+#
+# QAPI/QMP introspection
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.
+# See the COPYING file in the top-level directory.
+
+##
+# @query-qmp-schema
+#
+# Command query-qmp-schema exposes the QMP wire ABI as an array of
+# SchemaInfo.  This lets QMP clients figure out what commands and
+# events are available in this QEMU, and their parameters and results.
+#
+# However, the SchemaInfo can't reflect all the rules and restrictions
+# that apply to QMP.  It's interface introspection (figuring out
+# what's there), not interface specification.  The specification is in
+# the QAPI schema.
+#
+# Furthermore, while we strive to keep the QMP wire format
+# backwards-compatible across qemu versions, the introspection output
+# is not guaranteed to have the same stability.  For example, one
+# version of qemu may list an object member as an optional
+# non-variant, while another lists the same member only through the
+# object's variants; or the type of a member may change from a generic
+# string into a specific enum or from one specific type into an
+# alternate that includes the original type alongside something else.
+#
+# Returns: array of @SchemaInfo, where each element describes an
+# entity in the ABI: command, event, type, ...
+#
+# The order of the various SchemaInfo is unspecified; however, all
+# names are guaranteed to be unique (no name will be duplicated with
+# different meta-types).
+#
+# Note: the QAPI schema is also used to help define *internal*
+# interfaces, by defining QAPI types.  These are not part of the QMP
+# wire ABI, and therefore not returned by this command.
+#
+# Since: 2.5
+##
+{ 'command': 'query-qmp-schema',
+  'returns': [ 'SchemaInfo' ],
+  'gen': false }                # just to simplify qmp_query_json()
+
+##
+# @SchemaMetaType
+#
+# This is a @SchemaInfo's meta type, i.e. the kind of entity it
+# describes.
+#
+# @builtin: a predefined type such as 'int' or 'bool'.
+#
+# @enum: an enumeration type
+#
+# @array: an array type
+#
+# @object: an object type (struct or union)
+#
+# @alternate: an alternate type
+#
+# @command: a QMP command
+#
+# @event: a QMP event
+#
+# Since: 2.5
+##
+{ 'enum': 'SchemaMetaType',
+  'data': [ 'builtin', 'enum', 'array', 'object', 'alternate',
+            'command', 'event' ] }
+
+##
+# @SchemaInfo
+#
+# @name: the entity's name, inherited from @base.
+#        Commands and events have the name defined in the QAPI schema.
+#        Unlike command and event names, type names are not part of
+#        the wire ABI.  Consequently, type names are meaningless
+#        strings here, although they are still guaranteed unique
+#        regardless of @meta-type.
+#
+# All references to other SchemaInfo are by name.
+#
+# @meta-type: the entity's meta type, inherited from @base.
+#
+# Additional members depend on the value of @meta-type.
+#
+# Since: 2.5
+##
+{ 'union': 'SchemaInfo',
+  'base': { 'name': 'str', 'meta-type': 'SchemaMetaType' },
+  'discriminator': 'meta-type',
+  'data': {
+      'builtin': 'SchemaInfoBuiltin',
+      'enum': 'SchemaInfoEnum',
+      'array': 'SchemaInfoArray',
+      'object': 'SchemaInfoObject',
+      'alternate': 'SchemaInfoAlternate',
+      'command': 'SchemaInfoCommand',
+      'event': 'SchemaInfoEvent' } }
+
+##
+# @SchemaInfoBuiltin
+#
+# Additional SchemaInfo members for meta-type 'builtin'.
+#
+# @json-type: the JSON type used for this type on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoBuiltin',
+  'data': { 'json-type': 'JSONType' } }
+
+##
+# @JSONType
+#
+# The four primitive and two structured types according to RFC 7159
+# section 1, plus 'int' (split off 'number'), plus the obvious top
+# type 'value'.
+#
+# Since: 2.5
+##
+{ 'enum': 'JSONType',
+  'data': [ 'string', 'number', 'int', 'boolean', 'null',
+            'object', 'array', 'value' ] }
+
+##
+# @SchemaInfoEnum
+#
+# Additional SchemaInfo members for meta-type 'enum'.
+#
+# @values: the enumeration type's values, in no particular order.
+#
+# Values of this type are JSON string on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoEnum',
+  'data': { 'values': ['str'] } }
+
+##
+# @SchemaInfoArray
+#
+# Additional SchemaInfo members for meta-type 'array'.
+#
+# @element-type: the array type's element type.
+#
+# Values of this type are JSON array on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoArray',
+  'data': { 'element-type': 'str' } }
+
+##
+# @SchemaInfoObject
+#
+# Additional SchemaInfo members for meta-type 'object'.
+#
+# @members: the object type's (non-variant) members, in no particular order.
+#
+# @tag: #optional the name of the member serving as type tag.
+#       An element of @members with this name must exist.
+#
+# @variants: #optional variant members, i.e. additional members that
+#            depend on the type tag's value.  Present exactly when
+#            @tag is present.  The variants are in no particular order,
+#            and may even differ from the order of the values of the
+#            enum type of the @tag.
+#
+# Values of this type are JSON object on the wire.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObject',
+  'data': { 'members': [ 'SchemaInfoObjectMember' ],
+            '*tag': 'str',
+            '*variants': [ 'SchemaInfoObjectVariant' ] } }
+
+##
+# @SchemaInfoObjectMember
+#
+# An object member.
+#
+# @name: the member's name, as defined in the QAPI schema.
+#
+# @type: the name of the member's type.
+#
+# @default: #optional default when used as command parameter.
+#           If absent, the parameter is mandatory.
+#           If present, the value must be null.  The parameter is
+#           optional, and behavior when it's missing is not specified
+#           here.
+#           Future extension: if present and non-null, the parameter
+#           is optional, and defaults to this value.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObjectMember',
+  'data': { 'name': 'str', 'type': 'str', '*default': 'any' } }
+# @default's type must be null or match @type
+
+##
+# @SchemaInfoObjectVariant
+#
+# The variant members for a value of the type tag.
+#
+# @case: a value of the type tag.
+#
+# @type: the name of the object type that provides the variant members
+#        when the type tag has value @case.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoObjectVariant',
+  'data': { 'case': 'str', 'type': 'str' } }
+
+##
+# @SchemaInfoAlternate
+#
+# Additional SchemaInfo members for meta-type 'alternate'.
+#
+# @members: the alternate type's members, in no particular order.
+#           The members' wire encoding is distinct, see
+#           docs/qapi-code-gen.txt section Alternate types.
+#
+# On the wire, this can be any of the members.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoAlternate',
+  'data': { 'members': [ 'SchemaInfoAlternateMember' ] } }
+
+##
+# @SchemaInfoAlternateMember
+#
+# An alternate member.
+#
+# @type: the name of the member's type.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoAlternateMember',
+  'data': { 'type': 'str' } }
+
+##
+# @SchemaInfoCommand
+#
+# Additional SchemaInfo members for meta-type 'command'.
+#
+# @arg-type: the name of the object type that provides the command's
+#            parameters.
+#
+# @ret-type: the name of the command's result type.
+#
+# TODO @success-response (currently irrelevant, because it's QGA, not QMP)
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoCommand',
+  'data': { 'arg-type': 'str', 'ret-type': 'str' } }
+
+##
+# @SchemaInfoEvent
+#
+# Additional SchemaInfo members for meta-type 'event'.
+#
+# @arg-type: the name of the object type that provides the event's
+#            parameters.
+#
+# Since: 2.5
+##
+{ 'struct': 'SchemaInfoEvent',
+  'data': { 'arg-type': 'str' } }
diff --git a/qapi/opts-visitor.c b/qapi/opts-visitor.c
index f2ad6d7..1048bbc 100644
--- a/qapi/opts-visitor.c
+++ b/qapi/opts-visitor.c
@@ -1,7 +1,7 @@
 /*
  * Options Visitor
  *
- * Copyright Red Hat, Inc. 2012, 2013
+ * Copyright Red Hat, Inc. 2012-2016
  *
  * Author: Laszlo Ersek <lersek@redhat.com>
  *
@@ -10,7 +10,9 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/opts-visitor.h"
 #include "qemu/queue.h"
@@ -21,9 +23,8 @@
 enum ListMode
 {
     LM_NONE,             /* not traversing a list of repeated options */
-    LM_STARTED,          /* opts_start_list() succeeded */
 
-    LM_IN_PROGRESS,      /* opts_next_list() has been called.
+    LM_IN_PROGRESS,      /* opts_next_list() ready to be called.
                           *
                           * Generating the next list link will consume the most
                           * recently parsed QemuOpt instance of the repeated
@@ -89,6 +90,12 @@
 };
 
 
+static OptsVisitor *to_ov(Visitor *v)
+{
+    return container_of(v, OptsVisitor, visitor);
+}
+
+
 static void
 destroy_list(gpointer list)
 {
@@ -118,14 +125,14 @@
 
 
 static void
-opts_start_struct(Visitor *v, void **obj, const char *kind,
-                  const char *name, size_t size, Error **errp)
+opts_start_struct(Visitor *v, const char *name, void **obj,
+                  size_t size, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
 
     if (obj) {
-        *obj = g_malloc0(size > 0 ? size : 1);
+        *obj = g_malloc0(size);
     }
     if (ov->depth++ > 0) {
         return;
@@ -150,31 +157,37 @@
 }
 
 
-static gboolean
-ghr_true(gpointer ign_key, gpointer ign_value, gpointer ign_user_data)
+static void
+opts_check_struct(Visitor *v, Error **errp)
 {
-    return TRUE;
+    OptsVisitor *ov = to_ov(v);
+    GHashTableIter iter;
+    GQueue *any;
+
+    if (ov->depth > 0) {
+        return;
+    }
+
+    /* we should have processed all (distinct) QemuOpt instances */
+    g_hash_table_iter_init(&iter, ov->unprocessed_opts);
+    if (g_hash_table_iter_next(&iter, NULL, (void **)&any)) {
+        const QemuOpt *first;
+
+        first = g_queue_peek_head(any);
+        error_setg(errp, QERR_INVALID_PARAMETER, first->name);
+    }
 }
 
 
 static void
-opts_end_struct(Visitor *v, Error **errp)
+opts_end_struct(Visitor *v, void **obj)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
-    GQueue *any;
+    OptsVisitor *ov = to_ov(v);
 
     if (--ov->depth > 0) {
         return;
     }
 
-    /* we should have processed all (distinct) QemuOpt instances */
-    any = g_hash_table_find(ov->unprocessed_opts, &ghr_true, NULL);
-    if (any) {
-        const QemuOpt *first;
-
-        first = g_queue_peek_head(any);
-        error_set(errp, QERR_INVALID_PARAMETER, first->name);
-    }
     g_hash_table_destroy(ov->unprocessed_opts);
     ov->unprocessed_opts = NULL;
     if (ov->fake_id_opt) {
@@ -193,42 +206,40 @@
 
     list = g_hash_table_lookup(ov->unprocessed_opts, name);
     if (!list) {
-        error_set(errp, QERR_MISSING_PARAMETER, name);
+        error_setg(errp, QERR_MISSING_PARAMETER, name);
     }
     return list;
 }
 
 
 static void
-opts_start_list(Visitor *v, const char *name, Error **errp)
+opts_start_list(Visitor *v, const char *name, GenericList **list, size_t size,
+                Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
 
     /* we can't traverse a list in a list */
     assert(ov->list_mode == LM_NONE);
+    /* we don't support visits without a list */
+    assert(list);
     ov->repeated_opts = lookup_distinct(ov, name, errp);
-    if (ov->repeated_opts != NULL) {
-        ov->list_mode = LM_STARTED;
+    if (ov->repeated_opts) {
+        ov->list_mode = LM_IN_PROGRESS;
+        *list = g_malloc0(size);
+    } else {
+        *list = NULL;
     }
 }
 
 
 static GenericList *
-opts_next_list(Visitor *v, GenericList **list, Error **errp)
+opts_next_list(Visitor *v, GenericList *tail, size_t size)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
-    GenericList **link;
+    OptsVisitor *ov = to_ov(v);
 
     switch (ov->list_mode) {
-    case LM_STARTED:
-        ov->list_mode = LM_IN_PROGRESS;
-        link = list;
-        break;
-
     case LM_SIGNED_INTERVAL:
     case LM_UNSIGNED_INTERVAL:
-        link = &(*list)->next;
-
         if (ov->list_mode == LM_SIGNED_INTERVAL) {
             if (ov->range_next.s < ov->range_limit.s) {
                 ++ov->range_next.s;
@@ -249,7 +260,6 @@
             g_hash_table_remove(ov->unprocessed_opts, opt->name);
             return NULL;
         }
-        link = &(*list)->next;
         break;
     }
 
@@ -257,18 +267,17 @@
         abort();
     }
 
-    *link = g_malloc0(sizeof **link);
-    return *link;
+    tail->next = g_malloc0(size);
+    return tail->next;
 }
 
 
 static void
-opts_end_list(Visitor *v, Error **errp)
+opts_end_list(Visitor *v, void **obj)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
 
-    assert(ov->list_mode == LM_STARTED ||
-           ov->list_mode == LM_IN_PROGRESS ||
+    assert(ov->list_mode == LM_IN_PROGRESS ||
            ov->list_mode == LM_SIGNED_INTERVAL ||
            ov->list_mode == LM_UNSIGNED_INTERVAL);
     ov->repeated_opts = NULL;
@@ -305,25 +314,31 @@
 
 
 static void
-opts_type_str(Visitor *v, char **obj, const char *name, Error **errp)
+opts_type_str(Visitor *v, const char *name, char **obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
 
     opt = lookup_scalar(ov, name, errp);
     if (!opt) {
+        *obj = NULL;
         return;
     }
     *obj = g_strdup(opt->str ? opt->str : "");
+    /* Note that we consume a string even if this is called as part of
+     * an enum visit that later fails because the string is not a
+     * valid enum value; this is harmless because tracking what gets
+     * consumed only matters to visit_end_struct() as the final error
+     * check if there were no other failures during the visit.  */
     processed(ov, name);
 }
 
 
 /* mimics qemu-option.c::parse_option_bool() */
 static void
-opts_type_bool(Visitor *v, bool *obj, const char *name, Error **errp)
+opts_type_bool(Visitor *v, const char *name, bool *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
 
     opt = lookup_scalar(ov, name, errp);
@@ -341,8 +356,8 @@
             strcmp(opt->str, "n") == 0) {
             *obj = false;
         } else {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
-                "on|yes|y|off|no|n");
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
+                       "on|yes|y|off|no|n");
             return;
         }
     } else {
@@ -354,9 +369,9 @@
 
 
 static void
-opts_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp)
+opts_type_int64(Visitor *v, const char *name, int64_t *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
     const char *str;
     long long val;
@@ -403,16 +418,16 @@
             }
         }
     }
-    error_set(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
-              (ov->list_mode == LM_NONE) ? "an int64 value" :
-                                           "an int64 value or range");
+    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
+               (ov->list_mode == LM_NONE) ? "an int64 value" :
+                                            "an int64 value or range");
 }
 
 
 static void
-opts_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp)
+opts_type_uint64(Visitor *v, const char *name, uint64_t *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
     const char *str;
     unsigned long long val;
@@ -455,16 +470,16 @@
             }
         }
     }
-    error_set(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
-              (ov->list_mode == LM_NONE) ? "a uint64 value" :
-                                           "a uint64 value or range");
+    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
+               (ov->list_mode == LM_NONE) ? "a uint64 value" :
+                                            "a uint64 value or range");
 }
 
 
 static void
-opts_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
+opts_type_size(Visitor *v, const char *name, uint64_t *obj, Error **errp)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
     const QemuOpt *opt;
     int64_t val;
     char *endptr;
@@ -474,11 +489,11 @@
         return;
     }
 
-    val = strtosz_suffix(opt->str ? opt->str : "", &endptr,
-                         STRTOSZ_DEFSUFFIX_B);
+    val = qemu_strtosz_suffix(opt->str ? opt->str : "", &endptr,
+                         QEMU_STRTOSZ_DEFSUFFIX_B);
     if (val < 0 || *endptr) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
-                  "a size value representible as a non-negative int64");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, opt->name,
+                   "a size value representible as a non-negative int64");
         return;
     }
 
@@ -488,9 +503,9 @@
 
 
 static void
-opts_optional(Visitor *v, bool *present, const char *name, Error **errp)
+opts_optional(Visitor *v, const char *name, bool *present)
 {
-    OptsVisitor *ov = DO_UPCAST(OptsVisitor, visitor, v);
+    OptsVisitor *ov = to_ov(v);
 
     /* we only support a single mandatory scalar field in a list node */
     assert(ov->list_mode == LM_NONE);
@@ -498,50 +513,11 @@
 }
 
 
-OptsVisitor *
-opts_visitor_new(const QemuOpts *opts)
+static void
+opts_free(Visitor *v)
 {
-    OptsVisitor *ov;
+    OptsVisitor *ov = to_ov(v);
 
-    ov = g_malloc0(sizeof *ov);
-
-    ov->visitor.start_struct = &opts_start_struct;
-    ov->visitor.end_struct   = &opts_end_struct;
-
-    ov->visitor.start_list = &opts_start_list;
-    ov->visitor.next_list  = &opts_next_list;
-    ov->visitor.end_list   = &opts_end_list;
-
-    /* input_type_enum() covers both "normal" enums and union discriminators.
-     * The union discriminator field is always generated as "type"; it should
-     * match the "type" QemuOpt child of any QemuOpts.
-     *
-     * input_type_enum() will remove the looked-up key from the
-     * "unprocessed_opts" hash even if the lookup fails, because the removal is
-     * done earlier in opts_type_str(). This should be harmless.
-     */
-    ov->visitor.type_enum = &input_type_enum;
-
-    ov->visitor.type_int    = &opts_type_int;
-    ov->visitor.type_uint64 = &opts_type_uint64;
-    ov->visitor.type_size   = &opts_type_size;
-    ov->visitor.type_bool   = &opts_type_bool;
-    ov->visitor.type_str    = &opts_type_str;
-
-    /* type_number() is not filled in, but this is not the first visitor to
-     * skip some mandatory methods... */
-
-    ov->visitor.optional = &opts_optional;
-
-    ov->opts_root = opts;
-
-    return ov;
-}
-
-
-void
-opts_visitor_cleanup(OptsVisitor *ov)
-{
     if (ov->unprocessed_opts != NULL) {
         g_hash_table_destroy(ov->unprocessed_opts);
     }
@@ -551,7 +527,35 @@
 
 
 Visitor *
-opts_get_visitor(OptsVisitor *ov)
+opts_visitor_new(const QemuOpts *opts)
 {
+    OptsVisitor *ov;
+
+    ov = g_malloc0(sizeof *ov);
+
+    ov->visitor.type = VISITOR_INPUT;
+
+    ov->visitor.start_struct = &opts_start_struct;
+    ov->visitor.check_struct = &opts_check_struct;
+    ov->visitor.end_struct   = &opts_end_struct;
+
+    ov->visitor.start_list = &opts_start_list;
+    ov->visitor.next_list  = &opts_next_list;
+    ov->visitor.end_list   = &opts_end_list;
+
+    ov->visitor.type_int64  = &opts_type_int64;
+    ov->visitor.type_uint64 = &opts_type_uint64;
+    ov->visitor.type_size   = &opts_type_size;
+    ov->visitor.type_bool   = &opts_type_bool;
+    ov->visitor.type_str    = &opts_type_str;
+
+    /* type_number() is not filled in, but this is not the first visitor to
+     * skip some mandatory methods... */
+
+    ov->visitor.optional = &opts_optional;
+    ov->visitor.free = opts_free;
+
+    ov->opts_root = opts;
+
     return &ov->visitor;
 }
diff --git a/qapi/qapi-clone-visitor.c b/qapi/qapi-clone-visitor.c
new file mode 100644
index 0000000..0bb8216
--- /dev/null
+++ b/qapi/qapi-clone-visitor.c
@@ -0,0 +1,182 @@
+/*
+ * Copy one QAPI object to another
+ *
+ * Copyright (C) 2016 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/clone-visitor.h"
+#include "qapi/visitor-impl.h"
+#include "qapi/error.h"
+
+struct QapiCloneVisitor {
+    Visitor visitor;
+    size_t depth;
+};
+
+static QapiCloneVisitor *to_qcv(Visitor *v)
+{
+    return container_of(v, QapiCloneVisitor, visitor);
+}
+
+static void qapi_clone_start_struct(Visitor *v, const char *name, void **obj,
+                                    size_t size, Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    if (!obj) {
+        assert(qcv->depth);
+        /* Only possible when visiting an alternate's object
+         * branch. Nothing further to do here, since the earlier
+         * visit_start_alternate() already copied memory. */
+        return;
+    }
+
+    *obj = g_memdup(*obj, size);
+    qcv->depth++;
+}
+
+static void qapi_clone_end(Visitor *v, void **obj)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    if (obj) {
+        qcv->depth--;
+    }
+}
+
+static void qapi_clone_start_list(Visitor *v, const char *name,
+                                  GenericList **listp, size_t size,
+                                  Error **errp)
+{
+    qapi_clone_start_struct(v, name, (void **)listp, size, errp);
+}
+
+static GenericList *qapi_clone_next_list(Visitor *v, GenericList *tail,
+                                         size_t size)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /* Unshare the tail of the list cloned by g_memdup() */
+    tail->next = g_memdup(tail->next, size);
+    return tail->next;
+}
+
+static void qapi_clone_start_alternate(Visitor *v, const char *name,
+                                       GenericAlternate **obj, size_t size,
+                                       bool promote_int, Error **errp)
+{
+    qapi_clone_start_struct(v, name, (void **)obj, size, errp);
+}
+
+static void qapi_clone_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                   Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /* Value was already cloned by g_memdup() */
+}
+
+static void qapi_clone_type_uint64(Visitor *v, const char *name,
+                                    uint64_t *obj, Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /* Value was already cloned by g_memdup() */
+}
+
+static void qapi_clone_type_bool(Visitor *v, const char *name, bool *obj,
+                                  Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /* Value was already cloned by g_memdup() */
+}
+
+static void qapi_clone_type_str(Visitor *v, const char *name, char **obj,
+                                 Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /*
+     * Pointer was already cloned by g_memdup; create fresh copy.
+     * Note that as long as qmp-output-visitor accepts NULL instead of
+     * "", then we must do likewise. However, we want to obey the
+     * input visitor semantics of never producing NULL when the empty
+     * string is intended.
+     */
+    *obj = g_strdup(*obj ?: "");
+}
+
+static void qapi_clone_type_number(Visitor *v, const char *name, double *obj,
+                                    Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /* Value was already cloned by g_memdup() */
+}
+
+static void qapi_clone_type_null(Visitor *v, const char *name, Error **errp)
+{
+    QapiCloneVisitor *qcv = to_qcv(v);
+
+    assert(qcv->depth);
+    /* Nothing to do */
+}
+
+static void qapi_clone_free(Visitor *v)
+{
+    g_free(v);
+}
+
+static Visitor *qapi_clone_visitor_new(void)
+{
+    QapiCloneVisitor *v;
+
+    v = g_malloc0(sizeof(*v));
+
+    v->visitor.type = VISITOR_CLONE;
+    v->visitor.start_struct = qapi_clone_start_struct;
+    v->visitor.end_struct = qapi_clone_end;
+    v->visitor.start_list = qapi_clone_start_list;
+    v->visitor.next_list = qapi_clone_next_list;
+    v->visitor.end_list = qapi_clone_end;
+    v->visitor.start_alternate = qapi_clone_start_alternate;
+    v->visitor.end_alternate = qapi_clone_end;
+    v->visitor.type_int64 = qapi_clone_type_int64;
+    v->visitor.type_uint64 = qapi_clone_type_uint64;
+    v->visitor.type_bool = qapi_clone_type_bool;
+    v->visitor.type_str = qapi_clone_type_str;
+    v->visitor.type_number = qapi_clone_type_number;
+    v->visitor.type_null = qapi_clone_type_null;
+    v->visitor.free = qapi_clone_free;
+
+    return &v->visitor;
+}
+
+void *qapi_clone(const void *src, void (*visit_type)(Visitor *, const char *,
+                                                     void **, Error **))
+{
+    Visitor *v;
+    void *dst = (void *) src; /* Cast away const */
+
+    if (!src) {
+        return NULL;
+    }
+
+    v = qapi_clone_visitor_new();
+    visit_type(v, NULL, &dst, &error_abort);
+    visit_free(v);
+    return dst;
+}
diff --git a/qapi/qapi-dealloc-visitor.c b/qapi/qapi-dealloc-visitor.c
index d875611..e39457b 100644
--- a/qapi/qapi-dealloc-visitor.c
+++ b/qapi/qapi-dealloc-visitor.c
@@ -1,6 +1,7 @@
 /*
  * Dealloc Visitor
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,125 +12,62 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/dealloc-visitor.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
 #include "qapi/qmp/types.h"
 #include "qapi/visitor-impl.h"
 
-typedef struct StackEntry
-{
-    void *value;
-    bool is_list_head;
-    QTAILQ_ENTRY(StackEntry) node;
-} StackEntry;
-
 struct QapiDeallocVisitor
 {
     Visitor visitor;
-    QTAILQ_HEAD(, StackEntry) stack;
-    bool is_list_head;
 };
 
-static QapiDeallocVisitor *to_qov(Visitor *v)
+static void qapi_dealloc_start_struct(Visitor *v, const char *name, void **obj,
+                                      size_t unused, Error **errp)
 {
-    return container_of(v, QapiDeallocVisitor, visitor);
 }
 
-static void qapi_dealloc_push(QapiDeallocVisitor *qov, void *value)
+static void qapi_dealloc_end_struct(Visitor *v, void **obj)
 {
-    StackEntry *e = g_malloc0(sizeof(*e));
-
-    e->value = value;
-
-    /* see if we're just pushing a list head tracker */
-    if (value == NULL) {
-        e->is_list_head = true;
-    }
-    QTAILQ_INSERT_HEAD(&qov->stack, e, node);
-}
-
-static void *qapi_dealloc_pop(QapiDeallocVisitor *qov)
-{
-    StackEntry *e = QTAILQ_FIRST(&qov->stack);
-    QObject *value;
-    QTAILQ_REMOVE(&qov->stack, e, node);
-    value = e->value;
-    g_free(e);
-    return value;
-}
-
-static void qapi_dealloc_start_struct(Visitor *v, void **obj, const char *kind,
-                                      const char *name, size_t unused,
-                                      Error **errp)
-{
-    QapiDeallocVisitor *qov = to_qov(v);
-    qapi_dealloc_push(qov, obj);
-}
-
-static void qapi_dealloc_end_struct(Visitor *v, Error **errp)
-{
-    QapiDeallocVisitor *qov = to_qov(v);
-    void **obj = qapi_dealloc_pop(qov);
     if (obj) {
         g_free(*obj);
     }
 }
 
-static void qapi_dealloc_start_implicit_struct(Visitor *v,
-                                               void **obj,
-                                               size_t size,
-                                               Error **errp)
+static void qapi_dealloc_start_alternate(Visitor *v, const char *name,
+                                         GenericAlternate **obj, size_t size,
+                                         bool promote_int, Error **errp)
 {
-    QapiDeallocVisitor *qov = to_qov(v);
-    qapi_dealloc_push(qov, obj);
 }
 
-static void qapi_dealloc_end_implicit_struct(Visitor *v, Error **errp)
+static void qapi_dealloc_end_alternate(Visitor *v, void **obj)
 {
-    QapiDeallocVisitor *qov = to_qov(v);
-    void **obj = qapi_dealloc_pop(qov);
     if (obj) {
         g_free(*obj);
     }
 }
 
-static void qapi_dealloc_start_list(Visitor *v, const char *name, Error **errp)
+static void qapi_dealloc_start_list(Visitor *v, const char *name,
+                                    GenericList **list, size_t size,
+                                    Error **errp)
 {
-    QapiDeallocVisitor *qov = to_qov(v);
-    qapi_dealloc_push(qov, NULL);
 }
 
-static GenericList *qapi_dealloc_next_list(Visitor *v, GenericList **listp,
-                                           Error **errp)
+static GenericList *qapi_dealloc_next_list(Visitor *v, GenericList *tail,
+                                           size_t size)
 {
-    GenericList *list = *listp;
-    QapiDeallocVisitor *qov = to_qov(v);
-    StackEntry *e = QTAILQ_FIRST(&qov->stack);
-
-    if (e && e->is_list_head) {
-        e->is_list_head = false;
-        return list;
-    }
-
-    if (list) {
-        list = list->next;
-        g_free(*listp);
-        return list;
-    }
-
-    return NULL;
+    GenericList *next = tail->next;
+    g_free(tail);
+    return next;
 }
 
-static void qapi_dealloc_end_list(Visitor *v, Error **errp)
+static void qapi_dealloc_end_list(Visitor *v, void **obj)
 {
-    QapiDeallocVisitor *qov = to_qov(v);
-    void *obj = qapi_dealloc_pop(qov);
-    assert(obj == NULL); /* should've been list head tracker with no payload */
-    (void)obj;
 }
 
-static void qapi_dealloc_type_str(Visitor *v, char **obj, const char *name,
+static void qapi_dealloc_type_str(Visitor *v, const char *name, char **obj,
                                   Error **errp)
 {
     if (obj) {
@@ -137,89 +75,65 @@
     }
 }
 
-static void qapi_dealloc_type_int(Visitor *v, int64_t *obj, const char *name,
-                                  Error **errp)
+static void qapi_dealloc_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                    Error **errp)
 {
 }
 
-static void qapi_dealloc_type_bool(Visitor *v, bool *obj, const char *name,
+static void qapi_dealloc_type_uint64(Visitor *v, const char *name,
+                                     uint64_t *obj, Error **errp)
+{
+}
+
+static void qapi_dealloc_type_bool(Visitor *v, const char *name, bool *obj,
                                    Error **errp)
 {
 }
 
-static void qapi_dealloc_type_number(Visitor *v, double *obj, const char *name,
+static void qapi_dealloc_type_number(Visitor *v, const char *name, double *obj,
                                      Error **errp)
 {
 }
 
-static void qapi_dealloc_type_size(Visitor *v, uint64_t *obj, const char *name,
-                                   Error **errp)
+static void qapi_dealloc_type_anything(Visitor *v, const char *name,
+                                       QObject **obj, Error **errp)
+{
+    if (obj) {
+        qobject_decref(*obj);
+    }
+}
+
+static void qapi_dealloc_type_null(Visitor *v, const char *name, Error **errp)
 {
 }
 
-static void qapi_dealloc_type_enum(Visitor *v, int *obj, const char *strings[],
-                                   const char *kind, const char *name,
-                                   Error **errp)
+static void qapi_dealloc_free(Visitor *v)
 {
+    g_free(container_of(v, QapiDeallocVisitor, visitor));
 }
 
-/* If there's no data present, the dealloc visitor has nothing to free.
- * Thus, indicate to visitor code that the subsequent union fields can
- * be skipped. This is not an error condition, since the cleanup of the
- * rest of an object can continue unhindered, so leave errp unset in
- * these cases.
- *
- * NOTE: In cases where we're attempting to deallocate an object that
- * may have missing fields, the field indicating the union type may
- * be missing. In such a case, it's possible we don't have enough
- * information to differentiate data_present == false from a case where
- * data *is* present but happens to be a scalar with a value of 0.
- * This is okay, since in the case of the dealloc visitor there's no
- * work that needs to done in either situation.
- *
- * The current inability in QAPI code to more thoroughly verify a union
- * type in such cases will likely need to be addressed if we wish to
- * implement this interface for other types of visitors in the future,
- * however.
- */
-static bool qapi_dealloc_start_union(Visitor *v, bool data_present,
-                                     Error **errp)
-{
-    return data_present;
-}
-
-Visitor *qapi_dealloc_get_visitor(QapiDeallocVisitor *v)
-{
-    return &v->visitor;
-}
-
-void qapi_dealloc_visitor_cleanup(QapiDeallocVisitor *v)
-{
-    g_free(v);
-}
-
-QapiDeallocVisitor *qapi_dealloc_visitor_new(void)
+Visitor *qapi_dealloc_visitor_new(void)
 {
     QapiDeallocVisitor *v;
 
     v = g_malloc0(sizeof(*v));
 
+    v->visitor.type = VISITOR_DEALLOC;
     v->visitor.start_struct = qapi_dealloc_start_struct;
     v->visitor.end_struct = qapi_dealloc_end_struct;
-    v->visitor.start_implicit_struct = qapi_dealloc_start_implicit_struct;
-    v->visitor.end_implicit_struct = qapi_dealloc_end_implicit_struct;
+    v->visitor.start_alternate = qapi_dealloc_start_alternate;
+    v->visitor.end_alternate = qapi_dealloc_end_alternate;
     v->visitor.start_list = qapi_dealloc_start_list;
     v->visitor.next_list = qapi_dealloc_next_list;
     v->visitor.end_list = qapi_dealloc_end_list;
-    v->visitor.type_enum = qapi_dealloc_type_enum;
-    v->visitor.type_int = qapi_dealloc_type_int;
+    v->visitor.type_int64 = qapi_dealloc_type_int64;
+    v->visitor.type_uint64 = qapi_dealloc_type_uint64;
     v->visitor.type_bool = qapi_dealloc_type_bool;
     v->visitor.type_str = qapi_dealloc_type_str;
     v->visitor.type_number = qapi_dealloc_type_number;
-    v->visitor.type_size = qapi_dealloc_type_size;
-    v->visitor.start_union = qapi_dealloc_start_union;
+    v->visitor.type_any = qapi_dealloc_type_anything;
+    v->visitor.type_null = qapi_dealloc_type_null;
+    v->visitor.free = qapi_dealloc_free;
 
-    QTAILQ_INIT(&v->stack);
-
-    return v;
+    return &v->visitor;
 }
diff --git a/qapi/qapi-util.c b/qapi/qapi-util.c
index 1d8fb96..818730a 100644
--- a/qapi/qapi-util.c
+++ b/qapi/qapi-util.c
@@ -10,11 +10,12 @@
  *
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
 #include "qapi/error.h"
+#include "qemu-common.h"
 #include "qapi/util.h"
 
-int qapi_enum_parse(const char *lookup[], const char *buf,
+int qapi_enum_parse(const char * const lookup[], const char *buf,
                     int max, int def, Error **errp)
 {
     int i;
diff --git a/qapi/qapi-visit-core.c b/qapi/qapi-visit-core.c
index b66b93a..55f5876 100644
--- a/qapi/qapi-visit-core.c
+++ b/qapi/qapi-visit-core.c
@@ -1,6 +1,7 @@
 /*
  * Core Definitions for QAPI Visitor Classes
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,285 +12,303 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/visitor.h"
 #include "qapi/visitor-impl.h"
 
-void visit_start_struct(Visitor *v, void **obj, const char *kind,
-                        const char *name, size_t size, Error **errp)
+void visit_complete(Visitor *v, void *opaque)
 {
-    v->start_struct(v, obj, kind, name, size, errp);
-}
-
-void visit_end_struct(Visitor *v, Error **errp)
-{
-    v->end_struct(v, errp);
-}
-
-void visit_start_implicit_struct(Visitor *v, void **obj, size_t size,
-                                 Error **errp)
-{
-    if (v->start_implicit_struct) {
-        v->start_implicit_struct(v, obj, size, errp);
+    assert(v->type != VISITOR_OUTPUT || v->complete);
+    if (v->complete) {
+        v->complete(v, opaque);
     }
 }
 
-void visit_end_implicit_struct(Visitor *v, Error **errp)
+void visit_free(Visitor *v)
 {
-    if (v->end_implicit_struct) {
-        v->end_implicit_struct(v, errp);
+    if (v) {
+        v->free(v);
     }
 }
 
-void visit_start_list(Visitor *v, const char *name, Error **errp)
+void visit_start_struct(Visitor *v, const char *name, void **obj,
+                        size_t size, Error **errp)
 {
-    v->start_list(v, name, errp);
-}
+    Error *err = NULL;
 
-GenericList *visit_next_list(Visitor *v, GenericList **list, Error **errp)
-{
-    return v->next_list(v, list, errp);
-}
-
-void visit_end_list(Visitor *v, Error **errp)
-{
-    v->end_list(v, errp);
-}
-
-bool visit_start_union(Visitor *v, bool data_present, Error **errp)
-{
-    if (v->start_union) {
-        return v->start_union(v, data_present, errp);
+    if (obj) {
+        assert(size);
+        assert(!(v->type & VISITOR_OUTPUT) || *obj);
     }
-    return true;
+    v->start_struct(v, name, obj, size, &err);
+    if (obj && (v->type & VISITOR_INPUT)) {
+        assert(!err != !*obj);
+    }
+    error_propagate(errp, err);
 }
 
-void visit_end_union(Visitor *v, bool data_present, Error **errp)
+void visit_check_struct(Visitor *v, Error **errp)
 {
-    if (v->end_union) {
-        v->end_union(v, data_present, errp);
+    if (v->check_struct) {
+        v->check_struct(v, errp);
     }
 }
 
-void visit_optional(Visitor *v, bool *present, const char *name,
-                    Error **errp)
+void visit_end_struct(Visitor *v, void **obj)
+{
+    v->end_struct(v, obj);
+}
+
+void visit_start_list(Visitor *v, const char *name, GenericList **list,
+                      size_t size, Error **errp)
+{
+    Error *err = NULL;
+
+    assert(!list || size >= sizeof(GenericList));
+    v->start_list(v, name, list, size, &err);
+    if (list && (v->type & VISITOR_INPUT)) {
+        assert(!(err && *list));
+    }
+    error_propagate(errp, err);
+}
+
+GenericList *visit_next_list(Visitor *v, GenericList *tail, size_t size)
+{
+    assert(tail && size >= sizeof(GenericList));
+    return v->next_list(v, tail, size);
+}
+
+void visit_end_list(Visitor *v, void **obj)
+{
+    v->end_list(v, obj);
+}
+
+void visit_start_alternate(Visitor *v, const char *name,
+                           GenericAlternate **obj, size_t size,
+                           bool promote_int, Error **errp)
+{
+    Error *err = NULL;
+
+    assert(obj && size >= sizeof(GenericAlternate));
+    assert(!(v->type & VISITOR_OUTPUT) || *obj);
+    if (v->start_alternate) {
+        v->start_alternate(v, name, obj, size, promote_int, &err);
+    }
+    if (v->type & VISITOR_INPUT) {
+        assert(v->start_alternate && !err != !*obj);
+    }
+    error_propagate(errp, err);
+}
+
+void visit_end_alternate(Visitor *v, void **obj)
+{
+    if (v->end_alternate) {
+        v->end_alternate(v, obj);
+    }
+}
+
+bool visit_optional(Visitor *v, const char *name, bool *present)
 {
     if (v->optional) {
-        v->optional(v, present, name, errp);
+        v->optional(v, name, present);
     }
+    return *present;
 }
 
-void visit_get_next_type(Visitor *v, int *obj, const int *qtypes,
-                         const char *name, Error **errp)
+bool visit_is_input(Visitor *v)
 {
-    if (v->get_next_type) {
-        v->get_next_type(v, obj, qtypes, name, errp);
-    }
+    return v->type == VISITOR_INPUT;
 }
 
-void visit_type_enum(Visitor *v, int *obj, const char *strings[],
-                     const char *kind, const char *name, Error **errp)
+void visit_type_int(Visitor *v, const char *name, int64_t *obj, Error **errp)
 {
-    v->type_enum(v, obj, strings, kind, name, errp);
+    assert(obj);
+    v->type_int64(v, name, obj, errp);
 }
 
-void visit_type_int(Visitor *v, int64_t *obj, const char *name, Error **errp)
+static void visit_type_uintN(Visitor *v, uint64_t *obj, const char *name,
+                             uint64_t max, const char *type, Error **errp)
 {
-    v->type_int(v, obj, name, errp);
-}
+    Error *err = NULL;
+    uint64_t value = *obj;
 
-void visit_type_uint8(Visitor *v, uint8_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_uint8) {
-        v->type_uint8(v, obj, name, errp);
+    v->type_uint64(v, name, &value, &err);
+    if (err) {
+        error_propagate(errp, err);
+    } else if (value > max) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   name ? name : "null", type);
     } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < 0 || value > UINT8_MAX) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "uint8_t");
-            return;
-        }
         *obj = value;
     }
 }
 
-void visit_type_uint16(Visitor *v, uint16_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_uint16) {
-        v->type_uint16(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < 0 || value > UINT16_MAX) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "uint16_t");
-            return;
-        }
-        *obj = value;
-    }
-}
-
-void visit_type_uint32(Visitor *v, uint32_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_uint32) {
-        v->type_uint32(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < 0 || value > UINT32_MAX) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "uint32_t");
-            return;
-        }
-        *obj = value;
-    }
-}
-
-void visit_type_uint64(Visitor *v, uint64_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_uint64) {
-        v->type_uint64(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        *obj = value;
-    }
-}
-
-void visit_type_int8(Visitor *v, int8_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_int8) {
-        v->type_int8(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < INT8_MIN || value > INT8_MAX) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "int8_t");
-            return;
-        }
-        *obj = value;
-    }
-}
-
-void visit_type_int16(Visitor *v, int16_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_int16) {
-        v->type_int16(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < INT16_MIN || value > INT16_MAX) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "int16_t");
-            return;
-        }
-        *obj = value;
-    }
-}
-
-void visit_type_int32(Visitor *v, int32_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_int32) {
-        v->type_int32(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        if (value < INT32_MIN || value > INT32_MAX) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
-                      "int32_t");
-            return;
-        }
-        *obj = value;
-    }
-}
-
-void visit_type_int64(Visitor *v, int64_t *obj, const char *name, Error **errp)
-{
-    if (v->type_int64) {
-        v->type_int64(v, obj, name, errp);
-    } else {
-        v->type_int(v, obj, name, errp);
-    }
-}
-
-void visit_type_size(Visitor *v, uint64_t *obj, const char *name, Error **errp)
-{
-    int64_t value;
-
-    if (v->type_size) {
-        v->type_size(v, obj, name, errp);
-    } else if (v->type_uint64) {
-        v->type_uint64(v, obj, name, errp);
-    } else {
-        value = *obj;
-        v->type_int(v, &value, name, errp);
-        *obj = value;
-    }
-}
-
-void visit_type_bool(Visitor *v, bool *obj, const char *name, Error **errp)
-{
-    v->type_bool(v, obj, name, errp);
-}
-
-void visit_type_str(Visitor *v, char **obj, const char *name, Error **errp)
-{
-    v->type_str(v, obj, name, errp);
-}
-
-void visit_type_number(Visitor *v, double *obj, const char *name, Error **errp)
-{
-    v->type_number(v, obj, name, errp);
-}
-
-void output_type_enum(Visitor *v, int *obj, const char *strings[],
-                      const char *kind, const char *name,
+void visit_type_uint8(Visitor *v, const char *name, uint8_t *obj,
                       Error **errp)
 {
+    uint64_t value = *obj;
+    visit_type_uintN(v, &value, name, UINT8_MAX, "uint8_t", errp);
+    *obj = value;
+}
+
+void visit_type_uint16(Visitor *v, const char *name, uint16_t *obj,
+                       Error **errp)
+{
+    uint64_t value = *obj;
+    visit_type_uintN(v, &value, name, UINT16_MAX, "uint16_t", errp);
+    *obj = value;
+}
+
+void visit_type_uint32(Visitor *v, const char *name, uint32_t *obj,
+                       Error **errp)
+{
+    uint64_t value = *obj;
+    visit_type_uintN(v, &value, name, UINT32_MAX, "uint32_t", errp);
+    *obj = value;
+}
+
+void visit_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                       Error **errp)
+{
+    assert(obj);
+    v->type_uint64(v, name, obj, errp);
+}
+
+static void visit_type_intN(Visitor *v, int64_t *obj, const char *name,
+                            int64_t min, int64_t max, const char *type,
+                            Error **errp)
+{
+    Error *err = NULL;
+    int64_t value = *obj;
+
+    v->type_int64(v, name, &value, &err);
+    if (err) {
+        error_propagate(errp, err);
+    } else if (value < min || value > max) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                   name ? name : "null", type);
+    } else {
+        *obj = value;
+    }
+}
+
+void visit_type_int8(Visitor *v, const char *name, int8_t *obj, Error **errp)
+{
+    int64_t value = *obj;
+    visit_type_intN(v, &value, name, INT8_MIN, INT8_MAX, "int8_t", errp);
+    *obj = value;
+}
+
+void visit_type_int16(Visitor *v, const char *name, int16_t *obj,
+                      Error **errp)
+{
+    int64_t value = *obj;
+    visit_type_intN(v, &value, name, INT16_MIN, INT16_MAX, "int16_t", errp);
+    *obj = value;
+}
+
+void visit_type_int32(Visitor *v, const char *name, int32_t *obj,
+                      Error **errp)
+{
+    int64_t value = *obj;
+    visit_type_intN(v, &value, name, INT32_MIN, INT32_MAX, "int32_t", errp);
+    *obj = value;
+}
+
+void visit_type_int64(Visitor *v, const char *name, int64_t *obj,
+                      Error **errp)
+{
+    assert(obj);
+    v->type_int64(v, name, obj, errp);
+}
+
+void visit_type_size(Visitor *v, const char *name, uint64_t *obj,
+                     Error **errp)
+{
+    assert(obj);
+    if (v->type_size) {
+        v->type_size(v, name, obj, errp);
+    } else {
+        v->type_uint64(v, name, obj, errp);
+    }
+}
+
+void visit_type_bool(Visitor *v, const char *name, bool *obj, Error **errp)
+{
+    assert(obj);
+    v->type_bool(v, name, obj, errp);
+}
+
+void visit_type_str(Visitor *v, const char *name, char **obj, Error **errp)
+{
+    Error *err = NULL;
+
+    assert(obj);
+    /* TODO: Fix callers to not pass NULL when they mean "", so that we
+     * can enable:
+    assert(!(v->type & VISITOR_OUTPUT) || *obj);
+     */
+    v->type_str(v, name, obj, &err);
+    if (v->type & VISITOR_INPUT) {
+        assert(!err != !*obj);
+    }
+    error_propagate(errp, err);
+}
+
+void visit_type_number(Visitor *v, const char *name, double *obj,
+                       Error **errp)
+{
+    assert(obj);
+    v->type_number(v, name, obj, errp);
+}
+
+void visit_type_any(Visitor *v, const char *name, QObject **obj, Error **errp)
+{
+    Error *err = NULL;
+
+    assert(obj);
+    assert(v->type != VISITOR_OUTPUT || *obj);
+    v->type_any(v, name, obj, &err);
+    if (v->type == VISITOR_INPUT) {
+        assert(!err != !*obj);
+    }
+    error_propagate(errp, err);
+}
+
+void visit_type_null(Visitor *v, const char *name, Error **errp)
+{
+    v->type_null(v, name, errp);
+}
+
+static void output_type_enum(Visitor *v, const char *name, int *obj,
+                             const char *const strings[], Error **errp)
+{
     int i = 0;
     int value = *obj;
     char *enum_str;
 
-    assert(strings);
     while (strings[i++] != NULL);
     if (value < 0 || value >= i - 1) {
-        error_set(errp, QERR_INVALID_PARAMETER, name ? name : "null");
+        error_setg(errp, QERR_INVALID_PARAMETER, name ? name : "null");
         return;
     }
 
     enum_str = (char *)strings[value];
-    visit_type_str(v, &enum_str, name, errp);
+    visit_type_str(v, name, &enum_str, errp);
 }
 
-void input_type_enum(Visitor *v, int *obj, const char *strings[],
-                     const char *kind, const char *name,
-                     Error **errp)
+static void input_type_enum(Visitor *v, const char *name, int *obj,
+                            const char *const strings[], Error **errp)
 {
     Error *local_err = NULL;
     int64_t value = 0;
     char *enum_str;
 
-    assert(strings);
-
-    visit_type_str(v, &enum_str, name, &local_err);
+    visit_type_str(v, name, &enum_str, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -303,7 +322,7 @@
     }
 
     if (strings[value] == NULL) {
-        error_set(errp, QERR_INVALID_PARAMETER, enum_str);
+        error_setg(errp, QERR_INVALID_PARAMETER, enum_str);
         g_free(enum_str);
         return;
     }
@@ -311,3 +330,24 @@
     g_free(enum_str);
     *obj = value;
 }
+
+void visit_type_enum(Visitor *v, const char *name, int *obj,
+                     const char *const strings[], Error **errp)
+{
+    assert(obj && strings);
+    switch (v->type) {
+    case VISITOR_INPUT:
+        input_type_enum(v, name, obj, strings, errp);
+        break;
+    case VISITOR_OUTPUT:
+        output_type_enum(v, name, obj, strings, errp);
+        break;
+    case VISITOR_CLONE:
+        /* nothing further to do, scalar value was already copied by
+         * g_memdup() during visit_start_*() */
+        break;
+    case VISITOR_DEALLOC:
+        /* nothing to deallocate for a scalar */
+        break;
+    }
+}
diff --git a/qapi/qmp-dispatch.c b/qapi/qmp-dispatch.c
index 168b083..505eb41 100644
--- a/qapi/qmp-dispatch.c
+++ b/qapi/qmp-dispatch.c
@@ -11,11 +11,13 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qapi/qmp/types.h"
 #include "qapi/qmp/dispatch.h"
 #include "qapi/qmp/json-parser.h"
+#include "qapi/qmp/qjson.h"
 #include "qapi-types.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 
 static QDict *qmp_dispatch_check_obj(const QObject *request, Error **errp)
@@ -27,8 +29,8 @@
     QDict *dict = NULL;
 
     if (qobject_type(request) != QTYPE_QDICT) {
-        error_set(errp, QERR_QMP_BAD_INPUT_OBJECT,
-                  "request is not a dictionary");
+        error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT,
+                   "request is not a dictionary");
         return NULL;
     }
 
@@ -41,19 +43,19 @@
 
         if (!strcmp(arg_name, "execute")) {
             if (qobject_type(arg_obj) != QTYPE_QSTRING) {
-                error_set(errp, QERR_QMP_BAD_INPUT_OBJECT_MEMBER, "execute",
-                          "string");
+                error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT_MEMBER, "execute",
+                           "string");
                 return NULL;
             }
             has_exec_key = true;
         } else if (strcmp(arg_name, "arguments")) {
-            error_set(errp, QERR_QMP_EXTRA_MEMBER, arg_name);
+            error_setg(errp, QERR_QMP_EXTRA_MEMBER, arg_name);
             return NULL;
         }
     }
 
     if (!has_exec_key) {
-        error_set(errp, QERR_QMP_BAD_INPUT_OBJECT, "execute");
+        error_setg(errp, QERR_QMP_BAD_INPUT_OBJECT, "execute");
         return NULL;
     }
 
@@ -76,7 +78,8 @@
     command = qdict_get_str(dict, "execute");
     cmd = qmp_find_command(command);
     if (cmd == NULL) {
-        error_set(errp, QERR_COMMAND_NOT_FOUND, command);
+        error_set(errp, ERROR_CLASS_COMMAND_NOT_FOUND,
+                  "The command %s has not been found", command);
         return NULL;
     }
     if (!cmd->enabled) {
@@ -92,17 +95,13 @@
         QINCREF(args);
     }
 
-    switch (cmd->type) {
-    case QCT_NORMAL:
-        cmd->fn(args, &ret, &local_err);
-        if (local_err) {
-            error_propagate(errp, local_err);
-        } else if (cmd->options & QCO_NO_SUCCESS_RESP) {
-            g_assert(!ret);
-        } else if (!ret) {
-            ret = QOBJECT(qdict_new());
-        }
-        break;
+    cmd->fn(args, &ret, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+    } else if (cmd->options & QCO_NO_SUCCESS_RESP) {
+        g_assert(!ret);
+    } else if (!ret) {
+        ret = QOBJECT(qdict_new());
     }
 
     QDECREF(args);
@@ -113,7 +112,7 @@
 QObject *qmp_build_error_object(Error *err)
 {
     return qobject_from_jsonf("{ 'class': %s, 'desc': %s }",
-                              ErrorClass_lookup[error_get_class(err)],
+                              QapiErrorClass_lookup[error_get_class(err)],
                               error_get_pretty(err));
 }
 
diff --git a/qapi/qmp-event.c b/qapi/qmp-event.c
index 0d1ce0b..8bba165 100644
--- a/qapi/qmp-event.c
+++ b/qapi/qmp-event.c
@@ -11,21 +11,13 @@
  *
  */
 
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "qapi/qmp-event.h"
 #include "qapi/qmp/qstring.h"
 #include "qapi/qmp/qjson.h"
 
-#ifdef _WIN32
-#include "sysemu/os-win32.h"
-#endif
-
-#ifdef CONFIG_POSIX
-#include "sysemu/os-posix.h"
-#endif
-
 static QMPEventFuncEmit qmp_emit;
 
 void qmp_event_set_func_emit(QMPEventFuncEmit emit)
diff --git a/qapi/qmp-input-visitor.c b/qapi/qmp-input-visitor.c
index d861206..64dd392 100644
--- a/qapi/qmp-input-visitor.c
+++ b/qapi/qmp-input-visitor.c
@@ -1,6 +1,7 @@
 /*
  * Input Visitor
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,6 +12,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/visitor-impl.h"
 #include "qemu/queue.h"
@@ -22,16 +25,27 @@
 
 typedef struct StackObject
 {
-    QObject *obj;
-    const QListEntry *entry;
-    GHashTable *h;
+    QObject *obj; /* Object being visited */
+    void *qapi; /* sanity check that caller uses same pointer */
+
+    GHashTable *h;           /* If obj is dict: unvisited keys */
+    const QListEntry *entry; /* If obj is list: unvisited tail */
+
+    QSLIST_ENTRY(StackObject) node;
 } StackObject;
 
 struct QmpInputVisitor
 {
     Visitor visitor;
-    StackObject stack[QIV_STACK_SIZE];
-    int nb_stack;
+
+    /* Root of visit at visitor creation. */
+    QObject *root;
+
+    /* Stack of objects being visited (all entries will be either
+     * QDict or QList). */
+    QSLIST_HEAD(, StackObject) stack;
+
+    /* True to reject parse in visit_end_struct() if unvisited keys remain. */
     bool strict;
 };
 
@@ -44,20 +58,37 @@
                                      const char *name,
                                      bool consume)
 {
-    QObject *qobj = qiv->stack[qiv->nb_stack - 1].obj;
+    StackObject *tos;
+    QObject *qobj;
+    QObject *ret;
 
-    if (qobj) {
-        if (name && qobject_type(qobj) == QTYPE_QDICT) {
-            if (qiv->stack[qiv->nb_stack - 1].h && consume) {
-                g_hash_table_remove(qiv->stack[qiv->nb_stack - 1].h, name);
-            }
-            return qdict_get(qobject_to_qdict(qobj), name);
-        } else if (qiv->stack[qiv->nb_stack - 1].entry) {
-            return qlist_entry_obj(qiv->stack[qiv->nb_stack - 1].entry);
+    if (QSLIST_EMPTY(&qiv->stack)) {
+        /* Starting at root, name is ignored. */
+        return qiv->root;
+    }
+
+    /* We are in a container; find the next element. */
+    tos = QSLIST_FIRST(&qiv->stack);
+    qobj = tos->obj;
+    assert(qobj);
+
+    if (qobject_type(qobj) == QTYPE_QDICT) {
+        assert(name);
+        ret = qdict_get(qobject_to_qdict(qobj), name);
+        if (tos->h && consume && ret) {
+            bool removed = g_hash_table_remove(tos->h, name);
+            assert(removed);
+        }
+    } else {
+        assert(qobject_type(qobj) == QTYPE_QLIST);
+        assert(!name);
+        ret = qlist_entry_obj(tos->entry);
+        if (consume) {
+            tos->entry = qlist_next(tos->entry);
         }
     }
 
-    return qobj;
+    return ret;
 }
 
 static void qdict_add_key(const char *key, QObject *obj, void *opaque)
@@ -66,68 +97,85 @@
     g_hash_table_insert(h, (gpointer) key, NULL);
 }
 
-static void qmp_input_push(QmpInputVisitor *qiv, QObject *obj, Error **errp)
+static const QListEntry *qmp_input_push(QmpInputVisitor *qiv, QObject *obj,
+                                        void *qapi, Error **errp)
 {
     GHashTable *h;
+    StackObject *tos = g_new0(StackObject, 1);
 
-    if (qiv->nb_stack >= QIV_STACK_SIZE) {
-        error_setg(errp, "An internal buffer overran");
-        return;
-    }
-
-    qiv->stack[qiv->nb_stack].obj = obj;
-    qiv->stack[qiv->nb_stack].entry = NULL;
-    qiv->stack[qiv->nb_stack].h = NULL;
+    assert(obj);
+    tos->obj = obj;
+    tos->qapi = qapi;
 
     if (qiv->strict && qobject_type(obj) == QTYPE_QDICT) {
         h = g_hash_table_new(g_str_hash, g_str_equal);
         qdict_iter(qobject_to_qdict(obj), qdict_add_key, h);
-        qiv->stack[qiv->nb_stack].h = h;
+        tos->h = h;
+    } else if (qobject_type(obj) == QTYPE_QLIST) {
+        tos->entry = qlist_first(qobject_to_qlist(obj));
     }
 
-    qiv->nb_stack++;
+    QSLIST_INSERT_HEAD(&qiv->stack, tos, node);
+    return tos->entry;
 }
 
-/** Only for qmp_input_pop. */
-static gboolean always_true(gpointer key, gpointer val, gpointer user_pkey)
-{
-    *(const char **)user_pkey = (const char *)key;
-    return TRUE;
-}
 
-static void qmp_input_pop(QmpInputVisitor *qiv, Error **errp)
+static void qmp_input_check_struct(Visitor *v, Error **errp)
 {
-    assert(qiv->nb_stack > 0);
+    QmpInputVisitor *qiv = to_qiv(v);
+    StackObject *tos = QSLIST_FIRST(&qiv->stack);
 
+    assert(tos && !tos->entry);
     if (qiv->strict) {
-        GHashTable * const top_ht = qiv->stack[qiv->nb_stack - 1].h;
+        GHashTable *const top_ht = tos->h;
         if (top_ht) {
-            if (g_hash_table_size(top_ht)) {
-                const char *key;
-                g_hash_table_find(top_ht, always_true, &key);
-                error_set(errp, QERR_QMP_EXTRA_MEMBER, key);
+            GHashTableIter iter;
+            const char *key;
+
+            g_hash_table_iter_init(&iter, top_ht);
+            if (g_hash_table_iter_next(&iter, (void **)&key, NULL)) {
+                error_setg(errp, QERR_QMP_EXTRA_MEMBER, key);
             }
-            g_hash_table_unref(top_ht);
         }
     }
-
-    qiv->nb_stack--;
 }
 
-static void qmp_input_start_struct(Visitor *v, void **obj, const char *kind,
-                                   const char *name, size_t size, Error **errp)
+static void qmp_input_stack_object_free(StackObject *tos)
+{
+    if (tos->h) {
+        g_hash_table_unref(tos->h);
+    }
+
+    g_free(tos);
+}
+
+static void qmp_input_pop(Visitor *v, void **obj)
+{
+    QmpInputVisitor *qiv = to_qiv(v);
+    StackObject *tos = QSLIST_FIRST(&qiv->stack);
+
+    assert(tos && tos->qapi == obj);
+    QSLIST_REMOVE_HEAD(&qiv->stack, node);
+    qmp_input_stack_object_free(tos);
+}
+
+static void qmp_input_start_struct(Visitor *v, const char *name, void **obj,
+                                   size_t size, Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
     Error *err = NULL;
 
+    if (obj) {
+        *obj = NULL;
+    }
     if (!qobj || qobject_type(qobj) != QTYPE_QDICT) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "QDict");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "QDict");
         return;
     }
 
-    qmp_input_push(qiv, qobj, &err);
+    qmp_input_push(qiv, qobj, obj, &err);
     if (err) {
         error_propagate(errp, err);
         return;
@@ -138,160 +186,178 @@
     }
 }
 
-static void qmp_input_end_struct(Visitor *v, Error **errp)
-{
-    QmpInputVisitor *qiv = to_qiv(v);
 
-    qmp_input_pop(qiv, errp);
-}
-
-static void qmp_input_start_implicit_struct(Visitor *v, void **obj,
-                                            size_t size, Error **errp)
-{
-    if (obj) {
-        *obj = g_malloc0(size);
-    }
-}
-
-static void qmp_input_end_implicit_struct(Visitor *v, Error **errp)
-{
-}
-
-static void qmp_input_start_list(Visitor *v, const char *name, Error **errp)
+static void qmp_input_start_list(Visitor *v, const char *name,
+                                 GenericList **list, size_t size, Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
+    const QListEntry *entry;
 
     if (!qobj || qobject_type(qobj) != QTYPE_QLIST) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "list");
+        if (list) {
+            *list = NULL;
+        }
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "list");
         return;
     }
 
-    qmp_input_push(qiv, qobj, errp);
+    entry = qmp_input_push(qiv, qobj, list, errp);
+    if (list) {
+        if (entry) {
+            *list = g_malloc0(size);
+        } else {
+            *list = NULL;
+        }
+    }
 }
 
-static GenericList *qmp_input_next_list(Visitor *v, GenericList **list,
-                                        Error **errp)
+static GenericList *qmp_input_next_list(Visitor *v, GenericList *tail,
+                                        size_t size)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    GenericList *entry;
-    StackObject *so = &qiv->stack[qiv->nb_stack - 1];
-    bool first;
+    StackObject *so = QSLIST_FIRST(&qiv->stack);
 
-    if (so->entry == NULL) {
-        so->entry = qlist_first(qobject_to_qlist(so->obj));
-        first = true;
-    } else {
-        so->entry = qlist_next(so->entry);
-        first = false;
-    }
-
-    if (so->entry == NULL) {
+    if (!so->entry) {
         return NULL;
     }
-
-    entry = g_malloc0(sizeof(*entry));
-    if (first) {
-        *list = entry;
-    } else {
-        (*list)->next = entry;
-    }
-
-    return entry;
+    tail->next = g_malloc0(size);
+    return tail->next;
 }
 
-static void qmp_input_end_list(Visitor *v, Error **errp)
-{
-    QmpInputVisitor *qiv = to_qiv(v);
 
-    qmp_input_pop(qiv, errp);
-}
-
-static void qmp_input_get_next_type(Visitor *v, int *kind, const int *qobjects,
-                                    const char *name, Error **errp)
+static void qmp_input_start_alternate(Visitor *v, const char *name,
+                                      GenericAlternate **obj, size_t size,
+                                      bool promote_int, Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, false);
 
     if (!qobj) {
-        error_set(errp, QERR_MISSING_PARAMETER, name ? name : "null");
+        *obj = NULL;
+        error_setg(errp, QERR_MISSING_PARAMETER, name ? name : "null");
         return;
     }
-    *kind = qobjects[qobject_type(qobj)];
+    *obj = g_malloc0(size);
+    (*obj)->type = qobject_type(qobj);
+    if (promote_int && (*obj)->type == QTYPE_QINT) {
+        (*obj)->type = QTYPE_QFLOAT;
+    }
 }
 
-static void qmp_input_type_int(Visitor *v, int64_t *obj, const char *name,
-                               Error **errp)
+static void qmp_input_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                 Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QInt *qint = qobject_to_qint(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QINT) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "integer");
+    if (!qint) {
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "integer");
         return;
     }
 
-    *obj = qint_get_int(qobject_to_qint(qobj));
+    *obj = qint_get_int(qint);
 }
 
-static void qmp_input_type_bool(Visitor *v, bool *obj, const char *name,
+static void qmp_input_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                                  Error **errp)
+{
+    /* FIXME: qobject_to_qint mishandles values over INT64_MAX */
+    QmpInputVisitor *qiv = to_qiv(v);
+    QInt *qint = qobject_to_qint(qmp_input_get_object(qiv, name, true));
+
+    if (!qint) {
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "integer");
+        return;
+    }
+
+    *obj = qint_get_int(qint);
+}
+
+static void qmp_input_type_bool(Visitor *v, const char *name, bool *obj,
                                 Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QBool *qbool = qobject_to_qbool(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QBOOL) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "boolean");
+    if (!qbool) {
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "boolean");
         return;
     }
 
-    *obj = qbool_get_int(qobject_to_qbool(qobj));
+    *obj = qbool_get_bool(qbool);
 }
 
-static void qmp_input_type_str(Visitor *v, char **obj, const char *name,
+static void qmp_input_type_str(Visitor *v, const char *name, char **obj,
                                Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
-    QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QString *qstr = qobject_to_qstring(qmp_input_get_object(qiv, name, true));
 
-    if (!qobj || qobject_type(qobj) != QTYPE_QSTRING) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "string");
+    if (!qstr) {
+        *obj = NULL;
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "string");
         return;
     }
 
-    *obj = g_strdup(qstring_get_str(qobject_to_qstring(qobj)));
+    *obj = g_strdup(qstring_get_str(qstr));
 }
 
-static void qmp_input_type_number(Visitor *v, double *obj, const char *name,
+static void qmp_input_type_number(Visitor *v, const char *name, double *obj,
                                   Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
+    QInt *qint;
+    QFloat *qfloat;
 
-    if (!qobj || (qobject_type(qobj) != QTYPE_QFLOAT &&
-        qobject_type(qobj) != QTYPE_QINT)) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "number");
+    qint = qobject_to_qint(qobj);
+    if (qint) {
+        *obj = qint_get_int(qobject_to_qint(qobj));
         return;
     }
 
-    if (qobject_type(qobj) == QTYPE_QINT) {
-        *obj = qint_get_int(qobject_to_qint(qobj));
-    } else {
+    qfloat = qobject_to_qfloat(qobj);
+    if (qfloat) {
         *obj = qfloat_get_double(qobject_to_qfloat(qobj));
+        return;
     }
+
+    error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+               "number");
 }
 
-static void qmp_input_optional(Visitor *v, bool *present, const char *name,
+static void qmp_input_type_any(Visitor *v, const char *name, QObject **obj,
                                Error **errp)
 {
     QmpInputVisitor *qiv = to_qiv(v);
     QObject *qobj = qmp_input_get_object(qiv, name, true);
 
+    qobject_incref(qobj);
+    *obj = qobj;
+}
+
+static void qmp_input_type_null(Visitor *v, const char *name, Error **errp)
+{
+    QmpInputVisitor *qiv = to_qiv(v);
+    QObject *qobj = qmp_input_get_object(qiv, name, true);
+
+    if (qobject_type(qobj) != QTYPE_QNULL) {
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "null");
+    }
+}
+
+static void qmp_input_optional(Visitor *v, const char *name, bool *present)
+{
+    QmpInputVisitor *qiv = to_qiv(v);
+    QObject *qobj = qmp_input_get_object(qiv, name, false);
+
     if (!qobj) {
         *present = false;
         return;
@@ -300,50 +366,47 @@
     *present = true;
 }
 
-Visitor *qmp_input_get_visitor(QmpInputVisitor *v)
+static void qmp_input_free(Visitor *v)
 {
-    return &v->visitor;
+    QmpInputVisitor *qiv = to_qiv(v);
+    while (!QSLIST_EMPTY(&qiv->stack)) {
+        StackObject *tos = QSLIST_FIRST(&qiv->stack);
+
+        QSLIST_REMOVE_HEAD(&qiv->stack, node);
+        qmp_input_stack_object_free(tos);
+    }
+
+    qobject_decref(qiv->root);
+    g_free(qiv);
 }
 
-void qmp_input_visitor_cleanup(QmpInputVisitor *v)
-{
-    qobject_decref(v->stack[0].obj);
-    g_free(v);
-}
-
-QmpInputVisitor *qmp_input_visitor_new(QObject *obj)
+Visitor *qmp_input_visitor_new(QObject *obj, bool strict)
 {
     QmpInputVisitor *v;
 
     v = g_malloc0(sizeof(*v));
 
+    v->visitor.type = VISITOR_INPUT;
     v->visitor.start_struct = qmp_input_start_struct;
-    v->visitor.end_struct = qmp_input_end_struct;
-    v->visitor.start_implicit_struct = qmp_input_start_implicit_struct;
-    v->visitor.end_implicit_struct = qmp_input_end_implicit_struct;
+    v->visitor.check_struct = qmp_input_check_struct;
+    v->visitor.end_struct = qmp_input_pop;
     v->visitor.start_list = qmp_input_start_list;
     v->visitor.next_list = qmp_input_next_list;
-    v->visitor.end_list = qmp_input_end_list;
-    v->visitor.type_enum = input_type_enum;
-    v->visitor.type_int = qmp_input_type_int;
+    v->visitor.end_list = qmp_input_pop;
+    v->visitor.start_alternate = qmp_input_start_alternate;
+    v->visitor.type_int64 = qmp_input_type_int64;
+    v->visitor.type_uint64 = qmp_input_type_uint64;
     v->visitor.type_bool = qmp_input_type_bool;
     v->visitor.type_str = qmp_input_type_str;
     v->visitor.type_number = qmp_input_type_number;
+    v->visitor.type_any = qmp_input_type_any;
+    v->visitor.type_null = qmp_input_type_null;
     v->visitor.optional = qmp_input_optional;
-    v->visitor.get_next_type = qmp_input_get_next_type;
+    v->visitor.free = qmp_input_free;
+    v->strict = strict;
 
-    qmp_input_push(v, obj, NULL);
+    v->root = obj;
     qobject_incref(obj);
 
-    return v;
-}
-
-QmpInputVisitor *qmp_input_visitor_new_strict(QObject *obj)
-{
-    QmpInputVisitor *v;
-
-    v = qmp_input_visitor_new(obj);
-    v->strict = true;
-
-    return v;
+    return &v->visitor;
 }
diff --git a/qapi/qmp-output-visitor.c b/qapi/qmp-output-visitor.c
index 96b3384..9e3b67c 100644
--- a/qapi/qmp-output-visitor.c
+++ b/qapi/qmp-output-visitor.c
@@ -1,6 +1,7 @@
 /*
  * Core Definitions for QAPI/QMP Command Registry
  *
+ * Copyright (C) 2012-2016 Red Hat, Inc.
  * Copyright IBM, Corp. 2011
  *
  * Authors:
@@ -11,165 +12,159 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/visitor-impl.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
 #include "qapi/qmp/types.h"
-#include "qapi/qmp/qerror.h"
 
 typedef struct QStackEntry
 {
     QObject *value;
-    bool is_list_head;
-    QTAILQ_ENTRY(QStackEntry) node;
+    void *qapi; /* sanity check that caller uses same pointer */
+    QSLIST_ENTRY(QStackEntry) node;
 } QStackEntry;
 
-typedef QTAILQ_HEAD(QStack, QStackEntry) QStack;
-
 struct QmpOutputVisitor
 {
     Visitor visitor;
-    QStack stack;
+    QSLIST_HEAD(, QStackEntry) stack; /* Stack of unfinished containers */
+    QObject *root; /* Root of the output visit */
+    QObject **result; /* User's storage location for result */
 };
 
 #define qmp_output_add(qov, name, value) \
     qmp_output_add_obj(qov, name, QOBJECT(value))
-#define qmp_output_push(qov, value) qmp_output_push_obj(qov, QOBJECT(value))
+#define qmp_output_push(qov, value, qapi) \
+    qmp_output_push_obj(qov, QOBJECT(value), qapi)
 
 static QmpOutputVisitor *to_qov(Visitor *v)
 {
     return container_of(v, QmpOutputVisitor, visitor);
 }
 
-static void qmp_output_push_obj(QmpOutputVisitor *qov, QObject *value)
+/* Push @value onto the stack of current QObjects being built */
+static void qmp_output_push_obj(QmpOutputVisitor *qov, QObject *value,
+                                void *qapi)
 {
     QStackEntry *e = g_malloc0(sizeof(*e));
 
+    assert(qov->root);
+    assert(value);
     e->value = value;
-    if (qobject_type(e->value) == QTYPE_QLIST) {
-        e->is_list_head = true;
-    }
-    QTAILQ_INSERT_HEAD(&qov->stack, e, node);
+    e->qapi = qapi;
+    QSLIST_INSERT_HEAD(&qov->stack, e, node);
 }
 
-static QObject *qmp_output_pop(QmpOutputVisitor *qov)
+/* Pop a value off the stack of QObjects being built, and return it. */
+static QObject *qmp_output_pop(QmpOutputVisitor *qov, void *qapi)
 {
-    QStackEntry *e = QTAILQ_FIRST(&qov->stack);
+    QStackEntry *e = QSLIST_FIRST(&qov->stack);
     QObject *value;
-    QTAILQ_REMOVE(&qov->stack, e, node);
+
+    assert(e);
+    assert(e->qapi == qapi);
+    QSLIST_REMOVE_HEAD(&qov->stack, node);
     value = e->value;
+    assert(value);
     g_free(e);
     return value;
 }
 
-static QObject *qmp_output_first(QmpOutputVisitor *qov)
-{
-    QStackEntry *e = QTAILQ_LAST(&qov->stack, QStack);
-
-    /* FIXME - find a better way to deal with NULL values */
-    if (!e) {
-        return NULL;
-    }
-
-    return e->value;
-}
-
-static QObject *qmp_output_last(QmpOutputVisitor *qov)
-{
-    QStackEntry *e = QTAILQ_FIRST(&qov->stack);
-    return e->value;
-}
-
+/* Add @value to the current QObject being built.
+ * If the stack is visiting a dictionary or list, @value is now owned
+ * by that container. Otherwise, @value is now the root.  */
 static void qmp_output_add_obj(QmpOutputVisitor *qov, const char *name,
                                QObject *value)
 {
-    QObject *cur;
+    QStackEntry *e = QSLIST_FIRST(&qov->stack);
+    QObject *cur = e ? e->value : NULL;
 
-    if (QTAILQ_EMPTY(&qov->stack)) {
-        qmp_output_push_obj(qov, value);
-        return;
-    }
-
-    cur = qmp_output_last(qov);
-
-    switch (qobject_type(cur)) {
-    case QTYPE_QDICT:
-        qdict_put_obj(qobject_to_qdict(cur), name, value);
-        break;
-    case QTYPE_QLIST:
-        qlist_append_obj(qobject_to_qlist(cur), value);
-        break;
-    default:
-        qobject_decref(qmp_output_pop(qov));
-        qmp_output_push_obj(qov, value);
-        break;
+    if (!cur) {
+        /* Don't allow reuse of visitor on more than one root */
+        assert(!qov->root);
+        qov->root = value;
+    } else {
+        switch (qobject_type(cur)) {
+        case QTYPE_QDICT:
+            assert(name);
+            qdict_put_obj(qobject_to_qdict(cur), name, value);
+            break;
+        case QTYPE_QLIST:
+            assert(!name);
+            qlist_append_obj(qobject_to_qlist(cur), value);
+            break;
+        default:
+            g_assert_not_reached();
+        }
     }
 }
 
-static void qmp_output_start_struct(Visitor *v, void **obj, const char *kind,
-                                    const char *name, size_t unused,
-                                    Error **errp)
+static void qmp_output_start_struct(Visitor *v, const char *name, void **obj,
+                                    size_t unused, Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     QDict *dict = qdict_new();
 
     qmp_output_add(qov, name, dict);
-    qmp_output_push(qov, dict);
+    qmp_output_push(qov, dict, obj);
 }
 
-static void qmp_output_end_struct(Visitor *v, Error **errp)
+static void qmp_output_end_struct(Visitor *v, void **obj)
 {
     QmpOutputVisitor *qov = to_qov(v);
-    qmp_output_pop(qov);
+    QObject *value = qmp_output_pop(qov, obj);
+    assert(qobject_type(value) == QTYPE_QDICT);
 }
 
-static void qmp_output_start_list(Visitor *v, const char *name, Error **errp)
+static void qmp_output_start_list(Visitor *v, const char *name,
+                                  GenericList **listp, size_t size,
+                                  Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     QList *list = qlist_new();
 
     qmp_output_add(qov, name, list);
-    qmp_output_push(qov, list);
+    qmp_output_push(qov, list, listp);
 }
 
-static GenericList *qmp_output_next_list(Visitor *v, GenericList **listp,
-                                         Error **errp)
+static GenericList *qmp_output_next_list(Visitor *v, GenericList *tail,
+                                         size_t size)
 {
-    GenericList *list = *listp;
-    QmpOutputVisitor *qov = to_qov(v);
-    QStackEntry *e = QTAILQ_FIRST(&qov->stack);
-
-    assert(e);
-    if (e->is_list_head) {
-        e->is_list_head = false;
-        return list;
-    }
-
-    return list ? list->next : NULL;
+    return tail->next;
 }
 
-static void qmp_output_end_list(Visitor *v, Error **errp)
+static void qmp_output_end_list(Visitor *v, void **obj)
 {
     QmpOutputVisitor *qov = to_qov(v);
-    qmp_output_pop(qov);
+    QObject *value = qmp_output_pop(qov, obj);
+    assert(qobject_type(value) == QTYPE_QLIST);
 }
 
-static void qmp_output_type_int(Visitor *v, int64_t *obj, const char *name,
-                                Error **errp)
+static void qmp_output_type_int64(Visitor *v, const char *name, int64_t *obj,
+                                  Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     qmp_output_add(qov, name, qint_from_int(*obj));
 }
 
-static void qmp_output_type_bool(Visitor *v, bool *obj, const char *name,
+static void qmp_output_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                                   Error **errp)
+{
+    /* FIXME: QMP outputs values larger than INT64_MAX as negative */
+    QmpOutputVisitor *qov = to_qov(v);
+    qmp_output_add(qov, name, qint_from_int(*obj));
+}
+
+static void qmp_output_type_bool(Visitor *v, const char *name, bool *obj,
                                  Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
-    qmp_output_add(qov, name, qbool_from_int(*obj));
+    qmp_output_add(qov, name, qbool_from_bool(*obj));
 }
 
-static void qmp_output_type_str(Visitor *v, char **obj, const char *name,
+static void qmp_output_type_str(Visitor *v, const char *name, char **obj,
                                 Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
@@ -180,62 +175,82 @@
     }
 }
 
-static void qmp_output_type_number(Visitor *v, double *obj, const char *name,
+static void qmp_output_type_number(Visitor *v, const char *name, double *obj,
                                    Error **errp)
 {
     QmpOutputVisitor *qov = to_qov(v);
     qmp_output_add(qov, name, qfloat_from_double(*obj));
 }
 
-QObject *qmp_output_get_qobject(QmpOutputVisitor *qov)
+static void qmp_output_type_any(Visitor *v, const char *name, QObject **obj,
+                                Error **errp)
 {
-    QObject *obj = qmp_output_first(qov);
-    if (obj) {
-        qobject_incref(obj);
-    }
-    return obj;
+    QmpOutputVisitor *qov = to_qov(v);
+    qobject_incref(*obj);
+    qmp_output_add_obj(qov, name, *obj);
 }
 
-Visitor *qmp_output_get_visitor(QmpOutputVisitor *v)
+static void qmp_output_type_null(Visitor *v, const char *name, Error **errp)
 {
-    return &v->visitor;
+    QmpOutputVisitor *qov = to_qov(v);
+    qmp_output_add_obj(qov, name, qnull());
 }
 
-void qmp_output_visitor_cleanup(QmpOutputVisitor *v)
+/* Finish building, and return the root object.
+ * The root object is never null. The caller becomes the object's
+ * owner, and should use qobject_decref() when done with it.  */
+static void qmp_output_complete(Visitor *v, void *opaque)
 {
-    QStackEntry *e, *tmp;
+    QmpOutputVisitor *qov = to_qov(v);
 
-    /* The bottom QStackEntry, if any, owns the root QObject. See the
-     * qmp_output_push_obj() invocations in qmp_output_add_obj(). */
-    QObject *root = QTAILQ_EMPTY(&v->stack) ? NULL : qmp_output_first(v);
+    /* A visit must have occurred, with each start paired with end.  */
+    assert(qov->root && QSLIST_EMPTY(&qov->stack));
+    assert(opaque == qov->result);
 
-    QTAILQ_FOREACH_SAFE(e, &v->stack, node, tmp) {
-        QTAILQ_REMOVE(&v->stack, e, node);
+    qobject_incref(qov->root);
+    *qov->result = qov->root;
+    qov->result = NULL;
+}
+
+static void qmp_output_free(Visitor *v)
+{
+    QmpOutputVisitor *qov = to_qov(v);
+    QStackEntry *e;
+
+    while (!QSLIST_EMPTY(&qov->stack)) {
+        e = QSLIST_FIRST(&qov->stack);
+        QSLIST_REMOVE_HEAD(&qov->stack, node);
         g_free(e);
     }
 
-    qobject_decref(root);
-    g_free(v);
+    qobject_decref(qov->root);
+    g_free(qov);
 }
 
-QmpOutputVisitor *qmp_output_visitor_new(void)
+Visitor *qmp_output_visitor_new(QObject **result)
 {
     QmpOutputVisitor *v;
 
     v = g_malloc0(sizeof(*v));
 
+    v->visitor.type = VISITOR_OUTPUT;
     v->visitor.start_struct = qmp_output_start_struct;
     v->visitor.end_struct = qmp_output_end_struct;
     v->visitor.start_list = qmp_output_start_list;
     v->visitor.next_list = qmp_output_next_list;
     v->visitor.end_list = qmp_output_end_list;
-    v->visitor.type_enum = output_type_enum;
-    v->visitor.type_int = qmp_output_type_int;
+    v->visitor.type_int64 = qmp_output_type_int64;
+    v->visitor.type_uint64 = qmp_output_type_uint64;
     v->visitor.type_bool = qmp_output_type_bool;
     v->visitor.type_str = qmp_output_type_str;
     v->visitor.type_number = qmp_output_type_number;
+    v->visitor.type_any = qmp_output_type_any;
+    v->visitor.type_null = qmp_output_type_null;
+    v->visitor.complete = qmp_output_complete;
+    v->visitor.free = qmp_output_free;
 
-    QTAILQ_INIT(&v->stack);
+    *result = NULL;
+    v->result = result;
 
-    return v;
+    return &v->visitor;
 }
diff --git a/qapi/qmp-registry.c b/qapi/qmp-registry.c
index 3e4498a..68b24c9 100644
--- a/qapi/qmp-registry.c
+++ b/qapi/qmp-registry.c
@@ -12,8 +12,7 @@
  *
  */
 
-#include <glib.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include "qapi/qmp/dispatch.h"
 
 static QTAILQ_HEAD(QmpCommandList, QmpCommand) qmp_commands =
@@ -25,7 +24,6 @@
     QmpCommand *cmd = g_malloc0(sizeof(*cmd));
 
     cmd->name = name;
-    cmd->type = QCT_NORMAL;
     cmd->fn = fn;
     cmd->enabled = true;
     cmd->options = options;
diff --git a/qapi/rocker.json b/qapi/rocker.json
new file mode 100644
index 0000000..2fe7fdf
--- /dev/null
+++ b/qapi/rocker.json
@@ -0,0 +1,286 @@
+##
+# @Rocker:
+#
+# Rocker switch information.
+#
+# @name: switch name
+#
+# @id: switch ID
+#
+# @ports: number of front-panel ports
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerSwitch',
+  'data': { 'name': 'str', 'id': 'uint64', 'ports': 'uint32' } }
+
+##
+# @query-rocker:
+#
+# Return rocker switch information.
+#
+# Returns: @Rocker information
+#
+# Since: 2.4
+##
+{ 'command': 'query-rocker',
+  'data': { 'name': 'str' },
+  'returns': 'RockerSwitch' }
+
+##
+# @RockerPortDuplex:
+#
+# An eumeration of port duplex states.
+#
+# @half: half duplex
+#
+# @full: full duplex
+#
+# Since: 2.4
+##
+{ 'enum': 'RockerPortDuplex', 'data': [ 'half', 'full' ] }
+
+##
+# @RockerPortAutoneg:
+#
+# An eumeration of port autoneg states.
+#
+# @off: autoneg is off
+#
+# @on: autoneg is on
+#
+# Since: 2.4
+##
+{ 'enum': 'RockerPortAutoneg', 'data': [ 'off', 'on' ] }
+
+##
+# @RockerPort:
+#
+# Rocker switch port information.
+#
+# @name: port name
+#
+# @enabled: port is enabled for I/O
+#
+# @link-up: physical link is UP on port
+#
+# @speed: port link speed in Mbps
+#
+# @duplex: port link duplex
+#
+# @autoneg: port link autoneg
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerPort',
+  'data': { 'name': 'str', 'enabled': 'bool', 'link-up': 'bool',
+            'speed': 'uint32', 'duplex': 'RockerPortDuplex',
+            'autoneg': 'RockerPortAutoneg' } }
+
+##
+# @query-rocker-ports:
+#
+# Return rocker switch information.
+#
+# Returns: @Rocker information
+#
+# Since: 2.4
+##
+{ 'command': 'query-rocker-ports',
+  'data': { 'name': 'str' },
+  'returns': ['RockerPort'] }
+
+##
+# @RockerOfDpaFlowKey:
+#
+# Rocker switch OF-DPA flow key
+#
+# @priority: key priority, 0 being lowest priority
+#
+# @tbl-id: flow table ID
+#
+# @in-pport: #optional physical input port
+#
+# @tunnel-id: #optional tunnel ID
+#
+# @vlan-id: #optional VLAN ID
+#
+# @eth-type: #optional Ethernet header type
+#
+# @eth-src: #optional Ethernet header source MAC address
+#
+# @eth-dst: #optional Ethernet header destination MAC address
+#
+# @ip-proto: #optional IP Header protocol field
+#
+# @ip-tos: #optional IP header TOS field
+#
+# @ip-dst: #optional IP header destination address
+#
+# Note: fields are marked #optional to indicate that they may or may not
+# appear in the flow key depending if they're relevant to the flow key.
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerOfDpaFlowKey',
+  'data' : { 'priority': 'uint32', 'tbl-id': 'uint32', '*in-pport': 'uint32',
+             '*tunnel-id': 'uint32', '*vlan-id': 'uint16',
+             '*eth-type': 'uint16', '*eth-src': 'str', '*eth-dst': 'str',
+             '*ip-proto': 'uint8', '*ip-tos': 'uint8', '*ip-dst': 'str' } }
+
+##
+# @RockerOfDpaFlowMask:
+#
+# Rocker switch OF-DPA flow mask
+#
+# @in-pport: #optional physical input port
+#
+# @tunnel-id: #optional tunnel ID
+#
+# @vlan-id: #optional VLAN ID
+#
+# @eth-src: #optional Ethernet header source MAC address
+#
+# @eth-dst: #optional Ethernet header destination MAC address
+#
+# @ip-proto: #optional IP Header protocol field
+#
+# @ip-tos: #optional IP header TOS field
+#
+# Note: fields are marked #optional to indicate that they may or may not
+# appear in the flow mask depending if they're relevant to the flow mask.
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerOfDpaFlowMask',
+  'data' : { '*in-pport': 'uint32', '*tunnel-id': 'uint32',
+             '*vlan-id': 'uint16', '*eth-src': 'str', '*eth-dst': 'str',
+             '*ip-proto': 'uint8', '*ip-tos': 'uint8' } }
+
+##
+# @RockerOfDpaFlowAction:
+#
+# Rocker switch OF-DPA flow action
+#
+# @goto-tbl: #optional next table ID
+#
+# @group-id: #optional group ID
+#
+# @tunnel-lport: #optional tunnel logical port ID
+#
+# @vlan-id: #optional VLAN ID
+#
+# @new-vlan-id: #optional new VLAN ID
+#
+# @out-pport: #optional physical output port
+#
+# Note: fields are marked #optional to indicate that they may or may not
+# appear in the flow action depending if they're relevant to the flow action.
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerOfDpaFlowAction',
+  'data' : { '*goto-tbl': 'uint32', '*group-id': 'uint32',
+             '*tunnel-lport': 'uint32', '*vlan-id': 'uint16',
+             '*new-vlan-id': 'uint16', '*out-pport': 'uint32' } }
+
+##
+# @RockerOfDpaFlow:
+#
+# Rocker switch OF-DPA flow
+#
+# @cookie: flow unique cookie ID
+#
+# @hits: count of matches (hits) on flow
+#
+# @key: flow key
+#
+# @mask: flow mask
+#
+# @action: flow action
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerOfDpaFlow',
+  'data': { 'cookie': 'uint64', 'hits': 'uint64', 'key': 'RockerOfDpaFlowKey',
+            'mask': 'RockerOfDpaFlowMask', 'action': 'RockerOfDpaFlowAction' } }
+
+##
+# @query-rocker-of-dpa-flows:
+#
+# Return rocker OF-DPA flow information.
+#
+# @name: switch name
+#
+# @tbl-id: #optional flow table ID.  If tbl-id is not specified, returns
+# flow information for all tables.
+#
+# Returns: @Rocker OF-DPA flow information
+#
+# Since: 2.4
+##
+{ 'command': 'query-rocker-of-dpa-flows',
+  'data': { 'name': 'str', '*tbl-id': 'uint32' },
+  'returns': ['RockerOfDpaFlow'] }
+
+##
+# @RockerOfDpaGroup:
+#
+# Rocker switch OF-DPA group
+#
+# @id: group unique ID
+#
+# @type: group type
+#
+# @vlan-id: #optional VLAN ID
+#
+# @pport: #optional physical port number
+#
+# @index: #optional group index, unique with group type
+#
+# @out-pport: #optional output physical port number
+#
+# @group-id: #optional next group ID
+#
+# @set-vlan-id: #optional VLAN ID to set
+#
+# @pop-vlan: #optional pop VLAN headr from packet
+#
+# @group-ids: #optional list of next group IDs
+#
+# @set-eth-src: #optional set source MAC address in Ethernet header
+#
+# @set-eth-dst: #optional set destination MAC address in Ethernet header
+#
+# @ttl-check: #optional perform TTL check
+#
+# Note: fields are marked #optional to indicate that they may or may not
+# appear in the group depending if they're relevant to the group type.
+#
+# Since: 2.4
+##
+{ 'struct': 'RockerOfDpaGroup',
+  'data': { 'id': 'uint32',  'type': 'uint8', '*vlan-id': 'uint16',
+            '*pport': 'uint32', '*index': 'uint32', '*out-pport': 'uint32',
+            '*group-id': 'uint32', '*set-vlan-id': 'uint16',
+            '*pop-vlan': 'uint8', '*group-ids': ['uint32'],
+            '*set-eth-src': 'str', '*set-eth-dst': 'str',
+            '*ttl-check': 'uint8' } }
+
+##
+# @query-rocker-of-dpa-groups:
+#
+# Return rocker OF-DPA group information.
+#
+# @name: switch name
+#
+# @type: #optional group type.  If type is not specified, returns
+# group information for all group types.
+#
+# Returns: @Rocker OF-DPA group information
+#
+# Since: 2.4
+##
+{ 'command': 'query-rocker-of-dpa-groups',
+  'data': { 'name': 'str', '*type': 'uint8' },
+  'returns': ['RockerOfDpaGroup'] }
diff --git a/qapi/string-input-visitor.c b/qapi/string-input-visitor.c
index d8a8db0..8dfa561 100644
--- a/qapi/string-input-visitor.c
+++ b/qapi/string-input-visitor.c
@@ -1,7 +1,7 @@
 /*
  * String parsing visitor
  *
- * Copyright Red Hat, Inc. 2012
+ * Copyright Red Hat, Inc. 2012-2016
  *
  * Author: Paolo Bonzini <pbonzini@redhat.com>
  *
@@ -10,6 +10,8 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qapi/string-input-visitor.h"
 #include "qapi/visitor-impl.h"
@@ -23,21 +25,25 @@
 {
     Visitor visitor;
 
-    bool head;
-
     GList *ranges;
     GList *cur_range;
     int64_t cur;
 
     const char *string;
+    void *list; /* Only needed for sanity checking the caller */
 };
 
+static StringInputVisitor *to_siv(Visitor *v)
+{
+    return container_of(v, StringInputVisitor, visitor);
+}
+
 static void free_range(void *range, void *dummy)
 {
     g_free(range);
 }
 
-static void parse_str(StringInputVisitor *siv, Error **errp)
+static int parse_str(StringInputVisitor *siv, const char *name, Error **errp)
 {
     char *str = (char *) siv->string;
     long long start, end;
@@ -45,7 +51,7 @@
     char *endptr;
 
     if (siv->ranges) {
-        return;
+        return 0;
     }
 
     do {
@@ -54,10 +60,8 @@
         if (errno == 0 && endptr > str) {
             if (*endptr == '\0') {
                 cur = g_malloc0(sizeof(*cur));
-                cur->begin = start;
-                cur->end = start + 1;
-                siv->ranges = g_list_insert_sorted_merged(siv->ranges, cur,
-                                                          range_compare);
+                range_set_bounds(cur, start, start);
+                siv->ranges = range_list_insert(siv->ranges, cur);
                 cur = NULL;
                 str = NULL;
             } else if (*endptr == '-') {
@@ -69,23 +73,15 @@
                      end < start + 65536)) {
                     if (*endptr == '\0') {
                         cur = g_malloc0(sizeof(*cur));
-                        cur->begin = start;
-                        cur->end = end + 1;
-                        siv->ranges =
-                            g_list_insert_sorted_merged(siv->ranges,
-                                                        cur,
-                                                        range_compare);
+                        range_set_bounds(cur, start, end);
+                        siv->ranges = range_list_insert(siv->ranges, cur);
                         cur = NULL;
                         str = NULL;
                     } else if (*endptr == ',') {
                         str = endptr + 1;
                         cur = g_malloc0(sizeof(*cur));
-                        cur->begin = start;
-                        cur->end = end + 1;
-                        siv->ranges =
-                            g_list_insert_sorted_merged(siv->ranges,
-                                                        cur,
-                                                        range_compare);
+                        range_set_bounds(cur, start, end);
+                        siv->ranges = range_list_insert(siv->ranges, cur);
                         cur = NULL;
                     } else {
                         goto error;
@@ -96,11 +92,8 @@
             } else if (*endptr == ',') {
                 str = endptr + 1;
                 cur = g_malloc0(sizeof(*cur));
-                cur->begin = start;
-                cur->end = start + 1;
-                siv->ranges = g_list_insert_sorted_merged(siv->ranges,
-                                                          cur,
-                                                          range_compare);
+                range_set_bounds(cur, start, start);
+                siv->ranges = range_list_insert(siv->ranges, cur);
                 cur = NULL;
             } else {
                 goto error;
@@ -110,34 +103,46 @@
         }
     } while (str);
 
-    return;
+    return 0;
 error:
     g_list_foreach(siv->ranges, free_range, NULL);
     g_list_free(siv->ranges);
     siv->ranges = NULL;
+    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
+               "an int64 value or range");
+    return -1;
 }
 
 static void
-start_list(Visitor *v, const char *name, Error **errp)
+start_list(Visitor *v, const char *name, GenericList **list, size_t size,
+           Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
-    parse_str(siv, errp);
+    /* We don't support visits without a list */
+    assert(list);
+    siv->list = list;
+
+    if (parse_str(siv, name, errp) < 0) {
+        *list = NULL;
+        return;
+    }
 
     siv->cur_range = g_list_first(siv->ranges);
     if (siv->cur_range) {
         Range *r = siv->cur_range->data;
         if (r) {
-            siv->cur = r->begin;
+            siv->cur = range_lob(r);
         }
+        *list = g_malloc0(size);
+    } else {
+        *list = NULL;
     }
 }
 
-static GenericList *
-next_list(Visitor *v, GenericList **list, Error **errp)
+static GenericList *next_list(Visitor *v, GenericList *tail, size_t size)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
-    GenericList **link;
+    StringInputVisitor *siv = to_siv(v);
     Range *r;
 
     if (!siv->ranges || !siv->cur_range) {
@@ -149,7 +154,7 @@
         return NULL;
     }
 
-    if (siv->cur < r->begin || siv->cur >= r->end) {
+    if (!range_contains(r, siv->cur)) {
         siv->cur_range = g_list_next(siv->cur_range);
         if (!siv->cur_range) {
             return NULL;
@@ -158,39 +163,34 @@
         if (!r) {
             return NULL;
         }
-        siv->cur = r->begin;
+        siv->cur = range_lob(r);
     }
 
-    if (siv->head) {
-        link = list;
-        siv->head = false;
-    } else {
-        link = &(*list)->next;
-    }
-
-    *link = g_malloc0(sizeof **link);
-    return *link;
+    tail->next = g_malloc0(size);
+    return tail->next;
 }
 
-static void
-end_list(Visitor *v, Error **errp)
+static void end_list(Visitor *v, void **obj)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
-    siv->head = true;
+    StringInputVisitor *siv = to_siv(v);
+
+    assert(siv->list == obj);
 }
 
-static void parse_type_int(Visitor *v, int64_t *obj, const char *name,
-                           Error **errp)
+static void parse_type_int64(Visitor *v, const char *name, int64_t *obj,
+                             Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     if (!siv->string) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "integer");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "integer");
         return;
     }
 
-    parse_str(siv, errp);
+    if (parse_str(siv, name, errp) < 0) {
+        return;
+    }
 
     if (!siv->ranges) {
         goto error;
@@ -209,7 +209,7 @@
             goto error;
         }
 
-        siv->cur = r->begin;
+        siv->cur = range_lob(r);
     }
 
     *obj = siv->cur;
@@ -217,22 +217,36 @@
     return;
 
 error:
-    error_set(errp, QERR_INVALID_PARAMETER_VALUE, name,
-              "an int64 value or range");
+    error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name ? name : "null",
+               "an int64 value or range");
 }
 
-static void parse_type_size(Visitor *v, uint64_t *obj, const char *name,
+static void parse_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                              Error **errp)
+{
+    /* FIXME: parse_type_int64 mishandles values over INT64_MAX */
+    int64_t i;
+    Error *err = NULL;
+    parse_type_int64(v, name, &i, &err);
+    if (err) {
+        error_propagate(errp, err);
+    } else {
+        *obj = i;
+    }
+}
+
+static void parse_type_size(Visitor *v, const char *name, uint64_t *obj,
                             Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     Error *err = NULL;
     uint64_t val;
 
     if (siv->string) {
         parse_option_size(name, siv->string, &val, &err);
     } else {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "size");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "size");
         return;
     }
     if (err) {
@@ -243,10 +257,10 @@
     *obj = val;
 }
 
-static void parse_type_bool(Visitor *v, bool *obj, const char *name,
+static void parse_type_bool(Visitor *v, const char *name, bool *obj,
                             Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     if (siv->string) {
         if (!strcasecmp(siv->string, "on") ||
@@ -263,26 +277,27 @@
         }
     }
 
-    error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-              "boolean");
+    error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+               "boolean");
 }
 
-static void parse_type_str(Visitor *v, char **obj, const char *name,
+static void parse_type_str(Visitor *v, const char *name, char **obj,
                            Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     if (siv->string) {
         *obj = g_strdup(siv->string);
     } else {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "string");
+        *obj = NULL;
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "string");
     }
 }
 
-static void parse_type_number(Visitor *v, double *obj, const char *name,
+static void parse_type_number(Visitor *v, const char *name, double *obj,
                               Error **errp)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
     char *endp = (char *) siv->string;
     double val;
 
@@ -291,18 +306,17 @@
         val = strtod(siv->string, &endp);
     }
     if (!siv->string || errno || endp == siv->string || *endp) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
-                  "number");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "number");
         return;
     }
 
     *obj = val;
 }
 
-static void parse_optional(Visitor *v, bool *present, const char *name,
-                           Error **errp)
+static void parse_optional(Visitor *v, const char *name, bool *present)
 {
-    StringInputVisitor *siv = DO_UPCAST(StringInputVisitor, visitor, v);
+    StringInputVisitor *siv = to_siv(v);
 
     if (!siv->string) {
         *present = false;
@@ -312,26 +326,24 @@
     *present = true;
 }
 
-Visitor *string_input_get_visitor(StringInputVisitor *v)
+static void string_input_free(Visitor *v)
 {
-    return &v->visitor;
+    StringInputVisitor *siv = to_siv(v);
+
+    g_list_foreach(siv->ranges, free_range, NULL);
+    g_list_free(siv->ranges);
+    g_free(siv);
 }
 
-void string_input_visitor_cleanup(StringInputVisitor *v)
-{
-    g_list_foreach(v->ranges, free_range, NULL);
-    g_list_free(v->ranges);
-    g_free(v);
-}
-
-StringInputVisitor *string_input_visitor_new(const char *str)
+Visitor *string_input_visitor_new(const char *str)
 {
     StringInputVisitor *v;
 
     v = g_malloc0(sizeof(*v));
 
-    v->visitor.type_enum = input_type_enum;
-    v->visitor.type_int = parse_type_int;
+    v->visitor.type = VISITOR_INPUT;
+    v->visitor.type_int64 = parse_type_int64;
+    v->visitor.type_uint64 = parse_type_uint64;
     v->visitor.type_size = parse_type_size;
     v->visitor.type_bool = parse_type_bool;
     v->visitor.type_str = parse_type_str;
@@ -340,8 +352,8 @@
     v->visitor.next_list = next_list;
     v->visitor.end_list = end_list;
     v->visitor.optional = parse_optional;
+    v->visitor.free = string_input_free;
 
     v->string = str;
-    v->head = true;
-    return v;
+    return &v->visitor;
 }
diff --git a/qapi/string-output-visitor.c b/qapi/string-output-visitor.c
index 1ab8574..94ac821 100644
--- a/qapi/string-output-visitor.c
+++ b/qapi/string-output-visitor.c
@@ -1,7 +1,7 @@
 /*
  * String printing Visitor
  *
- * Copyright Red Hat, Inc. 2012
+ * Copyright Red Hat, Inc. 2012-2016
  *
  * Author: Paolo Bonzini <pbonzini@redhat.com>
  *
@@ -10,17 +10,17 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/string-output-visitor.h"
 #include "qapi/visitor-impl.h"
-#include "qapi/qmp/qerror.h"
 #include "qemu/host-utils.h"
 #include <math.h>
 #include "qemu/range.h"
 
 enum ListMode {
     LM_NONE,             /* not traversing a list of repeated options */
-    LM_STARTED,          /* start_list() succeeded */
+    LM_STARTED,          /* next_list() ready to be called */
 
     LM_IN_PROGRESS,      /* next_list() has been called.
                           *
@@ -48,7 +48,7 @@
 
     LM_UNSIGNED_INTERVAL,/* Same as above, only for an unsigned interval. */
 
-    LM_END
+    LM_END,              /* next_list() called, about to see last element. */
 };
 
 typedef enum ListMode ListMode;
@@ -58,15 +58,21 @@
     Visitor visitor;
     bool human;
     GString *string;
-    bool head;
+    char **result;
     ListMode list_mode;
     union {
         int64_t s;
         uint64_t u;
     } range_start, range_end;
     GList *ranges;
+    void *list; /* Only needed for sanity checking the caller */
 };
 
+static StringOutputVisitor *to_sov(Visitor *v)
+{
+    return container_of(v, StringOutputVisitor, visitor);
+}
+
 static void string_output_set(StringOutputVisitor *sov, char *string)
 {
     if (sov->string) {
@@ -79,37 +85,37 @@
 static void string_output_append(StringOutputVisitor *sov, int64_t a)
 {
     Range *r = g_malloc0(sizeof(*r));
-    r->begin = a;
-    r->end = a + 1;
-    sov->ranges = g_list_insert_sorted_merged(sov->ranges, r, range_compare);
+
+    range_set_bounds(r, a, a);
+    sov->ranges = range_list_insert(sov->ranges, r);
 }
 
 static void string_output_append_range(StringOutputVisitor *sov,
                                        int64_t s, int64_t e)
 {
     Range *r = g_malloc0(sizeof(*r));
-    r->begin = s;
-    r->end = e + 1;
-    sov->ranges = g_list_insert_sorted_merged(sov->ranges, r, range_compare);
+
+    range_set_bounds(r, s, e);
+    sov->ranges = range_list_insert(sov->ranges, r);
 }
 
 static void format_string(StringOutputVisitor *sov, Range *r, bool next,
                           bool human)
 {
-    if (r->end - r->begin > 1) {
+    if (range_lob(r) != range_upb(r)) {
         if (human) {
             g_string_append_printf(sov->string, "0x%" PRIx64 "-0x%" PRIx64,
-                                   r->begin, r->end - 1);
+                                   range_lob(r), range_upb(r));
 
         } else {
             g_string_append_printf(sov->string, "%" PRId64 "-%" PRId64,
-                                   r->begin, r->end - 1);
+                                   range_lob(r), range_upb(r));
         }
     } else {
         if (human) {
-            g_string_append_printf(sov->string, "0x%" PRIx64, r->begin);
+            g_string_append_printf(sov->string, "0x%" PRIx64, range_lob(r));
         } else {
-            g_string_append_printf(sov->string, "%" PRId64, r->begin);
+            g_string_append_printf(sov->string, "%" PRId64, range_lob(r));
         }
     }
     if (next) {
@@ -117,10 +123,10 @@
     }
 }
 
-static void print_type_int(Visitor *v, int64_t *obj, const char *name,
-                           Error **errp)
+static void print_type_int64(Visitor *v, const char *name, int64_t *obj,
+                             Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     GList *l;
 
     switch (sov->list_mode) {
@@ -193,10 +199,18 @@
     }
 }
 
-static void print_type_size(Visitor *v, uint64_t *obj, const char *name,
-                           Error **errp)
+static void print_type_uint64(Visitor *v, const char *name, uint64_t *obj,
+                             Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    /* FIXME: print_type_int64 mishandles values over INT64_MAX */
+    int64_t i = *obj;
+    print_type_int64(v, name, &i, errp);
+}
+
+static void print_type_size(Visitor *v, const char *name, uint64_t *obj,
+                            Error **errp)
+{
+    StringOutputVisitor *sov = to_sov(v);
     static const char suffixes[] = { 'B', 'K', 'M', 'G', 'T', 'P', 'E' };
     uint64_t div, val;
     char *out;
@@ -224,17 +238,17 @@
     string_output_set(sov, out);
 }
 
-static void print_type_bool(Visitor *v, bool *obj, const char *name,
+static void print_type_bool(Visitor *v, const char *name, bool *obj,
                             Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     string_output_set(sov, g_strdup(*obj ? "true" : "false"));
 }
 
-static void print_type_str(Visitor *v, char **obj, const char *name,
+static void print_type_str(Visitor *v, const char *name, char **obj,
                            Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     char *out;
 
     if (sov->human) {
@@ -245,75 +259,60 @@
     string_output_set(sov, out);
 }
 
-static void print_type_number(Visitor *v, double *obj, const char *name,
+static void print_type_number(Visitor *v, const char *name, double *obj,
                               Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
     string_output_set(sov, g_strdup_printf("%f", *obj));
 }
 
 static void
-start_list(Visitor *v, const char *name, Error **errp)
+start_list(Visitor *v, const char *name, GenericList **list, size_t size,
+           Error **errp)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
 
     /* we can't traverse a list in a list */
     assert(sov->list_mode == LM_NONE);
-    sov->list_mode = LM_STARTED;
-    sov->head = true;
+    /* We don't support visits without a list */
+    assert(list);
+    sov->list = list;
+    /* List handling is only needed if there are at least two elements */
+    if (*list && (*list)->next) {
+        sov->list_mode = LM_STARTED;
+    }
 }
 
-static GenericList *
-next_list(Visitor *v, GenericList **list, Error **errp)
+static GenericList *next_list(Visitor *v, GenericList *tail, size_t size)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
-    GenericList *ret = NULL;
-    if (*list) {
-        if (sov->head) {
-            ret = *list;
-        } else {
-            ret = (*list)->next;
-        }
+    StringOutputVisitor *sov = to_sov(v);
+    GenericList *ret = tail->next;
 
-        if (sov->head) {
-            if (ret && ret->next == NULL) {
-                sov->list_mode = LM_NONE;
-            }
-            sov->head = false;
-        } else {
-            if (ret && ret->next == NULL) {
-                sov->list_mode = LM_END;
-            }
-        }
+    if (ret && !ret->next) {
+        sov->list_mode = LM_END;
     }
-
     return ret;
 }
 
-static void
-end_list(Visitor *v, Error **errp)
+static void end_list(Visitor *v, void **obj)
 {
-    StringOutputVisitor *sov = DO_UPCAST(StringOutputVisitor, visitor, v);
+    StringOutputVisitor *sov = to_sov(v);
 
+    assert(sov->list == obj);
     assert(sov->list_mode == LM_STARTED ||
            sov->list_mode == LM_END ||
            sov->list_mode == LM_NONE ||
            sov->list_mode == LM_IN_PROGRESS);
     sov->list_mode = LM_NONE;
-    sov->head = true;
-
 }
 
-char *string_output_get_string(StringOutputVisitor *sov)
+static void string_output_complete(Visitor *v, void *opaque)
 {
-    char *string = g_string_free(sov->string, false);
+    StringOutputVisitor *sov = to_sov(v);
+
+    assert(opaque == sov->result);
+    *sov->result = g_string_free(sov->string, false);
     sov->string = NULL;
-    return string;
-}
-
-Visitor *string_output_get_visitor(StringOutputVisitor *sov)
-{
-    return &sov->visitor;
 }
 
 static void free_range(void *range, void *dummy)
@@ -321,8 +320,10 @@
     g_free(range);
 }
 
-void string_output_visitor_cleanup(StringOutputVisitor *sov)
+static void string_output_free(Visitor *v)
 {
+    StringOutputVisitor *sov = to_sov(v);
+
     if (sov->string) {
         g_string_free(sov->string, true);
     }
@@ -332,7 +333,7 @@
     g_free(sov);
 }
 
-StringOutputVisitor *string_output_visitor_new(bool human)
+Visitor *string_output_visitor_new(bool human, char **result)
 {
     StringOutputVisitor *v;
 
@@ -340,8 +341,12 @@
 
     v->string = g_string_new(NULL);
     v->human = human;
-    v->visitor.type_enum = output_type_enum;
-    v->visitor.type_int = print_type_int;
+    v->result = result;
+    *result = NULL;
+
+    v->visitor.type = VISITOR_OUTPUT;
+    v->visitor.type_int64 = print_type_int64;
+    v->visitor.type_uint64 = print_type_uint64;
     v->visitor.type_size = print_type_size;
     v->visitor.type_bool = print_type_bool;
     v->visitor.type_str = print_type_str;
@@ -349,6 +354,8 @@
     v->visitor.start_list = start_list;
     v->visitor.next_list = next_list;
     v->visitor.end_list = end_list;
+    v->visitor.complete = string_output_complete;
+    v->visitor.free = string_output_free;
 
-    return v;
+    return &v->visitor;
 }
diff --git a/qapi/trace.json b/qapi/trace.json
index 06c613c..e872146 100644
--- a/qapi/trace.json
+++ b/qapi/trace.json
@@ -1,6 +1,6 @@
 # -*- mode: python -*-
 #
-# Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
+# Copyright (C) 2011-2016 Lluís Vilanova <vilanova@ac.upc.edu>
 #
 # This work is licensed under the terms of the GNU GPL, version 2 or later.
 # See the COPYING file in the top-level directory.
@@ -29,11 +29,15 @@
 #
 # @name: Event name.
 # @state: Tracing state.
+# @vcpu: Whether this is a per-vCPU event (since 2.7).
+#
+# An event is per-vCPU if it has the "vcpu" property in the "trace-events"
+# files.
 #
 # Since 2.2
 ##
-{ 'type': 'TraceEventInfo',
-  'data': {'name': 'str', 'state': 'TraceEventState'} }
+{ 'struct': 'TraceEventInfo',
+  'data': {'name': 'str', 'state': 'TraceEventState', 'vcpu': 'bool'} }
 
 ##
 # @trace-event-get-state:
@@ -41,13 +45,23 @@
 # Query the state of events.
 #
 # @name: Event name pattern (case-sensitive glob).
+# @vcpu: #optional The vCPU to query (any by default; since 2.7).
 #
 # Returns: a list of @TraceEventInfo for the matching events
 #
+# An event is returned if:
+# - its name matches the @name pattern, and
+# - if @vcpu is given, the event has the "vcpu" property.
+#
+# Therefore, if @vcpu is given, the operation will only match per-vCPU events,
+# returning their state on the specified vCPU. Special case: if @name is an
+# exact match, @vcpu is given and the event does not have the "vcpu" property,
+# an error is returned.
+#
 # Since 2.2
 ##
 { 'command': 'trace-event-get-state',
-  'data': {'name': 'str'},
+  'data': {'name': 'str', '*vcpu': 'int'},
   'returns': ['TraceEventInfo'] }
 
 ##
@@ -58,8 +72,19 @@
 # @name: Event name pattern (case-sensitive glob).
 # @enable: Whether to enable tracing.
 # @ignore-unavailable: #optional Do not match unavailable events with @name.
+# @vcpu: #optional The vCPU to act upon (all by default; since 2.7).
+#
+# An event's state is modified if:
+# - its name matches the @name pattern, and
+# - if @vcpu is given, the event has the "vcpu" property.
+#
+# Therefore, if @vcpu is given, the operation will only match per-vCPU events,
+# setting their state on the specified vCPU. Special case: if @name is an exact
+# match, @vcpu is given and the event does not have the "vcpu" property, an
+# error is returned.
 #
 # Since 2.2
 ##
 { 'command': 'trace-event-set-state',
-  'data': {'name': 'str', 'enable': 'bool', '*ignore-unavailable': 'bool'} }
+  'data': {'name': 'str', 'enable': 'bool', '*ignore-unavailable': 'bool',
+           '*vcpu': 'int'} }
diff --git a/qdev-monitor.c b/qdev-monitor.c
index ebfa701..e19617f 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -17,13 +17,17 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "hw/sysbus.h"
 #include "monitor/monitor.h"
 #include "monitor/qdev.h"
 #include "qmp-commands.h"
 #include "sysemu/arch_init.h"
+#include "qapi/qmp/qerror.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
+#include "qemu/help_option.h"
 
 /*
  * Aliases were a bad idea from the start.  Let's keep them
@@ -36,18 +40,39 @@
     uint32_t arch_mask;
 } QDevAlias;
 
+/* Please keep this table sorted by typename. */
 static const QDevAlias qdev_alias_table[] = {
-    { "virtio-blk-pci", "virtio-blk", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
-    { "virtio-net-pci", "virtio-net", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
-    { "virtio-serial-pci", "virtio-serial", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
-    { "virtio-balloon-pci", "virtio-balloon",
-            QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
-    { "virtio-blk-s390", "virtio-blk", QEMU_ARCH_S390X },
-    { "virtio-net-s390", "virtio-net", QEMU_ARCH_S390X },
-    { "virtio-serial-s390", "virtio-serial", QEMU_ARCH_S390X },
-    { "lsi53c895a", "lsi" },
+    { "e1000", "e1000-82540em" },
     { "ich9-ahci", "ahci" },
     { "kvm-pci-assign", "pci-assign" },
+    { "lsi53c895a", "lsi" },
+    { "virtio-9p-ccw", "virtio-9p", QEMU_ARCH_S390X },
+    { "virtio-9p-pci", "virtio-9p", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-balloon-ccw", "virtio-balloon", QEMU_ARCH_S390X },
+    { "virtio-balloon-pci", "virtio-balloon",
+            QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-blk-ccw", "virtio-blk", QEMU_ARCH_S390X },
+    { "virtio-blk-pci", "virtio-blk", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-gpu-ccw", "virtio-gpu", QEMU_ARCH_S390X },
+    { "virtio-gpu-pci", "virtio-gpu", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-input-host-ccw", "virtio-input-host", QEMU_ARCH_S390X },
+    { "virtio-input-host-pci", "virtio-input-host",
+            QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-keyboard-ccw", "virtio-keyboard", QEMU_ARCH_S390X },
+    { "virtio-keyboard-pci", "virtio-keyboard",
+            QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-mouse-ccw", "virtio-mouse", QEMU_ARCH_S390X },
+    { "virtio-mouse-pci", "virtio-mouse", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-net-ccw", "virtio-net", QEMU_ARCH_S390X },
+    { "virtio-net-pci", "virtio-net", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-rng-ccw", "virtio-rng", QEMU_ARCH_S390X },
+    { "virtio-rng-pci", "virtio-rng", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-scsi-ccw", "virtio-scsi", QEMU_ARCH_S390X },
+    { "virtio-scsi-pci", "virtio-scsi", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-serial-ccw", "virtio-serial", QEMU_ARCH_S390X },
+    { "virtio-serial-pci", "virtio-serial", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
+    { "virtio-tablet-ccw", "virtio-tablet", QEMU_ARCH_S390X },
+    { "virtio-tablet-pci", "virtio-tablet", QEMU_ARCH_ALL & ~QEMU_ARCH_S390X },
     { }
 };
 
@@ -143,7 +168,8 @@
     g_slist_free(list);
 }
 
-static int set_property(const char *name, const char *value, void *opaque)
+static int set_property(void *opaque, const char *name, const char *value,
+                        Error **errp)
 {
     Object *obj = opaque;
     Error *err = NULL;
@@ -155,8 +181,7 @@
 
     object_property_parse(obj, value, name, &err);
     if (err != NULL) {
-        qerror_report_err(err);
-        error_free(err);
+        error_propagate(errp, err);
         return -1;
     }
     return 0;
@@ -184,6 +209,7 @@
 {
     ObjectClass *oc;
     DeviceClass *dc;
+    const char *original_name = *driver;
 
     oc = object_class_by_name(*driver);
     if (!oc) {
@@ -196,21 +222,26 @@
     }
 
     if (!object_class_dynamic_cast(oc, TYPE_DEVICE)) {
-        error_setg(errp, "'%s' is not a valid device model name", *driver);
+        if (*driver != original_name) {
+            error_setg(errp, "'%s' (alias '%s') is not a valid device model"
+                       " name", original_name, *driver);
+        } else {
+            error_setg(errp, "'%s' is not a valid device model name", *driver);
+        }
         return NULL;
     }
 
     if (object_class_is_abstract(oc)) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "driver",
-                  "non-abstract device type");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "driver",
+                   "non-abstract device type");
         return NULL;
     }
 
     dc = DEVICE_CLASS(oc);
     if (dc->cannot_instantiate_with_device_add_yet ||
         (qdev_hotplug && !dc->hotpluggable)) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "driver",
-                  "pluggable device type");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "driver",
+                   "pluggable device type");
         return NULL;
     }
 
@@ -235,9 +266,12 @@
         return 0;
     }
 
-    qdev_get_device_class(&driver, &local_err);
-    if (local_err) {
-        goto error;
+    if (!object_class_by_name(driver)) {
+        const char *typename = find_typename_by_alias(driver);
+
+        if (typename) {
+            driver = typename;
+        }
     }
 
     prop_list = qmp_device_list_properties(driver, &local_err);
@@ -260,8 +294,7 @@
     return 1;
 
 error:
-    error_printf("%s\n", error_get_pretty(local_err));
-    error_free(local_err);
+    error_report_err(local_err);
     return 1;
 }
 
@@ -287,34 +320,36 @@
     return dev;
 }
 
-static void qbus_list_bus(DeviceState *dev)
+static void qbus_list_bus(DeviceState *dev, Error **errp)
 {
     BusState *child;
     const char *sep = " ";
 
-    error_printf("child busses at \"%s\":",
-                 dev->id ? dev->id : object_get_typename(OBJECT(dev)));
+    error_append_hint(errp, "child buses at \"%s\":",
+                      dev->id ? dev->id : object_get_typename(OBJECT(dev)));
     QLIST_FOREACH(child, &dev->child_bus, sibling) {
-        error_printf("%s\"%s\"", sep, child->name);
+        error_append_hint(errp, "%s\"%s\"", sep, child->name);
         sep = ", ";
     }
-    error_printf("\n");
+    error_append_hint(errp, "\n");
 }
 
-static void qbus_list_dev(BusState *bus)
+static void qbus_list_dev(BusState *bus, Error **errp)
 {
     BusChild *kid;
     const char *sep = " ";
 
-    error_printf("devices at \"%s\":", bus->name);
+    error_append_hint(errp, "devices at \"%s\":", bus->name);
     QTAILQ_FOREACH(kid, &bus->children, sibling) {
         DeviceState *dev = kid->child;
-        error_printf("%s\"%s\"", sep, object_get_typename(OBJECT(dev)));
-        if (dev->id)
-            error_printf("/\"%s\"", dev->id);
+        error_append_hint(errp, "%s\"%s\"", sep,
+                          object_get_typename(OBJECT(dev)));
+        if (dev->id) {
+            error_append_hint(errp, "/\"%s\"", dev->id);
+        }
         sep = ", ";
     }
-    error_printf("\n");
+    error_append_hint(errp, "\n");
 }
 
 static BusState *qbus_find_bus(DeviceState *dev, char *elem)
@@ -363,46 +398,58 @@
     return NULL;
 }
 
+static inline bool qbus_is_full(BusState *bus)
+{
+    BusClass *bus_class = BUS_GET_CLASS(bus);
+    return bus_class->max_dev && bus->max_index >= bus_class->max_dev;
+}
+
+/*
+ * Search the tree rooted at @bus for a bus.
+ * If @name, search for a bus with that name.  Note that bus names
+ * need not be unique.  Yes, that's screwed up.
+ * Else search for a bus that is a subtype of @bus_typename.
+ * If more than one exists, prefer one that can take another device.
+ * Return the bus if found, else %NULL.
+ */
 static BusState *qbus_find_recursive(BusState *bus, const char *name,
                                      const char *bus_typename)
 {
-    BusClass *bus_class = BUS_GET_CLASS(bus);
     BusChild *kid;
-    BusState *child, *ret;
-    int match = 1;
+    BusState *pick, *child, *ret;
+    bool match;
 
-    if (name && (strcmp(bus->name, name) != 0)) {
-        match = 0;
-    } else if (bus_typename && !object_dynamic_cast(OBJECT(bus), bus_typename)) {
-        match = 0;
-    } else if ((bus_class->max_dev != 0) && (bus_class->max_dev <= bus->max_index)) {
-        if (name != NULL) {
-            /* bus was explicitly specified: return an error. */
-            qerror_report(ERROR_CLASS_GENERIC_ERROR, "Bus '%s' is full",
-                          bus->name);
-            return NULL;
-        } else {
-            /* bus was not specified: try to find another one. */
-            match = 0;
-        }
+    assert(name || bus_typename);
+    if (name) {
+        match = !strcmp(bus->name, name);
+    } else {
+        match = !!object_dynamic_cast(OBJECT(bus), bus_typename);
     }
-    if (match) {
-        return bus;
+
+    if (match && !qbus_is_full(bus)) {
+        return bus;             /* root matches and isn't full */
     }
 
+    pick = match ? bus : NULL;
+
     QTAILQ_FOREACH(kid, &bus->children, sibling) {
         DeviceState *dev = kid->child;
         QLIST_FOREACH(child, &dev->child_bus, sibling) {
             ret = qbus_find_recursive(child, name, bus_typename);
-            if (ret) {
-                return ret;
+            if (ret && !qbus_is_full(ret)) {
+                return ret;     /* a descendant matches and isn't full */
+            }
+            if (ret && !pick) {
+                pick = ret;
             }
         }
     }
-    return NULL;
+
+    /* root or a descendant matches, but is full */
+    return pick;
 }
 
-static BusState *qbus_find(const char *path)
+static BusState *qbus_find(const char *path, Error **errp)
 {
     DeviceState *dev;
     BusState *bus;
@@ -420,7 +467,7 @@
         }
         bus = qbus_find_recursive(sysbus_get_default(), elem, NULL);
         if (!bus) {
-            qerror_report(QERR_BUS_NOT_FOUND, elem);
+            error_setg(errp, "Bus '%s' not found", elem);
             return NULL;
         }
         pos = len;
@@ -432,7 +479,7 @@
             pos++;
         }
         if (path[pos] == '\0') {
-            return bus;
+            break;
         }
 
         /* find device */
@@ -443,10 +490,9 @@
         pos += len;
         dev = qbus_find_dev(bus, elem);
         if (!dev) {
-            qerror_report(QERR_DEVICE_NOT_FOUND, elem);
-            if (!monitor_cur_is_qmp()) {
-                qbus_list_dev(bus);
-            }
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "Device '%s' not found", elem);
+            qbus_list_dev(bus, errp);
             return NULL;
         }
 
@@ -457,21 +503,18 @@
         if (path[pos] == '\0') {
             /* last specified element is a device.  If it has exactly
              * one child bus accept it nevertheless */
-            switch (dev->num_child_bus) {
-            case 0:
-                qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                              "Device '%s' has no child bus", elem);
-                return NULL;
-            case 1:
-                return QLIST_FIRST(&dev->child_bus);
-            default:
-                qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                              "Device '%s' has multiple child busses", elem);
-                if (!monitor_cur_is_qmp()) {
-                    qbus_list_bus(dev);
-                }
-                return NULL;
+            if (dev->num_child_bus == 1) {
+                bus = QLIST_FIRST(&dev->child_bus);
+                break;
             }
+            if (dev->num_child_bus) {
+                error_setg(errp, "Device '%s' has multiple child buses",
+                           elem);
+                qbus_list_bus(dev, errp);
+            } else {
+                error_setg(errp, "Device '%s' has no child bus", elem);
+            }
+            return NULL;
         }
 
         /* find bus */
@@ -482,16 +525,20 @@
         pos += len;
         bus = qbus_find_bus(dev, elem);
         if (!bus) {
-            qerror_report(QERR_BUS_NOT_FOUND, elem);
-            if (!monitor_cur_is_qmp()) {
-                qbus_list_bus(dev);
-            }
+            error_setg(errp, "Bus '%s' not found", elem);
+            qbus_list_bus(dev, errp);
             return NULL;
         }
     }
+
+    if (qbus_is_full(bus)) {
+        error_setg(errp, "Bus '%s' is full", path);
+        return NULL;
+    }
+    return bus;
 }
 
-DeviceState *qdev_device_add(QemuOpts *opts)
+DeviceState *qdev_device_add(QemuOpts *opts, Error **errp)
 {
     DeviceClass *dc;
     const char *driver, *path, *id;
@@ -501,42 +548,38 @@
 
     driver = qemu_opt_get(opts, "driver");
     if (!driver) {
-        qerror_report(QERR_MISSING_PARAMETER, "driver");
+        error_setg(errp, QERR_MISSING_PARAMETER, "driver");
         return NULL;
     }
 
     /* find driver */
-    dc = qdev_get_device_class(&driver, &err);
-    if (err) {
-        qerror_report_err(err);
-        error_free(err);
+    dc = qdev_get_device_class(&driver, errp);
+    if (!dc) {
         return NULL;
     }
 
     /* find bus */
     path = qemu_opt_get(opts, "bus");
     if (path != NULL) {
-        bus = qbus_find(path);
+        bus = qbus_find(path, errp);
         if (!bus) {
             return NULL;
         }
         if (!object_dynamic_cast(OBJECT(bus), dc->bus_type)) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                          "Device '%s' can't go on a %s bus",
-                          driver, object_get_typename(OBJECT(bus)));
+            error_setg(errp, "Device '%s' can't go on %s bus",
+                       driver, object_get_typename(OBJECT(bus)));
             return NULL;
         }
     } else if (dc->bus_type != NULL) {
         bus = qbus_find_recursive(sysbus_get_default(), NULL, dc->bus_type);
-        if (!bus) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                          "No '%s' bus found for device '%s'",
-                          dc->bus_type, driver);
+        if (!bus || qbus_is_full(bus)) {
+            error_setg(errp, "No '%s' bus found for device '%s'",
+                       dc->bus_type, driver);
             return NULL;
         }
     }
     if (qdev_hotplug && bus && !qbus_is_hotpluggable(bus)) {
-        qerror_report(QERR_BUS_NO_HOTPLUG, bus->name);
+        error_setg(errp, QERR_BUS_NO_HOTPLUG, bus->name);
         return NULL;
     }
 
@@ -564,7 +607,8 @@
     }
 
     /* set properties */
-    if (qemu_opt_foreach(opts, set_property, dev, 1) != 0) {
+    if (qemu_opt_foreach(opts, set_property, dev, &err)) {
+        error_propagate(errp, err);
         object_unparent(OBJECT(dev));
         object_unref(OBJECT(dev));
         return NULL;
@@ -573,12 +617,10 @@
     dev->opts = opts;
     object_property_set_bool(OBJECT(dev), true, "realized", &err);
     if (err != NULL) {
-        qerror_report_err(err);
-        error_free(err);
+        error_propagate(errp, err);
         dev->opts = NULL;
         object_unparent(OBJECT(dev));
         object_unref(OBJECT(dev));
-        qerror_report(QERR_DEVICE_INIT_FAILED, driver);
         return NULL;
     }
     return dev;
@@ -667,18 +709,75 @@
 }
 #undef qdev_printf
 
-void do_info_qtree(Monitor *mon, const QDict *qdict)
+void hmp_info_qtree(Monitor *mon, const QDict *qdict)
 {
     if (sysbus_get_default())
         qbus_print(mon, sysbus_get_default(), 0);
 }
 
-void do_info_qdm(Monitor *mon, const QDict *qdict)
+void hmp_info_qdm(Monitor *mon, const QDict *qdict)
 {
     qdev_print_devinfos(true);
 }
 
-int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data)
+typedef struct QOMCompositionState {
+    Monitor *mon;
+    int indent;
+} QOMCompositionState;
+
+static void print_qom_composition(Monitor *mon, Object *obj, int indent);
+
+static int print_qom_composition_child(Object *obj, void *opaque)
+{
+    QOMCompositionState *s = opaque;
+
+    print_qom_composition(s->mon, obj, s->indent);
+
+    return 0;
+}
+
+static void print_qom_composition(Monitor *mon, Object *obj, int indent)
+{
+    QOMCompositionState s = {
+        .mon = mon,
+        .indent = indent + 2,
+    };
+    char *name;
+
+    if (obj == object_get_root()) {
+        name = g_strdup("");
+    } else {
+        name = object_get_canonical_path_component(obj);
+    }
+    monitor_printf(mon, "%*s/%s (%s)\n", indent, "", name,
+                   object_get_typename(obj));
+    g_free(name);
+    object_child_foreach(obj, print_qom_composition_child, &s);
+}
+
+void hmp_info_qom_tree(Monitor *mon, const QDict *dict)
+{
+    const char *path = qdict_get_try_str(dict, "path");
+    Object *obj;
+    bool ambiguous = false;
+
+    if (path) {
+        obj = object_resolve_path(path, &ambiguous);
+        if (!obj) {
+            monitor_printf(mon, "Path '%s' could not be resolved.\n", path);
+            return;
+        }
+        if (ambiguous) {
+            monitor_printf(mon, "Warning: Path '%s' is ambiguous.\n", path);
+            return;
+        }
+    } else {
+        obj = qdev_get_machine();
+    }
+    print_qom_composition(mon, obj, 0);
+}
+
+void qmp_device_add(QDict *qdict, QObject **ret_data, Error **errp)
 {
     Error *local_err = NULL;
     QemuOpts *opts;
@@ -686,35 +785,45 @@
 
     opts = qemu_opts_from_qdict(qemu_find_opts("device"), qdict, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
+        error_propagate(errp, local_err);
+        return;
     }
     if (!monitor_cur_is_qmp() && qdev_device_help(opts)) {
         qemu_opts_del(opts);
-        return 0;
+        return;
     }
-    dev = qdev_device_add(opts);
+    dev = qdev_device_add(opts, &local_err);
     if (!dev) {
+        error_propagate(errp, local_err);
         qemu_opts_del(opts);
-        return -1;
+        return;
     }
     object_unref(OBJECT(dev));
-    return 0;
 }
 
 void qmp_device_del(const char *id, Error **errp)
 {
     Object *obj;
-    char *root_path = object_get_canonical_path(qdev_get_peripheral());
-    char *path = g_strdup_printf("%s/%s", root_path, id);
 
-    g_free(root_path);
-    obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
-    g_free(path);
+    if (id[0] == '/') {
+        obj = object_resolve_path(id, NULL);
+    } else {
+        char *root_path = object_get_canonical_path(qdev_get_peripheral());
+        char *path = g_strdup_printf("%s/%s", root_path, id);
+
+        g_free(root_path);
+        obj = object_resolve_path_type(path, TYPE_DEVICE, NULL);
+        g_free(path);
+    }
 
     if (!obj) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, id);
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", id);
+        return;
+    }
+
+    if (!object_dynamic_cast(obj, TYPE_DEVICE)) {
+        error_setg(errp, "%s is not a hotpluggable device", id);
         return;
     }
 
@@ -765,15 +874,19 @@
     QemuOpts *opts;
     int rc, offset;
 
-    rc = sscanf(str, "%63[^.].%63[^=]%n", driver, property, &offset);
-    if (rc < 2 || str[offset] != '=') {
-        error_report("can't parse: \"%s\"", str);
+    rc = sscanf(str, "%63[^.=].%63[^=]%n", driver, property, &offset);
+    if (rc == 2 && str[offset] == '=') {
+        opts = qemu_opts_create(&qemu_global_opts, NULL, 0, &error_abort);
+        qemu_opt_set(opts, "driver", driver, &error_abort);
+        qemu_opt_set(opts, "property", property, &error_abort);
+        qemu_opt_set(opts, "value", str + offset + 1, &error_abort);
+        return 0;
+    }
+
+    opts = qemu_opts_parse_noisily(&qemu_global_opts, str, false);
+    if (!opts) {
         return -1;
     }
 
-    opts = qemu_opts_create(&qemu_global_opts, NULL, 0, &error_abort);
-    qemu_opt_set(opts, "driver", driver);
-    qemu_opt_set(opts, "property", property);
-    qemu_opt_set(opts, "value", str+offset+1);
     return 0;
 }
diff --git a/qemu-bridge-helper.c b/qemu-bridge-helper.c
index 36eb3bc..5396fbf 100644
--- a/qemu-bridge-helper.c
+++ b/qemu-bridge-helper.c
@@ -13,19 +13,9 @@
  *
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
-#include <stdio.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdbool.h>
-#include <ctype.h>
-#include <glib.h>
 
-#include <sys/types.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/un.h>
diff --git a/qemu-char.c b/qemu-char.c
index 22218d5..71a78bc 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -21,40 +21,40 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
+#include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "sysemu/char.h"
 #include "hw/usb.h"
 #include "qmp-commands.h"
-#include "qapi/qmp-input-visitor.h"
-#include "qapi/qmp-output-visitor.h"
+#include "qapi/clone-visitor.h"
 #include "qapi-visit.h"
+#include "qemu/base64.h"
+#include "io/channel-socket.h"
+#include "io/channel-file.h"
+#include "io/channel-tls.h"
+#include "sysemu/replay.h"
 
-#include <unistd.h>
-#include <fcntl.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
 #include <zlib.h>
 
 #ifndef _WIN32
 #include <sys/times.h>
 #include <sys/wait.h>
 #include <termios.h>
-#include <sys/mman.h>
 #include <sys/ioctl.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <net/if.h>
 #include <arpa/inet.h>
-#include <dirent.h>
 #include <netdb.h>
 #include <sys/select.h>
 #ifdef CONFIG_BSD
-#include <sys/stat.h>
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 #include <dev/ppbus/ppi.h>
 #include <dev/ppbus/ppbconf.h>
@@ -68,7 +68,6 @@
 #include <linux/parport.h>
 #endif
 #ifdef __sun__
-#include <sys/stat.h>
 #include <sys/ethernet.h>
 #include <sys/sockio.h>
 #include <netinet/arp.h>
@@ -87,64 +86,39 @@
 
 #define READ_BUF_LEN 4096
 #define READ_RETRIES 10
-#define CHR_MAX_FILENAME_SIZE 256
 #define TCP_MAX_FDS 16
 
 /***********************************************************/
 /* Socket address helpers */
-static void qapi_copy_SocketAddress(SocketAddress **p_dest,
-                                    SocketAddress *src)
+
+static char *SocketAddress_to_str(const char *prefix, SocketAddress *addr,
+                                  bool is_listen, bool is_telnet)
 {
-    QmpOutputVisitor *qov;
-    QmpInputVisitor *qiv;
-    Visitor *ov, *iv;
-    QObject *obj;
-
-    *p_dest = NULL;
-
-    qov = qmp_output_visitor_new();
-    ov = qmp_output_get_visitor(qov);
-    visit_type_SocketAddress(ov, &src, NULL, &error_abort);
-    obj = qmp_output_get_qobject(qov);
-    qmp_output_visitor_cleanup(qov);
-    if (!obj) {
-        return;
-    }
-
-    qiv = qmp_input_visitor_new(obj);
-    iv = qmp_input_get_visitor(qiv);
-    visit_type_SocketAddress(iv, p_dest, NULL, &error_abort);
-    qmp_input_visitor_cleanup(qiv);
-    qobject_decref(obj);
-}
-
-static int SocketAddress_to_str(char *dest, int max_len,
-                                const char *prefix, SocketAddress *addr,
-                                bool is_listen, bool is_telnet)
-{
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        return snprintf(dest, max_len, "%s%s:%s:%s%s", prefix,
-                        is_telnet ? "telnet" : "tcp", addr->inet->host,
-                        addr->inet->port, is_listen ? ",server" : "");
+        return g_strdup_printf("%s%s:%s:%s%s", prefix,
+                               is_telnet ? "telnet" : "tcp",
+                               addr->u.inet.data->host,
+                               addr->u.inet.data->port,
+                               is_listen ? ",server" : "");
         break;
     case SOCKET_ADDRESS_KIND_UNIX:
-        return snprintf(dest, max_len, "%sunix:%s%s", prefix,
-                        addr->q_unix->path, is_listen ? ",server" : "");
+        return g_strdup_printf("%sunix:%s%s", prefix,
+                               addr->u.q_unix.data->path,
+                               is_listen ? ",server" : "");
         break;
     case SOCKET_ADDRESS_KIND_FD:
-        return snprintf(dest, max_len, "%sfd:%s%s", prefix, addr->fd->str,
-                        is_listen ? ",server" : "");
+        return g_strdup_printf("%sfd:%s%s", prefix, addr->u.fd.data->str,
+                               is_listen ? ",server" : "");
         break;
     default:
         abort();
     }
 }
 
-static int sockaddr_to_str(char *dest, int max_len,
-                           struct sockaddr_storage *ss, socklen_t ss_len,
-                           struct sockaddr_storage *ps, socklen_t ps_len,
-                           bool is_listen, bool is_telnet)
+static char *sockaddr_to_str(struct sockaddr_storage *ss, socklen_t ss_len,
+                             struct sockaddr_storage *ps, socklen_t ps_len,
+                             bool is_listen, bool is_telnet)
 {
     char shost[NI_MAXHOST], sserv[NI_MAXSERV];
     char phost[NI_MAXHOST], pserv[NI_MAXSERV];
@@ -153,9 +127,9 @@
     switch (ss->ss_family) {
 #ifndef _WIN32
     case AF_UNIX:
-        return snprintf(dest, max_len, "unix:%s%s",
-                        ((struct sockaddr_un *)(ss))->sun_path,
-                        is_listen ? ",server" : "");
+        return g_strdup_printf("unix:%s%s",
+                               ((struct sockaddr_un *)(ss))->sun_path,
+                               is_listen ? ",server" : "");
 #endif
     case AF_INET6:
         left  = "[";
@@ -166,14 +140,14 @@
                     sserv, sizeof(sserv), NI_NUMERICHOST | NI_NUMERICSERV);
         getnameinfo((struct sockaddr *) ps, ps_len, phost, sizeof(phost),
                     pserv, sizeof(pserv), NI_NUMERICHOST | NI_NUMERICSERV);
-        return snprintf(dest, max_len, "%s:%s%s%s:%s%s <-> %s%s%s:%s",
-                        is_telnet ? "telnet" : "tcp",
-                        left, shost, right, sserv,
-                        is_listen ? ",server" : "",
-                        left, phost, right, pserv);
+        return g_strdup_printf("%s:%s%s%s:%s%s <-> %s%s%s:%s",
+                               is_telnet ? "telnet" : "tcp",
+                               left, shost, right, sserv,
+                               is_listen ? ",server" : "",
+                               left, phost, right, pserv);
 
     default:
-        return snprintf(dest, max_len, "unknown");
+        return g_strdup_printf("unknown");
     }
 }
 
@@ -183,10 +157,33 @@
 static QTAILQ_HEAD(CharDriverStateHead, CharDriverState) chardevs =
     QTAILQ_HEAD_INITIALIZER(chardevs);
 
-CharDriverState *qemu_chr_alloc(void)
+static void qemu_chr_free_common(CharDriverState *chr);
+
+CharDriverState *qemu_chr_alloc(ChardevCommon *backend, Error **errp)
 {
     CharDriverState *chr = g_malloc0(sizeof(CharDriverState));
     qemu_mutex_init(&chr->chr_write_lock);
+
+    if (backend->has_logfile) {
+        int flags = O_WRONLY | O_CREAT;
+        if (backend->has_logappend &&
+            backend->logappend) {
+            flags |= O_APPEND;
+        } else {
+            flags |= O_TRUNC;
+        }
+        chr->logfd = qemu_open(backend->logfile, flags, 0666);
+        if (chr->logfd < 0) {
+            error_setg_errno(errp, errno,
+                             "Unable to open logfile %s",
+                             backend->logfile);
+            g_free(chr);
+            return NULL;
+        }
+    } else {
+        chr->logfd = -1;
+    }
+
     return chr;
 }
 
@@ -212,38 +209,108 @@
     qemu_chr_be_event(s, CHR_EVENT_OPENED);
 }
 
-int qemu_chr_fe_write(CharDriverState *s, const uint8_t *buf, int len)
-{
-    int ret;
 
-    qemu_mutex_lock(&s->chr_write_lock);
-    ret = s->chr_write(s, buf, len);
-    qemu_mutex_unlock(&s->chr_write_lock);
-    return ret;
+/* Not reporting errors from writing to logfile, as logs are
+ * defined to be "best effort" only */
+static void qemu_chr_fe_write_log(CharDriverState *s,
+                                  const uint8_t *buf, size_t len)
+{
+    size_t done = 0;
+    ssize_t ret;
+
+    if (s->logfd < 0) {
+        return;
+    }
+
+    while (done < len) {
+    retry:
+        ret = write(s->logfd, buf + done, len - done);
+        if (ret == -1 && errno == EAGAIN) {
+            g_usleep(100);
+            goto retry;
+        }
+
+        if (ret <= 0) {
+            return;
+        }
+        done += ret;
+    }
 }
 
-int qemu_chr_fe_write_all(CharDriverState *s, const uint8_t *buf, int len)
+static int qemu_chr_fe_write_buffer(CharDriverState *s, const uint8_t *buf, int len, int *offset)
 {
-    int offset = 0;
     int res = 0;
+    *offset = 0;
 
     qemu_mutex_lock(&s->chr_write_lock);
-    while (offset < len) {
-        do {
-            res = s->chr_write(s, buf + offset, len - offset);
-            if (res == -1 && errno == EAGAIN) {
-                g_usleep(100);
-            }
-        } while (res == -1 && errno == EAGAIN);
+    while (*offset < len) {
+    retry:
+        res = s->chr_write(s, buf + *offset, len - *offset);
+        if (res < 0 && errno == EAGAIN) {
+            g_usleep(100);
+            goto retry;
+        }
 
         if (res <= 0) {
             break;
         }
 
-        offset += res;
+        *offset += res;
+    }
+    if (*offset > 0) {
+        qemu_chr_fe_write_log(s, buf, *offset);
     }
     qemu_mutex_unlock(&s->chr_write_lock);
 
+    return res;
+}
+
+int qemu_chr_fe_write(CharDriverState *s, const uint8_t *buf, int len)
+{
+    int ret;
+
+    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
+        int offset;
+        replay_char_write_event_load(&ret, &offset);
+        assert(offset <= len);
+        qemu_chr_fe_write_buffer(s, buf, offset, &offset);
+        return ret;
+    }
+
+    qemu_mutex_lock(&s->chr_write_lock);
+    ret = s->chr_write(s, buf, len);
+
+    if (ret > 0) {
+        qemu_chr_fe_write_log(s, buf, ret);
+    }
+
+    qemu_mutex_unlock(&s->chr_write_lock);
+    
+    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+        replay_char_write_event_save(ret, ret < 0 ? 0 : ret);
+    }
+    
+    return ret;
+}
+
+int qemu_chr_fe_write_all(CharDriverState *s, const uint8_t *buf, int len)
+{
+    int offset;
+    int res;
+
+    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
+        replay_char_write_event_load(&res, &offset);
+        assert(offset <= len);
+        qemu_chr_fe_write_buffer(s, buf, offset, &offset);
+        return res;
+    }
+
+    res = qemu_chr_fe_write_buffer(s, buf, len, &offset);
+
+    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+        replay_char_write_event_save(res, offset);
+    }
+
     if (res < 0) {
         return res;
     }
@@ -258,20 +325,27 @@
     if (!s->chr_sync_read) {
         return 0;
     }
+    
+    if (s->replay && replay_mode == REPLAY_MODE_PLAY) {
+        return replay_char_read_all_load(buf);
+    }
 
     while (offset < len) {
-        do {
-            res = s->chr_sync_read(s, buf + offset, len - offset);
-            if (res == -1 && errno == EAGAIN) {
-                g_usleep(100);
-            }
-        } while (res == -1 && errno == EAGAIN);
+    retry:
+        res = s->chr_sync_read(s, buf + offset, len - offset);
+        if (res == -1 && errno == EAGAIN) {
+            g_usleep(100);
+            goto retry;
+        }
 
         if (res == 0) {
             break;
         }
 
         if (res < 0) {
+            if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+                replay_char_read_all_save_error(res);
+            }
             return res;
         }
 
@@ -282,14 +356,22 @@
         }
     }
 
+    if (s->replay && replay_mode == REPLAY_MODE_RECORD) {
+        replay_char_read_all_save_buf(buf, offset);
+    }
     return offset;
 }
 
 int qemu_chr_fe_ioctl(CharDriverState *s, int cmd, void *arg)
 {
-    if (!s->chr_ioctl)
-        return -ENOTSUP;
-    return s->chr_ioctl(s, cmd, arg);
+    int res;
+    if (!s->chr_ioctl || s->replay) {
+        res = -ENOTSUP;
+    } else {
+        res = s->chr_ioctl(s, cmd, arg);
+    }
+
+    return res;
 }
 
 int qemu_chr_be_can_write(CharDriverState *s)
@@ -299,17 +381,35 @@
     return s->chr_can_read(s->handler_opaque);
 }
 
-void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+void qemu_chr_be_write_impl(CharDriverState *s, uint8_t *buf, int len)
 {
     if (s->chr_read) {
         s->chr_read(s->handler_opaque, buf, len);
     }
 }
 
+void qemu_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+{
+    if (s->replay) {
+        if (replay_mode == REPLAY_MODE_PLAY) {
+            return;
+        }
+        replay_chr_be_write(s, buf, len);
+    } else {
+        qemu_chr_be_write_impl(s, buf, len);
+    }
+}
+
 int qemu_chr_fe_get_msgfd(CharDriverState *s)
 {
     int fd;
-    return (qemu_chr_fe_get_msgfds(s, &fd, 1) == 1) ? fd : -1;
+    int res = (qemu_chr_fe_get_msgfds(s, &fd, 1) == 1) ? fd : -1;
+    if (s->replay) {
+        fprintf(stderr,
+                "Replay: get msgfd is not supported for serial devices yet\n");
+        exit(1);
+    }
+    return res;
 }
 
 int qemu_chr_fe_get_msgfds(CharDriverState *s, int *fds, int len)
@@ -327,11 +427,6 @@
     return s->chr_add_client ? s->chr_add_client(s, fd) : -1;
 }
 
-int qemu_chr_del_client(CharDriverState *s)
-{
-    return s->chr_del_client ? s->chr_del_client(s) : -1;
-}
-
 void qemu_chr_accept_input(CharDriverState *s)
 {
     if (s->chr_accept_input)
@@ -388,11 +483,18 @@
     return len;
 }
 
-static CharDriverState *qemu_chr_open_null(void)
+static CharDriverState *qemu_chr_open_null(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
     CharDriverState *chr;
+    ChardevCommon *common = backend->u.null.data;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     chr->chr_write = null_chr_write;
     chr->explicit_be_open = true;
     return chr;
@@ -528,7 +630,7 @@
                  break;
             }
         case 's':
-            bdrv_commit_all();
+            blk_commit_all();
             break;
         case 'b':
             qemu_chr_be_event(chr, CHR_EVENT_BREAK);
@@ -684,17 +786,38 @@
     return d->drv->chr_add_watch(d->drv, cond);
 }
 
-static CharDriverState *qemu_chr_open_mux(CharDriverState *drv)
+static void mux_chr_close(struct CharDriverState *chr)
 {
-    CharDriverState *chr;
-    MuxDriver *d;
+    MuxDriver *d = chr->opaque;
 
-    chr = qemu_chr_alloc();
-    d = g_malloc0(sizeof(MuxDriver));
+    g_free(d);
+}
+
+static CharDriverState *qemu_chr_open_mux(const char *id,
+                                          ChardevBackend *backend,
+                                          ChardevReturn *ret, Error **errp)
+{
+    ChardevMux *mux = backend->u.mux.data;
+    CharDriverState *chr, *drv;
+    MuxDriver *d;
+    ChardevCommon *common = qapi_ChardevMux_base(mux);
+
+    drv = qemu_chr_find(mux->chardev);
+    if (drv == NULL) {
+        error_setg(errp, "mux: base chardev %s not found", mux->chardev);
+        return NULL;
+    }
+
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    d = g_new0(MuxDriver, 1);
 
     chr->opaque = d;
     d->drv = drv;
     d->focus = -1;
+    chr->chr_close = mux_chr_close;
     chr->chr_write = mux_chr_write;
     chr->chr_update_read_handler = mux_chr_update_read_handler;
     chr->chr_accept_input = mux_chr_accept_input;
@@ -713,107 +836,11 @@
 }
 
 
-#ifdef _WIN32
-int send_all(int fd, const void *buf, int len1)
-{
-    int ret, len;
-
-    len = len1;
-    while (len > 0) {
-        ret = send(fd, buf, len, 0);
-        if (ret < 0) {
-            errno = WSAGetLastError();
-            if (errno != WSAEWOULDBLOCK) {
-                return -1;
-            }
-        } else if (ret == 0) {
-            break;
-        } else {
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-int recv_all(int fd, void *_buf, int len1, bool single_read)
-{
-    int ret, len;
-    char *buf = _buf;
-
-    len = len1;
-    while (len > 0) {
-        ret = recv(fd, buf, len, 0);
-        if (ret < 0) {
-            errno = WSAGetLastError();
-            if (errno != WSAEWOULDBLOCK) {
-                return -1;
-            }
-            continue;
-        } else {
-            if (single_read) {
-                return ret;
-            }
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-#else
-
-int send_all(int fd, const void *_buf, int len1)
-{
-    int ret, len;
-    const uint8_t *buf = _buf;
-
-    len = len1;
-    while (len > 0) {
-        ret = write(fd, buf, len);
-        if (ret < 0) {
-            if (errno != EINTR && errno != EAGAIN)
-                return -1;
-        } else if (ret == 0) {
-            break;
-        } else {
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-int recv_all(int fd, void *_buf, int len1, bool single_read)
-{
-    int ret, len;
-    uint8_t *buf = _buf;
-
-    len = len1;
-    while ((len > 0) && (ret = read(fd, buf, len)) != 0) {
-        if (ret < 0) {
-            if (errno != EINTR && errno != EAGAIN) {
-                return -1;
-            }
-            continue;
-        } else {
-            if (single_read) {
-                return ret;
-            }
-            buf += ret;
-            len -= ret;
-        }
-    }
-    return len1 - len;
-}
-
-#endif /* !_WIN32 */
-
 typedef struct IOWatchPoll
 {
     GSource parent;
 
-    GIOChannel *channel;
+    QIOChannel *ioc;
     GSource *src;
 
     IOCanReadHandler *fd_can_read;
@@ -836,7 +863,8 @@
     }
 
     if (now_active) {
-        iwp->src = g_io_create_watch(iwp->channel, G_IO_IN | G_IO_ERR | G_IO_HUP);
+        iwp->src = qio_channel_create_watch(
+            iwp->ioc, G_IO_IN | G_IO_ERR | G_IO_HUP | G_IO_NVAL);
         g_source_set_callback(iwp->src, iwp->fd_read, iwp->opaque, NULL);
         g_source_attach(iwp->src, NULL);
     } else {
@@ -883,9 +911,9 @@
 };
 
 /* Can only be used for read */
-static guint io_add_watch_poll(GIOChannel *channel,
+static guint io_add_watch_poll(QIOChannel *ioc,
                                IOCanReadHandler *fd_can_read,
-                               GIOFunc fd_read,
+                               QIOChannelFunc fd_read,
                                gpointer user_data)
 {
     IOWatchPoll *iwp;
@@ -894,7 +922,7 @@
     iwp = (IOWatchPoll *) g_source_new(&io_watch_poll_funcs, sizeof(IOWatchPoll));
     iwp->fd_can_read = fd_can_read;
     iwp->opaque = user_data;
-    iwp->channel = channel;
+    iwp->ioc = ioc;
     iwp->fd_read = (GSourceFunc) fd_read;
     iwp->src = NULL;
 
@@ -930,81 +958,51 @@
     }
 }
 
-#ifndef _WIN32
-static GIOChannel *io_channel_from_fd(int fd)
-{
-    GIOChannel *chan;
 
-    if (fd == -1) {
-        return NULL;
-    }
-
-    chan = g_io_channel_unix_new(fd);
-
-    g_io_channel_set_encoding(chan, NULL, NULL);
-    g_io_channel_set_buffered(chan, FALSE);
-
-    return chan;
-}
-#endif
-
-static GIOChannel *io_channel_from_socket(int fd)
-{
-    GIOChannel *chan;
-
-    if (fd == -1) {
-        return NULL;
-    }
-
-#ifdef _WIN32
-    chan = g_io_channel_win32_new_socket(fd);
-#else
-    chan = g_io_channel_unix_new(fd);
-#endif
-
-    g_io_channel_set_encoding(chan, NULL, NULL);
-    g_io_channel_set_buffered(chan, FALSE);
-
-    return chan;
-}
-
-static int io_channel_send(GIOChannel *fd, const void *buf, size_t len)
+static int io_channel_send_full(QIOChannel *ioc,
+                                const void *buf, size_t len,
+                                int *fds, size_t nfds)
 {
     size_t offset = 0;
-    GIOStatus status = G_IO_STATUS_NORMAL;
 
-    while (offset < len && status == G_IO_STATUS_NORMAL) {
-        gsize bytes_written = 0;
+    while (offset < len) {
+        ssize_t ret = 0;
+        struct iovec iov = { .iov_base = (char *)buf + offset,
+                             .iov_len = len - offset };
 
-        status = g_io_channel_write_chars(fd, buf + offset, len - offset,
-                                          &bytes_written, NULL);
-        offset += bytes_written;
+        ret = qio_channel_writev_full(
+            ioc, &iov, 1,
+            fds, nfds, NULL);
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            if (offset) {
+                return offset;
+            }
+
+            errno = EAGAIN;
+            return -1;
+        } else if (ret < 0) {
+            errno = EINVAL;
+            return -1;
+        }
+
+        offset += ret;
     }
 
-    if (offset > 0) {
-        return offset;
-    }
-    switch (status) {
-    case G_IO_STATUS_NORMAL:
-        g_assert(len == 0);
-        return 0;
-    case G_IO_STATUS_AGAIN:
-        errno = EAGAIN;
-        return -1;
-    default:
-        break;
-    }
-    errno = EINVAL;
-    return -1;
+    return offset;
 }
 
+
 #ifndef _WIN32
+static int io_channel_send(QIOChannel *ioc, const void *buf, size_t len)
+{
+    return io_channel_send_full(ioc, buf, len, NULL, 0);
+}
+
 
 typedef struct FDCharDriver {
     CharDriverState *chr;
-    GIOChannel *fd_in, *fd_out;
+    QIOChannel *ioc_in, *ioc_out;
     int max_size;
-    QTAILQ_ENTRY(FDCharDriver) node;
 } FDCharDriver;
 
 /* Called with chr_write_lock held.  */
@@ -1012,17 +1010,16 @@
 {
     FDCharDriver *s = chr->opaque;
     
-    return io_channel_send(s->fd_out, buf, len);
+    return io_channel_send(s->ioc_out, buf, len);
 }
 
-static gboolean fd_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean fd_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     FDCharDriver *s = chr->opaque;
     int len;
     uint8_t buf[READ_BUF_LEN];
-    GIOStatus status;
-    gsize bytes_read;
+    ssize_t ret;
 
     len = sizeof(buf);
     if (len > s->max_size) {
@@ -1032,15 +1029,15 @@
         return TRUE;
     }
 
-    status = g_io_channel_read_chars(chan, (gchar *)buf,
-                                     len, &bytes_read, NULL);
-    if (status == G_IO_STATUS_EOF) {
+    ret = qio_channel_read(
+        chan, (gchar *)buf, len, NULL);
+    if (ret == 0) {
         remove_fd_in_watch(chr);
         qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
         return FALSE;
     }
-    if (status == G_IO_STATUS_NORMAL) {
-        qemu_chr_be_write(chr, buf, bytes_read);
+    if (ret > 0) {
+        qemu_chr_be_write(chr, buf, ret);
     }
 
     return TRUE;
@@ -1058,7 +1055,7 @@
 static GSource *fd_chr_add_watch(CharDriverState *chr, GIOCondition cond)
 {
     FDCharDriver *s = chr->opaque;
-    return g_io_create_watch(s->fd_out, cond);
+    return qio_channel_create_watch(s->ioc_out, cond);
 }
 
 static void fd_chr_update_read_handler(CharDriverState *chr)
@@ -1066,8 +1063,9 @@
     FDCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->fd_in) {
-        chr->fd_in_tag = io_add_watch_poll(s->fd_in, fd_chr_read_poll,
+    if (s->ioc_in) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc_in,
+                                           fd_chr_read_poll,
                                            fd_chr_read, chr);
     }
 }
@@ -1077,11 +1075,11 @@
     FDCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->fd_in) {
-        g_io_channel_unref(s->fd_in);
+    if (s->ioc_in) {
+        object_unref(OBJECT(s->ioc_in));
     }
-    if (s->fd_out) {
-        g_io_channel_unref(s->fd_out);
+    if (s->ioc_out) {
+        object_unref(OBJECT(s->ioc_out));
     }
 
     g_free(s);
@@ -1089,15 +1087,19 @@
 }
 
 /* open a character device to a unix fd */
-static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out)
+static CharDriverState *qemu_chr_open_fd(int fd_in, int fd_out,
+                                         ChardevCommon *backend, Error **errp)
 {
     CharDriverState *chr;
     FDCharDriver *s;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(FDCharDriver));
-    s->fd_in = io_channel_from_fd(fd_in);
-    s->fd_out = io_channel_from_fd(fd_out);
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(FDCharDriver, 1);
+    s->ioc_in = QIO_CHANNEL(qio_channel_file_new_fd(fd_in));
+    s->ioc_out = QIO_CHANNEL(qio_channel_file_new_fd(fd_out));
     qemu_set_nonblock(fd_out);
     s->chr = chr;
     chr->opaque = s;
@@ -1109,22 +1111,25 @@
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
+static CharDriverState *qemu_chr_open_pipe(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
+    ChardevHostdev *opts = backend->u.pipe.data;
     int fd_in, fd_out;
-    char filename_in[CHR_MAX_FILENAME_SIZE];
-    char filename_out[CHR_MAX_FILENAME_SIZE];
+    char *filename_in;
+    char *filename_out;
     const char *filename = opts->device;
+    ChardevCommon *common = qapi_ChardevHostdev_base(opts);
 
-    if (filename == NULL) {
-        fprintf(stderr, "chardev: pipe: no filename given\n");
-        return NULL;
-    }
 
-    snprintf(filename_in, CHR_MAX_FILENAME_SIZE, "%s.in", filename);
-    snprintf(filename_out, CHR_MAX_FILENAME_SIZE, "%s.out", filename);
+    filename_in = g_strdup_printf("%s.in", filename);
+    filename_out = g_strdup_printf("%s.out", filename);
     TFR(fd_in = qemu_open(filename_in, O_RDWR | O_BINARY));
     TFR(fd_out = qemu_open(filename_out, O_RDWR | O_BINARY));
+    g_free(filename_in);
+    g_free(filename_out);
     if (fd_in < 0 || fd_out < 0) {
 	if (fd_in >= 0)
 	    close(fd_in);
@@ -1132,10 +1137,11 @@
 	    close(fd_out);
         TFR(fd_in = fd_out = qemu_open(filename, O_RDWR | O_BINARY));
         if (fd_in < 0) {
+            error_setg_file_open(errp, errno, filename);
             return NULL;
         }
     }
-    return qemu_chr_open_fd(fd_in, fd_out);
+    return qemu_chr_open_fd(fd_in, fd_out, common, errp);
 }
 
 /* init terminal so that we can grab keys */
@@ -1143,6 +1149,9 @@
 static int old_fd0_flags;
 static bool stdio_in_use;
 static bool stdio_allow_signal;
+static bool stdio_echo_state;
+
+static void qemu_chr_set_echo_stdio(CharDriverState *chr, bool echo);
 
 static void term_exit(void)
 {
@@ -1150,10 +1159,17 @@
     fcntl(0, F_SETFL, old_fd0_flags);
 }
 
+static void term_stdio_handler(int sig)
+{
+    /* restore echo after resume from suspend. */
+    qemu_chr_set_echo_stdio(NULL, stdio_echo_state);
+}
+
 static void qemu_chr_set_echo_stdio(CharDriverState *chr, bool echo)
 {
     struct termios tty;
 
+    stdio_echo_state = echo;
     tty = oldtty;
     if (!echo) {
         tty.c_iflag &= ~(IGNBRK|BRKINT|PARMRK|ISTRIP
@@ -1177,18 +1193,24 @@
     fd_chr_close(chr);
 }
 
-static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
+static CharDriverState *qemu_chr_open_stdio(const char *id,
+                                            ChardevBackend *backend,
+                                            ChardevReturn *ret,
+                                            Error **errp)
 {
+    ChardevStdio *opts = backend->u.stdio.data;
     CharDriverState *chr;
+    struct sigaction act;
+    ChardevCommon *common = qapi_ChardevStdio_base(opts);
 
     if (is_daemonized()) {
-        error_report("cannot use stdio with -daemonize");
+        error_setg(errp, "cannot use stdio with -daemonize");
         return NULL;
     }
 
     if (stdio_in_use) {
-        error_report("cannot use stdio by multiple character devices");
-        exit(1);
+        error_setg(errp, "cannot use stdio by multiple character devices");
+        return NULL;
     }
 
     stdio_in_use = true;
@@ -1197,7 +1219,11 @@
     qemu_set_nonblock(0);
     atexit(term_exit);
 
-    chr = qemu_chr_open_fd(0, 1);
+    memset(&act, 0, sizeof(act));
+    act.sa_handler = term_stdio_handler;
+    sigaction(SIGCONT, &act, NULL);
+
+    chr = qemu_chr_open_fd(0, 1, common, errp);
     chr->chr_close = qemu_chr_close_stdio;
     chr->chr_set_echo = qemu_chr_set_echo_stdio;
     if (opts->has_signal) {
@@ -1212,10 +1238,11 @@
     || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) \
     || defined(__GLIBC__)
 
-#define HAVE_CHARDEV_TTY 1
+#define HAVE_CHARDEV_SERIAL 1
+#define HAVE_CHARDEV_PTY 1
 
 typedef struct {
-    GIOChannel *fd;
+    QIOChannel *ioc;
     int read_bytes;
 
     /* Protected by the CharDriverState chr_write_lock.  */
@@ -1265,11 +1292,17 @@
 {
     PtyCharDriver *s = chr->opaque;
     GPollFD pfd;
+    int rc;
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(s->ioc);
 
-    pfd.fd = g_io_channel_unix_get_fd(s->fd);
+    pfd.fd = fioc->fd;
     pfd.events = G_IO_OUT;
     pfd.revents = 0;
-    g_poll(&pfd, 1, 0);
+    do {
+        rc = g_poll(&pfd, 1, 0);
+    } while (rc == -1 && errno == EINTR);
+    assert(rc >= 0);
+
     if (pfd.revents & G_IO_HUP) {
         pty_chr_state(chr, 0);
     } else {
@@ -1296,7 +1329,7 @@
             return 0;
         }
     }
-    return io_channel_send(s->fd, buf, len);
+    return io_channel_send(s->ioc, buf, len);
 }
 
 static GSource *pty_chr_add_watch(CharDriverState *chr, GIOCondition cond)
@@ -1305,7 +1338,7 @@
     if (!s->connected) {
         return NULL;
     }
-    return g_io_create_watch(s->fd, cond);
+    return qio_channel_create_watch(s->ioc, cond);
 }
 
 static int pty_chr_read_poll(void *opaque)
@@ -1317,13 +1350,13 @@
     return s->read_bytes;
 }
 
-static gboolean pty_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean pty_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     PtyCharDriver *s = chr->opaque;
-    gsize size, len;
+    gsize len;
     uint8_t buf[READ_BUF_LEN];
-    GIOStatus status;
+    ssize_t ret;
 
     len = sizeof(buf);
     if (len > s->read_bytes)
@@ -1331,13 +1364,13 @@
     if (len == 0) {
         return TRUE;
     }
-    status = g_io_channel_read_chars(s->fd, (gchar *)buf, len, &size, NULL);
-    if (status != G_IO_STATUS_NORMAL) {
+    ret = qio_channel_read(s->ioc, (char *)buf, len, NULL);
+    if (ret <= 0) {
         pty_chr_state(chr, 0);
         return FALSE;
     } else {
         pty_chr_state(chr, 1);
-        qemu_chr_be_write(chr, buf, size);
+        qemu_chr_be_write(chr, buf, ret);
     }
     return TRUE;
 }
@@ -1379,7 +1412,8 @@
             s->open_tag = g_idle_add(qemu_chr_be_generic_open_func, chr);
         }
         if (!chr->fd_in_tag) {
-            chr->fd_in_tag = io_add_watch_poll(s->fd, pty_chr_read_poll,
+            chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                               pty_chr_read_poll,
                                                pty_chr_read, chr);
         }
     }
@@ -1388,13 +1422,10 @@
 static void pty_chr_close(struct CharDriverState *chr)
 {
     PtyCharDriver *s = chr->opaque;
-    int fd;
 
     qemu_mutex_lock(&chr->chr_write_lock);
     pty_chr_state(chr, 0);
-    fd = g_io_channel_unix_get_fd(s->fd);
-    g_io_channel_unref(s->fd);
-    close(fd);
+    object_unref(OBJECT(s->ioc));
     if (s->timer_tag) {
         g_source_remove(s->timer_tag);
         s->timer_tag = 0;
@@ -1405,21 +1436,30 @@
 }
 
 static CharDriverState *qemu_chr_open_pty(const char *id,
-                                          ChardevReturn *ret)
+                                          ChardevBackend *backend,
+                                          ChardevReturn *ret,
+                                          Error **errp)
 {
     CharDriverState *chr;
     PtyCharDriver *s;
     int master_fd, slave_fd;
     char pty_name[PATH_MAX];
+    ChardevCommon *common = backend->u.pty.data;
 
     master_fd = qemu_openpty_raw(&slave_fd, pty_name);
     if (master_fd < 0) {
+        error_setg_errno(errp, errno, "Failed to create PTY");
         return NULL;
     }
 
     close(slave_fd);
+    qemu_set_nonblock(master_fd);
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        close(master_fd);
+        return NULL;
+    }
 
     chr->filename = g_strdup_printf("pty:%s", pty_name);
     ret->pty = g_strdup(pty_name);
@@ -1428,7 +1468,7 @@
     fprintf(stderr, "char device redirected to %s (label %s)\n",
             pty_name, id);
 
-    s = g_malloc0(sizeof(PtyCharDriver));
+    s = g_new0(PtyCharDriver, 1);
     chr->opaque = s;
     chr->chr_write = pty_chr_write;
     chr->chr_update_read_handler = pty_chr_update_read_handler;
@@ -1436,7 +1476,7 @@
     chr->chr_add_watch = pty_chr_add_watch;
     chr->explicit_be_open = true;
 
-    s->fd = io_channel_from_fd(master_fd);
+    s->ioc = QIO_CHANNEL(qio_channel_file_new_fd(master_fd));
     s->timer_tag = 0;
 
     return chr;
@@ -1560,12 +1600,13 @@
 static int tty_serial_ioctl(CharDriverState *chr, int cmd, void *arg)
 {
     FDCharDriver *s = chr->opaque;
+    QIOChannelFile *fioc = QIO_CHANNEL_FILE(s->ioc_in);
 
     switch(cmd) {
     case CHR_IOCTL_SERIAL_SET_PARAMS:
         {
             QEMUSerialSetParams *ssp = arg;
-            tty_serial_init(g_io_channel_unix_get_fd(s->fd_in),
+            tty_serial_init(fioc->fd,
                             ssp->speed, ssp->parity,
                             ssp->data_bits, ssp->stop_bits);
         }
@@ -1574,7 +1615,7 @@
         {
             int enable = *(int *)arg;
             if (enable) {
-                tcsendbreak(g_io_channel_unix_get_fd(s->fd_in), 1);
+                tcsendbreak(fioc->fd, 1);
             }
         }
         break;
@@ -1582,7 +1623,7 @@
         {
             int sarg = 0;
             int *targ = (int *)arg;
-            ioctl(g_io_channel_unix_get_fd(s->fd_in), TIOCMGET, &sarg);
+            ioctl(fioc->fd, TIOCMGET, &sarg);
             *targ = 0;
             if (sarg & TIOCM_CTS)
                 *targ |= CHR_TIOCM_CTS;
@@ -1602,7 +1643,7 @@
         {
             int sarg = *(int *)arg;
             int targ = 0;
-            ioctl(g_io_channel_unix_get_fd(s->fd_in), TIOCMGET, &targ);
+            ioctl(fioc->fd, TIOCMGET, &targ);
             targ &= ~(CHR_TIOCM_CTS | CHR_TIOCM_CAR | CHR_TIOCM_DSR
                      | CHR_TIOCM_RI | CHR_TIOCM_DTR | CHR_TIOCM_RTS);
             if (sarg & CHR_TIOCM_CTS)
@@ -1617,7 +1658,7 @@
                 targ |= TIOCM_DTR;
             if (sarg & CHR_TIOCM_RTS)
                 targ |= TIOCM_RTS;
-            ioctl(g_io_channel_unix_get_fd(s->fd_in), TIOCMSET, &targ);
+            ioctl(fioc->fd, TIOCMSET, &targ);
         }
         break;
     default:
@@ -1628,26 +1669,17 @@
 
 static void qemu_chr_close_tty(CharDriverState *chr)
 {
-    FDCharDriver *s = chr->opaque;
-    int fd = -1;
-
-    if (s) {
-        fd = g_io_channel_unix_get_fd(s->fd_in);
-    }
-
     fd_chr_close(chr);
-
-    if (fd >= 0) {
-        close(fd);
-    }
 }
 
-static CharDriverState *qemu_chr_open_tty_fd(int fd)
+static CharDriverState *qemu_chr_open_tty_fd(int fd,
+                                             ChardevCommon *backend,
+                                             Error **errp)
 {
     CharDriverState *chr;
 
     tty_serial_init(fd, 115200, 'N', 8, 1);
-    chr = qemu_chr_open_fd(fd, fd);
+    chr = qemu_chr_open_fd(fd, fd, backend, errp);
     chr->chr_ioctl = tty_serial_ioctl;
     chr->chr_close = qemu_chr_close_tty;
     return chr;
@@ -1767,25 +1799,32 @@
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_pp_fd(int fd)
+static CharDriverState *qemu_chr_open_pp_fd(int fd,
+                                            ChardevCommon *backend,
+                                            Error **errp)
 {
     CharDriverState *chr;
     ParallelCharDriver *drv;
 
     if (ioctl(fd, PPCLAIM) < 0) {
+        error_setg_errno(errp, errno, "not a parallel port");
         close(fd);
         return NULL;
     }
 
-    drv = g_malloc0(sizeof(ParallelCharDriver));
-    drv->fd = fd;
-    drv->mode = IEEE1284_MODE_COMPAT;
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
 
-    chr = qemu_chr_alloc();
+    drv = g_new0(ParallelCharDriver, 1);
+    chr->opaque = drv;
     chr->chr_write = null_chr_write;
     chr->chr_ioctl = pp_ioctl;
     chr->chr_close = pp_close;
-    chr->opaque = drv;
+
+    drv->fd = fd;
+    drv->mode = IEEE1284_MODE_COMPAT;
 
     return chr;
 }
@@ -1832,11 +1871,16 @@
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_pp_fd(int fd)
+static CharDriverState *qemu_chr_open_pp_fd(int fd,
+                                            ChardevCommon *backend,
+                                            Error **errp)
 {
     CharDriverState *chr;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
     chr->opaque = (void *)(intptr_t)fd;
     chr->chr_write = null_chr_write;
     chr->chr_ioctl = pp_ioctl;
@@ -1847,6 +1891,8 @@
 
 #else /* _WIN32 */
 
+#define HAVE_CHARDEV_SERIAL 1
+
 typedef struct {
     int max_size;
     HANDLE hcom, hrecv, hsend;
@@ -1898,7 +1944,7 @@
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static int win_chr_init(CharDriverState *chr, const char *filename)
+static int win_chr_init(CharDriverState *chr, const char *filename, Error **errp)
 {
     WinCharState *s = chr->opaque;
     COMMCONFIG comcfg;
@@ -1909,25 +1955,25 @@
 
     s->hsend = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hsend) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
     s->hrecv = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hrecv) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
 
-    s->hcom = CreateFile(filename, GENERIC_READ|GENERIC_WRITE, 0, NULL,
+    s->hcom = win32CreateFile(filename, GENERIC_READ|GENERIC_WRITE, 0, NULL,
                       OPEN_EXISTING, FILE_FLAG_OVERLAPPED, 0);
     if (s->hcom == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "Failed CreateFile (%lu)\n", GetLastError());
+        error_setg(errp, "Failed CreateFile (%lu)", GetLastError());
         s->hcom = NULL;
         goto fail;
     }
 
     if (!SetupComm(s->hcom, NRECVBUF, NSENDBUF)) {
-        fprintf(stderr, "Failed SetupComm\n");
+        error_setg(errp, "Failed SetupComm");
         goto fail;
     }
 
@@ -1938,23 +1984,23 @@
     CommConfigDialog(filename, NULL, &comcfg);
 
     if (!SetCommState(s->hcom, &comcfg.dcb)) {
-        fprintf(stderr, "Failed SetCommState\n");
+        error_setg(errp, "Failed SetCommState");
         goto fail;
     }
 
     if (!SetCommMask(s->hcom, EV_ERR)) {
-        fprintf(stderr, "Failed SetCommMask\n");
+        error_setg(errp, "Failed SetCommMask");
         goto fail;
     }
 
     cto.ReadIntervalTimeout = MAXDWORD;
     if (!SetCommTimeouts(s->hcom, &cto)) {
-        fprintf(stderr, "Failed SetCommTimeouts\n");
+        error_setg(errp, "Failed SetCommTimeouts");
         goto fail;
     }
 
     if (!ClearCommError(s->hcom, &err, &comstat)) {
-        fprintf(stderr, "Failed ClearCommError\n");
+        error_setg(errp, "Failed ClearCommError");
         goto fail;
     }
     qemu_add_polling_cb(win_chr_poll, chr);
@@ -2059,20 +2105,25 @@
     return 0;
 }
 
-static CharDriverState *qemu_chr_open_win_path(const char *filename)
+static CharDriverState *qemu_chr_open_win_path(const char *filename,
+                                               ChardevCommon *backend,
+                                               Error **errp)
 {
     CharDriverState *chr;
     WinCharState *s;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(WinCharState, 1);
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     chr->chr_close = win_chr_close;
 
-    if (win_chr_init(chr, filename) < 0) {
+    if (win_chr_init(chr, filename, errp) < 0) {
         g_free(s);
-        g_free(chr);
+        qemu_chr_free_common(chr);
         return NULL;
     }
     return chr;
@@ -2094,34 +2145,36 @@
     return 0;
 }
 
-static int win_chr_pipe_init(CharDriverState *chr, const char *filename)
+static int win_chr_pipe_init(CharDriverState *chr, const char *filename,
+                             Error **errp)
 {
     WinCharState *s = chr->opaque;
     OVERLAPPED ov;
     int ret;
     DWORD size;
-    char openname[CHR_MAX_FILENAME_SIZE];
+    char *openname;
 
     s->fpipe = TRUE;
 
     s->hsend = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hsend) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
     s->hrecv = CreateEvent(NULL, TRUE, FALSE, NULL);
     if (!s->hrecv) {
-        fprintf(stderr, "Failed CreateEvent\n");
+        error_setg(errp, "Failed CreateEvent");
         goto fail;
     }
 
-    snprintf(openname, sizeof(openname), "\\\\.\\pipe\\%s", filename);
+    openname = g_strdup_printf("\\\\.\\pipe\\%s", filename);
     s->hcom = CreateNamedPipe(openname, PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED,
                               PIPE_TYPE_BYTE | PIPE_READMODE_BYTE |
                               PIPE_WAIT,
                               MAXCONNECT, NSENDBUF, NRECVBUF, NTIMEOUT, NULL);
+    g_free(openname);
     if (s->hcom == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "Failed CreateNamedPipe (%lu)\n", GetLastError());
+        error_setg(errp, "Failed CreateNamedPipe (%lu)", GetLastError());
         s->hcom = NULL;
         goto fail;
     }
@@ -2130,13 +2183,13 @@
     ov.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
     ret = ConnectNamedPipe(s->hcom, &ov);
     if (ret) {
-        fprintf(stderr, "Failed ConnectNamedPipe\n");
+        error_setg(errp, "Failed ConnectNamedPipe");
         goto fail;
     }
 
     ret = GetOverlappedResult(s->hcom, &ov, &size, TRUE);
     if (!ret) {
-        fprintf(stderr, "Failed GetOverlappedResult\n");
+        error_setg(errp, "Failed GetOverlappedResult");
         if (ov.hEvent) {
             CloseHandle(ov.hEvent);
             ov.hEvent = NULL;
@@ -2157,42 +2210,60 @@
 }
 
 
-static CharDriverState *qemu_chr_open_pipe(ChardevHostdev *opts)
+static CharDriverState *qemu_chr_open_pipe(const char *id,
+                                           ChardevBackend *backend,
+                                           ChardevReturn *ret,
+                                           Error **errp)
 {
+    ChardevHostdev *opts = backend->u.pipe.data;
     const char *filename = opts->device;
     CharDriverState *chr;
     WinCharState *s;
+    ChardevCommon *common = qapi_ChardevHostdev_base(opts);
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(WinCharState, 1);
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     chr->chr_close = win_chr_close;
 
-    if (win_chr_pipe_init(chr, filename) < 0) {
+    if (win_chr_pipe_init(chr, filename, errp) < 0) {
         g_free(s);
-        g_free(chr);
+        qemu_chr_free_common(chr);
         return NULL;
     }
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_win_file(HANDLE fd_out)
+static CharDriverState *qemu_chr_open_win_file(HANDLE fd_out,
+                                               ChardevCommon *backend,
+                                               Error **errp)
 {
     CharDriverState *chr;
     WinCharState *s;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(WinCharState));
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(WinCharState, 1);
     s->hcom = fd_out;
     chr->opaque = s;
     chr->chr_write = win_chr_write;
     return chr;
 }
 
-static CharDriverState *qemu_chr_open_win_con(void)
+static CharDriverState *qemu_chr_open_win_con(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
-    return qemu_chr_open_win_file(GetStdHandle(STD_OUTPUT_HANDLE));
+    ChardevCommon *common = backend->u.console.data;
+    return qemu_chr_open_win_file(GetStdHandle(STD_OUTPUT_HANDLE),
+                                  common, errp);
 }
 
 static int win_stdio_write(CharDriverState *chr, const uint8_t *buf, int len)
@@ -2349,20 +2420,27 @@
     g_free(chr);
 }
 
-static CharDriverState *qemu_chr_open_stdio(ChardevStdio *opts)
+static CharDriverState *qemu_chr_open_stdio(const char *id,
+                                            ChardevBackend *backend,
+                                            ChardevReturn *ret,
+                                            Error **errp)
 {
     CharDriverState   *chr;
     WinStdioCharState *stdio;
     DWORD              dwMode;
     int                is_console = 0;
+    ChardevCommon *common = qapi_ChardevStdio_base(backend->u.stdio.data);
 
-    chr   = qemu_chr_alloc();
-    stdio = g_malloc0(sizeof(WinStdioCharState));
+    chr   = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    stdio = g_new0(WinStdioCharState, 1);
 
     stdio->hStdIn = GetStdHandle(STD_INPUT_HANDLE);
     if (stdio->hStdIn == INVALID_HANDLE_VALUE) {
-        fprintf(stderr, "cannot open stdio: invalid handle\n");
-        exit(1);
+        error_setg(errp, "cannot open stdio: invalid handle");
+        return NULL;
     }
 
     is_console = GetConsoleMode(stdio->hStdIn, &dwMode) != 0;
@@ -2374,25 +2452,30 @@
     if (is_console) {
         if (qemu_add_wait_object(stdio->hStdIn,
                                  win_stdio_wait_func, chr)) {
-            fprintf(stderr, "qemu_add_wait_object: failed\n");
+            error_setg(errp, "qemu_add_wait_object: failed");
+            goto err1;
         }
     } else {
         DWORD   dwId;
             
         stdio->hInputReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL);
         stdio->hInputDoneEvent  = CreateEvent(NULL, FALSE, FALSE, NULL);
-        stdio->hInputThread     = CreateThread(NULL, 0, win_stdio_thread,
-                                               chr, 0, &dwId);
-
-        if (stdio->hInputThread == INVALID_HANDLE_VALUE
-            || stdio->hInputReadyEvent == INVALID_HANDLE_VALUE
+        if (stdio->hInputReadyEvent == INVALID_HANDLE_VALUE
             || stdio->hInputDoneEvent == INVALID_HANDLE_VALUE) {
-            fprintf(stderr, "cannot create stdio thread or event\n");
-            exit(1);
+            error_setg(errp, "cannot create event");
+            goto err2;
         }
         if (qemu_add_wait_object(stdio->hInputReadyEvent,
                                  win_stdio_thread_wait_func, chr)) {
-            fprintf(stderr, "qemu_add_wait_object: failed\n");
+            error_setg(errp, "qemu_add_wait_object: failed");
+            goto err2;
+        }
+        stdio->hInputThread     = CreateThread(NULL, 0, win_stdio_thread,
+                                               chr, 0, &dwId);
+
+        if (stdio->hInputThread == INVALID_HANDLE_VALUE) {
+            error_setg(errp, "cannot create stdio thread");
+            goto err3;
         }
     }
 
@@ -2410,6 +2493,15 @@
     qemu_chr_fe_set_echo(chr, false);
 
     return chr;
+
+err3:
+    qemu_del_wait_object(stdio->hInputReadyEvent, NULL, NULL);
+err2:
+    CloseHandle(stdio->hInputReadyEvent);
+    CloseHandle(stdio->hInputDoneEvent);
+err1:
+    qemu_del_wait_object(stdio->hStdIn, NULL, NULL);
+    return NULL;
 }
 #endif /* !_WIN32 */
 
@@ -2418,8 +2510,7 @@
 /* UDP Net console */
 
 typedef struct {
-    int fd;
-    GIOChannel *chan;
+    QIOChannel *ioc;
     uint8_t buf[READ_BUF_LEN];
     int bufcnt;
     int bufptr;
@@ -2430,17 +2521,9 @@
 static int udp_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     NetCharDriver *s = chr->opaque;
-    gsize bytes_written;
-    GIOStatus status;
 
-    status = g_io_channel_write_chars(s->chan, (const gchar *)buf, len, &bytes_written, NULL);
-    if (status == G_IO_STATUS_EOF) {
-        return 0;
-    } else if (status != G_IO_STATUS_NORMAL) {
-        return -1;
-    }
-
-    return bytes_written;
+    return qio_channel_write(
+        s->ioc, (const char *)buf, len, NULL);
 }
 
 static int udp_chr_read_poll(void *opaque)
@@ -2461,24 +2544,22 @@
     return s->max_size;
 }
 
-static gboolean udp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean udp_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     NetCharDriver *s = chr->opaque;
-    gsize bytes_read = 0;
-    GIOStatus status;
+    ssize_t ret;
 
     if (s->max_size == 0) {
         return TRUE;
     }
-    status = g_io_channel_read_chars(s->chan, (gchar *)s->buf, sizeof(s->buf),
-                                     &bytes_read, NULL);
-    s->bufcnt = bytes_read;
-    s->bufptr = s->bufcnt;
-    if (status != G_IO_STATUS_NORMAL) {
+    ret = qio_channel_read(
+        s->ioc, (char *)s->buf, sizeof(s->buf), NULL);
+    if (ret <= 0) {
         remove_fd_in_watch(chr);
         return FALSE;
     }
+    s->bufcnt = ret;
 
     s->bufptr = 0;
     while (s->max_size > 0 && s->bufptr < s->bufcnt) {
@@ -2495,8 +2576,9 @@
     NetCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->chan) {
-        chr->fd_in_tag = io_add_watch_poll(s->chan, udp_chr_read_poll,
+    if (s->ioc) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                           udp_chr_read_poll,
                                            udp_chr_read, chr);
     }
 }
@@ -2506,24 +2588,27 @@
     NetCharDriver *s = chr->opaque;
 
     remove_fd_in_watch(chr);
-    if (s->chan) {
-        g_io_channel_unref(s->chan);
-        closesocket(s->fd);
+    if (s->ioc) {
+        object_unref(OBJECT(s->ioc));
     }
     g_free(s);
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static CharDriverState *qemu_chr_open_udp_fd(int fd)
+static CharDriverState *qemu_chr_open_udp(QIOChannelSocket *sioc,
+                                          ChardevCommon *backend,
+                                          Error **errp)
 {
     CharDriverState *chr = NULL;
     NetCharDriver *s = NULL;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(NetCharDriver));
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(NetCharDriver, 1);
 
-    s->fd = fd;
-    s->chan = io_channel_from_socket(s->fd);
+    s->ioc = QIO_CHANNEL(sioc);
     s->bufcnt = 0;
     s->bufptr = 0;
     chr->opaque = s;
@@ -2539,19 +2624,20 @@
 /* TCP Net console */
 
 typedef struct {
-
-    GIOChannel *chan, *listen_chan;
+    QIOChannel *ioc; /* Client I/O channel */
+    QIOChannelSocket *sioc; /* Client master channel */
+    QIOChannelSocket *listen_ioc;
     guint listen_tag;
-    int fd, listen_fd;
+    QCryptoTLSCreds *tls_creds;
     int connected;
     int max_size;
     int do_telnetopt;
     int do_nodelay;
     int is_unix;
     int *read_msgfds;
-    int read_msgfds_num;
+    size_t read_msgfds_num;
     int *write_msgfds;
-    int write_msgfds_num;
+    size_t write_msgfds_num;
 
     SocketAddress *addr;
     bool is_listen;
@@ -2585,68 +2671,27 @@
     qemu_chr_socket_restart_timer(chr);
 }
 
-static gboolean tcp_chr_accept(GIOChannel *chan, GIOCondition cond, void *opaque);
-
-#ifndef _WIN32
-static int unix_send_msgfds(CharDriverState *chr, const uint8_t *buf, int len)
-{
-    TCPCharDriver *s = chr->opaque;
-    struct msghdr msgh;
-    struct iovec iov;
-    int r;
-
-    size_t fd_size = s->write_msgfds_num * sizeof(int);
-    char control[CMSG_SPACE(fd_size)];
-    struct cmsghdr *cmsg;
-
-    memset(&msgh, 0, sizeof(msgh));
-    memset(control, 0, sizeof(control));
-
-    /* set the payload */
-    iov.iov_base = (uint8_t *) buf;
-    iov.iov_len = len;
-
-    msgh.msg_iov = &iov;
-    msgh.msg_iovlen = 1;
-
-    msgh.msg_control = control;
-    msgh.msg_controllen = sizeof(control);
-
-    cmsg = CMSG_FIRSTHDR(&msgh);
-
-    cmsg->cmsg_len = CMSG_LEN(fd_size);
-    cmsg->cmsg_level = SOL_SOCKET;
-    cmsg->cmsg_type = SCM_RIGHTS;
-    memcpy(CMSG_DATA(cmsg), s->write_msgfds, fd_size);
-
-    do {
-        r = sendmsg(s->fd, &msgh, 0);
-    } while (r < 0 && errno == EINTR);
-
-    /* free the written msgfds, no matter what */
-    if (s->write_msgfds_num) {
-        g_free(s->write_msgfds);
-        s->write_msgfds = 0;
-        s->write_msgfds_num = 0;
-    }
-
-    return r;
-}
-#endif
+static gboolean tcp_chr_accept(QIOChannel *chan,
+                               GIOCondition cond,
+                               void *opaque);
 
 /* Called with chr_write_lock held.  */
 static int tcp_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     TCPCharDriver *s = chr->opaque;
     if (s->connected) {
-#ifndef _WIN32
-        if (s->is_unix && s->write_msgfds_num) {
-            return unix_send_msgfds(chr, buf, len);
-        } else
-#endif
-        {
-            return io_channel_send(s->chan, buf, len);
+        int ret =  io_channel_send_full(s->ioc, buf, len,
+                                        s->write_msgfds,
+                                        s->write_msgfds_num);
+
+        /* free the written msgfds, no matter what */
+        if (s->write_msgfds_num) {
+            g_free(s->write_msgfds);
+            s->write_msgfds = 0;
+            s->write_msgfds_num = 0;
         }
+
+        return ret;
     } else {
         /* XXX: indicate an error ? */
         return len;
@@ -2743,12 +2788,18 @@
     TCPCharDriver *s = chr->opaque;
 
     /* clear old pending fd array */
-    if (s->write_msgfds) {
-        g_free(s->write_msgfds);
+    g_free(s->write_msgfds);
+    s->write_msgfds = NULL;
+    s->write_msgfds_num = 0;
+
+    if (!s->connected ||
+        !qio_channel_has_feature(s->ioc,
+                                 QIO_CHANNEL_FEATURE_FD_PASS)) {
+        return -1;
     }
 
     if (num) {
-        s->write_msgfds = g_malloc(num * sizeof(int));
+        s->write_msgfds = g_new(int, num);
         memcpy(s->write_msgfds, fds, num * sizeof(int));
     }
 
@@ -2757,27 +2808,33 @@
     return 0;
 }
 
-#ifndef _WIN32
-static void unix_process_msgfd(CharDriverState *chr, struct msghdr *msg)
+static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
 {
     TCPCharDriver *s = chr->opaque;
-    struct cmsghdr *cmsg;
+    struct iovec iov = { .iov_base = buf, .iov_len = len };
+    int ret;
+    size_t i;
+    int *msgfds = NULL;
+    size_t msgfds_num = 0;
 
-    for (cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {
-        int fd_size, i;
+    if (qio_channel_has_feature(s->ioc, QIO_CHANNEL_FEATURE_FD_PASS)) {
+        ret = qio_channel_readv_full(s->ioc, &iov, 1,
+                                     &msgfds, &msgfds_num,
+                                     NULL);
+    } else {
+        ret = qio_channel_readv_full(s->ioc, &iov, 1,
+                                     NULL, NULL,
+                                     NULL);
+    }
 
-        if (cmsg->cmsg_len < CMSG_LEN(sizeof(int)) ||
-            cmsg->cmsg_level != SOL_SOCKET ||
-            cmsg->cmsg_type != SCM_RIGHTS) {
-            continue;
-        }
+    if (ret == QIO_CHANNEL_ERR_BLOCK) {
+        errno = EAGAIN;
+        ret = -1;
+    } else if (ret == -1) {
+        errno = EIO;
+    }
 
-        fd_size = cmsg->cmsg_len - CMSG_LEN(0);
-
-        if (!fd_size) {
-            continue;
-        }
-
+    if (msgfds_num) {
         /* close and clean read_msgfds */
         for (i = 0; i < s->read_msgfds_num; i++) {
             close(s->read_msgfds[i]);
@@ -2787,110 +2844,91 @@
             g_free(s->read_msgfds);
         }
 
-        s->read_msgfds_num = fd_size / sizeof(int);
-        s->read_msgfds = g_malloc(fd_size);
-        memcpy(s->read_msgfds, CMSG_DATA(cmsg), fd_size);
-
-        for (i = 0; i < s->read_msgfds_num; i++) {
-            int fd = s->read_msgfds[i];
-            if (fd < 0) {
-                continue;
-            }
-
-            /* O_NONBLOCK is preserved across SCM_RIGHTS so reset it */
-            qemu_set_block(fd);
-
-    #ifndef MSG_CMSG_CLOEXEC
-            qemu_set_cloexec(fd);
-    #endif
-        }
+        s->read_msgfds = msgfds;
+        s->read_msgfds_num = msgfds_num;
     }
-}
 
-static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
-{
-    TCPCharDriver *s = chr->opaque;
-    struct msghdr msg = { NULL, };
-    struct iovec iov[1];
-    union {
-        struct cmsghdr cmsg;
-        char control[CMSG_SPACE(sizeof(int) * TCP_MAX_FDS)];
-    } msg_control;
-    int flags = 0;
-    ssize_t ret;
+    for (i = 0; i < s->read_msgfds_num; i++) {
+        int fd = s->read_msgfds[i];
+        if (fd < 0) {
+            continue;
+        }
 
-    iov[0].iov_base = buf;
-    iov[0].iov_len = len;
+        /* O_NONBLOCK is preserved across SCM_RIGHTS so reset it */
+        qemu_set_block(fd);
 
-    msg.msg_iov = iov;
-    msg.msg_iovlen = 1;
-    msg.msg_control = &msg_control;
-    msg.msg_controllen = sizeof(msg_control);
-
-#ifdef MSG_CMSG_CLOEXEC
-    flags |= MSG_CMSG_CLOEXEC;
+#ifndef MSG_CMSG_CLOEXEC
+        qemu_set_cloexec(fd);
 #endif
-    ret = recvmsg(s->fd, &msg, flags);
-    if (ret > 0 && s->is_unix) {
-        unix_process_msgfd(chr, &msg);
     }
 
     return ret;
 }
-#else
-static ssize_t tcp_chr_recv(CharDriverState *chr, char *buf, size_t len)
-{
-    TCPCharDriver *s = chr->opaque;
-    return qemu_recv(s->fd, buf, len, 0);
-}
-#endif
 
 static GSource *tcp_chr_add_watch(CharDriverState *chr, GIOCondition cond)
 {
     TCPCharDriver *s = chr->opaque;
-    return g_io_create_watch(s->chan, cond);
+    return qio_channel_create_watch(s->ioc, cond);
 }
 
-static int tcp_chr_del_client(CharDriverState *chr)
+static void tcp_chr_free_connection(CharDriverState *chr)
+{
+    TCPCharDriver *s = chr->opaque;
+    int i;
+
+    if (!s->connected) {
+        return;
+    }
+
+    if (s->read_msgfds_num) {
+        for (i = 0; i < s->read_msgfds_num; i++) {
+            close(s->read_msgfds[i]);
+        }
+        g_free(s->read_msgfds);
+        s->read_msgfds = NULL;
+        s->read_msgfds_num = 0;
+    }
+
+    tcp_set_msgfds(chr, NULL, 0);
+    remove_fd_in_watch(chr);
+    object_unref(OBJECT(s->sioc));
+    s->sioc = NULL;
+    object_unref(OBJECT(s->ioc));
+    s->ioc = NULL;
+    g_free(chr->filename);
+    chr->filename = NULL;
+    s->connected = 0;
+}
+
+static void tcp_chr_disconnect(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
 
     if (!s->connected) {
-        return -1;
+        return;
     }
 
-    s->connected = 0;
-    if (s->listen_chan) {
-        s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN,
-                                       tcp_chr_accept, chr);
+    tcp_chr_free_connection(chr);
+
+    if (s->listen_ioc) {
+        s->listen_tag = qio_channel_add_watch(
+            QIO_CHANNEL(s->listen_ioc), G_IO_IN, tcp_chr_accept, chr, NULL);
     }
-    remove_fd_in_watch(chr);
-    g_io_channel_unref(s->chan);
-    s->chan = NULL;
-    closesocket(s->fd);
-    s->fd = -1;
-    SocketAddress_to_str(chr->filename, CHR_MAX_FILENAME_SIZE,
-                         "disconnected:", s->addr, s->is_listen, s->is_telnet);
+    chr->filename = SocketAddress_to_str("disconnected:", s->addr,
+                                         s->is_listen, s->is_telnet);
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
     if (s->reconnect_time) {
         qemu_chr_socket_restart_timer(chr);
     }
-    return 0;
 }
 
-static gboolean tcp_chr_read(GIOChannel *chan, GIOCondition cond, void *opaque)
+static gboolean tcp_chr_read(QIOChannel *chan, GIOCondition cond, void *opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
     uint8_t buf[READ_BUF_LEN];
     int len, size;
 
-    if (cond & G_IO_HUP) {
-        /* connection closed */
-        tcp_chr_del_client(chr);
-        return TRUE;
-    }
-
     if (!s->connected || s->max_size <= 0) {
         return TRUE;
     }
@@ -2898,9 +2936,9 @@
     if (len > s->max_size)
         len = s->max_size;
     size = tcp_chr_recv(chr, (void *)buf, len);
-    if (size == 0) {
+    if (size == 0 || size == -1) {
         /* connection closed */
-        tcp_chr_del_client(chr);
+        tcp_chr_disconnect(chr);
     } else if (size > 0) {
         if (s->do_telnetopt)
             tcp_chr_process_IAC_bytes(chr, s, buf, &size);
@@ -2923,48 +2961,27 @@
     size = tcp_chr_recv(chr, (void *) buf, len);
     if (size == 0) {
         /* connection closed */
-        tcp_chr_del_client(chr);
+        tcp_chr_disconnect(chr);
     }
 
     return size;
 }
 
-#ifndef _WIN32
-CharDriverState *qemu_chr_open_eventfd(int eventfd)
-{
-    CharDriverState *chr = qemu_chr_open_fd(eventfd, eventfd);
-
-    if (chr) {
-        chr->avail_connections = 1;
-    }
-
-    return chr;
-}
-#endif
-
 static void tcp_chr_connect(void *opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
-    struct sockaddr_storage ss, ps;
-    socklen_t ss_len = sizeof(ss), ps_len = sizeof(ps);
 
-    memset(&ss, 0, ss_len);
-    if (getsockname(s->fd, (struct sockaddr *) &ss, &ss_len) != 0) {
-        snprintf(chr->filename, CHR_MAX_FILENAME_SIZE,
-                 "Error in getsockname: %s\n", strerror(errno));
-    } else if (getpeername(s->fd, (struct sockaddr *) &ps, &ps_len) != 0) {
-        snprintf(chr->filename, CHR_MAX_FILENAME_SIZE,
-                 "Error in getpeername: %s\n", strerror(errno));
-    } else {
-        sockaddr_to_str(chr->filename, CHR_MAX_FILENAME_SIZE,
-                        &ss, ss_len, &ps, ps_len,
-                        s->is_listen, s->is_telnet);
-    }
+    g_free(chr->filename);
+    chr->filename = sockaddr_to_str(
+        &s->sioc->localAddr, s->sioc->localAddrLen,
+        &s->sioc->remoteAddr, s->sioc->remoteAddrLen,
+        s->is_listen, s->is_telnet);
 
     s->connected = 1;
-    if (s->chan) {
-        chr->fd_in_tag = io_add_watch_poll(s->chan, tcp_chr_read_poll,
+    if (s->ioc) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                           tcp_chr_read_poll,
                                            tcp_chr_read, chr);
     }
     qemu_chr_be_generic_open(chr);
@@ -2974,179 +2991,285 @@
 {
     TCPCharDriver *s = chr->opaque;
 
+    if (!s->connected) {
+        return;
+    }
+
     remove_fd_in_watch(chr);
-    if (s->chan) {
-        chr->fd_in_tag = io_add_watch_poll(s->chan, tcp_chr_read_poll,
+    if (s->ioc) {
+        chr->fd_in_tag = io_add_watch_poll(s->ioc,
+                                           tcp_chr_read_poll,
                                            tcp_chr_read, chr);
     }
 }
 
-#define IACSET(x,a,b,c) x[0] = a; x[1] = b; x[2] = c;
-static void tcp_chr_telnet_init(int fd)
+typedef struct {
+    CharDriverState *chr;
+    char buf[12];
+    size_t buflen;
+} TCPCharDriverTelnetInit;
+
+static gboolean tcp_chr_telnet_init_io(QIOChannel *ioc,
+                                       GIOCondition cond G_GNUC_UNUSED,
+                                       gpointer user_data)
 {
-    char buf[3];
-    /* Send the telnet negotion to put telnet in binary, no echo, single char mode */
-    IACSET(buf, 0xff, 0xfb, 0x01);  /* IAC WILL ECHO */
-    send(fd, (char *)buf, 3, 0);
-    IACSET(buf, 0xff, 0xfb, 0x03);  /* IAC WILL Suppress go ahead */
-    send(fd, (char *)buf, 3, 0);
-    IACSET(buf, 0xff, 0xfb, 0x00);  /* IAC WILL Binary */
-    send(fd, (char *)buf, 3, 0);
-    IACSET(buf, 0xff, 0xfd, 0x00);  /* IAC DO Binary */
-    send(fd, (char *)buf, 3, 0);
+    TCPCharDriverTelnetInit *init = user_data;
+    ssize_t ret;
+
+    ret = qio_channel_write(ioc, init->buf, init->buflen, NULL);
+    if (ret < 0) {
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            ret = 0;
+        } else {
+            tcp_chr_disconnect(init->chr);
+            return FALSE;
+        }
+    }
+    init->buflen -= ret;
+
+    if (init->buflen == 0) {
+        tcp_chr_connect(init->chr);
+        return FALSE;
+    }
+
+    memmove(init->buf, init->buf + ret, init->buflen);
+
+    return TRUE;
 }
 
-static int tcp_chr_add_client(CharDriverState *chr, int fd)
+static void tcp_chr_telnet_init(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
-    if (s->fd != -1)
-	return -1;
+    TCPCharDriverTelnetInit *init =
+        g_new0(TCPCharDriverTelnetInit, 1);
+    size_t n = 0;
 
-    qemu_set_nonblock(fd);
-    if (s->do_nodelay)
-        socket_set_nodelay(fd);
-    s->fd = fd;
-    s->chan = io_channel_from_socket(fd);
+    init->chr = chr;
+    init->buflen = 12;
+
+#define IACSET(x, a, b, c)                      \
+    do {                                        \
+        x[n++] = a;                             \
+        x[n++] = b;                             \
+        x[n++] = c;                             \
+    } while (0)
+
+    /* Prep the telnet negotion to put telnet in binary,
+     * no echo, single char mode */
+    IACSET(init->buf, 0xff, 0xfb, 0x01);  /* IAC WILL ECHO */
+    IACSET(init->buf, 0xff, 0xfb, 0x03);  /* IAC WILL Suppress go ahead */
+    IACSET(init->buf, 0xff, 0xfb, 0x00);  /* IAC WILL Binary */
+    IACSET(init->buf, 0xff, 0xfd, 0x00);  /* IAC DO Binary */
+
+#undef IACSET
+
+    qio_channel_add_watch(
+        s->ioc, G_IO_OUT,
+        tcp_chr_telnet_init_io,
+        init, NULL);
+}
+
+
+static void tcp_chr_tls_handshake(Object *source,
+                                  Error *err,
+                                  gpointer user_data)
+{
+    CharDriverState *chr = user_data;
+    TCPCharDriver *s = chr->opaque;
+
+    if (err) {
+        tcp_chr_disconnect(chr);
+    } else {
+        if (s->do_telnetopt) {
+            tcp_chr_telnet_init(chr);
+        } else {
+            tcp_chr_connect(chr);
+        }
+    }
+}
+
+
+static void tcp_chr_tls_init(CharDriverState *chr)
+{
+    TCPCharDriver *s = chr->opaque;
+    QIOChannelTLS *tioc;
+    Error *err = NULL;
+
+    if (s->is_listen) {
+        tioc = qio_channel_tls_new_server(
+            s->ioc, s->tls_creds,
+            NULL, /* XXX Use an ACL */
+            &err);
+    } else {
+        tioc = qio_channel_tls_new_client(
+            s->ioc, s->tls_creds,
+            s->addr->u.inet.data->host,
+            &err);
+    }
+    if (tioc == NULL) {
+        error_free(err);
+        tcp_chr_disconnect(chr);
+    }
+    object_unref(OBJECT(s->ioc));
+    s->ioc = QIO_CHANNEL(tioc);
+
+    qio_channel_tls_handshake(tioc,
+                              tcp_chr_tls_handshake,
+                              chr,
+                              NULL);
+}
+
+
+static int tcp_chr_new_client(CharDriverState *chr, QIOChannelSocket *sioc)
+{
+    TCPCharDriver *s = chr->opaque;
+    if (s->ioc != NULL) {
+	return -1;
+    }
+
+    s->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(sioc));
+    s->sioc = sioc;
+    object_ref(OBJECT(sioc));
+
+    qio_channel_set_blocking(s->ioc, false, NULL);
+
+    if (s->do_nodelay) {
+        qio_channel_set_delay(s->ioc, false);
+    }
     if (s->listen_tag) {
         g_source_remove(s->listen_tag);
         s->listen_tag = 0;
     }
-    tcp_chr_connect(chr);
+
+    if (s->tls_creds) {
+        tcp_chr_tls_init(chr);
+    } else {
+        if (s->do_telnetopt) {
+            tcp_chr_telnet_init(chr);
+        } else {
+            tcp_chr_connect(chr);
+        }
+    }
 
     return 0;
 }
 
-static gboolean tcp_chr_accept(GIOChannel *channel, GIOCondition cond, void *opaque)
+
+static int tcp_chr_add_client(CharDriverState *chr, int fd)
+{
+    int ret;
+    QIOChannelSocket *sioc;
+
+    sioc = qio_channel_socket_new_fd(fd, NULL);
+    if (!sioc) {
+        return -1;
+    }
+    ret = tcp_chr_new_client(chr, sioc);
+    object_unref(OBJECT(sioc));
+    return ret;
+}
+
+static gboolean tcp_chr_accept(QIOChannel *channel,
+                               GIOCondition cond,
+                               void *opaque)
 {
     CharDriverState *chr = opaque;
-    TCPCharDriver *s = chr->opaque;
-    struct sockaddr_in saddr;
-#ifndef _WIN32
-    struct sockaddr_un uaddr;
-#endif
-    struct sockaddr *addr;
-    socklen_t len;
-    int fd;
+    QIOChannelSocket *sioc;
 
-    for(;;) {
-#ifndef _WIN32
-	if (s->is_unix) {
-	    len = sizeof(uaddr);
-	    addr = (struct sockaddr *)&uaddr;
-	} else
-#endif
-	{
-	    len = sizeof(saddr);
-	    addr = (struct sockaddr *)&saddr;
-	}
-        fd = qemu_accept(s->listen_fd, addr, &len);
-        if (fd < 0 && errno != EINTR) {
-            s->listen_tag = 0;
-            return FALSE;
-        } else if (fd >= 0) {
-            if (s->do_telnetopt)
-                tcp_chr_telnet_init(fd);
-            break;
-        }
+    sioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(channel),
+                                     NULL);
+    if (!sioc) {
+        return TRUE;
     }
-    if (tcp_chr_add_client(chr, fd) < 0)
-	close(fd);
+
+    tcp_chr_new_client(chr, sioc);
+
+    object_unref(OBJECT(sioc));
 
     return TRUE;
 }
 
+static int tcp_chr_wait_connected(CharDriverState *chr, Error **errp)
+{
+    TCPCharDriver *s = chr->opaque;
+    QIOChannelSocket *sioc;
+
+    /* It can't wait on s->connected, since it is set asynchronously
+     * in TLS and telnet cases, only wait for an accepted socket */
+    while (!s->ioc) {
+        if (s->is_listen) {
+            fprintf(stderr, "QEMU waiting for connection on: %s\n",
+                    chr->filename);
+            qio_channel_set_blocking(QIO_CHANNEL(s->listen_ioc), true, NULL);
+            tcp_chr_accept(QIO_CHANNEL(s->listen_ioc), G_IO_IN, chr);
+            qio_channel_set_blocking(QIO_CHANNEL(s->listen_ioc), false, NULL);
+        } else {
+            sioc = qio_channel_socket_new();
+            if (qio_channel_socket_connect_sync(sioc, s->addr, errp) < 0) {
+                object_unref(OBJECT(sioc));
+                return -1;
+            }
+            tcp_chr_new_client(chr, sioc);
+            object_unref(OBJECT(sioc));
+        }
+    }
+
+    return 0;
+}
+
+int qemu_chr_wait_connected(CharDriverState *chr, Error **errp)
+{
+    if (chr->chr_wait_connected) {
+        return chr->chr_wait_connected(chr, errp);
+    }
+
+    return 0;
+}
+
 static void tcp_chr_close(CharDriverState *chr)
 {
     TCPCharDriver *s = chr->opaque;
-    int i;
+
+    tcp_chr_free_connection(chr);
 
     if (s->reconnect_timer) {
         g_source_remove(s->reconnect_timer);
         s->reconnect_timer = 0;
     }
     qapi_free_SocketAddress(s->addr);
-    if (s->fd >= 0) {
-        remove_fd_in_watch(chr);
-        if (s->chan) {
-            g_io_channel_unref(s->chan);
-        }
-        closesocket(s->fd);
+    if (s->listen_tag) {
+        g_source_remove(s->listen_tag);
+        s->listen_tag = 0;
     }
-    if (s->listen_fd >= 0) {
-        if (s->listen_tag) {
-            g_source_remove(s->listen_tag);
-            s->listen_tag = 0;
-        }
-        if (s->listen_chan) {
-            g_io_channel_unref(s->listen_chan);
-        }
-        closesocket(s->listen_fd);
+    if (s->listen_ioc) {
+        object_unref(OBJECT(s->listen_ioc));
     }
-    if (s->read_msgfds_num) {
-        for (i = 0; i < s->read_msgfds_num; i++) {
-            close(s->read_msgfds[i]);
-        }
-        g_free(s->read_msgfds);
-    }
-    if (s->write_msgfds_num) {
-        g_free(s->write_msgfds);
+    if (s->tls_creds) {
+        object_unref(OBJECT(s->tls_creds));
     }
     g_free(s);
     qemu_chr_be_event(chr, CHR_EVENT_CLOSED);
 }
 
-static void qemu_chr_finish_socket_connection(CharDriverState *chr, int fd)
-{
-    TCPCharDriver *s = chr->opaque;
 
-    if (s->is_listen) {
-        s->listen_fd = fd;
-        s->listen_chan = io_channel_from_socket(s->listen_fd);
-        s->listen_tag = g_io_add_watch(s->listen_chan, G_IO_IN,
-                                       tcp_chr_accept, chr);
-    } else {
-        s->connected = 1;
-        s->fd = fd;
-        socket_set_nodelay(fd);
-        s->chan = io_channel_from_socket(s->fd);
-        tcp_chr_connect(chr);
-    }
-}
-
-static void qemu_chr_socket_connected(int fd, Error *err, void *opaque)
+static void qemu_chr_socket_connected(Object *src, Error *err, void *opaque)
 {
+    QIOChannelSocket *sioc = QIO_CHANNEL_SOCKET(src);
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
 
-    if (fd < 0) {
+    if (err) {
         check_report_connect_error(chr, err);
+        object_unref(src);
         return;
     }
 
     s->connect_err_reported = false;
-    qemu_chr_finish_socket_connection(chr, fd);
+    tcp_chr_new_client(chr, sioc);
+    object_unref(OBJECT(sioc));
 }
 
-static bool qemu_chr_open_socket_fd(CharDriverState *chr, Error **errp)
-{
-    TCPCharDriver *s = chr->opaque;
-    int fd;
-
-    if (s->is_listen) {
-        fd = socket_listen_addr(s->addr, errp);
-    } else if (s->reconnect_time) {
-        fd = socket_connect_addr(s->addr, errp, qemu_chr_socket_connected, chr);
-        return fd >= 0;
-    } else {
-        fd = socket_connect_addr(s->addr, errp, NULL, NULL);
-    }
-    if (fd < 0) {
-        return false;
-    }
-
-    qemu_chr_finish_socket_connection(chr, fd);
-    return true;
-}
 
 /*********************************************************/
 /* Ring buffer chardev */
@@ -3208,13 +3331,20 @@
     chr->opaque = NULL;
 }
 
-static CharDriverState *qemu_chr_open_ringbuf(ChardevRingbuf *opts,
+static CharDriverState *qemu_chr_open_ringbuf(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
                                               Error **errp)
 {
+    ChardevRingbuf *opts = backend->u.ringbuf.data;
+    ChardevCommon *common = qapi_ChardevRingbuf_base(opts);
     CharDriverState *chr;
     RingBufCharDriver *d;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
     d = g_malloc(sizeof(*d));
 
     d->size = opts->has_size ? opts->size : 65536;
@@ -3237,7 +3367,7 @@
 
 fail:
     g_free(d);
-    g_free(chr);
+    qemu_chr_free_common(chr);
     return NULL;
 }
 
@@ -3267,7 +3397,12 @@
     }
 
     if (has_format && (format == DATA_FORMAT_BASE64)) {
-        write_data = g_base64_decode(data, &write_count);
+        write_data = qbase64_decode(data, -1,
+                                    &write_count,
+                                    errp);
+        if (!write_data) {
+            return;
+        }
     } else {
         write_data = (uint8_t *)data;
         write_count = strlen(data);
@@ -3344,21 +3479,20 @@
 
     opts = qemu_opts_create(qemu_find_opts("chardev"), label, 1, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return NULL;
     }
 
     if (strstart(filename, "mon:", &p)) {
         filename = p;
-        qemu_opt_set(opts, "mux", "on");
+        qemu_opt_set(opts, "mux", "on", &error_abort);
         if (strcmp(filename, "stdio") == 0) {
             /* Monitor is muxed to stdio: do not exit on Ctrl+C by default
              * but pass it to the guest.  Handle this only for compat syntax,
              * for -chardev syntax we have special option for this.
              * This is what -nographic did, redirecting+muxing serial+monitor
              * to stdio causing Ctrl+C to be passed to guest. */
-            qemu_opt_set(opts, "signal", "off");
+            qemu_opt_set(opts, "signal", "off", &error_abort);
         }
     }
 
@@ -3368,20 +3502,20 @@
         strcmp(filename, "braille") == 0 ||
         strcmp(filename, "testdev") == 0 ||
         strcmp(filename, "stdio")   == 0) {
-        qemu_opt_set(opts, "backend", filename);
+        qemu_opt_set(opts, "backend", filename, &error_abort);
         return opts;
     }
     if (strstart(filename, "vc", &p)) {
-        qemu_opt_set(opts, "backend", "vc");
+        qemu_opt_set(opts, "backend", "vc", &error_abort);
         if (*p == ':') {
             if (sscanf(p+1, "%7[0-9]x%7[0-9]", width, height) == 2) {
                 /* pixels */
-                qemu_opt_set(opts, "width", width);
-                qemu_opt_set(opts, "height", height);
+                qemu_opt_set(opts, "width", width, &error_abort);
+                qemu_opt_set(opts, "height", height, &error_abort);
             } else if (sscanf(p+1, "%7[0-9]Cx%7[0-9]C", width, height) == 2) {
                 /* chars */
-                qemu_opt_set(opts, "cols", width);
-                qemu_opt_set(opts, "rows", height);
+                qemu_opt_set(opts, "cols", width, &error_abort);
+                qemu_opt_set(opts, "rows", height, &error_abort);
             } else {
                 goto fail;
             }
@@ -3389,22 +3523,22 @@
         return opts;
     }
     if (strcmp(filename, "con:") == 0) {
-        qemu_opt_set(opts, "backend", "console");
+        qemu_opt_set(opts, "backend", "console", &error_abort);
         return opts;
     }
     if (strstart(filename, "COM", NULL)) {
-        qemu_opt_set(opts, "backend", "serial");
-        qemu_opt_set(opts, "path", filename);
+        qemu_opt_set(opts, "backend", "serial", &error_abort);
+        qemu_opt_set(opts, "path", filename, &error_abort);
         return opts;
     }
     if (strstart(filename, "file:", &p)) {
-        qemu_opt_set(opts, "backend", "file");
-        qemu_opt_set(opts, "path", p);
+        qemu_opt_set(opts, "backend", "file", &error_abort);
+        qemu_opt_set(opts, "path", p, &error_abort);
         return opts;
     }
     if (strstart(filename, "pipe:", &p)) {
-        qemu_opt_set(opts, "backend", "pipe");
-        qemu_opt_set(opts, "path", p);
+        qemu_opt_set(opts, "backend", "pipe", &error_abort);
+        qemu_opt_set(opts, "path", p, &error_abort);
         return opts;
     }
     if (strstart(filename, "tcp:", &p) ||
@@ -3414,27 +3548,30 @@
             if (sscanf(p, ":%32[^,]%n", port, &pos) < 1)
                 goto fail;
         }
-        qemu_opt_set(opts, "backend", "socket");
-        qemu_opt_set(opts, "host", host);
-        qemu_opt_set(opts, "port", port);
+        qemu_opt_set(opts, "backend", "socket", &error_abort);
+        qemu_opt_set(opts, "host", host, &error_abort);
+        qemu_opt_set(opts, "port", port, &error_abort);
         if (p[pos] == ',') {
-            if (qemu_opts_do_parse(opts, p+pos+1, NULL) != 0)
+            qemu_opts_do_parse(opts, p+pos+1, NULL, &local_err);
+            if (local_err) {
+                error_report_err(local_err);
                 goto fail;
+            }
         }
         if (strstart(filename, "telnet:", &p))
-            qemu_opt_set(opts, "telnet", "on");
+            qemu_opt_set(opts, "telnet", "on", &error_abort);
         return opts;
     }
     if (strstart(filename, "udp:", &p)) {
-        qemu_opt_set(opts, "backend", "udp");
+        qemu_opt_set(opts, "backend", "udp", &error_abort);
         if (sscanf(p, "%64[^:]:%32[^@,]%n", host, port, &pos) < 2) {
             host[0] = 0;
             if (sscanf(p, ":%32[^@,]%n", port, &pos) < 1) {
                 goto fail;
             }
         }
-        qemu_opt_set(opts, "host", host);
-        qemu_opt_set(opts, "port", port);
+        qemu_opt_set(opts, "host", host, &error_abort);
+        qemu_opt_set(opts, "port", port, &error_abort);
         if (p[pos] == '@') {
             p += pos + 1;
             if (sscanf(p, "%64[^:]:%32[^,]%n", host, port, &pos) < 2) {
@@ -3443,26 +3580,29 @@
                     goto fail;
                 }
             }
-            qemu_opt_set(opts, "localaddr", host);
-            qemu_opt_set(opts, "localport", port);
+            qemu_opt_set(opts, "localaddr", host, &error_abort);
+            qemu_opt_set(opts, "localport", port, &error_abort);
         }
         return opts;
     }
     if (strstart(filename, "unix:", &p)) {
-        qemu_opt_set(opts, "backend", "socket");
-        if (qemu_opts_do_parse(opts, p, "path") != 0)
+        qemu_opt_set(opts, "backend", "socket", &error_abort);
+        qemu_opts_do_parse(opts, p, "path", &local_err);
+        if (local_err) {
+            error_report_err(local_err);
             goto fail;
+        }
         return opts;
     }
     if (strstart(filename, "/dev/parport", NULL) ||
         strstart(filename, "/dev/ppi", NULL)) {
-        qemu_opt_set(opts, "backend", "parport");
-        qemu_opt_set(opts, "path", filename);
+        qemu_opt_set(opts, "backend", "parport", &error_abort);
+        qemu_opt_set(opts, "path", filename, &error_abort);
         return opts;
     }
     if (strstart(filename, "/dev/", NULL)) {
-        qemu_opt_set(opts, "backend", "tty");
-        qemu_opt_set(opts, "path", filename);
+        qemu_opt_set(opts, "backend", "tty", &error_abort);
+        qemu_opt_set(opts, "path", filename, &error_abort);
         return opts;
     }
 
@@ -3471,77 +3611,109 @@
     return NULL;
 }
 
+void qemu_chr_parse_common(QemuOpts *opts, ChardevCommon *backend)
+{
+    const char *logfile = qemu_opt_get(opts, "logfile");
+
+    backend->has_logfile = logfile != NULL;
+    backend->logfile = logfile ? g_strdup(logfile) : NULL;
+
+    backend->has_logappend = true;
+    backend->logappend = qemu_opt_get_bool(opts, "logappend", false);
+}
+
+
 static void qemu_chr_parse_file_out(QemuOpts *opts, ChardevBackend *backend,
                                     Error **errp)
 {
     const char *path = qemu_opt_get(opts, "path");
+    ChardevFile *file;
 
     if (path == NULL) {
         error_setg(errp, "chardev: file: no filename given");
         return;
     }
-    backend->file = g_new0(ChardevFile, 1);
-    backend->file->out = g_strdup(path);
+    file = backend->u.file.data = g_new0(ChardevFile, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevFile_base(file));
+    file->out = g_strdup(path);
+
+    file->has_append = true;
+    file->append = qemu_opt_get_bool(opts, "append", false);
 }
 
 static void qemu_chr_parse_stdio(QemuOpts *opts, ChardevBackend *backend,
                                  Error **errp)
 {
-    backend->stdio = g_new0(ChardevStdio, 1);
-    backend->stdio->has_signal = true;
-    backend->stdio->signal = qemu_opt_get_bool(opts, "signal", true);
+    ChardevStdio *stdio;
+
+    stdio = backend->u.stdio.data = g_new0(ChardevStdio, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevStdio_base(stdio));
+    stdio->has_signal = true;
+    stdio->signal = qemu_opt_get_bool(opts, "signal", true);
 }
 
+#ifdef HAVE_CHARDEV_SERIAL
 static void qemu_chr_parse_serial(QemuOpts *opts, ChardevBackend *backend,
                                   Error **errp)
 {
     const char *device = qemu_opt_get(opts, "path");
+    ChardevHostdev *serial;
 
     if (device == NULL) {
         error_setg(errp, "chardev: serial/tty: no device path given");
         return;
     }
-    backend->serial = g_new0(ChardevHostdev, 1);
-    backend->serial->device = g_strdup(device);
+    serial = backend->u.serial.data = g_new0(ChardevHostdev, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevHostdev_base(serial));
+    serial->device = g_strdup(device);
 }
+#endif
 
+#ifdef HAVE_CHARDEV_PARPORT
 static void qemu_chr_parse_parallel(QemuOpts *opts, ChardevBackend *backend,
                                     Error **errp)
 {
     const char *device = qemu_opt_get(opts, "path");
+    ChardevHostdev *parallel;
 
     if (device == NULL) {
         error_setg(errp, "chardev: parallel: no device path given");
         return;
     }
-    backend->parallel = g_new0(ChardevHostdev, 1);
-    backend->parallel->device = g_strdup(device);
+    parallel = backend->u.parallel.data = g_new0(ChardevHostdev, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevHostdev_base(parallel));
+    parallel->device = g_strdup(device);
 }
+#endif
 
 static void qemu_chr_parse_pipe(QemuOpts *opts, ChardevBackend *backend,
                                 Error **errp)
 {
     const char *device = qemu_opt_get(opts, "path");
+    ChardevHostdev *dev;
 
     if (device == NULL) {
         error_setg(errp, "chardev: pipe: no device path given");
         return;
     }
-    backend->pipe = g_new0(ChardevHostdev, 1);
-    backend->pipe->device = g_strdup(device);
+    dev = backend->u.pipe.data = g_new0(ChardevHostdev, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevHostdev_base(dev));
+    dev->device = g_strdup(device);
 }
 
 static void qemu_chr_parse_ringbuf(QemuOpts *opts, ChardevBackend *backend,
                                    Error **errp)
 {
     int val;
+    ChardevRingbuf *ringbuf;
 
-    backend->ringbuf = g_new0(ChardevRingbuf, 1);
+    ringbuf = backend->u.ringbuf.data = g_new0(ChardevRingbuf, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevRingbuf_base(ringbuf));
 
     val = qemu_opt_get_size(opts, "size", 0);
     if (val != 0) {
-        backend->ringbuf->has_size = true;
-        backend->ringbuf->size = val;
+        ringbuf->has_size = true;
+        ringbuf->size = val;
     }
 }
 
@@ -3549,13 +3721,15 @@
                                Error **errp)
 {
     const char *chardev = qemu_opt_get(opts, "chardev");
+    ChardevMux *mux;
 
     if (chardev == NULL) {
         error_setg(errp, "chardev: mux: no chardev given");
         return;
     }
-    backend->mux = g_new0(ChardevMux, 1);
-    backend->mux->chardev = g_strdup(chardev);
+    mux = backend->u.mux.data = g_new0(ChardevMux, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevMux_base(mux));
+    mux->chardev = g_strdup(chardev);
 }
 
 static void qemu_chr_parse_socket(QemuOpts *opts, ChardevBackend *backend,
@@ -3569,7 +3743,9 @@
     const char *path = qemu_opt_get(opts, "path");
     const char *host = qemu_opt_get(opts, "host");
     const char *port = qemu_opt_get(opts, "port");
+    const char *tls_creds = qemu_opt_get(opts, "tls-creds");
     SocketAddress *addr;
+    ChardevSocket *sock;
 
     if (!path) {
         if (!host) {
@@ -3580,39 +3756,49 @@
             error_setg(errp, "chardev: socket: no port given");
             return;
         }
+    } else {
+        if (tls_creds) {
+            error_setg(errp, "TLS can only be used over TCP socket");
+            return;
+        }
     }
 
-    backend->socket = g_new0(ChardevSocket, 1);
+    sock = backend->u.socket.data = g_new0(ChardevSocket, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevSocket_base(sock));
 
-    backend->socket->has_nodelay = true;
-    backend->socket->nodelay = do_nodelay;
-    backend->socket->has_server = true;
-    backend->socket->server = is_listen;
-    backend->socket->has_telnet = true;
-    backend->socket->telnet = is_telnet;
-    backend->socket->has_wait = true;
-    backend->socket->wait = is_waitconnect;
-    backend->socket->has_reconnect = true;
-    backend->socket->reconnect = reconnect;
+    sock->has_nodelay = true;
+    sock->nodelay = do_nodelay;
+    sock->has_server = true;
+    sock->server = is_listen;
+    sock->has_telnet = true;
+    sock->telnet = is_telnet;
+    sock->has_wait = true;
+    sock->wait = is_waitconnect;
+    sock->has_reconnect = true;
+    sock->reconnect = reconnect;
+    sock->tls_creds = g_strdup(tls_creds);
 
     addr = g_new0(SocketAddress, 1);
     if (path) {
-        addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-        addr->q_unix = g_new0(UnixSocketAddress, 1);
-        addr->q_unix->path = g_strdup(path);
+        UnixSocketAddress *q_unix;
+        addr->type = SOCKET_ADDRESS_KIND_UNIX;
+        q_unix = addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+        q_unix->path = g_strdup(path);
     } else {
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = g_new0(InetSocketAddress, 1);
-        addr->inet->host = g_strdup(host);
-        addr->inet->port = g_strdup(port);
-        addr->inet->has_to = qemu_opt_get(opts, "to");
-        addr->inet->to = qemu_opt_get_number(opts, "to", 0);
-        addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
-        addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
-        addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
-        addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet.data = g_new(InetSocketAddress, 1);
+        *addr->u.inet.data = (InetSocketAddress) {
+            .host = g_strdup(host),
+            .port = g_strdup(port),
+            .has_to = qemu_opt_get(opts, "to"),
+            .to = qemu_opt_get_number(opts, "to", 0),
+            .has_ipv4 = qemu_opt_get(opts, "ipv4"),
+            .ipv4 = qemu_opt_get_bool(opts, "ipv4", 0),
+            .has_ipv6 = qemu_opt_get(opts, "ipv6"),
+            .ipv6 = qemu_opt_get_bool(opts, "ipv6", 0),
+        };
     }
-    backend->socket->addr = addr;
+    sock->addr = addr;
 }
 
 static void qemu_chr_parse_udp(QemuOpts *opts, ChardevBackend *backend,
@@ -3624,6 +3810,7 @@
     const char *localport = qemu_opt_get(opts, "localport");
     bool has_local = false;
     SocketAddress *addr;
+    ChardevUdp *udp;
 
     if (host == NULL || strlen(host) == 0) {
         host = "localhost";
@@ -3643,27 +3830,32 @@
         has_local = true;
     }
 
-    backend->udp = g_new0(ChardevUdp, 1);
+    udp = backend->u.udp.data = g_new0(ChardevUdp, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevUdp_base(udp));
 
     addr = g_new0(SocketAddress, 1);
-    addr->kind = SOCKET_ADDRESS_KIND_INET;
-    addr->inet = g_new0(InetSocketAddress, 1);
-    addr->inet->host = g_strdup(host);
-    addr->inet->port = g_strdup(port);
-    addr->inet->has_ipv4 = qemu_opt_get(opts, "ipv4");
-    addr->inet->ipv4 = qemu_opt_get_bool(opts, "ipv4", 0);
-    addr->inet->has_ipv6 = qemu_opt_get(opts, "ipv6");
-    addr->inet->ipv6 = qemu_opt_get_bool(opts, "ipv6", 0);
-    backend->udp->remote = addr;
+    addr->type = SOCKET_ADDRESS_KIND_INET;
+    addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup(host),
+        .port = g_strdup(port),
+        .has_ipv4 = qemu_opt_get(opts, "ipv4"),
+        .ipv4 = qemu_opt_get_bool(opts, "ipv4", 0),
+        .has_ipv6 = qemu_opt_get(opts, "ipv6"),
+        .ipv6 = qemu_opt_get_bool(opts, "ipv6", 0),
+    };
+    udp->remote = addr;
 
     if (has_local) {
-        backend->udp->has_local = true;
+        udp->has_local = true;
         addr = g_new0(SocketAddress, 1);
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = g_new0(InetSocketAddress, 1);
-        addr->inet->host = g_strdup(localaddr);
-        addr->inet->port = g_strdup(localport);
-        backend->udp->local = addr;
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet.data = g_new(InetSocketAddress, 1);
+        *addr->u.inet.data = (InetSocketAddress) {
+            .host = g_strdup(localaddr),
+            .port = g_strdup(localport),
+        };
+        udp->local = addr;
     }
 }
 
@@ -3671,12 +3863,16 @@
     const char *name;
     ChardevBackendKind kind;
     void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp);
+    CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                               ChardevReturn *ret, Error **errp);
 } CharDriver;
 
 static GSList *backends;
 
 void register_char_driver(const char *name, ChardevBackendKind kind,
-        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp))
+        void (*parse)(QemuOpts *opts, ChardevBackend *backend, Error **errp),
+        CharDriverState *(*create)(const char *id, ChardevBackend *backend,
+                                   ChardevReturn *ret, Error **errp))
 {
     CharDriver *s;
 
@@ -3684,6 +3880,7 @@
     s->name = g_strdup(name);
     s->kind = kind;
     s->parse = parse;
+    s->create = create;
 
     backends = g_slist_append(backends, s);
 }
@@ -3731,14 +3928,19 @@
     }
 
     chr = NULL;
-    backend->kind = cd->kind;
+    backend->type = cd->kind;
     if (cd->parse) {
         cd->parse(opts, backend, &local_err);
         if (local_err) {
             error_propagate(errp, local_err);
             goto qapi_out;
         }
+    } else {
+        ChardevCommon *cc = g_new0(ChardevCommon, 1);
+        qemu_chr_parse_common(opts, cc);
+        backend->u.null.data = cc; /* Any ChardevCommon member would work */
     }
+
     ret = qmp_chardev_add(bid ? bid : id, backend, errp);
     if (!ret) {
         goto qapi_out;
@@ -3748,9 +3950,9 @@
         qapi_free_ChardevBackend(backend);
         qapi_free_ChardevReturn(ret);
         backend = g_new0(ChardevBackend, 1);
-        backend->mux = g_new0(ChardevMux, 1);
-        backend->kind = CHARDEV_BACKEND_KIND_MUX;
-        backend->mux->chardev = g_strdup(bid);
+        backend->u.mux.data = g_new0(ChardevMux, 1);
+        backend->type = CHARDEV_BACKEND_KIND_MUX;
+        backend->u.mux.data->chardev = g_strdup(bid);
         ret = qmp_chardev_add(id, backend, errp);
         if (!ret) {
             chr = qemu_chr_find(bid);
@@ -3774,7 +3976,8 @@
     return NULL;
 }
 
-CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
+CharDriverState *qemu_chr_new_noreplay(const char *label, const char *filename,
+                                       void (*init)(struct CharDriverState *s))
 {
     const char *p;
     CharDriverState *chr;
@@ -3791,8 +3994,7 @@
 
     chr = qemu_chr_new_from_opts(opts, init, &err);
     if (err) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
     }
     if (chr && qemu_opt_get_bool(opts, "mux", 0)) {
         qemu_chr_fe_claim_no_fail(chr);
@@ -3801,6 +4003,21 @@
     return chr;
 }
 
+CharDriverState *qemu_chr_new(const char *label, const char *filename, void (*init)(struct CharDriverState *s))
+{
+    CharDriverState *chr;
+    chr = qemu_chr_new_noreplay(label, filename, init);
+    if (chr) {
+        chr->replay = replay_mode != REPLAY_MODE_NONE;
+        if (chr->replay && chr->chr_ioctl) {
+            fprintf(stderr,
+                    "Replay: ioctl is not supported for serial devices yet\n");
+        }
+        replay_register_char_driver(chr);
+    }
+    return chr;
+}
+
 void qemu_chr_fe_set_echo(struct CharDriverState *chr, bool echo)
 {
     if (chr->chr_set_echo) {
@@ -3826,19 +4043,19 @@
     }
 }
 
-int qemu_chr_fe_add_watch(CharDriverState *s, GIOCondition cond,
-                          GIOFunc func, void *user_data)
+guint qemu_chr_fe_add_watch(CharDriverState *s, GIOCondition cond,
+                            GIOFunc func, void *user_data)
 {
     GSource *src;
     guint tag;
 
     if (s->chr_add_watch == NULL) {
-        return -ENOSYS;
+        return 0;
     }
 
     src = s->chr_add_watch(s, cond);
     if (!src) {
-        return -EINVAL;
+        return 0;
     }
 
     g_source_set_callback(src, (GSourceFunc)func, user_data, NULL);
@@ -3871,18 +4088,37 @@
     s->avail_connections++;
 }
 
-void qemu_chr_delete(CharDriverState *chr)
+void qemu_chr_disconnect(CharDriverState *chr)
 {
-    QTAILQ_REMOVE(&chardevs, chr, next);
+    if (chr->chr_disconnect) {
+        chr->chr_disconnect(chr);
+    }
+}
+
+static void qemu_chr_free_common(CharDriverState *chr)
+{
+    g_free(chr->filename);
+    g_free(chr->label);
+    qemu_opts_del(chr->opts);
+    if (chr->logfd != -1) {
+        close(chr->logfd);
+    }
+    qemu_mutex_destroy(&chr->chr_write_lock);
+    g_free(chr);
+}
+
+void qemu_chr_free(CharDriverState *chr)
+{
     if (chr->chr_close) {
         chr->chr_close(chr);
     }
-    g_free(chr->filename);
-    g_free(chr->label);
-    if (chr->opts) {
-        qemu_opts_del(chr->opts);
-    }
-    g_free(chr);
+    qemu_chr_free_common(chr);
+}
+
+void qemu_chr_delete(CharDriverState *chr)
+{
+    QTAILQ_REMOVE(&chardevs, chr, next);
+    qemu_chr_free(chr);
 }
 
 ChardevInfoList *qmp_query_chardev(Error **errp)
@@ -3935,22 +4171,6 @@
     return NULL;
 }
 
-/* Get a character (serial) device interface.  */
-CharDriverState *qemu_char_get_next_serial(void)
-{
-    static int next_serial;
-    CharDriverState *chr;
-
-    /* FIXME: This function needs to go away: use chardev properties!  */
-
-    while (next_serial < MAX_SERIAL_PORTS && serial_hds[next_serial]) {
-        chr = serial_hds[next_serial++];
-        qemu_chr_fe_claim_no_fail(chr);
-        return chr;
-    }
-    return NULL;
-}
-
 QemuOptsList qemu_chardev_opts = {
     .name = "chardev",
     .implied_opt_name = "backend",
@@ -3999,6 +4219,9 @@
             .name = "telnet",
             .type = QEMU_OPT_BOOL,
         },{
+            .name = "tls-creds",
+            .type = QEMU_OPT_STRING,
+        },{
             .name = "width",
             .type = QEMU_OPT_NUMBER,
         },{
@@ -4028,6 +4251,15 @@
         },{
             .name = "chardev",
             .type = QEMU_OPT_STRING,
+        },{
+            .name = "append",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "logfile",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "logappend",
+            .type = QEMU_OPT_BOOL,
         },
         { /* end of list */ }
     },
@@ -4035,35 +4267,49 @@
 
 #ifdef _WIN32
 
-static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+static CharDriverState *qmp_chardev_open_file(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevFile *file = backend->u.file.data;
+    ChardevCommon *common = qapi_ChardevFile_base(file);
     HANDLE out;
+    DWORD accessmode;
+    DWORD flags;
 
     if (file->has_in) {
         error_setg(errp, "input file not supported");
         return NULL;
     }
 
-    out = CreateFile(file->out, GENERIC_WRITE, FILE_SHARE_READ, NULL,
-                     OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (file->has_append && file->append) {
+        /* Append to file if it already exists. */
+        accessmode = FILE_GENERIC_WRITE & ~FILE_WRITE_DATA;
+        flags = OPEN_ALWAYS;
+    } else {
+        /* Truncate file if it already exists. */
+        accessmode = GENERIC_WRITE;
+        flags = CREATE_ALWAYS;
+    }
+
+    out = win32CreateFile(file->out, accessmode, FILE_SHARE_READ, NULL, flags,
+                     FILE_ATTRIBUTE_NORMAL, NULL);
     if (out == INVALID_HANDLE_VALUE) {
         error_setg(errp, "open %s failed", file->out);
         return NULL;
     }
-    return qemu_chr_open_win_file(out);
+    return qemu_chr_open_win_file(out, common, errp);
 }
 
-static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+static CharDriverState *qmp_chardev_open_serial(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
-    return qemu_chr_open_win_path(serial->device);
-}
-
-static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
-                                                  Error **errp)
-{
-    error_setg(errp, "character device backend type 'parallel' not supported");
-    return NULL;
+    ChardevHostdev *serial = backend->u.serial.data;
+    ChardevCommon *common = qapi_ChardevHostdev_base(serial);
+    return qemu_chr_open_win_path(serial->device, common, errp);
 }
 
 #else /* WIN32 */
@@ -4080,11 +4326,22 @@
     return fd;
 }
 
-static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
+static CharDriverState *qmp_chardev_open_file(const char *id,
+                                              ChardevBackend *backend,
+                                              ChardevReturn *ret,
+                                              Error **errp)
 {
+    ChardevFile *file = backend->u.file.data;
+    ChardevCommon *common = qapi_ChardevFile_base(file);
     int flags, in = -1, out;
 
-    flags = O_WRONLY | O_TRUNC | O_CREAT | O_BINARY;
+    flags = O_WRONLY | O_CREAT | O_BINARY;
+    if (file->has_append && file->append) {
+        flags |= O_APPEND;
+    } else {
+        flags |= O_TRUNC;
+    }
+
     out = qmp_chardev_open_file_source(file->out, flags, errp);
     if (out < 0) {
         return NULL;
@@ -4099,13 +4356,17 @@
         }
     }
 
-    return qemu_chr_open_fd(in, out);
+    return qemu_chr_open_fd(in, out, common, errp);
 }
 
-static CharDriverState *qmp_chardev_open_serial(ChardevHostdev *serial,
+#ifdef HAVE_CHARDEV_SERIAL
+static CharDriverState *qmp_chardev_open_serial(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
-#ifdef HAVE_CHARDEV_TTY
+    ChardevHostdev *serial = backend->u.serial.data;
+    ChardevCommon *common = qapi_ChardevHostdev_base(serial);
     int fd;
 
     fd = qmp_chardev_open_file_source(serial->device, O_RDWR, errp);
@@ -4113,45 +4374,35 @@
         return NULL;
     }
     qemu_set_nonblock(fd);
-    return qemu_chr_open_tty_fd(fd);
-#else
-    error_setg(errp, "character device backend type 'serial' not supported");
-    return NULL;
-#endif
+    return qemu_chr_open_tty_fd(fd, common, errp);
 }
+#endif
 
-static CharDriverState *qmp_chardev_open_parallel(ChardevHostdev *parallel,
+#ifdef HAVE_CHARDEV_PARPORT
+static CharDriverState *qmp_chardev_open_parallel(const char *id,
+                                                  ChardevBackend *backend,
+                                                  ChardevReturn *ret,
                                                   Error **errp)
 {
-#ifdef HAVE_CHARDEV_PARPORT
+    ChardevHostdev *parallel = backend->u.parallel.data;
+    ChardevCommon *common = qapi_ChardevHostdev_base(parallel);
     int fd;
 
     fd = qmp_chardev_open_file_source(parallel->device, O_RDWR, errp);
     if (fd < 0) {
         return NULL;
     }
-    return qemu_chr_open_pp_fd(fd);
-#else
-    error_setg(errp, "character device backend type 'parallel' not supported");
-    return NULL;
-#endif
+    return qemu_chr_open_pp_fd(fd, common, errp);
 }
+#endif
 
 #endif /* WIN32 */
 
-static void socket_try_connect(CharDriverState *chr)
-{
-    Error *err = NULL;
-
-    if (!qemu_chr_open_socket_fd(chr, &err)) {
-        check_report_connect_error(chr, err);
-    }
-}
-
 static gboolean socket_reconnect_timeout(gpointer opaque)
 {
     CharDriverState *chr = opaque;
     TCPCharDriver *s = chr->opaque;
+    QIOChannelSocket *sioc;
 
     s->reconnect_timer = 0;
 
@@ -4159,50 +4410,92 @@
         return false;
     }
 
-    socket_try_connect(chr);
+    sioc = qio_channel_socket_new();
+    qio_channel_socket_connect_async(sioc, s->addr,
+                                     qemu_chr_socket_connected,
+                                     chr, NULL);
 
     return false;
 }
 
-static CharDriverState *qmp_chardev_open_socket(ChardevSocket *sock,
+static CharDriverState *qmp_chardev_open_socket(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
                                                 Error **errp)
 {
     CharDriverState *chr;
     TCPCharDriver *s;
+    ChardevSocket *sock = backend->u.socket.data;
     SocketAddress *addr = sock->addr;
     bool do_nodelay     = sock->has_nodelay ? sock->nodelay : false;
     bool is_listen      = sock->has_server  ? sock->server  : true;
     bool is_telnet      = sock->has_telnet  ? sock->telnet  : false;
     bool is_waitconnect = sock->has_wait    ? sock->wait    : false;
     int64_t reconnect   = sock->has_reconnect ? sock->reconnect : 0;
+    ChardevCommon *common = qapi_ChardevSocket_base(sock);
+    QIOChannelSocket *sioc = NULL;
 
-    chr = qemu_chr_alloc();
-    s = g_malloc0(sizeof(TCPCharDriver));
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+    s = g_new0(TCPCharDriver, 1);
 
-    s->fd = -1;
-    s->listen_fd = -1;
-    s->is_unix = addr->kind == SOCKET_ADDRESS_KIND_UNIX;
+    s->is_unix = addr->type == SOCKET_ADDRESS_KIND_UNIX;
     s->is_listen = is_listen;
     s->is_telnet = is_telnet;
     s->do_nodelay = do_nodelay;
-    qapi_copy_SocketAddress(&s->addr, sock->addr);
+    if (sock->tls_creds) {
+        Object *creds;
+        creds = object_resolve_path_component(
+            object_get_objects_root(), sock->tls_creds);
+        if (!creds) {
+            error_setg(errp, "No TLS credentials with id '%s'",
+                       sock->tls_creds);
+            goto error;
+        }
+        s->tls_creds = (QCryptoTLSCreds *)
+            object_dynamic_cast(creds,
+                                TYPE_QCRYPTO_TLS_CREDS);
+        if (!s->tls_creds) {
+            error_setg(errp, "Object with id '%s' is not TLS credentials",
+                       sock->tls_creds);
+            goto error;
+        }
+        object_ref(OBJECT(s->tls_creds));
+        if (is_listen) {
+            if (s->tls_creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+                error_setg(errp, "%s",
+                           "Expected TLS credentials for server endpoint");
+                goto error;
+            }
+        } else {
+            if (s->tls_creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT) {
+                error_setg(errp, "%s",
+                           "Expected TLS credentials for client endpoint");
+                goto error;
+            }
+        }
+    }
+
+    s->addr = QAPI_CLONE(SocketAddress, sock->addr);
 
     chr->opaque = s;
+    chr->chr_wait_connected = tcp_chr_wait_connected;
     chr->chr_write = tcp_chr_write;
     chr->chr_sync_read = tcp_chr_sync_read;
     chr->chr_close = tcp_chr_close;
+    chr->chr_disconnect = tcp_chr_disconnect;
     chr->get_msgfds = tcp_get_msgfds;
     chr->set_msgfds = tcp_set_msgfds;
     chr->chr_add_client = tcp_chr_add_client;
-    chr->chr_del_client = tcp_chr_del_client;
     chr->chr_add_watch = tcp_chr_add_watch;
     chr->chr_update_read_handler = tcp_chr_update_read_handler;
     /* be isn't opened until we get a connection */
     chr->explicit_be_open = true;
 
-    chr->filename = g_malloc(CHR_MAX_FILENAME_SIZE);
-    SocketAddress_to_str(chr->filename, CHR_MAX_FILENAME_SIZE, "disconnected:",
-                         addr, is_listen, is_telnet);
+    chr->filename = SocketAddress_to_str("disconnected:",
+                                         addr, is_listen, is_telnet);
 
     if (is_listen) {
         if (is_telnet) {
@@ -4213,41 +4506,71 @@
     }
 
     if (s->reconnect_time) {
-        socket_try_connect(chr);
-    } else if (!qemu_chr_open_socket_fd(chr, errp)) {
-        g_free(s);
-        g_free(chr->filename);
-        g_free(chr);
-        return NULL;
-    }
-
-    if (is_listen && is_waitconnect) {
-        fprintf(stderr, "QEMU waiting for connection on: %s\n",
-                chr->filename);
-        tcp_chr_accept(s->listen_chan, G_IO_IN, chr);
-        qemu_set_nonblock(s->listen_fd);
+        sioc = qio_channel_socket_new();
+        qio_channel_socket_connect_async(sioc, s->addr,
+                                         qemu_chr_socket_connected,
+                                         chr, NULL);
+    } else {
+        if (s->is_listen) {
+            sioc = qio_channel_socket_new();
+            if (qio_channel_socket_listen_sync(sioc, s->addr, errp) < 0) {
+                goto error;
+            }
+            s->listen_ioc = sioc;
+            if (is_waitconnect &&
+                qemu_chr_wait_connected(chr, errp) < 0) {
+                goto error;
+            }
+            if (!s->ioc) {
+                s->listen_tag = qio_channel_add_watch(
+                    QIO_CHANNEL(s->listen_ioc), G_IO_IN,
+                    tcp_chr_accept, chr, NULL);
+            }
+        } else if (qemu_chr_wait_connected(chr, errp) < 0) {
+            goto error;
+        }
     }
 
     return chr;
+
+ error:
+    if (sioc) {
+        object_unref(OBJECT(sioc));
+    }
+    if (s->tls_creds) {
+        object_unref(OBJECT(s->tls_creds));
+    }
+    g_free(s);
+    qemu_chr_free_common(chr);
+    return NULL;
 }
 
-static CharDriverState *qmp_chardev_open_udp(ChardevUdp *udp,
+static CharDriverState *qmp_chardev_open_udp(const char *id,
+                                             ChardevBackend *backend,
+                                             ChardevReturn *ret,
                                              Error **errp)
 {
-    int fd;
+    ChardevUdp *udp = backend->u.udp.data;
+    ChardevCommon *common = qapi_ChardevUdp_base(udp);
+    QIOChannelSocket *sioc = qio_channel_socket_new();
 
-    fd = socket_dgram(udp->remote, udp->local, errp);
-    if (fd < 0) {
+    if (qio_channel_socket_dgram_sync(sioc,
+                                      udp->local, udp->remote,
+                                      errp) < 0) {
+        object_unref(OBJECT(sioc));
         return NULL;
     }
-    return qemu_chr_open_udp_fd(fd);
+    return qemu_chr_open_udp(sioc, common, errp);
 }
 
 ChardevReturn *qmp_chardev_add(const char *id, ChardevBackend *backend,
                                Error **errp)
 {
     ChardevReturn *ret = g_new0(ChardevReturn, 1);
-    CharDriverState *base, *chr = NULL;
+    CharDriverState *chr = NULL;
+    Error *local_err = NULL;
+    GSList *i;
+    CharDriver *cd;
 
     chr = qemu_chr_find(id);
     if (chr) {
@@ -4256,106 +4579,40 @@
         return NULL;
     }
 
-    switch (backend->kind) {
-    case CHARDEV_BACKEND_KIND_FILE:
-        chr = qmp_chardev_open_file(backend->file, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_SERIAL:
-        chr = qmp_chardev_open_serial(backend->serial, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_PARALLEL:
-        chr = qmp_chardev_open_parallel(backend->parallel, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_PIPE:
-        chr = qemu_chr_open_pipe(backend->pipe);
-        break;
-    case CHARDEV_BACKEND_KIND_SOCKET:
-        chr = qmp_chardev_open_socket(backend->socket, errp);
-        break;
-    case CHARDEV_BACKEND_KIND_UDP:
-        chr = qmp_chardev_open_udp(backend->udp, errp);
-        break;
-#ifdef HAVE_CHARDEV_TTY
-    case CHARDEV_BACKEND_KIND_PTY:
-        chr = qemu_chr_open_pty(id, ret);
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_NULL:
-        chr = qemu_chr_open_null();
-        break;
-    case CHARDEV_BACKEND_KIND_MUX:
-        base = qemu_chr_find(backend->mux->chardev);
-        if (base == NULL) {
-            error_setg(errp, "mux: base chardev %s not found",
-                       backend->mux->chardev);
+    for (i = backends; i; i = i->next) {
+        cd = i->data;
+
+        if (cd->kind == backend->type) {
+            chr = cd->create(id, backend, ret, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
+                goto out_error;
+            }
             break;
         }
-        chr = qemu_chr_open_mux(base);
-        break;
-    case CHARDEV_BACKEND_KIND_MSMOUSE:
-        chr = qemu_chr_open_msmouse();
-        break;
-#ifdef CONFIG_BRLAPI
-    case CHARDEV_BACKEND_KIND_BRAILLE:
-        chr = chr_baum_init();
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_TESTDEV:
-        chr = chr_testdev_init();
-        break;
-    case CHARDEV_BACKEND_KIND_STDIO:
-        chr = qemu_chr_open_stdio(backend->stdio);
-        break;
-#ifdef _WIN32
-    case CHARDEV_BACKEND_KIND_CONSOLE:
-        chr = qemu_chr_open_win_con();
-        break;
-#endif
-#ifdef CONFIG_SPICE
-    case CHARDEV_BACKEND_KIND_SPICEVMC:
-        chr = qemu_chr_open_spice_vmc(backend->spicevmc->type);
-        break;
-    case CHARDEV_BACKEND_KIND_SPICEPORT:
-        chr = qemu_chr_open_spice_port(backend->spiceport->fqdn);
-        break;
-#endif
-    case CHARDEV_BACKEND_KIND_VC:
-        chr = vc_init(backend->vc);
-        break;
-    case CHARDEV_BACKEND_KIND_RINGBUF:
-    case CHARDEV_BACKEND_KIND_MEMORY:
-        chr = qemu_chr_open_ringbuf(backend->ringbuf, errp);
-        break;
-    default:
-        error_setg(errp, "unknown chardev backend (%d)", backend->kind);
-        break;
     }
 
-    /*
-     * Character backend open hasn't been fully converted to the Error
-     * API.  Some opens fail without setting an error.  Set a generic
-     * error then.
-     * TODO full conversion to Error API
-     */
-    if (chr == NULL && errp && !*errp) {
-        error_setg(errp, "Failed to create chardev");
+    if (chr == NULL) {
+        assert(!i);
+        error_setg(errp, "chardev backend not available");
+        goto out_error;
     }
-    if (chr) {
-        chr->label = g_strdup(id);
-        chr->avail_connections =
-            (backend->kind == CHARDEV_BACKEND_KIND_MUX) ? MAX_MUX : 1;
-        if (!chr->filename) {
-            chr->filename = g_strdup(ChardevBackendKind_lookup[backend->kind]);
-        }
-        if (!chr->explicit_be_open) {
-            qemu_chr_be_event(chr, CHR_EVENT_OPENED);
-        }
-        QTAILQ_INSERT_TAIL(&chardevs, chr, next);
-        return ret;
-    } else {
-        g_free(ret);
-        return NULL;
+
+    chr->label = g_strdup(id);
+    chr->avail_connections =
+        (backend->type == CHARDEV_BACKEND_KIND_MUX) ? MAX_MUX : 1;
+    if (!chr->filename) {
+        chr->filename = g_strdup(ChardevBackendKind_lookup[backend->type]);
     }
+    if (!chr->explicit_be_open) {
+        qemu_chr_be_event(chr, CHR_EVENT_OPENED);
+    }
+    QTAILQ_INSERT_TAIL(&chardevs, chr, next);
+    return ret;
+
+out_error:
+    g_free(ret);
+    return NULL;
 }
 
 void qmp_chardev_remove(const char *id, Error **errp)
@@ -4372,37 +4629,64 @@
         error_setg(errp, "Chardev '%s' is busy", id);
         return;
     }
+    if (chr->replay) {
+        error_setg(errp,
+            "Chardev '%s' cannot be unplugged in record/replay mode", id);
+        return;
+    }
     qemu_chr_delete(chr);
 }
 
+void qemu_chr_cleanup(void)
+{
+    CharDriverState *chr, *tmp;
+
+    QTAILQ_FOREACH_SAFE(chr, &chardevs, next, tmp) {
+        qemu_chr_delete(chr);
+    }
+}
+
 static void register_types(void)
 {
-    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL);
+    register_char_driver("null", CHARDEV_BACKEND_KIND_NULL, NULL,
+                         qemu_chr_open_null);
     register_char_driver("socket", CHARDEV_BACKEND_KIND_SOCKET,
-                         qemu_chr_parse_socket);
-    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp);
+                         qemu_chr_parse_socket, qmp_chardev_open_socket);
+    register_char_driver("udp", CHARDEV_BACKEND_KIND_UDP, qemu_chr_parse_udp,
+                         qmp_chardev_open_udp);
     register_char_driver("ringbuf", CHARDEV_BACKEND_KIND_RINGBUF,
-                         qemu_chr_parse_ringbuf);
+                         qemu_chr_parse_ringbuf, qemu_chr_open_ringbuf);
     register_char_driver("file", CHARDEV_BACKEND_KIND_FILE,
-                         qemu_chr_parse_file_out);
+                         qemu_chr_parse_file_out, qmp_chardev_open_file);
     register_char_driver("stdio", CHARDEV_BACKEND_KIND_STDIO,
-                         qemu_chr_parse_stdio);
+                         qemu_chr_parse_stdio, qemu_chr_open_stdio);
+#if defined HAVE_CHARDEV_SERIAL
     register_char_driver("serial", CHARDEV_BACKEND_KIND_SERIAL,
-                         qemu_chr_parse_serial);
+                         qemu_chr_parse_serial, qmp_chardev_open_serial);
     register_char_driver("tty", CHARDEV_BACKEND_KIND_SERIAL,
-                         qemu_chr_parse_serial);
+                         qemu_chr_parse_serial, qmp_chardev_open_serial);
+#endif
+#ifdef HAVE_CHARDEV_PARPORT
     register_char_driver("parallel", CHARDEV_BACKEND_KIND_PARALLEL,
-                         qemu_chr_parse_parallel);
+                         qemu_chr_parse_parallel, qmp_chardev_open_parallel);
     register_char_driver("parport", CHARDEV_BACKEND_KIND_PARALLEL,
-                         qemu_chr_parse_parallel);
-    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL);
-    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL);
+                         qemu_chr_parse_parallel, qmp_chardev_open_parallel);
+#endif
+#ifdef HAVE_CHARDEV_PTY
+    register_char_driver("pty", CHARDEV_BACKEND_KIND_PTY, NULL,
+                         qemu_chr_open_pty);
+#endif
+#ifdef _WIN32
+    register_char_driver("console", CHARDEV_BACKEND_KIND_CONSOLE, NULL,
+                         qemu_chr_open_win_con);
+#endif
     register_char_driver("pipe", CHARDEV_BACKEND_KIND_PIPE,
-                         qemu_chr_parse_pipe);
-    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux);
+                         qemu_chr_parse_pipe, qemu_chr_open_pipe);
+    register_char_driver("mux", CHARDEV_BACKEND_KIND_MUX, qemu_chr_parse_mux,
+                         qemu_chr_open_mux);
     /* Bug-compatibility: */
     register_char_driver("memory", CHARDEV_BACKEND_KIND_MEMORY,
-                         qemu_chr_parse_ringbuf);
+                         qemu_chr_parse_ringbuf, qemu_chr_open_ringbuf);
     /* this must be done after machine init, since we register FEs with muxes
      * as part of realize functions like serial_isa_realizefn when -nographic
      * is specified
diff --git a/qemu-coroutine-io.c b/qemu-coroutine-io.c
deleted file mode 100644
index d404926..0000000
--- a/qemu-coroutine-io.c
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Coroutine-aware I/O functions
- *
- * Copyright (C) 2009-2010 Nippon Telegraph and Telephone Corporation.
- * Copyright (c) 2011, Red Hat, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "qemu/sockets.h"
-#include "block/coroutine.h"
-#include "qemu/iov.h"
-#include "qemu/main-loop.h"
-
-ssize_t coroutine_fn
-qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
-                    size_t offset, size_t bytes, bool do_send)
-{
-    size_t done = 0;
-    ssize_t ret;
-    int err;
-    while (done < bytes) {
-        ret = iov_send_recv(sockfd, iov, iov_cnt,
-                            offset + done, bytes - done, do_send);
-        if (ret > 0) {
-            done += ret;
-        } else if (ret < 0) {
-            err = socket_error();
-            if (err == EAGAIN || err == EWOULDBLOCK) {
-                qemu_coroutine_yield();
-            } else if (done == 0) {
-                return -1;
-            } else {
-                break;
-            }
-        } else if (ret == 0 && !do_send) {
-            /* write (send) should never return 0.
-             * read (recv) returns 0 for end-of-file (-data).
-             * In both cases there's little point retrying,
-             * but we do for write anyway, just in case */
-            break;
-        }
-    }
-    return done;
-}
-
-ssize_t coroutine_fn
-qemu_co_send_recv(int sockfd, void *buf, size_t bytes, bool do_send)
-{
-    struct iovec iov = { .iov_base = buf, .iov_len = bytes };
-    return qemu_co_sendv_recvv(sockfd, &iov, 1, 0, bytes, do_send);
-}
-
-typedef struct {
-    Coroutine *co;
-    int fd;
-} FDYieldUntilData;
-
-static void fd_coroutine_enter(void *opaque)
-{
-    FDYieldUntilData *data = opaque;
-    qemu_set_fd_handler(data->fd, NULL, NULL, NULL);
-    qemu_coroutine_enter(data->co, NULL);
-}
-
-void coroutine_fn yield_until_fd_readable(int fd)
-{
-    FDYieldUntilData data;
-
-    assert(qemu_in_coroutine());
-    data.co = qemu_coroutine_self();
-    data.fd = fd;
-    qemu_set_fd_handler(fd, fd_coroutine_enter, NULL, &data);
-    qemu_coroutine_yield();
-}
diff --git a/qemu-coroutine-lock.c b/qemu-coroutine-lock.c
deleted file mode 100644
index e4860ae..0000000
--- a/qemu-coroutine-lock.c
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * coroutine queues and locks
- *
- * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-common.h"
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
-#include "qemu/queue.h"
-#include "trace.h"
-
-void qemu_co_queue_init(CoQueue *queue)
-{
-    QTAILQ_INIT(&queue->entries);
-}
-
-void coroutine_fn qemu_co_queue_wait(CoQueue *queue)
-{
-    Coroutine *self = qemu_coroutine_self();
-    QTAILQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
-    qemu_coroutine_yield();
-    assert(qemu_in_coroutine());
-}
-
-/**
- * qemu_co_queue_run_restart:
- *
- * Enter each coroutine that was previously marked for restart by
- * qemu_co_queue_next() or qemu_co_queue_restart_all().  This function is
- * invoked by the core coroutine code when the current coroutine yields or
- * terminates.
- */
-void qemu_co_queue_run_restart(Coroutine *co)
-{
-    Coroutine *next;
-
-    trace_qemu_co_queue_run_restart(co);
-    while ((next = QTAILQ_FIRST(&co->co_queue_wakeup))) {
-        QTAILQ_REMOVE(&co->co_queue_wakeup, next, co_queue_next);
-        qemu_coroutine_enter(next, NULL);
-    }
-}
-
-static bool qemu_co_queue_do_restart(CoQueue *queue, bool single)
-{
-    Coroutine *self = qemu_coroutine_self();
-    Coroutine *next;
-
-    if (QTAILQ_EMPTY(&queue->entries)) {
-        return false;
-    }
-
-    while ((next = QTAILQ_FIRST(&queue->entries)) != NULL) {
-        QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
-        QTAILQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);
-        trace_qemu_co_queue_next(next);
-        if (single) {
-            break;
-        }
-    }
-    return true;
-}
-
-bool coroutine_fn qemu_co_queue_next(CoQueue *queue)
-{
-    assert(qemu_in_coroutine());
-    return qemu_co_queue_do_restart(queue, true);
-}
-
-void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue)
-{
-    assert(qemu_in_coroutine());
-    qemu_co_queue_do_restart(queue, false);
-}
-
-bool qemu_co_enter_next(CoQueue *queue)
-{
-    Coroutine *next;
-
-    next = QTAILQ_FIRST(&queue->entries);
-    if (!next) {
-        return false;
-    }
-
-    QTAILQ_REMOVE(&queue->entries, next, co_queue_next);
-    qemu_coroutine_enter(next, NULL);
-    return true;
-}
-
-bool qemu_co_queue_empty(CoQueue *queue)
-{
-    return (QTAILQ_FIRST(&queue->entries) == NULL);
-}
-
-void qemu_co_mutex_init(CoMutex *mutex)
-{
-    memset(mutex, 0, sizeof(*mutex));
-    qemu_co_queue_init(&mutex->queue);
-}
-
-void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)
-{
-    Coroutine *self = qemu_coroutine_self();
-
-    trace_qemu_co_mutex_lock_entry(mutex, self);
-
-    while (mutex->locked) {
-        qemu_co_queue_wait(&mutex->queue);
-    }
-
-    mutex->locked = true;
-
-    trace_qemu_co_mutex_lock_return(mutex, self);
-}
-
-void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
-{
-    Coroutine *self = qemu_coroutine_self();
-
-    trace_qemu_co_mutex_unlock_entry(mutex, self);
-
-    assert(mutex->locked == true);
-    assert(qemu_in_coroutine());
-
-    mutex->locked = false;
-    qemu_co_queue_next(&mutex->queue);
-
-    trace_qemu_co_mutex_unlock_return(mutex, self);
-}
-
-void qemu_co_rwlock_init(CoRwlock *lock)
-{
-    memset(lock, 0, sizeof(*lock));
-    qemu_co_queue_init(&lock->queue);
-}
-
-void qemu_co_rwlock_rdlock(CoRwlock *lock)
-{
-    while (lock->writer) {
-        qemu_co_queue_wait(&lock->queue);
-    }
-    lock->reader++;
-}
-
-void qemu_co_rwlock_unlock(CoRwlock *lock)
-{
-    assert(qemu_in_coroutine());
-    if (lock->writer) {
-        lock->writer = false;
-        qemu_co_queue_restart_all(&lock->queue);
-    } else {
-        lock->reader--;
-        assert(lock->reader >= 0);
-        /* Wakeup only one waiting writer */
-        if (!lock->reader) {
-            qemu_co_queue_next(&lock->queue);
-        }
-    }
-}
-
-void qemu_co_rwlock_wrlock(CoRwlock *lock)
-{
-    while (lock->writer || lock->reader) {
-        qemu_co_queue_wait(&lock->queue);
-    }
-    lock->writer = true;
-}
diff --git a/qemu-coroutine-sleep.c b/qemu-coroutine-sleep.c
deleted file mode 100644
index 9abb7fd..0000000
--- a/qemu-coroutine-sleep.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * QEMU coroutine sleep
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "block/coroutine.h"
-#include "qemu/timer.h"
-#include "block/aio.h"
-
-typedef struct CoSleepCB {
-    QEMUTimer *ts;
-    Coroutine *co;
-} CoSleepCB;
-
-static void co_sleep_cb(void *opaque)
-{
-    CoSleepCB *sleep_cb = opaque;
-
-    qemu_coroutine_enter(sleep_cb->co, NULL);
-}
-
-void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
-                                  int64_t ns)
-{
-    CoSleepCB sleep_cb = {
-        .co = qemu_coroutine_self(),
-    };
-    sleep_cb.ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, &sleep_cb);
-    timer_mod(sleep_cb.ts, qemu_clock_get_ns(type) + ns);
-    qemu_coroutine_yield();
-    timer_del(sleep_cb.ts);
-    timer_free(sleep_cb.ts);
-}
diff --git a/qemu-coroutine.c b/qemu-coroutine.c
deleted file mode 100644
index bd574aa..0000000
--- a/qemu-coroutine.c
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * QEMU coroutines
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
- *  Kevin Wolf         <kwolf@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#include "trace.h"
-#include "qemu-common.h"
-#include "qemu/thread.h"
-#include "block/coroutine.h"
-#include "block/coroutine_int.h"
-
-enum {
-    POOL_DEFAULT_SIZE = 64,
-};
-
-/** Free list to speed up creation */
-static QemuMutex pool_lock;
-static QSLIST_HEAD(, Coroutine) pool = QSLIST_HEAD_INITIALIZER(pool);
-static unsigned int pool_size;
-static unsigned int pool_max_size = POOL_DEFAULT_SIZE;
-
-Coroutine *qemu_coroutine_create(CoroutineEntry *entry)
-{
-    Coroutine *co = NULL;
-
-    if (CONFIG_COROUTINE_POOL) {
-        qemu_mutex_lock(&pool_lock);
-        co = QSLIST_FIRST(&pool);
-        if (co) {
-            QSLIST_REMOVE_HEAD(&pool, pool_next);
-            pool_size--;
-        }
-        qemu_mutex_unlock(&pool_lock);
-    }
-
-    if (!co) {
-        co = qemu_coroutine_new();
-    }
-
-    co->entry = entry;
-    QTAILQ_INIT(&co->co_queue_wakeup);
-    return co;
-}
-
-static void coroutine_delete(Coroutine *co)
-{
-    if (CONFIG_COROUTINE_POOL) {
-        qemu_mutex_lock(&pool_lock);
-        if (pool_size < pool_max_size) {
-            QSLIST_INSERT_HEAD(&pool, co, pool_next);
-            co->caller = NULL;
-            pool_size++;
-            qemu_mutex_unlock(&pool_lock);
-            return;
-        }
-        qemu_mutex_unlock(&pool_lock);
-    }
-
-    qemu_coroutine_delete(co);
-}
-
-static void __attribute__((constructor)) coroutine_pool_init(void)
-{
-    qemu_mutex_init(&pool_lock);
-}
-
-static void __attribute__((destructor)) coroutine_pool_cleanup(void)
-{
-    Coroutine *co;
-    Coroutine *tmp;
-
-    QSLIST_FOREACH_SAFE(co, &pool, pool_next, tmp) {
-        QSLIST_REMOVE_HEAD(&pool, pool_next);
-        qemu_coroutine_delete(co);
-    }
-
-    qemu_mutex_destroy(&pool_lock);
-}
-
-static void coroutine_swap(Coroutine *from, Coroutine *to)
-{
-    CoroutineAction ret;
-
-    ret = qemu_coroutine_switch(from, to, COROUTINE_YIELD);
-
-    qemu_co_queue_run_restart(to);
-
-    switch (ret) {
-    case COROUTINE_YIELD:
-        return;
-    case COROUTINE_TERMINATE:
-        trace_qemu_coroutine_terminate(to);
-        coroutine_delete(to);
-        return;
-    default:
-        abort();
-    }
-}
-
-void qemu_coroutine_enter(Coroutine *co, void *opaque)
-{
-    Coroutine *self = qemu_coroutine_self();
-
-    trace_qemu_coroutine_enter(self, co, opaque);
-
-    if (co->caller) {
-        fprintf(stderr, "Co-routine re-entered recursively\n");
-        abort();
-    }
-
-    co->caller = self;
-    co->entry_arg = opaque;
-    coroutine_swap(self, co);
-}
-
-void coroutine_fn qemu_coroutine_yield(void)
-{
-    Coroutine *self = qemu_coroutine_self();
-    Coroutine *to = self->caller;
-
-    trace_qemu_coroutine_yield(self, to);
-
-    if (!to) {
-        fprintf(stderr, "Co-routine is yielding to no one\n");
-        abort();
-    }
-
-    self->caller = NULL;
-    coroutine_swap(self, to);
-}
-
-void qemu_coroutine_adjust_pool_size(int n)
-{
-    qemu_mutex_lock(&pool_lock);
-
-    pool_max_size += n;
-
-    /* Callers should never take away more than they added */
-    assert(pool_max_size >= POOL_DEFAULT_SIZE);
-
-    /* Trim oversized pool down to new max */
-    while (pool_size > pool_max_size) {
-        Coroutine *co = QSLIST_FIRST(&pool);
-        QSLIST_REMOVE_HEAD(&pool, pool_next);
-        pool_size--;
-        qemu_coroutine_delete(co);
-    }
-
-    qemu_mutex_unlock(&pool_lock);
-}
diff --git a/qemu-doc.texi b/qemu-doc.texi
index ad418f8..f37fd31 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -158,7 +158,8 @@
 * pcsys_introduction:: Introduction
 * pcsys_quickstart::   Quick Start
 * sec_invocation::     Invocation
-* pcsys_keys::         Keys
+* pcsys_keys::         Keys in the graphical frontends
+* mux_keys::           Keys in the character backend multiplexer
 * pcsys_monitor::      QEMU Monitor
 * disk_images::        Disk Images
 * pcsys_network::      Network emulation
@@ -195,6 +196,8 @@
 @item
 Serial ports
 @item
+IPMI BMC, either and internal or external one
+@item
 Creative SoundBlaster 16 sound card
 @item
 ENSONIQ AudioPCI ES1370 sound card
@@ -257,7 +260,7 @@
 
 @example
 @c man begin SYNOPSIS
-usage: qemu-system-i386 [options] [@var{disk_image}]
+@command{qemu-system-i386} [@var{options}] [@var{disk_image}]
 @c man end
 @end example
 
@@ -270,7 +273,7 @@
 @c man end
 
 @node pcsys_keys
-@section Keys
+@section Keys in the graphical frontends
 
 @c man begin OPTIONS
 
@@ -320,15 +323,23 @@
 In the virtual consoles, you can use @key{Ctrl-Up}, @key{Ctrl-Down},
 @key{Ctrl-PageUp} and @key{Ctrl-PageDown} to move in the back log.
 
-@kindex Ctrl-a h
-During emulation, if you are using the @option{-nographic} option, use
-@key{Ctrl-a h} to get terminal commands:
+@c man end
+
+@node mux_keys
+@section Keys in the character backend multiplexer
+
+@c man begin OPTIONS
+
+During emulation, if you are using a character backend multiplexer
+(which is the default if you are using @option{-nographic}) then
+several commands are available via an escape sequence. These
+key sequences all start with an escape character, which is @key{Ctrl-a}
+by default, but can be changed with @option{-echr}. The list below assumes
+you're using the default.
 
 @table @key
 @item Ctrl-a h
 @kindex Ctrl-a h
-@item Ctrl-a ?
-@kindex Ctrl-a ?
 Print this help
 @item Ctrl-a x
 @kindex Ctrl-a x
@@ -344,10 +355,11 @@
 Send break (magic sysrq in Linux)
 @item Ctrl-a c
 @kindex Ctrl-a c
-Switch between console and monitor
+Rotate between the frontends connected to the multiplexer (usually
+this switches between the monitor and the console)
 @item Ctrl-a Ctrl-a
-@kindex Ctrl-a a
-Send Ctrl-a
+@kindex Ctrl-a Ctrl-a
+Send the escape character to the frontend
 @end table
 @c man end
 
@@ -391,6 +403,8 @@
 
 @include qemu-monitor.texi
 
+@include qemu-monitor-info.texi
+
 @subsection Integer expressions
 
 The monitor understands integers expressions for every integer
@@ -412,6 +426,7 @@
 * vm_snapshots::              VM snapshots
 * qemu_img_invocation::       qemu-img Invocation
 * qemu_nbd_invocation::       qemu-nbd Invocation
+* qemu_ga_invocation::        qemu-ga Invocation
 * disk_images_formats::       Disk image file formats
 * host_drives::               Using host drives
 * disk_images_fat_images::    Virtual FAT disk images
@@ -505,6 +520,11 @@
 
 @include qemu-nbd.texi
 
+@node qemu_ga_invocation
+@subsection @code{qemu-ga} Invocation
+
+@include qemu-ga.texi
+
 @node disk_images_formats
 @subsection Disk image file formats
 
@@ -539,8 +559,8 @@
 @item qcow2
 QEMU image format, the most versatile format. Use it to have smaller
 images (useful if your filesystem does not supports holes, for example
-on Windows), optional AES encryption, zlib based compression and
-support of multiple VM snapshots.
+on Windows), zlib based compression and support of multiple VM
+snapshots.
 
 Supported options:
 @table @code
@@ -574,9 +594,10 @@
 though even this is ineffective with many modern storage technologies.
 @end itemize
 
-Use of qcow / qcow2 encryption is thus strongly discouraged. Users are
-recommended to use an alternative encryption technology such as the
-Linux dm-crypt / LUKS system.
+Use of qcow / qcow2 encryption with QEMU is deprecated, and support for
+it will go away in a future release.  Users are recommended to use an
+alternative encryption technology such as the Linux dm-crypt / LUKS
+system.
 
 @item cluster_size
 Changes the qcow2 cluster size (must be between 512 and 2M). Smaller cluster
@@ -672,6 +693,9 @@
 File name of a base image (see @option{create} subcommand).
 @item compat6
 Create a VMDK version 6 image (instead of version 4)
+@item hwversion
+Specify vmdk virtual hardware version. Compat6 flag cannot be enabled
+if hwversion is specified.
 @item subformat
 Specifies which VMDK subformat to use. Valid options are
 @code{monolithicSparse} (default),
@@ -698,7 +722,11 @@
 Specifies which VHDX subformat to use. Valid options are
 @code{dynamic} (default) and @code{fixed}.
 @item block_state_zero
-Force use of payload blocks of type 'ZERO'.
+Force use of payload blocks of type 'ZERO'.  Can be set to @code{on} (default)
+or @code{off}.  When set to @code{off}, new blocks will be created as
+@code{PAYLOAD_BLOCK_NOT_PRESENT}, which means parsers are free to return
+arbitrary data for those blocks.  Do not set to @code{off} when using
+@code{qemu-img convert} with @code{subformat=dynamic}.
 @item block_size
 Block size; min 1 MB, max 256 MB.  0 means auto-calculate based on image size.
 @item log_size
@@ -731,8 +759,7 @@
 
 On Linux, you can directly use the host device filename instead of a
 disk image filename provided you have enough privileges to access
-it. For example, use @file{/dev/cdrom} to access to the CDROM or
-@file{/dev/fd0} for the floppy.
+it. For example, use @file{/dev/cdrom} to access to the CDROM.
 
 @table @code
 @item CD
@@ -744,6 +771,8 @@
 removal is currently not detected accurately (if you change floppy
 without doing floppy access while the floppy is not loaded, the guest
 OS will think that the same floppy is loaded).
+Use of the host's floppy device is deprecated, and support for it will
+be removed in a future release.
 @item Hard disks
 Hard disks can be used. Normally you must specify the whole disk
 (@file{/dev/hdb} instead of @file{/dev/hdb1}) so that the guest OS can
@@ -1055,7 +1084,7 @@
 @var{server} specifies the server where the volume file specification for
 the given volume resides. This can be either hostname, ipv4 address
 or ipv6 address. ipv6 address needs to be within square brackets [ ].
-If transport type is unix, then @var{server} field should not be specifed.
+If transport type is unix, then @var{server} field should not be specified.
 Instead @var{socket} field needs to be populated with the path to unix domain
 socket.
 
@@ -1221,9 +1250,9 @@
 When using the built-in TFTP server, the router is also the TFTP
 server.
 
-When using the @option{-redir} option, TCP or UDP connections can be
-redirected from the host to the guest. It allows for example to
-redirect X11, telnet or SSH connections.
+When using the @option{'-netdev user,hostfwd=...'} option, TCP or UDP
+connections can be redirected from the host to the guest. It allows for
+example to redirect X11, telnet or SSH connections.
 
 @subsection Connecting VLANs between QEMU instances
 
@@ -1236,13 +1265,18 @@
 
 @subsection Inter-VM Shared Memory device
 
-With KVM enabled on a Linux host, a shared memory device is available.  Guests
-map a POSIX shared memory region into the guest as a PCI device that enables
-zero-copy communication to the application level of the guests.  The basic
-syntax is:
+On Linux hosts, a shared memory device is available.  The basic syntax
+is:
 
 @example
-qemu-system-i386 -device ivshmem,size=<size in format accepted by -m>[,shm=<shm name>]
+qemu-system-x86_64 -device ivshmem-plain,memdev=@var{hostmem}
+@end example
+
+where @var{hostmem} names a host memory backend.  For a POSIX shared
+memory backend, use something like
+
+@example
+-object memory-backend-file,size=1M,share,mem-path=/dev/shm/ivshmem,id=@var{hostmem}
 @end example
 
 If desired, interrupts can be sent between guest VMs accessing the same shared
@@ -1252,28 +1286,41 @@
 memory server is:
 
 @example
-qemu-system-i386 -device ivshmem,size=<size in format accepted by -m>[,chardev=<id>]
-                 [,msi=on][,ioeventfd=on][,vectors=n][,role=peer|master]
-qemu-system-i386 -chardev socket,path=<path>,id=<id>
+# First start the ivshmem server once and for all
+ivshmem-server -p @var{pidfile} -S @var{path} -m @var{shm-name} -l @var{shm-size} -n @var{vectors}
+
+# Then start your qemu instances with matching arguments
+qemu-system-x86_64 -device ivshmem-doorbell,vectors=@var{vectors},chardev=@var{id}
+                 -chardev socket,path=@var{path},id=@var{id}
 @end example
 
 When using the server, the guest will be assigned a VM ID (>=0) that allows guests
 using the same server to communicate via interrupts.  Guests can read their
-VM ID from a device register (see example code).  Since receiving the shared
-memory region from the server is asynchronous, there is a (small) chance the
-guest may boot before the shared memory is attached.  To allow an application
-to ensure shared memory is attached, the VM ID register will return -1 (an
-invalid VM ID) until the memory is attached.  Once the shared memory is
-attached, the VM ID will return the guest's valid VM ID.  With these semantics,
-the guest application can check to ensure the shared memory is attached to the
-guest before proceeding.
+VM ID from a device register (see ivshmem-spec.txt).
 
-The @option{role} argument can be set to either master or peer and will affect
-how the shared memory is migrated.  With @option{role=master}, the guest will
-copy the shared memory on migration to the destination host.  With
-@option{role=peer}, the guest will not be able to migrate with the device attached.
-With the @option{peer} case, the device should be detached and then reattached
-after migration using the PCI hotplug support.
+@subsubsection Migration with ivshmem
+
+With device property @option{master=on}, the guest will copy the shared
+memory on migration to the destination host.  With @option{master=off},
+the guest will not be able to migrate with the device attached.  In the
+latter case, the device should be detached and then reattached after
+migration using the PCI hotplug support.
+
+At most one of the devices sharing the same memory can be master.  The
+master must complete migration before you plug back the other devices.
+
+@subsubsection ivshmem and hugepages
+
+Instead of specifying the <shm size> using POSIX shm, you may specify
+a memory backend that has hugepage support:
+
+@example
+qemu-system-x86_64 -object memory-backend-file,size=1G,mem-path=/dev/hugepages/my-shmem-file,share,id=mb1
+                 -device ivshmem-plain,memdev=mb1
+@end example
+
+ivshmem-server also supports hugepages mount points with the
+@option{-m} memory path argument.
 
 @node direct_linux_boot
 @section Direct Linux Boot
@@ -1373,7 +1420,7 @@
 This USB device implements the USB Transport Layer of HCI.  Example
 usage:
 @example
-qemu-system-i386 [...OPTIONS...] -usbdevice bt:hci,vlan=3 -bt device:keyboard,vlan=3
+@command{qemu-system-i386} [...@var{OPTIONS}...] @option{-usbdevice} bt:hci,vlan=3 @option{-bt} device:keyboard,vlan=3
 @end example
 @end table
 
@@ -1760,7 +1807,7 @@
 
 Advanced debugging options:
 
-The default single stepping behavior is step with the IRQs and timer service routines off.  It is set this way because when gdb executes a single step it expects to advance beyond the current instruction.  With the IRQs and and timer service routines on, a single step might jump into the one of the interrupt or exception vectors instead of executing the current instruction. This means you may hit the same breakpoint a number of times before executing the instruction gdb wants to have executed.  Because there are rare circumstances where you want to single step into an interrupt vector the behavior can be controlled from GDB.  There are three commands you can query and set the single step behavior:
+The default single stepping behavior is step with the IRQs and timer service routines off.  It is set this way because when gdb executes a single step it expects to advance beyond the current instruction.  With the IRQs and timer service routines on, a single step might jump into the one of the interrupt or exception vectors instead of executing the current instruction. This means you may hit the same breakpoint a number of times before executing the instruction gdb wants to have executed.  Because there are rare circumstances where you want to single step into an interrupt vector the behavior can be controlled from GDB.  There are three commands you can query and set the single step behavior:
 @table @code
 @item maintenance packet qqemu.sstepbits
 
@@ -1856,7 +1903,8 @@
 
 @subsubsection Share a directory between Unix and Windows
 
-See @ref{sec_invocation} about the help of the option @option{-smb}.
+See @ref{sec_invocation} about the help of the option
+@option{'-netdev user,smb=...'}.
 
 @subsubsection Windows XP security problem
 
@@ -2048,7 +2096,7 @@
 
 A sample Linux 2.6 series kernel and ram disk image are available on
 the QEMU web site. There are still issues with NetBSD and OpenBSD, but
-some kernel versions work. Please note that currently older Solaris kernels
+most kernel versions work. Please note that currently older Solaris kernels
 don't work probably due to interface issues between OpenBIOS and
 Solaris.
 
@@ -2087,8 +2135,9 @@
 
 Use the executable @file{qemu-system-sparc64} to simulate a Sun4u
 (UltraSPARC PC-like machine), Sun4v (T1 PC-like machine), or generic
-Niagara (T1) machine. The emulator is not usable for anything yet, but
-it can launch some kernels.
+Niagara (T1) machine. The Sun4u emulator is mostly complete, being
+able to run Linux, NetBSD and OpenBSD in headless (-nographic) mode. The
+Sun4v and Niagara emulators are still a work in progress.
 
 QEMU emulates the following peripherals:
 
@@ -2721,7 +2770,7 @@
 @subsection Command line options
 
 @example
-usage: qemu-i386 [-h] [-d] [-L path] [-s size] [-cpu model] [-g port] [-B offset] [-R size] program [arguments...]
+@command{qemu-i386} [@option{-h]} [@option{-d]} [@option{-L} @var{path}] [@option{-s} @var{size}] [@option{-cpu} @var{model}] [@option{-g} @var{port}] [@option{-B} @var{offset}] [@option{-R} @var{size}] @var{program} [@var{arguments}...]
 @end example
 
 @table @option
@@ -2863,7 +2912,7 @@
 @subsection Command line options
 
 @example
-usage: qemu-sparc64 [-h] [-d] [-L path] [-s size] [-bsd type] program [arguments...]
+@command{qemu-sparc64} [@option{-h]} [@option{-d]} [@option{-L} @var{path}] [@option{-s} @var{size}] [@option{-bsd} @var{type}] @var{program} [@var{arguments}...]
 @end example
 
 @table @option
@@ -3014,9 +3063,62 @@
 @node Mac OS X
 @section Mac OS X
 
-The Mac OS X patches are not fully merged in QEMU, so you should look
-at the QEMU mailing list archive to have all the necessary
-information.
+System Requirements:
+@itemize
+@item Mac OS 10.5 or higher
+@item The clang compiler shipped with Xcode 4.2 or higher,
+or GCC 4.3 or higher
+@end itemize
+
+Additional Requirements (install in order):
+@enumerate
+@item libffi: @uref{https://sourceware.org/libffi/}
+@item gettext: @uref{http://www.gnu.org/software/gettext/}
+@item glib: @uref{http://ftp.gnome.org/pub/GNOME/sources/glib/}
+@item pkg-config: @uref{http://www.freedesktop.org/wiki/Software/pkg-config/}
+@item autoconf: @uref{http://www.gnu.org/software/autoconf/autoconf.html}
+@item automake: @uref{http://www.gnu.org/software/automake/}
+@item pixman: @uref{http://www.pixman.org/}
+@end enumerate
+
+* You may find it easiest to get these from a third-party packager
+such as Homebrew, Macports, or Fink.
+
+After downloading the QEMU source code, double-click it to expand it.
+
+Then configure and make QEMU:
+@example
+./configure
+make
+@end example
+
+If you have a recent version of Mac OS X (OSX 10.7 or better
+with Xcode 4.2 or better) we recommend building QEMU with the
+default compiler provided by Apple, for your version of Mac OS X
+(which will be 'clang'). The configure script will
+automatically pick this.
+
+Note: If after the configure step you see a message like this:
+@example
+ERROR: Your compiler does not support the __thread specifier for
+       Thread-Local Storage (TLS). Please upgrade to a version that does.
+@end example
+you may have to build your own version of gcc from source. Expect that to take
+several hours. More information can be found here:
+@uref{https://gcc.gnu.org/install/} @*
+
+These are some of the third party binaries of gcc available for download:
+@itemize
+@item Homebrew: @uref{http://brew.sh/}
+@item @uref{https://www.litebeam.net/gcc/gcc_472.pkg}
+@item @uref{http://www.macports.org/ports.php?by=name&substr=gcc}
+@end itemize
+
+You can have several versions of GCC on your system. To specify a certain version,
+use the --cc and --cxx options.
+@example
+./configure --cxx=<path of your c++ compiler> --cc=<path of your c compiler> <other options>
+@end example
 
 @node Make targets
 @section Make targets
diff --git a/qemu-file-stdio.c b/qemu-file-stdio.c
deleted file mode 100644
index 73fec60..0000000
--- a/qemu-file-stdio.c
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "block/coroutine.h"
-#include "migration/qemu-file.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/utils/file_io.h"
-#endif  // CONFIG_ANDROID
-
-typedef struct QEMUFileStdio {
-    FILE *stdio_file;
-    QEMUFile *file;
-} QEMUFileStdio;
-
-static int stdio_get_fd(void *opaque)
-{
-    QEMUFileStdio *s = opaque;
-
-    return fileno(s->stdio_file);
-}
-
-static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos,
-                            int size)
-{
-    QEMUFileStdio *s = opaque;
-    int res;
-
-    res = fwrite(buf, 1, size, s->stdio_file);
-
-    if (res != size) {
-        return -errno;
-    }
-    return res;
-}
-
-static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileStdio *s = opaque;
-    FILE *fp = s->stdio_file;
-    int bytes;
-
-    for (;;) {
-        clearerr(fp);
-        bytes = fread(buf, 1, size, fp);
-        if (bytes != 0 || !ferror(fp)) {
-            break;
-        }
-        if (errno == EAGAIN) {
-            yield_until_fd_readable(fileno(fp));
-        } else if (errno != EINTR) {
-            break;
-        }
-    }
-    return bytes;
-}
-
-static int stdio_pclose(void *opaque)
-{
-    QEMUFileStdio *s = opaque;
-    int ret;
-    ret = pclose(s->stdio_file);
-    if (ret == -1) {
-        ret = -errno;
-    } else if (!WIFEXITED(ret) || WEXITSTATUS(ret) != 0) {
-        /* close succeeded, but non-zero exit code: */
-        ret = -EIO; /* fake errno value */
-    }
-    g_free(s);
-    return ret;
-}
-
-static int stdio_fclose(void *opaque)
-{
-    QEMUFileStdio *s = opaque;
-    int ret = 0;
-
-    if (qemu_file_is_writable(s->file)) {
-        int fd = fileno(s->stdio_file);
-        struct stat st;
-
-        ret = fstat(fd, &st);
-        if (ret == 0 && S_ISREG(st.st_mode)) {
-            /*
-             * If the file handle is a regular file make sure the
-             * data is flushed to disk before signaling success.
-             */
-            ret = fsync(fd);
-            if (ret != 0) {
-                ret = -errno;
-                return ret;
-            }
-        }
-    }
-    if (fclose(s->stdio_file) == EOF) {
-        ret = -errno;
-    }
-    g_free(s);
-    return ret;
-}
-
-static const QEMUFileOps stdio_pipe_read_ops = {
-    .get_fd =     stdio_get_fd,
-    .get_buffer = stdio_get_buffer,
-    .close =      stdio_pclose
-};
-
-static const QEMUFileOps stdio_pipe_write_ops = {
-    .get_fd =     stdio_get_fd,
-    .put_buffer = stdio_put_buffer,
-    .close =      stdio_pclose
-};
-
-QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
-{
-    FILE *stdio_file;
-    QEMUFileStdio *s;
-
-    if (mode == NULL || (mode[0] != 'r' && mode[0] != 'w') || mode[1] != 0) {
-        fprintf(stderr, "qemu_popen: Argument validity check failed\n");
-        return NULL;
-    }
-
-#ifdef CONFIG_ANDROID
-    stdio_file = android_popen(command, mode);
-#else  // !CONFIG_ANDROID
-    stdio_file = popen(command, mode);
-#endif  // CONFIG_ANDROID
-    if (stdio_file == NULL) {
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileStdio));
-
-    s->stdio_file = stdio_file;
-
-    if (mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, &stdio_pipe_read_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &stdio_pipe_write_ops);
-    }
-    return s->file;
-}
-
-static const QEMUFileOps stdio_file_read_ops = {
-    .get_fd =     stdio_get_fd,
-    .get_buffer = stdio_get_buffer,
-    .close =      stdio_fclose
-};
-
-static const QEMUFileOps stdio_file_write_ops = {
-    .get_fd =     stdio_get_fd,
-    .put_buffer = stdio_put_buffer,
-    .close =      stdio_fclose
-};
-
-QEMUFile *qemu_fopen(const char *filename, const char *mode)
-{
-    QEMUFileStdio *s;
-
-    if (qemu_file_mode_is_not_valid(mode)) {
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileStdio));
-
-    s->stdio_file = fopen(filename, mode);
-    if (!s->stdio_file) {
-        goto fail;
-    }
-
-    if (mode[0] == 'w') {
-        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
-    }
-    return s->file;
-fail:
-    g_free(s);
-    return NULL;
-}
diff --git a/qemu-file-unix.c b/qemu-file-unix.c
deleted file mode 100644
index 9682396..0000000
--- a/qemu-file-unix.c
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "qemu/iov.h"
-#include "qemu/sockets.h"
-#include "block/coroutine.h"
-#include "migration/qemu-file.h"
-
-typedef struct QEMUFileSocket {
-    int fd;
-    QEMUFile *file;
-} QEMUFileSocket;
-
-static ssize_t socket_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
-                                    int64_t pos)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len;
-    ssize_t size = iov_size(iov, iovcnt);
-
-    len = iov_send(s->fd, iov, iovcnt, 0, size);
-    if (len < size) {
-        len = -socket_error();
-    }
-    return len;
-}
-
-static int socket_get_fd(void *opaque)
-{
-    QEMUFileSocket *s = opaque;
-
-    return s->fd;
-}
-
-static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len;
-
-    for (;;) {
-        len = qemu_recv(s->fd, buf, size, 0);
-        if (len != -1) {
-            break;
-        }
-        if (socket_error() == EAGAIN) {
-            yield_until_fd_readable(s->fd);
-        } else if (socket_error() != EINTR) {
-            break;
-        }
-    }
-
-    if (len == -1) {
-        len = -socket_error();
-    }
-    return len;
-}
-
-static int socket_close(void *opaque)
-{
-    QEMUFileSocket *s = opaque;
-    closesocket(s->fd);
-    g_free(s);
-    return 0;
-}
-
-static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
-                                  int64_t pos)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len, offset;
-    ssize_t size = iov_size(iov, iovcnt);
-    ssize_t total = 0;
-
-    assert(iovcnt > 0);
-    offset = 0;
-    while (size > 0) {
-        /* Find the next start position; skip all full-sized vector elements  */
-        while (offset >= iov[0].iov_len) {
-            offset -= iov[0].iov_len;
-            iov++, iovcnt--;
-        }
-
-        /* skip `offset' bytes from the (now) first element, undo it on exit */
-        assert(iovcnt > 0);
-        iov[0].iov_base += offset;
-        iov[0].iov_len -= offset;
-
-        do {
-            len = writev(s->fd, iov, iovcnt);
-        } while (len == -1 && errno == EINTR);
-        if (len == -1) {
-            return -errno;
-        }
-
-        /* Undo the changes above */
-        iov[0].iov_base -= offset;
-        iov[0].iov_len += offset;
-
-        /* Prepare for the next iteration */
-        offset += len;
-        total += len;
-        size -= len;
-    }
-
-    return total;
-}
-
-static int unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileSocket *s = opaque;
-    ssize_t len;
-
-    for (;;) {
-        len = read(s->fd, buf, size);
-        if (len != -1) {
-            break;
-        }
-        if (errno == EAGAIN) {
-            yield_until_fd_readable(s->fd);
-        } else if (errno != EINTR) {
-            break;
-        }
-    }
-
-    if (len == -1) {
-        len = -errno;
-    }
-    return len;
-}
-
-static int unix_close(void *opaque)
-{
-    QEMUFileSocket *s = opaque;
-    close(s->fd);
-    g_free(s);
-    return 0;
-}
-
-static const QEMUFileOps unix_read_ops = {
-    .get_fd =     socket_get_fd,
-    .get_buffer = unix_get_buffer,
-    .close =      unix_close
-};
-
-static const QEMUFileOps unix_write_ops = {
-    .get_fd =     socket_get_fd,
-    .writev_buffer = unix_writev_buffer,
-    .close =      unix_close
-};
-
-QEMUFile *qemu_fdopen(int fd, const char *mode)
-{
-    QEMUFileSocket *s;
-
-    if (mode == NULL ||
-        (mode[0] != 'r' && mode[0] != 'w') ||
-        mode[1] != 'b' || mode[2] != 0) {
-        fprintf(stderr, "qemu_fdopen: Argument validity check failed\n");
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileSocket));
-    s->fd = fd;
-
-    if (mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, &unix_read_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &unix_write_ops);
-    }
-    return s->file;
-}
-
-static const QEMUFileOps socket_read_ops = {
-    .get_fd =     socket_get_fd,
-    .get_buffer = socket_get_buffer,
-    .close =      socket_close
-};
-
-static const QEMUFileOps socket_write_ops = {
-    .get_fd =     socket_get_fd,
-    .writev_buffer = socket_writev_buffer,
-    .close =      socket_close
-};
-
-QEMUFile *qemu_fopen_socket(int fd, const char *mode)
-{
-    QEMUFileSocket *s;
-
-    if (qemu_file_mode_is_not_valid(mode)) {
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUFileSocket));
-    s->fd = fd;
-    if (mode[0] == 'w') {
-        qemu_set_block(s->fd);
-        s->file = qemu_fopen_ops(s, &socket_write_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &socket_read_ops);
-    }
-    return s->file;
-}
diff --git a/qemu-file.c b/qemu-file.c
deleted file mode 100644
index f938e36..0000000
--- a/qemu-file.c
+++ /dev/null
@@ -1,995 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#include "qemu-common.h"
-#include "qemu/iov.h"
-#include "qemu/sockets.h"
-#include "block/coroutine.h"
-#include "migration/migration.h"
-#include "migration/qemu-file.h"
-#include "trace.h"
-
-#define IO_BUF_SIZE 32768
-#define MAX_IOV_SIZE MIN(IOV_MAX, 64)
-
-struct QEMUFile {
-    const QEMUFileOps *ops;
-    void *opaque;
-
-    int64_t bytes_xfer;
-    int64_t xfer_limit;
-
-    int64_t pos; /* start of buffer when writing, end of buffer
-                    when reading */
-    int buf_index;
-    int buf_size; /* 0 when writing */
-    uint8_t buf[IO_BUF_SIZE];
-
-    struct iovec iov[MAX_IOV_SIZE];
-    unsigned int iovcnt;
-
-    int last_error;
-};
-
-bool qemu_file_mode_is_not_valid(const char *mode)
-{
-    if (mode == NULL ||
-        (mode[0] != 'r' && mode[0] != 'w') ||
-        mode[1] != 'b' || mode[2] != 0) {
-        fprintf(stderr, "qemu_fopen: Argument validity check failed\n");
-        return true;
-    }
-
-    return false;
-}
-
-QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
-{
-    QEMUFile *f;
-
-    f = g_malloc0(sizeof(QEMUFile));
-
-    f->opaque = opaque;
-    f->ops = ops;
-    return f;
-}
-
-/*
- * Get last error for stream f
- *
- * Return negative error value if there has been an error on previous
- * operations, return 0 if no error happened.
- *
- */
-int qemu_file_get_error(QEMUFile *f)
-{
-    return f->last_error;
-}
-
-void qemu_file_set_error(QEMUFile *f, int ret)
-{
-    if (f->last_error == 0) {
-        f->last_error = ret;
-    }
-}
-
-bool qemu_file_is_writable(QEMUFile *f)
-{
-    return f->ops->writev_buffer || f->ops->put_buffer;
-}
-
-/**
- * Flushes QEMUFile buffer
- *
- * If there is writev_buffer QEMUFileOps it uses it otherwise uses
- * put_buffer ops.
- */
-void qemu_fflush(QEMUFile *f)
-{
-    ssize_t ret = 0;
-
-    if (!qemu_file_is_writable(f)) {
-        return;
-    }
-
-    if (f->ops->writev_buffer) {
-        if (f->iovcnt > 0) {
-            ret = f->ops->writev_buffer(f->opaque, f->iov, f->iovcnt, f->pos);
-        }
-    } else {
-        if (f->buf_index > 0) {
-            ret = f->ops->put_buffer(f->opaque, f->buf, f->pos, f->buf_index);
-        }
-    }
-    if (ret >= 0) {
-        f->pos += ret;
-    }
-    f->buf_index = 0;
-    f->iovcnt = 0;
-    if (ret < 0) {
-        qemu_file_set_error(f, ret);
-    }
-}
-
-void ram_control_before_iterate(QEMUFile *f, uint64_t flags)
-{
-    int ret = 0;
-
-    if (f->ops->before_ram_iterate) {
-        ret = f->ops->before_ram_iterate(f, f->opaque, flags);
-        if (ret < 0) {
-            qemu_file_set_error(f, ret);
-        }
-    }
-}
-
-void ram_control_after_iterate(QEMUFile *f, uint64_t flags)
-{
-    int ret = 0;
-
-    if (f->ops->after_ram_iterate) {
-        ret = f->ops->after_ram_iterate(f, f->opaque, flags);
-        if (ret < 0) {
-            qemu_file_set_error(f, ret);
-        }
-    }
-}
-
-void ram_control_load_hook(QEMUFile *f, uint64_t flags)
-{
-    int ret = -EINVAL;
-
-    if (f->ops->hook_ram_load) {
-        ret = f->ops->hook_ram_load(f, f->opaque, flags);
-        if (ret < 0) {
-            qemu_file_set_error(f, ret);
-        }
-    } else {
-        qemu_file_set_error(f, ret);
-    }
-}
-
-size_t ram_control_save_page(QEMUFile *f, ram_addr_t block_offset,
-                         ram_addr_t offset, size_t size, int *bytes_sent)
-{
-    if (f->ops->save_page) {
-        int ret = f->ops->save_page(f, f->opaque, block_offset,
-                                    offset, size, bytes_sent);
-
-        if (ret != RAM_SAVE_CONTROL_DELAYED) {
-            if (bytes_sent && *bytes_sent > 0) {
-                qemu_update_position(f, *bytes_sent);
-            } else if (ret < 0) {
-                qemu_file_set_error(f, ret);
-            }
-        }
-
-        return ret;
-    }
-
-    return RAM_SAVE_CONTROL_NOT_SUPP;
-}
-
-/*
- * Attempt to fill the buffer from the underlying file
- * Returns the number of bytes read, or negative value for an error.
- *
- * Note that it can return a partially full buffer even in a not error/not EOF
- * case if the underlying file descriptor gives a short read, and that can
- * happen even on a blocking fd.
- */
-static ssize_t qemu_fill_buffer(QEMUFile *f)
-{
-    int len;
-    int pending;
-
-    assert(!qemu_file_is_writable(f));
-
-    pending = f->buf_size - f->buf_index;
-    if (pending > 0) {
-        memmove(f->buf, f->buf + f->buf_index, pending);
-    }
-    f->buf_index = 0;
-    f->buf_size = pending;
-
-    len = f->ops->get_buffer(f->opaque, f->buf + pending, f->pos,
-                        IO_BUF_SIZE - pending);
-    if (len > 0) {
-        f->buf_size += len;
-        f->pos += len;
-    } else if (len == 0) {
-        qemu_file_set_error(f, -EIO);
-    } else if (len != -EAGAIN) {
-        qemu_file_set_error(f, len);
-    }
-
-    return len;
-}
-
-int qemu_get_fd(QEMUFile *f)
-{
-    if (f->ops->get_fd) {
-        return f->ops->get_fd(f->opaque);
-    }
-    return -1;
-}
-
-void qemu_update_position(QEMUFile *f, size_t size)
-{
-    f->pos += size;
-}
-
-/** Closes the file
- *
- * Returns negative error value if any error happened on previous operations or
- * while closing the file. Returns 0 or positive number on success.
- *
- * The meaning of return value on success depends on the specific backend
- * being used.
- */
-int qemu_fclose(QEMUFile *f)
-{
-    int ret;
-    qemu_fflush(f);
-    ret = qemu_file_get_error(f);
-
-    if (f->ops->close) {
-        int ret2 = f->ops->close(f->opaque);
-        if (ret >= 0) {
-            ret = ret2;
-        }
-    }
-    /* If any error was spotted before closing, we should report it
-     * instead of the close() return value.
-     */
-    if (f->last_error) {
-        ret = f->last_error;
-    }
-    g_free(f);
-    trace_qemu_file_fclose();
-    return ret;
-}
-
-static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size)
-{
-    /* check for adjacent buffer and coalesce them */
-    if (f->iovcnt > 0 && buf == f->iov[f->iovcnt - 1].iov_base +
-        f->iov[f->iovcnt - 1].iov_len) {
-        f->iov[f->iovcnt - 1].iov_len += size;
-    } else {
-        f->iov[f->iovcnt].iov_base = (uint8_t *)buf;
-        f->iov[f->iovcnt++].iov_len = size;
-    }
-
-    if (f->iovcnt >= MAX_IOV_SIZE) {
-        qemu_fflush(f);
-    }
-}
-
-void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size)
-{
-    if (!f->ops->writev_buffer) {
-        qemu_put_buffer(f, buf, size);
-        return;
-    }
-
-    if (f->last_error) {
-        return;
-    }
-
-    f->bytes_xfer += size;
-    add_to_iovec(f, buf, size);
-}
-
-void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
-{
-    int l;
-
-    if (f->last_error) {
-        return;
-    }
-
-    while (size > 0) {
-        l = IO_BUF_SIZE - f->buf_index;
-        if (l > size) {
-            l = size;
-        }
-        memcpy(f->buf + f->buf_index, buf, l);
-        f->bytes_xfer += l;
-        if (f->ops->writev_buffer) {
-            add_to_iovec(f, f->buf + f->buf_index, l);
-        }
-        f->buf_index += l;
-        if (f->buf_index == IO_BUF_SIZE) {
-            qemu_fflush(f);
-        }
-        if (qemu_file_get_error(f)) {
-            break;
-        }
-        buf += l;
-        size -= l;
-    }
-}
-
-void qemu_put_byte(QEMUFile *f, int v)
-{
-    if (f->last_error) {
-        return;
-    }
-
-    f->buf[f->buf_index] = v;
-    f->bytes_xfer++;
-    if (f->ops->writev_buffer) {
-        add_to_iovec(f, f->buf + f->buf_index, 1);
-    }
-    f->buf_index++;
-    if (f->buf_index == IO_BUF_SIZE) {
-        qemu_fflush(f);
-    }
-}
-
-void qemu_file_skip(QEMUFile *f, int size)
-{
-    if (f->buf_index + size <= f->buf_size) {
-        f->buf_index += size;
-    }
-}
-
-/*
- * Read 'size' bytes from file (at 'offset') into buf without moving the
- * pointer.
- *
- * It will return size bytes unless there was an error, in which case it will
- * return as many as it managed to read (assuming blocking fd's which
- * all current QEMUFile are)
- */
-int qemu_peek_buffer(QEMUFile *f, uint8_t *buf, int size, size_t offset)
-{
-    int pending;
-    int index;
-
-    assert(!qemu_file_is_writable(f));
-    assert(offset < IO_BUF_SIZE);
-    assert(size <= IO_BUF_SIZE - offset);
-
-    /* The 1st byte to read from */
-    index = f->buf_index + offset;
-    /* The number of available bytes starting at index */
-    pending = f->buf_size - index;
-
-    /*
-     * qemu_fill_buffer might return just a few bytes, even when there isn't
-     * an error, so loop collecting them until we get enough.
-     */
-    while (pending < size) {
-        int received = qemu_fill_buffer(f);
-
-        if (received <= 0) {
-            break;
-        }
-
-        index = f->buf_index + offset;
-        pending = f->buf_size - index;
-    }
-
-    if (pending <= 0) {
-        return 0;
-    }
-    if (size > pending) {
-        size = pending;
-    }
-
-    memcpy(buf, f->buf + index, size);
-    return size;
-}
-
-/*
- * Read 'size' bytes of data from the file into buf.
- * 'size' can be larger than the internal buffer.
- *
- * It will return size bytes unless there was an error, in which case it will
- * return as many as it managed to read (assuming blocking fd's which
- * all current QEMUFile are)
- */
-int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size)
-{
-    int pending = size;
-    int done = 0;
-
-    while (pending > 0) {
-        int res;
-
-        res = qemu_peek_buffer(f, buf, MIN(pending, IO_BUF_SIZE), 0);
-        if (res == 0) {
-            return done;
-        }
-        qemu_file_skip(f, res);
-        buf += res;
-        pending -= res;
-        done += res;
-    }
-    return done;
-}
-
-/*
- * Peeks a single byte from the buffer; this isn't guaranteed to work if
- * offset leaves a gap after the previous read/peeked data.
- */
-int qemu_peek_byte(QEMUFile *f, int offset)
-{
-    int index = f->buf_index + offset;
-
-    assert(!qemu_file_is_writable(f));
-    assert(offset < IO_BUF_SIZE);
-
-    if (index >= f->buf_size) {
-        qemu_fill_buffer(f);
-        index = f->buf_index + offset;
-        if (index >= f->buf_size) {
-            return 0;
-        }
-    }
-    return f->buf[index];
-}
-
-int qemu_get_byte(QEMUFile *f)
-{
-    int result;
-
-    result = qemu_peek_byte(f, 0);
-    qemu_file_skip(f, 1);
-    return result;
-}
-
-int64_t qemu_ftell(QEMUFile *f)
-{
-    qemu_fflush(f);
-    return f->pos;
-}
-
-int qemu_file_rate_limit(QEMUFile *f)
-{
-    if (qemu_file_get_error(f)) {
-        return 1;
-    }
-    if (f->xfer_limit > 0 && f->bytes_xfer > f->xfer_limit) {
-        return 1;
-    }
-    return 0;
-}
-
-int64_t qemu_file_get_rate_limit(QEMUFile *f)
-{
-    return f->xfer_limit;
-}
-
-void qemu_file_set_rate_limit(QEMUFile *f, int64_t limit)
-{
-    f->xfer_limit = limit;
-}
-
-void qemu_file_reset_rate_limit(QEMUFile *f)
-{
-    f->bytes_xfer = 0;
-}
-
-void qemu_put_be16(QEMUFile *f, unsigned int v)
-{
-    qemu_put_byte(f, v >> 8);
-    qemu_put_byte(f, v);
-}
-
-void qemu_put_be32(QEMUFile *f, unsigned int v)
-{
-    qemu_put_byte(f, v >> 24);
-    qemu_put_byte(f, v >> 16);
-    qemu_put_byte(f, v >> 8);
-    qemu_put_byte(f, v);
-}
-
-void qemu_put_be64(QEMUFile *f, uint64_t v)
-{
-    qemu_put_be32(f, v >> 32);
-    qemu_put_be32(f, v);
-}
-
-unsigned int qemu_get_be16(QEMUFile *f)
-{
-    unsigned int v;
-    v = qemu_get_byte(f) << 8;
-    v |= qemu_get_byte(f);
-    return v;
-}
-
-unsigned int qemu_get_be32(QEMUFile *f)
-{
-    unsigned int v;
-    v = qemu_get_byte(f) << 24;
-    v |= qemu_get_byte(f) << 16;
-    v |= qemu_get_byte(f) << 8;
-    v |= qemu_get_byte(f);
-    return v;
-}
-
-uint64_t qemu_get_be64(QEMUFile *f)
-{
-    uint64_t v;
-    v = (uint64_t)qemu_get_be32(f) << 32;
-    v |= qemu_get_be32(f);
-    return v;
-}
-
-#define QSB_CHUNK_SIZE      (1 << 10)
-#define QSB_MAX_CHUNK_SIZE  (16 * QSB_CHUNK_SIZE)
-
-/**
- * Create a QEMUSizedBuffer
- * This type of buffer uses scatter-gather lists internally and
- * can grow to any size. Any data array in the scatter-gather list
- * can hold different amount of bytes.
- *
- * @buffer: Optional buffer to copy into the QSB
- * @len: size of initial buffer; if @buffer is given, buffer must
- *       hold at least len bytes
- *
- * Returns a pointer to a QEMUSizedBuffer or NULL on allocation failure
- */
-QEMUSizedBuffer *qsb_create(const uint8_t *buffer, size_t len)
-{
-    QEMUSizedBuffer *qsb;
-    size_t alloc_len, num_chunks, i, to_copy;
-    size_t chunk_size = (len > QSB_MAX_CHUNK_SIZE)
-                        ? QSB_MAX_CHUNK_SIZE
-                        : QSB_CHUNK_SIZE;
-
-    num_chunks = DIV_ROUND_UP(len ? len : QSB_CHUNK_SIZE, chunk_size);
-    alloc_len = num_chunks * chunk_size;
-
-    qsb = g_try_new0(QEMUSizedBuffer, 1);
-    if (!qsb) {
-        return NULL;
-    }
-
-    qsb->iov = g_try_new0(struct iovec, num_chunks);
-    if (!qsb->iov) {
-        g_free(qsb);
-        return NULL;
-    }
-
-    qsb->n_iov = num_chunks;
-
-    for (i = 0; i < num_chunks; i++) {
-        qsb->iov[i].iov_base = g_try_malloc0(chunk_size);
-        if (!qsb->iov[i].iov_base) {
-            /* qsb_free is safe since g_free can cope with NULL */
-            qsb_free(qsb);
-            return NULL;
-        }
-
-        qsb->iov[i].iov_len = chunk_size;
-        if (buffer) {
-            to_copy = (len - qsb->used) > chunk_size
-                      ? chunk_size : (len - qsb->used);
-            memcpy(qsb->iov[i].iov_base, &buffer[qsb->used], to_copy);
-            qsb->used += to_copy;
-        }
-    }
-
-    qsb->size = alloc_len;
-
-    return qsb;
-}
-
-/**
- * Free the QEMUSizedBuffer
- *
- * @qsb: The QEMUSizedBuffer to free
- */
-void qsb_free(QEMUSizedBuffer *qsb)
-{
-    size_t i;
-
-    if (!qsb) {
-        return;
-    }
-
-    for (i = 0; i < qsb->n_iov; i++) {
-        g_free(qsb->iov[i].iov_base);
-    }
-    g_free(qsb->iov);
-    g_free(qsb);
-}
-
-/**
- * Get the number of used bytes in the QEMUSizedBuffer
- *
- * @qsb: A QEMUSizedBuffer
- *
- * Returns the number of bytes currently used in this buffer
- */
-size_t qsb_get_length(const QEMUSizedBuffer *qsb)
-{
-    return qsb->used;
-}
-
-/**
- * Set the length of the buffer; the primary usage of this
- * function is to truncate the number of used bytes in the buffer.
- * The size will not be extended beyond the current number of
- * allocated bytes in the QEMUSizedBuffer.
- *
- * @qsb: A QEMUSizedBuffer
- * @new_len: The new length of bytes in the buffer
- *
- * Returns the number of bytes the buffer was truncated or extended
- * to.
- */
-size_t qsb_set_length(QEMUSizedBuffer *qsb, size_t new_len)
-{
-    if (new_len <= qsb->size) {
-        qsb->used = new_len;
-    } else {
-        qsb->used = qsb->size;
-    }
-    return qsb->used;
-}
-
-/**
- * Get the iovec that holds the data for a given position @pos.
- *
- * @qsb: A QEMUSizedBuffer
- * @pos: The index of a byte in the buffer
- * @d_off: Pointer to an offset that this function will indicate
- *         at what position within the returned iovec the byte
- *         is to be found
- *
- * Returns the index of the iovec that holds the byte at the given
- * index @pos in the byte stream; a negative number if the iovec
- * for the given position @pos does not exist.
- */
-static ssize_t qsb_get_iovec(const QEMUSizedBuffer *qsb,
-                             off_t pos, off_t *d_off)
-{
-    ssize_t i;
-    off_t curr = 0;
-
-    if (pos > qsb->used) {
-        return -1;
-    }
-
-    for (i = 0; i < qsb->n_iov; i++) {
-        if (curr + qsb->iov[i].iov_len > pos) {
-            *d_off = pos - curr;
-            return i;
-        }
-        curr += qsb->iov[i].iov_len;
-    }
-    return -1;
-}
-
-/*
- * Convert the QEMUSizedBuffer into a flat buffer.
- *
- * Note: If at all possible, try to avoid this function since it
- *       may unnecessarily copy memory around.
- *
- * @qsb: pointer to QEMUSizedBuffer
- * @start: offset to start at
- * @count: number of bytes to copy
- * @buf: a pointer to a buffer to write into (at least @count bytes)
- *
- * Returns the number of bytes copied into the output buffer
- */
-ssize_t qsb_get_buffer(const QEMUSizedBuffer *qsb, off_t start,
-                       size_t count, uint8_t *buffer)
-{
-    const struct iovec *iov;
-    size_t to_copy, all_copy;
-    ssize_t index;
-    off_t s_off;
-    off_t d_off = 0;
-    char *s;
-
-    if (start > qsb->used) {
-        return 0;
-    }
-
-    all_copy = qsb->used - start;
-    if (all_copy > count) {
-        all_copy = count;
-    } else {
-        count = all_copy;
-    }
-
-    index = qsb_get_iovec(qsb, start, &s_off);
-    if (index < 0) {
-        return 0;
-    }
-
-    while (all_copy > 0) {
-        iov = &qsb->iov[index];
-
-        s = iov->iov_base;
-
-        to_copy = iov->iov_len - s_off;
-        if (to_copy > all_copy) {
-            to_copy = all_copy;
-        }
-        memcpy(&buffer[d_off], &s[s_off], to_copy);
-
-        d_off += to_copy;
-        all_copy -= to_copy;
-
-        s_off = 0;
-        index++;
-    }
-
-    return count;
-}
-
-/**
- * Grow the QEMUSizedBuffer to the given size and allocate
- * memory for it.
- *
- * @qsb: A QEMUSizedBuffer
- * @new_size: The new size of the buffer
- *
- * Return:
- *    a negative error code in case of memory allocation failure
- * or
- *    the new size of the buffer. The returned size may be greater or equal
- *    to @new_size.
- */
-static ssize_t qsb_grow(QEMUSizedBuffer *qsb, size_t new_size)
-{
-    size_t needed_chunks, i;
-
-    if (qsb->size < new_size) {
-        struct iovec *new_iov;
-        size_t size_diff = new_size - qsb->size;
-        size_t chunk_size = (size_diff > QSB_MAX_CHUNK_SIZE)
-                             ? QSB_MAX_CHUNK_SIZE : QSB_CHUNK_SIZE;
-
-        needed_chunks = DIV_ROUND_UP(size_diff, chunk_size);
-
-        new_iov = g_try_new(struct iovec, qsb->n_iov + needed_chunks);
-        if (new_iov == NULL) {
-            return -ENOMEM;
-        }
-
-        /* Allocate new chunks as needed into new_iov */
-        for (i = qsb->n_iov; i < qsb->n_iov + needed_chunks; i++) {
-            new_iov[i].iov_base = g_try_malloc0(chunk_size);
-            new_iov[i].iov_len = chunk_size;
-            if (!new_iov[i].iov_base) {
-                size_t j;
-
-                /* Free previously allocated new chunks */
-                for (j = qsb->n_iov; j < i; j++) {
-                    g_free(new_iov[j].iov_base);
-                }
-                g_free(new_iov);
-
-                return -ENOMEM;
-            }
-        }
-
-        /*
-         * Now we can't get any allocation errors, copy over to new iov
-         * and switch.
-         */
-        for (i = 0; i < qsb->n_iov; i++) {
-            new_iov[i] = qsb->iov[i];
-        }
-
-        qsb->n_iov += needed_chunks;
-        g_free(qsb->iov);
-        qsb->iov = new_iov;
-        qsb->size += (needed_chunks * chunk_size);
-    }
-
-    return qsb->size;
-}
-
-/**
- * Write into the QEMUSizedBuffer at a given position and a given
- * number of bytes. This function will automatically grow the
- * QEMUSizedBuffer.
- *
- * @qsb: A QEMUSizedBuffer
- * @source: A byte array to copy data from
- * @pos: The position within the @qsb to write data to
- * @size: The number of bytes to copy into the @qsb
- *
- * Returns @size or a negative error code in case of memory allocation failure,
- *           or with an invalid 'pos'
- */
-ssize_t qsb_write_at(QEMUSizedBuffer *qsb, const uint8_t *source,
-                     off_t pos, size_t count)
-{
-    ssize_t rc = qsb_grow(qsb, pos + count);
-    size_t to_copy;
-    size_t all_copy = count;
-    const struct iovec *iov;
-    ssize_t index;
-    char *dest;
-    off_t d_off, s_off = 0;
-
-    if (rc < 0) {
-        return rc;
-    }
-
-    if (pos + count > qsb->used) {
-        qsb->used = pos + count;
-    }
-
-    index = qsb_get_iovec(qsb, pos, &d_off);
-    if (index < 0) {
-        return -EINVAL;
-    }
-
-    while (all_copy > 0) {
-        iov = &qsb->iov[index];
-
-        dest = iov->iov_base;
-
-        to_copy = iov->iov_len - d_off;
-        if (to_copy > all_copy) {
-            to_copy = all_copy;
-        }
-
-        memcpy(&dest[d_off], &source[s_off], to_copy);
-
-        s_off += to_copy;
-        all_copy -= to_copy;
-
-        d_off = 0;
-        index++;
-    }
-
-    return count;
-}
-
-/**
- * Create a deep copy of the given QEMUSizedBuffer.
- *
- * @qsb: A QEMUSizedBuffer
- *
- * Returns a clone of @qsb or NULL on allocation failure
- */
-QEMUSizedBuffer *qsb_clone(const QEMUSizedBuffer *qsb)
-{
-    QEMUSizedBuffer *out = qsb_create(NULL, qsb_get_length(qsb));
-    size_t i;
-    ssize_t res;
-    off_t pos = 0;
-
-    if (!out) {
-        return NULL;
-    }
-
-    for (i = 0; i < qsb->n_iov; i++) {
-        res =  qsb_write_at(out, qsb->iov[i].iov_base,
-                            pos, qsb->iov[i].iov_len);
-        if (res < 0) {
-            qsb_free(out);
-            return NULL;
-        }
-        pos += res;
-    }
-
-    return out;
-}
-
-typedef struct QEMUBuffer {
-    QEMUSizedBuffer *qsb;
-    QEMUFile *file;
-} QEMUBuffer;
-
-static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUBuffer *s = opaque;
-    ssize_t len = qsb_get_length(s->qsb) - pos;
-
-    if (len <= 0) {
-        return 0;
-    }
-
-    if (len > size) {
-        len = size;
-    }
-    return qsb_get_buffer(s->qsb, pos, len, buf);
-}
-
-static int buf_put_buffer(void *opaque, const uint8_t *buf,
-                          int64_t pos, int size)
-{
-    QEMUBuffer *s = opaque;
-
-    return qsb_write_at(s->qsb, buf, pos, size);
-}
-
-static int buf_close(void *opaque)
-{
-    QEMUBuffer *s = opaque;
-
-    qsb_free(s->qsb);
-
-    g_free(s);
-
-    return 0;
-}
-
-const QEMUSizedBuffer *qemu_buf_get(QEMUFile *f)
-{
-    QEMUBuffer *p;
-
-    qemu_fflush(f);
-
-    p = f->opaque;
-
-    return p->qsb;
-}
-
-static const QEMUFileOps buf_read_ops = {
-    .get_buffer = buf_get_buffer,
-    .close =      buf_close,
-};
-
-static const QEMUFileOps buf_write_ops = {
-    .put_buffer = buf_put_buffer,
-    .close =      buf_close,
-};
-
-QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input)
-{
-    QEMUBuffer *s;
-
-    if (mode == NULL || (mode[0] != 'r' && mode[0] != 'w') ||
-        mode[1] != '\0') {
-        error_report("qemu_bufopen: Argument validity check failed");
-        return NULL;
-    }
-
-    s = g_malloc0(sizeof(QEMUBuffer));
-    if (mode[0] == 'r') {
-        s->qsb = input;
-    }
-
-    if (s->qsb == NULL) {
-        s->qsb = qsb_create(NULL, 0);
-    }
-    if (!s->qsb) {
-        g_free(s);
-        error_report("qemu_bufopen: qsb_create failed");
-        return NULL;
-    }
-
-
-    if (mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, &buf_read_ops);
-    } else {
-        s->file = qemu_fopen_ops(s, &buf_write_ops);
-    }
-    return s->file;
-}
diff --git a/qemu-ga.texi b/qemu-ga.texi
new file mode 100644
index 0000000..0e53bf6
--- /dev/null
+++ b/qemu-ga.texi
@@ -0,0 +1,137 @@
+@example
+@c man begin SYNOPSIS
+@command{qemu-ga} [@var{OPTIONS}]
+@c man end
+@end example
+
+@c man begin DESCRIPTION
+
+The QEMU Guest Agent is a daemon intended to be run within virtual
+machines. It allows the hypervisor host to perform various operations
+in the guest, such as:
+
+@itemize
+@item
+get information from the guest
+@item
+set the guest's system time
+@item
+read/write a file
+@item
+sync and freeze the filesystems
+@item
+suspend the guest
+@item
+reconfigure guest local processors
+@item
+set user's password
+@item
+...
+@end itemize
+
+qemu-ga will read a system configuration file on startup (located at
+q@file{/etc/qemu/qemu-ga.conf} by default), then parse remaining
+configuration options on the command line. For the same key, the last
+option wins, but the lists accumulate (see below for configuration
+file format).
+
+@c man end
+
+@c man begin OPTIONS
+@table @option
+@item -m, --method=@var{method}
+  Transport method: one of @samp{unix-listen}, @samp{virtio-serial}, or
+  @samp{isa-serial} (@samp{virtio-serial} is the default).
+
+@item -p, --path=@var{path}
+  Device/socket path (the default for virtio-serial is
+  @samp{/dev/virtio-ports/org.qemu.guest_agent.0},
+  the default for isa-serial is @samp{/dev/ttyS0})
+
+@item -l, --logfile=@var{path}
+  Set log file path (default is stderr).
+
+@item -f, --pidfile=@var{path}
+  Specify pid file (default is @samp{/var/run/qemu-ga.pid}).
+
+@item -F, --fsfreeze-hook=@var{path}
+  Enable fsfreeze hook. Accepts an optional argument that specifies
+  script to run on freeze/thaw. Script will be called with
+  'freeze'/'thaw' arguments accordingly (default is
+  @samp{/etc/qemu/fsfreeze-hook}). If using -F with an argument, do
+  not follow -F with a space (for example:
+  @samp{-F/var/run/fsfreezehook.sh}).
+
+@item -t, --statedir=@var{path}
+  Specify the directory to store state information (absolute paths only,
+  default is @samp{/var/run}).
+
+@item -v, --verbose
+  Log extra debugging information.
+
+@item -V, --version
+  Print version information and exit.
+
+@item -d, --daemon
+  Daemonize after startup (detach from terminal).
+
+@item -b, --blacklist=@var{list}
+  Comma-separated list of RPCs to disable (no spaces, @samp{?} to list
+  available RPCs).
+
+@item -D, --dump-conf
+  Dump the configuration in a format compatible with @file{qemu-ga.conf}
+  and exit.
+
+@item -h, --help
+  Display this help and exit.
+@end table
+
+@c man end
+
+@c man begin FILES
+
+The syntax of the @file{qemu-ga.conf} configuration file follows the
+Desktop Entry Specification, here is a quick summary: it consists of
+groups of key-value pairs, interspersed with comments.
+
+@example
+# qemu-ga configuration sample
+[general]
+daemonize = 0
+pidfile = /var/run/qemu-ga.pid
+verbose = 0
+method = virtio-serial
+path = /dev/virtio-ports/org.qemu.guest_agent.0
+statedir = /var/run
+@end example
+
+The list of keys follows the command line options:
+@table @option
+@item daemon= boolean
+@item method= string
+@item path= string
+@item logfile= string
+@item pidfile= string
+@item fsfreeze-hook= string
+@item statedir= string
+@item verbose= boolean
+@item blacklist= string list
+@end table
+
+@c man end
+
+@ignore
+
+@setfilename qemu-ga
+@settitle QEMU Guest Agent
+
+@c man begin AUTHOR
+Michael Roth <mdroth@linux.vnet.ibm.com>
+@c man end
+
+@c man begin SEEALSO
+qemu(1)
+@c man end
+
+@end ignore
diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index 9567774..7e95b2d 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -9,69 +9,75 @@
 @table @option
 ETEXI
 
-DEF("check", img_check,
-    "check [-q] [-f fmt] [--output=ofmt] [-r [leaks | all]] [-T src_cache] filename")
+DEF("bench", img_bench,
+    "bench [-c count] [-d depth] [-f fmt] [--flush-interval=flush_interval] [-n] [--no-drain] [-o offset] [--pattern=pattern] [-q] [-s buffer_size] [-S step_size] [-t cache] [-w] filename")
 STEXI
-@item check [-q] [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
+@item bench [-c @var{count}] [-d @var{depth}] [-f @var{fmt}] [--flush-interval=@var{flush_interval}] [-n] [--no-drain] [-o @var{offset}] [--pattern=@var{pattern}] [-q] [-s @var{buffer_size}] [-S @var{step_size}] [-t @var{cache}] [-w] @var{filename}
+ETEXI
+
+DEF("check", img_check,
+    "check [-q] [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] [-r [leaks | all]] [-T src_cache] filename")
+STEXI
+@item check [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
 ETEXI
 
 DEF("create", img_create,
-    "create [-q] [-f fmt] [-o options] filename [size]")
+    "create [-q] [--object objectdef] [--image-opts] [-f fmt] [-o options] filename [size]")
 STEXI
-@item create [-q] [-f @var{fmt}] [-o @var{options}] @var{filename} [@var{size}]
+@item create [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [-o @var{options}] @var{filename} [@var{size}]
 ETEXI
 
 DEF("commit", img_commit,
-    "commit [-q] [-f fmt] [-t cache] [-b base] [-d] [-p] filename")
+    "commit [-q] [--object objectdef] [--image-opts] [-f fmt] [-t cache] [-b base] [-d] [-p] filename")
 STEXI
-@item commit [-q] [-f @var{fmt}] [-t @var{cache}] [-b @var{base}] [-d] [-p] @var{filename}
+@item commit [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [-t @var{cache}] [-b @var{base}] [-d] [-p] @var{filename}
 ETEXI
 
 DEF("compare", img_compare,
-    "compare [-f fmt] [-F fmt] [-T src_cache] [-p] [-q] [-s] filename1 filename2")
+    "compare [--object objectdef] [--image-opts] [-f fmt] [-F fmt] [-T src_cache] [-p] [-q] [-s] filename1 filename2")
 STEXI
-@item compare [-f @var{fmt}] [-F @var{fmt}] [-T @var{src_cache}] [-p] [-q] [-s] @var{filename1} @var{filename2}
+@item compare [--object @var{objectdef}] [--image-opts] [-f @var{fmt}] [-F @var{fmt}] [-T @var{src_cache}] [-p] [-q] [-s] @var{filename1} @var{filename2}
 ETEXI
 
 DEF("convert", img_convert,
-    "convert [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-o options] [-s snapshot_id_or_name] [-l snapshot_param] [-S sparse_size] filename [filename2 [...]] output_filename")
+    "convert [--object objectdef] [--image-opts] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-o options] [-s snapshot_id_or_name] [-l snapshot_param] [-S sparse_size] filename [filename2 [...]] output_filename")
 STEXI
-@item convert [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
+@item convert [--object @var{objectdef}] [--image-opts] [-c] [-p] [-q] [-n] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-O @var{output_fmt}] [-o @var{options}] [-s @var{snapshot_id_or_name}] [-l @var{snapshot_param}] [-S @var{sparse_size}] @var{filename} [@var{filename2} [...]] @var{output_filename}
 ETEXI
 
 DEF("info", img_info,
-    "info [-f fmt] [--output=ofmt] [--backing-chain] filename")
+    "info [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] [--backing-chain] filename")
 STEXI
-@item info [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
+@item info [--object @var{objectdef}] [--image-opts] [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
 ETEXI
 
 DEF("map", img_map,
-    "map [-f fmt] [--output=ofmt] filename")
+    "map [--object objectdef] [--image-opts] [-f fmt] [--output=ofmt] filename")
 STEXI
-@item map [-f @var{fmt}] [--output=@var{ofmt}] @var{filename}
+@item map [--object @var{objectdef}] [--image-opts] [-f @var{fmt}] [--output=@var{ofmt}] @var{filename}
 ETEXI
 
 DEF("snapshot", img_snapshot,
-    "snapshot [-q] [-l | -a snapshot | -c snapshot | -d snapshot] filename")
+    "snapshot [--object objectdef] [--image-opts] [-q] [-l | -a snapshot | -c snapshot | -d snapshot] filename")
 STEXI
-@item snapshot [-q] [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot}] @var{filename}
+@item snapshot [--object @var{objectdef}] [--image-opts] [-q] [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot}] @var{filename}
 ETEXI
 
 DEF("rebase", img_rebase,
-    "rebase [-q] [-f fmt] [-t cache] [-T src_cache] [-p] [-u] -b backing_file [-F backing_fmt] filename")
+    "rebase [--object objectdef] [--image-opts] [-q] [-f fmt] [-t cache] [-T src_cache] [-p] [-u] -b backing_file [-F backing_fmt] filename")
 STEXI
-@item rebase [-q] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
+@item rebase [--object @var{objectdef}] [--image-opts] [-q] [-f @var{fmt}] [-t @var{cache}] [-T @var{src_cache}] [-p] [-u] -b @var{backing_file} [-F @var{backing_fmt}] @var{filename}
 ETEXI
 
 DEF("resize", img_resize,
-    "resize [-q] filename [+ | -]size")
+    "resize [--object objectdef] [--image-opts] [-q] filename [+ | -]size")
 STEXI
-@item resize [-q] @var{filename} [+ | -]@var{size}
+@item resize [--object @var{objectdef}] [--image-opts] [-q] @var{filename} [+ | -]@var{size}
 ETEXI
 
 DEF("amend", img_amend,
-    "amend [-p] [-q] [-f fmt] [-t cache] -o options filename")
+    "amend [--object objectdef] [--image-opts] [-p] [-q] [-f fmt] [-t cache] -o options filename")
 STEXI
-@item amend [-p] [-q] [-f @var{fmt}] [-t @var{cache}] -o @var{options} @var{filename}
+@item amend [--object @var{objectdef}] [--image-opts] [-p] [-q] [-f @var{fmt}] [-t @var{cache}] -o @var{options} @var{filename}
 @end table
 ETEXI
diff --git a/qemu-img.c b/qemu-img.c
index a42335c..f204d041 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -21,22 +21,30 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
+#include "qemu-version.h"
+#include "qapi/error.h"
 #include "qapi-visit.h"
 #include "qapi/qmp-output-visitor.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qjson.h"
-#include "qemu-common.h"
+#include "qemu/cutils.h"
+#include "qemu/config-file.h"
 #include "qemu/option.h"
 #include "qemu/error-report.h"
-#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "qom/object_interfaces.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "block/blockjob.h"
 #include "block/qapi.h"
+#include "crypto/init.h"
+#include "trace/control.h"
 #include <getopt.h>
 
-#define QEMU_IMG_VERSION "qemu-img version " QEMU_VERSION \
-                          ", Copyright (c) 2004-2008 Fabrice Bellard\n"
+#define QEMU_IMG_VERSION "qemu-img version " QEMU_VERSION QEMU_PKGVERSION \
+                          ", " QEMU_COPYRIGHT "\n"
 
 typedef struct img_cmd_t {
     const char *name;
@@ -46,6 +54,11 @@
 enum {
     OPTION_OUTPUT = 256,
     OPTION_BACKING_CHAIN = 257,
+    OPTION_OBJECT = 258,
+    OPTION_IMAGE_OPTS = 259,
+    OPTION_PATTERN = 260,
+    OPTION_FLUSH_INTERVAL = 261,
+    OPTION_NO_DRAIN = 262,
 };
 
 typedef enum OutputFormat {
@@ -53,8 +66,7 @@
     OFORMAT_HUMAN,
 } OutputFormat;
 
-/* Default to cache=writeback as data integrity is not important for qemu-tcg. */
-#define BDRV_O_FLAGS BDRV_O_CACHE_WB
+/* Default to cache=writeback as data integrity is not important for qemu-img */
 #define BDRV_DEFAULT_CACHE "writeback"
 
 static void format_print(void *opaque, const char *name)
@@ -81,9 +93,14 @@
 {
     const char *help_msg =
            QEMU_IMG_VERSION
-           "usage: qemu-img command [command options]\n"
+           "usage: qemu-img [standard options] command [command options]\n"
            "QEMU disk image utility\n"
            "\n"
+           "    '-h', '--help'       display this help and exit\n"
+           "    '-V', '--version'    output version information and exit\n"
+           "    '-T', '--trace'      [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
+           "                         specify tracing options\n"
+           "\n"
            "Command syntax:\n"
 #define DEF(option, callback, arg_string)        \
            "  " arg_string "\n"
@@ -93,6 +110,10 @@
            "\n"
            "Command parameters:\n"
            "  'filename' is a disk image filename\n"
+           "  'objectdef' is a QEMU user creatable object definition. See the qemu(1)\n"
+           "    manual page for a description of the object properties. The most common\n"
+           "    object type is a 'secret', which is used to supply passwords and/or\n"
+           "    encryption keys.\n"
            "  'fmt' is the disk image format. It is guessed automatically in most cases\n"
            "  'cache' is the cache mode used to write the output disk image, the valid\n"
            "    options are: 'none', 'writeback' (default, except for convert), 'writethrough',\n"
@@ -153,6 +174,24 @@
     exit(EXIT_SUCCESS);
 }
 
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+static QemuOptsList qemu_source_opts = {
+    .name = "source",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_source_opts.head),
+    .desc = {
+        { }
+    },
+};
+
 static int GCC_FMT_ATTR(2, 3) qprintf(bool quiet, const char *fmt, ...)
 {
     int ret = 0;
@@ -165,102 +204,12 @@
     return ret;
 }
 
-#if defined(WIN32)
-/* XXX: put correct support for win32 */
-static int read_password(char *buf, int buf_size)
-{
-    int c, i;
-
-    printf("Password: ");
-    fflush(stdout);
-    i = 0;
-    for(;;) {
-        c = getchar();
-        if (c < 0) {
-            buf[i] = '\0';
-            return -1;
-        } else if (c == '\n') {
-            break;
-        } else if (i < (buf_size - 1)) {
-            buf[i++] = c;
-        }
-    }
-    buf[i] = '\0';
-    return 0;
-}
-
-#else
-
-#include <termios.h>
-
-static struct termios oldtty;
-
-static void term_exit(void)
-{
-    tcsetattr (0, TCSANOW, &oldtty);
-}
-
-static void term_init(void)
-{
-    struct termios tty;
-
-    tcgetattr (0, &tty);
-    oldtty = tty;
-
-    tty.c_iflag &= ~(IGNBRK|BRKINT|PARMRK|ISTRIP
-                          |INLCR|IGNCR|ICRNL|IXON);
-    tty.c_oflag |= OPOST;
-    tty.c_lflag &= ~(ECHO|ECHONL|ICANON|IEXTEN);
-    tty.c_cflag &= ~(CSIZE|PARENB);
-    tty.c_cflag |= CS8;
-    tty.c_cc[VMIN] = 1;
-    tty.c_cc[VTIME] = 0;
-
-    tcsetattr (0, TCSANOW, &tty);
-
-    atexit(term_exit);
-}
-
-static int read_password(char *buf, int buf_size)
-{
-    uint8_t ch;
-    int i, ret;
-
-    printf("password: ");
-    fflush(stdout);
-    term_init();
-    i = 0;
-    for(;;) {
-        ret = read(0, &ch, 1);
-        if (ret == -1) {
-            if (errno == EAGAIN || errno == EINTR) {
-                continue;
-            } else {
-                break;
-            }
-        } else if (ret == 0) {
-            ret = -1;
-            break;
-        } else {
-            if (ch == '\r') {
-                ret = 0;
-                break;
-            }
-            if (i < (buf_size - 1))
-                buf[i++] = ch;
-        }
-    }
-    term_exit();
-    buf[i] = '\0';
-    printf("\n");
-    return ret;
-}
-#endif
 
 static int print_block_option_help(const char *filename, const char *fmt)
 {
     BlockDriver *drv, *proto_drv;
     QemuOptsList *create_opts = NULL;
+    Error *local_err = NULL;
 
     /* Find driver and parse its options */
     drv = bdrv_find_format(fmt);
@@ -271,9 +220,9 @@
 
     create_opts = qemu_opts_append(create_opts, drv->create_opts);
     if (filename) {
-        proto_drv = bdrv_find_protocol(filename, true);
+        proto_drv = bdrv_find_protocol(filename, true, &local_err);
         if (!proto_drv) {
-            error_report("Unknown protocol '%s'", filename);
+            error_report_err(local_err);
             qemu_opts_free(create_opts);
             return 1;
         }
@@ -285,70 +234,126 @@
     return 0;
 }
 
-static BlockBackend *img_open(const char *id, const char *filename,
-                              const char *fmt, int flags,
-                              bool require_io, bool quiet)
+
+static int img_open_password(BlockBackend *blk, const char *filename,
+                             int flags, bool quiet)
 {
-    BlockBackend *blk;
     BlockDriverState *bs;
-    BlockDriver *drv;
     char password[256];
-    Error *local_err = NULL;
-    int ret;
 
-    blk = blk_new_with_bs(id, &error_abort);
     bs = blk_bs(blk);
-
-    if (fmt) {
-        drv = bdrv_find_format(fmt);
-        if (!drv) {
-            error_report("Unknown file format '%s'", fmt);
-            goto fail;
-        }
-    } else {
-        drv = NULL;
-    }
-
-    ret = bdrv_open(&bs, filename, NULL, NULL, flags, drv, &local_err);
-    if (ret < 0) {
-        error_report("Could not open '%s': %s", filename,
-                     error_get_pretty(local_err));
-        error_free(local_err);
-        goto fail;
-    }
-
-    if (bdrv_is_encrypted(bs) && require_io) {
+    if (bdrv_is_encrypted(bs) && bdrv_key_required(bs) &&
+        !(flags & BDRV_O_NO_IO)) {
         qprintf(quiet, "Disk image '%s' is encrypted.\n", filename);
-        if (read_password(password, sizeof(password)) < 0) {
+        if (qemu_read_password(password, sizeof(password)) < 0) {
             error_report("No password given");
-            goto fail;
+            return -1;
         }
         if (bdrv_set_key(bs, password) < 0) {
             error_report("invalid password");
-            goto fail;
+            return -1;
         }
     }
-    return blk;
-fail:
-    blk_unref(blk);
-    return NULL;
+    return 0;
 }
 
+
+static BlockBackend *img_open_opts(const char *optstr,
+                                   QemuOpts *opts, int flags, bool writethrough,
+                                   bool quiet)
+{
+    QDict *options;
+    Error *local_err = NULL;
+    BlockBackend *blk;
+    options = qemu_opts_to_qdict(opts, NULL);
+    blk = blk_new_open(NULL, NULL, options, flags, &local_err);
+    if (!blk) {
+        error_reportf_err(local_err, "Could not open '%s': ", optstr);
+        return NULL;
+    }
+    blk_set_enable_write_cache(blk, !writethrough);
+
+    if (img_open_password(blk, optstr, flags, quiet) < 0) {
+        blk_unref(blk);
+        return NULL;
+    }
+    return blk;
+}
+
+static BlockBackend *img_open_file(const char *filename,
+                                   const char *fmt, int flags,
+                                   bool writethrough, bool quiet)
+{
+    BlockBackend *blk;
+    Error *local_err = NULL;
+    QDict *options = NULL;
+
+    if (fmt) {
+        options = qdict_new();
+        qdict_put(options, "driver", qstring_from_str(fmt));
+    }
+
+    blk = blk_new_open(filename, NULL, options, flags, &local_err);
+    if (!blk) {
+        error_reportf_err(local_err, "Could not open '%s': ", filename);
+        return NULL;
+    }
+    blk_set_enable_write_cache(blk, !writethrough);
+
+    if (img_open_password(blk, filename, flags, quiet) < 0) {
+        blk_unref(blk);
+        return NULL;
+    }
+    return blk;
+}
+
+
+static BlockBackend *img_open(bool image_opts,
+                              const char *filename,
+                              const char *fmt, int flags, bool writethrough,
+                              bool quiet)
+{
+    BlockBackend *blk;
+    if (image_opts) {
+        QemuOpts *opts;
+        if (fmt) {
+            error_report("--image-opts and --format are mutually exclusive");
+            return NULL;
+        }
+        opts = qemu_opts_parse_noisily(qemu_find_opts("source"),
+                                       filename, true);
+        if (!opts) {
+            return NULL;
+        }
+        blk = img_open_opts(filename, opts, flags, writethrough, quiet);
+    } else {
+        blk = img_open_file(filename, fmt, flags, writethrough, quiet);
+    }
+    return blk;
+}
+
+
 static int add_old_style_options(const char *fmt, QemuOpts *opts,
                                  const char *base_filename,
                                  const char *base_fmt)
 {
+    Error *err = NULL;
+
     if (base_filename) {
-        if (qemu_opt_set(opts, BLOCK_OPT_BACKING_FILE, base_filename)) {
+        qemu_opt_set(opts, BLOCK_OPT_BACKING_FILE, base_filename, &err);
+        if (err) {
             error_report("Backing file not supported for file format '%s'",
                          fmt);
+            error_free(err);
             return -1;
         }
     }
     if (base_fmt) {
-        if (qemu_opt_set(opts, BLOCK_OPT_BACKING_FMT, base_fmt)) {
+        qemu_opt_set(opts, BLOCK_OPT_BACKING_FMT, base_fmt, &err);
+        if (err) {
             error_report("Backing file format not supported for file "
                          "format '%s'", fmt);
+            error_free(err);
             return -1;
         }
     }
@@ -368,7 +373,13 @@
     bool quiet = false;
 
     for(;;) {
-        c = getopt(argc, argv, "F:b:f:he6o:q");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "F:b:f:he6o:q",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -410,6 +421,14 @@
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                goto fail;
+            }
+        }   break;
         }
     }
 
@@ -425,11 +444,18 @@
     }
     optind++;
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        goto fail;
+    }
+
     /* Get image size, if specified */
     if (optind < argc) {
         int64_t sval;
         char *end;
-        sval = strtosz_suffix(argv[optind++], &end, STRTOSZ_DEFSUFFIX_B);
+        sval = qemu_strtosz_suffix(argv[optind++], &end,
+                                   QEMU_STRTOSZ_DEFSUFFIX_B);
         if (sval < 0 || *end) {
             if (sval == -ERANGE) {
                 error_report("Image size must be less than 8 EiB!");
@@ -448,10 +474,9 @@
     }
 
     bdrv_img_create(filename, fmt, base_filename, base_fmt,
-                    options, img_size, BDRV_O_FLAGS, &local_err, quiet);
+                    options, img_size, 0, &local_err, quiet);
     if (local_err) {
-        error_report("%s: %s", filename, error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err, "%s: ", filename);
         goto fail;
     }
 
@@ -465,18 +490,17 @@
 
 static void dump_json_image_check(ImageCheck *check, bool quiet)
 {
-    Error *local_err = NULL;
     QString *str;
-    QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj;
-    visit_type_ImageCheck(qmp_output_get_visitor(ov),
-                          &check, NULL, &local_err);
-    obj = qmp_output_get_qobject(ov);
+    Visitor *v = qmp_output_visitor_new(&obj);
+
+    visit_type_ImageCheck(v, NULL, &check, &error_abort);
+    visit_complete(v, &obj);
     str = qobject_to_json_pretty(obj);
     assert(str != NULL);
     qprintf(quiet, "%s\n", qstring_get_str(str));
     qobject_decref(obj);
-    qmp_output_visitor_cleanup(ov);
+    visit_free(v);
     QDECREF(str);
 }
 
@@ -579,13 +603,16 @@
     BlockBackend *blk;
     BlockDriverState *bs;
     int fix = 0;
-    int flags = BDRV_O_FLAGS | BDRV_O_CHECK;
+    int flags = BDRV_O_CHECK;
+    bool writethrough;
     ImageCheck *check;
     bool quiet = false;
+    bool image_opts = false;
 
     fmt = NULL;
     output = NULL;
     cache = BDRV_DEFAULT_CACHE;
+
     for(;;) {
         int option_index = 0;
         static const struct option long_options[] = {
@@ -593,6 +620,8 @@
             {"format", required_argument, 0, 'f'},
             {"repair", required_argument, 0, 'r'},
             {"output", required_argument, 0, OPTION_OUTPUT},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
             {0, 0, 0, 0}
         };
         c = getopt_long(argc, argv, "hf:r:T:q",
@@ -629,6 +658,17 @@
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -645,13 +685,19 @@
         return 1;
     }
 
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", cache);
         return 1;
     }
 
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         return 1;
     }
@@ -735,18 +781,16 @@
     if (ret < 0) {
         error_setg_errno(cbi->errp, -ret, "Block job failed");
     }
-
-    /* Drop this block job's reference */
-    bdrv_unref(cbi->bs);
 }
 
 static void run_block_job(BlockJob *job, Error **errp)
 {
-    AioContext *aio_context = bdrv_get_aio_context(job->bs);
+    AioContext *aio_context = blk_get_aio_context(job->blk);
 
     do {
         aio_poll(aio_context, true);
-        qemu_progress_print((float)job->offset / job->len * 100.f, 0);
+        qemu_progress_print(job->len ?
+                            ((float)job->offset / job->len * 100.f) : 0.0f, 0);
     } while (!job->ready);
 
     block_job_complete_sync(job, errp);
@@ -763,14 +807,23 @@
     BlockBackend *blk;
     BlockDriverState *bs, *base_bs;
     bool progress = false, quiet = false, drop = false;
+    bool writethrough;
     Error *local_err = NULL;
     CommonBlockJobCBInfo cbi;
+    bool image_opts = false;
 
     fmt = NULL;
     cache = BDRV_DEFAULT_CACHE;
     base = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:ht:b:dpq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "f:ht:b:dpq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -799,6 +852,17 @@
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -812,14 +876,20 @@
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     flags = BDRV_O_RDWR | BDRV_O_UNMAP;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         return 1;
     }
 
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         return 1;
     }
@@ -831,14 +901,14 @@
     if (base) {
         base_bs = bdrv_find_backing_image(bs, base);
         if (!base_bs) {
-            error_set(&local_err, QERR_BASE_NOT_FOUND, base);
+            error_setg(&local_err, QERR_BASE_NOT_FOUND, base);
             goto done;
         }
     } else {
         /* This is different from QMP, which by default uses the deepest file in
          * the backing chain (i.e., the very base); however, the traditional
          * behavior of qemu-img commit is using the immediate backing file. */
-        base_bs = bs->backing_hd;
+        base_bs = backing_bs(bs);
         if (!base_bs) {
             error_setg(&local_err, "Image does not have a backing file");
             goto done;
@@ -850,18 +920,18 @@
         .bs   = bs,
     };
 
-    commit_active_start(bs, base_bs, 0, BLOCKDEV_ON_ERROR_REPORT,
+    commit_active_start("commit", bs, base_bs, 0, BLOCKDEV_ON_ERROR_REPORT,
                         common_block_job_cb, &cbi, &local_err);
     if (local_err) {
         goto done;
     }
 
-    /* The block job will swap base_bs and bs (which is not what we really want
-     * here, but okay) and unref base_bs (after the swap, i.e., the old top
-     * image). In order to still be able to empty that top image afterwards,
-     * increment the reference counter here preemptively. */
+    /* When the block job completes, the BlockBackend reference will point to
+     * the old backing file. In order to avoid that the top image is already
+     * deleted, so we can still empty it afterwards, increment the reference
+     * counter here preemptively. */
     if (!drop) {
-        bdrv_ref(base_bs);
+        bdrv_ref(bs);
     }
 
     run_block_job(bs->job, &local_err);
@@ -869,8 +939,8 @@
         goto unref_backing;
     }
 
-    if (!drop && base_bs->drv->bdrv_make_empty) {
-        ret = base_bs->drv->bdrv_make_empty(base_bs);
+    if (!drop && bs->drv->bdrv_make_empty) {
+        ret = bs->drv->bdrv_make_empty(bs);
         if (ret) {
             error_setg_errno(&local_err, -ret, "Could not empty %s",
                              filename);
@@ -880,7 +950,7 @@
 
 unref_backing:
     if (!drop) {
-        bdrv_unref(base_bs);
+        bdrv_unref(bs);
     }
 
 done:
@@ -889,8 +959,7 @@
     blk_unref(blk);
 
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return 1;
     }
 
@@ -977,7 +1046,8 @@
 static int compare_sectors(const uint8_t *buf1, const uint8_t *buf2, int n,
     int *pnum)
 {
-    int res, i;
+    bool res;
+    int i;
 
     if (n <= 0) {
         *pnum = 0;
@@ -1016,19 +1086,20 @@
  * Returns 0 in case sectors are filled with 0, 1 if sectors contain non-zero
  * data and negative value on error.
  *
- * @param bs:  Driver used for accessing file
+ * @param blk:  BlockBackend for the image
  * @param sect_num: Number of first sector to check
  * @param sect_count: Number of sectors to check
  * @param filename: Name of disk file we are checking (logging purpose)
  * @param buffer: Allocated buffer for storing read data
  * @param quiet: Flag for quiet mode
  */
-static int check_empty_sectors(BlockDriverState *bs, int64_t sect_num,
+static int check_empty_sectors(BlockBackend *blk, int64_t sect_num,
                                int sect_count, const char *filename,
                                uint8_t *buffer, bool quiet)
 {
     int pnum, ret = 0;
-    ret = bdrv_read(bs, sect_num, buffer, sect_count);
+    ret = blk_pread(blk, sect_num << BDRV_SECTOR_BITS, buffer,
+                    sect_count << BDRV_SECTOR_BITS);
     if (ret < 0) {
         error_report("Error while reading offset %" PRId64 " of %s: %s",
                      sectors_to_bytes(sect_num), filename, strerror(-ret));
@@ -1063,15 +1134,24 @@
     int ret = 0; /* return value - 0 Ident, 1 Different, >1 Error */
     bool progress = false, quiet = false, strict = false;
     int flags;
+    bool writethrough;
     int64_t total_sectors;
     int64_t sector_num = 0;
     int64_t nb_sectors;
     int c, pnum;
     uint64_t progress_base;
+    bool image_opts = false;
 
     cache = BDRV_DEFAULT_CACHE;
     for (;;) {
-        c = getopt(argc, argv, "hf:F:T:pqs");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hf:F:T:pqs",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -1098,6 +1178,18 @@
         case 's':
             strict = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                ret = 2;
+                goto out4;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -1113,41 +1205,48 @@
     filename1 = argv[optind++];
     filename2 = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        ret = 2;
+        goto out4;
+    }
+
     /* Initialize before goto out */
     qemu_progress_init(progress, 2.0);
 
-    flags = BDRV_O_FLAGS;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    flags = 0;
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", cache);
         ret = 2;
         goto out3;
     }
 
-    blk1 = img_open("image_1", filename1, fmt1, flags, true, quiet);
+    blk1 = img_open(image_opts, filename1, fmt1, flags, writethrough, quiet);
     if (!blk1) {
         ret = 2;
         goto out3;
     }
-    bs1 = blk_bs(blk1);
 
-    blk2 = img_open("image_2", filename2, fmt2, flags, true, quiet);
+    blk2 = img_open(image_opts, filename2, fmt2, flags, writethrough, quiet);
     if (!blk2) {
         ret = 2;
         goto out2;
     }
+    bs1 = blk_bs(blk1);
     bs2 = blk_bs(blk2);
 
-    buf1 = qemu_blockalign(bs1, IO_BUF_SIZE);
-    buf2 = qemu_blockalign(bs2, IO_BUF_SIZE);
-    total_sectors1 = bdrv_nb_sectors(bs1);
+    buf1 = blk_blockalign(blk1, IO_BUF_SIZE);
+    buf2 = blk_blockalign(blk2, IO_BUF_SIZE);
+    total_sectors1 = blk_nb_sectors(blk1);
     if (total_sectors1 < 0) {
         error_report("Can't get size of %s: %s",
                      filename1, strerror(-total_sectors1));
         ret = 4;
         goto out;
     }
-    total_sectors2 = bdrv_nb_sectors(bs2);
+    total_sectors2 = blk_nb_sectors(blk2);
     if (total_sectors2 < 0) {
         error_report("Can't get size of %s: %s",
                      filename2, strerror(-total_sectors2));
@@ -1166,30 +1265,55 @@
     }
 
     for (;;) {
+        int64_t status1, status2;
+        BlockDriverState *file;
+
         nb_sectors = sectors_to_process(total_sectors, sector_num);
         if (nb_sectors <= 0) {
             break;
         }
-        allocated1 = bdrv_is_allocated_above(bs1, NULL, sector_num, nb_sectors,
-                                             &pnum1);
-        if (allocated1 < 0) {
+        status1 = bdrv_get_block_status_above(bs1, NULL, sector_num,
+                                              total_sectors1 - sector_num,
+                                              &pnum1, &file);
+        if (status1 < 0) {
             ret = 3;
             error_report("Sector allocation test failed for %s", filename1);
             goto out;
         }
+        allocated1 = status1 & BDRV_BLOCK_ALLOCATED;
 
-        allocated2 = bdrv_is_allocated_above(bs2, NULL, sector_num, nb_sectors,
-                                             &pnum2);
-        if (allocated2 < 0) {
+        status2 = bdrv_get_block_status_above(bs2, NULL, sector_num,
+                                              total_sectors2 - sector_num,
+                                              &pnum2, &file);
+        if (status2 < 0) {
             ret = 3;
             error_report("Sector allocation test failed for %s", filename2);
             goto out;
         }
-        nb_sectors = MIN(pnum1, pnum2);
+        allocated2 = status2 & BDRV_BLOCK_ALLOCATED;
+        if (pnum1) {
+            nb_sectors = MIN(nb_sectors, pnum1);
+        }
+        if (pnum2) {
+            nb_sectors = MIN(nb_sectors, pnum2);
+        }
 
-        if (allocated1 == allocated2) {
+        if (strict) {
+            if ((status1 & ~BDRV_BLOCK_OFFSET_MASK) !=
+                (status2 & ~BDRV_BLOCK_OFFSET_MASK)) {
+                ret = 1;
+                qprintf(quiet, "Strict mode: Offset %" PRId64
+                        " block status mismatch!\n",
+                        sectors_to_bytes(sector_num));
+                goto out;
+            }
+        }
+        if ((status1 & BDRV_BLOCK_ZERO) && (status2 & BDRV_BLOCK_ZERO)) {
+            nb_sectors = MIN(pnum1, pnum2);
+        } else if (allocated1 == allocated2) {
             if (allocated1) {
-                ret = bdrv_read(bs1, sector_num, buf1, nb_sectors);
+                ret = blk_pread(blk1, sector_num << BDRV_SECTOR_BITS, buf1,
+                                nb_sectors << BDRV_SECTOR_BITS);
                 if (ret < 0) {
                     error_report("Error while reading offset %" PRId64 " of %s:"
                                  " %s", sectors_to_bytes(sector_num), filename1,
@@ -1197,7 +1321,8 @@
                     ret = 4;
                     goto out;
                 }
-                ret = bdrv_read(bs2, sector_num, buf2, nb_sectors);
+                ret = blk_pread(blk2, sector_num << BDRV_SECTOR_BITS, buf2,
+                                nb_sectors << BDRV_SECTOR_BITS);
                 if (ret < 0) {
                     error_report("Error while reading offset %" PRId64
                                  " of %s: %s", sectors_to_bytes(sector_num),
@@ -1215,19 +1340,12 @@
                 }
             }
         } else {
-            if (strict) {
-                ret = 1;
-                qprintf(quiet, "Strict mode: Offset %" PRId64
-                        " allocation mismatch!\n",
-                        sectors_to_bytes(sector_num));
-                goto out;
-            }
 
             if (allocated1) {
-                ret = check_empty_sectors(bs1, sector_num, nb_sectors,
+                ret = check_empty_sectors(blk1, sector_num, nb_sectors,
                                           filename1, buf1, quiet);
             } else {
-                ret = check_empty_sectors(bs2, sector_num, nb_sectors,
+                ret = check_empty_sectors(blk2, sector_num, nb_sectors,
                                           filename2, buf1, quiet);
             }
             if (ret) {
@@ -1244,18 +1362,18 @@
     }
 
     if (total_sectors1 != total_sectors2) {
-        BlockDriverState *bs_over;
+        BlockBackend *blk_over;
         int64_t total_sectors_over;
         const char *filename_over;
 
         qprintf(quiet, "Warning: Image size mismatch!\n");
         if (total_sectors1 > total_sectors2) {
             total_sectors_over = total_sectors1;
-            bs_over = bs1;
+            blk_over = blk1;
             filename_over = filename1;
         } else {
             total_sectors_over = total_sectors2;
-            bs_over = bs2;
+            blk_over = blk2;
             filename_over = filename2;
         }
 
@@ -1264,7 +1382,7 @@
             if (nb_sectors <= 0) {
                 break;
             }
-            ret = bdrv_is_allocated_above(bs_over, NULL, sector_num,
+            ret = bdrv_is_allocated_above(blk_bs(blk_over), NULL, sector_num,
                                           nb_sectors, &pnum);
             if (ret < 0) {
                 ret = 3;
@@ -1275,7 +1393,7 @@
             }
             nb_sectors = pnum;
             if (ret) {
-                ret = check_empty_sectors(bs_over, sector_num, nb_sectors,
+                ret = check_empty_sectors(blk_over, sector_num, nb_sectors,
                                           filename_over, buf1, quiet);
                 if (ret) {
                     if (ret < 0) {
@@ -1303,23 +1421,337 @@
     blk_unref(blk1);
 out3:
     qemu_progress_end();
+out4:
+    return ret;
+}
+
+enum ImgConvertBlockStatus {
+    BLK_DATA,
+    BLK_ZERO,
+    BLK_BACKING_FILE,
+};
+
+typedef struct ImgConvertState {
+    BlockBackend **src;
+    int64_t *src_sectors;
+    int src_cur, src_num;
+    int64_t src_cur_offset;
+    int64_t total_sectors;
+    int64_t allocated_sectors;
+    enum ImgConvertBlockStatus status;
+    int64_t sector_next_status;
+    BlockBackend *target;
+    bool has_zero_init;
+    bool compressed;
+    bool target_has_backing;
+    int min_sparse;
+    size_t cluster_sectors;
+    size_t buf_sectors;
+} ImgConvertState;
+
+static void convert_select_part(ImgConvertState *s, int64_t sector_num)
+{
+    assert(sector_num >= s->src_cur_offset);
+    while (sector_num - s->src_cur_offset >= s->src_sectors[s->src_cur]) {
+        s->src_cur_offset += s->src_sectors[s->src_cur];
+        s->src_cur++;
+        assert(s->src_cur < s->src_num);
+    }
+}
+
+static int convert_iteration_sectors(ImgConvertState *s, int64_t sector_num)
+{
+    int64_t ret;
+    int n;
+
+    convert_select_part(s, sector_num);
+
+    assert(s->total_sectors > sector_num);
+    n = MIN(s->total_sectors - sector_num, BDRV_REQUEST_MAX_SECTORS);
+
+    if (s->sector_next_status <= sector_num) {
+        BlockDriverState *file;
+        ret = bdrv_get_block_status(blk_bs(s->src[s->src_cur]),
+                                    sector_num - s->src_cur_offset,
+                                    n, &n, &file);
+        if (ret < 0) {
+            return ret;
+        }
+
+        if (ret & BDRV_BLOCK_ZERO) {
+            s->status = BLK_ZERO;
+        } else if (ret & BDRV_BLOCK_DATA) {
+            s->status = BLK_DATA;
+        } else if (!s->target_has_backing) {
+            /* Without a target backing file we must copy over the contents of
+             * the backing file as well. */
+            /* Check block status of the backing file chain to avoid
+             * needlessly reading zeroes and limiting the iteration to the
+             * buffer size */
+            ret = bdrv_get_block_status_above(blk_bs(s->src[s->src_cur]), NULL,
+                                              sector_num - s->src_cur_offset,
+                                              n, &n, &file);
+            if (ret < 0) {
+                return ret;
+            }
+
+            if (ret & BDRV_BLOCK_ZERO) {
+                s->status = BLK_ZERO;
+            } else {
+                s->status = BLK_DATA;
+            }
+        } else {
+            s->status = BLK_BACKING_FILE;
+        }
+
+        s->sector_next_status = sector_num + n;
+    }
+
+    n = MIN(n, s->sector_next_status - sector_num);
+    if (s->status == BLK_DATA) {
+        n = MIN(n, s->buf_sectors);
+    }
+
+    /* We need to write complete clusters for compressed images, so if an
+     * unallocated area is shorter than that, we must consider the whole
+     * cluster allocated. */
+    if (s->compressed) {
+        if (n < s->cluster_sectors) {
+            n = MIN(s->cluster_sectors, s->total_sectors - sector_num);
+            s->status = BLK_DATA;
+        } else {
+            n = QEMU_ALIGN_DOWN(n, s->cluster_sectors);
+        }
+    }
+
+    return n;
+}
+
+static int convert_read(ImgConvertState *s, int64_t sector_num, int nb_sectors,
+                        uint8_t *buf)
+{
+    int n;
+    int ret;
+
+    assert(nb_sectors <= s->buf_sectors);
+    while (nb_sectors > 0) {
+        BlockBackend *blk;
+        int64_t bs_sectors;
+
+        /* In the case of compression with multiple source files, we can get a
+         * nb_sectors that spreads into the next part. So we must be able to
+         * read across multiple BDSes for one convert_read() call. */
+        convert_select_part(s, sector_num);
+        blk = s->src[s->src_cur];
+        bs_sectors = s->src_sectors[s->src_cur];
+
+        n = MIN(nb_sectors, bs_sectors - (sector_num - s->src_cur_offset));
+        ret = blk_pread(blk,
+                        (sector_num - s->src_cur_offset) << BDRV_SECTOR_BITS,
+                        buf, n << BDRV_SECTOR_BITS);
+        if (ret < 0) {
+            return ret;
+        }
+
+        sector_num += n;
+        nb_sectors -= n;
+        buf += n * BDRV_SECTOR_SIZE;
+    }
+
+    return 0;
+}
+
+static int convert_write(ImgConvertState *s, int64_t sector_num, int nb_sectors,
+                         const uint8_t *buf)
+{
+    int ret;
+
+    while (nb_sectors > 0) {
+        int n = nb_sectors;
+
+        switch (s->status) {
+        case BLK_BACKING_FILE:
+            /* If we have a backing file, leave clusters unallocated that are
+             * unallocated in the source image, so that the backing file is
+             * visible at the respective offset. */
+            assert(s->target_has_backing);
+            break;
+
+        case BLK_DATA:
+            /* We must always write compressed clusters as a whole, so don't
+             * try to find zeroed parts in the buffer. We can only save the
+             * write if the buffer is completely zeroed and we're allowed to
+             * keep the target sparse. */
+            if (s->compressed) {
+                if (s->has_zero_init && s->min_sparse &&
+                    buffer_is_zero(buf, n * BDRV_SECTOR_SIZE))
+                {
+                    assert(!s->target_has_backing);
+                    break;
+                }
+
+                ret = blk_write_compressed(s->target, sector_num, buf, n);
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+            }
+
+            /* If there is real non-zero data or we're told to keep the target
+             * fully allocated (-S 0), we must write it. Otherwise we can treat
+             * it as zero sectors. */
+            if (!s->min_sparse ||
+                is_allocated_sectors_min(buf, n, &n, s->min_sparse))
+            {
+                ret = blk_pwrite(s->target, sector_num << BDRV_SECTOR_BITS,
+                                 buf, n << BDRV_SECTOR_BITS, 0);
+                if (ret < 0) {
+                    return ret;
+                }
+                break;
+            }
+            /* fall-through */
+
+        case BLK_ZERO:
+            if (s->has_zero_init) {
+                break;
+            }
+            ret = blk_pwrite_zeroes(s->target, sector_num << BDRV_SECTOR_BITS,
+                                    n << BDRV_SECTOR_BITS, 0);
+            if (ret < 0) {
+                return ret;
+            }
+            break;
+        }
+
+        sector_num += n;
+        nb_sectors -= n;
+        buf += n * BDRV_SECTOR_SIZE;
+    }
+
+    return 0;
+}
+
+static int convert_do_copy(ImgConvertState *s)
+{
+    uint8_t *buf = NULL;
+    int64_t sector_num, allocated_done;
+    int ret;
+    int n;
+
+    /* Check whether we have zero initialisation or can get it efficiently */
+    s->has_zero_init = s->min_sparse && !s->target_has_backing
+                     ? bdrv_has_zero_init(blk_bs(s->target))
+                     : false;
+
+    if (!s->has_zero_init && !s->target_has_backing &&
+        bdrv_can_write_zeroes_with_unmap(blk_bs(s->target)))
+    {
+        ret = blk_make_zero(s->target, BDRV_REQ_MAY_UNMAP);
+        if (ret == 0) {
+            s->has_zero_init = true;
+        }
+    }
+
+    /* Allocate buffer for copied data. For compressed images, only one cluster
+     * can be copied at a time. */
+    if (s->compressed) {
+        if (s->cluster_sectors <= 0 || s->cluster_sectors > s->buf_sectors) {
+            error_report("invalid cluster size");
+            ret = -EINVAL;
+            goto fail;
+        }
+        s->buf_sectors = s->cluster_sectors;
+    }
+    buf = blk_blockalign(s->target, s->buf_sectors * BDRV_SECTOR_SIZE);
+
+    /* Calculate allocated sectors for progress */
+    s->allocated_sectors = 0;
+    sector_num = 0;
+    while (sector_num < s->total_sectors) {
+        n = convert_iteration_sectors(s, sector_num);
+        if (n < 0) {
+            ret = n;
+            goto fail;
+        }
+        if (s->status == BLK_DATA || (!s->min_sparse && s->status == BLK_ZERO))
+        {
+            s->allocated_sectors += n;
+        }
+        sector_num += n;
+    }
+
+    /* Do the copy */
+    s->src_cur = 0;
+    s->src_cur_offset = 0;
+    s->sector_next_status = 0;
+
+    sector_num = 0;
+    allocated_done = 0;
+
+    while (sector_num < s->total_sectors) {
+        n = convert_iteration_sectors(s, sector_num);
+        if (n < 0) {
+            ret = n;
+            goto fail;
+        }
+        if (s->status == BLK_DATA || (!s->min_sparse && s->status == BLK_ZERO))
+        {
+            allocated_done += n;
+            qemu_progress_print(100.0 * allocated_done / s->allocated_sectors,
+                                0);
+        }
+
+        if (s->status == BLK_DATA) {
+            ret = convert_read(s, sector_num, n, buf);
+            if (ret < 0) {
+                error_report("error while reading sector %" PRId64
+                             ": %s", sector_num, strerror(-ret));
+                goto fail;
+            }
+        } else if (!s->min_sparse && s->status == BLK_ZERO) {
+            n = MIN(n, s->buf_sectors);
+            memset(buf, 0, n * BDRV_SECTOR_SIZE);
+            s->status = BLK_DATA;
+        }
+
+        ret = convert_write(s, sector_num, n, buf);
+        if (ret < 0) {
+            error_report("error while writing sector %" PRId64
+                         ": %s", sector_num, strerror(-ret));
+            goto fail;
+        }
+
+        sector_num += n;
+    }
+
+    if (s->compressed) {
+        /* signal EOF to align */
+        ret = blk_write_compressed(s->target, 0, NULL, 0);
+        if (ret < 0) {
+            goto fail;
+        }
+    }
+
+    ret = 0;
+fail:
+    qemu_vfree(buf);
     return ret;
 }
 
 static int img_convert(int argc, char **argv)
 {
-    int c, n, n1, bs_n, bs_i, compress, cluster_sectors, skip_create;
+    int c, bs_n, bs_i, compress, cluster_sectors, skip_create;
     int64_t ret = 0;
     int progress = 0, flags, src_flags;
+    bool writethrough, src_writethrough;
     const char *fmt, *out_fmt, *cache, *src_cache, *out_baseimg, *out_filename;
     BlockDriver *drv, *proto_drv;
     BlockBackend **blk = NULL, *out_blk = NULL;
     BlockDriverState **bs = NULL, *out_bs = NULL;
-    int64_t total_sectors, nb_sectors, sector_num, bs_offset;
+    int64_t total_sectors;
     int64_t *bs_sectors = NULL;
-    uint8_t * buf = NULL;
     size_t bufsectors = IO_BUF_SIZE / BDRV_SECTOR_SIZE;
-    const uint8_t *buf1;
     BlockDriverInfo bdi;
     QemuOpts *opts = NULL;
     QemuOptsList *create_opts = NULL;
@@ -1330,6 +1762,8 @@
     bool quiet = false;
     Error *local_err = NULL;
     QemuOpts *sn_opts = NULL;
+    ImgConvertState state;
+    bool image_opts = false;
 
     fmt = NULL;
     out_fmt = "raw";
@@ -1339,7 +1773,14 @@
     compress = 0;
     skip_create = 0;
     for(;;) {
-        c = getopt(argc, argv, "hf:O:B:ce6o:s:l:S:pt:T:qn");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hf:O:B:ce6o:s:l:S:pt:T:qn",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -1389,7 +1830,8 @@
             break;
         case 'l':
             if (strstart(optarg, SNAPSHOT_OPT_BASE, NULL)) {
-                sn_opts = qemu_opts_parse(&internal_snapshot_opts, optarg, 0);
+                sn_opts = qemu_opts_parse_noisily(&internal_snapshot_opts,
+                                                  optarg, false);
                 if (!sn_opts) {
                     error_report("Failed in parsing snapshot param '%s'",
                                  optarg);
@@ -1404,7 +1846,7 @@
         {
             int64_t sval;
             char *end;
-            sval = strtosz_suffix(optarg, &end, STRTOSZ_DEFSUFFIX_B);
+            sval = qemu_strtosz_suffix(optarg, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
             if (sval < 0 || *end) {
                 error_report("Invalid minimum zero buffer size for sparse output specified");
                 ret = -1;
@@ -1429,16 +1871,31 @@
         case 'n':
             skip_create = 1;
             break;
+        case OPTION_OBJECT:
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                goto fail_getopt;
+            }
+            break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        goto fail_getopt;
+    }
+
     /* Initialize before goto out */
     if (quiet) {
         progress = 0;
     }
     qemu_progress_init(progress, 1.0);
 
-
     bs_n = argc - optind - 1;
     out_filename = bs_n >= 1 ? argv[argc - 1] : NULL;
 
@@ -1459,8 +1916,8 @@
         goto out;
     }
 
-    src_flags = BDRV_O_FLAGS;
-    ret = bdrv_parse_cache_flags(src_cache, &src_flags);
+    src_flags = 0;
+    ret = bdrv_parse_cache_mode(src_cache, &src_flags, &src_writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", src_cache);
         goto out;
@@ -1474,17 +1931,14 @@
 
     total_sectors = 0;
     for (bs_i = 0; bs_i < bs_n; bs_i++) {
-        char *id = bs_n > 1 ? g_strdup_printf("source_%d", bs_i)
-                            : g_strdup("source");
-        blk[bs_i] = img_open(id, argv[optind + bs_i], fmt, src_flags,
-                             true, quiet);
-        g_free(id);
+        blk[bs_i] = img_open(image_opts, argv[optind + bs_i],
+                             fmt, src_flags, src_writethrough, quiet);
         if (!blk[bs_i]) {
             ret = -1;
             goto out;
         }
         bs[bs_i] = blk_bs(blk[bs_i]);
-        bs_sectors[bs_i] = bdrv_nb_sectors(bs[bs_i]);
+        bs_sectors[bs_i] = blk_nb_sectors(blk[bs_i]);
         if (bs_sectors[bs_i] < 0) {
             error_report("Could not get size of %s: %s",
                          argv[optind + bs_i], strerror(-bs_sectors[bs_i]));
@@ -1509,9 +1963,7 @@
         bdrv_snapshot_load_tmp_by_id_or_name(bs[0], snapshot_name, &local_err);
     }
     if (local_err) {
-        error_report("Failed to load snapshot: %s",
-                     error_get_pretty(local_err));
-        error_free(local_err);
+        error_reportf_err(local_err, "Failed to load snapshot: ");
         ret = -1;
         goto out;
     }
@@ -1524,27 +1976,47 @@
         goto out;
     }
 
-    proto_drv = bdrv_find_protocol(out_filename, true);
+    proto_drv = bdrv_find_protocol(out_filename, true, &local_err);
     if (!proto_drv) {
-        error_report("Unknown protocol '%s'", out_filename);
+        error_report_err(local_err);
         ret = -1;
         goto out;
     }
 
-    create_opts = qemu_opts_append(create_opts, drv->create_opts);
-    create_opts = qemu_opts_append(create_opts, proto_drv->create_opts);
+    if (!skip_create) {
+        if (!drv->create_opts) {
+            error_report("Format driver '%s' does not support image creation",
+                         drv->format_name);
+            ret = -1;
+            goto out;
+        }
 
-    opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
-    if (options && qemu_opts_do_parse(opts, options, NULL)) {
-        error_report("Invalid options for file format '%s'", out_fmt);
-        ret = -1;
-        goto out;
-    }
+        if (!proto_drv->create_opts) {
+            error_report("Protocol driver '%s' does not support image creation",
+                         proto_drv->format_name);
+            ret = -1;
+            goto out;
+        }
 
-    qemu_opt_set_number(opts, BLOCK_OPT_SIZE, total_sectors * 512);
-    ret = add_old_style_options(out_fmt, opts, out_baseimg, NULL);
-    if (ret < 0) {
-        goto out;
+        create_opts = qemu_opts_append(create_opts, drv->create_opts);
+        create_opts = qemu_opts_append(create_opts, proto_drv->create_opts);
+
+        opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
+        if (options) {
+            qemu_opts_do_parse(opts, options, NULL, &local_err);
+            if (local_err) {
+                error_report_err(local_err);
+                ret = -1;
+                goto out;
+            }
+        }
+
+        qemu_opt_set_number(opts, BLOCK_OPT_SIZE, total_sectors * 512,
+                            &error_abort);
+        ret = add_old_style_options(out_fmt, opts, out_baseimg, NULL);
+        if (ret < 0) {
+            goto out;
+        }
     }
 
     /* Get backing file name if -o backing_file was used */
@@ -1587,44 +2059,44 @@
         /* Create the new image */
         ret = bdrv_create(drv, out_filename, opts, &local_err);
         if (ret < 0) {
-            error_report("%s: error while converting %s: %s",
-                         out_filename, out_fmt, error_get_pretty(local_err));
-            error_free(local_err);
+            error_reportf_err(local_err, "%s: error while converting %s: ",
+                              out_filename, out_fmt);
             goto out;
         }
     }
 
     flags = min_sparse ? (BDRV_O_RDWR | BDRV_O_UNMAP) : BDRV_O_RDWR;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         goto out;
     }
 
-    out_blk = img_open("target", out_filename, out_fmt, flags, true, quiet);
+    /* XXX we should allow --image-opts to trigger use of
+     * img_open() here, but then we have trouble with
+     * the bdrv_create() call which takes different params.
+     * Not critical right now, so fix can wait...
+     */
+    out_blk = img_open_file(out_filename, out_fmt, flags, writethrough, quiet);
     if (!out_blk) {
         ret = -1;
         goto out;
     }
     out_bs = blk_bs(out_blk);
 
-    bs_i = 0;
-    bs_offset = 0;
-
-    /* increase bufsectors from the default 4096 (2M) if opt_transfer_length
+    /* increase bufsectors from the default 4096 (2M) if opt_transfer
      * or discard_alignment of the out_bs is greater. Limit to 32768 (16MB)
      * as maximum. */
     bufsectors = MIN(32768,
-                     MAX(bufsectors, MAX(out_bs->bl.opt_transfer_length,
-                                         out_bs->bl.discard_alignment))
-                    );
-
-    buf = qemu_blockalign(out_bs, bufsectors * BDRV_SECTOR_SIZE);
+                     MAX(bufsectors,
+                         MAX(out_bs->bl.opt_transfer >> BDRV_SECTOR_BITS,
+                             out_bs->bl.pdiscard_alignment >>
+                             BDRV_SECTOR_BITS)));
 
     if (skip_create) {
-        int64_t output_sectors = bdrv_nb_sectors(out_bs);
+        int64_t output_sectors = blk_nb_sectors(out_blk);
         if (output_sectors < 0) {
-            error_report("unable to get output image length: %s\n",
+            error_report("unable to get output image length: %s",
                          strerror(-output_sectors));
             ret = -1;
             goto out;
@@ -1647,203 +2119,20 @@
         cluster_sectors = bdi.cluster_size / BDRV_SECTOR_SIZE;
     }
 
-    if (compress) {
-        if (cluster_sectors <= 0 || cluster_sectors > bufsectors) {
-            error_report("invalid cluster size");
-            ret = -1;
-            goto out;
-        }
-        sector_num = 0;
+    state = (ImgConvertState) {
+        .src                = blk,
+        .src_sectors        = bs_sectors,
+        .src_num            = bs_n,
+        .total_sectors      = total_sectors,
+        .target             = out_blk,
+        .compressed         = compress,
+        .target_has_backing = (bool) out_baseimg,
+        .min_sparse         = min_sparse,
+        .cluster_sectors    = cluster_sectors,
+        .buf_sectors        = bufsectors,
+    };
+    ret = convert_do_copy(&state);
 
-        nb_sectors = total_sectors;
-
-        for(;;) {
-            int64_t bs_num;
-            int remainder;
-            uint8_t *buf2;
-
-            nb_sectors = total_sectors - sector_num;
-            if (nb_sectors <= 0)
-                break;
-            if (nb_sectors >= cluster_sectors)
-                n = cluster_sectors;
-            else
-                n = nb_sectors;
-
-            bs_num = sector_num - bs_offset;
-            assert (bs_num >= 0);
-            remainder = n;
-            buf2 = buf;
-            while (remainder > 0) {
-                int nlow;
-                while (bs_num == bs_sectors[bs_i]) {
-                    bs_offset += bs_sectors[bs_i];
-                    bs_i++;
-                    assert (bs_i < bs_n);
-                    bs_num = 0;
-                    /* printf("changing part: sector_num=%" PRId64 ", "
-                       "bs_i=%d, bs_offset=%" PRId64 ", bs_sectors=%" PRId64
-                       "\n", sector_num, bs_i, bs_offset, bs_sectors[bs_i]); */
-                }
-                assert (bs_num < bs_sectors[bs_i]);
-
-                nlow = remainder > bs_sectors[bs_i] - bs_num
-                    ? bs_sectors[bs_i] - bs_num : remainder;
-
-                ret = bdrv_read(bs[bs_i], bs_num, buf2, nlow);
-                if (ret < 0) {
-                    error_report("error while reading sector %" PRId64 ": %s",
-                                 bs_num, strerror(-ret));
-                    goto out;
-                }
-
-                buf2 += nlow * 512;
-                bs_num += nlow;
-
-                remainder -= nlow;
-            }
-            assert (remainder == 0);
-
-            if (!buffer_is_zero(buf, n * BDRV_SECTOR_SIZE)) {
-                ret = bdrv_write_compressed(out_bs, sector_num, buf, n);
-                if (ret != 0) {
-                    error_report("error while compressing sector %" PRId64
-                                 ": %s", sector_num, strerror(-ret));
-                    goto out;
-                }
-            }
-            sector_num += n;
-            qemu_progress_print(100.0 * sector_num / total_sectors, 0);
-        }
-        /* signal EOF to align */
-        bdrv_write_compressed(out_bs, 0, NULL, 0);
-    } else {
-        int64_t sectors_to_read, sectors_read, sector_num_next_status;
-        bool count_allocated_sectors;
-        int has_zero_init = min_sparse ? bdrv_has_zero_init(out_bs) : 0;
-
-        if (!has_zero_init && bdrv_can_write_zeroes_with_unmap(out_bs)) {
-            ret = bdrv_make_zero(out_bs, BDRV_REQ_MAY_UNMAP);
-            if (ret < 0) {
-                goto out;
-            }
-            has_zero_init = 1;
-        }
-
-        sectors_to_read = total_sectors;
-        count_allocated_sectors = progress && (out_baseimg || has_zero_init);
-restart:
-        sector_num = 0; // total number of sectors converted so far
-        sectors_read = 0;
-        sector_num_next_status = 0;
-
-        for(;;) {
-            nb_sectors = total_sectors - sector_num;
-            if (nb_sectors <= 0) {
-                if (count_allocated_sectors) {
-                    sectors_to_read = sectors_read;
-                    count_allocated_sectors = false;
-                    goto restart;
-                }
-                ret = 0;
-                break;
-            }
-
-            while (sector_num - bs_offset >= bs_sectors[bs_i]) {
-                bs_offset += bs_sectors[bs_i];
-                bs_i ++;
-                assert (bs_i < bs_n);
-                /* printf("changing part: sector_num=%" PRId64 ", bs_i=%d, "
-                  "bs_offset=%" PRId64 ", bs_sectors=%" PRId64 "\n",
-                   sector_num, bs_i, bs_offset, bs_sectors[bs_i]); */
-            }
-
-            if ((out_baseimg || has_zero_init) &&
-                sector_num >= sector_num_next_status) {
-                n = nb_sectors > INT_MAX ? INT_MAX : nb_sectors;
-                ret = bdrv_get_block_status(bs[bs_i], sector_num - bs_offset,
-                                            n, &n1);
-                if (ret < 0) {
-                    error_report("error while reading block status of sector %"
-                                 PRId64 ": %s", sector_num - bs_offset,
-                                 strerror(-ret));
-                    goto out;
-                }
-                /* If the output image is zero initialized, we are not working
-                 * on a shared base and the input is zero we can skip the next
-                 * n1 sectors */
-                if (has_zero_init && !out_baseimg && (ret & BDRV_BLOCK_ZERO)) {
-                    sector_num += n1;
-                    continue;
-                }
-                /* If the output image is being created as a copy on write
-                 * image, assume that sectors which are unallocated in the
-                 * input image are present in both the output's and input's
-                 * base images (no need to copy them). */
-                if (out_baseimg) {
-                    if (!(ret & BDRV_BLOCK_DATA)) {
-                        sector_num += n1;
-                        continue;
-                    }
-                    /* The next 'n1' sectors are allocated in the input image.
-                     * Copy only those as they may be followed by unallocated
-                     * sectors. */
-                    nb_sectors = n1;
-                }
-                /* avoid redundant callouts to get_block_status */
-                sector_num_next_status = sector_num + n1;
-            }
-
-            n = MIN(nb_sectors, bufsectors);
-
-            /* round down request length to an aligned sector, but
-             * do not bother doing this on short requests. They happen
-             * when we found an all-zero area, and the next sector to
-             * write will not be sector_num + n. */
-            if (cluster_sectors > 0 && n >= cluster_sectors) {
-                int64_t next_aligned_sector = (sector_num + n);
-                next_aligned_sector -= next_aligned_sector % cluster_sectors;
-                if (sector_num + n > next_aligned_sector) {
-                    n = next_aligned_sector - sector_num;
-                }
-            }
-
-            n = MIN(n, bs_sectors[bs_i] - (sector_num - bs_offset));
-
-            sectors_read += n;
-            if (count_allocated_sectors) {
-                sector_num += n;
-                continue;
-            }
-
-            n1 = n;
-            ret = bdrv_read(bs[bs_i], sector_num - bs_offset, buf, n);
-            if (ret < 0) {
-                error_report("error while reading sector %" PRId64 ": %s",
-                             sector_num - bs_offset, strerror(-ret));
-                goto out;
-            }
-            /* NOTE: at the same time we convert, we do not write zero
-               sectors to have a chance to compress the image. Ideally, we
-               should add a specific call to have the info to go faster */
-            buf1 = buf;
-            while (n > 0) {
-                if (!has_zero_init ||
-                    is_allocated_sectors_min(buf1, n, &n1, min_sparse)) {
-                    ret = bdrv_write(out_bs, sector_num, buf1, n1);
-                    if (ret < 0) {
-                        error_report("error while writing sector %" PRId64
-                                     ": %s", sector_num, strerror(-ret));
-                        goto out;
-                    }
-                }
-                sector_num += n1;
-                n -= n1;
-                buf1 += n1 * 512;
-            }
-            qemu_progress_print(100.0 * sectors_read / sectors_to_read, 0);
-        }
-    }
 out:
     if (!ret) {
         qemu_progress_print(100, 0);
@@ -1851,7 +2140,6 @@
     qemu_progress_end();
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
-    qemu_vfree(buf);
     qemu_opts_del(sn_opts);
     blk_unref(out_blk);
     g_free(bs);
@@ -1893,35 +2181,33 @@
 
 static void dump_json_image_info_list(ImageInfoList *list)
 {
-    Error *local_err = NULL;
     QString *str;
-    QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj;
-    visit_type_ImageInfoList(qmp_output_get_visitor(ov),
-                             &list, NULL, &local_err);
-    obj = qmp_output_get_qobject(ov);
+    Visitor *v = qmp_output_visitor_new(&obj);
+
+    visit_type_ImageInfoList(v, NULL, &list, &error_abort);
+    visit_complete(v, &obj);
     str = qobject_to_json_pretty(obj);
     assert(str != NULL);
     printf("%s\n", qstring_get_str(str));
     qobject_decref(obj);
-    qmp_output_visitor_cleanup(ov);
+    visit_free(v);
     QDECREF(str);
 }
 
 static void dump_json_image_info(ImageInfo *info)
 {
-    Error *local_err = NULL;
     QString *str;
-    QmpOutputVisitor *ov = qmp_output_visitor_new();
     QObject *obj;
-    visit_type_ImageInfo(qmp_output_get_visitor(ov),
-                         &info, NULL, &local_err);
-    obj = qmp_output_get_qobject(ov);
+    Visitor *v = qmp_output_visitor_new(&obj);
+
+    visit_type_ImageInfo(v, NULL, &info, &error_abort);
+    visit_complete(v, &obj);
     str = qobject_to_json_pretty(obj);
     assert(str != NULL);
     printf("%s\n", qstring_get_str(str));
     qobject_decref(obj);
-    qmp_output_visitor_cleanup(ov);
+    visit_free(v);
     QDECREF(str);
 }
 
@@ -1957,7 +2243,8 @@
  * image file.  If there was an error a message will have been printed to
  * stderr.
  */
-static ImageInfoList *collect_image_info_list(const char *filename,
+static ImageInfoList *collect_image_info_list(bool image_opts,
+                                              const char *filename,
                                               const char *fmt,
                                               bool chain)
 {
@@ -1981,8 +2268,8 @@
         }
         g_hash_table_insert(filenames, (gpointer)filename, NULL);
 
-        blk = img_open("image", filename, fmt,
-                       BDRV_O_FLAGS | BDRV_O_NO_BACKING, false, false);
+        blk = img_open(image_opts, filename, fmt,
+                       BDRV_O_NO_BACKING | BDRV_O_NO_IO, false, false);
         if (!blk) {
             goto err;
         }
@@ -1990,8 +2277,7 @@
 
         bdrv_query_image_info(bs, &info, &err);
         if (err) {
-            error_report("%s", error_get_pretty(err));
-            error_free(err);
+            error_report_err(err);
             blk_unref(blk);
             goto err;
         }
@@ -2008,7 +2294,10 @@
             if (info->has_full_backing_filename) {
                 filename = info->full_backing_filename;
             } else if (info->has_backing_filename) {
-                filename = info->backing_filename;
+                error_report("Could not determine absolute backing filename,"
+                             " but backing filename '%s' present",
+                             info->backing_filename);
+                goto err;
             }
             if (info->has_backing_filename_format) {
                 fmt = info->backing_filename_format;
@@ -2031,6 +2320,7 @@
     bool chain = false;
     const char *filename, *fmt, *output;
     ImageInfoList *list;
+    bool image_opts = false;
 
     fmt = NULL;
     output = NULL;
@@ -2041,6 +2331,8 @@
             {"format", required_argument, 0, 'f'},
             {"output", required_argument, 0, OPTION_OUTPUT},
             {"backing-chain", no_argument, 0, OPTION_BACKING_CHAIN},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
             {0, 0, 0, 0}
         };
         c = getopt_long(argc, argv, "f:h",
@@ -2062,6 +2354,17 @@
         case OPTION_BACKING_CHAIN:
             chain = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -2078,7 +2381,13 @@
         return 1;
     }
 
-    list = collect_image_info_list(filename, fmt, chain);
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
+    list = collect_image_info_list(image_opts, filename, fmt, chain);
     if (!list) {
         return 1;
     }
@@ -2100,47 +2409,37 @@
     return 0;
 }
 
-
-typedef struct MapEntry {
-    int flags;
-    int depth;
-    int64_t start;
-    int64_t length;
-    int64_t offset;
-    BlockDriverState *bs;
-} MapEntry;
-
 static void dump_map_entry(OutputFormat output_format, MapEntry *e,
                            MapEntry *next)
 {
     switch (output_format) {
     case OFORMAT_HUMAN:
-        if ((e->flags & BDRV_BLOCK_DATA) &&
-            !(e->flags & BDRV_BLOCK_OFFSET_VALID)) {
+        if (e->data && !e->has_offset) {
             error_report("File contains external, encrypted or compressed clusters.");
             exit(1);
         }
-        if ((e->flags & (BDRV_BLOCK_DATA|BDRV_BLOCK_ZERO)) == BDRV_BLOCK_DATA) {
+        if (e->data && !e->zero) {
             printf("%#-16"PRIx64"%#-16"PRIx64"%#-16"PRIx64"%s\n",
-                   e->start, e->length, e->offset, e->bs->filename);
+                   e->start, e->length,
+                   e->has_offset ? e->offset : 0,
+                   e->has_filename ? e->filename : "");
         }
         /* This format ignores the distinction between 0, ZERO and ZERO|DATA.
          * Modify the flags here to allow more coalescing.
          */
-        if (next &&
-            (next->flags & (BDRV_BLOCK_DATA|BDRV_BLOCK_ZERO)) != BDRV_BLOCK_DATA) {
-            next->flags &= ~BDRV_BLOCK_DATA;
-            next->flags |= BDRV_BLOCK_ZERO;
+        if (next && (!next->data || next->zero)) {
+            next->data = false;
+            next->zero = true;
         }
         break;
     case OFORMAT_JSON:
-        printf("%s{ \"start\": %"PRId64", \"length\": %"PRId64", \"depth\": %d,"
-               " \"zero\": %s, \"data\": %s",
+        printf("%s{ \"start\": %"PRId64", \"length\": %"PRId64","
+               " \"depth\": %"PRId64", \"zero\": %s, \"data\": %s",
                (e->start == 0 ? "[" : ",\n"),
                e->start, e->length, e->depth,
-               (e->flags & BDRV_BLOCK_ZERO) ? "true" : "false",
-               (e->flags & BDRV_BLOCK_DATA) ? "true" : "false");
-        if (e->flags & BDRV_BLOCK_OFFSET_VALID) {
+               e->zero ? "true" : "false",
+               e->data ? "true" : "false");
+        if (e->has_offset) {
             printf(", \"offset\": %"PRId64"", e->offset);
         }
         putchar('}');
@@ -2157,6 +2456,8 @@
 {
     int64_t ret;
     int depth;
+    BlockDriverState *file;
+    bool has_offset;
 
     /* As an optimization, we could cache the current range of unallocated
      * clusters in each file of the chain, and avoid querying the same
@@ -2165,7 +2466,8 @@
 
     depth = 0;
     for (;;) {
-        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &nb_sectors);
+        ret = bdrv_get_block_status(bs, sector_num, nb_sectors, &nb_sectors,
+                                    &file);
         if (ret < 0) {
             return ret;
         }
@@ -2173,7 +2475,7 @@
         if (ret & (BDRV_BLOCK_ZERO|BDRV_BLOCK_DATA)) {
             break;
         }
-        bs = bs->backing_hd;
+        bs = backing_bs(bs);
         if (bs == NULL) {
             ret = 0;
             break;
@@ -2182,15 +2484,44 @@
         depth++;
     }
 
-    e->start = sector_num * BDRV_SECTOR_SIZE;
-    e->length = nb_sectors * BDRV_SECTOR_SIZE;
-    e->flags = ret & ~BDRV_BLOCK_OFFSET_MASK;
-    e->offset = ret & BDRV_BLOCK_OFFSET_MASK;
-    e->depth = depth;
-    e->bs = bs;
+    has_offset = !!(ret & BDRV_BLOCK_OFFSET_VALID);
+
+    *e = (MapEntry) {
+        .start = sector_num * BDRV_SECTOR_SIZE,
+        .length = nb_sectors * BDRV_SECTOR_SIZE,
+        .data = !!(ret & BDRV_BLOCK_DATA),
+        .zero = !!(ret & BDRV_BLOCK_ZERO),
+        .offset = ret & BDRV_BLOCK_OFFSET_MASK,
+        .has_offset = has_offset,
+        .depth = depth,
+        .has_filename = file && has_offset,
+        .filename = file && has_offset ? file->filename : NULL,
+    };
+
     return 0;
 }
 
+static inline bool entry_mergeable(const MapEntry *curr, const MapEntry *next)
+{
+    if (curr->length == 0) {
+        return false;
+    }
+    if (curr->zero != next->zero ||
+        curr->data != next->data ||
+        curr->depth != next->depth ||
+        curr->has_filename != next->has_filename ||
+        curr->has_offset != next->has_offset) {
+        return false;
+    }
+    if (curr->has_filename && strcmp(curr->filename, next->filename)) {
+        return false;
+    }
+    if (curr->has_offset && curr->offset + curr->length != next->offset) {
+        return false;
+    }
+    return true;
+}
+
 static int img_map(int argc, char **argv)
 {
     int c;
@@ -2201,6 +2532,7 @@
     int64_t length;
     MapEntry curr = { .length = 0 }, next;
     int ret = 0;
+    bool image_opts = false;
 
     fmt = NULL;
     output = NULL;
@@ -2210,6 +2542,8 @@
             {"help", no_argument, 0, 'h'},
             {"format", required_argument, 0, 'f'},
             {"output", required_argument, 0, OPTION_OUTPUT},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
             {0, 0, 0, 0}
         };
         c = getopt_long(argc, argv, "f:h",
@@ -2228,6 +2562,17 @@
         case OPTION_OUTPUT:
             output = optarg;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -2244,7 +2589,13 @@
         return 1;
     }
 
-    blk = img_open("image", filename, fmt, BDRV_O_FLAGS, true, false);
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
+    blk = img_open(image_opts, filename, fmt, 0, false, false);
     if (!blk) {
         return 1;
     }
@@ -2254,7 +2605,7 @@
         printf("%-16s%-16s%-16s%s\n", "Offset", "Length", "Mapped to", "File");
     }
 
-    length = bdrv_getlength(bs);
+    length = blk_getlength(blk);
     while (curr.start + curr.length < length) {
         int64_t nsectors_left;
         int64_t sector_num;
@@ -2272,10 +2623,7 @@
             goto out;
         }
 
-        if (curr.length != 0 && curr.flags == next.flags &&
-            curr.depth == next.depth &&
-            ((curr.flags & BDRV_BLOCK_OFFSET_VALID) == 0 ||
-             curr.offset + curr.length == next.offset)) {
+        if (entry_mergeable(&curr, &next)) {
             curr.length += next.length;
             continue;
         }
@@ -2309,11 +2657,19 @@
     qemu_timeval tv;
     bool quiet = false;
     Error *err = NULL;
+    bool image_opts = false;
 
-    bdrv_oflags = BDRV_O_FLAGS | BDRV_O_RDWR;
+    bdrv_oflags = BDRV_O_RDWR;
     /* Parse commandline parameters */
     for(;;) {
-        c = getopt(argc, argv, "la:c:d:hq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "la:c:d:hq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2357,6 +2713,17 @@
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -2365,8 +2732,14 @@
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     /* Open the image */
-    blk = img_open("image", filename, NULL, bdrv_oflags, true, quiet);
+    blk = img_open(image_opts, filename, NULL, bdrv_oflags, false, quiet);
     if (!blk) {
         return 1;
     }
@@ -2404,9 +2777,8 @@
     case SNAPSHOT_DELETE:
         bdrv_snapshot_delete_by_id_or_name(bs, snapshot_name, &err);
         if (err) {
-            error_report("Could not delete snapshot '%s': (%s)",
-                         snapshot_name, error_get_pretty(err));
-            error_free(err);
+            error_reportf_err(err, "Could not delete snapshot '%s': ",
+                              snapshot_name);
             ret = 1;
         }
         break;
@@ -2423,15 +2795,18 @@
 static int img_rebase(int argc, char **argv)
 {
     BlockBackend *blk = NULL, *blk_old_backing = NULL, *blk_new_backing = NULL;
-    BlockDriverState *bs = NULL, *bs_old_backing = NULL, *bs_new_backing = NULL;
-    BlockDriver *old_backing_drv, *new_backing_drv;
+    uint8_t *buf_old = NULL;
+    uint8_t *buf_new = NULL;
+    BlockDriverState *bs = NULL;
     char *filename;
     const char *fmt, *cache, *src_cache, *out_basefmt, *out_baseimg;
     int c, flags, src_flags, ret;
+    bool writethrough, src_writethrough;
     int unsafe = 0;
     int progress = 0;
     bool quiet = false;
     Error *local_err = NULL;
+    bool image_opts = false;
 
     /* Parse commandline parameters */
     fmt = NULL;
@@ -2440,7 +2815,14 @@
     out_baseimg = NULL;
     out_basefmt = NULL;
     for(;;) {
-        c = getopt(argc, argv, "hf:F:b:upt:T:q");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hf:F:b:upt:T:q",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2473,6 +2855,17 @@
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
 
@@ -2488,52 +2881,48 @@
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     qemu_progress_init(progress, 2.0);
     qemu_progress_print(0, 100);
 
     flags = BDRV_O_RDWR | (unsafe ? BDRV_O_NO_BACKING : 0);
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         goto out;
     }
 
-    src_flags = BDRV_O_FLAGS;
-    ret = bdrv_parse_cache_flags(src_cache, &src_flags);
+    src_flags = 0;
+    ret = bdrv_parse_cache_mode(src_cache, &src_flags, &src_writethrough);
     if (ret < 0) {
         error_report("Invalid source cache option: %s", src_cache);
         goto out;
     }
 
+    /* The source files are opened read-only, don't care about WCE */
+    assert((src_flags & BDRV_O_RDWR) == 0);
+    (void) src_writethrough;
+
     /*
      * Open the images.
      *
      * Ignore the old backing file for unsafe rebase in case we want to correct
      * the reference to a renamed or moved backing file.
      */
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         ret = -1;
         goto out;
     }
     bs = blk_bs(blk);
 
-    /* Find the right drivers for the backing files */
-    old_backing_drv = NULL;
-    new_backing_drv = NULL;
-
-    if (!unsafe && bs->backing_format[0] != '\0') {
-        old_backing_drv = bdrv_find_format(bs->backing_format);
-        if (old_backing_drv == NULL) {
-            error_report("Invalid format name: '%s'", bs->backing_format);
-            ret = -1;
-            goto out;
-        }
-    }
-
     if (out_basefmt != NULL) {
-        new_backing_drv = bdrv_find_format(out_basefmt);
-        if (new_backing_drv == NULL) {
+        if (bdrv_find_format(out_basefmt) == NULL) {
             error_report("Invalid format name: '%s'", out_basefmt);
             ret = -1;
             goto out;
@@ -2542,28 +2931,38 @@
 
     /* For safe rebasing we need to compare old and new backing file */
     if (!unsafe) {
-        char backing_name[1024];
+        char backing_name[PATH_MAX];
+        QDict *options = NULL;
 
-        blk_old_backing = blk_new_with_bs("old_backing", &error_abort);
-        bs_old_backing = blk_bs(blk_old_backing);
+        if (bs->backing_format[0] != '\0') {
+            options = qdict_new();
+            qdict_put(options, "driver", qstring_from_str(bs->backing_format));
+        }
+
         bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
-        ret = bdrv_open(&bs_old_backing, backing_name, NULL, NULL, src_flags,
-                        old_backing_drv, &local_err);
-        if (ret) {
-            error_report("Could not open old backing file '%s': %s",
-                         backing_name, error_get_pretty(local_err));
-            error_free(local_err);
+        blk_old_backing = blk_new_open(backing_name, NULL,
+                                       options, src_flags, &local_err);
+        if (!blk_old_backing) {
+            error_reportf_err(local_err,
+                              "Could not open old backing file '%s': ",
+                              backing_name);
             goto out;
         }
+
         if (out_baseimg[0]) {
-            blk_new_backing = blk_new_with_bs("new_backing", &error_abort);
-            bs_new_backing = blk_bs(blk_new_backing);
-            ret = bdrv_open(&bs_new_backing, out_baseimg, NULL, NULL, src_flags,
-                            new_backing_drv, &local_err);
-            if (ret) {
-                error_report("Could not open new backing file '%s': %s",
-                             out_baseimg, error_get_pretty(local_err));
-                error_free(local_err);
+            if (out_basefmt) {
+                options = qdict_new();
+                qdict_put(options, "driver", qstring_from_str(out_basefmt));
+            } else {
+                options = NULL;
+            }
+
+            blk_new_backing = blk_new_open(out_baseimg, NULL,
+                                           options, src_flags, &local_err);
+            if (!blk_new_backing) {
+                error_reportf_err(local_err,
+                                  "Could not open new backing file '%s': ",
+                                  out_baseimg);
                 goto out;
             }
         }
@@ -2584,23 +2983,21 @@
         int64_t new_backing_num_sectors = 0;
         uint64_t sector;
         int n;
-        uint8_t * buf_old;
-        uint8_t * buf_new;
         float local_progress = 0;
 
-        buf_old = qemu_blockalign(bs, IO_BUF_SIZE);
-        buf_new = qemu_blockalign(bs, IO_BUF_SIZE);
+        buf_old = blk_blockalign(blk, IO_BUF_SIZE);
+        buf_new = blk_blockalign(blk, IO_BUF_SIZE);
 
-        num_sectors = bdrv_nb_sectors(bs);
+        num_sectors = blk_nb_sectors(blk);
         if (num_sectors < 0) {
             error_report("Could not get size of '%s': %s",
                          filename, strerror(-num_sectors));
             ret = -1;
             goto out;
         }
-        old_backing_num_sectors = bdrv_nb_sectors(bs_old_backing);
+        old_backing_num_sectors = blk_nb_sectors(blk_old_backing);
         if (old_backing_num_sectors < 0) {
-            char backing_name[1024];
+            char backing_name[PATH_MAX];
 
             bdrv_get_backing_filename(bs, backing_name, sizeof(backing_name));
             error_report("Could not get size of '%s': %s",
@@ -2608,8 +3005,8 @@
             ret = -1;
             goto out;
         }
-        if (bs_new_backing) {
-            new_backing_num_sectors = bdrv_nb_sectors(bs_new_backing);
+        if (blk_new_backing) {
+            new_backing_num_sectors = blk_nb_sectors(blk_new_backing);
             if (new_backing_num_sectors < 0) {
                 error_report("Could not get size of '%s': %s",
                              out_baseimg, strerror(-new_backing_num_sectors));
@@ -2654,21 +3051,23 @@
                     n = old_backing_num_sectors - sector;
                 }
 
-                ret = bdrv_read(bs_old_backing, sector, buf_old, n);
+                ret = blk_pread(blk_old_backing, sector << BDRV_SECTOR_BITS,
+                                buf_old, n << BDRV_SECTOR_BITS);
                 if (ret < 0) {
                     error_report("error while reading from old backing file");
                     goto out;
                 }
             }
 
-            if (sector >= new_backing_num_sectors || !bs_new_backing) {
+            if (sector >= new_backing_num_sectors || !blk_new_backing) {
                 memset(buf_new, 0, n * BDRV_SECTOR_SIZE);
             } else {
                 if (sector + n > new_backing_num_sectors) {
                     n = new_backing_num_sectors - sector;
                 }
 
-                ret = bdrv_read(bs_new_backing, sector, buf_new, n);
+                ret = blk_pread(blk_new_backing, sector << BDRV_SECTOR_BITS,
+                                buf_new, n << BDRV_SECTOR_BITS);
                 if (ret < 0) {
                     error_report("error while reading from new backing file");
                     goto out;
@@ -2684,8 +3083,10 @@
                 if (compare_sectors(buf_old + written * 512,
                     buf_new + written * 512, n - written, &pnum))
                 {
-                    ret = bdrv_write(bs, sector + written,
-                        buf_old + written * 512, pnum);
+                    ret = blk_pwrite(blk,
+                                     (sector + written) << BDRV_SECTOR_BITS,
+                                     buf_old + written * 512,
+                                     pnum << BDRV_SECTOR_BITS, 0);
                     if (ret < 0) {
                         error_report("Error while writing to COW image: %s",
                             strerror(-ret));
@@ -2697,9 +3098,6 @@
             }
             qemu_progress_print(local_progress, 100);
         }
-
-        qemu_vfree(buf_old);
-        qemu_vfree(buf_new);
     }
 
     /*
@@ -2735,6 +3133,8 @@
         blk_unref(blk_old_backing);
         blk_unref(blk_new_backing);
     }
+    qemu_vfree(buf_old);
+    qemu_vfree(buf_new);
 
     blk_unref(blk);
     if (ret) {
@@ -2745,13 +3145,14 @@
 
 static int img_resize(int argc, char **argv)
 {
+    Error *err = NULL;
     int c, ret, relative;
     const char *filename, *fmt, *size;
     int64_t n, total_size;
     bool quiet = false;
     BlockBackend *blk = NULL;
-    BlockDriverState *bs = NULL;
     QemuOpts *param;
+
     static QemuOptsList resize_options = {
         .name = "resize_options",
         .head = QTAILQ_HEAD_INITIALIZER(resize_options.head),
@@ -2765,6 +3166,7 @@
             }
         },
     };
+    bool image_opts = false;
 
     /* Remove size from argv manually so that negative numbers are not treated
      * as options by getopt. */
@@ -2778,7 +3180,14 @@
     /* Parse getopt arguments */
     fmt = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:hq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "f:hq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2793,6 +3202,17 @@
         case 'q':
             quiet = true;
             break;
+        case OPTION_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                return 1;
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
         }
     }
     if (optind != argc - 1) {
@@ -2800,6 +3220,12 @@
     }
     filename = argv[optind++];
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        return 1;
+    }
+
     /* Choose grow, shrink, or absolute resize mode */
     switch (size[0]) {
     case '+':
@@ -2817,8 +3243,9 @@
 
     /* Parse size */
     param = qemu_opts_create(&resize_options, NULL, 0, &error_abort);
-    if (qemu_opt_set(param, BLOCK_OPT_SIZE, size)) {
-        /* Error message already printed when size parsing fails */
+    qemu_opt_set(param, BLOCK_OPT_SIZE, size, &err);
+    if (err) {
+        error_report_err(err);
         ret = -1;
         qemu_opts_del(param);
         goto out;
@@ -2826,16 +3253,15 @@
     n = qemu_opt_get_size(param, BLOCK_OPT_SIZE, 0);
     qemu_opts_del(param);
 
-    blk = img_open("image", filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR,
-                   true, quiet);
+    blk = img_open(image_opts, filename, fmt,
+                   BDRV_O_RDWR, false, quiet);
     if (!blk) {
         ret = -1;
         goto out;
     }
-    bs = blk_bs(blk);
 
     if (relative) {
-        total_size = bdrv_getlength(bs) + n * relative;
+        total_size = blk_getlength(blk) + n * relative;
     } else {
         total_size = n;
     }
@@ -2845,7 +3271,7 @@
         goto out;
     }
 
-    ret = bdrv_truncate(bs, total_size);
+    ret = blk_truncate(blk, total_size);
     switch (ret) {
     case 0:
         qprintf(quiet, "Image resized.\n");
@@ -2857,7 +3283,7 @@
         error_report("Image is read-only");
         break;
     default:
-        error_report("Error resizing image (%d)", -ret);
+        error_report("Error resizing image: %s", strerror(-ret));
         break;
     }
 out:
@@ -2869,26 +3295,37 @@
 }
 
 static void amend_status_cb(BlockDriverState *bs,
-                            int64_t offset, int64_t total_work_size)
+                            int64_t offset, int64_t total_work_size,
+                            void *opaque)
 {
     qemu_progress_print(100.f * offset / total_work_size, 0);
 }
 
 static int img_amend(int argc, char **argv)
 {
+    Error *err = NULL;
     int c, ret = 0;
     char *options = NULL;
     QemuOptsList *create_opts = NULL;
     QemuOpts *opts = NULL;
     const char *fmt = NULL, *filename, *cache;
     int flags;
+    bool writethrough;
     bool quiet = false, progress = false;
     BlockBackend *blk = NULL;
     BlockDriverState *bs = NULL;
+    bool image_opts = false;
 
     cache = BDRV_DEFAULT_CACHE;
     for (;;) {
-        c = getopt(argc, argv, "ho:f:t:pq");
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"object", required_argument, 0, OPTION_OBJECT},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "ho:f:t:pq",
+                        long_options, NULL);
         if (c == -1) {
             break;
         }
@@ -2902,7 +3339,7 @@
                 if (!is_valid_option_list(optarg)) {
                     error_report("Invalid option list: %s", optarg);
                     ret = -1;
-                    goto out;
+                    goto out_no_progress;
                 }
                 if (!options) {
                     options = g_strdup(optarg);
@@ -2924,6 +3361,17 @@
             case 'q':
                 quiet = true;
                 break;
+            case OPTION_OBJECT:
+                opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                               optarg, true);
+                if (!opts) {
+                    ret = -1;
+                    goto out_no_progress;
+                }
+                break;
+            case OPTION_IMAGE_OPTS:
+                image_opts = true;
+                break;
         }
     }
 
@@ -2931,6 +3379,13 @@
         error_exit("Must specify options (-o)");
     }
 
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        ret = -1;
+        goto out_no_progress;
+    }
+
     if (quiet) {
         progress = false;
     }
@@ -2950,14 +3405,14 @@
         goto out;
     }
 
-    flags = BDRV_O_FLAGS | BDRV_O_RDWR;
-    ret = bdrv_parse_cache_flags(cache, &flags);
+    flags = BDRV_O_RDWR;
+    ret = bdrv_parse_cache_mode(cache, &flags, &writethrough);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
         goto out;
     }
 
-    blk = img_open("image", filename, fmt, flags, true, quiet);
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
     if (!blk) {
         ret = -1;
         goto out;
@@ -2972,17 +3427,27 @@
         goto out;
     }
 
-    create_opts = qemu_opts_append(create_opts, bs->drv->create_opts);
-    opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
-    if (options && qemu_opts_do_parse(opts, options, NULL)) {
-        error_report("Invalid options for file format '%s'", fmt);
+    if (!bs->drv->create_opts) {
+        error_report("Format driver '%s' does not support any options to amend",
+                     fmt);
         ret = -1;
         goto out;
     }
 
+    create_opts = qemu_opts_append(create_opts, bs->drv->create_opts);
+    opts = qemu_opts_create(create_opts, NULL, 0, &error_abort);
+    if (options) {
+        qemu_opts_do_parse(opts, options, NULL, &err);
+        if (err) {
+            error_report_err(err);
+            ret = -1;
+            goto out;
+        }
+    }
+
     /* In case the driver does not call amend_status_cb() */
     qemu_progress_print(0.f, 0);
-    ret = bdrv_amend_options(bs, opts, &amend_status_cb);
+    ret = bdrv_amend_options(bs, opts, &amend_status_cb, NULL);
     qemu_progress_print(100.f, 0);
     if (ret < 0) {
         error_report("Error while amending options: %s", strerror(-ret));
@@ -2992,6 +3457,7 @@
 out:
     qemu_progress_end();
 
+out_no_progress:
     blk_unref(blk);
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
@@ -3003,6 +3469,332 @@
     return 0;
 }
 
+typedef struct BenchData {
+    BlockBackend *blk;
+    uint64_t image_size;
+    bool write;
+    int bufsize;
+    int step;
+    int nrreq;
+    int n;
+    int flush_interval;
+    bool drain_on_flush;
+    uint8_t *buf;
+    QEMUIOVector *qiov;
+
+    int in_flight;
+    bool in_flush;
+    uint64_t offset;
+} BenchData;
+
+static void bench_undrained_flush_cb(void *opaque, int ret)
+{
+    if (ret < 0) {
+        error_report("Failed flush request: %s", strerror(-ret));
+        exit(EXIT_FAILURE);
+    }
+}
+
+static void bench_cb(void *opaque, int ret)
+{
+    BenchData *b = opaque;
+    BlockAIOCB *acb;
+
+    if (ret < 0) {
+        error_report("Failed request: %s", strerror(-ret));
+        exit(EXIT_FAILURE);
+    }
+
+    if (b->in_flush) {
+        /* Just finished a flush with drained queue: Start next requests */
+        assert(b->in_flight == 0);
+        b->in_flush = false;
+    } else if (b->in_flight > 0) {
+        int remaining = b->n - b->in_flight;
+
+        b->n--;
+        b->in_flight--;
+
+        /* Time for flush? Drain queue if requested, then flush */
+        if (b->flush_interval && remaining % b->flush_interval == 0) {
+            if (!b->in_flight || !b->drain_on_flush) {
+                BlockCompletionFunc *cb;
+
+                if (b->drain_on_flush) {
+                    b->in_flush = true;
+                    cb = bench_cb;
+                } else {
+                    cb = bench_undrained_flush_cb;
+                }
+
+                acb = blk_aio_flush(b->blk, cb, b);
+                if (!acb) {
+                    error_report("Failed to issue flush request");
+                    exit(EXIT_FAILURE);
+                }
+            }
+            if (b->drain_on_flush) {
+                return;
+            }
+        }
+    }
+
+    while (b->n > b->in_flight && b->in_flight < b->nrreq) {
+        if (b->write) {
+            acb = blk_aio_pwritev(b->blk, b->offset, b->qiov, 0,
+                                  bench_cb, b);
+        } else {
+            acb = blk_aio_preadv(b->blk, b->offset, b->qiov, 0,
+                                 bench_cb, b);
+        }
+        if (!acb) {
+            error_report("Failed to issue request");
+            exit(EXIT_FAILURE);
+        }
+        b->in_flight++;
+        b->offset += b->step;
+        b->offset %= b->image_size;
+    }
+}
+
+static int img_bench(int argc, char **argv)
+{
+    int c, ret = 0;
+    const char *fmt = NULL, *filename;
+    bool quiet = false;
+    bool image_opts = false;
+    bool is_write = false;
+    int count = 75000;
+    int depth = 64;
+    int64_t offset = 0;
+    size_t bufsize = 4096;
+    int pattern = 0;
+    size_t step = 0;
+    int flush_interval = 0;
+    bool drain_on_flush = true;
+    int64_t image_size;
+    BlockBackend *blk = NULL;
+    BenchData data = {};
+    int flags = 0;
+    bool writethrough = false;
+    struct timeval t1, t2;
+    int i;
+
+    for (;;) {
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"flush-interval", required_argument, 0, OPTION_FLUSH_INTERVAL},
+            {"image-opts", no_argument, 0, OPTION_IMAGE_OPTS},
+            {"pattern", required_argument, 0, OPTION_PATTERN},
+            {"no-drain", no_argument, 0, OPTION_NO_DRAIN},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "hc:d:f:no:qs:S:t:w", long_options, NULL);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+        case 'h':
+        case '?':
+            help();
+            break;
+        case 'c':
+        {
+            char *end;
+            errno = 0;
+            count = strtoul(optarg, &end, 0);
+            if (errno || *end || count > INT_MAX) {
+                error_report("Invalid request count specified");
+                return 1;
+            }
+            break;
+        }
+        case 'd':
+        {
+            char *end;
+            errno = 0;
+            depth = strtoul(optarg, &end, 0);
+            if (errno || *end || depth > INT_MAX) {
+                error_report("Invalid queue depth specified");
+                return 1;
+            }
+            break;
+        }
+        case 'f':
+            fmt = optarg;
+            break;
+        case 'n':
+            flags |= BDRV_O_NATIVE_AIO;
+            break;
+        case 'o':
+        {
+            char *end;
+            errno = 0;
+            offset = qemu_strtosz_suffix(optarg, &end,
+                                         QEMU_STRTOSZ_DEFSUFFIX_B);
+            if (offset < 0|| *end) {
+                error_report("Invalid offset specified");
+                return 1;
+            }
+            break;
+        }
+            break;
+        case 'q':
+            quiet = true;
+            break;
+        case 's':
+        {
+            int64_t sval;
+            char *end;
+
+            sval = qemu_strtosz_suffix(optarg, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
+            if (sval < 0 || sval > INT_MAX || *end) {
+                error_report("Invalid buffer size specified");
+                return 1;
+            }
+
+            bufsize = sval;
+            break;
+        }
+        case 'S':
+        {
+            int64_t sval;
+            char *end;
+
+            sval = qemu_strtosz_suffix(optarg, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
+            if (sval < 0 || sval > INT_MAX || *end) {
+                error_report("Invalid step size specified");
+                return 1;
+            }
+
+            step = sval;
+            break;
+        }
+        case 't':
+            ret = bdrv_parse_cache_mode(optarg, &flags, &writethrough);
+            if (ret < 0) {
+                error_report("Invalid cache mode");
+                ret = -1;
+                goto out;
+            }
+            break;
+        case 'w':
+            flags |= BDRV_O_RDWR;
+            is_write = true;
+            break;
+        case OPTION_PATTERN:
+        {
+            char *end;
+            errno = 0;
+            pattern = strtoul(optarg, &end, 0);
+            if (errno || *end || pattern > 0xff) {
+                error_report("Invalid pattern byte specified");
+                return 1;
+            }
+            break;
+        }
+        case OPTION_FLUSH_INTERVAL:
+        {
+            char *end;
+            errno = 0;
+            flush_interval = strtoul(optarg, &end, 0);
+            if (errno || *end || flush_interval > INT_MAX) {
+                error_report("Invalid flush interval specified");
+                return 1;
+            }
+            break;
+        }
+        case OPTION_NO_DRAIN:
+            drain_on_flush = false;
+            break;
+        case OPTION_IMAGE_OPTS:
+            image_opts = true;
+            break;
+        }
+    }
+
+    if (optind != argc - 1) {
+        error_exit("Expecting one image file name");
+    }
+    filename = argv[argc - 1];
+
+    if (!is_write && flush_interval) {
+        error_report("--flush-interval is only available in write tests");
+        ret = -1;
+        goto out;
+    }
+    if (flush_interval && flush_interval < depth) {
+        error_report("Flush interval can't be smaller than depth");
+        ret = -1;
+        goto out;
+    }
+
+    blk = img_open(image_opts, filename, fmt, flags, writethrough, quiet);
+    if (!blk) {
+        ret = -1;
+        goto out;
+    }
+
+    image_size = blk_getlength(blk);
+    if (image_size < 0) {
+        ret = image_size;
+        goto out;
+    }
+
+    data = (BenchData) {
+        .blk            = blk,
+        .image_size     = image_size,
+        .bufsize        = bufsize,
+        .step           = step ?: bufsize,
+        .nrreq          = depth,
+        .n              = count,
+        .offset         = offset,
+        .write          = is_write,
+        .flush_interval = flush_interval,
+        .drain_on_flush = drain_on_flush,
+    };
+    printf("Sending %d %s requests, %d bytes each, %d in parallel "
+           "(starting at offset %" PRId64 ", step size %d)\n",
+           data.n, data.write ? "write" : "read", data.bufsize, data.nrreq,
+           data.offset, data.step);
+    if (flush_interval) {
+        printf("Sending flush every %d requests\n", flush_interval);
+    }
+
+    data.buf = blk_blockalign(blk, data.nrreq * data.bufsize);
+    memset(data.buf, pattern, data.nrreq * data.bufsize);
+
+    data.qiov = g_new(QEMUIOVector, data.nrreq);
+    for (i = 0; i < data.nrreq; i++) {
+        qemu_iovec_init(&data.qiov[i], 1);
+        qemu_iovec_add(&data.qiov[i],
+                       data.buf + i * data.bufsize, data.bufsize);
+    }
+
+    gettimeofday(&t1, NULL);
+    bench_cb(&data, 0);
+
+    while (data.n > 0) {
+        main_loop_wait(false);
+    }
+    gettimeofday(&t2, NULL);
+
+    printf("Run completed in %3.3f seconds.\n",
+           (t2.tv_sec - t1.tv_sec)
+           + ((double)(t2.tv_usec - t1.tv_usec) / 1000000));
+
+out:
+    qemu_vfree(data.buf);
+    blk_unref(blk);
+
+    if (ret) {
+        return 1;
+    }
+    return 0;
+}
+
+
 static const img_cmd_t img_cmds[] = {
 #define DEF(option, callback, arg_string)        \
     { option, callback },
@@ -3017,10 +3809,12 @@
     const img_cmd_t *cmd;
     const char *cmdname;
     Error *local_error = NULL;
+    char *trace_file = NULL;
     int c;
     static const struct option long_options[] = {
         {"help", no_argument, 0, 'h'},
-        {"version", no_argument, 0, 'v'},
+        {"version", no_argument, 0, 'V'},
+        {"trace", required_argument, NULL, 'T'},
         {0, 0, 0, 0}
     };
 
@@ -3032,34 +3826,60 @@
     qemu_init_exec_dir(argv[0]);
 
     if (qemu_init_main_loop(&local_error)) {
-        error_report("%s", error_get_pretty(local_error));
-        error_free(local_error);
+        error_report_err(local_error);
         exit(EXIT_FAILURE);
     }
 
+    qcrypto_init(&error_fatal);
+
+    module_call_init(MODULE_INIT_QOM);
     bdrv_init();
     if (argc < 2) {
         error_exit("Not enough arguments");
     }
-    cmdname = argv[1];
+
+    qemu_add_opts(&qemu_object_opts);
+    qemu_add_opts(&qemu_source_opts);
+    qemu_add_opts(&qemu_trace_opts);
+
+    while ((c = getopt_long(argc, argv, "+hVT:", long_options, NULL)) != -1) {
+        switch (c) {
+        case 'h':
+            help();
+            return 0;
+        case 'V':
+            printf(QEMU_IMG_VERSION);
+            return 0;
+        case 'T':
+            g_free(trace_file);
+            trace_file = trace_opt_parse(optarg);
+            break;
+        }
+    }
+
+    cmdname = argv[optind];
+
+    /* reset getopt_long scanning */
+    argc -= optind;
+    if (argc < 1) {
+        return 0;
+    }
+    argv += optind;
+    optind = 0;
+
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+    qemu_set_log(LOG_TRACE);
 
     /* find the command */
     for (cmd = img_cmds; cmd->name != NULL; cmd++) {
         if (!strcmp(cmdname, cmd->name)) {
-            return cmd->handler(argc - 1, argv + 1);
+            return cmd->handler(argc, argv);
         }
     }
 
-    c = getopt_long(argc, argv, "h", long_options, NULL);
-
-    if (c == 'h') {
-        help();
-    }
-    if (c == 'v') {
-        printf(QEMU_IMG_VERSION);
-        return 0;
-    }
-
     /* not found */
     error_exit("Command not found: %s", cmdname);
 }
diff --git a/qemu-img.texi b/qemu-img.texi
index 0a1ab35..449a19c 100644
--- a/qemu-img.texi
+++ b/qemu-img.texi
@@ -1,6 +1,6 @@
 @example
 @c man begin SYNOPSIS
-usage: qemu-img command [command options]
+@command{qemu-img} [@var{standard} @var{options}] @var{command} [@var{command} @var{options}]
 @c man end
 @end example
 
@@ -16,6 +16,17 @@
 
 @c man begin OPTIONS
 
+Standard options:
+@table @option
+@item -h, --help
+Display this help and exit
+@item -V, --version
+Display version information and exit
+@item -T, --trace [[enable=]@var{pattern}][,events=@var{file}][,file=@var{file}]
+@findex --trace
+@include qemu-option-trace.texi
+@end table
+
 The following commands are supported:
 
 @include qemu-img-cmds.texi
@@ -24,6 +35,20 @@
 @table @var
 @item filename
  is a disk image filename
+
+@item --object @var{objectdef}
+
+is a QEMU user creatable object definition. See the @code{qemu(1)} manual
+page for a description of the object properties. The most common object
+type is a @code{secret}, which is used to supply passwords and/or encryption
+keys.
+
+@item --image-opts
+
+Indicates that the @var{filename} parameter is to be interpreted as a
+full option string, not a plain filename. This parameter is mutually
+exclusive with the @var{-f} and @var{-F} parameters.
+
 @item fmt
 is the disk image format. It is guessed automatically in most cases. See below
 for a description of the supported disk formats.
@@ -103,7 +128,7 @@
 @item -F
 Second image format
 @item -s
-Strict mode - fail on on different image size or sector allocation
+Strict mode - fail on different image size or sector allocation
 @end table
 
 Parameters to convert subcommand:
@@ -117,6 +142,30 @@
 Command description:
 
 @table @option
+@item bench [-c @var{count}] [-d @var{depth}] [-f @var{fmt}] [--flush-interval=@var{flush_interval}] [-n] [--no-drain] [-o @var{offset}] [--pattern=@var{pattern}] [-q] [-s @var{buffer_size}] [-S @var{step_size}] [-t @var{cache}] [-w] @var{filename}
+
+Run a simple sequential I/O benchmark on the specified image. If @code{-w} is
+specified, a write test is performed, otherwise a read test is performed.
+
+A total number of @var{count} I/O requests is performed, each @var{buffer_size}
+bytes in size, and with @var{depth} requests in parallel. The first request
+starts at the position given by @var{offset}, each following request increases
+the current position by @var{step_size}. If @var{step_size} is not given,
+@var{buffer_size} is used for its value.
+
+If @var{flush_interval} is specified for a write test, the request queue is
+drained and a flush is issued before new writes are made whenever the number of
+remaining requests is a multiple of @var{flush_interval}. If additionally
+@code{--no-drain} is specified, a flush is issued without draining the request
+queue first.
+
+If @code{-n} is specified, the native AIO backend is used if possible. On
+Linux, this option only works if @code{-t none} or @code{-t directsync} is
+specified as well.
+
+For write tests, by default a buffer filled with zeros is written. This can be
+overridden with a pattern byte specified by @var{pattern}.
+
 @item check [-f @var{fmt}] [--output=@var{ofmt}] [-r [leaks | all]] [-T @var{src_cache}] @var{filename}
 
 Perform a consistency check on the disk image @var{filename}. The command can
diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index d94fb1e..25954f5 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -1,18 +1,25 @@
 /*
  * Command line utility to exercise the QEMU I/O path.
  *
- * Copyright (C) 2009 Red Hat, Inc.
+ * Copyright (C) 2009-2016 Red Hat, Inc.
  * Copyright (c) 2003-2005 Silicon Graphics, Inc.
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-io.h"
-#include "block/block_int.h"
+#include "sysemu/block-backend.h"
+#include "block/block.h"
+#include "block/block_int.h" /* for info_f() */
 #include "block/qapi.h"
+#include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "qemu/timer.h"
+#include "sysemu/block-backend.h"
+#include "qemu/cutils.h"
 
 #define CMD_NOFILE_OK   0x01
 
@@ -40,24 +47,24 @@
     return 0;
 }
 
-static int init_check_command(BlockDriverState *bs, const cmdinfo_t *ct)
+static int init_check_command(BlockBackend *blk, const cmdinfo_t *ct)
 {
     if (ct->flags & CMD_FLAG_GLOBAL) {
         return 1;
     }
-    if (!(ct->flags & CMD_NOFILE_OK) && !bs) {
+    if (!(ct->flags & CMD_NOFILE_OK) && !blk) {
         fprintf(stderr, "no file open, try 'help open'\n");
         return 0;
     }
     return 1;
 }
 
-static int command(BlockDriverState *bs, const cmdinfo_t *ct, int argc,
+static int command(BlockBackend *blk, const cmdinfo_t *ct, int argc,
                    char **argv)
 {
     char *cmd = argv[0];
 
-    if (!init_check_command(bs, ct)) {
+    if (!init_check_command(blk, ct)) {
         return 0;
     }
 
@@ -78,7 +85,7 @@
         return 0;
     }
     optind = 0;
-    return ct->cfunc(bs, argc, argv);
+    return ct->cfunc(blk, argc, argv);
 }
 
 static const cmdinfo_t *find_command(const char *cmd)
@@ -132,7 +139,29 @@
 static int64_t cvtnum(const char *s)
 {
     char *end;
-    return strtosz_suffix(s, &end, STRTOSZ_DEFSUFFIX_B);
+    int64_t ret;
+
+    ret = qemu_strtosz_suffix(s, &end, QEMU_STRTOSZ_DEFSUFFIX_B);
+    if (*end != '\0') {
+        /* Detritus at the end of the string */
+        return -EINVAL;
+    }
+    return ret;
+}
+
+static void print_cvtnum_err(int64_t rc, const char *arg)
+{
+    switch (rc) {
+    case -EINVAL:
+        printf("Parsing error: non-numeric argument,"
+               " or extraneous/unrecognized suffix -- %s\n", arg);
+        break;
+    case -ERANGE:
+        printf("Parsing error: argument too large -- %s\n", arg);
+        break;
+    default:
+        printf("Parsing error: %s\n", arg);
+    }
 }
 
 #define EXABYTES(x)     ((long long)(x) << 60)
@@ -267,14 +296,14 @@
  */
 
 #define MISALIGN_OFFSET     16
-static void *qemu_io_alloc(BlockDriverState *bs, size_t len, int pattern)
+static void *qemu_io_alloc(BlockBackend *blk, size_t len, int pattern)
 {
     void *buf;
 
     if (qemuio_misalign) {
         len += MISALIGN_OFFSET;
     }
-    buf = qemu_blockalign(bs, len);
+    buf = blk_blockalign(blk, len);
     memset(buf, pattern, len);
     if (qemuio_misalign) {
         buf += MISALIGN_OFFSET;
@@ -290,9 +319,10 @@
     qemu_vfree(p);
 }
 
-static void dump_buffer(const void *buffer, int64_t offset, int len)
+static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
 {
-    int i, j;
+    uint64_t i;
+    int j;
     const uint8_t *p;
 
     for (i = 0, p = buffer; i < len; i += 16) {
@@ -315,7 +345,7 @@
 }
 
 static void print_report(const char *op, struct timeval *t, int64_t offset,
-                         int count, int total, int cnt, int Cflag)
+                         int64_t count, int64_t total, int cnt, bool Cflag)
 {
     char s1[64], s2[64], ts[64];
 
@@ -323,12 +353,12 @@
     if (!Cflag) {
         cvtstr((double)total, s1, sizeof(s1));
         cvtstr(tdiv((double)total, *t), s2, sizeof(s2));
-        printf("%s %d/%d bytes at offset %" PRId64 "\n",
+        printf("%s %"PRId64"/%"PRId64" bytes at offset %" PRId64 "\n",
                op, total, count, offset);
         printf("%s, %d ops; %s (%s/sec and %.4f ops/sec)\n",
                s1, cnt, ts, s2, tdiv((double)cnt, *t));
     } else {/* bytes,ops,time,bytes/sec,ops/sec */
-        printf("%d,%d,%s,%.3f,%.3f\n",
+        printf("%"PRId64",%d,%s,%.3f,%.3f\n",
             total, cnt, ts,
             tdiv((double)total, *t),
             tdiv((double)cnt, *t));
@@ -340,7 +370,7 @@
  * vector matching it.
  */
 static void *
-create_iovec(BlockDriverState *bs, QEMUIOVector *qiov, char **argv, int nr_iov,
+create_iovec(BlockBackend *blk, QEMUIOVector *qiov, char **argv, int nr_iov,
              int pattern)
 {
     size_t *sizes = g_new0(size_t, nr_iov);
@@ -355,19 +385,13 @@
 
         len = cvtnum(arg);
         if (len < 0) {
-            printf("non-numeric length argument -- %s\n", arg);
+            print_cvtnum_err(len, arg);
             goto fail;
         }
 
-        /* should be SIZE_T_MAX, but that doesn't exist */
-        if (len > INT_MAX) {
-            printf("too large length argument -- %s\n", arg);
-            goto fail;
-        }
-
-        if (len & 0x1ff) {
-            printf("length argument %" PRId64
-                   " is not sector aligned\n", len);
+        if (len > SIZE_MAX) {
+            printf("Argument '%s' exceeds maximum size %llu\n", arg,
+                   (unsigned long long)SIZE_MAX);
             goto fail;
         }
 
@@ -377,7 +401,7 @@
 
     qemu_iovec_init(qiov, nr_iov);
 
-    buf = p = qemu_io_alloc(bs, count, pattern);
+    buf = p = qemu_io_alloc(blk, count, pattern);
 
     for (i = 0; i < nr_iov; i++) {
         qemu_iovec_add(qiov, p, sizes[i]);
@@ -389,46 +413,28 @@
     return buf;
 }
 
-static int do_read(BlockDriverState *bs, char *buf, int64_t offset, int count,
-                   int *total)
+static int do_pread(BlockBackend *blk, char *buf, int64_t offset,
+                    int64_t count, int64_t *total)
 {
-    int ret;
-
-    ret = bdrv_read(bs, offset >> 9, (uint8_t *)buf, count >> 9);
-    if (ret < 0) {
-        return ret;
+    if (count > INT_MAX) {
+        return -ERANGE;
     }
-    *total = count;
-    return 1;
-}
 
-static int do_write(BlockDriverState *bs, char *buf, int64_t offset, int count,
-                    int *total)
-{
-    int ret;
-
-    ret = bdrv_write(bs, offset >> 9, (uint8_t *)buf, count >> 9);
-    if (ret < 0) {
-        return ret;
-    }
-    *total = count;
-    return 1;
-}
-
-static int do_pread(BlockDriverState *bs, char *buf, int64_t offset, int count,
-                    int *total)
-{
-    *total = bdrv_pread(bs, offset, (uint8_t *)buf, count);
+    *total = blk_pread(blk, offset, (uint8_t *)buf, count);
     if (*total < 0) {
         return *total;
     }
     return 1;
 }
 
-static int do_pwrite(BlockDriverState *bs, char *buf, int64_t offset, int count,
-                     int *total)
+static int do_pwrite(BlockBackend *blk, char *buf, int64_t offset,
+                     int64_t count, int flags, int64_t *total)
 {
-    *total = bdrv_pwrite(bs, offset, (uint8_t *)buf, count);
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
+    *total = blk_pwrite(blk, offset, (uint8_t *)buf, count, flags);
     if (*total < 0) {
         return *total;
     }
@@ -436,20 +442,21 @@
 }
 
 typedef struct {
-    BlockDriverState *bs;
+    BlockBackend *blk;
     int64_t offset;
-    int count;
-    int *total;
+    int64_t count;
+    int64_t *total;
+    int flags;
     int ret;
     bool done;
 } CoWriteZeroes;
 
-static void coroutine_fn co_write_zeroes_entry(void *opaque)
+static void coroutine_fn co_pwrite_zeroes_entry(void *opaque)
 {
     CoWriteZeroes *data = opaque;
 
-    data->ret = bdrv_co_write_zeroes(data->bs, data->offset / BDRV_SECTOR_SIZE,
-                                     data->count / BDRV_SECTOR_SIZE, 0);
+    data->ret = blk_co_pwrite_zeroes(data->blk, data->offset, data->count,
+                                     data->flags);
     data->done = true;
     if (data->ret < 0) {
         *data->total = data->ret;
@@ -459,22 +466,27 @@
     *data->total = data->count;
 }
 
-static int do_co_write_zeroes(BlockDriverState *bs, int64_t offset, int count,
-                              int *total)
+static int do_co_pwrite_zeroes(BlockBackend *blk, int64_t offset,
+                               int64_t count, int flags, int64_t *total)
 {
     Coroutine *co;
     CoWriteZeroes data = {
-        .bs     = bs,
+        .blk    = blk,
         .offset = offset,
         .count  = count,
         .total  = total,
+        .flags  = flags,
         .done   = false,
     };
 
-    co = qemu_coroutine_create(co_write_zeroes_entry);
-    qemu_coroutine_enter(co, &data);
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
+    co = qemu_coroutine_create(co_pwrite_zeroes_entry, &data);
+    qemu_coroutine_enter(co);
     while (!data.done) {
-        aio_poll(bdrv_get_aio_context(bs), true);
+        aio_poll(blk_get_aio_context(blk), true);
     }
     if (data.ret < 0) {
         return data.ret;
@@ -483,12 +495,16 @@
     }
 }
 
-static int do_write_compressed(BlockDriverState *bs, char *buf, int64_t offset,
-                               int count, int *total)
+static int do_write_compressed(BlockBackend *blk, char *buf, int64_t offset,
+                               int64_t count, int64_t *total)
 {
     int ret;
 
-    ret = bdrv_write_compressed(bs, offset >> 9, (uint8_t *)buf, count >> 9);
+    if (count >> 9 > BDRV_REQUEST_MAX_SECTORS) {
+        return -ERANGE;
+    }
+
+    ret = blk_write_compressed(blk, offset >> 9, (uint8_t *)buf, count >> 9);
     if (ret < 0) {
         return ret;
     }
@@ -496,20 +512,28 @@
     return 1;
 }
 
-static int do_load_vmstate(BlockDriverState *bs, char *buf, int64_t offset,
-                           int count, int *total)
+static int do_load_vmstate(BlockBackend *blk, char *buf, int64_t offset,
+                           int64_t count, int64_t *total)
 {
-    *total = bdrv_load_vmstate(bs, (uint8_t *)buf, offset, count);
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
+    *total = blk_load_vmstate(blk, (uint8_t *)buf, offset, count);
     if (*total < 0) {
         return *total;
     }
     return 1;
 }
 
-static int do_save_vmstate(BlockDriverState *bs, char *buf, int64_t offset,
-                           int count, int *total)
+static int do_save_vmstate(BlockBackend *blk, char *buf, int64_t offset,
+                           int64_t count, int64_t *total)
 {
-    *total = bdrv_save_vmstate(bs, (uint8_t *)buf, offset, count);
+    if (count > INT_MAX) {
+        return -ERANGE;
+    }
+
+    *total = blk_save_vmstate(blk, (uint8_t *)buf, offset, count);
     if (*total < 0) {
         return *total;
     }
@@ -522,13 +546,12 @@
     *(int *)opaque = ret;
 }
 
-static int do_aio_readv(BlockDriverState *bs, QEMUIOVector *qiov,
+static int do_aio_readv(BlockBackend *blk, QEMUIOVector *qiov,
                         int64_t offset, int *total)
 {
     int async_ret = NOT_DONE;
 
-    bdrv_aio_readv(bs, offset >> 9, qiov, qiov->size >> 9,
-                   aio_rw_done, &async_ret);
+    blk_aio_preadv(blk, offset, qiov, 0, aio_rw_done, &async_ret);
     while (async_ret == NOT_DONE) {
         main_loop_wait(false);
     }
@@ -537,13 +560,12 @@
     return async_ret < 0 ? async_ret : 1;
 }
 
-static int do_aio_writev(BlockDriverState *bs, QEMUIOVector *qiov,
-                         int64_t offset, int *total)
+static int do_aio_writev(BlockBackend *blk, QEMUIOVector *qiov,
+                         int64_t offset, int flags, int *total)
 {
     int async_ret = NOT_DONE;
 
-    bdrv_aio_writev(bs, offset >> 9, qiov, qiov->size >> 9,
-                    aio_rw_done, &async_ret);
+    blk_aio_pwritev(blk, offset, qiov, flags, aio_rw_done, &async_ret);
     while (async_ret == NOT_DONE) {
         main_loop_wait(false);
     }
@@ -552,49 +574,6 @@
     return async_ret < 0 ? async_ret : 1;
 }
 
-struct multiwrite_async_ret {
-    int num_done;
-    int error;
-};
-
-static void multiwrite_cb(void *opaque, int ret)
-{
-    struct multiwrite_async_ret *async_ret = opaque;
-
-    async_ret->num_done++;
-    if (ret < 0) {
-        async_ret->error = ret;
-    }
-}
-
-static int do_aio_multiwrite(BlockDriverState *bs, BlockRequest* reqs,
-                             int num_reqs, int *total)
-{
-    int i, ret;
-    struct multiwrite_async_ret async_ret = {
-        .num_done = 0,
-        .error = 0,
-    };
-
-    *total = 0;
-    for (i = 0; i < num_reqs; i++) {
-        reqs[i].cb = multiwrite_cb;
-        reqs[i].opaque = &async_ret;
-        *total += reqs[i].qiov->size;
-    }
-
-    ret = bdrv_aio_multiwrite(bs, reqs, num_reqs);
-    if (ret < 0) {
-        return ret;
-    }
-
-    while (async_ret.num_done < num_reqs) {
-        main_loop_wait(false);
-    }
-
-    return async_ret.error < 0 ? async_ret.error : 1;
-}
-
 static void read_help(void)
 {
     printf(
@@ -609,7 +588,7 @@
 " -b, -- read from the VM state rather than the virtual disk\n"
 " -C, -- report statistics in a machine parsable format\n"
 " -l, -- length for pattern verification (only with -P)\n"
-" -p, -- use bdrv_pread to read the file\n"
+" -p, -- ignored for backwards compatibility\n"
 " -P, -- use a pattern to verify read data\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
 " -s, -- start offset for pattern verification (only with -P)\n"
@@ -617,7 +596,7 @@
 "\n");
 }
 
-static int read_f(BlockDriverState *bs, int argc, char **argv);
+static int read_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t read_cmd = {
     .name       = "read",
@@ -625,63 +604,64 @@
     .cfunc      = read_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-abCpqv] [-P pattern [-s off] [-l len]] off len",
+    .args       = "[-abCqv] [-P pattern [-s off] [-l len]] off len",
     .oneline    = "reads a number of bytes at a specified offset",
     .help       = read_help,
 };
 
-static int read_f(BlockDriverState *bs, int argc, char **argv)
+static int read_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timeval t1, t2;
-    int Cflag = 0, pflag = 0, qflag = 0, vflag = 0;
-    int Pflag = 0, sflag = 0, lflag = 0, bflag = 0;
+    bool Cflag = false, qflag = false, vflag = false;
+    bool Pflag = false, sflag = false, lflag = false, bflag = false;
     int c, cnt;
     char *buf;
     int64_t offset;
-    int count;
+    int64_t count;
     /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
-    int pattern = 0, pattern_offset = 0, pattern_count = 0;
+    int64_t total = 0;
+    int pattern = 0;
+    int64_t pattern_offset = 0, pattern_count = 0;
 
-    while ((c = getopt(argc, argv, "bCl:pP:qs:v")) != EOF) {
+    while ((c = getopt(argc, argv, "bCl:pP:qs:v")) != -1) {
         switch (c) {
         case 'b':
-            bflag = 1;
+            bflag = true;
             break;
         case 'C':
-            Cflag = 1;
+            Cflag = true;
             break;
         case 'l':
-            lflag = 1;
+            lflag = true;
             pattern_count = cvtnum(optarg);
             if (pattern_count < 0) {
-                printf("non-numeric length argument -- %s\n", optarg);
+                print_cvtnum_err(pattern_count, optarg);
                 return 0;
             }
             break;
         case 'p':
-            pflag = 1;
+            /* Ignored for backwards compatibility */
             break;
         case 'P':
-            Pflag = 1;
+            Pflag = true;
             pattern = parse_pattern(optarg);
             if (pattern < 0) {
                 return 0;
             }
             break;
         case 'q':
-            qflag = 1;
+            qflag = true;
             break;
         case 's':
-            sflag = 1;
+            sflag = true;
             pattern_offset = cvtnum(optarg);
             if (pattern_offset < 0) {
-                printf("non-numeric length argument -- %s\n", optarg);
+                print_cvtnum_err(pattern_offset, optarg);
                 return 0;
             }
             break;
         case 'v':
-            vflag = 1;
+            vflag = true;
             break;
         default:
             return qemuio_command_usage(&read_cmd);
@@ -692,21 +672,20 @@
         return qemuio_command_usage(&read_cmd);
     }
 
-    if (bflag && pflag) {
-        printf("-b and -p cannot be specified at the same time\n");
-        return 0;
-    }
-
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count > SIZE_MAX) {
+        printf("length cannot exceed %" PRIu64 ", given %s\n",
+               (uint64_t) SIZE_MAX, argv[optind]);
         return 0;
     }
 
@@ -723,28 +702,26 @@
         return 0;
     }
 
-    if (!pflag) {
+    if (bflag) {
         if (offset & 0x1ff) {
             printf("offset %" PRId64 " is not sector aligned\n",
                    offset);
             return 0;
         }
         if (count & 0x1ff) {
-            printf("count %d is not sector aligned\n",
+            printf("count %"PRId64" is not sector aligned\n",
                    count);
             return 0;
         }
     }
 
-    buf = qemu_io_alloc(bs, count, 0xab);
+    buf = qemu_io_alloc(blk, count, 0xab);
 
     gettimeofday(&t1, NULL);
-    if (pflag) {
-        cnt = do_pread(bs, buf, offset, count, &total);
-    } else if (bflag) {
-        cnt = do_load_vmstate(bs, buf, offset, count, &total);
+    if (bflag) {
+        cnt = do_load_vmstate(blk, buf, offset, count, &total);
     } else {
-        cnt = do_read(bs, buf, offset, count, &total);
+        cnt = do_pread(blk, buf, offset, count, &total);
     }
     gettimeofday(&t2, NULL);
 
@@ -758,7 +735,7 @@
         memset(cmp_buf, pattern, pattern_count);
         if (memcmp(buf + pattern_offset, cmp_buf, pattern_count)) {
             printf("Pattern verification failed at offset %"
-                   PRId64 ", %d bytes\n",
+                   PRId64 ", %"PRId64" bytes\n",
                    offset + pattern_offset, pattern_count);
         }
         g_free(cmp_buf);
@@ -801,22 +778,22 @@
 "\n");
 }
 
-static int readv_f(BlockDriverState *bs, int argc, char **argv);
+static int readv_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t readv_cmd = {
     .name       = "readv",
     .cfunc      = readv_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-Cqv] [-P pattern ] off len [len..]",
+    .args       = "[-Cqv] [-P pattern] off len [len..]",
     .oneline    = "reads a number of bytes at a specified offset",
     .help       = readv_help,
 };
 
-static int readv_f(BlockDriverState *bs, int argc, char **argv)
+static int readv_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timeval t1, t2;
-    int Cflag = 0, qflag = 0, vflag = 0;
+    bool Cflag = false, qflag = false, vflag = false;
     int c, cnt;
     char *buf;
     int64_t offset;
@@ -825,25 +802,25 @@
     int nr_iov;
     QEMUIOVector qiov;
     int pattern = 0;
-    int Pflag = 0;
+    bool Pflag = false;
 
-    while ((c = getopt(argc, argv, "CP:qv")) != EOF) {
+    while ((c = getopt(argc, argv, "CP:qv")) != -1) {
         switch (c) {
         case 'C':
-            Cflag = 1;
+            Cflag = true;
             break;
         case 'P':
-            Pflag = 1;
+            Pflag = true;
             pattern = parse_pattern(optarg);
             if (pattern < 0) {
                 return 0;
             }
             break;
         case 'q':
-            qflag = 1;
+            qflag = true;
             break;
         case 'v':
-            vflag = 1;
+            vflag = true;
             break;
         default:
             return qemuio_command_usage(&readv_cmd);
@@ -857,25 +834,19 @@
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
     optind++;
 
-    if (offset & 0x1ff) {
-        printf("offset %" PRId64 " is not sector aligned\n",
-               offset);
-        return 0;
-    }
-
     nr_iov = argc - optind;
-    buf = create_iovec(bs, &qiov, &argv[optind], nr_iov, 0xab);
+    buf = create_iovec(blk, &qiov, &argv[optind], nr_iov, 0xab);
     if (buf == NULL) {
         return 0;
     }
 
     gettimeofday(&t1, NULL);
-    cnt = do_aio_readv(bs, &qiov, offset, &total);
+    cnt = do_aio_readv(blk, &qiov, offset, &total);
     gettimeofday(&t2, NULL);
 
     if (cnt < 0) {
@@ -923,16 +894,18 @@
 " Writes into a segment of the currently open file, using a buffer\n"
 " filled with a set pattern (0xcdcdcdcd).\n"
 " -b, -- write to the VM state rather than the virtual disk\n"
-" -c, -- write compressed data with bdrv_write_compressed\n"
-" -p, -- use bdrv_pwrite to write the file\n"
+" -c, -- write compressed data with blk_write_compressed\n"
+" -f, -- use Force Unit Access semantics\n"
+" -p, -- ignored for backwards compatibility\n"
 " -P, -- use different pattern to fill file\n"
 " -C, -- report statistics in a machine parsable format\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
-" -z, -- write zeroes using bdrv_co_write_zeroes\n"
+" -u, -- with -z, allow unmapping\n"
+" -z, -- write zeroes using blk_co_pwrite_zeroes\n"
 "\n");
 }
 
-static int write_f(BlockDriverState *bs, int argc, char **argv);
+static int write_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t write_cmd = {
     .name       = "write",
@@ -940,50 +913,57 @@
     .cfunc      = write_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-bcCpqz] [-P pattern ] off len",
+    .args       = "[-bcCfquz] [-P pattern] off len",
     .oneline    = "writes a number of bytes at a specified offset",
     .help       = write_help,
 };
 
-static int write_f(BlockDriverState *bs, int argc, char **argv)
+static int write_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timeval t1, t2;
-    int Cflag = 0, pflag = 0, qflag = 0, bflag = 0, Pflag = 0, zflag = 0;
-    int cflag = 0;
+    bool Cflag = false, qflag = false, bflag = false;
+    bool Pflag = false, zflag = false, cflag = false;
+    int flags = 0;
     int c, cnt;
     char *buf = NULL;
     int64_t offset;
-    int count;
+    int64_t count;
     /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
+    int64_t total = 0;
     int pattern = 0xcd;
 
-    while ((c = getopt(argc, argv, "bcCpP:qz")) != EOF) {
+    while ((c = getopt(argc, argv, "bcCfpP:quz")) != -1) {
         switch (c) {
         case 'b':
-            bflag = 1;
+            bflag = true;
             break;
         case 'c':
-            cflag = 1;
+            cflag = true;
             break;
         case 'C':
-            Cflag = 1;
+            Cflag = true;
+            break;
+        case 'f':
+            flags |= BDRV_REQ_FUA;
             break;
         case 'p':
-            pflag = 1;
+            /* Ignored for backwards compatibility */
             break;
         case 'P':
-            Pflag = 1;
+            Pflag = true;
             pattern = parse_pattern(optarg);
             if (pattern < 0) {
                 return 0;
             }
             break;
         case 'q':
-            qflag = 1;
+            qflag = true;
+            break;
+        case 'u':
+            flags |= BDRV_REQ_MAY_UNMAP;
             break;
         case 'z':
-            zflag = 1;
+            zflag = true;
             break;
         default:
             return qemuio_command_usage(&write_cmd);
@@ -994,8 +974,18 @@
         return qemuio_command_usage(&write_cmd);
     }
 
-    if (bflag + pflag + zflag > 1) {
-        printf("-b, -p, or -z cannot be specified at the same time\n");
+    if (bflag && zflag) {
+        printf("-b and -z cannot be specified at the same time\n");
+        return 0;
+    }
+
+    if ((flags & BDRV_REQ_FUA) && (bflag || cflag)) {
+        printf("-f and -b or -c cannot be specified at the same time\n");
+        return 0;
+    }
+
+    if ((flags & BDRV_REQ_MAY_UNMAP) && !zflag) {
+        printf("-u requires -z to be specified\n");
         return 0;
     }
 
@@ -1006,18 +996,22 @@
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count > SIZE_MAX) {
+        printf("length cannot exceed %" PRIu64 ", given %s\n",
+               (uint64_t) SIZE_MAX, argv[optind]);
         return 0;
     }
 
-    if (!pflag) {
+    if (bflag || cflag) {
         if (offset & 0x1ff) {
             printf("offset %" PRId64 " is not sector aligned\n",
                    offset);
@@ -1025,27 +1019,25 @@
         }
 
         if (count & 0x1ff) {
-            printf("count %d is not sector aligned\n",
+            printf("count %"PRId64" is not sector aligned\n",
                    count);
             return 0;
         }
     }
 
     if (!zflag) {
-        buf = qemu_io_alloc(bs, count, pattern);
+        buf = qemu_io_alloc(blk, count, pattern);
     }
 
     gettimeofday(&t1, NULL);
-    if (pflag) {
-        cnt = do_pwrite(bs, buf, offset, count, &total);
-    } else if (bflag) {
-        cnt = do_save_vmstate(bs, buf, offset, count, &total);
+    if (bflag) {
+        cnt = do_save_vmstate(blk, buf, offset, count, &total);
     } else if (zflag) {
-        cnt = do_co_write_zeroes(bs, offset, count, &total);
+        cnt = do_co_pwrite_zeroes(blk, offset, count, flags, &total);
     } else if (cflag) {
-        cnt = do_write_compressed(bs, buf, offset, count, &total);
+        cnt = do_write_compressed(blk, buf, offset, count, &total);
     } else {
-        cnt = do_write(bs, buf, offset, count, &total);
+        cnt = do_pwrite(blk, buf, offset, count, flags, &total);
     }
     gettimeofday(&t2, NULL);
 
@@ -1084,26 +1076,28 @@
 " filled with a set pattern (0xcdcdcdcd).\n"
 " -P, -- use different pattern to fill file\n"
 " -C, -- report statistics in a machine parsable format\n"
+" -f, -- use Force Unit Access semantics\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
 "\n");
 }
 
-static int writev_f(BlockDriverState *bs, int argc, char **argv);
+static int writev_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t writev_cmd = {
     .name       = "writev",
     .cfunc      = writev_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-Cq] [-P pattern ] off len [len..]",
+    .args       = "[-Cfq] [-P pattern] off len [len..]",
     .oneline    = "writes a number of bytes at a specified offset",
     .help       = writev_help,
 };
 
-static int writev_f(BlockDriverState *bs, int argc, char **argv)
+static int writev_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timeval t1, t2;
-    int Cflag = 0, qflag = 0;
+    bool Cflag = false, qflag = false;
+    int flags = 0;
     int c, cnt;
     char *buf;
     int64_t offset;
@@ -1113,13 +1107,16 @@
     int pattern = 0xcd;
     QEMUIOVector qiov;
 
-    while ((c = getopt(argc, argv, "CqP:")) != EOF) {
+    while ((c = getopt(argc, argv, "CfqP:")) != -1) {
         switch (c) {
         case 'C':
-            Cflag = 1;
+            Cflag = true;
+            break;
+        case 'f':
+            flags |= BDRV_REQ_FUA;
             break;
         case 'q':
-            qflag = 1;
+            qflag = true;
             break;
         case 'P':
             pattern = parse_pattern(optarg);
@@ -1138,25 +1135,19 @@
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
     optind++;
 
-    if (offset & 0x1ff) {
-        printf("offset %" PRId64 " is not sector aligned\n",
-               offset);
-        return 0;
-    }
-
     nr_iov = argc - optind;
-    buf = create_iovec(bs, &qiov, &argv[optind], nr_iov, pattern);
+    buf = create_iovec(blk, &qiov, &argv[optind], nr_iov, pattern);
     if (buf == NULL) {
         return 0;
     }
 
     gettimeofday(&t1, NULL);
-    cnt = do_aio_writev(bs, &qiov, offset, &total);
+    cnt = do_aio_writev(blk, &qiov, offset, flags, &total);
     gettimeofday(&t2, NULL);
 
     if (cnt < 0) {
@@ -1177,173 +1168,17 @@
     return 0;
 }
 
-static void multiwrite_help(void)
-{
-    printf(
-"\n"
-" writes a range of bytes from the given offset source from multiple buffers,\n"
-" in a batch of requests that may be merged by qemu\n"
-"\n"
-" Example:\n"
-" 'multiwrite 512 1k 1k ; 4k 1k'\n"
-"  writes 2 kB at 512 bytes and 1 kB at 4 kB into the open file\n"
-"\n"
-" Writes into a segment of the currently open file, using a buffer\n"
-" filled with a set pattern (0xcdcdcdcd). The pattern byte is increased\n"
-" by one for each request contained in the multiwrite command.\n"
-" -P, -- use different pattern to fill file\n"
-" -C, -- report statistics in a machine parsable format\n"
-" -q, -- quiet mode, do not show I/O statistics\n"
-"\n");
-}
-
-static int multiwrite_f(BlockDriverState *bs, int argc, char **argv);
-
-static const cmdinfo_t multiwrite_cmd = {
-    .name       = "multiwrite",
-    .cfunc      = multiwrite_f,
-    .argmin     = 2,
-    .argmax     = -1,
-    .args       = "[-Cq] [-P pattern ] off len [len..] [; off len [len..]..]",
-    .oneline    = "issues multiple write requests at once",
-    .help       = multiwrite_help,
-};
-
-static int multiwrite_f(BlockDriverState *bs, int argc, char **argv)
-{
-    struct timeval t1, t2;
-    int Cflag = 0, qflag = 0;
-    int c, cnt;
-    char **buf;
-    int64_t offset, first_offset = 0;
-    /* Some compilers get confused and warn if this is not initialized.  */
-    int total = 0;
-    int nr_iov;
-    int nr_reqs;
-    int pattern = 0xcd;
-    QEMUIOVector *qiovs;
-    int i;
-    BlockRequest *reqs;
-
-    while ((c = getopt(argc, argv, "CqP:")) != EOF) {
-        switch (c) {
-        case 'C':
-            Cflag = 1;
-            break;
-        case 'q':
-            qflag = 1;
-            break;
-        case 'P':
-            pattern = parse_pattern(optarg);
-            if (pattern < 0) {
-                return 0;
-            }
-            break;
-        default:
-            return qemuio_command_usage(&writev_cmd);
-        }
-    }
-
-    if (optind > argc - 2) {
-        return qemuio_command_usage(&writev_cmd);
-    }
-
-    nr_reqs = 1;
-    for (i = optind; i < argc; i++) {
-        if (!strcmp(argv[i], ";")) {
-            nr_reqs++;
-        }
-    }
-
-    reqs = g_new0(BlockRequest, nr_reqs);
-    buf = g_new0(char *, nr_reqs);
-    qiovs = g_new(QEMUIOVector, nr_reqs);
-
-    for (i = 0; i < nr_reqs && optind < argc; i++) {
-        int j;
-
-        /* Read the offset of the request */
-        offset = cvtnum(argv[optind]);
-        if (offset < 0) {
-            printf("non-numeric offset argument -- %s\n", argv[optind]);
-            goto out;
-        }
-        optind++;
-
-        if (offset & 0x1ff) {
-            printf("offset %lld is not sector aligned\n",
-                   (long long)offset);
-            goto out;
-        }
-
-        if (i == 0) {
-            first_offset = offset;
-        }
-
-        /* Read lengths for qiov entries */
-        for (j = optind; j < argc; j++) {
-            if (!strcmp(argv[j], ";")) {
-                break;
-            }
-        }
-
-        nr_iov = j - optind;
-
-        /* Build request */
-        buf[i] = create_iovec(bs, &qiovs[i], &argv[optind], nr_iov, pattern);
-        if (buf[i] == NULL) {
-            goto out;
-        }
-
-        reqs[i].qiov = &qiovs[i];
-        reqs[i].sector = offset >> 9;
-        reqs[i].nb_sectors = reqs[i].qiov->size >> 9;
-
-        optind = j + 1;
-
-        pattern++;
-    }
-
-    /* If there were empty requests at the end, ignore them */
-    nr_reqs = i;
-
-    gettimeofday(&t1, NULL);
-    cnt = do_aio_multiwrite(bs, reqs, nr_reqs, &total);
-    gettimeofday(&t2, NULL);
-
-    if (cnt < 0) {
-        printf("aio_multiwrite failed: %s\n", strerror(-cnt));
-        goto out;
-    }
-
-    if (qflag) {
-        goto out;
-    }
-
-    /* Finally, report back -- -C gives a parsable format */
-    t2 = tsub(t2, t1);
-    print_report("wrote", &t2, first_offset, total, total, cnt, Cflag);
-out:
-    for (i = 0; i < nr_reqs; i++) {
-        qemu_io_free(buf[i]);
-        if (reqs[i].qiov != NULL) {
-            qemu_iovec_destroy(&qiovs[i]);
-        }
-    }
-    g_free(buf);
-    g_free(reqs);
-    g_free(qiovs);
-    return 0;
-}
-
 struct aio_ctx {
+    BlockBackend *blk;
     QEMUIOVector qiov;
     int64_t offset;
     char *buf;
-    int qflag;
-    int vflag;
-    int Cflag;
-    int Pflag;
+    bool qflag;
+    bool vflag;
+    bool Cflag;
+    bool Pflag;
+    bool zflag;
+    BlockAcctCookie acct;
     int pattern;
     struct timeval t1;
 };
@@ -1358,9 +1193,12 @@
 
     if (ret < 0) {
         printf("aio_write failed: %s\n", strerror(-ret));
+        block_acct_failed(blk_get_stats(ctx->blk), &ctx->acct);
         goto out;
     }
 
+    block_acct_done(blk_get_stats(ctx->blk), &ctx->acct);
+
     if (ctx->qflag) {
         goto out;
     }
@@ -1370,8 +1208,10 @@
     print_report("wrote", &t2, ctx->offset, ctx->qiov.size,
                  ctx->qiov.size, 1, ctx->Cflag);
 out:
-    qemu_io_free(ctx->buf);
-    qemu_iovec_destroy(&ctx->qiov);
+    if (!ctx->zflag) {
+        qemu_io_free(ctx->buf);
+        qemu_iovec_destroy(&ctx->qiov);
+    }
     g_free(ctx);
 }
 
@@ -1384,6 +1224,7 @@
 
     if (ret < 0) {
         printf("readv failed: %s\n", strerror(-ret));
+        block_acct_failed(blk_get_stats(ctx->blk), &ctx->acct);
         goto out;
     }
 
@@ -1398,6 +1239,8 @@
         g_free(cmp_buf);
     }
 
+    block_acct_done(blk_get_stats(ctx->blk), &ctx->acct);
+
     if (ctx->qflag) {
         goto out;
     }
@@ -1431,46 +1274,53 @@
 " used to ensure all outstanding aio requests have been completed.\n"
 " -C, -- report statistics in a machine parsable format\n"
 " -P, -- use a pattern to verify read data\n"
+" -i, -- treat request as invalid, for exercising stats\n"
 " -v, -- dump buffer to standard output\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
 "\n");
 }
 
-static int aio_read_f(BlockDriverState *bs, int argc, char **argv);
+static int aio_read_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t aio_read_cmd = {
     .name       = "aio_read",
     .cfunc      = aio_read_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-Cqv] [-P pattern ] off len [len..]",
+    .args       = "[-Ciqv] [-P pattern] off len [len..]",
     .oneline    = "asynchronously reads a number of bytes",
     .help       = aio_read_help,
 };
 
-static int aio_read_f(BlockDriverState *bs, int argc, char **argv)
+static int aio_read_f(BlockBackend *blk, int argc, char **argv)
 {
     int nr_iov, c;
     struct aio_ctx *ctx = g_new0(struct aio_ctx, 1);
 
-    while ((c = getopt(argc, argv, "CP:qv")) != EOF) {
+    ctx->blk = blk;
+    while ((c = getopt(argc, argv, "CP:iqv")) != -1) {
         switch (c) {
         case 'C':
-            ctx->Cflag = 1;
+            ctx->Cflag = true;
             break;
         case 'P':
-            ctx->Pflag = 1;
+            ctx->Pflag = true;
             ctx->pattern = parse_pattern(optarg);
             if (ctx->pattern < 0) {
                 g_free(ctx);
                 return 0;
             }
             break;
+        case 'i':
+            printf("injecting invalid read request\n");
+            block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
+            g_free(ctx);
+            return 0;
         case 'q':
-            ctx->qflag = 1;
+            ctx->qflag = true;
             break;
         case 'v':
-            ctx->vflag = 1;
+            ctx->vflag = true;
             break;
         default:
             g_free(ctx);
@@ -1485,29 +1335,24 @@
 
     ctx->offset = cvtnum(argv[optind]);
     if (ctx->offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(ctx->offset, argv[optind]);
         g_free(ctx);
         return 0;
     }
     optind++;
 
-    if (ctx->offset & 0x1ff) {
-        printf("offset %" PRId64 " is not sector aligned\n",
-               ctx->offset);
-        g_free(ctx);
-        return 0;
-    }
-
     nr_iov = argc - optind;
-    ctx->buf = create_iovec(bs, &ctx->qiov, &argv[optind], nr_iov, 0xab);
+    ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov, 0xab);
     if (ctx->buf == NULL) {
+        block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_READ);
         g_free(ctx);
         return 0;
     }
 
     gettimeofday(&ctx->t1, NULL);
-    bdrv_aio_readv(bs, ctx->offset >> 9, &ctx->qiov,
-                   ctx->qiov.size >> 9, aio_read_done, ctx);
+    block_acct_start(blk_get_stats(blk), &ctx->acct, ctx->qiov.size,
+                     BLOCK_ACCT_READ);
+    blk_aio_preadv(blk, ctx->offset, &ctx->qiov, 0, aio_read_done, ctx);
     return 0;
 }
 
@@ -1527,35 +1372,47 @@
 " used to ensure all outstanding aio requests have been completed.\n"
 " -P, -- use different pattern to fill file\n"
 " -C, -- report statistics in a machine parsable format\n"
+" -f, -- use Force Unit Access semantics\n"
+" -i, -- treat request as invalid, for exercising stats\n"
 " -q, -- quiet mode, do not show I/O statistics\n"
+" -u, -- with -z, allow unmapping\n"
+" -z, -- write zeroes using blk_aio_pwrite_zeroes\n"
 "\n");
 }
 
-static int aio_write_f(BlockDriverState *bs, int argc, char **argv);
+static int aio_write_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t aio_write_cmd = {
     .name       = "aio_write",
     .cfunc      = aio_write_f,
     .argmin     = 2,
     .argmax     = -1,
-    .args       = "[-Cq] [-P pattern ] off len [len..]",
+    .args       = "[-Cfiquz] [-P pattern] off len [len..]",
     .oneline    = "asynchronously writes a number of bytes",
     .help       = aio_write_help,
 };
 
-static int aio_write_f(BlockDriverState *bs, int argc, char **argv)
+static int aio_write_f(BlockBackend *blk, int argc, char **argv)
 {
     int nr_iov, c;
     int pattern = 0xcd;
     struct aio_ctx *ctx = g_new0(struct aio_ctx, 1);
+    int flags = 0;
 
-    while ((c = getopt(argc, argv, "CqP:")) != EOF) {
+    ctx->blk = blk;
+    while ((c = getopt(argc, argv, "CfiqP:uz")) != -1) {
         switch (c) {
         case 'C':
-            ctx->Cflag = 1;
+            ctx->Cflag = true;
+            break;
+        case 'f':
+            flags |= BDRV_REQ_FUA;
             break;
         case 'q':
-            ctx->qflag = 1;
+            ctx->qflag = true;
+            break;
+        case 'u':
+            flags |= BDRV_REQ_MAY_UNMAP;
             break;
         case 'P':
             pattern = parse_pattern(optarg);
@@ -1564,6 +1421,14 @@
                 return 0;
             }
             break;
+        case 'i':
+            printf("injecting invalid write request\n");
+            block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
+            g_free(ctx);
+            return 0;
+        case 'z':
+            ctx->zflag = true;
+            break;
         default:
             g_free(ctx);
             return qemuio_command_usage(&aio_write_cmd);
@@ -1575,37 +1440,69 @@
         return qemuio_command_usage(&aio_write_cmd);
     }
 
+    if (ctx->zflag && optind != argc - 2) {
+        printf("-z supports only a single length parameter\n");
+        g_free(ctx);
+        return 0;
+    }
+
+    if ((flags & BDRV_REQ_MAY_UNMAP) && !ctx->zflag) {
+        printf("-u requires -z to be specified\n");
+        g_free(ctx);
+        return 0;
+    }
+
+    if (ctx->zflag && ctx->Pflag) {
+        printf("-z and -P cannot be specified at the same time\n");
+        g_free(ctx);
+        return 0;
+    }
+
     ctx->offset = cvtnum(argv[optind]);
     if (ctx->offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(ctx->offset, argv[optind]);
         g_free(ctx);
         return 0;
     }
     optind++;
 
-    if (ctx->offset & 0x1ff) {
-        printf("offset %" PRId64 " is not sector aligned\n",
-               ctx->offset);
-        g_free(ctx);
-        return 0;
-    }
+    if (ctx->zflag) {
+        int64_t count = cvtnum(argv[optind]);
+        if (count < 0) {
+            print_cvtnum_err(count, argv[optind]);
+            g_free(ctx);
+            return 0;
+        }
 
-    nr_iov = argc - optind;
-    ctx->buf = create_iovec(bs, &ctx->qiov, &argv[optind], nr_iov, pattern);
-    if (ctx->buf == NULL) {
-        g_free(ctx);
-        return 0;
-    }
+        ctx->qiov.size = count;
+        blk_aio_pwrite_zeroes(blk, ctx->offset, count, flags, aio_write_done,
+                              ctx);
+    } else {
+        nr_iov = argc - optind;
+        ctx->buf = create_iovec(blk, &ctx->qiov, &argv[optind], nr_iov,
+                                pattern);
+        if (ctx->buf == NULL) {
+            block_acct_invalid(blk_get_stats(blk), BLOCK_ACCT_WRITE);
+            g_free(ctx);
+            return 0;
+        }
 
-    gettimeofday(&ctx->t1, NULL);
-    bdrv_aio_writev(bs, ctx->offset >> 9, &ctx->qiov,
-                    ctx->qiov.size >> 9, aio_write_done, ctx);
+        gettimeofday(&ctx->t1, NULL);
+        block_acct_start(blk_get_stats(blk), &ctx->acct, ctx->qiov.size,
+                         BLOCK_ACCT_WRITE);
+
+        blk_aio_pwritev(blk, ctx->offset, &ctx->qiov, flags, aio_write_done,
+                        ctx);
+    }
     return 0;
 }
 
-static int aio_flush_f(BlockDriverState *bs, int argc, char **argv)
+static int aio_flush_f(BlockBackend *blk, int argc, char **argv)
 {
-    bdrv_drain_all();
+    BlockAcctCookie cookie;
+    block_acct_start(blk_get_stats(blk), &cookie, 0, BLOCK_ACCT_FLUSH);
+    blk_drain_all();
+    block_acct_done(blk_get_stats(blk), &cookie);
     return 0;
 }
 
@@ -1615,9 +1512,9 @@
     .oneline    = "completes all outstanding aio requests"
 };
 
-static int flush_f(BlockDriverState *bs, int argc, char **argv)
+static int flush_f(BlockBackend *blk, int argc, char **argv)
 {
-    bdrv_flush(bs);
+    blk_flush(blk);
     return 0;
 }
 
@@ -1628,18 +1525,18 @@
     .oneline    = "flush all in-core file state to disk",
 };
 
-static int truncate_f(BlockDriverState *bs, int argc, char **argv)
+static int truncate_f(BlockBackend *blk, int argc, char **argv)
 {
     int64_t offset;
     int ret;
 
     offset = cvtnum(argv[1]);
     if (offset < 0) {
-        printf("non-numeric truncate argument -- %s\n", argv[1]);
+        print_cvtnum_err(offset, argv[1]);
         return 0;
     }
 
-    ret = bdrv_truncate(bs, offset);
+    ret = blk_truncate(blk, offset);
     if (ret < 0) {
         printf("truncate: %s\n", strerror(-ret));
         return 0;
@@ -1658,12 +1555,12 @@
     .oneline    = "truncates the current file at the given offset",
 };
 
-static int length_f(BlockDriverState *bs, int argc, char **argv)
+static int length_f(BlockBackend *blk, int argc, char **argv)
 {
     int64_t size;
     char s1[64];
 
-    size = bdrv_getlength(bs);
+    size = blk_getlength(blk);
     if (size < 0) {
         printf("getlength: %s\n", strerror(-size));
         return 0;
@@ -1683,8 +1580,9 @@
 };
 
 
-static int info_f(BlockDriverState *bs, int argc, char **argv)
+static int info_f(BlockBackend *blk, int argc, char **argv)
 {
+    BlockDriverState *bs = blk_bs(blk);
     BlockDriverInfo bdi;
     ImageInfoSpecific *spec_info;
     char s1[64], s2[64];
@@ -1742,7 +1640,7 @@
 "\n");
 }
 
-static int discard_f(BlockDriverState *bs, int argc, char **argv);
+static int discard_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t discard_cmd = {
     .name       = "discard",
@@ -1755,21 +1653,20 @@
     .help       = discard_help,
 };
 
-static int discard_f(BlockDriverState *bs, int argc, char **argv)
+static int discard_f(BlockBackend *blk, int argc, char **argv)
 {
     struct timeval t1, t2;
-    int Cflag = 0, qflag = 0;
+    bool Cflag = false, qflag = false;
     int c, ret;
-    int64_t offset;
-    int count;
+    int64_t offset, count;
 
-    while ((c = getopt(argc, argv, "Cq")) != EOF) {
+    while ((c = getopt(argc, argv, "Cq")) != -1) {
         switch (c) {
         case 'C':
-            Cflag = 1;
+            Cflag = true;
             break;
         case 'q':
-            qflag = 1;
+            qflag = true;
             break;
         default:
             return qemuio_command_usage(&discard_cmd);
@@ -1782,20 +1679,24 @@
 
     offset = cvtnum(argv[optind]);
     if (offset < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(offset, argv[optind]);
         return 0;
     }
 
     optind++;
     count = cvtnum(argv[optind]);
     if (count < 0) {
-        printf("non-numeric length argument -- %s\n", argv[optind]);
+        print_cvtnum_err(count, argv[optind]);
+        return 0;
+    } else if (count >> BDRV_SECTOR_BITS > BDRV_REQUEST_MAX_SECTORS) {
+        printf("length cannot exceed %"PRIu64", given %s\n",
+               (uint64_t)BDRV_REQUEST_MAX_SECTORS << BDRV_SECTOR_BITS,
+               argv[optind]);
         return 0;
     }
 
     gettimeofday(&t1, NULL);
-    ret = bdrv_discard(bs, offset >> BDRV_SECTOR_BITS,
-                       count >> BDRV_SECTOR_BITS);
+    ret = blk_pdiscard(blk, offset, count);
     gettimeofday(&t2, NULL);
 
     if (ret < 0) {
@@ -1813,17 +1714,17 @@
     return 0;
 }
 
-static int alloc_f(BlockDriverState *bs, int argc, char **argv)
+static int alloc_f(BlockBackend *blk, int argc, char **argv)
 {
-    int64_t offset, sector_num;
-    int nb_sectors, remaining;
+    BlockDriverState *bs = blk_bs(blk);
+    int64_t offset, sector_num, nb_sectors, remaining;
     char s1[64];
-    int num, sum_alloc;
-    int ret;
+    int num, ret;
+    int64_t sum_alloc;
 
     offset = cvtnum(argv[1]);
     if (offset < 0) {
-        printf("non-numeric offset argument -- %s\n", argv[1]);
+        print_cvtnum_err(offset, argv[1]);
         return 0;
     } else if (offset & 0x1ff) {
         printf("offset %" PRId64 " is not sector aligned\n",
@@ -1834,7 +1735,11 @@
     if (argc == 3) {
         nb_sectors = cvtnum(argv[2]);
         if (nb_sectors < 0) {
-            printf("non-numeric length argument -- %s\n", argv[2]);
+            print_cvtnum_err(nb_sectors, argv[2]);
+            return 0;
+        } else if (nb_sectors > INT_MAX) {
+            printf("length argument cannot exceed %d, given %s\n",
+                   INT_MAX, argv[2]);
             return 0;
         }
     } else {
@@ -1863,7 +1768,7 @@
 
     cvtstr(offset, s1, sizeof(s1));
 
-    printf("%d/%d sectors allocated at offset %s\n",
+    printf("%"PRId64"/%"PRId64" sectors allocated at offset %s\n",
            sum_alloc, nb_sectors, s1);
     return 0;
 }
@@ -1910,20 +1815,27 @@
     return firstret;
 }
 
-static int map_f(BlockDriverState *bs, int argc, char **argv)
+static int map_f(BlockBackend *blk, int argc, char **argv)
 {
     int64_t offset;
-    int64_t nb_sectors;
+    int64_t nb_sectors, total_sectors;
     char s1[64];
     int64_t num;
     int ret;
     const char *retstr;
 
     offset = 0;
-    nb_sectors = bs->total_sectors;
+    total_sectors = blk_nb_sectors(blk);
+    if (total_sectors < 0) {
+        error_report("Failed to query image length: %s",
+                     strerror(-total_sectors));
+        return 0;
+    }
+
+    nb_sectors = total_sectors;
 
     do {
-        ret = map_is_allocated(bs, offset, nb_sectors, &num);
+        ret = map_is_allocated(blk_bs(blk), offset, nb_sectors, &num);
         if (ret < 0) {
             error_report("Failed to get allocation status: %s", strerror(-ret));
             return 0;
@@ -1940,7 +1852,7 @@
 
         offset += num;
         nb_sectors -= num;
-    } while (offset < bs->total_sectors);
+    } while (offset < total_sectors);
 
     return 0;
 }
@@ -1954,11 +1866,111 @@
        .oneline        = "prints the allocated areas of a file",
 };
 
-static int break_f(BlockDriverState *bs, int argc, char **argv)
+static void reopen_help(void)
+{
+    printf(
+"\n"
+" Changes the open options of an already opened image\n"
+"\n"
+" Example:\n"
+" 'reopen -o lazy-refcounts=on' - activates lazy refcount writeback on a qcow2 image\n"
+"\n"
+" -r, -- Reopen the image read-only\n"
+" -c, -- Change the cache mode to the given value\n"
+" -o, -- Changes block driver options (cf. 'open' command)\n"
+"\n");
+}
+
+static int reopen_f(BlockBackend *blk, int argc, char **argv);
+
+static QemuOptsList reopen_opts = {
+    .name = "reopen",
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(reopen_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static const cmdinfo_t reopen_cmd = {
+       .name           = "reopen",
+       .argmin         = 0,
+       .argmax         = -1,
+       .cfunc          = reopen_f,
+       .args           = "[-r] [-c cache] [-o options]",
+       .oneline        = "reopens an image with new options",
+       .help           = reopen_help,
+};
+
+static int reopen_f(BlockBackend *blk, int argc, char **argv)
+{
+    BlockDriverState *bs = blk_bs(blk);
+    QemuOpts *qopts;
+    QDict *opts;
+    int c;
+    int flags = bs->open_flags;
+    bool writethrough = !blk_enable_write_cache(blk);
+
+    BlockReopenQueue *brq;
+    Error *local_err = NULL;
+
+    while ((c = getopt(argc, argv, "c:o:r")) != -1) {
+        switch (c) {
+        case 'c':
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) < 0) {
+                error_report("Invalid cache option: %s", optarg);
+                return 0;
+            }
+            break;
+        case 'o':
+            if (!qemu_opts_parse_noisily(&reopen_opts, optarg, 0)) {
+                qemu_opts_reset(&reopen_opts);
+                return 0;
+            }
+            break;
+        case 'r':
+            flags &= ~BDRV_O_RDWR;
+            break;
+        default:
+            qemu_opts_reset(&reopen_opts);
+            return qemuio_command_usage(&reopen_cmd);
+        }
+    }
+
+    if (optind != argc) {
+        qemu_opts_reset(&reopen_opts);
+        return qemuio_command_usage(&reopen_cmd);
+    }
+
+    if (writethrough != blk_enable_write_cache(blk) &&
+        blk_get_attached_dev(blk))
+    {
+        error_report("Cannot change cache.writeback: Device attached");
+        qemu_opts_reset(&reopen_opts);
+        return 0;
+    }
+
+    qopts = qemu_opts_find(&reopen_opts, NULL);
+    opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
+    qemu_opts_reset(&reopen_opts);
+
+    brq = bdrv_reopen_queue(NULL, bs, opts, flags);
+    bdrv_reopen_multiple(brq, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
+    } else {
+        blk_set_enable_write_cache(blk, !writethrough);
+    }
+
+    return 0;
+}
+
+static int break_f(BlockBackend *blk, int argc, char **argv)
 {
     int ret;
 
-    ret = bdrv_debug_breakpoint(bs, argv[1], argv[2]);
+    ret = bdrv_debug_breakpoint(blk_bs(blk), argv[1], argv[2]);
     if (ret < 0) {
         printf("Could not set breakpoint: %s\n", strerror(-ret));
     }
@@ -1966,11 +1978,11 @@
     return 0;
 }
 
-static int remove_break_f(BlockDriverState *bs, int argc, char **argv)
+static int remove_break_f(BlockBackend *blk, int argc, char **argv)
 {
     int ret;
 
-    ret = bdrv_debug_remove_breakpoint(bs, argv[1]);
+    ret = bdrv_debug_remove_breakpoint(blk_bs(blk), argv[1]);
     if (ret < 0) {
         printf("Could not remove breakpoint %s: %s\n", argv[1], strerror(-ret));
     }
@@ -1997,11 +2009,11 @@
        .oneline        = "remove a breakpoint by tag",
 };
 
-static int resume_f(BlockDriverState *bs, int argc, char **argv)
+static int resume_f(BlockBackend *blk, int argc, char **argv)
 {
     int ret;
 
-    ret = bdrv_debug_resume(bs, argv[1]);
+    ret = bdrv_debug_resume(blk_bs(blk), argv[1]);
     if (ret < 0) {
         printf("Could not resume request: %s\n", strerror(-ret));
     }
@@ -2018,10 +2030,10 @@
        .oneline        = "resumes the request tagged as tag",
 };
 
-static int wait_break_f(BlockDriverState *bs, int argc, char **argv)
+static int wait_break_f(BlockBackend *blk, int argc, char **argv)
 {
-    while (!bdrv_debug_is_suspended(bs, argv[1])) {
-        aio_poll(bdrv_get_aio_context(bs), true);
+    while (!bdrv_debug_is_suspended(blk_bs(blk), argv[1])) {
+        aio_poll(blk_get_aio_context(blk), true);
     }
 
     return 0;
@@ -2036,7 +2048,7 @@
        .oneline        = "waits for the suspension of a request",
 };
 
-static int abort_f(BlockDriverState *bs, int argc, char **argv)
+static int abort_f(BlockBackend *blk, int argc, char **argv)
 {
     abort();
 }
@@ -2048,13 +2060,62 @@
        .oneline        = "simulate a program crash using abort(3)",
 };
 
+static void sigraise_help(void)
+{
+    printf(
+"\n"
+" raises the given signal\n"
+"\n"
+" Example:\n"
+" 'sigraise %i' - raises SIGTERM\n"
+"\n"
+" Invokes raise(signal), where \"signal\" is the mandatory integer argument\n"
+" given to sigraise.\n"
+"\n", SIGTERM);
+}
+
+static int sigraise_f(BlockBackend *blk, int argc, char **argv);
+
+static const cmdinfo_t sigraise_cmd = {
+    .name       = "sigraise",
+    .cfunc      = sigraise_f,
+    .argmin     = 1,
+    .argmax     = 1,
+    .flags      = CMD_NOFILE_OK,
+    .args       = "signal",
+    .oneline    = "raises a signal",
+    .help       = sigraise_help,
+};
+
+static int sigraise_f(BlockBackend *blk, int argc, char **argv)
+{
+    int64_t sig = cvtnum(argv[1]);
+    if (sig < 0) {
+        print_cvtnum_err(sig, argv[1]);
+        return 0;
+    } else if (sig > NSIG) {
+        printf("signal argument '%s' is too large to be a valid signal\n",
+               argv[1]);
+        return 0;
+    }
+
+    /* Using raise() to kill this process does not necessarily flush all open
+     * streams. At least stdout and stderr (although the latter should be
+     * non-buffered anyway) should be flushed, though. */
+    fflush(stdout);
+    fflush(stderr);
+
+    raise(sig);
+    return 0;
+}
+
 static void sleep_cb(void *opaque)
 {
     bool *expired = opaque;
     *expired = true;
 }
 
-static int sleep_f(BlockDriverState *bs, int argc, char **argv)
+static int sleep_f(BlockBackend *blk, int argc, char **argv)
 {
     char *endptr;
     long ms;
@@ -2123,7 +2184,7 @@
     printf("\nUse 'help commandname' for extended help.\n");
 }
 
-static int help_f(BlockDriverState *bs, int argc, char **argv)
+static int help_f(BlockBackend *blk, int argc, char **argv)
 {
     const cmdinfo_t *ct;
 
@@ -2153,7 +2214,7 @@
     .oneline    = "help for one or all commands",
 };
 
-bool qemuio_command(BlockDriverState *bs, const char *cmd)
+bool qemuio_command(BlockBackend *blk, const char *cmd)
 {
     char *input;
     const cmdinfo_t *ct;
@@ -2166,7 +2227,7 @@
     if (c) {
         ct = find_command(v[0]);
         if (ct) {
-            done = command(bs, ct, c, v);
+            done = command(blk, ct, c, v);
         } else {
             fprintf(stderr, "command \"%s\" not found\n", v[0]);
         }
@@ -2185,7 +2246,6 @@
     qemuio_add_command(&readv_cmd);
     qemuio_add_command(&write_cmd);
     qemuio_add_command(&writev_cmd);
-    qemuio_add_command(&multiwrite_cmd);
     qemuio_add_command(&aio_read_cmd);
     qemuio_add_command(&aio_write_cmd);
     qemuio_add_command(&aio_flush_cmd);
@@ -2196,10 +2256,12 @@
     qemuio_add_command(&discard_cmd);
     qemuio_add_command(&alloc_cmd);
     qemuio_add_command(&map_cmd);
+    qemuio_add_command(&reopen_cmd);
     qemuio_add_command(&break_cmd);
     qemuio_add_command(&remove_break_cmd);
     qemuio_add_command(&resume_cmd);
     qemuio_add_command(&wait_break_cmd);
     qemuio_add_command(&abort_cmd);
     qemuio_add_command(&sleep_cmd);
+    qemuio_add_command(&sigraise_cmd);
 }
diff --git a/qemu-io.c b/qemu-io.c
index 60f84dd..db129ea 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -7,39 +7,41 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <sys/time.h>
-#include <sys/types.h>
-#include <stdarg.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <getopt.h>
 #include <libgen.h>
 
+#include "qapi/error.h"
 #include "qemu-io.h"
+#include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qemu/readline.h"
+#include "qemu/log.h"
+#include "qapi/qmp/qstring.h"
+#include "qom/object_interfaces.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "trace/control.h"
+#include "crypto/init.h"
 
 #define CMD_NOFILE_OK   0x01
 
 static char *progname;
 
 static BlockBackend *qemuio_blk;
-static BlockDriverState *qemuio_bs;
 
 /* qemu-io commands passed using -c */
 static int ncmdline;
 static char **cmdline;
+static bool imageOpts;
 
 static ReadLineState *readline_state;
 
-static int close_f(BlockDriverState *bs, int argc, char **argv)
+static int close_f(BlockBackend *blk, int argc, char **argv)
 {
     blk_unref(qemuio_blk);
-    qemuio_bs = NULL;
     qemuio_blk = NULL;
     return 0;
 }
@@ -51,35 +53,46 @@
     .oneline    = "close the current open file",
 };
 
-static int openfile(char *name, int flags, int growable, QDict *opts)
+static int openfile(char *name, int flags, bool writethrough, QDict *opts)
 {
     Error *local_err = NULL;
+    BlockDriverState *bs;
 
-    if (qemuio_bs) {
-        fprintf(stderr, "file open already, try 'help close'\n");
+    if (qemuio_blk) {
+        error_report("file open already, try 'help close'");
         QDECREF(opts);
         return 1;
     }
 
-    qemuio_blk = blk_new_with_bs("hda", &error_abort);
-    qemuio_bs = blk_bs(qemuio_blk);
-
-    if (growable) {
-        flags |= BDRV_O_PROTOCOL;
-    }
-
-    if (bdrv_open(&qemuio_bs, name, NULL, opts, flags, NULL, &local_err) < 0) {
-        fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
-                name ? " device " : "", name ?: "",
-                error_get_pretty(local_err));
-        error_free(local_err);
-        blk_unref(qemuio_blk);
-        qemuio_bs = NULL;
-        qemuio_blk = NULL;
+    qemuio_blk = blk_new_open(name, NULL, opts, flags, &local_err);
+    if (!qemuio_blk) {
+        error_reportf_err(local_err, "can't open%s%s: ",
+                          name ? " device " : "", name ?: "");
         return 1;
     }
 
+    bs = blk_bs(qemuio_blk);
+    if (bdrv_is_encrypted(bs) && bdrv_key_required(bs)) {
+        char password[256];
+        printf("Disk image '%s' is encrypted.\n", name);
+        if (qemu_read_password(password, sizeof(password)) < 0) {
+            error_report("No password given");
+            goto error;
+        }
+        if (bdrv_set_key(bs, password) < 0) {
+            error_report("invalid password");
+            goto error;
+        }
+    }
+
+    blk_set_enable_write_cache(qemuio_blk, !writethrough);
+
     return 0;
+
+ error:
+    blk_unref(qemuio_blk);
+    qemuio_blk = NULL;
+    return 1;
 }
 
 static void open_help(void)
@@ -89,18 +102,20 @@
 " opens a new file in the requested mode\n"
 "\n"
 " Example:\n"
-" 'open -Cn /tmp/data' - creates/opens data file read-write and uncached\n"
+" 'open -n -o driver=raw /tmp/data' - opens raw data file read-write, uncached\n"
 "\n"
 " Opens a file for subsequent use by all of the other qemu-io commands.\n"
 " -r, -- open file read-only\n"
 " -s, -- use snapshot file\n"
-" -n, -- disable host cache\n"
-" -g, -- allow file to grow (only applies to protocols)\n"
+" -n, -- disable host cache, short for -t none\n"
+" -k, -- use kernel AIO implementation (on Linux only)\n"
+" -t, -- use the given cache mode for the image\n"
+" -d, -- use the given discard mode for the image\n"
 " -o, -- options to be given to the block driver"
 "\n");
 }
 
-static int open_f(BlockDriverState *bs, int argc, char **argv);
+static int open_f(BlockBackend *blk, int argc, char **argv);
 
 static const cmdinfo_t open_cmd = {
     .name       = "open",
@@ -109,7 +124,7 @@
     .argmin     = 1,
     .argmax     = -1,
     .flags      = CMD_NOFILE_OK,
-    .args       = "[-Crsn] [-o options] [path]",
+    .args       = "[-rsnk] [-t cache] [-d discard] [-o options] [path]",
     .oneline    = "open the file specified by path",
     .help       = open_help,
 };
@@ -124,32 +139,51 @@
     },
 };
 
-static int open_f(BlockDriverState *bs, int argc, char **argv)
+static int open_f(BlockBackend *blk, int argc, char **argv)
 {
-    int flags = 0;
+    int flags = BDRV_O_UNMAP;
     int readonly = 0;
-    int growable = 0;
+    bool writethrough = true;
     int c;
     QemuOpts *qopts;
     QDict *opts;
 
-    while ((c = getopt(argc, argv, "snrgo:")) != EOF) {
+    while ((c = getopt(argc, argv, "snro:kt:d:")) != -1) {
         switch (c) {
         case 's':
             flags |= BDRV_O_SNAPSHOT;
             break;
         case 'n':
-            flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+            flags |= BDRV_O_NOCACHE;
+            writethrough = false;
             break;
         case 'r':
             readonly = 1;
             break;
-        case 'g':
-            growable = 1;
+        case 'k':
+            flags |= BDRV_O_NATIVE_AIO;
+            break;
+        case 't':
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) < 0) {
+                error_report("Invalid cache option: %s", optarg);
+                qemu_opts_reset(&empty_opts);
+                return 0;
+            }
+            break;
+        case 'd':
+            if (bdrv_parse_discard_flags(optarg, &flags) < 0) {
+                error_report("Invalid discard option: %s", optarg);
+                qemu_opts_reset(&empty_opts);
+                return 0;
+            }
             break;
         case 'o':
-            if (!qemu_opts_parse(&empty_opts, optarg, 0)) {
-                printf("could not parse option list -- %s\n", optarg);
+            if (imageOpts) {
+                printf("--image-opts and 'open -o' are mutually exclusive\n");
+                qemu_opts_reset(&empty_opts);
+                return 0;
+            }
+            if (!qemu_opts_parse_noisily(&empty_opts, optarg, false)) {
                 qemu_opts_reset(&empty_opts);
                 return 0;
             }
@@ -164,21 +198,29 @@
         flags |= BDRV_O_RDWR;
     }
 
+    if (imageOpts && (optind == argc - 1)) {
+        if (!qemu_opts_parse_noisily(&empty_opts, argv[optind], false)) {
+            qemu_opts_reset(&empty_opts);
+            return 0;
+        }
+        optind++;
+    }
+
     qopts = qemu_opts_find(&empty_opts, NULL);
     opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
     qemu_opts_reset(&empty_opts);
 
     if (optind == argc - 1) {
-        return openfile(argv[optind], flags, growable, opts);
+        return openfile(argv[optind], flags, writethrough, opts);
     } else if (optind == argc) {
-        return openfile(NULL, flags, growable, opts);
+        return openfile(NULL, flags, writethrough, opts);
     } else {
         QDECREF(opts);
         return qemuio_command_usage(&open_cmd);
     }
 }
 
-static int quit_f(BlockDriverState *bs, int argc, char **argv)
+static int quit_f(BlockBackend *blk, int argc, char **argv)
 {
     return 1;
 }
@@ -196,19 +238,25 @@
 static void usage(const char *name)
 {
     printf(
-"Usage: %s [-h] [-V] [-rsnm] [-c STRING] ... [file]\n"
+"Usage: %s [OPTIONS]... [-c STRING]... [file]\n"
 "QEMU Disk exerciser\n"
 "\n"
+"  --object OBJECTDEF   define an object such as 'secret' for\n"
+"                       passwords and/or encryption keys\n"
+"  --image-opts         treat file as option string\n"
 "  -c, --cmd STRING     execute command with its arguments\n"
 "                       from the given string\n"
+"  -f, --format FMT     specifies the block driver to use\n"
 "  -r, --read-only      export read-only\n"
 "  -s, --snapshot       use snapshot file\n"
-"  -n, --nocache        disable host cache\n"
-"  -g, --growable       allow file to grow (only applies to protocols)\n"
+"  -n, --nocache        disable host cache, short for -t none\n"
 "  -m, --misalign       misalign allocations for O_DIRECT\n"
 "  -k, --native-aio     use kernel AIO implementation (on Linux only)\n"
 "  -t, --cache=MODE     use the given cache mode for the image\n"
-"  -T, --trace FILE     enable trace events listed in the given file\n"
+"  -d, --discard=MODE   use the given discard mode for the image\n"
+"  -T, --trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
+"                       specify tracing options\n"
+"                       see qemu-img(1) man page for full description\n"
 "  -h, --help           display this help and exit\n"
 "  -V, --version        output version information and exit\n"
 "\n"
@@ -315,7 +363,7 @@
     char *input;
 
     for (i = 0; !done && i < ncmdline; i++) {
-        done = qemuio_command(qemuio_bs, cmdline[i]);
+        done = qemuio_command(qemuio_blk, cmdline[i]);
     }
     if (cmdline) {
         g_free(cmdline);
@@ -340,7 +388,7 @@
         if (input == NULL) {
             break;
         }
-        done = qemuio_command(qemuio_bs, input);
+        done = qemuio_command(qemuio_blk, input);
         g_free(input);
 
         prompted = 0;
@@ -360,31 +408,60 @@
     qemu_set_tty_echo(STDIN_FILENO, true);
 }
 
+enum {
+    OPTION_OBJECT = 256,
+    OPTION_IMAGE_OPTS = 257,
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+
+static QemuOptsList file_opts = {
+    .name = "file",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(file_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
 int main(int argc, char **argv)
 {
     int readonly = 0;
-    int growable = 0;
-    const char *sopt = "hVc:d:rsnmgkt:T:";
+    const char *sopt = "hVc:d:f:rsnmkt:T:";
     const struct option lopt[] = {
-        { "help", 0, NULL, 'h' },
-        { "version", 0, NULL, 'V' },
-        { "offset", 1, NULL, 'o' },
-        { "cmd", 1, NULL, 'c' },
-        { "read-only", 0, NULL, 'r' },
-        { "snapshot", 0, NULL, 's' },
-        { "nocache", 0, NULL, 'n' },
-        { "misalign", 0, NULL, 'm' },
-        { "growable", 0, NULL, 'g' },
-        { "native-aio", 0, NULL, 'k' },
-        { "discard", 1, NULL, 'd' },
-        { "cache", 1, NULL, 't' },
-        { "trace", 1, NULL, 'T' },
+        { "help", no_argument, NULL, 'h' },
+        { "version", no_argument, NULL, 'V' },
+        { "cmd", required_argument, NULL, 'c' },
+        { "format", required_argument, NULL, 'f' },
+        { "read-only", no_argument, NULL, 'r' },
+        { "snapshot", no_argument, NULL, 's' },
+        { "nocache", no_argument, NULL, 'n' },
+        { "misalign", no_argument, NULL, 'm' },
+        { "native-aio", no_argument, NULL, 'k' },
+        { "discard", required_argument, NULL, 'd' },
+        { "cache", required_argument, NULL, 't' },
+        { "trace", required_argument, NULL, 'T' },
+        { "object", required_argument, NULL, OPTION_OBJECT },
+        { "image-opts", no_argument, NULL, OPTION_IMAGE_OPTS },
         { NULL, 0, NULL, 0 }
     };
     int c;
     int opt_index = 0;
     int flags = BDRV_O_UNMAP;
+    bool writethrough = true;
     Error *local_error = NULL;
+    QDict *opts = NULL;
+    const char *format = NULL;
+    char *trace_file = NULL;
 
 #ifdef CONFIG_POSIX
     signal(SIGPIPE, SIG_IGN);
@@ -393,13 +470,21 @@
     progname = basename(argv[0]);
     qemu_init_exec_dir(argv[0]);
 
+    qcrypto_init(&error_fatal);
+
+    module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_object_opts);
+    qemu_add_opts(&qemu_trace_opts);
+    bdrv_init();
+
     while ((c = getopt_long(argc, argv, sopt, lopt, &opt_index)) != -1) {
         switch (c) {
         case 's':
             flags |= BDRV_O_SNAPSHOT;
             break;
         case 'n':
-            flags |= BDRV_O_NOCACHE | BDRV_O_CACHE_WB;
+            flags |= BDRV_O_NOCACHE;
+            writethrough = false;
             break;
         case 'd':
             if (bdrv_parse_discard_flags(optarg, &flags) < 0) {
@@ -407,6 +492,9 @@
                 exit(1);
             }
             break;
+        case 'f':
+            format = optarg;
+            break;
         case 'c':
             add_user_command(optarg);
             break;
@@ -416,22 +504,18 @@
         case 'm':
             qemuio_misalign = true;
             break;
-        case 'g':
-            growable = 1;
-            break;
         case 'k':
             flags |= BDRV_O_NATIVE_AIO;
             break;
         case 't':
-            if (bdrv_parse_cache_flags(optarg, &flags) < 0) {
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) < 0) {
                 error_report("Invalid cache option: %s", optarg);
                 exit(1);
             }
             break;
         case 'T':
-            if (!trace_init_backends(optarg, NULL)) {
-                exit(1); /* error message will have been printed */
-            }
+            g_free(trace_file);
+            trace_file = trace_opt_parse(optarg);
             break;
         case 'V':
             printf("%s version %s\n", progname, QEMU_VERSION);
@@ -439,6 +523,17 @@
         case 'h':
             usage(progname);
             exit(0);
+        case OPTION_OBJECT: {
+            QemuOpts *qopts;
+            qopts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                            optarg, true);
+            if (!qopts) {
+                exit(1);
+            }
+        }   break;
+        case OPTION_IMAGE_OPTS:
+            imageOpts = true;
+            break;
         default:
             usage(progname);
             exit(1);
@@ -450,12 +545,27 @@
         exit(1);
     }
 
-    if (qemu_init_main_loop(&local_error)) {
-        error_report("%s", error_get_pretty(local_error));
-        error_free(local_error);
+    if (format && imageOpts) {
+        error_report("--image-opts and -f are mutually exclusive");
         exit(1);
     }
-    bdrv_init();
+
+    if (qemu_init_main_loop(&local_error)) {
+        error_report_err(local_error);
+        exit(1);
+    }
+
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        exit(1);
+    }
+
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+    qemu_set_log(LOG_TRACE);
 
     /* initialize commands */
     qemuio_add_command(&quit_cmd);
@@ -477,7 +587,21 @@
     }
 
     if ((argc - optind) == 1) {
-        openfile(argv[optind], flags, growable, NULL);
+        if (imageOpts) {
+            QemuOpts *qopts = NULL;
+            qopts = qemu_opts_parse_noisily(&file_opts, argv[optind], false);
+            if (!qopts) {
+                exit(1);
+            }
+            opts = qemu_opts_to_qdict(qopts, NULL);
+            openfile(NULL, flags, writethrough, opts);
+        } else {
+            if (format) {
+                opts = qdict_new();
+                qdict_put(opts, "driver", qstring_from_str(format));
+            }
+            openfile(argv[optind], flags, writethrough, opts);
+        }
     }
     command_loop();
 
diff --git a/qemu-log.c b/qemu-log.c
deleted file mode 100644
index 797f2af..0000000
--- a/qemu-log.c
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Logging support
- *
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "qemu-common.h"
-#include "qemu/log.h"
-
-static char *logfilename;
-FILE *qemu_logfile;
-int qemu_loglevel;
-static int log_append = 0;
-
-void qemu_log(const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    if (qemu_logfile) {
-        vfprintf(qemu_logfile, fmt, ap);
-    }
-    va_end(ap);
-}
-
-void qemu_log_mask(int mask, const char *fmt, ...)
-{
-    va_list ap;
-
-    va_start(ap, fmt);
-    if ((qemu_loglevel & mask) && qemu_logfile) {
-        vfprintf(qemu_logfile, fmt, ap);
-    }
-    va_end(ap);
-}
-
-/* enable or disable low levels log */
-void do_qemu_set_log(int log_flags, bool use_own_buffers)
-{
-    qemu_loglevel = log_flags;
-    if (qemu_loglevel && !qemu_logfile) {
-        if (logfilename) {
-            qemu_logfile = fopen(logfilename, log_append ? "a" : "w");
-            if (!qemu_logfile) {
-                perror(logfilename);
-                _exit(1);
-            }
-        } else {
-            /* Default to stderr if no log file specified */
-            qemu_logfile = stderr;
-        }
-        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
-        if (use_own_buffers) {
-            static char logfile_buf[4096];
-
-            setvbuf(qemu_logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
-        } else {
-#if defined(_WIN32)
-            /* Win32 doesn't support line-buffering, so use unbuffered output. */
-            setvbuf(qemu_logfile, NULL, _IONBF, 0);
-#else
-            setvbuf(qemu_logfile, NULL, _IOLBF, 0);
-#endif
-            log_append = 1;
-        }
-    }
-    if (!qemu_loglevel && qemu_logfile) {
-        qemu_log_close();
-    }
-}
-
-void qemu_set_log_filename(const char *filename)
-{
-    g_free(logfilename);
-    logfilename = g_strdup(filename);
-    qemu_log_close();
-    qemu_set_log(qemu_loglevel);
-}
-
-const QEMULogItem qemu_log_items[] = {
-    { CPU_LOG_TB_OUT_ASM, "out_asm",
-      "show generated host assembly code for each compiled TB" },
-    { CPU_LOG_TB_IN_ASM, "in_asm",
-      "show target assembly code for each compiled TB" },
-    { CPU_LOG_TB_OP, "op",
-      "show micro ops for each compiled TB" },
-    { CPU_LOG_TB_OP_OPT, "op_opt",
-      "show micro ops (x86 only: before eflags optimization) and\n"
-      "after liveness analysis" },
-    { CPU_LOG_INT, "int",
-      "show interrupts/exceptions in short format" },
-    { CPU_LOG_EXEC, "exec",
-      "show trace before each executed TB (lots of logs)" },
-    { CPU_LOG_TB_CPU, "cpu",
-      "show CPU state before block translation" },
-    { CPU_LOG_PCALL, "pcall",
-      "x86 only: show protected mode far calls/returns/exceptions" },
-    { CPU_LOG_RESET, "cpu_reset",
-      "x86 only: show CPU state before CPU resets" },
-    { CPU_LOG_IOPORT, "ioport",
-      "show all i/o ports accesses" },
-    { LOG_UNIMP, "unimp",
-      "log unimplemented functionality" },
-    { LOG_GUEST_ERROR, "guest_errors",
-      "log when the guest OS does something invalid (eg accessing a\n"
-      "non-existent register)" },
-    { 0, NULL, NULL },
-};
-
-static int cmp1(const char *s1, int n, const char *s2)
-{
-    if (strlen(s2) != n) {
-        return 0;
-    }
-    return memcmp(s1, s2, n) == 0;
-}
-
-/* takes a comma separated list of log masks. Return 0 if error. */
-int qemu_str_to_log_mask(const char *str)
-{
-    const QEMULogItem *item;
-    int mask;
-    const char *p, *p1;
-
-    p = str;
-    mask = 0;
-    for (;;) {
-        p1 = strchr(p, ',');
-        if (!p1) {
-            p1 = p + strlen(p);
-        }
-        if (cmp1(p,p1-p,"all")) {
-            for (item = qemu_log_items; item->mask != 0; item++) {
-                mask |= item->mask;
-            }
-        } else {
-            for (item = qemu_log_items; item->mask != 0; item++) {
-                if (cmp1(p, p1 - p, item->name)) {
-                    goto found;
-                }
-            }
-            return 0;
-        }
-    found:
-        mask |= item->mask;
-        if (*p1 != ',') {
-            break;
-        }
-        p = p1 + 1;
-    }
-    return mask;
-}
-
-void qemu_print_log_usage(FILE *f)
-{
-    const QEMULogItem *item;
-    fprintf(f, "Log items (comma separated):\n");
-    for (item = qemu_log_items; item->mask != 0; item++) {
-        fprintf(f, "%-10s %s\n", item->name, item->help);
-    }
-}
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 5cd6c6d..e3571c2 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -16,43 +16,53 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "sysemu/block-backend.h"
 #include "block/block_int.h"
 #include "block/nbd.h"
 #include "qemu/main-loop.h"
-#include "qemu/sockets.h"
 #include "qemu/error-report.h"
+#include "qemu/config-file.h"
+#include "qemu/bswap.h"
+#include "qemu/log.h"
 #include "block/snapshot.h"
 #include "qapi/util.h"
+#include "qapi/qmp/qstring.h"
+#include "qom/object_interfaces.h"
+#include "io/channel-socket.h"
+#include "crypto/init.h"
+#include "trace/control.h"
 
-#include <stdarg.h>
-#include <stdio.h>
 #include <getopt.h>
-#include <err.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <netinet/tcp.h>
-#include <arpa/inet.h>
-#include <signal.h>
 #include <libgen.h>
 #include <pthread.h>
 
 #define SOCKET_PATH                "/var/lock/qemu-nbd-%s"
-#define QEMU_NBD_OPT_CACHE         1
-#define QEMU_NBD_OPT_AIO           2
-#define QEMU_NBD_OPT_DISCARD       3
-#define QEMU_NBD_OPT_DETECT_ZEROES 4
+#define QEMU_NBD_OPT_CACHE         256
+#define QEMU_NBD_OPT_AIO           257
+#define QEMU_NBD_OPT_DISCARD       258
+#define QEMU_NBD_OPT_DETECT_ZEROES 259
+#define QEMU_NBD_OPT_OBJECT        260
+#define QEMU_NBD_OPT_TLSCREDS      261
+#define QEMU_NBD_OPT_IMAGE_OPTS    262
+
+#define MBR_SIZE 512
 
 static NBDExport *exp;
+static bool newproto;
 static int verbose;
 static char *srcpath;
-static char *sockpath;
+static SocketAddress *saddr;
 static int persistent = 0;
 static enum { RUNNING, TERMINATE, TERMINATING, TERMINATED } state;
 static int shared = 1;
 static int nb_fds;
+static QIOChannelSocket *server_ioc;
+static int server_watch = -1;
+static QCryptoTLSCreds *tlscreds;
 
 static void usage(const char *name)
 {
@@ -71,11 +81,17 @@
 "  -e, --shared=NUM          device can be shared by NUM clients (default '1')\n"
 "  -t, --persistent          don't exit on the last connection\n"
 "  -v, --verbose             display extra debugging information\n"
+"  -x, --export-name=NAME    expose export by name\n"
 "\n"
 "Exposing part of the image:\n"
 "  -o, --offset=OFFSET       offset into the image\n"
 "  -P, --partition=NUM       only expose partition NUM\n"
 "\n"
+"General purpose options:\n"
+"  --object type,id=ID,...   define an object such as 'secret' for providing\n"
+"                            passwords and/or encryption keys\n"
+"  -T, --trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
+"                            specify tracing options\n"
 #ifdef __linux__
 "Kernel NBD client support:\n"
 "  -c, --connect=DEV         connect FILE to the local NBD device DEV\n"
@@ -96,11 +112,10 @@
 "                            '[ID_OR_NAME]'\n"
 "  -n, --nocache             disable host cache\n"
 "      --cache=MODE          set cache mode (none, writeback, ...)\n"
-#ifdef CONFIG_LINUX_AIO
 "      --aio=MODE            set AIO mode (native or threads)\n"
-#endif
 "      --discard=MODE        set discard mode (ignore, unmap)\n"
-"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, discard)\n"
+"      --detect-zeroes=MODE  set detect-zeroes mode (off, on, unmap)\n"
+"      --image-opts          treat FILE as a full set of image options\n"
 "\n"
 "Report bugs to <qemu-devel@nongnu.org>\n"
     , name, NBD_DEFAULT_PORT, "DEVICE");
@@ -142,22 +157,24 @@
     r->end_head = p[5];
     r->end_cylinder = p[7] | ((p[6] << 2) & 0x300);
     r->end_sector = p[6] & 0x3f;
-    r->start_sector_abs = p[8] | p[9] << 8 | p[10] << 16 | p[11] << 24;
-    r->nb_sectors_abs = p[12] | p[13] << 8 | p[14] << 16 | p[15] << 24;
+
+    r->start_sector_abs = ldl_le_p(p + 8);
+    r->nb_sectors_abs   = ldl_le_p(p + 12);
 }
 
-static int find_partition(BlockDriverState *bs, int partition,
+static int find_partition(BlockBackend *blk, int partition,
                           off_t *offset, off_t *size)
 {
     struct partition_record mbr[4];
-    uint8_t data[512];
+    uint8_t data[MBR_SIZE];
     int i;
     int ext_partnum = 4;
     int ret;
 
-    if ((ret = bdrv_read(bs, 0, data, 1)) < 0) {
-        errno = -ret;
-        err(EXIT_FAILURE, "error while reading");
+    ret = blk_pread(blk, 0, data, sizeof(data));
+    if (ret < 0) {
+        error_report("error while reading: %s", strerror(-ret));
+        exit(EXIT_FAILURE);
     }
 
     if (data[510] != 0x55 || data[511] != 0xaa) {
@@ -167,23 +184,27 @@
     for (i = 0; i < 4; i++) {
         read_partition(&data[446 + 16 * i], &mbr[i]);
 
-        if (!mbr[i].nb_sectors_abs)
+        if (!mbr[i].system || !mbr[i].nb_sectors_abs) {
             continue;
+        }
 
         if (mbr[i].system == 0xF || mbr[i].system == 0x5) {
             struct partition_record ext[4];
-            uint8_t data1[512];
+            uint8_t data1[MBR_SIZE];
             int j;
 
-            if ((ret = bdrv_read(bs, mbr[i].start_sector_abs, data1, 1)) < 0) {
-                errno = -ret;
-                err(EXIT_FAILURE, "error while reading");
+            ret = blk_pread(blk, mbr[i].start_sector_abs * MBR_SIZE,
+                            data1, sizeof(data1));
+            if (ret < 0) {
+                error_report("error while reading: %s", strerror(-ret));
+                exit(EXIT_FAILURE);
             }
 
             for (j = 0; j < 4; j++) {
                 read_partition(&data1[446 + 16 * j], &ext[j]);
-                if (!ext[j].nb_sectors_abs)
+                if (!ext[j].system || !ext[j].nb_sectors_abs) {
                     continue;
+                }
 
                 if ((ext_partnum + j + 1) == partition) {
                     *offset = (uint64_t)ext[j].start_sector_abs << 9;
@@ -204,59 +225,10 @@
 
 static void termsig_handler(int signum)
 {
-    state = TERMINATE;
+    atomic_cmpxchg(&state, RUNNING, TERMINATE);
     qemu_notify_event();
 }
 
-static void combine_addr(char *buf, size_t len, const char* address,
-                         uint16_t port)
-{
-    /* If the address-part contains a colon, it's an IPv6 IP so needs [] */
-    if (strstr(address, ":")) {
-        snprintf(buf, len, "[%s]:%u", address, port);
-    } else {
-        snprintf(buf, len, "%s:%u", address, port);
-    }
-}
-
-static int tcp_socket_incoming(const char *address, uint16_t port)
-{
-    char address_and_port[128];
-    Error *local_err = NULL;
-
-    combine_addr(address_and_port, 128, address, port);
-    int fd = inet_listen(address_and_port, NULL, 0, SOCK_STREAM, 0, &local_err);
-
-    if (local_err != NULL) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
-    }
-    return fd;
-}
-
-static int unix_socket_incoming(const char *path)
-{
-    Error *local_err = NULL;
-    int fd = unix_listen(path, NULL, 0, &local_err);
-
-    if (local_err != NULL) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
-    }
-    return fd;
-}
-
-static int unix_socket_outgoing(const char *path)
-{
-    Error *local_err = NULL;
-    int fd = unix_connect(path, &local_err);
-
-    if (local_err != NULL) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
-    }
-    return fd;
-}
 
 static void *show_parts(void *arg)
 {
@@ -279,31 +251,39 @@
 {
     char *device = arg;
     off_t size;
-    size_t blocksize;
-    uint32_t nbdflags;
-    int fd, sock;
+    uint16_t nbdflags;
+    QIOChannelSocket *sioc;
+    int fd;
     int ret;
     pthread_t show_parts_thread;
+    Error *local_error = NULL;
 
-    sock = unix_socket_outgoing(sockpath);
-    if (sock < 0) {
+    sioc = qio_channel_socket_new();
+    if (qio_channel_socket_connect_sync(sioc,
+                                        saddr,
+                                        &local_error) < 0) {
+        error_report_err(local_error);
         goto out;
     }
 
-    ret = nbd_receive_negotiate(sock, NULL, &nbdflags,
-                                &size, &blocksize);
+    ret = nbd_receive_negotiate(QIO_CHANNEL(sioc), NULL, &nbdflags,
+                                NULL, NULL, NULL,
+                                &size, &local_error);
     if (ret < 0) {
+        if (local_error) {
+            error_report_err(local_error);
+        }
         goto out_socket;
     }
 
     fd = open(device, O_RDWR);
     if (fd < 0) {
         /* Linux-only, we can use %m in printf.  */
-        fprintf(stderr, "Failed to open %s: %m\n", device);
+        error_report("Failed to open %s: %m", device);
         goto out_socket;
     }
 
-    ret = nbd_init(fd, sock, nbdflags, size, blocksize);
+    ret = nbd_init(fd, sioc, nbdflags, size);
     if (ret < 0) {
         goto out_fd;
     }
@@ -324,19 +304,20 @@
         goto out_fd;
     }
     close(fd);
+    object_unref(OBJECT(sioc));
     kill(getpid(), SIGTERM);
     return (void *) EXIT_SUCCESS;
 
 out_fd:
     close(fd);
 out_socket:
-    closesocket(sock);
+    object_unref(OBJECT(sioc));
 out:
     kill(getpid(), SIGTERM);
     return (void *) EXIT_FAILURE;
 }
 
-static int nbd_can_accept(void *opaque)
+static int nbd_can_accept(void)
 {
     return nb_fds < shared;
 }
@@ -347,99 +328,202 @@
     state = TERMINATED;
 }
 
+static void nbd_update_server_watch(void);
+
 static void nbd_client_closed(NBDClient *client)
 {
     nb_fds--;
     if (nb_fds == 0 && !persistent && state == RUNNING) {
         state = TERMINATE;
     }
-    qemu_notify_event();
+    nbd_update_server_watch();
     nbd_client_put(client);
 }
 
-static void nbd_accept(void *opaque)
+static gboolean nbd_accept(QIOChannel *ioc, GIOCondition cond, gpointer opaque)
 {
-    int server_fd = (uintptr_t) opaque;
-    struct sockaddr_in addr;
-    socklen_t addr_len = sizeof(addr);
+    QIOChannelSocket *cioc;
 
-    int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    if (fd < 0) {
-        perror("accept");
-        return;
+    cioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc),
+                                     NULL);
+    if (!cioc) {
+        return TRUE;
     }
 
     if (state >= TERMINATE) {
-        close(fd);
-        return;
+        object_unref(OBJECT(cioc));
+        return TRUE;
     }
 
-    if (nbd_client_new(exp, fd, nbd_client_closed)) {
-        nb_fds++;
+    nb_fds++;
+    nbd_update_server_watch();
+    nbd_client_new(newproto ? NULL : exp, cioc,
+                   tlscreds, NULL, nbd_client_closed);
+    object_unref(OBJECT(cioc));
+
+    return TRUE;
+}
+
+static void nbd_update_server_watch(void)
+{
+    if (nbd_can_accept()) {
+        if (server_watch == -1) {
+            server_watch = qio_channel_add_watch(QIO_CHANNEL(server_ioc),
+                                                 G_IO_IN,
+                                                 nbd_accept,
+                                                 NULL, NULL);
+        }
     } else {
-        shutdown(fd, 2);
-        close(fd);
+        if (server_watch != -1) {
+            g_source_remove(server_watch);
+            server_watch = -1;
+        }
     }
 }
 
+
+static SocketAddress *nbd_build_socket_address(const char *sockpath,
+                                               const char *bindto,
+                                               const char *port)
+{
+    SocketAddress *saddr;
+
+    saddr = g_new0(SocketAddress, 1);
+    if (sockpath) {
+        saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+        saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+        saddr->u.q_unix.data->path = g_strdup(sockpath);
+    } else {
+        InetSocketAddress *inet;
+        saddr->type = SOCKET_ADDRESS_KIND_INET;
+        inet = saddr->u.inet.data = g_new0(InetSocketAddress, 1);
+        inet->host = g_strdup(bindto);
+        if (port) {
+            inet->port = g_strdup(port);
+        } else  {
+            inet->port = g_strdup_printf("%d", NBD_DEFAULT_PORT);
+        }
+    }
+
+    return saddr;
+}
+
+
+static QemuOptsList file_opts = {
+    .name = "file",
+    .implied_opt_name = "file",
+    .head = QTAILQ_HEAD_INITIALIZER(file_opts.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
+
+
+static QCryptoTLSCreds *nbd_get_tls_creds(const char *id, Error **errp)
+{
+    Object *obj;
+    QCryptoTLSCreds *creds;
+
+    obj = object_resolve_path_component(
+        object_get_objects_root(), id);
+    if (!obj) {
+        error_setg(errp, "No TLS credentials with id '%s'",
+                   id);
+        return NULL;
+    }
+    creds = (QCryptoTLSCreds *)
+        object_dynamic_cast(obj, TYPE_QCRYPTO_TLS_CREDS);
+    if (!creds) {
+        error_setg(errp, "Object with id '%s' is not TLS credentials",
+                   id);
+        return NULL;
+    }
+
+    if (creds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+        error_setg(errp,
+                   "Expecting TLS credentials with a server endpoint");
+        return NULL;
+    }
+    object_ref(obj);
+    return creds;
+}
+
+
 int main(int argc, char **argv)
 {
     BlockBackend *blk;
     BlockDriverState *bs;
-    BlockDriver *drv;
     off_t dev_offset = 0;
-    uint32_t nbdflags = 0;
+    uint16_t nbdflags = 0;
     bool disconnect = false;
     const char *bindto = "0.0.0.0";
+    const char *port = NULL;
+    char *sockpath = NULL;
     char *device = NULL;
-    int port = NBD_DEFAULT_PORT;
     off_t fd_size;
     QemuOpts *sn_opts = NULL;
     const char *sn_id_or_name = NULL;
-    const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:";
+    const char *sopt = "hVb:o:p:rsnP:c:dvk:e:f:tl:x:T:";
     struct option lopt[] = {
-        { "help", 0, NULL, 'h' },
-        { "version", 0, NULL, 'V' },
-        { "bind", 1, NULL, 'b' },
-        { "port", 1, NULL, 'p' },
-        { "socket", 1, NULL, 'k' },
-        { "offset", 1, NULL, 'o' },
-        { "read-only", 0, NULL, 'r' },
-        { "partition", 1, NULL, 'P' },
-        { "connect", 1, NULL, 'c' },
-        { "disconnect", 0, NULL, 'd' },
-        { "snapshot", 0, NULL, 's' },
-        { "load-snapshot", 1, NULL, 'l' },
-        { "nocache", 0, NULL, 'n' },
-        { "cache", 1, NULL, QEMU_NBD_OPT_CACHE },
-#ifdef CONFIG_LINUX_AIO
-        { "aio", 1, NULL, QEMU_NBD_OPT_AIO },
-#endif
-        { "discard", 1, NULL, QEMU_NBD_OPT_DISCARD },
-        { "detect-zeroes", 1, NULL, QEMU_NBD_OPT_DETECT_ZEROES },
-        { "shared", 1, NULL, 'e' },
-        { "format", 1, NULL, 'f' },
-        { "persistent", 0, NULL, 't' },
-        { "verbose", 0, NULL, 'v' },
+        { "help", no_argument, NULL, 'h' },
+        { "version", no_argument, NULL, 'V' },
+        { "bind", required_argument, NULL, 'b' },
+        { "port", required_argument, NULL, 'p' },
+        { "socket", required_argument, NULL, 'k' },
+        { "offset", required_argument, NULL, 'o' },
+        { "read-only", no_argument, NULL, 'r' },
+        { "partition", required_argument, NULL, 'P' },
+        { "connect", required_argument, NULL, 'c' },
+        { "disconnect", no_argument, NULL, 'd' },
+        { "snapshot", no_argument, NULL, 's' },
+        { "load-snapshot", required_argument, NULL, 'l' },
+        { "nocache", no_argument, NULL, 'n' },
+        { "cache", required_argument, NULL, QEMU_NBD_OPT_CACHE },
+        { "aio", required_argument, NULL, QEMU_NBD_OPT_AIO },
+        { "discard", required_argument, NULL, QEMU_NBD_OPT_DISCARD },
+        { "detect-zeroes", required_argument, NULL,
+          QEMU_NBD_OPT_DETECT_ZEROES },
+        { "shared", required_argument, NULL, 'e' },
+        { "format", required_argument, NULL, 'f' },
+        { "persistent", no_argument, NULL, 't' },
+        { "verbose", no_argument, NULL, 'v' },
+        { "object", required_argument, NULL, QEMU_NBD_OPT_OBJECT },
+        { "export-name", required_argument, NULL, 'x' },
+        { "tls-creds", required_argument, NULL, QEMU_NBD_OPT_TLSCREDS },
+        { "image-opts", no_argument, NULL, QEMU_NBD_OPT_IMAGE_OPTS },
+        { "trace", required_argument, NULL, 'T' },
         { NULL, 0, NULL, 0 }
     };
     int ch;
     int opt_ind = 0;
-    int li;
     char *end;
     int flags = BDRV_O_RDWR;
     int partition = -1;
-    int ret;
-    int fd;
+    int ret = 0;
     bool seen_cache = false;
     bool seen_discard = false;
-#ifdef CONFIG_LINUX_AIO
     bool seen_aio = false;
-#endif
     pthread_t client_thread;
     const char *fmt = NULL;
     Error *local_err = NULL;
     BlockdevDetectZeroesOptions detect_zeroes = BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF;
+    QDict *options = NULL;
+    const char *export_name = NULL;
+    const char *tlscredsid = NULL;
+    bool imageOpts = false;
+    bool writethrough = true;
+    char *trace_file = NULL;
 
     /* The client thread uses SIGTERM to interrupt the server.  A signal
      * handler ensures that "qemu-nbd -v -c" exits with a nice status code.
@@ -448,6 +532,12 @@
     memset(&sa_sigterm, 0, sizeof(sa_sigterm));
     sa_sigterm.sa_handler = termsig_handler;
     sigaction(SIGTERM, &sa_sigterm, NULL);
+
+    qcrypto_init(&error_fatal);
+
+    module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_object_opts);
+    qemu_add_opts(&qemu_trace_opts);
     qemu_init_exec_dir(argv[0]);
 
     while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
@@ -460,17 +550,19 @@
             /* fallthrough */
         case QEMU_NBD_OPT_CACHE:
             if (seen_cache) {
-                errx(EXIT_FAILURE, "-n and --cache can only be specified once");
+                error_report("-n and --cache can only be specified once");
+                exit(EXIT_FAILURE);
             }
             seen_cache = true;
-            if (bdrv_parse_cache_flags(optarg, &flags) == -1) {
-                errx(EXIT_FAILURE, "Invalid cache mode `%s'", optarg);
+            if (bdrv_parse_cache_mode(optarg, &flags, &writethrough) == -1) {
+                error_report("Invalid cache mode `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             break;
-#ifdef CONFIG_LINUX_AIO
         case QEMU_NBD_OPT_AIO:
             if (seen_aio) {
-                errx(EXIT_FAILURE, "--aio can only be specified once");
+                error_report("--aio can only be specified once");
+                exit(EXIT_FAILURE);
             }
             seen_aio = true;
             if (!strcmp(optarg, "native")) {
@@ -478,64 +570,65 @@
             } else if (!strcmp(optarg, "threads")) {
                 /* this is the default */
             } else {
-               errx(EXIT_FAILURE, "invalid aio mode `%s'", optarg);
+               error_report("invalid aio mode `%s'", optarg);
+               exit(EXIT_FAILURE);
             }
             break;
-#endif
         case QEMU_NBD_OPT_DISCARD:
             if (seen_discard) {
-                errx(EXIT_FAILURE, "--discard can only be specified once");
+                error_report("--discard can only be specified once");
+                exit(EXIT_FAILURE);
             }
             seen_discard = true;
             if (bdrv_parse_discard_flags(optarg, &flags) == -1) {
-                errx(EXIT_FAILURE, "Invalid discard mode `%s'", optarg);
+                error_report("Invalid discard mode `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             break;
         case QEMU_NBD_OPT_DETECT_ZEROES:
             detect_zeroes =
                 qapi_enum_parse(BlockdevDetectZeroesOptions_lookup,
                                 optarg,
-                                BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
+                                BLOCKDEV_DETECT_ZEROES_OPTIONS__MAX,
                                 BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
                                 &local_err);
             if (local_err) {
-                errx(EXIT_FAILURE, "Failed to parse detect_zeroes mode: %s", 
-                     error_get_pretty(local_err));
+                error_reportf_err(local_err,
+                                  "Failed to parse detect_zeroes mode: ");
+                exit(EXIT_FAILURE);
             }
             if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
                 !(flags & BDRV_O_UNMAP)) {
-                errx(EXIT_FAILURE, "setting detect-zeroes to unmap is not allowed "
-                                   "without setting discard operation to unmap"); 
+                error_report("setting detect-zeroes to unmap is not allowed "
+                             "without setting discard operation to unmap");
+                exit(EXIT_FAILURE);
             }
             break;
         case 'b':
             bindto = optarg;
             break;
         case 'p':
-            li = strtol(optarg, &end, 0);
-            if (*end) {
-                errx(EXIT_FAILURE, "Invalid port `%s'", optarg);
-            }
-            if (li < 1 || li > 65535) {
-                errx(EXIT_FAILURE, "Port out of range `%s'", optarg);
-            }
-            port = (uint16_t)li;
+            port = optarg;
             break;
         case 'o':
                 dev_offset = strtoll (optarg, &end, 0);
             if (*end) {
-                errx(EXIT_FAILURE, "Invalid offset `%s'", optarg);
+                error_report("Invalid offset `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             if (dev_offset < 0) {
-                errx(EXIT_FAILURE, "Offset must be positive `%s'", optarg);
+                error_report("Offset must be positive `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             break;
         case 'l':
             if (strstart(optarg, SNAPSHOT_OPT_BASE, NULL)) {
-                sn_opts = qemu_opts_parse(&internal_snapshot_opts, optarg, 0);
+                sn_opts = qemu_opts_parse_noisily(&internal_snapshot_opts,
+                                                  optarg, false);
                 if (!sn_opts) {
-                    errx(EXIT_FAILURE, "Failed in parsing snapshot param `%s'",
-                         optarg);
+                    error_report("Failed in parsing snapshot param `%s'",
+                                 optarg);
+                    exit(EXIT_FAILURE);
                 }
             } else {
                 sn_id_or_name = optarg;
@@ -548,16 +641,19 @@
         case 'P':
             partition = strtol(optarg, &end, 0);
             if (*end) {
-                errx(EXIT_FAILURE, "Invalid partition `%s'", optarg);
+                error_report("Invalid partition `%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             if (partition < 1 || partition > 8) {
-                errx(EXIT_FAILURE, "Invalid partition %d", partition);
+                error_report("Invalid partition %d", partition);
+                exit(EXIT_FAILURE);
             }
             break;
         case 'k':
             sockpath = optarg;
             if (sockpath[0] != '/') {
-                errx(EXIT_FAILURE, "socket path must be absolute\n");
+                error_report("socket path must be absolute");
+                exit(EXIT_FAILURE);
             }
             break;
         case 'd':
@@ -569,10 +665,12 @@
         case 'e':
             shared = strtol(optarg, &end, 0);
             if (*end) {
-                errx(EXIT_FAILURE, "Invalid shared device number '%s'", optarg);
+                error_report("Invalid shared device number '%s'", optarg);
+                exit(EXIT_FAILURE);
             }
             if (shared < 1) {
-                errx(EXIT_FAILURE, "Shared device number must be greater than 0\n");
+                error_report("Shared device number must be greater than 0");
+                exit(EXIT_FAILURE);
             }
             break;
         case 'f':
@@ -581,6 +679,9 @@
         case 't':
             persistent = 1;
             break;
+        case 'x':
+            export_name = optarg;
+            break;
         case 'v':
             verbose = 1;
             break;
@@ -593,25 +694,79 @@
             exit(0);
             break;
         case '?':
-            errx(EXIT_FAILURE, "Try `%s --help' for more information.",
-                 argv[0]);
+            error_report("Try `%s --help' for more information.", argv[0]);
+            exit(EXIT_FAILURE);
+        case QEMU_NBD_OPT_OBJECT: {
+            QemuOpts *opts;
+            opts = qemu_opts_parse_noisily(&qemu_object_opts,
+                                           optarg, true);
+            if (!opts) {
+                exit(EXIT_FAILURE);
+            }
+        }   break;
+        case QEMU_NBD_OPT_TLSCREDS:
+            tlscredsid = optarg;
+            break;
+        case QEMU_NBD_OPT_IMAGE_OPTS:
+            imageOpts = true;
+            break;
+        case 'T':
+            g_free(trace_file);
+            trace_file = trace_opt_parse(optarg);
+            break;
         }
     }
 
     if ((argc - optind) != 1) {
-        errx(EXIT_FAILURE, "Invalid number of argument.\n"
-             "Try `%s --help' for more information.",
-             argv[0]);
+        error_report("Invalid number of arguments");
+        error_printf("Try `%s --help' for more information.\n", argv[0]);
+        exit(EXIT_FAILURE);
+    }
+
+    if (qemu_opts_foreach(&qemu_object_opts,
+                          user_creatable_add_opts_foreach,
+                          NULL, NULL)) {
+        exit(EXIT_FAILURE);
+    }
+
+    if (!trace_init_backends()) {
+        exit(1);
+    }
+    trace_init_file(trace_file);
+    qemu_set_log(LOG_TRACE);
+
+    if (tlscredsid) {
+        if (sockpath) {
+            error_report("TLS is only supported with IPv4/IPv6");
+            exit(EXIT_FAILURE);
+        }
+        if (device) {
+            error_report("TLS is not supported with a host device");
+            exit(EXIT_FAILURE);
+        }
+        if (!export_name) {
+            /* Set the default NBD protocol export name, since
+             * we *must* use new style protocol for TLS */
+            export_name = "";
+        }
+        tlscreds = nbd_get_tls_creds(tlscredsid, &local_err);
+        if (local_err) {
+            error_report("Failed to get TLS creds %s",
+                         error_get_pretty(local_err));
+            exit(EXIT_FAILURE);
+        }
     }
 
     if (disconnect) {
-        fd = open(argv[optind], O_RDWR);
-        if (fd < 0) {
-            err(EXIT_FAILURE, "Cannot open %s", argv[optind]);
+        int nbdfd = open(argv[optind], O_RDWR);
+        if (nbdfd < 0) {
+            error_report("Cannot open %s: %s", argv[optind],
+                         strerror(errno));
+            exit(EXIT_FAILURE);
         }
-        nbd_disconnect(fd);
+        nbd_disconnect(nbdfd);
 
-        close(fd);
+        close(nbdfd);
 
         printf("%s disconnected\n", argv[optind]);
 
@@ -624,21 +779,27 @@
         int ret;
 
         if (qemu_pipe(stderr_fd) < 0) {
-            err(EXIT_FAILURE, "Error setting up communication pipe");
+            error_report("Error setting up communication pipe: %s",
+                         strerror(errno));
+            exit(EXIT_FAILURE);
         }
 
         /* Now daemonize, but keep a communication channel open to
          * print errors and exit with the proper status code.
          */
         pid = fork();
-        if (pid == 0) {
+        if (pid < 0) {
+            error_report("Failed to fork: %s", strerror(errno));
+            exit(EXIT_FAILURE);
+        } else if (pid == 0) {
             close(stderr_fd[0]);
             ret = qemu_daemon(1, 0);
 
             /* Temporarily redirect stderr to the parent's pipe...  */
             dup2(stderr_fd[1], STDERR_FILENO);
             if (ret < 0) {
-                err(EXIT_FAILURE, "Failed to daemonize");
+                error_report("Failed to daemonize: %s", strerror(errno));
+                exit(EXIT_FAILURE);
             }
 
             /* ... close the descriptor we inherited and go on.  */
@@ -660,7 +821,9 @@
                 }
             }
             if (ret < 0) {
-                err(EXIT_FAILURE, "Cannot read from daemon");
+                error_report("Cannot read from daemon: %s",
+                             strerror(errno));
+                exit(EXIT_FAILURE);
             }
 
             /* Usually the daemon should not print any message.
@@ -675,33 +838,46 @@
         snprintf(sockpath, 128, SOCKET_PATH, basename(device));
     }
 
+    saddr = nbd_build_socket_address(sockpath, bindto, port);
+
     if (qemu_init_main_loop(&local_err)) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         exit(EXIT_FAILURE);
     }
     bdrv_init();
     atexit(bdrv_close_all);
 
-    if (fmt) {
-        drv = bdrv_find_format(fmt);
-        if (!drv) {
-            errx(EXIT_FAILURE, "Unknown file format '%s'", fmt);
+    srcpath = argv[optind];
+    if (imageOpts) {
+        QemuOpts *opts;
+        if (fmt) {
+            error_report("--image-opts and -f are mutually exclusive");
+            exit(EXIT_FAILURE);
         }
+        opts = qemu_opts_parse_noisily(&file_opts, srcpath, true);
+        if (!opts) {
+            qemu_opts_reset(&file_opts);
+            exit(EXIT_FAILURE);
+        }
+        options = qemu_opts_to_qdict(opts, NULL);
+        qemu_opts_reset(&file_opts);
+        blk = blk_new_open(NULL, NULL, options, flags, &local_err);
     } else {
-        drv = NULL;
+        if (fmt) {
+            options = qdict_new();
+            qdict_put(options, "driver", qstring_from_str(fmt));
+        }
+        blk = blk_new_open(srcpath, NULL, options, flags, &local_err);
     }
 
-    blk = blk_new_with_bs("hda", &error_abort);
+    if (!blk) {
+        error_reportf_err(local_err, "Failed to blk_new_open '%s': ",
+                          argv[optind]);
+        exit(EXIT_FAILURE);
+    }
     bs = blk_bs(blk);
 
-    srcpath = argv[optind];
-    ret = bdrv_open(&bs, srcpath, NULL, NULL, flags, drv, &local_err);
-    if (ret < 0) {
-        errno = -ret;
-        err(EXIT_FAILURE, "Failed to bdrv_open '%s': %s", argv[optind],
-            error_get_pretty(local_err));
-    }
+    blk_set_enable_write_cache(blk, !writethrough);
 
     if (sn_opts) {
         ret = bdrv_snapshot_load_tmp(bs,
@@ -713,32 +889,42 @@
                                                    &local_err);
     }
     if (ret < 0) {
-        errno = -ret;
-        err(EXIT_FAILURE,
-            "Failed to load snapshot: %s",
-            error_get_pretty(local_err));
+        error_reportf_err(local_err, "Failed to load snapshot: ");
+        exit(EXIT_FAILURE);
     }
 
     bs->detect_zeroes = detect_zeroes;
-    fd_size = bdrv_getlength(bs);
+    fd_size = blk_getlength(blk);
+    if (fd_size < 0) {
+        error_report("Failed to determine the image length: %s",
+                     strerror(-fd_size));
+        exit(EXIT_FAILURE);
+    }
 
     if (partition != -1) {
-        ret = find_partition(bs, partition, &dev_offset, &fd_size);
+        ret = find_partition(blk, partition, &dev_offset, &fd_size);
         if (ret < 0) {
-            errno = -ret;
-            err(EXIT_FAILURE, "Could not find partition %d", partition);
+            error_report("Could not find partition %d: %s", partition,
+                         strerror(-ret));
+            exit(EXIT_FAILURE);
         }
     }
 
-    exp = nbd_export_new(bs, dev_offset, fd_size, nbdflags, nbd_export_closed);
-
-    if (sockpath) {
-        fd = unix_socket_incoming(sockpath);
-    } else {
-        fd = tcp_socket_incoming(bindto, port);
+    exp = nbd_export_new(blk, dev_offset, fd_size, nbdflags, nbd_export_closed,
+                         &local_err);
+    if (!exp) {
+        error_report_err(local_err);
+        exit(EXIT_FAILURE);
+    }
+    if (export_name) {
+        nbd_export_set_name(exp, export_name);
+        newproto = true;
     }
 
-    if (fd < 0) {
+    server_ioc = qio_channel_socket_new();
+    if (qio_channel_socket_listen_sync(server_ioc, saddr, &local_err) < 0) {
+        object_unref(OBJECT(server_ioc));
+        error_report_err(local_err);
         return 1;
     }
 
@@ -747,21 +933,22 @@
 
         ret = pthread_create(&client_thread, NULL, nbd_client_thread, device);
         if (ret != 0) {
-            errx(EXIT_FAILURE, "Failed to create client thread: %s",
-                 strerror(ret));
+            error_report("Failed to create client thread: %s", strerror(ret));
+            exit(EXIT_FAILURE);
         }
     } else {
         /* Shut up GCC warnings.  */
         memset(&client_thread, 0, sizeof(client_thread));
     }
 
-    qemu_set_fd_handler2(fd, nbd_can_accept, nbd_accept, NULL,
-                         (void *)(uintptr_t)fd);
+    nbd_update_server_watch();
 
     /* now when the initialization is (almost) complete, chdir("/")
      * to free any busy filesystems */
     if (chdir("/") < 0) {
-        err(EXIT_FAILURE, "Could not chdir to root directory");
+        error_report("Could not chdir to root directory: %s",
+                     strerror(errno));
+        exit(EXIT_FAILURE);
     }
 
     state = RUNNING;
diff --git a/qemu-nbd.texi b/qemu-nbd.texi
index 46fd483..91ebf04 100644
--- a/qemu-nbd.texi
+++ b/qemu-nbd.texi
@@ -1,68 +1,100 @@
 @example
 @c man begin SYNOPSIS
-usage: qemu-nbd [OPTION]...  @var{filename}
+@command{qemu-nbd} [OPTION]... @var{filename}
+
+@command{qemu-nbd} @option{-d} @var{dev}
 @c man end
 @end example
 
 @c man begin DESCRIPTION
 
-Export QEMU disk image using NBD protocol.
+Export a QEMU disk image using the NBD protocol.
 
 @c man end
 
 @c man begin OPTIONS
+@var{filename} is a disk image filename, or a set of block
+driver options if @var{--image-opts} is specified.
+
+@var{dev} is an NBD device.
+
 @table @option
-@item @var{filename}
- is a disk image filename
+@item --object type,id=@var{id},...props...
+Define a new instance of the @var{type} object class identified by @var{id}.
+See the @code{qemu(1)} manual page for full details of the properties
+supported. The common object types that it makes sense to define are the
+@code{secret} object, which is used to supply passwords and/or encryption
+keys, and the @code{tls-creds} object, which is used to supply TLS
+credentials for the qemu-nbd server.
 @item -p, --port=@var{port}
-  port to listen on (default @samp{10809})
+The TCP port to listen on (default @samp{10809})
 @item -o, --offset=@var{offset}
-  offset into the image
+The offset into the image
 @item -b, --bind=@var{iface}
-  interface to bind to (default @samp{0.0.0.0})
+The interface to bind to (default @samp{0.0.0.0})
 @item -k, --socket=@var{path}
-  Use a unix socket with path @var{path}
-@item -f, --format=@var{format}
-  Set image format as @var{format}
+Use a unix socket with path @var{path}
+@item --image-opts
+Treat @var{filename} as a set of image options, instead of a plain
+filename. If this flag is specified, the @var{-f} flag should
+not be used, instead the '@code{format=}' option should be set.
+@item -f, --format=@var{fmt}
+Force the use of the block driver for format @var{fmt} instead of
+auto-detecting
 @item -r, --read-only
-  export read-only
+Export the disk as read-only
 @item -P, --partition=@var{num}
-  only expose partition @var{num}
+Only expose partition @var{num}
 @item -s, --snapshot
-  use @var{filename} as an external snapshot, create a temporary
-  file with backing_file=@var{filename}, redirect the write to
-  the temporary one
+Use @var{filename} as an external snapshot, create a temporary
+file with backing_file=@var{filename}, redirect the write to
+the temporary one
 @item -l, --load-snapshot=@var{snapshot_param}
-  load an internal snapshot inside @var{filename} and export it
-  as an read-only device, @var{snapshot_param} format is
-  'snapshot.id=[ID],snapshot.name=[NAME]' or '[ID_OR_NAME]'
+Load an internal snapshot inside @var{filename} and export it
+as an read-only device, @var{snapshot_param} format is
+'snapshot.id=[ID],snapshot.name=[NAME]' or '[ID_OR_NAME]'
 @item -n, --nocache
 @itemx --cache=@var{cache}
-  set cache mode to be used with the file.  See the documentation of
-  the emulator's @code{-drive cache=...} option for allowed values.
+The cache mode to be used with the file.  See the documentation of
+the emulator's @code{-drive cache=...} option for allowed values.
 @item --aio=@var{aio}
-  choose asynchronous I/O mode between @samp{threads} (the default)
-  and @samp{native} (Linux only).
+Set the asynchronous I/O mode between @samp{threads} (the default)
+and @samp{native} (Linux only).
 @item --discard=@var{discard}
-  toggles whether @dfn{discard} (also known as @dfn{trim} or @dfn{unmap})
-  requests are ignored or passed to the filesystem.  The default is no
-  (@samp{--discard=ignore}).
+Control whether @dfn{discard} (also known as @dfn{trim} or @dfn{unmap})
+requests are ignored or passed to the filesystem.  @var{discard} is one of
+@samp{ignore} (or @samp{off}), @samp{unmap} (or @samp{on}).  The default is
+@samp{ignore}.
+@item --detect-zeroes=@var{detect-zeroes}
+Control the automatic conversion of plain zero writes by the OS to
+driver-specific optimized zero write commands.  @var{detect-zeroes} is one of
+@samp{off}, @samp{on} or @samp{unmap}.  @samp{unmap}
+converts a zero write to an unmap operation and can only be used if
+@var{discard} is set to @samp{unmap}.  The default is @samp{off}.
 @item -c, --connect=@var{dev}
-  connect @var{filename} to NBD device @var{dev}
+Connect @var{filename} to NBD device @var{dev}
 @item -d, --disconnect
-  disconnect the specified device
+Disconnect the device @var{dev}
 @item -e, --shared=@var{num}
-  device can be shared by @var{num} clients (default @samp{1})
-@item -f, --format=@var{fmt}
-  force block driver for format @var{fmt} instead of auto-detecting
+Allow up to @var{num} clients to share the device (default @samp{1})
 @item -t, --persistent
-  don't exit on the last connection
+Don't exit on the last connection
+@item -x NAME, --export-name=NAME
+Set the NBD volume export name. This switches the server to use
+the new style NBD protocol negotiation
+@item --tls-creds=ID
+Enable mandatory TLS encryption for the server by setting the ID
+of the TLS credentials object previously created with the --object
+option.
 @item -v, --verbose
-  display extra debugging information
+Display extra debugging information
 @item -h, --help
-  display this help and exit
+Display this help and exit
 @item -V, --version
-  output version information and exit
+Display version information and exit
+@item -T, --trace [[enable=]@var{pattern}][,events=@var{file}][,file=@var{file}]
+@findex --trace
+@include qemu-option-trace.texi
 @end table
 
 @c man end
@@ -79,7 +111,7 @@
 @c man end
 
 @c man begin SEEALSO
-qemu-img(1)
+qemu(1), qemu-img(1)
 @c man end
 
 @end ignore
diff --git a/qemu-option-trace.texi b/qemu-option-trace.texi
new file mode 100644
index 0000000..693ab5a
--- /dev/null
+++ b/qemu-option-trace.texi
@@ -0,0 +1,25 @@
+Specify tracing options.
+
+@table @option
+@item [enable=]@var{pattern}
+Immediately enable events matching @var{pattern}.
+The file must contain one event name (as listed in the @file{trace-events-all}
+file) per line; globbing patterns are accepted too.  This option is only
+available if QEMU has been compiled with the @var{simple}, @var{stderr}
+or @var{ftrace} tracing backend.  To specify multiple events or patterns,
+specify the @option{-trace} option multiple times.
+
+Use @code{-trace help} to print a list of names of trace points.
+
+@item events=@var{file}
+Immediately enable events listed in @var{file}.
+The file must contain one event name (as listed in the @file{trace-events-all}
+file) per line; globbing patterns are accepted too.  This option is only
+available if QEMU has been compiled with the @var{simple}, @var{stderr} or
+@var{ftrace} tracing backend.
+
+@item file=@var{file}
+Log output traces to @var{file}.
+This option is only available if QEMU has been compiled with
+the @var{simple} tracing backend.
+@end table
diff --git a/qemu-options.def b/qemu-options.def
new file mode 100644
index 0000000..f187141
--- /dev/null
+++ b/qemu-options.def
@@ -0,0 +1,897 @@
+
+DEFHEADING(Standard options:)
+
+DEF("help", 0, QEMU_OPTION_h,
+"-h or -help     display this help and exit\n", QEMU_ARCH_ALL)
+
+DEF("version", 0, QEMU_OPTION_version,
+"-version        display version information and exit\n", QEMU_ARCH_ALL)
+
+DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
+"-machine [type=]name[,prop[=value][,...]]\n"
+"                selects emulated machine ('-machine help' for list)\n"
+"                property accel=accel1[:accel2[:...]] selects accelerator\n"
+"                supported accelerators are kvm, xen, tcg (default: tcg)\n"
+"                kernel_irqchip=on|off controls accelerated irqchip support\n"
+"                kernel_irqchip=on|off|split controls accelerated irqchip support (default=off)\n"
+"                vmport=on|off|auto controls emulation of vmport (default: auto)\n"
+"                kvm_shadow_mem=size of KVM shadow MMU in bytes\n"
+"                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
+"                mem-merge=on|off controls memory merge support (default: on)\n"
+"                igd-passthru=on|off controls IGD GFX passthrough support (default=off)\n"
+"                aes-key-wrap=on|off controls support for AES key wrapping (default=on)\n"
+"                dea-key-wrap=on|off controls support for DEA key wrapping (default=on)\n"
+"                suppress-vmdesc=on|off disables self-describing migration (default=off)\n"
+"                nvdimm=on|off controls NVDIMM support (default=off)\n"
+"                enforce-config-section=on|off enforce configuration section migration (default=off)\n",
+QEMU_ARCH_ALL)
+
+DEF("M", HAS_ARG, QEMU_OPTION_M, "", QEMU_ARCH_ALL)
+
+DEF("cpu", HAS_ARG, QEMU_OPTION_cpu,
+"-cpu cpu        select CPU ('-cpu help' for list)\n", QEMU_ARCH_ALL)
+
+DEF("smp", HAS_ARG, QEMU_OPTION_smp,
+"-smp [cpus=]n[,maxcpus=cpus][,cores=cores][,threads=threads][,sockets=sockets]\n"
+"                set the number of CPUs to 'n' [default=1]\n"
+"                maxcpus= maximum number of total cpus, including\n"
+"                offline CPUs for hotplug, etc\n"
+"                cores= number of CPU cores on one socket\n"
+"                threads= number of threads on one CPU core\n"
+"                sockets= number of discrete sockets in the system\n",
+QEMU_ARCH_ALL)
+
+DEF("numa", HAS_ARG, QEMU_OPTION_numa,
+"-numa node[,mem=size][,cpus=cpu[-cpu]][,nodeid=node]\n"
+"-numa node[,memdev=id][,cpus=cpu[-cpu]][,nodeid=node]\n", QEMU_ARCH_ALL)
+
+DEF("add-fd", HAS_ARG, QEMU_OPTION_add_fd,
+"-add-fd fd=fd,set=set[,opaque=opaque]\n"
+"                Add 'fd' to fd 'set'\n", QEMU_ARCH_ALL)
+
+DEF("set", HAS_ARG, QEMU_OPTION_set,
+"-set group.id.arg=value\n"
+"                set <arg> parameter for item <id> of type <group>\n"
+"                i.e. -set drive.$id.file=/path/to/image\n", QEMU_ARCH_ALL)
+
+DEF("global", HAS_ARG, QEMU_OPTION_global,
+"-global driver.property=value\n"
+"-global driver=driver,property=property,value=value\n"
+"                set a global default for a driver property\n",
+QEMU_ARCH_ALL)
+
+DEF("boot", HAS_ARG, QEMU_OPTION_boot,
+"-boot [order=drives][,once=drives][,menu=on|off]\n"
+"      [,splash=sp_name][,splash-time=sp_time][,reboot-timeout=rb_time][,strict=on|off]\n"
+"                'drives': floppy (a), hard disk (c), CD-ROM (d), network (n)\n"
+"                'sp_name': the file's name that would be passed to bios as logo picture, if menu=on\n"
+"                'sp_time': the period that splash picture last if menu=on, unit is ms\n"
+"                'rb_timeout': the timeout before guest reboot when boot failed, unit is ms\n",
+QEMU_ARCH_ALL)
+
+DEF("m", HAS_ARG, QEMU_OPTION_m,
+"-m[emory] [size=]megs[,slots=n,maxmem=size]\n"
+"                configure guest RAM\n"
+"                size: initial amount of guest memory\n"
+"                slots: number of hotplug slots (default: none)\n"
+"                maxmem: maximum amount of guest memory (default: none)\n"
+"NOTE: Some architectures might enforce a specific granularity\n",
+QEMU_ARCH_ALL)
+
+DEF("mem-path", HAS_ARG, QEMU_OPTION_mempath,
+"-mem-path FILE  provide backing storage for guest RAM\n", QEMU_ARCH_ALL)
+
+DEF("mem-prealloc", 0, QEMU_OPTION_mem_prealloc,
+"-mem-prealloc   preallocate guest memory (use with -mem-path)\n",
+QEMU_ARCH_ALL)
+
+DEF("k", HAS_ARG, QEMU_OPTION_k,
+"-k language     use keyboard layout (for example 'fr' for French)\n",
+QEMU_ARCH_ALL)
+
+
+DEF("audio-help", 0, QEMU_OPTION_audio_help,
+"-audio-help     print list of audio drivers and their options\n",
+QEMU_ARCH_ALL)
+
+DEF("no-audio", 0, QEMU_OPTION_audio_none,
+"-no-audio        disable audio support\n",
+QEMU_ARCH_ALL)
+
+DEF("soundhw", HAS_ARG, QEMU_OPTION_soundhw,
+"-soundhw c1,... enable audio support\n"
+"                and only specified sound cards (comma separated list)\n"
+"                use '-soundhw help' to get the list of supported cards\n"
+"                use '-soundhw all' to enable all of them\n", QEMU_ARCH_ALL)
+
+DEF("balloon", HAS_ARG, QEMU_OPTION_balloon,
+"-balloon none   disable balloon device\n"
+"-balloon virtio[,addr=str]\n"
+"                enable virtio balloon device (default)\n", QEMU_ARCH_ALL)
+
+DEF("device", HAS_ARG, QEMU_OPTION_device,
+"-device driver[,prop[=value][,...]]\n"
+"                add device (based on driver)\n"
+"                prop=value,... sets driver properties\n"
+"                use '-device help' to print all possible drivers\n"
+"                use '-device driver,help' to print all possible properties\n",
+QEMU_ARCH_ALL)
+
+DEF("name", HAS_ARG, QEMU_OPTION_name,
+"-name string1[,process=string2][,debug-threads=on|off]\n"
+"                set the name of the guest\n"
+"                string1 sets the window title and string2 the process name (on Linux)\n"
+"                When debug-threads is enabled, individual threads are given a separate name (on Linux)\n"
+"                NOTE: The thread names are for debugging and not a stable API.\n",
+QEMU_ARCH_ALL)
+
+DEF("uuid", HAS_ARG, QEMU_OPTION_uuid,
+"-uuid %08x-%04x-%04x-%04x-%012x\n"
+"                specify machine UUID\n", QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+DEFHEADING(Block device options:)
+
+DEF("fda", HAS_ARG, QEMU_OPTION_fda,
+"-fda/-fdb file  use 'file' as floppy disk 0/1 image\n", QEMU_ARCH_ALL)
+DEF("fdb", HAS_ARG, QEMU_OPTION_fdb, "", QEMU_ARCH_ALL)
+
+DEF("hda", HAS_ARG, QEMU_OPTION_hda,
+"-hda/-hdb file  use 'file' as IDE hard disk 0/1 image\n", QEMU_ARCH_ALL)
+DEF("hdb", HAS_ARG, QEMU_OPTION_hdb, "", QEMU_ARCH_ALL)
+DEF("hdc", HAS_ARG, QEMU_OPTION_hdc,
+"-hdc/-hdd file  use 'file' as IDE hard disk 2/3 image\n", QEMU_ARCH_ALL)
+DEF("hdd", HAS_ARG, QEMU_OPTION_hdd, "", QEMU_ARCH_ALL)
+
+DEF("cdrom", HAS_ARG, QEMU_OPTION_cdrom,
+"-cdrom file     use 'file' as IDE cdrom image (cdrom is ide1 master)\n",
+QEMU_ARCH_ALL)
+
+DEF("drive", HAS_ARG, QEMU_OPTION_drive,
+"-drive [file=file][,if=type][,bus=n][,unit=m][,media=d][,index=i]\n"
+"       [,cyls=c,heads=h,secs=s[,trans=t]][,snapshot=on|off]\n"
+"       [,cache=writethrough|writeback|none|directsync|unsafe][,format=f]\n"
+"       [,serial=s][,addr=A][,rerror=ignore|stop|report]\n"
+"       [,werror=ignore|stop|report|enospc][,id=name][,aio=threads|native]\n"
+"       [,readonly=on|off][,copy-on-read=on|off]\n"
+"       [,discard=ignore|unmap][,detect-zeroes=on|off|unmap]\n"
+"       [[,bps=b]|[[,bps_rd=r][,bps_wr=w]]]\n"
+"       [[,iops=i]|[[,iops_rd=r][,iops_wr=w]]]\n"
+"       [[,bps_max=bm]|[[,bps_rd_max=rm][,bps_wr_max=wm]]]\n"
+"       [[,iops_max=im]|[[,iops_rd_max=irm][,iops_wr_max=iwm]]]\n"
+"       [[,iops_size=is]]\n"
+"       [[,group=g]]\n"
+"                use 'file' as a drive image\n", QEMU_ARCH_ALL)
+
+DEF("mtdblock", HAS_ARG, QEMU_OPTION_mtdblock,
+"-mtdblock file  use 'file' as on-board Flash memory image\n",
+QEMU_ARCH_ALL)
+
+DEF("sd", HAS_ARG, QEMU_OPTION_sd,
+"-sd file        use 'file' as SecureDigital card image\n", QEMU_ARCH_ALL)
+
+DEF("pflash", HAS_ARG, QEMU_OPTION_pflash,
+"-pflash file    use 'file' as a parallel flash image\n", QEMU_ARCH_ALL)
+
+DEF("snapshot", 0, QEMU_OPTION_snapshot,
+"-snapshot       write to temporary files instead of disk image files\n",
+QEMU_ARCH_ALL)
+
+DEF("hdachs", HAS_ARG, QEMU_OPTION_hdachs, \
+"-hdachs c,h,s[,t]\n" \
+"                force hard disk 0 physical geometry and the optional BIOS\n" \
+"                translation (t=none or lba) (usually QEMU can guess them)\n",
+QEMU_ARCH_ALL)
+
+DEF("fsdev", HAS_ARG, QEMU_OPTION_fsdev,
+"-fsdev fsdriver,id=id[,path=path,][security_model={mapped-xattr|mapped-file|passthrough|none}]\n"
+" [,writeout=immediate][,readonly][,socket=socket|sock_fd=sock_fd]\n",
+QEMU_ARCH_ALL)
+
+
+DEF("virtfs", HAS_ARG, QEMU_OPTION_virtfs,
+"-virtfs local,path=path,mount_tag=tag,security_model=[mapped-xattr|mapped-file|passthrough|none]\n"
+"        [,writeout=immediate][,readonly][,socket=socket|sock_fd=sock_fd]\n",
+QEMU_ARCH_ALL)
+
+
+DEF("virtfs_synth", 0, QEMU_OPTION_virtfs_synth,
+"-virtfs_synth Create synthetic file system image\n",
+QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+DEFHEADING(USB options:)
+
+DEF("usb", 0, QEMU_OPTION_usb,
+"-usb            enable the USB driver (will be the default soon)\n",
+QEMU_ARCH_ALL)
+
+DEF("usbdevice", HAS_ARG, QEMU_OPTION_usbdevice,
+"-usbdevice name add the host or guest USB device 'name'\n",
+QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+DEFHEADING(Display options:)
+
+DEF("display", HAS_ARG, QEMU_OPTION_display,
+"-display sdl[,frame=on|off][,alt_grab=on|off][,ctrl_grab=on|off]\n"
+"            [,window_close=on|off][,gl=on|off]|curses|none|\n"
+"-display gtk[,grab_on_hover=on|off][,gl=on|off]|\n"
+"-display vnc=<display>[,<optargs>]\n"
+"-display curses\n"
+"-display none"
+"                select display type\n"
+"The default display is equivalent to\n"
+#if defined(CONFIG_GTK)
+"\t\"-display gtk\"\n"
+#elif defined(CONFIG_SDL)
+"\t\"-display sdl\"\n"
+#elif defined(CONFIG_COCOA)
+"\t\"-display cocoa\"\n"
+#elif defined(CONFIG_VNC)
+"\t\"-vnc localhost:0,to=99,id=default\"\n"
+#else
+"\t\"-display none\"\n"
+#endif
+, QEMU_ARCH_ALL)
+
+DEF("nographic", 0, QEMU_OPTION_nographic,
+"-nographic      disable graphical output and redirect serial I/Os to console\n",
+QEMU_ARCH_ALL)
+
+DEF("curses", 0, QEMU_OPTION_curses,
+"-curses         shorthand for -display curses\n",
+QEMU_ARCH_ALL)
+
+DEF("no-frame", 0, QEMU_OPTION_no_frame,
+"-no-frame       open SDL window without a frame and window decorations\n",
+QEMU_ARCH_ALL)
+
+DEF("alt-grab", 0, QEMU_OPTION_alt_grab,
+"-alt-grab       use Ctrl-Alt-Shift to grab mouse (instead of Ctrl-Alt)\n",
+QEMU_ARCH_ALL)
+
+DEF("ctrl-grab", 0, QEMU_OPTION_ctrl_grab,
+"-ctrl-grab      use Right-Ctrl to grab mouse (instead of Ctrl-Alt)\n",
+QEMU_ARCH_ALL)
+
+DEF("no-quit", 0, QEMU_OPTION_no_quit,
+"-no-quit        disable SDL window close capability\n", QEMU_ARCH_ALL)
+
+DEF("sdl", 0, QEMU_OPTION_sdl,
+"-sdl            shorthand for -display sdl\n", QEMU_ARCH_ALL)
+
+DEF("spice", HAS_ARG, QEMU_OPTION_spice,
+"-spice [port=port][,tls-port=secured-port][,x509-dir=<dir>]\n"
+"       [,x509-key-file=<file>][,x509-key-password=<file>]\n"
+"       [,x509-cert-file=<file>][,x509-cacert-file=<file>]\n"
+"       [,x509-dh-key-file=<file>][,addr=addr][,ipv4|ipv6|unix]\n"
+"       [,tls-ciphers=<list>]\n"
+"       [,tls-channel=[main|display|cursor|inputs|record|playback]]\n"
+"       [,plaintext-channel=[main|display|cursor|inputs|record|playback]]\n"
+"       [,sasl][,password=<secret>][,disable-ticketing]\n"
+"       [,image-compression=[auto_glz|auto_lz|quic|glz|lz|off]]\n"
+"       [,jpeg-wan-compression=[auto|never|always]]\n"
+"       [,zlib-glz-wan-compression=[auto|never|always]]\n"
+"       [,streaming-video=[off|all|filter]][,disable-copy-paste]\n"
+"       [,disable-agent-file-xfer][,agent-mouse=[on|off]]\n"
+"       [,playback-compression=[on|off]][,seamless-migration=[on|off]]\n"
+"       [,gl=[on|off]]\n"
+"   enable spice\n"
+"   at least one of {port, tls-port} is mandatory\n",
+QEMU_ARCH_ALL)
+
+DEF("portrait", 0, QEMU_OPTION_portrait,
+"-portrait       rotate graphical output 90 deg left (only PXA LCD)\n",
+QEMU_ARCH_ALL)
+
+DEF("rotate", HAS_ARG, QEMU_OPTION_rotate,
+"-rotate <deg>   rotate graphical output some deg left (only PXA LCD)\n",
+QEMU_ARCH_ALL)
+
+DEF("vga", HAS_ARG, QEMU_OPTION_vga,
+"-vga [std|cirrus|vmware|qxl|xenfb|tcx|cg3|virtio|none]\n"
+"                select video card type\n", QEMU_ARCH_ALL)
+
+DEF("full-screen", 0, QEMU_OPTION_full_screen,
+"-full-screen    start in full screen\n", QEMU_ARCH_ALL)
+
+DEF("g", 1, QEMU_OPTION_g ,
+"-g WxH[xDEPTH]  Set the initial graphical resolution and depth\n",
+QEMU_ARCH_PPC | QEMU_ARCH_SPARC)
+
+DEF("vnc", HAS_ARG, QEMU_OPTION_vnc ,
+"-vnc <display>  shorthand for -display vnc=<display>\n", QEMU_ARCH_ALL)
+
+ARCHHEADING(, QEMU_ARCH_I386)
+
+ARCHHEADING(i386 target only:, QEMU_ARCH_I386)
+
+DEF("win2k-hack", 0, QEMU_OPTION_win2k_hack,
+"-win2k-hack     use it when installing Windows 2000 to avoid a disk full bug\n",
+QEMU_ARCH_I386)
+
+DEF("rtc-td-hack", 0, QEMU_OPTION_rtc_td_hack, "", QEMU_ARCH_I386)
+
+DEF("no-fd-bootchk", 0, QEMU_OPTION_no_fd_bootchk,
+"-no-fd-bootchk  disable boot signature checking for floppy disks\n",
+QEMU_ARCH_I386)
+
+DEF("no-acpi", 0, QEMU_OPTION_no_acpi,
+"-no-acpi        disable ACPI\n", QEMU_ARCH_I386 | QEMU_ARCH_ARM)
+
+DEF("no-hpet", 0, QEMU_OPTION_no_hpet,
+"-no-hpet        disable HPET\n", QEMU_ARCH_I386)
+
+DEF("acpitable", HAS_ARG, QEMU_OPTION_acpitable,
+"-acpitable [sig=str][,rev=n][,oem_id=str][,oem_table_id=str][,oem_rev=n][,asl_compiler_id=str][,asl_compiler_rev=n][,{data|file}=file1[:file2]...]\n"
+"                ACPI table description\n", QEMU_ARCH_I386)
+
+DEF("smbios", HAS_ARG, QEMU_OPTION_smbios,
+"-smbios file=binary\n"
+"                load SMBIOS entry from binary file\n"
+"-smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d]\n"
+"              [,uefi=on|off]\n"
+"                specify SMBIOS type 0 fields\n"
+"-smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str]\n"
+"              [,uuid=uuid][,sku=str][,family=str]\n"
+"                specify SMBIOS type 1 fields\n"
+"-smbios type=2[,manufacturer=str][,product=str][,version=str][,serial=str]\n"
+"              [,asset=str][,location=str]\n"
+"                specify SMBIOS type 2 fields\n"
+"-smbios type=3[,manufacturer=str][,version=str][,serial=str][,asset=str]\n"
+"              [,sku=str]\n"
+"                specify SMBIOS type 3 fields\n"
+"-smbios type=4[,sock_pfx=str][,manufacturer=str][,version=str][,serial=str]\n"
+"              [,asset=str][,part=str]\n"
+"                specify SMBIOS type 4 fields\n"
+"-smbios type=17[,loc_pfx=str][,bank=str][,manufacturer=str][,serial=str]\n"
+"               [,asset=str][,part=str][,speed=%d]\n"
+"                specify SMBIOS type 17 fields\n",
+QEMU_ARCH_I386 | QEMU_ARCH_ARM)
+
+DEFHEADING()
+
+DEFHEADING(Network options:)
+
+#ifdef CONFIG_SLIRP
+DEF("tftp", HAS_ARG, QEMU_OPTION_tftp, "", QEMU_ARCH_ALL)
+DEF("bootp", HAS_ARG, QEMU_OPTION_bootp, "", QEMU_ARCH_ALL)
+DEF("redir", HAS_ARG, QEMU_OPTION_redir, "", QEMU_ARCH_ALL)
+#ifndef _WIN32
+DEF("smb", HAS_ARG, QEMU_OPTION_smb, "", QEMU_ARCH_ALL)
+#endif
+#endif
+
+DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
+#ifdef CONFIG_SLIRP
+"-netdev user,id=str[,ipv4[=on|off]][,net=addr[/mask]][,host=addr]\n"
+"         [,ipv6[=on|off]][,ipv6-net=addr[/int]][,ipv6-host=addr]\n"
+"         [,restrict=on|off][,hostname=host][,dhcpstart=addr]\n"
+"         [,dns=addr][,ipv6-dns=addr][,dnssearch=domain][,tftp=dir]\n"
+"         [,bootfile=f][,hostfwd=rule][,guestfwd=rule]"
+#ifndef _WIN32
+"[,smb=dir[,smbserver=addr]]\n"
+#endif
+"                configure a user mode network backend with ID 'str',\n"
+"                its DHCP server and optional services\n"
+#endif
+#ifdef _WIN32
+"-netdev tap,id=str,ifname=name\n"
+"                configure a host TAP network backend with ID 'str'\n"
+#else
+"-netdev tap,id=str[,fd=h][,fds=x:y:...:z][,ifname=name][,script=file][,downscript=dfile]\n"
+"         [,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off]\n"
+"         [,vhostfd=h][,vhostfds=x:y:...:z][,vhostforce=on|off][,queues=n]\n"
+"         [,poll-us=n]\n"
+"                configure a host TAP network backend with ID 'str'\n"
+"                use network scripts 'file' (default=" DEFAULT_NETWORK_SCRIPT ")\n"
+"                to configure it and 'dfile' (default=" DEFAULT_NETWORK_DOWN_SCRIPT ")\n"
+"                to deconfigure it\n"
+"                use '[down]script=no' to disable script execution\n"
+"                use network helper 'helper' (default=" DEFAULT_BRIDGE_HELPER ") to\n"
+"                configure it\n"
+"                use 'fd=h' to connect to an already opened TAP interface\n"
+"                use 'fds=x:y:...:z' to connect to already opened multiqueue capable TAP interfaces\n"
+"                use 'sndbuf=nbytes' to limit the size of the send buffer (the\n"
+"                default is disabled 'sndbuf=0' to enable flow control set 'sndbuf=1048576')\n"
+"                use vnet_hdr=off to avoid enabling the IFF_VNET_HDR tap flag\n"
+"                use vnet_hdr=on to make the lack of IFF_VNET_HDR support an error condition\n"
+"                use vhost=on to enable experimental in kernel accelerator\n"
+"                    (only has effect for virtio guests which use MSIX)\n"
+"                use vhostforce=on to force vhost on for non-MSIX virtio guests\n"
+"                use 'vhostfd=h' to connect to an already opened vhost net device\n"
+"                use 'vhostfds=x:y:...:z to connect to multiple already opened vhost net devices\n"
+"                use 'queues=n' to specify the number of queues to be created for multiqueue TAP\n"
+"                use 'poll-us=n' to speciy the maximum number of microseconds that could be\n"
+"                spent on busy polling for vhost net\n"
+"-netdev bridge,id=str[,br=bridge][,helper=helper]\n"
+"                configure a host TAP network backend with ID 'str' that is\n"
+"                connected to a bridge (default=" DEFAULT_BRIDGE_INTERFACE ")\n"
+"                using the program 'helper (default=" DEFAULT_BRIDGE_HELPER ")\n"
+#endif
+#ifdef __linux__
+"-netdev l2tpv3,id=str,src=srcaddr,dst=dstaddr[,srcport=srcport][,dstport=dstport]\n"
+"         [,rxsession=rxsession],txsession=txsession[,ipv6=on/off][,udp=on/off]\n"
+"         [,cookie64=on/off][,counter][,pincounter][,txcookie=txcookie]\n"
+"         [,rxcookie=rxcookie][,offset=offset]\n"
+"                configure a network backend with ID 'str' connected to\n"
+"                an Ethernet over L2TPv3 pseudowire.\n"
+"                Linux kernel 3.3+ as well as most routers can talk\n"
+"                L2TPv3. This transport allows connecting a VM to a VM,\n"
+"                VM to a router and even VM to Host. It is a nearly-universal\n"
+"                standard (RFC3391). Note - this implementation uses static\n"
+"                pre-configured tunnels (same as the Linux kernel).\n"
+"                use 'src=' to specify source address\n"
+"                use 'dst=' to specify destination address\n"
+"                use 'udp=on' to specify udp encapsulation\n"
+"                use 'srcport=' to specify source udp port\n"
+"                use 'dstport=' to specify destination udp port\n"
+"                use 'ipv6=on' to force v6\n"
+"                L2TPv3 uses cookies to prevent misconfiguration as\n"
+"                well as a weak security measure\n"
+"                use 'rxcookie=0x012345678' to specify a rxcookie\n"
+"                use 'txcookie=0x012345678' to specify a txcookie\n"
+"                use 'cookie64=on' to set cookie size to 64 bit, otherwise 32\n"
+"                use 'counter=off' to force a 'cut-down' L2TPv3 with no counter\n"
+"                use 'pincounter=on' to work around broken counter handling in peer\n"
+"                use 'offset=X' to add an extra offset between header and data\n"
+#endif
+"-netdev socket,id=str[,fd=h][,listen=[host]:port][,connect=host:port]\n"
+"                configure a network backend to connect to another network\n"
+"                using a socket connection\n"
+"-netdev socket,id=str[,fd=h][,mcast=maddr:port[,localaddr=addr]]\n"
+"                configure a network backend to connect to a multicast maddr and port\n"
+"                use 'localaddr=addr' to specify the host address to send packets from\n"
+"-netdev socket,id=str[,fd=h][,udp=host:port][,localaddr=host:port]\n"
+"                configure a network backend to connect to another network\n"
+"                using an UDP tunnel\n"
+#ifdef CONFIG_VDE
+"-netdev vde,id=str[,sock=socketpath][,port=n][,group=groupname][,mode=octalmode]\n"
+"                configure a network backend to connect to port 'n' of a vde switch\n"
+"                running on host and listening for incoming connections on 'socketpath'.\n"
+"                Use group 'groupname' and mode 'octalmode' to change default\n"
+"                ownership and permissions for communication port.\n"
+#endif
+#ifdef CONFIG_NETMAP
+"-netdev netmap,id=str,ifname=name[,devname=nmname]\n"
+"                attach to the existing netmap-enabled network interface 'name', or to a\n"
+"                VALE port (created on the fly) called 'name' ('nmname' is name of the \n"
+"                netmap device, defaults to '/dev/netmap')\n"
+#endif
+"-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
+"                configure a vhost-user network, backed by a chardev 'dev'\n"
+"-netdev hubport,id=str,hubid=n\n"
+"                configure a hub port on QEMU VLAN 'n'\n", QEMU_ARCH_ALL)
+DEF("net", HAS_ARG, QEMU_OPTION_net,
+"-net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v]\n"
+"                old way to create a new NIC and connect it to VLAN 'n'\n"
+"                (use the '-device devtype,netdev=str' option if possible instead)\n"
+"-net dump[,vlan=n][,file=f][,len=n]\n"
+"                dump traffic on vlan 'n' to file 'f' (max n bytes per packet)\n"
+"-net none       use it alone to have zero network devices. If no -net option\n"
+"                is provided, the default is '-net nic -net user'\n"
+"-net ["
+#ifdef CONFIG_SLIRP
+"user|"
+#endif
+"tap|"
+"bridge|"
+#ifdef CONFIG_VDE
+"vde|"
+#endif
+#ifdef CONFIG_NETMAP
+"netmap|"
+#endif
+"socket][,vlan=n][,option][,option][,...]\n"
+"                old way to initialize a host network interface\n"
+"                (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+DEFHEADING(Character device options:)
+
+DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
+"-chardev null,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev socket,id=id[,host=host],port=port[,to=to][,ipv4][,ipv6][,nodelay][,reconnect=seconds]\n"
+"         [,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off]\n"
+"         [,logfile=PATH][,logappend=on|off][,tls-creds=ID] (tcp)\n"
+"-chardev socket,id=id,path=path[,server][,nowait][,telnet][,reconnect=seconds]\n"
+"         [,mux=on|off][,logfile=PATH][,logappend=on|off] (unix)\n"
+"-chardev udp,id=id[,host=host],port=port[,localaddr=localaddr]\n"
+"         [,localport=localport][,ipv4][,ipv6][,mux=on|off]\n"
+"         [,logfile=PATH][,logappend=on|off]\n"
+"-chardev msmouse,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]]\n"
+"         [,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev ringbuf,id=id[,size=size][,logfile=PATH][,logappend=on|off]\n"
+"-chardev file,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev pipe,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+#ifdef _WIN32
+"-chardev console,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+#else
+"-chardev pty,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev stdio,id=id[,mux=on|off][,signal=on|off][,logfile=PATH][,logappend=on|off]\n"
+#endif
+#ifdef CONFIG_BRLAPI
+"-chardev braille,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+#endif
+#if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
+|| defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
+"-chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev tty,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+#endif
+#if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__)
+"-chardev parallel,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+"-chardev parport,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+#endif
+#if defined(CONFIG_SPICE)
+"-chardev spicevmc,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off]\n"
+"-chardev spiceport,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off]\n"
+#endif
+, QEMU_ARCH_ALL
+)
+
+
+DEFHEADING()
+
+DEFHEADING(Device URL Syntax:)
+DEF("iscsi", HAS_ARG, QEMU_OPTION_iscsi,
+"-iscsi [user=user][,password=password]\n"
+"       [,header-digest=CRC32C|CR32C-NONE|NONE-CRC32C|NONE\n"
+"       [,initiator-name=initiator-iqn][,id=target-iqn]\n"
+"       [,timeout=timeout]\n"
+"                iSCSI session parameters\n", QEMU_ARCH_ALL)
+
+
+DEFHEADING(Bluetooth(R) options:)
+
+DEF("bt", HAS_ARG, QEMU_OPTION_bt, \
+"-bt hci,null    dumb bluetooth HCI - doesn't respond to commands\n" \
+"-bt hci,host[:id]\n" \
+"                use host's HCI with the given name\n" \
+"-bt hci[,vlan=n]\n" \
+"                emulate a standard HCI in virtual scatternet 'n'\n" \
+"-bt vhci[,vlan=n]\n" \
+"                add host computer to virtual scatternet 'n' using VHCI\n" \
+"-bt device:dev[,vlan=n]\n" \
+"                emulate a bluetooth device 'dev' in scatternet 'n'\n",
+QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+#ifdef CONFIG_TPM
+DEFHEADING(TPM device options:)
+
+DEF("tpmdev", HAS_ARG, QEMU_OPTION_tpmdev, \
+"-tpmdev passthrough,id=id[,path=path][,cancel-path=path]\n"
+"                use path to provide path to a character device; default is /dev/tpm0\n"
+"                use cancel-path to provide path to TPM's cancel sysfs entry; if\n"
+"                not provided it will be searched for in /sys/class/misc/tpm?/device\n",
+QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+#endif
+
+DEFHEADING(Linux/Multiboot boot specific:)
+
+DEF("kernel", HAS_ARG, QEMU_OPTION_kernel, \
+"-kernel bzImage use 'bzImage' as kernel image\n", QEMU_ARCH_ALL)
+
+DEF("append", HAS_ARG, QEMU_OPTION_append, \
+"-append cmdline use 'cmdline' as kernel command line\n", QEMU_ARCH_ALL)
+
+DEF("initrd", HAS_ARG, QEMU_OPTION_initrd, \
+"-initrd file    use 'file' as initial ram disk\n", QEMU_ARCH_ALL)
+
+DEF("dtb", HAS_ARG, QEMU_OPTION_dtb, \
+"-dtb    file    use 'file' as device tree image\n", QEMU_ARCH_ALL)
+
+DEFHEADING()
+
+DEFHEADING(Debug/Expert options:)
+
+DEF("fw_cfg", HAS_ARG, QEMU_OPTION_fwcfg,
+"-fw_cfg [name=]<name>,file=<file>\n"
+"                add named fw_cfg entry with contents from file\n"
+"-fw_cfg [name=]<name>,string=<str>\n"
+"                add named fw_cfg entry with contents from string\n",
+QEMU_ARCH_ALL)
+
+DEF("serial", HAS_ARG, QEMU_OPTION_serial, \
+"-serial dev     redirect the serial port to char device 'dev'\n",
+QEMU_ARCH_ALL)
+
+DEF("parallel", HAS_ARG, QEMU_OPTION_parallel, \
+"-parallel dev   redirect the parallel port to char device 'dev'\n",
+QEMU_ARCH_ALL)
+
+DEF("monitor", HAS_ARG, QEMU_OPTION_monitor, \
+"-monitor dev    redirect the monitor to char device 'dev'\n",
+QEMU_ARCH_ALL)
+DEF("qmp", HAS_ARG, QEMU_OPTION_qmp, \
+"-qmp dev        like -monitor but opens in 'control' mode\n",
+QEMU_ARCH_ALL)
+DEF("qmp-pretty", HAS_ARG, QEMU_OPTION_qmp_pretty, \
+"-qmp-pretty dev like -qmp but uses pretty JSON formatting\n",
+QEMU_ARCH_ALL)
+
+DEF("mon", HAS_ARG, QEMU_OPTION_mon, \
+"-mon [chardev=]name[,mode=readline|control][,default]\n", QEMU_ARCH_ALL)
+
+DEF("debugcon", HAS_ARG, QEMU_OPTION_debugcon, \
+"-debugcon dev   redirect the debug console to char device 'dev'\n",
+QEMU_ARCH_ALL)
+
+DEF("pidfile", HAS_ARG, QEMU_OPTION_pidfile, \
+"-pidfile file   write PID to 'file'\n", QEMU_ARCH_ALL)
+
+DEF("singlestep", 0, QEMU_OPTION_singlestep, \
+"-singlestep     always run in singlestep mode\n", QEMU_ARCH_ALL)
+
+DEF("S", 0, QEMU_OPTION_S, \
+"-S              freeze CPU at startup (use 'c' to start execution)\n",
+QEMU_ARCH_ALL)
+
+DEF("realtime", HAS_ARG, QEMU_OPTION_realtime,
+"-realtime [mlock=on|off]\n"
+"                run qemu with realtime features\n"
+"                mlock=on|off controls mlock support (default: on)\n",
+QEMU_ARCH_ALL)
+
+DEF("gdb", HAS_ARG, QEMU_OPTION_gdb, \
+"-gdb dev        wait for gdb connection on 'dev'\n", QEMU_ARCH_ALL)
+
+DEF("s", 0, QEMU_OPTION_s, \
+"-s              shorthand for -gdb tcp::" DEFAULT_GDBSTUB_PORT "\n",
+QEMU_ARCH_ALL)
+
+DEF("d", HAS_ARG, QEMU_OPTION_d, \
+"-d item1,...    enable logging of specified items (use '-d help' for a list of log items)\n",
+QEMU_ARCH_ALL)
+
+DEF("D", HAS_ARG, QEMU_OPTION_D, \
+"-D logfile      output log to logfile (default stderr)\n",
+QEMU_ARCH_ALL)
+
+DEF("dfilter", HAS_ARG, QEMU_OPTION_DFILTER, \
+"-dfilter range,..  filter debug output to range of addresses (useful for -d cpu,exec,etc..)\n",
+QEMU_ARCH_ALL)
+
+DEF("L", HAS_ARG, QEMU_OPTION_L, \
+"-L path         set the directory for the BIOS, VGA BIOS and keymaps\n",
+QEMU_ARCH_ALL)
+
+DEF("bios", HAS_ARG, QEMU_OPTION_bios, \
+"-bios file      set the filename for the BIOS\n", QEMU_ARCH_ALL)
+
+DEF("enable-kvm", 0, QEMU_OPTION_enable_kvm, \
+"-enable-kvm     enable KVM full virtualization support\n", QEMU_ARCH_ALL)
+
+DEF("enable-hax", 0, QEMU_OPTION_enable_hax, \
+"-enable-hax     enable HAX virtualization support\n", QEMU_ARCH_I386)
+
+DEF("xen-domid", HAS_ARG, QEMU_OPTION_xen_domid,
+"-xen-domid id   specify xen guest domain id\n", QEMU_ARCH_ALL)
+DEF("xen-create", 0, QEMU_OPTION_xen_create,
+"-xen-create     create domain using xen hypercalls, bypassing xend\n"
+"                warning: should not be used when xend is in use\n",
+QEMU_ARCH_ALL)
+DEF("xen-attach", 0, QEMU_OPTION_xen_attach,
+"-xen-attach     attach to existing xen domain\n"
+"                xend will use this when starting QEMU\n",
+QEMU_ARCH_ALL)
+
+DEF("no-reboot", 0, QEMU_OPTION_no_reboot, \
+"-no-reboot      exit instead of rebooting\n", QEMU_ARCH_ALL)
+
+DEF("no-shutdown", 0, QEMU_OPTION_no_shutdown, \
+"-no-shutdown    stop before shutdown\n", QEMU_ARCH_ALL)
+
+DEF("loadvm", HAS_ARG, QEMU_OPTION_loadvm, \
+"-loadvm [tag|id]\n" \
+"                start right away with a saved state (loadvm in monitor)\n",
+QEMU_ARCH_ALL)
+
+#ifndef _WIN32
+DEF("daemonize", 0, QEMU_OPTION_daemonize, \
+"-daemonize      daemonize QEMU after initializing\n", QEMU_ARCH_ALL)
+#endif
+
+DEF("option-rom", HAS_ARG, QEMU_OPTION_option_rom, \
+"-option-rom rom load a file, rom, into the option ROM space\n",
+QEMU_ARCH_ALL)
+
+DEF("clock", HAS_ARG, QEMU_OPTION_clock, "", QEMU_ARCH_ALL)
+
+DEF("localtime", 0, QEMU_OPTION_localtime, "", QEMU_ARCH_ALL)
+DEF("startdate", HAS_ARG, QEMU_OPTION_startdate, "", QEMU_ARCH_ALL)
+
+DEF("rtc", HAS_ARG, QEMU_OPTION_rtc, \
+"-rtc [base=utc|localtime|date][,clock=host|rt|vm][,driftfix=none|slew]\n" \
+"                set the RTC base and clock, enable drift fix for clock ticks (x86 only)\n",
+QEMU_ARCH_ALL)
+
+
+DEF("icount", HAS_ARG, QEMU_OPTION_icount, \
+"-icount [shift=N|auto][,align=on|off][,sleep=on|off,rr=record|replay,rrfile=<filename>]\n" \
+"                enable virtual instruction counter with 2^N clock ticks per\n" \
+"                instruction, enable aligning the host and virtual clocks\n" \
+"                or disable real time cpu sleeping\n", QEMU_ARCH_ALL)
+
+DEF("watchdog", HAS_ARG, QEMU_OPTION_watchdog, \
+"-watchdog model\n" \
+"                enable virtual hardware watchdog [default=none]\n",
+QEMU_ARCH_ALL)
+
+DEF("watchdog-action", HAS_ARG, QEMU_OPTION_watchdog_action, \
+"-watchdog-action reset|shutdown|poweroff|pause|debug|none\n" \
+"                action when watchdog fires [default=reset]\n",
+QEMU_ARCH_ALL)
+
+DEF("echr", HAS_ARG, QEMU_OPTION_echr, \
+"-echr chr       set terminal escape character instead of ctrl-a\n",
+QEMU_ARCH_ALL)
+
+DEF("virtioconsole", HAS_ARG, QEMU_OPTION_virtiocon, \
+"-virtioconsole c\n" \
+"                set virtio console\n", QEMU_ARCH_ALL)
+
+DEF("show-cursor", 0, QEMU_OPTION_show_cursor, \
+"-show-cursor    show cursor\n", QEMU_ARCH_ALL)
+
+DEF("tb-size", HAS_ARG, QEMU_OPTION_tb_size, \
+"-tb-size n      set TB size\n", QEMU_ARCH_ALL)
+
+DEF("incoming", HAS_ARG, QEMU_OPTION_incoming, \
+"-incoming tcp:[host]:port[,to=maxport][,ipv4][,ipv6]\n" \
+"-incoming rdma:host:port[,ipv4][,ipv6]\n" \
+"-incoming unix:socketpath\n" \
+"                prepare for incoming migration, listen on\n" \
+"                specified protocol and socket address\n" \
+"-incoming fd:fd\n" \
+"-incoming exec:cmdline\n" \
+"                accept incoming migration on given file descriptor\n" \
+"                or from given external command\n" \
+"-incoming defer\n" \
+"                wait for the URI to be specified via migrate_incoming\n",
+QEMU_ARCH_ALL)
+
+DEF("nodefaults", 0, QEMU_OPTION_nodefaults, \
+"-nodefaults     don't create default devices\n", QEMU_ARCH_ALL)
+
+#ifndef _WIN32
+DEF("chroot", HAS_ARG, QEMU_OPTION_chroot, \
+"-chroot dir     chroot to dir just before starting the VM\n",
+QEMU_ARCH_ALL)
+#endif
+
+#ifndef _WIN32
+DEF("runas", HAS_ARG, QEMU_OPTION_runas, \
+"-runas user     change to user id user just before starting the VM\n",
+QEMU_ARCH_ALL)
+#endif
+
+DEF("prom-env", HAS_ARG, QEMU_OPTION_prom_env,
+"-prom-env variable=value\n"
+"                set OpenBIOS nvram variables\n",
+QEMU_ARCH_PPC | QEMU_ARCH_SPARC)
+DEF("semihosting", 0, QEMU_OPTION_semihosting,
+"-semihosting    semihosting mode\n",
+QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_LM32 |
+QEMU_ARCH_MIPS)
+DEF("semihosting-config", HAS_ARG, QEMU_OPTION_semihosting_config,
+"-semihosting-config [enable=on|off][,target=native|gdb|auto][,arg=str[,...]]\n" \
+"                semihosting configuration\n",
+QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_LM32 |
+QEMU_ARCH_MIPS)
+DEF("old-param", 0, QEMU_OPTION_old_param,
+"-old-param      old param mode\n", QEMU_ARCH_ARM)
+
+DEF("sandbox", HAS_ARG, QEMU_OPTION_sandbox, \
+"-sandbox <arg>  Enable seccomp mode 2 system call filter (default 'off').\n",
+QEMU_ARCH_ALL)
+
+DEF("readconfig", HAS_ARG, QEMU_OPTION_readconfig,
+"-readconfig <file>\n", QEMU_ARCH_ALL)
+DEF("writeconfig", HAS_ARG, QEMU_OPTION_writeconfig,
+"-writeconfig <file>\n"
+"                read/write config file\n", QEMU_ARCH_ALL)
+DEF("nodefconfig", 0, QEMU_OPTION_nodefconfig,
+"-nodefconfig\n"
+"                do not load default config files at startup\n",
+QEMU_ARCH_ALL)
+DEF("no-user-config", 0, QEMU_OPTION_nouserconfig,
+"-no-user-config\n"
+"                do not load user-provided config files at startup\n",
+QEMU_ARCH_ALL)
+DEF("trace", HAS_ARG, QEMU_OPTION_trace,
+"-trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
+"                specify tracing options\n",
+QEMU_ARCH_ALL)
+
+DEF("qtest", HAS_ARG, QEMU_OPTION_qtest, "", QEMU_ARCH_ALL)
+DEF("qtest-log", HAS_ARG, QEMU_OPTION_qtest_log, "", QEMU_ARCH_ALL)
+
+#ifdef __linux__
+DEF("enable-fips", 0, QEMU_OPTION_enablefips,
+"-enable-fips    enable FIPS 140-2 compliance\n",
+QEMU_ARCH_ALL)
+#endif
+
+DEF("no-kvm", 0, QEMU_OPTION_no_kvm, "", QEMU_ARCH_I386)
+
+DEF("no-kvm-pit-reinjection", 0, QEMU_OPTION_no_kvm_pit_reinjection,
+"", QEMU_ARCH_I386)
+
+DEF("no-kvm-pit", 0, QEMU_OPTION_no_kvm_pit, "", QEMU_ARCH_I386)
+
+DEF("no-kvm-irqchip", 0, QEMU_OPTION_no_kvm_irqchip, "", QEMU_ARCH_I386)
+
+DEF("tdf", 0, QEMU_OPTION_tdf,"", QEMU_ARCH_ALL)
+
+DEF("msg", HAS_ARG, QEMU_OPTION_msg,
+"-msg timestamp[=on|off]\n"
+"                change the format of messages\n"
+"                on|off controls leading timestamps (default:on)\n",
+QEMU_ARCH_ALL)
+
+DEF("dump-vmstate", HAS_ARG, QEMU_OPTION_dump_vmstate,
+"-dump-vmstate <file>\n"
+"                Output vmstate information in JSON format to file.\n"
+"                Use the scripts/vmstate-static-checker.py file to\n"
+"                check for possible regressions in migration code\n"
+"                by comparing two such vmstate dumps.\n",
+QEMU_ARCH_ALL)
+
+DEFHEADING(Generic object creation)
+
+DEF("object", HAS_ARG, QEMU_OPTION_object,
+"-object TYPENAME[,PROP1=VALUE1,...]\n"
+"                create a new object of type TYPENAME setting properties\n"
+"                in the order they are specified.  Note that the 'id'\n"
+"                property must be set.  These objects are placed in the\n"
+"                '/objects' path.\n",
+QEMU_ARCH_ALL)
+
+#ifdef CONFIG_ANDROID
+
+DEF("boot-property", HAS_ARG, QEMU_OPTION_boot_property, \
+"-boot-property <name>=<value> set system property on boot\n", QEMU_ARCH_ALL)
+
+DEF("lcd-density", HAS_ARG, QEMU_OPTION_lcd_density,
+"-lcd-density density\n"
+"                select lcd display density value (default is 160)\n", QEMU_ARCH_ALL)
+
+DEF("android-ports", HAS_ARG, QEMU_OPTION_android_ports,
+"-android-ports <consoleport>,<adbport>"
+" TCP ports used for the emulator instance and adb bridge\n", QEMU_ARCH_ALL)
+
+DEF("android-report-console", HAS_ARG, QEMU_OPTION_android_report_console, \
+"-android-report-console <socket>"
+" report console port to remote socket\n", QEMU_ARCH_ALL)
+
+DEF("http-proxy", HAS_ARG, QEMU_OPTION_http_proxy,
+"-http-proxy <proxy> make TCP connections through a HTTP/HTTPS proxy\n", QEMU_ARCH_ALL)
+
+DEF("dns-server", HAS_ARG, QEMU_OPTION_dns_server,
+"-dns-server <servers> use this DNS server(s) in the emulated system\n", QEMU_ARCH_ALL)
+
+DEF("android-hw", HAS_ARG, QEMU_OPTION_android_hw,
+"-android-hw <file>\n"
+"                specify the hw config ini file location\n", QEMU_ARCH_ALL)
+
+DEF("skip-adb-auth", 0, QEMU_OPTION_android_skip_adb_auth,
+"-skip-adb-auth  skip adb authentication dialogue\n", QEMU_ARCH_ALL)
+
+DEF("timezone", HAS_ARG, QEMU_OPTION_timezone,
+"-timezone <timezone> use this timezone instead of host's default\n", QEMU_ARCH_ALL)
+
+#endif  // CONFIG_ANDROID
+
+
diff --git a/qemu-options.h b/qemu-options.h
index 89a009e..b4ee63c 100644
--- a/qemu-options.h
+++ b/qemu-options.h
@@ -25,8 +25,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef _QEMU_OPTIONS_H_
-#define _QEMU_OPTIONS_H_
+#ifndef QEMU_OPTIONS_H
+#define QEMU_OPTIONS_H
 
 enum {
 #define QEMU_OPTIONS_GENERATE_ENUM
diff --git a/qemu-options.hx b/qemu-options.hx
index 92e99be..1b0e6a0 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -33,11 +33,17 @@
     "                property accel=accel1[:accel2[:...]] selects accelerator\n"
     "                supported accelerators are kvm, xen, tcg (default: tcg)\n"
     "                kernel_irqchip=on|off controls accelerated irqchip support\n"
+    "                kernel_irqchip=on|off|split controls accelerated irqchip support (default=off)\n"
     "                vmport=on|off|auto controls emulation of vmport (default: auto)\n"
-    "                kvm_shadow_mem=size of KVM shadow MMU\n"
+    "                kvm_shadow_mem=size of KVM shadow MMU in bytes\n"
     "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
     "                mem-merge=on|off controls memory merge support (default: on)\n"
-    "                iommu=on|off controls emulated Intel IOMMU (VT-d) support (default=off)\n",
+    "                igd-passthru=on|off controls IGD GFX passthrough support (default=off)\n"
+    "                aes-key-wrap=on|off controls support for AES key wrapping (default=on)\n"
+    "                dea-key-wrap=on|off controls support for DEA key wrapping (default=on)\n"
+    "                suppress-vmdesc=on|off disables self-describing migration (default=off)\n"
+    "                nvdimm=on|off controls NVDIMM support (default=off)\n"
+    "                enforce-config-section=on|off enforce configuration section migration (default=off)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -machine [type=]@var{name}[,prop=@var{value}[,...]]
@@ -51,7 +57,9 @@
 than one accelerator specified, the next one is used if the previous one fails
 to initialize.
 @item kernel_irqchip=on|off
-Enables in-kernel irqchip support for the chosen accelerator when available.
+Controls in-kernel irqchip support for the chosen accelerator when available.
+@item gfx_passthru=on|off
+Enables IGD GFX passthrough support for the chosen machine when available.
 @item vmport=on|off|auto
 Enables emulation of VMWare IO port, for vmmouse etc. auto says to select the
 value based on accel. For accel=xen the default is off otherwise the default
@@ -64,8 +72,16 @@
 Enables or disables memory merge support. This feature, when supported by
 the host, de-duplicates identical memory pages among VMs instances
 (enabled by default).
-@item iommu=on|off
-Enables or disables emulated Intel IOMMU (VT-d) support. The default is off.
+@item aes-key-wrap=on|off
+Enables or disables AES key wrapping support on s390-ccw hosts. This feature
+controls whether AES wrapping keys will be created to allow
+execution of AES cryptographic functions.  The default is on.
+@item dea-key-wrap=on|off
+Enables or disables DEA key wrapping support on s390-ccw hosts. This feature
+controls whether DEA wrapping keys will be created to allow
+execution of DEA cryptographic functions.  The default is on.
+@item nvdimm=on|off
+Enables or disables NVDIMM support. The default is off.
 @end table
 ETEXI
 
@@ -107,7 +123,7 @@
     "-numa node[,memdev=id][,cpus=cpu[-cpu]][,nodeid=node]\n", QEMU_ARCH_ALL)
 STEXI
 @item -numa node[,mem=@var{size}][,cpus=@var{cpu[-cpu]}][,nodeid=@var{node}]
-@item -numa node[,memdev=@var{id}][,cpus=@var{cpu[-cpu]}][,nodeid=@var{node}]
+@itemx -numa node[,memdev=@var{id}][,cpus=@var{cpu[-cpu]}][,nodeid=@var{node}]
 @findex -numa
 Simulate a multi node NUMA system. If @samp{mem}, @samp{memdev}
 and @samp{cpus} are omitted, resources are split equally. Also, note
@@ -160,11 +176,13 @@
 ETEXI
 
 DEF("global", HAS_ARG, QEMU_OPTION_global,
-    "-global driver.prop=value\n"
+    "-global driver.property=value\n"
+    "-global driver=driver,property=property,value=value\n"
     "                set a global default for a driver property\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -global @var{driver}.@var{prop}=@var{value}
+@itemx -global driver=@var{driver},property=@var{property},value=@var{value}
 @findex -global
 Set default value of @var{driver}'s property @var{prop} to @var{value}, e.g.:
 
@@ -175,6 +193,10 @@
 In particular, you can use this to set driver properties for devices which are
 created automatically by the machine model. To create a device which is not
 created automatically and set properties on it, use -@option{device}.
+
+-global @var{driver}.@var{prop}=@var{value} is shorthand for -global
+driver=@var{driver},property=@var{prop},value=@var{value}.  The
+longhand syntax works even when @var{driver} contains a dot.
 ETEXI
 
 DEF("boot", HAS_ARG, QEMU_OPTION_boot,
@@ -189,7 +211,7 @@
 @item -boot [order=@var{drives}][,once=@var{drives}][,menu=on|off][,splash=@var{sp_name}][,splash-time=@var{sp_time}][,reboot-timeout=@var{rb_timeout}][,strict=on|off]
 @findex -boot
 Specify boot order @var{drives} as a string of drive letters. Valid
-drive letters depend on the target achitecture. The x86 PC uses: a, b
+drive letters depend on the target architecture. The x86 PC uses: a, b
 (floppy 1 and 2), c (first hard disk), d (first CD-ROM), n-p (Etherboot
 from network adapter 1-4), hard disk boot is the default. To apply a
 particular boot order only on the first startup, specify it via
@@ -230,19 +252,30 @@
 DEF("m", HAS_ARG, QEMU_OPTION_m,
     "-m[emory] [size=]megs[,slots=n,maxmem=size]\n"
     "                configure guest RAM\n"
-    "                size: initial amount of guest memory (default: "
-    stringify(DEFAULT_RAM_SIZE) "MiB)\n"
+    "                size: initial amount of guest memory\n"
     "                slots: number of hotplug slots (default: none)\n"
     "                maxmem: maximum amount of guest memory (default: none)\n"
     "NOTE: Some architectures might enforce a specific granularity\n",
     QEMU_ARCH_ALL)
 STEXI
-@item -m [size=]@var{megs}
+@item -m [size=]@var{megs}[,slots=n,maxmem=size]
 @findex -m
-Set virtual RAM size to @var{megs} megabytes. Default is 128 MiB.  Optionally,
-a suffix of ``M'' or ``G'' can be used to signify a value in megabytes or
-gigabytes respectively. Optional pair @var{slots}, @var{maxmem} could be used
-to set amount of hotluggable memory slots and possible maximum amount of memory.
+Sets guest startup RAM size to @var{megs} megabytes. Default is 128 MiB.
+Optionally, a suffix of ``M'' or ``G'' can be used to signify a value in
+megabytes or gigabytes respectively. Optional pair @var{slots}, @var{maxmem}
+could be used to set amount of hotpluggable memory slots and maximum amount of
+memory. Note that @var{maxmem} must be aligned to the page size.
+
+For example, the following command-line sets the guest startup RAM size to
+1GB, creates 3 slots to hotplug additional memory and sets the maximum
+memory the guest can reach to 4GB:
+
+@example
+qemu-system-x86_64 -m 1G,slots=3,maxmem=4G
+@end example
+
+If @var{slots} and @var{maxmem} are not specified, memory hotplug won't
+be enabled and the guest startup RAM will never increase.
 ETEXI
 
 DEF("mem-path", HAS_ARG, QEMU_OPTION_mempath,
@@ -360,6 +393,58 @@
 properties.  Valid properties depend on the driver.  To get help on
 possible drivers and properties, use @code{-device help} and
 @code{-device @var{driver},help}.
+
+Some drivers are:
+@item -device ipmi-bmc-sim,id=@var{id}[,slave_addr=@var{val}]
+
+Add an IPMI BMC.  This is a simulation of a hardware management
+interface processor that normally sits on a system.  It provides
+a watchdog and the ability to reset and power control the system.
+You need to connect this to an IPMI interface to make it useful
+
+The IPMI slave address to use for the BMC.  The default is 0x20.
+This address is the BMC's address on the I2C network of management
+controllers.  If you don't know what this means, it is safe to ignore
+it.
+
+@item -device ipmi-bmc-extern,id=@var{id},chardev=@var{id}[,slave_addr=@var{val}]
+
+Add a connection to an external IPMI BMC simulator.  Instead of
+locally emulating the BMC like the above item, instead connect
+to an external entity that provides the IPMI services.
+
+A connection is made to an external BMC simulator.  If you do this, it
+is strongly recommended that you use the "reconnect=" chardev option
+to reconnect to the simulator if the connection is lost.  Note that if
+this is not used carefully, it can be a security issue, as the
+interface has the ability to send resets, NMIs, and power off the VM.
+It's best if QEMU makes a connection to an external simulator running
+on a secure port on localhost, so neither the simulator nor QEMU is
+exposed to any outside network.
+
+See the "lanserv/README.vm" file in the OpenIPMI library for more
+details on the external interface.
+
+@item -device isa-ipmi-kcs,bmc=@var{id}[,ioport=@var{val}][,irq=@var{val}]
+
+Add a KCS IPMI interafce on the ISA bus.  This also adds a
+corresponding ACPI and SMBIOS entries, if appropriate.
+
+@table @option
+@item bmc=@var{id}
+The BMC to connect to, one of ipmi-bmc-sim or ipmi-bmc-extern above.
+@item ioport=@var{val}
+Define the I/O address of the interface.  The default is 0xca0 for KCS.
+@item irq=@var{val}
+Define the interrupt to use.  The default is 5.  To disable interrupts,
+set this to 0.
+@end table
+
+@item -device isa-ipmi-bt,bmc=@var{id}[,ioport=@var{val}][,irq=@var{val}]
+
+Like the KCS interface, but defines a BT interface.  The default port is
+0xe4 and the default interrupt is 5.
+
 ETEXI
 
 DEF("name", HAS_ARG, QEMU_OPTION_name,
@@ -403,11 +488,10 @@
 DEF("fdb", HAS_ARG, QEMU_OPTION_fdb, "", QEMU_ARCH_ALL)
 STEXI
 @item -fda @var{file}
-@item -fdb @var{file}
+@itemx -fdb @var{file}
 @findex -fda
 @findex -fdb
-Use @var{file} as floppy disk 0/1 image (@pxref{disk_images}). You can
-use the host floppy by using @file{/dev/fd0} as filename (@pxref{host_drives}).
+Use @var{file} as floppy disk 0/1 image (@pxref{disk_images}).
 ETEXI
 
 DEF("hda", HAS_ARG, QEMU_OPTION_hda,
@@ -418,9 +502,9 @@
 DEF("hdd", HAS_ARG, QEMU_OPTION_hdd, "", QEMU_ARCH_ALL)
 STEXI
 @item -hda @var{file}
-@item -hdb @var{file}
-@item -hdc @var{file}
-@item -hdd @var{file}
+@itemx -hdb @var{file}
+@itemx -hdc @var{file}
+@itemx -hdd @var{file}
 @findex -hda
 @findex -hdb
 @findex -hdc
@@ -452,6 +536,7 @@
     "       [[,bps_max=bm]|[[,bps_rd_max=rm][,bps_wr_max=wm]]]\n"
     "       [[,iops_max=im]|[[,iops_rd_max=irm][,iops_wr_max=iwm]]]\n"
     "       [[,iops_size=is]]\n"
+    "       [[,group=g]]\n"
     "                use 'file' as a drive image\n", QEMU_ARCH_ALL)
 STEXI
 @item -drive @var{option}[,@var{option}[,@var{option}[,...]]]
@@ -491,7 +576,7 @@
 @var{discard} is one of "ignore" (or "off") or "unmap" (or "on") and controls whether @dfn{discard} (also known as @dfn{trim} or @dfn{unmap}) requests are ignored or passed to the filesystem.  Some machine types may not support discard requests.
 @item format=@var{format}
 Specify which disk @var{format} will be used rather than detecting
-the format.  Can be used to specifiy format=raw to avoid interpreting
+the format.  Can be used to specify format=raw to avoid interpreting
 an untrusted format header.
 @item serial=@var{serial}
 This option specifies the serial number to assign to the device.
@@ -816,7 +901,7 @@
 
 @item disk:[format=@var{format}]:@var{file}
 Mass storage device based on file. The optional @var{format} argument
-will be used rather than detecting the format. Can be used to specifiy
+will be used rather than detecting the format. Can be used to specify
 @code{format=raw} to avoid interpreting an untrusted format header.
 
 @item host:@var{bus}.@var{addr}
@@ -852,10 +937,25 @@
 
 DEF("display", HAS_ARG, QEMU_OPTION_display,
     "-display sdl[,frame=on|off][,alt_grab=on|off][,ctrl_grab=on|off]\n"
-    "            [,window_close=on|off]|curses|none|\n"
-    "            gtk[,grab_on_hover=on|off]|\n"
-    "            vnc=<display>[,<optargs>]\n"
-    "                select display type\n", QEMU_ARCH_ALL)
+    "            [,window_close=on|off][,gl=on|off]|curses|none|\n"
+    "-display gtk[,grab_on_hover=on|off][,gl=on|off]|\n"
+    "-display vnc=<display>[,<optargs>]\n"
+    "-display curses\n"
+    "-display none"
+    "                select display type\n"
+    "The default display is equivalent to\n"
+#if defined(CONFIG_GTK)
+            "\t\"-display gtk\"\n"
+#elif defined(CONFIG_SDL)
+            "\t\"-display sdl\"\n"
+#elif defined(CONFIG_COCOA)
+            "\t\"-display cocoa\"\n"
+#elif defined(CONFIG_VNC)
+            "\t\"-vnc localhost:0,to=99,id=default\"\n"
+#else
+            "\t\"-display none\"\n"
+#endif
+    , QEMU_ARCH_ALL)
 STEXI
 @item -display @var{type}
 @findex -display
@@ -902,7 +1002,7 @@
 ETEXI
 
 DEF("curses", 0, QEMU_OPTION_curses,
-    "-curses         use a curses/ncurses interface instead of SDL\n",
+    "-curses         shorthand for -display curses\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -curses
@@ -952,7 +1052,7 @@
 ETEXI
 
 DEF("sdl", 0, QEMU_OPTION_sdl,
-    "-sdl            enable SDL\n", QEMU_ARCH_ALL)
+    "-sdl            shorthand for -display sdl\n", QEMU_ARCH_ALL)
 STEXI
 @item -sdl
 @findex -sdl
@@ -963,7 +1063,7 @@
     "-spice [port=port][,tls-port=secured-port][,x509-dir=<dir>]\n"
     "       [,x509-key-file=<file>][,x509-key-password=<file>]\n"
     "       [,x509-cert-file=<file>][,x509-cacert-file=<file>]\n"
-    "       [,x509-dh-key-file=<file>][,addr=addr][,ipv4|ipv6]\n"
+    "       [,x509-dh-key-file=<file>][,addr=addr][,ipv4|ipv6|unix]\n"
     "       [,tls-ciphers=<list>]\n"
     "       [,tls-channel=[main|display|cursor|inputs|record|playback]]\n"
     "       [,plaintext-channel=[main|display|cursor|inputs|record|playback]]\n"
@@ -974,6 +1074,7 @@
     "       [,streaming-video=[off|all|filter]][,disable-copy-paste]\n"
     "       [,disable-agent-file-xfer][,agent-mouse=[on|off]]\n"
     "       [,playback-compression=[on|off]][,seamless-migration=[on|off]]\n"
+    "       [,gl=[on|off]]\n"
     "   enable spice\n"
     "   at least one of {port, tls-port} is mandatory\n",
     QEMU_ARCH_ALL)
@@ -991,7 +1092,8 @@
 Set the IP address spice is listening on.  Default is any address.
 
 @item ipv4
-@item ipv6
+@itemx ipv6
+@itemx unix
 Force using the specified IP version.
 
 @item password=<secret>
@@ -1026,17 +1128,17 @@
 Set the x509 file directory. Expects same filenames as -vnc $display,x509=$dir
 
 @item x509-key-file=<file>
-@item x509-key-password=<file>
-@item x509-cert-file=<file>
-@item x509-cacert-file=<file>
-@item x509-dh-key-file=<file>
+@itemx x509-key-password=<file>
+@itemx x509-cert-file=<file>
+@itemx x509-cacert-file=<file>
+@itemx x509-dh-key-file=<file>
 The x509 file names can also be configured individually.
 
 @item tls-ciphers=<list>
 Specify which ciphers to use.
 
 @item tls-channel=[main|display|cursor|inputs|record|playback]
-@item plaintext-channel=[main|display|cursor|inputs|record|playback]
+@itemx plaintext-channel=[main|display|cursor|inputs|record|playback]
 Force specific channel to be used with or without TLS encryption.  The
 options can be specified multiple times to configure multiple
 channels.  The special name "default" can be used to set the default
@@ -1048,7 +1150,7 @@
 Default is auto_glz.
 
 @item jpeg-wan-compression=[auto|never|always]
-@item zlib-glz-wan-compression=[auto|never|always]
+@itemx zlib-glz-wan-compression=[auto|never|always]
 Configure wan image compression (lossy for slow links).
 Default is auto.
 
@@ -1064,6 +1166,9 @@
 @item seamless-migration=[on|off]
 Enable/disable spice seamless migration. Default is off.
 
+@item gl=[on|off]
+Enable/disable OpenGL context. Default is off.
+
 @end table
 ETEXI
 
@@ -1086,7 +1191,7 @@
 ETEXI
 
 DEF("vga", HAS_ARG, QEMU_OPTION_vga,
-    "-vga [std|cirrus|vmware|qxl|xenfb|tcx|cg3|none]\n"
+    "-vga [std|cirrus|vmware|qxl|xenfb|tcx|cg3|virtio|none]\n"
     "                select video card type\n", QEMU_ARCH_ALL)
 STEXI
 @item -vga @var{type}
@@ -1119,6 +1224,8 @@
 (sun4m only) Sun cgthree framebuffer. This is a simple 8-bit framebuffer
 for sun4m machines available in both 1024x768 (OpenBIOS) and 1152x900 (OBP)
 resolutions aimed at people wishing to run older Solaris versions.
+@item virtio
+Virtio VGA card.
 @item none
 Disable VGA card.
 @end table
@@ -1142,7 +1249,7 @@
 ETEXI
 
 DEF("vnc", HAS_ARG, QEMU_OPTION_vnc ,
-    "-vnc display    start a VNC server on display\n", QEMU_ARCH_ALL)
+    "-vnc <display>  shorthand for -display vnc=<display>\n", QEMU_ARCH_ALL)
 STEXI
 @item -vnc @var{display}[,@var{option}[,@var{option}[,...]]]
 @findex -vnc
@@ -1156,6 +1263,13 @@
 
 @table @option
 
+@item to=@var{L}
+
+With this option, QEMU will try next available VNC @var{display}s, until the
+number @var{L}, if the origianlly defined "-vnc @var{display}" is not
+available, e.g. port 5900+@var{display} is already used by another
+application. By default, to=0.
+
 @item @var{host}:@var{d}
 
 TCP connections will only be allowed from @var{host} on display @var{d}.
@@ -1193,8 +1307,9 @@
 specified connections will only be allowed from this host.
 As an alternative the Websocket port could be specified by using
 @code{websocket}=@var{port}.
-TLS encryption for the Websocket connection is supported if the required
-certificates are specified with the VNC option @option{x509}.
+If no TLS credentials are provided, the websocket connection runs in
+unencrypted mode. If TLS credentials are provided, the websocket connection
+requires encrypted client connections.
 
 @item password
 
@@ -1215,6 +1330,20 @@
 You can also use keywords "now" or "never" for the expiration time to
 allow <protocol> password to expire immediately or never expire.
 
+@item tls-creds=@var{ID}
+
+Provides the ID of a set of TLS credentials to use to secure the
+VNC server. They will apply to both the normal VNC server socket
+and the websocket socket (if enabled). Setting TLS credentials
+will cause the VNC server socket to enable the VeNCrypt auth
+mechanism.  The credentials should have been previously created
+using the @option{-object tls-creds} argument.
+
+The @option{tls-creds} parameter obsoletes the @option{tls},
+@option{x509}, and @option{x509verify} options, and as such
+it is not permitted to set both new and old type options at
+the same time.
+
 @item tls
 
 Require that client use TLS when communicating with the VNC server. This
@@ -1222,6 +1351,9 @@
 attack. It is recommended that this option be combined with either the
 @option{x509} or @option{x509verify} options.
 
+This option is now deprecated in favor of using the @option{tls-creds}
+argument.
+
 @item x509=@var{/path/to/certificate/dir}
 
 Valid if @option{tls} is specified. Require that x509 credentials are used
@@ -1231,6 +1363,9 @@
 this option specifies where the x509 certificates are to be loaded from.
 See the @ref{vnc_security} section for details on generating certificates.
 
+This option is now deprecated in favour of using the @option{tls-creds}
+argument.
+
 @item x509verify=@var{/path/to/certificate/dir}
 
 Valid if @option{tls} is specified. Require that x509 credentials are used
@@ -1244,6 +1379,9 @@
 be loaded from. See the @ref{vnc_security} section for details on generating
 certificates.
 
+This option is now deprecated in favour of using the @option{tls-creds}
+argument.
+
 @item sasl
 
 Require that the client use SASL to authenticate with the VNC server.
@@ -1301,6 +1439,14 @@
 allows everybody connect unconditionally.  Doesn't conform to the rfb
 spec but is traditional QEMU behavior.
 
+@item key-delay-ms
+
+Set keyboard delay, for key down and key up events, in milliseconds.
+Default is 1.  Keyboards are low-bandwidth devices, so this slowdown
+can help the device and guest to keep up and not lose events in case
+events are arriving in bulk.  Possible causes for the latter are flaky
+network connections, or scripts for automated testing.
+
 @end table
 ETEXI
 
@@ -1339,7 +1485,7 @@
 ETEXI
 
 DEF("no-acpi", 0, QEMU_OPTION_no_acpi,
-           "-no-acpi        disable ACPI\n", QEMU_ARCH_I386)
+           "-no-acpi        disable ACPI\n", QEMU_ARCH_I386 | QEMU_ARCH_ARM)
 STEXI
 @item -no-acpi
 @findex -no-acpi
@@ -1368,16 +1514,34 @@
 For data=, only data
 portion of the table is used, all header information is specified in the
 command line.
+If a SLIC table is supplied to QEMU, then the SLIC's oem_id and oem_table_id
+fields will override the same in the RSDT and the FADT (a.k.a. FACP), in order
+to ensure the field matches required by the Microsoft SLIC spec and the ACPI
+spec.
 ETEXI
 
 DEF("smbios", HAS_ARG, QEMU_OPTION_smbios,
     "-smbios file=binary\n"
     "                load SMBIOS entry from binary file\n"
-    "-smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d][,uefi=on|off]\n"
+    "-smbios type=0[,vendor=str][,version=str][,date=str][,release=%d.%d]\n"
+    "              [,uefi=on|off]\n"
     "                specify SMBIOS type 0 fields\n"
     "-smbios type=1[,manufacturer=str][,product=str][,version=str][,serial=str]\n"
     "              [,uuid=uuid][,sku=str][,family=str]\n"
-    "                specify SMBIOS type 1 fields\n", QEMU_ARCH_I386)
+    "                specify SMBIOS type 1 fields\n"
+    "-smbios type=2[,manufacturer=str][,product=str][,version=str][,serial=str]\n"
+    "              [,asset=str][,location=str]\n"
+    "                specify SMBIOS type 2 fields\n"
+    "-smbios type=3[,manufacturer=str][,version=str][,serial=str][,asset=str]\n"
+    "              [,sku=str]\n"
+    "                specify SMBIOS type 3 fields\n"
+    "-smbios type=4[,sock_pfx=str][,manufacturer=str][,version=str][,serial=str]\n"
+    "              [,asset=str][,part=str]\n"
+    "                specify SMBIOS type 4 fields\n"
+    "-smbios type=17[,loc_pfx=str][,bank=str][,manufacturer=str][,serial=str]\n"
+    "               [,asset=str][,part=str][,speed=%d]\n"
+    "                specify SMBIOS type 17 fields\n",
+    QEMU_ARCH_I386 | QEMU_ARCH_ARM)
 STEXI
 @item -smbios file=@var{binary}
 @findex -smbios
@@ -1386,8 +1550,20 @@
 @item -smbios type=0[,vendor=@var{str}][,version=@var{str}][,date=@var{str}][,release=@var{%d.%d}][,uefi=on|off]
 Specify SMBIOS type 0 fields
 
-@item -smbios type=1[,manufacturer=@var{str}][,product=@var{str}] [,version=@var{str}][,serial=@var{str}][,uuid=@var{uuid}][,sku=@var{str}] [,family=@var{str}]
+@item -smbios type=1[,manufacturer=@var{str}][,product=@var{str}][,version=@var{str}][,serial=@var{str}][,uuid=@var{uuid}][,sku=@var{str}][,family=@var{str}]
 Specify SMBIOS type 1 fields
+
+@item -smbios type=2[,manufacturer=@var{str}][,product=@var{str}][,version=@var{str}][,serial=@var{str}][,asset=@var{str}][,location=@var{str}][,family=@var{str}]
+Specify SMBIOS type 2 fields
+
+@item -smbios type=3[,manufacturer=@var{str}][,version=@var{str}][,serial=@var{str}][,asset=@var{str}][,sku=@var{str}]
+Specify SMBIOS type 3 fields
+
+@item -smbios type=4[,sock_pfx=@var{str}][,manufacturer=@var{str}][,version=@var{str}][,serial=@var{str}][,asset=@var{str}][,part=@var{str}]
+Specify SMBIOS type 4 fields
+
+@item -smbios type=17[,loc_pfx=@var{str}][,bank=@var{str}][,manufacturer=@var{str}][,serial=@var{str}][,asset=@var{str}][,part=@var{str}][,speed=@var{%d}]
+Specify SMBIOS type 17 fields
 ETEXI
 
 STEXI
@@ -1410,25 +1586,28 @@
 #endif
 #endif
 
-DEF("net", HAS_ARG, QEMU_OPTION_net,
-    "-net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v]\n"
-    "                create a new Network Interface Card and connect it to VLAN 'n'\n"
+DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
 #ifdef CONFIG_SLIRP
-    "-net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=on|off]\n"
-    "         [,hostname=host][,dhcpstart=addr][,dns=addr][,dnssearch=domain][,tftp=dir]\n"
+    "-netdev user,id=str[,ipv4[=on|off]][,net=addr[/mask]][,host=addr]\n"
+    "         [,ipv6[=on|off]][,ipv6-net=addr[/int]][,ipv6-host=addr]\n"
+    "         [,restrict=on|off][,hostname=host][,dhcpstart=addr]\n"
+    "         [,dns=addr][,ipv6-dns=addr][,dnssearch=domain][,tftp=dir]\n"
     "         [,bootfile=f][,hostfwd=rule][,guestfwd=rule]"
 #ifndef _WIN32
                                              "[,smb=dir[,smbserver=addr]]\n"
 #endif
-    "                connect the user mode network stack to VLAN 'n', configure its\n"
-    "                DHCP server and enabled optional services\n"
+    "                configure a user mode network backend with ID 'str',\n"
+    "                its DHCP server and optional services\n"
 #endif
 #ifdef _WIN32
-    "-net tap[,vlan=n][,name=str],ifname=name\n"
-    "                connect the host TAP network interface to VLAN 'n'\n"
+    "-netdev tap,id=str,ifname=name\n"
+    "                configure a host TAP network backend with ID 'str'\n"
 #else
-    "-net tap[,vlan=n][,name=str][,fd=h][,fds=x:y:...:z][,ifname=name][,script=file][,downscript=dfile][,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off][,vhostfd=h][,vhostfds=x:y:...:z][,vhostforce=on|off][,queues=n]\n"
-    "                connect the host TAP network interface to VLAN 'n'\n"
+    "-netdev tap,id=str[,fd=h][,fds=x:y:...:z][,ifname=name][,script=file][,downscript=dfile]\n"
+    "         [,helper=helper][,sndbuf=nbytes][,vnet_hdr=on|off][,vhost=on|off]\n"
+    "         [,vhostfd=h][,vhostfds=x:y:...:z][,vhostforce=on|off][,queues=n]\n"
+    "         [,poll-us=n]\n"
+    "                configure a host TAP network backend with ID 'str'\n"
     "                use network scripts 'file' (default=" DEFAULT_NETWORK_SCRIPT ")\n"
     "                to configure it and 'dfile' (default=" DEFAULT_NETWORK_DOWN_SCRIPT ")\n"
     "                to deconfigure it\n"
@@ -1447,14 +1626,20 @@
     "                use 'vhostfd=h' to connect to an already opened vhost net device\n"
     "                use 'vhostfds=x:y:...:z to connect to multiple already opened vhost net devices\n"
     "                use 'queues=n' to specify the number of queues to be created for multiqueue TAP\n"
-    "-net bridge[,vlan=n][,name=str][,br=bridge][,helper=helper]\n"
-    "                connects a host TAP network interface to a host bridge device 'br'\n"
-    "                (default=" DEFAULT_BRIDGE_INTERFACE ") using the program 'helper'\n"
-    "                (default=" DEFAULT_BRIDGE_HELPER ")\n"
+    "                use 'poll-us=n' to speciy the maximum number of microseconds that could be\n"
+    "                spent on busy polling for vhost net\n"
+    "-netdev bridge,id=str[,br=bridge][,helper=helper]\n"
+    "                configure a host TAP network backend with ID 'str' that is\n"
+    "                connected to a bridge (default=" DEFAULT_BRIDGE_INTERFACE ")\n"
+    "                using the program 'helper (default=" DEFAULT_BRIDGE_HELPER ")\n"
 #endif
 #ifdef __linux__
-    "-net l2tpv3[,vlan=n][,name=str],src=srcaddr,dst=dstaddr[,srcport=srcport][,dstport=dstport],txsession=txsession[,rxsession=rxsession][,ipv6=on/off][,udp=on/off][,cookie64=on/off][,counter][,pincounter][,txcookie=txcookie][,rxcookie=rxcookie][,offset=offset]\n"
-    "                connect the VLAN to an Ethernet over L2TPv3 pseudowire\n"
+    "-netdev l2tpv3,id=str,src=srcaddr,dst=dstaddr[,srcport=srcport][,dstport=dstport]\n"
+    "         [,rxsession=rxsession],txsession=txsession[,ipv6=on/off][,udp=on/off]\n"
+    "         [,cookie64=on/off][,counter][,pincounter][,txcookie=txcookie]\n"
+    "         [,rxcookie=rxcookie][,offset=offset]\n"
+    "                configure a network backend with ID 'str' connected to\n"
+    "                an Ethernet over L2TPv3 pseudowire.\n"
     "                Linux kernel 3.3+ as well as most routers can talk\n"
     "                L2TPv3. This transport allows connecting a VM to a VM,\n"
     "                VM to a router and even VM to Host. It is a nearly-universal\n"
@@ -1475,32 +1660,41 @@
     "                use 'pincounter=on' to work around broken counter handling in peer\n"
     "                use 'offset=X' to add an extra offset between header and data\n"
 #endif
-    "-net socket[,vlan=n][,name=str][,fd=h][,listen=[host]:port][,connect=host:port]\n"
-    "                connect the vlan 'n' to another VLAN using a socket connection\n"
-    "-net socket[,vlan=n][,name=str][,fd=h][,mcast=maddr:port[,localaddr=addr]]\n"
-    "                connect the vlan 'n' to multicast maddr and port\n"
+    "-netdev socket,id=str[,fd=h][,listen=[host]:port][,connect=host:port]\n"
+    "                configure a network backend to connect to another network\n"
+    "                using a socket connection\n"
+    "-netdev socket,id=str[,fd=h][,mcast=maddr:port[,localaddr=addr]]\n"
+    "                configure a network backend to connect to a multicast maddr and port\n"
     "                use 'localaddr=addr' to specify the host address to send packets from\n"
-    "-net socket[,vlan=n][,name=str][,fd=h][,udp=host:port][,localaddr=host:port]\n"
-    "                connect the vlan 'n' to another VLAN using an UDP tunnel\n"
+    "-netdev socket,id=str[,fd=h][,udp=host:port][,localaddr=host:port]\n"
+    "                configure a network backend to connect to another network\n"
+    "                using an UDP tunnel\n"
 #ifdef CONFIG_VDE
-    "-net vde[,vlan=n][,name=str][,sock=socketpath][,port=n][,group=groupname][,mode=octalmode]\n"
-    "                connect the vlan 'n' to port 'n' of a vde switch running\n"
-    "                on host and listening for incoming connections on 'socketpath'.\n"
+    "-netdev vde,id=str[,sock=socketpath][,port=n][,group=groupname][,mode=octalmode]\n"
+    "                configure a network backend to connect to port 'n' of a vde switch\n"
+    "                running on host and listening for incoming connections on 'socketpath'.\n"
     "                Use group 'groupname' and mode 'octalmode' to change default\n"
     "                ownership and permissions for communication port.\n"
 #endif
 #ifdef CONFIG_NETMAP
-    "-net netmap,ifname=name[,devname=nmname]\n"
+    "-netdev netmap,id=str,ifname=name[,devname=nmname]\n"
     "                attach to the existing netmap-enabled network interface 'name', or to a\n"
     "                VALE port (created on the fly) called 'name' ('nmname' is name of the \n"
     "                netmap device, defaults to '/dev/netmap')\n"
 #endif
+    "-netdev vhost-user,id=str,chardev=dev[,vhostforce=on|off]\n"
+    "                configure a vhost-user network, backed by a chardev 'dev'\n"
+    "-netdev hubport,id=str,hubid=n\n"
+    "                configure a hub port on QEMU VLAN 'n'\n", QEMU_ARCH_ALL)
+DEF("net", HAS_ARG, QEMU_OPTION_net,
+    "-net nic[,vlan=n][,macaddr=mac][,model=type][,name=str][,addr=str][,vectors=v]\n"
+    "                old way to create a new NIC and connect it to VLAN 'n'\n"
+    "                (use the '-device devtype,netdev=str' option if possible instead)\n"
     "-net dump[,vlan=n][,file=f][,len=n]\n"
     "                dump traffic on vlan 'n' to file 'f' (max n bytes per packet)\n"
     "-net none       use it alone to have zero network devices. If no -net option\n"
-    "                is provided, the default is '-net nic -net user'\n", QEMU_ARCH_ALL)
-DEF("netdev", HAS_ARG, QEMU_OPTION_netdev,
-    "-netdev ["
+    "                is provided, the default is '-net nic -net user'\n"
+    "-net ["
 #ifdef CONFIG_SLIRP
     "user|"
 #endif
@@ -1512,9 +1706,9 @@
 #ifdef CONFIG_NETMAP
     "netmap|"
 #endif
-    "vhost-user|"
-    "socket|"
-    "hubport],id=str[,option][,option][,...]\n", QEMU_ARCH_ALL)
+    "socket][,vlan=n][,option][,option][,...]\n"
+    "                old way to initialize a host network interface\n"
+    "                (use the -netdev option if possible instead)\n", QEMU_ARCH_ALL)
 STEXI
 @item -net nic[,vlan=@var{n}][,macaddr=@var{mac}][,model=@var{type}] [,name=@var{name}][,addr=@var{addr}][,vectors=@var{v}]
 @findex -net
@@ -1545,9 +1739,12 @@
 Connect user mode stack to VLAN @var{n} (@var{n} = 0 is the default).
 
 @item id=@var{id}
-@item name=@var{name}
+@itemx name=@var{name}
 Assign symbolic name for use in monitor commands.
 
+@option{ipv4} and @option{ipv6} specify that either IPv4 or IPv6 must
+be enabled.  If neither is specified both protocols are enabled.
+
 @item net=@var{addr}[/@var{mask}]
 Set IP network address the guest will see. Optionally specify the netmask,
 either in the form a.b.c.d or as number of valid top-most bits. Default is
@@ -1557,6 +1754,16 @@
 Specify the guest-visible address of the host. Default is the 2nd IP in the
 guest network, i.e. x.x.x.2.
 
+@item ipv6-net=@var{addr}[/@var{int}]
+Set IPv6 network address the guest will see (default is fec0::/64). The
+network prefix is given in the usual hexadecimal IPv6 address
+notation. The prefix size is optional, and is given as the number of
+valid top-most bits (default is 64).
+
+@item ipv6-host=@var{addr}
+Specify the guest-visible IPv6 address of the host. Default is the 2nd IPv6 in
+the guest network, i.e. xxxx::2.
+
 @item restrict=on|off
 If this option is enabled, the guest will be isolated, i.e. it will not be
 able to contact the host and no guest IP packets will be routed over the host
@@ -1574,6 +1781,11 @@
 be different from the host address. Default is the 3rd IP in the guest network,
 i.e. x.x.x.3.
 
+@item ipv6-dns=@var{addr}
+Specify the guest-visible address of the IPv6 virtual nameserver. The address
+must be different from the host address. Default is the 3rd IP in the guest
+network, i.e. xxxx::3.
+
 @item dnssearch=@var{domain}
 Provides an entry for the domain-search list sent by the built-in
 DHCP server. More than one domain suffix can be transmitted by specifying
@@ -1652,7 +1864,7 @@
 connect to the guest telnet server.
 
 @item guestfwd=[tcp]:@var{server}:@var{port}-@var{dev}
-@item guestfwd=[tcp]:@var{server}:@var{port}-@var{cmd:command}
+@itemx guestfwd=[tcp]:@var{server}:@var{port}-@var{cmd:command}
 Forward guest TCP connections to the IP address @var{server} on port @var{port}
 to the character device @var{dev} or to a program executed by @var{cmd:command}
 which gets spawned for each connection. This option can be given multiple times.
@@ -1683,7 +1895,7 @@
 as they will be removed from future versions.
 
 @item -netdev tap,id=@var{id}[,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
-@item -net tap[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
+@itemx -net tap[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
 Connect the host TAP network interface @var{name} to VLAN @var{n}.
 
 Use the network script @var{file} to configure it and the network script
@@ -1723,7 +1935,7 @@
 @end example
 
 @item -netdev bridge,id=@var{id}[,br=@var{bridge}][,helper=@var{helper}]
-@item -net bridge[,vlan=@var{n}][,name=@var{name}][,br=@var{bridge}][,helper=@var{helper}]
+@itemx -net bridge[,vlan=@var{n}][,name=@var{name}][,br=@var{bridge}][,helper=@var{helper}]
 Connect a host TAP network interface to a host bridge device.
 
 Use the network helper @var{helper} to configure the TAP interface and
@@ -1746,7 +1958,7 @@
 @end example
 
 @item -netdev socket,id=@var{id}[,fd=@var{h}][,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
-@item -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}] [,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
+@itemx -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}] [,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
 
 Connect the VLAN @var{n} to a remote VLAN in another QEMU virtual
 machine using a TCP socket connection. If @option{listen} is
@@ -1769,7 +1981,7 @@
 @end example
 
 @item -netdev socket,id=@var{id}[,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
-@item -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
+@itemx -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
 
 Create a VLAN @var{n} shared with another QEMU virtual
 machines using a UDP multicast socket, effectively making a bus for
@@ -1821,7 +2033,7 @@
 @end example
 
 @item -netdev l2tpv3,id=@var{id},src=@var{srcaddr},dst=@var{dstaddr}[,srcport=@var{srcport}][,dstport=@var{dstport}],txsession=@var{txsession}[,rxsession=@var{rxsession}][,ipv6][,udp][,cookie64][,counter][,pincounter][,txcookie=@var{txcookie}][,rxcookie=@var{rxcookie}][,offset=@var{offset}]
-@item -net l2tpv3[,vlan=@var{n}][,name=@var{name}],src=@var{srcaddr},dst=@var{dstaddr}[,srcport=@var{srcport}][,dstport=@var{dstport}],txsession=@var{txsession}[,rxsession=@var{rxsession}][,ipv6][,udp][,cookie64][,counter][,pincounter][,txcookie=@var{txcookie}][,rxcookie=@var{rxcookie}][,offset=@var{offset}]
+@itemx -net l2tpv3[,vlan=@var{n}][,name=@var{name}],src=@var{srcaddr},dst=@var{dstaddr}[,srcport=@var{srcport}][,dstport=@var{dstport}],txsession=@var{txsession}[,rxsession=@var{rxsession}][,ipv6][,udp][,cookie64][,counter][,pincounter][,txcookie=@var{txcookie}][,rxcookie=@var{rxcookie}][,offset=@var{offset}]
 Connect VLAN @var{n} to L2TPv3 pseudowire. L2TPv3 (RFC3391) is a popular
 protocol to transport Ethernet (and other Layer 2) data frames between
 two systems. It is present in routers, firewalls and the Linux kernel
@@ -1842,7 +2054,7 @@
 @item ipv6
     force v6, otherwise defaults to v4.
 @item rxcookie=@var{rxcookie}
-@item txcookie=@var{txcookie}
+@itemx txcookie=@var{txcookie}
     Cookies are a weak form of security in the l2tpv3 specification.
 Their function is mostly to prevent misconfiguration. By default they are 32
 bit.
@@ -1880,7 +2092,7 @@
 @end example
 
 @item -netdev vde,id=@var{id}[,sock=@var{socketpath}][,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
-@item -net vde[,vlan=@var{n}][,name=@var{name}][,sock=@var{socketpath}] [,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
+@itemx -net vde[,vlan=@var{n}][,name=@var{name}][,sock=@var{socketpath}] [,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
 Connect VLAN @var{n} to PORT @var{n} of a vde switch running on host and
 listening for incoming connections on @var{socketpath}. Use GROUP @var{groupname}
 and MODE @var{octalmode} to change default ownership and permissions for
@@ -1903,13 +2115,14 @@
 netdev.  @code{-net} and @code{-device} with parameter @option{vlan} create the
 required hub automatically.
 
-@item -netdev vhost-user,chardev=@var{id}[,vhostforce=on|off]
+@item -netdev vhost-user,chardev=@var{id}[,vhostforce=on|off][,queues=n]
 
 Establish a vhost-user netdev, backed by a chardev @var{id}. The chardev should
 be a unix domain socket backed one. The vhost-user uses a specifically defined
 protocol to pass vhost ioctl replacement messages to an application on the other
 end of the socket. On non-MSIX guests, the feature can be forced with
-@var{vhostforce}.
+@var{vhostforce}. Use 'queues=@var{n}' to specify the number of queues to
+be created for multiqueue vhost-user.
 
 Example:
 @example
@@ -1924,6 +2137,7 @@
 Dump network traffic on VLAN @var{n} to file @var{file} (@file{qemu-vlan0.pcap} by default).
 At most @var{len} bytes (64k by default) per packet are stored. The file format is
 libpcap, so it can be analyzed with tools such as tcpdump or Wireshark.
+Note: For devices created with '-netdev', use '-object filter-dump,...' instead.
 
 @item -net none
 Indicate that no network devices should be configured. It is used to
@@ -1944,40 +2158,43 @@
 ETEXI
 
 DEF("chardev", HAS_ARG, QEMU_OPTION_chardev,
-    "-chardev null,id=id[,mux=on|off]\n"
+    "-chardev null,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
     "-chardev socket,id=id[,host=host],port=port[,to=to][,ipv4][,ipv6][,nodelay][,reconnect=seconds]\n"
-    "         [,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off] (tcp)\n"
-    "-chardev socket,id=id,path=path[,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off] (unix)\n"
+    "         [,server][,nowait][,telnet][,reconnect=seconds][,mux=on|off]\n"
+    "         [,logfile=PATH][,logappend=on|off][,tls-creds=ID] (tcp)\n"
+    "-chardev socket,id=id,path=path[,server][,nowait][,telnet][,reconnect=seconds]\n"
+    "         [,mux=on|off][,logfile=PATH][,logappend=on|off] (unix)\n"
     "-chardev udp,id=id[,host=host],port=port[,localaddr=localaddr]\n"
     "         [,localport=localport][,ipv4][,ipv6][,mux=on|off]\n"
-    "-chardev msmouse,id=id[,mux=on|off]\n"
+    "         [,logfile=PATH][,logappend=on|off]\n"
+    "-chardev msmouse,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
     "-chardev vc,id=id[[,width=width][,height=height]][[,cols=cols][,rows=rows]]\n"
-    "         [,mux=on|off]\n"
-    "-chardev ringbuf,id=id[,size=size]\n"
-    "-chardev file,id=id,path=path[,mux=on|off]\n"
-    "-chardev pipe,id=id,path=path[,mux=on|off]\n"
+    "         [,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev ringbuf,id=id[,size=size][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev file,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev pipe,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #ifdef _WIN32
-    "-chardev console,id=id[,mux=on|off]\n"
-    "-chardev serial,id=id,path=path[,mux=on|off]\n"
+    "-chardev console,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #else
-    "-chardev pty,id=id[,mux=on|off]\n"
-    "-chardev stdio,id=id[,mux=on|off][,signal=on|off]\n"
+    "-chardev pty,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev stdio,id=id[,mux=on|off][,signal=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #ifdef CONFIG_BRLAPI
-    "-chardev braille,id=id[,mux=on|off]\n"
+    "-chardev braille,id=id[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #if defined(__linux__) || defined(__sun__) || defined(__FreeBSD__) \
         || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
-    "-chardev serial,id=id,path=path[,mux=on|off]\n"
-    "-chardev tty,id=id,path=path[,mux=on|off]\n"
+    "-chardev serial,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev tty,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #if defined(__linux__) || defined(__FreeBSD__) || defined(__DragonFly__)
-    "-chardev parallel,id=id,path=path[,mux=on|off]\n"
-    "-chardev parport,id=id,path=path[,mux=on|off]\n"
+    "-chardev parallel,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev parport,id=id,path=path[,mux=on|off][,logfile=PATH][,logappend=on|off]\n"
 #endif
 #if defined(CONFIG_SPICE)
-    "-chardev spicevmc,id=id,name=name[,debug=debug]\n"
-    "-chardev spiceport,id=id,name=name[,debug=debug]\n"
+    "-chardev spicevmc,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off]\n"
+    "-chardev spiceport,id=id,name=name[,debug=debug][,logfile=PATH][,logappend=on|off]\n"
 #endif
     , QEMU_ARCH_ALL
 )
@@ -2010,16 +2227,62 @@
 It is used to uniquely identify this device in other command line directives.
 
 A character device may be used in multiplexing mode by multiple front-ends.
-The key sequence of @key{Control-a} and @key{c} will rotate the input focus
-between attached front-ends. Specify @option{mux=on} to enable this mode.
+Specify @option{mux=on} to enable this mode.
+A multiplexer is a "1:N" device, and here the "1" end is your specified chardev
+backend, and the "N" end is the various parts of QEMU that can talk to a chardev.
+If you create a chardev with @option{id=myid} and @option{mux=on}, QEMU will
+create a multiplexer with your specified ID, and you can then configure multiple
+front ends to use that chardev ID for their input/output. Up to four different
+front ends can be connected to a single multiplexed chardev. (Without
+multiplexing enabled, a chardev can only be used by a single front end.)
+For instance you could use this to allow a single stdio chardev to be used by
+two serial ports and the QEMU monitor:
 
-Options to each backend are described below.
+@example
+-chardev stdio,mux=on,id=char0 \
+-mon chardev=char0,mode=readline,default \
+-serial chardev:char0 \
+-serial chardev:char0
+@end example
+
+You can have more than one multiplexer in a system configuration; for instance
+you could have a TCP port multiplexed between UART 0 and UART 1, and stdio
+multiplexed between the QEMU monitor and a parallel port:
+
+@example
+-chardev stdio,mux=on,id=char0 \
+-mon chardev=char0,mode=readline,default \
+-parallel chardev:char0 \
+-chardev tcp,...,mux=on,id=char1 \
+-serial chardev:char1 \
+-serial chardev:char1
+@end example
+
+When you're using a multiplexed character device, some escape sequences are
+interpreted in the input. @xref{mux_keys, Keys in the character backend
+multiplexer}.
+
+Note that some other command line options may implicitly create multiplexed
+character backends; for instance @option{-serial mon:stdio} creates a
+multiplexed stdio backend connected to the serial port and the QEMU monitor,
+and @option{-nographic} also multiplexes the console and the monitor to
+stdio.
+
+There is currently no support for multiplexing in the other direction
+(where a single QEMU front end takes input and output from multiple chardevs).
+
+Every backend supports the @option{logfile} option, which supplies the path
+to a file to record all data transmitted via the backend. The @option{logappend}
+option controls whether the log file will be truncated or appended to when
+opened.
+
+Further options to each backend are described below.
 
 @item -chardev null ,id=@var{id}
 A void device. This device will not emit any data, and will drop any data it
 receives. The null backend does not take any options.
 
-@item -chardev socket ,id=@var{id} [@var{TCP options} or @var{unix options}] [,server] [,nowait] [,telnet] [,reconnect=@var{seconds}]
+@item -chardev socket ,id=@var{id} [@var{TCP options} or @var{unix options}] [,server] [,nowait] [,telnet] [,reconnect=@var{seconds}] [,tls-creds=@var{id}]
 
 Create a two-way stream socket, which can be either a TCP or a unix socket. A
 unix socket will be created if @option{path} is specified. Behaviour is
@@ -2037,6 +2300,11 @@
 the remote end goes away.  qemu will delay this many seconds and then attempt
 to reconnect.  Zero disables reconnecting, and is the default.
 
+@option{tls-creds} requests enablement of the TLS protocol for encryption,
+and specifies the id of the TLS credentials to use for the handshake. The
+credentials must be previously created with the @option{-object tls-creds}
+argument.
+
 TCP and unix socket options are given below:
 
 @table @option
@@ -2178,7 +2446,7 @@
 @option{path} specifies the path to the tty. @option{path} is required.
 
 @item -chardev parallel ,id=@var{id} ,path=@var{path}
-@item -chardev parport ,id=@var{id} ,path=@var{path}
+@itemx -chardev parport ,id=@var{id} ,path=@var{path}
 
 @option{parallel} is only available on Linux, FreeBSD and DragonFlyBSD hosts.
 
@@ -2233,6 +2501,10 @@
 'iqn.2008-11.org.linux-kvm[:<name>]' but this can also be set from the command
 line or a configuration file.
 
+Since version Qemu 2.4 it is possible to specify a iSCSI request timeout to detect
+stalled requests and force a reestablishment of the session. The timeout
+is specified in seconds. The default is 0 which means no timeout. Libiscsi
+1.15.0 or greater is required for this feature.
 
 Example (without authentication):
 @example
@@ -2260,6 +2532,7 @@
     "-iscsi [user=user][,password=password]\n"
     "       [,header-digest=CRC32C|CR32C-NONE|NONE-CRC32C|NONE\n"
     "       [,initiator-name=initiator-iqn][,id=target-iqn]\n"
+    "       [,timeout=timeout]\n"
     "                iSCSI session parameters\n", QEMU_ARCH_ALL)
 STEXI
 
@@ -2629,6 +2902,36 @@
 @table @option
 ETEXI
 
+DEF("fw_cfg", HAS_ARG, QEMU_OPTION_fwcfg,
+    "-fw_cfg [name=]<name>,file=<file>\n"
+    "                add named fw_cfg entry with contents from file\n"
+    "-fw_cfg [name=]<name>,string=<str>\n"
+    "                add named fw_cfg entry with contents from string\n",
+    QEMU_ARCH_ALL)
+STEXI
+
+@item -fw_cfg [name=]@var{name},file=@var{file}
+@findex -fw_cfg
+Add named fw_cfg entry with contents from file @var{file}.
+
+@item -fw_cfg [name=]@var{name},string=@var{str}
+Add named fw_cfg entry with contents from string @var{str}.
+
+The terminating NUL character of the contents of @var{str} will not be
+included as part of the fw_cfg item data. To insert contents with
+embedded NUL characters, you have to use the @var{file} parameter.
+
+The fw_cfg entries are passed by QEMU through to the guest.
+
+Example:
+@example
+    -fw_cfg name=opt/com.mycompany/blob,file=./my_blob.bin
+@end example
+creates an fw_cfg entry named opt/com.mycompany/blob with contents
+from ./my_blob.bin.
+
+ETEXI
+
 DEF("serial", HAS_ARG, QEMU_OPTION_serial, \
     "-serial dev     redirect the serial port to char device 'dev'\n",
     QEMU_ARCH_ALL)
@@ -2800,6 +3103,14 @@
 @findex -qmp
 Like -monitor but opens in 'control' mode.
 ETEXI
+DEF("qmp-pretty", HAS_ARG, QEMU_OPTION_qmp_pretty, \
+    "-qmp-pretty dev like -qmp but uses pretty JSON formatting\n",
+    QEMU_ARCH_ALL)
+STEXI
+@item -qmp-pretty @var{dev}
+@findex -qmp-pretty
+Like -qmp but uses pretty JSON formatting.
+ETEXI
 
 DEF("mon", HAS_ARG, QEMU_OPTION_mon, \
     "-mon [chardev=]name[,mode=readline|control][,default]\n", QEMU_ARCH_ALL)
@@ -2903,6 +3214,24 @@
 Output log in @var{logfile} instead of to stderr
 ETEXI
 
+DEF("dfilter", HAS_ARG, QEMU_OPTION_DFILTER, \
+    "-dfilter range,..  filter debug output to range of addresses (useful for -d cpu,exec,etc..)\n",
+    QEMU_ARCH_ALL)
+STEXI
+@item -dfilter @var{range1}[,...]
+@findex -dfilter
+Filter debug output to that relevant to a range of target addresses. The filter
+spec can be either @var{start}+@var{size}, @var{start}-@var{size} or
+@var{start}..@var{end} where @var{start} @var{end} and @var{size} are the
+addresses and sizes required. For example:
+@example
+    -dfilter 0x8000..0x8fff,0xffffffc000080000+0x200,0xffffffc000060000-0x1000
+@end example
+Will dump output for any code in the 0x1000 sized block starting at 0x8000 and
+the 0x200 sized block starting at 0xffffffc000080000 and another 0x1000 sized
+block starting at 0xffffffc00005f000.
+ETEXI
+
 DEF("L", HAS_ARG, QEMU_OPTION_L, \
     "-L path         set the directory for the BIOS, VGA BIOS and keymaps\n",
     QEMU_ARCH_ALL)
@@ -2910,6 +3239,8 @@
 @item -L  @var{path}
 @findex -L
 Set the directory for the BIOS, VGA BIOS and keymaps.
+
+To list all the data directories, use @code{-L help}.
 ETEXI
 
 DEF("bios", HAS_ARG, QEMU_OPTION_bios, \
@@ -2934,10 +3265,9 @@
 STEXI
 @item -enable-hax
 @findex -enable-hax
-Enable HAX (Hardware-based Acceleration eXecution) support. This option
-is only available if HAX support is enabled when compiling. HAX is only
-applicable to MAC and Windows platform, and thus does not conflict with
-KVM.
+Enable HAX (Hardware-based Acceleration eXecution) support, used for full
+virtualization support on OS X and Windows.  This option
+is only available if HAX support is enabled when compiling.
 ETEXI
 
 DEF("xen-domid", HAS_ARG, QEMU_OPTION_xen_domid,
@@ -3050,36 +3380,48 @@
 ETEXI
 
 DEF("icount", HAS_ARG, QEMU_OPTION_icount, \
-    "-icount [shift=N|auto][,align=on|off]\n" \
+    "-icount [shift=N|auto][,align=on|off][,sleep=on|off,rr=record|replay,rrfile=<filename>]\n" \
     "                enable virtual instruction counter with 2^N clock ticks per\n" \
-    "                instruction and enable aligning the host and virtual clocks\n", QEMU_ARCH_ALL)
+    "                instruction, enable aligning the host and virtual clocks\n" \
+    "                or disable real time cpu sleeping\n", QEMU_ARCH_ALL)
 STEXI
-@item -icount [shift=@var{N}|auto]
+@item -icount [shift=@var{N}|auto][,rr=record|replay,rrfile=@var{filename}]
 @findex -icount
 Enable virtual instruction counter.  The virtual cpu will execute one
 instruction every 2^@var{N} ns of virtual time.  If @code{auto} is specified
 then the virtual cpu speed will be automatically adjusted to keep virtual
 time within a few seconds of real time.
 
+When the virtual cpu is sleeping, the virtual time will advance at default
+speed unless @option{sleep=on|off} is specified.
+With @option{sleep=on|off}, the virtual time will jump to the next timer deadline
+instantly whenever the virtual cpu goes to sleep mode and will not advance
+if no timer is enabled. This behavior give deterministic execution times from
+the guest point of view.
+
 Note that while this option can give deterministic behavior, it does not
 provide cycle accurate emulation.  Modern CPUs contain superscalar out of
 order cores with complex cache hierarchies.  The number of instructions
 executed often has little or no correlation with actual performance.
 
-@option{align=on} will activate the delay algorithm which will try to
+@option{align=on} will activate the delay algorithm which will try
 to synchronise the host clock and the virtual clock. The goal is to
 have a guest running at the real frequency imposed by the shift option.
 Whenever the guest clock is behind the host clock and if
-@option{align=on} is specified then we print a messsage to the user
+@option{align=on} is specified then we print a message to the user
 to inform about the delay.
 Currently this option does not work when @option{shift} is @code{auto}.
 Note: The sync algorithm will work for those shift values for which
 the guest clock runs ahead of the host clock. Typically this happens
 when the shift value is high (how high depends on the host machine).
+
+When @option{rr} option is specified deterministic record/replay is enabled.
+Replay log is written into @var{filename} file in record mode and
+read from this file in replay mode.
 ETEXI
 
 DEF("watchdog", HAS_ARG, QEMU_OPTION_watchdog, \
-    "-watchdog i6300esb|ib700\n" \
+    "-watchdog model\n" \
     "                enable virtual hardware watchdog [default=none]\n",
     QEMU_ARCH_ALL)
 STEXI
@@ -3087,16 +3429,24 @@
 @findex -watchdog
 Create a virtual hardware watchdog device.  Once enabled (by a guest
 action), the watchdog must be periodically polled by an agent inside
-the guest or else the guest will be restarted.
+the guest or else the guest will be restarted. Choose a model for
+which your guest has drivers.
 
-The @var{model} is the model of hardware watchdog to emulate.  Choices
-for model are: @code{ib700} (iBASE 700) which is a very simple ISA
-watchdog with a single timer, or @code{i6300esb} (Intel 6300ESB I/O
-controller hub) which is a much more featureful PCI-based dual-timer
-watchdog.  Choose a model for which your guest has drivers.
-
-Use @code{-watchdog help} to list available hardware models.  Only one
+The @var{model} is the model of hardware watchdog to emulate. Use
+@code{-watchdog help} to list available hardware models. Only one
 watchdog can be enabled for a guest.
+
+The following models may be available:
+@table @option
+@item ib700
+iBASE 700 is a very simple ISA watchdog with a single timer.
+@item i6300esb
+Intel 6300ESB I/O controller hub is a much more featureful PCI-based
+dual-timer watchdog.
+@item diag288
+A virtual watchdog for s390x backed by the diagnose 288 hypercall
+(currently KVM only).
+@end table
 ETEXI
 
 DEF("watchdog-action", HAS_ARG, QEMU_OPTION_watchdog_action, \
@@ -3127,7 +3477,7 @@
 
 @table @code
 @item -watchdog i6300esb -watchdog-action pause
-@item -watchdog ib700
+@itemx -watchdog ib700
 @end table
 ETEXI
 
@@ -3147,7 +3497,7 @@
 character to Control-t.
 @table @code
 @item -echr 0x14
-@item -echr 20
+@itemx -echr 20
 @end table
 ETEXI
 
@@ -3181,12 +3531,37 @@
 ETEXI
 
 DEF("incoming", HAS_ARG, QEMU_OPTION_incoming, \
-    "-incoming p     prepare for incoming migration, listen on port p\n",
+    "-incoming tcp:[host]:port[,to=maxport][,ipv4][,ipv6]\n" \
+    "-incoming rdma:host:port[,ipv4][,ipv6]\n" \
+    "-incoming unix:socketpath\n" \
+    "                prepare for incoming migration, listen on\n" \
+    "                specified protocol and socket address\n" \
+    "-incoming fd:fd\n" \
+    "-incoming exec:cmdline\n" \
+    "                accept incoming migration on given file descriptor\n" \
+    "                or from given external command\n" \
+    "-incoming defer\n" \
+    "                wait for the URI to be specified via migrate_incoming\n",
     QEMU_ARCH_ALL)
 STEXI
-@item -incoming @var{port}
+@item -incoming tcp:[@var{host}]:@var{port}[,to=@var{maxport}][,ipv4][,ipv6]
+@itemx -incoming rdma:@var{host}:@var{port}[,ipv4][,ipv6]
 @findex -incoming
-Prepare for incoming migration, listen on @var{port}.
+Prepare for incoming migration, listen on a given tcp port.
+
+@item -incoming unix:@var{socketpath}
+Prepare for incoming migration, listen on a given unix socket.
+
+@item -incoming fd:@var{fd}
+Accept incoming migration from a given filedescriptor.
+
+@item -incoming exec:@var{cmdline}
+Accept incoming migration as an output from specified external command.
+
+@item -incoming defer
+Wait for the URI to be specified via migrate_incoming.  The monitor can
+be used to change settings (such as migration parameters) prior to issuing
+the migrate_incoming to allow the migration to begin.
 ETEXI
 
 DEF("nodefaults", 0, QEMU_OPTION_nodefaults, \
@@ -3235,11 +3610,34 @@
 ETEXI
 DEF("semihosting", 0, QEMU_OPTION_semihosting,
     "-semihosting    semihosting mode\n",
-    QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_LM32)
+    QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_LM32 |
+    QEMU_ARCH_MIPS)
 STEXI
 @item -semihosting
 @findex -semihosting
-Semihosting mode (ARM, M68K, Xtensa only).
+Enable semihosting mode (ARM, M68K, Xtensa, MIPS only).
+ETEXI
+DEF("semihosting-config", HAS_ARG, QEMU_OPTION_semihosting_config,
+    "-semihosting-config [enable=on|off][,target=native|gdb|auto][,arg=str[,...]]\n" \
+    "                semihosting configuration\n",
+QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_LM32 |
+QEMU_ARCH_MIPS)
+STEXI
+@item -semihosting-config [enable=on|off][,target=native|gdb|auto][,arg=str[,...]]
+@findex -semihosting-config
+Enable and configure semihosting (ARM, M68K, Xtensa, MIPS only).
+@table @option
+@item target=@code{native|gdb|auto}
+Defines where the semihosting calls will be addressed, to QEMU (@code{native})
+or to GDB (@code{gdb}). The default is @code{auto}, which means @code{gdb}
+during debug sessions and @code{native} otherwise.
+@item arg=@var{str1},arg=@var{str2},...
+Allows the user to pass input arguments, and can be used multiple times to build
+up a list. The old-style @code{-kernel}/@code{-append} method of passing a
+command line is still supported for backward compatibility. If both the
+@code{--semihosting-config arg} and the @code{-kernel}/@code{-append} are
+specified, the former is passed to semihosting as it always takes precedence.
+@end table
 ETEXI
 DEF("old-param", 0, QEMU_OPTION_old_param,
     "-old-param      old param mode\n", QEMU_ARCH_ARM)
@@ -3300,30 +3698,15 @@
 files from @var{datadir}.
 ETEXI
 DEF("trace", HAS_ARG, QEMU_OPTION_trace,
-    "-trace [events=<file>][,file=<file>]\n"
+    "-trace [[enable=]<pattern>][,events=<file>][,file=<file>]\n"
     "                specify tracing options\n",
     QEMU_ARCH_ALL)
 STEXI
 HXCOMM This line is not accurate, as some sub-options are backend-specific but
 HXCOMM HX does not support conditional compilation of text.
-@item -trace [events=@var{file}][,file=@var{file}]
+@item -trace [[enable=]@var{pattern}][,events=@var{file}][,file=@var{file}]
 @findex -trace
-
-Specify tracing options.
-
-@table @option
-@item events=@var{file}
-Immediately enable events listed in @var{file}.
-The file must contain one event name (as listed in the @var{trace-events} file)
-per line.
-This option is only available if QEMU has been compiled with
-either @var{simple} or @var{stderr} tracing backend.
-@item file=@var{file}
-Log output traces to @var{file}.
-
-This option is only available if QEMU has been compiled with
-the @var{simple} tracing backend.
-@end table
+@include qemu-option-trace.texi
 ETEXI
 
 HXCOMM Internal use
@@ -3357,22 +3740,6 @@
 HXCOMM Deprecated (ignored)
 DEF("tdf", 0, QEMU_OPTION_tdf,"", QEMU_ARCH_ALL)
 
-DEF("object", HAS_ARG, QEMU_OPTION_object,
-    "-object TYPENAME[,PROP1=VALUE1,...]\n"
-    "                create an new object of type TYPENAME setting properties\n"
-    "                in the order they are specified.  Note that the 'id'\n"
-    "                property must be set.  These objects are placed in the\n"
-    "                '/objects' path.\n",
-    QEMU_ARCH_ALL)
-STEXI
-@item -object @var{typename}[,@var{prop1}=@var{value1},...]
-@findex -object
-Create an new object of type @var{typename} setting properties
-in the order they are specified.  Note that the 'id'
-property must be set.  These objects are placed in the
-'/objects' path.
-ETEXI
-
 DEF("msg", HAS_ARG, QEMU_OPTION_msg,
     "-msg timestamp[=on|off]\n"
     "                change the format of messages\n"
@@ -3398,6 +3765,229 @@
 in @var{file}
 ETEXI
 
+DEFHEADING(Generic object creation)
+
+DEF("object", HAS_ARG, QEMU_OPTION_object,
+    "-object TYPENAME[,PROP1=VALUE1,...]\n"
+    "                create a new object of type TYPENAME setting properties\n"
+    "                in the order they are specified.  Note that the 'id'\n"
+    "                property must be set.  These objects are placed in the\n"
+    "                '/objects' path.\n",
+    QEMU_ARCH_ALL)
+STEXI
+@item -object @var{typename}[,@var{prop1}=@var{value1},...]
+@findex -object
+Create a new object of type @var{typename} setting properties
+in the order they are specified.  Note that the 'id'
+property must be set.  These objects are placed in the
+'/objects' path.
+
+@table @option
+
+@item -object memory-backend-file,id=@var{id},size=@var{size},mem-path=@var{dir},share=@var{on|off}
+
+Creates a memory file backend object, which can be used to back
+the guest RAM with huge pages. The @option{id} parameter is a
+unique ID that will be used to reference this memory region
+when configuring the @option{-numa} argument. The @option{size}
+option provides the size of the memory region, and accepts
+common suffixes, eg @option{500M}. The @option{mem-path} provides
+the path to either a shared memory or huge page filesystem mount.
+The @option{share} boolean option determines whether the memory
+region is marked as private to QEMU, or shared. The latter allows
+a co-operating external process to access the QEMU memory region.
+
+@item -object rng-random,id=@var{id},filename=@var{/dev/random}
+
+Creates a random number generator backend which obtains entropy from
+a device on the host. The @option{id} parameter is a unique ID that
+will be used to reference this entropy backend from the @option{virtio-rng}
+device. The @option{filename} parameter specifies which file to obtain
+entropy from and if omitted defaults to @option{/dev/random}.
+
+@item -object rng-egd,id=@var{id},chardev=@var{chardevid}
+
+Creates a random number generator backend which obtains entropy from
+an external daemon running on the host. The @option{id} parameter is
+a unique ID that will be used to reference this entropy backend from
+the @option{virtio-rng} device. The @option{chardev} parameter is
+the unique ID of a character device backend that provides the connection
+to the RNG daemon.
+
+@item -object tls-creds-anon,id=@var{id},endpoint=@var{endpoint},dir=@var{/path/to/cred/dir},verify-peer=@var{on|off}
+
+Creates a TLS anonymous credentials object, which can be used to provide
+TLS support on network backends. The @option{id} parameter is a unique
+ID which network backends will use to access the credentials. The
+@option{endpoint} is either @option{server} or @option{client} depending
+on whether the QEMU network backend that uses the credentials will be
+acting as a client or as a server. If @option{verify-peer} is enabled
+(the default) then once the handshake is completed, the peer credentials
+will be verified, though this is a no-op for anonymous credentials.
+
+The @var{dir} parameter tells QEMU where to find the credential
+files. For server endpoints, this directory may contain a file
+@var{dh-params.pem} providing diffie-hellman parameters to use
+for the TLS server. If the file is missing, QEMU will generate
+a set of DH parameters at startup. This is a computationally
+expensive operation that consumes random pool entropy, so it is
+recommended that a persistent set of parameters be generated
+upfront and saved.
+
+@item -object tls-creds-x509,id=@var{id},endpoint=@var{endpoint},dir=@var{/path/to/cred/dir},verify-peer=@var{on|off},passwordid=@var{id}
+
+Creates a TLS anonymous credentials object, which can be used to provide
+TLS support on network backends. The @option{id} parameter is a unique
+ID which network backends will use to access the credentials. The
+@option{endpoint} is either @option{server} or @option{client} depending
+on whether the QEMU network backend that uses the credentials will be
+acting as a client or as a server. If @option{verify-peer} is enabled
+(the default) then once the handshake is completed, the peer credentials
+will be verified. With x509 certificates, this implies that the clients
+must be provided with valid client certificates too.
+
+The @var{dir} parameter tells QEMU where to find the credential
+files. For server endpoints, this directory may contain a file
+@var{dh-params.pem} providing diffie-hellman parameters to use
+for the TLS server. If the file is missing, QEMU will generate
+a set of DH parameters at startup. This is a computationally
+expensive operation that consumes random pool entropy, so it is
+recommended that a persistent set of parameters be generated
+upfront and saved.
+
+For x509 certificate credentials the directory will contain further files
+providing the x509 certificates. The certificates must be stored
+in PEM format, in filenames @var{ca-cert.pem}, @var{ca-crl.pem} (optional),
+@var{server-cert.pem} (only servers), @var{server-key.pem} (only servers),
+@var{client-cert.pem} (only clients), and @var{client-key.pem} (only clients).
+
+For the @var{server-key.pem} and @var{client-key.pem} files which
+contain sensitive private keys, it is possible to use an encrypted
+version by providing the @var{passwordid} parameter. This provides
+the ID of a previously created @code{secret} object containing the
+password for decryption.
+
+@item -object filter-buffer,id=@var{id},netdev=@var{netdevid},interval=@var{t}[,queue=@var{all|rx|tx}][,status=@var{on|off}]
+
+Interval @var{t} can't be 0, this filter batches the packet delivery: all
+packets arriving in a given interval on netdev @var{netdevid} are delayed
+until the end of the interval. Interval is in microseconds.
+@option{status} is optional that indicate whether the netfilter is
+on (enabled) or off (disabled), the default status for netfilter will be 'on'.
+
+queue @var{all|rx|tx} is an option that can be applied to any netfilter.
+
+@option{all}: the filter is attached both to the receive and the transmit
+              queue of the netdev (default).
+
+@option{rx}: the filter is attached to the receive queue of the netdev,
+             where it will receive packets sent to the netdev.
+
+@option{tx}: the filter is attached to the transmit queue of the netdev,
+             where it will receive packets sent by the netdev.
+
+@item -object filter-mirror,id=@var{id},netdev=@var{netdevid},outdev=@var{chardevid}[,queue=@var{all|rx|tx}]
+
+filter-mirror on netdev @var{netdevid},mirror net packet to chardev
+@var{chardevid}
+
+@item -object filter-redirector,id=@var{id},netdev=@var{netdevid},indev=@var{chardevid},
+outdev=@var{chardevid}[,queue=@var{all|rx|tx}]
+
+filter-redirector on netdev @var{netdevid},redirect filter's net packet to chardev
+@var{chardevid},and redirect indev's packet to filter.
+Create a filter-redirector we need to differ outdev id from indev id, id can not
+be the same. we can just use indev or outdev, but at least one of indev or outdev
+need to be specified.
+
+@item -object filter-dump,id=@var{id},netdev=@var{dev},file=@var{filename}][,maxlen=@var{len}]
+
+Dump the network traffic on netdev @var{dev} to the file specified by
+@var{filename}. At most @var{len} bytes (64k by default) per packet are stored.
+The file format is libpcap, so it can be analyzed with tools such as tcpdump
+or Wireshark.
+
+@item -object secret,id=@var{id},data=@var{string},format=@var{raw|base64}[,keyid=@var{secretid},iv=@var{string}]
+@item -object secret,id=@var{id},file=@var{filename},format=@var{raw|base64}[,keyid=@var{secretid},iv=@var{string}]
+
+Defines a secret to store a password, encryption key, or some other sensitive
+data. The sensitive data can either be passed directly via the @var{data}
+parameter, or indirectly via the @var{file} parameter. Using the @var{data}
+parameter is insecure unless the sensitive data is encrypted.
+
+The sensitive data can be provided in raw format (the default), or base64.
+When encoded as JSON, the raw format only supports valid UTF-8 characters,
+so base64 is recommended for sending binary data. QEMU will convert from
+which ever format is provided to the format it needs internally. eg, an
+RBD password can be provided in raw format, even though it will be base64
+encoded when passed onto the RBD sever.
+
+For added protection, it is possible to encrypt the data associated with
+a secret using the AES-256-CBC cipher. Use of encryption is indicated
+by providing the @var{keyid} and @var{iv} parameters. The @var{keyid}
+parameter provides the ID of a previously defined secret that contains
+the AES-256 decryption key. This key should be 32-bytes long and be
+base64 encoded. The @var{iv} parameter provides the random initialization
+vector used for encryption of this particular secret and should be a
+base64 encrypted string of the 16-byte IV.
+
+The simplest (insecure) usage is to provide the secret inline
+
+@example
+
+ # $QEMU -object secret,id=sec0,data=letmein,format=raw
+
+@end example
+
+The simplest secure usage is to provide the secret via a file
+
+ # echo -n "letmein" > mypasswd.txt
+ # $QEMU -object secret,id=sec0,file=mypasswd.txt,format=raw
+
+For greater security, AES-256-CBC should be used. To illustrate usage,
+consider the openssl command line tool which can encrypt the data. Note
+that when encrypting, the plaintext must be padded to the cipher block
+size (32 bytes) using the standard PKCS#5/6 compatible padding algorithm.
+
+First a master key needs to be created in base64 encoding:
+
+@example
+ # openssl rand -base64 32 > key.b64
+ # KEY=$(base64 -d key.b64 | hexdump  -v -e '/1 "%02X"')
+@end example
+
+Each secret to be encrypted needs to have a random initialization vector
+generated. These do not need to be kept secret
+
+@example
+ # openssl rand -base64 16 > iv.b64
+ # IV=$(base64 -d iv.b64 | hexdump  -v -e '/1 "%02X"')
+@end example
+
+The secret to be defined can now be encrypted, in this case we're
+telling openssl to base64 encode the result, but it could be left
+as raw bytes if desired.
+
+@example
+ # SECRET=$(echo -n "letmein" |
+            openssl enc -aes-256-cbc -a -K $KEY -iv $IV)
+@end example
+
+When launching QEMU, create a master secret pointing to @code{key.b64}
+and specify that to be used to decrypt the user password. Pass the
+contents of @code{iv.b64} to the second secret
+
+@example
+ # $QEMU \
+     -object secret,id=secmaster0,format=base64,file=key.b64 \
+     -object secret,id=sec0,keyid=secmaster0,format=base64,\
+         data=$SECRET,iv=$(<iv.b64)
+@end example
+
+@end table
+
+ETEXI
+
 #ifdef CONFIG_ANDROID
 
 DEF("boot-property", HAS_ARG, QEMU_OPTION_boot_property, \
@@ -3476,7 +4066,7 @@
 Set timezone instead of using host's default
 ETEXI
 
-#endif
+#endif  // CONFIG_ANDROID
 
 HXCOMM This is the last statement. Insert new options before this line!
 STEXI
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index af6a375..cb569dc 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -12,10 +12,20 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <seccomp.h>
 #include "sysemu/seccomp.h"
 
+/* For some architectures (notably ARM) cacheflush is not supported until
+ * libseccomp 2.2.3, but configure enforces that we are using a more recent
+ * version on those hosts, so it is OK for this check to be less strict.
+ */
+#if SCMP_VER_MAJOR >= 3
+  #define HAVE_CACHEFLUSH
+#elif SCMP_VER_MAJOR == 2 && SCMP_VER_MINOR >= 2
+  #define HAVE_CACHEFLUSH
+#endif
+
 struct QemuSeccompSyscall {
     int32_t num;
     uint8_t priority;
@@ -229,13 +239,20 @@
     { SCMP_SYS(shmdt), 240 },
     { SCMP_SYS(timerfd_create), 240 },
     { SCMP_SYS(shmctl), 240 },
+    { SCMP_SYS(mlockall), 240 },
     { SCMP_SYS(mlock), 240 },
     { SCMP_SYS(munlock), 240 },
     { SCMP_SYS(semctl), 240 },
     { SCMP_SYS(fallocate), 240 },
     { SCMP_SYS(fadvise64), 240 },
     { SCMP_SYS(inotify_init1), 240 },
-    { SCMP_SYS(inotify_add_watch), 240 }
+    { SCMP_SYS(inotify_add_watch), 240 },
+    { SCMP_SYS(mbind), 240 },
+    { SCMP_SYS(memfd_create), 240 },
+#ifdef HAVE_CACHEFLUSH
+    { SCMP_SYS(cacheflush), 240 },
+#endif
+    { SCMP_SYS(sysinfo), 240 },
 };
 
 int seccomp_start(void)
diff --git a/qemu-tech.texi b/qemu-tech.texi
index 8aefa74..bdb2285 100644
--- a/qemu-tech.texi
+++ b/qemu-tech.texi
@@ -320,7 +320,7 @@
 @node QEMU compared to other emulators
 @section QEMU compared to other emulators
 
-Like bochs [3], QEMU emulates an x86 CPU. But QEMU is much faster than
+Like bochs [1], QEMU emulates an x86 CPU. But QEMU is much faster than
 bochs as it uses dynamic compilation. Bochs is closely tied to x86 PC
 emulation while QEMU can emulate several processors.
 
@@ -333,25 +333,25 @@
 tied to an x86 host and target and has no support for precise exceptions
 and system emulation.
 
-EM86 [4] is the closest project to user space QEMU (and QEMU still uses
+EM86 [3] is the closest project to user space QEMU (and QEMU still uses
 some of its code, in particular the ELF file loader). EM86 was limited
 to an alpha host and used a proprietary and slow interpreter (the
-interpreter part of the FX!32 Digital Win32 code translator [5]).
+interpreter part of the FX!32 Digital Win32 code translator [4]).
 
-TWIN [6] is a Windows API emulator like Wine. It is less accurate than
-Wine but includes a protected mode x86 interpreter to launch x86 Windows
-executables. Such an approach has greater potential because most of the
-Windows API is executed natively but it is far more difficult to develop
-because all the data structures and function parameters exchanged
+TWIN from Willows Software was a Windows API emulator like Wine. It is less
+accurate than Wine but includes a protected mode x86 interpreter to launch
+x86 Windows executables. Such an approach has greater potential because most
+of the Windows API is executed natively but it is far more difficult to
+develop because all the data structures and function parameters exchanged
 between the API and the x86 code must be converted.
 
-User mode Linux [7] was the only solution before QEMU to launch a
+User mode Linux [5] was the only solution before QEMU to launch a
 Linux kernel as a process while not needing any host kernel
 patches. However, user mode Linux requires heavy kernel patches while
 QEMU accepts unpatched Linux kernels. The price to pay is that QEMU is
 slower.
 
-The Plex86 [8] PC virtualizer is done in the same spirit as the now
+The Plex86 [6] PC virtualizer is done in the same spirit as the now
 obsolete qemu-fast system emulator. It requires a patched Linux kernel
 to work (you cannot launch the same kernel on your PC), but the
 patches are really small. As it is a PC virtualizer (no emulation is
@@ -359,13 +359,13 @@
 being faster than QEMU. The downside is that a complicated (and
 potentially unsafe) host kernel patch is needed.
 
-The commercial PC Virtualizers (VMWare [9], VirtualPC [10], TwoOStwo
-[11]) are faster than QEMU, but they all need specific, proprietary
+The commercial PC Virtualizers (VMWare [7], VirtualPC [8]) are faster
+than QEMU (without virtualization), but they all need specific, proprietary
 and potentially unsafe host drivers. Moreover, they are unable to
 provide cycle exact simulation as an emulator can.
 
-VirtualBox [12], Xen [13] and KVM [14] are based on QEMU. QEMU-SystemC
-[15] uses QEMU to simulate a system where some hardware devices are
+VirtualBox [9], Xen [10] and KVM [11] are based on QEMU. QEMU-SystemC
+[12] uses QEMU to simulate a system where some hardware devices are
 developed in SystemC.
 
 @node Portable dynamic translation
@@ -385,7 +385,7 @@
 performed at this stage, including liveness analysis and trivial
 constant expression evaluation. TCG ops are then implemented in the
 host CPU back end, also known as TCG target (see
-@code{tcg/i386/tcg-target.c}). For more information, please take a
+@code{tcg/i386/tcg-target.inc.c}). For more information, please take a
 look at @code{tcg/README}.
 
 @node Condition code optimisations
@@ -440,7 +440,7 @@
 @section Direct block chaining
 
 After each translated basic block is executed, QEMU uses the simulated
-Program Counter (PC) and other cpu state informations (such as the CS
+Program Counter (PC) and other cpu state information (such as the CS
 segment base value) to find the next basic block.
 
 In order to accelerate the most common cases where the new simulated PC
@@ -608,64 +608,51 @@
 @table @asis
 
 @item [1]
-@url{http://citeseer.nj.nec.com/piumarta98optimizing.html}, Optimizing
-direct threaded code by selective inlining (1998) by Ian Piumarta, Fabio
-Riccardi.
-
-@item [2]
-@url{http://developer.kde.org/~sewardj/}, Valgrind, an open-source
-memory debugger for x86-GNU/Linux, by Julian Seward.
-
-@item [3]
 @url{http://bochs.sourceforge.net/}, the Bochs IA-32 Emulator Project,
 by Kevin Lawton et al.
 
-@item [4]
-@url{http://www.cs.rose-hulman.edu/~donaldlf/em86/index.html}, the EM86
-x86 emulator on Alpha-Linux.
+@item [2]
+@url{http://www.valgrind.org/}, Valgrind, an open-source memory debugger
+for GNU/Linux.
 
-@item [5]
+@item [3]
+@url{http://ftp.dreamtime.org/pub/linux/Linux-Alpha/em86/v0.2/docs/em86.html},
+the EM86 x86 emulator on Alpha-Linux.
+
+@item [4]
 @url{http://www.usenix.org/publications/library/proceedings/usenix-nt97/@/full_papers/chernoff/chernoff.pdf},
 DIGITAL FX!32: Running 32-Bit x86 Applications on Alpha NT, by Anton
 Chernoff and Ray Hookway.
 
-@item [6]
-@url{http://www.willows.com/}, Windows API library emulation from
-Willows Software.
-
-@item [7]
+@item [5]
 @url{http://user-mode-linux.sourceforge.net/},
 The User-mode Linux Kernel.
 
-@item [8]
+@item [6]
 @url{http://www.plex86.org/},
 The new Plex86 project.
 
-@item [9]
+@item [7]
 @url{http://www.vmware.com/},
 The VMWare PC virtualizer.
 
-@item [10]
-@url{http://www.microsoft.com/windowsxp/virtualpc/},
+@item [8]
+@url{https://www.microsoft.com/download/details.aspx?id=3702},
 The VirtualPC PC virtualizer.
 
-@item [11]
-@url{http://www.twoostwo.org/},
-The TwoOStwo PC virtualizer.
-
-@item [12]
+@item [9]
 @url{http://virtualbox.org/},
 The VirtualBox PC virtualizer.
 
-@item [13]
+@item [10]
 @url{http://www.xen.org/},
 The Xen hypervisor.
 
-@item [14]
-@url{http://kvm.qumranet.com/kvmwiki/Front_Page},
+@item [11]
+@url{http://www.linux-kvm.org/},
 Kernel Based Virtual Machine (KVM).
 
-@item [15]
+@item [12]
 @url{http://www.greensocs.com/projects/QEMUSystemC},
 QEMU-SystemC, a hardware co-simulator.
 
diff --git a/qemu-timer.c b/qemu-timer.c
index c77de64..9299cdc 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -22,13 +22,12 @@
  * THE SOFTWARE.
  */
 
-#include "sysemu/sysemu.h"
-#include "monitor/monitor.h"
-#include "ui/console.h"
-
-#include "hw/hw.h"
-
+#include "qemu/osdep.h"
+#include "qemu/main-loop.h"
 #include "qemu/timer.h"
+#include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
+
 #ifdef CONFIG_POSIX
 #include <pthread.h>
 #endif
@@ -103,7 +102,7 @@
     QEMUClock *clock = qemu_clock_ptr(type);
 
     timer_list = g_malloc0(sizeof(QEMUTimerList));
-    qemu_event_init(&timer_list->timers_done_ev, false);
+    qemu_event_init(&timer_list->timers_done_ev, true);
     timer_list->clock = clock;
     timer_list->notify_cb = cb;
     timer_list->notify_opaque = opaque;
@@ -130,7 +129,7 @@
     assert(main_loop_tlg.tl[type] == NULL);
 
     clock->type = type;
-    clock->enabled = true;
+    clock->enabled = (type == QEMU_CLOCK_VIRTUAL ? false : true);
     clock->last = INT64_MIN;
     QLIST_INIT(&clock->timerlists);
     notifier_list_init(&clock->reset_notifiers);
@@ -293,7 +292,7 @@
     /* Always round up, because it's better to wait too long than to wait too
      * little and effectively busy-wait
      */
-    ms = (ns + SCALE_MS - 1) / SCALE_MS;
+    ms = DIV_ROUND_UP(ns, SCALE_MS);
 
     /* To avoid overflow problems, limit this to 2^31, i.e. approx 25 days */
     if (ms > (int64_t) INT32_MAX) {
@@ -331,9 +330,9 @@
 }
 
 
-void timer_init(QEMUTimer *ts,
-                QEMUTimerList *timer_list, int scale,
-                QEMUTimerCB *cb, void *opaque)
+void timer_init_tl(QEMUTimer *ts,
+                   QEMUTimerList *timer_list, int scale,
+                   QEMUTimerCB *cb, void *opaque)
 {
     ts->timer_list = timer_list;
     ts->cb = cb;
@@ -342,6 +341,12 @@
     ts->expire_time = -1;
 }
 
+void timer_deinit(QEMUTimer *ts)
+{
+    assert(ts->expire_time == -1);
+    ts->timer_list = NULL;
+}
+
 void timer_free(QEMUTimer *ts)
 {
     g_free(ts);
@@ -389,7 +394,9 @@
 static void timerlist_rearm(QEMUTimerList *timer_list)
 {
     /* Interrupt execution to force deadline recalculation.  */
-    qemu_clock_warp(timer_list->clock->type);
+    if (timer_list->clock->type == QEMU_CLOCK_VIRTUAL) {
+        qemu_start_warp_timer();
+    }
     timerlist_notify(timer_list);
 }
 
@@ -398,9 +405,11 @@
 {
     QEMUTimerList *timer_list = ts->timer_list;
 
-    qemu_mutex_lock(&timer_list->active_timers_lock);
-    timer_del_locked(timer_list, ts);
-    qemu_mutex_unlock(&timer_list->active_timers_lock);
+    if (timer_list) {
+        qemu_mutex_lock(&timer_list->active_timers_lock);
+        timer_del_locked(timer_list, ts);
+        qemu_mutex_unlock(&timer_list->active_timers_lock);
+    }
 }
 
 /* modify the current timer so that it will be fired when current_time
@@ -473,10 +482,31 @@
     void *opaque;
 
     qemu_event_reset(&timer_list->timers_done_ev);
-    if (!timer_list->clock->enabled) {
+    if (!timer_list->clock->enabled || !timer_list->active_timers) {
         goto out;
     }
 
+    switch (timer_list->clock->type) {
+    case QEMU_CLOCK_REALTIME:
+        break;
+    default:
+    case QEMU_CLOCK_VIRTUAL:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_HOST:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_HOST)) {
+            goto out;
+        }
+        break;
+    case QEMU_CLOCK_VIRTUAL_RT:
+        if (!replay_checkpoint(CHECKPOINT_CLOCK_VIRTUAL_RT)) {
+            goto out;
+        }
+        break;
+    }
+
     current_time = qemu_clock_get_ns(timer_list->clock->type);
     for(;;) {
         qemu_mutex_lock(&timer_list->active_timers_lock);
@@ -540,11 +570,17 @@
 {
     int64_t deadline = -1;
     QEMUClockType type;
+    bool play = replay_mode == REPLAY_MODE_PLAY;
     for (type = 0; type < QEMU_CLOCK_MAX; type++) {
-        if (qemu_clock_use_for_deadline(tlg->tl[type]->clock->type)) {
-            deadline = qemu_soonest_timeout(deadline,
-                                            timerlist_deadline_ns(
-                                                tlg->tl[type]));
+        if (qemu_clock_use_for_deadline(type)) {
+            if (!play || type == QEMU_CLOCK_REALTIME) {
+                deadline = qemu_soonest_timeout(deadline,
+                                                timerlist_deadline_ns(tlg->tl[type]));
+            } else {
+                /* Read clock from the replay file and
+                   do not calculate the deadline, based on virtual clock. */
+                qemu_clock_get_ns(type);
+            }
         }
     }
     return deadline;
@@ -566,13 +602,15 @@
             return cpu_get_clock();
         }
     case QEMU_CLOCK_HOST:
-        now = get_clock_realtime();
+        now = REPLAY_CLOCK(REPLAY_CLOCK_HOST, get_clock_realtime());
         last = clock->last;
         clock->last = now;
-        if (now < last) {
+        if (now < last || now > (last + get_max_clock_jump())) {
             notifier_list_notify(&clock->reset_notifiers, &now);
         }
         return now;
+    case QEMU_CLOCK_VIRTUAL_RT:
+        return REPLAY_CLOCK(REPLAY_CLOCK_VIRTUAL_RT, cpu_get_clock());
     }
 }
 
diff --git a/qemu.nsi b/qemu.nsi
index cc5fafd..a20f6ef 100644
--- a/qemu.nsi
+++ b/qemu.nsi
@@ -139,6 +139,9 @@
 
     ; Write the uninstall keys for Windows
     WriteRegStr HKLM "${UNINST_KEY}" "DisplayName" "QEMU"
+!ifdef DISPLAYVERSION
+    WriteRegStr HKLM "${UNINST_KEY}" "DisplayVersion" "${DISPLAYVERSION}"
+!endif
     WriteRegStr HKLM "${UNINST_KEY}" "UninstallString" '"${UNINST_EXE}"'
     WriteRegDWORD HKLM "${UNINST_KEY}" "NoModify" 1
     WriteRegDWORD HKLM "${UNINST_KEY}" "NoRepair" 1
diff --git a/qga/channel-posix.c b/qga/channel-posix.c
index 8aad4fe..bb65d8b 100644
--- a/qga/channel-posix.c
+++ b/qga/channel-posix.c
@@ -1,11 +1,6 @@
-#include <glib.h>
-#include <termios.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <string.h>
 #include "qemu/osdep.h"
+#include <termios.h>
+#include "qapi/error.h"
 #include "qemu/sockets.h"
 #include "qga/channel.h"
 
@@ -217,25 +212,24 @@
     GIOStatus status = G_IO_STATUS_NORMAL;
 
     while (size) {
+        g_debug("sending data, count: %d", (int)size);
         status = g_io_channel_write_chars(c->client_channel, buf, size,
                                           &written, &err);
-        g_debug("sending data, count: %d", (int)size);
-        if (err != NULL) {
+        if (status == G_IO_STATUS_NORMAL) {
+            size -= written;
+            buf += written;
+        } else if (status != G_IO_STATUS_AGAIN) {
             g_warning("error writing to channel: %s", err->message);
-            return G_IO_STATUS_ERROR;
+            return status;
         }
-        if (status != G_IO_STATUS_NORMAL) {
-            break;
-        }
-        size -= written;
     }
 
-    if (status == G_IO_STATUS_NORMAL) {
+    do {
         status = g_io_channel_flush(c->client_channel, &err);
-        if (err != NULL) {
-            g_warning("error flushing channel: %s", err->message);
-            return G_IO_STATUS_ERROR;
-        }
+    } while (status == G_IO_STATUS_AGAIN);
+
+    if (status != G_IO_STATUS_NORMAL) {
+        g_warning("error flushing channel: %s", err->message);
     }
 
     return status;
@@ -249,7 +243,7 @@
 GAChannel *ga_channel_new(GAChannelMethod method, const gchar *path,
                           GAChannelCallback cb, gpointer opaque)
 {
-    GAChannel *c = g_malloc0(sizeof(GAChannel));
+    GAChannel *c = g_new0(GAChannel, 1);
     c->event_cb = cb;
     c->user_data = opaque;
 
diff --git a/qga/channel-win32.c b/qga/channel-win32.c
index 0d5e5f5..7cc23d3 100644
--- a/qga/channel-win32.c
+++ b/qga/channel-win32.c
@@ -1,9 +1,5 @@
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdbool.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include <windows.h>
-#include <errno.h>
 #include <io.h>
 #include "qga/guest-agent-core.h"
 #include "qga/channel.h"
@@ -80,7 +76,7 @@
     }
 
 out:
-    /* dont block forever, iterate the main loop every once and a while */
+    /* don't block forever, iterate the main loop every once in a while */
     *timeout_ms = 500;
     /* if there's data in the read buffer, or another event is pending,
      * skip polling and issue user cb.
@@ -269,7 +265,7 @@
 GIOStatus ga_channel_write_all(GAChannel *c, const char *buf, size_t size)
 {
     GIOStatus status = G_IO_STATUS_NORMAL;
-    size_t count;
+    size_t count = 0;
 
     while (size) {
         status = ga_channel_write(c, buf, size, &count);
@@ -302,11 +298,11 @@
         g_strlcpy(newpath, path, sizeof(newpath));
     }
 
-    c->handle = CreateFile(newpath, GENERIC_READ | GENERIC_WRITE, 0, NULL,
+    c->handle = win32CreateFile(newpath, GENERIC_READ | GENERIC_WRITE, 0, NULL,
                            OPEN_EXISTING,
                            FILE_FLAG_NO_BUFFERING | FILE_FLAG_OVERLAPPED, NULL);
     if (c->handle == INVALID_HANDLE_VALUE) {
-        g_critical("error opening path");
+        g_critical("error opening path %s", newpath);
         return false;
     }
 
@@ -322,7 +318,7 @@
 GAChannel *ga_channel_new(GAChannelMethod method, const gchar *path,
                           GAChannelCallback cb, gpointer opaque)
 {
-    GAChannel *c = g_malloc0(sizeof(GAChannel));
+    GAChannel *c = g_new0(GAChannel, 1);
     SECURITY_ATTRIBUTES sec_attrs;
 
     if (!ga_channel_open(c, method, path)) {
diff --git a/qga/channel.h b/qga/channel.h
index 3704ea9..ae8cf0f 100644
--- a/qga/channel.h
+++ b/qga/channel.h
@@ -12,7 +12,6 @@
 #ifndef QGA_CHANNEL_H
 #define QGA_CHANNEL_H
 
-#include <glib.h>
 
 typedef struct GAChannel GAChannel;
 
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index f6f3e3c..0113dd9 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -11,23 +11,18 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
 #include <sys/wait.h>
-#include <unistd.h>
-#include <errno.h>
-#include <fcntl.h>
 #include <dirent.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <inttypes.h>
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/queue.h"
 #include "qemu/host-utils.h"
+#include "qemu/sockets.h"
+#include "qemu/base64.h"
+#include "qemu/cutils.h"
 
 #ifndef CONFIG_HAS_ENVIRON
 #ifdef __APPLE__
@@ -132,7 +127,6 @@
 {
    int ret;
    qemu_timeval tq;
-   int64_t time_ns;
 
    ret = qemu_gettimeofday(&tq);
    if (ret < 0) {
@@ -140,8 +134,7 @@
        return -1;
    }
 
-   time_ns = tq.tv_sec * 1000000000LL + tq.tv_usec * 1000;
-   return time_ns;
+   return tq.tv_sec * 1000000000LL + tq.tv_usec * 1000;
 }
 
 void qmp_guest_set_time(bool has_time, int64_t time_ns, Error **errp)
@@ -154,6 +147,8 @@
 
     /* If user has passed a time, validate and set it. */
     if (has_time) {
+        GDate date = { 0, };
+
         /* year-2038 will overflow in case time_t is 32bit */
         if (time_ns / 1000000000 != (time_t)(time_ns / 1000000000)) {
             error_setg(errp, "Time %" PRId64 " is too large", time_ns);
@@ -162,6 +157,11 @@
 
         tv.tv_sec = time_ns / 1000000000;
         tv.tv_usec = (time_ns % 1000000000) / 1000;
+        g_date_set_time_t(&date, tv.tv_sec);
+        if (date.year < 1970 || date.year >= 2070) {
+            error_setg_errno(errp, errno, "Invalid time");
+            return;
+        }
 
         ret = settimeofday(&tv, NULL);
         if (ret < 0) {
@@ -208,15 +208,24 @@
     }
 }
 
+typedef enum {
+    RW_STATE_NEW,
+    RW_STATE_READING,
+    RW_STATE_WRITING,
+} RwState;
+
 typedef struct GuestFileHandle {
     uint64_t id;
     FILE *fh;
+    RwState state;
     QTAILQ_ENTRY(GuestFileHandle) next;
 } GuestFileHandle;
 
 static struct {
     QTAILQ_HEAD(, GuestFileHandle) filehandles;
-} guest_file_state;
+} guest_file_state = {
+    .filehandles = QTAILQ_HEAD_INITIALIZER(guest_file_state.filehandles),
+};
 
 static int64_t guest_file_handle_add(FILE *fh, Error **errp)
 {
@@ -228,7 +237,7 @@
         return -1;
     }
 
-    gfh = g_malloc0(sizeof(GuestFileHandle));
+    gfh = g_new0(GuestFileHandle, 1);
     gfh->id = handle;
     gfh->fh = fh;
     QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
@@ -381,8 +390,7 @@
 {
     FILE *fh;
     Error *local_err = NULL;
-    int fd;
-    int64_t ret = -1, handle;
+    int64_t handle;
 
     if (!has_mode) {
         mode = "r";
@@ -397,15 +405,7 @@
     /* set fd non-blocking to avoid common use cases (like reading from a
      * named pipe) from hanging the agent
      */
-    fd = fileno(fh);
-    ret = fcntl(fd, F_GETFL);
-    ret = fcntl(fd, F_SETFL, ret | O_NONBLOCK);
-    if (ret == -1) {
-        error_setg_errno(errp, errno, "failed to make file '%s' non-blocking",
-                         path);
-        fclose(fh);
-        return -1;
-    }
+    qemu_set_nonblock(fileno(fh));
 
     handle = guest_file_handle_add(fh, errp);
     if (handle < 0) {
@@ -459,6 +459,17 @@
     }
 
     fh = gfh->fh;
+
+    /* explicitly flush when switching from writing to reading */
+    if (gfh->state == RW_STATE_WRITING) {
+        int ret = fflush(fh);
+        if (ret == EOF) {
+            error_setg_errno(errp, errno, "failed to flush file");
+            return NULL;
+        }
+        gfh->state = RW_STATE_NEW;
+    }
+
     buf = g_malloc0(count+1);
     read_count = fread(buf, 1, count, fh);
     if (ferror(fh)) {
@@ -466,12 +477,13 @@
         slog("guest-file-read failed, handle: %" PRId64, handle);
     } else {
         buf[read_count] = 0;
-        read_data = g_malloc0(sizeof(GuestFileRead));
+        read_data = g_new0(GuestFileRead, 1);
         read_data->count = read_count;
         read_data->eof = feof(fh);
         if (read_count) {
             read_data->buf_b64 = g_base64_encode(buf, read_count);
         }
+        gfh->state = RW_STATE_READING;
     }
     g_free(buf);
     clearerr(fh);
@@ -495,7 +507,20 @@
     }
 
     fh = gfh->fh;
-    buf = g_base64_decode(buf_b64, &buf_len);
+
+    if (gfh->state == RW_STATE_READING) {
+        int ret = fseek(fh, 0, SEEK_CUR);
+        if (ret == -1) {
+            error_setg_errno(errp, errno, "failed to seek file");
+            return NULL;
+        }
+        gfh->state = RW_STATE_NEW;
+    }
+
+    buf = qbase64_decode(buf_b64, -1, &buf_len, errp);
+    if (!buf) {
+        return NULL;
+    }
 
     if (!has_count) {
         count = buf_len;
@@ -511,9 +536,10 @@
         error_setg_errno(errp, errno, "failed to write to file");
         slog("guest-file-write failed, handle: %" PRId64, handle);
     } else {
-        write_data = g_malloc0(sizeof(GuestFileWrite));
+        write_data = g_new0(GuestFileWrite, 1);
         write_data->count = write_count;
         write_data->eof = feof(fh);
+        gfh->state = RW_STATE_WRITING;
     }
     g_free(buf);
     clearerr(fh);
@@ -522,25 +548,40 @@
 }
 
 struct GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
-                                          int64_t whence, Error **errp)
+                                          GuestFileWhence *whence_code,
+                                          Error **errp)
 {
     GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
     GuestFileSeek *seek_data = NULL;
     FILE *fh;
     int ret;
+    int whence;
+    Error *err = NULL;
 
     if (!gfh) {
         return NULL;
     }
 
+    /* We stupidly exposed 'whence':'int' in our qapi */
+    whence = ga_parse_whence(whence_code, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
     fh = gfh->fh;
     ret = fseek(fh, offset, whence);
     if (ret == -1) {
         error_setg_errno(errp, errno, "failed to seek file");
+        if (errno == ESPIPE) {
+            /* file is non-seekable, stdio shouldn't be buffering anyways */
+            gfh->state = RW_STATE_NEW;
+        }
     } else {
         seek_data = g_new0(GuestFileSeek, 1);
         seek_data->position = ftell(fh);
         seek_data->eof = feof(fh);
+        gfh->state = RW_STATE_NEW;
     }
     clearerr(fh);
 
@@ -561,14 +602,11 @@
     ret = fflush(fh);
     if (ret == EOF) {
         error_setg_errno(errp, errno, "failed to flush file");
+    } else {
+        gfh->state = RW_STATE_NEW;
     }
 }
 
-static void guest_file_init(void)
-{
-    QTAILQ_INIT(&guest_file_state.filehandles);
-}
-
 /* linux-specific implementations. avoid this if at all possible. */
 #if defined(__linux__)
 
@@ -606,7 +644,7 @@
     *devmajor = 0;
     *devminor = 0;
 
-    if (stat(devpath, &st) < 0) {
+    if (qemu_stat(devpath, &st) < 0) {
         slog("failed to stat device file '%s': %s", devpath, strerror(errno));
         return -1;
     }
@@ -656,7 +694,7 @@
             continue;
         }
 
-        mount = g_malloc0(sizeof(FsMount));
+        mount = g_new0(FsMount, 1);
         mount->dirname = g_strdup(ment->mnt_dir);
         mount->devtype = g_strdup(ment->mnt_type);
         mount->devmajor = devmajor;
@@ -735,7 +773,7 @@
             }
         }
 
-        mount = g_malloc0(sizeof(FsMount));
+        mount = g_new0(FsMount, 1);
         mount->dirname = g_strdup(line + dir_s);
         mount->devtype = g_strdup(dash + type_s);
         mount->devmajor = devmajor;
@@ -1201,8 +1239,8 @@
             goto error;
         }
 
-        /* we try to cull filesytems we know won't work in advance, but other
-         * filesytems may not implement fsfreeze for less obvious reasons.
+        /* we try to cull filesystems we know won't work in advance, but other
+         * filesystems may not implement fsfreeze for less obvious reasons.
          * these will report EOPNOTSUPP. we simply ignore these when tallying
          * the number of frozen filesystems.
          *
@@ -1310,18 +1348,18 @@
 /*
  * Walk list of mounted file systems in the guest, and trim them.
  */
-void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
+GuestFilesystemTrimResponse *
+qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
 {
+    GuestFilesystemTrimResponse *response;
+    GuestFilesystemTrimResultList *list;
+    GuestFilesystemTrimResult *result;
     int ret = 0;
     FsMountList mounts;
     struct FsMount *mount;
     int fd;
     Error *local_err = NULL;
-    struct fstrim_range r = {
-        .start = 0,
-        .len = -1,
-        .minlen = has_minimum ? minimum : 0,
-    };
+    struct fstrim_range r;
 
     slog("guest-fstrim called");
 
@@ -1329,36 +1367,59 @@
     build_fs_mount_list(&mounts, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
-        return;
+        return NULL;
     }
 
+    response = g_malloc0(sizeof(*response));
+
     QTAILQ_FOREACH(mount, &mounts, next) {
+        result = g_malloc0(sizeof(*result));
+        result->path = g_strdup(mount->dirname);
+
+        list = g_malloc0(sizeof(*list));
+        list->value = result;
+        list->next = response->paths;
+        response->paths = list;
+
         fd = qemu_open(mount->dirname, O_RDONLY);
         if (fd == -1) {
-            error_setg_errno(errp, errno, "failed to open %s", mount->dirname);
-            goto error;
+            result->error = g_strdup_printf("failed to open: %s",
+                                            strerror(errno));
+            result->has_error = true;
+            continue;
         }
 
-        /* We try to cull filesytems we know won't work in advance, but other
-         * filesytems may not implement fstrim for less obvious reasons.  These
-         * will report EOPNOTSUPP; we simply ignore these errors.  Any other
-         * error means an unexpected error, so return it in those cases.  In
-         * some other cases ENOTTY will be reported (e.g. CD-ROMs).
+        /* We try to cull filesystems we know won't work in advance, but other
+         * filesystems may not implement fstrim for less obvious reasons.
+         * These will report EOPNOTSUPP; while in some other cases ENOTTY
+         * will be reported (e.g. CD-ROMs).
+         * Any other error means an unexpected error.
          */
+        r.start = 0;
+        r.len = -1;
+        r.minlen = has_minimum ? minimum : 0;
         ret = ioctl(fd, FITRIM, &r);
         if (ret == -1) {
-            if (errno != ENOTTY && errno != EOPNOTSUPP) {
-                error_setg_errno(errp, errno, "failed to trim %s",
-                                 mount->dirname);
-                close(fd);
-                goto error;
+            result->has_error = true;
+            if (errno == ENOTTY || errno == EOPNOTSUPP) {
+                result->error = g_strdup("trim not supported");
+            } else {
+                result->error = g_strdup_printf("failed to trim: %s",
+                                                strerror(errno));
             }
+            close(fd);
+            continue;
         }
+
+        result->has_minimum = true;
+        result->minimum = r.minlen;
+        result->has_trimmed = true;
+        result->trimmed = r.len;
         close(fd);
     }
 
-error:
     free_fs_mount_list(&mounts);
+    return response;
 }
 #endif /* CONFIG_FSTRIM */
 
@@ -1875,61 +1936,505 @@
     return processed;
 }
 
+void qmp_guest_set_user_password(const char *username,
+                                 const char *password,
+                                 bool crypted,
+                                 Error **errp)
+{
+    Error *local_err = NULL;
+    char *passwd_path = NULL;
+    pid_t pid;
+    int status;
+    int datafd[2] = { -1, -1 };
+    char *rawpasswddata = NULL;
+    size_t rawpasswdlen;
+    char *chpasswddata = NULL;
+    size_t chpasswdlen;
+
+    rawpasswddata = (char *)qbase64_decode(password, -1, &rawpasswdlen, errp);
+    if (!rawpasswddata) {
+        return;
+    }
+    rawpasswddata = g_renew(char, rawpasswddata, rawpasswdlen + 1);
+    rawpasswddata[rawpasswdlen] = '\0';
+
+    if (strchr(rawpasswddata, '\n')) {
+        error_setg(errp, "forbidden characters in raw password");
+        goto out;
+    }
+
+    if (strchr(username, '\n') ||
+        strchr(username, ':')) {
+        error_setg(errp, "forbidden characters in username");
+        goto out;
+    }
+
+    chpasswddata = g_strdup_printf("%s:%s\n", username, rawpasswddata);
+    chpasswdlen = strlen(chpasswddata);
+
+    passwd_path = g_find_program_in_path("chpasswd");
+
+    if (!passwd_path) {
+        error_setg(errp, "cannot find 'passwd' program in PATH");
+        goto out;
+    }
+
+    if (pipe(datafd) < 0) {
+        error_setg(errp, "cannot create pipe FDs");
+        goto out;
+    }
+
+    pid = fork();
+    if (pid == 0) {
+        close(datafd[1]);
+        /* child */
+        setsid();
+        dup2(datafd[0], 0);
+        reopen_fd_to_null(1);
+        reopen_fd_to_null(2);
+
+        if (crypted) {
+            execle(passwd_path, "chpasswd", "-e", NULL, environ);
+        } else {
+            execle(passwd_path, "chpasswd", NULL, environ);
+        }
+        _exit(EXIT_FAILURE);
+    } else if (pid < 0) {
+        error_setg_errno(errp, errno, "failed to create child process");
+        goto out;
+    }
+    close(datafd[0]);
+    datafd[0] = -1;
+
+    if (qemu_write_full(datafd[1], chpasswddata, chpasswdlen) != chpasswdlen) {
+        error_setg_errno(errp, errno, "cannot write new account password");
+        goto out;
+    }
+    close(datafd[1]);
+    datafd[1] = -1;
+
+    ga_wait_child(pid, &status, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        goto out;
+    }
+
+    if (!WIFEXITED(status)) {
+        error_setg(errp, "child process has terminated abnormally");
+        goto out;
+    }
+
+    if (WEXITSTATUS(status)) {
+        error_setg(errp, "child process has failed to set user password");
+        goto out;
+    }
+
+out:
+    g_free(chpasswddata);
+    g_free(rawpasswddata);
+    g_free(passwd_path);
+    if (datafd[0] != -1) {
+        close(datafd[0]);
+    }
+    if (datafd[1] != -1) {
+        close(datafd[1]);
+    }
+}
+
+static void ga_read_sysfs_file(int dirfd, const char *pathname, char *buf,
+                               int size, Error **errp)
+{
+    int fd;
+    int res;
+
+    errno = 0;
+    fd = openat(dirfd, pathname, O_RDONLY);
+    if (fd == -1) {
+        error_setg_errno(errp, errno, "open sysfs file \"%s\"", pathname);
+        return;
+    }
+
+    res = pread(fd, buf, size, 0);
+    if (res == -1) {
+        error_setg_errno(errp, errno, "pread sysfs file \"%s\"", pathname);
+    } else if (res == 0) {
+        error_setg(errp, "pread sysfs file \"%s\": unexpected EOF", pathname);
+    }
+    close(fd);
+}
+
+static void ga_write_sysfs_file(int dirfd, const char *pathname,
+                                const char *buf, int size, Error **errp)
+{
+    int fd;
+
+    errno = 0;
+    fd = openat(dirfd, pathname, O_WRONLY);
+    if (fd == -1) {
+        error_setg_errno(errp, errno, "open sysfs file \"%s\"", pathname);
+        return;
+    }
+
+    if (pwrite(fd, buf, size, 0) == -1) {
+        error_setg_errno(errp, errno, "pwrite sysfs file \"%s\"", pathname);
+    }
+
+    close(fd);
+}
+
+/* Transfer online/offline status between @mem_blk and the guest system.
+ *
+ * On input either @errp or *@errp must be NULL.
+ *
+ * In system-to-@mem_blk direction, the following @mem_blk fields are accessed:
+ * - R: mem_blk->phys_index
+ * - W: mem_blk->online
+ * - W: mem_blk->can_offline
+ *
+ * In @mem_blk-to-system direction, the following @mem_blk fields are accessed:
+ * - R: mem_blk->phys_index
+ * - R: mem_blk->online
+ *-  R: mem_blk->can_offline
+ * Written members remain unmodified on error.
+ */
+static void transfer_memory_block(GuestMemoryBlock *mem_blk, bool sys2memblk,
+                                  GuestMemoryBlockResponse *result,
+                                  Error **errp)
+{
+    char *dirpath;
+    int dirfd;
+    char *status;
+    Error *local_err = NULL;
+
+    if (!sys2memblk) {
+        DIR *dp;
+
+        if (!result) {
+            error_setg(errp, "Internal error, 'result' should not be NULL");
+            return;
+        }
+        errno = 0;
+        dp = opendir("/sys/devices/system/memory/");
+         /* if there is no 'memory' directory in sysfs,
+         * we think this VM does not support online/offline memory block,
+         * any other solution?
+         */
+        if (!dp && errno == ENOENT) {
+            result->response =
+                GUEST_MEMORY_BLOCK_RESPONSE_TYPE_OPERATION_NOT_SUPPORTED;
+            goto out1;
+        }
+        closedir(dp);
+    }
+
+    dirpath = g_strdup_printf("/sys/devices/system/memory/memory%" PRId64 "/",
+                              mem_blk->phys_index);
+    dirfd = open(dirpath, O_RDONLY | O_DIRECTORY);
+    if (dirfd == -1) {
+        if (sys2memblk) {
+            error_setg_errno(errp, errno, "open(\"%s\")", dirpath);
+        } else {
+            if (errno == ENOENT) {
+                result->response = GUEST_MEMORY_BLOCK_RESPONSE_TYPE_NOT_FOUND;
+            } else {
+                result->response =
+                    GUEST_MEMORY_BLOCK_RESPONSE_TYPE_OPERATION_FAILED;
+            }
+        }
+        g_free(dirpath);
+        goto out1;
+    }
+    g_free(dirpath);
+
+    status = g_malloc0(10);
+    ga_read_sysfs_file(dirfd, "state", status, 10, &local_err);
+    if (local_err) {
+        /* treat with sysfs file that not exist in old kernel */
+        if (errno == ENOENT) {
+            error_free(local_err);
+            if (sys2memblk) {
+                mem_blk->online = true;
+                mem_blk->can_offline = false;
+            } else if (!mem_blk->online) {
+                result->response =
+                    GUEST_MEMORY_BLOCK_RESPONSE_TYPE_OPERATION_NOT_SUPPORTED;
+            }
+        } else {
+            if (sys2memblk) {
+                error_propagate(errp, local_err);
+            } else {
+                result->response =
+                    GUEST_MEMORY_BLOCK_RESPONSE_TYPE_OPERATION_FAILED;
+            }
+        }
+        goto out2;
+    }
+
+    if (sys2memblk) {
+        char removable = '0';
+
+        mem_blk->online = (strncmp(status, "online", 6) == 0);
+
+        ga_read_sysfs_file(dirfd, "removable", &removable, 1, &local_err);
+        if (local_err) {
+            /* if no 'removable' file, it doesn't support offline mem blk */
+            if (errno == ENOENT) {
+                error_free(local_err);
+                mem_blk->can_offline = false;
+            } else {
+                error_propagate(errp, local_err);
+            }
+        } else {
+            mem_blk->can_offline = (removable != '0');
+        }
+    } else {
+        if (mem_blk->online != (strncmp(status, "online", 6) == 0)) {
+            char *new_state = mem_blk->online ? g_strdup("online") :
+                                                g_strdup("offline");
+
+            ga_write_sysfs_file(dirfd, "state", new_state, strlen(new_state),
+                                &local_err);
+            g_free(new_state);
+            if (local_err) {
+                error_free(local_err);
+                result->response =
+                    GUEST_MEMORY_BLOCK_RESPONSE_TYPE_OPERATION_FAILED;
+                goto out2;
+            }
+
+            result->response = GUEST_MEMORY_BLOCK_RESPONSE_TYPE_SUCCESS;
+            result->has_error_code = false;
+        } /* otherwise pretend successful re-(on|off)-lining */
+    }
+    g_free(status);
+    close(dirfd);
+    return;
+
+out2:
+    g_free(status);
+    close(dirfd);
+out1:
+    if (!sys2memblk) {
+        result->has_error_code = true;
+        result->error_code = errno;
+    }
+}
+
+GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
+{
+    GuestMemoryBlockList *head, **link;
+    Error *local_err = NULL;
+    struct dirent *de;
+    DIR *dp;
+
+    head = NULL;
+    link = &head;
+
+    dp = opendir("/sys/devices/system/memory/");
+    if (!dp) {
+        /* it's ok if this happens to be a system that doesn't expose
+         * memory blocks via sysfs, but otherwise we should report
+         * an error
+         */
+        if (errno != ENOENT) {
+            error_setg_errno(errp, errno, "Can't open directory"
+                             "\"/sys/devices/system/memory/\"");
+        }
+        return NULL;
+    }
+
+    /* Note: the phys_index of memory block may be discontinuous,
+     * this is because a memblk is the unit of the Sparse Memory design, which
+     * allows discontinuous memory ranges (ex. NUMA), so here we should
+     * traverse the memory block directory.
+     */
+    while ((de = readdir(dp)) != NULL) {
+        GuestMemoryBlock *mem_blk;
+        GuestMemoryBlockList *entry;
+
+        if ((strncmp(de->d_name, "memory", 6) != 0) ||
+            !(de->d_type & DT_DIR)) {
+            continue;
+        }
+
+        mem_blk = g_malloc0(sizeof *mem_blk);
+        /* The d_name is "memoryXXX",  phys_index is block id, same as XXX */
+        mem_blk->phys_index = strtoul(&de->d_name[6], NULL, 10);
+        mem_blk->has_can_offline = true; /* lolspeak ftw */
+        transfer_memory_block(mem_blk, true, NULL, &local_err);
+
+        entry = g_malloc0(sizeof *entry);
+        entry->value = mem_blk;
+
+        *link = entry;
+        link = &entry->next;
+    }
+
+    closedir(dp);
+    if (local_err == NULL) {
+        /* there's no guest with zero memory blocks */
+        if (head == NULL) {
+            error_setg(errp, "guest reported zero memory blocks!");
+        }
+        return head;
+    }
+
+    qapi_free_GuestMemoryBlockList(head);
+    error_propagate(errp, local_err);
+    return NULL;
+}
+
+GuestMemoryBlockResponseList *
+qmp_guest_set_memory_blocks(GuestMemoryBlockList *mem_blks, Error **errp)
+{
+    GuestMemoryBlockResponseList *head, **link;
+    Error *local_err = NULL;
+
+    head = NULL;
+    link = &head;
+
+    while (mem_blks != NULL) {
+        GuestMemoryBlockResponse *result;
+        GuestMemoryBlockResponseList *entry;
+        GuestMemoryBlock *current_mem_blk = mem_blks->value;
+
+        result = g_malloc0(sizeof(*result));
+        result->phys_index = current_mem_blk->phys_index;
+        transfer_memory_block(current_mem_blk, false, result, &local_err);
+        if (local_err) { /* should never happen */
+            goto err;
+        }
+        entry = g_malloc0(sizeof *entry);
+        entry->value = result;
+
+        *link = entry;
+        link = &entry->next;
+        mem_blks = mem_blks->next;
+    }
+
+    return head;
+err:
+    qapi_free_GuestMemoryBlockResponseList(head);
+    error_propagate(errp, local_err);
+    return NULL;
+}
+
+GuestMemoryBlockInfo *qmp_guest_get_memory_block_info(Error **errp)
+{
+    Error *local_err = NULL;
+    char *dirpath;
+    int dirfd;
+    char *buf;
+    GuestMemoryBlockInfo *info;
+
+    dirpath = g_strdup_printf("/sys/devices/system/memory/");
+    dirfd = open(dirpath, O_RDONLY | O_DIRECTORY);
+    if (dirfd == -1) {
+        error_setg_errno(errp, errno, "open(\"%s\")", dirpath);
+        g_free(dirpath);
+        return NULL;
+    }
+    g_free(dirpath);
+
+    buf = g_malloc0(20);
+    ga_read_sysfs_file(dirfd, "block_size_bytes", buf, 20, &local_err);
+    close(dirfd);
+    if (local_err) {
+        g_free(buf);
+        error_propagate(errp, local_err);
+        return NULL;
+    }
+
+    info = g_new0(GuestMemoryBlockInfo, 1);
+    info->size = strtol(buf, NULL, 16); /* the unit is bytes */
+
+    g_free(buf);
+
+    return info;
+}
+
 #else /* defined(__linux__) */
 
 void qmp_guest_suspend_disk(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 }
 
 void qmp_guest_suspend_ram(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 }
 
 void qmp_guest_suspend_hybrid(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 }
 
 GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return NULL;
 }
 
 GuestLogicalProcessorList *qmp_guest_get_vcpus(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return NULL;
 }
 
 int64_t qmp_guest_set_vcpus(GuestLogicalProcessorList *vcpus, Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return -1;
 }
 
+void qmp_guest_set_user_password(const char *username,
+                                 const char *password,
+                                 bool crypted,
+                                 Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+}
+
+GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
+GuestMemoryBlockResponseList *
+qmp_guest_set_memory_blocks(GuestMemoryBlockList *mem_blks, Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
+GuestMemoryBlockInfo *qmp_guest_get_memory_block_info(Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
 #endif
 
 #if !defined(CONFIG_FSFREEZE)
 
 GuestFilesystemInfoList *qmp_guest_get_fsinfo(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return NULL;
 }
 
 GuestFsfreezeStatus qmp_guest_fsfreeze_status(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 
     return 0;
 }
 
 int64_t qmp_guest_fsfreeze_freeze(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 
     return 0;
 }
@@ -1938,23 +2443,25 @@
                                        strList *mountpoints,
                                        Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 
     return 0;
 }
 
 int64_t qmp_guest_fsfreeze_thaw(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 
     return 0;
 }
 #endif /* CONFIG_FSFREEZE */
 
 #if !defined(CONFIG_FSTRIM)
-void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
+GuestFilesystemTrimResponse *
+qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
 }
 #endif
 
@@ -1966,11 +2473,13 @@
         const char *list[] = {
             "guest-suspend-disk", "guest-suspend-ram",
             "guest-suspend-hybrid", "guest-network-get-interfaces",
-            "guest-get-vcpus", "guest-set-vcpus", NULL};
+            "guest-get-vcpus", "guest-set-vcpus",
+            "guest-get-memory-blocks", "guest-set-memory-blocks",
+            "guest-get-memory-block-size", NULL};
         char **p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 #endif
@@ -1984,13 +2493,13 @@
         char **p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 #endif
 
 #if !defined(CONFIG_FSTRIM)
-    blacklist = g_list_append(blacklist, (char *)"guest-fstrim");
+    blacklist = g_list_append(blacklist, g_strdup("guest-fstrim"));
 #endif
 
     return blacklist;
@@ -2002,5 +2511,4 @@
 #if defined(CONFIG_FSFREEZE)
     ga_command_state_add(cs, NULL, guest_fsfreeze_cleanup);
 #endif
-    ga_command_state_add(cs, guest_file_init, NULL);
 }
diff --git a/qga/commands-win32.c b/qga/commands-win32.c
index 3bcbeae..cba342a 100644
--- a/qga/commands-win32.c
+++ b/qga/commands-win32.c
@@ -11,13 +11,28 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include <wtypes.h>
 #include <powrprof.h>
+#include <winsock2.h>
+#include <ws2tcpip.h>
+#include <iptypes.h>
+#include <iphlpapi.h>
+#ifdef CONFIG_QGA_NTDDSCSI
+#include <winioctl.h>
+#include <ntddscsi.h>
+#include <setupapi.h>
+#include <initguid.h>
+#endif
+#include <lm.h>
+
 #include "qga/guest-agent-core.h"
 #include "qga/vss-win32.h"
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/queue.h"
+#include "qemu/host-utils.h"
+#include "qemu/base64.h"
 
 #ifndef SHTDN_REASON_FLAG_PLANNED
 #define SHTDN_REASON_FLAG_PLANNED 0x80000000
@@ -29,6 +44,176 @@
                        (365 * (1970 - 1601) +       \
                         (1970 - 1601) / 4 - 3))
 
+#define INVALID_SET_FILE_POINTER ((DWORD)-1)
+
+typedef struct GuestFileHandle {
+    int64_t id;
+    HANDLE fh;
+    QTAILQ_ENTRY(GuestFileHandle) next;
+} GuestFileHandle;
+
+static struct {
+    QTAILQ_HEAD(, GuestFileHandle) filehandles;
+} guest_file_state = {
+    .filehandles = QTAILQ_HEAD_INITIALIZER(guest_file_state.filehandles),
+};
+
+#define FILE_GENERIC_APPEND (FILE_GENERIC_WRITE & ~FILE_WRITE_DATA)
+
+typedef struct OpenFlags {
+    const char *forms;
+    DWORD desired_access;
+    DWORD creation_disposition;
+} OpenFlags;
+static OpenFlags guest_file_open_modes[] = {
+    {"r",   GENERIC_READ,                     OPEN_EXISTING},
+    {"rb",  GENERIC_READ,                     OPEN_EXISTING},
+    {"w",   GENERIC_WRITE,                    CREATE_ALWAYS},
+    {"wb",  GENERIC_WRITE,                    CREATE_ALWAYS},
+    {"a",   FILE_GENERIC_APPEND,              OPEN_ALWAYS  },
+    {"r+",  GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"rb+", GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"r+b", GENERIC_WRITE|GENERIC_READ,       OPEN_EXISTING},
+    {"w+",  GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"wb+", GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"w+b", GENERIC_WRITE|GENERIC_READ,       CREATE_ALWAYS},
+    {"a+",  FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  },
+    {"ab+", FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  },
+    {"a+b", FILE_GENERIC_APPEND|GENERIC_READ, OPEN_ALWAYS  }
+};
+
+static OpenFlags *find_open_flag(const char *mode_str)
+{
+    int mode;
+    Error **errp = NULL;
+
+    for (mode = 0; mode < ARRAY_SIZE(guest_file_open_modes); ++mode) {
+        OpenFlags *flags = guest_file_open_modes + mode;
+
+        if (strcmp(flags->forms, mode_str) == 0) {
+            return flags;
+        }
+    }
+
+    error_setg(errp, "invalid file open mode '%s'", mode_str);
+    return NULL;
+}
+
+static int64_t guest_file_handle_add(HANDLE fh, Error **errp)
+{
+    GuestFileHandle *gfh;
+    int64_t handle;
+
+    handle = ga_get_fd_handle(ga_state, errp);
+    if (handle < 0) {
+        return -1;
+    }
+    gfh = g_new0(GuestFileHandle, 1);
+    gfh->id = handle;
+    gfh->fh = fh;
+    QTAILQ_INSERT_TAIL(&guest_file_state.filehandles, gfh, next);
+
+    return handle;
+}
+
+static GuestFileHandle *guest_file_handle_find(int64_t id, Error **errp)
+{
+    GuestFileHandle *gfh;
+    QTAILQ_FOREACH(gfh, &guest_file_state.filehandles, next) {
+        if (gfh->id == id) {
+            return gfh;
+        }
+    }
+    error_setg(errp, "handle '%" PRId64 "' has not been found", id);
+    return NULL;
+}
+
+static void handle_set_nonblocking(HANDLE fh)
+{
+    DWORD file_type, pipe_state;
+    file_type = GetFileType(fh);
+    if (file_type != FILE_TYPE_PIPE) {
+        return;
+    }
+    /* If file_type == FILE_TYPE_PIPE, according to MSDN
+     * the specified file is socket or named pipe */
+    if (!GetNamedPipeHandleState(fh, &pipe_state, NULL,
+                                 NULL, NULL, NULL, 0)) {
+        return;
+    }
+    /* The fd is named pipe fd */
+    if (pipe_state & PIPE_NOWAIT) {
+        return;
+    }
+
+    pipe_state |= PIPE_NOWAIT;
+    SetNamedPipeHandleState(fh, &pipe_state, NULL, NULL);
+}
+
+int64_t qmp_guest_file_open(const char *path, bool has_mode,
+                            const char *mode, Error **errp)
+{
+    int64_t fd;
+    HANDLE fh;
+    HANDLE templ_file = NULL;
+    DWORD share_mode = FILE_SHARE_READ;
+    DWORD flags_and_attr = FILE_ATTRIBUTE_NORMAL;
+    LPSECURITY_ATTRIBUTES sa_attr = NULL;
+    OpenFlags *guest_flags;
+
+    if (!has_mode) {
+        mode = "r";
+    }
+    slog("guest-file-open called, filepath: %s, mode: %s", path, mode);
+    guest_flags = find_open_flag(mode);
+    if (guest_flags == NULL) {
+        error_setg(errp, "invalid file open mode");
+        return -1;
+    }
+
+    fh = win32CreateFile(path, guest_flags->desired_access, share_mode, sa_attr,
+                    guest_flags->creation_disposition, flags_and_attr,
+                    templ_file);
+    if (fh == INVALID_HANDLE_VALUE) {
+        error_setg_win32(errp, GetLastError(), "failed to open file '%s'",
+                         path);
+        return -1;
+    }
+
+    /* set fd non-blocking to avoid common use cases (like reading from a
+     * named pipe) from hanging the agent
+     */
+    handle_set_nonblocking(fh);
+
+    fd = guest_file_handle_add(fh, errp);
+    if (fd < 0) {
+        CloseHandle(fh);
+        error_setg(errp, "failed to add handle to qmp handle table");
+        return -1;
+    }
+
+    slog("guest-file-open, handle: % " PRId64, fd);
+    return fd;
+}
+
+void qmp_guest_file_close(int64_t handle, Error **errp)
+{
+    bool ret;
+    GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
+    slog("guest-file-close called, handle: %" PRId64, handle);
+    if (gfh == NULL) {
+        return;
+    }
+    ret = CloseHandle(gfh->fh);
+    if (!ret) {
+        error_setg_win32(errp, GetLastError(), "failed close handle");
+        return;
+    }
+
+    QTAILQ_REMOVE(&guest_file_state.filehandles, gfh, next);
+    g_free(gfh);
+}
+
 static void acquire_privilege(const char *name, Error **errp)
 {
     HANDLE token = NULL;
@@ -39,8 +224,8 @@
         TOKEN_ADJUST_PRIVILEGES|TOKEN_QUERY, &token))
     {
         if (!LookupPrivilegeValue(NULL, name, &priv.Privileges[0].Luid)) {
-            error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                      "no luid for requested privilege");
+            error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                       "no luid for requested privilege");
             goto out;
         }
 
@@ -48,23 +233,21 @@
         priv.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
 
         if (!AdjustTokenPrivileges(token, FALSE, &priv, 0, NULL, 0)) {
-            error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                      "unable to acquire requested privilege");
+            error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                       "unable to acquire requested privilege");
             goto out;
         }
 
     } else {
-        error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                  "failed to open privilege token");
+        error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                   "failed to open privilege token");
     }
 
 out:
     if (token) {
         CloseHandle(token);
     }
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
 static void execute_async(DWORD WINAPI (*func)(LPVOID), LPVOID opaque,
@@ -74,8 +257,8 @@
 
     HANDLE thread = CreateThread(NULL, 0, func, opaque, 0, NULL);
     if (!thread) {
-        error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                  "failed to dispatch asynchronous command");
+        error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                   "failed to dispatch asynchronous command");
         error_propagate(errp, local_err);
     }
 }
@@ -94,8 +277,8 @@
     } else if (strcmp(mode, "reboot") == 0) {
         shutdown_flag |= EWX_REBOOT;
     } else {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, "mode",
-                  "halt|powerdown|reboot");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "mode",
+                   "halt|powerdown|reboot");
         return;
     }
 
@@ -109,53 +292,444 @@
 
     if (!ExitWindowsEx(shutdown_flag, SHTDN_REASON_FLAG_PLANNED)) {
         slog("guest-shutdown failed: %lu", GetLastError());
-        error_set(errp, QERR_UNDEFINED_ERROR);
+        error_setg(errp, QERR_UNDEFINED_ERROR);
     }
 }
 
-int64_t qmp_guest_file_open(const char *path, bool has_mode, const char *mode,
-                            Error **errp)
-{
-    error_set(errp, QERR_UNSUPPORTED);
-    return 0;
-}
-
-void qmp_guest_file_close(int64_t handle, Error **errp)
-{
-    error_set(errp, QERR_UNSUPPORTED);
-}
-
 GuestFileRead *qmp_guest_file_read(int64_t handle, bool has_count,
                                    int64_t count, Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
-    return 0;
+    GuestFileRead *read_data = NULL;
+    guchar *buf;
+    HANDLE fh;
+    bool is_ok;
+    DWORD read_count;
+    GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
+
+    if (!gfh) {
+        return NULL;
+    }
+    if (!has_count) {
+        count = QGA_READ_COUNT_DEFAULT;
+    } else if (count < 0) {
+        error_setg(errp, "value '%" PRId64
+                   "' is invalid for argument count", count);
+        return NULL;
+    }
+
+    fh = gfh->fh;
+    buf = g_malloc0(count+1);
+    is_ok = ReadFile(fh, buf, count, &read_count, NULL);
+    if (!is_ok) {
+        error_setg_win32(errp, GetLastError(), "failed to read file");
+        slog("guest-file-read failed, handle %" PRId64, handle);
+    } else {
+        buf[read_count] = 0;
+        read_data = g_new0(GuestFileRead, 1);
+        read_data->count = (size_t)read_count;
+        read_data->eof = read_count == 0;
+
+        if (read_count != 0) {
+            read_data->buf_b64 = g_base64_encode(buf, read_count);
+        }
+    }
+    g_free(buf);
+
+    return read_data;
 }
 
 GuestFileWrite *qmp_guest_file_write(int64_t handle, const char *buf_b64,
                                      bool has_count, int64_t count,
                                      Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
-    return 0;
+    GuestFileWrite *write_data = NULL;
+    guchar *buf;
+    gsize buf_len;
+    bool is_ok;
+    DWORD write_count;
+    GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
+    HANDLE fh;
+
+    if (!gfh) {
+        return NULL;
+    }
+    fh = gfh->fh;
+    buf = qbase64_decode(buf_b64, -1, &buf_len, errp);
+    if (!buf) {
+        return NULL;
+    }
+
+    if (!has_count) {
+        count = buf_len;
+    } else if (count < 0 || count > buf_len) {
+        error_setg(errp, "value '%" PRId64
+                   "' is invalid for argument count", count);
+        goto done;
+    }
+
+    is_ok = WriteFile(fh, buf, count, &write_count, NULL);
+    if (!is_ok) {
+        error_setg_win32(errp, GetLastError(), "failed to write to file");
+        slog("guest-file-write-failed, handle: %" PRId64, handle);
+    } else {
+        write_data = g_new0(GuestFileWrite, 1);
+        write_data->count = (size_t) write_count;
+    }
+
+done:
+    g_free(buf);
+    return write_data;
 }
 
 GuestFileSeek *qmp_guest_file_seek(int64_t handle, int64_t offset,
-                                   int64_t whence, Error **errp)
+                                   GuestFileWhence *whence_code,
+                                   Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
-    return 0;
+    GuestFileHandle *gfh;
+    GuestFileSeek *seek_data;
+    HANDLE fh;
+    LARGE_INTEGER new_pos, off_pos;
+    off_pos.QuadPart = offset;
+    BOOL res;
+    int whence;
+    Error *err = NULL;
+
+    gfh = guest_file_handle_find(handle, errp);
+    if (!gfh) {
+        return NULL;
+    }
+
+    /* We stupidly exposed 'whence':'int' in our qapi */
+    whence = ga_parse_whence(whence_code, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
+    fh = gfh->fh;
+    res = SetFilePointerEx(fh, off_pos, &new_pos, whence);
+    if (!res) {
+        error_setg_win32(errp, GetLastError(), "failed to seek file");
+        return NULL;
+    }
+    seek_data = g_new0(GuestFileSeek, 1);
+    seek_data->position = new_pos.QuadPart;
+    return seek_data;
 }
 
 void qmp_guest_file_flush(int64_t handle, Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    HANDLE fh;
+    GuestFileHandle *gfh = guest_file_handle_find(handle, errp);
+    if (!gfh) {
+        return;
+    }
+
+    fh = gfh->fh;
+    if (!FlushFileBuffers(fh)) {
+        error_setg_win32(errp, GetLastError(), "failed to flush file");
+    }
+}
+
+#ifdef CONFIG_QGA_NTDDSCSI
+
+static STORAGE_BUS_TYPE win2qemu[] = {
+    [BusTypeUnknown] = GUEST_DISK_BUS_TYPE_UNKNOWN,
+    [BusTypeScsi] = GUEST_DISK_BUS_TYPE_SCSI,
+    [BusTypeAtapi] = GUEST_DISK_BUS_TYPE_IDE,
+    [BusTypeAta] = GUEST_DISK_BUS_TYPE_IDE,
+    [BusType1394] = GUEST_DISK_BUS_TYPE_IEEE1394,
+    [BusTypeSsa] = GUEST_DISK_BUS_TYPE_SSA,
+    [BusTypeFibre] = GUEST_DISK_BUS_TYPE_SSA,
+    [BusTypeUsb] = GUEST_DISK_BUS_TYPE_USB,
+    [BusTypeRAID] = GUEST_DISK_BUS_TYPE_RAID,
+#if (_WIN32_WINNT >= 0x0600)
+    [BusTypeiScsi] = GUEST_DISK_BUS_TYPE_ISCSI,
+    [BusTypeSas] = GUEST_DISK_BUS_TYPE_SAS,
+    [BusTypeSata] = GUEST_DISK_BUS_TYPE_SATA,
+    [BusTypeSd] =  GUEST_DISK_BUS_TYPE_SD,
+    [BusTypeMmc] = GUEST_DISK_BUS_TYPE_MMC,
+#endif
+#if (_WIN32_WINNT >= 0x0601)
+    [BusTypeVirtual] = GUEST_DISK_BUS_TYPE_VIRTUAL,
+    [BusTypeFileBackedVirtual] = GUEST_DISK_BUS_TYPE_FILE_BACKED_VIRTUAL,
+#endif
+};
+
+static GuestDiskBusType find_bus_type(STORAGE_BUS_TYPE bus)
+{
+    if (bus > ARRAY_SIZE(win2qemu) || (int)bus < 0) {
+        return GUEST_DISK_BUS_TYPE_UNKNOWN;
+    }
+    return win2qemu[(int)bus];
+}
+
+DEFINE_GUID(GUID_DEVINTERFACE_VOLUME,
+        0x53f5630dL, 0xb6bf, 0x11d0, 0x94, 0xf2,
+        0x00, 0xa0, 0xc9, 0x1e, 0xfb, 0x8b);
+
+static GuestPCIAddress *get_pci_info(char *guid, Error **errp)
+{
+    HDEVINFO dev_info;
+    SP_DEVINFO_DATA dev_info_data;
+    DWORD size = 0;
+    int i;
+    char dev_name[MAX_PATH];
+    char *buffer = NULL;
+    GuestPCIAddress *pci = NULL;
+    char *name = g_strdup(&guid[4]);
+
+    if (!QueryDosDevice(name, dev_name, ARRAY_SIZE(dev_name))) {
+        error_setg_win32(errp, GetLastError(), "failed to get dos device name");
+        goto out;
+    }
+
+    dev_info = SetupDiGetClassDevs(&GUID_DEVINTERFACE_VOLUME, 0, 0,
+                                   DIGCF_PRESENT | DIGCF_DEVICEINTERFACE);
+    if (dev_info == INVALID_HANDLE_VALUE) {
+        error_setg_win32(errp, GetLastError(), "failed to get devices tree");
+        goto out;
+    }
+
+    dev_info_data.cbSize = sizeof(SP_DEVINFO_DATA);
+    for (i = 0; SetupDiEnumDeviceInfo(dev_info, i, &dev_info_data); i++) {
+        DWORD addr, bus, slot, func, dev, data, size2;
+        while (!SetupDiGetDeviceRegistryProperty(dev_info, &dev_info_data,
+                                            SPDRP_PHYSICAL_DEVICE_OBJECT_NAME,
+                                            &data, (PBYTE)buffer, size,
+                                            &size2)) {
+            size = MAX(size, size2);
+            if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
+                g_free(buffer);
+                /* Double the size to avoid problems on
+                 * W2k MBCS systems per KB 888609.
+                 * https://support.microsoft.com/en-us/kb/259695 */
+                buffer = g_malloc(size * 2);
+            } else {
+                error_setg_win32(errp, GetLastError(),
+                        "failed to get device name");
+                goto out;
+            }
+        }
+
+        if (g_strcmp0(buffer, dev_name)) {
+            continue;
+        }
+
+        /* There is no need to allocate buffer in the next functions. The size
+         * is known and ULONG according to
+         * https://support.microsoft.com/en-us/kb/253232
+         * https://msdn.microsoft.com/en-us/library/windows/hardware/ff543095(v=vs.85).aspx
+         */
+        if (!SetupDiGetDeviceRegistryProperty(dev_info, &dev_info_data,
+                   SPDRP_BUSNUMBER, &data, (PBYTE)&bus, size, NULL)) {
+            break;
+        }
+
+        /* The function retrieves the device's address. This value will be
+         * transformed into device function and number */
+        if (!SetupDiGetDeviceRegistryProperty(dev_info, &dev_info_data,
+                   SPDRP_ADDRESS, &data, (PBYTE)&addr, size, NULL)) {
+            break;
+        }
+
+        /* This call returns UINumber of DEVICE_CAPABILITIES structure.
+         * This number is typically a user-perceived slot number. */
+        if (!SetupDiGetDeviceRegistryProperty(dev_info, &dev_info_data,
+                   SPDRP_UI_NUMBER, &data, (PBYTE)&slot, size, NULL)) {
+            break;
+        }
+
+        /* SetupApi gives us the same information as driver with
+         * IoGetDeviceProperty. According to Microsoft
+         * https://support.microsoft.com/en-us/kb/253232
+         * FunctionNumber = (USHORT)((propertyAddress) & 0x0000FFFF);
+         * DeviceNumber = (USHORT)(((propertyAddress) >> 16) & 0x0000FFFF);
+         * SPDRP_ADDRESS is propertyAddress, so we do the same.*/
+
+        func = addr & 0x0000FFFF;
+        dev = (addr >> 16) & 0x0000FFFF;
+        pci = g_malloc0(sizeof(*pci));
+        pci->domain = dev;
+        pci->slot = slot;
+        pci->function = func;
+        pci->bus = bus;
+        break;
+    }
+out:
+    g_free(buffer);
+    g_free(name);
+    return pci;
+}
+
+static int get_disk_bus_type(HANDLE vol_h, Error **errp)
+{
+    STORAGE_PROPERTY_QUERY query;
+    STORAGE_DEVICE_DESCRIPTOR *dev_desc, buf;
+    DWORD received;
+
+    dev_desc = &buf;
+    dev_desc->Size = sizeof(buf);
+    query.PropertyId = StorageDeviceProperty;
+    query.QueryType = PropertyStandardQuery;
+
+    if (!DeviceIoControl(vol_h, IOCTL_STORAGE_QUERY_PROPERTY, &query,
+                         sizeof(STORAGE_PROPERTY_QUERY), dev_desc,
+                         dev_desc->Size, &received, NULL)) {
+        error_setg_win32(errp, GetLastError(), "failed to get bus type");
+        return -1;
+    }
+
+    return dev_desc->BusType;
+}
+
+/* VSS provider works with volumes, thus there is no difference if
+ * the volume consist of spanned disks. Info about the first disk in the
+ * volume is returned for the spanned disk group (LVM) */
+static GuestDiskAddressList *build_guest_disk_info(char *guid, Error **errp)
+{
+    GuestDiskAddressList *list = NULL;
+    GuestDiskAddress *disk;
+    SCSI_ADDRESS addr, *scsi_ad;
+    DWORD len;
+    int bus;
+    HANDLE vol_h;
+
+    scsi_ad = &addr;
+    char *name = g_strndup(guid, strlen(guid)-1);
+
+    vol_h = win32CreateFile(name, 0, FILE_SHARE_READ, NULL, OPEN_EXISTING,
+                       0, NULL);
+    if (vol_h == INVALID_HANDLE_VALUE) {
+        error_setg_win32(errp, GetLastError(), "failed to open volume");
+        goto out_free;
+    }
+
+    bus = get_disk_bus_type(vol_h, errp);
+    if (bus < 0) {
+        goto out_close;
+    }
+
+    disk = g_malloc0(sizeof(*disk));
+    disk->bus_type = find_bus_type(bus);
+    if (bus == BusTypeScsi || bus == BusTypeAta || bus == BusTypeRAID
+#if (_WIN32_WINNT >= 0x0600)
+            /* This bus type is not supported before Windows Server 2003 SP1 */
+            || bus == BusTypeSas
+#endif
+        ) {
+        /* We are able to use the same ioctls for different bus types
+         * according to Microsoft docs
+         * https://technet.microsoft.com/en-us/library/ee851589(v=ws.10).aspx */
+        if (DeviceIoControl(vol_h, IOCTL_SCSI_GET_ADDRESS, NULL, 0, scsi_ad,
+                            sizeof(SCSI_ADDRESS), &len, NULL)) {
+            disk->unit = addr.Lun;
+            disk->target = addr.TargetId;
+            disk->bus = addr.PathId;
+            disk->pci_controller = get_pci_info(name, errp);
+        }
+        /* We do not set error in this case, because we still have enough
+         * information about volume. */
+    } else {
+         disk->pci_controller = NULL;
+    }
+
+    list = g_malloc0(sizeof(*list));
+    list->value = disk;
+    list->next = NULL;
+out_close:
+    CloseHandle(vol_h);
+out_free:
+    g_free(name);
+    return list;
+}
+
+#else
+
+static GuestDiskAddressList *build_guest_disk_info(char *guid, Error **errp)
+{
+    return NULL;
+}
+
+#endif /* CONFIG_QGA_NTDDSCSI */
+
+static GuestFilesystemInfo *build_guest_fsinfo(char *guid, Error **errp)
+{
+    DWORD info_size;
+    char mnt, *mnt_point;
+    char fs_name[32];
+    char vol_info[MAX_PATH+1];
+    size_t len;
+    GuestFilesystemInfo *fs = NULL;
+
+    GetVolumePathNamesForVolumeName(guid, (LPCH)&mnt, 0, &info_size);
+    if (GetLastError() != ERROR_MORE_DATA) {
+        error_setg_win32(errp, GetLastError(), "failed to get volume name");
+        return NULL;
+    }
+
+    mnt_point = g_malloc(info_size + 1);
+    if (!GetVolumePathNamesForVolumeName(guid, mnt_point, info_size,
+                                         &info_size)) {
+        error_setg_win32(errp, GetLastError(), "failed to get volume name");
+        goto free;
+    }
+
+    len = strlen(mnt_point);
+    mnt_point[len] = '\\';
+    mnt_point[len+1] = 0;
+    if (!GetVolumeInformation(mnt_point, vol_info, sizeof(vol_info), NULL, NULL,
+                              NULL, (LPSTR)&fs_name, sizeof(fs_name))) {
+        if (GetLastError() != ERROR_NOT_READY) {
+            error_setg_win32(errp, GetLastError(), "failed to get volume info");
+        }
+        goto free;
+    }
+
+    fs_name[sizeof(fs_name) - 1] = 0;
+    fs = g_malloc(sizeof(*fs));
+    fs->name = g_strdup(guid);
+    if (len == 0) {
+        fs->mountpoint = g_strdup("System Reserved");
+    } else {
+        fs->mountpoint = g_strndup(mnt_point, len);
+    }
+    fs->type = g_strdup(fs_name);
+    fs->disk = build_guest_disk_info(guid, errp);
+free:
+    g_free(mnt_point);
+    return fs;
 }
 
 GuestFilesystemInfoList *qmp_guest_get_fsinfo(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
-    return NULL;
+    HANDLE vol_h;
+    GuestFilesystemInfoList *new, *ret = NULL;
+    char guid[256];
+
+    vol_h = FindFirstVolume(guid, sizeof(guid));
+    if (vol_h == INVALID_HANDLE_VALUE) {
+        error_setg_win32(errp, GetLastError(), "failed to find any volume");
+        return NULL;
+    }
+
+    do {
+        GuestFilesystemInfo *info = build_guest_fsinfo(guid, errp);
+        if (info == NULL) {
+            continue;
+        }
+        new = g_malloc(sizeof(*ret));
+        new->value = info;
+        new->next = ret;
+        ret = new;
+    } while (FindNextVolume(vol_h, guid, sizeof(guid)));
+
+    if (GetLastError() != ERROR_NO_MORE_FILES) {
+        error_setg_win32(errp, GetLastError(), "failed to find next volume");
+    }
+
+    FindVolumeClose(vol_h);
+    return ret;
 }
 
 /*
@@ -164,7 +738,7 @@
 GuestFsfreezeStatus qmp_guest_fsfreeze_status(Error **errp)
 {
     if (!vss_initialized()) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         return 0;
     }
 
@@ -185,7 +759,7 @@
     Error *local_err = NULL;
 
     if (!vss_initialized()) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         return 0;
     }
 
@@ -216,7 +790,7 @@
                                        strList *mountpoints,
                                        Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 
     return 0;
 }
@@ -229,7 +803,7 @@
     int i;
 
     if (!vss_initialized()) {
-        error_set(errp, QERR_UNSUPPORTED);
+        error_setg(errp, QERR_UNSUPPORTED);
         return 0;
     }
 
@@ -263,9 +837,11 @@
  * Walk list of mounted file systems in the guest, and discard unused
  * areas.
  */
-void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
+GuestFilesystemTrimResponse *
+qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
 }
 
 typedef enum {
@@ -280,33 +856,31 @@
 
     ZeroMemory(&sys_pwr_caps, sizeof(sys_pwr_caps));
     if (!GetPwrCapabilities(&sys_pwr_caps)) {
-        error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                  "failed to determine guest suspend capabilities");
+        error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                   "failed to determine guest suspend capabilities");
         goto out;
     }
 
     switch (mode) {
     case GUEST_SUSPEND_MODE_DISK:
         if (!sys_pwr_caps.SystemS4) {
-            error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                      "suspend-to-disk not supported by OS");
+            error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                       "suspend-to-disk not supported by OS");
         }
         break;
     case GUEST_SUSPEND_MODE_RAM:
         if (!sys_pwr_caps.SystemS3) {
-            error_set(&local_err, QERR_QGA_COMMAND_FAILED,
-                      "suspend-to-ram not supported by OS");
+            error_setg(&local_err, QERR_QGA_COMMAND_FAILED,
+                       "suspend-to-ram not supported by OS");
         }
         break;
     default:
-        error_set(&local_err, QERR_INVALID_PARAMETER_VALUE, "mode",
-                  "GuestSuspendMode");
+        error_setg(&local_err, QERR_INVALID_PARAMETER_VALUE, "mode",
+                   "GuestSuspendMode");
     }
 
 out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
+    error_propagate(errp, local_err);
 }
 
 static DWORD WINAPI do_suspend(LPVOID opaque)
@@ -325,7 +899,7 @@
 void qmp_guest_suspend_disk(Error **errp)
 {
     Error *local_err = NULL;
-    GuestSuspendMode *mode = g_malloc(sizeof(GuestSuspendMode));
+    GuestSuspendMode *mode = g_new(GuestSuspendMode, 1);
 
     *mode = GUEST_SUSPEND_MODE_DISK;
     check_suspend_mode(*mode, &local_err);
@@ -341,7 +915,7 @@
 void qmp_guest_suspend_ram(Error **errp)
 {
     Error *local_err = NULL;
-    GuestSuspendMode *mode = g_malloc(sizeof(GuestSuspendMode));
+    GuestSuspendMode *mode = g_new(GuestSuspendMode, 1);
 
     *mode = GUEST_SUSPEND_MODE_RAM;
     check_suspend_mode(*mode, &local_err);
@@ -356,19 +930,226 @@
 
 void qmp_guest_suspend_hybrid(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
 }
 
+static IP_ADAPTER_ADDRESSES *guest_get_adapters_addresses(Error **errp)
+{
+    IP_ADAPTER_ADDRESSES *adptr_addrs = NULL;
+    ULONG adptr_addrs_len = 0;
+    DWORD ret;
+
+    /* Call the first time to get the adptr_addrs_len. */
+    GetAdaptersAddresses(AF_UNSPEC, GAA_FLAG_INCLUDE_PREFIX,
+                         NULL, adptr_addrs, &adptr_addrs_len);
+
+    adptr_addrs = g_malloc(adptr_addrs_len);
+    ret = GetAdaptersAddresses(AF_UNSPEC, GAA_FLAG_INCLUDE_PREFIX,
+                               NULL, adptr_addrs, &adptr_addrs_len);
+    if (ret != ERROR_SUCCESS) {
+        error_setg_win32(errp, ret, "failed to get adapters addresses");
+        g_free(adptr_addrs);
+        adptr_addrs = NULL;
+    }
+    return adptr_addrs;
+}
+
+static char *guest_wctomb_dup(WCHAR *wstr)
+{
+    char *str;
+    size_t i;
+
+    i = wcslen(wstr) + 1;
+    str = g_malloc(i);
+    WideCharToMultiByte(CP_ACP, WC_COMPOSITECHECK,
+                        wstr, -1, str, i, NULL, NULL);
+    return str;
+}
+
+static char *guest_addr_to_str(IP_ADAPTER_UNICAST_ADDRESS *ip_addr,
+                               Error **errp)
+{
+    char addr_str[INET6_ADDRSTRLEN + INET_ADDRSTRLEN];
+    DWORD len;
+    int ret;
+
+    if (ip_addr->Address.lpSockaddr->sa_family == AF_INET ||
+            ip_addr->Address.lpSockaddr->sa_family == AF_INET6) {
+        len = sizeof(addr_str);
+        ret = WSAAddressToString(ip_addr->Address.lpSockaddr,
+                                 ip_addr->Address.iSockaddrLength,
+                                 NULL,
+                                 addr_str,
+                                 &len);
+        if (ret != 0) {
+            error_setg_win32(errp, WSAGetLastError(),
+                "failed address presentation form conversion");
+            return NULL;
+        }
+        return g_strdup(addr_str);
+    }
+    return NULL;
+}
+
+#if (_WIN32_WINNT >= 0x0600)
+static int64_t guest_ip_prefix(IP_ADAPTER_UNICAST_ADDRESS *ip_addr)
+{
+    /* For Windows Vista/2008 and newer, use the OnLinkPrefixLength
+     * field to obtain the prefix.
+     */
+    return ip_addr->OnLinkPrefixLength;
+}
+#else
+/* When using the Windows XP and 2003 build environment, do the best we can to
+ * figure out the prefix.
+ */
+static IP_ADAPTER_INFO *guest_get_adapters_info(void)
+{
+    IP_ADAPTER_INFO *adptr_info = NULL;
+    ULONG adptr_info_len = 0;
+    DWORD ret;
+
+    /* Call the first time to get the adptr_info_len. */
+    GetAdaptersInfo(adptr_info, &adptr_info_len);
+
+    adptr_info = g_malloc(adptr_info_len);
+    ret = GetAdaptersInfo(adptr_info, &adptr_info_len);
+    if (ret != ERROR_SUCCESS) {
+        g_free(adptr_info);
+        adptr_info = NULL;
+    }
+    return adptr_info;
+}
+
+static int64_t guest_ip_prefix(IP_ADAPTER_UNICAST_ADDRESS *ip_addr)
+{
+    int64_t prefix = -1; /* Use for AF_INET6 and unknown/undetermined values. */
+    IP_ADAPTER_INFO *adptr_info, *info;
+    IP_ADDR_STRING *ip;
+    struct in_addr *p;
+
+    if (ip_addr->Address.lpSockaddr->sa_family != AF_INET) {
+        return prefix;
+    }
+    adptr_info = guest_get_adapters_info();
+    if (adptr_info == NULL) {
+        return prefix;
+    }
+
+    /* Match up the passed in ip_addr with one found in adaptr_info.
+     * The matching one in adptr_info will have the netmask.
+     */
+    p = &((struct sockaddr_in *)ip_addr->Address.lpSockaddr)->sin_addr;
+    for (info = adptr_info; info; info = info->Next) {
+        for (ip = &info->IpAddressList; ip; ip = ip->Next) {
+            if (p->S_un.S_addr == inet_addr(ip->IpAddress.String)) {
+                prefix = ctpop32(inet_addr(ip->IpMask.String));
+                goto out;
+            }
+        }
+    }
+out:
+    g_free(adptr_info);
+    return prefix;
+}
+#endif
+
 GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
-    return NULL;
+    IP_ADAPTER_ADDRESSES *adptr_addrs, *addr;
+    IP_ADAPTER_UNICAST_ADDRESS *ip_addr = NULL;
+    GuestNetworkInterfaceList *head = NULL, *cur_item = NULL;
+    GuestIpAddressList *head_addr, *cur_addr;
+    GuestNetworkInterfaceList *info;
+    GuestIpAddressList *address_item = NULL;
+    unsigned char *mac_addr;
+    char *addr_str;
+    WORD wsa_version;
+    WSADATA wsa_data;
+    int ret;
+
+    adptr_addrs = guest_get_adapters_addresses(errp);
+    if (adptr_addrs == NULL) {
+        return NULL;
+    }
+
+    /* Make WSA APIs available. */
+    wsa_version = MAKEWORD(2, 2);
+    ret = WSAStartup(wsa_version, &wsa_data);
+    if (ret != 0) {
+        error_setg_win32(errp, ret, "failed socket startup");
+        goto out;
+    }
+
+    for (addr = adptr_addrs; addr; addr = addr->Next) {
+        info = g_malloc0(sizeof(*info));
+
+        if (cur_item == NULL) {
+            head = cur_item = info;
+        } else {
+            cur_item->next = info;
+            cur_item = info;
+        }
+
+        info->value = g_malloc0(sizeof(*info->value));
+        info->value->name = guest_wctomb_dup(addr->FriendlyName);
+
+        if (addr->PhysicalAddressLength != 0) {
+            mac_addr = addr->PhysicalAddress;
+
+            info->value->hardware_address =
+                g_strdup_printf("%02x:%02x:%02x:%02x:%02x:%02x",
+                                (int) mac_addr[0], (int) mac_addr[1],
+                                (int) mac_addr[2], (int) mac_addr[3],
+                                (int) mac_addr[4], (int) mac_addr[5]);
+
+            info->value->has_hardware_address = true;
+        }
+
+        head_addr = NULL;
+        cur_addr = NULL;
+        for (ip_addr = addr->FirstUnicastAddress;
+                ip_addr;
+                ip_addr = ip_addr->Next) {
+            addr_str = guest_addr_to_str(ip_addr, errp);
+            if (addr_str == NULL) {
+                continue;
+            }
+
+            address_item = g_malloc0(sizeof(*address_item));
+
+            if (!cur_addr) {
+                head_addr = cur_addr = address_item;
+            } else {
+                cur_addr->next = address_item;
+                cur_addr = address_item;
+            }
+
+            address_item->value = g_malloc0(sizeof(*address_item->value));
+            address_item->value->ip_address = addr_str;
+            address_item->value->prefix = guest_ip_prefix(ip_addr);
+            if (ip_addr->Address.lpSockaddr->sa_family == AF_INET) {
+                address_item->value->ip_address_type =
+                    GUEST_IP_ADDRESS_TYPE_IPV4;
+            } else if (ip_addr->Address.lpSockaddr->sa_family == AF_INET6) {
+                address_item->value->ip_address_type =
+                    GUEST_IP_ADDRESS_TYPE_IPV6;
+            }
+        }
+        if (head_addr) {
+            info->value->has_ip_addresses = true;
+            info->value->ip_addresses = head_addr;
+        }
+    }
+    WSACleanup();
+out:
+    g_free(adptr_addrs);
+    return head;
 }
 
 int64_t qmp_guest_get_time(Error **errp)
 {
     SYSTEMTIME ts = {0};
-    int64_t time_ns;
     FILETIME tf;
 
     GetSystemTime(&ts);
@@ -382,10 +1163,8 @@
         return -1;
     }
 
-    time_ns = ((((int64_t)tf.dwHighDateTime << 32) | tf.dwLowDateTime)
+    return ((((int64_t)tf.dwHighDateTime << 32) | tf.dwLowDateTime)
                 - W32_FT_OFFSET) * 100;
-
-    return time_ns;
 }
 
 void qmp_guest_set_time(bool has_time, int64_t time_ns, Error **errp)
@@ -395,31 +1174,31 @@
     FILETIME tf;
     LONGLONG time;
 
-    if (has_time) {
-        /* Okay, user passed a time to set. Validate it. */
-        if (time_ns < 0 || time_ns / 100 > INT64_MAX - W32_FT_OFFSET) {
-            error_setg(errp, "Time %" PRId64 "is invalid", time_ns);
-            return;
-        }
+    if (!has_time) {
+        /* Unfortunately, Windows libraries don't provide an easy way to access
+         * RTC yet:
+         *
+         * https://msdn.microsoft.com/en-us/library/aa908981.aspx
+         */
+        error_setg(errp, "Time argument is required on this platform");
+        return;
+    }
 
-        time = time_ns / 100 + W32_FT_OFFSET;
+    /* Validate time passed by user. */
+    if (time_ns < 0 || time_ns / 100 > INT64_MAX - W32_FT_OFFSET) {
+        error_setg(errp, "Time %" PRId64 "is invalid", time_ns);
+        return;
+    }
 
-        tf.dwLowDateTime = (DWORD) time;
-        tf.dwHighDateTime = (DWORD) (time >> 32);
+    time = time_ns / 100 + W32_FT_OFFSET;
 
-        if (!FileTimeToSystemTime(&tf, &ts)) {
-            error_setg(errp, "Failed to convert system time %d",
-                       (int)GetLastError());
-            return;
-        }
-    } else {
-        /* Otherwise read the time from RTC which contains the correct value.
-         * Hopefully. */
-        GetSystemTime(&ts);
-        if (ts.wYear < 1601 || ts.wYear > 30827) {
-            error_setg(errp, "Failed to get time");
-            return;
-        }
+    tf.dwLowDateTime = (DWORD) time;
+    tf.dwHighDateTime = (DWORD) (time >> 32);
+
+    if (!FileTimeToSystemTime(&tf, &ts)) {
+        error_setg(errp, "Failed to convert system time %d",
+                   (int)GetLastError());
+        return;
     }
 
     acquire_privilege(SE_SYSTEMTIME_NAME, &local_err);
@@ -436,40 +1215,223 @@
 
 GuestLogicalProcessorList *qmp_guest_get_vcpus(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    PSYSTEM_LOGICAL_PROCESSOR_INFORMATION pslpi, ptr;
+    DWORD length;
+    GuestLogicalProcessorList *head, **link;
+    Error *local_err = NULL;
+    int64_t current;
+
+    ptr = pslpi = NULL;
+    length = 0;
+    current = 0;
+    head = NULL;
+    link = &head;
+
+    if ((GetLogicalProcessorInformation(pslpi, &length) == FALSE) &&
+        (GetLastError() == ERROR_INSUFFICIENT_BUFFER) &&
+        (length > sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION))) {
+        ptr = pslpi = g_malloc0(length);
+        if (GetLogicalProcessorInformation(pslpi, &length) == FALSE) {
+            error_setg(&local_err, "Failed to get processor information: %d",
+                       (int)GetLastError());
+        }
+    } else {
+        error_setg(&local_err,
+                   "Failed to get processor information buffer length: %d",
+                   (int)GetLastError());
+    }
+
+    while ((local_err == NULL) && (length > 0)) {
+        if (pslpi->Relationship == RelationProcessorCore) {
+            ULONG_PTR cpu_bits = pslpi->ProcessorMask;
+
+            while (cpu_bits > 0) {
+                if (!!(cpu_bits & 1)) {
+                    GuestLogicalProcessor *vcpu;
+                    GuestLogicalProcessorList *entry;
+
+                    vcpu = g_malloc0(sizeof *vcpu);
+                    vcpu->logical_id = current++;
+                    vcpu->online = true;
+                    vcpu->has_can_offline = false;
+
+                    entry = g_malloc0(sizeof *entry);
+                    entry->value = vcpu;
+
+                    *link = entry;
+                    link = &entry->next;
+                }
+                cpu_bits >>= 1;
+            }
+        }
+        length -= sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
+        pslpi++; /* next entry */
+    }
+
+    g_free(ptr);
+
+    if (local_err == NULL) {
+        if (head != NULL) {
+            return head;
+        }
+        /* there's no guest with zero VCPUs */
+        error_setg(&local_err, "Guest reported zero VCPUs");
+    }
+
+    qapi_free_GuestLogicalProcessorList(head);
+    error_propagate(errp, local_err);
     return NULL;
 }
 
 int64_t qmp_guest_set_vcpus(GuestLogicalProcessorList *vcpus, Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return -1;
 }
 
+static gchar *
+get_net_error_message(gint error)
+{
+    HMODULE module = NULL;
+    gchar *retval = NULL;
+    wchar_t *msg = NULL;
+    int flags;
+    size_t nchars;
+
+    flags = FORMAT_MESSAGE_ALLOCATE_BUFFER |
+        FORMAT_MESSAGE_IGNORE_INSERTS |
+        FORMAT_MESSAGE_FROM_SYSTEM;
+
+    if (error >= NERR_BASE && error <= MAX_NERR) {
+        module = LoadLibraryExW(L"netmsg.dll", NULL, LOAD_LIBRARY_AS_DATAFILE);
+
+        if (module != NULL) {
+            flags |= FORMAT_MESSAGE_FROM_HMODULE;
+        }
+    }
+
+    FormatMessageW(flags, module, error, 0, (LPWSTR)&msg, 0, NULL);
+
+    if (msg != NULL) {
+        nchars = wcslen(msg);
+
+        if (nchars >= 2 &&
+            msg[nchars - 1] == L'\n' &&
+            msg[nchars - 2] == L'\r') {
+            msg[nchars - 2] = L'\0';
+        }
+
+        retval = g_utf16_to_utf8(msg, -1, NULL, NULL, NULL);
+
+        LocalFree(msg);
+    }
+
+    if (module != NULL) {
+        FreeLibrary(module);
+    }
+
+    return retval;
+}
+
+void qmp_guest_set_user_password(const char *username,
+                                 const char *password,
+                                 bool crypted,
+                                 Error **errp)
+{
+    NET_API_STATUS nas;
+    char *rawpasswddata = NULL;
+    size_t rawpasswdlen;
+    wchar_t *user = NULL, *wpass = NULL;
+    USER_INFO_1003 pi1003 = { 0, };
+    GError *gerr = NULL;
+
+    if (crypted) {
+        error_setg(errp, QERR_UNSUPPORTED);
+        return;
+    }
+
+    rawpasswddata = (char *)qbase64_decode(password, -1, &rawpasswdlen, errp);
+    if (!rawpasswddata) {
+        return;
+    }
+    rawpasswddata = g_renew(char, rawpasswddata, rawpasswdlen + 1);
+    rawpasswddata[rawpasswdlen] = '\0';
+
+    user = g_utf8_to_utf16(username, -1, NULL, NULL, &gerr);
+    if (!user) {
+        goto done;
+    }
+
+    wpass = g_utf8_to_utf16(rawpasswddata, -1, NULL, NULL, &gerr);
+    if (!wpass) {
+        goto done;
+    }
+
+    pi1003.usri1003_password = wpass;
+    nas = NetUserSetInfo(NULL, user,
+                         1003, (LPBYTE)&pi1003,
+                         NULL);
+
+    if (nas != NERR_Success) {
+        gchar *msg = get_net_error_message(nas);
+        error_setg(errp, "failed to set password: %s", msg);
+        g_free(msg);
+    }
+
+done:
+    if (gerr) {
+        error_setg(errp, QERR_QGA_COMMAND_FAILED, gerr->message);
+        g_error_free(gerr);
+    }
+    g_free(user);
+    g_free(wpass);
+    g_free(rawpasswddata);
+}
+
+GuestMemoryBlockList *qmp_guest_get_memory_blocks(Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
+GuestMemoryBlockResponseList *
+qmp_guest_set_memory_blocks(GuestMemoryBlockList *mem_blks, Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
+GuestMemoryBlockInfo *qmp_guest_get_memory_block_info(Error **errp)
+{
+    error_setg(errp, QERR_UNSUPPORTED);
+    return NULL;
+}
+
 /* add unsupported commands to the blacklist */
 GList *ga_command_blacklist_init(GList *blacklist)
 {
     const char *list_unsupported[] = {
-        "guest-file-open", "guest-file-close", "guest-file-read",
-        "guest-file-write", "guest-file-seek", "guest-file-flush",
-        "guest-suspend-hybrid", "guest-network-get-interfaces",
-        "guest-get-vcpus", "guest-set-vcpus",
-        "guest-fsfreeze-freeze-list", "guest-get-fsinfo",
+        "guest-suspend-hybrid",
+        "guest-set-vcpus",
+        "guest-get-memory-blocks", "guest-set-memory-blocks",
+        "guest-get-memory-block-size",
+        "guest-fsfreeze-freeze-list",
         "guest-fstrim", NULL};
     char **p = (char **)list_unsupported;
 
     while (*p) {
-        blacklist = g_list_append(blacklist, *p++);
+        blacklist = g_list_append(blacklist, g_strdup(*p++));
     }
 
     if (!vss_init(true)) {
+        g_debug("vss_init failed, vss commands are going to be disabled");
         const char *list[] = {
             "guest-get-fsinfo", "guest-fsfreeze-status",
             "guest-fsfreeze-freeze", "guest-fsfreeze-thaw", NULL};
         p = (char **)list;
 
         while (*p) {
-            blacklist = g_list_append(blacklist, *p++);
+            blacklist = g_list_append(blacklist, g_strdup(*p++));
         }
     }
 
diff --git a/qga/commands.c b/qga/commands.c
index 7834967..50fd26a 100644
--- a/qga/commands.c
+++ b/qga/commands.c
@@ -10,10 +10,17 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "qga/guest-agent-core.h"
 #include "qga-qmp-commands.h"
 #include "qapi/qmp/qerror.h"
+#include "qemu/base64.h"
+#include "qemu/cutils.h"
+
+/* Maximum captured guest-exec out_data/err_data - 16MB */
+#define GUEST_EXEC_MAX_OUTPUT (16*1024*1024)
+/* Allocation and I/O buffer for reading guest-exec out_data/err_data - 4KB */
+#define GUEST_EXEC_IO_SIZE (4*1024)
 
 /* Note: in some situations, like with the fsfreeze, logging may be
  * temporarilly disabled. if it is necessary that a command be able
@@ -51,12 +58,12 @@
     GuestAgentCommandInfo *cmd_info;
     GuestAgentCommandInfoList *cmd_info_list;
 
-    cmd_info = g_malloc0(sizeof(GuestAgentCommandInfo));
+    cmd_info = g_new0(GuestAgentCommandInfo, 1);
     cmd_info->name = g_strdup(qmp_command_name(cmd));
     cmd_info->enabled = qmp_command_is_enabled(cmd);
     cmd_info->success_response = qmp_has_success_response(cmd);
 
-    cmd_info_list = g_malloc0(sizeof(GuestAgentCommandInfoList));
+    cmd_info_list = g_new0(GuestAgentCommandInfoList, 1);
     cmd_info_list->value = cmd_info;
     cmd_info_list->next = info->supported_commands;
     info->supported_commands = cmd_info_list;
@@ -64,9 +71,430 @@
 
 struct GuestAgentInfo *qmp_guest_info(Error **errp)
 {
-    GuestAgentInfo *info = g_malloc0(sizeof(GuestAgentInfo));
+    GuestAgentInfo *info = g_new0(GuestAgentInfo, 1);
 
     info->version = g_strdup(QEMU_VERSION);
     qmp_for_each_command(qmp_command_info, info);
     return info;
 }
+
+struct GuestExecIOData {
+    guchar *data;
+    gsize size;
+    gsize length;
+    gint closed;
+    bool truncated;
+    const char *name;
+};
+typedef struct GuestExecIOData GuestExecIOData;
+
+struct GuestExecInfo {
+    GPid pid;
+    int64_t pid_numeric;
+    gint status;
+    bool has_output;
+    gint finished;
+    GuestExecIOData in;
+    GuestExecIOData out;
+    GuestExecIOData err;
+    QTAILQ_ENTRY(GuestExecInfo) next;
+};
+typedef struct GuestExecInfo GuestExecInfo;
+
+static struct {
+    QTAILQ_HEAD(, GuestExecInfo) processes;
+} guest_exec_state = {
+    .processes = QTAILQ_HEAD_INITIALIZER(guest_exec_state.processes),
+};
+
+static int64_t gpid_to_int64(GPid pid)
+{
+#ifdef G_OS_WIN32
+    return GetProcessId(pid);
+#else
+    return (int64_t)pid;
+#endif
+}
+
+static GuestExecInfo *guest_exec_info_add(GPid pid)
+{
+    GuestExecInfo *gei;
+
+    gei = g_new0(GuestExecInfo, 1);
+    gei->pid = pid;
+    gei->pid_numeric = gpid_to_int64(pid);
+    QTAILQ_INSERT_TAIL(&guest_exec_state.processes, gei, next);
+
+    return gei;
+}
+
+static GuestExecInfo *guest_exec_info_find(int64_t pid_numeric)
+{
+    GuestExecInfo *gei;
+
+    QTAILQ_FOREACH(gei, &guest_exec_state.processes, next) {
+        if (gei->pid_numeric == pid_numeric) {
+            return gei;
+        }
+    }
+
+    return NULL;
+}
+
+GuestExecStatus *qmp_guest_exec_status(int64_t pid, Error **err)
+{
+    GuestExecInfo *gei;
+    GuestExecStatus *ges;
+
+    slog("guest-exec-status called, pid: %u", (uint32_t)pid);
+
+    gei = guest_exec_info_find(pid);
+    if (gei == NULL) {
+        error_setg(err, QERR_INVALID_PARAMETER, "pid");
+        return NULL;
+    }
+
+    ges = g_new0(GuestExecStatus, 1);
+
+    bool finished = g_atomic_int_get(&gei->finished);
+
+    /* need to wait till output channels are closed
+     * to be sure we captured all output at this point */
+    if (gei->has_output) {
+        finished = finished && g_atomic_int_get(&gei->out.closed);
+        finished = finished && g_atomic_int_get(&gei->err.closed);
+    }
+
+    ges->exited = finished;
+    if (finished) {
+        /* Glib has no portable way to parse exit status.
+         * On UNIX, we can get either exit code from normal termination
+         * or signal number.
+         * On Windows, it is either the same exit code or the exception
+         * value for an unhandled exception that caused the process
+         * to terminate.
+         * See MSDN for GetExitCodeProcess() and ntstatus.h for possible
+         * well-known codes, e.g. C0000005 ACCESS_DENIED - analog of SIGSEGV
+         * References:
+         *   https://msdn.microsoft.com/en-us/library/windows/desktop/ms683189(v=vs.85).aspx
+         *   https://msdn.microsoft.com/en-us/library/aa260331(v=vs.60).aspx
+         */
+#ifdef G_OS_WIN32
+        /* Additionally WIN32 does not provide any additional information
+         * on whether the child exited or terminated via signal.
+         * We use this simple range check to distinguish application exit code
+         * (usually value less then 256) and unhandled exception code with
+         * ntstatus (always value greater then 0xC0000005). */
+        if ((uint32_t)gei->status < 0xC0000000U) {
+            ges->has_exitcode = true;
+            ges->exitcode = gei->status;
+        } else {
+            ges->has_signal = true;
+            ges->signal = gei->status;
+        }
+#else
+        if (WIFEXITED(gei->status)) {
+            ges->has_exitcode = true;
+            ges->exitcode = WEXITSTATUS(gei->status);
+        } else if (WIFSIGNALED(gei->status)) {
+            ges->has_signal = true;
+            ges->signal = WTERMSIG(gei->status);
+        }
+#endif
+        if (gei->out.length > 0) {
+            ges->has_out_data = true;
+            ges->out_data = g_base64_encode(gei->out.data, gei->out.length);
+            g_free(gei->out.data);
+            ges->has_out_truncated = gei->out.truncated;
+        }
+
+        if (gei->err.length > 0) {
+            ges->has_err_data = true;
+            ges->err_data = g_base64_encode(gei->err.data, gei->err.length);
+            g_free(gei->err.data);
+            ges->has_err_truncated = gei->err.truncated;
+        }
+
+        QTAILQ_REMOVE(&guest_exec_state.processes, gei, next);
+        g_free(gei);
+    }
+
+    return ges;
+}
+
+/* Get environment variables or arguments array for execve(). */
+static char **guest_exec_get_args(const strList *entry, bool log)
+{
+    const strList *it;
+    int count = 1, i = 0;  /* reserve for NULL terminator */
+    char **args;
+    char *str; /* for logging array of arguments */
+    size_t str_size = 1;
+
+    for (it = entry; it != NULL; it = it->next) {
+        count++;
+        str_size += 1 + strlen(it->value);
+    }
+
+    str = g_malloc(str_size);
+    *str = 0;
+    args = g_malloc(count * sizeof(char *));
+    for (it = entry; it != NULL; it = it->next) {
+        args[i++] = it->value;
+        pstrcat(str, str_size, it->value);
+        if (it->next) {
+            pstrcat(str, str_size, " ");
+        }
+    }
+    args[i] = NULL;
+
+    if (log) {
+        slog("guest-exec called: \"%s\"", str);
+    }
+    g_free(str);
+
+    return args;
+}
+
+static void guest_exec_child_watch(GPid pid, gint status, gpointer data)
+{
+    GuestExecInfo *gei = (GuestExecInfo *)data;
+
+    g_debug("guest_exec_child_watch called, pid: %d, status: %u",
+            (int32_t)gpid_to_int64(pid), (uint32_t)status);
+
+    gei->status = status;
+    gei->finished = true;
+
+    g_spawn_close_pid(pid);
+}
+
+/** Reset ignored signals back to default. */
+static void guest_exec_task_setup(gpointer data)
+{
+#if !defined(G_OS_WIN32)
+    struct sigaction sigact;
+
+    memset(&sigact, 0, sizeof(struct sigaction));
+    sigact.sa_handler = SIG_DFL;
+
+    if (sigaction(SIGPIPE, &sigact, NULL) != 0) {
+        slog("sigaction() failed to reset child process's SIGPIPE: %s",
+             strerror(errno));
+    }
+#endif
+}
+
+static gboolean guest_exec_input_watch(GIOChannel *ch,
+        GIOCondition cond, gpointer p_)
+{
+    GuestExecIOData *p = (GuestExecIOData *)p_;
+    gsize bytes_written = 0;
+    GIOStatus status;
+    GError *gerr = NULL;
+
+    /* nothing left to write */
+    if (p->size == p->length) {
+        goto done;
+    }
+
+    status = g_io_channel_write_chars(ch, (gchar *)p->data + p->length,
+            p->size - p->length, &bytes_written, &gerr);
+
+    /* can be not 0 even if not G_IO_STATUS_NORMAL */
+    if (bytes_written != 0) {
+        p->length += bytes_written;
+    }
+
+    /* continue write, our callback will be called again */
+    if (status == G_IO_STATUS_NORMAL || status == G_IO_STATUS_AGAIN) {
+        return true;
+    }
+
+    if (gerr) {
+        g_warning("qga: i/o error writing to input_data channel: %s",
+                gerr->message);
+        g_error_free(gerr);
+    }
+
+done:
+    g_io_channel_shutdown(ch, true, NULL);
+    g_io_channel_unref(ch);
+    g_atomic_int_set(&p->closed, 1);
+    g_free(p->data);
+
+    return false;
+}
+
+static gboolean guest_exec_output_watch(GIOChannel *ch,
+        GIOCondition cond, gpointer p_)
+{
+    GuestExecIOData *p = (GuestExecIOData *)p_;
+    gsize bytes_read;
+    GIOStatus gstatus;
+
+    if (cond == G_IO_HUP || cond == G_IO_ERR) {
+        goto close;
+    }
+
+    if (p->size == p->length) {
+        gpointer t = NULL;
+        if (!p->truncated && p->size < GUEST_EXEC_MAX_OUTPUT) {
+            t = g_try_realloc(p->data, p->size + GUEST_EXEC_IO_SIZE);
+        }
+        if (t == NULL) {
+            /* ignore truncated output */
+            gchar buf[GUEST_EXEC_IO_SIZE];
+
+            p->truncated = true;
+            gstatus = g_io_channel_read_chars(ch, buf, sizeof(buf),
+                                              &bytes_read, NULL);
+            if (gstatus == G_IO_STATUS_EOF || gstatus == G_IO_STATUS_ERROR) {
+                goto close;
+            }
+
+            return true;
+        }
+        p->size += GUEST_EXEC_IO_SIZE;
+        p->data = t;
+    }
+
+    /* Calling read API once.
+     * On next available data our callback will be called again */
+    gstatus = g_io_channel_read_chars(ch, (gchar *)p->data + p->length,
+            p->size - p->length, &bytes_read, NULL);
+    if (gstatus == G_IO_STATUS_EOF || gstatus == G_IO_STATUS_ERROR) {
+        goto close;
+    }
+
+    p->length += bytes_read;
+
+    return true;
+
+close:
+    g_io_channel_shutdown(ch, true, NULL);
+    g_io_channel_unref(ch);
+    g_atomic_int_set(&p->closed, 1);
+    return false;
+}
+
+GuestExec *qmp_guest_exec(const char *path,
+                       bool has_arg, strList *arg,
+                       bool has_env, strList *env,
+                       bool has_input_data, const char *input_data,
+                       bool has_capture_output, bool capture_output,
+                       Error **err)
+{
+    GPid pid;
+    GuestExec *ge = NULL;
+    GuestExecInfo *gei;
+    char **argv, **envp;
+    strList arglist;
+    gboolean ret;
+    GError *gerr = NULL;
+    gint in_fd, out_fd, err_fd;
+    GIOChannel *in_ch, *out_ch, *err_ch;
+    GSpawnFlags flags;
+    bool has_output = (has_capture_output && capture_output);
+    uint8_t *input = NULL;
+    size_t ninput = 0;
+
+    arglist.value = (char *)path;
+    arglist.next = has_arg ? arg : NULL;
+
+    if (has_input_data) {
+        input = qbase64_decode(input_data, -1, &ninput, err);
+        if (!input) {
+            return NULL;
+        }
+    }
+
+    argv = guest_exec_get_args(&arglist, true);
+    envp = has_env ? guest_exec_get_args(env, false) : NULL;
+
+    flags = G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD;
+#if GLIB_CHECK_VERSION(2, 33, 2)
+    flags |= G_SPAWN_SEARCH_PATH_FROM_ENVP;
+#endif
+    if (!has_output) {
+        flags |= G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL;
+    }
+
+    ret = g_spawn_async_with_pipes(NULL, argv, envp, flags,
+            guest_exec_task_setup, NULL, &pid, has_input_data ? &in_fd : NULL,
+            has_output ? &out_fd : NULL, has_output ? &err_fd : NULL, &gerr);
+    if (!ret) {
+        error_setg(err, QERR_QGA_COMMAND_FAILED, gerr->message);
+        g_error_free(gerr);
+        goto done;
+    }
+
+    ge = g_new0(GuestExec, 1);
+    ge->pid = gpid_to_int64(pid);
+
+    gei = guest_exec_info_add(pid);
+    gei->has_output = has_output;
+    g_child_watch_add(pid, guest_exec_child_watch, gei);
+
+    if (has_input_data) {
+        gei->in.data = input;
+        gei->in.size = ninput;
+#ifdef G_OS_WIN32
+        in_ch = g_io_channel_win32_new_fd(in_fd);
+#else
+        in_ch = g_io_channel_unix_new(in_fd);
+#endif
+        g_io_channel_set_encoding(in_ch, NULL, NULL);
+        g_io_channel_set_buffered(in_ch, false);
+        g_io_channel_set_flags(in_ch, G_IO_FLAG_NONBLOCK, NULL);
+        g_io_channel_set_close_on_unref(in_ch, true);
+        g_io_add_watch(in_ch, G_IO_OUT, guest_exec_input_watch, &gei->in);
+    }
+
+    if (has_output) {
+#ifdef G_OS_WIN32
+        out_ch = g_io_channel_win32_new_fd(out_fd);
+        err_ch = g_io_channel_win32_new_fd(err_fd);
+#else
+        out_ch = g_io_channel_unix_new(out_fd);
+        err_ch = g_io_channel_unix_new(err_fd);
+#endif
+        g_io_channel_set_encoding(out_ch, NULL, NULL);
+        g_io_channel_set_encoding(err_ch, NULL, NULL);
+        g_io_channel_set_buffered(out_ch, false);
+        g_io_channel_set_buffered(err_ch, false);
+        g_io_channel_set_close_on_unref(out_ch, true);
+        g_io_channel_set_close_on_unref(err_ch, true);
+        g_io_add_watch(out_ch, G_IO_IN | G_IO_HUP,
+                guest_exec_output_watch, &gei->out);
+        g_io_add_watch(err_ch, G_IO_IN | G_IO_HUP,
+                guest_exec_output_watch, &gei->err);
+    }
+
+done:
+    g_free(argv);
+    g_free(envp);
+
+    return ge;
+}
+
+/* Convert GuestFileWhence (either a raw integer or an enum value) into
+ * the guest's SEEK_ constants.  */
+int ga_parse_whence(GuestFileWhence *whence, Error **errp)
+{
+    /* Exploit the fact that we picked values to match QGA_SEEK_*. */
+    if (whence->type == QTYPE_QSTRING) {
+        whence->type = QTYPE_QINT;
+        whence->u.value = whence->u.name;
+    }
+    switch (whence->u.value) {
+    case QGA_SEEK_SET:
+        return SEEK_SET;
+    case QGA_SEEK_CUR:
+        return SEEK_CUR;
+    case QGA_SEEK_END:
+        return SEEK_END;
+    }
+    error_setg(errp, "invalid whence code %"PRId64, whence->u.value);
+    return -1;
+}
diff --git a/qga/guest-agent-command-state.c b/qga/guest-agent-command-state.c
index 969da23..4de229c 100644
--- a/qga/guest-agent-command-state.c
+++ b/qga/guest-agent-command-state.c
@@ -9,7 +9,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "qga/guest-agent-core.h"
 
 struct GACommandState {
@@ -27,7 +27,7 @@
                           void (*init)(void),
                           void (*cleanup)(void))
 {
-    GACommandGroup *cg = g_malloc0(sizeof(GACommandGroup));
+    GACommandGroup *cg = g_new0(GACommandGroup, 1);
     cg->init = init;
     cg->cleanup = cleanup;
     cs->groups = g_slist_append(cs->groups, cg);
@@ -67,7 +67,7 @@
 
 GACommandState *ga_command_state_new(void)
 {
-    GACommandState *cs = g_malloc0(sizeof(GACommandState));
+    GACommandState *cs = g_new0(GACommandState, 1);
     cs->groups = NULL;
     return cs;
 }
diff --git a/qga/guest-agent-core.h b/qga/guest-agent-core.h
index e92c6ab..0a49516 100644
--- a/qga/guest-agent-core.h
+++ b/qga/guest-agent-core.h
@@ -12,6 +12,7 @@
  */
 #include "qapi/qmp/dispatch.h"
 #include "qemu-common.h"
+#include "qga-qmp-commands.h"
 
 #define QGA_READ_COUNT_DEFAULT 4096
 
@@ -37,6 +38,7 @@
 void ga_unset_frozen(GAState *s);
 const char *ga_fsfreeze_hook(GAState *s);
 int64_t ga_get_fd_handle(GAState *s, Error **errp);
+int ga_parse_whence(GuestFileWhence *whence, Error **errp);
 
 #ifndef _WIN32
 void reopen_fd_to_null(int fd);
diff --git a/qga/installer/qemu-ga.wxs b/qga/installer/qemu-ga.wxs
new file mode 100644
index 0000000..fa2260c
--- /dev/null
+++ b/qga/installer/qemu-ga.wxs
@@ -0,0 +1,185 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+  <?ifndef env.QEMU_GA_VERSION ?>
+    <?error Environment variable QEMU_GA_VERSION undefined?>
+  <?endif?>
+
+  <?ifndef env.QEMU_GA_DISTRO ?>
+    <?error Environment variable QEMU_GA_DISTRO undefined?>
+  <?endif?>
+
+  <?ifndef env.QEMU_GA_MANUFACTURER ?>
+    <?error Environment variable QEMU_GA_MANUFACTURER undefined?>
+  <?endif?>
+
+  <?ifndef var.Arch?>
+    <?error Define Arch to 32 or 64?>
+  <?endif?>
+
+  <?ifndef var.Mingw_bin?>
+    <?if $(var.Arch) = "64"?>
+      <?define Mingw_bin=/usr/x86_64-w64-mingw32/sys-root/mingw/bin ?>
+    <?endif?>
+    <?if $(var.Arch) = "32"?>
+      <?define Mingw_bin=/usr/i686-w64-mingw32/sys-root/mingw/bin ?>
+    <?endif?>
+  <?endif?>
+
+  <?if $(var.Arch) = "64"?>
+    <?define ArchLib=libgcc_s_seh-1.dll?>
+    <?define GaProgramFilesFolder="ProgramFiles64Folder" ?>
+  <?endif?>
+
+  <?if $(var.Arch) = "32"?>
+    <?define ArchLib=libgcc_s_sjlj-1.dll?>
+    <?define GaProgramFilesFolder="ProgramFilesFolder" ?>
+  <?endif?>
+
+  <?ifndef var.ArchLib ?>
+    <?error Unexpected Arch value $(var.Arch)?>
+  <?endif?>
+
+  <Product
+    Name="QEMU guest agent"
+    Id="{DF9974AD-E41A-4304-81AD-69AA8F299766}"
+    UpgradeCode="{EB6B8302-C06E-4BEC-ADAC-932C68A3A98D}"
+    Manufacturer="$(env.QEMU_GA_MANUFACTURER)"
+    Version="$(env.QEMU_GA_VERSION)"
+    Language="1033">
+    <?if $(var.Arch) = 32 ?>
+    <Condition Message="Error: 32-bit version of Qemu GA can not be installed on 64-bit Windows.">NOT VersionNT64</Condition>
+    <?endif?>
+    <Package
+      Manufacturer="$(env.QEMU_GA_MANUFACTURER)"
+      InstallerVersion="200"
+      Languages="1033"
+      Compressed="yes"
+      InstallScope="perMachine"
+      />
+    <Media Id="1" Cabinet="qemu_ga.$(env.QEMU_GA_VERSION).cab" EmbedCab="yes" />
+    <Property Id="WHSLogo">1</Property>
+    <MajorUpgrade
+      DowngradeErrorMessage="Error: A newer version of QEMU guest agent is already installed."
+      />
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+      <Directory Id="$(var.GaProgramFilesFolder)" Name="QEMU Guest Agent">
+        <Directory Id="qemu_ga_directory" Name="Qemu-ga">
+          <Component Id="qemu_ga" Guid="{908B7199-DE2A-4DC6-A8D0-27A5AE444FEA}">
+            <File Id="qemu_ga.exe" Name="qemu-ga.exe" Source="$(env.BUILD_DIR)/qemu-ga.exe" KeyPath="yes" DiskId="1"/>
+            <ServiceInstall
+              Id="ServiceInstaller"
+              Type="ownProcess"
+              Vital="yes"
+              Name="QEMU-GA"
+              DisplayName="QEMU Guest Agent"
+              Description="QEMU Guest Agent"
+              Start="auto"
+              Account="LocalSystem"
+              ErrorControl="ignore"
+              Interactive="no"
+              Arguments="-d"
+              >
+            </ServiceInstall>
+            <ServiceControl Id="StartService" Start="install" Stop="both" Remove="uninstall" Name="QEMU-GA" Wait="no" />
+          </Component>
+          <?ifdef var.InstallVss?>
+          <Component Id="qga_vss_dll" Guid="{CB19C453-FABB-4BB1-ABAB-6B74F687BFBB}">
+            <File Id="qga_vss.dll" Name="qga-vss.dll" Source="$(env.BUILD_DIR)/qga/vss-win32/qga-vss.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="qga_vss_tlb" Guid="{D8D584B1-59C2-4FB7-A91F-636FF7BFA66E}">
+            <File Id="qga_vss.tlb" Name="qga-vss.tlb" Source="$(env.BUILD_DIR)/qga/vss-win32/qga-vss.tlb" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <?if $(var.Arch) = "32"?>
+          <Component Id="gspawn-helper-console" Guid="{446185B3-87BE-43D2-96B8-0FEFD9E8696D}">
+            <File Id="gspawn-win32-helper-console.exe" Name="gspawn-win32-helper-console.exe" Source="$(var.Mingw_bin)/gspawn-win32-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="gspawn-helper" Guid="{CD67A5A3-2DB1-4DA1-A67A-8D71E797B466}">
+            <File Id="gspawn-win32-helper.exe" Name="gspawn-win32-helper.exe" Source="$(var.Mingw_bin)/gspawn-win32-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <?if $(var.Arch) = "64"?>
+          <Component Id="gspawn-helper-console" Guid="{9E615A9F-349A-4992-A5C2-C10BAD173660}">
+            <File Id="gspawn-win64-helper-console.exe" Name="gspawn-win64-helper-console.exe" Source="$(var.Mingw_bin)/gspawn-win64-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="gspawn-helper" Guid="{D201AD22-1846-4E4F-B6E1-C7A908ED2457}">
+            <File Id="gspawn-win64-helper.exe" Name="gspawn-win64-helper.exe" Source="$(var.Mingw_bin)/gspawn-win64-helper-console.exe" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <?endif?>
+          <Component Id="iconv" Guid="{35EE3558-D34B-4F0A-B8BD-430FF0775246}">
+            <File Id="iconv.dll" Name="iconv.dll" Source="$(var.Mingw_bin)/iconv.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libgcc_arch_lib" Guid="{ADD4D07D-4515-4AB6-AF3E-C904961B4BB0}">
+            <File Id="libgcc_arch_lib" Name="$(var.ArchLib)" Source="$(var.Mingw_bin)/$(var.ArchLib)" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libglib" Guid="{D31BFD83-2773-4B65-B45A-E0D2ADA58679}">
+            <File Id="libglib_2.0_0.dll" Name="libglib-2.0-0.dll" Source="$(var.Mingw_bin)/libglib-2.0-0.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libintl" Guid="{A641BC2D-A907-4A94-9149-F30ED430878F}">
+            <File Id="libintl_8.dll" Name="libintl-8.dll" Source="$(var.Mingw_bin)/libintl-8.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libssp" Guid="{7880087B-02B4-4EF6-A5D3-D18F8E3D90E1}">
+            <File Id="libssp_0.dll" Name="libssp-0.dll" Source="$(var.Mingw_bin)/libssp-0.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="libwinpthread" Guid="{6C117C78-0F47-4B07-8F34-6BEE11643829}">
+            <File Id="libwinpthread_1.dll" Name="libwinpthread-1.dll" Source="$(var.Mingw_bin)/libwinpthread-1.dll" KeyPath="yes" DiskId="1"/>
+          </Component>
+          <Component Id="registry_entries" Guid="{D075D109-51CA-11E3-9F8B-000C29858960}">
+            <RegistryKey Root="HKLM"
+                         Key="Software\$(env.QEMU_GA_MANUFACTURER)\$(env.QEMU_GA_DISTRO)\Tools\QemuGA">
+              <RegistryValue Type="string" Name="ProductID" Value="fb0a0d66-c7fb-4e2e-a16b-c4a3bfe8d13b" />
+              <RegistryValue Type="string" Name="Version" Value="$(env.QEMU_GA_VERSION)" />
+            </RegistryKey>
+          </Component>
+        </Directory>
+      </Directory>
+    </Directory>
+
+    <Property Id="cmd" Value="cmd.exe"/>
+    <Property Id="REINSTALLMODE" Value="amus"/>
+
+    <?ifdef var.InstallVss?>
+    <CustomAction Id="RegisterCom"
+              ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-install'
+              Execute="deferred"
+              Property="cmd"
+              Impersonate="no"
+              Return="check"
+              >
+    </CustomAction>
+    <CustomAction Id="UnRegisterCom"
+              ExeCommand='/c "[qemu_ga_directory]qemu-ga.exe" -s vss-uninstall'
+              Execute="deferred"
+              Property="cmd"
+              Impersonate="no"
+              Return="check"
+              >
+    </CustomAction>
+    <?endif?>
+
+    <Feature Id="QEMUFeature" Title="QEMU Guest Agent" Level="1">
+      <ComponentRef Id="qemu_ga" />
+      <?ifdef var.InstallVss?>
+      <ComponentRef Id="qga_vss_dll" />
+      <ComponentRef Id="qga_vss_tlb" />
+      <?endif?>
+      <ComponentRef Id="gspawn-helper-console" />
+      <ComponentRef Id="gspawn-helper" />
+      <ComponentRef Id="iconv" />
+      <ComponentRef Id="libgcc_arch_lib" />
+      <ComponentRef Id="libglib" />
+      <ComponentRef Id="libintl" />
+      <ComponentRef Id="libssp" />
+      <ComponentRef Id="libwinpthread" />
+      <ComponentRef Id="registry_entries" />
+    </Feature>
+
+    <InstallExecuteSequence>
+      <?ifdef var.InstallVss?>
+      <Custom Action="UnRegisterCom" After="StopServices">Installed</Custom>
+      <Custom Action="RegisterCom" After="InstallServices">NOT REMOVE</Custom>
+      <?endif?>
+    </InstallExecuteSequence>
+  </Product>
+</Wix>
diff --git a/qga/main.c b/qga/main.c
index 9939a2b..cf7c168 100644
--- a/qga/main.c
+++ b/qga/main.c
@@ -10,16 +10,12 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdbool.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include <getopt.h>
 #include <glib/gstdio.h>
 #ifndef _WIN32
 #include <syslog.h>
 #include <sys/wait.h>
-#include <sys/stat.h>
 #endif
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/json-parser.h"
@@ -27,11 +23,11 @@
 #include "qapi/qmp/qjson.h"
 #include "qga/guest-agent-core.h"
 #include "qemu/module.h"
-#include "signal.h"
 #include "qapi/qmp/qerror.h"
 #include "qapi/qmp/dispatch.h"
 #include "qga/channel.h"
 #include "qemu/bswap.h"
+#include "qemu/help_option.h"
 #ifdef _WIN32
 #include "qga/service-win32.h"
 #include "qga/vss-win32.h"
@@ -56,6 +52,7 @@
 #define QGA_FSFREEZE_HOOK_DEFAULT CONFIG_QEMU_CONFDIR "/fsfreeze-hook"
 #endif
 #define QGA_SENTINEL_BYTE 0xFF
+#define QGA_CONF_DEFAULT CONFIG_QEMU_CONFDIR G_DIR_SEPARATOR_S "qemu-ga.conf"
 
 static struct {
     const char *state_dir;
@@ -82,7 +79,7 @@
     bool delimit_response;
     bool frozen;
     GList *blacklist;
-    const char *state_filepath_isfrozen;
+    char *state_filepath_isfrozen;
     struct {
         const char *log_filepath;
         const char *pid_filepath;
@@ -90,7 +87,7 @@
 #ifdef CONFIG_FSFREEZE
     const char *fsfreeze_hook;
 #endif
-    const gchar *pstate_filepath;
+    gchar *pstate_filepath;
     GAPersistentState pstate;
 };
 
@@ -160,6 +157,12 @@
         g_error("error configuring signal handler: %s", strerror(errno));
     }
 
+    sigact.sa_handler = SIG_IGN;
+    if (sigaction(SIGPIPE, &sigact, NULL) != 0) {
+        g_error("error configuring SIGPIPE signal handler: %s",
+                strerror(errno));
+    }
+
     return true;
 }
 
@@ -211,10 +214,12 @@
 "  -V, --version     print version information and exit\n"
 "  -d, --daemonize   become a daemon\n"
 #ifdef _WIN32
-"  -s, --service     service commands: install, uninstall\n"
+"  -s, --service     service commands: install, uninstall, vss-install, vss-uninstall\n"
 #endif
 "  -b, --blacklist   comma-separated list of RPCs to disable (no spaces, \"?\"\n"
 "                    to list available RPCs)\n"
+"  -D, --dump-conf   dump a qemu-ga config file based on current config\n"
+"                    options / command-line parameters to stdout\n"
 "  -h, --help        display this help and exit\n"
 "\n"
 "Report bugs to <mdroth@linux.vnet.ibm.com>\n"
@@ -274,7 +279,7 @@
 
     level &= G_LOG_LEVEL_MASK;
 #ifndef _WIN32
-    if (domain && strcmp(domain, "syslog") == 0) {
+    if (g_strcmp0(domain, "syslog") == 0) {
         syslog(LOG_INFO, "%s: %s", level_str, msg);
     } else if (level & s->log_level) {
 #else
@@ -561,10 +566,9 @@
 }
 
 /* handle requests/control events coming in over the channel */
-static void process_event(JSONMessageParser *parser, QList *tokens)
+static void process_event(JSONMessageParser *parser, GQueue *tokens)
 {
     GAState *s = container_of(parser, GAState, parser);
-    QObject *obj;
     QDict *qdict;
     Error *err = NULL;
     int ret;
@@ -572,24 +576,20 @@
     g_assert(s && parser);
 
     g_debug("process_event: called");
-    obj = json_parser_parse_err(tokens, NULL, &err);
-    if (err || !obj || qobject_type(obj) != QTYPE_QDICT) {
-        qobject_decref(obj);
+    qdict = qobject_to_qdict(json_parser_parse_err(tokens, NULL, &err));
+    if (err || !qdict) {
+        QDECREF(qdict);
         qdict = qdict_new();
         if (!err) {
             g_warning("failed to parse event: unknown error");
-            error_set(&err, QERR_JSON_PARSING);
+            error_setg(&err, QERR_JSON_PARSING);
         } else {
             g_warning("failed to parse event: %s", error_get_pretty(err));
         }
         qdict_put_obj(qdict, "error", qmp_build_error_object(err));
         error_free(err);
-    } else {
-        qdict = qobject_to_qdict(obj);
     }
 
-    g_assert(qdict);
-
     /* handle host->guest commands */
     if (qdict_haskey(qdict, "execute")) {
         process_command(s, qdict);
@@ -598,7 +598,7 @@
             QDECREF(qdict);
             qdict = qdict_new();
             g_warning("unrecognized payload format");
-            error_set(&err, QERR_UNSUPPORTED);
+            error_setg(&err, QERR_UNSUPPORTED);
             qdict_put_obj(qdict, "error", qmp_build_error_object(err));
             error_free(err);
         }
@@ -617,13 +617,7 @@
     GAState *s = data;
     gchar buf[QGA_READ_COUNT_DEFAULT+1];
     gsize count;
-    GError *err = NULL;
     GIOStatus status = ga_channel_read(s->channel, buf, QGA_READ_COUNT_DEFAULT, &count);
-    if (err != NULL) {
-        g_warning("error reading channel: %s", err->message);
-        g_error_free(err);
-        return false;
-    }
     switch (status) {
     case G_IO_STATUS_ERROR:
         g_warning("error reading channel");
@@ -658,23 +652,6 @@
 {
     GAChannelMethod channel_method;
 
-    if (method == NULL) {
-        method = "virtio-serial";
-    }
-
-    if (path == NULL) {
-        if (strcmp(method, "virtio-serial") == 0 ) {
-            /* try the default path for the virtio-serial port */
-            path = QGA_VIRTIO_PATH_DEFAULT;
-        } else if (strcmp(method, "isa-serial") == 0){
-            /* try the default path for the serial port - COM1 */
-            path = QGA_SERIAL_PATH_DEFAULT;
-        } else {
-            g_critical("must specify a path for this channel");
-            return false;
-        }
-    }
-
     if (strcmp(method, "virtio-serial") == 0) {
         s->virtio = true; /* virtio requires special handling in some cases */
         channel_method = GA_CHANNEL_VIRTIO_SERIAL;
@@ -829,7 +806,7 @@
 
     g_assert(pstate);
 
-    if (stat(path, &st) == -1) {
+    if (qemu_stat(path, &st) == -1) {
         /* it's okay if state file doesn't exist, but any other error
          * indicates a permissions issue or some other misconfiguration
          * that we likely won't be able to recover from.
@@ -921,22 +898,165 @@
     printf("%s\n", qmp_command_name(cmd));
 }
 
-int main(int argc, char **argv)
+static GList *split_list(const gchar *str, const gchar *delim)
 {
-    const char *sopt = "hVvdm:p:l:f:F::b:s:t:";
-    const char *method = NULL, *path = NULL;
-    const char *log_filepath = NULL;
-    const char *pid_filepath;
+    GList *list = NULL;
+    int i;
+    gchar **strv;
+
+    strv = g_strsplit(str, delim, -1);
+    for (i = 0; strv[i]; i++) {
+        list = g_list_prepend(list, strv[i]);
+    }
+    g_free(strv);
+
+    return list;
+}
+
+typedef struct GAConfig {
+    char *channel_path;
+    char *method;
+    char *log_filepath;
+    char *pid_filepath;
 #ifdef CONFIG_FSFREEZE
-    const char *fsfreeze_hook = NULL;
+    char *fsfreeze_hook;
 #endif
-    const char *state_dir;
+    char *state_dir;
 #ifdef _WIN32
-    const char *service = NULL;
+    const char *service;
 #endif
+    gchar *bliststr; /* blacklist may point to this string */
+    GList *blacklist;
+    int daemonize;
+    GLogLevelFlags log_level;
+    int dumpconf;
+} GAConfig;
+
+static void config_load(GAConfig *config)
+{
+    GError *gerr = NULL;
+    GKeyFile *keyfile;
+    const char *conf = g_getenv("QGA_CONF") ?: QGA_CONF_DEFAULT;
+
+    /* read system config */
+    keyfile = g_key_file_new();
+    if (!g_key_file_load_from_file(keyfile, conf, 0, &gerr)) {
+        goto end;
+    }
+    if (g_key_file_has_key(keyfile, "general", "daemon", NULL)) {
+        config->daemonize =
+            g_key_file_get_boolean(keyfile, "general", "daemon", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "method", NULL)) {
+        config->method =
+            g_key_file_get_string(keyfile, "general", "method", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "path", NULL)) {
+        config->channel_path =
+            g_key_file_get_string(keyfile, "general", "path", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "logfile", NULL)) {
+        config->log_filepath =
+            g_key_file_get_string(keyfile, "general", "logfile", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "pidfile", NULL)) {
+        config->pid_filepath =
+            g_key_file_get_string(keyfile, "general", "pidfile", &gerr);
+    }
+#ifdef CONFIG_FSFREEZE
+    if (g_key_file_has_key(keyfile, "general", "fsfreeze-hook", NULL)) {
+        config->fsfreeze_hook =
+            g_key_file_get_string(keyfile,
+                                  "general", "fsfreeze-hook", &gerr);
+    }
+#endif
+    if (g_key_file_has_key(keyfile, "general", "statedir", NULL)) {
+        config->state_dir =
+            g_key_file_get_string(keyfile, "general", "statedir", &gerr);
+    }
+    if (g_key_file_has_key(keyfile, "general", "verbose", NULL) &&
+        g_key_file_get_boolean(keyfile, "general", "verbose", &gerr)) {
+        /* enable all log levels */
+        config->log_level = G_LOG_LEVEL_MASK;
+    }
+    if (g_key_file_has_key(keyfile, "general", "blacklist", NULL)) {
+        config->bliststr =
+            g_key_file_get_string(keyfile, "general", "blacklist", &gerr);
+        config->blacklist = g_list_concat(config->blacklist,
+                                          split_list(config->bliststr, ","));
+    }
+
+end:
+    g_key_file_free(keyfile);
+    if (gerr &&
+        !(gerr->domain == G_FILE_ERROR && gerr->code == G_FILE_ERROR_NOENT)) {
+        g_critical("error loading configuration from path: %s, %s",
+                   QGA_CONF_DEFAULT, gerr->message);
+        exit(EXIT_FAILURE);
+    }
+    g_clear_error(&gerr);
+}
+
+static gchar *list_join(GList *list, const gchar separator)
+{
+    GString *str = g_string_new("");
+
+    while (list) {
+        str = g_string_append(str, (gchar *)list->data);
+        list = g_list_next(list);
+        if (list) {
+            str = g_string_append_c(str, separator);
+        }
+    }
+
+    return g_string_free(str, FALSE);
+}
+
+static void config_dump(GAConfig *config)
+{
+    GError *error = NULL;
+    GKeyFile *keyfile;
+    gchar *tmp;
+
+    keyfile = g_key_file_new();
+    g_assert(keyfile);
+
+    g_key_file_set_boolean(keyfile, "general", "daemon", config->daemonize);
+    g_key_file_set_string(keyfile, "general", "method", config->method);
+    g_key_file_set_string(keyfile, "general", "path", config->channel_path);
+    if (config->log_filepath) {
+        g_key_file_set_string(keyfile, "general", "logfile",
+                              config->log_filepath);
+    }
+    g_key_file_set_string(keyfile, "general", "pidfile", config->pid_filepath);
+#ifdef CONFIG_FSFREEZE
+    if (config->fsfreeze_hook) {
+        g_key_file_set_string(keyfile, "general", "fsfreeze-hook",
+                              config->fsfreeze_hook);
+    }
+#endif
+    g_key_file_set_string(keyfile, "general", "statedir", config->state_dir);
+    g_key_file_set_boolean(keyfile, "general", "verbose",
+                           config->log_level == G_LOG_LEVEL_MASK);
+    tmp = list_join(config->blacklist, ',');
+    g_key_file_set_string(keyfile, "general", "blacklist", tmp);
+    g_free(tmp);
+
+    tmp = g_key_file_to_data(keyfile, NULL, &error);
+    printf("%s", tmp);
+
+    g_free(tmp);
+    g_key_file_free(keyfile);
+}
+
+static void config_parse(GAConfig *config, int argc, char **argv)
+{
+    const char *sopt = "hVvdm:p:l:f:F::b:s:t:D";
+    int opt_ind = 0, ch;
     const struct option lopt[] = {
         { "help", 0, NULL, 'h' },
         { "version", 0, NULL, 'V' },
+        { "dump-conf", 0, NULL, 'D' },
         { "logfile", 1, NULL, 'l' },
         { "pidfile", 1, NULL, 'f' },
 #ifdef CONFIG_FSFREEZE
@@ -953,133 +1073,113 @@
         { "statedir", 1, NULL, 't' },
         { NULL, 0, NULL, 0 }
     };
-    int opt_ind = 0, ch, daemonize = 0, i, j, len;
-    GLogLevelFlags log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
-    GList *blacklist = NULL;
-    GAState *s;
-
-    module_call_init(MODULE_INIT_QAPI);
-
-    init_dfl_pathnames();
-    pid_filepath = dfl_pathnames.pidfile;
-    state_dir = dfl_pathnames.state_dir;
 
     while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
         switch (ch) {
         case 'm':
-            method = optarg;
+            g_free(config->method);
+            config->method = g_strdup(optarg);
             break;
         case 'p':
-            path = optarg;
+            g_free(config->channel_path);
+            config->channel_path = g_strdup(optarg);
             break;
         case 'l':
-            log_filepath = optarg;
+            g_free(config->log_filepath);
+            config->log_filepath = g_strdup(optarg);
             break;
         case 'f':
-            pid_filepath = optarg;
+            g_free(config->pid_filepath);
+            config->pid_filepath = g_strdup(optarg);
             break;
 #ifdef CONFIG_FSFREEZE
         case 'F':
-            fsfreeze_hook = optarg ? optarg : QGA_FSFREEZE_HOOK_DEFAULT;
+            g_free(config->fsfreeze_hook);
+            config->fsfreeze_hook = g_strdup(optarg ?: QGA_FSFREEZE_HOOK_DEFAULT);
             break;
 #endif
         case 't':
-             state_dir = optarg;
-             break;
+            g_free(config->state_dir);
+            config->state_dir = g_strdup(optarg);
+            break;
         case 'v':
             /* enable all log levels */
-            log_level = G_LOG_LEVEL_MASK;
+            config->log_level = G_LOG_LEVEL_MASK;
             break;
         case 'V':
             printf("QEMU Guest Agent %s\n", QEMU_VERSION);
-            return 0;
+            exit(EXIT_SUCCESS);
         case 'd':
-            daemonize = 1;
+            config->daemonize = 1;
+            break;
+        case 'D':
+            config->dumpconf = 1;
             break;
         case 'b': {
             if (is_help_option(optarg)) {
                 qmp_for_each_command(ga_print_cmd, NULL);
-                return 0;
+                exit(EXIT_SUCCESS);
             }
-            for (j = 0, i = 0, len = strlen(optarg); i < len; i++) {
-                if (optarg[i] == ',') {
-                    optarg[i] = 0;
-                    blacklist = g_list_append(blacklist, &optarg[j]);
-                    j = i + 1;
-                }
-            }
-            if (j < i) {
-                blacklist = g_list_append(blacklist, &optarg[j]);
-            }
+            config->blacklist = g_list_concat(config->blacklist,
+                                             split_list(optarg, ","));
             break;
         }
 #ifdef _WIN32
         case 's':
-            service = optarg;
-            if (strcmp(service, "install") == 0) {
-                const char *fixed_state_dir;
-
-                /* If the user passed the "-t" option, we save that state dir
-                 * in the service. Otherwise we let the service fetch the state
-                 * dir from the environment when it starts.
-                 */
-                fixed_state_dir = (state_dir == dfl_pathnames.state_dir) ?
-                                  NULL :
-                                  state_dir;
+            config->service = optarg;
+            if (strcmp(config->service, "install") == 0) {
                 if (ga_install_vss_provider()) {
-                    return EXIT_FAILURE;
+                    exit(EXIT_FAILURE);
                 }
-                if (ga_install_service(path, log_filepath, fixed_state_dir)) {
-                    return EXIT_FAILURE;
+                if (ga_install_service(config->channel_path,
+                                       config->log_filepath, config->state_dir)) {
+                    exit(EXIT_FAILURE);
                 }
-                return 0;
-            } else if (strcmp(service, "uninstall") == 0) {
+                exit(EXIT_SUCCESS);
+            } else if (strcmp(config->service, "uninstall") == 0) {
                 ga_uninstall_vss_provider();
-                return ga_uninstall_service();
+                exit(ga_uninstall_service());
+            } else if (strcmp(config->service, "vss-install") == 0) {
+                if (ga_install_vss_provider()) {
+                    exit(EXIT_FAILURE);
+                }
+                exit(EXIT_SUCCESS);
+            } else if (strcmp(config->service, "vss-uninstall") == 0) {
+                ga_uninstall_vss_provider();
+                exit(EXIT_SUCCESS);
             } else {
                 printf("Unknown service command.\n");
-                return EXIT_FAILURE;
+                exit(EXIT_FAILURE);
             }
             break;
 #endif
         case 'h':
             usage(argv[0]);
-            return 0;
+            exit(EXIT_SUCCESS);
         case '?':
             g_print("Unknown option, try '%s --help' for more information.\n",
                     argv[0]);
-            return EXIT_FAILURE;
+            exit(EXIT_FAILURE);
         }
     }
+}
 
-#ifdef _WIN32
-    /* On win32 the state directory is application specific (be it the default
-     * or a user override). We got past the command line parsing; let's create
-     * the directory (with any intermediate directories). If we run into an
-     * error later on, we won't try to clean up the directory, it is considered
-     * persistent.
-     */
-    if (g_mkdir_with_parents(state_dir, S_IRWXU) == -1) {
-        g_critical("unable to create (an ancestor of) the state directory"
-                   " '%s': %s", state_dir, strerror(errno));
-        return EXIT_FAILURE;
-    }
-#endif
-
-    s = g_malloc0(sizeof(GAState));
-    s->log_level = log_level;
-    s->log_file = stderr;
+static void config_free(GAConfig *config)
+{
+    g_free(config->method);
+    g_free(config->log_filepath);
+    g_free(config->pid_filepath);
+    g_free(config->state_dir);
+    g_free(config->channel_path);
+    g_free(config->bliststr);
 #ifdef CONFIG_FSFREEZE
-    s->fsfreeze_hook = fsfreeze_hook;
+    g_free(config->fsfreeze_hook);
 #endif
-    g_log_set_default_handler(ga_log, s);
-    g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
-    ga_enable_logging(s);
-    s->state_filepath_isfrozen = g_strdup_printf("%s/qga.state.isfrozen",
-                                                 state_dir);
-    s->pstate_filepath = g_strdup_printf("%s/qga.state", state_dir);
-    s->frozen = false;
+    g_free(config);
+}
 
+static bool check_is_frozen(GAState *s)
+{
 #ifndef _WIN32
     /* check if a previous instance of qemu-ga exited with filesystems' state
      * marked as frozen. this could be a stale value (a non-qemu-ga process
@@ -1087,7 +1187,7 @@
      * uneeded unfreeze than to risk hanging on start-up
      */
     struct stat st;
-    if (stat(s->state_filepath_isfrozen, &st) == -1) {
+    if (qemu_stat(s->state_filepath_isfrozen, &st) == -1) {
         /* it's okay if the file doesn't exist, but if we can't access for
          * some other reason, such as permissions, there's a configuration
          * that needs to be addressed. so just bail now before we get into
@@ -1105,32 +1205,56 @@
                   " guest-fsfreeze-thaw is issued, or filesystems are"
                   " manually unfrozen and the file %s is removed",
                   s->state_filepath_isfrozen);
-        s->frozen = true;
+        return true;
+    }
+#endif
+    return false;
+}
+
+static int run_agent(GAState *s, GAConfig *config)
+{
+    ga_state = s;
+
+    g_log_set_default_handler(ga_log, s);
+    g_log_set_fatal_mask(NULL, G_LOG_LEVEL_ERROR);
+    ga_enable_logging(s);
+
+#ifdef _WIN32
+    /* On win32 the state directory is application specific (be it the default
+     * or a user override). We got past the command line parsing; let's create
+     * the directory (with any intermediate directories). If we run into an
+     * error later on, we won't try to clean up the directory, it is considered
+     * persistent.
+     */
+    if (g_mkdir_with_parents(config->state_dir, S_IRWXU) == -1) {
+        g_critical("unable to create (an ancestor of) the state directory"
+                   " '%s': %s", config->state_dir, strerror(errno));
+        return EXIT_FAILURE;
     }
 #endif
 
     if (ga_is_frozen(s)) {
-        if (daemonize) {
+        if (config->daemonize) {
             /* delay opening/locking of pidfile till filesystems are unfrozen */
-            s->deferred_options.pid_filepath = pid_filepath;
+            s->deferred_options.pid_filepath = config->pid_filepath;
             become_daemon(NULL);
         }
-        if (log_filepath) {
+        if (config->log_filepath) {
             /* delay opening the log file till filesystems are unfrozen */
-            s->deferred_options.log_filepath = log_filepath;
+            s->deferred_options.log_filepath = config->log_filepath;
         }
         ga_disable_logging(s);
         qmp_for_each_command(ga_disable_non_whitelisted, NULL);
     } else {
-        if (daemonize) {
-            become_daemon(pid_filepath);
+        if (config->daemonize) {
+            become_daemon(config->pid_filepath);
         }
-        if (log_filepath) {
-            FILE *log_file = ga_open_logfile(log_filepath);
+        if (config->log_filepath) {
+            FILE *log_file = ga_open_logfile(config->log_filepath);
             if (!log_file) {
                 g_critical("unable to open specified log file: %s",
                            strerror(errno));
-                goto out_bad;
+                return EXIT_FAILURE;
             }
             s->log_file = log_file;
         }
@@ -1141,17 +1265,18 @@
                                s->pstate_filepath,
                                ga_is_frozen(s))) {
         g_critical("failed to load persistent state");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 
-    blacklist = ga_command_blacklist_init(blacklist);
-    if (blacklist) {
-        s->blacklist = blacklist;
+    config->blacklist = ga_command_blacklist_init(config->blacklist);
+    if (config->blacklist) {
+        GList *l = config->blacklist;
+        s->blacklist = config->blacklist;
         do {
-            g_debug("disabling command: %s", (char *)blacklist->data);
-            qmp_disable_command(blacklist->data);
-            blacklist = g_list_next(blacklist);
-        } while (blacklist);
+            g_debug("disabling command: %s", (char *)l->data);
+            qmp_disable_command(l->data);
+            l = g_list_next(l);
+        } while (l);
     }
     s->command_state = ga_command_state_new();
     ga_command_state_init(s, s->command_state);
@@ -1161,19 +1286,19 @@
 #ifndef _WIN32
     if (!register_signal_handlers()) {
         g_critical("failed to register signal handlers");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 #endif
 
     s->main_loop = g_main_loop_new(NULL, false);
-    if (!channel_init(ga_state, method, path)) {
+    if (!channel_init(ga_state, config->method, config->channel_path)) {
         g_critical("failed to initialize guest agent channel");
-        goto out_bad;
+        return EXIT_FAILURE;
     }
 #ifndef _WIN32
     g_main_loop_run(ga_state->main_loop);
 #else
-    if (daemonize) {
+    if (config->daemonize) {
         SERVICE_TABLE_ENTRY service_table[] = {
             { (char *)QGA_SERVICE_NAME, service_main }, { NULL, NULL } };
         StartServiceCtrlDispatcher(service_table);
@@ -1182,17 +1307,87 @@
     }
 #endif
 
-    ga_command_state_cleanup_all(ga_state->command_state);
-    ga_channel_free(ga_state->channel);
+    return EXIT_SUCCESS;
+}
 
-    if (daemonize) {
-        unlink(pid_filepath);
-    }
-    return 0;
+static void free_blacklist_entry(gpointer entry, gpointer unused)
+{
+    g_free(entry);
+}
 
-out_bad:
-    if (daemonize) {
-        unlink(pid_filepath);
+int main(int argc, char **argv)
+{
+    int ret = EXIT_SUCCESS;
+    GAState *s = g_new0(GAState, 1);
+    GAConfig *config = g_new0(GAConfig, 1);
+
+    config->log_level = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL;
+
+    module_call_init(MODULE_INIT_QAPI);
+
+    init_dfl_pathnames();
+    config_load(config);
+    config_parse(config, argc, argv);
+
+    if (config->pid_filepath == NULL) {
+        config->pid_filepath = g_strdup(dfl_pathnames.pidfile);
     }
-    return EXIT_FAILURE;
+
+    if (config->state_dir == NULL) {
+        config->state_dir = g_strdup(dfl_pathnames.state_dir);
+    }
+
+    if (config->method == NULL) {
+        config->method = g_strdup("virtio-serial");
+    }
+
+    if (config->channel_path == NULL) {
+        if (strcmp(config->method, "virtio-serial") == 0) {
+            /* try the default path for the virtio-serial port */
+            config->channel_path = g_strdup(QGA_VIRTIO_PATH_DEFAULT);
+        } else if (strcmp(config->method, "isa-serial") == 0) {
+            /* try the default path for the serial port - COM1 */
+            config->channel_path = g_strdup(QGA_SERIAL_PATH_DEFAULT);
+        } else {
+            g_critical("must specify a path for this channel");
+            ret = EXIT_FAILURE;
+            goto end;
+        }
+    }
+
+    s->log_level = config->log_level;
+    s->log_file = stderr;
+#ifdef CONFIG_FSFREEZE
+    s->fsfreeze_hook = config->fsfreeze_hook;
+#endif
+    s->pstate_filepath = g_strdup_printf("%s/qga.state", config->state_dir);
+    s->state_filepath_isfrozen = g_strdup_printf("%s/qga.state.isfrozen",
+                                                 config->state_dir);
+    s->frozen = check_is_frozen(s);
+
+    if (config->dumpconf) {
+        config_dump(config);
+        goto end;
+    }
+
+    ret = run_agent(s, config);
+
+end:
+    if (s->command_state) {
+        ga_command_state_cleanup_all(s->command_state);
+    }
+    if (s->channel) {
+        ga_channel_free(s->channel);
+    }
+    g_list_foreach(config->blacklist, free_blacklist_entry, NULL);
+    g_free(s->pstate_filepath);
+    g_free(s->state_filepath_isfrozen);
+
+    if (config->daemonize) {
+        unlink(config->pid_filepath);
+    }
+
+    config_free(config);
+
+    return ret;
 }
diff --git a/qga/qapi-schema.json b/qga/qapi-schema.json
index 376e79f..c21f308 100644
--- a/qga/qapi-schema.json
+++ b/qga/qapi-schema.json
@@ -12,6 +12,7 @@
 ##
 
 ##
+# @guest-sync-delimited:
 #
 # Echo back a unique integer value, and prepend to response a
 # leading sentinel byte (0xFF) the client can check scan for.
@@ -41,7 +42,7 @@
 # Returns: The unique integer id passed in by the client
 #
 # Since: 1.1
-# ##
+##
 { 'command': 'guest-sync-delimited',
   'data':    { 'id': 'int' },
   'returns': 'int' }
@@ -121,7 +122,10 @@
 # given value, then sets the Hardware Clock (RTC) to the
 # current System Time. This will make it easier for a guest
 # to resynchronize without waiting for NTP. If no @time is
-# specified, then the time to set is read from RTC.
+# specified, then the time to set is read from RTC. However,
+# this may not be supported on all platforms (i.e. Windows).
+# If that's the case users are advised to always pass a
+# value.
 #
 # @time: #optional time of nanoseconds, relative to the Epoch
 #        of 1970-01-01 in UTC.
@@ -147,7 +151,7 @@
 #
 # Since 1.1.0
 ##
-{ 'type': 'GuestAgentCommandInfo',
+{ 'struct': 'GuestAgentCommandInfo',
   'data': { 'name': 'str', 'enabled': 'bool', 'success-response': 'bool' } }
 
 ##
@@ -161,7 +165,7 @@
 #
 # Since 0.15.0
 ##
-{ 'type': 'GuestAgentInfo',
+{ 'struct': 'GuestAgentInfo',
   'data': { 'version': 'str',
             'supported_commands': ['GuestAgentCommandInfo'] } }
 ##
@@ -192,7 +196,7 @@
 # Since: 0.15.0
 ##
 { 'command': 'guest-shutdown', 'data': { '*mode': 'str' },
-  'success-response': 'no' }
+  'success-response': false }
 
 ##
 # @guest-file-open:
@@ -239,7 +243,7 @@
 #
 # Since: 0.15.0
 ##
-{ 'type': 'GuestFileRead',
+{ 'struct': 'GuestFileRead',
   'data': { 'count': 'int', 'buf-b64': 'str', 'eof': 'bool' } }
 
 ##
@@ -271,7 +275,7 @@
 #
 # Since: 0.15.0
 ##
-{ 'type': 'GuestFileWrite',
+{ 'struct': 'GuestFileWrite',
   'data': { 'count': 'int', 'eof': 'bool' } }
 
 ##
@@ -306,28 +310,57 @@
 #
 # Since: 0.15.0
 ##
-{ 'type': 'GuestFileSeek',
+{ 'struct': 'GuestFileSeek',
   'data': { 'position': 'int', 'eof': 'bool' } }
 
 ##
+# @QGASeek:
+#
+# Symbolic names for use in @guest-file-seek
+#
+# @set: Set to the specified offset (same effect as 'whence':0)
+# @cur: Add offset to the current location (same effect as 'whence':1)
+# @end: Add offset to the end of the file (same effect as 'whence':2)
+#
+# Since: 2.6
+##
+{ 'enum': 'QGASeek', 'data': [ 'set', 'cur', 'end' ] }
+
+##
+# @GuestFileWhence:
+#
+# Controls the meaning of offset to @guest-file-seek.
+#
+# @value: Integral value (0 for set, 1 for cur, 2 for end), available
+#         for historical reasons, and might differ from the host's or
+#         guest's SEEK_* values (since: 0.15)
+# @name: Symbolic name, and preferred interface
+#
+# Since: 2.6
+##
+{ 'alternate': 'GuestFileWhence',
+  'data': { 'value': 'int', 'name': 'QGASeek' } }
+
+##
 # @guest-file-seek:
 #
 # Seek to a position in the file, as with fseek(), and return the
 # current file position afterward. Also encapsulates ftell()'s
-# functionality, just Set offset=0, whence=SEEK_CUR.
+# functionality, with offset=0 and whence=1.
 #
 # @handle: filehandle returned by guest-file-open
 #
 # @offset: bytes to skip over in the file stream
 #
-# @whence: SEEK_SET, SEEK_CUR, or SEEK_END, as with fseek()
+# @whence: Symbolic or numeric code for interpreting offset
 #
 # Returns: @GuestFileSeek on success.
 #
 # Since: 0.15.0
 ##
 { 'command': 'guest-file-seek',
-  'data':    { 'handle': 'int', 'offset': 'int', 'whence': 'int' },
+  'data':    { 'handle': 'int', 'offset': 'int',
+               'whence': 'GuestFileWhence' },
   'returns': 'GuestFileSeek' }
 
 ##
@@ -422,6 +455,30 @@
   'returns': 'int' }
 
 ##
+# @GuestFilesystemTrimResult
+#
+# @path: path that was trimmed
+# @error: an error message when trim failed
+# @trimmed: bytes trimmed for this path
+# @minimum: reported effective minimum for this path
+#
+# Since: 2.4
+##
+{ 'struct': 'GuestFilesystemTrimResult',
+  'data': {'path': 'str',
+           '*trimmed': 'int', '*minimum': 'int', '*error': 'str'} }
+
+##
+# @GuestFilesystemTrimResponse
+#
+# @paths: list of @GuestFilesystemTrimResult per path that was trimmed
+#
+# Since: 2.4
+##
+{ 'struct': 'GuestFilesystemTrimResponse',
+  'data': {'paths': ['GuestFilesystemTrimResult']} }
+
+##
 # @guest-fstrim:
 #
 # Discard (or "trim") blocks which are not in use by the filesystem.
@@ -434,12 +491,14 @@
 #       fragmented free space, although not all blocks will be discarded.
 #       The default value is zero, meaning "discard every free block".
 #
-# Returns: Nothing.
+# Returns: A @GuestFilesystemTrimResponse which contains the
+#          status of all trimmed paths. (since 2.4)
 #
 # Since: 1.2
 ##
 { 'command': 'guest-fstrim',
-  'data': { '*minimum': 'int' } }
+  'data': { '*minimum': 'int' },
+  'returns': 'GuestFilesystemTrimResponse' }
 
 ##
 # @guest-suspend-disk
@@ -467,7 +526,7 @@
 #
 # Since: 1.1
 ##
-{ 'command': 'guest-suspend-disk', 'success-response': 'no' }
+{ 'command': 'guest-suspend-disk', 'success-response': false }
 
 ##
 # @guest-suspend-ram
@@ -499,7 +558,7 @@
 #
 # Since: 1.1
 ##
-{ 'command': 'guest-suspend-ram', 'success-response': 'no' }
+{ 'command': 'guest-suspend-ram', 'success-response': false }
 
 ##
 # @guest-suspend-hybrid
@@ -526,7 +585,7 @@
 #
 # Since: 1.1
 ##
-{ 'command': 'guest-suspend-hybrid', 'success-response': 'no' }
+{ 'command': 'guest-suspend-hybrid', 'success-response': false }
 
 ##
 # @GuestIpAddressType:
@@ -553,7 +612,7 @@
 #
 # Since: 1.1
 ##
-{ 'type': 'GuestIpAddress',
+{ 'struct': 'GuestIpAddress',
   'data': {'ip-address': 'str',
            'ip-address-type': 'GuestIpAddressType',
            'prefix': 'int'} }
@@ -569,7 +628,7 @@
 #
 # Since: 1.1
 ##
-{ 'type': 'GuestNetworkInterface',
+{ 'struct': 'GuestNetworkInterface',
   'data': {'name': 'str',
            '*hardware-address': 'str',
            '*ip-addresses': ['GuestIpAddress'] } }
@@ -601,7 +660,7 @@
 #
 # Since: 1.5
 ##
-{ 'type': 'GuestLogicalProcessor',
+{ 'struct': 'GuestLogicalProcessor',
   'data': {'logical-id': 'int',
            'online': 'bool',
            '*can-offline': 'bool'} }
@@ -674,12 +733,24 @@
 # @uml: UML disks
 # @sata: SATA disks
 # @sd: SD cards
+# @unknown: Unknown bus type
+# @ieee1394: Win IEEE 1394 bus type
+# @ssa: Win SSA bus type
+# @fibre: Win fiber channel bus type
+# @raid: Win RAID bus type
+# @iscsi: Win iScsi bus type
+# @sas: Win serial-attaches SCSI bus type
+# @mmc: Win multimedia card (MMC) bus type
+# @virtual: Win virtual bus type
+# @file-backed virtual: Win file-backed bus type
 #
-# Since: 2.2
+# Since: 2.2; 'Unknown' and all entries below since 2.4
 ##
 { 'enum': 'GuestDiskBusType',
   'data': [ 'ide', 'fdc', 'scsi', 'virtio', 'xen', 'usb', 'uml', 'sata',
-            'sd' ] }
+            'sd', 'unknown', 'ieee1394', 'ssa', 'fibre', 'raid', 'iscsi',
+            'sas', 'mmc', 'virtual', 'file-backed-virtual' ] }
+
 
 ##
 # @GuestPCIAddress:
@@ -691,7 +762,7 @@
 #
 # Since: 2.2
 ##
-{ 'type': 'GuestPCIAddress',
+{ 'struct': 'GuestPCIAddress',
   'data': {'domain': 'int', 'bus': 'int',
            'slot': 'int', 'function': 'int'} }
 
@@ -706,7 +777,7 @@
 #
 # Since: 2.2
 ##
-{ 'type': 'GuestDiskAddress',
+{ 'struct': 'GuestDiskAddress',
   'data': {'pci-controller': 'GuestPCIAddress',
            'bus-type': 'GuestDiskBusType',
            'bus': 'int', 'target': 'int', 'unit': 'int'} }
@@ -722,7 +793,7 @@
 #
 # Since: 2.2
 ##
-{ 'type': 'GuestFilesystemInfo',
+{ 'struct': 'GuestFilesystemInfo',
   'data': {'name': 'str', 'mountpoint': 'str', 'type': 'str',
            'disk': ['GuestDiskAddress']} }
 
@@ -738,3 +809,220 @@
 ##
 { 'command': 'guest-get-fsinfo',
   'returns': ['GuestFilesystemInfo'] }
+
+##
+# @guest-set-user-password
+#
+# @username: the user account whose password to change
+# @password: the new password entry string, base64 encoded
+# @crypted: true if password is already crypt()d, false if raw
+#
+# If the @crypted flag is true, it is the caller's responsibility
+# to ensure the correct crypt() encryption scheme is used. This
+# command does not attempt to interpret or report on the encryption
+# scheme. Refer to the documentation of the guest operating system
+# in question to determine what is supported.
+#
+# Not all guest operating systems will support use of the
+# @crypted flag, as they may require the clear-text password
+#
+# The @password parameter must always be base64 encoded before
+# transmission, even if already crypt()d, to ensure it is 8-bit
+# safe when passed as JSON.
+#
+# Returns: Nothing on success.
+#
+# Since 2.3
+##
+{ 'command': 'guest-set-user-password',
+  'data': { 'username': 'str', 'password': 'str', 'crypted': 'bool' } }
+
+# @GuestMemoryBlock:
+#
+# @phys-index: Arbitrary guest-specific unique identifier of the MEMORY BLOCK.
+#
+# @online: Whether the MEMORY BLOCK is enabled in guest.
+#
+# @can-offline: #optional Whether offlining the MEMORY BLOCK is possible.
+#               This member is always filled in by the guest agent when the
+#               structure is returned, and always ignored on input (hence it
+#               can be omitted then).
+#
+# Since: 2.3
+##
+{ 'struct': 'GuestMemoryBlock',
+  'data': {'phys-index': 'uint64',
+           'online': 'bool',
+           '*can-offline': 'bool'} }
+
+##
+# @guest-get-memory-blocks:
+#
+# Retrieve the list of the guest's memory blocks.
+#
+# This is a read-only operation.
+#
+# Returns: The list of all memory blocks the guest knows about.
+# Each memory block is put on the list exactly once, but their order
+# is unspecified.
+#
+# Since: 2.3
+##
+{ 'command': 'guest-get-memory-blocks',
+  'returns': ['GuestMemoryBlock'] }
+
+##
+# @GuestMemoryBlockResponseType
+#
+# An enumeration of memory block operation result.
+#
+# @success: the operation of online/offline memory block is successful.
+# @not-found: can't find the corresponding memoryXXX directory in sysfs.
+# @operation-not-supported: for some old kernels, it does not support
+#                           online or offline memory block.
+# @operation-failed: the operation of online/offline memory block fails,
+#                    because of some errors happen.
+#
+# Since: 2.3
+##
+{ 'enum': 'GuestMemoryBlockResponseType',
+  'data': ['success', 'not-found', 'operation-not-supported',
+           'operation-failed'] }
+
+##
+# @GuestMemoryBlockResponse:
+#
+# @phys-index: same with the 'phys-index' member of @GuestMemoryBlock.
+#
+# @response: the result of memory block operation.
+#
+# @error-code: #optional the error number.
+#               When memory block operation fails, we assign the value of
+#               'errno' to this member, it indicates what goes wrong.
+#               When the operation succeeds, it will be omitted.
+#
+# Since: 2.3
+##
+{ 'struct': 'GuestMemoryBlockResponse',
+  'data': { 'phys-index': 'uint64',
+            'response': 'GuestMemoryBlockResponseType',
+            '*error-code': 'int' }}
+
+##
+# @guest-set-memory-blocks:
+#
+# Attempt to reconfigure (currently: enable/disable) state of memory blocks
+# inside the guest.
+#
+# The input list is processed node by node in order. In each node @phys-index
+# is used to look up the guest MEMORY BLOCK, for which @online specifies the
+# requested state. The set of distinct @phys-index's is only required to be a
+# subset of the guest-supported identifiers. There's no restriction on list
+# length or on repeating the same @phys-index (with possibly different @online
+# field).
+# Preferably the input list should describe a modified subset of
+# @guest-get-memory-blocks' return value.
+#
+# Returns: The operation results, it is a list of @GuestMemoryBlockResponse,
+#          which is corresponding to the input list.
+#
+#          Note: it will return NULL if the @mem-blks list was empty on input,
+#          or there is an error, and in this case, guest state will not be
+#          changed.
+#
+# Since: 2.3
+##
+{ 'command': 'guest-set-memory-blocks',
+  'data':    {'mem-blks': ['GuestMemoryBlock'] },
+  'returns': ['GuestMemoryBlockResponse'] }
+
+# @GuestMemoryBlockInfo:
+#
+# @size: the size (in bytes) of the guest memory blocks,
+#        which are the minimal units of memory block online/offline
+#        operations (also called Logical Memory Hotplug).
+#
+# Since: 2.3
+##
+{ 'struct': 'GuestMemoryBlockInfo',
+  'data': {'size': 'uint64'} }
+
+##
+# @guest-get-memory-block-info:
+#
+# Get information relating to guest memory blocks.
+#
+# Returns: memory block size in bytes.
+# Returns: @GuestMemoryBlockInfo
+#
+# Since 2.3
+##
+{ 'command': 'guest-get-memory-block-info',
+  'returns': 'GuestMemoryBlockInfo' }
+
+# @GuestExecStatus:
+#
+# @exited: true if process has already terminated.
+# @exitcode: #optional process exit code if it was normally terminated.
+# @signal: #optional signal number (linux) or unhandled exception code
+#       (windows) if the process was abnormally terminated.
+# @out-data: #optional base64-encoded stdout of the process
+# @err-data: #optional base64-encoded stderr of the process
+#       Note: @out-data and @err-data are present only
+#       if 'capture-output' was specified for 'guest-exec'
+# @out-truncated: #optional true if stdout was not fully captured
+#       due to size limitation.
+# @err-truncated: #optional true if stderr was not fully captured
+#       due to size limitation.
+#
+# Since: 2.5
+##
+{ 'struct': 'GuestExecStatus',
+  'data': { 'exited': 'bool', '*exitcode': 'int', '*signal': 'int',
+            '*out-data': 'str', '*err-data': 'str',
+            '*out-truncated': 'bool', '*err-truncated': 'bool' }}
+##
+# @guest-exec-status
+#
+# Check status of process associated with PID retrieved via guest-exec.
+# Reap the process and associated metadata if it has exited.
+#
+# @pid: pid returned from guest-exec
+#
+# Returns: GuestExecStatus on success.
+#
+# Since 2.5
+##
+{ 'command': 'guest-exec-status',
+  'data':    { 'pid': 'int' },
+  'returns': 'GuestExecStatus' }
+
+##
+# @GuestExec:
+# @pid: pid of child process in guest OS
+#
+#Since: 2.5
+##
+{ 'struct': 'GuestExec',
+  'data': { 'pid': 'int'} }
+
+##
+# @guest-exec:
+#
+# Execute a command in the guest
+#
+# @path: path or executable name to execute
+# @arg: #optional argument list to pass to executable
+# @env: #optional environment variables to pass to executable
+# @input-data: #optional data to be passed to process stdin (base64 encoded)
+# @capture-output: #optional bool flag to enable capture of
+#                  stdout/stderr of running process. defaults to false.
+#
+# Returns: PID on success.
+#
+# Since: 2.5
+##
+{ 'command': 'guest-exec',
+  'data':    { 'path': 'str', '*arg': ['str'], '*env': ['str'],
+               '*input-data': 'str', '*capture-output': 'bool' },
+  'returns': 'GuestExec' }
diff --git a/qga/service-win32.c b/qga/service-win32.c
index aef41f0..f44ca03 100644
--- a/qga/service-win32.c
+++ b/qga/service-win32.c
@@ -10,9 +10,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include <windows.h>
 #include "qga/service-win32.h"
 
@@ -108,7 +106,7 @@
     GString *cmdline;
     SERVICE_DESCRIPTION desc = { (char *)QGA_SERVICE_DESCRIPTION };
 
-    if (GetModuleFileName(NULL, module_fname, MAX_PATH) == 0) {
+    if (win32GetModuleFileName(NULL, module_fname, MAX_PATH) == 0) {
         printf_win_error("No full path to service's executable");
         return EXIT_FAILURE;
     }
diff --git a/qga/service-win32.h b/qga/service-win32.h
index 3b9e870..89e99df 100644
--- a/qga/service-win32.h
+++ b/qga/service-win32.h
@@ -10,8 +10,9 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#ifndef QGA_SERVICE_H
-#define QGA_SERVICE_H
+
+#ifndef QGA_SERVICE_WIN32_H
+#define QGA_SERVICE_WIN32_H
 
 #include <windows.h>
 
diff --git a/qga/vss-win32.c b/qga/vss-win32.c
index 0e40957..9a0e463 100644
--- a/qga/vss-win32.c
+++ b/qga/vss-win32.c
@@ -10,7 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <windows.h>
 #include "qga/guest-agent-core.h"
 #include "qga/vss-win32.h"
@@ -150,11 +150,11 @@
     const char *func_name = freeze ? "requester_freeze" : "requester_thaw";
     QGAVSSRequesterFunc func;
     ErrorSet errset = {
-        .error_set = (ErrorSetFunc)error_set_win32,
-        .errp = (void **)errp,
-        .err_class = ERROR_CLASS_GENERIC_ERROR
+        .error_setg_win32_wrapper = error_setg_win32_internal,
+        .errp = errp,
     };
 
+    g_assert(errp);             /* requester.cpp requires it */
     func = (QGAVSSRequesterFunc)GetProcAddress(provider_lib, func_name);
     if (!func) {
         error_setg_win32(errp, GetLastError(), "failed to load %s from %s",
diff --git a/qga/vss-win32.h b/qga/vss-win32.h
index 298927d..4d1d150 100644
--- a/qga/vss-win32.h
+++ b/qga/vss-win32.h
@@ -13,7 +13,6 @@
 #ifndef VSS_WIN32_H
 #define VSS_WIN32_H
 
-#include "qapi/error.h"
 
 bool vss_init(bool init_requester);
 void vss_deinit(bool deinit_requester);
diff --git a/qga/vss-win32/Makefile.objs b/qga/vss-win32/Makefile.objs
index 6a69d50..7c96c6b 100644
--- a/qga/vss-win32/Makefile.objs
+++ b/qga/vss-win32/Makefile.objs
@@ -3,7 +3,7 @@
 qga-vss-dll-obj-y += requester.o provider.o install.o
 
 obj-qga-vss-dll-obj-y = $(addprefix $(obj)/, $(qga-vss-dll-obj-y))
-$(obj-qga-vss-dll-obj-y): QEMU_CXXFLAGS = $(filter-out -Wstrict-prototypes -Wmissing-prototypes -Wnested-externs -Wold-style-declaration -Wold-style-definition -Wredundant-decls -fstack-protector-all, $(QEMU_CFLAGS)) -Wno-unknown-pragmas -Wno-delete-non-virtual-dtor
+$(obj-qga-vss-dll-obj-y): QEMU_CXXFLAGS = $(filter-out -Wstrict-prototypes -Wmissing-prototypes -Wnested-externs -Wold-style-declaration -Wold-style-definition -Wredundant-decls -fstack-protector-all -fstack-protector-strong, $(QEMU_CFLAGS)) -Wno-unknown-pragmas -Wno-delete-non-virtual-dtor
 
 $(obj)/qga-vss.dll: LDFLAGS = -shared -Wl,--add-stdcall-alias,--enable-stdcall-fixup -lole32 -loleaut32 -lshlwapi -luuid -static
 $(obj)/qga-vss.dll: $(obj-qga-vss-dll-obj-y) $(SRC_PATH)/$(obj)/qga-vss.def
diff --git a/qga/vss-win32/install.cpp b/qga/vss-win32/install.cpp
index b0e4426..931ffb3 100644
--- a/qga/vss-win32/install.cpp
+++ b/qga/vss-win32/install.cpp
@@ -10,12 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "vss-common.h"
-#include "inc/win2003/vscoordint.h"
-
+#include <inc/win2003/vscoordint.h>
 #include <comadmin.h>
 #include <wbemidl.h>
 #include <comdef.h>
@@ -254,7 +252,7 @@
 
     chk(pObj->get_Key(&key));
 
-    if (!GetModuleFileName(g_hinstDll, dllPath, sizeof(dllPath))) {
+    if (!win32GetModuleFileName(g_hinstDll, dllPath, sizeof(dllPath))) {
         hr = HRESULT_FROM_WIN32(GetLastError());
         errmsg(hr, "GetModuleFileName failed");
         goto out;
@@ -369,7 +367,7 @@
         goto out;
     }
 
-    if (!GetModuleFileName(g_hinstDll, dllPath, sizeof(dllPath))) {
+    if (!win32GetModuleFileName(g_hinstDll, dllPath, sizeof(dllPath))) {
         errmsg_dialog(GetLastError(), "GetModuleFileName failed");
         goto out;
     }
diff --git a/qga/vss-win32/provider.cpp b/qga/vss-win32/provider.cpp
index d5129f8..ef94669 100644
--- a/qga/vss-win32/provider.cpp
+++ b/qga/vss-win32/provider.cpp
@@ -10,10 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "vss-common.h"
-#include "inc/win2003/vscoordint.h"
-#include "inc/win2003/vsprov.h"
+#include <inc/win2003/vscoordint.h>
+#include <inc/win2003/vsprov.h>
 
 #define VSS_TIMEOUT_MSEC (60*1000)
 
diff --git a/qga/vss-win32/requester.cpp b/qga/vss-win32/requester.cpp
index 922e74d..0cd2f0e 100644
--- a/qga/vss-win32/requester.cpp
+++ b/qga/vss-win32/requester.cpp
@@ -10,12 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "vss-common.h"
 #include "requester.h"
-#include "assert.h"
-#include "inc/win2003/vswriter.h"
-#include "inc/win2003/vsbackup.h"
+#include <inc/win2003/vswriter.h>
+#include <inc/win2003/vsbackup.h>
 
 /* Max wait time for frozen event (VSS can only hold writes for 10 seconds) */
 #define VSS_TIMEOUT_FREEZE_MSEC 10000
@@ -23,10 +22,12 @@
 /* Call QueryStatus every 10 ms while waiting for frozen event */
 #define VSS_TIMEOUT_EVENT_MSEC 10
 
-#define err_set(e, err, fmt, ...) \
-    ((e)->error_set((e)->errp, err, (e)->err_class, fmt, ## __VA_ARGS__))
+#define err_set(e, err, fmt, ...)                                           \
+    ((e)->error_setg_win32_wrapper((e)->errp, __FILE__, __LINE__, __func__, \
+                                   err, fmt, ## __VA_ARGS__))
+/* Bad idea, works only when (e)->errp != NULL: */
 #define err_is_set(e) ((e)->errp && *(e)->errp)
-
+/* To lift this restriction, error_propagate(), like we do in QEMU code */
 
 /* Handle to VSSAPI.DLL */
 static HMODULE hLib;
diff --git a/qga/vss-win32/requester.h b/qga/vss-win32/requester.h
index 374f9b8..2a39d73 100644
--- a/qga/vss-win32/requester.h
+++ b/qga/vss-win32/requester.h
@@ -14,19 +14,21 @@
 #define VSS_WIN32_REQUESTER_H
 
 #include <basetyps.h>           /* STDAPI */
-#include "qemu/compiler.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+struct Error;
+
 /* Callback to set Error; used to avoid linking glib to the DLL */
-typedef void (*ErrorSetFunc)(void **errp, int win32_err, int err_class,
-                             const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+typedef void (*ErrorSetFunc)(struct Error **errp,
+                             const char *src, int line, const char *func,
+                             int win32_err, const char *fmt, ...)
+    GCC_FMT_ATTR(6, 7);
 typedef struct ErrorSet {
-    ErrorSetFunc error_set;
-    void **errp;
-    int err_class;
+    ErrorSetFunc error_setg_win32_wrapper;
+    struct Error **errp;        /* restriction: must not be null */
 } ErrorSet;
 
 STDAPI requester_init(void);
diff --git a/qga/vss-win32/vss-common.h b/qga/vss-win32/vss-common.h
index ce14e14..c81a856 100644
--- a/qga/vss-win32/vss-common.h
+++ b/qga/vss-win32/vss-common.h
@@ -10,11 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
-#ifndef VSS_WIN32_H
-#define VSS_WIN32_H
+#ifndef VSS_COMMON_H
+#define VSS_COMMON_H
 
 #define __MIDL_user_allocate_free_DEFINED__
-#include "config-host.h"
 #include <windows.h>
 #include <shlwapi.h>
 
@@ -51,7 +50,7 @@
  * VSS headers must be installed from Microsoft VSS SDK 7.2 available at:
  * http://www.microsoft.com/en-us/download/details.aspx?id=23490
  */
-#include "inc/win2003/vss.h"
+#include <inc/win2003/vss.h>
 
 /* Macros to convert char definitions to wchar */
 #define _L(a) L##a
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 718dd92..6866264 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -1,5 +1,5 @@
 HXCOMM QMP dispatch table and documentation
-HXCOMM Text between SQMP and EQMP is copied to the QMP documention file and
+HXCOMM Text between SQMP and EQMP is copied to the QMP documentation file and
 HXCOMM does not show up in the other formats.
 
 SQMP
@@ -63,7 +63,7 @@
     {
         .name       = "quit",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_quit,
+        .mhandler.cmd_new = qmp_marshal_quit,
     },
 
 SQMP
@@ -84,7 +84,7 @@
     {
         .name       = "eject",
         .args_type  = "force:-f,device:B",
-        .mhandler.cmd_new = qmp_marshal_input_eject,
+        .mhandler.cmd_new = qmp_marshal_eject,
     },
 
 SQMP
@@ -110,7 +110,7 @@
     {
         .name       = "change",
         .args_type  = "device:B,target:F,arg:s?",
-        .mhandler.cmd_new = qmp_marshal_input_change,
+        .mhandler.cmd_new = qmp_marshal_change,
     },
 
 SQMP
@@ -146,7 +146,7 @@
     {
         .name       = "screendump",
         .args_type  = "filename:F",
-        .mhandler.cmd_new = qmp_marshal_input_screendump,
+        .mhandler.cmd_new = qmp_marshal_screendump,
     },
 
 SQMP
@@ -169,7 +169,7 @@
     {
         .name       = "stop",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_stop,
+        .mhandler.cmd_new = qmp_marshal_stop,
     },
 
 SQMP
@@ -190,7 +190,7 @@
     {
         .name       = "cont",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_cont,
+        .mhandler.cmd_new = qmp_marshal_cont,
     },
 
 SQMP
@@ -211,7 +211,7 @@
     {
         .name       = "system_wakeup",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_wakeup,
+        .mhandler.cmd_new = qmp_marshal_system_wakeup,
     },
 
 SQMP
@@ -232,7 +232,7 @@
     {
         .name       = "system_reset",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_reset,
+        .mhandler.cmd_new = qmp_marshal_system_reset,
     },
 
 SQMP
@@ -253,7 +253,7 @@
     {
         .name       = "system_powerdown",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_system_powerdown,
+        .mhandler.cmd_new = qmp_marshal_system_powerdown,
     },
 
 SQMP
@@ -276,8 +276,7 @@
         .args_type  = "device:O",
         .params     = "driver[,prop=value][,...]",
         .help       = "add device, like -device on the command line",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_device_add,
+        .mhandler.cmd_new = qmp_device_add,
     },
 
 SQMP
@@ -311,7 +310,7 @@
     {
         .name       = "device_del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_device_del,
+        .mhandler.cmd_new = qmp_marshal_device_del,
     },
 
 SQMP
@@ -322,19 +321,24 @@
 
 Arguments:
 
-- "id": the device's ID (json-string)
+- "id": the device's ID or QOM path (json-string)
 
 Example:
 
 -> { "execute": "device_del", "arguments": { "id": "net1" } }
 <- { "return": {} }
 
+Example:
+
+-> { "execute": "device_del", "arguments": { "id": "/machine/peripheral-anon/device[0]" } }
+<- { "return": {} }
+
 EQMP
 
     {
         .name       = "send-key",
-        .args_type  = "keys:O,hold-time:i?",
-        .mhandler.cmd_new = qmp_marshal_input_send_key,
+        .args_type  = "keys:q,hold-time:i?",
+        .mhandler.cmd_new = qmp_marshal_send_key,
     },
 
 SQMP
@@ -365,7 +369,7 @@
     {
         .name       = "cpu",
         .args_type  = "index:i",
-        .mhandler.cmd_new = qmp_marshal_input_cpu,
+        .mhandler.cmd_new = qmp_marshal_cpu,
     },
 
 SQMP
@@ -390,7 +394,7 @@
     {
         .name       = "cpu-add",
         .args_type  = "id:i",
-        .mhandler.cmd_new = qmp_marshal_input_cpu_add,
+        .mhandler.cmd_new = qmp_marshal_cpu_add,
     },
 
 SQMP
@@ -413,7 +417,7 @@
     {
         .name       = "memsave",
         .args_type  = "val:l,size:i,filename:s,cpu:i?",
-        .mhandler.cmd_new = qmp_marshal_input_memsave,
+        .mhandler.cmd_new = qmp_marshal_memsave,
     },
 
 SQMP
@@ -442,7 +446,7 @@
     {
         .name       = "pmemsave",
         .args_type  = "val:l,size:i,filename:s",
-        .mhandler.cmd_new = qmp_marshal_input_pmemsave,
+        .mhandler.cmd_new = qmp_marshal_pmemsave,
     },
 
 SQMP
@@ -470,7 +474,7 @@
     {
         .name       = "inject-nmi",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_inject_nmi,
+        .mhandler.cmd_new = qmp_marshal_inject_nmi,
     },
 
 SQMP
@@ -493,7 +497,7 @@
     {
         .name       = "ringbuf-write",
         .args_type  = "device:s,data:s,format:s?",
-        .mhandler.cmd_new = qmp_marshal_input_ringbuf_write,
+        .mhandler.cmd_new = qmp_marshal_ringbuf_write,
     },
 
 SQMP
@@ -508,8 +512,6 @@
 - "data": data to write (json-string)
 - "format": data format (json-string, optional)
           - Possible values: "utf8" (default), "base64"
-            Bug: invalid base64 is currently not rejected.
-            Whitespace *is* invalid.
 
 Example:
 
@@ -524,7 +526,7 @@
     {
         .name       = "ringbuf-read",
         .args_type  = "device:s,size:i,format:s?",
-        .mhandler.cmd_new = qmp_marshal_input_ringbuf_read,
+        .mhandler.cmd_new = qmp_marshal_ringbuf_read,
     },
 
 SQMP
@@ -560,7 +562,7 @@
     {
         .name       = "xen-save-devices-state",
         .args_type  = "filename:F",
-    .mhandler.cmd_new = qmp_marshal_input_xen_save_devices_state,
+    .mhandler.cmd_new = qmp_marshal_xen_save_devices_state,
     },
 
 SQMP
@@ -585,9 +587,36 @@
 EQMP
 
     {
+        .name       = "xen-load-devices-state",
+        .args_type  = "filename:F",
+        .mhandler.cmd_new = qmp_marshal_xen_load_devices_state,
+    },
+
+SQMP
+xen-load-devices-state
+----------------------
+
+Load the state of all devices from file. The RAM and the block devices
+of the VM are not loaded by this command.
+
+Arguments:
+
+- "filename": the file to load the state of the devices from as binary
+data. See xen-save-devices-state.txt for a description of the binary
+format.
+
+Example:
+
+-> { "execute": "xen-load-devices-state",
+     "arguments": { "filename": "/tmp/resume" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "xen-set-global-dirty-log",
         .args_type  = "enable:b",
-        .mhandler.cmd_new = qmp_marshal_input_xen_set_global_dirty_log,
+        .mhandler.cmd_new = qmp_marshal_xen_set_global_dirty_log,
     },
 
 SQMP
@@ -611,7 +640,7 @@
     {
         .name       = "migrate",
         .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
-        .mhandler.cmd_new = qmp_marshal_input_migrate,
+        .mhandler.cmd_new = qmp_marshal_migrate,
     },
 
 SQMP
@@ -644,7 +673,7 @@
     {
         .name       = "migrate_cancel",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_cancel,
+        .mhandler.cmd_new = qmp_marshal_migrate_cancel,
     },
 
 SQMP
@@ -661,10 +690,39 @@
 <- { "return": {} }
 
 EQMP
-{
+
+    {
+        .name       = "migrate-incoming",
+        .args_type  = "uri:s",
+        .mhandler.cmd_new = qmp_marshal_migrate_incoming,
+    },
+
+SQMP
+migrate-incoming
+----------------
+
+Continue an incoming migration
+
+Arguments:
+
+- "uri": Source/listening URI (json-string)
+
+Example:
+
+-> { "execute": "migrate-incoming", "arguments": { "uri": "tcp::4446" } }
+<- { "return": {} }
+
+Notes:
+
+(1) QEMU must be started with -incoming defer to allow migrate-incoming to
+    be used
+(2) The uri format is the same as for -incoming
+
+EQMP
+    {
         .name       = "migrate-set-cache-size",
         .args_type  = "value:o",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_cache_size,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_cache_size,
     },
 
 SQMP
@@ -685,9 +743,28 @@
 
 EQMP
     {
+        .name       = "migrate-start-postcopy",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_migrate_start_postcopy,
+    },
+
+SQMP
+migrate-start-postcopy
+----------------------
+
+Switch an in-progress migration to postcopy mode. Ignored after the end of
+migration (or once already in postcopy).
+
+Example:
+-> { "execute": "migrate-start-postcopy" }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-migrate-cache-size",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_cache_size,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_cache_size,
     },
 
 SQMP
@@ -709,7 +786,7 @@
     {
         .name       = "migrate_set_speed",
         .args_type  = "value:o",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_speed,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_speed,
     },
 
 SQMP
@@ -732,7 +809,7 @@
     {
         .name       = "migrate_set_downtime",
         .args_type  = "value:T",
-        .mhandler.cmd_new = qmp_marshal_input_migrate_set_downtime,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_downtime,
     },
 
 SQMP
@@ -756,25 +833,23 @@
         .name       = "client_migrate_info",
         .args_type  = "protocol:s,hostname:s,port:i?,tls-port:i?,cert-subject:s?",
         .params     = "protocol hostname port tls-port cert-subject",
-        .help       = "send migration info to spice/vnc client",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_async = client_migrate_info,
-        .flags      = MONITOR_CMD_ASYNC,
+        .help       = "set migration information for remote display",
+        .mhandler.cmd_new = qmp_marshal_client_migrate_info,
     },
 
 SQMP
 client_migrate_info
-------------------
+-------------------
 
-Set the spice/vnc connection info for the migration target.  The spice/vnc
-server will ask the spice/vnc client to automatically reconnect using the
-new parameters (if specified) once the vm migration finished successfully.
+Set migration information for remote display.  This makes the server
+ask the client to automatically reconnect using the new parameters
+once migration finished successfully.  Only implemented for SPICE.
 
 Arguments:
 
-- "protocol":     protocol: "spice" or "vnc" (json-string)
+- "protocol":     must be "spice" (json-string)
 - "hostname":     migration target hostname (json-string)
-- "port":         spice/vnc tcp port for plaintext channels (json-int, optional)
+- "port":         spice tcp port for plaintext channels (json-int, optional)
 - "tls-port":     spice tcp port for tls-secured channels (json-int, optional)
 - "cert-subject": server certificate subject (json-string, optional)
 
@@ -790,11 +865,10 @@
 
     {
         .name       = "dump-guest-memory",
-        .args_type  = "paging:b,protocol:s,begin:i?,end:i?,format:s?",
-        .params     = "-p protocol [begin] [length] [format]",
+        .args_type  = "paging:b,protocol:s,detach:b?,begin:i?,end:i?,format:s?",
+        .params     = "-p protocol [-d] [begin] [length] [format]",
         .help       = "dump guest memory to file",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = qmp_marshal_input_dump_guest_memory,
+        .mhandler.cmd_new = qmp_marshal_dump_guest_memory,
     },
 
 SQMP
@@ -808,6 +882,9 @@
 - "paging": do paging to get guest's memory mapping (json-bool)
 - "protocol": destination file(started with "file:") or destination file
               descriptor (started with "fd:") (json-string)
+- "detach": if specified, command will return immediately, without waiting
+            for the dump to finish. The user can track progress using
+            "query-dump". (json-bool)
 - "begin": the starting physical address. It's optional, and should be specified
            with length together (json-int)
 - "length": the memory size, in bytes. It's optional, and should be specified
@@ -830,7 +907,7 @@
     {
         .name       = "query-dump-guest-memory-capability",
         .args_type  = "",
-    .mhandler.cmd_new = qmp_marshal_input_query_dump_guest_memory_capability,
+    .mhandler.cmd_new = qmp_marshal_query_dump_guest_memory_capability,
     },
 
 SQMP
@@ -848,6 +925,55 @@
 EQMP
 
     {
+        .name       = "query-dump",
+        .args_type  = "",
+        .params     = "",
+        .help       = "query background dump status",
+        .mhandler.cmd_new = qmp_marshal_query_dump,
+    },
+
+SQMP
+query-dump
+----------
+
+Query background dump status.
+
+Arguments: None.
+
+Example:
+
+-> { "execute": "query-dump" }
+<- { "return": { "status": "active", "completed": 1024000,
+                 "total": 2048000 } }
+
+EQMP
+
+#if defined TARGET_S390X
+    {
+        .name       = "dump-skeys",
+        .args_type  = "filename:F",
+        .mhandler.cmd_new = qmp_marshal_dump_skeys,
+    },
+#endif
+
+SQMP
+dump-skeys
+----------
+
+Save guest storage keys to file.
+
+Arguments:
+
+- "filename": file path (json-string)
+
+Example:
+
+-> { "execute": "dump-skeys", "arguments": { "filename": "/tmp/skeys" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "netdev_add",
         .args_type  = "netdev:O",
         .mhandler.cmd_new = qmp_netdev_add,
@@ -867,7 +993,9 @@
 
 Example:
 
--> { "execute": "netdev_add", "arguments": { "type": "user", "id": "netdev1" } }
+-> { "execute": "netdev_add",
+     "arguments": { "type": "user", "id": "netdev1",
+                    "dnssearch": "example.org" } }
 <- { "return": {} }
 
 Note: The supported device options are the same ones supported by the '-netdev'
@@ -879,7 +1007,7 @@
     {
         .name       = "netdev_del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_netdev_del,
+        .mhandler.cmd_new = qmp_marshal_netdev_del,
     },
 
 SQMP
@@ -903,7 +1031,7 @@
     {
         .name       = "object-add",
         .args_type  = "qom-type:s,id:s,props:q?",
-        .mhandler.cmd_new = qmp_object_add,
+        .mhandler.cmd_new = qmp_marshal_object_add,
     },
 
 SQMP
@@ -929,7 +1057,7 @@
     {
         .name       = "object-del",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_object_del,
+        .mhandler.cmd_new = qmp_marshal_object_del,
     },
 
 SQMP
@@ -954,7 +1082,7 @@
     {
         .name       = "block_resize",
         .args_type  = "device:s?,node-name:s?,size:o",
-        .mhandler.cmd_new = qmp_marshal_input_block_resize,
+        .mhandler.cmd_new = qmp_marshal_block_resize,
     },
 
 SQMP
@@ -978,14 +1106,53 @@
 
     {
         .name       = "block-stream",
-        .args_type  = "device:B,base:s?,speed:o?,backing-file:s?,on-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_block_stream,
+        .args_type  = "job-id:s?,device:B,base:s?,speed:o?,backing-file:s?,on-error:s?",
+        .mhandler.cmd_new = qmp_marshal_block_stream,
     },
 
+SQMP
+block-stream
+------------
+
+Copy data from a backing file into a block device.
+
+Arguments:
+
+- "job-id": Identifier for the newly-created block job. If omitted,
+            the device name will be used. (json-string, optional)
+- "device": The device's ID, must be unique (json-string)
+- "base": The file name of the backing image above which copying starts
+          (json-string, optional)
+- "backing-file": The backing file string to write into the active layer. This
+                  filename is not validated.
+
+                  If a pathname string is such that it cannot be resolved by
+                  QEMU, that means that subsequent QMP or HMP commands must use
+                  node-names for the image in question, as filename lookup
+                  methods will fail.
+
+                  If not specified, QEMU will automatically determine the
+                  backing file string to use, or error out if there is no
+                  obvious choice.  Care should be taken when specifying the
+                  string, to specify a valid filename or protocol.
+                  (json-string, optional) (Since 2.1)
+- "speed":  the maximum speed, in bytes per second (json-int, optional)
+- "on-error": the action to take on an error (default 'report').  'stop' and
+              'enospc' can only be used if the block device supports io-status.
+              (json-string, optional) (Since 2.1)
+
+Example:
+
+-> { "execute": "block-stream", "arguments": { "device": "virtio0",
+                                               "base": "/tmp/master.qcow2" } }
+<- { "return": {} }
+
+EQMP
+
     {
         .name       = "block-commit",
-        .args_type  = "device:B,base:s?,top:s?,backing-file:s?,speed:o?",
-        .mhandler.cmd_new = qmp_marshal_input_block_commit,
+        .args_type  = "job-id:s?,device:B,base:s?,top:s?,backing-file:s?,speed:o?",
+        .mhandler.cmd_new = qmp_marshal_block_commit,
     },
 
 SQMP
@@ -997,6 +1164,8 @@
 
 Arguments:
 
+- "job-id": Identifier for the newly-created block job. If omitted,
+            the device name will be used. (json-string, optional)
 - "device": The device's ID, must be unique (json-string)
 - "base": The file name of the backing image to write data into.
           If not specified, this is the deepest backing image
@@ -1047,9 +1216,9 @@
 
     {
         .name       = "drive-backup",
-        .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
-                      "on-source-error:s?,on-target-error:s?",
-        .mhandler.cmd_new = qmp_marshal_input_drive_backup,
+        .args_type  = "job-id:s?,sync:s,device:B,target:s,speed:i?,mode:s?,"
+                      "format:s?,bitmap:s?,on-source-error:s?,on-target-error:s?",
+        .mhandler.cmd_new = qmp_marshal_drive_backup,
     },
 
 SQMP
@@ -1064,6 +1233,8 @@
 
 Arguments:
 
+- "job-id": Identifier for the newly-created block job. If omitted,
+            the device name will be used. (json-string, optional)
 - "device": the name of the device which should be copied.
             (json-string)
 - "target": the target of the new image. If the file exists, or if it is a
@@ -1075,8 +1246,10 @@
             (json-string, optional)
 - "sync": what parts of the disk image should be copied to the destination;
   possibilities include "full" for all the disk, "top" for only the sectors
-  allocated in the topmost image, or "none" to only replicate new I/O
-  (MirrorSyncMode).
+  allocated in the topmost image, "incremental" for only the dirty sectors in
+  the bitmap, or "none" to only replicate new I/O (MirrorSyncMode).
+- "bitmap": dirty bitmap name for sync==incremental. Must be present if sync
+            is "incremental", must NOT be present otherwise.
 - "mode": whether and how QEMU should create a new image
           (NewImageMode, optional, default 'absolute-paths')
 - "speed": the maximum speed, in bytes per second (json-int, optional)
@@ -1094,49 +1267,104 @@
                                                "sync": "full",
                                                "target": "backup.img" } }
 <- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-backup",
+        .args_type  = "job-id:s?,sync:s,device:B,target:B,speed:i?,"
+                      "on-source-error:s?,on-target-error:s?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_backup,
+    },
+
+SQMP
+blockdev-backup
+---------------
+
+The device version of drive-backup: this command takes an existing named device
+as backup target.
+
+Arguments:
+
+- "job-id": Identifier for the newly-created block job. If omitted,
+            the device name will be used. (json-string, optional)
+- "device": the name of the device which should be copied.
+            (json-string)
+- "target": the name of the backup target device. (json-string)
+- "sync": what parts of the disk image should be copied to the destination;
+          possibilities include "full" for all the disk, "top" for only the
+          sectors allocated in the topmost image, or "none" to only replicate
+          new I/O (MirrorSyncMode).
+- "speed": the maximum speed, in bytes per second (json-int, optional)
+- "on-source-error": the action to take on an error on the source, default
+                     'report'.  'stop' and 'enospc' can only be used
+                     if the block device supports io-status.
+                     (BlockdevOnError, optional)
+- "on-target-error": the action to take on an error on the target, default
+                     'report' (no limitations, since this applies to
+                     a different block device than device).
+                     (BlockdevOnError, optional)
+
+Example:
+-> { "execute": "blockdev-backup", "arguments": { "device": "src-id",
+                                                  "sync": "full",
+                                                  "target": "tgt-id" } }
+<- { "return": {} }
+
 EQMP
 
     {
         .name       = "block-job-set-speed",
         .args_type  = "device:B,speed:o",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_set_speed,
+        .mhandler.cmd_new = qmp_marshal_block_job_set_speed,
     },
 
     {
         .name       = "block-job-cancel",
         .args_type  = "device:B,force:b?",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_cancel,
+        .mhandler.cmd_new = qmp_marshal_block_job_cancel,
     },
     {
         .name       = "block-job-pause",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_pause,
+        .mhandler.cmd_new = qmp_marshal_block_job_pause,
     },
     {
         .name       = "block-job-resume",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_resume,
+        .mhandler.cmd_new = qmp_marshal_block_job_resume,
     },
     {
         .name       = "block-job-complete",
         .args_type  = "device:B",
-        .mhandler.cmd_new = qmp_marshal_input_block_job_complete,
+        .mhandler.cmd_new = qmp_marshal_block_job_complete,
     },
     {
         .name       = "transaction",
-        .args_type  = "actions:q",
-        .mhandler.cmd_new = qmp_marshal_input_transaction,
+        .args_type  = "actions:q,properties:q?",
+        .mhandler.cmd_new = qmp_marshal_transaction,
     },
 
 SQMP
 transaction
 -----------
 
-Atomically operate on one or more block devices.  The only supported operations
-for now are drive-backup, internal and external snapshotting.  A list of
-dictionaries is accepted, that contains the actions to be performed.
-If there is any failure performing any of the operations, all operations
-for the group are abandoned.
+Atomically operate on one or more block devices.  Operations that are
+currently supported:
+
+    - drive-backup
+    - blockdev-backup
+    - blockdev-snapshot-sync
+    - blockdev-snapshot-internal-sync
+    - abort
+    - block-dirty-bitmap-add
+    - block-dirty-bitmap-clear
+
+Refer to the qemu/qapi-schema.json file for minimum required QEMU
+versions for these operations.  A list of dictionaries is accepted,
+that contains the actions to be performed.  If there is any failure
+performing any of the operations, all operations for the group are
+abandoned.
 
 For external snapshots, the dictionary contains the device, the file to use for
 the new snapshot, and the format.  The default format, if not specified, is
@@ -1163,8 +1391,12 @@
 Arguments:
 
 actions array:
-    - "type": the operation to perform.  The only supported
-      value is "blockdev-snapshot-sync". (json-string)
+    - "type": the operation to perform (json-string).  Possible
+              values: "drive-backup", "blockdev-backup",
+                      "blockdev-snapshot-sync",
+                      "blockdev-snapshot-internal-sync",
+                      "abort", "block-dirty-bitmap-add",
+                      "block-dirty-bitmap-clear"
     - "data": a dictionary.  The contents depend on the value
       of "type".  When "type" is "blockdev-snapshot-sync":
       - "device": device name to snapshot (json-string)
@@ -1202,9 +1434,94 @@
 EQMP
 
     {
+        .name       = "block-dirty-bitmap-add",
+        .args_type  = "node:B,name:s,granularity:i?",
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_add,
+    },
+
+SQMP
+
+block-dirty-bitmap-add
+----------------------
+Since 2.4
+
+Create a dirty bitmap with a name on the device, and start tracking the writes.
+
+Arguments:
+
+- "node": device/node on which to create dirty bitmap (json-string)
+- "name": name of the new dirty bitmap (json-string)
+- "granularity": granularity to track writes with (int, optional)
+
+Example:
+
+-> { "execute": "block-dirty-bitmap-add", "arguments": { "node": "drive0",
+                                                   "name": "bitmap0" } }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "block-dirty-bitmap-remove",
+        .args_type  = "node:B,name:s",
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_remove,
+    },
+
+SQMP
+
+block-dirty-bitmap-remove
+-------------------------
+Since 2.4
+
+Stop write tracking and remove the dirty bitmap that was created with
+block-dirty-bitmap-add.
+
+Arguments:
+
+- "node": device/node on which to remove dirty bitmap (json-string)
+- "name": name of the dirty bitmap to remove (json-string)
+
+Example:
+
+-> { "execute": "block-dirty-bitmap-remove", "arguments": { "node": "drive0",
+                                                      "name": "bitmap0" } }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "block-dirty-bitmap-clear",
+        .args_type  = "node:B,name:s",
+        .mhandler.cmd_new = qmp_marshal_block_dirty_bitmap_clear,
+    },
+
+SQMP
+
+block-dirty-bitmap-clear
+------------------------
+Since 2.4
+
+Reset the dirty bitmap associated with a node so that an incremental backup
+from this point in time forward will only backup clusters modified after this
+clear operation.
+
+Arguments:
+
+- "node": device/node on which to remove dirty bitmap (json-string)
+- "name": name of the dirty bitmap to remove (json-string)
+
+Example:
+
+-> { "execute": "block-dirty-bitmap-clear", "arguments": { "node": "drive0",
+                                                           "name": "bitmap0" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "blockdev-snapshot-sync",
         .args_type  = "device:s?,node-name:s?,snapshot-file:s,snapshot-node-name:s?,format:s?,mode:s?",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_sync,
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot_sync,
     },
 
 SQMP
@@ -1238,9 +1555,47 @@
 EQMP
 
     {
+        .name       = "blockdev-snapshot",
+        .args_type  = "node:s,overlay:s",
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot,
+    },
+
+SQMP
+blockdev-snapshot
+-----------------
+Since 2.5
+
+Create a snapshot, by installing 'node' as the backing image of
+'overlay'. Additionally, if 'node' is associated with a block
+device, the block device changes to using 'overlay' as its new active
+image.
+
+Arguments:
+
+- "node": device that will have a snapshot created (json-string)
+- "overlay": device that will have 'node' as its backing image (json-string)
+
+Example:
+
+-> { "execute": "blockdev-add",
+                "arguments": { "options": { "driver": "qcow2",
+                                            "node-name": "node1534",
+                                            "file": { "driver": "file",
+                                                      "filename": "hd1.qcow2" },
+                                            "backing": "" } } }
+
+<- { "return": {} }
+
+-> { "execute": "blockdev-snapshot", "arguments": { "node": "ide-hd0",
+                                                    "overlay": "node1534" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "blockdev-snapshot-internal-sync",
         .args_type  = "device:B,name:s",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_snapshot_internal_sync,
+        .mhandler.cmd_new = qmp_marshal_blockdev_snapshot_internal_sync,
     },
 
 SQMP
@@ -1270,7 +1625,7 @@
         .name       = "blockdev-snapshot-delete-internal-sync",
         .args_type  = "device:B,id:s?,name:s?",
         .mhandler.cmd_new =
-                      qmp_marshal_input_blockdev_snapshot_delete_internal_sync,
+                      qmp_marshal_blockdev_snapshot_delete_internal_sync,
     },
 
 SQMP
@@ -1309,11 +1664,12 @@
 
     {
         .name       = "drive-mirror",
-        .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
-                      "node-name:s?,replaces:s?,"
+        .args_type  = "job-id:s?,sync:s,device:B,target:s,speed:i?,mode:s?,"
+                      "format:s?,node-name:s?,replaces:s?,"
                       "on-source-error:s?,on-target-error:s?,"
+                      "unmap:b?,"
                       "granularity:i?,buf-size:i?",
-        .mhandler.cmd_new = qmp_marshal_input_drive_mirror,
+        .mhandler.cmd_new = qmp_marshal_drive_mirror,
     },
 
 SQMP
@@ -1329,6 +1685,8 @@
 
 Arguments:
 
+- "job-id": Identifier for the newly-created block job. If omitted,
+            the device name will be used. (json-string, optional)
 - "device": device name to operate on (json-string)
 - "target": name of new image file (json-string)
 - "format": format of new image (json-string, optional)
@@ -1341,6 +1699,61 @@
 - "speed": maximum speed of the streaming job, in bytes per second
   (json-int)
 - "granularity": granularity of the dirty bitmap, in bytes (json-int, optional)
+- "buf-size": maximum amount of data in flight from source to target, in bytes
+  (json-int, default 10M)
+- "sync": what parts of the disk image should be copied to the destination;
+  possibilities include "full" for all the disk, "top" for only the sectors
+  allocated in the topmost image, or "none" to only replicate new I/O
+  (MirrorSyncMode).
+- "on-source-error": the action to take on an error on the source
+  (BlockdevOnError, default 'report')
+- "on-target-error": the action to take on an error on the target
+  (BlockdevOnError, default 'report')
+- "unmap": whether the target sectors should be discarded where source has only
+  zeroes. (json-bool, optional, default true)
+
+The default value of the granularity is the image cluster size clamped
+between 4096 and 65536, if the image format defines one.  If the format
+does not define a cluster size, the default value of the granularity
+is 65536.
+
+
+Example:
+
+-> { "execute": "drive-mirror", "arguments": { "device": "ide-hd0",
+                                               "target": "/some/place/my-image",
+                                               "sync": "full",
+                                               "format": "qcow2" } }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-mirror",
+        .args_type  = "job-id:s?,sync:s,device:B,target:B,replaces:s?,speed:i?,"
+                      "on-source-error:s?,on-target-error:s?,"
+                      "granularity:i?,buf-size:i?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_mirror,
+    },
+
+SQMP
+blockdev-mirror
+------------
+
+Start mirroring a block device's writes to another block device. target
+specifies the target of mirror operation.
+
+Arguments:
+
+- "job-id": Identifier for the newly-created block job. If omitted,
+            the device name will be used. (json-string, optional)
+- "device": device name to operate on (json-string)
+- "target": device name to mirror to (json-string)
+- "replaces": the block driver node name to replace when finished
+              (json-string, optional)
+- "speed": maximum speed of the streaming job, in bytes per second
+  (json-int)
+- "granularity": granularity of the dirty bitmap, in bytes (json-int, optional)
 - "buf_size": maximum amount of data in flight from source to target, in bytes
   (json-int, default 10M)
 - "sync": what parts of the disk image should be copied to the destination;
@@ -1357,21 +1770,18 @@
 does not define a cluster size, the default value of the granularity
 is 65536.
 
-
 Example:
 
--> { "execute": "drive-mirror", "arguments": { "device": "ide-hd0",
-                                               "target": "/some/place/my-image",
-                                               "sync": "full",
-                                               "format": "qcow2" } }
+-> { "execute": "blockdev-mirror", "arguments": { "device": "ide-hd0",
+                                                  "target": "target0",
+                                                  "sync": "full" } }
 <- { "return": {} }
 
 EQMP
-
     {
         .name       = "change-backing-file",
         .args_type  = "device:s,image-node-name:s,backing-file:s",
-        .mhandler.cmd_new = qmp_marshal_input_change_backing_file,
+        .mhandler.cmd_new = qmp_marshal_change_backing_file,
     },
 
 SQMP
@@ -1410,7 +1820,7 @@
     {
         .name       = "balloon",
         .args_type  = "value:M",
-        .mhandler.cmd_new = qmp_marshal_input_balloon,
+        .mhandler.cmd_new = qmp_marshal_balloon,
     },
 
 SQMP
@@ -1433,7 +1843,7 @@
     {
         .name       = "set_link",
         .args_type  = "name:s,up:b",
-        .mhandler.cmd_new = qmp_marshal_input_set_link,
+        .mhandler.cmd_new = qmp_marshal_set_link,
     },
 
 SQMP
@@ -1459,7 +1869,7 @@
         .args_type  = "fdname:s",
         .params     = "getfd name",
         .help       = "receive a file descriptor via SCM rights and assign it a name",
-        .mhandler.cmd_new = qmp_marshal_input_getfd,
+        .mhandler.cmd_new = qmp_marshal_getfd,
     },
 
 SQMP
@@ -1492,7 +1902,7 @@
         .args_type  = "fdname:s",
         .params     = "closefd name",
         .help       = "close a file descriptor previously passed via SCM rights",
-        .mhandler.cmd_new = qmp_marshal_input_closefd,
+        .mhandler.cmd_new = qmp_marshal_closefd,
     },
 
 SQMP
@@ -1517,7 +1927,7 @@
         .args_type  = "fdset-id:i?,opaque:s?",
         .params     = "add-fd fdset-id opaque",
         .help       = "Add a file descriptor, that was passed via SCM rights, to an fd set",
-        .mhandler.cmd_new = qmp_marshal_input_add_fd,
+        .mhandler.cmd_new = qmp_marshal_add_fd,
     },
 
 SQMP
@@ -1556,7 +1966,7 @@
         .args_type  = "fdset-id:i,fd:i?",
         .params     = "remove-fd fdset-id fd",
         .help       = "Remove a file descriptor from an fd set",
-        .mhandler.cmd_new = qmp_marshal_input_remove_fd,
+        .mhandler.cmd_new = qmp_marshal_remove_fd,
     },
 
 SQMP
@@ -1588,7 +1998,7 @@
         .name       = "query-fdsets",
         .args_type  = "",
         .help       = "Return information describing all fd sets",
-        .mhandler.cmd_new = qmp_marshal_input_query_fdsets,
+        .mhandler.cmd_new = qmp_marshal_query_fdsets,
     },
 
 SQMP
@@ -1637,7 +2047,7 @@
     {
         .name       = "block_passwd",
         .args_type  = "device:s?,node-name:s?,password:s",
-        .mhandler.cmd_new = qmp_marshal_input_block_passwd,
+        .mhandler.cmd_new = qmp_marshal_block_passwd,
     },
 
 SQMP
@@ -1662,8 +2072,8 @@
 
     {
         .name       = "block_set_io_throttle",
-        .args_type  = "device:B,bps:l,bps_rd:l,bps_wr:l,iops:l,iops_rd:l,iops_wr:l,bps_max:l?,bps_rd_max:l?,bps_wr_max:l?,iops_max:l?,iops_rd_max:l?,iops_wr_max:l?,iops_size:l?",
-        .mhandler.cmd_new = qmp_marshal_input_block_set_io_throttle,
+        .args_type  = "device:B,bps:l,bps_rd:l,bps_wr:l,iops:l,iops_rd:l,iops_wr:l,bps_max:l?,bps_rd_max:l?,bps_wr_max:l?,iops_max:l?,iops_rd_max:l?,iops_wr_max:l?,bps_max_length:l?,bps_rd_max_length:l?,bps_wr_max_length:l?,iops_max_length:l?,iops_rd_max_length:l?,iops_wr_max_length:l?,iops_size:l?,group:s?",
+        .mhandler.cmd_new = qmp_marshal_block_set_io_throttle,
     },
 
 SQMP
@@ -1681,13 +2091,20 @@
 - "iops": total I/O operations per second (json-int)
 - "iops_rd": read I/O operations per second (json-int)
 - "iops_wr": write I/O operations per second (json-int)
-- "bps_max":  total max in bytes (json-int)
-- "bps_rd_max":  read max in bytes (json-int)
-- "bps_wr_max":  write max in bytes (json-int)
-- "iops_max":  total I/O operations max (json-int)
-- "iops_rd_max":  read I/O operations max (json-int)
-- "iops_wr_max":  write I/O operations max (json-int)
-- "iops_size":  I/O size in bytes when limiting (json-int)
+- "bps_max": total throughput limit during bursts, in bytes (json-int, optional)
+- "bps_rd_max": read throughput limit during bursts, in bytes (json-int, optional)
+- "bps_wr_max": write throughput limit during bursts, in bytes (json-int, optional)
+- "iops_max": total I/O operations per second during bursts (json-int, optional)
+- "iops_rd_max": read I/O operations per second during bursts (json-int, optional)
+- "iops_wr_max": write I/O operations per second during bursts (json-int, optional)
+- "bps_max_length": maximum length of the @bps_max burst period, in seconds (json-int, optional)
+- "bps_rd_max_length": maximum length of the @bps_rd_max burst period, in seconds (json-int, optional)
+- "bps_wr_max_length": maximum length of the @bps_wr_max burst period, in seconds (json-int, optional)
+- "iops_max_length": maximum length of the @iops_max burst period, in seconds (json-int, optional)
+- "iops_rd_max_length": maximum length of the @iops_rd_max burst period, in seconds (json-int, optional)
+- "iops_wr_max_length": maximum length of the @iops_wr_max burst period, in seconds (json-int, optional)
+- "iops_size":  I/O size in bytes when limiting (json-int, optional)
+- "group": throttle group name (json-string, optional)
 
 Example:
 
@@ -1704,6 +2121,7 @@
                                                "iops_max": 0,
                                                "iops_rd_max": 0,
                                                "iops_wr_max": 0,
+                                               "bps_max_length": 60,
                                                "iops_size": 0 } }
 <- { "return": {} }
 
@@ -1712,7 +2130,7 @@
     {
         .name       = "set_password",
         .args_type  = "protocol:s,password:s,connected:s?",
-        .mhandler.cmd_new = qmp_marshal_input_set_password,
+        .mhandler.cmd_new = qmp_marshal_set_password,
     },
 
 SQMP
@@ -1725,7 +2143,7 @@
 
 - "protocol": protocol name (json-string)
 - "password": password (json-string)
-- "connected": [ keep | disconnect | fail ] (josn-string, optional)
+- "connected": [ keep | disconnect | fail ] (json-string, optional)
 
 Example:
 
@@ -1738,7 +2156,7 @@
     {
         .name       = "expire_password",
         .args_type  = "protocol:s,time:s",
-        .mhandler.cmd_new = qmp_marshal_input_expire_password,
+        .mhandler.cmd_new = qmp_marshal_expire_password,
     },
 
 SQMP
@@ -1763,7 +2181,7 @@
     {
         .name       = "add_client",
         .args_type  = "protocol:s,fdname:s,skipauth:b?,tls:b?",
-        .mhandler.cmd_new = qmp_marshal_input_add_client,
+        .mhandler.cmd_new = qmp_marshal_add_client,
     },
 
 SQMP
@@ -1791,8 +2209,7 @@
         .args_type  = "",
         .params     = "",
         .help       = "enable QMP capabilities",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_qmp_capabilities,
+        .mhandler.cmd_new = qmp_capabilities,
     },
 
 SQMP
@@ -1815,7 +2232,7 @@
     {
         .name       = "human-monitor-command",
         .args_type  = "command-line:s,cpu-index:i?",
-        .mhandler.cmd_new = qmp_marshal_input_human_monitor_command,
+        .mhandler.cmd_new = qmp_marshal_human_monitor_command,
     },
 
 SQMP
@@ -1894,7 +2311,7 @@
     {
         .name       = "query-version",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_version,
+        .mhandler.cmd_new = qmp_marshal_query_version,
     },
 
 SQMP
@@ -1931,7 +2348,7 @@
     {
         .name       = "query-commands",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_commands,
+        .mhandler.cmd_new = qmp_marshal_query_commands,
     },
 
 SQMP
@@ -1968,7 +2385,24 @@
     {
         .name       = "query-events",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_events,
+        .mhandler.cmd_new = qmp_marshal_query_events,
+    },
+
+SQMP
+query-qmp-schema
+----------------
+
+Return the QMP wire schema.  The returned value is a json-array of
+named schema entities.  Entities are commands, events and various
+types.  See docs/qapi-code-gen.txt for information on their structure
+and intended use.
+
+EQMP
+
+    {
+        .name       = "query-qmp-schema",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_query_qmp_schema,
     },
 
 SQMP
@@ -2013,7 +2447,7 @@
     {
         .name       = "query-chardev",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_chardev,
+        .mhandler.cmd_new = qmp_marshal_query_chardev,
     },
 
 SQMP
@@ -2054,7 +2488,7 @@
     {
         .name       = "query-chardev-backends",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_chardev_backends,
+        .mhandler.cmd_new = qmp_marshal_query_chardev_backends,
     },
 
 SQMP
@@ -2083,7 +2517,7 @@
          - "drv": driver format name (json-string)
              - Possible values: "blkdebug", "bochs", "cloop", "dmg",
                                 "file", "file", "ftp", "ftps", "host_cdrom",
-                                "host_device", "host_floppy", "http", "https",
+                                "host_device", "http", "https",
                                 "nbd", "parallels", "qcow", "qcow2", "raw",
                                 "tftp", "vdi", "vmdk", "vpc", "vvfat"
          - "backing_file": backing file name (json-string, optional)
@@ -2104,6 +2538,8 @@
          - "iops_size": I/O size when limiting by iops (json-int)
          - "detect_zeroes": detect and optimize zero writing (json-string)
              - Possible values: "off", "on", "unmap"
+         - "write_threshold": write offset threshold in bytes, a event will be
+                              emitted if crossed. Zero if disabled (json-int)
          - "image": the detail of the image, it is a json-object containing
             the following:
              - "filename": image file name (json-string)
@@ -2181,13 +2617,14 @@
                "iops_wr_max": 0,
                "iops_size": 0,
                "detect_zeroes": "on",
+               "write_threshold": 0,
                "image":{
                   "filename":"disks/test.qcow2",
                   "format":"qcow2",
                   "virtual-size":2048000,
                   "backing_file":"base.qcow2",
                   "full-backing-filename":"disks/base.qcow2",
-                  "backing-filename-format:"qcow2",
+                  "backing-filename-format":"qcow2",
                   "snapshots":[
                      {
                         "id": "1",
@@ -2235,7 +2672,7 @@
     {
         .name       = "query-block",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_block,
+        .mhandler.cmd_new = qmp_marshal_query_block,
     },
 
 SQMP
@@ -2259,8 +2696,70 @@
     - "wr_total_time_ns": total time spend on writes in nano-seconds (json-int)
     - "rd_total_time_ns": total time spend on reads in nano-seconds (json-int)
     - "flush_total_time_ns": total time spend on cache flushes in nano-seconds (json-int)
-    - "wr_highest_offset": Highest offset of a sector written since the
-                           BlockDriverState has been opened (json-int)
+    - "wr_highest_offset": The offset after the greatest byte written to the
+                           BlockDriverState since it has been opened (json-int)
+    - "rd_merged": number of read requests that have been merged into
+                   another request (json-int)
+    - "wr_merged": number of write requests that have been merged into
+                   another request (json-int)
+    - "idle_time_ns": time since the last I/O operation, in
+                      nanoseconds. If the field is absent it means
+                      that there haven't been any operations yet
+                      (json-int, optional)
+    - "failed_rd_operations": number of failed read operations
+                              (json-int)
+    - "failed_wr_operations": number of failed write operations
+                              (json-int)
+    - "failed_flush_operations": number of failed flush operations
+                               (json-int)
+    - "invalid_rd_operations": number of invalid read operations
+                               (json-int)
+    - "invalid_wr_operations": number of invalid write operations
+                               (json-int)
+    - "invalid_flush_operations": number of invalid flush operations
+                                  (json-int)
+    - "account_invalid": whether invalid operations are included in
+                         the last access statistics (json-bool)
+    - "account_failed": whether failed operations are included in the
+                         latency and last access statistics
+                         (json-bool)
+    - "timed_stats": A json-array containing statistics collected in
+                     specific intervals, with the following members:
+        - "interval_length": interval used for calculating the
+                             statistics, in seconds (json-int)
+        - "min_rd_latency_ns": minimum latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "min_wr_latency_ns": minimum latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "min_flush_latency_ns": minimum latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "max_rd_latency_ns": maximum latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "max_wr_latency_ns": maximum latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "max_flush_latency_ns": maximum latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "avg_rd_latency_ns": average latency of read operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "avg_wr_latency_ns": average latency of write operations in
+                               the defined interval, in nanoseconds
+                               (json-int)
+        - "avg_flush_latency_ns": average latency of flush operations
+                                  in the defined interval, in
+                                  nanoseconds (json-int)
+        - "avg_rd_queue_depth": average number of pending read
+                                operations in the defined interval
+                                (json-number)
+        - "avg_wr_queue_depth": average number of pending write
+                                operations in the defined interval
+                                (json-number).
 - "parent": Contains recursively the statistics of the underlying
             protocol (e.g. the host file for a qcow2 image). If there is
             no underlying protocol, this field is omitted
@@ -2284,6 +2783,11 @@
                   "rd_total_times_ns":3465673657
                   "flush_total_times_ns":49653
                   "flush_operations":61,
+                  "rd_merged":0,
+                  "wr_merged":0,
+                  "idle_time_ns":2953431879,
+                  "account_invalid":true,
+                  "account_failed":false
                }
             },
             "stats":{
@@ -2295,7 +2799,12 @@
                "flush_operations":51,
                "wr_total_times_ns":313253456
                "rd_total_times_ns":3465673657
-               "flush_total_times_ns":49653
+               "flush_total_times_ns":49653,
+               "rd_merged":0,
+               "wr_merged":0,
+               "idle_time_ns":2953431879,
+               "account_invalid":true,
+               "account_failed":false
             }
          },
          {
@@ -2309,7 +2818,11 @@
                "flush_operations":0,
                "wr_total_times_ns":0
                "rd_total_times_ns":0
-               "flush_total_times_ns":0
+               "flush_total_times_ns":0,
+               "rd_merged":0,
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          },
          {
@@ -2323,7 +2836,11 @@
                "flush_operations":0,
                "wr_total_times_ns":0
                "rd_total_times_ns":0
-               "flush_total_times_ns":0
+               "flush_total_times_ns":0,
+               "rd_merged":0,
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          },
          {
@@ -2337,7 +2854,11 @@
                "flush_operations":0,
                "wr_total_times_ns":0
                "rd_total_times_ns":0
-               "flush_total_times_ns":0
+               "flush_total_times_ns":0,
+               "rd_merged":0,
+               "wr_merged":0,
+               "account_invalid":false,
+               "account_failed":false
             }
          }
       ]
@@ -2347,8 +2868,8 @@
 
     {
         .name       = "query-blockstats",
-        .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_blockstats,
+        .args_type  = "query-nodes:b?",
+        .mhandler.cmd_new = qmp_marshal_query_blockstats,
     },
 
 SQMP
@@ -2362,6 +2883,9 @@
 - "CPU": CPU index (json-int)
 - "current": true if this is the current CPU, false otherwise (json-bool)
 - "halted": true if the cpu is halted, false otherwise (json-bool)
+- "qom_path": path to the CPU object in the QOM tree (json-str)
+- "arch": architecture of the cpu, which determines what additional
+          keys will be present (json-str)
 - Current program counter. The key's name depends on the architecture:
      "pc": i386/x86_64 (json-int)
      "nip": PPC (json-int)
@@ -2378,14 +2902,18 @@
             "CPU":0,
             "current":true,
             "halted":false,
-            "pc":3227107138
+            "qom_path":"/machine/unattached/device[0]",
+            "arch":"x86",
+            "pc":3227107138,
             "thread_id":3134
          },
          {
             "CPU":1,
             "current":false,
             "halted":true,
-            "pc":7108165
+            "qom_path":"/machine/unattached/device[2]",
+            "arch":"x86",
+            "pc":7108165,
             "thread_id":3135
          }
       ]
@@ -2396,7 +2924,7 @@
     {
         .name       = "query-cpus",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_cpus,
+        .mhandler.cmd_new = qmp_marshal_query_cpus,
     },
 
 SQMP
@@ -2435,7 +2963,7 @@
     {
         .name       = "query-iothreads",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_iothreads,
+        .mhandler.cmd_new = qmp_marshal_query_iothreads,
     },
 
 SQMP
@@ -2652,7 +3180,7 @@
     {
         .name       = "query-pci",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_pci,
+        .mhandler.cmd_new = qmp_marshal_query_pci,
     },
 
 SQMP
@@ -2676,7 +3204,7 @@
     {
         .name       = "query-kvm",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_kvm,
+        .mhandler.cmd_new = qmp_marshal_query_kvm,
     },
 
 SQMP
@@ -2716,7 +3244,7 @@
     {
         .name       = "query-status",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_status,
+        .mhandler.cmd_new = qmp_marshal_query_status,
     },
 
 SQMP
@@ -2760,7 +3288,7 @@
     {
         .name       = "query-mice",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_mice,
+        .mhandler.cmd_new = qmp_marshal_query_mice,
     },
 
 SQMP
@@ -2823,7 +3351,12 @@
     {
         .name       = "query-vnc",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_vnc,
+        .mhandler.cmd_new = qmp_marshal_query_vnc,
+    },
+    {
+        .name       = "query-vnc-servers",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_query_vnc_servers,
     },
 
 SQMP
@@ -2858,7 +3391,7 @@
 - "channel-id": channel id.  Usually "0", might be different needed when
                 multiple channels of the same type exist, such as multiple
                 display channels in a multihead setup (json-int)
-- "tls": whevener the channel is encrypted (json-bool)
+- "tls": whether the channel is encrypted (json-bool)
 
 Example:
 
@@ -2900,7 +3433,7 @@
     {
         .name       = "query-spice",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_spice,
+        .mhandler.cmd_new = qmp_marshal_query_spice,
     },
 #endif
 
@@ -2924,7 +3457,7 @@
     {
         .name       = "query-name",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_name,
+        .mhandler.cmd_new = qmp_marshal_query_name,
     },
 
 SQMP
@@ -2947,7 +3480,7 @@
     {
         .name       = "query-uuid",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_uuid,
+        .mhandler.cmd_new = qmp_marshal_query_uuid,
     },
 
 SQMP
@@ -2996,7 +3529,7 @@
     {
         .name       = "query-command-line-options",
         .args_type  = "option:s?",
-        .mhandler.cmd_new = qmp_marshal_input_query_command_line_options,
+        .mhandler.cmd_new = qmp_marshal_query_command_line_options,
     },
 
 SQMP
@@ -3174,7 +3707,7 @@
     {
         .name       = "query-migrate",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate,
+        .mhandler.cmd_new = qmp_marshal_query_migrate,
     },
 
 SQMP
@@ -3184,6 +3717,12 @@
 Enable/Disable migration capabilities
 
 - "xbzrle": XBZRLE support
+- "rdma-pin-all": pin all pages when using RDMA during migration
+- "auto-converge": throttle down guest to help convergence of migration
+- "zero-blocks": compress zero blocks during block migration
+- "compress": use multiple compression threads to accelerate live migration
+- "events": generate events for each migration state change
+- "postcopy-ram": postcopy mode for live migration
 
 Arguments:
 
@@ -3196,9 +3735,9 @@
 
     {
         .name       = "migrate-set-capabilities",
-        .args_type  = "capabilities:O",
+        .args_type  = "capabilities:q",
         .params     = "capability:s,state:b",
-	.mhandler.cmd_new = qmp_marshal_input_migrate_set_capabilities,
+        .mhandler.cmd_new = qmp_marshal_migrate_set_capabilities,
     },
 SQMP
 query-migrate-capabilities
@@ -3208,20 +3747,101 @@
 
 - "capabilities": migration capabilities state
          - "xbzrle" : XBZRLE state (json-bool)
+         - "rdma-pin-all" : RDMA Pin Page state (json-bool)
+         - "auto-converge" : Auto Converge state (json-bool)
+         - "zero-blocks" : Zero Blocks state (json-bool)
+         - "compress": Multiple compression threads state (json-bool)
+         - "events": Migration state change event state (json-bool)
+         - "postcopy-ram": postcopy ram state (json-bool)
 
 Arguments:
 
 Example:
 
 -> { "execute": "query-migrate-capabilities" }
-<- { "return": [ { "state": false, "capability": "xbzrle" } ] }
+<- {"return": [
+     {"state": false, "capability": "xbzrle"},
+     {"state": false, "capability": "rdma-pin-all"},
+     {"state": false, "capability": "auto-converge"},
+     {"state": false, "capability": "zero-blocks"},
+     {"state": false, "capability": "compress"},
+     {"state": true, "capability": "events"},
+     {"state": false, "capability": "postcopy-ram"}
+   ]}
 
 EQMP
 
     {
         .name       = "query-migrate-capabilities",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_migrate_capabilities,
+        .mhandler.cmd_new = qmp_marshal_query_migrate_capabilities,
+    },
+
+SQMP
+migrate-set-parameters
+----------------------
+
+Set migration parameters
+
+- "compress-level": set compression level during migration (json-int)
+- "compress-threads": set compression thread count for migration (json-int)
+- "decompress-threads": set decompression thread count for migration (json-int)
+- "cpu-throttle-initial": set initial percentage of time guest cpus are
+                          throttled for auto-converge (json-int)
+- "cpu-throttle-increment": set throttle increasing percentage for
+                            auto-converge (json-int)
+
+Arguments:
+
+Example:
+
+-> { "execute": "migrate-set-parameters" , "arguments":
+      { "compress-level": 1 } }
+
+EQMP
+
+    {
+        .name       = "migrate-set-parameters",
+        .args_type  =
+            "compress-level:i?,compress-threads:i?,decompress-threads:i?,cpu-throttle-initial:i?,cpu-throttle-increment:i?",
+        .mhandler.cmd_new = qmp_marshal_migrate_set_parameters,
+    },
+SQMP
+query-migrate-parameters
+------------------------
+
+Query current migration parameters
+
+- "parameters": migration parameters value
+         - "compress-level" : compression level value (json-int)
+         - "compress-threads" : compression thread count value (json-int)
+         - "decompress-threads" : decompression thread count value (json-int)
+         - "cpu-throttle-initial" : initial percentage of time guest cpus are
+                                    throttled (json-int)
+         - "cpu-throttle-increment" : throttle increasing percentage for
+                                      auto-converge (json-int)
+
+Arguments:
+
+Example:
+
+-> { "execute": "query-migrate-parameters" }
+<- {
+      "return": {
+         "decompress-threads": 2,
+         "cpu-throttle-increment": 10,
+         "compress-threads": 8,
+         "compress-level": 1,
+         "cpu-throttle-initial": 20
+      }
+   }
+
+EQMP
+
+    {
+        .name       = "query-migrate-parameters",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_query_migrate_parameters,
     },
 
 SQMP
@@ -3249,88 +3869,88 @@
     {
         .name       = "query-balloon",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_balloon,
+        .mhandler.cmd_new = qmp_marshal_query_balloon,
     },
 
     {
         .name       = "query-block-jobs",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_block_jobs,
+        .mhandler.cmd_new = qmp_marshal_query_block_jobs,
     },
 
     {
         .name       = "qom-list",
         .args_type  = "path:s",
-        .mhandler.cmd_new = qmp_marshal_input_qom_list,
+        .mhandler.cmd_new = qmp_marshal_qom_list,
     },
 
     {
         .name       = "qom-set",
 	.args_type  = "path:s,property:s,value:q",
-	.mhandler.cmd_new = qmp_qom_set,
+        .mhandler.cmd_new = qmp_marshal_qom_set,
     },
 
     {
         .name       = "qom-get",
 	.args_type  = "path:s,property:s",
-	.mhandler.cmd_new = qmp_qom_get,
+        .mhandler.cmd_new = qmp_marshal_qom_get,
     },
 
     {
         .name       = "nbd-server-start",
-        .args_type  = "addr:q",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_start,
+        .args_type  = "addr:q,tls-creds:s?",
+        .mhandler.cmd_new = qmp_marshal_nbd_server_start,
     },
     {
         .name       = "nbd-server-add",
         .args_type  = "device:B,writable:b?",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_add,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_add,
     },
     {
         .name       = "nbd-server-stop",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_nbd_server_stop,
+        .mhandler.cmd_new = qmp_marshal_nbd_server_stop,
     },
 
     {
         .name       = "change-vnc-password",
         .args_type  = "password:s",
-        .mhandler.cmd_new = qmp_marshal_input_change_vnc_password,
+        .mhandler.cmd_new = qmp_marshal_change_vnc_password,
     },
     {
         .name       = "qom-list-types",
         .args_type  = "implements:s?,abstract:b?",
-        .mhandler.cmd_new = qmp_marshal_input_qom_list_types,
+        .mhandler.cmd_new = qmp_marshal_qom_list_types,
     },
 
     {
         .name       = "device-list-properties",
         .args_type  = "typename:s",
-        .mhandler.cmd_new = qmp_marshal_input_device_list_properties,
+        .mhandler.cmd_new = qmp_marshal_device_list_properties,
     },
 
     {
         .name       = "query-machines",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_machines,
+        .mhandler.cmd_new = qmp_marshal_query_machines,
     },
 
     {
         .name       = "query-cpu-definitions",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_cpu_definitions,
+        .mhandler.cmd_new = qmp_marshal_query_cpu_definitions,
     },
 
     {
         .name       = "query-target",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_target,
+        .mhandler.cmd_new = qmp_marshal_query_target,
     },
 
     {
         .name       = "query-tpm",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm,
+        .mhandler.cmd_new = qmp_marshal_query_tpm,
     },
 
 SQMP
@@ -3364,7 +3984,7 @@
     {
         .name       = "query-tpm-models",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm_models,
+        .mhandler.cmd_new = qmp_marshal_query_tpm_models,
     },
 
 SQMP
@@ -3385,7 +4005,7 @@
     {
         .name       = "query-tpm-types",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_tpm_types,
+        .mhandler.cmd_new = qmp_marshal_query_tpm_types,
     },
 
 SQMP
@@ -3406,7 +4026,7 @@
     {
         .name       = "chardev-add",
         .args_type  = "id:s,backend:q",
-        .mhandler.cmd_new = qmp_marshal_input_chardev_add,
+        .mhandler.cmd_new = qmp_marshal_chardev_add,
     },
 
 SQMP
@@ -3443,7 +4063,7 @@
     {
         .name       = "chardev-remove",
         .args_type  = "id:s",
-        .mhandler.cmd_new = qmp_marshal_input_chardev_remove,
+        .mhandler.cmd_new = qmp_marshal_chardev_remove,
     },
 
 
@@ -3466,7 +4086,7 @@
     {
         .name       = "query-rx-filter",
         .args_type  = "name:s?",
-        .mhandler.cmd_new = qmp_marshal_input_query_rx_filter,
+        .mhandler.cmd_new = qmp_marshal_query_rx_filter,
     },
 
 SQMP
@@ -3532,7 +4152,7 @@
     {
         .name       = "blockdev-add",
         .args_type  = "options:q",
-        .mhandler.cmd_new = qmp_marshal_input_blockdev_add,
+        .mhandler.cmd_new = qmp_marshal_blockdev_add,
     },
 
 SQMP
@@ -3541,6 +4161,10 @@
 
 Add a block device.
 
+This command is still a work in progress.  It doesn't support all
+block drivers among other things.  Stay away from it unless you want
+to help with its development.
+
 Arguments:
 
 - "options": block driver options
@@ -3585,9 +4209,290 @@
 EQMP
 
     {
+        .name       = "x-blockdev-del",
+        .args_type  = "id:s?,node-name:s?",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_del,
+    },
+
+SQMP
+x-blockdev-del
+------------
+Since 2.5
+
+Deletes a block device thas has been added using blockdev-add.
+The selected device can be either a block backend or a graph node.
+
+In the former case the backend will be destroyed, along with its
+inserted medium if there's any. The command will fail if the backend
+or its medium are in use.
+
+In the latter case the node will be destroyed. The command will fail
+if the node is attached to a block backend or is otherwise being
+used.
+
+One of "id" or "node-name" must be specified, but not both.
+
+This command is still a work in progress and is considered
+experimental. Stay away from it unless you want to help with its
+development.
+
+Arguments:
+
+- "id": Name of the block backend device to delete (json-string, optional)
+- "node-name": Name of the graph node to delete (json-string, optional)
+
+Example:
+
+-> { "execute": "blockdev-add",
+     "arguments": {
+         "options": {
+             "driver": "qcow2",
+             "id": "drive0",
+             "file": {
+                 "driver": "file",
+                 "filename": "test.qcow2"
+             }
+         }
+     }
+   }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-del",
+     "arguments": { "id": "drive0" }
+   }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-open-tray",
+        .args_type  = "device:s,force:b?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_open_tray,
+    },
+
+SQMP
+blockdev-open-tray
+------------------
+
+Opens a block device's tray. If there is a block driver state tree inserted as a
+medium, it will become inaccessible to the guest (but it will remain associated
+to the block device, so closing the tray will make it accessible again).
+
+If the tray was already open before, this will be a no-op.
+
+Once the tray opens, a DEVICE_TRAY_MOVED event is emitted. There are cases in
+which no such event will be generated, these include:
+- if the guest has locked the tray, @force is false and the guest does not
+  respond to the eject request
+- if the BlockBackend denoted by @device does not have a guest device attached
+  to it
+- if the guest device does not have an actual tray and is empty, for instance
+  for floppy disk drives
+
+Arguments:
+
+- "device": block device name (json-string)
+- "force": if false (the default), an eject request will be sent to the guest if
+           it has locked the tray (and the tray will not be opened immediately);
+           if true, the tray will be opened regardless of whether it is locked
+           (json-bool, optional)
+
+Example:
+
+-> { "execute": "blockdev-open-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751016,
+                    "microseconds": 716996 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": true } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "blockdev-close-tray",
+        .args_type  = "device:s",
+        .mhandler.cmd_new = qmp_marshal_blockdev_close_tray,
+    },
+
+SQMP
+blockdev-close-tray
+-------------------
+
+Closes a block device's tray. If there is a block driver state tree associated
+with the block device (which is currently ejected), that tree will be loaded as
+the medium.
+
+If the tray was already closed before, this will be a no-op.
+
+Arguments:
+
+- "device": block device name (json-string)
+
+Example:
+
+-> { "execute": "blockdev-close-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751345,
+                    "microseconds": 272147 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": false } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-remove-medium",
+        .args_type  = "device:s",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_remove_medium,
+    },
+
+SQMP
+x-blockdev-remove-medium
+------------------------
+
+Removes a medium (a block driver state tree) from a block device. That block
+device's tray must currently be open (unless there is no attached guest device).
+
+If the tray is open and there is no medium inserted, this will be a no-op.
+
+This command is still a work in progress and is considered experimental.
+Stay away from it unless you want to help with its development.
+
+Arguments:
+
+- "device": block device name (json-string)
+
+Example:
+
+-> { "execute": "x-blockdev-remove-medium",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "error": { "class": "GenericError",
+                "desc": "Tray of device 'ide1-cd0' is not open" } }
+
+-> { "execute": "blockdev-open-tray",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "timestamp": { "seconds": 1418751627,
+                    "microseconds": 549958 },
+     "event": "DEVICE_TRAY_MOVED",
+     "data": { "device": "ide1-cd0",
+               "tray-open": true } }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-remove-medium",
+     "arguments": { "device": "ide1-cd0" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-insert-medium",
+        .args_type  = "device:s,node-name:s",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_insert_medium,
+    },
+
+SQMP
+x-blockdev-insert-medium
+------------------------
+
+Inserts a medium (a block driver state tree) into a block device. That block
+device's tray must currently be open (unless there is no attached guest device)
+and there must be no medium inserted already.
+
+This command is still a work in progress and is considered experimental.
+Stay away from it unless you want to help with its development.
+
+Arguments:
+
+- "device": block device name (json-string)
+- "node-name": root node of the BDS tree to insert into the block device
+
+Example:
+
+-> { "execute": "blockdev-add",
+     "arguments": { "options": { "node-name": "node0",
+                                 "driver": "raw",
+                                 "file": { "driver": "file",
+                                           "filename": "fedora.iso" } } } }
+
+<- { "return": {} }
+
+-> { "execute": "x-blockdev-insert-medium",
+     "arguments": { "device": "ide1-cd0",
+                    "node-name": "node0" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "x-blockdev-change",
+        .args_type  = "parent:B,child:B?,node:B?",
+        .mhandler.cmd_new = qmp_marshal_x_blockdev_change,
+    },
+
+SQMP
+x-blockdev-change
+-----------------
+
+Dynamically reconfigure the block driver state graph. It can be used
+to add, remove, insert or replace a graph node. Currently only the
+Quorum driver implements this feature to add or remove its child. This
+is useful to fix a broken quorum child.
+
+If @node is specified, it will be inserted under @parent. @child
+may not be specified in this case. If both @parent and @child are
+specified but @node is not, @child will be detached from @parent.
+
+Arguments:
+- "parent": the id or name of the parent node (json-string)
+- "child": the name of a child under the given parent node (json-string, optional)
+- "node": the name of the node that will be added (json-string, optional)
+
+Note: this command is experimental, and not a stable API. It doesn't
+support all kinds of operations, all kinds of children, nor all block
+drivers.
+
+Warning: The data in a new quorum child MUST be consistent with that of
+the rest of the array.
+
+Example:
+
+Add a new node to a quorum
+-> { "execute": "blockdev-add",
+     "arguments": { "options": { "driver": "raw",
+                                 "node-name": "new_node",
+                                 "file": { "driver": "file",
+                                           "filename": "test.raw" } } } }
+<- { "return": {} }
+-> { "execute": "x-blockdev-change",
+     "arguments": { "parent": "disk1",
+                    "node": "new_node" } }
+<- { "return": {} }
+
+Delete a quorum's node
+-> { "execute": "x-blockdev-change",
+     "arguments": { "parent": "disk1",
+                    "child": "children.1" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-named-block-nodes",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_named_block_nodes,
+        .mhandler.cmd_new = qmp_marshal_query_named_block_nodes,
     },
 
 SQMP
@@ -3618,13 +4523,14 @@
                    "iops_rd_max": 0,
                    "iops_wr_max": 0,
                    "iops_size": 0,
+                   "write_threshold": 0,
                    "image":{
                       "filename":"disks/test.qcow2",
                       "format":"qcow2",
                       "virtual-size":2048000,
                       "backing_file":"base.qcow2",
                       "full-backing-filename":"disks/base.qcow2",
-                      "backing-filename-format:"qcow2",
+                      "backing-filename-format":"qcow2",
                       "snapshots":[
                          {
                             "id": "1",
@@ -3646,9 +4552,62 @@
 EQMP
 
     {
+        .name       = "blockdev-change-medium",
+        .args_type  = "device:B,filename:F,format:s?,read-only-mode:s?",
+        .mhandler.cmd_new = qmp_marshal_blockdev_change_medium,
+    },
+
+SQMP
+blockdev-change-medium
+----------------------
+
+Changes the medium inserted into a block device by ejecting the current medium
+and loading a new image file which is inserted as the new medium.
+
+Arguments:
+
+- "device": device name (json-string)
+- "filename": filename of the new image (json-string)
+- "format": format of the new image (json-string, optional)
+- "read-only-mode": new read-only mode (json-string, optional)
+          - Possible values: "retain" (default), "read-only", "read-write"
+
+Examples:
+
+1. Change a removable medium
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "ide1-cd0",
+                            "filename": "/srv/images/Fedora-12-x86_64-DVD.iso",
+                            "format": "raw" } }
+<- { "return": {} }
+
+2. Load a read-only medium into a writable drive
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "isa-fd0",
+                            "filename": "/srv/images/ro.img",
+                            "format": "raw",
+                            "read-only-mode": "retain" } }
+
+<- { "error":
+     { "class": "GenericError",
+       "desc": "Could not open '/srv/images/ro.img': Permission denied" } }
+
+-> { "execute": "blockdev-change-medium",
+             "arguments": { "device": "isa-fd0",
+                            "filename": "/srv/images/ro.img",
+                            "format": "raw",
+                            "read-only-mode": "read-only" } }
+
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "query-memdev",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_memdev,
+        .mhandler.cmd_new = qmp_marshal_query_memdev,
     },
 
 SQMP
@@ -3686,7 +4645,7 @@
     {
         .name       = "query-memory-devices",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_memory_devices,
+        .mhandler.cmd_new = qmp_marshal_query_memory_devices,
     },
 
 SQMP
@@ -3713,7 +4672,7 @@
     {
         .name       = "query-acpi-ospm-status",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_query_acpi_ospm_status,
+        .mhandler.cmd_new = qmp_marshal_query_acpi_ospm_status,
     },
 
 SQMP
@@ -3736,7 +4695,7 @@
     {
         .name       = "rtc-reset-reinjection",
         .args_type  = "",
-        .mhandler.cmd_new = qmp_marshal_input_rtc_reset_reinjection,
+        .mhandler.cmd_new = qmp_marshal_rtc_reset_reinjection,
     },
 #endif
 
@@ -3756,8 +4715,8 @@
 
     {
         .name       = "trace-event-get-state",
-        .args_type  = "name:s",
-        .mhandler.cmd_new = qmp_marshal_input_trace_event_get_state,
+        .args_type  = "name:s,vcpu:i?",
+        .mhandler.cmd_new = qmp_marshal_trace_event_get_state,
     },
 
 SQMP
@@ -3766,6 +4725,20 @@
 
 Query the state of events.
 
+Arguments:
+
+- "name": Event name pattern (json-string).
+- "vcpu": The vCPU to query, any vCPU by default (json-int, optional).
+
+An event is returned if:
+- its name matches the "name" pattern, and
+- if "vcpu" is given, the event has the "vcpu" property.
+
+Therefore, if "vcpu" is given, the operation will only match per-vCPU events,
+returning their state on the specified vCPU. Special case: if "name" is an exact
+match, "vcpu" is given and the event does not have the "vcpu" property, an error
+is returned.
+
 Example:
 
 -> { "execute": "trace-event-get-state", "arguments": { "name": "qemu_memalign" } }
@@ -3774,8 +4747,8 @@
 
     {
         .name       = "trace-event-set-state",
-        .args_type  = "name:s,enable:b,ignore-unavailable:b?",
-        .mhandler.cmd_new = qmp_marshal_input_trace_event_set_state,
+        .args_type  = "name:s,enable:b,ignore-unavailable:b?,vcpu:i?",
+        .mhandler.cmd_new = qmp_marshal_trace_event_set_state,
     },
 
 SQMP
@@ -3784,6 +4757,23 @@
 
 Set the state of events.
 
+Arguments:
+
+- "name": Event name pattern (json-string).
+- "enable": Whether to enable or disable the event (json-bool).
+- "ignore-unavailable": Whether to ignore errors for events that cannot be
+  changed (json-bool, optional).
+- "vcpu": The vCPU to act upon, all vCPUs by default (json-int, optional).
+
+An event's state is modified if:
+- its name matches the "name" pattern, and
+- if "vcpu" is given, the event has the "vcpu" property.
+
+Therefore, if "vcpu" is given, the operation will only match per-vCPU events,
+setting their state on the specified vCPU. Special case: if "name" is an exact
+match, "vcpu" is given and the event does not have the "vcpu" property, an error
+is returned.
+
 Example:
 
 -> { "execute": "trace-event-set-state", "arguments": { "name": "qemu_memalign", "enable": "true" } }
@@ -3791,50 +4781,49 @@
 EQMP
 
     {
-        .name       = "x-input-send-event",
+        .name       = "input-send-event",
         .args_type  = "console:i?,events:q",
-        .mhandler.cmd_new = qmp_marshal_input_x_input_send_event,
+        .mhandler.cmd_new = qmp_marshal_input_send_event,
     },
 
 SQMP
-@x-input-send-event
+@input-send-event
 -----------------
 
 Send input event to guest.
 
 Arguments:
 
-- "console": console index. (json-int, optional)
-- "events": list of input events.
+- "device": display device (json-string, optional)
+- "head": display head (json-int, optional)
+- "events": list of input events
 
 The consoles are visible in the qom tree, under
 /backend/console[$index]. They have a device link and head property, so
 it is possible to map which console belongs to which device and display.
 
-Note: this command is experimental, and not a stable API.
-
 Example (1):
 
 Press left mouse button.
 
--> { "execute": "x-input-send-event",
-    "arguments": { "console": 0,
+-> { "execute": "input-send-event",
+    "arguments": { "device": "video0",
                    "events": [ { "type": "btn",
-                    "data" : { "down": true, "button": "Left" } } } }
+                   "data" : { "down": true, "button": "left" } } ] } }
 <- { "return": {} }
 
--> { "execute": "x-input-send-event",
-    "arguments": { "console": 0,
+-> { "execute": "input-send-event",
+    "arguments": { "device": "video0",
                    "events": [ { "type": "btn",
-                    "data" : { "down": false, "button": "Left" } } } }
+                   "data" : { "down": false, "button": "left" } } ] } }
 <- { "return": {} }
 
 Example (2):
 
 Press ctrl-alt-del.
 
--> { "execute": "x-input-send-event",
-     "arguments": { "console": 0, "events": [
+-> { "execute": "input-send-event",
+     "arguments": { "events": [
         { "type": "key", "data" : { "down": true,
           "key": {"type": "qcode", "data": "ctrl" } } },
         { "type": "key", "data" : { "down": true,
@@ -3847,10 +4836,206 @@
 
 Move mouse pointer to absolute coordinates (20000, 400).
 
--> { "execute": "x-input-send-event" ,
-  "arguments": { "console": 0, "events": [
-               { "type": "abs", "data" : { "axis": "X", "value" : 20000 } },
-               { "type": "abs", "data" : { "axis": "Y", "value" : 400 } } ] } }
+-> { "execute": "input-send-event" ,
+  "arguments": { "events": [
+               { "type": "abs", "data" : { "axis": "x", "value" : 20000 } },
+               { "type": "abs", "data" : { "axis": "y", "value" : 400 } } ] } }
 <- { "return": {} }
 
 EQMP
+
+    {
+        .name       = "block-set-write-threshold",
+        .args_type  = "node-name:s,write-threshold:l",
+        .mhandler.cmd_new = qmp_marshal_block_set_write_threshold,
+    },
+
+SQMP
+block-set-write-threshold
+------------
+
+Change the write threshold for a block drive. The threshold is an offset,
+thus must be non-negative. Default is no write threshold.
+Setting the threshold to zero disables it.
+
+Arguments:
+
+- "node-name": the node name in the block driver state graph (json-string)
+- "write-threshold": the write threshold in bytes (json-int)
+
+Example:
+
+-> { "execute": "block-set-write-threshold",
+  "arguments": { "node-name": "mydev",
+                 "write-threshold": 17179869184 } }
+<- { "return": {} }
+
+EQMP
+
+    {
+        .name       = "query-rocker",
+        .args_type  = "name:s",
+        .mhandler.cmd_new = qmp_marshal_query_rocker,
+    },
+
+SQMP
+Show rocker switch
+------------------
+
+Arguments:
+
+- "name": switch name
+
+Example:
+
+-> { "execute": "query-rocker", "arguments": { "name": "sw1" } }
+<- { "return": {"name": "sw1", "ports": 2, "id": 1327446905938}}
+
+EQMP
+
+    {
+        .name       = "query-rocker-ports",
+        .args_type  = "name:s",
+        .mhandler.cmd_new = qmp_marshal_query_rocker_ports,
+    },
+
+SQMP
+Show rocker switch ports
+------------------------
+
+Arguments:
+
+- "name": switch name
+
+Example:
+
+-> { "execute": "query-rocker-ports", "arguments": { "name": "sw1" } }
+<- { "return": [ {"duplex": "full", "enabled": true, "name": "sw1.1",
+                  "autoneg": "off", "link-up": true, "speed": 10000},
+                 {"duplex": "full", "enabled": true, "name": "sw1.2",
+                  "autoneg": "off", "link-up": true, "speed": 10000}
+   ]}
+
+EQMP
+
+    {
+        .name       = "query-rocker-of-dpa-flows",
+        .args_type  = "name:s,tbl-id:i?",
+        .mhandler.cmd_new = qmp_marshal_query_rocker_of_dpa_flows,
+    },
+
+SQMP
+Show rocker switch OF-DPA flow tables
+-------------------------------------
+
+Arguments:
+
+- "name": switch name
+- "tbl-id": (optional) flow table ID
+
+Example:
+
+-> { "execute": "query-rocker-of-dpa-flows", "arguments": { "name": "sw1" } }
+<- { "return": [ {"key": {"in-pport": 0, "priority": 1, "tbl-id": 0},
+                  "hits": 138,
+                  "cookie": 0,
+                  "action": {"goto-tbl": 10},
+                  "mask": {"in-pport": 4294901760}
+                 },
+                 {...more...},
+   ]}
+
+EQMP
+
+    {
+        .name       = "query-rocker-of-dpa-groups",
+        .args_type  = "name:s,type:i?",
+        .mhandler.cmd_new = qmp_marshal_query_rocker_of_dpa_groups,
+    },
+
+SQMP
+Show rocker OF-DPA group tables
+-------------------------------
+
+Arguments:
+
+- "name": switch name
+- "type": (optional) group type
+
+Example:
+
+-> { "execute": "query-rocker-of-dpa-groups", "arguments": { "name": "sw1" } }
+<- { "return": [ {"type": 0, "out-pport": 2, "pport": 2, "vlan-id": 3841,
+                  "pop-vlan": 1, "id": 251723778},
+                 {"type": 0, "out-pport": 0, "pport": 0, "vlan-id": 3841,
+                  "pop-vlan": 1, "id": 251723776},
+                 {"type": 0, "out-pport": 1, "pport": 1, "vlan-id": 3840,
+                  "pop-vlan": 1, "id": 251658241},
+                 {"type": 0, "out-pport": 0, "pport": 0, "vlan-id": 3840,
+                  "pop-vlan": 1, "id": 251658240}
+   ]}
+
+EQMP
+
+#if defined TARGET_ARM
+    {
+        .name       = "query-gic-capabilities",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_query_gic_capabilities,
+    },
+#endif
+
+SQMP
+query-gic-capabilities
+---------------
+
+Return a list of GICCapability objects, describing supported GIC
+(Generic Interrupt Controller) versions.
+
+Arguments: None
+
+Example:
+
+-> { "execute": "query-gic-capabilities" }
+<- { "return": [{ "version": 2, "emulated": true, "kernel": false },
+                { "version": 3, "emulated": false, "kernel": true } ] }
+
+EQMP
+
+    {
+        .name       = "query-hotpluggable-cpus",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_query_hotpluggable_cpus,
+    },
+
+SQMP
+Show existing/possible CPUs
+---------------------------
+
+Arguments: None.
+
+Example for pseries machine type started with
+-smp 2,cores=2,maxcpus=4 -cpu POWER8:
+
+-> { "execute": "query-hotpluggable-cpus" }
+<- {"return": [
+     { "props": { "core-id": 8 }, "type": "POWER8-spapr-cpu-core",
+       "vcpus-count": 1 },
+     { "props": { "core-id": 0 }, "type": "POWER8-spapr-cpu-core",
+       "vcpus-count": 1, "qom-path": "/machine/unattached/device[0]"}
+   ]}'
+
+Example for pc machine type started with
+-smp 1,maxcpus=2:
+    -> { "execute": "query-hotpluggable-cpus" }
+    <- {"return": [
+         {
+            "type": "qemu64-x86_64-cpu", "vcpus-count": 1,
+            "props": {"core-id": 0, "socket-id": 1, "thread-id": 0}
+         },
+         {
+            "qom-path": "/machine/unattached/device[0]",
+            "type": "qemu64-x86_64-cpu", "vcpus-count": 1,
+            "props": {"core-id": 0, "socket-id": 0, "thread-id": 0}
+         }
+       ]}
diff --git a/qmp.c b/qmp.c
index 0b4f131..b6d531e 100644
--- a/qmp.c
+++ b/qmp.c
@@ -13,7 +13,10 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu-version.h"
+#include "qemu/cutils.h"
+#include "monitor/monitor.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
 #include "sysemu/char.h"
@@ -23,7 +26,9 @@
 #include "sysemu/arch_init.h"
 #include "hw/qdev.h"
 #include "sysemu/blockdev.h"
+#include "sysemu/block-backend.h"
 #include "qom/qom-qobject.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp-input-visitor.h"
 #include "hw/boards.h"
@@ -45,15 +50,22 @@
 
 VersionInfo *qmp_query_version(Error **errp)
 {
-    VersionInfo *info = g_malloc0(sizeof(*info));
+    VersionInfo *info = g_new0(VersionInfo, 1);
     const char *version = QEMU_VERSION;
-    char *tmp;
+    const char *tmp;
+    int err;
 
-    info->qemu.major = strtol(version, &tmp, 10);
+    info->qemu = g_new0(VersionTriple, 1);
+    err = qemu_strtoll(version, &tmp, 10, &info->qemu->major);
+    assert(err == 0);
     tmp++;
-    info->qemu.minor = strtol(tmp, &tmp, 10);
+
+    err = qemu_strtoll(tmp, &tmp, 10, &info->qemu->minor);
+    assert(err == 0);
     tmp++;
-    info->qemu.micro = strtol(tmp, &tmp, 10);
+
+    err = qemu_strtoll(tmp, &tmp, 10, &info->qemu->micro);
+    assert(err == 0);
     info->package = g_strdup(QEMU_PKGVERSION);
 
     return info;
@@ -92,6 +104,13 @@
 
 void qmp_stop(Error **errp)
 {
+    /* if there is a dump in background, we should wait until the dump
+     * finished */
+    if (dump_in_progress()) {
+        error_setg(errp, "There is a dump in process, please wait.");
+        return;
+    }
+
     if (runstate_check(RUN_STATE_INMIGRATE)) {
         autostart = 0;
     } else {
@@ -131,26 +150,46 @@
    defined in the VNC subsystem */
 VncInfo *qmp_query_vnc(Error **errp)
 {
-    error_set(errp, QERR_FEATURE_DISABLED, "vnc");
+    error_setg(errp, QERR_FEATURE_DISABLED, "vnc");
+    return NULL;
+};
+
+VncInfo2List *qmp_query_vnc_servers(Error **errp)
+{
+    error_setg(errp, QERR_FEATURE_DISABLED, "vnc");
     return NULL;
 };
 #endif
 
 #ifndef CONFIG_SPICE
-/* If SPICE support is enabled, the "true" query-spice command is
-   defined in the SPICE subsystem. Also note that we use a small
-   trick to maintain query-spice's original behavior, which is not
-   to be available in the namespace if SPICE is not compiled in */
+/*
+ * qmp-commands.hx ensures that QMP command query-spice exists only
+ * #ifdef CONFIG_SPICE.  Necessary for an accurate query-commands
+ * result.  However, the QAPI schema is blissfully unaware of that,
+ * and the QAPI code generator happily generates a dead
+ * qmp_marshal_query_spice() that calls qmp_query_spice().  Provide it
+ * one, or else linking fails.  FIXME Educate the QAPI schema on
+ * CONFIG_SPICE.
+ */
 SpiceInfo *qmp_query_spice(Error **errp)
 {
-    error_set(errp, QERR_COMMAND_NOT_FOUND, "query-spice");
-    return NULL;
+    abort();
 };
 #endif
 
 void qmp_cont(Error **errp)
 {
+    Error *local_err = NULL;
+    BlockBackend *blk;
     BlockDriverState *bs;
+    BdrvNextIterator it;
+
+    /* if there is a dump in background, we should wait until the dump
+     * finished */
+    if (dump_in_progress()) {
+        error_setg(errp, "There is a dump in process, please wait.");
+        return;
+    }
 
     if (runstate_needs_reset()) {
         error_setg(errp, "Resetting the Virtual Machine is required");
@@ -159,14 +198,26 @@
         return;
     }
 
-    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-        bdrv_iostatus_reset(bs);
+    for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
+        blk_iostatus_reset(blk);
     }
-    for (bs = bdrv_next(NULL); bs; bs = bdrv_next(bs)) {
-        if (bdrv_key_required(bs)) {
-            error_set(errp, QERR_DEVICE_ENCRYPTED,
-                      bdrv_get_device_name(bs),
-                      bdrv_get_encrypted_filename(bs));
+
+    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
+        bdrv_add_key(bs, NULL, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+    /* Continuing after completed migration. Images have been inactivated to
+     * allow the destination to take control. Need to get control back now. */
+    if (runstate_check(RUN_STATE_FINISH_MIGRATE) ||
+        runstate_check(RUN_STATE_POSTMIGRATE))
+    {
+        bdrv_invalidate_cache_all(&local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
             return;
         }
     }
@@ -189,18 +240,21 @@
     bool ambiguous = false;
     ObjectPropertyInfoList *props = NULL;
     ObjectProperty *prop;
+    ObjectPropertyIterator iter;
 
     obj = object_resolve_path(path, &ambiguous);
     if (obj == NULL) {
         if (ambiguous) {
             error_setg(errp, "Path '%s' is ambiguous", path);
         } else {
-            error_set(errp, QERR_DEVICE_NOT_FOUND, path);
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "Device '%s' not found", path);
         }
         return NULL;
     }
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    object_property_iter_init(&iter, obj);
+    while ((prop = object_property_iter_next(&iter))) {
         ObjectPropertyInfoList *entry = g_malloc0(sizeof(*entry));
 
         entry->value = g_malloc0(sizeof(ObjectPropertyInfo));
@@ -214,56 +268,33 @@
     return props;
 }
 
-/* FIXME: teach qapi about how to pass through Visitors */
-int qmp_qom_set(Monitor *mon, const QDict *qdict, QObject **ret)
+void qmp_qom_set(const char *path, const char *property, QObject *value,
+                 Error **errp)
 {
-    const char *path = qdict_get_str(qdict, "path");
-    const char *property = qdict_get_str(qdict, "property");
-    QObject *value = qdict_get(qdict, "value");
-    Error *local_err = NULL;
     Object *obj;
 
     obj = object_resolve_path(path, NULL);
     if (!obj) {
-        error_set(&local_err, QERR_DEVICE_NOT_FOUND, path);
-        goto out;
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", path);
+        return;
     }
 
-    object_property_set_qobject(obj, value, property, &local_err);
-
-out:
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
-    }
-
-    return 0;
+    object_property_set_qobject(obj, value, property, errp);
 }
 
-int qmp_qom_get(Monitor *mon, const QDict *qdict, QObject **ret)
+QObject *qmp_qom_get(const char *path, const char *property, Error **errp)
 {
-    const char *path = qdict_get_str(qdict, "path");
-    const char *property = qdict_get_str(qdict, "property");
-    Error *local_err = NULL;
     Object *obj;
 
     obj = object_resolve_path(path, NULL);
     if (!obj) {
-        error_set(&local_err, QERR_DEVICE_NOT_FOUND, path);
-        goto out;
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", path);
+        return NULL;
     }
 
-    *ret = object_property_get_qobject(obj, property, &local_err);
-
-out:
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
-    }
-
-    return 0;
+    return object_property_get_qobject(obj, property, errp);
 }
 
 void qmp_set_password(const char *protocol, const char *password,
@@ -281,21 +312,19 @@
         } else if (strcmp(connected, "keep") == 0) {
             /* nothing */
         } else {
-            error_set(errp, QERR_INVALID_PARAMETER, "connected");
+            error_setg(errp, QERR_INVALID_PARAMETER, "connected");
             return;
         }
     }
 
     if (strcmp(protocol, "spice") == 0) {
-        if (!using_spice) {
-            /* correct one? spice isn't a device ,,, */
-            error_set(errp, QERR_DEVICE_NOT_ACTIVE, "spice");
+        if (!qemu_using_spice(errp)) {
             return;
         }
         rc = qemu_spice_set_passwd(password, fail_if_connected,
                                    disconnect_if_connected);
         if (rc != 0) {
-            error_set(errp, QERR_SET_PASSWD_FAILED);
+            error_setg(errp, QERR_SET_PASSWD_FAILED);
         }
         return;
     }
@@ -303,19 +332,19 @@
     if (strcmp(protocol, "vnc") == 0) {
         if (fail_if_connected || disconnect_if_connected) {
             /* vnc supports "connected=keep" only */
-            error_set(errp, QERR_INVALID_PARAMETER, "connected");
+            error_setg(errp, QERR_INVALID_PARAMETER, "connected");
             return;
         }
         /* Note that setting an empty password will not disable login through
          * this interface. */
         rc = vnc_display_password(NULL, password);
         if (rc < 0) {
-            error_set(errp, QERR_SET_PASSWD_FAILED);
+            error_setg(errp, QERR_SET_PASSWD_FAILED);
         }
         return;
     }
 
-    error_set(errp, QERR_INVALID_PARAMETER, "protocol");
+    error_setg(errp, QERR_INVALID_PARAMETER, "protocol");
 }
 
 void qmp_expire_password(const char *protocol, const char *whenstr,
@@ -335,14 +364,12 @@
     }
 
     if (strcmp(protocol, "spice") == 0) {
-        if (!using_spice) {
-            /* correct one? spice isn't a device ,,, */
-            error_set(errp, QERR_DEVICE_NOT_ACTIVE, "spice");
+        if (!qemu_using_spice(errp)) {
             return;
         }
         rc = qemu_spice_set_pw_expire(when);
         if (rc != 0) {
-            error_set(errp, QERR_SET_PASSWD_FAILED);
+            error_setg(errp, QERR_SET_PASSWD_FAILED);
         }
         return;
     }
@@ -350,25 +377,42 @@
     if (strcmp(protocol, "vnc") == 0) {
         rc = vnc_display_pw_expire(NULL, when);
         if (rc != 0) {
-            error_set(errp, QERR_SET_PASSWD_FAILED);
+            error_setg(errp, QERR_SET_PASSWD_FAILED);
         }
         return;
     }
 
-    error_set(errp, QERR_INVALID_PARAMETER, "protocol");
+    error_setg(errp, QERR_INVALID_PARAMETER, "protocol");
 }
 
 #ifdef CONFIG_VNC
 void qmp_change_vnc_password(const char *password, Error **errp)
 {
     if (vnc_display_password(NULL, password) < 0) {
-        error_set(errp, QERR_SET_PASSWD_FAILED);
+        error_setg(errp, QERR_SET_PASSWD_FAILED);
     }
 }
 
 static void qmp_change_vnc_listen(const char *target, Error **errp)
 {
-    vnc_display_open(NULL, target, errp);
+    QemuOptsList *olist = qemu_find_opts("vnc");
+    QemuOpts *opts;
+
+    if (strstr(target, "id=")) {
+        error_setg(errp, "id not supported");
+        return;
+    }
+
+    opts = qemu_opts_find(olist, "default");
+    if (opts) {
+        qemu_opts_del(opts);
+    }
+    opts = vnc_parse(target, errp);
+    if (!opts) {
+        return;
+    }
+
+    vnc_display_open("default", errp);
 }
 
 static void qmp_change_vnc(const char *target, bool has_arg, const char *arg,
@@ -376,7 +420,7 @@
 {
     if (strcmp(target, "passwd") == 0 || strcmp(target, "password") == 0) {
         if (!has_arg) {
-            error_set(errp, QERR_MISSING_PARAMETER, "password");
+            error_setg(errp, QERR_MISSING_PARAMETER, "password");
         } else {
             qmp_change_vnc_password(arg, errp);
         }
@@ -387,12 +431,12 @@
 #else
 void qmp_change_vnc_password(const char *password, Error **errp)
 {
-    error_set(errp, QERR_FEATURE_DISABLED, "vnc");
+    error_setg(errp, QERR_FEATURE_DISABLED, "vnc");
 }
 static void qmp_change_vnc(const char *target, bool has_arg, const char *arg,
                            Error **errp)
 {
-    error_set(errp, QERR_FEATURE_DISABLED, "vnc");
+    error_setg(errp, QERR_FEATURE_DISABLED, "vnc");
 }
 #endif /* !CONFIG_VNC */
 
@@ -402,7 +446,8 @@
     if (strcmp(device, "vnc") == 0) {
         qmp_change_vnc(target, has_arg, arg, errp);
     } else {
-        qmp_change_blockdev(device, target, arg, errp);
+        qmp_blockdev_change_medium(device, target, has_arg, arg, false, 0,
+                                   errp);
     }
 }
 
@@ -490,24 +535,37 @@
     ObjectClass *klass;
     Object *obj;
     ObjectProperty *prop;
+    ObjectPropertyIterator iter;
     DevicePropertyInfoList *prop_list = NULL;
 
     klass = object_class_by_name(typename);
     if (klass == NULL) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, typename);
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", typename);
         return NULL;
     }
 
     klass = object_class_dynamic_cast(klass, TYPE_DEVICE);
     if (klass == NULL) {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
-                  "name", TYPE_DEVICE);
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "name", TYPE_DEVICE);
+        return NULL;
+    }
+
+    if (object_class_is_abstract(klass)) {
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "name",
+                   "non-abstract device type");
+        return NULL;
+    }
+
+    if (DEVICE_CLASS(klass)->cannot_destroy_with_object_finalize_yet) {
+        error_setg(errp, "Can't list properties of device '%s'", typename);
         return NULL;
     }
 
     obj = object_new(typename);
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    object_property_iter_init(&iter, obj);
+    while ((prop = object_property_iter_next(&iter))) {
         DevicePropertyInfo *info;
         DevicePropertyInfoList *entry;
 
@@ -562,8 +620,7 @@
     }
 
     if (strcmp(protocol, "spice") == 0) {
-        if (!using_spice) {
-            error_set(errp, QERR_DEVICE_NOT_ACTIVE, "spice");
+        if (!qemu_using_spice(errp)) {
             close(fd);
             return;
         }
@@ -593,103 +650,33 @@
     close(fd);
 }
 
-void object_add(const char *type, const char *id, const QDict *qdict,
-                Visitor *v, Error **errp)
+
+void qmp_object_add(const char *type, const char *id,
+                    bool has_props, QObject *props, Error **errp)
 {
-    Object *obj;
-    ObjectClass *klass;
-    const QDictEntry *e;
-    Error *local_err = NULL;
-
-    klass = object_class_by_name(type);
-    if (!klass) {
-        error_setg(errp, "invalid object type: %s", type);
-        return;
-    }
-
-    if (!object_class_dynamic_cast(klass, TYPE_USER_CREATABLE)) {
-        error_setg(errp, "object type '%s' isn't supported by object-add",
-                   type);
-        return;
-    }
-
-    if (object_class_is_abstract(klass)) {
-        error_setg(errp, "object type '%s' is abstract", type);
-        return;
-    }
-
-    obj = object_new(type);
-    if (qdict) {
-        for (e = qdict_first(qdict); e; e = qdict_next(qdict, e)) {
-            object_property_set(obj, v, e->key, &local_err);
-            if (local_err) {
-                goto out;
-            }
-        }
-    }
-
-    object_property_add_child(container_get(object_get_root(), "/objects"),
-                              id, obj, &local_err);
-    if (local_err) {
-        goto out;
-    }
-
-    user_creatable_complete(obj, &local_err);
-    if (local_err) {
-        object_property_del(container_get(object_get_root(), "/objects"),
-                            id, &error_abort);
-        goto out;
-    }
-out:
-    if (local_err) {
-        error_propagate(errp, local_err);
-    }
-    object_unref(obj);
-}
-
-int qmp_object_add(Monitor *mon, const QDict *qdict, QObject **ret)
-{
-    const char *type = qdict_get_str(qdict, "qom-type");
-    const char *id = qdict_get_str(qdict, "id");
-    QObject *props = qdict_get(qdict, "props");
     const QDict *pdict = NULL;
-    Error *local_err = NULL;
-    QmpInputVisitor *qiv;
+    Visitor *v;
+    Object *obj;
 
     if (props) {
         pdict = qobject_to_qdict(props);
         if (!pdict) {
-            error_set(&local_err, QERR_INVALID_PARAMETER_TYPE, "props", "dict");
-            goto out;
+            error_setg(errp, QERR_INVALID_PARAMETER_TYPE, "props", "dict");
+            return;
         }
     }
 
-    qiv = qmp_input_visitor_new(props);
-    object_add(type, id, pdict, qmp_input_get_visitor(qiv), &local_err);
-    qmp_input_visitor_cleanup(qiv);
-
-out:
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
+    v = qmp_input_visitor_new(props, true);
+    obj = user_creatable_add_type(type, id, pdict, v, errp);
+    visit_free(v);
+    if (obj) {
+        object_unref(obj);
     }
-
-    return 0;
 }
 
 void qmp_object_del(const char *id, Error **errp)
 {
-    Object *container;
-    Object *obj;
-
-    container = container_get(object_get_root(), "/objects");
-    obj = object_resolve_path_component(container, id);
-    if (!obj) {
-        error_setg(errp, "object id not found");
-        return;
-    }
-    object_unparent(obj);
+    user_creatable_del(id, errp);
 }
 
 MemoryDeviceInfoList *qmp_query_memory_devices(Error **errp)
diff --git a/qobject/Makefile.objs b/qobject/Makefile.objs
index c9ff59c..bed5508 100644
--- a/qobject/Makefile.objs
+++ b/qobject/Makefile.objs
@@ -1,3 +1,2 @@
-util-obj-y = qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
-util-obj-y += qjson.o json-lexer.o json-streamer.o json-parser.o
-util-obj-y += qerror.o
+util-obj-y = qnull.o qint.o qstring.o qdict.o qlist.o qfloat.o qbool.o
+util-obj-y += qjson.o qobject.o json-lexer.o json-streamer.o json-parser.o
diff --git a/qobject/json-lexer.c b/qobject/json-lexer.c
index b19623e..af4a75e 100644
--- a/qobject/json-lexer.c
+++ b/qobject/json-lexer.c
@@ -11,26 +11,32 @@
  *
  */
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qdict.h"
-#include "qapi/qmp/qint.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
 
 /*
- * \"([^\\\"]|(\\\"\\'\\\\\\/\\b\\f\\n\\r\\t\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]))*\"
- * '([^\\']|(\\\"\\'\\\\\\/\\b\\f\\n\\r\\t\\u[0-9a-fA-F][0-9a-fA-F][0-9a-fA-F][0-9a-fA-F]))*'
- * 0|([1-9][0-9]*(.[0-9]+)?([eE]([-+])?[0-9]+))
+ * Required by JSON (RFC 7159):
+ *
+ * \"([^\\\"]|\\[\"'\\/bfnrt]|\\u[0-9a-fA-F]{4})*\"
+ * -?(0|[1-9][0-9]*)(.[0-9]+)?([eE][-+]?[0-9]+)?
  * [{}\[\],:]
- * [a-z]+
+ * [a-z]+   # covers null, true, false
+ *
+ * Extension of '' strings:
+ *
+ * '([^\\']|\\[\"'\\/bfnrt]|\\u[0-9a-fA-F]{4})*'
+ *
+ * Extension for vararg handling in JSON construction:
+ *
+ * %((l|ll|I64)?d|[ipsf])
  *
  */
 
 enum json_lexer_state {
-    IN_ERROR = 0,
+    IN_ERROR = 0,               /* must really be 0, see json_lexer[] */
     IN_DQ_UCODE3,
     IN_DQ_UCODE2,
     IN_DQ_UCODE1,
@@ -62,6 +68,8 @@
     IN_START,
 };
 
+QEMU_BUILD_BUG_ON((int)JSON_MIN <= (int)IN_START);
+
 #define TERMINAL(state) [0 ... 0x7F] = (state)
 
 /* Return whether TERMINAL is a terminal state and the transition to it
@@ -71,6 +79,8 @@
             (json_lexer[(old_state)][0] == (terminal))
 
 static const uint8_t json_lexer[][256] =  {
+    /* Relies on default initialization to IN_ERROR! */
+
     /* double quote string */
     [IN_DQ_UCODE3] = {
         ['0' ... '9'] = IN_DQ_STRING,
@@ -212,7 +222,7 @@
         ['\t'] = IN_WHITESPACE,
         ['\r'] = IN_WHITESPACE,
         ['\n'] = IN_WHITESPACE,
-    },        
+    },
 
     /* escape */
     [IN_ESCAPE_LL] = {
@@ -253,12 +263,12 @@
         ['0'] = IN_ZERO,
         ['1' ... '9'] = IN_NONZERO_NUMBER,
         ['-'] = IN_NEG_NONZERO_NUMBER,
-        ['{'] = JSON_OPERATOR,
-        ['}'] = JSON_OPERATOR,
-        ['['] = JSON_OPERATOR,
-        [']'] = JSON_OPERATOR,
-        [','] = JSON_OPERATOR,
-        [':'] = JSON_OPERATOR,
+        ['{'] = JSON_LCURLY,
+        ['}'] = JSON_RCURLY,
+        ['['] = JSON_LSQUARE,
+        [']'] = JSON_RSQUARE,
+        [','] = JSON_COMMA,
+        [':'] = JSON_COLON,
         ['a' ... 'z'] = IN_KEYWORD,
         ['%'] = IN_ESCAPE,
         [' '] = IN_WHITESPACE,
@@ -272,7 +282,7 @@
 {
     lexer->emit = func;
     lexer->state = IN_START;
-    lexer->token = qstring_new();
+    lexer->token = g_string_sized_new(3);
     lexer->x = lexer->y = 0;
 }
 
@@ -287,14 +297,20 @@
     }
 
     do {
+        assert(lexer->state <= ARRAY_SIZE(json_lexer));
         new_state = json_lexer[lexer->state][(uint8_t)ch];
         char_consumed = !TERMINAL_NEEDED_LOOKAHEAD(lexer->state, new_state);
         if (char_consumed) {
-            qstring_append_chr(lexer->token, ch);
+            g_string_append_c(lexer->token, ch);
         }
 
         switch (new_state) {
-        case JSON_OPERATOR:
+        case JSON_LCURLY:
+        case JSON_RCURLY:
+        case JSON_LSQUARE:
+        case JSON_RSQUARE:
+        case JSON_COLON:
+        case JSON_COMMA:
         case JSON_ESCAPE:
         case JSON_INTEGER:
         case JSON_FLOAT:
@@ -303,8 +319,7 @@
             lexer->emit(lexer, lexer->token, new_state, lexer->x, lexer->y);
             /* fall through */
         case JSON_SKIP:
-            QDECREF(lexer->token);
-            lexer->token = qstring_new();
+            g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             break;
         case IN_ERROR:
@@ -322,8 +337,7 @@
              * induce an error/flush state.
              */
             lexer->emit(lexer, lexer->token, JSON_ERROR, lexer->x, lexer->y);
-            QDECREF(lexer->token);
-            lexer->token = qstring_new();
+            g_string_truncate(lexer->token, 0);
             new_state = IN_START;
             lexer->state = new_state;
             return 0;
@@ -336,10 +350,9 @@
     /* Do not let a single token grow to an arbitrarily large size,
      * this is a security consideration.
      */
-    if (lexer->token->length > MAX_TOKEN_SIZE) {
+    if (lexer->token->len > MAX_TOKEN_SIZE) {
         lexer->emit(lexer, lexer->token, lexer->state, lexer->x, lexer->y);
-        QDECREF(lexer->token);
-        lexer->token = qstring_new();
+        g_string_truncate(lexer->token, 0);
         lexer->state = IN_START;
     }
 
@@ -369,5 +382,5 @@
 
 void json_lexer_destroy(JSONLexer *lexer)
 {
-    QDECREF(lexer->token);
+    g_string_free(lexer->token, true);
 }
diff --git a/qobject/json-parser.c b/qobject/json-parser.c
index 4288267..c18e48a 100644
--- a/qobject/json-parser.c
+++ b/qobject/json-parser.c
@@ -1,5 +1,5 @@
 /*
- * JSON Parser 
+ * JSON Parser
  *
  * Copyright IBM, Corp. 2009
  *
@@ -11,27 +11,19 @@
  *
  */
 
-#include <stdarg.h>
-
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qdict.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qfloat.h"
-#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/types.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-lexer.h"
-#include "qapi/qmp/qerror.h"
+#include "qapi/qmp/json-streamer.h"
 
 typedef struct JSONParserContext
 {
     Error *err;
-    struct {
-        QObject **buf;
-        size_t pos;
-        size_t count;
-    } tokens;
+    JSONToken *current;
+    GQueue *buf;
 } JSONParserContext;
 
 #define BUG_ON(cond) assert(!(cond))
@@ -48,58 +40,10 @@
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap);
 
 /**
- * Token manipulators
- *
- * tokens are dictionaries that contain a type, a string value, and geometry information
- * about a token identified by the lexer.  These are routines that make working with
- * these objects a bit easier.
- */
-static const char *token_get_value(QObject *obj)
-{
-    return qdict_get_str(qobject_to_qdict(obj), "token");
-}
-
-static JSONTokenType token_get_type(QObject *obj)
-{
-    return qdict_get_int(qobject_to_qdict(obj), "type");
-}
-
-static int token_is_operator(QObject *obj, char op)
-{
-    const char *val;
-
-    if (token_get_type(obj) != JSON_OPERATOR) {
-        return 0;
-    }
-
-    val = token_get_value(obj);
-
-    return (val[0] == op) && (val[1] == 0);
-}
-
-static int token_is_keyword(QObject *obj, const char *value)
-{
-    if (token_get_type(obj) != JSON_KEYWORD) {
-        return 0;
-    }
-
-    return strcmp(token_get_value(obj), value) == 0;
-}
-
-static int token_is_escape(QObject *obj, const char *value)
-{
-    if (token_get_type(obj) != JSON_ESCAPE) {
-        return 0;
-    }
-
-    return (strcmp(token_get_value(obj), value) == 0);
-}
-
-/**
  * Error handler
  */
 static void GCC_FMT_ATTR(3, 4) parse_error(JSONParserContext *ctxt,
-                                           QObject *token, const char *msg, ...)
+                                           JSONToken *token, const char *msg, ...)
 {
     va_list ap;
     char message[1024];
@@ -177,9 +121,10 @@
  *      \t
  *      \u four-hex-digits 
  */
-static QString *qstring_from_escaped_str(JSONParserContext *ctxt, QObject *token)
+static QString *qstring_from_escaped_str(JSONParserContext *ctxt,
+                                         JSONToken *token)
 {
-    const char *ptr = token_get_value(token);
+    const char *ptr = token->str;
     QString *str;
     int double_quote = 1;
 
@@ -275,73 +220,34 @@
     return NULL;
 }
 
-static QObject *parser_context_pop_token(JSONParserContext *ctxt)
-{
-    QObject *token;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    token = ctxt->tokens.buf[ctxt->tokens.pos];
-    ctxt->tokens.pos++;
-    return token;
-}
-
-/* Note: parser_context_{peek|pop}_token do not increment the
- * token object's refcount. In both cases the references will continue
- * to be tracked and cleaned up in parser_context_free(), so do not
- * attempt to free the token object.
+/* Note: the token object returned by parser_context_peek_token or
+ * parser_context_pop_token is deleted as soon as parser_context_pop_token
+ * is called again.
  */
-static QObject *parser_context_peek_token(JSONParserContext *ctxt)
+static JSONToken *parser_context_pop_token(JSONParserContext *ctxt)
 {
-    QObject *token;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    token = ctxt->tokens.buf[ctxt->tokens.pos];
-    return token;
+    g_free(ctxt->current);
+    assert(!g_queue_is_empty(ctxt->buf));
+    ctxt->current = g_queue_pop_head(ctxt->buf);
+    return ctxt->current;
 }
 
-static JSONParserContext parser_context_save(JSONParserContext *ctxt)
+static JSONToken *parser_context_peek_token(JSONParserContext *ctxt)
 {
-    JSONParserContext saved_ctxt = {0};
-    saved_ctxt.tokens.pos = ctxt->tokens.pos;
-    saved_ctxt.tokens.count = ctxt->tokens.count;
-    saved_ctxt.tokens.buf = ctxt->tokens.buf;
-    return saved_ctxt;
+    assert(!g_queue_is_empty(ctxt->buf));
+    return g_queue_peek_head(ctxt->buf);
 }
 
-static void parser_context_restore(JSONParserContext *ctxt,
-                                   JSONParserContext saved_ctxt)
-{
-    ctxt->tokens.pos = saved_ctxt.tokens.pos;
-    ctxt->tokens.count = saved_ctxt.tokens.count;
-    ctxt->tokens.buf = saved_ctxt.tokens.buf;
-}
-
-static void tokens_append_from_iter(QObject *obj, void *opaque)
-{
-    JSONParserContext *ctxt = opaque;
-    g_assert(ctxt->tokens.pos < ctxt->tokens.count);
-    ctxt->tokens.buf[ctxt->tokens.pos++] = obj;
-    qobject_incref(obj);
-}
-
-static JSONParserContext *parser_context_new(QList *tokens)
+static JSONParserContext *parser_context_new(GQueue *tokens)
 {
     JSONParserContext *ctxt;
-    size_t count;
 
     if (!tokens) {
         return NULL;
     }
 
-    count = qlist_size(tokens);
-    if (count == 0) {
-        return NULL;
-    }
-
     ctxt = g_malloc0(sizeof(JSONParserContext));
-    ctxt->tokens.pos = 0;
-    ctxt->tokens.count = count;
-    ctxt->tokens.buf = g_malloc(count * sizeof(QObject *));
-    qlist_iter(tokens, tokens_append_from_iter, ctxt);
-    ctxt->tokens.pos = 0;
+    ctxt->buf = tokens;
 
     return ctxt;
 }
@@ -349,12 +255,12 @@
 /* to support error propagation, ctxt->err must be freed separately */
 static void parser_context_free(JSONParserContext *ctxt)
 {
-    int i;
     if (ctxt) {
-        for (i = 0; i < ctxt->tokens.count; i++) {
-            qobject_decref(ctxt->tokens.buf[i]);
+        while (!g_queue_is_empty(ctxt->buf)) {
+            parser_context_pop_token(ctxt);
         }
-        g_free(ctxt->tokens.buf);
+        g_free(ctxt->current);
+        g_queue_free(ctxt->buf);
         g_free(ctxt);
     }
 }
@@ -364,8 +270,8 @@
  */
 static int parse_pair(JSONParserContext *ctxt, QDict *dict, va_list *ap)
 {
-    QObject *key = NULL, *token = NULL, *value, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    QObject *key = NULL, *value;
+    JSONToken *peek, *token;
 
     peek = parser_context_peek_token(ctxt);
     if (peek == NULL) {
@@ -385,7 +291,7 @@
         goto out;
     }
 
-    if (!token_is_operator(token, ':')) {
+    if (token->type != JSON_COLON) {
         parse_error(ctxt, token, "missing : in object pair");
         goto out;
     }
@@ -403,7 +309,6 @@
     return 0;
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     qobject_decref(key);
 
     return -1;
@@ -412,17 +317,10 @@
 static QObject *parse_object(JSONParserContext *ctxt, va_list *ap)
 {
     QDict *dict = NULL;
-    QObject *token, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token, *peek;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (!token_is_operator(token, '{')) {
-        goto out;
-    }
+    assert(token && token->type == JSON_LCURLY);
 
     dict = qdict_new();
 
@@ -432,7 +330,7 @@
         goto out;
     }
 
-    if (!token_is_operator(peek, '}')) {
+    if (peek->type != JSON_RCURLY) {
         if (parse_pair(ctxt, dict, ap) == -1) {
             goto out;
         }
@@ -443,8 +341,8 @@
             goto out;
         }
 
-        while (!token_is_operator(token, '}')) {
-            if (!token_is_operator(token, ',')) {
+        while (token->type != JSON_RCURLY) {
+            if (token->type != JSON_COMMA) {
                 parse_error(ctxt, token, "expected separator in dict");
                 goto out;
             }
@@ -466,7 +364,6 @@
     return QOBJECT(dict);
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     QDECREF(dict);
     return NULL;
 }
@@ -474,17 +371,10 @@
 static QObject *parse_array(JSONParserContext *ctxt, va_list *ap)
 {
     QList *list = NULL;
-    QObject *token, *peek;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token, *peek;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
-
-    if (!token_is_operator(token, '[')) {
-        goto out;
-    }
+    assert(token && token->type == JSON_LSQUARE);
 
     list = qlist_new();
 
@@ -494,7 +384,7 @@
         goto out;
     }
 
-    if (!token_is_operator(peek, ']')) {
+    if (peek->type != JSON_RSQUARE) {
         QObject *obj;
 
         obj = parse_value(ctxt, ap);
@@ -511,8 +401,8 @@
             goto out;
         }
 
-        while (!token_is_operator(token, ']')) {
-            if (!token_is_operator(token, ',')) {
+        while (token->type != JSON_RSQUARE) {
+            if (token->type != JSON_COMMA) {
                 parse_error(ctxt, token, "expected separator in list");
                 goto out;
             }
@@ -538,97 +428,68 @@
     return QOBJECT(list);
 
 out:
-    parser_context_restore(ctxt, saved_ctxt);
     QDECREF(list);
     return NULL;
 }
 
 static QObject *parse_keyword(JSONParserContext *ctxt)
 {
-    QObject *token, *ret;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
+    assert(token && token->type == JSON_KEYWORD);
+
+    if (!strcmp(token->str, "true")) {
+        return QOBJECT(qbool_from_bool(true));
+    } else if (!strcmp(token->str, "false")) {
+        return QOBJECT(qbool_from_bool(false));
+    } else if (!strcmp(token->str, "null")) {
+        return qnull();
     }
-
-    if (token_get_type(token) != JSON_KEYWORD) {
-        goto out;
-    }
-
-    if (token_is_keyword(token, "true")) {
-        ret = QOBJECT(qbool_from_int(true));
-    } else if (token_is_keyword(token, "false")) {
-        ret = QOBJECT(qbool_from_int(false));
-    } else {
-        parse_error(ctxt, token, "invalid keyword `%s'", token_get_value(token));
-        goto out;
-    }
-
-    return ret;
-
-out: 
-    parser_context_restore(ctxt, saved_ctxt);
-
+    parse_error(ctxt, token, "invalid keyword '%s'", token->str);
     return NULL;
 }
 
 static QObject *parse_escape(JSONParserContext *ctxt, va_list *ap)
 {
-    QObject *token = NULL, *obj;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     if (ap == NULL) {
-        goto out;
+        return NULL;
     }
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
+    assert(token && token->type == JSON_ESCAPE);
+
+    if (!strcmp(token->str, "%p")) {
+        return va_arg(*ap, QObject *);
+    } else if (!strcmp(token->str, "%i")) {
+        return QOBJECT(qbool_from_bool(va_arg(*ap, int)));
+    } else if (!strcmp(token->str, "%d")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, int)));
+    } else if (!strcmp(token->str, "%ld")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, long)));
+    } else if (!strcmp(token->str, "%lld") ||
+               !strcmp(token->str, "%I64d")) {
+        return QOBJECT(qint_from_int(va_arg(*ap, long long)));
+    } else if (!strcmp(token->str, "%s")) {
+        return QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
+    } else if (!strcmp(token->str, "%f")) {
+        return QOBJECT(qfloat_from_double(va_arg(*ap, double)));
     }
-
-    if (token_is_escape(token, "%p")) {
-        obj = va_arg(*ap, QObject *);
-    } else if (token_is_escape(token, "%i")) {
-        obj = QOBJECT(qbool_from_int(va_arg(*ap, int)));
-    } else if (token_is_escape(token, "%d")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, int)));
-    } else if (token_is_escape(token, "%ld")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, long)));
-    } else if (token_is_escape(token, "%lld") ||
-               token_is_escape(token, "%I64d")) {
-        obj = QOBJECT(qint_from_int(va_arg(*ap, long long)));
-    } else if (token_is_escape(token, "%s")) {
-        obj = QOBJECT(qstring_from_str(va_arg(*ap, const char *)));
-    } else if (token_is_escape(token, "%f")) {
-        obj = QOBJECT(qfloat_from_double(va_arg(*ap, double)));
-    } else {
-        goto out;
-    }
-
-    return obj;
-
-out:
-    parser_context_restore(ctxt, saved_ctxt);
-
     return NULL;
 }
 
 static QObject *parse_literal(JSONParserContext *ctxt)
 {
-    QObject *token, *obj;
-    JSONParserContext saved_ctxt = parser_context_save(ctxt);
+    JSONToken *token;
 
     token = parser_context_pop_token(ctxt);
-    if (token == NULL) {
-        goto out;
-    }
+    assert(token);
 
-    switch (token_get_type(token)) {
+    switch (token->type) {
     case JSON_STRING:
-        obj = QOBJECT(qstring_from_escaped_str(ctxt, token));
-        break;
+        return QOBJECT(qstring_from_escaped_str(ctxt, token));
     case JSON_INTEGER: {
         /* A possibility exists that this is a whole-valued float where the
          * fractional part was left out due to being 0 (.0). It's not a big
@@ -645,56 +506,57 @@
         int64_t value;
 
         errno = 0; /* strtoll doesn't set errno on success */
-        value = strtoll(token_get_value(token), NULL, 10);
+        value = strtoll(token->str, NULL, 10);
         if (errno != ERANGE) {
-            obj = QOBJECT(qint_from_int(value));
-            break;
+            return QOBJECT(qint_from_int(value));
         }
         /* fall through to JSON_FLOAT */
     }
     case JSON_FLOAT:
-        /* FIXME dependent on locale */
-        obj = QOBJECT(qfloat_from_double(strtod(token_get_value(token), NULL)));
-        break;
+        /* FIXME dependent on locale; a pervasive issue in QEMU */
+        /* FIXME our lexer matches RFC 7159 in forbidding Inf or NaN,
+         * but those might be useful extensions beyond JSON */
+        return QOBJECT(qfloat_from_double(strtod(token->str, NULL)));
     default:
-        goto out;
+        abort();
     }
-
-    return obj;
-
-out:
-    parser_context_restore(ctxt, saved_ctxt);
-
-    return NULL;
 }
 
 static QObject *parse_value(JSONParserContext *ctxt, va_list *ap)
 {
-    QObject *obj;
+    JSONToken *token;
 
-    obj = parse_object(ctxt, ap);
-    if (obj == NULL) {
-        obj = parse_array(ctxt, ap);
-    }
-    if (obj == NULL) {
-        obj = parse_escape(ctxt, ap);
-    }
-    if (obj == NULL) {
-        obj = parse_keyword(ctxt);
-    } 
-    if (obj == NULL) {
-        obj = parse_literal(ctxt);
+    token = parser_context_peek_token(ctxt);
+    if (token == NULL) {
+        parse_error(ctxt, NULL, "premature EOI");
+        return NULL;
     }
 
-    return obj;
+    switch (token->type) {
+    case JSON_LCURLY:
+        return parse_object(ctxt, ap);
+    case JSON_LSQUARE:
+        return parse_array(ctxt, ap);
+    case JSON_ESCAPE:
+        return parse_escape(ctxt, ap);
+    case JSON_INTEGER:
+    case JSON_FLOAT:
+    case JSON_STRING:
+        return parse_literal(ctxt);
+    case JSON_KEYWORD:
+        return parse_keyword(ctxt);
+    default:
+        parse_error(ctxt, token, "expecting value");
+        return NULL;
+    }
 }
 
-QObject *json_parser_parse(QList *tokens, va_list *ap)
+QObject *json_parser_parse(GQueue *tokens, va_list *ap)
 {
     return json_parser_parse_err(tokens, ap, NULL);
 }
 
-QObject *json_parser_parse_err(QList *tokens, va_list *ap, Error **errp)
+QObject *json_parser_parse_err(GQueue *tokens, va_list *ap, Error **errp)
 {
     JSONParserContext *ctxt = parser_context_new(tokens);
     QObject *result;
diff --git a/qobject/json-streamer.c b/qobject/json-streamer.c
index 1b2f9b1..c51c202 100644
--- a/qobject/json-streamer.c
+++ b/qobject/json-streamer.c
@@ -11,50 +11,63 @@
  *
  */
 
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qdict.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/json-lexer.h"
 #include "qapi/qmp/json-streamer.h"
 
 #define MAX_TOKEN_SIZE (64ULL << 20)
+#define MAX_TOKEN_COUNT (2ULL << 20)
 #define MAX_NESTING (1ULL << 10)
 
-static void json_message_process_token(JSONLexer *lexer, QString *token, JSONTokenType type, int x, int y)
+static void json_message_free_token(void *token, void *opaque)
+{
+    g_free(token);
+}
+
+static void json_message_free_tokens(JSONMessageParser *parser)
+{
+    if (parser->tokens) {
+        g_queue_foreach(parser->tokens, json_message_free_token, NULL);
+        g_queue_free(parser->tokens);
+        parser->tokens = NULL;
+    }
+}
+
+static void json_message_process_token(JSONLexer *lexer, GString *input,
+                                       JSONTokenType type, int x, int y)
 {
     JSONMessageParser *parser = container_of(lexer, JSONMessageParser, lexer);
-    QDict *dict;
+    JSONToken *token;
+    GQueue *tokens;
 
-    if (type == JSON_OPERATOR) {
-        switch (qstring_get_str(token)[0]) {
-        case '{':
-            parser->brace_count++;
-            break;
-        case '}':
-            parser->brace_count--;
-            break;
-        case '[':
-            parser->bracket_count++;
-            break;
-        case ']':
-            parser->bracket_count--;
-            break;
-        default:
-            break;
-        }
+    switch (type) {
+    case JSON_LCURLY:
+        parser->brace_count++;
+        break;
+    case JSON_RCURLY:
+        parser->brace_count--;
+        break;
+    case JSON_LSQUARE:
+        parser->bracket_count++;
+        break;
+    case JSON_RSQUARE:
+        parser->bracket_count--;
+        break;
+    default:
+        break;
     }
 
-    dict = qdict_new();
-    qdict_put(dict, "type", qint_from_int(type));
-    QINCREF(token);
-    qdict_put(dict, "token", token);
-    qdict_put(dict, "x", qint_from_int(x));
-    qdict_put(dict, "y", qint_from_int(y));
+    token = g_malloc(sizeof(JSONToken) + input->len + 1);
+    token->type = type;
+    memcpy(token->str, input->str, input->len);
+    token->str[input->len] = 0;
+    token->x = x;
+    token->y = y;
 
-    parser->token_size += token->length;
+    parser->token_size += input->len;
 
-    qlist_append(parser->tokens, dict);
+    g_queue_push_tail(parser->tokens, token);
 
     if (type == JSON_ERROR) {
         goto out_emit_bad;
@@ -64,41 +77,42 @@
          parser->bracket_count == 0)) {
         goto out_emit;
     } else if (parser->token_size > MAX_TOKEN_SIZE ||
-               parser->bracket_count > MAX_NESTING ||
-               parser->brace_count > MAX_NESTING) {
+               g_queue_get_length(parser->tokens) > MAX_TOKEN_COUNT ||
+               parser->bracket_count + parser->brace_count > MAX_NESTING) {
         /* Security consideration, we limit total memory allocated per object
          * and the maximum recursion depth that a message can force.
          */
-        goto out_emit;
+        goto out_emit_bad;
     }
 
     return;
 
 out_emit_bad:
-    /* clear out token list and tell the parser to emit and error
+    /*
+     * Clear out token list and tell the parser to emit an error
      * indication by passing it a NULL list
      */
-    QDECREF(parser->tokens);
-    parser->tokens = NULL;
+    json_message_free_tokens(parser);
 out_emit:
     /* send current list of tokens to parser and reset tokenizer */
     parser->brace_count = 0;
     parser->bracket_count = 0;
-    parser->emit(parser, parser->tokens);
-    if (parser->tokens) {
-        QDECREF(parser->tokens);
-    }
-    parser->tokens = qlist_new();
+    /* parser->emit takes ownership of parser->tokens.  Remove our own
+     * reference to parser->tokens before handing it out to parser->emit.
+     */
+    tokens = parser->tokens;
+    parser->tokens = g_queue_new();
+    parser->emit(parser, tokens);
     parser->token_size = 0;
 }
 
 void json_message_parser_init(JSONMessageParser *parser,
-                              void (*func)(JSONMessageParser *, QList *))
+                              void (*func)(JSONMessageParser *, GQueue *))
 {
     parser->emit = func;
     parser->brace_count = 0;
     parser->bracket_count = 0;
-    parser->tokens = qlist_new();
+    parser->tokens = g_queue_new();
     parser->token_size = 0;
 
     json_lexer_init(&parser->lexer, json_message_process_token);
@@ -118,5 +132,5 @@
 void json_message_parser_destroy(JSONMessageParser *parser)
 {
     json_lexer_destroy(&parser->lexer);
-    QDECREF(parser->tokens);
+    json_message_free_tokens(parser);
 }
diff --git a/qobject/qbool.c b/qobject/qbool.c
index a3d2afa..0606bbd 100644
--- a/qobject/qbool.c
+++ b/qobject/qbool.c
@@ -11,37 +11,31 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qbool.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
-static void qbool_destroy_obj(QObject *obj);
-
-static const QType qbool_type = {
-    .code = QTYPE_QBOOL,
-    .destroy = qbool_destroy_obj,
-};
-
 /**
- * qbool_from_int(): Create a new QBool from an int
+ * qbool_from_bool(): Create a new QBool from a bool
  *
  * Return strong reference.
  */
-QBool *qbool_from_int(int value)
+QBool *qbool_from_bool(bool value)
 {
     QBool *qb;
 
     qb = g_malloc(sizeof(*qb));
+    qobject_init(QOBJECT(qb), QTYPE_QBOOL);
     qb->value = value;
-    QOBJECT_INIT(qb, &qbool_type);
 
     return qb;
 }
 
 /**
- * qbool_get_int(): Get the stored int
+ * qbool_get_bool(): Get the stored bool
  */
-int qbool_get_int(const QBool *qb)
+bool qbool_get_bool(const QBool *qb)
 {
     return qb->value;
 }
@@ -51,9 +45,9 @@
  */
 QBool *qobject_to_qbool(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QBOOL)
+    if (!obj || qobject_type(obj) != QTYPE_QBOOL) {
         return NULL;
-
+    }
     return container_of(obj, QBool, base);
 }
 
@@ -61,7 +55,7 @@
  * qbool_destroy_obj(): Free all memory allocated by a
  * QBool object
  */
-static void qbool_destroy_obj(QObject *obj)
+void qbool_destroy_obj(QObject *obj)
 {
     assert(obj != NULL);
     g_free(qobject_to_qbool(obj));
diff --git a/qobject/qdict.c b/qobject/qdict.c
index 24ea1c6..6d66255 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -10,6 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qfloat.h"
 #include "qapi/qmp/qdict.h"
@@ -18,13 +19,7 @@
 #include "qapi/qmp/qobject.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
-
-static void qdict_destroy_obj(QObject *obj);
-
-static const QType qdict_type = {
-    .code = QTYPE_QDICT,
-    .destroy = qdict_destroy_obj,
-};
+#include "qemu/cutils.h"
 
 /**
  * qdict_new(): Create a new QDict
@@ -36,7 +31,7 @@
     QDict *qdict;
 
     qdict = g_malloc0(sizeof(*qdict));
-    QOBJECT_INIT(qdict, &qdict_type);
+    qobject_init(QOBJECT(qdict), QTYPE_QDICT);
 
     return qdict;
 }
@@ -46,9 +41,9 @@
  */
 QDict *qobject_to_qdict(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QDICT)
+    if (!obj || qobject_type(obj) != QTYPE_QDICT) {
         return NULL;
-
+    }
     return container_of(obj, QDict, base);
 }
 
@@ -184,8 +179,7 @@
 /**
  * qdict_get_obj(): Get a QObject of a specific type
  */
-static QObject *qdict_get_obj(const QDict *qdict, const char *key,
-                              qtype_code type)
+static QObject *qdict_get_obj(const QDict *qdict, const char *key, QType type)
 {
     QObject *obj;
 
@@ -229,8 +223,7 @@
  */
 int64_t qdict_get_int(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QINT);
-    return qint_get_int(qobject_to_qint(obj));
+    return qint_get_int(qobject_to_qint(qdict_get(qdict, key)));
 }
 
 /**
@@ -241,10 +234,9 @@
  *
  * Return bool mapped by 'key'.
  */
-int qdict_get_bool(const QDict *qdict, const char *key)
+bool qdict_get_bool(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QBOOL);
-    return qbool_get_int(qobject_to_qbool(obj));
+    return qbool_get_bool(qobject_to_qbool(qdict_get(qdict, key)));
 }
 
 /**
@@ -270,7 +262,7 @@
  */
 QDict *qdict_get_qdict(const QDict *qdict, const char *key)
 {
-    return qobject_to_qdict(qdict_get_obj(qdict, key, QTYPE_QDICT));
+    return qobject_to_qdict(qdict_get(qdict, key));
 }
 
 /**
@@ -284,8 +276,7 @@
  */
 const char *qdict_get_str(const QDict *qdict, const char *key)
 {
-    QObject *obj = qdict_get_obj(qdict, key, QTYPE_QSTRING);
-    return qstring_get_str(qobject_to_qstring(obj));
+    return qstring_get_str(qobject_to_qstring(qdict_get(qdict, key)));
 }
 
 /**
@@ -298,13 +289,9 @@
 int64_t qdict_get_try_int(const QDict *qdict, const char *key,
                           int64_t def_value)
 {
-    QObject *obj;
+    QInt *qint = qobject_to_qint(qdict_get(qdict, key));
 
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QINT)
-        return def_value;
-
-    return qint_get_int(qobject_to_qint(obj));
+    return qint ? qint_get_int(qint) : def_value;
 }
 
 /**
@@ -314,15 +301,11 @@
  * dictionary or if the stored object is not of QBool type
  * 'def_value' will be returned.
  */
-int qdict_get_try_bool(const QDict *qdict, const char *key, int def_value)
+bool qdict_get_try_bool(const QDict *qdict, const char *key, bool def_value)
 {
-    QObject *obj;
+    QBool *qbool = qobject_to_qbool(qdict_get(qdict, key));
 
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QBOOL)
-        return def_value;
-
-    return qbool_get_int(qobject_to_qbool(obj));
+    return qbool ? qbool_get_bool(qbool) : def_value;
 }
 
 /**
@@ -335,13 +318,9 @@
  */
 const char *qdict_get_try_str(const QDict *qdict, const char *key)
 {
-    QObject *obj;
+    QString *qstr = qobject_to_qstring(qdict_get(qdict, key));
 
-    obj = qdict_get(qdict, key);
-    if (!obj || qobject_type(obj) != QTYPE_QSTRING)
-        return NULL;
-
-    return qstring_get_str(qobject_to_qstring(obj));
+    return qstr ? qstring_get_str(qstr) : NULL;
 }
 
 /**
@@ -456,7 +435,7 @@
 /**
  * qdict_destroy_obj(): Free all the memory allocated by a QDict
  */
-static void qdict_destroy_obj(QObject *obj)
+void qdict_destroy_obj(QObject *obj)
 {
     int i;
     QDict *qdict;
@@ -477,6 +456,39 @@
     g_free(qdict);
 }
 
+/**
+ * qdict_copy_default(): If no entry mapped by 'key' exists in 'dst' yet, the
+ * value of 'key' in 'src' is copied there (and the refcount increased
+ * accordingly).
+ */
+void qdict_copy_default(QDict *dst, QDict *src, const char *key)
+{
+    QObject *val;
+
+    if (qdict_haskey(dst, key)) {
+        return;
+    }
+
+    val = qdict_get(src, key);
+    if (val) {
+        qobject_incref(val);
+        qdict_put_obj(dst, key, val);
+    }
+}
+
+/**
+ * qdict_set_default_str(): If no entry mapped by 'key' exists in 'dst' yet, a
+ * new QString initialised by 'val' is put there.
+ */
+void qdict_set_default_str(QDict *dst, const char *key, const char *val)
+{
+    if (qdict_haskey(dst, key)) {
+        return;
+    }
+
+    qdict_put(dst, key, qstring_from_str(val));
+}
+
 static void qdict_flatten_qdict(QDict *qdict, QDict *target,
                                 const char *prefix);
 
@@ -597,17 +609,21 @@
     }
 }
 
-static bool qdict_has_prefixed_entries(const QDict *src, const char *start)
+static int qdict_count_prefixed_entries(const QDict *src, const char *start)
 {
     const QDictEntry *entry;
+    int count = 0;
 
     for (entry = qdict_first(src); entry; entry = qdict_next(src, entry)) {
         if (strstart(entry->key, start, NULL)) {
-            return true;
+            if (count == INT_MAX) {
+                return -ERANGE;
+            }
+            count++;
         }
     }
 
-    return false;
+    return count;
 }
 
 /**
@@ -648,7 +664,8 @@
         assert(snprintf_ret < 32);
         (void)snprintf_ret;
 
-        is_subqdict = qdict_has_prefixed_entries(src, prefix);
+        /* Overflow is the same as positive non-zero results */
+        is_subqdict = qdict_count_prefixed_entries(src, prefix);
 
         // There may be either a single subordinate object (named "%u") or
         // multiple objects (each with a key prefixed "%u."), but not both.
@@ -669,6 +686,68 @@
 }
 
 /**
+ * qdict_array_entries(): Returns the number of direct array entries if the
+ * sub-QDict of src specified by the prefix in subqdict (or src itself for
+ * prefix == "") is valid as an array, i.e. the length of the created list if
+ * the sub-QDict would become empty after calling qdict_array_split() on it. If
+ * the array is not valid, -EINVAL is returned.
+ */
+int qdict_array_entries(QDict *src, const char *subqdict)
+{
+    const QDictEntry *entry;
+    unsigned i;
+    unsigned entries = 0;
+    size_t subqdict_len = strlen(subqdict);
+
+    assert(!subqdict_len || subqdict[subqdict_len - 1] == '.');
+
+    /* qdict_array_split() loops until UINT_MAX, but as we want to return
+     * negative errors, we only have a signed return value here. Any additional
+     * entries will lead to -EINVAL. */
+    for (i = 0; i < INT_MAX; i++) {
+        QObject *subqobj;
+        int subqdict_entries;
+        char *prefix = g_strdup_printf("%s%u.", subqdict, i);
+
+        subqdict_entries = qdict_count_prefixed_entries(src, prefix);
+
+        /* Remove ending "." */
+        prefix[strlen(prefix) - 1] = 0;
+        subqobj = qdict_get(src, prefix);
+
+        g_free(prefix);
+
+        if (subqdict_entries < 0) {
+            return subqdict_entries;
+        }
+
+        /* There may be either a single subordinate object (named "%u") or
+         * multiple objects (each with a key prefixed "%u."), but not both. */
+        if (subqobj && subqdict_entries) {
+            return -EINVAL;
+        } else if (!subqobj && !subqdict_entries) {
+            break;
+        }
+
+        entries += subqdict_entries ? subqdict_entries : 1;
+    }
+
+    /* Consider everything handled that isn't part of the given sub-QDict */
+    for (entry = qdict_first(src); entry; entry = qdict_next(src, entry)) {
+        if (!strstart(qdict_entry_key(entry), subqdict, NULL)) {
+            entries++;
+        }
+    }
+
+    /* Anything left in the sub-QDict that wasn't handled? */
+    if (qdict_size(src) != entries) {
+        return -EINVAL;
+    }
+
+    return i;
+}
+
+/**
  * qdict_join(): Absorb the src QDict into the dest QDict, that is, move all
  * elements from src to dest.
  *
diff --git a/qobject/qerror.c b/qobject/qerror.c
deleted file mode 100644
index e3608e2..0000000
--- a/qobject/qerror.c
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * QError Module
- *
- * Copyright (C) 2009 Red Hat Inc.
- *
- * Authors:
- *  Luiz Capitulino <lcapitulino@redhat.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- */
-
-#include "monitor/monitor.h"
-#include "qapi/qmp/qjson.h"
-#include "qapi/qmp/qerror.h"
-#include "qemu-common.h"
-
-static void qerror_destroy_obj(QObject *obj);
-
-static const QType qerror_type = {
-    .code = QTYPE_QERROR,
-    .destroy = qerror_destroy_obj,
-};
-
-/**
- * qerror_new(): Create a new QError
- *
- * Return strong reference.
- */
-static QError *qerror_new(void)
-{
-    QError *qerr;
-
-    qerr = g_malloc0(sizeof(*qerr));
-    QOBJECT_INIT(qerr, &qerror_type);
-
-    return qerr;
-}
-
-/**
- * qerror_from_info(): Create a new QError from error information
- *
- * Return strong reference.
- */
-static QError * GCC_FMT_ATTR(2, 0)
-qerror_from_info(ErrorClass err_class, const char *fmt, va_list *va)
-{
-    QError *qerr;
-
-    qerr = qerror_new();
-    loc_save(&qerr->loc);
-
-    qerr->err_msg = g_strdup_vprintf(fmt, *va);
-    qerr->err_class = err_class;
-
-    return qerr;
-}
-
-/**
- * qerror_human(): Format QError data into human-readable string.
- */
-QString *qerror_human(const QError *qerror)
-{
-    return qstring_from_str(qerror->err_msg);
-}
-
-/**
- * qerror_print(): Print QError data
- *
- * This function will print the member 'desc' of the specified QError object,
- * it uses error_report() for this, so that the output is routed to the right
- * place (ie. stderr or Monitor's device).
- */
-static void qerror_print(QError *qerror)
-{
-    QString *qstring = qerror_human(qerror);
-    loc_push_restore(&qerror->loc);
-    error_report("%s", qstring_get_str(qstring));
-    loc_pop(&qerror->loc);
-    QDECREF(qstring);
-}
-
-void qerror_report(ErrorClass eclass, const char *fmt, ...)
-{
-    va_list va;
-    QError *qerror;
-
-    va_start(va, fmt);
-    qerror = qerror_from_info(eclass, fmt, &va);
-    va_end(va);
-
-    if (monitor_cur_is_qmp()) {
-        monitor_set_error(cur_mon, qerror);
-    } else {
-        qerror_print(qerror);
-        QDECREF(qerror);
-    }
-}
-
-/* Evil... */
-struct Error
-{
-    char *msg;
-    ErrorClass err_class;
-};
-
-void qerror_report_err(Error *err)
-{
-    QError *qerr;
-
-    qerr = qerror_new();
-    loc_save(&qerr->loc);
-    qerr->err_msg = g_strdup(err->msg);
-    qerr->err_class = err->err_class;
-
-    if (monitor_cur_is_qmp()) {
-        monitor_set_error(cur_mon, qerr);
-    } else {
-        qerror_print(qerr);
-        QDECREF(qerr);
-    }
-}
-
-/**
- * qobject_to_qerror(): Convert a QObject into a QError
- */
-static QError *qobject_to_qerror(const QObject *obj)
-{
-    if (qobject_type(obj) != QTYPE_QERROR) {
-        return NULL;
-    }
-
-    return container_of(obj, QError, base);
-}
-
-/**
- * qerror_destroy_obj(): Free all memory allocated by a QError
- */
-static void qerror_destroy_obj(QObject *obj)
-{
-    QError *qerr;
-
-    assert(obj != NULL);
-    qerr = qobject_to_qerror(obj);
-
-    g_free(qerr->err_msg);
-    g_free(qerr);
-}
diff --git a/qobject/qfloat.c b/qobject/qfloat.c
index 7de0992..d5da847 100644
--- a/qobject/qfloat.c
+++ b/qobject/qfloat.c
@@ -11,17 +11,11 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qfloat.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
-static void qfloat_destroy_obj(QObject *obj);
-
-static const QType qfloat_type = {
-    .code = QTYPE_QFLOAT,
-    .destroy = qfloat_destroy_obj,
-};
-
 /**
  * qfloat_from_int(): Create a new QFloat from a float
  *
@@ -32,8 +26,8 @@
     QFloat *qf;
 
     qf = g_malloc(sizeof(*qf));
+    qobject_init(QOBJECT(qf), QTYPE_QFLOAT);
     qf->value = value;
-    QOBJECT_INIT(qf, &qfloat_type);
 
     return qf;
 }
@@ -51,9 +45,9 @@
  */
 QFloat *qobject_to_qfloat(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QFLOAT)
+    if (!obj || qobject_type(obj) != QTYPE_QFLOAT) {
         return NULL;
-
+    }
     return container_of(obj, QFloat, base);
 }
 
@@ -61,7 +55,7 @@
  * qfloat_destroy_obj(): Free all memory allocated by a
  * QFloat object
  */
-static void qfloat_destroy_obj(QObject *obj)
+void qfloat_destroy_obj(QObject *obj)
 {
     assert(obj != NULL);
     g_free(qobject_to_qfloat(obj));
diff --git a/qobject/qint.c b/qobject/qint.c
index 86b9b04..d7d1b30 100644
--- a/qobject/qint.c
+++ b/qobject/qint.c
@@ -10,17 +10,11 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu-common.h"
 
-static void qint_destroy_obj(QObject *obj);
-
-static const QType qint_type = {
-    .code = QTYPE_QINT,
-    .destroy = qint_destroy_obj,
-};
-
 /**
  * qint_from_int(): Create a new QInt from an int64_t
  *
@@ -31,8 +25,8 @@
     QInt *qi;
 
     qi = g_malloc(sizeof(*qi));
+    qobject_init(QOBJECT(qi), QTYPE_QINT);
     qi->value = value;
-    QOBJECT_INIT(qi, &qint_type);
 
     return qi;
 }
@@ -50,9 +44,9 @@
  */
 QInt *qobject_to_qint(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QINT)
+    if (!obj || qobject_type(obj) != QTYPE_QINT) {
         return NULL;
-
+    }
     return container_of(obj, QInt, base);
 }
 
@@ -60,7 +54,7 @@
  * qint_destroy_obj(): Free all memory allocated by a
  * QInt object
  */
-static void qint_destroy_obj(QObject *obj)
+void qint_destroy_obj(QObject *obj)
 {
     assert(obj != NULL);
     g_free(qobject_to_qint(obj));
diff --git a/qobject/qjson.c b/qobject/qjson.c
index 6cf2511..9a0de89 100644
--- a/qobject/qjson.c
+++ b/qobject/qjson.c
@@ -11,15 +11,13 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/json-lexer.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/qjson.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qbool.h"
-#include "qapi/qmp/qfloat.h"
-#include "qapi/qmp/qdict.h"
+#include "qapi/qmp/types.h"
+#include "qemu/unicode.h"
 
 typedef struct JSONParsingState
 {
@@ -28,7 +26,7 @@
     QObject *result;
 } JSONParsingState;
 
-static void parse_json(JSONMessageParser *parser, QList *tokens)
+static void parse_json(JSONMessageParser *parser, GQueue *tokens)
 {
     JSONParsingState *s = container_of(parser, JSONParsingState, parser);
     s->result = json_parser_parse(tokens, s->ap);
@@ -86,8 +84,9 @@
     QString *qkey;
     int j;
 
-    if (s->count)
-        qstring_append(s->str, ", ");
+    if (s->count) {
+        qstring_append(s->str, s->pretty ? "," : ", ");
+    }
 
     if (s->pretty) {
         qstring_append(s->str, "\n");
@@ -109,8 +108,9 @@
     ToJsonIterState *s = opaque;
     int j;
 
-    if (s->count)
-        qstring_append(s->str, ", ");
+    if (s->count) {
+        qstring_append(s->str, s->pretty ? "," : ", ");
+    }
 
     if (s->pretty) {
         qstring_append(s->str, "\n");
@@ -125,6 +125,9 @@
 static void to_json(const QObject *obj, QString *str, int pretty, int indent)
 {
     switch (qobject_type(obj)) {
+    case QTYPE_QNULL:
+        qstring_append(str, "null");
+        break;
     case QTYPE_QINT: {
         QInt *val = qobject_to_qint(obj);
         char buffer[1024];
@@ -232,6 +235,15 @@
         char buffer[1024];
         int len;
 
+        /* FIXME: snprintf() is locale dependent; but JSON requires
+         * numbers to be formatted as if in the C locale. Dependence
+         * on C locale is a pervasive issue in QEMU. */
+        /* FIXME: This risks printing Inf or NaN, which are not valid
+         * JSON values. */
+        /* FIXME: the default precision of 6 for %f often causes
+         * rounding errors; we should be using DBL_DECIMAL_DIG (17),
+         * and only rounding to a shorter number if the result would
+         * still produce the same floating point value.  */
         len = snprintf(buffer, sizeof(buffer), "%f", qfloat_get_double(val));
         while (len > 0 && buffer[len - 1] == '0') {
             len--;
@@ -242,25 +254,21 @@
         } else {
             buffer[len] = 0;
         }
-        
+
         qstring_append(str, buffer);
         break;
     }
     case QTYPE_QBOOL: {
         QBool *val = qobject_to_qbool(obj);
 
-        if (qbool_get_int(val)) {
+        if (qbool_get_bool(val)) {
             qstring_append(str, "true");
         } else {
             qstring_append(str, "false");
         }
         break;
     }
-    case QTYPE_QERROR:
-        /* XXX: should QError be emitted? */
-    case QTYPE_NONE:
-        break;
-    case QTYPE_MAX:
+    default:
         abort();
     }
 }
diff --git a/qobject/qlist.c b/qobject/qlist.c
index 1ced0de..86b60cb 100644
--- a/qobject/qlist.c
+++ b/qobject/qlist.c
@@ -10,18 +10,12 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qlist.h"
 #include "qapi/qmp/qobject.h"
 #include "qemu/queue.h"
 #include "qemu-common.h"
 
-static void qlist_destroy_obj(QObject *obj);
-
-static const QType qlist_type = {
-    .code = QTYPE_QLIST,
-    .destroy = qlist_destroy_obj,
-};
- 
 /**
  * qlist_new(): Create a new QList
  *
@@ -32,8 +26,8 @@
     QList *qlist;
 
     qlist = g_malloc(sizeof(*qlist));
+    qobject_init(QOBJECT(qlist), QTYPE_QLIST);
     QTAILQ_INIT(&qlist->head);
-    QOBJECT_INIT(qlist, &qlist_type);
 
     return qlist;
 }
@@ -106,7 +100,6 @@
 QObject *qlist_peek(QList *qlist)
 {
     QListEntry *entry;
-    QObject *ret;
 
     if (qlist == NULL || QTAILQ_EMPTY(&qlist->head)) {
         return NULL;
@@ -114,9 +107,7 @@
 
     entry = QTAILQ_FIRST(&qlist->head);
 
-    ret = entry->value;
-
-    return ret;
+    return entry->value;
 }
 
 int qlist_empty(const QList *qlist)
@@ -142,17 +133,16 @@
  */
 QList *qobject_to_qlist(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QLIST) {
+    if (!obj || qobject_type(obj) != QTYPE_QLIST) {
         return NULL;
     }
-
     return container_of(obj, QList, base);
 }
 
 /**
  * qlist_destroy_obj(): Free all the memory allocated by a QList
  */
-static void qlist_destroy_obj(QObject *obj)
+void qlist_destroy_obj(QObject *obj)
 {
     QList *qlist;
     QListEntry *entry, *next_entry;
diff --git a/qobject/qnull.c b/qobject/qnull.c
new file mode 100644
index 0000000..c124d05
--- /dev/null
+++ b/qobject/qnull.c
@@ -0,0 +1,20 @@
+/*
+ * QNull
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1
+ * or later.  See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qobject.h"
+
+QObject qnull_ = {
+    .type = QTYPE_QNULL,
+    .refcnt = 1,
+};
diff --git a/qobject/qobject.c b/qobject/qobject.c
new file mode 100644
index 0000000..fe4fa10
--- /dev/null
+++ b/qobject/qobject.c
@@ -0,0 +1,30 @@
+/*
+ * QObject
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1
+ * or later.  See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qapi/qmp/types.h"
+
+static void (*qdestroy[QTYPE__MAX])(QObject *) = {
+    [QTYPE_NONE] = NULL,               /* No such object exists */
+    [QTYPE_QNULL] = NULL,              /* qnull_ is indestructible */
+    [QTYPE_QINT] = qint_destroy_obj,
+    [QTYPE_QSTRING] = qstring_destroy_obj,
+    [QTYPE_QDICT] = qdict_destroy_obj,
+    [QTYPE_QLIST] = qlist_destroy_obj,
+    [QTYPE_QFLOAT] = qfloat_destroy_obj,
+    [QTYPE_QBOOL] = qbool_destroy_obj,
+};
+
+void qobject_destroy(QObject *obj)
+{
+    assert(!obj->refcnt);
+    assert(QTYPE_QNULL < obj->type && obj->type < QTYPE__MAX);
+    qdestroy[obj->type](obj);
+}
diff --git a/qobject/qstring.c b/qobject/qstring.c
index 607b7a1..5da7b5f 100644
--- a/qobject/qstring.c
+++ b/qobject/qstring.c
@@ -10,17 +10,11 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
 
-static void qstring_destroy_obj(QObject *obj);
-
-static const QType qstring_type = {
-    .code = QTYPE_QSTRING,
-    .destroy = qstring_destroy_obj,
-};
-
 /**
  * qstring_new(): Create a new empty QString
  *
@@ -49,6 +43,7 @@
     QString *qstring;
 
     qstring = g_malloc(sizeof(*qstring));
+    qobject_init(QOBJECT(qstring), QTYPE_QSTRING);
 
     qstring->length = end - start + 1;
     qstring->capacity = qstring->length;
@@ -57,7 +52,6 @@
     memcpy(qstring->string, str + start, qstring->length);
     qstring->string[qstring->length] = 0;
 
-    QOBJECT_INIT(qstring, &qstring_type);
 
     return qstring;
 }
@@ -117,9 +111,9 @@
  */
 QString *qobject_to_qstring(const QObject *obj)
 {
-    if (qobject_type(obj) != QTYPE_QSTRING)
+    if (!obj || qobject_type(obj) != QTYPE_QSTRING) {
         return NULL;
-
+    }
     return container_of(obj, QString, base);
 }
 
@@ -138,7 +132,7 @@
  * qstring_destroy_obj(): Free all memory allocated by a QString
  * object
  */
-static void qstring_destroy_obj(QObject *obj)
+void qstring_destroy_obj(QObject *obj)
 {
     QString *qs;
 
diff --git a/qom/Makefile.objs b/qom/Makefile.objs
index 985003b..516349e 100644
--- a/qom/Makefile.objs
+++ b/qom/Makefile.objs
@@ -1,3 +1,4 @@
-common-obj-y = object.o container.o qom-qobject.o
-common-obj-y += cpu.o
-common-obj-y += object_interfaces.o
+qom-obj-y = object.o container.o qom-qobject.o
+qom-obj-y += object_interfaces.o
+
+common-obj-y = cpu.o
diff --git a/qom/container.c b/qom/container.c
index 62b1648..c9eb49b 100644
--- a/qom/container.c
+++ b/qom/container.c
@@ -10,9 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "qemu/module.h"
-#include <assert.h>
 
 static const TypeInfo container_info = {
     .name          = "container",
diff --git a/qom/cpu.c b/qom/cpu.c
index 79d2228..2553247 100644
--- a/qom/cpu.c
+++ b/qom/cpu.c
@@ -18,13 +18,17 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qom/cpu.h"
 #include "sysemu/kvm.h"
 #include "qemu/notify.h"
 #include "qemu/log.h"
+#include "exec/log.h"
 #include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
+#include "hw/qdev-properties.h"
 
 bool cpu_exists(int64_t id)
 {
@@ -43,7 +47,7 @@
 CPUState *cpu_generic_init(const char *typename, const char *cpu_model)
 {
     char *str, *name, *featurestr;
-    CPUState *cpu;
+    CPUState *cpu = NULL;
     ObjectClass *oc;
     CPUClass *cc;
     Error *err = NULL;
@@ -57,22 +61,23 @@
         return NULL;
     }
 
-    cpu = CPU(object_new(object_class_get_name(oc)));
-    cc = CPU_GET_CLASS(cpu);
-
+    cc = CPU_CLASS(oc);
     featurestr = strtok(NULL, ",");
-    cc->parse_features(cpu, featurestr, &err);
+    /* TODO: all callers of cpu_generic_init() need to be converted to
+     * call parse_features() only once, before calling cpu_generic_init().
+     */
+    cc->parse_features(object_class_get_name(oc), featurestr, &err);
     g_free(str);
     if (err != NULL) {
         goto out;
     }
 
+    cpu = CPU(object_new(object_class_get_name(oc)));
     object_property_set_bool(OBJECT(cpu), true, "realized", &err);
 
 out:
     if (err != NULL) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         object_unref(OBJECT(cpu));
         return NULL;
     }
@@ -97,7 +102,7 @@
 {
     CPUClass *cc = CPU_GET_CLASS(cpu);
 
-    return cc->get_memory_mapping(cpu, list, errp);
+    cc->get_memory_mapping(cpu, list, errp);
 }
 
 static void cpu_common_get_memory_mapping(CPUState *cpu,
@@ -115,6 +120,8 @@
 void cpu_exit(CPUState *cpu)
 {
     cpu->exit_request = 1;
+    /* Ensure cpu_exec will see the exit request after TCG has exited.  */
+    smp_wmb();
     cpu->tcg_exit_req = 1;
 }
 
@@ -129,7 +136,7 @@
 static int cpu_common_write_elf32_qemunote(WriteCoreDumpFunction f,
                                            CPUState *cpu, void *opaque)
 {
-    return -1;
+    return 0;
 }
 
 int cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cpu,
@@ -158,7 +165,7 @@
 static int cpu_common_write_elf64_qemunote(WriteCoreDumpFunction f,
                                            CPUState *cpu, void *opaque)
 {
-    return -1;
+    return 0;
 }
 
 int cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cpu,
@@ -187,6 +194,14 @@
     return 0;
 }
 
+static bool cpu_common_debug_check_watchpoint(CPUState *cpu, CPUWatchpoint *wp)
+{
+    /* If no extra check is required, QEMU watchpoint match can be considered
+     * as an architectural match.
+     */
+    return true;
+}
+
 bool target_words_bigendian(void);
 static bool cpu_common_virtio_is_big_endian(CPUState *cpu)
 {
@@ -242,13 +257,14 @@
     }
 
     cpu->interrupt_request = 0;
-    cpu->current_tb = NULL;
     cpu->halted = 0;
     cpu->mem_io_pc = 0;
     cpu->mem_io_vaddr = 0;
     cpu->icount_extra = 0;
     cpu->icount_decr.u32 = 0;
-    cpu->can_do_io = 0;
+    cpu->can_do_io = 1;
+    cpu->exception_index = -1;
+    cpu->crash_occurred = false;
     memset(cpu->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
 }
 
@@ -269,25 +285,37 @@
     return NULL;
 }
 
-static void cpu_common_parse_features(CPUState *cpu, char *features,
+static void cpu_common_parse_features(const char *typename, char *features,
                                       Error **errp)
 {
     char *featurestr; /* Single "key=value" string being parsed */
     char *val;
-    Error *err = NULL;
+    static bool cpu_globals_initialized;
+
+    /* TODO: all callers of ->parse_features() need to be changed to
+     * call it only once, so we can remove this check (or change it
+     * to assert(!cpu_globals_initialized).
+     * Current callers of ->parse_features() are:
+     * - cpu_generic_init()
+     */
+    if (cpu_globals_initialized) {
+        return;
+    }
+    cpu_globals_initialized = true;
 
     featurestr = features ? strtok(features, ",") : NULL;
 
     while (featurestr) {
         val = strchr(featurestr, '=');
         if (val) {
+            GlobalProperty *prop = g_new0(typeof(*prop), 1);
             *val = 0;
             val++;
-            object_property_parse(OBJECT(cpu), val, featurestr, &err);
-            if (err) {
-                error_propagate(errp, err);
-                return;
-            }
+            prop->driver = typename;
+            prop->property = g_strdup(featurestr);
+            prop->value = g_strdup(val);
+            prop->errp = &error_fatal;
+            qdev_prop_register_global(prop);
         } else {
             error_setg(errp, "Expected key=value format, found %s.",
                        featurestr);
@@ -312,7 +340,17 @@
     CPUState *cpu = CPU(obj);
     CPUClass *cc = CPU_GET_CLASS(obj);
 
+    cpu->cpu_index = UNASSIGNED_CPU_INDEX;
     cpu->gdb_num_regs = cpu->gdb_num_g_regs = cc->gdb_num_core_regs;
+    qemu_mutex_init(&cpu->work_mutex);
+    QTAILQ_INIT(&cpu->breakpoints);
+    QTAILQ_INIT(&cpu->watchpoints);
+    bitmap_zero(cpu->trace_dstate, TRACE_VCPU_EVENT_COUNT);
+}
+
+static void cpu_common_finalize(Object *obj)
+{
+    cpu_exec_exit(CPU(obj));
 }
 
 static int64_t cpu_common_get_arch_id(CPUState *cpu)
@@ -340,6 +378,7 @@
     k->gdb_write_register = cpu_common_gdb_write_register;
     k->virtio_is_big_endian = cpu_common_virtio_is_big_endian;
     k->debug_excp_handler = cpu_common_noop;
+    k->debug_check_watchpoint = cpu_common_debug_check_watchpoint;
     k->cpu_exec_enter = cpu_common_noop;
     k->cpu_exec_exit = cpu_common_noop;
     k->cpu_exec_interrupt = cpu_common_exec_interrupt;
@@ -356,6 +395,7 @@
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(CPUState),
     .instance_init = cpu_common_initfn,
+    .instance_finalize = cpu_common_finalize,
     .abstract = true,
     .class_size = sizeof(CPUClass),
     .class_init = cpu_class_init,
diff --git a/qom/object.c b/qom/object.c
index d242264..106aa8d 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -10,8 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qom/object.h"
-#include "qemu-common.h"
+#include "qom/object_interfaces.h"
+#include "qemu/cutils.h"
 #include "qapi/visitor.h"
 #include "qapi-visit.h"
 #include "qapi/string-input-visitor.h"
@@ -60,7 +63,7 @@
     const char *parent;
     TypeImpl *parent_type;
 
-    ObjectClass *class;
+    ObjectClass *clazz;
 
     int num_interfaces;
     InterfaceImpl interfaces[MAX_INTERFACES];
@@ -199,11 +202,19 @@
     return 0;
 }
 
+size_t object_type_get_instance_size(const char *typename)
+{
+    TypeImpl *type = type_get_by_name(typename);
+
+    g_assert(type != NULL);
+    return type_object_get_size(type);
+}
+
 static bool type_is_ancestor(TypeImpl *type, TypeImpl *target_type)
 {
     assert(target_type);
 
-    /* Check if typename is a direct ancestor of type */
+    /* Check if target_type is a direct ancestor of type */
     while (type) {
         if (type == target_type) {
             return true;
@@ -233,26 +244,36 @@
     type_initialize(iface_impl);
     g_free((char *)info.name);
 
-    new_iface = (InterfaceClass *)iface_impl->class;
-    new_iface->concrete_class = ti->class;
+    new_iface = (InterfaceClass *)iface_impl->clazz;
+    new_iface->concrete_class = ti->clazz;
     new_iface->interface_type = interface_type;
 
-    ti->class->interfaces = g_slist_append(ti->class->interfaces,
-                                           iface_impl->class);
+    ti->clazz->interfaces = g_slist_append(ti->clazz->interfaces,
+                                           iface_impl->clazz);
+}
+
+static void object_property_free(gpointer data)
+{
+    ObjectProperty *prop = data;
+
+    g_free(prop->name);
+    g_free(prop->type);
+    g_free(prop->description);
+    g_free(prop);
 }
 
 static void type_initialize(TypeImpl *ti)
 {
     TypeImpl *parent;
 
-    if (ti->class) {
+    if (ti->clazz) {
         return;
     }
 
     ti->class_size = type_class_get_size(ti);
     ti->instance_size = type_object_get_size(ti);
 
-    ti->class = g_malloc0(ti->class_size);
+    ti->clazz = g_malloc0(ti->class_size);
 
     parent = type_get_parent(ti);
     if (parent) {
@@ -260,11 +281,13 @@
         GSList *e;
         int i;
 
-        g_assert(parent->class_size <= ti->class_size);
-        memcpy(ti->class, parent->class, parent->class_size);
-        ti->class->interfaces = NULL;
+        g_assert_cmpint(parent->class_size, <=, ti->class_size);
+        memcpy(ti->clazz, parent->clazz, parent->class_size);
+        ti->clazz->interfaces = NULL;
+        ti->clazz->properties = g_hash_table_new_full(
+            g_str_hash, g_str_equal, g_free, object_property_free);
 
-        for (e = parent->class->interfaces; e; e = e->next) {
+        for (e = parent->clazz->interfaces; e; e = e->next) {
             InterfaceClass *iface = e->data;
             ObjectClass *klass = OBJECT_CLASS(iface);
 
@@ -273,7 +296,7 @@
 
         for (i = 0; i < ti->num_interfaces; i++) {
             TypeImpl *t = type_get_by_name(ti->interfaces[i].typename);
-            for (e = ti->class->interfaces; e; e = e->next) {
+            for (e = ti->clazz->interfaces; e; e = e->next) {
                 TypeImpl *target_type = OBJECT_CLASS(e->data)->type;
 
                 if (type_is_ancestor(target_type, t)) {
@@ -287,19 +310,22 @@
 
             type_initialize_interface(ti, t, t);
         }
+    } else {
+        ti->clazz->properties = g_hash_table_new_full(
+            g_str_hash, g_str_equal, g_free, object_property_free);
     }
 
-    ti->class->type = ti;
+    ti->clazz->type = ti;
 
     while (parent) {
         if (parent->class_base_init) {
-            parent->class_base_init(ti->class, ti->class_data);
+            parent->class_base_init(ti->clazz, ti->class_data);
         }
         parent = type_get_parent(parent);
     }
 
     if (ti->class_init) {
-        ti->class_init(ti->class, ti->class_data);
+        ti->class_init(ti->clazz, ti->class_data);
     }
 }
 
@@ -332,14 +358,15 @@
     g_assert(type != NULL);
     type_initialize(type);
 
-    g_assert(type->instance_size >= sizeof(Object));
+    g_assert_cmpint(type->instance_size, >=, sizeof(Object));
     g_assert(type->abstract == false);
-    g_assert(size >= type->instance_size);
+    g_assert_cmpint(size, >=, type->instance_size);
 
     memset(obj, 0, type->instance_size);
-    obj->klass = type->class;
+    obj->clazz = type->clazz;
     object_ref(obj);
-    QTAILQ_INIT(&obj->properties);
+    obj->properties = g_hash_table_new_full(g_str_hash, g_str_equal,
+                                            NULL, object_property_free);
     object_init_with_type(obj, type);
     object_post_init_with_type(obj, type);
 }
@@ -358,29 +385,51 @@
 
 static void object_property_del_all(Object *obj)
 {
-    while (!QTAILQ_EMPTY(&obj->properties)) {
-        ObjectProperty *prop = QTAILQ_FIRST(&obj->properties);
+    ObjectProperty *prop;
+    GHashTableIter iter;
+    gpointer key, value;
+    bool released;
 
-        QTAILQ_REMOVE(&obj->properties, prop, node);
-
-        if (prop->release) {
-            prop->release(obj, prop->name, prop->opaque);
+    do {
+        released = false;
+        g_hash_table_iter_init(&iter, obj->properties);
+        while (g_hash_table_iter_next(&iter, &key, &value)) {
+            prop = value;
+            if (prop->release) {
+                prop->release(obj, prop->name, prop->opaque);
+                prop->release = NULL;
+                released = true;
+                break;
+            }
+            g_hash_table_iter_remove(&iter);
         }
+    } while (released);
 
-        g_free(prop->name);
-        g_free(prop->type);
-        g_free(prop->description);
-        g_free(prop);
-    }
+    g_hash_table_unref(obj->properties);
 }
 
 static void object_property_del_child(Object *obj, Object *child, Error **errp)
 {
     ObjectProperty *prop;
+    GHashTableIter iter;
+    gpointer key, value;
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        prop = value;
         if (object_property_is_child(prop) && prop->opaque == child) {
-            object_property_del(obj, prop->name, errp);
+            if (prop->release) {
+                prop->release(obj, prop->name, prop->opaque);
+                prop->release = NULL;
+            }
+            break;
+        }
+    }
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, &key, &value)) {
+        prop = value;
+        if (object_property_is_child(prop) && prop->opaque == child) {
+            g_hash_table_iter_remove(&iter);
             break;
         }
     }
@@ -407,12 +456,12 @@
 static void object_finalize(void *data)
 {
     Object *obj = data;
-    TypeImpl *ti = obj->klass->type;
+    TypeImpl *ti = obj->clazz->type;
 
     object_property_del_all(obj);
     object_deinit(obj, ti);
 
-    g_assert(obj->ref == 0);
+    g_assert_cmpint(obj->ref, ==, 0);
     if (obj->free) {
         obj->free(obj);
     }
@@ -439,6 +488,112 @@
     return object_new_with_type(ti);
 }
 
+
+Object *object_new_with_props(const char *typename,
+                              Object *parent,
+                              const char *id,
+                              Error **errp,
+                              ...)
+{
+    va_list vargs;
+    Object *obj;
+
+    va_start(vargs, errp);
+    obj = object_new_with_propv(typename, parent, id, errp, vargs);
+    va_end(vargs);
+
+    return obj;
+}
+
+
+Object *object_new_with_propv(const char *typename,
+                              Object *parent,
+                              const char *id,
+                              Error **errp,
+                              va_list vargs)
+{
+    Object *obj;
+    ObjectClass *klass;
+    Error *local_err = NULL;
+
+    klass = object_class_by_name(typename);
+    if (!klass) {
+        error_setg(errp, "invalid object type: %s", typename);
+        return NULL;
+    }
+
+    if (object_class_is_abstract(klass)) {
+        error_setg(errp, "object type '%s' is abstract", typename);
+        return NULL;
+    }
+    obj = object_new(typename);
+
+    if (object_set_propv(obj, &local_err, vargs) < 0) {
+        goto error;
+    }
+
+    object_property_add_child(parent, id, obj, &local_err);
+    if (local_err) {
+        goto error;
+    }
+
+    if (object_dynamic_cast(obj, TYPE_USER_CREATABLE)) {
+        user_creatable_complete(obj, &local_err);
+        if (local_err) {
+            object_unparent(obj);
+            goto error;
+        }
+    }
+
+    object_unref(OBJECT(obj));
+    return obj;
+
+ error:
+    error_propagate(errp, local_err);
+    object_unref(obj);
+    return NULL;
+}
+
+
+int object_set_props(Object *obj,
+                     Error **errp,
+                     ...)
+{
+    va_list vargs;
+    int ret;
+
+    va_start(vargs, errp);
+    ret = object_set_propv(obj, errp, vargs);
+    va_end(vargs);
+
+    return ret;
+}
+
+
+int object_set_propv(Object *obj,
+                     Error **errp,
+                     va_list vargs)
+{
+    const char *propname;
+    Error *local_err = NULL;
+
+    propname = va_arg(vargs, char *);
+    while (propname != NULL) {
+        const char *value = va_arg(vargs, char *);
+
+        g_assert(value != NULL);
+        object_property_parse(obj, value, propname, &local_err);
+        if (local_err) {
+            error_propagate(errp, local_err);
+            return -1;
+        }
+        propname = va_arg(vargs, char *);
+    }
+
+    return 0;
+}
+
+
 Object *object_dynamic_cast(Object *obj, const char *typename)
 {
     if (obj && object_class_dynamic_cast(object_get_class(obj), typename)) {
@@ -451,7 +606,7 @@
 Object *object_dynamic_cast_assert(Object *obj, const char *typename,
                                    const char *file, int line, const char *func)
 {
-    trace_object_dynamic_cast_assert(obj ? obj->klass->type->name : "(null)",
+    trace_object_dynamic_cast_assert(obj ? obj->clazz->type->name : "(null)",
                                      typename, file, line, func);
 
 #ifdef CONFIG_QOM_CAST_DEBUG
@@ -459,7 +614,7 @@
     Object *inst;
 
     for (i = 0; obj && i < OBJECT_CLASS_CAST_CACHE; i++) {
-        if (obj->klass->object_cast_cache[i] == typename) {
+        if (obj->clazz->object_cast_cache[i] == typename) {
             goto out;
         }
     }
@@ -476,10 +631,10 @@
 
     if (obj && obj == inst) {
         for (i = 1; i < OBJECT_CLASS_CAST_CACHE; i++) {
-            obj->klass->object_cast_cache[i - 1] =
-                    obj->klass->object_cast_cache[i];
+            obj->clazz->object_cast_cache[i - 1] =
+                    obj->clazz->object_cast_cache[i];
         }
-        obj->klass->object_cast_cache[i - 1] = typename;
+        obj->clazz->object_cast_cache[i - 1] = typename;
     }
 
 out:
@@ -510,7 +665,7 @@
         return NULL;
     }
 
-    if (type->class->interfaces &&
+    if (type->clazz->interfaces &&
             type_is_ancestor(target_type, type_interface)) {
         int found = 0;
         GSList *i;
@@ -581,12 +736,12 @@
 
 const char *object_get_typename(Object *obj)
 {
-    return obj->klass->type->name;
+    return obj->clazz->type->name;
 }
 
 ObjectClass *object_get_class(Object *obj)
 {
-    return obj->klass;
+    return obj->clazz;
 }
 
 bool object_class_is_abstract(ObjectClass *klass)
@@ -609,7 +764,7 @@
 
     type_initialize(type);
 
-    return type->class;
+    return type->clazz;
 }
 
 ObjectClass *object_class_get_parent(ObjectClass *class)
@@ -622,7 +777,7 @@
 
     type_initialize(type);
 
-    return type->class;
+    return type->clazz;
 }
 
 typedef struct OCFData
@@ -641,7 +796,7 @@
     ObjectClass *k;
 
     type_initialize(type);
-    k = type->class;
+    k = type->clazz;
 
     if (!data->include_abstract && type->abstract) {
         return;
@@ -666,23 +821,44 @@
     enumerating_types = false;
 }
 
-int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
-                         void *opaque)
+static int do_object_child_foreach(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque, bool recurse)
 {
-    ObjectProperty *prop, *next;
+    GHashTableIter iter;
+    ObjectProperty *prop;
     int ret = 0;
 
-    QTAILQ_FOREACH_SAFE(prop, &obj->properties, node, next) {
+    g_hash_table_iter_init(&iter, obj->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         if (object_property_is_child(prop)) {
-            ret = fn(prop->opaque, opaque);
+            Object *child = prop->opaque;
+
+            ret = fn(child, opaque);
             if (ret != 0) {
                 break;
             }
+            if (recurse) {
+                do_object_child_foreach(child, fn, opaque, true);
+            }
         }
     }
     return ret;
 }
 
+int object_child_foreach(Object *obj, int (*fn)(Object *child, void *opaque),
+                         void *opaque)
+{
+    return do_object_child_foreach(obj, fn, opaque, false);
+}
+
+int object_child_foreach_recursive(Object *obj,
+                                   int (*fn)(Object *child, void *opaque),
+                                   void *opaque)
+{
+    return do_object_child_foreach(obj, fn, opaque, true);
+}
+
 static void object_class_get_list_tramp(ObjectClass *klass, void *opaque)
 {
     GSList **list = opaque;
@@ -705,7 +881,7 @@
     if (!obj) {
         return;
     }
-     atomic_inc(&obj->ref);
+    atomic_inc(&obj->ref);
 }
 
 void object_unref(Object *obj)
@@ -713,7 +889,7 @@
     if (!obj) {
         return;
     }
-    g_assert(obj->ref > 0);
+    g_assert_cmpint(obj->ref, >, 0);
 
     /* parent always holds a reference to its children */
     if (atomic_fetch_dec(&obj->ref) == 1) {
@@ -751,13 +927,11 @@
         return ret;
     }
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
-        if (strcmp(prop->name, name) == 0) {
-            error_setg(errp, "attempt to add duplicate property '%s'"
-                       " to object (type '%s')", name,
-                       object_get_typename(obj));
-            return NULL;
-        }
+    if (object_property_find(obj, name, NULL) != NULL) {
+        error_setg(errp, "attempt to add duplicate property '%s'"
+                   " to object (type '%s')", name,
+                   object_get_typename(obj));
+        return NULL;
     }
 
     prop = g_malloc0(sizeof(*prop));
@@ -770,7 +944,41 @@
     prop->release = release;
     prop->opaque = opaque;
 
-    QTAILQ_INSERT_TAIL(&obj->properties, prop, node);
+    g_hash_table_insert(obj->properties, prop->name, prop);
+    return prop;
+}
+
+ObjectProperty *
+object_class_property_add(ObjectClass *klass,
+                          const char *name,
+                          const char *type,
+                          ObjectPropertyAccessor *get,
+                          ObjectPropertyAccessor *set,
+                          ObjectPropertyRelease *release,
+                          void *opaque,
+                          Error **errp)
+{
+    ObjectProperty *prop;
+
+    if (object_class_property_find(klass, name, NULL) != NULL) {
+        error_setg(errp, "attempt to add duplicate property '%s'"
+                   " to object (type '%s')", name,
+                   object_class_get_name(klass));
+        return NULL;
+    }
+
+    prop = g_malloc0(sizeof(*prop));
+
+    prop->name = g_strdup(name);
+    prop->type = g_strdup(type);
+
+    prop->get = get;
+    prop->set = set;
+    prop->release = release;
+    prop->opaque = opaque;
+
+    g_hash_table_insert(klass->properties, g_strdup(name), prop);
+
     return prop;
 }
 
@@ -778,34 +986,76 @@
                                      Error **errp)
 {
     ObjectProperty *prop;
+    ObjectClass *klass = object_get_class(obj);
 
-    QTAILQ_FOREACH(prop, &obj->properties, node) {
-        if (strcmp(prop->name, name) == 0) {
-            return prop;
-        }
+    prop = object_class_property_find(klass, name, NULL);
+    if (prop) {
+        return prop;
+    }
+
+    prop = g_hash_table_lookup(obj->properties, name);
+    if (prop) {
+        return prop;
     }
 
     error_setg(errp, "Property '.%s' not found", name);
     return NULL;
 }
 
+void object_property_iter_init(ObjectPropertyIterator *iter,
+                               Object *obj)
+{
+    g_hash_table_iter_init(&iter->iter, obj->properties);
+    iter->nextclass = object_get_class(obj);
+}
+
+ObjectProperty *object_property_iter_next(ObjectPropertyIterator *iter)
+{
+    gpointer key, val;
+    while (!g_hash_table_iter_next(&iter->iter, &key, &val)) {
+        if (!iter->nextclass) {
+            return NULL;
+        }
+        g_hash_table_iter_init(&iter->iter, iter->nextclass->properties);
+        iter->nextclass = object_class_get_parent(iter->nextclass);
+    }
+    return val;
+}
+
+ObjectProperty *object_class_property_find(ObjectClass *klass, const char *name,
+                                           Error **errp)
+{
+    ObjectProperty *prop;
+    ObjectClass *parent_klass;
+
+    parent_klass = object_class_get_parent(klass);
+    if (parent_klass) {
+        prop = object_class_property_find(parent_klass, name, NULL);
+        if (prop) {
+            return prop;
+        }
+    }
+
+    prop = g_hash_table_lookup(klass->properties, name);
+    if (!prop) {
+        error_setg(errp, "Property '.%s' not found", name);
+    }
+    return prop;
+}
+
 void object_property_del(Object *obj, const char *name, Error **errp)
 {
-    ObjectProperty *prop = object_property_find(obj, name, errp);
-    if (prop == NULL) {
+    ObjectProperty *prop = g_hash_table_lookup(obj->properties, name);
+
+    if (!prop) {
+        error_setg(errp, "Property '.%s' not found", name);
         return;
     }
 
     if (prop->release) {
         prop->release(obj, name, prop->opaque);
     }
-
-    QTAILQ_REMOVE(&obj->properties, prop, node);
-
-    g_free(prop->name);
-    g_free(prop->type);
-    g_free(prop->description);
-    g_free(prop);
+    g_hash_table_remove(obj->properties, name);
 }
 
 void object_property_get(Object *obj, Visitor *v, const char *name,
@@ -817,9 +1067,9 @@
     }
 
     if (!prop->get) {
-        error_set(errp, QERR_PERMISSION_DENIED);
+        error_setg(errp, QERR_PERMISSION_DENIED);
     } else {
-        prop->get(obj, v, prop->opaque, name, errp);
+        prop->get(obj, v, name, prop->opaque, errp);
     }
 }
 
@@ -832,9 +1082,9 @@
     }
 
     if (!prop->set) {
-        error_set(errp, QERR_PERMISSION_DENIED);
+        error_setg(errp, QERR_PERMISSION_DENIED);
     } else {
-        prop->set(obj, v, prop->opaque, name, errp);
+        prop->set(obj, v, name, prop->opaque, errp);
     }
 }
 
@@ -859,7 +1109,7 @@
     }
     qstring = qobject_to_qstring(ret);
     if (!qstring) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name, "string");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name, "string");
         retval = NULL;
     } else {
         retval = g_strdup(qstring_get_str(qstring));
@@ -890,7 +1140,8 @@
     if (str && *str) {
         target = object_resolve_path(str, NULL);
         if (!target) {
-            error_set(errp, QERR_DEVICE_NOT_FOUND, str);
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "Device '%s' not found", str);
         }
     }
 
@@ -901,7 +1152,7 @@
 void object_property_set_bool(Object *obj, bool value,
                               const char *name, Error **errp)
 {
-    QBool *qbool = qbool_from_int(value);
+    QBool *qbool = qbool_from_bool(value);
     object_property_set_qobject(obj, QOBJECT(qbool), name, errp);
 
     QDECREF(qbool);
@@ -919,10 +1170,10 @@
     }
     qbool = qobject_to_qbool(ret);
     if (!qbool) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name, "boolean");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name, "boolean");
         retval = false;
     } else {
-        retval = qbool_get_int(qbool);
+        retval = qbool_get_bool(qbool);
     }
 
     QDECREF(qbool);
@@ -950,7 +1201,7 @@
     }
     qint = qobject_to_qint(ret);
     if (!qint) {
-        error_set(errp, QERR_INVALID_PARAMETER_TYPE, name, "int");
+        error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name, "int");
         retval = -1;
     } else {
         retval = qint_get_int(qint);
@@ -960,24 +1211,49 @@
     return retval;
 }
 
+typedef struct EnumProperty {
+    const char * const *strings;
+    int (*get)(Object *, Error **);
+    void (*set)(Object *, int, Error **);
+} EnumProperty;
+
 int object_property_get_enum(Object *obj, const char *name,
-                             const char *strings[], Error **errp)
+                             const char *typename, Error **errp)
 {
-    StringOutputVisitor *sov;
-    StringInputVisitor *siv;
+    Error *err = NULL;
+    Visitor *v;
     char *str;
     int ret;
+    ObjectProperty *prop = object_property_find(obj, name, errp);
+    EnumProperty *enumprop;
 
-    sov = string_output_visitor_new(false);
-    object_property_get(obj, string_output_get_visitor(sov), name, errp);
-    str = string_output_get_string(sov);
-    siv = string_input_visitor_new(str);
-    string_output_visitor_cleanup(sov);
-    visit_type_enum(string_input_get_visitor(siv),
-                    &ret, strings, NULL, name, errp);
+    if (prop == NULL) {
+        return 0;
+    }
+
+    if (!g_str_equal(prop->type, typename)) {
+        error_setg(errp, "Property %s on %s is not '%s' enum type",
+                   name, object_class_get_name(
+                       object_get_class(obj)), typename);
+        return 0;
+    }
+
+    enumprop = prop->opaque;
+
+    v = string_output_visitor_new(false, &str);
+    object_property_get(obj, v, name, &err);
+    if (err) {
+        error_propagate(errp, err);
+        visit_free(v);
+        return 0;
+    }
+    visit_complete(v, &str);
+    visit_free(v);
+    v = string_input_visitor_new(str);
+    visit_type_enum(v, name, &ret, enumprop->strings, errp);
 
     g_free(str);
-    string_input_visitor_cleanup(siv);
+    visit_free(v);
 
     return ret;
 }
@@ -985,51 +1261,52 @@
 void object_property_get_uint16List(Object *obj, const char *name,
                                     uint16List **list, Error **errp)
 {
-    StringOutputVisitor *ov;
-    StringInputVisitor *iv;
+    Error *err = NULL;
+    Visitor *v;
     char *str;
 
-    ov = string_output_visitor_new(false);
-    object_property_get(obj, string_output_get_visitor(ov),
-                        name, errp);
-    str = string_output_get_string(ov);
-    iv = string_input_visitor_new(str);
-    visit_type_uint16List(string_input_get_visitor(iv),
-                          list, NULL, errp);
+    v = string_output_visitor_new(false, &str);
+    object_property_get(obj, v, name, &err);
+    if (err) {
+        error_propagate(errp, err);
+        goto out;
+    }
+    visit_complete(v, &str);
+    visit_free(v);
+    v = string_input_visitor_new(str);
+    visit_type_uint16List(v, NULL, list, errp);
 
     g_free(str);
-    string_output_visitor_cleanup(ov);
-    string_input_visitor_cleanup(iv);
+out:
+    visit_free(v);
 }
 
 void object_property_parse(Object *obj, const char *string,
                            const char *name, Error **errp)
 {
-    StringInputVisitor *mi;
-    mi = string_input_visitor_new(string);
-    object_property_set(obj, string_input_get_visitor(mi), name, errp);
-
-    string_input_visitor_cleanup(mi);
+    Visitor *v = string_input_visitor_new(string);
+    object_property_set(obj, v, name, errp);
+    visit_free(v);
 }
 
 char *object_property_print(Object *obj, const char *name, bool human,
                             Error **errp)
 {
-    StringOutputVisitor *mo;
+    Visitor *v;
     char *string = NULL;
     Error *local_err = NULL;
 
-    mo = string_output_visitor_new(human);
-    object_property_get(obj, string_output_get_visitor(mo), name, &local_err);
+    v = string_output_visitor_new(human, &string);
+    object_property_get(obj, v, name, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         goto out;
     }
 
-    string = string_output_get_string(mo);
+    visit_complete(v, &string);
 
 out:
-    string_output_visitor_cleanup(mo);
+    visit_free(v);
     return string;
 }
 
@@ -1054,14 +1331,20 @@
     return root;
 }
 
-static void object_get_child_property(Object *obj, Visitor *v, void *opaque,
-                                      const char *name, Error **errp)
+Object *object_get_objects_root(void)
+{
+    return container_get(object_get_root(), "/objects");
+}
+
+static void object_get_child_property(Object *obj, Visitor *v,
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     Object *child = opaque;
     gchar *path;
 
     path = object_get_canonical_path(child);
-    visit_type_str(v, &path, name, errp);
+    visit_type_str(v, name, &path, errp);
     g_free(path);
 }
 
@@ -1075,8 +1358,8 @@
 {
     Object *child = opaque;
 
-    if (child->klass->unparent) {
-        (child->klass->unparent)(child);
+    if (child->clazz->unparent) {
+        (child->clazz->unparent)(child);
     }
     child->parent = NULL;
     object_unref(child);
@@ -1123,8 +1406,9 @@
     ObjectPropertyLinkFlags flags;
 } LinkProperty;
 
-static void object_get_link_property(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void object_get_link_property(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     LinkProperty *lprop = opaque;
     Object **child = lprop->child;
@@ -1132,11 +1416,11 @@
 
     if (*child) {
         path = object_get_canonical_path(*child);
-        visit_type_str(v, &path, name, errp);
+        visit_type_str(v, name, &path, errp);
         g_free(path);
     } else {
         path = (gchar *)"";
-        visit_type_str(v, &path, name, errp);
+        visit_type_str(v, name, &path, errp);
     }
 }
 
@@ -1163,14 +1447,15 @@
     target = object_resolve_path_type(path, target_type, &ambiguous);
 
     if (ambiguous) {
-        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
-                  "Path '%s' does not uniquely identify an object", path);
+        error_setg(errp, "Path '%s' does not uniquely identify an object",
+                   path);
     } else if (!target) {
         target = object_resolve_path(path, &ambiguous);
         if (target || ambiguous) {
-            error_set(errp, QERR_INVALID_PARAMETER_TYPE, name, target_type);
+            error_setg(errp, QERR_INVALID_PARAMETER_TYPE, name, target_type);
         } else {
-            error_set(errp, QERR_DEVICE_NOT_FOUND, path);
+            error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                      "Device '%s' not found", path);
         }
         target = NULL;
     }
@@ -1179,8 +1464,9 @@
     return target;
 }
 
-static void object_set_link_property(Object *obj, Visitor *v, void *opaque,
-                                     const char *name, Error **errp)
+static void object_set_link_property(Object *obj, Visitor *v,
+                                     const char *name, void *opaque,
+                                     Error **errp)
 {
     Error *local_err = NULL;
     LinkProperty *prop = opaque;
@@ -1189,7 +1475,7 @@
     Object *new_target = NULL;
     char *path = NULL;
 
-    visit_type_str(v, &path, name, &local_err);
+    visit_type_str(v, name, &path, &local_err);
 
     if (!local_err && strcmp(path, "") != 0) {
         new_target = object_resolve_link(obj, name, path, &local_err);
@@ -1266,14 +1552,32 @@
     g_free(full_type);
 }
 
+void object_property_add_const_link(Object *obj, const char *name,
+                                    Object *target, Error **errp)
+{
+    char *link_type;
+    ObjectProperty *op;
+
+    link_type = g_strdup_printf("link<%s>", object_get_typename(target));
+    op = object_property_add(obj, name, link_type,
+                             object_get_child_property, NULL,
+                             NULL, target, errp);
+    if (op != NULL) {
+        op->resolve = object_resolve_child_property;
+    }
+    g_free(link_type);
+}
+
 gchar *object_get_canonical_path_component(Object *obj)
 {
     ObjectProperty *prop = NULL;
+    GHashTableIter iter;
 
     g_assert(obj);
     g_assert(obj->parent != NULL);
 
-    QTAILQ_FOREACH(prop, &obj->parent->properties, node) {
+    g_hash_table_iter_init(&iter, obj->parent->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         if (!object_property_is_child(prop)) {
             continue;
         }
@@ -1357,11 +1661,13 @@
                                               bool *ambiguous)
 {
     Object *obj;
+    GHashTableIter iter;
     ObjectProperty *prop;
 
     obj = object_resolve_abs_path(parent, parts, typename, 0);
 
-    QTAILQ_FOREACH(prop, &parent->properties, node) {
+    g_hash_table_iter_init(&iter, parent->properties);
+    while (g_hash_table_iter_next(&iter, NULL, (gpointer *)&prop)) {
         Object *found;
 
         if (!object_property_is_child(prop)) {
@@ -1423,27 +1729,31 @@
     void (*set)(Object *, const char *, Error **);
 } StringProperty;
 
-static void property_get_str(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+static void property_get_str(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     StringProperty *prop = opaque;
     char *value;
+    Error *err = NULL;
 
-    value = prop->get(obj, errp);
-    if (value) {
-        visit_type_str(v, &value, name, errp);
-        g_free(value);
+    value = prop->get(obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
     }
+
+    visit_type_str(v, name, &value, errp);
+    g_free(value);
 }
 
-static void property_set_str(Object *obj, Visitor *v, void *opaque,
-                             const char *name, Error **errp)
+static void property_set_str(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
 {
     StringProperty *prop = opaque;
     char *value;
     Error *local_err = NULL;
 
-    visit_type_str(v, &value, name, &local_err);
+    visit_type_str(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1482,30 +1792,59 @@
     }
 }
 
+void object_class_property_add_str(ObjectClass *klass, const char *name,
+                                   char *(*get)(Object *, Error **),
+                                   void (*set)(Object *, const char *,
+                                               Error **),
+                                   Error **errp)
+{
+    Error *local_err = NULL;
+    StringProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+    prop->set = set;
+
+    object_class_property_add(klass, name, "string",
+                              get ? property_get_str : NULL,
+                              set ? property_set_str : NULL,
+                              property_release_str,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
 typedef struct BoolProperty
 {
     bool (*get)(Object *, Error **);
     void (*set)(Object *, bool, Error **);
 } BoolProperty;
 
-static void property_get_bool(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void property_get_bool(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     BoolProperty *prop = opaque;
     bool value;
+    Error *err = NULL;
 
-    value = prop->get(obj, errp);
-    visit_type_bool(v, &value, name, errp);
+    value = prop->get(obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    visit_type_bool(v, name, &value, errp);
 }
 
-static void property_set_bool(Object *obj, Visitor *v, void *opaque,
-                              const char *name, Error **errp)
+static void property_set_bool(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
 {
     BoolProperty *prop = opaque;
     bool value;
     Error *local_err = NULL;
 
-    visit_type_bool(v, &value, name, &local_err);
+    visit_type_bool(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
@@ -1543,41 +1882,244 @@
     }
 }
 
+void object_class_property_add_bool(ObjectClass *klass, const char *name,
+                                    bool (*get)(Object *, Error **),
+                                    void (*set)(Object *, bool, Error **),
+                                    Error **errp)
+{
+    Error *local_err = NULL;
+    BoolProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+    prop->set = set;
+
+    object_class_property_add(klass, name, "bool",
+                              get ? property_get_bool : NULL,
+                              set ? property_set_bool : NULL,
+                              property_release_bool,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
+static void property_get_enum(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
+{
+    EnumProperty *prop = opaque;
+    int value;
+    Error *err = NULL;
+
+    value = prop->get(obj, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+
+    visit_type_enum(v, name, &value, prop->strings, errp);
+}
+
+static void property_set_enum(Object *obj, Visitor *v, const char *name,
+                              void *opaque, Error **errp)
+{
+    EnumProperty *prop = opaque;
+    int value;
+    Error *err = NULL;
+
+    visit_type_enum(v, name, &value, prop->strings, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    prop->set(obj, value, errp);
+}
+
+static void property_release_enum(Object *obj, const char *name,
+                                  void *opaque)
+{
+    EnumProperty *prop = opaque;
+    g_free(prop);
+}
+
+void object_property_add_enum(Object *obj, const char *name,
+                              const char *typename,
+                              const char * const *strings,
+                              int (*get)(Object *, Error **),
+                              void (*set)(Object *, int, Error **),
+                              Error **errp)
+{
+    Error *local_err = NULL;
+    EnumProperty *prop = g_malloc(sizeof(*prop));
+
+    prop->strings = strings;
+    prop->get = get;
+    prop->set = set;
+
+    object_property_add(obj, name, typename,
+                        get ? property_get_enum : NULL,
+                        set ? property_set_enum : NULL,
+                        property_release_enum,
+                        prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
+void object_class_property_add_enum(ObjectClass *klass, const char *name,
+                                    const char *typename,
+                                    const char * const *strings,
+                                    int (*get)(Object *, Error **),
+                                    void (*set)(Object *, int, Error **),
+                                    Error **errp)
+{
+    Error *local_err = NULL;
+    EnumProperty *prop = g_malloc(sizeof(*prop));
+
+    prop->strings = strings;
+    prop->get = get;
+    prop->set = set;
+
+    object_class_property_add(klass, name, typename,
+                              get ? property_get_enum : NULL,
+                              set ? property_set_enum : NULL,
+                              property_release_enum,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
+typedef struct TMProperty {
+    void (*get)(Object *, struct tm *, Error **);
+} TMProperty;
+
+static void property_get_tm(Object *obj, Visitor *v, const char *name,
+                            void *opaque, Error **errp)
+{
+    TMProperty *prop = opaque;
+    Error *err = NULL;
+    struct tm value;
+
+    prop->get(obj, &value, &err);
+    if (err) {
+        goto out;
+    }
+
+    visit_start_struct(v, name, NULL, 0, &err);
+    if (err) {
+        goto out;
+    }
+    visit_type_int32(v, "tm_year", &value.tm_year, &err);
+    if (err) {
+        goto out_end;
+    }
+    visit_type_int32(v, "tm_mon", &value.tm_mon, &err);
+    if (err) {
+        goto out_end;
+    }
+    visit_type_int32(v, "tm_mday", &value.tm_mday, &err);
+    if (err) {
+        goto out_end;
+    }
+    visit_type_int32(v, "tm_hour", &value.tm_hour, &err);
+    if (err) {
+        goto out_end;
+    }
+    visit_type_int32(v, "tm_min", &value.tm_min, &err);
+    if (err) {
+        goto out_end;
+    }
+    visit_type_int32(v, "tm_sec", &value.tm_sec, &err);
+    if (err) {
+        goto out_end;
+    }
+    visit_check_struct(v, &err);
+out_end:
+    visit_end_struct(v, NULL);
+out:
+    error_propagate(errp, err);
+
+}
+
+static void property_release_tm(Object *obj, const char *name,
+                                void *opaque)
+{
+    TMProperty *prop = opaque;
+    g_free(prop);
+}
+
+void object_property_add_tm(Object *obj, const char *name,
+                            void (*get)(Object *, struct tm *, Error **),
+                            Error **errp)
+{
+    Error *local_err = NULL;
+    TMProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+
+    object_property_add(obj, name, "struct tm",
+                        get ? property_get_tm : NULL, NULL,
+                        property_release_tm,
+                        prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
+void object_class_property_add_tm(ObjectClass *klass, const char *name,
+                                  void (*get)(Object *, struct tm *, Error **),
+                                  Error **errp)
+{
+    Error *local_err = NULL;
+    TMProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+
+    object_class_property_add(klass, name, "struct tm",
+                              get ? property_get_tm : NULL, NULL,
+                              property_release_tm,
+                              prop, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        g_free(prop);
+    }
+}
+
 static char *qdev_get_type(Object *obj, Error **errp)
 {
     return g_strdup(object_get_typename(obj));
 }
 
-static void property_get_uint8_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint8_ptr(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     uint8_t value = *(uint8_t *)opaque;
-    visit_type_uint8(v, &value, name, errp);
+    visit_type_uint8(v, name, &value, errp);
 }
 
-static void property_get_uint16_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint16_ptr(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     uint16_t value = *(uint16_t *)opaque;
-    visit_type_uint16(v, &value, name, errp);
+    visit_type_uint16(v, name, &value, errp);
 }
 
-static void property_get_uint32_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint32_ptr(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     uint32_t value = *(uint32_t *)opaque;
-    visit_type_uint32(v, &value, name, errp);
+    visit_type_uint32(v, name, &value, errp);
 }
 
-static void property_get_uint64_ptr(Object *obj, Visitor *v,
-                                   void *opaque, const char *name,
-                                   Error **errp)
+static void property_get_uint64_ptr(Object *obj, Visitor *v, const char *name,
+                                    void *opaque, Error **errp)
 {
     uint64_t value = *(uint64_t *)opaque;
-    visit_type_uint64(v, &value, name, errp);
+    visit_type_uint64(v, name, &value, errp);
 }
 
 void object_property_add_uint8_ptr(Object *obj, const char *name,
@@ -1587,6 +2129,13 @@
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint8_ptr(ObjectClass *klass, const char *name,
+                                         const uint8_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint8", property_get_uint8_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 void object_property_add_uint16_ptr(Object *obj, const char *name,
                                     const uint16_t *v, Error **errp)
 {
@@ -1594,6 +2143,13 @@
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint16_ptr(ObjectClass *klass, const char *name,
+                                          const uint16_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint16", property_get_uint16_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 void object_property_add_uint32_ptr(Object *obj, const char *name,
                                     const uint32_t *v, Error **errp)
 {
@@ -1601,6 +2157,13 @@
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint32_ptr(ObjectClass *klass, const char *name,
+                                          const uint32_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint32", property_get_uint32_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 void object_property_add_uint64_ptr(Object *obj, const char *name,
                                     const uint64_t *v, Error **errp)
 {
@@ -1608,21 +2171,28 @@
                         NULL, NULL, (void *)v, errp);
 }
 
+void object_class_property_add_uint64_ptr(ObjectClass *klass, const char *name,
+                                          const uint64_t *v, Error **errp)
+{
+    object_class_property_add(klass, name, "uint64", property_get_uint64_ptr,
+                              NULL, NULL, (void *)v, errp);
+}
+
 typedef struct {
     Object *target_obj;
-    const char *target_name;
+    char *target_name;
 } AliasProperty;
 
-static void property_get_alias(Object *obj, struct Visitor *v, void *opaque,
-                               const char *name, Error **errp)
+static void property_get_alias(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     AliasProperty *prop = opaque;
 
     object_property_get(prop->target_obj, v, prop->target_name, errp);
 }
 
-static void property_set_alias(Object *obj, struct Visitor *v, void *opaque,
-                               const char *name, Error **errp)
+static void property_set_alias(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     AliasProperty *prop = opaque;
 
@@ -1641,6 +2211,7 @@
 {
     AliasProperty *prop = opaque;
 
+    g_free(prop->target_name);
     g_free(prop);
 }
 
@@ -1668,7 +2239,7 @@
 
     prop = g_malloc(sizeof(*prop));
     prop->target_obj = target_obj;
-    prop->target_name = target_name;
+    prop->target_name = g_strdup(target_name);
 
     op = object_property_add(obj, name, prop_type,
                              property_get_alias,
@@ -1682,7 +2253,7 @@
     }
     op->resolve = property_resolve_alias;
 
-    object_property_set_description(obj, name,
+    object_property_set_description(obj, op->name,
                                     target_prop->description,
                                     &error_abort);
 
@@ -1704,6 +2275,23 @@
     op->description = g_strdup(description);
 }
 
+void object_class_property_set_description(ObjectClass *klass,
+                                           const char *name,
+                                           const char *description,
+                                           Error **errp)
+{
+    ObjectProperty *op;
+
+    op = g_hash_table_lookup(klass->properties, name);
+    if (!op) {
+        error_setg(errp, "Property '.%s' not found", name);
+        return;
+    }
+
+    g_free(op->description);
+    op->description = g_strdup(description);
+}
+
 static void object_instance_init(Object *obj)
 {
     object_property_add_str(obj, "type", qdev_get_type, NULL, NULL);
diff --git a/qom/object_interfaces.c b/qom/object_interfaces.c
index 6360818..bf59846 100644
--- a/qom/object_interfaces.c
+++ b/qom/object_interfaces.c
@@ -1,5 +1,10 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qom/object_interfaces.h"
 #include "qemu/module.h"
+#include "qapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/opts-visitor.h"
 
 void user_creatable_complete(Object *obj, Error **errp)
 {
@@ -18,6 +23,195 @@
     }
 }
 
+bool user_creatable_can_be_deleted(UserCreatable *uc, Error **errp)
+{
+
+    UserCreatableClass *ucc = USER_CREATABLE_GET_CLASS(uc);
+
+    if (ucc->can_be_deleted) {
+        return ucc->can_be_deleted(uc, errp);
+    } else {
+        return true;
+    }
+}
+
+
+Object *user_creatable_add(const QDict *qdict,
+                           Visitor *v, Error **errp)
+{
+    char *type = NULL;
+    char *id = NULL;
+    Object *obj = NULL;
+    Error *local_err = NULL;
+    QDict *pdict;
+
+    pdict = qdict_clone_shallow(qdict);
+
+    visit_start_struct(v, NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    qdict_del(pdict, "qom-type");
+    visit_type_str(v, "qom-type", &type, &local_err);
+    if (local_err) {
+        goto out_visit;
+    }
+
+    qdict_del(pdict, "id");
+    visit_type_str(v, "id", &id, &local_err);
+    if (local_err) {
+        goto out_visit;
+    }
+    visit_check_struct(v, &local_err);
+    if (local_err) {
+        goto out_visit;
+    }
+
+    obj = user_creatable_add_type(type, id, pdict, v, &local_err);
+
+out_visit:
+    visit_end_struct(v, NULL);
+
+out:
+    QDECREF(pdict);
+    g_free(id);
+    g_free(type);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        object_unref(obj);
+        return NULL;
+    }
+    return obj;
+}
+
+
+Object *user_creatable_add_type(const char *type, const char *id,
+                                const QDict *qdict,
+                                Visitor *v, Error **errp)
+{
+    Object *obj;
+    ObjectClass *klass;
+    const QDictEntry *e;
+    Error *local_err = NULL;
+
+    klass = object_class_by_name(type);
+    if (!klass) {
+        error_setg(errp, "invalid object type: %s", type);
+        return NULL;
+    }
+
+    if (!object_class_dynamic_cast(klass, TYPE_USER_CREATABLE)) {
+        error_setg(errp, "object type '%s' isn't supported by object-add",
+                   type);
+        return NULL;
+    }
+
+    if (object_class_is_abstract(klass)) {
+        error_setg(errp, "object type '%s' is abstract", type);
+        return NULL;
+    }
+
+    assert(qdict);
+    obj = object_new(type);
+    visit_start_struct(v, NULL, NULL, 0, &local_err);
+    if (local_err) {
+        goto out;
+    }
+    for (e = qdict_first(qdict); e; e = qdict_next(qdict, e)) {
+        object_property_set(obj, v, e->key, &local_err);
+        if (local_err) {
+            break;
+        }
+    }
+    if (!local_err) {
+        visit_check_struct(v, &local_err);
+    }
+    visit_end_struct(v, NULL);
+    if (local_err) {
+        goto out;
+    }
+
+    object_property_add_child(object_get_objects_root(),
+                              id, obj, &local_err);
+    if (local_err) {
+        goto out;
+    }
+
+    user_creatable_complete(obj, &local_err);
+    if (local_err) {
+        object_property_del(object_get_objects_root(),
+                            id, &error_abort);
+        goto out;
+    }
+out:
+    if (local_err) {
+        error_propagate(errp, local_err);
+        object_unref(obj);
+        return NULL;
+    }
+    return obj;
+}
+
+
+Object *user_creatable_add_opts(QemuOpts *opts, Error **errp)
+{
+    Visitor *v;
+    QDict *pdict;
+    Object *obj = NULL;
+
+    v = opts_visitor_new(opts);
+    pdict = qemu_opts_to_qdict(opts, NULL);
+
+    obj = user_creatable_add(pdict, v, errp);
+    visit_free(v);
+    QDECREF(pdict);
+    return obj;
+}
+
+
+int user_creatable_add_opts_foreach(void *opaque, QemuOpts *opts, Error **errp)
+{
+    bool (*type_predicate)(const char *) = opaque;
+    Object *obj = NULL;
+    Error *err = NULL;
+    const char *type;
+
+    type = qemu_opt_get(opts, "qom-type");
+    if (type && type_predicate &&
+        !type_predicate(type)) {
+        return 0;
+    }
+
+    obj = user_creatable_add_opts(opts, &err);
+    if (!obj) {
+        error_report_err(err);
+        return -1;
+    }
+    object_unref(obj);
+    return 0;
+}
+
+
+void user_creatable_del(const char *id, Error **errp)
+{
+    Object *container;
+    Object *obj;
+
+    container = object_get_objects_root();
+    obj = object_resolve_path_component(container, id);
+    if (!obj) {
+        error_setg(errp, "object '%s' not found", id);
+        return;
+    }
+
+    if (!user_creatable_can_be_deleted(USER_CREATABLE(obj), errp)) {
+        error_setg(errp, "object '%s' is in use, can not be deleted", id);
+        return;
+    }
+    object_unparent(obj);
+}
+
 static void register_types(void)
 {
     static const TypeInfo uc_interface_info = {
diff --git a/qom/qom-qobject.c b/qom/qom-qobject.c
index 6384b8e..c225abc 100644
--- a/qom/qom-qobject.c
+++ b/qom/qom-qobject.c
@@ -9,6 +9,8 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qom/object.h"
 #include "qom/qom-qobject.h"
@@ -19,11 +21,11 @@
 void object_property_set_qobject(Object *obj, QObject *value,
                                  const char *name, Error **errp)
 {
-    QmpInputVisitor *mi;
-    mi = qmp_input_visitor_new(value);
-    object_property_set(obj, qmp_input_get_visitor(mi), name, errp);
-
-    qmp_input_visitor_cleanup(mi);
+    Visitor *v;
+    /* TODO: Should we reject, rather than ignore, excess input? */
+    v = qmp_input_visitor_new(value, false);
+    object_property_set(obj, v, name, errp);
+    visit_free(v);
 }
 
 QObject *object_property_get_qobject(Object *obj, const char *name,
@@ -31,14 +33,14 @@
 {
     QObject *ret = NULL;
     Error *local_err = NULL;
-    QmpOutputVisitor *mo;
+    Visitor *v;
 
-    mo = qmp_output_visitor_new();
-    object_property_get(obj, qmp_output_get_visitor(mo), name, &local_err);
+    v = qmp_output_visitor_new(&ret);
+    object_property_get(obj, v, name, &local_err);
     if (!local_err) {
-        ret = qmp_output_get_qobject(mo);
+        visit_complete(v, &ret);
     }
     error_propagate(errp, local_err);
-    qmp_output_visitor_cleanup(mo);
+    visit_free(v);
     return ret;
 }
diff --git a/qom/trace-events b/qom/trace-events
new file mode 100644
index 0000000..97db357
--- /dev/null
+++ b/qom/trace-events
@@ -0,0 +1,5 @@
+# See docs/tracing.txt for syntax documentation.
+
+# qom/object.c
+object_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
+object_class_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
diff --git a/qtest.c b/qtest.c
index 2bca04e..da4826c 100644
--- a/qtest.c
+++ b/qtest.c
@@ -11,6 +11,10 @@
  *
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "sysemu/qtest.h"
 #include "hw/qdev.h"
 #include "sysemu/char.h"
@@ -119,12 +123,24 @@
  *  > write ADDR SIZE DATA
  *  < OK
  *
+ *  > b64read ADDR SIZE
+ *  < OK B64_DATA
+ *
+ *  > b64write ADDR SIZE B64_DATA
+ *  < OK
+ *
+ *  > memset ADDR SIZE VALUE
+ *  < OK
+ *
  * ADDR, SIZE, VALUE are all integers parsed with strtoul() with a base of 0.
  *
  * DATA is an arbitrarily long hex number prefixed with '0x'.  If it's smaller
  * than the expected size, the value will be zero filled at the end of the data
  * sequence.
  *
+ * B64_DATA is an arbitrarily long base64 encoded string.
+ * If the sizes do not match, the data will be truncated.
+ *
  * IRQ management:
  *
  *  > irq_intercept_in QOM-PATH
@@ -182,23 +198,46 @@
             (long) tv.tv_sec, (long) tv.tv_usec);
 }
 
-static void GCC_FMT_ATTR(2, 3) qtest_send(CharDriverState *chr,
-                                          const char *fmt, ...)
+static void GCC_FMT_ATTR(1, 2) qtest_log_send(const char *fmt, ...)
 {
     va_list ap;
-    char buffer[1024];
-    size_t len;
+
+    if (!qtest_log_fp || !qtest_opened) {
+        return;
+    }
+
+    qtest_send_prefix(NULL);
 
     va_start(ap, fmt);
-    len = vsnprintf(buffer, sizeof(buffer), fmt, ap);
+    vfprintf(qtest_log_fp, fmt, ap);
     va_end(ap);
+}
 
-    qemu_chr_fe_write_all(chr, (uint8_t *)buffer, len);
+static void do_qtest_send(CharDriverState *chr, const char *str, size_t len)
+{
+    qemu_chr_fe_write_all(chr, (uint8_t *)str, len);
     if (qtest_log_fp && qtest_opened) {
-        fprintf(qtest_log_fp, "%s", buffer);
+        fprintf(qtest_log_fp, "%s", str);
     }
 }
 
+static void qtest_send(CharDriverState *chr, const char *str)
+{
+    do_qtest_send(chr, str, strlen(str));
+}
+
+static void GCC_FMT_ATTR(2, 3) qtest_sendf(CharDriverState *chr,
+                                           const char *fmt, ...)
+{
+    va_list ap;
+    gchar *buffer;
+
+    va_start(ap, fmt);
+    buffer = g_strdup_vprintf(fmt, ap);
+    qtest_send(chr, buffer);
+    va_end(ap);
+}
+
 static void qtest_irq_handler(void *opaque, int n, int level)
 {
     qemu_irq old_irq = *(qemu_irq *)opaque;
@@ -208,8 +247,8 @@
         CharDriverState *chr = qtest_chr;
         irq_levels[n] = level;
         qtest_send_prefix(chr);
-        qtest_send(chr, "IRQ %s %d\n",
-                   level ? "raise" : "lower", n);
+        qtest_sendf(chr, "IRQ %s %d\n",
+                    level ? "raise" : "lower", n);
     }
 }
 
@@ -318,7 +357,7 @@
             value = cpu_inl(addr);
         }
         qtest_send_prefix(chr);
-        qtest_send(chr, "OK 0x%04x\n", value);
+        qtest_sendf(chr, "OK 0x%04x\n", value);
     } else if (strcmp(words[0], "writeb") == 0 ||
                strcmp(words[0], "writew") == 0 ||
                strcmp(words[0], "writel") == 0 ||
@@ -375,10 +414,11 @@
             tswap64s(&value);
         }
         qtest_send_prefix(chr);
-        qtest_send(chr, "OK 0x%016" PRIx64 "\n", value);
+        qtest_sendf(chr, "OK 0x%016" PRIx64 "\n", value);
     } else if (strcmp(words[0], "read") == 0) {
         uint64_t addr, len, i;
         uint8_t *data;
+        char *enc;
 
         g_assert(words[1] && words[2]);
         addr = strtoull(words[1], NULL, 0);
@@ -387,14 +427,33 @@
         data = g_malloc(len);
         cpu_physical_memory_read(addr, data, len);
 
-        qtest_send_prefix(chr);
-        qtest_send(chr, "OK 0x");
+        enc = g_malloc(2 * len + 1);
         for (i = 0; i < len; i++) {
-            qtest_send(chr, "%02x", data[i]);
+            sprintf(&enc[i * 2], "%02x", data[i]);
         }
-        qtest_send(chr, "\n");
+
+        qtest_send_prefix(chr);
+        qtest_sendf(chr, "OK 0x%s\n", enc);
 
         g_free(data);
+        g_free(enc);
+    } else if (strcmp(words[0], "b64read") == 0) {
+        uint64_t addr, len;
+        uint8_t *data;
+        gchar *b64_data;
+
+        g_assert(words[1] && words[2]);
+        addr = strtoull(words[1], NULL, 0);
+        len = strtoull(words[2], NULL, 0);
+
+        data = g_malloc(len);
+        cpu_physical_memory_read(addr, data, len);
+        b64_data = g_base64_encode(data, len);
+        qtest_send_prefix(chr);
+        qtest_sendf(chr, "OK %s\n", b64_data);
+
+        g_free(data);
+        g_free(b64_data);
     } else if (strcmp(words[0], "write") == 0) {
         uint64_t addr, len, i;
         uint8_t *data;
@@ -424,6 +483,51 @@
 
         qtest_send_prefix(chr);
         qtest_send(chr, "OK\n");
+    } else if (strcmp(words[0], "memset") == 0) {
+        uint64_t addr, len;
+        uint8_t *data;
+        uint8_t pattern;
+
+        g_assert(words[1] && words[2] && words[3]);
+        addr = strtoull(words[1], NULL, 0);
+        len = strtoull(words[2], NULL, 0);
+        pattern = strtoull(words[3], NULL, 0);
+
+        data = g_malloc(len);
+        memset(data, pattern, len);
+        cpu_physical_memory_write(addr, data, len);
+        g_free(data);
+
+        qtest_send_prefix(chr);
+        qtest_send(chr, "OK\n");
+    }  else if (strcmp(words[0], "b64write") == 0) {
+        uint64_t addr, len;
+        uint8_t *data;
+        size_t data_len;
+        gsize out_len;
+
+        g_assert(words[1] && words[2] && words[3]);
+        addr = strtoull(words[1], NULL, 0);
+        len = strtoull(words[2], NULL, 0);
+
+        data_len = strlen(words[3]);
+        if (data_len < 3) {
+            qtest_send(chr, "ERR invalid argument size\n");
+            return;
+        }
+
+        data = g_base64_decode_inplace(words[3], &out_len);
+        if (out_len != len) {
+            qtest_log_send("b64write: data length mismatch (told %"PRIu64", "
+                           "found %zu)\n",
+                           len, out_len);
+            out_len = MIN(out_len, len);
+        }
+
+        cpu_physical_memory_write(addr, data, out_len);
+
+        qtest_send_prefix(chr);
+        qtest_send(chr, "OK\n");
     } else if (qtest_enabled() && strcmp(words[0], "clock_step") == 0) {
         int64_t ns;
 
@@ -434,7 +538,8 @@
         }
         qtest_clock_warp(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + ns);
         qtest_send_prefix(chr);
-        qtest_send(chr, "OK %"PRIi64"\n", (int64_t)qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+        qtest_sendf(chr, "OK %"PRIi64"\n",
+                    (int64_t)qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
     } else if (qtest_enabled() && strcmp(words[0], "clock_set") == 0) {
         int64_t ns;
 
@@ -442,10 +547,11 @@
         ns = strtoll(words[1], NULL, 0);
         qtest_clock_warp(ns);
         qtest_send_prefix(chr);
-        qtest_send(chr, "OK %"PRIi64"\n", (int64_t)qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
+        qtest_sendf(chr, "OK %"PRIi64"\n",
+                    (int64_t)qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
     } else {
         qtest_send_prefix(chr);
-        qtest_send(chr, "FAIL Unknown command `%s'\n", words[0]);
+        qtest_sendf(chr, "FAIL Unknown command '%s'\n", words[0]);
     }
 }
 
@@ -520,16 +626,13 @@
     }
 }
 
-static void configure_qtest_icount(const char *options)
-{
-    QemuOpts *opts  = qemu_opts_parse(qemu_find_opts("icount"), options, 1);
-    configure_icount(opts, &error_abort);
-    qemu_opts_del(opts);
-}
-
 static int qtest_init_accel(MachineState *ms)
 {
-    configure_qtest_icount("0");
+    QemuOpts *opts = qemu_opts_create(qemu_find_opts("icount"), NULL, 0,
+                                      &error_abort);
+    qemu_opt_set(opts, "shift", "0", &error_abort);
+    configure_icount(opts, &error_abort);
+    qemu_opts_del(opts);
     return 0;
 }
 
diff --git a/replay/Makefile.objs b/replay/Makefile.objs
new file mode 100644
index 0000000..fcb3f74
--- /dev/null
+++ b/replay/Makefile.objs
@@ -0,0 +1,6 @@
+common-obj-y += replay.o
+common-obj-y += replay-internal.o
+common-obj-y += replay-events.o
+common-obj-y += replay-time.o
+common-obj-y += replay-input.o
+common-obj-y += replay-char.o
diff --git a/replay/replay-char.c b/replay/replay-char.c
new file mode 100755
index 0000000..edf46ab
--- /dev/null
+++ b/replay/replay-char.c
@@ -0,0 +1,164 @@
+/*
+ * replay-char.c
+ *
+ * Copyright (c) 2010-2016 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/char.h"
+
+/* Char drivers that generate qemu_chr_be_write events
+   that should be saved into the log. */
+static CharDriverState **char_drivers;
+static int drivers_count;
+
+/* Char event attributes. */
+typedef struct CharEvent {
+    int id;
+    uint8_t *buf;
+    size_t len;
+} CharEvent;
+
+static int find_char_driver(CharDriverState *chr)
+{
+    int i = 0;
+    for ( ; i < drivers_count ; ++i) {
+        if (char_drivers[i] == chr) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+void replay_register_char_driver(CharDriverState *chr)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+    char_drivers = g_realloc(char_drivers,
+                             sizeof(*char_drivers) * (drivers_count + 1));
+    char_drivers[drivers_count++] = chr;
+}
+
+void replay_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+{
+    CharEvent *event = g_malloc0(sizeof(CharEvent));
+
+    event->id = find_char_driver(s);
+    if (event->id < 0) {
+        fprintf(stderr, "Replay: cannot find char driver\n");
+        exit(1);
+    }
+    event->buf = g_malloc(len);
+    memcpy(event->buf, buf, len);
+    event->len = len;
+
+    replay_add_event(REPLAY_ASYNC_EVENT_CHAR_READ, event, NULL, 0);
+}
+
+void replay_event_char_read_run(void *opaque)
+{
+    CharEvent *event = (CharEvent *)opaque;
+
+    qemu_chr_be_write_impl(char_drivers[event->id], event->buf,
+                           (int)event->len);
+
+    g_free(event->buf);
+    g_free(event);
+}
+
+void replay_event_char_read_save(void *opaque)
+{
+    CharEvent *event = (CharEvent *)opaque;
+
+    replay_put_byte(event->id);
+    replay_put_array(event->buf, event->len);
+}
+
+void *replay_event_char_read_load(void)
+{
+    CharEvent *event = g_malloc0(sizeof(CharEvent));
+
+    event->id = replay_get_byte();
+    replay_get_array_alloc(&event->buf, &event->len);
+
+    return event;
+}
+
+void replay_char_write_event_save(int res, int offset)
+{
+    replay_save_instructions();
+    replay_mutex_lock();
+    replay_put_event(EVENT_CHAR_WRITE);
+    replay_put_dword(res);
+    replay_put_dword(offset);
+    replay_mutex_unlock();
+}
+
+void replay_char_write_event_load(int *res, int *offset)
+{
+    replay_account_executed_instructions();
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_CHAR_WRITE)) {
+        *res = replay_get_dword();
+        *offset = replay_get_dword();
+        replay_finish_event();
+        replay_mutex_unlock();
+    } else {
+        replay_mutex_unlock();
+        error_report("Missing character write event in the replay log");
+        exit(1);
+    }
+}
+
+int replay_char_read_all_load(uint8_t *buf)
+{
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_CHAR_READ_ALL)) {
+        size_t size;
+        int res;
+        replay_get_array(buf, &size);
+        replay_finish_event();
+        replay_mutex_unlock();
+        res = (int)size;
+        assert(res >= 0);
+        return res;
+    } else if (replay_next_event_is(EVENT_CHAR_READ_ALL_ERROR)) {
+        int res = replay_get_dword();
+        replay_finish_event();
+        replay_mutex_unlock();
+        return res;
+    } else {
+        replay_mutex_unlock();
+        error_report("Missing character read all event in the replay log");
+        exit(1);
+    }
+}
+
+void replay_char_read_all_save_error(int res)
+{
+    assert(res < 0);
+    replay_save_instructions();
+    replay_mutex_lock();
+    replay_put_event(EVENT_CHAR_READ_ALL_ERROR);
+    replay_put_dword(res);
+    replay_mutex_unlock();
+}
+
+void replay_char_read_all_save_buf(uint8_t *buf, int offset)
+{
+    replay_save_instructions();
+    replay_mutex_lock();
+    replay_put_event(EVENT_CHAR_READ_ALL);
+    replay_put_array(buf, offset);
+    replay_mutex_unlock();
+}
diff --git a/replay/replay-events.c b/replay/replay-events.c
new file mode 100644
index 0000000..3807245
--- /dev/null
+++ b/replay/replay-events.c
@@ -0,0 +1,311 @@
+/*
+ * replay-events.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "block/aio.h"
+#include "ui/input.h"
+
+typedef struct Event {
+    ReplayAsyncEventKind event_kind;
+    void *opaque;
+    void *opaque2;
+    uint64_t id;
+
+    QTAILQ_ENTRY(Event) events;
+} Event;
+
+static QTAILQ_HEAD(, Event) events_list = QTAILQ_HEAD_INITIALIZER(events_list);
+static unsigned int read_event_kind = -1;
+static uint64_t read_id = -1;
+static int read_checkpoint = -1;
+
+static bool events_enabled;
+
+/* Functions */
+
+static void replay_run_event(Event *event)
+{
+    switch (event->event_kind) {
+    case REPLAY_ASYNC_EVENT_BH:
+        aio_bh_call(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT:
+        qemu_input_event_send_impl(NULL, (InputEvent *)event->opaque);
+        qapi_free_InputEvent((InputEvent *)event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+        qemu_input_event_sync_impl();
+        break;
+    case REPLAY_ASYNC_EVENT_CHAR_READ:
+        replay_event_char_read_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK:
+        aio_bh_call(event->opaque);
+        break;
+    default:
+        error_report("Replay: invalid async event ID (%d) in the queue",
+                    event->event_kind);
+        exit(1);
+        break;
+    }
+}
+
+void replay_enable_events(void)
+{
+    events_enabled = true;
+}
+
+bool replay_has_events(void)
+{
+    return !QTAILQ_EMPTY(&events_list);
+}
+
+void replay_flush_events(void)
+{
+    replay_mutex_lock();
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+        QTAILQ_REMOVE(&events_list, event, events);
+        g_free(event);
+    }
+    replay_mutex_unlock();
+}
+
+void replay_disable_events(void)
+{
+    if (replay_mode != REPLAY_MODE_NONE) {
+        events_enabled = false;
+        /* Flush events queue before waiting of completion */
+        replay_flush_events();
+    }
+}
+
+void replay_clear_events(void)
+{
+    replay_mutex_lock();
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        QTAILQ_REMOVE(&events_list, event, events);
+
+        g_free(event);
+    }
+    replay_mutex_unlock();
+}
+
+/*! Adds specified async event to the queue */
+void replay_add_event(ReplayAsyncEventKind event_kind,
+                      void *opaque,
+                      void *opaque2, uint64_t id)
+{
+    assert(event_kind < REPLAY_ASYNC_COUNT);
+
+    if (!replay_file || replay_mode == REPLAY_MODE_NONE
+        || !events_enabled) {
+        Event e;
+        e.event_kind = event_kind;
+        e.opaque = opaque;
+        e.opaque2 = opaque2;
+        e.id = id;
+        replay_run_event(&e);
+        return;
+    }
+
+    Event *event = g_malloc0(sizeof(Event));
+    event->event_kind = event_kind;
+    event->opaque = opaque;
+    event->opaque2 = opaque2;
+    event->id = id;
+
+    replay_mutex_lock();
+    QTAILQ_INSERT_TAIL(&events_list, event, events);
+    replay_mutex_unlock();
+}
+
+void replay_bh_schedule_event(QEMUBH *bh)
+{
+    if (replay_mode != REPLAY_MODE_NONE && events_enabled) {
+        uint64_t id = replay_get_current_step();
+        replay_add_event(REPLAY_ASYNC_EVENT_BH, bh, NULL, id);
+    } else {
+        qemu_bh_schedule(bh);
+    }
+}
+
+void replay_add_input_event(struct InputEvent *event)
+{
+    replay_add_event(REPLAY_ASYNC_EVENT_INPUT, event, NULL, 0);
+}
+
+void replay_add_input_sync_event(void)
+{
+    replay_add_event(REPLAY_ASYNC_EVENT_INPUT_SYNC, NULL, NULL, 0);
+}
+
+void replay_block_event(QEMUBH *bh, uint64_t id)
+{
+    if (replay_mode != REPLAY_MODE_NONE && events_enabled) {
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK, bh, NULL, id);
+    } else {
+        qemu_bh_schedule(bh);
+    }
+}
+
+static void replay_save_event(Event *event, int checkpoint)
+{
+    if (replay_mode != REPLAY_MODE_PLAY) {
+        /* put the event into the file */
+        replay_put_event(EVENT_ASYNC);
+        replay_put_byte(checkpoint);
+        replay_put_byte(event->event_kind);
+
+        /* save event-specific data */
+        switch (event->event_kind) {
+        case REPLAY_ASYNC_EVENT_BH:
+            replay_put_qword(event->id);
+            break;
+        case REPLAY_ASYNC_EVENT_INPUT:
+            replay_save_input_event(event->opaque);
+            break;
+        case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+            break;
+        case REPLAY_ASYNC_EVENT_CHAR_READ:
+            replay_event_char_read_save(event->opaque);
+            break;
+        case REPLAY_ASYNC_EVENT_BLOCK:
+            replay_put_qword(event->id);
+            break;
+        default:
+            error_report("Unknown ID %" PRId64 " of replay event", event->id);
+            exit(1);
+        }
+    }
+}
+
+/* Called with replay mutex locked */
+void replay_save_events(int checkpoint)
+{
+    while (!QTAILQ_EMPTY(&events_list)) {
+        Event *event = QTAILQ_FIRST(&events_list);
+        replay_save_event(event, checkpoint);
+
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+        QTAILQ_REMOVE(&events_list, event, events);
+        g_free(event);
+    }
+}
+
+static Event *replay_read_event(int checkpoint)
+{
+    Event *event;
+    if (read_event_kind == -1) {
+        read_checkpoint = replay_get_byte();
+        read_event_kind = replay_get_byte();
+        read_id = -1;
+        replay_check_error();
+    }
+
+    if (checkpoint != read_checkpoint) {
+        return NULL;
+    }
+
+    /* Events that has not to be in the queue */
+    switch (read_event_kind) {
+    case REPLAY_ASYNC_EVENT_BH:
+        if (read_id == -1) {
+            read_id = replay_get_qword();
+        }
+        break;
+    case REPLAY_ASYNC_EVENT_INPUT:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = replay_read_input_event();
+        return event;
+    case REPLAY_ASYNC_EVENT_INPUT_SYNC:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = 0;
+        return event;
+    case REPLAY_ASYNC_EVENT_CHAR_READ:
+        event = g_malloc0(sizeof(Event));
+        event->event_kind = read_event_kind;
+        event->opaque = replay_event_char_read_load();
+        return event;
+    case REPLAY_ASYNC_EVENT_BLOCK:
+        if (read_id == -1) {
+            read_id = replay_get_qword();
+        }
+        break;
+    default:
+        error_report("Unknown ID %d of replay event", read_event_kind);
+        exit(1);
+        break;
+    }
+
+    QTAILQ_FOREACH(event, &events_list, events) {
+        if (event->event_kind == read_event_kind
+            && (read_id == -1 || read_id == event->id)) {
+            break;
+        }
+    }
+
+    if (event) {
+        QTAILQ_REMOVE(&events_list, event, events);
+    } else {
+        return NULL;
+    }
+
+    /* Read event-specific data */
+
+    return event;
+}
+
+/* Called with replay mutex locked */
+void replay_read_events(int checkpoint)
+{
+    while (replay_data_kind == EVENT_ASYNC) {
+        Event *event = replay_read_event(checkpoint);
+        if (!event) {
+            break;
+        }
+        replay_mutex_unlock();
+        replay_run_event(event);
+        replay_mutex_lock();
+
+        g_free(event);
+        replay_finish_event();
+        read_event_kind = -1;
+    }
+}
+
+void replay_init_events(void)
+{
+    read_event_kind = -1;
+}
+
+void replay_finish_events(void)
+{
+    events_enabled = false;
+    replay_clear_events();
+}
+
+bool replay_events_enabled(void)
+{
+    return events_enabled;
+}
diff --git a/replay/replay-input.c b/replay/replay-input.c
new file mode 100644
index 0000000..bd93554
--- /dev/null
+++ b/replay/replay-input.c
@@ -0,0 +1,141 @@
+/*
+ * replay-input.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/notify.h"
+#include "ui/input.h"
+#include "qapi/clone-visitor.h"
+
+void replay_save_input_event(InputEvent *evt)
+{
+    InputKeyEvent *key;
+    InputBtnEvent *btn;
+    InputMoveEvent *move;
+    replay_put_dword(evt->type);
+
+    switch (evt->type) {
+    case INPUT_EVENT_KIND_KEY:
+        key = evt->u.key.data;
+        replay_put_dword(key->key->type);
+
+        switch (key->key->type) {
+        case KEY_VALUE_KIND_NUMBER:
+            replay_put_qword(key->key->u.number.data);
+            replay_put_byte(key->down);
+            break;
+        case KEY_VALUE_KIND_QCODE:
+            replay_put_dword(key->key->u.qcode.data);
+            replay_put_byte(key->down);
+            break;
+        case KEY_VALUE_KIND__MAX:
+            /* keep gcc happy */
+            break;
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        btn = evt->u.btn.data;
+        replay_put_dword(btn->button);
+        replay_put_byte(btn->down);
+        break;
+    case INPUT_EVENT_KIND_REL:
+        move = evt->u.rel.data;
+        replay_put_dword(move->axis);
+        replay_put_qword(move->value);
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        move = evt->u.abs.data;
+        replay_put_dword(move->axis);
+        replay_put_qword(move->value);
+        break;
+    case INPUT_EVENT_KIND__MAX:
+        /* keep gcc happy */
+        break;
+    }
+}
+
+InputEvent *replay_read_input_event(void)
+{
+    InputEvent evt;
+    KeyValue keyValue;
+    InputKeyEvent key;
+    key.key = &keyValue;
+    InputBtnEvent btn;
+    InputMoveEvent rel;
+    InputMoveEvent abs;
+
+    evt.type = replay_get_dword();
+    switch (evt.type) {
+    case INPUT_EVENT_KIND_KEY:
+        evt.u.key.data = &key;
+        evt.u.key.data->key->type = replay_get_dword();
+
+        switch (evt.u.key.data->key->type) {
+        case KEY_VALUE_KIND_NUMBER:
+            evt.u.key.data->key->u.number.data = replay_get_qword();
+            evt.u.key.data->down = replay_get_byte();
+            break;
+        case KEY_VALUE_KIND_QCODE:
+            evt.u.key.data->key->u.qcode.data = (QKeyCode)replay_get_dword();
+            evt.u.key.data->down = replay_get_byte();
+            break;
+        case KEY_VALUE_KIND__MAX:
+            /* keep gcc happy */
+            break;
+        }
+        break;
+    case INPUT_EVENT_KIND_BTN:
+        evt.u.btn.data = &btn;
+        evt.u.btn.data->button = (InputButton)replay_get_dword();
+        evt.u.btn.data->down = replay_get_byte();
+        break;
+    case INPUT_EVENT_KIND_REL:
+        evt.u.rel.data = &rel;
+        evt.u.rel.data->axis = (InputAxis)replay_get_dword();
+        evt.u.rel.data->value = replay_get_qword();
+        break;
+    case INPUT_EVENT_KIND_ABS:
+        evt.u.abs.data = &abs;
+        evt.u.abs.data->axis = (InputAxis)replay_get_dword();
+        evt.u.abs.data->value = replay_get_qword();
+        break;
+    case INPUT_EVENT_KIND__MAX:
+        /* keep gcc happy */
+        break;
+    }
+
+    return QAPI_CLONE(InputEvent, &evt);
+}
+
+void replay_input_event(QemuConsole *src, InputEvent *evt)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        /* Nothing */
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_add_input_event(QAPI_CLONE(InputEvent, evt));
+    } else {
+        qemu_input_event_send_impl(src, evt);
+    }
+}
+
+void replay_input_sync_event(void)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        /* Nothing */
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_add_input_sync_event();
+    } else {
+        qemu_input_event_sync_impl();
+    }
+}
diff --git a/replay/replay-internal.c b/replay/replay-internal.c
new file mode 100644
index 0000000..5835e8d
--- /dev/null
+++ b/replay/replay-internal.c
@@ -0,0 +1,207 @@
+/*
+ * replay-internal.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/error-report.h"
+#include "sysemu/sysemu.h"
+
+unsigned int replay_data_kind = -1;
+static unsigned int replay_has_unread_data;
+
+/* Mutex to protect reading and writing events to the log.
+   replay_data_kind and replay_has_unread_data are also protected
+   by this mutex.
+   It also protects replay events queue which stores events to be
+   written or read to the log. */
+static QemuMutex lock;
+
+/* File for replay writing */
+FILE *replay_file;
+
+void replay_put_byte(uint8_t byte)
+{
+    if (replay_file) {
+        putc(byte, replay_file);
+    }
+}
+
+void replay_put_event(uint8_t event)
+{
+    assert(event < EVENT_COUNT);
+    replay_put_byte(event);
+}
+
+
+void replay_put_word(uint16_t word)
+{
+    replay_put_byte(word >> 8);
+    replay_put_byte(word);
+}
+
+void replay_put_dword(uint32_t dword)
+{
+    replay_put_word(dword >> 16);
+    replay_put_word(dword);
+}
+
+void replay_put_qword(int64_t qword)
+{
+    replay_put_dword(qword >> 32);
+    replay_put_dword(qword);
+}
+
+void replay_put_array(const uint8_t *buf, size_t size)
+{
+    if (replay_file) {
+        replay_put_dword(size);
+        fwrite(buf, 1, size, replay_file);
+    }
+}
+
+uint8_t replay_get_byte(void)
+{
+    uint8_t byte = 0;
+    if (replay_file) {
+        byte = getc(replay_file);
+    }
+    return byte;
+}
+
+uint16_t replay_get_word(void)
+{
+    uint16_t word = 0;
+    if (replay_file) {
+        word = replay_get_byte();
+        word = (word << 8) + replay_get_byte();
+    }
+
+    return word;
+}
+
+uint32_t replay_get_dword(void)
+{
+    uint32_t dword = 0;
+    if (replay_file) {
+        dword = replay_get_word();
+        dword = (dword << 16) + replay_get_word();
+    }
+
+    return dword;
+}
+
+int64_t replay_get_qword(void)
+{
+    int64_t qword = 0;
+    if (replay_file) {
+        qword = replay_get_dword();
+        qword = (qword << 32) + replay_get_dword();
+    }
+
+    return qword;
+}
+
+void replay_get_array(uint8_t *buf, size_t *size)
+{
+    if (replay_file) {
+        *size = replay_get_dword();
+        if (fread(buf, 1, *size, replay_file) != *size) {
+            error_report("replay read error");
+        }
+    }
+}
+
+void replay_get_array_alloc(uint8_t **buf, size_t *size)
+{
+    if (replay_file) {
+        *size = replay_get_dword();
+        *buf = g_malloc(*size);
+        if (fread(*buf, 1, *size, replay_file) != *size) {
+            error_report("replay read error");
+        }
+    }
+}
+
+void replay_check_error(void)
+{
+    if (replay_file) {
+        if (feof(replay_file)) {
+            error_report("replay file is over");
+            qemu_system_vmstop_request_prepare();
+            qemu_system_vmstop_request(RUN_STATE_PAUSED);
+        } else if (ferror(replay_file)) {
+            error_report("replay file is over or something goes wrong");
+            qemu_system_vmstop_request_prepare();
+            qemu_system_vmstop_request(RUN_STATE_INTERNAL_ERROR);
+        }
+    }
+}
+
+void replay_fetch_data_kind(void)
+{
+    if (replay_file) {
+        if (!replay_has_unread_data) {
+            replay_data_kind = replay_get_byte();
+            if (replay_data_kind == EVENT_INSTRUCTION) {
+                replay_state.instructions_count = replay_get_dword();
+            }
+            replay_check_error();
+            replay_has_unread_data = 1;
+            if (replay_data_kind >= EVENT_COUNT) {
+                error_report("Replay: unknown event kind %d", replay_data_kind);
+                exit(1);
+            }
+        }
+    }
+}
+
+void replay_finish_event(void)
+{
+    replay_has_unread_data = 0;
+    replay_fetch_data_kind();
+}
+
+void replay_mutex_init(void)
+{
+    qemu_mutex_init(&lock);
+}
+
+void replay_mutex_destroy(void)
+{
+    qemu_mutex_destroy(&lock);
+}
+
+void replay_mutex_lock(void)
+{
+    qemu_mutex_lock(&lock);
+}
+
+void replay_mutex_unlock(void)
+{
+    qemu_mutex_unlock(&lock);
+}
+
+/*! Saves cached instructions. */
+void replay_save_instructions(void)
+{
+    if (replay_file && replay_mode == REPLAY_MODE_RECORD) {
+        replay_mutex_lock();
+        int diff = (int)(replay_get_current_step() - replay_state.current_step);
+        if (diff > 0) {
+            replay_put_event(EVENT_INSTRUCTION);
+            replay_put_dword(diff);
+            replay_state.current_step += diff;
+        }
+        replay_mutex_unlock();
+    }
+}
diff --git a/replay/replay-internal.h b/replay/replay-internal.h
new file mode 100644
index 0000000..efbf14c
--- /dev/null
+++ b/replay/replay-internal.h
@@ -0,0 +1,158 @@
+#ifndef REPLAY_INTERNAL_H
+#define REPLAY_INTERNAL_H
+
+/*
+ * replay-internal.h
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+
+enum ReplayEvents {
+    /* for instruction event */
+    EVENT_INSTRUCTION,
+    /* for software interrupt */
+    EVENT_INTERRUPT,
+    /* for emulated exceptions */
+    EVENT_EXCEPTION,
+    /* for async events */
+    EVENT_ASYNC,
+    /* for shutdown request */
+    EVENT_SHUTDOWN,
+    /* for character device write event */
+    EVENT_CHAR_WRITE,
+    /* for character device read all event */
+    EVENT_CHAR_READ_ALL,
+    EVENT_CHAR_READ_ALL_ERROR,
+    /* for clock read/writes */
+    /* some of greater codes are reserved for clocks */
+    EVENT_CLOCK,
+    EVENT_CLOCK_LAST = EVENT_CLOCK + REPLAY_CLOCK_COUNT - 1,
+    /* for checkpoint event */
+    /* some of greater codes are reserved for checkpoints */
+    EVENT_CHECKPOINT,
+    EVENT_CHECKPOINT_LAST = EVENT_CHECKPOINT + CHECKPOINT_COUNT - 1,
+    /* end of log event */
+    EVENT_END,
+    EVENT_COUNT
+};
+
+/* Asynchronous events IDs */
+
+enum ReplayAsyncEventKind {
+    REPLAY_ASYNC_EVENT_BH,
+    REPLAY_ASYNC_EVENT_INPUT,
+    REPLAY_ASYNC_EVENT_INPUT_SYNC,
+    REPLAY_ASYNC_EVENT_CHAR_READ,
+    REPLAY_ASYNC_EVENT_BLOCK,
+    REPLAY_ASYNC_COUNT
+};
+
+typedef enum ReplayAsyncEventKind ReplayAsyncEventKind;
+
+typedef struct ReplayState {
+    /*! Cached clock values. */
+    int64_t cached_clock[REPLAY_CLOCK_COUNT];
+    /*! Current step - number of processed instructions and timer events. */
+    uint64_t current_step;
+    /*! Number of instructions to be executed before other events happen. */
+    int instructions_count;
+} ReplayState;
+extern ReplayState replay_state;
+
+extern unsigned int replay_data_kind;
+
+/* File for replay writing */
+extern FILE *replay_file;
+
+void replay_put_byte(uint8_t byte);
+void replay_put_event(uint8_t event);
+void replay_put_word(uint16_t word);
+void replay_put_dword(uint32_t dword);
+void replay_put_qword(int64_t qword);
+void replay_put_array(const uint8_t *buf, size_t size);
+
+uint8_t replay_get_byte(void);
+uint16_t replay_get_word(void);
+uint32_t replay_get_dword(void);
+int64_t replay_get_qword(void);
+void replay_get_array(uint8_t *buf, size_t *size);
+void replay_get_array_alloc(uint8_t **buf, size_t *size);
+
+/* Mutex functions for protecting replay log file */
+
+void replay_mutex_init(void);
+void replay_mutex_destroy(void);
+void replay_mutex_lock(void);
+void replay_mutex_unlock(void);
+
+/*! Checks error status of the file. */
+void replay_check_error(void);
+
+/*! Finishes processing of the replayed event and fetches
+    the next event from the log. */
+void replay_finish_event(void);
+/*! Reads data type from the file and stores it in the
+    replay_data_kind variable. */
+void replay_fetch_data_kind(void);
+
+/*! Saves queued events (like instructions and sound). */
+void replay_save_instructions(void);
+
+/*! Skips async events until some sync event will be found.
+    \return true, if event was found */
+bool replay_next_event_is(int event);
+
+/*! Reads next clock value from the file.
+    If clock kind read from the file is different from the parameter,
+    the value is not used. */
+void replay_read_next_clock(unsigned int kind);
+
+/* Asynchronous events queue */
+
+/*! Initializes events' processing internals */
+void replay_init_events(void);
+/*! Clears internal data structures for events handling */
+void replay_finish_events(void);
+/*! Enables storing events in the queue */
+void replay_enable_events(void);
+/*! Flushes events queue */
+void replay_flush_events(void);
+/*! Clears events list before loading new VM state */
+void replay_clear_events(void);
+/*! Returns true if there are any unsaved events in the queue */
+bool replay_has_events(void);
+/*! Saves events from queue into the file */
+void replay_save_events(int checkpoint);
+/*! Read events from the file into the input queue */
+void replay_read_events(int checkpoint);
+/*! Adds specified async event to the queue */
+void replay_add_event(ReplayAsyncEventKind event_kind, void *opaque,
+                      void *opaque2, uint64_t id);
+
+/* Input events */
+
+/*! Saves input event to the log */
+void replay_save_input_event(InputEvent *evt);
+/*! Reads input event from the log */
+InputEvent *replay_read_input_event(void);
+/*! Adds input event to the queue */
+void replay_add_input_event(struct InputEvent *event);
+/*! Adds input sync event to the queue */
+void replay_add_input_sync_event(void);
+
+/* Character devices */
+
+/*! Called to run char device read event. */
+void replay_event_char_read_run(void *opaque);
+/*! Writes char read event to the file. */
+void replay_event_char_read_save(void *opaque);
+/*! Reads char event read from the file. */
+void *replay_event_char_read_load(void);
+
+#endif
diff --git a/replay/replay-time.c b/replay/replay-time.c
new file mode 100644
index 0000000..fffe072
--- /dev/null
+++ b/replay/replay-time.c
@@ -0,0 +1,65 @@
+/*
+ * replay-time.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/error-report.h"
+
+int64_t replay_save_clock(ReplayClockKind kind, int64_t clock)
+{
+    replay_save_instructions();
+
+    if (replay_file) {
+        replay_mutex_lock();
+        replay_put_event(EVENT_CLOCK + kind);
+        replay_put_qword(clock);
+        replay_mutex_unlock();
+    }
+
+    return clock;
+}
+
+void replay_read_next_clock(ReplayClockKind kind)
+{
+    unsigned int read_kind = replay_data_kind - EVENT_CLOCK;
+
+    assert(read_kind == kind);
+
+    int64_t clock = replay_get_qword();
+
+    replay_check_error();
+    replay_finish_event();
+
+    replay_state.cached_clock[read_kind] = clock;
+}
+
+/*! Reads next clock event from the input. */
+int64_t replay_read_clock(ReplayClockKind kind)
+{
+    replay_account_executed_instructions();
+
+    if (replay_file) {
+        int64_t ret;
+        replay_mutex_lock();
+        if (replay_next_event_is(EVENT_CLOCK + kind)) {
+            replay_read_next_clock(kind);
+        }
+        ret = replay_state.cached_clock[kind];
+        replay_mutex_unlock();
+
+        return ret;
+    }
+
+    error_report("REPLAY INTERNAL ERROR %d", __LINE__);
+    exit(1);
+}
diff --git a/replay/replay.c b/replay/replay.c
new file mode 100644
index 0000000..167fd29
--- /dev/null
+++ b/replay/replay.c
@@ -0,0 +1,354 @@
+/*
+ * replay.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "sysemu/replay.h"
+#include "replay-internal.h"
+#include "qemu/timer.h"
+#include "qemu/main-loop.h"
+#include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
+
+/* Current version of the replay mechanism.
+   Increase it when file format changes. */
+#define REPLAY_VERSION              0xe02004
+/* Size of replay log header */
+#define HEADER_SIZE                 (sizeof(uint32_t) + sizeof(uint64_t))
+
+ReplayMode replay_mode = REPLAY_MODE_NONE;
+
+/* Name of replay file  */
+static char *replay_filename;
+ReplayState replay_state;
+static GSList *replay_blockers;
+
+bool replay_next_event_is(int event)
+{
+    bool res = false;
+
+    /* nothing to skip - not all instructions used */
+    if (replay_state.instructions_count != 0) {
+        assert(replay_data_kind == EVENT_INSTRUCTION);
+        return event == EVENT_INSTRUCTION;
+    }
+
+    while (true) {
+        if (event == replay_data_kind) {
+            res = true;
+        }
+        switch (replay_data_kind) {
+        case EVENT_SHUTDOWN:
+            replay_finish_event();
+            qemu_system_shutdown_request();
+            break;
+        default:
+            /* clock, time_t, checkpoint and other events */
+            return res;
+        }
+    }
+    return res;
+}
+
+uint64_t replay_get_current_step(void)
+{
+    return cpu_get_icount_raw();
+}
+
+int replay_get_instructions(void)
+{
+    int res = 0;
+    replay_mutex_lock();
+    if (replay_next_event_is(EVENT_INSTRUCTION)) {
+        res = replay_state.instructions_count;
+    }
+    replay_mutex_unlock();
+    return res;
+}
+
+void replay_account_executed_instructions(void)
+{
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_mutex_lock();
+        if (replay_state.instructions_count > 0) {
+            int count = (int)(replay_get_current_step()
+                              - replay_state.current_step);
+            replay_state.instructions_count -= count;
+            replay_state.current_step += count;
+            if (replay_state.instructions_count == 0) {
+                assert(replay_data_kind == EVENT_INSTRUCTION);
+                replay_finish_event();
+                /* Wake up iothread. This is required because
+                   timers will not expire until clock counters
+                   will be read from the log. */
+                qemu_notify_event();
+            }
+        }
+        replay_mutex_unlock();
+    }
+}
+
+bool replay_exception(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_save_instructions();
+        replay_mutex_lock();
+        replay_put_event(EVENT_EXCEPTION);
+        replay_mutex_unlock();
+        return true;
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        bool res = replay_has_exception();
+        if (res) {
+            replay_mutex_lock();
+            replay_finish_event();
+            replay_mutex_unlock();
+        }
+        return res;
+    }
+
+    return true;
+}
+
+bool replay_has_exception(void)
+{
+    bool res = false;
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_account_executed_instructions();
+        replay_mutex_lock();
+        res = replay_next_event_is(EVENT_EXCEPTION);
+        replay_mutex_unlock();
+    }
+
+    return res;
+}
+
+bool replay_interrupt(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_save_instructions();
+        replay_mutex_lock();
+        replay_put_event(EVENT_INTERRUPT);
+        replay_mutex_unlock();
+        return true;
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        bool res = replay_has_interrupt();
+        if (res) {
+            replay_mutex_lock();
+            replay_finish_event();
+            replay_mutex_unlock();
+        }
+        return res;
+    }
+
+    return true;
+}
+
+bool replay_has_interrupt(void)
+{
+    bool res = false;
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        replay_account_executed_instructions();
+        replay_mutex_lock();
+        res = replay_next_event_is(EVENT_INTERRUPT);
+        replay_mutex_unlock();
+    }
+    return res;
+}
+
+void replay_shutdown_request(void)
+{
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_mutex_lock();
+        replay_put_event(EVENT_SHUTDOWN);
+        replay_mutex_unlock();
+    }
+}
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    bool res = false;
+    assert(EVENT_CHECKPOINT + checkpoint <= EVENT_CHECKPOINT_LAST);
+    replay_save_instructions();
+
+    if (!replay_file) {
+        return true;
+    }
+
+    replay_mutex_lock();
+
+    if (replay_mode == REPLAY_MODE_PLAY) {
+        if (replay_next_event_is(EVENT_CHECKPOINT + checkpoint)) {
+            replay_finish_event();
+        } else if (replay_data_kind != EVENT_ASYNC) {
+            res = false;
+            goto out;
+        }
+        replay_read_events(checkpoint);
+        /* replay_read_events may leave some unread events.
+           Return false if not all of the events associated with
+           checkpoint were processed */
+        res = replay_data_kind != EVENT_ASYNC;
+    } else if (replay_mode == REPLAY_MODE_RECORD) {
+        replay_put_event(EVENT_CHECKPOINT + checkpoint);
+        replay_save_events(checkpoint);
+        res = true;
+    }
+out:
+    replay_mutex_unlock();
+    return res;
+}
+
+static void replay_enable(const char *fname, int mode)
+{
+    const char *fmode = NULL;
+    assert(!replay_file);
+
+    switch (mode) {
+    case REPLAY_MODE_RECORD:
+        fmode = "wb";
+        break;
+    case REPLAY_MODE_PLAY:
+        fmode = "rb";
+        break;
+    default:
+        fprintf(stderr, "Replay: internal error: invalid replay mode\n");
+        exit(1);
+    }
+
+    atexit(replay_finish);
+
+    replay_mutex_init();
+
+    replay_file = fopen(fname, fmode);
+    if (replay_file == NULL) {
+        fprintf(stderr, "Replay: open %s: %s\n", fname, strerror(errno));
+        exit(1);
+    }
+
+    replay_filename = g_strdup(fname);
+
+    replay_mode = mode;
+    replay_data_kind = -1;
+    replay_state.instructions_count = 0;
+    replay_state.current_step = 0;
+
+    /* skip file header for RECORD and check it for PLAY */
+    if (replay_mode == REPLAY_MODE_RECORD) {
+        fseek(replay_file, HEADER_SIZE, SEEK_SET);
+    } else if (replay_mode == REPLAY_MODE_PLAY) {
+        unsigned int version = replay_get_dword();
+        if (version != REPLAY_VERSION) {
+            fprintf(stderr, "Replay: invalid input log file version\n");
+            exit(1);
+        }
+        /* go to the beginning */
+        fseek(replay_file, HEADER_SIZE, SEEK_SET);
+        replay_fetch_data_kind();
+    }
+
+    replay_init_events();
+}
+
+void replay_configure(QemuOpts *opts)
+{
+    const char *fname;
+    const char *rr;
+    ReplayMode mode = REPLAY_MODE_NONE;
+    Location loc;
+
+    if (!opts) {
+        return;
+    }
+
+    loc_push_none(&loc);
+    qemu_opts_loc_restore(opts);
+
+    rr = qemu_opt_get(opts, "rr");
+    if (!rr) {
+        /* Just enabling icount */
+        goto out;
+    } else if (!strcmp(rr, "record")) {
+        mode = REPLAY_MODE_RECORD;
+    } else if (!strcmp(rr, "replay")) {
+        mode = REPLAY_MODE_PLAY;
+    } else {
+        error_report("Invalid icount rr option: %s", rr);
+        exit(1);
+    }
+
+    fname = qemu_opt_get(opts, "rrfile");
+    if (!fname) {
+        error_report("File name not specified for replay");
+        exit(1);
+    }
+
+    replay_enable(fname, mode);
+
+out:
+    loc_pop(&loc);
+}
+
+void replay_start(void)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+
+    if (replay_blockers) {
+        error_reportf_err(replay_blockers->data, "Record/replay: ");
+        exit(1);
+    }
+    if (!use_icount) {
+        error_report("Please enable icount to use record/replay");
+        exit(1);
+    }
+
+    /* Timer for snapshotting will be set up here. */
+
+    replay_enable_events();
+}
+
+void replay_finish(void)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return;
+    }
+
+    replay_save_instructions();
+
+    /* finalize the file */
+    if (replay_file) {
+        if (replay_mode == REPLAY_MODE_RECORD) {
+            /* write end event */
+            replay_put_event(EVENT_END);
+
+            /* write header */
+            fseek(replay_file, 0, SEEK_SET);
+            replay_put_dword(REPLAY_VERSION);
+        }
+
+        fclose(replay_file);
+        replay_file = NULL;
+    }
+    if (replay_filename) {
+        g_free(replay_filename);
+        replay_filename = NULL;
+    }
+
+    replay_finish_events();
+    replay_mutex_destroy();
+}
+
+void replay_add_blocker(Error *reason)
+{
+    replay_blockers = g_slist_prepend(replay_blockers, reason);
+}
diff --git a/roms/Makefile b/roms/Makefile
index 610b534..88b3709 100644
--- a/roms/Makefile
+++ b/roms/Makefile
@@ -1,11 +1,13 @@
 
-vgabios_variants := stdvga cirrus vmware qxl isavga
+vgabios_variants := stdvga cirrus vmware qxl isavga virtio
 vgabios_targets  := $(subst -isavga,,$(patsubst %,vgabios-%.bin,$(vgabios_variants)))
-pxerom_variants  := e1000 eepro100 ne2k_pci pcnet rtl8139 virtio
-pxerom_targets   := 8086100e 80861209 10500940 10222000 10ec8139 1af41000
+pxerom_variants  := e1000 e1000e eepro100 ne2k_pci pcnet rtl8139 virtio vmxnet3
+pxerom_targets   := 8086100e 808610d3 80861209 10500940 10222000 10ec8139 1af41000 15ad07b0
 
 pxe-rom-e1000    efi-rom-e1000    : VID := 8086
 pxe-rom-e1000    efi-rom-e1000    : DID := 100e
+pxe-rom-e1000e   efi-rom-e1000e   : VID := 8086
+pxe-rom-e1000e   efi-rom-e1000e   : DID := 10d3
 pxe-rom-eepro100 efi-rom-eepro100 : VID := 8086
 pxe-rom-eepro100 efi-rom-eepro100 : DID := 1209
 pxe-rom-ne2k_pci efi-rom-ne2k_pci : VID := 1050
@@ -16,6 +18,8 @@
 pxe-rom-rtl8139  efi-rom-rtl8139  : DID := 8139
 pxe-rom-virtio   efi-rom-virtio   : VID := 1af4
 pxe-rom-virtio   efi-rom-virtio   : DID := 1000
+pxe-rom-vmxnet3  efi-rom-vmxnet3  : VID := 15ad
+pxe-rom-vmxnet3  efi-rom-vmxnet3  : DID := 07b0
 
 #
 # cross compiler auto detection
@@ -34,6 +38,9 @@
 powerpc_cross_prefix := $(call find-cross-prefix,powerpc)
 x86_64_cross_prefix := $(call find-cross-prefix,x86_64)
 
+# tag our seabios builds
+SEABIOS_EXTRAVERSION="-prebuilt.qemu-project.org"
+
 #
 # EfiRom utility is shipped with edk2 / tianocore, in BaseTools/
 #
@@ -61,7 +68,6 @@
 bios: build-seabios-config-seabios-128k build-seabios-config-seabios-256k
 	cp seabios/builds/seabios-128k/bios.bin ../pc-bios/bios.bin
 	cp seabios/builds/seabios-256k/bios.bin ../pc-bios/bios-256k.bin
-	cp seabios/builds/seabios-256k/src/fw/*dsdt.aml ../pc-bios/
 
 seavgabios: $(patsubst %,seavgabios-%,$(vgabios_variants))
 
@@ -75,10 +81,12 @@
 	mkdir -p seabios/builds/$*
 	cp $< seabios/builds/$*/.config
 	$(MAKE) -C seabios \
+		EXTRAVERSION=$(SEABIOS_EXTRAVERSION) \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		KCONFIG_CONFIG=$(CURDIR)/seabios/builds/$*/.config \
 		OUT=$(CURDIR)/seabios/builds/$*/ oldnoconfig
 	$(MAKE) -C seabios \
+		EXTRAVERSION=$(SEABIOS_EXTRAVERSION) \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		KCONFIG_CONFIG=$(CURDIR)/seabios/builds/$*/.config \
 		OUT=$(CURDIR)/seabios/builds/$*/ all
@@ -115,20 +123,17 @@
 		-ec ipxe/src/bin-x86_64-efi/$(VID)$(DID).efidrv \
 		-o ../pc-bios/efi-$*.rom
 
-build-pxe-roms: ipxe/src/config/local/general.h
-	$(MAKE) -C ipxe/src GITVERSION="" \
+build-pxe-roms:
+	$(MAKE) -C ipxe/src CONFIG=qemu \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		$(patsubst %,bin/%.rom,$(pxerom_targets))
 
-build-efi-roms: build-pxe-roms ipxe/src/config/local/general.h
-	$(MAKE) -C ipxe/src GITVERSION="" \
+build-efi-roms: build-pxe-roms
+	$(MAKE) -C ipxe/src CONFIG=qemu \
 		CROSS_COMPILE=$(x86_64_cross_prefix) \
 		$(patsubst %,bin-i386-efi/%.efidrv,$(pxerom_targets)) \
 		$(patsubst %,bin-x86_64-efi/%.efidrv,$(pxerom_targets))
 
-ipxe/src/config/local/%: config.ipxe.%
-	cp $< $@
-
 
 slof:
 	$(MAKE) -C SLOF CROSS=$(powerpc64_cross_prefix) qemu
diff --git a/roms/SLOF b/roms/SLOF
index f284ab3..e3d0572 160000
--- a/roms/SLOF
+++ b/roms/SLOF
@@ -1 +1 @@
-Subproject commit f284ab3f03ae69a20e1ae966f6ddf76da33cbf72
+Subproject commit e3d05727a074619fc12d0a67f05cf2c42c875cce
diff --git a/roms/config.ipxe.general.h b/roms/config.ipxe.general.h
deleted file mode 100644
index 619ee4c..0000000
--- a/roms/config.ipxe.general.h
+++ /dev/null
@@ -1,4 +0,0 @@
-#undef BANNER_TIMEOUT
-#define BANNER_TIMEOUT 30
-#undef ROM_BANNER_TIMEOUT
-#define ROM_BANNER_TIMEOUT 0
diff --git a/roms/config.seabios-128k b/roms/config.seabios-128k
index e653c75..93203af 100644
--- a/roms/config.seabios-128k
+++ b/roms/config.seabios-128k
@@ -2,6 +2,11 @@
 # need to turn off features (xhci,uas) to make it fit into 128k
 CONFIG_QEMU=y
 CONFIG_ROM_SIZE=128
+CONFIG_BOOTSPLASH=n
 CONFIG_XEN=n
+CONFIG_USB_OHCI=n
 CONFIG_USB_XHCI=n
 CONFIG_USB_UAS=n
+CONFIG_SDCARD=n
+CONFIG_TCGBIOS=n
+CONFIG_MPT_SCSI=n
diff --git a/roms/config.vga-virtio b/roms/config.vga-virtio
new file mode 100644
index 0000000..aa7a15b
--- /dev/null
+++ b/roms/config.vga-virtio
@@ -0,0 +1,6 @@
+CONFIG_BUILD_VGABIOS=y
+CONFIG_VGA_BOCHS=y
+CONFIG_VGA_PCI=y
+CONFIG_OVERRIDE_PCI_ID=y
+CONFIG_VGA_VID=0x1af4
+CONFIG_VGA_DID=0x1050
diff --git a/roms/ipxe b/roms/ipxe
index 69313ed..0418631 160000
--- a/roms/ipxe
+++ b/roms/ipxe
@@ -1 +1 @@
-Subproject commit 69313edad85f8958acc8a47272b3c3da494835ec
+Subproject commit 04186319181298083ef28695a8309028b26fe83c
diff --git a/roms/openbios b/roms/openbios
index 038aa78..e79bca6 160000
--- a/roms/openbios
+++ b/roms/openbios
@@ -1 +1 @@
-Subproject commit 038aa78d3c331731733378a73e778ee620a5b9da
+Subproject commit e79bca64838c96ec44fd7acd508879c5284233dd
diff --git a/roms/seabios b/roms/seabios
index 8936dbb..e2fc41e 160000
--- a/roms/seabios
+++ b/roms/seabios
@@ -1 +1 @@
-Subproject commit 8936dbb2cd5403c9a8279c849f7f4b0ccc0796ac
+Subproject commit e2fc41e24ee0ada60fc511d60b15a41b294538be
diff --git a/rules.mak b/rules.mak
index f500fef..99cd0b3 100644
--- a/rules.mak
+++ b/rules.mak
@@ -1,4 +1,6 @@
 
+COMMA := ,
+
 # Don't use implicit rules or variables
 # we have explicit rules for everything
 MAKEFLAGS += -rR
@@ -17,7 +19,7 @@
 QEMU_CXXFLAGS = -D__STDC_LIMIT_MACROS $(filter-out -Wstrict-prototypes -Wmissing-prototypes -Wnested-externs -Wold-style-declaration -Wold-style-definition -Wredundant-decls, $(QEMU_CFLAGS))
 
 # Flags for dependency generation
-QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(*D)/$(*F).d
+QEMU_DGFLAGS += -MMD -MP -MT $@ -MF $(@D)/$(*F).d
 
 # Same as -I$(SRC_PATH) -I., but for the nested source/object directories
 QEMU_INCLUDES += -I$(<D) -I$(@D)
@@ -62,33 +64,12 @@
 # must link with the C++ compiler, not the plain C compiler.
 LINKPROG = $(or $(CXX),$(CC))
 
-ifeq ($(LIBTOOL),)
 LINK = $(call quiet-command, $(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
        $(call process-archive-undefs, $1) \
        $(version-obj-y) $(call extract-libs,$1) $(LIBS),"  LINK  $(TARGET_DIR)$@")
-else
-LIBTOOL += $(if $(V),,--quiet)
-%.lo: %.c
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC $(CC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) $($*.o-cflags) -c -o $@ $<,"  lt CC $@")
-%.lo: %.rc
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=RC $(WINDRES) -I. -o $@ $<,"lt RC   $(TARGET_DIR)$@")
-%.lo: %.dtrace
-	$(call quiet-command,$(LIBTOOL) --mode=compile --tag=CC dtrace -o $@ -G -s $<, " lt GEN $(TARGET_DIR)$@")
 
-LINK = $(call quiet-command,\
-       $(if $(filter %.lo %.la,$1),$(LIBTOOL) --mode=link --tag=CC \
-       )$(LINKPROG) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ \
-       $(call process-archive-undefs, $1)\
-       $(if $(filter %.lo %.la,$1),$(version-lobj-y),$(version-obj-y)) \
-       $(if $(filter %.lo %.la,$1),$(LIBTOOLFLAGS)) \
-       $(call extract-libs,$(1:.lo=.o)) $(LIBS),$(if $(filter %.lo %.la,$1),"lt LINK ", "  LINK  ")"$(TARGET_DIR)$@")
-endif
-
-%.asm: %.S
-	$(call quiet-command,$(CPP) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -o $@ $<,"  CPP   $(TARGET_DIR)$@")
-
-%.o: %.asm
-	$(call quiet-command,$(AS) $(ASFLAGS) -o $@ $<,"  AS    $(TARGET_DIR)$@")
+%.o: %.S
+	$(call quiet-command,$(CCAS) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  CCAS  $(TARGET_DIR)$@")
 
 %.o: %.cc
 	$(call quiet-command,$(CXX) $(QEMU_INCLUDES) $(QEMU_CXXFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) $($@-cflags) -c -o $@ $<,"  CXX   $(TARGET_DIR)$@")
@@ -102,7 +83,8 @@
 %.o: %.dtrace
 	$(call quiet-command,dtrace -o $@ -G -s $<, "  GEN   $(TARGET_DIR)$@")
 
-%$(DSOSUF): CFLAGS += -fPIC -DBUILD_DSO
+DSO_OBJ_CFLAGS := -fPIC -DBUILD_DSO
+module-common.o: CFLAGS += $(DSO_OBJ_CFLAGS)
 %$(DSOSUF): LDFLAGS += $(LDFLAGS_SHARED)
 %$(DSOSUF): %.mo
 	$(call LINK,$^)
@@ -110,7 +92,7 @@
 	$(if $(findstring /,$@),$(call quiet-command,cp $@ $(subst /,-,$@), "  CP    $(subst /,-,$@)"))
 
 
-LD_REL := $(CC) -nostdlib -Wl,-r
+LD_REL := $(CC) -nostdlib -Wl,-r $(LD_REL_FLAGS)
 
 %.mo:
 	$(call quiet-command,$(LD_REL) -o $@ $^,"  LD -r $(TARGET_DIR)$@")
@@ -119,7 +101,7 @@
 modules:
 
 %$(EXESUF): %.o
-	$(call LINK,$^)
+	$(call LINK,$(filter %.o %.a %.mo, $^))
 
 %.a:
 	$(call quiet-command,rm -f $@ && $(AR) rcs $@ $^,"  AR    $(TARGET_DIR)$@")
@@ -131,6 +113,8 @@
 
 cc-option = $(if $(shell $(CC) $1 $2 -S -o /dev/null -xc /dev/null \
               >/dev/null 2>&1 && echo OK), $2, $3)
+cc-c-option = $(if $(shell $(CC) $1 $2 -c -o /dev/null -xc /dev/null \
+                >/dev/null 2>&1 && echo OK), $2, $3)
 
 VPATH_SUFFIXES = %.c %.h %.S %.cc %.cpp %.m %.mak %.texi %.sh %.rc
 set-vpath = $(if $1,$(foreach PATTERN,$(VPATH_SUFFIXES),$(eval vpath $(PATTERN) $1)))
@@ -187,7 +171,7 @@
 config-%.h: config-%.h-timestamp
 	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
 
-config-%.h-timestamp: config-%.mak
+config-%.h-timestamp: config-%.mak $(SRC_PATH)/scripts/create_config
 	$(call quiet-command, sh $(SRC_PATH)/scripts/create_config < $< > $@, "  GEN   $(TARGET_DIR)config-$*.h")
 
 .PHONY: clean-timestamp
@@ -326,7 +310,17 @@
     $(if $1,$(call fix-paths,$1/,,$2))
 
     # Descend and include every subdir Makefile.objs
-    $(foreach v, $2, $(call unnest-var-recursive,$1,$2,$v))
+    $(foreach v, $2,
+        $(call unnest-var-recursive,$1,$2,$v)
+        # Pass the .mo-cflags and .mo-libs along to its member objects
+        $(foreach o, $(filter %.mo,$($v)),
+            $(foreach p,$($o-objs),
+                $(if $($o-cflags), $(eval $p-cflags += $($o-cflags)))
+                $(if $($o-libs), $(eval $p-libs += $($o-libs))))))
+
+    # For all %.mo objects that are directly added into -y, just expand them
+    $(foreach v,$(filter %-y,$2),
+        $(eval $v := $(foreach o,$($v),$(if $($o-objs),$($o-objs),$o))))
 
     $(foreach v,$(filter %-m,$2),
         # All .o found in *-m variables are single object modules, create .mo
@@ -341,6 +335,7 @@
         # For non-module build, add -m to -y
         $(if $(CONFIG_MODULES),
              $(foreach o,$($v),
+                   $(eval $($o-objs): CFLAGS += $(DSO_OBJ_CFLAGS))
                    $(eval $o: $($o-objs)))
              $(eval $(patsubst %-m,%-y,$v) += $($v))
              $(eval modules: $($v:%.mo=%$(DSOSUF))),
@@ -353,18 +348,9 @@
             # according to .mo-objs. Report error if not set
             $(if $($o-objs),
                 $(eval $(o:%.mo=%$(DSOSUF)): module-common.o $($o-objs)),
-                $(error $o added in $v but $o-objs is not set))
-            # Pass the .mo-cflags and .mo-libs along to member objects
-            $(foreach p,$($o-objs),
-                $(if $($o-cflags), $(eval $p-cflags += $($o-cflags)))
-                $(if $($o-libs), $(eval $p-libs += $($o-libs)))))
+                $(error $o added in $v but $o-objs is not set)))
         $(shell mkdir -p ./ $(sort $(dir $($v))))
         # Include all the .d files
-        $(eval -include $(addsuffix *.d, $(sort $(dir $($v)))))
+        $(eval -include $(patsubst %.o,%.d,$(patsubst %.mo,%.d,$($v))))
         $(eval $v := $(filter-out %/,$($v))))
-
-    # For all %.mo objects that are directly added into -y, expand them to %.mo-objs
-    $(foreach v,$2,
-        $(eval $v := $(foreach o,$($v),$(if $($o-objs),$($o-objs),$o))))
-
 endef
diff --git a/savevm.c b/savevm.c
deleted file mode 100644
index ec30f49..0000000
--- a/savevm.c
+++ /dev/null
@@ -1,1373 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "config-host.h"
-#include "qemu-common.h"
-#include "hw/boards.h"
-#include "hw/hw.h"
-#include "hw/qdev.h"
-#include "net/net.h"
-#include "monitor/monitor.h"
-#include "sysemu/sysemu.h"
-#include "qemu/timer.h"
-#include "audio/audio.h"
-#include "migration/migration.h"
-#include "qemu/sockets.h"
-#include "qemu/queue.h"
-#include "sysemu/cpus.h"
-#include "exec/memory.h"
-#include "qmp-commands.h"
-#include "trace.h"
-#include "qemu/iov.h"
-#include "block/snapshot.h"
-#include "block/qapi.h"
-
-
-#ifndef ETH_P_RARP
-#define ETH_P_RARP 0x8035
-#endif
-#define ARP_HTYPE_ETH 0x0001
-#define ARP_PTYPE_IP 0x0800
-#define ARP_OP_REQUEST_REV 0x3
-
-static int announce_self_create(uint8_t *buf,
-                                uint8_t *mac_addr)
-{
-    /* Ethernet header. */
-    memset(buf, 0xff, 6);         /* destination MAC addr */
-    memcpy(buf + 6, mac_addr, 6); /* source MAC addr */
-    *(uint16_t *)(buf + 12) = htons(ETH_P_RARP); /* ethertype */
-
-    /* RARP header. */
-    *(uint16_t *)(buf + 14) = htons(ARP_HTYPE_ETH); /* hardware addr space */
-    *(uint16_t *)(buf + 16) = htons(ARP_PTYPE_IP); /* protocol addr space */
-    *(buf + 18) = 6; /* hardware addr length (ethernet) */
-    *(buf + 19) = 4; /* protocol addr length (IPv4) */
-    *(uint16_t *)(buf + 20) = htons(ARP_OP_REQUEST_REV); /* opcode */
-    memcpy(buf + 22, mac_addr, 6); /* source hw addr */
-    memset(buf + 28, 0x00, 4);     /* source protocol addr */
-    memcpy(buf + 32, mac_addr, 6); /* target hw addr */
-    memset(buf + 38, 0x00, 4);     /* target protocol addr */
-
-    /* Padding to get up to 60 bytes (ethernet min packet size, minus FCS). */
-    memset(buf + 42, 0x00, 18);
-
-    return 60; /* len (FCS will be added by hardware) */
-}
-
-static void qemu_announce_self_iter(NICState *nic, void *opaque)
-{
-    uint8_t buf[60];
-    int len;
-
-    trace_qemu_announce_self_iter(qemu_ether_ntoa(&nic->conf->macaddr));
-    len = announce_self_create(buf, nic->conf->macaddr.a);
-
-    qemu_send_packet_raw(qemu_get_queue(nic), buf, len);
-}
-
-
-static void qemu_announce_self_once(void *opaque)
-{
-    static int count = SELF_ANNOUNCE_ROUNDS;
-    QEMUTimer *timer = *(QEMUTimer **)opaque;
-
-    qemu_foreach_nic(qemu_announce_self_iter, NULL);
-
-    if (--count) {
-        /* delay 50ms, 150ms, 250ms, ... */
-        timer_mod(timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME) +
-                  self_announce_delay(count));
-    } else {
-            timer_del(timer);
-            timer_free(timer);
-    }
-}
-
-void qemu_announce_self(void)
-{
-    static QEMUTimer *timer;
-    timer = timer_new_ms(QEMU_CLOCK_REALTIME, qemu_announce_self_once, &timer);
-    qemu_announce_self_once(&timer);
-}
-
-/***********************************************************/
-/* savevm/loadvm support */
-
-static ssize_t block_writev_buffer(void *opaque, struct iovec *iov, int iovcnt,
-                                   int64_t pos)
-{
-    int ret;
-    QEMUIOVector qiov;
-
-    qemu_iovec_init_external(&qiov, iov, iovcnt);
-    ret = bdrv_writev_vmstate(opaque, &qiov, pos);
-    if (ret < 0) {
-        return ret;
-    }
-
-    return qiov.size;
-}
-
-static int block_put_buffer(void *opaque, const uint8_t *buf,
-                           int64_t pos, int size)
-{
-    bdrv_save_vmstate(opaque, buf, pos, size);
-    return size;
-}
-
-static int block_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    return bdrv_load_vmstate(opaque, buf, pos, size);
-}
-
-static int bdrv_fclose(void *opaque)
-{
-    return bdrv_flush(opaque);
-}
-
-static const QEMUFileOps bdrv_read_ops = {
-    .get_buffer = block_get_buffer,
-    .close =      bdrv_fclose
-};
-
-static const QEMUFileOps bdrv_write_ops = {
-    .put_buffer     = block_put_buffer,
-    .writev_buffer  = block_writev_buffer,
-    .close          = bdrv_fclose
-};
-
-static QEMUFile *qemu_fopen_bdrv(BlockDriverState *bs, int is_writable)
-{
-    if (is_writable) {
-        return qemu_fopen_ops(bs, &bdrv_write_ops);
-    }
-    return qemu_fopen_ops(bs, &bdrv_read_ops);
-}
-
-
-/* QEMUFile timer support.
- * Not in qemu-file.c to not add qemu-timer.c as dependency to qemu-file.c
- */
-
-void timer_put(QEMUFile *f, QEMUTimer *ts)
-{
-    uint64_t expire_time;
-
-    expire_time = timer_expire_time_ns(ts);
-    qemu_put_be64(f, expire_time);
-}
-
-void timer_get(QEMUFile *f, QEMUTimer *ts)
-{
-    uint64_t expire_time;
-
-    expire_time = qemu_get_be64(f);
-    if (expire_time != -1) {
-        timer_mod_ns(ts, expire_time);
-    } else {
-        timer_del(ts);
-    }
-}
-
-
-/* VMState timer support.
- * Not in vmstate.c to not add qemu-timer.c as dependency to vmstate.c
- */
-
-static int get_timer(QEMUFile *f, void *pv, size_t size)
-{
-    QEMUTimer *v = pv;
-    timer_get(f, v);
-    return 0;
-}
-
-static void put_timer(QEMUFile *f, void *pv, size_t size)
-{
-    QEMUTimer *v = pv;
-    timer_put(f, v);
-}
-
-const VMStateInfo vmstate_info_timer = {
-    .name = "timer",
-    .get  = get_timer,
-    .put  = put_timer,
-};
-
-
-typedef struct CompatEntry {
-    char idstr[256];
-    int instance_id;
-} CompatEntry;
-
-typedef struct SaveStateEntry {
-    QTAILQ_ENTRY(SaveStateEntry) entry;
-    char idstr[256];
-    int instance_id;
-    int alias_id;
-    int version_id;
-    int section_id;
-    SaveVMHandlers *ops;
-    const VMStateDescription *vmsd;
-    void *opaque;
-    CompatEntry *compat;
-    int is_ram;
-} SaveStateEntry;
-
-
-static QTAILQ_HEAD(savevm_handlers, SaveStateEntry) savevm_handlers =
-    QTAILQ_HEAD_INITIALIZER(savevm_handlers);
-static int global_section_id;
-
-static void dump_vmstate_vmsd(FILE *out_file,
-                              const VMStateDescription *vmsd, int indent,
-                              bool is_subsection);
-
-static void dump_vmstate_vmsf(FILE *out_file, const VMStateField *field,
-                              int indent)
-{
-    fprintf(out_file, "%*s{\n", indent, "");
-    indent += 2;
-    fprintf(out_file, "%*s\"field\": \"%s\",\n", indent, "", field->name);
-    fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "",
-            field->version_id);
-    fprintf(out_file, "%*s\"field_exists\": %s,\n", indent, "",
-            field->field_exists ? "true" : "false");
-    fprintf(out_file, "%*s\"size\": %zu", indent, "", field->size);
-    if (field->vmsd != NULL) {
-        fprintf(out_file, ",\n");
-        dump_vmstate_vmsd(out_file, field->vmsd, indent, false);
-    }
-    fprintf(out_file, "\n%*s}", indent - 2, "");
-}
-
-static void dump_vmstate_vmss(FILE *out_file,
-                              const VMStateSubsection *subsection,
-                              int indent)
-{
-    if (subsection->vmsd != NULL) {
-        dump_vmstate_vmsd(out_file, subsection->vmsd, indent, true);
-    }
-}
-
-static void dump_vmstate_vmsd(FILE *out_file,
-                              const VMStateDescription *vmsd, int indent,
-                              bool is_subsection)
-{
-    if (is_subsection) {
-        fprintf(out_file, "%*s{\n", indent, "");
-    } else {
-        fprintf(out_file, "%*s\"%s\": {\n", indent, "", "Description");
-    }
-    indent += 2;
-    fprintf(out_file, "%*s\"name\": \"%s\",\n", indent, "", vmsd->name);
-    fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "",
-            vmsd->version_id);
-    fprintf(out_file, "%*s\"minimum_version_id\": %d", indent, "",
-            vmsd->minimum_version_id);
-    if (vmsd->fields != NULL) {
-        const VMStateField *field = vmsd->fields;
-        bool first;
-
-        fprintf(out_file, ",\n%*s\"Fields\": [\n", indent, "");
-        first = true;
-        while (field->name != NULL) {
-            if (field->flags & VMS_MUST_EXIST) {
-                /* Ignore VMSTATE_VALIDATE bits; these don't get migrated */
-                field++;
-                continue;
-            }
-            if (!first) {
-                fprintf(out_file, ",\n");
-            }
-            dump_vmstate_vmsf(out_file, field, indent + 2);
-            field++;
-            first = false;
-        }
-        fprintf(out_file, "\n%*s]", indent, "");
-    }
-    if (vmsd->subsections != NULL) {
-        const VMStateSubsection *subsection = vmsd->subsections;
-        bool first;
-
-        fprintf(out_file, ",\n%*s\"Subsections\": [\n", indent, "");
-        first = true;
-        while (subsection->vmsd != NULL) {
-            if (!first) {
-                fprintf(out_file, ",\n");
-            }
-            dump_vmstate_vmss(out_file, subsection, indent + 2);
-            subsection++;
-            first = false;
-        }
-        fprintf(out_file, "\n%*s]", indent, "");
-    }
-    fprintf(out_file, "\n%*s}", indent - 2, "");
-}
-
-static void dump_machine_type(FILE *out_file)
-{
-    MachineClass *mc;
-
-    mc = MACHINE_GET_CLASS(current_machine);
-
-    fprintf(out_file, "  \"vmschkmachine\": {\n");
-    fprintf(out_file, "    \"Name\": \"%s\"\n", mc->name);
-    fprintf(out_file, "  },\n");
-}
-
-void dump_vmstate_json_to_file(FILE *out_file)
-{
-    GSList *list, *elt;
-    bool first;
-
-    fprintf(out_file, "{\n");
-    dump_machine_type(out_file);
-
-    first = true;
-    list = object_class_get_list(TYPE_DEVICE, true);
-    for (elt = list; elt; elt = elt->next) {
-        DeviceClass *dc = OBJECT_CLASS_CHECK(DeviceClass, elt->data,
-                                             TYPE_DEVICE);
-        const char *name;
-        int indent = 2;
-
-        if (!dc->vmsd) {
-            continue;
-        }
-
-        if (!first) {
-            fprintf(out_file, ",\n");
-        }
-        name = object_class_get_name(OBJECT_CLASS(dc));
-        fprintf(out_file, "%*s\"%s\": {\n", indent, "", name);
-        indent += 2;
-        fprintf(out_file, "%*s\"Name\": \"%s\",\n", indent, "", name);
-        fprintf(out_file, "%*s\"version_id\": %d,\n", indent, "",
-                dc->vmsd->version_id);
-        fprintf(out_file, "%*s\"minimum_version_id\": %d,\n", indent, "",
-                dc->vmsd->minimum_version_id);
-
-        dump_vmstate_vmsd(out_file, dc->vmsd, indent, false);
-
-        fprintf(out_file, "\n%*s}", indent - 2, "");
-        first = false;
-    }
-    fprintf(out_file, "\n}\n");
-    fclose(out_file);
-}
-
-static int calculate_new_instance_id(const char *idstr)
-{
-    SaveStateEntry *se;
-    int instance_id = 0;
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (strcmp(idstr, se->idstr) == 0
-            && instance_id <= se->instance_id) {
-            instance_id = se->instance_id + 1;
-        }
-    }
-    return instance_id;
-}
-
-static int calculate_compat_instance_id(const char *idstr)
-{
-    SaveStateEntry *se;
-    int instance_id = 0;
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (!se->compat) {
-            continue;
-        }
-
-        if (strcmp(idstr, se->compat->idstr) == 0
-            && instance_id <= se->compat->instance_id) {
-            instance_id = se->compat->instance_id + 1;
-        }
-    }
-    return instance_id;
-}
-
-/* TODO: Individual devices generally have very little idea about the rest
-   of the system, so instance_id should be removed/replaced.
-   Meanwhile pass -1 as instance_id if you do not already have a clearly
-   distinguishing id for all instances of your device class. */
-int register_savevm_live(DeviceState *dev,
-                         const char *idstr,
-                         int instance_id,
-                         int version_id,
-                         SaveVMHandlers *ops,
-                         void *opaque)
-{
-    SaveStateEntry *se;
-
-    se = g_malloc0(sizeof(SaveStateEntry));
-    se->version_id = version_id;
-    se->section_id = global_section_id++;
-    se->ops = ops;
-    se->opaque = opaque;
-    se->vmsd = NULL;
-    /* if this is a live_savem then set is_ram */
-    if (ops->save_live_setup != NULL) {
-        se->is_ram = 1;
-    }
-
-    if (dev) {
-        char *id = qdev_get_dev_path(dev);
-        if (id) {
-            pstrcpy(se->idstr, sizeof(se->idstr), id);
-            pstrcat(se->idstr, sizeof(se->idstr), "/");
-            g_free(id);
-
-            se->compat = g_malloc0(sizeof(CompatEntry));
-            pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), idstr);
-            se->compat->instance_id = instance_id == -1 ?
-                         calculate_compat_instance_id(idstr) : instance_id;
-            instance_id = -1;
-        }
-    }
-    pstrcat(se->idstr, sizeof(se->idstr), idstr);
-
-    if (instance_id == -1) {
-        se->instance_id = calculate_new_instance_id(se->idstr);
-    } else {
-        se->instance_id = instance_id;
-    }
-    assert(!se->compat || se->instance_id == 0);
-    /* add at the end of list */
-    QTAILQ_INSERT_TAIL(&savevm_handlers, se, entry);
-    return 0;
-}
-
-int register_savevm(DeviceState *dev,
-                    const char *idstr,
-                    int instance_id,
-                    int version_id,
-                    SaveStateHandler *save_state,
-                    LoadStateHandler *load_state,
-                    void *opaque)
-{
-    SaveVMHandlers *ops = g_malloc0(sizeof(SaveVMHandlers));
-    ops->save_state = save_state;
-    ops->load_state = load_state;
-    return register_savevm_live(dev, idstr, instance_id, version_id,
-                                ops, opaque);
-}
-
-int register_savevm_with_post_load(DeviceState *dev,
-                                   const char *idstr,
-                                   int instance_id,
-                                   int version_id,
-                                   SaveStateHandler *save_state,
-                                   LoadStateHandler *load_state,
-                                   PostLoadHandler *post_load,
-                                   void *opaque)
-{
-    SaveVMHandlers *ops = g_malloc0(sizeof(SaveVMHandlers));
-    ops->save_state = save_state;
-    ops->load_state = load_state;
-    ops->post_load = post_load;
-    return register_savevm_live(dev, idstr, instance_id, version_id,
-                                ops, opaque);
-}
-
-void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque)
-{
-    SaveStateEntry *se, *new_se;
-    char id[256] = "";
-
-    if (dev) {
-        char *path = qdev_get_dev_path(dev);
-        if (path) {
-            pstrcpy(id, sizeof(id), path);
-            pstrcat(id, sizeof(id), "/");
-            g_free(path);
-        }
-    }
-    pstrcat(id, sizeof(id), idstr);
-
-    QTAILQ_FOREACH_SAFE(se, &savevm_handlers, entry, new_se) {
-        if (strcmp(se->idstr, id) == 0 && se->opaque == opaque) {
-            QTAILQ_REMOVE(&savevm_handlers, se, entry);
-            if (se->compat) {
-                g_free(se->compat);
-            }
-            g_free(se->ops);
-            g_free(se);
-        }
-    }
-}
-
-int vmstate_register_with_alias_id(DeviceState *dev, int instance_id,
-                                   const VMStateDescription *vmsd,
-                                   void *opaque, int alias_id,
-                                   int required_for_version)
-{
-    SaveStateEntry *se;
-
-    /* If this triggers, alias support can be dropped for the vmsd. */
-    assert(alias_id == -1 || required_for_version >= vmsd->minimum_version_id);
-
-    se = g_malloc0(sizeof(SaveStateEntry));
-    se->version_id = vmsd->version_id;
-    se->section_id = global_section_id++;
-    se->opaque = opaque;
-    se->vmsd = vmsd;
-    se->alias_id = alias_id;
-
-    if (dev) {
-        char *id = qdev_get_dev_path(dev);
-        if (id) {
-            pstrcpy(se->idstr, sizeof(se->idstr), id);
-            pstrcat(se->idstr, sizeof(se->idstr), "/");
-            g_free(id);
-
-            se->compat = g_malloc0(sizeof(CompatEntry));
-            pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), vmsd->name);
-            se->compat->instance_id = instance_id == -1 ?
-                         calculate_compat_instance_id(vmsd->name) : instance_id;
-            instance_id = -1;
-        }
-    }
-    pstrcat(se->idstr, sizeof(se->idstr), vmsd->name);
-
-    if (instance_id == -1) {
-        se->instance_id = calculate_new_instance_id(se->idstr);
-    } else {
-        se->instance_id = instance_id;
-    }
-    assert(!se->compat || se->instance_id == 0);
-    /* add at the end of list */
-    QTAILQ_INSERT_TAIL(&savevm_handlers, se, entry);
-    return 0;
-}
-
-void vmstate_unregister(DeviceState *dev, const VMStateDescription *vmsd,
-                        void *opaque)
-{
-    SaveStateEntry *se, *new_se;
-
-    QTAILQ_FOREACH_SAFE(se, &savevm_handlers, entry, new_se) {
-        if (se->vmsd == vmsd && se->opaque == opaque) {
-            QTAILQ_REMOVE(&savevm_handlers, se, entry);
-            if (se->compat) {
-                g_free(se->compat);
-            }
-            g_free(se);
-        }
-    }
-}
-
-static int vmstate_load(QEMUFile *f, SaveStateEntry *se, int version_id)
-{
-    trace_vmstate_load(se->idstr, se->vmsd ? se->vmsd->name : "(old)");
-    if (!se->vmsd) {         /* Old style */
-        return se->ops->load_state(f, se->opaque, version_id);
-    }
-    return vmstate_load_state(f, se->vmsd, se->opaque, version_id);
-}
-
-static void vmstate_save(QEMUFile *f, SaveStateEntry *se)
-{
-    trace_vmstate_save(se->idstr, se->vmsd ? se->vmsd->name : "(old)");
-    if (!se->vmsd) {         /* Old style */
-        se->ops->save_state(f, se->opaque);
-        return;
-    }
-    vmstate_save_state(f, se->vmsd, se->opaque);
-}
-
-bool qemu_savevm_state_blocked(Error **errp)
-{
-    SaveStateEntry *se;
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (se->vmsd && se->vmsd->unmigratable) {
-            error_setg(errp, "State blocked by non-migratable device '%s'",
-                       se->idstr);
-            return true;
-        }
-    }
-    return false;
-}
-
-void qemu_savevm_state_begin(QEMUFile *f,
-                             const MigrationParams *params)
-{
-    SaveStateEntry *se;
-    int ret;
-
-    trace_savevm_state_begin();
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (!se->ops || !se->ops->set_params) {
-            continue;
-        }
-        se->ops->set_params(params, se->opaque);
-    }
-
-    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
-    qemu_put_be32(f, QEMU_VM_FILE_VERSION);
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        int len;
-
-        if (!se->ops || !se->ops->save_live_setup) {
-            continue;
-        }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
-                continue;
-            }
-        }
-        /* Section type */
-        qemu_put_byte(f, QEMU_VM_SECTION_START);
-        qemu_put_be32(f, se->section_id);
-
-        /* ID string */
-        len = strlen(se->idstr);
-        qemu_put_byte(f, len);
-        qemu_put_buffer(f, (uint8_t *)se->idstr, len);
-
-        qemu_put_be32(f, se->instance_id);
-        qemu_put_be32(f, se->version_id);
-
-        ret = se->ops->save_live_setup(f, se->opaque);
-        if (ret < 0) {
-            qemu_file_set_error(f, ret);
-            break;
-        }
-    }
-}
-
-/*
- * this function has three return values:
- *   negative: there was one error, and we have -errno.
- *   0 : We haven't finished, caller have to go again
- *   1 : We have finished, we can go to complete phase
- */
-int qemu_savevm_state_iterate(QEMUFile *f)
-{
-    SaveStateEntry *se;
-    int ret = 1;
-
-    trace_savevm_state_iterate();
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (!se->ops || !se->ops->save_live_iterate) {
-            continue;
-        }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
-                continue;
-            }
-        }
-        if (qemu_file_rate_limit(f)) {
-            return 0;
-        }
-        trace_savevm_section_start(se->idstr, se->section_id);
-        /* Section type */
-        qemu_put_byte(f, QEMU_VM_SECTION_PART);
-        qemu_put_be32(f, se->section_id);
-
-        ret = se->ops->save_live_iterate(f, se->opaque);
-        trace_savevm_section_end(se->idstr, se->section_id);
-
-        if (ret < 0) {
-            qemu_file_set_error(f, ret);
-        }
-        if (ret <= 0) {
-            /* Do not proceed to the next vmstate before this one reported
-               completion of the current stage. This serializes the migration
-               and reduces the probability that a faster changing state is
-               synchronized over and over again. */
-            break;
-        }
-    }
-    return ret;
-}
-
-void qemu_savevm_state_complete(QEMUFile *f)
-{
-    SaveStateEntry *se;
-    int ret;
-
-    trace_savevm_state_complete();
-
-    cpu_synchronize_all_states();
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (!se->ops || !se->ops->save_live_complete) {
-            continue;
-        }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
-                continue;
-            }
-        }
-        trace_savevm_section_start(se->idstr, se->section_id);
-        /* Section type */
-        qemu_put_byte(f, QEMU_VM_SECTION_END);
-        qemu_put_be32(f, se->section_id);
-
-        ret = se->ops->save_live_complete(f, se->opaque);
-        trace_savevm_section_end(se->idstr, se->section_id);
-        if (ret < 0) {
-            qemu_file_set_error(f, ret);
-            return;
-        }
-    }
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        int len;
-
-        if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
-            continue;
-        }
-        trace_savevm_section_start(se->idstr, se->section_id);
-        /* Section type */
-        qemu_put_byte(f, QEMU_VM_SECTION_FULL);
-        qemu_put_be32(f, se->section_id);
-
-        /* ID string */
-        len = strlen(se->idstr);
-        qemu_put_byte(f, len);
-        qemu_put_buffer(f, (uint8_t *)se->idstr, len);
-
-        qemu_put_be32(f, se->instance_id);
-        qemu_put_be32(f, se->version_id);
-
-        vmstate_save(f, se);
-        trace_savevm_section_end(se->idstr, se->section_id);
-    }
-
-    qemu_put_byte(f, QEMU_VM_EOF);
-    qemu_fflush(f);
-}
-
-uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size)
-{
-    SaveStateEntry *se;
-    uint64_t ret = 0;
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (!se->ops || !se->ops->save_live_pending) {
-            continue;
-        }
-        if (se->ops && se->ops->is_active) {
-            if (!se->ops->is_active(se->opaque)) {
-                continue;
-            }
-        }
-        ret += se->ops->save_live_pending(f, se->opaque, max_size);
-    }
-    return ret;
-}
-
-void qemu_savevm_state_cancel(void)
-{
-    SaveStateEntry *se;
-
-    trace_savevm_state_cancel();
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (se->ops && se->ops->cancel) {
-            se->ops->cancel(se->opaque);
-        }
-    }
-}
-
-static int qemu_savevm_state(QEMUFile *f)
-{
-    int ret;
-    MigrationParams params = {
-        .blk = 0,
-        .shared = 0
-    };
-
-    if (qemu_savevm_state_blocked(NULL)) {
-        return -EINVAL;
-    }
-
-    qemu_mutex_unlock_iothread();
-    qemu_savevm_state_begin(f, &params);
-    qemu_mutex_lock_iothread();
-
-    while (qemu_file_get_error(f) == 0) {
-        if (qemu_savevm_state_iterate(f) > 0) {
-            break;
-        }
-    }
-
-    ret = qemu_file_get_error(f);
-    if (ret == 0) {
-        qemu_savevm_state_complete(f);
-        ret = qemu_file_get_error(f);
-    }
-    if (ret != 0) {
-        qemu_savevm_state_cancel();
-    }
-    return ret;
-}
-
-static int qemu_save_device_state(QEMUFile *f)
-{
-    SaveStateEntry *se;
-
-    qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
-    qemu_put_be32(f, QEMU_VM_FILE_VERSION);
-
-    cpu_synchronize_all_states();
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        int len;
-
-        if (se->is_ram) {
-            continue;
-        }
-        if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
-            continue;
-        }
-
-        /* Section type */
-        qemu_put_byte(f, QEMU_VM_SECTION_FULL);
-        qemu_put_be32(f, se->section_id);
-
-        /* ID string */
-        len = strlen(se->idstr);
-        qemu_put_byte(f, len);
-        qemu_put_buffer(f, (uint8_t *)se->idstr, len);
-
-        qemu_put_be32(f, se->instance_id);
-        qemu_put_be32(f, se->version_id);
-
-        vmstate_save(f, se);
-    }
-
-    qemu_put_byte(f, QEMU_VM_EOF);
-
-    return qemu_file_get_error(f);
-}
-
-static SaveStateEntry *find_se(const char *idstr, int instance_id)
-{
-    SaveStateEntry *se;
-
-    QTAILQ_FOREACH(se, &savevm_handlers, entry) {
-        if (!strcmp(se->idstr, idstr) &&
-            (instance_id == se->instance_id ||
-             instance_id == se->alias_id))
-            return se;
-        /* Migrating from an older version? */
-        if (strstr(se->idstr, idstr) && se->compat) {
-            if (!strcmp(se->compat->idstr, idstr) &&
-                (instance_id == se->compat->instance_id ||
-                 instance_id == se->alias_id))
-                return se;
-        }
-    }
-    return NULL;
-}
-
-typedef struct LoadStateEntry {
-    QLIST_ENTRY(LoadStateEntry) entry;
-    SaveStateEntry *se;
-    int section_id;
-    int version_id;
-} LoadStateEntry;
-
-int qemu_loadvm_state(QEMUFile *f)
-{
-    QLIST_HEAD(, LoadStateEntry) loadvm_handlers =
-        QLIST_HEAD_INITIALIZER(loadvm_handlers);
-    LoadStateEntry *le, *new_le;
-    uint8_t section_type;
-    unsigned int v;
-    int ret;
-
-    if (qemu_savevm_state_blocked(NULL)) {
-        return -EINVAL;
-    }
-
-    v = qemu_get_be32(f);
-    if (v != QEMU_VM_FILE_MAGIC) {
-        return -EINVAL;
-    }
-
-    v = qemu_get_be32(f);
-    if (v == QEMU_VM_FILE_VERSION_COMPAT) {
-        fprintf(stderr, "SaveVM v2 format is obsolete and don't work anymore\n");
-        return -ENOTSUP;
-    }
-    if (v != QEMU_VM_FILE_VERSION) {
-        return -ENOTSUP;
-    }
-
-    while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) {
-        uint32_t instance_id, version_id, section_id;
-        SaveStateEntry *se;
-        char idstr[257];
-        int len;
-
-        switch (section_type) {
-        case QEMU_VM_SECTION_START:
-        case QEMU_VM_SECTION_FULL:
-            /* Read section start */
-            section_id = qemu_get_be32(f);
-            len = qemu_get_byte(f);
-            qemu_get_buffer(f, (uint8_t *)idstr, len);
-            idstr[len] = 0;
-            instance_id = qemu_get_be32(f);
-            version_id = qemu_get_be32(f);
-
-            /* Find savevm section */
-            se = find_se(idstr, instance_id);
-            if (se == NULL) {
-                fprintf(stderr, "Unknown savevm section or instance '%s' %d\n", idstr, instance_id);
-                ret = -EINVAL;
-                goto out;
-            }
-
-            /* Validate version */
-            if (version_id > se->version_id) {
-                fprintf(stderr, "savevm: unsupported version %d for '%s' v%d\n",
-                        version_id, idstr, se->version_id);
-                ret = -EINVAL;
-                goto out;
-            }
-
-            /* Add entry */
-            le = g_malloc0(sizeof(*le));
-
-            le->se = se;
-            le->section_id = section_id;
-            le->version_id = version_id;
-            QLIST_INSERT_HEAD(&loadvm_handlers, le, entry);
-
-            ret = vmstate_load(f, le->se, le->version_id);
-            if (ret < 0) {
-                fprintf(stderr, "qemu: warning: error while loading state for instance 0x%x of device '%s'\n",
-                        instance_id, idstr);
-                goto out;
-            }
-            break;
-        case QEMU_VM_SECTION_PART:
-        case QEMU_VM_SECTION_END:
-            section_id = qemu_get_be32(f);
-
-            QLIST_FOREACH(le, &loadvm_handlers, entry) {
-                if (le->section_id == section_id) {
-                    break;
-                }
-            }
-            if (le == NULL) {
-                fprintf(stderr, "Unknown savevm section %d\n", section_id);
-                ret = -EINVAL;
-                goto out;
-            }
-
-            ret = vmstate_load(f, le->se, le->version_id);
-            if (ret < 0) {
-                fprintf(stderr, "qemu: warning: error while loading state section id %d\n",
-                        section_id);
-                goto out;
-            }
-            break;
-        default:
-            fprintf(stderr, "Unknown savevm section type %d\n", section_type);
-            ret = -EINVAL;
-            goto out;
-        }
-    }
-
-    cpu_synchronize_all_post_init();
-
-    ret = 0;
-
-out:
-    QLIST_FOREACH_SAFE(le, &loadvm_handlers, entry, new_le) {
-        if (!le->se->vmsd && le->se->ops->post_load) {
-            le->se->ops->post_load(le->se->opaque);
-        }
-        QLIST_REMOVE(le, entry);
-        g_free(le);
-    }
-
-    if (ret == 0) {
-        ret = qemu_file_get_error(f);
-    }
-
-    return ret;
-}
-
-static BlockDriverState *find_vmstate_bs(void)
-{
-    BlockDriverState *bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            return bs;
-        }
-    }
-    return NULL;
-}
-
-/*
- * Deletes snapshots of a given name in all opened images.
- */
-static int del_existing_snapshots(Monitor *mon, const char *name)
-{
-    BlockDriverState *bs;
-    QEMUSnapshotInfo sn1, *snapshot = &sn1;
-    Error *err = NULL;
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs) &&
-            bdrv_snapshot_find(bs, snapshot, name) >= 0) {
-            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
-            if (err) {
-                monitor_printf(mon,
-                               "Error while deleting snapshot on device '%s':"
-                               " %s\n",
-                               bdrv_get_device_name(bs),
-                               error_get_pretty(err));
-                error_free(err);
-                return -1;
-            }
-        }
-    }
-
-    return 0;
-}
-
-void do_savevm(Monitor *mon, const QDict *qdict)
-{
-    BlockDriverState *bs, *bs1;
-    QEMUSnapshotInfo sn1, *sn = &sn1, old_sn1, *old_sn = &old_sn1;
-    int ret;
-    QEMUFile *f;
-    int saved_vm_running;
-    uint64_t vm_state_size;
-    qemu_timeval tv;
-    struct tm tm;
-    const char *name = qdict_get_try_str(qdict, "name");
-
-    /* Verify if there is a device that doesn't support snapshots and is writable */
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-
-        if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-            continue;
-        }
-
-        if (!bdrv_can_snapshot(bs)) {
-            monitor_printf(mon, "Device '%s' is writable but does not support snapshots.\n",
-                               bdrv_get_device_name(bs));
-            return;
-        }
-    }
-
-    bs = find_vmstate_bs();
-    if (!bs) {
-        monitor_printf(mon, "No block device can accept snapshots\n");
-        return;
-    }
-
-    saved_vm_running = runstate_is_running();
-    vm_stop(RUN_STATE_SAVE_VM);
-
-    memset(sn, 0, sizeof(*sn));
-
-    /* fill auxiliary fields */
-    qemu_gettimeofday(&tv);
-    sn->date_sec = tv.tv_sec;
-    sn->date_nsec = tv.tv_usec * 1000;
-    sn->vm_clock_nsec = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
-
-    if (name) {
-        ret = bdrv_snapshot_find(bs, old_sn, name);
-        if (ret >= 0) {
-            pstrcpy(sn->name, sizeof(sn->name), old_sn->name);
-            pstrcpy(sn->id_str, sizeof(sn->id_str), old_sn->id_str);
-        } else {
-            pstrcpy(sn->name, sizeof(sn->name), name);
-        }
-    } else {
-        /* cast below needed for OpenBSD where tv_sec is still 'long' */
-        localtime_r((const time_t *)&tv.tv_sec, &tm);
-        strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm);
-    }
-
-    /* Delete old snapshots of the same name */
-    if (name && del_existing_snapshots(mon, name) < 0) {
-        goto the_end;
-    }
-
-    /* save the VM state */
-    f = qemu_fopen_bdrv(bs, 1);
-    if (!f) {
-        monitor_printf(mon, "Could not open VM state file\n");
-        goto the_end;
-    }
-    ret = qemu_savevm_state(f);
-    vm_state_size = qemu_ftell(f);
-    qemu_fclose(f);
-    if (ret < 0) {
-        monitor_printf(mon, "Error %d while writing VM\n", ret);
-        goto the_end;
-    }
-
-    /* create the snapshots */
-
-    bs1 = NULL;
-    while ((bs1 = bdrv_next(bs1))) {
-        if (bdrv_can_snapshot(bs1)) {
-            /* Write VM state size only to the image that contains the state */
-            sn->vm_state_size = (bs == bs1 ? vm_state_size : 0);
-            ret = bdrv_snapshot_create(bs1, sn);
-            if (ret < 0) {
-                monitor_printf(mon, "Error while creating snapshot on '%s'\n",
-                               bdrv_get_device_name(bs1));
-            }
-        }
-    }
-
- the_end:
-    if (saved_vm_running) {
-        vm_start();
-    }
-}
-
-void qmp_xen_save_devices_state(const char *filename, Error **errp)
-{
-    QEMUFile *f;
-    int saved_vm_running;
-    int ret;
-
-    saved_vm_running = runstate_is_running();
-    vm_stop(RUN_STATE_SAVE_VM);
-
-    f = qemu_fopen(filename, "wb");
-    if (!f) {
-        error_setg_file_open(errp, errno, filename);
-        goto the_end;
-    }
-    ret = qemu_save_device_state(f);
-    qemu_fclose(f);
-    if (ret < 0) {
-        error_set(errp, QERR_IO_ERROR);
-    }
-
- the_end:
-    if (saved_vm_running) {
-        vm_start();
-    }
-}
-
-int load_vmstate(const char *name)
-{
-    BlockDriverState *bs, *bs_vm_state;
-    QEMUSnapshotInfo sn;
-    QEMUFile *f;
-    int ret;
-
-    bs_vm_state = find_vmstate_bs();
-    if (!bs_vm_state) {
-        error_report("No block device supports snapshots");
-        return -ENOTSUP;
-    }
-
-    /* Don't even try to load empty VM states */
-    ret = bdrv_snapshot_find(bs_vm_state, &sn, name);
-    if (ret < 0) {
-        return ret;
-    } else if (sn.vm_state_size == 0) {
-        error_report("This is a disk-only snapshot. Revert to it offline "
-            "using qemu-img.");
-        return -EINVAL;
-    }
-
-    /* Verify if there is any device that doesn't support snapshots and is
-    writable and check if the requested snapshot is available too. */
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-
-        if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) {
-            continue;
-        }
-
-        if (!bdrv_can_snapshot(bs)) {
-            error_report("Device '%s' is writable but does not support snapshots.",
-                               bdrv_get_device_name(bs));
-            return -ENOTSUP;
-        }
-
-        ret = bdrv_snapshot_find(bs, &sn, name);
-        if (ret < 0) {
-            error_report("Device '%s' does not have the requested snapshot '%s'",
-                           bdrv_get_device_name(bs), name);
-            return ret;
-        }
-    }
-
-    /* Flush all IO requests so they don't interfere with the new state.  */
-    bdrv_drain_all();
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            ret = bdrv_snapshot_goto(bs, name);
-            if (ret < 0) {
-                error_report("Error %d while activating snapshot '%s' on '%s'",
-                             ret, name, bdrv_get_device_name(bs));
-                return ret;
-            }
-        }
-    }
-
-    /* restore the VM state */
-    f = qemu_fopen_bdrv(bs_vm_state, 0);
-    if (!f) {
-        error_report("Could not open VM state file");
-        return -EINVAL;
-    }
-
-    qemu_system_reset(VMRESET_SILENT);
-    ret = qemu_loadvm_state(f);
-
-    qemu_fclose(f);
-    if (ret < 0) {
-        error_report("Error %d while loading VM state", ret);
-        return ret;
-    }
-
-    return 0;
-}
-
-void do_delvm(Monitor *mon, const QDict *qdict)
-{
-    BlockDriverState *bs;
-    Error *err;
-    const char *name = qdict_get_str(qdict, "name");
-
-    if (!find_vmstate_bs()) {
-        monitor_printf(mon, "No block device supports snapshots\n");
-        return;
-    }
-
-    bs = NULL;
-    while ((bs = bdrv_next(bs))) {
-        if (bdrv_can_snapshot(bs)) {
-            err = NULL;
-            bdrv_snapshot_delete_by_id_or_name(bs, name, &err);
-            if (err) {
-                monitor_printf(mon,
-                               "Error while deleting snapshot on device '%s':"
-                               " %s\n",
-                               bdrv_get_device_name(bs),
-                               error_get_pretty(err));
-                error_free(err);
-            }
-        }
-    }
-}
-
-void do_info_snapshots(Monitor *mon, const QDict *qdict)
-{
-    BlockDriverState *bs, *bs1;
-    QEMUSnapshotInfo *sn_tab, *sn, s, *sn_info = &s;
-    int nb_sns, i, ret, available;
-    int total;
-    int *available_snapshots;
-
-    bs = find_vmstate_bs();
-    if (!bs) {
-        monitor_printf(mon, "No available block device supports snapshots\n");
-        return;
-    }
-
-    nb_sns = bdrv_snapshot_list(bs, &sn_tab);
-    if (nb_sns < 0) {
-        monitor_printf(mon, "bdrv_snapshot_list: error %d\n", nb_sns);
-        return;
-    }
-
-    if (nb_sns == 0) {
-        monitor_printf(mon, "There is no snapshot available.\n");
-        return;
-    }
-
-    available_snapshots = g_malloc0(sizeof(int) * nb_sns);
-    total = 0;
-    for (i = 0; i < nb_sns; i++) {
-        sn = &sn_tab[i];
-        available = 1;
-        bs1 = NULL;
-
-        while ((bs1 = bdrv_next(bs1))) {
-            if (bdrv_can_snapshot(bs1) && bs1 != bs) {
-                ret = bdrv_snapshot_find(bs1, sn_info, sn->id_str);
-                if (ret < 0) {
-                    available = 0;
-                    break;
-                }
-            }
-        }
-
-        if (available) {
-            available_snapshots[total] = i;
-            total++;
-        }
-    }
-
-    if (total > 0) {
-        bdrv_snapshot_dump((fprintf_function)monitor_printf, mon, NULL);
-        monitor_printf(mon, "\n");
-        for (i = 0; i < total; i++) {
-            sn = &sn_tab[available_snapshots[i]];
-            bdrv_snapshot_dump((fprintf_function)monitor_printf, mon, sn);
-            monitor_printf(mon, "\n");
-        }
-    } else {
-        monitor_printf(mon, "There is no suitable snapshot available\n");
-    }
-
-    g_free(sn_tab);
-    g_free(available_snapshots);
-
-}
-
-void vmstate_register_ram(MemoryRegion *mr, DeviceState *dev)
-{
-    qemu_ram_set_idstr(memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK,
-                       memory_region_name(mr), dev);
-}
-
-void vmstate_unregister_ram(MemoryRegion *mr, DeviceState *dev)
-{
-    qemu_ram_unset_idstr(memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK);
-}
-
-void vmstate_register_ram_global(MemoryRegion *mr)
-{
-    vmstate_register_ram(mr, NULL);
-}
diff --git a/scripts/acpi_extract.py b/scripts/acpi_extract.py
deleted file mode 100755
index 10c1ffb..0000000
--- a/scripts/acpi_extract.py
+++ /dev/null
@@ -1,367 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, see <http://www.gnu.org/licenses/>.
-
-# Process mixed ASL/AML listing (.lst file) produced by iasl -l
-# Locate and execute ACPI_EXTRACT directives, output offset info
-#
-# Documentation of ACPI_EXTRACT_* directive tags:
-#
-# These directive tags output offset information from AML for BIOS runtime
-# table generation.
-# Each directive is of the form:
-# ACPI_EXTRACT_<TYPE> <array_name> <Operator> (...)
-# and causes the extractor to create an array
-# named <array_name> with offset, in the generated AML,
-# of an object of a given type in the following <Operator>.
-#
-# A directive must fit on a single code line.
-#
-# Object type in AML is verified, a mismatch causes a build failure.
-#
-# Directives and operators currently supported are:
-# ACPI_EXTRACT_NAME_DWORD_CONST - extract a Dword Const object from Name()
-# ACPI_EXTRACT_NAME_WORD_CONST - extract a Word Const object from Name()
-# ACPI_EXTRACT_NAME_BYTE_CONST - extract a Byte Const object from Name()
-# ACPI_EXTRACT_METHOD_STRING - extract a NameString from Method()
-# ACPI_EXTRACT_NAME_STRING - extract a NameString from Name()
-# ACPI_EXTRACT_PROCESSOR_START - start of Processor() block
-# ACPI_EXTRACT_PROCESSOR_STRING - extract a NameString from Processor()
-# ACPI_EXTRACT_PROCESSOR_END - offset at last byte of Processor() + 1
-# ACPI_EXTRACT_PKG_START - start of Package block
-#
-# ACPI_EXTRACT_ALL_CODE - create an array storing the generated AML bytecode
-#
-# ACPI_EXTRACT is not allowed anywhere else in code, except in comments.
-
-import re;
-import sys;
-import fileinput;
-
-aml = []
-asl = []
-output = {}
-debug = ""
-
-class asl_line:
-    line = None
-    lineno = None
-    aml_offset = None
-
-def die(diag):
-    sys.stderr.write("Error: %s; %s\n" % (diag, debug))
-    sys.exit(1)
-
-#Store an ASL command, matching AML offset, and input line (for debugging)
-def add_asl(lineno, line):
-    l = asl_line()
-    l.line = line
-    l.lineno = lineno
-    l.aml_offset = len(aml)
-    asl.append(l)
-
-#Store an AML byte sequence
-#Verify that offset output by iasl matches # of bytes so far
-def add_aml(offset, line):
-    o = int(offset, 16);
-    # Sanity check: offset must match size of code so far
-    if (o != len(aml)):
-        die("Offset 0x%x != 0x%x" % (o, len(aml)))
-    # Strip any trailing dots and ASCII dump after "
-    line = re.sub(r'\s*\.*\s*".*$',"", line)
-    # Strip traling whitespace
-    line = re.sub(r'\s+$',"", line)
-    # Strip leading whitespace
-    line = re.sub(r'^\s+',"", line)
-    # Split on whitespace
-    code = re.split(r'\s+', line)
-    for c in code:
-        # Require a legal hex number, two digits
-        if (not(re.search(r'^[0-9A-Fa-f][0-9A-Fa-f]$', c))):
-            die("Unexpected octet %s" % c);
-        aml.append(int(c, 16));
-
-# Process aml bytecode array, decoding AML
-def aml_pkglen_bytes(offset):
-    # PkgLength can be multibyte. Bits 8-7 give the # of extra bytes.
-    pkglenbytes = aml[offset] >> 6;
-    return pkglenbytes + 1
-
-def aml_pkglen(offset):
-    pkgstart = offset
-    pkglenbytes = aml_pkglen_bytes(offset)
-    pkglen = aml[offset] & 0x3F
-    # If multibyte, first nibble only uses bits 0-3
-    if ((pkglenbytes > 1) and (pkglen & 0x30)):
-        die("PkgLen bytes 0x%x but first nibble 0x%x expected 0x0X" %
-            (pkglen, pkglen))
-    offset += 1
-    pkglenbytes -= 1
-    for i in range(pkglenbytes):
-        pkglen |= aml[offset + i] << (i * 8 + 4)
-    if (len(aml) < pkgstart + pkglen):
-        die("PckgLen 0x%x at offset 0x%x exceeds AML size 0x%x" %
-            (pkglen, offset, len(aml)))
-    return pkglen
-
-# Given method offset, find its NameString offset
-def aml_method_string(offset):
-    #0x14 MethodOp PkgLength NameString MethodFlags TermList
-    if (aml[offset] != 0x14):
-        die( "Method offset 0x%x: expected 0x14 actual 0x%x" %
-             (offset, aml[offset]));
-    offset += 1;
-    pkglenbytes = aml_pkglen_bytes(offset)
-    offset += pkglenbytes;
-    return offset;
-
-# Given name offset, find its NameString offset
-def aml_name_string(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x08):
-        die( "Name offset 0x%x: expected 0x08 actual 0x%x" %
-             (offset, aml[offset]));
-    offset += 1
-    # Block Name Modifier. Skip it.
-    if (aml[offset] == 0x5c or aml[offset] == 0x5e):
-        offset += 1
-    return offset;
-
-# Given data offset, find variable length byte buffer offset
-def aml_data_buffer(offset, length):
-    #0x11 PkgLength BufferSize ByteList
-    if (length > 63):
-        die( "Name offset 0x%x: expected a one byte PkgLength (length<=63)" %
-             (offset));
-    expect = [0x11, length+3, 0x0A, length]
-    if (aml[offset:offset+4] != expect):
-        die( "Name offset 0x%x: expected %s actual %s" %
-             (offset, expect, aml[offset:offset+4]))
-    return offset + len(expect)
-
-# Given data offset, find dword const offset
-def aml_data_dword_const(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x0C):
-        die( "Name offset 0x%x: expected 0x0C actual 0x%x" %
-             (offset, aml[offset]));
-    return offset + 1;
-
-# Given data offset, find word const offset
-def aml_data_word_const(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x0B):
-        die( "Name offset 0x%x: expected 0x0B actual 0x%x" %
-             (offset, aml[offset]));
-    return offset + 1;
-
-# Given data offset, find byte const offset
-def aml_data_byte_const(offset):
-    #0x08 NameOp NameString DataRef
-    if (aml[offset] != 0x0A):
-        die( "Name offset 0x%x: expected 0x0A actual 0x%x" %
-             (offset, aml[offset]));
-    return offset + 1;
-
-# Find name'd buffer
-def aml_name_buffer(offset, length):
-    return aml_data_buffer(aml_name_string(offset) + 4, length)
-
-# Given name offset, find dword const offset
-def aml_name_dword_const(offset):
-    return aml_data_dword_const(aml_name_string(offset) + 4)
-
-# Given name offset, find word const offset
-def aml_name_word_const(offset):
-    return aml_data_word_const(aml_name_string(offset) + 4)
-
-# Given name offset, find byte const offset
-def aml_name_byte_const(offset):
-    return aml_data_byte_const(aml_name_string(offset) + 4)
-
-def aml_device_start(offset):
-    #0x5B 0x82 DeviceOp PkgLength NameString
-    if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x82)):
-        die( "Name offset 0x%x: expected 0x5B 0x82 actual 0x%x 0x%x" %
-             (offset, aml[offset], aml[offset + 1]));
-    return offset
-
-def aml_device_string(offset):
-    #0x5B 0x82 DeviceOp PkgLength NameString
-    start = aml_device_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    offset += pkglenbytes
-    return offset
-
-def aml_device_end(offset):
-    start = aml_device_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    pkglen = aml_pkglen(offset)
-    return offset + pkglen
-
-def aml_processor_start(offset):
-    #0x5B 0x83 ProcessorOp PkgLength NameString ProcID
-    if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x83)):
-        die( "Name offset 0x%x: expected 0x5B 0x83 actual 0x%x 0x%x" %
-             (offset, aml[offset], aml[offset + 1]));
-    return offset
-
-def aml_processor_string(offset):
-    #0x5B 0x83 ProcessorOp PkgLength NameString ProcID
-    start = aml_processor_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    offset += pkglenbytes
-    return offset
-
-def aml_processor_end(offset):
-    start = aml_processor_start(offset)
-    offset += 2
-    pkglenbytes = aml_pkglen_bytes(offset)
-    pkglen = aml_pkglen(offset)
-    return offset + pkglen
-
-def aml_package_start(offset):
-    offset = aml_name_string(offset) + 4
-    # 0x12 PkgLength NumElements PackageElementList
-    if (aml[offset] != 0x12):
-        die( "Name offset 0x%x: expected 0x12 actual 0x%x" %
-             (offset, aml[offset]));
-    offset += 1
-    return offset + aml_pkglen_bytes(offset) + 1
-
-lineno = 0
-for line in fileinput.input():
-    # Strip trailing newline
-    line = line.rstrip();
-    # line number and debug string to output in case of errors
-    lineno = lineno + 1
-    debug = "input line %d: %s" % (lineno, line)
-    #ASL listing: space, then line#, then ...., then code
-    pasl = re.compile('^\s+([0-9]+)(:\s\s|\.\.\.\.)\s*')
-    m = pasl.search(line)
-    if (m):
-        add_asl(lineno, pasl.sub("", line));
-    # AML listing: offset in hex, then ...., then code
-    paml = re.compile('^([0-9A-Fa-f]+)(:\s\s|\.\.\.\.)\s*')
-    m = paml.search(line)
-    if (m):
-        add_aml(m.group(1), paml.sub("", line))
-
-# Now go over code
-# Track AML offset of a previous non-empty ASL command
-prev_aml_offset = -1
-for i in range(len(asl)):
-    debug = "input line %d: %s" % (asl[i].lineno, asl[i].line)
-
-    l = asl[i].line
-
-    # skip if not an extract directive
-    a = len(re.findall(r'ACPI_EXTRACT', l))
-    if (not a):
-        # If not empty, store AML offset. Will be used for sanity checks
-        # IASL seems to put {}. at random places in the listing.
-        # Ignore any non-words for the purpose of this test.
-        m = re.search(r'\w+', l)
-        if (m):
-                prev_aml_offset = asl[i].aml_offset
-        continue
-
-    if (a > 1):
-        die("Expected at most one ACPI_EXTRACT per line, actual %d" % a)
-
-    mext = re.search(r'''
-                      ^\s* # leading whitespace
-                      /\*\s* # start C comment
-                      (ACPI_EXTRACT_\w+) # directive: group(1)
-                      \s+ # whitspace separates directive from array name
-                      (\w+) # array name: group(2)
-                      \s*\*/ # end of C comment
-                      \s*$ # trailing whitespace
-                      ''', l, re.VERBOSE)
-    if (not mext):
-        die("Stray ACPI_EXTRACT in input")
-
-    # previous command must have produced some AML,
-    # otherwise we are in a middle of a block
-    if (prev_aml_offset == asl[i].aml_offset):
-        die("ACPI_EXTRACT directive in the middle of a block")
-
-    directive = mext.group(1)
-    array = mext.group(2)
-    offset = asl[i].aml_offset
-
-    if (directive == "ACPI_EXTRACT_ALL_CODE"):
-        if array in output:
-            die("%s directive used more than once" % directive)
-        output[array] = aml
-        continue
-    if (directive == "ACPI_EXTRACT_NAME_BUFFER8"):
-        offset = aml_name_buffer(offset, 8)
-    elif (directive == "ACPI_EXTRACT_NAME_BUFFER16"):
-        offset = aml_name_buffer(offset, 16)
-    elif (directive == "ACPI_EXTRACT_NAME_DWORD_CONST"):
-        offset = aml_name_dword_const(offset)
-    elif (directive == "ACPI_EXTRACT_NAME_WORD_CONST"):
-        offset = aml_name_word_const(offset)
-    elif (directive == "ACPI_EXTRACT_NAME_BYTE_CONST"):
-        offset = aml_name_byte_const(offset)
-    elif (directive == "ACPI_EXTRACT_NAME_STRING"):
-        offset = aml_name_string(offset)
-    elif (directive == "ACPI_EXTRACT_METHOD_STRING"):
-        offset = aml_method_string(offset)
-    elif (directive == "ACPI_EXTRACT_DEVICE_START"):
-        offset = aml_device_start(offset)
-    elif (directive == "ACPI_EXTRACT_DEVICE_STRING"):
-        offset = aml_device_string(offset)
-    elif (directive == "ACPI_EXTRACT_DEVICE_END"):
-        offset = aml_device_end(offset)
-    elif (directive == "ACPI_EXTRACT_PROCESSOR_START"):
-        offset = aml_processor_start(offset)
-    elif (directive == "ACPI_EXTRACT_PROCESSOR_STRING"):
-        offset = aml_processor_string(offset)
-    elif (directive == "ACPI_EXTRACT_PROCESSOR_END"):
-        offset = aml_processor_end(offset)
-    elif (directive == "ACPI_EXTRACT_PKG_START"):
-        offset = aml_package_start(offset)
-    else:
-        die("Unsupported directive %s" % directive)
-
-    if array not in output:
-        output[array] = []
-    output[array].append(offset)
-
-debug = "at end of file"
-
-def get_value_type(maxvalue):
-    #Use type large enough to fit the table
-    if (maxvalue >= 0x10000):
-            return "int"
-    elif (maxvalue >= 0x100):
-            return "short"
-    else:
-            return "char"
-
-# Pretty print output
-for array in output.keys():
-    otype = get_value_type(max(output[array]))
-    odata = []
-    for value in output[array]:
-        odata.append("0x%x" % value)
-    sys.stdout.write("static unsigned %s %s[] = {\n" % (otype, array))
-    sys.stdout.write(",\n".join(odata))
-    sys.stdout.write('\n};\n');
diff --git a/scripts/acpi_extract_preprocess.py b/scripts/acpi_extract_preprocess.py
deleted file mode 100755
index 69d10d6..0000000
--- a/scripts/acpi_extract_preprocess.py
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com>
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, see <http://www.gnu.org/licenses/>.
-
-# Read a preprocessed ASL listing and put each ACPI_EXTRACT
-# directive in a comment, to make iasl skip it.
-# We also put each directive on a new line, the machinery
-# in tools/acpi_extract.py requires this.
-
-import re;
-import sys;
-import fileinput;
-
-def die(diag):
-    sys.stderr.write("Error: %s\n" % (diag))
-    sys.exit(1)
-
-# Note: () around pattern make split return matched string as part of list
-psplit = re.compile(r''' (
-                          \b # At word boundary
-                          ACPI_EXTRACT_\w+ # directive
-                          \s+ # some whitespace
-                          \w+ # array name
-                         )''', re.VERBOSE);
-
-lineno = 0
-for line in fileinput.input():
-    # line number and debug string to output in case of errors
-    lineno = lineno + 1
-    debug = "input line %d: %s" % (lineno, line.rstrip())
-
-    s = psplit.split(line);
-    # The way split works, each odd item is the matching ACPI_EXTRACT directive.
-    # Put each in a comment, and on a line by itself.
-    for i in range(len(s)):
-        if (i % 2):
-            sys.stdout.write("\n/* %s */\n" % s[i])
-        else:
-            sys.stdout.write(s[i])
diff --git a/scripts/analyze-inclusions b/scripts/analyze-inclusions
new file mode 100644
index 0000000..a8108d9
--- /dev/null
+++ b/scripts/analyze-inclusions
@@ -0,0 +1,102 @@
+#! /bin/sh
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# Author: Paolo Bonzini <pbonzini@redhat.com>
+#
+# Print statistics about header file inclusions.
+#
+# The script has two modes of execution:
+#
+# 1) if invoked with a path on the command line (possibly
+# preceded by a "--" argument), it will run the analysis on
+# an existing build directory
+#
+# 2) otherwise, it will configure and builds QEMU itself in a
+# "+build" subdirectory which is left around when the script
+# exits.  In this case the command line is passed directly to
+# "make" (typically used for a "-j" argument suitable for your
+# system).
+#
+# Inspired by a post by Markus Armbruster.
+
+case "x$1" in
+x--)
+  shift
+  cd "$1" || exit $?
+  ;;
+x-* | x)
+  mkdir -p +build
+  cd +build
+  test -f Makefile && make distclean
+  ../configure
+  make "$@"
+  ;;
+*)
+  cd "$1" || exit $?
+esac
+
+QEMU_CFLAGS=$(sed -n s/^QEMU_CFLAGS=//p config-host.mak)
+QEMU_INCLUDES=$(sed -n s/^QEMU_INCLUDES=//p config-host.mak | \
+    sed 's/$(SRC_PATH)/../g' )
+CFLAGS=$(sed -n s/^CFLAGS=//p config-host.mak)
+
+grep_include() {
+  find . -name "*.d" -exec grep -l "$@" {} + | wc -l
+}
+
+echo Found $(find . -name "*.d" | wc -l) object files
+echo $(grep_include -F 'include/qemu-common.h') files include qemu-common.h
+echo $(grep_include -F 'hw/hw.h') files include hw/hw.h
+echo $(grep_include 'target-[a-z0-9]*/cpu\.h') files include cpu.h
+echo $(grep_include -F 'qapi-types.h') files include qapi-types.h
+echo $(grep_include -F 'trace/generated-tracers.h') files include generated-tracers.h
+echo $(grep_include -F 'qapi/error.h') files include qapi/error.h
+echo $(grep_include -F 'qom/object.h') files include qom/object.h
+echo $(grep_include -F 'block/aio.h') files include block/aio.h
+echo $(grep_include -F 'exec/memory.h') files include exec/memory.h
+echo $(grep_include -F 'fpu/softfloat.h') files include fpu/softfloat.h
+echo $(grep_include -F 'qemu/bswap.h') files include qemu/bswap.h
+echo
+
+awk1='
+    /^# / { file = $3;next }
+    NR>1 { bytes[file]+=length()+1; lines[file]++ }
+    END { for(i in lines) print i,lines[i],bytes[i] }'
+
+awk2='
+    {tot_l+=$2;tot_b+=$3;tot_f++}
+    /\/usr.*\/glib/ {glib_l+=$2;glib_b+=$3;glib_f++;next}
+    /\/usr/ {sys_l+=$2;sys_b+=$3;sys_f++;next}
+    {qemu_l+=$2;qemu_b+=$3;qemu_f++;next}
+    END {
+      printf "%s\t %s\t %s\t %s\n", "lines", "bytes", "files", "source"
+      printf "%s\t %s\t %s\t %s\n", qemu_l, qemu_b, qemu_f, "QEMU"
+      printf "%s\t %s\t %s\t %s\n", sys_l, sys_b, sys_f, "system"
+      printf "%s\t %s\t %s\t %s\n", glib_l, glib_b, glib_f, "glib"
+      printf "%s\t %s\t %s\t %s\n", tot_l, tot_b, tot_f, "total"
+    }'
+
+analyze() {
+  cc $QEMU_CFLAGS $QEMU_INCLUDES $CFLAGS  -E -o - "$@" | \
+    awk "$awk1" | awk "$awk2"
+  echo
+}
+
+echo osdep.h:
+analyze ../include/qemu/osdep.h
+
+echo qemu-common.h:
+analyze  -include ../include/qemu/osdep.h ../include/qemu-common.h
+
+echo hw/hw.h:
+analyze -include ../include/qemu/osdep.h ../include/hw/hw.h
+
+echo trace/generated-tracers.h:
+analyze -include ../include/qemu/osdep.h trace/generated-tracers.h
+
+echo target-i386/cpu.h:
+analyze -DNEED_CPU_H -I../target-i386 -Ii386-softmmu -include ../include/qemu/osdep.h ../target-i386/cpu.h
+
+echo hw/hw.h + NEED_CPU_H:
+analyze -DNEED_CPU_H -I../target-i386 -Ii386-softmmu -include ../include/qemu/osdep.h ../include/hw/hw.h
diff --git a/scripts/analyze-migration.py b/scripts/analyze-migration.py
new file mode 100755
index 0000000..1455387
--- /dev/null
+++ b/scripts/analyze-migration.py
@@ -0,0 +1,610 @@
+#!/usr/bin/env python
+#
+#  Migration Stream Analyzer
+#
+#  Copyright (c) 2015 Alexander Graf <agraf@suse.de>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+
+import numpy as np
+import json
+import os
+import argparse
+import collections
+import pprint
+
+def mkdir_p(path):
+    try:
+        os.makedirs(path)
+    except OSError:
+        pass
+
+class MigrationFile(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self.file = open(self.filename, "rb")
+
+    def read64(self):
+        return np.asscalar(np.fromfile(self.file, count=1, dtype='>i8')[0])
+
+    def read32(self):
+        return np.asscalar(np.fromfile(self.file, count=1, dtype='>i4')[0])
+
+    def read16(self):
+        return np.asscalar(np.fromfile(self.file, count=1, dtype='>i2')[0])
+
+    def read8(self):
+        return np.asscalar(np.fromfile(self.file, count=1, dtype='>i1')[0])
+
+    def readstr(self, len = None):
+        if len is None:
+            len = self.read8()
+        if len == 0:
+            return ""
+        return np.fromfile(self.file, count=1, dtype=('S%d' % len))[0]
+
+    def readvar(self, size = None):
+        if size is None:
+            size = self.read8()
+        if size == 0:
+            return ""
+        value = self.file.read(size)
+        if len(value) != size:
+            raise Exception("Unexpected end of %s at 0x%x" % (self.filename, self.file.tell()))
+        return value
+
+    def tell(self):
+        return self.file.tell()
+
+    # The VMSD description is at the end of the file, after EOF. Look for
+    # the last NULL byte, then for the beginning brace of JSON.
+    def read_migration_debug_json(self):
+        QEMU_VM_VMDESCRIPTION = 0x06
+
+        # Remember the offset in the file when we started
+        entrypos = self.file.tell()
+
+        # Read the last 10MB
+        self.file.seek(0, os.SEEK_END)
+        endpos = self.file.tell()
+        self.file.seek(max(-endpos, -10 * 1024 * 1024), os.SEEK_END)
+        datapos = self.file.tell()
+        data = self.file.read()
+        # The full file read closed the file as well, reopen it
+        self.file = open(self.filename, "rb")
+
+        # Find the last NULL byte, then the first brace after that. This should
+        # be the beginning of our JSON data.
+        nulpos = data.rfind("\0")
+        jsonpos = data.find("{", nulpos)
+
+        # Check backwards from there and see whether we guessed right
+        self.file.seek(datapos + jsonpos - 5, 0)
+        if self.read8() != QEMU_VM_VMDESCRIPTION:
+            raise Exception("No Debug Migration device found")
+
+        jsonlen = self.read32()
+
+        # Seek back to where we were at the beginning
+        self.file.seek(entrypos, 0)
+
+        return data[jsonpos:jsonpos + jsonlen]
+
+    def close(self):
+        self.file.close()
+
+class RamSection(object):
+    RAM_SAVE_FLAG_COMPRESS = 0x02
+    RAM_SAVE_FLAG_MEM_SIZE = 0x04
+    RAM_SAVE_FLAG_PAGE     = 0x08
+    RAM_SAVE_FLAG_EOS      = 0x10
+    RAM_SAVE_FLAG_CONTINUE = 0x20
+    RAM_SAVE_FLAG_XBZRLE   = 0x40
+    RAM_SAVE_FLAG_HOOK     = 0x80
+
+    def __init__(self, file, version_id, ramargs, section_key):
+        if version_id != 4:
+            raise Exception("Unknown RAM version %d" % version_id)
+
+        self.file = file
+        self.section_key = section_key
+        self.TARGET_PAGE_SIZE = ramargs['page_size']
+        self.dump_memory = ramargs['dump_memory']
+        self.write_memory = ramargs['write_memory']
+        self.sizeinfo = collections.OrderedDict()
+        self.data = collections.OrderedDict()
+        self.data['section sizes'] = self.sizeinfo
+        self.name = ''
+        if self.write_memory:
+            self.files = { }
+        if self.dump_memory:
+            self.memory = collections.OrderedDict()
+            self.data['memory'] = self.memory
+
+    def __repr__(self):
+        return self.data.__repr__()
+
+    def __str__(self):
+        return self.data.__str__()
+
+    def getDict(self):
+        return self.data
+
+    def read(self):
+        # Read all RAM sections
+        while True:
+            addr = self.file.read64()
+            flags = addr & (self.TARGET_PAGE_SIZE - 1)
+            addr &= ~(self.TARGET_PAGE_SIZE - 1)
+
+            if flags & self.RAM_SAVE_FLAG_MEM_SIZE:
+                while True:
+                    namelen = self.file.read8()
+                    # We assume that no RAM chunk is big enough to ever
+                    # hit the first byte of the address, so when we see
+                    # a zero here we know it has to be an address, not the
+                    # length of the next block.
+                    if namelen == 0:
+                        self.file.file.seek(-1, 1)
+                        break
+                    self.name = self.file.readstr(len = namelen)
+                    len = self.file.read64()
+                    self.sizeinfo[self.name] = '0x%016x' % len
+                    if self.write_memory:
+                        print self.name
+                        mkdir_p('./' + os.path.dirname(self.name))
+                        f = open('./' + self.name, "wb")
+                        f.truncate(0)
+                        f.truncate(len)
+                        self.files[self.name] = f
+                flags &= ~self.RAM_SAVE_FLAG_MEM_SIZE
+
+            if flags & self.RAM_SAVE_FLAG_COMPRESS:
+                if flags & self.RAM_SAVE_FLAG_CONTINUE:
+                    flags &= ~self.RAM_SAVE_FLAG_CONTINUE
+                else:
+                    self.name = self.file.readstr()
+                fill_char = self.file.read8()
+                # The page in question is filled with fill_char now
+                if self.write_memory and fill_char != 0:
+                    self.files[self.name].seek(addr, os.SEEK_SET)
+                    self.files[self.name].write(chr(fill_char) * self.TARGET_PAGE_SIZE)
+                if self.dump_memory:
+                    self.memory['%s (0x%016x)' % (self.name, addr)] = 'Filled with 0x%02x' % fill_char
+                flags &= ~self.RAM_SAVE_FLAG_COMPRESS
+            elif flags & self.RAM_SAVE_FLAG_PAGE:
+                if flags & self.RAM_SAVE_FLAG_CONTINUE:
+                    flags &= ~self.RAM_SAVE_FLAG_CONTINUE
+                else:
+                    self.name = self.file.readstr()
+
+                if self.write_memory or self.dump_memory:
+                    data = self.file.readvar(size = self.TARGET_PAGE_SIZE)
+                else: # Just skip RAM data
+                    self.file.file.seek(self.TARGET_PAGE_SIZE, 1)
+
+                if self.write_memory:
+                    self.files[self.name].seek(addr, os.SEEK_SET)
+                    self.files[self.name].write(data)
+                if self.dump_memory:
+                    hexdata = " ".join("{0:02x}".format(ord(c)) for c in data)
+                    self.memory['%s (0x%016x)' % (self.name, addr)] = hexdata
+
+                flags &= ~self.RAM_SAVE_FLAG_PAGE
+            elif flags & self.RAM_SAVE_FLAG_XBZRLE:
+                raise Exception("XBZRLE RAM compression is not supported yet")
+            elif flags & self.RAM_SAVE_FLAG_HOOK:
+                raise Exception("RAM hooks don't make sense with files")
+
+            # End of RAM section
+            if flags & self.RAM_SAVE_FLAG_EOS:
+                break
+
+            if flags != 0:
+                raise Exception("Unknown RAM flags: %x" % flags)
+
+    def __del__(self):
+        if self.write_memory:
+            for key in self.files:
+                self.files[key].close()
+
+
+class HTABSection(object):
+    HASH_PTE_SIZE_64       = 16
+
+    def __init__(self, file, version_id, device, section_key):
+        if version_id != 1:
+            raise Exception("Unknown HTAB version %d" % version_id)
+
+        self.file = file
+        self.section_key = section_key
+
+    def read(self):
+
+        header = self.file.read32()
+
+        if (header > 0):
+            # First section, just the hash shift
+            return
+
+        # Read until end marker
+        while True:
+            index = self.file.read32()
+            n_valid = self.file.read16()
+            n_invalid = self.file.read16()
+
+            if index == 0 and n_valid == 0 and n_invalid == 0:
+                break
+
+            self.file.readvar(n_valid * self.HASH_PTE_SIZE_64)
+
+    def getDict(self):
+        return ""
+
+
+class ConfigurationSection(object):
+    def __init__(self, file):
+        self.file = file
+
+    def read(self):
+        name_len = self.file.read32()
+        name = self.file.readstr(len = name_len)
+
+class VMSDFieldGeneric(object):
+    def __init__(self, desc, file):
+        self.file = file
+        self.desc = desc
+        self.data = ""
+
+    def __repr__(self):
+        return str(self.__str__())
+
+    def __str__(self):
+        return " ".join("{0:02x}".format(ord(c)) for c in self.data)
+
+    def getDict(self):
+        return self.__str__()
+
+    def read(self):
+        size = int(self.desc['size'])
+        self.data = self.file.readvar(size)
+        return self.data
+
+class VMSDFieldInt(VMSDFieldGeneric):
+    def __init__(self, desc, file):
+        super(VMSDFieldInt, self).__init__(desc, file)
+        self.size = int(desc['size'])
+        self.format = '0x%%0%dx' % (self.size * 2)
+        self.sdtype = '>i%d' % self.size
+        self.udtype = '>u%d' % self.size
+
+    def __repr__(self):
+        if self.data < 0:
+            return ('%s (%d)' % ((self.format % self.udata), self.data))
+        else:
+            return self.format % self.data
+
+    def __str__(self):
+        return self.__repr__()
+
+    def getDict(self):
+        return self.__str__()
+
+    def read(self):
+        super(VMSDFieldInt, self).read()
+        self.sdata = np.fromstring(self.data, count=1, dtype=(self.sdtype))[0]
+        self.udata = np.fromstring(self.data, count=1, dtype=(self.udtype))[0]
+        self.data = self.sdata
+        return self.data
+
+class VMSDFieldUInt(VMSDFieldInt):
+    def __init__(self, desc, file):
+        super(VMSDFieldUInt, self).__init__(desc, file)
+
+    def read(self):
+        super(VMSDFieldUInt, self).read()
+        self.data = self.udata
+        return self.data
+
+class VMSDFieldIntLE(VMSDFieldInt):
+    def __init__(self, desc, file):
+        super(VMSDFieldIntLE, self).__init__(desc, file)
+        self.dtype = '<i%d' % self.size
+
+class VMSDFieldBool(VMSDFieldGeneric):
+    def __init__(self, desc, file):
+        super(VMSDFieldBool, self).__init__(desc, file)
+
+    def __repr__(self):
+        return self.data.__repr__()
+
+    def __str__(self):
+        return self.data.__str__()
+
+    def getDict(self):
+        return self.data
+
+    def read(self):
+        super(VMSDFieldBool, self).read()
+        if self.data[0] == 0:
+            self.data = False
+        else:
+            self.data = True
+        return self.data
+
+class VMSDFieldStruct(VMSDFieldGeneric):
+    QEMU_VM_SUBSECTION    = 0x05
+
+    def __init__(self, desc, file):
+        super(VMSDFieldStruct, self).__init__(desc, file)
+        self.data = collections.OrderedDict()
+
+        # When we see compressed array elements, unfold them here
+        new_fields = []
+        for field in self.desc['struct']['fields']:
+            if not 'array_len' in field:
+                new_fields.append(field)
+                continue
+            array_len = field.pop('array_len')
+            field['index'] = 0
+            new_fields.append(field)
+            for i in xrange(1, array_len):
+                c = field.copy()
+                c['index'] = i
+                new_fields.append(c)
+
+        self.desc['struct']['fields'] = new_fields
+
+    def __repr__(self):
+        return self.data.__repr__()
+
+    def __str__(self):
+        return self.data.__str__()
+
+    def read(self):
+        for field in self.desc['struct']['fields']:
+            try:
+                reader = vmsd_field_readers[field['type']]
+            except:
+                reader = VMSDFieldGeneric
+
+            field['data'] = reader(field, self.file)
+            field['data'].read()
+
+            if 'index' in field:
+                if field['name'] not in self.data:
+                    self.data[field['name']] = []
+                a = self.data[field['name']]
+                if len(a) != int(field['index']):
+                    raise Exception("internal index of data field unmatched (%d/%d)" % (len(a), int(field['index'])))
+                a.append(field['data'])
+            else:
+                self.data[field['name']] = field['data']
+
+        if 'subsections' in self.desc['struct']:
+            for subsection in self.desc['struct']['subsections']:
+                if self.file.read8() != self.QEMU_VM_SUBSECTION:
+                    raise Exception("Subsection %s not found at offset %x" % ( subsection['vmsd_name'], self.file.tell()))
+                name = self.file.readstr()
+                version_id = self.file.read32()
+                self.data[name] = VMSDSection(self.file, version_id, subsection, (name, 0))
+                self.data[name].read()
+
+    def getDictItem(self, value):
+       # Strings would fall into the array category, treat
+       # them specially
+       if value.__class__ is ''.__class__:
+           return value
+
+       try:
+           return self.getDictOrderedDict(value)
+       except:
+           try:
+               return self.getDictArray(value)
+           except:
+               try:
+                   return value.getDict()
+               except:
+                   return value
+
+    def getDictArray(self, array):
+        r = []
+        for value in array:
+           r.append(self.getDictItem(value))
+        return r
+
+    def getDictOrderedDict(self, dict):
+        r = collections.OrderedDict()
+        for (key, value) in dict.items():
+            r[key] = self.getDictItem(value)
+        return r
+
+    def getDict(self):
+        return self.getDictOrderedDict(self.data)
+
+vmsd_field_readers = {
+    "bool" : VMSDFieldBool,
+    "int8" : VMSDFieldInt,
+    "int16" : VMSDFieldInt,
+    "int32" : VMSDFieldInt,
+    "int32 equal" : VMSDFieldInt,
+    "int32 le" : VMSDFieldIntLE,
+    "int64" : VMSDFieldInt,
+    "uint8" : VMSDFieldUInt,
+    "uint16" : VMSDFieldUInt,
+    "uint32" : VMSDFieldUInt,
+    "uint32 equal" : VMSDFieldUInt,
+    "uint64" : VMSDFieldUInt,
+    "int64 equal" : VMSDFieldInt,
+    "uint8 equal" : VMSDFieldInt,
+    "uint16 equal" : VMSDFieldInt,
+    "float64" : VMSDFieldGeneric,
+    "timer" : VMSDFieldGeneric,
+    "buffer" : VMSDFieldGeneric,
+    "unused_buffer" : VMSDFieldGeneric,
+    "bitmap" : VMSDFieldGeneric,
+    "struct" : VMSDFieldStruct,
+    "unknown" : VMSDFieldGeneric,
+}
+
+class VMSDSection(VMSDFieldStruct):
+    def __init__(self, file, version_id, device, section_key):
+        self.file = file
+        self.data = ""
+        self.vmsd_name = ""
+        self.section_key = section_key
+        desc = device
+        if 'vmsd_name' in device:
+            self.vmsd_name = device['vmsd_name']
+
+        # A section really is nothing but a FieldStruct :)
+        super(VMSDSection, self).__init__({ 'struct' : desc }, file)
+
+###############################################################################
+
+class MigrationDump(object):
+    QEMU_VM_FILE_MAGIC    = 0x5145564d
+    QEMU_VM_FILE_VERSION  = 0x00000003
+    QEMU_VM_EOF           = 0x00
+    QEMU_VM_SECTION_START = 0x01
+    QEMU_VM_SECTION_PART  = 0x02
+    QEMU_VM_SECTION_END   = 0x03
+    QEMU_VM_SECTION_FULL  = 0x04
+    QEMU_VM_SUBSECTION    = 0x05
+    QEMU_VM_VMDESCRIPTION = 0x06
+    QEMU_VM_CONFIGURATION = 0x07
+    QEMU_VM_SECTION_FOOTER= 0x7e
+
+    def __init__(self, filename):
+        self.section_classes = { ( 'ram', 0 ) : [ RamSection, None ],
+                                 ( 'spapr/htab', 0) : ( HTABSection, None ) }
+        self.filename = filename
+        self.vmsd_desc = None
+
+    def read(self, desc_only = False, dump_memory = False, write_memory = False):
+        # Read in the whole file
+        file = MigrationFile(self.filename)
+
+        # File magic
+        data = file.read32()
+        if data != self.QEMU_VM_FILE_MAGIC:
+            raise Exception("Invalid file magic %x" % data)
+
+        # Version (has to be v3)
+        data = file.read32()
+        if data != self.QEMU_VM_FILE_VERSION:
+            raise Exception("Invalid version number %d" % data)
+
+        self.load_vmsd_json(file)
+
+        # Read sections
+        self.sections = collections.OrderedDict()
+
+        if desc_only:
+            return
+
+        ramargs = {}
+        ramargs['page_size'] = self.vmsd_desc['page_size']
+        ramargs['dump_memory'] = dump_memory
+        ramargs['write_memory'] = write_memory
+        self.section_classes[('ram',0)][1] = ramargs
+
+        while True:
+            section_type = file.read8()
+            if section_type == self.QEMU_VM_EOF:
+                break
+            elif section_type == self.QEMU_VM_CONFIGURATION:
+                section = ConfigurationSection(file)
+                section.read()
+            elif section_type == self.QEMU_VM_SECTION_START or section_type == self.QEMU_VM_SECTION_FULL:
+                section_id = file.read32()
+                name = file.readstr()
+                instance_id = file.read32()
+                version_id = file.read32()
+                section_key = (name, instance_id)
+                classdesc = self.section_classes[section_key]
+                section = classdesc[0](file, version_id, classdesc[1], section_key)
+                self.sections[section_id] = section
+                section.read()
+            elif section_type == self.QEMU_VM_SECTION_PART or section_type == self.QEMU_VM_SECTION_END:
+                section_id = file.read32()
+                self.sections[section_id].read()
+            elif section_type == self.QEMU_VM_SECTION_FOOTER:
+                read_section_id = file.read32()
+                if read_section_id != section_id:
+                    raise Exception("Mismatched section footer: %x vs %x" % (read_section_id, section_id))
+            else:
+                raise Exception("Unknown section type: %d" % section_type)
+        file.close()
+
+    def load_vmsd_json(self, file):
+        vmsd_json = file.read_migration_debug_json()
+        self.vmsd_desc = json.loads(vmsd_json, object_pairs_hook=collections.OrderedDict)
+        for device in self.vmsd_desc['devices']:
+            key = (device['name'], device['instance_id'])
+            value = ( VMSDSection, device )
+            self.section_classes[key] = value
+
+    def getDict(self):
+        r = collections.OrderedDict()
+        for (key, value) in self.sections.items():
+           key = "%s (%d)" % ( value.section_key[0], key )
+           r[key] = value.getDict()
+        return r
+
+###############################################################################
+
+class JSONEncoder(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, VMSDFieldGeneric):
+            return str(o)
+        return json.JSONEncoder.default(self, o)
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-f", "--file", help='migration dump to read from', required=True)
+parser.add_argument("-m", "--memory", help='dump RAM contents as well', action='store_true')
+parser.add_argument("-d", "--dump", help='what to dump ("state" or "desc")', default='state')
+parser.add_argument("-x", "--extract", help='extract contents into individual files', action='store_true')
+args = parser.parse_args()
+
+jsonenc = JSONEncoder(indent=4, separators=(',', ': '))
+
+if args.extract:
+    dump = MigrationDump(args.file)
+
+    dump.read(desc_only = True)
+    print "desc.json"
+    f = open("desc.json", "wb")
+    f.truncate()
+    f.write(jsonenc.encode(dump.vmsd_desc))
+    f.close()
+
+    dump.read(write_memory = True)
+    dict = dump.getDict()
+    print "state.json"
+    f = open("state.json", "wb")
+    f.truncate()
+    f.write(jsonenc.encode(dict))
+    f.close()
+elif args.dump == "state":
+    dump = MigrationDump(args.file)
+    dump.read(dump_memory = args.memory)
+    dict = dump.getDict()
+    print jsonenc.encode(dict)
+elif args.dump == "desc":
+    dump = MigrationDump(args.file)
+    dump.read(desc_only = True)
+    print jsonenc.encode(dump.vmsd_desc)
+else:
+    raise Exception("Please specify either -x, -d state or -d dump")
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 053e432..b0096a4 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -22,7 +22,7 @@
 my $emacs = 0;
 my $terse = 0;
 my $file = 0;
-my $check = 0;
+my $no_warnings = 0;
 my $summary = 1;
 my $mailback = 0;
 my $summary_file = 0;
@@ -45,7 +45,7 @@
   --emacs                    emacs compile window format
   --terse                    one line per report
   -f, --file                 treat FILE as regular source file
-  --subjective, --strict     enable more subjective tests
+  --strict                   fail if only warnings are found
   --root=PATH                PATH to the kernel tree root
   --no-summary               suppress the per-file summary
   --mailback                 only produce a report in case of warnings/errors
@@ -71,8 +71,7 @@
 	'emacs!'	=> \$emacs,
 	'terse!'	=> \$terse,
 	'f|file!'	=> \$file,
-	'subjective!'	=> \$check,
-	'strict!'	=> \$check,
+	'strict!'	=> \$no_warnings,
 	'root=s'	=> \$root,
 	'summary!'	=> \$summary,
 	'mailback!'	=> \$mailback,
@@ -141,44 +140,22 @@
 		}x;
 our $Storage	= qr{extern|static|asmlinkage};
 our $Sparse	= qr{
-			__user|
-			__kernel|
-			__force|
-			__iomem|
-			__must_check|
-			__init_refok|
-			__kprobes|
-			__ref
+			__force
 		}x;
 
 # Notes to $Attribute:
-# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check
 our $Attribute	= qr{
 			const|
-			__percpu|
-			__nocast|
-			__safe|
-			__bitwise__|
-			__packed__|
-			__packed2__|
-			__naked|
-			__maybe_unused|
-			__always_unused|
-			__noreturn|
-			__used|
-			__cold|
-			__noclone|
-			__deprecated|
-			__read_mostly|
-			__kprobes|
-			__(?:mem|cpu|dev|)(?:initdata|initconst|init\b)|
-			____cacheline_aligned|
-			____cacheline_aligned_in_smp|
-			____cacheline_internodealigned_in_smp|
-			__weak
+			volatile|
+			QEMU_NORETURN|
+			QEMU_WARN_UNUSED_RESULT|
+			QEMU_SENTINEL|
+			QEMU_ARTIFICIAL|
+			QEMU_PACKED|
+			GCC_FMT_ATTR
 		  }x;
 our $Modifier;
-our $Inline	= qr{inline|__always_inline|noinline};
+our $Inline	= qr{inline};
 our $Member	= qr{->$Ident|\.$Ident|\[[^]]*\]};
 our $Lval	= qr{$Ident(?:$Member)*};
 
@@ -215,14 +192,6 @@
         | QEMUBH                            # all uppercase
 )};
 
-our $logFunctions = qr{(?x:
-	printk|
-	pr_(debug|dbg|vdbg|devel|info|warning|err|notice|alert|crit|emerg|cont)|
-	(dev|netdev|netif)_(printk|dbg|vdbg|info|warn|err|notice|alert|crit|emerg|WARN)|
-	WARN|
-	panic
-)};
-
 our @typeList = (
 	qr{void},
 	qr{(?:unsigned\s+)?char},
@@ -242,21 +211,22 @@
 	qr{${Ident}_t},
 	qr{${Ident}_handler},
 	qr{${Ident}_handler_fn},
-);
-our @modifierList = (
-	qr{fastcall},
+	qr{target_(?:u)?long},
 );
 
-our $allowed_asm_includes = qr{(?x:
-	irq|
-	memory
-)};
-# memory.h: ARM has a custom one
+# This can be modified by sub possible.  Since it can be empty, be careful
+# about regexes that always match, because they can cause infinite loops.
+our @modifierList = (
+);
 
 sub build_types {
-	my $mods = "(?x:  \n" . join("|\n  ", @modifierList) . "\n)";
 	my $all = "(?x:  \n" . join("|\n  ", @typeList) . "\n)";
-	$Modifier	= qr{(?:$Attribute|$Sparse|$mods)};
+	if (@modifierList > 0) {
+		my $mods = "(?x:  \n" . join("|\n  ", @modifierList) . "\n)";
+		$Modifier = qr{(?:$Attribute|$Sparse|$mods)};
+	} else {
+		$Modifier = qr{(?:$Attribute|$Sparse)};
+	}
 	$NonptrType	= qr{
 			(?:$Modifier\s+|const\s+)*
 			(?:
@@ -277,27 +247,6 @@
 
 $chk_signoff = 0 if ($file);
 
-my @dep_includes = ();
-my @dep_functions = ();
-my $removal = "Documentation/feature-removal-schedule.txt";
-if ($tree && -f "$root/$removal") {
-	open(my $REMOVE, '<', "$root/$removal") ||
-				die "$P: $removal: open failed - $!\n";
-	while (<$REMOVE>) {
-		if (/^Check:\s+(.*\S)/) {
-			for my $entry (split(/[, ]+/, $1)) {
-				if ($entry =~ m@include/(.*)@) {
-					push(@dep_includes, $1);
-
-				} elsif ($entry !~ m@/@) {
-					push(@dep_functions, $entry);
-				}
-			}
-		}
-	}
-	close($REMOVE);
-}
-
 my @rawlines = ();
 my @lines = ();
 my $vname;
@@ -413,7 +362,7 @@
 	for ($off = 1; $off < length($line); $off++) {
 		$c = substr($line, $off, 1);
 
-		# Comments we are wacking completly including the begin
+		# Comments we are wacking completely including the begin
 		# and end, all to $;.
 		if ($sanitise_quote eq '' && substr($line, $off, 2) eq '/*') {
 			$sanitise_quote = '*/';
@@ -633,7 +582,7 @@
 	my ($stmt) = @_;
 
 	$stmt =~ s/(^|\n)./$1/g;
-	$stmt =~ s/^\s*{//;
+	$stmt =~ s/^\s*\{//;
 	$stmt =~ s/}\s*$//;
 	$stmt =~ s/^\s*//;
 	$stmt =~ s/\s*$//;
@@ -1061,7 +1010,9 @@
 			case|
 			else|
 			asm|__asm__|
-			do
+			do|
+			\#|
+			\#\#
 		)(?:\s|$)|
 		^(?:typedef|struct|enum)\b
 	    )}x;
@@ -1120,39 +1071,6 @@
 		our $cnt_warn++;
 	}
 }
-sub CHK {
-	if ($check && report("CHECK: $_[0]\n")) {
-		our $clean = 0;
-		our $cnt_chk++;
-	}
-}
-
-sub check_absolute_file {
-	my ($absolute, $herecurr) = @_;
-	my $file = $absolute;
-
-	##print "absolute<$absolute>\n";
-
-	# See if any suffix of this path is a path within the tree.
-	while ($file =~ s@^[^/]*/@@) {
-		if (-f "$root/$file") {
-			##print "file<$file>\n";
-			last;
-		}
-	}
-	if (! -f _)  {
-		return 0;
-	}
-
-	# It is, so see if the prefix is acceptable.
-	my $prefix = $absolute;
-	substr($prefix, -length($file)) = '';
-
-	##print "prefix<$prefix>\n";
-	if ($prefix ne ".../") {
-		WARN("use relative pathname instead of absolute in changelog text\n" . $herecurr);
-	}
-}
 
 sub process {
 	my $filename = shift;
@@ -1196,10 +1114,6 @@
 	my %suppress_export;
 
 	# Pre-scan the patch sanitizing the lines.
-	# Pre-scan the patch looking for any __setup documentation.
-	#
-	my @setup_docs = ();
-	my $setup_docs = 0;
 
 	sanitise_line_reset();
 	my $line;
@@ -1207,13 +1121,6 @@
 		$linenr++;
 		$line = $rawline;
 
-		if ($rawline=~/^\+\+\+\s+(\S+)/) {
-			$setup_docs = 0;
-			if ($1 =~ m@Documentation/kernel-parameters.txt$@) {
-				$setup_docs = 1;
-			}
-			#next;
-		}
 		if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) {
 			$realline=$1-1;
 			if (defined $2) {
@@ -1272,10 +1179,6 @@
 
 		#print "==>$rawline\n";
 		#print "-->$line\n";
-
-		if ($setup_docs && $line =~ /^\+/) {
-			push(@setup_docs, $line);
-		}
 	}
 
 	$prefix = '';
@@ -1350,9 +1253,6 @@
 				WARN("patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n");
 			}
 
-			if ($realfile =~ m@^include/asm/@) {
-				ERROR("do not modify files in include/asm, change architecture specific files in include/asm-<architecture>\n" . "$here$rawline\n");
-			}
 			next;
 		}
 
@@ -1367,21 +1267,26 @@
 # Check for incorrect file permissions
 		if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) {
 			my $permhere = $here . "FILE: $realfile\n";
-			if ($realfile =~ /(Makefile|Kconfig|\.c|\.cpp|\.h|\.S|\.tmpl)$/) {
+			if ($realfile =~ /(\bMakefile(?:\.objs)?|\.c|\.cc|\.cpp|\.h|\.mak|\.[sS])$/) {
 				ERROR("do not set execute permissions for source files\n" . $permhere);
 			}
 		}
 
+# Accept git diff extended headers as valid patches
+		if ($line =~ /^(?:rename|copy) (?:from|to) [\w\/\.\-]+\s*$/) {
+			$is_patch = 1;
+		}
+
 #check the patch for a signoff:
 		if ($line =~ /^\s*signed-off-by:/i) {
 			# This is a signoff, if ugly, so do not double report.
 			$signoff++;
 			if (!($line =~ /^\s*Signed-off-by:/)) {
-				WARN("Signed-off-by: is the preferred form\n" .
+				ERROR("The correct form is \"Signed-off-by\"\n" .
 					$herecurr);
 			}
 			if ($line =~ /^\s*signed-off-by:\S/i) {
-				WARN("space required after Signed-off-by:\n" .
+				ERROR("space required after Signed-off-by:\n" .
 					$herecurr);
 			}
 		}
@@ -1392,20 +1297,6 @@
 				$herecurr) if (!$emitted_corrupt++);
 		}
 
-# Check for absolute kernel paths.
-		if ($tree) {
-			while ($line =~ m{(?:^|\s)(/\S*)}g) {
-				my $file = $1;
-
-				if ($file =~ m{^(.*?)(?::\d+)+:?$} &&
-				    check_absolute_file($1, $herecurr)) {
-					#
-				} else {
-					check_absolute_file($file, $herecurr);
-				}
-			}
-		}
-
 # UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php
 		if (($realfile =~ /^$/ || $line =~ /^\+/) &&
 		    $rawline !~ m/^$UTF8*$/) {
@@ -1421,6 +1312,9 @@
 # ignore non-hunk lines and lines being removed
 		next if (!$hunk_line || $line =~ /^-/);
 
+# ignore files that are being periodically imported from Linux
+		next if ($realfile =~ /^(linux-headers|include\/standard-headers)\//);
+
 #trailing whitespace
 		if ($line =~ /^\+.*\015/) {
 			my $herevet = "$here\n" . cat_vet($rawline) . "\n";
@@ -1432,76 +1326,41 @@
 			$rpt_cleaners = 1;
 		}
 
-# check for Kconfig help text having a real description
-# Only applies when adding the entry originally, after that we do not have
-# sufficient context to determine whether it is indeed long enough.
-		if ($realfile =~ /Kconfig/ &&
-		    $line =~ /\+\s*(?:---)?help(?:---)?$/) {
-			my $length = 0;
-			my $cnt = $realcnt;
-			my $ln = $linenr + 1;
-			my $f;
-			my $is_end = 0;
-			while ($cnt > 0 && defined $lines[$ln - 1]) {
-				$f = $lines[$ln - 1];
-				$cnt-- if ($lines[$ln - 1] !~ /^-/);
-				$is_end = $lines[$ln - 1] =~ /^\+/;
-				$ln++;
-
-				next if ($f =~ /^-/);
-				$f =~ s/^.//;
-				$f =~ s/#.*//;
-				$f =~ s/^\s+//;
-				next if ($f =~ /^$/);
-				if ($f =~ /^\s*config\s/) {
-					$is_end = 1;
-					last;
-				}
-				$length++;
-			}
-			WARN("please write a paragraph that describes the config symbol fully\n" . $herecurr) if ($is_end && $length < 4);
-			#print "is_end<$is_end> length<$length>\n";
-		}
-
 # check we are in a valid source file if not then ignore this hunk
-		next if ($realfile !~ /\.(h|c|cpp|s|S|pl|sh)$/);
+		next if ($realfile !~ /\.(h|c|cpp|s|S|pl|py|sh)$/);
 
-#80 column limit
-		if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ &&
-		    $rawline !~ /^.\s*\*\s*\@$Ident\s/ &&
-		    !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:,|\)\s*;)\s*$/ ||
-		    $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
+#90 column limit
+		if ($line =~ /^\+/ &&
+		    !($line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) &&
 		    $length > 80)
 		{
-			WARN("line over 80 characters\n" . $herecurr);
+			if ($length > 90) {
+				ERROR("line over 90 characters\n" . $herecurr);
+			} else {
+				WARN("line over 80 characters\n" . $herecurr);
+			}
 		}
 
 # check for spaces before a quoted newline
 		if ($rawline =~ /^.*\".*\s\\n/) {
-			WARN("unnecessary whitespace before a quoted newline\n" . $herecurr);
+			ERROR("unnecessary whitespace before a quoted newline\n" . $herecurr);
 		}
 
 # check for adding lines without a newline.
 		if ($line =~ /^\+/ && defined $lines[$linenr] && $lines[$linenr] =~ /^\\ No newline at end of file/) {
-			WARN("adding a line without newline at end of file\n" . $herecurr);
+			ERROR("adding a line without newline at end of file\n" . $herecurr);
 		}
 
-# Blackfin: use hi/lo macros
-		if ($realfile =~ m@arch/blackfin/.*\.S$@) {
-			if ($line =~ /\.[lL][[:space:]]*=.*&[[:space:]]*0x[fF][fF][fF][fF]/) {
-				my $herevet = "$here\n" . cat_vet($line) . "\n";
-				ERROR("use the LO() macro, not (... & 0xFFFF)\n" . $herevet);
-			}
-			if ($line =~ /\.[hH][[:space:]]*=.*>>[[:space:]]*16/) {
-				my $herevet = "$here\n" . cat_vet($line) . "\n";
-				ERROR("use the HI() macro, not (... >> 16)\n" . $herevet);
-			}
+# check for RCS/CVS revision markers
+		if ($rawline =~ /^\+.*\$(Revision|Log|Id)(?:\$|\b)/) {
+			ERROR("CVS style keyword markers, these will _not_ be updated\n". $herecurr);
 		}
 
-# check we are in a valid source file C or perl if not then ignore this hunk
-		next if ($realfile !~ /\.(h|c|cpp|pl)$/);
+# tabs are only allowed in assembly source code, and in
+# some scripts we imported from other projects.
+		next if ($realfile =~ /\.(s|S)$/);
+		next if ($realfile =~ /(checkpatch|get_maintainer|texi2pod)\.pl$/);
 
-# in QEMU, no tabs are allowed
 		if ($rawline =~ /^\+.*\t/) {
 			my $herevet = "$here\n" . cat_vet($rawline) . "\n";
 			ERROR("code indent should never use tabs\n" . $herevet);
@@ -1511,21 +1370,6 @@
 # check we are in a valid C source file if not then ignore this hunk
 		next if ($realfile !~ /\.(h|c|cpp)$/);
 
-# check for RCS/CVS revision markers
-		if ($rawline =~ /^\+.*\$(Revision|Log|Id)(?:\$|)/) {
-			WARN("CVS style keyword markers, these will _not_ be updated\n". $herecurr);
-		}
-
-# Blackfin: don't use __builtin_bfin_[cs]sync
-		if ($line =~ /__builtin_bfin_csync/) {
-			my $herevet = "$here\n" . cat_vet($line) . "\n";
-			ERROR("use the CSYNC() macro in asm/blackfin.h\n" . $herevet);
-		}
-		if ($line =~ /__builtin_bfin_ssync/) {
-			my $herevet = "$here\n" . cat_vet($line) . "\n";
-			ERROR("use the SSYNC() macro in asm/blackfin.h\n" . $herevet);
-		}
-
 # Check for potential 'bare' types
 		my ($stat, $cond, $line_nr_next, $remain_next, $off_next,
 		    $realline_next);
@@ -1639,7 +1483,13 @@
 			#print "realcnt<$realcnt> ctx_cnt<$ctx_cnt>\n";
 			#print "pre<$pre_ctx>\nline<$line>\nctx<$ctx>\nnext<$lines[$ctx_ln - 1]>\n";
 
-			if ($ctx !~ /{\s*/ && defined($lines[$ctx_ln -1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/) {
+			# The length of the "previous line" is checked against 80 because it
+			# includes the + at the beginning of the line (if the actual line has
+			# 79 or 80 characters, it is no longer possible to add a space and an
+			# opening brace there)
+			if ($#ctx == 0 && $ctx !~ /{\s*/ &&
+			    defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*\{/ &&
+			    defined($lines[$ctx_ln - 2]) && length($lines[$ctx_ln - 2]) < 80) {
 				ERROR("that open brace { should be on the previous line\n" .
 					"$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
 			}
@@ -1649,7 +1499,7 @@
 			{
 				my ($nlength, $nindent) = line_stats($lines[$ctx_ln - 1]);
 				if ($nindent > $indent) {
-					WARN("trailing semicolon indicates no statements, indent implies otherwise\n" .
+					ERROR("trailing semicolon indicates no statements, indent implies otherwise\n" .
 						"$here\n$ctx\n$rawlines[$ctx_ln - 1]\n");
 				}
 			}
@@ -1678,7 +1528,7 @@
 			my $continuation = 0;
 			my $check = 0;
 			$s =~ s/^.*\bdo\b//;
-			$s =~ s/^\s*{//;
+			$s =~ s/^\s*\{//;
 			if ($s =~ s/^\s*\\//) {
 				$continuation = 1;
 			}
@@ -1737,7 +1587,7 @@
 
 			if ($check && (($sindent % 4) != 0 ||
 			    ($sindent <= $indent && $s ne ''))) {
-				WARN("suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
+				ERROR("suspect code indent for conditional statements ($indent, $sindent)\n" . $herecurr . "$stat_real\n");
 			}
 		}
 
@@ -1777,7 +1627,7 @@
 		}
 
 # check for initialisation to aggregates open brace on the next line
-		if ($line =~ /^.\s*{/ &&
+		if ($line =~ /^.\s*\{/ &&
 		    $prevline =~ /(?:^|[^=])=\s*$/) {
 			ERROR("that open brace { should be on the previous line\n" . $hereprev);
 		}
@@ -1803,50 +1653,6 @@
 		$line =~ s@//.*@@;
 		$opline =~ s@//.*@@;
 
-# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider
-# the whole statement.
-#print "APW <$lines[$realline_next - 1]>\n";
-		if (defined $realline_next &&
-		    exists $lines[$realline_next - 1] &&
-		    !defined $suppress_export{$realline_next} &&
-		    ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
-		     $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
-			# Handle definitions which produce identifiers with
-			# a prefix:
-			#   XXX(foo);
-			#   EXPORT_SYMBOL(something_foo);
-			my $name = $1;
-			if ($stat =~ /^.([A-Z_]+)\s*\(\s*($Ident)/ &&
-			    $name =~ /^${Ident}_$2/) {
-#print "FOO C name<$name>\n";
-				$suppress_export{$realline_next} = 1;
-
-			} elsif ($stat !~ /(?:
-				\n.}\s*$|
-				^.DEFINE_$Ident\(\Q$name\E\)|
-				^.DECLARE_$Ident\(\Q$name\E\)|
-				^.LIST_HEAD\(\Q$name\E\)|
-				^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(|
-				\b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\()
-			    )/x) {
-#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n";
-				$suppress_export{$realline_next} = 2;
-			} else {
-				$suppress_export{$realline_next} = 1;
-			}
-		}
-		if (!defined $suppress_export{$linenr} &&
-		    $prevline =~ /^.\s*$/ &&
-		    ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ ||
-		     $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) {
-#print "FOO B <$lines[$linenr - 1]>\n";
-			$suppress_export{$linenr} = 2;
-		}
-		if (defined $suppress_export{$linenr} &&
-		    $suppress_export{$linenr} == 2) {
-			WARN("EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr);
-		}
-
 # check for global initialisers.
 		if ($line =~ /^.$Type\s*$Ident\s*(?:\s+$Modifier)*\s*=\s*(0|NULL|false)\s*;/) {
 			ERROR("do not initialise globals to 0 or NULL\n" .
@@ -1894,67 +1700,37 @@
 			}
 		}
 
-# # no BUG() or BUG_ON()
-# 		if ($line =~ /\b(BUG|BUG_ON)\b/) {
-# 			print "Try to use WARN_ON & Recovery code rather than BUG() or BUG_ON()\n";
-# 			print "$herecurr";
-# 			$clean = 0;
-# 		}
-
-		if ($line =~ /\bLINUX_VERSION_CODE\b/) {
-			WARN("LINUX_VERSION_CODE should be avoided, code should be for the version to which it is merged\n" . $herecurr);
-		}
-
-# printk should use KERN_* levels.  Note that follow on printk's on the
-# same line do not need a level, so we use the current block context
-# to try and find and validate the current printk.  In summary the current
-# printk includes all preceding printk's which have no newline on the end.
-# we assume the first bad printk is the one to report.
-		if ($line =~ /\bprintk\((?!KERN_)\s*"/) {
-			my $ok = 0;
-			for (my $ln = $linenr - 1; $ln >= $first_line; $ln--) {
-				#print "CHECK<$lines[$ln - 1]\n";
-				# we have a preceding printk if it ends
-				# with "\n" ignore it, else it is to blame
-				if ($lines[$ln - 1] =~ m{\bprintk\(}) {
-					if ($rawlines[$ln - 1] !~ m{\\n"}) {
-						$ok = 1;
-					}
-					last;
-				}
-			}
-			if ($ok == 0) {
-				WARN("printk() should include KERN_ facility level\n" . $herecurr);
-			}
-		}
-
 # function brace can't be on same line, except for #defines of do while,
 # or if closed on same line
-		if (($line=~/$Type\s*$Ident\(.*\).*\s{/) and
-		    !($line=~/\#\s*define.*do\s{/) and !($line=~/}/)) {
+		if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and
+		    !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) {
 			ERROR("open brace '{' following function declarations go on the next line\n" . $herecurr);
 		}
 
 # open braces for enum, union and struct go on the same line.
-		if ($line =~ /^.\s*{/ &&
+		if ($line =~ /^.\s*\{/ &&
 		    $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) {
 			ERROR("open brace '{' following $1 go on the same line\n" . $hereprev);
 		}
 
 # missing space after union, struct or enum definition
 		if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?(?:\s+$Ident)?[=\{]/) {
-		    WARN("missing space after $1 definition\n" . $herecurr);
+		    ERROR("missing space after $1 definition\n" . $herecurr);
 		}
 
 # check for spacing round square brackets; allowed:
 #  1. with a type on the left -- int [] a;
 #  2. at the beginning of a line for slice initialisers -- [0...10] = 5,
 #  3. inside a curly brace -- = { [0...10] = 5 }
+#  4. after a comma -- [1] = 5, [2] = 6
+#  5. in a macro definition -- #define abc(x) [x] = y
 		while ($line =~ /(.*?\s)\[/g) {
 			my ($where, $prefix) = ($-[1], $1);
 			if ($prefix !~ /$Type\s+$/ &&
 			    ($where != 0 || $prefix !~ /^.\s+$/) &&
-			    $prefix !~ /{\s+$/) {
+			    $prefix !~ /{\s+$/ &&
+			    $prefix !~ /\#\s*define[^(]*\([^)]*\)\s+$/ &&
+			    $prefix !~ /,\s+$/) {
 				ERROR("space prohibited before open square bracket '['\n" . $herecurr);
 			}
 		}
@@ -1989,7 +1765,7 @@
 			} elsif ($ctx =~ /$Type$/) {
 
 			} else {
-				WARN("space prohibited between function name and open parenthesis '('\n" . $herecurr);
+				ERROR("space prohibited between function name and open parenthesis '('\n" . $herecurr);
 			}
 		}
 # Check operator spacing.
@@ -2085,7 +1861,7 @@
                                 # not required when having a single },{ on one line
 				} elsif ($op eq ',') {
 					if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/ &&
-                                            ($elements[$n] . $elements[$n + 2]) !~ " *}{") {
+                                            ($elements[$n] . $elements[$n + 2]) !~ " *}\\{") {
 						ERROR("space required after that '$op' $at\n" . $hereptr);
 					}
 
@@ -2125,19 +1901,6 @@
 						ERROR("space prohibited after that '$op' $at\n" . $hereptr);
 					}
 
-
-				# << and >> may either have or not have spaces both sides
-				} elsif ($op eq '<<' or $op eq '>>' or
-					 $op eq '&' or $op eq '^' or $op eq '|' or
-					 $op eq '+' or $op eq '-' or
-					 $op eq '*' or $op eq '/' or
-					 $op eq '%')
-				{
-					if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) {
-						ERROR("need consistent spacing around '$op' $at\n" .
-							$hereptr);
-					}
-
 				# A colon needs no spaces before when it is
 				# terminating a case value or a label.
 				} elsif ($opv eq ':C' || $opv eq ':L') {
@@ -2184,29 +1947,9 @@
 			}
 		}
 
-# check for multiple assignments
-		if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) {
-			CHK("multiple assignments should be avoided\n" . $herecurr);
-		}
-
-## # check for multiple declarations, allowing for a function declaration
-## # continuation.
-## 		if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ &&
-## 		    $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) {
-##
-## 			# Remove any bracketed sections to ensure we do not
-## 			# falsly report the parameters of functions.
-## 			my $ln = $line;
-## 			while ($ln =~ s/\([^\(\)]*\)//g) {
-## 			}
-## 			if ($ln =~ /,/) {
-## 				WARN("declaring multiple variables together should be avoided\n" . $herecurr);
-## 			}
-## 		}
-
 #need space before brace following if, while, etc
-		if (($line =~ /\(.*\){/ && $line !~ /\($Type\){/) ||
-		    $line =~ /do{/) {
+		if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) ||
+		    $line =~ /do\{/) {
 			ERROR("space required before the open brace '{'\n" . $herecurr);
 		}
 
@@ -2261,7 +2004,7 @@
 		if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) {
 			my $name = $1;
 			if ($name ne 'EOF' && $name ne 'ERROR') {
-				CHK("return of an errno should typically be -ve (return -$1)\n" . $herecurr);
+				ERROR("return of an errno should typically be -ve (return -$1)\n" . $herecurr);
 			}
 		}
 
@@ -2333,7 +2076,7 @@
 				(?:\&\&|\|\||\)|\])
 			)/x)
 		{
-			WARN("boolean test with hexadecimal, perhaps just 1 \& or \|?\n" . $herecurr);
+			ERROR("boolean test with hexadecimal, perhaps just 1 \& or \|?\n" . $herecurr);
 		}
 
 # if and else should not have general statements after it
@@ -2389,23 +2132,7 @@
 
 #no spaces allowed after \ in define
 		if ($line=~/\#\s*define.*\\\s$/) {
-			WARN("Whitepspace after \\ makes next lines useless\n" . $herecurr);
-		}
-
-#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line)
-		if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) {
-			my $file = "$1.h";
-			my $checkfile = "include/linux/$file";
-			if (-f "$root/$checkfile" &&
-			    $realfile ne $checkfile &&
-			    $1 !~ /$allowed_asm_includes/)
-			{
-				if ($realfile =~ m{^arch/}) {
-					CHK("Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
-				} else {
-					WARN("Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr);
-				}
-			}
+			ERROR("Whitespace after \\ makes next lines useless\n" . $herecurr);
 		}
 
 # multi-statement macros should be enclosed in a do while loop, grab the
@@ -2502,15 +2229,6 @@
 			}
 		}
 
-# make sure symbols are always wrapped with VMLINUX_SYMBOL() ...
-# all assignments may have only one of the following with an assignment:
-#	.
-#	ALIGN(...)
-#	VMLINUX_SYMBOL(...)
-		if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) {
-			WARN("vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr);
-		}
-
 # check for missing bracing round if etc
 		if ($line =~ /(^.*)\bif\b/ && $line !~ /\#\s*if/) {
 			my ($level, $endln, @chunks) =
@@ -2542,7 +2260,10 @@
 
 					substr($block, 0, length($cond), '');
 
-					$seen++ if ($block =~ /^\s*{/);
+					my $spaced_block = $block;
+					$spaced_block =~ s/\n\+/ /g;
+
+					$seen++ if ($spaced_block =~ /^\s*\{/);
 
                                         print "APW: cond<$cond> block<$block> allowed<$allowed>\n"
                                             if $dbg_adv_apw;
@@ -2563,7 +2284,7 @@
 					}
 				}
 				if ($seen != ($#chunks + 1)) {
-					WARN("braces {} are necessary for all arms of this statement\n" . $herectx);
+					ERROR("braces {} are necessary for all arms of this statement\n" . $herectx);
 				}
 			}
 		}
@@ -2631,68 +2352,27 @@
 					$herectx .= raw_line($linenr, $n) . "\n";;
 				}
 
-				WARN("braces {} are necessary even for single statement blocks\n" . $herectx);
-			}
-		}
-
-# don't include deprecated include files (uses RAW line)
-		for my $inc (@dep_includes) {
-			if ($rawline =~ m@^.\s*\#\s*include\s*\<$inc>@) {
-				ERROR("Don't use <$inc>: see Documentation/feature-removal-schedule.txt\n" . $herecurr);
-			}
-		}
-
-# don't use deprecated functions
-		for my $func (@dep_functions) {
-			if ($line =~ /\b$func\b/) {
-				ERROR("Don't use $func(): see Documentation/feature-removal-schedule.txt\n" . $herecurr);
+				ERROR("braces {} are necessary even for single statement blocks\n" . $herectx);
 			}
 		}
 
 # no volatiles please
 		my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b};
 		if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) {
-			WARN("Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
-		}
-
-# SPIN_LOCK_UNLOCKED & RW_LOCK_UNLOCKED are deprecated
-		if ($line =~ /\b(SPIN_LOCK_UNLOCKED|RW_LOCK_UNLOCKED)/) {
-			ERROR("Use of $1 is deprecated: see Documentation/spinlocks.txt\n" . $herecurr);
+			ERROR("Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr);
 		}
 
 # warn about #if 0
 		if ($line =~ /^.\s*\#\s*if\s+0\b/) {
-			CHK("if this code is redundant consider removing it\n" .
+			ERROR("if this code is redundant consider removing it\n" .
 				$herecurr);
 		}
 
-# check for needless kfree() checks
+# check for needless g_free() checks
 		if ($prevline =~ /\bif\s*\(([^\)]*)\)/) {
 			my $expr = $1;
-			if ($line =~ /\bkfree\(\Q$expr\E\);/) {
-				WARN("kfree(NULL) is safe this check is probably not required\n" . $hereprev);
-			}
-		}
-# check for needless usb_free_urb() checks
-		if ($prevline =~ /\bif\s*\(([^\)]*)\)/) {
-			my $expr = $1;
-			if ($line =~ /\busb_free_urb\(\Q$expr\E\);/) {
-				WARN("usb_free_urb(NULL) is safe this check is probably not required\n" . $hereprev);
-			}
-		}
-
-# prefer usleep_range over udelay
-		if ($line =~ /\budelay\s*\(\s*(\w+)\s*\)/) {
-			# ignore udelay's < 10, however
-			if (! (($1 =~ /(\d+)/) && ($1 < 10)) ) {
-				CHK("usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $line);
-			}
-		}
-
-# warn about unexpectedly long msleep's
-		if ($line =~ /\bmsleep\s*\((\d+)\);/) {
-			if ($1 < 20) {
-				WARN("msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $line);
+			if ($line =~ /\bg_free\(\Q$expr\E\);/) {
+				ERROR("g_free(NULL) is safe this check is probably not required\n" . $hereprev);
 			}
 		}
 
@@ -2707,29 +2387,22 @@
 		if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) {
 			ERROR("exactly one space required after that #$1\n" . $herecurr);
 		}
-
-# check for spinlock_t definitions without a comment.
-		if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ ||
-		    $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) {
-			my $which = $1;
-			if (!ctx_has_comment($first_line, $linenr)) {
-				CHK("$1 definition without comment\n" . $herecurr);
-			}
-		}
 # check for memory barriers without a comment.
-		if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
+		if ($line =~ /\b(smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) {
 			if (!ctx_has_comment($first_line, $linenr)) {
-				CHK("memory barrier without comment\n" . $herecurr);
+				ERROR("memory barrier without comment\n" . $herecurr);
 			}
 		}
 # check of hardware specific defines
-		if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) {
-			CHK("architecture specific defines should be avoided\n" .  $herecurr);
+# we have e.g. CONFIG_LINUX and CONFIG_WIN32 for common cases
+# where they might be necessary.
+		if ($line =~ m@^.\s*\#\s*if.*\b__@) {
+			ERROR("architecture specific defines should be avoided\n" .  $herecurr);
 		}
 
 # Check that the storage class is at the beginning of a declaration
 		if ($line =~ /\b$Storage\b/ && $line !~ /^.\s*$Storage\b/) {
-			WARN("storage class should be at the beginning of the declaration\n" . $herecurr)
+			ERROR("storage class should be at the beginning of the declaration\n" . $herecurr)
 		}
 
 # check the location of the inline attribute, that it is between
@@ -2739,14 +2412,9 @@
 			ERROR("inline keyword should sit between storage class and type\n" . $herecurr);
 		}
 
-# Check for __inline__ and __inline, prefer inline
-		if ($line =~ /\b(__inline__|__inline)\b/) {
-			WARN("plain inline is preferred over $1\n" . $herecurr);
-		}
-
 # check for sizeof(&)
 		if ($line =~ /\bsizeof\s*\(\s*\&/) {
-			WARN("sizeof(& should be avoided\n" . $herecurr);
+			ERROR("sizeof(& should be avoided\n" . $herecurr);
 		}
 
 # check for new externs in .c files.
@@ -2763,144 +2431,135 @@
 			if ($s =~ /^\s*;/ &&
 			    $function_name ne 'uninitialized_var')
 			{
-				WARN("externs should be avoided in .c files\n" .  $herecurr);
+				ERROR("externs should be avoided in .c files\n" .  $herecurr);
 			}
 
 			if ($paren_space =~ /\n/) {
-				WARN("arguments for function declarations should follow identifier\n" . $herecurr);
+				ERROR("arguments for function declarations should follow identifier\n" . $herecurr);
 			}
 
 		} elsif ($realfile =~ /\.c$/ && defined $stat &&
 		    $stat =~ /^.\s*extern\s+/)
 		{
-			WARN("externs should be avoided in .c files\n" .  $herecurr);
+			ERROR("externs should be avoided in .c files\n" .  $herecurr);
 		}
 
-# checks for new __setup's
-		if ($rawline =~ /\b__setup\("([^"]*)"/) {
-			my $name = $1;
-
-			if (!grep(/$name/, @setup_docs)) {
-				CHK("__setup appears un-documented -- check Documentation/kernel-parameters.txt\n" . $herecurr);
+# check for pointless casting of g_malloc return
+		if ($line =~ /\*\s*\)\s*g_(try)?(m|re)alloc(0?)(_n)?\b/) {
+			if ($2 == 'm') {
+				ERROR("unnecessary cast may hide bugs, use g_$1new$3 instead\n" . $herecurr);
+			} else {
+				ERROR("unnecessary cast may hide bugs, use g_$1renew$3 instead\n" . $herecurr);
 			}
 		}
 
-# check for pointless casting of kmalloc return
-		if ($line =~ /\*\s*\)\s*k[czm]alloc\b/) {
-			WARN("unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr);
-		}
-
 # check for gcc specific __FUNCTION__
 		if ($line =~ /__FUNCTION__/) {
-			WARN("__func__ should be used instead of gcc specific __FUNCTION__\n"  . $herecurr);
+			ERROR("__func__ should be used instead of gcc specific __FUNCTION__\n"  . $herecurr);
 		}
 
-# check for semaphores used as mutexes
-		if ($line =~ /^.\s*(DECLARE_MUTEX|init_MUTEX)\s*\(/) {
-			WARN("mutexes are preferred for single holder semaphores\n" . $herecurr);
+# recommend qemu_strto* over strto* for numeric conversions
+		if ($line =~ /\b(strto[^kd].*?)\s*\(/) {
+			ERROR("consider using qemu_$1 in preference to $1\n" . $herecurr);
 		}
-# check for semaphores used as mutexes
-		if ($line =~ /^.\s*init_MUTEX_LOCKED\s*\(/) {
-			WARN("consider using a completion\n" . $herecurr);
-
-		}
-# recommend strict_strto* over simple_strto*
-		if ($line =~ /\bsimple_(strto.*?)\s*\(/) {
-			WARN("consider using strict_$1 in preference to simple_$1\n" . $herecurr);
-		}
-# check for __initcall(), use device_initcall() explicitly please
-		if ($line =~ /^.\s*__initcall\s*\(/) {
-			WARN("please use device_initcall() instead of __initcall()\n" . $herecurr);
+# check for module_init(), use category-specific init macros explicitly please
+		if ($line =~ /^module_init\s*\(/) {
+			ERROR("please use block_init(), type_init() etc. instead of module_init()\n" . $herecurr);
 		}
 # check for various ops structs, ensure they are const.
-		my $struct_ops = qr{acpi_dock_ops|
-				address_space_operations|
-				backlight_ops|
-				block_device_operations|
-				dentry_operations|
-				dev_pm_ops|
-				dma_map_ops|
-				extent_io_ops|
-				file_lock_operations|
-				file_operations|
-				hv_ops|
-				ide_dma_ops|
-				intel_dvo_dev_ops|
-				item_operations|
-				iwl_ops|
-				kgdb_arch|
-				kgdb_io|
-				kset_uevent_ops|
-				lock_manager_operations|
-				microcode_ops|
-				mtrr_ops|
-				neigh_ops|
-				nlmsvc_binding|
-				pci_raw_ops|
-				pipe_buf_operations|
-				platform_hibernation_ops|
-				platform_suspend_ops|
-				proto_ops|
-				rpc_pipe_ops|
-				seq_operations|
-				snd_ac97_build_ops|
-				soc_pcmcia_socket_ops|
-				stacktrace_ops|
-				sysfs_ops|
-				tty_operations|
-				usb_mon_operations|
-				wd_ops}x;
+		my $struct_ops = qr{AIOCBInfo|
+				BdrvActionOps|
+				BlockDevOps|
+				BlockJobDriver|
+				DisplayChangeListenerOps|
+				GraphicHwOps|
+				IDEDMAOps|
+				KVMCapabilityInfo|
+				MemoryRegionIOMMUOps|
+				MemoryRegionOps|
+				MemoryRegionPortio|
+				QEMUFileOps|
+				SCSIBusInfo|
+				SCSIReqOps|
+				Spice[A-Z][a-zA-Z0-9]*Interface|
+				TPMDriverOps|
+				USBDesc[A-Z][a-zA-Z0-9]*|
+				VhostOps|
+				VMStateDescription|
+				VMStateInfo}x;
 		if ($line !~ /\bconst\b/ &&
-		    $line =~ /\bstruct\s+($struct_ops)\b/) {
-			WARN("struct $1 should normally be const\n" .
+		    $line =~ /\b($struct_ops)\b/) {
+			ERROR("struct $1 should normally be const\n" .
 				$herecurr);
 		}
 
-# use of NR_CPUS is usually wrong
-# ignore definitions of NR_CPUS and usage to define arrays as likely right
-		if ($line =~ /\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*\s*#\s*if\b.*\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*\s*#\s*define\b.*\bNR_CPUS\b/ &&
-		    $line !~ /^.\s*$Declare\s.*\[[^\]]*NR_CPUS[^\]]*\]/ &&
-		    $line !~ /\[[^\]]*\.\.\.[^\]]*NR_CPUS[^\]]*\]/ &&
-		    $line !~ /\[[^\]]*NR_CPUS[^\]]*\.\.\.[^\]]*\]/)
-		{
-			WARN("usage of NR_CPUS is often wrong - consider using cpu_possible(), num_possible_cpus(), for_each_possible_cpu(), etc\n" . $herecurr);
-		}
-
 # check for %L{u,d,i} in strings
 		my $string;
 		while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) {
 			$string = substr($rawline, $-[1], $+[1] - $-[1]);
 			$string =~ s/%%/__/g;
 			if ($string =~ /(?<!%)%L[udi]/) {
-				WARN("\%Ld/%Lu are not-standard C, use %lld/%llu\n" . $herecurr);
+				ERROR("\%Ld/%Lu are not-standard C, use %lld/%llu\n" . $herecurr);
 				last;
 			}
 		}
 
-# whine mightly about in_atomic
-		if ($line =~ /\bin_atomic\s*\(/) {
-			if ($realfile =~ m@^drivers/@) {
-				ERROR("do not use in_atomic in drivers\n" . $herecurr);
-			} elsif ($realfile !~ m@^kernel/@) {
-				WARN("use of in_atomic() is incorrect outside core kernel code\n" . $herecurr);
-			}
-		}
-
-# check for lockdep_set_novalidate_class
-		if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ ||
-		    $line =~ /__lockdep_no_validate__\s*\)/ ) {
-			if ($realfile !~ m@^kernel/lockdep@ &&
-			    $realfile !~ m@^include/linux/lockdep@ &&
-			    $realfile !~ m@^drivers/base/core@) {
-				ERROR("lockdep_no_validate class is reserved for device->mutex.\n" . $herecurr);
-			}
-		}
-
 # QEMU specific tests
 		if ($rawline =~ /\b(?:Qemu|QEmu)\b/) {
-			WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr);
+			ERROR("use QEMU instead of Qemu or QEmu\n" . $herecurr);
+		}
+
+# Qemu error function tests
+
+	# Find newlines in error messages
+	my $qemu_error_funcs = qr{error_setg|
+				error_setg_errno|
+				error_setg_win32|
+				error_setg_file_open|
+				error_set|
+				error_prepend|
+				error_reportf_err|
+				error_vreport|
+				error_report}x;
+
+	if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(.*\".*\\n/) {
+		ERROR("Error messages should not contain newlines\n" . $herecurr);
+	}
+
+	# Continue checking for error messages that contains newlines. This
+	# check handles cases where string literals are spread over multiple lines.
+	# Example:
+	# error_report("Error msg line #1"
+	#              "Error msg line #2\n");
+	my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"};
+	my $continued_str_literal = qr{\+\s*\".*\"};
+
+	if ($rawline =~ /$quoted_newline_regex/) {
+		# Backtrack to first line that does not contain only a quoted literal
+		# and assume that it is the start of the statement.
+		my $i = $linenr - 2;
+
+		while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) {
+			$i--;
+		}
+
+		if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) {
+			ERROR("Error messages should not contain newlines\n" . $herecurr);
+		}
+	}
+
+# check for non-portable libc calls that have portable alternatives in QEMU
+		if ($line =~ /\bffs\(/) {
+			ERROR("use ctz32() instead of ffs()\n" . $herecurr);
+		}
+		if ($line =~ /\bffsl\(/) {
+			ERROR("use ctz32() or ctz64() instead of ffsl()\n" . $herecurr);
+		}
+		if ($line =~ /\bffsll\(/) {
+			ERROR("use ctz64() instead of ffsll()\n" . $herecurr);
+		}
+		if ($line =~ /\bbzero\(/) {
+			ERROR("use memset() instead of bzero()\n" . $herecurr);
 		}
 	}
 
@@ -2933,7 +2592,6 @@
 	if ($summary && !($clean == 1 && $quiet == 1)) {
 		print "$filename " if ($summary_file);
 		print "total: $cnt_error errors, $cnt_warn warnings, " .
-			(($check)? "$cnt_chk checks, " : "") .
 			"$cnt_lines lines checked\n";
 		print "\n" if ($quiet == 0);
 	}
@@ -2956,5 +2614,5 @@
 		print "CHECKPATCH in MAINTAINERS.\n";
 	}
 
-	return $clean;
+	return ($no_warnings ? $clean : $cnt_error == 0);
 }
diff --git a/scripts/clean-header-guards.pl b/scripts/clean-header-guards.pl
new file mode 100755
index 0000000..54ab99a
--- /dev/null
+++ b/scripts/clean-header-guards.pl
@@ -0,0 +1,213 @@
+#!/usr/bin/perl -w
+#
+# Clean up include guards in headers
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# (at your option) any later version. See the COPYING file in the
+# top-level directory.
+#
+# Usage: scripts/clean-header-guards.pl [OPTION]... [FILE]...
+#     -c CC     Use a compiler other than cc
+#     -n        Suppress actual cleanup
+#     -v        Show which files are cleaned up, and which are skipped
+#
+# Does the following:
+# - Header files without a recognizable header guard are skipped.
+# - Clean up any untidy header guards in-place.  Warn if the cleanup
+#   renames guard symbols, and explain how to find occurences of these
+#   symbols that may have to be updated manually.
+# - Warn about duplicate header guard symbols.  To make full use of
+#   this warning, you should clean up *all* headers in one run.
+# - Warn when preprocessing a header with its guard symbol defined
+#   produces anything but whitespace.  The preprocessor is run like
+#   "cc -E -DGUARD_H -c -P -", and fed the test program on stdin.
+
+use strict;
+use Getopt::Std;
+
+# Stuff we don't want to clean because we import it into our tree:
+my $exclude = qr,^(disas/libvixl/|include/standard-headers/
+    |linux-headers/|pc-bios/|tests/tcg/|tests/multiboot/),x;
+# Stuff that is expected to fail the preprocessing test:
+my $exclude_cpp = qr,^include/libdecnumber/decNumberLocal.h,;
+
+my %guarded = ();
+my %old_guard = ();
+
+our $opt_c = "cc";
+our $opt_n = 0;
+our $opt_v = 0;
+getopts("c:nv");
+
+sub skipping {
+    my ($fname, $msg, $line1, $line2) = @_;
+
+    return if !$opt_v or $fname =~ $exclude;
+    print "$fname skipped: $msg\n";
+    print "    $line1" if defined $line1;
+    print "    $line2" if defined $line2;
+}
+
+sub gripe {
+    my ($fname, $msg) = @_;
+    return if $fname =~ $exclude;
+    print STDERR "$fname: warning: $msg\n";
+}
+
+sub slurp {
+    my ($fname) = @_;
+    local $/;                   # slurp
+    open(my $in, "<", $fname)
+        or die "can't open $fname for reading: $!";
+    return <$in>;
+}
+
+sub unslurp {
+    my ($fname, $contents) = @_;
+    open (my $out, ">", $fname)
+        or die "can't open $fname for writing: $!";
+    print $out $contents
+        or die "error writing $fname: $!";
+    close $out
+        or die "error writing $fname: $!";
+}
+
+sub fname2guard {
+    my ($fname) = @_;
+    $fname =~ tr/a-z/A-Z/;
+    $fname =~ tr/A-Z0-9/_/cs;
+    return $fname;
+}
+
+sub preprocess {
+    my ($fname, $guard) = @_;
+
+    open(my $pipe, "-|", "$opt_c -E -D$guard -c -P - <$fname")
+        or die "can't run $opt_c: $!";
+    while (<$pipe>) {
+        if ($_ =~ /\S/) {
+            gripe($fname, "not blank after preprocessing");
+            last;
+        }
+    }
+    close $pipe
+        or gripe($fname, "preprocessing failed ($opt_c exit status $?)");
+}
+
+for my $fname (@ARGV) {
+    my $text = slurp($fname);
+
+    $text =~ m,\A(\s*\n|\s*//\N*\n|\s*/\*.*?\*/\s*\n)*|,msg;
+    my $pre = $&;
+    unless ($text =~ /\G(.*\n)/g) {
+        $text =~ /\G.*/;
+        skipping($fname, "no recognizable header guard", "$&\n");
+        next;
+    }
+    my $line1 = $1;
+    unless ($text =~ /\G(.*\n)/g) {
+        $text =~ /\G.*/;
+        skipping($fname, "no recognizable header guard", "$&\n");
+        next;
+    }
+    my $line2 = $1;
+    my $body = substr($text, pos($text));
+
+    unless ($line1 =~ /^\s*\#\s*(if\s*\!\s*defined(\s*\()?|ifndef)\s*
+                       ([A-Za-z0-9_]+)/x) {
+        skipping($fname, "no recognizable header guard", $line1, $line2);
+        next;
+    }
+    my $guard = $3;
+    unless ($line2 =~ /^\s*\#\s*define\s+([A-Za-z0-9_]+)/) {
+        skipping($fname, "no recognizable header guard", $line1, $line2);
+        next;
+    }
+    my $guard2 = $1;
+    unless ($guard2 eq $guard) {
+        skipping($fname, "mismatched header guard ($guard vs. $guard2) ",
+                 $line1, $line2);
+        next;
+    }
+
+    unless ($body =~ m,\A((.*\n)*)
+                       (\s*\#\s*endif\s*(/\*\s*.*\s*\*/\s*)?\n?)
+                       (\n|\s)*\Z,x) {
+        skipping($fname, "can't find end of header guard");
+        next;
+    }
+    $body = $1;
+    my $line3 = $3;
+    my $endif_comment = $4;
+
+    my $oldg = $guard;
+
+    unless ($fname =~ $exclude) {
+        my @issues = ();
+        $guard =~ tr/a-z/A-Z/
+            and push @issues, "contains lowercase letters";
+        $guard =~ s/^_+//
+            and push @issues, "is a reserved identifier";
+        $guard =~ s/(_H)?_*$/_H/
+            and $& ne "_H" and push @issues, "doesn't end with _H";
+        unless ($guard =~ /^[A-Z][A-Z0-9_]*_H/) {
+            skipping($fname, "can't clean up odd guard symbol $oldg\n",
+                     $line1, $line2);
+            next;
+        }
+
+        my $exp = fname2guard($fname =~ s,.*/,,r);
+        unless ($guard =~ /\Q$exp\E\Z/) {
+            $guard = fname2guard($fname =~ s,^include/,,r);
+            push @issues, "doesn't match the file name";
+        }
+        if (@issues and $opt_v) {
+            print "$fname guard $oldg needs cleanup:\n    ",
+                join(", ", @issues), "\n";
+        }
+    }
+
+    $old_guard{$guard} = $oldg
+        if $guard ne $oldg;
+
+    if (exists $guarded{$guard}) {
+        gripe($fname, "guard $guard also used by $guarded{$guard}");
+    } else {
+        $guarded{$guard} = $fname;
+    }
+
+    unless ($fname =~ $exclude) {
+        my $newl1 = "#ifndef $guard\n";
+        my $newl2 = "#define $guard\n";
+        my $newl3 = "#endif\n";
+        $newl3 =~ s,\Z, /* $guard */, if defined $endif_comment;
+        if ($line1 ne $newl1 or $line2 ne $newl2 or $line3 ne $newl3) {
+            $pre =~ s/\n*\Z/\n\n/ if $pre =~ /\N/;
+            $body =~ s/\A\n*/\n/;
+            if ($opt_n) {
+                print "$fname would be cleaned up\n" if $opt_v;
+            } else {
+                unslurp($fname, "$pre$newl1$newl2$body$newl3");
+                print "$fname cleaned up\n" if $opt_v;
+            }
+        }
+    }
+
+    preprocess($fname, $opt_n ? $oldg : $guard)
+        unless $fname =~ $exclude or $fname =~ $exclude_cpp;
+}
+
+if (%old_guard) {
+    print STDERR "warning: guard symbol renaming may break things\n";
+    for my $guard (sort keys %old_guard) {
+        print STDERR "    $old_guard{$guard} -> $guard\n";
+    }
+    print STDERR "To find uses that may have to be updated try:\n";
+    print STDERR "    git grep -Ew '", join("|", sort values %old_guard),
+        "'\n";
+}
diff --git a/scripts/clean-includes b/scripts/clean-includes
new file mode 100755
index 0000000..4412a55
--- /dev/null
+++ b/scripts/clean-includes
@@ -0,0 +1,169 @@
+#!/bin/sh -e
+#
+# Clean up QEMU #include lines by ensuring that qemu/osdep.h
+# is the first include listed in .c files, and no headers provided
+# by osdep.h itself are redundantly included in either .c or .h files.
+#
+# Copyright (c) 2015 Linaro Limited
+#
+# Authors:
+#  Peter Maydell <peter.maydell@linaro.org>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+# Usage:
+#   clean-includes [--git subjectprefix] file ...
+# or
+#   clean-includes [--git subjectprefix] --all
+#
+# If the --git subjectprefix option is given, then after making
+# the changes to the files this script will create a git commit
+# with the subject line "subjectprefix: Clean up includes"
+# and a boilerplate commit message.
+#
+# Using --all will cause clean-includes to run on the whole source
+# tree (excluding certain directories which are known not to need
+# handling).
+
+# This script requires Coccinelle to be installed.
+
+# .c files will have the osdep.h included added, and redundant
+# includes removed.
+# .h files will have redundant includes (including includes of osdep.h)
+# removed.
+# Other files (including C++ and ObjectiveC) can't be handled by this script.
+
+# The following one-liner may be handy for finding files to run this on.
+# However some caution is required regarding files that might be part
+# of the guest agent or standalone tests.
+
+# for i in $(git ls-tree --name-only HEAD) ; do test -f $i && \
+#   grep -E '^# *include' $i | head -1 | grep 'osdep.h' ; test $? != 0 && \
+#   echo $i ; done
+
+
+GIT=no
+
+# Extended regular expression defining files to ignore when using --all
+XDIRREGEX='^(tests/tcg|tests/multiboot|pc-bios|disas/libvixl)'
+
+if [ $# -ne 0 ] && [ "$1" = "--git" ]; then
+    if [ $# -eq 1 ]; then
+        echo "--git option requires an argument"
+        exit 1
+    fi
+    GITSUBJ="$2"
+    GIT=yes
+    shift
+    shift
+fi
+
+if [ $# -eq 0 ]; then
+    echo "Usage: clean-includes [--git subjectprefix] [--all | foo.c ...]"
+    echo "(modifies the files in place)"
+    exit 1
+fi
+
+if [ "$1" = "--all" ]; then
+    # We assume there are no files in the tree with spaces in their name
+    set -- $(git ls-files '*.[ch]' | grep -E -v "$XDIRREGEX")
+fi
+
+# Annoyingly coccinelle won't read a scriptfile unless its
+# name ends '.cocci', so write it out to a tempfile with the
+# right kind of name.
+COCCIFILE="$(mktemp --suffix=.cocci)"
+
+trap 'rm -f -- "$COCCIFILE"' INT TERM HUP EXIT
+
+cat >"$COCCIFILE" <<EOT
+@@
+@@
+
+(
++ #include "qemu/osdep.h"
+ #include "..."
+|
++ #include "qemu/osdep.h"
+ #include <...>
+)
+EOT
+
+
+for f in "$@"; do
+  case "$f" in
+    *.inc.c)
+      # These aren't standalone C source files
+      echo "SKIPPING $f (not a standalone source file)"
+      continue
+      ;;
+    *.c)
+      MODE=c
+      ;;
+    *include/qemu/osdep.h | \
+    *include/qemu/compiler.h | \
+    *include/glib-compat.h | \
+    *include/sysemu/os-posix.h | \
+    *include/sysemu/os-win32.h | \
+    *include/standard-headers/ )
+      # Removing include lines from osdep.h itself would be counterproductive.
+      echo "SKIPPING $f (special case header)"
+      continue
+      ;;
+    *include/standard-headers/*)
+      echo "SKIPPING $f (autogenerated header)"
+      continue
+      ;;
+    *.h)
+      MODE=h
+      ;;
+    *)
+      echo "WARNING: ignoring $f (cannot handle non-C files)"
+      continue
+      ;;
+  esac
+
+  if [ "$MODE" = "c" ]; then
+    # First, use Coccinelle to add qemu/osdep.h before the first existing include
+    # (this will add two lines if the file uses both "..." and <...> #includes,
+    # but we will remove the extras in the next step)
+    spatch  --in-place --no-show-diff --cocci-file "$COCCIFILE" "$f"
+
+    # Now remove any duplicate osdep.h includes
+    perl -n -i -e 'print if !/#include "qemu\/osdep.h"/ || !$n++;' "$f"
+  else
+    # Remove includes of osdep.h itself
+    perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ ||
+                            ! (grep { $_ eq $1 } qw ("qemu/osdep.h"))' "$f"
+  fi
+
+  # Remove includes that osdep.h already provides
+  perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ ||
+                          ! (grep { $_ eq $1 } qw (
+           "config-host.h" "config-target.h" "qemu/compiler.h"
+           <setjmp.h> <stdarg.h> <stddef.h> <stdbool.h> <stdint.h> <sys/types.h>
+           <stdlib.h> <stdio.h> <string.h> <strings.h> <inttypes.h>
+           <limits.h> <unistd.h> <time.h> <ctype.h> <errno.h> <fcntl.h>
+           <sys/stat.h> <sys/time.h> <assert.h> <signal.h> <glib.h>
+           <sys/stat.h> <sys/time.h> <assert.h> <signal.h> <glib.h> <sys/mman.h>
+           "sysemu/os-posix.h, sysemu/os-win32.h "glib-compat.h"
+           "qemu/typedefs.h"
+            ))' "$f"
+
+done
+
+if [ "$GIT" = "yes" ]; then
+    git add -- "$@"
+    git commit --signoff -F - <<EOF
+$GITSUBJ: Clean up includes
+
+Clean up includes so that osdep.h is included first and headers
+which it implies are not included manually.
+
+This commit was created with scripts/clean-includes.
+
+EOF
+
+fi
diff --git a/scripts/cocci-macro-file.h b/scripts/cocci-macro-file.h
new file mode 100644
index 0000000..9f2e72e
--- /dev/null
+++ b/scripts/cocci-macro-file.h
@@ -0,0 +1,125 @@
+/* Macro file for Coccinelle
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or, at your
+ * option, any later version.  See the COPYING file in the top-level directory.
+ */
+
+/* Coccinelle only does limited parsing of headers, and chokes on some idioms
+ * defined in compiler.h and queue.h.  Macros that Coccinelle must know about
+ * in order to parse .c files must be in a separate macro file---which is
+ * exactly what you're staring at now.
+ *
+ * To use this file, add the "--macro-file scripts/cocci-macro-file.h" to the
+ * Coccinelle command line.
+ */
+
+/* From qemu/compiler.h */
+#define QEMU_GNUC_PREREQ(maj, min) 1
+#define QEMU_NORETURN __attribute__ ((__noreturn__))
+#define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result))
+#define QEMU_SENTINEL __attribute__((sentinel))
+#define QEMU_ARTIFICIAL __attribute__((always_inline, artificial))
+#define QEMU_PACKED __attribute__((gcc_struct, packed))
+
+#define cat(x,y) x ## y
+#define cat2(x,y) cat(x,y)
+#define QEMU_BUILD_BUG_ON(x) \
+    typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused));
+
+#define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m)))
+
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)	tostring(s)
+#define tostring(s)	#s
+
+#define typeof_field(type, field) typeof(((type *)0)->field)
+#define type_check(t1,t2) ((t1*)0 - (t2*)0)
+
+/* From qemu/queue.h */
+
+#define QLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *lh_first;  /* first element */                     \
+}
+
+#define QLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *le_next;   /* next element */                      \
+        struct type **le_prev;  /* address of previous next element */  \
+}
+
+/*
+ * Singly-linked List definitions.
+ */
+#define QSLIST_HEAD(name, type)                                          \
+struct name {                                                           \
+        struct type *slh_first; /* first element */                     \
+}
+
+#define QSLIST_HEAD_INITIALIZER(head)                                    \
+        { NULL }
+
+#define QSLIST_ENTRY(type)                                               \
+struct {                                                                \
+        struct type *sle_next;  /* next element */                      \
+}
+
+/*
+ * Simple queue definitions.
+ */
+#define QSIMPLEQ_HEAD(name, type)                                       \
+struct name {                                                           \
+    struct type *sqh_first;    /* first element */                      \
+    struct type **sqh_last;    /* addr of last next element */          \
+}
+
+#define QSIMPLEQ_HEAD_INITIALIZER(head)                                 \
+    { NULL, &(head).sqh_first }
+
+#define QSIMPLEQ_ENTRY(type)                                            \
+struct {                                                                \
+    struct type *sqe_next;    /* next element */                        \
+}
+
+/*
+ * Tail queue definitions.
+ */
+#define Q_TAILQ_HEAD(name, type, qual)                                  \
+struct name {                                                           \
+        qual type *tqh_first;           /* first element */             \
+        qual type *qual *tqh_last;      /* addr of last next element */ \
+}
+#define QTAILQ_HEAD(name, type)                                         \
+struct name {                                                           \
+        type *tqh_first;      /* first element */                       \
+        type **tqh_last;      /* addr of last next element */           \
+}
+
+#define QTAILQ_HEAD_INITIALIZER(head)                                   \
+        { NULL, &(head).tqh_first }
+
+#define Q_TAILQ_ENTRY(type, qual)                                       \
+struct {                                                                \
+        qual type *tqe_next;            /* next element */              \
+        qual type *qual *tqe_prev;      /* address of previous next element */\
+}
+#define QTAILQ_ENTRY(type)                                              \
+struct {                                                                \
+        type *tqe_next;       /* next element */                        \
+        type **tqe_prev;      /* address of previous next element */    \
+}
+
+/* From glib */
+#define g_assert_cmpint(a, op, b)   g_assert(a op b)
+#define g_assert_cmpuint(a, op, b)   g_assert(a op b)
+#define g_assert_cmphex(a, op, b)   g_assert(a op b)
+#define g_assert_cmpstr(a, op, b)   g_assert(strcmp(a, b) op 0)
diff --git a/scripts/coccinelle/err-bad-newline.cocci b/scripts/coccinelle/err-bad-newline.cocci
new file mode 100644
index 0000000..1316cc8
--- /dev/null
+++ b/scripts/coccinelle/err-bad-newline.cocci
@@ -0,0 +1,29 @@
+// Error messages should not contain newlines.  This script finds
+// messages that do.  Fixing them is manual.
+@r@
+expression errp, eno, cls, fmt;
+position p;
+@@
+(
+error_report(fmt, ...)@p
+|
+error_setg(errp, fmt, ...)@p
+|
+error_setg_errno(errp, eno, fmt, ...)@p
+|
+error_setg_win32(errp, eno, cls, fmt, ...)@p
+|
+error_prepend(errp, fmt, ...)@p
+|
+error_setg_file_open(errp, eno, cls, fmt, ...)@p
+|
+error_reportf_err(errp, fmt, ...)@p
+|
+error_set(errp, cls, fmt, ...)@p
+)
+@script:python@
+fmt << r.fmt;
+p << r.p;
+@@
+if "\\n" in str(fmt):
+    print "%s:%s:%s:%s" % (p[0].file, p[0].line, p[0].column, fmt)
diff --git a/scripts/coccinelle/error_propagate_null.cocci b/scripts/coccinelle/error_propagate_null.cocci
new file mode 100644
index 0000000..c236380
--- /dev/null
+++ b/scripts/coccinelle/error_propagate_null.cocci
@@ -0,0 +1,10 @@
+// error_propagate() already ignores local_err==NULL, so there's
+// no need to check it before calling.
+
+@@
+identifier L;
+expression E;
+@@
+-if (L) {
+     error_propagate(E, L);
+-}
diff --git a/scripts/coccinelle/overflow_muldiv64.cocci b/scripts/coccinelle/overflow_muldiv64.cocci
new file mode 100644
index 0000000..08ec4a8
--- /dev/null
+++ b/scripts/coccinelle/overflow_muldiv64.cocci
@@ -0,0 +1,16 @@
+// Find muldiv64(i64, i64, x) for potential overflow
+@filter@
+typedef uint64_t;
+typedef int64_t;
+{ uint64_t, int64_t, long, unsigned long } a, b;
+expression c;
+position p;
+@@
+
+muldiv64(a,b,c)@p
+
+@script:python@
+p << filter.p;
+@@
+
+cocci.print_main("potential muldiv64() overflow", p)
diff --git a/scripts/coccinelle/remove_local_err.cocci b/scripts/coccinelle/remove_local_err.cocci
new file mode 100644
index 0000000..9261c99
--- /dev/null
+++ b/scripts/coccinelle/remove_local_err.cocci
@@ -0,0 +1,29 @@
+// Replace unnecessary usage of local_err variable with
+// direct usage of errp argument
+
+@@
+identifier F;
+expression list ARGS;
+expression F2;
+identifier LOCAL_ERR;
+identifier ERRP;
+idexpression V;
+typedef Error;
+@@
+ F(..., Error **ERRP)
+ {
+     ...
+-    Error *LOCAL_ERR;
+     ... when != LOCAL_ERR
+         when != ERRP
+(
+-    F2(ARGS, &LOCAL_ERR);
+-    error_propagate(ERRP, LOCAL_ERR);
++    F2(ARGS, ERRP);
+|
+-    V = F2(ARGS, &LOCAL_ERR);
+-    error_propagate(ERRP, LOCAL_ERR);
++    V = F2(ARGS, ERRP);
+)
+     ... when != LOCAL_ERR
+ }
diff --git a/scripts/coccinelle/remove_muldiv64.cocci b/scripts/coccinelle/remove_muldiv64.cocci
new file mode 100644
index 0000000..4c10bd5
--- /dev/null
+++ b/scripts/coccinelle/remove_muldiv64.cocci
@@ -0,0 +1,6 @@
+// replace muldiv64(a, 1, b) by "a / b"
+@@
+expression a, b;
+@@
+-muldiv64(a, 1, b)
++a / b
diff --git a/scripts/coccinelle/return_directly.cocci b/scripts/coccinelle/return_directly.cocci
new file mode 100644
index 0000000..48680f2
--- /dev/null
+++ b/scripts/coccinelle/return_directly.cocci
@@ -0,0 +1,19 @@
+// replace 'R = X; return R;' with 'return R;'
+@@
+identifier VAR;
+expression E;
+type T;
+identifier F;
+@@
+ T F(...)
+ {
+     ...
+-    T VAR;
+     ... when != VAR
+
+-    VAR =
++    return
+     E;
+-    return VAR;
+     ... when != VAR
+ }
diff --git a/scripts/coccinelle/round.cocci b/scripts/coccinelle/round.cocci
new file mode 100644
index 0000000..ed06773
--- /dev/null
+++ b/scripts/coccinelle/round.cocci
@@ -0,0 +1,19 @@
+// Use macro DIV_ROUND_UP instead of (((n) + (d) - 1) /(d))
+@@
+expression e1;
+expression e2;
+@@
+(
+- ((e1) + e2 - 1) / (e2)
++ DIV_ROUND_UP(e1,e2)
+|
+- ((e1) + (e2 - 1)) / (e2)
++ DIV_ROUND_UP(e1,e2)
+)
+
+@@
+expression e1;
+expression e2;
+@@
+-(DIV_ROUND_UP(e1,e2))
++DIV_ROUND_UP(e1,e2)
diff --git a/scripts/coccinelle/simplify_muldiv64.cocci b/scripts/coccinelle/simplify_muldiv64.cocci
new file mode 100644
index 0000000..3d7c974
--- /dev/null
+++ b/scripts/coccinelle/simplify_muldiv64.cocci
@@ -0,0 +1,11 @@
+// replace muldiv64(i32, i32, x) by (uint64_t)i32 * i32 / x
+@@
+typedef uint32_t;
+typedef int32_t;
+{ uint32_t, int32_t, int, unsigned int } a, b;
+typedef uint64_t;
+expression c;
+@@
+
+-muldiv64(a,b,c)
++(uint64_t) a * b / c
diff --git a/scripts/coccinelle/swap_muldiv64.cocci b/scripts/coccinelle/swap_muldiv64.cocci
new file mode 100644
index 0000000..b48b0d0
--- /dev/null
+++ b/scripts/coccinelle/swap_muldiv64.cocci
@@ -0,0 +1,13 @@
+// replace muldiv64(i32, i64, x) by muldiv64(i64, i32, x)
+@@
+typedef uint64_t;
+typedef int64_t;
+typedef uint32_t;
+typedef int32_t;
+{ uint32_t, int32_t, int, unsigned int } a;
+{ uint64_t, int64_t, long, unsigned long } b;
+expression c;
+@@
+
+-muldiv64(a,b,c)
++muldiv64(b,a,c)
diff --git a/scripts/coverity-model.c b/scripts/coverity-model.c
index 4c99a85..ee5bf9d 100644
--- a/scripts/coverity-model.c
+++ b/scripts/coverity-model.c
@@ -40,12 +40,16 @@
 typedef long long int64_t;
 typedef _Bool bool;
 
+typedef struct va_list_str *va_list;
+
 /* exec.c */
 
 typedef struct AddressSpace AddressSpace;
 typedef uint64_t hwaddr;
+typedef uint32_t MemTxResult;
+typedef uint64_t MemTxAttrs;
 
-static void __write(uint8_t *buf, ssize_t len)
+static void __bufwrite(uint8_t *buf, ssize_t len)
 {
     int first, last;
     __coverity_negative_sink__(len);
@@ -55,7 +59,7 @@
     __coverity_writeall__(buf);
 }
 
-static void __read(uint8_t *buf, ssize_t len)
+static void __bufread(uint8_t *buf, ssize_t len)
 {
     __coverity_negative_sink__(len);
     if (len == 0) return;
@@ -63,14 +67,14 @@
     int last = buf[len-1];
 }
 
-bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
-                      int len, bool is_write)
+MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs,
+                             uint8_t *buf, int len, bool is_write)
 {
-    bool result;
+    MemTxResult result;
 
     // TODO: investigate impact of treating reads as producing
     // tainted data, with __coverity_tainted_data_argument__(buf).
-    if (is_write) __write(buf, len); else __read(buf, len);
+    if (is_write) __bufread(buf, len); else __bufwrite(buf, len);
 
     return result;
 }
@@ -90,7 +94,8 @@
     }
 }
 
-/* glib memory allocation functions.
+/*
+ * GLib memory allocation functions.
  *
  * Note that we ignore the fact that g_malloc of 0 bytes returns NULL,
  * and g_realloc of 0 bytes frees the pointer.
@@ -107,64 +112,245 @@
  * we'll get a buffer overflow reported anyway.
  */
 
-void *malloc(size_t);
-void *calloc(size_t, size_t);
-void *realloc(void *, size_t);
-void free(void *);
+/*
+ * Allocation primitives, cannot return NULL
+ * See also Coverity's library/generic/libc/all/all.c
+ */
 
-void *
-g_malloc(size_t n_bytes)
+void *g_malloc_n(size_t nmemb, size_t size)
 {
-    void *mem;
-    __coverity_negative_sink__(n_bytes);
-    mem = malloc(n_bytes == 0 ? 1 : n_bytes);
-    if (!mem) __coverity_panic__();
-    return mem;
+    size_t sz;
+    void *ptr;
+
+    __coverity_negative_sink__(nmemb);
+    __coverity_negative_sink__(size);
+    sz = nmemb * size;
+    ptr = __coverity_alloc__(sz);
+    __coverity_mark_as_uninitialized_buffer__(ptr);
+    __coverity_mark_as_afm_allocated__(ptr, "g_free");
+    return ptr;
 }
 
-void *
-g_malloc0(size_t n_bytes)
+void *g_malloc0_n(size_t nmemb, size_t size)
 {
-    void *mem;
-    __coverity_negative_sink__(n_bytes);
-    mem = calloc(1, n_bytes == 0 ? 1 : n_bytes);
-    if (!mem) __coverity_panic__();
-    return mem;
+    size_t sz;
+    void *ptr;
+
+    __coverity_negative_sink__(nmemb);
+    __coverity_negative_sink__(size);
+    sz = nmemb * size;
+    ptr = __coverity_alloc__(sz);
+    __coverity_writeall0__(ptr);
+    __coverity_mark_as_afm_allocated__(ptr, "g_free");
+    return ptr;
 }
 
-void g_free(void *mem)
+void *g_realloc_n(void *ptr, size_t nmemb, size_t size)
 {
-    free(mem);
+    size_t sz;
+
+    __coverity_negative_sink__(nmemb);
+    __coverity_negative_sink__(size);
+    sz = nmemb * size;
+    __coverity_escape__(ptr);
+    ptr = __coverity_alloc__(sz);
+    /*
+     * Memory beyond the old size isn't actually initialized.  Can't
+     * model that.  See Coverity's realloc() model
+     */
+    __coverity_writeall__(ptr);
+    __coverity_mark_as_afm_allocated__(ptr, "g_free");
+    return ptr;
 }
 
-void *g_realloc(void * mem, size_t n_bytes)
+void g_free(void *ptr)
 {
-    __coverity_negative_sink__(n_bytes);
-    mem = realloc(mem, n_bytes == 0 ? 1 : n_bytes);
-    if (!mem) __coverity_panic__();
-    return mem;
+    __coverity_free__(ptr);
+    __coverity_mark_as_afm_freed__(ptr, "g_free");
 }
 
-void *g_try_malloc(size_t n_bytes)
+/*
+ * Derive the g_try_FOO_n() from the g_FOO_n() by adding indeterminate
+ * out of memory conditions
+ */
+
+void *g_try_malloc_n(size_t nmemb, size_t size)
 {
-    __coverity_negative_sink__(n_bytes);
-    return malloc(n_bytes == 0 ? 1 : n_bytes);
+    int nomem;
+
+    if (nomem) {
+        return NULL;
+    }
+    return g_malloc_n(nmemb, size);
 }
 
-void *g_try_malloc0(size_t n_bytes)
+void *g_try_malloc0_n(size_t nmemb, size_t size)
 {
-    __coverity_negative_sink__(n_bytes);
-    return calloc(1, n_bytes == 0 ? 1 : n_bytes);
+    int nomem;
+
+    if (nomem) {
+        return NULL;
+    }
+    return g_malloc0_n(nmemb, size);
 }
 
-void *g_try_realloc(void *mem, size_t n_bytes)
+void *g_try_realloc_n(void *ptr, size_t nmemb, size_t size)
 {
-    __coverity_negative_sink__(n_bytes);
-    return realloc(mem, n_bytes == 0 ? 1 : n_bytes);
+    int nomem;
+
+    if (nomem) {
+        return NULL;
+    }
+    return g_realloc_n(ptr, nmemb, size);
+}
+
+/* Trivially derive the g_FOO() from the g_FOO_n() */
+
+void *g_malloc(size_t size)
+{
+    return g_malloc_n(1, size);
+}
+
+void *g_malloc0(size_t size)
+{
+    return g_malloc0_n(1, size);
+}
+
+void *g_realloc(void *ptr, size_t size)
+{
+    return g_realloc_n(ptr, 1, size);
+}
+
+void *g_try_malloc(size_t size)
+{
+    return g_try_malloc_n(1, size);
+}
+
+void *g_try_malloc0(size_t size)
+{
+    return g_try_malloc0_n(1, size);
+}
+
+void *g_try_realloc(void *ptr, size_t size)
+{
+    return g_try_realloc_n(ptr, 1, size);
+}
+
+/* Other memory allocation functions */
+
+void *g_memdup(const void *ptr, unsigned size)
+{
+    unsigned char *dup;
+    unsigned i;
+
+    if (!ptr) {
+        return NULL;
+    }
+
+    dup = g_malloc(size);
+    for (i = 0; i < size; i++)
+        dup[i] = ((unsigned char *)ptr)[i];
+    return dup;
+}
+
+/*
+ * GLib string allocation functions
+ */
+
+char *g_strdup(const char *s)
+{
+    char *dup;
+    size_t i;
+
+    if (!s) {
+        return NULL;
+    }
+
+    __coverity_string_null_sink__(s);
+    __coverity_string_size_sink__(s);
+    dup = __coverity_alloc_nosize__();
+    __coverity_mark_as_afm_allocated__(dup, "g_free");
+    for (i = 0; (dup[i] = s[i]); i++) ;
+    return dup;
+}
+
+char *g_strndup(const char *s, size_t n)
+{
+    char *dup;
+    size_t i;
+
+    __coverity_negative_sink__(n);
+
+    if (!s) {
+        return NULL;
+    }
+
+    dup = g_malloc(n + 1);
+    for (i = 0; i < n && (dup[i] = s[i]); i++) ;
+    dup[i] = 0;
+    return dup;
+}
+
+char *g_strdup_printf(const char *format, ...)
+{
+    char ch, *s;
+    size_t len;
+
+    __coverity_string_null_sink__(format);
+    __coverity_string_size_sink__(format);
+
+    ch = *format;
+
+    s = __coverity_alloc_nosize__();
+    __coverity_writeall__(s);
+    __coverity_mark_as_afm_allocated__(s, "g_free");
+    return s;
+}
+
+char *g_strdup_vprintf(const char *format, va_list ap)
+{
+    char ch, *s;
+    size_t len;
+
+    __coverity_string_null_sink__(format);
+    __coverity_string_size_sink__(format);
+
+    ch = *format;
+    ch = *(char *)ap;
+
+    s = __coverity_alloc_nosize__();
+    __coverity_writeall__(s);
+    __coverity_mark_as_afm_allocated__(s, "g_free");
+
+    return len;
+}
+
+char *g_strconcat(const char *s, ...)
+{
+    char *s;
+
+    /*
+     * Can't model: last argument must be null, the others
+     * null-terminated strings
+     */
+
+    s = __coverity_alloc_nosize__();
+    __coverity_writeall__(s);
+    __coverity_mark_as_afm_allocated__(s, "g_free");
+    return s;
 }
 
 /* Other glib functions */
 
+typedef struct pollfd GPollFD;
+
+int poll();
+
+int g_poll (GPollFD *fds, unsigned nfds, int timeout)
+{
+    return poll(fds, nfds, timeout);
+}
+
 typedef struct _GIOChannel GIOChannel;
 GIOChannel *g_io_channel_unix_new(int fd)
 {
diff --git a/scripts/create_config b/scripts/create_config
index 546f889..1dd6a35 100755
--- a/scripts/create_config
+++ b/scripts/create_config
@@ -9,14 +9,10 @@
     version=${line#*=}
     echo "#define QEMU_VERSION \"$version\""
     ;;
- PKGVERSION=*) # configuration
-    pkgversion=${line#*=}
-    echo "#define QEMU_PKGVERSION \"$pkgversion\""
-    ;;
  qemu_*dir=*) # qemu-specific directory configuration
     name=${line%=*}
     value=${line#*=}
-    define_name=`echo $name | LC_ALL=C tr '[a-z]' '[A-Z]'`
+    define_name=$(echo $name | LC_ALL=C tr '[a-z]' '[A-Z]')
     eval "define_value=\"$value\""
     echo "#define CONFIG_$define_name \"$define_value\""
     # save for the next definitions
@@ -52,7 +48,7 @@
     done
     echo "    NULL"
     ;;
- CONFIG_*=y) # configuration
+ CONFIG_*='$(CONFIG_SOFTMMU)'|CONFIG_*=y) # configuration
     name=${line%=*}
     echo "#define $name 1"
     ;;
@@ -61,9 +57,18 @@
     value=${line#*=}
     echo "#define $name $value"
     ;;
+ HAVE_*=y) # configuration
+    name=${line%=*}
+    echo "#define $name 1"
+    ;;
+ HAVE_*=*) # configuration
+    name=${line%=*}
+    value=${line#*=}
+    echo "#define $name $value"
+    ;;
  ARCH=*) # configuration
     arch=${line#*=}
-    arch_name=`echo $arch | LC_ALL=C tr '[a-z]' '[A-Z]'`
+    arch_name=$(echo $arch | LC_ALL=C tr '[a-z]' '[A-Z]')
     echo "#define HOST_$arch_name 1"
     ;;
  HOST_USB=*)
@@ -83,7 +88,7 @@
     ;;
  TARGET_BASE_ARCH=*) # configuration
     target_base_arch=${line#*=}
-    base_arch_name=`echo $target_base_arch | LC_ALL=C tr '[a-z]' '[A-Z]'`
+    base_arch_name=$(echo $target_base_arch | LC_ALL=C tr '[a-z]' '[A-Z]')
     echo "#define TARGET_$base_arch_name 1"
     ;;
  TARGET_XML_FILES=*)
diff --git a/scripts/dump-guest-memory.py b/scripts/dump-guest-memory.py
index 1ed8b67..9956fc0 100644
--- a/scripts/dump-guest-memory.py
+++ b/scripts/dump-guest-memory.py
@@ -1,39 +1,443 @@
-# This python script adds a new gdb command, "dump-guest-memory". It
-# should be loaded with "source dump-guest-memory.py" at the (gdb)
-# prompt.
-#
-# Copyright (C) 2013, Red Hat, Inc.
-#
-# Authors:
-#   Laszlo Ersek <lersek@redhat.com>
-#
-# This work is licensed under the terms of the GNU GPL, version 2 or later. See
-# the COPYING file in the top-level directory.
-#
+"""
+This python script adds a new gdb command, "dump-guest-memory". It
+should be loaded with "source dump-guest-memory.py" at the (gdb)
+prompt.
+
+Copyright (C) 2013, Red Hat, Inc.
+
+Authors:
+   Laszlo Ersek <lersek@redhat.com>
+   Janosch Frank <frankja@linux.vnet.ibm.com>
+
+This work is licensed under the terms of the GNU GPL, version 2 or later. See
+the COPYING file in the top-level directory.
+"""
+
+import ctypes
+
+UINTPTR_T = gdb.lookup_type("uintptr_t")
+
+TARGET_PAGE_SIZE = 0x1000
+TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000
+
+# Special value for e_phnum. This indicates that the real number of
+# program headers is too large to fit into e_phnum. Instead the real
+# value is in the field sh_info of section 0.
+PN_XNUM = 0xFFFF
+
+EV_CURRENT = 1
+
+ELFCLASS32 = 1
+ELFCLASS64 = 2
+
+ELFDATA2LSB = 1
+ELFDATA2MSB = 2
+
+ET_CORE = 4
+
+PT_LOAD = 1
+PT_NOTE = 4
+
+EM_386 = 3
+EM_PPC = 20
+EM_PPC64 = 21
+EM_S390 = 22
+EM_AARCH = 183
+EM_X86_64 = 62
+
+class ELF(object):
+    """Representation of a ELF file."""
+
+    def __init__(self, arch):
+        self.ehdr = None
+        self.notes = []
+        self.segments = []
+        self.notes_size = 0
+        self.endianness = None
+        self.elfclass = ELFCLASS64
+
+        if arch == 'aarch64-le':
+            self.endianness = ELFDATA2LSB
+            self.elfclass = ELFCLASS64
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_AARCH
+
+        elif arch == 'aarch64-be':
+            self.endianness = ELFDATA2MSB
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_AARCH
+
+        elif arch == 'X86_64':
+            self.endianness = ELFDATA2LSB
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_X86_64
+
+        elif arch == '386':
+            self.endianness = ELFDATA2LSB
+            self.elfclass = ELFCLASS32
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_386
+
+        elif arch == 's390':
+            self.endianness = ELFDATA2MSB
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_S390
+
+        elif arch == 'ppc64-le':
+            self.endianness = ELFDATA2LSB
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_PPC64
+
+        elif arch == 'ppc64-be':
+            self.endianness = ELFDATA2MSB
+            self.ehdr = get_arch_ehdr(self.endianness, self.elfclass)
+            self.ehdr.e_machine = EM_PPC64
+
+        else:
+            raise gdb.GdbError("No valid arch type specified.\n"
+                               "Currently supported types:\n"
+                               "aarch64-be, aarch64-le, X86_64, 386, s390, "
+                               "ppc64-be, ppc64-le")
+
+        self.add_segment(PT_NOTE, 0, 0)
+
+    def add_note(self, n_name, n_desc, n_type):
+        """Adds a note to the ELF."""
+
+        note = get_arch_note(self.endianness, len(n_name), len(n_desc))
+        note.n_namesz = len(n_name) + 1
+        note.n_descsz = len(n_desc)
+        note.n_name = n_name.encode()
+        note.n_type = n_type
+
+        # Desc needs to be 4 byte aligned (although the 64bit spec
+        # specifies 8 byte). When defining n_desc as uint32 it will be
+        # automatically aligned but we need the memmove to copy the
+        # string into it.
+        ctypes.memmove(note.n_desc, n_desc.encode(), len(n_desc))
+
+        self.notes.append(note)
+        self.segments[0].p_filesz += ctypes.sizeof(note)
+        self.segments[0].p_memsz += ctypes.sizeof(note)
+
+    def add_segment(self, p_type, p_paddr, p_size):
+        """Adds a segment to the elf."""
+
+        phdr = get_arch_phdr(self.endianness, self.elfclass)
+        phdr.p_type = p_type
+        phdr.p_paddr = p_paddr
+        phdr.p_filesz = p_size
+        phdr.p_memsz = p_size
+        self.segments.append(phdr)
+        self.ehdr.e_phnum += 1
+
+    def to_file(self, elf_file):
+        """Writes all ELF structures to the the passed file.
+
+        Structure:
+        Ehdr
+        Segment 0:PT_NOTE
+        Segment 1:PT_LOAD
+        Segment N:PT_LOAD
+        Note    0..N
+        Dump contents
+        """
+        elf_file.write(self.ehdr)
+        off = ctypes.sizeof(self.ehdr) + \
+              len(self.segments) * ctypes.sizeof(self.segments[0])
+
+        for phdr in self.segments:
+            phdr.p_offset = off
+            elf_file.write(phdr)
+            off += phdr.p_filesz
+
+        for note in self.notes:
+            elf_file.write(note)
+
+
+def get_arch_note(endianness, len_name, len_desc):
+    """Returns a Note class with the specified endianness."""
+
+    if endianness == ELFDATA2LSB:
+        superclass = ctypes.LittleEndianStructure
+    else:
+        superclass = ctypes.BigEndianStructure
+
+    len_name = len_name + 1
+
+    class Note(superclass):
+        """Represents an ELF note, includes the content."""
+
+        _fields_ = [("n_namesz", ctypes.c_uint32),
+                    ("n_descsz", ctypes.c_uint32),
+                    ("n_type", ctypes.c_uint32),
+                    ("n_name", ctypes.c_char * len_name),
+                    ("n_desc", ctypes.c_uint32 * ((len_desc + 3) // 4))]
+    return Note()
+
+
+class Ident(ctypes.Structure):
+    """Represents the ELF ident array in the ehdr structure."""
+
+    _fields_ = [('ei_mag0', ctypes.c_ubyte),
+                ('ei_mag1', ctypes.c_ubyte),
+                ('ei_mag2', ctypes.c_ubyte),
+                ('ei_mag3', ctypes.c_ubyte),
+                ('ei_class', ctypes.c_ubyte),
+                ('ei_data', ctypes.c_ubyte),
+                ('ei_version', ctypes.c_ubyte),
+                ('ei_osabi', ctypes.c_ubyte),
+                ('ei_abiversion', ctypes.c_ubyte),
+                ('ei_pad', ctypes.c_ubyte * 7)]
+
+    def __init__(self, endianness, elfclass):
+        self.ei_mag0 = 0x7F
+        self.ei_mag1 = ord('E')
+        self.ei_mag2 = ord('L')
+        self.ei_mag3 = ord('F')
+        self.ei_class = elfclass
+        self.ei_data = endianness
+        self.ei_version = EV_CURRENT
+
+
+def get_arch_ehdr(endianness, elfclass):
+    """Returns a EHDR64 class with the specified endianness."""
+
+    if endianness == ELFDATA2LSB:
+        superclass = ctypes.LittleEndianStructure
+    else:
+        superclass = ctypes.BigEndianStructure
+
+    class EHDR64(superclass):
+        """Represents the 64 bit ELF header struct."""
+
+        _fields_ = [('e_ident', Ident),
+                    ('e_type', ctypes.c_uint16),
+                    ('e_machine', ctypes.c_uint16),
+                    ('e_version', ctypes.c_uint32),
+                    ('e_entry', ctypes.c_uint64),
+                    ('e_phoff', ctypes.c_uint64),
+                    ('e_shoff', ctypes.c_uint64),
+                    ('e_flags', ctypes.c_uint32),
+                    ('e_ehsize', ctypes.c_uint16),
+                    ('e_phentsize', ctypes.c_uint16),
+                    ('e_phnum', ctypes.c_uint16),
+                    ('e_shentsize', ctypes.c_uint16),
+                    ('e_shnum', ctypes.c_uint16),
+                    ('e_shstrndx', ctypes.c_uint16)]
+
+        def __init__(self):
+            super(superclass, self).__init__()
+            self.e_ident = Ident(endianness, elfclass)
+            self.e_type = ET_CORE
+            self.e_version = EV_CURRENT
+            self.e_ehsize = ctypes.sizeof(self)
+            self.e_phoff = ctypes.sizeof(self)
+            self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianness, elfclass))
+            self.e_phnum = 0
+
+
+    class EHDR32(superclass):
+        """Represents the 32 bit ELF header struct."""
+
+        _fields_ = [('e_ident', Ident),
+                    ('e_type', ctypes.c_uint16),
+                    ('e_machine', ctypes.c_uint16),
+                    ('e_version', ctypes.c_uint32),
+                    ('e_entry', ctypes.c_uint32),
+                    ('e_phoff', ctypes.c_uint32),
+                    ('e_shoff', ctypes.c_uint32),
+                    ('e_flags', ctypes.c_uint32),
+                    ('e_ehsize', ctypes.c_uint16),
+                    ('e_phentsize', ctypes.c_uint16),
+                    ('e_phnum', ctypes.c_uint16),
+                    ('e_shentsize', ctypes.c_uint16),
+                    ('e_shnum', ctypes.c_uint16),
+                    ('e_shstrndx', ctypes.c_uint16)]
+
+        def __init__(self):
+            super(superclass, self).__init__()
+            self.e_ident = Ident(endianness, elfclass)
+            self.e_type = ET_CORE
+            self.e_version = EV_CURRENT
+            self.e_ehsize = ctypes.sizeof(self)
+            self.e_phoff = ctypes.sizeof(self)
+            self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianness, elfclass))
+            self.e_phnum = 0
+
+    # End get_arch_ehdr
+    if elfclass == ELFCLASS64:
+        return EHDR64()
+    else:
+        return EHDR32()
+
+
+def get_arch_phdr(endianness, elfclass):
+    """Returns a 32 or 64 bit PHDR class with the specified endianness."""
+
+    if endianness == ELFDATA2LSB:
+        superclass = ctypes.LittleEndianStructure
+    else:
+        superclass = ctypes.BigEndianStructure
+
+    class PHDR64(superclass):
+        """Represents the 64 bit ELF program header struct."""
+
+        _fields_ = [('p_type', ctypes.c_uint32),
+                    ('p_flags', ctypes.c_uint32),
+                    ('p_offset', ctypes.c_uint64),
+                    ('p_vaddr', ctypes.c_uint64),
+                    ('p_paddr', ctypes.c_uint64),
+                    ('p_filesz', ctypes.c_uint64),
+                    ('p_memsz', ctypes.c_uint64),
+                    ('p_align', ctypes.c_uint64)]
+
+    class PHDR32(superclass):
+        """Represents the 32 bit ELF program header struct."""
+
+        _fields_ = [('p_type', ctypes.c_uint32),
+                    ('p_offset', ctypes.c_uint32),
+                    ('p_vaddr', ctypes.c_uint32),
+                    ('p_paddr', ctypes.c_uint32),
+                    ('p_filesz', ctypes.c_uint32),
+                    ('p_memsz', ctypes.c_uint32),
+                    ('p_flags', ctypes.c_uint32),
+                    ('p_align', ctypes.c_uint32)]
+
+    # End get_arch_phdr
+    if elfclass == ELFCLASS64:
+        return PHDR64()
+    else:
+        return PHDR32()
+
+
+def int128_get64(val):
+    """Returns low 64bit part of Int128 struct."""
+
+    assert val["hi"] == 0
+    return val["lo"]
+
+
+def qlist_foreach(head, field_str):
+    """Generator for qlists."""
+
+    var_p = head["lh_first"]
+    while var_p != 0:
+        var = var_p.dereference()
+        var_p = var[field_str]["le_next"]
+        yield var
+
+
+def qemu_map_ram_ptr(block, offset):
+    """Returns qemu vaddr for given guest physical address."""
+
+    return block["host"] + offset
+
+
+def memory_region_get_ram_ptr(memory_region):
+    if memory_region["alias"] != 0:
+        return (memory_region_get_ram_ptr(memory_region["alias"].dereference())
+                + memory_region["alias_offset"])
+
+    return qemu_map_ram_ptr(memory_region["ram_block"], 0)
+
+
+def get_guest_phys_blocks():
+    """Returns a list of ram blocks.
+
+    Each block entry contains:
+    'target_start': guest block phys start address
+    'target_end':   guest block phys end address
+    'host_addr':    qemu vaddr of the block's start
+    """
+
+    guest_phys_blocks = []
+
+    print("guest RAM blocks:")
+    print("target_start     target_end       host_addr        message "
+          "count")
+    print("---------------- ---------------- ---------------- ------- "
+          "-----")
+
+    current_map_p = gdb.parse_and_eval("address_space_memory.current_map")
+    current_map = current_map_p.dereference()
+
+    # Conversion to int is needed for python 3
+    # compatibility. Otherwise range doesn't cast the value itself and
+    # breaks.
+    for cur in range(int(current_map["nr"])):
+        flat_range = (current_map["ranges"] + cur).dereference()
+        memory_region = flat_range["mr"].dereference()
+
+        # we only care about RAM
+        if not memory_region["ram"]:
+            continue
+
+        section_size = int128_get64(flat_range["addr"]["size"])
+        target_start = int128_get64(flat_range["addr"]["start"])
+        target_end = target_start + section_size
+        host_addr = (memory_region_get_ram_ptr(memory_region)
+                     + flat_range["offset_in_region"])
+        predecessor = None
+
+        # find continuity in guest physical address space
+        if len(guest_phys_blocks) > 0:
+            predecessor = guest_phys_blocks[-1]
+            predecessor_size = (predecessor["target_end"] -
+                                predecessor["target_start"])
+
+            # the memory API guarantees monotonically increasing
+            # traversal
+            assert predecessor["target_end"] <= target_start
+
+            # we want continuity in both guest-physical and
+            # host-virtual memory
+            if (predecessor["target_end"] < target_start or
+                predecessor["host_addr"] + predecessor_size != host_addr):
+                predecessor = None
+
+        if predecessor is None:
+            # isolated mapping, add it to the list
+            guest_phys_blocks.append({"target_start": target_start,
+                                      "target_end":   target_end,
+                                      "host_addr":    host_addr})
+            message = "added"
+        else:
+            # expand predecessor until @target_end; predecessor's
+            # start doesn't change
+            predecessor["target_end"] = target_end
+            message = "joined"
+
+        print("%016x %016x %016x %-7s %5u" %
+              (target_start, target_end, host_addr.cast(UINTPTR_T),
+               message, len(guest_phys_blocks)))
+
+    return guest_phys_blocks
+
+
 # The leading docstring doesn't have idiomatic Python formatting. It is
 # printed by gdb's "help" command (the first line is printed in the
 # "help data" summary), and it should match how other help texts look in
 # gdb.
-
-import struct
-
 class DumpGuestMemory(gdb.Command):
     """Extract guest vmcore from qemu process coredump.
 
-The sole argument is FILE, identifying the target file to write the
-guest vmcore to.
+The two required arguments are FILE and ARCH:
+FILE identifies the target file to write the guest vmcore to.
+ARCH specifies the architecture for which the core will be generated.
 
 This GDB command reimplements the dump-guest-memory QMP command in
 python, using the representation of guest memory as captured in the qemu
 coredump. The qemu process that has been dumped must have had the
-command line option "-machine dump-guest-core=on".
+command line option "-machine dump-guest-core=on" which is the default.
 
 For simplicity, the "paging", "begin" and "end" parameters of the QMP
 command are not supported -- no attempt is made to get the guest's
 internal paging structures (ie. paging=false is hard-wired), and guest
 memory is always fully dumped.
 
-Only x86_64 guests are supported.
+Currently aarch64-be, aarch64-le, X86_64, 386, s390, ppc64-be,
+ppc64-le guests are supported.
 
 The CORE/NT_PRSTATUS and QEMU notes (that is, the VCPUs' statuses) are
 not written to the vmcore. Preparing these would require context that is
@@ -47,293 +451,66 @@
 a halfway fortunate point, then its coredump should be in reasonable
 shape and this command should mostly work."""
 
-    TARGET_PAGE_SIZE = 0x1000
-    TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000
-
-    # Various ELF constants
-    EM_X86_64   = 62        # AMD x86-64 target machine
-    ELFDATA2LSB = 1         # little endian
-    ELFCLASS64  = 2
-    ELFMAG      = "\x7FELF"
-    EV_CURRENT  = 1
-    ET_CORE     = 4
-    PT_LOAD     = 1
-    PT_NOTE     = 4
-
-    # Special value for e_phnum. This indicates that the real number of
-    # program headers is too large to fit into e_phnum. Instead the real
-    # value is in the field sh_info of section 0.
-    PN_XNUM = 0xFFFF
-
-    # Format strings for packing and header size calculation.
-    ELF64_EHDR = ("4s" # e_ident/magic
-                  "B"  # e_ident/class
-                  "B"  # e_ident/data
-                  "B"  # e_ident/version
-                  "B"  # e_ident/osabi
-                  "8s" # e_ident/pad
-                  "H"  # e_type
-                  "H"  # e_machine
-                  "I"  # e_version
-                  "Q"  # e_entry
-                  "Q"  # e_phoff
-                  "Q"  # e_shoff
-                  "I"  # e_flags
-                  "H"  # e_ehsize
-                  "H"  # e_phentsize
-                  "H"  # e_phnum
-                  "H"  # e_shentsize
-                  "H"  # e_shnum
-                  "H"  # e_shstrndx
-                 )
-    ELF64_PHDR = ("I"  # p_type
-                  "I"  # p_flags
-                  "Q"  # p_offset
-                  "Q"  # p_vaddr
-                  "Q"  # p_paddr
-                  "Q"  # p_filesz
-                  "Q"  # p_memsz
-                  "Q"  # p_align
-                 )
-
     def __init__(self):
         super(DumpGuestMemory, self).__init__("dump-guest-memory",
                                               gdb.COMMAND_DATA,
                                               gdb.COMPLETE_FILENAME)
-        self.uintptr_t     = gdb.lookup_type("uintptr_t")
-        self.elf64_ehdr_le = struct.Struct("<%s" % self.ELF64_EHDR)
-        self.elf64_phdr_le = struct.Struct("<%s" % self.ELF64_PHDR)
+        self.elf = None
+        self.guest_phys_blocks = None
 
-    def int128_get64(self, val):
-        assert (val["hi"] == 0)
-        return val["lo"]
+    def dump_init(self, vmcore):
+        """Prepares and writes ELF structures to core file."""
 
-    def qtailq_foreach(self, head, field_str):
-        var_p = head["tqh_first"]
-        while (var_p != 0):
-            var = var_p.dereference()
-            yield var
-            var_p = var[field_str]["tqe_next"]
+        # Needed to make crash happy, data for more useful notes is
+        # not available in a qemu core.
+        self.elf.add_note("NONE", "EMPTY", 0)
 
-    def qemu_get_ram_block(self, ram_addr):
-        ram_blocks = gdb.parse_and_eval("ram_list.blocks")
-        for block in self.qtailq_foreach(ram_blocks, "next"):
-            if (ram_addr - block["offset"] < block["length"]):
-                return block
-        raise gdb.GdbError("Bad ram offset %x" % ram_addr)
+        # We should never reach PN_XNUM for paging=false dumps,
+        # there's just a handful of discontiguous ranges after
+        # merging.
+        # The constant is needed to account for the PT_NOTE segment.
+        phdr_num = len(self.guest_phys_blocks) + 1
+        assert phdr_num < PN_XNUM
 
-    def qemu_get_ram_ptr(self, ram_addr):
-        block = self.qemu_get_ram_block(ram_addr)
-        return block["host"] + (ram_addr - block["offset"])
-
-    def memory_region_get_ram_ptr(self, mr):
-        if (mr["alias"] != 0):
-            return (self.memory_region_get_ram_ptr(mr["alias"].dereference()) +
-                    mr["alias_offset"])
-        return self.qemu_get_ram_ptr(mr["ram_addr"] & self.TARGET_PAGE_MASK)
-
-    def guest_phys_blocks_init(self):
-        self.guest_phys_blocks = []
-
-    def guest_phys_blocks_append(self):
-        print "guest RAM blocks:"
-        print ("target_start     target_end       host_addr        message "
-               "count")
-        print ("---------------- ---------------- ---------------- ------- "
-               "-----")
-
-        current_map_p = gdb.parse_and_eval("address_space_memory.current_map")
-        current_map = current_map_p.dereference()
-        for cur in range(current_map["nr"]):
-            flat_range   = (current_map["ranges"] + cur).dereference()
-            mr           = flat_range["mr"].dereference()
-
-            # we only care about RAM
-            if (not mr["ram"]):
-                continue
-
-            section_size = self.int128_get64(flat_range["addr"]["size"])
-            target_start = self.int128_get64(flat_range["addr"]["start"])
-            target_end   = target_start + section_size
-            host_addr    = (self.memory_region_get_ram_ptr(mr) +
-                            flat_range["offset_in_region"])
-            predecessor = None
-
-            # find continuity in guest physical address space
-            if (len(self.guest_phys_blocks) > 0):
-                predecessor = self.guest_phys_blocks[-1]
-                predecessor_size = (predecessor["target_end"] -
-                                    predecessor["target_start"])
-
-                # the memory API guarantees monotonically increasing
-                # traversal
-                assert (predecessor["target_end"] <= target_start)
-
-                # we want continuity in both guest-physical and
-                # host-virtual memory
-                if (predecessor["target_end"] < target_start or
-                    predecessor["host_addr"] + predecessor_size != host_addr):
-                    predecessor = None
-
-            if (predecessor is None):
-                # isolated mapping, add it to the list
-                self.guest_phys_blocks.append({"target_start": target_start,
-                                               "target_end"  : target_end,
-                                               "host_addr"   : host_addr})
-                message = "added"
-            else:
-                # expand predecessor until @target_end; predecessor's
-                # start doesn't change
-                predecessor["target_end"] = target_end
-                message = "joined"
-
-            print ("%016x %016x %016x %-7s %5u" %
-                   (target_start, target_end, host_addr.cast(self.uintptr_t),
-                    message, len(self.guest_phys_blocks)))
-
-    def cpu_get_dump_info(self):
-        # We can't synchronize the registers with KVM post-mortem, and
-        # the bits in (first_x86_cpu->env.hflags) seem to be stale; they
-        # may not reflect long mode for example. Hence just assume the
-        # most common values. This also means that instruction pointer
-        # etc. will be bogus in the dump, but at least the RAM contents
-        # should be valid.
-        self.dump_info = {"d_machine": self.EM_X86_64,
-                          "d_endian" : self.ELFDATA2LSB,
-                          "d_class"  : self.ELFCLASS64}
-
-    def encode_elf64_ehdr_le(self):
-        return self.elf64_ehdr_le.pack(
-                                 self.ELFMAG,                 # e_ident/magic
-                                 self.dump_info["d_class"],   # e_ident/class
-                                 self.dump_info["d_endian"],  # e_ident/data
-                                 self.EV_CURRENT,             # e_ident/version
-                                 0,                           # e_ident/osabi
-                                 "",                          # e_ident/pad
-                                 self.ET_CORE,                # e_type
-                                 self.dump_info["d_machine"], # e_machine
-                                 self.EV_CURRENT,             # e_version
-                                 0,                           # e_entry
-                                 self.elf64_ehdr_le.size,     # e_phoff
-                                 0,                           # e_shoff
-                                 0,                           # e_flags
-                                 self.elf64_ehdr_le.size,     # e_ehsize
-                                 self.elf64_phdr_le.size,     # e_phentsize
-                                 self.phdr_num,               # e_phnum
-                                 0,                           # e_shentsize
-                                 0,                           # e_shnum
-                                 0                            # e_shstrndx
-                                )
-
-    def encode_elf64_note_le(self):
-        return self.elf64_phdr_le.pack(self.PT_NOTE,         # p_type
-                                       0,                    # p_flags
-                                       (self.memory_offset -
-                                        len(self.note)),     # p_offset
-                                       0,                    # p_vaddr
-                                       0,                    # p_paddr
-                                       len(self.note),       # p_filesz
-                                       len(self.note),       # p_memsz
-                                       0                     # p_align
-                                      )
-
-    def encode_elf64_load_le(self, offset, start_hwaddr, range_size):
-        return self.elf64_phdr_le.pack(self.PT_LOAD, # p_type
-                                       0,            # p_flags
-                                       offset,       # p_offset
-                                       0,            # p_vaddr
-                                       start_hwaddr, # p_paddr
-                                       range_size,   # p_filesz
-                                       range_size,   # p_memsz
-                                       0             # p_align
-                                      )
-
-    def note_init(self, name, desc, type):
-        # name must include a trailing NUL
-        namesz = (len(name) + 1 + 3) / 4 * 4
-        descsz = (len(desc)     + 3) / 4 * 4
-        fmt = ("<"   # little endian
-               "I"   # n_namesz
-               "I"   # n_descsz
-               "I"   # n_type
-               "%us" # name
-               "%us" # desc
-               % (namesz, descsz))
-        self.note = struct.pack(fmt,
-                                len(name) + 1, len(desc), type, name, desc)
-
-    def dump_init(self):
-        self.guest_phys_blocks_init()
-        self.guest_phys_blocks_append()
-        self.cpu_get_dump_info()
-        # we have no way to retrieve the VCPU status from KVM
-        # post-mortem
-        self.note_init("NONE", "EMPTY", 0)
-
-        # Account for PT_NOTE.
-        self.phdr_num = 1
-
-        # We should never reach PN_XNUM for paging=false dumps: there's
-        # just a handful of discontiguous ranges after merging.
-        self.phdr_num += len(self.guest_phys_blocks)
-        assert (self.phdr_num < self.PN_XNUM)
-
-        # Calculate the ELF file offset where the memory dump commences:
-        #
-        #   ELF header
-        #   PT_NOTE
-        #   PT_LOAD: 1
-        #   PT_LOAD: 2
-        #   ...
-        #   PT_LOAD: len(self.guest_phys_blocks)
-        #   ELF note
-        #   memory dump
-        self.memory_offset = (self.elf64_ehdr_le.size +
-                              self.elf64_phdr_le.size * self.phdr_num +
-                              len(self.note))
-
-    def dump_begin(self, vmcore):
-        vmcore.write(self.encode_elf64_ehdr_le())
-        vmcore.write(self.encode_elf64_note_le())
-        running = self.memory_offset
         for block in self.guest_phys_blocks:
-            range_size = block["target_end"] - block["target_start"]
-            vmcore.write(self.encode_elf64_load_le(running,
-                                                   block["target_start"],
-                                                   range_size))
-            running += range_size
-        vmcore.write(self.note)
+            block_size = block["target_end"] - block["target_start"]
+            self.elf.add_segment(PT_LOAD, block["target_start"], block_size)
+
+        self.elf.to_file(vmcore)
 
     def dump_iterate(self, vmcore):
+        """Writes guest core to file."""
+
         qemu_core = gdb.inferiors()[0]
         for block in self.guest_phys_blocks:
-            cur  = block["host_addr"]
+            cur = block["host_addr"]
             left = block["target_end"] - block["target_start"]
-            print ("dumping range at %016x for length %016x" %
-                   (cur.cast(self.uintptr_t), left))
-            while (left > 0):
-                chunk_size = min(self.TARGET_PAGE_SIZE, left)
+            print("dumping range at %016x for length %016x" %
+                  (cur.cast(UINTPTR_T), left))
+
+            while left > 0:
+                chunk_size = min(TARGET_PAGE_SIZE, left)
                 chunk = qemu_core.read_memory(cur, chunk_size)
                 vmcore.write(chunk)
-                cur  += chunk_size
+                cur += chunk_size
                 left -= chunk_size
 
-    def create_vmcore(self, filename):
-        vmcore = open(filename, "wb")
-        self.dump_begin(vmcore)
-        self.dump_iterate(vmcore)
-        vmcore.close()
-
     def invoke(self, args, from_tty):
+        """Handles command invocation from gdb."""
+
         # Unwittingly pressing the Enter key after the command should
         # not dump the same multi-gig coredump to the same file.
         self.dont_repeat()
 
         argv = gdb.string_to_argv(args)
-        if (len(argv) != 1):
-            raise gdb.GdbError("usage: dump-guest-memory FILE")
+        if len(argv) != 2:
+            raise gdb.GdbError("usage: dump-guest-memory FILE ARCH")
 
-        self.dump_init()
-        self.create_vmcore(argv[0])
+        self.elf = ELF(argv[1])
+        self.guest_phys_blocks = get_guest_phys_blocks()
+
+        with open(argv[0], "wb") as vmcore:
+            self.dump_init(vmcore)
+            self.dump_iterate(vmcore)
 
 DumpGuestMemory()
diff --git a/scripts/feature_to_c.sh b/scripts/feature_to_c.sh
index 888548e..c8ce9b8 100644
--- a/scripts/feature_to_c.sh
+++ b/scripts/feature_to_c.sh
@@ -33,12 +33,10 @@
 fi
 
 for input; do
-  arrayname=xml_feature_`echo $input | sed 's,.*/,,; s/[-.]/_/g'`
+  arrayname=xml_feature_$(echo $input | sed 's,.*/,,; s/[-.]/_/g')
 
   ${AWK:-awk} 'BEGIN { n = 0
-      printf "#include \"config.h\"\n"
-      printf "#include \"qemu-common.h\"\n"
-      printf "#include \"exec/gdbstub.h\"\n"
+      printf "#include \"qemu/osdep.h\"\n"
       print "static const char '$arrayname'[] = {"
       for (i = 0; i < 255; i++)
         _ord_[sprintf("%c", i)] = i
@@ -69,8 +67,8 @@
 echo "const char *const xml_builtin[][2] = {" >> $output
 
 for input; do
-  basename=`echo $input | sed 's,.*/,,'`
-  arrayname=xml_feature_`echo $input | sed 's,.*/,,; s/[-.]/_/g'`
+  basename=$(echo $input | sed 's,.*/,,')
+  arrayname=xml_feature_$(echo $input | sed 's,.*/,,; s/[-.]/_/g')
   echo "  { \"$basename\", $arrayname }," >> $output
 done
 
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index af68c6c..8261bcb 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -23,7 +23,6 @@
 my $email_maintainer = 1;
 my $email_list = 1;
 my $email_subscriber_list = 0;
-my $email_git_penguin_chiefs = 0;
 my $email_git = 0;
 my $email_git_all_signature_types = 0;
 my $email_git_blame = 0;
@@ -60,21 +59,6 @@
 my %commit_author_hash;
 my %commit_signer_hash;
 
-my @penguin_chief = ();
-push(@penguin_chief, "Linus Torvalds:torvalds\@linux-foundation.org");
-#Andrew wants in on most everything - 2009/01/14
-#push(@penguin_chief, "Andrew Morton:akpm\@linux-foundation.org");
-
-my @penguin_chief_names = ();
-foreach my $chief (@penguin_chief) {
-    if ($chief =~ m/^(.*):(.*)/) {
-	my $chief_name = $1;
-	my $chief_addr = $2;
-	push(@penguin_chief_names, $chief_name);
-    }
-}
-my $penguin_chiefs = "\(" . join("|", @penguin_chief_names) . "\)";
-
 # Signature types of people who are either
 # 	a) responsible for the code in question, or
 # 	b) familiar enough with it to give relevant feedback
@@ -187,7 +171,6 @@
 		'git-blame!' => \$email_git_blame,
 		'git-blame-signatures!' => \$email_git_blame_signatures,
 		'git-fallback!' => \$email_git_fallback,
-		'git-chief-penguins!' => \$email_git_penguin_chiefs,
 		'git-min-signatures=i' => \$email_git_min_signatures,
 		'git-max-maintainers=i' => \$email_git_max_maintainers,
 		'git-min-percent=i' => \$email_git_min_percent,
@@ -256,7 +239,7 @@
 
 if ($email &&
     ($email_maintainer + $email_list + $email_subscriber_list +
-     $email_git + $email_git_penguin_chiefs + $email_git_blame) == 0) {
+     $email_git + $email_git_blame) == 0) {
     die "$P: Please select at least 1 email option\n";
 }
 
@@ -275,7 +258,7 @@
 while (<$maint>) {
     my $line = $_;
 
-    if ($line =~ m/^(\C):\s*(.*)/) {
+    if ($line =~ m/^(.):\s*(.*)/) {
 	my $type = $1;
 	my $value = $2;
 
@@ -510,7 +493,7 @@
 
     for (my $i = $start; $i < $end; $i++) {
 	my $line = $typevalue[$i];
-	if ($line =~ m/^(\C):\s*(.*)/) {
+	if ($line =~ m/^(.):\s*(.*)/) {
 	    my $type = $1;
 	    my $value = $2;
 	    if ($type eq 'S') {
@@ -528,7 +511,7 @@
 
     for (my $i = $start; $i < $end; $i++) {
 	my $line = $typevalue[$i];
-	if ($line =~ m/^(\C):\s*(.*)/) {
+	if ($line =~ m/^(.):\s*(.*)/) {
 	    my $type = $1;
 	    my $value = $2;
 	    if ($type eq 'M') {
@@ -577,7 +560,7 @@
 
 	    for ($i = $start; $i < $end; $i++) {
 		my $line = $typevalue[$i];
-		if ($line =~ m/^(\C):\s*(.*)/) {
+		if ($line =~ m/^(.):\s*(.*)/) {
 		    my $type = $1;
 		    my $value = $2;
 		    if ($type eq 'X') {
@@ -592,7 +575,7 @@
 	    if (!$exclude) {
 		for ($i = $start; $i < $end; $i++) {
 		    my $line = $typevalue[$i];
-		    if ($line =~ m/^(\C):\s*(.*)/) {
+		    if ($line =~ m/^(.):\s*(.*)/) {
 			my $type = $1;
 			my $value = $2;
 			if ($type eq 'F') {
@@ -653,7 +636,7 @@
 
     if ($email) {
 	if (! $interactive) {
-	    $email_git_fallback = 0 if @email_to > 0 || @list_to > 0 || $email_git || $email_git_blame;
+	    $email_git_fallback = 0 if @email_to > 0 || $email_git || $email_git_blame;
 	    if ($email_git_fallback) {
 	        print STDERR "get_maintainer.pl: No maintainers found, printing recent contributors.\n";
 	        print STDERR "get_maintainer.pl: Do not blindly cc: them on patches!  Use common sense.\n";
@@ -671,19 +654,6 @@
 	    }
 	}
 
-	foreach my $chief (@penguin_chief) {
-	    if ($chief =~ m/^(.*):(.*)/) {
-		my $email_address;
-
-		$email_address = format_email($1, $2, $email_usename);
-		if ($email_git_penguin_chiefs) {
-		    push(@email_to, [$email_address, 'chief penguin']);
-		} else {
-		    @email_to = grep($_->[0] !~ /${email_address}/, @email_to);
-		}
-	    }
-	}
-
 	foreach my $email (@file_emails) {
 	    my ($name, $address) = parse_email($email);
 
@@ -740,7 +710,6 @@
     --git-all-signature-types => include signers regardless of signature type
         or use only ${signature_pattern} signers (default: $email_git_all_signature_types)
     --git-fallback => use git when no exact MAINTAINERS pattern (default: $email_git_fallback)
-    --git-chief-penguins => include ${penguin_chiefs}
     --git-min-signatures => number of signatures required (default: $email_git_min_signatures)
     --git-max-maintainers => maximum maintainers to add (default: $email_git_max_maintainers)
     --git-min-percent => minimum percentage of commits required (default: $email_git_min_percent)
@@ -886,7 +855,7 @@
 
     while ($index < @typevalue) {
 	my $tv = $typevalue[$index];
-	if (($tv =~ m/^(\C):\s*(.*)/)) {
+	if (($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index++;
@@ -900,7 +869,7 @@
 
     while ($index > 0) {
 	my $tv = $typevalue[$index];
-	if (!($tv =~ m/^(\C):\s*(.*)/)) {
+	if (!($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index--;
@@ -914,7 +883,7 @@
 
     while ($index < @typevalue) {
 	my $tv = $typevalue[$index];
-	if (!($tv =~ m/^(\C):\s*(.*)/)) {
+	if (!($tv =~ m/^(.):\s*(.*)/)) {
 	    last;
 	}
 	$index++;
@@ -940,7 +909,7 @@
 
     for ($i = $start + 1; $i < $end; $i++) {
 	my $tv = $typevalue[$i];
-	if ($tv =~ m/^(\C):\s*(.*)/) {
+	if ($tv =~ m/^(.):\s*(.*)/) {
 	    my $ptype = $1;
 	    my $pvalue = $2;
 	    if ($ptype eq "S") {
@@ -999,7 +968,7 @@
 
     for ($i = $start + 1; $i < $end; $i++) {
 	my $tv = $typevalue[$i];
-	if ($tv =~ m/^(\C):\s*(.*)/) {
+	if ($tv =~ m/^(.):\s*(.*)/) {
 	    my $ptype = $1;
 	    my $pvalue = $2;
 	    if ($ptype eq "L") {
@@ -1041,7 +1010,7 @@
 		if ($name eq "") {
 		    if ($i > 0) {
 			my $tv = $typevalue[$i - 1];
-			if ($tv =~ m/^(\C):\s*(.*)/) {
+			if ($tv =~ m/^(.):\s*(.*)/) {
 			    if ($1 eq "P") {
 				$name = $2;
 				$pvalue = format_email($name, $address, $email_usename);
@@ -1281,10 +1250,6 @@
     save_commits_by_author(@lines) if ($interactive);
     save_commits_by_signer(@lines) if ($interactive);
 
-    if (!$email_git_penguin_chiefs) {
-	@signatures = grep(!/${penguin_chiefs}/i, @signatures);
-    }
-
     my ($types_ref, $signers_ref) = extract_formatted_signatures(@signatures);
 
     return ($commits, @$signers_ref);
@@ -1296,10 +1261,6 @@
 
     @lines = &{$VCS_cmds{"execute_cmd"}}($cmd);
 
-    if (!$email_git_penguin_chiefs) {
-	@lines = grep(!/${penguin_chiefs}/i, @lines);
-    }
-
     return @lines if !@lines;
 
     my @authors = ();
@@ -1925,10 +1886,6 @@
 
 		@lines = &{$VCS_cmds{"execute_cmd"}}($cmd);
 
-		if (!$email_git_penguin_chiefs) {
-		    @lines = grep(!/${penguin_chiefs}/i, @lines);
-		}
-
 		last if !@lines;
 
 		my @authors = ();
diff --git a/scripts/kvm/kvm_stat b/scripts/kvm/kvm_stat
deleted file mode 100755
index 7b1437c..0000000
--- a/scripts/kvm/kvm_stat
+++ /dev/null
@@ -1,581 +0,0 @@
-#!/usr/bin/python
-#
-# top-like utility for displaying kvm statistics
-#
-# Copyright 2006-2008 Qumranet Technologies
-# Copyright 2008-2011 Red Hat, Inc.
-#
-# Authors:
-#  Avi Kivity <avi@redhat.com>
-#
-# This work is licensed under the terms of the GNU GPL, version 2.  See
-# the COPYING file in the top-level directory.
-
-import curses
-import sys, os, time, optparse, ctypes
-
-class DebugfsProvider(object):
-    def __init__(self):
-        self.base = '/sys/kernel/debug/kvm'
-        self._fields = os.listdir(self.base)
-    def fields(self):
-        return self._fields
-    def select(self, fields):
-        self._fields = fields
-    def read(self):
-        def val(key):
-            return int(file(self.base + '/' + key).read())
-        return dict([(key, val(key)) for key in self._fields])
-
-vmx_exit_reasons = {
-    0: 'EXCEPTION_NMI',
-    1: 'EXTERNAL_INTERRUPT',
-    2: 'TRIPLE_FAULT',
-    7: 'PENDING_INTERRUPT',
-    8: 'NMI_WINDOW',
-    9: 'TASK_SWITCH',
-    10: 'CPUID',
-    12: 'HLT',
-    14: 'INVLPG',
-    15: 'RDPMC',
-    16: 'RDTSC',
-    18: 'VMCALL',
-    19: 'VMCLEAR',
-    20: 'VMLAUNCH',
-    21: 'VMPTRLD',
-    22: 'VMPTRST',
-    23: 'VMREAD',
-    24: 'VMRESUME',
-    25: 'VMWRITE',
-    26: 'VMOFF',
-    27: 'VMON',
-    28: 'CR_ACCESS',
-    29: 'DR_ACCESS',
-    30: 'IO_INSTRUCTION',
-    31: 'MSR_READ',
-    32: 'MSR_WRITE',
-    33: 'INVALID_STATE',
-    36: 'MWAIT_INSTRUCTION',
-    39: 'MONITOR_INSTRUCTION',
-    40: 'PAUSE_INSTRUCTION',
-    41: 'MCE_DURING_VMENTRY',
-    43: 'TPR_BELOW_THRESHOLD',
-    44: 'APIC_ACCESS',
-    48: 'EPT_VIOLATION',
-    49: 'EPT_MISCONFIG',
-    54: 'WBINVD',
-    55: 'XSETBV',
-}
-
-svm_exit_reasons = {
-    0x000: 'READ_CR0',
-    0x003: 'READ_CR3',
-    0x004: 'READ_CR4',
-    0x008: 'READ_CR8',
-    0x010: 'WRITE_CR0',
-    0x013: 'WRITE_CR3',
-    0x014: 'WRITE_CR4',
-    0x018: 'WRITE_CR8',
-    0x020: 'READ_DR0',
-    0x021: 'READ_DR1',
-    0x022: 'READ_DR2',
-    0x023: 'READ_DR3',
-    0x024: 'READ_DR4',
-    0x025: 'READ_DR5',
-    0x026: 'READ_DR6',
-    0x027: 'READ_DR7',
-    0x030: 'WRITE_DR0',
-    0x031: 'WRITE_DR1',
-    0x032: 'WRITE_DR2',
-    0x033: 'WRITE_DR3',
-    0x034: 'WRITE_DR4',
-    0x035: 'WRITE_DR5',
-    0x036: 'WRITE_DR6',
-    0x037: 'WRITE_DR7',
-    0x040: 'EXCP_BASE',
-    0x060: 'INTR',
-    0x061: 'NMI',
-    0x062: 'SMI',
-    0x063: 'INIT',
-    0x064: 'VINTR',
-    0x065: 'CR0_SEL_WRITE',
-    0x066: 'IDTR_READ',
-    0x067: 'GDTR_READ',
-    0x068: 'LDTR_READ',
-    0x069: 'TR_READ',
-    0x06a: 'IDTR_WRITE',
-    0x06b: 'GDTR_WRITE',
-    0x06c: 'LDTR_WRITE',
-    0x06d: 'TR_WRITE',
-    0x06e: 'RDTSC',
-    0x06f: 'RDPMC',
-    0x070: 'PUSHF',
-    0x071: 'POPF',
-    0x072: 'CPUID',
-    0x073: 'RSM',
-    0x074: 'IRET',
-    0x075: 'SWINT',
-    0x076: 'INVD',
-    0x077: 'PAUSE',
-    0x078: 'HLT',
-    0x079: 'INVLPG',
-    0x07a: 'INVLPGA',
-    0x07b: 'IOIO',
-    0x07c: 'MSR',
-    0x07d: 'TASK_SWITCH',
-    0x07e: 'FERR_FREEZE',
-    0x07f: 'SHUTDOWN',
-    0x080: 'VMRUN',
-    0x081: 'VMMCALL',
-    0x082: 'VMLOAD',
-    0x083: 'VMSAVE',
-    0x084: 'STGI',
-    0x085: 'CLGI',
-    0x086: 'SKINIT',
-    0x087: 'RDTSCP',
-    0x088: 'ICEBP',
-    0x089: 'WBINVD',
-    0x08a: 'MONITOR',
-    0x08b: 'MWAIT',
-    0x08c: 'MWAIT_COND',
-    0x400: 'NPF',
-}
-
-# From include/uapi/linux/kvm.h, KVM_EXIT_xxx
-userspace_exit_reasons = {
-     0: 'UNKNOWN',
-     1: 'EXCEPTION',
-     2: 'IO',
-     3: 'HYPERCALL',
-     4: 'DEBUG',
-     5: 'HLT',
-     6: 'MMIO',
-     7: 'IRQ_WINDOW_OPEN',
-     8: 'SHUTDOWN',
-     9: 'FAIL_ENTRY',
-    10: 'INTR',
-    11: 'SET_TPR',
-    12: 'TPR_ACCESS',
-    13: 'S390_SIEIC',
-    14: 'S390_RESET',
-    15: 'DCR',
-    16: 'NMI',
-    17: 'INTERNAL_ERROR',
-    18: 'OSI',
-    19: 'PAPR_HCALL',
-    20: 'S390_UCONTROL',
-    21: 'WATCHDOG',
-    22: 'S390_TSCH',
-    23: 'EPR',
-}
-
-x86_exit_reasons = {
-    'vmx': vmx_exit_reasons,
-    'svm': svm_exit_reasons,
-}
-
-sc_perf_evt_open = None
-exit_reasons = None
-
-ioctl_numbers = {
-    'SET_FILTER' : 0x40082406,
-    'ENABLE'     : 0x00002400,
-    'DISABLE'    : 0x00002401,
-}
-
-def x86_init(flag):
-    globals().update({
-        'sc_perf_evt_open' : 298,
-        'exit_reasons' : x86_exit_reasons[flag],
-    })
-
-def s390_init():
-    globals().update({
-        'sc_perf_evt_open' : 331
-    })
-
-def ppc_init():
-    globals().update({
-        'sc_perf_evt_open' : 319,
-        'ioctl_numbers' : {
-            'SET_FILTER' : 0x80002406 | (ctypes.sizeof(ctypes.c_char_p) << 16),
-            'ENABLE'     : 0x20002400,
-            'DISABLE'    : 0x20002401,
-        }
-    })
-
-def detect_platform():
-    if os.uname()[4].startswith('ppc'):
-        ppc_init()
-        return
-
-    for line in file('/proc/cpuinfo').readlines():
-        if line.startswith('flags'):
-            for flag in line.split():
-                if flag in x86_exit_reasons:
-                    x86_init(flag)
-                    return
-        elif line.startswith('vendor_id'):
-            for flag in line.split():
-                if flag == 'IBM/S390':
-                    s390_init()
-                    return
-
-detect_platform()
-
-def invert(d):
-    return dict((x[1], x[0]) for x in d.iteritems())
-
-filters = {}
-filters['kvm_userspace_exit'] = ('reason', invert(userspace_exit_reasons))
-if exit_reasons:
-    filters['kvm_exit'] = ('exit_reason', invert(exit_reasons))
-
-import struct, array
-
-libc = ctypes.CDLL('libc.so.6')
-syscall = libc.syscall
-class perf_event_attr(ctypes.Structure):
-    _fields_ = [('type', ctypes.c_uint32),
-                ('size', ctypes.c_uint32),
-                ('config', ctypes.c_uint64),
-                ('sample_freq', ctypes.c_uint64),
-                ('sample_type', ctypes.c_uint64),
-                ('read_format', ctypes.c_uint64),
-                ('flags', ctypes.c_uint64),
-                ('wakeup_events', ctypes.c_uint32),
-                ('bp_type', ctypes.c_uint32),
-                ('bp_addr', ctypes.c_uint64),
-                ('bp_len', ctypes.c_uint64),
-                ]
-def _perf_event_open(attr, pid, cpu, group_fd, flags):
-    return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid),
-                   ctypes.c_int(cpu), ctypes.c_int(group_fd),
-                   ctypes.c_long(flags))
-
-PERF_TYPE_HARDWARE              = 0
-PERF_TYPE_SOFTWARE              = 1
-PERF_TYPE_TRACEPOINT            = 2
-PERF_TYPE_HW_CACHE              = 3
-PERF_TYPE_RAW                   = 4
-PERF_TYPE_BREAKPOINT            = 5
-
-PERF_SAMPLE_IP                  = 1 << 0
-PERF_SAMPLE_TID                 = 1 << 1
-PERF_SAMPLE_TIME                = 1 << 2
-PERF_SAMPLE_ADDR                = 1 << 3
-PERF_SAMPLE_READ                = 1 << 4
-PERF_SAMPLE_CALLCHAIN           = 1 << 5
-PERF_SAMPLE_ID                  = 1 << 6
-PERF_SAMPLE_CPU                 = 1 << 7
-PERF_SAMPLE_PERIOD              = 1 << 8
-PERF_SAMPLE_STREAM_ID           = 1 << 9
-PERF_SAMPLE_RAW                 = 1 << 10
-
-PERF_FORMAT_TOTAL_TIME_ENABLED  = 1 << 0
-PERF_FORMAT_TOTAL_TIME_RUNNING  = 1 << 1
-PERF_FORMAT_ID                  = 1 << 2
-PERF_FORMAT_GROUP               = 1 << 3
-
-import re
-
-sys_tracing = '/sys/kernel/debug/tracing'
-
-class Group(object):
-    def __init__(self, cpu):
-        self.events = []
-        self.group_leader = None
-        self.cpu = cpu
-    def add_event(self, name, event_set, tracepoint, filter = None):
-        self.events.append(Event(group = self,
-                                 name = name, event_set = event_set,
-                                 tracepoint = tracepoint, filter = filter))
-        if len(self.events) == 1:
-            self.file = os.fdopen(self.events[0].fd)
-    def read(self):
-        bytes = 8 * (1 + len(self.events))
-        fmt = 'xxxxxxxx' + 'q' * len(self.events)
-        return dict(zip([event.name for event in self.events],
-                        struct.unpack(fmt, self.file.read(bytes))))
-
-class Event(object):
-    def __init__(self, group, name, event_set, tracepoint, filter = None):
-        self.name = name
-        attr = perf_event_attr()
-        attr.type = PERF_TYPE_TRACEPOINT
-        attr.size = ctypes.sizeof(attr)
-        id_path = os.path.join(sys_tracing, 'events', event_set,
-                               tracepoint, 'id')
-        id = int(file(id_path).read())
-        attr.config = id
-        attr.sample_type = (PERF_SAMPLE_RAW
-                            | PERF_SAMPLE_TIME
-                            | PERF_SAMPLE_CPU)
-        attr.sample_period = 1
-        attr.read_format = PERF_FORMAT_GROUP
-        group_leader = -1
-        if group.events:
-            group_leader = group.events[0].fd
-        fd = _perf_event_open(attr, -1, group.cpu, group_leader, 0)
-        if fd == -1:
-            raise Exception('perf_event_open failed')
-        if filter:
-            import fcntl
-            fcntl.ioctl(fd, ioctl_numbers['SET_FILTER'], filter)
-        self.fd = fd
-    def enable(self):
-        import fcntl
-        fcntl.ioctl(self.fd, ioctl_numbers['ENABLE'], 0)
-    def disable(self):
-        import fcntl
-        fcntl.ioctl(self.fd, ioctl_numbers['DISABLE'], 0)
-
-class TracepointProvider(object):
-    def __init__(self):
-        path = os.path.join(sys_tracing, 'events', 'kvm')
-        fields = [f
-                  for f in os.listdir(path)
-                  if os.path.isdir(os.path.join(path, f))]
-        extra = []
-        for f in fields:
-            if f in filters:
-                subfield, values = filters[f]
-                for name, number in values.iteritems():
-                    extra.append(f + '(' + name + ')')
-        fields += extra
-        self._setup(fields)
-        self.select(fields)
-    def fields(self):
-        return self._fields
-
-    def _online_cpus(self):
-        l = []
-        pattern = r'cpu([0-9]+)'
-        basedir = '/sys/devices/system/cpu'
-        for entry in os.listdir(basedir):
-            match = re.match(pattern, entry)
-            if not match:
-                continue
-            path = os.path.join(basedir, entry, 'online')
-            if os.path.exists(path) and open(path).read().strip() != '1':
-                continue
-            l.append(int(match.group(1)))
-        return l
-
-    def _setup(self, _fields):
-        self._fields = _fields
-        cpus = self._online_cpus()
-        import resource
-        nfiles = len(cpus) * 1000
-        resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles))
-        events = []
-        self.group_leaders = []
-        for cpu in cpus:
-            group = Group(cpu)
-            for name in _fields:
-                tracepoint = name
-                filter = None
-                m = re.match(r'(.*)\((.*)\)', name)
-                if m:
-                    tracepoint, sub = m.groups()
-                    filter = '%s==%d\0' % (filters[tracepoint][0],
-                                           filters[tracepoint][1][sub])
-                event = group.add_event(name, event_set = 'kvm',
-                                        tracepoint = tracepoint,
-                                        filter = filter)
-            self.group_leaders.append(group)
-    def select(self, fields):
-        for group in self.group_leaders:
-            for event in group.events:
-                if event.name in fields:
-                    event.enable()
-                else:
-                    event.disable()
-    def read(self):
-        from collections import defaultdict
-        ret = defaultdict(int)
-        for group in self.group_leaders:
-            for name, val in group.read().iteritems():
-                ret[name] += val
-        return ret
-
-class Stats:
-    def __init__(self, providers, fields = None):
-        self.providers = providers
-        self.fields_filter = fields
-        self._update()
-    def _update(self):
-        def wanted(key):
-            import re
-            if not self.fields_filter:
-                return True
-            return re.match(self.fields_filter, key) is not None
-        self.values = dict()
-        for d in providers:
-            provider_fields = [key for key in d.fields() if wanted(key)]
-            for key in provider_fields:
-                self.values[key] = None
-            d.select(provider_fields)
-    def set_fields_filter(self, fields_filter):
-        self.fields_filter = fields_filter
-        self._update()
-    def get(self):
-        for d in providers:
-            new = d.read()
-            for key in d.fields():
-                oldval = self.values.get(key, (0, 0))
-                newval = new[key]
-                newdelta = None
-                if oldval is not None:
-                    newdelta = newval - oldval[0]
-                self.values[key] = (newval, newdelta)
-        return self.values
-
-if not os.access('/sys/kernel/debug', os.F_OK):
-    print 'Please enable CONFIG_DEBUG_FS in your kernel'
-    sys.exit(1)
-if not os.access('/sys/kernel/debug/kvm', os.F_OK):
-    print "Please mount debugfs ('mount -t debugfs debugfs /sys/kernel/debug')"
-    print "and ensure the kvm modules are loaded"
-    sys.exit(1)
-
-label_width = 40
-number_width = 10
-
-def tui(screen, stats):
-    curses.use_default_colors()
-    curses.noecho()
-    drilldown = False
-    fields_filter = stats.fields_filter
-    def update_drilldown():
-        if not fields_filter:
-            if drilldown:
-                stats.set_fields_filter(None)
-            else:
-                stats.set_fields_filter(r'^[^\(]*$')
-    update_drilldown()
-    def refresh(sleeptime):
-        screen.erase()
-        screen.addstr(0, 0, 'kvm statistics')
-        row = 2
-        s = stats.get()
-        def sortkey(x):
-            if s[x][1]:
-                return (-s[x][1], -s[x][0])
-            else:
-                return (0, -s[x][0])
-        for key in sorted(s.keys(), key = sortkey):
-            if row >= screen.getmaxyx()[0]:
-                break
-            values = s[key]
-            if not values[0] and not values[1]:
-                break
-            col = 1
-            screen.addstr(row, col, key)
-            col += label_width
-            screen.addstr(row, col, '%10d' % (values[0],))
-            col += number_width
-            if values[1] is not None:
-                screen.addstr(row, col, '%8d' % (values[1] / sleeptime,))
-            row += 1
-        screen.refresh()
-
-    sleeptime = 0.25
-    while True:
-        refresh(sleeptime)
-        curses.halfdelay(int(sleeptime * 10))
-        sleeptime = 3
-        try:
-            c = screen.getkey()
-            if c == 'x':
-                drilldown = not drilldown
-                update_drilldown()
-            if c == 'q':
-                break
-        except KeyboardInterrupt:
-            break
-        except curses.error:
-            continue
-
-def batch(stats):
-    s = stats.get()
-    time.sleep(1)
-    s = stats.get()
-    for key in sorted(s.keys()):
-        values = s[key]
-        print '%-22s%10d%10d' % (key, values[0], values[1])
-
-def log(stats):
-    keys = sorted(stats.get().iterkeys())
-    def banner():
-        for k in keys:
-            print '%10s' % k[0:9],
-        print
-    def statline():
-        s = stats.get()
-        for k in keys:
-            print ' %9d' % s[k][1],
-        print
-    line = 0
-    banner_repeat = 20
-    while True:
-        time.sleep(1)
-        if line % banner_repeat == 0:
-            banner()
-        statline()
-        line += 1
-
-options = optparse.OptionParser()
-options.add_option('-1', '--once', '--batch',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'once',
-                   help = 'run in batch mode for one second',
-                   )
-options.add_option('-l', '--log',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'log',
-                   help = 'run in logging mode (like vmstat)',
-                   )
-options.add_option('-t', '--tracepoints',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'tracepoints',
-                   help = 'retrieve statistics from tracepoints',
-                   )
-options.add_option('-d', '--debugfs',
-                   action = 'store_true',
-                   default = False,
-                   dest = 'debugfs',
-                   help = 'retrieve statistics from debugfs',
-                   )
-options.add_option('-f', '--fields',
-                   action = 'store',
-                   default = None,
-                   dest = 'fields',
-                   help = 'fields to display (regex)',
-                   )
-(options, args) = options.parse_args(sys.argv)
-
-providers = []
-if options.tracepoints:
-    providers.append(TracepointProvider())
-if options.debugfs:
-    providers.append(DebugfsProvider())
-
-if len(providers) == 0:
-    try:
-        providers = [TracepointProvider()]
-    except:
-        providers = [DebugfsProvider()]
-
-stats = Stats(providers, fields = options.fields)
-
-if options.log:
-    log(stats)
-elif not options.once:
-    import curses.wrapper
-    curses.wrapper(tui, stats)
-else:
-    batch(stats)
diff --git a/scripts/make_device_config.sh b/scripts/make_device_config.sh
index 7242707..354af31 100644
--- a/scripts/make_device_config.sh
+++ b/scripts/make_device_config.sh
@@ -1,11 +1,13 @@
 #! /bin/sh
-# Construct a target device config file from a default, pulling in any
-# files from include directives.
+# Writes a target device config file to stdout, from a default and from
+# include directives therein.  Also emits Makefile dependencies.
+#
+# Usage: make_device_config.sh SRC DEPFILE-NAME DEPFILE-TARGET > DEST
 
-dest=$1.tmp
-dep=`dirname $1`-`basename $1`.d
-src=$2
-src_dir=`dirname $src`
+src=$1
+dep=$2
+target=$3
+src_dir=$(dirname $src)
 all_includes=
 
 process_includes () {
@@ -18,11 +20,11 @@
 
 f=$src
 while [ -n "$f" ] ; do
-  f=`cat $f | tr -d '\r' | awk '/^include / {printf "'$src_dir'/%s ", $2}'`
+  f=$(cat $f | tr -d '\r' | awk '/^include / {printf "'$src_dir'/%s ", $2}')
   [ $? = 0 ] || exit 1
   all_includes="$all_includes $f"
 done
-process_includes $src > $dest
+process_includes $src
 
-cat $src $all_includes | grep -v '^include' > $dest
-echo "$1: $all_includes" > $dep
+cat $src $all_includes | grep -v '^include'
+echo "$target: $all_includes" > $dep
diff --git a/scripts/ordereddict.py b/scripts/ordereddict.py
index 7242b50..2d1d813 100644
--- a/scripts/ordereddict.py
+++ b/scripts/ordereddict.py
@@ -22,6 +22,7 @@
 
 from UserDict import DictMixin
 
+
 class OrderedDict(dict, DictMixin):
 
     def __init__(self, *args, **kwds):
@@ -117,7 +118,7 @@
         if isinstance(other, OrderedDict):
             if len(self) != len(other):
                 return False
-            for p, q in  zip(self.items(), other.items()):
+            for p, q in zip(self.items(), other.items()):
                 if p != q:
                     return False
             return True
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index 053ba85..a06a2c4 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -2,7 +2,7 @@
 # QAPI command marshaller generator
 #
 # Copyright IBM, Corp. 2011
-# Copyright (C) 2014 Red Hat, Inc.
+# Copyright (C) 2014-2016 Red Hat, Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
@@ -12,349 +12,233 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 import re
-import sys
-import os
-import getopt
-import errno
 
-def type_visitor(name):
-    if type(name) == list:
-        return 'visit_type_%sList' % name[0]
-    else:
-        return 'visit_type_%s' % name
 
-def generate_command_decl(name, args, ret_type):
-    arglist=""
-    for argname, argtype, optional, structured in parse_args(args):
-        argtype = c_type(argtype, is_param=True)
-        if optional:
-            arglist += "bool has_%s, " % c_var(argname)
-        arglist += "%s %s, " % (argtype, c_var(argname))
+def gen_command_decl(name, arg_type, boxed, ret_type):
     return mcgen('''
-%(ret_type)s qmp_%(name)s(%(args)sError **errp);
+%(c_type)s qmp_%(c_name)s(%(params)s);
 ''',
-                 ret_type=c_type(ret_type), name=c_fun(name), args=arglist).strip()
+                 c_type=(ret_type and ret_type.c_type()) or 'void',
+                 c_name=c_name(name),
+                 params=gen_params(arg_type, boxed, 'Error **errp'))
 
-def gen_err_check(errvar):
-    if errvar:
-        return mcgen('''
-if (local_err) {
-    goto out;
-}
-''')
-    return ''
 
-def gen_sync_call(name, args, ret_type, indent=0):
-    ret = ""
-    arglist=""
-    retval=""
+def gen_call(name, arg_type, boxed, ret_type):
+    ret = ''
+
+    argstr = ''
+    if boxed:
+        assert arg_type and not arg_type.is_empty()
+        argstr = '&arg, '
+    elif arg_type:
+        assert not arg_type.variants
+        for memb in arg_type.members:
+            if memb.optional:
+                argstr += 'arg.has_%s, ' % c_name(memb.name)
+            argstr += 'arg.%s, ' % c_name(memb.name)
+
+    lhs = ''
     if ret_type:
-        retval = "retval = "
-    for argname, argtype, optional, structured in parse_args(args):
-        if optional:
-            arglist += "has_%s, " % c_var(argname)
-        arglist += "%s, " % (c_var(argname))
-    push_indent(indent)
-    ret = mcgen('''
-%(retval)sqmp_%(name)s(%(args)s&local_err);
-
-''',
-                name=c_fun(name), args=arglist, retval=retval).rstrip()
-    if ret_type:
-        ret += "\n" + gen_err_check('local_err')
-        ret += "\n" + mcgen(''''
-%(marshal_output_call)s
-''',
-                            marshal_output_call=gen_marshal_output_call(name, ret_type)).rstrip()
-    pop_indent(indent)
-    return ret.rstrip()
-
-
-def gen_marshal_output_call(name, ret_type):
-    if not ret_type:
-        return ""
-    return "qmp_marshal_output_%s(retval, ret, &local_err);" % c_fun(name)
-
-def gen_visitor_input_containers_decl(args, obj):
-    ret = ""
-
-    push_indent()
-    if len(args) > 0:
-        ret += mcgen('''
-QmpInputVisitor *mi = qmp_input_visitor_new_strict(%(obj)s);
-QapiDeallocVisitor *md;
-Visitor *v;
-''',
-                     obj=obj)
-    pop_indent()
-
-    return ret.rstrip()
-
-def gen_visitor_input_vars_decl(args):
-    ret = ""
-    push_indent()
-    for argname, argtype, optional, structured in parse_args(args):
-        if optional:
-            ret += mcgen('''
-bool has_%(argname)s = false;
-''',
-                         argname=c_var(argname))
-        if is_c_ptr(argtype):
-            ret += mcgen('''
-%(argtype)s %(argname)s = NULL;
-''',
-                         argname=c_var(argname), argtype=c_type(argtype))
-        else:
-            ret += mcgen('''
-%(argtype)s %(argname)s = {0};
-''',
-                         argname=c_var(argname), argtype=c_type(argtype))
-
-    pop_indent()
-    return ret.rstrip()
-
-def gen_visitor_input_block(args, dealloc=False):
-    ret = ""
-    errparg = '&local_err'
-    errarg = 'local_err'
-
-    if len(args) == 0:
-        return ret
-
-    push_indent()
-
-    if dealloc:
-        errparg = 'NULL'
-        errarg = None;
-        ret += mcgen('''
-qmp_input_visitor_cleanup(mi);
-md = qapi_dealloc_visitor_new();
-v = qapi_dealloc_get_visitor(md);
-''')
-    else:
-        ret += mcgen('''
-v = qmp_input_get_visitor(mi);
-''')
-
-    for argname, argtype, optional, structured in parse_args(args):
-        if optional:
-            ret += mcgen('''
-visit_optional(v, &has_%(c_name)s, "%(name)s", %(errp)s);
-''',
-                         c_name=c_var(argname), name=argname, errp=errparg)
-            ret += gen_err_check(errarg)
-            ret += mcgen('''
-if (has_%(c_name)s) {
-''',
-                         c_name=c_var(argname))
-            push_indent()
-        ret += mcgen('''
-%(visitor)s(v, &%(c_name)s, "%(name)s", %(errp)s);
-''',
-                     c_name=c_var(argname), name=argname, argtype=argtype,
-                     visitor=type_visitor(argtype), errp=errparg)
-        ret += gen_err_check(errarg)
-        if optional:
-            pop_indent()
-            ret += mcgen('''
-}
-''')
-
-    if dealloc:
-        ret += mcgen('''
-qapi_dealloc_visitor_cleanup(md);
-''')
-    pop_indent()
-    return ret.rstrip()
-
-def gen_marshal_output(name, args, ret_type, middle_mode):
-    if not ret_type:
-        return ""
+        lhs = 'retval = '
 
     ret = mcgen('''
-static void qmp_marshal_output_%(c_name)s(%(c_ret_type)s ret_in, QObject **ret_out, Error **errp)
-{
-    Error *local_err = NULL;
-    QmpOutputVisitor *mo = qmp_output_visitor_new();
-    QapiDeallocVisitor *md;
-    Visitor *v;
 
-    v = qmp_output_get_visitor(mo);
-    %(visitor)s(v, &ret_in, "unused", &local_err);
-    if (local_err) {
+    %(lhs)sqmp_%(c_name)s(%(args)s&err);
+''',
+                c_name=c_name(name), args=argstr, lhs=lhs)
+    if ret_type:
+        ret += mcgen('''
+    if (err) {
         goto out;
     }
-    *ret_out = qmp_output_get_qobject(mo);
 
-out:
-    error_propagate(errp, local_err);
-    qmp_output_visitor_cleanup(mo);
-    md = qapi_dealloc_visitor_new();
-    v = qapi_dealloc_get_visitor(md);
-    %(visitor)s(v, &ret_in, "unused", NULL);
-    qapi_dealloc_visitor_cleanup(md);
-}
+    qmp_marshal_output_%(c_name)s(retval, ret, &err);
 ''',
-                c_ret_type=c_type(ret_type), c_name=c_fun(name),
-                visitor=type_visitor(ret_type))
-
+                     c_name=ret_type.c_name())
     return ret
 
-def gen_marshal_input_decl(name, args, ret_type, middle_mode):
-    if middle_mode:
-        return 'int qmp_marshal_input_%s(Monitor *mon, const QDict *qdict, QObject **ret)' % c_fun(name)
-    else:
-        return 'static void qmp_marshal_input_%s(QDict *args, QObject **ret, Error **errp)' % c_fun(name)
 
+def gen_marshal_output(ret_type):
+    return mcgen('''
 
-
-def gen_marshal_input(name, args, ret_type, middle_mode):
-    hdr = gen_marshal_input_decl(name, args, ret_type, middle_mode)
-
-    ret = mcgen('''
-%(header)s
+static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Error **errp)
 {
-    Error *local_err = NULL;
-''',
-                header=hdr)
+    Error *err = NULL;
+    Visitor *v;
 
-    if middle_mode:
-        ret += mcgen('''
-    QDict *args = (QDict *)qdict;
-''')
+    v = qmp_output_visitor_new(ret_out);
+    visit_type_%(c_name)s(v, "unused", &ret_in, &err);
+    if (!err) {
+        visit_complete(v, ret_out);
+    }
+    error_propagate(errp, err);
+    visit_free(v);
+    v = qapi_dealloc_visitor_new();
+    visit_type_%(c_name)s(v, "unused", &ret_in, NULL);
+    visit_free(v);
+}
+''',
+                 c_type=ret_type.c_type(), c_name=ret_type.c_name())
+
+
+def gen_marshal_proto(name):
+    ret = 'void qmp_marshal_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name)
+    if not middle_mode:
+        ret = 'static ' + ret
+    return ret
+
+
+def gen_marshal_decl(name):
+    return mcgen('''
+%(proto)s;
+''',
+                 proto=gen_marshal_proto(name))
+
+
+def gen_marshal(name, arg_type, boxed, ret_type):
+    ret = mcgen('''
+
+%(proto)s
+{
+    Error *err = NULL;
+''',
+                proto=gen_marshal_proto(name))
 
     if ret_type:
-        if is_c_ptr(ret_type):
-            retval = "    %s retval = NULL;" % c_type(ret_type)
-        else:
-            retval = "    %s retval;" % c_type(ret_type)
         ret += mcgen('''
-%(retval)s
+    %(c_type)s retval;
 ''',
-                     retval=retval)
+                     c_type=ret_type.c_type())
 
-    if len(args) > 0:
+    if arg_type and not arg_type.is_empty():
         ret += mcgen('''
-%(visitor_input_containers_decl)s
-%(visitor_input_vars_decl)s
+    Visitor *v;
+    %(c_name)s arg = {0};
 
-%(visitor_input_block)s
-
+    v = qmp_input_visitor_new(QOBJECT(args), true);
+    visit_start_struct(v, NULL, NULL, 0, &err);
+    if (err) {
+        goto out;
+    }
+    visit_type_%(c_name)s_members(v, &arg, &err);
+    if (!err) {
+        visit_check_struct(v, &err);
+    }
+    visit_end_struct(v, NULL);
+    if (err) {
+        goto out;
+    }
 ''',
-                     visitor_input_containers_decl=gen_visitor_input_containers_decl(args, "QOBJECT(args)"),
-                     visitor_input_vars_decl=gen_visitor_input_vars_decl(args),
-                     visitor_input_block=gen_visitor_input_block(args))
+                     c_name=arg_type.c_name())
+
     else:
         ret += mcgen('''
 
     (void)args;
 ''')
 
-    ret += mcgen('''
-%(sync_call)s
-''',
-                 sync_call=gen_sync_call(name, args, ret_type, indent=4))
-    if re.search('^ *goto out\\;', ret, re.MULTILINE):
+    ret += gen_call(name, arg_type, boxed, ret_type)
+
+    # 'goto out' produced above for arg_type, and by gen_call() for ret_type
+    if (arg_type and not arg_type.is_empty()) or ret_type:
         ret += mcgen('''
 
 out:
 ''')
-    if not middle_mode:
-        ret += mcgen('''
-    error_propagate(errp, local_err);
-''')
     ret += mcgen('''
-%(visitor_input_block_cleanup)s
+    error_propagate(errp, err);
+''')
+    if arg_type and not arg_type.is_empty():
+        ret += mcgen('''
+    visit_free(v);
+    v = qapi_dealloc_visitor_new();
+    visit_start_struct(v, NULL, NULL, 0, NULL);
+    visit_type_%(c_name)s_members(v, &arg, NULL);
+    visit_end_struct(v, NULL);
+    visit_free(v);
 ''',
-                 visitor_input_block_cleanup=gen_visitor_input_block(args,
-                                                                     dealloc=True))
-
-    if middle_mode:
-        ret += mcgen('''
-
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
-    }
-    return 0;
-''')
-    else:
-        ret += mcgen('''
-    return;
-''')
+                     c_name=arg_type.c_name())
 
     ret += mcgen('''
 }
 ''')
-
     return ret
 
-def option_value_matches(opt, val, cmd):
-    if opt in cmd and cmd[opt] == val:
-        return True
-    return False
 
-def gen_registry(commands):
-    registry=""
-    push_indent()
-    for cmd in commands:
-        options = 'QCO_NO_OPTIONS'
-        if option_value_matches('success-response', 'no', cmd):
-            options = 'QCO_NO_SUCCESS_RESP'
+def gen_register_command(name, success_response):
+    options = 'QCO_NO_OPTIONS'
+    if not success_response:
+        options = 'QCO_NO_SUCCESS_RESP'
 
-        registry += mcgen('''
-qmp_register_command("%(name)s", qmp_marshal_input_%(c_name)s, %(opts)s);
-''',
-                     name=cmd['command'], c_name=c_fun(cmd['command']),
-                     opts=options)
-    pop_indent()
     ret = mcgen('''
+    qmp_register_command("%(name)s", qmp_marshal_%(c_name)s, %(opts)s);
+''',
+                name=name, c_name=c_name(name),
+                opts=options)
+    return ret
+
+
+def gen_registry(registry):
+    ret = mcgen('''
+
 static void qmp_init_marshal(void)
 {
-%(registry)s
+''')
+    ret += registry
+    ret += mcgen('''
 }
 
 qapi_init(qmp_init_marshal);
-''',
-                registry=registry.rstrip())
+''')
     return ret
 
-def gen_command_decl_prologue(header, guard, prefix=""):
-    ret = mcgen('''
-/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
 
-/*
- * schema-defined QAPI function prototypes
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
+class QAPISchemaGenCommandVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._regy = None
+        self._visited_ret_types = None
 
-#ifndef %(guard)s
-#define %(guard)s
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._regy = ''
+        self._visited_ret_types = set()
 
-#include "%(prefix)sqapi-types.h"
-#include "qapi/qmp/qdict.h"
-#include "qapi/error.h"
+    def visit_end(self):
+        if not middle_mode:
+            self.defn += gen_registry(self._regy)
+        self._regy = None
+        self._visited_ret_types = None
 
-''',
-                 header=basename(header), guard=guardname(header), prefix=prefix)
-    return ret
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response, boxed):
+        if not gen:
+            return
+        self.decl += gen_command_decl(name, arg_type, boxed, ret_type)
+        if ret_type and ret_type not in self._visited_ret_types:
+            self._visited_ret_types.add(ret_type)
+            self.defn += gen_marshal_output(ret_type)
+        if middle_mode:
+            self.decl += gen_marshal_decl(name)
+        self.defn += gen_marshal(name, arg_type, boxed, ret_type)
+        if not middle_mode:
+            self._regy += gen_register_command(name, success_response)
 
-def gen_command_def_prologue(prefix="", proxy=False):
-    ret = mcgen('''
-/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
 
+middle_mode = False
+
+(input_file, output_dir, do_c, do_h, prefix, opts) = \
+    parse_command_line("m", ["middle"])
+
+for o, a in opts:
+    if o in ("-m", "--middle"):
+        middle_mode = True
+
+c_comment = '''
 /*
  * schema-defined QMP->QAPI command dispatch
  *
@@ -367,10 +251,30 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+'''
+h_comment = '''
+/*
+ * schema-defined QAPI function prototypes
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
 
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qmp-marshal.c', 'qmp-commands.h',
+                            c_comment, h_comment)
+
+fdef.write(mcgen('''
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/module.h"
-#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/types.h"
 #include "qapi/qmp/dispatch.h"
 #include "qapi/visitor.h"
@@ -379,107 +283,23 @@
 #include "qapi/dealloc-visitor.h"
 #include "%(prefix)sqapi-types.h"
 #include "%(prefix)sqapi-visit.h"
+#include "%(prefix)sqmp-commands.h"
 
 ''',
-                prefix=prefix)
-    if not proxy:
-        ret += '#include "%sqmp-commands.h"' % prefix
-    return ret + "\n\n"
+                 prefix=prefix))
 
+fdecl.write(mcgen('''
+#include "%(prefix)sqapi-types.h"
+#include "qapi/qmp/qdict.h"
+#include "qapi/error.h"
 
-try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:i:o:m",
-                                   ["source", "header", "prefix=",
-                                    "input-file=", "output-dir=",
-                                    "type=", "middle"])
-except getopt.GetoptError, err:
-    print str(err)
-    sys.exit(1)
+''',
+                  prefix=prefix))
 
-output_dir = ""
-prefix = ""
-dispatch_type = "sync"
-c_file = 'qmp-marshal.c'
-h_file = 'qmp-commands.h'
-middle_mode = False
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenCommandVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
-do_c = False
-do_h = False
-
-for o, a in opts:
-    if o in ("-p", "--prefix"):
-        prefix = a
-    elif o in ("-i", "--input-file"):
-        input_file = a
-    elif o in ("-o", "--output-dir"):
-        output_dir = a + "/"
-    elif o in ("-t", "--type"):
-        dispatch_type = a
-    elif o in ("-m", "--middle"):
-        middle_mode = True
-    elif o in ("-c", "--source"):
-        do_c = True
-    elif o in ("-h", "--header"):
-        do_h = True
-
-if not do_c and not do_h:
-    do_c = True
-    do_h = True
-
-c_file = output_dir + prefix + c_file
-h_file = output_dir + prefix + h_file
-
-def maybe_open(really, name, opt):
-    if really:
-        return open(name, opt)
-    else:
-        import StringIO
-        return StringIO.StringIO()
-
-try:
-    os.makedirs(output_dir)
-except os.error, e:
-    if e.errno != errno.EEXIST:
-        raise
-
-exprs = parse_schema(input_file)
-commands = filter(lambda expr: expr.has_key('command'), exprs)
-commands = filter(lambda expr: not expr.has_key('gen'), commands)
-
-if dispatch_type == "sync":
-    fdecl = maybe_open(do_h, h_file, 'w')
-    fdef = maybe_open(do_c, c_file, 'w')
-    ret = gen_command_decl_prologue(header=basename(h_file), guard=guardname(h_file), prefix=prefix)
-    fdecl.write(ret)
-    ret = gen_command_def_prologue(prefix=prefix)
-    fdef.write(ret)
-
-    for cmd in commands:
-        arglist = []
-        ret_type = None
-        if cmd.has_key('data'):
-            arglist = cmd['data']
-        if cmd.has_key('returns'):
-            ret_type = cmd['returns']
-        ret = generate_command_decl(cmd['command'], arglist, ret_type) + "\n"
-        fdecl.write(ret)
-        if ret_type:
-            ret = gen_marshal_output(cmd['command'], arglist, ret_type, middle_mode) + "\n"
-            fdef.write(ret)
-
-        if middle_mode:
-            fdecl.write('%s;\n' % gen_marshal_input_decl(cmd['command'], arglist, ret_type, middle_mode))
-
-        ret = gen_marshal_input(cmd['command'], arglist, ret_type, middle_mode) + "\n"
-        fdef.write(ret)
-
-    fdecl.write("\n#endif\n");
-
-    if not middle_mode:
-        ret = gen_registry(commands)
-        fdef.write(ret)
-
-    fdef.flush()
-    fdef.close()
-    fdecl.flush()
-    fdecl.close()
+close_output(fdef, fdecl)
diff --git a/scripts/qapi-event.py b/scripts/qapi-event.py
index 601e307..38d8211 100644
--- a/scripts/qapi-event.py
+++ b/scripts/qapi-event.py
@@ -2,281 +2,176 @@
 # QAPI event generator
 #
 # Copyright (c) 2014 Wenchao Xia
+# Copyright (c) 2015-2016 Red Hat Inc.
 #
 # Authors:
 #  Wenchao Xia <wenchaoqemu@gmail.com>
+#  Markus Armbruster <armbru@redhat.com>
 #
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
-import sys
-import os
-import getopt
-import errno
-
-def _generate_event_api_name(event_name, params):
-    api_name = "void qapi_event_send_%s(" % c_fun(event_name).lower();
-    l = len(api_name)
-
-    if params:
-        for argname, argentry, optional, structured in parse_args(params):
-            if optional:
-                api_name += "bool has_%s,\n" % c_var(argname)
-                api_name += "".ljust(l)
-
-            api_name += "%s %s,\n" % (c_type(argentry, is_param=True),
-                                      c_var(argname))
-            api_name += "".ljust(l)
-
-    api_name += "Error **errp)"
-    return api_name;
 
 
-# Following are the core functions that generate C APIs to emit event.
+def gen_event_send_proto(name, arg_type, boxed):
+    return 'void qapi_event_send_%(c_name)s(%(param)s)' % {
+        'c_name': c_name(name.lower()),
+        'param': gen_params(arg_type, boxed, 'Error **errp')}
 
-def generate_event_declaration(api_name):
+
+def gen_event_send_decl(name, arg_type, boxed):
     return mcgen('''
 
-%(api_name)s;
+%(proto)s;
 ''',
-                 api_name = api_name)
+                 proto=gen_event_send_proto(name, arg_type, boxed))
 
-def generate_event_implement(api_name, event_name, params):
-    # step 1: declare any variables
-    ret = mcgen("""
 
-%(api_name)s
+# Declare and initialize an object 'qapi' using parameters from gen_params()
+def gen_param_var(typ):
+    assert not typ.variants
+    ret = mcgen('''
+    %(c_name)s param = {
+''',
+                c_name=typ.c_name())
+    sep = '        '
+    for memb in typ.members:
+        ret += sep
+        sep = ', '
+        if memb.optional:
+            ret += 'has_' + c_name(memb.name) + sep
+        if memb.type.name == 'str':
+            # Cast away const added in gen_params()
+            ret += '(char *)'
+        ret += c_name(memb.name)
+    ret += mcgen('''
+
+    };
+''')
+    if not typ.is_implicit():
+        ret += mcgen('''
+    %(c_name)s *arg = &param;
+''',
+                     c_name=typ.c_name())
+    return ret
+
+
+def gen_event_send(name, arg_type, boxed):
+    # FIXME: Our declaration of local variables (and of 'errp' in the
+    # parameter list) can collide with exploded members of the event's
+    # data type passed in as parameters.  If this collision ever hits in
+    # practice, we can rename our local variables with a leading _ prefix,
+    # or split the code into a wrapper function that creates a boxed
+    # 'param' object then calls another to do the real work.
+    ret = mcgen('''
+
+%(proto)s
 {
     QDict *qmp;
-    Error *local_err = NULL;
+    Error *err = NULL;
     QMPEventFuncEmit emit;
-""",
-                api_name = api_name)
+''',
+                proto=gen_event_send_proto(name, arg_type, boxed))
 
-    if params:
-        ret += mcgen("""
-    QmpOutputVisitor *qov;
-    Visitor *v;
+    if arg_type and not arg_type.is_empty():
+        ret += mcgen('''
     QObject *obj;
+    Visitor *v;
+''')
+        if not boxed:
+            ret += gen_param_var(arg_type)
+    else:
+        assert not boxed
 
-""")
+    ret += mcgen('''
 
-    # step 2: check emit function, create a dict
-    ret += mcgen("""
     emit = qmp_event_get_func_emit();
     if (!emit) {
         return;
     }
 
-    qmp = qmp_event_build_dict("%(event_name)s");
+    qmp = qmp_event_build_dict("%(name)s");
 
-""",
-                 event_name = event_name)
-
-    # step 3: visit the params if params != None
-    if params:
-        ret += mcgen("""
-    qov = qmp_output_visitor_new();
-    g_assert(qov);
-
-    v = qmp_output_get_visitor(qov);
-    g_assert(v);
-
-    /* Fake visit, as if all members are under a structure */
-    visit_start_struct(v, NULL, "", "%(event_name)s", 0, &local_err);
-    if (local_err) {
-        goto clean;
-    }
-
-""",
-                event_name = event_name)
-
-        for argname, argentry, optional, structured in parse_args(params):
-            if optional:
-                ret += mcgen("""
-    if (has_%(var)s) {
-""",
-                             var = c_var(argname))
-                push_indent()
-
-            if argentry == "str":
-                var_type = "(char **)"
-            else:
-                var_type = ""
-
-            ret += mcgen("""
-    visit_type_%(type)s(v, %(var_type)s&%(var)s, "%(name)s", &local_err);
-    if (local_err) {
-        goto clean;
-    }
-""",
-                         var_type = var_type,
-                         var = c_var(argname),
-                         type = type_name(argentry),
-                         name = argname)
-
-            if optional:
-                pop_indent()
-                ret += mcgen("""
-    }
-""")
-
-        ret += mcgen("""
-
-    visit_end_struct(v, &local_err);
-    if (local_err) {
-        goto clean;
-    }
-
-    obj = qmp_output_get_qobject(qov);
-    g_assert(obj != NULL);
-
-    qdict_put_obj(qmp, "data", obj);
-""")
-
-    # step 4: call qmp event api
-    ret += mcgen("""
-    emit(%(event_enum_value)s, qmp, &local_err);
-
-""",
-                 event_enum_value = event_enum_value)
-
-    # step 5: clean up
-    if params:
-        ret += mcgen("""
- clean:
-    qmp_output_visitor_cleanup(qov);
-""")
-    ret += mcgen("""
-    error_propagate(errp, local_err);
-    QDECREF(qmp);
-}
-""")
-
-    return ret
-
-
-# Following are the functions that generate an enum type for all defined
-# events, similar to qapi-types.py. Here we already have enum name and
-# values which were generated before and recorded in event_enum_*. It also
-# works around the issue that "import qapi-types" can't work.
-
-def generate_event_enum_decl(event_enum_name, event_enum_values):
-    lookup_decl = mcgen('''
-
-extern const char *%(event_enum_name)s_lookup[];
 ''',
-                        event_enum_name = event_enum_name)
+                 name=name)
 
-    enum_decl = mcgen('''
-typedef enum %(event_enum_name)s
-{
-''',
-                      event_enum_name = event_enum_name)
-
-    # append automatically generated _MAX value
-    enum_max_value = generate_enum_full_value(event_enum_name, "MAX")
-    enum_values = event_enum_values + [ enum_max_value ]
-
-    i = 0
-    for value in enum_values:
-        enum_decl += mcgen('''
-    %(value)s = %(i)d,
-''',
-                     value = value,
-                     i = i)
-        i += 1
-
-    enum_decl += mcgen('''
-} %(event_enum_name)s;
-''',
-                       event_enum_name = event_enum_name)
-
-    return lookup_decl + enum_decl
-
-def generate_event_enum_lookup(event_enum_name, event_enum_strings):
-    ret = mcgen('''
-
-const char *%(event_enum_name)s_lookup[] = {
-''',
-                event_enum_name = event_enum_name)
-
-    i = 0
-    for string in event_enum_strings:
+    if arg_type and not arg_type.is_empty():
         ret += mcgen('''
-    "%(string)s",
+    v = qmp_output_visitor_new(&obj);
+''')
+        if not arg_type.is_implicit():
+            ret += mcgen('''
+    visit_type_%(c_name)s(v, "%(name)s", &arg, &err);
 ''',
-                     string = string)
+                         name=name, c_name=arg_type.c_name())
+        else:
+            ret += mcgen('''
+
+    visit_start_struct(v, "%(name)s", NULL, 0, &err);
+    if (err) {
+        goto out;
+    }
+    visit_type_%(c_name)s_members(v, &param, &err);
+    if (!err) {
+        visit_check_struct(v, &err);
+    }
+    visit_end_struct(v, NULL);
+''',
+                         name=name, c_name=arg_type.c_name())
+        ret += mcgen('''
+    if (err) {
+        goto out;
+    }
+
+    visit_complete(v, &obj);
+    qdict_put_obj(qmp, "data", obj);
+''')
 
     ret += mcgen('''
-    NULL,
-};
+    emit(%(c_enum)s, qmp, &err);
+
+''',
+                 c_enum=c_enum_const(event_enum_name, name))
+
+    if arg_type and not arg_type.is_empty():
+        ret += mcgen('''
+out:
+    visit_free(v);
+''')
+    ret += mcgen('''
+    error_propagate(errp, err);
+    QDECREF(qmp);
+}
 ''')
     return ret
 
 
-# Start the real job
+class QAPISchemaGenEventVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._event_names = None
 
-try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:i:o:",
-                                   ["source", "header", "builtins", "prefix=",
-                                    "input-file=", "output-dir="])
-except getopt.GetoptError, err:
-    print str(err)
-    sys.exit(1)
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._event_names = []
 
-input_file = ""
-output_dir = ""
-prefix = ""
-c_file = 'qapi-event.c'
-h_file = 'qapi-event.h'
+    def visit_end(self):
+        self.decl += gen_enum(event_enum_name, self._event_names)
+        self.defn += gen_enum_lookup(event_enum_name, self._event_names)
+        self._event_names = None
 
-do_c = False
-do_h = False
-do_builtins = False
+    def visit_event(self, name, info, arg_type, boxed):
+        self.decl += gen_event_send_decl(name, arg_type, boxed)
+        self.defn += gen_event_send(name, arg_type, boxed)
+        self._event_names.append(name)
 
-for o, a in opts:
-    if o in ("-p", "--prefix"):
-        prefix = a
-    elif o in ("-i", "--input-file"):
-        input_file = a
-    elif o in ("-o", "--output-dir"):
-        output_dir = a + "/"
-    elif o in ("-c", "--source"):
-        do_c = True
-    elif o in ("-h", "--header"):
-        do_h = True
-    elif o in ("-b", "--builtins"):
-        do_builtins = True
 
-if not do_c and not do_h:
-    do_c = True
-    do_h = True
+(input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line()
 
-c_file = output_dir + prefix + c_file
-h_file = output_dir + prefix + h_file
-
-try:
-    os.makedirs(output_dir)
-except os.error, e:
-    if e.errno != errno.EEXIST:
-        raise
-
-def maybe_open(really, name, opt):
-    if really:
-        return open(name, opt)
-    else:
-        import StringIO
-        return StringIO.StringIO()
-
-fdef = maybe_open(do_c, c_file, 'w')
-fdecl = maybe_open(do_h, h_file, 'w')
-
-fdef.write(mcgen('''
-/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
-
+c_comment = '''
 /*
  * schema-defined QAPI event functions
  *
@@ -289,19 +184,8 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
-
-#include "qemu-common.h"
-#include "%(header)s"
-#include "%(prefix)sqapi-visit.h"
-#include "qapi/qmp-output-visitor.h"
-#include "qapi/qmp-event.h"
-
-''',
-                 prefix=prefix, header=basename(h_file)))
-
-fdecl.write(mcgen('''
-/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
-
+'''
+h_comment = '''
 /*
  * schema-defined QAPI event functions
  *
@@ -314,55 +198,37 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+'''
 
-#ifndef %(guard)s
-#define %(guard)s
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qapi-event.c', 'qapi-event.h',
+                            c_comment, h_comment)
 
+fdef.write(mcgen('''
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "%(prefix)sqapi-event.h"
+#include "%(prefix)sqapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/qmp-event.h"
+
+''',
+                 prefix=prefix))
+
+fdecl.write(mcgen('''
 #include "qapi/error.h"
 #include "qapi/qmp/qdict.h"
 #include "%(prefix)sqapi-types.h"
 
 ''',
-                  prefix=prefix, guard=guardname(h_file)))
+                  prefix=prefix))
 
-exprs = parse_schema(input_file)
+event_enum_name = c_name(prefix + "QAPIEvent", protect=False)
 
-event_enum_name = prefix.upper().replace('-', '_') + "QAPIEvent"
-event_enum_values = []
-event_enum_strings = []
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenEventVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
-for expr in exprs:
-    if expr.has_key('event'):
-        event_name = expr['event']
-        params = expr.get('data')
-        if params and len(params) == 0:
-            params = None
-
-        api_name = _generate_event_api_name(event_name, params)
-        ret = generate_event_declaration(api_name)
-        fdecl.write(ret)
-
-        # We need an enum value per event
-        event_enum_value = generate_enum_full_value(event_enum_name,
-                                                    event_name)
-        ret = generate_event_implement(api_name, event_name, params)
-        fdef.write(ret)
-
-        # Record it, and generate enum later
-        event_enum_values.append(event_enum_value)
-        event_enum_strings.append(event_name)
-
-ret = generate_event_enum_decl(event_enum_name, event_enum_values)
-fdecl.write(ret)
-ret = generate_event_enum_lookup(event_enum_name, event_enum_strings)
-fdef.write(ret)
-
-fdecl.write('''
-#endif
-''')
-
-fdecl.flush()
-fdecl.close()
-
-fdef.flush()
-fdef.close()
+close_output(fdef, fdecl)
diff --git a/scripts/qapi-introspect.py b/scripts/qapi-introspect.py
new file mode 100644
index 0000000..541644e
--- /dev/null
+++ b/scripts/qapi-introspect.py
@@ -0,0 +1,219 @@
+#
+# QAPI introspection generator
+#
+# Copyright (C) 2015-2016 Red Hat, Inc.
+#
+# Authors:
+#  Markus Armbruster <armbru@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.
+# See the COPYING file in the top-level directory.
+
+from qapi import *
+
+
+# Caveman's json.dumps() replacement (we're stuck at Python 2.4)
+# TODO try to use json.dumps() once we get unstuck
+def to_json(obj, level=0):
+    if obj is None:
+        ret = 'null'
+    elif isinstance(obj, str):
+        ret = '"' + obj.replace('"', r'\"') + '"'
+    elif isinstance(obj, list):
+        elts = [to_json(elt, level + 1)
+                for elt in obj]
+        ret = '[' + ', '.join(elts) + ']'
+    elif isinstance(obj, dict):
+        elts = ['"%s": %s' % (key.replace('"', r'\"'),
+                              to_json(obj[key], level + 1))
+                for key in sorted(obj.keys())]
+        ret = '{' + ', '.join(elts) + '}'
+    else:
+        assert False                # not implemented
+    if level == 1:
+        ret = '\n' + ret
+    return ret
+
+
+def to_c_string(string):
+    return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"'
+
+
+class QAPISchemaGenIntrospectVisitor(QAPISchemaVisitor):
+    def __init__(self, unmask):
+        self._unmask = unmask
+        self.defn = None
+        self.decl = None
+        self._schema = None
+        self._jsons = None
+        self._used_types = None
+        self._name_map = None
+
+    def visit_begin(self, schema):
+        self._schema = schema
+        self._jsons = []
+        self._used_types = []
+        self._name_map = {}
+
+    def visit_end(self):
+        # visit the types that are actually used
+        jsons = self._jsons
+        self._jsons = []
+        for typ in self._used_types:
+            typ.visit(self)
+        # generate C
+        # TODO can generate awfully long lines
+        jsons.extend(self._jsons)
+        name = prefix + 'qmp_schema_json'
+        self.decl = mcgen('''
+extern const char %(c_name)s[];
+''',
+                          c_name=c_name(name))
+        lines = to_json(jsons).split('\n')
+        c_string = '\n    '.join([to_c_string(line) for line in lines])
+        self.defn = mcgen('''
+const char %(c_name)s[] = %(c_string)s;
+''',
+                          c_name=c_name(name),
+                          c_string=c_string)
+        self._schema = None
+        self._jsons = None
+        self._used_types = None
+        self._name_map = None
+
+    def visit_needed(self, entity):
+        # Ignore types on first pass; visit_end() will pick up used types
+        return not isinstance(entity, QAPISchemaType)
+
+    def _name(self, name):
+        if self._unmask:
+            return name
+        if name not in self._name_map:
+            self._name_map[name] = '%d' % len(self._name_map)
+        return self._name_map[name]
+
+    def _use_type(self, typ):
+        # Map the various integer types to plain int
+        if typ.json_type() == 'int':
+            typ = self._schema.lookup_type('int')
+        elif (isinstance(typ, QAPISchemaArrayType) and
+              typ.element_type.json_type() == 'int'):
+            typ = self._schema.lookup_type('intList')
+        # Add type to work queue if new
+        if typ not in self._used_types:
+            self._used_types.append(typ)
+        # Clients should examine commands and events, not types.  Hide
+        # type names to reduce the temptation.  Also saves a few
+        # characters.
+        if isinstance(typ, QAPISchemaBuiltinType):
+            return typ.name
+        if isinstance(typ, QAPISchemaArrayType):
+            return '[' + self._use_type(typ.element_type) + ']'
+        return self._name(typ.name)
+
+    def _gen_json(self, name, mtype, obj):
+        if mtype not in ('command', 'event', 'builtin', 'array'):
+            name = self._name(name)
+        obj['name'] = name
+        obj['meta-type'] = mtype
+        self._jsons.append(obj)
+
+    def _gen_member(self, member):
+        ret = {'name': member.name, 'type': self._use_type(member.type)}
+        if member.optional:
+            ret['default'] = None
+        return ret
+
+    def _gen_variants(self, tag_name, variants):
+        return {'tag': tag_name,
+                'variants': [self._gen_variant(v) for v in variants]}
+
+    def _gen_variant(self, variant):
+        return {'case': variant.name, 'type': self._use_type(variant.type)}
+
+    def visit_builtin_type(self, name, info, json_type):
+        self._gen_json(name, 'builtin', {'json-type': json_type})
+
+    def visit_enum_type(self, name, info, values, prefix):
+        self._gen_json(name, 'enum', {'values': values})
+
+    def visit_array_type(self, name, info, element_type):
+        element = self._use_type(element_type)
+        self._gen_json('[' + element + ']', 'array', {'element-type': element})
+
+    def visit_object_type_flat(self, name, info, members, variants):
+        obj = {'members': [self._gen_member(m) for m in members]}
+        if variants:
+            obj.update(self._gen_variants(variants.tag_member.name,
+                                          variants.variants))
+        self._gen_json(name, 'object', obj)
+
+    def visit_alternate_type(self, name, info, variants):
+        self._gen_json(name, 'alternate',
+                       {'members': [{'type': self._use_type(m.type)}
+                                    for m in variants.variants]})
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response, boxed):
+        arg_type = arg_type or self._schema.the_empty_object_type
+        ret_type = ret_type or self._schema.the_empty_object_type
+        self._gen_json(name, 'command',
+                       {'arg-type': self._use_type(arg_type),
+                        'ret-type': self._use_type(ret_type)})
+
+    def visit_event(self, name, info, arg_type, boxed):
+        arg_type = arg_type or self._schema.the_empty_object_type
+        self._gen_json(name, 'event', {'arg-type': self._use_type(arg_type)})
+
+# Debugging aid: unmask QAPI schema's type names
+# We normally mask them, because they're not QMP wire ABI
+opt_unmask = False
+
+(input_file, output_dir, do_c, do_h, prefix, opts) = \
+    parse_command_line("u", ["unmask-non-abi-names"])
+
+for o, a in opts:
+    if o in ("-u", "--unmask-non-abi-names"):
+        opt_unmask = True
+
+c_comment = '''
+/*
+ * QAPI/QMP schema introspection
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+h_comment = '''
+/*
+ * QAPI/QMP schema introspection
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qmp-introspect.c', 'qmp-introspect.h',
+                            c_comment, h_comment)
+
+fdef.write(mcgen('''
+#include "qemu/osdep.h"
+#include "%(prefix)sqmp-introspect.h"
+
+''',
+                 prefix=prefix))
+
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenIntrospectVisitor(opt_unmask)
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
+
+close_output(fdef, fdecl)
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index d2f815b..dabc42e 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -2,342 +2,255 @@
 # QAPI types generator
 #
 # Copyright IBM, Corp. 2011
+# Copyright (c) 2013-2016 Red Hat Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
+#  Markus Armbruster <armbru@redhat.com>
 #
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
-import sys
-import os
-import getopt
-import errno
 
-def generate_fwd_struct(name, members, builtin_type=False):
-    if builtin_type:
-        return mcgen('''
 
-typedef struct %(name)sList
-{
-    union {
-        %(type)s value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
-''',
-                     type=c_type(name),
-                     name=name)
+# variants must be emitted before their container; track what has already
+# been output
+objects_seen = set()
 
+
+def gen_fwd_object_or_array(name):
     return mcgen('''
 
-typedef struct %(name)s %(name)s;
-
-typedef struct %(name)sList
-{
-    union {
-        %(name)s *value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
+typedef struct %(c_name)s %(c_name)s;
 ''',
-                 name=name)
+                 c_name=c_name(name))
 
-def generate_fwd_enum_struct(name, members):
+
+def gen_array(name, element_type):
     return mcgen('''
-typedef struct %(name)sList
-{
-    union {
-        %(name)s value;
-        uint64_t padding;
-    };
-    struct %(name)sList *next;
-} %(name)sList;
-''',
-                 name=name)
 
-def generate_struct_fields(members):
+struct %(c_name)s {
+    %(c_name)s *next;
+    %(c_type)s value;
+};
+''',
+                 c_name=c_name(name), c_type=element_type.c_type())
+
+
+def gen_struct_members(members):
     ret = ''
-
-    for argname, argentry, optional, structured in parse_args(members):
-        if optional:
+    for memb in members:
+        if memb.optional:
             ret += mcgen('''
     bool has_%(c_name)s;
 ''',
-                         c_name=c_var(argname))
-        if structured:
-            push_indent()
-            ret += generate_struct({ "field": argname, "data": argentry})
-            pop_indent()
-        else:
-            ret += mcgen('''
+                         c_name=c_name(memb.name))
+        ret += mcgen('''
     %(c_type)s %(c_name)s;
 ''',
-                     c_type=c_type(argentry), c_name=c_var(argname))
-
+                     c_type=memb.type.c_type(), c_name=c_name(memb.name))
     return ret
 
-def generate_struct(expr):
 
-    structname = expr.get('type', "")
-    fieldname = expr.get('field', "")
-    members = expr['data']
-    base = expr.get('base')
+def gen_object(name, base, members, variants):
+    if name in objects_seen:
+        return ''
+    objects_seen.add(name)
 
-    ret = mcgen('''
-struct %(name)s
-{
+    ret = ''
+    if variants:
+        for v in variants.variants:
+            if isinstance(v.type, QAPISchemaObjectType):
+                ret += gen_object(v.type.name, v.type.base,
+                                  v.type.local_members, v.type.variants)
+
+    ret += mcgen('''
+
+struct %(c_name)s {
 ''',
-          name=structname)
+                 c_name=c_name(name))
 
     if base:
-        ret += generate_struct_fields({'base': base})
-
-    ret += generate_struct_fields(members)
-
-    if len(fieldname):
-        fieldname = " " + fieldname
-    ret += mcgen('''
-}%(field)s;
+        if not base.is_implicit():
+            ret += mcgen('''
+    /* Members inherited from %(c_name)s: */
 ''',
-            field=fieldname)
-
-    return ret
-
-def generate_enum_lookup(name, values):
-    ret = mcgen('''
-const char *%(name)s_lookup[] = {
-''',
-                         name=name)
-    i = 0
-    for value in values:
-        ret += mcgen('''
-    "%(value)s",
-''',
-                     value=value)
-
-    ret += mcgen('''
-    NULL,
-};
-
+                         c_name=base.c_name())
+        ret += gen_struct_members(base.members)
+        if not base.is_implicit():
+            ret += mcgen('''
+    /* Own members: */
 ''')
-    return ret
+    ret += gen_struct_members(members)
 
-def generate_enum(name, values):
-    lookup_decl = mcgen('''
-extern const char *%(name)s_lookup[];
-''',
-                name=name)
+    if variants:
+        ret += gen_variants(variants)
 
-    enum_decl = mcgen('''
-typedef enum %(name)s
-{
-''',
-                name=name)
-
-    # append automatically generated _MAX value
-    enum_values = values + [ 'MAX' ]
-
-    i = 0
-    for value in enum_values:
-        enum_full_value = generate_enum_full_value(name, value)
-        enum_decl += mcgen('''
-    %(enum_full_value)s = %(i)d,
-''',
-                     enum_full_value = enum_full_value,
-                     i=i)
-        i += 1
-
-    enum_decl += mcgen('''
-} %(name)s;
-''',
-                 name=name)
-
-    return lookup_decl + enum_decl
-
-def generate_anon_union_qtypes(expr):
-
-    name = expr['union']
-    members = expr['data']
-
-    ret = mcgen('''
-const int %(name)s_qtypes[QTYPE_MAX] = {
-''',
-    name=name)
-
-    for key in members:
-        qapi_type = members[key]
-        if builtin_type_qtypes.has_key(qapi_type):
-            qtype = builtin_type_qtypes[qapi_type]
-        elif find_struct(qapi_type):
-            qtype = "QTYPE_QDICT"
-        elif find_union(qapi_type):
-            qtype = "QTYPE_QDICT"
-        elif find_enum(qapi_type):
-            qtype = "QTYPE_QSTRING"
-        else:
-            assert False, "Invalid anonymous union member"
-
+    # Make sure that all structs have at least one member; this avoids
+    # potential issues with attempting to malloc space for zero-length
+    # structs in C, and also incompatibility with C++ (where an empty
+    # struct is size 1).
+    if (not base or base.is_empty()) and not members and not variants:
         ret += mcgen('''
-    [ %(qtype)s ] = %(abbrev)s_KIND_%(enum)s,
-''',
-        qtype = qtype,
-        abbrev = de_camel_case(name).upper(),
-        enum = c_fun(de_camel_case(key),False).upper())
+    char qapi_dummy_for_empty_struct;
+''')
 
     ret += mcgen('''
 };
 ''')
+
     return ret
 
 
-def generate_union(expr):
+def gen_upcast(name, base):
+    # C makes const-correctness ugly.  We have to cast away const to let
+    # this function work for both const and non-const obj.
+    return mcgen('''
 
-    name = expr['union']
-    typeinfo = expr['data']
-
-    base = expr.get('base')
-    discriminator = expr.get('discriminator')
-
-    enum_define = discriminator_find_enum_define(expr)
-    if enum_define:
-        discriminator_type_name = enum_define['enum_name']
-    else:
-        discriminator_type_name = '%sKind' % (name)
-
-    ret = mcgen('''
-struct %(name)s
+static inline %(base)s *qapi_%(c_name)s_base(const %(c_name)s *obj)
 {
-    %(discriminator_type_name)s kind;
-    union {
-        void *data;
+    return (%(base)s *)obj;
+}
 ''',
-                name=name,
-                discriminator_type_name=discriminator_type_name)
+                 c_name=c_name(name), base=base.c_name())
 
-    for key in typeinfo:
+
+def gen_variants(variants):
+    ret = mcgen('''
+    union { /* union tag is @%(c_name)s */
+''',
+                c_name=c_name(variants.tag_member.name))
+
+    for var in variants.variants:
         ret += mcgen('''
         %(c_type)s %(c_name)s;
 ''',
-                     c_type=c_type(typeinfo[key]),
-                     c_name=c_fun(key))
+                     c_type=var.type.c_unboxed_type(),
+                     c_name=c_name(var.name))
 
     ret += mcgen('''
-    };
+    } u;
 ''')
 
-    if base:
-        base_fields = find_struct(base)['data']
-        if discriminator:
-            base_fields = base_fields.copy()
-            del base_fields[discriminator]
-        ret += generate_struct_fields(base_fields)
-    else:
-        assert not discriminator
-
-    ret += mcgen('''
-};
-''')
-    if discriminator == {}:
-        ret += mcgen('''
-extern const int %(name)s_qtypes[];
-''',
-            name=name)
-
-
     return ret
 
-def generate_type_cleanup_decl(name):
+
+def gen_type_cleanup_decl(name):
     ret = mcgen('''
-void qapi_free_%(type)s(%(c_type)s obj);
+
+void qapi_free_%(c_name)s(%(c_name)s *obj);
 ''',
-                c_type=c_type(name),type=name)
+                c_name=c_name(name))
     return ret
 
-def generate_type_cleanup(name):
+
+def gen_type_cleanup(name):
     ret = mcgen('''
 
-void qapi_free_%(type)s(%(c_type)s obj)
+void qapi_free_%(c_name)s(%(c_name)s *obj)
 {
-    QapiDeallocVisitor *md;
     Visitor *v;
 
     if (!obj) {
         return;
     }
 
-    md = qapi_dealloc_visitor_new();
-    v = qapi_dealloc_get_visitor(md);
-    visit_type_%(type)s(v, &obj, NULL, NULL);
-    qapi_dealloc_visitor_cleanup(md);
+    v = qapi_dealloc_visitor_new();
+    visit_type_%(c_name)s(v, NULL, &obj, NULL);
+    visit_free(v);
 }
 ''',
-                c_type=c_type(name),type=name)
+                c_name=c_name(name))
     return ret
 
 
-try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:i:o:",
-                                   ["source", "header", "builtins",
-                                    "prefix=", "input-file=", "output-dir="])
-except getopt.GetoptError, err:
-    print str(err)
-    sys.exit(1)
+class QAPISchemaGenTypeVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._fwdecl = None
+        self._btin = None
 
-output_dir = ""
-input_file = ""
-prefix = ""
-c_file = 'qapi-types.c'
-h_file = 'qapi-types.h'
+    def visit_begin(self, schema):
+        # gen_object() is recursive, ensure it doesn't visit the empty type
+        objects_seen.add(schema.the_empty_object_type.name)
+        self.decl = ''
+        self.defn = ''
+        self._fwdecl = ''
+        self._btin = guardstart('QAPI_TYPES_BUILTIN')
 
-do_c = False
-do_h = False
+    def visit_end(self):
+        self.decl = self._fwdecl + self.decl
+        self._fwdecl = None
+        # To avoid header dependency hell, we always generate
+        # declarations for built-in types in our header files and
+        # simply guard them.  See also do_builtins (command line
+        # option -b).
+        self._btin += guardend('QAPI_TYPES_BUILTIN')
+        self.decl = self._btin + self.decl
+        self._btin = None
+
+    def _gen_type_cleanup(self, name):
+        self.decl += gen_type_cleanup_decl(name)
+        self.defn += gen_type_cleanup(name)
+
+    def visit_enum_type(self, name, info, values, prefix):
+        # Special case for our lone builtin enum type
+        # TODO use something cleaner than existence of info
+        if not info:
+            self._btin += gen_enum(name, values, prefix)
+            if do_builtins:
+                self.defn += gen_enum_lookup(name, values, prefix)
+        else:
+            self._fwdecl += gen_enum(name, values, prefix)
+            self.defn += gen_enum_lookup(name, values, prefix)
+
+    def visit_array_type(self, name, info, element_type):
+        if isinstance(element_type, QAPISchemaBuiltinType):
+            self._btin += gen_fwd_object_or_array(name)
+            self._btin += gen_array(name, element_type)
+            self._btin += gen_type_cleanup_decl(name)
+            if do_builtins:
+                self.defn += gen_type_cleanup(name)
+        else:
+            self._fwdecl += gen_fwd_object_or_array(name)
+            self.decl += gen_array(name, element_type)
+            self._gen_type_cleanup(name)
+
+    def visit_object_type(self, name, info, base, members, variants):
+        # Nothing to do for the special empty builtin
+        if name == 'q_empty':
+            return
+        self._fwdecl += gen_fwd_object_or_array(name)
+        self.decl += gen_object(name, base, members, variants)
+        if base and not base.is_implicit():
+            self.decl += gen_upcast(name, base)
+        # TODO Worth changing the visitor signature, so we could
+        # directly use rather than repeat type.is_implicit()?
+        if not name.startswith('q_'):
+            # implicit types won't be directly allocated/freed
+            self._gen_type_cleanup(name)
+
+    def visit_alternate_type(self, name, info, variants):
+        self._fwdecl += gen_fwd_object_or_array(name)
+        self.decl += gen_object(name, None, [variants.tag_member], variants)
+        self._gen_type_cleanup(name)
+
+# If you link code generated from multiple schemata, you want only one
+# instance of the code for built-in types.  Generate it only when
+# do_builtins, enabled by command line option -b.  See also
+# QAPISchemaGenTypeVisitor.visit_end().
 do_builtins = False
 
+(input_file, output_dir, do_c, do_h, prefix, opts) = \
+    parse_command_line("b", ["builtins"])
+
 for o, a in opts:
-    if o in ("-p", "--prefix"):
-        prefix = a
-    elif o in ("-i", "--input-file"):
-        input_file = a
-    elif o in ("-o", "--output-dir"):
-        output_dir = a + "/"
-    elif o in ("-c", "--source"):
-        do_c = True
-    elif o in ("-h", "--header"):
-        do_h = True
-    elif o in ("-b", "--builtins"):
+    if o in ("-b", "--builtins"):
         do_builtins = True
 
-if not do_c and not do_h:
-    do_c = True
-    do_h = True
-
-c_file = output_dir + prefix + c_file
-h_file = output_dir + prefix + h_file
-
-try:
-    os.makedirs(output_dir)
-except os.error, e:
-    if e.errno != errno.EEXIST:
-        raise
-
-def maybe_open(really, name, opt):
-    if really:
-        return open(name, opt)
-    else:
-        import StringIO
-        return StringIO.StringIO()
-
-fdef = maybe_open(do_c, c_file, 'w')
-fdecl = maybe_open(do_h, h_file, 'w')
-
-fdef.write(mcgen('''
-/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
-
+c_comment = '''
 /*
  * deallocation functions for schema-defined QAPI types
  *
@@ -351,16 +264,8 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
-
-#include "qapi/dealloc-visitor.h"
-#include "%(prefix)sqapi-types.h"
-#include "%(prefix)sqapi-visit.h"
-
-''',             prefix=prefix))
-
-fdecl.write(mcgen('''
-/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
-
+'''
+h_comment = '''
 /*
  * schema-defined QAPI types
  *
@@ -373,88 +278,24 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
+'''
 
-#ifndef %(guard)s
-#define %(guard)s
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qapi-types.c', 'qapi-types.h',
+                            c_comment, h_comment)
 
-#include <stdbool.h>
-#include <stdint.h>
-
+fdef.write(mcgen('''
+#include "qemu/osdep.h"
+#include "qapi/dealloc-visitor.h"
+#include "%(prefix)sqapi-types.h"
+#include "%(prefix)sqapi-visit.h"
 ''',
-                  guard=guardname(h_file)))
+                 prefix=prefix))
 
-exprs = parse_schema(input_file)
-exprs = filter(lambda expr: not expr.has_key('gen'), exprs)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenTypeVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
-fdecl.write(guardstart("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
-for typename in builtin_types:
-    fdecl.write(generate_fwd_struct(typename, None, builtin_type=True))
-fdecl.write(guardend("QAPI_TYPES_BUILTIN_STRUCT_DECL"))
-
-for expr in exprs:
-    ret = "\n"
-    if expr.has_key('type'):
-        ret += generate_fwd_struct(expr['type'], expr['data'])
-    elif expr.has_key('enum'):
-        ret += generate_enum(expr['enum'], expr['data']) + "\n"
-        ret += generate_fwd_enum_struct(expr['enum'], expr['data'])
-        fdef.write(generate_enum_lookup(expr['enum'], expr['data']))
-    elif expr.has_key('union'):
-        ret += generate_fwd_struct(expr['union'], expr['data']) + "\n"
-        enum_define = discriminator_find_enum_define(expr)
-        if not enum_define:
-            ret += generate_enum('%sKind' % expr['union'], expr['data'].keys())
-            fdef.write(generate_enum_lookup('%sKind' % expr['union'],
-                                            expr['data'].keys()))
-        if expr.get('discriminator') == {}:
-            fdef.write(generate_anon_union_qtypes(expr))
-    else:
-        continue
-    fdecl.write(ret)
-
-# to avoid header dependency hell, we always generate declarations
-# for built-in types in our header files and simply guard them
-fdecl.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DECL"))
-for typename in builtin_types:
-    fdecl.write(generate_type_cleanup_decl(typename + "List"))
-fdecl.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DECL"))
-
-# ...this doesn't work for cases where we link in multiple objects that
-# have the functions defined, so we use -b option to provide control
-# over these cases
-if do_builtins:
-    fdef.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DEF"))
-    for typename in builtin_types:
-        fdef.write(generate_type_cleanup(typename + "List"))
-    fdef.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DEF"))
-
-for expr in exprs:
-    ret = "\n"
-    if expr.has_key('type'):
-        ret += generate_struct(expr) + "\n"
-        ret += generate_type_cleanup_decl(expr['type'] + "List")
-        fdef.write(generate_type_cleanup(expr['type'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['type'])
-        fdef.write(generate_type_cleanup(expr['type']) + "\n")
-    elif expr.has_key('union'):
-        ret += generate_union(expr)
-        ret += generate_type_cleanup_decl(expr['union'] + "List")
-        fdef.write(generate_type_cleanup(expr['union'] + "List") + "\n")
-        ret += generate_type_cleanup_decl(expr['union'])
-        fdef.write(generate_type_cleanup(expr['union']) + "\n")
-    elif expr.has_key('enum'):
-        ret += generate_type_cleanup_decl(expr['enum'] + "List")
-        fdef.write(generate_type_cleanup(expr['enum'] + "List") + "\n")
-    else:
-        continue
-    fdecl.write(ret)
-
-fdecl.write('''
-#endif
-''')
-
-fdecl.flush()
-fdecl.close()
-
-fdef.flush()
-fdef.close()
+close_output(fdef, fdecl)
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 8f845a2..96f2491 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -2,7 +2,7 @@
 # QAPI visitor generator
 #
 # Copyright IBM, Corp. 2011
-# Copyright (C) 2014 Red Hat, Inc.
+# Copyright (C) 2014-2016 Red Hat, Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
@@ -12,582 +12,385 @@
 # This work is licensed under the terms of the GNU GPL, version 2.
 # See the COPYING file in the top-level directory.
 
-from ordereddict import OrderedDict
 from qapi import *
 import re
-import sys
-import os
-import getopt
-import errno
 
-implicit_structs = []
 
-def generate_visit_implicit_struct(type):
-    global implicit_structs
-    if type in implicit_structs:
-        return ''
-    implicit_structs.append(type)
+def gen_visit_decl(name, scalar=False):
+    c_type = c_name(name) + ' *'
+    if not scalar:
+        c_type += '*'
+    return mcgen('''
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_type)sobj, Error **errp);
+''',
+                 c_name=c_name(name), c_type=c_type)
+
+
+def gen_visit_members_decl(name):
     return mcgen('''
 
-static void visit_type_implicit_%(c_type)s(Visitor *m, %(c_type)s **obj, Error **errp)
+void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp);
+''',
+                 c_name=c_name(name))
+
+
+def gen_visit_object_members(name, base, members, variants):
+    ret = mcgen('''
+
+void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp)
 {
     Error *err = NULL;
 
-    visit_start_implicit_struct(m, (void **)obj, sizeof(%(c_type)s), &err);
-    if (!err) {
-        visit_type_%(c_type)s_fields(m, obj, errp);
-        visit_end_implicit_struct(m, &err);
-    }
-    error_propagate(errp, err);
-}
 ''',
-                 c_type=type_name(type))
-
-def generate_visit_struct_fields(name, field_prefix, fn_prefix, members, base = None):
-    substructs = []
-    ret = ''
-    if not fn_prefix:
-        full_name = name
-    else:
-        full_name = "%s_%s" % (name, fn_prefix)
-
-    for argname, argentry, optional, structured in parse_args(members):
-        if structured:
-            if not fn_prefix:
-                nested_fn_prefix = argname
-            else:
-                nested_fn_prefix = "%s_%s" % (fn_prefix, argname)
-
-            nested_field_prefix = "%s%s." % (field_prefix, argname)
-            ret += generate_visit_struct_fields(name, nested_field_prefix,
-                                                nested_fn_prefix, argentry)
-            ret += mcgen('''
-
-static void visit_type_%(full_name)s_field_%(c_name)s(Visitor *m, %(name)s **obj, Error **errp)
-{
-''',
-                         name=name, full_name=full_name, c_name=c_var(argname))
-            ret += generate_visit_struct_body(full_name, argname, argentry)
-            ret += mcgen('''
-}
-''')
-
-    if base:
-        ret += generate_visit_implicit_struct(base)
-
-    ret += mcgen('''
-
-static void visit_type_%(full_name)s_fields(Visitor *m, %(name)s **obj, Error **errp)
-{
-    Error *err = NULL;
-''',
-        name=name, full_name=full_name)
-    push_indent()
+                c_name=c_name(name))
 
     if base:
         ret += mcgen('''
-visit_type_implicit_%(type)s(m, &(*obj)->%(c_prefix)s%(c_name)s, &err);
-if (err) {
-    goto out;
-}
+    visit_type_%(c_type)s_members(v, (%(c_type)s *)obj, &err);
+    if (err) {
+        goto out;
+    }
 ''',
-                     c_prefix=c_var(field_prefix),
-                     type=type_name(base), c_name=c_var('base'))
+                     c_type=base.c_name())
 
-    for argname, argentry, optional, structured in parse_args(members):
-        if optional:
+    for memb in members:
+        if memb.optional:
             ret += mcgen('''
-visit_optional(m, &(*obj)->%(c_prefix)shas_%(c_name)s, "%(name)s", &err);
-if (!err && (*obj)->%(prefix)shas_%(c_name)s) {
+    if (visit_optional(v, "%(name)s", &obj->has_%(c_name)s)) {
 ''',
-                         c_prefix=c_var(field_prefix), prefix=field_prefix,
-                         c_name=c_var(argname), name=argname)
+                         name=memb.name, c_name=c_name(memb.name))
             push_indent()
-
-        if structured:
-            ret += mcgen('''
-visit_type_%(full_name)s_field_%(c_name)s(m, obj, &err);
+        ret += mcgen('''
+    visit_type_%(c_type)s(v, "%(name)s", &obj->%(c_name)s, &err);
+    if (err) {
+        goto out;
+    }
 ''',
-                         full_name=full_name, c_name=c_var(argname))
-        else:
-            ret += mcgen('''
-visit_type_%(type)s(m, &(*obj)->%(c_prefix)s%(c_name)s, "%(name)s", &err);
-''',
-                         c_prefix=c_var(field_prefix), prefix=field_prefix,
-                         type=type_name(argentry), c_name=c_var(argname),
-                         name=argname)
-
-        if optional:
+                     c_type=memb.type.c_name(), name=memb.name,
+                     c_name=c_name(memb.name))
+        if memb.optional:
             pop_indent()
             ret += mcgen('''
-}
-''')
-        ret += mcgen('''
-if (err) {
-    goto out;
-}
-''')
-
-    pop_indent()
-    if re.search('^ *goto out\\;', ret, re.MULTILINE):
-        ret += mcgen('''
-
-out:
-''')
-    ret += mcgen('''
-    error_propagate(errp, err);
-}
-''')
-    return ret
-
-
-def generate_visit_struct_body(field_prefix, name, members):
-    ret = mcgen('''
-    Error *err = NULL;
-
-''')
-
-    if not field_prefix:
-        full_name = name
-    else:
-        full_name = "%s_%s" % (field_prefix, name)
-
-    if len(field_prefix):
-        ret += mcgen('''
-    visit_start_struct(m, NULL, "", "%(name)s", 0, &err);
-''',
-                name=name)
-    else:
-        ret += mcgen('''
-    visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err);
-''',
-                name=name)
-
-    ret += mcgen('''
-    if (!err) {
-        if (*obj) {
-            visit_type_%(name)s_fields(m, obj, errp);
-        }
-        visit_end_struct(m, &err);
     }
-    error_propagate(errp, err);
-''',
-        name=full_name)
-
-    return ret
-
-def generate_visit_struct(expr):
-
-    name = expr['type']
-    members = expr['data']
-    base = expr.get('base')
-
-    ret = generate_visit_struct_fields(name, "", "", members, base)
-
-    ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
-{
-''',
-                name=name)
-
-    ret += generate_visit_struct_body("", name, members)
-
-    ret += mcgen('''
-}
 ''')
-    return ret
 
-def generate_visit_list(name, members):
-    return mcgen('''
-
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp)
-{
-    Error *err = NULL;
-    GenericList *i, **prev;
-
-    visit_start_list(m, name, &err);
-    if (err) {
-        goto out;
-    }
-
-    for (prev = (GenericList **)obj;
-         !err && (i = visit_next_list(m, prev, &err)) != NULL;
-         prev = &i) {
-        %(name)sList *native_i = (%(name)sList *)i;
-        visit_type_%(name)s(m, &native_i->value, NULL, &err);
-    }
-
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_list(m, &err);
-out:
-    error_propagate(errp, err);
-}
-''',
-                name=name)
-
-def generate_visit_enum(name, members):
-    return mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp)
-{
-    visit_type_enum(m, (int *)obj, %(name)s_lookup, "%(name)s", name, errp);
-}
-''',
-                 name=name)
-
-def generate_visit_anon_union(name, members):
-    ret = mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_implicit_struct(m, (void**) obj, sizeof(%(name)s), &err);
-    if (err) {
-        goto out;
-    }
-    visit_get_next_type(m, (int*) &(*obj)->kind, %(name)s_qtypes, name, &err);
-    if (err) {
-        goto out_end;
-    }
-    switch ((*obj)->kind) {
-''',
-    name=name)
-
-    # For anon union, always use the default enum type automatically generated
-    # as "'%sKind' % (name)"
-    disc_type = '%sKind' % (name)
-
-    for key in members:
-        assert (members[key] in builtin_types
-            or find_struct(members[key])
-            or find_union(members[key])
-            or find_enum(members[key])), "Invalid anonymous union member"
-
-        enum_full_value = generate_enum_full_value(disc_type, key)
+    if variants:
         ret += mcgen('''
-    case %(enum_full_value)s:
-        visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, name, &err);
+    switch (obj->%(c_name)s) {
+''',
+                     c_name=c_name(variants.tag_member.name))
+
+        for var in variants.variants:
+            ret += mcgen('''
+    case %(case)s:
+        visit_type_%(c_type)s_members(v, &obj->u.%(c_name)s, &err);
         break;
 ''',
-                enum_full_value = enum_full_value,
-                c_type = type_name(members[key]),
-                c_name = c_fun(key))
+                         case=c_enum_const(variants.tag_member.type.name,
+                                           var.name,
+                                           variants.tag_member.type.prefix),
+                         c_type=var.type.c_name(), c_name=c_name(var.name))
 
-    ret += mcgen('''
+        ret += mcgen('''
     default:
         abort();
     }
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_implicit_struct(m, &err);
+''')
+
+    # 'goto out' produced for base, for each member, and if variants were
+    # present
+    if base or members or variants:
+        ret += mcgen('''
+
 out:
+''')
+    ret += mcgen('''
     error_propagate(errp, err);
 }
 ''')
-
     return ret
 
 
-def generate_visit_union(expr):
+def gen_visit_list(name, element_type):
+    return mcgen('''
 
-    name = expr['union']
-    members = expr['data']
-
-    base = expr.get('base')
-    discriminator = expr.get('discriminator')
-
-    if discriminator == {}:
-        assert not base
-        return generate_visit_anon_union(name, members)
-
-    enum_define = discriminator_find_enum_define(expr)
-    if enum_define:
-        # Use the enum type as discriminator
-        ret = ""
-        disc_type = enum_define['enum_name']
-    else:
-        # There will always be a discriminator in the C switch code, by default it
-        # is an enum type generated silently as "'%sKind' % (name)"
-        ret = generate_visit_enum('%sKind' % name, members.keys())
-        disc_type = '%sKind' % (name)
-
-    if base:
-        base_fields = find_struct(base)['data']
-        if discriminator:
-            base_fields = base_fields.copy()
-            del base_fields[discriminator]
-        ret += generate_visit_struct_fields(name, "", "", base_fields)
-
-    if discriminator:
-        for key in members:
-            ret += generate_visit_implicit_struct(members[key])
-
-    ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp)
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
 {
     Error *err = NULL;
+    %(c_name)s *tail;
+    size_t size = sizeof(**obj);
 
-    visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err);
+    visit_start_list(v, name, (GenericList **)obj, size, &err);
     if (err) {
         goto out;
     }
-    if (*obj) {
-''',
-                 name=name)
 
-    if base:
-        ret += mcgen('''
-        visit_type_%(name)s_fields(m, obj, &err);
+    for (tail = *obj; tail;
+         tail = (%(c_name)s *)visit_next_list(v, (GenericList *)tail, size)) {
+        visit_type_%(c_elt_type)s(v, NULL, &tail->value, &err);
         if (err) {
-            goto out_obj;
-        }
-''',
-            name=name)
-
-    if not discriminator:
-        disc_key = "type"
-    else:
-        disc_key = discriminator
-    ret += mcgen('''
-        visit_type_%(disc_type)s(m, &(*obj)->kind, "%(disc_key)s", &err);
-        if (err) {
-            goto out_obj;
-        }
-        if (!visit_start_union(m, !!(*obj)->data, &err) || err) {
-            goto out_obj;
-        }
-        switch ((*obj)->kind) {
-''',
-                 disc_type = disc_type,
-                 disc_key = disc_key)
-
-    for key in members:
-        if not discriminator:
-            fmt = 'visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, "data", &err);'
-        else:
-            fmt = 'visit_type_implicit_%(c_type)s(m, &(*obj)->%(c_name)s, &err);'
-
-        enum_full_value = generate_enum_full_value(disc_type, key)
-        ret += mcgen('''
-        case %(enum_full_value)s:
-            ''' + fmt + '''
             break;
-''',
-                enum_full_value = enum_full_value,
-                c_type=type_name(members[key]),
-                c_name=c_fun(key))
-
-    ret += mcgen('''
-        default:
-            abort();
         }
-out_obj:
-        error_propagate(errp, err);
-        err = NULL;
-        visit_end_union(m, !!(*obj)->data, &err);
-        error_propagate(errp, err);
-        err = NULL;
     }
-    visit_end_struct(m, &err);
+
+    visit_end_list(v, (void **)obj);
+    if (err && visit_is_input(v)) {
+        qapi_free_%(c_name)s(*obj);
+        *obj = NULL;
+    }
 out:
     error_propagate(errp, err);
 }
-''')
-
-    return ret
-
-def generate_declaration(name, members, genlist=True, builtin_type=False):
-    ret = ""
-    if not builtin_type:
-        ret += mcgen('''
-
-void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp);
 ''',
-                    name=name)
+                 c_name=c_name(name), c_elt_type=element_type.c_name())
 
-    if genlist:
-        ret += mcgen('''
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp);
-''',
-                 name=name)
 
-    return ret
-
-def generate_enum_declaration(name, members, genlist=True):
-    ret = ""
-    if genlist:
-        ret += mcgen('''
-void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp);
-''',
-                     name=name)
-
-    return ret
-
-def generate_decl_enum(name, members, genlist=True):
+def gen_visit_enum(name):
     return mcgen('''
 
-void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp);
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error **errp)
+{
+    int value = *obj;
+    visit_type_enum(v, name, &value, %(c_name)s_lookup, errp);
+    *obj = value;
+}
 ''',
-                name=name)
+                 c_name=c_name(name))
 
-try:
-    opts, args = getopt.gnu_getopt(sys.argv[1:], "chbp:i:o:",
-                                   ["source", "header", "builtins", "prefix=",
-                                    "input-file=", "output-dir="])
-except getopt.GetoptError, err:
-    print str(err)
-    sys.exit(1)
 
-input_file = ""
-output_dir = ""
-prefix = ""
-c_file = 'qapi-visit.c'
-h_file = 'qapi-visit.h'
+def gen_visit_alternate(name, variants):
+    promote_int = 'true'
+    ret = ''
+    for var in variants.variants:
+        if var.type.alternate_qtype() == 'QTYPE_QINT':
+            promote_int = 'false'
 
-do_c = False
-do_h = False
+    ret += mcgen('''
+
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
+{
+    Error *err = NULL;
+
+    visit_start_alternate(v, name, (GenericAlternate **)obj, sizeof(**obj),
+                          %(promote_int)s, &err);
+    if (err) {
+        goto out;
+    }
+    if (!*obj) {
+        goto out_obj;
+    }
+    switch ((*obj)->type) {
+''',
+                 c_name=c_name(name), promote_int=promote_int)
+
+    for var in variants.variants:
+        ret += mcgen('''
+    case %(case)s:
+''',
+                     case=var.type.alternate_qtype())
+        if isinstance(var.type, QAPISchemaObjectType):
+            ret += mcgen('''
+        visit_start_struct(v, name, NULL, 0, &err);
+        if (err) {
+            break;
+        }
+        visit_type_%(c_type)s_members(v, &(*obj)->u.%(c_name)s, &err);
+        if (!err) {
+            visit_check_struct(v, &err);
+        }
+        visit_end_struct(v, NULL);
+''',
+                         c_type=var.type.c_name(),
+                         c_name=c_name(var.name))
+        else:
+            ret += mcgen('''
+        visit_type_%(c_type)s(v, name, &(*obj)->u.%(c_name)s, &err);
+''',
+                         c_type=var.type.c_name(),
+                         c_name=c_name(var.name))
+        ret += mcgen('''
+        break;
+''')
+
+    ret += mcgen('''
+    case QTYPE_NONE:
+        abort();
+    default:
+        error_setg(&err, QERR_INVALID_PARAMETER_TYPE, name ? name : "null",
+                   "%(name)s");
+    }
+out_obj:
+    visit_end_alternate(v, (void **)obj);
+    if (err && visit_is_input(v)) {
+        qapi_free_%(c_name)s(*obj);
+        *obj = NULL;
+    }
+out:
+    error_propagate(errp, err);
+}
+''',
+                 name=name, c_name=c_name(name))
+
+    return ret
+
+
+def gen_visit_object(name, base, members, variants):
+    return mcgen('''
+
+void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp)
+{
+    Error *err = NULL;
+
+    visit_start_struct(v, name, (void **)obj, sizeof(%(c_name)s), &err);
+    if (err) {
+        goto out;
+    }
+    if (!*obj) {
+        goto out_obj;
+    }
+    visit_type_%(c_name)s_members(v, *obj, &err);
+    if (err) {
+        goto out_obj;
+    }
+    visit_check_struct(v, &err);
+out_obj:
+    visit_end_struct(v, (void **)obj);
+    if (err && visit_is_input(v)) {
+        qapi_free_%(c_name)s(*obj);
+        *obj = NULL;
+    }
+out:
+    error_propagate(errp, err);
+}
+''',
+                 c_name=c_name(name))
+
+
+class QAPISchemaGenVisitVisitor(QAPISchemaVisitor):
+    def __init__(self):
+        self.decl = None
+        self.defn = None
+        self._btin = None
+
+    def visit_begin(self, schema):
+        self.decl = ''
+        self.defn = ''
+        self._btin = guardstart('QAPI_VISIT_BUILTIN')
+
+    def visit_end(self):
+        # To avoid header dependency hell, we always generate
+        # declarations for built-in types in our header files and
+        # simply guard them.  See also do_builtins (command line
+        # option -b).
+        self._btin += guardend('QAPI_VISIT_BUILTIN')
+        self.decl = self._btin + self.decl
+        self._btin = None
+
+    def visit_enum_type(self, name, info, values, prefix):
+        # Special case for our lone builtin enum type
+        # TODO use something cleaner than existence of info
+        if not info:
+            self._btin += gen_visit_decl(name, scalar=True)
+            if do_builtins:
+                self.defn += gen_visit_enum(name)
+        else:
+            self.decl += gen_visit_decl(name, scalar=True)
+            self.defn += gen_visit_enum(name)
+
+    def visit_array_type(self, name, info, element_type):
+        decl = gen_visit_decl(name)
+        defn = gen_visit_list(name, element_type)
+        if isinstance(element_type, QAPISchemaBuiltinType):
+            self._btin += decl
+            if do_builtins:
+                self.defn += defn
+        else:
+            self.decl += decl
+            self.defn += defn
+
+    def visit_object_type(self, name, info, base, members, variants):
+        # Nothing to do for the special empty builtin
+        if name == 'q_empty':
+            return
+        self.decl += gen_visit_members_decl(name)
+        self.defn += gen_visit_object_members(name, base, members, variants)
+        # TODO Worth changing the visitor signature, so we could
+        # directly use rather than repeat type.is_implicit()?
+        if not name.startswith('q_'):
+            # only explicit types need an allocating visit
+            self.decl += gen_visit_decl(name)
+            self.defn += gen_visit_object(name, base, members, variants)
+
+    def visit_alternate_type(self, name, info, variants):
+        self.decl += gen_visit_decl(name)
+        self.defn += gen_visit_alternate(name, variants)
+
+# If you link code generated from multiple schemata, you want only one
+# instance of the code for built-in types.  Generate it only when
+# do_builtins, enabled by command line option -b.  See also
+# QAPISchemaGenVisitVisitor.visit_end().
 do_builtins = False
 
+(input_file, output_dir, do_c, do_h, prefix, opts) = \
+    parse_command_line("b", ["builtins"])
+
 for o, a in opts:
-    if o in ("-p", "--prefix"):
-        prefix = a
-    elif o in ("-i", "--input-file"):
-        input_file = a
-    elif o in ("-o", "--output-dir"):
-        output_dir = a + "/"
-    elif o in ("-c", "--source"):
-        do_c = True
-    elif o in ("-h", "--header"):
-        do_h = True
-    elif o in ("-b", "--builtins"):
+    if o in ("-b", "--builtins"):
         do_builtins = True
 
-if not do_c and not do_h:
-    do_c = True
-    do_h = True
+c_comment = '''
+/*
+ * schema-defined QAPI visitor functions
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
+h_comment = '''
+/*
+ * schema-defined QAPI visitor functions
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+'''
 
-c_file = output_dir + prefix + c_file
-h_file = output_dir + prefix + h_file
-
-try:
-    os.makedirs(output_dir)
-except os.error, e:
-    if e.errno != errno.EEXIST:
-        raise
-
-def maybe_open(really, name, opt):
-    if really:
-        return open(name, opt)
-    else:
-        import StringIO
-        return StringIO.StringIO()
-
-fdef = maybe_open(do_c, c_file, 'w')
-fdecl = maybe_open(do_h, h_file, 'w')
+(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix,
+                            'qapi-visit.c', 'qapi-visit.h',
+                            c_comment, h_comment)
 
 fdef.write(mcgen('''
-/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
-
-/*
- * schema-defined QAPI visitor functions
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "%(header)s"
+#include "qapi/error.h"
+#include "%(prefix)sqapi-visit.h"
 ''',
-                 header=basename(h_file)))
+                 prefix=prefix))
 
 fdecl.write(mcgen('''
-/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT MODIFY */
-
-/*
- * schema-defined QAPI visitor functions
- *
- * Copyright IBM, Corp. 2011
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
- * See the COPYING.LIB file in the top-level directory.
- *
- */
-
-#ifndef %(guard)s
-#define %(guard)s
-
 #include "qapi/visitor.h"
+#include "qapi/qmp/qerror.h"
 #include "%(prefix)sqapi-types.h"
 
 ''',
-                  prefix=prefix, guard=guardname(h_file)))
+                  prefix=prefix))
 
-exprs = parse_schema(input_file)
+schema = QAPISchema(input_file)
+gen = QAPISchemaGenVisitVisitor()
+schema.visit(gen)
+fdef.write(gen.defn)
+fdecl.write(gen.decl)
 
-# to avoid header dependency hell, we always generate declarations
-# for built-in types in our header files and simply guard them
-fdecl.write(guardstart("QAPI_VISIT_BUILTIN_VISITOR_DECL"))
-for typename in builtin_types:
-    fdecl.write(generate_declaration(typename, None, genlist=True,
-                                     builtin_type=True))
-fdecl.write(guardend("QAPI_VISIT_BUILTIN_VISITOR_DECL"))
-
-# ...this doesn't work for cases where we link in multiple objects that
-# have the functions defined, so we use -b option to provide control
-# over these cases
-if do_builtins:
-    for typename in builtin_types:
-        fdef.write(generate_visit_list(typename, None))
-
-for expr in exprs:
-    if expr.has_key('type'):
-        ret = generate_visit_struct(expr)
-        ret += generate_visit_list(expr['type'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_declaration(expr['type'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('union'):
-        ret = generate_visit_union(expr)
-        ret += generate_visit_list(expr['union'], expr['data'])
-        fdef.write(ret)
-
-        enum_define = discriminator_find_enum_define(expr)
-        ret = ""
-        if not enum_define:
-            ret = generate_decl_enum('%sKind' % expr['union'],
-                                     expr['data'].keys())
-        ret += generate_declaration(expr['union'], expr['data'])
-        fdecl.write(ret)
-    elif expr.has_key('enum'):
-        ret = generate_visit_list(expr['enum'], expr['data'])
-        ret += generate_visit_enum(expr['enum'], expr['data'])
-        fdef.write(ret)
-
-        ret = generate_decl_enum(expr['enum'], expr['data'])
-        ret += generate_enum_declaration(expr['enum'], expr['data'])
-        fdecl.write(ret)
-
-fdecl.write('''
-#endif
-''')
-
-fdecl.flush()
-fdecl.close()
-
-fdef.flush()
-fdef.close()
+close_output(fdef, fdecl)
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 77d46aa..21bc32f 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -2,7 +2,7 @@
 # QAPI helper library
 #
 # Copyright IBM, Corp. 2011
-# Copyright (c) 2013 Red Hat Inc.
+# Copyright (c) 2013-2016 Red Hat Inc.
 #
 # Authors:
 #  Anthony Liguori <aliguori@us.ibm.com>
@@ -13,16 +13,13 @@
 
 import re
 from ordereddict import OrderedDict
+import errno
+import getopt
 import os
 import sys
+import string
 
-builtin_types = [
-    'str', 'int', 'number', 'bool',
-    'int8', 'int16', 'int32', 'int64',
-    'uint8', 'uint16', 'uint32', 'uint64'
-]
-
-builtin_type_qtypes = {
+builtin_types = {
     'str':      'QTYPE_QSTRING',
     'int':      'QTYPE_QINT',
     'number':   'QTYPE_QFLOAT',
@@ -35,8 +32,56 @@
     'uint16':   'QTYPE_QINT',
     'uint32':   'QTYPE_QINT',
     'uint64':   'QTYPE_QINT',
+    'size':     'QTYPE_QINT',
+    'any':      None,           # any QType possible, actually
+    'QType':    'QTYPE_QSTRING',
 }
 
+# Whitelist of commands allowed to return a non-dictionary
+returns_whitelist = [
+    # From QMP:
+    'human-monitor-command',
+    'qom-get',
+    'query-migrate-cache-size',
+    'query-tpm-models',
+    'query-tpm-types',
+    'ringbuf-read',
+
+    # From QGA:
+    'guest-file-open',
+    'guest-fsfreeze-freeze',
+    'guest-fsfreeze-freeze-list',
+    'guest-fsfreeze-status',
+    'guest-fsfreeze-thaw',
+    'guest-get-time',
+    'guest-set-vcpus',
+    'guest-sync',
+    'guest-sync-delimited',
+]
+
+# Whitelist of entities allowed to violate case conventions
+case_whitelist = [
+    # From QMP:
+    'ACPISlotType',         # DIMM, visible through query-acpi-ospm-status
+    'CpuInfoMIPS',          # PC, visible through query-cpu
+    'CpuInfoTricore',       # PC, visible through query-cpu
+    'QapiErrorClass',       # all members, visible through errors
+    'UuidInfo',             # UUID, visible through query-uuid
+    'X86CPURegister32',     # all members, visible indirectly through qom-get
+    'q_obj_CpuInfo-base',   # CPU, visible through query-cpu
+]
+
+enum_types = []
+struct_types = []
+union_types = []
+events = []
+all_names = {}
+
+#
+# Parsing the schema into expressions
+#
+
+
 def error_path(parent):
     res = ""
     while parent:
@@ -45,9 +90,11 @@
         parent = parent['parent']
     return res
 
+
 class QAPISchemaError(Exception):
     def __init__(self, schema, msg):
-        self.input_file = schema.input_file
+        Exception.__init__(self)
+        self.fname = schema.fname
         self.msg = msg
         self.col = 1
         self.line = schema.line
@@ -56,14 +103,17 @@
                 self.col = (self.col + 7) % 8 + 1
             else:
                 self.col += 1
-        self.info = schema.parent_info
+        self.info = schema.incl_info
 
     def __str__(self):
         return error_path(self.info) + \
-            "%s:%d:%d: %s" % (self.input_file, self.line, self.col, self.msg)
+            "%s:%d:%d: %s" % (self.fname, self.line, self.col, self.msg)
+
 
 class QAPIExprError(Exception):
     def __init__(self, expr_info, msg):
+        Exception.__init__(self)
+        assert expr_info
         self.info = expr_info
         self.msg = msg
 
@@ -71,21 +121,15 @@
         return error_path(self.info['parent']) + \
             "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg)
 
-class QAPISchema:
 
-    def __init__(self, fp, input_relname=None, include_hist=[],
-                 previously_included=[], parent_info=None):
-        """ include_hist is a stack used to detect inclusion cycles
-            previously_included is a global state used to avoid multiple
-                                inclusions of the same file"""
-        input_fname = os.path.abspath(fp.name)
-        if input_relname is None:
-            input_relname = fp.name
-        self.input_dir = os.path.dirname(input_fname)
-        self.input_file = input_relname
-        self.include_hist = include_hist + [(input_relname, input_fname)]
-        previously_included.append(input_fname)
-        self.parent_info = parent_info
+class QAPISchemaParser(object):
+
+    def __init__(self, fp, previously_included=[], incl_info=None):
+        abs_fname = os.path.abspath(fp.name)
+        fname = fp.name
+        self.fname = fname
+        previously_included.append(abs_fname)
+        self.incl_info = incl_info
         self.src = fp.read()
         if self.src == '' or self.src[-1] != '\n':
             self.src += '\n'
@@ -95,32 +139,37 @@
         self.exprs = []
         self.accept()
 
-        while self.tok != None:
-            expr_info = {'file': input_relname, 'line': self.line, 'parent': self.parent_info}
+        while self.tok is not None:
+            expr_info = {'file': fname, 'line': self.line,
+                         'parent': self.incl_info}
             expr = self.get_expr(False)
             if isinstance(expr, dict) and "include" in expr:
                 if len(expr) != 1:
-                    raise QAPIExprError(expr_info, "Invalid 'include' directive")
+                    raise QAPIExprError(expr_info,
+                                        "Invalid 'include' directive")
                 include = expr["include"]
                 if not isinstance(include, str):
                     raise QAPIExprError(expr_info,
-                                        'Expected a file name (string), got: %s'
-                                        % include)
-                include_path = os.path.join(self.input_dir, include)
-                for elem in self.include_hist:
-                    if include_path == elem[1]:
+                                        "Value of 'include' must be a string")
+                incl_abs_fname = os.path.join(os.path.dirname(abs_fname),
+                                              include)
+                # catch inclusion cycle
+                inf = expr_info
+                while inf:
+                    if incl_abs_fname == os.path.abspath(inf['file']):
                         raise QAPIExprError(expr_info, "Inclusion loop for %s"
                                             % include)
+                    inf = inf['parent']
                 # skip multiple include of the same file
-                if include_path in previously_included:
+                if incl_abs_fname in previously_included:
                     continue
                 try:
-                    fobj = open(include_path, 'r')
-                except IOError, e:
+                    fobj = open(incl_abs_fname, 'r')
+                except IOError as e:
                     raise QAPIExprError(expr_info,
                                         '%s: %s' % (e.strerror, include))
-                exprs_include = QAPISchema(fobj, include, self.include_hist,
-                                           previously_included, expr_info)
+                exprs_include = QAPISchemaParser(fobj, previously_included,
+                                                 expr_info)
                 self.exprs.extend(exprs_include.exprs)
             else:
                 expr_elem = {'expr': expr,
@@ -136,7 +185,7 @@
 
             if self.tok == '#':
                 self.cursor = self.src.find('\n', self.cursor)
-            elif self.tok in ['{', '}', ':', ',', '[', ']']:
+            elif self.tok in "{}:,[]":
                 return
             elif self.tok == "'":
                 string = ''
@@ -148,7 +197,41 @@
                         raise QAPISchemaError(self,
                                               'Missing terminating "\'"')
                     if esc:
-                        string += ch
+                        if ch == 'b':
+                            string += '\b'
+                        elif ch == 'f':
+                            string += '\f'
+                        elif ch == 'n':
+                            string += '\n'
+                        elif ch == 'r':
+                            string += '\r'
+                        elif ch == 't':
+                            string += '\t'
+                        elif ch == 'u':
+                            value = 0
+                            for _ in range(0, 4):
+                                ch = self.src[self.cursor]
+                                self.cursor += 1
+                                if ch not in "0123456789abcdefABCDEF":
+                                    raise QAPISchemaError(self,
+                                                          '\\u escape needs 4 '
+                                                          'hex digits')
+                                value = (value << 4) + int(ch, 16)
+                            # If Python 2 and 3 didn't disagree so much on
+                            # how to handle Unicode, then we could allow
+                            # Unicode string defaults.  But most of QAPI is
+                            # ASCII-only, so we aren't losing much for now.
+                            if not value or value > 0x7f:
+                                raise QAPISchemaError(self,
+                                                      'For now, \\u escape '
+                                                      'only supports non-zero '
+                                                      'values up to \\u007f')
+                            string += chr(value)
+                        elif ch in "\\/'\"":
+                            string += ch
+                        else:
+                            raise QAPISchemaError(self,
+                                                  "Unknown escape \\%s" % ch)
                         esc = False
                     elif ch == "\\":
                         esc = True
@@ -157,6 +240,18 @@
                         return
                     else:
                         string += ch
+            elif self.src.startswith("true", self.pos):
+                self.val = True
+                self.cursor += 3
+                return
+            elif self.src.startswith("false", self.pos):
+                self.val = False
+                self.cursor += 4
+                return
+            elif self.src.startswith("null", self.pos):
+                self.val = None
+                self.cursor += 3
+                return
             elif self.tok == '\n':
                 if self.cursor == len(self.src):
                     self.tok = None
@@ -196,8 +291,9 @@
         if self.tok == ']':
             self.accept()
             return expr
-        if not self.tok in [ '{', '[', "'" ]:
-            raise QAPISchemaError(self, 'Expected "{", "[", "]" or string')
+        if self.tok not in "{['tfn":
+            raise QAPISchemaError(self, 'Expected "{", "[", "]", string, '
+                                  'boolean or "null"')
         while True:
             expr.append(self.get_expr(True))
             if self.tok == ']':
@@ -216,19 +312,42 @@
         elif self.tok == '[':
             self.accept()
             expr = self.get_values()
-        elif self.tok == "'":
+        elif self.tok in "'tfn":
             expr = self.val
             self.accept()
         else:
             raise QAPISchemaError(self, 'Expected "{", "[" or string')
         return expr
 
-def find_base_fields(base):
+#
+# Semantic analysis of schema expressions
+# TODO fold into QAPISchema
+# TODO catching name collisions in generated code would be nice
+#
+
+
+def find_base_members(base):
+    if isinstance(base, dict):
+        return base
     base_struct_define = find_struct(base)
     if not base_struct_define:
         return None
     return base_struct_define['data']
 
+
+# Return the qtype of an alternate branch, or None on error.
+def find_alternate_member_qtype(qapi_type):
+    if qapi_type in builtin_types:
+        return builtin_types[qapi_type]
+    elif find_struct(qapi_type):
+        return "QTYPE_QDICT"
+    elif find_enum(qapi_type):
+        return "QTYPE_QSTRING"
+    elif find_union(qapi_type):
+        return "QTYPE_QDICT"
+    return None
+
+
 # Return the discriminator enum define if discriminator is specified as an
 # enum type, otherwise return None.
 def discriminator_find_enum_define(expr):
@@ -238,25 +357,200 @@
     if not (discriminator and base):
         return None
 
-    base_fields = find_base_fields(base)
-    if not base_fields:
+    base_members = find_base_members(base)
+    if not base_members:
         return None
 
-    discriminator_type = base_fields.get(discriminator)
+    discriminator_type = base_members.get(discriminator)
     if not discriminator_type:
         return None
 
     return find_enum(discriminator_type)
 
+
+# Names must be letters, numbers, -, and _.  They must start with letter,
+# except for downstream extensions which must start with __RFQDN_.
+# Dots are only valid in the downstream extension prefix.
+valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?'
+                        '[a-zA-Z][a-zA-Z0-9_-]*$')
+
+
+def check_name(expr_info, source, name, allow_optional=False,
+               enum_member=False):
+    global valid_name
+    membername = name
+
+    if not isinstance(name, str):
+        raise QAPIExprError(expr_info,
+                            "%s requires a string name" % source)
+    if name.startswith('*'):
+        membername = name[1:]
+        if not allow_optional:
+            raise QAPIExprError(expr_info,
+                                "%s does not allow optional name '%s'"
+                                % (source, name))
+    # Enum members can start with a digit, because the generated C
+    # code always prefixes it with the enum name
+    if enum_member and membername[0].isdigit():
+        membername = 'D' + membername
+    # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty'
+    # and 'q_obj_*' implicit type names.
+    if not valid_name.match(membername) or \
+       c_name(membername, False).startswith('q_'):
+        raise QAPIExprError(expr_info,
+                            "%s uses invalid name '%s'" % (source, name))
+
+
+def add_name(name, info, meta, implicit=False):
+    global all_names
+    check_name(info, "'%s'" % meta, name)
+    # FIXME should reject names that differ only in '_' vs. '.'
+    # vs. '-', because they're liable to clash in generated C.
+    if name in all_names:
+        raise QAPIExprError(info,
+                            "%s '%s' is already defined"
+                            % (all_names[name], name))
+    if not implicit and (name.endswith('Kind') or name.endswith('List')):
+        raise QAPIExprError(info,
+                            "%s '%s' should not end in '%s'"
+                            % (meta, name, name[-4:]))
+    all_names[name] = meta
+
+
+def add_struct(definition, info):
+    global struct_types
+    name = definition['struct']
+    add_name(name, info, 'struct')
+    struct_types.append(definition)
+
+
+def find_struct(name):
+    global struct_types
+    for struct in struct_types:
+        if struct['struct'] == name:
+            return struct
+    return None
+
+
+def add_union(definition, info):
+    global union_types
+    name = definition['union']
+    add_name(name, info, 'union')
+    union_types.append(definition)
+
+
+def find_union(name):
+    global union_types
+    for union in union_types:
+        if union['union'] == name:
+            return union
+    return None
+
+
+def add_enum(name, info, enum_values=None, implicit=False):
+    global enum_types
+    add_name(name, info, 'enum', implicit)
+    enum_types.append({"enum_name": name, "enum_values": enum_values})
+
+
+def find_enum(name):
+    global enum_types
+    for enum in enum_types:
+        if enum['enum_name'] == name:
+            return enum
+    return None
+
+
+def is_enum(name):
+    return find_enum(name) is not None
+
+
+def check_type(expr_info, source, value, allow_array=False,
+               allow_dict=False, allow_optional=False,
+               allow_metas=[]):
+    global all_names
+
+    if value is None:
+        return
+
+    # Check if array type for value is okay
+    if isinstance(value, list):
+        if not allow_array:
+            raise QAPIExprError(expr_info,
+                                "%s cannot be an array" % source)
+        if len(value) != 1 or not isinstance(value[0], str):
+            raise QAPIExprError(expr_info,
+                                "%s: array type must contain single type name"
+                                % source)
+        value = value[0]
+
+    # Check if type name for value is okay
+    if isinstance(value, str):
+        if value not in all_names:
+            raise QAPIExprError(expr_info,
+                                "%s uses unknown type '%s'"
+                                % (source, value))
+        if not all_names[value] in allow_metas:
+            raise QAPIExprError(expr_info,
+                                "%s cannot use %s type '%s'"
+                                % (source, all_names[value], value))
+        return
+
+    if not allow_dict:
+        raise QAPIExprError(expr_info,
+                            "%s should be a type name" % source)
+
+    if not isinstance(value, OrderedDict):
+        raise QAPIExprError(expr_info,
+                            "%s should be a dictionary or type name" % source)
+
+    # value is a dictionary, check that each member is okay
+    for (key, arg) in value.items():
+        check_name(expr_info, "Member of %s" % source, key,
+                   allow_optional=allow_optional)
+        if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'):
+            raise QAPIExprError(expr_info,
+                                "Member of %s uses reserved name '%s'"
+                                % (source, key))
+        # Todo: allow dictionaries to represent default values of
+        # an optional argument.
+        check_type(expr_info, "Member '%s' of %s" % (key, source), arg,
+                   allow_array=True,
+                   allow_metas=['built-in', 'union', 'alternate', 'struct',
+                                'enum'])
+
+
+def check_command(expr, expr_info):
+    name = expr['command']
+    boxed = expr.get('boxed', False)
+
+    args_meta = ['struct']
+    if boxed:
+        args_meta += ['union', 'alternate']
+    check_type(expr_info, "'data' for command '%s'" % name,
+               expr.get('data'), allow_dict=not boxed, allow_optional=True,
+               allow_metas=args_meta)
+    returns_meta = ['union', 'struct']
+    if name in returns_whitelist:
+        returns_meta += ['built-in', 'alternate', 'enum']
+    check_type(expr_info, "'returns' for command '%s'" % name,
+               expr.get('returns'), allow_array=True,
+               allow_optional=True, allow_metas=returns_meta)
+
+
 def check_event(expr, expr_info):
-    params = expr.get('data')
-    if params:
-        for argname, argentry, optional, structured in parse_args(params):
-            if structured:
-                raise QAPIExprError(expr_info,
-                                    "Nested structure define in event is not "
-                                    "supported, event '%s', argname '%s'"
-                                    % (expr['event'], argname))
+    global events
+    name = expr['event']
+    boxed = expr.get('boxed', False)
+
+    meta = ['struct']
+    if boxed:
+        meta += ['union', 'alternate']
+    events.append(name)
+    check_type(expr_info, "'data' for event '%s'" % name,
+               expr.get('data'), allow_dict=not boxed, allow_optional=True,
+               allow_metas=meta)
+
 
 def check_union(expr, expr_info):
     name = expr['union']
@@ -264,128 +558,922 @@
     discriminator = expr.get('discriminator')
     members = expr['data']
 
-    # If the object has a member 'base', its value must name a complex type.
-    if base:
-        base_fields = find_base_fields(base)
-        if not base_fields:
+    # Two types of unions, determined by discriminator.
+
+    # With no discriminator it is a simple union.
+    if discriminator is None:
+        enum_define = None
+        allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum']
+        if base is not None:
             raise QAPIExprError(expr_info,
-                                "Base '%s' is not a valid type"
-                                % base)
-
-    # If the union object has no member 'discriminator', it's an
-    # ordinary union.
-    if not discriminator:
-        enum_define = None
-
-    # Else if the value of member 'discriminator' is {}, it's an
-    # anonymous union.
-    elif discriminator == {}:
-        enum_define = None
+                                "Simple union '%s' must not have a base"
+                                % name)
 
     # Else, it's a flat union.
     else:
-        # The object must have a member 'base'.
+        # The object must have a string or dictionary 'base'.
+        check_type(expr_info, "'base' for union '%s'" % name,
+                   base, allow_dict=True, allow_optional=True,
+                   allow_metas=['struct'])
         if not base:
             raise QAPIExprError(expr_info,
-                                "Flat union '%s' must have a base field"
+                                "Flat union '%s' must have a base"
                                 % name)
-        # The value of member 'discriminator' must name a member of the
-        # base type.
-        discriminator_type = base_fields.get(discriminator)
+        base_members = find_base_members(base)
+        assert base_members
+
+        # The value of member 'discriminator' must name a non-optional
+        # member of the base struct.
+        check_name(expr_info, "Discriminator of flat union '%s'" % name,
+                   discriminator)
+        discriminator_type = base_members.get(discriminator)
         if not discriminator_type:
             raise QAPIExprError(expr_info,
                                 "Discriminator '%s' is not a member of base "
-                                "type '%s'"
+                                "struct '%s'"
                                 % (discriminator, base))
         enum_define = find_enum(discriminator_type)
+        allow_metas = ['struct']
         # Do not allow string discriminator
         if not enum_define:
             raise QAPIExprError(expr_info,
                                 "Discriminator '%s' must be of enumeration "
                                 "type" % discriminator)
 
-    # Check every branch
+    # Check every branch; don't allow an empty union
+    if len(members) == 0:
+        raise QAPIExprError(expr_info,
+                            "Union '%s' cannot have empty 'data'" % name)
     for (key, value) in members.items():
-        # If this named member's value names an enum type, then all members
+        check_name(expr_info, "Member of union '%s'" % name, key)
+
+        # Each value must name a known type
+        check_type(expr_info, "Member '%s' of union '%s'" % (key, name),
+                   value, allow_array=not base, allow_metas=allow_metas)
+
+        # If the discriminator names an enum type, then all members
         # of 'data' must also be members of the enum type.
-        if enum_define and not key in enum_define['enum_values']:
+        if enum_define:
+            if key not in enum_define['enum_values']:
+                raise QAPIExprError(expr_info,
+                                    "Discriminator value '%s' is not found in "
+                                    "enum '%s'" %
+                                    (key, enum_define["enum_name"]))
+
+    # If discriminator is user-defined, ensure all values are covered
+    if enum_define:
+        for value in enum_define['enum_values']:
+            if value not in members.keys():
+                raise QAPIExprError(expr_info,
+                                    "Union '%s' data missing '%s' branch"
+                                    % (name, value))
+
+
+def check_alternate(expr, expr_info):
+    name = expr['alternate']
+    members = expr['data']
+    types_seen = {}
+
+    # Check every branch; require at least two branches
+    if len(members) < 2:
+        raise QAPIExprError(expr_info,
+                            "Alternate '%s' should have at least two branches "
+                            "in 'data'" % name)
+    for (key, value) in members.items():
+        check_name(expr_info, "Member of alternate '%s'" % name, key)
+
+        # Ensure alternates have no type conflicts.
+        check_type(expr_info, "Member '%s' of alternate '%s'" % (key, name),
+                   value,
+                   allow_metas=['built-in', 'union', 'struct', 'enum'])
+        qtype = find_alternate_member_qtype(value)
+        if not qtype:
             raise QAPIExprError(expr_info,
-                                "Discriminator value '%s' is not found in "
-                                "enum '%s'" %
-                                (key, enum_define["enum_name"]))
-        # Todo: add checking for values. Key is checked as above, value can be
-        # also checked here, but we need more functions to handle array case.
+                                "Alternate '%s' member '%s' cannot use "
+                                "type '%s'" % (name, key, value))
+        if qtype in types_seen:
+            raise QAPIExprError(expr_info,
+                                "Alternate '%s' member '%s' can't "
+                                "be distinguished from member '%s'"
+                                % (name, key, types_seen[qtype]))
+        types_seen[qtype] = key
 
-def check_exprs(schema):
-    for expr_elem in schema.exprs:
+
+def check_enum(expr, expr_info):
+    name = expr['enum']
+    members = expr.get('data')
+    prefix = expr.get('prefix')
+
+    if not isinstance(members, list):
+        raise QAPIExprError(expr_info,
+                            "Enum '%s' requires an array for 'data'" % name)
+    if prefix is not None and not isinstance(prefix, str):
+        raise QAPIExprError(expr_info,
+                            "Enum '%s' requires a string for 'prefix'" % name)
+    for member in members:
+        check_name(expr_info, "Member of enum '%s'" % name, member,
+                   enum_member=True)
+
+
+def check_struct(expr, expr_info):
+    name = expr['struct']
+    members = expr['data']
+
+    check_type(expr_info, "'data' for struct '%s'" % name, members,
+               allow_dict=True, allow_optional=True)
+    check_type(expr_info, "'base' for struct '%s'" % name, expr.get('base'),
+               allow_metas=['struct'])
+
+
+def check_keys(expr_elem, meta, required, optional=[]):
+    expr = expr_elem['expr']
+    info = expr_elem['info']
+    name = expr[meta]
+    if not isinstance(name, str):
+        raise QAPIExprError(info,
+                            "'%s' key must have a string value" % meta)
+    required = required + [meta]
+    for (key, value) in expr.items():
+        if key not in required and key not in optional:
+            raise QAPIExprError(info,
+                                "Unknown key '%s' in %s '%s'"
+                                % (key, meta, name))
+        if (key == 'gen' or key == 'success-response') and value is not False:
+            raise QAPIExprError(info,
+                                "'%s' of %s '%s' should only use false value"
+                                % (key, meta, name))
+        if key == 'boxed' and value is not True:
+            raise QAPIExprError(info,
+                                "'%s' of %s '%s' should only use true value"
+                                % (key, meta, name))
+    for key in required:
+        if key not in expr:
+            raise QAPIExprError(info,
+                                "Key '%s' is missing from %s '%s'"
+                                % (key, meta, name))
+
+
+def check_exprs(exprs):
+    global all_names
+
+    # Learn the types and check for valid expression keys
+    for builtin in builtin_types.keys():
+        all_names[builtin] = 'built-in'
+    for expr_elem in exprs:
         expr = expr_elem['expr']
-        if expr.has_key('union'):
-            check_union(expr, expr_elem['info'])
-        if expr.has_key('event'):
-            check_event(expr, expr_elem['info'])
-
-def parse_schema(input_file):
-    try:
-        schema = QAPISchema(open(input_file, "r"))
-    except (QAPISchemaError, QAPIExprError), e:
-        print >>sys.stderr, e
-        exit(1)
-
-    exprs = []
-
-    for expr_elem in schema.exprs:
-        expr = expr_elem['expr']
-        if expr.has_key('enum'):
-            add_enum(expr['enum'], expr['data'])
-        elif expr.has_key('union'):
-            add_union(expr)
-        elif expr.has_key('type'):
-            add_struct(expr)
-        exprs.append(expr)
+        info = expr_elem['info']
+        if 'enum' in expr:
+            check_keys(expr_elem, 'enum', ['data'], ['prefix'])
+            add_enum(expr['enum'], info, expr['data'])
+        elif 'union' in expr:
+            check_keys(expr_elem, 'union', ['data'],
+                       ['base', 'discriminator'])
+            add_union(expr, info)
+        elif 'alternate' in expr:
+            check_keys(expr_elem, 'alternate', ['data'])
+            add_name(expr['alternate'], info, 'alternate')
+        elif 'struct' in expr:
+            check_keys(expr_elem, 'struct', ['data'], ['base'])
+            add_struct(expr, info)
+        elif 'command' in expr:
+            check_keys(expr_elem, 'command', [],
+                       ['data', 'returns', 'gen', 'success-response', 'boxed'])
+            add_name(expr['command'], info, 'command')
+        elif 'event' in expr:
+            check_keys(expr_elem, 'event', [], ['data', 'boxed'])
+            add_name(expr['event'], info, 'event')
+        else:
+            raise QAPIExprError(expr_elem['info'],
+                                "Expression is missing metatype")
 
     # Try again for hidden UnionKind enum
-    for expr_elem in schema.exprs:
+    for expr_elem in exprs:
         expr = expr_elem['expr']
-        if expr.has_key('union'):
+        if 'union' in expr:
             if not discriminator_find_enum_define(expr):
-                add_enum('%sKind' % expr['union'])
+                add_enum('%sKind' % expr['union'], expr_elem['info'],
+                         implicit=True)
+        elif 'alternate' in expr:
+            add_enum('%sKind' % expr['alternate'], expr_elem['info'],
+                     implicit=True)
 
-    try:
-        check_exprs(schema)
-    except QAPIExprError, e:
-        print >>sys.stderr, e
-        exit(1)
+    # Validate that exprs make sense
+    for expr_elem in exprs:
+        expr = expr_elem['expr']
+        info = expr_elem['info']
+
+        if 'enum' in expr:
+            check_enum(expr, info)
+        elif 'union' in expr:
+            check_union(expr, info)
+        elif 'alternate' in expr:
+            check_alternate(expr, info)
+        elif 'struct' in expr:
+            check_struct(expr, info)
+        elif 'command' in expr:
+            check_command(expr, info)
+        elif 'event' in expr:
+            check_event(expr, info)
+        else:
+            assert False, 'unexpected meta type'
 
     return exprs
 
-def parse_args(typeinfo):
-    if isinstance(typeinfo, basestring):
-        struct = find_struct(typeinfo)
-        assert struct != None
-        typeinfo = struct['data']
 
-    for member in typeinfo:
-        argname = member
-        argentry = typeinfo[member]
+#
+# Schema compiler frontend
+#
+
+class QAPISchemaEntity(object):
+    def __init__(self, name, info):
+        assert isinstance(name, str)
+        self.name = name
+        # For explicitly defined entities, info points to the (explicit)
+        # definition.  For builtins (and their arrays), info is None.
+        # For implicitly defined entities, info points to a place that
+        # triggered the implicit definition (there may be more than one
+        # such place).
+        self.info = info
+
+    def c_name(self):
+        return c_name(self.name)
+
+    def check(self, schema):
+        pass
+
+    def is_implicit(self):
+        return not self.info
+
+    def visit(self, visitor):
+        pass
+
+
+class QAPISchemaVisitor(object):
+    def visit_begin(self, schema):
+        pass
+
+    def visit_end(self):
+        pass
+
+    def visit_needed(self, entity):
+        # Default to visiting everything
+        return True
+
+    def visit_builtin_type(self, name, info, json_type):
+        pass
+
+    def visit_enum_type(self, name, info, values, prefix):
+        pass
+
+    def visit_array_type(self, name, info, element_type):
+        pass
+
+    def visit_object_type(self, name, info, base, members, variants):
+        pass
+
+    def visit_object_type_flat(self, name, info, members, variants):
+        pass
+
+    def visit_alternate_type(self, name, info, variants):
+        pass
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response, boxed):
+        pass
+
+    def visit_event(self, name, info, arg_type, boxed):
+        pass
+
+
+class QAPISchemaType(QAPISchemaEntity):
+    # Return the C type for common use.
+    # For the types we commonly box, this is a pointer type.
+    def c_type(self):
+        pass
+
+    # Return the C type to be used in a parameter list.
+    def c_param_type(self):
+        return self.c_type()
+
+    # Return the C type to be used where we suppress boxing.
+    def c_unboxed_type(self):
+        return self.c_type()
+
+    def json_type(self):
+        pass
+
+    def alternate_qtype(self):
+        json2qtype = {
+            'string':  'QTYPE_QSTRING',
+            'number':  'QTYPE_QFLOAT',
+            'int':     'QTYPE_QINT',
+            'boolean': 'QTYPE_QBOOL',
+            'object':  'QTYPE_QDICT'
+        }
+        return json2qtype.get(self.json_type())
+
+
+class QAPISchemaBuiltinType(QAPISchemaType):
+    def __init__(self, name, json_type, c_type):
+        QAPISchemaType.__init__(self, name, None)
+        assert not c_type or isinstance(c_type, str)
+        assert json_type in ('string', 'number', 'int', 'boolean', 'null',
+                             'value')
+        self._json_type_name = json_type
+        self._c_type_name = c_type
+
+    def c_name(self):
+        return self.name
+
+    def c_type(self):
+        return self._c_type_name
+
+    def c_param_type(self):
+        if self.name == 'str':
+            return 'const ' + self._c_type_name
+        return self._c_type_name
+
+    def json_type(self):
+        return self._json_type_name
+
+    def visit(self, visitor):
+        visitor.visit_builtin_type(self.name, self.info, self.json_type())
+
+
+class QAPISchemaEnumType(QAPISchemaType):
+    def __init__(self, name, info, values, prefix):
+        QAPISchemaType.__init__(self, name, info)
+        for v in values:
+            assert isinstance(v, QAPISchemaMember)
+            v.set_owner(name)
+        assert prefix is None or isinstance(prefix, str)
+        self.values = values
+        self.prefix = prefix
+
+    def check(self, schema):
+        seen = {}
+        for v in self.values:
+            v.check_clash(self.info, seen)
+
+    def is_implicit(self):
+        # See QAPISchema._make_implicit_enum_type()
+        return self.name.endswith('Kind')
+
+    def c_type(self):
+        return c_name(self.name)
+
+    def member_names(self):
+        return [v.name for v in self.values]
+
+    def json_type(self):
+        return 'string'
+
+    def visit(self, visitor):
+        visitor.visit_enum_type(self.name, self.info,
+                                self.member_names(), self.prefix)
+
+
+class QAPISchemaArrayType(QAPISchemaType):
+    def __init__(self, name, info, element_type):
+        QAPISchemaType.__init__(self, name, info)
+        assert isinstance(element_type, str)
+        self._element_type_name = element_type
+        self.element_type = None
+
+    def check(self, schema):
+        self.element_type = schema.lookup_type(self._element_type_name)
+        assert self.element_type
+
+    def is_implicit(self):
+        return True
+
+    def c_type(self):
+        return c_name(self.name) + pointer_suffix
+
+    def json_type(self):
+        return 'array'
+
+    def visit(self, visitor):
+        visitor.visit_array_type(self.name, self.info, self.element_type)
+
+
+class QAPISchemaObjectType(QAPISchemaType):
+    def __init__(self, name, info, base, local_members, variants):
+        # struct has local_members, optional base, and no variants
+        # flat union has base, variants, and no local_members
+        # simple union has local_members, variants, and no base
+        QAPISchemaType.__init__(self, name, info)
+        assert base is None or isinstance(base, str)
+        for m in local_members:
+            assert isinstance(m, QAPISchemaObjectTypeMember)
+            m.set_owner(name)
+        if variants is not None:
+            assert isinstance(variants, QAPISchemaObjectTypeVariants)
+            variants.set_owner(name)
+        self._base_name = base
+        self.base = None
+        self.local_members = local_members
+        self.variants = variants
+        self.members = None
+
+    def check(self, schema):
+        if self.members is False:               # check for cycles
+            raise QAPIExprError(self.info,
+                                "Object %s contains itself" % self.name)
+        if self.members:
+            return
+        self.members = False                    # mark as being checked
+        seen = OrderedDict()
+        if self._base_name:
+            self.base = schema.lookup_type(self._base_name)
+            assert isinstance(self.base, QAPISchemaObjectType)
+            self.base.check(schema)
+            self.base.check_clash(schema, self.info, seen)
+        for m in self.local_members:
+            m.check(schema)
+            m.check_clash(self.info, seen)
+        self.members = seen.values()
+        if self.variants:
+            self.variants.check(schema, seen)
+            assert self.variants.tag_member in self.members
+            self.variants.check_clash(schema, self.info, seen)
+
+    # Check that the members of this type do not cause duplicate JSON members,
+    # and update seen to track the members seen so far. Report any errors
+    # on behalf of info, which is not necessarily self.info
+    def check_clash(self, schema, info, seen):
+        assert not self.variants       # not implemented
+        for m in self.members:
+            m.check_clash(info, seen)
+
+    def is_implicit(self):
+        # See QAPISchema._make_implicit_object_type(), as well as
+        # _def_predefineds()
+        return self.name.startswith('q_')
+
+    def is_empty(self):
+        assert self.members is not None
+        return not self.members and not self.variants
+
+    def c_name(self):
+        assert self.name != 'q_empty'
+        return QAPISchemaType.c_name(self)
+
+    def c_type(self):
+        assert not self.is_implicit()
+        return c_name(self.name) + pointer_suffix
+
+    def c_unboxed_type(self):
+        return c_name(self.name)
+
+    def json_type(self):
+        return 'object'
+
+    def visit(self, visitor):
+        visitor.visit_object_type(self.name, self.info,
+                                  self.base, self.local_members, self.variants)
+        visitor.visit_object_type_flat(self.name, self.info,
+                                       self.members, self.variants)
+
+
+class QAPISchemaMember(object):
+    role = 'member'
+
+    def __init__(self, name):
+        assert isinstance(name, str)
+        self.name = name
+        self.owner = None
+
+    def set_owner(self, name):
+        assert not self.owner
+        self.owner = name
+
+    def check_clash(self, info, seen):
+        cname = c_name(self.name)
+        if cname.lower() != cname and self.owner not in case_whitelist:
+            raise QAPIExprError(info,
+                                "%s should not use uppercase" % self.describe())
+        if cname in seen:
+            raise QAPIExprError(info,
+                                "%s collides with %s"
+                                % (self.describe(), seen[cname].describe()))
+        seen[cname] = self
+
+    def _pretty_owner(self):
+        owner = self.owner
+        if owner.startswith('q_obj_'):
+            # See QAPISchema._make_implicit_object_type() - reverse the
+            # mapping there to create a nice human-readable description
+            owner = owner[6:]
+            if owner.endswith('-arg'):
+                return '(parameter of %s)' % owner[:-4]
+            elif owner.endswith('-base'):
+                return '(base of %s)' % owner[:-5]
+            else:
+                assert owner.endswith('-wrapper')
+                # Unreachable and not implemented
+                assert False
+        if owner.endswith('Kind'):
+            # See QAPISchema._make_implicit_enum_type()
+            return '(branch of %s)' % owner[:-4]
+        return '(%s of %s)' % (self.role, owner)
+
+    def describe(self):
+        return "'%s' %s" % (self.name, self._pretty_owner())
+
+
+class QAPISchemaObjectTypeMember(QAPISchemaMember):
+    def __init__(self, name, typ, optional):
+        QAPISchemaMember.__init__(self, name)
+        assert isinstance(typ, str)
+        assert isinstance(optional, bool)
+        self._type_name = typ
+        self.type = None
+        self.optional = optional
+
+    def check(self, schema):
+        assert self.owner
+        self.type = schema.lookup_type(self._type_name)
+        assert self.type
+
+
+class QAPISchemaObjectTypeVariants(object):
+    def __init__(self, tag_name, tag_member, variants):
+        # Flat unions pass tag_name but not tag_member.
+        # Simple unions and alternates pass tag_member but not tag_name.
+        # After check(), tag_member is always set, and tag_name remains
+        # a reliable witness of being used by a flat union.
+        assert bool(tag_member) != bool(tag_name)
+        assert (isinstance(tag_name, str) or
+                isinstance(tag_member, QAPISchemaObjectTypeMember))
+        assert len(variants) > 0
+        for v in variants:
+            assert isinstance(v, QAPISchemaObjectTypeVariant)
+        self._tag_name = tag_name
+        self.tag_member = tag_member
+        self.variants = variants
+
+    def set_owner(self, name):
+        for v in self.variants:
+            v.set_owner(name)
+
+    def check(self, schema, seen):
+        if not self.tag_member:    # flat union
+            self.tag_member = seen[c_name(self._tag_name)]
+            assert self._tag_name == self.tag_member.name
+        assert isinstance(self.tag_member.type, QAPISchemaEnumType)
+        for v in self.variants:
+            v.check(schema)
+            # Union names must match enum values; alternate names are
+            # checked separately. Use 'seen' to tell the two apart.
+            if seen:
+                assert v.name in self.tag_member.type.member_names()
+                assert isinstance(v.type, QAPISchemaObjectType)
+                v.type.check(schema)
+
+    def check_clash(self, schema, info, seen):
+        for v in self.variants:
+            # Reset seen map for each variant, since qapi names from one
+            # branch do not affect another branch
+            assert isinstance(v.type, QAPISchemaObjectType)
+            v.type.check_clash(schema, info, dict(seen))
+
+
+class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember):
+    role = 'branch'
+
+    def __init__(self, name, typ):
+        QAPISchemaObjectTypeMember.__init__(self, name, typ, False)
+
+
+class QAPISchemaAlternateType(QAPISchemaType):
+    def __init__(self, name, info, variants):
+        QAPISchemaType.__init__(self, name, info)
+        assert isinstance(variants, QAPISchemaObjectTypeVariants)
+        assert variants.tag_member
+        variants.set_owner(name)
+        variants.tag_member.set_owner(self.name)
+        self.variants = variants
+
+    def check(self, schema):
+        self.variants.tag_member.check(schema)
+        # Not calling self.variants.check_clash(), because there's nothing
+        # to clash with
+        self.variants.check(schema, {})
+        # Alternate branch names have no relation to the tag enum values;
+        # so we have to check for potential name collisions ourselves.
+        seen = {}
+        for v in self.variants.variants:
+            v.check_clash(self.info, seen)
+
+    def c_type(self):
+        return c_name(self.name) + pointer_suffix
+
+    def json_type(self):
+        return 'value'
+
+    def visit(self, visitor):
+        visitor.visit_alternate_type(self.name, self.info, self.variants)
+
+    def is_empty(self):
+        return False
+
+
+class QAPISchemaCommand(QAPISchemaEntity):
+    def __init__(self, name, info, arg_type, ret_type, gen, success_response,
+                 boxed):
+        QAPISchemaEntity.__init__(self, name, info)
+        assert not arg_type or isinstance(arg_type, str)
+        assert not ret_type or isinstance(ret_type, str)
+        self._arg_type_name = arg_type
+        self.arg_type = None
+        self._ret_type_name = ret_type
+        self.ret_type = None
+        self.gen = gen
+        self.success_response = success_response
+        self.boxed = boxed
+
+    def check(self, schema):
+        if self._arg_type_name:
+            self.arg_type = schema.lookup_type(self._arg_type_name)
+            assert (isinstance(self.arg_type, QAPISchemaObjectType) or
+                    isinstance(self.arg_type, QAPISchemaAlternateType))
+            self.arg_type.check(schema)
+            if self.boxed:
+                if self.arg_type.is_empty():
+                    raise QAPIExprError(self.info,
+                                        "Cannot use 'boxed' with empty type")
+            else:
+                assert not isinstance(self.arg_type, QAPISchemaAlternateType)
+                assert not self.arg_type.variants
+        elif self.boxed:
+            raise QAPIExprError(self.info,
+                                "Use of 'boxed' requires 'data'")
+        if self._ret_type_name:
+            self.ret_type = schema.lookup_type(self._ret_type_name)
+            assert isinstance(self.ret_type, QAPISchemaType)
+
+    def visit(self, visitor):
+        visitor.visit_command(self.name, self.info,
+                              self.arg_type, self.ret_type,
+                              self.gen, self.success_response, self.boxed)
+
+
+class QAPISchemaEvent(QAPISchemaEntity):
+    def __init__(self, name, info, arg_type, boxed):
+        QAPISchemaEntity.__init__(self, name, info)
+        assert not arg_type or isinstance(arg_type, str)
+        self._arg_type_name = arg_type
+        self.arg_type = None
+        self.boxed = boxed
+
+    def check(self, schema):
+        if self._arg_type_name:
+            self.arg_type = schema.lookup_type(self._arg_type_name)
+            assert (isinstance(self.arg_type, QAPISchemaObjectType) or
+                    isinstance(self.arg_type, QAPISchemaAlternateType))
+            self.arg_type.check(schema)
+            if self.boxed:
+                if self.arg_type.is_empty():
+                    raise QAPIExprError(self.info,
+                                        "Cannot use 'boxed' with empty type")
+            else:
+                assert not isinstance(self.arg_type, QAPISchemaAlternateType)
+                assert not self.arg_type.variants
+        elif self.boxed:
+            raise QAPIExprError(self.info,
+                                "Use of 'boxed' requires 'data'")
+
+    def visit(self, visitor):
+        visitor.visit_event(self.name, self.info, self.arg_type, self.boxed)
+
+
+class QAPISchema(object):
+    def __init__(self, fname):
+        try:
+            self.exprs = check_exprs(QAPISchemaParser(open(fname, "r")).exprs)
+            self._entity_dict = {}
+            self._predefining = True
+            self._def_predefineds()
+            self._predefining = False
+            self._def_exprs()
+            self.check()
+        except (QAPISchemaError, QAPIExprError) as err:
+            print >>sys.stderr, err
+            exit(1)
+
+    def _def_entity(self, ent):
+        # Only the predefined types are allowed to not have info
+        assert ent.info or self._predefining
+        assert ent.name not in self._entity_dict
+        self._entity_dict[ent.name] = ent
+
+    def lookup_entity(self, name, typ=None):
+        ent = self._entity_dict.get(name)
+        if typ and not isinstance(ent, typ):
+            return None
+        return ent
+
+    def lookup_type(self, name):
+        return self.lookup_entity(name, QAPISchemaType)
+
+    def _def_builtin_type(self, name, json_type, c_type):
+        self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type))
+        # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple
+        # qapi-types.h from a single .c, all arrays of builtins must be
+        # declared in the first file whether or not they are used.  Nicer
+        # would be to use lazy instantiation, while figuring out how to
+        # avoid compilation issues with multiple qapi-types.h.
+        self._make_array_type(name, None)
+
+    def _def_predefineds(self):
+        for t in [('str',    'string',  'char' + pointer_suffix),
+                  ('number', 'number',  'double'),
+                  ('int',    'int',     'int64_t'),
+                  ('int8',   'int',     'int8_t'),
+                  ('int16',  'int',     'int16_t'),
+                  ('int32',  'int',     'int32_t'),
+                  ('int64',  'int',     'int64_t'),
+                  ('uint8',  'int',     'uint8_t'),
+                  ('uint16', 'int',     'uint16_t'),
+                  ('uint32', 'int',     'uint32_t'),
+                  ('uint64', 'int',     'uint64_t'),
+                  ('size',   'int',     'uint64_t'),
+                  ('bool',   'boolean', 'bool'),
+                  ('any',    'value',   'QObject' + pointer_suffix)]:
+            self._def_builtin_type(*t)
+        self.the_empty_object_type = QAPISchemaObjectType('q_empty', None,
+                                                          None, [], None)
+        self._def_entity(self.the_empty_object_type)
+        qtype_values = self._make_enum_members(['none', 'qnull', 'qint',
+                                                'qstring', 'qdict', 'qlist',
+                                                'qfloat', 'qbool'])
+        self._def_entity(QAPISchemaEnumType('QType', None, qtype_values,
+                                            'QTYPE'))
+
+    def _make_enum_members(self, values):
+        return [QAPISchemaMember(v) for v in values]
+
+    def _make_implicit_enum_type(self, name, info, values):
+        # See also QAPISchemaObjectTypeMember._pretty_owner()
+        name = name + 'Kind'   # Use namespace reserved by add_name()
+        self._def_entity(QAPISchemaEnumType(
+            name, info, self._make_enum_members(values), None))
+        return name
+
+    def _make_array_type(self, element_type, info):
+        name = element_type + 'List'   # Use namespace reserved by add_name()
+        if not self.lookup_type(name):
+            self._def_entity(QAPISchemaArrayType(name, info, element_type))
+        return name
+
+    def _make_implicit_object_type(self, name, info, role, members):
+        if not members:
+            return None
+        # See also QAPISchemaObjectTypeMember._pretty_owner()
+        name = 'q_obj_%s-%s' % (name, role)
+        if not self.lookup_entity(name, QAPISchemaObjectType):
+            self._def_entity(QAPISchemaObjectType(name, info, None,
+                                                  members, None))
+        return name
+
+    def _def_enum_type(self, expr, info):
+        name = expr['enum']
+        data = expr['data']
+        prefix = expr.get('prefix')
+        self._def_entity(QAPISchemaEnumType(
+            name, info, self._make_enum_members(data), prefix))
+
+    def _make_member(self, name, typ, info):
         optional = False
-        structured = False
-        if member.startswith('*'):
-            argname = member[1:]
+        if name.startswith('*'):
+            name = name[1:]
             optional = True
-        if isinstance(argentry, OrderedDict):
-            structured = True
-        yield (argname, argentry, optional, structured)
+        if isinstance(typ, list):
+            assert len(typ) == 1
+            typ = self._make_array_type(typ[0], info)
+        return QAPISchemaObjectTypeMember(name, typ, optional)
 
-def de_camel_case(name):
-    new_name = ''
-    for ch in name:
-        if ch.isupper() and new_name:
-            new_name += '_'
-        if ch == '-':
-            new_name += '_'
+    def _make_members(self, data, info):
+        return [self._make_member(key, value, info)
+                for (key, value) in data.iteritems()]
+
+    def _def_struct_type(self, expr, info):
+        name = expr['struct']
+        base = expr.get('base')
+        data = expr['data']
+        self._def_entity(QAPISchemaObjectType(name, info, base,
+                                              self._make_members(data, info),
+                                              None))
+
+    def _make_variant(self, case, typ):
+        return QAPISchemaObjectTypeVariant(case, typ)
+
+    def _make_simple_variant(self, case, typ, info):
+        if isinstance(typ, list):
+            assert len(typ) == 1
+            typ = self._make_array_type(typ[0], info)
+        typ = self._make_implicit_object_type(
+            typ, info, 'wrapper', [self._make_member('data', typ, info)])
+        return QAPISchemaObjectTypeVariant(case, typ)
+
+    def _def_union_type(self, expr, info):
+        name = expr['union']
+        data = expr['data']
+        base = expr.get('base')
+        tag_name = expr.get('discriminator')
+        tag_member = None
+        if isinstance(base, dict):
+            base = (self._make_implicit_object_type(
+                    name, info, 'base', self._make_members(base, info)))
+        if tag_name:
+            variants = [self._make_variant(key, value)
+                        for (key, value) in data.iteritems()]
+            members = []
         else:
-            new_name += ch.lower()
-    return new_name
+            variants = [self._make_simple_variant(key, value, info)
+                        for (key, value) in data.iteritems()]
+            typ = self._make_implicit_enum_type(name, info,
+                                                [v.name for v in variants])
+            tag_member = QAPISchemaObjectTypeMember('type', typ, False)
+            members = [tag_member]
+        self._def_entity(
+            QAPISchemaObjectType(name, info, base, members,
+                                 QAPISchemaObjectTypeVariants(tag_name,
+                                                              tag_member,
+                                                              variants)))
+
+    def _def_alternate_type(self, expr, info):
+        name = expr['alternate']
+        data = expr['data']
+        variants = [self._make_variant(key, value)
+                    for (key, value) in data.iteritems()]
+        tag_member = QAPISchemaObjectTypeMember('type', 'QType', False)
+        self._def_entity(
+            QAPISchemaAlternateType(name, info,
+                                    QAPISchemaObjectTypeVariants(None,
+                                                                 tag_member,
+                                                                 variants)))
+
+    def _def_command(self, expr, info):
+        name = expr['command']
+        data = expr.get('data')
+        rets = expr.get('returns')
+        gen = expr.get('gen', True)
+        success_response = expr.get('success-response', True)
+        boxed = expr.get('boxed', False)
+        if isinstance(data, OrderedDict):
+            data = self._make_implicit_object_type(
+                name, info, 'arg', self._make_members(data, info))
+        if isinstance(rets, list):
+            assert len(rets) == 1
+            rets = self._make_array_type(rets[0], info)
+        self._def_entity(QAPISchemaCommand(name, info, data, rets, gen,
+                                           success_response, boxed))
+
+    def _def_event(self, expr, info):
+        name = expr['event']
+        data = expr.get('data')
+        boxed = expr.get('boxed', False)
+        if isinstance(data, OrderedDict):
+            data = self._make_implicit_object_type(
+                name, info, 'arg', self._make_members(data, info))
+        self._def_entity(QAPISchemaEvent(name, info, data, boxed))
+
+    def _def_exprs(self):
+        for expr_elem in self.exprs:
+            expr = expr_elem['expr']
+            info = expr_elem['info']
+            if 'enum' in expr:
+                self._def_enum_type(expr, info)
+            elif 'struct' in expr:
+                self._def_struct_type(expr, info)
+            elif 'union' in expr:
+                self._def_union_type(expr, info)
+            elif 'alternate' in expr:
+                self._def_alternate_type(expr, info)
+            elif 'command' in expr:
+                self._def_command(expr, info)
+            elif 'event' in expr:
+                self._def_event(expr, info)
+            else:
+                assert False
+
+    def check(self):
+        for ent in self._entity_dict.values():
+            ent.check(self)
+
+    def visit(self, visitor):
+        visitor.visit_begin(self)
+        for (name, entity) in sorted(self._entity_dict.items()):
+            if visitor.visit_needed(entity):
+                entity.visit(visitor)
+        visitor.visit_end()
+
+
+#
+# Code generation helpers
+#
 
 def camel_case(name):
     new_name = ''
@@ -400,18 +1488,59 @@
             new_name += ch.lower()
     return new_name
 
-def c_var(name, protect=True):
+
+# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
+# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
+# ENUM24_Name -> ENUM24_NAME
+def camel_to_upper(value):
+    c_fun_str = c_name(value, False)
+    if value.isupper():
+        return c_fun_str
+
+    new_name = ''
+    l = len(c_fun_str)
+    for i in range(l):
+        c = c_fun_str[i]
+        # When c is upper and no "_" appears before, do more checks
+        if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
+            if i < l - 1 and c_fun_str[i + 1].islower():
+                new_name += '_'
+            elif c_fun_str[i - 1].isdigit():
+                new_name += '_'
+        new_name += c
+    return new_name.lstrip('_').upper()
+
+
+def c_enum_const(type_name, const_name, prefix=None):
+    if prefix is not None:
+        type_name = prefix
+    return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper()
+
+c_name_trans = string.maketrans('.-', '__')
+
+
+# Map @name to a valid C identifier.
+# If @protect, avoid returning certain ticklish identifiers (like
+# C keywords) by prepending "q_".
+#
+# Used for converting 'name' from a 'name':'type' qapi definition
+# into a generated struct member, as well as converting type names
+# into substrings of a generated C function name.
+# '__a.b_c' -> '__a_b_c', 'x-foo' -> 'x_foo'
+# protect=True: 'int' -> 'q_int'; protect=False: 'int' -> 'int'
+def c_name(name, protect=True):
     # ANSI X3J11/88-090, 3.1.1
     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
-                     'default', 'do', 'double', 'else', 'enum', 'extern', 'float',
-                     'for', 'goto', 'if', 'int', 'long', 'register', 'return',
-                     'short', 'signed', 'sizeof', 'static', 'struct', 'switch',
-                     'typedef', 'union', 'unsigned', 'void', 'volatile', 'while'])
+                     'default', 'do', 'double', 'else', 'enum', 'extern',
+                     'float', 'for', 'goto', 'if', 'int', 'long', 'register',
+                     'return', 'short', 'signed', 'sizeof', 'static',
+                     'struct', 'switch', 'typedef', 'union', 'unsigned',
+                     'void', 'volatile', 'while'])
     # ISO/IEC 9899:1999, 6.4.1
     c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary'])
     # ISO/IEC 9899:2011, 6.4.1
-    c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic', '_Noreturn',
-                     '_Static_assert', '_Thread_local'])
+    c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic',
+                     '_Noreturn', '_Static_assert', '_Thread_local'])
     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
     # excluding _.*
     gcc_words = set(['asm', 'typeof'])
@@ -426,134 +1555,58 @@
                      'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not',
                      'not_eq', 'or', 'or_eq', 'xor', 'xor_eq'])
     # namespace pollution:
-    polluted_words = set(['unix', 'errno'])
-    if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words):
+    polluted_words = set(['unix', 'errno', 'mips', 'sparc'])
+    name = name.translate(c_name_trans)
+    if protect and (name in c89_words | c99_words | c11_words | gcc_words
+                    | cpp_words | polluted_words):
         return "q_" + name
-    return name.replace('-', '_').lstrip("*")
-
-def c_fun(name, protect=True):
-    return c_var(name, protect).replace('.', '_')
-
-def c_list_type(name):
-    return '%sList' % name
-
-def type_name(name):
-    if type(name) == list:
-        return c_list_type(name[0])
     return name
 
-enum_types = []
-struct_types = []
-union_types = []
-
-def add_struct(definition):
-    global struct_types
-    struct_types.append(definition)
-
-def find_struct(name):
-    global struct_types
-    for struct in struct_types:
-        if struct['type'] == name:
-            return struct
-    return None
-
-def add_union(definition):
-    global union_types
-    union_types.append(definition)
-
-def find_union(name):
-    global union_types
-    for union in union_types:
-        if union['union'] == name:
-            return union
-    return None
-
-def add_enum(name, enum_values = None):
-    global enum_types
-    enum_types.append({"enum_name": name, "enum_values": enum_values})
-
-def find_enum(name):
-    global enum_types
-    for enum in enum_types:
-        if enum['enum_name'] == name:
-            return enum
-    return None
-
-def is_enum(name):
-    return find_enum(name) != None
-
 eatspace = '\033EATSPACE.'
+pointer_suffix = ' *' + eatspace
 
-# A special suffix is added in c_type() for pointer types, and it's
-# stripped in mcgen(). So please notice this when you check the return
-# value of c_type() outside mcgen().
-def c_type(name, is_param=False):
-    if name == 'str':
-        if is_param:
-            return 'const char *' + eatspace
-        return 'char *' + eatspace
-
-    elif name == 'int':
-        return 'int64_t'
-    elif (name == 'int8' or name == 'int16' or name == 'int32' or
-          name == 'int64' or name == 'uint8' or name == 'uint16' or
-          name == 'uint32' or name == 'uint64'):
-        return name + '_t'
-    elif name == 'size':
-        return 'uint64_t'
-    elif name == 'bool':
-        return 'bool'
-    elif name == 'number':
-        return 'double'
-    elif type(name) == list:
-        return '%s *%s' % (c_list_type(name[0]), eatspace)
-    elif is_enum(name):
-        return name
-    elif name == None or len(name) == 0:
-        return 'void'
-    elif name == name.upper():
-        return '%sEvent *%s' % (camel_case(name), eatspace)
-    else:
-        return '%s *%s' % (name, eatspace)
-
-def is_c_ptr(name):
-    suffix = "*" + eatspace
-    return c_type(name).endswith(suffix)
 
 def genindent(count):
     ret = ""
-    for i in range(count):
+    for _ in range(count):
         ret += " "
     return ret
 
 indent_level = 0
 
+
 def push_indent(indent_amount=4):
     global indent_level
     indent_level += indent_amount
 
+
 def pop_indent(indent_amount=4):
     global indent_level
     indent_level -= indent_amount
 
-def cgen(code, **kwds):
-    indent = genindent(indent_level)
-    lines = code.split('\n')
-    lines = map(lambda x: indent + x, lines)
-    return '\n'.join(lines) % kwds + '\n'
 
-def mcgen(code, **kwds):
-    raw = cgen('\n'.join(code.split('\n')[1:-1]), **kwds)
+# Generate @code with @kwds interpolated.
+# Obey indent_level, and strip eatspace.
+def cgen(code, **kwds):
+    raw = code % kwds
+    if indent_level:
+        indent = genindent(indent_level)
+        # re.subn() lacks flags support before Python 2.7, use re.compile()
+        raw = re.subn(re.compile("^.", re.MULTILINE),
+                      indent + r'\g<0>', raw)
+        raw = raw[0]
     return re.sub(re.escape(eatspace) + ' *', '', raw)
 
-def basename(filename):
-    return filename.split("/")[-1]
+
+def mcgen(code, **kwds):
+    if code[0] == '\n':
+        code = code[1:]
+    return cgen(code, **kwds)
+
 
 def guardname(filename):
-    guard = basename(filename).rsplit(".", 1)[0]
-    for substr in [".", " ", "-"]:
-        guard = guard.replace(substr, "_")
-    return guard.upper() + '_H'
+    return c_name(filename, protect=False).upper()
+
 
 def guardstart(name):
     return mcgen('''
@@ -564,6 +1617,7 @@
 ''',
                  name=guardname(name))
 
+
 def guardend(name):
     return mcgen('''
 
@@ -572,29 +1626,185 @@
 ''',
                  name=guardname(name))
 
-# ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1
-# ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2
-# ENUM24_Name -> ENUM24_NAME
-def _generate_enum_string(value):
-    c_fun_str = c_fun(value, False)
-    if value.isupper():
-        return c_fun_str
 
-    new_name = ''
-    l = len(c_fun_str)
-    for i in range(l):
-        c = c_fun_str[i]
-        # When c is upper and no "_" appears before, do more checks
-        if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_":
-            # Case 1: next string is lower
-            # Case 2: previous string is digit
-            if (i < (l - 1) and c_fun_str[i + 1].islower()) or \
-            c_fun_str[i - 1].isdigit():
-                new_name += '_'
-        new_name += c
-    return new_name.lstrip('_').upper()
+def gen_enum_lookup(name, values, prefix=None):
+    ret = mcgen('''
 
-def generate_enum_full_value(enum_name, enum_value):
-    abbrev_string = _generate_enum_string(enum_name)
-    value_string = _generate_enum_string(enum_value)
-    return "%s_%s" % (abbrev_string, value_string)
+const char *const %(c_name)s_lookup[] = {
+''',
+                c_name=c_name(name))
+    for value in values:
+        index = c_enum_const(name, value, prefix)
+        ret += mcgen('''
+    [%(index)s] = "%(value)s",
+''',
+                     index=index, value=value)
+
+    max_index = c_enum_const(name, '_MAX', prefix)
+    ret += mcgen('''
+    [%(max_index)s] = NULL,
+};
+''',
+                 max_index=max_index)
+    return ret
+
+
+def gen_enum(name, values, prefix=None):
+    # append automatically generated _MAX value
+    enum_values = values + ['_MAX']
+
+    ret = mcgen('''
+
+typedef enum %(c_name)s {
+''',
+                c_name=c_name(name))
+
+    i = 0
+    for value in enum_values:
+        ret += mcgen('''
+    %(c_enum)s = %(i)d,
+''',
+                     c_enum=c_enum_const(name, value, prefix),
+                     i=i)
+        i += 1
+
+    ret += mcgen('''
+} %(c_name)s;
+''',
+                 c_name=c_name(name))
+
+    ret += mcgen('''
+
+extern const char *const %(c_name)s_lookup[];
+''',
+                 c_name=c_name(name))
+    return ret
+
+
+def gen_params(arg_type, boxed, extra):
+    if not arg_type:
+        assert not boxed
+        return extra
+    ret = ''
+    sep = ''
+    if boxed:
+        ret += '%s arg' % arg_type.c_param_type()
+        sep = ', '
+    else:
+        assert not arg_type.variants
+        for memb in arg_type.members:
+            ret += sep
+            sep = ', '
+            if memb.optional:
+                ret += 'bool has_%s, ' % c_name(memb.name)
+            ret += '%s %s' % (memb.type.c_param_type(),
+                              c_name(memb.name))
+    if extra:
+        ret += sep + extra
+    return ret
+
+
+#
+# Common command line parsing
+#
+
+
+def parse_command_line(extra_options="", extra_long_options=[]):
+
+    try:
+        opts, args = getopt.gnu_getopt(sys.argv[1:],
+                                       "chp:o:" + extra_options,
+                                       ["source", "header", "prefix=",
+                                        "output-dir="] + extra_long_options)
+    except getopt.GetoptError as err:
+        print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err))
+        sys.exit(1)
+
+    output_dir = ""
+    prefix = ""
+    do_c = False
+    do_h = False
+    extra_opts = []
+
+    for oa in opts:
+        o, a = oa
+        if o in ("-p", "--prefix"):
+            match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a)
+            if match.end() != len(a):
+                print >>sys.stderr, \
+                    "%s: 'funny character '%s' in argument of --prefix" \
+                    % (sys.argv[0], a[match.end()])
+                sys.exit(1)
+            prefix = a
+        elif o in ("-o", "--output-dir"):
+            output_dir = a + "/"
+        elif o in ("-c", "--source"):
+            do_c = True
+        elif o in ("-h", "--header"):
+            do_h = True
+        else:
+            extra_opts.append(oa)
+
+    if not do_c and not do_h:
+        do_c = True
+        do_h = True
+
+    if len(args) != 1:
+        print >>sys.stderr, "%s: need exactly one argument" % sys.argv[0]
+        sys.exit(1)
+    fname = args[0]
+
+    return (fname, output_dir, do_c, do_h, prefix, extra_opts)
+
+#
+# Generate output files with boilerplate
+#
+
+
+def open_output(output_dir, do_c, do_h, prefix, c_file, h_file,
+                c_comment, h_comment):
+    guard = guardname(prefix + h_file)
+    c_file = output_dir + prefix + c_file
+    h_file = output_dir + prefix + h_file
+
+    if output_dir:
+        try:
+            os.makedirs(output_dir)
+        except os.error as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+    def maybe_open(really, name, opt):
+        if really:
+            return open(name, opt)
+        else:
+            import StringIO
+            return StringIO.StringIO()
+
+    fdef = maybe_open(do_c, c_file, 'w')
+    fdecl = maybe_open(do_h, h_file, 'w')
+
+    fdef.write(mcgen('''
+/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
+%(comment)s
+''',
+                     comment=c_comment))
+
+    fdecl.write(mcgen('''
+/* AUTOMATICALLY GENERATED, DO NOT MODIFY */
+%(comment)s
+#ifndef %(guard)s
+#define %(guard)s
+
+''',
+                      comment=h_comment, guard=guard))
+
+    return (fdef, fdecl)
+
+
+def close_output(fdef, fdecl):
+    fdecl.write('''
+#endif
+''')
+    fdecl.close()
+    fdef.close()
diff --git a/scripts/qemu-binfmt-conf.sh b/scripts/qemu-binfmt-conf.sh
old mode 100644
new mode 100755
index 289b1a3..de4d1c1
--- a/scripts/qemu-binfmt-conf.sh
+++ b/scripts/qemu-binfmt-conf.sh
@@ -1,72 +1,323 @@
 #!/bin/sh
 # enable automatic i386/ARM/M68K/MIPS/SPARC/PPC/s390 program execution by the kernel
 
-# load the binfmt_misc module
-if [ ! -d /proc/sys/fs/binfmt_misc ]; then
-  /sbin/modprobe binfmt_misc
-fi
-if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
-  mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
-fi
+qemu_target_list="i386 i486 alpha arm sparc32plus ppc ppc64 ppc64le m68k \
+mips mipsel mipsn32 mipsn32el mips64 mips64el \
+sh4 sh4eb s390x aarch64"
 
-# probe cpu type
-cpu=`uname -m`
-case "$cpu" in
-  i386|i486|i586|i686|i86pc|BePC|x86_64)
-    cpu="i386"
-  ;;
-  m68k)
-    cpu="m68k"
-  ;;
-  mips*)
-    cpu="mips"
-  ;;
-  "Power Macintosh"|ppc|ppc64)
-    cpu="ppc"
-  ;;
-  armv[4-9]*)
-    cpu="arm"
-  ;;
-esac
+i386_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00'
+i386_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+i386_family=i386
 
-# register the interpreter for each cpu except for the native one
-if [ $cpu != "i386" ] ; then
-    echo ':i386:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x03\x00:\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-i386:' > /proc/sys/fs/binfmt_misc/register
-    echo ':i486:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00:\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-i386:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "alpha" ] ; then
-    echo ':alpha:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90:\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-alpha:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "arm" ] ; then
-    echo   ':arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-arm:' > /proc/sys/fs/binfmt_misc/register
-    echo   ':armeb:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-armeb:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "aarch64" ] ; then
-    echo ':aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-aarch64:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "sparc" ] ; then
-    echo   ':sparc:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-sparc:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "ppc" ] ; then
-    echo   ':ppc:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-ppc:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "m68k" ] ; then
-    echo   'Please check cpu value and header information for m68k!'
-    echo   ':m68k:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-m68k:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "mips" ] ; then
-    # FIXME: We could use the other endianness on a MIPS host.
-    echo   ':mips:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-mips:' > /proc/sys/fs/binfmt_misc/register
-    echo   ':mipsel:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-mipsel:' > /proc/sys/fs/binfmt_misc/register
-    echo   ':mipsn32:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-mipsn32:' > /proc/sys/fs/binfmt_misc/register
-    echo   ':mipsn32el:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-mipsn32el:' > /proc/sys/fs/binfmt_misc/register
-    echo   ':mips64:M::\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-mips64:' > /proc/sys/fs/binfmt_misc/register
-    echo   ':mips64el:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-mips64el:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "sh" ] ; then
-    echo    ':sh4:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-sh4:' > /proc/sys/fs/binfmt_misc/register
-    echo    ':sh4eb:M::\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-sh4eb:' > /proc/sys/fs/binfmt_misc/register
-fi
-if [ $cpu != "s390x" ] ; then
-    echo   ':s390x:M::\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16:\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff:/usr/local/bin/qemu-s390x:' > /proc/sys/fs/binfmt_misc/register
-fi
+i486_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x06\x00'
+i486_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+i486_family=i386
+
+alpha_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x26\x90'
+alpha_mask='\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+alpha_family=alpha
+
+arm_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00'
+arm_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+arm_family=arm
+
+armeb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28'
+armeb_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+armeb_family=arm
+
+sparc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x02'
+sparc_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sparc_family=sparc
+
+sparc32plus_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x12'
+sparc32plus_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sparc32plus_family=sparc
+
+ppc_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x14'
+ppc_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+ppc_family=ppc
+
+ppc64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15'
+ppc64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+ppc64_family=ppc
+
+ppc64le_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x15\x00'
+ppc64le_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\x00'
+ppc64le_family=ppcle
+
+m68k_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x04'
+m68k_mask='\xff\xff\xff\xff\xff\xff\xfe\xfe\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+m68k_family=m68k
+
+# FIXME: We could use the other endianness on a MIPS host.
+
+mips_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
+mips_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+mips_family=mips
+
+mipsel_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
+mipsel_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+mipsel_family=mips
+
+mipsn32_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
+mipsn32_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+mipsn32_family=mips
+
+mipsn32el_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
+mipsn32el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+mipsn32el_family=mips
+
+mips64_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08'
+mips64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+mips64_family=mips
+
+mips64el_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x08\x00'
+mips64el_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+mips64el_family=mips
+
+sh4_magic='\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a\x00'
+sh4_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+sh4_family=sh4
+
+sh4eb_magic='\x7fELF\x01\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x2a'
+sh4eb_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+sh4eb_family=sh4
+
+s390x_magic='\x7fELF\x02\x02\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x16'
+s390x_mask='\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff'
+s390x_family=s390x
+
+aarch64_magic='\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00'
+aarch64_mask='\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff'
+aarch64_family=arm
+
+qemu_get_family() {
+    cpu=${HOST_ARCH:-$(uname -m)}
+    case "$cpu" in
+    amd64|i386|i486|i586|i686|i86pc|BePC|x86_64)
+        echo "i386"
+        ;;
+    mips*)
+        echo "mips"
+        ;;
+    "Power Macintosh"|ppc64|powerpc|ppc)
+        echo "ppc"
+        ;;
+    ppc64el|ppc64le)
+        echo "ppcle"
+        ;;
+    arm|armel|armhf|arm64|armv[4-9]*)
+        echo "arm"
+        ;;
+    sparc*)
+        echo "sparc"
+        ;;
+    *)
+        echo "$cpu"
+        ;;
+    esac
+}
+
+usage() {
+    cat <<EOF
+Usage: qemu-binfmt-conf.sh [--qemu-path PATH][--debian][--systemd CPU]
+                           [--help][--credential yes|no][--exportdir PATH]
+
+       Configure binfmt_misc to use qemu interpreter
+
+       --help:       display this usage
+       --qemu-path:  set path to qemu interpreter ($QEMU_PATH)
+       --debian:     don't write into /proc,
+                     instead generate update-binfmts templates
+       --systemd:    don't write into /proc,
+                     instead generate file for systemd-binfmt.service
+                     for the given CPU
+       --exportdir:  define where to write configuration files
+                     (default: $SYSTEMDDIR or $DEBIANDIR)
+       --credential: if yes, credential and security tokens are
+                     calculated according to the binary to interpret
+
+    To import templates with update-binfmts, use :
+
+        sudo update-binfmts --importdir ${EXPORTDIR:-$DEBIANDIR} --import qemu-CPU
+
+    To remove interpreter, use :
+
+        sudo update-binfmts --package qemu-CPU --remove qemu-CPU $QEMU_PATH
+
+    With systemd, binfmt files are loaded by systemd-binfmt.service
+
+    The environment variable HOST_ARCH allows to override 'uname' to generate
+    configuration files for a different architecture than the current one.
+
+    where CPU is one of:
+
+        $qemu_target_list
+
+EOF
+}
+
+qemu_check_access() {
+    if [ ! -w "$1" ] ; then
+        echo "ERROR: cannot write to $1" 1>&2
+        exit 1
+    fi
+}
+
+qemu_check_bintfmt_misc() {
+    # load the binfmt_misc module
+    if [ ! -d /proc/sys/fs/binfmt_misc ]; then
+      if ! /sbin/modprobe binfmt_misc ; then
+          exit 1
+      fi
+    fi
+    if [ ! -f /proc/sys/fs/binfmt_misc/register ]; then
+      if ! mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc ; then
+          exit 1
+      fi
+    fi
+
+    qemu_check_access /proc/sys/fs/binfmt_misc/register
+}
+
+installed_dpkg() {
+    dpkg --status "$1" > /dev/null 2>&1
+}
+
+qemu_check_debian() {
+    if [ ! -e /etc/debian_version ] ; then
+        echo "WARNING: your system is not a Debian based distro" 1>&2
+    elif ! installed_dpkg binfmt-support ; then
+        echo "WARNING: package binfmt-support is needed" 1>&2
+    fi
+    qemu_check_access "$EXPORTDIR"
+}
+
+qemu_check_systemd() {
+    if ! systemctl -q is-enabled systemd-binfmt.service ; then
+        echo "WARNING: systemd-binfmt.service is missing or disabled" 1>&2
+    fi
+    qemu_check_access "$EXPORTDIR"
+}
+
+qemu_generate_register() {
+    echo ":qemu-$cpu:M::$magic:$mask:$qemu:$FLAGS"
+}
+
+qemu_register_interpreter() {
+    echo "Setting $qemu as binfmt interpreter for $cpu"
+    qemu_generate_register > /proc/sys/fs/binfmt_misc/register
+}
+
+qemu_generate_systemd() {
+    echo "Setting $qemu as binfmt interpreter for $cpu for systemd-binfmt.service"
+    qemu_generate_register > "$EXPORTDIR/qemu-$cpu.conf"
+}
+
+qemu_generate_debian() {
+    cat > "$EXPORTDIR/qemu-$cpu" <<EOF
+package qemu-$cpu
+interpreter $qemu
+magic $magic
+mask $mask
+EOF
+    if [ "$FLAGS" = "OC" ] ; then
+        echo "credentials yes" >> "$EXPORTDIR/qemu-$cpu"
+    fi
+}
+
+qemu_set_binfmts() {
+    # probe cpu type
+    host_family=$(qemu_get_family)
+
+    # register the interpreter for each cpu except for the native one
+
+    for cpu in ${qemu_target_list} ; do
+        magic=$(eval echo \$${cpu}_magic)
+        mask=$(eval echo \$${cpu}_mask)
+        family=$(eval echo \$${cpu}_family)
+
+        if [ "$magic" = "" ] || [ "$mask" = "" ] || [ "$family" = "" ] ; then
+            echo "INTERNAL ERROR: unknown cpu $cpu" 1>&2
+            continue
+        fi
+
+        qemu="$QEMU_PATH/qemu-$cpu"
+        if [ "$cpu" = "i486" ] ; then
+            qemu="$QEMU_PATH/qemu-i386"
+        fi
+
+        if [ "$host_family" != "$family" ] ; then
+            $BINFMT_SET
+        fi
+    done
+}
+
+CHECK=qemu_check_bintfmt_misc
+BINFMT_SET=qemu_register_interpreter
+
+SYSTEMDDIR="/etc/binfmt.d"
+DEBIANDIR="/usr/share/binfmts"
+
+QEMU_PATH=/usr/local/bin
+FLAGS=""
+
+options=$(getopt -o ds:Q:e:hc: -l debian,systemd:,qemu-path:,exportdir:,help,credential: -- "$@")
+eval set -- "$options"
+
+while true ; do
+    case "$1" in
+    -d|--debian)
+        CHECK=qemu_check_debian
+        BINFMT_SET=qemu_generate_debian
+        EXPORTDIR=${EXPORTDIR:-$DEBIANDIR}
+        ;;
+    -s|--systemd)
+        CHECK=qemu_check_systemd
+        BINFMT_SET=qemu_generate_systemd
+        EXPORTDIR=${EXPORTDIR:-$SYSTEMDDIR}
+        shift
+        # check given cpu is in the supported CPU list
+        for cpu in ${qemu_target_list} ; do
+            if [ "$cpu" == "$1" ] ; then
+                break
+            fi
+        done
+
+        if [ "$cpu" == "$1" ] ; then
+            qemu_target_list="$1"
+        else
+            echo "ERROR: unknown CPU \"$1\"" 1>&2
+            usage
+            exit 1
+        fi
+        ;;
+    -Q|--qemu-path)
+        shift
+        QEMU_PATH="$1"
+        ;;
+    -e|--exportdir)
+        shift
+        EXPORTDIR="$1"
+        ;;
+    -h|--help)
+        usage
+        exit 1
+        ;;
+    -c|--credential)
+        shift
+        if [ "$1" = "yes" ] ; then
+            FLAGS="OC"
+        else
+            FLAGS=""
+        fi
+        ;;
+    *)
+        break
+        ;;
+    esac
+    shift
+done
+
+$CHECK
+qemu_set_binfmts
diff --git a/scripts/qemu-gdb.py b/scripts/qemu-gdb.py
index 8a0f305..b3f8e04 100644
--- a/scripts/qemu-gdb.py
+++ b/scripts/qemu-gdb.py
@@ -13,14 +13,20 @@
 # Contributions after 2012-01-13 are licensed under the terms of the
 # GNU GPL, version 2 or (at your option) any later version.
 
+# Usage:
+# At the (gdb) prompt, type "source scripts/qemu-gdb.py".
+# "help qemu" should then list the supported QEMU debug support commands.
 
 import gdb
 
-def isnull(ptr):
-    return ptr == gdb.Value(0).cast(ptr.type)
+import os, sys
 
-def int128(p):
-    return long(p['lo']) + (long(p['hi']) << 64)
+# Annoyingly, gdb doesn't put the directory of scripts onto the
+# module search path. Do it manually.
+
+sys.path.append(os.path.dirname(__file__))
+
+from qemugdb import aio, mtree, coroutine
 
 class QemuCommand(gdb.Command):
     '''Prefix for QEMU debug support commands'''
@@ -28,62 +34,14 @@
         gdb.Command.__init__(self, 'qemu', gdb.COMMAND_DATA,
                              gdb.COMPLETE_NONE, True)
 
-class MtreeCommand(gdb.Command):
-    '''Display the memory tree hierarchy'''
-    def __init__(self):
-        gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA,
-                             gdb.COMPLETE_NONE)
-        self.queue = []
-    def invoke(self, arg, from_tty):
-        self.seen = set()
-        self.queue_root('address_space_memory')
-        self.queue_root('address_space_io')
-        self.process_queue()
-    def queue_root(self, varname):
-        ptr = gdb.parse_and_eval(varname)['root']
-        self.queue.append(ptr)
-    def process_queue(self):
-        while self.queue:
-            ptr = self.queue.pop(0)
-            if long(ptr) in self.seen:
-                continue
-            self.print_item(ptr)
-    def print_item(self, ptr, offset = gdb.Value(0), level = 0):
-        self.seen.add(long(ptr))
-        addr = ptr['addr']
-        addr += offset
-        size = int128(ptr['size'])
-        alias = ptr['alias']
-        klass = ''
-        if not isnull(alias):
-            klass = ' (alias)'
-        elif not isnull(ptr['ops']):
-            klass = ' (I/O)'
-        elif bool(ptr['ram']):
-            klass = ' (RAM)'
-        gdb.write('%s%016x-%016x %s%s (@ %s)\n'
-                  % ('  ' * level,
-                     long(addr),
-                     long(addr + (size - 1)),
-                     ptr['name'].string(),
-                     klass,
-                     ptr,
-                     ),
-                  gdb.STDOUT)
-        if not isnull(alias):
-            gdb.write('%s    alias: %s@%016x (@ %s)\n' %
-                      ('  ' * level,
-                       alias['name'].string(),
-                       ptr['alias_offset'],
-                       alias,
-                       ),
-                      gdb.STDOUT)
-            self.queue.append(alias)
-        subregion = ptr['subregions']['tqh_first']
-        level += 1
-        while not isnull(subregion):
-            self.print_item(subregion, addr, level)
-            subregion = subregion['subregions_link']['tqe_next']
-
 QemuCommand()
-MtreeCommand()
+coroutine.CoroutineCommand()
+mtree.MtreeCommand()
+aio.HandlersCommand()
+
+coroutine.CoroutineSPFunction()
+coroutine.CoroutinePCFunction()
+
+# Default to silently passing through SIGUSR1, because QEMU sends it
+# to itself a lot.
+gdb.execute('handle SIGUSR1 pass noprint nostop')
diff --git a/scripts/qemu.py b/scripts/qemu.py
new file mode 100644
index 0000000..6d1b623
--- /dev/null
+++ b/scripts/qemu.py
@@ -0,0 +1,222 @@
+# QEMU library
+#
+# Copyright (C) 2015-2016 Red Hat Inc.
+# Copyright (C) 2012 IBM Corp.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Based on qmp.py.
+#
+
+import errno
+import string
+import os
+import sys
+import subprocess
+import qmp.qmp
+
+
+class QEMUMachine(object):
+    '''A QEMU VM'''
+
+    def __init__(self, binary, args=[], wrapper=[], name=None, test_dir="/var/tmp",
+                 monitor_address=None, socket_scm_helper=None, debug=False):
+        if name is None:
+            name = "qemu-%d" % os.getpid()
+        if monitor_address is None:
+            monitor_address = os.path.join(test_dir, name + "-monitor.sock")
+        self._monitor_address = monitor_address
+        self._qemu_log_path = os.path.join(test_dir, name + ".log")
+        self._popen = None
+        self._binary = binary
+        self._args = list(args) # Force copy args in case we modify them
+        self._wrapper = wrapper
+        self._events = []
+        self._iolog = None
+        self._socket_scm_helper = socket_scm_helper
+        self._debug = debug
+
+    # This can be used to add an unused monitor instance.
+    def add_monitor_telnet(self, ip, port):
+        args = 'tcp:%s:%d,server,nowait,telnet' % (ip, port)
+        self._args.append('-monitor')
+        self._args.append(args)
+
+    def add_fd(self, fd, fdset, opaque, opts=''):
+        '''Pass a file descriptor to the VM'''
+        options = ['fd=%d' % fd,
+                   'set=%d' % fdset,
+                   'opaque=%s' % opaque]
+        if opts:
+            options.append(opts)
+
+        self._args.append('-add-fd')
+        self._args.append(','.join(options))
+        return self
+
+    def send_fd_scm(self, fd_file_path):
+        # In iotest.py, the qmp should always use unix socket.
+        assert self._qmp.is_scm_available()
+        if self._socket_scm_helper is None:
+            print >>sys.stderr, "No path to socket_scm_helper set"
+            return -1
+        if os.path.exists(self._socket_scm_helper) == False:
+            print >>sys.stderr, "%s does not exist" % self._socket_scm_helper
+            return -1
+        fd_param = ["%s" % self._socket_scm_helper,
+                    "%d" % self._qmp.get_sock_fd(),
+                    "%s" % fd_file_path]
+        devnull = open('/dev/null', 'rb')
+        p = subprocess.Popen(fd_param, stdin=devnull, stdout=sys.stdout,
+                             stderr=sys.stderr)
+        return p.wait()
+
+    @staticmethod
+    def _remove_if_exists(path):
+        '''Remove file object at path if it exists'''
+        try:
+            os.remove(path)
+        except OSError as exception:
+            if exception.errno == errno.ENOENT:
+                return
+            raise
+
+    def get_pid(self):
+        if not self._popen:
+            return None
+        return self._popen.pid
+
+    def _load_io_log(self):
+        with open(self._qemu_log_path, "r") as fh:
+            self._iolog = fh.read()
+
+    def _base_args(self):
+        if isinstance(self._monitor_address, tuple):
+            moncdev = "socket,id=mon,host=%s,port=%s" % (
+                self._monitor_address[0],
+                self._monitor_address[1])
+        else:
+            moncdev = 'socket,id=mon,path=%s' % self._monitor_address
+        return ['-chardev', moncdev,
+                '-mon', 'chardev=mon,mode=control',
+                '-display', 'none', '-vga', 'none']
+
+    def _pre_launch(self):
+        self._qmp = qmp.qmp.QEMUMonitorProtocol(self._monitor_address, server=True,
+                                                debug=self._debug)
+
+    def _post_launch(self):
+        self._qmp.accept()
+
+    def _post_shutdown(self):
+        if not isinstance(self._monitor_address, tuple):
+            self._remove_if_exists(self._monitor_address)
+        self._remove_if_exists(self._qemu_log_path)
+
+    def launch(self):
+        '''Launch the VM and establish a QMP connection'''
+        devnull = open('/dev/null', 'rb')
+        qemulog = open(self._qemu_log_path, 'wb')
+        try:
+            self._pre_launch()
+            args = self._wrapper + [self._binary] + self._base_args() + self._args
+            self._popen = subprocess.Popen(args, stdin=devnull, stdout=qemulog,
+                                           stderr=subprocess.STDOUT, shell=False)
+            self._post_launch()
+        except:
+            if self._popen:
+                self._popen.kill()
+            self._load_io_log()
+            self._post_shutdown()
+            self._popen = None
+            raise
+
+    def shutdown(self):
+        '''Terminate the VM and clean up'''
+        if not self._popen is None:
+            try:
+                self._qmp.cmd('quit')
+                self._qmp.close()
+            except:
+                self._popen.kill()
+
+            exitcode = self._popen.wait()
+            if exitcode < 0:
+                sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, ' '.join(self._args)))
+            self._load_io_log()
+            self._post_shutdown()
+            self._popen = None
+
+    underscore_to_dash = string.maketrans('_', '-')
+    def qmp(self, cmd, conv_keys=True, **args):
+        '''Invoke a QMP command and return the result dict'''
+        qmp_args = dict()
+        for k in args.keys():
+            if conv_keys:
+                qmp_args[k.translate(self.underscore_to_dash)] = args[k]
+            else:
+                qmp_args[k] = args[k]
+
+        return self._qmp.cmd(cmd, args=qmp_args)
+
+    def command(self, cmd, conv_keys=True, **args):
+        reply = self.qmp(cmd, conv_keys, **args)
+        if reply is None:
+            raise Exception("Monitor is closed")
+        if "error" in reply:
+            raise Exception(reply["error"]["desc"])
+        return reply["return"]
+
+    def get_qmp_event(self, wait=False):
+        '''Poll for one queued QMP events and return it'''
+        if len(self._events) > 0:
+            return self._events.pop(0)
+        return self._qmp.pull_event(wait=wait)
+
+    def get_qmp_events(self, wait=False):
+        '''Poll for queued QMP events and return a list of dicts'''
+        events = self._qmp.get_events(wait=wait)
+        events.extend(self._events)
+        del self._events[:]
+        self._qmp.clear_events()
+        return events
+
+    def event_wait(self, name, timeout=60.0, match=None):
+        # Test if 'match' is a recursive subset of 'event'
+        def event_match(event, match=None):
+            if match is None:
+                return True
+
+            for key in match:
+                if key in event:
+                    if isinstance(event[key], dict):
+                        if not event_match(event[key], match[key]):
+                            return False
+                    elif event[key] != match[key]:
+                        return False
+                else:
+                    return False
+
+            return True
+
+        # Search cached events
+        for event in self._events:
+            if (event['event'] == name) and event_match(event, match):
+                self._events.remove(event)
+                return event
+
+        # Poll for new events
+        while True:
+            event = self._qmp.pull_event(wait=timeout)
+            if (event['event'] == name) and event_match(event, match):
+                return event
+            self._events.append(event)
+
+        return None
+
+    def get_log(self):
+        return self._iolog
diff --git a/scripts/qemugdb/__init__.py b/scripts/qemugdb/__init__.py
new file mode 100644
index 0000000..969f552
--- /dev/null
+++ b/scripts/qemugdb/__init__.py
@@ -0,0 +1,28 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright (c) 2015 Linaro Ltd
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see
+# <http://www.gnu.org/licenses/gpl-2.0.html>
+#
+
+# We don't need to do anything in our init file currently.
+
+"""
+Support routines for debugging QEMU under GDB
+"""
+
+__license__    = "GPL version 2 or (at your option) any later version"
diff --git a/scripts/qemugdb/aio.py b/scripts/qemugdb/aio.py
new file mode 100644
index 0000000..2ba00c4
--- /dev/null
+++ b/scripts/qemugdb/aio.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+
+# GDB debugging support: aio/iohandler debug
+#
+# Copyright (c) 2015 Red Hat, Inc.
+#
+# Author: Dr. David Alan Gilbert <dgilbert@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+#
+
+import gdb
+from qemugdb import coroutine
+
+def isnull(ptr):
+    return ptr == gdb.Value(0).cast(ptr.type)
+
+def dump_aiocontext(context, verbose):
+    '''Display a dump and backtrace for an aiocontext'''
+    cur = context['aio_handlers']['lh_first']
+    # Get pointers to functions we're going to process specially
+    sym_fd_coroutine_enter = gdb.parse_and_eval('fd_coroutine_enter')
+
+    while not isnull(cur):
+        entry = cur.dereference()
+        gdb.write('----\n%s\n' % entry)
+        if verbose and cur['io_read'] == sym_fd_coroutine_enter:
+            coptr = (cur['opaque'].cast(gdb.lookup_type('FDYieldUntilData').pointer()))['co']
+            coptr = coptr.cast(gdb.lookup_type('CoroutineUContext').pointer())
+            coroutine.bt_jmpbuf(coptr['env']['__jmpbuf'])
+        cur = cur['node']['le_next'];
+
+    gdb.write('----\n')
+
+class HandlersCommand(gdb.Command):
+    '''Display aio handlers'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu handlers', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+
+    def invoke(self, arg, from_tty):
+        verbose = False
+        argv = gdb.string_to_argv(arg)
+
+        if len(argv) > 0 and argv[0] == '--verbose':
+            verbose = True
+            argv.pop(0)
+
+        if len(argv) > 1:
+            gdb.write('usage: qemu handlers [--verbose] [handler]\n')
+            return
+
+        if len(argv) == 1:
+            handlers_name = argv[0]
+        else:
+            handlers_name = 'qemu_aio_context'
+        dump_aiocontext(gdb.parse_and_eval(handlers_name), verbose)
diff --git a/scripts/qemugdb/coroutine.py b/scripts/qemugdb/coroutine.py
new file mode 100644
index 0000000..ab69979
--- /dev/null
+++ b/scripts/qemugdb/coroutine.py
@@ -0,0 +1,119 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright 2012 Red Hat, Inc. and/or its affiliates
+#
+# Authors:
+#  Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Contributions after 2012-01-13 are licensed under the terms of the
+# GNU GPL, version 2 or (at your option) any later version.
+
+import gdb
+
+VOID_PTR = gdb.lookup_type('void').pointer()
+
+def get_fs_base():
+    '''Fetch %fs base value using arch_prctl(ARCH_GET_FS).  This is
+       pthread_self().'''
+    # %rsp - 120 is scratch space according to the SystemV ABI
+    old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
+    gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True)
+    fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)')
+    gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True)
+    return fs_base
+
+def pthread_self():
+    '''Fetch pthread_self() from the glibc start_thread function.'''
+    f = gdb.newest_frame()
+    while f.name() != 'start_thread':
+        f = f.older()
+        if f is None:
+            return get_fs_base()
+
+    try:
+        return f.read_var("arg")
+    except ValueError:
+        return get_fs_base()
+
+def get_glibc_pointer_guard():
+    '''Fetch glibc pointer guard value'''
+    fs_base = pthread_self()
+    return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base)
+
+def glibc_ptr_demangle(val, pointer_guard):
+    '''Undo effect of glibc's PTR_MANGLE()'''
+    return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard))
+
+def get_jmpbuf_regs(jmpbuf):
+    JB_RBX  = 0
+    JB_RBP  = 1
+    JB_R12  = 2
+    JB_R13  = 3
+    JB_R14  = 4
+    JB_R15  = 5
+    JB_RSP  = 6
+    JB_PC   = 7
+
+    pointer_guard = get_glibc_pointer_guard()
+    return {'rbx': jmpbuf[JB_RBX],
+        'rbp': glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard),
+        'rsp': glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard),
+        'r12': jmpbuf[JB_R12],
+        'r13': jmpbuf[JB_R13],
+        'r14': jmpbuf[JB_R14],
+        'r15': jmpbuf[JB_R15],
+        'rip': glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard) }
+
+def bt_jmpbuf(jmpbuf):
+    '''Backtrace a jmpbuf'''
+    regs = get_jmpbuf_regs(jmpbuf)
+    old = dict()
+
+    for i in regs:
+        old[i] = gdb.parse_and_eval('(uint64_t)$%s' % i)
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, regs[i]))
+
+    gdb.execute('bt')
+
+    for i in regs:
+        gdb.execute('set $%s = %s' % (i, old[i]))
+
+def coroutine_to_jmpbuf(co):
+    coroutine_pointer = co.cast(gdb.lookup_type('CoroutineUContext').pointer())
+    return coroutine_pointer['env']['__jmpbuf']
+
+
+class CoroutineCommand(gdb.Command):
+    '''Display coroutine backtrace'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+
+    def invoke(self, arg, from_tty):
+        argv = gdb.string_to_argv(arg)
+        if len(argv) != 1:
+            gdb.write('usage: qemu coroutine <coroutine-pointer>\n')
+            return
+
+        bt_jmpbuf(coroutine_to_jmpbuf(gdb.parse_and_eval(argv[0])))
+
+class CoroutineSPFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_sp')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rsp'].cast(VOID_PTR)
+
+class CoroutinePCFunction(gdb.Function):
+    def __init__(self):
+        gdb.Function.__init__(self, 'qemu_coroutine_pc')
+
+    def invoke(self, addr):
+        return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rip'].cast(VOID_PTR)
diff --git a/scripts/qemugdb/mtree.py b/scripts/qemugdb/mtree.py
new file mode 100644
index 0000000..cc8131c
--- /dev/null
+++ b/scripts/qemugdb/mtree.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+# GDB debugging support
+#
+# Copyright 2012 Red Hat, Inc. and/or its affiliates
+#
+# Authors:
+#  Avi Kivity <avi@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Contributions after 2012-01-13 are licensed under the terms of the
+# GNU GPL, version 2 or (at your option) any later version.
+
+# 'qemu mtree' -- display the memory hierarchy
+
+import gdb
+
+def isnull(ptr):
+    return ptr == gdb.Value(0).cast(ptr.type)
+
+def int128(p):
+    return int(p['lo']) + (int(p['hi']) << 64)
+
+class MtreeCommand(gdb.Command):
+    '''Display the memory tree hierarchy'''
+    def __init__(self):
+        gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA,
+                             gdb.COMPLETE_NONE)
+        self.queue = []
+    def invoke(self, arg, from_tty):
+        self.seen = set()
+        self.queue_root('address_space_memory')
+        self.queue_root('address_space_io')
+        self.process_queue()
+    def queue_root(self, varname):
+        ptr = gdb.parse_and_eval(varname)['root']
+        self.queue.append(ptr)
+    def process_queue(self):
+        while self.queue:
+            ptr = self.queue.pop(0)
+            if int(ptr) in self.seen:
+                continue
+            self.print_item(ptr)
+    def print_item(self, ptr, offset = gdb.Value(0), level = 0):
+        self.seen.add(int(ptr))
+        addr = ptr['addr']
+        addr += offset
+        size = int128(ptr['size'])
+        alias = ptr['alias']
+        klass = ''
+        if not isnull(alias):
+            klass = ' (alias)'
+        elif not isnull(ptr['ops']):
+            klass = ' (I/O)'
+        elif bool(ptr['ram']):
+            klass = ' (RAM)'
+        gdb.write('%s%016x-%016x %s%s (@ %s)\n'
+                  % ('  ' * level,
+                     int(addr),
+                     int(addr + (size - 1)),
+                     ptr['name'].string(),
+                     klass,
+                     ptr,
+                     ),
+                  gdb.STDOUT)
+        if not isnull(alias):
+            gdb.write('%s    alias: %s@%016x (@ %s)\n' %
+                      ('  ' * level,
+                       alias['name'].string(),
+                       ptr['alias_offset'],
+                       alias,
+                       ),
+                      gdb.STDOUT)
+            self.queue.append(alias)
+        subregion = ptr['subregions']['tqh_first']
+        level += 1
+        while not isnull(subregion):
+            self.print_item(subregion, addr, level)
+            subregion = subregion['subregions_link']['tqe_next']
+
diff --git a/sysconfigs/target/target-x86_64.conf b/scripts/qmp/__init__.py
similarity index 100%
rename from sysconfigs/target/target-x86_64.conf
rename to scripts/qmp/__init__.py
diff --git a/scripts/qmp/qemu-ga-client b/scripts/qmp/qemu-ga-client
index 9908f21..fd05605 100755
--- a/scripts/qmp/qemu-ga-client
+++ b/scripts/qmp/qemu-ga-client
@@ -259,7 +259,7 @@
 
     try:
         client = QemuGuestAgentClient(address)
-    except QemuGuestAgent.error, e:
+    except QemuGuestAgent.error as e:
         import errno
 
         print(e)
diff --git a/scripts/qmp/qmp b/scripts/qmp/qmp
index 1db3c7f..514b539 100755
--- a/scripts/qmp/qmp
+++ b/scripts/qmp/qmp
@@ -91,8 +91,8 @@
         try:
             os.environ['QMP_PATH'] = path
             os.execvp(fullcmd, [fullcmd] + args)
-        except OSError, (errno, msg):
-            if errno == 2:
+        except OSError as exc:
+            if exc.errno == 2:
                 print 'Command "%s" not found.' % (fullcmd)
                 return 1
             raise
diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell
index e0e848b..0373b24 100755
--- a/scripts/qmp/qmp-shell
+++ b/scripts/qmp/qmp-shell
@@ -29,12 +29,47 @@
 # (QEMU) device_add driver=e1000 id=net1
 # {u'return': {}}
 # (QEMU)
+#
+# key=value pairs also support Python or JSON object literal subset notations,
+# without spaces. Dictionaries/objects {} are supported as are arrays [].
+#
+#    example-command arg-name1={'key':'value','obj'={'prop':"value"}}
+#
+# Both JSON and Python formatting should work, including both styles of
+# string literal quotes. Both paradigms of literal values should work,
+# including null/true/false for JSON and None/True/False for Python.
+#
+#
+# Transactions have the following multi-line format:
+#
+#    transaction(
+#    action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ]
+#    ...
+#    action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ]
+#    )
+#
+# One line transactions are also supported:
+#
+#    transaction( action-name1 ... )
+#
+# For example:
+#
+#     (QEMU) transaction(
+#     TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1
+#     TRANS> block-dirty-bitmap-clear node=drive0 name=bitmap0
+#     TRANS> )
+#     {"return": {}}
+#     (QEMU)
+#
+# Use the -v and -p options to activate the verbose and pretty-print options,
+# which will echo back the properly formatted JSON-compliant QMP that is being
+# sent to QEMU, which is useful for debugging and documentation generation.
 
 import qmp
 import json
+import ast
 import readline
 import sys
-import pprint
 
 class QMPCompleter(list):
     def complete(self, text, state):
@@ -51,14 +86,29 @@
 class QMPShellBadPort(QMPShellError):
     pass
 
+class FuzzyJSON(ast.NodeTransformer):
+    '''This extension of ast.NodeTransformer filters literal "true/false/null"
+    values in an AST and replaces them by proper "True/False/None" values that
+    Python can properly evaluate.'''
+    def visit_Name(self, node):
+        if node.id == 'true':
+            node.id = 'True'
+        if node.id == 'false':
+            node.id = 'False'
+        if node.id == 'null':
+            node.id = 'None'
+        return node
+
 # TODO: QMPShell's interface is a bit ugly (eg. _fill_completion() and
 #       _execute_cmd()). Let's design a better one.
 class QMPShell(qmp.QEMUMonitorProtocol):
-    def __init__(self, address, pp=None):
+    def __init__(self, address, pretty=False):
         qmp.QEMUMonitorProtocol.__init__(self, self.__get_address(address))
         self._greeting = None
         self._completer = None
-        self._pp = pp
+        self._pretty = pretty
+        self._transmode = False
+        self._actions = list()
 
     def __get_address(self, arg):
         """
@@ -88,32 +138,40 @@
         # clearing everything as it doesn't seem to matter
         readline.set_completer_delims('')
 
-    def __build_cmd(self, cmdline):
-        """
-        Build a QMP input object from a user provided command-line in the
-        following format:
+    def __parse_value(self, val):
+        try:
+            return int(val)
+        except ValueError:
+            pass
 
-            < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ]
-        """
-        cmdargs = cmdline.split()
-        qmpcmd = { 'execute': cmdargs[0], 'arguments': {} }
-        for arg in cmdargs[1:]:
-            opt = arg.split('=')
+        if val.lower() == 'true':
+            return True
+        if val.lower() == 'false':
+            return False
+        if val.startswith(('{', '[')):
+            # Try first as pure JSON:
             try:
-                if(len(opt) > 2):
-                    opt[1] = '='.join(opt[1:])
-                value = int(opt[1])
+                return json.loads(val)
             except ValueError:
-                if opt[1] == 'true':
-                    value = True
-                elif opt[1] == 'false':
-                    value = False
-                elif opt[1].startswith('{'):
-                    value = json.loads(opt[1])
-                else:
-                    value = opt[1]
-            optpath = opt[0].split('.')
-            parent = qmpcmd['arguments']
+                pass
+            # Try once again as FuzzyJSON:
+            try:
+                st = ast.parse(val, mode='eval')
+                return ast.literal_eval(FuzzyJSON().visit(st))
+            except SyntaxError:
+                pass
+            except ValueError:
+                pass
+        return val
+
+    def __cli_expr(self, tokens, parent):
+        for arg in tokens:
+            (key, _, val) = arg.partition('=')
+            if not val:
+                raise QMPShellError("Expected a key=value pair, got '%s'" % arg)
+
+            value = self.__parse_value(val)
+            optpath = key.split('.')
             curpath = []
             for p in optpath[:-1]:
                 curpath.append(p)
@@ -126,27 +184,76 @@
                 if type(parent[optpath[-1]]) is dict:
                     raise QMPShellError('Cannot use "%s" as both leaf and non-leaf key' % '.'.join(curpath))
                 else:
-                    raise QMPShellError('Cannot set "%s" multiple times' % opt[0])
+                    raise QMPShellError('Cannot set "%s" multiple times' % key)
             parent[optpath[-1]] = value
+
+    def __build_cmd(self, cmdline):
+        """
+        Build a QMP input object from a user provided command-line in the
+        following format:
+
+            < command-name > [ arg-name1=arg1 ] ... [ arg-nameN=argN ]
+        """
+        cmdargs = cmdline.split()
+
+        # Transactional CLI entry/exit:
+        if cmdargs[0] == 'transaction(':
+            self._transmode = True
+            cmdargs.pop(0)
+        elif cmdargs[0] == ')' and self._transmode:
+            self._transmode = False
+            if len(cmdargs) > 1:
+                raise QMPShellError("Unexpected input after close of Transaction sub-shell")
+            qmpcmd = { 'execute': 'transaction',
+                       'arguments': { 'actions': self._actions } }
+            self._actions = list()
+            return qmpcmd
+
+        # Nothing to process?
+        if not cmdargs:
+            return None
+
+        # Parse and then cache this Transactional Action
+        if self._transmode:
+            finalize = False
+            action = { 'type': cmdargs[0], 'data': {} }
+            if cmdargs[-1] == ')':
+                cmdargs.pop(-1)
+                finalize = True
+            self.__cli_expr(cmdargs[1:], action['data'])
+            self._actions.append(action)
+            return self.__build_cmd(')') if finalize else None
+
+        # Standard command: parse and return it to be executed.
+        qmpcmd = { 'execute': cmdargs[0], 'arguments': {} }
+        self.__cli_expr(cmdargs[1:], qmpcmd['arguments'])
         return qmpcmd
 
+    def _print(self, qmp):
+        indent = None
+        if self._pretty:
+            indent = 4
+        jsobj = json.dumps(qmp, indent=indent)
+        print str(jsobj)
+
     def _execute_cmd(self, cmdline):
         try:
             qmpcmd = self.__build_cmd(cmdline)
-        except Exception, e:
+        except Exception as e:
             print 'Error while parsing command line: %s' % e
             print 'command format: <command-name> ',
             print '[arg-name1=arg1] ... [arg-nameN=argN]'
             return True
+        # For transaction mode, we may have just cached the action:
+        if qmpcmd is None:
+            return True
+        if self._verbose:
+            self._print(qmpcmd)
         resp = self.cmd_obj(qmpcmd)
         if resp is None:
             print 'Disconnected'
             return False
-
-        if self._pp is not None:
-            self._pp.pprint(resp)
-        else:
-            print resp
+        self._print(resp)
         return True
 
     def connect(self):
@@ -158,6 +265,11 @@
         version = self._greeting['QMP']['version']['qemu']
         print 'Connected to QEMU %d.%d.%d\n' % (version['major'],version['minor'],version['micro'])
 
+    def get_prompt(self):
+        if self._transmode:
+            return "TRANS> "
+        return "(QEMU) "
+
     def read_exec_command(self, prompt):
         """
         Read and execute a command.
@@ -177,6 +289,9 @@
         else:
             return self._execute_cmd(cmdline)
 
+    def set_verbosity(self, verbose):
+        self._verbose = verbose
+
 class HMPShell(QMPShell):
     def __init__(self, address):
         QMPShell.__init__(self, address)
@@ -254,14 +369,15 @@
 def fail_cmdline(option=None):
     if option:
         sys.stderr.write('ERROR: bad command-line option \'%s\'\n' % option)
-    sys.stderr.write('qemu-shell [ -p ] [ -H ] < UNIX socket path> | < TCP address:port >\n')
+    sys.stderr.write('qemu-shell [ -v ] [ -p ] [ -H ] < UNIX socket path> | < TCP address:port >\n')
     sys.exit(1)
 
 def main():
     addr = ''
     qemu = None
     hmp = False
-    pp = None
+    pretty = False
+    verbose = False
 
     try:
         for arg in sys.argv[1:]:
@@ -270,16 +386,16 @@
                     fail_cmdline(arg)
                 hmp = True
             elif arg == "-p":
-                if pp is not None:
-                    fail_cmdline(arg)
-                pp = pprint.PrettyPrinter(indent=4)
+                pretty = True
+            elif arg == "-v":
+                verbose = True
             else:
                 if qemu is not None:
                     fail_cmdline(arg)
                 if hmp:
                     qemu = HMPShell(arg)
                 else:
-                    qemu = QMPShell(arg, pp)
+                    qemu = QMPShell(arg, pretty)
                 addr = arg
 
         if qemu is None:
@@ -297,7 +413,8 @@
         die('Could not connect to %s' % addr)
 
     qemu.show_banner()
-    while qemu.read_exec_command('(QEMU) '):
+    qemu.set_verbosity(verbose)
+    while qemu.read_exec_command(qemu.get_prompt()):
         pass
     qemu.close()
 
diff --git a/scripts/qmp/qmp.py b/scripts/qmp/qmp.py
index 20b6ec7..62d3651 100644
--- a/scripts/qmp/qmp.py
+++ b/scripts/qmp/qmp.py
@@ -11,6 +11,7 @@
 import json
 import errno
 import socket
+import sys
 
 class QMPError(Exception):
     pass
@@ -21,8 +22,11 @@
 class QMPCapabilitiesError(QMPError):
     pass
 
+class QMPTimeoutError(QMPError):
+    pass
+
 class QEMUMonitorProtocol:
-    def __init__(self, address, server=False):
+    def __init__(self, address, server=False, debug=False):
         """
         Create a QEMUMonitorProtocol class.
 
@@ -36,8 +40,10 @@
         """
         self.__events = []
         self.__address = address
+        self._debug = debug
         self.__sock = self.__get_sock()
         if server:
+            self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
             self.__sock.bind(self.__address)
             self.__sock.listen(1)
 
@@ -65,6 +71,8 @@
                 return
             resp = json.loads(data)
             if 'event' in resp:
+                if self._debug:
+                    print >>sys.stderr, "QMP:<<< %s" % resp
                 self.__events.append(resp)
                 if not only_event:
                     continue
@@ -72,6 +80,44 @@
 
     error = socket.error
 
+    def __get_events(self, wait=False):
+        """
+        Check for new events in the stream and cache them in __events.
+
+        @param wait (bool): block until an event is available.
+        @param wait (float): If wait is a float, treat it as a timeout value.
+
+        @raise QMPTimeoutError: If a timeout float is provided and the timeout
+                                period elapses.
+        @raise QMPConnectError: If wait is True but no events could be retrieved
+                                or if some other error occurred.
+        """
+
+        # Check for new events regardless and pull them into the cache:
+        self.__sock.setblocking(0)
+        try:
+            self.__json_read()
+        except socket.error as err:
+            if err[0] == errno.EAGAIN:
+                # No data available
+                pass
+        self.__sock.setblocking(1)
+
+        # Wait for new events, if needed.
+        # if wait is 0.0, this means "no wait" and is also implicitly false.
+        if not self.__events and wait:
+            if isinstance(wait, float):
+                self.__sock.settimeout(wait)
+            try:
+                ret = self.__json_read(only_event=True)
+            except socket.timeout:
+                raise QMPTimeoutError("Timeout waiting for event")
+            except:
+                raise QMPConnectError("Error while reading from socket")
+            if ret is None:
+                raise QMPConnectError("Error while reading from socket")
+            self.__sock.settimeout(None)
+
     def connect(self, negotiate=True):
         """
         Connect to the QMP Monitor and perform capabilities negotiation.
@@ -95,6 +141,7 @@
         @raise QMPConnectError if the greeting is not received
         @raise QMPCapabilitiesError if fails to negotiate capabilities
         """
+        self.__sock.settimeout(15)
         self.__sock, _ = self.__sock.accept()
         self.__sockfile = self.__sock.makefile()
         return self.__negotiate_capabilities()
@@ -107,13 +154,18 @@
         @return QMP response as a Python dict or None if the connection has
                 been closed
         """
+        if self._debug:
+            print >>sys.stderr, "QMP:>>> %s" % qmp_cmd
         try:
             self.__sock.sendall(json.dumps(qmp_cmd))
-        except socket.error, err:
+        except socket.error as err:
             if err[0] == errno.EPIPE:
                 return
             raise socket.error(err)
-        return self.__json_read()
+        resp = self.__json_read()
+        if self._debug:
+            print >>sys.stderr, "QMP:<<< %s" % resp
+        return resp
 
     def cmd(self, name, args=None, id=None):
         """
@@ -140,43 +192,37 @@
         """
         Get and delete the first available QMP event.
 
-        @param wait: block until an event is available (bool)
+        @param wait (bool): block until an event is available.
+        @param wait (float): If wait is a float, treat it as a timeout value.
+
+        @raise QMPTimeoutError: If a timeout float is provided and the timeout
+                                period elapses.
+        @raise QMPConnectError: If wait is True but no events could be retrieved
+                                or if some other error occurred.
+
+        @return The first available QMP event, or None.
         """
-        self.__sock.setblocking(0)
-        try:
-            self.__json_read()
-        except socket.error, err:
-            if err[0] == errno.EAGAIN:
-                # No data available
-                pass
-        self.__sock.setblocking(1)
-        if not self.__events and wait:
-            self.__json_read(only_event=True)
-        event = self.__events[0]
-        del self.__events[0]
-        return event
+        self.__get_events(wait)
+
+        if self.__events:
+            return self.__events.pop(0)
+        return None
 
     def get_events(self, wait=False):
         """
         Get a list of available QMP events.
 
-        @param wait: block until an event is available (bool)
-        """
-        self.__sock.setblocking(0)
-        try:
-            self.__json_read()
-        except socket.error, err:
-            if err[0] == errno.EAGAIN:
-                # No data available
-                pass
-        self.__sock.setblocking(1)
-        if not self.__events and wait:
-            ret = self.__json_read(only_event=True)
-            if ret == None:
-                # We are in blocking mode, if don't get anything, something
-                # went wrong
-                raise QMPConnectError("Error while reading from socket")
+        @param wait (bool): block until an event is available.
+        @param wait (float): If wait is a float, treat it as a timeout value.
 
+        @raise QMPTimeoutError: If a timeout float is provided and the timeout
+                                period elapses.
+        @raise QMPConnectError: If wait is True but no events could be retrieved
+                                or if some other error occurred.
+
+        @return The list of available QMP events.
+        """
+        self.__get_events(wait)
         return self.__events
 
     def clear_events(self):
diff --git a/scripts/qmp/qom-tree b/scripts/qmp/qom-tree
new file mode 100755
index 0000000..906fcd2
--- /dev/null
+++ b/scripts/qmp/qom-tree
@@ -0,0 +1,75 @@
+#!/usr/bin/python
+##
+# QEMU Object Model test tools
+#
+# Copyright IBM, Corp. 2011
+# Copyright (c) 2013 SUSE LINUX Products GmbH
+#
+# Authors:
+#  Anthony Liguori   <aliguori@amazon.com>
+#  Andreas Faerber   <afaerber@suse.de>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or later.  See
+# the COPYING file in the top-level directory.
+##
+
+import sys
+import os
+from qmp import QEMUMonitorProtocol
+
+cmd, args = sys.argv[0], sys.argv[1:]
+socket_path = None
+path = None
+prop = None
+
+def usage():
+    return '''environment variables:
+    QMP_SOCKET=<path | addr:port>
+usage:
+    %s [-h] [-s <QMP socket path | addr:port>] [<path>]
+''' % cmd
+
+def usage_error(error_msg = "unspecified error"):
+    sys.stderr.write('%s\nERROR: %s\n' % (usage(), error_msg))
+    exit(1)
+
+if len(args) > 0:
+    if args[0] == "-h":
+        print usage()
+        exit(0);
+    elif args[0] == "-s":
+        try:
+            socket_path = args[1]
+        except:
+            usage_error("missing argument: QMP socket path or address");
+        args = args[2:]
+
+if not socket_path:
+    if os.environ.has_key('QMP_SOCKET'):
+        socket_path = os.environ['QMP_SOCKET']
+    else:
+        usage_error("no QMP socket path or address given");
+
+srv = QEMUMonitorProtocol(socket_path)
+srv.connect()
+
+def list_node(path):
+    print '%s' % path
+    items = srv.command('qom-list', path=path)
+    for item in items:
+        if not item['type'].startswith('child<'):
+            try:
+                print '  %s: %s (%s)' % (item['name'], srv.command('qom-get', path=path, property=item['name']), item['type'])
+            except:
+                print '  %s: <EXCEPTION> (%s)' % (item['name'], item['type'])
+    print ''
+    for item in items:
+        if item['type'].startswith('child<'):
+            list_node((path if (path != '/') else '')  + '/' + item['name'])
+
+if len(args) == 0:
+    path = '/'
+else:
+    path = args[0]
+
+list_node(path)
diff --git a/scripts/qtest.py b/scripts/qtest.py
new file mode 100644
index 0000000..d5aecb5
--- /dev/null
+++ b/scripts/qtest.py
@@ -0,0 +1,110 @@
+# QEMU qtest library
+#
+# Copyright (C) 2015 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Based on qmp.py.
+#
+
+import errno
+import socket
+import string
+import os
+import subprocess
+import qmp.qmp
+import qemu
+
+class QEMUQtestProtocol(object):
+    def __init__(self, address, server=False):
+        """
+        Create a QEMUQtestProtocol object.
+
+        @param address: QEMU address, can be either a unix socket path (string)
+                        or a tuple in the form ( address, port ) for a TCP
+                        connection
+        @param server: server mode, listens on the socket (bool)
+        @raise socket.error on socket connection errors
+        @note No connection is established, this is done by the connect() or
+              accept() methods
+        """
+        self._address = address
+        self._sock = self._get_sock()
+        if server:
+            self._sock.bind(self._address)
+            self._sock.listen(1)
+
+    def _get_sock(self):
+        if isinstance(self._address, tuple):
+            family = socket.AF_INET
+        else:
+            family = socket.AF_UNIX
+        return socket.socket(family, socket.SOCK_STREAM)
+
+    def connect(self):
+        """
+        Connect to the qtest socket.
+
+        @raise socket.error on socket connection errors
+        """
+        self._sock.connect(self._address)
+
+    def accept(self):
+        """
+        Await connection from QEMU.
+
+        @raise socket.error on socket connection errors
+        """
+        self._sock, _ = self._sock.accept()
+
+    def cmd(self, qtest_cmd):
+        """
+        Send a qtest command on the wire.
+
+        @param qtest_cmd: qtest command text to be sent
+        """
+        self._sock.sendall(qtest_cmd + "\n")
+
+    def close(self):
+        self._sock.close()
+
+    def settimeout(self, timeout):
+        self._sock.settimeout(timeout)
+
+
+class QEMUQtestMachine(qemu.QEMUMachine):
+    '''A QEMU VM'''
+
+    def __init__(self, binary, args=[], name=None, test_dir="/var/tmp",
+                 socket_scm_helper=None):
+        if name is None:
+            name = "qemu-%d" % os.getpid()
+        super(QEMUQtestMachine, self).__init__(binary, args, name=name, test_dir=test_dir,
+                                               socket_scm_helper=socket_scm_helper)
+        self._qtest_path = os.path.join(test_dir, name + "-qtest.sock")
+
+    def _base_args(self):
+        args = super(QEMUQtestMachine, self)._base_args()
+        args.extend(['-qtest', 'unix:path=' + self._qtest_path,
+                     '-machine', 'accel=qtest'])
+        return args
+
+    def _pre_launch(self):
+        super(QEMUQtestMachine, self)._pre_launch()
+        self._qtest = QEMUQtestProtocol(self._qtest_path, server=True)
+
+    def _post_launch(self):
+        super(QEMUQtestMachine, self)._post_launch()
+        self._qtest.accept()
+
+    def _post_shutdown(self):
+        super(QEMUQtestMachine, self)._post_shutdown()
+        self._remove_if_exists(self._qtest_path)
+
+    def qtest(self, cmd):
+        '''Send a qtest command to guest'''
+        return self._qtest.cmd(cmd)
diff --git a/scripts/shaderinclude.pl b/scripts/shaderinclude.pl
new file mode 100644
index 0000000..81b5146
--- /dev/null
+++ b/scripts/shaderinclude.pl
@@ -0,0 +1,16 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+
+my $file = shift;
+open FILE, "<", $file or die "open $file: $!";
+my $name = $file;
+$name =~ s|.*/||;
+$name =~ s/[-.]/_/g;
+print "static GLchar ${name}_src[] =\n";
+while (<FILE>) {
+    chomp;
+    printf "    \"%s\\n\"\n", $_;
+}
+print "    \"\\n\";\n";
+close FILE;
diff --git a/scripts/signrom.py b/scripts/signrom.py
index f9c35cc..d1dabe0 100644
--- a/scripts/signrom.py
+++ b/scripts/signrom.py
@@ -17,11 +17,28 @@
 fin = open(sys.argv[1], 'rb')
 fout = open(sys.argv[2], 'wb')
 
-fin.seek(2)
-size = ord(fin.read(1)) * 512 - 1
+magic = fin.read(2)
+if magic != '\x55\xaa':
+    sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1])
 
+size_byte = ord(fin.read(1))
 fin.seek(0)
-data = fin.read(size)
+data = fin.read()
+
+size = size_byte * 512
+if len(data) > size:
+    sys.stderr.write('error: ROM is too large (%d > %d)\n' % (len(data), size))
+    sys.exit(1)
+elif len(data) < size:
+    # Add padding if necessary, rounding the whole input to a multiple of
+    # 512 bytes according to the third byte of the input.
+    # size-1 because a final byte is added below to store the checksum.
+    data = data.ljust(size-1, '\0')
+else:
+    if ord(data[-1:]) != 0:
+        sys.stderr.write('WARNING: ROM includes nonzero checksum\n')
+    data = data[:size-1]
+
 fout.write(data)
 
 checksum = 0
diff --git a/scripts/texi2pod.pl b/scripts/texi2pod.pl
index 94097fb..8767662 100755
--- a/scripts/texi2pod.pl
+++ b/scripts/texi2pod.pl
@@ -317,7 +317,7 @@
 	@columns = ();
 	for $column (split (/\s*\@tab\s*/, $1)) {
 	    # @strong{...} is used a @headitem work-alike
-	    $column =~ s/^\@strong{(.*)}$/$1/;
+	    $column =~ s/^\@strong\{(.*)\}$/$1/;
 	    push @columns, $column;
 	}
 	$_ = "\n=item ".join (" : ", @columns)."\n";
diff --git a/scripts/tracetool.py b/scripts/tracetool.py
index 83bde7b..7b82959 100755
--- a/scripts/tracetool.py
+++ b/scripts/tracetool.py
@@ -71,7 +71,7 @@
 
     try:
         opts, args = getopt.getopt(args[1:], "", long_opts)
-    except getopt.GetoptError, err:
+    except getopt.GetoptError as err:
         error_opt(str(err))
 
     check_backends = False
@@ -132,7 +132,7 @@
     try:
         tracetool.generate(sys.stdin, arg_format, arg_backends,
                            binary=binary, probe_prefix=probe_prefix)
-    except tracetool.TracetoolError, e:
+    except tracetool.TracetoolError as e:
         error_opt(str(e))
 
 if __name__ == "__main__":
diff --git a/scripts/tracetool/__init__.py b/scripts/tracetool/__init__.py
index 181675f..be24039 100644
--- a/scripts/tracetool/__init__.py
+++ b/scripts/tracetool/__init__.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -50,9 +50,14 @@
         Parameters
         ----------
         args :
-            List of (type, name) tuples.
+            List of (type, name) tuples or Arguments objects.
         """
-        self._args = args
+        self._args = []
+        for arg in args:
+            if isinstance(arg, Arguments):
+                self._args.extend(arg._args)
+            else:
+                self._args.append(arg)
 
     def copy(self):
         """Create a new copy."""
@@ -83,6 +88,12 @@
             res.append((arg_type, identifier))
         return Arguments(res)
 
+    def __getitem__(self, index):
+        if isinstance(index, slice):
+            return Arguments(self._args[index])
+        else:
+            return self._args[index]
+
     def __iter__(self):
         """Iterate over the (type, name) pairs."""
         return iter(self._args)
@@ -110,6 +121,10 @@
         """List of argument types."""
         return [ type_ for type_, _ in self._args ]
 
+    def casted(self):
+        """List of argument names casted to their type."""
+        return ["(%s)%s" % (type_, name) for type_, name in self._args]
+
     def transform(self, *trans):
         """Return a new Arguments instance with transformed types.
 
@@ -146,9 +161,10 @@
                       "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?"
                       "\s*")
 
-    _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec"])
+    _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"])
 
-    def __init__(self, name, props, fmt, args, orig=None):
+    def __init__(self, name, props, fmt, args, orig=None,
+                 event_trans=None, event_exec=None):
         """
         Parameters
         ----------
@@ -161,13 +177,19 @@
         args : Arguments
             Event arguments.
         orig : Event or None
-            Original Event before transformation.
+            Original Event before transformation/generation.
+        event_trans : Event or None
+            Generated translation-time event ("tcg" property).
+        event_exec : Event or None
+            Generated execution-time event ("tcg" property).
 
         """
         self.name = name
         self.properties = props
         self.fmt = fmt
         self.args = args
+        self.event_trans = event_trans
+        self.event_exec = event_exec
 
         if orig is None:
             self.original = weakref.ref(self)
@@ -183,7 +205,7 @@
     def copy(self):
         """Create a new copy."""
         return Event(self.name, list(self.properties), self.fmt,
-                     self.args.copy(), self)
+                     self.args.copy(), self, self.event_trans, self.event_exec)
 
     @staticmethod
     def build(line_str):
@@ -215,7 +237,13 @@
         if "tcg" in props and isinstance(fmt, str):
             raise ValueError("Events with 'tcg' property must have two formats")
 
-        return Event(name, props, fmt, args)
+        event = Event(name, props, fmt, args)
+
+        # add implicit arguments when using the 'vcpu' property
+        import tracetool.vcpu
+        event = tracetool.vcpu.transform_event(event)
+
+        return event
 
     def __repr__(self):
         """Evaluable string representation for this object."""
@@ -270,6 +298,7 @@
             event_trans.name += "_trans"
             event_trans.properties += ["tcg-trans"]
             event_trans.fmt = event.fmt[0]
+            # ignore TCG arguments
             args_trans = []
             for atrans, aorig in zip(
                     event_trans.transform(tracetool.transform.TCG_2_HOST).args,
@@ -277,13 +306,12 @@
                 if atrans == aorig:
                     args_trans.append(atrans)
             event_trans.args = Arguments(args_trans)
-            event_trans = event_trans.copy()
 
             event_exec = event.copy()
             event_exec.name += "_exec"
             event_exec.properties += ["tcg-exec"]
             event_exec.fmt = event.fmt[1]
-            event_exec = event_exec.transform(tracetool.transform.TCG_2_HOST)
+            event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST)
 
             new_event = [event_trans, event_exec]
             event.event_trans, event.event_exec = new_event
diff --git a/scripts/tracetool/backend/dtrace.py b/scripts/tracetool/backend/dtrace.py
index fabfe99..ab9ecfa 100644
--- a/scripts/tracetool/backend/dtrace.py
+++ b/scripts/tracetool/backend/dtrace.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -41,6 +41,6 @@
 
 
 def generate_h(event):
-    out('    QEMU_%(uppername)s(%(argnames)s);',
+    out('        QEMU_%(uppername)s(%(argnames)s);',
         uppername=event.name.upper(),
         argnames=", ".join(event.args.names()))
diff --git a/scripts/tracetool/backend/ftrace.py b/scripts/tracetool/backend/ftrace.py
index d798c71..80dcf30 100644
--- a/scripts/tracetool/backend/ftrace.py
+++ b/scripts/tracetool/backend/ftrace.py
@@ -30,17 +30,17 @@
     if len(event.args) > 0:
         argnames = ", " + argnames
 
-    out('    {',
-        '        char ftrace_buf[MAX_TRACE_STRLEN];',
-        '        int unused __attribute__ ((unused));',
-        '        int trlen;',
-        '        if (trace_event_get_state(%(event_id)s)) {',
-        '            trlen = snprintf(ftrace_buf, MAX_TRACE_STRLEN,',
-        '                             "%(name)s " %(fmt)s "\\n" %(argnames)s);',
-        '            trlen = MIN(trlen, MAX_TRACE_STRLEN - 1);',
-        '            unused = write(trace_marker_fd, ftrace_buf, trlen);',
+    out('        {',
+        '            char ftrace_buf[MAX_TRACE_STRLEN];',
+        '            int unused __attribute__ ((unused));',
+        '            int trlen;',
+        '            if (trace_event_get_state(%(event_id)s)) {',
+        '                trlen = snprintf(ftrace_buf, MAX_TRACE_STRLEN,',
+        '                                 "%(name)s " %(fmt)s "\\n" %(argnames)s);',
+        '                trlen = MIN(trlen, MAX_TRACE_STRLEN - 1);',
+        '                unused = write(trace_marker_fd, ftrace_buf, trlen);',
+        '            }',
         '        }',
-        '    }',
         name=event.name,
         args=event.args,
         event_id="TRACE_" + event.name.upper(),
diff --git a/scripts/tracetool/backend/log.py b/scripts/tracetool/backend/log.py
new file mode 100644
index 0000000..b3ff064
--- /dev/null
+++ b/scripts/tracetool/backend/log.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Stderr built-in backend.
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import out
+
+
+PUBLIC = True
+
+
+def generate_h_begin(events):
+    out('#include "trace/control.h"',
+        '#include "qemu/log.h"',
+        '')
+
+
+def generate_h(event):
+    argnames = ", ".join(event.args.names())
+    if len(event.args) > 0:
+        argnames = ", " + argnames
+
+    if "vcpu" in event.properties:
+        # already checked on the generic format code
+        cond = "true"
+    else:
+        cond = "trace_event_get_state(%s)" % ("TRACE_" + event.name.upper())
+
+    out('        if (%(cond)s) {',
+        '            struct timeval _now;',
+        '            gettimeofday(&_now, NULL);',
+        '            qemu_log_mask(LOG_TRACE, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",',
+        '                          getpid(),',
+        '                          (size_t)_now.tv_sec, (size_t)_now.tv_usec',
+        '                          %(argnames)s);',
+        '        }',
+        cond=cond,
+        name=event.name,
+        fmt=event.fmt.rstrip("\n"),
+        argnames=argnames)
diff --git a/scripts/tracetool/backend/simple.py b/scripts/tracetool/backend/simple.py
index e8c2cd5..1bccada 100644
--- a/scripts/tracetool/backend/simple.py
+++ b/scripts/tracetool/backend/simple.py
@@ -36,13 +36,14 @@
 
 
 def generate_h(event):
-    out('    _simple_%(api)s(%(args)s);',
+    out('        _simple_%(api)s(%(args)s);',
         api=event.api(),
         args=", ".join(event.args.names()))
 
 
 def generate_c_begin(events):
-    out('#include "trace.h"',
+    out('#include "qemu/osdep.h"',
+        '#include "trace.h"',
         '#include "trace/control.h"',
         '#include "trace/simple.h"',
         '')
@@ -67,16 +68,23 @@
     if len(event.args) == 0:
         sizestr = '0'
 
+    event_id = 'TRACE_' + event.name.upper()
+    if "vcpu" in event.properties:
+        # already checked on the generic format code
+        cond = "true"
+    else:
+        cond = "trace_event_get_state(%s)" % event_id
 
     out('',
-        '    if (!trace_event_get_state(%(event_id)s)) {',
+        '    if (!%(cond)s) {',
         '        return;',
         '    }',
         '',
         '    if (trace_record_start(&rec, %(event_id)s, %(size_str)s)) {',
         '        return; /* Trace Buffer Full, Event Dropped ! */',
         '    }',
-        event_id='TRACE_' + event.name.upper(),
+        cond=cond,
+        event_id=event_id,
         size_str=sizestr)
 
     if len(event.args) > 0:
diff --git a/scripts/tracetool/backend/stderr.py b/scripts/tracetool/backend/stderr.py
deleted file mode 100644
index 2a1e906..0000000
--- a/scripts/tracetool/backend/stderr.py
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""
-Stderr built-in backend.
-"""
-
-__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
-__license__    = "GPL version 2 or (at your option) any later version"
-
-__maintainer__ = "Stefan Hajnoczi"
-__email__      = "stefanha@linux.vnet.ibm.com"
-
-
-from tracetool import out
-
-
-PUBLIC = True
-
-
-def generate_h_begin(events):
-    out('#include <stdio.h>',
-        '#include "trace/control.h"',
-        '')
-
-
-def generate_h(event):
-    argnames = ", ".join(event.args.names())
-    if len(event.args) > 0:
-        argnames = ", " + argnames
-
-    out('    if (trace_event_get_state(%(event_id)s)) {',
-        '        fprintf(stderr, "%(name)s " %(fmt)s "\\n" %(argnames)s);',
-        '    }',
-        event_id="TRACE_" + event.name.upper(),
-        name=event.name,
-        fmt=event.fmt.rstrip("\n"),
-        argnames=argnames)
diff --git a/scripts/tracetool/backend/ust.py b/scripts/tracetool/backend/ust.py
index 2f8f44a..ed4c227 100644
--- a/scripts/tracetool/backend/ust.py
+++ b/scripts/tracetool/backend/ust.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -30,6 +30,6 @@
     if len(event.args) > 0:
         argnames = ", " + argnames
 
-    out('    tracepoint(qemu, %(name)s%(tp_args)s);',
+    out('        tracepoint(qemu, %(name)s%(tp_args)s);',
         name=event.name,
         tp_args=argnames)
diff --git a/scripts/tracetool/format/d.py b/scripts/tracetool/format/d.py
index 46eebb1..c77d5b7 100644
--- a/scripts/tracetool/format/d.py
+++ b/scripts/tracetool/format/d.py
@@ -16,6 +16,19 @@
 from tracetool import out
 
 
+# Reserved keywords from
+# https://wikis.oracle.com/display/DTrace/Types,+Operators+and+Expressions
+RESERVED_WORDS = (
+    'auto', 'goto', 'sizeof', 'break', 'if', 'static', 'case', 'import',
+    'string', 'char', 'inline', 'stringof', 'const', 'int', 'struct',
+    'continue', 'long', 'switch', 'counter', 'offsetof', 'this',
+    'default', 'probe', 'translator', 'do', 'provider', 'typedef',
+    'double', 'register', 'union', 'else', 'restrict', 'unsigned',
+    'enum', 'return', 'void', 'extern', 'self', 'volatile', 'float',
+    'short', 'while', 'for', 'signed', 'xlate',
+)
+
+
 def generate(events, backend):
     events = [e for e in events
               if "disable" not in e.properties]
@@ -25,18 +38,17 @@
         'provider qemu {')
 
     for e in events:
-        args = str(e.args)
-
-        # DTrace provider syntax expects foo() for empty
-        # params, not foo(void)
-        if args == 'void':
-            args = ''
+        args = []
+        for type_, name in e.args:
+            if name in RESERVED_WORDS:
+                name += '_'
+            args.append(type_ + ' ' + name)
 
         # Define prototype for probe arguments
         out('',
             'probe %(name)s(%(args)s);',
             name=e.name,
-            args=args)
+            args=','.join(args))
 
     out('',
         '};')
diff --git a/scripts/tracetool/format/events_c.py b/scripts/tracetool/format/events_c.py
index 2d97fa3..4012063 100644
--- a/scripts/tracetool/format/events_c.py
+++ b/scripts/tracetool/format/events_c.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -19,6 +19,7 @@
 def generate(events, backend):
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
+        '#include "qemu/osdep.h"',
         '#include "trace.h"',
         '#include "trace/generated-events.h"',
         '#include "trace/control.h"',
@@ -27,8 +28,15 @@
     out('TraceEvent trace_events[TRACE_EVENT_COUNT] = {')
 
     for e in events:
-        out('    { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s, .dstate = 0 },',
+        if "vcpu" in e.properties:
+            vcpu_id = "TRACE_VCPU_" + e.name.upper()
+        else:
+            vcpu_id = "TRACE_VCPU_EVENT_COUNT"
+        out('    { .id = %(id)s, .vcpu_id = %(vcpu_id)s,'
+            ' .name = \"%(name)s\",'
+            ' .sstate = %(sstate)s },',
             id = "TRACE_" + e.name.upper(),
+            vcpu_id = vcpu_id,
             name = e.name,
             sstate = "TRACE_%s_ENABLED" % e.name.upper())
 
diff --git a/scripts/tracetool/format/events_h.py b/scripts/tracetool/format/events_h.py
index 9f114a3..a9da60b 100644
--- a/scripts/tracetool/format/events_h.py
+++ b/scripts/tracetool/format/events_h.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -21,8 +21,6 @@
         '',
         '#ifndef TRACE__GENERATED_EVENTS_H',
         '#define TRACE__GENERATED_EVENTS_H',
-        '',
-        '#include <stdbool.h>',
         '')
 
     # event identifiers
@@ -34,16 +32,26 @@
     out('    TRACE_EVENT_COUNT',
         '} TraceEventID;')
 
+    # per-vCPU event identifiers
+    out('typedef enum {')
+
+    for e in events:
+        if "vcpu" in e.properties:
+            out('    TRACE_VCPU_%s,' % e.name.upper())
+
+    out('    TRACE_VCPU_EVENT_COUNT',
+        '} TraceEventVCPUID;')
+
     # static state
     for e in events:
         if 'disable' in e.properties:
             enabled = 0
         else:
             enabled = 1
-        if "tcg-trans" in e.properties:
+        if "tcg-exec" in e.properties:
             # a single define for the two "sub-events"
             out('#define TRACE_%(name)s_ENABLED %(enabled)d',
-                name=e.original.original.name.upper(),
+                name=e.original.name.upper(),
                 enabled=enabled)
         out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled))
 
diff --git a/scripts/tracetool/format/h.py b/scripts/tracetool/format/h.py
index 9b39430..3763e9a 100644
--- a/scripts/tracetool/format/h.py
+++ b/scripts/tracetool/format/h.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -23,21 +23,36 @@
         '#define TRACE__GENERATED_TRACERS_H',
         '',
         '#include "qemu-common.h"',
+        '#include "trace/control.h"',
         '')
 
     backend.generate_begin(events)
 
     for e in events:
+        if "vcpu" in e.properties:
+            trace_cpu = next(iter(e.args))[1]
+            cond = "trace_event_get_vcpu_state(%(cpu)s,"\
+                   " TRACE_%(id)s,"\
+                   " TRACE_VCPU_%(id)s)"\
+                   % dict(
+                       cpu=trace_cpu,
+                       id=e.name.upper())
+        else:
+            cond = "true"
+
         out('',
             'static inline void %(api)s(%(args)s)',
             '{',
+            '    if (%(cond)s) {',
             api=e.api(),
-            args=e.args)
+            args=e.args,
+            cond=cond)
 
         if "disable" not in e.properties:
             backend.generate(e)
 
-        out('}')
+        out('    }',
+            '}')
 
     backend.generate_end(events)
 
diff --git a/scripts/tracetool/format/tcg_h.py b/scripts/tracetool/format/tcg_h.py
index f676b66..e2331f2 100644
--- a/scripts/tracetool/format/tcg_h.py
+++ b/scripts/tracetool/format/tcg_h.py
@@ -6,14 +6,25 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
 __email__      = "stefanha@linux.vnet.ibm.com"
 
 
-from tracetool import out
+from tracetool import out, Arguments
+import tracetool.vcpu
+
+
+def vcpu_transform_args(args):
+    assert len(args) == 1
+    return Arguments([
+        args,
+        # NOTE: this name must be kept in sync with the one in "tcg_h"
+        # NOTE: Current helper code uses TCGv_env (CPUArchState*)
+        ("TCGv_env", "__tcg_" + args.names()[0]),
+    ])
 
 
 def generate(events, backend):
@@ -23,8 +34,6 @@
         '#ifndef TRACE__GENERATED_TCG_TRACERS_H',
         '#define TRACE__GENERATED_TCG_TRACERS_H',
         '',
-        '#include <stdint.h>',
-        '',
         '#include "trace.h"',
         '#include "exec/helper-proto.h"',
         '',
@@ -35,21 +44,21 @@
         if "tcg-trans" not in e.properties:
             continue
 
-        # get the original event definition
-        e = e.original.original
-
         out('static inline void %(name_tcg)s(%(args)s)',
             '{',
-            name_tcg=e.api(e.QEMU_TRACE_TCG),
-            args=e.args)
+            name_tcg=e.original.api(e.QEMU_TRACE_TCG),
+            args=tracetool.vcpu.transform_args("tcg_h", e.original))
 
         if "disable" not in e.properties:
+            args_trans = e.original.event_trans.args
+            args_exec = tracetool.vcpu.transform_args(
+                "tcg_helper_c", e.original.event_exec, "wrapper")
             out('    %(name_trans)s(%(argnames_trans)s);',
                 '    gen_helper_%(name_exec)s(%(argnames_exec)s);',
-                name_trans=e.event_trans.api(e.QEMU_TRACE),
-                name_exec=e.event_exec.api(e.QEMU_TRACE),
-                argnames_trans=", ".join(e.event_trans.args.names()),
-                argnames_exec=", ".join(e.event_exec.args.names()))
+                name_trans=e.original.event_trans.api(e.QEMU_TRACE),
+                name_exec=e.original.event_exec.api(e.QEMU_TRACE),
+                argnames_trans=", ".join(args_trans.names()),
+                argnames_exec=", ".join(args_exec.names()))
 
         out('}')
 
diff --git a/scripts/tracetool/format/tcg_helper_c.py b/scripts/tracetool/format/tcg_helper_c.py
index 96655a0..e3485b7 100644
--- a/scripts/tracetool/format/tcg_helper_c.py
+++ b/scripts/tracetool/format/tcg_helper_c.py
@@ -6,15 +6,38 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
 __email__      = "stefanha@linux.vnet.ibm.com"
 
 
-from tracetool import out
+from tracetool import Arguments, out
 from tracetool.transform import *
+import tracetool.vcpu
+
+
+def vcpu_transform_args(args, mode):
+    assert len(args) == 1
+    # NOTE: this name must be kept in sync with the one in "tcg_h"
+    args = Arguments([(args.types()[0], "__tcg_" + args.names()[0])])
+    if mode == "code":
+        return Arguments([
+            # Does cast from helper requirements to tracing types
+            ("CPUState *", "ENV_GET_CPU(%s)" % args.names()[0]),
+        ])
+    else:
+        args = Arguments([
+            # NOTE: Current helper code uses TCGv_env (CPUArchState*)
+            ("CPUArchState *", args.names()[0]),
+        ])
+        if mode == "header":
+            return args
+        elif mode == "wrapper":
+            return args.transform(HOST_2_TCG)
+        else:
+            assert False
 
 
 def generate(events, backend):
@@ -23,7 +46,9 @@
 
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
+        '#include "qemu/osdep.h"',
         '#include "qemu-common.h"',
+        '#include "cpu.h"',
         '#include "trace.h"',
         '#include "exec/helper-proto.h"',
         '',
@@ -33,18 +58,18 @@
         if "tcg-exec" not in e.properties:
             continue
 
-        # tracetool.generate always transforms types to host
-        e_args = e.original.args
+        e_args_api = tracetool.vcpu.transform_args(
+            "tcg_helper_c", e.original, "header").transform(
+                HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF)
+        e_args_call = tracetool.vcpu.transform_args(
+            "tcg_helper_c", e, "code")
 
-        values = ["(%s)%s" % (t, n)
-                  for t, n in e.args.transform(TCG_2_TCG_HELPER_DEF)]
-
-        out('void %(name_tcg)s(%(args)s)',
+        out('void %(name_tcg)s(%(args_api)s)',
             '{',
-            '    %(name)s(%(values)s);',
+            '    %(name)s(%(args_call)s);',
             '}',
             name_tcg="helper_%s_proxy" % e.api(),
             name=e.api(),
-            args=e_args.transform(HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF),
-            values=", ".join(values),
+            args_api=e_args_api,
+            args_call=", ".join(e_args_call.casted()),
             )
diff --git a/scripts/tracetool/format/tcg_helper_h.py b/scripts/tracetool/format/tcg_helper_h.py
index a8ba7ba..dc76c15 100644
--- a/scripts/tracetool/format/tcg_helper_h.py
+++ b/scripts/tracetool/format/tcg_helper_h.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -15,6 +15,7 @@
 
 from tracetool import out
 from tracetool.transform import *
+import tracetool.vcpu
 
 
 def generate(events, backend):
@@ -29,11 +30,9 @@
         if "tcg-exec" not in e.properties:
             continue
 
-        # tracetool.generate always transforms types to host
-        e_args = e.original.args
-
         # TCG helper proxy declaration
         fmt = "DEF_HELPER_FLAGS_%(argc)d(%(name)s, %(flags)svoid%(types)s)"
+        e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "header")
         args = e_args.transform(HOST_2_TCG_COMPAT, HOST_2_TCG,
                                 TCG_2_TCG_HELPER_DECL)
         types = ", ".join(args.types())
diff --git a/scripts/tracetool/format/tcg_helper_wrapper_h.py b/scripts/tracetool/format/tcg_helper_wrapper_h.py
index cac5a87..020f442 100644
--- a/scripts/tracetool/format/tcg_helper_wrapper_h.py
+++ b/scripts/tracetool/format/tcg_helper_wrapper_h.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -15,6 +15,7 @@
 
 from tracetool import out
 from tracetool.transform import *
+import tracetool.vcpu
 
 
 def generate(events, backend):
@@ -33,7 +34,7 @@
             continue
 
         # tracetool.generate always transforms types to host
-        e_args = e.original.args
+        e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "wrapper")
 
         # mixed-type to TCG helper bridge
         args_tcg_compat = e_args.transform(HOST_2_TCG_COMPAT)
diff --git a/scripts/tracetool/format/ust_events_c.py b/scripts/tracetool/format/ust_events_c.py
index bc97093..9967c7a 100644
--- a/scripts/tracetool/format/ust_events_c.py
+++ b/scripts/tracetool/format/ust_events_c.py
@@ -22,6 +22,8 @@
 
     out('/* This file is autogenerated by tracetool, do not edit. */',
         '',
+        '#include "qemu/osdep.h"',
+        '',
         '#define TRACEPOINT_DEFINE',
         '#define TRACEPOINT_CREATE_PROBES',
         '',
diff --git a/scripts/tracetool/transform.py b/scripts/tracetool/transform.py
index fc5e679..e18b053 100644
--- a/scripts/tracetool/transform.py
+++ b/scripts/tracetool/transform.py
@@ -6,7 +6,7 @@
 """
 
 __author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
-__copyright__  = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>"
 __license__    = "GPL version 2 or (at your option) any later version"
 
 __maintainer__ = "Stefan Hajnoczi"
@@ -98,6 +98,7 @@
     "uint32_t": "TCGv_i32",
     "uint64_t": "TCGv_i64",
     "void *"  : "TCGv_ptr",
+    "CPUArchState *": "TCGv_env",
     None: _host_2_tcg,
     }
 
@@ -130,6 +131,7 @@
     "TCGv_ptr": "ptr",
     "TCGv_i32": "i32",
     "TCGv_i64": "i64",
+    "TCGv_env": "env",
     None: _tcg_2_tcg_helper_decl_error,
     }
 
diff --git a/scripts/tracetool/vcpu.py b/scripts/tracetool/vcpu.py
new file mode 100644
index 0000000..452c7f5
--- /dev/null
+++ b/scripts/tracetool/vcpu.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+Generic management for the 'vcpu' property.
+
+"""
+
+__author__     = "Lluís Vilanova <vilanova@ac.upc.edu>"
+__copyright__  = "Copyright 2016, Lluís Vilanova <vilanova@ac.upc.edu>"
+__license__    = "GPL version 2 or (at your option) any later version"
+
+__maintainer__ = "Stefan Hajnoczi"
+__email__      = "stefanha@linux.vnet.ibm.com"
+
+
+from tracetool import Arguments, try_import
+
+
+def transform_event(event):
+    """Transform event to comply with the 'vcpu' property (if present)."""
+    if "vcpu" in event.properties:
+        # events with 'tcg-trans' and 'tcg-exec' are auto-generated from
+        # already-patched events
+        assert "tcg-trans" not in event.properties
+        assert "tcg-exec" not in event.properties
+
+        event.args = Arguments([("CPUState *", "__cpu"), event.args])
+        if "tcg" in event.properties:
+            fmt = "\"cpu=%p \""
+            event.fmt = [fmt + event.fmt[0],
+                         fmt + event.fmt[1]]
+        else:
+            fmt = "\"cpu=%p \""
+            event.fmt = fmt + event.fmt
+    return event
+
+
+def transform_args(format, event, *args, **kwargs):
+    """Transforms the arguments to suit the specified format.
+
+    The format module must implement function 'vcpu_args', which receives the
+    implicit arguments added by the 'vcpu' property, and must return suitable
+    arguments for the given format.
+
+    The function is only called for events with the 'vcpu' property.
+
+    Parameters
+    ==========
+    format : str
+        Format module name.
+    event : Event
+    args, kwargs
+        Passed to 'vcpu_transform_args'.
+
+    Returns
+    =======
+    Arguments
+        The transformed arguments, including the non-implicit ones.
+
+    """
+    if "vcpu" in event.properties:
+        ok, func = try_import("tracetool.format." + format,
+                              "vcpu_transform_args")
+        assert ok
+        assert func
+        return Arguments([func(event.args[:1], *args, **kwargs),
+                          event.args[1:]])
+    else:
+        return event.args
diff --git a/scripts/update-acpi.sh b/scripts/update-acpi.sh
deleted file mode 100644
index b5f05ff..0000000
--- a/scripts/update-acpi.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-cd x86_64-softmmu
-for file in hw/i386/*.hex; do
-    cp -f $file ../$file.generated
-done
diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh
index c8e026d..08c4c4a 100755
--- a/scripts/update-linux-headers.sh
+++ b/scripts/update-linux-headers.sh
@@ -10,7 +10,7 @@
 # This work is licensed under the terms of the GNU GPL version 2.
 # See the COPYING file in the top-level directory.
 
-tmpdir=`mktemp -d`
+tmpdir=$(mktemp -d)
 linux="$1"
 output="$2"
 
@@ -28,6 +28,37 @@
     output="$PWD"
 fi
 
+cp_portable() {
+    f=$1
+    to=$2
+    if
+        grep '#include' "$f" | grep -v -e 'linux/virtio' \
+                                     -e 'linux/types' \
+                                     -e 'stdint' \
+                                     -e 'linux/if_ether' \
+                                     -e 'input-event-codes' \
+                                     -e 'sys/' \
+                                     > /dev/null
+    then
+        echo "Unexpected #include in input file $f".
+        exit 2
+    fi
+
+    header=$(basename "$f");
+    sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \
+        -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \
+        -e 's/"\(input-event-codes\.h\)"/"standard-headers\/linux\/\1"/' \
+        -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \
+        -e 's/__bitwise__//' \
+        -e 's/__attribute__((packed))/QEMU_PACKED/' \
+        -e 's/__inline__/inline/' \
+        -e '/sys\/ioctl.h/d' \
+        -e 's/SW_MAX/SW_MAX_/' \
+        "$f" > "$to/$header";
+}
+
 # This will pick up non-directories too (eg "Kconfig") but we will
 # ignore them in the next loop.
 ARCHLIST=$(cd "$linux/arch" && echo *)
@@ -40,7 +71,7 @@
     fi
 
     # Blacklist architectures which have KVM headers but are actually dead
-    if [ "$arch" = "ia64" ]; then
+    if [ "$arch" = "ia64" -o "$arch" = "mips" ]; then
         continue
     fi
 
@@ -48,21 +79,31 @@
 
     rm -rf "$output/linux-headers/asm-$arch"
     mkdir -p "$output/linux-headers/asm-$arch"
-    for header in kvm.h kvm_para.h; do
+    for header in kvm.h kvm_para.h unistd.h; do
         cp "$tmpdir/include/asm/$header" "$output/linux-headers/asm-$arch"
     done
-    if [ $arch = x86 ]; then
-        cp "$tmpdir/include/asm/hyperv.h" "$output/linux-headers/asm-x86"
-    fi
     if [ $arch = powerpc ]; then
         cp "$tmpdir/include/asm/epapr_hcalls.h" "$output/linux-headers/asm-powerpc/"
     fi
+
+    rm -rf "$output/include/standard-headers/asm-$arch"
+    mkdir -p "$output/include/standard-headers/asm-$arch"
+    if [ $arch = s390 ]; then
+        cp_portable "$tmpdir/include/asm/kvm_virtio.h" "$output/include/standard-headers/asm-s390/"
+        cp_portable "$tmpdir/include/asm/virtio-ccw.h" "$output/include/standard-headers/asm-s390/"
+    fi
+    if [ $arch = x86 ]; then
+        cp_portable "$tmpdir/include/asm/hyperv.h" "$output/include/standard-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_32.h" "$output/linux-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_x32.h" "$output/linux-headers/asm-x86/"
+        cp "$tmpdir/include/asm/unistd_64.h" "$output/linux-headers/asm-x86/"
+    fi
 done
 
 rm -rf "$output/linux-headers/linux"
 mkdir -p "$output/linux-headers/linux"
-for header in kvm.h kvm_para.h vfio.h vhost.h virtio_config.h virtio_ring.h \
-              psci.h; do
+for header in kvm.h kvm_para.h vfio.h vhost.h \
+              psci.h userfaultfd.h; do
     cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux"
 done
 rm -rf "$output/linux-headers/asm-generic"
@@ -76,4 +117,31 @@
     cp "$linux/COPYING" "$output/linux-headers"
 fi
 
+cat <<EOF >$output/linux-headers/asm-x86/hyperv.h
+#include "standard-headers/asm-x86/hyperv.h"
+EOF
+cat <<EOF >$output/linux-headers/linux/virtio_config.h
+#include "standard-headers/linux/virtio_config.h"
+EOF
+cat <<EOF >$output/linux-headers/linux/virtio_ring.h
+#include "standard-headers/linux/virtio_ring.h"
+EOF
+
+rm -rf "$output/include/standard-headers/linux"
+mkdir -p "$output/include/standard-headers/linux"
+for i in "$tmpdir"/include/linux/*virtio*.h "$tmpdir/include/linux/input.h" \
+         "$tmpdir/include/linux/input-event-codes.h" \
+         "$tmpdir/include/linux/pci_regs.h"; do
+    cp_portable "$i" "$output/include/standard-headers/linux"
+done
+
+cat <<EOF >$output/include/standard-headers/linux/types.h
+/* For QEMU all types are already defined via osdep.h, so this
+ * header does not need to do anything.
+ */
+EOF
+cat <<EOF >$output/include/standard-headers/linux/if_ether.h
+#define ETH_ALEN    6
+EOF
+
 rm -rf "$tmpdir"
diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py
index f7ce3fc..14a27e7 100755
--- a/scripts/vmstate-static-checker.py
+++ b/scripts/vmstate-static-checker.py
@@ -53,6 +53,8 @@
                                        'parent_obj.parent_obj.parent_obj',
                                        'port.br.dev.exp.aer_log',
                                 'parent_obj.parent_obj.parent_obj.exp.aer_log'],
+        'cirrus_vga': ['hw_cursor_x', 'vga.hw_cursor_x',
+                       'hw_cursor_y', 'vga.hw_cursor_y'],
         'lsiscsi': ['dev', 'parent_obj'],
         'mch': ['d', 'parent_obj'],
         'pci_bridge': ['bridge.dev', 'parent_obj', 'bridge.dev.shpc', 'shpc'],
@@ -97,6 +99,7 @@
     # Section names can change -- see commit 292b1634 for an example.
     changes = {
         "ICH9 LPC": "ICH9-LPC",
+        "e1000-82540em": "e1000",
     }
 
     for item in changes:
@@ -182,7 +185,7 @@
             if unused_count == 0:
                 advance_dest = True
 
-        if unused_count > 0:
+        if unused_count != 0:
             if advance_dest == False:
                 unused_count = unused_count - s_item["size"]
                 if unused_count == 0:
diff --git a/slirp/Makefile.objs b/slirp/Makefile.objs
index 2daa9dc..1baa1f1 100644
--- a/slirp/Makefile.objs
+++ b/slirp/Makefile.objs
@@ -1,3 +1,5 @@
-common-obj-y = cksum.o if.o ip_icmp.o ip_input.o ip_output.o dnssearch.o
+common-obj-y = cksum.o if.o ip_icmp.o ip6_icmp.o ip6_input.o ip6_output.o \
+               ip_input.o ip_output.o dnssearch.o dhcpv6.o
 common-obj-y += slirp.o mbuf.o misc.o sbuf.o socket.o tcp_input.o tcp_output.o
-common-obj-y += tcp_subr.o tcp_timer.o udp.o bootp.o tftp.o arp_table.o
+common-obj-y += tcp_subr.o tcp_timer.o udp.o udp6.o bootp.o tftp.o arp_table.o \
+                ndp_table.o
diff --git a/slirp/arp_table.c b/slirp/arp_table.c
index bcaeb44..3547043 100644
--- a/slirp/arp_table.c
+++ b/slirp/arp_table.c
@@ -22,6 +22,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "slirp.h"
 
 void arp_table_add(Slirp *slirp, uint32_t ip_addr, uint8_t ethaddr[ETH_ALEN])
diff --git a/slirp/bootp.c b/slirp/bootp.c
index b7db9fa..5a4646c 100644
--- a/slirp/bootp.c
+++ b/slirp/bootp.c
@@ -21,7 +21,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
+
+#if defined(_WIN32)
+/* Windows ntohl() returns an u_long value.
+ * Add a type cast to match the format strings. */
+# define ntohl(n) ((uint32_t)ntohl(n))
+#endif
 
 /* XXX: only DHCP is supported */
 
@@ -155,7 +162,7 @@
     dhcp_decode(bp, &dhcp_msg_type, &preq_addr);
     DPRINTF("bootp packet op=%d msgtype=%d", bp->bp_op, dhcp_msg_type);
     if (preq_addr.s_addr != htonl(0L))
-        DPRINTF(" req_addr=%08x\n", ntohl(preq_addr.s_addr));
+        DPRINTF(" req_addr=%08" PRIx32 "\n", ntohl(preq_addr.s_addr));
     else
         DPRINTF("\n");
 
@@ -234,7 +241,7 @@
     q += 4;
 
     if (bc) {
-        DPRINTF("%s addr=%08x\n",
+        DPRINTF("%s addr=%08" PRIx32 "\n",
                 (dhcp_msg_type == DHCPDISCOVER) ? "offered" : "ack'ed",
                 ntohl(daddr.sin_addr.s_addr));
 
@@ -302,7 +309,7 @@
     } else {
         static const char nak_msg[] = "requested address not available";
 
-        DPRINTF("nak'ed addr=%08x\n", ntohl(preq_addr.s_addr));
+        DPRINTF("nak'ed addr=%08" PRIx32 "\n", ntohl(preq_addr.s_addr));
 
         *q++ = RFC2132_MSG_TYPE;
         *q++ = 1;
@@ -319,7 +326,7 @@
 
     m->m_len = sizeof(struct bootp_t) -
         sizeof(struct ip) - sizeof(struct udphdr);
-    udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+    udp_output(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
 }
 
 void bootp_input(struct mbuf *m)
diff --git a/slirp/bootp.h b/slirp/bootp.h
index ec3b687..3945257 100644
--- a/slirp/bootp.h
+++ b/slirp/bootp.h
@@ -1,6 +1,7 @@
 /* bootp/dhcp defines */
+
 #ifndef SLIRP_BOOTP_H
-#define SLIRP_BOOTP_H 1
+#define SLIRP_BOOTP_H
 
 #define BOOTP_SERVER	67
 #define BOOTP_CLIENT	68
diff --git a/slirp/cksum.c b/slirp/cksum.c
index 6328660..6d73abf 100644
--- a/slirp/cksum.c
+++ b/slirp/cksum.c
@@ -30,7 +30,8 @@
  * in_cksum.c,v 1.2 1994/08/02 07:48:16 davidg Exp
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 
 /*
  * Checksum routine for Internet Protocol family headers (Portable Version).
@@ -137,3 +138,28 @@
 	REDUCE;
 	return (~sum & 0xffff);
 }
+
+int ip6_cksum(struct mbuf *m)
+{
+    /* TODO: Optimize this by being able to pass the ip6_pseudohdr to cksum
+     * separately from the mbuf */
+    struct ip6 save_ip, *ip = mtod(m, struct ip6 *);
+    struct ip6_pseudohdr *ih = mtod(m, struct ip6_pseudohdr *);
+    int sum;
+
+    save_ip = *ip;
+
+    ih->ih_src = save_ip.ip_src;
+    ih->ih_dst = save_ip.ip_dst;
+    ih->ih_pl = htonl((uint32_t)ntohs(save_ip.ip_pl));
+    ih->ih_zero_hi = 0;
+    ih->ih_zero_lo = 0;
+    ih->ih_nh = save_ip.ip_nh;
+
+    sum = cksum(m, ((int)sizeof(struct ip6_pseudohdr))
+                    + ntohl(ih->ih_pl));
+
+    *ip = save_ip;
+
+    return sum;
+}
diff --git a/slirp/dhcpv6.c b/slirp/dhcpv6.c
new file mode 100644
index 0000000..02c51c7
--- /dev/null
+++ b/slirp/dhcpv6.c
@@ -0,0 +1,209 @@
+/*
+ * SLIRP stateless DHCPv6
+ *
+ * We only support stateless DHCPv6, e.g. for network booting.
+ * See RFC 3315, RFC 3736, RFC 3646 and RFC 5970 for details.
+ *
+ * Copyright 2016 Thomas Huth, Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License,
+ * or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/log.h"
+#include "slirp.h"
+#include "dhcpv6.h"
+
+/* DHCPv6 message types */
+#define MSGTYPE_REPLY        7
+#define MSGTYPE_INFO_REQUEST 11
+
+/* DHCPv6 option types */
+#define OPTION_CLIENTID      1
+#define OPTION_IAADDR        5
+#define OPTION_ORO           6
+#define OPTION_DNS_SERVERS   23
+#define OPTION_BOOTFILE_URL  59
+
+struct requested_infos {
+    uint8_t *client_id;
+    int client_id_len;
+    bool want_dns;
+    bool want_boot_url;
+};
+
+/**
+ * Analyze the info request message sent by the client to see what data it
+ * provided and what it wants to have. The information is gathered in the
+ * "requested_infos" struct. Note that client_id (if provided) points into
+ * the odata region, thus the caller must keep odata valid as long as it
+ * needs to access the requested_infos struct.
+ */
+static int dhcpv6_parse_info_request(uint8_t *odata, int olen,
+                                     struct requested_infos *ri)
+{
+    int i, req_opt;
+
+    while (olen > 4) {
+        /* Parse one option */
+        int option = odata[0] << 8 | odata[1];
+        int len = odata[2] << 8 | odata[3];
+
+        if (len + 4 > olen) {
+            qemu_log_mask(LOG_GUEST_ERROR, "Guest sent bad DHCPv6 packet!\n");
+            return -E2BIG;
+        }
+
+        switch (option) {
+        case OPTION_IAADDR:
+            /* According to RFC3315, we must discard requests with IA option */
+            return -EINVAL;
+        case OPTION_CLIENTID:
+            if (len > 256) {
+                /* Avoid very long IDs which could cause problems later */
+                return -E2BIG;
+            }
+            ri->client_id = odata + 4;
+            ri->client_id_len = len;
+            break;
+        case OPTION_ORO:        /* Option request option */
+            if (len & 1) {
+                return -EINVAL;
+            }
+            /* Check which options the client wants to have */
+            for (i = 0; i < len; i += 2) {
+                req_opt = odata[4 + i] << 8 | odata[4 + i + 1];
+                switch (req_opt) {
+                case OPTION_DNS_SERVERS:
+                    ri->want_dns = true;
+                    break;
+                case OPTION_BOOTFILE_URL:
+                    ri->want_boot_url = true;
+                    break;
+                default:
+                    DEBUG_MISC((dfd, "dhcpv6: Unsupported option request %d\n",
+                                req_opt));
+                }
+            }
+            break;
+        default:
+            DEBUG_MISC((dfd, "dhcpv6 info req: Unsupported option %d, len=%d\n",
+                        option, len));
+        }
+
+        odata += len + 4;
+        olen -= len + 4;
+    }
+
+    return 0;
+}
+
+
+/**
+ * Handle information request messages
+ */
+static void dhcpv6_info_request(Slirp *slirp, struct sockaddr_in6 *srcsas,
+                                uint32_t xid, uint8_t *odata, int olen)
+{
+    struct requested_infos ri = { NULL };
+    struct sockaddr_in6 sa6, da6;
+    struct mbuf *m;
+    uint8_t *resp;
+
+    if (dhcpv6_parse_info_request(odata, olen, &ri) < 0) {
+        return;
+    }
+
+    m = m_get(slirp);
+    if (!m) {
+        return;
+    }
+    memset(m->m_data, 0, m->m_size);
+    m->m_data += IF_MAXLINKHDR;
+    resp = (uint8_t *)m->m_data + sizeof(struct ip6) + sizeof(struct udphdr);
+
+    /* Fill in response */
+    *resp++ = MSGTYPE_REPLY;
+    *resp++ = (uint8_t)(xid >> 16);
+    *resp++ = (uint8_t)(xid >> 8);
+    *resp++ = (uint8_t)xid;
+
+    if (ri.client_id) {
+        *resp++ = OPTION_CLIENTID >> 8;         /* option-code high byte */
+        *resp++ = OPTION_CLIENTID;              /* option-code low byte */
+        *resp++ = ri.client_id_len >> 8;        /* option-len high byte */
+        *resp++ = ri.client_id_len;             /* option-len low byte */
+        memcpy(resp, ri.client_id, ri.client_id_len);
+        resp += ri.client_id_len;
+    }
+    if (ri.want_dns) {
+        *resp++ = OPTION_DNS_SERVERS >> 8;      /* option-code high byte */
+        *resp++ = OPTION_DNS_SERVERS;           /* option-code low byte */
+        *resp++ = 0;                            /* option-len high byte */
+        *resp++ = 16;                           /* option-len low byte */
+        memcpy(resp, &slirp->vnameserver_addr6, 16);
+        resp += 16;
+    }
+    if (ri.want_boot_url) {
+        uint8_t *sa = slirp->vhost_addr6.s6_addr;
+        int slen, smaxlen;
+
+        *resp++ = OPTION_BOOTFILE_URL >> 8;     /* option-code high byte */
+        *resp++ = OPTION_BOOTFILE_URL;          /* option-code low byte */
+        smaxlen = (uint8_t *)m->m_data + IF_MTU - (resp + 2);
+        slen = snprintf((char *)resp + 2, smaxlen,
+                        "tftp://[%02x%02x:%02x%02x:%02x%02x:%02x%02x:"
+                                "%02x%02x:%02x%02x:%02x%02x:%02x%02x]/%s",
+                        sa[0], sa[1], sa[2], sa[3], sa[4], sa[5], sa[6], sa[7],
+                        sa[8], sa[9], sa[10], sa[11], sa[12], sa[13], sa[14],
+                        sa[15], slirp->bootp_filename);
+        slen = min(slen, smaxlen);
+        *resp++ = slen >> 8;                    /* option-len high byte */
+        *resp++ = slen;                         /* option-len low byte */
+        resp += slen;
+    }
+
+    sa6.sin6_addr = slirp->vhost_addr6;
+    sa6.sin6_port = DHCPV6_SERVER_PORT;
+    da6.sin6_addr = srcsas->sin6_addr;
+    da6.sin6_port = srcsas->sin6_port;
+    m->m_data += sizeof(struct ip6) + sizeof(struct udphdr);
+    m->m_len = resp - (uint8_t *)m->m_data;
+    udp6_output(NULL, m, &sa6, &da6);
+}
+
+/**
+ * Handle DHCPv6 messages sent by the client
+ */
+void dhcpv6_input(struct sockaddr_in6 *srcsas, struct mbuf *m)
+{
+    uint8_t *data = (uint8_t *)m->m_data + sizeof(struct udphdr);
+    int data_len = m->m_len - sizeof(struct udphdr);
+    uint32_t xid;
+
+    if (data_len < 4) {
+        return;
+    }
+
+    xid = ntohl(*(uint32_t *)data) & 0xffffff;
+
+    switch (data[0]) {
+    case MSGTYPE_INFO_REQUEST:
+        dhcpv6_info_request(m->slirp, srcsas, xid, &data[4], data_len - 4);
+        break;
+    default:
+        DEBUG_MISC((dfd, "dhcpv6_input: Unsupported message type 0x%x\n",
+                    data[0]));
+    }
+}
diff --git a/slirp/dhcpv6.h b/slirp/dhcpv6.h
new file mode 100644
index 0000000..9189cd3
--- /dev/null
+++ b/slirp/dhcpv6.h
@@ -0,0 +1,22 @@
+/*
+ * Definitions and prototypes for SLIRP stateless DHCPv6
+ *
+ * Copyright 2016 Thomas Huth, Red Hat Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2
+ * or later. See the COPYING file in the top-level directory.
+ */
+#ifndef SLIRP_DHCPV6_H
+#define SLIRP_DHCPV6_H
+
+#define DHCPV6_SERVER_PORT 547
+
+#define ALLDHCP_MULTICAST { .s6_addr = \
+                            { 0xff, 0x02, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x01, 0x00, 0x02 } }
+
+void dhcpv6_input(struct sockaddr_in6 *srcsas, struct mbuf *m);
+
+#endif
diff --git a/slirp/dnssearch.c b/slirp/dnssearch.c
index 4c9064e..8fb5633 100644
--- a/slirp/dnssearch.c
+++ b/slirp/dnssearch.c
@@ -22,10 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdlib.h>
-#include <string.h>
-#include <stdio.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "slirp.h"
 
 static const uint8_t RFC3397_OPT_DOMAIN_SEARCH = 119;
@@ -264,7 +261,7 @@
     }
 
     /* reserve extra 2 header bytes for each 255 bytes of output */
-    memreq += ((memreq + MAX_OPT_LEN - 1) / MAX_OPT_LEN) * OPT_HEADER_LEN;
+    memreq += DIV_ROUND_UP(memreq, MAX_OPT_LEN) * OPT_HEADER_LEN;
     result = g_malloc(memreq * sizeof(*result));
 
     outptr = result;
@@ -291,7 +288,7 @@
     domain_mkxrefs(domains, domains + num_domains - 1, 0);
     memreq = domain_compactify(domains, num_domains);
 
-    blocks = (memreq + MAX_OPT_LEN - 1) / MAX_OPT_LEN;
+    blocks = DIV_ROUND_UP(memreq, MAX_OPT_LEN);
     bsrc_end = memreq;
     bsrc_start = (blocks - 1) * MAX_OPT_LEN;
     bdst_start = bsrc_start + blocks * OPT_HEADER_LEN;
diff --git a/slirp/if.c b/slirp/if.c
index fb7acf8..51ae0d0 100644
--- a/slirp/if.c
+++ b/slirp/if.c
@@ -5,7 +5,8 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 #include "qemu/timer.h"
 
 static void
@@ -27,9 +28,9 @@
 void
 if_init(Slirp *slirp)
 {
-    slirp->if_fastq.ifq_next = slirp->if_fastq.ifq_prev = &slirp->if_fastq;
-    slirp->if_batchq.ifq_next = slirp->if_batchq.ifq_prev = &slirp->if_batchq;
-    slirp->next_m = &slirp->if_batchq;
+    slirp->if_fastq.qh_link = slirp->if_fastq.qh_rlink = &slirp->if_fastq;
+    slirp->if_batchq.qh_link = slirp->if_batchq.qh_rlink = &slirp->if_batchq;
+    slirp->next_m = (struct mbuf *) &slirp->if_batchq;
 }
 
 /*
@@ -53,8 +54,8 @@
 	int on_fastq = 1;
 
 	DEBUG_CALL("if_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("ifm = %lx", (long)ifm);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("ifm = %p", ifm);
 
 	/*
 	 * First remove the mbuf from m_usedlist,
@@ -73,7 +74,8 @@
 	 * We mustn't put this packet back on the fastq (or we'll send it out of order)
 	 * XXX add cache here?
 	 */
-	for (ifq = slirp->if_batchq.ifq_prev; ifq != &slirp->if_batchq;
+	for (ifq = (struct mbuf *) slirp->if_batchq.qh_rlink;
+	     (struct quehead *) ifq != &slirp->if_batchq;
 	     ifq = ifq->ifq_prev) {
 		if (so == ifq->ifq_so) {
 			/* A match! */
@@ -85,7 +87,7 @@
 
 	/* No match, check which queue to put it on */
 	if (so && (so->so_iptos & IPTOS_LOWDELAY)) {
-		ifq = slirp->if_fastq.ifq_prev;
+		ifq = (struct mbuf *) slirp->if_fastq.qh_rlink;
 		on_fastq = 1;
 		/*
 		 * Check if this packet is a part of the last
@@ -97,9 +99,9 @@
 			goto diddit;
 		}
         } else {
-		ifq = slirp->if_batchq.ifq_prev;
+		ifq = (struct mbuf *) slirp->if_batchq.qh_rlink;
                 /* Set next_m if the queue was empty so far */
-                if (slirp->next_m == &slirp->if_batchq) {
+                if ((struct quehead *) slirp->next_m == &slirp->if_batchq) {
                     slirp->next_m = ifm;
                 }
         }
@@ -165,10 +167,10 @@
     }
     slirp->if_start_busy = true;
 
-    if (slirp->if_fastq.ifq_next != &slirp->if_fastq) {
-        ifm_next = slirp->if_fastq.ifq_next;
+    if (slirp->if_fastq.qh_link != &slirp->if_fastq) {
+        ifm_next = (struct mbuf *) slirp->if_fastq.qh_link;
         next_from_batchq = false;
-    } else if (slirp->next_m != &slirp->if_batchq) {
+    } else if ((struct quehead *) slirp->next_m != &slirp->if_batchq) {
         /* Nothing on fastq, pick up from batchq via next_m */
         ifm_next = slirp->next_m;
         next_from_batchq = true;
@@ -181,19 +183,19 @@
         from_batchq = next_from_batchq;
 
         ifm_next = ifm->ifq_next;
-        if (ifm_next == &slirp->if_fastq) {
+        if ((struct quehead *) ifm_next == &slirp->if_fastq) {
             /* No more packets in fastq, switch to batchq */
             ifm_next = slirp->next_m;
             next_from_batchq = true;
         }
-        if (ifm_next == &slirp->if_batchq) {
+        if ((struct quehead *) ifm_next == &slirp->if_batchq) {
             /* end of batchq */
             ifm_next = NULL;
         }
 
         /* Try to send packet unless it already expired */
         if (ifm->expiration_date >= now && !if_encap(slirp, ifm)) {
-            /* Packet is delayed due to pending ARP resolution */
+            /* Packet is delayed due to pending ARP or NDP resolution */
             continue;
         }
 
@@ -217,7 +219,7 @@
                 /* Next packet in fastq is from the same session */
                 ifm_next = next;
                 next_from_batchq = false;
-            } else if (slirp->next_m == &slirp->if_batchq) {
+            } else if ((struct quehead *) slirp->next_m == &slirp->if_batchq) {
                 /* Set next_m and ifm_next if the session packet is now the
                  * only one on batchq */
                 slirp->next_m = ifm_next = next;
diff --git a/slirp/if.h b/slirp/if.h
index 3327023..69569c1 100644
--- a/slirp/if.h
+++ b/slirp/if.h
@@ -5,8 +5,8 @@
  * terms and conditions of the copyright.
  */
 
-#ifndef _IF_H_
-#define _IF_H_
+#ifndef IF_H
+#define IF_H
 
 #define IF_COMPRESS	0x01	/* We want compression */
 #define IF_NOCOMPRESS	0x02	/* Do not do compression */
@@ -17,7 +17,7 @@
 #define IF_MRU 1500
 #define	IF_COMP IF_AUTOCOMP	/* Flags for compression */
 
-/* 2 for alignment, 14 for ethernet, 40 for TCP/IP */
-#define IF_MAXLINKHDR (2 + 14 + 40)
+/* 2 for alignment, 14 for ethernet */
+#define IF_MAXLINKHDR (2 + ETH_HLEN)
 
 #endif
diff --git a/slirp/ip.h b/slirp/ip.h
index e2ee5e3..1df6723 100644
--- a/slirp/ip.h
+++ b/slirp/ip.h
@@ -30,8 +30,8 @@
  * ip.h,v 1.3 1994/08/21 05:27:30 paul Exp
  */
 
-#ifndef _IP_H_
-#define _IP_H_
+#ifndef IP_H
+#define IP_H
 
 #ifdef HOST_WORDS_BIGENDIAN
 # undef NTOHL
diff --git a/slirp/ip6.h b/slirp/ip6.h
new file mode 100644
index 0000000..0908855
--- /dev/null
+++ b/slirp/ip6.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#ifndef SLIRP_IP6_H
+#define SLIRP_IP6_H
+
+#include "net/eth.h"
+
+#define ALLNODES_MULTICAST  { .s6_addr = \
+                            { 0xff, 0x02, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x01 } }
+
+#define SOLICITED_NODE_PREFIX { .s6_addr = \
+                            { 0xff, 0x02, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x00,\
+                            0x00, 0x00, 0x00, 0x01,\
+                            0xff, 0x00, 0x00, 0x00 } }
+
+#define LINKLOCAL_ADDR  { .s6_addr = \
+                        { 0xfe, 0x80, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x02 } }
+
+#define ZERO_ADDR  { .s6_addr = \
+                        { 0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00,\
+                        0x00, 0x00, 0x00, 0x00 } }
+
+static inline bool in6_equal(const struct in6_addr *a, const struct in6_addr *b)
+{
+    return memcmp(a, b, sizeof(*a)) == 0;
+}
+
+static inline bool in6_equal_net(const struct in6_addr *a,
+                                 const struct in6_addr *b,
+                                 int prefix_len)
+{
+    if (memcmp(a, b, prefix_len / 8) != 0) {
+        return 0;
+    }
+
+    if (prefix_len % 8 == 0) {
+        return 1;
+    }
+
+    return a->s6_addr[prefix_len / 8] >> (8 - (prefix_len % 8))
+        == b->s6_addr[prefix_len / 8] >> (8 - (prefix_len % 8));
+}
+
+static inline bool in6_equal_mach(const struct in6_addr *a,
+                                  const struct in6_addr *b,
+                                  int prefix_len)
+{
+    if (memcmp(&(a->s6_addr[(prefix_len + 7) / 8]),
+               &(b->s6_addr[(prefix_len + 7) / 8]),
+               16 - (prefix_len + 7) / 8) != 0) {
+        return 0;
+    }
+
+    if (prefix_len % 8 == 0) {
+        return 1;
+    }
+
+    return (a->s6_addr[prefix_len / 8] & ((1U << (8 - (prefix_len % 8))) - 1))
+        == (b->s6_addr[prefix_len / 8] & ((1U << (8 - (prefix_len % 8))) - 1));
+}
+
+
+#define in6_equal_router(a)\
+    ((in6_equal_net(a, &slirp->vprefix_addr6, slirp->vprefix_len)\
+      && in6_equal_mach(a, &slirp->vhost_addr6, slirp->vprefix_len))\
+  || (in6_equal_net(a, &(struct in6_addr)LINKLOCAL_ADDR, 64)\
+      && in6_equal_mach(a, &slirp->vhost_addr6, 64)))
+
+#define in6_equal_dns(a)\
+    ((in6_equal_net(a, &slirp->vprefix_addr6, slirp->vprefix_len)\
+      && in6_equal_mach(a, &slirp->vnameserver_addr6, slirp->vprefix_len))\
+  || (in6_equal_net(a, &(struct in6_addr)LINKLOCAL_ADDR, 64)\
+      && in6_equal_mach(a, &slirp->vnameserver_addr6, 64)))
+
+#define in6_equal_host(a)\
+    (in6_equal_router(a) || in6_equal_dns(a))
+
+#define in6_solicitednode_multicast(a)\
+    (in6_equal_net(a, &(struct in6_addr)SOLICITED_NODE_PREFIX, 104))
+
+#define in6_zero(a)\
+    (in6_equal(a, &(struct in6_addr)ZERO_ADDR))
+
+/* Compute emulated host MAC address from its ipv6 address */
+static inline void in6_compute_ethaddr(struct in6_addr ip,
+                                       uint8_t eth[ETH_ALEN])
+{
+    eth[0] = 0x52;
+    eth[1] = 0x56;
+    memcpy(&eth[2], &ip.s6_addr[16 - (ETH_ALEN - 2)], ETH_ALEN - 2);
+}
+
+/*
+ * Definitions for internet protocol version 6.
+ * Per RFC 2460, December 1998.
+ */
+#define IP6VERSION      6
+#define IP6_HOP_LIMIT 255
+
+/*
+ * Structure of an internet header, naked of options.
+ */
+struct ip6 {
+#ifdef HOST_WORDS_BIGENDIAN
+    uint32_t
+        ip_v:4,         /* version */
+        ip_tc_hi:4,     /* traffic class */
+        ip_tc_lo:4,
+        ip_fl_hi:4,     /* flow label */
+        ip_fl_lo:16;
+#else
+    uint32_t
+        ip_tc_hi:4,
+        ip_v:4,
+        ip_fl_hi:4,
+        ip_tc_lo:4,
+        ip_fl_lo:16;
+#endif
+    uint16_t    ip_pl;               /* payload length */
+    uint8_t     ip_nh;               /* next header */
+    uint8_t     ip_hl;               /* hop limit */
+    struct in6_addr ip_src, ip_dst;  /* source and dest address */
+} QEMU_PACKED;
+
+/*
+ * IPv6 pseudo-header used by upper-layer protocols
+ */
+struct ip6_pseudohdr {
+    struct      in6_addr ih_src;  /* source internet address */
+    struct      in6_addr ih_dst;  /* destination internet address */
+    uint32_t    ih_pl;            /* upper-layer packet length */
+    uint16_t    ih_zero_hi;       /* zero */
+    uint8_t     ih_zero_lo;       /* zero */
+    uint8_t     ih_nh;            /* next header */
+} QEMU_PACKED;
+
+
+#endif
diff --git a/slirp/ip6_icmp.c b/slirp/ip6_icmp.c
new file mode 100644
index 0000000..6d18e28
--- /dev/null
+++ b/slirp/ip6_icmp.c
@@ -0,0 +1,444 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "slirp.h"
+#include "ip6_icmp.h"
+#include "qemu/timer.h"
+#include "qemu/error-report.h"
+#include "qemu/log.h"
+
+#define NDP_Interval g_rand_int_range(slirp->grand, \
+        NDP_MinRtrAdvInterval, NDP_MaxRtrAdvInterval)
+
+static void ra_timer_handler(void *opaque)
+{
+    Slirp *slirp = opaque;
+    timer_mod(slirp->ra_timer,
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + NDP_Interval);
+    ndp_send_ra(slirp);
+}
+
+void icmp6_init(Slirp *slirp)
+{
+    if (!slirp->in6_enabled) {
+        return;
+    }
+
+    slirp->ra_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, ra_timer_handler, slirp);
+    timer_mod(slirp->ra_timer,
+              qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + NDP_Interval);
+}
+
+void icmp6_cleanup(Slirp *slirp)
+{
+    if (!slirp->in6_enabled) {
+        return;
+    }
+
+    timer_del(slirp->ra_timer);
+    timer_free(slirp->ra_timer);
+}
+
+static void icmp6_send_echoreply(struct mbuf *m, Slirp *slirp, struct ip6 *ip,
+        struct icmp6 *icmp)
+{
+    struct mbuf *t = m_get(slirp);
+    t->m_len = sizeof(struct ip6) + ntohs(ip->ip_pl);
+    memcpy(t->m_data, m->m_data, t->m_len);
+
+    /* IPv6 Packet */
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_dst = ip->ip_src;
+    rip->ip_src = ip->ip_dst;
+
+    /* ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_ECHO_REPLY;
+    ricmp->icmp6_cksum = 0;
+
+    /* Checksum */
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+void icmp6_send_error(struct mbuf *m, uint8_t type, uint8_t code)
+{
+    Slirp *slirp = m->slirp;
+    struct mbuf *t;
+    struct ip6 *ip = mtod(m, struct ip6 *);
+
+    DEBUG_CALL("icmp6_send_error");
+    DEBUG_ARGS((dfd, " type = %d, code = %d\n", type, code));
+
+    if (IN6_IS_ADDR_MULTICAST(&ip->ip_src) ||
+            IN6_IS_ADDR_UNSPECIFIED(&ip->ip_src)) {
+        /* TODO icmp error? */
+        return;
+    }
+
+    t = m_get(slirp);
+
+    /* IPv6 packet */
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = (struct in6_addr)LINKLOCAL_ADDR;
+    rip->ip_dst = ip->ip_src;
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &rip->ip_dst, addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("target = %s", addrstr);
+#endif
+
+    rip->ip_nh = IPPROTO_ICMPV6;
+    const int error_data_len = min(m->m_len,
+            IF_MTU - (sizeof(struct ip6) + ICMP6_ERROR_MINLEN));
+    rip->ip_pl = htons(ICMP6_ERROR_MINLEN + error_data_len);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = type;
+    ricmp->icmp6_code = code;
+    ricmp->icmp6_cksum = 0;
+
+    switch (type) {
+    case ICMP6_UNREACH:
+    case ICMP6_TIMXCEED:
+        ricmp->icmp6_err.unused = 0;
+        break;
+    case ICMP6_TOOBIG:
+        ricmp->icmp6_err.mtu = htonl(IF_MTU);
+        break;
+    case ICMP6_PARAMPROB:
+        /* TODO: Handle this case */
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+    t->m_data += ICMP6_ERROR_MINLEN;
+    memcpy(t->m_data, m->m_data, error_data_len);
+
+    /* Checksum */
+    t->m_data -= ICMP6_ERROR_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+/*
+ * Send NDP Router Advertisement
+ */
+void ndp_send_ra(Slirp *slirp)
+{
+    DEBUG_CALL("ndp_send_ra");
+
+    /* Build IPv6 packet */
+    struct mbuf *t = m_get(slirp);
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = (struct in6_addr)LINKLOCAL_ADDR;
+    rip->ip_dst = (struct in6_addr)ALLNODES_MULTICAST;
+    rip->ip_nh = IPPROTO_ICMPV6;
+    rip->ip_pl = htons(ICMP6_NDP_RA_MINLEN
+                        + NDPOPT_LINKLAYER_LEN
+                        + NDPOPT_PREFIXINFO_LEN
+#ifndef _WIN32
+                        + NDPOPT_RDNSS_LEN
+#endif
+                        );
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* Build ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_NDP_RA;
+    ricmp->icmp6_code = 0;
+    ricmp->icmp6_cksum = 0;
+
+    /* NDP */
+    ricmp->icmp6_nra.chl = NDP_AdvCurHopLimit;
+    ricmp->icmp6_nra.M = NDP_AdvManagedFlag;
+    ricmp->icmp6_nra.O = NDP_AdvOtherConfigFlag;
+    ricmp->icmp6_nra.reserved = 0;
+    ricmp->icmp6_nra.lifetime = htons(NDP_AdvDefaultLifetime);
+    ricmp->icmp6_nra.reach_time = htonl(NDP_AdvReachableTime);
+    ricmp->icmp6_nra.retrans_time = htonl(NDP_AdvRetransTime);
+    t->m_data += ICMP6_NDP_RA_MINLEN;
+
+    /* Source link-layer address (NDP option) */
+    struct ndpopt *opt = mtod(t, struct ndpopt *);
+    opt->ndpopt_type = NDPOPT_LINKLAYER_SOURCE;
+    opt->ndpopt_len = NDPOPT_LINKLAYER_LEN / 8;
+    in6_compute_ethaddr(rip->ip_src, opt->ndpopt_linklayer);
+    t->m_data += NDPOPT_LINKLAYER_LEN;
+
+    /* Prefix information (NDP option) */
+    struct ndpopt *opt2 = mtod(t, struct ndpopt *);
+    opt2->ndpopt_type = NDPOPT_PREFIX_INFO;
+    opt2->ndpopt_len = NDPOPT_PREFIXINFO_LEN / 8;
+    opt2->ndpopt_prefixinfo.prefix_length = slirp->vprefix_len;
+    opt2->ndpopt_prefixinfo.L = 1;
+    opt2->ndpopt_prefixinfo.A = 1;
+    opt2->ndpopt_prefixinfo.reserved1 = 0;
+    opt2->ndpopt_prefixinfo.valid_lt = htonl(NDP_AdvValidLifetime);
+    opt2->ndpopt_prefixinfo.pref_lt = htonl(NDP_AdvPrefLifetime);
+    opt2->ndpopt_prefixinfo.reserved2 = 0;
+    opt2->ndpopt_prefixinfo.prefix = slirp->vprefix_addr6;
+    t->m_data += NDPOPT_PREFIXINFO_LEN;
+
+#ifndef _WIN32
+    /* Prefix information (NDP option) */
+    /* disabled for windows for now, until get_dns6_addr is implemented */
+    struct ndpopt *opt3 = mtod(t, struct ndpopt *);
+    opt3->ndpopt_type = NDPOPT_RDNSS;
+    opt3->ndpopt_len = NDPOPT_RDNSS_LEN / 8;
+    opt3->ndpopt_rdnss.reserved = 0;
+    opt3->ndpopt_rdnss.lifetime = htonl(2 * NDP_MaxRtrAdvInterval);
+    opt3->ndpopt_rdnss.addr = slirp->vnameserver_addr6;
+    t->m_data += NDPOPT_RDNSS_LEN;
+#endif
+
+    /* ICMPv6 Checksum */
+#ifndef _WIN32
+    t->m_data -= NDPOPT_RDNSS_LEN;
+#endif
+    t->m_data -= NDPOPT_PREFIXINFO_LEN;
+    t->m_data -= NDPOPT_LINKLAYER_LEN;
+    t->m_data -= ICMP6_NDP_RA_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+/*
+ * Send NDP Neighbor Solitication
+ */
+void ndp_send_ns(Slirp *slirp, struct in6_addr addr)
+{
+    DEBUG_CALL("ndp_send_ns");
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &addr, addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("target = %s", addrstr);
+#endif
+
+    /* Build IPv6 packet */
+    struct mbuf *t = m_get(slirp);
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = slirp->vhost_addr6;
+    rip->ip_dst = (struct in6_addr)SOLICITED_NODE_PREFIX;
+    memcpy(&rip->ip_dst.s6_addr[13], &addr.s6_addr[13], 3);
+    rip->ip_nh = IPPROTO_ICMPV6;
+    rip->ip_pl = htons(ICMP6_NDP_NS_MINLEN + NDPOPT_LINKLAYER_LEN);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* Build ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_NDP_NS;
+    ricmp->icmp6_code = 0;
+    ricmp->icmp6_cksum = 0;
+
+    /* NDP */
+    ricmp->icmp6_nns.reserved = 0;
+    ricmp->icmp6_nns.target = addr;
+
+    /* Build NDP option */
+    t->m_data += ICMP6_NDP_NS_MINLEN;
+    struct ndpopt *opt = mtod(t, struct ndpopt *);
+    opt->ndpopt_type = NDPOPT_LINKLAYER_SOURCE;
+    opt->ndpopt_len = NDPOPT_LINKLAYER_LEN / 8;
+    in6_compute_ethaddr(slirp->vhost_addr6, opt->ndpopt_linklayer);
+
+    /* ICMPv6 Checksum */
+    t->m_data -= ICMP6_NDP_NA_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 1);
+}
+
+/*
+ * Send NDP Neighbor Advertisement
+ */
+static void ndp_send_na(Slirp *slirp, struct ip6 *ip, struct icmp6 *icmp)
+{
+    /* Build IPv6 packet */
+    struct mbuf *t = m_get(slirp);
+    struct ip6 *rip = mtod(t, struct ip6 *);
+    rip->ip_src = icmp->icmp6_nns.target;
+    if (IN6_IS_ADDR_UNSPECIFIED(&ip->ip_src)) {
+        rip->ip_dst = (struct in6_addr)ALLNODES_MULTICAST;
+    } else {
+        rip->ip_dst = ip->ip_src;
+    }
+    rip->ip_nh = IPPROTO_ICMPV6;
+    rip->ip_pl = htons(ICMP6_NDP_NA_MINLEN
+                        + NDPOPT_LINKLAYER_LEN);
+    t->m_len = sizeof(struct ip6) + ntohs(rip->ip_pl);
+
+    /* Build ICMPv6 packet */
+    t->m_data += sizeof(struct ip6);
+    struct icmp6 *ricmp = mtod(t, struct icmp6 *);
+    ricmp->icmp6_type = ICMP6_NDP_NA;
+    ricmp->icmp6_code = 0;
+    ricmp->icmp6_cksum = 0;
+
+    /* NDP */
+    ricmp->icmp6_nna.R = NDP_IsRouter;
+    ricmp->icmp6_nna.S = !IN6_IS_ADDR_MULTICAST(&rip->ip_dst);
+    ricmp->icmp6_nna.O = 1;
+    ricmp->icmp6_nna.reserved_hi = 0;
+    ricmp->icmp6_nna.reserved_lo = 0;
+    ricmp->icmp6_nna.target = icmp->icmp6_nns.target;
+
+    /* Build NDP option */
+    t->m_data += ICMP6_NDP_NA_MINLEN;
+    struct ndpopt *opt = mtod(t, struct ndpopt *);
+    opt->ndpopt_type = NDPOPT_LINKLAYER_TARGET;
+    opt->ndpopt_len = NDPOPT_LINKLAYER_LEN / 8;
+    in6_compute_ethaddr(ricmp->icmp6_nna.target,
+                    opt->ndpopt_linklayer);
+
+    /* ICMPv6 Checksum */
+    t->m_data -= ICMP6_NDP_NA_MINLEN;
+    t->m_data -= sizeof(struct ip6);
+    ricmp->icmp6_cksum = ip6_cksum(t);
+
+    ip6_output(NULL, t, 0);
+}
+
+/*
+ * Process a NDP message
+ */
+static void ndp_input(struct mbuf *m, Slirp *slirp, struct ip6 *ip,
+        struct icmp6 *icmp)
+{
+    m->m_len += ETH_HLEN;
+    m->m_data -= ETH_HLEN;
+    struct ethhdr *eth = mtod(m, struct ethhdr *);
+    m->m_len -= ETH_HLEN;
+    m->m_data += ETH_HLEN;
+
+    switch (icmp->icmp6_type) {
+    case ICMP6_NDP_RS:
+        DEBUG_CALL(" type = Router Solicitation");
+        if (ip->ip_hl == 255
+                && icmp->icmp6_code == 0
+                && ntohs(ip->ip_pl) >= ICMP6_NDP_RS_MINLEN) {
+            /* Gratuitous NDP */
+            ndp_table_add(slirp, ip->ip_src, eth->h_source);
+
+            ndp_send_ra(slirp);
+        }
+        break;
+
+    case ICMP6_NDP_RA:
+        DEBUG_CALL(" type = Router Advertisement");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                "Warning: guest sent NDP RA, but shouldn't");
+        break;
+
+    case ICMP6_NDP_NS:
+        DEBUG_CALL(" type = Neighbor Solicitation");
+        if (ip->ip_hl == 255
+                && icmp->icmp6_code == 0
+                && !IN6_IS_ADDR_MULTICAST(&icmp->icmp6_nns.target)
+                && ntohs(ip->ip_pl) >= ICMP6_NDP_NS_MINLEN
+                && (!IN6_IS_ADDR_UNSPECIFIED(&ip->ip_src)
+                    || in6_solicitednode_multicast(&ip->ip_dst))) {
+            if (in6_equal_host(&icmp->icmp6_nns.target)) {
+                /* Gratuitous NDP */
+                ndp_table_add(slirp, ip->ip_src, eth->h_source);
+                ndp_send_na(slirp, ip, icmp);
+            }
+        }
+        break;
+
+    case ICMP6_NDP_NA:
+        DEBUG_CALL(" type = Neighbor Advertisement");
+        if (ip->ip_hl == 255
+                && icmp->icmp6_code == 0
+                && ntohs(ip->ip_pl) >= ICMP6_NDP_NA_MINLEN
+                && !IN6_IS_ADDR_MULTICAST(&icmp->icmp6_nna.target)
+                && (!IN6_IS_ADDR_MULTICAST(&ip->ip_dst)
+                    || icmp->icmp6_nna.S == 0)) {
+            ndp_table_add(slirp, ip->ip_src, eth->h_source);
+        }
+        break;
+
+    case ICMP6_NDP_REDIRECT:
+        DEBUG_CALL(" type = Redirect");
+        qemu_log_mask(LOG_GUEST_ERROR,
+                "Warning: guest sent NDP REDIRECT, but shouldn't");
+        break;
+    }
+}
+
+/*
+ * Process a received ICMPv6 message.
+ */
+void icmp6_input(struct mbuf *m)
+{
+    struct icmp6 *icmp;
+    struct ip6 *ip = mtod(m, struct ip6 *);
+    Slirp *slirp = m->slirp;
+    int hlen = sizeof(struct ip6);
+
+    DEBUG_CALL("icmp6_input");
+    DEBUG_ARG("m = %lx", (long) m);
+    DEBUG_ARG("m_len = %d", m->m_len);
+
+    if (ntohs(ip->ip_pl) < ICMP6_MINLEN) {
+        goto end;
+    }
+
+    if (ip6_cksum(m)) {
+        goto end;
+    }
+
+    m->m_len -= hlen;
+    m->m_data += hlen;
+    icmp = mtod(m, struct icmp6 *);
+    m->m_len += hlen;
+    m->m_data -= hlen;
+
+    DEBUG_ARG("icmp6_type = %d", icmp->icmp6_type);
+    switch (icmp->icmp6_type) {
+    case ICMP6_ECHO_REQUEST:
+        if (in6_equal_host(&ip->ip_dst)) {
+            icmp6_send_echoreply(m, slirp, ip, icmp);
+        } else {
+            /* TODO */
+            error_report("external icmpv6 not supported yet");
+        }
+        break;
+
+    case ICMP6_NDP_RS:
+    case ICMP6_NDP_RA:
+    case ICMP6_NDP_NS:
+    case ICMP6_NDP_NA:
+    case ICMP6_NDP_REDIRECT:
+        ndp_input(m, slirp, ip, icmp);
+        break;
+
+    case ICMP6_UNREACH:
+    case ICMP6_TOOBIG:
+    case ICMP6_TIMXCEED:
+    case ICMP6_PARAMPROB:
+        /* XXX? report error? close socket? */
+    default:
+        break;
+    }
+
+end:
+    m_free(m);
+}
diff --git a/slirp/ip6_icmp.h b/slirp/ip6_icmp.h
new file mode 100644
index 0000000..b3378b1
--- /dev/null
+++ b/slirp/ip6_icmp.h
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#ifndef SLIRP_IP6_ICMP_H
+#define SLIRP_IP6_ICMP_H
+
+/*
+ * Interface Control Message Protocol version 6 Definitions.
+ * Per RFC 4443, March 2006.
+ *
+ * Network Discover Protocol Definitions.
+ * Per RFC 4861, September 2007.
+ */
+
+struct icmp6_echo { /* Echo Messages */
+    uint16_t id;
+    uint16_t seq_num;
+};
+
+union icmp6_error_body {
+    uint32_t unused;
+    uint32_t pointer;
+    uint32_t mtu;
+};
+
+/*
+ * NDP Messages
+ */
+struct ndp_rs {     /* Router Solicitation Message */
+    uint32_t reserved;
+};
+
+struct ndp_ra {     /* Router Advertisement Message */
+    uint8_t chl;    /* Cur Hop Limit */
+#ifdef HOST_WORDS_BIGENDIAN
+    uint8_t
+        M:1,
+        O:1,
+        reserved:6;
+#else
+    uint8_t
+        reserved:6,
+        O:1,
+        M:1;
+#endif
+    uint16_t lifetime;      /* Router Lifetime */
+    uint32_t reach_time;    /* Reachable Time */
+    uint32_t retrans_time;  /* Retrans Timer */
+} QEMU_PACKED;
+
+struct ndp_ns {     /* Neighbor Solicitation Message */
+    uint32_t reserved;
+    struct in6_addr target; /* Target Address */
+} QEMU_PACKED;
+
+struct ndp_na {     /* Neighbor Advertisement Message */
+#ifdef HOST_WORDS_BIGENDIAN
+    uint32_t
+        R:1,                /* Router Flag */
+        S:1,                /* Solicited Flag */
+        O:1,                /* Override Flag */
+        reserved_hi:5,
+        reserved_lo:24;
+#else
+    uint32_t
+        reserved_hi:5,
+        O:1,
+        S:1,
+        R:1,
+        reserved_lo:24;
+#endif
+    struct in6_addr target; /* Target Address */
+} QEMU_PACKED;
+
+struct ndp_redirect {
+    uint32_t reserved;
+    struct in6_addr target; /* Target Address */
+    struct in6_addr dest;   /* Destination Address */
+} QEMU_PACKED;
+
+/*
+ * Structure of an icmpv6 header.
+ */
+struct icmp6 {
+    uint8_t     icmp6_type;         /* type of message, see below */
+    uint8_t     icmp6_code;         /* type sub code */
+    uint16_t    icmp6_cksum;        /* ones complement cksum of struct */
+    union {
+        union icmp6_error_body error_body;
+        struct icmp6_echo echo;
+        struct ndp_rs ndp_rs;
+        struct ndp_ra ndp_ra;
+        struct ndp_ns ndp_ns;
+        struct ndp_na ndp_na;
+        struct ndp_redirect ndp_redirect;
+    } icmp6_body;
+#define icmp6_err icmp6_body.error_body
+#define icmp6_echo icmp6_body.echo
+#define icmp6_nrs icmp6_body.ndp_rs
+#define icmp6_nra icmp6_body.ndp_ra
+#define icmp6_nns icmp6_body.ndp_ns
+#define icmp6_nna icmp6_body.ndp_na
+#define icmp6_redirect icmp6_body.ndp_redirect
+} QEMU_PACKED;
+
+#define ICMP6_MINLEN    4
+#define ICMP6_ERROR_MINLEN  8
+#define ICMP6_ECHO_MINLEN   8
+#define ICMP6_NDP_RS_MINLEN 8
+#define ICMP6_NDP_RA_MINLEN 16
+#define ICMP6_NDP_NS_MINLEN 24
+#define ICMP6_NDP_NA_MINLEN 24
+#define ICMP6_NDP_REDIRECT_MINLEN 40
+
+/*
+ * NDP Options
+ */
+struct ndpopt {
+    uint8_t     ndpopt_type;                    /* Option type */
+    uint8_t     ndpopt_len;                     /* /!\ In units of 8 octets */
+    union {
+        unsigned char   linklayer_addr[6];      /* Source/Target Link-layer */
+#define ndpopt_linklayer ndpopt_body.linklayer_addr
+        struct prefixinfo {                     /* Prefix Information */
+            uint8_t     prefix_length;
+#ifdef HOST_WORDS_BIGENDIAN
+            uint8_t     L:1, A:1, reserved1:6;
+#else
+            uint8_t     reserved1:6, A:1, L:1;
+#endif
+            uint32_t    valid_lt;               /* Valid Lifetime */
+            uint32_t    pref_lt;                /* Preferred Lifetime */
+            uint32_t    reserved2;
+            struct in6_addr prefix;
+        } QEMU_PACKED prefixinfo;
+#define ndpopt_prefixinfo ndpopt_body.prefixinfo
+        struct rdnss {
+            uint16_t reserved;
+            uint32_t lifetime;
+            struct in6_addr addr;
+        } QEMU_PACKED rdnss;
+#define ndpopt_rdnss ndpopt_body.rdnss
+    } ndpopt_body;
+} QEMU_PACKED;
+
+/* NDP options type */
+#define NDPOPT_LINKLAYER_SOURCE     1   /* Source Link-Layer Address */
+#define NDPOPT_LINKLAYER_TARGET     2   /* Target Link-Layer Address */
+#define NDPOPT_PREFIX_INFO          3   /* Prefix Information */
+#define NDPOPT_RDNSS                25  /* Recursive DNS Server Address */
+
+/* NDP options size, in octets. */
+#define NDPOPT_LINKLAYER_LEN    8
+#define NDPOPT_PREFIXINFO_LEN   32
+#define NDPOPT_RDNSS_LEN        24
+
+/*
+ * Definition of type and code field values.
+ * Per https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xml
+ * Last Updated 2012-11-12
+ */
+
+/* Errors */
+#define ICMP6_UNREACH   1   /* Destination Unreachable */
+#define     ICMP6_UNREACH_NO_ROUTE      0   /* no route to dest */
+#define     ICMP6_UNREACH_DEST_PROHIB   1   /* com with dest prohibited */
+#define     ICMP6_UNREACH_SCOPE         2   /* beyond scope of src addr */
+#define     ICMP6_UNREACH_ADDRESS       3   /* address unreachable */
+#define     ICMP6_UNREACH_PORT          4   /* port unreachable */
+#define     ICMP6_UNREACH_SRC_FAIL      5   /* src addr failed */
+#define     ICMP6_UNREACH_REJECT_ROUTE  6   /* reject route to dest */
+#define     ICMP6_UNREACH_SRC_HDR_ERROR 7   /* error in src routing header */
+#define ICMP6_TOOBIG    2   /* Packet Too Big */
+#define ICMP6_TIMXCEED  3   /* Time Exceeded */
+#define     ICMP6_TIMXCEED_INTRANS      0   /* hop limit exceeded in transit */
+#define     ICMP6_TIMXCEED_REASS        1   /* ttl=0 in reass */
+#define ICMP6_PARAMPROB 4   /* Parameter Problem */
+#define     ICMP6_PARAMPROB_HDR_FIELD   0   /* err header field */
+#define     ICMP6_PARAMPROB_NXTHDR_TYPE 1   /* unrecognized Next Header type */
+#define     ICMP6_PARAMPROB_IPV6_OPT    2   /* unrecognized IPv6 option */
+
+/* Informational Messages */
+#define ICMP6_ECHO_REQUEST      128 /* Echo Request */
+#define ICMP6_ECHO_REPLY        129 /* Echo Reply */
+#define ICMP6_NDP_RS            133 /* Router Solicitation (NDP) */
+#define ICMP6_NDP_RA            134 /* Router Advertisement (NDP) */
+#define ICMP6_NDP_NS            135 /* Neighbor Solicitation (NDP) */
+#define ICMP6_NDP_NA            136 /* Neighbor Advertisement (NDP) */
+#define ICMP6_NDP_REDIRECT      137 /* Redirect Message (NDP) */
+
+/*
+ * Router Configuration Variables (rfc4861#section-6)
+ */
+#define NDP_IsRouter                1
+#define NDP_AdvSendAdvertisements   1
+#define NDP_MaxRtrAdvInterval       600000
+#define NDP_MinRtrAdvInterval       ((NDP_MaxRtrAdvInterval >= 9) ? \
+                                        NDP_MaxRtrAdvInterval / 3 : \
+                                        NDP_MaxRtrAdvInterval)
+#define NDP_AdvManagedFlag          0
+#define NDP_AdvOtherConfigFlag      0
+#define NDP_AdvLinkMTU              0
+#define NDP_AdvReachableTime        0
+#define NDP_AdvRetransTime          0
+#define NDP_AdvCurHopLimit          64
+#define NDP_AdvDefaultLifetime      ((3 * NDP_MaxRtrAdvInterval) / 1000)
+#define NDP_AdvValidLifetime        86400
+#define NDP_AdvOnLinkFlag           1
+#define NDP_AdvPrefLifetime         14400
+#define NDP_AdvAutonomousFlag       1
+
+void icmp6_init(Slirp *slirp);
+void icmp6_cleanup(Slirp *slirp);
+void icmp6_input(struct mbuf *);
+void icmp6_send_error(struct mbuf *m, uint8_t type, uint8_t code);
+void ndp_send_ra(Slirp *slirp);
+void ndp_send_ns(Slirp *slirp, struct in6_addr addr);
+
+#endif
diff --git a/slirp/ip6_input.c b/slirp/ip6_input.c
new file mode 100644
index 0000000..ac2e3ea
--- /dev/null
+++ b/slirp/ip6_input.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "slirp.h"
+#include "ip6_icmp.h"
+
+/*
+ * IP initialization: fill in IP protocol switch table.
+ * All protocols not implemented in kernel go to raw IP protocol handler.
+ */
+void ip6_init(Slirp *slirp)
+{
+    icmp6_init(slirp);
+}
+
+void ip6_cleanup(Slirp *slirp)
+{
+    icmp6_cleanup(slirp);
+}
+
+void ip6_input(struct mbuf *m)
+{
+    struct ip6 *ip6;
+    Slirp *slirp = m->slirp;
+
+    if (!slirp->in6_enabled) {
+        goto bad;
+    }
+
+    DEBUG_CALL("ip6_input");
+    DEBUG_ARG("m = %lx", (long)m);
+    DEBUG_ARG("m_len = %d", m->m_len);
+
+    if (m->m_len < sizeof(struct ip6)) {
+        goto bad;
+    }
+
+    ip6 = mtod(m, struct ip6 *);
+
+    if (ip6->ip_v != IP6VERSION) {
+        goto bad;
+    }
+
+    if (ntohs(ip6->ip_pl) > IF_MTU) {
+        icmp6_send_error(m, ICMP6_TOOBIG, 0);
+        goto bad;
+    }
+
+    /* check ip_ttl for a correct ICMP reply */
+    if (ip6->ip_hl == 0) {
+        icmp6_send_error(m, ICMP6_TIMXCEED, ICMP6_TIMXCEED_INTRANS);
+        goto bad;
+    }
+
+    /*
+     * Switch out to protocol's input routine.
+     */
+    switch (ip6->ip_nh) {
+    case IPPROTO_TCP:
+        NTOHS(ip6->ip_pl);
+        tcp_input(m, sizeof(struct ip6), (struct socket *)NULL, AF_INET6);
+        break;
+    case IPPROTO_UDP:
+        udp6_input(m);
+        break;
+    case IPPROTO_ICMPV6:
+        icmp6_input(m);
+        break;
+    default:
+        m_free(m);
+    }
+    return;
+bad:
+    m_free(m);
+}
diff --git a/slirp/ip6_output.c b/slirp/ip6_output.c
new file mode 100644
index 0000000..762cbfe
--- /dev/null
+++ b/slirp/ip6_output.c
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "slirp.h"
+
+/* Number of packets queued before we start sending
+ * (to prevent allocing too many mbufs) */
+#define IF6_THRESH 10
+
+/*
+ * IPv6 output. The packet in mbuf chain m contains a IP header
+ */
+int ip6_output(struct socket *so, struct mbuf *m, int fast)
+{
+    struct ip6 *ip = mtod(m, struct ip6 *);
+
+    DEBUG_CALL("ip6_output");
+    DEBUG_ARG("so = %lx", (long)so);
+    DEBUG_ARG("m = %lx", (long)m);
+
+    /* Fill IPv6 header */
+    ip->ip_v = IP6VERSION;
+    ip->ip_hl = IP6_HOP_LIMIT;
+    ip->ip_tc_hi = 0;
+    ip->ip_tc_lo = 0;
+    ip->ip_fl_hi = 0;
+    ip->ip_fl_lo = 0;
+
+    if (fast) {
+        if_encap(m->slirp, m);
+    } else {
+        if_output(so, m);
+    }
+
+    return 0;
+}
diff --git a/slirp/ip_icmp.c b/slirp/ip_icmp.c
index 3556e73..5ffc7a6 100644
--- a/slirp/ip_icmp.c
+++ b/slirp/ip_icmp.c
@@ -30,6 +30,7 @@
  * ip_icmp.c,v 1.7 1995/05/30 08:09:42 rgrimes Exp
  */
 
+#include "qemu/osdep.h"
 #include "slirp.h"
 #include "ip_icmp.h"
 
@@ -37,7 +38,7 @@
 /* Be nice and tell them it's just a pseudo-ping packet */
 static const char icmp_ping_msg[] = "This is a pseudo-PING packet used by Slirp to emulate ICMP ECHO-REQUEST packets.\n";
 
-/* list of actions for icmp_error() on RX of an icmp message */
+/* list of actions for icmp_send_error() on RX of an icmp message */
 static const int icmp_flush[19] = {
 /*  ECHO REPLY (0)  */   0,
 		         1,
@@ -100,7 +101,7 @@
                (struct sockaddr *)&addr, sizeof(addr)) == -1) {
         DEBUG_MISC((dfd, "icmp_input icmp sendto tx errno = %d-%s\n",
                     errno, strerror(errno)));
-        icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0, strerror(errno));
+        icmp_send_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0, strerror(errno));
         icmp_detach(so);
     }
 
@@ -125,7 +126,7 @@
   Slirp *slirp = m->slirp;
 
   DEBUG_CALL("icmp_input");
-  DEBUG_ARG("m = %lx", (long )m);
+  DEBUG_ARG("m = %p", m);
   DEBUG_ARG("m_len = %d", m->m_len);
 
   /*
@@ -157,12 +158,12 @@
         goto freeit;
     } else {
       struct socket *so;
-      struct sockaddr_in addr;
+      struct sockaddr_storage addr;
       if ((so = socreate(slirp)) == NULL) goto freeit;
       if (icmp_send(so, m, hlen) == 0) {
         return;
       }
-      if(udp_attach(so) == -1) {
+      if (udp_attach(so, AF_INET) == -1) {
 	DEBUG_MISC((dfd,"icmp_input udp_attach errno = %d-%s\n",
 		    errno,strerror(errno)));
 	sofree(so);
@@ -170,8 +171,10 @@
 	goto end_error;
       }
       so->so_m = m;
+      so->so_ffamily = AF_INET;
       so->so_faddr = ip->ip_dst;
       so->so_fport = htons(7);
+      so->so_lfamily = AF_INET;
       so->so_laddr = ip->ip_src;
       so->so_lport = htons(9);
       so->so_iptos = ip->ip_tos;
@@ -179,25 +182,14 @@
       so->so_state = SS_ISFCONNECTED;
 
       /* Send the packet */
-      addr.sin_family = AF_INET;
-      if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-          slirp->vnetwork_addr.s_addr) {
-	/* It's an alias */
-	if (is_dns_addr(slirp, &so->so_faddr)) {
-	  if (get_dns_addr(&so->so_faddr, &addr.sin_addr) < 0)
-	    addr.sin_addr = loopback_addr;
-	} else {
-	  addr.sin_addr = loopback_addr;
-	}
-      } else {
-	addr.sin_addr = so->so_faddr;
-      }
-      addr.sin_port = so->so_fport;
+      addr = so->fhost.ss;
+      sotranslate_out(so, &addr);
+
       if(sendto(so->s, icmp_ping_msg, strlen(icmp_ping_msg), 0,
-		(struct sockaddr *)&addr, sizeof(addr)) == -1) {
+		(struct sockaddr *)&addr, sockaddr_size(&addr)) == -1) {
 	DEBUG_MISC((dfd,"icmp_input udp sendto tx errno = %d-%s\n",
 		    errno,strerror(errno)));
-	icmp_error(m, ICMP_UNREACH,ICMP_UNREACH_NET, 0,strerror(errno));
+	icmp_send_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0, strerror(errno));
 	udp_detach(so);
       }
     } /* if ip->ip_dst.s_addr == alias_addr.s_addr */
@@ -243,7 +235,7 @@
 
 #define ICMP_MAXDATALEN (IP_MSS-28)
 void
-icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
+icmp_send_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
            const char *message)
 {
   unsigned hlen, shlen, s_ip_len;
@@ -251,8 +243,8 @@
   register struct icmp *icp;
   register struct mbuf *m;
 
-  DEBUG_CALL("icmp_error");
-  DEBUG_ARG("msrc = %lx", (long )msrc);
+  DEBUG_CALL("icmp_send_error");
+  DEBUG_ARG("msrc = %p", msrc);
   DEBUG_ARG("msrc_len = %d", msrc->m_len);
 
   if(type!=ICMP_UNREACH && type!=ICMP_TIMXCEED) goto end_error;
@@ -441,7 +433,7 @@
         }
         DEBUG_MISC((dfd, " udp icmp rx errno = %d-%s\n", errno,
                     strerror(errno)));
-        icmp_error(so->so_m, ICMP_UNREACH, error_code, 0, strerror(errno));
+        icmp_send_error(so->so_m, ICMP_UNREACH, error_code, 0, strerror(errno));
     } else {
         icmp_reflect(so->so_m);
         so->so_m = NULL; /* Don't m_free() it again! */
diff --git a/slirp/ip_icmp.h b/slirp/ip_icmp.h
index be4426b..d88ab34 100644
--- a/slirp/ip_icmp.h
+++ b/slirp/ip_icmp.h
@@ -30,8 +30,8 @@
  * ip_icmp.h,v 1.4 1995/05/30 08:09:43 rgrimes Exp
  */
 
-#ifndef _NETINET_IP_ICMP_H_
-#define _NETINET_IP_ICMP_H_
+#ifndef NETINET_IP_ICMP_H
+#define NETINET_IP_ICMP_H
 
 /*
  * Interface Control Message Protocol Definitions.
@@ -156,8 +156,8 @@
 void icmp_init(Slirp *slirp);
 void icmp_cleanup(Slirp *slirp);
 void icmp_input(struct mbuf *, int);
-void icmp_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
-                const char *message);
+void icmp_send_error(struct mbuf *msrc, u_char type, u_char code, int minsize,
+                     const char *message);
 void icmp_reflect(struct mbuf *);
 void icmp_receive(struct socket *so);
 void icmp_detach(struct socket *so);
diff --git a/slirp/ip_input.c b/slirp/ip_input.c
index 880bdfd..348e1dc 100644
--- a/slirp/ip_input.c
+++ b/slirp/ip_input.c
@@ -38,8 +38,8 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
-#include <qemu/osdep.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 #include "ip_icmp.h"
 
 static struct ip *ip_reass(Slirp *slirp, struct ip *ip, struct ipq *fp);
@@ -79,12 +79,16 @@
 	register struct ip *ip;
 	int hlen;
 
+	if (!slirp->in_enabled) {
+		goto bad;
+	}
+
 	DEBUG_CALL("ip_input");
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("m_len = %d", m->m_len);
 
 	if (m->m_len < sizeof (struct ip)) {
-		return;
+		goto bad;
 	}
 
 	ip = mtod(m, struct ip *);
@@ -131,9 +135,9 @@
 	   m_adj(m, ip->ip_len - m->m_len);
 
 	/* check ip_ttl for a correct ICMP reply */
-	if(ip->ip_ttl==0) {
-	  icmp_error(m, ICMP_TIMXCEED,ICMP_TIMXCEED_INTRANS, 0,"ttl");
-	  goto bad;
+	if (ip->ip_ttl == 0) {
+	    icmp_send_error(m, ICMP_TIMXCEED, ICMP_TIMXCEED_INTRANS, 0, "ttl");
+	    goto bad;
 	}
 
 	/*
@@ -199,7 +203,7 @@
 	 */
 	switch (ip->ip_p) {
 	 case IPPROTO_TCP:
-		tcp_input(m, hlen, (struct socket *)NULL);
+		tcp_input(m, hlen, (struct socket *)NULL, AF_INET);
 		break;
 	 case IPPROTO_UDP:
 		udp_input(m, hlen);
@@ -232,9 +236,9 @@
 	int i, next;
 
 	DEBUG_CALL("ip_reass");
-	DEBUG_ARG("ip = %lx", (long)ip);
-	DEBUG_ARG("fp = %lx", (long)fp);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("ip = %p", ip);
+	DEBUG_ARG("fp = %p", fp);
+	DEBUG_ARG("m = %p", m);
 
 	/*
 	 * Presence of header sizes in mbufs
@@ -400,7 +404,7 @@
 ip_enq(register struct ipasfrag *p, register struct ipasfrag *prev)
 {
 	DEBUG_CALL("ip_enq");
-	DEBUG_ARG("prev = %lx", (long)prev);
+	DEBUG_ARG("prev = %p", prev);
 	p->ipf_prev =  prev;
 	p->ipf_next = prev->ipf_next;
 	((struct ipasfrag *)(prev->ipf_next))->ipf_prev = p;
@@ -636,7 +640,7 @@
 	}
 	return (0);
 bad:
- 	icmp_error(m, type, code, 0, 0);
+	icmp_send_error(m, type, code, 0, 0);
 
 	return (1);
 }
diff --git a/slirp/ip_output.c b/slirp/ip_output.c
index c82830f..db403f0 100644
--- a/slirp/ip_output.c
+++ b/slirp/ip_output.c
@@ -38,7 +38,8 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 
 /* Number of packets queued before we start sending
  * (to prevent allocing too many mbufs) */
@@ -60,8 +61,8 @@
 	int len, off, error = 0;
 
 	DEBUG_CALL("ip_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m0 = %lx", (long)m0);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m0 = %p", m0);
 
 	ip = mtod(m, struct ip *);
 	/*
diff --git a/slirp/libslirp.h b/slirp/libslirp.h
index c2e81c1..5f1b4ab 100644
--- a/slirp/libslirp.h
+++ b/slirp/libslirp.h
@@ -1,26 +1,44 @@
-#ifndef _LIBSLIRP_H
-#define _LIBSLIRP_H
+#ifndef LIBSLIRP_H
+#define LIBSLIRP_H
 
 #include "qemu-common.h"
 
-int is_dns_addr(Slirp* slirp, const struct in_addr* guest_addr);
+struct Slirp;
+typedef struct Slirp Slirp;
 
-int get_dns_addr(const struct in_addr* guest_addr, struct in_addr* host_addr);
-
-int slirp_get_system_dns_servers(void);
-
-int slirp_parse_dns_servers(const char* servers);
-
-int slirp_get_max_dns_servers(void);
-
-Slirp *slirp_init(int restricted, struct in_addr vnetwork,
+Slirp *slirp_init(int restricted, bool in_enabled, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
-                  const char *vhostname, const char *tftp_path,
-                  const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, const char **vdnssearch,
+                  bool in6_enabled,
+                  struct in6_addr vprefix_addr6, uint8_t vprefix_len,
+                  struct in6_addr vhost6, const char *vhostname,
+                  const char *tftp_path, const char *bootfile,
+                  struct in_addr vdhcp_start, struct in_addr vnameserver,
+                  struct in6_addr vnameserver6, const char **vdnssearch,
                   void *opaque);
 void slirp_cleanup(Slirp *slirp);
 
+/* Setup custom DNS servers to be used. |dns_servers| is an array of
+ * |num_dns_servers| IPv4 host addresses for DNS servers to be used
+ * when translating guest IPv4 DNS addresses (typically from 10.0.2.3
+ * to 10.0.2.(3 + num_dns_servers - 1). */
+void slirp_init_custom_dns_servers(Slirp *slirp,
+                                   const struct sockaddr_storage *dns_servers,
+                                   int num_dns_servers);
+
+
+/* Check whether |guest_ip| is the IPv4 address of a guest DNS server.
+ * If that's the case, set |*host_ip| to the corresponding host DNS server
+ * address and return 0. Otherwise, leave |host_ip| untouched and
+ * return -1 */
+int slirp_translate_guest_dns(Slirp *slirp,
+                              const struct sockaddr_in *guest_ip,
+                              struct sockaddr_in *host_ip);
+
+/* Same for IPv6 addresses */
+int slirp_translate_guest_dns6(Slirp *slirp,
+                               const struct sockaddr_in6 *guest_ip6,
+                               struct sockaddr_in6 *host_ip6);
+
 void slirp_pollfds_fill(GArray *pollfds, uint32_t *timeout);
 
 void slirp_pollfds_poll(GArray *pollfds, int select_error);
diff --git a/slirp/main.h b/slirp/main.h
index f2e58cf..90053ce 100644
--- a/slirp/main.h
+++ b/slirp/main.h
@@ -4,8 +4,9 @@
  * Please read the file COPYRIGHT for the
  * terms and conditions of the copyright.
  */
+
 #ifndef SLIRP_MAIN_H
-#define SLIRP_MAIN_H 1
+#define SLIRP_MAIN_H
 
 #ifdef HAVE_SYS_SELECT_H
 #include <sys/select.h>
diff --git a/slirp/mbuf.c b/slirp/mbuf.c
index 4fefb04..7eddc21 100644
--- a/slirp/mbuf.c
+++ b/slirp/mbuf.c
@@ -15,29 +15,30 @@
  * the flags
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 
 #define MBUF_THRESH 30
 
 /*
  * Find a nice value for msize
- * XXX if_maxlinkhdr already in mtu
  */
-#define SLIRP_MSIZE (IF_MTU + IF_MAXLINKHDR + offsetof(struct mbuf, m_dat) + 6)
+#define SLIRP_MSIZE\
+    (offsetof(struct mbuf, m_dat) + IF_MAXLINKHDR + TCPIPHDR_DELTA + IF_MTU)
 
 void
 m_init(Slirp *slirp)
 {
-    slirp->m_freelist.m_next = slirp->m_freelist.m_prev = &slirp->m_freelist;
-    slirp->m_usedlist.m_next = slirp->m_usedlist.m_prev = &slirp->m_usedlist;
+    slirp->m_freelist.qh_link = slirp->m_freelist.qh_rlink = &slirp->m_freelist;
+    slirp->m_usedlist.qh_link = slirp->m_usedlist.qh_rlink = &slirp->m_usedlist;
 }
 
 void m_cleanup(Slirp *slirp)
 {
     struct mbuf *m, *next;
 
-    m = slirp->m_usedlist.m_next;
-    while (m != &slirp->m_usedlist) {
+    m = (struct mbuf *) slirp->m_usedlist.qh_link;
+    while ((struct quehead *) m != &slirp->m_usedlist) {
         next = m->m_next;
         if (m->m_flags & M_EXT) {
             free(m->m_ext);
@@ -45,8 +46,8 @@
         free(m);
         m = next;
     }
-    m = slirp->m_freelist.m_next;
-    while (m != &slirp->m_freelist) {
+    m = (struct mbuf *) slirp->m_freelist.qh_link;
+    while ((struct quehead *) m != &slirp->m_freelist) {
         next = m->m_next;
         free(m);
         m = next;
@@ -69,7 +70,7 @@
 
 	DEBUG_CALL("m_get");
 
-	if (slirp->m_freelist.m_next == &slirp->m_freelist) {
+	if (slirp->m_freelist.qh_link == &slirp->m_freelist) {
 		m = (struct mbuf *)malloc(SLIRP_MSIZE);
 		if (m == NULL) goto end_error;
 		slirp->mbuf_alloced++;
@@ -77,7 +78,7 @@
 			flags = M_DOFREE;
 		m->slirp = slirp;
 	} else {
-		m = slirp->m_freelist.m_next;
+		m = (struct mbuf *) slirp->m_freelist.qh_link;
 		remque(m);
 	}
 
@@ -91,10 +92,10 @@
 	m->m_len = 0;
         m->m_nextpkt = NULL;
         m->m_prevpkt = NULL;
-        m->arp_requested = false;
+        m->resolution_requested = false;
         m->expiration_date = (uint64_t)-1;
 end_error:
-	DEBUG_ARG("m = %lx", (long )m);
+	DEBUG_ARG("m = %p", m);
 	return m;
 }
 
@@ -103,7 +104,7 @@
 {
 
   DEBUG_CALL("m_free");
-  DEBUG_ARG("m = %lx", (long )m);
+  DEBUG_ARG("m = %p", m);
 
   if(m) {
 	/* Remove from m_usedlist */
@@ -221,10 +222,11 @@
 	struct mbuf *m;
 
 	DEBUG_CALL("dtom");
-	DEBUG_ARG("dat = %lx", (long )dat);
+	DEBUG_ARG("dat = %p", dat);
 
 	/* bug corrected for M_EXT buffers */
-	for (m = slirp->m_usedlist.m_next; m != &slirp->m_usedlist;
+	for (m = (struct mbuf *) slirp->m_usedlist.qh_link;
+	     (struct quehead *) m != &slirp->m_usedlist;
 	     m = m->m_next) {
 	  if (m->m_flags & M_EXT) {
 	    if( (char *)dat>=m->m_ext && (char *)dat<(m->m_ext + m->m_size) )
diff --git a/slirp/mbuf.h b/slirp/mbuf.h
index b144f1c..893601f 100644
--- a/slirp/mbuf.h
+++ b/slirp/mbuf.h
@@ -30,8 +30,8 @@
  * mbuf.h,v 1.9 1994/11/14 13:54:20 bde Exp
  */
 
-#ifndef _MBUF_H_
-#define _MBUF_H_
+#ifndef MBUF_H
+#define MBUF_H
 
 #define MINCSIZE 4096	/* Amount to increase mbuf if too small */
 
@@ -79,13 +79,11 @@
 	int	m_len;			/* Amount of data in this mbuf */
 
 	Slirp *slirp;
-	bool	arp_requested;
+	bool	resolution_requested;
 	uint64_t expiration_date;
+	char   *m_ext;
 	/* start of dynamic buffer area, must be last element */
-	union {
-		char	m_dat[1]; /* ANSI don't like 0 sized arrays */
-		char	*m_ext;
-	};
+	char    m_dat[];
 };
 
 #define ifq_prev m_prev
diff --git a/slirp/misc.c b/slirp/misc.c
index 6543dc7..88e9d94 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -5,21 +5,17 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
-#include <libslirp.h>
-
+#include "qemu/osdep.h"
+#include "slirp.h"
+#include "libslirp.h"
 #include "monitor/monitor.h"
+#include "qemu/error-report.h"
 #include "qemu/main-loop.h"
 
 #ifdef DEBUG
 int slirp_debug = DBG_CALL|DBG_MISC|DBG_ERROR;
 #endif
 
-struct quehead {
-	struct quehead *qh_link;
-	struct quehead *qh_rlink;
-};
-
 inline void
 insque(void *a, void *b)
 {
@@ -63,27 +59,6 @@
 	return 0;
 }
 
-#ifndef HAVE_STRERROR
-
-/*
- * For systems with no strerror
- */
-
-extern int sys_nerr;
-extern char *sys_errlist[];
-
-char *
-strerror(error)
-	int error;
-{
-	if (error < sys_nerr)
-	   return sys_errlist[error];
-	else
-	   return "Unknown error.";
-}
-
-#endif
-
 
 #ifdef _WIN32
 
@@ -122,9 +97,9 @@
 	pid_t pid;
 
 	DEBUG_CALL("fork_exec");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("ex = %lx", (long)ex);
-	DEBUG_ARG("do_pty = %lx", (long)do_pty);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("ex = %p", ex);
+	DEBUG_ARG("do_pty = %x", do_pty);
 
 	if (do_pty == 2) {
                 return 0;
diff --git a/slirp/misc.h b/slirp/misc.h
index 41a3258..5211bbd 100644
--- a/slirp/misc.h
+++ b/slirp/misc.h
@@ -5,8 +5,8 @@
  * terms and conditions of the copyright.
  */
 
-#ifndef _MISC_H_
-#define _MISC_H_
+#ifndef MISC_H
+#define MISC_H
 
 struct ex_list {
 	int ex_pty;			/* Do we want a pty? */
@@ -45,6 +45,11 @@
     struct emu_t *next;
 };
 
+struct slirp_quehead {
+    struct slirp_quehead *qh_link;
+    struct slirp_quehead *qh_rlink;
+};
+
 void slirp_insque(void *, void *);
 void slirp_remque(void *);
 int add_exec(struct ex_list **, int, char *, struct in_addr, int);
diff --git a/slirp/ndp_table.c b/slirp/ndp_table.c
new file mode 100644
index 0000000..9d4c39b
--- /dev/null
+++ b/slirp/ndp_table.c
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron, Yann Bordenave, Serigne Modou Wagne.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "slirp.h"
+
+void ndp_table_add(Slirp *slirp, struct in6_addr ip_addr,
+                    uint8_t ethaddr[ETH_ALEN])
+{
+    NdpTable *ndp_table = &slirp->ndp_table;
+    int i;
+
+    DEBUG_CALL("ndp_table_add");
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &(ip_addr), addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("ip = %s", addrstr);
+#endif
+    DEBUG_ARGS((dfd, " hw addr = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                ethaddr[0], ethaddr[1], ethaddr[2],
+                ethaddr[3], ethaddr[4], ethaddr[5]));
+
+    if (IN6_IS_ADDR_MULTICAST(&ip_addr) || IN6_IS_ADDR_UNSPECIFIED(&ip_addr)) {
+        /* Do not register multicast or unspecified addresses */
+        DEBUG_CALL(" abort: do not register multicast or unspecified address");
+        return;
+    }
+
+    /* Search for an entry */
+    for (i = 0; i < NDP_TABLE_SIZE; i++) {
+        if (in6_equal(&ndp_table->table[i].ip_addr, &ip_addr)) {
+            DEBUG_CALL(" already in table: update the entry");
+            /* Update the entry */
+            memcpy(ndp_table->table[i].eth_addr, ethaddr, ETH_ALEN);
+            return;
+        }
+    }
+
+    /* No entry found, create a new one */
+    DEBUG_CALL(" create new entry");
+    ndp_table->table[ndp_table->next_victim].ip_addr = ip_addr;
+    memcpy(ndp_table->table[ndp_table->next_victim].eth_addr,
+            ethaddr, ETH_ALEN);
+    ndp_table->next_victim = (ndp_table->next_victim + 1) % NDP_TABLE_SIZE;
+}
+
+bool ndp_table_search(Slirp *slirp, struct in6_addr ip_addr,
+                      uint8_t out_ethaddr[ETH_ALEN])
+{
+    NdpTable *ndp_table = &slirp->ndp_table;
+    int i;
+
+    DEBUG_CALL("ndp_table_search");
+#if !defined(_WIN32) || (_WIN32_WINNT >= 0x0600)
+    char addrstr[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &(ip_addr), addrstr, INET6_ADDRSTRLEN);
+    DEBUG_ARG("ip = %s", addrstr);
+#endif
+
+    assert(!IN6_IS_ADDR_UNSPECIFIED(&ip_addr));
+
+    /* Multicast address: fec0::abcd:efgh/8 -> 33:33:ab:cd:ef:gh */
+    if (IN6_IS_ADDR_MULTICAST(&ip_addr)) {
+        out_ethaddr[0] = 0x33; out_ethaddr[1] = 0x33;
+        out_ethaddr[2] = ip_addr.s6_addr[12];
+        out_ethaddr[3] = ip_addr.s6_addr[13];
+        out_ethaddr[4] = ip_addr.s6_addr[14];
+        out_ethaddr[5] = ip_addr.s6_addr[15];
+        DEBUG_ARGS((dfd, " multicast addr = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                    out_ethaddr[0], out_ethaddr[1], out_ethaddr[2],
+                    out_ethaddr[3], out_ethaddr[4], out_ethaddr[5]));
+        return 1;
+    }
+
+    for (i = 0; i < NDP_TABLE_SIZE; i++) {
+        if (in6_equal(&ndp_table->table[i].ip_addr, &ip_addr)) {
+            memcpy(out_ethaddr, ndp_table->table[i].eth_addr,  ETH_ALEN);
+            DEBUG_ARGS((dfd, " found hw addr = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                        out_ethaddr[0], out_ethaddr[1], out_ethaddr[2],
+                        out_ethaddr[3], out_ethaddr[4], out_ethaddr[5]));
+            return 1;
+        }
+    }
+
+    DEBUG_CALL(" ip not found in table");
+    return 0;
+}
diff --git a/slirp/sbuf.c b/slirp/sbuf.c
index 08ec2b4..10119d3 100644
--- a/slirp/sbuf.c
+++ b/slirp/sbuf.c
@@ -5,8 +5,9 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
-#include <qemu/main-loop.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
+#include "qemu/main-loop.h"
 
 static void sbappendsb(struct sbuf *sb, struct mbuf *m);
 
@@ -72,8 +73,8 @@
 	int ret = 0;
 
 	DEBUG_CALL("sbappend");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("m->m_len = %d", m->m_len);
 
 	/* Shouldn't happen, but...  e.g. foreign host closes connection */
diff --git a/slirp/sbuf.h b/slirp/sbuf.h
index 4f22e7c..efcec39 100644
--- a/slirp/sbuf.h
+++ b/slirp/sbuf.h
@@ -5,8 +5,8 @@
  * terms and conditions of the copyright.
  */
 
-#ifndef _SBUF_H_
-#define _SBUF_H_
+#ifndef SBUF_H
+#define SBUF_H
 
 #define sbflush(sb) sbdrop((sb),(sb)->sb_cc)
 #define sbspace(sb) ((sb)->sb_datalen - (sb)->sb_cc)
diff --git a/slirp/slirp.c b/slirp/slirp.c
index a247af8..3e4048d 100644
--- a/slirp/slirp.c
+++ b/slirp/slirp.c
@@ -21,17 +21,18 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifdef CONFIG_ANDROID
-#include "android/utils/dns.h"
-#include "android/utils/ipaddr.h"
-#include "android/utils/sockets.h"
-#endif  // CONFIG_ANDROID
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
 #include "sysemu/char.h"
 #include "slirp.h"
 #include "hw/hw.h"
+#include "qemu/cutils.h"
+
+#ifndef _WIN32
+#include <net/if.h>
+#endif
 
 /* host loopback address */
 struct in_addr loopback_addr;
@@ -48,18 +49,14 @@
 static QTAILQ_HEAD(slirp_instances, Slirp) slirp_instances =
     QTAILQ_HEAD_INITIALIZER(slirp_instances);
 
-#ifdef CONFIG_ANDROID
-#define SPECIAL_ADDRESS_IP "10.0.2.0"
-#define DNS_ADDR_MAX 4
-#define DNS_ADDR_BASE 3
-
-static uint32_t dns_addr[DNS_ADDR_MAX];
-static size_t dns_addr_count = 0;
-static struct in_addr special_addr_ip;
-#else  // !(CONFIG_ANDROID)
 static struct in_addr dns_addr;
+#ifndef _WIN32
+static struct in6_addr dns6_addr;
+#endif
 static u_int dns_addr_time;
-#endif  // CONFIG_ANDROID
+#ifndef _WIN32
+static u_int dns6_addr_time;
+#endif
 
 #define TIMEOUT_FAST 2  /* milliseconds */
 #define TIMEOUT_SLOW 499  /* milliseconds */
@@ -67,70 +64,8 @@
 #define TIMEOUT_DEFAULT 1000  /* milliseconds */
 
 #ifdef _WIN32
-static void winsock_cleanup(void)
-{
-    WSACleanup();
-}
-#endif
 
-#ifdef CONFIG_ANDROID
-static size_t get_dns_index(uint32_t addr) {
-    /* Use unsigned underflow so that values less than DNS_BASE_ADDR become
-     * huge and invalid */
-    return (addr & 0xff) - DNS_ADDR_BASE;
-}
-
-// The Slirp parameter is just there to maintain the same interface that would
-// be used for non-Android compiles. This makes calling code cleaner.
-int is_dns_addr(Slirp* slirp, const struct in_addr* address)  {
-    /* special_addr is stored in host byte order but the incoming parameter is
-     * in network order. Make sure they're both in host order */
-    uint32_t addr = ntohl(address->s_addr);
-    uint32_t network = addr & 0xffffff00;
-    return network == special_addr_ip.s_addr &&
-            get_dns_index(addr) < DNS_ADDR_MAX;
-}
-
-int get_dns_addr(const struct in_addr* guest_addr,
-                 struct in_addr* pdns_addr)
-{
-    size_t dns_index = get_dns_index(ntohl(guest_addr->s_addr));
-
-    if (dns_index < dns_addr_count && dns_addr[dns_index] != 0) {
-        pdns_addr->s_addr = htonl(dns_addr[dns_index]);
-        return 0;
-    }
-    return -1;
-}
-
-int slirp_get_system_dns_servers(void)
-{
-    int num_servers = android_dns_get_system_servers(dns_addr, DNS_ADDR_MAX);
-    if (num_servers >= 0)
-        dns_addr_count = num_servers;
-    return num_servers;
-}
-
-int slirp_parse_dns_servers(const char* servers) {
-    int num_servers = android_dns_parse_servers(servers, dns_addr, DNS_ADDR_MAX);
-    if (num_servers >= 0)
-        dns_addr_count = num_servers;
-    return num_servers;
-}
-
-int slirp_get_max_dns_servers(void) {
-    return DNS_ADDR_MAX;
-}
-
-#else  // !(CONFIG_ANDROID)
-
-int is_dns_addr(Slirp* slirp, const struct in_addr* address)  {
-    return address->s_addr == slirp->vnameserver_addr.s_addr;
-}
-
-#ifdef _WIN32
-int get_dns_addr(const struct in_addr* guest_addr,
-                 struct in_addr* pdns_addr)
+static int get_dns_addr(struct in_addr *pdns_addr)
 {
     FIXED_INFO *FixedInfo=NULL;
     ULONG    BufLen;
@@ -138,8 +73,6 @@
     IP_ADDR_STRING *pIPAddr;
     struct in_addr tmp_addr;
 
-    (void)guest_addr;
-
     if (dns_addr.s_addr != 0 && (curtime - dns_addr_time) < TIMEOUT_DEFAULT) {
         *pdns_addr = dns_addr;
         return 0;
@@ -177,36 +110,51 @@
     return 0;
 }
 
-#else  // !(_WIN32)
+static int get_dns6_addr(struct in6_addr *pdns6_addr, uint32_t *scope_id)
+{
+    return -1;
+}
 
-static struct stat dns_addr_stat;
+static void winsock_cleanup(void)
+{
+    WSACleanup();
+}
 
-int get_dns_addr(const struct in_addr* guest_addr,
-                 struct in_addr* pdns_addr)
+#else
+
+static int get_dns_addr_cached(void *pdns_addr, void *cached_addr,
+                               socklen_t addrlen,
+                               struct stat *cached_stat, u_int *cached_time)
+{
+    struct stat old_stat;
+    if (curtime - *cached_time < TIMEOUT_DEFAULT) {
+        memcpy(pdns_addr, cached_addr, addrlen);
+        return 0;
+    }
+    old_stat = *cached_stat;
+    if (qemu_stat("/etc/resolv.conf", cached_stat) != 0) {
+        return -1;
+    }
+    if (cached_stat->st_dev == old_stat.st_dev
+        && cached_stat->st_ino == old_stat.st_ino
+        && cached_stat->st_size == old_stat.st_size
+        && cached_stat->st_mtime == old_stat.st_mtime) {
+        memcpy(pdns_addr, cached_addr, addrlen);
+        return 0;
+    }
+    return 1;
+}
+
+static int get_dns_addr_resolv_conf(int af, void *pdns_addr, void *cached_addr,
+                                    socklen_t addrlen, uint32_t *scope_id,
+                                    u_int *cached_time)
 {
     char buff[512];
     char buff2[257];
     FILE *f;
     int found = 0;
-    struct in_addr tmp_addr;
-
-    if (dns_addr.s_addr != 0) {
-        struct stat old_stat;
-        if ((curtime - dns_addr_time) < TIMEOUT_DEFAULT) {
-            *pdns_addr = dns_addr;
-            return 0;
-        }
-        old_stat = dns_addr_stat;
-        if (stat("/etc/resolv.conf", &dns_addr_stat) != 0)
-            return -1;
-        if ((dns_addr_stat.st_dev == old_stat.st_dev)
-            && (dns_addr_stat.st_ino == old_stat.st_ino)
-            && (dns_addr_stat.st_size == old_stat.st_size)
-            && (dns_addr_stat.st_mtime == old_stat.st_mtime)) {
-            *pdns_addr = dns_addr;
-            return 0;
-        }
-    }
+    void *tmp_addr = alloca(addrlen);
+    unsigned if_index;
 
     f = fopen("/etc/resolv.conf", "r");
     if (!f)
@@ -217,13 +165,25 @@
 #endif
     while (fgets(buff, 512, f) != NULL) {
         if (sscanf(buff, "nameserver%*[ \t]%256s", buff2) == 1) {
-            if (!inet_aton(buff2, &tmp_addr))
+            char *c = strchr(buff2, '%');
+            if (c) {
+                if_index = if_nametoindex(c + 1);
+                *c = '\0';
+            } else {
+                if_index = 0;
+            }
+
+            if (!inet_pton(af, buff2, tmp_addr)) {
                 continue;
+            }
             /* If it's the first one, set it to dns_addr */
             if (!found) {
-                *pdns_addr = tmp_addr;
-                dns_addr = tmp_addr;
-                dns_addr_time = curtime;
+                memcpy(pdns_addr, tmp_addr, addrlen);
+                memcpy(cached_addr, tmp_addr, addrlen);
+                if (scope_id) {
+                    *scope_id = if_index;
+                }
+                *cached_time = curtime;
             }
 #ifdef DEBUG
             else
@@ -236,8 +196,14 @@
                 break;
             }
 #ifdef DEBUG
-            else
-                fprintf(stderr, "%s", inet_ntoa(tmp_addr));
+            else {
+                char s[INET6_ADDRSTRLEN];
+                char *res = inet_ntop(af, tmp_addr, s, sizeof(s));
+                if (!res) {
+                    res = "(string conversion error)";
+                }
+                fprintf(stderr, "%s", res);
+            }
 #endif
         }
     }
@@ -246,8 +212,141 @@
         return -1;
     return 0;
 }
-#endif  // _WIN32
-#endif  // CONFIG_ANDROID
+
+static int get_dns_addr(struct in_addr *pdns_addr)
+{
+    static struct stat dns_addr_stat;
+
+    if (dns_addr.s_addr != 0) {
+        int ret;
+        ret = get_dns_addr_cached(pdns_addr, &dns_addr, sizeof(dns_addr),
+                                  &dns_addr_stat, &dns_addr_time);
+        if (ret <= 0) {
+            return ret;
+        }
+    }
+    return get_dns_addr_resolv_conf(AF_INET, pdns_addr, &dns_addr,
+                                    sizeof(dns_addr), NULL, &dns_addr_time);
+}
+
+static int get_dns6_addr(struct in6_addr *pdns6_addr, uint32_t *scope_id)
+{
+    static struct stat dns6_addr_stat;
+
+    if (!in6_zero(&dns6_addr)) {
+        int ret;
+        ret = get_dns_addr_cached(pdns6_addr, &dns6_addr, sizeof(dns6_addr),
+                                  &dns6_addr_stat, &dns6_addr_time);
+        if (ret <= 0) {
+            return ret;
+        }
+    }
+    return get_dns_addr_resolv_conf(AF_INET6, pdns6_addr, &dns6_addr,
+                                    sizeof(dns6_addr),
+                                    scope_id, &dns6_addr_time);
+}
+
+#endif
+
+int slirp_translate_guest_dns(Slirp *slirp,
+                              const struct sockaddr_in *guest_ip,
+                              struct sockaddr_in *host_ip)
+{
+    int dns_index = -1;
+
+    if (slirp->host_dns_count > 0) {
+        /* Use custom DNS servers. */
+        uint32_t dns_base = ntohl(slirp->vnameserver_addr.s_addr);
+        uint32_t guest = ntohl(guest_ip->sin_addr.s_addr);
+        dns_index = (int)(guest - dns_base);
+        if (dns_index < 0 || dns_index >= slirp->host_dns_count)
+            return -1;
+
+        host_ip->sin_addr = slirp->host_dns[dns_index];
+        return 0;
+    }
+
+    /* Use system-provided DNS server. */
+    if (guest_ip->sin_addr.s_addr != slirp->vnameserver_addr.s_addr) {
+        return -1;
+    }
+    if (get_dns_addr(&host_ip->sin_addr) < 0) {
+        host_ip->sin_addr = loopback_addr;
+    }
+    return 0;
+}
+
+int slirp_translate_guest_dns6(Slirp *slirp,
+                               const struct sockaddr_in6 *guest_ip6,
+                               struct sockaddr_in6 *host_ip6)
+{
+    uint32_t scope_id;
+
+    if (slirp->host_dns_count > 0) {
+        /* Use custom IPv6 DNS servers. */
+        struct in6_addr dns_base = slirp->vnameserver_addr6;
+        int n, dns_index = -1;
+        for (n = 0; n < slirp->host_dns_count; ++n) {
+            if (in6_equal(&guest_ip6->sin6_addr, &dns_base)) {
+                dns_index = n;
+                break;
+            }
+            /* Increment DNS Ipv6 address */
+            int offset = 15;
+            while (offset >= 0) {
+                dns_base.s6_addr[offset]++;
+                if (dns_base.s6_addr[offset] != 0) {
+                    break;
+                }
+                offset--;
+            }
+        }
+        if (dns_index < 0) {
+            return -1;
+        }
+        host_ip6->sin6_addr = slirp->host_dns6[dns_index];
+        return 0;
+    }
+
+    /* Use system-provided DNS server */
+    if (!in6_equal(&guest_ip6->sin6_addr, &slirp->vnameserver_addr6)) {
+        return -1;
+    }
+    if (get_dns6_addr(&host_ip6->sin6_addr, &scope_id) >= 0) {
+        host_ip6->sin6_scope_id = scope_id;
+    } else {
+        host_ip6->sin6_addr = in6addr_loopback;
+    }
+    return 0;
+}
+
+void slirp_init_custom_dns_servers(Slirp *slirp,
+                                   const struct sockaddr_storage* dns,
+                                   int dns_count)
+{
+    int n = 0;
+    for (; n < dns_count; ++n) {
+        switch (dns[n].ss_family) {
+        case AF_INET:
+            if (slirp->host_dns_count < SLIRP_MAX_DNS_SERVERS) {
+                slirp->host_dns[slirp->host_dns_count++] =
+                        ((const struct sockaddr_in *)&dns[n])->sin_addr;
+            }
+            break;
+
+        case AF_INET6:
+            if (slirp->host_dns6_count < SLIRP_MAX_DNS_SERVERS) {
+                slirp->host_dns6[slirp->host_dns6_count++] =
+                        ((const struct sockaddr_in6 *)&dns[n])->sin6_addr;
+            }
+            break;
+
+        default:
+            /* Ignore this one */
+            ;
+        }
+    }
+}
 
 static void slirp_init_once(void)
 {
@@ -273,21 +372,29 @@
 static void slirp_state_save(QEMUFile *f, void *opaque);
 static int slirp_state_load(QEMUFile *f, void *opaque, int version_id);
 
-Slirp *slirp_init(int restricted, struct in_addr vnetwork,
+Slirp *slirp_init(int restricted, bool in_enabled, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
-                  const char *vhostname, const char *tftp_path,
-                  const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, const char **vdnssearch,
+                  bool in6_enabled,
+                  struct in6_addr vprefix_addr6, uint8_t vprefix_len,
+                  struct in6_addr vhost6, const char *vhostname,
+                  const char *tftp_path, const char *bootfile,
+                  struct in_addr vdhcp_start, struct in_addr vnameserver,
+                  struct in6_addr vnameserver6, const char **vdnssearch,
                   void *opaque)
 {
     Slirp *slirp = g_malloc0(sizeof(Slirp));
 
     slirp_init_once();
 
+    slirp->grand = g_rand_new();
     slirp->restricted = restricted;
 
+    slirp->in_enabled = in_enabled;
+    slirp->in6_enabled = in6_enabled;
+
     if_init(slirp);
     ip_init(slirp);
+    ip6_init(slirp);
 
     /* Initialise mbufs *after* setting the MTU */
     m_init(slirp);
@@ -295,6 +402,9 @@
     slirp->vnetwork_addr = vnetwork;
     slirp->vnetwork_mask = vnetmask;
     slirp->vhost_addr = vhost;
+    slirp->vprefix_addr6 = vprefix_addr6;
+    slirp->vprefix_len = vprefix_len;
+    slirp->vhost_addr6 = vhost6;
     if (vhostname) {
         pstrcpy(slirp->client_hostname, sizeof(slirp->client_hostname),
                 vhostname);
@@ -303,20 +413,18 @@
     slirp->bootp_filename = g_strdup(bootfile);
     slirp->vdhcp_startaddr = vdhcp_start;
     slirp->vnameserver_addr = vnameserver;
+    slirp->vnameserver_addr6 = vnameserver6;
 
     if (vdnssearch) {
         translate_dnssearch(slirp, vdnssearch);
     }
 
+    slirp->host_dns_count = 0;
+    slirp->host_dns6_count = 0;
+
     slirp->opaque = opaque;
 
-#ifdef CONFIG_ANDROID
-    uint32_t special_ip = 0;
-    inet_strtoip(SPECIAL_ADDRESS_IP, &special_ip);
-    special_addr_ip.s_addr = special_ip;
-#endif
-
-    register_savevm(NULL, "slirp", 0, 3,
+    register_savevm(NULL, "slirp", 0, 4,
                     slirp_state_save, slirp_state_load, slirp);
 
     QTAILQ_INSERT_TAIL(&slirp_instances, slirp, entry);
@@ -331,8 +439,11 @@
     unregister_savevm(NULL, "slirp", slirp);
 
     ip_cleanup(slirp);
+    ip6_cleanup(slirp);
     m_cleanup(slirp);
 
+    g_rand_free(slirp->grand);
+
     g_free(slirp->vdnssearch);
     g_free(slirp->tftp_prefix);
     g_free(slirp->bootp_filename);
@@ -598,7 +709,12 @@
                  * test for G_IO_IN below if this succeeds
                  */
                 if (revents & G_IO_PRI) {
-                    sorecvoob(so);
+                    ret = sorecvoob(so);
+                    if (ret < 0) {
+                        /* Socket error might have resulted in the socket being
+                         * removed, do not try to do anything more with it. */
+                        continue;
+                    }
                 }
                 /*
                  * Check sockets for reading
@@ -617,6 +733,11 @@
                     if (ret > 0) {
                         tcp_output(sototcpcb(so));
                     }
+                    if (ret < 0) {
+                        /* Socket error might have resulted in the socket being
+                         * removed, do not try to do anything more with it. */
+                        continue;
+                    }
                 }
 
                 /*
@@ -648,7 +769,8 @@
                         /*
                          * Continue tcp_input
                          */
-                        tcp_input((struct mbuf *)NULL, sizeof(struct ip), so);
+                        tcp_input((struct mbuf *)NULL, sizeof(struct ip), so,
+                                  so->so_ffamily);
                         /* continue; */
                     } else {
                         ret = sowrite(so);
@@ -697,7 +819,8 @@
                         }
 
                     }
-                    tcp_input((struct mbuf *)NULL, sizeof(struct ip), so);
+                    tcp_input((struct mbuf *)NULL, sizeof(struct ip), so,
+                              so->so_ffamily);
                 } /* SS_ISFCONNECTING */
 #endif
             }
@@ -753,13 +876,17 @@
 
 static void arp_input(Slirp *slirp, const uint8_t *pkt, int pkt_len)
 {
-    struct arphdr *ah = (struct arphdr *)(pkt + ETH_HLEN);
-    uint8_t arp_reply[max(ETH_HLEN + sizeof(struct arphdr), 64)];
+    struct slirp_arphdr *ah = (struct slirp_arphdr *)(pkt + ETH_HLEN);
+    uint8_t arp_reply[max(ETH_HLEN + sizeof(struct slirp_arphdr), 64)];
     struct ethhdr *reh = (struct ethhdr *)arp_reply;
-    struct arphdr *rah = (struct arphdr *)(arp_reply + ETH_HLEN);
+    struct slirp_arphdr *rah = (struct slirp_arphdr *)(arp_reply + ETH_HLEN);
     int ar_op;
     struct ex_list *ex_ptr;
 
+    if (!slirp->in_enabled) {
+        return;
+    }
+
     ar_op = ntohs(ah->ar_op);
     switch(ar_op) {
     case ARPOP_REQUEST:
@@ -824,51 +951,53 @@
         arp_input(slirp, pkt, pkt_len);
         break;
     case ETH_P_IP:
+    case ETH_P_IPV6:
         m = m_get(slirp);
         if (!m)
             return;
-        /* Note: we add to align the IP header */
-        if (M_FREEROOM(m) < pkt_len + 2) {
-            m_inc(m, pkt_len + 2);
+        /* Note: we add 2 to align the IP header on 4 bytes,
+         * and add the margin for the tcpiphdr overhead  */
+        if (M_FREEROOM(m) < pkt_len + TCPIPHDR_DELTA + 2) {
+            m_inc(m, pkt_len + TCPIPHDR_DELTA + 2);
         }
-        m->m_len = pkt_len + 2;
-        memcpy(m->m_data + 2, pkt, pkt_len);
+        m->m_len = pkt_len + TCPIPHDR_DELTA + 2;
+        memcpy(m->m_data + TCPIPHDR_DELTA + 2, pkt, pkt_len);
 
-        m->m_data += 2 + ETH_HLEN;
-        m->m_len -= 2 + ETH_HLEN;
+        m->m_data += TCPIPHDR_DELTA + 2 + ETH_HLEN;
+        m->m_len -= TCPIPHDR_DELTA + 2 + ETH_HLEN;
 
-        ip_input(m);
+        if (proto == ETH_P_IP) {
+            ip_input(m);
+        } else if (proto == ETH_P_IPV6) {
+            ip6_input(m);
+        }
         break;
+
     default:
         break;
     }
 }
 
-/* Output the IP packet to the ethernet device. Returns 0 if the packet must be
- * re-queued.
+/* Prepare the IPv4 packet to be sent to the ethernet device. Returns 1 if no
+ * packet should be sent, 0 if the packet must be re-queued, 2 if the packet
+ * is ready to go.
  */
-int if_encap(Slirp *slirp, struct mbuf *ifm)
+static int if_encap4(Slirp *slirp, struct mbuf *ifm, struct ethhdr *eh,
+        uint8_t ethaddr[ETH_ALEN])
 {
-    uint8_t buf[1600];
-    struct ethhdr *eh = (struct ethhdr *)buf;
-    uint8_t ethaddr[ETH_ALEN];
     const struct ip *iph = (const struct ip *)ifm->m_data;
 
-    if (ifm->m_len + ETH_HLEN > sizeof(buf)) {
-        return 1;
-    }
-
     if (iph->ip_dst.s_addr == 0) {
         /* 0.0.0.0 can not be a destination address, something went wrong,
          * avoid making it worse */
         return 1;
     }
     if (!arp_table_search(slirp, iph->ip_dst.s_addr, ethaddr)) {
-        uint8_t arp_req[ETH_HLEN + sizeof(struct arphdr)];
+        uint8_t arp_req[ETH_HLEN + sizeof(struct slirp_arphdr)];
         struct ethhdr *reh = (struct ethhdr *)arp_req;
-        struct arphdr *rah = (struct arphdr *)(arp_req + ETH_HLEN);
+        struct slirp_arphdr *rah = (struct slirp_arphdr *)(arp_req + ETH_HLEN);
 
-        if (!ifm->arp_requested) {
+        if (!ifm->resolution_requested) {
             /* If the client addr is not known, send an ARP request */
             memset(reh->h_dest, 0xff, ETH_ALEN);
             memcpy(reh->h_source, special_ethaddr, ETH_ALEN - 4);
@@ -894,22 +1023,93 @@
             rah->ar_tip = iph->ip_dst.s_addr;
             slirp->client_ipaddr = iph->ip_dst;
             slirp_output(slirp->opaque, arp_req, sizeof(arp_req));
-            ifm->arp_requested = true;
+            ifm->resolution_requested = true;
 
             /* Expire request and drop outgoing packet after 1 second */
             ifm->expiration_date = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + 1000000000ULL;
         }
         return 0;
     } else {
-        memcpy(eh->h_dest, ethaddr, ETH_ALEN);
         memcpy(eh->h_source, special_ethaddr, ETH_ALEN - 4);
         /* XXX: not correct */
         memcpy(&eh->h_source[2], &slirp->vhost_addr, 4);
         eh->h_proto = htons(ETH_P_IP);
-        memcpy(buf + sizeof(struct ethhdr), ifm->m_data, ifm->m_len);
-        slirp_output(slirp->opaque, buf, ifm->m_len + ETH_HLEN);
+
+        /* Send this */
+        return 2;
+    }
+}
+
+/* Prepare the IPv6 packet to be sent to the ethernet device. Returns 1 if no
+ * packet should be sent, 0 if the packet must be re-queued, 2 if the packet
+ * is ready to go.
+ */
+static int if_encap6(Slirp *slirp, struct mbuf *ifm, struct ethhdr *eh,
+        uint8_t ethaddr[ETH_ALEN])
+{
+    const struct ip6 *ip6h = mtod(ifm, const struct ip6 *);
+    if (!ndp_table_search(slirp, ip6h->ip_dst, ethaddr)) {
+        if (!ifm->resolution_requested) {
+            ndp_send_ns(slirp, ip6h->ip_dst);
+            ifm->resolution_requested = true;
+            ifm->expiration_date =
+                qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + 1000000000ULL;
+        }
+        return 0;
+    } else {
+        eh->h_proto = htons(ETH_P_IPV6);
+        in6_compute_ethaddr(ip6h->ip_src, eh->h_source);
+
+        /* Send this */
+        return 2;
+    }
+}
+
+/* Output the IP packet to the ethernet device. Returns 0 if the packet must be
+ * re-queued.
+ */
+int if_encap(Slirp *slirp, struct mbuf *ifm)
+{
+    uint8_t buf[1600];
+    struct ethhdr *eh = (struct ethhdr *)buf;
+    uint8_t ethaddr[ETH_ALEN];
+    const struct ip *iph = (const struct ip *)ifm->m_data;
+    int ret;
+
+    if (ifm->m_len + ETH_HLEN > sizeof(buf)) {
         return 1;
     }
+
+    switch (iph->ip_v) {
+    case IPVERSION:
+        ret = if_encap4(slirp, ifm, eh, ethaddr);
+        if (ret < 2) {
+            return ret;
+        }
+        break;
+
+    case IP6VERSION:
+        ret = if_encap6(slirp, ifm, eh, ethaddr);
+        if (ret < 2) {
+            return ret;
+        }
+        break;
+
+    default:
+        g_assert_not_reached();
+        break;
+    }
+
+    memcpy(eh->h_dest, ethaddr, ETH_ALEN);
+    DEBUG_ARGS((dfd, " src = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                eh->h_source[0], eh->h_source[1], eh->h_source[2],
+                eh->h_source[3], eh->h_source[4], eh->h_source[5]));
+    DEBUG_ARGS((dfd, " dst = %02x:%02x:%02x:%02x:%02x:%02x\n",
+                eh->h_dest[0], eh->h_dest[1], eh->h_dest[2],
+                eh->h_dest[3], eh->h_dest[4], eh->h_dest[5]));
+    memcpy(buf + sizeof(struct ethhdr), ifm->m_data, ifm->m_len);
+    slirp_output(slirp->opaque, buf, ifm->m_len + ETH_HLEN);
+    return 1;
 }
 
 /* Drop host forwarding rule, return 0 if found. */
@@ -1093,10 +1293,26 @@
 static void slirp_socket_save(QEMUFile *f, struct socket *so)
 {
     qemu_put_be32(f, so->so_urgc);
-    qemu_put_be32(f, so->so_faddr.s_addr);
-    qemu_put_be32(f, so->so_laddr.s_addr);
-    qemu_put_be16(f, so->so_fport);
-    qemu_put_be16(f, so->so_lport);
+    qemu_put_be16(f, so->so_ffamily);
+    switch (so->so_ffamily) {
+    case AF_INET:
+        qemu_put_be32(f, so->so_faddr.s_addr);
+        qemu_put_be16(f, so->so_fport);
+        break;
+    default:
+        error_report("so_ffamily unknown, unable to save so_faddr and"
+                     " so_fport");
+    }
+    qemu_put_be16(f, so->so_lfamily);
+    switch (so->so_lfamily) {
+    case AF_INET:
+        qemu_put_be32(f, so->so_laddr.s_addr);
+        qemu_put_be16(f, so->so_lport);
+        break;
+    default:
+        error_report("so_ffamily unknown, unable to save so_laddr and"
+                     " so_lport");
+    }
     qemu_put_byte(f, so->so_iptos);
     qemu_put_byte(f, so->so_emu);
     qemu_put_byte(f, so->so_type);
@@ -1210,16 +1426,40 @@
     return 0;
 }
 
-static int slirp_socket_load(QEMUFile *f, struct socket *so)
+static int slirp_socket_load(QEMUFile *f, struct socket *so, int version_id)
 {
     if (tcp_attach(so) < 0)
         return -ENOMEM;
 
     so->so_urgc = qemu_get_be32(f);
-    so->so_faddr.s_addr = qemu_get_be32(f);
-    so->so_laddr.s_addr = qemu_get_be32(f);
-    so->so_fport = qemu_get_be16(f);
-    so->so_lport = qemu_get_be16(f);
+    if (version_id <= 3) {
+        so->so_ffamily = AF_INET;
+        so->so_faddr.s_addr = qemu_get_be32(f);
+        so->so_laddr.s_addr = qemu_get_be32(f);
+        so->so_fport = qemu_get_be16(f);
+        so->so_lport = qemu_get_be16(f);
+    } else {
+        so->so_ffamily = qemu_get_be16(f);
+        switch (so->so_ffamily) {
+        case AF_INET:
+            so->so_faddr.s_addr = qemu_get_be32(f);
+            so->so_fport = qemu_get_be16(f);
+            break;
+        default:
+            error_report(
+                "so_ffamily unknown, unable to restore so_faddr and so_lport");
+        }
+        so->so_lfamily = qemu_get_be16(f);
+        switch (so->so_lfamily) {
+        case AF_INET:
+            so->so_laddr.s_addr = qemu_get_be32(f);
+            so->so_lport = qemu_get_be16(f);
+            break;
+        default:
+            error_report(
+                "so_ffamily unknown, unable to restore so_laddr and so_lport");
+        }
+    }
     so->so_iptos = qemu_get_byte(f);
     so->so_emu = qemu_get_byte(f);
     so->so_type = qemu_get_byte(f);
@@ -1255,7 +1495,7 @@
         if (!so)
             return -ENOMEM;
 
-        ret = slirp_socket_load(f, so);
+        ret = slirp_socket_load(f, so, version_id);
 
         if (ret < 0)
             return ret;
diff --git a/slirp/slirp.h b/slirp/slirp.h
index 6589d7e..7e890b4 100644
--- a/slirp/slirp.h
+++ b/slirp/slirp.h
@@ -1,11 +1,10 @@
-#ifndef __COMMON_H__
-#define __COMMON_H__
+#ifndef SLIRP_H
+#define SLIRP_H
 
-#include "config-host.h"
+#include "qemu/host-utils.h"
 #include "slirp_config.h"
 
 #ifdef _WIN32
-# include <inttypes.h>
 
 typedef char *caddr_t;
 
@@ -16,52 +15,15 @@
 # include <iphlpapi.h>
 
 #else
-# define ioctlsocket ioctl
-# define closesocket(s) close(s)
 # if !defined(__HAIKU__)
 #  define O_BINARY 0
 # endif
 #endif
 
-#include <sys/types.h>
 #ifdef HAVE_SYS_BITYPES_H
 # include <sys/bitypes.h>
 #endif
 
-#include <sys/time.h>
-
-#ifdef HAVE_UNISTD_H
-# include <unistd.h>
-#endif
-
-#ifdef HAVE_STDLIB_H
-# include <stdlib.h>
-#endif
-
-#include <stdio.h>
-#include <errno.h>
-
-#ifndef HAVE_MEMMOVE
-#define memmove(x, y, z) bcopy(y, x, z)
-#endif
-
-#if TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# ifdef HAVE_SYS_TIME_H
-#  include <sys/time.h>
-# else
-#  include <time.h>
-# endif
-#endif
-
-#ifdef HAVE_STRING_H
-# include <string.h>
-#else
-# include <strings.h>
-#endif
-
 #ifndef _WIN32
 #include <sys/uio.h>
 #endif
@@ -71,22 +33,9 @@
 #include <arpa/inet.h>
 #endif
 
-/* Systems lacking strdup() definition in <string.h>. */
-#if defined(ultrix)
-char *strdup(const char *);
-#endif
-
-/* Systems lacking malloc() definition in <stdlib.h>. */
-#if defined(ultrix) || defined(hcx)
-void *malloc(size_t arg);
-void free(void *ptr);
-#endif
-
-#include <fcntl.h>
 #ifndef NO_UNIX_SOCKETS
 #include <sys/un.h>
 #endif
-#include <signal.h>
 #ifdef HAVE_SYS_SIGNAL_H
 # include <sys/signal.h>
 #endif
@@ -110,60 +59,43 @@
 # include <sys/filio.h>
 #endif
 
-#ifdef USE_PPP
-#include <ppp/slirppp.h>
-#endif
-
-#ifdef __STDC__
-#include <stdarg.h>
-#else
-#include <varargs.h>
-#endif
-
-#include <sys/stat.h>
-
 /* Avoid conflicting with the libc insque() and remque(), which
    have different prototypes. */
 #define insque slirp_insque
 #define remque slirp_remque
+#define quehead slirp_quehead
 
 #ifdef HAVE_SYS_STROPTS_H
 #include <sys/stropts.h>
 #endif
 
+
 #include "debug.h"
 
 #include "qemu/queue.h"
 #include "qemu/sockets.h"
+#include "net/eth.h"
 
 #include "libslirp.h"
 #include "ip.h"
+#include "ip6.h"
 #include "tcp.h"
 #include "tcp_timer.h"
 #include "tcp_var.h"
 #include "tcpip.h"
 #include "udp.h"
 #include "ip_icmp.h"
+#include "ip6_icmp.h"
 #include "mbuf.h"
 #include "sbuf.h"
 #include "socket.h"
 #include "if.h"
 #include "main.h"
 #include "misc.h"
-#ifdef USE_PPP
-#include "ppp/pppd.h"
-#include "ppp/ppp.h"
-#endif
 
 #include "bootp.h"
 #include "tftp.h"
 
-#define ETH_ALEN 6
-#define ETH_HLEN 14
-
-#define ETH_P_IP  0x0800        /* Internet Protocol packet  */
-#define ETH_P_ARP 0x0806        /* Address Resolution packet */
-
 #define ARPOP_REQUEST 1         /* ARP request */
 #define ARPOP_REPLY   2         /* ARP reply   */
 
@@ -173,7 +105,7 @@
     unsigned short h_proto;            /* packet type ID field */
 };
 
-struct arphdr {
+struct slirp_arphdr {
     unsigned short ar_hrd;      /* format of hardware address */
     unsigned short ar_pro;      /* format of protocol address */
     unsigned char  ar_hln;      /* length of hardware address */
@@ -192,7 +124,7 @@
 #define ARP_TABLE_SIZE 16
 
 typedef struct ArpTable {
-    struct arphdr table[ARP_TABLE_SIZE];
+    struct slirp_arphdr table[ARP_TABLE_SIZE];
     int next_victim;
 } ArpTable;
 
@@ -201,32 +133,64 @@
 bool arp_table_search(Slirp *slirp, uint32_t ip_addr,
                       uint8_t out_ethaddr[ETH_ALEN]);
 
+struct ndpentry {
+    unsigned char   eth_addr[ETH_ALEN];     /* sender hardware address */
+    struct in6_addr ip_addr;                /* sender IP address       */
+} QEMU_PACKED;
+
+#define NDP_TABLE_SIZE 16
+
+typedef struct NdpTable {
+    struct ndpentry table[NDP_TABLE_SIZE];
+    int next_victim;
+} NdpTable;
+
+void ndp_table_add(Slirp *slirp, struct in6_addr ip_addr,
+                   uint8_t ethaddr[ETH_ALEN]);
+bool ndp_table_search(Slirp *slirp, struct in6_addr ip_addr,
+                      uint8_t out_ethaddr[ETH_ALEN]);
+
+#define SLIRP_MAX_DNS_SERVERS 4
+
 struct Slirp {
     QTAILQ_ENTRY(Slirp) entry;
     u_int time_fasttimo;
     u_int last_slowtimo;
     bool do_slowtimo;
 
+    bool in_enabled, in6_enabled;
+
     /* virtual network configuration */
     struct in_addr vnetwork_addr;
     struct in_addr vnetwork_mask;
     struct in_addr vhost_addr;
+    struct in6_addr vprefix_addr6;
+    uint8_t vprefix_len;
+    struct in6_addr vhost_addr6;
     struct in_addr vdhcp_startaddr;
     struct in_addr vnameserver_addr;
+    struct in6_addr vnameserver_addr6;
 
     struct in_addr client_ipaddr;
     char client_hostname[33];
 
+    struct in_addr host_dns[SLIRP_MAX_DNS_SERVERS];
+    int host_dns_count;
+
+    struct in6_addr host_dns6[SLIRP_MAX_DNS_SERVERS];
+    int host_dns6_count;
+
     int restricted;
     struct ex_list *exec_list;
 
     /* mbuf states */
-    struct mbuf m_freelist, m_usedlist;
+    struct quehead m_freelist;
+    struct quehead m_usedlist;
     int mbuf_alloced;
 
     /* if states */
-    struct mbuf if_fastq;   /* fast queue (for interactive data) */
-    struct mbuf if_batchq;  /* queue for non-interactive data */
+    struct quehead if_fastq;   /* fast queue (for interactive data) */
+    struct quehead if_batchq;  /* queue for non-interactive data */
     struct mbuf *next_m;    /* pointer to next mbuf to output */
     bool if_start_busy;     /* avoid if_start recursion */
 
@@ -259,6 +223,10 @@
     struct tftp_session tftp_sessions[TFTP_SESSIONS_MAX];
 
     ArpTable arp_table;
+    NdpTable ndp_table;
+
+    GRand *grand;
+    QEMUTimer *ra_timer;
 
     void *opaque;
 };
@@ -269,30 +237,12 @@
 #define NULL (void *)0
 #endif
 
-#ifndef FULL_BOLT
 void if_start(Slirp *);
-#else
-void if_start(struct ttys *);
-#endif
-
-#ifndef HAVE_STRERROR
- char *strerror(int error);
-#endif
-
-#ifndef HAVE_INDEX
- char *index(const char *, int);
-#endif
-
-#ifndef HAVE_GETHOSTID
- long gethostid(void);
-#endif
 
 #ifndef _WIN32
 #include <netdb.h>
 #endif
 
-#define DEFAULT_BAUD 115200
-
 #define SO_OPTIONS DO_KEEPALIVE
 #define TCP_MAXIDLE (TCPTV_KEEPCNT * TCPTV_KEEPINTVL)
 
@@ -301,6 +251,7 @@
 
 /* cksum.c */
 int cksum(struct mbuf *m, int len);
+int ip6_cksum(struct mbuf *m);
 
 /* if.c */
 void if_init(Slirp *);
@@ -316,8 +267,16 @@
 /* ip_output.c */
 int ip_output(struct socket *, struct mbuf *);
 
+/* ip6_input.c */
+void ip6_init(Slirp *);
+void ip6_cleanup(Slirp *);
+void ip6_input(struct mbuf *);
+
+/* ip6_output */
+int ip6_output(struct socket *, struct mbuf *, int fast);
+
 /* tcp_input.c */
-void tcp_input(register struct mbuf *, int, struct socket *);
+void tcp_input(register struct mbuf *, int, struct socket *, unsigned short af);
 int tcp_mss(register struct tcpcb *, u_int);
 
 /* tcp_output.c */
@@ -328,11 +287,12 @@
 void tcp_init(Slirp *);
 void tcp_cleanup(Slirp *);
 void tcp_template(struct tcpcb *);
-void tcp_respond(struct tcpcb *, register struct tcpiphdr *, register struct mbuf *, tcp_seq, tcp_seq, int);
+void tcp_respond(struct tcpcb *, register struct tcpiphdr *,
+        register struct mbuf *, tcp_seq, tcp_seq, int, unsigned short);
 struct tcpcb * tcp_newtcpcb(struct socket *);
 struct tcpcb * tcp_close(register struct tcpcb *);
 void tcp_sockclosed(struct tcpcb *);
-int tcp_fconnect(struct socket *);
+int tcp_fconnect(struct socket *, unsigned short af);
 void tcp_connect(struct socket *);
 int tcp_attach(struct socket *);
 uint8_t tcp_tos(struct socket *);
@@ -340,22 +300,9 @@
 int tcp_ctl(struct socket *);
 struct tcpcb *tcp_drop(struct tcpcb *tp, int err);
 
-#ifdef USE_PPP
-#define MIN_MRU MINMRU
-#define MAX_MRU MAXMRU
-#else
-#define MIN_MRU 128
-#define MAX_MRU 16384
-#endif
-
 #ifndef _WIN32
 #define min(x,y) ((x) < (y) ? (x) : (y))
 #define max(x,y) ((x) > (y) ? (x) : (y))
 #endif
 
-#ifdef _WIN32
-#undef errno
-#define errno (WSAGetLastError())
-#endif
-
 #endif
diff --git a/slirp/slirp_config.h b/slirp/slirp_config.h
index 896d802..c59f655 100644
--- a/slirp/slirp_config.h
+++ b/slirp/slirp_config.h
@@ -9,19 +9,6 @@
 /* Define to 1 if you want KEEPALIVE timers */
 #define DO_KEEPALIVE 0
 
-/* Define to MAX interfaces you expect to use at once */
-/* MAX_INTERFACES determines the max. TOTAL number of interfaces (SLIP and PPP) */
-/* MAX_PPP_INTERFACES determines max. number of PPP interfaces */
-#define MAX_INTERFACES 1
-#define MAX_PPP_INTERFACES 1
-
-/* Define if you want slirp's socket in /tmp */
-/* XXXXXX Do this in ./configure */
-#undef USE_TMPSOCKET
-
-/* Define if you want slirp to use cfsetXspeed() on the terminal */
-#undef DO_CFSETSPEED
-
 /* Define this if you want slirp to write to the tty as fast as it can */
 /* This should only be set if you are using load-balancing, slirp does a */
 /* pretty good job on single modems already, and seting this will make */
@@ -29,34 +16,12 @@
 /* XXXXX Talk about having fast modem as unit 0 */
 #undef FULL_BOLT
 
-/*
- * Define if you want slirp to use less CPU
- * You will notice a small lag in interactive sessions, but it's not that bad
- * Things like Netscape/ftp/etc. are completely unaffected
- * This is mainly for sysadmins who have many slirp users
- */
-#undef USE_LOWCPU
-
-/* Define this if your compiler doesn't like prototypes */
-#ifndef __STDC__
-#define NO_PROTOTYPES
-#endif
-
 /*********************************************************/
 /*
  * Autoconf defined configuration options
  * You shouldn't need to touch any of these
  */
 
-/* Ignore this */
-#undef DUMMY_PPP
-
-/* Define if you have unistd.h */
-#define HAVE_UNISTD_H
-
-/* Define if you have stdlib.h */
-#define HAVE_STDLIB_H
-
 /* Define if you have sys/ioctl.h */
 #undef HAVE_SYS_IOCTL_H
 #ifndef _WIN32
@@ -69,13 +34,6 @@
 #define HAVE_SYS_FILIO_H
 #endif
 
-/* Define if you have strerror */
-#define HAVE_STRERROR
-
-/* Define according to how time.h should be included */
-#define TIME_WITH_SYS_TIME 0
-#undef HAVE_SYS_TIME_H
-
 /* Define if you have sys/bitypes.h */
 #undef HAVE_SYS_BITYPES_H
 
@@ -100,9 +58,6 @@
 #define HAVE_SYS_SELECT_H
 #endif
 
-/* Define if you have strings.h */
-#define HAVE_STRING_H
-
 /* Define if you have arpa/inet.h */
 #undef HAVE_ARPA_INET_H
 #ifndef _WIN32
@@ -115,71 +70,17 @@
 /* Define if you have sys/stropts.h */
 #undef HAVE_SYS_STROPTS_H
 
-/* Define to whatever your compiler thinks inline should be */
-//#define inline inline
-
-/* Define to whatever your compiler thinks const should be */
-//#define const const
-
-/* Define if your compiler doesn't like prototypes */
-#undef NO_PROTOTYPES
-
-/* Define to sizeof(char) */
-#define SIZEOF_CHAR 1
-
-/* Define to sizeof(short) */
-#define SIZEOF_SHORT 2
-
-/* Define to sizeof(int) */
-#define SIZEOF_INT 4
-
 /* Define to sizeof(char *) */
 #define SIZEOF_CHAR_P (HOST_LONG_BITS / 8)
 
-/* Define if you have random() */
-#undef HAVE_RANDOM
-
-/* Define if you have srandom() */
-#undef HAVE_SRANDOM
-
 /* Define if you have inet_aton */
 #undef HAVE_INET_ATON
 #ifndef _WIN32
 #define HAVE_INET_ATON
 #endif
 
-/* Define if you have setenv */
-#undef HAVE_SETENV
-
-/* Define if you have index() */
-#define HAVE_INDEX
-
-/* Define if you have bcmp() */
-#undef HAVE_BCMP
-
-/* Define if you have drand48 */
-#undef HAVE_DRAND48
-
-/* Define if you have memmove */
-#define HAVE_MEMMOVE
-
-/* Define if you have gethostid */
-#define HAVE_GETHOSTID
-
 /* Define if you DON'T have unix-domain sockets */
 #undef NO_UNIX_SOCKETS
 #ifdef _WIN32
 #define NO_UNIX_SOCKETS
 #endif
-
-/* Define if you have revoke() */
-#undef HAVE_REVOKE
-
-/* Define if you have the sysv method of opening pty's (/dev/ptmx, etc.) */
-#undef HAVE_GRANTPT
-
-/* Define if you have fchmod */
-#undef HAVE_FCHMOD
-
-/* Define if you have <sys/type32.h> */
-#undef HAVE_SYS_TYPES32_H
diff --git a/slirp/socket.c b/slirp/socket.c
index 109feed..180099a 100644
--- a/slirp/socket.c
+++ b/slirp/socket.c
@@ -5,37 +5,37 @@
  * terms and conditions of the copyright.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include <slirp.h>
+#include "slirp.h"
 #include "ip_icmp.h"
 #ifdef __sun__
 #include <sys/filio.h>
 #endif
-#ifdef CONFIG_ANDROID
-#include "android/proxy/proxy_common.h"
-#endif  // CONFIG_ANDROID
 
 static void sofcantrcvmore(struct socket *so);
 static void sofcantsendmore(struct socket *so);
 
-struct socket *
-solookup(struct socket *head, struct in_addr laddr, u_int lport,
-         struct in_addr faddr, u_int fport)
+struct socket *solookup(struct socket **last, struct socket *head,
+        struct sockaddr_storage *lhost, struct sockaddr_storage *fhost)
 {
-	struct socket *so;
+    struct socket *so = *last;
 
-	for (so = head->so_next; so != head; so = so->so_next) {
-		if (so->so_lport == lport &&
-		    so->so_laddr.s_addr == laddr.s_addr &&
-		    so->so_faddr.s_addr == faddr.s_addr &&
-		    so->so_fport == fport)
-		   break;
-	}
+    /* Optimisation */
+    if (so != head && sockaddr_equal(&(so->lhost.ss), lhost)
+            && (!fhost || sockaddr_equal(&so->fhost.ss, fhost))) {
+        return so;
+    }
 
-	if (so == head)
-	   return (struct socket *)NULL;
-	return so;
+    for (so = head->so_next; so != head; so = so->so_next) {
+        if (sockaddr_equal(&(so->lhost.ss), lhost)
+                && (!fhost || sockaddr_equal(&so->fhost.ss, fhost))) {
+            *last = so;
+            return so;
+        }
+    }
 
+    return (struct socket *)NULL;
 }
 
 /*
@@ -67,11 +67,6 @@
 {
   Slirp *slirp = so->slirp;
 
-#ifdef CONFIG_ANDROID
-  if (so->so_state & SS_PROXIFIED)
-    proxy_manager_del(so);
-#endif
-
   if (so->so_emu==EMU_RSH && so->extra) {
 	sofree(so->extra);
 	so->extra=NULL;
@@ -99,7 +94,7 @@
 	int mss = so->so_tcpcb->t_maxseg;
 
 	DEBUG_CALL("sopreprbuf");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	if (len <= 0)
 		return 0;
@@ -163,7 +158,7 @@
 	struct iovec iov[2];
 
 	DEBUG_CALL("soread");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * No need to check if there's enough room to read.
@@ -181,9 +176,24 @@
 		if (nn < 0 && (errno == EINTR || errno == EAGAIN))
 			return 0;
 		else {
+			int err;
+			socklen_t slen = sizeof err;
+
+			err = errno;
+			if (nn == 0) {
+				getsockopt(so->s, SOL_SOCKET, SO_ERROR,
+					   &err, &slen);
+			}
+
 			DEBUG_MISC((dfd, " --- soread() disconnected, nn = %d, errno = %d-%s\n", nn, errno,strerror(errno)));
 			sofcantrcvmore(so);
-			tcp_sockclosed(sototcpcb(so));
+
+			if (err == ECONNRESET || err == ECONNREFUSED
+			    || err == ENOTCONN || err == EPIPE) {
+				tcp_drop(sototcpcb(so), err);
+			} else {
+				tcp_sockclosed(sototcpcb(so));
+			}
 			return -1;
 		}
 	}
@@ -196,7 +206,7 @@
 	 * We don't test for <= 0 this time, because there legitimately
 	 * might not be any more data (since the socket is non-blocking),
 	 * a close will be detected on next iteration.
-	 * A return of -1 wont (shouldn't) happen, since it didn't happen above
+	 * A return of -1 won't (shouldn't) happen, since it didn't happen above
 	 */
 	if (n == 2 && nn == iov[0].iov_len) {
             int ret;
@@ -223,7 +233,7 @@
 	struct iovec iov[2];
 
 	DEBUG_CALL("soreadbuf");
-	DEBUG_ARG("so = %lx", (long )so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * No need to check if there's enough room to read.
@@ -265,13 +275,14 @@
  * so when OOB data arrives, we soread() it and everything
  * in the send buffer is sent as urgent data
  */
-void
+int
 sorecvoob(struct socket *so)
 {
 	struct tcpcb *tp = sototcpcb(so);
+	int ret;
 
 	DEBUG_CALL("sorecvoob");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	/*
 	 * We take a guess at how much urgent data has arrived.
@@ -281,11 +292,15 @@
 	 * urgent data, or the read() doesn't return all the
 	 * urgent data.
 	 */
-	soread(so);
-	tp->snd_up = tp->snd_una + so->so_snd.sb_cc;
-	tp->t_force = 1;
-	tcp_output(tp);
-	tp->t_force = 0;
+	ret = soread(so);
+	if (ret > 0) {
+	    tp->snd_up = tp->snd_una + so->so_snd.sb_cc;
+	    tp->t_force = 1;
+	    tcp_output(tp);
+	    tp->t_force = 0;
+	}
+
+	return ret;
 }
 
 /*
@@ -301,7 +316,7 @@
 	int n, len;
 
 	DEBUG_CALL("sosendoob");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 	DEBUG_ARG("sb->sb_cc = %d", sb->sb_cc);
 
 	if (so->so_urgc > 2048)
@@ -359,7 +374,7 @@
 	struct iovec iov[2];
 
 	DEBUG_CALL("sowrite");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	if (so->so_urgc) {
 		sosendoob(so);
@@ -445,11 +460,12 @@
 void
 sorecvfrom(struct socket *so)
 {
-	struct sockaddr_in addr;
-	socklen_t addrlen = sizeof(struct sockaddr_in);
+	struct sockaddr_storage addr;
+	struct sockaddr_storage saddr, daddr;
+	socklen_t addrlen = sizeof(struct sockaddr_storage);
 
 	DEBUG_CALL("sorecvfrom");
-	DEBUG_ARG("so = %lx", (long)so);
+	DEBUG_ARG("so = %p", so);
 
 	if (so->so_type == IPPROTO_ICMP) {   /* This is a "ping" reply */
 	  char buff[256];
@@ -467,7 +483,7 @@
 
 	    DEBUG_MISC((dfd," udp icmp rx errno = %d-%s\n",
 			errno,strerror(errno)));
-	    icmp_error(so->so_m, ICMP_UNREACH,code, 0,strerror(errno));
+	    icmp_send_error(so->so_m, ICMP_UNREACH, code, 0, strerror(errno));
 	  } else {
 	    icmp_reflect(so->so_m);
             so->so_m = NULL; /* Don't m_free() it again! */
@@ -487,7 +503,18 @@
 	  if (!m) {
 	      return;
 	  }
-	  m->m_data += IF_MAXLINKHDR;
+	  switch (so->so_ffamily) {
+	  case AF_INET:
+	      m->m_data += IF_MAXLINKHDR + sizeof(struct udpiphdr);
+	      break;
+	  case AF_INET6:
+	      m->m_data += IF_MAXLINKHDR + sizeof(struct ip6)
+	                                 + sizeof(struct udphdr);
+	      break;
+	  default:
+	      g_assert_not_reached();
+	      break;
+	  }
 
 	  /*
 	   * XXX Shouldn't FIONREAD packets destined for port 53,
@@ -509,13 +536,37 @@
 	  DEBUG_MISC((dfd, " did recvfrom %d, errno = %d-%s\n",
 		      m->m_len, errno,strerror(errno)));
 	  if(m->m_len<0) {
-	    u_char code=ICMP_UNREACH_PORT;
+	    /* Report error as ICMP */
+	    switch (so->so_lfamily) {
+	    uint8_t code;
+	    case AF_INET:
+	      code = ICMP_UNREACH_PORT;
 
-	    if(errno == EHOSTUNREACH) code=ICMP_UNREACH_HOST;
-	    else if(errno == ENETUNREACH) code=ICMP_UNREACH_NET;
+	      if (errno == EHOSTUNREACH) {
+		code = ICMP_UNREACH_HOST;
+	      } else if (errno == ENETUNREACH) {
+		code = ICMP_UNREACH_NET;
+	      }
 
-	    DEBUG_MISC((dfd," rx error, tx icmp ICMP_UNREACH:%i\n", code));
-	    icmp_error(so->so_m, ICMP_UNREACH,code, 0,strerror(errno));
+	      DEBUG_MISC((dfd, " rx error, tx icmp ICMP_UNREACH:%i\n", code));
+	      icmp_send_error(so->so_m, ICMP_UNREACH, code, 0, strerror(errno));
+	      break;
+	    case AF_INET6:
+	      code = ICMP6_UNREACH_PORT;
+
+	      if (errno == EHOSTUNREACH) {
+		code = ICMP6_UNREACH_ADDRESS;
+	      } else if (errno == ENETUNREACH) {
+		code = ICMP6_UNREACH_NO_ROUTE;
+	      }
+
+	      DEBUG_MISC((dfd, " rx error, tx icmp6 ICMP_UNREACH:%i\n", code));
+	      icmp6_send_error(so->so_m, ICMP6_UNREACH, code);
+	      break;
+	    default:
+	      g_assert_not_reached();
+	      break;
+	    }
 	    m_free(m);
 	  } else {
 	  /*
@@ -533,9 +584,26 @@
 
 	    /*
 	     * If this packet was destined for CTL_ADDR,
-	     * make it look like that's where it came from, done by udp_output
+	     * make it look like that's where it came from
 	     */
-	    udp_output(so, m, &addr);
+	    saddr = addr;
+	    sotranslate_in(so, &saddr);
+	    daddr = so->lhost.ss;
+
+	    switch (so->so_ffamily) {
+	    case AF_INET:
+	        udp_output(so, m, (struct sockaddr_in *) &saddr,
+	                   (struct sockaddr_in *) &daddr,
+	                   so->so_iptos);
+	        break;
+	    case AF_INET6:
+	        udp6_output(so, m, (struct sockaddr_in6 *) &saddr,
+	                    (struct sockaddr_in6 *) &daddr);
+	        break;
+	    default:
+	        g_assert_not_reached();
+	        break;
+	    }
 	  } /* rx error */
 	} /* if ping packet */
 }
@@ -546,33 +614,20 @@
 int
 sosendto(struct socket *so, struct mbuf *m)
 {
-	Slirp *slirp = so->slirp;
 	int ret;
-	struct sockaddr_in addr;
+	struct sockaddr_storage addr;
 
 	DEBUG_CALL("sosendto");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 
-        addr.sin_family = AF_INET;
-	if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-	    slirp->vnetwork_addr.s_addr) {
-	  /* It's an alias */
-	  if (is_dns_addr(slirp, &so->so_faddr)) {
-	    if (get_dns_addr(&so->so_faddr, &addr.sin_addr) < 0)
-	      addr.sin_addr = loopback_addr;
-	  } else {
-	    addr.sin_addr = loopback_addr;
-	  }
-	} else
-	  addr.sin_addr = so->so_faddr;
-	addr.sin_port = so->so_fport;
-
-	DEBUG_MISC((dfd, " sendto()ing, addr.sin_port=%d, addr.sin_addr.s_addr=%.16s\n", ntohs(addr.sin_port), inet_ntoa(addr.sin_addr)));
+	addr = so->fhost.ss;
+	DEBUG_CALL(" sendto()ing)");
+	sotranslate_out(so, &addr);
 
 	/* Don't care what port we get */
 	ret = sendto(so->s, m->m_data, m->m_len, 0,
-		     (struct sockaddr *)&addr, sizeof (struct sockaddr));
+		     (struct sockaddr *)&addr, sockaddr_size(&addr));
 	if (ret < 0)
 		return -1;
 
@@ -627,6 +682,7 @@
 
 	so->so_state &= SS_PERSISTENT_MASK;
 	so->so_state |= (SS_FACCEPTCONN | flags);
+	so->so_lfamily = AF_INET;
 	so->so_lport = lport; /* Kept in network format */
 	so->so_laddr.s_addr = laddr; /* Ditto */
 
@@ -653,6 +709,7 @@
 	qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(int));
 
 	getsockname(s,(struct sockaddr *)&addr,&addrlen);
+	so->so_ffamily = AF_INET;
 	so->so_fport = addr.sin_port;
 	if (addr.sin_addr.s_addr == 0 || addr.sin_addr.s_addr == loopback_addr.s_addr)
 	   so->so_faddr = slirp->vhost_addr;
@@ -726,3 +783,104 @@
 	else
 		sofcantsendmore(so);
 }
+
+/*
+ * Translate addr in host addr when it is a virtual address
+ */
+void sotranslate_out(struct socket *so, struct sockaddr_storage *addr)
+{
+    Slirp *slirp = so->slirp;
+    struct sockaddr_in *sin = (struct sockaddr_in *)addr;
+    struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
+
+    switch (addr->ss_family) {
+    case AF_INET:
+        if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
+                slirp->vnetwork_addr.s_addr) {
+            if (slirp_translate_guest_dns(slirp, &so->fhost.sin, sin) < 0) {
+                sin->sin_addr = loopback_addr;
+            }
+        }
+
+        DEBUG_MISC((dfd, " addr.sin_port=%d, "
+            "addr.sin_addr.s_addr=%.16s\n",
+            ntohs(sin->sin_port), inet_ntoa(sin->sin_addr)));
+        break;
+
+    case AF_INET6:
+        if (in6_equal_net(&so->so_faddr6, &slirp->vprefix_addr6,
+                    slirp->vprefix_len)) {
+            if (slirp_translate_guest_dns6(slirp, &so->fhost.sin6, sin6) < 0) {
+                sin6->sin6_addr = in6addr_loopback;
+            }
+        }
+        break;
+
+    default:
+        break;
+    }
+}
+
+void sotranslate_in(struct socket *so, struct sockaddr_storage *addr)
+{
+    Slirp *slirp = so->slirp;
+    struct sockaddr_in *sin = (struct sockaddr_in *)addr;
+    struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
+
+    switch (addr->ss_family) {
+    case AF_INET:
+        if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
+            slirp->vnetwork_addr.s_addr) {
+            uint32_t inv_mask = ~slirp->vnetwork_mask.s_addr;
+
+            if ((so->so_faddr.s_addr & inv_mask) == inv_mask) {
+                sin->sin_addr = slirp->vhost_addr;
+            } else if (sin->sin_addr.s_addr == loopback_addr.s_addr ||
+                       so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
+                sin->sin_addr = so->so_faddr;
+            }
+        }
+        break;
+
+    case AF_INET6:
+        if (in6_equal_net(&so->so_faddr6, &slirp->vprefix_addr6,
+                    slirp->vprefix_len)) {
+            if (in6_equal(&sin6->sin6_addr, &in6addr_loopback)
+                    || !in6_equal(&so->so_faddr6, &slirp->vhost_addr6)) {
+                sin6->sin6_addr = so->so_faddr6;
+            }
+        }
+        break;
+
+    default:
+        break;
+    }
+}
+
+/*
+ * Translate connections from localhost to the real hostname
+ */
+void sotranslate_accept(struct socket *so)
+{
+    Slirp *slirp = so->slirp;
+
+    switch (so->so_ffamily) {
+    case AF_INET:
+        if (so->so_faddr.s_addr == INADDR_ANY ||
+            (so->so_faddr.s_addr & loopback_mask) ==
+            (loopback_addr.s_addr & loopback_mask)) {
+           so->so_faddr = slirp->vhost_addr;
+        }
+        break;
+
+   case AF_INET6:
+        if (in6_equal(&so->so_faddr6, &in6addr_any) ||
+                in6_equal(&so->so_faddr6, &in6addr_loopback)) {
+           so->so_faddr6 = slirp->vhost_addr6;
+        }
+        break;
+
+    default:
+        break;
+    }
+}
diff --git a/slirp/socket.h b/slirp/socket.h
index c85b8b3..8feed2a 100644
--- a/slirp/socket.h
+++ b/slirp/socket.h
@@ -5,8 +5,8 @@
  * terms and conditions of the copyright.
  */
 
-#ifndef _SLIRP_SOCKET_H_
-#define _SLIRP_SOCKET_H_
+#ifndef SLIRP_SOCKET_H
+#define SLIRP_SOCKET_H
 
 #define SO_EXPIRE 240000
 #define SO_EXPIREFAST 10000
@@ -31,10 +31,27 @@
   struct tcpiphdr *so_ti;	   /* Pointer to the original ti within
 				    * so_mconn, for non-blocking connections */
   int so_urgc;
-  struct in_addr so_faddr;	   /* foreign host table entry */
-  struct in_addr so_laddr;	   /* local host table entry */
-  uint16_t so_fport;		   /* foreign port */
-  uint16_t so_lport;		   /* local port */
+  union {   /* foreign host */
+      struct sockaddr_storage ss;
+      struct sockaddr_in sin;
+      struct sockaddr_in6 sin6;
+  } fhost;
+#define so_faddr fhost.sin.sin_addr
+#define so_fport fhost.sin.sin_port
+#define so_faddr6 fhost.sin6.sin6_addr
+#define so_fport6 fhost.sin6.sin6_port
+#define so_ffamily fhost.ss.ss_family
+
+  union {   /* local host */
+      struct sockaddr_storage ss;
+      struct sockaddr_in sin;
+      struct sockaddr_in6 sin6;
+  } lhost;
+#define so_laddr lhost.sin.sin_addr
+#define so_lport lhost.sin.sin_port
+#define so_laddr6 lhost.sin6.sin6_addr
+#define so_lport6 lhost.sin6.sin6_port
+#define so_lfamily lhost.ss.ss_family
 
   uint8_t	so_iptos;	/* Type of service */
   uint8_t	so_emu;		/* Is the socket emulated? */
@@ -71,18 +88,58 @@
 #define SS_CTL			0x080
 #define SS_FACCEPTCONN		0x100	/* Socket is accepting connections from a host on the internet */
 #define SS_FACCEPTONCE		0x200	/* If set, the SS_FACCEPTCONN socket will die after one accept */
-#define SS_PROXIFIED           0x400   /* Socket is trying to connect through a proxy, only makes sense
-                                          when SS_ISFCONNECTING is also set */
 
 #define SS_PERSISTENT_MASK	0xf000	/* Unremovable state bits */
 #define SS_HOSTFWD		0x1000	/* Socket describes host->guest forwarding */
 #define SS_INCOMING		0x2000	/* Connection was initiated by a host on the internet */
 
-struct socket * solookup(struct socket *, struct in_addr, u_int, struct in_addr, u_int);
-struct socket * socreate(Slirp *);
+static inline int sockaddr_equal(struct sockaddr_storage *a,
+        struct sockaddr_storage *b)
+{
+    if (a->ss_family != b->ss_family) {
+        return 0;
+    }
+
+    switch (a->ss_family) {
+    case AF_INET:
+    {
+        struct sockaddr_in *a4 = (struct sockaddr_in *) a;
+        struct sockaddr_in *b4 = (struct sockaddr_in *) b;
+        return a4->sin_addr.s_addr == b4->sin_addr.s_addr
+               && a4->sin_port == b4->sin_port;
+    }
+    case AF_INET6:
+    {
+        struct sockaddr_in6 *a6 = (struct sockaddr_in6 *) a;
+        struct sockaddr_in6 *b6 = (struct sockaddr_in6 *) b;
+        return (in6_equal(&a6->sin6_addr, &b6->sin6_addr)
+                && a6->sin6_port == b6->sin6_port);
+    }
+    default:
+        g_assert_not_reached();
+    }
+
+    return 0;
+}
+
+static inline socklen_t sockaddr_size(struct sockaddr_storage *a)
+{
+    switch (a->ss_family) {
+    case AF_INET:
+        return sizeof(struct sockaddr_in);
+    case AF_INET6:
+        return sizeof(struct sockaddr_in6);
+    default:
+        g_assert_not_reached();
+    }
+}
+
+struct socket *solookup(struct socket **, struct socket *,
+        struct sockaddr_storage *, struct sockaddr_storage *);
+struct socket *socreate(Slirp *);
 void sofree(struct socket *);
 int soread(struct socket *);
-void sorecvoob(struct socket *);
+int sorecvoob(struct socket *);
 int sosendoob(struct socket *);
 int sowrite(struct socket *);
 void sorecvfrom(struct socket *);
@@ -96,4 +153,9 @@
 size_t sopreprbuf(struct socket *so, struct iovec *iov, int *np);
 int soreadbuf(struct socket *so, const char *buf, int size);
 
-#endif /* _SOCKET_H_ */
+void sotranslate_out(struct socket *, struct sockaddr_storage *);
+void sotranslate_in(struct socket *, struct sockaddr_storage *);
+void sotranslate_accept(struct socket *);
+
+
+#endif /* SLIRP_SOCKET_H */
diff --git a/slirp/tcp.h b/slirp/tcp.h
index 2e2b403..174d3d9 100644
--- a/slirp/tcp.h
+++ b/slirp/tcp.h
@@ -30,8 +30,8 @@
  * tcp.h,v 1.3 1994/08/21 05:27:34 paul Exp
  */
 
-#ifndef _TCP_H_
-#define _TCP_H_
+#ifndef TCP_H
+#define TCP_H
 
 typedef	uint32_t tcp_seq;
 
@@ -106,6 +106,8 @@
  */
 #undef TCP_MSS
 #define	TCP_MSS	1460
+#undef TCP6_MSS
+#define TCP6_MSS 1440
 
 #undef TCP_MAXWIN
 #define	TCP_MAXWIN	65535	/* largest value for (unscaled) window */
diff --git a/slirp/tcp_input.c b/slirp/tcp_input.c
index b30dacb..c5063a9 100644
--- a/slirp/tcp_input.c
+++ b/slirp/tcp_input.c
@@ -38,7 +38,8 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 #include "ip_icmp.h"
 
 #define	TCPREXMTTHRESH 3
@@ -213,9 +214,10 @@
  * protocol specification dated September, 1981 very closely.
  */
 void
-tcp_input(struct mbuf *m, int iphlen, struct socket *inso)
+tcp_input(struct mbuf *m, int iphlen, struct socket *inso, unsigned short af)
 {
-  	struct ip save_ip, *ip;
+	struct ip save_ip, *ip;
+	struct ip6 save_ip6, *ip6;
 	register struct tcpiphdr *ti;
 	caddr_t optp = NULL;
 	int optlen = 0;
@@ -227,12 +229,15 @@
 	int iss = 0;
 	u_long tiwin;
 	int ret;
+	struct sockaddr_storage lhost, fhost;
+	struct sockaddr_in *lhost4, *fhost4;
+	struct sockaddr_in6 *lhost6, *fhost6;
     struct ex_list *ex_ptr;
     Slirp *slirp;
 
 	DEBUG_CALL("tcp_input");
-	DEBUG_ARGS((dfd, " m = %8lx  iphlen = %2d  inso = %lx\n",
-		    (long )m, iphlen, (long )inso ));
+	DEBUG_ARGS((dfd, " m = %p  iphlen = %2d  inso = %p\n",
+		    m, iphlen, inso));
 
 	/*
 	 * If called with m == 0, then we're continuing the connect
@@ -253,37 +258,83 @@
 	}
 	slirp = m->slirp;
 
-	/*
-	 * Get IP and TCP header together in first mbuf.
-	 * Note: IP leaves IP header in first mbuf.
-	 */
-	ti = mtod(m, struct tcpiphdr *);
-	if (iphlen > sizeof(struct ip )) {
-	  ip_stripoptions(m, (struct mbuf *)0);
-	  iphlen=sizeof(struct ip );
+	ip = mtod(m, struct ip *);
+	ip6 = mtod(m, struct ip6 *);
+
+	switch (af) {
+	case AF_INET:
+	    if (iphlen > sizeof(struct ip)) {
+	        ip_stripoptions(m, (struct mbuf *)0);
+	        iphlen = sizeof(struct ip);
+	    }
+	    /* XXX Check if too short */
+
+
+	    /*
+	     * Save a copy of the IP header in case we want restore it
+	     * for sending an ICMP error message in response.
+	     */
+	    save_ip = *ip;
+	    save_ip.ip_len += iphlen;
+
+	    /*
+	     * Get IP and TCP header together in first mbuf.
+	     * Note: IP leaves IP header in first mbuf.
+	     */
+	    m->m_data -= sizeof(struct tcpiphdr) - sizeof(struct ip)
+	                                         - sizeof(struct tcphdr);
+	    m->m_len += sizeof(struct tcpiphdr) - sizeof(struct ip)
+	                                        - sizeof(struct tcphdr);
+	    ti = mtod(m, struct tcpiphdr *);
+
+	    /*
+	     * Checksum extended TCP header and data.
+	     */
+	    tlen = ip->ip_len;
+	    tcpiphdr2qlink(ti)->next = tcpiphdr2qlink(ti)->prev = NULL;
+	    memset(&ti->ih_mbuf, 0 , sizeof(struct mbuf_ptr));
+	    memset(&ti->ti, 0, sizeof(ti->ti));
+	    ti->ti_x0 = 0;
+	    ti->ti_src = save_ip.ip_src;
+	    ti->ti_dst = save_ip.ip_dst;
+	    ti->ti_pr = save_ip.ip_p;
+	    ti->ti_len = htons((uint16_t)tlen);
+	    break;
+
+	case AF_INET6:
+	    /*
+	     * Save a copy of the IP header in case we want restore it
+	     * for sending an ICMP error message in response.
+	     */
+	    save_ip6 = *ip6;
+	    /*
+	     * Get IP and TCP header together in first mbuf.
+	     * Note: IP leaves IP header in first mbuf.
+	     */
+	    m->m_data -= sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+	                                         + sizeof(struct tcphdr));
+	    m->m_len  += sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+	                                         + sizeof(struct tcphdr));
+	    ti = mtod(m, struct tcpiphdr *);
+
+	    tlen = ip6->ip_pl;
+	    tcpiphdr2qlink(ti)->next = tcpiphdr2qlink(ti)->prev = NULL;
+	    memset(&ti->ih_mbuf, 0 , sizeof(struct mbuf_ptr));
+	    memset(&ti->ti, 0, sizeof(ti->ti));
+	    ti->ti_x0 = 0;
+	    ti->ti_src6 = save_ip6.ip_src;
+	    ti->ti_dst6 = save_ip6.ip_dst;
+	    ti->ti_nh6 = save_ip6.ip_nh;
+	    ti->ti_len = htons((uint16_t)tlen);
+	    break;
+
+	default:
+	    g_assert_not_reached();
 	}
-	/* XXX Check if too short */
 
-
-	/*
-	 * Save a copy of the IP header in case we want restore it
-	 * for sending an ICMP error message in response.
-	 */
-	ip=mtod(m, struct ip *);
-	save_ip = *ip;
-	save_ip.ip_len+= iphlen;
-
-	/*
-	 * Checksum extended TCP header and data.
-	 */
-	tlen = ((struct ip *)ti)->ip_len;
-        tcpiphdr2qlink(ti)->next = tcpiphdr2qlink(ti)->prev = NULL;
-        memset(&ti->ti_i.ih_mbuf, 0 , sizeof(struct mbuf_ptr));
-	ti->ti_x1 = 0;
-	ti->ti_len = htons((uint16_t)tlen);
-	len = sizeof(struct ip ) + tlen;
-	if(cksum(m, len)) {
-	  goto drop;
+	len = ((sizeof(struct tcpiphdr) - sizeof(struct tcphdr)) + tlen);
+	if (cksum(m, len)) {
+	    goto drop;
 	}
 
 	/*
@@ -320,17 +371,31 @@
 	 * Locate pcb for segment.
 	 */
 findso:
-	so = slirp->tcp_last_so;
-	if (so->so_fport != ti->ti_dport ||
-	    so->so_lport != ti->ti_sport ||
-	    so->so_laddr.s_addr != ti->ti_src.s_addr ||
-	    so->so_faddr.s_addr != ti->ti_dst.s_addr) {
-		so = solookup(&slirp->tcb, ti->ti_src, ti->ti_sport,
-			       ti->ti_dst, ti->ti_dport);
-		if (so)
-			slirp->tcp_last_so = so;
+	lhost.ss_family = af;
+	fhost.ss_family = af;
+	switch (af) {
+	case AF_INET:
+	    lhost4 = (struct sockaddr_in *) &lhost;
+	    lhost4->sin_addr = ti->ti_src;
+	    lhost4->sin_port = ti->ti_sport;
+	    fhost4 = (struct sockaddr_in *) &fhost;
+	    fhost4->sin_addr = ti->ti_dst;
+	    fhost4->sin_port = ti->ti_dport;
+	    break;
+	case AF_INET6:
+	    lhost6 = (struct sockaddr_in6 *) &lhost;
+	    lhost6->sin6_addr = ti->ti_src6;
+	    lhost6->sin6_port = ti->ti_sport;
+	    fhost6 = (struct sockaddr_in6 *) &fhost;
+	    fhost6->sin6_addr = ti->ti_dst6;
+	    fhost6->sin6_port = ti->ti_dport;
+	    break;
+	default:
+	    g_assert_not_reached();
 	}
 
+	so = solookup(&slirp->tcp_last_so, &slirp->tcb, &lhost, &fhost);
+
 	/*
 	 * If the state is CLOSED (i.e., TCB does not exist) then
 	 * all data in the incoming segment is discarded.
@@ -374,13 +439,21 @@
 	  sbreserve(&so->so_snd, TCP_SNDSPACE);
 	  sbreserve(&so->so_rcv, TCP_RCVSPACE);
 
-	  so->so_laddr = ti->ti_src;
-	  so->so_lport = ti->ti_sport;
-	  so->so_faddr = ti->ti_dst;
-	  so->so_fport = ti->ti_dport;
+	  so->lhost.ss = lhost;
+	  so->fhost.ss = fhost;
 
-	  if ((so->so_iptos = tcp_tos(so)) == 0)
-	    so->so_iptos = ((struct ip *)ti)->ip_tos;
+	  so->so_iptos = tcp_tos(so);
+	  if (so->so_iptos == 0) {
+	      switch (af) {
+	      case AF_INET:
+	          so->so_iptos = ((struct ip *)ti)->ip_tos;
+	          break;
+	      case AF_INET6:
+	          break;
+	      default:
+	          g_assert_not_reached();
+	      }
+	  }
 
 	  tp = sototcpcb(so);
 	  tp->t_state = TCPS_LISTEN;
@@ -559,8 +632,9 @@
 	   * If this is destined for the control address, then flag to
 	   * tcp_ctl once connected, otherwise connect
 	   */
-	  if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-	      slirp->vnetwork_addr.s_addr) {
+	  if (af == AF_INET &&
+	         (so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
+	         slirp->vnetwork_addr.s_addr) {
 	    if (so->so_faddr.s_addr != slirp->vhost_addr.s_addr &&
 		so->so_faddr.s_addr != slirp->vnameserver_addr.s_addr) {
 		/* May be an add exec */
@@ -584,30 +658,60 @@
 	    goto cont_input;
 	  }
 
-          if ((tcp_fconnect(so) == -1) &&
-#if defined(_WIN32)
-              socket_error() != WSAEINPROGRESS && socket_error() != WSAEWOULDBLOCK
-#else
+	  if ((tcp_fconnect(so, so->so_ffamily) == -1) &&
+              (errno != EAGAIN) &&
               (errno != EINPROGRESS) && (errno != EWOULDBLOCK)
-#endif
           ) {
-	    u_char code=ICMP_UNREACH_NET;
+	    uint8_t code;
 	    DEBUG_MISC((dfd, " tcp fconnect errno = %d-%s\n",
 			errno,strerror(errno)));
 	    if(errno == ECONNREFUSED) {
 	      /* ACK the SYN, send RST to refuse the connection */
-	      tcp_respond(tp, ti, m, ti->ti_seq+1, (tcp_seq)0,
-			  TH_RST|TH_ACK);
+	      tcp_respond(tp, ti, m, ti->ti_seq + 1, (tcp_seq) 0,
+			  TH_RST | TH_ACK, af);
 	    } else {
-	      if(errno == EHOSTUNREACH) code=ICMP_UNREACH_HOST;
+	      switch (af) {
+	      case AF_INET:
+		code = ICMP_UNREACH_NET;
+		if (errno == EHOSTUNREACH) {
+		  code = ICMP_UNREACH_HOST;
+		}
+		break;
+	      case AF_INET6:
+		code = ICMP6_UNREACH_NO_ROUTE;
+		if (errno == EHOSTUNREACH) {
+		  code = ICMP6_UNREACH_ADDRESS;
+		}
+		break;
+	      default:
+		g_assert_not_reached();
+	      }
 	      HTONL(ti->ti_seq);             /* restore tcp header */
 	      HTONL(ti->ti_ack);
 	      HTONS(ti->ti_win);
 	      HTONS(ti->ti_urp);
 	      m->m_data -= sizeof(struct tcpiphdr)+off-sizeof(struct tcphdr);
 	      m->m_len  += sizeof(struct tcpiphdr)+off-sizeof(struct tcphdr);
-	      *ip=save_ip;
-	      icmp_error(m, ICMP_UNREACH,code, 0,strerror(errno));
+	      switch (af) {
+	      case AF_INET:
+		m->m_data += sizeof(struct tcpiphdr) - sizeof(struct ip)
+						     - sizeof(struct tcphdr);
+		m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct ip)
+						     - sizeof(struct tcphdr);
+		*ip = save_ip;
+		icmp_send_error(m, ICMP_UNREACH, code, 0, strerror(errno));
+		break;
+	      case AF_INET6:
+		m->m_data += sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+						     + sizeof(struct tcphdr));
+		m->m_len  -= sizeof(struct tcpiphdr) - (sizeof(struct ip6)
+						     + sizeof(struct tcphdr));
+		*ip6 = save_ip6;
+		icmp6_send_error(m, ICMP6_UNREACH, code);
+		break;
+	      default:
+		g_assert_not_reached();
+	      }
 	    }
             tcp_close(tp);
 	    m_free(m);
@@ -622,6 +726,12 @@
 	    so->so_ti = ti;
 	    tp->t_timer[TCPT_KEEP] = TCPTV_KEEP_INIT;
 	    tp->t_state = TCPS_SYN_RECEIVED;
+	    /*
+	     * Initialize receive sequence numbers now so that we can send a
+	     * valid RST if the remote end rejects our connection.
+	     */
+	    tp->irs = ti->ti_seq;
+	    tcp_rcvseqinit(tp);
 	    tcp_template(tp);
 	  }
 	  return;
@@ -923,8 +1033,8 @@
 
 		if (SEQ_LEQ(ti->ti_ack, tp->snd_una)) {
 			if (ti->ti_len == 0 && tiwin == tp->snd_wnd) {
-			  DEBUG_MISC((dfd, " dup ack  m = %lx  so = %lx\n",
-				      (long )m, (long )so));
+			  DEBUG_MISC((dfd, " dup ack  m = %p  so = %p\n",
+				      m, so));
 				/*
 				 * If we have outstanding data (other than
 				 * a window probe), this is a completely
@@ -1279,11 +1389,11 @@
 dropwithreset:
 	/* reuses m if m!=NULL, m_free() unnecessary */
 	if (tiflags & TH_ACK)
-		tcp_respond(tp, ti, m, (tcp_seq)0, ti->ti_ack, TH_RST);
+		tcp_respond(tp, ti, m, (tcp_seq)0, ti->ti_ack, TH_RST, af);
 	else {
 		if (tiflags & TH_SYN) ti->ti_len++;
-		tcp_respond(tp, ti, m, ti->ti_seq+ti->ti_len, (tcp_seq)0,
-		    TH_RST|TH_ACK);
+		tcp_respond(tp, ti, m, ti->ti_seq + ti->ti_len, (tcp_seq) 0,
+		    TH_RST | TH_ACK, af);
 	}
 
 	return;
@@ -1302,7 +1412,7 @@
 	int opt, optlen;
 
 	DEBUG_CALL("tcp_dooptions");
-	DEBUG_ARGS((dfd, " tp = %lx  cnt=%i\n", (long)tp, cnt));
+	DEBUG_ARGS((dfd, " tp = %p  cnt=%i\n", tp, cnt));
 
 	for (; cnt > 0; cnt -= optlen, cp += optlen) {
 		opt = cp[0];
@@ -1383,7 +1493,7 @@
 	register short delta;
 
 	DEBUG_CALL("tcp_xmit_timer");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("rtt = %d", rtt);
 
 	if (tp->t_srtt != 0) {
@@ -1471,10 +1581,22 @@
 	int mss;
 
 	DEBUG_CALL("tcp_mss");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("offer = %d", offer);
 
-	mss = min(IF_MTU, IF_MRU) - sizeof(struct tcpiphdr);
+	switch (so->so_ffamily) {
+	case AF_INET:
+	    mss = min(IF_MTU, IF_MRU) - sizeof(struct tcphdr)
+	                              + sizeof(struct ip);
+	    break;
+	case AF_INET6:
+	    mss = min(IF_MTU, IF_MRU) - sizeof(struct tcphdr)
+	                              + sizeof(struct ip6);
+	    break;
+	default:
+	    g_assert_not_reached();
+	}
+
 	if (offer)
 		mss = min(mss, offer);
 	mss = max(mss, 32);
diff --git a/slirp/tcp_output.c b/slirp/tcp_output.c
index 31c86de..363496b 100644
--- a/slirp/tcp_output.c
+++ b/slirp/tcp_output.c
@@ -38,11 +38,9 @@
  * terms and conditions of the copyright.
  */
 
-#if defined(CONFIG_ANDROID)
-#include "android/crashreport/crash-handler.h"
-#endif
-
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "qemu/abort.h"
+#include "slirp.h"
 
 static const u_char  tcp_outflags[TCP_NSTATES] = {
 	TH_RST|TH_ACK, 0,      TH_SYN,        TH_SYN|TH_ACK,
@@ -64,25 +62,22 @@
 	register long len, win;
 	int off, flags, error;
 	register struct mbuf *m;
-	register struct tcpiphdr *ti;
+	register struct tcpiphdr *ti, tcpiph_save;
+	struct ip *ip;
+	struct ip6 *ip6;
 	u_char opt[MAX_TCPOPTLEN];
 	unsigned optlen, hdrlen;
 	int idle, sendalot;
 
-#ifdef CONFIG_ANDROID
-	if (tp == NULL) {
-		crashhandler_die("QEMU-2 tcp_output() invoked with tp==NULL");
-	}
-#endif
-	so = tp->t_socket;
-#ifdef CONFIG_ANDROID
-	if (so == NULL) {
-		crashhandler_die("QEMU-2 tcp_output() invoked "
-		                 "with tp->t_socket==NULL");
-	}
-#endif
+        if (tp == NULL) {
+            qemu_abort("%s: should not happen: tp == NULL\n", __func__);
+        }
+        so = tp->t_socket;
+        if (so == NULL) {
+            qemu_abort("%s: should not happen: so == NULL\n", __func__);
+        }
 	DEBUG_CALL("tcp_output");
-	DEBUG_ARG("tp = %lx", (long )tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	/*
 	 * Determine length of data that should be transmitted,
@@ -462,16 +457,45 @@
 	 * the template, but need a way to checksum without them.
 	 */
 	m->m_len = hdrlen + len; /* XXX Needed? m_len should be correct */
+	tcpiph_save = *mtod(m, struct tcpiphdr *);
 
-    {
+	switch (so->so_ffamily) {
+	case AF_INET:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    ip = mtod(m, struct ip *);
 
-	((struct ip *)ti)->ip_len = m->m_len;
+	    ip->ip_len = m->m_len;
+	    ip->ip_dst = tcpiph_save.ti_dst;
+	    ip->ip_src = tcpiph_save.ti_src;
+	    ip->ip_p = tcpiph_save.ti_pr;
 
-	((struct ip *)ti)->ip_ttl = IPDEFTTL;
-	((struct ip *)ti)->ip_tos = so->so_iptos;
+	    ip->ip_ttl = IPDEFTTL;
+	    ip->ip_tos = so->so_iptos;
+	    error = ip_output(so, m);
+	    break;
 
-	error = ip_output(so, m);
-    }
+	case AF_INET6:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    ip6 = mtod(m, struct ip6 *);
+
+	    ip6->ip_pl = tcpiph_save.ti_len;
+	    ip6->ip_dst = tcpiph_save.ti_dst6;
+	    ip6->ip_src = tcpiph_save.ti_src6;
+	    ip6->ip_nh = tcpiph_save.ti_nh6;
+
+	    error = ip6_output(so, m, 0);
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
+
 	if (error) {
 out:
 		return (error);
diff --git a/slirp/tcp_subr.c b/slirp/tcp_subr.c
index 822a358..ed16e18 100644
--- a/slirp/tcp_subr.c
+++ b/slirp/tcp_subr.c
@@ -38,11 +38,8 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
-
-#ifdef CONFIG_ANDROID
-#include "android/proxy/proxy_common.h"
-#endif  // CONFIG_ANDROID
+#include "qemu/osdep.h"
+#include "slirp.h"
 
 /* patchable/settable parameters for tcp */
 /* Don't do rfc1323 performance enhancements */
@@ -79,13 +76,30 @@
 	register struct tcpiphdr *n = &tp->t_template;
 
 	n->ti_mbuf = NULL;
-	n->ti_x1 = 0;
-	n->ti_pr = IPPROTO_TCP;
-	n->ti_len = htons(sizeof (struct tcpiphdr) - sizeof (struct ip));
-	n->ti_src = so->so_faddr;
-	n->ti_dst = so->so_laddr;
-	n->ti_sport = so->so_fport;
-	n->ti_dport = so->so_lport;
+	memset(&n->ti, 0, sizeof(n->ti));
+	n->ti_x0 = 0;
+	switch (so->so_ffamily) {
+	case AF_INET:
+	    n->ti_pr = IPPROTO_TCP;
+	    n->ti_len = htons(sizeof(struct tcphdr));
+	    n->ti_src = so->so_faddr;
+	    n->ti_dst = so->so_laddr;
+	    n->ti_sport = so->so_fport;
+	    n->ti_dport = so->so_lport;
+	    break;
+
+	case AF_INET6:
+	    n->ti_nh6 = IPPROTO_TCP;
+	    n->ti_len = htons(sizeof(struct tcphdr));
+	    n->ti_src6 = so->so_faddr6;
+	    n->ti_dst6 = so->so_laddr6;
+	    n->ti_sport = so->so_fport6;
+	    n->ti_dport = so->so_lport6;
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
 
 	n->ti_seq = 0;
 	n->ti_ack = 0;
@@ -112,7 +126,7 @@
  */
 void
 tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
-            tcp_seq ack, tcp_seq seq, int flags)
+            tcp_seq ack, tcp_seq seq, int flags, unsigned short af)
 {
 	register int tlen;
 	int win = 0;
@@ -134,6 +148,7 @@
 		m->m_data += IF_MAXLINKHDR;
 		*mtod(m, struct tcpiphdr *) = *ti;
 		ti = mtod(m, struct tcpiphdr *);
+		memset(&ti->ti, 0, sizeof(ti->ti));
 		flags = TH_ACK;
 	} else {
 		/*
@@ -145,16 +160,26 @@
 		m->m_len = sizeof (struct tcpiphdr);
 		tlen = 0;
 #define xchg(a,b,type) { type t; t=a; a=b; b=t; }
-		xchg(ti->ti_dst.s_addr, ti->ti_src.s_addr, uint32_t);
-		xchg(ti->ti_dport, ti->ti_sport, uint16_t);
+		switch (af) {
+		case AF_INET:
+		    xchg(ti->ti_dst.s_addr, ti->ti_src.s_addr, uint32_t);
+		    xchg(ti->ti_dport, ti->ti_sport, uint16_t);
+		    break;
+		case AF_INET6:
+		    xchg(ti->ti_dst6, ti->ti_src6, struct in6_addr);
+		    xchg(ti->ti_dport, ti->ti_sport, uint16_t);
+		    break;
+		default:
+		    g_assert_not_reached();
+		}
 #undef xchg
 	}
 	ti->ti_len = htons((u_short)(sizeof (struct tcphdr) + tlen));
 	tlen += sizeof (struct tcpiphdr);
 	m->m_len = tlen;
 
-        ti->ti_mbuf = NULL;
-	ti->ti_x1 = 0;
+	ti->ti_mbuf = NULL;
+	ti->ti_x0 = 0;
 	ti->ti_seq = htonl(seq);
 	ti->ti_ack = htonl(ack);
 	ti->ti_x2 = 0;
@@ -167,14 +192,49 @@
 	ti->ti_urp = 0;
 	ti->ti_sum = 0;
 	ti->ti_sum = cksum(m, tlen);
-	((struct ip *)ti)->ip_len = tlen;
 
-	if(flags & TH_RST)
-	  ((struct ip *)ti)->ip_ttl = MAXTTL;
-	else
-	  ((struct ip *)ti)->ip_ttl = IPDEFTTL;
+	struct tcpiphdr tcpiph_save = *(mtod(m, struct tcpiphdr *));
+	struct ip *ip;
+	struct ip6 *ip6;
 
-	(void) ip_output((struct socket *)0, m);
+	switch (af) {
+	case AF_INET:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip);
+	    ip = mtod(m, struct ip *);
+	    ip->ip_len = tlen;
+	    ip->ip_dst = tcpiph_save.ti_dst;
+	    ip->ip_src = tcpiph_save.ti_src;
+	    ip->ip_p = tcpiph_save.ti_pr;
+
+	    if (flags & TH_RST) {
+	        ip->ip_ttl = MAXTTL;
+	    } else {
+	        ip->ip_ttl = IPDEFTTL;
+	    }
+
+	    ip_output(NULL, m);
+	    break;
+
+	case AF_INET6:
+	    m->m_data += sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    m->m_len  -= sizeof(struct tcpiphdr) - sizeof(struct tcphdr)
+	                                         - sizeof(struct ip6);
+	    ip6 = mtod(m, struct ip6 *);
+	    ip6->ip_pl = tlen;
+	    ip6->ip_dst = tcpiph_save.ti_dst6;
+	    ip6->ip_src = tcpiph_save.ti_src6;
+	    ip6->ip_nh = tcpiph_save.ti_nh6;
+
+	    ip6_output(NULL, m, 0);
+	    break;
+
+	default:
+	    g_assert_not_reached();
+	}
 }
 
 /*
@@ -193,7 +253,7 @@
 
 	memset((char *) tp, 0, sizeof(struct tcpcb));
 	tp->seg_next = tp->seg_prev = (struct tcpiphdr*)tp;
-	tp->t_maxseg = TCP_MSS;
+	tp->t_maxseg = (so->so_ffamily == AF_INET) ? TCP_MSS : TCP6_MSS;
 
 	tp->t_flags = TCP_DO_RFC1323 ? (TF_REQ_SCALE|TF_REQ_TSTMP) : 0;
 	tp->t_socket = so;
@@ -228,7 +288,7 @@
 struct tcpcb *tcp_drop(struct tcpcb *tp, int err)
 {
 	DEBUG_CALL("tcp_drop");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
 	DEBUG_ARG("errno = %d", errno);
 
 	if (TCPS_HAVERCVDSYN(tp->t_state)) {
@@ -253,7 +313,7 @@
 	register struct mbuf *m;
 
 	DEBUG_CALL("tcp_close");
-	DEBUG_ARG("tp = %lx", (long )tp);
+	DEBUG_ARG("tp = %p", tp);
 
 	/* free the reassembly queue, if any */
 	t = tcpfrag_list_first(tp);
@@ -294,7 +354,11 @@
 {
 
 	DEBUG_CALL("tcp_sockclosed");
-	DEBUG_ARG("tp = %lx", (long)tp);
+	DEBUG_ARG("tp = %p", tp);
+
+	if (!tp) {
+		return;
+	}
 
 	switch (tp->t_state) {
 
@@ -314,27 +378,9 @@
 		tp->t_state = TCPS_LAST_ACK;
 		break;
 	}
-	if (tp)
-		tcp_output(tp);
+	tcp_output(tp);
 }
 
-#ifdef CONFIG_ANDROID
-static void
-tcp_proxy_event(struct socket*  so, int s, ProxyEvent event) {
-    so->so_state &= ~SS_PROXIFIED;
-
-    if (event == PROXY_EVENT_CONNECTED) {
-        so->s         = s;
-        so->so_state &= ~(SS_ISFCONNECTING);
-    } else {
-        so->so_state = SS_NOFDREF;
-    }
-
-    /* continue the connect */
-    tcp_input(NULL, sizeof(struct ip), so);
-}
-#endif  // CONFIG_ANDROID
-
 /*
  * Connect to a host on the Internet
  * Called by tcp_input
@@ -345,64 +391,29 @@
  * nonblocking.  Connect returns after the SYN is sent, and does
  * not wait for ACK+SYN.
  */
-int tcp_fconnect(struct socket *so)
+int tcp_fconnect(struct socket *so, unsigned short af)
 {
-  Slirp *slirp = so->slirp;
   int ret=0;
-  int try_proxy = 0;
 
   DEBUG_CALL("tcp_fconnect");
-  DEBUG_ARG("so = %lx", (long )so);
+  DEBUG_ARG("so = %p", so);
 
-  if( (ret = so->s = qemu_socket(AF_INET,SOCK_STREAM,0)) >= 0) {
+  ret = so->s = qemu_socket(af, SOCK_STREAM, 0);
+  if (ret >= 0) {
     int opt, s=so->s;
-    struct sockaddr_in addr;
+    struct sockaddr_storage addr;
 
     qemu_set_nonblock(s);
     socket_set_fast_reuse(s);
     opt = 1;
     qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(opt));
 
-    addr.sin_family = AF_INET;
-    if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-        slirp->vnetwork_addr.s_addr) {
-      /* It's an alias */
-      if (is_dns_addr(slirp, &so->so_faddr)) {
-	if (get_dns_addr(&so->so_faddr, &addr.sin_addr) < 0)
-	  addr.sin_addr = loopback_addr;
-      } else {
-	addr.sin_addr = loopback_addr;
-      }
-    } else {
-      try_proxy = 1;
-      addr.sin_addr = so->so_faddr;
-    }
-    addr.sin_port = so->so_fport;
-
-    DEBUG_MISC((dfd, " connect()ing, addr.sin_port=%d, "
-        "addr.sin_addr.s_addr=%.16s, proxy=%d\n",
-        ntohs(addr.sin_port), inet_ntoa(addr.sin_addr), try_proxy));
-
-#ifdef CONFIG_ANDROID
-    if (try_proxy) {
-        SockAddress sockaddr;
-        sock_address_init_inet(&sockaddr,
-                               ntohl(addr.sin_addr.s_addr),
-                               ntohs(addr.sin_port));
-        if (!proxy_manager_add(&sockaddr, SOCKET_STREAM,
-                               (ProxyEventFunc)tcp_proxy_event, so)) {
-            soisfconnecting(so);
-            so->s = -1;
-            so->so_state |= SS_PROXIFIED;
-            return 0;
-        }
-    }
-#else
-    (void)try_proxy;  /* make compiler happy */
-#endif  // CONFIG_ANDROID
+    addr = so->fhost.ss;
+    DEBUG_CALL(" connect()ing")
+    sotranslate_out(so, &addr);
 
     /* We don't care what port we get */
-    ret = connect(s,(struct sockaddr *)&addr,sizeof (addr));
+    ret = connect(s, (struct sockaddr *)&addr, sockaddr_size(&addr));
 
     /*
      * If it's not in progress, it failed, so we just return 0,
@@ -430,13 +441,13 @@
 {
     Slirp *slirp = inso->slirp;
     struct socket *so;
-    struct sockaddr_in addr;
-    socklen_t addrlen = sizeof(struct sockaddr_in);
+    struct sockaddr_storage addr;
+    socklen_t addrlen = sizeof(struct sockaddr_storage);
     struct tcpcb *tp;
     int s, opt;
 
     DEBUG_CALL("tcp_connect");
-    DEBUG_ARG("inso = %lx", (long)inso);
+    DEBUG_ARG("inso = %p", inso);
 
     /*
      * If it's an SS_ACCEPTONCE socket, no need to socreate()
@@ -456,8 +467,8 @@
             free(so); /* NOT sofree */
             return;
         }
-        so->so_laddr = inso->so_laddr;
-        so->so_lport = inso->so_lport;
+        so->lhost = inso->lhost;
+        so->so_ffamily = inso->so_ffamily;
     }
 
     tcp_mss(sototcpcb(so), 0);
@@ -473,14 +484,8 @@
     qemu_setsockopt(s, SOL_SOCKET, SO_OOBINLINE, &opt, sizeof(int));
     socket_set_nodelay(s);
 
-    so->so_fport = addr.sin_port;
-    so->so_faddr = addr.sin_addr;
-    /* Translate connections from localhost to the real hostname */
-    if (so->so_faddr.s_addr == 0 ||
-        (so->so_faddr.s_addr & loopback_mask) ==
-        (loopback_addr.s_addr & loopback_mask)) {
-        so->so_faddr = slirp->vhost_addr;
-    }
+    so->fhost.ss = addr;
+    sotranslate_accept(so);
 
     /* Close the accept() socket, set right state */
     if (inso->so_state & SS_FACCEPTONCE) {
@@ -607,8 +612,8 @@
 	char *bptr;
 
 	DEBUG_CALL("tcp_emu");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 
 	switch(so->so_emu) {
 		int x, i;
@@ -943,7 +948,7 @@
     int do_pty;
 
     DEBUG_CALL("tcp_ctl");
-    DEBUG_ARG("so = %lx", (long )so);
+    DEBUG_ARG("so = %p", so);
 
     if (so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
         /* Check if it's pty_exec */
diff --git a/slirp/tcp_timer.c b/slirp/tcp_timer.c
index 6c5bb11..f9060c7 100644
--- a/slirp/tcp_timer.c
+++ b/slirp/tcp_timer.c
@@ -30,7 +30,8 @@
  * tcp_timer.c,v 1.2 1994/08/02 07:49:10 davidg Exp
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 
 static struct tcpcb *tcp_timers(register struct tcpcb *tp, int timer);
 
@@ -277,7 +278,8 @@
 			 * correspondent TCP to respond.
 			 */
 			tcp_respond(tp, &tp->t_template, (struct mbuf *)NULL,
-			    tp->rcv_nxt, tp->snd_una - 1, 0);
+			    tp->rcv_nxt, tp->snd_una - 1, 0,
+			    tp->t_socket->so_ffamily);
 			tp->t_timer[TCPT_KEEP] = TCPTV_KEEPINTVL;
 		} else
 			tp->t_timer[TCPT_KEEP] = TCPTV_KEEP_IDLE;
diff --git a/slirp/tcp_timer.h b/slirp/tcp_timer.h
index ff17914..b25b391 100644
--- a/slirp/tcp_timer.h
+++ b/slirp/tcp_timer.h
@@ -30,8 +30,8 @@
  * tcp_timer.h,v 1.4 1994/08/21 05:27:38 paul Exp
  */
 
-#ifndef _TCP_TIMER_H_
-#define _TCP_TIMER_H_
+#ifndef TCP_TIMER_H
+#define TCP_TIMER_H
 
 /*
  * Definitions of the TCP timers.  These timers are counted
diff --git a/slirp/tcp_var.h b/slirp/tcp_var.h
index 004193f..0f8f187 100644
--- a/slirp/tcp_var.h
+++ b/slirp/tcp_var.h
@@ -30,8 +30,8 @@
  * tcp_var.h,v 1.3 1994/08/21 05:27:39 paul Exp
  */
 
-#ifndef _TCP_VAR_H_
-#define _TCP_VAR_H_
+#ifndef TCP_VAR_H
+#define TCP_VAR_H
 
 #include "tcpip.h"
 #include "tcp_timer.h"
diff --git a/slirp/tcpip.h b/slirp/tcpip.h
index 7974ce3..7bdb971 100644
--- a/slirp/tcpip.h
+++ b/slirp/tcpip.h
@@ -30,22 +30,39 @@
  * tcpip.h,v 1.3 1994/08/21 05:27:40 paul Exp
  */
 
-#ifndef _TCPIP_H_
-#define _TCPIP_H_
+#ifndef TCPIP_H
+#define TCPIP_H
 
 /*
  * Tcp+ip header, after ip options removed.
  */
 struct tcpiphdr {
-	struct 	ipovly ti_i;		/* overlaid ip structure */
-	struct	tcphdr ti_t;		/* tcp header */
+    struct mbuf_ptr ih_mbuf;	/* backpointer to mbuf */
+    union {
+        struct {
+            struct  in_addr ih_src; /* source internet address */
+            struct  in_addr ih_dst; /* destination internet address */
+            uint8_t ih_x1;          /* (unused) */
+            uint8_t ih_pr;          /* protocol */
+        } ti_i4;
+        struct {
+            struct  in6_addr ih_src;
+            struct  in6_addr ih_dst;
+            uint8_t ih_x1;
+            uint8_t ih_nh;
+        } ti_i6;
+    } ti;
+    uint16_t    ti_x0;
+    uint16_t    ti_len;             /* protocol length */
+    struct      tcphdr ti_t;        /* tcp header */
 };
-#define	ti_mbuf		ti_i.ih_mbuf.mptr
-#define	ti_x1		ti_i.ih_x1
-#define	ti_pr		ti_i.ih_pr
-#define	ti_len		ti_i.ih_len
-#define	ti_src		ti_i.ih_src
-#define	ti_dst		ti_i.ih_dst
+#define	ti_mbuf		ih_mbuf.mptr
+#define	ti_pr		ti.ti_i4.ih_pr
+#define	ti_src		ti.ti_i4.ih_src
+#define	ti_dst		ti.ti_i4.ih_dst
+#define	ti_src6		ti.ti_i6.ih_src
+#define	ti_dst6		ti.ti_i6.ih_dst
+#define	ti_nh6		ti.ti_i6.ih_nh
 #define	ti_sport	ti_t.th_sport
 #define	ti_dport	ti_t.th_dport
 #define	ti_seq		ti_t.th_seq
@@ -65,6 +82,13 @@
 #define tcpfrag_list_end(F, T) (tcpiphdr2qlink(F) == (struct qlink*)(T))
 #define tcpfrag_list_empty(T) ((T)->seg_next == (struct tcpiphdr*)(T))
 
+/* This is the difference between the size of a tcpiphdr structure, and the
+ * size of actual ip+tcp headers, rounded up since we need to align data.  */
+#define TCPIPHDR_DELTA\
+    (max(0,\
+         (sizeof(struct tcpiphdr)\
+          - sizeof(struct ip) - sizeof(struct tcphdr) + 3) & ~3))
+
 /*
  * Just a clean way to get to the first byte
  * of the packet
diff --git a/slirp/tftp.c b/slirp/tftp.c
index 0960541..d539fff 100644
--- a/slirp/tftp.c
+++ b/slirp/tftp.c
@@ -22,11 +22,10 @@
  * THE SOFTWARE.
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 #include "qemu-common.h"
-#ifdef CONFIG_ANDROID
-#include "android/utils/file_io.h"
-#endif  // CONFIG_ANDROID
+#include "qemu/cutils.h"
 
 static inline int tftp_session_in_use(struct tftp_session *spt)
 {
@@ -48,7 +47,8 @@
     spt->slirp = NULL;
 }
 
-static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
+static int tftp_session_allocate(Slirp *slirp, struct sockaddr_storage *srcsas,
+                                 struct tftp_t *tp)
 {
   struct tftp_session *spt;
   int k;
@@ -70,7 +70,7 @@
 
  found:
   memset(spt, 0, sizeof(*spt));
-  memcpy(&spt->client_ip, &tp->ip.ip_src, sizeof(spt->client_ip));
+  spt->client_addr = *srcsas;
   spt->fd = -1;
   spt->client_port = tp->udp.uh_sport;
   spt->slirp = slirp;
@@ -80,7 +80,8 @@
   return k;
 }
 
-static int tftp_session_find(Slirp *slirp, struct tftp_t *tp)
+static int tftp_session_find(Slirp *slirp, struct sockaddr_storage *srcsas,
+                             struct tftp_t *tp)
 {
   struct tftp_session *spt;
   int k;
@@ -89,7 +90,7 @@
     spt = &slirp->tftp_sessions[k];
 
     if (tftp_session_in_use(spt)) {
-      if (!memcmp(&spt->client_ip, &tp->ip.ip_src, sizeof(spt->client_ip))) {
+      if (sockaddr_equal(&spt->client_addr, srcsas)) {
 	if (spt->client_port == tp->udp.uh_sport) {
 	  return k;
 	}
@@ -122,11 +123,53 @@
     return bytes_read;
 }
 
+static struct tftp_t *tftp_prep_mbuf_data(struct tftp_session *spt,
+                                          struct mbuf *m)
+{
+    struct tftp_t *tp;
+
+    memset(m->m_data, 0, m->m_size);
+
+    m->m_data += IF_MAXLINKHDR;
+    if (spt->client_addr.ss_family == AF_INET6) {
+        m->m_data += sizeof(struct ip6);
+    } else {
+        m->m_data += sizeof(struct ip);
+    }
+    tp = (void *)m->m_data;
+    m->m_data += sizeof(struct udphdr);
+
+    return tp;
+}
+
+static void tftp_udp_output(struct tftp_session *spt, struct mbuf *m,
+                            struct tftp_t *recv_tp)
+{
+    if (spt->client_addr.ss_family == AF_INET6) {
+        struct sockaddr_in6 sa6, da6;
+
+        sa6.sin6_addr = spt->slirp->vhost_addr6;
+        sa6.sin6_port = recv_tp->udp.uh_dport;
+        da6.sin6_addr = ((struct sockaddr_in6 *)&spt->client_addr)->sin6_addr;
+        da6.sin6_port = spt->client_port;
+
+        udp6_output(NULL, m, &sa6, &da6);
+    } else {
+        struct sockaddr_in sa4, da4;
+
+        sa4.sin_addr = spt->slirp->vhost_addr;
+        sa4.sin_port = recv_tp->udp.uh_dport;
+        da4.sin_addr = ((struct sockaddr_in *)&spt->client_addr)->sin_addr;
+        da4.sin_port = spt->client_port;
+
+        udp_output(NULL, m, &sa4, &da4, IPTOS_LOWDELAY);
+    }
+}
+
 static int tftp_send_oack(struct tftp_session *spt,
                           const char *keys[], uint32_t values[], int nb,
                           struct tftp_t *recv_tp)
 {
-    struct sockaddr_in saddr, daddr;
     struct mbuf *m;
     struct tftp_t *tp;
     int i, n = 0;
@@ -134,13 +177,9 @@
     m = m_get(spt->slirp);
 
     if (!m)
-	return -1;
+        return -1;
 
-    memset(m->m_data, 0, m->m_size);
-
-    m->m_data += IF_MAXLINKHDR;
-    tp = (void *)m->m_data;
-    m->m_data += sizeof(struct udpiphdr);
+    tp = tftp_prep_mbuf_data(spt, m);
 
     tp->tp_op = htons(TFTP_OACK);
     for (i = 0; i < nb; i++) {
@@ -150,15 +189,8 @@
                       values[i]) + 1;
     }
 
-    saddr.sin_addr = recv_tp->ip.ip_dst;
-    saddr.sin_port = recv_tp->udp.uh_dport;
-
-    daddr.sin_addr = spt->client_ip;
-    daddr.sin_port = spt->client_port;
-
-    m->m_len = sizeof(struct tftp_t) - 514 + n -
-        sizeof(struct ip) - sizeof(struct udphdr);
-    udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+    m->m_len = sizeof(struct tftp_t) - 514 + n - sizeof(struct udphdr);
+    tftp_udp_output(spt, m, recv_tp);
 
     return 0;
 }
@@ -167,7 +199,6 @@
                             uint16_t errorcode, const char *msg,
                             struct tftp_t *recv_tp)
 {
-  struct sockaddr_in saddr, daddr;
   struct mbuf *m;
   struct tftp_t *tp;
 
@@ -177,26 +208,15 @@
     goto out;
   }
 
-  memset(m->m_data, 0, m->m_size);
-
-  m->m_data += IF_MAXLINKHDR;
-  tp = (void *)m->m_data;
-  m->m_data += sizeof(struct udpiphdr);
+  tp = tftp_prep_mbuf_data(spt, m);
 
   tp->tp_op = htons(TFTP_ERROR);
   tp->x.tp_error.tp_error_code = htons(errorcode);
   pstrcpy((char *)tp->x.tp_error.tp_msg, sizeof(tp->x.tp_error.tp_msg), msg);
 
-  saddr.sin_addr = recv_tp->ip.ip_dst;
-  saddr.sin_port = recv_tp->udp.uh_dport;
-
-  daddr.sin_addr = spt->client_ip;
-  daddr.sin_port = spt->client_port;
-
-  m->m_len = sizeof(struct tftp_t) - 514 + 3 + strlen(msg) -
-        sizeof(struct ip) - sizeof(struct udphdr);
-
-  udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+  m->m_len = sizeof(struct tftp_t) - 514 + 3 + strlen(msg)
+             - sizeof(struct udphdr);
+  tftp_udp_output(spt, m, recv_tp);
 
 out:
   tftp_session_terminate(spt);
@@ -205,7 +225,6 @@
 static void tftp_send_next_block(struct tftp_session *spt,
                                  struct tftp_t *recv_tp)
 {
-  struct sockaddr_in saddr, daddr;
   struct mbuf *m;
   struct tftp_t *tp;
   int nobytes;
@@ -216,21 +235,11 @@
     return;
   }
 
-  memset(m->m_data, 0, m->m_size);
-
-  m->m_data += IF_MAXLINKHDR;
-  tp = (void *)m->m_data;
-  m->m_data += sizeof(struct udpiphdr);
+  tp = tftp_prep_mbuf_data(spt, m);
 
   tp->tp_op = htons(TFTP_DATA);
   tp->x.tp_data.tp_block_nr = htons((spt->block_nr + 1) & 0xffff);
 
-  saddr.sin_addr = recv_tp->ip.ip_dst;
-  saddr.sin_port = recv_tp->udp.uh_dport;
-
-  daddr.sin_addr = spt->client_ip;
-  daddr.sin_port = spt->client_port;
-
   nobytes = tftp_read_data(spt, spt->block_nr, tp->x.tp_data.tp_buf, 512);
 
   if (nobytes < 0) {
@@ -243,10 +252,8 @@
     return;
   }
 
-  m->m_len = sizeof(struct tftp_t) - (512 - nobytes) -
-        sizeof(struct ip) - sizeof(struct udphdr);
-
-  udp_output2(NULL, m, &saddr, &daddr, IPTOS_LOWDELAY);
+  m->m_len = sizeof(struct tftp_t) - (512 - nobytes) - sizeof(struct udphdr);
+  tftp_udp_output(spt, m, recv_tp);
 
   if (nobytes == 512) {
     tftp_session_update(spt);
@@ -258,7 +265,8 @@
   spt->block_nr++;
 }
 
-static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
+static void tftp_handle_rrq(Slirp *slirp, struct sockaddr_storage *srcsas,
+                            struct tftp_t *tp, int pktlen)
 {
   struct tftp_session *spt;
   int s, k;
@@ -269,12 +277,12 @@
   int nb_options = 0;
 
   /* check if a session already exists and if so terminate it */
-  s = tftp_session_find(slirp, tp);
+  s = tftp_session_find(slirp, srcsas, tp);
   if (s >= 0) {
     tftp_session_terminate(&slirp->tftp_sessions[s]);
   }
 
-  s = tftp_session_allocate(slirp, tp);
+  s = tftp_session_allocate(slirp, srcsas, tp);
 
   if (s < 0) {
     return;
@@ -363,11 +371,7 @@
 	  struct stat stat_p;
 
 	  if (tsize == 0) {
-#ifdef CONFIG_ANDROID
-	      if (android_stat(spt->filename, &stat_p) == 0)
-#else  // !CONFIG_ANDROID
-	      if (stat(spt->filename, &stat_p) == 0)
-#endif  // CONFIG_ANDROID
+	      if (qemu_stat(spt->filename, &stat_p) == 0)
 		  tsize = stat_p.st_size;
 	      else {
 		  tftp_send_error(spt, 1, "File not found", tp);
@@ -403,11 +407,12 @@
   tftp_send_next_block(spt, tp);
 }
 
-static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
+static void tftp_handle_ack(Slirp *slirp, struct sockaddr_storage *srcsas,
+                            struct tftp_t *tp, int pktlen)
 {
   int s;
 
-  s = tftp_session_find(slirp, tp);
+  s = tftp_session_find(slirp, srcsas, tp);
 
   if (s < 0) {
     return;
@@ -416,11 +421,12 @@
   tftp_send_next_block(&slirp->tftp_sessions[s], tp);
 }
 
-static void tftp_handle_error(Slirp *slirp, struct tftp_t *tp, int pktlen)
+static void tftp_handle_error(Slirp *slirp, struct sockaddr_storage *srcsas,
+                              struct tftp_t *tp, int pktlen)
 {
   int s;
 
-  s = tftp_session_find(slirp, tp);
+  s = tftp_session_find(slirp, srcsas, tp);
 
   if (s < 0) {
     return;
@@ -429,21 +435,21 @@
   tftp_session_terminate(&slirp->tftp_sessions[s]);
 }
 
-void tftp_input(struct mbuf *m)
+void tftp_input(struct sockaddr_storage *srcsas, struct mbuf *m)
 {
   struct tftp_t *tp = (struct tftp_t *)m->m_data;
 
   switch(ntohs(tp->tp_op)) {
   case TFTP_RRQ:
-    tftp_handle_rrq(m->slirp, tp, m->m_len);
+    tftp_handle_rrq(m->slirp, srcsas, tp, m->m_len);
     break;
 
   case TFTP_ACK:
-    tftp_handle_ack(m->slirp, tp, m->m_len);
+    tftp_handle_ack(m->slirp, srcsas, tp, m->m_len);
     break;
 
   case TFTP_ERROR:
-    tftp_handle_error(m->slirp, tp, m->m_len);
+    tftp_handle_error(m->slirp, srcsas, tp, m->m_len);
     break;
   }
 }
diff --git a/slirp/tftp.h b/slirp/tftp.h
index e1cc24b..2cd276d 100644
--- a/slirp/tftp.h
+++ b/slirp/tftp.h
@@ -1,6 +1,7 @@
 /* tftp defines */
+
 #ifndef SLIRP_TFTP_H
-#define SLIRP_TFTP_H 1
+#define SLIRP_TFTP_H
 
 #define TFTP_SESSIONS_MAX 20
 
@@ -16,7 +17,6 @@
 #define TFTP_FILENAME_MAX 512
 
 struct tftp_t {
-  struct ip ip;
   struct udphdr udp;
   uint16_t tp_op;
   union {
@@ -30,20 +30,20 @@
     } tp_error;
     char tp_buf[512 + 2];
   } x;
-};
+} __attribute__((packed));
 
 struct tftp_session {
     Slirp *slirp;
     char *filename;
     int fd;
 
-    struct in_addr client_ip;
+    struct sockaddr_storage client_addr;
     uint16_t client_port;
     uint32_t block_nr;
 
     int timestamp;
 };
 
-void tftp_input(struct mbuf *m);
+void tftp_input(struct sockaddr_storage *srcsas, struct mbuf *m);
 
 #endif
diff --git a/slirp/udp.c b/slirp/udp.c
index f77e00f..93d7224 100644
--- a/slirp/udp.c
+++ b/slirp/udp.c
@@ -38,7 +38,8 @@
  * terms and conditions of the copyright.
  */
 
-#include <slirp.h>
+#include "qemu/osdep.h"
+#include "slirp.h"
 #include "ip_icmp.h"
 
 static uint8_t udp_tos(struct socket *so);
@@ -70,9 +71,11 @@
 	int len;
 	struct ip save_ip;
 	struct socket *so;
+	struct sockaddr_storage lhost;
+	struct sockaddr_in *lhost4;
 
 	DEBUG_CALL("udp_input");
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("iphlen = %d", iphlen);
 
 	/*
@@ -125,6 +128,11 @@
 	  }
 	}
 
+	lhost.ss_family = AF_INET;
+	lhost4 = (struct sockaddr_in *) &lhost;
+	lhost4->sin_addr = ip->ip_src;
+	lhost4->sin_port = uh->uh_sport;
+
         /*
          *  handle DHCP/BOOTP
          */
@@ -140,7 +148,11 @@
          */
         if (ntohs(uh->uh_dport) == TFTP_SERVER &&
             ip->ip_dst.s_addr == slirp->vhost_addr.s_addr) {
-            tftp_input(m);
+            m->m_data += iphlen;
+            m->m_len -= iphlen;
+            tftp_input(&lhost, m);
+            m->m_data -= iphlen;
+            m->m_len += iphlen;
             goto bad;
         }
 
@@ -151,25 +163,7 @@
 	/*
 	 * Locate pcb for datagram.
 	 */
-	so = slirp->udp_last_so;
-	if (so == &slirp->udb || so->so_lport != uh->uh_sport ||
-	    so->so_laddr.s_addr != ip->ip_src.s_addr) {
-		struct socket *tmp;
-
-		for (tmp = slirp->udb.so_next; tmp != &slirp->udb;
-		     tmp = tmp->so_next) {
-			if (tmp->so_lport == uh->uh_sport &&
-			    tmp->so_laddr.s_addr == ip->ip_src.s_addr) {
-				so = tmp;
-				break;
-			}
-		}
-		if (tmp == &slirp->udb) {
-		  so = NULL;
-		} else {
-		  slirp->udp_last_so = so;
-		}
-	}
+	so = solookup(&slirp->udp_last_so, &slirp->udb, &lhost, NULL);
 
 	if (so == NULL) {
 	  /*
@@ -180,7 +174,7 @@
 	  if (!so) {
 	      goto bad;
 	  }
-	  if(udp_attach(so) == -1) {
+	  if (udp_attach(so, AF_INET) == -1) {
 	    DEBUG_MISC((dfd," udp_attach errno = %d-%s\n",
 			errno,strerror(errno)));
 	    sofree(so);
@@ -190,6 +184,7 @@
 	  /*
 	   * Setup fields
 	   */
+	  so->so_lfamily = AF_INET;
 	  so->so_laddr = ip->ip_src;
 	  so->so_lport = uh->uh_sport;
 
@@ -202,6 +197,7 @@
 	   */
 	}
 
+        so->so_ffamily = AF_INET;
         so->so_faddr = ip->ip_dst; /* XXX */
         so->so_fport = uh->uh_dport; /* XXX */
 
@@ -217,7 +213,9 @@
 	  m->m_data -= iphlen;
 	  *ip=save_ip;
 	  DEBUG_MISC((dfd,"udp tx errno = %d-%s\n",errno,strerror(errno)));
-	  icmp_error(m, ICMP_UNREACH,ICMP_UNREACH_NET, 0,strerror(errno));
+	  icmp_send_error(m, ICMP_UNREACH, ICMP_UNREACH_NET, 0,
+	                  strerror(errno));
+	  goto bad;
 	}
 
 	m_free(so->so_m);   /* used for ICMP if error on sorecvfrom */
@@ -233,7 +231,7 @@
 	m_free(m);
 }
 
-int udp_output2(struct socket *so, struct mbuf *m,
+int udp_output(struct socket *so, struct mbuf *m,
                 struct sockaddr_in *saddr, struct sockaddr_in *daddr,
                 int iptos)
 {
@@ -241,8 +239,8 @@
 	int error = 0;
 
 	DEBUG_CALL("udp_output");
-	DEBUG_ARG("so = %lx", (long)so);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("so = %p", so);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("saddr = %lx", (long)saddr->sin_addr.s_addr);
 	DEBUG_ARG("daddr = %lx", (long)daddr->sin_addr.s_addr);
 
@@ -284,35 +282,11 @@
 	return (error);
 }
 
-int udp_output(struct socket *so, struct mbuf *m,
-               struct sockaddr_in *addr)
-
-{
-    Slirp *slirp = so->slirp;
-    struct sockaddr_in saddr, daddr;
-
-    saddr = *addr;
-    if ((so->so_faddr.s_addr & slirp->vnetwork_mask.s_addr) ==
-        slirp->vnetwork_addr.s_addr) {
-        uint32_t inv_mask = ~slirp->vnetwork_mask.s_addr;
-
-        if ((so->so_faddr.s_addr & inv_mask) == inv_mask) {
-            saddr.sin_addr = slirp->vhost_addr;
-        } else if (addr->sin_addr.s_addr == loopback_addr.s_addr ||
-                   so->so_faddr.s_addr != slirp->vhost_addr.s_addr) {
-            saddr.sin_addr = so->so_faddr;
-        }
-    }
-    daddr.sin_addr = so->so_laddr;
-    daddr.sin_port = so->so_lport;
-
-    return udp_output2(so, m, &saddr, &daddr, so->so_iptos);
-}
-
 int
-udp_attach(struct socket *so)
+udp_attach(struct socket *so, unsigned short af)
 {
-  if((so->s = qemu_socket(AF_INET,SOCK_DGRAM,0)) != -1) {
+  so->s = qemu_socket(af, SOCK_DGRAM, 0);
+  if (so->s != -1) {
     so->so_expire = curtime + SO_EXPIRE;
     insque(so, &so->slirp->udb);
   }
@@ -375,13 +349,9 @@
 	socket_set_fast_reuse(so->s);
 
 	getsockname(so->s,(struct sockaddr *)&addr,&addrlen);
-	so->so_fport = addr.sin_port;
-	if (addr.sin_addr.s_addr == 0 ||
-	    addr.sin_addr.s_addr == loopback_addr.s_addr) {
-	   so->so_faddr = slirp->vhost_addr;
-	} else {
-	   so->so_faddr = addr.sin_addr;
-	}
+	so->fhost.sin = addr;
+	sotranslate_accept(so);
+	so->so_lfamily = AF_INET;
 	so->so_lport = lport;
 	so->so_laddr.s_addr = laddr;
 	if (flags != SS_FACCEPTONCE)
diff --git a/slirp/udp.h b/slirp/udp.h
index 9bf31fe..be657cf 100644
--- a/slirp/udp.h
+++ b/slirp/udp.h
@@ -30,8 +30,8 @@
  * udp.h,v 1.3 1994/08/21 05:27:41 paul Exp
  */
 
-#ifndef _UDP_H_
-#define _UDP_H_
+#ifndef UDP_H
+#define UDP_H
 
 #define UDP_TTL 0x60
 #define UDP_UDPDATALEN 16192
@@ -76,12 +76,16 @@
 void udp_init(Slirp *);
 void udp_cleanup(Slirp *);
 void udp_input(register struct mbuf *, int);
-int udp_output(struct socket *, struct mbuf *, struct sockaddr_in *);
-int udp_attach(struct socket *);
+int udp_attach(struct socket *, unsigned short af);
 void udp_detach(struct socket *);
 struct socket * udp_listen(Slirp *, uint32_t, u_int, uint32_t, u_int,
                            int);
-int udp_output2(struct socket *so, struct mbuf *m,
+int udp_output(struct socket *so, struct mbuf *m,
                 struct sockaddr_in *saddr, struct sockaddr_in *daddr,
                 int iptos);
+
+void udp6_input(register struct mbuf *);
+int udp6_output(struct socket *so, struct mbuf *m,
+                struct sockaddr_in6 *saddr, struct sockaddr_in6 *daddr);
+
 #endif
diff --git a/slirp/udp6.c b/slirp/udp6.c
new file mode 100644
index 0000000..9fa314b
--- /dev/null
+++ b/slirp/udp6.c
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2013
+ * Guillaume Subiron
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "slirp.h"
+#include "udp.h"
+#include "dhcpv6.h"
+
+void udp6_input(struct mbuf *m)
+{
+    Slirp *slirp = m->slirp;
+    struct ip6 *ip, save_ip;
+    struct udphdr *uh;
+    int iphlen = sizeof(struct ip6);
+    int len;
+    struct socket *so;
+    struct sockaddr_in6 lhost;
+
+    DEBUG_CALL("udp6_input");
+    DEBUG_ARG("m = %lx", (long)m);
+
+    if (slirp->restricted) {
+        goto bad;
+    }
+
+    ip = mtod(m, struct ip6 *);
+    m->m_len -= iphlen;
+    m->m_data += iphlen;
+    uh = mtod(m, struct udphdr *);
+    m->m_len += iphlen;
+    m->m_data -= iphlen;
+
+    if (ip6_cksum(m)) {
+        goto bad;
+    }
+
+    len = ntohs((uint16_t)uh->uh_ulen);
+
+    /*
+     * Make mbuf data length reflect UDP length.
+     * If not enough data to reflect UDP length, drop.
+     */
+    if (ntohs(ip->ip_pl) != len) {
+        if (len > ntohs(ip->ip_pl)) {
+            goto bad;
+        }
+        m_adj(m, len - ntohs(ip->ip_pl));
+        ip->ip_pl = htons(len);
+    }
+
+    /*
+     * Save a copy of the IP header in case we want restore it
+     * for sending an ICMP error message in response.
+     */
+    save_ip = *ip;
+
+    /* Locate pcb for datagram. */
+    lhost.sin6_family = AF_INET6;
+    lhost.sin6_addr = ip->ip_src;
+    lhost.sin6_port = uh->uh_sport;
+
+    /* handle DHCPv6 */
+    if (ntohs(uh->uh_dport) == DHCPV6_SERVER_PORT &&
+        (in6_equal(&ip->ip_dst, &slirp->vhost_addr6) ||
+         in6_equal(&ip->ip_dst, &(struct in6_addr)ALLDHCP_MULTICAST))) {
+        m->m_data += iphlen;
+        m->m_len -= iphlen;
+        dhcpv6_input(&lhost, m);
+        m->m_data -= iphlen;
+        m->m_len += iphlen;
+        goto bad;
+    }
+
+    /* handle TFTP */
+    if (ntohs(uh->uh_dport) == TFTP_SERVER &&
+        !memcmp(ip->ip_dst.s6_addr, slirp->vhost_addr6.s6_addr, 16)) {
+        m->m_data += iphlen;
+        m->m_len -= iphlen;
+        tftp_input((struct sockaddr_storage *)&lhost, m);
+        m->m_data -= iphlen;
+        m->m_len += iphlen;
+        goto bad;
+    }
+
+    so = solookup(&slirp->udp_last_so, &slirp->udb,
+                  (struct sockaddr_storage *) &lhost, NULL);
+
+    if (so == NULL) {
+        /* If there's no socket for this packet, create one. */
+        so = socreate(slirp);
+        if (!so) {
+            goto bad;
+        }
+        if (udp_attach(so, AF_INET6) == -1) {
+            DEBUG_MISC((dfd, " udp6_attach errno = %d-%s\n",
+                        errno, strerror(errno)));
+            sofree(so);
+            goto bad;
+        }
+
+        /* Setup fields */
+        so->so_lfamily = AF_INET6;
+        so->so_laddr6 = ip->ip_src;
+        so->so_lport6 = uh->uh_sport;
+    }
+
+    so->so_ffamily = AF_INET6;
+    so->so_faddr6 = ip->ip_dst; /* XXX */
+    so->so_fport6 = uh->uh_dport; /* XXX */
+
+    iphlen += sizeof(struct udphdr);
+    m->m_len -= iphlen;
+    m->m_data += iphlen;
+
+    /*
+     * Now we sendto() the packet.
+     */
+    if (sosendto(so, m) == -1) {
+        m->m_len += iphlen;
+        m->m_data -= iphlen;
+        *ip = save_ip;
+        DEBUG_MISC((dfd, "udp tx errno = %d-%s\n", errno, strerror(errno)));
+        icmp6_send_error(m, ICMP6_UNREACH, ICMP6_UNREACH_NO_ROUTE);
+        goto bad;
+    }
+
+    m_free(so->so_m);   /* used for ICMP if error on sorecvfrom */
+
+    /* restore the orig mbuf packet */
+    m->m_len += iphlen;
+    m->m_data -= iphlen;
+    *ip = save_ip;
+    so->so_m = m;
+
+    return;
+bad:
+    m_free(m);
+}
+
+int udp6_output(struct socket *so, struct mbuf *m,
+        struct sockaddr_in6 *saddr, struct sockaddr_in6 *daddr)
+{
+    struct ip6 *ip;
+    struct udphdr *uh;
+
+    DEBUG_CALL("udp6_output");
+    DEBUG_ARG("so = %lx", (long)so);
+    DEBUG_ARG("m = %lx", (long)m);
+
+    /* adjust for header */
+    m->m_data -= sizeof(struct udphdr);
+    m->m_len += sizeof(struct udphdr);
+    uh = mtod(m, struct udphdr *);
+    m->m_data -= sizeof(struct ip6);
+    m->m_len += sizeof(struct ip6);
+    ip = mtod(m, struct ip6 *);
+
+    /* Build IP header */
+    ip->ip_pl = htons(m->m_len - sizeof(struct ip6));
+    ip->ip_nh = IPPROTO_UDP;
+    ip->ip_src = saddr->sin6_addr;
+    ip->ip_dst = daddr->sin6_addr;
+
+    /* Build UDP header */
+    uh->uh_sport = saddr->sin6_port;
+    uh->uh_dport = daddr->sin6_port;
+    uh->uh_ulen = ip->ip_pl;
+    uh->uh_sum = 0;
+    uh->uh_sum = ip6_cksum(m);
+    if (uh->uh_sum == 0) {
+        uh->uh_sum = 0xffff;
+    }
+
+    return ip6_output(so, m, 0);
+}
diff --git a/softmmu_template.h b/softmmu_template.h
index df3e171..c20d94e 100644
--- a/softmmu_template.h
+++ b/softmmu_template.h
@@ -116,56 +116,30 @@
 # define helper_te_st_name  helper_le_st_name
 #endif
 
-/* macro to check the victim tlb */
-#define VICTIM_TLB_HIT(ty)                                                    \
-({                                                                            \
-    /* we are about to do a page table walk. our last hope is the             \
-     * victim tlb. try to refill from the victim tlb before walking the       \
-     * page table. */                                                         \
-    int vidx;                                                                 \
-    hwaddr tmpiotlb;                                                          \
-    CPUTLBEntry tmptlb;                                                       \
-    for (vidx = CPU_VTLB_SIZE-1; vidx >= 0; --vidx) {                         \
-        if (env->tlb_v_table[mmu_idx][vidx].ty == (addr & TARGET_PAGE_MASK)) {\
-            /* found entry in victim tlb, swap tlb and iotlb */               \
-            tmptlb = env->tlb_table[mmu_idx][index];                          \
-            env->tlb_table[mmu_idx][index] = env->tlb_v_table[mmu_idx][vidx]; \
-            env->tlb_v_table[mmu_idx][vidx] = tmptlb;                         \
-            tmpiotlb = env->iotlb[mmu_idx][index];                            \
-            env->iotlb[mmu_idx][index] = env->iotlb_v[mmu_idx][vidx];         \
-            env->iotlb_v[mmu_idx][vidx] = tmpiotlb;                           \
-            break;                                                            \
-        }                                                                     \
-    }                                                                         \
-    /* return true when there is a vtlb hit, i.e. vidx >=0 */                 \
-    vidx >= 0;                                                                \
-})
-
 #ifndef SOFTMMU_CODE_ACCESS
 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
-                                              hwaddr physaddr,
+                                              CPUIOTLBEntry *iotlbentry,
                                               target_ulong addr,
                                               uintptr_t retaddr)
 {
     uint64_t val;
     CPUState *cpu = ENV_GET_CPU(env);
-    MemoryRegion *mr = iotlb_to_region(cpu->as, physaddr);
+    hwaddr physaddr = iotlbentry->addr;
+    MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
 
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
     cpu->mem_io_pc = retaddr;
-    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
+    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
         cpu_io_recompile(cpu, retaddr);
     }
 
     cpu->mem_io_vaddr = addr;
-    io_mem_read(mr, physaddr, &val, 1 << SHIFT);
+    memory_region_dispatch_read(mr, physaddr, &val, 1 << SHIFT,
+                                iotlbentry->attrs);
     return val;
 }
 #endif
 
-#ifdef SOFTMMU_CODE_ACCESS
-static __attribute__((unused))
-#endif
 WORD_TYPE helper_le_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
@@ -181,13 +155,13 @@
 
     if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {
         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
-                             mmu_idx, retaddr);
+                             mmu_idx, retaddr, DATA_SIZE);
     }
 
     /* If the TLB entry is for a different page, reload and try again.  */
     if ((addr & TARGET_PAGE_MASK)
          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        if (!VICTIM_TLB_HIT(ADDR_READ)) {
+        if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
                      mmu_idx, retaddr);
         }
@@ -196,15 +170,15 @@
 
     /* Handle an IO access.  */
     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
-        hwaddr ioaddr;
+        CPUIOTLBEntry *iotlbentry;
         if ((addr & (DATA_SIZE - 1)) != 0) {
             goto do_unaligned_access;
         }
-        ioaddr = env->iotlb[mmu_idx][index];
+        iotlbentry = &env->iotlb[mmu_idx][index];
 
         /* ??? Note that the io helpers always read data in the target
            byte ordering.  We should push the LE/BE request down into io.  */
-        res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
+        res = glue(io_read, SUFFIX)(env, iotlbentry, addr, retaddr);
         res = TGT_LE(res);
         return res;
     }
@@ -240,9 +214,6 @@
 }
 
 #if DATA_SIZE > 1
-#ifdef SOFTMMU_CODE_ACCESS
-static __attribute__((unused))
-#endif
 WORD_TYPE helper_be_ld_name(CPUArchState *env, target_ulong addr,
                             TCGMemOpIdx oi, uintptr_t retaddr)
 {
@@ -258,13 +229,13 @@
 
     if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {
         cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
-                             mmu_idx, retaddr);
+                             mmu_idx, retaddr, DATA_SIZE);
     }
 
     /* If the TLB entry is for a different page, reload and try again.  */
     if ((addr & TARGET_PAGE_MASK)
          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        if (!VICTIM_TLB_HIT(ADDR_READ)) {
+        if (!VICTIM_TLB_HIT(ADDR_READ, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
                      mmu_idx, retaddr);
         }
@@ -273,15 +244,15 @@
 
     /* Handle an IO access.  */
     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
-        hwaddr ioaddr;
+        CPUIOTLBEntry *iotlbentry;
         if ((addr & (DATA_SIZE - 1)) != 0) {
             goto do_unaligned_access;
         }
-        ioaddr = env->iotlb[mmu_idx][index];
+        iotlbentry = &env->iotlb[mmu_idx][index];
 
         /* ??? Note that the io helpers always read data in the target
            byte ordering.  We should push the LE/BE request down into io.  */
-        res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
+        res = glue(io_read, SUFFIX)(env, iotlbentry, addr, retaddr);
         res = TGT_BE(res);
         return res;
     }
@@ -313,14 +284,6 @@
 }
 #endif /* DATA_SIZE > 1 */
 
-DATA_TYPE
-glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
-                                         int mmu_idx)
-{
-    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
-    return helper_te_ld_name (env, addr, oi, GETRA());
-}
-
 #ifndef SOFTMMU_CODE_ACCESS
 
 /* Provide signed versions of the load routines as well.  We can of course
@@ -342,22 +305,24 @@
 #endif
 
 static inline void glue(io_write, SUFFIX)(CPUArchState *env,
-                                          hwaddr physaddr,
+                                          CPUIOTLBEntry *iotlbentry,
                                           DATA_TYPE val,
                                           target_ulong addr,
                                           uintptr_t retaddr)
 {
     CPUState *cpu = ENV_GET_CPU(env);
-    MemoryRegion *mr = iotlb_to_region(cpu->as, physaddr);
+    hwaddr physaddr = iotlbentry->addr;
+    MemoryRegion *mr = iotlb_to_region(cpu, physaddr, iotlbentry->attrs);
 
     physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
-    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu_can_do_io(cpu)) {
+    if (mr != &io_mem_rom && mr != &io_mem_notdirty && !cpu->can_do_io) {
         cpu_io_recompile(cpu, retaddr);
     }
 
     cpu->mem_io_vaddr = addr;
     cpu->mem_io_pc = retaddr;
-    io_mem_write(mr, physaddr, val, 1 << SHIFT);
+    memory_region_dispatch_write(mr, physaddr, val, 1 << SHIFT,
+                                 iotlbentry->attrs);
 }
 
 void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val,
@@ -374,13 +339,13 @@
 
     if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {
         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
-                             mmu_idx, retaddr);
+                             mmu_idx, retaddr, DATA_SIZE);
     }
 
     /* If the TLB entry is for a different page, reload and try again.  */
     if ((addr & TARGET_PAGE_MASK)
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        if (!VICTIM_TLB_HIT(addr_write)) {
+        if (!VICTIM_TLB_HIT(addr_write, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
         }
         tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
@@ -388,16 +353,16 @@
 
     /* Handle an IO access.  */
     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
-        hwaddr ioaddr;
+        CPUIOTLBEntry *iotlbentry;
         if ((addr & (DATA_SIZE - 1)) != 0) {
             goto do_unaligned_access;
         }
-        ioaddr = env->iotlb[mmu_idx][index];
+        iotlbentry = &env->iotlb[mmu_idx][index];
 
         /* ??? Note that the io helpers always read data in the target
            byte ordering.  We should push the LE/BE request down into io.  */
         val = TGT_LE(val);
-        glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
+        glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
         return;
     }
 
@@ -405,12 +370,25 @@
     if (DATA_SIZE > 1
         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                      >= TARGET_PAGE_SIZE)) {
-        int i;
+        int i, index2;
+        target_ulong page2, tlb_addr2;
     do_unaligned_access:
-        /* XXX: not efficient, but simple */
-        /* Note: relies on the fact that tlb_fill() does not remove the
-         * previous page from the TLB cache.  */
-        for (i = DATA_SIZE - 1; i >= 0; i--) {
+        /* Ensure the second page is in the TLB.  Note that the first page
+           is already guaranteed to be filled, and that the second page
+           cannot evict the first.  */
+        page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
+        index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+        tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
+        if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
+            && !VICTIM_TLB_HIT(addr_write, page2)) {
+            tlb_fill(ENV_GET_CPU(env), page2, MMU_DATA_STORE,
+                     mmu_idx, retaddr);
+        }
+
+        /* XXX: not efficient, but simple.  */
+        /* This loop must go in the forward direction to avoid issues
+           with self-modifying code in Windows 64-bit.  */
+        for (i = 0; i < DATA_SIZE; ++i) {
             /* Little-endian extract.  */
             uint8_t val8 = val >> (i * 8);
             /* Note the adjustment at the beginning of the function.
@@ -444,13 +422,13 @@
 
     if (a_bits > 0 && (addr & ((1 << a_bits) - 1)) != 0) {
         cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
-                             mmu_idx, retaddr);
+                             mmu_idx, retaddr, DATA_SIZE);
     }
 
     /* If the TLB entry is for a different page, reload and try again.  */
     if ((addr & TARGET_PAGE_MASK)
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        if (!VICTIM_TLB_HIT(addr_write)) {
+        if (!VICTIM_TLB_HIT(addr_write, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
         }
         tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
@@ -458,16 +436,16 @@
 
     /* Handle an IO access.  */
     if (unlikely(tlb_addr & ~TARGET_PAGE_MASK)) {
-        hwaddr ioaddr;
+        CPUIOTLBEntry *iotlbentry;
         if ((addr & (DATA_SIZE - 1)) != 0) {
             goto do_unaligned_access;
         }
-        ioaddr = env->iotlb[mmu_idx][index];
+        iotlbentry = &env->iotlb[mmu_idx][index];
 
         /* ??? Note that the io helpers always read data in the target
            byte ordering.  We should push the LE/BE request down into io.  */
         val = TGT_BE(val);
-        glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
+        glue(io_write, SUFFIX)(env, iotlbentry, val, addr, retaddr);
         return;
     }
 
@@ -475,12 +453,25 @@
     if (DATA_SIZE > 1
         && unlikely((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1
                      >= TARGET_PAGE_SIZE)) {
-        int i;
+        int i, index2;
+        target_ulong page2, tlb_addr2;
     do_unaligned_access:
+        /* Ensure the second page is in the TLB.  Note that the first page
+           is already guaranteed to be filled, and that the second page
+           cannot evict the first.  */
+        page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK;
+        index2 = (page2 >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+        tlb_addr2 = env->tlb_table[mmu_idx][index2].addr_write;
+        if (page2 != (tlb_addr2 & (TARGET_PAGE_MASK | TLB_INVALID_MASK))
+            && !VICTIM_TLB_HIT(addr_write, page2)) {
+            tlb_fill(ENV_GET_CPU(env), page2, MMU_DATA_STORE,
+                     mmu_idx, retaddr);
+        }
+
         /* XXX: not efficient, but simple */
-        /* Note: relies on the fact that tlb_fill() does not remove the
-         * previous page from the TLB cache.  */
-        for (i = DATA_SIZE - 1; i >= 0; i--) {
+        /* This loop must go in the forward direction to avoid issues
+           with self-modifying code.  */
+        for (i = 0; i < DATA_SIZE; ++i) {
             /* Big-endian extract.  */
             uint8_t val8 = val >> (((DATA_SIZE - 1) * 8) - (i * 8));
             /* Note the adjustment at the beginning of the function.
@@ -496,14 +487,6 @@
 }
 #endif /* DATA_SIZE > 1 */
 
-void
-glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
-                                         DATA_TYPE val, int mmu_idx)
-{
-    TCGMemOpIdx oi = make_memop_idx(SHIFT, mmu_idx);
-    helper_te_st_name(env, addr, val, oi, GETRA());
-}
-
 #if DATA_SIZE == 1
 /* Probe for whether the specified guest write access is permitted.
  * If it is not permitted then an exception will be taken in the same
@@ -520,7 +503,7 @@
     if ((addr & TARGET_PAGE_MASK)
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
         /* TLB entry is for a different page */
-        if (!VICTIM_TLB_HIT(addr_write)) {
+        if (!VICTIM_TLB_HIT(addr_write, addr)) {
             tlb_fill(ENV_GET_CPU(env), addr, MMU_DATA_STORE, mmu_idx, retaddr);
         }
     }
diff --git a/spice-qemu-char.c b/spice-qemu-char.c
index 8106e06..351fcaa 100644
--- a/spice-qemu-char.c
+++ b/spice-qemu-char.c
@@ -1,12 +1,10 @@
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "ui/qemu-spice.h"
 #include "sysemu/char.h"
 #include <spice.h>
-#include <spice-experimental.h>
 #include <spice/protocol.h>
 
-#include "qemu/osdep.h"
 
 typedef struct SpiceCharDriver {
     CharDriverState*      chr;
@@ -111,6 +109,9 @@
 #if SPICE_SERVER_VERSION >= 0x000c02
     .event              = vmc_event,
 #endif
+#if SPICE_SERVER_VERSION >= 0x000c06
+    .flags              = SPICE_CHAR_DEVICE_NOTIFY_WRITABLE,
+#endif
 };
 
 
@@ -159,7 +160,7 @@
     return func(NULL, G_IO_OUT, user_data);
 }
 
-GSourceFuncs SpiceCharSourceFuncs = {
+static GSourceFuncs SpiceCharSourceFuncs = {
     .prepare  = spice_char_source_prepare,
     .check    = spice_char_source_check,
     .dispatch = spice_char_source_dispatch,
@@ -170,7 +171,7 @@
     SpiceCharDriver *scd = chr->opaque;
     SpiceCharSource *src;
 
-    assert(cond == G_IO_OUT);
+    assert(cond & G_IO_OUT);
 
     src = (SpiceCharSource *)g_source_new(&SpiceCharSourceFuncs,
                                           sizeof(SpiceCharSource));
@@ -261,14 +262,26 @@
     fprintf(stderr, "\n");
 }
 
-static CharDriverState *chr_open(const char *subtype,
-    void (*set_fe_open)(struct CharDriverState *, int))
+static void spice_chr_accept_input(struct CharDriverState *chr)
+{
+    SpiceCharDriver *s = chr->opaque;
 
+    spice_server_char_device_wakeup(&s->sin);
+}
+
+static CharDriverState *chr_open(const char *subtype,
+                                 void (*set_fe_open)(struct CharDriverState *,
+                                                     int),
+                                 ChardevCommon *backend,
+                                 Error **errp)
 {
     CharDriverState *chr;
     SpiceCharDriver *s;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(backend, errp);
+    if (!chr) {
+        return NULL;
+    }
     s = g_malloc0(sizeof(SpiceCharDriver));
     s->chr = chr;
     s->active = false;
@@ -280,21 +293,23 @@
     chr->chr_set_fe_open = set_fe_open;
     chr->explicit_be_open = true;
     chr->chr_fe_event = spice_chr_fe_event;
+    chr->chr_accept_input = spice_chr_accept_input;
 
     QLIST_INSERT_HEAD(&spice_chars, s, next);
 
     return chr;
 }
 
-CharDriverState *qemu_chr_open_spice_vmc(const char *type)
+static CharDriverState *qemu_chr_open_spice_vmc(const char *id,
+                                                ChardevBackend *backend,
+                                                ChardevReturn *ret,
+                                                Error **errp)
 {
+    ChardevSpiceChannel *spicevmc = backend->u.spicevmc.data;
+    const char *type = spicevmc->type;
     const char **psubtype = spice_server_char_device_recognized_subtypes();
+    ChardevCommon *common = qapi_ChardevSpiceChannel_base(spicevmc);
 
-    if (type == NULL) {
-        fprintf(stderr, "spice-qemu-char: missing name parameter\n");
-        print_allowed_subtypes();
-        return NULL;
-    }
     for (; *psubtype != NULL; ++psubtype) {
         if (strcmp(type, *psubtype) == 0) {
             break;
@@ -306,12 +321,18 @@
         return NULL;
     }
 
-    return chr_open(type, spice_vmc_set_fe_open);
+    return chr_open(type, spice_vmc_set_fe_open, common, errp);
 }
 
 #if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name)
+static CharDriverState *qemu_chr_open_spice_port(const char *id,
+                                                 ChardevBackend *backend,
+                                                 ChardevReturn *ret,
+                                                 Error **errp)
 {
+    ChardevSpicePort *spiceport = backend->u.spiceport.data;
+    const char *name = spiceport->fqdn;
+    ChardevCommon *common = qapi_ChardevSpicePort_base(spiceport);
     CharDriverState *chr;
     SpiceCharDriver *s;
 
@@ -320,7 +341,10 @@
         return NULL;
     }
 
-    chr = chr_open("port", spice_port_set_fe_open);
+    chr = chr_open("port", spice_port_set_fe_open, common, errp);
+    if (!chr) {
+        return NULL;
+    }
     s = chr->opaque;
     s->sin.portname = g_strdup(name);
 
@@ -344,34 +368,38 @@
                                      Error **errp)
 {
     const char *name = qemu_opt_get(opts, "name");
+    ChardevSpiceChannel *spicevmc;
 
     if (name == NULL) {
         error_setg(errp, "chardev: spice channel: no name given");
         return;
     }
-    backend->spicevmc = g_new0(ChardevSpiceChannel, 1);
-    backend->spicevmc->type = g_strdup(name);
+    spicevmc = backend->u.spicevmc.data = g_new0(ChardevSpiceChannel, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevSpiceChannel_base(spicevmc));
+    spicevmc->type = g_strdup(name);
 }
 
 static void qemu_chr_parse_spice_port(QemuOpts *opts, ChardevBackend *backend,
                                       Error **errp)
 {
     const char *name = qemu_opt_get(opts, "name");
+    ChardevSpicePort *spiceport;
 
     if (name == NULL) {
         error_setg(errp, "chardev: spice port: no name given");
         return;
     }
-    backend->spiceport = g_new0(ChardevSpicePort, 1);
-    backend->spiceport->fqdn = g_strdup(name);
+    spiceport = backend->u.spiceport.data = g_new0(ChardevSpicePort, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevSpicePort_base(spiceport));
+    spiceport->fqdn = g_strdup(name);
 }
 
 static void register_types(void)
 {
     register_char_driver("spicevmc", CHARDEV_BACKEND_KIND_SPICEVMC,
-                         qemu_chr_parse_spice_vmc);
+                         qemu_chr_parse_spice_vmc, qemu_chr_open_spice_vmc);
     register_char_driver("spiceport", CHARDEV_BACKEND_KIND_SPICEPORT,
-                         qemu_chr_parse_spice_port);
+                         qemu_chr_parse_spice_port, qemu_chr_open_spice_port);
 }
 
 type_init(register_types);
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
index 5e347d0..85cde33 100644
--- a/stubs/Makefile.objs
+++ b/stubs/Makefile.objs
@@ -1,8 +1,7 @@
 stub-obj-y += arch-query-cpu-def.o
-stub-obj-y += bdrv-commit-all.o
-stub-obj-y += chr-baum-init.o
-stub-obj-y += chr-msmouse.o
-stub-obj-y += chr-testdev.o
+stub-obj-y += bdrv-next-monitor-owned.o
+stub-obj-y += blk-commit-all.o
+stub-obj-y += blockdev-close-all-bdrv-states.o
 stub-obj-y += clock-warp.o
 stub-obj-y += cpu-get-clock.o
 stub-obj-y += cpu-get-icount.o
@@ -21,22 +20,29 @@
 stub-obj-y += migr-blocker.o
 stub-obj-y += mon-is-qmp.o
 stub-obj-y += mon-printf.o
-stub-obj-y += mon-set-error.o
 stub-obj-y += monitor-init.o
 stub-obj-y += notify-event.o
-stub-obj-y += pci-drive-hot-add.o
-stub-obj-$(CONFIG_SPICE) += qemu-chr-open-spice.o
 stub-obj-y += qtest.o
+stub-obj-y += replay.o
+stub-obj-y += replay-user.o
 stub-obj-y += reset.o
 stub-obj-y += runstate-check.o
 stub-obj-y += set-fd-handler.o
 stub-obj-y += slirp.o
 stub-obj-y += sysbus.o
+stub-obj-y += trace-control.o
 stub-obj-y += uuid.o
-stub-obj-y += vc-init.o
 stub-obj-y += vm-stop.o
 stub-obj-y += vmstate.o
 stub-obj-$(CONFIG_WIN32) += fd-register.o
+stub-obj-$(CONFIG_WIN32) += win32-stubs.o
 stub-obj-y += cpus.o
 stub-obj-y += kvm.o
 stub-obj-y += qmp_pc_dimm_device_list.o
+stub-obj-y += target-monitor-defs.o
+stub-obj-y += target-get-monitor-def.o
+stub-obj-y += vhost.o
+stub-obj-y += iohandler.o
+stub-obj-y += smbios_type_38.o
+stub-obj-y += ipmi.o
+stub-obj-y += pc_madt_cpu_entry.o
diff --git a/stubs/arch-query-cpu-def.c b/stubs/arch-query-cpu-def.c
index 22e0b43..cefe4be 100644
--- a/stubs/arch-query-cpu-def.c
+++ b/stubs/arch-query-cpu-def.c
@@ -1,9 +1,10 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/arch_init.h"
 #include "qapi/qmp/qerror.h"
 
 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
 {
-    error_set(errp, QERR_UNSUPPORTED);
+    error_setg(errp, QERR_UNSUPPORTED);
     return NULL;
 }
diff --git a/stubs/bdrv-commit-all.c b/stubs/bdrv-commit-all.c
deleted file mode 100644
index a8e0a95..0000000
--- a/stubs/bdrv-commit-all.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "block/block.h"
-
-int bdrv_commit_all(void)
-{
-    return 0;
-}
diff --git a/stubs/bdrv-next-monitor-owned.c b/stubs/bdrv-next-monitor-owned.c
new file mode 100644
index 0000000..2acf6c3
--- /dev/null
+++ b/stubs/bdrv-next-monitor-owned.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "block/block.h"
+
+BlockDriverState *bdrv_next_monitor_owned(BlockDriverState *bs)
+{
+    return NULL;
+}
diff --git a/stubs/blk-commit-all.c b/stubs/blk-commit-all.c
new file mode 100644
index 0000000..c82fb7f
--- /dev/null
+++ b/stubs/blk-commit-all.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "sysemu/block-backend.h"
+
+int blk_commit_all(void)
+{
+    return 0;
+}
diff --git a/stubs/blockdev-close-all-bdrv-states.c b/stubs/blockdev-close-all-bdrv-states.c
new file mode 100644
index 0000000..f1f1d9c
--- /dev/null
+++ b/stubs/blockdev-close-all-bdrv-states.c
@@ -0,0 +1,6 @@
+#include "qemu/osdep.h"
+#include "block/block_int.h"
+
+void blockdev_close_all_bdrv_states(void)
+{
+}
diff --git a/stubs/chr-baum-init.c b/stubs/chr-baum-init.c
deleted file mode 100644
index f5cc6ce..0000000
--- a/stubs/chr-baum-init.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *chr_baum_init(void)
-{
-    return NULL;
-}
diff --git a/stubs/chr-msmouse.c b/stubs/chr-msmouse.c
deleted file mode 100644
index 812f8b0..0000000
--- a/stubs/chr-msmouse.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *qemu_chr_open_msmouse(void)
-{
-    return 0;
-}
diff --git a/stubs/chr-testdev.c b/stubs/chr-testdev.c
deleted file mode 100644
index 23112a2..0000000
--- a/stubs/chr-testdev.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *chr_testdev_init(void)
-{
-    return 0;
-}
diff --git a/stubs/clock-warp.c b/stubs/clock-warp.c
index 5565118..8acb58a 100644
--- a/stubs/clock-warp.c
+++ b/stubs/clock-warp.c
@@ -1,7 +1,8 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
-void qemu_clock_warp(QEMUClockType type)
+void qemu_start_warp_timer(void)
 {
 }
 
diff --git a/stubs/cpu-get-clock.c b/stubs/cpu-get-clock.c
index 5b34c97..1d07523 100644
--- a/stubs/cpu-get-clock.c
+++ b/stubs/cpu-get-clock.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
 
diff --git a/stubs/cpu-get-icount.c b/stubs/cpu-get-icount.c
index d685859..2e8b63b 100644
--- a/stubs/cpu-get-icount.c
+++ b/stubs/cpu-get-icount.c
@@ -1,5 +1,7 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "sysemu/cpus.h"
 
 int use_icount;
 
diff --git a/stubs/cpus.c b/stubs/cpus.c
index 8e6f06b..e192722 100644
--- a/stubs/cpus.c
+++ b/stubs/cpus.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qom/cpu.h"
 
diff --git a/stubs/dump.c b/stubs/dump.c
index fac7019..d9ee23f 100644
--- a/stubs/dump.c
+++ b/stubs/dump.c
@@ -11,9 +11,9 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/dump-arch.h"
-#include "qapi/qmp/qerror.h"
 #include "qmp-commands.h"
 
 int cpu_get_dump_info(ArchDumpInfo *info,
diff --git a/stubs/fd-register.c b/stubs/fd-register.c
index d0c34fd..f91aa34 100644
--- a/stubs/fd-register.c
+++ b/stubs/fd-register.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
diff --git a/stubs/fdset-add-fd.c b/stubs/fdset-add-fd.c
index ee16437..bf9e60a 100644
--- a/stubs/fdset-add-fd.c
+++ b/stubs/fdset-add-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/stubs/fdset-find-fd.c b/stubs/fdset-find-fd.c
index 4f18344..1d9caf3 100644
--- a/stubs/fdset-find-fd.c
+++ b/stubs/fdset-find-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/stubs/fdset-get-fd.c b/stubs/fdset-get-fd.c
index 7112c15..5325044 100644
--- a/stubs/fdset-get-fd.c
+++ b/stubs/fdset-get-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/stubs/fdset-remove-fd.c b/stubs/fdset-remove-fd.c
index 7f6d61e..47ea297 100644
--- a/stubs/fdset-remove-fd.c
+++ b/stubs/fdset-remove-fd.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/stubs/gdbstub.c b/stubs/gdbstub.c
index 7c40267..2b7aee5 100644
--- a/stubs/gdbstub.c
+++ b/stubs/gdbstub.c
@@ -1,5 +1,5 @@
-#include "stdbool.h"            /* bool (in exec/gdbstub.h) */
-#include "stddef.h"             /* NULL */
+#include "qemu/osdep.h"
+#include "exec/gdbstub.h"       /* xml_builtin */
 
 const char *const xml_builtin[][2] = {
   { NULL, NULL }
diff --git a/stubs/get-fd.c b/stubs/get-fd.c
index 9f2c65c..7dfdfb5 100644
--- a/stubs/get-fd.c
+++ b/stubs/get-fd.c
@@ -1,3 +1,5 @@
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/stubs/get-next-serial.c b/stubs/get-next-serial.c
index 40c56d1..6ff6a6d 100644
--- a/stubs/get-next-serial.c
+++ b/stubs/get-next-serial.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 CharDriverState *serial_hds[0];
diff --git a/stubs/get-vm-name.c b/stubs/get-vm-name.c
index e5f619f..fa99013 100644
--- a/stubs/get-vm-name.c
+++ b/stubs/get-vm-name.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 const char *qemu_get_vm_name(void)
diff --git a/stubs/iohandler.c b/stubs/iohandler.c
new file mode 100644
index 0000000..22b0ee5
--- /dev/null
+++ b/stubs/iohandler.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/main-loop.h"
+
+AioContext *iohandler_get_aio_context(void)
+{
+    abort();
+}
diff --git a/stubs/iothread-lock.c b/stubs/iothread-lock.c
index 5d8aca1..9b6db2e 100644
--- a/stubs/iothread-lock.c
+++ b/stubs/iothread-lock.c
@@ -1,6 +1,12 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
+bool qemu_mutex_iothread_locked(void)
+{
+    return true;
+}
+
 void qemu_mutex_lock_iothread(void)
 {
 }
diff --git a/stubs/ipmi.c b/stubs/ipmi.c
new file mode 100644
index 0000000..98b6dce
--- /dev/null
+++ b/stubs/ipmi.c
@@ -0,0 +1,14 @@
+/*
+ * IPMI ACPI firmware handling
+ *
+ * Copyright (c) 2015,2016 Corey Minyard, MontaVista Software, LLC
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "hw/acpi/ipmi.h"
+
+void build_acpi_ipmi_devices(Aml *table, BusState *bus)
+{
+}
diff --git a/stubs/is-daemonized.c b/stubs/is-daemonized.c
index c0ee917..d5cd1dc 100644
--- a/stubs/is-daemonized.c
+++ b/stubs/is-daemonized.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 /* Win32 has its own inline stub */
diff --git a/stubs/kvm.c b/stubs/kvm.c
index e7c60b6..ddd6204 100644
--- a/stubs/kvm.c
+++ b/stubs/kvm.c
@@ -1,7 +1,8 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/kvm.h"
 
-int kvm_arch_irqchip_create(KVMState *s)
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
 {
     return 0;
 }
diff --git a/stubs/machine-init-done.c b/stubs/machine-init-done.c
index 28a9255..9a0d625 100644
--- a/stubs/machine-init-done.c
+++ b/stubs/machine-init-done.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 
diff --git a/stubs/migr-blocker.c b/stubs/migr-blocker.c
index 300df6e..8ab3604 100644
--- a/stubs/migr-blocker.c
+++ b/stubs/migr-blocker.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "migration/migration.h"
 
diff --git a/stubs/mon-is-qmp.c b/stubs/mon-is-qmp.c
index 1f0a8fd..a8344ce 100644
--- a/stubs/mon-is-qmp.c
+++ b/stubs/mon-is-qmp.c
@@ -1,7 +1,10 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
-int monitor_cur_is_qmp(void)
+Monitor *cur_mon;
+
+bool monitor_cur_is_qmp(void)
 {
-    return 0;
+    return false;
 }
diff --git a/stubs/mon-printf.c b/stubs/mon-printf.c
index 0ce2ca6..e7c1e0c 100644
--- a/stubs/mon-printf.c
+++ b/stubs/mon-printf.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
diff --git a/stubs/mon-set-error.c b/stubs/mon-set-error.c
deleted file mode 100644
index d0411f9..0000000
--- a/stubs/mon-set-error.c
+++ /dev/null
@@ -1,8 +0,0 @@
-#include "qemu-common.h"
-#include "monitor/monitor.h"
-
-Monitor *cur_mon;
-
-void monitor_set_error(Monitor *mon, QError *qerror)
-{
-}
diff --git a/stubs/monitor-init.c b/stubs/monitor-init.c
index 83ac5c1..de1bc7c 100644
--- a/stubs/monitor-init.c
+++ b/stubs/monitor-init.c
@@ -1,7 +1,7 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "monitor/monitor.h"
 
-Monitor * monitor_init(CharDriverState *chr, int flags)
+void monitor_init(CharDriverState *chr, int flags)
 {
-    return NULL;
 }
diff --git a/stubs/notify-event.c b/stubs/notify-event.c
index 32f7289..14e5226 100644
--- a/stubs/notify-event.c
+++ b/stubs/notify-event.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
diff --git a/stubs/pc_madt_cpu_entry.c b/stubs/pc_madt_cpu_entry.c
new file mode 100644
index 0000000..427e772
--- /dev/null
+++ b/stubs/pc_madt_cpu_entry.c
@@ -0,0 +1,7 @@
+#include "qemu/osdep.h"
+#include "hw/i386/pc.h"
+
+void pc_madt_cpu_entry(AcpiDeviceIf *adev, int uid,
+                       CPUArchIdList *apic_ids, GArray *entry)
+{
+}
diff --git a/stubs/pci-drive-hot-add.c b/stubs/pci-drive-hot-add.c
deleted file mode 100644
index 1d98145..0000000
--- a/stubs/pci-drive-hot-add.c
+++ /dev/null
@@ -1,10 +0,0 @@
-#include <monitor/monitor.h>
-#include <sysemu/sysemu.h>
-#include <sysemu/blockdev.h>
-
-int pci_drive_hot_add(Monitor *mon, const QDict *qdict, DriveInfo *dinfo)
-{
-    /* On non-x86 we don't do PCI hotplug */
-    monitor_printf(mon, "Can't hot-add drive to type %d\n", dinfo->type);
-    return -1;
-}
diff --git a/stubs/qemu-chr-open-spice.c b/stubs/qemu-chr-open-spice.c
deleted file mode 100644
index f1c4849..0000000
--- a/stubs/qemu-chr-open-spice.c
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "qemu-common.h"
-#include "ui/qemu-spice.h"
-
-CharDriverState *qemu_chr_open_spice_vmc(const char *type)
-{
-    return NULL;
-}
-
-#if SPICE_SERVER_VERSION >= 0x000c02
-CharDriverState *qemu_chr_open_spice_port(const char *name)
-{
-    return NULL;
-}
-#endif
diff --git a/stubs/qmp_pc_dimm_device_list.c b/stubs/qmp_pc_dimm_device_list.c
index 5cb220c..def2115 100644
--- a/stubs/qmp_pc_dimm_device_list.c
+++ b/stubs/qmp_pc_dimm_device_list.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qom/object.h"
 #include "hw/mem/pc-dimm.h"
 
diff --git a/stubs/qtest.c b/stubs/qtest.c
index e671ed8..891eb95 100644
--- a/stubs/qtest.c
+++ b/stubs/qtest.c
@@ -8,7 +8,13 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "sysemu/qtest.h"
 
 /* Needed for qtest_allowed() */
 bool qtest_allowed;
+
+bool qtest_driver(void)
+{
+    return false;
+}
diff --git a/stubs/replay-user.c b/stubs/replay-user.c
new file mode 100644
index 0000000..b29e7eb
--- /dev/null
+++ b/stubs/replay-user.c
@@ -0,0 +1,33 @@
+/*
+ * replay.c
+ *
+ * Copyright (c) 2010-2015 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "sysemu/replay.h"
+
+bool replay_exception(void)
+{
+    return true;
+}
+
+bool replay_has_exception(void)
+{
+    return false;
+}
+
+bool replay_interrupt(void)
+{
+    return true;
+}
+
+bool replay_has_interrupt(void)
+{
+    return false;
+}
diff --git a/stubs/replay.c b/stubs/replay.c
new file mode 100644
index 0000000..de9fa1e
--- /dev/null
+++ b/stubs/replay.c
@@ -0,0 +1,69 @@
+#include "qemu/osdep.h"
+#include "sysemu/replay.h"
+#include "sysemu/sysemu.h"
+
+ReplayMode replay_mode;
+
+int64_t replay_save_clock(unsigned int kind, int64_t clock)
+{
+    abort();
+    return 0;
+}
+
+int64_t replay_read_clock(unsigned int kind)
+{
+    abort();
+    return 0;
+}
+
+bool replay_checkpoint(ReplayCheckpoint checkpoint)
+{
+    return true;
+}
+
+bool replay_events_enabled(void)
+{
+    return false;
+}
+
+void replay_finish(void)
+{
+}
+
+void replay_register_char_driver(CharDriverState *chr)
+{
+}
+
+void replay_chr_be_write(CharDriverState *s, uint8_t *buf, int len)
+{
+    abort();
+}
+
+void replay_char_write_event_save(int res, int offset)
+{
+    abort();
+}
+
+void replay_char_write_event_load(int *res, int *offset)
+{
+    abort();
+}
+
+int replay_char_read_all_load(uint8_t *buf)
+{
+    abort();
+}
+
+void replay_char_read_all_save_error(int res)
+{
+    abort();
+}
+
+void replay_char_read_all_save_buf(uint8_t *buf, int offset)
+{
+    abort();
+}
+
+void replay_block_event(QEMUBH *bh, uint64_t id)
+{
+}
diff --git a/stubs/reset.c b/stubs/reset.c
index ad28725..5d47711 100644
--- a/stubs/reset.c
+++ b/stubs/reset.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 
 /* Stub functions for binaries that never call qemu_devices_reset(),
diff --git a/stubs/runstate-check.c b/stubs/runstate-check.c
index bd2e375..7c5227e 100644
--- a/stubs/runstate-check.c
+++ b/stubs/runstate-check.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 
 bool runstate_check(RunState state)
diff --git a/stubs/set-fd-handler.c b/stubs/set-fd-handler.c
index fc874d3..06a5da4 100644
--- a/stubs/set-fd-handler.c
+++ b/stubs/set-fd-handler.c
@@ -1,11 +1,21 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/main-loop.h"
 
-int qemu_set_fd_handler2(int fd,
-                         IOCanReadHandler *fd_read_poll,
+void qemu_set_fd_handler(int fd,
                          IOHandler *fd_read,
                          IOHandler *fd_write,
                          void *opaque)
 {
     abort();
 }
+
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        bool is_external,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        void *opaque)
+{
+    abort();
+}
diff --git a/stubs/slirp.c b/stubs/slirp.c
index bd0ac7f..42f7e1a 100644
--- a/stubs/slirp.c
+++ b/stubs/slirp.c
@@ -1,4 +1,6 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/host-utils.h"
 #include "slirp/slirp.h"
 
 void slirp_pollfds_fill(GArray *pollfds, uint32_t *timeout)
diff --git a/stubs/smbios_type_38.c b/stubs/smbios_type_38.c
new file mode 100644
index 0000000..9528c2c
--- /dev/null
+++ b/stubs/smbios_type_38.c
@@ -0,0 +1,14 @@
+/*
+ * IPMI SMBIOS firmware handling
+ *
+ * Copyright (c) 2015,2016 Corey Minyard, MontaVista Software, LLC
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "hw/smbios/ipmi.h"
+
+void smbios_build_type_38_table(void)
+{
+}
diff --git a/stubs/sysbus.c b/stubs/sysbus.c
index e134965..d8da90c 100644
--- a/stubs/sysbus.c
+++ b/stubs/sysbus.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "hw/qdev-core.h"
 
 BusState *sysbus_get_default(void)
diff --git a/stubs/target-get-monitor-def.c b/stubs/target-get-monitor-def.c
new file mode 100644
index 0000000..394e0f9
--- /dev/null
+++ b/stubs/target-get-monitor-def.c
@@ -0,0 +1,29 @@
+/*
+ *  Stub for target_get_monitor_def.
+ *
+ *  Copyright IBM Corp., 2015
+ *
+ *  Author: Alexey Kardashevskiy <aik@ozlabs.ru>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License,
+ *  or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval);
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval)
+{
+    return -1;
+}
diff --git a/stubs/target-monitor-defs.c b/stubs/target-monitor-defs.c
new file mode 100644
index 0000000..ac07b19
--- /dev/null
+++ b/stubs/target-monitor-defs.c
@@ -0,0 +1,8 @@
+#include "qemu/osdep.h"
+
+const MonitorDef *target_monitor_defs(void);
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return NULL;
+}
diff --git a/stubs/trace-control.c b/stubs/trace-control.c
new file mode 100644
index 0000000..fe59836
--- /dev/null
+++ b/stubs/trace-control.c
@@ -0,0 +1,28 @@
+/*
+ * Interface for configuring and controlling the state of tracing events.
+ *
+ * Copyright (C) 2014-2016 Lluís Vilanova <vilanova@ac.upc.edu>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "trace/control.h"
+
+
+void trace_event_set_state_dynamic(TraceEvent *ev, bool state)
+{
+    TraceEventID id;
+    assert(trace_event_get_state_static(ev));
+    id = trace_event_get_id(ev);
+    trace_events_enabled_count += state - trace_events_dstate[id];
+    trace_events_dstate[id] = state;
+}
+
+void trace_event_set_vcpu_state_dynamic(CPUState *vcpu,
+                                        TraceEvent *ev, bool state)
+{
+    /* should never be called on non-target binaries */
+    abort();
+}
diff --git a/stubs/uuid.c b/stubs/uuid.c
index ffc0ed4..92ad717 100644
--- a/stubs/uuid.c
+++ b/stubs/uuid.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
diff --git a/stubs/vc-init.c b/stubs/vc-init.c
deleted file mode 100644
index 308dfa0..0000000
--- a/stubs/vc-init.c
+++ /dev/null
@@ -1,7 +0,0 @@
-#include "qemu-common.h"
-#include "sysemu/char.h"
-
-CharDriverState *vc_init(ChardevVC *vc)
-{
-    return 0;
-}
diff --git a/stubs/vhost.c b/stubs/vhost.c
new file mode 100644
index 0000000..2d76cde
--- /dev/null
+++ b/stubs/vhost.c
@@ -0,0 +1,7 @@
+#include "qemu/osdep.h"
+#include "hw/virtio/vhost.h"
+
+bool vhost_has_free_slot(void)
+{
+    return true;
+}
diff --git a/stubs/vm-stop.c b/stubs/vm-stop.c
index 69fd86b..8271cad 100644
--- a/stubs/vm-stop.c
+++ b/stubs/vm-stop.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "sysemu/sysemu.h"
 
diff --git a/stubs/vmstate.c b/stubs/vmstate.c
index 778bc3f..6590627 100644
--- a/stubs/vmstate.c
+++ b/stubs/vmstate.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
 
diff --git a/stubs/win32-stubs.c b/stubs/win32-stubs.c
new file mode 100644
index 0000000..e7a33a3
--- /dev/null
+++ b/stubs/win32-stubs.c
@@ -0,0 +1,31 @@
+#include "qemu/osdep.h"
+
+/* NOTE: This file should not be linked against the Android emulator binaries */
+HANDLE win32CreateFile(
+        LPCTSTR               lpFileName,
+        DWORD                 dwDesiredAccess,
+        DWORD                 dwShareMode,
+        LPSECURITY_ATTRIBUTES lpSecurityAttributes,
+        DWORD                 dwCreationDisposition,
+        DWORD                 dwFlagsAndAttributes,
+        HANDLE                hTemplateFile)
+{
+    return CreateFile(lpFileName, dwDesiredAccess, dwShareMode,
+                      lpSecurityAttributes, dwCreationDisposition,
+                      dwFlagsAndAttributes, hTemplateFile);
+}
+
+DWORD win32GetCurrentDirectory(
+        DWORD  nBufferLength,
+        LPTSTR lpBuffer)
+{
+    return GetCurrentDirectory(nBufferLength, lpBuffer);
+}
+
+DWORD win32GetModuleFileName(
+        HMODULE hModule,
+        LPTSTR  lpFilename,
+        DWORD   nSize)
+{
+    return GetModuleFileName(hModule, lpFilename, nSize);
+}
diff --git a/target-alpha/Makefile.objs b/target-alpha/Makefile.objs
index b96c5da..6366462 100644
--- a/target-alpha/Makefile.objs
+++ b/target-alpha/Makefile.objs
@@ -1,4 +1,4 @@
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-y += translate.o helper.o cpu.o
-obj-y += int_helper.o fpu_helper.o sys_helper.o mem_helper.o
+obj-y += int_helper.o fpu_helper.o vax_helper.o sys_helper.o mem_helper.o
 obj-y += gdbstub.o
diff --git a/target-alpha/cpu-qom.h b/target-alpha/cpu-qom.h
index b01c6c8..bae4945 100644
--- a/target-alpha/cpu-qom.h
+++ b/target-alpha/cpu-qom.h
@@ -21,7 +21,6 @@
 #define QEMU_ALPHA_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #define TYPE_ALPHA_CPU "alpha-cpu"
 
@@ -48,44 +47,6 @@
     void (*parent_reset)(CPUState *cpu);
 } AlphaCPUClass;
 
-/**
- * AlphaCPU:
- * @env: #CPUAlphaState
- *
- * An Alpha CPU.
- */
-typedef struct AlphaCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUAlphaState env;
-
-    /* This alarm doesn't exist in real hardware; we wish it did.  */
-    QEMUTimer *alarm_timer;
-} AlphaCPU;
-
-static inline AlphaCPU *alpha_env_get_cpu(CPUAlphaState *env)
-{
-    return container_of(env, AlphaCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(alpha_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(AlphaCPU, env)
-
-#ifndef CONFIG_USER_ONLY
-extern const struct VMStateDescription vmstate_alpha_cpu;
-#endif
-
-void alpha_cpu_do_interrupt(CPUState *cpu);
-bool alpha_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void alpha_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
-                          int flags);
-hwaddr alpha_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int alpha_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int alpha_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-void alpha_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
-                                   int is_write, int is_user, uintptr_t retaddr);
+typedef struct AlphaCPU AlphaCPU;
 
 #endif
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index a98b7d8..6d01d7f 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -19,9 +19,12 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
+#include "exec/exec-all.h"
 
 
 static void alpha_cpu_set_pc(CPUState *cs, vaddr value)
@@ -46,6 +49,12 @@
                                     | CPU_INTERRUPT_MCHK);
 }
 
+static void alpha_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_alpha_ev6;
+    info->print_insn = print_insn_alpha;
+}
+
 static void alpha_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -257,7 +266,7 @@
     CPUAlphaState *env = &cpu->env;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
     tlb_flush(cs, 1);
 
     alpha_translate_init();
@@ -297,7 +306,16 @@
     cc->get_phys_page_debug = alpha_cpu_get_phys_page_debug;
     dc->vmsd = &vmstate_alpha_cpu;
 #endif
+    cc->disas_set_info = alpha_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = 67;
+
+    /*
+     * Reason: alpha_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo alpha_cpu_type_info = {
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index d9b861f..f6150c8 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -17,11 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#if !defined (__CPU_ALPHA_H__)
-#define __CPU_ALPHA_H__
+#ifndef ALPHA_CPU_H
+#define ALPHA_CPU_H
 
-#include "config.h"
 #include "qemu-common.h"
+#include "cpu-qom.h"
 
 #define TARGET_LONG_BITS 64
 #define ALIGNED_ONLY
@@ -32,10 +32,6 @@
 
 #include "fpu/softfloat.h"
 
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE     EM_ALPHA
-
 #define ICACHE_LINE_SIZE 32
 #define DCACHE_LINE_SIZE 32
 
@@ -152,54 +148,54 @@
     FP_ROUND_DYNAMIC = 0x3,
 };
 
-/* FPCR bits */
-#define FPCR_SUM		(1ULL << 63)
-#define FPCR_INED		(1ULL << 62)
-#define FPCR_UNFD		(1ULL << 61)
-#define FPCR_UNDZ		(1ULL << 60)
-#define FPCR_DYN_SHIFT		58
-#define FPCR_DYN_CHOPPED	(0ULL << FPCR_DYN_SHIFT)
-#define FPCR_DYN_MINUS		(1ULL << FPCR_DYN_SHIFT)
-#define FPCR_DYN_NORMAL		(2ULL << FPCR_DYN_SHIFT)
-#define FPCR_DYN_PLUS		(3ULL << FPCR_DYN_SHIFT)
-#define FPCR_DYN_MASK		(3ULL << FPCR_DYN_SHIFT)
-#define FPCR_IOV		(1ULL << 57)
-#define FPCR_INE		(1ULL << 56)
-#define FPCR_UNF		(1ULL << 55)
-#define FPCR_OVF		(1ULL << 54)
-#define FPCR_DZE		(1ULL << 53)
-#define FPCR_INV		(1ULL << 52)
-#define FPCR_OVFD		(1ULL << 51)
-#define FPCR_DZED		(1ULL << 50)
-#define FPCR_INVD		(1ULL << 49)
-#define FPCR_DNZ		(1ULL << 48)
-#define FPCR_DNOD		(1ULL << 47)
-#define FPCR_STATUS_MASK	(FPCR_IOV | FPCR_INE | FPCR_UNF \
-				 | FPCR_OVF | FPCR_DZE | FPCR_INV)
+/* FPCR bits -- right-shifted 32 so we can use a uint32_t.  */
+#define FPCR_SUM                (1U << (63 - 32))
+#define FPCR_INED               (1U << (62 - 32))
+#define FPCR_UNFD               (1U << (61 - 32))
+#define FPCR_UNDZ               (1U << (60 - 32))
+#define FPCR_DYN_SHIFT          (58 - 32)
+#define FPCR_DYN_CHOPPED        (0U << FPCR_DYN_SHIFT)
+#define FPCR_DYN_MINUS          (1U << FPCR_DYN_SHIFT)
+#define FPCR_DYN_NORMAL         (2U << FPCR_DYN_SHIFT)
+#define FPCR_DYN_PLUS           (3U << FPCR_DYN_SHIFT)
+#define FPCR_DYN_MASK           (3U << FPCR_DYN_SHIFT)
+#define FPCR_IOV                (1U << (57 - 32))
+#define FPCR_INE                (1U << (56 - 32))
+#define FPCR_UNF                (1U << (55 - 32))
+#define FPCR_OVF                (1U << (54 - 32))
+#define FPCR_DZE                (1U << (53 - 32))
+#define FPCR_INV                (1U << (52 - 32))
+#define FPCR_OVFD               (1U << (51 - 32))
+#define FPCR_DZED               (1U << (50 - 32))
+#define FPCR_INVD               (1U << (49 - 32))
+#define FPCR_DNZ                (1U << (48 - 32))
+#define FPCR_DNOD               (1U << (47 - 32))
+#define FPCR_STATUS_MASK        (FPCR_IOV | FPCR_INE | FPCR_UNF \
+                                 | FPCR_OVF | FPCR_DZE | FPCR_INV)
 
 /* The silly software trap enables implemented by the kernel emulation.
    These are more or less architecturally required, since the real hardware
    has read-as-zero bits in the FPCR when the features aren't implemented.
    For the purposes of QEMU, we pretend the FPCR can hold everything.  */
-#define SWCR_TRAP_ENABLE_INV	(1ULL << 1)
-#define SWCR_TRAP_ENABLE_DZE	(1ULL << 2)
-#define SWCR_TRAP_ENABLE_OVF	(1ULL << 3)
-#define SWCR_TRAP_ENABLE_UNF	(1ULL << 4)
-#define SWCR_TRAP_ENABLE_INE	(1ULL << 5)
-#define SWCR_TRAP_ENABLE_DNO	(1ULL << 6)
-#define SWCR_TRAP_ENABLE_MASK	((1ULL << 7) - (1ULL << 1))
+#define SWCR_TRAP_ENABLE_INV    (1U << 1)
+#define SWCR_TRAP_ENABLE_DZE    (1U << 2)
+#define SWCR_TRAP_ENABLE_OVF    (1U << 3)
+#define SWCR_TRAP_ENABLE_UNF    (1U << 4)
+#define SWCR_TRAP_ENABLE_INE    (1U << 5)
+#define SWCR_TRAP_ENABLE_DNO    (1U << 6)
+#define SWCR_TRAP_ENABLE_MASK   ((1U << 7) - (1U << 1))
 
-#define SWCR_MAP_DMZ		(1ULL << 12)
-#define SWCR_MAP_UMZ		(1ULL << 13)
-#define SWCR_MAP_MASK		(SWCR_MAP_DMZ | SWCR_MAP_UMZ)
+#define SWCR_MAP_DMZ            (1U << 12)
+#define SWCR_MAP_UMZ            (1U << 13)
+#define SWCR_MAP_MASK           (SWCR_MAP_DMZ | SWCR_MAP_UMZ)
 
-#define SWCR_STATUS_INV		(1ULL << 17)
-#define SWCR_STATUS_DZE		(1ULL << 18)
-#define SWCR_STATUS_OVF		(1ULL << 19)
-#define SWCR_STATUS_UNF		(1ULL << 20)
-#define SWCR_STATUS_INE		(1ULL << 21)
-#define SWCR_STATUS_DNO		(1ULL << 22)
-#define SWCR_STATUS_MASK	((1ULL << 23) - (1ULL << 17))
+#define SWCR_STATUS_INV         (1U << 17)
+#define SWCR_STATUS_DZE         (1U << 18)
+#define SWCR_STATUS_OVF         (1U << 19)
+#define SWCR_STATUS_UNF         (1U << 20)
+#define SWCR_STATUS_INE         (1U << 21)
+#define SWCR_STATUS_DNO         (1U << 22)
+#define SWCR_STATUS_MASK        ((1U << 23) - (1U << 17))
 
 #define SWCR_MASK  (SWCR_TRAP_ENABLE_MASK | SWCR_MAP_MASK | SWCR_STATUS_MASK)
 
@@ -240,14 +236,13 @@
     uint64_t lock_addr;
     uint64_t lock_st_addr;
     uint64_t lock_value;
+
+    /* The FPCR, and disassembled portions thereof.  */
+    uint32_t fpcr;
+    uint32_t fpcr_exc_enable;
     float_status fp_status;
-    /* The following fields make up the FPCR, but in FP_STATUS format.  */
-    uint8_t fpcr_exc_status;
-    uint8_t fpcr_exc_mask;
     uint8_t fpcr_dyn_round;
     uint8_t fpcr_flush_to_zero;
-    uint8_t fpcr_dnod;
-    uint8_t fpcr_undz;
 
     /* The Internal Processor Registers.  Some of these we assume always
        exist for use in user-mode.  */
@@ -290,13 +285,52 @@
     int implver;
 };
 
+/**
+ * AlphaCPU:
+ * @env: #CPUAlphaState
+ *
+ * An Alpha CPU.
+ */
+struct AlphaCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUAlphaState env;
+
+    /* This alarm doesn't exist in real hardware; we wish it did.  */
+    QEMUTimer *alarm_timer;
+};
+
+static inline AlphaCPU *alpha_env_get_cpu(CPUAlphaState *env)
+{
+    return container_of(env, AlphaCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(alpha_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(AlphaCPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_alpha_cpu;
+#endif
+
+void alpha_cpu_do_interrupt(CPUState *cpu);
+bool alpha_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void alpha_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
+                          int flags);
+hwaddr alpha_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int alpha_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int alpha_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void alpha_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                   MMUAccessType access_type,
+                                   int mmu_idx, uintptr_t retaddr,
+                                   unsigned size);
+
 #define cpu_list alpha_cpu_list
-#define cpu_exec cpu_alpha_exec
-#define cpu_gen_code cpu_alpha_gen_code
 #define cpu_signal_handler cpu_alpha_signal_handler
 
 #include "exec/cpu-all.h"
-#include "cpu-qom.h"
 
 enum {
     FEATURE_ASN    = 0x00000001,
@@ -379,7 +413,7 @@
     PS_USER_MODE = 8
 };
 
-static inline int cpu_mmu_index(CPUAlphaState *env)
+static inline int cpu_mmu_index(CPUAlphaState *env, bool ifetch)
 {
     if (env->pal_mode) {
         return MMU_KERNEL_IDX;
@@ -431,17 +465,9 @@
 
 AlphaCPU *cpu_alpha_init(const char *cpu_model);
 
-static inline CPUAlphaState *cpu_init(const char *cpu_model)
-{
-    AlphaCPU *cpu = cpu_alpha_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_alpha_init(cpu_model))
 
 void alpha_cpu_list(FILE *f, fprintf_function cpu_fprintf);
-int cpu_alpha_exec(CPUAlphaState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
@@ -455,8 +481,9 @@
 
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env);
 void cpu_alpha_store_fpcr (CPUAlphaState *env, uint64_t val);
+uint64_t cpu_alpha_load_gr(CPUAlphaState *env, unsigned reg);
+void cpu_alpha_store_gr(CPUAlphaState *env, unsigned reg, uint64_t val);
 #ifndef CONFIG_USER_ONLY
-void swap_shadow_regs(CPUAlphaState *env);
 QEMU_NORETURN void alpha_cpu_unassigned_access(CPUState *cpu, hwaddr addr,
                                                bool is_write, bool is_exec,
                                                int unused, unsigned size);
@@ -478,7 +505,7 @@
 };
 
 static inline void cpu_get_tb_cpu_state(CPUAlphaState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *pflags)
+                                        target_ulong *cs_base, uint32_t *pflags)
 {
     int flags = 0;
 
@@ -498,6 +525,4 @@
     *pflags = flags;
 }
 
-#include "exec/exec-all.h"
-
-#endif /* !defined (__CPU_ALPHA_H__) */
+#endif /* ALPHA_CPU_H */
diff --git a/target-alpha/fpu_helper.c b/target-alpha/fpu_helper.c
index d2d776c..9645978 100644
--- a/target-alpha/fpu_helper.c
+++ b/target-alpha/fpu_helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 #include "fpu/softfloat.h"
 
@@ -34,57 +36,65 @@
     set_flush_to_zero(val, &FP_STATUS);
 }
 
-void helper_fp_exc_clear(CPUAlphaState *env)
+#define CONVERT_BIT(X, SRC, DST) \
+    (SRC > DST ? (X) / (SRC / DST) & (DST) : ((X) & SRC) * (DST / SRC))
+
+static uint32_t soft_to_fpcr_exc(CPUAlphaState *env)
 {
-    set_float_exception_flags(0, &FP_STATUS);
-}
+    uint8_t exc = get_float_exception_flags(&FP_STATUS);
+    uint32_t ret = 0;
 
-uint32_t helper_fp_exc_get(CPUAlphaState *env)
-{
-    return get_float_exception_flags(&FP_STATUS);
-}
-
-static inline void inline_fp_exc_raise(CPUAlphaState *env, uintptr_t retaddr,
-                                       uint32_t exc, uint32_t regno)
-{
-    if (exc) {
-        uint32_t hw_exc = 0;
-
-        if (exc & float_flag_invalid) {
-            hw_exc |= EXC_M_INV;
-        }
-        if (exc & float_flag_divbyzero) {
-            hw_exc |= EXC_M_DZE;
-        }
-        if (exc & float_flag_overflow) {
-            hw_exc |= EXC_M_FOV;
-        }
-        if (exc & float_flag_underflow) {
-            hw_exc |= EXC_M_UNF;
-        }
-        if (exc & float_flag_inexact) {
-            hw_exc |= EXC_M_INE;
-        }
-
-        arith_excp(env, retaddr, hw_exc, 1ull << regno);
+    if (unlikely(exc)) {
+        set_float_exception_flags(0, &FP_STATUS);
+        ret |= CONVERT_BIT(exc, float_flag_invalid, FPCR_INV);
+        ret |= CONVERT_BIT(exc, float_flag_divbyzero, FPCR_DZE);
+        ret |= CONVERT_BIT(exc, float_flag_overflow, FPCR_OVF);
+        ret |= CONVERT_BIT(exc, float_flag_underflow, FPCR_UNF);
+        ret |= CONVERT_BIT(exc, float_flag_inexact, FPCR_INE);
     }
+
+    return ret;
+}
+
+static void fp_exc_raise1(CPUAlphaState *env, uintptr_t retaddr,
+                          uint32_t exc, uint32_t regno, uint32_t hw_exc)
+{
+    hw_exc |= CONVERT_BIT(exc, FPCR_INV, EXC_M_INV);
+    hw_exc |= CONVERT_BIT(exc, FPCR_DZE, EXC_M_DZE);
+    hw_exc |= CONVERT_BIT(exc, FPCR_OVF, EXC_M_FOV);
+    hw_exc |= CONVERT_BIT(exc, FPCR_UNF, EXC_M_UNF);
+    hw_exc |= CONVERT_BIT(exc, FPCR_INE, EXC_M_INE);
+    hw_exc |= CONVERT_BIT(exc, FPCR_IOV, EXC_M_IOV);
+
+    arith_excp(env, retaddr, hw_exc, 1ull << regno);
 }
 
 /* Raise exceptions for ieee fp insns without software completion.
    In that case there are no exceptions that don't trap; the mask
    doesn't apply.  */
-void helper_fp_exc_raise(CPUAlphaState *env, uint32_t exc, uint32_t regno)
+void helper_fp_exc_raise(CPUAlphaState *env, uint32_t ignore, uint32_t regno)
 {
-    inline_fp_exc_raise(env, GETPC(), exc, regno);
+    uint32_t exc = env->error_code;
+    if (exc) {
+        env->fpcr |= exc;
+        exc &= ~ignore;
+        if (exc) {
+            fp_exc_raise1(env, GETPC(), exc, regno, 0);
+        }
+    }
 }
 
 /* Raise exceptions for ieee fp insns with software completion.  */
-void helper_fp_exc_raise_s(CPUAlphaState *env, uint32_t exc, uint32_t regno)
+void helper_fp_exc_raise_s(CPUAlphaState *env, uint32_t ignore, uint32_t regno)
 {
+    uint32_t exc = env->error_code & ~ignore;
     if (exc) {
-        env->fpcr_exc_status |= exc;
-        exc &= ~env->fpcr_exc_mask;
-        inline_fp_exc_raise(env, GETPC(), exc, regno);
+        env->fpcr |= exc;
+        exc &= ~ignore;
+        if (exc) {
+            exc &= env->fpcr_exc_enable;
+            fp_exc_raise1(env, GETPC(), exc, regno, EXC_M_SWC);
+        }
     }
 }
 
@@ -96,16 +106,14 @@
     uint64_t frac = val & 0xfffffffffffffull;
 
     if (exp == 0) {
-        /* Denormals without DNZ set raise an exception.  */
-        if (frac != 0 && !env->fp_status.flush_inputs_to_zero) {
-            arith_excp(env, GETPC(), EXC_M_UNF, 0);
+        /* Denormals without /S raise an exception.  */
+        if (frac != 0) {
+            arith_excp(env, GETPC(), EXC_M_INV, 0);
         }
     } else if (exp == 0x7ff) {
         /* Infinity or NaN.  */
-        /* ??? I'm not sure these exception bit flags are correct.  I do
-           know that the Linux kernel, at least, doesn't rely on them and
-           just emulates the insn to figure out what exception to use.  */
-        arith_excp(env, GETPC(), frac ? EXC_M_INV : EXC_M_FOV, 0);
+        env->fpcr |= FPCR_INV;
+        arith_excp(env, GETPC(), EXC_M_INV, 0);
     }
 }
 
@@ -116,274 +124,31 @@
     uint64_t frac = val & 0xfffffffffffffull;
 
     if (exp == 0) {
-        /* Denormals without DNZ set raise an exception.  */
-        if (frac != 0 && !env->fp_status.flush_inputs_to_zero) {
-            arith_excp(env, GETPC(), EXC_M_UNF, 0);
+        /* Denormals without /S raise an exception.  */
+        if (frac != 0) {
+            arith_excp(env, GETPC(), EXC_M_INV, 0);
         }
     } else if (exp == 0x7ff && frac) {
         /* NaN.  */
+        env->fpcr |= FPCR_INV;
         arith_excp(env, GETPC(), EXC_M_INV, 0);
     }
 }
 
-/* F floating (VAX) */
-static uint64_t float32_to_f(float32 fa)
+/* Input handing with software completion.  Trap for denorms, unless DNZ
+   is set.  If we try to support DNOD (which none of the produced hardware
+   did, AFAICS), we'll need to suppress the trap when FPCR.DNOD is set;
+   then the code downstream of that will need to cope with denorms sans
+   flush_input_to_zero.  Most of it should work sanely, but there's
+   nothing to compare with.  */
+void helper_ieee_input_s(CPUAlphaState *env, uint64_t val)
 {
-    uint64_t r, exp, mant, sig;
-    CPU_FloatU a;
-
-    a.f = fa;
-    sig = ((uint64_t)a.l & 0x80000000) << 32;
-    exp = (a.l >> 23) & 0xff;
-    mant = ((uint64_t)a.l & 0x007fffff) << 29;
-
-    if (exp == 255) {
-        /* NaN or infinity */
-        r = 1; /* VAX dirty zero */
-    } else if (exp == 0) {
-        if (mant == 0) {
-            /* Zero */
-            r = 0;
-        } else {
-            /* Denormalized */
-            r = sig | ((exp + 1) << 52) | mant;
-        }
-    } else {
-        if (exp >= 253) {
-            /* Overflow */
-            r = 1; /* VAX dirty zero */
-        } else {
-            r = sig | ((exp + 2) << 52);
-        }
+    if (unlikely(2 * val - 1 < 0x1fffffffffffffull)
+        && !env->fp_status.flush_inputs_to_zero) {
+        arith_excp(env, GETPC(), EXC_M_INV | EXC_M_SWC, 0);
     }
-
-    return r;
 }
 
-static float32 f_to_float32(CPUAlphaState *env, uintptr_t retaddr, uint64_t a)
-{
-    uint32_t exp, mant_sig;
-    CPU_FloatU r;
-
-    exp = ((a >> 55) & 0x80) | ((a >> 52) & 0x7f);
-    mant_sig = ((a >> 32) & 0x80000000) | ((a >> 29) & 0x007fffff);
-
-    if (unlikely(!exp && mant_sig)) {
-        /* Reserved operands / Dirty zero */
-        dynamic_excp(env, retaddr, EXCP_OPCDEC, 0);
-    }
-
-    if (exp < 3) {
-        /* Underflow */
-        r.l = 0;
-    } else {
-        r.l = ((exp - 2) << 23) | mant_sig;
-    }
-
-    return r.f;
-}
-
-uint32_t helper_f_to_memory(uint64_t a)
-{
-    uint32_t r;
-    r =  (a & 0x00001fffe0000000ull) >> 13;
-    r |= (a & 0x07ffe00000000000ull) >> 45;
-    r |= (a & 0xc000000000000000ull) >> 48;
-    return r;
-}
-
-uint64_t helper_memory_to_f(uint32_t a)
-{
-    uint64_t r;
-    r =  ((uint64_t)(a & 0x0000c000)) << 48;
-    r |= ((uint64_t)(a & 0x003fffff)) << 45;
-    r |= ((uint64_t)(a & 0xffff0000)) << 13;
-    if (!(a & 0x00004000)) {
-        r |= 0x7ll << 59;
-    }
-    return r;
-}
-
-/* ??? Emulating VAX arithmetic with IEEE arithmetic is wrong.  We should
-   either implement VAX arithmetic properly or just signal invalid opcode.  */
-
-uint64_t helper_addf(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float32 fa, fb, fr;
-
-    fa = f_to_float32(env, GETPC(), a);
-    fb = f_to_float32(env, GETPC(), b);
-    fr = float32_add(fa, fb, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-uint64_t helper_subf(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float32 fa, fb, fr;
-
-    fa = f_to_float32(env, GETPC(), a);
-    fb = f_to_float32(env, GETPC(), b);
-    fr = float32_sub(fa, fb, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-uint64_t helper_mulf(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float32 fa, fb, fr;
-
-    fa = f_to_float32(env, GETPC(), a);
-    fb = f_to_float32(env, GETPC(), b);
-    fr = float32_mul(fa, fb, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-uint64_t helper_divf(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float32 fa, fb, fr;
-
-    fa = f_to_float32(env, GETPC(), a);
-    fb = f_to_float32(env, GETPC(), b);
-    fr = float32_div(fa, fb, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-uint64_t helper_sqrtf(CPUAlphaState *env, uint64_t t)
-{
-    float32 ft, fr;
-
-    ft = f_to_float32(env, GETPC(), t);
-    fr = float32_sqrt(ft, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-
-/* G floating (VAX) */
-static uint64_t float64_to_g(float64 fa)
-{
-    uint64_t r, exp, mant, sig;
-    CPU_DoubleU a;
-
-    a.d = fa;
-    sig = a.ll & 0x8000000000000000ull;
-    exp = (a.ll >> 52) & 0x7ff;
-    mant = a.ll & 0x000fffffffffffffull;
-
-    if (exp == 2047) {
-        /* NaN or infinity */
-        r = 1; /* VAX dirty zero */
-    } else if (exp == 0) {
-        if (mant == 0) {
-            /* Zero */
-            r = 0;
-        } else {
-            /* Denormalized */
-            r = sig | ((exp + 1) << 52) | mant;
-        }
-    } else {
-        if (exp >= 2045) {
-            /* Overflow */
-            r = 1; /* VAX dirty zero */
-        } else {
-            r = sig | ((exp + 2) << 52);
-        }
-    }
-
-    return r;
-}
-
-static float64 g_to_float64(CPUAlphaState *env, uintptr_t retaddr, uint64_t a)
-{
-    uint64_t exp, mant_sig;
-    CPU_DoubleU r;
-
-    exp = (a >> 52) & 0x7ff;
-    mant_sig = a & 0x800fffffffffffffull;
-
-    if (!exp && mant_sig) {
-        /* Reserved operands / Dirty zero */
-        dynamic_excp(env, retaddr, EXCP_OPCDEC, 0);
-    }
-
-    if (exp < 3) {
-        /* Underflow */
-        r.ll = 0;
-    } else {
-        r.ll = ((exp - 2) << 52) | mant_sig;
-    }
-
-    return r.d;
-}
-
-uint64_t helper_g_to_memory(uint64_t a)
-{
-    uint64_t r;
-    r =  (a & 0x000000000000ffffull) << 48;
-    r |= (a & 0x00000000ffff0000ull) << 16;
-    r |= (a & 0x0000ffff00000000ull) >> 16;
-    r |= (a & 0xffff000000000000ull) >> 48;
-    return r;
-}
-
-uint64_t helper_memory_to_g(uint64_t a)
-{
-    uint64_t r;
-    r =  (a & 0x000000000000ffffull) << 48;
-    r |= (a & 0x00000000ffff0000ull) << 16;
-    r |= (a & 0x0000ffff00000000ull) >> 16;
-    r |= (a & 0xffff000000000000ull) >> 48;
-    return r;
-}
-
-uint64_t helper_addg(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb, fr;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-    fr = float64_add(fa, fb, &FP_STATUS);
-    return float64_to_g(fr);
-}
-
-uint64_t helper_subg(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb, fr;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-    fr = float64_sub(fa, fb, &FP_STATUS);
-    return float64_to_g(fr);
-}
-
-uint64_t helper_mulg(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb, fr;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-    fr = float64_mul(fa, fb, &FP_STATUS);
-    return float64_to_g(fr);
-}
-
-uint64_t helper_divg(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb, fr;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-    fr = float64_div(fa, fb, &FP_STATUS);
-    return float64_to_g(fr);
-}
-
-uint64_t helper_sqrtg(CPUAlphaState *env, uint64_t a)
-{
-    float64 fa, fr;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fr = float64_sqrt(fa, &FP_STATUS);
-    return float64_to_g(fr);
-}
-
-
 /* S floating (single) */
 
 /* Taken from linux/arch/alpha/kernel/traps.c, s_mem_to_reg.  */
@@ -447,6 +212,8 @@
     fa = s_to_float32(a);
     fb = s_to_float32(b);
     fr = float32_add(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
@@ -457,6 +224,8 @@
     fa = s_to_float32(a);
     fb = s_to_float32(b);
     fr = float32_sub(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
@@ -467,6 +236,8 @@
     fa = s_to_float32(a);
     fb = s_to_float32(b);
     fr = float32_mul(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
@@ -477,6 +248,8 @@
     fa = s_to_float32(a);
     fb = s_to_float32(b);
     fr = float32_div(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
@@ -486,6 +259,8 @@
 
     fa = s_to_float32(a);
     fr = float32_sqrt(fa, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
@@ -514,6 +289,8 @@
     fa = t_to_float64(a);
     fb = t_to_float64(b);
     fr = float64_add(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float64_to_t(fr);
 }
 
@@ -524,6 +301,8 @@
     fa = t_to_float64(a);
     fb = t_to_float64(b);
     fr = float64_sub(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float64_to_t(fr);
 }
 
@@ -534,6 +313,8 @@
     fa = t_to_float64(a);
     fb = t_to_float64(b);
     fr = float64_mul(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float64_to_t(fr);
 }
 
@@ -544,6 +325,8 @@
     fa = t_to_float64(a);
     fb = t_to_float64(b);
     fr = float64_div(fa, fb, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float64_to_t(fr);
 }
 
@@ -553,6 +336,8 @@
 
     fa = t_to_float64(a);
     fr = float64_sqrt(fa, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float64_to_t(fr);
 }
 
@@ -560,99 +345,65 @@
 uint64_t helper_cmptun(CPUAlphaState *env, uint64_t a, uint64_t b)
 {
     float64 fa, fb;
+    uint64_t ret = 0;
 
     fa = t_to_float64(a);
     fb = t_to_float64(b);
 
     if (float64_unordered_quiet(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
+        ret = 0x4000000000000000ULL;
     }
+    env->error_code = soft_to_fpcr_exc(env);
+
+    return ret;
 }
 
 uint64_t helper_cmpteq(CPUAlphaState *env, uint64_t a, uint64_t b)
 {
     float64 fa, fb;
+    uint64_t ret = 0;
 
     fa = t_to_float64(a);
     fb = t_to_float64(b);
 
     if (float64_eq_quiet(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
+        ret = 0x4000000000000000ULL;
     }
+    env->error_code = soft_to_fpcr_exc(env);
+
+    return ret;
 }
 
 uint64_t helper_cmptle(CPUAlphaState *env, uint64_t a, uint64_t b)
 {
     float64 fa, fb;
+    uint64_t ret = 0;
 
     fa = t_to_float64(a);
     fb = t_to_float64(b);
 
     if (float64_le(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
+        ret = 0x4000000000000000ULL;
     }
+    env->error_code = soft_to_fpcr_exc(env);
+
+    return ret;
 }
 
 uint64_t helper_cmptlt(CPUAlphaState *env, uint64_t a, uint64_t b)
 {
     float64 fa, fb;
+    uint64_t ret = 0;
 
     fa = t_to_float64(a);
     fb = t_to_float64(b);
 
     if (float64_lt(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
+        ret = 0x4000000000000000ULL;
     }
-}
+    env->error_code = soft_to_fpcr_exc(env);
 
-uint64_t helper_cmpgeq(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-
-    if (float64_eq_quiet(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
-    }
-}
-
-uint64_t helper_cmpgle(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-
-    if (float64_le(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
-    }
-}
-
-uint64_t helper_cmpglt(CPUAlphaState *env, uint64_t a, uint64_t b)
-{
-    float64 fa, fb;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fb = g_to_float64(env, GETPC(), b);
-
-    if (float64_lt(fa, fb, &FP_STATUS)) {
-        return 0x4000000000000000ULL;
-    } else {
-        return 0;
-    }
+    return ret;
 }
 
 /* Floating point format conversion */
@@ -663,6 +414,8 @@
 
     fa = t_to_float64(a);
     fr = float64_to_float32(fa, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
@@ -673,23 +426,24 @@
 
     fa = s_to_float32(a);
     fr = float32_to_float64(fa, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float64_to_t(fr);
 }
 
 uint64_t helper_cvtqs(CPUAlphaState *env, uint64_t a)
 {
     float32 fr = int64_to_float32(a, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
+
     return float32_to_s(fr);
 }
 
-/* Implement float64 to uint64 conversion without saturation -- we must
+/* Implement float64 to uint64_t conversion without saturation -- we must
    supply the truncated result.  This behaviour is used by the compiler
-   to get unsigned conversion for free with the same instruction.
+   to get unsigned conversion for free with the same instruction.  */
 
-   The VI flag is set when overflow or inexact exceptions should be raised.  */
-
-static inline uint64_t inline_cvttq(CPUAlphaState *env, uint64_t a,
-                                    int roundmode, int VI)
+static uint64_t do_cvttq(CPUAlphaState *env, uint64_t a, int roundmode)
 {
     uint64_t frac, ret = 0;
     uint32_t exp, sign, exc = 0;
@@ -700,11 +454,11 @@
     frac = a & 0xfffffffffffffull;
 
     if (exp == 0) {
-        if (unlikely(frac != 0)) {
+        if (unlikely(frac != 0) && !env->fp_status.flush_inputs_to_zero) {
             goto do_underflow;
         }
     } else if (exp == 0x7ff) {
-        exc = (frac ? float_flag_invalid : VI ? float_flag_overflow : 0);
+        exc = FPCR_INV;
     } else {
         /* Restore implicit bit.  */
         frac |= 0x10000000000000ull;
@@ -713,11 +467,12 @@
         if (shift >= 0) {
             /* In this case the number is so large that we must shift
                the fraction left.  There is no rounding to do.  */
-            if (shift < 63) {
+            if (shift < 64) {
                 ret = frac << shift;
-                if (VI && (ret >> shift) != frac) {
-                    exc = float_flag_overflow;
-                }
+            }
+            /* Check for overflow.  Note the special case of -0x1p63.  */
+            if (shift >= 11 && a != 0xC3E0000000000000ull) {
+                exc = FPCR_IOV | FPCR_INE;
             }
         } else {
             uint64_t round;
@@ -739,7 +494,7 @@
             }
 
             if (round) {
-                exc = (VI ? float_flag_inexact : 0);
+                exc = FPCR_INE;
                 switch (roundmode) {
                 case float_round_nearest_even:
                     if (round == (1ull << 63)) {
@@ -764,66 +519,35 @@
             ret = -ret;
         }
     }
-    if (unlikely(exc)) {
-        float_raise(exc, &FP_STATUS);
-    }
+    env->error_code = exc;
 
     return ret;
 }
 
 uint64_t helper_cvttq(CPUAlphaState *env, uint64_t a)
 {
-    return inline_cvttq(env, a, FP_STATUS.float_rounding_mode, 1);
+    return do_cvttq(env, a, FP_STATUS.float_rounding_mode);
 }
 
 uint64_t helper_cvttq_c(CPUAlphaState *env, uint64_t a)
 {
-    return inline_cvttq(env, a, float_round_to_zero, 0);
-}
-
-uint64_t helper_cvttq_svic(CPUAlphaState *env, uint64_t a)
-{
-    return inline_cvttq(env, a, float_round_to_zero, 1);
+    return do_cvttq(env, a, float_round_to_zero);
 }
 
 uint64_t helper_cvtqt(CPUAlphaState *env, uint64_t a)
 {
     float64 fr = int64_to_float64(a, &FP_STATUS);
+    env->error_code = soft_to_fpcr_exc(env);
     return float64_to_t(fr);
 }
 
-uint64_t helper_cvtqf(CPUAlphaState *env, uint64_t a)
+uint64_t helper_cvtql(CPUAlphaState *env, uint64_t val)
 {
-    float32 fr = int64_to_float32(a, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-uint64_t helper_cvtgf(CPUAlphaState *env, uint64_t a)
-{
-    float64 fa;
-    float32 fr;
-
-    fa = g_to_float64(env, GETPC(), a);
-    fr = float64_to_float32(fa, &FP_STATUS);
-    return float32_to_f(fr);
-}
-
-uint64_t helper_cvtgq(CPUAlphaState *env, uint64_t a)
-{
-    float64 fa = g_to_float64(env, GETPC(), a);
-    return float64_to_int64_round_to_zero(fa, &FP_STATUS);
-}
-
-uint64_t helper_cvtqg(CPUAlphaState *env, uint64_t a)
-{
-    float64 fr;
-    fr = int64_to_float64(a, &FP_STATUS);
-    return float64_to_g(fr);
-}
-
-void helper_fcvtql_v_input(CPUAlphaState *env, uint64_t val)
-{
+    uint32_t exc = 0;
     if (val != (int32_t)val) {
-        arith_excp(env, GETPC(), EXC_M_IOV, 0);
+        exc = FPCR_IOV | FPCR_INE;
     }
+    env->error_code = exc;
+
+    return ((val & 0xc0000000) << 32) | ((val & 0x3fffffff) << 29);
 }
diff --git a/target-alpha/gdbstub.c b/target-alpha/gdbstub.c
index 980f140..d64bccc 100644
--- a/target-alpha/gdbstub.c
+++ b/target-alpha/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int alpha_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
@@ -30,7 +31,7 @@
 
     switch (n) {
     case 0 ... 30:
-        val = env->ir[n];
+        val = cpu_alpha_load_gr(env, n);
         break;
     case 32 ... 62:
         d.d = env->fir[n - 32];
@@ -66,7 +67,7 @@
 
     switch (n) {
     case 0 ... 30:
-        env->ir[n] = tmp;
+        cpu_alpha_store_gr(env, n, tmp);
         break;
     case 32 ... 62:
         d.ll = tmp;
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index a8aa782..85168b7 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -17,144 +17,55 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdint.h>
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "fpu/softfloat.h"
 #include "exec/helper-proto.h"
 
+
+#define CONVERT_BIT(X, SRC, DST) \
+    (SRC > DST ? (X) / (SRC / DST) & (DST) : ((X) & SRC) * (DST / SRC))
+
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env)
 {
-    uint64_t r = 0;
-    uint8_t t;
-
-    t = env->fpcr_exc_status;
-    if (t) {
-        r = FPCR_SUM;
-        if (t & float_flag_invalid) {
-            r |= FPCR_INV;
-        }
-        if (t & float_flag_divbyzero) {
-            r |= FPCR_DZE;
-        }
-        if (t & float_flag_overflow) {
-            r |= FPCR_OVF;
-        }
-        if (t & float_flag_underflow) {
-            r |= FPCR_UNF;
-        }
-        if (t & float_flag_inexact) {
-            r |= FPCR_INE;
-        }
-    }
-
-    t = env->fpcr_exc_mask;
-    if (t & float_flag_invalid) {
-        r |= FPCR_INVD;
-    }
-    if (t & float_flag_divbyzero) {
-        r |= FPCR_DZED;
-    }
-    if (t & float_flag_overflow) {
-        r |= FPCR_OVFD;
-    }
-    if (t & float_flag_underflow) {
-        r |= FPCR_UNFD;
-    }
-    if (t & float_flag_inexact) {
-        r |= FPCR_INED;
-    }
-
-    switch (env->fpcr_dyn_round) {
-    case float_round_nearest_even:
-        r |= FPCR_DYN_NORMAL;
-        break;
-    case float_round_down:
-        r |= FPCR_DYN_MINUS;
-        break;
-    case float_round_up:
-        r |= FPCR_DYN_PLUS;
-        break;
-    case float_round_to_zero:
-        r |= FPCR_DYN_CHOPPED;
-        break;
-    }
-
-    if (env->fp_status.flush_inputs_to_zero) {
-        r |= FPCR_DNZ;
-    }
-    if (env->fpcr_dnod) {
-        r |= FPCR_DNOD;
-    }
-    if (env->fpcr_undz) {
-        r |= FPCR_UNDZ;
-    }
-
-    return r;
+    return (uint64_t)env->fpcr << 32;
 }
 
 void cpu_alpha_store_fpcr (CPUAlphaState *env, uint64_t val)
 {
-    uint8_t t;
+    uint32_t fpcr = val >> 32;
+    uint32_t t = 0;
 
-    t = 0;
-    if (val & FPCR_INV) {
-        t |= float_flag_invalid;
-    }
-    if (val & FPCR_DZE) {
-        t |= float_flag_divbyzero;
-    }
-    if (val & FPCR_OVF) {
-        t |= float_flag_overflow;
-    }
-    if (val & FPCR_UNF) {
-        t |= float_flag_underflow;
-    }
-    if (val & FPCR_INE) {
-        t |= float_flag_inexact;
-    }
-    env->fpcr_exc_status = t;
+    t |= CONVERT_BIT(fpcr, FPCR_INED, FPCR_INE);
+    t |= CONVERT_BIT(fpcr, FPCR_UNFD, FPCR_UNF);
+    t |= CONVERT_BIT(fpcr, FPCR_OVFD, FPCR_OVF);
+    t |= CONVERT_BIT(fpcr, FPCR_DZED, FPCR_DZE);
+    t |= CONVERT_BIT(fpcr, FPCR_INVD, FPCR_INV);
 
-    t = 0;
-    if (val & FPCR_INVD) {
-        t |= float_flag_invalid;
-    }
-    if (val & FPCR_DZED) {
-        t |= float_flag_divbyzero;
-    }
-    if (val & FPCR_OVFD) {
-        t |= float_flag_overflow;
-    }
-    if (val & FPCR_UNFD) {
-        t |= float_flag_underflow;
-    }
-    if (val & FPCR_INED) {
-        t |= float_flag_inexact;
-    }
-    env->fpcr_exc_mask = t;
+    env->fpcr = fpcr;
+    env->fpcr_exc_enable = ~t & FPCR_STATUS_MASK;
 
-    switch (val & FPCR_DYN_MASK) {
+    switch (fpcr & FPCR_DYN_MASK) {
+    case FPCR_DYN_NORMAL:
+    default:
+        t = float_round_nearest_even;
+        break;
     case FPCR_DYN_CHOPPED:
         t = float_round_to_zero;
         break;
     case FPCR_DYN_MINUS:
         t = float_round_down;
         break;
-    case FPCR_DYN_NORMAL:
-        t = float_round_nearest_even;
-        break;
     case FPCR_DYN_PLUS:
         t = float_round_up;
         break;
     }
     env->fpcr_dyn_round = t;
 
-    env->fpcr_dnod = (val & FPCR_DNOD) != 0;
-    env->fpcr_undz = (val & FPCR_UNDZ) != 0;
-    env->fpcr_flush_to_zero = env->fpcr_dnod & env->fpcr_undz;
-    env->fp_status.flush_inputs_to_zero = (val & FPCR_DNZ) != 0;
+    env->fpcr_flush_to_zero = (fpcr & FPCR_UNFD) && (fpcr & FPCR_UNDZ);
+    env->fp_status.flush_inputs_to_zero = (fpcr & FPCR_DNZ) != 0;
 }
 
 uint64_t helper_load_fpcr(CPUAlphaState *env)
@@ -167,6 +78,30 @@
     cpu_alpha_store_fpcr(env, val);
 }
 
+static uint64_t *cpu_alpha_addr_gr(CPUAlphaState *env, unsigned reg)
+{
+#ifndef CONFIG_USER_ONLY
+    if (env->pal_mode) {
+        if (reg >= 8 && reg <= 14) {
+            return &env->shadow[reg - 8];
+        } else if (reg == 25) {
+            return &env->shadow[7];
+        }
+    }
+#endif
+    return &env->ir[reg];
+}
+
+uint64_t cpu_alpha_load_gr(CPUAlphaState *env, unsigned reg)
+{
+    return *cpu_alpha_addr_gr(env, reg);
+}
+
+void cpu_alpha_store_gr(CPUAlphaState *env, unsigned reg, uint64_t val)
+{
+    *cpu_alpha_addr_gr(env, reg) = val;
+}
+
 #if defined(CONFIG_USER_ONLY)
 int alpha_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
                                int rw, int mmu_idx)
@@ -178,38 +113,6 @@
     return 1;
 }
 #else
-void swap_shadow_regs(CPUAlphaState *env)
-{
-    uint64_t i0, i1, i2, i3, i4, i5, i6, i7;
-
-    i0 = env->ir[8];
-    i1 = env->ir[9];
-    i2 = env->ir[10];
-    i3 = env->ir[11];
-    i4 = env->ir[12];
-    i5 = env->ir[13];
-    i6 = env->ir[14];
-    i7 = env->ir[25];
-
-    env->ir[8]  = env->shadow[0];
-    env->ir[9]  = env->shadow[1];
-    env->ir[10] = env->shadow[2];
-    env->ir[11] = env->shadow[3];
-    env->ir[12] = env->shadow[4];
-    env->ir[13] = env->shadow[5];
-    env->ir[14] = env->shadow[6];
-    env->ir[25] = env->shadow[7];
-
-    env->shadow[0] = i0;
-    env->shadow[1] = i1;
-    env->shadow[2] = i2;
-    env->shadow[3] = i3;
-    env->shadow[4] = i4;
-    env->shadow[5] = i5;
-    env->shadow[6] = i6;
-    env->shadow[7] = i7;
-}
-
 /* Returns the OSF/1 entMM failure indication, or -1 on success.  */
 static int get_physical_address(CPUAlphaState *env, target_ulong addr,
                                 int prot_need, int mmu_idx,
@@ -463,10 +366,7 @@
     env->pc = env->palbr + i;
 
     /* Switch to PALmode.  */
-    if (!env->pal_mode) {
-        env->pal_mode = 1;
-        swap_shadow_regs(env);
-    }
+    env->pal_mode = 1;
 #endif /* !USER_ONLY */
 }
 
@@ -531,7 +431,7 @@
                 env->pc, env->ps);
     for (i = 0; i < 31; i++) {
         cpu_fprintf(f, "IR%02d %s " TARGET_FMT_lx " ", i,
-                    linux_reg_names[i], env->ir[i]);
+                    linux_reg_names[i], cpu_alpha_load_gr(env, i));
         if ((i % 3) == 2)
             cpu_fprintf(f, "\n");
     }
@@ -571,6 +471,8 @@
     env->error_code = error;
     if (retaddr) {
         cpu_restore_state(cs, retaddr);
+        /* Floating-point exceptions (our only users) point to the next PC.  */
+        env->pc += 4;
     }
     cpu_loop_exit(cs);
 }
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index a451cfe..c3d8a3e 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -1,12 +1,7 @@
 DEF_HELPER_3(excp, noreturn, env, int, int)
 DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_NO_RWG_SE, i64, env)
 
-DEF_HELPER_FLAGS_3(addqv, TCG_CALL_NO_WG, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(addlv, TCG_CALL_NO_WG, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subqv, TCG_CALL_NO_WG, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(sublv, TCG_CALL_NO_WG, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mullv, TCG_CALL_NO_WG, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mulqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(check_overflow, TCG_CALL_NO_WG, void, env, i64, i64)
 
 DEF_HELPER_FLAGS_1(ctpop, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_1(ctlz, TCG_CALL_NO_RWG_SE, i64, i64)
@@ -15,6 +10,7 @@
 DEF_HELPER_FLAGS_2(zap, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
+DEF_HELPER_FLAGS_1(cmpbe0, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
 DEF_HELPER_FLAGS_2(minub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
@@ -83,23 +79,19 @@
 
 DEF_HELPER_FLAGS_2(cvttq, TCG_CALL_NO_RWG, i64, env, i64)
 DEF_HELPER_FLAGS_2(cvttq_c, TCG_CALL_NO_RWG, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvttq_svic, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_2(cvtql, TCG_CALL_NO_RWG, i64, env, i64)
 
 DEF_HELPER_FLAGS_2(setroundmode, TCG_CALL_NO_RWG, void, env, i32)
 DEF_HELPER_FLAGS_2(setflushzero, TCG_CALL_NO_RWG, void, env, i32)
-DEF_HELPER_FLAGS_1(fp_exc_clear, TCG_CALL_NO_RWG, void, env)
-DEF_HELPER_FLAGS_1(fp_exc_get, TCG_CALL_NO_RWG_SE, i32, env)
 DEF_HELPER_FLAGS_3(fp_exc_raise, TCG_CALL_NO_WG, void, env, i32, i32)
 DEF_HELPER_FLAGS_3(fp_exc_raise_s, TCG_CALL_NO_WG, void, env, i32, i32)
 
 DEF_HELPER_FLAGS_2(ieee_input, TCG_CALL_NO_WG, void, env, i64)
 DEF_HELPER_FLAGS_2(ieee_input_cmp, TCG_CALL_NO_WG, void, env, i64)
-DEF_HELPER_FLAGS_2(fcvtql_v_input, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_2(ieee_input_s, TCG_CALL_NO_WG, void, env, i64)
 
 #if !defined (CONFIG_USER_ONLY)
-DEF_HELPER_2(hw_ret, void, env, i64)
-DEF_HELPER_3(call_pal, void, env, i64, i64)
-
 DEF_HELPER_2(ldl_phys, i64, env, i64)
 DEF_HELPER_2(ldq_phys, i64, env, i64)
 DEF_HELPER_2(ldl_l_phys, i64, env, i64)
diff --git a/target-alpha/int_helper.c b/target-alpha/int_helper.c
index 7a205eb..19bebfe 100644
--- a/target-alpha/int_helper.c
+++ b/target-alpha/int_helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 
@@ -37,47 +39,68 @@
     return ctz64(arg);
 }
 
-static inline uint64_t byte_zap(uint64_t op, uint8_t mskb)
+uint64_t helper_zapnot(uint64_t val, uint64_t mskb)
 {
     uint64_t mask;
 
-    mask = 0;
-    mask |= ((mskb >> 0) & 1) * 0x00000000000000FFULL;
-    mask |= ((mskb >> 1) & 1) * 0x000000000000FF00ULL;
-    mask |= ((mskb >> 2) & 1) * 0x0000000000FF0000ULL;
-    mask |= ((mskb >> 3) & 1) * 0x00000000FF000000ULL;
-    mask |= ((mskb >> 4) & 1) * 0x000000FF00000000ULL;
-    mask |= ((mskb >> 5) & 1) * 0x0000FF0000000000ULL;
-    mask |= ((mskb >> 6) & 1) * 0x00FF000000000000ULL;
-    mask |= ((mskb >> 7) & 1) * 0xFF00000000000000ULL;
+    mask  = -(mskb & 0x01) & 0x00000000000000ffull;
+    mask |= -(mskb & 0x02) & 0x000000000000ff00ull;
+    mask |= -(mskb & 0x04) & 0x0000000000ff0000ull;
+    mask |= -(mskb & 0x08) & 0x00000000ff000000ull;
+    mask |= -(mskb & 0x10) & 0x000000ff00000000ull;
+    mask |= -(mskb & 0x20) & 0x0000ff0000000000ull;
+    mask |= -(mskb & 0x40) & 0x00ff000000000000ull;
+    mask |= -(mskb & 0x80) & 0xff00000000000000ull;
 
-    return op & ~mask;
+    return val & mask;
 }
 
 uint64_t helper_zap(uint64_t val, uint64_t mask)
 {
-    return byte_zap(val, mask);
+    return helper_zapnot(val, ~mask);
 }
 
-uint64_t helper_zapnot(uint64_t val, uint64_t mask)
+uint64_t helper_cmpbe0(uint64_t a)
 {
-    return byte_zap(val, ~mask);
+    uint64_t m = 0x7f7f7f7f7f7f7f7fULL;
+    uint64_t c = ~(((a & m) + m) | a | m);
+    /* a.......b.......c.......d.......e.......f.......g.......h....... */
+    c |= c << 7;
+    /* ab......bc......cd......de......ef......fg......gh......h....... */
+    c |= c << 14;
+    /* abcd....bcde....cdef....defg....efgh....fgh.....gh......h....... */
+    c |= c << 28;
+    /* abcdefghbcdefgh.cdefgh..defgh...efgh....fgh.....gh......h....... */
+    return c >> 56;
 }
 
-uint64_t helper_cmpbge(uint64_t op1, uint64_t op2)
+uint64_t helper_cmpbge(uint64_t a, uint64_t b)
 {
-    uint8_t opa, opb, res;
-    int i;
+    uint64_t mask = 0x00ff00ff00ff00ffULL;
+    uint64_t test = 0x0100010001000100ULL;
+    uint64_t al, ah, bl, bh, cl, ch;
 
-    res = 0;
-    for (i = 0; i < 8; i++) {
-        opa = op1 >> (i * 8);
-        opb = op2 >> (i * 8);
-        if (opa >= opb) {
-            res |= 1 << i;
-        }
-    }
-    return res;
+    /* Separate the bytes to avoid false positives.  */
+    al = a & mask;
+    bl = b & mask;
+    ah = (a >> 8) & mask;
+    bh = (b >> 8) & mask;
+
+    /* "Compare".  If a byte in B is greater than a byte in A,
+       it will clear the test bit.  */
+    cl = ((al | test) - bl) & test;
+    ch = ((ah | test) - bh) & test;
+
+    /* Fold all of the test bits into a contiguous set.  */
+    /* ch=.......a...............c...............e...............g........ */
+    /* cl=.......b...............d...............f...............h........ */
+    cl += ch << 1;
+    /* cl=......ab..............cd..............ef..............gh........ */
+    cl |= cl << 14;
+    /* cl=......abcd............cdef............efgh............gh........ */
+    cl |= cl << 28;
+    /* cl=......abcdefgh........cdefgh..........efgh............gh........ */
+    return cl >> 50;
 }
 
 uint64_t helper_minub8(uint64_t op1, uint64_t op2)
@@ -249,64 +272,9 @@
             | ((op1 & 0xff000000) << 24));
 }
 
-uint64_t helper_addqv(CPUAlphaState *env, uint64_t op1, uint64_t op2)
+void helper_check_overflow(CPUAlphaState *env, uint64_t op1, uint64_t op2)
 {
-    uint64_t tmp = op1;
-    op1 += op2;
-    if (unlikely((tmp ^ op2 ^ (-1ULL)) & (tmp ^ op1) & (1ULL << 63))) {
+    if (unlikely(op1 != op2)) {
         arith_excp(env, GETPC(), EXC_M_IOV, 0);
     }
-    return op1;
-}
-
-uint64_t helper_addlv(CPUAlphaState *env, uint64_t op1, uint64_t op2)
-{
-    uint64_t tmp = op1;
-    op1 = (uint32_t)(op1 + op2);
-    if (unlikely((tmp ^ op2 ^ (-1UL)) & (tmp ^ op1) & (1UL << 31))) {
-        arith_excp(env, GETPC(), EXC_M_IOV, 0);
-    }
-    return op1;
-}
-
-uint64_t helper_subqv(CPUAlphaState *env, uint64_t op1, uint64_t op2)
-{
-    uint64_t res;
-    res = op1 - op2;
-    if (unlikely((op1 ^ op2) & (res ^ op1) & (1ULL << 63))) {
-        arith_excp(env, GETPC(), EXC_M_IOV, 0);
-    }
-    return res;
-}
-
-uint64_t helper_sublv(CPUAlphaState *env, uint64_t op1, uint64_t op2)
-{
-    uint32_t res;
-    res = op1 - op2;
-    if (unlikely((op1 ^ op2) & (res ^ op1) & (1UL << 31))) {
-        arith_excp(env, GETPC(), EXC_M_IOV, 0);
-    }
-    return res;
-}
-
-uint64_t helper_mullv(CPUAlphaState *env, uint64_t op1, uint64_t op2)
-{
-    int64_t res = (int64_t)op1 * (int64_t)op2;
-
-    if (unlikely((int32_t)res != res)) {
-        arith_excp(env, GETPC(), EXC_M_IOV, 0);
-    }
-    return (int64_t)((int32_t)res);
-}
-
-uint64_t helper_mulqv(CPUAlphaState *env, uint64_t op1, uint64_t op2)
-{
-    uint64_t tl, th;
-
-    muls64(&tl, &th, op1, op2);
-    /* If th != 0 && th != -1, then we had an overflow */
-    if (unlikely((th + 1) > 1)) {
-        arith_excp(env, GETPC(), EXC_M_IOV, 0);
-    }
-    return tl;
 }
diff --git a/target-alpha/machine.c b/target-alpha/machine.c
index e796bbe..710b783 100644
--- a/target-alpha/machine.c
+++ b/target-alpha/machine.c
@@ -1,5 +1,9 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "migration/cpu.h"
 
 static int get_fpcr(QEMUFile *f, void *opaque, size_t size)
 {
@@ -70,8 +74,8 @@
 
 static const VMStateDescription vmstate_env = {
     .name = "env",
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
     .fields = vmstate_env_fields,
 };
 
diff --git a/target-alpha/mem_helper.c b/target-alpha/mem_helper.c
index fc4f57a..92bc119 100644
--- a/target-alpha/mem_helper.c
+++ b/target-alpha/mem_helper.c
@@ -17,8 +17,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 /* Softmmu support */
@@ -97,13 +99,17 @@
 }
 
 void alpha_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
-                                   int is_write, int is_user, uintptr_t retaddr)
+                                   MMUAccessType access_type,
+                                   int mmu_idx, uintptr_t retaddr,
+                                   unsigned size)
 {
     AlphaCPU *cpu = ALPHA_CPU(cs);
     CPUAlphaState *env = &cpu->env;
     uint64_t pc;
     uint32_t insn;
 
+    (void)size;
+
     if (retaddr) {
         cpu_restore_state(cs, retaddr);
     }
@@ -128,19 +134,26 @@
 
     env->trap_arg0 = addr;
     env->trap_arg1 = is_write ? 1 : 0;
-    dynamic_excp(env, 0, EXCP_MCHK, 0);
+    cs->exception_index = EXCP_MCHK;
+    env->error_code = 0;
+
+    /* ??? We should cpu_restore_state to the faulting insn, but this hook
+       does not have access to the retaddr value from the original helper.
+       It's all moot until the QEMU PALcode grows an MCHK handler.  */
+
+    cpu_loop_exit(cs);
 }
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
 /* XXX: fix it to restore all registers */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write,
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
               int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = alpha_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = alpha_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret != 0)) {
         if (retaddr) {
             cpu_restore_state(cs, retaddr);
diff --git a/target-alpha/sys_helper.c b/target-alpha/sys_helper.c
index ae2e174..bec1e17 100644
--- a/target-alpha/sys_helper.c
+++ b/target-alpha/sys_helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 #include "sysemu/sysemu.h"
 #include "qemu/timer.h"
@@ -34,34 +36,12 @@
 #else
     /* In user-mode, QEMU_CLOCK_VIRTUAL doesn't exist.  Just pass through the host cpu
        clock ticks.  Also, don't bother taking PCC_OFS into account.  */
-    return (uint32_t)cpu_get_real_ticks();
+    return (uint32_t)cpu_get_host_ticks();
 #endif
 }
 
 /* PALcode support special instructions */
 #ifndef CONFIG_USER_ONLY
-void helper_hw_ret(CPUAlphaState *env, uint64_t a)
-{
-    env->pc = a & ~3;
-    env->intr_flag = 0;
-    env->lock_addr = -1;
-    if ((a & 1) == 0) {
-        env->pal_mode = 0;
-        swap_shadow_regs(env);
-    }
-}
-
-void helper_call_pal(CPUAlphaState *env, uint64_t pc, uint64_t entry_ofs)
-{
-    int pal_mode = env->pal_mode;
-    env->exc_addr = pc | pal_mode;
-    env->pc = env->palbr + entry_ofs;
-    if (!pal_mode) {
-        env->pal_mode = 1;
-        swap_shadow_regs(env);
-    }
-}
-
 void helper_tbia(CPUAlphaState *env)
 {
     tlb_flush(CPU(alpha_env_get_cpu(env)), 1);
@@ -74,7 +54,7 @@
 
 void helper_tb_flush(CPUAlphaState *env)
 {
-    tb_flush(env);
+    tb_flush(CPU(alpha_env_get_cpu(env)));
 }
 
 void helper_halt(uint64_t restart)
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 5bec586..0ea0e6e 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -17,9 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "qemu/host-utils.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/cpu_ldst.h"
 
@@ -27,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #undef ALPHA_DEBUG_DISAS
@@ -42,6 +45,9 @@
 struct DisasContext {
     struct TranslationBlock *tb;
     uint64_t pc;
+#ifndef CONFIG_USER_ONLY
+    uint64_t palbr;
+#endif
     int mem_idx;
 
     /* Current rounding mode for this TB.  */
@@ -52,6 +58,9 @@
     /* implver value for this CPU.  */
     int implver;
 
+    /* The set of registers active in the current context.  */
+    TCGv *ir;
+
     /* Temporaries for $31 and $f31 as source and destination.  */
     TCGv zero;
     TCGv sink;
@@ -85,14 +94,18 @@
 } ExitStatus;
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
-static TCGv cpu_ir[31];
+static TCGv_env cpu_env;
+static TCGv cpu_std_ir[31];
 static TCGv cpu_fir[31];
 static TCGv cpu_pc;
 static TCGv cpu_lock_addr;
 static TCGv cpu_lock_st_addr;
 static TCGv cpu_lock_value;
 
+#ifndef CONFIG_USER_ONLY
+static TCGv cpu_pal_ir[31];
+#endif
+
 #include "exec/gen-icount.h"
 
 void alpha_translate_init(void)
@@ -122,6 +135,12 @@
         "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
         "f24", "f25", "f26", "f27", "f28", "f29", "f30"
     };
+#ifndef CONFIG_USER_ONLY
+    static const char shadow_names[8][8] = {
+        "pal_t7", "pal_s0", "pal_s1", "pal_s2",
+        "pal_s3", "pal_s4", "pal_s5", "pal_t11"
+    };
+#endif
 
     static bool done_init = 0;
     int i;
@@ -132,22 +151,34 @@
     done_init = 1;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
     for (i = 0; i < 31; i++) {
-        cpu_ir[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                           offsetof(CPUAlphaState, ir[i]),
-                                           greg_names[i]);
+        cpu_std_ir[i] = tcg_global_mem_new_i64(cpu_env,
+                                               offsetof(CPUAlphaState, ir[i]),
+                                               greg_names[i]);
     }
 
     for (i = 0; i < 31; i++) {
-        cpu_fir[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_fir[i] = tcg_global_mem_new_i64(cpu_env,
                                             offsetof(CPUAlphaState, fir[i]),
                                             freg_names[i]);
     }
 
+#ifndef CONFIG_USER_ONLY
+    memcpy(cpu_pal_ir, cpu_std_ir, sizeof(cpu_pal_ir));
+    for (i = 0; i < 8; i++) {
+        int r = (i == 7 ? 25 : i + 8);
+        cpu_pal_ir[r] = tcg_global_mem_new_i64(cpu_env,
+                                               offsetof(CPUAlphaState,
+                                                        shadow[i]),
+                                               shadow_names[i]);
+    }
+#endif
+
     for (i = 0; i < ARRAY_SIZE(vars); ++i) {
         const GlobalVar *v = &vars[i];
-        *v->var = tcg_global_mem_new_i64(TCG_AREG0, v->ofs, v->name);
+        *v->var = tcg_global_mem_new_i64(cpu_env, v->ofs, v->name);
     }
 }
 
@@ -170,7 +201,7 @@
 static TCGv load_gpr(DisasContext *ctx, unsigned reg)
 {
     if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return load_zero(ctx);
     }
@@ -183,7 +214,7 @@
         ctx->lit = tcg_const_i64(lit);
         return ctx->lit;
     } else if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return load_zero(ctx);
     }
@@ -192,7 +223,7 @@
 static TCGv dest_gpr(DisasContext *ctx, unsigned reg)
 {
     if (likely(reg < 31)) {
-        return cpu_ir[reg];
+        return ctx->ir[reg];
     } else {
         return dest_sink(ctx);
     }
@@ -304,7 +335,7 @@
         addr = tmp;
     }
 
-    va = (fp ? cpu_fir[ra] : cpu_ir[ra]);
+    va = (fp ? cpu_fir[ra] : ctx->ir[ra]);
     tcg_gen_qemu_load(va, addr, ctx->mem_idx);
 
     tcg_temp_free(tmp);
@@ -388,7 +419,7 @@
     /* ??? In system mode we are never multi-threaded, so CAS can be
        implemented via a non-atomic load-compare-store sequence.  */
     {
-        int lab_fail, lab_done;
+        TCGLabel *lab_fail, *lab_done;
         TCGv val;
 
         lab_fail = gen_new_label();
@@ -399,13 +430,13 @@
         tcg_gen_qemu_ld_i64(val, addr, ctx->mem_idx, quad ? MO_LEQ : MO_LESL);
         tcg_gen_brcond_i64(TCG_COND_NE, val, cpu_lock_value, lab_fail);
 
-        tcg_gen_qemu_st_i64(cpu_ir[ra], addr, ctx->mem_idx,
+        tcg_gen_qemu_st_i64(ctx->ir[ra], addr, ctx->mem_idx,
                             quad ? MO_LEQ : MO_LEUL);
-        tcg_gen_movi_i64(cpu_ir[ra], 1);
+        tcg_gen_movi_i64(ctx->ir[ra], 1);
         tcg_gen_br(lab_done);
 
         gen_set_label(lab_fail);
-        tcg_gen_movi_i64(cpu_ir[ra], 0);
+        tcg_gen_movi_i64(ctx->ir[ra], 0);
 
         gen_set_label(lab_done);
         tcg_gen_movi_i64(cpu_lock_addr, -1);
@@ -418,10 +449,13 @@
 
 static bool in_superpage(DisasContext *ctx, int64_t addr)
 {
+#ifndef CONFIG_USER_ONLY
     return ((ctx->tb->flags & TB_FLAGS_USER_MODE) == 0
-            && addr < 0
-            && ((addr >> 41) & 3) == 2
-            && addr >> TARGET_VIRT_ADDR_SPACE_BITS == addr >> 63);
+            && addr >> TARGET_VIRT_ADDR_SPACE_BITS == -1
+            && ((addr >> 41) & 3) == 2);
+#else
+    return false;
+#endif
 }
 
 static bool use_goto_tb(DisasContext *ctx, uint64_t dest)
@@ -431,12 +465,16 @@
         || ctx->singlestep_enabled || singlestep) {
         return false;
     }
+#ifndef CONFIG_USER_ONLY
     /* If the destination is in the superpage, the page perms can't change.  */
     if (in_superpage(ctx, dest)) {
         return true;
     }
     /* Check for the dest on the same page as the start of the TB.  */
     return ((ctx->tb->pc ^ dest) & TARGET_PAGE_MASK) == 0;
+#else
+    return true;
+#endif
 }
 
 static ExitStatus gen_bdirect(DisasContext *ctx, int ra, int32_t disp)
@@ -444,7 +482,7 @@
     uint64_t dest = ctx->pc + (disp << 2);
 
     if (ra != 31) {
-        tcg_gen_movi_i64(cpu_ir[ra], ctx->pc);
+        tcg_gen_movi_i64(ctx->ir[ra], ctx->pc);
     }
 
     /* Notice branch-to-next; used to initialize RA with the PC.  */
@@ -465,7 +503,7 @@
                                      TCGv cmp, int32_t disp)
 {
     uint64_t dest = ctx->pc + (disp << 2);
-    int lab_true = gen_new_label();
+    TCGLabel *lab_true = gen_new_label();
 
     if (use_goto_tb(ctx, dest)) {
         tcg_gen_brcondi_i64(cond, cmp, 0, lab_true);
@@ -658,43 +696,36 @@
             } else {
                 gen_helper_ieee_input(cpu_env, val);
             }
+        } else {
+#ifndef CONFIG_USER_ONLY
+            /* In system mode, raise exceptions for denormals like real
+               hardware.  In user mode, proceed as if the OS completion
+               handler is handling the denormal as per spec.  */
+            gen_helper_ieee_input_s(cpu_env, val);
+#endif
         }
     }
     return val;
 }
 
-static void gen_fp_exc_clear(void)
-{
-#if defined(CONFIG_SOFTFLOAT_INLINE)
-    TCGv_i32 zero = tcg_const_i32(0);
-    tcg_gen_st8_i32(zero, cpu_env,
-                    offsetof(CPUAlphaState, fp_status.float_exception_flags));
-    tcg_temp_free_i32(zero);
-#else
-    gen_helper_fp_exc_clear(cpu_env);
-#endif
-}
-
-static void gen_fp_exc_raise_ignore(int rc, int fn11, int ignore)
+static void gen_fp_exc_raise(int rc, int fn11)
 {
     /* ??? We ought to be able to do something with imprecise exceptions.
        E.g. notice we're still in the trap shadow of something within the
        TB and do not generate the code to signal the exception; end the TB
        when an exception is forced to arrive, either by consumption of a
        register value or TRAPB or EXCB.  */
-    TCGv_i32 exc = tcg_temp_new_i32();
-    TCGv_i32 reg;
+    TCGv_i32 reg, ign;
+    uint32_t ignore = 0;
 
-#if defined(CONFIG_SOFTFLOAT_INLINE)
-    tcg_gen_ld8u_i32(exc, cpu_env,
-                     offsetof(CPUAlphaState, fp_status.float_exception_flags));
-#else
-    gen_helper_fp_exc_get(exc, cpu_env);
-#endif
-
-    if (ignore) {
-        tcg_gen_andi_i32(exc, exc, ~ignore);
+    if (!(fn11 & QUAL_U)) {
+        /* Note that QUAL_U == QUAL_V, so ignore either.  */
+        ignore |= FPCR_UNF | FPCR_IOV;
     }
+    if (!(fn11 & QUAL_I)) {
+        ignore |= FPCR_INE;
+    }
+    ign = tcg_const_i32(ignore);
 
     /* ??? Pass in the regno of the destination so that the helper can
        set EXC_MASK, which contains a bitmask of destination registers
@@ -702,23 +733,17 @@
        does not require this.  We do need it for a guest kernel's entArith,
        or if we were to do something clever with imprecise exceptions.  */
     reg = tcg_const_i32(rc + 32);
-
     if (fn11 & QUAL_S) {
-        gen_helper_fp_exc_raise_s(cpu_env, exc, reg);
+        gen_helper_fp_exc_raise_s(cpu_env, ign, reg);
     } else {
-        gen_helper_fp_exc_raise(cpu_env, exc, reg);
+        gen_helper_fp_exc_raise(cpu_env, ign, reg);
     }
 
     tcg_temp_free_i32(reg);
-    tcg_temp_free_i32(exc);
+    tcg_temp_free_i32(ign);
 }
 
-static inline void gen_fp_exc_raise(int rc, int fn11)
-{
-    gen_fp_exc_raise_ignore(rc, fn11, fn11 & QUAL_I ? 0 : float_flag_inexact);
-}
-
-static void gen_fcvtlq(TCGv vc, TCGv vb)
+static void gen_cvtlq(TCGv vc, TCGv vb)
 {
     TCGv tmp = tcg_temp_new();
 
@@ -733,19 +758,6 @@
     tcg_temp_free(tmp);
 }
 
-static void gen_fcvtql(TCGv vc, TCGv vb)
-{
-    TCGv tmp = tcg_temp_new();
-
-    tcg_gen_andi_i64(tmp, vb, (int32_t)0xc0000000);
-    tcg_gen_andi_i64(vc, vb, 0x3FFFFFFF);
-    tcg_gen_shli_i64(tmp, tmp, 32);
-    tcg_gen_shli_i64(vc, vc, 29);
-    tcg_gen_or_i64(vc, vc, tmp);
-
-    tcg_temp_free(tmp);
-}
-
 static void gen_ieee_arith2(DisasContext *ctx,
                             void (*helper)(TCGv, TCGv_ptr, TCGv),
                             int rb, int rc, int fn11)
@@ -754,7 +766,6 @@
 
     gen_qual_roundmode(ctx, fn11);
     gen_qual_flushzero(ctx, fn11);
-    gen_fp_exc_clear();
 
     vb = gen_ieee_input(ctx, rb, fn11, 0);
     helper(dest_fpr(ctx, rc), cpu_env, vb);
@@ -763,8 +774,8 @@
 }
 
 #define IEEE_ARITH2(name)                                       \
-static inline void glue(gen_f, name)(DisasContext *ctx,         \
-                                     int rb, int rc, int fn11)  \
+static inline void glue(gen_, name)(DisasContext *ctx,          \
+                                    int rb, int rc, int fn11)   \
 {                                                               \
     gen_ieee_arith2(ctx, gen_helper_##name, rb, rc, fn11);      \
 }
@@ -773,38 +784,23 @@
 IEEE_ARITH2(cvtst)
 IEEE_ARITH2(cvtts)
 
-static void gen_fcvttq(DisasContext *ctx, int rb, int rc, int fn11)
+static void gen_cvttq(DisasContext *ctx, int rb, int rc, int fn11)
 {
     TCGv vb, vc;
-    int ignore = 0;
 
     /* No need to set flushzero, since we have an integer output.  */
-    gen_fp_exc_clear();
     vb = gen_ieee_input(ctx, rb, fn11, 0);
     vc = dest_fpr(ctx, rc);
 
-    /* Almost all integer conversions use cropped rounding, and most
-       also do not have integer overflow enabled.  Special case that.  */
-    switch (fn11) {
-    case QUAL_RM_C:
+    /* Almost all integer conversions use cropped rounding;
+       special case that.  */
+    if ((fn11 & QUAL_RM_MASK) == QUAL_RM_C) {
         gen_helper_cvttq_c(vc, cpu_env, vb);
-        break;
-    case QUAL_V | QUAL_RM_C:
-    case QUAL_S | QUAL_V | QUAL_RM_C:
-        ignore = float_flag_inexact;
-        /* FALLTHRU */
-    case QUAL_S | QUAL_V | QUAL_I | QUAL_RM_C:
-        gen_helper_cvttq_svic(vc, cpu_env, vb);
-        break;
-    default:
+    } else {
         gen_qual_roundmode(ctx, fn11);
         gen_helper_cvttq(vc, cpu_env, vb);
-        ignore |= (fn11 & QUAL_V ? 0 : float_flag_overflow);
-        ignore |= (fn11 & QUAL_I ? 0 : float_flag_inexact);
-        break;
     }
-
-    gen_fp_exc_raise_ignore(rc, fn11, ignore);
+    gen_fp_exc_raise(rc, fn11);
 }
 
 static void gen_ieee_intcvt(DisasContext *ctx,
@@ -821,7 +817,6 @@
        is inexact.  Thus we only need to worry about exceptions when
        inexact handling is requested.  */
     if (fn11 & QUAL_I) {
-        gen_fp_exc_clear();
         helper(vc, cpu_env, vb);
         gen_fp_exc_raise(rc, fn11);
     } else {
@@ -830,8 +825,8 @@
 }
 
 #define IEEE_INTCVT(name)                                       \
-static inline void glue(gen_f, name)(DisasContext *ctx,         \
-                                     int rb, int rc, int fn11)  \
+static inline void glue(gen_, name)(DisasContext *ctx,          \
+                                    int rb, int rc, int fn11)   \
 {                                                               \
     gen_ieee_intcvt(ctx, gen_helper_##name, rb, rc, fn11);      \
 }
@@ -864,7 +859,6 @@
 
     gen_qual_roundmode(ctx, fn11);
     gen_qual_flushzero(ctx, fn11);
-    gen_fp_exc_clear();
 
     va = gen_ieee_input(ctx, ra, fn11, 0);
     vb = gen_ieee_input(ctx, rb, fn11, 0);
@@ -875,8 +869,8 @@
 }
 
 #define IEEE_ARITH3(name)                                               \
-static inline void glue(gen_f, name)(DisasContext *ctx,                 \
-                                     int ra, int rb, int rc, int fn11)  \
+static inline void glue(gen_, name)(DisasContext *ctx,                  \
+                                    int ra, int rb, int rc, int fn11)   \
 {                                                                       \
     gen_ieee_arith3(ctx, gen_helper_##name, ra, rb, rc, fn11);          \
 }
@@ -895,8 +889,6 @@
 {
     TCGv va, vb, vc;
 
-    gen_fp_exc_clear();
-
     va = gen_ieee_input(ctx, ra, fn11, 1);
     vb = gen_ieee_input(ctx, rb, fn11, 1);
     vc = dest_fpr(ctx, rc);
@@ -906,8 +898,8 @@
 }
 
 #define IEEE_CMP3(name)                                                 \
-static inline void glue(gen_f, name)(DisasContext *ctx,                 \
-                                     int ra, int rb, int rc, int fn11)  \
+static inline void glue(gen_, name)(DisasContext *ctx,                  \
+                                    int ra, int rb, int rc, int fn11)   \
 {                                                                       \
     gen_ieee_compare(ctx, gen_helper_##name, ra, rb, rc, fn11);         \
 }
@@ -1105,12 +1097,13 @@
     }
 }
 
-static void gen_rx(int ra, int set)
+static void gen_rx(DisasContext *ctx, int ra, int set)
 {
     TCGv_i32 tmp;
 
     if (ra != 31) {
-        tcg_gen_ld8u_i64(cpu_ir[ra], cpu_env, offsetof(CPUAlphaState, intr_flag));
+        tcg_gen_ld8u_i64(ctx->ir[ra], cpu_env,
+                         offsetof(CPUAlphaState, intr_flag));
     }
 
     tmp = tcg_const_i32(set);
@@ -1132,12 +1125,12 @@
             break;
         case 0x9E:
             /* RDUNIQUE */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, unique));
             break;
         case 0x9F:
             /* WRUNIQUE */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, unique));
             break;
         default:
@@ -1161,17 +1154,17 @@
             break;
         case 0x2D:
             /* WRVPTPTR */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, vptptr));
             break;
         case 0x31:
             /* WRVAL */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, sysval));
             break;
         case 0x32:
             /* RDVAL */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, sysval));
             break;
 
@@ -1181,12 +1174,12 @@
 
             /* Note that we already know we're in kernel mode, so we know
                that PS only contains the 3 IPL bits.  */
-            tcg_gen_ld8u_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld8u_i64(ctx->ir[IR_V0], cpu_env,
                              offsetof(CPUAlphaState, ps));
 
             /* But make sure and store only the 3 IPL bits from the user.  */
             tmp = tcg_temp_new();
-            tcg_gen_andi_i64(tmp, cpu_ir[IR_A0], PS_INT_MASK);
+            tcg_gen_andi_i64(tmp, ctx->ir[IR_A0], PS_INT_MASK);
             tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, ps));
             tcg_temp_free(tmp);
             break;
@@ -1194,22 +1187,22 @@
 
         case 0x36:
             /* RDPS */
-            tcg_gen_ld8u_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld8u_i64(ctx->ir[IR_V0], cpu_env,
                              offsetof(CPUAlphaState, ps));
             break;
         case 0x38:
             /* WRUSP */
-            tcg_gen_st_i64(cpu_ir[IR_A0], cpu_env,
+            tcg_gen_st_i64(ctx->ir[IR_A0], cpu_env,
                            offsetof(CPUAlphaState, usp));
             break;
         case 0x3A:
             /* RDUSP */
-            tcg_gen_ld_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld_i64(ctx->ir[IR_V0], cpu_env,
                            offsetof(CPUAlphaState, usp));
             break;
         case 0x3C:
             /* WHAMI */
-            tcg_gen_ld32s_i64(cpu_ir[IR_V0], cpu_env,
+            tcg_gen_ld32s_i64(ctx->ir[IR_V0], cpu_env,
                 -offsetof(AlphaCPU, env) + offsetof(CPUState, cpu_index));
             break;
 
@@ -1227,15 +1220,24 @@
     return gen_excp(ctx, EXCP_CALL_PAL, palcode);
 #else
     {
-        TCGv pc = tcg_const_i64(ctx->pc);
-        TCGv entry = tcg_const_i64(palcode & 0x80
-                                   ? 0x2000 + (palcode - 0x80) * 64
-                                   : 0x1000 + palcode * 64);
+        TCGv tmp = tcg_temp_new();
+        uint64_t exc_addr = ctx->pc;
+        uint64_t entry = ctx->palbr;
 
-        gen_helper_call_pal(cpu_env, pc, entry);
+        if (ctx->tb->flags & TB_FLAGS_PAL_MODE) {
+            exc_addr |= 1;
+        } else {
+            tcg_gen_movi_i64(tmp, 1);
+            tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, pal_mode));
+        }
 
-        tcg_temp_free(entry);
-        tcg_temp_free(pc);
+        tcg_gen_movi_i64(tmp, exc_addr);
+        tcg_gen_st_i64(tmp, cpu_env, offsetof(CPUAlphaState, exc_addr));
+        tcg_temp_free(tmp);
+
+        entry += (palcode & 0x80
+                  ? 0x2000 + (palcode - 0x80) * 64
+                  : 0x1000 + palcode * 64);
 
         /* Since the destination is running in PALmode, we don't really
            need the page permissions check.  We'll see the existence of
@@ -1243,11 +1245,13 @@
            we change the PAL base register.  */
         if (!ctx->singlestep_enabled && !(ctx->tb->cflags & CF_LAST_IO)) {
             tcg_gen_goto_tb(0);
+            tcg_gen_movi_i64(cpu_pc, entry);
             tcg_gen_exit_tb((uintptr_t)ctx->tb);
             return EXIT_GOTO_TB;
+        } else {
+            tcg_gen_movi_i64(cpu_pc, entry);
+            return EXIT_PC_UPDATED;
         }
-
-        return EXIT_PC_UPDATED;
     }
 #endif
 }
@@ -1274,8 +1278,6 @@
     case 11: return offsetof(CPUAlphaState, sysval);
     case 12: return offsetof(CPUAlphaState, usp);
 
-    case 32 ... 39:
-        return offsetof(CPUAlphaState, shadow[pr - 32]);
     case 40 ... 63:
         return offsetof(CPUAlphaState, scratch[pr - 40]);
 
@@ -1285,16 +1287,24 @@
     return 0;
 }
 
-static ExitStatus gen_mfpr(TCGv va, int regno)
+static ExitStatus gen_mfpr(DisasContext *ctx, TCGv va, int regno)
 {
-    int data = cpu_pr_data(regno);
+    void (*helper)(TCGv);
+    int data;
 
-    /* Special help for VMTIME and WALLTIME.  */
-    if (regno == 250 || regno == 249) {
-	void (*helper)(TCGv) = gen_helper_get_walltime;
-	if (regno == 249) {
-		helper = gen_helper_get_vmtime;
-	}
+    switch (regno) {
+    case 32 ... 39:
+        /* Accessing the "non-shadow" general registers.  */
+        regno = regno == 39 ? 25 : regno - 32 + 8;
+        tcg_gen_mov_i64(va, cpu_std_ir[regno]);
+        break;
+
+    case 250: /* WALLTIME */
+        helper = gen_helper_get_walltime;
+        goto do_helper;
+    case 249: /* VMTIME */
+        helper = gen_helper_get_vmtime;
+    do_helper:
         if (use_icount) {
             gen_io_start();
             helper(va);
@@ -1302,21 +1312,25 @@
             return EXIT_PC_STALE;
         } else {
             helper(va);
-            return NO_EXIT;
         }
+        break;
+
+    default:
+        /* The basic registers are data only, and unknown registers
+           are read-zero, write-ignore.  */
+        data = cpu_pr_data(regno);
+        if (data == 0) {
+            tcg_gen_movi_i64(va, 0);
+        } else if (data & PR_BYTE) {
+            tcg_gen_ld8u_i64(va, cpu_env, data & ~PR_BYTE);
+        } else if (data & PR_LONG) {
+            tcg_gen_ld32s_i64(va, cpu_env, data & ~PR_LONG);
+        } else {
+            tcg_gen_ld_i64(va, cpu_env, data);
+        }
+        break;
     }
 
-    /* The basic registers are data only, and unknown registers
-       are read-zero, write-ignore.  */
-    if (data == 0) {
-        tcg_gen_movi_i64(va, 0);
-    } else if (data & PR_BYTE) {
-        tcg_gen_ld8u_i64(va, cpu_env, data & ~PR_BYTE);
-    } else if (data & PR_LONG) {
-        tcg_gen_ld32s_i64(va, cpu_env, data & ~PR_LONG);
-    } else {
-        tcg_gen_ld_i64(va, cpu_env, data);
-    }
     return NO_EXIT;
 }
 
@@ -1362,6 +1376,12 @@
         gen_helper_tb_flush(cpu_env);
         return EXIT_PC_STALE;
 
+    case 32 ... 39:
+        /* Accessing the "non-shadow" general registers.  */
+        regno = regno == 39 ? 25 : regno - 32 + 8;
+        tcg_gen_mov_i64(cpu_std_ir[regno], vb);
+        break;
+
     default:
         /* The basic registers are data only, and unknown registers
            are read-zero, write-ignore.  */
@@ -1382,6 +1402,13 @@
 }
 #endif /* !USER_ONLY*/
 
+#define REQUIRE_NO_LIT                          \
+    do {                                        \
+        if (real_islit) {                       \
+            goto invalid_opc;                   \
+        }                                       \
+    } while (0)
+
 #define REQUIRE_TB_FLAG(FLAG)                   \
     do {                                        \
         if ((ctx->tb->flags & (FLAG)) == 0) {   \
@@ -1401,8 +1428,8 @@
     int32_t disp21, disp16, disp12 __attribute__((unused));
     uint16_t fn11;
     uint8_t opc, ra, rb, rc, fpfn, fn7, lit;
-    bool islit;
-    TCGv va, vb, vc, tmp;
+    bool islit, real_islit;
+    TCGv va, vb, vc, tmp, tmp2;
     TCGv_i32 t32;
     ExitStatus ret;
 
@@ -1411,7 +1438,7 @@
     ra = extract32(insn, 21, 5);
     rb = extract32(insn, 16, 5);
     rc = extract32(insn, 0, 5);
-    islit = extract32(insn, 12, 1);
+    real_islit = islit = extract32(insn, 12, 1);
     lit = extract32(insn, 13, 8);
 
     disp21 = sextract32(insn, 0, 21);
@@ -1546,7 +1573,12 @@
             break;
         case 0x0F:
             /* CMPBGE */
-            gen_helper_cmpbge(vc, va, vb);
+            if (ra == 31) {
+                /* Special case 0 >= X as X == 0.  */
+                gen_helper_cmpbe0(vc, vb);
+            } else {
+                gen_helper_cmpbge(vc, va, vb);
+            }
             break;
         case 0x12:
             /* S8ADDL */
@@ -1614,11 +1646,23 @@
             break;
         case 0x40:
             /* ADDL/V */
-            gen_helper_addlv(vc, cpu_env, va, vb);
+            tmp = tcg_temp_new();
+            tcg_gen_ext32s_i64(tmp, va);
+            tcg_gen_ext32s_i64(vc, vb);
+            tcg_gen_add_i64(tmp, tmp, vc);
+            tcg_gen_ext32s_i64(vc, tmp);
+            gen_helper_check_overflow(cpu_env, vc, tmp);
+            tcg_temp_free(tmp);
             break;
         case 0x49:
             /* SUBL/V */
-            gen_helper_sublv(vc, cpu_env, va, vb);
+            tmp = tcg_temp_new();
+            tcg_gen_ext32s_i64(tmp, va);
+            tcg_gen_ext32s_i64(vc, vb);
+            tcg_gen_sub_i64(tmp, tmp, vc);
+            tcg_gen_ext32s_i64(vc, tmp);
+            gen_helper_check_overflow(cpu_env, vc, tmp);
+            tcg_temp_free(tmp);
             break;
         case 0x4D:
             /* CMPLT */
@@ -1626,11 +1670,33 @@
             break;
         case 0x60:
             /* ADDQ/V */
-            gen_helper_addqv(vc, cpu_env, va, vb);
+            tmp = tcg_temp_new();
+            tmp2 = tcg_temp_new();
+            tcg_gen_eqv_i64(tmp, va, vb);
+            tcg_gen_mov_i64(tmp2, va);
+            tcg_gen_add_i64(vc, va, vb);
+            tcg_gen_xor_i64(tmp2, tmp2, vc);
+            tcg_gen_and_i64(tmp, tmp, tmp2);
+            tcg_gen_shri_i64(tmp, tmp, 63);
+            tcg_gen_movi_i64(tmp2, 0);
+            gen_helper_check_overflow(cpu_env, tmp, tmp2);
+            tcg_temp_free(tmp);
+            tcg_temp_free(tmp2);
             break;
         case 0x69:
             /* SUBQ/V */
-            gen_helper_subqv(vc, cpu_env, va, vb);
+            tmp = tcg_temp_new();
+            tmp2 = tcg_temp_new();
+            tcg_gen_xor_i64(tmp, va, vb);
+            tcg_gen_mov_i64(tmp2, va);
+            tcg_gen_sub_i64(vc, va, vb);
+            tcg_gen_xor_i64(tmp2, tmp2, vc);
+            tcg_gen_and_i64(tmp, tmp, tmp2);
+            tcg_gen_shri_i64(tmp, tmp, 63);
+            tcg_gen_movi_i64(tmp2, 0);
+            gen_helper_check_overflow(cpu_env, tmp, tmp2);
+            tcg_temp_free(tmp);
+            tcg_temp_free(tmp2);
             break;
         case 0x6D:
             /* CMPLE */
@@ -1925,11 +1991,23 @@
             break;
         case 0x40:
             /* MULL/V */
-            gen_helper_mullv(vc, cpu_env, va, vb);
+            tmp = tcg_temp_new();
+            tcg_gen_ext32s_i64(tmp, va);
+            tcg_gen_ext32s_i64(vc, vb);
+            tcg_gen_mul_i64(tmp, tmp, vc);
+            tcg_gen_ext32s_i64(vc, tmp);
+            gen_helper_check_overflow(cpu_env, vc, tmp);
+            tcg_temp_free(tmp);
             break;
         case 0x60:
             /* MULQ/V */
-            gen_helper_mulqv(vc, cpu_env, va, vb);
+            tmp = tcg_temp_new();
+            tmp2 = tcg_temp_new();
+            tcg_gen_muls2_i64(vc, tmp, va, vb);
+            tcg_gen_sari_i64(tmp2, vc, 63);
+            gen_helper_check_overflow(cpu_env, tmp, tmp2);
+            tcg_temp_free(tmp);
+            tcg_temp_free(tmp2);
             break;
         default:
             goto invalid_opc;
@@ -1945,7 +2023,7 @@
             REQUIRE_REG_31(rb);
             t32 = tcg_temp_new_i32();
             va = load_gpr(ctx, ra);
-            tcg_gen_trunc_i64_i32(t32, va);
+            tcg_gen_extrl_i64_i32(t32, va);
             gen_helper_memory_to_s(vc, t32);
             tcg_temp_free_i32(t32);
             break;
@@ -1958,14 +2036,14 @@
         case 0x0B:
             /* SQRTS */
             REQUIRE_REG_31(ra);
-            gen_fsqrts(ctx, rb, rc, fn11);
+            gen_sqrts(ctx, rb, rc, fn11);
             break;
         case 0x14:
             /* ITOFF */
             REQUIRE_REG_31(rb);
             t32 = tcg_temp_new_i32();
             va = load_gpr(ctx, ra);
-            tcg_gen_trunc_i64_i32(t32, va);
+            tcg_gen_extrl_i64_i32(t32, va);
             gen_helper_memory_to_f(vc, t32);
             tcg_temp_free_i32(t32);
             break;
@@ -1984,7 +2062,7 @@
         case 0x02B:
             /* SQRTT */
             REQUIRE_REG_31(ra);
-            gen_fsqrtt(ctx, rb, rc, fn11);
+            gen_sqrtt(ctx, rb, rc, fn11);
             break;
         default:
             goto invalid_opc;
@@ -2080,76 +2158,76 @@
         switch (fpfn) { /* fn11 & 0x3F */
         case 0x00:
             /* ADDS */
-            gen_fadds(ctx, ra, rb, rc, fn11);
+            gen_adds(ctx, ra, rb, rc, fn11);
             break;
         case 0x01:
             /* SUBS */
-            gen_fsubs(ctx, ra, rb, rc, fn11);
+            gen_subs(ctx, ra, rb, rc, fn11);
             break;
         case 0x02:
             /* MULS */
-            gen_fmuls(ctx, ra, rb, rc, fn11);
+            gen_muls(ctx, ra, rb, rc, fn11);
             break;
         case 0x03:
             /* DIVS */
-            gen_fdivs(ctx, ra, rb, rc, fn11);
+            gen_divs(ctx, ra, rb, rc, fn11);
             break;
         case 0x20:
             /* ADDT */
-            gen_faddt(ctx, ra, rb, rc, fn11);
+            gen_addt(ctx, ra, rb, rc, fn11);
             break;
         case 0x21:
             /* SUBT */
-            gen_fsubt(ctx, ra, rb, rc, fn11);
+            gen_subt(ctx, ra, rb, rc, fn11);
             break;
         case 0x22:
             /* MULT */
-            gen_fmult(ctx, ra, rb, rc, fn11);
+            gen_mult(ctx, ra, rb, rc, fn11);
             break;
         case 0x23:
             /* DIVT */
-            gen_fdivt(ctx, ra, rb, rc, fn11);
+            gen_divt(ctx, ra, rb, rc, fn11);
             break;
         case 0x24:
             /* CMPTUN */
-            gen_fcmptun(ctx, ra, rb, rc, fn11);
+            gen_cmptun(ctx, ra, rb, rc, fn11);
             break;
         case 0x25:
             /* CMPTEQ */
-            gen_fcmpteq(ctx, ra, rb, rc, fn11);
+            gen_cmpteq(ctx, ra, rb, rc, fn11);
             break;
         case 0x26:
             /* CMPTLT */
-            gen_fcmptlt(ctx, ra, rb, rc, fn11);
+            gen_cmptlt(ctx, ra, rb, rc, fn11);
             break;
         case 0x27:
             /* CMPTLE */
-            gen_fcmptle(ctx, ra, rb, rc, fn11);
+            gen_cmptle(ctx, ra, rb, rc, fn11);
             break;
         case 0x2C:
             REQUIRE_REG_31(ra);
             if (fn11 == 0x2AC || fn11 == 0x6AC) {
                 /* CVTST */
-                gen_fcvtst(ctx, rb, rc, fn11);
+                gen_cvtst(ctx, rb, rc, fn11);
             } else {
                 /* CVTTS */
-                gen_fcvtts(ctx, rb, rc, fn11);
+                gen_cvtts(ctx, rb, rc, fn11);
             }
             break;
         case 0x2F:
             /* CVTTQ */
             REQUIRE_REG_31(ra);
-            gen_fcvttq(ctx, rb, rc, fn11);
+            gen_cvttq(ctx, rb, rc, fn11);
             break;
         case 0x3C:
             /* CVTQS */
             REQUIRE_REG_31(ra);
-            gen_fcvtqs(ctx, rb, rc, fn11);
+            gen_cvtqs(ctx, rb, rc, fn11);
             break;
         case 0x3E:
             /* CVTQT */
             REQUIRE_REG_31(ra);
-            gen_fcvtqt(ctx, rb, rc, fn11);
+            gen_cvtqt(ctx, rb, rc, fn11);
             break;
         default:
             goto invalid_opc;
@@ -2163,7 +2241,7 @@
             REQUIRE_REG_31(ra);
             vc = dest_fpr(ctx, rc);
             vb = load_fpr(ctx, rb);
-            gen_fcvtlq(vc, vb);
+            gen_cvtlq(vc, vb);
             break;
         case 0x020:
             /* CPYS */
@@ -2199,6 +2277,11 @@
             /* MT_FPCR */
             va = load_fpr(ctx, ra);
             gen_helper_store_fpcr(cpu_env, va);
+            if (ctx->tb_rm == QUAL_RM_D) {
+                /* Re-do the copy of the rounding mode to fp_status
+                   the next time we use dynamic rounding.  */
+                ctx->tb_rm = -1;
+            }
             break;
         case 0x025:
             /* MF_FPCR */
@@ -2229,25 +2312,14 @@
             /* FCMOVGT */
             gen_fcmov(ctx, TCG_COND_GT, ra, rb, rc);
             break;
-        case 0x030:
-            /* CVTQL */
+        case 0x030: /* CVTQL */
+        case 0x130: /* CVTQL/V */
+        case 0x530: /* CVTQL/SV */
             REQUIRE_REG_31(ra);
             vc = dest_fpr(ctx, rc);
             vb = load_fpr(ctx, rb);
-            gen_fcvtql(vc, vb);
-            break;
-        case 0x130:
-            /* CVTQL/V */
-        case 0x530:
-            /* CVTQL/SV */
-            REQUIRE_REG_31(ra);
-            /* ??? I'm pretty sure there's nothing that /sv needs to do that
-               /v doesn't do.  The only thing I can think is that /sv is a
-               valid instruction merely for completeness in the ISA.  */
-            vc = dest_fpr(ctx, rc);
-            vb = load_fpr(ctx, rb);
-            gen_helper_fcvtql_v_input(cpu_env, vb);
-            gen_fcvtql(vc, vb);
+            gen_helper_cvtql(vc, cpu_env, vb);
+            gen_fp_exc_raise(rc, fn11);
             break;
         default:
             goto invalid_opc;
@@ -2283,7 +2355,7 @@
         case 0xC000:
             /* RPCC */
             va = dest_gpr(ctx, ra);
-            if (use_icount) {
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_start();
                 gen_helper_load_pcc(va, cpu_env);
                 gen_io_end();
@@ -2294,19 +2366,23 @@
             break;
         case 0xE000:
             /* RC */
-            gen_rx(ra, 0);
+            gen_rx(ctx, ra, 0);
             break;
         case 0xE800:
             /* ECB */
             break;
         case 0xF000:
             /* RS */
-            gen_rx(ra, 1);
+            gen_rx(ctx, ra, 1);
             break;
         case 0xF800:
             /* WH64 */
             /* No-op */
             break;
+        case 0xFC00:
+            /* WH64EN */
+            /* No-op */
+            break;
         default:
             goto invalid_opc;
         }
@@ -2317,7 +2393,7 @@
 #ifndef CONFIG_USER_ONLY
         REQUIRE_TB_FLAG(TB_FLAGS_PAL_MODE);
         va = dest_gpr(ctx, ra);
-        ret = gen_mfpr(va, insn & 0xffff);
+        ret = gen_mfpr(ctx, va, insn & 0xffff);
         break;
 #else
         goto invalid_opc;
@@ -2329,7 +2405,7 @@
         vb = load_gpr(ctx, rb);
         tcg_gen_andi_i64(cpu_pc, vb, ~3);
         if (ra != 31) {
-            tcg_gen_movi_i64(cpu_ir[ra], ctx->pc);
+            tcg_gen_movi_i64(ctx->ir[ra], ctx->pc);
         }
         ret = EXIT_PC_UPDATED;
         break;
@@ -2369,10 +2445,10 @@
                 goto invalid_opc;
                 break;
             case 0x6:
-                /* Incpu_ir[ra]id */
+                /* Invalid */
                 goto invalid_opc;
             case 0x7:
-                /* Incpu_ir[ra]id */
+                /* Invaliid */
                 goto invalid_opc;
             case 0x8:
                 /* Longword virtual access (hw_ldl) */
@@ -2451,11 +2527,13 @@
             /* CTPOP */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_CIX);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_ctpop(vc, vb);
             break;
         case 0x31:
             /* PERR */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_MVI);
+            REQUIRE_NO_LIT;
             va = load_gpr(ctx, ra);
             gen_helper_perr(vc, va, vb);
             break;
@@ -2463,36 +2541,42 @@
             /* CTLZ */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_CIX);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_ctlz(vc, vb);
             break;
         case 0x33:
             /* CTTZ */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_CIX);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_cttz(vc, vb);
             break;
         case 0x34:
             /* UNPKBW */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_MVI);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_unpkbw(vc, vb);
             break;
         case 0x35:
             /* UNPKBL */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_MVI);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_unpkbl(vc, vb);
             break;
         case 0x36:
             /* PKWB */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_MVI);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_pkwb(vc, vb);
             break;
         case 0x37:
             /* PKLB */
             REQUIRE_TB_FLAG(TB_FLAGS_AMASK_MVI);
             REQUIRE_REG_31(ra);
+            REQUIRE_NO_LIT;
             gen_helper_pklb(vc, vb);
             break;
         case 0x38:
@@ -2567,13 +2651,18 @@
             /* Pre-EV6 CPUs interpreted this as HW_REI, loading the return
                address from EXC_ADDR.  This turns out to be useful for our
                emulation PALcode, so continue to accept it.  */
-            tmp = tcg_temp_new();
-            tcg_gen_ld_i64(tmp, cpu_env, offsetof(CPUAlphaState, exc_addr));
-            gen_helper_hw_ret(cpu_env, tmp);
-            tcg_temp_free(tmp);
+            ctx->lit = vb = tcg_temp_new();
+            tcg_gen_ld_i64(vb, cpu_env, offsetof(CPUAlphaState, exc_addr));
         } else {
-            gen_helper_hw_ret(cpu_env, load_gpr(ctx, rb));
+            vb = load_gpr(ctx, rb);
         }
+        tmp = tcg_temp_new();
+        tcg_gen_movi_i64(tmp, 0);
+        tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, intr_flag));
+        tcg_gen_movi_i64(cpu_lock_addr, -1);
+        tcg_gen_andi_i64(tmp, vb, 1);
+        tcg_gen_st8_i64(tmp, cpu_env, offsetof(CPUAlphaState, pal_mode));
+        tcg_gen_andi_i64(cpu_pc, vb, ~3);
         ret = EXIT_PC_UPDATED;
         break;
 #else
@@ -2780,18 +2869,14 @@
     return ret;
 }
 
-static inline void gen_intermediate_code_internal(AlphaCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUAlphaState *env, struct TranslationBlock *tb)
 {
+    AlphaCPU *cpu = alpha_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUAlphaState *env = &cpu->env;
     DisasContext ctx, *ctxp = &ctx;
     target_ulong pc_start;
     target_ulong pc_mask;
     uint32_t insn;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     ExitStatus ret;
     int num_insns;
     int max_insns;
@@ -2800,10 +2885,17 @@
 
     ctx.tb = tb;
     ctx.pc = pc_start;
-    ctx.mem_idx = cpu_mmu_index(env);
+    ctx.mem_idx = cpu_mmu_index(env, false);
     ctx.implver = env->implver;
     ctx.singlestep_enabled = cs->singlestep_enabled;
 
+#ifdef CONFIG_USER_ONLY
+    ctx.ir = cpu_std_ir;
+#else
+    ctx.palbr = env->palbr;
+    ctx.ir = (tb->flags & TB_FLAGS_PAL_MODE ? cpu_pal_ir : cpu_std_ir);
+#endif
+
     /* ??? Every TB begins with unset rounding mode, to be initialized on
        the first fp insn of the TB.  Alternately we could define a proper
        default for every TB (e.g. QUAL_RM_N or QUAL_RM_D) and make sure
@@ -2819,6 +2911,9 @@
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     if (in_superpage(&ctx, pc_start)) {
         pc_mask = (1ULL << 41) - 1;
@@ -2826,37 +2921,24 @@
         pc_mask = ~TARGET_PAGE_MASK;
     }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.pc) {
-                    gen_excp(&ctx, EXCP_DEBUG, 0);
-                    break;
-                }
-            }
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            ret = gen_excp(&ctx, EXCP_DEBUG, 0);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 4;
+            break;
         }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         insn = cpu_ldl_code(env, ctx.pc);
-        num_insns++;
-
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(ctx.pc);
-        }
 
         TCGV_UNUSED_I64(ctx.zero);
         TCGV_UNUSED_I64(ctx.sink);
@@ -2912,37 +2994,21 @@
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, ctx.pc - pc_start, 1);
+        log_target_disas(cs, pc_start, ctx.pc - pc_start, 1);
         qemu_log("\n");
     }
 #endif
 }
 
-void gen_intermediate_code (CPUAlphaState *env, struct TranslationBlock *tb)
+void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(alpha_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUAlphaState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(alpha_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUAlphaState *env, TranslationBlock *tb, int pc_pos)
-{
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-alpha/vax_helper.c b/target-alpha/vax_helper.c
new file mode 100644
index 0000000..2b0c178
--- /dev/null
+++ b/target-alpha/vax_helper.c
@@ -0,0 +1,355 @@
+/*
+ *  Helpers for vax floating point instructions.
+ *
+ *  Copyright (c) 2007 Jocelyn Mayer
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
+#include "exec/helper-proto.h"
+#include "fpu/softfloat.h"
+
+#define FP_STATUS (env->fp_status)
+
+
+/* F floating (VAX) */
+static uint64_t float32_to_f(float32 fa)
+{
+    uint64_t r, exp, mant, sig;
+    CPU_FloatU a;
+
+    a.f = fa;
+    sig = ((uint64_t)a.l & 0x80000000) << 32;
+    exp = (a.l >> 23) & 0xff;
+    mant = ((uint64_t)a.l & 0x007fffff) << 29;
+
+    if (exp == 255) {
+        /* NaN or infinity */
+        r = 1; /* VAX dirty zero */
+    } else if (exp == 0) {
+        if (mant == 0) {
+            /* Zero */
+            r = 0;
+        } else {
+            /* Denormalized */
+            r = sig | ((exp + 1) << 52) | mant;
+        }
+    } else {
+        if (exp >= 253) {
+            /* Overflow */
+            r = 1; /* VAX dirty zero */
+        } else {
+            r = sig | ((exp + 2) << 52);
+        }
+    }
+
+    return r;
+}
+
+static float32 f_to_float32(CPUAlphaState *env, uintptr_t retaddr, uint64_t a)
+{
+    uint32_t exp, mant_sig;
+    CPU_FloatU r;
+
+    exp = ((a >> 55) & 0x80) | ((a >> 52) & 0x7f);
+    mant_sig = ((a >> 32) & 0x80000000) | ((a >> 29) & 0x007fffff);
+
+    if (unlikely(!exp && mant_sig)) {
+        /* Reserved operands / Dirty zero */
+        dynamic_excp(env, retaddr, EXCP_OPCDEC, 0);
+    }
+
+    if (exp < 3) {
+        /* Underflow */
+        r.l = 0;
+    } else {
+        r.l = ((exp - 2) << 23) | mant_sig;
+    }
+
+    return r.f;
+}
+
+uint32_t helper_f_to_memory(uint64_t a)
+{
+    uint32_t r;
+    r =  (a & 0x00001fffe0000000ull) >> 13;
+    r |= (a & 0x07ffe00000000000ull) >> 45;
+    r |= (a & 0xc000000000000000ull) >> 48;
+    return r;
+}
+
+uint64_t helper_memory_to_f(uint32_t a)
+{
+    uint64_t r;
+    r =  ((uint64_t)(a & 0x0000c000)) << 48;
+    r |= ((uint64_t)(a & 0x003fffff)) << 45;
+    r |= ((uint64_t)(a & 0xffff0000)) << 13;
+    if (!(a & 0x00004000)) {
+        r |= 0x7ll << 59;
+    }
+    return r;
+}
+
+/* ??? Emulating VAX arithmetic with IEEE arithmetic is wrong.  We should
+   either implement VAX arithmetic properly or just signal invalid opcode.  */
+
+uint64_t helper_addf(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float32 fa, fb, fr;
+
+    fa = f_to_float32(env, GETPC(), a);
+    fb = f_to_float32(env, GETPC(), b);
+    fr = float32_add(fa, fb, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+uint64_t helper_subf(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float32 fa, fb, fr;
+
+    fa = f_to_float32(env, GETPC(), a);
+    fb = f_to_float32(env, GETPC(), b);
+    fr = float32_sub(fa, fb, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+uint64_t helper_mulf(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float32 fa, fb, fr;
+
+    fa = f_to_float32(env, GETPC(), a);
+    fb = f_to_float32(env, GETPC(), b);
+    fr = float32_mul(fa, fb, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+uint64_t helper_divf(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float32 fa, fb, fr;
+
+    fa = f_to_float32(env, GETPC(), a);
+    fb = f_to_float32(env, GETPC(), b);
+    fr = float32_div(fa, fb, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+uint64_t helper_sqrtf(CPUAlphaState *env, uint64_t t)
+{
+    float32 ft, fr;
+
+    ft = f_to_float32(env, GETPC(), t);
+    fr = float32_sqrt(ft, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+
+/* G floating (VAX) */
+static uint64_t float64_to_g(float64 fa)
+{
+    uint64_t r, exp, mant, sig;
+    CPU_DoubleU a;
+
+    a.d = fa;
+    sig = a.ll & 0x8000000000000000ull;
+    exp = (a.ll >> 52) & 0x7ff;
+    mant = a.ll & 0x000fffffffffffffull;
+
+    if (exp == 2047) {
+        /* NaN or infinity */
+        r = 1; /* VAX dirty zero */
+    } else if (exp == 0) {
+        if (mant == 0) {
+            /* Zero */
+            r = 0;
+        } else {
+            /* Denormalized */
+            r = sig | ((exp + 1) << 52) | mant;
+        }
+    } else {
+        if (exp >= 2045) {
+            /* Overflow */
+            r = 1; /* VAX dirty zero */
+        } else {
+            r = sig | ((exp + 2) << 52);
+        }
+    }
+
+    return r;
+}
+
+static float64 g_to_float64(CPUAlphaState *env, uintptr_t retaddr, uint64_t a)
+{
+    uint64_t exp, mant_sig;
+    CPU_DoubleU r;
+
+    exp = (a >> 52) & 0x7ff;
+    mant_sig = a & 0x800fffffffffffffull;
+
+    if (!exp && mant_sig) {
+        /* Reserved operands / Dirty zero */
+        dynamic_excp(env, retaddr, EXCP_OPCDEC, 0);
+    }
+
+    if (exp < 3) {
+        /* Underflow */
+        r.ll = 0;
+    } else {
+        r.ll = ((exp - 2) << 52) | mant_sig;
+    }
+
+    return r.d;
+}
+
+uint64_t helper_g_to_memory(uint64_t a)
+{
+    uint64_t r;
+    r =  (a & 0x000000000000ffffull) << 48;
+    r |= (a & 0x00000000ffff0000ull) << 16;
+    r |= (a & 0x0000ffff00000000ull) >> 16;
+    r |= (a & 0xffff000000000000ull) >> 48;
+    return r;
+}
+
+uint64_t helper_memory_to_g(uint64_t a)
+{
+    uint64_t r;
+    r =  (a & 0x000000000000ffffull) << 48;
+    r |= (a & 0x00000000ffff0000ull) << 16;
+    r |= (a & 0x0000ffff00000000ull) >> 16;
+    r |= (a & 0xffff000000000000ull) >> 48;
+    return r;
+}
+
+uint64_t helper_addg(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb, fr;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+    fr = float64_add(fa, fb, &FP_STATUS);
+    return float64_to_g(fr);
+}
+
+uint64_t helper_subg(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb, fr;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+    fr = float64_sub(fa, fb, &FP_STATUS);
+    return float64_to_g(fr);
+}
+
+uint64_t helper_mulg(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb, fr;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+    fr = float64_mul(fa, fb, &FP_STATUS);
+    return float64_to_g(fr);
+}
+
+uint64_t helper_divg(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb, fr;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+    fr = float64_div(fa, fb, &FP_STATUS);
+    return float64_to_g(fr);
+}
+
+uint64_t helper_sqrtg(CPUAlphaState *env, uint64_t a)
+{
+    float64 fa, fr;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fr = float64_sqrt(fa, &FP_STATUS);
+    return float64_to_g(fr);
+}
+
+uint64_t helper_cmpgeq(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+
+    if (float64_eq_quiet(fa, fb, &FP_STATUS)) {
+        return 0x4000000000000000ULL;
+    } else {
+        return 0;
+    }
+}
+
+uint64_t helper_cmpgle(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+
+    if (float64_le(fa, fb, &FP_STATUS)) {
+        return 0x4000000000000000ULL;
+    } else {
+        return 0;
+    }
+}
+
+uint64_t helper_cmpglt(CPUAlphaState *env, uint64_t a, uint64_t b)
+{
+    float64 fa, fb;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fb = g_to_float64(env, GETPC(), b);
+
+    if (float64_lt(fa, fb, &FP_STATUS)) {
+        return 0x4000000000000000ULL;
+    } else {
+        return 0;
+    }
+}
+
+uint64_t helper_cvtqf(CPUAlphaState *env, uint64_t a)
+{
+    float32 fr = int64_to_float32(a, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+uint64_t helper_cvtgf(CPUAlphaState *env, uint64_t a)
+{
+    float64 fa;
+    float32 fr;
+
+    fa = g_to_float64(env, GETPC(), a);
+    fr = float64_to_float32(fa, &FP_STATUS);
+    return float32_to_f(fr);
+}
+
+uint64_t helper_cvtgq(CPUAlphaState *env, uint64_t a)
+{
+    float64 fa = g_to_float64(env, GETPC(), a);
+    return float64_to_int64_round_to_zero(fa, &FP_STATUS);
+}
+
+uint64_t helper_cvtqg(CPUAlphaState *env, uint64_t a)
+{
+    float64 fr;
+    fr = int64_to_float64(a, &FP_STATUS);
+    return float64_to_g(fr);
+}
diff --git a/target-arm/Makefile.objs b/target-arm/Makefile.objs
index 9460b40..f206411 100644
--- a/target-arm/Makefile.objs
+++ b/target-arm/Makefile.objs
@@ -1,5 +1,5 @@
 obj-y += arm-semi.o
-obj-$(CONFIG_SOFTMMU) += machine.o
+obj-$(CONFIG_SOFTMMU) += machine.o psci.o arch_dump.o monitor.o
 obj-$(CONFIG_KVM) += kvm.o
 obj-$(call land,$(CONFIG_KVM),$(call lnot,$(TARGET_AARCH64))) += kvm32.o
 obj-$(call land,$(CONFIG_KVM),$(TARGET_AARCH64)) += kvm64.o
@@ -7,6 +7,6 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += psci.o
 obj-$(TARGET_AARCH64) += cpu64.o translate-a64.o helper-a64.o gdbstub64.o
 obj-y += crypto_helper.o
+obj-y += arm-powerctl.o
diff --git a/target-arm/arch_dump.c b/target-arm/arch_dump.c
new file mode 100644
index 0000000..1a9861f
--- /dev/null
+++ b/target-arm/arch_dump.c
@@ -0,0 +1,337 @@
+/* Support for writing ELF notes for ARM architectures
+ *
+ * Copyright (C) 2015 Red Hat Inc.
+ *
+ * Author: Andrew Jones <drjones@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "elf.h"
+#include "sysemu/dump.h"
+
+/* struct user_pt_regs from arch/arm64/include/uapi/asm/ptrace.h */
+struct aarch64_user_regs {
+    uint64_t regs[31];
+    uint64_t sp;
+    uint64_t pc;
+    uint64_t pstate;
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct aarch64_user_regs) != 272);
+
+/* struct elf_prstatus from include/uapi/linux/elfcore.h */
+struct aarch64_elf_prstatus {
+    char pad1[32]; /* 32 == offsetof(struct elf_prstatus, pr_pid) */
+    uint32_t pr_pid;
+    char pad2[76]; /* 76 == offsetof(struct elf_prstatus, pr_reg) -
+                            offsetof(struct elf_prstatus, pr_ppid) */
+    struct aarch64_user_regs pr_reg;
+    uint32_t pr_fpvalid;
+    char pad3[4];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct aarch64_elf_prstatus) != 392);
+
+/* struct user_fpsimd_state from arch/arm64/include/uapi/asm/ptrace.h
+ *
+ * While the vregs member of user_fpsimd_state is of type __uint128_t,
+ * QEMU uses an array of uint64_t, where the high half of the 128-bit
+ * value is always in the 2n+1'th index. Thus we also break the 128-
+ * bit values into two halves in this reproduction of user_fpsimd_state.
+ */
+struct aarch64_user_vfp_state {
+    uint64_t vregs[64];
+    uint32_t fpsr;
+    uint32_t fpcr;
+    char pad[8];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct aarch64_user_vfp_state) != 528);
+
+struct aarch64_note {
+    Elf64_Nhdr hdr;
+    char name[8]; /* align_up(sizeof("CORE"), 4) */
+    union {
+        struct aarch64_elf_prstatus prstatus;
+        struct aarch64_user_vfp_state vfp;
+    };
+} QEMU_PACKED;
+
+#define AARCH64_NOTE_HEADER_SIZE offsetof(struct aarch64_note, prstatus)
+#define AARCH64_PRSTATUS_NOTE_SIZE \
+            (AARCH64_NOTE_HEADER_SIZE + sizeof(struct aarch64_elf_prstatus))
+#define AARCH64_PRFPREG_NOTE_SIZE \
+            (AARCH64_NOTE_HEADER_SIZE + sizeof(struct aarch64_user_vfp_state))
+
+static void aarch64_note_init(struct aarch64_note *note, DumpState *s,
+                              const char *name, Elf64_Word namesz,
+                              Elf64_Word type, Elf64_Word descsz)
+{
+    memset(note, 0, sizeof(*note));
+
+    note->hdr.n_namesz = cpu_to_dump32(s, namesz);
+    note->hdr.n_descsz = cpu_to_dump32(s, descsz);
+    note->hdr.n_type = cpu_to_dump32(s, type);
+
+    memcpy(note->name, name, namesz);
+}
+
+static int aarch64_write_elf64_prfpreg(WriteCoreDumpFunction f,
+                                       CPUARMState *env, int cpuid,
+                                       DumpState *s)
+{
+    struct aarch64_note note;
+    int ret, i;
+
+    aarch64_note_init(&note, s, "CORE", 5, NT_PRFPREG, sizeof(note.vfp));
+
+    for (i = 0; i < 64; ++i) {
+        note.vfp.vregs[i] = cpu_to_dump64(s, float64_val(env->vfp.regs[i]));
+    }
+
+    if (s->dump_info.d_endian == ELFDATA2MSB) {
+        /* For AArch64 we must always swap the vfp.regs's 2n and 2n+1
+         * entries when generating BE notes, because even big endian
+         * hosts use 2n+1 for the high half.
+         */
+        for (i = 0; i < 32; ++i) {
+            uint64_t tmp = note.vfp.vregs[2*i];
+            note.vfp.vregs[2*i] = note.vfp.vregs[2*i+1];
+            note.vfp.vregs[2*i+1] = tmp;
+        }
+    }
+
+    note.vfp.fpsr = cpu_to_dump32(s, vfp_get_fpsr(env));
+    note.vfp.fpcr = cpu_to_dump32(s, vfp_get_fpcr(env));
+
+    ret = f(&note, AARCH64_PRFPREG_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+int arm_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque)
+{
+    struct aarch64_note note;
+    CPUARMState *env = &ARM_CPU(cs)->env;
+    DumpState *s = opaque;
+    uint64_t pstate, sp;
+    int ret, i;
+
+    aarch64_note_init(&note, s, "CORE", 5, NT_PRSTATUS, sizeof(note.prstatus));
+
+    note.prstatus.pr_pid = cpu_to_dump32(s, cpuid);
+    note.prstatus.pr_fpvalid = cpu_to_dump32(s, 1);
+
+    if (!is_a64(env)) {
+        aarch64_sync_32_to_64(env);
+        pstate = cpsr_read(env);
+        sp = 0;
+    } else {
+        pstate = pstate_read(env);
+        sp = env->xregs[31];
+    }
+
+    for (i = 0; i < 31; ++i) {
+        note.prstatus.pr_reg.regs[i] = cpu_to_dump64(s, env->xregs[i]);
+    }
+    note.prstatus.pr_reg.sp = cpu_to_dump64(s, sp);
+    note.prstatus.pr_reg.pc = cpu_to_dump64(s, env->pc);
+    note.prstatus.pr_reg.pstate = cpu_to_dump64(s, pstate);
+
+    ret = f(&note, AARCH64_PRSTATUS_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return aarch64_write_elf64_prfpreg(f, env, cpuid, s);
+}
+
+/* struct pt_regs from arch/arm/include/asm/ptrace.h */
+struct arm_user_regs {
+    uint32_t regs[17];
+    char pad[4];
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct arm_user_regs) != 72);
+
+/* struct elf_prstatus from include/uapi/linux/elfcore.h */
+struct arm_elf_prstatus {
+    char pad1[24]; /* 24 == offsetof(struct elf_prstatus, pr_pid) */
+    uint32_t pr_pid;
+    char pad2[44]; /* 44 == offsetof(struct elf_prstatus, pr_reg) -
+                            offsetof(struct elf_prstatus, pr_ppid) */
+    struct arm_user_regs pr_reg;
+    uint32_t pr_fpvalid;
+} QEMU_PACKED arm_elf_prstatus;
+
+QEMU_BUILD_BUG_ON(sizeof(struct arm_elf_prstatus) != 148);
+
+/* struct user_vfp from arch/arm/include/asm/user.h */
+struct arm_user_vfp_state {
+    uint64_t vregs[32];
+    uint32_t fpscr;
+} QEMU_PACKED;
+
+QEMU_BUILD_BUG_ON(sizeof(struct arm_user_vfp_state) != 260);
+
+struct arm_note {
+    Elf32_Nhdr hdr;
+    char name[8]; /* align_up(sizeof("LINUX"), 4) */
+    union {
+        struct arm_elf_prstatus prstatus;
+        struct arm_user_vfp_state vfp;
+    };
+} QEMU_PACKED;
+
+#define ARM_NOTE_HEADER_SIZE offsetof(struct arm_note, prstatus)
+#define ARM_PRSTATUS_NOTE_SIZE \
+            (ARM_NOTE_HEADER_SIZE + sizeof(struct arm_elf_prstatus))
+#define ARM_VFP_NOTE_SIZE \
+            (ARM_NOTE_HEADER_SIZE + sizeof(struct arm_user_vfp_state))
+
+static void arm_note_init(struct arm_note *note, DumpState *s,
+                          const char *name, Elf32_Word namesz,
+                          Elf32_Word type, Elf32_Word descsz)
+{
+    memset(note, 0, sizeof(*note));
+
+    note->hdr.n_namesz = cpu_to_dump32(s, namesz);
+    note->hdr.n_descsz = cpu_to_dump32(s, descsz);
+    note->hdr.n_type = cpu_to_dump32(s, type);
+
+    memcpy(note->name, name, namesz);
+}
+
+static int arm_write_elf32_vfp(WriteCoreDumpFunction f, CPUARMState *env,
+                               int cpuid, DumpState *s)
+{
+    struct arm_note note;
+    int ret, i;
+
+    arm_note_init(&note, s, "LINUX", 6, NT_ARM_VFP, sizeof(note.vfp));
+
+    for (i = 0; i < 32; ++i) {
+        note.vfp.vregs[i] = cpu_to_dump64(s, float64_val(env->vfp.regs[i]));
+    }
+
+    note.vfp.fpscr = cpu_to_dump32(s, vfp_get_fpscr(env));
+
+    ret = f(&note, ARM_VFP_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+int arm_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque)
+{
+    struct arm_note note;
+    CPUARMState *env = &ARM_CPU(cs)->env;
+    DumpState *s = opaque;
+    int ret, i, fpvalid = !!arm_feature(env, ARM_FEATURE_VFP);
+
+    arm_note_init(&note, s, "CORE", 5, NT_PRSTATUS, sizeof(note.prstatus));
+
+    note.prstatus.pr_pid = cpu_to_dump32(s, cpuid);
+    note.prstatus.pr_fpvalid = cpu_to_dump32(s, fpvalid);
+
+    for (i = 0; i < 16; ++i) {
+        note.prstatus.pr_reg.regs[i] = cpu_to_dump32(s, env->regs[i]);
+    }
+    note.prstatus.pr_reg.regs[16] = cpu_to_dump32(s, cpsr_read(env));
+
+    ret = f(&note, ARM_PRSTATUS_NOTE_SIZE, s);
+    if (ret < 0) {
+        return -1;
+    } else if (fpvalid) {
+        return arm_write_elf32_vfp(f, env, cpuid, s);
+    }
+
+    return 0;
+}
+
+int cpu_get_dump_info(ArchDumpInfo *info,
+                      const GuestPhysBlockList *guest_phys_blocks)
+{
+    ARMCPU *cpu = ARM_CPU(first_cpu);
+    CPUARMState *env = &cpu->env;
+    GuestPhysBlock *block;
+    hwaddr lowest_addr = ULLONG_MAX;
+
+    /* Take a best guess at the phys_base. If we get it wrong then crash
+     * will need '--machdep phys_offset=<phys-offset>' added to its command
+     * line, which isn't any worse than assuming we can use zero, but being
+     * wrong. This is the same algorithm the crash utility uses when
+     * attempting to guess as it loads non-dumpfile formatted files.
+     */
+    QTAILQ_FOREACH(block, &guest_phys_blocks->head, next) {
+        if (block->target_start < lowest_addr) {
+            lowest_addr = block->target_start;
+        }
+    }
+
+    if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+        info->d_machine = EM_AARCH64;
+        info->d_class = ELFCLASS64;
+        info->page_size = (1 << 16); /* aarch64 max pagesize */
+        if (lowest_addr != ULLONG_MAX) {
+            info->phys_base = lowest_addr;
+        }
+    } else {
+        info->d_machine = EM_ARM;
+        info->d_class = ELFCLASS32;
+        info->page_size = (1 << 12);
+        if (lowest_addr < UINT_MAX) {
+            info->phys_base = lowest_addr;
+        }
+    }
+
+    /* We assume the relevant endianness is that of EL1; this is right
+     * for kernels, but might give the wrong answer if you're trying to
+     * dump a hypervisor that happens to be running an opposite-endian
+     * kernel.
+     */
+    info->d_endian = (env->cp15.sctlr_el[1] & SCTLR_EE) != 0
+                     ? ELFDATA2MSB : ELFDATA2LSB;
+
+    return 0;
+}
+
+ssize_t cpu_get_note_size(int class, int machine, int nr_cpus)
+{
+    ARMCPU *cpu = ARM_CPU(first_cpu);
+    CPUARMState *env = &cpu->env;
+    size_t note_size;
+
+    if (class == ELFCLASS64) {
+        note_size = AARCH64_PRSTATUS_NOTE_SIZE;
+        note_size += AARCH64_PRFPREG_NOTE_SIZE;
+    } else {
+        note_size = ARM_PRSTATUS_NOTE_SIZE;
+        if (arm_feature(env, ARM_FEATURE_VFP)) {
+            note_size += ARM_VFP_NOTE_SIZE;
+        }
+    }
+
+    return note_size * nr_cpus;
+}
diff --git a/target-arm/arm-powerctl.c b/target-arm/arm-powerctl.c
new file mode 100644
index 0000000..6519d52
--- /dev/null
+++ b/target-arm/arm-powerctl.c
@@ -0,0 +1,226 @@
+/*
+ * QEMU support -- ARM Power Control specific functions.
+ *
+ * Copyright (c) 2016 Jean-Christophe Dubois
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "cpu-qom.h"
+#include "internals.h"
+#include "arm-powerctl.h"
+#include "qemu/log.h"
+#include "exec/exec-all.h"
+
+#ifndef DEBUG_ARM_POWERCTL
+#define DEBUG_ARM_POWERCTL 0
+#endif
+
+#define DPRINTF(fmt, args...) \
+    do { \
+        if (DEBUG_ARM_POWERCTL) { \
+            fprintf(stderr, "[ARM]%s: " fmt , __func__, ##args); \
+        } \
+    } while (0)
+
+CPUState *arm_get_cpu_by_id(uint64_t id)
+{
+    CPUState *cpu;
+
+    DPRINTF("cpu %" PRId64 "\n", id);
+
+    CPU_FOREACH(cpu) {
+        ARMCPU *armcpu = ARM_CPU(cpu);
+
+        if (armcpu->mp_affinity == id) {
+            return cpu;
+        }
+    }
+
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "[ARM]%s: Requesting unknown CPU %" PRId64 "\n",
+                  __func__, id);
+
+    return NULL;
+}
+
+int arm_set_cpu_on(uint64_t cpuid, uint64_t entry, uint64_t context_id,
+                   uint32_t target_el, bool target_aa64)
+{
+    CPUState *target_cpu_state;
+    ARMCPU *target_cpu;
+
+    DPRINTF("cpu %" PRId64 " (EL %d, %s) @ 0x%" PRIx64 " with R0 = 0x%" PRIx64
+            "\n", cpuid, target_el, target_aa64 ? "aarch64" : "aarch32", entry,
+            context_id);
+
+    /* requested EL level need to be in the 1 to 3 range */
+    assert((target_el > 0) && (target_el < 4));
+
+    if (target_aa64 && (entry & 3)) {
+        /*
+         * if we are booting in AArch64 mode then "entry" needs to be 4 bytes
+         * aligned.
+         */
+        return QEMU_ARM_POWERCTL_INVALID_PARAM;
+    }
+
+    /* Retrieve the cpu we are powering up */
+    target_cpu_state = arm_get_cpu_by_id(cpuid);
+    if (!target_cpu_state) {
+        /* The cpu was not found */
+        return QEMU_ARM_POWERCTL_INVALID_PARAM;
+    }
+
+    target_cpu = ARM_CPU(target_cpu_state);
+    if (!target_cpu->powered_off) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "[ARM]%s: CPU %" PRId64 " is already on\n",
+                      __func__, cpuid);
+        return QEMU_ARM_POWERCTL_ALREADY_ON;
+    }
+
+    /*
+     * The newly brought CPU is requested to enter the exception level
+     * "target_el" and be in the requested mode (AArch64 or AArch32).
+     */
+
+    if (((target_el == 3) && !arm_feature(&target_cpu->env, ARM_FEATURE_EL3)) ||
+        ((target_el == 2) && !arm_feature(&target_cpu->env, ARM_FEATURE_EL2))) {
+        /*
+         * The CPU does not support requested level
+         */
+        return QEMU_ARM_POWERCTL_INVALID_PARAM;
+    }
+
+    if (!target_aa64 && arm_feature(&target_cpu->env, ARM_FEATURE_AARCH64)) {
+        /*
+         * For now we don't support booting an AArch64 CPU in AArch32 mode
+         * TODO: We should add this support later
+         */
+        qemu_log_mask(LOG_UNIMP,
+                      "[ARM]%s: Starting AArch64 CPU %" PRId64
+                      " in AArch32 mode is not supported yet\n",
+                      __func__, cpuid);
+        return QEMU_ARM_POWERCTL_INVALID_PARAM;
+    }
+
+    /* Initialize the cpu we are turning on */
+    cpu_reset(target_cpu_state);
+    target_cpu->powered_off = false;
+    target_cpu_state->halted = 0;
+
+    if (target_aa64) {
+        if ((target_el < 3) && arm_feature(&target_cpu->env, ARM_FEATURE_EL3)) {
+            /*
+             * As target mode is AArch64, we need to set lower
+             * exception level (the requested level 2) to AArch64
+             */
+            target_cpu->env.cp15.scr_el3 |= SCR_RW;
+        }
+
+        if ((target_el < 2) && arm_feature(&target_cpu->env, ARM_FEATURE_EL2)) {
+            /*
+             * As target mode is AArch64, we need to set lower
+             * exception level (the requested level 1) to AArch64
+             */
+            target_cpu->env.cp15.hcr_el2 |= HCR_RW;
+        }
+
+        target_cpu->env.pstate = aarch64_pstate_mode(target_el, true);
+    } else {
+        /* We are requested to boot in AArch32 mode */
+        static uint32_t mode_for_el[] = { 0,
+                                          ARM_CPU_MODE_SVC,
+                                          ARM_CPU_MODE_HYP,
+                                          ARM_CPU_MODE_SVC };
+
+        cpsr_write(&target_cpu->env, mode_for_el[target_el], CPSR_M,
+                   CPSRWriteRaw);
+    }
+
+    if (target_el == 3) {
+        /* Processor is in secure mode */
+        target_cpu->env.cp15.scr_el3 &= ~SCR_NS;
+    } else {
+        /* Processor is not in secure mode */
+        target_cpu->env.cp15.scr_el3 |= SCR_NS;
+    }
+
+    /* We check if the started CPU is now at the correct level */
+    assert(target_el == arm_current_el(&target_cpu->env));
+
+    if (target_aa64) {
+        target_cpu->env.xregs[0] = context_id;
+        target_cpu->env.thumb = false;
+    } else {
+        target_cpu->env.regs[0] = context_id;
+        target_cpu->env.thumb = entry & 1;
+        entry &= 0xfffffffe;
+    }
+
+    /* Start the new CPU at the requested address */
+    cpu_set_pc(target_cpu_state, entry);
+
+    /* We are good to go */
+    return QEMU_ARM_POWERCTL_RET_SUCCESS;
+}
+
+int arm_set_cpu_off(uint64_t cpuid)
+{
+    CPUState *target_cpu_state;
+    ARMCPU *target_cpu;
+
+    DPRINTF("cpu %" PRId64 "\n", cpuid);
+
+    /* change to the cpu we are powering up */
+    target_cpu_state = arm_get_cpu_by_id(cpuid);
+    if (!target_cpu_state) {
+        return QEMU_ARM_POWERCTL_INVALID_PARAM;
+    }
+    target_cpu = ARM_CPU(target_cpu_state);
+    if (target_cpu->powered_off) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "[ARM]%s: CPU %" PRId64 " is already off\n",
+                      __func__, cpuid);
+        return QEMU_ARM_POWERCTL_IS_OFF;
+    }
+
+    target_cpu->powered_off = true;
+    target_cpu_state->halted = 1;
+    target_cpu_state->exception_index = EXCP_HLT;
+    cpu_loop_exit(target_cpu_state);
+    /* notreached */
+
+    return QEMU_ARM_POWERCTL_RET_SUCCESS;
+}
+
+int arm_reset_cpu(uint64_t cpuid)
+{
+    CPUState *target_cpu_state;
+    ARMCPU *target_cpu;
+
+    DPRINTF("cpu %" PRId64 "\n", cpuid);
+
+    /* change to the cpu we are resetting */
+    target_cpu_state = arm_get_cpu_by_id(cpuid);
+    if (!target_cpu_state) {
+        return QEMU_ARM_POWERCTL_INVALID_PARAM;
+    }
+    target_cpu = ARM_CPU(target_cpu_state);
+    if (target_cpu->powered_off) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "[ARM]%s: CPU %" PRId64 " is off\n",
+                      __func__, cpuid);
+        return QEMU_ARM_POWERCTL_IS_OFF;
+    }
+
+    /* Reset the cpu */
+    cpu_reset(target_cpu_state);
+
+    return QEMU_ARM_POWERCTL_RET_SUCCESS;
+}
diff --git a/target-arm/arm-powerctl.h b/target-arm/arm-powerctl.h
new file mode 100644
index 0000000..98ee049
--- /dev/null
+++ b/target-arm/arm-powerctl.h
@@ -0,0 +1,75 @@
+/*
+ * QEMU support -- ARM Power Control specific functions.
+ *
+ * Copyright (c) 2016 Jean-Christophe Dubois
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef QEMU_ARM_POWERCTL_H
+#define QEMU_ARM_POWERCTL_H
+
+#include "kvm-consts.h"
+
+#define QEMU_ARM_POWERCTL_RET_SUCCESS QEMU_PSCI_RET_SUCCESS
+#define QEMU_ARM_POWERCTL_INVALID_PARAM QEMU_PSCI_RET_INVALID_PARAMS
+#define QEMU_ARM_POWERCTL_ALREADY_ON QEMU_PSCI_RET_ALREADY_ON
+#define QEMU_ARM_POWERCTL_IS_OFF QEMU_PSCI_RET_DENIED
+
+/*
+ * arm_get_cpu_by_id:
+ * @cpuid: the id of the CPU we want to retrieve the state
+ *
+ * Retrieve a CPUState object from its CPU ID provided in @cpuid.
+ *
+ * Returns: a pointer to the CPUState structure of the requested CPU.
+ */
+CPUState *arm_get_cpu_by_id(uint64_t cpuid);
+
+/*
+ * arm_set_cpu_on:
+ * @cpuid: the id of the CPU we want to start/wake up.
+ * @entry: the address the CPU shall start from.
+ * @context_id: the value to put in r0/x0.
+ * @target_el: The desired exception level.
+ * @target_aa64: 1 if the requested mode is AArch64. 0 otherwise.
+ *
+ * Start the cpu designated by @cpuid in @target_el exception level. The mode
+ * shall be AArch64 if @target_aa64 is set to 1. Otherwise the mode is
+ * AArch32. The CPU shall start at @entry with @context_id in r0/x0.
+ *
+ * Returns: QEMU_ARM_POWERCTL_RET_SUCCESS on success.
+ * QEMU_ARM_POWERCTL_INVALID_PARAM if bad parameters are provided.
+ * QEMU_ARM_POWERCTL_ALREADY_ON if the CPU was already started.
+ */
+int arm_set_cpu_on(uint64_t cpuid, uint64_t entry, uint64_t context_id,
+                   uint32_t target_el, bool target_aa64);
+
+/*
+ * arm_set_cpu_off:
+ * @cpuid: the id of the CPU we want to stop/shut down.
+ *
+ * Stop the cpu designated by @cpuid.
+ *
+ * Returns: QEMU_ARM_POWERCTL_RET_SUCCESS on success.
+ * QEMU_ARM_POWERCTL_INVALID_PARAM if bad parameters are provided.
+ * QEMU_ARM_POWERCTL_IS_OFF if CPU is already off
+ */
+
+int arm_set_cpu_off(uint64_t cpuid);
+
+/*
+ * arm_reset_cpu:
+ * @cpuid: the id of the CPU we want to reset.
+ *
+ * Reset the cpu designated by @cpuid.
+ *
+ * Returns: QEMU_ARM_POWERCTL_RET_SUCCESS on success.
+ * QEMU_ARM_POWERCTL_INVALID_PARAM if bad parameters are provided.
+ * QEMU_ARM_POWERCTL_IS_OFF if CPU is off
+ */
+int arm_reset_cpu(uint64_t cpuid);
+
+#endif
diff --git a/target-arm/arm-semi.c b/target-arm/arm-semi.c
index ebb5235..7cac873 100644
--- a/target-arm/arm-semi.c
+++ b/target-arm/arm-semi.c
@@ -18,15 +18,10 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
+#include "exec/semihost.h"
 #ifdef CONFIG_USER_ONLY
 #include "qemu.h"
 
@@ -35,6 +30,7 @@
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 #include "hw/arm/arm.h"
+#include "qemu/cutils.h"
 #endif
 
 #define TARGET_SYS_OPEN        0x01
@@ -57,6 +53,11 @@
 #define TARGET_SYS_GET_CMDLINE 0x15
 #define TARGET_SYS_HEAPINFO    0x16
 #define TARGET_SYS_EXIT        0x18
+#define TARGET_SYS_SYNCCACHE   0x19
+
+/* ADP_Stopped_ApplicationExit is used for exit(0),
+ * anything else is implemented as exit(1) */
+#define ADP_Stopped_ApplicationExit     (0x20026)
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -129,6 +130,7 @@
 #ifdef CONFIG_USER_ONLY
     TaskState *ts = cs->opaque;
 #endif
+    target_ulong reg0 = is_a64(env) ? env->xregs[0] : env->regs[0];
 
     if (ret == (target_ulong)-1) {
 #ifdef CONFIG_USER_ONLY
@@ -136,22 +138,46 @@
 #else
 	syscall_err = err;
 #endif
-        env->regs[0] = ret;
+        reg0 = ret;
     } else {
         /* Fixup syscalls that use nonstardard return conventions.  */
-        switch (env->regs[0]) {
+        switch (reg0) {
         case TARGET_SYS_WRITE:
         case TARGET_SYS_READ:
-            env->regs[0] = arm_semi_syscall_len - ret;
+            reg0 = arm_semi_syscall_len - ret;
             break;
         case TARGET_SYS_SEEK:
-            env->regs[0] = 0;
+            reg0 = 0;
             break;
         default:
-            env->regs[0] = ret;
+            reg0 = ret;
             break;
         }
     }
+    if (is_a64(env)) {
+        env->xregs[0] = reg0;
+    } else {
+        env->regs[0] = reg0;
+    }
+}
+
+static target_ulong arm_flen_buf(ARMCPU *cpu)
+{
+    /* Return an address in target memory of 64 bytes where the remote
+     * gdb should write its stat struct. (The format of this structure
+     * is defined by GDB's remote protocol and is not target-specific.)
+     * We put this on the guest's stack just below SP.
+     */
+    CPUARMState *env = &cpu->env;
+    target_ulong sp;
+
+    if (is_a64(env)) {
+        sp = env->xregs[31];
+    } else {
+        sp = env->regs[13];
+    }
+
+    return sp - 64;
 }
 
 static void arm_semi_flen_cb(CPUState *cs, target_ulong ret, target_ulong err)
@@ -161,8 +187,13 @@
     /* The size is always stored in big-endian order, extract
        the value. We assume the size always fit in 32 bits.  */
     uint32_t size;
-    cpu_memory_rw_debug(cs, env->regs[13]-64+32, (uint8_t *)&size, 4, 0);
-    env->regs[0] = be32_to_cpu(size);
+    cpu_memory_rw_debug(cs, arm_flen_buf(cpu) + 32, (uint8_t *)&size, 4, 0);
+    size = be32_to_cpu(size);
+    if (is_a64(env)) {
+        env->xregs[0] = size;
+    } else {
+        env->regs[0] = size;
+    }
 #ifdef CONFIG_USER_ONLY
     ((TaskState *)cs->opaque)->swi_errno = err;
 #else
@@ -170,17 +201,46 @@
 #endif
 }
 
+static target_ulong arm_gdb_syscall(ARMCPU *cpu, gdb_syscall_complete_cb cb,
+                                    const char *fmt, ...)
+{
+    va_list va;
+    CPUARMState *env = &cpu->env;
+
+    va_start(va, fmt);
+    gdb_do_syscallv(cb, fmt, va);
+    va_end(va);
+
+    /* FIXME: we are implicitly relying on the syscall completing
+     * before this point, which is not guaranteed. We should
+     * put in an explicit synchronization between this and
+     * the callback function.
+     */
+
+    return is_a64(env) ? env->xregs[0] : env->regs[0];
+}
+
 /* Read the input value from the argument block; fail the semihosting
  * call if the memory read fails.
  */
 #define GET_ARG(n) do {                                 \
-    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
-        return (uint32_t)-1;                            \
+    if (is_a64(env)) {                                  \
+        if (get_user_u64(arg ## n, args + (n) * 8)) {   \
+            return -1;                                  \
+        }                                               \
+    } else {                                            \
+        if (get_user_u32(arg ## n, args + (n) * 4)) {   \
+            return -1;                                  \
+        }                                               \
     }                                                   \
 } while (0)
 
-#define SET_ARG(n, val) put_user_ual(val, args + (n) * 4)
-uint32_t do_arm_semihosting(CPUARMState *env)
+#define SET_ARG(n, val)                                 \
+    (is_a64(env) ?                                      \
+     put_user_u64(val, args + (n) * 8) :                \
+     put_user_u32(val, args + (n) * 4))
+
+target_ulong do_arm_semihosting(CPUARMState *env)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
@@ -196,8 +256,15 @@
     CPUARMState *ts = env;
 #endif
 
-    nr = env->regs[0];
-    args = env->regs[1];
+    if (is_a64(env)) {
+        /* Note that the syscall number is in W0, not X0 */
+        nr = env->xregs[0] & 0xffffffffU;
+        args = env->xregs[1];
+    } else {
+        nr = env->regs[0];
+        args = env->regs[1];
+    }
+
     switch (nr) {
     case TARGET_SYS_OPEN:
         GET_ARG(0);
@@ -218,9 +285,8 @@
             return result_fileno;
         }
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", arg0,
-                           (int)arg2+1, gdb_open_modeflags[arg1]);
-            ret = env->regs[0];
+            ret = arm_gdb_syscall(cpu, arm_semi_cb, "open,%s,%x,1a4", arg0,
+                                  (int)arg2+1, gdb_open_modeflags[arg1]);
         } else {
             ret = set_swi_errno(ts, open(s, open_modeflags[arg1], 0644));
         }
@@ -229,8 +295,7 @@
     case TARGET_SYS_CLOSE:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "close,%x", arg0);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "close,%x", arg0);
         } else {
             return set_swi_errno(ts, close(arg0));
         }
@@ -243,8 +308,7 @@
               return (uint32_t)-1;
           /* Write to debug console.  stderr is near enough.  */
           if (use_gdb_syscalls()) {
-                gdb_do_syscall(arm_semi_cb, "write,2,%x,1", args);
-                return env->regs[0];
+                return arm_gdb_syscall(cpu, arm_semi_cb, "write,2,%x,1", args);
           } else {
                 return write(STDERR_FILENO, &c, 1);
           }
@@ -255,8 +319,8 @@
             return (uint32_t)-1;
         len = strlen(s);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "write,2,%x,%x\n", args, len);
-            ret = env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "write,2,%x,%x",
+                                   args, len);
         } else {
             ret = write(STDERR_FILENO, s, len);
         }
@@ -269,8 +333,8 @@
         len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", arg0, arg1, len);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "write,%x,%x,%x",
+                                   arg0, arg1, len);
         } else {
             s = lock_user(VERIFY_READ, arg1, len, 1);
             if (!s) {
@@ -290,8 +354,8 @@
         len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", arg0, arg1, len);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "read,%x,%x,%x",
+                                   arg0, arg1, len);
         } else {
             s = lock_user(VERIFY_WRITE, arg1, len, 0);
             if (!s) {
@@ -312,8 +376,7 @@
     case TARGET_SYS_ISTTY:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "isatty,%x", arg0);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "isatty,%x", arg0);
         } else {
             return isatty(arg0);
         }
@@ -321,8 +384,8 @@
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", arg0, arg1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "lseek,%x,%x,0",
+                                   arg0, arg1);
         } else {
             ret = set_swi_errno(ts, lseek(arg0, arg1, SEEK_SET));
             if (ret == (uint32_t)-1)
@@ -332,9 +395,8 @@
     case TARGET_SYS_FLEN:
         GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_flen_cb, "fstat,%x,%x",
-                           arg0, env->regs[13]-64);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_flen_cb, "fstat,%x,%x",
+                                   arg0, arm_flen_buf(cpu));
         } else {
             struct stat buf;
             ret = set_swi_errno(ts, fstat(arg0, &buf));
@@ -349,8 +411,8 @@
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "unlink,%s", arg0, (int)arg1+1);
-            ret = env->regs[0];
+            ret = arm_gdb_syscall(cpu, arm_semi_cb, "unlink,%s",
+                                  arg0, (int)arg1+1);
         } else {
             s = lock_user_string(arg0);
             if (!s) {
@@ -367,9 +429,8 @@
         GET_ARG(2);
         GET_ARG(3);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "rename,%s,%s",
-                           arg0, (int)arg1+1, arg2, (int)arg3+1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "rename,%s,%s",
+                                   arg0, (int)arg1+1, arg2, (int)arg3+1);
         } else {
             char *s2;
             s = lock_user_string(arg0);
@@ -393,8 +454,8 @@
         GET_ARG(0);
         GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "system,%s", arg0, (int)arg1+1);
-            return env->regs[0];
+            return arm_gdb_syscall(cpu, arm_semi_cb, "system,%s",
+                                   arg0, (int)arg1+1);
         } else {
             s = lock_user_string(arg0);
             if (!s) {
@@ -431,15 +492,19 @@
             size_t input_size;
             size_t output_size;
             int status = 0;
+#if !defined(CONFIG_USER_ONLY)
+            const char *cmdline;
+#endif
             GET_ARG(0);
             GET_ARG(1);
             input_size = arg1;
             /* Compute the size of the output string.  */
 #if !defined(CONFIG_USER_ONLY)
-            output_size = strlen(ts->boot_info->kernel_filename)
-                        + 1  /* Separating space.  */
-                        + strlen(ts->boot_info->kernel_cmdline)
-                        + 1; /* Terminating null byte.  */
+            cmdline = semihosting_get_cmdline();
+            if (cmdline == NULL) {
+                cmdline = ""; /* Default to an empty line. */
+            }
+            output_size = strlen(cmdline) + 1; /* Count terminating 0. */
 #else
             unsigned int i;
 
@@ -470,9 +535,7 @@
 
             /* Copy the command-line arguments.  */
 #if !defined(CONFIG_USER_ONLY)
-            pstrcpy(output_buffer, output_size, ts->boot_info->kernel_filename);
-            pstrcat(output_buffer, output_size, " ");
-            pstrcat(output_buffer, output_size, ts->boot_info->kernel_cmdline);
+            pstrcpy(output_buffer, output_size, cmdline);
 #else
             if (output_size == 1) {
                 /* Empty command-line.  */
@@ -501,8 +564,10 @@
         }
     case TARGET_SYS_HEAPINFO:
         {
-            uint32_t *ptr;
-            uint32_t limit;
+            target_ulong retvals[4];
+            target_ulong limit;
+            int i;
+
             GET_ARG(0);
 
 #ifdef CONFIG_USER_ONLY
@@ -524,35 +589,65 @@
                 ts->heap_limit = limit;
             }
 
-            ptr = lock_user(VERIFY_WRITE, arg0, 16, 0);
-            if (!ptr) {
-                /* FIXME - should this error code be -TARGET_EFAULT ? */
-                return (uint32_t)-1;
-            }
-            ptr[0] = tswap32(ts->heap_base);
-            ptr[1] = tswap32(ts->heap_limit);
-            ptr[2] = tswap32(ts->stack_base);
-            ptr[3] = tswap32(0); /* Stack limit.  */
-            unlock_user(ptr, arg0, 16);
+            retvals[0] = ts->heap_base;
+            retvals[1] = ts->heap_limit;
+            retvals[2] = ts->stack_base;
+            retvals[3] = 0; /* Stack limit.  */
 #else
             limit = ram_size;
-            ptr = lock_user(VERIFY_WRITE, arg0, 16, 0);
-            if (!ptr) {
-                /* FIXME - should this error code be -TARGET_EFAULT ? */
-                return (uint32_t)-1;
-            }
             /* TODO: Make this use the limit of the loaded application.  */
-            ptr[0] = tswap32(limit / 2);
-            ptr[1] = tswap32(limit);
-            ptr[2] = tswap32(limit); /* Stack base */
-            ptr[3] = tswap32(0); /* Stack limit.  */
-            unlock_user(ptr, arg0, 16);
+            retvals[0] = limit / 2;
+            retvals[1] = limit;
+            retvals[2] = limit; /* Stack base */
+            retvals[3] = 0; /* Stack limit.  */
 #endif
+
+            for (i = 0; i < ARRAY_SIZE(retvals); i++) {
+                bool fail;
+
+                if (is_a64(env)) {
+                    fail = put_user_u64(retvals[i], arg0 + i * 8);
+                } else {
+                    fail = put_user_u32(retvals[i], arg0 + i * 4);
+                }
+
+                if (fail) {
+                    /* Couldn't write back to argument block */
+                    return -1;
+                }
+            }
             return 0;
         }
     case TARGET_SYS_EXIT:
-        gdb_exit(env, 0);
-        exit(0);
+        if (is_a64(env)) {
+            /* The A64 version of this call takes a parameter block,
+             * so the application-exit type can return a subcode which
+             * is the exit status code from the application.
+             */
+            GET_ARG(0);
+            GET_ARG(1);
+
+            if (arg0 == ADP_Stopped_ApplicationExit) {
+                ret = arg1;
+            } else {
+                ret = 1;
+            }
+        } else {
+            /* ARM specifies only Stopped_ApplicationExit as normal
+             * exit, everything else is considered an error */
+            ret = (args == ADP_Stopped_ApplicationExit) ? 0 : 1;
+        }
+        gdb_exit(env, ret);
+        exit(ret);
+    case TARGET_SYS_SYNCCACHE:
+        /* Clean the D-cache and invalidate the I-cache for the specified
+         * virtual address range. This is a nop for us since we don't
+         * implement caches. This is only present on A64.
+         */
+        if (is_a64(env)) {
+            return 0;
+        }
+        /* fall through -- invalid for A32/T32 */
     default:
         fprintf(stderr, "qemu: Unsupported SemiHosting SWI 0x%02x\n", nr);
         cpu_dump_state(cs, stderr, fprintf, 0);
diff --git a/target-arm/arm_ldst.h b/target-arm/arm_ldst.h
index b1ece01..a76d89f 100644
--- a/target-arm/arm_ldst.h
+++ b/target-arm/arm_ldst.h
@@ -20,15 +20,16 @@
 #ifndef ARM_LDST_H
 #define ARM_LDST_H
 
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "qemu/bswap.h"
 
 /* Load an instruction and return it in the standard little-endian order */
 static inline uint32_t arm_ldl_code(CPUARMState *env, target_ulong addr,
-                                    bool do_swap)
+                                    bool sctlr_b)
 {
     uint32_t insn = cpu_ldl_code(env, addr);
-    if (do_swap) {
+    if (bswap_code(sctlr_b)) {
         return bswap32(insn);
     }
     return insn;
@@ -36,10 +37,10 @@
 
 /* Ditto, for a halfword (Thumb) instruction */
 static inline uint16_t arm_lduw_code(CPUARMState *env, target_ulong addr,
-                                     bool do_swap)
+                                     bool sctlr_b)
 {
     uint16_t insn = cpu_lduw_code(env, addr);
-    if (do_swap) {
+    if (bswap_code(sctlr_b)) {
         return bswap16(insn);
     }
     return insn;
diff --git a/target-arm/cpu-qom.h b/target-arm/cpu-qom.h
index dcfda7d..3991173 100644
--- a/target-arm/cpu-qom.h
+++ b/target-arm/cpu-qom.h
@@ -22,6 +22,8 @@
 
 #include "qom/cpu.h"
 
+struct arm_boot_info;
+
 #define TYPE_ARM_CPU "arm-cpu"
 
 #define ARM_CPU_CLASS(klass) \
@@ -47,127 +49,7 @@
     void (*parent_reset)(CPUState *cpu);
 } ARMCPUClass;
 
-/**
- * ARMCPU:
- * @env: #CPUARMState
- *
- * An ARM CPU core.
- */
-typedef struct ARMCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUARMState env;
-
-    /* Coprocessor information */
-    GHashTable *cp_regs;
-    /* For marshalling (mostly coprocessor) register state between the
-     * kernel and QEMU (for KVM) and between two QEMUs (for migration),
-     * we use these arrays.
-     */
-    /* List of register indexes managed via these arrays; (full KVM style
-     * 64 bit indexes, not CPRegInfo 32 bit indexes)
-     */
-    uint64_t *cpreg_indexes;
-    /* Values of the registers (cpreg_indexes[i]'s value is cpreg_values[i]) */
-    uint64_t *cpreg_values;
-    /* Length of the indexes, values, reset_values arrays */
-    int32_t cpreg_array_len;
-    /* These are used only for migration: incoming data arrives in
-     * these fields and is sanity checked in post_load before copying
-     * to the working data structures above.
-     */
-    uint64_t *cpreg_vmstate_indexes;
-    uint64_t *cpreg_vmstate_values;
-    int32_t cpreg_vmstate_array_len;
-
-    /* Timers used by the generic (architected) timer */
-    QEMUTimer *gt_timer[NUM_GTIMERS];
-    /* GPIO outputs for generic timer */
-    qemu_irq gt_timer_outputs[NUM_GTIMERS];
-
-    /* 'compatible' string for this CPU for Linux device trees */
-    const char *dtb_compatible;
-
-    /* PSCI version for this CPU
-     * Bits[31:16] = Major Version
-     * Bits[15:0] = Minor Version
-     */
-    uint32_t psci_version;
-
-    /* Should CPU start in PSCI powered-off state? */
-    bool start_powered_off;
-    /* CPU currently in PSCI powered-off state */
-    bool powered_off;
-
-    /* PSCI conduit used to invoke PSCI methods
-     * 0 - disabled, 1 - smc, 2 - hvc
-     */
-    uint32_t psci_conduit;
-
-    /* [QEMU_]KVM_ARM_TARGET_* constant for this CPU, or
-     * QEMU_KVM_ARM_TARGET_NONE if the kernel doesn't support this CPU type.
-     */
-    uint32_t kvm_target;
-
-    /* KVM init features for this CPU */
-    uint32_t kvm_init_features[7];
-
-    /* The instance init functions for implementation-specific subclasses
-     * set these fields to specify the implementation-dependent values of
-     * various constant registers and reset values of non-constant
-     * registers.
-     * Some of these might become QOM properties eventually.
-     * Field names match the official register names as defined in the
-     * ARMv7AR ARM Architecture Reference Manual. A reset_ prefix
-     * is used for reset values of non-constant registers; no reset_
-     * prefix means a constant register.
-     */
-    uint32_t midr;
-    uint32_t reset_fpsid;
-    uint32_t mvfr0;
-    uint32_t mvfr1;
-    uint32_t mvfr2;
-    uint32_t ctr;
-    uint32_t reset_sctlr;
-    uint32_t id_pfr0;
-    uint32_t id_pfr1;
-    uint32_t id_dfr0;
-    uint32_t id_afr0;
-    uint32_t id_mmfr0;
-    uint32_t id_mmfr1;
-    uint32_t id_mmfr2;
-    uint32_t id_mmfr3;
-    uint32_t id_isar0;
-    uint32_t id_isar1;
-    uint32_t id_isar2;
-    uint32_t id_isar3;
-    uint32_t id_isar4;
-    uint32_t id_isar5;
-    uint64_t id_aa64pfr0;
-    uint64_t id_aa64pfr1;
-    uint64_t id_aa64dfr0;
-    uint64_t id_aa64dfr1;
-    uint64_t id_aa64afr0;
-    uint64_t id_aa64afr1;
-    uint64_t id_aa64isar0;
-    uint64_t id_aa64isar1;
-    uint64_t id_aa64mmfr0;
-    uint64_t id_aa64mmfr1;
-    uint32_t dbgdidr;
-    uint32_t clidr;
-    /* The elements of this array are the CCSIDR values for each cache,
-     * in the order L1DCache, L1ICache, L2DCache, L2ICache, etc.
-     */
-    uint32_t ccsidr[16];
-    uint64_t reset_cbar;
-    uint32_t reset_auxcr;
-    bool reset_hivecs;
-    /* DCZ blocksize, in log_2(words), ie low 4 bits of DCZID_EL0 */
-    uint32_t dcz_blocksize;
-    uint64_t rvbar;
-} ARMCPU;
+typedef struct ARMCPU ARMCPU;
 
 #define TYPE_AARCH64_CPU "aarch64-cpu"
 #define AARCH64_CPU_CLASS(klass) \
@@ -181,43 +63,26 @@
     /*< public >*/
 } AArch64CPUClass;
 
-static inline ARMCPU *arm_env_get_cpu(CPUARMState *env)
-{
-    return container_of(env, ARMCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(arm_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(ARMCPU, env)
-
-#ifndef CONFIG_USER_ONLY
-extern const struct VMStateDescription vmstate_arm_cpu;
-#endif
-
 void register_cp_regs_for_features(ARMCPU *cpu);
 void init_cpreg_list(ARMCPU *cpu);
 
-void arm_cpu_do_interrupt(CPUState *cpu);
-void arm_v7m_cpu_do_interrupt(CPUState *cpu);
-bool arm_cpu_exec_interrupt(CPUState *cpu, int int_req);
-
-void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
-                        int flags);
-
-hwaddr arm_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-
-int arm_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int arm_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-
 /* Callback functions for the generic timer's timers. */
 void arm_gt_ptimer_cb(void *opaque);
 void arm_gt_vtimer_cb(void *opaque);
+void arm_gt_htimer_cb(void *opaque);
+void arm_gt_stimer_cb(void *opaque);
 
-#ifdef TARGET_AARCH64
-int aarch64_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int aarch64_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+#define ARM_AFF0_SHIFT 0
+#define ARM_AFF0_MASK  (0xFFULL << ARM_AFF0_SHIFT)
+#define ARM_AFF1_SHIFT 8
+#define ARM_AFF1_MASK  (0xFFULL << ARM_AFF1_SHIFT)
+#define ARM_AFF2_SHIFT 16
+#define ARM_AFF2_MASK  (0xFFULL << ARM_AFF2_SHIFT)
+#define ARM_AFF3_SHIFT 32
+#define ARM_AFF3_MASK  (0xFFULL << ARM_AFF3_SHIFT)
 
-void aarch64_cpu_do_interrupt(CPUState *cs);
-#endif
+#define ARM32_AFFINITY_MASK (ARM_AFF0_MASK|ARM_AFF1_MASK|ARM_AFF2_MASK)
+#define ARM64_AFFINITY_MASK \
+    (ARM_AFF0_MASK|ARM_AFF1_MASK|ARM_AFF2_MASK|ARM_AFF3_MASK)
 
 #endif
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index e88737b..ce8b8f4 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -18,11 +18,13 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "internals.h"
 #include "qemu-common.h"
+#include "exec/exec-all.h"
 #include "hw/qdev-properties.h"
-#include "qapi/qmp/qerror.h"
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/loader.h"
 #endif
@@ -49,13 +51,22 @@
          | CPU_INTERRUPT_EXITTB);
 }
 
+void arm_register_el_change_hook(ARMCPU *cpu, ARMELChangeHook *hook,
+                                 void *opaque)
+{
+    /* We currently only support registering a single hook function */
+    assert(!cpu->el_change_hook);
+    cpu->el_change_hook = hook;
+    cpu->el_change_hook_opaque = opaque;
+}
+
 static void cp_reg_reset(gpointer key, gpointer value, gpointer opaque)
 {
     /* Reset a single ARMCPRegInfo register */
     ARMCPRegInfo *ri = value;
     ARMCPU *cpu = opaque;
 
-    if (ri->type & ARM_CP_SPECIAL) {
+    if (ri->type & (ARM_CP_SPECIAL | ARM_CP_ALIAS)) {
         return;
     }
 
@@ -80,6 +91,27 @@
     }
 }
 
+static void cp_reg_check_reset(gpointer key, gpointer value,  gpointer opaque)
+{
+    /* Purely an assertion check: we've already done reset once,
+     * so now check that running the reset for the cpreg doesn't
+     * change its value. This traps bugs where two different cpregs
+     * both try to reset the same state field but to different values.
+     */
+    ARMCPRegInfo *ri = value;
+    ARMCPU *cpu = opaque;
+    uint64_t oldvalue, newvalue;
+
+    if (ri->type & (ARM_CP_SPECIAL | ARM_CP_ALIAS | ARM_CP_NO_RAW)) {
+        return;
+    }
+
+    oldvalue = read_raw_cp_reg(&cpu->env, ri);
+    cp_reg_reset(key, value, opaque);
+    newvalue = read_raw_cp_reg(&cpu->env, ri);
+    assert(oldvalue == newvalue);
+}
+
 /* CPUClass::reset() */
 static void arm_cpu_reset(CPUState *s)
 {
@@ -91,6 +123,8 @@
 
     memset(env, 0, offsetof(CPUARMState, features));
     g_hash_table_foreach(cpu->cp_regs, cp_reg_reset, cpu);
+    g_hash_table_foreach(cpu->cp_regs, cp_reg_check_reset, cpu);
+
     env->vfp.xregs[ARM_VFP_FPSID] = cpu->reset_fpsid;
     env->vfp.xregs[ARM_VFP_MVFR0] = cpu->mvfr0;
     env->vfp.xregs[ARM_VFP_MVFR1] = cpu->mvfr1;
@@ -109,17 +143,24 @@
 #if defined(CONFIG_USER_ONLY)
         env->pstate = PSTATE_MODE_EL0t;
         /* Userspace expects access to DC ZVA, CTL_EL0 and the cache ops */
-        env->cp15.c1_sys |= SCTLR_UCT | SCTLR_UCI | SCTLR_DZE;
+        env->cp15.sctlr_el[1] |= SCTLR_UCT | SCTLR_UCI | SCTLR_DZE;
         /* and to the FP/Neon instructions */
-        env->cp15.c1_coproc = deposit64(env->cp15.c1_coproc, 20, 2, 3);
+        env->cp15.cpacr_el1 = deposit64(env->cp15.cpacr_el1, 20, 2, 3);
 #else
-        env->pstate = PSTATE_MODE_EL1h;
+        /* Reset into the highest available EL */
+        if (arm_feature(env, ARM_FEATURE_EL3)) {
+            env->pstate = PSTATE_MODE_EL3h;
+        } else if (arm_feature(env, ARM_FEATURE_EL2)) {
+            env->pstate = PSTATE_MODE_EL2h;
+        } else {
+            env->pstate = PSTATE_MODE_EL1h;
+        }
         env->pc = cpu->rvbar;
 #endif
     } else {
 #if defined(CONFIG_USER_ONLY)
         /* Userspace expects access to cp10 and cp11 for FP/Neon */
-        env->cp15.c1_coproc = deposit64(env->cp15.c1_coproc, 20, 4, 0xf);
+        env->cp15.cpacr_el1 = deposit64(env->cp15.cpacr_el1, 20, 4, 0xf);
 #endif
     }
 
@@ -167,7 +208,11 @@
         env->thumb = initial_pc & 1;
     }
 
-    if (env->cp15.c1_sys & SCTLR_V) {
+    /* AArch32 has a hard highvec setting of 0xFFFF0000.  If we are currently
+     * executing as AArch32 then check if highvecs are enabled and
+     * adjust the PC accordingly.
+     */
+    if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
         env->regs[15] = 0xFFFF0000;
     }
 
@@ -195,31 +240,52 @@
 bool arm_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
 {
     CPUClass *cc = CPU_GET_CLASS(cs);
+    CPUARMState *env = cs->env_ptr;
+    uint32_t cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+    uint32_t target_el;
+    uint32_t excp_idx;
     bool ret = false;
 
-    if (interrupt_request & CPU_INTERRUPT_FIQ
-        && arm_excp_unmasked(cs, EXCP_FIQ)) {
-        cs->exception_index = EXCP_FIQ;
-        cc->do_interrupt(cs);
-        ret = true;
+    if (interrupt_request & CPU_INTERRUPT_FIQ) {
+        excp_idx = EXCP_FIQ;
+        target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure);
+        if (arm_excp_unmasked(cs, excp_idx, target_el)) {
+            cs->exception_index = excp_idx;
+            env->exception.target_el = target_el;
+            cc->do_interrupt(cs);
+            ret = true;
+        }
     }
-    if (interrupt_request & CPU_INTERRUPT_HARD
-        && arm_excp_unmasked(cs, EXCP_IRQ)) {
-        cs->exception_index = EXCP_IRQ;
-        cc->do_interrupt(cs);
-        ret = true;
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        excp_idx = EXCP_IRQ;
+        target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure);
+        if (arm_excp_unmasked(cs, excp_idx, target_el)) {
+            cs->exception_index = excp_idx;
+            env->exception.target_el = target_el;
+            cc->do_interrupt(cs);
+            ret = true;
+        }
     }
-    if (interrupt_request & CPU_INTERRUPT_VIRQ
-        && arm_excp_unmasked(cs, EXCP_VIRQ)) {
-        cs->exception_index = EXCP_VIRQ;
-        cc->do_interrupt(cs);
-        ret = true;
+    if (interrupt_request & CPU_INTERRUPT_VIRQ) {
+        excp_idx = EXCP_VIRQ;
+        target_el = 1;
+        if (arm_excp_unmasked(cs, excp_idx, target_el)) {
+            cs->exception_index = excp_idx;
+            env->exception.target_el = target_el;
+            cc->do_interrupt(cs);
+            ret = true;
+        }
     }
-    if (interrupt_request & CPU_INTERRUPT_VFIQ
-        && arm_excp_unmasked(cs, EXCP_VFIQ)) {
-        cs->exception_index = EXCP_VFIQ;
-        cc->do_interrupt(cs);
-        ret = true;
+    if (interrupt_request & CPU_INTERRUPT_VFIQ) {
+        excp_idx = EXCP_VFIQ;
+        target_el = 1;
+        if (arm_excp_unmasked(cs, excp_idx, target_el)) {
+            cs->exception_index = excp_idx;
+            env->exception.target_el = target_el;
+            cc->do_interrupt(cs);
+            ret = true;
+        }
     }
 
     return ret;
@@ -277,10 +343,7 @@
     switch (irq) {
     case ARM_CPU_VIRQ:
     case ARM_CPU_VFIQ:
-        if (!arm_feature(env, ARM_FEATURE_EL2)) {
-            hw_error("%s: Virtual interrupt line %d with no EL2 support\n",
-                     __func__, irq);
-        }
+        assert(arm_feature(env, ARM_FEATURE_EL2));
         /* fall through */
     case ARM_CPU_IRQ:
     case ARM_CPU_FIQ:
@@ -291,7 +354,7 @@
         }
         break;
     default:
-        hw_error("arm_cpu_set_irq: Bad interrupt line %d\n", irq);
+        g_assert_not_reached();
     }
 }
 
@@ -310,12 +373,22 @@
         kvm_irq |= KVM_ARM_IRQ_CPU_FIQ;
         break;
     default:
-        hw_error("arm_cpu_kvm_set_irq: Bad interrupt line %d\n", irq);
+        g_assert_not_reached();
     }
     kvm_irq |= cs->cpu_index << KVM_ARM_IRQ_VCPU_SHIFT;
     kvm_set_irq(kvm_state, kvm_irq, level ? 1 : 0);
 #endif
 }
+
+static bool arm_cpu_virtio_is_big_endian(CPUState *cs)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    cpu_synchronize_state(cs);
+    return arm_cpu_data_is_big_endian(env);
+}
+
 #endif
 
 static inline void set_feature(CPUARMState *env, int feature)
@@ -323,17 +396,67 @@
     env->features |= 1ULL << feature;
 }
 
+static inline void unset_feature(CPUARMState *env, int feature)
+{
+    env->features &= ~(1ULL << feature);
+}
+
+static int
+print_insn_thumb1(bfd_vma pc, disassemble_info *info)
+{
+  return print_insn_arm(pc | 1, info);
+}
+
+static void arm_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    ARMCPU *ac = ARM_CPU(cpu);
+    CPUARMState *env = &ac->env;
+
+    if (is_a64(env)) {
+        /* We might not be compiled with the A64 disassembler
+         * because it needs a C++ compiler. Leave print_insn
+         * unset in this case to use the caller default behaviour.
+         */
+#if defined(CONFIG_ARM_A64_DIS)
+        info->print_insn = print_insn_arm_a64;
+#endif
+    } else if (env->thumb) {
+        info->print_insn = print_insn_thumb1;
+    } else {
+        info->print_insn = print_insn_arm;
+    }
+    if (bswap_code(arm_sctlr_b(env))) {
+#ifdef TARGET_WORDS_BIGENDIAN
+        info->endian = BFD_ENDIAN_LITTLE;
+#else
+        info->endian = BFD_ENDIAN_BIG;
+#endif
+    }
+}
+
+#define ARM_CPUS_PER_CLUSTER 8
+
 static void arm_cpu_initfn(Object *obj)
 {
     CPUState *cs = CPU(obj);
     ARMCPU *cpu = ARM_CPU(obj);
     static bool inited;
+    uint32_t Aff1, Aff0;
 
     cs->env_ptr = &cpu->env;
-    cpu_exec_init(&cpu->env);
+    cpu_exec_init(cs, &error_abort);
     cpu->cp_regs = g_hash_table_new_full(g_int_hash, g_int_equal,
                                          g_free, g_free);
 
+    /* This cpu-id-to-MPIDR affinity is used only for TCG; KVM will override it.
+     * We don't support setting cluster ID ([16..23]) (known as Aff2
+     * in later ARM ARM versions), or any of the higher affinity level fields,
+     * so these bits always RAZ.
+     */
+    Aff1 = cs->cpu_index / ARM_CPUS_PER_CLUSTER;
+    Aff0 = cs->cpu_index % ARM_CPUS_PER_CLUSTER;
+    cpu->mp_affinity = (Aff1 << ARM_AFF1_SHIFT) | Aff0;
+
 #ifndef CONFIG_USER_ONLY
     /* Our inbound IRQ and FIQ lines */
     if (kvm_enabled()) {
@@ -349,6 +472,10 @@
                                                 arm_gt_ptimer_cb, cpu);
     cpu->gt_timer[GTIMER_VIRT] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
                                                 arm_gt_vtimer_cb, cpu);
+    cpu->gt_timer[GTIMER_HYP] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
+                                                arm_gt_htimer_cb, cpu);
+    cpu->gt_timer[GTIMER_SEC] = timer_new(QEMU_CLOCK_VIRTUAL, GTIMER_SCALE,
+                                                arm_gt_stimer_cb, cpu);
     qdev_init_gpio_out(DEVICE(cpu), cpu->gt_timer_outputs,
                        ARRAY_SIZE(cpu->gt_timer_outputs));
 #endif
@@ -379,6 +506,15 @@
 static Property arm_cpu_rvbar_property =
             DEFINE_PROP_UINT64("rvbar", ARMCPU, rvbar, 0);
 
+static Property arm_cpu_has_el3_property =
+            DEFINE_PROP_BOOL("has_el3", ARMCPU, has_el3, true);
+
+static Property arm_cpu_has_mpu_property =
+            DEFINE_PROP_BOOL("has-mpu", ARMCPU, has_mpu, true);
+
+static Property arm_cpu_pmsav7_dregion_property =
+            DEFINE_PROP_UINT32("pmsav7-dregion", ARMCPU, pmsav7_dregion, 16);
+
 static void arm_cpu_post_init(Object *obj)
 {
     ARMCPU *cpu = ARM_CPU(obj);
@@ -398,6 +534,34 @@
         qdev_property_add_static(DEVICE(obj), &arm_cpu_rvbar_property,
                                  &error_abort);
     }
+
+    if (arm_feature(&cpu->env, ARM_FEATURE_EL3)) {
+        /* Add the has_el3 state CPU property only if EL3 is allowed.  This will
+         * prevent "has_el3" from existing on CPUs which cannot support EL3.
+         */
+        qdev_property_add_static(DEVICE(obj), &arm_cpu_has_el3_property,
+                                 &error_abort);
+
+#ifndef CONFIG_USER_ONLY
+        object_property_add_link(obj, "secure-memory",
+                                 TYPE_MEMORY_REGION,
+                                 (Object **)&cpu->secure_memory,
+                                 qdev_prop_allow_set_link_before_realize,
+                                 OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                                 &error_abort);
+#endif
+    }
+
+    if (arm_feature(&cpu->env, ARM_FEATURE_MPU)) {
+        qdev_property_add_static(DEVICE(obj), &arm_cpu_has_mpu_property,
+                                 &error_abort);
+        if (arm_feature(&cpu->env, ARM_FEATURE_V7)) {
+            qdev_property_add_static(DEVICE(obj),
+                                     &arm_cpu_pmsav7_dregion_property,
+                                     &error_abort);
+        }
+    }
+
 }
 
 static void arm_cpu_finalizefn(Object *obj)
@@ -462,16 +626,85 @@
     if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
         set_feature(env, ARM_FEATURE_CBAR);
     }
+    if (arm_feature(env, ARM_FEATURE_THUMB2) &&
+        !arm_feature(env, ARM_FEATURE_M)) {
+        set_feature(env, ARM_FEATURE_THUMB_DSP);
+    }
 
     if (cpu->reset_hivecs) {
             cpu->reset_sctlr |= (1 << 13);
     }
 
+    if (!cpu->has_el3) {
+        /* If the has_el3 CPU property is disabled then we need to disable the
+         * feature.
+         */
+        unset_feature(env, ARM_FEATURE_EL3);
+
+        /* Disable the security extension feature bits in the processor feature
+         * registers as well. These are id_pfr1[7:4] and id_aa64pfr0[15:12].
+         */
+        cpu->id_pfr1 &= ~0xf0;
+        cpu->id_aa64pfr0 &= ~0xf000;
+    }
+
+    if (!arm_feature(env, ARM_FEATURE_EL2)) {
+        /* Disable the hypervisor feature bits in the processor feature
+         * registers if we don't have EL2. These are id_pfr1[15:12] and
+         * id_aa64pfr0_el1[11:8].
+         */
+        cpu->id_aa64pfr0 &= ~0xf00;
+        cpu->id_pfr1 &= ~0xf000;
+    }
+
+    if (!cpu->has_mpu) {
+        unset_feature(env, ARM_FEATURE_MPU);
+    }
+
+    if (arm_feature(env, ARM_FEATURE_MPU) &&
+        arm_feature(env, ARM_FEATURE_V7)) {
+        uint32_t nr = cpu->pmsav7_dregion;
+
+        if (nr > 0xff) {
+            error_setg(errp, "PMSAv7 MPU #regions invalid %" PRIu32, nr);
+            return;
+        }
+
+        if (nr) {
+            env->pmsav7.drbar = g_new0(uint32_t, nr);
+            env->pmsav7.drsr = g_new0(uint32_t, nr);
+            env->pmsav7.dracr = g_new0(uint32_t, nr);
+        }
+    }
+
     register_cp_regs_for_features(cpu);
     arm_cpu_register_gdb_regs_for_features(cpu);
 
     init_cpreg_list(cpu);
 
+#ifndef CONFIG_USER_ONLY
+    if (cpu->has_el3) {
+        cs->num_ases = 2;
+    } else {
+        cs->num_ases = 1;
+    }
+
+    if (cpu->has_el3) {
+        AddressSpace *as;
+
+        if (!cpu->secure_memory) {
+            cpu->secure_memory = cs->memory;
+        }
+        as = address_space_init_shareable(cpu->secure_memory,
+                                          "cpu-secure-memory");
+        cpu_address_space_init(cs, as, ARMASIdx_S);
+    }
+    cpu_address_space_init(cs,
+                           address_space_init_shareable(cs->memory,
+                                                        "cpu-memory"),
+                           ARMASIdx_NS);
+#endif
+
     qemu_init_vcpu(cs);
     cpu_reset(cs);
 
@@ -482,13 +715,16 @@
 {
     ObjectClass *oc;
     char *typename;
+    char **cpuname;
 
     if (!cpu_model) {
         return NULL;
     }
 
-    typename = g_strdup_printf("%s-" TYPE_ARM_CPU, cpu_model);
+    cpuname = g_strsplit(cpu_model, ",", 1);
+    typename = g_strdup_printf("%s-" TYPE_ARM_CPU, cpuname[0]);
     oc = object_class_by_name(typename);
+    g_strfreev(cpuname);
     g_free(typename);
     if (!oc || !object_class_dynamic_cast(oc, TYPE_ARM_CPU) ||
         object_class_is_abstract(oc)) {
@@ -548,7 +784,7 @@
         ARMCPRegInfo ifar = {
             .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
             .access = PL1_RW,
-            .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el[1]),
+            .fieldoffset = offsetof(CPUARMState, cp15.ifar_ns),
             .resetvalue = 0
         };
         define_one_arm_cp_reg(cpu, &ifar);
@@ -636,6 +872,7 @@
     set_feature(&cpu->env, ARM_FEATURE_DUMMY_C15_REGS);
     set_feature(&cpu->env, ARM_FEATURE_CACHE_DIRTY_REG);
     set_feature(&cpu->env, ARM_FEATURE_CACHE_BLOCK_OPS);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     cpu->midr = 0x410fb767;
     cpu->reset_fpsid = 0x410120b5;
     cpu->mvfr0 = 0x11111111;
@@ -695,6 +932,15 @@
     cpu->midr = 0x410fc231;
 }
 
+static void cortex_m4_initfn(Object *obj)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+
+    set_feature(&cpu->env, ARM_FEATURE_V7);
+    set_feature(&cpu->env, ARM_FEATURE_M);
+    set_feature(&cpu->env, ARM_FEATURE_THUMB_DSP);
+    cpu->midr = 0x410fc240; /* r0p0 */
+}
 static void arm_v7m_class_init(ObjectClass *oc, void *data)
 {
     CPUClass *cc = CPU_CLASS(oc);
@@ -706,6 +952,43 @@
     cc->cpu_exec_interrupt = arm_v7m_cpu_exec_interrupt;
 }
 
+static const ARMCPRegInfo cortexr5_cp_reginfo[] = {
+    /* Dummy the TCM region regs for the moment */
+    { .name = "ATCM", .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
+      .access = PL1_RW, .type = ARM_CP_CONST },
+    { .name = "BTCM", .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
+      .access = PL1_RW, .type = ARM_CP_CONST },
+    REGINFO_SENTINEL
+};
+
+static void cortex_r5_initfn(Object *obj)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+
+    set_feature(&cpu->env, ARM_FEATURE_V7);
+    set_feature(&cpu->env, ARM_FEATURE_THUMB_DIV);
+    set_feature(&cpu->env, ARM_FEATURE_ARM_DIV);
+    set_feature(&cpu->env, ARM_FEATURE_V7MP);
+    set_feature(&cpu->env, ARM_FEATURE_MPU);
+    cpu->midr = 0x411fc153; /* r1p3 */
+    cpu->id_pfr0 = 0x0131;
+    cpu->id_pfr1 = 0x001;
+    cpu->id_dfr0 = 0x010400;
+    cpu->id_afr0 = 0x0;
+    cpu->id_mmfr0 = 0x0210030;
+    cpu->id_mmfr1 = 0x00000000;
+    cpu->id_mmfr2 = 0x01200000;
+    cpu->id_mmfr3 = 0x0211;
+    cpu->id_isar0 = 0x2101111;
+    cpu->id_isar1 = 0x13112111;
+    cpu->id_isar2 = 0x21232141;
+    cpu->id_isar3 = 0x01112131;
+    cpu->id_isar4 = 0x0010142;
+    cpu->id_isar5 = 0x0;
+    cpu->mp_is_up = true;
+    define_arm_cp_regs(cpu, cortexr5_cp_reginfo);
+}
+
 static const ARMCPRegInfo cortexa8_cp_reginfo[] = {
     { .name = "L2LOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 1, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
@@ -724,6 +1007,7 @@
     set_feature(&cpu->env, ARM_FEATURE_NEON);
     set_feature(&cpu->env, ARM_FEATURE_THUMB2EE);
     set_feature(&cpu->env, ARM_FEATURE_DUMMY_C15_REGS);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     cpu->midr = 0x410fc080;
     cpu->reset_fpsid = 0x410330c0;
     cpu->mvfr0 = 0x11110222;
@@ -791,6 +1075,7 @@
     set_feature(&cpu->env, ARM_FEATURE_VFP_FP16);
     set_feature(&cpu->env, ARM_FEATURE_NEON);
     set_feature(&cpu->env, ARM_FEATURE_THUMB2EE);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     /* Note that A9 supports the MP extensions even for
      * A9UP and single-core A9MP (which are both different
      * and valid configurations; we don't model A9UP).
@@ -858,6 +1143,7 @@
     set_feature(&cpu->env, ARM_FEATURE_DUMMY_C15_REGS);
     set_feature(&cpu->env, ARM_FEATURE_CBAR_RO);
     set_feature(&cpu->env, ARM_FEATURE_LPAE);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A15;
     cpu->midr = 0x412fc0f1;
     cpu->reset_fpsid = 0x410430f0;
@@ -868,6 +1154,8 @@
     cpu->id_pfr0 = 0x00001131;
     cpu->id_pfr1 = 0x00011011;
     cpu->id_dfr0 = 0x02010555;
+    cpu->pmceid0 = 0x0000000;
+    cpu->pmceid1 = 0x00000000;
     cpu->id_afr0 = 0x00000000;
     cpu->id_mmfr0 = 0x10201105;
     cpu->id_mmfr1 = 0x20000000;
@@ -1094,6 +1382,9 @@
     { .name = "arm11mpcore", .initfn = arm11mpcore_initfn },
     { .name = "cortex-m3",   .initfn = cortex_m3_initfn,
                              .class_init = arm_v7m_class_init },
+    { .name = "cortex-m4",   .initfn = cortex_m4_initfn,
+                             .class_init = arm_v7m_class_init },
+    { .name = "cortex-r5",   .initfn = cortex_r5_initfn },
     { .name = "cortex-a8",   .initfn = cortex_a8_initfn },
     { .name = "cortex-a9",   .initfn = cortex_a9_initfn },
     { .name = "cortex-a15",  .initfn = cortex_a15_initfn },
@@ -1124,9 +1415,27 @@
     DEFINE_PROP_BOOL("start-powered-off", ARMCPU, start_powered_off, false),
     DEFINE_PROP_UINT32("psci-conduit", ARMCPU, psci_conduit, 0),
     DEFINE_PROP_UINT32("midr", ARMCPU, midr, 0),
+    DEFINE_PROP_UINT64("mp-affinity", ARMCPU, mp_affinity, 0),
     DEFINE_PROP_END_OF_LIST()
 };
 
+#ifdef CONFIG_USER_ONLY
+static int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
+                                    int mmu_idx)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    env->exception.vaddress = address;
+    if (rw == 2) {
+        cs->exception_index = EXCP_PREFETCH_ABORT;
+    } else {
+        cs->exception_index = EXCP_DATA_ABORT;
+    }
+    return 1;
+}
+#endif
+
 static gchar *arm_gdb_arch_name(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
@@ -1162,14 +1471,33 @@
     cc->handle_mmu_fault = arm_cpu_handle_mmu_fault;
 #else
     cc->do_interrupt = arm_cpu_do_interrupt;
-    cc->get_phys_page_debug = arm_cpu_get_phys_page_debug;
+    cc->do_unaligned_access = arm_cpu_do_unaligned_access;
+    cc->get_phys_page_attrs_debug = arm_cpu_get_phys_page_attrs_debug;
+    cc->asidx_from_attrs = arm_asidx_from_attrs;
     cc->vmsd = &vmstate_arm_cpu;
+    cc->virtio_is_big_endian = arm_cpu_virtio_is_big_endian;
+    cc->write_elf64_note = arm_cpu_write_elf64_note;
+    cc->write_elf32_note = arm_cpu_write_elf32_note;
 #endif
     cc->gdb_num_core_regs = 26;
     cc->gdb_core_xml_file = "arm-core.xml";
     cc->gdb_arch_name = arm_gdb_arch_name;
     cc->gdb_stop_before_watchpoint = true;
     cc->debug_excp_handler = arm_debug_excp_handler;
+    cc->debug_check_watchpoint = arm_debug_check_watchpoint;
+
+    cc->disas_set_info = arm_disas_set_info;
+
+    /*
+     * Reason: arm_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     *
+     * Once this is fixed, the devices that create ARM CPUs should be
+     * updated not to set cannot_destroy_with_object_finalize_yet,
+     * unless they still screw up something else.
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void cpu_register(const ARMCPUInfo *info)
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index 4f4b161..fc637cf 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -16,31 +16,27 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef CPU_ARM_H
-#define CPU_ARM_H
 
-#include "config.h"
+#ifndef ARM_CPU_H
+#define ARM_CPU_H
 
 #include "kvm-consts.h"
 
 #if defined(TARGET_AARCH64)
   /* AArch64 definitions */
 #  define TARGET_LONG_BITS 64
-#  define ELF_MACHINE EM_AARCH64
 #else
 #  define TARGET_LONG_BITS 32
-#  define ELF_MACHINE EM_ARM
 #endif
 
 #define CPUArchState struct CPUARMState
 
 #include "qemu-common.h"
+#include "cpu-qom.h"
 #include "exec/cpu-defs.h"
 
 #include "fpu/softfloat.h"
 
-#define TARGET_HAS_ICE 1
-
 #define EXCP_UDEF            1   /* undefined instruction */
 #define EXCP_SWI             2   /* software interrupt */
 #define EXCP_PREFETCH_ABORT  3
@@ -56,6 +52,7 @@
 #define EXCP_SMC            13   /* Secure Monitor Call */
 #define EXCP_VIRQ           14
 #define EXCP_VFIQ           15
+#define EXCP_SEMIHOST       16   /* semihosting call (A64 only) */
 
 #define ARMV7M_EXCP_RESET   1
 #define ARMV7M_EXCP_NMI     2
@@ -93,14 +90,20 @@
 #define ARM_CPU_VIRQ 2
 #define ARM_CPU_VFIQ 3
 
-typedef void ARMWriteCPFunc(void *opaque, int cp_info,
-                            int srcreg, int operand, uint32_t value);
-typedef uint32_t ARMReadCPFunc(void *opaque, int cp_info,
-                               int dstreg, int operand);
+#define NB_MMU_MODES 7
+/* ARM-specific extra insn start words:
+ * 1: Conditional execution bits
+ * 2: Partial exception syndrome for data aborts
+ */
+#define TARGET_INSN_START_EXTRA_WORDS 2
 
-struct arm_boot_info;
-
-#define NB_MMU_MODES 4
+/* The 2nd extra word holding syndrome info for data aborts does not use
+ * the upper 6 bits nor the lower 14 bits. We mask and shift it down to
+ * help the sleb128 encoder do a better job.
+ * When restoring the CPU state, we shift it back up.
+ */
+#define ARM_INSN_START_WORD2_MASK ((1 << 26) - 1)
+#define ARM_INSN_START_WORD2_SHIFT 14
 
 /* We currently assume float and double are IEEE single and double
    precision respectively.
@@ -118,7 +121,15 @@
 
 #define GTIMER_PHYS 0
 #define GTIMER_VIRT 1
-#define NUM_GTIMERS 2
+#define GTIMER_HYP  2
+#define GTIMER_SEC  3
+#define NUM_GTIMERS 4
+
+typedef struct {
+    uint64_t raw_tcr;
+    uint32_t mask;
+    uint32_t base_mask;
+} TCR;
 
 typedef struct CPUARMState {
     /* Regs for current mode.  */
@@ -169,7 +180,7 @@
     uint32_t GE; /* cpsr[19:16] */
     uint32_t thumb; /* cpsr[5]. 0 = arm mode, 1 = thumb mode. */
     uint32_t condexec_bits; /* IT bits.  cpsr[15:10,26:25].  */
-    uint64_t daif; /* exception masks, in the bits they are in in PSTATE */
+    uint64_t daif; /* exception masks, in the bits they are in PSTATE */
 
     uint64_t elr_el[4]; /* AArch64 exception link regs  */
     uint64_t sp_el[4]; /* AArch64 banked stack pointers */
@@ -177,28 +188,119 @@
     /* System control coprocessor (cp15) */
     struct {
         uint32_t c0_cpuid;
-        uint64_t c0_cssel; /* Cache size selection.  */
-        uint64_t c1_sys; /* System control register.  */
-        uint64_t c1_coproc; /* Coprocessor access register.  */
+        union { /* Cache size selection */
+            struct {
+                uint64_t _unused_csselr0;
+                uint64_t csselr_ns;
+                uint64_t _unused_csselr1;
+                uint64_t csselr_s;
+            };
+            uint64_t csselr_el[4];
+        };
+        union { /* System control register. */
+            struct {
+                uint64_t _unused_sctlr;
+                uint64_t sctlr_ns;
+                uint64_t hsctlr;
+                uint64_t sctlr_s;
+            };
+            uint64_t sctlr_el[4];
+        };
+        uint64_t cpacr_el1; /* Architectural feature access control register */
+        uint64_t cptr_el[4];  /* ARMv8 feature trap registers */
         uint32_t c1_xscaleauxcr; /* XScale auxiliary control register.  */
-        uint64_t ttbr0_el1; /* MMU translation table base 0. */
-        uint64_t ttbr1_el1; /* MMU translation table base 1. */
-        uint64_t c2_control; /* MMU translation table base control.  */
-        uint32_t c2_mask; /* MMU translation table base selection mask.  */
-        uint32_t c2_base_mask; /* MMU translation table base 0 mask. */
-        uint32_t c2_data; /* MPU data cachable bits.  */
-        uint32_t c2_insn; /* MPU instruction cachable bits.  */
-        uint32_t c3; /* MMU domain access control register
-                        MPU write buffer control.  */
+        uint64_t sder; /* Secure debug enable register. */
+        uint32_t nsacr; /* Non-secure access control register. */
+        union { /* MMU translation table base 0. */
+            struct {
+                uint64_t _unused_ttbr0_0;
+                uint64_t ttbr0_ns;
+                uint64_t _unused_ttbr0_1;
+                uint64_t ttbr0_s;
+            };
+            uint64_t ttbr0_el[4];
+        };
+        union { /* MMU translation table base 1. */
+            struct {
+                uint64_t _unused_ttbr1_0;
+                uint64_t ttbr1_ns;
+                uint64_t _unused_ttbr1_1;
+                uint64_t ttbr1_s;
+            };
+            uint64_t ttbr1_el[4];
+        };
+        uint64_t vttbr_el2; /* Virtualization Translation Table Base.  */
+        /* MMU translation table base control. */
+        TCR tcr_el[4];
+        TCR vtcr_el2; /* Virtualization Translation Control.  */
+        uint32_t c2_data; /* MPU data cacheable bits.  */
+        uint32_t c2_insn; /* MPU instruction cacheable bits.  */
+        union { /* MMU domain access control register
+                 * MPU write buffer control.
+                 */
+            struct {
+                uint64_t dacr_ns;
+                uint64_t dacr_s;
+            };
+            struct {
+                uint64_t dacr32_el2;
+            };
+        };
         uint32_t pmsav5_data_ap; /* PMSAv5 MPU data access permissions */
         uint32_t pmsav5_insn_ap; /* PMSAv5 MPU insn access permissions */
         uint64_t hcr_el2; /* Hypervisor configuration register */
         uint64_t scr_el3; /* Secure configuration register.  */
-        uint32_t ifsr_el2; /* Fault status registers.  */
-        uint64_t esr_el[4];
+        union { /* Fault status registers.  */
+            struct {
+                uint64_t ifsr_ns;
+                uint64_t ifsr_s;
+            };
+            struct {
+                uint64_t ifsr32_el2;
+            };
+        };
+        union {
+            struct {
+                uint64_t _unused_dfsr;
+                uint64_t dfsr_ns;
+                uint64_t hsr;
+                uint64_t dfsr_s;
+            };
+            uint64_t esr_el[4];
+        };
         uint32_t c6_region[8]; /* MPU base/size registers.  */
-        uint64_t far_el[4]; /* Fault address registers.  */
-        uint64_t par_el1;  /* Translation result. */
+        union { /* Fault address registers. */
+            struct {
+                uint64_t _unused_far0;
+#ifdef HOST_WORDS_BIGENDIAN
+                uint32_t ifar_ns;
+                uint32_t dfar_ns;
+                uint32_t ifar_s;
+                uint32_t dfar_s;
+#else
+                uint32_t dfar_ns;
+                uint32_t ifar_ns;
+                uint32_t dfar_s;
+                uint32_t ifar_s;
+#endif
+                uint64_t _unused_far3;
+            };
+            uint64_t far_el[4];
+        };
+        uint64_t hpfar_el2;
+        uint64_t hstr_el2;
+        union { /* Translation result. */
+            struct {
+                uint64_t _unused_par_0;
+                uint64_t par_ns;
+                uint64_t _unused_par_1;
+                uint64_t par_s;
+            };
+            uint64_t par_el[4];
+        };
+
+        uint32_t c6_rgnr;
+
         uint32_t c9_insn; /* Cache lockdown registers.  */
         uint32_t c9_data;
         uint64_t c9_pmcr; /* performance monitor control register */
@@ -207,15 +309,71 @@
         uint32_t c9_pmxevtyper; /* perf monitor event type */
         uint32_t c9_pmuserenr; /* perf monitor user enable */
         uint32_t c9_pminten; /* perf monitor interrupt enables */
-        uint64_t mair_el1;
-        uint64_t vbar_el[4]; /* vector base address register */
-        uint32_t c13_fcse; /* FCSE PID.  */
-        uint64_t contextidr_el1; /* Context ID.  */
-        uint64_t tpidr_el0; /* User RW Thread register.  */
-        uint64_t tpidrro_el0; /* User RO Thread register.  */
-        uint64_t tpidr_el1; /* Privileged Thread register.  */
+        union { /* Memory attribute redirection */
+            struct {
+#ifdef HOST_WORDS_BIGENDIAN
+                uint64_t _unused_mair_0;
+                uint32_t mair1_ns;
+                uint32_t mair0_ns;
+                uint64_t _unused_mair_1;
+                uint32_t mair1_s;
+                uint32_t mair0_s;
+#else
+                uint64_t _unused_mair_0;
+                uint32_t mair0_ns;
+                uint32_t mair1_ns;
+                uint64_t _unused_mair_1;
+                uint32_t mair0_s;
+                uint32_t mair1_s;
+#endif
+            };
+            uint64_t mair_el[4];
+        };
+        union { /* vector base address register */
+            struct {
+                uint64_t _unused_vbar;
+                uint64_t vbar_ns;
+                uint64_t hvbar;
+                uint64_t vbar_s;
+            };
+            uint64_t vbar_el[4];
+        };
+        uint32_t mvbar; /* (monitor) vector base address register */
+        struct { /* FCSE PID. */
+            uint32_t fcseidr_ns;
+            uint32_t fcseidr_s;
+        };
+        union { /* Context ID. */
+            struct {
+                uint64_t _unused_contextidr_0;
+                uint64_t contextidr_ns;
+                uint64_t _unused_contextidr_1;
+                uint64_t contextidr_s;
+            };
+            uint64_t contextidr_el[4];
+        };
+        union { /* User RW Thread register. */
+            struct {
+                uint64_t tpidrurw_ns;
+                uint64_t tpidrprw_ns;
+                uint64_t htpidr;
+                uint64_t _tpidr_el3;
+            };
+            uint64_t tpidr_el[4];
+        };
+        /* The secure banks of these registers don't map anywhere */
+        uint64_t tpidrurw_s;
+        uint64_t tpidrprw_s;
+        uint64_t tpidruro_s;
+
+        union { /* User RO Thread register. */
+            uint64_t tpidruro_ns;
+            uint64_t tpidrro_el[1];
+        };
         uint64_t c14_cntfrq; /* Counter Frequency register */
         uint64_t c14_cntkctl; /* Timer Control register */
+        uint32_t cnthctl_el2; /* Counter/Timer Hyp Control register */
+        uint64_t cntvoff_el2; /* Counter Virtual Offset register */
         ARMGenericTimer c14_timer[NUM_GTIMERS];
         uint32_t c15_cpar; /* XScale Coprocessor Access Register */
         uint32_t c15_ticonfig; /* TI925T configuration byte.  */
@@ -231,11 +389,16 @@
         uint64_t dbgwvr[16]; /* watchpoint value registers */
         uint64_t dbgwcr[16]; /* watchpoint control registers */
         uint64_t mdscr_el1;
+        uint64_t oslsr_el1; /* OS Lock Status */
+        uint64_t mdcr_el2;
+        uint64_t mdcr_el3;
         /* If the counter is enabled, this stores the last time the counter
          * was reset. Otherwise it stores the counter value
          */
         uint64_t c15_ccnt;
         uint64_t pmccfiltr_el0; /* Performance Monitor Filter Register */
+        uint64_t vpidr_el2; /* Virtualization Processor ID Register */
+        uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */
     } cp15;
 
     struct {
@@ -245,7 +408,6 @@
         uint32_t control;
         int current_sp;
         int exception;
-        int pending_exception;
     } v7m;
 
     /* Information associated with an exception about to be taken:
@@ -258,6 +420,7 @@
         uint32_t syndrome; /* AArch64 format syndrome register */
         uint32_t fsr; /* AArch32 format fault status register info */
         uint64_t vaddress; /* virtual addr associated with exception, if any */
+        uint32_t target_el; /* EL the exception should be targeted for */
         /* If we implement EL2 we will also need to store information
          * about the intermediate physical address for stage 2 faults.
          */
@@ -325,9 +488,6 @@
         uint32_t cregs[16];
     } iwmmxt;
 
-    /* For mixed endian mode.  */
-    bool bswap_code;
-
 #if defined(CONFIG_USER_ONLY)
     /* For usermode syscall translation.  */
     int eabi;
@@ -343,15 +503,209 @@
     /* Internal CPU feature flags.  */
     uint64_t features;
 
+    /* PMSAv7 MPU */
+    struct {
+        uint32_t *drbar;
+        uint32_t *drsr;
+        uint32_t *dracr;
+    } pmsav7;
+
     void *nvic;
     const struct arm_boot_info *boot_info;
 } CPUARMState;
 
-#include "cpu-qom.h"
+/**
+ * ARMELChangeHook:
+ * type of a function which can be registered via arm_register_el_change_hook()
+ * to get callbacks when the CPU changes its exception level or mode.
+ */
+typedef void ARMELChangeHook(ARMCPU *cpu, void *opaque);
+
+/**
+ * ARMCPU:
+ * @env: #CPUARMState
+ *
+ * An ARM CPU core.
+ */
+struct ARMCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUARMState env;
+
+    /* Coprocessor information */
+    GHashTable *cp_regs;
+    /* For marshalling (mostly coprocessor) register state between the
+     * kernel and QEMU (for KVM) and between two QEMUs (for migration),
+     * we use these arrays.
+     */
+    /* List of register indexes managed via these arrays; (full KVM style
+     * 64 bit indexes, not CPRegInfo 32 bit indexes)
+     */
+    uint64_t *cpreg_indexes;
+    /* Values of the registers (cpreg_indexes[i]'s value is cpreg_values[i]) */
+    uint64_t *cpreg_values;
+    /* Length of the indexes, values, reset_values arrays */
+    int32_t cpreg_array_len;
+    /* These are used only for migration: incoming data arrives in
+     * these fields and is sanity checked in post_load before copying
+     * to the working data structures above.
+     */
+    uint64_t *cpreg_vmstate_indexes;
+    uint64_t *cpreg_vmstate_values;
+    int32_t cpreg_vmstate_array_len;
+
+    /* Timers used by the generic (architected) timer */
+    QEMUTimer *gt_timer[NUM_GTIMERS];
+    /* GPIO outputs for generic timer */
+    qemu_irq gt_timer_outputs[NUM_GTIMERS];
+
+    /* MemoryRegion to use for secure physical accesses */
+    MemoryRegion *secure_memory;
+
+    /* 'compatible' string for this CPU for Linux device trees */
+    const char *dtb_compatible;
+
+    /* PSCI version for this CPU
+     * Bits[31:16] = Major Version
+     * Bits[15:0] = Minor Version
+     */
+    uint32_t psci_version;
+
+    /* Should CPU start in PSCI powered-off state? */
+    bool start_powered_off;
+    /* CPU currently in PSCI powered-off state */
+    bool powered_off;
+    /* CPU has security extension */
+    bool has_el3;
+    /* CPU has PMU (Performance Monitor Unit) */
+    bool has_pmu;
+
+    /* CPU has memory protection unit */
+    bool has_mpu;
+    /* PMSAv7 MPU number of supported regions */
+    uint32_t pmsav7_dregion;
+
+    /* PSCI conduit used to invoke PSCI methods
+     * 0 - disabled, 1 - smc, 2 - hvc
+     */
+    uint32_t psci_conduit;
+
+    /* [QEMU_]KVM_ARM_TARGET_* constant for this CPU, or
+     * QEMU_KVM_ARM_TARGET_NONE if the kernel doesn't support this CPU type.
+     */
+    uint32_t kvm_target;
+
+    /* KVM init features for this CPU */
+    uint32_t kvm_init_features[7];
+
+    /* Uniprocessor system with MP extensions */
+    bool mp_is_up;
+
+    /* The instance init functions for implementation-specific subclasses
+     * set these fields to specify the implementation-dependent values of
+     * various constant registers and reset values of non-constant
+     * registers.
+     * Some of these might become QOM properties eventually.
+     * Field names match the official register names as defined in the
+     * ARMv7AR ARM Architecture Reference Manual. A reset_ prefix
+     * is used for reset values of non-constant registers; no reset_
+     * prefix means a constant register.
+     */
+    uint32_t midr;
+    uint32_t revidr;
+    uint32_t reset_fpsid;
+    uint32_t mvfr0;
+    uint32_t mvfr1;
+    uint32_t mvfr2;
+    uint32_t ctr;
+    uint32_t reset_sctlr;
+    uint32_t id_pfr0;
+    uint32_t id_pfr1;
+    uint32_t id_dfr0;
+    uint32_t pmceid0;
+    uint32_t pmceid1;
+    uint32_t id_afr0;
+    uint32_t id_mmfr0;
+    uint32_t id_mmfr1;
+    uint32_t id_mmfr2;
+    uint32_t id_mmfr3;
+    uint32_t id_mmfr4;
+    uint32_t id_isar0;
+    uint32_t id_isar1;
+    uint32_t id_isar2;
+    uint32_t id_isar3;
+    uint32_t id_isar4;
+    uint32_t id_isar5;
+    uint64_t id_aa64pfr0;
+    uint64_t id_aa64pfr1;
+    uint64_t id_aa64dfr0;
+    uint64_t id_aa64dfr1;
+    uint64_t id_aa64afr0;
+    uint64_t id_aa64afr1;
+    uint64_t id_aa64isar0;
+    uint64_t id_aa64isar1;
+    uint64_t id_aa64mmfr0;
+    uint64_t id_aa64mmfr1;
+    uint32_t dbgdidr;
+    uint32_t clidr;
+    uint64_t mp_affinity; /* MP ID without feature bits */
+    /* The elements of this array are the CCSIDR values for each cache,
+     * in the order L1DCache, L1ICache, L2DCache, L2ICache, etc.
+     */
+    uint32_t ccsidr[16];
+    uint64_t reset_cbar;
+    uint32_t reset_auxcr;
+    bool reset_hivecs;
+    /* DCZ blocksize, in log_2(words), ie low 4 bits of DCZID_EL0 */
+    uint32_t dcz_blocksize;
+    uint64_t rvbar;
+
+    ARMELChangeHook *el_change_hook;
+    void *el_change_hook_opaque;
+};
+
+static inline ARMCPU *arm_env_get_cpu(CPUARMState *env)
+{
+    return container_of(env, ARMCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(arm_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(ARMCPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_arm_cpu;
+#endif
+
+void arm_cpu_do_interrupt(CPUState *cpu);
+void arm_v7m_cpu_do_interrupt(CPUState *cpu);
+bool arm_cpu_exec_interrupt(CPUState *cpu, int int_req);
+
+void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
+                        int flags);
+
+hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cpu, vaddr addr,
+                                         MemTxAttrs *attrs);
+
+int arm_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int arm_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+
+int arm_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque);
+int arm_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cs,
+                             int cpuid, void *opaque);
+
+#ifdef TARGET_AARCH64
+int aarch64_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int aarch64_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+#endif
 
 ARMCPU *cpu_arm_init(const char *cpu_model);
-int cpu_arm_exec(CPUARMState *s);
-uint32_t do_arm_semihosting(CPUARMState *env);
+target_ulong do_arm_semihosting(CPUARMState *env);
+void aarch64_sync_32_to_64(CPUARMState *env);
+void aarch64_sync_64_to_32(CPUARMState *env);
 
 static inline bool is_a64(CPUARMState *env)
 {
@@ -363,8 +717,6 @@
    is returned if the signal was handled by the virtual CPU.  */
 int cpu_arm_signal_handler(int host_signum, void *pinfo,
                            void *puc);
-int arm_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
-                             int mmu_idx);
 
 /**
  * pmccntr_sync
@@ -411,6 +763,7 @@
 #define SCTLR_DT      (1U << 16) /* up to ??, RAO in v6 and v7 */
 #define SCTLR_nTWI    (1U << 16) /* v8 onward */
 #define SCTLR_HA      (1U << 17)
+#define SCTLR_BR      (1U << 17) /* PMSA only */
 #define SCTLR_IT      (1U << 18) /* up to ??, RAO in v6 and v7 */
 #define SCTLR_nTWE    (1U << 18) /* v8 onward */
 #define SCTLR_WXN     (1U << 19)
@@ -429,6 +782,26 @@
 #define SCTLR_AFE     (1U << 29)
 #define SCTLR_TE      (1U << 30)
 
+#define CPTR_TCPAC    (1U << 31)
+#define CPTR_TTA      (1U << 20)
+#define CPTR_TFP      (1U << 10)
+
+#define MDCR_EPMAD    (1U << 21)
+#define MDCR_EDAD     (1U << 20)
+#define MDCR_SPME     (1U << 17)
+#define MDCR_SDD      (1U << 16)
+#define MDCR_SPD      (3U << 14)
+#define MDCR_TDRA     (1U << 11)
+#define MDCR_TDOSA    (1U << 10)
+#define MDCR_TDA      (1U << 9)
+#define MDCR_TDE      (1U << 8)
+#define MDCR_HPME     (1U << 7)
+#define MDCR_TPM      (1U << 6)
+#define MDCR_TPMCR    (1U << 5)
+
+/* Not all of the MDCR_EL3 bits are present in the 32-bit SDCR */
+#define SDCR_VALID_MASK (MDCR_EPMAD | MDCR_EDAD | MDCR_SPME | MDCR_SPD)
+
 #define CPSR_M (0x1fU)
 #define CPSR_T (1U << 5)
 #define CPSR_F (1U << 6)
@@ -541,8 +914,17 @@
 
 /* Return the current CPSR value.  */
 uint32_t cpsr_read(CPUARMState *env);
-/* Set the CPSR.  Note that some bits of mask must be all-set or all-clear.  */
-void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask);
+
+typedef enum CPSRWriteType {
+    CPSRWriteByInstr = 0,         /* from guest MSR or CPS */
+    CPSRWriteExceptionReturn = 1, /* from guest exception return insn */
+    CPSRWriteRaw = 2,             /* trust values, do not switch reg banks */
+    CPSRWriteByGDBStub = 3,       /* from the GDB stub */
+} CPSRWriteType;
+
+/* Set the CPSR.  Note that some bits of mask must be all-set or all-clear.*/
+void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
+                CPSRWriteType write_type);
 
 /* Return the current xPSR value.  */
 static inline uint32_t xpsr_read(CPUARMState *env)
@@ -746,6 +1128,7 @@
     ARM_FEATURE_V8_SHA1, /* implements SHA1 part of v8 Crypto Extensions */
     ARM_FEATURE_V8_SHA256, /* implements SHA256 part of v8 Crypto Extensions */
     ARM_FEATURE_V8_PMULL, /* implements PMULL part of v8 Crypto Extensions */
+    ARM_FEATURE_THUMB_DSP, /* DSP insns supported in the Thumb encodings */
 };
 
 static inline int arm_feature(CPUARMState *env, int feature)
@@ -765,15 +1148,15 @@
     if (arm_feature(env, ARM_FEATURE_EL3)) {
         return !(env->cp15.scr_el3 & SCR_NS);
     } else {
-        /* If EL2 is not supported then the secure state is implementation
+        /* If EL3 is not supported then the secure state is implementation
          * defined, in which case QEMU defaults to non-secure.
          */
         return false;
     }
 }
 
-/* Return true if the processor is in secure state */
-static inline bool arm_is_secure(CPUARMState *env)
+/* Return true if the CPU is AArch64 EL3 or AArch32 Mon */
+static inline bool arm_is_el3_or_mon(CPUARMState *env)
 {
     if (arm_feature(env, ARM_FEATURE_EL3)) {
         if (is_a64(env) && extract32(env->pstate, 2, 2) == 3) {
@@ -785,6 +1168,15 @@
             return true;
         }
     }
+    return false;
+}
+
+/* Return true if the processor is in secure state */
+static inline bool arm_is_secure(CPUARMState *env)
+{
+    if (arm_is_el3_or_mon(env)) {
+        return true;
+    }
     return arm_is_secure_below_el3(env);
 }
 
@@ -803,22 +1195,81 @@
 /* Return true if the specified exception level is running in AArch64 state. */
 static inline bool arm_el_is_aa64(CPUARMState *env, int el)
 {
-    /* We don't currently support EL2, and this isn't valid for EL0
-     * (if we're in EL0, is_a64() is what you want, and if we're not in EL0
-     * then the state of EL0 isn't well defined.)
+    /* This isn't valid for EL0 (if we're in EL0, is_a64() is what you want,
+     * and if we're not in EL0 then the state of EL0 isn't well defined.)
      */
-    assert(el == 1 || el == 3);
+    assert(el >= 1 && el <= 3);
+    bool aa64 = arm_feature(env, ARM_FEATURE_AARCH64);
 
-    /* AArch64-capable CPUs always run with EL1 in AArch64 mode. This
-     * is a QEMU-imposed simplification which we may wish to change later.
-     * If we in future support EL2 and/or EL3, then the state of lower
-     * exception levels is controlled by the HCR.RW and SCR.RW bits.
+    /* The highest exception level is always at the maximum supported
+     * register width, and then lower levels have a register width controlled
+     * by bits in the SCR or HCR registers.
      */
-    return arm_feature(env, ARM_FEATURE_AARCH64);
+    if (el == 3) {
+        return aa64;
+    }
+
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        aa64 = aa64 && (env->cp15.scr_el3 & SCR_RW);
+    }
+
+    if (el == 2) {
+        return aa64;
+    }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !arm_is_secure_below_el3(env)) {
+        aa64 = aa64 && (env->cp15.hcr_el2 & HCR_RW);
+    }
+
+    return aa64;
 }
 
+/* Function for determing whether guest cp register reads and writes should
+ * access the secure or non-secure bank of a cp register.  When EL3 is
+ * operating in AArch32 state, the NS-bit determines whether the secure
+ * instance of a cp register should be used. When EL3 is AArch64 (or if
+ * it doesn't exist at all) then there is no register banking, and all
+ * accesses are to the non-secure version.
+ */
+static inline bool access_secure_reg(CPUARMState *env)
+{
+    bool ret = (arm_feature(env, ARM_FEATURE_EL3) &&
+                !arm_el_is_aa64(env, 3) &&
+                !(env->cp15.scr_el3 & SCR_NS));
+
+    return ret;
+}
+
+/* Macros for accessing a specified CP register bank */
+#define A32_BANKED_REG_GET(_env, _regname, _secure)    \
+    ((_secure) ? (_env)->cp15._regname##_s : (_env)->cp15._regname##_ns)
+
+#define A32_BANKED_REG_SET(_env, _regname, _secure, _val)   \
+    do {                                                \
+        if (_secure) {                                   \
+            (_env)->cp15._regname##_s = (_val);            \
+        } else {                                        \
+            (_env)->cp15._regname##_ns = (_val);           \
+        }                                               \
+    } while (0)
+
+/* Macros for automatically accessing a specific CP register bank depending on
+ * the current secure state of the system.  These macros are not intended for
+ * supporting instruction translation reads/writes as these are dependent
+ * solely on the SCR.NS bit and not the mode.
+ */
+#define A32_BANKED_CURRENT_REG_GET(_env, _regname)        \
+    A32_BANKED_REG_GET((_env), _regname,                \
+                       (arm_is_secure(_env) && !arm_el_is_aa64((_env), 3)))
+
+#define A32_BANKED_CURRENT_REG_SET(_env, _regname, _val)                       \
+    A32_BANKED_REG_SET((_env), _regname,                                    \
+                       (arm_is_secure(_env) && !arm_el_is_aa64((_env), 3)), \
+                       (_val))
+
 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf);
-unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx);
+uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
+                                 uint32_t cur_el, bool secure);
 
 /* Interface between CPU and Interrupt controller.  */
 void armv7m_nvic_set_pending(void *opaque, int irq);
@@ -836,6 +1287,7 @@
  *  Crn, Crm, opc1, opc2 fields
  *  32 or 64 bit register (ie is it accessed via MRC/MCR
  *    or via MRRC/MCRR?)
+ *  non-secure/secure bank (AArch32 only)
  * We allow 4 bits for opc1 because MRRC/MCRR have a 4 bit field.
  * (In this case crn and opc2 should be zero.)
  * For AArch64, there is no 32/64 bit size distinction;
@@ -853,9 +1305,16 @@
 #define CP_REG_AA64_SHIFT 28
 #define CP_REG_AA64_MASK (1 << CP_REG_AA64_SHIFT)
 
-#define ENCODE_CP_REG(cp, is64, crn, crm, opc1, opc2)   \
-    (((cp) << 16) | ((is64) << 15) | ((crn) << 11) |    \
-     ((crm) << 7) | ((opc1) << 3) | (opc2))
+/* To enable banking of coprocessor registers depending on ns-bit we
+ * add a bit to distinguish between secure and non-secure cpregs in the
+ * hashtable.
+ */
+#define CP_REG_NS_SHIFT 29
+#define CP_REG_NS_MASK (1 << CP_REG_NS_SHIFT)
+
+#define ENCODE_CP_REG(cp, is64, ns, crn, crm, opc1, opc2)   \
+    ((ns) << CP_REG_NS_SHIFT | ((cp) << 16) | ((is64) << 15) |   \
+     ((crn) << 11) | ((crm) << 7) | ((opc1) << 3) | (opc2))
 
 #define ENCODE_AA64_CP_REG(cp, crn, crm, op0, op1, op2) \
     (CP_REG_AA64_MASK |                                 \
@@ -874,8 +1333,15 @@
     uint32_t cpregid = kvmid;
     if ((kvmid & CP_REG_ARCH_MASK) == CP_REG_ARM64) {
         cpregid |= CP_REG_AA64_MASK;
-    } else if ((kvmid & CP_REG_SIZE_MASK) == CP_REG_SIZE_U64) {
-        cpregid |= (1 << 15);
+    } else {
+        if ((kvmid & CP_REG_SIZE_MASK) == CP_REG_SIZE_U64) {
+            cpregid |= (1 << 15);
+        }
+
+        /* KVM is always non-secure so add the NS flag on AArch32 register
+         * entries.
+         */
+         cpregid |= 1 << CP_REG_NS_SHIFT;
     }
     return cpregid;
 }
@@ -911,8 +1377,14 @@
  * a register definition to override a previous definition for the
  * same (cp, is64, crn, crm, opc1, opc2) tuple: either the new or the
  * old must have the OVERRIDE bit set.
- * NO_MIGRATE indicates that this register should be ignored for migration;
- * (eg because any state is accessed via some other coprocessor register).
+ * ALIAS indicates that this register is an alias view of some underlying
+ * state which is also visible via another register, and that the other
+ * register is handling migration and reset; registers marked ALIAS will not be
+ * migrated but may have their state set by syncing of register state from KVM.
+ * NO_RAW indicates that this register has no underlying state and does not
+ * support raw access for state saving/loading; it will not be used for either
+ * migration or KVM state synchronization. (Typically this is for "registers"
+ * which are actually used as instructions for cache maintenance and so on.)
  * IO indicates that this register does I/O and therefore its accesses
  * need to be surrounded by gen_io_start()/gen_io_end(). In particular,
  * registers which implement clocks or timers require this.
@@ -922,8 +1394,9 @@
 #define ARM_CP_64BIT 4
 #define ARM_CP_SUPPRESS_TB_END 8
 #define ARM_CP_OVERRIDE 16
-#define ARM_CP_NO_MIGRATE 32
+#define ARM_CP_ALIAS 32
 #define ARM_CP_IO 64
+#define ARM_CP_NO_RAW 128
 #define ARM_CP_NOP (ARM_CP_SPECIAL | (1 << 8))
 #define ARM_CP_WFI (ARM_CP_SPECIAL | (2 << 8))
 #define ARM_CP_NZCV (ARM_CP_SPECIAL | (3 << 8))
@@ -933,7 +1406,7 @@
 /* Used only as a terminator for ARMCPRegInfo lists */
 #define ARM_CP_SENTINEL 0xffff
 /* Mask of only the flag bits in a type field */
-#define ARM_CP_FLAG_MASK 0x7f
+#define ARM_CP_FLAG_MASK 0xff
 
 /* Valid values for ARMCPRegInfo state field, indicating which of
  * the AArch32 and AArch64 execution states this register is visible in.
@@ -950,6 +1423,21 @@
     ARM_CP_STATE_BOTH = 2,
 };
 
+/* ARM CP register secure state flags.  These flags identify security state
+ * attributes for a given CP register entry.
+ * The existence of both or neither secure and non-secure flags indicates that
+ * the register has both a secure and non-secure hash entry.  A single one of
+ * these flags causes the register to only be hashed for the specified
+ * security state.
+ * Although definitions may have any combination of the S/NS bits, each
+ * registered entry will only have one to identify whether the entry is secure
+ * or non-secure.
+ */
+enum {
+    ARM_CP_SECSTATE_S =   (1 << 0), /* bit[0]: Secure state register */
+    ARM_CP_SECSTATE_NS =  (1 << 1), /* bit[1]: Non-secure state register */
+};
+
 /* Return true if cptype is a valid type field. This is used to try to
  * catch errors where the sentinel has been accidentally left off the end
  * of a list of registers.
@@ -992,11 +1480,27 @@
 #define PL1_RW (PL1_R | PL1_W)
 #define PL0_RW (PL0_R | PL0_W)
 
+/* Return the highest implemented Exception Level */
+static inline int arm_highest_el(CPUARMState *env)
+{
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        return 3;
+    }
+    if (arm_feature(env, ARM_FEATURE_EL2)) {
+        return 2;
+    }
+    return 1;
+}
+
 /* Return the current Exception Level (as per ARMv8; note that this differs
  * from the ARMv7 Privilege Level).
  */
 static inline int arm_current_el(CPUARMState *env)
 {
+    if (arm_feature(env, ARM_FEATURE_M)) {
+        return !((env->v7m.exception == 0) && (env->v7m.control & 1));
+    }
+
     if (is_a64(env)) {
         return extract32(env->pstate, 2, 2);
     }
@@ -1028,7 +1532,8 @@
     /* Access fails due to a configurable trap or enable which would
      * result in a categorized exception syndrome giving information about
      * the failing instruction (ie syndrome category 0x3, 0x4, 0x5, 0x6,
-     * 0xc or 0x18).
+     * 0xc or 0x18). The exception is taken to the usual target EL (EL1 or
+     * PL1 if in EL0, otherwise to the current EL).
      */
     CP_ACCESS_TRAP = 1,
     /* Access fails and results in an exception syndrome 0x0 ("uncategorized").
@@ -1036,6 +1541,17 @@
      * result in this failure is specifically defined by the architecture.
      */
     CP_ACCESS_TRAP_UNCATEGORIZED = 2,
+    /* As CP_ACCESS_TRAP, but for traps directly to EL2 or EL3 */
+    CP_ACCESS_TRAP_EL2 = 3,
+    CP_ACCESS_TRAP_EL3 = 4,
+    /* As CP_ACCESS_UNCATEGORIZED, but for traps directly to EL2 or EL3 */
+    CP_ACCESS_TRAP_UNCATEGORIZED_EL2 = 5,
+    CP_ACCESS_TRAP_UNCATEGORIZED_EL3 = 6,
+    /* Access fails and results in an exception syndrome for an FP access,
+     * trapped directly to EL2 or EL3
+     */
+    CP_ACCESS_TRAP_FP_EL2 = 7,
+    CP_ACCESS_TRAP_FP_EL3 = 8,
 } CPAccessResult;
 
 /* Access functions for coprocessor registers. These cannot fail and
@@ -1045,7 +1561,9 @@
 typedef void CPWriteFn(CPUARMState *env, const ARMCPRegInfo *opaque,
                        uint64_t value);
 /* Access permission check functions for coprocessor registers. */
-typedef CPAccessResult CPAccessFn(CPUARMState *env, const ARMCPRegInfo *opaque);
+typedef CPAccessResult CPAccessFn(CPUARMState *env,
+                                  const ARMCPRegInfo *opaque,
+                                  bool isread);
 /* Hook function for register reset */
 typedef void CPResetFn(CPUARMState *env, const ARMCPRegInfo *opaque);
 
@@ -1084,6 +1602,8 @@
     int type;
     /* Access rights: PL*_[RW] */
     int access;
+    /* Security state: ARM_CP_SECSTATE_* bits/values */
+    int secure;
     /* The opaque pointer passed to define_arm_cp_regs_with_opaque() when
      * this register was defined: can be used to hand data through to the
      * register read/write functions, since they are passed the ARMCPRegInfo*.
@@ -1093,12 +1613,27 @@
      * fieldoffset is non-zero, the reset value of the register.
      */
     uint64_t resetvalue;
-    /* Offset of the field in CPUARMState for this register. This is not
-     * needed if either:
+    /* Offset of the field in CPUARMState for this register.
+     *
+     * This is not needed if either:
      *  1. type is ARM_CP_CONST or one of the ARM_CP_SPECIALs
      *  2. both readfn and writefn are specified
      */
     ptrdiff_t fieldoffset; /* offsetof(CPUARMState, field) */
+
+    /* Offsets of the secure and non-secure fields in CPUARMState for the
+     * register if it is banked.  These fields are only used during the static
+     * registration of a register.  During hashing the bank associated
+     * with a given security state is copied to fieldoffset which is used from
+     * there on out.
+     *
+     * It is expected that register definitions use either fieldoffset or
+     * bank_fieldoffsets in the definition but not both.  It is also expected
+     * that both bank offsets are set when defining a banked register.  This
+     * use indicates that a register is banked.
+     */
+    ptrdiff_t bank_fieldoffsets[2];
+
     /* Function for making any access checks for this register in addition to
      * those specified by the 'access' permissions bits. If NULL, no extra
      * checks required. The access check is performed at runtime, not at
@@ -1184,6 +1719,9 @@
     return (ri->access >> ((current_el * 2) + isread)) & 1;
 }
 
+/* Raw read of a coprocessor register (as needed for migration, etc) */
+uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri);
+
 /**
  * write_list_to_cpustate
  * @cpu: ARMCPU
@@ -1216,7 +1754,7 @@
  */
 bool write_cpustate_to_list(ARMCPU *cpu);
 
-/* Does the core conform to the the "MicroController" profile. e.g. Cortex-M3.
+/* Does the core conform to the "MicroController" profile. e.g. Cortex-M3.
    Note the M in older cores (eg. ARM7TDMI) stands for Multiply. These are
    conventional cores (ie. Application or Realtime profile).  */
 
@@ -1249,32 +1787,32 @@
 #  define TARGET_VIRT_ADDR_SPACE_BITS 32
 #endif
 
-static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx)
+static inline bool arm_excp_unmasked(CPUState *cs, unsigned int excp_idx,
+                                     unsigned int target_el)
 {
     CPUARMState *env = cs->env_ptr;
     unsigned int cur_el = arm_current_el(env);
-    unsigned int target_el = arm_excp_target_el(cs, excp_idx);
-    /* FIXME: Use actual secure state.  */
-    bool secure = false;
-    /* If in EL1/0, Physical IRQ routing to EL2 only happens from NS state.  */
-    bool irq_can_hyp = !secure && cur_el < 2 && target_el == 2;
+    bool secure = arm_is_secure(env);
+    bool pstate_unmasked;
+    int8_t unmasked = 0;
 
-    /* Don't take exceptions if they target a lower EL.  */
+    /* Don't take exceptions if they target a lower EL.
+     * This check should catch any exceptions that would not be taken but left
+     * pending.
+     */
     if (cur_el > target_el) {
         return false;
     }
 
     switch (excp_idx) {
     case EXCP_FIQ:
-        if (irq_can_hyp && (env->cp15.hcr_el2 & HCR_FMO)) {
-            return true;
-        }
-        return !(env->daif & PSTATE_F);
+        pstate_unmasked = !(env->daif & PSTATE_F);
+        break;
+
     case EXCP_IRQ:
-        if (irq_can_hyp && (env->cp15.hcr_el2 & HCR_IMO)) {
-            return true;
-        }
-        return !(env->daif & PSTATE_I);
+        pstate_unmasked = !(env->daif & PSTATE_I);
+        break;
+
     case EXCP_VFIQ:
         if (secure || !(env->cp15.hcr_el2 & HCR_FMO)) {
             /* VFIQs are only taken when hypervized and non-secure.  */
@@ -1290,41 +1828,201 @@
     default:
         g_assert_not_reached();
     }
-}
 
-static inline CPUARMState *cpu_init(const char *cpu_model)
-{
-    ARMCPU *cpu = cpu_arm_init(cpu_model);
-    if (cpu) {
-        return &cpu->env;
+    /* Use the target EL, current execution state and SCR/HCR settings to
+     * determine whether the corresponding CPSR bit is used to mask the
+     * interrupt.
+     */
+    if ((target_el > cur_el) && (target_el != 1)) {
+        /* Exceptions targeting a higher EL may not be maskable */
+        if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+            /* 64-bit masking rules are simple: exceptions to EL3
+             * can't be masked, and exceptions to EL2 can only be
+             * masked from Secure state. The HCR and SCR settings
+             * don't affect the masking logic, only the interrupt routing.
+             */
+            if (target_el == 3 || !secure) {
+                unmasked = 1;
+            }
+        } else {
+            /* The old 32-bit-only environment has a more complicated
+             * masking setup. HCR and SCR bits not only affect interrupt
+             * routing but also change the behaviour of masking.
+             */
+            bool hcr, scr;
+
+            switch (excp_idx) {
+            case EXCP_FIQ:
+                /* If FIQs are routed to EL3 or EL2 then there are cases where
+                 * we override the CPSR.F in determining if the exception is
+                 * masked or not. If neither of these are set then we fall back
+                 * to the CPSR.F setting otherwise we further assess the state
+                 * below.
+                 */
+                hcr = (env->cp15.hcr_el2 & HCR_FMO);
+                scr = (env->cp15.scr_el3 & SCR_FIQ);
+
+                /* When EL3 is 32-bit, the SCR.FW bit controls whether the
+                 * CPSR.F bit masks FIQ interrupts when taken in non-secure
+                 * state. If SCR.FW is set then FIQs can be masked by CPSR.F
+                 * when non-secure but only when FIQs are only routed to EL3.
+                 */
+                scr = scr && !((env->cp15.scr_el3 & SCR_FW) && !hcr);
+                break;
+            case EXCP_IRQ:
+                /* When EL3 execution state is 32-bit, if HCR.IMO is set then
+                 * we may override the CPSR.I masking when in non-secure state.
+                 * The SCR.IRQ setting has already been taken into consideration
+                 * when setting the target EL, so it does not have a further
+                 * affect here.
+                 */
+                hcr = (env->cp15.hcr_el2 & HCR_IMO);
+                scr = false;
+                break;
+            default:
+                g_assert_not_reached();
+            }
+
+            if ((scr || hcr) && !secure) {
+                unmasked = 1;
+            }
+        }
     }
-    return NULL;
+
+    /* The PSTATE bits only mask the interrupt if we have not overriden the
+     * ability above.
+     */
+    return unmasked || pstate_unmasked;
 }
 
-#define cpu_exec cpu_arm_exec
-#define cpu_gen_code cpu_arm_gen_code
+#define cpu_init(cpu_model) CPU(cpu_arm_init(cpu_model))
+
 #define cpu_signal_handler cpu_arm_signal_handler
 #define cpu_list arm_cpu_list
 
-/* MMU modes definitions */
-#define MMU_MODE0_SUFFIX _user
-#define MMU_MODE1_SUFFIX _kernel
+/* ARM has the following "translation regimes" (as the ARM ARM calls them):
+ *
+ * If EL3 is 64-bit:
+ *  + NonSecure EL1 & 0 stage 1
+ *  + NonSecure EL1 & 0 stage 2
+ *  + NonSecure EL2
+ *  + Secure EL1 & EL0
+ *  + Secure EL3
+ * If EL3 is 32-bit:
+ *  + NonSecure PL1 & 0 stage 1
+ *  + NonSecure PL1 & 0 stage 2
+ *  + NonSecure PL2
+ *  + Secure PL0 & PL1
+ * (reminder: for 32 bit EL3, Secure PL1 is *EL3*, not EL1.)
+ *
+ * For QEMU, an mmu_idx is not quite the same as a translation regime because:
+ *  1. we need to split the "EL1 & 0" regimes into two mmu_idxes, because they
+ *     may differ in access permissions even if the VA->PA map is the same
+ *  2. we want to cache in our TLB the full VA->IPA->PA lookup for a stage 1+2
+ *     translation, which means that we have one mmu_idx that deals with two
+ *     concatenated translation regimes [this sort of combined s1+2 TLB is
+ *     architecturally permitted]
+ *  3. we don't need to allocate an mmu_idx to translations that we won't be
+ *     handling via the TLB. The only way to do a stage 1 translation without
+ *     the immediate stage 2 translation is via the ATS or AT system insns,
+ *     which can be slow-pathed and always do a page table walk.
+ *  4. we can also safely fold together the "32 bit EL3" and "64 bit EL3"
+ *     translation regimes, because they map reasonably well to each other
+ *     and they can't both be active at the same time.
+ * This gives us the following list of mmu_idx values:
+ *
+ * NS EL0 (aka NS PL0) stage 1+2
+ * NS EL1 (aka NS PL1) stage 1+2
+ * NS EL2 (aka NS PL2)
+ * S EL3 (aka S PL1)
+ * S EL0 (aka S PL0)
+ * S EL1 (not used if EL3 is 32 bit)
+ * NS EL0+1 stage 2
+ *
+ * (The last of these is an mmu_idx because we want to be able to use the TLB
+ * for the accesses done as part of a stage 1 page table walk, rather than
+ * having to walk the stage 2 page table over and over.)
+ *
+ * Our enumeration includes at the end some entries which are not "true"
+ * mmu_idx values in that they don't have corresponding TLBs and are only
+ * valid for doing slow path page table walks.
+ *
+ * The constant names here are patterned after the general style of the names
+ * of the AT/ATS operations.
+ * The values used are carefully arranged to make mmu_idx => EL lookup easy.
+ */
+typedef enum ARMMMUIdx {
+    ARMMMUIdx_S12NSE0 = 0,
+    ARMMMUIdx_S12NSE1 = 1,
+    ARMMMUIdx_S1E2 = 2,
+    ARMMMUIdx_S1E3 = 3,
+    ARMMMUIdx_S1SE0 = 4,
+    ARMMMUIdx_S1SE1 = 5,
+    ARMMMUIdx_S2NS = 6,
+    /* Indexes below here don't have TLBs and are used only for AT system
+     * instructions or for the first stage of an S12 page table walk.
+     */
+    ARMMMUIdx_S1NSE0 = 7,
+    ARMMMUIdx_S1NSE1 = 8,
+} ARMMMUIdx;
+
 #define MMU_USER_IDX 0
-static inline int cpu_mmu_index (CPUARMState *env)
+
+/* Return the exception level we're running at if this is our mmu_idx */
+static inline int arm_mmu_idx_to_el(ARMMMUIdx mmu_idx)
 {
-    return arm_current_el(env);
+    assert(mmu_idx < ARMMMUIdx_S2NS);
+    return mmu_idx & 3;
 }
 
-/* Return the Exception Level targeted by debug exceptions;
- * currently always EL1 since we don't implement EL2 or EL3.
- */
+/* Determine the current mmu_idx to use for normal loads/stores */
+static inline int cpu_mmu_index(CPUARMState *env, bool ifetch)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && arm_is_secure_below_el3(env)) {
+        return ARMMMUIdx_S1SE0 + el;
+    }
+    return el;
+}
+
+/* Indexes used when registering address spaces with cpu_address_space_init */
+typedef enum ARMASIdx {
+    ARMASIdx_NS = 0,
+    ARMASIdx_S = 1,
+} ARMASIdx;
+
+/* Return the Exception Level targeted by debug exceptions. */
 static inline int arm_debug_target_el(CPUARMState *env)
 {
-    return 1;
+    bool secure = arm_is_secure(env);
+    bool route_to_el2 = false;
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !secure) {
+        route_to_el2 = env->cp15.hcr_el2 & HCR_TGE ||
+                       env->cp15.mdcr_el2 & (1 << 8);
+    }
+
+    if (route_to_el2) {
+        return 2;
+    } else if (arm_feature(env, ARM_FEATURE_EL3) &&
+               !arm_el_is_aa64(env, 3) && secure) {
+        return 3;
+    } else {
+        return 1;
+    }
 }
 
 static inline bool aa64_generate_debug_exceptions(CPUARMState *env)
 {
+    if (arm_is_secure(env)) {
+        /* MDCR_EL3.SDD disables debug events from Secure state */
+        if (extract32(env->cp15.mdcr_el3, 16, 1) != 0
+            || arm_current_el(env) == 3) {
+            return false;
+        }
+    }
+
     if (arm_current_el(env) == arm_debug_target_el(env)) {
         if ((extract32(env->cp15.mdscr_el1, 13, 1) == 0)
             || (env->daif & PSTATE_D)) {
@@ -1336,10 +2034,42 @@
 
 static inline bool aa32_generate_debug_exceptions(CPUARMState *env)
 {
-    if (arm_current_el(env) == 0 && arm_el_is_aa64(env, 1)) {
+    int el = arm_current_el(env);
+
+    if (el == 0 && arm_el_is_aa64(env, 1)) {
         return aa64_generate_debug_exceptions(env);
     }
-    return arm_current_el(env) != 2;
+
+    if (arm_is_secure(env)) {
+        int spd;
+
+        if (el == 0 && (env->cp15.sder & 1)) {
+            /* SDER.SUIDEN means debug exceptions from Secure EL0
+             * are always enabled. Otherwise they are controlled by
+             * SDCR.SPD like those from other Secure ELs.
+             */
+            return true;
+        }
+
+        spd = extract32(env->cp15.mdcr_el3, 14, 2);
+        switch (spd) {
+        case 1:
+            /* SPD == 0b01 is reserved, but behaves as 0b00. */
+        case 0:
+            /* For 0b00 we return true if external secure invasive debug
+             * is enabled. On real hardware this is controlled by external
+             * signals to the core. QEMU always permits debug, and behaves
+             * as if DBGEN, SPIDEN, NIDEN and SPNIDEN are all tied high.
+             */
+            return true;
+        case 2:
+            return false;
+        case 3:
+            return true;
+        }
+    }
+
+    return el != 2;
 }
 
 /* Return true if debugging exceptions are currently enabled.
@@ -1375,13 +2105,71 @@
         && arm_generate_debug_exceptions(env);
 }
 
+static inline bool arm_sctlr_b(CPUARMState *env)
+{
+    return
+        /* We need not implement SCTLR.ITD in user-mode emulation, so
+         * let linux-user ignore the fact that it conflicts with SCTLR_B.
+         * This lets people run BE32 binaries with "-cpu any".
+         */
+#ifndef CONFIG_USER_ONLY
+        !arm_feature(env, ARM_FEATURE_V7) &&
+#endif
+        (env->cp15.sctlr_el[1] & SCTLR_B) != 0;
+}
+
+/* Return true if the processor is in big-endian mode. */
+static inline bool arm_cpu_data_is_big_endian(CPUARMState *env)
+{
+    int cur_el;
+
+    /* In 32bit endianness is determined by looking at CPSR's E bit */
+    if (!is_a64(env)) {
+        return
+#ifdef CONFIG_USER_ONLY
+            /* In system mode, BE32 is modelled in line with the
+             * architecture (as word-invariant big-endianness), where loads
+             * and stores are done little endian but from addresses which
+             * are adjusted by XORing with the appropriate constant. So the
+             * endianness to use for the raw data access is not affected by
+             * SCTLR.B.
+             * In user mode, however, we model BE32 as byte-invariant
+             * big-endianness (because user-only code cannot tell the
+             * difference), and so we need to use a data access endianness
+             * that depends on SCTLR.B.
+             */
+            arm_sctlr_b(env) ||
+#endif
+                ((env->uncached_cpsr & CPSR_E) ? 1 : 0);
+    }
+
+    cur_el = arm_current_el(env);
+
+    if (cur_el == 0) {
+        return (env->cp15.sctlr_el[1] & SCTLR_E0E) != 0;
+    }
+
+    return (env->cp15.sctlr_el[cur_el] & SCTLR_EE) != 0;
+}
+
 #include "exec/cpu-all.h"
 
 /* Bit usage in the TB flags field: bit 31 indicates whether we are
  * in 32 or 64 bit mode. The meaning of the other bits depends on that.
+ * We put flags which are shared between 32 and 64 bit mode at the top
+ * of the word, and flags which apply to only one mode at the bottom.
  */
 #define ARM_TBFLAG_AARCH64_STATE_SHIFT 31
 #define ARM_TBFLAG_AARCH64_STATE_MASK  (1U << ARM_TBFLAG_AARCH64_STATE_SHIFT)
+#define ARM_TBFLAG_MMUIDX_SHIFT 28
+#define ARM_TBFLAG_MMUIDX_MASK (0x7 << ARM_TBFLAG_MMUIDX_SHIFT)
+#define ARM_TBFLAG_SS_ACTIVE_SHIFT 27
+#define ARM_TBFLAG_SS_ACTIVE_MASK (1 << ARM_TBFLAG_SS_ACTIVE_SHIFT)
+#define ARM_TBFLAG_PSTATE_SS_SHIFT 26
+#define ARM_TBFLAG_PSTATE_SS_MASK (1 << ARM_TBFLAG_PSTATE_SS_SHIFT)
+/* Target EL if we take a floating-point-disabled exception */
+#define ARM_TBFLAG_FPEXC_EL_SHIFT 24
+#define ARM_TBFLAG_FPEXC_EL_MASK (0x3 << ARM_TBFLAG_FPEXC_EL_SHIFT)
 
 /* Bit usage when in AArch32 state: */
 #define ARM_TBFLAG_THUMB_SHIFT      0
@@ -1390,154 +2178,204 @@
 #define ARM_TBFLAG_VECLEN_MASK      (0x7 << ARM_TBFLAG_VECLEN_SHIFT)
 #define ARM_TBFLAG_VECSTRIDE_SHIFT  4
 #define ARM_TBFLAG_VECSTRIDE_MASK   (0x3 << ARM_TBFLAG_VECSTRIDE_SHIFT)
-#define ARM_TBFLAG_PRIV_SHIFT       6
-#define ARM_TBFLAG_PRIV_MASK        (1 << ARM_TBFLAG_PRIV_SHIFT)
 #define ARM_TBFLAG_VFPEN_SHIFT      7
 #define ARM_TBFLAG_VFPEN_MASK       (1 << ARM_TBFLAG_VFPEN_SHIFT)
 #define ARM_TBFLAG_CONDEXEC_SHIFT   8
 #define ARM_TBFLAG_CONDEXEC_MASK    (0xff << ARM_TBFLAG_CONDEXEC_SHIFT)
-#define ARM_TBFLAG_BSWAP_CODE_SHIFT 16
-#define ARM_TBFLAG_BSWAP_CODE_MASK  (1 << ARM_TBFLAG_BSWAP_CODE_SHIFT)
-#define ARM_TBFLAG_CPACR_FPEN_SHIFT 17
-#define ARM_TBFLAG_CPACR_FPEN_MASK  (1 << ARM_TBFLAG_CPACR_FPEN_SHIFT)
-#define ARM_TBFLAG_SS_ACTIVE_SHIFT 18
-#define ARM_TBFLAG_SS_ACTIVE_MASK (1 << ARM_TBFLAG_SS_ACTIVE_SHIFT)
-#define ARM_TBFLAG_PSTATE_SS_SHIFT 19
-#define ARM_TBFLAG_PSTATE_SS_MASK (1 << ARM_TBFLAG_PSTATE_SS_SHIFT)
+#define ARM_TBFLAG_SCTLR_B_SHIFT    16
+#define ARM_TBFLAG_SCTLR_B_MASK     (1 << ARM_TBFLAG_SCTLR_B_SHIFT)
 /* We store the bottom two bits of the CPAR as TB flags and handle
  * checks on the other bits at runtime
  */
-#define ARM_TBFLAG_XSCALE_CPAR_SHIFT 20
+#define ARM_TBFLAG_XSCALE_CPAR_SHIFT 17
 #define ARM_TBFLAG_XSCALE_CPAR_MASK (3 << ARM_TBFLAG_XSCALE_CPAR_SHIFT)
+/* Indicates whether cp register reads and writes by guest code should access
+ * the secure or nonsecure bank of banked registers; note that this is not
+ * the same thing as the current security state of the processor!
+ */
+#define ARM_TBFLAG_NS_SHIFT         19
+#define ARM_TBFLAG_NS_MASK          (1 << ARM_TBFLAG_NS_SHIFT)
+#define ARM_TBFLAG_BE_DATA_SHIFT    20
+#define ARM_TBFLAG_BE_DATA_MASK     (1 << ARM_TBFLAG_BE_DATA_SHIFT)
 
-/* Bit usage when in AArch64 state */
-#define ARM_TBFLAG_AA64_EL_SHIFT    0
-#define ARM_TBFLAG_AA64_EL_MASK     (0x3 << ARM_TBFLAG_AA64_EL_SHIFT)
-#define ARM_TBFLAG_AA64_FPEN_SHIFT  2
-#define ARM_TBFLAG_AA64_FPEN_MASK   (1 << ARM_TBFLAG_AA64_FPEN_SHIFT)
-#define ARM_TBFLAG_AA64_SS_ACTIVE_SHIFT 3
-#define ARM_TBFLAG_AA64_SS_ACTIVE_MASK (1 << ARM_TBFLAG_AA64_SS_ACTIVE_SHIFT)
-#define ARM_TBFLAG_AA64_PSTATE_SS_SHIFT 4
-#define ARM_TBFLAG_AA64_PSTATE_SS_MASK (1 << ARM_TBFLAG_AA64_PSTATE_SS_SHIFT)
+/* Bit usage when in AArch64 state: currently we have no A64 specific bits */
 
 /* some convenience accessor macros */
 #define ARM_TBFLAG_AARCH64_STATE(F) \
     (((F) & ARM_TBFLAG_AARCH64_STATE_MASK) >> ARM_TBFLAG_AARCH64_STATE_SHIFT)
+#define ARM_TBFLAG_MMUIDX(F) \
+    (((F) & ARM_TBFLAG_MMUIDX_MASK) >> ARM_TBFLAG_MMUIDX_SHIFT)
+#define ARM_TBFLAG_SS_ACTIVE(F) \
+    (((F) & ARM_TBFLAG_SS_ACTIVE_MASK) >> ARM_TBFLAG_SS_ACTIVE_SHIFT)
+#define ARM_TBFLAG_PSTATE_SS(F) \
+    (((F) & ARM_TBFLAG_PSTATE_SS_MASK) >> ARM_TBFLAG_PSTATE_SS_SHIFT)
+#define ARM_TBFLAG_FPEXC_EL(F) \
+    (((F) & ARM_TBFLAG_FPEXC_EL_MASK) >> ARM_TBFLAG_FPEXC_EL_SHIFT)
 #define ARM_TBFLAG_THUMB(F) \
     (((F) & ARM_TBFLAG_THUMB_MASK) >> ARM_TBFLAG_THUMB_SHIFT)
 #define ARM_TBFLAG_VECLEN(F) \
     (((F) & ARM_TBFLAG_VECLEN_MASK) >> ARM_TBFLAG_VECLEN_SHIFT)
 #define ARM_TBFLAG_VECSTRIDE(F) \
     (((F) & ARM_TBFLAG_VECSTRIDE_MASK) >> ARM_TBFLAG_VECSTRIDE_SHIFT)
-#define ARM_TBFLAG_PRIV(F) \
-    (((F) & ARM_TBFLAG_PRIV_MASK) >> ARM_TBFLAG_PRIV_SHIFT)
 #define ARM_TBFLAG_VFPEN(F) \
     (((F) & ARM_TBFLAG_VFPEN_MASK) >> ARM_TBFLAG_VFPEN_SHIFT)
 #define ARM_TBFLAG_CONDEXEC(F) \
     (((F) & ARM_TBFLAG_CONDEXEC_MASK) >> ARM_TBFLAG_CONDEXEC_SHIFT)
-#define ARM_TBFLAG_BSWAP_CODE(F) \
-    (((F) & ARM_TBFLAG_BSWAP_CODE_MASK) >> ARM_TBFLAG_BSWAP_CODE_SHIFT)
-#define ARM_TBFLAG_CPACR_FPEN(F) \
-    (((F) & ARM_TBFLAG_CPACR_FPEN_MASK) >> ARM_TBFLAG_CPACR_FPEN_SHIFT)
-#define ARM_TBFLAG_SS_ACTIVE(F) \
-    (((F) & ARM_TBFLAG_SS_ACTIVE_MASK) >> ARM_TBFLAG_SS_ACTIVE_SHIFT)
-#define ARM_TBFLAG_PSTATE_SS(F) \
-    (((F) & ARM_TBFLAG_PSTATE_SS_MASK) >> ARM_TBFLAG_PSTATE_SS_SHIFT)
+#define ARM_TBFLAG_SCTLR_B(F) \
+    (((F) & ARM_TBFLAG_SCTLR_B_MASK) >> ARM_TBFLAG_SCTLR_B_SHIFT)
 #define ARM_TBFLAG_XSCALE_CPAR(F) \
     (((F) & ARM_TBFLAG_XSCALE_CPAR_MASK) >> ARM_TBFLAG_XSCALE_CPAR_SHIFT)
-#define ARM_TBFLAG_AA64_EL(F) \
-    (((F) & ARM_TBFLAG_AA64_EL_MASK) >> ARM_TBFLAG_AA64_EL_SHIFT)
-#define ARM_TBFLAG_AA64_FPEN(F) \
-    (((F) & ARM_TBFLAG_AA64_FPEN_MASK) >> ARM_TBFLAG_AA64_FPEN_SHIFT)
-#define ARM_TBFLAG_AA64_SS_ACTIVE(F) \
-    (((F) & ARM_TBFLAG_AA64_SS_ACTIVE_MASK) >> ARM_TBFLAG_AA64_SS_ACTIVE_SHIFT)
-#define ARM_TBFLAG_AA64_PSTATE_SS(F) \
-    (((F) & ARM_TBFLAG_AA64_PSTATE_SS_MASK) >> ARM_TBFLAG_AA64_PSTATE_SS_SHIFT)
+#define ARM_TBFLAG_NS(F) \
+    (((F) & ARM_TBFLAG_NS_MASK) >> ARM_TBFLAG_NS_SHIFT)
+#define ARM_TBFLAG_BE_DATA(F) \
+    (((F) & ARM_TBFLAG_BE_DATA_MASK) >> ARM_TBFLAG_BE_DATA_SHIFT)
 
-static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+static inline bool bswap_code(bool sctlr_b)
+{
+#ifdef CONFIG_USER_ONLY
+    /* BE8 (SCTLR.B = 0, TARGET_WORDS_BIGENDIAN = 1) is mixed endian.
+     * The invalid combination SCTLR.B=1/CPSR.E=1/TARGET_WORDS_BIGENDIAN=0
+     * would also end up as a mixed-endian mode with BE code, LE data.
+     */
+    return
+#ifdef TARGET_WORDS_BIGENDIAN
+        1 ^
+#endif
+        sctlr_b;
+#else
+    /* All code access in ARM is little endian, and there are no loaders
+     * doing swaps that need to be reversed
+     */
+    return 0;
+#endif
+}
+
+/* Return the exception level to which FP-disabled exceptions should
+ * be taken, or 0 if FP is enabled.
+ */
+static inline int fp_exception_el(CPUARMState *env)
 {
     int fpen;
+    int cur_el = arm_current_el(env);
 
-    if (arm_feature(env, ARM_FEATURE_V6)) {
-        fpen = extract32(env->cp15.c1_coproc, 20, 2);
-    } else {
-        /* CPACR doesn't exist before v6, so VFP is always accessible */
-        fpen = 3;
+    /* CPACR and the CPTR registers don't exist before v6, so FP is
+     * always accessible
+     */
+    if (!arm_feature(env, ARM_FEATURE_V6)) {
+        return 0;
     }
 
+    /* The CPACR controls traps to EL1, or PL1 if we're 32 bit:
+     * 0, 2 : trap EL0 and EL1/PL1 accesses
+     * 1    : trap only EL0 accesses
+     * 3    : trap no accesses
+     */
+    fpen = extract32(env->cp15.cpacr_el1, 20, 2);
+    switch (fpen) {
+    case 0:
+    case 2:
+        if (cur_el == 0 || cur_el == 1) {
+            /* Trap to PL1, which might be EL1 or EL3 */
+            if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) {
+                return 3;
+            }
+            return 1;
+        }
+        if (cur_el == 3 && !is_a64(env)) {
+            /* Secure PL1 running at EL3 */
+            return 3;
+        }
+        break;
+    case 1:
+        if (cur_el == 0) {
+            return 1;
+        }
+        break;
+    case 3:
+        break;
+    }
+
+    /* For the CPTR registers we don't need to guard with an ARM_FEATURE
+     * check because zero bits in the registers mean "don't trap".
+     */
+
+    /* CPTR_EL2 : present in v7VE or v8 */
+    if (cur_el <= 2 && extract32(env->cp15.cptr_el[2], 10, 1)
+        && !arm_is_secure_below_el3(env)) {
+        /* Trap FP ops at EL2, NS-EL1 or NS-EL0 to EL2 */
+        return 2;
+    }
+
+    /* CPTR_EL3 : present in v8 */
+    if (extract32(env->cp15.cptr_el[3], 10, 1)) {
+        /* Trap all FP ops to EL3 */
+        return 3;
+    }
+
+    return 0;
+}
+
+#ifdef CONFIG_USER_ONLY
+static inline bool arm_cpu_bswap_data(CPUARMState *env)
+{
+    return
+#ifdef TARGET_WORDS_BIGENDIAN
+       1 ^
+#endif
+       arm_cpu_data_is_big_endian(env);
+}
+#endif
+
+static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
+                                        target_ulong *cs_base, uint32_t *flags)
+{
     if (is_a64(env)) {
         *pc = env->pc;
-        *flags = ARM_TBFLAG_AARCH64_STATE_MASK
-            | (arm_current_el(env) << ARM_TBFLAG_AA64_EL_SHIFT);
-        if (fpen == 3 || (fpen == 1 && arm_current_el(env) != 0)) {
-            *flags |= ARM_TBFLAG_AA64_FPEN_MASK;
-        }
-        /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
-         * states defined in the ARM ARM for software singlestep:
-         *  SS_ACTIVE   PSTATE.SS   State
-         *     0            x       Inactive (the TB flag for SS is always 0)
-         *     1            0       Active-pending
-         *     1            1       Active-not-pending
-         */
-        if (arm_singlestep_active(env)) {
-            *flags |= ARM_TBFLAG_AA64_SS_ACTIVE_MASK;
-            if (env->pstate & PSTATE_SS) {
-                *flags |= ARM_TBFLAG_AA64_PSTATE_SS_MASK;
-            }
-        }
+        *flags = ARM_TBFLAG_AARCH64_STATE_MASK;
     } else {
-        int privmode;
         *pc = env->regs[15];
         *flags = (env->thumb << ARM_TBFLAG_THUMB_SHIFT)
             | (env->vfp.vec_len << ARM_TBFLAG_VECLEN_SHIFT)
             | (env->vfp.vec_stride << ARM_TBFLAG_VECSTRIDE_SHIFT)
             | (env->condexec_bits << ARM_TBFLAG_CONDEXEC_SHIFT)
-            | (env->bswap_code << ARM_TBFLAG_BSWAP_CODE_SHIFT);
-        if (arm_feature(env, ARM_FEATURE_M)) {
-            privmode = !((env->v7m.exception == 0) && (env->v7m.control & 1));
-        } else {
-            privmode = (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR;
-        }
-        if (privmode) {
-            *flags |= ARM_TBFLAG_PRIV_MASK;
+            | (arm_sctlr_b(env) << ARM_TBFLAG_SCTLR_B_SHIFT);
+        if (!(access_secure_reg(env))) {
+            *flags |= ARM_TBFLAG_NS_MASK;
         }
         if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30)
             || arm_el_is_aa64(env, 1)) {
             *flags |= ARM_TBFLAG_VFPEN_MASK;
         }
-        if (fpen == 3 || (fpen == 1 && arm_current_el(env) != 0)) {
-            *flags |= ARM_TBFLAG_CPACR_FPEN_MASK;
-        }
-        /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
-         * states defined in the ARM ARM for software singlestep:
-         *  SS_ACTIVE   PSTATE.SS   State
-         *     0            x       Inactive (the TB flag for SS is always 0)
-         *     1            0       Active-pending
-         *     1            1       Active-not-pending
-         */
-        if (arm_singlestep_active(env)) {
-            *flags |= ARM_TBFLAG_SS_ACTIVE_MASK;
-            if (env->uncached_cpsr & PSTATE_SS) {
-                *flags |= ARM_TBFLAG_PSTATE_SS_MASK;
-            }
-        }
         *flags |= (extract32(env->cp15.c15_cpar, 0, 2)
                    << ARM_TBFLAG_XSCALE_CPAR_SHIFT);
     }
 
-    *cs_base = 0;
-}
-
-#include "exec/exec-all.h"
-
-static inline void cpu_pc_from_tb(CPUARMState *env, TranslationBlock *tb)
-{
-    if (ARM_TBFLAG_AARCH64_STATE(tb->flags)) {
-        env->pc = tb->pc;
-    } else {
-        env->regs[15] = tb->pc;
+    *flags |= (cpu_mmu_index(env, false) << ARM_TBFLAG_MMUIDX_SHIFT);
+    /* The SS_ACTIVE and PSTATE_SS bits correspond to the state machine
+     * states defined in the ARM ARM for software singlestep:
+     *  SS_ACTIVE   PSTATE.SS   State
+     *     0            x       Inactive (the TB flag for SS is always 0)
+     *     1            0       Active-pending
+     *     1            1       Active-not-pending
+     */
+    if (arm_singlestep_active(env)) {
+        *flags |= ARM_TBFLAG_SS_ACTIVE_MASK;
+        if (is_a64(env)) {
+            if (env->pstate & PSTATE_SS) {
+                *flags |= ARM_TBFLAG_PSTATE_SS_MASK;
+            }
+        } else {
+            if (env->uncached_cpsr & PSTATE_SS) {
+                *flags |= ARM_TBFLAG_PSTATE_SS_MASK;
+            }
+        }
     }
+    if (arm_cpu_data_is_big_endian(env)) {
+        *flags |= ARM_TBFLAG_BE_DATA_MASK;
+    }
+    *flags |= fp_exception_el(env) << ARM_TBFLAG_FPEXC_EL_SHIFT;
+
+    *cs_base = 0;
 }
 
 enum {
@@ -1546,4 +2384,45 @@
     QEMU_PSCI_CONDUIT_HVC = 2,
 };
 
+#ifndef CONFIG_USER_ONLY
+/* Return the address space index to use for a memory access */
+static inline int arm_asidx_from_attrs(CPUState *cs, MemTxAttrs attrs)
+{
+    return attrs.secure ? ARMASIdx_S : ARMASIdx_NS;
+}
+
+/* Return the AddressSpace to use for a memory access
+ * (which depends on whether the access is S or NS, and whether
+ * the board gave us a separate AddressSpace for S accesses).
+ */
+static inline AddressSpace *arm_addressspace(CPUState *cs, MemTxAttrs attrs)
+{
+    return cpu_get_address_space(cs, arm_asidx_from_attrs(cs, attrs));
+}
+#endif
+
+/**
+ * arm_register_el_change_hook:
+ * Register a hook function which will be called back whenever this
+ * CPU changes exception level or mode. The hook function will be
+ * passed a pointer to the ARMCPU and the opaque data pointer passed
+ * to this function when the hook was registered.
+ *
+ * Note that we currently only support registering a single hook function,
+ * and will assert if this function is called twice.
+ * This facility is intended for the use of the GICv3 emulation.
+ */
+void arm_register_el_change_hook(ARMCPU *cpu, ARMELChangeHook *hook,
+                                 void *opaque);
+
+/**
+ * arm_get_el_change_hook_opaque:
+ * Return the opaque data that will be used by the el_change_hook
+ * for this CPU.
+ */
+static inline void *arm_get_el_change_hook_opaque(ARMCPU *cpu)
+{
+    return cpu->el_change_hook_opaque;
+}
+
 #endif
diff --git a/target-arm/cpu64.c b/target-arm/cpu64.c
index e1141c6..1635deb 100644
--- a/target-arm/cpu64.c
+++ b/target-arm/cpu64.c
@@ -18,6 +18,8 @@
  * <http://www.gnu.org/licenses/gpl-2.0.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #if !defined(CONFIG_USER_ONLY)
@@ -32,23 +34,28 @@
     env->features |= 1ULL << feature;
 }
 
+static inline void unset_feature(CPUARMState *env, int feature)
+{
+    env->features &= ~(1ULL << feature);
+}
+
 #ifndef CONFIG_USER_ONLY
-static uint64_t a57_l2ctlr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t a57_a53_l2ctlr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     /* Number of processors is in [25:24]; otherwise we RAZ */
     return (smp_cpus - 1) << 24;
 }
 #endif
 
-static const ARMCPRegInfo cortexa57_cp_reginfo[] = {
+static const ARMCPRegInfo cortex_a57_a53_cp_reginfo[] = {
 #ifndef CONFIG_USER_ONLY
     { .name = "L2CTLR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 1, .crn = 11, .crm = 0, .opc2 = 2,
-      .access = PL1_RW, .readfn = a57_l2ctlr_read,
+      .access = PL1_RW, .readfn = a57_a53_l2ctlr_read,
       .writefn = arm_cp_write_ignore },
     { .name = "L2CTLR",
       .cp = 15, .opc1 = 1, .crn = 9, .crm = 0, .opc2 = 2,
-      .access = PL1_RW, .readfn = a57_l2ctlr_read,
+      .access = PL1_RW, .readfn = a57_a53_l2ctlr_read,
       .writefn = arm_cp_write_ignore },
 #endif
     { .name = "L2ECTLR_EL1", .state = ARM_CP_STATE_AA64,
@@ -91,6 +98,7 @@
 {
     ARMCPU *cpu = ARM_CPU(obj);
 
+    cpu->dtb_compatible = "arm,cortex-a57";
     set_feature(&cpu->env, ARM_FEATURE_V8);
     set_feature(&cpu->env, ARM_FEATURE_VFP4);
     set_feature(&cpu->env, ARM_FEATURE_NEON);
@@ -102,8 +110,10 @@
     set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
     set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
     set_feature(&cpu->env, ARM_FEATURE_CRC);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
     cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A57;
     cpu->midr = 0x411fd070;
+    cpu->revidr = 0x00000000;
     cpu->reset_fpsid = 0x41034070;
     cpu->mvfr0 = 0x10110222;
     cpu->mvfr1 = 0x12111111;
@@ -126,6 +136,8 @@
     cpu->id_isar5 = 0x00011121;
     cpu->id_aa64pfr0 = 0x00002222;
     cpu->id_aa64dfr0 = 0x10305106;
+    cpu->pmceid0 = 0x00000000;
+    cpu->pmceid1 = 0x00000000;
     cpu->id_aa64isar0 = 0x00011120;
     cpu->id_aa64mmfr0 = 0x00001124;
     cpu->dbgdidr = 0x3516d000;
@@ -134,7 +146,60 @@
     cpu->ccsidr[1] = 0x201fe012; /* 48KB L1 icache */
     cpu->ccsidr[2] = 0x70ffe07a; /* 2048KB L2 cache */
     cpu->dcz_blocksize = 4; /* 64 bytes */
-    define_arm_cp_regs(cpu, cortexa57_cp_reginfo);
+    define_arm_cp_regs(cpu, cortex_a57_a53_cp_reginfo);
+}
+
+static void aarch64_a53_initfn(Object *obj)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+
+    cpu->dtb_compatible = "arm,cortex-a53";
+    set_feature(&cpu->env, ARM_FEATURE_V8);
+    set_feature(&cpu->env, ARM_FEATURE_VFP4);
+    set_feature(&cpu->env, ARM_FEATURE_NEON);
+    set_feature(&cpu->env, ARM_FEATURE_GENERIC_TIMER);
+    set_feature(&cpu->env, ARM_FEATURE_AARCH64);
+    set_feature(&cpu->env, ARM_FEATURE_CBAR_RO);
+    set_feature(&cpu->env, ARM_FEATURE_V8_AES);
+    set_feature(&cpu->env, ARM_FEATURE_V8_SHA1);
+    set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
+    set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
+    set_feature(&cpu->env, ARM_FEATURE_CRC);
+    set_feature(&cpu->env, ARM_FEATURE_EL3);
+    cpu->kvm_target = QEMU_KVM_ARM_TARGET_CORTEX_A53;
+    cpu->midr = 0x410fd034;
+    cpu->revidr = 0x00000000;
+    cpu->reset_fpsid = 0x41034070;
+    cpu->mvfr0 = 0x10110222;
+    cpu->mvfr1 = 0x12111111;
+    cpu->mvfr2 = 0x00000043;
+    cpu->ctr = 0x84448004; /* L1Ip = VIPT */
+    cpu->reset_sctlr = 0x00c50838;
+    cpu->id_pfr0 = 0x00000131;
+    cpu->id_pfr1 = 0x00011011;
+    cpu->id_dfr0 = 0x03010066;
+    cpu->id_afr0 = 0x00000000;
+    cpu->id_mmfr0 = 0x10101105;
+    cpu->id_mmfr1 = 0x40000000;
+    cpu->id_mmfr2 = 0x01260000;
+    cpu->id_mmfr3 = 0x02102211;
+    cpu->id_isar0 = 0x02101110;
+    cpu->id_isar1 = 0x13112111;
+    cpu->id_isar2 = 0x21232042;
+    cpu->id_isar3 = 0x01112131;
+    cpu->id_isar4 = 0x00011142;
+    cpu->id_isar5 = 0x00011121;
+    cpu->id_aa64pfr0 = 0x00002222;
+    cpu->id_aa64dfr0 = 0x10305106;
+    cpu->id_aa64isar0 = 0x00011120;
+    cpu->id_aa64mmfr0 = 0x00001122; /* 40 bit physical addr */
+    cpu->dbgdidr = 0x3516d000;
+    cpu->clidr = 0x0a200023;
+    cpu->ccsidr[0] = 0x700fe01a; /* 32KB L1 dcache */
+    cpu->ccsidr[1] = 0x201fe00a; /* 32KB L1 icache */
+    cpu->ccsidr[2] = 0x707fe07a; /* 1024KB L2 cache */
+    cpu->dcz_blocksize = 4; /* 64 bytes */
+    define_arm_cp_regs(cpu, cortex_a57_a53_cp_reginfo);
 }
 
 #ifdef CONFIG_USER_ONLY
@@ -164,14 +229,49 @@
 
 static const ARMCPUInfo aarch64_cpus[] = {
     { .name = "cortex-a57",         .initfn = aarch64_a57_initfn },
+    { .name = "cortex-a53",         .initfn = aarch64_a53_initfn },
 #ifdef CONFIG_USER_ONLY
     { .name = "any",         .initfn = aarch64_any_initfn },
 #endif
     { .name = NULL }
 };
 
+static bool aarch64_cpu_get_aarch64(Object *obj, Error **errp)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+
+    return arm_feature(&cpu->env, ARM_FEATURE_AARCH64);
+}
+
+static void aarch64_cpu_set_aarch64(Object *obj, bool value, Error **errp)
+{
+    ARMCPU *cpu = ARM_CPU(obj);
+
+    /* At this time, this property is only allowed if KVM is enabled.  This
+     * restriction allows us to avoid fixing up functionality that assumes a
+     * uniform execution state like do_interrupt.
+     */
+    if (!kvm_enabled()) {
+        error_setg(errp, "'aarch64' feature cannot be disabled "
+                         "unless KVM is enabled");
+        return;
+    }
+
+    if (value == false) {
+        unset_feature(&cpu->env, ARM_FEATURE_AARCH64);
+    } else {
+        set_feature(&cpu->env, ARM_FEATURE_AARCH64);
+    }
+}
+
 static void aarch64_cpu_initfn(Object *obj)
 {
+    object_property_add_bool(obj, "aarch64", aarch64_cpu_get_aarch64,
+                             aarch64_cpu_set_aarch64, NULL);
+    object_property_set_description(obj, "aarch64",
+                                    "Set on/off to enable/disable aarch64 "
+                                    "execution state ",
+                                    NULL);
 }
 
 static void aarch64_cpu_finalizefn(Object *obj)
@@ -201,9 +301,6 @@
 {
     CPUClass *cc = CPU_CLASS(oc);
 
-#if !defined(CONFIG_USER_ONLY)
-    cc->do_interrupt = aarch64_cpu_do_interrupt;
-#endif
     cc->cpu_exec_interrupt = arm_cpu_exec_interrupt;
     cc->set_pc = aarch64_cpu_set_pc;
     cc->gdb_read_register = aarch64_cpu_gdb_read_register;
diff --git a/target-arm/crypto_helper.c b/target-arm/crypto_helper.c
index dd60d0b..3b6df3f 100644
--- a/target-arm/crypto_helper.c
+++ b/target-arm/crypto_helper.c
@@ -9,12 +9,12 @@
  * version 2 of the License, or (at your option) any later version.
  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/helper-proto.h"
-#include "qemu/aes.h"
+#include "crypto/aes.h"
 
 union CRYPTO_STATE {
     uint8_t    bytes[16];
@@ -22,6 +22,14 @@
     uint64_t   l[2];
 };
 
+#ifdef HOST_WORDS_BIGENDIAN
+#define CR_ST_BYTE(state, i)   (state.bytes[(15 - (i)) ^ 8])
+#define CR_ST_WORD(state, i)   (state.words[(3 - (i)) ^ 2])
+#else
+#define CR_ST_BYTE(state, i)   (state.bytes[i])
+#define CR_ST_WORD(state, i)   (state.words[i])
+#endif
+
 void HELPER(crypto_aese)(CPUARMState *env, uint32_t rd, uint32_t rm,
                          uint32_t decrypt)
 {
@@ -46,7 +54,7 @@
 
     /* combine ShiftRows operation and sbox substitution */
     for (i = 0; i < 16; i++) {
-        st.bytes[i] = sbox[decrypt][rk.bytes[shift[decrypt][i]]];
+        CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])];
     }
 
     env->vfp.regs[rd] = make_float64(st.l[0]);
@@ -198,11 +206,11 @@
     assert(decrypt < 2);
 
     for (i = 0; i < 16; i += 4) {
-        st.words[i >> 2] = cpu_to_le32(
-            mc[decrypt][st.bytes[i]] ^
-            rol32(mc[decrypt][st.bytes[i + 1]], 8) ^
-            rol32(mc[decrypt][st.bytes[i + 2]], 16) ^
-            rol32(mc[decrypt][st.bytes[i + 3]], 24));
+        CR_ST_WORD(st, i >> 2) =
+            mc[decrypt][CR_ST_BYTE(st, i)] ^
+            rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^
+            rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^
+            rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24);
     }
 
     env->vfp.regs[rd] = make_float64(st.l[0]);
@@ -255,24 +263,25 @@
 
             switch (op) {
             case 0: /* sha1c */
-                t = cho(d.words[1], d.words[2], d.words[3]);
+                t = cho(CR_ST_WORD(d, 1), CR_ST_WORD(d, 2), CR_ST_WORD(d, 3));
                 break;
             case 1: /* sha1p */
-                t = par(d.words[1], d.words[2], d.words[3]);
+                t = par(CR_ST_WORD(d, 1), CR_ST_WORD(d, 2), CR_ST_WORD(d, 3));
                 break;
             case 2: /* sha1m */
-                t = maj(d.words[1], d.words[2], d.words[3]);
+                t = maj(CR_ST_WORD(d, 1), CR_ST_WORD(d, 2), CR_ST_WORD(d, 3));
                 break;
             default:
                 g_assert_not_reached();
             }
-            t += rol32(d.words[0], 5) + n.words[0] + m.words[i];
+            t += rol32(CR_ST_WORD(d, 0), 5) + CR_ST_WORD(n, 0)
+                 + CR_ST_WORD(m, i);
 
-            n.words[0] = d.words[3];
-            d.words[3] = d.words[2];
-            d.words[2] = ror32(d.words[1], 2);
-            d.words[1] = d.words[0];
-            d.words[0] = t;
+            CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3);
+            CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
+            CR_ST_WORD(d, 2) = ror32(CR_ST_WORD(d, 1), 2);
+            CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
+            CR_ST_WORD(d, 0) = t;
         }
     }
     env->vfp.regs[rd] = make_float64(d.l[0]);
@@ -286,8 +295,8 @@
         float64_val(env->vfp.regs[rm + 1])
     } };
 
-    m.words[0] = ror32(m.words[0], 2);
-    m.words[1] = m.words[2] = m.words[3] = 0;
+    CR_ST_WORD(m, 0) = ror32(CR_ST_WORD(m, 0), 2);
+    CR_ST_WORD(m, 1) = CR_ST_WORD(m, 2) = CR_ST_WORD(m, 3) = 0;
 
     env->vfp.regs[rd] = make_float64(m.l[0]);
     env->vfp.regs[rd + 1] = make_float64(m.l[1]);
@@ -304,10 +313,10 @@
         float64_val(env->vfp.regs[rm + 1])
     } };
 
-    d.words[0] = rol32(d.words[0] ^ m.words[1], 1);
-    d.words[1] = rol32(d.words[1] ^ m.words[2], 1);
-    d.words[2] = rol32(d.words[2] ^ m.words[3], 1);
-    d.words[3] = rol32(d.words[3] ^ d.words[0], 1);
+    CR_ST_WORD(d, 0) = rol32(CR_ST_WORD(d, 0) ^ CR_ST_WORD(m, 1), 1);
+    CR_ST_WORD(d, 1) = rol32(CR_ST_WORD(d, 1) ^ CR_ST_WORD(m, 2), 1);
+    CR_ST_WORD(d, 2) = rol32(CR_ST_WORD(d, 2) ^ CR_ST_WORD(m, 3), 1);
+    CR_ST_WORD(d, 3) = rol32(CR_ST_WORD(d, 3) ^ CR_ST_WORD(d, 0), 1);
 
     env->vfp.regs[rd] = make_float64(d.l[0]);
     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
@@ -356,20 +365,22 @@
     int i;
 
     for (i = 0; i < 4; i++) {
-        uint32_t t = cho(n.words[0], n.words[1], n.words[2]) + n.words[3]
-                     + S1(n.words[0]) + m.words[i];
+        uint32_t t = cho(CR_ST_WORD(n, 0), CR_ST_WORD(n, 1), CR_ST_WORD(n, 2))
+                     + CR_ST_WORD(n, 3) + S1(CR_ST_WORD(n, 0))
+                     + CR_ST_WORD(m, i);
 
-        n.words[3] = n.words[2];
-        n.words[2] = n.words[1];
-        n.words[1] = n.words[0];
-        n.words[0] = d.words[3] + t;
+        CR_ST_WORD(n, 3) = CR_ST_WORD(n, 2);
+        CR_ST_WORD(n, 2) = CR_ST_WORD(n, 1);
+        CR_ST_WORD(n, 1) = CR_ST_WORD(n, 0);
+        CR_ST_WORD(n, 0) = CR_ST_WORD(d, 3) + t;
 
-        t += maj(d.words[0], d.words[1], d.words[2]) + S0(d.words[0]);
+        t += maj(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
+             + S0(CR_ST_WORD(d, 0));
 
-        d.words[3] = d.words[2];
-        d.words[2] = d.words[1];
-        d.words[1] = d.words[0];
-        d.words[0] = t;
+        CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
+        CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
+        CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
+        CR_ST_WORD(d, 0) = t;
     }
 
     env->vfp.regs[rd] = make_float64(d.l[0]);
@@ -394,13 +405,14 @@
     int i;
 
     for (i = 0; i < 4; i++) {
-        uint32_t t = cho(d.words[0], d.words[1], d.words[2]) + d.words[3]
-                     + S1(d.words[0]) + m.words[i];
+        uint32_t t = cho(CR_ST_WORD(d, 0), CR_ST_WORD(d, 1), CR_ST_WORD(d, 2))
+                     + CR_ST_WORD(d, 3) + S1(CR_ST_WORD(d, 0))
+                     + CR_ST_WORD(m, i);
 
-        d.words[3] = d.words[2];
-        d.words[2] = d.words[1];
-        d.words[1] = d.words[0];
-        d.words[0] = n.words[3 - i] + t;
+        CR_ST_WORD(d, 3) = CR_ST_WORD(d, 2);
+        CR_ST_WORD(d, 2) = CR_ST_WORD(d, 1);
+        CR_ST_WORD(d, 1) = CR_ST_WORD(d, 0);
+        CR_ST_WORD(d, 0) = CR_ST_WORD(n, 3 - i) + t;
     }
 
     env->vfp.regs[rd] = make_float64(d.l[0]);
@@ -418,10 +430,10 @@
         float64_val(env->vfp.regs[rm + 1])
     } };
 
-    d.words[0] += s0(d.words[1]);
-    d.words[1] += s0(d.words[2]);
-    d.words[2] += s0(d.words[3]);
-    d.words[3] += s0(m.words[0]);
+    CR_ST_WORD(d, 0) += s0(CR_ST_WORD(d, 1));
+    CR_ST_WORD(d, 1) += s0(CR_ST_WORD(d, 2));
+    CR_ST_WORD(d, 2) += s0(CR_ST_WORD(d, 3));
+    CR_ST_WORD(d, 3) += s0(CR_ST_WORD(m, 0));
 
     env->vfp.regs[rd] = make_float64(d.l[0]);
     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
@@ -443,10 +455,10 @@
         float64_val(env->vfp.regs[rm + 1])
     } };
 
-    d.words[0] += s1(m.words[2]) + n.words[1];
-    d.words[1] += s1(m.words[3]) + n.words[2];
-    d.words[2] += s1(d.words[0]) + n.words[3];
-    d.words[3] += s1(d.words[1]) + m.words[0];
+    CR_ST_WORD(d, 0) += s1(CR_ST_WORD(m, 2)) + CR_ST_WORD(n, 1);
+    CR_ST_WORD(d, 1) += s1(CR_ST_WORD(m, 3)) + CR_ST_WORD(n, 2);
+    CR_ST_WORD(d, 2) += s1(CR_ST_WORD(d, 0)) + CR_ST_WORD(n, 3);
+    CR_ST_WORD(d, 3) += s1(CR_ST_WORD(d, 1)) + CR_ST_WORD(m, 0);
 
     env->vfp.regs[rd] = make_float64(d.l[0]);
     env->vfp.regs[rd + 1] = make_float64(d.l[1]);
diff --git a/target-arm/gdbstub.c b/target-arm/gdbstub.c
index 1c34396..04c1208 100644
--- a/target-arm/gdbstub.c
+++ b/target-arm/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 /* Old gdb always expect FPA registers.  Newer (xml-aware) gdb only expect
@@ -94,7 +95,7 @@
         return 4;
     case 25:
         /* CPSR */
-        cpsr_write(env, tmp, 0xffffffff);
+        cpsr_write(env, tmp, 0xffffffff, CPSRWriteByGDBStub);
         return 4;
     }
     /* Unknown register.  */
diff --git a/target-arm/gdbstub64.c b/target-arm/gdbstub64.c
index 8f3b8d1..49bc3fc 100644
--- a/target-arm/gdbstub64.c
+++ b/target-arm/gdbstub64.c
@@ -16,8 +16,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int aarch64_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
diff --git a/target-arm/helper-a64.c b/target-arm/helper-a64.c
index 81066ca..41e48a4 100644
--- a/target-arm/helper-a64.c
+++ b/target-arm/helper-a64.c
@@ -17,10 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/gdbstub.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
+#include "qemu/log.h"
 #include "sysemu/sysemu.h"
 #include "qemu/bitops.h"
 #include "internals.h"
@@ -70,20 +72,7 @@
 
 uint64_t HELPER(rbit64)(uint64_t x)
 {
-    /* assign the correct byte position */
-    x = bswap64(x);
-
-    /* assign the correct nibble position */
-    x = ((x & 0xf0f0f0f0f0f0f0f0ULL) >> 4)
-        | ((x & 0x0f0f0f0f0f0f0f0fULL) << 4);
-
-    /* assign the correct bit position */
-    x = ((x & 0x8888888888888888ULL) >> 3)
-        | ((x & 0x4444444444444444ULL) >> 1)
-        | ((x & 0x2222222222222222ULL) << 1)
-        | ((x & 0x1111111111111111ULL) << 3);
-
-    return x;
+    return revbit64(x);
 }
 
 /* Convert a softfloat float_relation_ (as returned by
@@ -135,6 +124,9 @@
 {
     float_status *fpst = fpstp;
 
+    a = float32_squash_input_denormal(a, fpst);
+    b = float32_squash_input_denormal(b, fpst);
+
     if ((float32_is_zero(a) && float32_is_infinity(b)) ||
         (float32_is_infinity(a) && float32_is_zero(b))) {
         /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
@@ -148,6 +140,9 @@
 {
     float_status *fpst = fpstp;
 
+    a = float64_squash_input_denormal(a, fpst);
+    b = float64_squash_input_denormal(b, fpst);
+
     if ((float64_is_zero(a) && float64_is_infinity(b)) ||
         (float64_is_infinity(a) && float64_is_zero(b))) {
         /* 2.0 with the sign bit set to sign(A) XOR sign(B) */
@@ -223,6 +218,9 @@
 {
     float_status *fpst = fpstp;
 
+    a = float32_squash_input_denormal(a, fpst);
+    b = float32_squash_input_denormal(b, fpst);
+
     a = float32_chs(a);
     if ((float32_is_infinity(a) && float32_is_zero(b)) ||
         (float32_is_infinity(b) && float32_is_zero(a))) {
@@ -235,6 +233,9 @@
 {
     float_status *fpst = fpstp;
 
+    a = float64_squash_input_denormal(a, fpst);
+    b = float64_squash_input_denormal(b, fpst);
+
     a = float64_chs(a);
     if ((float64_is_infinity(a) && float64_is_zero(b)) ||
         (float64_is_infinity(b) && float64_is_zero(a))) {
@@ -247,6 +248,9 @@
 {
     float_status *fpst = fpstp;
 
+    a = float32_squash_input_denormal(a, fpst);
+    b = float32_squash_input_denormal(b, fpst);
+
     a = float32_chs(a);
     if ((float32_is_infinity(a) && float32_is_zero(b)) ||
         (float32_is_infinity(b) && float32_is_zero(a))) {
@@ -259,6 +263,9 @@
 {
     float_status *fpst = fpstp;
 
+    a = float64_squash_input_denormal(a, fpst);
+    b = float64_squash_input_denormal(b, fpst);
+
     a = float64_chs(a);
     if ((float64_is_infinity(a) && float64_is_zero(b)) ||
         (float64_is_infinity(b) && float64_is_zero(a))) {
@@ -337,12 +344,12 @@
 
     if (float32_is_any_nan(a)) {
         float32 nan = a;
-        if (float32_is_signaling_nan(a)) {
+        if (float32_is_signaling_nan(a, fpst)) {
             float_raise(float_flag_invalid, fpst);
-            nan = float32_maybe_silence_nan(a);
+            nan = float32_maybe_silence_nan(a, fpst);
         }
         if (fpst->default_nan_mode) {
-            nan = float32_default_nan;
+            nan = float32_default_nan(fpst);
         }
         return nan;
     }
@@ -366,12 +373,12 @@
 
     if (float64_is_any_nan(a)) {
         float64 nan = a;
-        if (float64_is_signaling_nan(a)) {
+        if (float64_is_signaling_nan(a, fpst)) {
             float_raise(float_flag_invalid, fpst);
-            nan = float64_maybe_silence_nan(a);
+            nan = float64_maybe_silence_nan(a, fpst);
         }
         if (fpst->default_nan_mode) {
-            nan = float64_default_nan;
+            nan = float64_default_nan(fpst);
         }
         return nan;
     }
@@ -400,7 +407,7 @@
     set_float_rounding_mode(float_round_to_zero, &tstat);
     set_float_exception_flags(0, &tstat);
     r = float64_to_float32(a, &tstat);
-    r = float32_maybe_silence_nan(r);
+    r = float32_maybe_silence_nan(r, &tstat);
     exflags = get_float_exception_flags(&tstat);
     if (exflags & float_flag_inexact) {
         r = make_float32(float32_val(r) | 1);
@@ -437,93 +444,3 @@
     /* Linux crc32c converts the output to one's complement.  */
     return crc32c(acc, buf, bytes) ^ 0xffffffff;
 }
-
-#if !defined(CONFIG_USER_ONLY)
-
-/* Handle a CPU exception.  */
-void aarch64_cpu_do_interrupt(CPUState *cs)
-{
-    ARMCPU *cpu = ARM_CPU(cs);
-    CPUARMState *env = &cpu->env;
-    unsigned int new_el = arm_excp_target_el(cs, cs->exception_index);
-    target_ulong addr = env->cp15.vbar_el[new_el];
-    unsigned int new_mode = aarch64_pstate_mode(new_el, true);
-    int i;
-
-    if (arm_current_el(env) < new_el) {
-        if (env->aarch64) {
-            addr += 0x400;
-        } else {
-            addr += 0x600;
-        }
-    } else if (pstate_read(env) & PSTATE_SP) {
-        addr += 0x200;
-    }
-
-    arm_log_exception(cs->exception_index);
-    qemu_log_mask(CPU_LOG_INT, "...from EL%d\n", arm_current_el(env));
-    if (qemu_loglevel_mask(CPU_LOG_INT)
-        && !excp_is_internal(cs->exception_index)) {
-        qemu_log_mask(CPU_LOG_INT, "...with ESR 0x%" PRIx32 "\n",
-                      env->exception.syndrome);
-    }
-
-    if (arm_is_psci_call(cpu, cs->exception_index)) {
-        arm_handle_psci_call(cpu);
-        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
-        return;
-    }
-
-    switch (cs->exception_index) {
-    case EXCP_PREFETCH_ABORT:
-    case EXCP_DATA_ABORT:
-        env->cp15.far_el[new_el] = env->exception.vaddress;
-        qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
-                      env->cp15.far_el[new_el]);
-        /* fall through */
-    case EXCP_BKPT:
-    case EXCP_UDEF:
-    case EXCP_SWI:
-    case EXCP_HVC:
-    case EXCP_HYP_TRAP:
-    case EXCP_SMC:
-        env->cp15.esr_el[new_el] = env->exception.syndrome;
-        break;
-    case EXCP_IRQ:
-    case EXCP_VIRQ:
-        addr += 0x80;
-        break;
-    case EXCP_FIQ:
-    case EXCP_VFIQ:
-        addr += 0x100;
-        break;
-    default:
-        cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
-    }
-
-    if (is_a64(env)) {
-        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
-        aarch64_save_sp(env, arm_current_el(env));
-        env->elr_el[new_el] = env->pc;
-    } else {
-        env->banked_spsr[0] = cpsr_read(env);
-        if (!env->thumb) {
-            env->cp15.esr_el[new_el] |= 1 << 25;
-        }
-        env->elr_el[new_el] = env->regs[15];
-
-        for (i = 0; i < 15; i++) {
-            env->xregs[i] = env->regs[i];
-        }
-
-        env->condexec_bits = 0;
-    }
-
-    pstate_write(env, PSTATE_DAIF | new_mode);
-    env->aarch64 = 1;
-    aarch64_restore_sp(env, new_el);
-
-    env->pc = addr;
-    cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
-}
-#endif
diff --git a/target-arm/helper.c b/target-arm/helper.c
index d436e34..addd042 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "internals.h"
 #include "exec/gdbstub.h"
@@ -7,15 +8,27 @@
 #include "sysemu/sysemu.h"
 #include "qemu/bitops.h"
 #include "qemu/crc32c.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "arm_ldst.h"
 #include <zlib.h> /* For crc32 */
+#include "exec/semihost.h"
+#include "sysemu/kvm.h"
+
+#define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
 
 #ifndef CONFIG_USER_ONLY
-static inline int get_phys_addr(CPUARMState *env, target_ulong address,
-                                int access_type, int is_user,
-                                hwaddr *phys_ptr, int *prot,
-                                target_ulong *page_size);
+static bool get_phys_addr(CPUARMState *env, target_ulong address,
+                          int access_type, ARMMMUIdx mmu_idx,
+                          hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                          target_ulong *page_size, uint32_t *fsr,
+                          ARMMMUFaultInfo *fi);
+
+static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
+                               int access_type, ARMMMUIdx mmu_idx,
+                               hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
+                               target_ulong *page_size_ptr, uint32_t *fsr,
+                               ARMMMUFaultInfo *fi);
 
 /* Definitions for the PMCCNTR and PMCR registers */
 #define PMCRD   0x8
@@ -119,6 +132,7 @@
 
 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
+    assert(ri->fieldoffset);
     if (cpreg_field_is_64bit(ri)) {
         return CPREG_FIELD64(env, ri);
     } else {
@@ -129,6 +143,7 @@
 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
                       uint64_t value)
 {
+    assert(ri->fieldoffset);
     if (cpreg_field_is_64bit(ri)) {
         CPREG_FIELD64(env, ri) = value;
     } else {
@@ -136,7 +151,12 @@
     }
 }
 
-static uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
+static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return (char *)env + ri->fieldoffset;
+}
+
+uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     /* Raw read of a coprocessor register (as needed for migration, etc). */
     if (ri->type & ARM_CP_CONST) {
@@ -169,6 +189,27 @@
     }
 }
 
+static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
+{
+   /* Return true if the regdef would cause an assertion if you called
+    * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
+    * program bug for it not to have the NO_RAW flag).
+    * NB that returning false here doesn't necessarily mean that calling
+    * read/write_raw_cp_reg() is safe, because we can't distinguish "has
+    * read/write access functions which are safe for raw use" from "has
+    * read/write access functions which have side effects but has forgotten
+    * to provide raw access functions".
+    * The tests here line up with the conditions in read/write_raw_cp_reg()
+    * and assertions in raw_read()/raw_write().
+    */
+    if ((ri->type & ARM_CP_CONST) ||
+        ri->fieldoffset ||
+        ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
+        return false;
+    }
+    return true;
+}
+
 bool write_cpustate_to_list(ARMCPU *cpu)
 {
     /* Write the coprocessor state from cpu->env to the (index,value) list. */
@@ -184,7 +225,7 @@
             ok = false;
             continue;
         }
-        if (ri->type & ARM_CP_NO_MIGRATE) {
+        if (ri->type & ARM_CP_NO_RAW) {
             continue;
         }
         cpu->cpreg_values[i] = read_raw_cp_reg(&cpu->env, ri);
@@ -207,7 +248,7 @@
             ok = false;
             continue;
         }
-        if (ri->type & ARM_CP_NO_MIGRATE) {
+        if (ri->type & ARM_CP_NO_RAW) {
             continue;
         }
         /* Write value and confirm it reads back as written
@@ -231,7 +272,7 @@
     regidx = *(uint32_t *)key;
     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 
-    if (!(ri->type & ARM_CP_NO_MIGRATE)) {
+    if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
         /* The value array need not be initialized at this point */
         cpu->cpreg_array_len++;
@@ -247,7 +288,7 @@
     regidx = *(uint32_t *)key;
     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 
-    if (!(ri->type & ARM_CP_NO_MIGRATE)) {
+    if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
         cpu->cpreg_array_len++;
     }
 }
@@ -266,23 +307,15 @@
     return 0;
 }
 
-static void cpreg_make_keylist(gpointer key, gpointer value, gpointer udata)
-{
-    GList **plist = udata;
-
-    *plist = g_list_prepend(*plist, key);
-}
-
 void init_cpreg_list(ARMCPU *cpu)
 {
     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
      * Note that we require cpreg_tuples[] to be sorted by key ID.
      */
-    GList *keys = NULL;
+    GList *keys;
     int arraylen;
 
-    g_hash_table_foreach(cpu->cp_regs, cpreg_make_keylist, &keys);
-
+    keys = g_hash_table_get_keys(cpu->cp_regs);
     keys = g_list_sort(keys, cpreg_key_compare);
 
     cpu->cpreg_array_len = 0;
@@ -304,6 +337,127 @@
     g_list_free(keys);
 }
 
+/*
+ * Some registers are not accessible if EL3.NS=0 and EL3 is using AArch32 but
+ * they are accessible when EL3 is using AArch64 regardless of EL3.NS.
+ *
+ * access_el3_aa32ns: Used to check AArch32 register views.
+ * access_el3_aa32ns_aa64any: Used to check both AArch32/64 register views.
+ */
+static CPAccessResult access_el3_aa32ns(CPUARMState *env,
+                                        const ARMCPRegInfo *ri,
+                                        bool isread)
+{
+    bool secure = arm_is_secure_below_el3(env);
+
+    assert(!arm_el_is_aa64(env, 3));
+    if (secure) {
+        return CP_ACCESS_TRAP_UNCATEGORIZED;
+    }
+    return CP_ACCESS_OK;
+}
+
+static CPAccessResult access_el3_aa32ns_aa64any(CPUARMState *env,
+                                                const ARMCPRegInfo *ri,
+                                                bool isread)
+{
+    if (!arm_el_is_aa64(env, 3)) {
+        return access_el3_aa32ns(env, ri, isread);
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Some secure-only AArch32 registers trap to EL3 if used from
+ * Secure EL1 (but are just ordinary UNDEF in other non-EL3 contexts).
+ * Note that an access from Secure EL1 can only happen if EL3 is AArch64.
+ * We assume that the .access field is set to PL1_RW.
+ */
+static CPAccessResult access_trap_aa32s_el1(CPUARMState *env,
+                                            const ARMCPRegInfo *ri,
+                                            bool isread)
+{
+    if (arm_current_el(env) == 3) {
+        return CP_ACCESS_OK;
+    }
+    if (arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    /* This will be EL1 NS and EL2 NS, which just UNDEF */
+    return CP_ACCESS_TRAP_UNCATEGORIZED;
+}
+
+/* Check for traps to "powerdown debug" registers, which are controlled
+ * by MDCR.TDOSA
+ */
+static CPAccessResult access_tdosa(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDOSA)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDOSA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Check for traps to "debug ROM" registers, which are controlled
+ * by MDCR_EL2.TDRA for EL2 but by the more general MDCR_EL3.TDA for EL3.
+ */
+static CPAccessResult access_tdra(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDRA)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Check for traps to general debug registers, which are controlled
+ * by MDCR_EL2.TDA for EL2 and MDCR_EL3.TDA for EL3.
+ */
+static CPAccessResult access_tda(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDA)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+/* Check for traps to performance monitor registers, which are controlled
+ * by MDCR_EL2.TPM for EL2 and MDCR_EL3.TPM for EL3.
+ */
+static CPAccessResult access_tpm(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 bool isread)
+{
+    int el = arm_current_el(env);
+
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -418,14 +572,133 @@
     }
 }
 
+static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    CPUState *cs = ENV_GET_CPU(env);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                        ARMMMUIdx_S2NS, -1);
+}
+
+static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                            ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
+    }
+}
+
+static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                            uint64_t value)
+{
+    /* Invalidate by IPA. This has to invalidate any structures that
+     * contain only stage 2 translation information, but does not need
+     * to apply to structures that contain combined stage 1 and stage 2
+     * translation information.
+     * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
+     */
+    CPUState *cs = ENV_GET_CPU(env);
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 40);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
+}
+
+static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    CPUState *other_cs;
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 40);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
+    }
+}
+
+static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    CPUState *cs = ENV_GET_CPU(env);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
+}
+
+static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
+    }
+}
+
+static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    CPUState *cs = ENV_GET_CPU(env);
+    uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
+}
+
+static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
+{
+    CPUState *other_cs;
+    uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
+    }
+}
+
 static const ARMCPRegInfo cp_reginfo[] = {
-    { .name = "FCSEIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_fcse),
+    /* Define the secure and non-secure FCSE identifier CP registers
+     * separately because there is no secure bank in V8 (no _EL3).  This allows
+     * the secure register to be properly reset and migrated. There is also no
+     * v8 EL1 version of the register so the non-secure instance stands alone.
+     */
+    { .name = "FCSEIDR(NS)",
+      .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
+      .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
+      .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
-    { .name = "CONTEXTIDR", .state = ARM_CP_STATE_BOTH,
+    { .name = "FCSEIDR(S)",
+      .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
+      .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
+      .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
+      .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
+    /* Define the secure and non-secure context identifier CP registers
+     * separately because there is no secure bank in V8 (no _EL3).  This allows
+     * the secure register to be properly reset and migrated.  In the
+     * non-secure case, the 32-bit register will have reset and migration
+     * disabled during registration as it is handled by the 64-bit instance.
+     */
+    { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
-      .access = PL1_RW,
-      .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el1),
+      .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
+      .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
+      .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
+    { .name = "CONTEXTIDR(S)", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
+      .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
+      .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
     REGINFO_SENTINEL
 };
@@ -435,14 +708,22 @@
      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
      */
     /* MMU Domain access control / MPU write buffer control */
-    { .name = "DACR", .cp = 15,
-      .crn = 3, .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c3),
-      .resetvalue = 0, .writefn = dacr_write, .raw_writefn = raw_write, },
-    /* ??? This covers not just the impdef TLB lockdown registers but also
-     * some v7VMSA registers relating to TEX remap, so it is overly broad.
+    { .name = "DACR",
+      .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
+      .access = PL1_RW, .resetvalue = 0,
+      .writefn = dacr_write, .raw_writefn = raw_write,
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
+                             offsetoflow32(CPUARMState, cp15.dacr_ns) } },
+    /* ARMv7 allocates a range of implementation defined TLB LOCKDOWN regs.
+     * For v6 and v5, these mappings are overly broad.
      */
-    { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = CP_ANY,
+    { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 0,
+      .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
+    { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 1,
+      .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
+    { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 4,
+      .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
+    { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 8,
       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
     /* Cache maintenance ops; some of this space may be overridden later. */
     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
@@ -478,7 +759,7 @@
       .resetvalue = 0 },
     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
-      .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
       .resetvalue = 0 },
     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
      * implementing it as RAZ means the "debug architecture version" bits
@@ -492,16 +773,20 @@
      */
     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
-      .type = ARM_CP_NO_MIGRATE },
+      .type = ARM_CP_NO_RAW },
     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
-      .type = ARM_CP_NO_MIGRATE },
+      .type = ARM_CP_NO_RAW },
     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
-      .type = ARM_CP_NO_MIGRATE },
+      .type = ARM_CP_NO_RAW },
     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
-      .type = ARM_CP_NO_MIGRATE },
+      .type = ARM_CP_NO_RAW },
+    { .name = "PRRR", .cp = 15, .crn = 10, .crm = 2,
+      .opc1 = 0, .opc2 = 0, .access = PL1_RW, .type = ARM_CP_NOP },
+    { .name = "NMRR", .cp = 15, .crn = 10, .crm = 2,
+      .opc1 = 0, .opc2 = 1, .access = PL1_RW, .type = ARM_CP_NOP },
     REGINFO_SENTINEL
 };
 
@@ -536,7 +821,36 @@
         }
         value &= mask;
     }
-    env->cp15.c1_coproc = value;
+    env->cp15.cpacr_el1 = value;
+}
+
+static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
+{
+    if (arm_feature(env, ARM_FEATURE_V8)) {
+        /* Check if CPACR accesses are to be trapped to EL2 */
+        if (arm_current_el(env) == 1 &&
+            (env->cp15.cptr_el[2] & CPTR_TCPAC) && !arm_is_secure(env)) {
+            return CP_ACCESS_TRAP_EL2;
+        /* Check if CPACR accesses are to be trapped to EL3 */
+        } else if (arm_current_el(env) < 3 &&
+                   (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
+            return CP_ACCESS_TRAP_EL3;
+        }
+    }
+
+    return CP_ACCESS_OK;
+}
+
+static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  bool isread)
+{
+    /* Check if CPTR accesses are set to trap to EL3 */
+    if (arm_current_el(env) == 2 && (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+
+    return CP_ACCESS_OK;
 }
 
 static const ARMCPRegInfo v6_cp_reginfo[] = {
@@ -544,15 +858,20 @@
     { .name = "MVA_prefetch",
       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
       .access = PL1_W, .type = ARM_CP_NOP },
+    /* We need to break the TB after ISB to execute self-modifying code
+     * correctly and also to take any pending interrupts immediately.
+     * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
+     */
     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
-      .access = PL0_W, .type = ARM_CP_NOP },
+      .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
       .access = PL0_W, .type = ARM_CP_NOP },
     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
       .access = PL0_W, .type = ARM_CP_NOP },
     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL1_RW,
-      .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el[1]),
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
+                             offsetof(CPUARMState, cp15.ifar_ns) },
       .resetvalue = 0, },
     /* Watchpoint Fault Address Register : should actually only be present
      * for 1136, 1176, 11MPCore.
@@ -560,20 +879,32 @@
     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
-      .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_coproc),
+      .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2, .accessfn = cpacr_access,
+      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
       .resetvalue = 0, .writefn = cpacr_write },
     REGINFO_SENTINEL
 };
 
-static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
 {
     /* Performance monitor registers user accessibility is controlled
-     * by PMUSERENR.
+     * by PMUSERENR. MDCR_EL2.TPM and MDCR_EL3.TPM allow configurable
+     * trapping to EL2 or EL3 for other accesses.
      */
-    if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
+    int el = arm_current_el(env);
+
+    if (el == 0 && !env->cp15.c9_pmuserenr) {
         return CP_ACCESS_TRAP;
     }
+    if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
+        && !arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
+    if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+
     return CP_ACCESS_OK;
 }
 
@@ -594,8 +925,8 @@
 {
     uint64_t temp_ticks;
 
-    temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                          get_ticks_per_sec(), 1000000);
+    temp_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                          ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -633,8 +964,8 @@
         return env->cp15.c15_ccnt;
     }
 
-    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                           get_ticks_per_sec(), 1000000);
+    total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -654,8 +985,8 @@
         return;
     }
 
-    total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
-                           get_ticks_per_sec(), 1000000);
+    total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
+                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 
     if (env->cp15.c9_pmcr & PMCRD) {
         /* Increment once every 64 processor clock cycles */
@@ -762,8 +1093,10 @@
          * supported if EL2 exists. The bit is UNK/SBZP when
          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
          * when EL2 is unavailable.
+         * On ARMv8, this bit is always available.
          */
-        if (arm_feature(env, ARM_FEATURE_V7)) {
+        if (arm_feature(env, ARM_FEATURE_V7) &&
+            !arm_feature(env, ARM_FEATURE_V8)) {
             valid_mask &= ~SCR_SMD;
         }
     }
@@ -776,7 +1109,14 @@
 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    return cpu->ccsidr[env->cp15.c0_cssel];
+
+    /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
+     * bank
+     */
+    uint32_t index = A32_BANKED_REG_GET(env, csselr,
+                                        ri->secure & ARM_CP_SECSTATE_S);
+
+    return cpu->ccsidr[index];
 }
 
 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -816,7 +1156,7 @@
      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
      */
     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
-      .access = PL0_RW, .type = ARM_CP_NO_MIGRATE,
+      .access = PL0_RW, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
       .writefn = pmcntenset_write,
       .accessfn = pmreg_access,
@@ -831,11 +1171,11 @@
       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
       .accessfn = pmreg_access,
       .writefn = pmcntenclr_write,
-      .type = ARM_CP_NO_MIGRATE },
+      .type = ARM_CP_ALIAS },
     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
       .access = PL0_RW, .accessfn = pmreg_access,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
       .writefn = pmcntenclr_write },
     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
@@ -843,6 +1183,13 @@
       .accessfn = pmreg_access,
       .writefn = pmovsr_write,
       .raw_writefn = raw_write },
+    { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3,
+      .access = PL0_RW, .accessfn = pmreg_access,
+      .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
+      .writefn = pmovsr_write,
+      .raw_writefn = raw_write },
     /* Unimplemented so WI. */
     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
@@ -880,34 +1227,44 @@
       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
       .accessfn = pmreg_access },
     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R | PL1_RW,
+      .access = PL0_R | PL1_RW, .accessfn = access_tpm,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
+      .resetvalue = 0,
+      .writefn = pmuserenr_write, .raw_writefn = raw_write },
+    { .name = "PMUSERENR_EL0", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 0,
+      .access = PL0_R | PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
       .resetvalue = 0,
       .writefn = pmuserenr_write, .raw_writefn = raw_write },
     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW,
+      .access = PL1_RW, .accessfn = access_tpm,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
       .resetvalue = 0,
       .writefn = pmintenset_write, .raw_writefn = raw_write },
     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
-      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
+      .access = PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
-      .resetvalue = 0, .writefn = pmintenclr_write, },
+      .writefn = pmintenclr_write, },
+    { .name = "PMINTENCLR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 2,
+      .access = PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
+      .writefn = pmintenclr_write },
     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .writefn = vbar_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[1]),
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
+                             offsetof(CPUARMState, cp15.vbar_ns) },
       .resetvalue = 0 },
-    { .name = "SCR", .cp = 15, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
-      .resetvalue = 0, .writefn = scr_write },
     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
-      .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_MIGRATE },
+      .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c0_cssel),
-      .writefn = csselr_write, .resetvalue = 0 },
+      .access = PL1_RW, .writefn = csselr_write, .resetvalue = 0,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
+                             offsetof(CPUARMState, cp15.csselr_ns) } },
     /* Auxiliary ID register: this actually has an IMPDEF value but for now
      * just RAZ for all cores:
      */
@@ -928,61 +1285,69 @@
      */
     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el1),
+      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
+      .resetvalue = 0 },
+    { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
       .resetvalue = 0 },
     /* For non-long-descriptor page tables these are PRRR and NMRR;
      * regardless they still act as reads-as-written for QEMU.
-     * The override is necessary because of the overly-broad TLB_LOCKDOWN
-     * definition.
      */
-    { .name = "MAIR0", .state = ARM_CP_STATE_AA32, .type = ARM_CP_OVERRIDE,
+     /* MAIR0/1 are defined separately from their 64-bit counterpart which
+      * allows them to assign the correct fieldoffset based on the endianness
+      * handled in the field definitions.
+      */
+    { .name = "MAIR0", .state = ARM_CP_STATE_AA32,
       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0, .access = PL1_RW,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.mair_el1),
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
+                             offsetof(CPUARMState, cp15.mair0_ns) },
       .resetfn = arm_cp_reset_ignore },
-    { .name = "MAIR1", .state = ARM_CP_STATE_AA32, .type = ARM_CP_OVERRIDE,
+    { .name = "MAIR1", .state = ARM_CP_STATE_AA32,
       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
-      .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el1),
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
+                             offsetof(CPUARMState, cp15.mair1_ns) },
       .resetfn = arm_cp_reset_ignore },
     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_R, .readfn = isr_read },
+      .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
     /* 32 bit ITLB invalidates */
     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
     /* 32 bit DTLB invalidates */
     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
     /* 32 bit TLB invalidates */
     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
     REGINFO_SENTINEL
 };
 
 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
     /* 32 bit TLB invalidates, Inner Shareable */
     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_is_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_is_write },
     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_is_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
+      .type = ARM_CP_NO_RAW, .access = PL1_W,
       .writefn = tlbiasid_is_write },
     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
+      .type = ARM_CP_NO_RAW, .access = PL1_W,
       .writefn = tlbimvaa_is_write },
     REGINFO_SENTINEL
 };
@@ -994,7 +1359,8 @@
     env->teecr = value;
 }
 
-static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    bool isread)
 {
     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
         return CP_ACCESS_TRAP;
@@ -1017,79 +1383,162 @@
     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
       .access = PL0_RW,
-      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el0), .resetvalue = 0 },
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL0_RW,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.tpidr_el0),
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
+                             offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
       .resetfn = arm_cp_reset_ignore },
     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
       .access = PL0_R|PL1_W,
-      .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el0), .resetvalue = 0 },
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
+      .resetvalue = 0},
     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
       .access = PL0_R|PL1_W,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.tpidrro_el0),
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
+                             offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
       .resetfn = arm_cp_reset_ignore },
-    { .name = "TPIDR_EL1", .state = ARM_CP_STATE_BOTH,
+    { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
       .access = PL1_RW,
-      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el1), .resetvalue = 0 },
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
+    { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
+      .access = PL1_RW,
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
+                             offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
+      .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
 #ifndef CONFIG_USER_ONLY
 
-static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero */
-    if (arm_current_el(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
-        return CP_ACCESS_TRAP;
+    /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero.
+     * Writable only at the highest implemented exception level.
+     */
+    int el = arm_current_el(env);
+
+    switch (el) {
+    case 0:
+        if (!extract32(env->cp15.c14_cntkctl, 0, 2)) {
+            return CP_ACCESS_TRAP;
+        }
+        break;
+    case 1:
+        if (!isread && ri->state == ARM_CP_STATE_AA32 &&
+            arm_is_secure_below_el3(env)) {
+            /* Accesses from 32-bit Secure EL1 UNDEF (*not* trap to EL3!) */
+            return CP_ACCESS_TRAP_UNCATEGORIZED;
+        }
+        break;
+    case 2:
+    case 3:
+        break;
     }
+
+    if (!isread && el < arm_highest_el(env)) {
+        return CP_ACCESS_TRAP_UNCATEGORIZED;
+    }
+
     return CP_ACCESS_OK;
 }
 
-static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
+static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx,
+                                        bool isread)
 {
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
-    if (arm_current_el(env) == 0 &&
+    if (cur_el == 0 &&
         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) &&
+        timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
+        !extract32(env->cp15.cnthctl_el2, 0, 1)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
     return CP_ACCESS_OK;
 }
 
-static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
+static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx,
+                                      bool isread)
 {
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
      * EL0[PV]TEN is zero.
      */
-    if (arm_current_el(env) == 0 &&
+    if (cur_el == 0 &&
         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
         return CP_ACCESS_TRAP;
     }
+
+    if (arm_feature(env, ARM_FEATURE_EL2) &&
+        timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
+        !extract32(env->cp15.cnthctl_el2, 1, 1)) {
+        return CP_ACCESS_TRAP_EL2;
+    }
     return CP_ACCESS_OK;
 }
 
 static CPAccessResult gt_pct_access(CPUARMState *env,
-                                         const ARMCPRegInfo *ri)
+                                    const ARMCPRegInfo *ri,
+                                    bool isread)
 {
-    return gt_counter_access(env, GTIMER_PHYS);
+    return gt_counter_access(env, GTIMER_PHYS, isread);
 }
 
 static CPAccessResult gt_vct_access(CPUARMState *env,
-                                         const ARMCPRegInfo *ri)
+                                    const ARMCPRegInfo *ri,
+                                    bool isread)
 {
-    return gt_counter_access(env, GTIMER_VIRT);
+    return gt_counter_access(env, GTIMER_VIRT, isread);
 }
 
-static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    return gt_timer_access(env, GTIMER_PHYS);
+    return gt_timer_access(env, GTIMER_PHYS, isread);
 }
 
-static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    return gt_timer_access(env, GTIMER_VIRT);
+    return gt_timer_access(env, GTIMER_VIRT, isread);
+}
+
+static CPAccessResult gt_stimer_access(CPUARMState *env,
+                                       const ARMCPRegInfo *ri,
+                                       bool isread)
+{
+    /* The AArch64 register view of the secure physical timer is
+     * always accessible from EL3, and configurably accessible from
+     * Secure EL1.
+     */
+    switch (arm_current_el(env)) {
+    case 1:
+        if (!arm_is_secure(env)) {
+            return CP_ACCESS_TRAP;
+        }
+        if (!(env->cp15.scr_el3 & SCR_ST)) {
+            return CP_ACCESS_TRAP_EL3;
+        }
+        return CP_ACCESS_OK;
+    case 0:
+    case 2:
+        return CP_ACCESS_TRAP;
+    case 3:
+        return CP_ACCESS_OK;
+    default:
+        g_assert_not_reached();
+    }
 }
 
 static uint64_t gt_get_countervalue(CPUARMState *env)
@@ -1105,9 +1554,11 @@
         /* Timer enabled: calculate and set current ISTATUS, irq, and
          * reset timer to when ISTATUS next has to change
          */
+        uint64_t offset = timeridx == GTIMER_VIRT ?
+                                      cpu->env.cp15.cntvoff_el2 : 0;
         uint64_t count = gt_get_countervalue(&cpu->env);
         /* Note that this must be unsigned 64 bit arithmetic: */
-        int istatus = count >= gt->cval;
+        int istatus = count - offset >= gt->cval;
         uint64_t nexttick;
 
         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
@@ -1118,7 +1569,7 @@
             nexttick = UINT64_MAX;
         } else {
             /* Next transition is when we hit cval */
-            nexttick = gt->cval;
+            nexttick = gt->cval + offset;
         }
         /* Note that the desired next expiry time might be beyond the
          * signed-64-bit range of a QEMUTimer -- in this case we just
@@ -1137,10 +1588,10 @@
     }
 }
 
-static void gt_cnt_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
+                           int timeridx)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int timeridx = ri->opc1 & 1;
 
     timer_del(cpu->gt_timer[timeridx]);
 }
@@ -1150,38 +1601,44 @@
     return gt_get_countervalue(env);
 }
 
+static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_get_countervalue(env) - env->cp15.cntvoff_el2;
+}
+
 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          int timeridx,
                           uint64_t value)
 {
-    int timeridx = ri->opc1 & 1;
-
     env->cp15.c14_timer[timeridx].cval = value;
     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
 }
 
-static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
+                             int timeridx)
 {
-    int timeridx = ri->crm & 1;
+    uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
 
     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
-                      gt_get_countervalue(env));
+                      (gt_get_countervalue(env) - offset));
 }
 
 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                          int timeridx,
                           uint64_t value)
 {
-    int timeridx = ri->crm & 1;
+    uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
 
-    env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) +
-        + sextract64(value, 0, 32);
+    env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
+                                         sextract64(value, 0, 32);
     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
 }
 
 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         int timeridx,
                          uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int timeridx = ri->crm & 1;
     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
 
     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
@@ -1197,6 +1654,127 @@
     }
 }
 
+static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_PHYS);
+}
+
+static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_PHYS, value);
+}
+
+static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_PHYS);
+}
+
+static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_PHYS, value);
+}
+
+static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_PHYS, value);
+}
+
+static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_VIRT);
+}
+
+static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_VIRT, value);
+}
+
+static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_VIRT);
+}
+
+static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                               uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_VIRT, value);
+}
+
+static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_VIRT, value);
+}
+
+static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    raw_write(env, ri, value);
+    gt_recalc_timer(cpu, GTIMER_VIRT);
+}
+
+static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_HYP);
+}
+
+static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_HYP, value);
+}
+
+static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_HYP);
+}
+
+static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_HYP, value);
+}
+
+static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_HYP, value);
+}
+
+static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    gt_timer_reset(env, ri, GTIMER_SEC);
+}
+
+static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_cval_write(env, ri, GTIMER_SEC, value);
+}
+
+static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    return gt_tval_read(env, ri, GTIMER_SEC);
+}
+
+static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_tval_write(env, ri, GTIMER_SEC, value);
+}
+
+static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    gt_ctl_write(env, ri, GTIMER_SEC, value);
+}
+
 void arm_gt_ptimer_cb(void *opaque)
 {
     ARMCPU *cpu = opaque;
@@ -1211,16 +1789,29 @@
     gt_recalc_timer(cpu, GTIMER_VIRT);
 }
 
+void arm_gt_htimer_cb(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    gt_recalc_timer(cpu, GTIMER_HYP);
+}
+
+void arm_gt_stimer_cb(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+
+    gt_recalc_timer(cpu, GTIMER_SEC);
+}
+
 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
     /* Note that CNTFRQ is purely reads-as-written for the benefit
      * of software; writing it doesn't actually change the timer frequency.
      * Our reset value matches the fixed frequency we implement the timer at.
      */
     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
-      .resetfn = arm_cp_reset_ignore,
     },
     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
@@ -1237,12 +1828,21 @@
     },
     /* per-timer control */
     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
-      .type = ARM_CP_IO | ARM_CP_NO_MIGRATE, .access = PL1_RW | PL0_R,
+      .secure = ARM_CP_SECSTATE_NS,
+      .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
       .accessfn = gt_ptimer_access,
       .fieldoffset = offsetoflow32(CPUARMState,
                                    cp15.c14_timer[GTIMER_PHYS].ctl),
-      .resetfn = arm_cp_reset_ignore,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTP_CTL(S)",
+      .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
+      .secure = ARM_CP_SECSTATE_S,
+      .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access,
+      .fieldoffset = offsetoflow32(CPUARMState,
+                                   cp15.c14_timer[GTIMER_SEC].ctl),
+      .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
@@ -1250,15 +1850,14 @@
       .accessfn = gt_ptimer_access,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
       .resetvalue = 0,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
-      .type = ARM_CP_IO | ARM_CP_NO_MIGRATE, .access = PL1_RW | PL0_R,
+      .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
       .accessfn = gt_vtimer_access,
       .fieldoffset = offsetoflow32(CPUARMState,
                                    cp15.c14_timer[GTIMER_VIRT].ctl),
-      .resetfn = arm_cp_reset_ignore,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
@@ -1266,74 +1865,91 @@
       .accessfn = gt_vtimer_access,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
       .resetvalue = 0,
-      .writefn = gt_ctl_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
     },
     /* TimerValue views: a 32 bit downcounting view of the underlying state */
     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .secure = ARM_CP_SECSTATE_NS,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
       .accessfn = gt_ptimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
+    },
+    { .name = "CNTP_TVAL(S)",
+      .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
+      .secure = ARM_CP_SECSTATE_S,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access,
+      .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
     },
     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
+      .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
     },
     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
       .accessfn = gt_vtimer_access,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
     },
     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
-      .readfn = gt_tval_read, .writefn = gt_tval_write,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
+      .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
+      .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
     },
     /* The counter itself */
     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
-      .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE | ARM_CP_IO,
+      .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
       .accessfn = gt_pct_access,
       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
     },
     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
-      .access = PL0_R, .type = ARM_CP_NO_MIGRATE | ARM_CP_IO,
-      .accessfn = gt_pct_access,
-      .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
+      .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
+      .accessfn = gt_pct_access, .readfn = gt_cnt_read,
     },
     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
-      .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE | ARM_CP_IO,
+      .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
       .accessfn = gt_vct_access,
-      .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
+      .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
     },
     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
-      .access = PL0_R, .type = ARM_CP_NO_MIGRATE | ARM_CP_IO,
-      .accessfn = gt_vct_access,
-      .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
+      .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
+      .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
     },
     /* Comparison value, indicating when the timer goes off */
     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
+      .secure = ARM_CP_SECSTATE_NS,
       .access = PL1_RW | PL0_R,
-      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
-      .accessfn = gt_ptimer_access, .resetfn = arm_cp_reset_ignore,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .accessfn = gt_ptimer_access,
+      .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTP_CVAL(S)", .cp = 15, .crm = 14, .opc1 = 2,
+      .secure = ARM_CP_SECSTATE_S,
+      .access = PL1_RW | PL0_R,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
+      .accessfn = gt_ptimer_access,
+      .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
       .access = PL1_RW | PL0_R,
       .type = ARM_CP_IO,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
-      .resetvalue = 0, .accessfn = gt_vtimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .resetvalue = 0, .accessfn = gt_ptimer_access,
+      .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
       .access = PL1_RW | PL0_R,
-      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
-      .accessfn = gt_vtimer_access, .resetfn = arm_cp_reset_ignore,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .accessfn = gt_vtimer_access,
+      .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
     },
     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
@@ -1341,7 +1957,33 @@
       .type = ARM_CP_IO,
       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
       .resetvalue = 0, .accessfn = gt_vtimer_access,
-      .writefn = gt_cval_write, .raw_writefn = raw_write,
+      .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
+    },
+    /* Secure timer -- this is actually restricted to only EL3
+     * and configurably Secure-EL1 via the accessfn.
+     */
+    { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .readfn = gt_sec_tval_read,
+      .writefn = gt_sec_tval_write,
+      .resetfn = gt_sec_timer_reset,
+    },
+    { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
+      .type = ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
+      .resetvalue = 0,
+      .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
+    },
+    { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
+      .type = ARM_CP_IO, .access = PL1_RW,
+      .accessfn = gt_stimer_access,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
+      .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
     },
     REGINFO_SENTINEL
 };
@@ -1371,78 +2013,212 @@
 #ifndef CONFIG_USER_ONLY
 /* get_phys_addr() isn't present for user-mode-only targets */
 
-static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 bool isread)
 {
     if (ri->opc2 & 4) {
-        /* Other states are only available with TrustZone; in
-         * a non-TZ implementation these registers don't exist
-         * at all, which is an Uncategorized trap. This underdecoding
-         * is safe because the reginfo is NO_MIGRATE.
+        /* The ATS12NSO* operations must trap to EL3 if executed in
+         * Secure EL1 (which can only happen if EL3 is AArch64).
+         * They are simply UNDEF if executed from NS EL1.
+         * They function normally from EL2 or EL3.
          */
-        return CP_ACCESS_TRAP_UNCATEGORIZED;
+        if (arm_current_el(env) == 1) {
+            if (arm_is_secure_below_el3(env)) {
+                return CP_ACCESS_TRAP_UNCATEGORIZED_EL3;
+            }
+            return CP_ACCESS_TRAP_UNCATEGORIZED;
+        }
     }
     return CP_ACCESS_OK;
 }
 
-static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
+static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
+                             int access_type, ARMMMUIdx mmu_idx)
 {
     hwaddr phys_addr;
     target_ulong page_size;
     int prot;
-    int ret, is_user = ri->opc2 & 2;
-    int access_type = ri->opc2 & 1;
+    uint32_t fsr;
+    bool ret;
+    uint64_t par64;
+    MemTxAttrs attrs = {};
+    ARMMMUFaultInfo fi = {};
 
-    ret = get_phys_addr(env, value, access_type, is_user,
-                        &phys_addr, &prot, &page_size);
+    ret = get_phys_addr(env, value, access_type, mmu_idx,
+                        &phys_addr, &attrs, &prot, &page_size, &fsr, &fi);
     if (extended_addresses_enabled(env)) {
-        /* ret is a DFSR/IFSR value for the long descriptor
+        /* fsr is a DFSR/IFSR value for the long descriptor
          * translation table format, but with WnR always clear.
          * Convert it to a 64-bit PAR.
          */
-        uint64_t par64 = (1 << 11); /* LPAE bit always set */
-        if (ret == 0) {
+        par64 = (1 << 11); /* LPAE bit always set */
+        if (!ret) {
             par64 |= phys_addr & ~0xfffULL;
+            if (!attrs.secure) {
+                par64 |= (1 << 9); /* NS */
+            }
             /* We don't set the ATTR or SH fields in the PAR. */
         } else {
             par64 |= 1; /* F */
-            par64 |= (ret & 0x3f) << 1; /* FS */
+            par64 |= (fsr & 0x3f) << 1; /* FS */
             /* Note that S2WLK and FSTAGE are always zero, because we don't
              * implement virtualization and therefore there can't be a stage 2
              * fault.
              */
         }
-        env->cp15.par_el1 = par64;
     } else {
-        /* ret is a DFSR/IFSR value for the short descriptor
+        /* fsr is a DFSR/IFSR value for the short descriptor
          * translation table format (with WnR always clear).
          * Convert it to a 32-bit PAR.
          */
-        if (ret == 0) {
+        if (!ret) {
             /* We do not set any attribute bits in the PAR */
             if (page_size == (1 << 24)
                 && arm_feature(env, ARM_FEATURE_V7)) {
-                env->cp15.par_el1 = (phys_addr & 0xff000000) | 1 << 1;
+                par64 = (phys_addr & 0xff000000) | (1 << 1);
             } else {
-                env->cp15.par_el1 = phys_addr & 0xfffff000;
+                par64 = phys_addr & 0xfffff000;
+            }
+            if (!attrs.secure) {
+                par64 |= (1 << 9); /* NS */
             }
         } else {
-            env->cp15.par_el1 = ((ret & (1 << 10)) >> 5) |
-                ((ret & (1 << 12)) >> 6) |
-                ((ret & 0xf) << 1) | 1;
+            par64 = ((fsr & (1 << 10)) >> 5) | ((fsr & (1 << 12)) >> 6) |
+                    ((fsr & 0xf) << 1) | 1;
         }
     }
+    return par64;
+}
+
+static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
+{
+    int access_type = ri->opc2 & 1;
+    uint64_t par64;
+    ARMMMUIdx mmu_idx;
+    int el = arm_current_el(env);
+    bool secure = arm_is_secure_below_el3(env);
+
+    switch (ri->opc2 & 6) {
+    case 0:
+        /* stage 1 current state PL1: ATS1CPR, ATS1CPW */
+        switch (el) {
+        case 3:
+            mmu_idx = ARMMMUIdx_S1E3;
+            break;
+        case 2:
+            mmu_idx = ARMMMUIdx_S1NSE1;
+            break;
+        case 1:
+            mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+    case 2:
+        /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
+        switch (el) {
+        case 3:
+            mmu_idx = ARMMMUIdx_S1SE0;
+            break;
+        case 2:
+            mmu_idx = ARMMMUIdx_S1NSE0;
+            break;
+        case 1:
+            mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+    case 4:
+        /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
+        mmu_idx = ARMMMUIdx_S12NSE1;
+        break;
+    case 6:
+        /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
+        mmu_idx = ARMMMUIdx_S12NSE0;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    par64 = do_ats_write(env, value, access_type, mmu_idx);
+
+    A32_BANKED_CURRENT_REG_SET(env, par, par64);
+}
+
+static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    int access_type = ri->opc2 & 1;
+    uint64_t par64;
+
+    par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S2NS);
+
+    A32_BANKED_CURRENT_REG_SET(env, par, par64);
+}
+
+static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                     bool isread)
+{
+    if (arm_current_el(env) == 3 && !(env->cp15.scr_el3 & SCR_NS)) {
+        return CP_ACCESS_TRAP;
+    }
+    return CP_ACCESS_OK;
+}
+
+static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    int access_type = ri->opc2 & 1;
+    ARMMMUIdx mmu_idx;
+    int secure = arm_is_secure_below_el3(env);
+
+    switch (ri->opc2 & 6) {
+    case 0:
+        switch (ri->opc1) {
+        case 0: /* AT S1E1R, AT S1E1W */
+            mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
+            break;
+        case 4: /* AT S1E2R, AT S1E2W */
+            mmu_idx = ARMMMUIdx_S1E2;
+            break;
+        case 6: /* AT S1E3R, AT S1E3W */
+            mmu_idx = ARMMMUIdx_S1E3;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+    case 2: /* AT S1E0R, AT S1E0W */
+        mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
+        break;
+    case 4: /* AT S12E1R, AT S12E1W */
+        mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1;
+        break;
+    case 6: /* AT S12E0R, AT S12E0W */
+        mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    env->cp15.par_el[1] = do_ats_write(env, value, access_type, mmu_idx);
 }
 #endif
 
 static const ARMCPRegInfo vapa_cp_reginfo[] = {
     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .resetvalue = 0,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.par_el1),
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
+                             offsetoflow32(CPUARMState, cp15.par_ns) },
       .writefn = par_write },
 #ifndef CONFIG_USER_ONLY
+    /* This underdecoding is safe because the reginfo is NO_RAW. */
     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
       .access = PL1_W, .accessfn = ats_access,
-      .writefn = ats_write, .type = ARM_CP_NO_MIGRATE },
+      .writefn = ats_write, .type = ARM_CP_NO_RAW },
 #endif
     REGINFO_SENTINEL
 };
@@ -1499,16 +2275,89 @@
     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
 }
 
+static uint64_t pmsav7_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
+
+    if (!u32p) {
+        return 0;
+    }
+
+    u32p += env->cp15.c6_rgnr;
+    return *u32p;
+}
+
+static void pmsav7_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                         uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
+
+    if (!u32p) {
+        return;
+    }
+
+    u32p += env->cp15.c6_rgnr;
+    tlb_flush(CPU(cpu), 1); /* Mappings may have changed - purge! */
+    *u32p = value;
+}
+
+static void pmsav7_reset(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
+
+    if (!u32p) {
+        return;
+    }
+
+    memset(u32p, 0, sizeof(*u32p) * cpu->pmsav7_dregion);
+}
+
+static void pmsav7_rgnr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                              uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    uint32_t nrgs = cpu->pmsav7_dregion;
+
+    if (value >= nrgs) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "PMSAv7 RGNR write >= # supported regions, %" PRIu32
+                      " > %" PRIu32 "\n", (uint32_t)value, nrgs);
+        return;
+    }
+
+    raw_write(env, ri, value);
+}
+
+static const ARMCPRegInfo pmsav7_cp_reginfo[] = {
+    { .name = "DRBAR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 0,
+      .access = PL1_RW, .type = ARM_CP_NO_RAW,
+      .fieldoffset = offsetof(CPUARMState, pmsav7.drbar),
+      .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
+    { .name = "DRSR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 2,
+      .access = PL1_RW, .type = ARM_CP_NO_RAW,
+      .fieldoffset = offsetof(CPUARMState, pmsav7.drsr),
+      .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
+    { .name = "DRACR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 4,
+      .access = PL1_RW, .type = ARM_CP_NO_RAW,
+      .fieldoffset = offsetof(CPUARMState, pmsav7.dracr),
+      .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
+    { .name = "RGNR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 2, .opc2 = 0,
+      .access = PL1_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.c6_rgnr),
+      .writefn = pmsav7_rgnr_write },
+    REGINFO_SENTINEL
+};
+
 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
+      .access = PL1_RW, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
-      .resetvalue = 0,
       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
+      .access = PL1_RW, .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
-      .resetvalue = 0,
       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL1_RW,
@@ -1555,6 +2404,7 @@
 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t value)
 {
+    TCR *tcr = raw_ptr(env, ri);
     int maskshift = extract32(value, 0, 3);
 
     if (!arm_feature(env, ARM_FEATURE_V8)) {
@@ -1573,14 +2423,15 @@
         }
     }
 
-    /* Note that we always calculate c2_mask and c2_base_mask, but
+    /* Update the masks corresponding to the TCR bank being written
+     * Note that we always calculate mask and base_mask, but
      * they are only used for short-descriptor tables (ie if EAE is 0);
-     * for long-descriptor tables the TTBCR fields are used differently
-     * and the c2_mask and c2_base_mask values are meaningless.
+     * for long-descriptor tables the TCR fields are used differently
+     * and the mask and base_mask values are meaningless.
      */
-    raw_write(env, ri, value);
-    env->cp15.c2_mask = ~(((uint32_t)0xffffffffu) >> maskshift);
-    env->cp15.c2_base_mask = ~((uint32_t)0x3fffu >> maskshift);
+    tcr->raw_tcr = value;
+    tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
+    tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
 }
 
 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -1599,19 +2450,25 @@
 
 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-    env->cp15.c2_base_mask = 0xffffc000u;
-    raw_write(env, ri, 0);
-    env->cp15.c2_mask = 0;
+    TCR *tcr = raw_ptr(env, ri);
+
+    /* Reset both the TCR as well as the masks corresponding to the bank of
+     * the TCR being reset.
+     */
+    tcr->raw_tcr = 0;
+    tcr->mask = 0;
+    tcr->base_mask = 0xffffc000u;
 }
 
 static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                uint64_t value)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
+    TCR *tcr = raw_ptr(env, ri);
 
     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
     tlb_flush(CPU(cpu), 1);
-    raw_write(env, ri, value);
+    tcr->raw_tcr = value;
 }
 
 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -1628,40 +2485,65 @@
     raw_write(env, ri, value);
 }
 
-static const ARMCPRegInfo vmsa_cp_reginfo[] = {
+static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    /* Accesses to VTTBR may change the VMID so we must flush the TLB.  */
+    if (raw_read(env, ri) != value) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                            ARMMMUIdx_S2NS, -1);
+        raw_write(env, ri, value);
+    }
+}
+
+static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
-      .resetfn = arm_cp_reset_ignore, },
+      .access = PL1_RW, .type = ARM_CP_ALIAS,
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
+                             offsetoflow32(CPUARMState, cp15.dfsr_ns) }, },
     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW,
-      .fieldoffset = offsetof(CPUARMState, cp15.ifsr_el2), .resetvalue = 0, },
+      .access = PL1_RW, .resetvalue = 0,
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
+                             offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
+    { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
+      .access = PL1_RW, .resetvalue = 0,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
+                             offsetof(CPUARMState, cp15.dfar_ns) } },
+    { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
+      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
+      .resetvalue = 0, },
+    REGINFO_SENTINEL
+};
+
+static const ARMCPRegInfo vmsa_cp_reginfo[] = {
     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW,
       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
-      .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1),
-      .writefn = vmsa_ttbr_write, .resetvalue = 0 },
+      .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
+      .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
+                             offsetof(CPUARMState, cp15.ttbr0_ns) } },
     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
-      .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el1),
-      .writefn = vmsa_ttbr_write, .resetvalue = 0 },
+      .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
+      .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
+                             offsetof(CPUARMState, cp15.ttbr1_ns) } },
     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
       .access = PL1_RW, .writefn = vmsa_tcr_el1_write,
       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.c2_control) },
+      .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
-      .access = PL1_RW, .type = ARM_CP_NO_MIGRATE, .writefn = vmsa_ttbcr_write,
-      .resetfn = arm_cp_reset_ignore, .raw_writefn = vmsa_ttbcr_raw_write,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.c2_control) },
-    /* 64-bit FAR; this entry also gives us the AArch32 DFAR */
-    { .name = "FAR_EL1", .state = ARM_CP_STATE_BOTH,
-      .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
-      .resetvalue = 0, },
+      .access = PL1_RW, .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
+      .raw_writefn = vmsa_ttbcr_raw_write,
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
+                             offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
     REGINFO_SENTINEL
 };
 
@@ -1720,7 +2602,7 @@
       .writefn = omap_threadid_write },
     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_NO_RAW,
       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
     /* TODO: Peripheral port remap register:
      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
@@ -1729,7 +2611,7 @@
      */
     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
-      .type = ARM_CP_OVERRIDE | ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
       .writefn = omap_cachemaint_write },
     { .name = "C9", .cp = 15, .crn = 9,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
@@ -1779,7 +2661,7 @@
     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
       .access = PL1_RW,
-      .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE | ARM_CP_OVERRIDE,
+      .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
       .resetvalue = 0 },
     REGINFO_SENTINEL
 };
@@ -1787,7 +2669,7 @@
 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
     /* Cache status: RAZ because we have no cache so it's always clean */
     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
-      .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
       .resetvalue = 0 },
     REGINFO_SENTINEL
 };
@@ -1795,7 +2677,7 @@
 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
     /* We never have a a block transfer operation in progress */
     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
-      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
       .resetvalue = 0 },
     /* The cache ops themselves: these all NOP for QEMU */
     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
@@ -1818,10 +2700,10 @@
      * to indicate that there are no dirty cache lines.
      */
     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
-      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
       .resetvalue = (1 << 30) },
     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
-      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
+      .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
       .resetvalue = (1 << 30) },
     REGINFO_SENTINEL
 };
@@ -1831,59 +2713,82 @@
     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
       .access = PL1_RW, .resetvalue = 0,
-      .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_MIGRATE },
+      .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
     REGINFO_SENTINEL
 };
 
-static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 {
-    CPUState *cs = CPU(arm_env_get_cpu(env));
-    uint32_t mpidr = cs->cpu_index;
-    /* We don't support setting cluster ID ([8..11]) (known as Aff1
-     * in later ARM ARM versions), or any of the higher affinity level fields,
-     * so these bits always RAZ.
-     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
+    if (arm_feature(&cpu->env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
+        return env->cp15.vpidr_el2;
+    }
+    return raw_read(env, ri);
+}
+
+static uint64_t mpidr_read_val(CPUARMState *env)
+{
+    ARMCPU *cpu = ARM_CPU(arm_env_get_cpu(env));
+    uint64_t mpidr = cpu->mp_affinity;
+
     if (arm_feature(env, ARM_FEATURE_V7MP)) {
         mpidr |= (1U << 31);
         /* Cores which are uniprocessor (non-coherent)
          * but still implement the MP extensions set
-         * bit 30. (For instance, A9UP.) However we do
-         * not currently model any of those cores.
+         * bit 30. (For instance, Cortex-R5).
          */
+        if (cpu->mp_is_up) {
+            mpidr |= (1u << 30);
+        }
     }
     return mpidr;
 }
 
+static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+    unsigned int cur_el = arm_current_el(env);
+    bool secure = arm_is_secure(env);
+
+    if (arm_feature(env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
+        return env->cp15.vmpidr_el2;
+    }
+    return mpidr_read_val(env);
+}
+
 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
-      .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_MIGRATE },
+      .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
     REGINFO_SENTINEL
 };
 
 static const ARMCPRegInfo lpae_cp_reginfo[] = {
-    /* NOP AMAIR0/1: the override is because these clash with the rather
-     * broadly specified TLB_LOCKDOWN entry in the generic cp_reginfo.
-     */
+    /* NOP AMAIR0/1 */
     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
-      .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
+      .access = PL1_RW, .type = ARM_CP_CONST,
       .resetvalue = 0 },
     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
-      .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
+      .access = PL1_RW, .type = ARM_CP_CONST,
       .resetvalue = 0 },
     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
-      .access = PL1_RW, .type = ARM_CP_64BIT,
-      .fieldoffset = offsetof(CPUARMState, cp15.par_el1), .resetvalue = 0 },
+      .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
+                             offsetof(CPUARMState, cp15.par_ns)} },
     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
-      .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE,
-      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1),
-      .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
+      .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
+                             offsetof(CPUARMState, cp15.ttbr0_ns) },
+      .writefn = vmsa_ttbr_write, },
     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
-      .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE,
-      .fieldoffset = offsetof(CPUARMState, cp15.ttbr1_el1),
-      .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
+      .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
+      .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
+                             offsetof(CPUARMState, cp15.ttbr1_ns) },
+      .writefn = vmsa_ttbr_write, },
     REGINFO_SENTINEL
 };
 
@@ -1909,9 +2814,10 @@
     vfp_set_fpsr(env, value);
 }
 
-static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                       bool isread)
 {
-    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -1924,12 +2830,13 @@
 }
 
 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
-                                          const ARMCPRegInfo *ri)
+                                          const ARMCPRegInfo *ri,
+                                          bool isread)
 {
     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
      * SCTLR_EL1.UCI is set.
      */
-    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UCI)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -1939,74 +2846,254 @@
  * Page D4-1736 (DDI0487A.b)
  */
 
-static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                               uint64_t value)
+static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
 {
-    /* Invalidate by VA (AArch64 version) */
     ARMCPU *cpu = arm_env_get_cpu(env);
-    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+    CPUState *cs = CPU(cpu);
 
-    tlb_flush_page(CPU(cpu), pageaddr);
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+    } else {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
+    }
 }
 
-static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                uint64_t value)
+static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t value)
 {
-    /* Invalidate by VA, all ASIDs (AArch64 version) */
-    ARMCPU *cpu = arm_env_get_cpu(env);
-    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+    bool sec = arm_is_secure_below_el3(env);
+    CPUState *other_cs;
 
-    tlb_flush_page(CPU(cpu), pageaddr);
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+        } else {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, -1);
+        }
+    }
 }
 
-static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
+static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    /* Note that the 'ALL' scope must invalidate both stage 1 and
+     * stage 2 translations, whereas most other scopes only invalidate
+     * stage 1 translations.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+    } else {
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
+                                ARMMMUIdx_S2NS, -1);
+        } else {
+            tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
+}
+
+static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                  uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E3, -1);
+}
+
+static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    /* Note that the 'ALL' scope must invalidate both stage 1 and
+     * stage 2 translations, whereas most other scopes only invalidate
+     * stage 1 translations.
+     */
+    bool sec = arm_is_secure_below_el3(env);
+    bool has_el2 = arm_feature(env, ARM_FEATURE_EL2);
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
+        } else if (has_el2) {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
+        } else {
+            tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
+                                ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
+    }
+}
+
+static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    CPUState *other_cs;
+
+    CPU_FOREACH(other_cs) {
+        tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E3, -1);
+    }
+}
+
+static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
                                  uint64_t value)
 {
-    /* Invalidate by ASID (AArch64 version) */
+    /* Invalidate by VA, EL1&0 (AArch64 version).
+     * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
+     * since we don't support flush-for-specific-ASID-only or
+     * flush-last-level-only.
+     */
     ARMCPU *cpu = arm_env_get_cpu(env);
-    int asid = extract64(value, 48, 16);
-    tlb_flush(CPU(cpu), asid == 0);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    if (arm_is_secure_below_el3(env)) {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1SE1,
+                                 ARMMMUIdx_S1SE0, -1);
+    } else {
+        tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S12NSE1,
+                                 ARMMMUIdx_S12NSE0, -1);
+    }
 }
 
-static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
+{
+    /* Invalidate by VA, EL2
+     * Currently handles both VAE2 and VALE2, since we don't support
+     * flush-last-level-only.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
+}
+
+static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                 uint64_t value)
+{
+    /* Invalidate by VA, EL3
+     * Currently handles both VAE3 and VALE3, since we don't support
+     * flush-last-level-only.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E3, -1);
+}
+
+static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
+{
+    bool sec = arm_is_secure_below_el3(env);
+    CPUState *other_cs;
+    uint64_t pageaddr = sextract64(value << 12, 0, 56);
+
+    CPU_FOREACH(other_cs) {
+        if (sec) {
+            tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1SE1,
+                                     ARMMMUIdx_S1SE0, -1);
+        } else {
+            tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S12NSE1,
+                                     ARMMMUIdx_S12NSE0, -1);
+        }
+    }
+}
+
+static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
     CPUState *other_cs;
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush_page(other_cs, pageaddr);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
     }
 }
 
-static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   uint64_t value)
 {
     CPUState *other_cs;
     uint64_t pageaddr = sextract64(value << 12, 0, 56);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush_page(other_cs, pageaddr);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E3, -1);
     }
 }
 
-static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                                  uint64_t value)
+static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    uint64_t value)
+{
+    /* Invalidate by IPA. This has to invalidate any structures that
+     * contain only stage 2 translation information, but does not need
+     * to apply to structures that contain combined stage 1 and stage 2
+     * translation information.
+     * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
+     */
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 48);
+
+    tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
+}
+
+static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      uint64_t value)
 {
     CPUState *other_cs;
-    int asid = extract64(value, 48, 16);
+    uint64_t pageaddr;
+
+    if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
+        return;
+    }
+
+    pageaddr = sextract64(value << 12, 0, 48);
 
     CPU_FOREACH(other_cs) {
-        tlb_flush(other_cs, asid == 0);
+        tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
     }
 }
 
-static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                      bool isread)
 {
     /* We don't implement EL2, so the only control on DC ZVA is the
      * bit in the SCTLR which can prohibit access for EL0.
      */
-    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_DZE)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
@@ -2018,13 +3105,14 @@
     int dzp_bit = 1 << 4;
 
     /* DZP indicates whether DC ZVA access is allowed */
-    if (aa64_zva_access(env, NULL) == CP_ACCESS_OK) {
+    if (aa64_zva_access(env, NULL, false) == CP_ACCESS_OK) {
         dzp_bit = 0;
     }
     return cpu->dcz_blocksize | dzp_bit;
 }
 
-static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                    bool isread)
 {
     if (!(env->pstate & PSTATE_SP)) {
         /* Access to SP_EL0 is undefined if it's being used as
@@ -2045,6 +3133,42 @@
     update_spsel(env, val);
 }
 
+static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    if (raw_read(env, ri) == value) {
+        /* Skip the TLB flush if nothing actually changed; Linux likes
+         * to do a lot of pointless SCTLR writes.
+         */
+        return;
+    }
+
+    raw_write(env, ri, value);
+    /* ??? Lots of these bits are not implemented.  */
+    /* This may enable/disable the MMU, so do a TLB flush.  */
+    tlb_flush(CPU(cpu), 1);
+}
+
+static CPAccessResult fpexc32_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                     bool isread)
+{
+    if ((env->cp15.cptr_el[2] & CPTR_TFP) && arm_current_el(env) == 2) {
+        return CP_ACCESS_TRAP_FP_EL2;
+    }
+    if (env->cp15.cptr_el[3] & CPTR_TFP) {
+        return CP_ACCESS_TRAP_FP_EL3;
+    }
+    return CP_ACCESS_OK;
+}
+
+static void sdcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                       uint64_t value)
+{
+    env->cp15.mdcr_el3 = value & SDCR_VALID_MASK;
+}
+
 static const ARMCPRegInfo v8_cp_reginfo[] = {
     /* Minimal set of EL0-visible registers. This will need to be expanded
      * significantly for system emulation of AArch64 CPUs.
@@ -2054,7 +3178,7 @@
       .access = PL0_RW, .type = ARM_CP_NZCV },
     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_NO_RAW,
       .access = PL0_RW, .accessfn = aa64_daif_access,
       .fieldoffset = offsetof(CPUARMState, daif),
       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
@@ -2066,7 +3190,7 @@
       .access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
-      .access = PL0_R, .type = ARM_CP_NO_MIGRATE,
+      .access = PL0_R, .type = ARM_CP_NO_RAW,
       .readfn = aa64_dczid_read },
     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
@@ -2117,77 +3241,157 @@
     /* TLBI operations */
     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbiall_is_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_va_is_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_asid_is_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vmalle1is_write },
     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_vaa_is_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_va_is_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_vaa_is_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1is_write },
     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbiall_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_va_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_asid_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vmalle1_write },
     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_vaa_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_va_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1_write },
     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
-      .writefn = tlbi_aa64_vaa_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae1_write },
+    { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1is_write },
+    { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1is_write },
+    { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
+    { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
+    { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1_write },
+    { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_ipas2e1_write },
+    { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1_write },
+    { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle1is_write },
 #ifndef CONFIG_USER_ONLY
     /* 64 bit address translation operations */
     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
-      .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+      .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
+      .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
+    { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
+      .access = PL1_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
+      .writefn = par_write },
 #endif
     /* TLB invalidate last level of translation table walk */
     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_is_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
+      .type = ARM_CP_NO_RAW, .access = PL1_W,
       .writefn = tlbimvaa_is_write },
     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
-      .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+      .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
+    { .name = "TLBIMVALH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbimva_hyp_write },
+    { .name = "TLBIMVALHIS",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbimva_hyp_is_write },
+    { .name = "TLBIIPAS2",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiipas2_write },
+    { .name = "TLBIIPAS2IS",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiipas2_is_write },
+    { .name = "TLBIIPAS2L",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiipas2_write },
+    { .name = "TLBIIPAS2LIS",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiipas2_is_write },
     /* 32 bit cache operations */
     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
       .type = ARM_CP_NOP, .access = PL1_W },
@@ -2216,19 +3420,21 @@
     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
       .type = ARM_CP_NOP, .access = PL1_W },
     /* MMU Domain access control / MPU write buffer control */
-    { .name = "DACR", .cp = 15,
-      .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c3),
-      .resetvalue = 0, .writefn = dacr_write, .raw_writefn = raw_write, },
+    { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
+      .access = PL1_RW, .resetvalue = 0,
+      .writefn = dacr_write, .raw_writefn = raw_write,
+      .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
+                             offsetoflow32(CPUARMState, cp15.dacr_ns) } },
     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
       .access = PL1_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[0]) },
+      .access = PL1_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
     /* We rely on the access checks not allowing the guest to write to the
      * state field when SPSel indicates that it's being used as the stack
      * pointer.
@@ -2236,26 +3442,157 @@
     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
       .access = PL1_RW, .accessfn = sp_el0_access,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
+    { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_NO_RAW,
       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
+    { .name = "FPEXC32_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 3, .opc2 = 0,
+      .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, vfp.xregs[ARM_VFP_FPEXC]),
+      .access = PL2_RW, .accessfn = fpexc32_access },
+    { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .resetvalue = 0,
+      .writefn = dacr_write, .raw_writefn = raw_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
+    { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
+    { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
+    { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
+    { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
+    { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_ALIAS,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
+    { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
+      .resetvalue = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
+    { .name = "SDCR", .type = ARM_CP_ALIAS,
+      .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
+      .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
+      .writefn = sdcr_write,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.mdcr_el3) },
     REGINFO_SENTINEL
 };
 
 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
-static const ARMCPRegInfo v8_el3_no_el2_cp_reginfo[] = {
+static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL2_RW,
       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_NO_RAW,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
       .access = PL2_RW,
       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
+    { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "VTCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 6, .crm = 2,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .type = ARM_CP_CONST | ARM_CP_64BIT, .resetvalue = 0 },
+    { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .accessfn = access_tda,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "HPFAR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
+    { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
     REGINFO_SENTINEL
 };
 
@@ -2284,94 +3621,392 @@
     raw_write(env, ri, value);
 }
 
-static const ARMCPRegInfo v8_el2_cp_reginfo[] = {
+static const ARMCPRegInfo el2_cp_reginfo[] = {
     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
       .writefn = hcr_write },
     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
       .access = PL2_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL2_RW, .writefn = vbar_write,
       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
       .resetvalue = 0 },
+    { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
+    { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
+      .access = PL2_RW, .accessfn = cptr_access, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[2]) },
+    { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
+      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[2]),
+      .resetvalue = 0 },
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_ALIAS,
+      .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
+    { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
+    { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
+      .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
+      .access = PL2_RW,
+      /* no .writefn needed as this can't cause an ASID change;
+       * no .raw_writefn or .resetfn needed as we never use mask/base_mask
+       */
+      .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
+    { .name = "VTCR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .type = ARM_CP_ALIAS,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
+    { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
+      .access = PL2_RW,
+      /* no .writefn needed as this can't cause an ASID change;
+       * no .raw_writefn or .resetfn needed as we never use mask/base_mask
+       */
+      .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
+    { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 6, .crm = 2,
+      .type = ARM_CP_64BIT | ARM_CP_ALIAS,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
+      .writefn = vttbr_write },
+    { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
+      .access = PL2_RW, .writefn = vttbr_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
+    { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[2]) },
+    { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
+    { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
+    { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
+      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
+    { .name = "TLBIALLNSNH",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiall_nsnh_write },
+    { .name = "TLBIALLNSNHIS",
+      .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiall_nsnh_is_write },
+    { .name = "TLBIALLH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiall_hyp_write },
+    { .name = "TLBIALLHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbiall_hyp_is_write },
+    { .name = "TLBIMVAH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbimva_hyp_write },
+    { .name = "TLBIMVAHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbimva_hyp_is_write },
+    { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbi_aa64_alle2_write },
+    { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbi_aa64_vae2_write },
+    { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae2_write },
+    { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle2is_write },
+    { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
+      .type = ARM_CP_NO_RAW, .access = PL2_W,
+      .writefn = tlbi_aa64_vae2is_write },
+    { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
+      .access = PL2_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae2is_write },
+#ifndef CONFIG_USER_ONLY
+    /* Unlike the other EL2-related AT operations, these must
+     * UNDEF from EL3 if EL2 is not implemented, which is why we
+     * define them here rather than with the rest of the AT ops.
+     */
+    { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL2_W, .accessfn = at_s1e2_access,
+      .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL2_W, .accessfn = at_s1e2_access,
+      .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
+    /* The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
+     * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
+     * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
+     * to behave as if SCR.NS was 1.
+     */
+    { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
+      .access = PL2_W,
+      .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
+    { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
+      .access = PL2_W,
+      .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
+    { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
+      /* ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
+       * reset values as IMPDEF. We choose to reset to 3 to comply with
+       * both ARMv7 and ARMv8.
+       */
+      .access = PL2_RW, .resetvalue = 3,
+      .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
+    { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
+      .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
+      .writefn = gt_cntvoff_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
+    { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
+      .writefn = gt_cntvoff_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
+    { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
+      .type = ARM_CP_IO, .access = PL2_RW,
+      .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
+    { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
+      .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
+      .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
+    { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
+      .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
+      .resetfn = gt_hyp_timer_reset,
+      .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
+    { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
+      .type = ARM_CP_IO,
+      .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
+      .resetvalue = 0,
+      .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
+#endif
+    /* The only field of MDCR_EL2 that has a defined architectural reset value
+     * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
+     * don't impelment any PMU event counters, so using zero as a reset
+     * value for MDCR_EL2 is okay
+     */
+    { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL2_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
+    { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
+      .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW, .accessfn = access_el3_aa32ns,
+      .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
+    { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
+    { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
+      .cp = 15, .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, cp15.hstr_el2) },
     REGINFO_SENTINEL
 };
 
-static const ARMCPRegInfo v8_el3_cp_reginfo[] = {
+static CPAccessResult nsacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                   bool isread)
+{
+    /* The NSACR is RW at EL3, and RO for NS EL1 and NS EL2.
+     * At Secure EL1 it traps to EL3.
+     */
+    if (arm_current_el(env) == 3) {
+        return CP_ACCESS_OK;
+    }
+    if (arm_is_secure_below_el3(env)) {
+        return CP_ACCESS_TRAP_EL3;
+    }
+    /* Accesses from EL1 NS and EL2 NS are UNDEF for write but allow reads. */
+    if (isread) {
+        return CP_ACCESS_OK;
+    }
+    return CP_ACCESS_TRAP_UNCATEGORIZED;
+}
+
+static const ARMCPRegInfo el3_cp_reginfo[] = {
+    { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
+      .resetvalue = 0, .writefn = scr_write },
+    { .name = "SCR",  .type = ARM_CP_ALIAS,
+      .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
+      .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
+      .writefn = scr_write },
+    { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL3_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.sder) },
+    { .name = "SDER",
+      .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
+      .access = PL3_RW, .resetvalue = 0,
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
+    { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
+      .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
+      .writefn = vbar_write, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
+    { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
+      .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
+    { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
+      .access = PL3_RW,
+      /* no .writefn needed as this can't cause an ASID change;
+       * we must provide a .raw_writefn and .resetfn because we handle
+       * reset and migration for the AArch32 TTBCR(S), which might be
+       * using mask and base_mask.
+       */
+      .resetfn = vmsa_ttbcr_reset, .raw_writefn = vmsa_ttbcr_raw_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
       .access = PL3_RW,
       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
+      .type = ARM_CP_ALIAS,
       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
-      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
+      .access = PL3_RW,
+      .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
       .access = PL3_RW, .writefn = vbar_write,
       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
       .resetvalue = 0 },
-    { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
-      .type = ARM_CP_NO_MIGRATE,
-      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
-      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
-      .writefn = scr_write },
+    { .name = "CPTR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
+      .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
+    { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
+      .access = PL3_RW, .resetvalue = 0,
+      .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
+    { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
+      .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
+      .access = PL3_RW, .type = ARM_CP_CONST,
+      .resetvalue = 0 },
+    { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle3is_write },
+    { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3is_write },
+    { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3is_write },
+    { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_alle3_write },
+    { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3_write },
+    { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
+      .access = PL3_W, .type = ARM_CP_NO_RAW,
+      .writefn = tlbi_aa64_vae3_write },
     REGINFO_SENTINEL
 };
 
-static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
-                        uint64_t value)
-{
-    ARMCPU *cpu = arm_env_get_cpu(env);
-
-    if (raw_read(env, ri) == value) {
-        /* Skip the TLB flush if nothing actually changed; Linux likes
-         * to do a lot of pointless SCTLR writes.
-         */
-        return;
-    }
-
-    raw_write(env, ri, value);
-    /* ??? Lots of these bits are not implemented.  */
-    /* This may enable/disable the MMU, so do a TLB flush.  */
-    tlb_flush(CPU(cpu), 1);
-}
-
-static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
+static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
+                                     bool isread)
 {
     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
      * but the AArch32 CTR has its own reginfo struct)
      */
-    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UCT)) {
+    if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
         return CP_ACCESS_TRAP;
     }
     return CP_ACCESS_OK;
 }
 
+static void oslar_write(CPUARMState *env, const ARMCPRegInfo *ri,
+                        uint64_t value)
+{
+    /* Writes to OSLAR_EL1 may update the OS lock status, which can be
+     * read via a bit in OSLSR_EL1.
+     */
+    int oslock;
+
+    if (ri->state == ARM_CP_STATE_AA32) {
+        oslock = (value == 0xC5ACCE55);
+    } else {
+        oslock = value & 1;
+    }
+
+    env->cp15.oslsr_el1 = deposit32(env->cp15.oslsr_el1, 1, 1, oslock);
+}
+
 static const ARMCPRegInfo debug_cp_reginfo[] = {
     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
@@ -2380,16 +4015,19 @@
      * accessor.
      */
     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL0_R, .accessfn = access_tdra,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
-      .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL1_R, .accessfn = access_tdra,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
-      .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+      .access = PL0_R, .accessfn = access_tdra,
+      .type = ARM_CP_CONST, .resetvalue = 0 },
     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
-      .access = PL1_RW,
+      .access = PL1_RW, .accessfn = access_tda,
       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
       .resetvalue = 0 },
     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
@@ -2397,24 +4035,31 @@
      */
     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
-      .type = ARM_CP_NO_MIGRATE,
-      .access = PL1_R,
-      .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
-      .resetfn = arm_cp_reset_ignore },
-    /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
+      .type = ARM_CP_ALIAS,
+      .access = PL1_R, .accessfn = access_tda,
+      .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1), },
     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
-      .access = PL1_W, .type = ARM_CP_NOP },
+      .access = PL1_W, .type = ARM_CP_NO_RAW,
+      .accessfn = access_tdosa,
+      .writefn = oslar_write },
+    { .name = "OSLSR_EL1", .state = ARM_CP_STATE_BOTH,
+      .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 4,
+      .access = PL1_R, .resetvalue = 10,
+      .accessfn = access_tdosa,
+      .fieldoffset = offsetof(CPUARMState, cp15.oslsr_el1) },
     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
-      .access = PL1_RW, .type = ARM_CP_NOP },
+      .access = PL1_RW, .accessfn = access_tdosa,
+      .type = ARM_CP_NOP },
     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
      * implement vector catch debug events yet.
      */
     { .name = "DBGVCR",
       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
-      .access = PL1_RW, .type = ARM_CP_NOP },
+      .access = PL1_RW, .accessfn = access_tda,
+      .type = ARM_CP_NOP },
     REGINFO_SENTINEL
 };
 
@@ -2679,7 +4324,8 @@
     int wrps, brps, ctx_cmps;
     ARMCPRegInfo dbgdidr = {
         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
-        .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
+        .access = PL0_R, .accessfn = access_tda,
+        .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
     };
 
     /* Note that all these register fields hold "number of Xs minus 1". */
@@ -2711,13 +4357,13 @@
         ARMCPRegInfo dbgregs[] = {
             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
               .writefn = dbgbvr_write, .raw_writefn = raw_write
             },
             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
               .writefn = dbgbcr_write, .raw_writefn = raw_write
             },
@@ -2730,13 +4376,13 @@
         ARMCPRegInfo dbgregs[] = {
             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
               .writefn = dbgwvr_write, .raw_writefn = raw_write
             },
             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
-              .access = PL1_RW,
+              .access = PL1_RW, .accessfn = access_tda,
               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
               .writefn = dbgwcr_write, .raw_writefn = raw_write
             },
@@ -2822,12 +4468,14 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_isar5 },
-            /* 6..7 are as yet unallocated and must RAZ */
-            { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
-              .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
-              .resetvalue = 0 },
-            { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
-              .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
+            { .name = "ID_MMFR4", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = cpu->id_mmfr4 },
+            /* 7 is as yet unallocated and must RAZ */
+            { .name = "ID_ISAR7_RESERVED", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = 0 },
             REGINFO_SENTINEL
         };
@@ -2839,7 +4487,8 @@
     if (arm_feature(env, ARM_FEATURE_V6K)) {
         define_arm_cp_regs(cpu, v6k_cp_reginfo);
     }
-    if (arm_feature(env, ARM_FEATURE_V7MP)) {
+    if (arm_feature(env, ARM_FEATURE_V7MP) &&
+        !arm_feature(env, ARM_FEATURE_MPU)) {
         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_V7)) {
@@ -2851,7 +4500,7 @@
         ARMCPRegInfo pmcr = {
             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
             .access = PL0_RW,
-            .type = ARM_CP_IO | ARM_CP_NO_MIGRATE,
+            .type = ARM_CP_IO | ARM_CP_ALIAS,
             .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
             .accessfn = pmreg_access, .writefn = pmcr_write,
             .raw_writefn = raw_write,
@@ -2880,7 +4529,11 @@
         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_V8)) {
-        /* AArch64 ID registers, which all have impdef reset values */
+        /* AArch64 ID registers, which all have impdef reset values.
+         * Note that within the ID register ranges the unused slots
+         * must all RAZ, not UNDEF; future architecture versions may
+         * define new registers here.
+         */
         ARMCPRegInfo v8_idregs[] = {
             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
@@ -2890,6 +4543,30 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64pfr1},
+            { .name = "ID_AA64PFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64PFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -2903,6 +4580,14 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64dfr1 },
+            { .name = "ID_AA64DFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64DFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -2911,6 +4596,14 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64afr1 },
+            { .name = "ID_AA64AFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64AFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -2919,6 +4612,30 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64isar1 },
+            { .name = "ID_AA64ISAR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64ISAR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -2927,6 +4644,30 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->id_aa64mmfr1 },
+            { .name = "ID_AA64MMFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 2,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ID_AA64MMFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
               .access = PL1_R, .type = ARM_CP_CONST,
@@ -2939,39 +4680,184 @@
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
               .access = PL1_R, .type = ARM_CP_CONST,
               .resetvalue = cpu->mvfr2 },
+            { .name = "MVFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 5,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 6,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "MVFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 7,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "PMCEID0", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 6,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid0 },
+            { .name = "PMCEID0_EL0", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 6,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid0 },
+            { .name = "PMCEID1", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 7,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid1 },
+            { .name = "PMCEID1_EL0", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 7,
+              .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmceid1 },
             REGINFO_SENTINEL
         };
-        ARMCPRegInfo rvbar = {
-            .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
-            .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
-            .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
-        };
-        define_one_arm_cp_reg(cpu, &rvbar);
+        /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
+        if (!arm_feature(env, ARM_FEATURE_EL3) &&
+            !arm_feature(env, ARM_FEATURE_EL2)) {
+            ARMCPRegInfo rvbar = {
+                .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
+                .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
+                .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
+            };
+            define_one_arm_cp_reg(cpu, &rvbar);
+        }
         define_arm_cp_regs(cpu, v8_idregs);
         define_arm_cp_regs(cpu, v8_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_EL2)) {
-        define_arm_cp_regs(cpu, v8_el2_cp_reginfo);
+        uint64_t vmpidr_def = mpidr_read_val(env);
+        ARMCPRegInfo vpidr_regs[] = {
+            { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+              .access = PL2_RW, .accessfn = access_el3_aa32ns,
+              .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+            { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+              .access = PL2_RW, .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+            { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
+              .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+              .access = PL2_RW, .accessfn = access_el3_aa32ns,
+              .resetvalue = vmpidr_def,
+              .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
+            { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+              .access = PL2_RW,
+              .resetvalue = vmpidr_def,
+              .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
+            REGINFO_SENTINEL
+        };
+        define_arm_cp_regs(cpu, vpidr_regs);
+        define_arm_cp_regs(cpu, el2_cp_reginfo);
+        /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
+        if (!arm_feature(env, ARM_FEATURE_EL3)) {
+            ARMCPRegInfo rvbar = {
+                .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
+                .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
+                .type = ARM_CP_CONST, .access = PL2_R, .resetvalue = cpu->rvbar
+            };
+            define_one_arm_cp_reg(cpu, &rvbar);
+        }
     } else {
         /* If EL2 is missing but higher ELs are enabled, we need to
          * register the no_el2 reginfos.
          */
         if (arm_feature(env, ARM_FEATURE_EL3)) {
-            define_arm_cp_regs(cpu, v8_el3_no_el2_cp_reginfo);
+            /* When EL3 exists but not EL2, VPIDR and VMPIDR take the value
+             * of MIDR_EL1 and MPIDR_EL1.
+             */
+            ARMCPRegInfo vpidr_regs[] = {
+                { .name = "VPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+                  .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
+                  .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+                  .type = ARM_CP_CONST, .resetvalue = cpu->midr,
+                  .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
+                { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_BOTH,
+                  .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
+                  .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
+                  .type = ARM_CP_NO_RAW,
+                  .writefn = arm_cp_write_ignore, .readfn = mpidr_read },
+                REGINFO_SENTINEL
+            };
+            define_arm_cp_regs(cpu, vpidr_regs);
+            define_arm_cp_regs(cpu, el3_no_el2_cp_reginfo);
         }
     }
     if (arm_feature(env, ARM_FEATURE_EL3)) {
-        define_arm_cp_regs(cpu, v8_el3_cp_reginfo);
+        define_arm_cp_regs(cpu, el3_cp_reginfo);
+        ARMCPRegInfo el3_regs[] = {
+            { .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
+              .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar },
+            { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
+              .access = PL3_RW,
+              .raw_writefn = raw_write, .writefn = sctlr_write,
+              .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]),
+              .resetvalue = cpu->reset_sctlr },
+            REGINFO_SENTINEL
+        };
+
+        define_arm_cp_regs(cpu, el3_regs);
     }
-    if (arm_feature(env, ARM_FEATURE_MPU)) {
-        /* These are the MPU registers prior to PMSAv6. Any new
-         * PMSA core later than the ARM946 will require that we
-         * implement the PMSAv6 or PMSAv7 registers, which are
-         * completely different.
-         */
-        assert(!arm_feature(env, ARM_FEATURE_V6));
-        define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
+    /* The behaviour of NSACR is sufficiently various that we don't
+     * try to describe it in a single reginfo:
+     *  if EL3 is 64 bit, then trap to EL3 from S EL1,
+     *     reads as constant 0xc00 from NS EL1 and NS EL2
+     *  if EL3 is 32 bit, then RW at EL3, RO at NS EL1 and NS EL2
+     *  if v7 without EL3, register doesn't exist
+     *  if v8 without EL3, reads as constant 0xc00 from NS EL1 and NS EL2
+     */
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+            ARMCPRegInfo nsacr = {
+                .name = "NSACR", .type = ARM_CP_CONST,
+                .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
+                .access = PL1_RW, .accessfn = nsacr_access,
+                .resetvalue = 0xc00
+            };
+            define_one_arm_cp_reg(cpu, &nsacr);
+        } else {
+            ARMCPRegInfo nsacr = {
+                .name = "NSACR",
+                .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
+                .access = PL3_RW | PL1_R,
+                .resetvalue = 0,
+                .fieldoffset = offsetof(CPUARMState, cp15.nsacr)
+            };
+            define_one_arm_cp_reg(cpu, &nsacr);
+        }
     } else {
+        if (arm_feature(env, ARM_FEATURE_V8)) {
+            ARMCPRegInfo nsacr = {
+                .name = "NSACR", .type = ARM_CP_CONST,
+                .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
+                .access = PL1_R,
+                .resetvalue = 0xc00
+            };
+            define_one_arm_cp_reg(cpu, &nsacr);
+        }
+    }
+
+    if (arm_feature(env, ARM_FEATURE_MPU)) {
+        if (arm_feature(env, ARM_FEATURE_V6)) {
+            /* PMSAv6 not implemented */
+            assert(arm_feature(env, ARM_FEATURE_V7));
+            define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
+            define_arm_cp_regs(cpu, pmsav7_cp_reginfo);
+        } else {
+            define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
+        }
+    } else {
+        define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
     }
     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
@@ -3026,6 +4912,7 @@
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
               .access = PL1_R, .resetvalue = cpu->midr,
               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
+              .readfn = midr_read,
               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
               .type = ARM_CP_OVERRIDE },
             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
@@ -3047,16 +4934,21 @@
             REGINFO_SENTINEL
         };
         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
-            /* v8 MIDR -- the wildcard isn't necessary, and nor is the
-             * variable-MIDR TI925 behaviour. Instead we have a single
-             * (strictly speaking IMPDEF) alias of the MIDR, REVIDR.
-             */
             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
-              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
+              .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
+              .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
+              .readfn = midr_read },
+            /* crn = 0 op1 = 0 crm = 0 op2 = 4,7 : AArch32 aliases of MIDR */
+            { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
+              .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
+              .access = PL1_R, .resetvalue = cpu->midr },
+            { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
+              .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 7,
+              .access = PL1_R, .resetvalue = cpu->midr },
             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
-              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
+              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->revidr },
             REGINFO_SENTINEL
         };
         ARMCPRegInfo id_cp_reginfo[] = {
@@ -3072,11 +4964,21 @@
             { .name = "TCMTR",
               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
-            { .name = "TLBTR",
-              .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
-              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
             REGINFO_SENTINEL
         };
+        /* TLBTR is specific to VMSA */
+        ARMCPRegInfo id_tlbtr_reginfo = {
+              .name = "TLBTR",
+              .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
+              .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0,
+        };
+        /* MPUIR is specific to PMSA V6+ */
+        ARMCPRegInfo id_mpuir_reginfo = {
+              .name = "MPUIR",
+              .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
+              .access = PL1_R, .type = ARM_CP_CONST,
+              .resetvalue = cpu->pmsav7_dregion << 8
+        };
         ARMCPRegInfo crn0_wi_reginfo = {
             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
@@ -3098,6 +5000,8 @@
             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
                 r->access = PL1_RW;
             }
+            id_tlbtr_reginfo.access = PL1_RW;
+            id_tlbtr_reginfo.access = PL1_RW;
         }
         if (arm_feature(env, ARM_FEATURE_V8)) {
             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
@@ -3105,6 +5009,11 @@
             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
         }
         define_arm_cp_regs(cpu, id_cp_reginfo);
+        if (!arm_feature(env, ARM_FEATURE_MPU)) {
+            define_one_arm_cp_reg(cpu, &id_tlbtr_reginfo);
+        } else if (arm_feature(env, ARM_FEATURE_V7)) {
+            define_one_arm_cp_reg(cpu, &id_mpuir_reginfo);
+        }
     }
 
     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
@@ -3112,13 +5021,22 @@
     }
 
     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
-        ARMCPRegInfo auxcr = {
-            .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
-            .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
-            .access = PL1_RW, .type = ARM_CP_CONST,
-            .resetvalue = cpu->reset_auxcr
+        ARMCPRegInfo auxcr_reginfo[] = {
+            { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL1_RW, .type = ARM_CP_CONST,
+              .resetvalue = cpu->reset_auxcr },
+            { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
+              .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL2_RW, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
+              .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
+              .access = PL3_RW, .type = ARM_CP_CONST,
+              .resetvalue = 0 },
+            REGINFO_SENTINEL
         };
-        define_one_arm_cp_reg(cpu, &auxcr);
+        define_arm_cp_regs(cpu, auxcr_reginfo);
     }
 
     if (arm_feature(env, ARM_FEATURE_CBAR)) {
@@ -3161,8 +5079,10 @@
     {
         ARMCPRegInfo sctlr = {
             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
-            .opc0 = 3, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
-            .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_sys),
+            .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
+            .access = PL1_RW,
+            .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
+                                   offsetof(CPUARMState, cp15.sctlr_ns) },
             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
             .raw_writefn = raw_write,
         };
@@ -3288,7 +5208,7 @@
 }
 
 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
-                                   void *opaque, int state,
+                                   void *opaque, int state, int secstate,
                                    int crm, int opc1, int opc2)
 {
     /* Private utility function for define_one_arm_cp_reg_with_opaque():
@@ -3297,22 +5217,57 @@
     uint32_t *key = g_new(uint32_t, 1);
     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
-    if (r->state == ARM_CP_STATE_BOTH && state == ARM_CP_STATE_AA32) {
-        /* The AArch32 view of a shared register sees the lower 32 bits
-         * of a 64 bit backing field. It is not migratable as the AArch64
-         * view handles that. AArch64 also handles reset.
-         * We assume it is a cp15 register if the .cp field is left unset.
+    int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
+
+    /* Reset the secure state to the specific incoming state.  This is
+     * necessary as the register may have been defined with both states.
+     */
+    r2->secure = secstate;
+
+    if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
+        /* Register is banked (using both entries in array).
+         * Overwriting fieldoffset as the array is only used to define
+         * banked registers but later only fieldoffset is used.
          */
-        if (r2->cp == 0) {
-            r2->cp = 15;
+        r2->fieldoffset = r->bank_fieldoffsets[ns];
+    }
+
+    if (state == ARM_CP_STATE_AA32) {
+        if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
+            /* If the register is banked then we don't need to migrate or
+             * reset the 32-bit instance in certain cases:
+             *
+             * 1) If the register has both 32-bit and 64-bit instances then we
+             *    can count on the 64-bit instance taking care of the
+             *    non-secure bank.
+             * 2) If ARMv8 is enabled then we can count on a 64-bit version
+             *    taking care of the secure bank.  This requires that separate
+             *    32 and 64-bit definitions are provided.
+             */
+            if ((r->state == ARM_CP_STATE_BOTH && ns) ||
+                (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
+                r2->type |= ARM_CP_ALIAS;
+            }
+        } else if ((secstate != r->secure) && !ns) {
+            /* The register is not banked so we only want to allow migration of
+             * the non-secure instance.
+             */
+            r2->type |= ARM_CP_ALIAS;
         }
-        r2->type |= ARM_CP_NO_MIGRATE;
-        r2->resetfn = arm_cp_reset_ignore;
+
+        if (r->state == ARM_CP_STATE_BOTH) {
+            /* We assume it is a cp15 register if the .cp field is left unset.
+             */
+            if (r2->cp == 0) {
+                r2->cp = 15;
+            }
+
 #ifdef HOST_WORDS_BIGENDIAN
-        if (r2->fieldoffset) {
-            r2->fieldoffset += sizeof(uint32_t);
-        }
+            if (r2->fieldoffset) {
+                r2->fieldoffset += sizeof(uint32_t);
+            }
 #endif
+        }
     }
     if (state == ARM_CP_STATE_AA64) {
         /* To allow abbreviation of ARMCPRegInfo
@@ -3328,7 +5283,7 @@
         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
                                   r2->opc0, opc1, opc2);
     } else {
-        *key = ENCODE_CP_REG(r2->cp, is64, r2->crn, crm, opc1, opc2);
+        *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
     }
     if (opaque) {
         r2->opaque = opaque;
@@ -3345,15 +5300,25 @@
     r2->opc2 = opc2;
     /* By convention, for wildcarded registers only the first
      * entry is used for migration; the others are marked as
-     * NO_MIGRATE so we don't try to transfer the register
+     * ALIAS so we don't try to transfer the register
      * multiple times. Special registers (ie NOP/WFI) are
-     * never migratable.
+     * never migratable and not even raw-accessible.
      */
-    if ((r->type & ARM_CP_SPECIAL) ||
-        ((r->crm == CP_ANY) && crm != 0) ||
+    if ((r->type & ARM_CP_SPECIAL)) {
+        r2->type |= ARM_CP_NO_RAW;
+    }
+    if (((r->crm == CP_ANY) && crm != 0) ||
         ((r->opc1 == CP_ANY) && opc1 != 0) ||
         ((r->opc2 == CP_ANY) && opc2 != 0)) {
-        r2->type |= ARM_CP_NO_MIGRATE;
+        r2->type |= ARM_CP_ALIAS;
+    }
+
+    /* Check that raw accesses are either forbidden or handled. Note that
+     * we can't assert this earlier because the setup of fieldoffset for
+     * banked registers has to be done first.
+     */
+    if (!(r2->type & ARM_CP_NO_RAW)) {
+        assert(!raw_accessors_invalid(r2));
     }
 
     /* Overriding of an existing definition must be explicitly
@@ -3462,10 +5427,14 @@
      */
     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
         if (r->access & PL3_R) {
-            assert(r->fieldoffset || r->readfn);
+            assert((r->fieldoffset ||
+                   (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
+                   r->readfn);
         }
         if (r->access & PL3_W) {
-            assert(r->fieldoffset || r->writefn);
+            assert((r->fieldoffset ||
+                   (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
+                   r->writefn);
         }
     }
     /* Bad type field probably means missing sentinel at end of reg list */
@@ -3478,8 +5447,32 @@
                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
                         continue;
                     }
-                    add_cpreg_to_hashtable(cpu, r, opaque, state,
-                                           crm, opc1, opc2);
+                    if (state == ARM_CP_STATE_AA32) {
+                        /* Under AArch32 CP registers can be common
+                         * (same for secure and non-secure world) or banked.
+                         */
+                        switch (r->secure) {
+                        case ARM_CP_SECSTATE_S:
+                        case ARM_CP_SECSTATE_NS:
+                            add_cpreg_to_hashtable(cpu, r, opaque, state,
+                                                   r->secure, crm, opc1, opc2);
+                            break;
+                        default:
+                            add_cpreg_to_hashtable(cpu, r, opaque, state,
+                                                   ARM_CP_SECSTATE_S,
+                                                   crm, opc1, opc2);
+                            add_cpreg_to_hashtable(cpu, r, opaque, state,
+                                                   ARM_CP_SECSTATE_NS,
+                                                   crm, opc1, opc2);
+                            break;
+                        }
+                    } else {
+                        /* AArch64 registers get mapped to non-secure instance
+                         * of AArch32 */
+                        add_cpreg_to_hashtable(cpu, r, opaque, state,
+                                               ARM_CP_SECSTATE_NS,
+                                               crm, opc1, opc2);
+                    }
                 }
             }
         }
@@ -3518,23 +5511,47 @@
     /* Helper coprocessor reset function for do-nothing-on-reset registers */
 }
 
-static int bad_mode_switch(CPUARMState *env, int mode)
+static int bad_mode_switch(CPUARMState *env, int mode, CPSRWriteType write_type)
 {
     /* Return true if it is not valid for us to switch to
      * this CPU mode (ie all the UNPREDICTABLE cases in
      * the ARM ARM CPSRWriteByInstr pseudocode).
      */
+
+    /* Changes to or from Hyp via MSR and CPS are illegal. */
+    if (write_type == CPSRWriteByInstr &&
+        ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_HYP ||
+         mode == ARM_CPU_MODE_HYP)) {
+        return 1;
+    }
+
     switch (mode) {
     case ARM_CPU_MODE_USR:
+        return 0;
     case ARM_CPU_MODE_SYS:
     case ARM_CPU_MODE_SVC:
     case ARM_CPU_MODE_ABT:
     case ARM_CPU_MODE_UND:
     case ARM_CPU_MODE_IRQ:
     case ARM_CPU_MODE_FIQ:
+        /* Note that we don't implement the IMPDEF NSACR.RFR which in v7
+         * allows FIQ mode to be Secure-only. (In v8 this doesn't exist.)
+         */
+        /* If HCR.TGE is set then changes from Monitor to NS PL1 via MSR
+         * and CPS are treated as illegal mode changes.
+         */
+        if (write_type == CPSRWriteByInstr &&
+            (env->cp15.hcr_el2 & HCR_TGE) &&
+            (env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON &&
+            !arm_is_secure_below_el3(env)) {
+            return 1;
+        }
         return 0;
+    case ARM_CPU_MODE_HYP:
+        return !arm_feature(env, ARM_FEATURE_EL2)
+            || arm_current_el(env) < 2 || arm_is_secure(env);
     case ARM_CPU_MODE_MON:
-        return !arm_is_secure(env);
+        return arm_current_el(env) < 3;
     default:
         return 1;
     }
@@ -3551,8 +5568,11 @@
         | (env->GE << 16) | (env->daif & CPSR_AIF);
 }
 
-void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
+void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
+                CPSRWriteType write_type)
 {
+    uint32_t changed_daif;
+
     if (mask & CPSR_NZCV) {
         env->ZF = (~val) & CPSR_Z;
         env->NF = val;
@@ -3575,16 +5595,86 @@
         env->GE = (val >> 16) & 0xf;
     }
 
+    /* In a V7 implementation that includes the security extensions but does
+     * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
+     * whether non-secure software is allowed to change the CPSR_F and CPSR_A
+     * bits respectively.
+     *
+     * In a V8 implementation, it is permitted for privileged software to
+     * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
+     */
+    if (write_type != CPSRWriteRaw && !arm_feature(env, ARM_FEATURE_V8) &&
+        arm_feature(env, ARM_FEATURE_EL3) &&
+        !arm_feature(env, ARM_FEATURE_EL2) &&
+        !arm_is_secure(env)) {
+
+        changed_daif = (env->daif ^ val) & mask;
+
+        if (changed_daif & CPSR_A) {
+            /* Check to see if we are allowed to change the masking of async
+             * abort exceptions from a non-secure state.
+             */
+            if (!(env->cp15.scr_el3 & SCR_AW)) {
+                qemu_log_mask(LOG_GUEST_ERROR,
+                              "Ignoring attempt to switch CPSR_A flag from "
+                              "non-secure world with SCR.AW bit clear\n");
+                mask &= ~CPSR_A;
+            }
+        }
+
+        if (changed_daif & CPSR_F) {
+            /* Check to see if we are allowed to change the masking of FIQ
+             * exceptions from a non-secure state.
+             */
+            if (!(env->cp15.scr_el3 & SCR_FW)) {
+                qemu_log_mask(LOG_GUEST_ERROR,
+                              "Ignoring attempt to switch CPSR_F flag from "
+                              "non-secure world with SCR.FW bit clear\n");
+                mask &= ~CPSR_F;
+            }
+
+            /* Check whether non-maskable FIQ (NMFI) support is enabled.
+             * If this bit is set software is not allowed to mask
+             * FIQs, but is allowed to set CPSR_F to 0.
+             */
+            if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
+                (val & CPSR_F)) {
+                qemu_log_mask(LOG_GUEST_ERROR,
+                              "Ignoring attempt to enable CPSR_F flag "
+                              "(non-maskable FIQ [NMFI] support enabled)\n");
+                mask &= ~CPSR_F;
+            }
+        }
+    }
+
     env->daif &= ~(CPSR_AIF & mask);
     env->daif |= val & CPSR_AIF & mask;
 
-    if ((env->uncached_cpsr ^ val) & mask & CPSR_M) {
-        if (bad_mode_switch(env, val & CPSR_M)) {
-            /* Attempt to switch to an invalid mode: this is UNPREDICTABLE.
-             * We choose to ignore the attempt and leave the CPSR M field
-             * untouched.
+    if (write_type != CPSRWriteRaw &&
+        ((env->uncached_cpsr ^ val) & mask & CPSR_M)) {
+        if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR) {
+            /* Note that we can only get here in USR mode if this is a
+             * gdb stub write; for this case we follow the architectural
+             * behaviour for guest writes in USR mode of ignoring an attempt
+             * to switch mode. (Those are caught by translate.c for writes
+             * triggered by guest instructions.)
              */
             mask &= ~CPSR_M;
+        } else if (bad_mode_switch(env, val & CPSR_M, write_type)) {
+            /* Attempt to switch to an invalid mode: this is UNPREDICTABLE in
+             * v7, and has defined behaviour in v8:
+             *  + leave CPSR.M untouched
+             *  + allow changes to the other CPSR fields
+             *  + set PSTATE.IL
+             * For user changes via the GDB stub, we don't set PSTATE.IL,
+             * as this would be unnecessarily harsh for a user error.
+             */
+            mask &= ~CPSR_M;
+            if (write_type != CPSRWriteByGDBStub &&
+                arm_feature(env, ARM_FEATURE_V8)) {
+                mask |= CPSR_IL;
+                val |= CPSR_IL;
+            }
         } else {
             switch_mode(env, val & CPSR_M);
         }
@@ -3633,36 +5723,11 @@
 
 uint32_t HELPER(rbit)(uint32_t x)
 {
-    x =  ((x & 0xff000000) >> 24)
-       | ((x & 0x00ff0000) >> 8)
-       | ((x & 0x0000ff00) << 8)
-       | ((x & 0x000000ff) << 24);
-    x =  ((x & 0xf0f0f0f0) >> 4)
-       | ((x & 0x0f0f0f0f) << 4);
-    x =  ((x & 0x88888888) >> 3)
-       | ((x & 0x44444444) >> 1)
-       | ((x & 0x22222222) << 1)
-       | ((x & 0x11111111) << 3);
-    return x;
+    return revbit32(x);
 }
 
 #if defined(CONFIG_USER_ONLY)
 
-int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
-                             int mmu_idx)
-{
-    ARMCPU *cpu = ARM_CPU(cs);
-    CPUARMState *env = &cpu->env;
-
-    env->exception.vaddress = address;
-    if (rw == 2) {
-        cs->exception_index = EXCP_PREFETCH_ABORT;
-    } else {
-        cs->exception_index = EXCP_DATA_ABORT;
-    }
-    return 1;
-}
-
 /* These should probably raise undefined insn exceptions.  */
 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
 {
@@ -3688,53 +5753,19 @@
     }
 }
 
-void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
-{
-    ARMCPU *cpu = arm_env_get_cpu(env);
-
-    cpu_abort(CPU(cpu), "banked r13 write\n");
-}
-
-uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
-{
-    ARMCPU *cpu = arm_env_get_cpu(env);
-
-    cpu_abort(CPU(cpu), "banked r13 read\n");
-    return 0;
-}
-
-unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
+uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
+                                 uint32_t cur_el, bool secure)
 {
     return 1;
 }
 
-#else
-
-/* Map CPU modes onto saved register banks.  */
-int bank_number(int mode)
+void aarch64_sync_64_to_32(CPUARMState *env)
 {
-    switch (mode) {
-    case ARM_CPU_MODE_USR:
-    case ARM_CPU_MODE_SYS:
-        return 0;
-    case ARM_CPU_MODE_SVC:
-        return 1;
-    case ARM_CPU_MODE_ABT:
-        return 2;
-    case ARM_CPU_MODE_UND:
-        return 3;
-    case ARM_CPU_MODE_IRQ:
-        return 4;
-    case ARM_CPU_MODE_FIQ:
-        return 5;
-    case ARM_CPU_MODE_HYP:
-        return 6;
-    case ARM_CPU_MODE_MON:
-        return 7;
-    }
-    hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
+    g_assert_not_reached();
 }
 
+#else
+
 void switch_mode(CPUARMState *env, int mode)
 {
     int old_mode;
@@ -3763,54 +5794,109 @@
     env->spsr = env->banked_spsr[i];
 }
 
-/*
- * Determine the target EL for a given exception type.
+/* Physical Interrupt Target EL Lookup Table
+ *
+ * [ From ARM ARM section G1.13.4 (Table G1-15) ]
+ *
+ * The below multi-dimensional table is used for looking up the target
+ * exception level given numerous condition criteria.  Specifically, the
+ * target EL is based on SCR and HCR routing controls as well as the
+ * currently executing EL and secure state.
+ *
+ *    Dimensions:
+ *    target_el_table[2][2][2][2][2][4]
+ *                    |  |  |  |  |  +--- Current EL
+ *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
+ *                    |  |  |  +--------- HCR mask override
+ *                    |  |  +------------ SCR exec state control
+ *                    |  +--------------- SCR mask override
+ *                    +------------------ 32-bit(0)/64-bit(1) EL3
+ *
+ *    The table values are as such:
+ *    0-3 = EL0-EL3
+ *     -1 = Cannot occur
+ *
+ * The ARM ARM target EL table includes entries indicating that an "exception
+ * is not taken".  The two cases where this is applicable are:
+ *    1) An exception is taken from EL3 but the SCR does not have the exception
+ *    routed to EL3.
+ *    2) An exception is taken from EL2 but the HCR does not have the exception
+ *    routed to EL2.
+ * In these two cases, the below table contain a target of EL1.  This value is
+ * returned as it is expected that the consumer of the table data will check
+ * for "target EL >= current EL" to ensure the exception is not taken.
+ *
+ *            SCR     HCR
+ *         64  EA     AMO                 From
+ *        BIT IRQ     IMO      Non-secure         Secure
+ *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
  */
-unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
-{
-    ARMCPU *cpu = ARM_CPU(cs);
-    CPUARMState *env = &cpu->env;
-    unsigned int cur_el = arm_current_el(env);
-    unsigned int target_el;
-    /* FIXME: Use actual secure state.  */
-    bool secure = false;
+static const int8_t target_el_table[2][2][2][2][2][4] = {
+    {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
+       {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
+      {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
+       {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
+     {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
+       {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
+      {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
+       {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
+    {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
+       {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
+      {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
+       {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
+     {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
+       {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
+      {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
+       {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
+};
 
-    if (!env->aarch64) {
-        /* TODO: Add EL2 and 3 exception handling for AArch32.  */
-        return 1;
+/*
+ * Determine the target EL for physical exceptions
+ */
+uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
+                                 uint32_t cur_el, bool secure)
+{
+    CPUARMState *env = cs->env_ptr;
+    int rw;
+    int scr;
+    int hcr;
+    int target_el;
+    /* Is the highest EL AArch64? */
+    int is64 = arm_feature(env, ARM_FEATURE_AARCH64);
+
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
+    } else {
+        /* Either EL2 is the highest EL (and so the EL2 register width
+         * is given by is64); or there is no EL2 or EL3, in which case
+         * the value of 'rw' does not affect the table lookup anyway.
+         */
+        rw = is64;
     }
 
     switch (excp_idx) {
-    case EXCP_HVC:
-    case EXCP_HYP_TRAP:
-        target_el = 2;
-        break;
-    case EXCP_SMC:
-        target_el = 3;
+    case EXCP_IRQ:
+        scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
+        hcr = ((env->cp15.hcr_el2 & HCR_IMO) == HCR_IMO);
         break;
     case EXCP_FIQ:
-    case EXCP_IRQ:
-    {
-        const uint64_t hcr_mask = excp_idx == EXCP_FIQ ? HCR_FMO : HCR_IMO;
-        const uint32_t scr_mask = excp_idx == EXCP_FIQ ? SCR_FIQ : SCR_IRQ;
-
-        target_el = 1;
-        if (!secure && (env->cp15.hcr_el2 & hcr_mask)) {
-            target_el = 2;
-        }
-        if (env->cp15.scr_el3 & scr_mask) {
-            target_el = 3;
-        }
-        break;
-    }
-    case EXCP_VIRQ:
-    case EXCP_VFIQ:
-        target_el = 1;
+        scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
+        hcr = ((env->cp15.hcr_el2 & HCR_FMO) == HCR_FMO);
         break;
     default:
-        target_el = MAX(cur_el, 1);
+        scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
+        hcr = ((env->cp15.hcr_el2 & HCR_AMO) == HCR_AMO);
         break;
-    }
+    };
+
+    /* If HCR.TGE is set then HCR is treated as being 1 */
+    hcr |= ((env->cp15.hcr_el2 & HCR_TGE) == HCR_TGE);
+
+    /* Perform a table-lookup for the target EL given the current state */
+    target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
+
+    assert(target_el > 0);
+
     return target_el;
 }
 
@@ -3863,6 +5949,16 @@
     env->regs[12] = v7m_pop(env);
     env->regs[14] = v7m_pop(env);
     env->regs[15] = v7m_pop(env);
+    if (env->regs[15] & 1) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "M profile return from interrupt with misaligned "
+                      "PC is UNPREDICTABLE\n");
+        /* Actual hardware seems to ignore the lsbit, and there are several
+         * RTOSes out there which incorrectly assume the r15 in the stack
+         * frame should be a Thumb-style "lsbit indicates ARM/Thumb" value.
+         */
+        env->regs[15] &= ~1U;
+    }
     xpsr = v7m_pop(env);
     xpsr_write(env, xpsr, 0xfffffdff);
     /* Undo stack alignment.  */
@@ -3875,6 +5971,21 @@
        pointer.  */
 }
 
+static void arm_log_exception(int idx)
+{
+    if (qemu_loglevel_mask(CPU_LOG_INT)) {
+        const char *exc = NULL;
+
+        if (idx >= 0 && idx < ARRAY_SIZE(excnames)) {
+            exc = excnames[idx];
+        }
+        if (!exc) {
+            exc = "unknown";
+        }
+        qemu_log_mask(CPU_LOG_INT, "Taking exception %d [%s]\n", idx, exc);
+    }
+}
+
 void arm_v7m_cpu_do_interrupt(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
@@ -3911,13 +6022,15 @@
         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
         return;
     case EXCP_BKPT:
-        if (semihosting_enabled) {
+        if (semihosting_enabled()) {
             int nr;
-            nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
+            nr = arm_lduw_code(env, env->regs[15], arm_sctlr_b(env)) & 0xff;
             if (nr == 0xab) {
                 env->regs[15] += 2;
+                qemu_log_mask(CPU_LOG_INT,
+                              "...handling as semihosting call 0x%x\n",
+                              env->regs[0]);
                 env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
                 return;
             }
         }
@@ -3959,8 +6072,213 @@
     env->thumb = addr & 1;
 }
 
-/* Handle a CPU exception.  */
-void arm_cpu_do_interrupt(CPUState *cs)
+/* Function used to synchronize QEMU's AArch64 register set with AArch32
+ * register set.  This is necessary when switching between AArch32 and AArch64
+ * execution state.
+ */
+void aarch64_sync_32_to_64(CPUARMState *env)
+{
+    int i;
+    uint32_t mode = env->uncached_cpsr & CPSR_M;
+
+    /* We can blanket copy R[0:7] to X[0:7] */
+    for (i = 0; i < 8; i++) {
+        env->xregs[i] = env->regs[i];
+    }
+
+    /* Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
+     * Otherwise, they come from the banked user regs.
+     */
+    if (mode == ARM_CPU_MODE_FIQ) {
+        for (i = 8; i < 13; i++) {
+            env->xregs[i] = env->usr_regs[i - 8];
+        }
+    } else {
+        for (i = 8; i < 13; i++) {
+            env->xregs[i] = env->regs[i];
+        }
+    }
+
+    /* Registers x13-x23 are the various mode SP and FP registers. Registers
+     * r13 and r14 are only copied if we are in that mode, otherwise we copy
+     * from the mode banked register.
+     */
+    if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
+        env->xregs[13] = env->regs[13];
+        env->xregs[14] = env->regs[14];
+    } else {
+        env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
+        /* HYP is an exception in that it is copied from r14 */
+        if (mode == ARM_CPU_MODE_HYP) {
+            env->xregs[14] = env->regs[14];
+        } else {
+            env->xregs[14] = env->banked_r14[bank_number(ARM_CPU_MODE_USR)];
+        }
+    }
+
+    if (mode == ARM_CPU_MODE_HYP) {
+        env->xregs[15] = env->regs[13];
+    } else {
+        env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
+    }
+
+    if (mode == ARM_CPU_MODE_IRQ) {
+        env->xregs[16] = env->regs[14];
+        env->xregs[17] = env->regs[13];
+    } else {
+        env->xregs[16] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
+        env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
+    }
+
+    if (mode == ARM_CPU_MODE_SVC) {
+        env->xregs[18] = env->regs[14];
+        env->xregs[19] = env->regs[13];
+    } else {
+        env->xregs[18] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
+        env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
+    }
+
+    if (mode == ARM_CPU_MODE_ABT) {
+        env->xregs[20] = env->regs[14];
+        env->xregs[21] = env->regs[13];
+    } else {
+        env->xregs[20] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
+        env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
+    }
+
+    if (mode == ARM_CPU_MODE_UND) {
+        env->xregs[22] = env->regs[14];
+        env->xregs[23] = env->regs[13];
+    } else {
+        env->xregs[22] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
+        env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
+    }
+
+    /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
+     * mode, then we can copy from r8-r14.  Otherwise, we copy from the
+     * FIQ bank for r8-r14.
+     */
+    if (mode == ARM_CPU_MODE_FIQ) {
+        for (i = 24; i < 31; i++) {
+            env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
+        }
+    } else {
+        for (i = 24; i < 29; i++) {
+            env->xregs[i] = env->fiq_regs[i - 24];
+        }
+        env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
+        env->xregs[30] = env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)];
+    }
+
+    env->pc = env->regs[15];
+}
+
+/* Function used to synchronize QEMU's AArch32 register set with AArch64
+ * register set.  This is necessary when switching between AArch32 and AArch64
+ * execution state.
+ */
+void aarch64_sync_64_to_32(CPUARMState *env)
+{
+    int i;
+    uint32_t mode = env->uncached_cpsr & CPSR_M;
+
+    /* We can blanket copy X[0:7] to R[0:7] */
+    for (i = 0; i < 8; i++) {
+        env->regs[i] = env->xregs[i];
+    }
+
+    /* Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
+     * Otherwise, we copy x8-x12 into the banked user regs.
+     */
+    if (mode == ARM_CPU_MODE_FIQ) {
+        for (i = 8; i < 13; i++) {
+            env->usr_regs[i - 8] = env->xregs[i];
+        }
+    } else {
+        for (i = 8; i < 13; i++) {
+            env->regs[i] = env->xregs[i];
+        }
+    }
+
+    /* Registers r13 & r14 depend on the current mode.
+     * If we are in a given mode, we copy the corresponding x registers to r13
+     * and r14.  Otherwise, we copy the x register to the banked r13 and r14
+     * for the mode.
+     */
+    if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
+        env->regs[13] = env->xregs[13];
+        env->regs[14] = env->xregs[14];
+    } else {
+        env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
+
+        /* HYP is an exception in that it does not have its own banked r14 but
+         * shares the USR r14
+         */
+        if (mode == ARM_CPU_MODE_HYP) {
+            env->regs[14] = env->xregs[14];
+        } else {
+            env->banked_r14[bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
+        }
+    }
+
+    if (mode == ARM_CPU_MODE_HYP) {
+        env->regs[13] = env->xregs[15];
+    } else {
+        env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
+    }
+
+    if (mode == ARM_CPU_MODE_IRQ) {
+        env->regs[14] = env->xregs[16];
+        env->regs[13] = env->xregs[17];
+    } else {
+        env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
+        env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
+    }
+
+    if (mode == ARM_CPU_MODE_SVC) {
+        env->regs[14] = env->xregs[18];
+        env->regs[13] = env->xregs[19];
+    } else {
+        env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
+        env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
+    }
+
+    if (mode == ARM_CPU_MODE_ABT) {
+        env->regs[14] = env->xregs[20];
+        env->regs[13] = env->xregs[21];
+    } else {
+        env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
+        env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
+    }
+
+    if (mode == ARM_CPU_MODE_UND) {
+        env->regs[14] = env->xregs[22];
+        env->regs[13] = env->xregs[23];
+    } else {
+        env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
+        env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
+    }
+
+    /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
+     * mode, then we can copy to r8-r14.  Otherwise, we copy to the
+     * FIQ bank for r8-r14.
+     */
+    if (mode == ARM_CPU_MODE_FIQ) {
+        for (i = 24; i < 31; i++) {
+            env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
+        }
+    } else {
+        for (i = 24; i < 29; i++) {
+            env->fiq_regs[i - 24] = env->xregs[i];
+        }
+        env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
+        env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
+    }
+
+    env->regs[15] = env->pc;
+}
+
+static void arm_cpu_do_interrupt_aarch32(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
@@ -3970,16 +6288,6 @@
     uint32_t offset;
     uint32_t moe;
 
-    assert(!IS_M(env));
-
-    arm_log_exception(cs->exception_index);
-
-    if (arm_is_psci_call(cpu, cs->exception_index)) {
-        arm_handle_psci_call(cpu);
-        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
-        return;
-    }
-
     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
     case EC_BREAKPOINT:
@@ -4017,25 +6325,6 @@
             offset = 4;
         break;
     case EXCP_SWI:
-        if (semihosting_enabled) {
-            /* Check for semihosting interrupt.  */
-            if (env->thumb) {
-                mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
-                    & 0xff;
-            } else {
-                mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
-                    & 0xffffff;
-            }
-            /* Only intercept calls from privileged modes, to provide some
-               semblance of security.  */
-            if (((mask == 0x123456 && !env->thumb)
-                    || (mask == 0xab && env->thumb))
-                  && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
-                env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
-                return;
-            }
-        }
         new_mode = ARM_CPU_MODE_SVC;
         addr = 0x08;
         mask = CPSR_I;
@@ -4043,36 +6332,23 @@
         offset = 0;
         break;
     case EXCP_BKPT:
-        /* See if this is a semihosting syscall.  */
-        if (env->thumb && semihosting_enabled) {
-            mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
-            if (mask == 0xab
-                  && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
-                env->regs[15] += 2;
-                env->regs[0] = do_arm_semihosting(env);
-                qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
-                return;
-            }
-        }
         env->exception.fsr = 2;
         /* Fall through to prefetch abort.  */
     case EXCP_PREFETCH_ABORT:
-        env->cp15.ifsr_el2 = env->exception.fsr;
-        env->cp15.far_el[1] = deposit64(env->cp15.far_el[1], 32, 32,
-                                        env->exception.vaddress);
+        A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
+        A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
-                      env->cp15.ifsr_el2, (uint32_t)env->exception.vaddress);
+                      env->exception.fsr, (uint32_t)env->exception.vaddress);
         new_mode = ARM_CPU_MODE_ABT;
         addr = 0x0c;
         mask = CPSR_A | CPSR_I;
         offset = 4;
         break;
     case EXCP_DATA_ABORT:
-        env->cp15.esr_el[1] = env->exception.fsr;
-        env->cp15.far_el[1] = deposit64(env->cp15.far_el[1], 0, 32,
-                                        env->exception.vaddress);
+        A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
+        A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
-                      (uint32_t)env->cp15.esr_el[1],
+                      env->exception.fsr,
                       (uint32_t)env->exception.vaddress);
         new_mode = ARM_CPU_MODE_ABT;
         addr = 0x10;
@@ -4085,12 +6361,21 @@
         /* Disable IRQ and imprecise data aborts.  */
         mask = CPSR_A | CPSR_I;
         offset = 4;
+        if (env->cp15.scr_el3 & SCR_IRQ) {
+            /* IRQ routed to monitor mode */
+            new_mode = ARM_CPU_MODE_MON;
+            mask |= CPSR_F;
+        }
         break;
     case EXCP_FIQ:
         new_mode = ARM_CPU_MODE_FIQ;
         addr = 0x1c;
         /* Disable FIQ, IRQ and imprecise data aborts.  */
         mask = CPSR_A | CPSR_I | CPSR_F;
+        if (env->cp15.scr_el3 & SCR_FIQ) {
+            /* FIQ routed to monitor mode */
+            new_mode = ARM_CPU_MODE_MON;
+        }
         offset = 4;
         break;
     case EXCP_SMC:
@@ -4103,19 +6388,19 @@
         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
         return; /* Never happens.  Keep compiler happy.  */
     }
-    /* High vectors.  */
-    if (env->cp15.c1_sys & SCTLR_V) {
-        /* when enabled, base address cannot be remapped.  */
+
+    if (new_mode == ARM_CPU_MODE_MON) {
+        addr += env->cp15.mvbar;
+    } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
+        /* High vectors. When enabled, base address cannot be remapped. */
         addr += 0xffff0000;
     } else {
         /* ARM v7 architectures provide a vector base address register to remap
          * the interrupt vector table.
-         * This register is only followed in non-monitor mode, and has a secure
-         * and un-secure copy. Since the cpu is always in a un-secure operation
-         * and is never in monitor mode this feature is always active.
+         * This register is only followed in non-monitor mode, and is banked.
          * Note: only bits 31:5 are valid.
          */
-        addr += env->cp15.vbar_el[1];
+        addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
     }
 
     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
@@ -4132,96 +6417,660 @@
     env->condexec_bits = 0;
     /* Switch to the new mode, and to the correct instruction set.  */
     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
+    /* Set new mode endianness */
+    env->uncached_cpsr &= ~CPSR_E;
+    if (env->cp15.sctlr_el[arm_current_el(env)] & SCTLR_EE) {
+        env->uncached_cpsr |= ~CPSR_E;
+    }
     env->daif |= mask;
     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
      * and we should just guard the thumb mode on V4 */
     if (arm_feature(env, ARM_FEATURE_V4T)) {
-        env->thumb = (env->cp15.c1_sys & SCTLR_TE) != 0;
+        env->thumb = (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
     }
     env->regs[14] = env->regs[15] + offset;
     env->regs[15] = addr;
-    cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
 }
 
-/* Check section/page access permissions.
-   Returns the page protection flags, or zero if the access is not
-   permitted.  */
-static inline int check_ap(CPUARMState *env, int ap, int domain_prot,
-                           int access_type, int is_user)
+/* Handle exception entry to a target EL which is using AArch64 */
+static void arm_cpu_do_interrupt_aarch64(CPUState *cs)
 {
-  int prot_ro;
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    unsigned int new_el = env->exception.target_el;
+    target_ulong addr = env->cp15.vbar_el[new_el];
+    unsigned int new_mode = aarch64_pstate_mode(new_el, true);
 
-  if (domain_prot == 3) {
-    return PAGE_READ | PAGE_WRITE;
-  }
+    if (arm_current_el(env) < new_el) {
+        /* Entry vector offset depends on whether the implemented EL
+         * immediately lower than the target level is using AArch32 or AArch64
+         */
+        bool is_aa64;
 
-  if (access_type == 1)
-      prot_ro = 0;
-  else
-      prot_ro = PAGE_READ;
+        switch (new_el) {
+        case 3:
+            is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0;
+            break;
+        case 2:
+            is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0;
+            break;
+        case 1:
+            is_aa64 = is_a64(env);
+            break;
+        default:
+            g_assert_not_reached();
+        }
 
-  switch (ap) {
-  case 0:
-      if (arm_feature(env, ARM_FEATURE_V7)) {
-          return 0;
-      }
-      if (access_type == 1)
-          return 0;
-      switch (env->cp15.c1_sys & (SCTLR_S | SCTLR_R)) {
-      case SCTLR_S:
-          return is_user ? 0 : PAGE_READ;
-      case SCTLR_R:
-          return PAGE_READ;
-      default:
-          return 0;
-      }
-  case 1:
-      return is_user ? 0 : PAGE_READ | PAGE_WRITE;
-  case 2:
-      if (is_user)
-          return prot_ro;
-      else
-          return PAGE_READ | PAGE_WRITE;
-  case 3:
-      return PAGE_READ | PAGE_WRITE;
-  case 4: /* Reserved.  */
-      return 0;
-  case 5:
-      return is_user ? 0 : prot_ro;
-  case 6:
-      return prot_ro;
-  case 7:
-      if (!arm_feature (env, ARM_FEATURE_V6K))
-          return 0;
-      return prot_ro;
-  default:
-      abort();
-  }
+        if (is_aa64) {
+            addr += 0x400;
+        } else {
+            addr += 0x600;
+        }
+    } else if (pstate_read(env) & PSTATE_SP) {
+        addr += 0x200;
+    }
+
+    switch (cs->exception_index) {
+    case EXCP_PREFETCH_ABORT:
+    case EXCP_DATA_ABORT:
+        env->cp15.far_el[new_el] = env->exception.vaddress;
+        qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
+                      env->cp15.far_el[new_el]);
+        /* fall through */
+    case EXCP_BKPT:
+    case EXCP_UDEF:
+    case EXCP_SWI:
+    case EXCP_HVC:
+    case EXCP_HYP_TRAP:
+    case EXCP_SMC:
+        env->cp15.esr_el[new_el] = env->exception.syndrome;
+        break;
+    case EXCP_IRQ:
+    case EXCP_VIRQ:
+        addr += 0x80;
+        break;
+    case EXCP_FIQ:
+    case EXCP_VFIQ:
+        addr += 0x100;
+        break;
+    case EXCP_SEMIHOST:
+        qemu_log_mask(CPU_LOG_INT,
+                      "...handling as semihosting call 0x%" PRIx64 "\n",
+                      env->xregs[0]);
+        env->xregs[0] = do_arm_semihosting(env);
+        return;
+    default:
+        cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
+    }
+
+    if (is_a64(env)) {
+        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
+        aarch64_save_sp(env, arm_current_el(env));
+        env->elr_el[new_el] = env->pc;
+    } else {
+        env->banked_spsr[aarch64_banked_spsr_index(new_el)] = cpsr_read(env);
+        env->elr_el[new_el] = env->regs[15];
+
+        aarch64_sync_32_to_64(env);
+
+        env->condexec_bits = 0;
+    }
+    qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
+                  env->elr_el[new_el]);
+
+    pstate_write(env, PSTATE_DAIF | new_mode);
+    env->aarch64 = 1;
+    aarch64_restore_sp(env, new_el);
+
+    env->pc = addr;
+
+    qemu_log_mask(CPU_LOG_INT, "...to EL%d PC 0x%" PRIx64 " PSTATE 0x%x\n",
+                  new_el, env->pc, pstate_read(env));
 }
 
-static bool get_level1_table_address(CPUARMState *env, uint32_t *table,
-                                         uint32_t address)
+static inline bool check_for_semihosting(CPUState *cs)
 {
-    if (address & env->cp15.c2_mask) {
-        if ((env->cp15.c2_control & TTBCR_PD1)) {
+    /* Check whether this exception is a semihosting call; if so
+     * then handle it and return true; otherwise return false.
+     */
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+
+    if (is_a64(env)) {
+        if (cs->exception_index == EXCP_SEMIHOST) {
+            /* This is always the 64-bit semihosting exception.
+             * The "is this usermode" and "is semihosting enabled"
+             * checks have been done at translate time.
+             */
+            qemu_log_mask(CPU_LOG_INT,
+                          "...handling as semihosting call 0x%" PRIx64 "\n",
+                          env->xregs[0]);
+            env->xregs[0] = do_arm_semihosting(env);
+            return true;
+        }
+        return false;
+    } else {
+        uint32_t imm;
+
+        /* Only intercept calls from privileged modes, to provide some
+         * semblance of security.
+         */
+        if (!semihosting_enabled() ||
+            ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)) {
+            return false;
+        }
+
+        switch (cs->exception_index) {
+        case EXCP_SWI:
+            /* Check for semihosting interrupt.  */
+            if (env->thumb) {
+                imm = arm_lduw_code(env, env->regs[15] - 2, arm_sctlr_b(env))
+                    & 0xff;
+                if (imm == 0xab) {
+                    break;
+                }
+            } else {
+                imm = arm_ldl_code(env, env->regs[15] - 4, arm_sctlr_b(env))
+                    & 0xffffff;
+                if (imm == 0x123456) {
+                    break;
+                }
+            }
+            return false;
+        case EXCP_BKPT:
+            /* See if this is a semihosting syscall.  */
+            if (env->thumb) {
+                imm = arm_lduw_code(env, env->regs[15], arm_sctlr_b(env))
+                    & 0xff;
+                if (imm == 0xab) {
+                    env->regs[15] += 2;
+                    break;
+                }
+            }
+            return false;
+        default:
+            return false;
+        }
+
+        qemu_log_mask(CPU_LOG_INT,
+                      "...handling as semihosting call 0x%x\n",
+                      env->regs[0]);
+        env->regs[0] = do_arm_semihosting(env);
+        return true;
+    }
+}
+
+/* Handle a CPU exception for A and R profile CPUs.
+ * Do any appropriate logging, handle PSCI calls, and then hand off
+ * to the AArch64-entry or AArch32-entry function depending on the
+ * target exception level's register width.
+ */
+void arm_cpu_do_interrupt(CPUState *cs)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    unsigned int new_el = env->exception.target_el;
+
+    assert(!IS_M(env));
+
+    arm_log_exception(cs->exception_index);
+    qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
+                  new_el);
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && !excp_is_internal(cs->exception_index)) {
+        qemu_log_mask(CPU_LOG_INT, "...with ESR %x/0x%" PRIx32 "\n",
+                      env->exception.syndrome >> ARM_EL_EC_SHIFT,
+                      env->exception.syndrome);
+    }
+
+    if (arm_is_psci_call(cpu, cs->exception_index)) {
+        arm_handle_psci_call(cpu);
+        qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
+        return;
+    }
+
+    /* Semihosting semantics depend on the register width of the
+     * code that caused the exception, not the target exception level,
+     * so must be handled here.
+     */
+    if (check_for_semihosting(cs)) {
+        return;
+    }
+
+    assert(!excp_is_internal(cs->exception_index));
+    if (arm_el_is_aa64(env, new_el)) {
+        arm_cpu_do_interrupt_aarch64(cs);
+    } else {
+        arm_cpu_do_interrupt_aarch32(cs);
+    }
+
+    arm_call_el_change_hook(cpu);
+
+    if (!kvm_enabled()) {
+        cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
+    }
+}
+
+/* Return the exception level which controls this address translation regime */
+static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    switch (mmu_idx) {
+    case ARMMMUIdx_S2NS:
+    case ARMMMUIdx_S1E2:
+        return 2;
+    case ARMMMUIdx_S1E3:
+        return 3;
+    case ARMMMUIdx_S1SE0:
+        return arm_el_is_aa64(env, 3) ? 1 : 3;
+    case ARMMMUIdx_S1SE1:
+    case ARMMMUIdx_S1NSE0:
+    case ARMMMUIdx_S1NSE1:
+        return 1;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+/* Return true if this address translation regime is secure */
+static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    switch (mmu_idx) {
+    case ARMMMUIdx_S12NSE0:
+    case ARMMMUIdx_S12NSE1:
+    case ARMMMUIdx_S1NSE0:
+    case ARMMMUIdx_S1NSE1:
+    case ARMMMUIdx_S1E2:
+    case ARMMMUIdx_S2NS:
+        return false;
+    case ARMMMUIdx_S1E3:
+    case ARMMMUIdx_S1SE0:
+    case ARMMMUIdx_S1SE1:
+        return true;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+/* Return the SCTLR value which controls this address translation regime */
+static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    return env->cp15.sctlr_el[regime_el(env, mmu_idx)];
+}
+
+/* Return true if the specified stage of address translation is disabled */
+static inline bool regime_translation_disabled(CPUARMState *env,
+                                               ARMMMUIdx mmu_idx)
+{
+    if (mmu_idx == ARMMMUIdx_S2NS) {
+        return (env->cp15.hcr_el2 & HCR_VM) == 0;
+    }
+    return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
+}
+
+static inline bool regime_translation_big_endian(CPUARMState *env,
+                                                 ARMMMUIdx mmu_idx)
+{
+    return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0;
+}
+
+/* Return the TCR controlling this translation regime */
+static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    if (mmu_idx == ARMMMUIdx_S2NS) {
+        return &env->cp15.vtcr_el2;
+    }
+    return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
+}
+
+/* Return the TTBR associated with this translation regime */
+static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
+                                   int ttbrn)
+{
+    if (mmu_idx == ARMMMUIdx_S2NS) {
+        return env->cp15.vttbr_el2;
+    }
+    if (ttbrn == 0) {
+        return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
+    } else {
+        return env->cp15.ttbr1_el[regime_el(env, mmu_idx)];
+    }
+}
+
+/* Return true if the translation regime is using LPAE format page tables */
+static inline bool regime_using_lpae_format(CPUARMState *env,
+                                            ARMMMUIdx mmu_idx)
+{
+    int el = regime_el(env, mmu_idx);
+    if (el == 2 || arm_el_is_aa64(env, el)) {
+        return true;
+    }
+    if (arm_feature(env, ARM_FEATURE_LPAE)
+        && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) {
+        return true;
+    }
+    return false;
+}
+
+/* Returns true if the stage 1 translation regime is using LPAE format page
+ * tables. Used when raising alignment exceptions, whose FSR changes depending
+ * on whether the long or short descriptor format is in use. */
+bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
+        mmu_idx += ARMMMUIdx_S1NSE0;
+    }
+
+    return regime_using_lpae_format(env, mmu_idx);
+}
+
+static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
+{
+    switch (mmu_idx) {
+    case ARMMMUIdx_S1SE0:
+    case ARMMMUIdx_S1NSE0:
+        return true;
+    default:
+        return false;
+    case ARMMMUIdx_S12NSE0:
+    case ARMMMUIdx_S12NSE1:
+        g_assert_not_reached();
+    }
+}
+
+/* Translate section/page access permissions to page
+ * R/W protection flags
+ *
+ * @env:         CPUARMState
+ * @mmu_idx:     MMU index indicating required translation regime
+ * @ap:          The 3-bit access permissions (AP[2:0])
+ * @domain_prot: The 2-bit domain access permissions
+ */
+static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
+                                int ap, int domain_prot)
+{
+    bool is_user = regime_is_user(env, mmu_idx);
+
+    if (domain_prot == 3) {
+        return PAGE_READ | PAGE_WRITE;
+    }
+
+    switch (ap) {
+    case 0:
+        if (arm_feature(env, ARM_FEATURE_V7)) {
+            return 0;
+        }
+        switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
+        case SCTLR_S:
+            return is_user ? 0 : PAGE_READ;
+        case SCTLR_R:
+            return PAGE_READ;
+        default:
+            return 0;
+        }
+    case 1:
+        return is_user ? 0 : PAGE_READ | PAGE_WRITE;
+    case 2:
+        if (is_user) {
+            return PAGE_READ;
+        } else {
+            return PAGE_READ | PAGE_WRITE;
+        }
+    case 3:
+        return PAGE_READ | PAGE_WRITE;
+    case 4: /* Reserved.  */
+        return 0;
+    case 5:
+        return is_user ? 0 : PAGE_READ;
+    case 6:
+        return PAGE_READ;
+    case 7:
+        if (!arm_feature(env, ARM_FEATURE_V6K)) {
+            return 0;
+        }
+        return PAGE_READ;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+/* Translate section/page access permissions to page
+ * R/W protection flags.
+ *
+ * @ap:      The 2-bit simple AP (AP[2:1])
+ * @is_user: TRUE if accessing from PL0
+ */
+static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
+{
+    switch (ap) {
+    case 0:
+        return is_user ? 0 : PAGE_READ | PAGE_WRITE;
+    case 1:
+        return PAGE_READ | PAGE_WRITE;
+    case 2:
+        return is_user ? 0 : PAGE_READ;
+    case 3:
+        return PAGE_READ;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+static inline int
+simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
+{
+    return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
+}
+
+/* Translate S2 section/page access permissions to protection flags
+ *
+ * @env:     CPUARMState
+ * @s2ap:    The 2-bit stage2 access permissions (S2AP)
+ * @xn:      XN (execute-never) bit
+ */
+static int get_S2prot(CPUARMState *env, int s2ap, int xn)
+{
+    int prot = 0;
+
+    if (s2ap & 1) {
+        prot |= PAGE_READ;
+    }
+    if (s2ap & 2) {
+        prot |= PAGE_WRITE;
+    }
+    if (!xn) {
+        if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) {
+            prot |= PAGE_EXEC;
+        }
+    }
+    return prot;
+}
+
+/* Translate section/page access permissions to protection flags
+ *
+ * @env:     CPUARMState
+ * @mmu_idx: MMU index indicating required translation regime
+ * @is_aa64: TRUE if AArch64
+ * @ap:      The 2-bit simple AP (AP[2:1])
+ * @ns:      NS (non-secure) bit
+ * @xn:      XN (execute-never) bit
+ * @pxn:     PXN (privileged execute-never) bit
+ */
+static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
+                      int ap, int ns, int xn, int pxn)
+{
+    bool is_user = regime_is_user(env, mmu_idx);
+    int prot_rw, user_rw;
+    bool have_wxn;
+    int wxn = 0;
+
+    assert(mmu_idx != ARMMMUIdx_S2NS);
+
+    user_rw = simple_ap_to_rw_prot_is_user(ap, true);
+    if (is_user) {
+        prot_rw = user_rw;
+    } else {
+        prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
+    }
+
+    if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
+        return prot_rw;
+    }
+
+    /* TODO have_wxn should be replaced with
+     *   ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
+     * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
+     * compatible processors have EL2, which is required for [U]WXN.
+     */
+    have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
+
+    if (have_wxn) {
+        wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
+    }
+
+    if (is_aa64) {
+        switch (regime_el(env, mmu_idx)) {
+        case 1:
+            if (!is_user) {
+                xn = pxn || (user_rw & PAGE_WRITE);
+            }
+            break;
+        case 2:
+        case 3:
+            break;
+        }
+    } else if (arm_feature(env, ARM_FEATURE_V7)) {
+        switch (regime_el(env, mmu_idx)) {
+        case 1:
+        case 3:
+            if (is_user) {
+                xn = xn || !(user_rw & PAGE_READ);
+            } else {
+                int uwxn = 0;
+                if (have_wxn) {
+                    uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
+                }
+                xn = xn || !(prot_rw & PAGE_READ) || pxn ||
+                     (uwxn && (user_rw & PAGE_WRITE));
+            }
+            break;
+        case 2:
+            break;
+        }
+    } else {
+        xn = wxn = 0;
+    }
+
+    if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
+        return prot_rw;
+    }
+    return prot_rw | PAGE_EXEC;
+}
+
+static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
+                                     uint32_t *table, uint32_t address)
+{
+    /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
+    TCR *tcr = regime_tcr(env, mmu_idx);
+
+    if (address & tcr->mask) {
+        if (tcr->raw_tcr & TTBCR_PD1) {
             /* Translation table walk disabled for TTBR1 */
             return false;
         }
-        *table = env->cp15.ttbr1_el1 & 0xffffc000;
+        *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
     } else {
-        if ((env->cp15.c2_control & TTBCR_PD0)) {
+        if (tcr->raw_tcr & TTBCR_PD0) {
             /* Translation table walk disabled for TTBR0 */
             return false;
         }
-        *table = env->cp15.ttbr0_el1 & env->cp15.c2_base_mask;
+        *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask;
     }
     *table |= (address >> 18) & 0x3ffc;
     return true;
 }
 
-static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
-                            int is_user, hwaddr *phys_ptr,
-                            int *prot, target_ulong *page_size)
+/* Translate a S1 pagetable walk through S2 if needed.  */
+static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx,
+                               hwaddr addr, MemTxAttrs txattrs,
+                               uint32_t *fsr,
+                               ARMMMUFaultInfo *fi)
+{
+    if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) &&
+        !regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
+        target_ulong s2size;
+        hwaddr s2pa;
+        int s2prot;
+        int ret;
+
+        ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa,
+                                 &txattrs, &s2prot, &s2size, fsr, fi);
+        if (ret) {
+            fi->s2addr = addr;
+            fi->stage2 = true;
+            fi->s1ptw = true;
+            return ~0;
+        }
+        addr = s2pa;
+    }
+    return addr;
+}
+
+/* All loads done in the course of a page table walk go through here.
+ * TODO: rather than ignoring errors from physical memory reads (which
+ * are external aborts in ARM terminology) we should propagate this
+ * error out so that we can turn it into a Data Abort if this walk
+ * was being done for a CPU load/store or an address translation instruction
+ * (but not if it was for a debug access).
+ */
+static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure,
+                            ARMMMUIdx mmu_idx, uint32_t *fsr,
+                            ARMMMUFaultInfo *fi)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    MemTxAttrs attrs = {};
+    AddressSpace *as;
+
+    attrs.secure = is_secure;
+    as = arm_addressspace(cs, attrs);
+    addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
+    if (fi->s1ptw) {
+        return 0;
+    }
+    if (regime_translation_big_endian(env, mmu_idx)) {
+        return address_space_ldl_be(as, addr, attrs, NULL);
+    } else {
+        return address_space_ldl_le(as, addr, attrs, NULL);
+    }
+}
+
+static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure,
+                            ARMMMUIdx mmu_idx, uint32_t *fsr,
+                            ARMMMUFaultInfo *fi)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    MemTxAttrs attrs = {};
+    AddressSpace *as;
+
+    attrs.secure = is_secure;
+    as = arm_addressspace(cs, attrs);
+    addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
+    if (fi->s1ptw) {
+        return 0;
+    }
+    if (regime_translation_big_endian(env, mmu_idx)) {
+        return address_space_ldq_be(as, addr, attrs, NULL);
+    } else {
+        return address_space_ldq_le(as, addr, attrs, NULL);
+    }
+}
+
+static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
+                             int access_type, ARMMMUIdx mmu_idx,
+                             hwaddr *phys_ptr, int *prot,
+                             target_ulong *page_size, uint32_t *fsr,
+                             ARMMMUFaultInfo *fi)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int code;
@@ -4232,18 +7081,25 @@
     int domain = 0;
     int domain_prot;
     hwaddr phys_addr;
+    uint32_t dacr;
 
     /* Pagetable walk.  */
     /* Lookup l1 descriptor.  */
-    if (!get_level1_table_address(env, &table, address)) {
+    if (!get_level1_table_address(env, mmu_idx, &table, address)) {
         /* Section translation fault if page walk is disabled by PD0 or PD1 */
         code = 5;
         goto do_fault;
     }
-    desc = ldl_phys(cs->as, table);
+    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                       mmu_idx, fsr, fi);
     type = (desc & 3);
     domain = (desc >> 5) & 0x0f;
-    domain_prot = (env->cp15.c3 >> (domain * 2)) & 3;
+    if (regime_el(env, mmu_idx) == 1) {
+        dacr = env->cp15.dacr_ns;
+    } else {
+        dacr = env->cp15.dacr_s;
+    }
+    domain_prot = (dacr >> (domain * 2)) & 3;
     if (type == 0) {
         /* Section translation fault.  */
         code = 5;
@@ -4264,14 +7120,15 @@
         *page_size = 1024 * 1024;
     } else {
         /* Lookup l2 entry.  */
-	if (type == 1) {
-	    /* Coarse pagetable.  */
-	    table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
-	} else {
-	    /* Fine pagetable.  */
-	    table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
-	}
-        desc = ldl_phys(cs->as, table);
+        if (type == 1) {
+            /* Coarse pagetable.  */
+            table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
+        } else {
+            /* Fine pagetable.  */
+            table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
+        }
+        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                           mmu_idx, fsr, fi);
         switch (desc & 3) {
         case 0: /* Page translation fault.  */
             code = 7;
@@ -4286,20 +7143,25 @@
             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
             *page_size = 0x1000;
             break;
-        case 3: /* 1k page.  */
-	    if (type == 1) {
-		if (arm_feature(env, ARM_FEATURE_XSCALE)) {
-		    phys_addr = (desc & 0xfffff000) | (address & 0xfff);
-		} else {
-		    /* Page translation fault.  */
-		    code = 7;
-		    goto do_fault;
-		}
-	    } else {
-		phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
-	    }
+        case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */
+            if (type == 1) {
+                /* ARMv6/XScale extended small page format */
+                if (arm_feature(env, ARM_FEATURE_XSCALE)
+                    || arm_feature(env, ARM_FEATURE_V6)) {
+                    phys_addr = (desc & 0xfffff000) | (address & 0xfff);
+                    *page_size = 0x1000;
+                } else {
+                    /* UNPREDICTABLE in ARMv5; we choose to take a
+                     * page translation fault.
+                     */
+                    code = 7;
+                    goto do_fault;
+                }
+            } else {
+                phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
+                *page_size = 0x400;
+            }
             ap = (desc >> 4) & 3;
-            *page_size = 0x400;
             break;
         default:
             /* Never happens, but compiler isn't smart enough to tell.  */
@@ -4307,21 +7169,24 @@
         }
         code = 15;
     }
-    *prot = check_ap(env, ap, domain_prot, access_type, is_user);
-    if (!*prot) {
+    *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
+    *prot |= *prot ? PAGE_EXEC : 0;
+    if (!(*prot & (1 << access_type))) {
         /* Access permission fault.  */
         goto do_fault;
     }
-    *prot |= PAGE_EXEC;
     *phys_ptr = phys_addr;
-    return 0;
+    return false;
 do_fault:
-    return code | (domain << 4);
+    *fsr = code | (domain << 4);
+    return true;
 }
 
-static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
-                            int is_user, hwaddr *phys_ptr,
-                            int *prot, target_ulong *page_size)
+static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
+                             int access_type, ARMMMUIdx mmu_idx,
+                             hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                             target_ulong *page_size, uint32_t *fsr,
+                             ARMMMUFaultInfo *fi)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
     int code;
@@ -4334,15 +7199,18 @@
     int domain = 0;
     int domain_prot;
     hwaddr phys_addr;
+    uint32_t dacr;
+    bool ns;
 
     /* Pagetable walk.  */
     /* Lookup l1 descriptor.  */
-    if (!get_level1_table_address(env, &table, address)) {
+    if (!get_level1_table_address(env, mmu_idx, &table, address)) {
         /* Section translation fault if page walk is disabled by PD0 or PD1 */
         code = 5;
         goto do_fault;
     }
-    desc = ldl_phys(cs->as, table);
+    desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                       mmu_idx, fsr, fi);
     type = (desc & 3);
     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
         /* Section translation fault, or attempt to use the encoding
@@ -4355,7 +7223,12 @@
         /* Page or Section.  */
         domain = (desc >> 5) & 0x0f;
     }
-    domain_prot = (env->cp15.c3 >> (domain * 2)) & 3;
+    if (regime_el(env, mmu_idx) == 1) {
+        dacr = env->cp15.dacr_ns;
+    } else {
+        dacr = env->cp15.dacr_s;
+    }
+    domain_prot = (dacr >> (domain * 2)) & 3;
     if (domain_prot == 0 || domain_prot == 2) {
         if (type != 1) {
             code = 9; /* Section domain fault.  */
@@ -4368,6 +7241,8 @@
         if (desc & (1 << 18)) {
             /* Supersection.  */
             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
+            phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32;
+            phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36;
             *page_size = 0x1000000;
         } else {
             /* Section.  */
@@ -4378,13 +7253,16 @@
         xn = desc & (1 << 4);
         pxn = desc & 1;
         code = 13;
+        ns = extract32(desc, 19, 1);
     } else {
         if (arm_feature(env, ARM_FEATURE_PXN)) {
             pxn = (desc >> 2) & 1;
         }
+        ns = extract32(desc, 3, 1);
         /* Lookup l2 entry.  */
         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
-        desc = ldl_phys(cs->as, table);
+        desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
+                           mmu_idx, fsr, fi);
         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
         switch (desc & 3) {
         case 0: /* Page translation fault.  */
@@ -4409,31 +7287,44 @@
     if (domain_prot == 3) {
         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
     } else {
-        if (pxn && !is_user) {
+        if (pxn && !regime_is_user(env, mmu_idx)) {
             xn = 1;
         }
         if (xn && access_type == 2)
             goto do_fault;
 
-        /* The simplified model uses AP[0] as an access control bit.  */
-        if ((env->cp15.c1_sys & SCTLR_AFE) && (ap & 1) == 0) {
-            /* Access flag fault.  */
-            code = (code == 15) ? 6 : 3;
-            goto do_fault;
+        if (arm_feature(env, ARM_FEATURE_V6K) &&
+                (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
+            /* The simplified model uses AP[0] as an access control bit.  */
+            if ((ap & 1) == 0) {
+                /* Access flag fault.  */
+                code = (code == 15) ? 6 : 3;
+                goto do_fault;
+            }
+            *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
+        } else {
+            *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
         }
-        *prot = check_ap(env, ap, domain_prot, access_type, is_user);
-        if (!*prot) {
+        if (*prot && !xn) {
+            *prot |= PAGE_EXEC;
+        }
+        if (!(*prot & (1 << access_type))) {
             /* Access permission fault.  */
             goto do_fault;
         }
-        if (!xn) {
-            *prot |= PAGE_EXEC;
-        }
+    }
+    if (ns) {
+        /* The NS bit will (as required by the architecture) have no effect if
+         * the CPU doesn't support TZ or this is a non-secure translation
+         * regime, because the attribute will already be non-secure.
+         */
+        attrs->secure = false;
     }
     *phys_ptr = phys_addr;
-    return 0;
+    return false;
 do_fault:
-    return code | (domain << 4);
+    *fsr = code | (domain << 4);
+    return true;
 }
 
 /* Fault type for long-descriptor MMU fault reporting; this corresponds
@@ -4445,35 +7336,140 @@
     permission_fault = 3,
 } MMUFaultType;
 
-static int get_phys_addr_lpae(CPUARMState *env, target_ulong address,
-                              int access_type, int is_user,
-                              hwaddr *phys_ptr, int *prot,
-                              target_ulong *page_size_ptr)
+/*
+ * check_s2_mmu_setup
+ * @cpu:        ARMCPU
+ * @is_aa64:    True if the translation regime is in AArch64 state
+ * @startlevel: Suggested starting level
+ * @inputsize:  Bitsize of IPAs
+ * @stride:     Page-table stride (See the ARM ARM)
+ *
+ * Returns true if the suggested S2 translation parameters are OK and
+ * false otherwise.
+ */
+static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level,
+                               int inputsize, int stride)
 {
-    CPUState *cs = CPU(arm_env_get_cpu(env));
+    const int grainsize = stride + 3;
+    int startsizecheck;
+
+    /* Negative levels are never allowed.  */
+    if (level < 0) {
+        return false;
+    }
+
+    startsizecheck = inputsize - ((3 - level) * stride + grainsize);
+    if (startsizecheck < 1 || startsizecheck > stride + 4) {
+        return false;
+    }
+
+    if (is_aa64) {
+        CPUARMState *env = &cpu->env;
+        unsigned int pamax = arm_pamax(cpu);
+
+        switch (stride) {
+        case 13: /* 64KB Pages.  */
+            if (level == 0 || (level == 1 && pamax <= 42)) {
+                return false;
+            }
+            break;
+        case 11: /* 16KB Pages.  */
+            if (level == 0 || (level == 1 && pamax <= 40)) {
+                return false;
+            }
+            break;
+        case 9: /* 4KB Pages.  */
+            if (level == 0 && pamax <= 42) {
+                return false;
+            }
+            break;
+        default:
+            g_assert_not_reached();
+        }
+
+        /* Inputsize checks.  */
+        if (inputsize > pamax &&
+            (arm_el_is_aa64(env, 1) || inputsize > 40)) {
+            /* This is CONSTRAINED UNPREDICTABLE and we choose to fault.  */
+            return false;
+        }
+    } else {
+        /* AArch32 only supports 4KB pages. Assert on that.  */
+        assert(stride == 9);
+
+        if (level == 0) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
+                               int access_type, ARMMMUIdx mmu_idx,
+                               hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
+                               target_ulong *page_size_ptr, uint32_t *fsr,
+                               ARMMMUFaultInfo *fi)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
     /* Read an LPAE long-descriptor translation table. */
     MMUFaultType fault_type = translation_fault;
-    uint32_t level = 1;
-    uint32_t epd;
-    int32_t tsz;
+    uint32_t level;
+    uint32_t epd = 0;
+    int32_t t0sz, t1sz;
     uint32_t tg;
     uint64_t ttbr;
     int ttbr_select;
-    hwaddr descaddr, descmask;
+    hwaddr descaddr, indexmask, indexmask_grainsize;
     uint32_t tableattrs;
     target_ulong page_size;
     uint32_t attrs;
-    int32_t granule_sz = 9;
-    int32_t va_size = 32;
+    int32_t stride = 9;
+    int32_t addrsize;
+    int inputsize;
     int32_t tbi = 0;
+    TCR *tcr = regime_tcr(env, mmu_idx);
+    int ap, ns, xn, pxn;
+    uint32_t el = regime_el(env, mmu_idx);
+    bool ttbr1_valid = true;
+    uint64_t descaddrmask;
+    bool aarch64 = arm_el_is_aa64(env, el);
 
-    if (arm_el_is_aa64(env, 1)) {
-        va_size = 64;
-        if (extract64(address, 55, 1))
-            tbi = extract64(env->cp15.c2_control, 38, 1);
-        else
-            tbi = extract64(env->cp15.c2_control, 37, 1);
+    /* TODO:
+     * This code does not handle the different format TCR for VTCR_EL2.
+     * This code also does not support shareability levels.
+     * Attribute and permission bit handling should also be checked when adding
+     * support for those page table walks.
+     */
+    if (aarch64) {
+        level = 0;
+        addrsize = 64;
+        if (el > 1) {
+            if (mmu_idx != ARMMMUIdx_S2NS) {
+                tbi = extract64(tcr->raw_tcr, 20, 1);
+            }
+        } else {
+            if (extract64(address, 55, 1)) {
+                tbi = extract64(tcr->raw_tcr, 38, 1);
+            } else {
+                tbi = extract64(tcr->raw_tcr, 37, 1);
+            }
+        }
         tbi *= 8;
+
+        /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it
+         * invalid.
+         */
+        if (el > 1) {
+            ttbr1_valid = false;
+        }
+    } else {
+        level = 1;
+        addrsize = 32;
+        /* There is no TTBR1 for EL2 */
+        if (el == 2) {
+            ttbr1_valid = false;
+        }
     }
 
     /* Determine whether this address is in the region controlled by
@@ -4481,26 +7477,48 @@
      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
      */
-    uint32_t t0sz = extract32(env->cp15.c2_control, 0, 6);
-    if (arm_el_is_aa64(env, 1)) {
+    if (aarch64) {
+        /* AArch64 translation.  */
+        t0sz = extract32(tcr->raw_tcr, 0, 6);
         t0sz = MIN(t0sz, 39);
         t0sz = MAX(t0sz, 16);
+    } else if (mmu_idx != ARMMMUIdx_S2NS) {
+        /* AArch32 stage 1 translation.  */
+        t0sz = extract32(tcr->raw_tcr, 0, 3);
+    } else {
+        /* AArch32 stage 2 translation.  */
+        bool sext = extract32(tcr->raw_tcr, 4, 1);
+        bool sign = extract32(tcr->raw_tcr, 3, 1);
+        /* Address size is 40-bit for a stage 2 translation,
+         * and t0sz can be negative (from -8 to 7),
+         * so we need to adjust it to use the TTBR selecting logic below.
+         */
+        addrsize = 40;
+        t0sz = sextract32(tcr->raw_tcr, 0, 4) + 8;
+
+        /* If the sign-extend bit is not the same as t0sz[3], the result
+         * is unpredictable. Flag this as a guest error.  */
+        if (sign != sext) {
+            qemu_log_mask(LOG_GUEST_ERROR,
+                          "AArch32: VTCR.S / VTCR.T0SZ[3] missmatch\n");
+        }
     }
-    uint32_t t1sz = extract32(env->cp15.c2_control, 16, 6);
-    if (arm_el_is_aa64(env, 1)) {
+    t1sz = extract32(tcr->raw_tcr, 16, 6);
+    if (aarch64) {
         t1sz = MIN(t1sz, 39);
         t1sz = MAX(t1sz, 16);
     }
-    if (t0sz && !extract64(address, va_size - t0sz, t0sz - tbi)) {
+    if (t0sz && !extract64(address, addrsize - t0sz, t0sz - tbi)) {
         /* there is a ttbr0 region and we are in it (high bits all zero) */
         ttbr_select = 0;
-    } else if (t1sz && !extract64(~address, va_size - t1sz, t1sz - tbi)) {
+    } else if (ttbr1_valid && t1sz &&
+               !extract64(~address, addrsize - t1sz, t1sz - tbi)) {
         /* there is a ttbr1 region and we are in it (high bits all one) */
         ttbr_select = 1;
     } else if (!t0sz) {
         /* ttbr0 region is "everything not in the ttbr1 region" */
         ttbr_select = 0;
-    } else if (!t1sz) {
+    } else if (!t1sz && ttbr1_valid) {
         /* ttbr1 region is "everything not in the ttbr0 region" */
         ttbr_select = 1;
     } else {
@@ -4517,76 +7535,125 @@
      * we will always flush the TLB any time the ASID is changed).
      */
     if (ttbr_select == 0) {
-        ttbr = env->cp15.ttbr0_el1;
-        epd = extract32(env->cp15.c2_control, 7, 1);
-        tsz = t0sz;
+        ttbr = regime_ttbr(env, mmu_idx, 0);
+        if (el < 2) {
+            epd = extract32(tcr->raw_tcr, 7, 1);
+        }
+        inputsize = addrsize - t0sz;
 
-        tg = extract32(env->cp15.c2_control, 14, 2);
+        tg = extract32(tcr->raw_tcr, 14, 2);
         if (tg == 1) { /* 64KB pages */
-            granule_sz = 13;
+            stride = 13;
         }
         if (tg == 2) { /* 16KB pages */
-            granule_sz = 11;
+            stride = 11;
         }
     } else {
-        ttbr = env->cp15.ttbr1_el1;
-        epd = extract32(env->cp15.c2_control, 23, 1);
-        tsz = t1sz;
+        /* We should only be here if TTBR1 is valid */
+        assert(ttbr1_valid);
 
-        tg = extract32(env->cp15.c2_control, 30, 2);
+        ttbr = regime_ttbr(env, mmu_idx, 1);
+        epd = extract32(tcr->raw_tcr, 23, 1);
+        inputsize = addrsize - t1sz;
+
+        tg = extract32(tcr->raw_tcr, 30, 2);
         if (tg == 3)  { /* 64KB pages */
-            granule_sz = 13;
+            stride = 13;
         }
         if (tg == 1) { /* 16KB pages */
-            granule_sz = 11;
+            stride = 11;
         }
     }
 
+    /* Here we should have set up all the parameters for the translation:
+     * inputsize, ttbr, epd, stride, tbi
+     */
+
     if (epd) {
-        /* Translation table walk disabled => Translation fault on TLB miss */
+        /* Translation table walk disabled => Translation fault on TLB miss
+         * Note: This is always 0 on 64-bit EL2 and EL3.
+         */
         goto do_fault;
     }
 
-    /* The starting level depends on the virtual address size (which can be
-     * up to 48 bits) and the translation granule size. It indicates the number
-     * of strides (granule_sz bits at a time) needed to consume the bits
-     * of the input address. In the pseudocode this is:
-     *  level = 4 - RoundUp((inputsize - grainsize) / stride)
-     * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
-     * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
-     * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
-     *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
-     *     = 4 - (va_size - tsz - 4) / granule_sz;
-     */
-    level = 4 - (va_size - tsz - 4) / granule_sz;
+    if (mmu_idx != ARMMMUIdx_S2NS) {
+        /* The starting level depends on the virtual address size (which can
+         * be up to 48 bits) and the translation granule size. It indicates
+         * the number of strides (stride bits at a time) needed to
+         * consume the bits of the input address. In the pseudocode this is:
+         *  level = 4 - RoundUp((inputsize - grainsize) / stride)
+         * where their 'inputsize' is our 'inputsize', 'grainsize' is
+         * our 'stride + 3' and 'stride' is our 'stride'.
+         * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
+         * = 4 - (inputsize - stride - 3 + stride - 1) / stride
+         * = 4 - (inputsize - 4) / stride;
+         */
+        level = 4 - (inputsize - 4) / stride;
+    } else {
+        /* For stage 2 translations the starting level is specified by the
+         * VTCR_EL2.SL0 field (whose interpretation depends on the page size)
+         */
+        uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2);
+        uint32_t startlevel;
+        bool ok;
 
-    /* Clear the vaddr bits which aren't part of the within-region address,
-     * so that we don't have to special case things when calculating the
-     * first descriptor address.
-     */
-    if (tsz) {
-        address &= (1ULL << (va_size - tsz)) - 1;
+        if (!aarch64 || stride == 9) {
+            /* AArch32 or 4KB pages */
+            startlevel = 2 - sl0;
+        } else {
+            /* 16KB or 64KB pages */
+            startlevel = 3 - sl0;
+        }
+
+        /* Check that the starting level is valid. */
+        ok = check_s2_mmu_setup(cpu, aarch64, startlevel,
+                                inputsize, stride);
+        if (!ok) {
+            fault_type = translation_fault;
+            goto do_fault;
+        }
+        level = startlevel;
     }
 
-    descmask = (1ULL << (granule_sz + 3)) - 1;
+    indexmask_grainsize = (1ULL << (stride + 3)) - 1;
+    indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1;
 
     /* Now we can extract the actual base address from the TTBR */
     descaddr = extract64(ttbr, 0, 48);
-    descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
+    descaddr &= ~indexmask;
 
-    tableattrs = 0;
+    /* The address field in the descriptor goes up to bit 39 for ARMv7
+     * but up to bit 47 for ARMv8, but we use the descaddrmask
+     * up to bit 39 for AArch32, because we don't need other bits in that case
+     * to construct next descriptor address (anyway they should be all zeroes).
+     */
+    descaddrmask = ((1ull << (aarch64 ? 48 : 40)) - 1) &
+                   ~indexmask_grainsize;
+
+    /* Secure accesses start with the page table in secure memory and
+     * can be downgraded to non-secure at any step. Non-secure accesses
+     * remain non-secure. We implement this by just ORing in the NSTable/NS
+     * bits at each step.
+     */
+    tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4);
     for (;;) {
         uint64_t descriptor;
+        bool nstable;
 
-        descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
+        descaddr |= (address >> (stride * (4 - level))) & indexmask;
         descaddr &= ~7ULL;
-        descriptor = ldq_phys(cs->as, descaddr);
+        nstable = extract32(tableattrs, 4, 1);
+        descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fsr, fi);
+        if (fi->s1ptw) {
+            goto do_fault;
+        }
+
         if (!(descriptor & 1) ||
             (!(descriptor & 2) && (level == 3))) {
             /* Invalid, or the Reserved level 3 encoding */
             goto do_fault;
         }
-        descaddr = descriptor & 0xfffffff000ULL;
+        descaddr = descriptor & descaddrmask;
 
         if ((descriptor & 2) && (level < 3)) {
             /* Table entry. The top five bits are attributes which  may
@@ -4596,17 +7663,24 @@
              */
             tableattrs |= extract64(descriptor, 59, 5);
             level++;
+            indexmask = indexmask_grainsize;
             continue;
         }
         /* Block entry at level 1 or 2, or page entry at level 3.
          * These are basically the same thing, although the number
          * of bits we pull in from the vaddr varies.
          */
-        page_size = (1ULL << ((granule_sz * (4 - level)) + 3));
+        page_size = (1ULL << ((stride * (4 - level)) + 3));
         descaddr |= (address & (page_size - 1));
-        /* Extract attributes from the descriptor and merge with table attrs */
+        /* Extract attributes from the descriptor */
         attrs = extract64(descriptor, 2, 10)
             | (extract64(descriptor, 52, 12) << 10);
+
+        if (mmu_idx == ARMMMUIdx_S2NS) {
+            /* Stage 2 table descriptors do not include any attribute fields */
+            break;
+        }
+        /* Merge in attributes from table descriptors */
         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
@@ -4615,7 +7689,7 @@
         if (extract32(tableattrs, 2, 1)) {
             attrs &= ~(1 << 4);
         }
-        /* Since we're always in the Non-secure state, NSTable is ignored. */
+        attrs |= nstable << 3; /* NS */
         break;
     }
     /* Here descaddr is the final physical address, and attributes
@@ -4626,62 +7700,231 @@
         /* Access flag */
         goto do_fault;
     }
-    fault_type = permission_fault;
-    if (is_user && !(attrs & (1 << 4))) {
-        /* Unprivileged access not enabled */
-        goto do_fault;
-    }
-    *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
-    if ((arm_feature(env, ARM_FEATURE_V8) && is_user && (attrs & (1 << 12))) ||
-        (!arm_feature(env, ARM_FEATURE_V8) && (attrs & (1 << 12))) ||
-        (!is_user && (attrs & (1 << 11)))) {
-        /* XN/UXN or PXN. Since we only implement EL0/EL1 we unconditionally
-         * treat XN/UXN as UXN for v8.
-         */
-        if (access_type == 2) {
-            goto do_fault;
-        }
-        *prot &= ~PAGE_EXEC;
-    }
-    if (attrs & (1 << 5)) {
-        /* Write access forbidden */
-        if (access_type == 1) {
-            goto do_fault;
-        }
-        *prot &= ~PAGE_WRITE;
+
+    ap = extract32(attrs, 4, 2);
+    xn = extract32(attrs, 12, 1);
+
+    if (mmu_idx == ARMMMUIdx_S2NS) {
+        ns = true;
+        *prot = get_S2prot(env, ap, xn);
+    } else {
+        ns = extract32(attrs, 3, 1);
+        pxn = extract32(attrs, 11, 1);
+        *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn);
     }
 
+    fault_type = permission_fault;
+    if (!(*prot & (1 << access_type))) {
+        goto do_fault;
+    }
+
+    if (ns) {
+        /* The NS bit will (as required by the architecture) have no effect if
+         * the CPU doesn't support TZ or this is a non-secure translation
+         * regime, because the attribute will already be non-secure.
+         */
+        txattrs->secure = false;
+    }
     *phys_ptr = descaddr;
     *page_size_ptr = page_size;
-    return 0;
+    return false;
 
 do_fault:
     /* Long-descriptor format IFSR/DFSR value */
-    return (1 << 9) | (fault_type << 2) | level;
+    *fsr = (1 << 9) | (fault_type << 2) | level;
+    /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2.  */
+    fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS);
+    return true;
 }
 
-static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
-                             int access_type, int is_user,
-                             hwaddr *phys_ptr, int *prot)
+static inline void get_phys_addr_pmsav7_default(CPUARMState *env,
+                                                ARMMMUIdx mmu_idx,
+                                                int32_t address, int *prot)
+{
+    *prot = PAGE_READ | PAGE_WRITE;
+    switch (address) {
+    case 0xF0000000 ... 0xFFFFFFFF:
+        if (regime_sctlr(env, mmu_idx) & SCTLR_V) { /* hivecs execing is ok */
+            *prot |= PAGE_EXEC;
+        }
+        break;
+    case 0x00000000 ... 0x7FFFFFFF:
+        *prot |= PAGE_EXEC;
+        break;
+    }
+
+}
+
+static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address,
+                                 int access_type, ARMMMUIdx mmu_idx,
+                                 hwaddr *phys_ptr, int *prot, uint32_t *fsr)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    int n;
+    bool is_user = regime_is_user(env, mmu_idx);
+
+    *phys_ptr = address;
+    *prot = 0;
+
+    if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */
+        get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
+    } else { /* MPU enabled */
+        for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
+            /* region search */
+            uint32_t base = env->pmsav7.drbar[n];
+            uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5);
+            uint32_t rmask;
+            bool srdis = false;
+
+            if (!(env->pmsav7.drsr[n] & 0x1)) {
+                continue;
+            }
+
+            if (!rsize) {
+                qemu_log_mask(LOG_GUEST_ERROR, "DRSR.Rsize field can not be 0");
+                continue;
+            }
+            rsize++;
+            rmask = (1ull << rsize) - 1;
+
+            if (base & rmask) {
+                qemu_log_mask(LOG_GUEST_ERROR, "DRBAR %" PRIx32 " misaligned "
+                              "to DRSR region size, mask = %" PRIx32,
+                              base, rmask);
+                continue;
+            }
+
+            if (address < base || address > base + rmask) {
+                continue;
+            }
+
+            /* Region matched */
+
+            if (rsize >= 8) { /* no subregions for regions < 256 bytes */
+                int i, snd;
+                uint32_t srdis_mask;
+
+                rsize -= 3; /* sub region size (power of 2) */
+                snd = ((address - base) >> rsize) & 0x7;
+                srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1);
+
+                srdis_mask = srdis ? 0x3 : 0x0;
+                for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) {
+                    /* This will check in groups of 2, 4 and then 8, whether
+                     * the subregion bits are consistent. rsize is incremented
+                     * back up to give the region size, considering consistent
+                     * adjacent subregions as one region. Stop testing if rsize
+                     * is already big enough for an entire QEMU page.
+                     */
+                    int snd_rounded = snd & ~(i - 1);
+                    uint32_t srdis_multi = extract32(env->pmsav7.drsr[n],
+                                                     snd_rounded + 8, i);
+                    if (srdis_mask ^ srdis_multi) {
+                        break;
+                    }
+                    srdis_mask = (srdis_mask << i) | srdis_mask;
+                    rsize++;
+                }
+            }
+            if (rsize < TARGET_PAGE_BITS) {
+                qemu_log_mask(LOG_UNIMP, "No support for MPU (sub)region"
+                              "alignment of %" PRIu32 " bits. Minimum is %d\n",
+                              rsize, TARGET_PAGE_BITS);
+                continue;
+            }
+            if (srdis) {
+                continue;
+            }
+            break;
+        }
+
+        if (n == -1) { /* no hits */
+            if (cpu->pmsav7_dregion &&
+                (is_user || !(regime_sctlr(env, mmu_idx) & SCTLR_BR))) {
+                /* background fault */
+                *fsr = 0;
+                return true;
+            }
+            get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
+        } else { /* a MPU hit! */
+            uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3);
+
+            if (is_user) { /* User mode AP bit decoding */
+                switch (ap) {
+                case 0:
+                case 1:
+                case 5:
+                    break; /* no access */
+                case 3:
+                    *prot |= PAGE_WRITE;
+                    /* fall through */
+                case 2:
+                case 6:
+                    *prot |= PAGE_READ | PAGE_EXEC;
+                    break;
+                default:
+                    qemu_log_mask(LOG_GUEST_ERROR,
+                                  "Bad value for AP bits in DRACR %"
+                                  PRIx32 "\n", ap);
+                }
+            } else { /* Priv. mode AP bits decoding */
+                switch (ap) {
+                case 0:
+                    break; /* no access */
+                case 1:
+                case 2:
+                case 3:
+                    *prot |= PAGE_WRITE;
+                    /* fall through */
+                case 5:
+                case 6:
+                    *prot |= PAGE_READ | PAGE_EXEC;
+                    break;
+                default:
+                    qemu_log_mask(LOG_GUEST_ERROR,
+                                  "Bad value for AP bits in DRACR %"
+                                  PRIx32 "\n", ap);
+                }
+            }
+
+            /* execute never */
+            if (env->pmsav7.dracr[n] & (1 << 12)) {
+                *prot &= ~PAGE_EXEC;
+            }
+        }
+    }
+
+    *fsr = 0x00d; /* Permission fault */
+    return !(*prot & (1 << access_type));
+}
+
+static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
+                                 int access_type, ARMMMUIdx mmu_idx,
+                                 hwaddr *phys_ptr, int *prot, uint32_t *fsr)
 {
     int n;
     uint32_t mask;
     uint32_t base;
+    bool is_user = regime_is_user(env, mmu_idx);
 
     *phys_ptr = address;
     for (n = 7; n >= 0; n--) {
-	base = env->cp15.c6_region[n];
-	if ((base & 1) == 0)
-	    continue;
-	mask = 1 << ((base >> 1) & 0x1f);
-	/* Keep this shift separate from the above to avoid an
-	   (undefined) << 32.  */
-	mask = (mask << 1) - 1;
-	if (((base ^ address) & ~mask) == 0)
-	    break;
+        base = env->cp15.c6_region[n];
+        if ((base & 1) == 0) {
+            continue;
+        }
+        mask = 1 << ((base >> 1) & 0x1f);
+        /* Keep this shift separate from the above to avoid an
+           (undefined) << 32.  */
+        mask = (mask << 1) - 1;
+        if (((base ^ address) & ~mask) == 0) {
+            break;
+        }
     }
-    if (n < 0)
-	return 2;
+    if (n < 0) {
+        *fsr = 2;
+        return true;
+    }
 
     if (access_type == 2) {
         mask = env->cp15.pmsav5_insn_ap;
@@ -4691,34 +7934,41 @@
     mask = (mask >> (n * 4)) & 0xf;
     switch (mask) {
     case 0:
-	return 1;
+        *fsr = 1;
+        return true;
     case 1:
-	if (is_user)
-	  return 1;
-	*prot = PAGE_READ | PAGE_WRITE;
-	break;
+        if (is_user) {
+            *fsr = 1;
+            return true;
+        }
+        *prot = PAGE_READ | PAGE_WRITE;
+        break;
     case 2:
-	*prot = PAGE_READ;
-	if (!is_user)
-	    *prot |= PAGE_WRITE;
-	break;
+        *prot = PAGE_READ;
+        if (!is_user) {
+            *prot |= PAGE_WRITE;
+        }
+        break;
     case 3:
-	*prot = PAGE_READ | PAGE_WRITE;
-	break;
+        *prot = PAGE_READ | PAGE_WRITE;
+        break;
     case 5:
-	if (is_user)
-	    return 1;
-	*prot = PAGE_READ;
-	break;
+        if (is_user) {
+            *fsr = 1;
+            return true;
+        }
+        *prot = PAGE_READ;
+        break;
     case 6:
-	*prot = PAGE_READ;
-	break;
+        *prot = PAGE_READ;
+        break;
     default:
-	/* Bad permission.  */
-	return 1;
+        /* Bad permission.  */
+        *fsr = 1;
+        return true;
     }
     *prot |= PAGE_EXEC;
-    return 0;
+    return false;
 }
 
 /* get_phys_addr - get the physical address for this virtual address
@@ -4727,135 +7977,175 @@
  * by doing a translation table walk on MMU based systems or using the
  * MPU state on MPU based systems.
  *
- * Returns 0 if the translation was successful. Otherwise, phys_ptr,
- * prot and page_size are not filled in, and the return value provides
+ * Returns false if the translation was successful. Otherwise, phys_ptr, attrs,
+ * prot and page_size may not be filled in, and the populated fsr value provides
  * information on why the translation aborted, in the format of a
  * DFSR/IFSR fault register, with the following caveats:
  *  * we honour the short vs long DFSR format differences.
  *  * the WnR bit is never set (the caller must do this).
- *  * for MPU based systems we don't bother to return a full FSR format
+ *  * for PSMAv5 based systems we don't bother to return a full FSR format
  *    value.
  *
  * @env: CPUARMState
  * @address: virtual address to get physical address for
  * @access_type: 0 for read, 1 for write, 2 for execute
- * @is_user: 0 for privileged access, 1 for user
+ * @mmu_idx: MMU index indicating required translation regime
  * @phys_ptr: set to the physical address corresponding to the virtual address
+ * @attrs: set to the memory transaction attributes to use
  * @prot: set to the permissions for the page containing phys_ptr
  * @page_size: set to the size of the page containing phys_ptr
+ * @fsr: set to the DFSR/IFSR value on failure
  */
-static inline int get_phys_addr(CPUARMState *env, target_ulong address,
-                                int access_type, int is_user,
-                                hwaddr *phys_ptr, int *prot,
-                                target_ulong *page_size)
+static bool get_phys_addr(CPUARMState *env, target_ulong address,
+                          int access_type, ARMMMUIdx mmu_idx,
+                          hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
+                          target_ulong *page_size, uint32_t *fsr,
+                          ARMMMUFaultInfo *fi)
 {
-    /* Fast Context Switch Extension.  */
-    if (address < 0x02000000)
-        address += env->cp15.c13_fcse;
+    if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
+        /* Call ourselves recursively to do the stage 1 and then stage 2
+         * translations.
+         */
+        if (arm_feature(env, ARM_FEATURE_EL2)) {
+            hwaddr ipa;
+            int s2_prot;
+            int ret;
 
-    if ((env->cp15.c1_sys & SCTLR_M) == 0) {
+            ret = get_phys_addr(env, address, access_type,
+                                mmu_idx + ARMMMUIdx_S1NSE0, &ipa, attrs,
+                                prot, page_size, fsr, fi);
+
+            /* If S1 fails or S2 is disabled, return early.  */
+            if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
+                *phys_ptr = ipa;
+                return ret;
+            }
+
+            /* S1 is done. Now do S2 translation.  */
+            ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS,
+                                     phys_ptr, attrs, &s2_prot,
+                                     page_size, fsr, fi);
+            fi->s2addr = ipa;
+            /* Combine the S1 and S2 perms.  */
+            *prot &= s2_prot;
+            return ret;
+        } else {
+            /*
+             * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
+             */
+            mmu_idx += ARMMMUIdx_S1NSE0;
+        }
+    }
+
+    /* The page table entries may downgrade secure to non-secure, but
+     * cannot upgrade an non-secure translation regime's attributes
+     * to secure.
+     */
+    attrs->secure = regime_is_secure(env, mmu_idx);
+    attrs->user = regime_is_user(env, mmu_idx);
+
+    /* Fast Context Switch Extension. This doesn't exist at all in v8.
+     * In v7 and earlier it affects all stage 1 translations.
+     */
+    if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS
+        && !arm_feature(env, ARM_FEATURE_V8)) {
+        if (regime_el(env, mmu_idx) == 3) {
+            address += env->cp15.fcseidr_s;
+        } else {
+            address += env->cp15.fcseidr_ns;
+        }
+    }
+
+    /* pmsav7 has special handling for when MPU is disabled so call it before
+     * the common MMU/MPU disabled check below.
+     */
+    if (arm_feature(env, ARM_FEATURE_MPU) &&
+        arm_feature(env, ARM_FEATURE_V7)) {
+        *page_size = TARGET_PAGE_SIZE;
+        return get_phys_addr_pmsav7(env, address, access_type, mmu_idx,
+                                    phys_ptr, prot, fsr);
+    }
+
+    if (regime_translation_disabled(env, mmu_idx)) {
         /* MMU/MPU disabled.  */
         *phys_ptr = address;
         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
         *page_size = TARGET_PAGE_SIZE;
         return 0;
-    } else if (arm_feature(env, ARM_FEATURE_MPU)) {
+    }
+
+    if (arm_feature(env, ARM_FEATURE_MPU)) {
+        /* Pre-v7 MPU */
         *page_size = TARGET_PAGE_SIZE;
-	return get_phys_addr_mpu(env, address, access_type, is_user, phys_ptr,
-				 prot);
-    } else if (extended_addresses_enabled(env)) {
-        return get_phys_addr_lpae(env, address, access_type, is_user, phys_ptr,
-                                  prot, page_size);
-    } else if (env->cp15.c1_sys & SCTLR_XP) {
-        return get_phys_addr_v6(env, address, access_type, is_user, phys_ptr,
-                                prot, page_size);
+        return get_phys_addr_pmsav5(env, address, access_type, mmu_idx,
+                                    phys_ptr, prot, fsr);
+    }
+
+    if (regime_using_lpae_format(env, mmu_idx)) {
+        return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
+                                  attrs, prot, page_size, fsr, fi);
+    } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
+        return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
+                                attrs, prot, page_size, fsr, fi);
     } else {
-        return get_phys_addr_v5(env, address, access_type, is_user, phys_ptr,
-                                prot, page_size);
+        return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
+                                prot, page_size, fsr, fi);
     }
 }
 
-int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
-                             int access_type, int mmu_idx)
+/* Walk the page table and (if the mapping exists) add the page
+ * to the TLB. Return false on success, or true on failure. Populate
+ * fsr with ARM DFSR/IFSR fault register format value on failure.
+ */
+bool arm_tlb_fill(CPUState *cs, vaddr address,
+                  int access_type, int mmu_idx, uint32_t *fsr,
+                  ARMMMUFaultInfo *fi)
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
     hwaddr phys_addr;
     target_ulong page_size;
     int prot;
-    int ret, is_user;
-    uint32_t syn;
-    bool same_el = (arm_current_el(env) != 0);
+    int ret;
+    MemTxAttrs attrs = {};
 
-    is_user = mmu_idx == MMU_USER_IDX;
-    ret = get_phys_addr(env, address, access_type, is_user, &phys_addr, &prot,
-                        &page_size);
-    if (ret == 0) {
+    ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
+                        &attrs, &prot, &page_size, fsr, fi);
+    if (!ret) {
         /* Map a single [sub]page.  */
         phys_addr &= TARGET_PAGE_MASK;
         address &= TARGET_PAGE_MASK;
-        tlb_set_page(cs, address, phys_addr, prot, mmu_idx, page_size);
+        tlb_set_page_with_attrs(cs, address, phys_addr, attrs,
+                                prot, mmu_idx, page_size);
         return 0;
     }
 
-    /* AArch64 syndrome does not have an LPAE bit */
-    syn = ret & ~(1 << 9);
-
-    /* For insn and data aborts we assume there is no instruction syndrome
-     * information; this is always true for exceptions reported to EL1.
-     */
-    if (access_type == 2) {
-        syn = syn_insn_abort(same_el, 0, 0, syn);
-        cs->exception_index = EXCP_PREFETCH_ABORT;
-    } else {
-        syn = syn_data_abort(same_el, 0, 0, 0, access_type == 1, syn);
-        if (access_type == 1 && arm_feature(env, ARM_FEATURE_V6)) {
-            ret |= (1 << 11);
-        }
-        cs->exception_index = EXCP_DATA_ABORT;
-    }
-
-    env->exception.syndrome = syn;
-    env->exception.vaddress = address;
-    env->exception.fsr = ret;
-    return 1;
+    return ret;
 }
 
-hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
+hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
+                                         MemTxAttrs *attrs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
     hwaddr phys_addr;
     target_ulong page_size;
     int prot;
-    int ret;
+    bool ret;
+    uint32_t fsr;
+    ARMMMUFaultInfo fi = {};
 
-    ret = get_phys_addr(&cpu->env, addr, 0, 0, &phys_addr, &prot, &page_size);
+    *attrs = (MemTxAttrs) {};
 
-    if (ret != 0) {
+    ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env, false), &phys_addr,
+                        attrs, &prot, &page_size, &fsr, &fi);
+
+    if (ret) {
         return -1;
     }
-
     return phys_addr;
 }
 
-void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
-{
-    if ((env->uncached_cpsr & CPSR_M) == mode) {
-        env->regs[13] = val;
-    } else {
-        env->banked_r13[bank_number(mode)] = val;
-    }
-}
-
-uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
-{
-    if ((env->uncached_cpsr & CPSR_M) == mode) {
-        return env->regs[13];
-    } else {
-        return env->banked_r13[bank_number(mode)];
-    }
-}
-
 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
 {
     ARMCPU *cpu = arm_env_get_cpu(env);
@@ -4993,7 +8283,7 @@
         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
         void *hostaddr[maxidx];
         int try, i;
-        unsigned mmu_idx = cpu_mmu_index(env);
+        unsigned mmu_idx = cpu_mmu_index(env, false);
         TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
 
         for (try = 0; try < 2; try++) {
@@ -5529,7 +8819,7 @@
     /* ARM requires that S<->D conversion of any kind of NaN generates
      * a quiet NaN by forcing the most significant frac bit to 1.
      */
-    return float64_maybe_silence_nan(r);
+    return float64_maybe_silence_nan(r, &env->vfp.fp_status);
 }
 
 float32 VFP_HELPER(fcvts, d)(float64 x, CPUARMState *env)
@@ -5538,7 +8828,7 @@
     /* ARM requires that S<->D conversion of any kind of NaN generates
      * a quiet NaN by forcing the most significant frac bit to 1.
      */
-    return float32_maybe_silence_nan(r);
+    return float32_maybe_silence_nan(r, &env->vfp.fp_status);
 }
 
 /* VFP3 fixed point conversion.  */
@@ -5637,7 +8927,7 @@
     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
     float32 r = float16_to_float32(make_float16(a), ieee, s);
     if (ieee) {
-        return float32_maybe_silence_nan(r);
+        return float32_maybe_silence_nan(r, s);
     }
     return r;
 }
@@ -5647,7 +8937,7 @@
     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
     float16 r = float32_to_float16(a, ieee, s);
     if (ieee) {
-        r = float16_maybe_silence_nan(r);
+        r = float16_maybe_silence_nan(r, s);
     }
     return float16_val(r);
 }
@@ -5677,7 +8967,7 @@
     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
     float64 r = float16_to_float64(make_float16(a), ieee, &env->vfp.fp_status);
     if (ieee) {
-        return float64_maybe_silence_nan(r);
+        return float64_maybe_silence_nan(r, &env->vfp.fp_status);
     }
     return r;
 }
@@ -5687,7 +8977,7 @@
     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
     float16 r = float64_to_float16(a, ieee, &env->vfp.fp_status);
     if (ieee) {
-        r = float16_maybe_silence_nan(r);
+        r = float16_maybe_silence_nan(r, &env->vfp.fp_status);
     }
     return float16_val(r);
 }
@@ -5837,12 +9127,12 @@
 
     if (float32_is_any_nan(f32)) {
         float32 nan = f32;
-        if (float32_is_signaling_nan(f32)) {
+        if (float32_is_signaling_nan(f32, fpst)) {
             float_raise(float_flag_invalid, fpst);
-            nan = float32_maybe_silence_nan(f32);
+            nan = float32_maybe_silence_nan(f32, fpst);
         }
         if (fpst->default_nan_mode) {
-            nan =  float32_default_nan;
+            nan =  float32_default_nan(fpst);
         }
         return nan;
     } else if (float32_is_infinity(f32)) {
@@ -5891,12 +9181,12 @@
     /* Deal with any special cases */
     if (float64_is_any_nan(f64)) {
         float64 nan = f64;
-        if (float64_is_signaling_nan(f64)) {
+        if (float64_is_signaling_nan(f64, fpst)) {
             float_raise(float_flag_invalid, fpst);
-            nan = float64_maybe_silence_nan(f64);
+            nan = float64_maybe_silence_nan(f64, fpst);
         }
         if (fpst->default_nan_mode) {
-            nan =  float64_default_nan;
+            nan =  float64_default_nan(fpst);
         }
         return nan;
     } else if (float64_is_infinity(f64)) {
@@ -5912,7 +9202,7 @@
         } else {
             return float64_set_sign(float64_maxnorm, float64_is_neg(f64));
         }
-    } else if (f64_exp >= 1023 && fpst->flush_to_zero) {
+    } else if (f64_exp >= 2045 && fpst->flush_to_zero) {
         float_raise(float_flag_underflow, fpst);
         return float64_set_sign(float64_zero, float64_is_neg(f64));
     }
@@ -5998,12 +9288,12 @@
 
     if (float32_is_any_nan(f32)) {
         float32 nan = f32;
-        if (float32_is_signaling_nan(f32)) {
+        if (float32_is_signaling_nan(f32, s)) {
             float_raise(float_flag_invalid, s);
-            nan = float32_maybe_silence_nan(f32);
+            nan = float32_maybe_silence_nan(f32, s);
         }
         if (s->default_nan_mode) {
-            nan =  float32_default_nan;
+            nan =  float32_default_nan(s);
         }
         return nan;
     } else if (float32_is_zero(f32)) {
@@ -6011,7 +9301,7 @@
         return float32_set_sign(float32_infinity, float32_is_neg(f32));
     } else if (float32_is_neg(f32)) {
         float_raise(float_flag_invalid, s);
-        return float32_default_nan;
+        return float32_default_nan(s);
     } else if (float32_is_infinity(f32)) {
         return float32_zero;
     }
@@ -6062,12 +9352,12 @@
 
     if (float64_is_any_nan(f64)) {
         float64 nan = f64;
-        if (float64_is_signaling_nan(f64)) {
+        if (float64_is_signaling_nan(f64, s)) {
             float_raise(float_flag_invalid, s);
-            nan = float64_maybe_silence_nan(f64);
+            nan = float64_maybe_silence_nan(f64, s);
         }
         if (s->default_nan_mode) {
-            nan =  float64_default_nan;
+            nan =  float64_default_nan(s);
         }
         return nan;
     } else if (float64_is_zero(f64)) {
@@ -6075,7 +9365,7 @@
         return float64_set_sign(float64_infinity, float64_is_neg(f64));
     } else if (float64_is_neg(f64)) {
         float_raise(float_flag_invalid, s);
-        return float64_default_nan;
+        return float64_default_nan(s);
     } else if (float64_is_infinity(f64)) {
         return float64_zero;
     }
diff --git a/target-arm/helper.h b/target-arm/helper.h
index dec3728..84aa637 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -47,19 +47,24 @@
 DEF_HELPER_FLAGS_3(sel_flags, TCG_CALL_NO_RWG_SE,
                    i32, i32, i32, i32)
 DEF_HELPER_2(exception_internal, void, env, i32)
-DEF_HELPER_3(exception_with_syndrome, void, env, i32, i32)
+DEF_HELPER_4(exception_with_syndrome, void, env, i32, i32, i32)
+DEF_HELPER_1(setend, void, env)
 DEF_HELPER_1(wfi, void, env)
 DEF_HELPER_1(wfe, void, env)
+DEF_HELPER_1(yield, void, env)
 DEF_HELPER_1(pre_hvc, void, env)
 DEF_HELPER_2(pre_smc, void, env, i32)
 
+DEF_HELPER_1(check_breakpoints, void, env)
+
 DEF_HELPER_3(cpsr_write, void, env, i32, i32)
+DEF_HELPER_2(cpsr_write_eret, void, env, i32)
 DEF_HELPER_1(cpsr_read, i32, env)
 
 DEF_HELPER_3(v7m_msr, void, env, i32, i32)
 DEF_HELPER_2(v7m_mrs, i32, env, i32)
 
-DEF_HELPER_3(access_check_cp_reg, void, env, ptr, i32)
+DEF_HELPER_4(access_check_cp_reg, void, env, ptr, i32, i32)
 DEF_HELPER_3(set_cp_reg, void, env, ptr, i32)
 DEF_HELPER_2(get_cp_reg, i32, env, ptr)
 DEF_HELPER_3(set_cp_reg64, void, env, ptr, i64)
@@ -72,6 +77,9 @@
 DEF_HELPER_2(get_r13_banked, i32, env, i32)
 DEF_HELPER_3(set_r13_banked, void, env, i32, i32)
 
+DEF_HELPER_3(mrs_banked, i32, env, i32, i32)
+DEF_HELPER_4(msr_banked, void, env, i32, i32, i32)
+
 DEF_HELPER_2(get_user_reg, i32, env, i32)
 DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
diff --git a/target-arm/internals.h b/target-arm/internals.h
index 2dff4ff..ba8ef63 100644
--- a/target-arm/internals.h
+++ b/target-arm/internals.h
@@ -25,6 +25,16 @@
 #ifndef TARGET_ARM_INTERNALS_H
 #define TARGET_ARM_INTERNALS_H
 
+/* register banks for CPU modes */
+#define BANK_USRSYS 0
+#define BANK_SVC    1
+#define BANK_ABT    2
+#define BANK_UND    3
+#define BANK_IRQ    4
+#define BANK_FIQ    5
+#define BANK_HYP    6
+#define BANK_MON    7
+
 static inline bool excp_is_internal(int excp)
 {
     /* Return true if this exception number represents a QEMU-internal
@@ -36,6 +46,7 @@
         || excp == EXCP_HALTED
         || excp == EXCP_EXCEPTION_EXIT
         || excp == EXCP_KERNEL_TRAP
+        || excp == EXCP_SEMIHOST
         || excp == EXCP_STREX;
 }
 
@@ -58,23 +69,9 @@
     [EXCP_SMC] = "Secure Monitor Call",
     [EXCP_VIRQ] = "Virtual IRQ",
     [EXCP_VFIQ] = "Virtual FIQ",
+    [EXCP_SEMIHOST] = "Semihosting call",
 };
 
-static inline void arm_log_exception(int idx)
-{
-    if (qemu_loglevel_mask(CPU_LOG_INT)) {
-        const char *exc = NULL;
-
-        if (idx >= 0 && idx < ARRAY_SIZE(excnames)) {
-            exc = excnames[idx];
-        }
-        if (!exc) {
-            exc = "unknown";
-        }
-        qemu_log_mask(CPU_LOG_INT, "Taking exception %d [%s]\n", idx, exc);
-    }
-}
-
 /* Scale factor for generic timers, ie number of ns per tick.
  * This gives a 62.5MHz timer.
  */
@@ -82,19 +79,46 @@
 
 /*
  * For AArch64, map a given EL to an index in the banked_spsr array.
+ * Note that this mapping and the AArch32 mapping defined in bank_number()
+ * must agree such that the AArch64<->AArch32 SPSRs have the architecturally
+ * mandated mapping between each other.
  */
 static inline unsigned int aarch64_banked_spsr_index(unsigned int el)
 {
     static const unsigned int map[4] = {
-        [1] = 0, /* EL1.  */
-        [2] = 6, /* EL2.  */
-        [3] = 7, /* EL3.  */
+        [1] = BANK_SVC, /* EL1.  */
+        [2] = BANK_HYP, /* EL2.  */
+        [3] = BANK_MON, /* EL3.  */
     };
     assert(el >= 1 && el <= 3);
     return map[el];
 }
 
-int bank_number(int mode);
+/* Map CPU modes onto saved register banks.  */
+static inline int bank_number(int mode)
+{
+    switch (mode) {
+    case ARM_CPU_MODE_USR:
+    case ARM_CPU_MODE_SYS:
+        return BANK_USRSYS;
+    case ARM_CPU_MODE_SVC:
+        return BANK_SVC;
+    case ARM_CPU_MODE_ABT:
+        return BANK_ABT;
+    case ARM_CPU_MODE_UND:
+        return BANK_UND;
+    case ARM_CPU_MODE_IRQ:
+        return BANK_IRQ;
+    case ARM_CPU_MODE_FIQ:
+        return BANK_FIQ;
+    case ARM_CPU_MODE_HYP:
+        return BANK_HYP;
+    case ARM_CPU_MODE_MON:
+        return BANK_MON;
+    }
+    g_assert_not_reached();
+}
+
 void switch_mode(CPUARMState *, int);
 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu);
 void arm_translate_init(void);
@@ -147,15 +171,40 @@
     aarch64_restore_sp(env, cur_el);
 }
 
+/*
+ * arm_pamax
+ * @cpu: ARMCPU
+ *
+ * Returns the implementation defined bit-width of physical addresses.
+ * The ARMv8 reference manuals refer to this as PAMax().
+ */
+static inline unsigned int arm_pamax(ARMCPU *cpu)
+{
+    static const unsigned int pamax_map[] = {
+        [0] = 32,
+        [1] = 36,
+        [2] = 40,
+        [3] = 42,
+        [4] = 44,
+        [5] = 48,
+    };
+    unsigned int parange = extract32(cpu->id_aa64mmfr0, 0, 4);
+
+    /* id_aa64mmfr0 is a read-only register so values outside of the
+     * supported mappings can be considered an implementation error.  */
+    assert(parange < ARRAY_SIZE(pamax_map));
+    return pamax_map[parange];
+}
+
 /* Return true if extended addresses are enabled.
  * This is always the case if our translation regime is 64 bit,
  * but depends on TTBCR.EAE for 32 bit.
  */
 static inline bool extended_addresses_enabled(CPUARMState *env)
 {
-    return arm_el_is_aa64(env, 1)
-        || ((arm_feature(env, ARM_FEATURE_LPAE)
-             && (env->cp15.c2_control & TTBCR_EAE)));
+    TCR *tcr = &env->cp15.tcr_el[arm_is_secure(env) ? 3 : 1];
+    return arm_el_is_aa64(env, 1) ||
+           (arm_feature(env, ARM_FEATURE_LPAE) && (tcr->raw_tcr & TTBCR_EAE));
 }
 
 /* Valid Syndrome Register EC field values */
@@ -199,7 +248,9 @@
 
 #define ARM_EL_EC_SHIFT 26
 #define ARM_EL_IL_SHIFT 25
+#define ARM_EL_ISV_SHIFT 24
 #define ARM_EL_IL (1 << ARM_EL_IL_SHIFT)
+#define ARM_EL_ISV (1 << ARM_EL_ISV_SHIFT)
 
 /* Utility functions for constructing various kinds of syndrome value.
  * Note that in general we follow the AArch64 syndrome values; in a
@@ -230,10 +281,10 @@
     return (EC_AA64_SMC << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
 }
 
-static inline uint32_t syn_aa32_svc(uint32_t imm16, bool is_thumb)
+static inline uint32_t syn_aa32_svc(uint32_t imm16, bool is_16bit)
 {
     return (EC_AA32_SVC << ARM_EL_EC_SHIFT) | (imm16 & 0xffff)
-        | (is_thumb ? 0 : ARM_EL_IL);
+        | (is_16bit ? 0 : ARM_EL_IL);
 }
 
 static inline uint32_t syn_aa32_hvc(uint32_t imm16)
@@ -251,10 +302,10 @@
     return (EC_AA64_BKPT << ARM_EL_EC_SHIFT) | ARM_EL_IL | (imm16 & 0xffff);
 }
 
-static inline uint32_t syn_aa32_bkpt(uint32_t imm16, bool is_thumb)
+static inline uint32_t syn_aa32_bkpt(uint32_t imm16, bool is_16bit)
 {
     return (EC_AA32_BKPT << ARM_EL_EC_SHIFT) | (imm16 & 0xffff)
-        | (is_thumb ? 0 : ARM_EL_IL);
+        | (is_16bit ? 0 : ARM_EL_IL);
 }
 
 static inline uint32_t syn_aa64_sysregtrap(int op0, int op1, int op2,
@@ -268,74 +319,90 @@
 
 static inline uint32_t syn_cp14_rt_trap(int cv, int cond, int opc1, int opc2,
                                         int crn, int crm, int rt, int isread,
-                                        bool is_thumb)
+                                        bool is_16bit)
 {
     return (EC_CP14RTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc2 << 17) | (opc1 << 14)
         | (crn << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
 static inline uint32_t syn_cp15_rt_trap(int cv, int cond, int opc1, int opc2,
                                         int crn, int crm, int rt, int isread,
-                                        bool is_thumb)
+                                        bool is_16bit)
 {
     return (EC_CP15RTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc2 << 17) | (opc1 << 14)
         | (crn << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
 static inline uint32_t syn_cp14_rrt_trap(int cv, int cond, int opc1, int crm,
                                          int rt, int rt2, int isread,
-                                         bool is_thumb)
+                                         bool is_16bit)
 {
     return (EC_CP14RRTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc1 << 16)
         | (rt2 << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
 static inline uint32_t syn_cp15_rrt_trap(int cv, int cond, int opc1, int crm,
                                          int rt, int rt2, int isread,
-                                         bool is_thumb)
+                                         bool is_16bit)
 {
     return (EC_CP15RRTTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20) | (opc1 << 16)
         | (rt2 << 10) | (rt << 5) | (crm << 1) | isread;
 }
 
-static inline uint32_t syn_fp_access_trap(int cv, int cond, bool is_thumb)
+static inline uint32_t syn_fp_access_trap(int cv, int cond, bool is_16bit)
 {
     return (EC_ADVSIMDFPACCESSTRAP << ARM_EL_EC_SHIFT)
-        | (is_thumb ? 0 : ARM_EL_IL)
+        | (is_16bit ? 0 : ARM_EL_IL)
         | (cv << 24) | (cond << 20);
 }
 
 static inline uint32_t syn_insn_abort(int same_el, int ea, int s1ptw, int fsc)
 {
     return (EC_INSNABORT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
-        | (ea << 9) | (s1ptw << 7) | fsc;
+        | ARM_EL_IL | (ea << 9) | (s1ptw << 7) | fsc;
 }
 
-static inline uint32_t syn_data_abort(int same_el, int ea, int cm, int s1ptw,
-                                      int wnr, int fsc)
+static inline uint32_t syn_data_abort_no_iss(int same_el,
+                                             int ea, int cm, int s1ptw,
+                                             int wnr, int fsc)
 {
     return (EC_DATAABORT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
-        | (ea << 9) | (cm << 8) | (s1ptw << 7) | (wnr << 6) | fsc;
+           | ARM_EL_IL
+           | (ea << 9) | (cm << 8) | (s1ptw << 7) | (wnr << 6) | fsc;
+}
+
+static inline uint32_t syn_data_abort_with_iss(int same_el,
+                                               int sas, int sse, int srt,
+                                               int sf, int ar,
+                                               int ea, int cm, int s1ptw,
+                                               int wnr, int fsc,
+                                               bool is_16bit)
+{
+    return (EC_DATAABORT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
+           | (is_16bit ? 0 : ARM_EL_IL)
+           | ARM_EL_ISV | (sas << 22) | (sse << 21) | (srt << 16)
+           | (sf << 15) | (ar << 14)
+           | (ea << 9) | (cm << 8) | (s1ptw << 7) | (wnr << 6) | fsc;
 }
 
 static inline uint32_t syn_swstep(int same_el, int isv, int ex)
 {
     return (EC_SOFTWARESTEP << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
-        | (isv << 24) | (ex << 6) | 0x22;
+        | ARM_EL_IL | (isv << 24) | (ex << 6) | 0x22;
 }
 
 static inline uint32_t syn_watchpoint(int same_el, int cm, int wnr)
 {
     return (EC_WATCHPOINT << ARM_EL_EC_SHIFT) | (same_el << ARM_EL_EC_SHIFT)
-        | (cm << 8) | (wnr << 6) | 0x22;
+        | ARM_EL_IL | (cm << 8) | (wnr << 6) | 0x22;
 }
 
 static inline uint32_t syn_breakpoint(int same_el)
@@ -344,6 +411,12 @@
         | ARM_EL_IL | 0x22;
 }
 
+static inline uint32_t syn_wfx(int cv, int cond, int ti)
+{
+    return (EC_WFX_TRAP << ARM_EL_EC_SHIFT) |
+           (cv << 24) | (cond << 20) | ti;
+}
+
 /* Update a QEMU watchpoint based on the information the guest has set in the
  * DBGWCR<n>_EL1 and DBGWVR<n>_EL1 registers.
  */
@@ -363,6 +436,9 @@
  */
 void hw_breakpoint_update_all(ARMCPU *cpu);
 
+/* Callback function for checking if a watchpoint should trigger. */
+bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp);
+
 /* Callback function for when a watchpoint or breakpoint triggers. */
 void arm_debug_excp_handler(CPUState *cs);
 
@@ -378,4 +454,39 @@
 void arm_handle_psci_call(ARMCPU *cpu);
 #endif
 
+/**
+ * ARMMMUFaultInfo: Information describing an ARM MMU Fault
+ * @s2addr: Address that caused a fault at stage 2
+ * @stage2: True if we faulted at stage 2
+ * @s1ptw: True if we faulted at stage 2 while doing a stage 1 page-table walk
+ */
+typedef struct ARMMMUFaultInfo ARMMMUFaultInfo;
+struct ARMMMUFaultInfo {
+    target_ulong s2addr;
+    bool stage2;
+    bool s1ptw;
+};
+
+/* Do a page table walk and add page to TLB if possible */
+bool arm_tlb_fill(CPUState *cpu, vaddr address, int rw, int mmu_idx,
+                  uint32_t *fsr, ARMMMUFaultInfo *fi);
+
+/* Return true if the stage 1 translation regime is using LPAE format page
+ * tables */
+bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx);
+
+/* Raise a data fault alignment exception for the specified virtual address */
+void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr,
+                                 MMUAccessType access_type,
+                                 int mmu_idx, uintptr_t retaddr,
+                                 unsigned size);
+
+/* Call the EL change hook if one has been registered */
+static inline void arm_call_el_change_hook(ARMCPU *cpu)
+{
+    if (cpu->el_change_hook) {
+        cpu->el_change_hook(cpu, cpu->el_change_hook_opaque);
+    }
+}
+
 #endif
diff --git a/target-arm/iwmmxt_helper.c b/target-arm/iwmmxt_helper.c
index a506914..7d87e1a 100644
--- a/target-arm/iwmmxt_helper.c
+++ b/target-arm/iwmmxt_helper.c
@@ -19,8 +19,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
diff --git a/target-arm/kvm-consts.h b/target-arm/kvm-consts.h
index aea12f1..a2c9518 100644
--- a/target-arm/kvm-consts.h
+++ b/target-arm/kvm-consts.h
@@ -15,7 +15,6 @@
 #define ARM_KVM_CONSTS_H
 
 #ifdef CONFIG_KVM
-#include "qemu/compiler.h"
 #include <linux/kvm.h>
 #include <linux/psci.h>
 
@@ -127,6 +126,8 @@
 #define QEMU_KVM_ARM_TARGET_AEM_V8 0
 #define QEMU_KVM_ARM_TARGET_FOUNDATION_V8 1
 #define QEMU_KVM_ARM_TARGET_CORTEX_A57 2
+#define QEMU_KVM_ARM_TARGET_XGENE_POTENZA 3
+#define QEMU_KVM_ARM_TARGET_CORTEX_A53 4
 
 /* There's no kernel define for this: sentinel value which
  * matches no KVM target value for either 64 or 32 bit
@@ -137,6 +138,8 @@
 MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_AEM_V8, KVM_ARM_TARGET_AEM_V8)
 MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_FOUNDATION_V8, KVM_ARM_TARGET_FOUNDATION_V8)
 MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_CORTEX_A57, KVM_ARM_TARGET_CORTEX_A57)
+MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_XGENE_POTENZA, KVM_ARM_TARGET_XGENE_POTENZA)
+MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_CORTEX_A53, KVM_ARM_TARGET_CORTEX_A53)
 #else
 MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_CORTEX_A15, KVM_ARM_TARGET_CORTEX_A15)
 MISMATCH_CHECK(QEMU_KVM_ARM_TARGET_CORTEX_A7, KVM_ARM_TARGET_CORTEX_A7)
diff --git a/target-arm/kvm-stub.c b/target-arm/kvm-stub.c
index cd1849f..b2c66df 100644
--- a/target-arm/kvm-stub.c
+++ b/target-arm/kvm-stub.c
@@ -9,7 +9,9 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "kvm_arm.h"
 
 bool write_kvmstate_to_list(ARMCPU *cpu)
@@ -17,7 +19,7 @@
     abort();
 }
 
-bool write_list_to_kvmstate(ARMCPU *cpu)
+bool write_list_to_kvmstate(ARMCPU *cpu, int level)
 {
     abort();
 }
diff --git a/target-arm/kvm.c b/target-arm/kvm.c
index f622289..815e279 100644
--- a/target-arm/kvm.c
+++ b/target-arm/kvm.c
@@ -8,25 +8,30 @@
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "cpu.h"
+#include "internals.h"
 #include "hw/arm/arm.h"
+#include "exec/memattrs.h"
+#include "hw/boards.h"
+#include "qemu/log.h"
 
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
 
+static bool cap_has_mp_state;
+
 int kvm_arm_vcpu_init(CPUState *cs)
 {
     ARMCPU *cpu = ARM_CPU(cs);
@@ -60,13 +65,18 @@
         goto err;
     }
 
+    if (!init) {
+        /* Caller doesn't want the VCPU to be initialized, so skip it */
+        goto finish;
+    }
+
     ret = ioctl(vmfd, KVM_ARM_PREFERRED_TARGET, init);
     if (ret >= 0) {
         ret = ioctl(cpufd, KVM_ARM_VCPU_INIT, init);
         if (ret < 0) {
             goto err;
         }
-    } else {
+    } else if (cpus_to_try) {
         /* Old kernel which doesn't know about the
          * PREFERRED_TARGET ioctl: we know it will only support
          * creating one kind of guest CPU which is its preferred
@@ -83,8 +93,15 @@
         if (ret < 0) {
             goto err;
         }
+    } else {
+        /* Treat a NULL cpus_to_try argument the same as an empty
+         * list, which means we will fail the call since this must
+         * be an old kernel which doesn't support PREFERRED_TARGET.
+         */
+        goto err;
     }
 
+finish:
     fdarray[0] = kvmfd;
     fdarray[1] = vmfd;
     fdarray[2] = cpufd;
@@ -152,13 +169,15 @@
     .class_size = sizeof(ARMHostCPUClass),
 };
 
-int kvm_arch_init(KVMState *s)
+int kvm_arch_init(MachineState *ms, KVMState *s)
 {
     /* For ARM interrupt delivery is always asynchronous,
      * whether we are using an in-kernel VGIC or not.
      */
     kvm_async_interrupts_allowed = true;
 
+    cap_has_mp_state = kvm_check_extension(s, KVM_CAP_MP_STATE);
+
     type_register_static(&host_arm_cpu_type_info);
 
     return 0;
@@ -282,6 +301,94 @@
     memory_region_ref(kd->mr);
 }
 
+static int compare_u64(const void *a, const void *b)
+{
+    if (*(uint64_t *)a > *(uint64_t *)b) {
+        return 1;
+    }
+    if (*(uint64_t *)a < *(uint64_t *)b) {
+        return -1;
+    }
+    return 0;
+}
+
+/* Initialize the CPUState's cpreg list according to the kernel's
+ * definition of what CPU registers it knows about (and throw away
+ * the previous TCG-created cpreg list).
+ */
+int kvm_arm_init_cpreg_list(ARMCPU *cpu)
+{
+    struct kvm_reg_list rl;
+    struct kvm_reg_list *rlp;
+    int i, ret, arraylen;
+    CPUState *cs = CPU(cpu);
+
+    rl.n = 0;
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
+    if (ret != -E2BIG) {
+        return ret;
+    }
+    rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
+    rlp->n = rl.n;
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
+    if (ret) {
+        goto out;
+    }
+    /* Sort the list we get back from the kernel, since cpreg_tuples
+     * must be in strictly ascending order.
+     */
+    qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
+
+    for (i = 0, arraylen = 0; i < rlp->n; i++) {
+        if (!kvm_arm_reg_syncs_via_cpreg_list(rlp->reg[i])) {
+            continue;
+        }
+        switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
+        case KVM_REG_SIZE_U32:
+        case KVM_REG_SIZE_U64:
+            break;
+        default:
+            fprintf(stderr, "Can't handle size of register in kernel list\n");
+            ret = -EINVAL;
+            goto out;
+        }
+
+        arraylen++;
+    }
+
+    cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
+    cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
+    cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
+                                         arraylen);
+    cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
+                                        arraylen);
+    cpu->cpreg_array_len = arraylen;
+    cpu->cpreg_vmstate_array_len = arraylen;
+
+    for (i = 0, arraylen = 0; i < rlp->n; i++) {
+        uint64_t regidx = rlp->reg[i];
+        if (!kvm_arm_reg_syncs_via_cpreg_list(regidx)) {
+            continue;
+        }
+        cpu->cpreg_indexes[arraylen] = regidx;
+        arraylen++;
+    }
+    assert(cpu->cpreg_array_len == arraylen);
+
+    if (!write_kvmstate_to_list(cpu)) {
+        /* Shouldn't happen unless kernel is inconsistent about
+         * what registers exist.
+         */
+        fprintf(stderr, "Initial read of kernel register state failed\n");
+        ret = -EINVAL;
+        goto out;
+    }
+
+out:
+    g_free(rlp);
+    return ret;
+}
+
 bool write_kvmstate_to_list(ARMCPU *cpu)
 {
     CPUState *cs = CPU(cpu);
@@ -318,7 +425,7 @@
     return ok;
 }
 
-bool write_list_to_kvmstate(ARMCPU *cpu)
+bool write_list_to_kvmstate(ARMCPU *cpu, int level)
 {
     CPUState *cs = CPU(cpu);
     int i;
@@ -330,6 +437,10 @@
         uint32_t v32;
         int ret;
 
+        if (kvm_arm_cpreg_level(regidx) > level) {
+            continue;
+        }
+
         r.id = regidx;
         switch (regidx & KVM_REG_SIZE_MASK) {
         case KVM_REG_SIZE_U32:
@@ -354,17 +465,90 @@
     return ok;
 }
 
+void kvm_arm_reset_vcpu(ARMCPU *cpu)
+{
+    int ret;
+
+    /* Re-init VCPU so that all registers are set to
+     * their respective reset values.
+     */
+    ret = kvm_arm_vcpu_init(CPU(cpu));
+    if (ret < 0) {
+        fprintf(stderr, "kvm_arm_vcpu_init failed: %s\n", strerror(-ret));
+        abort();
+    }
+    if (!write_kvmstate_to_list(cpu)) {
+        fprintf(stderr, "write_kvmstate_to_list failed\n");
+        abort();
+    }
+}
+
+/*
+ * Update KVM's MP_STATE based on what QEMU thinks it is
+ */
+int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu)
+{
+    if (cap_has_mp_state) {
+        struct kvm_mp_state mp_state = {
+            .mp_state =
+            cpu->powered_off ? KVM_MP_STATE_STOPPED : KVM_MP_STATE_RUNNABLE
+        };
+        int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MP_STATE, &mp_state);
+        if (ret) {
+            fprintf(stderr, "%s: failed to set MP_STATE %d/%s\n",
+                    __func__, ret, strerror(-ret));
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
+/*
+ * Sync the KVM MP_STATE into QEMU
+ */
+int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu)
+{
+    if (cap_has_mp_state) {
+        struct kvm_mp_state mp_state;
+        int ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MP_STATE, &mp_state);
+        if (ret) {
+            fprintf(stderr, "%s: failed to get MP_STATE %d/%s\n",
+                    __func__, ret, strerror(-ret));
+            abort();
+        }
+        cpu->powered_off = (mp_state.mp_state == KVM_MP_STATE_STOPPED);
+    }
+
+    return 0;
+}
+
 void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run)
 {
 }
 
-void kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
+MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
+    return MEMTXATTRS_UNSPECIFIED;
 }
 
+
 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
 {
-    return 0;
+    int ret = 0;
+
+    switch (run->exit_reason) {
+    case KVM_EXIT_DEBUG:
+        if (kvm_arm_handle_debug(cs, &run->debug.arch)) {
+            ret = EXCP_DEBUG;
+        } /* otherwise return to guest */
+        break;
+    default:
+        qemu_log_mask(LOG_UNIMP, "%s: un-handled exit reason %d\n",
+                      __func__, run->exit_reason);
+        break;
+    }
+    return ret;
 }
 
 bool kvm_arch_stop_on_emulation_error(CPUState *cs)
@@ -387,60 +571,72 @@
     return 1;
 }
 
+/* The #ifdef protections are until 32bit headers are imported and can
+ * be removed once both 32 and 64 bit reach feature parity.
+ */
 void kvm_arch_update_guest_debug(CPUState *cs, struct kvm_guest_debug *dbg)
 {
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-}
-
-int kvm_arch_insert_sw_breakpoint(CPUState *cs,
-                                  struct kvm_sw_breakpoint *bp)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-int kvm_arch_insert_hw_breakpoint(target_ulong addr,
-                                  target_ulong len, int type)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-int kvm_arch_remove_hw_breakpoint(target_ulong addr,
-                                  target_ulong len, int type)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-int kvm_arch_remove_sw_breakpoint(CPUState *cs,
-                                  struct kvm_sw_breakpoint *bp)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
-    return -EINVAL;
-}
-
-void kvm_arch_remove_all_hw_breakpoints(void)
-{
-    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+#ifdef KVM_GUESTDBG_USE_SW_BP
+    if (kvm_sw_breakpoints_active(cs)) {
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_SW_BP;
+    }
+#endif
+#ifdef KVM_GUESTDBG_USE_HW
+    if (kvm_arm_hw_debug_active(cs)) {
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW;
+        kvm_arm_copy_hw_debug_data(&dbg->arch);
+    }
+#endif
 }
 
 void kvm_arch_init_irq_routing(KVMState *s)
 {
 }
 
-int kvm_arch_irqchip_create(KVMState *s)
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
 {
-    int ret;
+     if (machine_kernel_irqchip_split(ms)) {
+         perror("-machine kernel_irqchip=split is not supported on ARM.");
+         exit(1);
+    }
 
     /* If we can create the VGIC using the newer device control API, we
      * let the device do this when it initializes itself, otherwise we
      * fall back to the old API */
+    return kvm_check_extension(s, KVM_CAP_DEVICE_CTRL);
+}
 
-    ret = kvm_create_device(s, KVM_DEV_TYPE_ARM_VGIC_V2, true);
-    if (ret == 0) {
-        return 1;
+int kvm_arm_vgic_probe(void)
+{
+    if (kvm_create_device(kvm_state,
+                          KVM_DEV_TYPE_ARM_VGIC_V3, true) == 0) {
+        return 3;
+    } else if (kvm_create_device(kvm_state,
+                                 KVM_DEV_TYPE_ARM_VGIC_V2, true) == 0) {
+        return 2;
+    } else {
+        return 0;
     }
+}
 
+int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
+                             uint64_t address, uint32_t data, PCIDevice *dev)
+{
     return 0;
 }
+
+int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
+                                int vector, PCIDevice *dev)
+{
+    return 0;
+}
+
+int kvm_arch_release_virq_post(int virq)
+{
+    return 0;
+}
+
+int kvm_arch_msi_data_to_gsi(uint32_t data)
+{
+    return (data - 32) & 0xffff;
+}
diff --git a/target-arm/kvm32.c b/target-arm/kvm32.c
index 5ec4eb1..069da0c 100644
--- a/target-arm/kvm32.c
+++ b/target-arm/kvm32.c
@@ -8,21 +8,20 @@
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
-#include "cpu.h"
 #include "internals.h"
 #include "hw/arm/arm.h"
+#include "qemu/log.h"
 
 static inline void set_feature(uint64_t *features, int feature)
 {
@@ -51,17 +50,17 @@
     struct kvm_one_reg idregs[] = {
         {
             .id = KVM_REG_ARM | KVM_REG_SIZE_U32
-            | ENCODE_CP_REG(15, 0, 0, 0, 0, 0),
+            | ENCODE_CP_REG(15, 0, 0, 0, 0, 0, 0),
             .addr = (uintptr_t)&midr,
         },
         {
             .id = KVM_REG_ARM | KVM_REG_SIZE_U32
-            | ENCODE_CP_REG(15, 0, 0, 1, 0, 0),
+            | ENCODE_CP_REG(15, 0, 0, 0, 1, 0, 0),
             .addr = (uintptr_t)&id_pfr0,
         },
         {
             .id = KVM_REG_ARM | KVM_REG_SIZE_U32
-            | ENCODE_CP_REG(15, 0, 0, 2, 0, 0),
+            | ENCODE_CP_REG(15, 0, 0, 0, 2, 0, 0),
             .addr = (uintptr_t)&id_isar0,
         },
         {
@@ -138,7 +137,7 @@
     return true;
 }
 
-static bool reg_syncs_via_tuple_list(uint64_t regidx)
+bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
 {
     /* Return true if the regidx is a register we should synchronize
      * via the cpreg_tuples array (ie is not a core reg we sync by
@@ -153,24 +152,42 @@
     }
 }
 
-static int compare_u64(const void *a, const void *b)
+typedef struct CPRegStateLevel {
+    uint64_t regidx;
+    int level;
+} CPRegStateLevel;
+
+/* All coprocessor registers not listed in the following table are assumed to
+ * be of the level KVM_PUT_RUNTIME_STATE. If a register should be written less
+ * often, you must add it to this table with a state of either
+ * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
+ */
+static const CPRegStateLevel non_runtime_cpregs[] = {
+    { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE },
+};
+
+int kvm_arm_cpreg_level(uint64_t regidx)
 {
-    if (*(uint64_t *)a > *(uint64_t *)b) {
-        return 1;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) {
+        const CPRegStateLevel *l = &non_runtime_cpregs[i];
+        if (l->regidx == regidx) {
+            return l->level;
+        }
     }
-    if (*(uint64_t *)a < *(uint64_t *)b) {
-        return -1;
-    }
-    return 0;
+
+    return KVM_PUT_RUNTIME_STATE;
 }
 
+#define ARM_CPU_ID_MPIDR       0, 0, 0, 5
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
-    int i, ret, arraylen;
+    int ret;
     uint64_t v;
+    uint32_t mpidr;
     struct kvm_one_reg r;
-    struct kvm_reg_list rl;
-    struct kvm_reg_list *rlp;
     ARMCPU *cpu = ARM_CPU(cs);
 
     if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE) {
@@ -206,73 +223,18 @@
         return -EINVAL;
     }
 
-    /* Populate the cpreg list based on the kernel's idea
-     * of what registers exist (and throw away the TCG-created list).
+    /*
+     * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
+     * Currently KVM has its own idea about MPIDR assignment, so we
+     * override our defaults with what we get from KVM.
      */
-    rl.n = 0;
-    ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, &rl);
-    if (ret != -E2BIG) {
+    ret = kvm_get_one_reg(cs, ARM_CP15_REG32(ARM_CPU_ID_MPIDR), &mpidr);
+    if (ret) {
         return ret;
     }
-    rlp = g_malloc(sizeof(struct kvm_reg_list) + rl.n * sizeof(uint64_t));
-    rlp->n = rl.n;
-    ret = kvm_vcpu_ioctl(cs, KVM_GET_REG_LIST, rlp);
-    if (ret) {
-        goto out;
-    }
-    /* Sort the list we get back from the kernel, since cpreg_tuples
-     * must be in strictly ascending order.
-     */
-    qsort(&rlp->reg, rlp->n, sizeof(rlp->reg[0]), compare_u64);
+    cpu->mp_affinity = mpidr & ARM32_AFFINITY_MASK;
 
-    for (i = 0, arraylen = 0; i < rlp->n; i++) {
-        if (!reg_syncs_via_tuple_list(rlp->reg[i])) {
-            continue;
-        }
-        switch (rlp->reg[i] & KVM_REG_SIZE_MASK) {
-        case KVM_REG_SIZE_U32:
-        case KVM_REG_SIZE_U64:
-            break;
-        default:
-            fprintf(stderr, "Can't handle size of register in kernel list\n");
-            ret = -EINVAL;
-            goto out;
-        }
-
-        arraylen++;
-    }
-
-    cpu->cpreg_indexes = g_renew(uint64_t, cpu->cpreg_indexes, arraylen);
-    cpu->cpreg_values = g_renew(uint64_t, cpu->cpreg_values, arraylen);
-    cpu->cpreg_vmstate_indexes = g_renew(uint64_t, cpu->cpreg_vmstate_indexes,
-                                         arraylen);
-    cpu->cpreg_vmstate_values = g_renew(uint64_t, cpu->cpreg_vmstate_values,
-                                        arraylen);
-    cpu->cpreg_array_len = arraylen;
-    cpu->cpreg_vmstate_array_len = arraylen;
-
-    for (i = 0, arraylen = 0; i < rlp->n; i++) {
-        uint64_t regidx = rlp->reg[i];
-        if (!reg_syncs_via_tuple_list(regidx)) {
-            continue;
-        }
-        cpu->cpreg_indexes[arraylen] = regidx;
-        arraylen++;
-    }
-    assert(cpu->cpreg_array_len == arraylen);
-
-    if (!write_kvmstate_to_list(cpu)) {
-        /* Shouldn't happen unless kernel is inconsistent about
-         * what registers exist.
-         */
-        fprintf(stderr, "Initial read of kernel register state failed\n");
-        ret = -EINVAL;
-        goto out;
-    }
-
-out:
-    g_free(rlp);
-    return ret;
+    return kvm_arm_init_cpreg_list(cpu);
 }
 
 typedef struct Reg {
@@ -317,30 +279,30 @@
     COREREG(usr_regs.uregs[10], usr_regs[2]),
     COREREG(usr_regs.uregs[11], usr_regs[3]),
     COREREG(usr_regs.uregs[12], usr_regs[4]),
-    COREREG(usr_regs.uregs[13], banked_r13[0]),
-    COREREG(usr_regs.uregs[14], banked_r14[0]),
+    COREREG(usr_regs.uregs[13], banked_r13[BANK_USRSYS]),
+    COREREG(usr_regs.uregs[14], banked_r14[BANK_USRSYS]),
     /* R13, R14, SPSR for SVC, ABT, UND, IRQ banks */
-    COREREG(svc_regs[0], banked_r13[1]),
-    COREREG(svc_regs[1], banked_r14[1]),
-    COREREG64(svc_regs[2], banked_spsr[1]),
-    COREREG(abt_regs[0], banked_r13[2]),
-    COREREG(abt_regs[1], banked_r14[2]),
-    COREREG64(abt_regs[2], banked_spsr[2]),
-    COREREG(und_regs[0], banked_r13[3]),
-    COREREG(und_regs[1], banked_r14[3]),
-    COREREG64(und_regs[2], banked_spsr[3]),
-    COREREG(irq_regs[0], banked_r13[4]),
-    COREREG(irq_regs[1], banked_r14[4]),
-    COREREG64(irq_regs[2], banked_spsr[4]),
+    COREREG(svc_regs[0], banked_r13[BANK_SVC]),
+    COREREG(svc_regs[1], banked_r14[BANK_SVC]),
+    COREREG64(svc_regs[2], banked_spsr[BANK_SVC]),
+    COREREG(abt_regs[0], banked_r13[BANK_ABT]),
+    COREREG(abt_regs[1], banked_r14[BANK_ABT]),
+    COREREG64(abt_regs[2], banked_spsr[BANK_ABT]),
+    COREREG(und_regs[0], banked_r13[BANK_UND]),
+    COREREG(und_regs[1], banked_r14[BANK_UND]),
+    COREREG64(und_regs[2], banked_spsr[BANK_UND]),
+    COREREG(irq_regs[0], banked_r13[BANK_IRQ]),
+    COREREG(irq_regs[1], banked_r14[BANK_IRQ]),
+    COREREG64(irq_regs[2], banked_spsr[BANK_IRQ]),
     /* R8_fiq .. R14_fiq and SPSR_fiq */
     COREREG(fiq_regs[0], fiq_regs[0]),
     COREREG(fiq_regs[1], fiq_regs[1]),
     COREREG(fiq_regs[2], fiq_regs[2]),
     COREREG(fiq_regs[3], fiq_regs[3]),
     COREREG(fiq_regs[4], fiq_regs[4]),
-    COREREG(fiq_regs[5], banked_r13[5]),
-    COREREG(fiq_regs[6], banked_r14[5]),
-    COREREG64(fiq_regs[7], banked_spsr[5]),
+    COREREG(fiq_regs[5], banked_r13[BANK_FIQ]),
+    COREREG(fiq_regs[6], banked_r14[BANK_FIQ]),
+    COREREG64(fiq_regs[7], banked_spsr[BANK_FIQ]),
     /* R15 */
     COREREG(usr_regs.uregs[15], regs[15]),
     /* VFP system registers */
@@ -431,10 +393,12 @@
      * managed to update the CPUARMState with, and only allowing those
      * to be written back up into the kernel).
      */
-    if (!write_list_to_kvmstate(cpu)) {
+    if (!write_list_to_kvmstate(cpu, level)) {
         return EINVAL;
     }
 
+    kvm_arm_sync_mpstate_to_kvm(cpu);
+
     return ret;
 }
 
@@ -464,7 +428,7 @@
     if (ret) {
         return ret;
     }
-    cpsr_write(env, cpsr, 0xffffffff);
+    cpsr_write(env, cpsr, 0xffffffff, CPSRWriteRaw);
 
     /* Make sure the current mode regs are properly set */
     mode = env->uncached_cpsr & CPSR_M;
@@ -506,14 +470,60 @@
      */
     write_list_to_cpustate(cpu);
 
+    kvm_arm_sync_mpstate_to_qemu(cpu);
+
     return 0;
 }
 
-void kvm_arm_reset_vcpu(ARMCPU *cpu)
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
-    /* Re-init VCPU so that all registers are set to
-     * their respective reset values.
-     */
-    kvm_arm_vcpu_init(CPU(cpu));
-    write_kvmstate_to_list(cpu);
+    qemu_log_mask(LOG_UNIMP, "%s: guest debug not yet implemented\n", __func__);
+    return -EINVAL;
+}
+
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: guest debug not yet implemented\n", __func__);
+    return -EINVAL;
+}
+
+bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: guest debug not yet implemented\n", __func__);
+    return false;
+}
+
+int kvm_arch_insert_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+    return -EINVAL;
+}
+
+int kvm_arch_remove_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+    return -EINVAL;
+}
+
+void kvm_arch_remove_all_hw_breakpoints(void)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+}
+
+void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+}
+
+bool kvm_arm_hw_debug_active(CPUState *cs)
+{
+    return false;
+}
+
+int kvm_arm_pmu_create(CPUState *cs, int irq)
+{
+    qemu_log_mask(LOG_UNIMP, "%s: not implemented\n", __func__);
+    return 0;
 }
diff --git a/target-arm/kvm64.c b/target-arm/kvm64.c
index c615286..5faa76c 100644
--- a/target-arm/kvm64.c
+++ b/target-arm/kvm64.c
@@ -2,28 +2,427 @@
  * ARM implementation of KVM hooks, 64 bit specific code
  *
  * Copyright Mian-M. Hamayun 2013, Virtual Open Systems
+ * Copyright Alex Bennée 2014, Linaro
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  *
  */
 
-#include <stdio.h>
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
+#include <sys/ptrace.h>
 
+#include <linux/elf.h>
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/timer.h"
+#include "qemu/error-report.h"
+#include "qemu/host-utils.h"
+#include "exec/gdbstub.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
-#include "cpu.h"
 #include "internals.h"
 #include "hw/arm/arm.h"
 
+static bool have_guest_debug;
+
+/*
+ * Although the ARM implementation of hardware assisted debugging
+ * allows for different breakpoints per-core, the current GDB
+ * interface treats them as a global pool of registers (which seems to
+ * be the case for x86, ppc and s390). As a result we store one copy
+ * of registers which is used for all active cores.
+ *
+ * Write access is serialised by virtue of the GDB protocol which
+ * updates things. Read access (i.e. when the values are copied to the
+ * vCPU) is also gated by GDB's run control.
+ *
+ * This is not unreasonable as most of the time debugging kernels you
+ * never know which core will eventually execute your function.
+ */
+
+typedef struct {
+    uint64_t bcr;
+    uint64_t bvr;
+} HWBreakpoint;
+
+/* The watchpoint registers can cover more area than the requested
+ * watchpoint so we need to store the additional information
+ * somewhere. We also need to supply a CPUWatchpoint to the GDB stub
+ * when the watchpoint is hit.
+ */
+typedef struct {
+    uint64_t wcr;
+    uint64_t wvr;
+    CPUWatchpoint details;
+} HWWatchpoint;
+
+/* Maximum and current break/watch point counts */
+int max_hw_bps, max_hw_wps;
+GArray *hw_breakpoints, *hw_watchpoints;
+
+#define cur_hw_wps      (hw_watchpoints->len)
+#define cur_hw_bps      (hw_breakpoints->len)
+#define get_hw_bp(i)    (&g_array_index(hw_breakpoints, HWBreakpoint, i))
+#define get_hw_wp(i)    (&g_array_index(hw_watchpoints, HWWatchpoint, i))
+
+/**
+ * kvm_arm_init_debug() - check for guest debug capabilities
+ * @cs: CPUState
+ *
+ * kvm_check_extension returns the number of debug registers we have
+ * or 0 if we have none.
+ *
+ */
+static void kvm_arm_init_debug(CPUState *cs)
+{
+    have_guest_debug = kvm_check_extension(cs->kvm_state,
+                                           KVM_CAP_SET_GUEST_DEBUG);
+
+    max_hw_wps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_WPS);
+    hw_watchpoints = g_array_sized_new(true, true,
+                                       sizeof(HWWatchpoint), max_hw_wps);
+
+    max_hw_bps = kvm_check_extension(cs->kvm_state, KVM_CAP_GUEST_DEBUG_HW_BPS);
+    hw_breakpoints = g_array_sized_new(true, true,
+                                       sizeof(HWBreakpoint), max_hw_bps);
+    return;
+}
+
+/**
+ * insert_hw_breakpoint()
+ * @addr: address of breakpoint
+ *
+ * See ARM ARM D2.9.1 for details but here we are only going to create
+ * simple un-linked breakpoints (i.e. we don't chain breakpoints
+ * together to match address and context or vmid). The hardware is
+ * capable of fancier matching but that will require exposing that
+ * fanciness to GDB's interface
+ *
+ * D7.3.2 DBGBCR<n>_EL1, Debug Breakpoint Control Registers
+ *
+ *  31  24 23  20 19   16 15 14  13  12   9 8   5 4    3 2   1  0
+ * +------+------+-------+-----+----+------+-----+------+-----+---+
+ * | RES0 |  BT  |  LBN  | SSC | HMC| RES0 | BAS | RES0 | PMC | E |
+ * +------+------+-------+-----+----+------+-----+------+-----+---+
+ *
+ * BT: Breakpoint type (0 = unlinked address match)
+ * LBN: Linked BP number (0 = unused)
+ * SSC/HMC/PMC: Security, Higher and Priv access control (Table D-12)
+ * BAS: Byte Address Select (RES1 for AArch64)
+ * E: Enable bit
+ */
+static int insert_hw_breakpoint(target_ulong addr)
+{
+    HWBreakpoint brk = {
+        .bcr = 0x1,                             /* BCR E=1, enable */
+        .bvr = addr
+    };
+
+    if (cur_hw_bps >= max_hw_bps) {
+        return -ENOBUFS;
+    }
+
+    brk.bcr = deposit32(brk.bcr, 1, 2, 0x3);   /* PMC = 11 */
+    brk.bcr = deposit32(brk.bcr, 5, 4, 0xf);   /* BAS = RES1 */
+
+    g_array_append_val(hw_breakpoints, brk);
+
+    return 0;
+}
+
+/**
+ * delete_hw_breakpoint()
+ * @pc: address of breakpoint
+ *
+ * Delete a breakpoint and shuffle any above down
+ */
+
+static int delete_hw_breakpoint(target_ulong pc)
+{
+    int i;
+    for (i = 0; i < hw_breakpoints->len; i++) {
+        HWBreakpoint *brk = get_hw_bp(i);
+        if (brk->bvr == pc) {
+            g_array_remove_index(hw_breakpoints, i);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+/**
+ * insert_hw_watchpoint()
+ * @addr: address of watch point
+ * @len: size of area
+ * @type: type of watch point
+ *
+ * See ARM ARM D2.10. As with the breakpoints we can do some advanced
+ * stuff if we want to. The watch points can be linked with the break
+ * points above to make them context aware. However for simplicity
+ * currently we only deal with simple read/write watch points.
+ *
+ * D7.3.11 DBGWCR<n>_EL1, Debug Watchpoint Control Registers
+ *
+ *  31  29 28   24 23  21  20  19 16 15 14  13   12  5 4   3 2   1  0
+ * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
+ * | RES0 |  MASK | RES0 | WT | LBN | SSC | HMC | BAS | LSC | PAC | E |
+ * +------+-------+------+----+-----+-----+-----+-----+-----+-----+---+
+ *
+ * MASK: num bits addr mask (0=none,01/10=res,11=3 bits (8 bytes))
+ * WT: 0 - unlinked, 1 - linked (not currently used)
+ * LBN: Linked BP number (not currently used)
+ * SSC/HMC/PAC: Security, Higher and Priv access control (Table D2-11)
+ * BAS: Byte Address Select
+ * LSC: Load/Store control (01: load, 10: store, 11: both)
+ * E: Enable
+ *
+ * The bottom 2 bits of the value register are masked. Therefore to
+ * break on any sizes smaller than an unaligned word you need to set
+ * MASK=0, BAS=bit per byte in question. For larger regions (^2) you
+ * need to ensure you mask the address as required and set BAS=0xff
+ */
+
+static int insert_hw_watchpoint(target_ulong addr,
+                                target_ulong len, int type)
+{
+    HWWatchpoint wp = {
+        .wcr = 1, /* E=1, enable */
+        .wvr = addr & (~0x7ULL),
+        .details = { .vaddr = addr, .len = len }
+    };
+
+    if (cur_hw_wps >= max_hw_wps) {
+        return -ENOBUFS;
+    }
+
+    /*
+     * HMC=0 SSC=0 PAC=3 will hit EL0 or EL1, any security state,
+     * valid whether EL3 is implemented or not
+     */
+    wp.wcr = deposit32(wp.wcr, 1, 2, 3);
+
+    switch (type) {
+    case GDB_WATCHPOINT_READ:
+        wp.wcr = deposit32(wp.wcr, 3, 2, 1);
+        wp.details.flags = BP_MEM_READ;
+        break;
+    case GDB_WATCHPOINT_WRITE:
+        wp.wcr = deposit32(wp.wcr, 3, 2, 2);
+        wp.details.flags = BP_MEM_WRITE;
+        break;
+    case GDB_WATCHPOINT_ACCESS:
+        wp.wcr = deposit32(wp.wcr, 3, 2, 3);
+        wp.details.flags = BP_MEM_ACCESS;
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+    if (len <= 8) {
+        /* we align the address and set the bits in BAS */
+        int off = addr & 0x7;
+        int bas = (1 << len) - 1;
+
+        wp.wcr = deposit32(wp.wcr, 5 + off, 8 - off, bas);
+    } else {
+        /* For ranges above 8 bytes we need to be a power of 2 */
+        if (is_power_of_2(len)) {
+            int bits = ctz64(len);
+
+            wp.wvr &= ~((1 << bits) - 1);
+            wp.wcr = deposit32(wp.wcr, 24, 4, bits);
+            wp.wcr = deposit32(wp.wcr, 5, 8, 0xff);
+        } else {
+            return -ENOBUFS;
+        }
+    }
+
+    g_array_append_val(hw_watchpoints, wp);
+    return 0;
+}
+
+
+static bool check_watchpoint_in_range(int i, target_ulong addr)
+{
+    HWWatchpoint *wp = get_hw_wp(i);
+    uint64_t addr_top, addr_bottom = wp->wvr;
+    int bas = extract32(wp->wcr, 5, 8);
+    int mask = extract32(wp->wcr, 24, 4);
+
+    if (mask) {
+        addr_top = addr_bottom + (1 << mask);
+    } else {
+        /* BAS must be contiguous but can offset against the base
+         * address in DBGWVR */
+        addr_bottom = addr_bottom + ctz32(bas);
+        addr_top = addr_bottom + clo32(bas);
+    }
+
+    if (addr >= addr_bottom && addr <= addr_top) {
+        return true;
+    }
+
+    return false;
+}
+
+/**
+ * delete_hw_watchpoint()
+ * @addr: address of breakpoint
+ *
+ * Delete a breakpoint and shuffle any above down
+ */
+
+static int delete_hw_watchpoint(target_ulong addr,
+                                target_ulong len, int type)
+{
+    int i;
+    for (i = 0; i < cur_hw_wps; i++) {
+        if (check_watchpoint_in_range(i, addr)) {
+            g_array_remove_index(hw_watchpoints, i);
+            return 0;
+        }
+    }
+    return -ENOENT;
+}
+
+
+int kvm_arch_insert_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        return insert_hw_breakpoint(addr);
+        break;
+    case GDB_WATCHPOINT_READ:
+    case GDB_WATCHPOINT_WRITE:
+    case GDB_WATCHPOINT_ACCESS:
+        return insert_hw_watchpoint(addr, len, type);
+    default:
+        return -ENOSYS;
+    }
+}
+
+int kvm_arch_remove_hw_breakpoint(target_ulong addr,
+                                  target_ulong len, int type)
+{
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        return delete_hw_breakpoint(addr);
+        break;
+    case GDB_WATCHPOINT_READ:
+    case GDB_WATCHPOINT_WRITE:
+    case GDB_WATCHPOINT_ACCESS:
+        return delete_hw_watchpoint(addr, len, type);
+    default:
+        return -ENOSYS;
+    }
+}
+
+
+void kvm_arch_remove_all_hw_breakpoints(void)
+{
+    if (cur_hw_wps > 0) {
+        g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
+    }
+    if (cur_hw_bps > 0) {
+        g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
+    }
+}
+
+void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr)
+{
+    int i;
+    memset(ptr, 0, sizeof(struct kvm_guest_debug_arch));
+
+    for (i = 0; i < max_hw_wps; i++) {
+        HWWatchpoint *wp = get_hw_wp(i);
+        ptr->dbg_wcr[i] = wp->wcr;
+        ptr->dbg_wvr[i] = wp->wvr;
+    }
+    for (i = 0; i < max_hw_bps; i++) {
+        HWBreakpoint *bp = get_hw_bp(i);
+        ptr->dbg_bcr[i] = bp->bcr;
+        ptr->dbg_bvr[i] = bp->bvr;
+    }
+}
+
+bool kvm_arm_hw_debug_active(CPUState *cs)
+{
+    return ((cur_hw_wps > 0) || (cur_hw_bps > 0));
+}
+
+static bool find_hw_breakpoint(CPUState *cpu, target_ulong pc)
+{
+    int i;
+
+    for (i = 0; i < cur_hw_bps; i++) {
+        HWBreakpoint *bp = get_hw_bp(i);
+        if (bp->bvr == pc) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static CPUWatchpoint *find_hw_watchpoint(CPUState *cpu, target_ulong addr)
+{
+    int i;
+
+    for (i = 0; i < cur_hw_wps; i++) {
+        if (check_watchpoint_in_range(i, addr)) {
+            return &get_hw_wp(i)->details;
+        }
+    }
+    return NULL;
+}
+
+static bool kvm_arm_pmu_support_ctrl(CPUState *cs, struct kvm_device_attr *attr)
+{
+    return kvm_vcpu_ioctl(cs, KVM_HAS_DEVICE_ATTR, attr) == 0;
+}
+
+int kvm_arm_pmu_create(CPUState *cs, int irq)
+{
+    int err;
+
+    struct kvm_device_attr attr = {
+        .group = KVM_ARM_VCPU_PMU_V3_CTRL,
+        .addr = (intptr_t)&irq,
+        .attr = KVM_ARM_VCPU_PMU_V3_IRQ,
+        .flags = 0,
+    };
+
+    if (!kvm_arm_pmu_support_ctrl(cs, &attr)) {
+        return 0;
+    }
+
+    err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, &attr);
+    if (err < 0) {
+        fprintf(stderr, "KVM_SET_DEVICE_ATTR failed: %s\n",
+                strerror(-err));
+        abort();
+    }
+
+    attr.group = KVM_ARM_VCPU_PMU_V3_CTRL;
+    attr.attr = KVM_ARM_VCPU_PMU_V3_INIT;
+    attr.addr = 0;
+    attr.flags = 0;
+
+    err = kvm_vcpu_ioctl(cs, KVM_SET_DEVICE_ATTR, &attr);
+    if (err < 0) {
+        fprintf(stderr, "KVM_SET_DEVICE_ATTR failed: %s\n",
+                strerror(-err));
+        abort();
+    }
+
+    return 1;
+}
+
 static inline void set_feature(uint64_t *features, int feature)
 {
     *features |= 1ULL << feature;
@@ -76,13 +475,16 @@
     return true;
 }
 
+#define ARM_CPU_ID_MPIDR       3, 0, 0, 0, 5
+
 int kvm_arch_init_vcpu(CPUState *cs)
 {
     int ret;
+    uint64_t mpidr;
     ARMCPU *cpu = ARM_CPU(cs);
 
     if (cpu->kvm_target == QEMU_KVM_ARM_TARGET_NONE ||
-        !arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
+        !object_dynamic_cast(OBJECT(cpu), TYPE_AARCH64_CPU)) {
         fprintf(stderr, "KVM is not supported for this guest CPU type\n");
         return -EINVAL;
     }
@@ -96,6 +498,14 @@
         cpu->psci_version = 2;
         cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PSCI_0_2;
     }
+    if (!arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
+        cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_EL1_32BIT;
+    }
+    if (kvm_irqchip_in_kernel() &&
+        kvm_check_extension(cs->kvm_state, KVM_CAP_ARM_PMU_V3)) {
+        cpu->has_pmu = true;
+        cpu->kvm_init_features[0] |= 1 << KVM_ARM_VCPU_PMU_V3;
+    }
 
     /* Do KVM_ARM_VCPU_INIT ioctl */
     ret = kvm_arm_vcpu_init(cs);
@@ -103,24 +513,92 @@
         return ret;
     }
 
-    /* TODO : support for save/restore/reset of system regs via tuple list */
+    /*
+     * When KVM is in use, PSCI is emulated in-kernel and not by qemu.
+     * Currently KVM has its own idea about MPIDR assignment, so we
+     * override our defaults with what we get from KVM.
+     */
+    ret = kvm_get_one_reg(cs, ARM64_SYS_REG(ARM_CPU_ID_MPIDR), &mpidr);
+    if (ret) {
+        return ret;
+    }
+    cpu->mp_affinity = mpidr & ARM64_AFFINITY_MASK;
 
-    return 0;
+    kvm_arm_init_debug(cs);
+
+    return kvm_arm_init_cpreg_list(cpu);
+}
+
+bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx)
+{
+    /* Return true if the regidx is a register we should synchronize
+     * via the cpreg_tuples array (ie is not a core reg we sync by
+     * hand in kvm_arch_get/put_registers())
+     */
+    switch (regidx & KVM_REG_ARM_COPROC_MASK) {
+    case KVM_REG_ARM_CORE:
+        return false;
+    default:
+        return true;
+    }
+}
+
+typedef struct CPRegStateLevel {
+    uint64_t regidx;
+    int level;
+} CPRegStateLevel;
+
+/* All system registers not listed in the following table are assumed to be
+ * of the level KVM_PUT_RUNTIME_STATE. If a register should be written less
+ * often, you must add it to this table with a state of either
+ * KVM_PUT_RESET_STATE or KVM_PUT_FULL_STATE.
+ */
+static const CPRegStateLevel non_runtime_cpregs[] = {
+    { KVM_REG_ARM_TIMER_CNT, KVM_PUT_FULL_STATE },
+};
+
+int kvm_arm_cpreg_level(uint64_t regidx)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(non_runtime_cpregs); i++) {
+        const CPRegStateLevel *l = &non_runtime_cpregs[i];
+        if (l->regidx == regidx) {
+            return l->level;
+        }
+    }
+
+    return KVM_PUT_RUNTIME_STATE;
 }
 
 #define AARCH64_CORE_REG(x)   (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
                  KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
 
+#define AARCH64_SIMD_CORE_REG(x)   (KVM_REG_ARM64 | KVM_REG_SIZE_U128 | \
+                 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
+
+#define AARCH64_SIMD_CTRL_REG(x)   (KVM_REG_ARM64 | KVM_REG_SIZE_U32 | \
+                 KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x))
+
 int kvm_arch_put_registers(CPUState *cs, int level)
 {
     struct kvm_one_reg reg;
+    uint32_t fpr;
     uint64_t val;
     int i;
     int ret;
+    unsigned int el;
 
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
 
+    /* If we are in AArch32 mode then we need to copy the AArch32 regs to the
+     * AArch64 registers before pushing them out to 64-bit KVM.
+     */
+    if (!is_a64(env)) {
+        aarch64_sync_32_to_64(env);
+    }
+
     for (i = 0; i < 31; i++) {
         reg.id = AARCH64_CORE_REG(regs.regs[i]);
         reg.addr = (uintptr_t) &env->xregs[i];
@@ -150,7 +628,11 @@
     }
 
     /* Note that KVM thinks pstate is 64 bit but we use a uint32_t */
-    val = pstate_read(env);
+    if (is_a64(env)) {
+        val = pstate_read(env);
+    } else {
+        val = cpsr_read(env);
+    }
     reg.id = AARCH64_CORE_REG(regs.pstate);
     reg.addr = (uintptr_t) &val;
     ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
@@ -172,19 +654,70 @@
         return ret;
     }
 
+    /* Saved Program State Registers
+     *
+     * Before we restore from the banked_spsr[] array we need to
+     * ensure that any modifications to env->spsr are correctly
+     * reflected in the banks.
+     */
+    el = arm_current_el(env);
+    if (el > 0 && !is_a64(env)) {
+        i = bank_number(env->uncached_cpsr & CPSR_M);
+        env->banked_spsr[i] = env->spsr;
+    }
+
+    /* KVM 0-4 map to QEMU banks 1-5 */
     for (i = 0; i < KVM_NR_SPSR; i++) {
         reg.id = AARCH64_CORE_REG(spsr[i]);
-        reg.addr = (uintptr_t) &env->banked_spsr[i - 1];
+        reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
         ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
         if (ret) {
             return ret;
         }
     }
 
-    /* TODO:
-     * FP state
-     * system registers
+    /* Advanced SIMD and FP registers
+     * We map Qn = regs[2n+1]:regs[2n]
      */
+    for (i = 0; i < 32; i++) {
+        int rd = i << 1;
+        uint64_t fp_val[2];
+#ifdef HOST_WORDS_BIGENDIAN
+        fp_val[0] = env->vfp.regs[rd + 1];
+        fp_val[1] = env->vfp.regs[rd];
+#else
+        fp_val[1] = env->vfp.regs[rd + 1];
+        fp_val[0] = env->vfp.regs[rd];
+#endif
+        reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
+        reg.addr = (uintptr_t)(&fp_val);
+        ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    reg.addr = (uintptr_t)(&fpr);
+    fpr = vfp_get_fpsr(env);
+    reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    fpr = vfp_get_fpcr(env);
+    reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
+    ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+
+    if (!write_list_to_kvmstate(cpu, level)) {
+        return EINVAL;
+    }
+
+    kvm_arm_sync_mpstate_to_kvm(cpu);
+
     return ret;
 }
 
@@ -192,6 +725,8 @@
 {
     struct kvm_one_reg reg;
     uint64_t val;
+    uint32_t fpr;
+    unsigned int el;
     int i;
     int ret;
 
@@ -227,7 +762,13 @@
     if (ret) {
         return ret;
     }
-    pstate_write(env, val);
+
+    env->aarch64 = ((val & PSTATE_nRW) == 0);
+    if (is_a64(env)) {
+        pstate_write(env, val);
+    } else {
+        cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
+    }
 
     /* KVM puts SP_EL0 in regs.sp and SP_EL1 in regs.sp_el1. On the
      * QEMU side we keep the current SP in xregs[31] as well.
@@ -241,6 +782,15 @@
         return ret;
     }
 
+    /* If we are in AArch32 mode then we need to sync the AArch32 regs with the
+     * incoming AArch64 regs received from 64-bit KVM.
+     * We must perform this after all of the registers have been acquired from
+     * the kernel.
+     */
+    if (!is_a64(env)) {
+        aarch64_sync_64_to_32(env);
+    }
+
     reg.id = AARCH64_CORE_REG(elr_el1);
     reg.addr = (uintptr_t) &env->elr_el[1];
     ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
@@ -248,23 +798,174 @@
         return ret;
     }
 
+    /* Fetch the SPSR registers
+     *
+     * KVM SPSRs 0-4 map to QEMU banks 1-5
+     */
     for (i = 0; i < KVM_NR_SPSR; i++) {
         reg.id = AARCH64_CORE_REG(spsr[i]);
-        reg.addr = (uintptr_t) &env->banked_spsr[i - 1];
+        reg.addr = (uintptr_t) &env->banked_spsr[i + 1];
         ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
         if (ret) {
             return ret;
         }
     }
 
+    el = arm_current_el(env);
+    if (el > 0 && !is_a64(env)) {
+        i = bank_number(env->uncached_cpsr & CPSR_M);
+        env->spsr = env->banked_spsr[i];
+    }
+
+    /* Advanced SIMD and FP registers
+     * We map Qn = regs[2n+1]:regs[2n]
+     */
+    for (i = 0; i < 32; i++) {
+        uint64_t fp_val[2];
+        reg.id = AARCH64_SIMD_CORE_REG(fp_regs.vregs[i]);
+        reg.addr = (uintptr_t)(&fp_val);
+        ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        } else {
+            int rd = i << 1;
+#ifdef HOST_WORDS_BIGENDIAN
+            env->vfp.regs[rd + 1] = fp_val[0];
+            env->vfp.regs[rd] = fp_val[1];
+#else
+            env->vfp.regs[rd + 1] = fp_val[1];
+            env->vfp.regs[rd] = fp_val[0];
+#endif
+        }
+    }
+
+    reg.addr = (uintptr_t)(&fpr);
+    reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpsr);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+    vfp_set_fpsr(env, fpr);
+
+    reg.id = AARCH64_SIMD_CTRL_REG(fp_regs.fpcr);
+    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
+    if (ret) {
+        return ret;
+    }
+    vfp_set_fpcr(env, fpr);
+
+    if (!write_kvmstate_to_list(cpu)) {
+        return EINVAL;
+    }
+    /* Note that it's OK to have registers which aren't in CPUState,
+     * so we can ignore a failure return here.
+     */
+    write_list_to_cpustate(cpu);
+
+    kvm_arm_sync_mpstate_to_qemu(cpu);
+
     /* TODO: other registers */
     return ret;
 }
 
-void kvm_arm_reset_vcpu(ARMCPU *cpu)
+/* C6.6.29 BRK instruction */
+static const uint32_t brk_insn = 0xd4200000;
+
+int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
-    /* Re-init VCPU so that all registers are set to
-     * their respective reset values.
+    if (have_guest_debug) {
+        if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
+            cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
+            return -EINVAL;
+        }
+        return 0;
+    } else {
+        error_report("guest debug not supported on this kernel");
+        return -EINVAL;
+    }
+}
+
+int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
+{
+    static uint32_t brk;
+
+    if (have_guest_debug) {
+        if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&brk, 4, 0) ||
+            brk != brk_insn ||
+            cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
+            return -EINVAL;
+        }
+        return 0;
+    } else {
+        error_report("guest debug not supported on this kernel");
+        return -EINVAL;
+    }
+}
+
+/* See v8 ARM ARM D7.2.27 ESR_ELx, Exception Syndrome Register
+ *
+ * To minimise translating between kernel and user-space the kernel
+ * ABI just provides user-space with the full exception syndrome
+ * register value to be decoded in QEMU.
+ */
+
+bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit)
+{
+    int hsr_ec = debug_exit->hsr >> ARM_EL_EC_SHIFT;
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUClass *cc = CPU_GET_CLASS(cs);
+    CPUARMState *env = &cpu->env;
+
+    /* Ensure PC is synchronised */
+    kvm_cpu_synchronize_state(cs);
+
+    switch (hsr_ec) {
+    case EC_SOFTWARESTEP:
+        if (cs->singlestep_enabled) {
+            return true;
+        } else {
+            /*
+             * The kernel should have suppressed the guest's ability to
+             * single step at this point so something has gone wrong.
+             */
+            error_report("%s: guest single-step while debugging unsupported"
+                         " (%"PRIx64", %"PRIx32")\n",
+                         __func__, env->pc, debug_exit->hsr);
+            return false;
+        }
+        break;
+    case EC_AA64_BKPT:
+        if (kvm_find_sw_breakpoint(cs, env->pc)) {
+            return true;
+        }
+        break;
+    case EC_BREAKPOINT:
+        if (find_hw_breakpoint(cs, env->pc)) {
+            return true;
+        }
+        break;
+    case EC_WATCHPOINT:
+    {
+        CPUWatchpoint *wp = find_hw_watchpoint(cs, debug_exit->far);
+        if (wp) {
+            cs->watchpoint_hit = wp;
+            return true;
+        }
+        break;
+    }
+    default:
+        error_report("%s: unhandled debug exit (%"PRIx32", %"PRIx64")\n",
+                     __func__, debug_exit->hsr, env->pc);
+    }
+
+    /* If we are not handling the debug exception it must belong to
+     * the guest. Let's re-use the existing TCG interrupt code to set
+     * everything up properly.
      */
-    kvm_arm_vcpu_init(CPU(cpu));
+    cs->exception_index = EXCP_BKPT;
+    env->exception.syndrome = debug_exit->hsr;
+    env->exception.vaddress = debug_exit->far;
+    cc->do_interrupt(cs);
+
+    return false;
 }
diff --git a/target-arm/kvm_arm.h b/target-arm/kvm_arm.h
index af93105..a419368 100644
--- a/target-arm/kvm_arm.h
+++ b/target-arm/kvm_arm.h
@@ -47,8 +47,40 @@
                              uint64_t attr, int dev_fd);
 
 /**
+ * kvm_arm_init_cpreg_list:
+ * @cs: CPUState
+ *
+ * Initialize the CPUState's cpreg list according to the kernel's
+ * definition of what CPU registers it knows about (and throw away
+ * the previous TCG-created cpreg list).
+ *
+ * Returns: 0 if success, else < 0 error code
+ */
+int kvm_arm_init_cpreg_list(ARMCPU *cpu);
+
+/**
+ * kvm_arm_reg_syncs_via_cpreg_list
+ * regidx: KVM register index
+ *
+ * Return true if this KVM register should be synchronized via the
+ * cpreg list of arbitrary system registers, false if it is synchronized
+ * by hand using code in kvm_arch_get/put_registers().
+ */
+bool kvm_arm_reg_syncs_via_cpreg_list(uint64_t regidx);
+
+/**
+ * kvm_arm_cpreg_level
+ * regidx: KVM register index
+ *
+ * Return the level of this coprocessor/system register.  Return value is
+ * either KVM_PUT_RUNTIME_STATE, KVM_PUT_RESET_STATE, or KVM_PUT_FULL_STATE.
+ */
+int kvm_arm_cpreg_level(uint64_t regidx);
+
+/**
  * write_list_to_kvmstate:
  * @cpu: ARMCPU
+ * @level: the state level to sync
  *
  * For each register listed in the ARMCPU cpreg_indexes list, write
  * its value from the cpreg_values list into the kernel (via ioctl).
@@ -61,7 +93,7 @@
  * Note that we do not stop early on failure -- we will attempt
  * writing all registers in the list.
  */
-bool write_list_to_kvmstate(ARMCPU *cpu);
+bool write_list_to_kvmstate(ARMCPU *cpu, int level);
 
 /**
  * write_kvmstate_to_list:
@@ -92,9 +124,12 @@
  * kvm_arm_create_scratch_host_vcpu:
  * @cpus_to_try: array of QEMU_KVM_ARM_TARGET_* values (terminated with
  * QEMU_KVM_ARM_TARGET_NONE) to try as fallback if the kernel does not
- * know the PREFERRED_TARGET ioctl
+ * know the PREFERRED_TARGET ioctl. Passing NULL is the same as passing
+ * an empty array.
  * @fdarray: filled in with kvmfd, vmfd, cpufd file descriptors in that order
- * @init: filled in with the necessary values for creating a host vcpu
+ * @init: filled in with the necessary values for creating a host
+ * vcpu. If NULL is provided, will not init the vCPU (though the cpufd
+ * will still be set up).
  *
  * Create a scratch vcpu in its own VM of the type preferred by the host
  * kernel (as would be used for '-cpu host'), for purposes of probing it
@@ -140,6 +175,84 @@
  */
 bool kvm_arm_get_host_cpu_features(ARMHostCPUClass *ahcc);
 
+
+/**
+ * kvm_arm_sync_mpstate_to_kvm
+ * @cpu: ARMCPU
+ *
+ * If supported set the KVM MP_STATE based on QEMU's model.
+ */
+int kvm_arm_sync_mpstate_to_kvm(ARMCPU *cpu);
+
+/**
+ * kvm_arm_sync_mpstate_to_qemu
+ * @cpu: ARMCPU
+ *
+ * If supported get the MP_STATE from KVM and store in QEMU's model.
+ */
+int kvm_arm_sync_mpstate_to_qemu(ARMCPU *cpu);
+
+int kvm_arm_vgic_probe(void);
+
+int kvm_arm_pmu_create(CPUState *cs, int irq);
+
+#else
+
+static inline int kvm_arm_vgic_probe(void)
+{
+    return 0;
+}
+
+static inline int kvm_arm_pmu_create(CPUState *cs, int irq)
+{
+    return 0;
+}
+
 #endif
 
+static inline const char *gic_class_name(void)
+{
+    return kvm_irqchip_in_kernel() ? "kvm-arm-gic" : "arm_gic";
+}
+
+/**
+ * gicv3_class_name
+ *
+ * Return name of GICv3 class to use depending on whether KVM acceleration is
+ * in use. May throw an error if the chosen implementation is not available.
+ *
+ * Returns: class name to use
+ */
+const char *gicv3_class_name(void);
+
+/**
+ * kvm_arm_handle_debug:
+ * @cs: CPUState
+ * @debug_exit: debug part of the KVM exit structure
+ *
+ * Returns: TRUE if the debug exception was handled.
+ */
+bool kvm_arm_handle_debug(CPUState *cs, struct kvm_debug_exit_arch *debug_exit);
+
+/**
+ * kvm_arm_hw_debug_active:
+ * @cs: CPU State
+ *
+ * Return: TRUE if any hardware breakpoints in use.
+ */
+
+bool kvm_arm_hw_debug_active(CPUState *cs);
+
+/**
+ * kvm_arm_copy_hw_debug_data:
+ *
+ * @ptr: kvm_guest_debug_arch structure
+ *
+ * Copy the architecture specific debug registers into the
+ * kvm_guest_debug ioctl structure.
+ */
+struct kvm_guest_debug_arch;
+
+void kvm_arm_copy_hw_debug_data(struct kvm_guest_debug_arch *ptr);
+
 #endif
diff --git a/target-arm/machine.c b/target-arm/machine.c
index 6437690..7a6ca31 100644
--- a/target-arm/machine.c
+++ b/target-arm/machine.c
@@ -1,8 +1,13 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_arm.h"
 #include "internals.h"
+#include "migration/cpu.h"
 
 static bool vfp_needed(void *opaque)
 {
@@ -40,6 +45,7 @@
     .name = "cpu/vfp",
     .version_id = 3,
     .minimum_version_id = 3,
+    .needed = vfp_needed,
     .fields = (VMStateField[]) {
         VMSTATE_FLOAT64_ARRAY(env.vfp.regs, ARMCPU, 64),
         /* The xregs array is a little awkward because element 1 (FPSCR)
@@ -72,6 +78,7 @@
     .name = "cpu/iwmmxt",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = iwmmxt_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64_ARRAY(env.iwmmxt.regs, ARMCPU, 16),
         VMSTATE_UINT32_ARRAY(env.iwmmxt.cregs, ARMCPU, 16),
@@ -91,6 +98,7 @@
     .name = "cpu/m",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = m_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(env.v7m.other_sp, ARMCPU),
         VMSTATE_UINT32(env.v7m.vecbase, ARMCPU),
@@ -114,6 +122,7 @@
     .name = "cpu/thumb2ee",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = thumb2ee_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32(env.teecr, ARMCPU),
         VMSTATE_UINT32(env.teehbr, ARMCPU),
@@ -121,15 +130,53 @@
     }
 };
 
+static bool pmsav7_needed(void *opaque)
+{
+    ARMCPU *cpu = opaque;
+    CPUARMState *env = &cpu->env;
+
+    return arm_feature(env, ARM_FEATURE_MPU) &&
+           arm_feature(env, ARM_FEATURE_V7);
+}
+
+static bool pmsav7_rgnr_vmstate_validate(void *opaque, int version_id)
+{
+    ARMCPU *cpu = opaque;
+
+    return cpu->env.cp15.c6_rgnr < cpu->pmsav7_dregion;
+}
+
+static const VMStateDescription vmstate_pmsav7 = {
+    .name = "cpu/pmsav7",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pmsav7_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_VARRAY_UINT32(env.pmsav7.drbar, ARMCPU, pmsav7_dregion, 0,
+                              vmstate_info_uint32, uint32_t),
+        VMSTATE_VARRAY_UINT32(env.pmsav7.drsr, ARMCPU, pmsav7_dregion, 0,
+                              vmstate_info_uint32, uint32_t),
+        VMSTATE_VARRAY_UINT32(env.pmsav7.dracr, ARMCPU, pmsav7_dregion, 0,
+                              vmstate_info_uint32, uint32_t),
+        VMSTATE_VALIDATE("rgnr is valid", pmsav7_rgnr_vmstate_validate),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static int get_cpsr(QEMUFile *f, void *opaque, size_t size)
 {
     ARMCPU *cpu = opaque;
     CPUARMState *env = &cpu->env;
     uint32_t val = qemu_get_be32(f);
 
-    /* Avoid mode switch when restoring CPSR */
-    env->uncached_cpsr = val & CPSR_M;
-    cpsr_write(env, val, 0xffffffff);
+    env->aarch64 = ((val & PSTATE_nRW) == 0);
+
+    if (is_a64(env)) {
+        pstate_write(env, val);
+        return 0;
+    }
+
+    cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
     return 0;
 }
 
@@ -137,8 +184,15 @@
 {
     ARMCPU *cpu = opaque;
     CPUARMState *env = &cpu->env;
+    uint32_t val;
 
-    qemu_put_be32(f, cpsr_read(env));
+    if (is_a64(env)) {
+        val = pstate_read(env);
+    } else {
+        val = cpsr_read(env);
+    }
+
+    qemu_put_be32(f, val);
 }
 
 static const VMStateInfo vmstate_cpsr = {
@@ -200,7 +254,7 @@
     }
 
     if (kvm_enabled()) {
-        if (!write_list_to_kvmstate(cpu)) {
+        if (!write_list_to_kvmstate(cpu, KVM_PUT_FULL_STATE)) {
             return -1;
         }
         /* Note that it's OK for the TCG side not to know about
@@ -222,12 +276,14 @@
 
 const VMStateDescription vmstate_arm_cpu = {
     .name = "cpu",
-    .version_id = 21,
-    .minimum_version_id = 21,
+    .version_id = 22,
+    .minimum_version_id = 22,
     .pre_save = cpu_pre_save,
     .post_load = cpu_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_UINT32_ARRAY(env.regs, ARMCPU, 16),
+        VMSTATE_UINT64_ARRAY(env.xregs, ARMCPU, 32),
+        VMSTATE_UINT64(env.pc, ARMCPU),
         {
             .name = "cpsr",
             .version_id = 0,
@@ -261,26 +317,32 @@
         VMSTATE_UINT32(env.exception.syndrome, ARMCPU),
         VMSTATE_UINT32(env.exception.fsr, ARMCPU),
         VMSTATE_UINT64(env.exception.vaddress, ARMCPU),
-        VMSTATE_TIMER(gt_timer[GTIMER_PHYS], ARMCPU),
-        VMSTATE_TIMER(gt_timer[GTIMER_VIRT], ARMCPU),
+        VMSTATE_TIMER_PTR(gt_timer[GTIMER_PHYS], ARMCPU),
+        VMSTATE_TIMER_PTR(gt_timer[GTIMER_VIRT], ARMCPU),
         VMSTATE_BOOL(powered_off, ARMCPU),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection[]) {
-        {
-            .vmsd = &vmstate_vfp,
-            .needed = vfp_needed,
-        } , {
-            .vmsd = &vmstate_iwmmxt,
-            .needed = iwmmxt_needed,
-        } , {
-            .vmsd = &vmstate_m,
-            .needed = m_needed,
-        } , {
-            .vmsd = &vmstate_thumb2ee,
-            .needed = thumb2ee_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_vfp,
+        &vmstate_iwmmxt,
+        &vmstate_m,
+        &vmstate_thumb2ee,
+        &vmstate_pmsav7,
+        NULL
     }
 };
+
+const char *gicv3_class_name(void)
+{
+    if (kvm_irqchip_in_kernel()) {
+#ifdef TARGET_AARCH64
+        return "kvm-arm-gicv3";
+#else
+        error_report("KVM GICv3 acceleration is not supported on this "
+                     "platform");
+        exit(1);
+#endif
+    } else {
+        return "arm-gicv3";
+    }
+}
diff --git a/target-arm/monitor.c b/target-arm/monitor.c
new file mode 100644
index 0000000..299cb80
--- /dev/null
+++ b/target-arm/monitor.c
@@ -0,0 +1,83 @@
+/*
+ * QEMU monitor.c for ARM.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qmp-commands.h"
+#include "hw/boards.h"
+#include "kvm_arm.h"
+
+static GICCapability *gic_cap_new(int version)
+{
+    GICCapability *cap = g_new0(GICCapability, 1);
+    cap->version = version;
+    /* by default, support none */
+    cap->emulated = false;
+    cap->kernel = false;
+    return cap;
+}
+
+static GICCapabilityList *gic_cap_list_add(GICCapabilityList *head,
+                                           GICCapability *cap)
+{
+    GICCapabilityList *item = g_new0(GICCapabilityList, 1);
+    item->value = cap;
+    item->next = head;
+    return item;
+}
+
+static inline void gic_cap_kvm_probe(GICCapability *v2, GICCapability *v3)
+{
+#ifdef CONFIG_KVM
+    int fdarray[3];
+
+    if (!kvm_arm_create_scratch_host_vcpu(NULL, fdarray, NULL)) {
+        return;
+    }
+
+    /* Test KVM GICv2 */
+    if (kvm_device_supported(fdarray[1], KVM_DEV_TYPE_ARM_VGIC_V2)) {
+        v2->kernel = true;
+    }
+
+    /* Test KVM GICv3 */
+    if (kvm_device_supported(fdarray[1], KVM_DEV_TYPE_ARM_VGIC_V3)) {
+        v3->kernel = true;
+    }
+
+    kvm_arm_destroy_scratch_host_vcpu(fdarray);
+#endif
+}
+
+GICCapabilityList *qmp_query_gic_capabilities(Error **errp)
+{
+    GICCapabilityList *head = NULL;
+    GICCapability *v2 = gic_cap_new(2), *v3 = gic_cap_new(3);
+
+    v2->emulated = true;
+    v3->emulated = true;
+
+    gic_cap_kvm_probe(v2, v3);
+
+    head = gic_cap_list_add(head, v2);
+    head = gic_cap_list_add(head, v3);
+
+    return head;
+}
diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index 47d13e9..ebdf7c9 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -6,8 +6,7 @@
  *
  * This code is licensed under the GNU GPL v2.
  */
-#include <stdlib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -1052,7 +1051,7 @@
     if (tmp >= (ssize_t)sizeof(src1) * 8) { \
         if (src1) { \
             SET_QC(); \
-            dest = (1 << (sizeof(src1) * 8 - 1)); \
+            dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
             if (src1 > 0) { \
                 dest--; \
             } \
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index 62012c3..db802ea 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -16,23 +16,42 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "internals.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
 
-static void raise_exception(CPUARMState *env, int tt)
+static void raise_exception(CPUARMState *env, uint32_t excp,
+                            uint32_t syndrome, uint32_t target_el)
 {
-    ARMCPU *cpu = arm_env_get_cpu(env);
-    CPUState *cs = CPU(cpu);
+    CPUState *cs = CPU(arm_env_get_cpu(env));
 
-    cs->exception_index = tt;
+    assert(!excp_is_internal(excp));
+    cs->exception_index = excp;
+    env->exception.syndrome = syndrome;
+    env->exception.target_el = target_el;
     cpu_loop_exit(cs);
 }
 
+static int exception_target_el(CPUARMState *env)
+{
+    int target_el = MAX(1, arm_current_el(env));
+
+    /* No such thing as secure EL1 if EL3 is aarch32, so update the target EL
+     * to EL3 in this case.
+     */
+    if (arm_is_secure(env) && !arm_el_is_aa64(env, 3) && target_el == 1) {
+        target_el = 3;
+    }
+
+    return target_el;
+}
+
 uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
                           uint32_t rn, uint32_t maxindex)
 {
@@ -57,28 +76,143 @@
 
 #if !defined(CONFIG_USER_ONLY)
 
+static inline uint32_t merge_syn_data_abort(uint32_t template_syn,
+                                            unsigned int target_el,
+                                            bool same_el,
+                                            bool s1ptw, bool is_write,
+                                            int fsc)
+{
+    uint32_t syn;
+
+    /* ISV is only set for data aborts routed to EL2 and
+     * never for stage-1 page table walks faulting on stage 2.
+     *
+     * Furthermore, ISV is only set for certain kinds of load/stores.
+     * If the template syndrome does not have ISV set, we should leave
+     * it cleared.
+     *
+     * See ARMv8 specs, D7-1974:
+     * ISS encoding for an exception from a Data Abort, the
+     * ISV field.
+     */
+    if (!(template_syn & ARM_EL_ISV) || target_el != 2 || s1ptw) {
+        syn = syn_data_abort_no_iss(same_el,
+                                    0, 0, s1ptw, is_write, fsc);
+    } else {
+        /* Fields: IL, ISV, SAS, SSE, SRT, SF and AR come from the template
+         * syndrome created at translation time.
+         * Now we create the runtime syndrome with the remaining fields.
+         */
+        syn = syn_data_abort_with_iss(same_el,
+                                      0, 0, 0, 0, 0,
+                                      0, 0, s1ptw, is_write, fsc,
+                                      false);
+        /* Merge the runtime syndrome with the template syndrome.  */
+        syn |= template_syn;
+    }
+    return syn;
+}
+
 /* try to fill the TLB and return an exception if error. If retaddr is
  * NULL, it means that the function was called in C code (i.e. not
  * from generated code or from helper.c)
  */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
-    int ret;
+    bool ret;
+    uint32_t fsr = 0;
+    ARMMMUFaultInfo fi = {};
 
-    ret = arm_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = arm_tlb_fill(cs, addr, access_type, mmu_idx, &fsr, &fi);
     if (unlikely(ret)) {
         ARMCPU *cpu = ARM_CPU(cs);
         CPUARMState *env = &cpu->env;
+        uint32_t syn, exc;
+        unsigned int target_el;
+        bool same_el;
 
         if (retaddr) {
             /* now we have a real cpu fault */
             cpu_restore_state(cs, retaddr);
         }
-        raise_exception(env, cs->exception_index);
+
+        target_el = exception_target_el(env);
+        if (fi.stage2) {
+            target_el = 2;
+            env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
+        }
+        same_el = arm_current_el(env) == target_el;
+        /* AArch64 syndrome does not have an LPAE bit */
+        syn = fsr & ~(1 << 9);
+
+        /* For insn and data aborts we assume there is no instruction syndrome
+         * information; this is always true for exceptions reported to EL1.
+         */
+        if (access_type == MMU_INST_FETCH) {
+            syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
+            exc = EXCP_PREFETCH_ABORT;
+        } else {
+            syn = merge_syn_data_abort(env->exception.syndrome, target_el,
+                                       same_el, fi.s1ptw,
+                                       access_type == MMU_DATA_STORE, syn);
+            if (access_type == MMU_DATA_STORE
+                && arm_feature(env, ARM_FEATURE_V6)) {
+                fsr |= (1 << 11);
+            }
+            exc = EXCP_DATA_ABORT;
+        }
+
+        env->exception.vaddress = addr;
+        env->exception.fsr = fsr;
+        raise_exception(env, exc, syn, target_el);
     }
 }
-#endif
+
+/* Raise a data fault alignment exception for the specified virtual address */
+void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr,
+                                 MMUAccessType access_type,
+                                 int mmu_idx, uintptr_t retaddr,
+                                 unsigned size)
+{
+    ARMCPU *cpu = ARM_CPU(cs);
+    CPUARMState *env = &cpu->env;
+    int target_el;
+    bool same_el;
+    uint32_t syn;
+
+    (void)size;
+
+    if (retaddr) {
+        /* now we have a real cpu fault */
+        cpu_restore_state(cs, retaddr);
+    }
+
+    target_el = exception_target_el(env);
+    same_el = (arm_current_el(env) == target_el);
+
+    env->exception.vaddress = vaddr;
+
+    /* the DFSR for an alignment fault depends on whether we're using
+     * the LPAE long descriptor format, or the short descriptor format
+     */
+    if (arm_s1_regime_using_lpae_format(env, cpu_mmu_index(env, false))) {
+        env->exception.fsr = 0x21;
+    } else {
+        env->exception.fsr = 0x1;
+    }
+
+    if (access_type == MMU_DATA_STORE && arm_feature(env, ARM_FEATURE_V6)) {
+        env->exception.fsr |= (1 << 11);
+    }
+
+    syn = merge_syn_data_abort(env->exception.syndrome, target_el,
+                               same_el, 0, access_type == MMU_DATA_STORE,
+                               0x21);
+    raise_exception(env, EXCP_DATA_ABORT, syn, target_el);
+}
+
+#endif /* !defined(CONFIG_USER_ONLY) */
 
 uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 {
@@ -209,9 +343,77 @@
     return res;
 }
 
+void HELPER(setend)(CPUARMState *env)
+{
+    env->uncached_cpsr ^= CPSR_E;
+}
+
+/* Function checks whether WFx (WFI/WFE) instructions are set up to be trapped.
+ * The function returns the target EL (1-3) if the instruction is to be trapped;
+ * otherwise it returns 0 indicating it is not trapped.
+ */
+static inline int check_wfx_trap(CPUARMState *env, bool is_wfe)
+{
+    int cur_el = arm_current_el(env);
+    uint64_t mask;
+
+    /* If we are currently in EL0 then we need to check if SCTLR is set up for
+     * WFx instructions being trapped to EL1. These trap bits don't exist in v7.
+     */
+    if (cur_el < 1 && arm_feature(env, ARM_FEATURE_V8)) {
+        int target_el;
+
+        mask = is_wfe ? SCTLR_nTWE : SCTLR_nTWI;
+        if (arm_is_secure_below_el3(env) && !arm_el_is_aa64(env, 3)) {
+            /* Secure EL0 and Secure PL1 is at EL3 */
+            target_el = 3;
+        } else {
+            target_el = 1;
+        }
+
+        if (!(env->cp15.sctlr_el[target_el] & mask)) {
+            return target_el;
+        }
+    }
+
+    /* We are not trapping to EL1; trap to EL2 if HCR_EL2 requires it
+     * No need for ARM_FEATURE check as if HCR_EL2 doesn't exist the
+     * bits will be zero indicating no trap.
+     */
+    if (cur_el < 2 && !arm_is_secure(env)) {
+        mask = (is_wfe) ? HCR_TWE : HCR_TWI;
+        if (env->cp15.hcr_el2 & mask) {
+            return 2;
+        }
+    }
+
+    /* We are not trapping to EL1 or EL2; trap to EL3 if SCR_EL3 requires it */
+    if (cur_el < 3) {
+        mask = (is_wfe) ? SCR_TWE : SCR_TWI;
+        if (env->cp15.scr_el3 & mask) {
+            return 3;
+        }
+    }
+
+    return 0;
+}
+
 void HELPER(wfi)(CPUARMState *env)
 {
     CPUState *cs = CPU(arm_env_get_cpu(env));
+    int target_el = check_wfx_trap(env, false);
+
+    if (cpu_has_work(cs)) {
+        /* Don't bother to go into our "low power state" if
+         * we would just wake up immediately.
+         */
+        return;
+    }
+
+    if (target_el) {
+        env->pc -= 4;
+        raise_exception(env, EXCP_UDEF, syn_wfx(1, 0xe, 0), target_el);
+    }
 
     cs->exception_index = EXCP_HLT;
     cs->halted = 1;
@@ -220,10 +422,24 @@
 
 void HELPER(wfe)(CPUARMState *env)
 {
-    CPUState *cs = CPU(arm_env_get_cpu(env));
+    /* This is a hint instruction that is semantically different
+     * from YIELD even though we currently implement it identically.
+     * Don't actually halt the CPU, just yield back to top
+     * level loop. This is not going into a "low power state"
+     * (ie halting until some event occurs), so we never take
+     * a configurable trap to a different exception level.
+     */
+    HELPER(yield)(env);
+}
 
-    /* Don't actually halt the CPU, just yield back to top
-     * level loop
+void HELPER(yield)(CPUARMState *env)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+    CPUState *cs = CPU(cpu);
+
+    /* This is a non-trappable hint instruction that generally indicates
+     * that the guest is currently busy-looping. Yield control back to the
+     * top level loop so that a more deserving VCPU has a chance to run.
      */
     cs->exception_index = EXCP_YIELD;
     cpu_loop_exit(cs);
@@ -246,14 +462,9 @@
 
 /* Raise an exception with the specified syndrome register value */
 void HELPER(exception_with_syndrome)(CPUARMState *env, uint32_t excp,
-                                     uint32_t syndrome)
+                                     uint32_t syndrome, uint32_t target_el)
 {
-    CPUState *cs = CPU(arm_env_get_cpu(env));
-
-    assert(!excp_is_internal(excp));
-    cs->exception_index = excp;
-    env->exception.syndrome = syndrome;
-    cpu_loop_exit(cs);
+    raise_exception(env, excp, syndrome, target_el);
 }
 
 uint32_t HELPER(cpsr_read)(CPUARMState *env)
@@ -263,7 +474,15 @@
 
 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 {
-    cpsr_write(env, val, mask);
+    cpsr_write(env, val, mask, CPSRWriteByInstr);
+}
+
+/* Write the CPSR for a 32-bit exception return */
+void HELPER(cpsr_write_eret)(CPUARMState *env, uint32_t val)
+{
+    cpsr_write(env, val, CPSR_ERET_MASK, CPSRWriteExceptionReturn);
+
+    arm_call_el_change_hook(arm_env_get_cpu(env));
 }
 
 /* Access to user mode registers from privileged modes.  */
@@ -272,9 +491,9 @@
     uint32_t val;
 
     if (regno == 13) {
-        val = env->banked_r13[0];
+        val = env->banked_r13[BANK_USRSYS];
     } else if (regno == 14) {
-        val = env->banked_r14[0];
+        val = env->banked_r14[BANK_USRSYS];
     } else if (regno >= 8
                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
         val = env->usr_regs[regno - 8];
@@ -287,9 +506,9 @@
 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 {
     if (regno == 13) {
-        env->banked_r13[0] = val;
+        env->banked_r13[BANK_USRSYS] = val;
     } else if (regno == 14) {
-        env->banked_r14[0] = val;
+        env->banked_r14[BANK_USRSYS] = val;
     } else if (regno >= 8
                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
         env->usr_regs[regno - 8] = val;
@@ -298,33 +517,213 @@
     }
 }
 
-void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome)
+void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
+{
+    if ((env->uncached_cpsr & CPSR_M) == mode) {
+        env->regs[13] = val;
+    } else {
+        env->banked_r13[bank_number(mode)] = val;
+    }
+}
+
+uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
+{
+    if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_SYS) {
+        /* SRS instruction is UNPREDICTABLE from System mode; we UNDEF.
+         * Other UNPREDICTABLE and UNDEF cases were caught at translate time.
+         */
+        raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+                        exception_target_el(env));
+    }
+
+    if ((env->uncached_cpsr & CPSR_M) == mode) {
+        return env->regs[13];
+    } else {
+        return env->banked_r13[bank_number(mode)];
+    }
+}
+
+static void msr_mrs_banked_exc_checks(CPUARMState *env, uint32_t tgtmode,
+                                      uint32_t regno)
+{
+    /* Raise an exception if the requested access is one of the UNPREDICTABLE
+     * cases; otherwise return. This broadly corresponds to the pseudocode
+     * BankedRegisterAccessValid() and SPSRAccessValid(),
+     * except that we have already handled some cases at translate time.
+     */
+    int curmode = env->uncached_cpsr & CPSR_M;
+
+    if (curmode == tgtmode) {
+        goto undef;
+    }
+
+    if (tgtmode == ARM_CPU_MODE_USR) {
+        switch (regno) {
+        case 8 ... 12:
+            if (curmode != ARM_CPU_MODE_FIQ) {
+                goto undef;
+            }
+            break;
+        case 13:
+            if (curmode == ARM_CPU_MODE_SYS) {
+                goto undef;
+            }
+            break;
+        case 14:
+            if (curmode == ARM_CPU_MODE_HYP || curmode == ARM_CPU_MODE_SYS) {
+                goto undef;
+            }
+            break;
+        default:
+            break;
+        }
+    }
+
+    if (tgtmode == ARM_CPU_MODE_HYP) {
+        switch (regno) {
+        case 17: /* ELR_Hyp */
+            if (curmode != ARM_CPU_MODE_HYP && curmode != ARM_CPU_MODE_MON) {
+                goto undef;
+            }
+            break;
+        default:
+            if (curmode != ARM_CPU_MODE_MON) {
+                goto undef;
+            }
+            break;
+        }
+    }
+
+    return;
+
+undef:
+    raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+                    exception_target_el(env));
+}
+
+void HELPER(msr_banked)(CPUARMState *env, uint32_t value, uint32_t tgtmode,
+                        uint32_t regno)
+{
+    msr_mrs_banked_exc_checks(env, tgtmode, regno);
+
+    switch (regno) {
+    case 16: /* SPSRs */
+        env->banked_spsr[bank_number(tgtmode)] = value;
+        break;
+    case 17: /* ELR_Hyp */
+        env->elr_el[2] = value;
+        break;
+    case 13:
+        env->banked_r13[bank_number(tgtmode)] = value;
+        break;
+    case 14:
+        env->banked_r14[bank_number(tgtmode)] = value;
+        break;
+    case 8 ... 12:
+        switch (tgtmode) {
+        case ARM_CPU_MODE_USR:
+            env->usr_regs[regno - 8] = value;
+            break;
+        case ARM_CPU_MODE_FIQ:
+            env->fiq_regs[regno - 8] = value;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+uint32_t HELPER(mrs_banked)(CPUARMState *env, uint32_t tgtmode, uint32_t regno)
+{
+    msr_mrs_banked_exc_checks(env, tgtmode, regno);
+
+    switch (regno) {
+    case 16: /* SPSRs */
+        return env->banked_spsr[bank_number(tgtmode)];
+    case 17: /* ELR_Hyp */
+        return env->elr_el[2];
+    case 13:
+        return env->banked_r13[bank_number(tgtmode)];
+    case 14:
+        return env->banked_r14[bank_number(tgtmode)];
+    case 8 ... 12:
+        switch (tgtmode) {
+        case ARM_CPU_MODE_USR:
+            return env->usr_regs[regno - 8];
+        case ARM_CPU_MODE_FIQ:
+            return env->fiq_regs[regno - 8];
+        default:
+            g_assert_not_reached();
+        }
+    default:
+        g_assert_not_reached();
+    }
+}
+
+void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome,
+                                 uint32_t isread)
 {
     const ARMCPRegInfo *ri = rip;
+    int target_el;
 
     if (arm_feature(env, ARM_FEATURE_XSCALE) && ri->cp < 14
         && extract32(env->cp15.c15_cpar, ri->cp, 1) == 0) {
-        env->exception.syndrome = syndrome;
-        raise_exception(env, EXCP_UDEF);
+        raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
     }
 
     if (!ri->accessfn) {
         return;
     }
 
-    switch (ri->accessfn(env, ri)) {
+    switch (ri->accessfn(env, ri, isread)) {
     case CP_ACCESS_OK:
         return;
     case CP_ACCESS_TRAP:
-        env->exception.syndrome = syndrome;
+        target_el = exception_target_el(env);
+        break;
+    case CP_ACCESS_TRAP_EL2:
+        /* Requesting a trap to EL2 when we're in EL3 or S-EL0/1 is
+         * a bug in the access function.
+         */
+        assert(!arm_is_secure(env) && arm_current_el(env) != 3);
+        target_el = 2;
+        break;
+    case CP_ACCESS_TRAP_EL3:
+        target_el = 3;
         break;
     case CP_ACCESS_TRAP_UNCATEGORIZED:
-        env->exception.syndrome = syn_uncategorized();
+        target_el = exception_target_el(env);
+        syndrome = syn_uncategorized();
+        break;
+    case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
+        target_el = 2;
+        syndrome = syn_uncategorized();
+        break;
+    case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
+        target_el = 3;
+        syndrome = syn_uncategorized();
+        break;
+    case CP_ACCESS_TRAP_FP_EL2:
+        target_el = 2;
+        /* Since we are an implementation that takes exceptions on a trapped
+         * conditional insn only if the insn has passed its condition code
+         * check, we take the IMPDEF choice to always report CV=1 COND=0xe
+         * (which is also the required value for AArch64 traps).
+         */
+        syndrome = syn_fp_access_trap(1, 0xe, false);
+        break;
+    case CP_ACCESS_TRAP_FP_EL3:
+        target_el = 3;
+        syndrome = syn_fp_access_trap(1, 0xe, false);
         break;
     default:
         g_assert_not_reached();
     }
-    raise_exception(env, EXCP_UDEF);
+
+    raise_exception(env, EXCP_UDEF, syndrome, target_el);
 }
 
 void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
@@ -361,8 +760,11 @@
      * Note that SPSel is never OK from EL0; we rely on handle_msr_i()
      * to catch that case at translate time.
      */
-    if (arm_current_el(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
-        raise_exception(env, EXCP_UDEF);
+    if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
+        uint32_t syndrome = syn_aa64_sysregtrap(0, extract32(op, 0, 3),
+                                                extract32(op, 3, 3), 4,
+                                                imm, 0x1f, 0);
+        raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
     }
 
     switch (op) {
@@ -420,8 +822,8 @@
     }
 
     if (undef) {
-        env->exception.syndrome = syn_uncategorized();
-        raise_exception(env, EXCP_UDEF);
+        raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+                        exception_target_el(env));
     }
 }
 
@@ -431,12 +833,14 @@
     int cur_el = arm_current_el(env);
     bool secure = arm_is_secure(env);
     bool smd = env->cp15.scr_el3 & SCR_SMD;
-    /* On ARMv8 AArch32, SMD only applies to NS state.
-     * On ARMv7 SMD only applies to NS state and only if EL2 is available.
-     * For ARMv7 non EL2, we force SMD to zero so we don't need to re-check
-     * the EL2 condition here.
+    /* On ARMv8 with EL3 AArch64, SMD applies to both S and NS state.
+     * On ARMv8 with EL3 AArch32, or ARMv7 with the Virtualization
+     *  extensions, SMD only applies to NS state.
+     * On ARMv7 without the Virtualization extensions, the SMD bit
+     * doesn't exist, but we forbid the guest to set it to 1 in scr_write(),
+     * so we need not special case this here.
      */
-    bool undef = is_a64(env) ? smd : (!secure && smd);
+    bool undef = arm_feature(env, ARM_FEATURE_AARCH64) ? smd : smd && !secure;
 
     if (arm_is_psci_call(cpu, EXCP_SMC)) {
         /* If PSCI is enabled and this looks like a valid PSCI call then
@@ -450,13 +854,50 @@
         undef = true;
     } else if (!secure && cur_el == 1 && (env->cp15.hcr_el2 & HCR_TSC)) {
         /* In NS EL1, HCR controlled routing to EL2 has priority over SMD. */
-        env->exception.syndrome = syndrome;
-        raise_exception(env, EXCP_HYP_TRAP);
+        raise_exception(env, EXCP_HYP_TRAP, syndrome, 2);
     }
 
     if (undef) {
-        env->exception.syndrome = syn_uncategorized();
-        raise_exception(env, EXCP_UDEF);
+        raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+                        exception_target_el(env));
+    }
+}
+
+static int el_from_spsr(uint32_t spsr)
+{
+    /* Return the exception level that this SPSR is requesting a return to,
+     * or -1 if it is invalid (an illegal return)
+     */
+    if (spsr & PSTATE_nRW) {
+        switch (spsr & CPSR_M) {
+        case ARM_CPU_MODE_USR:
+            return 0;
+        case ARM_CPU_MODE_HYP:
+            return 2;
+        case ARM_CPU_MODE_FIQ:
+        case ARM_CPU_MODE_IRQ:
+        case ARM_CPU_MODE_SVC:
+        case ARM_CPU_MODE_ABT:
+        case ARM_CPU_MODE_UND:
+        case ARM_CPU_MODE_SYS:
+            return 1;
+        case ARM_CPU_MODE_MON:
+            /* Returning to Mon from AArch64 is never possible,
+             * so this is an illegal return.
+             */
+        default:
+            return -1;
+        }
+    } else {
+        if (extract32(spsr, 1, 1)) {
+            /* Return with reserved M[1] bit set */
+            return -1;
+        }
+        if (extract32(spsr, 0, 4) == 1) {
+            /* return to EL0 with M[0] bit set */
+            return -1;
+        }
+        return extract32(spsr, 2, 2);
     }
 }
 
@@ -465,7 +906,8 @@
     int cur_el = arm_current_el(env);
     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
     uint32_t spsr = env->banked_spsr[spsr_idx];
-    int new_el, i;
+    int new_el;
+    bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
 
     aarch64_save_sp(env, cur_el);
 
@@ -482,37 +924,51 @@
         spsr &= ~PSTATE_SS;
     }
 
-    if (spsr & PSTATE_nRW) {
-        /* TODO: We currently assume EL1/2/3 are running in AArch64.  */
+    new_el = el_from_spsr(spsr);
+    if (new_el == -1) {
+        goto illegal_return;
+    }
+    if (new_el > cur_el
+        || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
+        /* Disallow return to an EL which is unimplemented or higher
+         * than the current one.
+         */
+        goto illegal_return;
+    }
+
+    if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
+        /* Return to an EL which is configured for a different register width */
+        goto illegal_return;
+    }
+
+    if (new_el == 2 && arm_is_secure_below_el3(env)) {
+        /* Return to the non-existent secure-EL2 */
+        goto illegal_return;
+    }
+
+    if (new_el == 1 && (env->cp15.hcr_el2 & HCR_TGE)
+        && !arm_is_secure_below_el3(env)) {
+        goto illegal_return;
+    }
+
+    if (!return_to_aa64) {
         env->aarch64 = 0;
-        new_el = 0;
-        env->uncached_cpsr = 0x10;
-        cpsr_write(env, spsr, ~0);
+        /* We do a raw CPSR write because aarch64_sync_64_to_32()
+         * will sort the register banks out for us, and we've already
+         * caught all the bad-mode cases in el_from_spsr().
+         */
+        cpsr_write(env, spsr, ~0, CPSRWriteRaw);
         if (!arm_singlestep_active(env)) {
             env->uncached_cpsr &= ~PSTATE_SS;
         }
-        for (i = 0; i < 15; i++) {
-            env->regs[i] = env->xregs[i];
-        }
+        aarch64_sync_64_to_32(env);
 
-        env->regs[15] = env->elr_el[1] & ~0x1;
+        if (spsr & CPSR_T) {
+            env->regs[15] = env->elr_el[cur_el] & ~0x1;
+        } else {
+            env->regs[15] = env->elr_el[cur_el] & ~0x3;
+        }
     } else {
-        new_el = extract32(spsr, 2, 2);
-        if (new_el > cur_el
-            || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
-            /* Disallow return to an EL which is unimplemented or higher
-             * than the current one.
-             */
-            goto illegal_return;
-        }
-        if (extract32(spsr, 1, 1)) {
-            /* Return with reserved M[1] bit set */
-            goto illegal_return;
-        }
-        if (new_el == 0 && (spsr & PSTATE_SP)) {
-            /* Return to EL0 with M[0] bit set */
-            goto illegal_return;
-        }
         env->aarch64 = 1;
         pstate_write(env, spsr);
         if (!arm_singlestep_active(env)) {
@@ -522,6 +978,8 @@
         env->pc = env->elr_el[cur_el];
     }
 
+    arm_call_el_change_hook(arm_env_get_cpu(env));
+
     return;
 
 illegal_return:
@@ -575,7 +1033,7 @@
      * short descriptor format (in which case it holds both PROCID and ASID),
      * since we don't implement the optional v7 context ID masking.
      */
-    contextidr = extract64(env->cp15.contextidr_el1, 0, 32);
+    contextidr = extract64(env->cp15.contextidr_el[1], 0, 32);
 
     switch (bt) {
     case 3: /* linked context ID match */
@@ -602,15 +1060,26 @@
     CPUARMState *env = &cpu->env;
     uint64_t cr;
     int pac, hmc, ssc, wt, lbn;
-    /* TODO: check against CPU security state when we implement TrustZone */
-    bool is_secure = false;
+    /* Note that for watchpoints the check is against the CPU security
+     * state, not the S/NS attribute on the offending data access.
+     */
+    bool is_secure = arm_is_secure(env);
+    int access_el = arm_current_el(env);
 
     if (is_wp) {
-        if (!env->cpu_watchpoint[n]
-            || !(env->cpu_watchpoint[n]->flags & BP_WATCHPOINT_HIT)) {
+        CPUWatchpoint *wp = env->cpu_watchpoint[n];
+
+        if (!wp || !(wp->flags & BP_WATCHPOINT_HIT)) {
             return false;
         }
         cr = env->cp15.dbgwcr[n];
+        if (wp->hitattrs.user) {
+            /* The LDRT/STRT/LDT/STT "unprivileged access" instructions should
+             * match watchpoints as if they were accesses done at EL0, even if
+             * the CPU is at EL1 or higher.
+             */
+            access_el = 0;
+        }
     } else {
         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
 
@@ -651,15 +1120,7 @@
         break;
     }
 
-    /* TODO: this is not strictly correct because the LDRT/STRT/LDT/STT
-     * "unprivileged access" instructions should match watchpoints as if
-     * they were accesses done at EL0, even if the CPU is at EL1 or higher.
-     * Implementing this would require reworking the core watchpoint code
-     * to plumb the mmu_idx through to this point. Luckily Linux does not
-     * rely on this behaviour currently.
-     * For breakpoints we do want to use the current CPU state.
-     */
-    switch (arm_current_el(env)) {
+    switch (access_el) {
     case 3:
     case 2:
         if (!hmc) {
@@ -732,6 +1193,25 @@
     return false;
 }
 
+void HELPER(check_breakpoints)(CPUARMState *env)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
+
+    if (check_breakpoints(cpu)) {
+        HELPER(exception_internal(env, EXCP_DEBUG));
+    }
+}
+
+bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
+{
+    /* Called by core code when a CPU watchpoint fires; need to check if this
+     * is also an architectural watchpoint match.
+     */
+    ARMCPU *cpu = ARM_CPU(cs);
+
+    return check_watchpoints(cpu);
+}
+
 void arm_debug_excp_handler(CPUState *cs)
 {
     /* Called by core code when a watchpoint or breakpoint fires;
@@ -743,35 +1223,44 @@
 
     if (wp_hit) {
         if (wp_hit->flags & BP_CPU) {
-            cs->watchpoint_hit = NULL;
-            if (check_watchpoints(cpu)) {
-                bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
-                bool same_el = arm_debug_target_el(env) == arm_current_el(env);
+            bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
+            bool same_el = arm_debug_target_el(env) == arm_current_el(env);
 
-                env->exception.syndrome = syn_watchpoint(same_el, 0, wnr);
-                if (extended_addresses_enabled(env)) {
-                    env->exception.fsr = (1 << 9) | 0x22;
-                } else {
-                    env->exception.fsr = 0x2;
-                }
-                env->exception.vaddress = wp_hit->hitaddr;
-                raise_exception(env, EXCP_DATA_ABORT);
-            } else {
-                cpu_resume_from_signal(cs, NULL);
-            }
-        }
-    } else {
-        if (check_breakpoints(cpu)) {
-            bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
-            env->exception.syndrome = syn_breakpoint(same_el);
+            cs->watchpoint_hit = NULL;
+
             if (extended_addresses_enabled(env)) {
                 env->exception.fsr = (1 << 9) | 0x22;
             } else {
                 env->exception.fsr = 0x2;
             }
-            /* FAR is UNKNOWN, so doesn't need setting */
-            raise_exception(env, EXCP_PREFETCH_ABORT);
+            env->exception.vaddress = wp_hit->hitaddr;
+            raise_exception(env, EXCP_DATA_ABORT,
+                    syn_watchpoint(same_el, 0, wnr),
+                    arm_debug_target_el(env));
         }
+    } else {
+        uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
+        bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
+
+        /* (1) GDB breakpoints should be handled first.
+         * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
+         * since singlestep is also done by generating a debug internal
+         * exception.
+         */
+        if (cpu_breakpoint_test(cs, pc, BP_GDB)
+            || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
+            return;
+        }
+
+        if (extended_addresses_enabled(env)) {
+            env->exception.fsr = (1 << 9) | 0x22;
+        } else {
+            env->exception.fsr = 0x2;
+        }
+        /* FAR is UNKNOWN, so doesn't need setting */
+        raise_exception(env, EXCP_PREFETCH_ABORT,
+                        syn_breakpoint(same_el),
+                        arm_debug_target_el(env));
     }
 }
 
diff --git a/target-arm/psci.c b/target-arm/psci.c
index d8fafab..14316eb 100644
--- a/target-arm/psci.c
+++ b/target-arm/psci.c
@@ -15,12 +15,14 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <cpu.h>
-#include <cpu-qom.h>
-#include <exec/helper-proto.h>
-#include <kvm-consts.h>
-#include <sysemu/sysemu.h>
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+#include "kvm-consts.h"
+#include "sysemu/sysemu.h"
 #include "internals.h"
+#include "arm-powerctl.h"
+#include "exec/exec-all.h"
 
 bool arm_is_psci_call(ARMCPU *cpu, int excp_type)
 {
@@ -82,7 +84,6 @@
      * Additional information about the calling convention used is available in
      * the document 'SMC Calling Convention' (ARM DEN 0028)
      */
-    CPUState *cs = CPU(cpu);
     CPUARMState *env = &cpu->env;
     uint64_t param[4];
     uint64_t context_id, mpidr;
@@ -107,7 +108,6 @@
     switch (param[0]) {
         CPUState *target_cpu_state;
         ARMCPU *target_cpu;
-        CPUClass *target_cpu_class;
 
     case QEMU_PSCI_0_2_FN_PSCI_VERSION:
         ret = QEMU_PSCI_0_2_RET_VERSION_0_2;
@@ -121,7 +121,7 @@
 
         switch (param[2]) {
         case 0:
-            target_cpu_state = qemu_get_cpu(mpidr & 0xff);
+            target_cpu_state = arm_get_cpu_by_id(mpidr);
             if (!target_cpu_state) {
                 ret = QEMU_PSCI_RET_INVALID_PARAMS;
                 break;
@@ -151,52 +151,13 @@
         mpidr = param[1];
         entry = param[2];
         context_id = param[3];
-
-        /* change to the cpu we are powering up */
-        target_cpu_state = qemu_get_cpu(mpidr & 0xff);
-        if (!target_cpu_state) {
-            ret = QEMU_PSCI_RET_INVALID_PARAMS;
-            break;
-        }
-        target_cpu = ARM_CPU(target_cpu_state);
-        if (!target_cpu->powered_off) {
-            ret = QEMU_PSCI_RET_ALREADY_ON;
-            break;
-        }
-        target_cpu_class = CPU_GET_CLASS(target_cpu);
-
-        /* Initialize the cpu we are turning on */
-        cpu_reset(target_cpu_state);
-        target_cpu->powered_off = false;
-        target_cpu_state->halted = 0;
-
         /*
          * The PSCI spec mandates that newly brought up CPUs enter the
          * exception level of the caller in the same execution mode as
          * the caller, with context_id in x0/r0, respectively.
-         *
-         * For now, it is sufficient to assert() that CPUs come out of
-         * reset in the same mode as the calling CPU, since we only
-         * implement EL1, which means that
-         * (a) there is no EL2 for the calling CPU to trap into to change
-         *     its state
-         * (b) the newly brought up CPU enters EL1 immediately after coming
-         *     out of reset in the default state
          */
-        assert(is_a64(env) == is_a64(&target_cpu->env));
-        if (is_a64(env)) {
-            if (entry & 1) {
-                ret = QEMU_PSCI_RET_INVALID_PARAMS;
-                break;
-            }
-            target_cpu->env.xregs[0] = context_id;
-        } else {
-            target_cpu->env.regs[0] = context_id;
-            target_cpu->env.thumb = entry & 1;
-        }
-        target_cpu_class->set_pc(target_cpu_state, entry);
-
-        ret = 0;
+        ret = arm_set_cpu_on(mpidr, entry, context_id, arm_current_el(env),
+                             is_a64(env));
         break;
     case QEMU_PSCI_0_1_FN_CPU_OFF:
     case QEMU_PSCI_0_2_FN_CPU_OFF:
@@ -234,9 +195,8 @@
     return;
 
 cpu_off:
-    cpu->powered_off = true;
-    cs->halted = 1;
-    cs->exception_index = EXCP_HLT;
-    cpu_loop_exit(cs);
+    ret = arm_set_cpu_off(cpu->mp_affinity);
     /* notreached */
+    /* sanity check in case something failed */
+    assert(ret == QEMU_ARM_POWERCTL_RET_SUCCESS);
 }
diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c
index 1e4e4ef..f5e29d2 100644
--- a/target-arm/translate-a64.c
+++ b/target-arm/translate-a64.c
@@ -16,13 +16,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "arm_ldst.h"
@@ -30,25 +27,20 @@
 #include "internals.h"
 #include "qemu/host-utils.h"
 
+#include "exec/semihost.h"
 #include "exec/gen-icount.h"
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
+#include "exec/log.h"
 
 #include "trace-tcg.h"
 
 static TCGv_i64 cpu_X[32];
 static TCGv_i64 cpu_pc;
-static TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;
 
 /* Load/store exclusive handling */
-static TCGv_i64 cpu_exclusive_addr;
-static TCGv_i64 cpu_exclusive_val;
 static TCGv_i64 cpu_exclusive_high;
-#ifdef CONFIG_USER_ONLY
-static TCGv_i64 cpu_exclusive_test;
-static TCGv_i32 cpu_exclusive_info;
-#endif
 
 static const char *regnames[] = {
     "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
@@ -95,32 +87,34 @@
 {
     int i;
 
-    cpu_pc = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new_i64(cpu_env,
                                     offsetof(CPUARMState, pc),
                                     "pc");
     for (i = 0; i < 32; i++) {
-        cpu_X[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_X[i] = tcg_global_mem_new_i64(cpu_env,
                                           offsetof(CPUARMState, xregs[i]),
                                           regnames[i]);
     }
 
-    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");
-    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");
-    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");
-    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");
-
-    cpu_exclusive_addr = tcg_global_mem_new_i64(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_addr), "exclusive_addr");
-    cpu_exclusive_val = tcg_global_mem_new_i64(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_val), "exclusive_val");
-    cpu_exclusive_high = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_high = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_high), "exclusive_high");
-#ifdef CONFIG_USER_ONLY
-    cpu_exclusive_test = tcg_global_mem_new_i64(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_test), "exclusive_test");
-    cpu_exclusive_info = tcg_global_mem_new_i32(TCG_AREG0,
-        offsetof(CPUARMState, exclusive_info), "exclusive_info");
-#endif
+}
+
+static inline ARMMMUIdx get_a64_user_mem_index(DisasContext *s)
+{
+    /* Return the mmu_idx to use for A64 "unprivileged load/store" insns:
+     *  if EL1, access as if EL0; otherwise access at current EL
+     */
+    switch (s->mmu_idx) {
+    case ARMMMUIdx_S12NSE1:
+        return ARMMMUIdx_S12NSE0;
+    case ARMMMUIdx_S1SE1:
+        return ARMMMUIdx_S1SE0;
+    case ARMMMUIdx_S2NS:
+        g_assert_not_reached();
+    default:
+        return s->mmu_idx;
+    }
 }
 
 void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
@@ -130,6 +124,8 @@
     CPUARMState *env = &cpu->env;
     uint32_t psr = pstate_read(env);
     int i;
+    int el = arm_current_el(env);
+    const char *ns_status;
 
     cpu_fprintf(f, "PC=%016"PRIx64"  SP=%016"PRIx64"\n",
             env->pc, env->xregs[31]);
@@ -141,13 +137,22 @@
             cpu_fprintf(f, " ");
         }
     }
-    cpu_fprintf(f, "PSTATE=%08x (flags %c%c%c%c)\n",
+
+    if (arm_feature(env, ARM_FEATURE_EL3) && el != 3) {
+        ns_status = env->cp15.scr_el3 & SCR_NS ? "NS " : "S ";
+    } else {
+        ns_status = "";
+    }
+
+    cpu_fprintf(f, "\nPSTATE=%08x %c%c%c%c %sEL%d%c\n",
                 psr,
                 psr & PSTATE_N ? 'N' : '-',
                 psr & PSTATE_Z ? 'Z' : '-',
                 psr & PSTATE_C ? 'C' : '-',
-                psr & PSTATE_V ? 'V' : '-');
-    cpu_fprintf(f, "\n");
+                psr & PSTATE_V ? 'V' : '-',
+                ns_status,
+                el,
+                psr & PSTATE_SP ? 'h' : 't');
 
     if (flags & CPU_DUMP_FPU) {
         int numvfpregs = 32;
@@ -171,6 +176,31 @@
     tcg_gen_movi_i64(cpu_pc, val);
 }
 
+typedef struct DisasCompare64 {
+    TCGCond cond;
+    TCGv_i64 value;
+} DisasCompare64;
+
+static void a64_test_cc(DisasCompare64 *c64, int cc)
+{
+    DisasCompare c32;
+
+    arm_test_cc(&c32, cc);
+
+    /* Sign-extend the 32-bit value so that the GE/LT comparisons work
+       * properly.  The NE/EQ comparisons are also fine with this choice.  */
+    c64->cond = c32.cond;
+    c64->value = tcg_temp_new_i64();
+    tcg_gen_ext_i32_i64(c64->value, c32.value);
+
+    arm_free_cc(&c32);
+}
+
+static void a64_free_cc(DisasCompare64 *c64)
+{
+    tcg_temp_free_i64(c64->value);
+}
+
 static void gen_exception_internal(int excp)
 {
     TCGv_i32 tcg_excp = tcg_const_i32(excp);
@@ -180,12 +210,15 @@
     tcg_temp_free_i32(tcg_excp);
 }
 
-static void gen_exception(int excp, uint32_t syndrome)
+static void gen_exception(int excp, uint32_t syndrome, uint32_t target_el)
 {
     TCGv_i32 tcg_excp = tcg_const_i32(excp);
     TCGv_i32 tcg_syn = tcg_const_i32(syndrome);
+    TCGv_i32 tcg_el = tcg_const_i32(target_el);
 
-    gen_helper_exception_with_syndrome(cpu_env, tcg_excp, tcg_syn);
+    gen_helper_exception_with_syndrome(cpu_env, tcg_excp,
+                                       tcg_syn, tcg_el);
+    tcg_temp_free_i32(tcg_el);
     tcg_temp_free_i32(tcg_syn);
     tcg_temp_free_i32(tcg_excp);
 }
@@ -198,10 +231,10 @@
 }
 
 static void gen_exception_insn(DisasContext *s, int offset, int excp,
-                               uint32_t syndrome)
+                               uint32_t syndrome, uint32_t target_el)
 {
     gen_a64_set_pc_im(s->pc - offset);
-    gen_exception(excp, syndrome);
+    gen_exception(excp, syndrome, target_el);
     s->is_jmp = DISAS_EXC;
 }
 
@@ -228,7 +261,8 @@
      * of the exception, and our syndrome information is always correct.
      */
     gen_ss_advance(s);
-    gen_exception(EXCP_UDEF, syn_swstep(s->ss_same_el, 1, s->is_ldex));
+    gen_exception(EXCP_UDEF, syn_swstep(s->ss_same_el, 1, s->is_ldex),
+                  default_exception_el(s));
     s->is_jmp = DISAS_EXC;
 }
 
@@ -241,10 +275,12 @@
         return false;
     }
 
+#ifndef CONFIG_USER_ONLY
     /* Only link tbs from inside the same guest page */
     if ((s->tb->pc & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
         return false;
     }
+#endif
 
     return true;
 }
@@ -272,10 +308,25 @@
     }
 }
 
+static void disas_set_insn_syndrome(DisasContext *s, uint32_t syn)
+{
+    /* We don't need to save all of the syndrome so we mask and shift
+     * out uneeded bits to help the sleb128 encoder do a better job.
+     */
+    syn &= ARM_INSN_START_WORD2_MASK;
+    syn >>= ARM_INSN_START_WORD2_SHIFT;
+
+    /* We check and clear insn_start_idx to catch multiple updates.  */
+    assert(s->insn_start_idx != 0);
+    tcg_set_insn_param(s->insn_start_idx, 2, syn);
+    s->insn_start_idx = 0;
+}
+
 static void unallocated_encoding(DisasContext *s)
 {
     /* Unallocated and reserved encodings are uncategorized */
-    gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized());
+    gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(),
+                       default_exception_el(s));
 }
 
 #define unsupported_encoding(s, insn)                                    \
@@ -390,7 +441,7 @@
 static inline void assert_fp_access_checked(DisasContext *s)
 {
 #ifdef CONFIG_DEBUG_TCG
-    if (unlikely(!s->fp_access_checked || !s->cpacr_fpen)) {
+    if (unlikely(!s->fp_access_checked || s->fp_excp_el)) {
         fprintf(stderr, "target-arm: FP access check missing for "
                 "instruction 0x%08x\n", s->insn);
         abort();
@@ -503,13 +554,8 @@
  */
 static inline void gen_set_NZ64(TCGv_i64 result)
 {
-    TCGv_i64 flag = tcg_temp_new_i64();
-
-    tcg_gen_setcondi_i64(TCG_COND_NE, flag, result, 0);
-    tcg_gen_trunc_i64_i32(cpu_ZF, flag);
-    tcg_gen_shri_i64(flag, result, 32);
-    tcg_gen_trunc_i64_i32(cpu_NF, flag);
-    tcg_temp_free_i64(flag);
+    tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
+    tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
 }
 
 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
@@ -518,8 +564,8 @@
     if (sf) {
         gen_set_NZ64(result);
     } else {
-        tcg_gen_trunc_i64_i32(cpu_ZF, result);
-        tcg_gen_trunc_i64_i32(cpu_NF, result);
+        tcg_gen_extrl_i64_i32(cpu_ZF, result);
+        tcg_gen_mov_i32(cpu_NF, cpu_ZF);
     }
     tcg_gen_movi_i32(cpu_CF, 0);
     tcg_gen_movi_i32(cpu_VF, 0);
@@ -537,7 +583,7 @@
         tcg_gen_movi_i64(tmp, 0);
         tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
 
-        tcg_gen_trunc_i64_i32(cpu_CF, flag);
+        tcg_gen_extrl_i64_i32(cpu_CF, flag);
 
         gen_set_NZ64(result);
 
@@ -545,8 +591,7 @@
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_andc_i64(flag, flag, tmp);
         tcg_temp_free_i64(tmp);
-        tcg_gen_shri_i64(flag, flag, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, flag);
+        tcg_gen_extrh_i64_i32(cpu_VF, flag);
 
         tcg_gen_mov_i64(dest, result);
         tcg_temp_free_i64(result);
@@ -558,8 +603,8 @@
         TCGv_i32 tmp = tcg_temp_new_i32();
 
         tcg_gen_movi_i32(tmp, 0);
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
         tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
@@ -587,15 +632,14 @@
         gen_set_NZ64(result);
 
         tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
-        tcg_gen_trunc_i64_i32(cpu_CF, flag);
+        tcg_gen_extrl_i64_i32(cpu_CF, flag);
 
         tcg_gen_xor_i64(flag, result, t0);
         tmp = tcg_temp_new_i64();
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_and_i64(flag, flag, tmp);
         tcg_temp_free_i64(tmp);
-        tcg_gen_shri_i64(flag, flag, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, flag);
+        tcg_gen_extrh_i64_i32(cpu_VF, flag);
         tcg_gen_mov_i64(dest, result);
         tcg_temp_free_i64(flag);
         tcg_temp_free_i64(result);
@@ -605,8 +649,8 @@
         TCGv_i32 t1_32 = tcg_temp_new_i32();
         TCGv_i32 tmp;
 
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
         tcg_gen_mov_i32(cpu_ZF, cpu_NF);
         tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
@@ -648,14 +692,13 @@
         tcg_gen_extu_i32_i64(cf_64, cpu_CF);
         tcg_gen_add2_i64(result, cf_64, t0, tmp, cf_64, tmp);
         tcg_gen_add2_i64(result, cf_64, result, cf_64, t1, tmp);
-        tcg_gen_trunc_i64_i32(cpu_CF, cf_64);
+        tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
         gen_set_NZ64(result);
 
         tcg_gen_xor_i64(vf_64, result, t0);
         tcg_gen_xor_i64(tmp, t0, t1);
         tcg_gen_andc_i64(vf_64, vf_64, tmp);
-        tcg_gen_shri_i64(vf_64, vf_64, 32);
-        tcg_gen_trunc_i64_i32(cpu_VF, vf_64);
+        tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
 
         tcg_gen_mov_i64(dest, result);
 
@@ -669,8 +712,8 @@
         t1_32 = tcg_temp_new_i32();
         tmp = tcg_const_i32(0);
 
-        tcg_gen_trunc_i64_i32(t0_32, t0);
-        tcg_gen_trunc_i64_i32(t1_32, t1);
+        tcg_gen_extrl_i64_i32(t0_32, t0);
+        tcg_gen_extrl_i64_i32(t1_32, t1);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, cpu_CF, tmp);
         tcg_gen_add2_i32(cpu_NF, cpu_CF, cpu_NF, cpu_CF, t1_32, tmp);
 
@@ -694,25 +737,49 @@
  * Store from GPR register to memory.
  */
 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
-                             TCGv_i64 tcg_addr, int size, int memidx)
+                             TCGv_i64 tcg_addr, int size, int memidx,
+                             bool iss_valid,
+                             unsigned int iss_srt,
+                             bool iss_sf, bool iss_ar)
 {
     g_assert(size <= 3);
-    tcg_gen_qemu_st_i64(source, tcg_addr, memidx, MO_TE + size);
+    tcg_gen_qemu_st_i64(source, tcg_addr, memidx, s->be_data + size);
+
+    if (iss_valid) {
+        uint32_t syn;
+
+        syn = syn_data_abort_with_iss(0,
+                                      size,
+                                      false,
+                                      iss_srt,
+                                      iss_sf,
+                                      iss_ar,
+                                      0, 0, 0, 0, 0, false);
+        disas_set_insn_syndrome(s, syn);
+    }
 }
 
 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
-                      TCGv_i64 tcg_addr, int size)
+                      TCGv_i64 tcg_addr, int size,
+                      bool iss_valid,
+                      unsigned int iss_srt,
+                      bool iss_sf, bool iss_ar)
 {
-    do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s));
+    do_gpr_st_memidx(s, source, tcg_addr, size, get_mem_index(s),
+                     iss_valid, iss_srt, iss_sf, iss_ar);
 }
 
 /*
  * Load from memory to GPR register
  */
-static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
-                             int size, bool is_signed, bool extend, int memidx)
+static void do_gpr_ld_memidx(DisasContext *s,
+                             TCGv_i64 dest, TCGv_i64 tcg_addr,
+                             int size, bool is_signed,
+                             bool extend, int memidx,
+                             bool iss_valid, unsigned int iss_srt,
+                             bool iss_sf, bool iss_ar)
 {
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
 
     g_assert(size <= 3);
 
@@ -726,13 +793,30 @@
         g_assert(size < 3);
         tcg_gen_ext32u_i64(dest, dest);
     }
+
+    if (iss_valid) {
+        uint32_t syn;
+
+        syn = syn_data_abort_with_iss(0,
+                                      size,
+                                      is_signed,
+                                      iss_srt,
+                                      iss_sf,
+                                      iss_ar,
+                                      0, 0, 0, 0, 0, false);
+        disas_set_insn_syndrome(s, syn);
+    }
 }
 
-static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
-                      int size, bool is_signed, bool extend)
+static void do_gpr_ld(DisasContext *s,
+                      TCGv_i64 dest, TCGv_i64 tcg_addr,
+                      int size, bool is_signed, bool extend,
+                      bool iss_valid, unsigned int iss_srt,
+                      bool iss_sf, bool iss_ar)
 {
     do_gpr_ld_memidx(s, dest, tcg_addr, size, is_signed, extend,
-                     get_mem_index(s));
+                     get_mem_index(s),
+                     iss_valid, iss_srt, iss_sf, iss_ar);
 }
 
 /*
@@ -744,13 +828,18 @@
     TCGv_i64 tmp = tcg_temp_new_i64();
     tcg_gen_ld_i64(tmp, cpu_env, fp_reg_offset(s, srcidx, MO_64));
     if (size < 4) {
-        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TE + size);
+        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s),
+                            s->be_data + size);
     } else {
+        bool be = s->be_data == MO_BE;
         TCGv_i64 tcg_hiaddr = tcg_temp_new_i64();
-        tcg_gen_qemu_st_i64(tmp, tcg_addr, get_mem_index(s), MO_TEQ);
-        tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
+
         tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
-        tcg_gen_qemu_st_i64(tmp, tcg_hiaddr, get_mem_index(s), MO_TEQ);
+        tcg_gen_qemu_st_i64(tmp, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
+                            s->be_data | MO_Q);
+        tcg_gen_ld_i64(tmp, cpu_env, fp_reg_hi_offset(s, srcidx));
+        tcg_gen_qemu_st_i64(tmp, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
+                            s->be_data | MO_Q);
         tcg_temp_free_i64(tcg_hiaddr);
     }
 
@@ -767,17 +856,21 @@
     TCGv_i64 tmphi;
 
     if (size < 4) {
-        TCGMemOp memop = MO_TE + size;
+        TCGMemOp memop = s->be_data + size;
         tmphi = tcg_const_i64(0);
         tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), memop);
     } else {
+        bool be = s->be_data == MO_BE;
         TCGv_i64 tcg_hiaddr;
+
         tmphi = tcg_temp_new_i64();
         tcg_hiaddr = tcg_temp_new_i64();
 
-        tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), MO_TEQ);
         tcg_gen_addi_i64(tcg_hiaddr, tcg_addr, 8);
-        tcg_gen_qemu_ld_i64(tmphi, tcg_hiaddr, get_mem_index(s), MO_TEQ);
+        tcg_gen_qemu_ld_i64(tmplo, be ? tcg_hiaddr : tcg_addr, get_mem_index(s),
+                            s->be_data | MO_Q);
+        tcg_gen_qemu_ld_i64(tmphi, be ? tcg_addr : tcg_hiaddr, get_mem_index(s),
+                            s->be_data | MO_Q);
         tcg_temp_free_i64(tcg_hiaddr);
     }
 
@@ -916,7 +1009,7 @@
 static void do_vec_st(DisasContext *s, int srcidx, int element,
                       TCGv_i64 tcg_addr, int size)
 {
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
     TCGv_i64 tcg_tmp = tcg_temp_new_i64();
 
     read_vec_element(s, tcg_tmp, srcidx, element, size);
@@ -929,7 +1022,7 @@
 static void do_vec_ld(DisasContext *s, int destidx, int element,
                       TCGv_i64 tcg_addr, int size)
 {
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
     TCGv_i64 tcg_tmp = tcg_temp_new_i64();
 
     tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), memop);
@@ -950,11 +1043,12 @@
     assert(!s->fp_access_checked);
     s->fp_access_checked = true;
 
-    if (s->cpacr_fpen) {
+    if (!s->fp_excp_el) {
         return true;
     }
 
-    gen_exception_insn(s, 4, EXCP_UDEF, syn_fp_access_trap(1, 0xe, false));
+    gen_exception_insn(s, 4, EXCP_UDEF, syn_fp_access_trap(1, 0xe, false),
+                       s->fp_excp_el);
     return false;
 }
 
@@ -1060,7 +1154,7 @@
 {
     uint64_t addr = s->pc + sextract32(insn, 0, 26) * 4 - 4;
 
-    if (insn & (1 << 31)) {
+    if (insn & (1U << 31)) {
         /* C5.6.26 BL Branch with link */
         tcg_gen_movi_i64(cpu_reg(s, 30), s->pc);
     }
@@ -1079,7 +1173,7 @@
 {
     unsigned int sf, op, rt;
     uint64_t addr;
-    int label_match;
+    TCGLabel *label_match;
     TCGv_i64 tcg_cmp;
 
     sf = extract32(insn, 31, 1);
@@ -1108,7 +1202,7 @@
 {
     unsigned int bit_pos, op, rt;
     uint64_t addr;
-    int label_match;
+    TCGLabel *label_match;
     TCGv_i64 tcg_cmp;
 
     bit_pos = (extract32(insn, 31, 1) << 5) | extract32(insn, 19, 5);
@@ -1147,7 +1241,7 @@
 
     if (cond < 0x0e) {
         /* genuinely conditional branches */
-        int label_match = gen_new_label();
+        TCGLabel *label_match = gen_new_label();
         arm_gen_test_cc(cond, label_match);
         gen_goto_tb(s, 0, s->pc);
         gen_set_label(label_match);
@@ -1176,6 +1270,8 @@
         s->is_jmp = DISAS_WFI;
         return;
     case 1: /* YIELD */
+        s->is_jmp = DISAS_YIELD;
+        return;
     case 2: /* WFE */
         s->is_jmp = DISAS_WFE;
         return;
@@ -1209,9 +1305,15 @@
         return;
     case 4: /* DSB */
     case 5: /* DMB */
-    case 6: /* ISB */
         /* We don't emulate caches so barriers are no-ops */
         return;
+    case 6: /* ISB */
+        /* We need to break the TB after this insn to execute
+         * a self-modified code correctly and also to take
+         * any pending interrupts immediately.
+         */
+        s->is_jmp = DISAS_UPDATE;
+        return;
     default:
         unallocated_encoding(s);
         return;
@@ -1254,7 +1356,7 @@
     TCGv_i32 nzcv = tcg_temp_new_i32();
 
     /* build bit 31, N */
-    tcg_gen_andi_i32(nzcv, cpu_NF, (1 << 31));
+    tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
     /* build bit 30, Z */
     tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
     tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
@@ -1276,10 +1378,10 @@
     TCGv_i32 nzcv = tcg_temp_new_i32();
 
     /* take NZCV from R[t] */
-    tcg_gen_trunc_i64_i32(nzcv, tcg_rt);
+    tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
 
     /* bit 31, N */
-    tcg_gen_andi_i32(cpu_NF, nzcv, (1 << 31));
+    tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
     /* bit 30, Z */
     tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
     tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
@@ -1332,16 +1434,18 @@
          * runtime; this may result in an exception.
          */
         TCGv_ptr tmpptr;
-        TCGv_i32 tcg_syn;
+        TCGv_i32 tcg_syn, tcg_isread;
         uint32_t syndrome;
 
         gen_a64_set_pc_im(s->pc - 4);
         tmpptr = tcg_const_ptr(ri);
         syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
         tcg_syn = tcg_const_i32(syndrome);
-        gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn);
+        tcg_isread = tcg_const_i32(isread);
+        gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn, tcg_isread);
         tcg_temp_free_ptr(tmpptr);
         tcg_temp_free_i32(tcg_syn);
+        tcg_temp_free_i32(tcg_isread);
     }
 
     /* Handle special cases first */
@@ -1372,7 +1476,7 @@
         break;
     }
 
-    if (use_icount && (ri->type & ARM_CP_IO)) {
+    if ((s->tb->cflags & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
         gen_io_start();
     }
 
@@ -1403,7 +1507,7 @@
         }
     }
 
-    if (use_icount && (ri->type & ARM_CP_IO)) {
+    if ((s->tb->cflags & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
         /* I/O operations must end the TB here (whether read or write) */
         gen_io_end();
         s->is_jmp = DISAS_UPDATE;
@@ -1481,7 +1585,8 @@
         switch (op2_ll) {
         case 1:
             gen_ss_advance(s);
-            gen_exception_insn(s, 0, EXCP_SWI, syn_aa64_svc(imm16));
+            gen_exception_insn(s, 0, EXCP_SWI, syn_aa64_svc(imm16),
+                               default_exception_el(s));
             break;
         case 2:
             if (s->current_el == 0) {
@@ -1494,7 +1599,7 @@
             gen_a64_set_pc_im(s->pc - 4);
             gen_helper_pre_hvc(cpu_env);
             gen_ss_advance(s);
-            gen_exception_insn(s, 0, EXCP_HVC, syn_aa64_hvc(imm16));
+            gen_exception_insn(s, 0, EXCP_HVC, syn_aa64_hvc(imm16), 2);
             break;
         case 3:
             if (s->current_el == 0) {
@@ -1506,7 +1611,7 @@
             gen_helper_pre_smc(cpu_env, tmp);
             tcg_temp_free_i32(tmp);
             gen_ss_advance(s);
-            gen_exception_insn(s, 0, EXCP_SMC, syn_aa64_smc(imm16));
+            gen_exception_insn(s, 0, EXCP_SMC, syn_aa64_smc(imm16), 3);
             break;
         default:
             unallocated_encoding(s);
@@ -1519,15 +1624,35 @@
             break;
         }
         /* BRK */
-        gen_exception_insn(s, 4, EXCP_BKPT, syn_aa64_bkpt(imm16));
+        gen_exception_insn(s, 4, EXCP_BKPT, syn_aa64_bkpt(imm16),
+                           default_exception_el(s));
         break;
     case 2:
         if (op2_ll != 0) {
             unallocated_encoding(s);
             break;
         }
-        /* HLT */
-        unsupported_encoding(s, insn);
+        /* HLT. This has two purposes.
+         * Architecturally, it is an external halting debug instruction.
+         * Since QEMU doesn't implement external debug, we treat this as
+         * it is required for halting debug disabled: it will UNDEF.
+         * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
+         */
+        if (semihosting_enabled() && imm16 == 0xf000) {
+#ifndef CONFIG_USER_ONLY
+            /* In system mode, don't allow userspace access to semihosting,
+             * to provide some semblance of security (and for consistency
+             * with our 32-bit semihosting).
+             */
+            if (s->current_el == 0) {
+                unsupported_encoding(s, insn);
+                break;
+            }
+#endif
+            gen_exception_internal_insn(s, 0, EXCP_SEMIHOST);
+        } else {
+            unsupported_encoding(s, insn);
+        }
         break;
     case 5:
         if (op2_ll < 1 || op2_ll > 3) {
@@ -1644,7 +1769,7 @@
                                TCGv_i64 addr, int size, bool is_pair)
 {
     TCGv_i64 tmp = tcg_temp_new_i64();
-    TCGMemOp memop = MO_TE + size;
+    TCGMemOp memop = s->be_data + size;
 
     g_assert(size <= 3);
     tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), memop);
@@ -1694,8 +1819,8 @@
      * }
      * env->exclusive_addr = -1;
      */
-    int fail_label = gen_new_label();
-    int done_label = gen_new_label();
+    TCGLabel *fail_label = gen_new_label();
+    TCGLabel *done_label = gen_new_label();
     TCGv_i64 addr = tcg_temp_local_new_i64();
     TCGv_i64 tmp;
 
@@ -1706,7 +1831,7 @@
     tcg_gen_brcond_i64(TCG_COND_NE, addr, cpu_exclusive_addr, fail_label);
 
     tmp = tcg_temp_new_i64();
-    tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), MO_TE + size);
+    tcg_gen_qemu_ld_i64(tmp, addr, get_mem_index(s), s->be_data + size);
     tcg_gen_brcond_i64(TCG_COND_NE, tmp, cpu_exclusive_val, fail_label);
     tcg_temp_free_i64(tmp);
 
@@ -1715,7 +1840,8 @@
         TCGv_i64 tmphi = tcg_temp_new_i64();
 
         tcg_gen_addi_i64(addrhi, addr, 1 << size);
-        tcg_gen_qemu_ld_i64(tmphi, addrhi, get_mem_index(s), MO_TE + size);
+        tcg_gen_qemu_ld_i64(tmphi, addrhi, get_mem_index(s),
+                            s->be_data + size);
         tcg_gen_brcond_i64(TCG_COND_NE, tmphi, cpu_exclusive_high, fail_label);
 
         tcg_temp_free_i64(tmphi);
@@ -1723,13 +1849,14 @@
     }
 
     /* We seem to still have the exclusive monitor, so do the store */
-    tcg_gen_qemu_st_i64(cpu_reg(s, rt), addr, get_mem_index(s), MO_TE + size);
+    tcg_gen_qemu_st_i64(cpu_reg(s, rt), addr, get_mem_index(s),
+                        s->be_data + size);
     if (is_pair) {
         TCGv_i64 addrhi = tcg_temp_new_i64();
 
         tcg_gen_addi_i64(addrhi, addr, 1 << size);
         tcg_gen_qemu_st_i64(cpu_reg(s, rt2), addrhi,
-                            get_mem_index(s), MO_TE + size);
+                            get_mem_index(s), s->be_data + size);
         tcg_temp_free_i64(addrhi);
     }
 
@@ -1745,6 +1872,22 @@
 }
 #endif
 
+/* Update the Sixty-Four bit (SF) registersize. This logic is derived
+ * from the ARMv8 specs for LDR (Shared decode for all encodings).
+ */
+static bool disas_ldst_compute_iss_sf(int size, bool is_signed, int opc)
+{
+    int opc0 = extract32(opc, 0, 1);
+    int regsize;
+
+    if (is_signed) {
+        regsize = opc0 ? 32 : 64;
+    } else {
+        regsize = size == 3 ? 64 : 32;
+    }
+    return regsize == 64;
+}
+
 /* C3.3.6 Load/store exclusive
  *
  *  31 30 29         24  23  22   21  20  16  15  14   10 9    5 4    0
@@ -1757,9 +1900,6 @@
  *  o2: 0 -> exclusive, 1 -> not
  *  o1: 0 -> single register, 1 -> register pair
  *  o0: 1 -> load-acquire/store-release, 0 -> not
- *
- *  o0 == 0 AND o2 == 1 is un-allocated
- *  o1 == 1 is un-allocated except for 32 and 64 bit sizes
  */
 static void disas_ldst_excl(DisasContext *s, uint32_t insn)
 {
@@ -1774,7 +1914,8 @@
     int size = extract32(insn, 30, 2);
     TCGv_i64 tcg_addr;
 
-    if ((!is_excl && !is_lasr) ||
+    if ((!is_excl && !is_pair && !is_lasr) ||
+        (!is_excl && is_pair) ||
         (is_pair && size < 2)) {
         unallocated_encoding(s);
         return;
@@ -1798,19 +1939,15 @@
         }
     } else {
         TCGv_i64 tcg_rt = cpu_reg(s, rt);
+        bool iss_sf = disas_ldst_compute_iss_sf(size, false, 0);
+
+        /* Generate ISS for non-exclusive accesses including LASR.  */
         if (is_store) {
-            do_gpr_st(s, tcg_rt, tcg_addr, size);
+            do_gpr_st(s, tcg_rt, tcg_addr, size,
+                      true, rt, iss_sf, is_lasr);
         } else {
-            do_gpr_ld(s, tcg_rt, tcg_addr, size, false, false);
-        }
-        if (is_pair) {
-            TCGv_i64 tcg_rt2 = cpu_reg(s, rt);
-            tcg_gen_addi_i64(tcg_addr, tcg_addr, 1 << size);
-            if (is_store) {
-                do_gpr_st(s, tcg_rt2, tcg_addr, size);
-            } else {
-                do_gpr_ld(s, tcg_rt2, tcg_addr, size, false, false);
-            }
+            do_gpr_ld(s, tcg_rt, tcg_addr, size, false, false,
+                      true, rt, iss_sf, is_lasr);
         }
     }
 }
@@ -1862,7 +1999,11 @@
     if (is_vector) {
         do_fp_ld(s, rt, tcg_addr, size);
     } else {
-        do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, false);
+        /* Only unsigned 32bit loads target 32bit registers.  */
+        bool iss_sf = opc == 0 ? 32 : 64;
+
+        do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, false,
+                  true, rt, iss_sf, false);
     }
     tcg_temp_free_i64(tcg_addr);
 }
@@ -1900,7 +2041,7 @@
     int rt = extract32(insn, 0, 5);
     int rn = extract32(insn, 5, 5);
     int rt2 = extract32(insn, 10, 5);
-    int64_t offset = sextract32(insn, 15, 7);
+    uint64_t offset = sextract64(insn, 15, 7);
     int index = extract32(insn, 23, 2);
     bool is_vector = extract32(insn, 26, 1);
     bool is_load = extract32(insn, 22, 1);
@@ -1981,9 +2122,11 @@
     } else {
         TCGv_i64 tcg_rt = cpu_reg(s, rt);
         if (is_load) {
-            do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, false);
+            do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, false,
+                      false, 0, false, false);
         } else {
-            do_gpr_st(s, tcg_rt, tcg_addr, size);
+            do_gpr_st(s, tcg_rt, tcg_addr, size,
+                      false, 0, false, false);
         }
     }
     tcg_gen_addi_i64(tcg_addr, tcg_addr, 1 << size);
@@ -1996,9 +2139,11 @@
     } else {
         TCGv_i64 tcg_rt2 = cpu_reg(s, rt2);
         if (is_load) {
-            do_gpr_ld(s, tcg_rt2, tcg_addr, size, is_signed, false);
+            do_gpr_ld(s, tcg_rt2, tcg_addr, size, is_signed, false,
+                      false, 0, false, false);
         } else {
-            do_gpr_st(s, tcg_rt2, tcg_addr, size);
+            do_gpr_st(s, tcg_rt2, tcg_addr, size,
+                      false, 0, false, false);
         }
     }
 
@@ -2028,19 +2173,20 @@
  * size: 00 -> 8 bit, 01 -> 16 bit, 10 -> 32 bit, 11 -> 64bit
  * opc: 00 -> store, 01 -> loadu, 10 -> loads 64, 11 -> loads 32
  */
-static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn)
+static void disas_ldst_reg_imm9(DisasContext *s, uint32_t insn,
+                                int opc,
+                                int size,
+                                int rt,
+                                bool is_vector)
 {
-    int rt = extract32(insn, 0, 5);
     int rn = extract32(insn, 5, 5);
     int imm9 = sextract32(insn, 12, 9);
-    int opc = extract32(insn, 22, 2);
-    int size = extract32(insn, 30, 2);
     int idx = extract32(insn, 10, 2);
     bool is_signed = false;
     bool is_store = false;
     bool is_extended = false;
     bool is_unpriv = (idx == 2);
-    bool is_vector = extract32(insn, 26, 1);
+    bool iss_valid = !is_vector;
     bool post_index;
     bool writeback;
 
@@ -2070,8 +2216,8 @@
             return;
         }
         is_store = (opc == 0);
-        is_signed = opc & (1<<1);
-        is_extended = (size < 3) && (opc & 1);
+        is_signed = extract32(opc, 1, 1);
+        is_extended = (size < 3) && extract32(opc, 0, 1);
     }
 
     switch (idx) {
@@ -2107,13 +2253,16 @@
         }
     } else {
         TCGv_i64 tcg_rt = cpu_reg(s, rt);
-        int memidx = is_unpriv ? 1 : get_mem_index(s);
+        int memidx = is_unpriv ? get_a64_user_mem_index(s) : get_mem_index(s);
+        bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
 
         if (is_store) {
-            do_gpr_st_memidx(s, tcg_rt, tcg_addr, size, memidx);
+            do_gpr_st_memidx(s, tcg_rt, tcg_addr, size, memidx,
+                             iss_valid, rt, iss_sf, false);
         } else {
             do_gpr_ld_memidx(s, tcg_rt, tcg_addr, size,
-                             is_signed, is_extended, memidx);
+                             is_signed, is_extended, memidx,
+                             iss_valid, rt, iss_sf, false);
         }
     }
 
@@ -2147,19 +2296,19 @@
  * Rn: address register or SP for base
  * Rm: offset register or ZR for offset
  */
-static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn)
+static void disas_ldst_reg_roffset(DisasContext *s, uint32_t insn,
+                                   int opc,
+                                   int size,
+                                   int rt,
+                                   bool is_vector)
 {
-    int rt = extract32(insn, 0, 5);
     int rn = extract32(insn, 5, 5);
     int shift = extract32(insn, 12, 1);
     int rm = extract32(insn, 16, 5);
-    int opc = extract32(insn, 22, 2);
     int opt = extract32(insn, 13, 3);
-    int size = extract32(insn, 30, 2);
     bool is_signed = false;
     bool is_store = false;
     bool is_extended = false;
-    bool is_vector = extract32(insn, 26, 1);
 
     TCGv_i64 tcg_rm;
     TCGv_i64 tcg_addr;
@@ -2211,10 +2360,14 @@
         }
     } else {
         TCGv_i64 tcg_rt = cpu_reg(s, rt);
+        bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
         if (is_store) {
-            do_gpr_st(s, tcg_rt, tcg_addr, size);
+            do_gpr_st(s, tcg_rt, tcg_addr, size,
+                      true, rt, iss_sf, false);
         } else {
-            do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, is_extended);
+            do_gpr_ld(s, tcg_rt, tcg_addr, size,
+                      is_signed, is_extended,
+                      true, rt, iss_sf, false);
         }
     }
 }
@@ -2236,14 +2389,14 @@
  * Rn: base address register (inc SP)
  * Rt: target register
  */
-static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn)
+static void disas_ldst_reg_unsigned_imm(DisasContext *s, uint32_t insn,
+                                        int opc,
+                                        int size,
+                                        int rt,
+                                        bool is_vector)
 {
-    int rt = extract32(insn, 0, 5);
     int rn = extract32(insn, 5, 5);
     unsigned int imm12 = extract32(insn, 10, 12);
-    bool is_vector = extract32(insn, 26, 1);
-    int size = extract32(insn, 30, 2);
-    int opc = extract32(insn, 22, 2);
     unsigned int offset;
 
     TCGv_i64 tcg_addr;
@@ -2291,10 +2444,13 @@
         }
     } else {
         TCGv_i64 tcg_rt = cpu_reg(s, rt);
+        bool iss_sf = disas_ldst_compute_iss_sf(size, is_signed, opc);
         if (is_store) {
-            do_gpr_st(s, tcg_rt, tcg_addr, size);
+            do_gpr_st(s, tcg_rt, tcg_addr, size,
+                      true, rt, iss_sf, false);
         } else {
-            do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, is_extended);
+            do_gpr_ld(s, tcg_rt, tcg_addr, size, is_signed, is_extended,
+                      true, rt, iss_sf, false);
         }
     }
 }
@@ -2302,20 +2458,25 @@
 /* Load/store register (all forms) */
 static void disas_ldst_reg(DisasContext *s, uint32_t insn)
 {
+    int rt = extract32(insn, 0, 5);
+    int opc = extract32(insn, 22, 2);
+    bool is_vector = extract32(insn, 26, 1);
+    int size = extract32(insn, 30, 2);
+
     switch (extract32(insn, 24, 2)) {
     case 0:
         if (extract32(insn, 21, 1) == 1 && extract32(insn, 10, 2) == 2) {
-            disas_ldst_reg_roffset(s, insn);
+            disas_ldst_reg_roffset(s, insn, opc, size, rt, is_vector);
         } else {
             /* Load/store register (unscaled immediate)
              * Load/store immediate pre/post-indexed
              * Load/store register unprivileged
              */
-            disas_ldst_reg_imm9(s, insn);
+            disas_ldst_reg_imm9(s, insn, opc, size, rt, is_vector);
         }
         break;
     case 1:
-        disas_ldst_reg_unsigned_imm(s, insn);
+        disas_ldst_reg_unsigned_imm(s, insn, opc, size, rt, is_vector);
         break;
     default:
         unallocated_encoding(s);
@@ -2555,7 +2716,7 @@
             TCGv_i64 tcg_tmp = tcg_temp_new_i64();
 
             tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr,
-                                get_mem_index(s), MO_TE + scale);
+                                get_mem_index(s), s->be_data + scale);
             switch (scale) {
             case 0:
                 mulconst = 0x0101010101010101ULL;
@@ -2585,9 +2746,9 @@
         } else {
             /* Load/store one element per register */
             if (is_load) {
-                do_vec_ld(s, rt, index, tcg_addr, MO_TE + scale);
+                do_vec_ld(s, rt, index, tcg_addr, s->be_data + scale);
             } else {
-                do_vec_st(s, rt, index, tcg_addr, MO_TE + scale);
+                do_vec_st(s, rt, index, tcg_addr, s->be_data + scale);
             }
         }
         tcg_gen_addi_i64(tcg_addr, tcg_addr, ebytes);
@@ -2645,11 +2806,12 @@
 {
     unsigned int page, rd;
     uint64_t base;
-    int64_t offset;
+    uint64_t offset;
 
     page = extract32(insn, 31, 1);
     /* SignExtend(immhi:immlo) -> offset */
-    offset = ((int64_t)sextract32(insn, 5, 19) << 2) | extract32(insn, 29, 2);
+    offset = sextract64(insn, 5, 19);
+    offset = offset << 2 | extract32(insn, 29, 2);
     rd = extract32(insn, 0, 5);
     base = s->pc - 4;
 
@@ -2803,7 +2965,10 @@
      * by r within the element (which is e bits wide)...
      */
     mask = bitmask64(s + 1);
-    mask = (mask >> r) | (mask << (e - r));
+    if (r) {
+        mask = (mask >> r) | (mask << (e - r));
+        mask &= bitmask64(e);
+    }
     /* ...then replicate the element over the whole 64 bit value */
     mask = bitfield_replicate(mask, e);
     *result = mask;
@@ -2961,9 +3126,51 @@
     }
 
     tcg_rd = cpu_reg(s, rd);
-    tcg_tmp = read_cpu_reg(s, rn, sf);
 
-    /* OPTME: probably worth recognizing common cases of ext{8,16,32}{u,s} */
+    /* Suppress the zero-extend for !sf.  Since RI and SI are constrained
+       to be smaller than bitsize, we'll never reference data outside the
+       low 32-bits anyway.  */
+    tcg_tmp = read_cpu_reg(s, rn, 1);
+
+    /* Recognize the common aliases.  */
+    if (opc == 0) { /* SBFM */
+        if (ri == 0) {
+            if (si == 7) { /* SXTB */
+                tcg_gen_ext8s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            } else if (si == 15) { /* SXTH */
+                tcg_gen_ext16s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            } else if (si == 31) { /* SXTW */
+                tcg_gen_ext32s_i64(tcg_rd, tcg_tmp);
+                goto done;
+            }
+        }
+        if (si == 63 || (si == 31 && ri <= si)) { /* ASR */
+            if (si == 31) {
+                tcg_gen_ext32s_i64(tcg_tmp, tcg_tmp);
+            }
+            tcg_gen_sari_i64(tcg_rd, tcg_tmp, ri);
+            goto done;
+        }
+    } else if (opc == 2) { /* UBFM */
+        if (ri == 0) { /* UXTB, UXTH, plus non-canonical AND */
+            tcg_gen_andi_i64(tcg_rd, tcg_tmp, bitmask64(si + 1));
+            return;
+        }
+        if (si == 63 || (si == 31 && ri <= si)) { /* LSR */
+            if (si == 31) {
+                tcg_gen_ext32u_i64(tcg_tmp, tcg_tmp);
+            }
+            tcg_gen_shri_i64(tcg_rd, tcg_tmp, ri);
+            return;
+        }
+        if (si + 1 == ri && si != bitsize - 1) { /* LSL */
+            int shift = bitsize - 1 - si;
+            tcg_gen_shli_i64(tcg_rd, tcg_tmp, shift);
+            goto done;
+        }
+    }
 
     if (opc != 1) { /* SBFM or UBFM */
         tcg_gen_movi_i64(tcg_rd, 0);
@@ -2988,6 +3195,7 @@
         tcg_gen_sari_i64(tcg_rd, tcg_rd, 64 - (pos + len));
     }
 
+ done:
     if (!sf) { /* zero extend final result */
         tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
     }
@@ -3020,17 +3228,7 @@
 
         tcg_rd = cpu_reg(s, rd);
 
-        if (imm) {
-            /* OPTME: we can special case rm==rn as a rotate */
-            tcg_rm = read_cpu_reg(s, rm, sf);
-            tcg_rn = read_cpu_reg(s, rn, sf);
-            tcg_gen_shri_i64(tcg_rm, tcg_rm, imm);
-            tcg_gen_shli_i64(tcg_rn, tcg_rn, bitsize - imm);
-            tcg_gen_or_i64(tcg_rd, tcg_rm, tcg_rn);
-            if (!sf) {
-                tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
-            }
-        } else {
+        if (unlikely(imm == 0)) {
             /* tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
              * so an extract from bit 0 is a special case.
              */
@@ -3039,8 +3237,27 @@
             } else {
                 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, rm));
             }
+        } else if (rm == rn) { /* ROR */
+            tcg_rm = cpu_reg(s, rm);
+            if (sf) {
+                tcg_gen_rotri_i64(tcg_rd, tcg_rm, imm);
+            } else {
+                TCGv_i32 tmp = tcg_temp_new_i32();
+                tcg_gen_extrl_i64_i32(tmp, tcg_rm);
+                tcg_gen_rotri_i32(tmp, tmp, imm);
+                tcg_gen_extu_i32_i64(tcg_rd, tmp);
+                tcg_temp_free_i32(tmp);
+            }
+        } else {
+            tcg_rm = read_cpu_reg(s, rm, sf);
+            tcg_rn = read_cpu_reg(s, rn, sf);
+            tcg_gen_shri_i64(tcg_rm, tcg_rm, imm);
+            tcg_gen_shli_i64(tcg_rn, tcg_rn, bitsize - imm);
+            tcg_gen_or_i64(tcg_rd, tcg_rm, tcg_rn);
+            if (!sf) {
+                tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
+            }
         }
-
     }
 }
 
@@ -3100,8 +3317,8 @@
             TCGv_i32 t0, t1;
             t0 = tcg_temp_new_i32();
             t1 = tcg_temp_new_i32();
-            tcg_gen_trunc_i64_i32(t0, src);
-            tcg_gen_trunc_i64_i32(t1, shift_amount);
+            tcg_gen_extrl_i64_i32(t0, src);
+            tcg_gen_extrl_i64_i32(t1, shift_amount);
             tcg_gen_rotr_i32(t0, t0, t1);
             tcg_gen_extu_i32_i64(dst, t0);
             tcg_temp_free_i32(t0);
@@ -3516,8 +3733,9 @@
 static void disas_cc(DisasContext *s, uint32_t insn)
 {
     unsigned int sf, op, y, cond, rn, nzcv, is_imm;
-    int label_continue = -1;
+    TCGv_i32 tcg_t0, tcg_t1, tcg_t2;
     TCGv_i64 tcg_tmp, tcg_y, tcg_rn;
+    DisasCompare c;
 
     if (!extract32(insn, 29, 1)) {
         unallocated_encoding(s);
@@ -3535,19 +3753,13 @@
     rn = extract32(insn, 5, 5);
     nzcv = extract32(insn, 0, 4);
 
-    if (cond < 0x0e) { /* not always */
-        int label_match = gen_new_label();
-        label_continue = gen_new_label();
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        tcg_tmp = tcg_temp_new_i64();
-        tcg_gen_movi_i64(tcg_tmp, nzcv << 28);
-        gen_set_nzcv(tcg_tmp);
-        tcg_temp_free_i64(tcg_tmp);
-        tcg_gen_br(label_continue);
-        gen_set_label(label_match);
-    }
-    /* match, or condition is always */
+    /* Set T0 = !COND.  */
+    tcg_t0 = tcg_temp_new_i32();
+    arm_test_cc(&c, cond);
+    tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
+    arm_free_cc(&c);
+
+    /* Load the arguments for the new comparison.  */
     if (is_imm) {
         tcg_y = new_tmp_a64(s);
         tcg_gen_movi_i64(tcg_y, y);
@@ -3556,6 +3768,7 @@
     }
     tcg_rn = cpu_reg(s, rn);
 
+    /* Set the flags for the new comparison.  */
     tcg_tmp = tcg_temp_new_i64();
     if (op) {
         gen_sub_CC(sf, tcg_tmp, tcg_rn, tcg_y);
@@ -3564,9 +3777,55 @@
     }
     tcg_temp_free_i64(tcg_tmp);
 
-    if (cond < 0x0e) { /* continue */
-        gen_set_label(label_continue);
+    /* If COND was false, force the flags to #nzcv.  Compute two masks
+     * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
+     * For tcg hosts that support ANDC, we can make do with just T1.
+     * In either case, allow the tcg optimizer to delete any unused mask.
+     */
+    tcg_t1 = tcg_temp_new_i32();
+    tcg_t2 = tcg_temp_new_i32();
+    tcg_gen_neg_i32(tcg_t1, tcg_t0);
+    tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
+
+    if (nzcv & 8) { /* N */
+        tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
+        }
     }
+    if (nzcv & 4) { /* Z */
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
+        }
+    } else {
+        tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
+    }
+    if (nzcv & 2) { /* C */
+        tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
+        }
+    }
+    if (nzcv & 1) { /* V */
+        tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
+    } else {
+        if (TCG_TARGET_HAS_andc_i32) {
+            tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
+        } else {
+            tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
+        }
+    }
+    tcg_temp_free_i32(tcg_t0);
+    tcg_temp_free_i32(tcg_t1);
+    tcg_temp_free_i32(tcg_t2);
 }
 
 /* C3.5.6 Conditional select
@@ -3578,7 +3837,8 @@
 static void disas_cond_select(DisasContext *s, uint32_t insn)
 {
     unsigned int sf, else_inv, rm, cond, else_inc, rn, rd;
-    TCGv_i64 tcg_rd, tcg_src;
+    TCGv_i64 tcg_rd, zero;
+    DisasCompare64 c;
 
     if (extract32(insn, 29, 1) || extract32(insn, 11, 1)) {
         /* S == 1 or op2<1> == 1 */
@@ -3593,48 +3853,35 @@
     rn = extract32(insn, 5, 5);
     rd = extract32(insn, 0, 5);
 
-    if (rd == 31) {
-        /* silly no-op write; until we use movcond we must special-case
-         * this to avoid a dead temporary across basic blocks.
-         */
-        return;
-    }
-
     tcg_rd = cpu_reg(s, rd);
 
-    if (cond >= 0x0e) { /* condition "always" */
-        tcg_src = read_cpu_reg(s, rn, sf);
-        tcg_gen_mov_i64(tcg_rd, tcg_src);
+    a64_test_cc(&c, cond);
+    zero = tcg_const_i64(0);
+
+    if (rn == 31 && rm == 31 && (else_inc ^ else_inv)) {
+        /* CSET & CSETM.  */
+        tcg_gen_setcond_i64(tcg_invert_cond(c.cond), tcg_rd, c.value, zero);
+        if (else_inv) {
+            tcg_gen_neg_i64(tcg_rd, tcg_rd);
+        }
     } else {
-        /* OPTME: we could use movcond here, at the cost of duplicating
-         * a lot of the arm_gen_test_cc() logic.
-         */
-        int label_match = gen_new_label();
-        int label_continue = gen_new_label();
-
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        tcg_src = cpu_reg(s, rm);
-
+        TCGv_i64 t_true = cpu_reg(s, rn);
+        TCGv_i64 t_false = read_cpu_reg(s, rm, 1);
         if (else_inv && else_inc) {
-            tcg_gen_neg_i64(tcg_rd, tcg_src);
+            tcg_gen_neg_i64(t_false, t_false);
         } else if (else_inv) {
-            tcg_gen_not_i64(tcg_rd, tcg_src);
+            tcg_gen_not_i64(t_false, t_false);
         } else if (else_inc) {
-            tcg_gen_addi_i64(tcg_rd, tcg_src, 1);
-        } else {
-            tcg_gen_mov_i64(tcg_rd, tcg_src);
+            tcg_gen_addi_i64(t_false, t_false, 1);
         }
-        if (!sf) {
-            tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
-        }
-        tcg_gen_br(label_continue);
-        /* match: */
-        gen_set_label(label_match);
-        tcg_src = read_cpu_reg(s, rn, sf);
-        tcg_gen_mov_i64(tcg_rd, tcg_src);
-        /* continue: */
-        gen_set_label(label_continue);
+        tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
+    }
+
+    tcg_temp_free_i64(zero);
+    a64_free_cc(&c);
+
+    if (!sf) {
+        tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
     }
 }
 
@@ -3649,7 +3896,7 @@
         gen_helper_clz64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_clz(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -3667,7 +3914,7 @@
         gen_helper_cls64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_cls32(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -3685,7 +3932,7 @@
         gen_helper_rbit64(tcg_rd, tcg_rn);
     } else {
         TCGv_i32 tcg_tmp32 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(tcg_tmp32, tcg_rn);
+        tcg_gen_extrl_i64_i32(tcg_tmp32, tcg_rn);
         gen_helper_rbit(tcg_tmp32, tcg_tmp32);
         tcg_gen_extu_i32_i64(tcg_rd, tcg_tmp32);
         tcg_temp_free_i32(tcg_tmp32);
@@ -4083,7 +4330,7 @@
 {
     unsigned int mos, type, rm, cond, rn, op, nzcv;
     TCGv_i64 tcg_flags;
-    int label_continue = -1;
+    TCGLabel *label_continue = NULL;
 
     mos = extract32(insn, 29, 3);
     type = extract32(insn, 22, 2); /* 0 = single, 1 = double */
@@ -4103,7 +4350,7 @@
     }
 
     if (cond < 0x0e) { /* not always */
-        int label_match = gen_new_label();
+        TCGLabel *label_match = gen_new_label();
         label_continue = gen_new_label();
         arm_gen_test_cc(cond, label_match);
         /* nomatch: */
@@ -4121,20 +4368,6 @@
     }
 }
 
-/* copy src FP register to dst FP register; type specifies single or double */
-static void gen_mov_fp2fp(DisasContext *s, int type, int dst, int src)
-{
-    if (type) {
-        TCGv_i64 v = read_fp_dreg(s, src);
-        write_fp_dreg(s, dst, v);
-        tcg_temp_free_i64(v);
-    } else {
-        TCGv_i32 v = read_fp_sreg(s, src);
-        write_fp_sreg(s, dst, v);
-        tcg_temp_free_i32(v);
-    }
-}
-
 /* C3.6.24 Floating point conditional select
  *   31  30  29 28       24 23  22  21 20  16 15  12 11 10 9    5 4    0
  * +---+---+---+-----------+------+---+------+------+-----+------+------+
@@ -4144,7 +4377,8 @@
 static void disas_fp_csel(DisasContext *s, uint32_t insn)
 {
     unsigned int mos, type, rm, cond, rn, rd;
-    int label_continue = -1;
+    TCGv_i64 t_true, t_false, t_zero;
+    DisasCompare64 c;
 
     mos = extract32(insn, 29, 3);
     type = extract32(insn, 22, 2); /* 0 = single, 1 = double */
@@ -4162,21 +4396,23 @@
         return;
     }
 
-    if (cond < 0x0e) { /* not always */
-        int label_match = gen_new_label();
-        label_continue = gen_new_label();
-        arm_gen_test_cc(cond, label_match);
-        /* nomatch: */
-        gen_mov_fp2fp(s, type, rd, rm);
-        tcg_gen_br(label_continue);
-        gen_set_label(label_match);
-    }
+    /* Zero extend sreg inputs to 64 bits now.  */
+    t_true = tcg_temp_new_i64();
+    t_false = tcg_temp_new_i64();
+    read_vec_element(s, t_true, rn, 0, type ? MO_64 : MO_32);
+    read_vec_element(s, t_false, rm, 0, type ? MO_64 : MO_32);
 
-    gen_mov_fp2fp(s, type, rd, rn);
+    a64_test_cc(&c, cond);
+    t_zero = tcg_const_i64(0);
+    tcg_gen_movcond_i64(c.cond, t_true, c.value, t_zero, t_true, t_false);
+    tcg_temp_free_i64(t_zero);
+    tcg_temp_free_i64(t_false);
+    a64_free_cc(&c);
 
-    if (cond < 0x0e) { /* continue */
-        gen_set_label(label_continue);
-    }
+    /* Note that sregs write back zeros to the high bits,
+       and we've already done the zero-extension.  */
+    write_fp_dreg(s, rd, t_true);
+    tcg_temp_free_i64(t_true);
 }
 
 /* C3.6.25 Floating-point data-processing (1 source) - single precision */
@@ -5444,16 +5680,16 @@
         assert(elements == 4);
 
         read_vec_element(s, tcg_elt, rn, 0, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt1, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt1, tcg_elt);
         read_vec_element(s, tcg_elt, rn, 1, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt2, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt2, tcg_elt);
 
         do_minmaxop(s, tcg_elt1, tcg_elt2, opcode, is_min, fpst);
 
         read_vec_element(s, tcg_elt, rn, 2, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt2, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt2, tcg_elt);
         read_vec_element(s, tcg_elt, rn, 3, MO_32);
-        tcg_gen_trunc_i64_i32(tcg_elt3, tcg_elt);
+        tcg_gen_extrl_i64_i32(tcg_elt3, tcg_elt);
 
         do_minmaxop(s, tcg_elt2, tcg_elt3, opcode, is_min, fpst);
 
@@ -7616,7 +7852,7 @@
             static NeonGenNarrowFn * const xtnfns[3] = {
                 gen_helper_neon_narrow_u8,
                 gen_helper_neon_narrow_u16,
-                tcg_gen_trunc_i64_i32,
+                tcg_gen_extrl_i64_i32,
             };
             static NeonGenNarrowEnvFn * const sqxtunfns[3] = {
                 gen_helper_neon_unarrow_sat8,
@@ -7650,10 +7886,8 @@
             } else {
                 TCGv_i32 tcg_lo = tcg_temp_new_i32();
                 TCGv_i32 tcg_hi = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tcg_lo, tcg_op);
+                tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, tcg_op);
                 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, cpu_env);
-                tcg_gen_shri_i64(tcg_op, tcg_op, 32);
-                tcg_gen_trunc_i64_i32(tcg_hi, tcg_op);
                 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, cpu_env);
                 tcg_gen_deposit_i32(tcg_res[pass], tcg_lo, tcg_hi, 16, 16);
                 tcg_temp_free_i32(tcg_lo);
@@ -8559,16 +8793,10 @@
     }
 }
 
-static void do_narrow_high_u32(TCGv_i32 res, TCGv_i64 in)
-{
-    tcg_gen_shri_i64(in, in, 32);
-    tcg_gen_trunc_i64_i32(res, in);
-}
-
 static void do_narrow_round_high_u32(TCGv_i32 res, TCGv_i64 in)
 {
     tcg_gen_addi_i64(in, in, 1U << 31);
-    do_narrow_high_u32(res, in);
+    tcg_gen_extrh_i64_i32(res, in);
 }
 
 static void handle_3rd_narrowing(DisasContext *s, int is_q, int is_u, int size,
@@ -8587,7 +8815,7 @@
               gen_helper_neon_narrow_round_high_u8 },
             { gen_helper_neon_narrow_high_u16,
               gen_helper_neon_narrow_round_high_u16 },
-            { do_narrow_high_u32, do_narrow_round_high_u32 },
+            { tcg_gen_extrh_i64_i32, do_narrow_round_high_u32 },
         };
         NeonGenNarrowFn *gennarrow = narrowfns[size][is_u];
 
@@ -10852,7 +11080,7 @@
 {
     uint32_t insn;
 
-    insn = arm_ldl_code(env, s->pc, s->bswap_code);
+    insn = arm_ldl_code(env, s->pc, s->sctlr_b);
     s->insn = insn;
     s->pc += 4;
 
@@ -10891,15 +11119,11 @@
     free_tmp_a64(s);
 }
 
-void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                        TranslationBlock *tb,
-                                        bool search_pc)
+void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb)
 {
     CPUState *cs = CPU(cpu);
     CPUARMState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     target_ulong next_page_start;
     int num_insns;
@@ -10915,18 +11139,25 @@
     dc->condjmp = 0;
 
     dc->aarch64 = 1;
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.
+     */
+    dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) &&
+                               !arm_el_is_aa64(env, 3);
     dc->thumb = 0;
-    dc->bswap_code = 0;
+    dc->sctlr_b = 0;
+    dc->be_data = ARM_TBFLAG_BE_DATA(tb->flags) ? MO_BE : MO_LE;
     dc->condexec_mask = 0;
     dc->condexec_cond = 0;
+    dc->mmu_idx = ARM_TBFLAG_MMUIDX(tb->flags);
+    dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx);
 #if !defined(CONFIG_USER_ONLY)
-    dc->user = (ARM_TBFLAG_AA64_EL(tb->flags) == 0);
+    dc->user = (dc->current_el == 0);
 #endif
-    dc->cpacr_fpen = ARM_TBFLAG_AA64_FPEN(tb->flags);
+    dc->fp_excp_el = ARM_TBFLAG_FPEXC_EL(tb->flags);
     dc->vec_len = 0;
     dc->vec_stride = 0;
     dc->cp_regs = cpu->cp_regs;
-    dc->current_el = arm_current_el(env);
     dc->features = env->features;
 
     /* Single step state. The code-generation logic here is:
@@ -10944,59 +11175,60 @@
      *   emit code to generate a software step exception
      *   end the TB
      */
-    dc->ss_active = ARM_TBFLAG_AA64_SS_ACTIVE(tb->flags);
-    dc->pstate_ss = ARM_TBFLAG_AA64_PSTATE_SS(tb->flags);
+    dc->ss_active = ARM_TBFLAG_SS_ACTIVE(tb->flags);
+    dc->pstate_ss = ARM_TBFLAG_PSTATE_SS(tb->flags);
     dc->is_ldex = false;
     dc->ss_same_el = (arm_debug_target_el(env) == dc->current_el);
 
     init_tmp_a64_array(dc);
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
 
     tcg_clear_temp_count();
 
     do {
+        dc->insn_start_idx = tcg_op_buf_count();
+        tcg_gen_insn_start(dc->pc, 0, 0);
+        num_insns++;
+
         if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
+            CPUBreakpoint *bp;
             QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
-                    gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2;
-                    goto done_generating;
+                    if (bp->flags & BP_CPU) {
+                        gen_a64_set_pc_im(dc->pc);
+                        gen_helper_check_breakpoints(cpu_env);
+                        /* End the TB early; it likely won't be executed */
+                        dc->is_jmp = DISAS_UPDATE;
+                    } else {
+                        gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
+                        /* The address covered by the breakpoint must be
+                           included in [tb->pc, tb->pc + tb->size) in order
+                           to for it to be properly cleared -- thus we
+                           increment the PC here so that the logic setting
+                           tb->size below does the right thing.  */
+                        dc->pc += 4;
+                        goto done_generating;
+                    }
+                    break;
                 }
             }
         }
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
-        }
-
         if (dc->ss_active && !dc->pstate_ss) {
             /* Singlestep state is Active-pending.
              * If we're in this state at the start of a TB then either
@@ -11008,8 +11240,9 @@
              * "did not step an insn" case, and so the syndrome ISV and EX
              * bits should be zero.
              */
-            assert(num_insns == 0);
-            gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0));
+            assert(num_insns == 1);
+            gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0),
+                          default_exception_el(dc));
             dc->is_jmp = DISAS_EXC;
             break;
         }
@@ -11026,7 +11259,6 @@
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.
          */
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -11075,12 +11307,20 @@
             gen_a64_set_pc_im(dc->pc);
             gen_helper_wfe(cpu_env);
             break;
+        case DISAS_YIELD:
+            gen_a64_set_pc_im(dc->pc);
+            gen_helper_yield(cpu_env);
+            break;
         case DISAS_WFI:
             /* This is a special case because we don't want to just halt the CPU
              * if trying to debug across a WFI.
              */
             gen_a64_set_pc_im(dc->pc);
             gen_helper_wfi(cpu_env);
+            /* The helper doesn't necessarily throw an exception, but we
+             * must go back to the main loop to check for interrupts anyway.
+             */
+            tcg_gen_exit_tb(0);
             break;
         }
     }
@@ -11089,22 +11329,15 @@
     gen_tb_end(tb, num_insns);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM) &&
+        qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, dc->pc - pc_start,
-                         4 | (dc->bswap_code << 1));
+        log_target_disas(cs, pc_start, dc->pc - pc_start,
+                         4 | (bswap_code(dc->sctlr_b) ? 2 : 0));
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
diff --git a/target-arm/translate.c b/target-arm/translate.c
index 32d8505..bd5d5cb 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -18,15 +18,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "internals.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "qemu/bitops.h"
@@ -36,6 +33,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define ENABLE_ARCH_4T    arm_dc_feature(s, ARM_FEATURE_V4T)
@@ -52,7 +50,6 @@
 #define ARCH(x) do { if (!ENABLE_ARCH_##x) goto illegal_op; } while(0)
 
 #include "translate.h"
-static uint32_t gen_opc_condexec_bits[OPC_BUF_SIZE];
 
 #if defined(CONFIG_USER_ONLY)
 #define IS_USER(s) 1
@@ -60,16 +57,16 @@
 #define IS_USER(s) (s->user)
 #endif
 
-TCGv_ptr cpu_env;
+TCGv_env cpu_env;
 /* We reuse the same 64-bit temporaries for efficiency.  */
 static TCGv_i64 cpu_V0, cpu_V1, cpu_M0;
 static TCGv_i32 cpu_R[16];
-static TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF;
-static TCGv_i64 cpu_exclusive_addr;
-static TCGv_i64 cpu_exclusive_val;
+TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF;
+TCGv_i64 cpu_exclusive_addr;
+TCGv_i64 cpu_exclusive_val;
 #ifdef CONFIG_USER_ONLY
-static TCGv_i64 cpu_exclusive_test;
-static TCGv_i32 cpu_exclusive_info;
+TCGv_i64 cpu_exclusive_test;
+TCGv_i32 cpu_exclusive_info;
 #endif
 
 /* FIXME:  These should be removed.  */
@@ -88,31 +85,54 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
     for (i = 0; i < 16; i++) {
-        cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new_i32(cpu_env,
                                           offsetof(CPUARMState, regs[i]),
                                           regnames[i]);
     }
-    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");
-    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");
-    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");
-    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");
+    cpu_CF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, CF), "CF");
+    cpu_NF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, NF), "NF");
+    cpu_VF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, VF), "VF");
+    cpu_ZF = tcg_global_mem_new_i32(cpu_env, offsetof(CPUARMState, ZF), "ZF");
 
-    cpu_exclusive_addr = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_addr = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_addr), "exclusive_addr");
-    cpu_exclusive_val = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_val = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_val), "exclusive_val");
 #ifdef CONFIG_USER_ONLY
-    cpu_exclusive_test = tcg_global_mem_new_i64(TCG_AREG0,
+    cpu_exclusive_test = tcg_global_mem_new_i64(cpu_env,
         offsetof(CPUARMState, exclusive_test), "exclusive_test");
-    cpu_exclusive_info = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_exclusive_info = tcg_global_mem_new_i32(cpu_env,
         offsetof(CPUARMState, exclusive_info), "exclusive_info");
 #endif
 
     a64_translate_init();
 }
 
+static inline ARMMMUIdx get_a32_user_mem_index(DisasContext *s)
+{
+    /* Return the mmu_idx to use for A32/T32 "unprivileged load/store"
+     * insns:
+     *  if PL2, UNPREDICTABLE (we choose to implement as if PL0)
+     *  otherwise, access as if at PL0.
+     */
+    switch (s->mmu_idx) {
+    case ARMMMUIdx_S1E2:        /* this one is UNPREDICTABLE */
+    case ARMMMUIdx_S12NSE0:
+    case ARMMMUIdx_S12NSE1:
+        return ARMMMUIdx_S12NSE0;
+    case ARMMMUIdx_S1E3:
+    case ARMMMUIdx_S1SE0:
+    case ARMMMUIdx_S1SE1:
+        return ARMMMUIdx_S1SE0;
+    case ARMMMUIdx_S2NS:
+    default:
+        g_assert_not_reached();
+    }
+}
+
 static inline TCGv_i32 load_cpu_offset(int offset)
 {
     TCGv_i32 tmp = tcg_temp_new_i32();
@@ -195,12 +215,16 @@
     tcg_temp_free_i32(tcg_excp);
 }
 
-static void gen_exception(int excp, uint32_t syndrome)
+static void gen_exception(int excp, uint32_t syndrome, uint32_t target_el)
 {
     TCGv_i32 tcg_excp = tcg_const_i32(excp);
     TCGv_i32 tcg_syn = tcg_const_i32(syndrome);
+    TCGv_i32 tcg_el = tcg_const_i32(target_el);
 
-    gen_helper_exception_with_syndrome(cpu_env, tcg_excp, tcg_syn);
+    gen_helper_exception_with_syndrome(cpu_env, tcg_excp,
+                                       tcg_syn, tcg_el);
+
+    tcg_temp_free_i32(tcg_el);
     tcg_temp_free_i32(tcg_syn);
     tcg_temp_free_i32(tcg_excp);
 }
@@ -228,7 +252,8 @@
      * of the exception, and our syndrome information is always correct.
      */
     gen_ss_advance(s);
-    gen_exception(EXCP_UDEF, syn_swstep(s->ss_same_el, 1, s->is_ldex));
+    gen_exception(EXCP_UDEF, syn_swstep(s->ss_same_el, 1, s->is_ldex),
+                  default_exception_el(s));
     s->is_jmp = DISAS_EXC;
 }
 
@@ -711,81 +736,113 @@
 #undef PAS_OP
 
 /*
- * generate a conditional branch based on ARM condition code cc.
+ * Generate a conditional based on ARM condition code cc.
  * This is common between ARM and Aarch64 targets.
  */
-void arm_gen_test_cc(int cc, int label)
+void arm_test_cc(DisasCompare *cmp, int cc)
 {
-    TCGv_i32 tmp;
-    int inv;
+    TCGv_i32 value;
+    TCGCond cond;
+    bool global = true;
 
     switch (cc) {
     case 0: /* eq: Z */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        break;
     case 1: /* ne: !Z */
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
+        cond = TCG_COND_EQ;
+        value = cpu_ZF;
         break;
+
     case 2: /* cs: C */
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_CF, 0, label);
-        break;
     case 3: /* cc: !C */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
+        cond = TCG_COND_NE;
+        value = cpu_CF;
         break;
+
     case 4: /* mi: N */
-        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_NF, 0, label);
-        break;
     case 5: /* pl: !N */
-        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_NF, 0, label);
+        cond = TCG_COND_LT;
+        value = cpu_NF;
         break;
+
     case 6: /* vs: V */
-        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_VF, 0, label);
-        break;
     case 7: /* vc: !V */
-        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_VF, 0, label);
+        cond = TCG_COND_LT;
+        value = cpu_VF;
         break;
+
     case 8: /* hi: C && !Z */
-        inv = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, inv);
-        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
-        gen_set_label(inv);
+    case 9: /* ls: !C || Z -> !(C && !Z) */
+        cond = TCG_COND_NE;
+        value = tcg_temp_new_i32();
+        global = false;
+        /* CF is 1 for C, so -CF is an all-bits-set mask for C;
+           ZF is non-zero for !Z; so AND the two subexpressions.  */
+        tcg_gen_neg_i32(value, cpu_CF);
+        tcg_gen_and_i32(value, value, cpu_ZF);
         break;
-    case 9: /* ls: !C || Z */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        break;
+
     case 10: /* ge: N == V -> N ^ V == 0 */
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        break;
     case 11: /* lt: N != V -> N ^ V != 0 */
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
+        /* Since we're only interested in the sign bit, == 0 is >= 0.  */
+        cond = TCG_COND_GE;
+        value = tcg_temp_new_i32();
+        global = false;
+        tcg_gen_xor_i32(value, cpu_VF, cpu_NF);
         break;
+
     case 12: /* gt: !Z && N == V */
-        inv = gen_new_label();
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, inv);
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        gen_set_label(inv);
-        break;
     case 13: /* le: Z || N != V */
-        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
-        tmp = tcg_temp_new_i32();
-        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
+        cond = TCG_COND_NE;
+        value = tcg_temp_new_i32();
+        global = false;
+        /* (N == V) is equal to the sign bit of ~(NF ^ VF).  Propagate
+         * the sign bit then AND with ZF to yield the result.  */
+        tcg_gen_xor_i32(value, cpu_VF, cpu_NF);
+        tcg_gen_sari_i32(value, value, 31);
+        tcg_gen_andc_i32(value, cpu_ZF, value);
         break;
+
+    case 14: /* always */
+    case 15: /* always */
+        /* Use the ALWAYS condition, which will fold early.
+         * It doesn't matter what we use for the value.  */
+        cond = TCG_COND_ALWAYS;
+        value = cpu_ZF;
+        goto no_invert;
+
     default:
         fprintf(stderr, "Bad condition code 0x%x\n", cc);
         abort();
     }
+
+    if (cc & 1) {
+        cond = tcg_invert_cond(cond);
+    }
+
+ no_invert:
+    cmp->cond = cond;
+    cmp->value = value;
+    cmp->value_global = global;
+}
+
+void arm_free_cc(DisasCompare *cmp)
+{
+    if (!cmp->value_global) {
+        tcg_temp_free_i32(cmp->value);
+    }
+}
+
+void arm_jump_cc(DisasCompare *cmp, TCGLabel *label)
+{
+    tcg_gen_brcondi_i32(cmp->cond, cmp->value, 0, label);
+}
+
+void arm_gen_test_cc(int cc, TCGLabel *label)
+{
+    DisasCompare cmp;
+    arm_test_cc(&cmp, cc);
+    arm_jump_cc(&cmp, label);
+    arm_free_cc(&cmp);
 }
 
 static const uint8_t table_logic_cc[16] = {
@@ -812,7 +869,7 @@
 {
     TCGv_i32 tmp;
 
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
     if (s->thumb != (addr & 1)) {
         tmp = tcg_temp_new_i32();
         tcg_gen_movi_i32(tmp, addr & 1);
@@ -825,7 +882,7 @@
 /* Set PC and Thumb state from var.  var is marked as dead.  */
 static inline void gen_bx(DisasContext *s, TCGv_i32 var)
 {
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
     tcg_gen_andi_i32(cpu_R[15], var, ~1);
     tcg_gen_andi_i32(var, var, 1);
     store_cpu_field(var, thumb);
@@ -856,6 +913,12 @@
     }
 }
 
+#ifdef CONFIG_USER_ONLY
+#define IS_USER_ONLY 1
+#else
+#define IS_USER_ONLY 0
+#endif
+
 /* Abstractions of "generate code to do a guest load/store for
  * AArch32", where a vaddr is always 32 bits (and is zero
  * extended if we're a 64 bit core) and  data is also
@@ -865,74 +928,143 @@
  */
 #if TARGET_LONG_BITS == 32
 
-#define DO_GEN_LD(SUFF, OPC)                                             \
-static inline void gen_aa32_ld##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_LD(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
-    tcg_gen_qemu_ld_i32(val, addr, index, OPC);                          \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        TCGv addr_be = tcg_temp_new();                                   \
+        tcg_gen_xori_i32(addr_be, addr, BE32_XOR);                       \
+        tcg_gen_qemu_ld_i32(val, addr_be, index, opc);                   \
+        tcg_temp_free(addr_be);                                          \
+        return;                                                          \
+    }                                                                    \
+    tcg_gen_qemu_ld_i32(val, addr, index, opc);                          \
 }
 
-#define DO_GEN_ST(SUFF, OPC)                                             \
-static inline void gen_aa32_st##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_ST(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
-    tcg_gen_qemu_st_i32(val, addr, index, OPC);                          \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        TCGv addr_be = tcg_temp_new();                                   \
+        tcg_gen_xori_i32(addr_be, addr, BE32_XOR);                       \
+        tcg_gen_qemu_st_i32(val, addr_be, index, opc);                   \
+        tcg_temp_free(addr_be);                                          \
+        return;                                                          \
+    }                                                                    \
+    tcg_gen_qemu_st_i32(val, addr, index, opc);                          \
 }
 
-static inline void gen_aa32_ld64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
-    tcg_gen_qemu_ld_i64(val, addr, index, MO_TEQ);
+    TCGMemOp opc = MO_Q | s->be_data;
+    tcg_gen_qemu_ld_i64(val, addr, index, opc);
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        tcg_gen_rotri_i64(val, val, 32);
+    }
 }
 
-static inline void gen_aa32_st64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_st64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
-    tcg_gen_qemu_st_i64(val, addr, index, MO_TEQ);
+    TCGMemOp opc = MO_Q | s->be_data;
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        TCGv_i64 tmp = tcg_temp_new_i64();
+        tcg_gen_rotri_i64(tmp, val, 32);
+        tcg_gen_qemu_st_i64(tmp, addr, index, opc);
+        tcg_temp_free_i64(tmp);
+        return;
+    }
+    tcg_gen_qemu_st_i64(val, addr, index, opc);
 }
 
 #else
 
-#define DO_GEN_LD(SUFF, OPC)                                             \
-static inline void gen_aa32_ld##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_LD(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_ld##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
     TCGv addr64 = tcg_temp_new();                                        \
     tcg_gen_extu_i32_i64(addr64, addr);                                  \
-    tcg_gen_qemu_ld_i32(val, addr64, index, OPC);                        \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        tcg_gen_xori_i64(addr64, addr64, BE32_XOR);                      \
+    }                                                                    \
+    tcg_gen_qemu_ld_i32(val, addr64, index, opc);                        \
     tcg_temp_free(addr64);                                               \
 }
 
-#define DO_GEN_ST(SUFF, OPC)                                             \
-static inline void gen_aa32_st##SUFF(TCGv_i32 val, TCGv_i32 addr, int index) \
+#define DO_GEN_ST(SUFF, OPC, BE32_XOR)                                   \
+static inline void gen_aa32_st##SUFF(DisasContext *s, TCGv_i32 val,      \
+                                     TCGv_i32 addr, int index)           \
 {                                                                        \
+    TCGMemOp opc = (OPC) | s->be_data;                                   \
     TCGv addr64 = tcg_temp_new();                                        \
     tcg_gen_extu_i32_i64(addr64, addr);                                  \
-    tcg_gen_qemu_st_i32(val, addr64, index, OPC);                        \
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */    \
+    if (!IS_USER_ONLY && s->sctlr_b && BE32_XOR) {                       \
+        tcg_gen_xori_i64(addr64, addr64, BE32_XOR);                      \
+    }                                                                    \
+    tcg_gen_qemu_st_i32(val, addr64, index, opc);                        \
     tcg_temp_free(addr64);                                               \
 }
 
-static inline void gen_aa32_ld64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_ld64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
+    TCGMemOp opc = MO_Q | s->be_data;
     TCGv addr64 = tcg_temp_new();
     tcg_gen_extu_i32_i64(addr64, addr);
-    tcg_gen_qemu_ld_i64(val, addr64, index, MO_TEQ);
+    tcg_gen_qemu_ld_i64(val, addr64, index, opc);
+
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        tcg_gen_rotri_i64(val, val, 32);
+    }
     tcg_temp_free(addr64);
 }
 
-static inline void gen_aa32_st64(TCGv_i64 val, TCGv_i32 addr, int index)
+static inline void gen_aa32_st64(DisasContext *s, TCGv_i64 val,
+                                 TCGv_i32 addr, int index)
 {
+    TCGMemOp opc = MO_Q | s->be_data;
     TCGv addr64 = tcg_temp_new();
     tcg_gen_extu_i32_i64(addr64, addr);
-    tcg_gen_qemu_st_i64(val, addr64, index, MO_TEQ);
+
+    /* Not needed for user-mode BE32, where we use MO_BE instead.  */
+    if (!IS_USER_ONLY && s->sctlr_b) {
+        TCGv tmp = tcg_temp_new();
+        tcg_gen_rotri_i64(tmp, val, 32);
+        tcg_gen_qemu_st_i64(tmp, addr64, index, opc);
+        tcg_temp_free(tmp);
+    } else {
+        tcg_gen_qemu_st_i64(val, addr64, index, opc);
+    }
     tcg_temp_free(addr64);
 }
 
 #endif
 
-DO_GEN_LD(8s, MO_SB)
-DO_GEN_LD(8u, MO_UB)
-DO_GEN_LD(16s, MO_TESW)
-DO_GEN_LD(16u, MO_TEUW)
-DO_GEN_LD(32u, MO_TEUL)
-DO_GEN_ST(8, MO_UB)
-DO_GEN_ST(16, MO_TEUW)
-DO_GEN_ST(32, MO_TEUL)
+DO_GEN_LD(8s, MO_SB, 3)
+DO_GEN_LD(8u, MO_UB, 3)
+DO_GEN_LD(16s, MO_SW, 2)
+DO_GEN_LD(16u, MO_UW, 2)
+DO_GEN_LD(32u, MO_UL, 0)
+/* 'a' variants include an alignment check */
+DO_GEN_LD(16ua, MO_UW | MO_ALIGN, 2)
+DO_GEN_LD(32ua, MO_UL | MO_ALIGN, 0)
+DO_GEN_ST(8, MO_UB, 3)
+DO_GEN_ST(16, MO_UW, 2)
+DO_GEN_ST(32, MO_UL, 0)
 
 static inline void gen_set_pc_im(DisasContext *s, target_ulong val)
 {
@@ -991,11 +1123,12 @@
     s->is_jmp = DISAS_JUMP;
 }
 
-static void gen_exception_insn(DisasContext *s, int offset, int excp, int syn)
+static void gen_exception_insn(DisasContext *s, int offset, int excp,
+                               int syn, uint32_t target_el)
 {
     gen_set_condexec(s);
     gen_set_pc_im(s, s->pc - offset);
-    gen_exception(excp, syn);
+    gen_exception(excp, syn, target_el);
     s->is_jmp = DISAS_JUMP;
 }
 
@@ -1003,7 +1136,7 @@
 static inline void gen_lookup_tb(DisasContext *s)
 {
     tcg_gen_movi_i32(cpu_R[15], s->pc & ~1);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 static inline void gen_add_data_offset(DisasContext *s, unsigned int insn,
@@ -1226,18 +1359,18 @@
 static inline void gen_vfp_ld(DisasContext *s, int dp, TCGv_i32 addr)
 {
     if (dp) {
-        gen_aa32_ld64(cpu_F0d, addr, get_mem_index(s));
+        gen_aa32_ld64(s, cpu_F0d, addr, get_mem_index(s));
     } else {
-        gen_aa32_ld32u(cpu_F0s, addr, get_mem_index(s));
+        gen_aa32_ld32u(s, cpu_F0s, addr, get_mem_index(s));
     }
 }
 
 static inline void gen_vfp_st(DisasContext *s, int dp, TCGv_i32 addr)
 {
     if (dp) {
-        gen_aa32_st64(cpu_F0d, addr, get_mem_index(s));
+        gen_aa32_st64(s, cpu_F0d, addr, get_mem_index(s));
     } else {
-        gen_aa32_st32(cpu_F0s, addr, get_mem_index(s));
+        gen_aa32_st32(s, cpu_F0s, addr, get_mem_index(s));
     }
 }
 
@@ -1529,7 +1662,7 @@
     } else {
         tmp = tcg_temp_new_i32();
         iwmmxt_load_reg(cpu_V0, rd);
-        tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+        tcg_gen_extrl_i64_i32(tmp, cpu_V0);
     }
     tcg_gen_andi_i32(tmp, tmp, mask);
     tcg_gen_mov_i32(dest, tmp);
@@ -1553,9 +1686,9 @@
             rdhi = (insn >> 16) & 0xf;
             if (insn & ARM_CP_RW_BIT) {			/* TMRRC */
                 iwmmxt_load_reg(cpu_V0, wrd);
-                tcg_gen_trunc_i64_i32(cpu_R[rdlo], cpu_V0);
+                tcg_gen_extrl_i64_i32(cpu_R[rdlo], cpu_V0);
                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                tcg_gen_trunc_i64_i32(cpu_R[rdhi], cpu_V0);
+                tcg_gen_extrl_i64_i32(cpu_R[rdhi], cpu_V0);
             } else {					/* TMCRR */
                 tcg_gen_concat_i32_i64(cpu_V0, cpu_R[rdlo], cpu_R[rdhi]);
                 iwmmxt_store_reg(cpu_V0, wrd);
@@ -1573,24 +1706,24 @@
         if (insn & ARM_CP_RW_BIT) {
             if ((insn >> 28) == 0xf) {			/* WLDRW wCx */
                 tmp = tcg_temp_new_i32();
-                gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                 iwmmxt_store_creg(wrd, tmp);
             } else {
                 i = 1;
                 if (insn & (1 << 8)) {
                     if (insn & (1 << 22)) {		/* WLDRD */
-                        gen_aa32_ld64(cpu_M0, addr, get_mem_index(s));
+                        gen_aa32_ld64(s, cpu_M0, addr, get_mem_index(s));
                         i = 0;
                     } else {				/* WLDRW wRd */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     }
                 } else {
                     tmp = tcg_temp_new_i32();
                     if (insn & (1 << 22)) {		/* WLDRH */
-                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                     } else {				/* WLDRB */
-                        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                     }
                 }
                 if (i) {
@@ -1602,24 +1735,24 @@
         } else {
             if ((insn >> 28) == 0xf) {			/* WSTRW wCx */
                 tmp = iwmmxt_load_creg(wrd);
-                gen_aa32_st32(tmp, addr, get_mem_index(s));
+                gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             } else {
                 gen_op_iwmmxt_movq_M0_wRn(wrd);
                 tmp = tcg_temp_new_i32();
                 if (insn & (1 << 8)) {
                     if (insn & (1 << 22)) {		/* WSTRD */
-                        gen_aa32_st64(cpu_M0, addr, get_mem_index(s));
+                        gen_aa32_st64(s, cpu_M0, addr, get_mem_index(s));
                     } else {				/* WSTRW wRd */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     }
                 } else {
                     if (insn & (1 << 22)) {		/* WSTRH */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st16(tmp, addr, get_mem_index(s));
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
+                        gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                     } else {				/* WSTRB */
-                        tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st8(tmp, addr, get_mem_index(s));
+                        tcg_gen_extrl_i64_i32(tmp, cpu_M0);
+                        gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                     }
                 }
             }
@@ -1918,7 +2051,7 @@
         switch ((insn >> 22) & 3) {
         case 0:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 7) << 3);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             if (insn & 8) {
                 tcg_gen_ext8s_i32(tmp, tmp);
             } else {
@@ -1927,7 +2060,7 @@
             break;
         case 1:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 3) << 4);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             if (insn & 8) {
                 tcg_gen_ext16s_i32(tmp, tmp);
             } else {
@@ -1936,7 +2069,7 @@
             break;
         case 2:
             tcg_gen_shri_i64(cpu_M0, cpu_M0, (insn & 1) << 5);
-            tcg_gen_trunc_i64_i32(tmp, cpu_M0);
+            tcg_gen_extrl_i64_i32(tmp, cpu_M0);
             break;
         }
         store_reg(s, rd, tmp);
@@ -2599,9 +2732,9 @@
 
         if (insn & ARM_CP_RW_BIT) {			/* MRA */
             iwmmxt_load_reg(cpu_V0, acc);
-            tcg_gen_trunc_i64_i32(cpu_R[rdlo], cpu_V0);
+            tcg_gen_extrl_i64_i32(cpu_R[rdlo], cpu_V0);
             tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-            tcg_gen_trunc_i64_i32(cpu_R[rdhi], cpu_V0);
+            tcg_gen_extrl_i64_i32(cpu_R[rdhi], cpu_V0);
             tcg_gen_andi_i32(cpu_R[rdhi], cpu_R[rdhi], (1 << (40 - 32)) - 1);
         } else {					/* MAR */
             tcg_gen_concat_i32_i64(cpu_V0, cpu_R[rdlo], cpu_R[rdhi]);
@@ -2684,15 +2817,15 @@
     TCGv_i32 tmp = tcg_temp_new_i32();
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
         gen_neon_dup_u8(tmp, 0);
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
         gen_neon_dup_low16(tmp);
         break;
     case 2:
-        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
         break;
     default: /* Avoid compiler warnings.  */
         abort();
@@ -2923,7 +3056,7 @@
         } else {
             gen_helper_vfp_tould(tcg_res, tcg_double, tcg_shift, fpst);
         }
-        tcg_gen_trunc_i64_i32(tcg_tmp, tcg_res);
+        tcg_gen_extrl_i64_i32(tcg_tmp, tcg_res);
         tcg_gen_st_f32(tcg_tmp, cpu_env, vfp_reg_offset(0, rd));
         tcg_temp_free_i32(tcg_tmp);
         tcg_temp_free_i64(tcg_res);
@@ -3016,9 +3149,9 @@
      * for invalid encodings; we will generate incorrect syndrome information
      * for attempts to execute invalid vfp/neon encodings with FP disabled.
      */
-    if (!s->cpacr_fpen) {
+    if (s->fp_excp_el) {
         gen_exception_insn(s, 4, EXCP_UDEF,
-                           syn_fp_access_trap(1, 0xe, s->thumb));
+                           syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
         return 0;
     }
 
@@ -3918,15 +4051,22 @@
     return 0;
 }
 
+static inline bool use_goto_tb(DisasContext *s, target_ulong dest)
+{
+#ifndef CONFIG_USER_ONLY
+    return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
+           ((s->pc - 1) & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(DisasContext *s, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-
-    tb = s->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+    if (use_goto_tb(s, dest)) {
         tcg_gen_goto_tb(n);
         gen_set_pc_im(s, dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)s->tb + n);
     } else {
         gen_set_pc_im(s, dest);
         tcg_gen_exit_tb(0);
@@ -4029,29 +4169,222 @@
     return gen_set_psr(s, mask, spsr, tmp);
 }
 
+static bool msr_banked_access_decode(DisasContext *s, int r, int sysm, int rn,
+                                     int *tgtmode, int *regno)
+{
+    /* Decode the r and sysm fields of MSR/MRS banked accesses into
+     * the target mode and register number, and identify the various
+     * unpredictable cases.
+     * MSR (banked) and MRS (banked) are CONSTRAINED UNPREDICTABLE if:
+     *  + executed in user mode
+     *  + using R15 as the src/dest register
+     *  + accessing an unimplemented register
+     *  + accessing a register that's inaccessible at current PL/security state*
+     *  + accessing a register that you could access with a different insn
+     * We choose to UNDEF in all these cases.
+     * Since we don't know which of the various AArch32 modes we are in
+     * we have to defer some checks to runtime.
+     * Accesses to Monitor mode registers from Secure EL1 (which implies
+     * that EL3 is AArch64) must trap to EL3.
+     *
+     * If the access checks fail this function will emit code to take
+     * an exception and return false. Otherwise it will return true,
+     * and set *tgtmode and *regno appropriately.
+     */
+    int exc_target = default_exception_el(s);
+
+    /* These instructions are present only in ARMv8, or in ARMv7 with the
+     * Virtualization Extensions.
+     */
+    if (!arm_dc_feature(s, ARM_FEATURE_V8) &&
+        !arm_dc_feature(s, ARM_FEATURE_EL2)) {
+        goto undef;
+    }
+
+    if (IS_USER(s) || rn == 15) {
+        goto undef;
+    }
+
+    /* The table in the v8 ARM ARM section F5.2.3 describes the encoding
+     * of registers into (r, sysm).
+     */
+    if (r) {
+        /* SPSRs for other modes */
+        switch (sysm) {
+        case 0xe: /* SPSR_fiq */
+            *tgtmode = ARM_CPU_MODE_FIQ;
+            break;
+        case 0x10: /* SPSR_irq */
+            *tgtmode = ARM_CPU_MODE_IRQ;
+            break;
+        case 0x12: /* SPSR_svc */
+            *tgtmode = ARM_CPU_MODE_SVC;
+            break;
+        case 0x14: /* SPSR_abt */
+            *tgtmode = ARM_CPU_MODE_ABT;
+            break;
+        case 0x16: /* SPSR_und */
+            *tgtmode = ARM_CPU_MODE_UND;
+            break;
+        case 0x1c: /* SPSR_mon */
+            *tgtmode = ARM_CPU_MODE_MON;
+            break;
+        case 0x1e: /* SPSR_hyp */
+            *tgtmode = ARM_CPU_MODE_HYP;
+            break;
+        default: /* unallocated */
+            goto undef;
+        }
+        /* We arbitrarily assign SPSR a register number of 16. */
+        *regno = 16;
+    } else {
+        /* general purpose registers for other modes */
+        switch (sysm) {
+        case 0x0 ... 0x6:   /* 0b00xxx : r8_usr ... r14_usr */
+            *tgtmode = ARM_CPU_MODE_USR;
+            *regno = sysm + 8;
+            break;
+        case 0x8 ... 0xe:   /* 0b01xxx : r8_fiq ... r14_fiq */
+            *tgtmode = ARM_CPU_MODE_FIQ;
+            *regno = sysm;
+            break;
+        case 0x10 ... 0x11: /* 0b1000x : r14_irq, r13_irq */
+            *tgtmode = ARM_CPU_MODE_IRQ;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x12 ... 0x13: /* 0b1001x : r14_svc, r13_svc */
+            *tgtmode = ARM_CPU_MODE_SVC;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x14 ... 0x15: /* 0b1010x : r14_abt, r13_abt */
+            *tgtmode = ARM_CPU_MODE_ABT;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x16 ... 0x17: /* 0b1011x : r14_und, r13_und */
+            *tgtmode = ARM_CPU_MODE_UND;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x1c ... 0x1d: /* 0b1110x : r14_mon, r13_mon */
+            *tgtmode = ARM_CPU_MODE_MON;
+            *regno = sysm & 1 ? 13 : 14;
+            break;
+        case 0x1e ... 0x1f: /* 0b1111x : elr_hyp, r13_hyp */
+            *tgtmode = ARM_CPU_MODE_HYP;
+            /* Arbitrarily pick 17 for ELR_Hyp (which is not a banked LR!) */
+            *regno = sysm & 1 ? 13 : 17;
+            break;
+        default: /* unallocated */
+            goto undef;
+        }
+    }
+
+    /* Catch the 'accessing inaccessible register' cases we can detect
+     * at translate time.
+     */
+    switch (*tgtmode) {
+    case ARM_CPU_MODE_MON:
+        if (!arm_dc_feature(s, ARM_FEATURE_EL3) || s->ns) {
+            goto undef;
+        }
+        if (s->current_el == 1) {
+            /* If we're in Secure EL1 (which implies that EL3 is AArch64)
+             * then accesses to Mon registers trap to EL3
+             */
+            exc_target = 3;
+            goto undef;
+        }
+        break;
+    case ARM_CPU_MODE_HYP:
+        /* Note that we can forbid accesses from EL2 here because they
+         * must be from Hyp mode itself
+         */
+        if (!arm_dc_feature(s, ARM_FEATURE_EL2) || s->current_el < 3) {
+            goto undef;
+        }
+        break;
+    default:
+        break;
+    }
+
+    return true;
+
+undef:
+    /* If we get here then some access check did not pass */
+    gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(), exc_target);
+    return false;
+}
+
+static void gen_msr_banked(DisasContext *s, int r, int sysm, int rn)
+{
+    TCGv_i32 tcg_reg, tcg_tgtmode, tcg_regno;
+    int tgtmode = 0, regno = 0;
+
+    if (!msr_banked_access_decode(s, r, sysm, rn, &tgtmode, &regno)) {
+        return;
+    }
+
+    /* Sync state because msr_banked() can raise exceptions */
+    gen_set_condexec(s);
+    gen_set_pc_im(s, s->pc - 4);
+    tcg_reg = load_reg(s, rn);
+    tcg_tgtmode = tcg_const_i32(tgtmode);
+    tcg_regno = tcg_const_i32(regno);
+    gen_helper_msr_banked(cpu_env, tcg_reg, tcg_tgtmode, tcg_regno);
+    tcg_temp_free_i32(tcg_tgtmode);
+    tcg_temp_free_i32(tcg_regno);
+    tcg_temp_free_i32(tcg_reg);
+    s->is_jmp = DISAS_UPDATE;
+}
+
+static void gen_mrs_banked(DisasContext *s, int r, int sysm, int rn)
+{
+    TCGv_i32 tcg_reg, tcg_tgtmode, tcg_regno;
+    int tgtmode = 0, regno = 0;
+
+    if (!msr_banked_access_decode(s, r, sysm, rn, &tgtmode, &regno)) {
+        return;
+    }
+
+    /* Sync state because mrs_banked() can raise exceptions */
+    gen_set_condexec(s);
+    gen_set_pc_im(s, s->pc - 4);
+    tcg_reg = tcg_temp_new_i32();
+    tcg_tgtmode = tcg_const_i32(tgtmode);
+    tcg_regno = tcg_const_i32(regno);
+    gen_helper_mrs_banked(tcg_reg, cpu_env, tcg_tgtmode, tcg_regno);
+    tcg_temp_free_i32(tcg_tgtmode);
+    tcg_temp_free_i32(tcg_regno);
+    store_reg(s, rn, tcg_reg);
+    s->is_jmp = DISAS_UPDATE;
+}
+
 /* Generate an old-style exception return. Marks pc as dead. */
 static void gen_exception_return(DisasContext *s, TCGv_i32 pc)
 {
     TCGv_i32 tmp;
     store_reg(s, 15, pc);
     tmp = load_cpu_field(spsr);
-    gen_set_cpsr(tmp, CPSR_ERET_MASK);
+    gen_helper_cpsr_write_eret(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 /* Generate a v6 exception return.  Marks both values as dead.  */
 static void gen_rfe(DisasContext *s, TCGv_i32 pc, TCGv_i32 cpsr)
 {
-    gen_set_cpsr(cpsr, CPSR_ERET_MASK);
+    gen_helper_cpsr_write_eret(cpu_env, cpsr);
     tcg_temp_free_i32(cpsr);
     store_reg(s, 15, pc);
-    s->is_jmp = DISAS_UPDATE;
+    s->is_jmp = DISAS_JUMP;
 }
 
 static void gen_nop_hint(DisasContext *s, int val)
 {
     switch (val) {
+    case 1: /* yield */
+        gen_set_pc_im(s, s->pc);
+        s->is_jmp = DISAS_YIELD;
+        break;
     case 3: /* wfi */
         gen_set_pc_im(s, s->pc);
         s->is_jmp = DISAS_WFI;
@@ -4334,9 +4667,9 @@
      * for invalid encodings; we will generate incorrect syndrome information
      * for attempts to execute invalid vfp/neon encodings with FP disabled.
      */
-    if (!s->cpacr_fpen) {
+    if (s->fp_excp_el) {
         gen_exception_insn(s, 4, EXCP_UDEF,
-                           syn_fp_access_trap(1, 0xe, s->thumb));
+                           syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
         return 0;
     }
 
@@ -4386,11 +4719,11 @@
             if (size == 3) {
                 tmp64 = tcg_temp_new_i64();
                 if (load) {
-                    gen_aa32_ld64(tmp64, addr, get_mem_index(s));
+                    gen_aa32_ld64(s, tmp64, addr, get_mem_index(s));
                     neon_store_reg64(tmp64, rd);
                 } else {
                     neon_load_reg64(tmp64, rd);
-                    gen_aa32_st64(tmp64, addr, get_mem_index(s));
+                    gen_aa32_st64(s, tmp64, addr, get_mem_index(s));
                 }
                 tcg_temp_free_i64(tmp64);
                 tcg_gen_addi_i32(addr, addr, stride);
@@ -4399,21 +4732,21 @@
                     if (size == 2) {
                         if (load) {
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                             neon_store_reg(rd, pass, tmp);
                         } else {
                             tmp = neon_load_reg(rd, pass);
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                         }
                         tcg_gen_addi_i32(addr, addr, stride);
                     } else if (size == 1) {
                         if (load) {
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                             tcg_gen_addi_i32(addr, addr, stride);
                             tmp2 = tcg_temp_new_i32();
-                            gen_aa32_ld16u(tmp2, addr, get_mem_index(s));
+                            gen_aa32_ld16u(s, tmp2, addr, get_mem_index(s));
                             tcg_gen_addi_i32(addr, addr, stride);
                             tcg_gen_shli_i32(tmp2, tmp2, 16);
                             tcg_gen_or_i32(tmp, tmp, tmp2);
@@ -4423,10 +4756,10 @@
                             tmp = neon_load_reg(rd, pass);
                             tmp2 = tcg_temp_new_i32();
                             tcg_gen_shri_i32(tmp2, tmp, 16);
-                            gen_aa32_st16(tmp, addr, get_mem_index(s));
+                            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                             tcg_gen_addi_i32(addr, addr, stride);
-                            gen_aa32_st16(tmp2, addr, get_mem_index(s));
+                            gen_aa32_st16(s, tmp2, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp2);
                             tcg_gen_addi_i32(addr, addr, stride);
                         }
@@ -4435,7 +4768,7 @@
                             TCGV_UNUSED_I32(tmp2);
                             for (n = 0; n < 4; n++) {
                                 tmp = tcg_temp_new_i32();
-                                gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                                gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                                 tcg_gen_addi_i32(addr, addr, stride);
                                 if (n == 0) {
                                     tmp2 = tmp;
@@ -4455,7 +4788,7 @@
                                 } else {
                                     tcg_gen_shri_i32(tmp, tmp2, n * 8);
                                 }
-                                gen_aa32_st8(tmp, addr, get_mem_index(s));
+                                gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                                 tcg_temp_free_i32(tmp);
                                 tcg_gen_addi_i32(addr, addr, stride);
                             }
@@ -4579,13 +4912,13 @@
                     tmp = tcg_temp_new_i32();
                     switch (size) {
                     case 0:
-                        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                         break;
                     case 1:
-                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         break;
                     default: /* Avoid compiler warnings.  */
                         abort();
@@ -4603,13 +4936,13 @@
                         tcg_gen_shri_i32(tmp, tmp, shift);
                     switch (size) {
                     case 0:
-                        gen_aa32_st8(tmp, addr, get_mem_index(s));
+                        gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                         break;
                     case 1:
-                        gen_aa32_st16(tmp, addr, get_mem_index(s));
+                        gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         break;
                     }
                     tcg_temp_free_i32(tmp);
@@ -4651,7 +4984,7 @@
     switch (size) {
     case 0: gen_helper_neon_narrow_u8(dest, src); break;
     case 1: gen_helper_neon_narrow_u16(dest, src); break;
-    case 2: tcg_gen_trunc_i64_i32(dest, src); break;
+    case 2: tcg_gen_extrl_i64_i32(dest, src); break;
     default: abort();
     }
 }
@@ -4979,6 +5312,30 @@
             op >= NEON_2RM_VRECPE_F);
 }
 
+static bool neon_2rm_is_v8_op(int op)
+{
+    /* Return true if this neon 2reg-misc op is ARMv8 and up */
+    switch (op) {
+    case NEON_2RM_VRINTN:
+    case NEON_2RM_VRINTA:
+    case NEON_2RM_VRINTM:
+    case NEON_2RM_VRINTP:
+    case NEON_2RM_VRINTZ:
+    case NEON_2RM_VRINTX:
+    case NEON_2RM_VCVTAU:
+    case NEON_2RM_VCVTAS:
+    case NEON_2RM_VCVTNU:
+    case NEON_2RM_VCVTNS:
+    case NEON_2RM_VCVTPU:
+    case NEON_2RM_VCVTPS:
+    case NEON_2RM_VCVTMU:
+    case NEON_2RM_VCVTMS:
+        return true;
+    default:
+        return false;
+    }
+}
+
 /* Each entry in this array has bit n set if the insn allows
  * size value n (otherwise it will UNDEF). Since unallocated
  * op values will have no bits set they always UNDEF.
@@ -5072,9 +5429,9 @@
      * for invalid encodings; we will generate incorrect syndrome information
      * for attempts to execute invalid vfp/neon encodings with FP disabled.
      */
-    if (!s->cpacr_fpen) {
+    if (s->fp_excp_el) {
         gen_exception_insn(s, 4, EXCP_UDEF,
-                           syn_fp_access_trap(1, 0xe, s->thumb));
+                           syn_fp_access_trap(1, 0xe, false), s->fp_excp_el);
         return 0;
     }
 
@@ -6222,7 +6579,7 @@
                                 break;
                             case 2:
                                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                                tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+                                tcg_gen_extrl_i64_i32(tmp, cpu_V0);
                                 break;
                             default: abort();
                             }
@@ -6237,7 +6594,7 @@
                             case 2:
                                 tcg_gen_addi_i64(cpu_V0, cpu_V0, 1u << 31);
                                 tcg_gen_shri_i64(cpu_V0, cpu_V0, 32);
-                                tcg_gen_trunc_i64_i32(tmp, cpu_V0);
+                                tcg_gen_extrl_i64_i32(tmp, cpu_V0);
                                 break;
                             default: abort();
                             }
@@ -6466,6 +6823,10 @@
                 if ((neon_2rm_sizes[op] & (1 << size)) == 0) {
                     return 1;
                 }
+                if (neon_2rm_is_v8_op(op) &&
+                    !arm_dc_feature(s, ARM_FEATURE_V8)) {
+                    return 1;
+                }
                 if ((op != NEON_2RM_VMOVN && op != NEON_2RM_VQMOVN) &&
                     q && ((rm | rd) & 1)) {
                     return 1;
@@ -7091,7 +7452,7 @@
     rt = (insn >> 12) & 0xf;
 
     ri = get_arm_cp_reginfo(s->cp_regs,
-                            ENCODE_CP_REG(cpnum, is64, crn, crm, opc1, opc2));
+            ENCODE_CP_REG(cpnum, is64, s->ns, crn, crm, opc1, opc2));
     if (ri) {
         /* Check access permissions */
         if (!cp_access_ok(s->current_el, ri, isread)) {
@@ -7106,7 +7467,7 @@
              * call in order to handle c15_cpar.
              */
             TCGv_ptr tmpptr;
-            TCGv_i32 tcg_syn;
+            TCGv_i32 tcg_syn, tcg_isread;
             uint32_t syndrome;
 
             /* Note that since we are an implementation which takes an
@@ -7121,19 +7482,19 @@
             case 14:
                 if (is64) {
                     syndrome = syn_cp14_rrt_trap(1, 0xe, opc1, crm, rt, rt2,
-                                                 isread, s->thumb);
+                                                 isread, false);
                 } else {
                     syndrome = syn_cp14_rt_trap(1, 0xe, opc1, opc2, crn, crm,
-                                                rt, isread, s->thumb);
+                                                rt, isread, false);
                 }
                 break;
             case 15:
                 if (is64) {
                     syndrome = syn_cp15_rrt_trap(1, 0xe, opc1, crm, rt, rt2,
-                                                 isread, s->thumb);
+                                                 isread, false);
                 } else {
                     syndrome = syn_cp15_rt_trap(1, 0xe, opc1, opc2, crn, crm,
-                                                rt, isread, s->thumb);
+                                                rt, isread, false);
                 }
                 break;
             default:
@@ -7147,12 +7508,16 @@
                 break;
             }
 
-            gen_set_pc_im(s, s->pc);
+            gen_set_condexec(s);
+            gen_set_pc_im(s, s->pc - 4);
             tmpptr = tcg_const_ptr(ri);
             tcg_syn = tcg_const_i32(syndrome);
-            gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn);
+            tcg_isread = tcg_const_i32(isread);
+            gen_helper_access_check_cp_reg(cpu_env, tmpptr, tcg_syn,
+                                           tcg_isread);
             tcg_temp_free_ptr(tmpptr);
             tcg_temp_free_i32(tcg_syn);
+            tcg_temp_free_i32(tcg_isread);
         }
 
         /* Handle special cases first */
@@ -7170,7 +7535,7 @@
             break;
         }
 
-        if (use_icount && (ri->type & ARM_CP_IO)) {
+        if ((s->tb->cflags & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
             gen_io_start();
         }
 
@@ -7192,11 +7557,11 @@
                     tcg_gen_ld_i64(tmp64, cpu_env, ri->fieldoffset);
                 }
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 store_reg(s, rt, tmp);
                 tcg_gen_shri_i64(tmp64, tmp64, 32);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 store_reg(s, rt2, tmp);
             } else {
@@ -7261,7 +7626,7 @@
             }
         }
 
-        if (use_icount && (ri->type & ARM_CP_IO)) {
+        if ((s->tb->cflags & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
             /* I/O operations must end the TB here (whether read or write) */
             gen_io_end();
             gen_lookup_tb(s);
@@ -7281,12 +7646,16 @@
      */
     if (is64) {
         qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch32 "
-                      "64 bit system register cp:%d opc1: %d crm:%d\n",
-                      isread ? "read" : "write", cpnum, opc1, crm);
+                      "64 bit system register cp:%d opc1: %d crm:%d "
+                      "(%s)\n",
+                      isread ? "read" : "write", cpnum, opc1, crm,
+                      s->ns ? "non-secure" : "secure");
     } else {
         qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch32 "
-                      "system register cp:%d opc1:%d crn:%d crm:%d opc2:%d\n",
-                      isread ? "read" : "write", cpnum, opc1, crn, crm, opc2);
+                      "system register cp:%d opc1:%d crn:%d crm:%d opc2:%d "
+                      "(%s)\n",
+                      isread ? "read" : "write", cpnum, opc1, crn, crm, opc2,
+                      s->ns ? "non-secure" : "secure");
     }
 
     return 1;
@@ -7298,11 +7667,11 @@
 {
     TCGv_i32 tmp;
     tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(tmp, val);
+    tcg_gen_extrl_i64_i32(tmp, val);
     store_reg(s, rlow, tmp);
     tmp = tcg_temp_new_i32();
     tcg_gen_shri_i64(val, val, 32);
-    tcg_gen_trunc_i64_i32(tmp, val);
+    tcg_gen_extrl_i64_i32(tmp, val);
     store_reg(s, rhigh, tmp);
 }
 
@@ -7364,14 +7733,14 @@
 
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld16ua(s, tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld32ua(s, tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7382,7 +7751,7 @@
         TCGv_i32 tmp3 = tcg_temp_new_i32();
 
         tcg_gen_addi_i32(tmp2, addr, 4);
-        gen_aa32_ld32u(tmp3, tmp2, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp3, tmp2, get_mem_index(s));
         tcg_temp_free_i32(tmp2);
         tcg_gen_concat_i32_i64(cpu_exclusive_val, tmp, tmp3);
         store_reg(s, rt2, tmp3);
@@ -7414,8 +7783,8 @@
 {
     TCGv_i32 tmp;
     TCGv_i64 val64, extaddr;
-    int done_label;
-    int fail_label;
+    TCGLabel *done_label;
+    TCGLabel *fail_label;
 
     /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]) {
          [addr] = {Rt};
@@ -7433,14 +7802,14 @@
     tmp = tcg_temp_new_i32();
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7451,7 +7820,7 @@
         TCGv_i32 tmp2 = tcg_temp_new_i32();
         TCGv_i32 tmp3 = tcg_temp_new_i32();
         tcg_gen_addi_i32(tmp2, addr, 4);
-        gen_aa32_ld32u(tmp3, tmp2, get_mem_index(s));
+        gen_aa32_ld32u(s, tmp3, tmp2, get_mem_index(s));
         tcg_temp_free_i32(tmp2);
         tcg_gen_concat_i32_i64(val64, tmp, tmp3);
         tcg_temp_free_i32(tmp3);
@@ -7466,14 +7835,14 @@
     tmp = load_reg(s, rt);
     switch (size) {
     case 0:
-        gen_aa32_st8(tmp, addr, get_mem_index(s));
+        gen_aa32_st8(s, tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_st16(tmp, addr, get_mem_index(s));
+        gen_aa32_st16(s, tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_st32(tmp, addr, get_mem_index(s));
+        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7482,7 +7851,7 @@
     if (size == 3) {
         tcg_gen_addi_i32(addr, addr, 4);
         tmp = load_reg(s, rt2);
-        gen_aa32_st32(tmp, addr, get_mem_index(s));
+        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
         tcg_temp_free_i32(tmp);
     }
     tcg_gen_movi_i32(cpu_R[rd], 0);
@@ -7507,8 +7876,68 @@
                     uint32_t mode, uint32_t amode, bool writeback)
 {
     int32_t offset;
-    TCGv_i32 addr = tcg_temp_new_i32();
-    TCGv_i32 tmp = tcg_const_i32(mode);
+    TCGv_i32 addr, tmp;
+    bool undef = false;
+
+    /* SRS is:
+     * - trapped to EL3 if EL3 is AArch64 and we are at Secure EL1
+     *   and specified mode is monitor mode
+     * - UNDEFINED in Hyp mode
+     * - UNPREDICTABLE in User or System mode
+     * - UNPREDICTABLE if the specified mode is:
+     * -- not implemented
+     * -- not a valid mode number
+     * -- a mode that's at a higher exception level
+     * -- Monitor, if we are Non-secure
+     * For the UNPREDICTABLE cases we choose to UNDEF.
+     */
+    if (s->current_el == 1 && !s->ns && mode == ARM_CPU_MODE_MON) {
+        gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(), 3);
+        return;
+    }
+
+    if (s->current_el == 0 || s->current_el == 2) {
+        undef = true;
+    }
+
+    switch (mode) {
+    case ARM_CPU_MODE_USR:
+    case ARM_CPU_MODE_FIQ:
+    case ARM_CPU_MODE_IRQ:
+    case ARM_CPU_MODE_SVC:
+    case ARM_CPU_MODE_ABT:
+    case ARM_CPU_MODE_UND:
+    case ARM_CPU_MODE_SYS:
+        break;
+    case ARM_CPU_MODE_HYP:
+        if (s->current_el == 1 || !arm_dc_feature(s, ARM_FEATURE_EL2)) {
+            undef = true;
+        }
+        break;
+    case ARM_CPU_MODE_MON:
+        /* No need to check specifically for "are we non-secure" because
+         * we've already made EL0 UNDEF and handled the trap for S-EL1;
+         * so if this isn't EL3 then we must be non-secure.
+         */
+        if (s->current_el != 3) {
+            undef = true;
+        }
+        break;
+    default:
+        undef = true;
+    }
+
+    if (undef) {
+        gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(),
+                           default_exception_el(s));
+        return;
+    }
+
+    addr = tcg_temp_new_i32();
+    tmp = tcg_const_i32(mode);
+    /* get_r13_banked() will raise an exception if called from System mode */
+    gen_set_condexec(s);
+    gen_set_pc_im(s, s->pc - 4);
     gen_helper_get_r13_banked(addr, cpu_env, tmp);
     tcg_temp_free_i32(tmp);
     switch (amode) {
@@ -7529,11 +7958,11 @@
     }
     tcg_gen_addi_i32(addr, addr, offset);
     tmp = load_reg(s, 14);
-    gen_aa32_st32(tmp, addr, get_mem_index(s));
+    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
     tcg_temp_free_i32(tmp);
     tmp = load_cpu_field(spsr);
     tcg_gen_addi_i32(addr, addr, 4);
-    gen_aa32_st32(tmp, addr, get_mem_index(s));
+    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
     tcg_temp_free_i32(tmp);
     if (writeback) {
         switch (amode) {
@@ -7558,6 +7987,7 @@
         tcg_temp_free_i32(tmp);
     }
     tcg_temp_free_i32(addr);
+    s->is_jmp = DISAS_UPDATE;
 }
 
 static void disas_arm_insn(DisasContext *s, unsigned int insn)
@@ -7639,10 +8069,9 @@
         if ((insn & 0x0ffffdff) == 0x01010000) {
             ARCH(6);
             /* setend */
-            if (((insn >> 9) & 1) != s->bswap_code) {
-                /* Dynamic endianness switching not implemented. */
-                qemu_log_mask(LOG_UNIMP, "arm: unimplemented setend\n");
-                goto illegal_op;
+            if (((insn >> 9) & 1) != !!(s->be_data == MO_BE)) {
+                gen_helper_setend(cpu_env);
+                s->is_jmp = DISAS_UPDATE;
             }
             return;
         } else if ((insn & 0x0fffff00) == 0x057ff000) {
@@ -7653,18 +8082,21 @@
                 return;
             case 4: /* dsb */
             case 5: /* dmb */
-            case 6: /* isb */
                 ARCH(7);
                 /* We don't emulate caches so these are a no-op.  */
                 return;
+            case 6: /* isb */
+                /* We need to break the TB after this insn to execute
+                 * self-modifying code correctly and also to take
+                 * any pending interrupts immediately.
+                 */
+                gen_lookup_tb(s);
+                return;
             default:
                 goto illegal_op;
             }
         } else if ((insn & 0x0e5fffe0) == 0x084d0500) {
             /* srs */
-            if (IS_USER(s)) {
-                goto illegal_op;
-            }
             ARCH(6);
             gen_srs(s, (insn & 0x1f), (insn >> 23) & 3, insn & (1 << 21));
             return;
@@ -7688,10 +8120,10 @@
                 tcg_gen_addi_i32(addr, addr, offset);
             /* Load PC into tmp and CPSR into tmp2.  */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             tcg_gen_addi_i32(addr, addr, 4);
             tmp2 = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp2, addr, get_mem_index(s));
             if (insn & (1 << 21)) {
                 /* Base writeback.  */
                 switch (i) {
@@ -7816,7 +8248,26 @@
         sh = (insn >> 4) & 0xf;
         rm = insn & 0xf;
         switch (sh) {
-        case 0x0: /* move program status register */
+        case 0x0: /* MSR, MRS */
+            if (insn & (1 << 9)) {
+                /* MSR (banked) and MRS (banked) */
+                int sysm = extract32(insn, 16, 4) |
+                    (extract32(insn, 8, 1) << 4);
+                int r = extract32(insn, 22, 1);
+
+                if (op1 & 1) {
+                    /* MSR (banked) */
+                    gen_msr_banked(s, r, sysm, rm);
+                } else {
+                    /* MRS (banked) */
+                    int rd = extract32(insn, 12, 4);
+
+                    gen_mrs_banked(s, r, sysm, rd);
+                }
+                break;
+            }
+
+            /* MSR, MRS (for PSRs) */
             if (op1 & 1) {
                 /* PSR = reg */
                 tmp = load_reg(s, rm);
@@ -7934,7 +8385,8 @@
                 /* bkpt */
                 ARCH(5);
                 gen_exception_insn(s, 4, EXCP_BKPT,
-                                   syn_aa32_bkpt(imm16, false));
+                                   syn_aa32_bkpt(imm16, false),
+                                   default_exception_el(s));
                 break;
             case 2:
                 /* Hypervisor call (v7) */
@@ -7976,7 +8428,7 @@
                 tmp64 = gen_muls_i64_i32(tmp, tmp2);
                 tcg_gen_shri_i64(tmp64, tmp64, 16);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 if ((sh & 2) == 0) {
                     tmp2 = load_reg(s, rn);
@@ -8306,13 +8758,16 @@
                                 tmp = tcg_temp_new_i32();
                                 switch (op1) {
                                 case 0: /* lda */
-                                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                                    gen_aa32_ld32u(s, tmp, addr,
+                                                   get_mem_index(s));
                                     break;
                                 case 2: /* ldab */
-                                    gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                                    gen_aa32_ld8u(s, tmp, addr,
+                                                  get_mem_index(s));
                                     break;
                                 case 3: /* ldah */
-                                    gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                                    gen_aa32_ld16u(s, tmp, addr,
+                                                   get_mem_index(s));
                                     break;
                                 default:
                                     abort();
@@ -8323,13 +8778,16 @@
                                 tmp = load_reg(s, rm);
                                 switch (op1) {
                                 case 0: /* stl */
-                                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                                    gen_aa32_st32(s, tmp, addr,
+                                                  get_mem_index(s));
                                     break;
                                 case 2: /* stlb */
-                                    gen_aa32_st8(tmp, addr, get_mem_index(s));
+                                    gen_aa32_st8(s, tmp, addr,
+                                                 get_mem_index(s));
                                     break;
                                 case 3: /* stlh */
-                                    gen_aa32_st16(tmp, addr, get_mem_index(s));
+                                    gen_aa32_st16(s, tmp, addr,
+                                                  get_mem_index(s));
                                     break;
                                 default:
                                     abort();
@@ -8384,11 +8842,11 @@
                         tmp = load_reg(s, rm);
                         tmp2 = tcg_temp_new_i32();
                         if (insn & (1 << 22)) {
-                            gen_aa32_ld8u(tmp2, addr, get_mem_index(s));
-                            gen_aa32_st8(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld8u(s, tmp2, addr, get_mem_index(s));
+                            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                         } else {
-                            gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp2, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         }
                         tcg_temp_free_i32(tmp);
                         tcg_temp_free_i32(addr);
@@ -8397,65 +8855,73 @@
                 }
             } else {
                 int address_offset;
-                int load;
+                bool load = insn & (1 << 20);
+                bool doubleword = false;
                 /* Misc load/store */
                 rn = (insn >> 16) & 0xf;
                 rd = (insn >> 12) & 0xf;
+
+                if (!load && (sh & 2)) {
+                    /* doubleword */
+                    ARCH(5TE);
+                    if (rd & 1) {
+                        /* UNPREDICTABLE; we choose to UNDEF */
+                        goto illegal_op;
+                    }
+                    load = (sh & 1) == 0;
+                    doubleword = true;
+                }
+
                 addr = load_reg(s, rn);
                 if (insn & (1 << 24))
                     gen_add_datah_offset(s, insn, 0, addr);
                 address_offset = 0;
-                if (insn & (1 << 20)) {
-                    /* load */
-                    tmp = tcg_temp_new_i32();
-                    switch(sh) {
-                    case 1:
-                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
-                        break;
-                    case 2:
-                        gen_aa32_ld8s(tmp, addr, get_mem_index(s));
-                        break;
-                    default:
-                    case 3:
-                        gen_aa32_ld16s(tmp, addr, get_mem_index(s));
-                        break;
-                    }
-                    load = 1;
-                } else if (sh & 2) {
-                    ARCH(5TE);
-                    /* doubleword */
-                    if (sh & 1) {
+
+                if (doubleword) {
+                    if (!load) {
                         /* store */
                         tmp = load_reg(s, rd);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                         tcg_gen_addi_i32(addr, addr, 4);
                         tmp = load_reg(s, rd + 1);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
-                        load = 0;
                     } else {
                         /* load */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         store_reg(s, rd, tmp);
                         tcg_gen_addi_i32(addr, addr, 4);
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         rd++;
-                        load = 1;
                     }
                     address_offset = -4;
+                } else if (load) {
+                    /* load */
+                    tmp = tcg_temp_new_i32();
+                    switch (sh) {
+                    case 1:
+                        gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
+                        break;
+                    case 2:
+                        gen_aa32_ld8s(s, tmp, addr, get_mem_index(s));
+                        break;
+                    default:
+                    case 3:
+                        gen_aa32_ld16s(s, tmp, addr, get_mem_index(s));
+                        break;
+                    }
                 } else {
                     /* store */
                     tmp = load_reg(s, rd);
-                    gen_aa32_st16(tmp, addr, get_mem_index(s));
+                    gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
-                    load = 0;
                 }
                 /* Perform base writeback before the loaded value to
                    ensure correct behavior with overlapping index registers.
-                   ldrd with base writeback is is undefined if the
+                   ldrd with base writeback is undefined if the
                    destination and index registers overlap.  */
                 if (!(insn & (1 << 24))) {
                     gen_add_datah_offset(s, insn, address_offset, addr);
@@ -8634,7 +9100,7 @@
                         }
                         tcg_gen_shri_i64(tmp64, tmp64, 32);
                         tmp = tcg_temp_new_i32();
-                        tcg_gen_trunc_i64_i32(tmp, tmp64);
+                        tcg_gen_extrl_i64_i32(tmp, tmp64);
                         tcg_temp_free_i64(tmp64);
                         store_reg(s, rn, tmp);
                         break;
@@ -8735,6 +9201,10 @@
                         ARCH(6T2);
                         shift = (insn >> 7) & 0x1f;
                         i = (insn >> 16) & 0x1f;
+                        if (i < shift) {
+                            /* UNPREDICTABLE; we choose to UNDEF */
+                            goto illegal_op;
+                        }
                         i = i + 1 - shift;
                         if (rm == 15) {
                             tmp = tcg_temp_new_i32();
@@ -8789,7 +9259,7 @@
             tmp2 = load_reg(s, rn);
             if ((insn & 0x01200000) == 0x00200000) {
                 /* ldrt/strt */
-                i = MMU_USER_IDX;
+                i = get_a32_user_mem_index(s);
             } else {
                 i = get_mem_index(s);
             }
@@ -8799,17 +9269,17 @@
                 /* load */
                 tmp = tcg_temp_new_i32();
                 if (insn & (1 << 22)) {
-                    gen_aa32_ld8u(tmp, tmp2, i);
+                    gen_aa32_ld8u(s, tmp, tmp2, i);
                 } else {
-                    gen_aa32_ld32u(tmp, tmp2, i);
+                    gen_aa32_ld32u(s, tmp, tmp2, i);
                 }
             } else {
                 /* store */
                 tmp = load_reg(s, rd);
                 if (insn & (1 << 22)) {
-                    gen_aa32_st8(tmp, tmp2, i);
+                    gen_aa32_st8(s, tmp, tmp2, i);
                 } else {
-                    gen_aa32_st32(tmp, tmp2, i);
+                    gen_aa32_st32(s, tmp, tmp2, i);
                 }
                 tcg_temp_free_i32(tmp);
             }
@@ -8829,17 +9299,23 @@
         case 0x08:
         case 0x09:
             {
-                int j, n, user, loaded_base;
+                int j, n, loaded_base;
+                bool exc_return = false;
+                bool is_load = extract32(insn, 20, 1);
+                bool user = false;
                 TCGv_i32 loaded_var;
                 /* load/store multiple words */
                 /* XXX: store correct base if write back */
-                user = 0;
                 if (insn & (1 << 22)) {
+                    /* LDM (user), LDM (exception return) and STM (user) */
                     if (IS_USER(s))
                         goto illegal_op; /* only usable in supervisor mode */
 
-                    if ((insn & (1 << 15)) == 0)
-                        user = 1;
+                    if (is_load && extract32(insn, 15, 1)) {
+                        exc_return = true;
+                    } else {
+                        user = true;
+                    }
                 }
                 rn = (insn >> 16) & 0xf;
                 addr = load_reg(s, rn);
@@ -8873,10 +9349,10 @@
                 j = 0;
                 for(i=0;i<16;i++) {
                     if (insn & (1 << i)) {
-                        if (insn & (1 << 20)) {
+                        if (is_load) {
                             /* load */
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                             if (user) {
                                 tmp2 = tcg_const_i32(i);
                                 gen_helper_set_user_reg(cpu_env, tmp2, tmp);
@@ -8903,7 +9379,7 @@
                             } else {
                                 tmp = load_reg(s, i);
                             }
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                         }
                         j++;
@@ -8938,12 +9414,12 @@
                 if (loaded_base) {
                     store_reg(s, rn, loaded_var);
                 }
-                if ((insn & (1 << 22)) && !user) {
+                if (exc_return) {
                     /* Restore CPSR from SPSR.  */
                     tmp = load_cpu_field(spsr);
-                    gen_set_cpsr(tmp, CPSR_ERET_MASK);
+                    gen_helper_cpsr_write_eret(cpu_env, tmp);
                     tcg_temp_free_i32(tmp);
-                    s->is_jmp = DISAS_UPDATE;
+                    s->is_jmp = DISAS_JUMP;
                 }
             }
             break;
@@ -8985,7 +9461,8 @@
             break;
         default:
         illegal_op:
-            gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized());
+            gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(),
+                               default_exception_el(s));
             break;
         }
     }
@@ -9132,7 +9609,7 @@
         /* Fall through to 32-bit decode.  */
     }
 
-    insn = arm_lduw_code(env, s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->sctlr_b);
     s->pc += 2;
     insn |= (uint32_t)insn_hw1 << 16;
 
@@ -9169,20 +9646,20 @@
                 if (insn & (1 << 20)) {
                     /* ldrd */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     store_reg(s, rs, tmp);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     store_reg(s, rd, tmp);
                 } else {
                     /* strd */
                     tmp = load_reg(s, rs);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = load_reg(s, rd);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 if (insn & (1 << 21)) {
@@ -9220,11 +9697,11 @@
                     tcg_gen_add_i32(addr, addr, tmp);
                     tcg_temp_free_i32(tmp);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                 } else { /* tbb */
                     tcg_temp_free_i32(tmp);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                 }
                 tcg_temp_free_i32(addr);
                 tcg_gen_shli_i32(tmp, tmp, 1);
@@ -9261,13 +9738,13 @@
                         tmp = tcg_temp_new_i32();
                         switch (op) {
                         case 0: /* ldab */
-                            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
                             break;
                         case 1: /* ldah */
-                            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
                             break;
                         case 2: /* lda */
-                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                             break;
                         default:
                             abort();
@@ -9277,13 +9754,13 @@
                         tmp = load_reg(s, rs);
                         switch (op) {
                         case 0: /* stlb */
-                            gen_aa32_st8(tmp, addr, get_mem_index(s));
+                            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
                             break;
                         case 1: /* stlh */
-                            gen_aa32_st16(tmp, addr, get_mem_index(s));
+                            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
                             break;
                         case 2: /* stl */
-                            gen_aa32_st32(tmp, addr, get_mem_index(s));
+                            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                             break;
                         default:
                             abort();
@@ -9311,10 +9788,10 @@
                         tcg_gen_addi_i32(addr, addr, -8);
                     /* Load PC into tmp and CPSR into tmp2.  */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp2 = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp2, addr, get_mem_index(s));
                     if (insn & (1 << 21)) {
                         /* Base writeback.  */
                         if (insn & (1 << 24)) {
@@ -9353,7 +9830,7 @@
                     if (insn & (1 << 20)) {
                         /* Load.  */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         if (i == 15) {
                             gen_bx(s, tmp);
                         } else if (i == rn) {
@@ -9365,7 +9842,7 @@
                     } else {
                         /* Store.  */
                         tmp = load_reg(s, i);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     }
                     tcg_gen_addi_i32(addr, addr, 4);
@@ -9392,6 +9869,9 @@
 
         op = (insn >> 21) & 0xf;
         if (op == 6) {
+            if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                goto illegal_op;
+            }
             /* Halfword pack.  */
             tmp = load_reg(s, rn);
             tmp2 = load_reg(s, rm);
@@ -9456,6 +9936,27 @@
             store_reg_bx(s, rd, tmp);
             break;
         case 1: /* Sign/zero extend.  */
+            op = (insn >> 20) & 7;
+            switch (op) {
+            case 0: /* SXTAH, SXTH */
+            case 1: /* UXTAH, UXTH */
+            case 4: /* SXTAB, SXTB */
+            case 5: /* UXTAB, UXTB */
+                break;
+            case 2: /* SXTAB16, SXTB16 */
+            case 3: /* UXTAB16, UXTB16 */
+                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                    goto illegal_op;
+                }
+                break;
+            default:
+                goto illegal_op;
+            }
+            if (rn != 15) {
+                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                    goto illegal_op;
+                }
+            }
             tmp = load_reg(s, rm);
             shift = (insn >> 4) & 3;
             /* ??? In many cases it's not necessary to do a
@@ -9470,7 +9971,8 @@
             case 3: gen_uxtb16(tmp); break;
             case 4: gen_sxtb(tmp);   break;
             case 5: gen_uxtb(tmp);   break;
-            default: goto illegal_op;
+            default:
+                g_assert_not_reached();
             }
             if (rn != 15) {
                 tmp2 = load_reg(s, rn);
@@ -9484,6 +9986,9 @@
             store_reg(s, rd, tmp);
             break;
         case 2: /* SIMD add/subtract.  */
+            if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                goto illegal_op;
+            }
             op = (insn >> 20) & 7;
             shift = (insn >> 4) & 7;
             if ((op & 3) == 3 || (shift & 3) == 3)
@@ -9498,6 +10003,9 @@
             op = ((insn >> 17) & 0x38) | ((insn >> 4) & 7);
             if (op < 4) {
                 /* Saturating add/subtract.  */
+                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                    goto illegal_op;
+                }
                 tmp = load_reg(s, rn);
                 tmp2 = load_reg(s, rm);
                 if (op & 1)
@@ -9508,6 +10016,31 @@
                     gen_helper_add_saturate(tmp, cpu_env, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
             } else {
+                switch (op) {
+                case 0x0a: /* rbit */
+                case 0x08: /* rev */
+                case 0x09: /* rev16 */
+                case 0x0b: /* revsh */
+                case 0x18: /* clz */
+                    break;
+                case 0x10: /* sel */
+                    if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                        goto illegal_op;
+                    }
+                    break;
+                case 0x20: /* crc32/crc32c */
+                case 0x21:
+                case 0x22:
+                case 0x28:
+                case 0x29:
+                case 0x2a:
+                    if (!arm_dc_feature(s, ARM_FEATURE_CRC)) {
+                        goto illegal_op;
+                    }
+                    break;
+                default:
+                    goto illegal_op;
+                }
                 tmp = load_reg(s, rn);
                 switch (op) {
                 case 0x0a: /* rbit */
@@ -9544,10 +10077,6 @@
                     uint32_t sz = op & 0x3;
                     uint32_t c = op & 0x8;
 
-                    if (!arm_dc_feature(s, ARM_FEATURE_CRC)) {
-                        goto illegal_op;
-                    }
-
                     tmp2 = load_reg(s, rm);
                     if (sz == 0) {
                         tcg_gen_andi_i32(tmp2, tmp2, 0xff);
@@ -9565,12 +10094,26 @@
                     break;
                 }
                 default:
-                    goto illegal_op;
+                    g_assert_not_reached();
                 }
             }
             store_reg(s, rd, tmp);
             break;
         case 4: case 5: /* 32-bit multiply.  Sum of absolute differences.  */
+            switch ((insn >> 20) & 7) {
+            case 0: /* 32 x 32 -> 32 */
+            case 7: /* Unsigned sum of absolute differences.  */
+                break;
+            case 1: /* 16 x 16 -> 32 */
+            case 2: /* Dual multiply add.  */
+            case 3: /* 32 * 16 -> 32msb */
+            case 4: /* Dual multiply subtract.  */
+            case 5: case 6: /* 32 * 32 -> 32msb (SMMUL, SMMLA, SMMLS) */
+                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                    goto illegal_op;
+                }
+                break;
+            }
             op = (insn >> 4) & 0xf;
             tmp = load_reg(s, rn);
             tmp2 = load_reg(s, rm);
@@ -9627,7 +10170,7 @@
                 tmp64 = gen_muls_i64_i32(tmp, tmp2);
                 tcg_gen_shri_i64(tmp64, tmp64, 16);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 if (rs != 15)
                   {
@@ -9651,7 +10194,7 @@
                 }
                 tcg_gen_shri_i64(tmp64, tmp64, 32);
                 tmp = tcg_temp_new_i32();
-                tcg_gen_trunc_i64_i32(tmp, tmp64);
+                tcg_gen_extrl_i64_i32(tmp, tmp64);
                 tcg_temp_free_i64(tmp64);
                 break;
             case 7: /* Unsigned sum of absolute differences.  */
@@ -9683,6 +10226,11 @@
                 store_reg(s, rd, tmp);
             } else if ((op & 0xe) == 0xc) {
                 /* Dual multiply accumulate long.  */
+                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                    tcg_temp_free_i32(tmp);
+                    tcg_temp_free_i32(tmp2);
+                    goto illegal_op;
+                }
                 if (op & 1)
                     gen_swap_half(tmp2);
                 gen_smul_dual(tmp, tmp2);
@@ -9706,6 +10254,11 @@
                 } else {
                     if (op & 8) {
                         /* smlalxy */
+                        if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                            tcg_temp_free_i32(tmp2);
+                            tcg_temp_free_i32(tmp);
+                            goto illegal_op;
+                        }
                         gen_mulxy(tmp, tmp2, op & 2, op & 1);
                         tcg_temp_free_i32(tmp2);
                         tmp64 = tcg_temp_new_i64();
@@ -9718,6 +10271,10 @@
                 }
                 if (op & 4) {
                     /* umaal */
+                    if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                        tcg_temp_free_i64(tmp64);
+                        goto illegal_op;
+                    }
                     gen_addq_lo(s, tmp64, rs);
                     gen_addq_lo(s, tmp64, rd);
                 } else if (op & 0x40) {
@@ -9821,6 +10378,18 @@
                         if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             goto illegal_op;
                         }
+
+                        if (extract32(insn, 5, 1)) {
+                            /* MSR (banked) */
+                            int sysm = extract32(insn, 8, 4) |
+                                (extract32(insn, 4, 1) << 4);
+                            int r = op & 1;
+
+                            gen_msr_banked(s, r, sysm, rm);
+                            break;
+                        }
+
+                        /* MSR (for PSRs) */
                         tmp = load_reg(s, rn);
                         if (gen_set_psr(s,
                               msr_mask(s, (insn >> 8) & 0xf, op == 1),
@@ -9863,9 +10432,16 @@
                             break;
                         case 4: /* dsb */
                         case 5: /* dmb */
-                        case 6: /* isb */
                             /* These execute as NOPs.  */
                             break;
+                        case 6: /* isb */
+                            /* We need to break the TB after this insn
+                             * to execute self-modifying code correctly
+                             * and also to take any pending interrupts
+                             * immediately.
+                             */
+                            gen_lookup_tb(s);
+                            break;
                         default:
                             goto illegal_op;
                         }
@@ -9886,7 +10462,17 @@
                         tcg_gen_subi_i32(tmp, tmp, insn & 0xff);
                         gen_exception_return(s, tmp);
                         break;
-                    case 6: /* mrs cpsr.  */
+                    case 6: /* MRS */
+                        if (extract32(insn, 5, 1)) {
+                            /* MRS (banked) */
+                            int sysm = extract32(insn, 16, 4) |
+                                (extract32(insn, 4, 1) << 4);
+
+                            gen_mrs_banked(s, 0, sysm, rd);
+                            break;
+                        }
+
+                        /* mrs cpsr */
                         tmp = tcg_temp_new_i32();
                         if (arm_dc_feature(s, ARM_FEATURE_M)) {
                             addr = tcg_const_i32(insn & 0xff);
@@ -9897,7 +10483,17 @@
                         }
                         store_reg(s, rd, tmp);
                         break;
-                    case 7: /* mrs spsr.  */
+                    case 7: /* MRS */
+                        if (extract32(insn, 5, 1)) {
+                            /* MRS (banked) */
+                            int sysm = extract32(insn, 16, 4) |
+                                (extract32(insn, 4, 1) << 4);
+
+                            gen_mrs_banked(s, 1, sysm, rd);
+                            break;
+                        }
+
+                        /* mrs spsr.  */
                         /* Not accessible in user mode.  */
                         if (IS_USER(s) || arm_dc_feature(s, ARM_FEATURE_M)) {
                             goto illegal_op;
@@ -9982,16 +10578,28 @@
                         tmp2 = tcg_const_i32(imm);
                         if (op & 4) {
                             /* Unsigned.  */
-                            if ((op & 1) && shift == 0)
+                            if ((op & 1) && shift == 0) {
+                                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                                    tcg_temp_free_i32(tmp);
+                                    tcg_temp_free_i32(tmp2);
+                                    goto illegal_op;
+                                }
                                 gen_helper_usat16(tmp, cpu_env, tmp, tmp2);
-                            else
+                            } else {
                                 gen_helper_usat(tmp, cpu_env, tmp, tmp2);
+                            }
                         } else {
                             /* Signed.  */
-                            if ((op & 1) && shift == 0)
+                            if ((op & 1) && shift == 0) {
+                                if (!arm_dc_feature(s, ARM_FEATURE_THUMB_DSP)) {
+                                    tcg_temp_free_i32(tmp);
+                                    tcg_temp_free_i32(tmp2);
+                                    goto illegal_op;
+                                }
                                 gen_helper_ssat16(tmp, cpu_env, tmp, tmp2);
-                            else
+                            } else {
                                 gen_helper_ssat(tmp, cpu_env, tmp, tmp2);
+                            }
                         }
                         tcg_temp_free_i32(tmp2);
                         break;
@@ -10169,7 +10777,7 @@
                     break;
                 case 0xe: /* User privilege.  */
                     tcg_gen_addi_i32(addr, addr, imm);
-                    memidx = MMU_USER_IDX;
+                    memidx = get_a32_user_mem_index(s);
                     break;
                 case 0x9: /* Post-decrement.  */
                     imm = -imm;
@@ -10196,19 +10804,19 @@
             tmp = tcg_temp_new_i32();
             switch (op) {
             case 0:
-                gen_aa32_ld8u(tmp, addr, memidx);
+                gen_aa32_ld8u(s, tmp, addr, memidx);
                 break;
             case 4:
-                gen_aa32_ld8s(tmp, addr, memidx);
+                gen_aa32_ld8s(s, tmp, addr, memidx);
                 break;
             case 1:
-                gen_aa32_ld16u(tmp, addr, memidx);
+                gen_aa32_ld16u(s, tmp, addr, memidx);
                 break;
             case 5:
-                gen_aa32_ld16s(tmp, addr, memidx);
+                gen_aa32_ld16s(s, tmp, addr, memidx);
                 break;
             case 2:
-                gen_aa32_ld32u(tmp, addr, memidx);
+                gen_aa32_ld32u(s, tmp, addr, memidx);
                 break;
             default:
                 tcg_temp_free_i32(tmp);
@@ -10225,13 +10833,13 @@
             tmp = load_reg(s, rs);
             switch (op) {
             case 0:
-                gen_aa32_st8(tmp, addr, memidx);
+                gen_aa32_st8(s, tmp, addr, memidx);
                 break;
             case 1:
-                gen_aa32_st16(tmp, addr, memidx);
+                gen_aa32_st16(s, tmp, addr, memidx);
                 break;
             case 2:
-                gen_aa32_st32(tmp, addr, memidx);
+                gen_aa32_st32(s, tmp, addr, memidx);
                 break;
             default:
                 tcg_temp_free_i32(tmp);
@@ -10275,7 +10883,7 @@
         }
     }
 
-    insn = arm_lduw_code(env, s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->sctlr_b);
     s->pc += 2;
 
     switch (insn >> 12) {
@@ -10368,7 +10976,7 @@
             addr = tcg_temp_new_i32();
             tcg_gen_movi_i32(addr, val);
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(addr);
             store_reg(s, rd, tmp);
             break;
@@ -10571,28 +11179,28 @@
 
         switch (op) {
         case 0: /* str */
-            gen_aa32_st32(tmp, addr, get_mem_index(s));
+            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             break;
         case 1: /* strh */
-            gen_aa32_st16(tmp, addr, get_mem_index(s));
+            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
             break;
         case 2: /* strb */
-            gen_aa32_st8(tmp, addr, get_mem_index(s));
+            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
             break;
         case 3: /* ldrsb */
-            gen_aa32_ld8s(tmp, addr, get_mem_index(s));
+            gen_aa32_ld8s(s, tmp, addr, get_mem_index(s));
             break;
         case 4: /* ldr */
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             break;
         case 5: /* ldrh */
-            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
             break;
         case 6: /* ldrb */
-            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
             break;
         case 7: /* ldrsh */
-            gen_aa32_ld16s(tmp, addr, get_mem_index(s));
+            gen_aa32_ld16s(s, tmp, addr, get_mem_index(s));
             break;
         }
         if (op >= 3) { /* load */
@@ -10614,12 +11222,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st32(tmp, addr, get_mem_index(s));
+            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10636,12 +11244,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld8u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st8(tmp, addr, get_mem_index(s));
+            gen_aa32_st8(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10658,12 +11266,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld16u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st16(tmp, addr, get_mem_index(s));
+            gen_aa32_st16(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10679,12 +11287,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+            gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st32(tmp, addr, get_mem_index(s));
+            gen_aa32_st32(s, tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10752,12 +11360,12 @@
                     if (insn & (1 << 11)) {
                         /* pop */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                        gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                         store_reg(s, i, tmp);
                     } else {
                         /* push */
                         tmp = load_reg(s, i);
-                        gen_aa32_st32(tmp, addr, get_mem_index(s));
+                        gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     }
                     /* advance to the next address.  */
@@ -10769,13 +11377,13 @@
                 if (insn & (1 << 11)) {
                     /* pop pc */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     /* don't set the pc until the rest of the instruction
                        has completed */
                 } else {
                     /* push lr */
                     tmp = load_reg(s, 14);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 tcg_gen_addi_i32(addr, addr, 4);
@@ -10822,7 +11430,8 @@
         {
             int imm8 = extract32(insn, 0, 8);
             ARCH(5);
-            gen_exception_insn(s, 2, EXCP_BKPT, syn_aa32_bkpt(imm8, true));
+            gen_exception_insn(s, 2, EXCP_BKPT, syn_aa32_bkpt(imm8, true),
+                               default_exception_el(s));
             break;
         }
 
@@ -10845,10 +11454,9 @@
             case 2:
                 /* setend */
                 ARCH(6);
-                if (((insn >> 3) & 1) != s->bswap_code) {
-                    /* Dynamic endianness switching not implemented. */
-                    qemu_log_mask(LOG_UNIMP, "arm: unimplemented setend\n");
-                    goto illegal_op;
+                if (((insn >> 3) & 1) != !!(s->be_data == MO_BE)) {
+                    gen_helper_setend(cpu_env);
+                    s->is_jmp = DISAS_UPDATE;
                 }
                 break;
             case 3:
@@ -10904,7 +11512,7 @@
                 if (insn & (1 << 11)) {
                     /* load */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
+                    gen_aa32_ld32u(s, tmp, addr, get_mem_index(s));
                     if (i == rn) {
                         loaded_var = tmp;
                     } else {
@@ -10913,7 +11521,7 @@
                 } else {
                     /* store */
                     tmp = load_reg(s, i);
-                    gen_aa32_st32(tmp, addr, get_mem_index(s));
+                    gen_aa32_st32(s, tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 /* advance to the next address */
@@ -10977,29 +11585,55 @@
     }
     return;
 undef32:
-    gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized());
+    gen_exception_insn(s, 4, EXCP_UDEF, syn_uncategorized(),
+                       default_exception_el(s));
     return;
 illegal_op:
 undef:
-    gen_exception_insn(s, 2, EXCP_UDEF, syn_uncategorized());
+    gen_exception_insn(s, 2, EXCP_UDEF, syn_uncategorized(),
+                       default_exception_el(s));
 }
 
-/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(ARMCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+static bool insn_crosses_page(CPUARMState *env, DisasContext *s)
 {
+    /* Return true if the insn at dc->pc might cross a page boundary.
+     * (False positives are OK, false negatives are not.)
+     */
+    uint16_t insn;
+
+    if ((s->pc & 3) == 0) {
+        /* At a 4-aligned address we can't be crossing a page */
+        return false;
+    }
+
+    /* This must be a Thumb insn */
+    insn = arm_lduw_code(env, s->pc, s->sctlr_b);
+
+    if ((insn >> 11) >= 0x1d) {
+        /* Top five bits 0b11101 / 0b11110 / 0b11111 : this is the
+         * First half of a 32-bit Thumb insn. Thumb-1 cores might
+         * end up actually treating this as two 16-bit insns (see the
+         * code at the start of disas_thumb2_insn()) but we don't bother
+         * to check for that as it is unlikely, and false positives here
+         * are harmless.
+         */
+        return true;
+    }
+    /* Definitely a 16-bit insn, can't be crossing a page. */
+    return false;
+}
+
+/* generate intermediate code for basic block 'tb'.  */
+void gen_intermediate_code(CPUARMState *env, TranslationBlock *tb)
+{
+    ARMCPU *cpu = arm_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUARMState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     target_ulong next_page_start;
     int num_insns;
     int max_insns;
+    bool end_of_page;
 
     /* generate intermediate code */
 
@@ -11007,7 +11641,7 @@
      * the A32/T32 complexity to do with conditional execution/IT blocks/etc.
      */
     if (ARM_TBFLAG_AARCH64_STATE(tb->flags)) {
-        gen_intermediate_code_internal_a64(cpu, tb, search_pc);
+        gen_intermediate_code_a64(cpu, tb);
         return;
     }
 
@@ -11021,20 +11655,28 @@
     dc->condjmp = 0;
 
     dc->aarch64 = 0;
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.
+     */
+    dc->secure_routed_to_el3 = arm_feature(env, ARM_FEATURE_EL3) &&
+                               !arm_el_is_aa64(env, 3);
     dc->thumb = ARM_TBFLAG_THUMB(tb->flags);
-    dc->bswap_code = ARM_TBFLAG_BSWAP_CODE(tb->flags);
+    dc->sctlr_b = ARM_TBFLAG_SCTLR_B(tb->flags);
+    dc->be_data = ARM_TBFLAG_BE_DATA(tb->flags) ? MO_BE : MO_LE;
     dc->condexec_mask = (ARM_TBFLAG_CONDEXEC(tb->flags) & 0xf) << 1;
     dc->condexec_cond = ARM_TBFLAG_CONDEXEC(tb->flags) >> 4;
+    dc->mmu_idx = ARM_TBFLAG_MMUIDX(tb->flags);
+    dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx);
 #if !defined(CONFIG_USER_ONLY)
-    dc->user = (ARM_TBFLAG_PRIV(tb->flags) == 0);
+    dc->user = (dc->current_el == 0);
 #endif
-    dc->cpacr_fpen = ARM_TBFLAG_CPACR_FPEN(tb->flags);
+    dc->ns = ARM_TBFLAG_NS(tb->flags);
+    dc->fp_excp_el = ARM_TBFLAG_FPEXC_EL(tb->flags);
     dc->vfp_enabled = ARM_TBFLAG_VFPEN(tb->flags);
     dc->vec_len = ARM_TBFLAG_VECLEN(tb->flags);
     dc->vec_stride = ARM_TBFLAG_VECSTRIDE(tb->flags);
     dc->c15_cpar = ARM_TBFLAG_XSCALE_CPAR(tb->flags);
     dc->cp_regs = cpu->cp_regs;
-    dc->current_el = arm_current_el(env);
     dc->features = env->features;
 
     /* Single step state. The code-generation logic here is:
@@ -11066,13 +11708,16 @@
     /* FIXME: cpu_M0 can probably be the same as cpu_V0.  */
     cpu_M0 = tcg_temp_new_i64();
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
 
     tcg_clear_temp_count();
 
@@ -11096,10 +11741,9 @@
      * (3) if we leave the TB unexpectedly (eg a data abort on a load)
      * then the CPUARMState will be wrong and we need to reset it.
      * This is handled in the same way as restoration of the
-     * PC in these situations: we will be called again with search_pc=1
-     * and generate a mapping of the condexec bits for each PC in
-     * gen_opc_condexec_bits[]. restore_state_to_opc() then uses
-     * this to restore the condexec bits.
+     * PC in these situations; we save the value of the condexec bits
+     * for each PC via tcg_gen_insn_start(), and restore_state_to_opc()
+     * then uses this to restore them after an exception.
      *
      * Note that there are no instructions which can read the condexec
      * bits, and none which can write non-static values to them, so
@@ -11116,13 +11760,18 @@
         store_cpu_field(tmp, condexec_bits);
       }
     do {
+        tcg_gen_insn_start(dc->pc,
+                           (dc->condexec_cond << 4) | (dc->condexec_mask >> 1),
+                           0);
+        num_insns++;
+
 #ifdef CONFIG_USER_ONLY
         /* Intercept jump to the magic kernel page.  */
         if (dc->pc >= 0xffff0000) {
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_KERNEL_TRAP);
-            dc->is_jmp = DISAS_UPDATE;
+            dc->is_jmp = DISAS_EXC;
             break;
         }
 #else
@@ -11130,40 +11779,40 @@
             /* We always get here via a jump, so know we are not in a
                conditional execution block.  */
             gen_exception_internal(EXCP_EXCEPTION_EXIT);
-            dc->is_jmp = DISAS_UPDATE;
+            dc->is_jmp = DISAS_EXC;
             break;
         }
 #endif
 
         if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
+            CPUBreakpoint *bp;
             QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
-                    gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2;
-                    goto done_generating;
+                    if (bp->flags & BP_CPU) {
+                        gen_set_condexec(dc);
+                        gen_set_pc_im(dc, dc->pc);
+                        gen_helper_check_breakpoints(cpu_env);
+                        /* End the TB early; it's likely not going to be executed */
+                        dc->is_jmp = DISAS_UPDATE;
+                    } else {
+                        gen_exception_internal_insn(dc, 0, EXCP_DEBUG);
+                        /* The address covered by the breakpoint must be
+                           included in [tb->pc, tb->pc + tb->size) in order
+                           to for it to be properly cleared -- thus we
+                           increment the PC here so that the logic setting
+                           tb->size below does the right thing.  */
+                        /* TODO: Advance PC by correct instruction length to
+                         * avoid disassembler error messages */
+                        dc->pc += 2;
+                        goto done_generating;
+                    }
+                    break;
                 }
             }
         }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            gen_opc_condexec_bits[lj] = (dc->condexec_cond << 4) | (dc->condexec_mask >> 1);
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
         }
 
         if (dc->ss_active && !dc->pstate_ss) {
@@ -11177,8 +11826,9 @@
              * "did not step an insn" case, and so the syndrome ISV and EX
              * bits should be zero.
              */
-            assert(num_insns == 0);
-            gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0));
+            assert(num_insns == 1);
+            gen_exception(EXCP_UDEF, syn_swstep(dc->ss_same_el, 0, 0),
+                          default_exception_el(dc));
             goto done_generating;
         }
 
@@ -11193,7 +11843,7 @@
                 }
             }
         } else {
-            unsigned int insn = arm_ldl_code(env, dc->pc, dc->bswap_code);
+            unsigned int insn = arm_ldl_code(env, dc->pc, dc->sctlr_b);
             dc->pc += 4;
             disas_arm_insn(dc, insn);
         }
@@ -11212,12 +11862,24 @@
          * Otherwise the subsequent code could get translated several times.
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
-        num_insns ++;
+
+        /* We want to stop the TB if the next insn starts in a new page,
+         * or if it spans between this page and the next. This means that
+         * if we're looking at the last halfword in the page we need to
+         * see if it's a 16-bit Thumb insn (which will fit in this TB)
+         * or a 32-bit Thumb insn (which won't).
+         * This is to avoid generating a silly TB with a single 16-bit insn
+         * in it at the end of this page (which would execute correctly
+         * but isn't very efficient).
+         */
+        end_of_page = (dc->pc >= next_page_start) ||
+            ((dc->pc >= next_page_start - 3) && insn_crosses_page(env, dc));
+
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
              !dc->ss_active &&
-             dc->pc < next_page_start &&
+             !end_of_page &&
              num_insns < max_insns);
 
     if (tb->cflags & CF_LAST_IO) {
@@ -11233,45 +11895,45 @@
        instruction was a conditional branch or trap, and the PC has
        already been written.  */
     if (unlikely(cs->singlestep_enabled || dc->ss_active)) {
-        /* Make sure the pc is updated, and raise a debug exception.  */
+        /* Unconditional and "condition passed" instruction codepath. */
+        gen_set_condexec(dc);
+        switch (dc->is_jmp) {
+        case DISAS_SWI:
+            gen_ss_advance(dc);
+            gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb),
+                          default_exception_el(dc));
+            break;
+        case DISAS_HVC:
+            gen_ss_advance(dc);
+            gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm), 2);
+            break;
+        case DISAS_SMC:
+            gen_ss_advance(dc);
+            gen_exception(EXCP_SMC, syn_aa32_smc(), 3);
+            break;
+        case DISAS_NEXT:
+        case DISAS_UPDATE:
+            gen_set_pc_im(dc, dc->pc);
+            /* fall through */
+        default:
+            if (dc->ss_active) {
+                gen_step_complete_exception(dc);
+            } else {
+                /* FIXME: Single stepping a WFI insn will not halt
+                   the CPU.  */
+                gen_exception_internal(EXCP_DEBUG);
+            }
+        }
         if (dc->condjmp) {
+            /* "Condition failed" instruction codepath. */
+            gen_set_label(dc->condlabel);
             gen_set_condexec(dc);
-            if (dc->is_jmp == DISAS_SWI) {
-                gen_ss_advance(dc);
-                gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb));
-            } else if (dc->is_jmp == DISAS_HVC) {
-                gen_ss_advance(dc);
-                gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm));
-            } else if (dc->is_jmp == DISAS_SMC) {
-                gen_ss_advance(dc);
-                gen_exception(EXCP_SMC, syn_aa32_smc());
-            } else if (dc->ss_active) {
+            gen_set_pc_im(dc, dc->pc);
+            if (dc->ss_active) {
                 gen_step_complete_exception(dc);
             } else {
                 gen_exception_internal(EXCP_DEBUG);
             }
-            gen_set_label(dc->condlabel);
-        }
-        if (dc->condjmp || !dc->is_jmp) {
-            gen_set_pc_im(dc, dc->pc);
-            dc->condjmp = 0;
-        }
-        gen_set_condexec(dc);
-        if (dc->is_jmp == DISAS_SWI && !dc->condjmp) {
-            gen_ss_advance(dc);
-            gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb));
-        } else if (dc->is_jmp == DISAS_HVC && !dc->condjmp) {
-            gen_ss_advance(dc);
-            gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm));
-        } else if (dc->is_jmp == DISAS_SMC && !dc->condjmp) {
-            gen_ss_advance(dc);
-            gen_exception(EXCP_SMC, syn_aa32_smc());
-        } else if (dc->ss_active) {
-            gen_step_complete_exception(dc);
-        } else {
-            /* FIXME: Single stepping a WFI insn will not halt
-               the CPU.  */
-            gen_exception_internal(EXCP_DEBUG);
         }
     } else {
         /* While branches must always occur at the end of an IT block,
@@ -11287,9 +11949,11 @@
         case DISAS_NEXT:
             gen_goto_tb(dc, 1, dc->pc);
             break;
-        default:
-        case DISAS_JUMP:
         case DISAS_UPDATE:
+            gen_set_pc_im(dc, dc->pc);
+            /* fall through */
+        case DISAS_JUMP:
+        default:
             /* indicate that the hash table must be used to find the next TB */
             tcg_gen_exit_tb(0);
             break;
@@ -11298,18 +11962,26 @@
             break;
         case DISAS_WFI:
             gen_helper_wfi(cpu_env);
+            /* The helper doesn't necessarily throw an exception, but we
+             * must go back to the main loop to check for interrupts anyway.
+             */
+            tcg_gen_exit_tb(0);
             break;
         case DISAS_WFE:
             gen_helper_wfe(cpu_env);
             break;
+        case DISAS_YIELD:
+            gen_helper_yield(cpu_env);
+            break;
         case DISAS_SWI:
-            gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb));
+            gen_exception(EXCP_SWI, syn_aa32_svc(dc->svc_imm, dc->thumb),
+                          default_exception_el(dc));
             break;
         case DISAS_HVC:
-            gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm));
+            gen_exception(EXCP_HVC, syn_aa32_hvc(dc->svc_imm), 2);
             break;
         case DISAS_SMC:
-            gen_exception(EXCP_SMC, syn_aa32_smc());
+            gen_exception(EXCP_SMC, syn_aa32_smc(), 3);
             break;
         }
         if (dc->condjmp) {
@@ -11324,33 +11996,17 @@
     gen_tb_end(tb, num_insns);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM) &&
+        qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, dc->pc - pc_start,
-                         dc->thumb | (dc->bswap_code << 1));
+        log_target_disas(cs, pc_start, dc->pc - pc_start,
+                         dc->thumb | (dc->sctlr_b << 1));
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUARMState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(arm_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUARMState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(arm_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 static const char *cpu_mode_names[16] = {
@@ -11365,6 +12021,7 @@
     CPUARMState *env = &cpu->env;
     int i;
     uint32_t psr;
+    const char *ns_status;
 
     if (is_a64(env)) {
         aarch64_cpu_dump_state(cs, f, cpu_fprintf, flags);
@@ -11379,13 +12036,22 @@
             cpu_fprintf(f, " ");
     }
     psr = cpsr_read(env);
-    cpu_fprintf(f, "PSR=%08x %c%c%c%c %c %s%d\n",
+
+    if (arm_feature(env, ARM_FEATURE_EL3) &&
+        (psr & CPSR_M) != ARM_CPU_MODE_MON) {
+        ns_status = env->cp15.scr_el3 & SCR_NS ? "NS " : "S ";
+    } else {
+        ns_status = "";
+    }
+
+    cpu_fprintf(f, "PSR=%08x %c%c%c%c %c %s%s%d\n",
                 psr,
                 psr & (1 << 31) ? 'N' : '-',
                 psr & (1 << 30) ? 'Z' : '-',
                 psr & (1 << 29) ? 'C' : '-',
                 psr & (1 << 28) ? 'V' : '-',
                 psr & CPSR_T ? 'T' : 'A',
+                ns_status,
                 cpu_mode_names[psr & 0xf], (psr & 0x10) ? 32 : 26);
 
     if (flags & CPU_DUMP_FPU) {
@@ -11407,13 +12073,16 @@
     }
 }
 
-void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
     if (is_a64(env)) {
-        env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+        env->pc = data[0];
         env->condexec_bits = 0;
+        env->exception.syndrome = data[2] << ARM_INSN_START_WORD2_SHIFT;
     } else {
-        env->regs[15] = tcg_ctx.gen_opc_pc[pc_pos];
-        env->condexec_bits = gen_opc_condexec_bits[pc_pos];
+        env->regs[15] = data[0];
+        env->condexec_bits = data[1];
+        env->exception.syndrome = data[2] << ARM_INSN_START_WORD2_SHIFT;
     }
 }
diff --git a/target-arm/translate.h b/target-arm/translate.h
index 41a9071..dbd7ac8 100644
--- a/target-arm/translate.h
+++ b/target-arm/translate.h
@@ -9,18 +9,23 @@
     /* Nonzero if this instruction has been conditionally skipped.  */
     int condjmp;
     /* The label that will be jumped to when the instruction is skipped.  */
-    int condlabel;
+    TCGLabel *condlabel;
     /* Thumb-2 conditional execution bits.  */
     int condexec_mask;
     int condexec_cond;
     struct TranslationBlock *tb;
     int singlestep_enabled;
     int thumb;
-    int bswap_code;
+    int sctlr_b;
+    TCGMemOp be_data;
 #if !defined(CONFIG_USER_ONLY)
     int user;
 #endif
-    bool cpacr_fpen; /* FP enabled via CPACR.FPEN */
+    ARMMMUIdx mmu_idx; /* MMU index to use for normal loads/stores */
+    bool ns;        /* Use non-secure CPREG bank on access */
+    int fp_excp_el; /* FP exception EL or 0 if enabled */
+    /* Flag indicating that exceptions from secure mode are routed to EL3. */
+    bool secure_routed_to_el3;
     bool vfp_enabled; /* FP enabled via FPSCR.EN */
     int vec_len;
     int vec_stride;
@@ -54,12 +59,28 @@
     bool ss_same_el;
     /* Bottom two bits of XScale c15_cpar coprocessor access control reg */
     int c15_cpar;
+    /* TCG op index of the current insn_start.  */
+    int insn_start_idx;
 #define TMP_A64_MAX 16
     int tmp_a64_count;
     TCGv_i64 tmp_a64[TMP_A64_MAX];
 } DisasContext;
 
-extern TCGv_ptr cpu_env;
+typedef struct DisasCompare {
+    TCGCond cond;
+    TCGv_i32 value;
+    bool value_global;
+} DisasCompare;
+
+/* Share the TCG temporaries common between 32 and 64 bit modes.  */
+extern TCGv_env cpu_env;
+extern TCGv_i32 cpu_NF, cpu_ZF, cpu_CF, cpu_VF;
+extern TCGv_i64 cpu_exclusive_addr;
+extern TCGv_i64 cpu_exclusive_val;
+#ifdef CONFIG_USER_ONLY
+extern TCGv_i64 cpu_exclusive_test;
+extern TCGv_i32 cpu_exclusive_info;
+#endif
 
 static inline int arm_dc_feature(DisasContext *dc, int feature)
 {
@@ -68,7 +89,21 @@
 
 static inline int get_mem_index(DisasContext *s)
 {
-    return s->current_el;
+    return s->mmu_idx;
+}
+
+/* Function used to determine the target exception EL when otherwise not known
+ * or default.
+ */
+static inline int default_exception_el(DisasContext *s)
+{
+    /* If we are coming from secure EL0 in a system with a 32-bit EL3, then
+     * there is no secure EL1, so we route exceptions to EL3.  Otherwise,
+     * exceptions can only be routed to ELs above 1, so we target the higher of
+     * 1 or the current EL.
+     */
+    return (s->mmu_idx == ARMMMUIdx_S1SE0 && s->secure_routed_to_el3)
+            ? 3 : MAX(1, s->current_el);
 }
 
 /* target-specific extra values for is_jmp */
@@ -86,12 +121,11 @@
 #define DISAS_WFE 7
 #define DISAS_HVC 8
 #define DISAS_SMC 9
+#define DISAS_YIELD 10
 
 #ifdef TARGET_AARCH64
 void a64_translate_init(void);
-void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                        TranslationBlock *tb,
-                                        bool search_pc);
+void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb);
 void gen_a64_set_pc_im(uint64_t val);
 void aarch64_cpu_dump_state(CPUState *cs, FILE *f,
                             fprintf_function cpu_fprintf, int flags);
@@ -100,9 +134,7 @@
 {
 }
 
-static inline void gen_intermediate_code_internal_a64(ARMCPU *cpu,
-                                                      TranslationBlock *tb,
-                                                      bool search_pc)
+static inline void gen_intermediate_code_a64(ARMCPU *cpu, TranslationBlock *tb)
 {
 }
 
@@ -117,6 +149,9 @@
 }
 #endif
 
-void arm_gen_test_cc(int cc, int label);
+void arm_test_cc(DisasCompare *cmp, int cc);
+void arm_free_cc(DisasCompare *cmp);
+void arm_jump_cc(DisasCompare *cmp, TCGLabel *label);
+void arm_gen_test_cc(int cc, TCGLabel *label);
 
 #endif /* TARGET_ARM_TRANSLATE_H */
diff --git a/target-cris/cpu-qom.h b/target-cris/cpu-qom.h
index 6fc30c2..7556e9f 100644
--- a/target-cris/cpu-qom.h
+++ b/target-cris/cpu-qom.h
@@ -50,40 +50,6 @@
     uint32_t vr;
 } CRISCPUClass;
 
-/**
- * CRISCPU:
- * @env: #CPUCRISState
- *
- * A CRIS CPU.
- */
-typedef struct CRISCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUCRISState env;
-} CRISCPU;
-
-static inline CRISCPU *cris_env_get_cpu(CPUCRISState *env)
-{
-    return container_of(env, CRISCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(cris_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(CRISCPU, env)
-
-void cris_cpu_do_interrupt(CPUState *cpu);
-void crisv10_cpu_do_interrupt(CPUState *cpu);
-bool cris_cpu_exec_interrupt(CPUState *cpu, int int_req);
-
-void cris_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
-                         int flags);
-
-hwaddr cris_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-
-int crisv10_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int cris_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int cris_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+typedef struct CRISCPU CRISCPU;
 
 #endif
diff --git a/target-cris/cpu.c b/target-cris/cpu.c
index 16cfba9..c5a656b 100644
--- a/target-cris/cpu.c
+++ b/target-cris/cpu.c
@@ -21,9 +21,12 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "mmu.h"
+#include "exec/exec-all.h"
 
 
 static void cris_cpu_set_pc(CPUState *cs, vaddr value)
@@ -161,6 +164,20 @@
 }
 #endif
 
+static void cris_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    CRISCPU *cc = CRIS_CPU(cpu);
+    CPUCRISState *env = &cc->env;
+
+    if (env->pregs[PR_VR] != 32) {
+        info->mach = bfd_mach_cris_v0_v10;
+        info->print_insn = print_insn_crisv10;
+    } else {
+        info->mach = bfd_mach_cris_v32;
+        info->print_insn = print_insn_crisv32;
+    }
+}
+
 static void cris_cpu_initfn(Object *obj)
 {
     CPUState *cs = CPU(obj);
@@ -170,7 +187,7 @@
     static bool tcg_initialized;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     env->pregs[PR_VR] = ccc->vr;
 
@@ -288,10 +305,20 @@
     cc->handle_mmu_fault = cris_cpu_handle_mmu_fault;
 #else
     cc->get_phys_page_debug = cris_cpu_get_phys_page_debug;
+    dc->vmsd = &vmstate_cris_cpu;
 #endif
 
     cc->gdb_num_core_regs = 49;
     cc->gdb_stop_before_watchpoint = true;
+
+    cc->disas_set_info = cris_disas_set_info;
+
+    /*
+     * Reason: cris_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo cris_cpu_type_info = {
diff --git a/target-cris/cpu.h b/target-cris/cpu.h
index b88c147..7d7fe6e 100644
--- a/target-cris/cpu.h
+++ b/target-cris/cpu.h
@@ -17,11 +17,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef CPU_CRIS_H
-#define CPU_CRIS_H
 
-#include "config.h"
+#ifndef CRIS_CPU_H
+#define CRIS_CPU_H
+
 #include "qemu-common.h"
+#include "cpu-qom.h"
 
 #define TARGET_LONG_BITS 32
 
@@ -29,10 +30,6 @@
 
 #include "exec/cpu-defs.h"
 
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE	EM_CRIS
-
 #define EXCP_NMI        1
 #define EXCP_GURU       2
 #define EXCP_BUSFAULT   3
@@ -110,6 +107,11 @@
 
 #define NB_MMU_MODES 2
 
+typedef struct {
+    uint32_t hi;
+    uint32_t lo;
+} TLBSet;
+
 typedef struct CPUCRISState {
 	uint32_t regs[16];
 	/* P0 - P15 are referred to as special registers in the docs.  */
@@ -153,7 +155,7 @@
 	uint32_t sregs[4][16];
 
 	/* Linear feedback shift reg in the mmu. Used to provide pseudo
-	   randomness for the 'hint' the mmu gives to sw for chosing valid
+	   randomness for the 'hint' the mmu gives to sw for choosing valid
 	   sets on TLB refills.  */
 	uint32_t mmu_rand_lfsr;
 
@@ -163,11 +165,7 @@
 	 *
 	 * One for I and another for D.
 	 */
-	struct
-	{
-		uint32_t hi;
-		uint32_t lo;
-	} tlbsets[2][4][16];
+        TLBSet tlbsets[2][4][16];
 
 	CPU_COMMON
 
@@ -175,10 +173,47 @@
     void *load_info;
 } CPUCRISState;
 
-#include "cpu-qom.h"
+/**
+ * CRISCPU:
+ * @env: #CPUCRISState
+ *
+ * A CRIS CPU.
+ */
+struct CRISCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUCRISState env;
+};
+
+static inline CRISCPU *cris_env_get_cpu(CPUCRISState *env)
+{
+    return container_of(env, CRISCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(cris_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(CRISCPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_cris_cpu;
+#endif
+
+void cris_cpu_do_interrupt(CPUState *cpu);
+void crisv10_cpu_do_interrupt(CPUState *cpu);
+bool cris_cpu_exec_interrupt(CPUState *cpu, int int_req);
+
+void cris_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
+                         int flags);
+
+hwaddr cris_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+
+int crisv10_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int cris_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int cris_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
 CRISCPU *cpu_cris_init(const char *cpu_model);
-int cpu_cris_exec(CPUCRISState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
@@ -223,26 +258,15 @@
 #define TARGET_PHYS_ADDR_SPACE_BITS 32
 #define TARGET_VIRT_ADDR_SPACE_BITS 32
 
-static inline CPUCRISState *cpu_init(const char *cpu_model)
-{
-    CRISCPU *cpu = cpu_cris_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_cris_init(cpu_model))
 
-#define cpu_exec cpu_cris_exec
-#define cpu_gen_code cpu_cris_gen_code
 #define cpu_signal_handler cpu_cris_signal_handler
 
-#define CPU_SAVE_VERSION 1
-
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUCRISState *env)
+static inline int cpu_mmu_index (CPUCRISState *env, bool ifetch)
 {
 	return !!(env->pregs[PR_CCS] & U_FLAG);
 }
@@ -263,7 +287,7 @@
 #include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->pc;
     *cs_base = 0;
@@ -275,6 +299,4 @@
 #define cpu_list cris_cpu_list
 void cris_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 
-#include "exec/exec-all.h"
-
 #endif
diff --git a/target-cris/crisv32-decode.h b/target-cris/crisv32-decode.h
index cdba377..cdc2f8c 100644
--- a/target-cris/crisv32-decode.h
+++ b/target-cris/crisv32-decode.h
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+
 #ifndef CRISV32_DECODE_H
-#define CRISV32_DECODE_H 1
+#define CRISV32_DECODE_H
 
 /* Convenient binary macros.  */
 #define HEX__(n) 0x##n##LU
diff --git a/target-cris/gdbstub.c b/target-cris/gdbstub.c
index 5db3683..3a72ee2 100644
--- a/target-cris/gdbstub.c
+++ b/target-cris/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int crisv10_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
diff --git a/target-cris/helper.c b/target-cris/helper.c
index e901c3a..af78cca 100644
--- a/target-cris/helper.c
+++ b/target-cris/helper.c
@@ -18,9 +18,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "qemu/host-utils.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 
@@ -84,8 +86,8 @@
     int r = -1;
     target_ulong phy;
 
-    D(printf("%s addr=%" VADDR_PRIx " pc=%x rw=%x\n",
-             __func__, address, env->pc, rw));
+    qemu_log_mask(CPU_LOG_MMU, "%s addr=%" VADDR_PRIx " pc=%x rw=%x\n",
+            __func__, address, env->pc, rw);
     miss = cris_mmu_translate(&res, env, address & TARGET_PAGE_MASK,
                               rw, mmu_idx, 0);
     if (miss) {
@@ -112,9 +114,10 @@
         r = 0;
     }
     if (r > 0) {
-        D_LOG("%s returns %d irqreq=%x addr=%" VADDR_PRIx " phy=%x vec=%x"
-              " pc=%x\n", __func__, r, cs->interrupt_request, address, res.phy,
-              res.bf_vec, env->pc);
+        qemu_log_mask(CPU_LOG_MMU,
+                "%s returns %d irqreq=%x addr=%" VADDR_PRIx " phy=%x vec=%x"
+                " pc=%x\n", __func__, r, cs->interrupt_request, address,
+                res.phy, res.bf_vec, env->pc);
     }
     return r;
 }
diff --git a/target-cris/helper.h b/target-cris/helper.h
index 0b383b2..ff35956 100644
--- a/target-cris/helper.h
+++ b/target-cris/helper.h
@@ -1,7 +1,6 @@
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_2(tlb_flush_pid, void, env, i32)
 DEF_HELPER_2(spc_write, void, env, i32)
-DEF_HELPER_3(dump, void, i32, i32, i32)
 DEF_HELPER_1(rfe, void, env)
 DEF_HELPER_1(rfn, void, env)
 
diff --git a/target-cris/machine.c b/target-cris/machine.c
index 8f9c0dd..6b797e8 100644
--- a/target-cris/machine.c
+++ b/target-cris/machine.c
@@ -1,90 +1,95 @@
+/*
+ *  CRIS virtual CPU state save/load support
+ *
+ *  Copyright (c) 2012 Red Hat, Inc.
+ *  Written by Juan Quintela <quintela@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
-#include "hw/boards.h"
+#include "migration/cpu.h"
 
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    CPUCRISState *env = opaque;
-    int i;
-    int s;
-    int mmu;
-
-    for (i = 0; i < 16; i++)
-        qemu_put_be32(f, env->regs[i]);
-    for (i = 0; i < 16; i++)
-        qemu_put_be32(f, env->pregs[i]);
-
-    qemu_put_be32(f, env->pc);
-    qemu_put_be32(f, env->ksp);
-
-    qemu_put_be32(f, env->dslot);
-    qemu_put_be32(f, env->btaken);
-    qemu_put_be32(f, env->btarget);
-
-    qemu_put_be32(f, env->cc_op);
-    qemu_put_be32(f, env->cc_mask);
-    qemu_put_be32(f, env->cc_dest);
-    qemu_put_be32(f, env->cc_src);
-    qemu_put_be32(f, env->cc_result);
-    qemu_put_be32(f, env->cc_size);
-    qemu_put_be32(f, env->cc_x);
-
-    for (s = 0; s < 4; s++) {
-        for (i = 0; i < 16; i++)
-            qemu_put_be32(f, env->sregs[s][i]);
+static const VMStateDescription vmstate_tlbset = {
+    .name = "cpu/tlbset",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(lo, TLBSet),
+        VMSTATE_UINT32(hi, TLBSet),
+        VMSTATE_END_OF_LIST()
     }
+};
 
-    qemu_put_be32(f, env->mmu_rand_lfsr);
-    for (mmu = 0; mmu < 2; mmu++) {
-        for (s = 0; s < 4; s++) {
-            for (i = 0; i < 16; i++) {
-                qemu_put_be32(f, env->tlbsets[mmu][s][i].lo);
-                qemu_put_be32(f, env->tlbsets[mmu][s][i].hi);
-            }
-        }
+static const VMStateDescription vmstate_cris_env = {
+    .name = "env",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32_ARRAY(regs, CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(pregs, CPUCRISState, 16),
+        VMSTATE_UINT32(pc, CPUCRISState),
+        VMSTATE_UINT32(ksp, CPUCRISState),
+        VMSTATE_INT32(dslot, CPUCRISState),
+        VMSTATE_INT32(btaken, CPUCRISState),
+        VMSTATE_UINT32(btarget, CPUCRISState),
+        VMSTATE_UINT32(cc_op, CPUCRISState),
+        VMSTATE_UINT32(cc_mask, CPUCRISState),
+        VMSTATE_UINT32(cc_dest, CPUCRISState),
+        VMSTATE_UINT32(cc_src, CPUCRISState),
+        VMSTATE_UINT32(cc_result, CPUCRISState),
+        VMSTATE_INT32(cc_size, CPUCRISState),
+        VMSTATE_INT32(cc_x, CPUCRISState),
+        VMSTATE_INT32(locked_irq, CPUCRISState),
+        VMSTATE_INT32(interrupt_vector, CPUCRISState),
+        VMSTATE_INT32(fault_vector, CPUCRISState),
+        VMSTATE_INT32(trap_vector, CPUCRISState),
+        VMSTATE_UINT32_ARRAY(sregs[0], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[1], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[2], CPUCRISState, 16),
+        VMSTATE_UINT32_ARRAY(sregs[3], CPUCRISState, 16),
+        VMSTATE_UINT32(mmu_rand_lfsr, CPUCRISState),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][0], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][1], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][2], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[0][3], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][0], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][1], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][2], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_STRUCT_ARRAY(tlbsets[1][3], CPUCRISState, 16, 0,
+                             vmstate_tlbset, TLBSet),
+        VMSTATE_END_OF_LIST()
     }
-}
+};
 
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-	CPUCRISState *env = opaque;
-    int i;
-    int s;
-    int mmu;
-
-    for (i = 0; i < 16; i++)
-        env->regs[i] = qemu_get_be32(f);
-    for (i = 0; i < 16; i++)
-        env->pregs[i] = qemu_get_be32(f);
-
-    env->pc = qemu_get_be32(f);
-    env->ksp = qemu_get_be32(f);
-
-    env->dslot = qemu_get_be32(f);
-    env->btaken = qemu_get_be32(f);
-    env->btarget = qemu_get_be32(f);
-
-    env->cc_op = qemu_get_be32(f);
-    env->cc_mask = qemu_get_be32(f);
-    env->cc_dest = qemu_get_be32(f);
-    env->cc_src = qemu_get_be32(f);
-    env->cc_result = qemu_get_be32(f);
-    env->cc_size = qemu_get_be32(f);
-    env->cc_x = qemu_get_be32(f);
-
-    for (s = 0; s < 4; s++) {
-        for (i = 0; i < 16; i++)
-            env->sregs[s][i] = qemu_get_be32(f);
+const VMStateDescription vmstate_cris_cpu = {
+    .name = "cpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_CPU(),
+        VMSTATE_STRUCT(env, CRISCPU, 1, vmstate_cris_env, CPUCRISState),
+        VMSTATE_END_OF_LIST()
     }
-
-    env->mmu_rand_lfsr = qemu_get_be32(f);
-    for (mmu = 0; mmu < 2; mmu++) {
-        for (s = 0; s < 4; s++) {
-            for (i = 0; i < 16; i++) {
-                env->tlbsets[mmu][s][i].lo = qemu_get_be32(f);
-                env->tlbsets[mmu][s][i].hi = qemu_get_be32(f);
-            }
-        }
-    }
-
-    return 0;
-}
+};
diff --git a/target-cris/mmu.c b/target-cris/mmu.c
index 1c95a41..b8db908 100644
--- a/target-cris/mmu.c
+++ b/target-cris/mmu.c
@@ -18,9 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef CONFIG_USER_ONLY
-
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "mmu.h"
 
 #ifdef DEBUG
@@ -360,4 +360,3 @@
 	env->pregs[PR_SRS] = old_srs;
 	return miss;
 }
-#endif
diff --git a/target-cris/op_helper.c b/target-cris/op_helper.c
index 5c0c14d..5043039 100644
--- a/target-cris/op_helper.c
+++ b/target-cris/op_helper.c
@@ -18,10 +18,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 //#define CRIS_OP_HELPER_DEBUG
@@ -39,8 +41,8 @@
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     CRISCPU *cpu = CRIS_CPU(cs);
     CPUCRISState *env = &cpu->env;
@@ -48,7 +50,7 @@
 
     D_LOG("%s pc=%x tpc=%x ra=%p\n", __func__,
           env->pc, env->pregs[PR_EDA], (void *)retaddr);
-    ret = cris_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = cris_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
@@ -91,11 +93,6 @@
 #endif
 }
 
-void helper_dump(uint32_t a0, uint32_t a1, uint32_t a2)
-{
-	qemu_log("%s: a0=%x a1=%x\n", __func__, a0, a1);
-}
-
 /* Used by the tlb decoder.  */
 #define EXTRACT_FIELD(src, start, end) \
 	    (((src) >> start) & ((1 << (end - start + 1)) - 1))
diff --git a/target-cris/opcode-cris.h b/target-cris/opcode-cris.h
index 779d4aa..e7ebb98 100644
--- a/target-cris/opcode-cris.h
+++ b/target-cris/opcode-cris.h
@@ -108,16 +108,6 @@
 };
 extern const struct cris_support_reg cris_support_regs[];
 
-struct cris_cond15
-{
-  /* The name of the condition.  */
-  const char *const name;
-
-  /* What CPU version this condition name applies to.  */
-  enum cris_insn_version_usage applicable_version;
-};
-extern const struct cris_cond15 cris_conds15[];
-
 /* Opcode-dependent constants.  */
 #define AUTOINCR_BIT (0x04)
 
diff --git a/target-cris/translate.c b/target-cris/translate.c
index b4c4ca3..f4a8d7d 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -23,8 +23,10 @@
  * The condition code translation is in need of attention.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/helper-proto.h"
 #include "mmu.h"
@@ -34,6 +36,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define DISAS_CRIS 0
@@ -58,7 +61,7 @@
 #define CC_MASK_NZVC 0xf
 #define CC_MASK_RNZV 0x10e
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_R[16];
 static TCGv cpu_PR[16];
 static TCGv cc_x;
@@ -102,9 +105,9 @@
 
     int cc_size_uptodate; /* -1 invalid or last written value.  */
 
-    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
-    int flags_uptodate; /* Wether or not $ccs is uptodate.  */
-    int flagx_known; /* Wether or not flags_x has the x flag known at
+    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not up-to-date.  */
+    int flags_uptodate; /* Whether or not $ccs is up-to-date.  */
+    int flagx_known; /* Whether or not flags_x has the x flag known at
                 translation time.  */
     int flags_x;
 
@@ -130,8 +133,10 @@
 
 static void gen_BUG(DisasContext *dc, const char *file, int line)
 {
-    printf("BUG: pc=%x %s %d\n", dc->pc, file, line);
-    qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    fprintf(stderr, "BUG: pc=%x %s %d\n", dc->pc, file, line);
+    if (qemu_log_separate()) {
+        qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    }
     cpu_abort(CPU(dc->cpu), "%s:%d\n", file, line);
 }
 
@@ -311,9 +316,7 @@
 
 static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
 {
-    int l1;
-
-    l1 = gen_new_label();
+    TCGv t = tcg_temp_new();
 
     /*
      * d <<= 1
@@ -321,9 +324,9 @@
      *    d -= s;
      */
     tcg_gen_shli_tl(d, a, 1);
-    tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
-    tcg_gen_sub_tl(d, d, b);
-    gen_set_label(l1);
+    tcg_gen_sub_tl(t, d, b);
+    tcg_gen_movcond_tl(TCG_COND_GEU, d, d, b, t, d);
+    tcg_temp_free(t);
 }
 
 static void t_gen_cris_mstep(TCGv d, TCGv a, TCGv b, TCGv ccs)
@@ -509,9 +512,7 @@
 
 static void t_gen_cc_jmp(TCGv pc_true, TCGv pc_false)
 {
-    int l1;
-
-    l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
 
     /* Conditional jmp.  */
     tcg_gen_mov_tl(env_pc, pc_false);
@@ -520,14 +521,22 @@
     gen_set_label(l1);
 }
 
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+#ifndef CONFIG_USER_ONLY
+    return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
+           (dc->ppc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = dc->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+    if (use_goto_tb(dc, dest)) {
         tcg_gen_goto_tb(n);
         tcg_gen_movi_tl(env_pc, dest);
-                tcg_gen_exit_tb((uintptr_t)tb + n);
+                tcg_gen_exit_tb((uintptr_t)dc->tb + n);
     } else {
         tcg_gen_movi_tl(env_pc, dest);
         tcg_gen_exit_tb(0);
@@ -773,14 +782,7 @@
         t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
         break;
     case CC_OP_BOUND:
-    {
-        int l1;
-        l1 = gen_new_label();
-        tcg_gen_mov_tl(dst, a);
-        tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
-        tcg_gen_mov_tl(dst, b);
-        gen_set_label(l1);
-    }
+        tcg_gen_movcond_tl(TCG_COND_LEU, dst, a, b, a, b);
         break;
     case CC_OP_CMP:
         tcg_gen_sub_tl(dst, a, b);
@@ -788,7 +790,7 @@
         t_gen_subx_carry(dc, dst);
         break;
     default:
-        qemu_log("illegal ALU op.\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "illegal ALU op.\n");
         BUG();
         break;
     }
@@ -1094,7 +1096,7 @@
 
 static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1108,7 +1110,7 @@
 static void gen_load(DisasContext *dc, TCGv dst, TCGv addr, 
              unsigned int size, int sign)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1123,7 +1125,7 @@
 static void gen_store (DisasContext *dc, TCGv addr, TCGv val,
                unsigned int size)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -1487,17 +1489,8 @@
     LOG_DIS("s%s $r%u\n",
             cc_name(cond), dc->op1);
 
-    if (cond != CC_A) {
-        int l1;
-
-        gen_tst_cc(dc, cpu_R[dc->op1], cond);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
-        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-        gen_set_label(l1);
-    } else {
-        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-    }
+    gen_tst_cc(dc, cpu_R[dc->op1], cond);
+    tcg_gen_setcondi_tl(TCG_COND_NE, cpu_R[dc->op1], cpu_R[dc->op1], 0);
 
     cris_cc_mask(dc, 0);
     return 2;
@@ -2611,9 +2604,9 @@
     tcg_temp_free(addr);
 
     for (i = 0; i < (nr >> 1); i++) {
-        tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
+        tcg_gen_extrl_i64_i32(cpu_R[i * 2], tmp[i]);
         tcg_gen_shri_i64(tmp[i], tmp[i], 32);
-        tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
+        tcg_gen_extrl_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
         tcg_temp_free_i64(tmp[i]);
     }
     if (nr & 1) {
@@ -3014,10 +3007,6 @@
     int insn_len = 2;
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-        }
-
     /* Load a halfword onto the instruction register.  */
         dc->ir = cris_fetch(env, dc, dc->pc, 2, 0);
 
@@ -3040,9 +3029,7 @@
 #if !defined(CONFIG_USER_ONLY)
     /* Single-stepping ?  */
     if (dc->tb_flags & S_FLAG) {
-        int l1;
-
-        l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
         tcg_gen_brcondi_tl(TCG_COND_NE, cpu_PR[PR_SPC], dc->pc, l1);
         /* We treat SPC as a break with an odd trap vector.  */
         cris_evaluate_flags(dc);
@@ -3056,23 +3043,6 @@
     return insn_len;
 }
 
-static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(cris_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                cris_evaluate_flags(dc);
-                tcg_gen_movi_tl(env_pc, dc->pc);
-                t_gen_raise_exception(EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-            }
-        }
-    }
-}
-
 #include "translate_v10.c"
 
 /*
@@ -3088,7 +3058,7 @@
  *
  * When the software returns from an exception, the branch will re-execute.
  * On QEMU care needs to be taken when a branch+delayslot sequence is broken
- * and the branch and delayslot dont share pages.
+ * and the branch and delayslot don't share pages.
  *
  * The TB contaning the branch insn will set up env->btarget and evaluate 
  * env->btaken. When the translation loop exits we will note that the branch 
@@ -3110,15 +3080,12 @@
  */
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(CRISCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUCRISState *env, struct TranslationBlock *tb)
 {
+    CRISCPU *cpu = cris_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUCRISState *env = &cpu->env;
     uint32_t pc_start;
     unsigned int insn_len;
-    int j, lj;
     struct DisasContext ctx;
     struct DisasContext *dc = &ctx;
     uint32_t next_page_start;
@@ -3170,13 +3137,13 @@
 
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log(
-                "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
+                "pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
                 "pid=%x usp=%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n"
                 "%x.%x.%x.%x\n",
-                search_pc, dc->pc, dc->ppc,
+                dc->pc, dc->ppc,
                 (uint64_t)tb->flags,
                 env->btarget, (unsigned)tb->flags & 7,
                 env->pregs[PR_CCS],
@@ -3192,38 +3159,38 @@
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do {
-        check_breakpoint(env, dc);
+        tcg_gen_insn_start(dc->delayed_branch == 1
+                           ? dc->ppc | 1 : dc->pc);
+        num_insns++;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            if (dc->delayed_branch == 1) {
-                tcg_ctx.gen_opc_pc[lj] = dc->ppc | 1;
-            } else {
-                tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            }
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            cris_evaluate_flags(dc);
+            tcg_gen_movi_tl(env_pc, dc->pc);
+            t_gen_raise_exception(EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 2;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         dc->clear_x = 1;
@@ -3235,7 +3202,6 @@
             cris_clear_x_flag(dc);
         }
 
-        num_insns++;
         /* Check for delayed branches here. If we do it before
            actually generating any host code, the simulator will just
            loop doing nothing for on this program location.  */
@@ -3256,9 +3222,7 @@
                 }
 
                 if (dc->jmp == JMP_DIRECT_CC) {
-                    int l1;
-
-                    l1 = gen_new_label();
+                    TCGLabel *l1 = gen_new_label();
                     cris_evaluate_flags(dc);
 
                     /* Conditional jmp.  */
@@ -3283,7 +3247,7 @@
         }
 
         /* If we are rexecuting a branch due to exceptions on
-           delay slots dont break.  */
+           delay slots don't break.  */
         if (!(tb->pc & 1) && cs->singlestep_enabled) {
             break;
         }
@@ -3342,21 +3306,14 @@
     }
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
 #if !DISAS_CRIS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
-        log_target_disas(env, pc_start, dc->pc - pc_start,
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
+        log_target_disas(cs, pc_start, dc->pc - pc_start,
                          env->pregs[PR_VR]);
         qemu_log("\nisize=%d osize=%d\n",
                  dc->pc - pc_start, tcg_op_buf_count());
@@ -3365,16 +3322,6 @@
 #endif
 }
 
-void gen_intermediate_code (CPUCRISState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(cris_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUCRISState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(cris_env_get_cpu(env), tb, true);
-}
-
 void cris_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -3427,47 +3374,49 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cc_x = tcg_global_mem_new(TCG_AREG0,
+    tcg_ctx.tcg_env = cpu_env;
+    cc_x = tcg_global_mem_new(cpu_env,
                               offsetof(CPUCRISState, cc_x), "cc_x");
-    cc_src = tcg_global_mem_new(TCG_AREG0,
+    cc_src = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUCRISState, cc_src), "cc_src");
-    cc_dest = tcg_global_mem_new(TCG_AREG0,
+    cc_dest = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUCRISState, cc_dest),
                                  "cc_dest");
-    cc_result = tcg_global_mem_new(TCG_AREG0,
+    cc_result = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUCRISState, cc_result),
                                    "cc_result");
-    cc_op = tcg_global_mem_new(TCG_AREG0,
+    cc_op = tcg_global_mem_new(cpu_env,
                                offsetof(CPUCRISState, cc_op), "cc_op");
-    cc_size = tcg_global_mem_new(TCG_AREG0,
+    cc_size = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUCRISState, cc_size),
                                  "cc_size");
-    cc_mask = tcg_global_mem_new(TCG_AREG0,
+    cc_mask = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUCRISState, cc_mask),
                                  "cc_mask");
 
-    env_pc = tcg_global_mem_new(TCG_AREG0,
+    env_pc = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUCRISState, pc),
                                 "pc");
-    env_btarget = tcg_global_mem_new(TCG_AREG0,
+    env_btarget = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUCRISState, btarget),
                                      "btarget");
-    env_btaken = tcg_global_mem_new(TCG_AREG0,
+    env_btaken = tcg_global_mem_new(cpu_env,
                                     offsetof(CPUCRISState, btaken),
                                     "btaken");
     for (i = 0; i < 16; i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                                       offsetof(CPUCRISState, regs[i]),
                                       regnames[i]);
     }
     for (i = 0; i < 16; i++) {
-        cpu_PR[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_PR[i] = tcg_global_mem_new(cpu_env,
                                        offsetof(CPUCRISState, pregs[i]),
                                        pregnames[i]);
     }
 }
 
-void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-cris/translate_v10.c b/target-cris/translate_v10.c
index efb3639..4707a18 100644
--- a/target-cris/translate_v10.c
+++ b/target-cris/translate_v10.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "crisv10-decode.h"
 
 static const char *regnames_v10[] =
@@ -58,14 +59,14 @@
 
 static inline void cris_illegal_insn(DisasContext *dc)
 {
-    qemu_log("illegal insn at pc=%x\n", dc->pc);
+    qemu_log_mask(LOG_GUEST_ERROR, "illegal insn at pc=%x\n", dc->pc);
     t_gen_raise_exception(EXCP_BREAK);
 }
 
 static void gen_store_v10_conditional(DisasContext *dc, TCGv addr, TCGv val,
                        unsigned int size, int mem_index)
 {
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv taddr = tcg_temp_local_new();
     TCGv tval = tcg_temp_local_new();
     TCGv t1 = tcg_temp_local_new();
@@ -96,7 +97,7 @@
 static void gen_store_v10(DisasContext *dc, TCGv addr, TCGv val,
                        unsigned int size)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     /* If we get a fault on a delayslot we must keep the jmp state in
        the cpu-state to be able to re-execute the jmp.  */
@@ -129,7 +130,7 @@
     dc->tb_flags |= PFIX_FLAG;
     tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], PFIX_FLAG);
 
-    /* prefix insns dont clear the x flag.  */
+    /* prefix insns don't clear the x flag.  */
     dc->clear_x = 0;
     cris_lock_irq(dc);
 }
@@ -535,18 +536,8 @@
 
     LOG_DIS("s%s $r%u\n", cc_name(cond), dc->src);
 
-    if (cond != CC_A)
-    {
-        int l1;
-
-        gen_tst_cc (dc, cpu_R[dc->src], cond);
-        l1 = gen_new_label();
-        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->src], 0, l1);
-        tcg_gen_movi_tl(cpu_R[dc->src], 1);
-        gen_set_label(l1);
-    } else {
-        tcg_gen_movi_tl(cpu_R[dc->src], 1);
-    }
+    gen_tst_cc(dc, cpu_R[dc->src], cond);
+    tcg_gen_setcondi_tl(TCG_COND_NE, cpu_R[dc->src], cpu_R[dc->src], 0);
 
     cris_cc_mask(dc, 0);
 }
@@ -1209,9 +1200,6 @@
 {
     unsigned int insn_len = 2;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
-        tcg_gen_debug_insn_start(dc->pc);
-
     /* Load a halfword onto the instruction register.  */
     dc->ir = cpu_lduw_code(env, dc->pc);
 
@@ -1259,45 +1247,46 @@
 
 void cris_initialize_crisv10_tcg(void)
 {
-	int i;
+    int i;
 
-	cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-	cc_x = tcg_global_mem_new(TCG_AREG0,
-				  offsetof(CPUCRISState, cc_x), "cc_x");
-	cc_src = tcg_global_mem_new(TCG_AREG0,
-				    offsetof(CPUCRISState, cc_src), "cc_src");
-	cc_dest = tcg_global_mem_new(TCG_AREG0,
-				     offsetof(CPUCRISState, cc_dest),
-				     "cc_dest");
-	cc_result = tcg_global_mem_new(TCG_AREG0,
-				       offsetof(CPUCRISState, cc_result),
-				       "cc_result");
-	cc_op = tcg_global_mem_new(TCG_AREG0,
-				   offsetof(CPUCRISState, cc_op), "cc_op");
-	cc_size = tcg_global_mem_new(TCG_AREG0,
-				     offsetof(CPUCRISState, cc_size),
-				     "cc_size");
-	cc_mask = tcg_global_mem_new(TCG_AREG0,
-				     offsetof(CPUCRISState, cc_mask),
-				     "cc_mask");
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
+    cc_x = tcg_global_mem_new(cpu_env,
+                              offsetof(CPUCRISState, cc_x), "cc_x");
+    cc_src = tcg_global_mem_new(cpu_env,
+                                offsetof(CPUCRISState, cc_src), "cc_src");
+    cc_dest = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUCRISState, cc_dest),
+                                 "cc_dest");
+    cc_result = tcg_global_mem_new(cpu_env,
+                                   offsetof(CPUCRISState, cc_result),
+                                   "cc_result");
+    cc_op = tcg_global_mem_new(cpu_env,
+                               offsetof(CPUCRISState, cc_op), "cc_op");
+    cc_size = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUCRISState, cc_size),
+                                 "cc_size");
+    cc_mask = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUCRISState, cc_mask),
+                                 "cc_mask");
 
-	env_pc = tcg_global_mem_new(TCG_AREG0, 
-				    offsetof(CPUCRISState, pc),
-				    "pc");
-	env_btarget = tcg_global_mem_new(TCG_AREG0,
-					 offsetof(CPUCRISState, btarget),
-					 "btarget");
-	env_btaken = tcg_global_mem_new(TCG_AREG0,
-					 offsetof(CPUCRISState, btaken),
-					 "btaken");
-	for (i = 0; i < 16; i++) {
-		cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
-					      offsetof(CPUCRISState, regs[i]),
-					      regnames_v10[i]);
-	}
-	for (i = 0; i < 16; i++) {
-		cpu_PR[i] = tcg_global_mem_new(TCG_AREG0,
-					       offsetof(CPUCRISState, pregs[i]),
-					       pregnames_v10[i]);
-	}
+    env_pc = tcg_global_mem_new(cpu_env,
+                                offsetof(CPUCRISState, pc),
+                                "pc");
+    env_btarget = tcg_global_mem_new(cpu_env,
+                                     offsetof(CPUCRISState, btarget),
+                                     "btarget");
+    env_btaken = tcg_global_mem_new(cpu_env,
+                                    offsetof(CPUCRISState, btaken),
+                                    "btaken");
+    for (i = 0; i < 16; i++) {
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
+                                      offsetof(CPUCRISState, regs[i]),
+                                      regnames_v10[i]);
+    }
+    for (i = 0; i < 16; i++) {
+        cpu_PR[i] = tcg_global_mem_new(cpu_env,
+                                       offsetof(CPUCRISState, pregs[i]),
+                                       pregnames_v10[i]);
+    }
 }
diff --git a/target-i386/Makefile.objs b/target-i386/Makefile.objs
index 027b94e..2820b49 100644
--- a/target-i386/Makefile.objs
+++ b/target-i386/Makefile.objs
@@ -1,9 +1,12 @@
-obj-y += translate.o helper.o cpu.o
+obj-y += translate.o helper.o cpu.o bpt_helper.o
 obj-y += excp_helper.o fpu_helper.o cc_helper.o int_helper.o svm_helper.o
-obj-y += smm_helper.o misc_helper.o mem_helper.o seg_helper.o
+obj-y += smm_helper.o misc_helper.o mem_helper.o seg_helper.o mpx_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o
-obj-$(CONFIG_KVM) += kvm.o
+obj-$(CONFIG_SOFTMMU) += machine.o arch_memory_mapping.o arch_dump.o monitor.o
+obj-$(CONFIG_KVM) += kvm.o hyperv.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
-obj-$(CONFIG_LINUX_USER) += ioport-user.o
-obj-$(CONFIG_BSD_USER) += ioport-user.o
+ifeq (y,$(CONFIG_HAX))
+obj-y += hax-all.o hax-slot.o
+obj-$(CONFIG_WIN32) += hax-windows.o
+obj-$(CONFIG_DARWIN) += hax-darwin.o
+endif
diff --git a/target-i386/arch_dump.c b/target-i386/arch_dump.c
index 0bbed23..5a2e4be 100644
--- a/target-i386/arch_dump.c
+++ b/target-i386/arch_dump.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
 #include "sysemu/dump.h"
@@ -78,9 +79,7 @@
     descsz = sizeof(x86_64_elf_prstatus);
     note_size = ((sizeof(Elf64_Nhdr) + 3) / 4 + (name_size + 3) / 4 +
                 (descsz + 3) / 4) * 4;
-    note = g_malloc(note_size);
-
-    memset(note, 0, note_size);
+    note = g_malloc0(note_size);
     note->n_namesz = cpu_to_le32(name_size);
     note->n_descsz = cpu_to_le32(descsz);
     note->n_type = cpu_to_le32(NT_PRSTATUS);
@@ -159,9 +158,7 @@
     descsz = sizeof(x86_elf_prstatus);
     note_size = ((sizeof(Elf64_Nhdr) + 3) / 4 + (name_size + 3) / 4 +
                 (descsz + 3) / 4) * 4;
-    note = g_malloc(note_size);
-
-    memset(note, 0, note_size);
+    note = g_malloc0(note_size);
     note->n_namesz = cpu_to_le32(name_size);
     note->n_descsz = cpu_to_le32(descsz);
     note->n_type = cpu_to_le32(NT_PRSTATUS);
@@ -216,9 +213,7 @@
     descsz = sizeof(x86_elf_prstatus);
     note_size = ((sizeof(Elf32_Nhdr) + 3) / 4 + (name_size + 3) / 4 +
                 (descsz + 3) / 4) * 4;
-    note = g_malloc(note_size);
-
-    memset(note, 0, note_size);
+    note = g_malloc0(note_size);
     note->n_namesz = cpu_to_le32(name_size);
     note->n_descsz = cpu_to_le32(descsz);
     note->n_type = cpu_to_le32(NT_PRSTATUS);
@@ -345,9 +340,7 @@
     }
     note_size = ((note_head_size + 3) / 4 + (name_size + 3) / 4 +
                 (descsz + 3) / 4) * 4;
-    note = g_malloc(note_size);
-
-    memset(note, 0, note_size);
+    note = g_malloc0(note_size);
     if (type == 0) {
         note32 = note;
         note32->n_namesz = cpu_to_le32(name_size);
diff --git a/target-i386/arch_memory_mapping.c b/target-i386/arch_memory_mapping.c
index 2d35f63..88f341e 100644
--- a/target-i386/arch_memory_mapping.c
+++ b/target-i386/arch_memory_mapping.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
 #include "sysemu/memory_mapping.h"
@@ -27,7 +28,7 @@
 
     for (i = 0; i < 512; i++) {
         pte_addr = (pte_start_addr + i * 8) & a20_mask;
-        pte = ldq_phys(as, pte_addr);
+        pte = address_space_ldq(as, pte_addr, MEMTXATTRS_UNSPECIFIED, NULL);
         if (!(pte & PG_PRESENT_MASK)) {
             /* not present */
             continue;
@@ -57,7 +58,7 @@
 
     for (i = 0; i < 1024; i++) {
         pte_addr = (pte_start_addr + i * 4) & a20_mask;
-        pte = ldl_phys(as, pte_addr);
+        pte = address_space_ldl(as, pte_addr, MEMTXATTRS_UNSPECIFIED, NULL);
         if (!(pte & PG_PRESENT_MASK)) {
             /* not present */
             continue;
@@ -89,7 +90,7 @@
 
     for (i = 0; i < 512; i++) {
         pde_addr = (pde_start_addr + i * 8) & a20_mask;
-        pde = ldq_phys(as, pde_addr);
+        pde = address_space_ldq(as, pde_addr, MEMTXATTRS_UNSPECIFIED, NULL);
         if (!(pde & PG_PRESENT_MASK)) {
             /* not present */
             continue;
@@ -126,7 +127,7 @@
 
     for (i = 0; i < 1024; i++) {
         pde_addr = (pde_start_addr + i * 4) & a20_mask;
-        pde = ldl_phys(as, pde_addr);
+        pde = address_space_ldl(as, pde_addr, MEMTXATTRS_UNSPECIFIED, NULL);
         if (!(pde & PG_PRESENT_MASK)) {
             /* not present */
             continue;
@@ -167,7 +168,7 @@
 
     for (i = 0; i < 4; i++) {
         pdpe_addr = (pdpe_start_addr + i * 8) & a20_mask;
-        pdpe = ldq_phys(as, pdpe_addr);
+        pdpe = address_space_ldq(as, pdpe_addr, MEMTXATTRS_UNSPECIFIED, NULL);
         if (!(pdpe & PG_PRESENT_MASK)) {
             /* not present */
             continue;
@@ -192,7 +193,7 @@
 
     for (i = 0; i < 512; i++) {
         pdpe_addr = (pdpe_start_addr + i * 8) & a20_mask;
-        pdpe = ldq_phys(as, pdpe_addr);
+        pdpe = address_space_ldq(as, pdpe_addr, MEMTXATTRS_UNSPECIFIED, NULL);
         if (!(pdpe & PG_PRESENT_MASK)) {
             /* not present */
             continue;
@@ -228,7 +229,8 @@
 
     for (i = 0; i < 512; i++) {
         pml4e_addr = (pml4e_start_addr + i * 8) & a20_mask;
-        pml4e = ldq_phys(as, pml4e_addr);
+        pml4e = address_space_ldq(as, pml4e_addr, MEMTXATTRS_UNSPECIFIED,
+                                  NULL);
         if (!(pml4e & PG_PRESENT_MASK)) {
             /* not present */
             continue;
diff --git a/target-i386/bpt_helper.c b/target-i386/bpt_helper.c
new file mode 100644
index 0000000..6fd7fe0
--- /dev/null
+++ b/target-i386/bpt_helper.c
@@ -0,0 +1,328 @@
+/*
+ *  i386 breakpoint helpers
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
+#include "exec/helper-proto.h"
+
+
+#ifndef CONFIG_USER_ONLY
+static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 1;
+}
+
+static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 2;
+
+}
+static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return hw_global_breakpoint_enabled(dr7, index) ||
+           hw_local_breakpoint_enabled(dr7, index);
+}
+
+static inline int hw_breakpoint_type(unsigned long dr7, int index)
+{
+    return (dr7 >> (DR7_TYPE_SHIFT + (index * 4))) & 3;
+}
+
+static inline int hw_breakpoint_len(unsigned long dr7, int index)
+{
+    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 4))) & 3);
+    return (len == 2) ? 8 : len + 1;
+}
+
+static int hw_breakpoint_insert(CPUX86State *env, int index)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+    target_ulong dr7 = env->dr[7];
+    target_ulong drN = env->dr[index];
+    int err = 0;
+
+    switch (hw_breakpoint_type(dr7, index)) {
+    case DR7_TYPE_BP_INST:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_breakpoint_insert(cs, drN, BP_CPU,
+                                        &env->cpu_breakpoint[index]);
+        }
+        break;
+
+    case DR7_TYPE_IO_RW:
+        /* Notice when we should enable calls to bpt_io.  */
+        return hw_breakpoint_enabled(env->dr[7], index)
+               ? HF_IOBPT_MASK : 0;
+
+    case DR7_TYPE_DATA_WR:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_watchpoint_insert(cs, drN,
+                                        hw_breakpoint_len(dr7, index),
+                                        BP_CPU | BP_MEM_WRITE,
+                                        &env->cpu_watchpoint[index]);
+        }
+        break;
+
+    case DR7_TYPE_DATA_RW:
+        if (hw_breakpoint_enabled(dr7, index)) {
+            err = cpu_watchpoint_insert(cs, drN,
+                                        hw_breakpoint_len(dr7, index),
+                                        BP_CPU | BP_MEM_ACCESS,
+                                        &env->cpu_watchpoint[index]);
+        }
+        break;
+    }
+    if (err) {
+        env->cpu_breakpoint[index] = NULL;
+    }
+    return 0;
+}
+
+static void hw_breakpoint_remove(CPUX86State *env, int index)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+
+    switch (hw_breakpoint_type(env->dr[7], index)) {
+    case DR7_TYPE_BP_INST:
+        if (env->cpu_breakpoint[index]) {
+            cpu_breakpoint_remove_by_ref(cs, env->cpu_breakpoint[index]);
+            env->cpu_breakpoint[index] = NULL;
+        }
+        break;
+
+    case DR7_TYPE_DATA_WR:
+    case DR7_TYPE_DATA_RW:
+        if (env->cpu_breakpoint[index]) {
+            cpu_watchpoint_remove_by_ref(cs, env->cpu_watchpoint[index]);
+            env->cpu_breakpoint[index] = NULL;
+        }
+        break;
+
+    case DR7_TYPE_IO_RW:
+        /* HF_IOBPT_MASK cleared elsewhere.  */
+        break;
+    }
+}
+
+void cpu_x86_update_dr7(CPUX86State *env, uint32_t new_dr7)
+{
+    target_ulong old_dr7 = env->dr[7];
+    int iobpt = 0;
+    int i;
+
+    new_dr7 |= DR7_FIXED_1;
+
+    /* If nothing is changing except the global/local enable bits,
+       then we can make the change more efficient.  */
+    if (((old_dr7 ^ new_dr7) & ~0xff) == 0) {
+        /* Fold the global and local enable bits together into the
+           global fields, then xor to show which registers have
+           changed collective enable state.  */
+        int mod = ((old_dr7 | old_dr7 * 2) ^ (new_dr7 | new_dr7 * 2)) & 0xff;
+
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if ((mod & (2 << i * 2)) && !hw_breakpoint_enabled(new_dr7, i)) {
+                hw_breakpoint_remove(env, i);
+            }
+        }
+        env->dr[7] = new_dr7;
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            if (mod & (2 << i * 2) && hw_breakpoint_enabled(new_dr7, i)) {
+                iobpt |= hw_breakpoint_insert(env, i);
+            } else if (hw_breakpoint_type(new_dr7, i) == DR7_TYPE_IO_RW
+                       && hw_breakpoint_enabled(new_dr7, i)) {
+                iobpt |= HF_IOBPT_MASK;
+            }
+        }
+    } else {
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            hw_breakpoint_remove(env, i);
+        }
+        env->dr[7] = new_dr7;
+        for (i = 0; i < DR7_MAX_BP; i++) {
+            iobpt |= hw_breakpoint_insert(env, i);
+        }
+    }
+
+    env->hflags = (env->hflags & ~HF_IOBPT_MASK) | iobpt;
+}
+
+static bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
+{
+    target_ulong dr6;
+    int reg;
+    bool hit_enabled = false;
+
+    dr6 = env->dr[6] & ~0xf;
+    for (reg = 0; reg < DR7_MAX_BP; reg++) {
+        bool bp_match = false;
+        bool wp_match = false;
+
+        switch (hw_breakpoint_type(env->dr[7], reg)) {
+        case DR7_TYPE_BP_INST:
+            if (env->dr[reg] == env->eip) {
+                bp_match = true;
+            }
+            break;
+        case DR7_TYPE_DATA_WR:
+        case DR7_TYPE_DATA_RW:
+            if (env->cpu_watchpoint[reg] &&
+                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
+                wp_match = true;
+            }
+            break;
+        case DR7_TYPE_IO_RW:
+            break;
+        }
+        if (bp_match || wp_match) {
+            dr6 |= 1 << reg;
+            if (hw_breakpoint_enabled(env->dr[7], reg)) {
+                hit_enabled = true;
+            }
+        }
+    }
+
+    if (hit_enabled || force_dr6_update) {
+        env->dr[6] = dr6;
+    }
+
+    return hit_enabled;
+}
+
+void breakpoint_handler(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    CPUBreakpoint *bp;
+
+    if (cs->watchpoint_hit) {
+        if (cs->watchpoint_hit->flags & BP_CPU) {
+            cs->watchpoint_hit = NULL;
+            if (check_hw_breakpoints(env, false)) {
+                raise_exception(env, EXCP01_DB);
+            } else {
+                cpu_loop_exit_noexc(cs);
+            }
+        }
+    } else {
+        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
+            if (bp->pc == env->eip) {
+                if (bp->flags & BP_CPU) {
+                    check_hw_breakpoints(env, true);
+                    raise_exception(env, EXCP01_DB);
+                }
+                break;
+            }
+        }
+    }
+}
+#endif
+
+void helper_single_step(CPUX86State *env)
+{
+#ifndef CONFIG_USER_ONLY
+    check_hw_breakpoints(env, true);
+    env->dr[6] |= DR6_BS;
+#endif
+    raise_exception(env, EXCP01_DB);
+}
+
+void helper_set_dr(CPUX86State *env, int reg, target_ulong t0)
+{
+#ifndef CONFIG_USER_ONLY
+    switch (reg) {
+    case 0: case 1: case 2: case 3:
+        if (hw_breakpoint_enabled(env->dr[7], reg)
+            && hw_breakpoint_type(env->dr[7], reg) != DR7_TYPE_IO_RW) {
+            hw_breakpoint_remove(env, reg);
+            env->dr[reg] = t0;
+            hw_breakpoint_insert(env, reg);
+        } else {
+            env->dr[reg] = t0;
+        }
+        return;
+    case 4:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        }
+        /* fallthru */
+    case 6:
+        env->dr[6] = t0 | DR6_FIXED_1;
+        return;
+    case 5:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        }
+        /* fallthru */
+    case 7:
+        cpu_x86_update_dr7(env, t0);
+        return;
+    }
+    raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+#endif
+}
+
+target_ulong helper_get_dr(CPUX86State *env, int reg)
+{
+    switch (reg) {
+    case 0: case 1: case 2: case 3: case 6: case 7:
+        return env->dr[reg];
+    case 4:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        } else {
+            return env->dr[6];
+        }
+    case 5:
+        if (env->cr[4] & CR4_DE_MASK) {
+            break;
+        } else {
+            return env->dr[7];
+        }
+    }
+    raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+}
+
+/* Check if Port I/O is trapped by a breakpoint.  */
+void helper_bpt_io(CPUX86State *env, uint32_t port,
+                   uint32_t size, target_ulong next_eip)
+{
+#ifndef CONFIG_USER_ONLY
+    target_ulong dr7 = env->dr[7];
+    int i, hit = 0;
+
+    for (i = 0; i < DR7_MAX_BP; ++i) {
+        if (hw_breakpoint_type(dr7, i) == DR7_TYPE_IO_RW
+            && hw_breakpoint_enabled(dr7, i)) {
+            int bpt_len = hw_breakpoint_len(dr7, i);
+            if (port + size - 1 >= env->dr[i]
+                && port <= env->dr[i] + bpt_len - 1) {
+                hit |= 1 << i;
+            }
+        }
+    }
+
+    if (hit) {
+        env->dr[6] = (env->dr[6] & ~0xf) | hit;
+        env->eip = next_eip;
+        raise_exception(env, EXCP01_DB);
+    }
+#endif
+}
diff --git a/target-i386/cc_helper.c b/target-i386/cc_helper.c
index ecbf0ec..83af223 100644
--- a/target-i386/cc_helper.c
+++ b/target-i386/cc_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -378,17 +379,7 @@
 {
     env->eflags |= VIF_MASK;
     if (env->eflags & VIP_MASK) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
 }
 #endif
-
-void helper_set_inhibit_irq(CPUX86State *env)
-{
-    env->hflags |= HF_INHIBIT_IRQ_MASK;
-}
-
-void helper_reset_inhibit_irq(CPUX86State *env)
-{
-    env->hflags &= ~HF_INHIBIT_IRQ_MASK;
-}
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index b557b61..5dde658 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -21,8 +21,7 @@
 #define QEMU_I386_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
-#include "qapi/error.h"
+#include "qemu/notify.h"
 
 #ifdef TARGET_X86_64
 #define TYPE_X86_CPU "x86_64-cpu"
@@ -68,91 +67,6 @@
     void (*parent_reset)(CPUState *cpu);
 } X86CPUClass;
 
-/**
- * X86CPU:
- * @env: #CPUX86State
- * @migratable: If set, only migratable flags will be accepted when "enforce"
- * mode is used, and only migratable flags will be included in the "host"
- * CPU model.
- *
- * An x86 CPU.
- */
-typedef struct X86CPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUX86State env;
-
-    bool hyperv_vapic;
-    bool hyperv_relaxed_timing;
-    int hyperv_spinlock_attempts;
-    bool hyperv_time;
-    bool check_cpuid;
-    bool enforce_cpuid;
-    bool expose_kvm;
-    bool migratable;
-    bool host_features;
-
-    /* if true the CPUID code directly forward host cache leaves to the guest */
-    bool cache_info_passthrough;
-
-    /* Features that were filtered out because of missing host capabilities */
-    uint32_t filtered_features[FEATURE_WORDS];
-
-    /* Enable PMU CPUID bits. This can't be enabled by default yet because
-     * it doesn't have ABI stability guarantees, as it passes all PMU CPUID
-     * bits returned by GET_SUPPORTED_CPUID (that depend on host CPU and kernel
-     * capabilities) directly to the guest.
-     */
-    bool enable_pmu;
-
-    /* in order to simplify APIC support, we leave this pointer to the
-       user */
-    struct DeviceState *apic_state;
-} X86CPU;
-
-static inline X86CPU *x86_env_get_cpu(CPUX86State *env)
-{
-    return container_of(env, X86CPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(x86_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(X86CPU, env)
-
-#ifndef CONFIG_USER_ONLY
-extern struct VMStateDescription vmstate_x86_cpu;
-#endif
-
-/**
- * x86_cpu_do_interrupt:
- * @cpu: vCPU the interrupt is to be handled by.
- */
-void x86_cpu_do_interrupt(CPUState *cpu);
-bool x86_cpu_exec_interrupt(CPUState *cpu, int int_req);
-
-int x86_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cpu,
-                             int cpuid, void *opaque);
-int x86_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cpu,
-                             int cpuid, void *opaque);
-int x86_cpu_write_elf64_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
-                                 void *opaque);
-int x86_cpu_write_elf32_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
-                                 void *opaque);
-
-void x86_cpu_get_memory_mapping(CPUState *cpu, MemoryMappingList *list,
-                                Error **errp);
-
-void x86_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
-                        int flags);
-
-hwaddr x86_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-
-int x86_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int x86_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-
-void x86_cpu_exec_enter(CPUState *cpu);
-void x86_cpu_exec_exit(CPUState *cpu);
+typedef struct X86CPU X86CPU;
 
 #endif
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 44791f2..14f2dbd 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -16,17 +16,16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
+#include "qemu/cutils.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "sysemu/kvm.h"
 #include "sysemu/cpus.h"
 #include "kvm_i386.h"
-#include "topology.h"
 
+#include "qemu/error-report.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qapi/qmp/qerror.h"
@@ -36,15 +35,16 @@
 #include "qapi/visitor.h"
 #include "sysemu/arch_init.h"
 
-#include "hw/hw.h"
 #if defined(CONFIG_KVM)
 #include <linux/kvm_para.h>
 #endif
 
 #include "sysemu/sysemu.h"
 #include "hw/qdev-properties.h"
-#include "hw/cpu/icc_bus.h"
+#include "hw/i386/topology.h"
 #ifndef CONFIG_USER_ONLY
+#include "exec/address-spaces.h"
+#include "hw/hw.h"
 #include "hw/xen/xen.h"
 #include "hw/i386/apic_internal.h"
 #endif
@@ -245,6 +245,47 @@
     NULL, NULL, NULL, NULL,
 };
 
+static const char *hyperv_priv_feature_name[] = {
+    NULL /* hv_msr_vp_runtime_access */, NULL /* hv_msr_time_refcount_access */,
+    NULL /* hv_msr_synic_access */, NULL /* hv_msr_stimer_access */,
+    NULL /* hv_msr_apic_access */, NULL /* hv_msr_hypercall_access */,
+    NULL /* hv_vpindex_access */, NULL /* hv_msr_reset_access */,
+    NULL /* hv_msr_stats_access */, NULL /* hv_reftsc_access */,
+    NULL /* hv_msr_idle_access */, NULL /* hv_msr_frequency_access */,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
+static const char *hyperv_ident_feature_name[] = {
+    NULL /* hv_create_partitions */, NULL /* hv_access_partition_id */,
+    NULL /* hv_access_memory_pool */, NULL /* hv_adjust_message_buffers */,
+    NULL /* hv_post_messages */, NULL /* hv_signal_events */,
+    NULL /* hv_create_port */, NULL /* hv_connect_port */,
+    NULL /* hv_access_stats */, NULL, NULL, NULL /* hv_debugging */,
+    NULL /* hv_cpu_power_management */, NULL /* hv_configure_profiler */,
+    NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
+static const char *hyperv_misc_feature_name[] = {
+    NULL /* hv_mwait */, NULL /* hv_guest_debugging */,
+    NULL /* hv_perf_monitor */, NULL /* hv_cpu_dynamic_part */,
+    NULL /* hv_hypercall_params_xmm */, NULL /* hv_guest_idle_state */,
+    NULL, NULL,
+    NULL, NULL, NULL /* hv_guest_crash_msr */, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
 static const char *svm_feature_name[] = {
     "npt", "lbrv", "svm_lock", "nrip_save",
     "tsc_scale", "vmcb_clean",  "flushbyasid", "decodeassists",
@@ -259,8 +300,19 @@
 static const char *cpuid_7_0_ebx_feature_name[] = {
     "fsgsbase", "tsc_adjust", NULL, "bmi1", "hle", "avx2", NULL, "smep",
     "bmi2", "erms", "invpcid", "rtm", NULL, NULL, "mpx", NULL,
-    "avx512f", NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
-    NULL, NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
+    "avx512f", NULL, "rdseed", "adx", "smap", NULL, "pcommit", "clflushopt",
+    "clwb", NULL, "avx512pf", "avx512er", "avx512cd", NULL, NULL, NULL,
+};
+
+static const char *cpuid_7_0_ecx_feature_name[] = {
+    NULL, NULL, "umip", "pku",
+    "ospke", NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, "rdpid", NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
 };
 
 static const char *cpuid_apm_edx_feature_name[] = {
@@ -274,6 +326,28 @@
     NULL, NULL, NULL, NULL,
 };
 
+static const char *cpuid_xsave_feature_name[] = {
+    "xsaveopt", "xsavec", "xgetbv1", "xsaves",
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
+static const char *cpuid_6_feature_name[] = {
+    NULL, NULL, "arat", NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+};
+
 #define I486_FEATURES (CPUID_FP87 | CPUID_VME | CPUID_PSE)
 #define PENTIUM_FEATURES (I486_FEATURES | CPUID_DE | CPUID_TSC | \
           CPUID_MSR | CPUID_MCE | CPUID_CX8 | CPUID_MMX | CPUID_APIC)
@@ -290,7 +364,7 @@
           CPUID_PAE | CPUID_MCE | CPUID_CX8 | CPUID_APIC | CPUID_SEP | \
           CPUID_MTRR | CPUID_PGE | CPUID_MCA | CPUID_CMOV | CPUID_PAT | \
           CPUID_PSE36 | CPUID_CLFLUSH | CPUID_ACPI | CPUID_MMX | \
-          CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS)
+          CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS | CPUID_DE)
           /* partly implemented:
           CPUID_MTRR, CPUID_MCA, CPUID_CLFLUSH (needed for Win64) */
           /* missing:
@@ -298,14 +372,14 @@
 #define TCG_EXT_FEATURES (CPUID_EXT_SSE3 | CPUID_EXT_PCLMULQDQ | \
           CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 | CPUID_EXT_CX16 | \
           CPUID_EXT_SSE41 | CPUID_EXT_SSE42 | CPUID_EXT_POPCNT | \
+          CPUID_EXT_XSAVE | /* CPUID_EXT_OSXSAVE is dynamic */   \
           CPUID_EXT_MOVBE | CPUID_EXT_AES | CPUID_EXT_HYPERVISOR)
           /* missing:
           CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_VMX, CPUID_EXT_SMX,
           CPUID_EXT_EST, CPUID_EXT_TM2, CPUID_EXT_CID, CPUID_EXT_FMA,
           CPUID_EXT_XTPR, CPUID_EXT_PDCM, CPUID_EXT_PCID, CPUID_EXT_DCA,
-          CPUID_EXT_X2APIC, CPUID_EXT_TSC_DEADLINE_TIMER, CPUID_EXT_XSAVE,
-          CPUID_EXT_OSXSAVE, CPUID_EXT_AVX, CPUID_EXT_F16C,
-          CPUID_EXT_RDRAND */
+          CPUID_EXT_X2APIC, CPUID_EXT_TSC_DEADLINE_TIMER, CPUID_EXT_AVX,
+          CPUID_EXT_F16C, CPUID_EXT_RDRAND */
 
 #ifdef TARGET_X86_64
 #define TCG_EXT2_X86_64_FEATURES (CPUID_EXT2_SYSCALL | CPUID_EXT2_LM)
@@ -323,13 +397,20 @@
 #define TCG_SVM_FEATURES 0
 #define TCG_KVM_FEATURES 0
 #define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP | \
-          CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX)
+          CPUID_7_0_EBX_BMI1 | CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ADX | \
+          CPUID_7_0_EBX_PCOMMIT | CPUID_7_0_EBX_CLFLUSHOPT |            \
+          CPUID_7_0_EBX_CLWB | CPUID_7_0_EBX_MPX | CPUID_7_0_EBX_FSGSBASE | \
+          CPUID_7_0_EBX_ERMS)
           /* missing:
-          CPUID_7_0_EBX_FSGSBASE, CPUID_7_0_EBX_HLE, CPUID_7_0_EBX_AVX2,
-          CPUID_7_0_EBX_ERMS, CPUID_7_0_EBX_INVPCID, CPUID_7_0_EBX_RTM,
+          CPUID_7_0_EBX_HLE, CPUID_7_0_EBX_AVX2,
+          CPUID_7_0_EBX_INVPCID, CPUID_7_0_EBX_RTM,
           CPUID_7_0_EBX_RDSEED */
+#define TCG_7_0_ECX_FEATURES (CPUID_7_0_ECX_PKU | CPUID_7_0_ECX_OSPKE)
 #define TCG_APM_FEATURES 0
-
+#define TCG_6_EAX_FEATURES CPUID_6_EAX_ARAT
+#define TCG_XSAVE_FEATURES (CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XGETBV1)
+          /* missing:
+          CPUID_XSAVE_XSAVEC, CPUID_XSAVE_XSAVES */
 
 typedef struct FeatureWordInfo {
     const char **feat_names;
@@ -372,6 +453,18 @@
         .cpuid_eax = KVM_CPUID_FEATURES, .cpuid_reg = R_EAX,
         .tcg_features = TCG_KVM_FEATURES,
     },
+    [FEAT_HYPERV_EAX] = {
+        .feat_names = hyperv_priv_feature_name,
+        .cpuid_eax = 0x40000003, .cpuid_reg = R_EAX,
+    },
+    [FEAT_HYPERV_EBX] = {
+        .feat_names = hyperv_ident_feature_name,
+        .cpuid_eax = 0x40000003, .cpuid_reg = R_EBX,
+    },
+    [FEAT_HYPERV_EDX] = {
+        .feat_names = hyperv_misc_feature_name,
+        .cpuid_eax = 0x40000003, .cpuid_reg = R_EDX,
+    },
     [FEAT_SVM] = {
         .feat_names = svm_feature_name,
         .cpuid_eax = 0x8000000A, .cpuid_reg = R_EDX,
@@ -384,6 +477,13 @@
         .cpuid_reg = R_EBX,
         .tcg_features = TCG_7_0_EBX_FEATURES,
     },
+    [FEAT_7_0_ECX] = {
+        .feat_names = cpuid_7_0_ecx_feature_name,
+        .cpuid_eax = 7,
+        .cpuid_needs_ecx = true, .cpuid_ecx = 0,
+        .cpuid_reg = R_ECX,
+        .tcg_features = TCG_7_0_ECX_FEATURES,
+    },
     [FEAT_8000_0007_EDX] = {
         .feat_names = cpuid_apm_edx_feature_name,
         .cpuid_eax = 0x80000007,
@@ -391,6 +491,18 @@
         .tcg_features = TCG_APM_FEATURES,
         .unmigratable_flags = CPUID_APM_INVTSC,
     },
+    [FEAT_XSAVE] = {
+        .feat_names = cpuid_xsave_feature_name,
+        .cpuid_eax = 0xd,
+        .cpuid_needs_ecx = true, .cpuid_ecx = 1,
+        .cpuid_reg = R_EAX,
+        .tcg_features = TCG_XSAVE_FEATURES,
+    },
+    [FEAT_6_EAX] = {
+        .feat_names = cpuid_6_feature_name,
+        .cpuid_eax = 6, .cpuid_reg = R_EAX,
+        .tcg_features = TCG_6_EAX_FEATURES,
+    },
 };
 
 typedef struct X86RegisterInfo32 {
@@ -414,24 +526,35 @@
 };
 #undef REGISTER
 
-typedef struct ExtSaveArea {
-    uint32_t feature, bits;
-    uint32_t offset, size;
-} ExtSaveArea;
-
-static const ExtSaveArea ext_save_areas[] = {
-    [2] = { .feature = FEAT_1_ECX, .bits = CPUID_EXT_AVX,
-            .offset = 0x240, .size = 0x100 },
-    [3] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
-            .offset = 0x3c0, .size = 0x40  },
-    [4] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
-            .offset = 0x400, .size = 0x40  },
-    [5] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
-            .offset = 0x440, .size = 0x40 },
-    [6] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
-            .offset = 0x480, .size = 0x200 },
-    [7] = { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
-            .offset = 0x680, .size = 0x400 },
+const ExtSaveArea x86_ext_save_areas[] = {
+    [XSTATE_YMM_BIT] =
+          { .feature = FEAT_1_ECX, .bits = CPUID_EXT_AVX,
+            .offset = offsetof(X86XSaveArea, avx_state),
+            .size = sizeof(XSaveAVX) },
+    [XSTATE_BNDREGS_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
+            .offset = offsetof(X86XSaveArea, bndreg_state),
+            .size = sizeof(XSaveBNDREG)  },
+    [XSTATE_BNDCSR_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_MPX,
+            .offset = offsetof(X86XSaveArea, bndcsr_state),
+            .size = sizeof(XSaveBNDCSR)  },
+    [XSTATE_OPMASK_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+            .offset = offsetof(X86XSaveArea, opmask_state),
+            .size = sizeof(XSaveOpmask) },
+    [XSTATE_ZMM_Hi256_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+            .offset = offsetof(X86XSaveArea, zmm_hi256_state),
+            .size = sizeof(XSaveZMM_Hi256) },
+    [XSTATE_Hi16_ZMM_BIT] =
+          { .feature = FEAT_7_0_EBX, .bits = CPUID_7_0_EBX_AVX512F,
+            .offset = offsetof(X86XSaveArea, hi16_zmm_state),
+            .size = sizeof(XSaveHi16_ZMM) },
+    [XSTATE_PKRU_BIT] =
+          { .feature = FEAT_7_0_ECX, .bits = CPUID_7_0_ECX_PKU,
+            .offset = offsetof(X86XSaveArea, pkru_state),
+            .size = sizeof(XSavePKRU) },
 };
 
 const char *get_register_name_32(unsigned int reg)
@@ -442,38 +565,6 @@
     return x86_reg_info_32[reg].name;
 }
 
-/* KVM-specific features that are automatically added to all CPU models
- * when KVM is enabled.
- */
-static uint32_t kvm_default_features[FEATURE_WORDS] = {
-    [FEAT_KVM] = (1 << KVM_FEATURE_CLOCKSOURCE) |
-        (1 << KVM_FEATURE_NOP_IO_DELAY) |
-        (1 << KVM_FEATURE_CLOCKSOURCE2) |
-        (1 << KVM_FEATURE_ASYNC_PF) |
-        (1 << KVM_FEATURE_STEAL_TIME) |
-        (1 << KVM_FEATURE_PV_EOI) |
-        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT),
-    [FEAT_1_ECX] = CPUID_EXT_X2APIC,
-};
-
-/* Features that are not added by default to any CPU model when KVM is enabled.
- */
-static uint32_t kvm_default_unset_features[FEATURE_WORDS] = {
-    [FEAT_1_EDX] = CPUID_ACPI,
-    [FEAT_1_ECX] = CPUID_EXT_MONITOR,
-    [FEAT_8000_0001_ECX] = CPUID_EXT3_SVM,
-};
-
-void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features)
-{
-    kvm_default_features[w] &= ~features;
-}
-
-void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features)
-{
-    kvm_default_unset_features[w] &= ~features;
-}
-
 /*
  * Returns the set of feature flags that are supported and migratable by
  * QEMU, for a given FeatureWord.
@@ -641,6 +732,14 @@
     return oc;
 }
 
+static char *x86_cpu_class_get_model_name(X86CPUClass *cc)
+{
+    const char *class_name = object_class_get_name(OBJECT_CLASS(cc));
+    assert(g_str_has_suffix(class_name, X86_CPU_TYPE_SUFFIX));
+    return g_strndup(class_name,
+                     strlen(class_name) - strlen(X86_CPU_TYPE_SUFFIX));
+}
+
 struct X86CPUDefinition {
     const char *name;
     uint32_t level;
@@ -653,13 +752,12 @@
     int stepping;
     FeatureWordArray features;
     char model_id[48];
-    bool cache_info_passthrough;
 };
 
 static X86CPUDefinition builtin_x86_defs[] = {
     {
         .name = "qemu64",
-        .level = 4,
+        .level = 0xd,
         .vendor = CPUID_VENDOR_AMD,
         .family = 6,
         .model = 6,
@@ -669,16 +767,15 @@
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
             CPUID_PSE36,
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT,
+            CPUID_EXT_SSE3 | CPUID_EXT_CX16,
         .features[FEAT_8000_0001_EDX] =
-            (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
-            CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
+            CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM,
         .xlevel = 0x8000000A,
+        .model_id = "QEMU Virtual CPU version " QEMU_HW_VERSION,
     },
-#ifdef CONFIG_ANDROID
+// ANDROID_BEGIN
     {
         /* A variant of qemu64 for Android emulation.
          * According to Table 1 of NDK Programmer's Guide, section "ABI
@@ -688,7 +785,7 @@
          *  SSSE3, SSE4.1, SSE4.2
          */
         .name = "android64",
-        .level = 4,  /* may be modified later if KVM is enabled */
+        .level = 4,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 6,
@@ -706,10 +803,10 @@
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
             CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
-        .xlevel = 0x8000000A,  /* may be modified later if KVM is enabled */
+        .xlevel = 0x8000000A,
         .model_id = "Android virtual processor"
     },
-#endif
+// ANDROID_END
     {
         .name = "phenom",
         .level = 5,
@@ -726,7 +823,6 @@
             CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_CX16 |
             CPUID_EXT_POPCNT,
         .features[FEAT_8000_0001_EDX] =
-            (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX |
             CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT | CPUID_EXT2_MMXEXT |
             CPUID_EXT2_FFXSR | CPUID_EXT2_PDPE1GB | CPUID_EXT2_RDTSCP,
@@ -769,14 +865,14 @@
     },
     {
         .name = "kvm64",
-        .level = 5,
+        .level = 0xd,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 15,
         .model = 6,
         .stepping = 1,
-        /* Missing: CPUID_VME, CPUID_HT */
+        /* Missing: CPUID_HT */
         .features[FEAT_1_EDX] =
-            PPRO_FEATURES |
+            PPRO_FEATURES | CPUID_VME |
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
             CPUID_PSE36,
         /* Missing: CPUID_EXT_POPCNT, CPUID_EXT_MONITOR */
@@ -784,7 +880,6 @@
             CPUID_EXT_SSE3 | CPUID_EXT_CX16,
         /* Missing: CPUID_EXT2_PDPE1GB, CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         /* Missing: CPUID_EXT3_LAHF_LM, CPUID_EXT3_CMP_LEG, CPUID_EXT3_EXTAPIC,
                     CPUID_EXT3_CR8LEG, CPUID_EXT3_ABM, CPUID_EXT3_SSE4A,
@@ -805,10 +900,11 @@
         .features[FEAT_1_EDX] =
             PPRO_FEATURES,
         .features[FEAT_1_ECX] =
-            CPUID_EXT_SSE3 | CPUID_EXT_POPCNT,
+            CPUID_EXT_SSE3,
         .xlevel = 0x80000004,
+        .model_id = "QEMU Virtual CPU version " QEMU_HW_VERSION,
     },
-#ifdef CONFIG_ANDROID
+// ANDROID_BEGIN
     {
         /* A variant of qemu32 for Android emulation.
          * According to Table 1 of NDK Programmer's Guide, section "ABI
@@ -820,7 +916,7 @@
          * required by Android and usually comes alongside SSE4.2.
          */
         .name = "android32",
-        .level = 4,  /* may be modified later if KVM is enabled */
+        .level = 4,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 6,
@@ -829,10 +925,10 @@
             PPRO_FEATURES,
         .features[FEAT_1_ECX] =
             CPUID_EXT_SSE3 | CPUID_EXT_SSSE3,
-        .xlevel = 0x80000004,  /* may be modified later if KVM is enabled */
+        .xlevel = 0x80000004,
         .model_id = "Android 32-bit virtual processor"
     },
-#endif
+// ANDROID_END
     {
         .name = "kvm32",
         .level = 5,
@@ -841,12 +937,10 @@
         .model = 6,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            PPRO_FEATURES |
+            PPRO_FEATURES | CPUID_VME |
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_PSE36,
         .features[FEAT_1_ECX] =
             CPUID_EXT_SSE3,
-        .features[FEAT_8000_0001_EDX] =
-            PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES,
         .features[FEAT_8000_0001_ECX] =
             0,
         .xlevel = 0x80000008,
@@ -908,7 +1002,7 @@
     },
     {
         .name = "pentium3",
-        .level = 2,
+        .level = 3,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 7,
@@ -928,14 +1022,13 @@
             PPRO_FEATURES | CPUID_PSE36 | CPUID_VME | CPUID_MTRR |
             CPUID_MCA,
         .features[FEAT_8000_0001_EDX] =
-            (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_MMXEXT | CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT,
         .xlevel = 0x80000008,
+        .model_id = "QEMU Virtual CPU version " QEMU_HW_VERSION,
     },
     {
         .name = "n270",
-        /* original is on level 10 */
-        .level = 5,
+        .level = 10,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 28,
@@ -952,22 +1045,21 @@
             CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 |
             CPUID_EXT_MOVBE,
         .features[FEAT_8000_0001_EDX] =
-            (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x8000000A,
+        .xlevel = 0x80000008,
         .model_id = "Intel(R) Atom(TM) CPU N270   @ 1.60GHz",
     },
     {
         .name = "Conroe",
-        .level = 4,
+        .level = 10,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 15,
         .stepping = 3,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -978,18 +1070,18 @@
             CPUID_EXT2_LM | CPUID_EXT2_NX | CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x8000000A,
+        .xlevel = 0x80000008,
         .model_id = "Intel Celeron_4x0 (Conroe/Merom Class Core 2)",
     },
     {
         .name = "Penryn",
-        .level = 4,
+        .level = 10,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 23,
         .stepping = 3,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1001,18 +1093,18 @@
             CPUID_EXT2_LM | CPUID_EXT2_NX | CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x8000000A,
+        .xlevel = 0x80000008,
         .model_id = "Intel Core 2 Duo P9xxx (Penryn Class Core 2)",
     },
     {
         .name = "Nehalem",
-        .level = 4,
+        .level = 11,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 26,
         .stepping = 3,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1024,7 +1116,7 @@
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x8000000A,
+        .xlevel = 0x80000008,
         .model_id = "Intel Core i7 9xx (Nehalem Class Core i7)",
     },
     {
@@ -1035,7 +1127,7 @@
         .model = 44,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1048,7 +1140,9 @@
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x8000000A,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
         .model_id = "Westmere E56xx/L56xx/X56xx (Nehalem-C)",
     },
     {
@@ -1059,7 +1153,7 @@
         .model = 42,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1075,18 +1169,56 @@
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
             CPUID_EXT3_LAHF_LM,
-        .xlevel = 0x8000000A,
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
         .model_id = "Intel Xeon E312xx (Sandy Bridge)",
     },
     {
-        .name = "Haswell",
+        .name = "IvyBridge",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 58,
+        .stepping = 9,
+        .features[FEAT_1_EDX] =
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+            CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+            CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+            CPUID_DE | CPUID_FP87,
+        .features[FEAT_1_ECX] =
+            CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+            CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_POPCNT |
+            CPUID_EXT_X2APIC | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+            CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
+            CPUID_EXT_SSE3 | CPUID_EXT_F16C | CPUID_EXT_RDRAND,
+        .features[FEAT_7_0_EBX] =
+            CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_SMEP |
+            CPUID_7_0_EBX_ERMS,
+        .features[FEAT_8000_0001_EDX] =
+            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+            CPUID_EXT2_SYSCALL,
+        .features[FEAT_8000_0001_ECX] =
+            CPUID_EXT3_LAHF_LM,
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
+        .model_id = "Intel Xeon E3-12xx v2 (Ivy Bridge)",
+    },
+    {
+        .name = "Haswell-noTSX",
         .level = 0xd,
         .vendor = CPUID_VENDOR_INTEL,
         .family = 6,
         .model = 60,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1097,21 +1229,98 @@
             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
             CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
             CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
-            CPUID_EXT_PCID,
+            CPUID_EXT_PCID | CPUID_EXT_F16C | CPUID_EXT_RDRAND,
         .features[FEAT_8000_0001_EDX] =
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM,
+        .features[FEAT_7_0_EBX] =
+            CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
+            CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
+            CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID,
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
+        .model_id = "Intel Core Processor (Haswell, no TSX)",
+    },    {
+        .name = "Haswell",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 60,
+        .stepping = 1,
+        .features[FEAT_1_EDX] =
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+            CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+            CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+            CPUID_DE | CPUID_FP87,
+        .features[FEAT_1_ECX] =
+            CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+            CPUID_EXT_POPCNT | CPUID_EXT_X2APIC | CPUID_EXT_SSE42 |
+            CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+            CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
+            CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
+            CPUID_EXT_PCID | CPUID_EXT_F16C | CPUID_EXT_RDRAND,
+        .features[FEAT_8000_0001_EDX] =
+            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+            CPUID_EXT2_SYSCALL,
+        .features[FEAT_8000_0001_ECX] =
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
             CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID |
             CPUID_7_0_EBX_RTM,
-        .xlevel = 0x8000000A,
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
         .model_id = "Intel Core Processor (Haswell)",
     },
     {
+        .name = "Broadwell-noTSX",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 61,
+        .stepping = 2,
+        .features[FEAT_1_EDX] =
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+            CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+            CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+            CPUID_DE | CPUID_FP87,
+        .features[FEAT_1_ECX] =
+            CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+            CPUID_EXT_POPCNT | CPUID_EXT_X2APIC | CPUID_EXT_SSE42 |
+            CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+            CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
+            CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
+            CPUID_EXT_PCID | CPUID_EXT_F16C | CPUID_EXT_RDRAND,
+        .features[FEAT_8000_0001_EDX] =
+            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+            CPUID_EXT2_SYSCALL,
+        .features[FEAT_8000_0001_ECX] =
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+        .features[FEAT_7_0_EBX] =
+            CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
+            CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
+            CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID |
+            CPUID_7_0_EBX_RDSEED | CPUID_7_0_EBX_ADX |
+            CPUID_7_0_EBX_SMAP,
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
+        .model_id = "Intel Core Processor (Broadwell, no TSX)",
+    },
+    {
         .name = "Broadwell",
         .level = 0xd,
         .vendor = CPUID_VENDOR_INTEL,
@@ -1119,7 +1328,7 @@
         .model = 61,
         .stepping = 2,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1130,22 +1339,71 @@
             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
             CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
             CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
-            CPUID_EXT_PCID,
+            CPUID_EXT_PCID | CPUID_EXT_F16C | CPUID_EXT_RDRAND,
         .features[FEAT_8000_0001_EDX] =
             CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
             CPUID_EXT2_SYSCALL,
         .features[FEAT_8000_0001_ECX] =
-            CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
         .features[FEAT_7_0_EBX] =
             CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
             CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
             CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID |
             CPUID_7_0_EBX_RTM | CPUID_7_0_EBX_RDSEED | CPUID_7_0_EBX_ADX |
             CPUID_7_0_EBX_SMAP,
-        .xlevel = 0x8000000A,
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
         .model_id = "Intel Core Processor (Broadwell)",
     },
     {
+        .name = "Skylake-Client",
+        .level = 0xd,
+        .vendor = CPUID_VENDOR_INTEL,
+        .family = 6,
+        .model = 94,
+        .stepping = 3,
+        .features[FEAT_1_EDX] =
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+            CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+            CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+            CPUID_DE | CPUID_FP87,
+        .features[FEAT_1_ECX] =
+            CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+            CPUID_EXT_POPCNT | CPUID_EXT_X2APIC | CPUID_EXT_SSE42 |
+            CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+            CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
+            CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
+            CPUID_EXT_PCID | CPUID_EXT_F16C | CPUID_EXT_RDRAND,
+        .features[FEAT_8000_0001_EDX] =
+            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+            CPUID_EXT2_SYSCALL,
+        .features[FEAT_8000_0001_ECX] =
+            CPUID_EXT3_ABM | CPUID_EXT3_LAHF_LM | CPUID_EXT3_3DNOWPREFETCH,
+        .features[FEAT_7_0_EBX] =
+            CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
+            CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
+            CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID |
+            CPUID_7_0_EBX_RTM | CPUID_7_0_EBX_RDSEED | CPUID_7_0_EBX_ADX |
+            CPUID_7_0_EBX_SMAP | CPUID_7_0_EBX_MPX,
+        /* Missing: XSAVES (not supported by some Linux versions,
+         * including v4.1 to v4.6).
+         * KVM doesn't yet expose any XSAVES state save component,
+         * and the only one defined in Skylake (processor tracing)
+         * probably will block migration anyway.
+         */
+        .features[FEAT_XSAVE] =
+            CPUID_XSAVE_XSAVEOPT | CPUID_XSAVE_XSAVEC |
+            CPUID_XSAVE_XGETBV1,
+        .features[FEAT_6_EAX] =
+            CPUID_6_EAX_ARAT,
+        .xlevel = 0x80000008,
+        .model_id = "Intel Core Processor (Skylake)",
+    },
+    {
         .name = "Opteron_G1",
         .level = 5,
         .vendor = CPUID_VENDOR_AMD,
@@ -1153,7 +1411,7 @@
         .model = 6,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1178,15 +1436,16 @@
         .model = 6,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
             CPUID_DE | CPUID_FP87,
         .features[FEAT_1_ECX] =
             CPUID_EXT_CX16 | CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+            CPUID_EXT2_LM | CPUID_EXT2_FXSR |
             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
@@ -1206,7 +1465,7 @@
         .model = 6,
         .stepping = 1,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1214,8 +1473,9 @@
         .features[FEAT_1_ECX] =
             CPUID_EXT_POPCNT | CPUID_EXT_CX16 | CPUID_EXT_MONITOR |
             CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+            CPUID_EXT2_LM | CPUID_EXT2_FXSR |
             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
@@ -1236,7 +1496,7 @@
         .model = 1,
         .stepping = 2,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1246,8 +1506,9 @@
             CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
             CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+            CPUID_EXT2_LM |
             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
@@ -1259,6 +1520,7 @@
             CPUID_EXT3_3DNOWPREFETCH | CPUID_EXT3_MISALIGNSSE |
             CPUID_EXT3_SSE4A | CPUID_EXT3_ABM | CPUID_EXT3_SVM |
             CPUID_EXT3_LAHF_LM,
+        /* no xsaveopt! */
         .xlevel = 0x8000001A,
         .model_id = "AMD Opteron 62xx class CPU",
     },
@@ -1270,7 +1532,7 @@
         .model = 2,
         .stepping = 0,
         .features[FEAT_1_EDX] =
-            CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+            CPUID_VME | CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
@@ -1280,8 +1542,9 @@
             CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_FMA |
             CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3,
+        /* Missing: CPUID_EXT2_RDTSCP */
         .features[FEAT_8000_0001_EDX] =
-            CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+            CPUID_EXT2_LM |
             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
@@ -1293,35 +1556,47 @@
             CPUID_EXT3_3DNOWPREFETCH | CPUID_EXT3_MISALIGNSSE |
             CPUID_EXT3_SSE4A | CPUID_EXT3_ABM | CPUID_EXT3_SVM |
             CPUID_EXT3_LAHF_LM,
+        /* no xsaveopt! */
         .xlevel = 0x8000001A,
         .model_id = "AMD Opteron 63xx class CPU",
     },
 };
 
-/**
- * x86_cpu_compat_set_features:
- * @cpu_model: CPU model name to be changed. If NULL, all CPU models are changed
- * @w: Identifies the feature word to be changed.
- * @feat_add: Feature bits to be added to feature word
- * @feat_remove: Feature bits to be removed from feature word
- *
- * Change CPU model feature bits for compatibility.
- *
- * This function may be used by machine-type compatibility functions
- * to enable or disable feature bits on specific CPU models.
+typedef struct PropValue {
+    const char *prop, *value;
+} PropValue;
+
+/* KVM-specific features that are automatically added/removed
+ * from all CPU models when KVM is enabled.
  */
-void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
-                                 uint32_t feat_add, uint32_t feat_remove)
+static PropValue kvm_default_props[] = {
+    { "kvmclock", "on" },
+    { "kvm-nopiodelay", "on" },
+    { "kvm-asyncpf", "on" },
+    { "kvm-steal-time", "on" },
+    { "kvm-pv-eoi", "on" },
+    { "kvmclock-stable-bit", "on" },
+    { "x2apic", "on" },
+    { "acpi", "off" },
+    { "monitor", "off" },
+    { "svm", "off" },
+    { NULL, NULL },
+};
+
+void x86_cpu_change_kvm_default(const char *prop, const char *value)
 {
-    X86CPUDefinition *def;
-    int i;
-    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); i++) {
-        def = &builtin_x86_defs[i];
-        if (!cpu_model || !strcmp(cpu_model, def->name)) {
-            def->features[w] |= feat_add;
-            def->features[w] &= ~feat_remove;
+    PropValue *pv;
+    for (pv = kvm_default_props; pv->prop; pv++) {
+        if (!strcmp(pv->prop, prop)) {
+            pv->value = value;
+            break;
         }
     }
+
+    /* It is valid to call this function only for properties that
+     * are already present in the kvm_default_props table.
+     */
+    assert(pv->prop);
 }
 
 static uint32_t x86_cpu_get_supported_feature_word(FeatureWord w,
@@ -1329,6 +1604,17 @@
 
 #ifdef CONFIG_KVM
 
+static bool lmce_supported(void)
+{
+    uint64_t mce_cap;
+
+    if (kvm_ioctl(kvm_state, KVM_X86_GET_MCE_CAP_SUPPORTED, &mce_cap) < 0) {
+        return false;
+    }
+
+    return !!(mce_cap & MCG_LMCE_P);
+}
+
 static int cpu_x86_fill_model_id(char *str)
 {
     uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
@@ -1348,6 +1634,7 @@
 
 static Property host_x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("migratable", X86CPU, migratable, true),
+    DEFINE_PROP_BOOL("host-cache-info", X86CPU, cache_info_passthrough, false),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -1374,13 +1661,14 @@
     cpu_x86_fill_model_id(host_cpudef.model_id);
 
     xcc->cpu_def = &host_cpudef;
-    host_cpudef.cache_info_passthrough = true;
 
     /* level, xlevel, xlevel2, and the feature words are initialized on
      * instance_init, because they require KVM to be initialized.
      */
 
     dc->props = host_x86_cpu_properties;
+    /* Reason: host_x86_cpu_initfn() dies when !kvm_enabled() */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void host_x86_cpu_initfn(Object *obj)
@@ -1389,16 +1677,21 @@
     CPUX86State *env = &cpu->env;
     KVMState *s = kvm_state;
 
-    assert(kvm_enabled());
-
     /* We can't fill the features array here because we don't know yet if
      * "migratable" is true or false.
      */
     cpu->host_features = true;
 
-    env->cpuid_level = kvm_arch_get_supported_cpuid(s, 0x0, 0, R_EAX);
-    env->cpuid_xlevel = kvm_arch_get_supported_cpuid(s, 0x80000000, 0, R_EAX);
-    env->cpuid_xlevel2 = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0, R_EAX);
+    /* If KVM is disabled, x86_cpu_realizefn() will report an error later */
+    if (kvm_enabled()) {
+        env->cpuid_level = kvm_arch_get_supported_cpuid(s, 0x0, 0, R_EAX);
+        env->cpuid_xlevel = kvm_arch_get_supported_cpuid(s, 0x80000000, 0, R_EAX);
+        env->cpuid_xlevel2 = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0, R_EAX);
+
+        if (lmce_supported()) {
+            object_property_set_bool(OBJECT(cpu), true, "lmce", &error_abort);
+        }
+    }
 
     object_property_set_bool(OBJECT(cpu), true, "pmu", &error_abort);
 }
@@ -1418,7 +1711,7 @@
     int i;
 
     for (i = 0; i < 32; ++i) {
-        if (1 << i & mask) {
+        if ((1UL << i) & mask) {
             const char *reg = get_register_name_32(f->cpuid_reg);
             assert(reg);
             fprintf(stderr, "warning: %s doesn't support requested feature: "
@@ -1431,8 +1724,9 @@
     }
 }
 
-static void x86_cpuid_version_get_family(Object *obj, Visitor *v, void *opaque,
-                                         const char *name, Error **errp)
+static void x86_cpuid_version_get_family(Object *obj, Visitor *v,
+                                         const char *name, void *opaque,
+                                         Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1442,11 +1736,12 @@
     if (value == 0xf) {
         value += (env->cpuid_version >> 20) & 0xff;
     }
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_version_set_family(Object *obj, Visitor *v, void *opaque,
-                                         const char *name, Error **errp)
+static void x86_cpuid_version_set_family(Object *obj, Visitor *v,
+                                         const char *name, void *opaque,
+                                         Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1455,14 +1750,14 @@
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
     if (value < min || value > max) {
-        error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
-                  name ? name : "null", value, min, max);
+        error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
+                   name ? name : "null", value, min, max);
         return;
     }
 
@@ -1474,8 +1769,9 @@
     }
 }
 
-static void x86_cpuid_version_get_model(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void x86_cpuid_version_get_model(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1483,11 +1779,12 @@
 
     value = (env->cpuid_version >> 4) & 0xf;
     value |= ((env->cpuid_version >> 16) & 0xf) << 4;
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_version_set_model(Object *obj, Visitor *v, void *opaque,
-                                        const char *name, Error **errp)
+static void x86_cpuid_version_set_model(Object *obj, Visitor *v,
+                                        const char *name, void *opaque,
+                                        Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
@@ -1496,14 +1793,14 @@
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
     if (value < min || value > max) {
-        error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
-                  name ? name : "null", value, min, max);
+        error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
+                   name ? name : "null", value, min, max);
         return;
     }
 
@@ -1512,7 +1809,7 @@
 }
 
 static void x86_cpuid_version_get_stepping(Object *obj, Visitor *v,
-                                           void *opaque, const char *name,
+                                           const char *name, void *opaque,
                                            Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
@@ -1520,11 +1817,11 @@
     int64_t value;
 
     value = env->cpuid_version & 0xf;
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
 static void x86_cpuid_version_set_stepping(Object *obj, Visitor *v,
-                                           void *opaque, const char *name,
+                                           const char *name, void *opaque,
                                            Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
@@ -1534,14 +1831,14 @@
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
     if (value < min || value > max) {
-        error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
-                  name ? name : "null", value, min, max);
+        error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
+                   name ? name : "null", value, min, max);
         return;
     }
 
@@ -1549,45 +1846,13 @@
     env->cpuid_version |= value & 0xf;
 }
 
-static void x86_cpuid_get_level(Object *obj, Visitor *v, void *opaque,
-                                const char *name, Error **errp)
-{
-    X86CPU *cpu = X86_CPU(obj);
-
-    visit_type_uint32(v, &cpu->env.cpuid_level, name, errp);
-}
-
-static void x86_cpuid_set_level(Object *obj, Visitor *v, void *opaque,
-                                const char *name, Error **errp)
-{
-    X86CPU *cpu = X86_CPU(obj);
-
-    visit_type_uint32(v, &cpu->env.cpuid_level, name, errp);
-}
-
-static void x86_cpuid_get_xlevel(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
-{
-    X86CPU *cpu = X86_CPU(obj);
-
-    visit_type_uint32(v, &cpu->env.cpuid_xlevel, name, errp);
-}
-
-static void x86_cpuid_set_xlevel(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
-{
-    X86CPU *cpu = X86_CPU(obj);
-
-    visit_type_uint32(v, &cpu->env.cpuid_xlevel, name, errp);
-}
-
 static char *x86_cpuid_get_vendor(Object *obj, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     CPUX86State *env = &cpu->env;
     char *value;
 
-    value = (char *)g_malloc(CPUID_VENDOR_SZ + 1);
+    value = g_malloc(CPUID_VENDOR_SZ + 1);
     x86_cpu_vendor_words2str(value, env->cpuid_vendor1, env->cpuid_vendor2,
                              env->cpuid_vendor3);
     return value;
@@ -1601,8 +1866,7 @@
     int i;
 
     if (strlen(value) != CPUID_VENDOR_SZ) {
-        error_set(errp, QERR_PROPERTY_VALUE_BAD, "",
-                  "vendor", value);
+        error_setg(errp, QERR_PROPERTY_VALUE_BAD, "", "vendor", value);
         return;
     }
 
@@ -1653,18 +1917,18 @@
     }
 }
 
-static void x86_cpuid_get_tsc_freq(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void x86_cpuid_get_tsc_freq(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     int64_t value;
 
     value = cpu->env.tsc_khz * 1000;
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
-                                   const char *name, Error **errp)
+static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, const char *name,
+                                   void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     const int64_t min = 0;
@@ -1672,71 +1936,27 @@
     Error *local_err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &local_err);
+    visit_type_int(v, name, &value, &local_err);
     if (local_err) {
         error_propagate(errp, local_err);
         return;
     }
     if (value < min || value > max) {
-        error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
-                  name ? name : "null", value, min, max);
+        error_setg(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE, "",
+                   name ? name : "null", value, min, max);
         return;
     }
 
-    cpu->env.tsc_khz = value / 1000;
-}
-
-static void x86_cpuid_get_apic_id(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
-{
-    X86CPU *cpu = X86_CPU(obj);
-    int64_t value = cpu->env.cpuid_apic_id;
-
-    visit_type_int(v, &value, name, errp);
-}
-
-static void x86_cpuid_set_apic_id(Object *obj, Visitor *v, void *opaque,
-                                  const char *name, Error **errp)
-{
-    X86CPU *cpu = X86_CPU(obj);
-    DeviceState *dev = DEVICE(obj);
-    const int64_t min = 0;
-    const int64_t max = UINT32_MAX;
-    Error *error = NULL;
-    int64_t value;
-
-    if (dev->realized) {
-        error_setg(errp, "Attempt to set property '%s' on '%s' after "
-                   "it was realized", name, object_get_typename(obj));
-        return;
-    }
-
-    visit_type_int(v, &value, name, &error);
-    if (error) {
-        error_propagate(errp, error);
-        return;
-    }
-    if (value < min || value > max) {
-        error_setg(errp, "Property %s.%s doesn't take value %" PRId64
-                   " (minimum: %" PRId64 ", maximum: %" PRId64 ")" ,
-                   object_get_typename(obj), name, value, min, max);
-        return;
-    }
-
-    if ((value != cpu->env.cpuid_apic_id) && cpu_exists(value)) {
-        error_setg(errp, "CPU with APIC ID %" PRIi64 " exists", value);
-        return;
-    }
-    cpu->env.cpuid_apic_id = value;
+    cpu->env.tsc_khz = cpu->env.user_tsc_khz = value / 1000;
 }
 
 /* Generic getter for "feature-words" and "filtered-features" properties */
-static void x86_cpu_get_feature_words(Object *obj, Visitor *v, void *opaque,
-                                      const char *name, Error **errp)
+static void x86_cpu_get_feature_words(Object *obj, Visitor *v,
+                                      const char *name, void *opaque,
+                                      Error **errp)
 {
     uint32_t *array = (uint32_t *)opaque;
     FeatureWord w;
-    Error *err = NULL;
     X86CPUFeatureWordInfo word_infos[FEATURE_WORDS] = { };
     X86CPUFeatureWordInfoList list_entries[FEATURE_WORDS] = { };
     X86CPUFeatureWordInfoList *list = NULL;
@@ -1756,21 +1976,20 @@
         list = &list_entries[w];
     }
 
-    visit_type_X86CPUFeatureWordInfoList(v, &list, "feature-words", &err);
-    error_propagate(errp, err);
+    visit_type_X86CPUFeatureWordInfoList(v, "feature-words", &list, errp);
 }
 
-static void x86_get_hv_spinlocks(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void x86_get_hv_spinlocks(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
     int64_t value = cpu->hyperv_spinlock_attempts;
 
-    visit_type_int(v, &value, name, errp);
+    visit_type_int(v, name, &value, errp);
 }
 
-static void x86_set_hv_spinlocks(Object *obj, Visitor *v, void *opaque,
-                                 const char *name, Error **errp)
+static void x86_set_hv_spinlocks(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
 {
     const int64_t min = 0xFFF;
     const int64_t max = UINT_MAX;
@@ -1778,7 +1997,7 @@
     Error *err = NULL;
     int64_t value;
 
-    visit_type_int(v, &value, name, &err);
+    visit_type_int(v, name, &value, &err);
     if (err) {
         error_propagate(errp, err);
         return;
@@ -1810,135 +2029,103 @@
     }
 }
 
+/* Compatibily hack to maintain legacy +-feat semantic,
+ * where +-feat overwrites any feature set by
+ * feat=on|feat even if the later is parsed after +-feat
+ * (i.e. "-x2apic,x2apic=on" will result in x2apic disabled)
+ */
+static FeatureWordArray plus_features = { 0 };
+static FeatureWordArray minus_features = { 0 };
+
 /* Parse "+feature,-feature,feature=foo" CPU feature string
  */
-static void x86_cpu_parse_featurestr(CPUState *cs, char *features,
+static void x86_cpu_parse_featurestr(const char *typename, char *features,
                                      Error **errp)
 {
-    X86CPU *cpu = X86_CPU(cs);
     char *featurestr; /* Single 'key=value" string being parsed */
-    FeatureWord w;
-    /* Features to be added */
-    FeatureWordArray plus_features = { 0 };
-    /* Features to be removed */
-    FeatureWordArray minus_features = { 0 };
-    uint32_t numvalue;
-    CPUX86State *env = &cpu->env;
     Error *local_err = NULL;
+    static bool cpu_globals_initialized;
 
-    featurestr = features ? strtok(features, ",") : NULL;
+    if (cpu_globals_initialized) {
+        return;
+    }
+    cpu_globals_initialized = true;
 
-    while (featurestr) {
-        char *val;
+    if (!features) {
+        return;
+    }
+
+    for (featurestr = strtok(features, ",");
+         featurestr  && !local_err;
+         featurestr = strtok(NULL, ",")) {
+        const char *name;
+        const char *val = NULL;
+        char *eq = NULL;
+        char num[32];
+        GlobalProperty *prop;
+
+        /* Compatibility syntax: */
         if (featurestr[0] == '+') {
             add_flagname_to_bitmaps(featurestr + 1, plus_features, &local_err);
+            continue;
         } else if (featurestr[0] == '-') {
             add_flagname_to_bitmaps(featurestr + 1, minus_features, &local_err);
-        } else if ((val = strchr(featurestr, '='))) {
-            *val = 0; val++;
-            feat2prop(featurestr);
-            if (!strcmp(featurestr, "xlevel")) {
-                char *err;
-                char num[32];
+            continue;
+        }
 
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    error_setg(errp, "bad numerical value %s", val);
-                    return;
-                }
-                if (numvalue < 0x80000000) {
-                    error_report("xlevel value shall always be >= 0x80000000"
-                                 ", fixup will be removed in future versions");
-                    numvalue += 0x80000000;
-                }
-                snprintf(num, sizeof(num), "%" PRIu32, numvalue);
-                object_property_parse(OBJECT(cpu), num, featurestr, &local_err);
-            } else if (!strcmp(featurestr, "tsc-freq")) {
-                int64_t tsc_freq;
-                char *err;
-                char num[32];
-
-                tsc_freq = strtosz_suffix_unit(val, &err,
-                                               STRTOSZ_DEFSUFFIX_B, 1000);
-                if (tsc_freq < 0 || *err) {
-                    error_setg(errp, "bad numerical value %s", val);
-                    return;
-                }
-                snprintf(num, sizeof(num), "%" PRId64, tsc_freq);
-                object_property_parse(OBJECT(cpu), num, "tsc-frequency",
-                                      &local_err);
-            } else if (!strcmp(featurestr, "hv-spinlocks")) {
-                char *err;
-                const int min = 0xFFF;
-                char num[32];
-                numvalue = strtoul(val, &err, 0);
-                if (!*val || *err) {
-                    error_setg(errp, "bad numerical value %s", val);
-                    return;
-                }
-                if (numvalue < min) {
-                    error_report("hv-spinlocks value shall always be >= 0x%x"
-                                 ", fixup will be removed in future versions",
-                                 min);
-                    numvalue = min;
-                }
-                snprintf(num, sizeof(num), "%" PRId32, numvalue);
-                object_property_parse(OBJECT(cpu), num, featurestr, &local_err);
-            } else {
-                object_property_parse(OBJECT(cpu), val, featurestr, &local_err);
-            }
+        eq = strchr(featurestr, '=');
+        if (eq) {
+            *eq++ = 0;
+            val = eq;
         } else {
-            feat2prop(featurestr);
-            object_property_parse(OBJECT(cpu), "on", featurestr, &local_err);
+            val = "on";
         }
-        if (local_err) {
-            error_propagate(errp, local_err);
-            return;
+
+        feat2prop(featurestr);
+        name = featurestr;
+
+        /* Special case: */
+        if (!strcmp(name, "tsc-freq")) {
+            int64_t tsc_freq;
+            char *err;
+
+            tsc_freq = qemu_strtosz_suffix_unit(val, &err,
+                                           QEMU_STRTOSZ_DEFSUFFIX_B, 1000);
+            if (tsc_freq < 0 || *err) {
+                error_setg(errp, "bad numerical value %s", val);
+                return;
+            }
+            snprintf(num, sizeof(num), "%" PRId64, tsc_freq);
+            val = num;
+            name = "tsc-frequency";
         }
-        featurestr = strtok(NULL, ",");
+
+        prop = g_new0(typeof(*prop), 1);
+        prop->driver = typename;
+        prop->property = g_strdup(name);
+        prop->value = g_strdup(val);
+        prop->errp = &error_fatal;
+        qdev_prop_register_global(prop);
     }
 
-    if (cpu->host_features) {
-        for (w = 0; w < FEATURE_WORDS; w++) {
-            env->features[w] =
-                x86_cpu_get_supported_feature_word(w, cpu->migratable);
-        }
-    }
-
-    for (w = 0; w < FEATURE_WORDS; w++) {
-        env->features[w] |= plus_features[w];
-        env->features[w] &= ~minus_features[w];
+    if (local_err) {
+        error_propagate(errp, local_err);
     }
 }
 
-/* generate a composite string into buf of all cpuid names in featureset
- * selected by fbits.  indicate truncation at bufsize in the event of overflow.
- * if flags, suppress names undefined in featureset.
+/* Print all cpuid feature names in featureset
  */
-static void listflags(char *buf, int bufsize, uint32_t fbits,
-                      const char **featureset, uint32_t flags)
+static void listflags(FILE *f, fprintf_function print, const char **featureset)
 {
-    const char **p = &featureset[31];
-    char *q, *b, bit;
-    int nc;
+    int bit;
+    bool first = true;
 
-    b = 4 <= bufsize ? buf + (bufsize -= 3) - 1 : NULL;
-    *buf = '\0';
-    for (q = buf, bit = 31; fbits && bufsize; --p, fbits &= ~(1 << bit), --bit)
-        if (fbits & 1 << bit && (*p || !flags)) {
-            if (*p)
-                nc = snprintf(q, bufsize, "%s%s", q == buf ? "" : " ", *p);
-            else
-                nc = snprintf(q, bufsize, "%s[%d]", q == buf ? "" : " ", bit);
-            if (bufsize <= nc) {
-                if (b) {
-                    memcpy(b, "...", sizeof("..."));
-                }
-                return;
-            }
-            q += nc;
-            bufsize -= nc;
+    for (bit = 0; bit < 32; bit++) {
+        if (featureset[bit]) {
+            print(f, "%s%s", first ? "" : " ", featureset[bit]);
+            first = false;
         }
+    }
 }
 
 /* generate CPU information. */
@@ -1963,8 +2150,9 @@
     for (i = 0; i < ARRAY_SIZE(feature_word_info); i++) {
         FeatureWordInfo *fw = &feature_word_info[i];
 
-        listflags(buf, sizeof(buf), (uint32_t)~0, fw->feat_names, 1);
-        (*cpu_fprintf)(f, "  %s\n", buf);
+        (*cpu_fprintf)(f, "  ");
+        listflags(f, cpu_fprintf, fw->feat_names);
+        (*cpu_fprintf)(f, "\n");
     }
 }
 
@@ -2040,6 +2228,18 @@
     return rv;
 }
 
+static void x86_cpu_apply_props(X86CPU *cpu, PropValue *props)
+{
+    PropValue *pv;
+    for (pv = props; pv->prop; pv++) {
+        if (!pv->value) {
+            continue;
+        }
+        object_property_parse(OBJECT(cpu), pv->value, pv->prop,
+                              &error_abort);
+    }
+}
+
 /* Load data from X86CPUDefinition
  */
 static void x86_cpu_load_def(X86CPU *cpu, X86CPUDefinition *def, Error **errp)
@@ -2054,8 +2254,7 @@
     object_property_set_int(OBJECT(cpu), def->model, "model", errp);
     object_property_set_int(OBJECT(cpu), def->stepping, "stepping", errp);
     object_property_set_int(OBJECT(cpu), def->xlevel, "xlevel", errp);
-    env->cpuid_xlevel2 = def->xlevel2;
-    cpu->cache_info_passthrough = def->cache_info_passthrough;
+    object_property_set_int(OBJECT(cpu), def->xlevel2, "xlevel2", errp);
     object_property_set_str(OBJECT(cpu), def->model_id, "model-id", errp);
     for (w = 0; w < FEATURE_WORDS; w++) {
         env->features[w] = def->features[w];
@@ -2063,11 +2262,11 @@
 
     /* Special cases not set in the X86CPUDefinition structs: */
     if (kvm_enabled()) {
-        FeatureWord w;
-        for (w = 0; w < FEATURE_WORDS; w++) {
-            env->features[w] |= kvm_default_features[w];
-            env->features[w] &= ~kvm_default_unset_features[w];
+        if (!kvm_irqchip_in_kernel()) {
+            x86_cpu_change_kvm_default("x2apic", "off");
         }
+
+        x86_cpu_apply_props(cpu, kvm_default_props);
     }
 
     env->features[FEAT_1_ECX] |= CPUID_EXT_HYPERVISOR;
@@ -2091,110 +2290,9 @@
 
 }
 
-X86CPU *cpu_x86_create(const char *cpu_model, DeviceState *icc_bridge,
-                       Error **errp)
-{
-    X86CPU *cpu = NULL;
-    X86CPUClass *xcc;
-    ObjectClass *oc;
-    gchar **model_pieces;
-    char *name, *features;
-    Error *error = NULL;
-
-    model_pieces = g_strsplit(cpu_model, ",", 2);
-    if (!model_pieces[0]) {
-        error_setg(&error, "Invalid/empty CPU model name");
-        goto out;
-    }
-    name = model_pieces[0];
-    features = model_pieces[1];
-
-    oc = x86_cpu_class_by_name(name);
-    if (oc == NULL) {
-        error_setg(&error, "Unable to find CPU definition: %s", name);
-        goto out;
-    }
-    xcc = X86_CPU_CLASS(oc);
-
-    if (xcc->kvm_required && !kvm_enabled()) {
-        error_setg(&error, "CPU model '%s' requires KVM", name);
-        goto out;
-    }
-
-    cpu = X86_CPU(object_new(object_class_get_name(oc)));
-
-#ifndef CONFIG_USER_ONLY
-    if (icc_bridge == NULL) {
-        error_setg(&error, "Invalid icc-bridge value");
-        goto out;
-    }
-    qdev_set_parent_bus(DEVICE(cpu), qdev_get_child_bus(icc_bridge, "icc"));
-    object_unref(OBJECT(cpu));
-#endif
-
-    x86_cpu_parse_featurestr(CPU(cpu), features, &error);
-    if (error) {
-        goto out;
-    }
-
-#ifdef CONFIG_ANDROID
-    char* intel_pmu_enabled =
-        getenv("ANDROID_EMU_FEATURE_IntelPerformanceMonitoringUnit");
-    if (kvm_enabled() &&
-        intel_pmu_enabled &&
-        !strcmp("on", intel_pmu_enabled)) {
-
-        CPUX86State *env = &cpu->env;
-        KVMState *s = kvm_state;
-
-        /* Use the host/KVM CPUID level in order to enable additional features
-         * supported by the host CPU, such as PMU. Cf. host_x86_cpu_initfn().
-         */
-        env->cpuid_level = kvm_arch_get_supported_cpuid(s, 0x0, 0, R_EAX);
-        env->cpuid_xlevel = kvm_arch_get_supported_cpuid(s, 0x80000000, 0,
-                                                         R_EAX);
-        env->cpuid_xlevel2 = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0,
-                                                          R_EAX);
-
-        /* Enable PMU (this has no effect if env->cpuid_level < 0xA) */
-        object_property_set_bool(OBJECT(cpu), true, "pmu", &error_abort);
-    }
-#endif
-
-out:
-    if (error != NULL) {
-        error_propagate(errp, error);
-        if (cpu) {
-            object_unref(OBJECT(cpu));
-            cpu = NULL;
-        }
-    }
-    g_strfreev(model_pieces);
-    return cpu;
-}
-
 X86CPU *cpu_x86_init(const char *cpu_model)
 {
-    Error *error = NULL;
-    X86CPU *cpu;
-
-    cpu = cpu_x86_create(cpu_model, NULL, &error);
-    if (error) {
-        goto out;
-    }
-
-    object_property_set_bool(OBJECT(cpu), true, "realized", &error);
-
-out:
-    if (error) {
-        error_report("%s", error_get_pretty(error));
-        error_free(error);
-        if (cpu != NULL) {
-            object_unref(OBJECT(cpu));
-            cpu = NULL;
-        }
-    }
-    return cpu;
+    return X86_CPU(cpu_generic_init(TYPE_X86_CPU, cpu_model));
 }
 
 static void x86_cpu_cpudef_class_init(ObjectClass *oc, void *data)
@@ -2228,38 +2326,6 @@
 
 #endif /* !CONFIG_USER_ONLY */
 
-/* Initialize list of CPU models, filling some non-static fields if necessary
- */
-void x86_cpudef_setup(void)
-{
-    int i, j;
-    static const char *model_with_versions[] = { "qemu32", "qemu64", "athlon" };
-
-    for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); ++i) {
-        X86CPUDefinition *def = &builtin_x86_defs[i];
-
-        /* Look for specific "cpudef" models that */
-        /* have the QEMU version in .model_id */
-        for (j = 0; j < ARRAY_SIZE(model_with_versions); j++) {
-            if (strcmp(model_with_versions[j], def->name) == 0) {
-                pstrcpy(def->model_id, sizeof(def->model_id),
-                        "QEMU Virtual CPU version ");
-                pstrcat(def->model_id, sizeof(def->model_id),
-                        qemu_get_version());
-                break;
-            }
-        }
-    }
-}
-
-static void get_cpuid_vendor(CPUX86State *env, uint32_t *ebx,
-                             uint32_t *ecx, uint32_t *edx)
-{
-    *ebx = env->cpuid_vendor1;
-    *edx = env->cpuid_vendor2;
-    *ecx = env->cpuid_vendor3;
-}
-
 void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                    uint32_t *eax, uint32_t *ebx,
                    uint32_t *ecx, uint32_t *edx)
@@ -2293,16 +2359,22 @@
     switch(index) {
     case 0:
         *eax = env->cpuid_level;
-        get_cpuid_vendor(env, ebx, ecx, edx);
+        *ebx = env->cpuid_vendor1;
+        *edx = env->cpuid_vendor2;
+        *ecx = env->cpuid_vendor3;
         break;
     case 1:
         *eax = env->cpuid_version;
-        *ebx = (env->cpuid_apic_id << 24) | 8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
+        *ebx = (cpu->apic_id << 24) |
+               8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
         *ecx = env->features[FEAT_1_ECX];
+        if ((*ecx & CPUID_EXT_XSAVE) && (env->cr[4] & CR4_OSXSAVE_MASK)) {
+            *ecx |= CPUID_EXT_OSXSAVE;
+        }
         *edx = env->features[FEAT_1_EDX];
         if (cs->nr_cores * cs->nr_threads > 1) {
             *ebx |= (cs->nr_cores * cs->nr_threads) << 16;
-            *edx |= 1 << 28;    /* HTT bit */
+            *edx |= CPUID_HT;
         }
         break;
     case 2:
@@ -2382,7 +2454,7 @@
         break;
     case 6:
         /* Thermal and Power Leaf */
-        *eax = 0;
+        *eax = env->features[FEAT_6_EAX];
         *ebx = 0;
         *ecx = 0;
         *edx = 0;
@@ -2392,7 +2464,10 @@
         if (count == 0) {
             *eax = 0; /* Maximum ECX value for sub-leaves */
             *ebx = env->features[FEAT_7_0_EBX]; /* Feature flags */
-            *ecx = 0; /* Reserved */
+            *ecx = env->features[FEAT_7_0_ECX]; /* Feature flags */
+            if ((*ecx & CPUID_7_0_ECX_PKU) && env->cr[4] & CR4_PKE_MASK) {
+                *ecx |= CPUID_7_0_ECX_OSPKE;
+            }
             *edx = 0; /* Reserved */
         } else {
             *eax = 0;
@@ -2424,9 +2499,39 @@
             *edx = 0;
         }
         break;
+    case 0xB:
+        /* Extended Topology Enumeration Leaf */
+        if (!cpu->enable_cpuid_0xb) {
+                *eax = *ebx = *ecx = *edx = 0;
+                break;
+        }
+
+        *ecx = count & 0xff;
+        *edx = cpu->apic_id;
+
+        switch (count) {
+        case 0:
+            *eax = apicid_core_offset(smp_cores, smp_threads);
+            *ebx = smp_threads;
+            *ecx |= CPUID_TOPOLOGY_LEVEL_SMT;
+            break;
+        case 1:
+            *eax = apicid_pkg_offset(smp_cores, smp_threads);
+            *ebx = smp_cores * smp_threads;
+            *ecx |= CPUID_TOPOLOGY_LEVEL_CORE;
+            break;
+        default:
+            *eax = 0;
+            *ebx = 0;
+            *ecx |= CPUID_TOPOLOGY_LEVEL_INVALID;
+        }
+
+        assert(!(*eax & ~0x1f));
+        *ebx &= 0xffff; /* The count doesn't need to be reliable. */
+        break;
     case 0xD: {
         KVMState *s = cs->kvm_state;
-        uint64_t kvm_mask;
+        uint64_t ena_mask;
         int i;
 
         /* Processor Extended State */
@@ -2434,35 +2539,39 @@
         *ebx = 0;
         *ecx = 0;
         *edx = 0;
-        if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) || !kvm_enabled()) {
+        if (!(env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE)) {
             break;
         }
-        kvm_mask =
-            kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX) |
-            ((uint64_t)kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX) << 32);
+        if (kvm_enabled()) {
+            ena_mask = kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EDX);
+            ena_mask <<= 32;
+            ena_mask |= kvm_arch_get_supported_cpuid(s, 0xd, 0, R_EAX);
+        } else {
+            ena_mask = -1;
+        }
 
         if (count == 0) {
             *ecx = 0x240;
-            for (i = 2; i < ARRAY_SIZE(ext_save_areas); i++) {
-                const ExtSaveArea *esa = &ext_save_areas[i];
-                if ((env->features[esa->feature] & esa->bits) == esa->bits &&
-                    (kvm_mask & (1 << i)) != 0) {
+            for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
+                const ExtSaveArea *esa = &x86_ext_save_areas[i];
+                if ((env->features[esa->feature] & esa->bits) == esa->bits
+                    && ((ena_mask >> i) & 1) != 0) {
                     if (i < 32) {
-                        *eax |= 1 << i;
+                        *eax |= 1u << i;
                     } else {
-                        *edx |= 1 << (i - 32);
+                        *edx |= 1u << (i - 32);
                     }
                     *ecx = MAX(*ecx, esa->offset + esa->size);
                 }
             }
-            *eax |= kvm_mask & (XSTATE_FP | XSTATE_SSE);
+            *eax |= ena_mask & (XSTATE_FP_MASK | XSTATE_SSE_MASK);
             *ebx = *ecx;
         } else if (count == 1) {
-            *eax = kvm_arch_get_supported_cpuid(s, 0xd, 1, R_EAX);
-        } else if (count < ARRAY_SIZE(ext_save_areas)) {
-            const ExtSaveArea *esa = &ext_save_areas[count];
-            if ((env->features[esa->feature] & esa->bits) == esa->bits &&
-                (kvm_mask & (1 << count)) != 0) {
+            *eax = env->features[FEAT_XSAVE];
+        } else if (count < ARRAY_SIZE(x86_ext_save_areas)) {
+            const ExtSaveArea *esa = &x86_ext_save_areas[count];
+            if ((env->features[esa->feature] & esa->bits) == esa->bits
+                && ((ena_mask >> count) & 1) != 0) {
                 *eax = esa->size;
                 *ebx = esa->offset;
             }
@@ -2483,14 +2592,12 @@
 
         /* The Linux kernel checks for the CMPLegacy bit and
          * discards multiple thread information if it is set.
-         * So dont set it here for Intel to make Linux guests happy.
+         * So don't set it here for Intel to make Linux guests happy.
          */
         if (cs->nr_cores * cs->nr_threads > 1) {
-            uint32_t tebx, tecx, tedx;
-            get_cpuid_vendor(env, &tebx, &tecx, &tedx);
-            if (tebx != CPUID_VENDOR_INTEL_1 ||
-                tedx != CPUID_VENDOR_INTEL_2 ||
-                tecx != CPUID_VENDOR_INTEL_3) {
+            if (env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1 ||
+                env->cpuid_vendor2 != CPUID_VENDOR_INTEL_2 ||
+                env->cpuid_vendor3 != CPUID_VENDOR_INTEL_3) {
                 *ecx |= 1 << 1;    /* CmpLegacy bit */
             }
         }
@@ -2547,17 +2654,13 @@
         break;
     case 0x80000008:
         /* virtual & phys address size in low 2 bytes. */
-/* XXX: This value must match the one used in the MMU code. */
         if (env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_LM) {
-            /* 64 bit processor */
-/* XXX: The physical address space is limited to 42 bits in exec.c. */
-            *eax = 0x00003028; /* 48 bits virtual, 40 bits physical */
+            /* 64 bit processor, 48 bits virtual, configurable
+             * physical bits.
+             */
+            *eax = 0x00003000 + cpu->phys_bits;
         } else {
-            if (env->features[FEAT_1_EDX] & CPUID_PSE36) {
-                *eax = 0x00000024; /* 36 bits physical */
-            } else {
-                *eax = 0x00000020; /* 32 bits physical */
-            }
+            *eax = cpu->phys_bits;
         }
         *ebx = 0;
         *ecx = 0;
@@ -2617,6 +2720,8 @@
     X86CPU *cpu = X86_CPU(s);
     X86CPUClass *xcc = X86_CPU_GET_CLASS(cpu);
     CPUX86State *env = &cpu->env;
+    target_ulong cr4;
+    uint64_t xcr0;
     int i;
 
     xcc->parent_reset(s);
@@ -2629,9 +2734,6 @@
 
     /* init to reset state */
 
-#ifdef CONFIG_SOFTMMU
-    env->hflags |= HF_SOFTMMU_MASK;
-#endif
     env->hflags2 |= HF2_GIF_MASK;
 
     cpu_x86_update_cr0(env, 0x60000010);
@@ -2676,7 +2778,8 @@
     cpu_set_fpuc(env, 0x37f);
 
     env->mxcsr = 0x1f80;
-    env->xstate_bv = XSTATE_FP | XSTATE_SSE;
+    /* All units are in INIT state.  */
+    env->xstate_bv = 0;
 
     env->pat = 0x0007040600070406ULL;
     env->msr_ia32_misc_enable = MSR_IA32_MISC_ENABLE_DEFAULT;
@@ -2687,7 +2790,31 @@
     cpu_breakpoint_remove_all(s, BP_CPU);
     cpu_watchpoint_remove_all(s, BP_CPU);
 
-    env->xcr0 = 1;
+    cr4 = 0;
+    xcr0 = XSTATE_FP_MASK;
+
+#ifdef CONFIG_USER_ONLY
+    /* Enable all the features for user-mode.  */
+    if (env->features[FEAT_1_EDX] & CPUID_SSE) {
+        xcr0 |= XSTATE_SSE_MASK;
+    }
+    for (i = 2; i < ARRAY_SIZE(x86_ext_save_areas); i++) {
+        const ExtSaveArea *esa = &x86_ext_save_areas[i];
+        if ((env->features[esa->feature] & esa->bits) == esa->bits) {
+            xcr0 |= 1ull << i;
+        }
+    }
+
+    if (env->features[FEAT_1_ECX] & CPUID_EXT_XSAVE) {
+        cr4 |= CR4_OSFXSR_MASK | CR4_OSXSAVE_MASK;
+    }
+    if (env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_FSGSBASE) {
+        cr4 |= CR4_FSGSBASE_MASK;
+    }
+#endif
+
+    env->xcr0 = xcr0;
+    cpu_x86_update_cr4(env, cr4);
 
     /*
      * SDM 11.11.5 requires:
@@ -2701,9 +2828,7 @@
 
 #if !defined(CONFIG_USER_ONLY)
     /* We hard-wire the BSP to the first CPU. */
-    if (s->cpu_index == 0) {
-        apic_designate_bsp(cpu->apic_state);
-    }
+    apic_designate_bsp(cpu->apic_state, s->cpu_index == 0);
 
     s->halted = !cpu_is_bsp(cpu);
 
@@ -2735,7 +2860,8 @@
     if (((cenv->cpuid_version >> 8) & 0xf) >= 6
         && (cenv->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) ==
             (CPUID_MCE | CPUID_MCA)) {
-        cenv->mcg_cap = MCE_CAP_DEF | MCE_BANKS_DEF;
+        cenv->mcg_cap = MCE_CAP_DEF | MCE_BANKS_DEF |
+                        (cpu->enable_lmce ? MCG_LMCE_P : 0);
         cenv->mcg_ctl = ~(uint64_t)0;
         for (bank = 0; bank < MCE_BANKS_DEF; bank++) {
             cenv->mce_banks[bank * 4] = ~(uint64_t)0;
@@ -2746,41 +2872,63 @@
 #ifndef CONFIG_USER_ONLY
 static void x86_cpu_apic_create(X86CPU *cpu, Error **errp)
 {
-    CPUX86State *env = &cpu->env;
-    DeviceState *dev = DEVICE(cpu);
     APICCommonState *apic;
     const char *apic_type = "apic";
 
-    if (kvm_irqchip_in_kernel()) {
+    if (kvm_apic_in_kernel()) {
         apic_type = "kvm-apic";
     } else if (xen_enabled()) {
         apic_type = "xen-apic";
     }
 
-    cpu->apic_state = qdev_try_create(qdev_get_parent_bus(dev), apic_type);
-    if (cpu->apic_state == NULL) {
-        error_setg(errp, "APIC device '%s' could not be created", apic_type);
-        return;
-    }
+    cpu->apic_state = DEVICE(object_new(apic_type));
 
-    object_property_add_child(OBJECT(cpu), "apic",
-                              OBJECT(cpu->apic_state), NULL);
-    qdev_prop_set_uint8(cpu->apic_state, "id", env->cpuid_apic_id);
+    object_property_add_child(OBJECT(cpu), "lapic",
+                              OBJECT(cpu->apic_state), &error_abort);
+    object_unref(OBJECT(cpu->apic_state));
+
+    qdev_prop_set_uint8(cpu->apic_state, "id", cpu->apic_id);
     /* TODO: convert to link<> */
     apic = APIC_COMMON(cpu->apic_state);
     apic->cpu = cpu;
+    apic->apicbase = APIC_DEFAULT_ADDRESS | MSR_IA32_APICBASE_ENABLE;
 }
 
 static void x86_cpu_apic_realize(X86CPU *cpu, Error **errp)
 {
+    APICCommonState *apic;
+    static bool apic_mmio_map_once;
+
     if (cpu->apic_state == NULL) {
         return;
     }
+    object_property_set_bool(OBJECT(cpu->apic_state), true, "realized",
+                             errp);
 
-    if (qdev_init(cpu->apic_state)) {
-        error_setg(errp, "APIC device '%s' could not be initialized",
-                   object_get_typename(OBJECT(cpu->apic_state)));
-        return;
+    /* Map APIC MMIO area */
+    apic = APIC_COMMON(cpu->apic_state);
+    if (!apic_mmio_map_once) {
+        memory_region_add_subregion_overlap(get_system_memory(),
+                                            apic->apicbase &
+                                            MSR_IA32_APICBASE_BASE,
+                                            &apic->io_memory,
+                                            0x1000);
+        apic_mmio_map_once = true;
+     }
+}
+
+static void x86_cpu_machine_done(Notifier *n, void *unused)
+{
+    X86CPU *cpu = container_of(n, X86CPU, machine_done);
+    MemoryRegion *smram =
+        (MemoryRegion *) object_resolve_path("/machine/smram", NULL);
+
+    if (smram) {
+        cpu->smram = g_new(MemoryRegion, 1);
+        memory_region_init_alias(cpu->smram, OBJECT(cpu), "smram",
+                                 smram, 0, 1ull << 32);
+        memory_region_set_enabled(cpu->smram, false);
+        memory_region_add_subregion_overlap(cpu->cpu_as_root, 0, cpu->smram, 1);
     }
 }
 #else
@@ -2789,6 +2937,31 @@
 }
 #endif
 
+/* Note: Only safe for use on x86(-64) hosts */
+static uint32_t x86_host_phys_bits(void)
+{
+    uint32_t eax;
+    uint32_t host_phys_bits;
+
+    host_cpuid(0x80000000, 0, &eax, NULL, NULL, NULL);
+    if (eax >= 0x80000008) {
+        host_cpuid(0x80000008, 0, &eax, NULL, NULL, NULL);
+        /* Note: According to AMD doc 25481 rev 2.34 they have a field
+         * at 23:16 that can specify a maximum physical address bits for
+         * the guest that can override this value; but I've not seen
+         * anything with that set.
+         */
+        host_phys_bits = eax & 0xff;
+    } else {
+        /* It's an odd 64 bit machine that doesn't have the leaf for
+         * physical address bits; fall back to 36 that's most older
+         * Intel.
+         */
+        host_phys_bits = 36;
+    }
+
+    return host_phys_bits;
+}
 
 #define IS_INTEL_CPU(env) ((env)->cpuid_vendor1 == CPUID_VENDOR_INTEL_1 && \
                            (env)->cpuid_vendor2 == CPUID_VENDOR_INTEL_2 && \
@@ -2804,11 +2977,49 @@
     CPUX86State *env = &cpu->env;
     Error *local_err = NULL;
     static bool ht_warned;
+    FeatureWord w;
+
+    if (xcc->kvm_required && !kvm_enabled()) {
+        char *name = x86_cpu_class_get_model_name(xcc);
+        error_setg(&local_err, "CPU model '%s' requires KVM", name);
+        g_free(name);
+        goto out;
+    }
+
+    if (cpu->apic_id == UNASSIGNED_APIC_ID) {
+        error_setg(errp, "apic-id property was not initialized properly");
+        return;
+    }
+
+    /*TODO: cpu->host_features incorrectly overwrites features
+     * set using "feat=on|off". Once we fix this, we can convert
+     * plus_features & minus_features to global properties
+     * inside x86_cpu_parse_featurestr() too.
+     */
+    if (cpu->host_features) {
+        for (w = 0; w < FEATURE_WORDS; w++) {
+            env->features[w] =
+                x86_cpu_get_supported_feature_word(w, cpu->migratable);
+        }
+    }
+
+    for (w = 0; w < FEATURE_WORDS; w++) {
+        cpu->env.features[w] |= plus_features[w];
+        cpu->env.features[w] &= ~minus_features[w];
+    }
 
     if (env->features[FEAT_7_0_EBX] && env->cpuid_level < 7) {
         env->cpuid_level = 7;
     }
 
+    if (x86_cpu_filter_features(cpu) && cpu->enforce_cpuid) {
+        error_setg(&local_err,
+                   kvm_enabled() ?
+                       "Host doesn't support requested features" :
+                       "TCG doesn't support requested features");
+        goto out;
+    }
+
     /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
      * CPUID[1].EDX.
      */
@@ -2818,13 +3029,74 @@
            & CPUID_EXT2_AMD_ALIASES);
     }
 
+    /* For 64bit systems think about the number of physical bits to present.
+     * ideally this should be the same as the host; anything other than matching
+     * the host can cause incorrect guest behaviour.
+     * QEMU used to pick the magic value of 40 bits that corresponds to
+     * consumer AMD devices but nothing else.
+     */
+    if (env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_LM) {
+        if (kvm_enabled()) {
+            uint32_t host_phys_bits = x86_host_phys_bits();
+            static bool warned;
 
-    if (x86_cpu_filter_features(cpu) && cpu->enforce_cpuid) {
-        error_setg(&local_err,
-                   kvm_enabled() ?
-                       "Host doesn't support requested features" :
-                       "TCG doesn't support requested features");
-        goto out;
+            if (cpu->host_phys_bits) {
+                /* The user asked for us to use the host physical bits */
+                cpu->phys_bits = host_phys_bits;
+            }
+
+            /* Print a warning if the user set it to a value that's not the
+             * host value.
+             */
+            if (cpu->phys_bits != host_phys_bits && cpu->phys_bits != 0 &&
+                !warned) {
+                error_report("Warning: Host physical bits (%u)"
+                                 " does not match phys-bits property (%u)",
+                                 host_phys_bits, cpu->phys_bits);
+                warned = true;
+            }
+
+            if (cpu->phys_bits &&
+                (cpu->phys_bits > TARGET_PHYS_ADDR_SPACE_BITS ||
+                cpu->phys_bits < 32)) {
+                error_setg(errp, "phys-bits should be between 32 and %u "
+                                 " (but is %u)",
+                                 TARGET_PHYS_ADDR_SPACE_BITS, cpu->phys_bits);
+                return;
+            }
+        } else {
+            if (cpu->phys_bits && cpu->phys_bits != TCG_PHYS_ADDR_BITS) {
+                error_setg(errp, "TCG only supports phys-bits=%u",
+                                  TCG_PHYS_ADDR_BITS);
+                return;
+            }
+        }
+        /* 0 means it was not explicitly set by the user (or by machine
+         * compat_props or by the host code above). In this case, the default
+         * is the value used by TCG (40).
+         */
+        if (cpu->phys_bits == 0) {
+            cpu->phys_bits = TCG_PHYS_ADDR_BITS;
+        }
+    } else {
+        /* For 32 bit systems don't use the user set value, but keep
+         * phys_bits consistent with what we tell the guest.
+         */
+        if (cpu->phys_bits != 0) {
+            error_setg(errp, "phys-bits is not user-configurable in 32 bit");
+            return;
+        }
+
+        if (env->features[FEAT_1_EDX] & CPUID_PSE36) {
+            cpu->phys_bits = 36;
+        } else {
+            cpu->phys_bits = 32;
+        }
+    }
+    cpu_exec_init(cs, &error_abort);
+
+    if (tcg_enabled()) {
+        tcg_x86_init();
     }
 
 #ifndef CONFIG_USER_ONLY
@@ -2839,6 +3111,35 @@
 #endif
 
     mce_init(cpu);
+
+#ifndef CONFIG_USER_ONLY
+    if (tcg_enabled()) {
+        AddressSpace *newas = g_new(AddressSpace, 1);
+
+        cpu->cpu_as_mem = g_new(MemoryRegion, 1);
+        cpu->cpu_as_root = g_new(MemoryRegion, 1);
+
+        /* Outer container... */
+        memory_region_init(cpu->cpu_as_root, OBJECT(cpu), "memory", ~0ull);
+        memory_region_set_enabled(cpu->cpu_as_root, true);
+
+        /* ... with two regions inside: normal system memory with low
+         * priority, and...
+         */
+        memory_region_init_alias(cpu->cpu_as_mem, OBJECT(cpu), "memory",
+                                 get_system_memory(), 0, ~0ull);
+        memory_region_add_subregion_overlap(cpu->cpu_as_root, 0, cpu->cpu_as_mem, 0);
+        memory_region_set_enabled(cpu->cpu_as_mem, true);
+        address_space_init(newas, cpu->cpu_as_root, "CPU");
+        cs->num_ases = 1;
+        cpu_address_space_init(cs, newas, 0);
+
+        /* ... SMRAM with higher priority, linked from /machine/smram.  */
+        cpu->machine_done.notify = x86_cpu_machine_done;
+        qemu_add_machine_init_done_notifier(&cpu->machine_done);
+    }
+#endif
+
     qemu_init_vcpu(cs);
 
     /* Only Intel CPUs support hyperthreading. Even though QEMU fixes this
@@ -2862,6 +3163,7 @@
     cpu_reset(cs);
 
     xcc->parent_realize(dev, &local_err);
+
 out:
     if (local_err != NULL) {
         error_propagate(errp, local_err);
@@ -2869,37 +3171,126 @@
     }
 }
 
-/* Enables contiguous-apic-ID mode, for compatibility */
-static bool compat_apic_id_mode;
-
-void enable_compat_apic_id_mode(void)
+static void x86_cpu_unrealizefn(DeviceState *dev, Error **errp)
 {
-    compat_apic_id_mode = true;
+    X86CPU *cpu = X86_CPU(dev);
+
+#ifndef CONFIG_USER_ONLY
+    cpu_remove_sync(CPU(dev));
+    qemu_unregister_reset(x86_cpu_machine_reset_cb, dev);
+#endif
+
+    if (cpu->apic_state) {
+        object_unparent(OBJECT(cpu->apic_state));
+        cpu->apic_state = NULL;
+    }
 }
 
-/* Calculates initial APIC ID for a specific CPU index
- *
- * Currently we need to be able to calculate the APIC ID from the CPU index
- * alone (without requiring a CPU object), as the QEMU<->Seabios interfaces have
- * no concept of "CPU index", and the NUMA tables on fw_cfg need the APIC ID of
- * all CPUs up to max_cpus.
- */
-uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index)
-{
-    uint32_t correct_id;
-    static bool warned;
+typedef struct BitProperty {
+    uint32_t *ptr;
+    uint32_t mask;
+} BitProperty;
 
-    correct_id = x86_apicid_from_cpu_idx(smp_cores, smp_threads, cpu_index);
-    if (compat_apic_id_mode) {
-        if (cpu_index != correct_id && !warned) {
-            error_report("APIC IDs set in compatibility mode, "
-                         "CPU topology won't match the configuration");
-            warned = true;
-        }
-        return cpu_index;
-    } else {
-        return correct_id;
+static void x86_cpu_get_bit_prop(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
+{
+    BitProperty *fp = opaque;
+    bool value = (*fp->ptr & fp->mask) == fp->mask;
+    visit_type_bool(v, name, &value, errp);
+}
+
+static void x86_cpu_set_bit_prop(Object *obj, Visitor *v, const char *name,
+                                 void *opaque, Error **errp)
+{
+    DeviceState *dev = DEVICE(obj);
+    BitProperty *fp = opaque;
+    Error *local_err = NULL;
+    bool value;
+
+    if (dev->realized) {
+        qdev_prop_set_after_realize(dev, name, errp);
+        return;
     }
+
+    visit_type_bool(v, name, &value, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    if (value) {
+        *fp->ptr |= fp->mask;
+    } else {
+        *fp->ptr &= ~fp->mask;
+    }
+}
+
+static void x86_cpu_release_bit_prop(Object *obj, const char *name,
+                                     void *opaque)
+{
+    BitProperty *prop = opaque;
+    g_free(prop);
+}
+
+/* Register a boolean property to get/set a single bit in a uint32_t field.
+ *
+ * The same property name can be registered multiple times to make it affect
+ * multiple bits in the same FeatureWord. In that case, the getter will return
+ * true only if all bits are set.
+ */
+static void x86_cpu_register_bit_prop(X86CPU *cpu,
+                                      const char *prop_name,
+                                      uint32_t *field,
+                                      int bitnr)
+{
+    BitProperty *fp;
+    ObjectProperty *op;
+    uint32_t mask = (1UL << bitnr);
+
+    op = object_property_find(OBJECT(cpu), prop_name, NULL);
+    if (op) {
+        fp = op->opaque;
+        assert(fp->ptr == field);
+        fp->mask |= mask;
+    } else {
+        fp = g_new0(BitProperty, 1);
+        fp->ptr = field;
+        fp->mask = mask;
+        object_property_add(OBJECT(cpu), prop_name, "bool",
+                            x86_cpu_get_bit_prop,
+                            x86_cpu_set_bit_prop,
+                            x86_cpu_release_bit_prop, fp, &error_abort);
+    }
+}
+
+static void x86_cpu_register_feature_bit_props(X86CPU *cpu,
+                                               FeatureWord w,
+                                               int bitnr)
+{
+    Object *obj = OBJECT(cpu);
+    int i;
+    char **names;
+    FeatureWordInfo *fi = &feature_word_info[w];
+
+    if (!fi->feat_names) {
+        return;
+    }
+    if (!fi->feat_names[bitnr]) {
+        return;
+    }
+
+    names = g_strsplit(fi->feat_names[bitnr], "|", 0);
+
+    feat2prop(names[0]);
+    x86_cpu_register_bit_prop(cpu, names[0], &cpu->env.features[w], bitnr);
+
+    for (i = 1; names[i]; i++) {
+        feat2prop(names[i]);
+        object_property_add_alias(obj, names[i], obj, names[0],
+                                  &error_abort);
+    }
+
+    g_strfreev(names);
 }
 
 static void x86_cpu_initfn(Object *obj)
@@ -2908,10 +3299,9 @@
     X86CPU *cpu = X86_CPU(obj);
     X86CPUClass *xcc = X86_CPU_GET_CLASS(obj);
     CPUX86State *env = &cpu->env;
-    static int inited;
+    FeatureWord w;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
 
     object_property_add(obj, "family", "int",
                         x86_cpuid_version_get_family,
@@ -2922,12 +3312,6 @@
     object_property_add(obj, "stepping", "int",
                         x86_cpuid_version_get_stepping,
                         x86_cpuid_version_set_stepping, NULL, NULL, NULL);
-    object_property_add(obj, "level", "int",
-                        x86_cpuid_get_level,
-                        x86_cpuid_set_level, NULL, NULL, NULL);
-    object_property_add(obj, "xlevel", "int",
-                        x86_cpuid_get_xlevel,
-                        x86_cpuid_set_xlevel, NULL, NULL, NULL);
     object_property_add_str(obj, "vendor",
                             x86_cpuid_get_vendor,
                             x86_cpuid_set_vendor, NULL);
@@ -2937,9 +3321,6 @@
     object_property_add(obj, "tsc-frequency", "int",
                         x86_cpuid_get_tsc_freq,
                         x86_cpuid_set_tsc_freq, NULL, NULL, NULL);
-    object_property_add(obj, "apic-id", "int",
-                        x86_cpuid_get_apic_id,
-                        x86_cpuid_set_apic_id, NULL, NULL, NULL);
     object_property_add(obj, "feature-words", "X86CPUFeatureWordInfo",
                         x86_cpu_get_feature_words,
                         NULL, NULL, (void *)env->features, NULL);
@@ -2948,23 +3329,23 @@
                         NULL, NULL, (void *)cpu->filtered_features, NULL);
 
     cpu->hyperv_spinlock_attempts = HYPERV_SPINLOCK_NEVER_RETRY;
-    env->cpuid_apic_id = x86_cpu_apic_id_from_index(cs->cpu_index);
+
+    for (w = 0; w < FEATURE_WORDS; w++) {
+        int bitnr;
+
+        for (bitnr = 0; bitnr < 32; bitnr++) {
+            x86_cpu_register_feature_bit_props(cpu, w, bitnr);
+        }
+    }
 
     x86_cpu_load_def(cpu, xcc->cpu_def, &error_abort);
-
-    /* init various static tables used in TCG mode */
-    if (tcg_enabled() && !inited) {
-        inited = 1;
-        optimize_flags_init();
-    }
 }
 
 static int64_t x86_cpu_get_arch_id(CPUState *cs)
 {
     X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
 
-    return env->cpuid_apic_id;
+    return cpu->apic_id;
 }
 
 static bool x86_cpu_get_paging_enabled(const CPUState *cs)
@@ -2993,30 +3374,53 @@
     X86CPU *cpu = X86_CPU(cs);
     CPUX86State *env = &cpu->env;
 
-#if !defined(CONFIG_USER_ONLY)
-    if (cs->interrupt_request & CPU_INTERRUPT_POLL) {
-        apic_poll_irq(cpu->apic_state);
-        cpu_reset_interrupt(cs, CPU_INTERRUPT_POLL);
-    }
-#endif
-
-    return ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
+    return ((cs->interrupt_request & (CPU_INTERRUPT_HARD |
+                                      CPU_INTERRUPT_POLL)) &&
             (env->eflags & IF_MASK)) ||
            (cs->interrupt_request & (CPU_INTERRUPT_NMI |
                                      CPU_INTERRUPT_INIT |
                                      CPU_INTERRUPT_SIPI |
-                                     CPU_INTERRUPT_MCE));
+                                     CPU_INTERRUPT_MCE)) ||
+           ((cs->interrupt_request & CPU_INTERRUPT_SMI) &&
+            !(env->hflags & HF_SMM_MASK));
 }
 
 static Property x86_cpu_properties[] = {
+#ifdef CONFIG_USER_ONLY
+    /* apic_id = 0 by default for *-user, see commit 9886e834 */
+    DEFINE_PROP_UINT32("apic-id", X86CPU, apic_id, 0),
+    DEFINE_PROP_INT32("thread-id", X86CPU, thread_id, 0),
+    DEFINE_PROP_INT32("core-id", X86CPU, core_id, 0),
+    DEFINE_PROP_INT32("socket-id", X86CPU, socket_id, 0),
+#else
+    DEFINE_PROP_UINT32("apic-id", X86CPU, apic_id, UNASSIGNED_APIC_ID),
+    DEFINE_PROP_INT32("thread-id", X86CPU, thread_id, -1),
+    DEFINE_PROP_INT32("core-id", X86CPU, core_id, -1),
+    DEFINE_PROP_INT32("socket-id", X86CPU, socket_id, -1),
+#endif
     DEFINE_PROP_BOOL("pmu", X86CPU, enable_pmu, false),
     { .name  = "hv-spinlocks", .info  = &qdev_prop_spinlocks },
     DEFINE_PROP_BOOL("hv-relaxed", X86CPU, hyperv_relaxed_timing, false),
     DEFINE_PROP_BOOL("hv-vapic", X86CPU, hyperv_vapic, false),
     DEFINE_PROP_BOOL("hv-time", X86CPU, hyperv_time, false),
-    DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, false),
+    DEFINE_PROP_BOOL("hv-crash", X86CPU, hyperv_crash, false),
+    DEFINE_PROP_BOOL("hv-reset", X86CPU, hyperv_reset, false),
+    DEFINE_PROP_BOOL("hv-vpindex", X86CPU, hyperv_vpindex, false),
+    DEFINE_PROP_BOOL("hv-runtime", X86CPU, hyperv_runtime, false),
+    DEFINE_PROP_BOOL("hv-synic", X86CPU, hyperv_synic, false),
+    DEFINE_PROP_BOOL("hv-stimer", X86CPU, hyperv_stimer, false),
+    DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, true),
     DEFINE_PROP_BOOL("enforce", X86CPU, enforce_cpuid, false),
     DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
+    DEFINE_PROP_UINT32("phys-bits", X86CPU, phys_bits, 0),
+    DEFINE_PROP_BOOL("host-phys-bits", X86CPU, host_phys_bits, false),
+    DEFINE_PROP_BOOL("fill-mtrr-mask", X86CPU, fill_mtrr_mask, true),
+    DEFINE_PROP_UINT32("level", X86CPU, env.cpuid_level, 0),
+    DEFINE_PROP_UINT32("xlevel", X86CPU, env.cpuid_xlevel, 0),
+    DEFINE_PROP_UINT32("xlevel2", X86CPU, env.cpuid_xlevel2, 0),
+    DEFINE_PROP_STRING("hv-vendor-id", X86CPU, hyperv_vendor_id),
+    DEFINE_PROP_BOOL("cpuid-0xb", X86CPU, enable_cpuid_0xb, true),
+    DEFINE_PROP_BOOL("lmce", X86CPU, enable_lmce, false),
     DEFINE_PROP_END_OF_LIST()
 };
 
@@ -3028,7 +3432,7 @@
 
     xcc->parent_realize = dc->realize;
     dc->realize = x86_cpu_realizefn;
-    dc->bus_type = TYPE_ICC_BUS;
+    dc->unrealize = x86_cpu_unrealizefn;
     dc->props = x86_cpu_properties;
 
     xcc->parent_reset = cc->reset;
@@ -3064,6 +3468,13 @@
 #endif
     cc->cpu_exec_enter = x86_cpu_exec_enter;
     cc->cpu_exec_exit = x86_cpu_exec_exit;
+
+    dc->cannot_instantiate_with_device_add_yet = false;
+    /*
+     * Reason: x86_cpu_initfn() calls cpu_exec_init(), which saves the
+     * object in cpus -> dangling pointer after final object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo x86_cpu_type_info = {
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 015f5b5..65615c0 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -16,11 +16,13 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef CPU_I386_H
-#define CPU_I386_H
 
-#include "config.h"
+#ifndef I386_CPU_H
+#define I386_CPU_H
+
 #include "qemu-common.h"
+#include "cpu-qom.h"
+#include "standard-headers/asm-x86/hyperv.h"
 
 #ifdef TARGET_X86_64
 #define TARGET_LONG_BITS 64
@@ -28,19 +30,18 @@
 #define TARGET_LONG_BITS 32
 #endif
 
-/* target supports implicit self modifying code */
-#define TARGET_HAS_SMC
+/* Maximum instruction code size */
+#define TARGET_MAX_INSN_SIZE 16
+
 /* support for self modifying code even if the modified instruction is
    close to the modifying instruction */
 #define TARGET_HAS_PRECISE_SMC
 
-#define TARGET_HAS_ICE 1
-
 #ifdef TARGET_X86_64
-#define ELF_MACHINE     EM_X86_64
+#define I386_ELF_MACHINE  EM_X86_64
 #define ELF_MACHINE_UNAME "x86_64"
 #else
-#define ELF_MACHINE     EM_386
+#define I386_ELF_MACHINE  EM_386
 #define ELF_MACHINE_UNAME "i686"
 #endif
 
@@ -129,8 +130,6 @@
    positions to ease oring with eflags. */
 /* current cpl */
 #define HF_CPL_SHIFT         0
-/* true if soft mmu is being used */
-#define HF_SOFTMMU_SHIFT     2
 /* true if hardware interrupts must be disabled for next instruction */
 #define HF_INHIBIT_IRQ_SHIFT 3
 /* 16 or 32 segments */
@@ -155,9 +154,11 @@
 #define HF_SVMI_SHIFT       21 /* SVM intercepts are active */
 #define HF_OSFXSR_SHIFT     22 /* CR4.OSFXSR */
 #define HF_SMAP_SHIFT       23 /* CR4.SMAP */
+#define HF_IOBPT_SHIFT      24 /* an io breakpoint enabled */
+#define HF_MPX_EN_SHIFT     25 /* MPX Enabled (CR4+XCR0+BNDCFGx) */
+#define HF_MPX_IU_SHIFT     26 /* BND registers in-use */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
-#define HF_SOFTMMU_MASK      (1 << HF_SOFTMMU_SHIFT)
 #define HF_INHIBIT_IRQ_MASK  (1 << HF_INHIBIT_IRQ_SHIFT)
 #define HF_CS32_MASK         (1 << HF_CS32_SHIFT)
 #define HF_SS32_MASK         (1 << HF_SS32_SHIFT)
@@ -178,18 +179,25 @@
 #define HF_SVMI_MASK         (1 << HF_SVMI_SHIFT)
 #define HF_OSFXSR_MASK       (1 << HF_OSFXSR_SHIFT)
 #define HF_SMAP_MASK         (1 << HF_SMAP_SHIFT)
+#define HF_IOBPT_MASK        (1 << HF_IOBPT_SHIFT)
+#define HF_MPX_EN_MASK       (1 << HF_MPX_EN_SHIFT)
+#define HF_MPX_IU_MASK       (1 << HF_MPX_IU_SHIFT)
 
 /* hflags2 */
 
-#define HF2_GIF_SHIFT        0 /* if set CPU takes interrupts */
-#define HF2_HIF_SHIFT        1 /* value of IF_MASK when entering SVM */
-#define HF2_NMI_SHIFT        2 /* CPU serving NMI */
-#define HF2_VINTR_SHIFT      3 /* value of V_INTR_MASKING bit */
+#define HF2_GIF_SHIFT            0 /* if set CPU takes interrupts */
+#define HF2_HIF_SHIFT            1 /* value of IF_MASK when entering SVM */
+#define HF2_NMI_SHIFT            2 /* CPU serving NMI */
+#define HF2_VINTR_SHIFT          3 /* value of V_INTR_MASKING bit */
+#define HF2_SMM_INSIDE_NMI_SHIFT 4 /* CPU serving SMI nested inside NMI */
+#define HF2_MPX_PR_SHIFT         5 /* BNDCFGx.BNDPRESERVE */
 
-#define HF2_GIF_MASK          (1 << HF2_GIF_SHIFT)
-#define HF2_HIF_MASK          (1 << HF2_HIF_SHIFT)
-#define HF2_NMI_MASK          (1 << HF2_NMI_SHIFT)
-#define HF2_VINTR_MASK        (1 << HF2_VINTR_SHIFT)
+#define HF2_GIF_MASK            (1 << HF2_GIF_SHIFT)
+#define HF2_HIF_MASK            (1 << HF2_HIF_SHIFT)
+#define HF2_NMI_MASK            (1 << HF2_NMI_SHIFT)
+#define HF2_VINTR_MASK          (1 << HF2_VINTR_SHIFT)
+#define HF2_SMM_INSIDE_NMI_MASK (1 << HF2_SMM_INSIDE_NMI_SHIFT)
+#define HF2_MPX_PR_MASK         (1 << HF2_MPX_PR_SHIFT)
 
 #define CR0_PE_SHIFT 0
 #define CR0_MP_SHIFT 1
@@ -223,6 +231,7 @@
 #define CR4_OSXSAVE_MASK (1U << 18)
 #define CR4_SMEP_MASK   (1U << 20)
 #define CR4_SMAP_MASK   (1U << 21)
+#define CR4_PKE_MASK   (1U << 22)
 
 #define DR6_BD          (1 << 13)
 #define DR6_BS          (1 << 14)
@@ -233,6 +242,7 @@
 #define DR7_TYPE_SHIFT  16
 #define DR7_LEN_SHIFT   18
 #define DR7_FIXED_1     0x00000400
+#define DR7_GLOBAL_BP_MASK   0xaa
 #define DR7_LOCAL_BP_MASK    0x55
 #define DR7_MAX_BP           4
 #define DR7_TYPE_BP_INST     0x0
@@ -250,6 +260,7 @@
 #define PG_PSE_BIT      7
 #define PG_GLOBAL_BIT   8
 #define PG_PSE_PAT_BIT  12
+#define PG_PKRU_BIT     59
 #define PG_NX_BIT       63
 
 #define PG_PRESENT_MASK  (1 << PG_PRESENT_BIT)
@@ -265,7 +276,8 @@
 #define PG_ADDRESS_MASK  0x000ffffffffff000LL
 #define PG_HI_RSVD_MASK  (PG_ADDRESS_MASK & ~PHYS_ADDR_MASK)
 #define PG_HI_USER_MASK  0x7ff0000000000000LL
-#define PG_NX_MASK       (1LL << PG_NX_BIT)
+#define PG_PKRU_MASK     (15ULL << PG_PKRU_BIT)
+#define PG_NX_MASK       (1ULL << PG_NX_BIT)
 
 #define PG_ERROR_W_BIT     1
 
@@ -274,16 +286,23 @@
 #define PG_ERROR_U_MASK    0x04
 #define PG_ERROR_RSVD_MASK 0x08
 #define PG_ERROR_I_D_MASK  0x10
+#define PG_ERROR_PK_MASK   0x20
 
 #define MCG_CTL_P       (1ULL<<8)   /* MCG_CAP register available */
 #define MCG_SER_P       (1ULL<<24) /* MCA recovery/new status bits */
+#define MCG_LMCE_P      (1ULL<<27) /* Local Machine Check Supported */
 
 #define MCE_CAP_DEF     (MCG_CTL_P|MCG_SER_P)
 #define MCE_BANKS_DEF   10
 
+#define MCG_CAP_BANKS_MASK 0xff
+
 #define MCG_STATUS_RIPV (1ULL<<0)   /* restart ip valid */
 #define MCG_STATUS_EIPV (1ULL<<1)   /* ip points to correct instruction */
 #define MCG_STATUS_MCIP (1ULL<<2)   /* machine check in progress */
+#define MCG_STATUS_LMCE (1ULL<<3)   /* Local MCE signaled */
+
+#define MCG_EXT_CTL_LMCE_EN (1ULL<<0) /* Local MCE enabled */
 
 #define MCI_STATUS_VAL   (1ULL<<63)  /* valid error */
 #define MCI_STATUS_OVER  (1ULL<<62)  /* previous errors lost */
@@ -306,13 +325,18 @@
 #define MSR_IA32_APICBASE               0x1b
 #define MSR_IA32_APICBASE_BSP           (1<<8)
 #define MSR_IA32_APICBASE_ENABLE        (1<<11)
-#define MSR_IA32_APICBASE_BASE          (0xfffff<<12)
+#define MSR_IA32_APICBASE_BASE          (0xfffffU<<12)
 #define MSR_IA32_FEATURE_CONTROL        0x0000003a
 #define MSR_TSC_ADJUST                  0x0000003b
 #define MSR_IA32_TSCDEADLINE            0x6e0
 
+#define FEATURE_CONTROL_LOCKED                    (1<<0)
+#define FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX (1<<2)
+#define FEATURE_CONTROL_LMCE                      (1<<20)
+
 #define MSR_P6_PERFCTR0                 0xc1
 
+#define MSR_IA32_SMBASE                 0x9e
 #define MSR_MTRRcap                     0xfe
 #define MSR_MTRRcap_VCNT                8
 #define MSR_MTRRcap_FIXRANGE_SUPPORT    (1 << 8)
@@ -325,6 +349,7 @@
 #define MSR_MCG_CAP                     0x179
 #define MSR_MCG_STATUS                  0x17a
 #define MSR_MCG_CTL                     0x17b
+#define MSR_MCG_EXT_CTL                 0x4d0
 
 #define MSR_P6_EVNTSEL0                 0x186
 
@@ -389,28 +414,45 @@
 #define MSR_VM_HSAVE_PA                 0xc0010117
 
 #define MSR_IA32_BNDCFGS                0x00000d90
+#define MSR_IA32_XSS                    0x00000da0
 
-#define XSTATE_FP                       (1ULL << 0)
-#define XSTATE_SSE                      (1ULL << 1)
-#define XSTATE_YMM                      (1ULL << 2)
-#define XSTATE_BNDREGS                  (1ULL << 3)
-#define XSTATE_BNDCSR                   (1ULL << 4)
-#define XSTATE_OPMASK                   (1ULL << 5)
-#define XSTATE_ZMM_Hi256                (1ULL << 6)
-#define XSTATE_Hi16_ZMM                 (1ULL << 7)
+#define XSTATE_FP_BIT                   0
+#define XSTATE_SSE_BIT                  1
+#define XSTATE_YMM_BIT                  2
+#define XSTATE_BNDREGS_BIT              3
+#define XSTATE_BNDCSR_BIT               4
+#define XSTATE_OPMASK_BIT               5
+#define XSTATE_ZMM_Hi256_BIT            6
+#define XSTATE_Hi16_ZMM_BIT             7
+#define XSTATE_PKRU_BIT                 9
 
+#define XSTATE_FP_MASK                  (1ULL << XSTATE_FP_BIT)
+#define XSTATE_SSE_MASK                 (1ULL << XSTATE_SSE_BIT)
+#define XSTATE_YMM_MASK                 (1ULL << XSTATE_YMM_BIT)
+#define XSTATE_BNDREGS_MASK             (1ULL << XSTATE_BNDREGS_BIT)
+#define XSTATE_BNDCSR_MASK              (1ULL << XSTATE_BNDCSR_BIT)
+#define XSTATE_OPMASK_MASK              (1ULL << XSTATE_OPMASK_BIT)
+#define XSTATE_ZMM_Hi256_MASK           (1ULL << XSTATE_ZMM_Hi256_BIT)
+#define XSTATE_Hi16_ZMM_MASK            (1ULL << XSTATE_Hi16_ZMM_BIT)
+#define XSTATE_PKRU_MASK                (1ULL << XSTATE_PKRU_BIT)
 
 /* CPUID feature words */
 typedef enum FeatureWord {
     FEAT_1_EDX,         /* CPUID[1].EDX */
     FEAT_1_ECX,         /* CPUID[1].ECX */
     FEAT_7_0_EBX,       /* CPUID[EAX=7,ECX=0].EBX */
+    FEAT_7_0_ECX,       /* CPUID[EAX=7,ECX=0].ECX */
     FEAT_8000_0001_EDX, /* CPUID[8000_0001].EDX */
     FEAT_8000_0001_ECX, /* CPUID[8000_0001].ECX */
     FEAT_8000_0007_EDX, /* CPUID[8000_0007].EDX */
     FEAT_C000_0001_EDX, /* CPUID[C000_0001].EDX */
     FEAT_KVM,           /* CPUID[4000_0001].EAX (KVM_CPUID_FEATURES) */
+    FEAT_HYPERV_EAX,    /* CPUID[4000_0003].EAX */
+    FEAT_HYPERV_EBX,    /* CPUID[4000_0003].EBX */
+    FEAT_HYPERV_EDX,    /* CPUID[4000_0003].EDX */
     FEAT_SVM,           /* CPUID[8000_000A].EDX */
+    FEAT_XSAVE,         /* CPUID[EAX=0xd,ECX=1].EAX */
+    FEAT_6_EAX,         /* CPUID[6].EAX */
     FEATURE_WORDS,
 } FeatureWord;
 
@@ -567,10 +609,25 @@
 #define CPUID_7_0_EBX_RDSEED   (1U << 18)
 #define CPUID_7_0_EBX_ADX      (1U << 19)
 #define CPUID_7_0_EBX_SMAP     (1U << 20)
+#define CPUID_7_0_EBX_PCOMMIT  (1U << 22) /* Persistent Commit */
+#define CPUID_7_0_EBX_CLFLUSHOPT (1U << 23) /* Flush a Cache Line Optimized */
+#define CPUID_7_0_EBX_CLWB     (1U << 24) /* Cache Line Write Back */
 #define CPUID_7_0_EBX_AVX512PF (1U << 26) /* AVX-512 Prefetch */
 #define CPUID_7_0_EBX_AVX512ER (1U << 27) /* AVX-512 Exponential and Reciprocal */
 #define CPUID_7_0_EBX_AVX512CD (1U << 28) /* AVX-512 Conflict Detection */
 
+#define CPUID_7_0_ECX_UMIP     (1U << 2)
+#define CPUID_7_0_ECX_PKU      (1U << 3)
+#define CPUID_7_0_ECX_OSPKE    (1U << 4)
+#define CPUID_7_0_ECX_RDPID    (1U << 22)
+
+#define CPUID_XSAVE_XSAVEOPT   (1U << 0)
+#define CPUID_XSAVE_XSAVEC     (1U << 1)
+#define CPUID_XSAVE_XGETBV1    (1U << 2)
+#define CPUID_XSAVE_XSAVES     (1U << 3)
+
+#define CPUID_6_EAX_ARAT       (1U << 2)
+
 /* CPUID[0x80000007].EDX flags: */
 #define CPUID_APM_INVTSC       (1U << 8)
 
@@ -591,6 +648,11 @@
 #define CPUID_MWAIT_IBE     (1U << 1) /* Interrupts can exit capability */
 #define CPUID_MWAIT_EMX     (1U << 0) /* enumeration supported */
 
+/* CPUID[0xB].ECX level types */
+#define CPUID_TOPOLOGY_LEVEL_INVALID  (0U << 8)
+#define CPUID_TOPOLOGY_LEVEL_SMT      (1U << 8)
+#define CPUID_TOPOLOGY_LEVEL_CORE     (2U << 8)
+
 #ifndef HYPERV_SPINLOCK_NEVER_RETRY
 #define HYPERV_SPINLOCK_NEVER_RETRY             0xFFFFFFFF
 #endif
@@ -704,40 +766,18 @@
     uint32_t flags;
 } SegmentCache;
 
-typedef union {
-    uint8_t _b[16];
-    uint16_t _w[8];
-    uint32_t _l[4];
-    uint64_t _q[2];
-    float32 _s[4];
-    float64 _d[2];
-} XMMReg;
+#define MMREG_UNION(n, bits)        \
+    union n {                       \
+        uint8_t  _b_##n[(bits)/8];  \
+        uint16_t _w_##n[(bits)/16]; \
+        uint32_t _l_##n[(bits)/32]; \
+        uint64_t _q_##n[(bits)/64]; \
+        float32  _s_##n[(bits)/32]; \
+        float64  _d_##n[(bits)/64]; \
+    }
 
-typedef union {
-    uint8_t _b[32];
-    uint16_t _w[16];
-    uint32_t _l[8];
-    uint64_t _q[4];
-    float32 _s[8];
-    float64 _d[4];
-} YMMReg;
-
-typedef union {
-    uint8_t _b[64];
-    uint16_t _w[32];
-    uint32_t _l[16];
-    uint64_t _q[8];
-    float32 _s[16];
-    float64 _d[8];
-} ZMMReg;
-
-typedef union {
-    uint8_t _b[8];
-    uint16_t _w[4];
-    uint32_t _l[2];
-    float32 _s[2];
-    uint64_t q;
-} MMXReg;
+typedef MMREG_UNION(ZMMReg, 512) ZMMReg;
+typedef MMREG_UNION(MMXReg, 64)  MMXReg;
 
 typedef struct BNDReg {
     uint64_t lb;
@@ -749,60 +789,36 @@
     uint64_t sts;
 } BNDCSReg;
 
+#define BNDCFG_ENABLE       1ULL
+#define BNDCFG_BNDPRESERVE  2ULL
+#define BNDCFG_BDIR_MASK    TARGET_PAGE_MASK
+
 #ifdef HOST_WORDS_BIGENDIAN
-#define ZMM_B(n) _b[63 - (n)]
-#define ZMM_W(n) _w[31 - (n)]
-#define ZMM_L(n) _l[15 - (n)]
-#define ZMM_S(n) _s[15 - (n)]
-#define ZMM_Q(n) _q[7 - (n)]
-#define ZMM_D(n) _d[7 - (n)]
+#define ZMM_B(n) _b_ZMMReg[63 - (n)]
+#define ZMM_W(n) _w_ZMMReg[31 - (n)]
+#define ZMM_L(n) _l_ZMMReg[15 - (n)]
+#define ZMM_S(n) _s_ZMMReg[15 - (n)]
+#define ZMM_Q(n) _q_ZMMReg[7 - (n)]
+#define ZMM_D(n) _d_ZMMReg[7 - (n)]
 
-#define YMM_B(n) _b[31 - (n)]
-#define YMM_W(n) _w[15 - (n)]
-#define YMM_L(n) _l[7 - (n)]
-#define YMM_S(n) _s[7 - (n)]
-#define YMM_Q(n) _q[3 - (n)]
-#define YMM_D(n) _d[3 - (n)]
-
-#define XMM_B(n) _b[15 - (n)]
-#define XMM_W(n) _w[7 - (n)]
-#define XMM_L(n) _l[3 - (n)]
-#define XMM_S(n) _s[3 - (n)]
-#define XMM_Q(n) _q[1 - (n)]
-#define XMM_D(n) _d[1 - (n)]
-
-#define MMX_B(n) _b[7 - (n)]
-#define MMX_W(n) _w[3 - (n)]
-#define MMX_L(n) _l[1 - (n)]
-#define MMX_S(n) _s[1 - (n)]
+#define MMX_B(n) _b_MMXReg[7 - (n)]
+#define MMX_W(n) _w_MMXReg[3 - (n)]
+#define MMX_L(n) _l_MMXReg[1 - (n)]
+#define MMX_S(n) _s_MMXReg[1 - (n)]
 #else
-#define ZMM_B(n) _b[n]
-#define ZMM_W(n) _w[n]
-#define ZMM_L(n) _l[n]
-#define ZMM_S(n) _s[n]
-#define ZMM_Q(n) _q[n]
-#define ZMM_D(n) _d[n]
+#define ZMM_B(n) _b_ZMMReg[n]
+#define ZMM_W(n) _w_ZMMReg[n]
+#define ZMM_L(n) _l_ZMMReg[n]
+#define ZMM_S(n) _s_ZMMReg[n]
+#define ZMM_Q(n) _q_ZMMReg[n]
+#define ZMM_D(n) _d_ZMMReg[n]
 
-#define YMM_B(n) _b[n]
-#define YMM_W(n) _w[n]
-#define YMM_L(n) _l[n]
-#define YMM_S(n) _s[n]
-#define YMM_Q(n) _q[n]
-#define YMM_D(n) _d[n]
-
-#define XMM_B(n) _b[n]
-#define XMM_W(n) _w[n]
-#define XMM_L(n) _l[n]
-#define XMM_S(n) _s[n]
-#define XMM_Q(n) _q[n]
-#define XMM_D(n) _d[n]
-
-#define MMX_B(n) _b[n]
-#define MMX_W(n) _w[n]
-#define MMX_L(n) _l[n]
-#define MMX_S(n) _s[n]
+#define MMX_B(n) _b_MMXReg[n]
+#define MMX_W(n) _w_MMXReg[n]
+#define MMX_L(n) _l_MMXReg[n]
+#define MMX_S(n) _s_MMXReg[n]
 #endif
-#define MMX_Q(n) q
+#define MMX_Q(n) _q_MMXReg[n]
 
 typedef union {
     floatx80 d __attribute__((aligned(16)));
@@ -827,9 +843,110 @@
 #define MAX_GP_COUNTERS    (MSR_IA32_PERF_STATUS - MSR_P6_EVNTSEL0)
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 #define NB_OPMASK_REGS 8
 
+/* CPU can't have 0xFFFFFFFF APIC ID, use that value to distinguish
+ * that APIC ID hasn't been set yet
+ */
+#define UNASSIGNED_APIC_ID 0xFFFFFFFF
+
+typedef union X86LegacyXSaveArea {
+    struct {
+        uint16_t fcw;
+        uint16_t fsw;
+        uint8_t ftw;
+        uint8_t reserved;
+        uint16_t fpop;
+        uint64_t fpip;
+        uint64_t fpdp;
+        uint32_t mxcsr;
+        uint32_t mxcsr_mask;
+        FPReg fpregs[8];
+        uint8_t xmm_regs[16][16];
+    };
+    uint8_t data[512];
+} X86LegacyXSaveArea;
+
+typedef struct X86XSaveHeader {
+    uint64_t xstate_bv;
+    uint64_t xcomp_bv;
+    uint8_t reserved[48];
+} X86XSaveHeader;
+
+/* Ext. save area 2: AVX State */
+typedef struct XSaveAVX {
+    uint8_t ymmh[16][16];
+} XSaveAVX;
+
+/* Ext. save area 3: BNDREG */
+typedef struct XSaveBNDREG {
+    BNDReg bnd_regs[4];
+} XSaveBNDREG;
+
+/* Ext. save area 4: BNDCSR */
+typedef union XSaveBNDCSR {
+    BNDCSReg bndcsr;
+    uint8_t data[64];
+} XSaveBNDCSR;
+
+/* Ext. save area 5: Opmask */
+typedef struct XSaveOpmask {
+    uint64_t opmask_regs[NB_OPMASK_REGS];
+} XSaveOpmask;
+
+/* Ext. save area 6: ZMM_Hi256 */
+typedef struct XSaveZMM_Hi256 {
+    uint8_t zmm_hi256[16][32];
+} XSaveZMM_Hi256;
+
+/* Ext. save area 7: Hi16_ZMM */
+typedef struct XSaveHi16_ZMM {
+    uint8_t hi16_zmm[16][64];
+} XSaveHi16_ZMM;
+
+/* Ext. save area 9: PKRU state */
+typedef struct XSavePKRU {
+    uint32_t pkru;
+    uint32_t padding;
+} XSavePKRU;
+
+typedef struct X86XSaveArea {
+    X86LegacyXSaveArea legacy;
+    X86XSaveHeader header;
+
+    /* Extended save areas: */
+
+    /* AVX State: */
+    XSaveAVX avx_state;
+    uint8_t padding[960 - 576 - sizeof(XSaveAVX)];
+    /* MPX State: */
+    XSaveBNDREG bndreg_state;
+    XSaveBNDCSR bndcsr_state;
+    /* AVX-512 State: */
+    XSaveOpmask opmask_state;
+    XSaveZMM_Hi256 zmm_hi256_state;
+    XSaveHi16_ZMM hi16_zmm_state;
+    /* PKRU State: */
+    XSavePKRU pkru_state;
+} X86XSaveArea;
+
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, avx_state) != 0x240);
+QEMU_BUILD_BUG_ON(sizeof(XSaveAVX) != 0x100);
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, bndreg_state) != 0x3c0);
+QEMU_BUILD_BUG_ON(sizeof(XSaveBNDREG) != 0x40);
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, bndcsr_state) != 0x400);
+QEMU_BUILD_BUG_ON(sizeof(XSaveBNDCSR) != 0x40);
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, opmask_state) != 0x440);
+QEMU_BUILD_BUG_ON(sizeof(XSaveOpmask) != 0x40);
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, zmm_hi256_state) != 0x480);
+QEMU_BUILD_BUG_ON(sizeof(XSaveZMM_Hi256) != 0x200);
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, hi16_zmm_state) != 0x680);
+QEMU_BUILD_BUG_ON(sizeof(XSaveHi16_ZMM) != 0x400);
+QEMU_BUILD_BUG_ON(offsetof(X86XSaveArea, pkru_state) != 0xA80);
+QEMU_BUILD_BUG_ON(sizeof(XSavePKRU) != 0x8);
+
 typedef enum TPRAccess {
     TPR_ACCESS_READ,
     TPR_ACCESS_WRITE,
@@ -866,6 +983,7 @@
     BNDReg bnd_regs[4];
     BNDCSReg bndcs_regs;
     uint64_t msr_bndcfgs;
+    uint64_t efer;
 
     /* Beginning of state preserved by INIT (dummy marker).  */
     struct {} start_init_save;
@@ -888,23 +1006,16 @@
     float_status mmx_status; /* for 3DNow! float ops */
     float_status sse_status;
     uint32_t mxcsr;
-    XMMReg xmm_regs[CPU_NB_REGS];
-    XMMReg xmm_t0;
+    ZMMReg xmm_regs[CPU_NB_REGS == 8 ? 8 : 32];
+    ZMMReg xmm_t0;
     MMXReg mmx_t0;
 
-    XMMReg ymmh_regs[CPU_NB_REGS];
-
     uint64_t opmask_regs[NB_OPMASK_REGS];
-    YMMReg zmmh_regs[CPU_NB_REGS];
-#ifdef TARGET_X86_64
-    ZMMReg hi16_zmm_regs[CPU_NB_REGS];
-#endif
 
     /* sysenter registers */
     uint32_t sysenter_cs;
     target_ulong sysenter_esp;
     target_ulong sysenter_eip;
-    uint64_t efer;
     uint64_t star;
 
     uint64_t vm_hsave;
@@ -948,12 +1059,21 @@
     uint64_t msr_hv_guest_os_id;
     uint64_t msr_hv_vapic;
     uint64_t msr_hv_tsc;
+    uint64_t msr_hv_crash_params[HV_X64_MSR_CRASH_PARAMS];
+    uint64_t msr_hv_runtime;
+    uint64_t msr_hv_synic_control;
+    uint64_t msr_hv_synic_version;
+    uint64_t msr_hv_synic_evt_page;
+    uint64_t msr_hv_synic_msg_page;
+    uint64_t msr_hv_synic_sint[HV_SYNIC_SINT_COUNT];
+    uint64_t msr_hv_stimer_config[HV_SYNIC_STIMER_COUNT];
+    uint64_t msr_hv_stimer_count[HV_SYNIC_STIMER_COUNT];
 
     /* exception/interrupt handling */
     int error_code;
     int exception_is_int;
     target_ulong exception_next_eip;
-    target_ulong dr[8]; /* debug registers */
+    target_ulong dr[8]; /* debug registers; note dr4 and dr5 are unused */
     union {
         struct CPUBreakpoint *cpu_breakpoint[4];
         struct CPUWatchpoint *cpu_watchpoint[4];
@@ -988,7 +1108,6 @@
     uint32_t cpuid_version;
     FeatureWordArray features;
     uint32_t cpuid_model[12];
-    uint32_t cpuid_apic_id;
 
     /* MTRRs */
     uint64_t mtrr_fixed[11];
@@ -1003,11 +1122,13 @@
     uint8_t has_error_code;
     uint32_t sipi_vector;
     bool tsc_valid;
-    int tsc_khz;
+    int64_t tsc_khz;
+    int64_t user_tsc_khz; /* for sanity check only */
     void *kvm_xsave_buf;
 
     uint64_t mcg_cap;
     uint64_t mcg_ctl;
+    uint64_t mcg_ext_ctl;
     uint64_t mce_banks[MCE_BANKS_DEF*4];
 
     uint64_t tsc_aux;
@@ -1019,18 +1140,138 @@
     uint64_t xstate_bv;
 
     uint64_t xcr0;
+    uint64_t xss;
+
+    uint32_t pkru;
 
     TPRAccess tpr_access_type;
 } CPUX86State;
 
-#include "cpu-qom.h"
+struct kvm_msrs;
+
+/**
+ * X86CPU:
+ * @env: #CPUX86State
+ * @migratable: If set, only migratable flags will be accepted when "enforce"
+ * mode is used, and only migratable flags will be included in the "host"
+ * CPU model.
+ *
+ * An x86 CPU.
+ */
+struct X86CPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUX86State env;
+
+    bool hyperv_vapic;
+    bool hyperv_relaxed_timing;
+    int hyperv_spinlock_attempts;
+    char *hyperv_vendor_id;
+    bool hyperv_time;
+    bool hyperv_crash;
+    bool hyperv_reset;
+    bool hyperv_vpindex;
+    bool hyperv_runtime;
+    bool hyperv_synic;
+    bool hyperv_stimer;
+    bool check_cpuid;
+    bool enforce_cpuid;
+    bool expose_kvm;
+    bool migratable;
+    bool host_features;
+    uint32_t apic_id;
+
+    /* if true the CPUID code directly forward host cache leaves to the guest */
+    bool cache_info_passthrough;
+
+    /* Features that were filtered out because of missing host capabilities */
+    uint32_t filtered_features[FEATURE_WORDS];
+
+    /* Enable PMU CPUID bits. This can't be enabled by default yet because
+     * it doesn't have ABI stability guarantees, as it passes all PMU CPUID
+     * bits returned by GET_SUPPORTED_CPUID (that depend on host CPU and kernel
+     * capabilities) directly to the guest.
+     */
+    bool enable_pmu;
+
+    /* LMCE support can be enabled/disabled via cpu option 'lmce=on/off'. It is
+     * disabled by default to avoid breaking migration between QEMU with
+     * different LMCE configurations.
+     */
+    bool enable_lmce;
+
+    /* Compatibility bits for old machine types: */
+    bool enable_cpuid_0xb;
+
+    /* if true fill the top bits of the MTRR_PHYSMASKn variable range */
+    bool fill_mtrr_mask;
+
+    /* if true override the phys_bits value with a value read from the host */
+    bool host_phys_bits;
+
+    /* Number of physical address bits supported */
+    uint32_t phys_bits;
+
+    /* in order to simplify APIC support, we leave this pointer to the
+       user */
+    struct DeviceState *apic_state;
+    struct MemoryRegion *cpu_as_root, *cpu_as_mem, *smram;
+    Notifier machine_done;
+
+    struct kvm_msrs *kvm_msr_buf;
+
+    int32_t socket_id;
+    int32_t core_id;
+    int32_t thread_id;
+};
+
+static inline X86CPU *x86_env_get_cpu(CPUX86State *env)
+{
+    return container_of(env, X86CPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(x86_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(X86CPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern struct VMStateDescription vmstate_x86_cpu;
+#endif
+
+/**
+ * x86_cpu_do_interrupt:
+ * @cpu: vCPU the interrupt is to be handled by.
+ */
+void x86_cpu_do_interrupt(CPUState *cpu);
+bool x86_cpu_exec_interrupt(CPUState *cpu, int int_req);
+
+int x86_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cpu,
+                             int cpuid, void *opaque);
+int x86_cpu_write_elf32_note(WriteCoreDumpFunction f, CPUState *cpu,
+                             int cpuid, void *opaque);
+int x86_cpu_write_elf64_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
+                                 void *opaque);
+int x86_cpu_write_elf32_qemunote(WriteCoreDumpFunction f, CPUState *cpu,
+                                 void *opaque);
+
+void x86_cpu_get_memory_mapping(CPUState *cpu, MemoryMappingList *list,
+                                Error **errp);
+
+void x86_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
+                        int flags);
+
+hwaddr x86_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+
+int x86_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int x86_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+
+void x86_cpu_exec_enter(CPUState *cpu);
+void x86_cpu_exec_exit(CPUState *cpu);
 
 X86CPU *cpu_x86_init(const char *cpu_model);
-X86CPU *cpu_x86_create(const char *cpu_model, DeviceState *icc_bridge,
-                       Error **errp);
-int cpu_x86_exec(CPUX86State *s);
 void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf);
-void x86_cpudef_setup(void);
 int cpu_x86_support_mca_broadcast(CPUX86State *env);
 
 int cpu_get_pic_interrupt(CPUX86State *s);
@@ -1139,7 +1380,14 @@
 int cpu_x86_signal_handler(int host_signum, void *pinfo,
                            void *puc);
 
-/* cpuid.c */
+/* cpu.c */
+typedef struct ExtSaveArea {
+    uint32_t feature, bits;
+    uint32_t offset, size;
+} ExtSaveArea;
+
+extern const ExtSaveArea x86_ext_save_areas[];
+
 void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
                    uint32_t *eax, uint32_t *ebx,
                    uint32_t *ecx, uint32_t *edx);
@@ -1152,45 +1400,27 @@
                              int is_write, int mmu_idx);
 void x86_cpu_set_a20(X86CPU *cpu, int a20_state);
 
-static inline bool hw_local_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return (dr7 >> (index * 2)) & 1;
-}
+#ifndef CONFIG_USER_ONLY
+uint8_t x86_ldub_phys(CPUState *cs, hwaddr addr);
+uint32_t x86_lduw_phys(CPUState *cs, hwaddr addr);
+uint32_t x86_ldl_phys(CPUState *cs, hwaddr addr);
+uint64_t x86_ldq_phys(CPUState *cs, hwaddr addr);
+void x86_stb_phys(CPUState *cs, hwaddr addr, uint8_t val);
+void x86_stl_phys_notdirty(CPUState *cs, hwaddr addr, uint32_t val);
+void x86_stw_phys(CPUState *cs, hwaddr addr, uint32_t val);
+void x86_stl_phys(CPUState *cs, hwaddr addr, uint32_t val);
+void x86_stq_phys(CPUState *cs, hwaddr addr, uint64_t val);
+#endif
 
-static inline bool hw_global_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return (dr7 >> (index * 2)) & 2;
-
-}
-static inline bool hw_breakpoint_enabled(unsigned long dr7, int index)
-{
-    return hw_global_breakpoint_enabled(dr7, index) ||
-           hw_local_breakpoint_enabled(dr7, index);
-}
-
-static inline int hw_breakpoint_type(unsigned long dr7, int index)
-{
-    return (dr7 >> (DR7_TYPE_SHIFT + (index * 4))) & 3;
-}
-
-static inline int hw_breakpoint_len(unsigned long dr7, int index)
-{
-    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 4))) & 3);
-    return (len == 2) ? 8 : len + 1;
-}
-
-void hw_breakpoint_insert(CPUX86State *env, int index);
-void hw_breakpoint_remove(CPUX86State *env, int index);
-bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update);
 void breakpoint_handler(CPUState *cs);
 
 /* will be suppressed */
 void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
 void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
+void cpu_x86_update_dr7(CPUX86State *env, uint32_t new_dr7);
 
 /* hw/pc.c */
-void cpu_smm_update(CPUX86State *env);
 uint64_t cpu_get_tsc(CPUX86State *env);
 
 #define TARGET_PAGE_BITS 12
@@ -1209,25 +1439,17 @@
 /* XXX: This value should match the one returned by CPUID
  * and in exec.c */
 # if defined(TARGET_X86_64)
-# define PHYS_ADDR_MASK 0xffffffffffLL
+# define TCG_PHYS_ADDR_BITS 40
 # else
-# define PHYS_ADDR_MASK 0xfffffffffLL
+# define TCG_PHYS_ADDR_BITS 36
 # endif
 
-static inline CPUX86State *cpu_init(const char *cpu_model)
-{
-    X86CPU *cpu = cpu_x86_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define PHYS_ADDR_MASK MAKE_64BIT_MASK(0, TCG_PHYS_ADDR_BITS)
 
-#define cpu_exec cpu_x86_exec
-#define cpu_gen_code cpu_x86_gen_code
+#define cpu_init(cpu_model) CPU(cpu_x86_init(cpu_model))
+
 #define cpu_signal_handler cpu_x86_signal_handler
 #define cpu_list x86_cpu_list
-#define cpudef_setup x86_cpudef_setup
 
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _ksmap
@@ -1236,7 +1458,7 @@
 #define MMU_KSMAP_IDX   0
 #define MMU_USER_IDX    1
 #define MMU_KNOSMAP_IDX 2
-static inline int cpu_mmu_index(CPUX86State *env)
+static inline int cpu_mmu_index(CPUX86State *env, bool ifetch)
 {
     return (env->hflags & HF_CPL_MASK) == 3 ? MMU_USER_IDX :
         (!(env->hflags & HF_SMAP_MASK) || (env->eflags & AC_MASK))
@@ -1272,7 +1494,7 @@
 #define ST1    ST(1)
 
 /* translate.c */
-void optimize_flags_init(void);
+void tcg_x86_init(void);
 
 #include "exec/cpu-all.h"
 #include "svm.h"
@@ -1281,10 +1503,8 @@
 #include "hw/i386/apic.h"
 #endif
 
-#include "exec/exec-all.h"
-
 static inline void cpu_get_tb_cpu_state(CPUX86State *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *cs_base = env->segs[R_CS].base;
     *pc = *cs_base + env->eip;
@@ -1304,8 +1524,12 @@
 
 /* excp_helper.c */
 void QEMU_NORETURN raise_exception(CPUX86State *env, int exception_index);
+void QEMU_NORETURN raise_exception_ra(CPUX86State *env, int exception_index,
+                                      uintptr_t retaddr);
 void QEMU_NORETURN raise_exception_err(CPUX86State *env, int exception_index,
                                        int error_code);
+void QEMU_NORETURN raise_exception_err_ra(CPUX86State *env, int exception_index,
+                                          int error_code, uintptr_t retaddr);
 void QEMU_NORETURN raise_interrupt(CPUX86State *nenv, int intno, int is_int,
                                    int error_code, int next_eip_addend);
 
@@ -1346,10 +1570,18 @@
     }
 }
 
+static inline MemTxAttrs cpu_get_mem_attrs(CPUX86State *env)
+{
+    return ((MemTxAttrs) { .secure = (env->hflags & HF_SMM_MASK) != 0 });
+}
+
 /* fpu_helper.c */
 void cpu_set_mxcsr(CPUX86State *env, uint32_t val);
 void cpu_set_fpuc(CPUX86State *env, uint16_t val);
 
+/* mem_helper.c */
+void helper_lock_init(void);
+
 /* svm_helper.c */
 void cpu_svm_check_intercept_param(CPUX86State *env1, uint32_t type,
                                    uint64_t param);
@@ -1358,24 +1590,41 @@
 /* seg_helper.c */
 void do_interrupt_x86_hardirq(CPUX86State *env, int intno, int is_hw);
 
+/* smm_helper.c */
 void do_smm_enter(X86CPU *cpu);
+void cpu_smm_update(X86CPU *cpu);
 
+/* apic.c */
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
+void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip,
+                                   TPRAccess access);
 
-void x86_cpu_compat_set_features(const char *cpu_model, FeatureWord w,
-                                 uint32_t feat_add, uint32_t feat_remove);
 
-void x86_cpu_compat_kvm_no_autoenable(FeatureWord w, uint32_t features);
-void x86_cpu_compat_kvm_no_autodisable(FeatureWord w, uint32_t features);
+/* Change the value of a KVM-specific default
+ *
+ * If value is NULL, no default will be set and the original
+ * value from the CPU model table will be kept.
+ *
+ * It is valid to call this function only for properties that
+ * are already present in the kvm_default_props table.
+ */
+void x86_cpu_change_kvm_default(const char *prop, const char *value);
 
+/* mpx_helper.c */
+void cpu_sync_bndcs_hflags(CPUX86State *env);
 
 /* Return name of 32-bit register, from a R_* constant */
 const char *get_register_name_32(unsigned int reg);
 
-uint32_t x86_cpu_apic_id_from_index(unsigned int cpu_index);
 void enable_compat_apic_id_mode(void);
 
 #define APIC_DEFAULT_ADDRESS 0xfee00000
 #define APIC_SPACE_SIZE      0x100000
 
-#endif /* CPU_I386_H */
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags);
+
+/* cpu.c */
+bool cpu_is_bsp(X86CPU *cpu);
+
+#endif /* I386_CPU_H */
diff --git a/target-i386/excp_helper.c b/target-i386/excp_helper.c
index 99fca84..f0dc499 100644
--- a/target-i386/excp_helper.c
+++ b/target-i386/excp_helper.c
@@ -17,19 +17,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
 #include "exec/helper-proto.h"
 
-#if 0
-#define raise_exception_err(env, a, b)                                  \
-    do {                                                                \
-        qemu_log("raise_exception line=%d\n", __LINE__);                \
-        (raise_exception_err)(env, a, b);                               \
-    } while (0)
-#endif
-
 void helper_raise_interrupt(CPUX86State *env, int intno, int next_eip_addend)
 {
     raise_interrupt(env, intno, 1, 0, next_eip_addend);
@@ -92,7 +86,8 @@
  */
 static void QEMU_NORETURN raise_interrupt2(CPUX86State *env, int intno,
                                            int is_int, int error_code,
-                                           int next_eip_addend)
+                                           int next_eip_addend,
+                                           uintptr_t retaddr)
 {
     CPUState *cs = CPU(x86_env_get_cpu(env));
 
@@ -108,7 +103,7 @@
     env->error_code = error_code;
     env->exception_is_int = is_int;
     env->exception_next_eip = env->eip + next_eip_addend;
-    cpu_loop_exit(cs);
+    cpu_loop_exit_restore(cs, retaddr);
 }
 
 /* shortcuts to generate exceptions */
@@ -116,16 +111,27 @@
 void QEMU_NORETURN raise_interrupt(CPUX86State *env, int intno, int is_int,
                                    int error_code, int next_eip_addend)
 {
-    raise_interrupt2(env, intno, is_int, error_code, next_eip_addend);
+    raise_interrupt2(env, intno, is_int, error_code, next_eip_addend, 0);
 }
 
 void raise_exception_err(CPUX86State *env, int exception_index,
                          int error_code)
 {
-    raise_interrupt2(env, exception_index, 0, error_code, 0);
+    raise_interrupt2(env, exception_index, 0, error_code, 0, 0);
+}
+
+void raise_exception_err_ra(CPUX86State *env, int exception_index,
+                            int error_code, uintptr_t retaddr)
+{
+    raise_interrupt2(env, exception_index, 0, error_code, 0, retaddr);
 }
 
 void raise_exception(CPUX86State *env, int exception_index)
 {
-    raise_interrupt2(env, exception_index, 0, 0, 0);
+    raise_interrupt2(env, exception_index, 0, 0, 0, 0);
+}
+
+void raise_exception_ra(CPUX86State *env, int exception_index, uintptr_t retaddr)
+{
+    raise_interrupt2(env, exception_index, 0, 0, 0, retaddr);
 }
diff --git a/target-i386/fpu_helper.c b/target-i386/fpu_helper.c
index 1d4eee3..929489b 100644
--- a/target-i386/fpu_helper.c
+++ b/target-i386/fpu_helper.c
@@ -17,11 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <math.h>
 #include "cpu.h"
 #include "exec/helper-proto.h"
-#include "qemu/aes.h"
 #include "qemu/host-utils.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #define FPU_RC_MASK         0xc00
@@ -68,22 +69,24 @@
     env->fpstt = (env->fpstt + 1) & 7;
 }
 
-static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr)
+static inline floatx80 helper_fldt(CPUX86State *env, target_ulong ptr,
+                                   uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
-    temp.l.lower = cpu_ldq_data(env, ptr);
-    temp.l.upper = cpu_lduw_data(env, ptr + 8);
+    temp.l.lower = cpu_ldq_data_ra(env, ptr, retaddr);
+    temp.l.upper = cpu_lduw_data_ra(env, ptr + 8, retaddr);
     return temp.d;
 }
 
-static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr)
+static inline void helper_fstt(CPUX86State *env, floatx80 f, target_ulong ptr,
+                               uintptr_t retaddr)
 {
     CPU_LDoubleU temp;
 
     temp.d = f;
-    cpu_stq_data(env, ptr, temp.l.lower);
-    cpu_stw_data(env, ptr + 8, temp.l.upper);
+    cpu_stq_data_ra(env, ptr, temp.l.lower, retaddr);
+    cpu_stw_data_ra(env, ptr + 8, temp.l.upper, retaddr);
 }
 
 /* x87 FPU helpers */
@@ -126,10 +129,10 @@
     return floatx80_div(a, b, &env->fp_status);
 }
 
-static void fpu_raise_exception(CPUX86State *env)
+static void fpu_raise_exception(CPUX86State *env, uintptr_t retaddr)
 {
     if (env->cr[0] & CR0_NE_MASK) {
-        raise_exception(env, EXCP10_COPR);
+        raise_exception_ra(env, EXCP10_COPR, retaddr);
     }
 #if !defined(CONFIG_USER_ONLY)
     else {
@@ -251,16 +254,34 @@
 int32_t helper_fistl_ST0(CPUX86State *env)
 {
     int32_t val;
+    signed char old_exp_flags;
+
+    old_exp_flags = get_float_exception_flags(&env->fp_status);
+    set_float_exception_flags(0, &env->fp_status);
 
     val = floatx80_to_int32(ST0, &env->fp_status);
+    if (get_float_exception_flags(&env->fp_status) & float_flag_invalid) {
+        val = 0x80000000;
+    }
+    set_float_exception_flags(get_float_exception_flags(&env->fp_status)
+                                | old_exp_flags, &env->fp_status);
     return val;
 }
 
 int64_t helper_fistll_ST0(CPUX86State *env)
 {
     int64_t val;
+    signed char old_exp_flags;
+
+    old_exp_flags = get_float_exception_flags(&env->fp_status);
+    set_float_exception_flags(0, &env->fp_status);
 
     val = floatx80_to_int64(ST0, &env->fp_status);
+    if (get_float_exception_flags(&env->fp_status) & float_flag_invalid) {
+        val = 0x8000000000000000ULL;
+    }
+    set_float_exception_flags(get_float_exception_flags(&env->fp_status)
+                                | old_exp_flags, &env->fp_status);
     return val;
 }
 
@@ -277,18 +298,12 @@
 
 int32_t helper_fisttl_ST0(CPUX86State *env)
 {
-    int32_t val;
-
-    val = floatx80_to_int32_round_to_zero(ST0, &env->fp_status);
-    return val;
+    return floatx80_to_int32_round_to_zero(ST0, &env->fp_status);
 }
 
 int64_t helper_fisttll_ST0(CPUX86State *env)
 {
-    int64_t val;
-
-    val = floatx80_to_int64_round_to_zero(ST0, &env->fp_status);
-    return val;
+    return floatx80_to_int64_round_to_zero(ST0, &env->fp_status);
 }
 
 void helper_fldt_ST0(CPUX86State *env, target_ulong ptr)
@@ -296,14 +311,14 @@
     int new_fpstt;
 
     new_fpstt = (env->fpstt - 1) & 7;
-    env->fpregs[new_fpstt].d = helper_fldt(env, ptr);
+    env->fpregs[new_fpstt].d = helper_fldt(env, ptr, GETPC());
     env->fpstt = new_fpstt;
     env->fptags[new_fpstt] = 0; /* validate stack entry */
 }
 
 void helper_fstt_ST0(CPUX86State *env, target_ulong ptr)
 {
-    helper_fstt(env, ST0, ptr);
+    helper_fstt(env, ST0, ptr, GETPC());
 }
 
 void helper_fpush(CPUX86State *env)
@@ -586,7 +601,7 @@
 void helper_fwait(CPUX86State *env)
 {
     if (env->fpus & FPUS_SE) {
-        fpu_raise_exception(env);
+        fpu_raise_exception(env, GETPC());
     }
 }
 
@@ -616,12 +631,12 @@
 
     val = 0;
     for (i = 8; i >= 0; i--) {
-        v = cpu_ldub_data(env, ptr + i);
+        v = cpu_ldub_data_ra(env, ptr + i, GETPC());
         val = (val * 100) + ((v >> 4) * 10) + (v & 0xf);
     }
     tmp = int64_to_floatx80(val, &env->fp_status);
-    if (cpu_ldub_data(env, ptr + 9) & 0x80) {
-        floatx80_chs(tmp);
+    if (cpu_ldub_data_ra(env, ptr + 9, GETPC()) & 0x80) {
+        tmp = floatx80_chs(tmp);
     }
     fpush(env);
     ST0 = tmp;
@@ -637,10 +652,10 @@
     mem_ref = ptr;
     mem_end = mem_ref + 9;
     if (val < 0) {
-        cpu_stb_data(env, mem_end, 0x80);
+        cpu_stb_data_ra(env, mem_end, 0x80, GETPC());
         val = -val;
     } else {
-        cpu_stb_data(env, mem_end, 0x00);
+        cpu_stb_data_ra(env, mem_end, 0x00, GETPC());
     }
     while (mem_ref < mem_end) {
         if (val == 0) {
@@ -649,10 +664,10 @@
         v = val % 100;
         val = val / 100;
         v = ((v / 10) << 4) | (v % 10);
-        cpu_stb_data(env, mem_ref++, v);
+        cpu_stb_data_ra(env, mem_ref++, v, GETPC());
     }
     while (mem_ref < mem_end) {
-        cpu_stb_data(env, mem_ref++, 0);
+        cpu_stb_data_ra(env, mem_ref++, 0, GETPC());
     }
 }
 
@@ -960,7 +975,8 @@
     }
 }
 
-void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
+static void do_fstenv(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     int fpus, fptag, exp, i;
     uint64_t mant;
@@ -988,37 +1004,43 @@
     }
     if (data32) {
         /* 32 bit */
-        cpu_stl_data(env, ptr, env->fpuc);
-        cpu_stl_data(env, ptr + 4, fpus);
-        cpu_stl_data(env, ptr + 8, fptag);
-        cpu_stl_data(env, ptr + 12, 0); /* fpip */
-        cpu_stl_data(env, ptr + 16, 0); /* fpcs */
-        cpu_stl_data(env, ptr + 20, 0); /* fpoo */
-        cpu_stl_data(env, ptr + 24, 0); /* fpos */
+        cpu_stl_data_ra(env, ptr, env->fpuc, retaddr);
+        cpu_stl_data_ra(env, ptr + 4, fpus, retaddr);
+        cpu_stl_data_ra(env, ptr + 8, fptag, retaddr);
+        cpu_stl_data_ra(env, ptr + 12, 0, retaddr); /* fpip */
+        cpu_stl_data_ra(env, ptr + 16, 0, retaddr); /* fpcs */
+        cpu_stl_data_ra(env, ptr + 20, 0, retaddr); /* fpoo */
+        cpu_stl_data_ra(env, ptr + 24, 0, retaddr); /* fpos */
     } else {
         /* 16 bit */
-        cpu_stw_data(env, ptr, env->fpuc);
-        cpu_stw_data(env, ptr + 2, fpus);
-        cpu_stw_data(env, ptr + 4, fptag);
-        cpu_stw_data(env, ptr + 6, 0);
-        cpu_stw_data(env, ptr + 8, 0);
-        cpu_stw_data(env, ptr + 10, 0);
-        cpu_stw_data(env, ptr + 12, 0);
+        cpu_stw_data_ra(env, ptr, env->fpuc, retaddr);
+        cpu_stw_data_ra(env, ptr + 2, fpus, retaddr);
+        cpu_stw_data_ra(env, ptr + 4, fptag, retaddr);
+        cpu_stw_data_ra(env, ptr + 6, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 8, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 10, 0, retaddr);
+        cpu_stw_data_ra(env, ptr + 12, 0, retaddr);
     }
 }
 
-void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
+void helper_fstenv(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fstenv(env, ptr, data32, GETPC());
+}
+
+static void do_fldenv(CPUX86State *env, target_ulong ptr, int data32,
+                      uintptr_t retaddr)
 {
     int i, fpus, fptag;
 
     if (data32) {
-        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-        fpus = cpu_lduw_data(env, ptr + 4);
-        fptag = cpu_lduw_data(env, ptr + 8);
+        cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+        fpus = cpu_lduw_data_ra(env, ptr + 4, retaddr);
+        fptag = cpu_lduw_data_ra(env, ptr + 8, retaddr);
     } else {
-        cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-        fpus = cpu_lduw_data(env, ptr + 2);
-        fptag = cpu_lduw_data(env, ptr + 4);
+        cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, retaddr));
+        fpus = cpu_lduw_data_ra(env, ptr + 2, retaddr);
+        fptag = cpu_lduw_data_ra(env, ptr + 4, retaddr);
     }
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800;
@@ -1028,17 +1050,22 @@
     }
 }
 
+void helper_fldenv(CPUX86State *env, target_ulong ptr, int data32)
+{
+    do_fldenv(env, ptr, data32, GETPC());
+}
+
 void helper_fsave(CPUX86State *env, target_ulong ptr, int data32)
 {
     floatx80 tmp;
     int i;
 
-    helper_fstenv(env, ptr, data32);
+    do_fstenv(env, ptr, data32, GETPC());
 
     ptr += (14 << data32);
     for (i = 0; i < 8; i++) {
         tmp = ST(i);
-        helper_fstt(env, tmp, ptr);
+        helper_fstt(env, tmp, ptr, GETPC());
         ptr += 10;
     }
 
@@ -1061,11 +1088,11 @@
     floatx80 tmp;
     int i;
 
-    helper_fldenv(env, ptr, data32);
+    do_fldenv(env, ptr, data32, GETPC());
     ptr += (14 << data32);
 
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, ptr);
+        tmp = helper_fldt(env, ptr, GETPC());
         ST(i) = tmp;
         ptr += 10;
     }
@@ -1083,82 +1110,183 @@
 }
 #endif
 
-void helper_fxsave(CPUX86State *env, target_ulong ptr, int data64)
+static void do_xsave_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
 {
-    int fpus, fptag, i, nb_xmm_regs;
-    floatx80 tmp;
+    int fpus, fptag, i;
     target_ulong addr;
 
-    /* The operand must be 16 byte aligned */
-    if (ptr & 0xf) {
-        raise_exception(env, EXCP0D_GPF);
-    }
-
     fpus = (env->fpus & ~0x3800) | (env->fpstt & 0x7) << 11;
     fptag = 0;
     for (i = 0; i < 8; i++) {
         fptag |= (env->fptags[i] << i);
     }
-    cpu_stw_data(env, ptr, env->fpuc);
-    cpu_stw_data(env, ptr + 2, fpus);
-    cpu_stw_data(env, ptr + 4, fptag ^ 0xff);
-#ifdef TARGET_X86_64
-    if (data64) {
-        cpu_stq_data(env, ptr + 0x08, 0); /* rip */
-        cpu_stq_data(env, ptr + 0x10, 0); /* rdp */
-    } else
-#endif
-    {
-        cpu_stl_data(env, ptr + 0x08, 0); /* eip */
-        cpu_stl_data(env, ptr + 0x0c, 0); /* sel  */
-        cpu_stl_data(env, ptr + 0x10, 0); /* dp */
-        cpu_stl_data(env, ptr + 0x14, 0); /* sel  */
-    }
+    cpu_stw_data_ra(env, ptr, env->fpuc, ra);
+    cpu_stw_data_ra(env, ptr + 2, fpus, ra);
+    cpu_stw_data_ra(env, ptr + 4, fptag ^ 0xff, ra);
+
+    /* In 32-bit mode this is eip, sel, dp, sel.
+       In 64-bit mode this is rip, rdp.
+       But in either case we don't write actual data, just zeros.  */
+    cpu_stq_data_ra(env, ptr + 0x08, 0, ra); /* eip+sel; rip */
+    cpu_stq_data_ra(env, ptr + 0x10, 0, ra); /* edp+sel; rdp */
 
     addr = ptr + 0x20;
     for (i = 0; i < 8; i++) {
-        tmp = ST(i);
-        helper_fstt(env, tmp, addr);
+        floatx80 tmp = ST(i);
+        helper_fstt(env, tmp, addr, ra);
         addr += 16;
     }
+}
+
+static void do_xsave_mxcsr(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    cpu_stl_data_ra(env, ptr + 0x18, env->mxcsr, ra); /* mxcsr */
+    cpu_stl_data_ra(env, ptr + 0x1c, 0x0000ffff, ra); /* mxcsr_mask */
+}
+
+static void do_xsave_sse(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, nb_xmm_regs;
+    target_ulong addr;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    addr = ptr + 0xa0;
+    for (i = 0; i < nb_xmm_regs; i++) {
+        cpu_stq_data_ra(env, addr, env->xmm_regs[i].ZMM_Q(0), ra);
+        cpu_stq_data_ra(env, addr + 8, env->xmm_regs[i].ZMM_Q(1), ra);
+        addr += 16;
+    }
+}
+
+static void do_xsave_bndregs(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    int i;
+
+    for (i = 0; i < 4; i++, addr += 16) {
+        cpu_stq_data_ra(env, addr, env->bnd_regs[i].lb, ra);
+        cpu_stq_data_ra(env, addr + 8, env->bnd_regs[i].ub, ra);
+    }
+}
+
+static void do_xsave_bndcsr(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    cpu_stq_data_ra(env, addr, env->bndcs_regs.cfgu, ra);
+    cpu_stq_data_ra(env, addr + 8, env->bndcs_regs.sts, ra);
+}
+
+static void do_xsave_pkru(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    cpu_stq_data_ra(env, addr, env->pkru, ra);
+}
+
+void helper_fxsave(CPUX86State *env, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+
+    /* The operand must be 16 byte aligned */
+    if (ptr & 0xf) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    do_xsave_fpu(env, ptr, ra);
 
     if (env->cr[4] & CR4_OSFXSR_MASK) {
-        /* XXX: finish it */
-        cpu_stl_data(env, ptr + 0x18, env->mxcsr); /* mxcsr */
-        cpu_stl_data(env, ptr + 0x1c, 0x0000ffff); /* mxcsr_mask */
-        if (env->hflags & HF_CS64_MASK) {
-            nb_xmm_regs = 16;
-        } else {
-            nb_xmm_regs = 8;
-        }
-        addr = ptr + 0xa0;
+        do_xsave_mxcsr(env, ptr, ra);
         /* Fast FXSAVE leaves out the XMM registers */
         if (!(env->efer & MSR_EFER_FFXSR)
             || (env->hflags & HF_CPL_MASK)
             || !(env->hflags & HF_LMA_MASK)) {
-            for (i = 0; i < nb_xmm_regs; i++) {
-                cpu_stq_data(env, addr, env->xmm_regs[i].XMM_Q(0));
-                cpu_stq_data(env, addr + 8, env->xmm_regs[i].XMM_Q(1));
-                addr += 16;
-            }
+            do_xsave_sse(env, ptr, ra);
         }
     }
 }
 
-void helper_fxrstor(CPUX86State *env, target_ulong ptr, int data64)
+static uint64_t get_xinuse(CPUX86State *env)
 {
-    int i, fpus, fptag, nb_xmm_regs;
-    floatx80 tmp;
-    target_ulong addr;
+    uint64_t inuse = -1;
 
-    /* The operand must be 16 byte aligned */
-    if (ptr & 0xf) {
-        raise_exception(env, EXCP0D_GPF);
+    /* For the most part, we don't track XINUSE.  We could calculate it
+       here for all components, but it's probably less work to simply
+       indicate in use.  That said, the state of BNDREGS is important
+       enough to track in HFLAGS, so we might as well use that here.  */
+    if ((env->hflags & HF_MPX_IU_MASK) == 0) {
+       inuse &= ~XSTATE_BNDREGS_MASK;
+    }
+    return inuse;
+}
+
+static void do_xsave(CPUX86State *env, target_ulong ptr, uint64_t rfbm,
+                     uint64_t inuse, uint64_t opt, uintptr_t ra)
+{
+    uint64_t old_bv, new_bv;
+
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, ra);
     }
 
-    cpu_set_fpuc(env, cpu_lduw_data(env, ptr));
-    fpus = cpu_lduw_data(env, ptr + 2);
-    fptag = cpu_lduw_data(env, ptr + 4);
+    /* The operand must be 64 byte aligned.  */
+    if (ptr & 63) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    /* Never save anything not enabled by XCR0.  */
+    rfbm &= env->xcr0;
+    opt &= rfbm;
+
+    if (opt & XSTATE_FP_MASK) {
+        do_xsave_fpu(env, ptr, ra);
+    }
+    if (rfbm & XSTATE_SSE_MASK) {
+        /* Note that saving MXCSR is not suppressed by XSAVEOPT.  */
+        do_xsave_mxcsr(env, ptr, ra);
+    }
+    if (opt & XSTATE_SSE_MASK) {
+        do_xsave_sse(env, ptr, ra);
+    }
+    if (opt & XSTATE_BNDREGS_MASK) {
+        target_ulong off = x86_ext_save_areas[XSTATE_BNDREGS_BIT].offset;
+        do_xsave_bndregs(env, ptr + off, ra);
+    }
+    if (opt & XSTATE_BNDCSR_MASK) {
+        target_ulong off = x86_ext_save_areas[XSTATE_BNDCSR_BIT].offset;
+        do_xsave_bndcsr(env, ptr + off, ra);
+    }
+    if (opt & XSTATE_PKRU_MASK) {
+        target_ulong off = x86_ext_save_areas[XSTATE_PKRU_BIT].offset;
+        do_xsave_pkru(env, ptr + off, ra);
+    }
+
+    /* Update the XSTATE_BV field.  */
+    old_bv = cpu_ldq_data_ra(env, ptr + 512, ra);
+    new_bv = (old_bv & ~rfbm) | (inuse & rfbm);
+    cpu_stq_data_ra(env, ptr + 512, new_bv, ra);
+}
+
+void helper_xsave(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    do_xsave(env, ptr, rfbm, get_xinuse(env), -1, GETPC());
+}
+
+void helper_xsaveopt(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    uint64_t inuse = get_xinuse(env);
+    do_xsave(env, ptr, rfbm, inuse, inuse, GETPC());
+}
+
+static void do_xrstor_fpu(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, fpus, fptag;
+    target_ulong addr;
+
+    cpu_set_fpuc(env, cpu_lduw_data_ra(env, ptr, ra));
+    fpus = cpu_lduw_data_ra(env, ptr + 2, ra);
+    fptag = cpu_lduw_data_ra(env, ptr + 4, ra);
     env->fpstt = (fpus >> 11) & 7;
     env->fpus = fpus & ~0x3800;
     fptag ^= 0xff;
@@ -1168,34 +1296,227 @@
 
     addr = ptr + 0x20;
     for (i = 0; i < 8; i++) {
-        tmp = helper_fldt(env, addr);
+        floatx80 tmp = helper_fldt(env, addr, ra);
         ST(i) = tmp;
         addr += 16;
     }
+}
+
+static void do_xrstor_mxcsr(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    cpu_set_mxcsr(env, cpu_ldl_data_ra(env, ptr + 0x18, ra));
+}
+
+static void do_xrstor_sse(CPUX86State *env, target_ulong ptr, uintptr_t ra)
+{
+    int i, nb_xmm_regs;
+    target_ulong addr;
+
+    if (env->hflags & HF_CS64_MASK) {
+        nb_xmm_regs = 16;
+    } else {
+        nb_xmm_regs = 8;
+    }
+
+    addr = ptr + 0xa0;
+    for (i = 0; i < nb_xmm_regs; i++) {
+        env->xmm_regs[i].ZMM_Q(0) = cpu_ldq_data_ra(env, addr, ra);
+        env->xmm_regs[i].ZMM_Q(1) = cpu_ldq_data_ra(env, addr + 8, ra);
+        addr += 16;
+    }
+}
+
+static void do_xrstor_bndregs(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    int i;
+
+    for (i = 0; i < 4; i++, addr += 16) {
+        env->bnd_regs[i].lb = cpu_ldq_data_ra(env, addr, ra);
+        env->bnd_regs[i].ub = cpu_ldq_data_ra(env, addr + 8, ra);
+    }
+}
+
+static void do_xrstor_bndcsr(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    /* FIXME: Extend highest implemented bit of linear address.  */
+    env->bndcs_regs.cfgu = cpu_ldq_data_ra(env, addr, ra);
+    env->bndcs_regs.sts = cpu_ldq_data_ra(env, addr + 8, ra);
+}
+
+static void do_xrstor_pkru(CPUX86State *env, target_ulong addr, uintptr_t ra)
+{
+    env->pkru = cpu_ldq_data_ra(env, addr, ra);
+}
+
+void helper_fxrstor(CPUX86State *env, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+
+    /* The operand must be 16 byte aligned */
+    if (ptr & 0xf) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    do_xrstor_fpu(env, ptr, ra);
 
     if (env->cr[4] & CR4_OSFXSR_MASK) {
-        /* XXX: finish it */
-        cpu_set_mxcsr(env, cpu_ldl_data(env, ptr + 0x18));
-        /* cpu_ldl_data(env, ptr + 0x1c); */
-        if (env->hflags & HF_CS64_MASK) {
-            nb_xmm_regs = 16;
-        } else {
-            nb_xmm_regs = 8;
-        }
-        addr = ptr + 0xa0;
-        /* Fast FXRESTORE leaves out the XMM registers */
+        do_xrstor_mxcsr(env, ptr, ra);
+        /* Fast FXRSTOR leaves out the XMM registers */
         if (!(env->efer & MSR_EFER_FFXSR)
             || (env->hflags & HF_CPL_MASK)
             || !(env->hflags & HF_LMA_MASK)) {
-            for (i = 0; i < nb_xmm_regs; i++) {
-                env->xmm_regs[i].XMM_Q(0) = cpu_ldq_data(env, addr);
-                env->xmm_regs[i].XMM_Q(1) = cpu_ldq_data(env, addr + 8);
-                addr += 16;
-            }
+            do_xrstor_sse(env, ptr, ra);
         }
     }
 }
 
+void helper_xrstor(CPUX86State *env, target_ulong ptr, uint64_t rfbm)
+{
+    uintptr_t ra = GETPC();
+    uint64_t xstate_bv, xcomp_bv0, xcomp_bv1;
+
+    rfbm &= env->xcr0;
+
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, ra);
+    }
+
+    /* The operand must be 64 byte aligned.  */
+    if (ptr & 63) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    xstate_bv = cpu_ldq_data_ra(env, ptr + 512, ra);
+
+    if ((int64_t)xstate_bv < 0) {
+        /* FIXME: Compact form.  */
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    /* Standard form.  */
+
+    /* The XSTATE field must not set bits not present in XCR0.  */
+    if (xstate_bv & ~env->xcr0) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    /* The XCOMP field must be zero.  */
+    xcomp_bv0 = cpu_ldq_data_ra(env, ptr + 520, ra);
+    xcomp_bv1 = cpu_ldq_data_ra(env, ptr + 528, ra);
+    if (xcomp_bv0 || xcomp_bv1) {
+        raise_exception_ra(env, EXCP0D_GPF, ra);
+    }
+
+    if (rfbm & XSTATE_FP_MASK) {
+        if (xstate_bv & XSTATE_FP_MASK) {
+            do_xrstor_fpu(env, ptr, ra);
+        } else {
+            helper_fninit(env);
+            memset(env->fpregs, 0, sizeof(env->fpregs));
+        }
+    }
+    if (rfbm & XSTATE_SSE_MASK) {
+        /* Note that the standard form of XRSTOR loads MXCSR from memory
+           whether or not the XSTATE_BV bit is set.  */
+        do_xrstor_mxcsr(env, ptr, ra);
+        if (xstate_bv & XSTATE_SSE_MASK) {
+            do_xrstor_sse(env, ptr, ra);
+        } else {
+            /* ??? When AVX is implemented, we may have to be more
+               selective in the clearing.  */
+            memset(env->xmm_regs, 0, sizeof(env->xmm_regs));
+        }
+    }
+    if (rfbm & XSTATE_BNDREGS_MASK) {
+        if (xstate_bv & XSTATE_BNDREGS_MASK) {
+            target_ulong off = x86_ext_save_areas[XSTATE_BNDREGS_BIT].offset;
+            do_xrstor_bndregs(env, ptr + off, ra);
+            env->hflags |= HF_MPX_IU_MASK;
+        } else {
+            memset(env->bnd_regs, 0, sizeof(env->bnd_regs));
+            env->hflags &= ~HF_MPX_IU_MASK;
+        }
+    }
+    if (rfbm & XSTATE_BNDCSR_MASK) {
+        if (xstate_bv & XSTATE_BNDCSR_MASK) {
+            target_ulong off = x86_ext_save_areas[XSTATE_BNDCSR_BIT].offset;
+            do_xrstor_bndcsr(env, ptr + off, ra);
+        } else {
+            memset(&env->bndcs_regs, 0, sizeof(env->bndcs_regs));
+        }
+        cpu_sync_bndcs_hflags(env);
+    }
+    if (rfbm & XSTATE_PKRU_MASK) {
+        uint64_t old_pkru = env->pkru;
+        if (xstate_bv & XSTATE_PKRU_MASK) {
+            target_ulong off = x86_ext_save_areas[XSTATE_PKRU_BIT].offset;
+            do_xrstor_pkru(env, ptr + off, ra);
+        } else {
+            env->pkru = 0;
+        }
+        if (env->pkru != old_pkru) {
+            CPUState *cs = CPU(x86_env_get_cpu(env));
+            tlb_flush(cs, 1);
+        }
+    }
+}
+
+uint64_t helper_xgetbv(CPUX86State *env, uint32_t ecx)
+{
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, GETPC());
+    }
+
+    switch (ecx) {
+    case 0:
+        return env->xcr0;
+    case 1:
+        if (env->features[FEAT_XSAVE] & CPUID_XSAVE_XGETBV1) {
+            return env->xcr0 & get_xinuse(env);
+        }
+        break;
+    }
+    raise_exception_ra(env, EXCP0D_GPF, GETPC());
+}
+
+void helper_xsetbv(CPUX86State *env, uint32_t ecx, uint64_t mask)
+{
+    uint32_t dummy, ena_lo, ena_hi;
+    uint64_t ena;
+
+    /* The OS must have enabled XSAVE.  */
+    if (!(env->cr[4] & CR4_OSXSAVE_MASK)) {
+        raise_exception_ra(env, EXCP06_ILLOP, GETPC());
+    }
+
+    /* Only XCR0 is defined at present; the FPU may not be disabled.  */
+    if (ecx != 0 || (mask & XSTATE_FP_MASK) == 0) {
+        goto do_gpf;
+    }
+
+    /* Disallow enabling unimplemented features.  */
+    cpu_x86_cpuid(env, 0x0d, 0, &ena_lo, &dummy, &dummy, &ena_hi);
+    ena = ((uint64_t)ena_hi << 32) | ena_lo;
+    if (mask & ~ena) {
+        goto do_gpf;
+    }
+
+    /* Disallow enabling only half of MPX.  */
+    if ((mask ^ (mask * (XSTATE_BNDCSR_MASK / XSTATE_BNDREGS_MASK)))
+        & XSTATE_BNDCSR_MASK) {
+        goto do_gpf;
+    }
+
+    env->xcr0 = mask;
+    cpu_sync_bndcs_hflags(env);
+    return;
+
+ do_gpf:
+    raise_exception_ra(env, EXCP0D_GPF, GETPC());
+}
+
 void cpu_get_fp80(uint64_t *pmant, uint16_t *pexp, floatx80 f)
 {
     CPU_LDoubleU temp;
diff --git a/target-i386/gdbstub.c b/target-i386/gdbstub.c
index ff99cfb..c494535 100644
--- a/target-i386/gdbstub.c
+++ b/target-i386/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 #ifdef TARGET_X86_64
@@ -61,8 +62,8 @@
         n -= IDX_XMM_REGS;
         if (n < CPU_NB_REGS32 ||
             (TARGET_LONG_BITS == 64 && env->hflags & HF_CS64_MASK)) {
-            stq_p(mem_buf, env->xmm_regs[n].XMM_Q(0));
-            stq_p(mem_buf + 8, env->xmm_regs[n].XMM_Q(1));
+            stq_p(mem_buf, env->xmm_regs[n].ZMM_Q(0));
+            stq_p(mem_buf + 8, env->xmm_regs[n].ZMM_Q(1));
             return 16;
         }
     } else {
@@ -170,8 +171,8 @@
         n -= IDX_XMM_REGS;
         if (n < CPU_NB_REGS32 ||
             (TARGET_LONG_BITS == 64 && env->hflags & HF_CS64_MASK)) {
-            env->xmm_regs[n].XMM_Q(0) = ldq_p(mem_buf);
-            env->xmm_regs[n].XMM_Q(1) = ldq_p(mem_buf + 8);
+            env->xmm_regs[n].ZMM_Q(0) = ldq_p(mem_buf);
+            env->xmm_regs[n].ZMM_Q(1) = ldq_p(mem_buf + 8);
             return 16;
         }
     } else {
diff --git a/target-i386/hax-all.c b/target-i386/hax-all.c
index 83b0943..197e817 100644
--- a/target-i386/hax-all.c
+++ b/target-i386/hax-all.c
@@ -23,25 +23,26 @@
  * HAX common code for both windows and darwin
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
-#include "strings.h"
+
 #include "hax-i386.h"
-#include "sysemu/accel.h"
-#include "exec/address-spaces.h"
-#include "qemu/main-loop.h"
 #include "hax-slot.h"
 
+#include "exec/address-spaces.h"
+#include "exec/exec-all.h"
+#include "exec/ioport.h"
+#include "qemu/main-loop.h"
+#include "strings.h"
+#include "sysemu/accel.h"
+
 #ifdef _WIN32
 #include "sysemu/os-win32.h"
 #endif
 
-#ifdef CONFIG_ANDROID
-#include "android/error-messages.h"
-#include "android/utils/debug.h"
-#else
 static const char kHaxVcpuSyncFailed[] = "Failed to sync HAX vcpu context";
+
 #define derror(msg) do { fprintf(stderr, (msg)); } while (0)
-#endif
 
 /* #define DEBUG_HAX */
 
@@ -116,7 +117,7 @@
 {
     /* Flush all emulation states */
     tlb_flush(ENV_GET_CPU(env), 1);
-    tb_flush(env);
+    tb_flush(ENV_GET_CPU(env));
     /* Sync the vcpu state from hax kernel module */
     hax_vcpu_sync_state(env, 0);
     return 0;
@@ -480,12 +481,14 @@
 }
 
 static void hax_log_start(MemoryListener * listener,
-                          MemoryRegionSection * section)
+                          MemoryRegionSection * section,
+                          int old, int new)
 {
 }
 
 static void hax_log_stop(MemoryListener * listener,
-                         MemoryRegionSection * section)
+                         MemoryRegionSection * section,
+                         int old, int new)
 {
 }
 
@@ -1315,8 +1318,12 @@
         env->fptags[i] = !((fpu.ftw >> i) & 1);
     memcpy(env->fpregs, fpu.st_mm, sizeof(env->fpregs));
 
-    memcpy(env->xmm_regs, fpu.mmx_1, sizeof(fpu.mmx_1));
-    memcpy((XMMReg *) (env->xmm_regs) + 8, fpu.mmx_2, sizeof(fpu.mmx_2));
+    for (i = 0; i < 8; ++i) {
+        memcpy(&env->xmm_regs[i], fpu.mmx_1[i], sizeof(fpu.mmx_1[i]));
+    }
+    for (i = 0; i < 8; ++i) {
+        memcpy(&env->xmm_regs[8 + i], fpu.mmx_2[i], sizeof(fpu.mmx_2[i]));
+    }
     env->mxcsr = fpu.mxcsr;
 
     return 0;
@@ -1336,8 +1343,13 @@
         fpu.ftw |= (!env->fptags[i]) << i;
 
     memcpy(fpu.st_mm, env->fpregs, sizeof(env->fpregs));
-    memcpy(fpu.mmx_1, env->xmm_regs, sizeof(fpu.mmx_1));
-    memcpy(fpu.mmx_2, (XMMReg *) (env->xmm_regs) + 8, sizeof(fpu.mmx_2));
+    
+    for (i = 0; i < 8; i++) {
+        memcpy(fpu.mmx_1[i], &env->xmm_regs[i], sizeof(fpu.mmx_1[i]));
+    }
+    for (i = 0; i < 8; i++) {
+        memcpy(fpu.mmx_2[i], &env->xmm_regs[i + 8], sizeof(fpu.mmx_2[i]));
+    }
 
     fpu.mxcsr = env->mxcsr;
 
diff --git a/target-i386/hax-i386.h b/target-i386/hax-i386.h
index 351e3db..3dc1981 100644
--- a/target-i386/hax-i386.h
+++ b/target-i386/hax-i386.h
@@ -14,6 +14,8 @@
 #define _HAX_I386_H
 
 #include "sysemu/hax.h"
+#include "target-i386/hax-interface.h"
+#include "cpu.h"
 
 #ifdef CONFIG_DARWIN
 typedef int hax_fd;
@@ -88,6 +90,4 @@
 #include "target-i386/hax-windows.h"
 #endif
 
-#include "target-i386/hax-interface.h"
-
 #endif
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 345bda1..1c250b8 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -17,16 +17,17 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "sysemu/kvm.h"
 #include "kvm_i386.h"
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/sysemu.h"
 #include "monitor/monitor.h"
+#include "hw/i386/apic_internal.h"
 #endif
 
-//#define DEBUG_MMU
-
 static void cpu_x86_version(CPUX86State *env, int *family, int *model)
 {
     int cpuver = env->cpuid_version;
@@ -179,6 +180,196 @@
     cpu_fprintf(f, "\n");
 }
 
+#ifndef CONFIG_USER_ONLY
+
+/* ARRAY_SIZE check is not required because
+ * DeliveryMode(dm) has a size of 3 bit.
+ */
+static inline const char *dm2str(uint32_t dm)
+{
+    static const char *str[] = {
+        "Fixed",
+        "...",
+        "SMI",
+        "...",
+        "NMI",
+        "INIT",
+        "...",
+        "ExtINT"
+    };
+    return str[dm];
+}
+
+static void dump_apic_lvt(FILE *f, fprintf_function cpu_fprintf,
+                          const char *name, uint32_t lvt, bool is_timer)
+{
+    uint32_t dm = (lvt & APIC_LVT_DELIV_MOD) >> APIC_LVT_DELIV_MOD_SHIFT;
+    cpu_fprintf(f,
+                "%s\t 0x%08x %s %-5s %-6s %-7s %-12s %-6s",
+                name, lvt,
+                lvt & APIC_LVT_INT_POLARITY ? "active-lo" : "active-hi",
+                lvt & APIC_LVT_LEVEL_TRIGGER ? "level" : "edge",
+                lvt & APIC_LVT_MASKED ? "masked" : "",
+                lvt & APIC_LVT_DELIV_STS ? "pending" : "",
+                !is_timer ?
+                    "" : lvt & APIC_LVT_TIMER_PERIODIC ?
+                            "periodic" : lvt & APIC_LVT_TIMER_TSCDEADLINE ?
+                                            "tsc-deadline" : "one-shot",
+                dm2str(dm));
+    if (dm != APIC_DM_NMI) {
+        cpu_fprintf(f, " (vec %u)\n", lvt & APIC_VECTOR_MASK);
+    } else {
+        cpu_fprintf(f, "\n");
+    }
+}
+
+/* ARRAY_SIZE check is not required because
+ * destination shorthand has a size of 2 bit.
+ */
+static inline const char *shorthand2str(uint32_t shorthand)
+{
+    const char *str[] = {
+        "no-shorthand", "self", "all-self", "all"
+    };
+    return str[shorthand];
+}
+
+static inline uint8_t divider_conf(uint32_t divide_conf)
+{
+    uint8_t divide_val = ((divide_conf & 0x8) >> 1) | (divide_conf & 0x3);
+
+    return divide_val == 7 ? 1 : 2 << divide_val;
+}
+
+static inline void mask2str(char *str, uint32_t val, uint8_t size)
+{
+    while (size--) {
+        *str++ = (val >> size) & 1 ? '1' : '0';
+    }
+    *str = 0;
+}
+
+#define MAX_LOGICAL_APIC_ID_MASK_SIZE 16
+
+static void dump_apic_icr(FILE *f, fprintf_function cpu_fprintf,
+                          APICCommonState *s, CPUX86State *env)
+{
+    uint32_t icr = s->icr[0], icr2 = s->icr[1];
+    uint8_t dest_shorthand = \
+        (icr & APIC_ICR_DEST_SHORT) >> APIC_ICR_DEST_SHORT_SHIFT;
+    bool logical_mod = icr & APIC_ICR_DEST_MOD;
+    char apic_id_str[MAX_LOGICAL_APIC_ID_MASK_SIZE + 1];
+    uint32_t dest_field;
+    bool x2apic;
+
+    cpu_fprintf(f, "ICR\t 0x%08x %s %s %s %s\n",
+                icr,
+                logical_mod ? "logical" : "physical",
+                icr & APIC_ICR_TRIGGER_MOD ? "level" : "edge",
+                icr & APIC_ICR_LEVEL ? "assert" : "de-assert",
+                shorthand2str(dest_shorthand));
+
+    cpu_fprintf(f, "ICR2\t 0x%08x", icr2);
+    if (dest_shorthand != 0) {
+        cpu_fprintf(f, "\n");
+        return;
+    }
+    x2apic = env->features[FEAT_1_ECX] & CPUID_EXT_X2APIC;
+    dest_field = x2apic ? icr2 : icr2 >> APIC_ICR_DEST_SHIFT;
+
+    if (!logical_mod) {
+        if (x2apic) {
+            cpu_fprintf(f, " cpu %u (X2APIC ID)\n", dest_field);
+        } else {
+            cpu_fprintf(f, " cpu %u (APIC ID)\n",
+                        dest_field & APIC_LOGDEST_XAPIC_ID);
+        }
+        return;
+    }
+
+    if (s->dest_mode == 0xf) { /* flat mode */
+        mask2str(apic_id_str, icr2 >> APIC_ICR_DEST_SHIFT, 8);
+        cpu_fprintf(f, " mask %s (APIC ID)\n", apic_id_str);
+    } else if (s->dest_mode == 0) { /* cluster mode */
+        if (x2apic) {
+            mask2str(apic_id_str, dest_field & APIC_LOGDEST_X2APIC_ID, 16);
+            cpu_fprintf(f, " cluster %u mask %s (X2APIC ID)\n",
+                        dest_field >> APIC_LOGDEST_X2APIC_SHIFT, apic_id_str);
+        } else {
+            mask2str(apic_id_str, dest_field & APIC_LOGDEST_XAPIC_ID, 4);
+            cpu_fprintf(f, " cluster %u mask %s (APIC ID)\n",
+                        dest_field >> APIC_LOGDEST_XAPIC_SHIFT, apic_id_str);
+        }
+    }
+}
+
+static void dump_apic_interrupt(FILE *f, fprintf_function cpu_fprintf,
+                                const char *name, uint32_t *ireg_tab,
+                                uint32_t *tmr_tab)
+{
+    int i, empty = true;
+
+    cpu_fprintf(f, "%s\t ", name);
+    for (i = 0; i < 256; i++) {
+        if (apic_get_bit(ireg_tab, i)) {
+            cpu_fprintf(f, "%u%s ", i,
+                        apic_get_bit(tmr_tab, i) ? "(level)" : "");
+            empty = false;
+        }
+    }
+    cpu_fprintf(f, "%s\n", empty ? "(none)" : "");
+}
+
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    APICCommonState *s = APIC_COMMON(cpu->apic_state);
+    uint32_t *lvt = s->lvt;
+
+    cpu_fprintf(f, "dumping local APIC state for CPU %-2u\n\n",
+                CPU(cpu)->cpu_index);
+    dump_apic_lvt(f, cpu_fprintf, "LVT0", lvt[APIC_LVT_LINT0], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVT1", lvt[APIC_LVT_LINT1], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTPC", lvt[APIC_LVT_PERFORM], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTERR", lvt[APIC_LVT_ERROR], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTTHMR", lvt[APIC_LVT_THERMAL], false);
+    dump_apic_lvt(f, cpu_fprintf, "LVTT", lvt[APIC_LVT_TIMER], true);
+
+    cpu_fprintf(f, "Timer\t DCR=0x%x (divide by %u) initial_count = %u\n",
+                s->divide_conf & APIC_DCR_MASK,
+                divider_conf(s->divide_conf),
+                s->initial_count);
+
+    cpu_fprintf(f, "SPIV\t 0x%08x APIC %s, focus=%s, spurious vec %u\n",
+                s->spurious_vec,
+                s->spurious_vec & APIC_SPURIO_ENABLED ? "enabled" : "disabled",
+                s->spurious_vec & APIC_SPURIO_FOCUS ? "on" : "off",
+                s->spurious_vec & APIC_VECTOR_MASK);
+
+    dump_apic_icr(f, cpu_fprintf, s, &cpu->env);
+
+    cpu_fprintf(f, "ESR\t 0x%08x\n", s->esr);
+
+    dump_apic_interrupt(f, cpu_fprintf, "ISR", s->isr, s->tmr);
+    dump_apic_interrupt(f, cpu_fprintf, "IRR", s->irr, s->tmr);
+
+    cpu_fprintf(f, "\nAPR 0x%02x TPR 0x%02x DFR 0x%02x LDR 0x%02x",
+                s->arb_id, s->tpr, s->dest_mode, s->log_dest);
+    if (s->dest_mode == 0) {
+        cpu_fprintf(f, "(cluster %u: id %u)",
+                    s->log_dest >> APIC_LOGDEST_XAPIC_SHIFT,
+                    s->log_dest & APIC_LOGDEST_XAPIC_ID);
+    }
+    cpu_fprintf(f, " PPR 0x%02x\n", apic_get_ppr(s));
+}
+#else
+void x86_cpu_dump_local_apic_state(CPUState *cs, FILE *f,
+                                   fprintf_function cpu_fprintf, int flags)
+{
+}
+#endif /* !CONFIG_USER_ONLY */
+
 #define DUMP_CODE_BYTES_TOTAL    50
 #define DUMP_CODE_BYTES_BACKWARD 20
 
@@ -346,10 +537,10 @@
         for(i=0;i<nb;i++) {
             cpu_fprintf(f, "XMM%02d=%08x%08x%08x%08x",
                         i,
-                        env->xmm_regs[i].XMM_L(3),
-                        env->xmm_regs[i].XMM_L(2),
-                        env->xmm_regs[i].XMM_L(1),
-                        env->xmm_regs[i].XMM_L(0));
+                        env->xmm_regs[i].ZMM_L(3),
+                        env->xmm_regs[i].ZMM_L(2),
+                        env->xmm_regs[i].ZMM_L(1),
+                        env->xmm_regs[i].ZMM_L(0));
             if ((i & 1) == 1)
                 cpu_fprintf(f, "\n");
             else
@@ -388,9 +579,7 @@
     if (a20_state != ((env->a20_mask >> 20) & 1)) {
         CPUState *cs = CPU(cpu);
 
-#if defined(DEBUG_MMU)
-        printf("A20 update: a20=%d\n", a20_state);
-#endif
+        qemu_log_mask(CPU_LOG_MMU, "A20 update: a20=%d\n", a20_state);
         /* if the cpu is currently executing code, we must unlink it and
            all the potentially executing TB */
         cpu_interrupt(cs, CPU_INTERRUPT_EXITTB);
@@ -407,9 +596,7 @@
     X86CPU *cpu = x86_env_get_cpu(env);
     int pe_state;
 
-#if defined(DEBUG_MMU)
-    printf("CR0 update: CR0=0x%08x\n", new_cr0);
-#endif
+    qemu_log_mask(CPU_LOG_MMU, "CR0 update: CR0=0x%08x\n", new_cr0);
     if ((new_cr0 & (CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK)) !=
         (env->cr[0] & (CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK))) {
         tlb_flush(CPU(cpu), 1);
@@ -452,9 +639,8 @@
 
     env->cr[3] = new_cr3;
     if (env->cr[0] & CR0_PG_MASK) {
-#if defined(DEBUG_MMU)
-        printf("CR3 update: CR3=" TARGET_FMT_lx "\n", new_cr3);
-#endif
+        qemu_log_mask(CPU_LOG_MMU,
+                        "CR3 update: CR3=" TARGET_FMT_lx "\n", new_cr3);
         tlb_flush(CPU(cpu), 0);
     }
 }
@@ -462,6 +648,7 @@
 void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
+    uint32_t hflags;
 
 #if defined(DEBUG_MMU)
     printf("CR4 update: CR4=%08x\n", (uint32_t)env->cr[4]);
@@ -471,24 +658,33 @@
          CR4_SMEP_MASK | CR4_SMAP_MASK)) {
         tlb_flush(CPU(cpu), 1);
     }
+
+    /* Clear bits we're going to recompute.  */
+    hflags = env->hflags & ~(HF_OSFXSR_MASK | HF_SMAP_MASK);
+
     /* SSE handling */
     if (!(env->features[FEAT_1_EDX] & CPUID_SSE)) {
         new_cr4 &= ~CR4_OSFXSR_MASK;
     }
-    env->hflags &= ~HF_OSFXSR_MASK;
     if (new_cr4 & CR4_OSFXSR_MASK) {
-        env->hflags |= HF_OSFXSR_MASK;
+        hflags |= HF_OSFXSR_MASK;
     }
 
     if (!(env->features[FEAT_7_0_EBX] & CPUID_7_0_EBX_SMAP)) {
         new_cr4 &= ~CR4_SMAP_MASK;
     }
-    env->hflags &= ~HF_SMAP_MASK;
     if (new_cr4 & CR4_SMAP_MASK) {
-        env->hflags |= HF_SMAP_MASK;
+        hflags |= HF_SMAP_MASK;
+    }
+
+    if (!(env->features[FEAT_7_0_ECX] & CPUID_7_0_ECX_PKU)) {
+        new_cr4 &= ~CR4_PKE_MASK;
     }
 
     env->cr[4] = new_cr4;
+    env->hflags = hflags;
+
+    cpu_sync_bndcs_hflags(env);
 }
 
 #if defined(CONFIG_USER_ONLY)
@@ -505,6 +701,8 @@
     env->error_code = (is_write << PG_ERROR_W_BIT);
     env->error_code |= PG_ERROR_U_MASK;
     cs->exception_index = EXCP0E_PAGE;
+    env->exception_is_int = 0;
+    env->exception_next_eip = -1;
     return 1;
 }
 
@@ -572,7 +770,7 @@
 
             pml4e_addr = ((env->cr[3] & ~0xfff) + (((addr >> 39) & 0x1ff) << 3)) &
                 env->a20_mask;
-            pml4e = ldq_phys(cs->as, pml4e_addr);
+            pml4e = x86_ldq_phys(cs, pml4e_addr);
             if (!(pml4e & PG_PRESENT_MASK)) {
                 goto do_fault;
             }
@@ -581,12 +779,12 @@
             }
             if (!(pml4e & PG_ACCESSED_MASK)) {
                 pml4e |= PG_ACCESSED_MASK;
-                stl_phys_notdirty(cs->as, pml4e_addr, pml4e);
+                x86_stl_phys_notdirty(cs, pml4e_addr, pml4e);
             }
             ptep = pml4e ^ PG_NX_MASK;
             pdpe_addr = ((pml4e & PG_ADDRESS_MASK) + (((addr >> 30) & 0x1ff) << 3)) &
                 env->a20_mask;
-            pdpe = ldq_phys(cs->as, pdpe_addr);
+            pdpe = x86_ldq_phys(cs, pdpe_addr);
             if (!(pdpe & PG_PRESENT_MASK)) {
                 goto do_fault;
             }
@@ -596,7 +794,7 @@
             ptep &= pdpe ^ PG_NX_MASK;
             if (!(pdpe & PG_ACCESSED_MASK)) {
                 pdpe |= PG_ACCESSED_MASK;
-                stl_phys_notdirty(cs->as, pdpe_addr, pdpe);
+                x86_stl_phys_notdirty(cs, pdpe_addr, pdpe);
             }
             if (pdpe & PG_PSE_MASK) {
                 /* 1 GB page */
@@ -611,7 +809,7 @@
             /* XXX: load them when cr3 is loaded ? */
             pdpe_addr = ((env->cr[3] & ~0x1f) + ((addr >> 27) & 0x18)) &
                 env->a20_mask;
-            pdpe = ldq_phys(cs->as, pdpe_addr);
+            pdpe = x86_ldq_phys(cs, pdpe_addr);
             if (!(pdpe & PG_PRESENT_MASK)) {
                 goto do_fault;
             }
@@ -624,7 +822,7 @@
 
         pde_addr = ((pdpe & PG_ADDRESS_MASK) + (((addr >> 21) & 0x1ff) << 3)) &
             env->a20_mask;
-        pde = ldq_phys(cs->as, pde_addr);
+        pde = x86_ldq_phys(cs, pde_addr);
         if (!(pde & PG_PRESENT_MASK)) {
             goto do_fault;
         }
@@ -642,11 +840,11 @@
         /* 4 KB page */
         if (!(pde & PG_ACCESSED_MASK)) {
             pde |= PG_ACCESSED_MASK;
-            stl_phys_notdirty(cs->as, pde_addr, pde);
+            x86_stl_phys_notdirty(cs, pde_addr, pde);
         }
         pte_addr = ((pde & PG_ADDRESS_MASK) + (((addr >> 12) & 0x1ff) << 3)) &
             env->a20_mask;
-        pte = ldq_phys(cs->as, pte_addr);
+        pte = x86_ldq_phys(cs, pte_addr);
         if (!(pte & PG_PRESENT_MASK)) {
             goto do_fault;
         }
@@ -662,7 +860,7 @@
         /* page directory entry */
         pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & 0xffc)) &
             env->a20_mask;
-        pde = ldl_phys(cs->as, pde_addr);
+        pde = x86_ldl_phys(cs, pde_addr);
         if (!(pde & PG_PRESENT_MASK)) {
             goto do_fault;
         }
@@ -676,20 +874,20 @@
             /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
              * Leave bits 20-13 in place for setting accessed/dirty bits below.
              */
-            pte = pde | ((pde & 0x1fe000) << (32 - 13));
+            pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
             rsvd_mask = 0x200000;
             goto do_check_protect_pse36;
         }
 
         if (!(pde & PG_ACCESSED_MASK)) {
             pde |= PG_ACCESSED_MASK;
-            stl_phys_notdirty(cs->as, pde_addr, pde);
+            x86_stl_phys_notdirty(cs, pde_addr, pde);
         }
 
         /* page directory entry */
         pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) &
             env->a20_mask;
-        pte = ldl_phys(cs->as, pte_addr);
+        pte = x86_ldl_phys(cs, pte_addr);
         if (!(pte & PG_PRESENT_MASK)) {
             goto do_fault;
         }
@@ -706,66 +904,66 @@
         goto do_fault_rsvd;
     }
     ptep ^= PG_NX_MASK;
-    if ((ptep & PG_NX_MASK) && is_write1 == 2) {
+
+    /* can the page can be put in the TLB?  prot will tell us */
+    if (is_user && !(ptep & PG_USER_MASK)) {
         goto do_fault_protect;
     }
-    switch (mmu_idx) {
-    case MMU_USER_IDX:
-        if (!(ptep & PG_USER_MASK)) {
-            goto do_fault_protect;
-        }
-        if (is_write && !(ptep & PG_RW_MASK)) {
-            goto do_fault_protect;
-        }
-        break;
 
-    case MMU_KSMAP_IDX:
-        if (is_write1 != 2 && (ptep & PG_USER_MASK)) {
-            goto do_fault_protect;
+    prot = 0;
+    if (mmu_idx != MMU_KSMAP_IDX || !(ptep & PG_USER_MASK)) {
+        prot |= PAGE_READ;
+        if ((ptep & PG_RW_MASK) || (!is_user && !(env->cr[0] & CR0_WP_MASK))) {
+            prot |= PAGE_WRITE;
         }
-        /* fall through */
-    case MMU_KNOSMAP_IDX:
-        if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
-            (ptep & PG_USER_MASK)) {
-            goto do_fault_protect;
-        }
-        if ((env->cr[0] & CR0_WP_MASK) &&
-            is_write && !(ptep & PG_RW_MASK)) {
-            goto do_fault_protect;
-        }
-        break;
-
-    default: /* cannot happen */
-        break;
     }
+    if (!(ptep & PG_NX_MASK) &&
+        (mmu_idx == MMU_USER_IDX ||
+         !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK)))) {
+        prot |= PAGE_EXEC;
+    }
+    if ((env->cr[4] & CR4_PKE_MASK) && (env->hflags & HF_LMA_MASK) &&
+        (ptep & PG_USER_MASK) && env->pkru) {
+        uint32_t pk = (pte & PG_PKRU_MASK) >> PG_PKRU_BIT;
+        uint32_t pkru_ad = (env->pkru >> pk * 2) & 1;
+        uint32_t pkru_wd = (env->pkru >> pk * 2) & 2;
+        uint32_t pkru_prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+
+        if (pkru_ad) {
+            pkru_prot &= ~(PAGE_READ | PAGE_WRITE);
+        } else if (pkru_wd && (is_user || env->cr[0] & CR0_WP_MASK)) {
+            pkru_prot &= ~PAGE_WRITE;
+        }
+
+        prot &= pkru_prot;
+        if ((pkru_prot & (1 << is_write1)) == 0) {
+            assert(is_write1 != 2);
+            error_code |= PG_ERROR_PK_MASK;
+            goto do_fault_protect;
+        }
+    }
+
+    if ((prot & (1 << is_write1)) == 0) {
+        goto do_fault_protect;
+    }
+
+    /* yes, it can! */
     is_dirty = is_write && !(pte & PG_DIRTY_MASK);
     if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
         pte |= PG_ACCESSED_MASK;
         if (is_dirty) {
             pte |= PG_DIRTY_MASK;
         }
-        stl_phys_notdirty(cs->as, pte_addr, pte);
+        x86_stl_phys_notdirty(cs, pte_addr, pte);
     }
 
-    /* the page can be put in the TLB */
-    prot = PAGE_READ;
-    if (!(ptep & PG_NX_MASK) &&
-        (mmu_idx == MMU_USER_IDX ||
-         !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK)))) {
-        prot |= PAGE_EXEC;
-    }
-    if (pte & PG_DIRTY_MASK) {
+    if (!(pte & PG_DIRTY_MASK)) {
         /* only set write access if already dirty... otherwise wait
            for dirty access */
-        if (is_user) {
-            if (ptep & PG_RW_MASK)
-                prot |= PAGE_WRITE;
-        } else {
-            if (!(env->cr[0] & CR0_WP_MASK) ||
-                (ptep & PG_RW_MASK))
-                prot |= PAGE_WRITE;
-        }
+        assert(!is_write);
+        prot &= ~PAGE_WRITE;
     }
+
  do_mapping:
     pte = pte & env->a20_mask;
 
@@ -778,7 +976,9 @@
     page_offset = vaddr & (page_size - 1);
     paddr = pte + page_offset;
 
-    tlb_set_page(cs, vaddr, paddr, prot, mmu_idx, page_size);
+    assert(prot & (1 << is_write1));
+    tlb_set_page_with_attrs(cs, vaddr, paddr, cpu_get_mem_attrs(env),
+                            prot, mmu_idx, page_size);
     return 0;
  do_fault_rsvd:
     error_code |= PG_ERROR_RSVD_MASK;
@@ -795,7 +995,7 @@
         error_code |= PG_ERROR_I_D_MASK;
     if (env->intercept_exceptions & (1 << EXCP0E_PAGE)) {
         /* cr2 is not modified in case of exceptions */
-        stq_phys(cs->as,
+        x86_stq_phys(cs,
                  env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
                  addr);
     } else {
@@ -834,13 +1034,13 @@
             }
             pml4e_addr = ((env->cr[3] & ~0xfff) + (((addr >> 39) & 0x1ff) << 3)) &
                 env->a20_mask;
-            pml4e = ldq_phys(cs->as, pml4e_addr);
+            pml4e = x86_ldq_phys(cs, pml4e_addr);
             if (!(pml4e & PG_PRESENT_MASK)) {
                 return -1;
             }
             pdpe_addr = ((pml4e & PG_ADDRESS_MASK) +
                          (((addr >> 30) & 0x1ff) << 3)) & env->a20_mask;
-            pdpe = ldq_phys(cs->as, pdpe_addr);
+            pdpe = x86_ldq_phys(cs, pdpe_addr);
             if (!(pdpe & PG_PRESENT_MASK)) {
                 return -1;
             }
@@ -855,14 +1055,14 @@
         {
             pdpe_addr = ((env->cr[3] & ~0x1f) + ((addr >> 27) & 0x18)) &
                 env->a20_mask;
-            pdpe = ldq_phys(cs->as, pdpe_addr);
+            pdpe = x86_ldq_phys(cs, pdpe_addr);
             if (!(pdpe & PG_PRESENT_MASK))
                 return -1;
         }
 
         pde_addr = ((pdpe & PG_ADDRESS_MASK) +
                     (((addr >> 21) & 0x1ff) << 3)) & env->a20_mask;
-        pde = ldq_phys(cs->as, pde_addr);
+        pde = x86_ldq_phys(cs, pde_addr);
         if (!(pde & PG_PRESENT_MASK)) {
             return -1;
         }
@@ -875,7 +1075,7 @@
             pte_addr = ((pde & PG_ADDRESS_MASK) +
                         (((addr >> 12) & 0x1ff) << 3)) & env->a20_mask;
             page_size = 4096;
-            pte = ldq_phys(cs->as, pte_addr);
+            pte = x86_ldq_phys(cs, pte_addr);
         }
         if (!(pte & PG_PRESENT_MASK)) {
             return -1;
@@ -885,16 +1085,16 @@
 
         /* page directory entry */
         pde_addr = ((env->cr[3] & ~0xfff) + ((addr >> 20) & 0xffc)) & env->a20_mask;
-        pde = ldl_phys(cs->as, pde_addr);
+        pde = x86_ldl_phys(cs, pde_addr);
         if (!(pde & PG_PRESENT_MASK))
             return -1;
         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-            pte = pde | ((pde & 0x1fe000) << (32 - 13));
+            pte = pde | ((pde & 0x1fe000LL) << (32 - 13));
             page_size = 4096 * 1024;
         } else {
             /* page directory entry */
             pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask;
-            pte = ldl_phys(cs->as, pte_addr);
+            pte = x86_ldl_phys(cs, pte_addr);
             if (!(pte & PG_PRESENT_MASK)) {
                 return -1;
             }
@@ -911,134 +1111,6 @@
     return pte | page_offset;
 }
 
-void hw_breakpoint_insert(CPUX86State *env, int index)
-{
-    CPUState *cs = CPU(x86_env_get_cpu(env));
-    int type = 0, err = 0;
-
-    switch (hw_breakpoint_type(env->dr[7], index)) {
-    case DR7_TYPE_BP_INST:
-        if (hw_breakpoint_enabled(env->dr[7], index)) {
-            err = cpu_breakpoint_insert(cs, env->dr[index], BP_CPU,
-                                        &env->cpu_breakpoint[index]);
-        }
-        break;
-    case DR7_TYPE_DATA_WR:
-        type = BP_CPU | BP_MEM_WRITE;
-        break;
-    case DR7_TYPE_IO_RW:
-        /* No support for I/O watchpoints yet */
-        break;
-    case DR7_TYPE_DATA_RW:
-        type = BP_CPU | BP_MEM_ACCESS;
-        break;
-    }
-
-    if (type != 0) {
-        err = cpu_watchpoint_insert(cs, env->dr[index],
-                                    hw_breakpoint_len(env->dr[7], index),
-                                    type, &env->cpu_watchpoint[index]);
-    }
-
-    if (err) {
-        env->cpu_breakpoint[index] = NULL;
-    }
-}
-
-void hw_breakpoint_remove(CPUX86State *env, int index)
-{
-    CPUState *cs;
-
-    if (!env->cpu_breakpoint[index]) {
-        return;
-    }
-    cs = CPU(x86_env_get_cpu(env));
-    switch (hw_breakpoint_type(env->dr[7], index)) {
-    case DR7_TYPE_BP_INST:
-        if (hw_breakpoint_enabled(env->dr[7], index)) {
-            cpu_breakpoint_remove_by_ref(cs, env->cpu_breakpoint[index]);
-        }
-        break;
-    case DR7_TYPE_DATA_WR:
-    case DR7_TYPE_DATA_RW:
-        cpu_watchpoint_remove_by_ref(cs, env->cpu_watchpoint[index]);
-        break;
-    case DR7_TYPE_IO_RW:
-        /* No support for I/O watchpoints yet */
-        break;
-    }
-}
-
-bool check_hw_breakpoints(CPUX86State *env, bool force_dr6_update)
-{
-    target_ulong dr6;
-    int reg;
-    bool hit_enabled = false;
-
-    dr6 = env->dr[6] & ~0xf;
-    for (reg = 0; reg < DR7_MAX_BP; reg++) {
-        bool bp_match = false;
-        bool wp_match = false;
-
-        switch (hw_breakpoint_type(env->dr[7], reg)) {
-        case DR7_TYPE_BP_INST:
-            if (env->dr[reg] == env->eip) {
-                bp_match = true;
-            }
-            break;
-        case DR7_TYPE_DATA_WR:
-        case DR7_TYPE_DATA_RW:
-            if (env->cpu_watchpoint[reg] &&
-                env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT) {
-                wp_match = true;
-            }
-            break;
-        case DR7_TYPE_IO_RW:
-            break;
-        }
-        if (bp_match || wp_match) {
-            dr6 |= 1 << reg;
-            if (hw_breakpoint_enabled(env->dr[7], reg)) {
-                hit_enabled = true;
-            }
-        }
-    }
-
-    if (hit_enabled || force_dr6_update) {
-        env->dr[6] = dr6;
-    }
-
-    return hit_enabled;
-}
-
-void breakpoint_handler(CPUState *cs)
-{
-    X86CPU *cpu = X86_CPU(cs);
-    CPUX86State *env = &cpu->env;
-    CPUBreakpoint *bp;
-
-    if (cs->watchpoint_hit) {
-        if (cs->watchpoint_hit->flags & BP_CPU) {
-            cs->watchpoint_hit = NULL;
-            if (check_hw_breakpoints(env, false)) {
-                raise_exception(env, EXCP01_DB);
-            } else {
-                cpu_resume_from_signal(cs, NULL);
-            }
-        }
-    } else {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == env->eip) {
-                if (bp->flags & BP_CPU) {
-                    check_hw_breakpoints(env, true);
-                    raise_exception(env, EXCP01_DB);
-                }
-                break;
-            }
-        }
-    }
-}
-
 typedef struct MCEInjectionParams {
     Monitor *mon;
     X86CPU *cpu;
@@ -1283,3 +1355,95 @@
 
     env->eflags = cpu_compute_eflags(env);
 }
+
+#ifndef CONFIG_USER_ONLY
+uint8_t x86_ldub_phys(CPUState *cs, hwaddr addr)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    return address_space_ldub(cs->as, addr,
+                              cpu_get_mem_attrs(env),
+                              NULL);
+}
+
+uint32_t x86_lduw_phys(CPUState *cs, hwaddr addr)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    return address_space_lduw(cs->as, addr,
+                              cpu_get_mem_attrs(env),
+                              NULL);
+}
+
+uint32_t x86_ldl_phys(CPUState *cs, hwaddr addr)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    return address_space_ldl(cs->as, addr,
+                             cpu_get_mem_attrs(env),
+                             NULL);
+}
+
+uint64_t x86_ldq_phys(CPUState *cs, hwaddr addr)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    return address_space_ldq(cs->as, addr,
+                             cpu_get_mem_attrs(env),
+                             NULL);
+}
+
+void x86_stb_phys(CPUState *cs, hwaddr addr, uint8_t val)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    address_space_stb(cs->as, addr, val,
+                      cpu_get_mem_attrs(env),
+                      NULL);
+}
+
+void x86_stl_phys_notdirty(CPUState *cs, hwaddr addr, uint32_t val)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    address_space_stl_notdirty(cs->as, addr, val,
+                               cpu_get_mem_attrs(env),
+                               NULL);
+}
+
+void x86_stw_phys(CPUState *cs, hwaddr addr, uint32_t val)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    address_space_stw(cs->as, addr, val,
+                      cpu_get_mem_attrs(env),
+                      NULL);
+}
+
+void x86_stl_phys(CPUState *cs, hwaddr addr, uint32_t val)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    address_space_stl(cs->as, addr, val,
+                      cpu_get_mem_attrs(env),
+                      NULL);
+}
+
+void x86_stq_phys(CPUState *cs, hwaddr addr, uint64_t val)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    address_space_stq(cs->as, addr, val,
+                      cpu_get_mem_attrs(env),
+                      NULL);
+}
+#endif
diff --git a/target-i386/helper.h b/target-i386/helper.h
index 8eb0145..1320edc 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -15,6 +15,14 @@
 DEF_HELPER_2(divq_EAX, void, env, tl)
 DEF_HELPER_2(idivq_EAX, void, env, tl)
 #endif
+DEF_HELPER_FLAGS_2(cr4_testbit, TCG_CALL_NO_WG, void, env, i32)
+
+DEF_HELPER_FLAGS_2(bndck, TCG_CALL_NO_WG, void, env, i32)
+DEF_HELPER_FLAGS_3(bndldx32, TCG_CALL_NO_WG, i64, env, tl, tl)
+DEF_HELPER_FLAGS_3(bndldx64, TCG_CALL_NO_WG, i64, env, tl, tl)
+DEF_HELPER_FLAGS_5(bndstx32, TCG_CALL_NO_WG, void, env, tl, tl, i64, i64)
+DEF_HELPER_FLAGS_5(bndstx64, TCG_CALL_NO_WG, void, env, tl, tl, i64, i64)
+DEF_HELPER_1(bnd_jmp, void, env)
 
 DEF_HELPER_2(aam, void, env, int)
 DEF_HELPER_2(aad, void, env, int)
@@ -30,9 +38,9 @@
 DEF_HELPER_2(lldt, void, env, int)
 DEF_HELPER_2(ltr, void, env, int)
 DEF_HELPER_3(load_seg, void, env, int, int)
-DEF_HELPER_4(ljmp_protected, void, env, int, tl, int)
+DEF_HELPER_4(ljmp_protected, void, env, int, tl, tl)
 DEF_HELPER_5(lcall_real, void, env, int, tl, int, int)
-DEF_HELPER_5(lcall_protected, void, env, int, tl, int, int)
+DEF_HELPER_5(lcall_protected, void, env, int, tl, int, tl)
 DEF_HELPER_2(iret_real, void, env, int)
 DEF_HELPER_3(iret_protected, void, env, int, int)
 DEF_HELPER_3(lret_protected, void, env, int, int)
@@ -40,13 +48,10 @@
 DEF_HELPER_3(write_crN, void, env, int, tl)
 DEF_HELPER_2(lmsw, void, env, tl)
 DEF_HELPER_1(clts, void, env)
-DEF_HELPER_3(movl_drN_T0, void, env, int, tl)
+DEF_HELPER_FLAGS_3(set_dr, TCG_CALL_NO_WG, void, env, int, tl)
+DEF_HELPER_FLAGS_2(get_dr, TCG_CALL_NO_WG, tl, env, int)
 DEF_HELPER_2(invlpg, void, env, tl)
 
-DEF_HELPER_4(enter_level, void, env, int, int, tl)
-#ifdef TARGET_X86_64
-DEF_HELPER_4(enter64_level, void, env, int, int, tl)
-#endif
 DEF_HELPER_1(sysenter, void, env)
 DEF_HELPER_2(sysexit, void, env, int)
 #ifdef TARGET_X86_64
@@ -65,8 +70,6 @@
 DEF_HELPER_1(sti, void, env)
 DEF_HELPER_1(clac, void, env)
 DEF_HELPER_1(stac, void, env)
-DEF_HELPER_1(set_inhibit_irq, void, env)
-DEF_HELPER_1(reset_inhibit_irq, void, env)
 DEF_HELPER_3(boundw, void, env, tl, int)
 DEF_HELPER_3(boundl, void, env, tl, int)
 DEF_HELPER_1(rsm, void, env)
@@ -86,12 +89,13 @@
 DEF_HELPER_2(check_iob, void, env, i32)
 DEF_HELPER_2(check_iow, void, env, i32)
 DEF_HELPER_2(check_iol, void, env, i32)
-DEF_HELPER_2(outb, void, i32, i32)
-DEF_HELPER_1(inb, tl, i32)
-DEF_HELPER_2(outw, void, i32, i32)
-DEF_HELPER_1(inw, tl, i32)
-DEF_HELPER_2(outl, void, i32, i32)
-DEF_HELPER_1(inl, tl, i32)
+DEF_HELPER_3(outb, void, env, i32, i32)
+DEF_HELPER_2(inb, tl, env, i32)
+DEF_HELPER_3(outw, void, env, i32, i32)
+DEF_HELPER_2(inw, tl, env, i32)
+DEF_HELPER_3(outl, void, env, i32, i32)
+DEF_HELPER_2(inl, tl, env, i32)
+DEF_HELPER_FLAGS_4(bpt_io, TCG_CALL_NO_WG, void, env, i32, i32, tl)
 
 DEF_HELPER_3(svm_check_intercept_param, void, env, i32, i64)
 DEF_HELPER_3(vmexit, void, env, i32, i64)
@@ -187,8 +191,15 @@
 DEF_HELPER_3(fldenv, void, env, tl, int)
 DEF_HELPER_3(fsave, void, env, tl, int)
 DEF_HELPER_3(frstor, void, env, tl, int)
-DEF_HELPER_3(fxsave, void, env, tl, int)
-DEF_HELPER_3(fxrstor, void, env, tl, int)
+DEF_HELPER_FLAGS_2(fxsave, TCG_CALL_NO_WG, void, env, tl)
+DEF_HELPER_FLAGS_2(fxrstor, TCG_CALL_NO_WG, void, env, tl)
+DEF_HELPER_FLAGS_3(xsave, TCG_CALL_NO_WG, void, env, tl, i64)
+DEF_HELPER_FLAGS_3(xsaveopt, TCG_CALL_NO_WG, void, env, tl, i64)
+DEF_HELPER_FLAGS_3(xrstor, TCG_CALL_NO_WG, void, env, tl, i64)
+DEF_HELPER_FLAGS_2(xgetbv, TCG_CALL_NO_WG, i64, env, i32)
+DEF_HELPER_FLAGS_3(xsetbv, TCG_CALL_NO_WG, void, env, i32, i64)
+DEF_HELPER_FLAGS_2(rdpkru, TCG_CALL_NO_WG, i64, env, i32)
+DEF_HELPER_FLAGS_3(wrpkru, TCG_CALL_NO_WG, void, env, i32, i64)
 
 DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_FLAGS_1(ctz, TCG_CALL_NO_RWG_SE, tl, tl)
diff --git a/target-i386/hyperv.c b/target-i386/hyperv.c
new file mode 100644
index 0000000..39a230f
--- /dev/null
+++ b/target-i386/hyperv.c
@@ -0,0 +1,140 @@
+/*
+ * QEMU KVM Hyper-V support
+ *
+ * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * Authors:
+ *  Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "hyperv.h"
+#include "standard-headers/asm-x86/hyperv.h"
+
+int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit)
+{
+    CPUX86State *env = &cpu->env;
+
+    switch (exit->type) {
+    case KVM_EXIT_HYPERV_SYNIC:
+        if (!cpu->hyperv_synic) {
+            return -1;
+        }
+
+        /*
+         * For now just track changes in SynIC control and msg/evt pages msr's.
+         * When SynIC messaging/events processing will be added in future
+         * here we will do messages queues flushing and pages remapping.
+         */
+        switch (exit->u.synic.msr) {
+        case HV_X64_MSR_SCONTROL:
+            env->msr_hv_synic_control = exit->u.synic.control;
+            break;
+        case HV_X64_MSR_SIMP:
+            env->msr_hv_synic_msg_page = exit->u.synic.msg_page;
+            break;
+        case HV_X64_MSR_SIEFP:
+            env->msr_hv_synic_evt_page = exit->u.synic.evt_page;
+            break;
+        default:
+            return -1;
+        }
+        return 0;
+    case KVM_EXIT_HYPERV_HCALL: {
+        uint16_t code;
+
+        code  = exit->u.hcall.input & 0xffff;
+        switch (code) {
+        case HVCALL_POST_MESSAGE:
+        case HVCALL_SIGNAL_EVENT:
+        default:
+            exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE;
+            return 0;
+        }
+    }
+    default:
+        return -1;
+    }
+}
+
+static void kvm_hv_sint_ack_handler(EventNotifier *notifier)
+{
+    HvSintRoute *sint_route = container_of(notifier, HvSintRoute,
+                                           sint_ack_notifier);
+    event_notifier_test_and_clear(notifier);
+    if (sint_route->sint_ack_clb) {
+        sint_route->sint_ack_clb(sint_route);
+    }
+}
+
+HvSintRoute *kvm_hv_sint_route_create(uint32_t vcpu_id, uint32_t sint,
+                                      HvSintAckClb sint_ack_clb)
+{
+    HvSintRoute *sint_route;
+    int r, gsi;
+
+    sint_route = g_malloc0(sizeof(*sint_route));
+    r = event_notifier_init(&sint_route->sint_set_notifier, false);
+    if (r) {
+        goto err;
+    }
+
+    r = event_notifier_init(&sint_route->sint_ack_notifier, false);
+    if (r) {
+        goto err_sint_set_notifier;
+    }
+
+    event_notifier_set_handler(&sint_route->sint_ack_notifier, false,
+                               kvm_hv_sint_ack_handler);
+
+    gsi = kvm_irqchip_add_hv_sint_route(kvm_state, vcpu_id, sint);
+    if (gsi < 0) {
+        goto err_gsi;
+    }
+
+    r = kvm_irqchip_add_irqfd_notifier_gsi(kvm_state,
+                                           &sint_route->sint_set_notifier,
+                                           &sint_route->sint_ack_notifier, gsi);
+    if (r) {
+        goto err_irqfd;
+    }
+    sint_route->gsi = gsi;
+    sint_route->sint_ack_clb = sint_ack_clb;
+    sint_route->vcpu_id = vcpu_id;
+    sint_route->sint = sint;
+
+    return sint_route;
+
+err_irqfd:
+    kvm_irqchip_release_virq(kvm_state, gsi);
+err_gsi:
+    event_notifier_set_handler(&sint_route->sint_ack_notifier, false, NULL);
+    event_notifier_cleanup(&sint_route->sint_ack_notifier);
+err_sint_set_notifier:
+    event_notifier_cleanup(&sint_route->sint_set_notifier);
+err:
+    g_free(sint_route);
+
+    return NULL;
+}
+
+void kvm_hv_sint_route_destroy(HvSintRoute *sint_route)
+{
+    kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state,
+                                          &sint_route->sint_set_notifier,
+                                          sint_route->gsi);
+    kvm_irqchip_release_virq(kvm_state, sint_route->gsi);
+    event_notifier_set_handler(&sint_route->sint_ack_notifier, false, NULL);
+    event_notifier_cleanup(&sint_route->sint_ack_notifier);
+    event_notifier_cleanup(&sint_route->sint_set_notifier);
+    g_free(sint_route);
+}
+
+int kvm_hv_sint_route_set_sint(HvSintRoute *sint_route)
+{
+    return event_notifier_set(&sint_route->sint_set_notifier);
+}
diff --git a/target-i386/hyperv.h b/target-i386/hyperv.h
new file mode 100644
index 0000000..0c3b562
--- /dev/null
+++ b/target-i386/hyperv.h
@@ -0,0 +1,42 @@
+/*
+ * QEMU KVM Hyper-V support
+ *
+ * Copyright (C) 2015 Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * Authors:
+ *  Andrey Smetanin <asmetanin@virtuozzo.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef TARGET_I386_HYPERV_H
+#define TARGET_I386_HYPERV_H
+
+#include "cpu.h"
+#include "sysemu/kvm.h"
+#include "qemu/event_notifier.h"
+
+typedef struct HvSintRoute HvSintRoute;
+typedef void (*HvSintAckClb)(HvSintRoute *sint_route);
+
+struct HvSintRoute {
+    uint32_t sint;
+    uint32_t vcpu_id;
+    int gsi;
+    EventNotifier sint_set_notifier;
+    EventNotifier sint_ack_notifier;
+    HvSintAckClb sint_ack_clb;
+};
+
+int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit);
+
+HvSintRoute *kvm_hv_sint_route_create(uint32_t vcpu_id, uint32_t sint,
+                                      HvSintAckClb sint_ack_clb);
+
+void kvm_hv_sint_route_destroy(HvSintRoute *sint_route);
+
+int kvm_hv_sint_route_set_sint(HvSintRoute *sint_route);
+
+#endif
diff --git a/target-i386/int_helper.c b/target-i386/int_helper.c
index b0d78e6..9e873ac 100644
--- a/target-i386/int_helper.c
+++ b/target-i386/int_helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 
@@ -48,11 +50,11 @@
     num = (env->regs[R_EAX] & 0xffff);
     den = (t0 & 0xff);
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q > 0xff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xff;
     r = (num % den) & 0xff;
@@ -66,11 +68,11 @@
     num = (int16_t)env->regs[R_EAX];
     den = (int8_t)t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q != (int8_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xff;
     r = (num % den) & 0xff;
@@ -84,11 +86,11 @@
     num = (env->regs[R_EAX] & 0xffff) | ((env->regs[R_EDX] & 0xffff) << 16);
     den = (t0 & 0xffff);
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q > 0xffff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xffff;
     r = (num % den) & 0xffff;
@@ -103,11 +105,11 @@
     num = (env->regs[R_EAX] & 0xffff) | ((env->regs[R_EDX] & 0xffff) << 16);
     den = (int16_t)t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     if (q != (int16_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q &= 0xffff;
     r = (num % den) & 0xffff;
@@ -123,12 +125,12 @@
     num = ((uint32_t)env->regs[R_EAX]) | ((uint64_t)((uint32_t)env->regs[R_EDX]) << 32);
     den = t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     r = (num % den);
     if (q > 0xffffffff) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = (uint32_t)q;
     env->regs[R_EDX] = (uint32_t)r;
@@ -142,12 +144,12 @@
     num = ((uint32_t)env->regs[R_EAX]) | ((uint64_t)((uint32_t)env->regs[R_EDX]) << 32);
     den = t0;
     if (den == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     q = (num / den);
     r = (num % den);
     if (q != (int32_t)q) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = (uint32_t)q;
     env->regs[R_EDX] = (uint32_t)r;
@@ -379,12 +381,12 @@
     uint64_t r0, r1;
 
     if (t0 == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     r0 = env->regs[R_EAX];
     r1 = env->regs[R_EDX];
     if (div64(&r0, &r1, t0)) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = r0;
     env->regs[R_EDX] = r1;
@@ -395,12 +397,12 @@
     uint64_t r0, r1;
 
     if (t0 == 0) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     r0 = env->regs[R_EAX];
     r1 = env->regs[R_EDX];
     if (idiv64(&r0, &r1, t0)) {
-        raise_exception(env, EXCP00_DIVZ);
+        raise_exception_ra(env, EXCP00_DIVZ, GETPC());
     }
     env->regs[R_EAX] = r0;
     env->regs[R_EDX] = r1;
@@ -469,3 +471,13 @@
 #include "shift_helper_template.h"
 #undef SHIFT
 #endif
+
+/* Test that BIT is enabled in CR4.  If not, raise an illegal opcode
+   exception.  This reduces the requirements for rare CR4 bits being
+   mapped into HFLAGS.  */
+void helper_cr4_testbit(CPUX86State *env, uint32_t bit)
+{
+    if (unlikely((env->cr[4] & bit) == 0)) {
+        raise_exception_ra(env, EXCP06_ILLOP, GETPC());
+    }
+}
diff --git a/target-i386/ioport-user.c b/target-i386/ioport-user.c
deleted file mode 100644
index f7636e0..0000000
--- a/target-i386/ioport-user.c
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- *  qemu user ioport functions
- *
- *  Copyright (c) 2003-2008 Fabrice Bellard
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <stdio.h>
-
-#include "qemu.h"
-#include "qemu-common.h"
-#include "exec/ioport.h"
-
-void cpu_outb(pio_addr_t addr, uint8_t val)
-{
-    fprintf(stderr, "outb: port=0x%04"FMT_pioaddr", data=%02"PRIx8"\n",
-            addr, val);
-}
-
-void cpu_outw(pio_addr_t addr, uint16_t val)
-{
-    fprintf(stderr, "outw: port=0x%04"FMT_pioaddr", data=%04"PRIx16"\n",
-            addr, val);
-}
-
-void cpu_outl(pio_addr_t addr, uint32_t val)
-{
-    fprintf(stderr, "outl: port=0x%04"FMT_pioaddr", data=%08"PRIx32"\n",
-            addr, val);
-}
-
-uint8_t cpu_inb(pio_addr_t addr)
-{
-    fprintf(stderr, "inb: port=0x%04"FMT_pioaddr"\n", addr);
-    return 0;
-}
-
-uint16_t cpu_inw(pio_addr_t addr)
-{
-    fprintf(stderr, "inw: port=0x%04"FMT_pioaddr"\n", addr);
-    return 0;
-}
-
-uint32_t cpu_inl(pio_addr_t addr)
-{
-    fprintf(stderr, "inl: port=0x%04"FMT_pioaddr"\n", addr);
-    return 0;
-}
diff --git a/target-i386/kvm-stub.c b/target-i386/kvm-stub.c
index 2b9e801..cdf1506 100644
--- a/target-i386/kvm-stub.c
+++ b/target-i386/kvm-stub.c
@@ -9,7 +9,9 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "kvm_i386.h"
 
 bool kvm_allows_irq0_override(void)
@@ -18,6 +20,11 @@
 }
 
 #ifndef __OPTIMIZE__
+bool kvm_has_smm(void)
+{
+    return 1;
+}
+
 /* This function is only called inside conditionals which we
  * rely on the compiler to optimize out when CONFIG_KVM is not
  * defined.
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index ccf36e8..d1a25c5 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -12,31 +12,39 @@
  *
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 #include <sys/utsname.h>
 
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
 
 #include "qemu-common.h"
-#include "sysemu/sysemu.h"
-#include "sysemu/kvm.h"
-#include "kvm_i386.h"
 #include "cpu.h"
+#include "sysemu/sysemu.h"
+#include "sysemu/kvm_int.h"
+#include "kvm_i386.h"
+#include "hyperv.h"
+
 #include "exec/gdbstub.h"
 #include "qemu/host-utils.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "hw/i386/pc.h"
 #include "hw/i386/apic.h"
 #include "hw/i386/apic_internal.h"
 #include "hw/i386/apic-msidef.h"
+#include "hw/i386/intel_iommu.h"
+#include "hw/i386/x86-iommu.h"
+
 #include "exec/ioport.h"
-#include <asm/hyperv.h>
+#include "standard-headers/asm-x86/hyperv.h"
 #include "hw/pci/pci.h"
+#include "hw/pci/msi.h"
 #include "migration/migration.h"
-#include "qapi/qmp/qerror.h"
+#include "exec/memattrs.h"
+#include "trace.h"
 
 //#define DEBUG_KVM
 
@@ -51,6 +59,10 @@
 #define MSR_KVM_WALL_CLOCK  0x11
 #define MSR_KVM_SYSTEM_TIME 0x12
 
+/* A 4096-byte buffer can hold the 8-byte kvm_msrs header, plus
+ * 255 kvm_msr_entry structs */
+#define MSR_BUF_SIZE 4096
+
 #ifndef BUS_MCEERR_AR
 #define BUS_MCEERR_AR 4
 #endif
@@ -67,35 +79,108 @@
 
 static bool has_msr_star;
 static bool has_msr_hsave_pa;
+static bool has_msr_tsc_aux;
 static bool has_msr_tsc_adjust;
 static bool has_msr_tsc_deadline;
 static bool has_msr_feature_control;
 static bool has_msr_async_pf_en;
 static bool has_msr_pv_eoi_en;
 static bool has_msr_misc_enable;
+static bool has_msr_smbase;
 static bool has_msr_bndcfgs;
 static bool has_msr_kvm_steal_time;
 static int lm_capable_kernel;
 static bool has_msr_hv_hypercall;
 static bool has_msr_hv_vapic;
 static bool has_msr_hv_tsc;
+static bool has_msr_hv_crash;
+static bool has_msr_hv_reset;
+static bool has_msr_hv_vpindex;
+static bool has_msr_hv_runtime;
+static bool has_msr_hv_synic;
+static bool has_msr_hv_stimer;
 static bool has_msr_mtrr;
+static bool has_msr_xss;
 
 static bool has_msr_architectural_pmu;
 static uint32_t num_architectural_pmu_counters;
 
+static int has_xsave;
+static int has_xcrs;
+static int has_pit_state2;
+
+static bool has_msr_mcg_ext_ctl;
+
+static struct kvm_cpuid2 *cpuid_cache;
+
+int kvm_has_pit_state2(void)
+{
+    return has_pit_state2;
+}
+
+bool kvm_has_smm(void)
+{
+    return kvm_check_extension(kvm_state, KVM_CAP_X86_SMM);
+}
+
 bool kvm_allows_irq0_override(void)
 {
     return !kvm_irqchip_in_kernel() || kvm_has_gsi_routing();
 }
 
+static int kvm_get_tsc(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    struct {
+        struct kvm_msrs info;
+        struct kvm_msr_entry entries[1];
+    } msr_data;
+    int ret;
+
+    if (env->tsc_valid) {
+        return 0;
+    }
+
+    msr_data.info.nmsrs = 1;
+    msr_data.entries[0].index = MSR_IA32_TSC;
+    env->tsc_valid = !runstate_is_running();
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    assert(ret == 1);
+    env->tsc = msr_data.entries[0].data;
+    return 0;
+}
+
+static inline void do_kvm_synchronize_tsc(void *arg)
+{
+    CPUState *cpu = arg;
+
+    kvm_get_tsc(cpu);
+}
+
+void kvm_synchronize_all_tsc(void)
+{
+    CPUState *cpu;
+
+    if (kvm_enabled()) {
+        CPU_FOREACH(cpu) {
+            run_on_cpu(cpu, do_kvm_synchronize_tsc, cpu);
+        }
+    }
+}
+
 static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
 {
     struct kvm_cpuid2 *cpuid;
     int r, size;
 
     size = sizeof(*cpuid) + max * sizeof(*cpuid->entries);
-    cpuid = (struct kvm_cpuid2 *)g_malloc0(size);
+    cpuid = g_malloc0(size);
     cpuid->nent = max;
     r = kvm_ioctl(s, KVM_GET_SUPPORTED_CPUID, cpuid);
     if (r == 0 && cpuid->nent >= max) {
@@ -121,9 +206,14 @@
 {
     struct kvm_cpuid2 *cpuid;
     int max = 1;
+
+    if (cpuid_cache != NULL) {
+        return cpuid_cache;
+    }
     while ((cpuid = try_get_cpuid(s, max)) == NULL) {
         max *= 2;
     }
+    cpuid_cache = cpuid;
     return cpuid;
 }
 
@@ -231,16 +321,23 @@
         if (!kvm_irqchip_in_kernel()) {
             ret &= ~CPUID_EXT_X2APIC;
         }
+    } else if (function == 6 && reg == R_EAX) {
+        ret |= CPUID_6_EAX_ARAT; /* safe to allow because of emulated APIC */
     } else if (function == 0x80000001 && reg == R_EDX) {
         /* On Intel, kvm returns cpuid according to the Intel spec,
          * so add missing bits according to the AMD spec:
          */
         cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
         ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
+    } else if (function == KVM_CPUID_FEATURES && reg == R_EAX) {
+        /* kvm_pv_unhalt is reported by GET_SUPPORTED_CPUID, but it can't
+         * be enabled without the in-kernel irqchip
+         */
+        if (!kvm_irqchip_in_kernel()) {
+            ret &= ~(1U << KVM_FEATURE_PV_UNHALT);
+        }
     }
 
-    g_free(cpuid);
-
     /* fallback for older kernels */
     if ((function == KVM_CPUID_FEATURES) && !found) {
         ret = get_para_features(s);
@@ -277,7 +374,7 @@
             return;
         }
     }
-    page = g_malloc(sizeof(HWPoisonPage));
+    page = g_new(HWPoisonPage, 1);
     page->ram_addr = ram_addr;
     QLIST_INSERT_HEAD(&hwpoison_page_list, page, list);
 }
@@ -297,10 +394,12 @@
 
 static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code)
 {
+    CPUState *cs = CPU(cpu);
     CPUX86State *env = &cpu->env;
     uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
                       MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
     uint64_t mcg_status = MCG_STATUS_MCIP;
+    int flags = 0;
 
     if (code == BUS_MCEERR_AR) {
         status |= MCI_STATUS_AR | 0x134;
@@ -309,10 +408,19 @@
         status |= 0xc0;
         mcg_status |= MCG_STATUS_RIPV;
     }
+
+    flags = cpu_x86_support_mca_broadcast(env) ? MCE_INJECT_BROADCAST : 0;
+    /* We need to read back the value of MSR_EXT_MCG_CTL that was set by the
+     * guest kernel back into env->mcg_ext_ctl.
+     */
+    cpu_synchronize_state(cs);
+    if (env->mcg_ext_ctl & MCG_EXT_CTL_LMCE_EN) {
+        mcg_status |= MCG_STATUS_LMCE;
+        flags = 0;
+    }
+
     cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr,
-                       (MCM_ADDR_PHYS << 6) | 0xc,
-                       cpu_x86_support_mca_broadcast(env) ?
-                       MCE_INJECT_BROADCAST : 0);
+                       (MCM_ADDR_PHYS << 6) | 0xc, flags);
 }
 
 static void hardware_memory_error(void)
@@ -330,7 +438,8 @@
 
     if ((env->mcg_cap & MCG_SER_P) && addr
         && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
-        if (qemu_ram_addr_from_host(addr, &ram_addr) == NULL ||
+        ram_addr = qemu_ram_addr_from_host(addr);
+        if (ram_addr == RAM_ADDR_INVALID ||
             !kvm_physical_memory_addr_from_host(c->kvm_state, addr, &paddr)) {
             fprintf(stderr, "Hardware memory error for memory used by "
                     "QEMU itself instead of guest system!\n");
@@ -364,7 +473,8 @@
         hwaddr paddr;
 
         /* Hope we are lucky for AO MCE */
-        if (qemu_ram_addr_from_host(addr, &ram_addr) == NULL ||
+        ram_addr = qemu_ram_addr_from_host(addr);
+        if (ram_addr == RAM_ADDR_INVALID ||
             !kvm_physical_memory_addr_from_host(first_cpu->kvm_state,
                                                 addr, &paddr)) {
             fprintf(stderr, "Hardware memory error for memory used by "
@@ -429,7 +539,7 @@
 unsigned long kvm_arch_vcpu_id(CPUState *cs)
 {
     X86CPU *cpu = X86_CPU(cs);
-    return cpu->env.cpuid_apic_id;
+    return cpu->apic_id;
 }
 
 #ifndef KVM_CPUID_SIGNATURE_NEXT
@@ -448,7 +558,103 @@
     return kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV) > 0 &&
            (hyperv_hypercall_available(cpu) ||
             cpu->hyperv_time  ||
-            cpu->hyperv_relaxed_timing);
+            cpu->hyperv_relaxed_timing ||
+            cpu->hyperv_crash ||
+            cpu->hyperv_reset ||
+            cpu->hyperv_vpindex ||
+            cpu->hyperv_runtime ||
+            cpu->hyperv_synic ||
+            cpu->hyperv_stimer);
+}
+
+static int kvm_arch_set_tsc_khz(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+    int r;
+
+    if (!env->tsc_khz) {
+        return 0;
+    }
+
+    r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL) ?
+        kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz) :
+        -ENOTSUP;
+    if (r < 0) {
+        /* When KVM_SET_TSC_KHZ fails, it's an error only if the current
+         * TSC frequency doesn't match the one we want.
+         */
+        int cur_freq = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
+                       kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
+                       -ENOTSUP;
+        if (cur_freq <= 0 || cur_freq != env->tsc_khz) {
+            error_report("warning: TSC frequency mismatch between "
+                         "VM (%" PRId64 " kHz) and host (%d kHz), "
+                         "and TSC scaling unavailable",
+                         env->tsc_khz, cur_freq);
+            return r;
+        }
+    }
+
+    return 0;
+}
+
+static int hyperv_handle_properties(CPUState *cs)
+{
+    X86CPU *cpu = X86_CPU(cs);
+    CPUX86State *env = &cpu->env;
+
+    if (cpu->hyperv_relaxed_timing) {
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_HYPERCALL_AVAILABLE;
+    }
+    if (cpu->hyperv_vapic) {
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_HYPERCALL_AVAILABLE;
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_APIC_ACCESS_AVAILABLE;
+        has_msr_hv_vapic = true;
+    }
+    if (cpu->hyperv_time &&
+            kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_TIME) > 0) {
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_HYPERCALL_AVAILABLE;
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_TIME_REF_COUNT_AVAILABLE;
+        env->features[FEAT_HYPERV_EAX] |= 0x200;
+        has_msr_hv_tsc = true;
+    }
+    if (cpu->hyperv_crash && has_msr_hv_crash) {
+        env->features[FEAT_HYPERV_EDX] |= HV_X64_GUEST_CRASH_MSR_AVAILABLE;
+    }
+    env->features[FEAT_HYPERV_EDX] |= HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE;
+    if (cpu->hyperv_reset && has_msr_hv_reset) {
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_RESET_AVAILABLE;
+    }
+    if (cpu->hyperv_vpindex && has_msr_hv_vpindex) {
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_VP_INDEX_AVAILABLE;
+    }
+    if (cpu->hyperv_runtime && has_msr_hv_runtime) {
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_VP_RUNTIME_AVAILABLE;
+    }
+    if (cpu->hyperv_synic) {
+        int sint;
+
+        if (!has_msr_hv_synic ||
+            kvm_vcpu_enable_cap(cs, KVM_CAP_HYPERV_SYNIC, 0)) {
+            fprintf(stderr, "Hyper-V SynIC is not supported by kernel\n");
+            return -ENOSYS;
+        }
+
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_SYNIC_AVAILABLE;
+        env->msr_hv_synic_version = HV_SYNIC_VERSION_1;
+        for (sint = 0; sint < ARRAY_SIZE(env->msr_hv_synic_sint); sint++) {
+            env->msr_hv_synic_sint[sint] = HV_SYNIC_SINT_MASKED;
+        }
+    }
+    if (cpu->hyperv_stimer) {
+        if (!has_msr_hv_stimer) {
+            fprintf(stderr, "Hyper-V timers aren't supported by kernel\n");
+            return -ENOSYS;
+        }
+        env->features[FEAT_HYPERV_EAX] |= HV_X64_MSR_SYNTIMER_AVAILABLE;
+    }
+    return 0;
 }
 
 static Error *invtsc_mig_blocker;
@@ -478,7 +684,18 @@
     if (hyperv_enabled(cpu)) {
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS;
-        memcpy(signature, "Microsoft Hv", 12);
+        if (!cpu->hyperv_vendor_id) {
+            memcpy(signature, "Microsoft Hv", 12);
+        } else {
+            size_t len = strlen(cpu->hyperv_vendor_id);
+
+            if (len > 12) {
+                error_report("hv-vendor-id truncated to 12 characters");
+                len = 12;
+            }
+            memset(signature, 0, 12);
+            memcpy(signature, cpu->hyperv_vendor_id, len);
+        }
         c->eax = HYPERV_CPUID_MIN;
         c->ebx = signature[0];
         c->ecx = signature[1];
@@ -499,21 +716,14 @@
 
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_FEATURES;
-        if (cpu->hyperv_relaxed_timing) {
-            c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
+        r = hyperv_handle_properties(cs);
+        if (r) {
+            return r;
         }
-        if (cpu->hyperv_vapic) {
-            c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
-            c->eax |= HV_X64_MSR_APIC_ACCESS_AVAILABLE;
-            has_msr_hv_vapic = true;
-        }
-        if (cpu->hyperv_time &&
-            kvm_check_extension(cs->kvm_state, KVM_CAP_HYPERV_TIME) > 0) {
-            c->eax |= HV_X64_MSR_HYPERCALL_AVAILABLE;
-            c->eax |= HV_X64_MSR_TIME_REF_COUNT_AVAILABLE;
-            c->eax |= 0x200;
-            has_msr_hv_tsc = true;
-        }
+        c->eax = env->features[FEAT_HYPERV_EAX];
+        c->ebx = env->features[FEAT_HYPERV_EBX];
+        c->edx = env->features[FEAT_HYPERV_EDX];
+
         c = &cpuid_data.entries[cpuid_i++];
         c->function = HYPERV_CPUID_ENLIGHTMENT_INFO;
         if (cpu->hyperv_relaxed_timing) {
@@ -678,7 +888,7 @@
         && (env->features[FEAT_1_EDX] & (CPUID_MCE | CPUID_MCA)) ==
            (CPUID_MCE | CPUID_MCA)
         && kvm_check_extension(cs->kvm_state, KVM_CAP_MCE) > 0) {
-        uint64_t mcg_cap;
+        uint64_t mcg_cap, unsupported_caps;
         int banks;
         int ret;
 
@@ -688,18 +898,28 @@
             return ret;
         }
 
-        if (banks > MCE_BANKS_DEF) {
-            banks = MCE_BANKS_DEF;
+        if (banks < (env->mcg_cap & MCG_CAP_BANKS_MASK)) {
+            error_report("kvm: Unsupported MCE bank count (QEMU = %d, KVM = %d)",
+                         (int)(env->mcg_cap & MCG_CAP_BANKS_MASK), banks);
+            return -ENOTSUP;
         }
-        mcg_cap &= MCE_CAP_DEF;
-        mcg_cap |= banks;
-        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &mcg_cap);
+
+        unsupported_caps = env->mcg_cap & ~(mcg_cap | MCG_CAP_BANKS_MASK);
+        if (unsupported_caps) {
+            if (unsupported_caps & MCG_LMCE_P) {
+                error_report("kvm: LMCE not supported");
+                return -ENOTSUP;
+            }
+            error_report("warning: Unsupported MCG_CAP bits: 0x%" PRIx64,
+                         unsupported_caps);
+        }
+
+        env->mcg_cap &= mcg_cap | MCG_CAP_BANKS_MASK;
+        ret = kvm_vcpu_ioctl(cs, KVM_X86_SETUP_MCE, &env->mcg_cap);
         if (ret < 0) {
             fprintf(stderr, "KVM_X86_SETUP_MCE: %s", strerror(-ret));
             return ret;
         }
-
-        env->mcg_cap = mcg_cap;
     }
 
     qemu_add_vm_change_state_handler(cpu_update_state, env);
@@ -710,6 +930,10 @@
                                   !!(c->ecx & CPUID_EXT_SMX);
     }
 
+    if (env->mcg_cap & MCG_LMCE_P) {
+        has_msr_mcg_ext_ctl = has_msr_feature_control = true;
+    }
+
     c = cpuid_find_entry(&cpuid_data.cpuid, 0x80000007, 0);
     if (c && (c->edx & 1<<8) && invtsc_mig_blocker == NULL) {
         /* for migration */
@@ -727,22 +951,36 @@
         return r;
     }
 
-    r = kvm_check_extension(cs->kvm_state, KVM_CAP_TSC_CONTROL);
-    if (r && env->tsc_khz) {
-        r = kvm_vcpu_ioctl(cs, KVM_SET_TSC_KHZ, env->tsc_khz);
-        if (r < 0) {
-            fprintf(stderr, "KVM_SET_TSC_KHZ failed\n");
-            return r;
+    r = kvm_arch_set_tsc_khz(cs);
+    if (r < 0) {
+        return r;
+    }
+
+    /* vcpu's TSC frequency is either specified by user, or following
+     * the value used by KVM if the former is not present. In the
+     * latter case, we query it from KVM and record in env->tsc_khz,
+     * so that vcpu's TSC frequency can be migrated later via this field.
+     */
+    if (!env->tsc_khz) {
+        r = kvm_check_extension(cs->kvm_state, KVM_CAP_GET_TSC_KHZ) ?
+            kvm_vcpu_ioctl(cs, KVM_GET_TSC_KHZ) :
+            -ENOTSUP;
+        if (r > 0) {
+            env->tsc_khz = r;
         }
     }
 
-    if (kvm_has_xsave()) {
+    if (has_xsave) {
         env->kvm_xsave_buf = qemu_memalign(4096, sizeof(struct kvm_xsave));
     }
+    cpu->kvm_msr_buf = g_malloc0(MSR_BUF_SIZE);
 
     if (env->features[FEAT_1_EDX] & CPUID_MTRR) {
         has_msr_mtrr = true;
     }
+    if (!(env->features[FEAT_8000_0001_EDX] & CPUID_EXT2_RDTSCP)) {
+        has_msr_tsc_aux = false;
+    }
 
     return 0;
 }
@@ -810,6 +1048,10 @@
                     has_msr_hsave_pa = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_TSC_AUX) {
+                    has_msr_tsc_aux = true;
+                    continue;
+                }
                 if (kvm_msr_list->indices[i] == MSR_TSC_ADJUST) {
                     has_msr_tsc_adjust = true;
                     continue;
@@ -818,6 +1060,10 @@
                     has_msr_tsc_deadline = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_IA32_SMBASE) {
+                    has_msr_smbase = true;
+                    continue;
+                }
                 if (kvm_msr_list->indices[i] == MSR_IA32_MISC_ENABLE) {
                     has_msr_misc_enable = true;
                     continue;
@@ -826,6 +1072,34 @@
                     has_msr_bndcfgs = true;
                     continue;
                 }
+                if (kvm_msr_list->indices[i] == MSR_IA32_XSS) {
+                    has_msr_xss = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_CRASH_CTL) {
+                    has_msr_hv_crash = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_RESET) {
+                    has_msr_hv_reset = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_VP_INDEX) {
+                    has_msr_hv_vpindex = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_VP_RUNTIME) {
+                    has_msr_hv_runtime = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_SCONTROL) {
+                    has_msr_hv_synic = true;
+                    continue;
+                }
+                if (kvm_msr_list->indices[i] == HV_X64_MSR_STIMER0_CONFIG) {
+                    has_msr_hv_stimer = true;
+                    continue;
+                }
             }
         }
 
@@ -835,13 +1109,59 @@
     return ret;
 }
 
-int kvm_arch_init(KVMState *s)
+static Notifier smram_machine_done;
+static KVMMemoryListener smram_listener;
+static AddressSpace smram_address_space;
+static MemoryRegion smram_as_root;
+static MemoryRegion smram_as_mem;
+
+static void register_smram_listener(Notifier *n, void *unused)
+{
+    MemoryRegion *smram =
+        (MemoryRegion *) object_resolve_path("/machine/smram", NULL);
+
+    /* Outer container... */
+    memory_region_init(&smram_as_root, OBJECT(kvm_state), "mem-container-smram", ~0ull);
+    memory_region_set_enabled(&smram_as_root, true);
+
+    /* ... with two regions inside: normal system memory with low
+     * priority, and...
+     */
+    memory_region_init_alias(&smram_as_mem, OBJECT(kvm_state), "mem-smram",
+                             get_system_memory(), 0, ~0ull);
+    memory_region_add_subregion_overlap(&smram_as_root, 0, &smram_as_mem, 0);
+    memory_region_set_enabled(&smram_as_mem, true);
+
+    if (smram) {
+        /* ... SMRAM with higher priority */
+        memory_region_add_subregion_overlap(&smram_as_root, 0, smram, 10);
+        memory_region_set_enabled(smram, true);
+    }
+
+    address_space_init(&smram_address_space, &smram_as_root, "KVM-SMRAM");
+    kvm_memory_listener_register(kvm_state, &smram_listener,
+                                 &smram_address_space, 1);
+}
+
+int kvm_arch_init(MachineState *ms, KVMState *s)
 {
     uint64_t identity_base = 0xfffbc000;
     uint64_t shadow_mem;
     int ret;
     struct utsname utsname;
 
+#ifdef KVM_CAP_XSAVE
+    has_xsave = kvm_check_extension(s, KVM_CAP_XSAVE);
+#endif
+
+#ifdef KVM_CAP_XCRS
+    has_xcrs = kvm_check_extension(s, KVM_CAP_XCRS);
+#endif
+
+#ifdef KVM_CAP_PIT_STATE2
+    has_pit_state2 = kvm_check_extension(s, KVM_CAP_PIT_STATE2);
+#endif
+
     ret = kvm_get_supported_msrs(s);
     if (ret < 0) {
         return ret;
@@ -885,8 +1205,7 @@
     }
     qemu_register_reset(kvm_unpoison_all, NULL);
 
-    shadow_mem = qemu_opt_get_size(qemu_get_machine_opts(),
-                                   "kvm_shadow_mem", -1);
+    shadow_mem = machine_kvm_shadow_mem(ms);
     if (shadow_mem != -1) {
         shadow_mem /= 4096;
         ret = kvm_vm_ioctl(s, KVM_SET_NR_MMU_PAGES, shadow_mem);
@@ -894,6 +1213,11 @@
             return ret;
         }
     }
+
+    if (kvm_check_extension(s, KVM_CAP_X86_SMM)) {
+        smram_machine_done.notify = register_smram_listener;
+        qemu_add_machine_init_done_notifier(&smram_machine_done);
+    }
     return 0;
 }
 
@@ -927,7 +1251,7 @@
     lhs->l = (flags >> DESC_L_SHIFT) & 1;
     lhs->g = (flags & DESC_G_MASK) != 0;
     lhs->avl = (flags & DESC_AVL_MASK) != 0;
-    lhs->unusable = 0;
+    lhs->unusable = !lhs->present;
     lhs->padding = 0;
 }
 
@@ -936,14 +1260,18 @@
     lhs->selector = rhs->selector;
     lhs->base = rhs->base;
     lhs->limit = rhs->limit;
-    lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
-                 (rhs->present * DESC_P_MASK) |
-                 (rhs->dpl << DESC_DPL_SHIFT) |
-                 (rhs->db << DESC_B_SHIFT) |
-                 (rhs->s * DESC_S_MASK) |
-                 (rhs->l << DESC_L_SHIFT) |
-                 (rhs->g * DESC_G_MASK) |
-                 (rhs->avl * DESC_AVL_MASK);
+    if (rhs->unusable) {
+        lhs->flags = 0;
+    } else {
+        lhs->flags = (rhs->type << DESC_TYPE_SHIFT) |
+                     (rhs->present * DESC_P_MASK) |
+                     (rhs->dpl << DESC_DPL_SHIFT) |
+                     (rhs->db << DESC_B_SHIFT) |
+                     (rhs->s * DESC_S_MASK) |
+                     (rhs->l << DESC_L_SHIFT) |
+                     (rhs->g * DESC_G_MASK) |
+                     (rhs->avl * DESC_AVL_MASK);
+    }
 }
 
 static void kvm_getput_reg(__u64 *kvm_reg, target_ulong *qemu_reg, int set)
@@ -1014,7 +1342,10 @@
         fpu.ftwx |= (!env->fptags[i]) << i;
     }
     memcpy(fpu.fpr, env->fpregs, sizeof env->fpregs);
-    memcpy(fpu.xmm, env->xmm_regs, sizeof env->xmm_regs);
+    for (i = 0; i < CPU_NB_REGS; i++) {
+        stq_p(&fpu.xmm[i][0], env->xmm_regs[i].ZMM_Q(0));
+        stq_p(&fpu.xmm[i][8], env->xmm_regs[i].ZMM_Q(1));
+    }
     fpu.mxcsr = env->mxcsr;
 
     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_FPU, &fpu);
@@ -1034,15 +1365,39 @@
 #define XSAVE_OPMASK      272
 #define XSAVE_ZMM_Hi256   288
 #define XSAVE_Hi16_ZMM    416
+#define XSAVE_PKRU        672
+
+#define XSAVE_BYTE_OFFSET(word_offset) \
+    ((word_offset) * sizeof(((struct kvm_xsave *)0)->region[0]))
+
+#define ASSERT_OFFSET(word_offset, field) \
+    QEMU_BUILD_BUG_ON(XSAVE_BYTE_OFFSET(word_offset) != \
+                      offsetof(X86XSaveArea, field))
+
+ASSERT_OFFSET(XSAVE_FCW_FSW, legacy.fcw);
+ASSERT_OFFSET(XSAVE_FTW_FOP, legacy.ftw);
+ASSERT_OFFSET(XSAVE_CWD_RIP, legacy.fpip);
+ASSERT_OFFSET(XSAVE_CWD_RDP, legacy.fpdp);
+ASSERT_OFFSET(XSAVE_MXCSR, legacy.mxcsr);
+ASSERT_OFFSET(XSAVE_ST_SPACE, legacy.fpregs);
+ASSERT_OFFSET(XSAVE_XMM_SPACE, legacy.xmm_regs);
+ASSERT_OFFSET(XSAVE_XSTATE_BV, header.xstate_bv);
+ASSERT_OFFSET(XSAVE_YMMH_SPACE, avx_state);
+ASSERT_OFFSET(XSAVE_BNDREGS, bndreg_state);
+ASSERT_OFFSET(XSAVE_BNDCSR, bndcsr_state);
+ASSERT_OFFSET(XSAVE_OPMASK, opmask_state);
+ASSERT_OFFSET(XSAVE_ZMM_Hi256, zmm_hi256_state);
+ASSERT_OFFSET(XSAVE_Hi16_ZMM, hi16_zmm_state);
+ASSERT_OFFSET(XSAVE_PKRU, pkru_state);
 
 static int kvm_put_xsave(X86CPU *cpu)
 {
     CPUX86State *env = &cpu->env;
-    struct kvm_xsave* xsave = env->kvm_xsave_buf;
+    X86XSaveArea *xsave = env->kvm_xsave_buf;
     uint16_t cwd, swd, twd;
-    int i, r;
+    int i;
 
-    if (!kvm_has_xsave()) {
+    if (!has_xsave) {
         return kvm_put_fpu(cpu);
     }
 
@@ -1054,40 +1409,50 @@
     for (i = 0; i < 8; ++i) {
         twd |= (!env->fptags[i]) << i;
     }
-    xsave->region[XSAVE_FCW_FSW] = (uint32_t)(swd << 16) + cwd;
-    xsave->region[XSAVE_FTW_FOP] = (uint32_t)(env->fpop << 16) + twd;
-    memcpy(&xsave->region[XSAVE_CWD_RIP], &env->fpip, sizeof(env->fpip));
-    memcpy(&xsave->region[XSAVE_CWD_RDP], &env->fpdp, sizeof(env->fpdp));
-    memcpy(&xsave->region[XSAVE_ST_SPACE], env->fpregs,
+    xsave->legacy.fcw = cwd;
+    xsave->legacy.fsw = swd;
+    xsave->legacy.ftw = twd;
+    xsave->legacy.fpop = env->fpop;
+    xsave->legacy.fpip = env->fpip;
+    xsave->legacy.fpdp = env->fpdp;
+    memcpy(&xsave->legacy.fpregs, env->fpregs,
             sizeof env->fpregs);
-    memcpy(&xsave->region[XSAVE_XMM_SPACE], env->xmm_regs,
-            sizeof env->xmm_regs);
-    xsave->region[XSAVE_MXCSR] = env->mxcsr;
-    *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV] = env->xstate_bv;
-    memcpy(&xsave->region[XSAVE_YMMH_SPACE], env->ymmh_regs,
-            sizeof env->ymmh_regs);
-    memcpy(&xsave->region[XSAVE_BNDREGS], env->bnd_regs,
+    xsave->legacy.mxcsr = env->mxcsr;
+    xsave->header.xstate_bv = env->xstate_bv;
+    memcpy(&xsave->bndreg_state.bnd_regs, env->bnd_regs,
             sizeof env->bnd_regs);
-    memcpy(&xsave->region[XSAVE_BNDCSR], &env->bndcs_regs,
-            sizeof(env->bndcs_regs));
-    memcpy(&xsave->region[XSAVE_OPMASK], env->opmask_regs,
+    xsave->bndcsr_state.bndcsr = env->bndcs_regs;
+    memcpy(&xsave->opmask_state.opmask_regs, env->opmask_regs,
             sizeof env->opmask_regs);
-    memcpy(&xsave->region[XSAVE_ZMM_Hi256], env->zmmh_regs,
-            sizeof env->zmmh_regs);
+
+    for (i = 0; i < CPU_NB_REGS; i++) {
+        uint8_t *xmm = xsave->legacy.xmm_regs[i];
+        uint8_t *ymmh = xsave->avx_state.ymmh[i];
+        uint8_t *zmmh = xsave->zmm_hi256_state.zmm_hi256[i];
+        stq_p(xmm,     env->xmm_regs[i].ZMM_Q(0));
+        stq_p(xmm+8,   env->xmm_regs[i].ZMM_Q(1));
+        stq_p(ymmh,    env->xmm_regs[i].ZMM_Q(2));
+        stq_p(ymmh+8,  env->xmm_regs[i].ZMM_Q(3));
+        stq_p(zmmh,    env->xmm_regs[i].ZMM_Q(4));
+        stq_p(zmmh+8,  env->xmm_regs[i].ZMM_Q(5));
+        stq_p(zmmh+16, env->xmm_regs[i].ZMM_Q(6));
+        stq_p(zmmh+24, env->xmm_regs[i].ZMM_Q(7));
+    }
+
 #ifdef TARGET_X86_64
-    memcpy(&xsave->region[XSAVE_Hi16_ZMM], env->hi16_zmm_regs,
-            sizeof env->hi16_zmm_regs);
+    memcpy(&xsave->hi16_zmm_state.hi16_zmm, &env->xmm_regs[16],
+            16 * sizeof env->xmm_regs[16]);
+    memcpy(&xsave->pkru_state, &env->pkru, sizeof env->pkru);
 #endif
-    r = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
-    return r;
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_XSAVE, xsave);
 }
 
 static int kvm_put_xcrs(X86CPU *cpu)
 {
     CPUX86State *env = &cpu->env;
-    struct kvm_xcrs xcrs;
+    struct kvm_xcrs xcrs = {};
 
-    if (!kvm_has_xcrs()) {
+    if (!has_xcrs) {
         return 0;
     }
 
@@ -1148,31 +1513,44 @@
     return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
 }
 
-static void kvm_msr_entry_set(struct kvm_msr_entry *entry,
-                              uint32_t index, uint64_t value)
+static void kvm_msr_buf_reset(X86CPU *cpu)
 {
+    memset(cpu->kvm_msr_buf, 0, MSR_BUF_SIZE);
+}
+
+static void kvm_msr_entry_add(X86CPU *cpu, uint32_t index, uint64_t value)
+{
+    struct kvm_msrs *msrs = cpu->kvm_msr_buf;
+    void *limit = ((void *)msrs) + MSR_BUF_SIZE;
+    struct kvm_msr_entry *entry = &msrs->entries[msrs->nmsrs];
+
+    assert((void *)(entry + 1) <= limit);
+
     entry->index = index;
+    entry->reserved = 0;
     entry->data = value;
+    msrs->nmsrs++;
 }
 
 static int kvm_put_tscdeadline_msr(X86CPU *cpu)
 {
     CPUX86State *env = &cpu->env;
-    struct {
-        struct kvm_msrs info;
-        struct kvm_msr_entry entries[1];
-    } msr_data;
-    struct kvm_msr_entry *msrs = msr_data.entries;
+    int ret;
 
     if (!has_msr_tsc_deadline) {
         return 0;
     }
 
-    kvm_msr_entry_set(&msrs[0], MSR_IA32_TSCDEADLINE, env->tsc_deadline);
+    kvm_msr_buf_reset(cpu);
+    kvm_msr_entry_add(cpu, MSR_IA32_TSCDEADLINE, env->tsc_deadline);
 
-    msr_data.info.nmsrs = 1;
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
+    if (ret < 0) {
+        return ret;
+    }
 
-    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+    assert(ret == 1);
+    return 0;
 }
 
 /*
@@ -1183,53 +1561,68 @@
  */
 static int kvm_put_msr_feature_control(X86CPU *cpu)
 {
-    struct {
-        struct kvm_msrs info;
-        struct kvm_msr_entry entry;
-    } msr_data;
+    int ret;
 
-    kvm_msr_entry_set(&msr_data.entry, MSR_IA32_FEATURE_CONTROL,
+    if (!has_msr_feature_control) {
+        return 0;
+    }
+
+    kvm_msr_buf_reset(cpu);
+    kvm_msr_entry_add(cpu, MSR_IA32_FEATURE_CONTROL,
                       cpu->env.msr_ia32_feature_control);
-    msr_data.info.nmsrs = 1;
-    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
+    if (ret < 0) {
+        return ret;
+    }
+
+    assert(ret == 1);
+    return 0;
 }
 
 static int kvm_put_msrs(X86CPU *cpu, int level)
 {
     CPUX86State *env = &cpu->env;
-    struct {
-        struct kvm_msrs info;
-        struct kvm_msr_entry entries[150];
-    } msr_data;
-    struct kvm_msr_entry *msrs = msr_data.entries;
-    int n = 0, i;
+    int i;
+    int ret;
 
-    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_CS, env->sysenter_cs);
-    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
-    kvm_msr_entry_set(&msrs[n++], MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
-    kvm_msr_entry_set(&msrs[n++], MSR_PAT, env->pat);
+    kvm_msr_buf_reset(cpu);
+
+    kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, env->sysenter_cs);
+    kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, env->sysenter_esp);
+    kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, env->sysenter_eip);
+    kvm_msr_entry_add(cpu, MSR_PAT, env->pat);
     if (has_msr_star) {
-        kvm_msr_entry_set(&msrs[n++], MSR_STAR, env->star);
+        kvm_msr_entry_add(cpu, MSR_STAR, env->star);
     }
     if (has_msr_hsave_pa) {
-        kvm_msr_entry_set(&msrs[n++], MSR_VM_HSAVE_PA, env->vm_hsave);
+        kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, env->vm_hsave);
+    }
+    if (has_msr_tsc_aux) {
+        kvm_msr_entry_add(cpu, MSR_TSC_AUX, env->tsc_aux);
     }
     if (has_msr_tsc_adjust) {
-        kvm_msr_entry_set(&msrs[n++], MSR_TSC_ADJUST, env->tsc_adjust);
+        kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, env->tsc_adjust);
     }
     if (has_msr_misc_enable) {
-        kvm_msr_entry_set(&msrs[n++], MSR_IA32_MISC_ENABLE,
+        kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE,
                           env->msr_ia32_misc_enable);
     }
+    if (has_msr_smbase) {
+        kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, env->smbase);
+    }
     if (has_msr_bndcfgs) {
-        kvm_msr_entry_set(&msrs[n++], MSR_IA32_BNDCFGS, env->msr_bndcfgs);
+        kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, env->msr_bndcfgs);
+    }
+    if (has_msr_xss) {
+        kvm_msr_entry_add(cpu, MSR_IA32_XSS, env->xss);
     }
 #ifdef TARGET_X86_64
     if (lm_capable_kernel) {
-        kvm_msr_entry_set(&msrs[n++], MSR_CSTAR, env->cstar);
-        kvm_msr_entry_set(&msrs[n++], MSR_KERNELGSBASE, env->kernelgsbase);
-        kvm_msr_entry_set(&msrs[n++], MSR_FMASK, env->fmask);
-        kvm_msr_entry_set(&msrs[n++], MSR_LSTAR, env->lstar);
+        kvm_msr_entry_add(cpu, MSR_CSTAR, env->cstar);
+        kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, env->kernelgsbase);
+        kvm_msr_entry_add(cpu, MSR_FMASK, env->fmask);
+        kvm_msr_entry_add(cpu, MSR_LSTAR, env->lstar);
     }
 #endif
     /*
@@ -1237,92 +1630,126 @@
      * for normal writeback. Limit them to reset or full state updates.
      */
     if (level >= KVM_PUT_RESET_STATE) {
-        kvm_msr_entry_set(&msrs[n++], MSR_IA32_TSC, env->tsc);
-        kvm_msr_entry_set(&msrs[n++], MSR_KVM_SYSTEM_TIME,
-                          env->system_time_msr);
-        kvm_msr_entry_set(&msrs[n++], MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
+        kvm_msr_entry_add(cpu, MSR_IA32_TSC, env->tsc);
+        kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, env->system_time_msr);
+        kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, env->wall_clock_msr);
         if (has_msr_async_pf_en) {
-            kvm_msr_entry_set(&msrs[n++], MSR_KVM_ASYNC_PF_EN,
-                              env->async_pf_en_msr);
+            kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, env->async_pf_en_msr);
         }
         if (has_msr_pv_eoi_en) {
-            kvm_msr_entry_set(&msrs[n++], MSR_KVM_PV_EOI_EN,
-                              env->pv_eoi_en_msr);
+            kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, env->pv_eoi_en_msr);
         }
         if (has_msr_kvm_steal_time) {
-            kvm_msr_entry_set(&msrs[n++], MSR_KVM_STEAL_TIME,
-                              env->steal_time_msr);
+            kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, env->steal_time_msr);
         }
         if (has_msr_architectural_pmu) {
             /* Stop the counter.  */
-            kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
-            kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_GLOBAL_CTRL, 0);
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0);
 
             /* Set the counter values.  */
             for (i = 0; i < MAX_FIXED_COUNTERS; i++) {
-                kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_FIXED_CTR0 + i,
+                kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i,
                                   env->msr_fixed_counters[i]);
             }
             for (i = 0; i < num_architectural_pmu_counters; i++) {
-                kvm_msr_entry_set(&msrs[n++], MSR_P6_PERFCTR0 + i,
+                kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i,
                                   env->msr_gp_counters[i]);
-                kvm_msr_entry_set(&msrs[n++], MSR_P6_EVNTSEL0 + i,
+                kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i,
                                   env->msr_gp_evtsel[i]);
             }
-            kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_GLOBAL_STATUS,
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS,
                               env->msr_global_status);
-            kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_GLOBAL_OVF_CTRL,
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL,
                               env->msr_global_ovf_ctrl);
 
             /* Now start the PMU.  */
-            kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_FIXED_CTR_CTRL,
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL,
                               env->msr_fixed_ctr_ctrl);
-            kvm_msr_entry_set(&msrs[n++], MSR_CORE_PERF_GLOBAL_CTRL,
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL,
                               env->msr_global_ctrl);
         }
         if (has_msr_hv_hypercall) {
-            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_GUEST_OS_ID,
+            kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID,
                               env->msr_hv_guest_os_id);
-            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_HYPERCALL,
+            kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL,
                               env->msr_hv_hypercall);
         }
         if (has_msr_hv_vapic) {
-            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_APIC_ASSIST_PAGE,
+            kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE,
                               env->msr_hv_vapic);
         }
         if (has_msr_hv_tsc) {
-            kvm_msr_entry_set(&msrs[n++], HV_X64_MSR_REFERENCE_TSC,
-                              env->msr_hv_tsc);
+            kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC, env->msr_hv_tsc);
+        }
+        if (has_msr_hv_crash) {
+            int j;
+
+            for (j = 0; j < HV_X64_MSR_CRASH_PARAMS; j++)
+                kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j,
+                                  env->msr_hv_crash_params[j]);
+
+            kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_CTL,
+                              HV_X64_MSR_CRASH_CTL_NOTIFY);
+        }
+        if (has_msr_hv_runtime) {
+            kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, env->msr_hv_runtime);
+        }
+        if (cpu->hyperv_synic) {
+            int j;
+
+            kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL,
+                              env->msr_hv_synic_control);
+            kvm_msr_entry_add(cpu, HV_X64_MSR_SVERSION,
+                              env->msr_hv_synic_version);
+            kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP,
+                              env->msr_hv_synic_evt_page);
+            kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP,
+                              env->msr_hv_synic_msg_page);
+
+            for (j = 0; j < ARRAY_SIZE(env->msr_hv_synic_sint); j++) {
+                kvm_msr_entry_add(cpu, HV_X64_MSR_SINT0 + j,
+                                  env->msr_hv_synic_sint[j]);
+            }
+        }
+        if (has_msr_hv_stimer) {
+            int j;
+
+            for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_config); j++) {
+                kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_CONFIG + j * 2,
+                                env->msr_hv_stimer_config[j]);
+            }
+
+            for (j = 0; j < ARRAY_SIZE(env->msr_hv_stimer_count); j++) {
+                kvm_msr_entry_add(cpu, HV_X64_MSR_STIMER0_COUNT + j * 2,
+                                env->msr_hv_stimer_count[j]);
+            }
         }
         if (has_msr_mtrr) {
-            kvm_msr_entry_set(&msrs[n++], MSR_MTRRdefType, env->mtrr_deftype);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix64K_00000, env->mtrr_fixed[0]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix16K_80000, env->mtrr_fixed[1]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix16K_A0000, env->mtrr_fixed[2]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_C0000, env->mtrr_fixed[3]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_C8000, env->mtrr_fixed[4]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_D0000, env->mtrr_fixed[5]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_D8000, env->mtrr_fixed[6]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_E0000, env->mtrr_fixed[7]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_E8000, env->mtrr_fixed[8]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_F0000, env->mtrr_fixed[9]);
-            kvm_msr_entry_set(&msrs[n++],
-                              MSR_MTRRfix4K_F8000, env->mtrr_fixed[10]);
+            uint64_t phys_mask = MAKE_64BIT_MASK(0, cpu->phys_bits);
+
+            kvm_msr_entry_add(cpu, MSR_MTRRdefType, env->mtrr_deftype);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, env->mtrr_fixed[0]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, env->mtrr_fixed[1]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, env->mtrr_fixed[2]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, env->mtrr_fixed[3]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, env->mtrr_fixed[4]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, env->mtrr_fixed[5]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, env->mtrr_fixed[6]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, env->mtrr_fixed[7]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, env->mtrr_fixed[8]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, env->mtrr_fixed[9]);
+            kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, env->mtrr_fixed[10]);
             for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
-                kvm_msr_entry_set(&msrs[n++],
-                                  MSR_MTRRphysBase(i), env->mtrr_var[i].base);
-                kvm_msr_entry_set(&msrs[n++],
-                                  MSR_MTRRphysMask(i), env->mtrr_var[i].mask);
+                /* The CPU GPs if we write to a bit above the physical limit of
+                 * the host CPU (and KVM emulates that)
+                 */
+                uint64_t mask = env->mtrr_var[i].mask;
+                mask &= phys_mask;
+
+                kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i),
+                                  env->mtrr_var[i].base);
+                kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), mask);
             }
         }
 
@@ -1332,17 +1759,23 @@
     if (env->mcg_cap) {
         int i;
 
-        kvm_msr_entry_set(&msrs[n++], MSR_MCG_STATUS, env->mcg_status);
-        kvm_msr_entry_set(&msrs[n++], MSR_MCG_CTL, env->mcg_ctl);
+        kvm_msr_entry_add(cpu, MSR_MCG_STATUS, env->mcg_status);
+        kvm_msr_entry_add(cpu, MSR_MCG_CTL, env->mcg_ctl);
+        if (has_msr_mcg_ext_ctl) {
+            kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, env->mcg_ext_ctl);
+        }
         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
-            kvm_msr_entry_set(&msrs[n++], MSR_MC0_CTL + i, env->mce_banks[i]);
+            kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, env->mce_banks[i]);
         }
     }
 
-    msr_data.info.nmsrs = n;
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, cpu->kvm_msr_buf);
+    if (ret < 0) {
+        return ret;
+    }
 
-    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_MSRS, &msr_data);
-
+    assert(ret == cpu->kvm_msr_buf->nmsrs);
+    return 0;
 }
 
 
@@ -1367,7 +1800,10 @@
         env->fptags[i] = !((fpu.ftwx >> i) & 1);
     }
     memcpy(env->fpregs, fpu.fpr, sizeof env->fpregs);
-    memcpy(env->xmm_regs, fpu.xmm, sizeof env->xmm_regs);
+    for (i = 0; i < CPU_NB_REGS; i++) {
+        env->xmm_regs[i].ZMM_Q(0) = ldq_p(&fpu.xmm[i][0]);
+        env->xmm_regs[i].ZMM_Q(1) = ldq_p(&fpu.xmm[i][8]);
+    }
     env->mxcsr = fpu.mxcsr;
 
     return 0;
@@ -1376,11 +1812,11 @@
 static int kvm_get_xsave(X86CPU *cpu)
 {
     CPUX86State *env = &cpu->env;
-    struct kvm_xsave* xsave = env->kvm_xsave_buf;
+    X86XSaveArea *xsave = env->kvm_xsave_buf;
     int ret, i;
     uint16_t cwd, swd, twd;
 
-    if (!kvm_has_xsave()) {
+    if (!has_xsave) {
         return kvm_get_fpu(cpu);
     }
 
@@ -1389,37 +1825,46 @@
         return ret;
     }
 
-    cwd = (uint16_t)xsave->region[XSAVE_FCW_FSW];
-    swd = (uint16_t)(xsave->region[XSAVE_FCW_FSW] >> 16);
-    twd = (uint16_t)xsave->region[XSAVE_FTW_FOP];
-    env->fpop = (uint16_t)(xsave->region[XSAVE_FTW_FOP] >> 16);
+    cwd = xsave->legacy.fcw;
+    swd = xsave->legacy.fsw;
+    twd = xsave->legacy.ftw;
+    env->fpop = xsave->legacy.fpop;
     env->fpstt = (swd >> 11) & 7;
     env->fpus = swd;
     env->fpuc = cwd;
     for (i = 0; i < 8; ++i) {
         env->fptags[i] = !((twd >> i) & 1);
     }
-    memcpy(&env->fpip, &xsave->region[XSAVE_CWD_RIP], sizeof(env->fpip));
-    memcpy(&env->fpdp, &xsave->region[XSAVE_CWD_RDP], sizeof(env->fpdp));
-    env->mxcsr = xsave->region[XSAVE_MXCSR];
-    memcpy(env->fpregs, &xsave->region[XSAVE_ST_SPACE],
+    env->fpip = xsave->legacy.fpip;
+    env->fpdp = xsave->legacy.fpdp;
+    env->mxcsr = xsave->legacy.mxcsr;
+    memcpy(env->fpregs, &xsave->legacy.fpregs,
             sizeof env->fpregs);
-    memcpy(env->xmm_regs, &xsave->region[XSAVE_XMM_SPACE],
-            sizeof env->xmm_regs);
-    env->xstate_bv = *(uint64_t *)&xsave->region[XSAVE_XSTATE_BV];
-    memcpy(env->ymmh_regs, &xsave->region[XSAVE_YMMH_SPACE],
-            sizeof env->ymmh_regs);
-    memcpy(env->bnd_regs, &xsave->region[XSAVE_BNDREGS],
+    env->xstate_bv = xsave->header.xstate_bv;
+    memcpy(env->bnd_regs, &xsave->bndreg_state.bnd_regs,
             sizeof env->bnd_regs);
-    memcpy(&env->bndcs_regs, &xsave->region[XSAVE_BNDCSR],
-            sizeof(env->bndcs_regs));
-    memcpy(env->opmask_regs, &xsave->region[XSAVE_OPMASK],
+    env->bndcs_regs = xsave->bndcsr_state.bndcsr;
+    memcpy(env->opmask_regs, &xsave->opmask_state.opmask_regs,
             sizeof env->opmask_regs);
-    memcpy(env->zmmh_regs, &xsave->region[XSAVE_ZMM_Hi256],
-            sizeof env->zmmh_regs);
+
+    for (i = 0; i < CPU_NB_REGS; i++) {
+        uint8_t *xmm = xsave->legacy.xmm_regs[i];
+        uint8_t *ymmh = xsave->avx_state.ymmh[i];
+        uint8_t *zmmh = xsave->zmm_hi256_state.zmm_hi256[i];
+        env->xmm_regs[i].ZMM_Q(0) = ldq_p(xmm);
+        env->xmm_regs[i].ZMM_Q(1) = ldq_p(xmm+8);
+        env->xmm_regs[i].ZMM_Q(2) = ldq_p(ymmh);
+        env->xmm_regs[i].ZMM_Q(3) = ldq_p(ymmh+8);
+        env->xmm_regs[i].ZMM_Q(4) = ldq_p(zmmh);
+        env->xmm_regs[i].ZMM_Q(5) = ldq_p(zmmh+8);
+        env->xmm_regs[i].ZMM_Q(6) = ldq_p(zmmh+16);
+        env->xmm_regs[i].ZMM_Q(7) = ldq_p(zmmh+24);
+    }
+
 #ifdef TARGET_X86_64
-    memcpy(env->hi16_zmm_regs, &xsave->region[XSAVE_Hi16_ZMM],
-            sizeof env->hi16_zmm_regs);
+    memcpy(&env->xmm_regs[16], &xsave->hi16_zmm_state.hi16_zmm,
+           16 * sizeof env->xmm_regs[16]);
+    memcpy(&env->pkru, &xsave->pkru_state, sizeof env->pkru);
 #endif
     return 0;
 }
@@ -1430,7 +1875,7 @@
     int i, ret;
     struct kvm_xcrs xcrs;
 
-    if (!kvm_has_xcrs()) {
+    if (!has_xcrs) {
         return 0;
     }
 
@@ -1502,13 +1947,16 @@
        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
 
-    hflags = (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
+    hflags = env->hflags & HFLAG_COPY_MASK;
+    hflags |= (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
     hflags |= (env->eflags & (HF_TF_MASK | HF_VM_MASK | HF_IOPL_MASK));
-    hflags |= (env->cr[4] & CR4_OSFXSR_MASK) <<
-                (HF_OSFXSR_SHIFT - CR4_OSFXSR_SHIFT);
+
+    if (env->cr[4] & CR4_OSFXSR_MASK) {
+        hflags |= HF_OSFXSR_MASK;
+    }
 
     if (env->efer & MSR_EFER_LMA) {
         hflags |= HF_LMA_MASK;
@@ -1529,7 +1977,7 @@
                         env->segs[R_SS].base) != 0) << HF_ADDSEG_SHIFT;
         }
     }
-    env->hflags = (env->hflags & HFLAG_COPY_MASK) | hflags;
+    env->hflags = hflags;
 
     return 0;
 }
@@ -1537,121 +1985,185 @@
 static int kvm_get_msrs(X86CPU *cpu)
 {
     CPUX86State *env = &cpu->env;
-    struct {
-        struct kvm_msrs info;
-        struct kvm_msr_entry entries[150];
-    } msr_data;
-    struct kvm_msr_entry *msrs = msr_data.entries;
-    int ret, i, n;
+    struct kvm_msr_entry *msrs = cpu->kvm_msr_buf->entries;
+    int ret, i;
+    uint64_t mtrr_top_bits;
 
-    n = 0;
-    msrs[n++].index = MSR_IA32_SYSENTER_CS;
-    msrs[n++].index = MSR_IA32_SYSENTER_ESP;
-    msrs[n++].index = MSR_IA32_SYSENTER_EIP;
-    msrs[n++].index = MSR_PAT;
+    kvm_msr_buf_reset(cpu);
+
+    kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_CS, 0);
+    kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_ESP, 0);
+    kvm_msr_entry_add(cpu, MSR_IA32_SYSENTER_EIP, 0);
+    kvm_msr_entry_add(cpu, MSR_PAT, 0);
     if (has_msr_star) {
-        msrs[n++].index = MSR_STAR;
+        kvm_msr_entry_add(cpu, MSR_STAR, 0);
     }
     if (has_msr_hsave_pa) {
-        msrs[n++].index = MSR_VM_HSAVE_PA;
+        kvm_msr_entry_add(cpu, MSR_VM_HSAVE_PA, 0);
+    }
+    if (has_msr_tsc_aux) {
+        kvm_msr_entry_add(cpu, MSR_TSC_AUX, 0);
     }
     if (has_msr_tsc_adjust) {
-        msrs[n++].index = MSR_TSC_ADJUST;
+        kvm_msr_entry_add(cpu, MSR_TSC_ADJUST, 0);
     }
     if (has_msr_tsc_deadline) {
-        msrs[n++].index = MSR_IA32_TSCDEADLINE;
+        kvm_msr_entry_add(cpu, MSR_IA32_TSCDEADLINE, 0);
     }
     if (has_msr_misc_enable) {
-        msrs[n++].index = MSR_IA32_MISC_ENABLE;
+        kvm_msr_entry_add(cpu, MSR_IA32_MISC_ENABLE, 0);
+    }
+    if (has_msr_smbase) {
+        kvm_msr_entry_add(cpu, MSR_IA32_SMBASE, 0);
     }
     if (has_msr_feature_control) {
-        msrs[n++].index = MSR_IA32_FEATURE_CONTROL;
+        kvm_msr_entry_add(cpu, MSR_IA32_FEATURE_CONTROL, 0);
     }
     if (has_msr_bndcfgs) {
-        msrs[n++].index = MSR_IA32_BNDCFGS;
+        kvm_msr_entry_add(cpu, MSR_IA32_BNDCFGS, 0);
+    }
+    if (has_msr_xss) {
+        kvm_msr_entry_add(cpu, MSR_IA32_XSS, 0);
     }
 
+
     if (!env->tsc_valid) {
-        msrs[n++].index = MSR_IA32_TSC;
+        kvm_msr_entry_add(cpu, MSR_IA32_TSC, 0);
         env->tsc_valid = !runstate_is_running();
     }
 
 #ifdef TARGET_X86_64
     if (lm_capable_kernel) {
-        msrs[n++].index = MSR_CSTAR;
-        msrs[n++].index = MSR_KERNELGSBASE;
-        msrs[n++].index = MSR_FMASK;
-        msrs[n++].index = MSR_LSTAR;
+        kvm_msr_entry_add(cpu, MSR_CSTAR, 0);
+        kvm_msr_entry_add(cpu, MSR_KERNELGSBASE, 0);
+        kvm_msr_entry_add(cpu, MSR_FMASK, 0);
+        kvm_msr_entry_add(cpu, MSR_LSTAR, 0);
     }
 #endif
-    msrs[n++].index = MSR_KVM_SYSTEM_TIME;
-    msrs[n++].index = MSR_KVM_WALL_CLOCK;
+    kvm_msr_entry_add(cpu, MSR_KVM_SYSTEM_TIME, 0);
+    kvm_msr_entry_add(cpu, MSR_KVM_WALL_CLOCK, 0);
     if (has_msr_async_pf_en) {
-        msrs[n++].index = MSR_KVM_ASYNC_PF_EN;
+        kvm_msr_entry_add(cpu, MSR_KVM_ASYNC_PF_EN, 0);
     }
     if (has_msr_pv_eoi_en) {
-        msrs[n++].index = MSR_KVM_PV_EOI_EN;
+        kvm_msr_entry_add(cpu, MSR_KVM_PV_EOI_EN, 0);
     }
     if (has_msr_kvm_steal_time) {
-        msrs[n++].index = MSR_KVM_STEAL_TIME;
+        kvm_msr_entry_add(cpu, MSR_KVM_STEAL_TIME, 0);
     }
     if (has_msr_architectural_pmu) {
-        msrs[n++].index = MSR_CORE_PERF_FIXED_CTR_CTRL;
-        msrs[n++].index = MSR_CORE_PERF_GLOBAL_CTRL;
-        msrs[n++].index = MSR_CORE_PERF_GLOBAL_STATUS;
-        msrs[n++].index = MSR_CORE_PERF_GLOBAL_OVF_CTRL;
+        kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR_CTRL, 0);
+        kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_CTRL, 0);
+        kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_STATUS, 0);
+        kvm_msr_entry_add(cpu, MSR_CORE_PERF_GLOBAL_OVF_CTRL, 0);
         for (i = 0; i < MAX_FIXED_COUNTERS; i++) {
-            msrs[n++].index = MSR_CORE_PERF_FIXED_CTR0 + i;
+            kvm_msr_entry_add(cpu, MSR_CORE_PERF_FIXED_CTR0 + i, 0);
         }
         for (i = 0; i < num_architectural_pmu_counters; i++) {
-            msrs[n++].index = MSR_P6_PERFCTR0 + i;
-            msrs[n++].index = MSR_P6_EVNTSEL0 + i;
+            kvm_msr_entry_add(cpu, MSR_P6_PERFCTR0 + i, 0);
+            kvm_msr_entry_add(cpu, MSR_P6_EVNTSEL0 + i, 0);
         }
     }
 
     if (env->mcg_cap) {
-        msrs[n++].index = MSR_MCG_STATUS;
-        msrs[n++].index = MSR_MCG_CTL;
+        kvm_msr_entry_add(cpu, MSR_MCG_STATUS, 0);
+        kvm_msr_entry_add(cpu, MSR_MCG_CTL, 0);
+        if (has_msr_mcg_ext_ctl) {
+            kvm_msr_entry_add(cpu, MSR_MCG_EXT_CTL, 0);
+        }
         for (i = 0; i < (env->mcg_cap & 0xff) * 4; i++) {
-            msrs[n++].index = MSR_MC0_CTL + i;
+            kvm_msr_entry_add(cpu, MSR_MC0_CTL + i, 0);
         }
     }
 
     if (has_msr_hv_hypercall) {
-        msrs[n++].index = HV_X64_MSR_HYPERCALL;
-        msrs[n++].index = HV_X64_MSR_GUEST_OS_ID;
+        kvm_msr_entry_add(cpu, HV_X64_MSR_HYPERCALL, 0);
+        kvm_msr_entry_add(cpu, HV_X64_MSR_GUEST_OS_ID, 0);
     }
     if (has_msr_hv_vapic) {
-        msrs[n++].index = HV_X64_MSR_APIC_ASSIST_PAGE;
+        kvm_msr_entry_add(cpu, HV_X64_MSR_APIC_ASSIST_PAGE, 0);
     }
     if (has_msr_hv_tsc) {
-        msrs[n++].index = HV_X64_MSR_REFERENCE_TSC;
+        kvm_msr_entry_add(cpu, HV_X64_MSR_REFERENCE_TSC, 0);
+    }
+    if (has_msr_hv_crash) {
+        int j;
+
+        for (j = 0; j < HV_X64_MSR_CRASH_PARAMS; j++) {
+            kvm_msr_entry_add(cpu, HV_X64_MSR_CRASH_P0 + j, 0);
+        }
+    }
+    if (has_msr_hv_runtime) {
+        kvm_msr_entry_add(cpu, HV_X64_MSR_VP_RUNTIME, 0);
+    }
+    if (cpu->hyperv_synic) {
+        uint32_t msr;
+
+        kvm_msr_entry_add(cpu, HV_X64_MSR_SCONTROL, 0);
+        kvm_msr_entry_add(cpu, HV_X64_MSR_SVERSION, 0);
+        kvm_msr_entry_add(cpu, HV_X64_MSR_SIEFP, 0);
+        kvm_msr_entry_add(cpu, HV_X64_MSR_SIMP, 0);
+        for (msr = HV_X64_MSR_SINT0; msr <= HV_X64_MSR_SINT15; msr++) {
+            kvm_msr_entry_add(cpu, msr, 0);
+        }
+    }
+    if (has_msr_hv_stimer) {
+        uint32_t msr;
+
+        for (msr = HV_X64_MSR_STIMER0_CONFIG; msr <= HV_X64_MSR_STIMER3_COUNT;
+             msr++) {
+            kvm_msr_entry_add(cpu, msr, 0);
+        }
     }
     if (has_msr_mtrr) {
-        msrs[n++].index = MSR_MTRRdefType;
-        msrs[n++].index = MSR_MTRRfix64K_00000;
-        msrs[n++].index = MSR_MTRRfix16K_80000;
-        msrs[n++].index = MSR_MTRRfix16K_A0000;
-        msrs[n++].index = MSR_MTRRfix4K_C0000;
-        msrs[n++].index = MSR_MTRRfix4K_C8000;
-        msrs[n++].index = MSR_MTRRfix4K_D0000;
-        msrs[n++].index = MSR_MTRRfix4K_D8000;
-        msrs[n++].index = MSR_MTRRfix4K_E0000;
-        msrs[n++].index = MSR_MTRRfix4K_E8000;
-        msrs[n++].index = MSR_MTRRfix4K_F0000;
-        msrs[n++].index = MSR_MTRRfix4K_F8000;
+        kvm_msr_entry_add(cpu, MSR_MTRRdefType, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix64K_00000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix16K_80000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix16K_A0000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C0000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_C8000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D0000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_D8000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E0000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_E8000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F0000, 0);
+        kvm_msr_entry_add(cpu, MSR_MTRRfix4K_F8000, 0);
         for (i = 0; i < MSR_MTRRcap_VCNT; i++) {
-            msrs[n++].index = MSR_MTRRphysBase(i);
-            msrs[n++].index = MSR_MTRRphysMask(i);
+            kvm_msr_entry_add(cpu, MSR_MTRRphysBase(i), 0);
+            kvm_msr_entry_add(cpu, MSR_MTRRphysMask(i), 0);
         }
     }
 
-    msr_data.info.nmsrs = n;
-    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, &msr_data);
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_MSRS, cpu->kvm_msr_buf);
     if (ret < 0) {
         return ret;
     }
 
+    assert(ret == cpu->kvm_msr_buf->nmsrs);
+    /*
+     * MTRR masks: Each mask consists of 5 parts
+     * a  10..0: must be zero
+     * b  11   : valid bit
+     * c n-1.12: actual mask bits
+     * d  51..n: reserved must be zero
+     * e  63.52: reserved must be zero
+     *
+     * 'n' is the number of physical bits supported by the CPU and is
+     * apparently always <= 52.   We know our 'n' but don't know what
+     * the destinations 'n' is; it might be smaller, in which case
+     * it masks (c) on loading. It might be larger, in which case
+     * we fill 'd' so that d..c is consistent irrespetive of the 'n'
+     * we're migrating to.
+     */
+
+    if (cpu->fill_mtrr_mask) {
+        QEMU_BUILD_BUG_ON(TARGET_PHYS_ADDR_SPACE_BITS > 52);
+        assert(cpu->phys_bits <= TARGET_PHYS_ADDR_SPACE_BITS);
+        mtrr_top_bits = MAKE_64BIT_MASK(cpu->phys_bits, 52 - cpu->phys_bits);
+    } else {
+        mtrr_top_bits = 0;
+    }
+
     for (i = 0; i < ret; i++) {
         uint32_t index = msrs[i].index;
         switch (index) {
@@ -1687,6 +2199,9 @@
         case MSR_IA32_TSC:
             env->tsc = msrs[i].data;
             break;
+        case MSR_TSC_AUX:
+            env->tsc_aux = msrs[i].data;
+            break;
         case MSR_TSC_ADJUST:
             env->tsc_adjust = msrs[i].data;
             break;
@@ -1708,15 +2223,24 @@
         case MSR_MCG_CTL:
             env->mcg_ctl = msrs[i].data;
             break;
+        case MSR_MCG_EXT_CTL:
+            env->mcg_ext_ctl = msrs[i].data;
+            break;
         case MSR_IA32_MISC_ENABLE:
             env->msr_ia32_misc_enable = msrs[i].data;
             break;
+        case MSR_IA32_SMBASE:
+            env->smbase = msrs[i].data;
+            break;
         case MSR_IA32_FEATURE_CONTROL:
             env->msr_ia32_feature_control = msrs[i].data;
             break;
         case MSR_IA32_BNDCFGS:
             env->msr_bndcfgs = msrs[i].data;
             break;
+        case MSR_IA32_XSS:
+            env->xss = msrs[i].data;
+            break;
         default:
             if (msrs[i].index >= MSR_MC0_CTL &&
                 msrs[i].index < MSR_MC0_CTL + (env->mcg_cap & 0xff) * 4) {
@@ -1765,6 +2289,41 @@
         case HV_X64_MSR_REFERENCE_TSC:
             env->msr_hv_tsc = msrs[i].data;
             break;
+        case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4:
+            env->msr_hv_crash_params[index - HV_X64_MSR_CRASH_P0] = msrs[i].data;
+            break;
+        case HV_X64_MSR_VP_RUNTIME:
+            env->msr_hv_runtime = msrs[i].data;
+            break;
+        case HV_X64_MSR_SCONTROL:
+            env->msr_hv_synic_control = msrs[i].data;
+            break;
+        case HV_X64_MSR_SVERSION:
+            env->msr_hv_synic_version = msrs[i].data;
+            break;
+        case HV_X64_MSR_SIEFP:
+            env->msr_hv_synic_evt_page = msrs[i].data;
+            break;
+        case HV_X64_MSR_SIMP:
+            env->msr_hv_synic_msg_page = msrs[i].data;
+            break;
+        case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15:
+            env->msr_hv_synic_sint[index - HV_X64_MSR_SINT0] = msrs[i].data;
+            break;
+        case HV_X64_MSR_STIMER0_CONFIG:
+        case HV_X64_MSR_STIMER1_CONFIG:
+        case HV_X64_MSR_STIMER2_CONFIG:
+        case HV_X64_MSR_STIMER3_CONFIG:
+            env->msr_hv_stimer_config[(index - HV_X64_MSR_STIMER0_CONFIG)/2] =
+                                msrs[i].data;
+            break;
+        case HV_X64_MSR_STIMER0_COUNT:
+        case HV_X64_MSR_STIMER1_COUNT:
+        case HV_X64_MSR_STIMER2_COUNT:
+        case HV_X64_MSR_STIMER3_COUNT:
+            env->msr_hv_stimer_count[(index - HV_X64_MSR_STIMER0_COUNT)/2] =
+                                msrs[i].data;
+            break;
         case MSR_MTRRdefType:
             env->mtrr_deftype = msrs[i].data;
             break;
@@ -1803,7 +2362,8 @@
             break;
         case MSR_MTRRphysBase(0) ... MSR_MTRRphysMask(MSR_MTRRcap_VCNT - 1):
             if (index & 1) {
-                env->mtrr_var[MSR_MTRRphysIndex(index)].mask = msrs[i].data;
+                env->mtrr_var[MSR_MTRRphysIndex(index)].mask = msrs[i].data |
+                                                               mtrr_top_bits;
             } else {
                 env->mtrr_var[MSR_MTRRphysIndex(index)].base = msrs[i].data;
             }
@@ -1871,8 +2431,9 @@
 
 static int kvm_put_vcpu_events(X86CPU *cpu, int level)
 {
+    CPUState *cs = CPU(cpu);
     CPUX86State *env = &cpu->env;
-    struct kvm_vcpu_events events;
+    struct kvm_vcpu_events events = {};
 
     if (!kvm_has_vcpu_events()) {
         return 0;
@@ -1895,6 +2456,24 @@
 
     events.sipi_vector = env->sipi_vector;
 
+    if (has_msr_smbase) {
+        events.smi.smm = !!(env->hflags & HF_SMM_MASK);
+        events.smi.smm_inside_nmi = !!(env->hflags2 & HF2_SMM_INSIDE_NMI_MASK);
+        if (kvm_irqchip_in_kernel()) {
+            /* As soon as these are moved to the kernel, remove them
+             * from cs->interrupt_request.
+             */
+            events.smi.pending = cs->interrupt_request & CPU_INTERRUPT_SMI;
+            events.smi.latched_init = cs->interrupt_request & CPU_INTERRUPT_INIT;
+            cs->interrupt_request &= ~(CPU_INTERRUPT_INIT | CPU_INTERRUPT_SMI);
+        } else {
+            /* Keep these in cs->interrupt_request.  */
+            events.smi.pending = 0;
+            events.smi.latched_init = 0;
+        }
+        events.flags |= KVM_VCPUEVENT_VALID_SMM;
+    }
+
     events.flags = 0;
     if (level >= KVM_PUT_RESET_STATE) {
         events.flags |=
@@ -1914,6 +2493,7 @@
         return 0;
     }
 
+    memset(&events, 0, sizeof(events));
     ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_VCPU_EVENTS, &events);
     if (ret < 0) {
        return ret;
@@ -1935,6 +2515,29 @@
         env->hflags2 &= ~HF2_NMI_MASK;
     }
 
+    if (events.flags & KVM_VCPUEVENT_VALID_SMM) {
+        if (events.smi.smm) {
+            env->hflags |= HF_SMM_MASK;
+        } else {
+            env->hflags &= ~HF_SMM_MASK;
+        }
+        if (events.smi.pending) {
+            cpu_interrupt(CPU(cpu), CPU_INTERRUPT_SMI);
+        } else {
+            cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_SMI);
+        }
+        if (events.smi.smm_inside_nmi) {
+            env->hflags2 |= HF2_SMM_INSIDE_NMI_MASK;
+        } else {
+            env->hflags2 &= ~HF2_SMM_INSIDE_NMI_MASK;
+        }
+        if (events.smi.latched_init) {
+            cpu_interrupt(CPU(cpu), CPU_INTERRUPT_INIT);
+        } else {
+            cpu_reset_interrupt(CPU(cpu), CPU_INTERRUPT_INIT);
+        }
+    }
+
     env->sipi_vector = events.sipi_vector;
 
     return 0;
@@ -2021,13 +2624,22 @@
 
     assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
-    if (level >= KVM_PUT_RESET_STATE && has_msr_feature_control) {
+    if (level >= KVM_PUT_RESET_STATE) {
         ret = kvm_put_msr_feature_control(x86_cpu);
         if (ret < 0) {
             return ret;
         }
     }
 
+    if (level == KVM_PUT_FULL_STATE) {
+        /* We don't check for kvm_arch_set_tsc_khz() errors here,
+         * because TSC frequency mismatch shouldn't abort migration,
+         * unless the user explicitly asked for a more strict TSC
+         * setting (e.g. using an explicit "tsc-freq" option).
+         */
+        kvm_arch_set_tsc_khz(cpu);
+    }
+
     ret = kvm_getput_regs(x86_cpu, 1);
     if (ret < 0) {
         return ret;
@@ -2094,41 +2706,44 @@
 
     ret = kvm_getput_regs(cpu, 0);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_xsave(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_xcrs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_sregs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_msrs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_mp_state(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_apic(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_vcpu_events(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
     ret = kvm_get_debugregs(cpu);
     if (ret < 0) {
-        return ret;
+        goto out;
     }
-    return 0;
+    ret = 0;
+ out:
+    cpu_sync_bndcs_hflags(&cpu->env);
+    return ret;
 }
 
 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
@@ -2138,14 +2753,33 @@
     int ret;
 
     /* Inject NMI */
-    if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
-        cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
-        DPRINTF("injected NMI\n");
-        ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
-        if (ret < 0) {
-            fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
-                    strerror(-ret));
+    if (cpu->interrupt_request & (CPU_INTERRUPT_NMI | CPU_INTERRUPT_SMI)) {
+        if (cpu->interrupt_request & CPU_INTERRUPT_NMI) {
+            qemu_mutex_lock_iothread();
+            cpu->interrupt_request &= ~CPU_INTERRUPT_NMI;
+            qemu_mutex_unlock_iothread();
+            DPRINTF("injected NMI\n");
+            ret = kvm_vcpu_ioctl(cpu, KVM_NMI);
+            if (ret < 0) {
+                fprintf(stderr, "KVM: injection failed, NMI lost (%s)\n",
+                        strerror(-ret));
+            }
         }
+        if (cpu->interrupt_request & CPU_INTERRUPT_SMI) {
+            qemu_mutex_lock_iothread();
+            cpu->interrupt_request &= ~CPU_INTERRUPT_SMI;
+            qemu_mutex_unlock_iothread();
+            DPRINTF("injected SMI\n");
+            ret = kvm_vcpu_ioctl(cpu, KVM_SMI);
+            if (ret < 0) {
+                fprintf(stderr, "KVM: injection failed, SMI lost (%s)\n",
+                        strerror(-ret));
+            }
+        }
+    }
+
+    if (!kvm_pic_in_kernel()) {
+        qemu_mutex_lock_iothread();
     }
 
     /* Force the VCPU out of its inner loop to process any INIT requests
@@ -2153,10 +2787,16 @@
      * pending TPR access reports.
      */
     if (cpu->interrupt_request & (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) {
-        cpu->exit_request = 1;
+        if ((cpu->interrupt_request & CPU_INTERRUPT_INIT) &&
+            !(env->hflags & HF_SMM_MASK)) {
+            cpu->exit_request = 1;
+        }
+        if (cpu->interrupt_request & CPU_INTERRUPT_TPR) {
+            cpu->exit_request = 1;
+        }
     }
 
-    if (!kvm_irqchip_in_kernel()) {
+    if (!kvm_pic_in_kernel()) {
         /* Try to inject an interrupt if the guest can accept it */
         if (run->ready_for_interrupt_injection &&
             (cpu->interrupt_request & CPU_INTERRUPT_HARD) &&
@@ -2191,21 +2831,38 @@
 
         DPRINTF("setting tpr\n");
         run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state);
+
+        qemu_mutex_unlock_iothread();
     }
 }
 
-void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
+MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
 {
     X86CPU *x86_cpu = X86_CPU(cpu);
     CPUX86State *env = &x86_cpu->env;
 
+    if (run->flags & KVM_RUN_X86_SMM) {
+        env->hflags |= HF_SMM_MASK;
+    } else {
+        env->hflags &= HF_SMM_MASK;
+    }
     if (run->if_flag) {
         env->eflags |= IF_MASK;
     } else {
         env->eflags &= ~IF_MASK;
     }
+
+    /* We need to protect the apic state against concurrent accesses from
+     * different threads in case the userspace irqchip is used. */
+    if (!kvm_irqchip_in_kernel()) {
+        qemu_mutex_lock_iothread();
+    }
     cpu_set_apic_tpr(x86_cpu->apic_state, run->cr8);
     cpu_set_apic_base(x86_cpu->apic_state, run->apic_base);
+    if (!kvm_irqchip_in_kernel()) {
+        qemu_mutex_unlock_iothread();
+    }
+    return cpu_get_mem_attrs(env);
 }
 
 int kvm_arch_process_async_events(CPUState *cs)
@@ -2236,7 +2893,8 @@
         }
     }
 
-    if (cs->interrupt_request & CPU_INTERRUPT_INIT) {
+    if ((cs->interrupt_request & CPU_INTERRUPT_INIT) &&
+        !(env->hflags & HF_SMM_MASK)) {
         kvm_cpu_synchronize_state(cs);
         do_cpu_init(cpu);
     }
@@ -2497,13 +3155,17 @@
     switch (run->exit_reason) {
     case KVM_EXIT_HLT:
         DPRINTF("handle_hlt\n");
+        qemu_mutex_lock_iothread();
         ret = kvm_handle_halt(cpu);
+        qemu_mutex_unlock_iothread();
         break;
     case KVM_EXIT_SET_TPR:
         ret = 0;
         break;
     case KVM_EXIT_TPR_ACCESS:
+        qemu_mutex_lock_iothread();
         ret = kvm_handle_tpr_access(cpu);
+        qemu_mutex_unlock_iothread();
         break;
     case KVM_EXIT_FAIL_ENTRY:
         code = run->fail_entry.hardware_entry_failure_reason;
@@ -2529,7 +3191,16 @@
         break;
     case KVM_EXIT_DEBUG:
         DPRINTF("kvm_exit_debug\n");
+        qemu_mutex_lock_iothread();
         ret = kvm_handle_debug(cpu, &run->debug.arch);
+        qemu_mutex_unlock_iothread();
+        break;
+    case KVM_EXIT_HYPERV:
+        ret = kvm_hv_handle_exit(cpu, &run->hyperv);
+        break;
+    case KVM_EXIT_IOAPIC_EOI:
+        ioapic_eoi_broadcast(run->eoi.vector);
+        ret = 0;
         break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
@@ -2563,9 +3234,40 @@
      * irqchip, so we can use irqfds, and on x86 we know
      * we can use msi via irqfd and GSI routing.
      */
-    kvm_irqfds_allowed = true;
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_routing_allowed = true;
+
+    if (kvm_irqchip_is_split()) {
+        int i;
+
+        /* If the ioapic is in QEMU and the lapics are in KVM, reserve
+           MSI routes for signaling interrupts to the local apics. */
+        for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+            if (kvm_irqchip_add_msi_route(s, 0, NULL) < 0) {
+                error_report("Could not enable split IRQ mode.");
+                exit(1);
+            }
+        }
+    }
+}
+
+int kvm_arch_irqchip_create(MachineState *ms, KVMState *s)
+{
+    int ret;
+    if (machine_kernel_irqchip_split(ms)) {
+        ret = kvm_vm_enable_cap(s, KVM_CAP_SPLIT_IRQCHIP, 0, 24);
+        if (ret) {
+            error_report("Could not enable split irqchip mode: %s",
+                         strerror(-ret));
+            exit(1);
+        } else {
+            DPRINTF("Enabled KVM_CAP_SPLIT_IRQCHIP\n");
+            kvm_split_irqchip = true;
+            return 1;
+        }
+    } else {
+        return 0;
+    }
 }
 
 /* Classic KVM device assignment interface. Will remain x86 only. */
@@ -2707,3 +3409,117 @@
     return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSIX |
                                                 KVM_DEV_IRQ_HOST_MSIX);
 }
+
+int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
+                             uint64_t address, uint32_t data, PCIDevice *dev)
+{
+    X86IOMMUState *iommu = x86_iommu_get_default();
+
+    if (iommu) {
+        int ret;
+        MSIMessage src, dst;
+        X86IOMMUClass *class = X86_IOMMU_GET_CLASS(iommu);
+
+        src.address = route->u.msi.address_hi;
+        src.address <<= VTD_MSI_ADDR_HI_SHIFT;
+        src.address |= route->u.msi.address_lo;
+        src.data = route->u.msi.data;
+
+        ret = class->int_remap(iommu, &src, &dst, dev ? \
+                               pci_requester_id(dev) : \
+                               X86_IOMMU_SID_INVALID);
+        if (ret) {
+            trace_kvm_x86_fixup_msi_error(route->gsi);
+            return 1;
+        }
+
+        route->u.msi.address_hi = dst.address >> VTD_MSI_ADDR_HI_SHIFT;
+        route->u.msi.address_lo = dst.address & VTD_MSI_ADDR_LO_MASK;
+        route->u.msi.data = dst.data;
+    }
+
+    return 0;
+}
+
+typedef struct MSIRouteEntry MSIRouteEntry;
+
+struct MSIRouteEntry {
+    PCIDevice *dev;             /* Device pointer */
+    int vector;                 /* MSI/MSIX vector index */
+    int virq;                   /* Virtual IRQ index */
+    QLIST_ENTRY(MSIRouteEntry) list;
+};
+
+/* List of used GSI routes */
+static QLIST_HEAD(, MSIRouteEntry) msi_route_list = \
+    QLIST_HEAD_INITIALIZER(msi_route_list);
+
+static void kvm_update_msi_routes_all(void *private, bool global,
+                                      uint32_t index, uint32_t mask)
+{
+    int cnt = 0;
+    MSIRouteEntry *entry;
+    MSIMessage msg;
+    /* TODO: explicit route update */
+    QLIST_FOREACH(entry, &msi_route_list, list) {
+        cnt++;
+        msg = pci_get_msi_message(entry->dev, entry->vector);
+        kvm_irqchip_update_msi_route(kvm_state, entry->virq,
+                                     msg, entry->dev);
+    }
+    kvm_irqchip_commit_routes(kvm_state);
+    trace_kvm_x86_update_msi_routes(cnt);
+}
+
+int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
+                                int vector, PCIDevice *dev)
+{
+    static bool notify_list_inited = false;
+    MSIRouteEntry *entry;
+
+    if (!dev) {
+        /* These are (possibly) IOAPIC routes only used for split
+         * kernel irqchip mode, while what we are housekeeping are
+         * PCI devices only. */
+        return 0;
+    }
+
+    entry = g_new0(MSIRouteEntry, 1);
+    entry->dev = dev;
+    entry->vector = vector;
+    entry->virq = route->gsi;
+    QLIST_INSERT_HEAD(&msi_route_list, entry, list);
+
+    trace_kvm_x86_add_msi_route(route->gsi);
+
+    if (!notify_list_inited) {
+        /* For the first time we do add route, add ourselves into
+         * IOMMU's IEC notify list if needed. */
+        X86IOMMUState *iommu = x86_iommu_get_default();
+        if (iommu) {
+            x86_iommu_iec_register_notifier(iommu,
+                                            kvm_update_msi_routes_all,
+                                            NULL);
+        }
+        notify_list_inited = true;
+    }
+    return 0;
+}
+
+int kvm_arch_release_virq_post(int virq)
+{
+    MSIRouteEntry *entry, *next;
+    QLIST_FOREACH_SAFE(entry, &msi_route_list, list, next) {
+        if (entry->virq == virq) {
+            trace_kvm_x86_remove_msi_route(virq);
+            QLIST_REMOVE(entry, list);
+            break;
+        }
+    }
+    return 0;
+}
+
+int kvm_arch_msi_data_to_gsi(uint32_t data)
+{
+    abort();
+}
diff --git a/target-i386/kvm_i386.h b/target-i386/kvm_i386.h
index cac30fd..42b00af 100644
--- a/target-i386/kvm_i386.h
+++ b/target-i386/kvm_i386.h
@@ -13,7 +13,11 @@
 
 #include "sysemu/kvm.h"
 
+#define kvm_apic_in_kernel() (kvm_irqchip_in_kernel())
+
 bool kvm_allows_irq0_override(void);
+bool kvm_has_smm(void);
+void kvm_synchronize_all_tsc(void);
 void kvm_arch_reset_vcpu(X86CPU *cs);
 void kvm_arch_do_init_vcpu(X86CPU *cs);
 
diff --git a/target-i386/machine.c b/target-i386/machine.c
index 1c13b14..71c0e4d 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -1,11 +1,20 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "hw/i386/pc.h"
 #include "hw/isa/isa.h"
+#include "migration/cpu.h"
+#include "exec/exec-all.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "sysemu/kvm.h"
 
+#include "qemu/error-report.h"
+
 static const VMStateDescription vmstate_segment = {
     .name = "segment",
     .version_id = 1,
@@ -36,45 +45,48 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(XMM_Q(0), XMMReg),
-        VMSTATE_UINT64(XMM_Q(1), XMMReg),
+        VMSTATE_UINT64(ZMM_Q(0), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(1), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
-#define VMSTATE_XMM_REGS(_field, _state, _n)                         \
-    VMSTATE_STRUCT_ARRAY(_field, _state, _n, 0, vmstate_xmm_reg, XMMReg)
+#define VMSTATE_XMM_REGS(_field, _state, _start)                         \
+    VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, 0,     \
+                             vmstate_xmm_reg, ZMMReg)
 
-/* YMMH format is the same as XMM */
+/* YMMH format is the same as XMM, but for bits 128-255 */
 static const VMStateDescription vmstate_ymmh_reg = {
     .name = "ymmh_reg",
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(XMM_Q(0), XMMReg),
-        VMSTATE_UINT64(XMM_Q(1), XMMReg),
+        VMSTATE_UINT64(ZMM_Q(2), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(3), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
-#define VMSTATE_YMMH_REGS_VARS(_field, _state, _n, _v)                         \
-    VMSTATE_STRUCT_ARRAY(_field, _state, _n, _v, vmstate_ymmh_reg, XMMReg)
+#define VMSTATE_YMMH_REGS_VARS(_field, _state, _start, _v)               \
+    VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, _v,    \
+                             vmstate_ymmh_reg, ZMMReg)
 
 static const VMStateDescription vmstate_zmmh_reg = {
     .name = "zmmh_reg",
     .version_id = 1,
     .minimum_version_id = 1,
     .fields = (VMStateField[]) {
-        VMSTATE_UINT64(YMM_Q(0), YMMReg),
-        VMSTATE_UINT64(YMM_Q(1), YMMReg),
-        VMSTATE_UINT64(YMM_Q(2), YMMReg),
-        VMSTATE_UINT64(YMM_Q(3), YMMReg),
+        VMSTATE_UINT64(ZMM_Q(4), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(5), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(6), ZMMReg),
+        VMSTATE_UINT64(ZMM_Q(7), ZMMReg),
         VMSTATE_END_OF_LIST()
     }
 };
 
-#define VMSTATE_ZMMH_REGS_VARS(_field, _state, _n)                             \
-    VMSTATE_STRUCT_ARRAY(_field, _state, _n, 0, vmstate_zmmh_reg, YMMReg)
+#define VMSTATE_ZMMH_REGS_VARS(_field, _state, _start)                   \
+    VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, 0,     \
+                             vmstate_zmmh_reg, ZMMReg)
 
 #ifdef TARGET_X86_64
 static const VMStateDescription vmstate_hi16_zmm_reg = {
@@ -94,8 +106,9 @@
     }
 };
 
-#define VMSTATE_Hi16_ZMM_REGS_VARS(_field, _state, _n)                         \
-    VMSTATE_STRUCT_ARRAY(_field, _state, _n, 0, vmstate_hi16_zmm_reg, ZMMReg)
+#define VMSTATE_Hi16_ZMM_REGS_VARS(_field, _state, _start)               \
+    VMSTATE_STRUCT_SUB_ARRAY(_field, _state, _start, CPU_NB_REGS, 0,     \
+                             vmstate_hi16_zmm_reg, ZMMReg)
 #endif
 
 static const VMStateDescription vmstate_bnd_regs = {
@@ -327,6 +340,13 @@
     CPUX86State *env = &cpu->env;
     int i;
 
+    if (env->tsc_khz && env->user_tsc_khz &&
+        env->tsc_khz != env->user_tsc_khz) {
+        error_report("Mismatch between user-specified TSC frequency and "
+                     "migrated TSC frequency");
+        return -EINVAL;
+    }
+
     /*
      * Real mode guest segments register DPL should be zero.
      * Older KVM version were setting it wrongly.
@@ -363,11 +383,18 @@
 
     cpu_breakpoint_remove_all(cs, BP_CPU);
     cpu_watchpoint_remove_all(cs, BP_CPU);
-    for (i = 0; i < DR7_MAX_BP; i++) {
-        hw_breakpoint_insert(env, i);
+    {
+        /* Indicate all breakpoints disabled, as they are, then
+           let the helper re-enable them.  */
+        target_ulong dr7 = env->dr[7];
+        env->dr[7] = dr7 & ~(DR7_GLOBAL_BP_MASK | DR7_LOCAL_BP_MASK);
+        cpu_x86_update_dr7(env, dr7);
     }
     tlb_flush(cs, 1);
 
+    if (tcg_enabled()) {
+        cpu_smm_update(cpu);
+    }
     return 0;
 }
 
@@ -396,6 +423,7 @@
     .name = "cpu/steal_time_msr",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = steal_time_msr_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.steal_time_msr, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -406,6 +434,7 @@
     .name = "cpu/async_pf_msr",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = async_pf_msr_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.async_pf_en_msr, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -416,6 +445,7 @@
     .name = "cpu/async_pv_eoi_msr",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = pv_eoi_msr_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.pv_eoi_en_msr, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -434,6 +464,7 @@
     .name = "cpu/fpop_ip_dp",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fpop_ip_dp_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT16(env.fpop, X86CPU),
         VMSTATE_UINT64(env.fpip, X86CPU),
@@ -454,6 +485,7 @@
     .name = "cpu/msr_tsc_adjust",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = tsc_adjust_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.tsc_adjust, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -472,6 +504,7 @@
     .name = "cpu/msr_tscdeadline",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = tscdeadline_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.tsc_deadline, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -498,6 +531,7 @@
     .name = "cpu/msr_ia32_misc_enable",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = misc_enable_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.msr_ia32_misc_enable, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -508,6 +542,7 @@
     .name = "cpu/msr_ia32_feature_control",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = feature_control_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.msr_ia32_feature_control, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -542,6 +577,7 @@
     .name = "cpu/msr_architectural_pmu",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = pmu_enable_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.msr_fixed_ctr_ctrl, X86CPU),
         VMSTATE_UINT64(env.msr_global_ctrl, X86CPU),
@@ -577,6 +613,7 @@
     .name = "cpu/mpx",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = mpx_needed,
     .fields = (VMStateField[]) {
         VMSTATE_BND_REGS(env.bnd_regs, X86CPU, 4),
         VMSTATE_UINT64(env.bndcs_regs.cfgu, X86CPU),
@@ -598,6 +635,7 @@
     .name = "cpu/msr_hyperv_hypercall",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = hyperv_hypercall_enable_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.msr_hv_guest_os_id, X86CPU),
         VMSTATE_UINT64(env.msr_hv_hypercall, X86CPU),
@@ -617,6 +655,7 @@
     .name = "cpu/msr_hyperv_vapic",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = hyperv_vapic_enable_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.msr_hv_vapic, X86CPU),
         VMSTATE_END_OF_LIST()
@@ -635,12 +674,122 @@
     .name = "cpu/msr_hyperv_time",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = hyperv_time_enable_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64(env.msr_hv_tsc, X86CPU),
         VMSTATE_END_OF_LIST()
     }
 };
 
+static bool hyperv_crash_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < HV_X64_MSR_CRASH_PARAMS; i++) {
+        if (env->msr_hv_crash_params[i]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_crash = {
+    .name = "cpu/msr_hyperv_crash",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_crash_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.msr_hv_crash_params,
+                             X86CPU, HV_X64_MSR_CRASH_PARAMS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_runtime_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->msr_hv_runtime != 0;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_runtime = {
+    .name = "cpu/msr_hyperv_runtime",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_runtime_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.msr_hv_runtime, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_synic_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    if (env->msr_hv_synic_control != 0 ||
+        env->msr_hv_synic_evt_page != 0 ||
+        env->msr_hv_synic_msg_page != 0) {
+        return true;
+    }
+
+    for (i = 0; i < ARRAY_SIZE(env->msr_hv_synic_sint); i++) {
+        if (env->msr_hv_synic_sint[i] != 0) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_synic = {
+    .name = "cpu/msr_hyperv_synic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_synic_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.msr_hv_synic_control, X86CPU),
+        VMSTATE_UINT64(env.msr_hv_synic_evt_page, X86CPU),
+        VMSTATE_UINT64(env.msr_hv_synic_msg_page, X86CPU),
+        VMSTATE_UINT64_ARRAY(env.msr_hv_synic_sint, X86CPU,
+                             HV_SYNIC_SINT_COUNT),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool hyperv_stimer_enable_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(env->msr_hv_stimer_config); i++) {
+        if (env->msr_hv_stimer_config[i] || env->msr_hv_stimer_count[i]) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static const VMStateDescription vmstate_msr_hyperv_stimer = {
+    .name = "cpu/msr_hyperv_stimer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyperv_stimer_enable_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64_ARRAY(env.msr_hv_stimer_config,
+                             X86CPU, HV_SYNIC_STIMER_COUNT),
+        VMSTATE_UINT64_ARRAY(env.msr_hv_stimer_count,
+                             X86CPU, HV_SYNIC_STIMER_COUNT),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static bool avx512_needed(void *opaque)
 {
     X86CPU *cpu = opaque;
@@ -654,17 +803,16 @@
     }
 
     for (i = 0; i < CPU_NB_REGS; i++) {
-#define ENV_ZMMH(reg, field) (env->zmmh_regs[reg].YMM_Q(field))
-        if (ENV_ZMMH(i, 0) || ENV_ZMMH(i, 1) ||
-            ENV_ZMMH(i, 2) || ENV_ZMMH(i, 3)) {
+#define ENV_XMM(reg, field) (env->xmm_regs[reg].ZMM_Q(field))
+        if (ENV_XMM(i, 4) || ENV_XMM(i, 6) ||
+            ENV_XMM(i, 5) || ENV_XMM(i, 7)) {
             return true;
         }
 #ifdef TARGET_X86_64
-#define ENV_Hi16_ZMM(reg, field) (env->hi16_zmm_regs[reg].ZMM_Q(field))
-        if (ENV_Hi16_ZMM(i, 0) || ENV_Hi16_ZMM(i, 1) ||
-            ENV_Hi16_ZMM(i, 2) || ENV_Hi16_ZMM(i, 3) ||
-            ENV_Hi16_ZMM(i, 4) || ENV_Hi16_ZMM(i, 5) ||
-            ENV_Hi16_ZMM(i, 6) || ENV_Hi16_ZMM(i, 7)) {
+        if (ENV_XMM(i+16, 0) || ENV_XMM(i+16, 1) ||
+            ENV_XMM(i+16, 2) || ENV_XMM(i+16, 3) ||
+            ENV_XMM(i+16, 4) || ENV_XMM(i+16, 5) ||
+            ENV_XMM(i+16, 6) || ENV_XMM(i+16, 7)) {
             return true;
         }
 #endif
@@ -677,16 +825,95 @@
     .name = "cpu/avx512",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = avx512_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64_ARRAY(env.opmask_regs, X86CPU, NB_OPMASK_REGS),
-        VMSTATE_ZMMH_REGS_VARS(env.zmmh_regs, X86CPU, CPU_NB_REGS),
+        VMSTATE_ZMMH_REGS_VARS(env.xmm_regs, X86CPU, 0),
 #ifdef TARGET_X86_64
-        VMSTATE_Hi16_ZMM_REGS_VARS(env.hi16_zmm_regs, X86CPU, CPU_NB_REGS),
+        VMSTATE_Hi16_ZMM_REGS_VARS(env.xmm_regs, X86CPU, 16),
 #endif
         VMSTATE_END_OF_LIST()
     }
 };
 
+static bool xss_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->xss != 0;
+}
+
+static const VMStateDescription vmstate_xss = {
+    .name = "cpu/xss",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = xss_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.xss, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+#ifdef TARGET_X86_64
+static bool pkru_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+
+    return env->pkru != 0;
+}
+
+static const VMStateDescription vmstate_pkru = {
+    .name = "cpu/pkru",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pkru_needed,
+    .fields = (VMStateField[]){
+        VMSTATE_UINT32(env.pkru, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+#endif
+
+static bool tsc_khz_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine());
+    PCMachineClass *pcmc = PC_MACHINE_CLASS(mc);
+    return env->tsc_khz && pcmc->save_tsc_khz;
+}
+
+static const VMStateDescription vmstate_tsc_khz = {
+    .name = "cpu/tsc_khz",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = tsc_khz_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT64(env.tsc_khz, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool mcg_ext_ctl_needed(void *opaque)
+{
+    X86CPU *cpu = opaque;
+    CPUX86State *env = &cpu->env;
+    return cpu->enable_lmce && env->mcg_ext_ctl;
+}
+
+static const VMStateDescription vmstate_mcg_ext_ctl = {
+    .name = "cpu/mcg_ext_ctl",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = mcg_ext_ctl_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.mcg_ext_ctl, X86CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 VMStateDescription vmstate_x86_cpu = {
     .name = "cpu",
     .version_id = 12,
@@ -732,7 +959,7 @@
         VMSTATE_INT32(env.a20_mask, X86CPU),
         /* XMM */
         VMSTATE_UINT32(env.mxcsr, X86CPU),
-        VMSTATE_XMM_REGS(env.xmm_regs, X86CPU, CPU_NB_REGS),
+        VMSTATE_XMM_REGS(env.xmm_regs, X86CPU, 0),
 
 #ifdef TARGET_X86_64
         VMSTATE_UINT64(env.efer, X86CPU),
@@ -785,55 +1012,35 @@
         /* XSAVE related fields */
         VMSTATE_UINT64_V(env.xcr0, X86CPU, 12),
         VMSTATE_UINT64_V(env.xstate_bv, X86CPU, 12),
-        VMSTATE_YMMH_REGS_VARS(env.ymmh_regs, X86CPU, CPU_NB_REGS, 12),
+        VMSTATE_YMMH_REGS_VARS(env.xmm_regs, X86CPU, 0, 12),
         VMSTATE_END_OF_LIST()
         /* The above list is not sorted /wrt version numbers, watch out! */
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_async_pf_msr,
-            .needed = async_pf_msr_needed,
-        } , {
-            .vmsd = &vmstate_pv_eoi_msr,
-            .needed = pv_eoi_msr_needed,
-        } , {
-            .vmsd = &vmstate_steal_time_msr,
-            .needed = steal_time_msr_needed,
-        } , {
-            .vmsd = &vmstate_fpop_ip_dp,
-            .needed = fpop_ip_dp_needed,
-        }, {
-            .vmsd = &vmstate_msr_tsc_adjust,
-            .needed = tsc_adjust_needed,
-        }, {
-            .vmsd = &vmstate_msr_tscdeadline,
-            .needed = tscdeadline_needed,
-        }, {
-            .vmsd = &vmstate_msr_ia32_misc_enable,
-            .needed = misc_enable_needed,
-        }, {
-            .vmsd = &vmstate_msr_ia32_feature_control,
-            .needed = feature_control_needed,
-        }, {
-            .vmsd = &vmstate_msr_architectural_pmu,
-            .needed = pmu_enable_needed,
-        } , {
-            .vmsd = &vmstate_mpx,
-            .needed = mpx_needed,
-        }, {
-            .vmsd = &vmstate_msr_hypercall_hypercall,
-            .needed = hyperv_hypercall_enable_needed,
-        }, {
-            .vmsd = &vmstate_msr_hyperv_vapic,
-            .needed = hyperv_vapic_enable_needed,
-        }, {
-            .vmsd = &vmstate_msr_hyperv_time,
-            .needed = hyperv_time_enable_needed,
-        }, {
-            .vmsd = &vmstate_avx512,
-            .needed = avx512_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_async_pf_msr,
+        &vmstate_pv_eoi_msr,
+        &vmstate_steal_time_msr,
+        &vmstate_fpop_ip_dp,
+        &vmstate_msr_tsc_adjust,
+        &vmstate_msr_tscdeadline,
+        &vmstate_msr_ia32_misc_enable,
+        &vmstate_msr_ia32_feature_control,
+        &vmstate_msr_architectural_pmu,
+        &vmstate_mpx,
+        &vmstate_msr_hypercall_hypercall,
+        &vmstate_msr_hyperv_vapic,
+        &vmstate_msr_hyperv_time,
+        &vmstate_msr_hyperv_crash,
+        &vmstate_msr_hyperv_runtime,
+        &vmstate_msr_hyperv_synic,
+        &vmstate_msr_hyperv_stimer,
+        &vmstate_avx512,
+        &vmstate_xss,
+        &vmstate_tsc_khz,
+#ifdef TARGET_X86_64
+        &vmstate_pkru,
+#endif
+        &vmstate_mcg_ext_ctl,
+        NULL
     }
 };
diff --git a/target-i386/mem_helper.c b/target-i386/mem_helper.c
index 1aec8a5..5bc0594 100644
--- a/target-i386/mem_helper.c
+++ b/target-i386/mem_helper.c
@@ -17,37 +17,59 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 /* broken thread support */
 
-static spinlock_t global_cpu_lock = SPIN_LOCK_UNLOCKED;
+#if defined(CONFIG_USER_ONLY)
+QemuMutex global_cpu_lock;
 
 void helper_lock(void)
 {
-    spin_lock(&global_cpu_lock);
+    qemu_mutex_lock(&global_cpu_lock);
 }
 
 void helper_unlock(void)
 {
-    spin_unlock(&global_cpu_lock);
+    qemu_mutex_unlock(&global_cpu_lock);
 }
 
+void helper_lock_init(void)
+{
+    qemu_mutex_init(&global_cpu_lock);
+}
+#else
+void helper_lock(void)
+{
+}
+
+void helper_unlock(void)
+{
+}
+
+void helper_lock_init(void)
+{
+}
+#endif
+
 void helper_cmpxchg8b(CPUX86State *env, target_ulong a0)
 {
     uint64_t d;
     int eflags;
 
     eflags = cpu_cc_compute_all(env, CC_OP);
-    d = cpu_ldq_data(env, a0);
+    d = cpu_ldq_data_ra(env, a0, GETPC());
     if (d == (((uint64_t)env->regs[R_EDX] << 32) | (uint32_t)env->regs[R_EAX])) {
-        cpu_stq_data(env, a0, ((uint64_t)env->regs[R_ECX] << 32) | (uint32_t)env->regs[R_EBX]);
+        cpu_stq_data_ra(env, a0, ((uint64_t)env->regs[R_ECX] << 32)
+                                  | (uint32_t)env->regs[R_EBX], GETPC());
         eflags |= CC_Z;
     } else {
         /* always do the store */
-        cpu_stq_data(env, a0, d);
+        cpu_stq_data_ra(env, a0, d, GETPC());
         env->regs[R_EDX] = (uint32_t)(d >> 32);
         env->regs[R_EAX] = (uint32_t)d;
         eflags &= ~CC_Z;
@@ -62,19 +84,19 @@
     int eflags;
 
     if ((a0 & 0xf) != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     eflags = cpu_cc_compute_all(env, CC_OP);
-    d0 = cpu_ldq_data(env, a0);
-    d1 = cpu_ldq_data(env, a0 + 8);
+    d0 = cpu_ldq_data_ra(env, a0, GETPC());
+    d1 = cpu_ldq_data_ra(env, a0 + 8, GETPC());
     if (d0 == env->regs[R_EAX] && d1 == env->regs[R_EDX]) {
-        cpu_stq_data(env, a0, env->regs[R_EBX]);
-        cpu_stq_data(env, a0 + 8, env->regs[R_ECX]);
+        cpu_stq_data_ra(env, a0, env->regs[R_EBX], GETPC());
+        cpu_stq_data_ra(env, a0 + 8, env->regs[R_ECX], GETPC());
         eflags |= CC_Z;
     } else {
         /* always do the store */
-        cpu_stq_data(env, a0, d0);
-        cpu_stq_data(env, a0 + 8, d1);
+        cpu_stq_data_ra(env, a0, d0, GETPC());
+        cpu_stq_data_ra(env, a0 + 8, d1, GETPC());
         env->regs[R_EDX] = d1;
         env->regs[R_EAX] = d0;
         eflags &= ~CC_Z;
@@ -87,11 +109,14 @@
 {
     int low, high;
 
-    low = cpu_ldsw_data(env, a0);
-    high = cpu_ldsw_data(env, a0 + 2);
+    low = cpu_ldsw_data_ra(env, a0, GETPC());
+    high = cpu_ldsw_data_ra(env, a0 + 2, GETPC());
     v = (int16_t)v;
     if (v < low || v > high) {
-        raise_exception(env, EXCP05_BOUND);
+        if (env->hflags & HF_MPX_EN_MASK) {
+            env->bndcs_regs.sts = 0;
+        }
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
     }
 }
 
@@ -99,10 +124,13 @@
 {
     int low, high;
 
-    low = cpu_ldl_data(env, a0);
-    high = cpu_ldl_data(env, a0 + 4);
+    low = cpu_ldl_data_ra(env, a0, GETPC());
+    high = cpu_ldl_data_ra(env, a0 + 4, GETPC());
     if (v < low || v > high) {
-        raise_exception(env, EXCP05_BOUND);
+        if (env->hflags & HF_MPX_EN_MASK) {
+            env->bndcs_regs.sts = 0;
+        }
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
     }
 }
 
@@ -112,21 +140,17 @@
  * from generated code or from helper.c)
  */
 /* XXX: fix it to restore all registers */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = x86_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = x86_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (ret) {
         X86CPU *cpu = X86_CPU(cs);
         CPUX86State *env = &cpu->env;
 
-        if (retaddr) {
-            /* now we have a real cpu fault */
-            cpu_restore_state(cs, retaddr);
-        }
-        raise_exception_err(env, cs->exception_index, env->error_code);
+        raise_exception_err_ra(env, cs->exception_index, env->error_code, retaddr);
     }
 }
 #endif
diff --git a/target-i386/misc_helper.c b/target-i386/misc_helper.c
index 4aaf1e4..3f666b4 100644
--- a/target-i386/misc_helper.c
+++ b/target-i386/misc_helper.c
@@ -17,39 +17,74 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
-#include "exec/ioport.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "exec/address-spaces.h"
 
-void helper_outb(uint32_t port, uint32_t data)
+void helper_outb(CPUX86State *env, uint32_t port, uint32_t data)
 {
-    cpu_outb(port, data & 0xff);
+#ifdef CONFIG_USER_ONLY
+    fprintf(stderr, "outb: port=0x%04x, data=%02x\n", port, data);
+#else
+    address_space_stb(&address_space_io, port, data,
+                      cpu_get_mem_attrs(env), NULL);
+#endif
 }
 
-target_ulong helper_inb(uint32_t port)
+target_ulong helper_inb(CPUX86State *env, uint32_t port)
 {
-    return cpu_inb(port);
+#ifdef CONFIG_USER_ONLY
+    fprintf(stderr, "inb: port=0x%04x\n", port);
+    return 0;
+#else
+    return address_space_ldub(&address_space_io, port,
+                              cpu_get_mem_attrs(env), NULL);
+#endif
 }
 
-void helper_outw(uint32_t port, uint32_t data)
+void helper_outw(CPUX86State *env, uint32_t port, uint32_t data)
 {
-    cpu_outw(port, data & 0xffff);
+#ifdef CONFIG_USER_ONLY
+    fprintf(stderr, "outw: port=0x%04x, data=%04x\n", port, data);
+#else
+    address_space_stw(&address_space_io, port, data,
+                      cpu_get_mem_attrs(env), NULL);
+#endif
 }
 
-target_ulong helper_inw(uint32_t port)
+target_ulong helper_inw(CPUX86State *env, uint32_t port)
 {
-    return cpu_inw(port);
+#ifdef CONFIG_USER_ONLY
+    fprintf(stderr, "inw: port=0x%04x\n", port);
+    return 0;
+#else
+    return address_space_lduw(&address_space_io, port,
+                              cpu_get_mem_attrs(env), NULL);
+#endif
 }
 
-void helper_outl(uint32_t port, uint32_t data)
+void helper_outl(CPUX86State *env, uint32_t port, uint32_t data)
 {
-    cpu_outl(port, data);
+#ifdef CONFIG_USER_ONLY
+    fprintf(stderr, "outw: port=0x%04x, data=%08x\n", port, data);
+#else
+    address_space_stl(&address_space_io, port, data,
+                      cpu_get_mem_attrs(env), NULL);
+#endif
 }
 
-target_ulong helper_inl(uint32_t port)
+target_ulong helper_inl(CPUX86State *env, uint32_t port)
 {
-    return cpu_inl(port);
+#ifdef CONFIG_USER_ONLY
+    fprintf(stderr, "inl: port=0x%04x\n", port);
+    return 0;
+#else
+    return address_space_ldl(&address_space_io, port,
+                             cpu_get_mem_attrs(env), NULL);
+#endif
 }
 
 void helper_into(CPUX86State *env, int next_eip_addend)
@@ -62,15 +97,6 @@
     }
 }
 
-void helper_single_step(CPUX86State *env)
-{
-#ifndef CONFIG_USER_ONLY
-    check_hw_breakpoints(env, true);
-    env->dr[6] |= DR6_BS;
-#endif
-    raise_exception(env, EXCP01_DB);
-}
-
 void helper_cpuid(CPUX86State *env)
 {
     uint32_t eax, ebx, ecx, edx;
@@ -94,10 +120,6 @@
 void helper_write_crN(CPUX86State *env, int reg, target_ulong t0)
 {
 }
-
-void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
-{
-}
 #else
 target_ulong helper_read_crN(CPUX86State *env, int reg)
 {
@@ -143,27 +165,6 @@
         break;
     }
 }
-
-void helper_movl_drN_T0(CPUX86State *env, int reg, target_ulong t0)
-{
-    int i;
-
-    if (reg < 4) {
-        hw_breakpoint_remove(env, reg);
-        env->dr[reg] = t0;
-        hw_breakpoint_insert(env, reg);
-    } else if (reg == 7) {
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            hw_breakpoint_remove(env, i);
-        }
-        env->dr[7] = t0;
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            hw_breakpoint_insert(env, i);
-        }
-    } else {
-        env->dr[reg] = t0;
-    }
-}
 #endif
 
 void helper_lmsw(CPUX86State *env, target_ulong t0)
@@ -187,7 +188,7 @@
     uint64_t val;
 
     if ((env->cr[4] & CR4_TSD_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_RDTSC, 0);
 
@@ -205,7 +206,7 @@
 void helper_rdpmc(CPUX86State *env)
 {
     if ((env->cr[4] & CR4_PCE_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_RDPMC, 0);
 
@@ -361,6 +362,12 @@
     case MSR_IA32_MISC_ENABLE:
         env->msr_ia32_misc_enable = val;
         break;
+    case MSR_IA32_BNDCFGS:
+        /* FIXME: #GP if reserved bits are set.  */
+        /* FIXME: Extend highest implemented bit of linear address.  */
+        env->msr_bndcfgs = val;
+        cpu_sync_bndcs_hflags(env);
+        break;
     default:
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
@@ -506,6 +513,9 @@
     case MSR_IA32_MISC_ENABLE:
         val = env->msr_ia32_misc_enable;
         break;
+    case MSR_IA32_BNDCFGS:
+        val = env->msr_bndcfgs;
+        break;
     default:
         if ((uint32_t)env->regs[R_ECX] >= MSR_MC0_CTL
             && (uint32_t)env->regs[R_ECX] < MSR_MC0_CTL +
@@ -556,7 +566,7 @@
 void helper_monitor(CPUX86State *env, target_ulong ptr)
 {
     if ((uint32_t)env->regs[R_ECX] != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     /* XXX: store address? */
     cpu_svm_check_intercept_param(env, SVM_EXIT_MONITOR, 0);
@@ -568,7 +578,7 @@
     X86CPU *cpu;
 
     if ((uint32_t)env->regs[R_ECX] != 0) {
-        raise_exception(env, EXCP0D_GPF);
+        raise_exception_ra(env, EXCP0D_GPF, GETPC());
     }
     cpu_svm_check_intercept_param(env, SVM_EXIT_MWAIT, 0);
     env->eip += next_eip_addend;
@@ -600,3 +610,30 @@
     cs->exception_index = EXCP_DEBUG;
     cpu_loop_exit(cs);
 }
+
+uint64_t helper_rdpkru(CPUX86State *env, uint32_t ecx)
+{
+    if ((env->cr[4] & CR4_PKE_MASK) == 0) {
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+    }
+    if (ecx != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
+    }
+
+    return env->pkru;
+}
+
+void helper_wrpkru(CPUX86State *env, uint32_t ecx, uint64_t val)
+{
+    CPUState *cs = CPU(x86_env_get_cpu(env));
+
+    if ((env->cr[4] & CR4_PKE_MASK) == 0) {
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
+    }
+    if (ecx != 0 || (val & 0xFFFFFFFF00000000ull)) {
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
+    }
+
+    env->pkru = val;
+    tlb_flush(cs, 1);
+}
diff --git a/target-i386/monitor.c b/target-i386/monitor.c
new file mode 100644
index 0000000..fccfe40
--- /dev/null
+++ b/target-i386/monitor.c
@@ -0,0 +1,512 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hw/i386/pc.h"
+#include "sysemu/kvm.h"
+#include "hmp.h"
+
+
+static void print_pte(Monitor *mon, hwaddr addr,
+                      hwaddr pte,
+                      hwaddr mask)
+{
+#ifdef TARGET_X86_64
+    if (addr & (1ULL << 47)) {
+        addr |= -1LL << 48;
+    }
+#endif
+    monitor_printf(mon, TARGET_FMT_plx ": " TARGET_FMT_plx
+                   " %c%c%c%c%c%c%c%c%c\n",
+                   addr,
+                   pte & mask,
+                   pte & PG_NX_MASK ? 'X' : '-',
+                   pte & PG_GLOBAL_MASK ? 'G' : '-',
+                   pte & PG_PSE_MASK ? 'P' : '-',
+                   pte & PG_DIRTY_MASK ? 'D' : '-',
+                   pte & PG_ACCESSED_MASK ? 'A' : '-',
+                   pte & PG_PCD_MASK ? 'C' : '-',
+                   pte & PG_PWT_MASK ? 'T' : '-',
+                   pte & PG_USER_MASK ? 'U' : '-',
+                   pte & PG_RW_MASK ? 'W' : '-');
+}
+
+static void tlb_info_32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2;
+    uint32_t pgd, pde, pte;
+
+    pgd = env->cr[3] & ~0xfff;
+    for(l1 = 0; l1 < 1024; l1++) {
+        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
+        pde = le32_to_cpu(pde);
+        if (pde & PG_PRESENT_MASK) {
+            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+                /* 4M pages */
+                print_pte(mon, (l1 << 22), pde, ~((1 << 21) - 1));
+            } else {
+                for(l2 = 0; l2 < 1024; l2++) {
+                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
+                    pte = le32_to_cpu(pte);
+                    if (pte & PG_PRESENT_MASK) {
+                        print_pte(mon, (l1 << 22) + (l2 << 12),
+                                  pte & ~PG_PSE_MASK,
+                                  ~0xfff);
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2, l3;
+    uint64_t pdpe, pde, pte;
+    uint64_t pdp_addr, pd_addr, pt_addr;
+
+    pdp_addr = env->cr[3] & ~0x1f;
+    for (l1 = 0; l1 < 4; l1++) {
+        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
+        pdpe = le64_to_cpu(pdpe);
+        if (pdpe & PG_PRESENT_MASK) {
+            pd_addr = pdpe & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
+                pde = le64_to_cpu(pde);
+                if (pde & PG_PRESENT_MASK) {
+                    if (pde & PG_PSE_MASK) {
+                        /* 2M pages with PAE, CR4.PSE is ignored */
+                        print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
+                                  ~((hwaddr)(1 << 20) - 1));
+                    } else {
+                        pt_addr = pde & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
+                            pte = le64_to_cpu(pte);
+                            if (pte & PG_PRESENT_MASK) {
+                                print_pte(mon, (l1 << 30 ) + (l2 << 21)
+                                          + (l3 << 12),
+                                          pte & ~PG_PSE_MASK,
+                                          ~(hwaddr)0xfff);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#ifdef TARGET_X86_64
+static void tlb_info_64(Monitor *mon, CPUArchState *env)
+{
+    uint64_t l1, l2, l3, l4;
+    uint64_t pml4e, pdpe, pde, pte;
+    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr;
+
+    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
+    for (l1 = 0; l1 < 512; l1++) {
+        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
+        pml4e = le64_to_cpu(pml4e);
+        if (pml4e & PG_PRESENT_MASK) {
+            pdp_addr = pml4e & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
+                pdpe = le64_to_cpu(pdpe);
+                if (pdpe & PG_PRESENT_MASK) {
+                    if (pdpe & PG_PSE_MASK) {
+                        /* 1G pages, CR4.PSE is ignored */
+                        print_pte(mon, (l1 << 39) + (l2 << 30), pdpe,
+                                  0x3ffffc0000000ULL);
+                    } else {
+                        pd_addr = pdpe & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
+                            pde = le64_to_cpu(pde);
+                            if (pde & PG_PRESENT_MASK) {
+                                if (pde & PG_PSE_MASK) {
+                                    /* 2M pages, CR4.PSE is ignored */
+                                    print_pte(mon, (l1 << 39) + (l2 << 30) +
+                                              (l3 << 21), pde,
+                                              0x3ffffffe00000ULL);
+                                } else {
+                                    pt_addr = pde & 0x3fffffffff000ULL;
+                                    for (l4 = 0; l4 < 512; l4++) {
+                                        cpu_physical_memory_read(pt_addr
+                                                                 + l4 * 8,
+                                                                 &pte, 8);
+                                        pte = le64_to_cpu(pte);
+                                        if (pte & PG_PRESENT_MASK) {
+                                            print_pte(mon, (l1 << 39) +
+                                                      (l2 << 30) +
+                                                      (l3 << 21) + (l4 << 12),
+                                                      pte & ~PG_PSE_MASK,
+                                                      0x3fffffffff000ULL);
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+#endif /* TARGET_X86_64 */
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env;
+
+    env = mon_get_cpu_env();
+
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        monitor_printf(mon, "PG disabled\n");
+        return;
+    }
+    if (env->cr[4] & CR4_PAE_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            tlb_info_64(mon, env);
+        } else
+#endif
+        {
+            tlb_info_pae32(mon, env);
+        }
+    } else {
+        tlb_info_32(mon, env);
+    }
+}
+
+static void mem_print(Monitor *mon, hwaddr *pstart,
+                      int *plast_prot,
+                      hwaddr end, int prot)
+{
+    int prot1;
+    prot1 = *plast_prot;
+    if (prot != prot1) {
+        if (*pstart != -1) {
+            monitor_printf(mon, TARGET_FMT_plx "-" TARGET_FMT_plx " "
+                           TARGET_FMT_plx " %c%c%c\n",
+                           *pstart, end, end - *pstart,
+                           prot1 & PG_USER_MASK ? 'u' : '-',
+                           'r',
+                           prot1 & PG_RW_MASK ? 'w' : '-');
+        }
+        if (prot != 0)
+            *pstart = end;
+        else
+            *pstart = -1;
+        *plast_prot = prot;
+    }
+}
+
+static void mem_info_32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2;
+    int prot, last_prot;
+    uint32_t pgd, pde, pte;
+    hwaddr start, end;
+
+    pgd = env->cr[3] & ~0xfff;
+    last_prot = 0;
+    start = -1;
+    for(l1 = 0; l1 < 1024; l1++) {
+        cpu_physical_memory_read(pgd + l1 * 4, &pde, 4);
+        pde = le32_to_cpu(pde);
+        end = l1 << 22;
+        if (pde & PG_PRESENT_MASK) {
+            if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
+                prot = pde & (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
+                mem_print(mon, &start, &last_prot, end, prot);
+            } else {
+                for(l2 = 0; l2 < 1024; l2++) {
+                    cpu_physical_memory_read((pde & ~0xfff) + l2 * 4, &pte, 4);
+                    pte = le32_to_cpu(pte);
+                    end = (l1 << 22) + (l2 << 12);
+                    if (pte & PG_PRESENT_MASK) {
+                        prot = pte & pde &
+                            (PG_USER_MASK | PG_RW_MASK | PG_PRESENT_MASK);
+                    } else {
+                        prot = 0;
+                    }
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
+}
+
+static void mem_info_pae32(Monitor *mon, CPUArchState *env)
+{
+    unsigned int l1, l2, l3;
+    int prot, last_prot;
+    uint64_t pdpe, pde, pte;
+    uint64_t pdp_addr, pd_addr, pt_addr;
+    hwaddr start, end;
+
+    pdp_addr = env->cr[3] & ~0x1f;
+    last_prot = 0;
+    start = -1;
+    for (l1 = 0; l1 < 4; l1++) {
+        cpu_physical_memory_read(pdp_addr + l1 * 8, &pdpe, 8);
+        pdpe = le64_to_cpu(pdpe);
+        end = l1 << 30;
+        if (pdpe & PG_PRESENT_MASK) {
+            pd_addr = pdpe & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pd_addr + l2 * 8, &pde, 8);
+                pde = le64_to_cpu(pde);
+                end = (l1 << 30) + (l2 << 21);
+                if (pde & PG_PRESENT_MASK) {
+                    if (pde & PG_PSE_MASK) {
+                        prot = pde & (PG_USER_MASK | PG_RW_MASK |
+                                      PG_PRESENT_MASK);
+                        mem_print(mon, &start, &last_prot, end, prot);
+                    } else {
+                        pt_addr = pde & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pt_addr + l3 * 8, &pte, 8);
+                            pte = le64_to_cpu(pte);
+                            end = (l1 << 30) + (l2 << 21) + (l3 << 12);
+                            if (pte & PG_PRESENT_MASK) {
+                                prot = pte & pde & (PG_USER_MASK | PG_RW_MASK |
+                                                    PG_PRESENT_MASK);
+                            } else {
+                                prot = 0;
+                            }
+                            mem_print(mon, &start, &last_prot, end, prot);
+                        }
+                    }
+                } else {
+                    prot = 0;
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
+}
+
+
+#ifdef TARGET_X86_64
+static void mem_info_64(Monitor *mon, CPUArchState *env)
+{
+    int prot, last_prot;
+    uint64_t l1, l2, l3, l4;
+    uint64_t pml4e, pdpe, pde, pte;
+    uint64_t pml4_addr, pdp_addr, pd_addr, pt_addr, start, end;
+
+    pml4_addr = env->cr[3] & 0x3fffffffff000ULL;
+    last_prot = 0;
+    start = -1;
+    for (l1 = 0; l1 < 512; l1++) {
+        cpu_physical_memory_read(pml4_addr + l1 * 8, &pml4e, 8);
+        pml4e = le64_to_cpu(pml4e);
+        end = l1 << 39;
+        if (pml4e & PG_PRESENT_MASK) {
+            pdp_addr = pml4e & 0x3fffffffff000ULL;
+            for (l2 = 0; l2 < 512; l2++) {
+                cpu_physical_memory_read(pdp_addr + l2 * 8, &pdpe, 8);
+                pdpe = le64_to_cpu(pdpe);
+                end = (l1 << 39) + (l2 << 30);
+                if (pdpe & PG_PRESENT_MASK) {
+                    if (pdpe & PG_PSE_MASK) {
+                        prot = pdpe & (PG_USER_MASK | PG_RW_MASK |
+                                       PG_PRESENT_MASK);
+                        prot &= pml4e;
+                        mem_print(mon, &start, &last_prot, end, prot);
+                    } else {
+                        pd_addr = pdpe & 0x3fffffffff000ULL;
+                        for (l3 = 0; l3 < 512; l3++) {
+                            cpu_physical_memory_read(pd_addr + l3 * 8, &pde, 8);
+                            pde = le64_to_cpu(pde);
+                            end = (l1 << 39) + (l2 << 30) + (l3 << 21);
+                            if (pde & PG_PRESENT_MASK) {
+                                if (pde & PG_PSE_MASK) {
+                                    prot = pde & (PG_USER_MASK | PG_RW_MASK |
+                                                  PG_PRESENT_MASK);
+                                    prot &= pml4e & pdpe;
+                                    mem_print(mon, &start, &last_prot, end, prot);
+                                } else {
+                                    pt_addr = pde & 0x3fffffffff000ULL;
+                                    for (l4 = 0; l4 < 512; l4++) {
+                                        cpu_physical_memory_read(pt_addr
+                                                                 + l4 * 8,
+                                                                 &pte, 8);
+                                        pte = le64_to_cpu(pte);
+                                        end = (l1 << 39) + (l2 << 30) +
+                                            (l3 << 21) + (l4 << 12);
+                                        if (pte & PG_PRESENT_MASK) {
+                                            prot = pte & (PG_USER_MASK | PG_RW_MASK |
+                                                          PG_PRESENT_MASK);
+                                            prot &= pml4e & pdpe & pde;
+                                        } else {
+                                            prot = 0;
+                                        }
+                                        mem_print(mon, &start, &last_prot, end, prot);
+                                    }
+                                }
+                            } else {
+                                prot = 0;
+                                mem_print(mon, &start, &last_prot, end, prot);
+                            }
+                        }
+                    }
+                } else {
+                    prot = 0;
+                    mem_print(mon, &start, &last_prot, end, prot);
+                }
+            }
+        } else {
+            prot = 0;
+            mem_print(mon, &start, &last_prot, end, prot);
+        }
+    }
+    /* Flush last range */
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
+}
+#endif /* TARGET_X86_64 */
+
+void hmp_info_mem(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env;
+
+    env = mon_get_cpu_env();
+
+    if (!(env->cr[0] & CR0_PG_MASK)) {
+        monitor_printf(mon, "PG disabled\n");
+        return;
+    }
+    if (env->cr[4] & CR4_PAE_MASK) {
+#ifdef TARGET_X86_64
+        if (env->hflags & HF_LMA_MASK) {
+            mem_info_64(mon, env);
+        } else
+#endif
+        {
+            mem_info_pae32(mon, env);
+        }
+    } else {
+        mem_info_32(mon, env);
+    }
+}
+
+void hmp_mce(Monitor *mon, const QDict *qdict)
+{
+    X86CPU *cpu;
+    CPUState *cs;
+    int cpu_index = qdict_get_int(qdict, "cpu_index");
+    int bank = qdict_get_int(qdict, "bank");
+    uint64_t status = qdict_get_int(qdict, "status");
+    uint64_t mcg_status = qdict_get_int(qdict, "mcg_status");
+    uint64_t addr = qdict_get_int(qdict, "addr");
+    uint64_t misc = qdict_get_int(qdict, "misc");
+    int flags = MCE_INJECT_UNCOND_AO;
+
+    if (qdict_get_try_bool(qdict, "broadcast", false)) {
+        flags |= MCE_INJECT_BROADCAST;
+    }
+    cs = qemu_get_cpu(cpu_index);
+    if (cs != NULL) {
+        cpu = X86_CPU(cs);
+        cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
+                           flags);
+    }
+}
+
+static target_long monitor_get_pc(const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return env->eip + env->segs[R_CS].base;
+}
+
+const MonitorDef monitor_defs[] = {
+#define SEG(name, seg) \
+    { name, offsetof(CPUX86State, segs[seg].selector), NULL, MD_I32 },\
+    { name ".base", offsetof(CPUX86State, segs[seg].base) },\
+    { name ".limit", offsetof(CPUX86State, segs[seg].limit), NULL, MD_I32 },
+
+    { "eax", offsetof(CPUX86State, regs[0]) },
+    { "ecx", offsetof(CPUX86State, regs[1]) },
+    { "edx", offsetof(CPUX86State, regs[2]) },
+    { "ebx", offsetof(CPUX86State, regs[3]) },
+    { "esp|sp", offsetof(CPUX86State, regs[4]) },
+    { "ebp|fp", offsetof(CPUX86State, regs[5]) },
+    { "esi", offsetof(CPUX86State, regs[6]) },
+    { "edi", offsetof(CPUX86State, regs[7]) },
+#ifdef TARGET_X86_64
+    { "r8", offsetof(CPUX86State, regs[8]) },
+    { "r9", offsetof(CPUX86State, regs[9]) },
+    { "r10", offsetof(CPUX86State, regs[10]) },
+    { "r11", offsetof(CPUX86State, regs[11]) },
+    { "r12", offsetof(CPUX86State, regs[12]) },
+    { "r13", offsetof(CPUX86State, regs[13]) },
+    { "r14", offsetof(CPUX86State, regs[14]) },
+    { "r15", offsetof(CPUX86State, regs[15]) },
+#endif
+    { "eflags", offsetof(CPUX86State, eflags) },
+    { "eip", offsetof(CPUX86State, eip) },
+    SEG("cs", R_CS)
+    SEG("ds", R_DS)
+    SEG("es", R_ES)
+    SEG("ss", R_SS)
+    SEG("fs", R_FS)
+    SEG("gs", R_GS)
+    { "pc", 0, monitor_get_pc, },
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
+
+void hmp_info_local_apic(Monitor *mon, const QDict *qdict)
+{
+    x86_cpu_dump_local_apic_state(mon_get_cpu(), (FILE *)mon, monitor_fprintf,
+                                  CPU_DUMP_FPU);
+}
+
+void hmp_info_io_apic(Monitor *mon, const QDict *qdict)
+{
+    if (kvm_irqchip_in_kernel()) {
+        kvm_ioapic_dump_state(mon, qdict);
+    } else {
+        ioapic_dump_state(mon, qdict);
+    }
+}
diff --git a/target-i386/mpx_helper.c b/target-i386/mpx_helper.c
new file mode 100644
index 0000000..7e44820
--- /dev/null
+++ b/target-i386/mpx_helper.c
@@ -0,0 +1,168 @@
+/*
+ *  x86 MPX helpers
+ *
+ *  Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
+#include "exec/exec-all.h"
+
+
+void cpu_sync_bndcs_hflags(CPUX86State *env)
+{
+    uint32_t hflags = env->hflags;
+    uint32_t hflags2 = env->hflags2;
+    uint32_t bndcsr;
+
+    if ((hflags & HF_CPL_MASK) == 3) {
+        bndcsr = env->bndcs_regs.cfgu;
+    } else {
+        bndcsr = env->msr_bndcfgs;
+    }
+
+    if ((env->cr[4] & CR4_OSXSAVE_MASK)
+        && (env->xcr0 & XSTATE_BNDCSR_MASK)
+        && (bndcsr & BNDCFG_ENABLE)) {
+        hflags |= HF_MPX_EN_MASK;
+    } else {
+        hflags &= ~HF_MPX_EN_MASK;
+    }
+
+    if (bndcsr & BNDCFG_BNDPRESERVE) {
+        hflags2 |= HF2_MPX_PR_MASK;
+    } else {
+        hflags2 &= ~HF2_MPX_PR_MASK;
+    }
+
+    env->hflags = hflags;
+    env->hflags2 = hflags2;
+}
+
+void helper_bndck(CPUX86State *env, uint32_t fail)
+{
+    if (unlikely(fail)) {
+        env->bndcs_regs.sts = 1;
+        raise_exception_ra(env, EXCP05_BOUND, GETPC());
+    }
+}
+
+static uint64_t lookup_bte64(CPUX86State *env, uint64_t base, uintptr_t ra)
+{
+    uint64_t bndcsr, bde, bt;
+
+    if ((env->hflags & HF_CPL_MASK) == 3) {
+        bndcsr = env->bndcs_regs.cfgu;
+    } else {
+        bndcsr = env->msr_bndcfgs;
+    }
+
+    bde = (extract64(base, 20, 28) << 3) + (extract64(bndcsr, 20, 44) << 12);
+    bt = cpu_ldq_data_ra(env, bde, ra);
+    if ((bt & 1) == 0) {
+        env->bndcs_regs.sts = bde | 2;
+        raise_exception_ra(env, EXCP05_BOUND, ra);
+    }
+
+    return (extract64(base, 3, 17) << 5) + (bt & ~7);
+}
+
+static uint32_t lookup_bte32(CPUX86State *env, uint32_t base, uintptr_t ra)
+{
+    uint32_t bndcsr, bde, bt;
+
+    if ((env->hflags & HF_CPL_MASK) == 3) {
+        bndcsr = env->bndcs_regs.cfgu;
+    } else {
+        bndcsr = env->msr_bndcfgs;
+    }
+
+    bde = (extract32(base, 12, 20) << 2) + (bndcsr & TARGET_PAGE_MASK);
+    bt = cpu_ldl_data_ra(env, bde, ra);
+    if ((bt & 1) == 0) {
+        env->bndcs_regs.sts = bde | 2;
+        raise_exception_ra(env, EXCP05_BOUND, ra);
+    }
+
+    return (extract32(base, 2, 10) << 4) + (bt & ~3);
+}
+
+uint64_t helper_bndldx64(CPUX86State *env, target_ulong base, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+    uint64_t bte, lb, ub, pt;
+
+    bte = lookup_bte64(env, base, ra);
+    lb = cpu_ldq_data_ra(env, bte, ra);
+    ub = cpu_ldq_data_ra(env, bte + 8, ra);
+    pt = cpu_ldq_data_ra(env, bte + 16, ra);
+
+    if (pt != ptr) {
+        lb = ub = 0;
+    }
+    env->mmx_t0.MMX_Q(0) = ub;
+    return lb;
+}
+
+uint64_t helper_bndldx32(CPUX86State *env, target_ulong base, target_ulong ptr)
+{
+    uintptr_t ra = GETPC();
+    uint32_t bte, lb, ub, pt;
+
+    bte = lookup_bte32(env, base, ra);
+    lb = cpu_ldl_data_ra(env, bte, ra);
+    ub = cpu_ldl_data_ra(env, bte + 4, ra);
+    pt = cpu_ldl_data_ra(env, bte + 8, ra);
+
+    if (pt != ptr) {
+        lb = ub = 0;
+    }
+    return ((uint64_t)ub << 32) | lb;
+}
+
+void helper_bndstx64(CPUX86State *env, target_ulong base, target_ulong ptr,
+                     uint64_t lb, uint64_t ub)
+{
+    uintptr_t ra = GETPC();
+    uint64_t bte;
+
+    bte = lookup_bte64(env, base, ra);
+    cpu_stq_data_ra(env, bte, lb, ra);
+    cpu_stq_data_ra(env, bte + 8, ub, ra);
+    cpu_stq_data_ra(env, bte + 16, ptr, ra);
+}
+
+void helper_bndstx32(CPUX86State *env, target_ulong base, target_ulong ptr,
+                     uint64_t lb, uint64_t ub)
+{
+    uintptr_t ra = GETPC();
+    uint32_t bte;
+
+    bte = lookup_bte32(env, base, ra);
+    cpu_stl_data_ra(env, bte, lb, ra);
+    cpu_stl_data_ra(env, bte + 4, ub, ra);
+    cpu_stl_data_ra(env, bte + 8, ptr, ra);
+}
+
+void helper_bnd_jmp(CPUX86State *env)
+{
+    if (!(env->hflags2 & HF2_MPX_PR_MASK)) {
+        memset(env->bnd_regs, 0, sizeof(env->bnd_regs));
+        env->hflags &= ~HF_MPX_IU_MASK;
+    }
+}
diff --git a/target-i386/ops_sse.h b/target-i386/ops_sse.h
index 886e0a8..7a98f53 100644
--- a/target-i386/ops_sse.h
+++ b/target-i386/ops_sse.h
@@ -18,7 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "qemu/aes.h"
+#include "crypto/aes.h"
 
 #if SHIFT == 0
 #define Reg MMXReg
@@ -26,15 +26,15 @@
 #define B(n) MMX_B(n)
 #define W(n) MMX_W(n)
 #define L(n) MMX_L(n)
-#define Q(n) q
+#define Q(n) MMX_Q(n)
 #define SUFFIX _mmx
 #else
-#define Reg XMMReg
+#define Reg ZMMReg
 #define XMM_ONLY(...) __VA_ARGS__
-#define B(n) XMM_B(n)
-#define W(n) XMM_W(n)
-#define L(n) XMM_L(n)
-#define Q(n) XMM_Q(n)
+#define B(n) ZMM_B(n)
+#define W(n) ZMM_W(n)
+#define L(n) ZMM_L(n)
+#define Q(n) ZMM_Q(n)
 #define SUFFIX _xmm
 #endif
 
@@ -483,7 +483,7 @@
 
     for (i = 0; i < (8 << SHIFT); i++) {
         if (s->B(i) & 0x80) {
-            cpu_stb_data(env, a0 + i, d->B(i));
+            cpu_stb_data_ra(env, a0 + i, d->B(i), GETPC());
         }
     }
 }
@@ -582,26 +582,26 @@
 #define SSE_HELPER_S(name, F)                                           \
     void helper_ ## name ## ps(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
-        d->XMM_S(1) = F(32, d->XMM_S(1), s->XMM_S(1));                  \
-        d->XMM_S(2) = F(32, d->XMM_S(2), s->XMM_S(2));                  \
-        d->XMM_S(3) = F(32, d->XMM_S(3), s->XMM_S(3));                  \
+        d->ZMM_S(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
+        d->ZMM_S(1) = F(32, d->ZMM_S(1), s->ZMM_S(1));                  \
+        d->ZMM_S(2) = F(32, d->ZMM_S(2), s->ZMM_S(2));                  \
+        d->ZMM_S(3) = F(32, d->ZMM_S(3), s->ZMM_S(3));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
+        d->ZMM_S(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## pd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
-        d->XMM_D(1) = F(64, d->XMM_D(1), s->XMM_D(1));                  \
+        d->ZMM_D(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
+        d->ZMM_D(1) = F(64, d->ZMM_D(1), s->ZMM_D(1));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
+        d->ZMM_D(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
     }
 
 #define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status)
@@ -633,216 +633,216 @@
 {
     float32 s0, s1;
 
-    s0 = s->XMM_S(0);
-    s1 = s->XMM_S(1);
-    d->XMM_D(0) = float32_to_float64(s0, &env->sse_status);
-    d->XMM_D(1) = float32_to_float64(s1, &env->sse_status);
+    s0 = s->ZMM_S(0);
+    s1 = s->ZMM_S(1);
+    d->ZMM_D(0) = float32_to_float64(s0, &env->sse_status);
+    d->ZMM_D(1) = float32_to_float64(s1, &env->sse_status);
 }
 
 void helper_cvtpd2ps(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
-    d->XMM_S(1) = float64_to_float32(s->XMM_D(1), &env->sse_status);
+    d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_S(1) = float64_to_float32(s->ZMM_D(1), &env->sse_status);
     d->Q(1) = 0;
 }
 
 void helper_cvtss2sd(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_D(0) = float32_to_float64(s->XMM_S(0), &env->sse_status);
+    d->ZMM_D(0) = float32_to_float64(s->ZMM_S(0), &env->sse_status);
 }
 
 void helper_cvtsd2ss(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
+    d->ZMM_S(0) = float64_to_float32(s->ZMM_D(0), &env->sse_status);
 }
 
 /* integer to float */
 void helper_cvtdq2ps(CPUX86State *env, Reg *d, Reg *s)
 {
-    d->XMM_S(0) = int32_to_float32(s->XMM_L(0), &env->sse_status);
-    d->XMM_S(1) = int32_to_float32(s->XMM_L(1), &env->sse_status);
-    d->XMM_S(2) = int32_to_float32(s->XMM_L(2), &env->sse_status);
-    d->XMM_S(3) = int32_to_float32(s->XMM_L(3), &env->sse_status);
+    d->ZMM_S(0) = int32_to_float32(s->ZMM_L(0), &env->sse_status);
+    d->ZMM_S(1) = int32_to_float32(s->ZMM_L(1), &env->sse_status);
+    d->ZMM_S(2) = int32_to_float32(s->ZMM_L(2), &env->sse_status);
+    d->ZMM_S(3) = int32_to_float32(s->ZMM_L(3), &env->sse_status);
 }
 
 void helper_cvtdq2pd(CPUX86State *env, Reg *d, Reg *s)
 {
     int32_t l0, l1;
 
-    l0 = (int32_t)s->XMM_L(0);
-    l1 = (int32_t)s->XMM_L(1);
-    d->XMM_D(0) = int32_to_float64(l0, &env->sse_status);
-    d->XMM_D(1) = int32_to_float64(l1, &env->sse_status);
+    l0 = (int32_t)s->ZMM_L(0);
+    l1 = (int32_t)s->ZMM_L(1);
+    d->ZMM_D(0) = int32_to_float64(l0, &env->sse_status);
+    d->ZMM_D(1) = int32_to_float64(l1, &env->sse_status);
 }
 
-void helper_cvtpi2ps(CPUX86State *env, XMMReg *d, MMXReg *s)
+void helper_cvtpi2ps(CPUX86State *env, ZMMReg *d, MMXReg *s)
 {
-    d->XMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
-    d->XMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
+    d->ZMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
+    d->ZMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
 }
 
-void helper_cvtpi2pd(CPUX86State *env, XMMReg *d, MMXReg *s)
+void helper_cvtpi2pd(CPUX86State *env, ZMMReg *d, MMXReg *s)
 {
-    d->XMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
-    d->XMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
+    d->ZMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
+    d->ZMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
 }
 
-void helper_cvtsi2ss(CPUX86State *env, XMMReg *d, uint32_t val)
+void helper_cvtsi2ss(CPUX86State *env, ZMMReg *d, uint32_t val)
 {
-    d->XMM_S(0) = int32_to_float32(val, &env->sse_status);
+    d->ZMM_S(0) = int32_to_float32(val, &env->sse_status);
 }
 
-void helper_cvtsi2sd(CPUX86State *env, XMMReg *d, uint32_t val)
+void helper_cvtsi2sd(CPUX86State *env, ZMMReg *d, uint32_t val)
 {
-    d->XMM_D(0) = int32_to_float64(val, &env->sse_status);
+    d->ZMM_D(0) = int32_to_float64(val, &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-void helper_cvtsq2ss(CPUX86State *env, XMMReg *d, uint64_t val)
+void helper_cvtsq2ss(CPUX86State *env, ZMMReg *d, uint64_t val)
 {
-    d->XMM_S(0) = int64_to_float32(val, &env->sse_status);
+    d->ZMM_S(0) = int64_to_float32(val, &env->sse_status);
 }
 
-void helper_cvtsq2sd(CPUX86State *env, XMMReg *d, uint64_t val)
+void helper_cvtsq2sd(CPUX86State *env, ZMMReg *d, uint64_t val)
 {
-    d->XMM_D(0) = int64_to_float64(val, &env->sse_status);
+    d->ZMM_D(0) = int64_to_float64(val, &env->sse_status);
 }
 #endif
 
 /* float to integer */
-void helper_cvtps2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvtps2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
-    d->XMM_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
-    d->XMM_L(2) = float32_to_int32(s->XMM_S(2), &env->sse_status);
-    d->XMM_L(3) = float32_to_int32(s->XMM_S(3), &env->sse_status);
+    d->ZMM_L(0) = float32_to_int32(s->ZMM_S(0), &env->sse_status);
+    d->ZMM_L(1) = float32_to_int32(s->ZMM_S(1), &env->sse_status);
+    d->ZMM_L(2) = float32_to_int32(s->ZMM_S(2), &env->sse_status);
+    d->ZMM_L(3) = float32_to_int32(s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_cvtpd2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvtpd2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
-    d->XMM_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
-    d->XMM_Q(1) = 0;
+    d->ZMM_L(0) = float64_to_int32(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_L(1) = float64_to_int32(s->ZMM_D(1), &env->sse_status);
+    d->ZMM_Q(1) = 0;
 }
 
-void helper_cvtps2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvtps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
-    d->MMX_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
+    d->MMX_L(0) = float32_to_int32(s->ZMM_S(0), &env->sse_status);
+    d->MMX_L(1) = float32_to_int32(s->ZMM_S(1), &env->sse_status);
 }
 
-void helper_cvtpd2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvtpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
-    d->MMX_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
+    d->MMX_L(0) = float64_to_int32(s->ZMM_D(0), &env->sse_status);
+    d->MMX_L(1) = float64_to_int32(s->ZMM_D(1), &env->sse_status);
 }
 
-int32_t helper_cvtss2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvtss2si(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int32(s->XMM_S(0), &env->sse_status);
+    return float32_to_int32(s->ZMM_S(0), &env->sse_status);
 }
 
-int32_t helper_cvtsd2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvtsd2si(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int32(s->XMM_D(0), &env->sse_status);
+    return float64_to_int32(s->ZMM_D(0), &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-int64_t helper_cvtss2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvtss2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int64(s->XMM_S(0), &env->sse_status);
+    return float32_to_int64(s->ZMM_S(0), &env->sse_status);
 }
 
-int64_t helper_cvtsd2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvtsd2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int64(s->XMM_D(0), &env->sse_status);
+    return float64_to_int64(s->ZMM_D(0), &env->sse_status);
 }
 #endif
 
 /* float to integer truncated */
-void helper_cvttps2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvttps2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
-    d->XMM_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
-    d->XMM_L(2) = float32_to_int32_round_to_zero(s->XMM_S(2), &env->sse_status);
-    d->XMM_L(3) = float32_to_int32_round_to_zero(s->XMM_S(3), &env->sse_status);
+    d->ZMM_L(0) = float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
+    d->ZMM_L(1) = float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status);
+    d->ZMM_L(2) = float32_to_int32_round_to_zero(s->ZMM_S(2), &env->sse_status);
+    d->ZMM_L(3) = float32_to_int32_round_to_zero(s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_cvttpd2dq(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_cvttpd2dq(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
-    d->XMM_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
-    d->XMM_Q(1) = 0;
+    d->ZMM_L(0) = float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_L(1) = float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status);
+    d->ZMM_Q(1) = 0;
 }
 
-void helper_cvttps2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvttps2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
-    d->MMX_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
+    d->MMX_L(0) = float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
+    d->MMX_L(1) = float32_to_int32_round_to_zero(s->ZMM_S(1), &env->sse_status);
 }
 
-void helper_cvttpd2pi(CPUX86State *env, MMXReg *d, XMMReg *s)
+void helper_cvttpd2pi(CPUX86State *env, MMXReg *d, ZMMReg *s)
 {
-    d->MMX_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
-    d->MMX_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
+    d->MMX_L(0) = float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
+    d->MMX_L(1) = float64_to_int32_round_to_zero(s->ZMM_D(1), &env->sse_status);
 }
 
-int32_t helper_cvttss2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvttss2si(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
+    return float32_to_int32_round_to_zero(s->ZMM_S(0), &env->sse_status);
 }
 
-int32_t helper_cvttsd2si(CPUX86State *env, XMMReg *s)
+int32_t helper_cvttsd2si(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
+    return float64_to_int32_round_to_zero(s->ZMM_D(0), &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-int64_t helper_cvttss2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvttss2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float32_to_int64_round_to_zero(s->XMM_S(0), &env->sse_status);
+    return float32_to_int64_round_to_zero(s->ZMM_S(0), &env->sse_status);
 }
 
-int64_t helper_cvttsd2sq(CPUX86State *env, XMMReg *s)
+int64_t helper_cvttsd2sq(CPUX86State *env, ZMMReg *s)
 {
-    return float64_to_int64_round_to_zero(s->XMM_D(0), &env->sse_status);
+    return float64_to_int64_round_to_zero(s->ZMM_D(0), &env->sse_status);
 }
 #endif
 
-void helper_rsqrtps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rsqrtps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(0), &env->sse_status),
+    d->ZMM_S(0) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(0), &env->sse_status),
                               &env->sse_status);
-    d->XMM_S(1) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(1), &env->sse_status),
+    d->ZMM_S(1) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(1), &env->sse_status),
                               &env->sse_status);
-    d->XMM_S(2) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(2), &env->sse_status),
+    d->ZMM_S(2) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(2), &env->sse_status),
                               &env->sse_status);
-    d->XMM_S(3) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(3), &env->sse_status),
+    d->ZMM_S(3) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(3), &env->sse_status),
                               &env->sse_status);
 }
 
-void helper_rsqrtss(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rsqrtss(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one,
-                              float32_sqrt(s->XMM_S(0), &env->sse_status),
+    d->ZMM_S(0) = float32_div(float32_one,
+                              float32_sqrt(s->ZMM_S(0), &env->sse_status),
                               &env->sse_status);
 }
 
-void helper_rcpps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rcpps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one, s->XMM_S(0), &env->sse_status);
-    d->XMM_S(1) = float32_div(float32_one, s->XMM_S(1), &env->sse_status);
-    d->XMM_S(2) = float32_div(float32_one, s->XMM_S(2), &env->sse_status);
-    d->XMM_S(3) = float32_div(float32_one, s->XMM_S(3), &env->sse_status);
+    d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status);
+    d->ZMM_S(1) = float32_div(float32_one, s->ZMM_S(1), &env->sse_status);
+    d->ZMM_S(2) = float32_div(float32_one, s->ZMM_S(2), &env->sse_status);
+    d->ZMM_S(3) = float32_div(float32_one, s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_rcpss(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_rcpss(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_div(float32_one, s->XMM_S(0), &env->sse_status);
+    d->ZMM_S(0) = float32_div(float32_one, s->ZMM_S(0), &env->sse_status);
 }
 
 static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
@@ -857,14 +857,14 @@
     return (src >> shift) & mask;
 }
 
-void helper_extrq_r(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_extrq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), s->XMM_B(1), s->XMM_B(0));
+    d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), s->ZMM_B(1), s->ZMM_B(0));
 }
 
-void helper_extrq_i(CPUX86State *env, XMMReg *d, int index, int length)
+void helper_extrq_i(CPUX86State *env, ZMMReg *d, int index, int length)
 {
-    d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), index, length);
+    d->ZMM_Q(0) = helper_extrq(d->ZMM_Q(0), index, length);
 }
 
 static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
@@ -879,94 +879,94 @@
     return (src & ~(mask << shift)) | ((src & mask) << shift);
 }
 
-void helper_insertq_r(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_insertq_r(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_Q(0) = helper_insertq(s->XMM_Q(0), s->XMM_B(9), s->XMM_B(8));
+    d->ZMM_Q(0) = helper_insertq(s->ZMM_Q(0), s->ZMM_B(9), s->ZMM_B(8));
 }
 
-void helper_insertq_i(CPUX86State *env, XMMReg *d, int index, int length)
+void helper_insertq_i(CPUX86State *env, ZMMReg *d, int index, int length)
 {
-    d->XMM_Q(0) = helper_insertq(d->XMM_Q(0), index, length);
+    d->ZMM_Q(0) = helper_insertq(d->ZMM_Q(0), index, length);
 }
 
-void helper_haddps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_haddps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_S(0) = float32_add(d->XMM_S(0), d->XMM_S(1), &env->sse_status);
-    r.XMM_S(1) = float32_add(d->XMM_S(2), d->XMM_S(3), &env->sse_status);
-    r.XMM_S(2) = float32_add(s->XMM_S(0), s->XMM_S(1), &env->sse_status);
-    r.XMM_S(3) = float32_add(s->XMM_S(2), s->XMM_S(3), &env->sse_status);
+    r.ZMM_S(0) = float32_add(d->ZMM_S(0), d->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(1) = float32_add(d->ZMM_S(2), d->ZMM_S(3), &env->sse_status);
+    r.ZMM_S(2) = float32_add(s->ZMM_S(0), s->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(3) = float32_add(s->ZMM_S(2), s->ZMM_S(3), &env->sse_status);
     *d = r;
 }
 
-void helper_haddpd(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_haddpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_D(0) = float64_add(d->XMM_D(0), d->XMM_D(1), &env->sse_status);
-    r.XMM_D(1) = float64_add(s->XMM_D(0), s->XMM_D(1), &env->sse_status);
+    r.ZMM_D(0) = float64_add(d->ZMM_D(0), d->ZMM_D(1), &env->sse_status);
+    r.ZMM_D(1) = float64_add(s->ZMM_D(0), s->ZMM_D(1), &env->sse_status);
     *d = r;
 }
 
-void helper_hsubps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_hsubps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_S(0) = float32_sub(d->XMM_S(0), d->XMM_S(1), &env->sse_status);
-    r.XMM_S(1) = float32_sub(d->XMM_S(2), d->XMM_S(3), &env->sse_status);
-    r.XMM_S(2) = float32_sub(s->XMM_S(0), s->XMM_S(1), &env->sse_status);
-    r.XMM_S(3) = float32_sub(s->XMM_S(2), s->XMM_S(3), &env->sse_status);
+    r.ZMM_S(0) = float32_sub(d->ZMM_S(0), d->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(1) = float32_sub(d->ZMM_S(2), d->ZMM_S(3), &env->sse_status);
+    r.ZMM_S(2) = float32_sub(s->ZMM_S(0), s->ZMM_S(1), &env->sse_status);
+    r.ZMM_S(3) = float32_sub(s->ZMM_S(2), s->ZMM_S(3), &env->sse_status);
     *d = r;
 }
 
-void helper_hsubpd(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_hsubpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    XMMReg r;
+    ZMMReg r;
 
-    r.XMM_D(0) = float64_sub(d->XMM_D(0), d->XMM_D(1), &env->sse_status);
-    r.XMM_D(1) = float64_sub(s->XMM_D(0), s->XMM_D(1), &env->sse_status);
+    r.ZMM_D(0) = float64_sub(d->ZMM_D(0), d->ZMM_D(1), &env->sse_status);
+    r.ZMM_D(1) = float64_sub(s->ZMM_D(0), s->ZMM_D(1), &env->sse_status);
     *d = r;
 }
 
-void helper_addsubps(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_addsubps(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_S(0) = float32_sub(d->XMM_S(0), s->XMM_S(0), &env->sse_status);
-    d->XMM_S(1) = float32_add(d->XMM_S(1), s->XMM_S(1), &env->sse_status);
-    d->XMM_S(2) = float32_sub(d->XMM_S(2), s->XMM_S(2), &env->sse_status);
-    d->XMM_S(3) = float32_add(d->XMM_S(3), s->XMM_S(3), &env->sse_status);
+    d->ZMM_S(0) = float32_sub(d->ZMM_S(0), s->ZMM_S(0), &env->sse_status);
+    d->ZMM_S(1) = float32_add(d->ZMM_S(1), s->ZMM_S(1), &env->sse_status);
+    d->ZMM_S(2) = float32_sub(d->ZMM_S(2), s->ZMM_S(2), &env->sse_status);
+    d->ZMM_S(3) = float32_add(d->ZMM_S(3), s->ZMM_S(3), &env->sse_status);
 }
 
-void helper_addsubpd(CPUX86State *env, XMMReg *d, XMMReg *s)
+void helper_addsubpd(CPUX86State *env, ZMMReg *d, ZMMReg *s)
 {
-    d->XMM_D(0) = float64_sub(d->XMM_D(0), s->XMM_D(0), &env->sse_status);
-    d->XMM_D(1) = float64_add(d->XMM_D(1), s->XMM_D(1), &env->sse_status);
+    d->ZMM_D(0) = float64_sub(d->ZMM_D(0), s->ZMM_D(0), &env->sse_status);
+    d->ZMM_D(1) = float64_add(d->ZMM_D(1), s->ZMM_D(1), &env->sse_status);
 }
 
 /* XXX: unordered */
 #define SSE_HELPER_CMP(name, F)                                         \
     void helper_ ## name ## ps(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
-        d->XMM_L(1) = F(32, d->XMM_S(1), s->XMM_S(1));                  \
-        d->XMM_L(2) = F(32, d->XMM_S(2), s->XMM_S(2));                  \
-        d->XMM_L(3) = F(32, d->XMM_S(3), s->XMM_S(3));                  \
+        d->ZMM_L(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
+        d->ZMM_L(1) = F(32, d->ZMM_S(1), s->ZMM_S(1));                  \
+        d->ZMM_L(2) = F(32, d->ZMM_S(2), s->ZMM_S(2));                  \
+        d->ZMM_L(3) = F(32, d->ZMM_S(3), s->ZMM_S(3));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## ss(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));                  \
+        d->ZMM_L(0) = F(32, d->ZMM_S(0), s->ZMM_S(0));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## pd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
-        d->XMM_Q(1) = F(64, d->XMM_D(1), s->XMM_D(1));                  \
+        d->ZMM_Q(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
+        d->ZMM_Q(1) = F(64, d->ZMM_D(1), s->ZMM_D(1));                  \
     }                                                                   \
                                                                         \
     void helper_ ## name ## sd(CPUX86State *env, Reg *d, Reg *s)        \
     {                                                                   \
-        d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));                  \
+        d->ZMM_Q(0) = F(64, d->ZMM_D(0), s->ZMM_D(0));                  \
     }
 
 #define FPU_CMPEQ(size, a, b)                                           \
@@ -1002,8 +1002,8 @@
     int ret;
     float32 s0, s1;
 
-    s0 = d->XMM_S(0);
-    s1 = s->XMM_S(0);
+    s0 = d->ZMM_S(0);
+    s1 = s->ZMM_S(0);
     ret = float32_compare_quiet(s0, s1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1013,8 +1013,8 @@
     int ret;
     float32 s0, s1;
 
-    s0 = d->XMM_S(0);
-    s1 = s->XMM_S(0);
+    s0 = d->ZMM_S(0);
+    s1 = s->ZMM_S(0);
     ret = float32_compare(s0, s1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1024,8 +1024,8 @@
     int ret;
     float64 d0, d1;
 
-    d0 = d->XMM_D(0);
-    d1 = s->XMM_D(0);
+    d0 = d->ZMM_D(0);
+    d1 = s->ZMM_D(0);
     ret = float64_compare_quiet(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1035,8 +1035,8 @@
     int ret;
     float64 d0, d1;
 
-    d0 = d->XMM_D(0);
-    d1 = s->XMM_D(0);
+    d0 = d->ZMM_D(0);
+    d1 = s->ZMM_D(0);
     ret = float64_compare(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
 }
@@ -1045,10 +1045,10 @@
 {
     int b0, b1, b2, b3;
 
-    b0 = s->XMM_L(0) >> 31;
-    b1 = s->XMM_L(1) >> 31;
-    b2 = s->XMM_L(2) >> 31;
-    b3 = s->XMM_L(3) >> 31;
+    b0 = s->ZMM_L(0) >> 31;
+    b1 = s->ZMM_L(1) >> 31;
+    b2 = s->ZMM_L(2) >> 31;
+    b3 = s->ZMM_L(3) >> 31;
     return b0 | (b1 << 1) | (b2 << 2) | (b3 << 3);
 }
 
@@ -1056,8 +1056,8 @@
 {
     int b0, b1;
 
-    b0 = s->XMM_L(1) >> 31;
-    b1 = s->XMM_L(3) >> 31;
+    b0 = s->ZMM_L(1) >> 31;
+    b1 = s->ZMM_L(3) >> 31;
     return b0 | (b1 << 1);
 }
 
@@ -1736,10 +1736,10 @@
         }
     }
 
-    d->XMM_S(0) = float32_round_to_int(s->XMM_S(0), &env->sse_status);
-    d->XMM_S(1) = float32_round_to_int(s->XMM_S(1), &env->sse_status);
-    d->XMM_S(2) = float32_round_to_int(s->XMM_S(2), &env->sse_status);
-    d->XMM_S(3) = float32_round_to_int(s->XMM_S(3), &env->sse_status);
+    d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status);
+    d->ZMM_S(1) = float32_round_to_int(s->ZMM_S(1), &env->sse_status);
+    d->ZMM_S(2) = float32_round_to_int(s->ZMM_S(2), &env->sse_status);
+    d->ZMM_S(3) = float32_round_to_int(s->ZMM_S(3), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1774,8 +1774,8 @@
         }
     }
 
-    d->XMM_D(0) = float64_round_to_int(s->XMM_D(0), &env->sse_status);
-    d->XMM_D(1) = float64_round_to_int(s->XMM_D(1), &env->sse_status);
+    d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status);
+    d->ZMM_D(1) = float64_round_to_int(s->ZMM_D(1), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1810,7 +1810,7 @@
         }
     }
 
-    d->XMM_S(0) = float32_round_to_int(s->XMM_S(0), &env->sse_status);
+    d->ZMM_S(0) = float32_round_to_int(s->ZMM_S(0), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1845,7 +1845,7 @@
         }
     }
 
-    d->XMM_D(0) = float64_round_to_int(s->XMM_D(0), &env->sse_status);
+    d->ZMM_D(0) = float64_round_to_int(s->ZMM_D(0), &env->sse_status);
 
 #if 0 /* TODO */
     if (mode & (1 << 3)) {
@@ -1868,32 +1868,32 @@
 
     if (mask & (1 << 4)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(0), s->XMM_S(0),
+                              float32_mul(d->ZMM_S(0), s->ZMM_S(0),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 5)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(1), s->XMM_S(1),
+                              float32_mul(d->ZMM_S(1), s->ZMM_S(1),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 6)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(2), s->XMM_S(2),
+                              float32_mul(d->ZMM_S(2), s->ZMM_S(2),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 7)) {
         iresult = float32_add(iresult,
-                              float32_mul(d->XMM_S(3), s->XMM_S(3),
+                              float32_mul(d->ZMM_S(3), s->ZMM_S(3),
                                           &env->sse_status),
                               &env->sse_status);
     }
-    d->XMM_S(0) = (mask & (1 << 0)) ? iresult : float32_zero;
-    d->XMM_S(1) = (mask & (1 << 1)) ? iresult : float32_zero;
-    d->XMM_S(2) = (mask & (1 << 2)) ? iresult : float32_zero;
-    d->XMM_S(3) = (mask & (1 << 3)) ? iresult : float32_zero;
+    d->ZMM_S(0) = (mask & (1 << 0)) ? iresult : float32_zero;
+    d->ZMM_S(1) = (mask & (1 << 1)) ? iresult : float32_zero;
+    d->ZMM_S(2) = (mask & (1 << 2)) ? iresult : float32_zero;
+    d->ZMM_S(3) = (mask & (1 << 3)) ? iresult : float32_zero;
 }
 
 void glue(helper_dppd, SUFFIX)(CPUX86State *env, Reg *d, Reg *s, uint32_t mask)
@@ -1902,18 +1902,18 @@
 
     if (mask & (1 << 4)) {
         iresult = float64_add(iresult,
-                              float64_mul(d->XMM_D(0), s->XMM_D(0),
+                              float64_mul(d->ZMM_D(0), s->ZMM_D(0),
                                           &env->sse_status),
                               &env->sse_status);
     }
     if (mask & (1 << 5)) {
         iresult = float64_add(iresult,
-                              float64_mul(d->XMM_D(1), s->XMM_D(1),
+                              float64_mul(d->ZMM_D(1), s->ZMM_D(1),
                                           &env->sse_status),
                               &env->sse_status);
     }
-    d->XMM_D(0) = (mask & (1 << 0)) ? iresult : float64_zero;
-    d->XMM_D(1) = (mask & (1 << 1)) ? iresult : float64_zero;
+    d->ZMM_D(0) = (mask & (1 << 0)) ? iresult : float64_zero;
+    d->ZMM_D(1) = (mask & (1 << 1)) ? iresult : float64_zero;
 }
 
 void glue(helper_mpsadbw, SUFFIX)(CPUX86State *env, Reg *d, Reg *s,
@@ -2037,10 +2037,10 @@
         }
         break;
     case 3:
-        for (j = valids - validd; j >= 0; j--) {
+        for (j = valids; j >= 0; j--) {
             res <<= 1;
             v = 1;
-            for (i = MIN(upper - j, validd); i >= 0; i--) {
+            for (i = MIN(valids - j, validd); i >= 0; i--) {
                 v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i));
             }
             res |= v;
@@ -2228,7 +2228,7 @@
     Reg rk = *s;
 
     for (i = 0; i < 16; i++) {
-        d->B(i) = rk.B(i) ^ (AES_Td4[st.B(AES_ishifts[i])] & 0xff);
+        d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i])]);
     }
 }
 
@@ -2253,7 +2253,7 @@
     Reg rk = *s;
 
     for (i = 0; i < 16; i++) {
-        d->B(i) = rk.B(i) ^ (AES_Te4[st.B(AES_shifts[i])] & 0xff);
+        d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i])]);
     }
 
 }
@@ -2264,10 +2264,10 @@
     Reg tmp = *s;
 
     for (i = 0 ; i < 4 ; i++) {
-        d->L(i) = bswap32(AES_Td0[AES_Te4[tmp.B(4*i+0)] & 0xff] ^
-                          AES_Td1[AES_Te4[tmp.B(4*i+1)] & 0xff] ^
-                          AES_Td2[AES_Te4[tmp.B(4*i+2)] & 0xff] ^
-                          AES_Td3[AES_Te4[tmp.B(4*i+3)] & 0xff]);
+        d->L(i) = bswap32(AES_imc[tmp.B(4*i+0)][0] ^
+                          AES_imc[tmp.B(4*i+1)][1] ^
+                          AES_imc[tmp.B(4*i+2)][2] ^
+                          AES_imc[tmp.B(4*i+3)][3]);
     }
 }
 
@@ -2278,8 +2278,8 @@
     Reg tmp = *s;
 
     for (i = 0 ; i < 4 ; i++) {
-        d->B(i) = AES_Te4[tmp.B(i + 4)] & 0xff;
-        d->B(i + 8) = AES_Te4[tmp.B(i + 12)] & 0xff;
+        d->B(i) = AES_sbox[tmp.B(i + 4)];
+        d->B(i + 8) = AES_sbox[tmp.B(i + 12)];
     }
     d->L(1) = (d->L(0) << 24 | d->L(0) >> 8) ^ ctrl;
     d->L(3) = (d->L(2) << 24 | d->L(2) >> 8) ^ ctrl;
diff --git a/target-i386/ops_sse_header.h b/target-i386/ops_sse_header.h
index a68c7cc..64c5857 100644
--- a/target-i386/ops_sse_header.h
+++ b/target-i386/ops_sse_header.h
@@ -20,18 +20,18 @@
 #define Reg MMXReg
 #define SUFFIX _mmx
 #else
-#define Reg XMMReg
+#define Reg ZMMReg
 #define SUFFIX _xmm
 #endif
 
 #define dh_alias_Reg ptr
-#define dh_alias_XMMReg ptr
+#define dh_alias_ZMMReg ptr
 #define dh_alias_MMXReg ptr
 #define dh_ctype_Reg Reg *
-#define dh_ctype_XMMReg XMMReg *
+#define dh_ctype_ZMMReg ZMMReg *
 #define dh_ctype_MMXReg MMXReg *
 #define dh_is_signed_Reg dh_is_signed_ptr
-#define dh_is_signed_XMMReg dh_is_signed_ptr
+#define dh_is_signed_ZMMReg dh_is_signed_ptr
 #define dh_is_signed_MMXReg dh_is_signed_ptr
 
 DEF_HELPER_3(glue(psrlw, SUFFIX), void, env, Reg, Reg)
@@ -154,52 +154,52 @@
 DEF_HELPER_3(cvtsd2ss, void, env, Reg, Reg)
 DEF_HELPER_3(cvtdq2ps, void, env, Reg, Reg)
 DEF_HELPER_3(cvtdq2pd, void, env, Reg, Reg)
-DEF_HELPER_3(cvtpi2ps, void, env, XMMReg, MMXReg)
-DEF_HELPER_3(cvtpi2pd, void, env, XMMReg, MMXReg)
-DEF_HELPER_3(cvtsi2ss, void, env, XMMReg, i32)
-DEF_HELPER_3(cvtsi2sd, void, env, XMMReg, i32)
+DEF_HELPER_3(cvtpi2ps, void, env, ZMMReg, MMXReg)
+DEF_HELPER_3(cvtpi2pd, void, env, ZMMReg, MMXReg)
+DEF_HELPER_3(cvtsi2ss, void, env, ZMMReg, i32)
+DEF_HELPER_3(cvtsi2sd, void, env, ZMMReg, i32)
 
 #ifdef TARGET_X86_64
-DEF_HELPER_3(cvtsq2ss, void, env, XMMReg, i64)
-DEF_HELPER_3(cvtsq2sd, void, env, XMMReg, i64)
+DEF_HELPER_3(cvtsq2ss, void, env, ZMMReg, i64)
+DEF_HELPER_3(cvtsq2sd, void, env, ZMMReg, i64)
 #endif
 
-DEF_HELPER_3(cvtps2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvtpd2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvtps2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_2(cvtss2si, s32, env, XMMReg)
-DEF_HELPER_2(cvtsd2si, s32, env, XMMReg)
+DEF_HELPER_3(cvtps2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvtpd2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvtps2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_3(cvtpd2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_2(cvtss2si, s32, env, ZMMReg)
+DEF_HELPER_2(cvtsd2si, s32, env, ZMMReg)
 #ifdef TARGET_X86_64
-DEF_HELPER_2(cvtss2sq, s64, env, XMMReg)
-DEF_HELPER_2(cvtsd2sq, s64, env, XMMReg)
+DEF_HELPER_2(cvtss2sq, s64, env, ZMMReg)
+DEF_HELPER_2(cvtsd2sq, s64, env, ZMMReg)
 #endif
 
-DEF_HELPER_3(cvttps2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvttpd2dq, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(cvttps2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, XMMReg)
-DEF_HELPER_2(cvttss2si, s32, env, XMMReg)
-DEF_HELPER_2(cvttsd2si, s32, env, XMMReg)
+DEF_HELPER_3(cvttps2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvttpd2dq, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(cvttps2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_3(cvttpd2pi, void, env, MMXReg, ZMMReg)
+DEF_HELPER_2(cvttss2si, s32, env, ZMMReg)
+DEF_HELPER_2(cvttsd2si, s32, env, ZMMReg)
 #ifdef TARGET_X86_64
-DEF_HELPER_2(cvttss2sq, s64, env, XMMReg)
-DEF_HELPER_2(cvttsd2sq, s64, env, XMMReg)
+DEF_HELPER_2(cvttss2sq, s64, env, ZMMReg)
+DEF_HELPER_2(cvttsd2sq, s64, env, ZMMReg)
 #endif
 
-DEF_HELPER_3(rsqrtps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(rsqrtss, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(rcpps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(rcpss, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(extrq_r, void, env, XMMReg, XMMReg)
-DEF_HELPER_4(extrq_i, void, env, XMMReg, int, int)
-DEF_HELPER_3(insertq_r, void, env, XMMReg, XMMReg)
-DEF_HELPER_4(insertq_i, void, env, XMMReg, int, int)
-DEF_HELPER_3(haddps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(haddpd, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(hsubps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(hsubpd, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(addsubps, void, env, XMMReg, XMMReg)
-DEF_HELPER_3(addsubpd, void, env, XMMReg, XMMReg)
+DEF_HELPER_3(rsqrtps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(rsqrtss, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(rcpps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(rcpss, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(extrq_r, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_4(extrq_i, void, env, ZMMReg, int, int)
+DEF_HELPER_3(insertq_r, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_4(insertq_i, void, env, ZMMReg, int, int)
+DEF_HELPER_3(haddps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(haddpd, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(hsubps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(hsubpd, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(addsubps, void, env, ZMMReg, ZMMReg)
+DEF_HELPER_3(addsubpd, void, env, ZMMReg, ZMMReg)
 
 #define SSE_HELPER_CMP(name, F)                           \
     DEF_HELPER_3(name ## ps, void, env, Reg, Reg)         \
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index 1108bfe..8b08393 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -18,11 +18,17 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/log.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "exec/log.h"
+#ifdef CONFIG_HAX
+#include "target-i386/hax-i386.h"
 #include "sysemu/hax.h"
+#endif
 
 //#define DEBUG_PCALL
 
@@ -35,7 +41,21 @@
 # define LOG_PCALL_STATE(cpu) do { } while (0)
 #endif
 
-#ifndef CONFIG_USER_ONLY
+#ifdef CONFIG_USER_ONLY
+#define MEMSUFFIX _kernel
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_useronly_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_useronly_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_useronly_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_useronly_template.h"
+#undef MEMSUFFIX
+#else
 #define CPU_MMU_INDEX (cpu_mmu_index_kernel(env))
 #define MEMSUFFIX _kernel
 #define DATA_SIZE 1
@@ -54,8 +74,9 @@
 #endif
 
 /* return non zero if error */
-static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
-                               uint32_t *e2_ptr, int selector)
+static inline int load_segment_ra(CPUX86State *env, uint32_t *e1_ptr,
+                               uint32_t *e2_ptr, int selector,
+                               uintptr_t retaddr)
 {
     SegmentCache *dt;
     int index;
@@ -71,11 +92,17 @@
         return -1;
     }
     ptr = dt->base + index;
-    *e1_ptr = cpu_ldl_kernel(env, ptr);
-    *e2_ptr = cpu_ldl_kernel(env, ptr + 4);
+    *e1_ptr = cpu_ldl_kernel_ra(env, ptr, retaddr);
+    *e2_ptr = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
     return 0;
 }
 
+static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
+                               uint32_t *e2_ptr, int selector)
+{
+    return load_segment_ra(env, e1_ptr, e2_ptr, selector, 0);
+}
+
 static inline unsigned int get_seg_limit(uint32_t e1, uint32_t e2)
 {
     unsigned int limit;
@@ -111,7 +138,8 @@
 }
 
 static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,
-                                       uint32_t *esp_ptr, int dpl)
+                                       uint32_t *esp_ptr, int dpl,
+                                       uintptr_t retaddr)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
     int type, index, shift;
@@ -140,60 +168,61 @@
     shift = type >> 3;
     index = (dpl * 4 + 2) << shift;
     if (index + (4 << shift) - 1 > env->tr.limit) {
-        raise_exception_err(env, EXCP0A_TSS, env->tr.selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, env->tr.selector & 0xfffc, retaddr);
     }
     if (shift == 0) {
-        *esp_ptr = cpu_lduw_kernel(env, env->tr.base + index);
-        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 2);
+        *esp_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index, retaddr);
+        *ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 2, retaddr);
     } else {
-        *esp_ptr = cpu_ldl_kernel(env, env->tr.base + index);
-        *ss_ptr = cpu_lduw_kernel(env, env->tr.base + index + 4);
+        *esp_ptr = cpu_ldl_kernel_ra(env, env->tr.base + index, retaddr);
+        *ss_ptr = cpu_lduw_kernel_ra(env, env->tr.base + index + 4, retaddr);
     }
 }
 
-static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
+static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl,
+                         uintptr_t retaddr)
 {
     uint32_t e1, e2;
     int rpl, dpl;
 
     if ((selector & 0xfffc) != 0) {
-        if (load_segment(env, &e1, &e2, selector) != 0) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, selector, retaddr) != 0) {
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
         if (!(e2 & DESC_S_MASK)) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (seg_reg == R_CS) {
             if (!(e2 & DESC_CS_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             if (dpl != rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
         } else if (seg_reg == R_SS) {
             /* SS must be writable data */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             if (dpl != cpl || dpl != rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
         } else {
             /* not readable code */
             if ((e2 & DESC_CS_MASK) && !(e2 & DESC_R_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
             }
             /* if data or non conforming code, checks the rights */
             if (((e2 >> DESC_TYPE_SHIFT) & 0xf) < 12) {
                 if (dpl < cpl || dpl < rpl) {
-                    raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+                    raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
                 }
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, retaddr);
         }
         cpu_x86_load_seg_cache(env, seg_reg, selector,
                                get_seg_base(e1, e2),
@@ -201,7 +230,7 @@
                                e2);
     } else {
         if (seg_reg == R_SS || seg_reg == R_CS) {
-            raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, selector & 0xfffc, retaddr);
         }
     }
 }
@@ -211,9 +240,9 @@
 #define SWITCH_TSS_CALL 2
 
 /* XXX: restore CPU state in registers (PowerPC case) */
-static void switch_tss(CPUX86State *env, int tss_selector,
-                       uint32_t e1, uint32_t e2, int source,
-                       uint32_t next_eip)
+static void switch_tss_ra(CPUX86State *env, int tss_selector,
+                          uint32_t e1, uint32_t e2, int source,
+                          uint32_t next_eip, uintptr_t retaddr)
 {
     int tss_limit, tss_limit_max, type, old_tss_limit_max, old_type, v1, v2, i;
     target_ulong tss_base;
@@ -231,26 +260,26 @@
     /* if task gate, we read the TSS segment and we load it */
     if (type == 5) {
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
         }
         tss_selector = e1 >> 16;
         if (tss_selector & 4) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
         }
-        if (load_segment(env, &e1, &e2, tss_selector) != 0) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, tss_selector, retaddr) != 0) {
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
         if (e2 & DESC_S_MASK) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
         type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
         if ((type & 7) != 1) {
-            raise_exception_err(env, EXCP0D_GPF, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, tss_selector & 0xfffc, retaddr);
         }
     }
 
     if (!(e2 & DESC_P_MASK)) {
-        raise_exception_err(env, EXCP0B_NOSEG, tss_selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0B_NOSEG, tss_selector & 0xfffc, retaddr);
     }
 
     if (type & 8) {
@@ -262,7 +291,7 @@
     tss_base = get_seg_base(e1, e2);
     if ((tss_selector & 4) != 0 ||
         tss_limit < tss_limit_max) {
-        raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, retaddr);
     }
     old_type = (env->tr.flags >> DESC_TYPE_SHIFT) & 0xf;
     if (old_type & 8) {
@@ -274,30 +303,33 @@
     /* read all the registers from the new TSS */
     if (type & 8) {
         /* 32 bit */
-        new_cr3 = cpu_ldl_kernel(env, tss_base + 0x1c);
-        new_eip = cpu_ldl_kernel(env, tss_base + 0x20);
-        new_eflags = cpu_ldl_kernel(env, tss_base + 0x24);
+        new_cr3 = cpu_ldl_kernel_ra(env, tss_base + 0x1c, retaddr);
+        new_eip = cpu_ldl_kernel_ra(env, tss_base + 0x20, retaddr);
+        new_eflags = cpu_ldl_kernel_ra(env, tss_base + 0x24, retaddr);
         for (i = 0; i < 8; i++) {
-            new_regs[i] = cpu_ldl_kernel(env, tss_base + (0x28 + i * 4));
+            new_regs[i] = cpu_ldl_kernel_ra(env, tss_base + (0x28 + i * 4),
+                                            retaddr);
         }
         for (i = 0; i < 6; i++) {
-            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x48 + i * 4));
+            new_segs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x48 + i * 4),
+                                             retaddr);
         }
-        new_ldt = cpu_lduw_kernel(env, tss_base + 0x60);
-        new_trap = cpu_ldl_kernel(env, tss_base + 0x64);
+        new_ldt = cpu_lduw_kernel_ra(env, tss_base + 0x60, retaddr);
+        new_trap = cpu_ldl_kernel_ra(env, tss_base + 0x64, retaddr);
     } else {
         /* 16 bit */
         new_cr3 = 0;
-        new_eip = cpu_lduw_kernel(env, tss_base + 0x0e);
-        new_eflags = cpu_lduw_kernel(env, tss_base + 0x10);
+        new_eip = cpu_lduw_kernel_ra(env, tss_base + 0x0e, retaddr);
+        new_eflags = cpu_lduw_kernel_ra(env, tss_base + 0x10, retaddr);
         for (i = 0; i < 8; i++) {
-            new_regs[i] = cpu_lduw_kernel(env, tss_base + (0x12 + i * 2)) |
-                0xffff0000;
+            new_regs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x12 + i * 2),
+                                             retaddr) | 0xffff0000;
         }
         for (i = 0; i < 4; i++) {
-            new_segs[i] = cpu_lduw_kernel(env, tss_base + (0x22 + i * 4));
+            new_segs[i] = cpu_lduw_kernel_ra(env, tss_base + (0x22 + i * 4),
+                                             retaddr);
         }
-        new_ldt = cpu_lduw_kernel(env, tss_base + 0x2a);
+        new_ldt = cpu_lduw_kernel_ra(env, tss_base + 0x2a, retaddr);
         new_segs[R_FS] = 0;
         new_segs[R_GS] = 0;
         new_trap = 0;
@@ -312,10 +344,10 @@
     /* XXX: it can still fail in some cases, so a bigger hack is
        necessary to valid the TLB after having done the accesses */
 
-    v1 = cpu_ldub_kernel(env, env->tr.base);
-    v2 = cpu_ldub_kernel(env, env->tr.base + old_tss_limit_max);
-    cpu_stb_kernel(env, env->tr.base, v1);
-    cpu_stb_kernel(env, env->tr.base + old_tss_limit_max, v2);
+    v1 = cpu_ldub_kernel_ra(env, env->tr.base, retaddr);
+    v2 = cpu_ldub_kernel_ra(env, env->tr.base + old_tss_limit_max, retaddr);
+    cpu_stb_kernel_ra(env, env->tr.base, v1, retaddr);
+    cpu_stb_kernel_ra(env, env->tr.base + old_tss_limit_max, v2, retaddr);
 
     /* clear busy bit (it is restartable) */
     if (source == SWITCH_TSS_JMP || source == SWITCH_TSS_IRET) {
@@ -323,9 +355,9 @@
         uint32_t e2;
 
         ptr = env->gdt.base + (env->tr.selector & ~7);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         e2 &= ~DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
     }
     old_eflags = cpu_compute_eflags(env);
     if (source == SWITCH_TSS_IRET) {
@@ -335,35 +367,35 @@
     /* save the current state in the old TSS */
     if (type & 8) {
         /* 32 bit */
-        cpu_stl_kernel(env, env->tr.base + 0x20, next_eip);
-        cpu_stl_kernel(env, env->tr.base + 0x24, old_eflags);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI]);
-        cpu_stl_kernel(env, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI]);
+        cpu_stl_kernel_ra(env, env->tr.base + 0x20, next_eip, retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + 0x24, old_eflags, retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 0 * 4), env->regs[R_EAX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 1 * 4), env->regs[R_ECX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 2 * 4), env->regs[R_EDX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 3 * 4), env->regs[R_EBX], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 4 * 4), env->regs[R_ESP], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 5 * 4), env->regs[R_EBP], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 6 * 4), env->regs[R_ESI], retaddr);
+        cpu_stl_kernel_ra(env, env->tr.base + (0x28 + 7 * 4), env->regs[R_EDI], retaddr);
         for (i = 0; i < 6; i++) {
-            cpu_stw_kernel(env, env->tr.base + (0x48 + i * 4),
-                           env->segs[i].selector);
+            cpu_stw_kernel_ra(env, env->tr.base + (0x48 + i * 4),
+                              env->segs[i].selector, retaddr);
         }
     } else {
         /* 16 bit */
-        cpu_stw_kernel(env, env->tr.base + 0x0e, next_eip);
-        cpu_stw_kernel(env, env->tr.base + 0x10, old_eflags);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI]);
-        cpu_stw_kernel(env, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI]);
+        cpu_stw_kernel_ra(env, env->tr.base + 0x0e, next_eip, retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + 0x10, old_eflags, retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 0 * 2), env->regs[R_EAX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 1 * 2), env->regs[R_ECX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 2 * 2), env->regs[R_EDX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 3 * 2), env->regs[R_EBX], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 4 * 2), env->regs[R_ESP], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 5 * 2), env->regs[R_EBP], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 6 * 2), env->regs[R_ESI], retaddr);
+        cpu_stw_kernel_ra(env, env->tr.base + (0x12 + 7 * 2), env->regs[R_EDI], retaddr);
         for (i = 0; i < 4; i++) {
-            cpu_stw_kernel(env, env->tr.base + (0x22 + i * 4),
-                           env->segs[i].selector);
+            cpu_stw_kernel_ra(env, env->tr.base + (0x22 + i * 4),
+                              env->segs[i].selector, retaddr);
         }
     }
 
@@ -371,7 +403,7 @@
        context */
 
     if (source == SWITCH_TSS_CALL) {
-        cpu_stw_kernel(env, tss_base, env->tr.selector);
+        cpu_stw_kernel_ra(env, tss_base, env->tr.selector, retaddr);
         new_eflags |= NT_MASK;
     }
 
@@ -381,9 +413,9 @@
         uint32_t e2;
 
         ptr = env->gdt.base + (tss_selector & ~7);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         e2 |= DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, retaddr);
     }
 
     /* set the new CPU state */
@@ -435,23 +467,23 @@
 
     /* load the LDT */
     if (new_ldt & 4) {
-        raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+        raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
     }
 
     if ((new_ldt & 0xfffc) != 0) {
         dt = &env->gdt;
         index = new_ldt & ~7;
         if ((index + 7) > dt->limit) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, retaddr);
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, retaddr);
         if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0A_TSS, new_ldt & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, new_ldt & 0xfffc, retaddr);
         }
         load_seg_cache_raw_dt(&env->ldt, e1, e2);
     }
@@ -459,34 +491,35 @@
     /* load the segments */
     if (!(new_eflags & VM_MASK)) {
         int cpl = new_segs[R_CS] & 3;
-        tss_load_seg(env, R_CS, new_segs[R_CS], cpl);
-        tss_load_seg(env, R_SS, new_segs[R_SS], cpl);
-        tss_load_seg(env, R_ES, new_segs[R_ES], cpl);
-        tss_load_seg(env, R_DS, new_segs[R_DS], cpl);
-        tss_load_seg(env, R_FS, new_segs[R_FS], cpl);
-        tss_load_seg(env, R_GS, new_segs[R_GS], cpl);
+        tss_load_seg(env, R_CS, new_segs[R_CS], cpl, retaddr);
+        tss_load_seg(env, R_SS, new_segs[R_SS], cpl, retaddr);
+        tss_load_seg(env, R_ES, new_segs[R_ES], cpl, retaddr);
+        tss_load_seg(env, R_DS, new_segs[R_DS], cpl, retaddr);
+        tss_load_seg(env, R_FS, new_segs[R_FS], cpl, retaddr);
+        tss_load_seg(env, R_GS, new_segs[R_GS], cpl, retaddr);
     }
 
     /* check that env->eip is in the CS segment limits */
     if (new_eip > env->segs[R_CS].limit) {
         /* XXX: different exception if CALL? */
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
     }
 
 #ifndef CONFIG_USER_ONLY
     /* reset local breakpoints */
     if (env->dr[7] & DR7_LOCAL_BP_MASK) {
-        for (i = 0; i < DR7_MAX_BP; i++) {
-            if (hw_local_breakpoint_enabled(env->dr[7], i) &&
-                !hw_global_breakpoint_enabled(env->dr[7], i)) {
-                hw_breakpoint_remove(env, i);
-            }
-        }
-        env->dr[7] &= ~DR7_LOCAL_BP_MASK;
+        cpu_x86_update_dr7(env, env->dr[7] & ~DR7_LOCAL_BP_MASK);
     }
 #endif
 }
 
+static void switch_tss(CPUX86State *env, int tss_selector,
+                       uint32_t e1, uint32_t e2, int source,
+                        uint32_t next_eip)
+{
+    switch_tss_ra(env, tss_selector, e1, e2, source, next_eip, 0);
+}
+
 static inline unsigned int get_sp_mask(unsigned int e2)
 {
     if (e2 & DESC_B_MASK) {
@@ -536,30 +569,35 @@
 #define SEG_ADDL(ssp, sp, sp_mask) ((uint32_t)((ssp) + (sp & (sp_mask))))
 
 /* XXX: add a is_user flag to have proper security support */
-#define PUSHW(ssp, sp, sp_mask, val)                             \
+#define PUSHW_RA(ssp, sp, sp_mask, val, ra)                      \
     {                                                            \
         sp -= 2;                                                 \
-        cpu_stw_kernel(env, (ssp) + (sp & (sp_mask)), (val));    \
+        cpu_stw_kernel_ra(env, (ssp) + (sp & (sp_mask)), (val), ra); \
     }
 
-#define PUSHL(ssp, sp, sp_mask, val)                                    \
+#define PUSHL_RA(ssp, sp, sp_mask, val, ra)                             \
     {                                                                   \
         sp -= 4;                                                        \
-        cpu_stl_kernel(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val)); \
+        cpu_stl_kernel_ra(env, SEG_ADDL(ssp, sp, sp_mask), (uint32_t)(val), ra); \
     }
 
-#define POPW(ssp, sp, sp_mask, val)                              \
+#define POPW_RA(ssp, sp, sp_mask, val, ra)                       \
     {                                                            \
-        val = cpu_lduw_kernel(env, (ssp) + (sp & (sp_mask)));    \
+        val = cpu_lduw_kernel_ra(env, (ssp) + (sp & (sp_mask)), ra); \
         sp += 2;                                                 \
     }
 
-#define POPL(ssp, sp, sp_mask, val)                                     \
+#define POPL_RA(ssp, sp, sp_mask, val, ra)                              \
     {                                                                   \
-        val = (uint32_t)cpu_ldl_kernel(env, SEG_ADDL(ssp, sp, sp_mask)); \
+        val = (uint32_t)cpu_ldl_kernel_ra(env, SEG_ADDL(ssp, sp, sp_mask), ra); \
         sp += 4;                                                        \
     }
 
+#define PUSHW(ssp, sp, sp_mask, val) PUSHW_RA(ssp, sp, sp_mask, val, 0)
+#define PUSHL(ssp, sp, sp_mask, val) PUSHL_RA(ssp, sp, sp_mask, val, 0)
+#define POPW(ssp, sp, sp_mask, val) POPW_RA(ssp, sp, sp_mask, val, 0)
+#define POPL(ssp, sp, sp_mask, val) POPL_RA(ssp, sp, sp_mask, val, 0)
+
 /* protected mode interrupt */
 static void do_interrupt_protected(CPUX86State *env, int intno, int is_int,
                                    int error_code, unsigned int next_eip,
@@ -660,7 +698,7 @@
     }
     if (!(e2 & DESC_C_MASK) && dpl < cpl) {
         /* to inner privilege */
-        get_ss_esp_from_tss(env, &ss, &esp, dpl);
+        get_ss_esp_from_tss(env, &ss, &esp, dpl, 0);
         if ((ss & 0xfffc) == 0) {
             raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
         }
@@ -778,18 +816,21 @@
 
 #ifdef TARGET_X86_64
 
-#define PUSHQ(sp, val)                          \
+#define PUSHQ_RA(sp, val, ra)                   \
     {                                           \
         sp -= 8;                                \
-        cpu_stq_kernel(env, sp, (val));         \
+        cpu_stq_kernel_ra(env, sp, (val), ra);  \
     }
 
-#define POPQ(sp, val)                           \
+#define POPQ_RA(sp, val, ra)                    \
     {                                           \
-        val = cpu_ldq_kernel(env, sp);          \
+        val = cpu_ldq_kernel_ra(env, sp, ra);   \
         sp += 8;                                \
     }
 
+#define PUSHQ(sp, val) PUSHQ_RA(sp, val, 0)
+#define POPQ(sp, val) POPQ_RA(sp, val, 0)
+
 static inline target_ulong get_rsp_from_tss(CPUX86State *env, int level)
 {
     X86CPU *cpu = x86_env_get_cpu(env);
@@ -948,7 +989,7 @@
     int selector;
 
     if (!(env->efer & MSR_EFER_SCE)) {
-        raise_exception_err(env, EXCP06_ILLOP, 0);
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
     }
     selector = (env->star >> 32) & 0xffff;
     if (env->hflags & HF_LMA_MASK) {
@@ -1003,11 +1044,11 @@
     int cpl, selector;
 
     if (!(env->efer & MSR_EFER_SCE)) {
-        raise_exception_err(env, EXCP06_ILLOP, 0);
+        raise_exception_err_ra(env, EXCP06_ILLOP, 0, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     if (!(env->cr[0] & CR0_PE_MASK) || cpl != 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
     selector = (env->star >> 48) & 0xffff;
     if (env->hflags & HF_LMA_MASK) {
@@ -1030,7 +1071,7 @@
                                    DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
             env->eip = (uint32_t)env->regs[R_ECX];
         }
-        cpu_x86_load_seg_cache(env, R_SS, selector + 8,
+        cpu_x86_load_seg_cache(env, R_SS, (selector + 8) | 3,
                                0, 0xffffffff,
                                DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                                DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
@@ -1043,7 +1084,7 @@
                                DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
                                DESC_CS_MASK | DESC_R_MASK | DESC_A_MASK);
         env->eip = (uint32_t)env->regs[R_ECX];
-        cpu_x86_load_seg_cache(env, R_SS, selector + 8,
+        cpu_x86_load_seg_cache(env, R_SS, (selector + 8) | 3,
                                0, 0xffffffff,
                                DESC_G_MASK | DESC_B_MASK | DESC_P_MASK |
                                DESC_S_MASK | (3 << DESC_DPL_SHIFT) |
@@ -1092,7 +1133,11 @@
 }
 
 #if defined(CONFIG_USER_ONLY)
-/* fake user mode interrupt */
+/* fake user mode interrupt. is_int is TRUE if coming from the int
+ * instruction. next_eip is the env->eip value AFTER the interrupt
+ * instruction. It is only relevant if is_int is TRUE or if intno
+ * is EXCP_SYSCALL.
+ */
 static void do_interrupt_user(CPUX86State *env, int intno, int is_int,
                               int error_code, target_ulong next_eip)
 {
@@ -1131,7 +1176,7 @@
                             int error_code, int is_hw, int rm)
 {
     CPUState *cs = CPU(x86_env_get_cpu(env));
-    uint32_t event_inj = ldl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    uint32_t event_inj = x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                           control.event_inj));
 
     if (!(event_inj & SVM_EVTINJ_VALID)) {
@@ -1145,11 +1190,11 @@
         event_inj = intno | type | SVM_EVTINJ_VALID;
         if (!rm && exception_has_error_code(intno)) {
             event_inj |= SVM_EVTINJ_VALID_ERR;
-            stl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+            x86_stl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                              control.event_inj_err),
                      error_code);
         }
-        stl_phys(cs->as,
+        x86_stl_phys(cs,
                  env->vm_vmcb + offsetof(struct vmcb, control.event_inj),
                  event_inj);
     }
@@ -1227,11 +1272,11 @@
 #if !defined(CONFIG_USER_ONLY)
     if (env->hflags & HF_SVMI_MASK) {
         CPUState *cs = CPU(cpu);
-        uint32_t event_inj = ldl_phys(cs->as, env->vm_vmcb +
+        uint32_t event_inj = x86_ldl_phys(cs, env->vm_vmcb +
                                       offsetof(struct vmcb,
                                                control.event_inj));
 
-        stl_phys(cs->as,
+        x86_stl_phys(cs,
                  env->vm_vmcb + offsetof(struct vmcb, control.event_inj),
                  event_inj & ~SVM_EVTINJ_VALID);
     }
@@ -1281,6 +1326,9 @@
     if (interrupt_request & CPU_INTERRUPT_POLL) {
         cs->interrupt_request &= ~CPU_INTERRUPT_POLL;
         apic_poll_irq(cpu->apic_state);
+        /* Don't process multiple interrupt requests in a single call.
+           This is required to make icount-driven execution deterministic. */
+        return true;
     }
 #endif
     if (interrupt_request & CPU_INTERRUPT_SIPI) {
@@ -1330,7 +1378,7 @@
             int intno;
             /* FIXME: this should respect TPR */
             cpu_svm_check_intercept_param(env, SVM_EXIT_VINTR, 0);
-            intno = ldl_phys(cs->as, env->vm_vmcb
+            intno = x86_ldl_phys(cs, env->vm_vmcb
                              + offsetof(struct vmcb, control.int_vector));
             qemu_log_mask(CPU_LOG_TB_IN_ASM,
                           "Servicing virtual hardware INT=0x%02x\n", intno);
@@ -1344,74 +1392,6 @@
     return ret;
 }
 
-void helper_enter_level(CPUX86State *env, int level, int data32,
-                        target_ulong t1)
-{
-    target_ulong ssp;
-    uint32_t esp_mask, esp, ebp;
-
-    esp_mask = get_sp_mask(env->segs[R_SS].flags);
-    ssp = env->segs[R_SS].base;
-    ebp = env->regs[R_EBP];
-    esp = env->regs[R_ESP];
-    if (data32) {
-        /* 32 bit */
-        esp -= 4;
-        while (--level) {
-            esp -= 4;
-            ebp -= 4;
-            cpu_stl_data(env, ssp + (esp & esp_mask),
-                         cpu_ldl_data(env, ssp + (ebp & esp_mask)));
-        }
-        esp -= 4;
-        cpu_stl_data(env, ssp + (esp & esp_mask), t1);
-    } else {
-        /* 16 bit */
-        esp -= 2;
-        while (--level) {
-            esp -= 2;
-            ebp -= 2;
-            cpu_stw_data(env, ssp + (esp & esp_mask),
-                         cpu_lduw_data(env, ssp + (ebp & esp_mask)));
-        }
-        esp -= 2;
-        cpu_stw_data(env, ssp + (esp & esp_mask), t1);
-    }
-}
-
-#ifdef TARGET_X86_64
-void helper_enter64_level(CPUX86State *env, int level, int data64,
-                          target_ulong t1)
-{
-    target_ulong esp, ebp;
-
-    ebp = env->regs[R_EBP];
-    esp = env->regs[R_ESP];
-
-    if (data64) {
-        /* 64 bit */
-        esp -= 8;
-        while (--level) {
-            esp -= 8;
-            ebp -= 8;
-            cpu_stq_data(env, esp, cpu_ldq_data(env, ebp));
-        }
-        esp -= 8;
-        cpu_stq_data(env, esp, t1);
-    } else {
-        /* 16 bit */
-        esp -= 2;
-        while (--level) {
-            esp -= 2;
-            ebp -= 2;
-            cpu_stw_data(env, esp, cpu_lduw_data(env, ebp));
-        }
-        esp -= 2;
-        cpu_stw_data(env, esp, t1);
-    }
-}
-#endif
-
 void helper_lldt(CPUX86State *env, int selector)
 {
     SegmentCache *dt;
@@ -1426,7 +1406,7 @@
         env->ldt.limit = 0;
     } else {
         if (selector & 0x4) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dt = &env->gdt;
         index = selector & ~7;
@@ -1439,22 +1419,22 @@
             entry_limit = 7;
         }
         if ((index + entry_limit) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
         if ((e2 & DESC_S_MASK) || ((e2 >> DESC_TYPE_SHIFT) & 0xf) != 2) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
             uint32_t e3;
 
-            e3 = cpu_ldl_kernel(env, ptr + 8);
+            e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
             load_seg_cache_raw_dt(&env->ldt, e1, e2);
             env->ldt.base |= (target_ulong)e3 << 32;
         } else
@@ -1481,7 +1461,7 @@
         env->tr.flags = 0;
     } else {
         if (selector & 0x4) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dt = &env->gdt;
         index = selector & ~7;
@@ -1494,27 +1474,27 @@
             entry_limit = 7;
         }
         if ((index + entry_limit) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
         type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
         if ((e2 & DESC_S_MASK) ||
             (type != 1 && type != 9)) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
             uint32_t e3, e4;
 
-            e3 = cpu_ldl_kernel(env, ptr + 8);
-            e4 = cpu_ldl_kernel(env, ptr + 12);
+            e3 = cpu_ldl_kernel_ra(env, ptr + 8, GETPC());
+            e4 = cpu_ldl_kernel_ra(env, ptr + 12, GETPC());
             if ((e4 >> DESC_TYPE_SHIFT) & 0xf) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
             load_seg_cache_raw_dt(&env->tr, e1, e2);
             env->tr.base |= (target_ulong)e3 << 32;
@@ -1524,7 +1504,7 @@
             load_seg_cache_raw_dt(&env->tr, e1, e2);
         }
         e2 |= DESC_TSS_BUSY_MASK;
-        cpu_stl_kernel(env, ptr + 4, e2);
+        cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
     }
     env->tr.selector = selector;
 }
@@ -1547,7 +1527,7 @@
             && (!(env->hflags & HF_CS64_MASK) || cpl == 3)
 #endif
             ) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
         cpu_x86_load_seg_cache(env, seg_reg, selector, 0, 0, 0);
     } else {
@@ -1559,51 +1539,51 @@
         }
         index = selector & ~7;
         if ((index + 7) > dt->limit) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         ptr = dt->base + index;
-        e1 = cpu_ldl_kernel(env, ptr);
-        e2 = cpu_ldl_kernel(env, ptr + 4);
+        e1 = cpu_ldl_kernel_ra(env, ptr, GETPC());
+        e2 = cpu_ldl_kernel_ra(env, ptr + 4, GETPC());
 
         if (!(e2 & DESC_S_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (seg_reg == R_SS) {
             /* must be writable segment */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
             if (rpl != cpl || dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
         } else {
             /* must be readable segment */
             if ((e2 & (DESC_CS_MASK | DESC_R_MASK)) == DESC_CS_MASK) {
-                raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
             }
 
             if (!(e2 & DESC_CS_MASK) || !(e2 & DESC_C_MASK)) {
                 /* if not conforming code, test rights */
                 if (dpl < cpl || dpl < rpl) {
-                    raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+                    raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
                 }
             }
         }
 
         if (!(e2 & DESC_P_MASK)) {
             if (seg_reg == R_SS) {
-                raise_exception_err(env, EXCP0C_STACK, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0C_STACK, selector & 0xfffc, GETPC());
             } else {
-                raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
             }
         }
 
         /* set the access bit if not already set */
         if (!(e2 & DESC_A_MASK)) {
             e2 |= DESC_A_MASK;
-            cpu_stl_kernel(env, ptr + 4, e2);
+            cpu_stl_kernel_ra(env, ptr + 4, e2, GETPC());
         }
 
         cpu_x86_load_seg_cache(env, seg_reg, selector,
@@ -1619,46 +1599,45 @@
 
 /* protected mode jump */
 void helper_ljmp_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
-                           int next_eip_addend)
+                           target_ulong next_eip)
 {
     int gate_cs, type;
     uint32_t e1, e2, cpl, dpl, rpl, limit;
-    target_ulong next_eip;
 
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (e2 & DESC_C_MASK) {
             /* conforming code segment */
             if (dpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         } else {
             /* non conforming code segment */
             rpl = new_cs & 3;
             if (rpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
         }
         limit = get_seg_limit(e1, e2);
         if (new_eip > limit &&
             !(env->hflags & HF_LMA_MASK) && !(e2 & DESC_L_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
                        get_seg_base(e1, e2), limit, e2);
@@ -1674,50 +1653,49 @@
         case 9: /* 386 TSS */
         case 5: /* task gate */
             if (dpl < cpl || dpl < rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
-            next_eip = env->eip + next_eip_addend;
-            switch_tss(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip);
+            switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip, GETPC());
             break;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
             if ((dpl < cpl) || (dpl < rpl)) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (!(e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
             }
             gate_cs = e1 >> 16;
             new_eip = (e1 & 0xffff);
             if (type == 12) {
                 new_eip |= (e2 & 0xffff0000);
             }
-            if (load_segment(env, &e1, &e2, gate_cs) != 0) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+            if (load_segment_ra(env, &e1, &e2, gate_cs, GETPC()) != 0) {
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             dpl = (e2 >> DESC_DPL_SHIFT) & 3;
             /* must be code segment */
             if (((e2 & (DESC_S_MASK | DESC_CS_MASK)) !=
                  (DESC_S_MASK | DESC_CS_MASK))) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             if (((e2 & DESC_C_MASK) && (dpl > cpl)) ||
                 (!(e2 & DESC_C_MASK) && (dpl != cpl))) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             if (!(e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, gate_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, gate_cs & 0xfffc, GETPC());
             }
             limit = get_seg_limit(e1, e2);
             if (new_eip > limit) {
-                raise_exception_err(env, EXCP0D_GPF, 0);
+                raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
             }
             cpu_x86_load_seg_cache(env, R_CS, (gate_cs & 0xfffc) | cpl,
                                    get_seg_base(e1, e2), limit, e2);
             env->eip = new_eip;
             break;
         default:
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             break;
         }
     }
@@ -1736,11 +1714,11 @@
     esp_mask = get_sp_mask(env->segs[R_SS].flags);
     ssp = env->segs[R_SS].base;
     if (shift) {
-        PUSHL(ssp, esp, esp_mask, env->segs[R_CS].selector);
-        PUSHL(ssp, esp, esp_mask, next_eip);
+        PUSHL_RA(ssp, esp, esp_mask, env->segs[R_CS].selector, GETPC());
+        PUSHL_RA(ssp, esp, esp_mask, next_eip, GETPC());
     } else {
-        PUSHW(ssp, esp, esp_mask, env->segs[R_CS].selector);
-        PUSHW(ssp, esp, esp_mask, next_eip);
+        PUSHW_RA(ssp, esp, esp_mask, env->segs[R_CS].selector, GETPC());
+        PUSHW_RA(ssp, esp, esp_mask, next_eip, GETPC());
     }
 
     SET_ESP(esp, esp_mask);
@@ -1751,47 +1729,46 @@
 
 /* protected mode call */
 void helper_lcall_protected(CPUX86State *env, int new_cs, target_ulong new_eip,
-                            int shift, int next_eip_addend)
+                            int shift, target_ulong next_eip)
 {
     int new_stack, i;
     uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
     uint32_t ss = 0, ss_e1 = 0, ss_e2 = 0, sp, type, ss_dpl, sp_mask;
     uint32_t val, limit, old_sp_mask;
-    target_ulong ssp, old_ssp, next_eip;
+    target_ulong ssp, old_ssp;
 
-    next_eip = env->eip + next_eip_addend;
     LOG_PCALL("lcall %04x:%08x s=%d\n", new_cs, (uint32_t)new_eip, shift);
     LOG_PCALL_STATE(CPU(x86_env_get_cpu(env)));
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, GETPC()) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
     }
     cpl = env->hflags & HF_CPL_MASK;
     LOG_PCALL("desc=%08x:%08x\n", e1, e2);
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK)) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (e2 & DESC_C_MASK) {
             /* conforming code segment */
             if (dpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         } else {
             /* non conforming code segment */
             rpl = new_cs & 3;
             if (rpl > cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             if (dpl != cpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, GETPC());
         }
 
 #ifdef TARGET_X86_64
@@ -1801,8 +1778,8 @@
 
             /* 64 bit case */
             rsp = env->regs[R_ESP];
-            PUSHQ(rsp, env->segs[R_CS].selector);
-            PUSHQ(rsp, next_eip);
+            PUSHQ_RA(rsp, env->segs[R_CS].selector, GETPC());
+            PUSHQ_RA(rsp, next_eip, GETPC());
             /* from this point, not restartable */
             env->regs[R_ESP] = rsp;
             cpu_x86_load_seg_cache(env, R_CS, (new_cs & 0xfffc) | cpl,
@@ -1816,16 +1793,16 @@
             sp_mask = get_sp_mask(env->segs[R_SS].flags);
             ssp = env->segs[R_SS].base;
             if (shift) {
-                PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
-                PUSHL(ssp, sp, sp_mask, next_eip);
+                PUSHL_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+                PUSHL_RA(ssp, sp, sp_mask, next_eip, GETPC());
             } else {
-                PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
-                PUSHW(ssp, sp, sp_mask, next_eip);
+                PUSHW_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+                PUSHW_RA(ssp, sp, sp_mask, next_eip, GETPC());
             }
 
             limit = get_seg_limit(e1, e2);
             if (new_eip > limit) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
             /* from this point, not restartable */
             SET_ESP(sp, sp_mask);
@@ -1843,73 +1820,73 @@
         case 9: /* available 386 TSS */
         case 5: /* task gate */
             if (dpl < cpl || dpl < rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             }
-            switch_tss(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip);
+            switch_tss_ra(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip, GETPC());
             return;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
             break;
         default:
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
             break;
         }
         shift = type >> 3;
 
         if (dpl < cpl || dpl < rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, GETPC());
         }
         /* check valid bit */
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG,  new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG,  new_cs & 0xfffc, GETPC());
         }
         selector = e1 >> 16;
         offset = (e2 & 0xffff0000) | (e1 & 0x0000ffff);
         param_count = e2 & 0x1f;
         if ((selector & 0xfffc) == 0) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
 
-        if (load_segment(env, &e1, &e2, selector) != 0) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_S_MASK) || !(e2 & (DESC_CS_MASK))) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
         if (dpl > cpl) {
-            raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, selector & 0xfffc, GETPC());
         }
         if (!(e2 & DESC_P_MASK)) {
-            raise_exception_err(env, EXCP0B_NOSEG, selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0B_NOSEG, selector & 0xfffc, GETPC());
         }
 
         if (!(e2 & DESC_C_MASK) && dpl < cpl) {
             /* to inner privilege */
-            get_ss_esp_from_tss(env, &ss, &sp, dpl);
+            get_ss_esp_from_tss(env, &ss, &sp, dpl, GETPC());
             LOG_PCALL("new ss:esp=%04x:%08x param_count=%d env->regs[R_ESP]="
                       TARGET_FMT_lx "\n", ss, sp, param_count,
                       env->regs[R_ESP]);
             if ((ss & 0xfffc) == 0) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if ((ss & 3) != dpl) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
-            if (load_segment(env, &ss_e1, &ss_e2, ss) != 0) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+            if (load_segment_ra(env, &ss_e1, &ss_e2, ss, GETPC()) != 0) {
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             ss_dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
             if (ss_dpl != dpl) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if (!(ss_e2 & DESC_S_MASK) ||
                 (ss_e2 & DESC_CS_MASK) ||
                 !(ss_e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
             if (!(ss_e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0A_TSS, ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0A_TSS, ss & 0xfffc, GETPC());
             }
 
             /* push_size = ((param_count * 2) + 8) << shift; */
@@ -1920,22 +1897,22 @@
             sp_mask = get_sp_mask(ss_e2);
             ssp = get_seg_base(ss_e1, ss_e2);
             if (shift) {
-                PUSHL(ssp, sp, sp_mask, env->segs[R_SS].selector);
-                PUSHL(ssp, sp, sp_mask, env->regs[R_ESP]);
+                PUSHL_RA(ssp, sp, sp_mask, env->segs[R_SS].selector, GETPC());
+                PUSHL_RA(ssp, sp, sp_mask, env->regs[R_ESP], GETPC());
                 for (i = param_count - 1; i >= 0; i--) {
-                    val = cpu_ldl_kernel(env, old_ssp +
-                                         ((env->regs[R_ESP] + i * 4) &
-                                          old_sp_mask));
-                    PUSHL(ssp, sp, sp_mask, val);
+                    val = cpu_ldl_kernel_ra(env, old_ssp +
+                                            ((env->regs[R_ESP] + i * 4) &
+                                             old_sp_mask), GETPC());
+                    PUSHL_RA(ssp, sp, sp_mask, val, GETPC());
                 }
             } else {
-                PUSHW(ssp, sp, sp_mask, env->segs[R_SS].selector);
-                PUSHW(ssp, sp, sp_mask, env->regs[R_ESP]);
+                PUSHW_RA(ssp, sp, sp_mask, env->segs[R_SS].selector, GETPC());
+                PUSHW_RA(ssp, sp, sp_mask, env->regs[R_ESP], GETPC());
                 for (i = param_count - 1; i >= 0; i--) {
-                    val = cpu_lduw_kernel(env, old_ssp +
-                                          ((env->regs[R_ESP] + i * 2) &
-                                           old_sp_mask));
-                    PUSHW(ssp, sp, sp_mask, val);
+                    val = cpu_lduw_kernel_ra(env, old_ssp +
+                                             ((env->regs[R_ESP] + i * 2) &
+                                              old_sp_mask), GETPC());
+                    PUSHW_RA(ssp, sp, sp_mask, val, GETPC());
                 }
             }
             new_stack = 1;
@@ -1949,11 +1926,11 @@
         }
 
         if (shift) {
-            PUSHL(ssp, sp, sp_mask, env->segs[R_CS].selector);
-            PUSHL(ssp, sp, sp_mask, next_eip);
+            PUSHL_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+            PUSHL_RA(ssp, sp, sp_mask, next_eip, GETPC());
         } else {
-            PUSHW(ssp, sp, sp_mask, env->segs[R_CS].selector);
-            PUSHW(ssp, sp, sp_mask, next_eip);
+            PUSHW_RA(ssp, sp, sp_mask, env->segs[R_CS].selector, GETPC());
+            PUSHW_RA(ssp, sp, sp_mask, next_eip, GETPC());
         }
 
         /* from this point, not restartable */
@@ -1988,15 +1965,15 @@
     ssp = env->segs[R_SS].base;
     if (shift == 1) {
         /* 32 bits */
-        POPL(ssp, sp, sp_mask, new_eip);
-        POPL(ssp, sp, sp_mask, new_cs);
+        POPL_RA(ssp, sp, sp_mask, new_eip, GETPC());
+        POPL_RA(ssp, sp, sp_mask, new_cs, GETPC());
         new_cs &= 0xffff;
-        POPL(ssp, sp, sp_mask, new_eflags);
+        POPL_RA(ssp, sp, sp_mask, new_eflags, GETPC());
     } else {
         /* 16 bits */
-        POPW(ssp, sp, sp_mask, new_eip);
-        POPW(ssp, sp, sp_mask, new_cs);
-        POPW(ssp, sp, sp_mask, new_eflags);
+        POPW_RA(ssp, sp, sp_mask, new_eip, GETPC());
+        POPW_RA(ssp, sp, sp_mask, new_cs, GETPC());
+        POPW_RA(ssp, sp, sp_mask, new_eflags, GETPC());
     }
     env->regs[R_ESP] = (env->regs[R_ESP] & ~sp_mask) | (sp & sp_mask);
     env->segs[R_CS].selector = new_cs;
@@ -2041,7 +2018,8 @@
 
 /* protected mode iret */
 static inline void helper_ret_protected(CPUX86State *env, int shift,
-                                        int is_iret, int addend)
+                                        int is_iret, int addend,
+                                        uintptr_t retaddr)
 {
     uint32_t new_cs, new_eflags, new_ss;
     uint32_t new_es, new_ds, new_fs, new_gs;
@@ -2062,32 +2040,32 @@
     new_eflags = 0; /* avoid warning */
 #ifdef TARGET_X86_64
     if (shift == 2) {
-        POPQ(sp, new_eip);
-        POPQ(sp, new_cs);
+        POPQ_RA(sp, new_eip, retaddr);
+        POPQ_RA(sp, new_cs, retaddr);
         new_cs &= 0xffff;
         if (is_iret) {
-            POPQ(sp, new_eflags);
+            POPQ_RA(sp, new_eflags, retaddr);
         }
     } else
 #endif
     {
         if (shift == 1) {
             /* 32 bits */
-            POPL(ssp, sp, sp_mask, new_eip);
-            POPL(ssp, sp, sp_mask, new_cs);
+            POPL_RA(ssp, sp, sp_mask, new_eip, retaddr);
+            POPL_RA(ssp, sp, sp_mask, new_cs, retaddr);
             new_cs &= 0xffff;
             if (is_iret) {
-                POPL(ssp, sp, sp_mask, new_eflags);
+                POPL_RA(ssp, sp, sp_mask, new_eflags, retaddr);
                 if (new_eflags & VM_MASK) {
                     goto return_to_vm86;
                 }
             }
         } else {
             /* 16 bits */
-            POPW(ssp, sp, sp_mask, new_eip);
-            POPW(ssp, sp, sp_mask, new_cs);
+            POPW_RA(ssp, sp, sp_mask, new_eip, retaddr);
+            POPW_RA(ssp, sp, sp_mask, new_cs, retaddr);
             if (is_iret) {
-                POPW(ssp, sp, sp_mask, new_eflags);
+                POPW_RA(ssp, sp, sp_mask, new_eflags, retaddr);
             }
         }
     }
@@ -2095,32 +2073,32 @@
               new_cs, new_eip, shift, addend);
     LOG_PCALL_STATE(CPU(x86_env_get_cpu(env)));
     if ((new_cs & 0xfffc) == 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
-    if (load_segment(env, &e1, &e2, new_cs) != 0) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+    if (load_segment_ra(env, &e1, &e2, new_cs, retaddr) != 0) {
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     if (!(e2 & DESC_S_MASK) ||
         !(e2 & DESC_CS_MASK)) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     cpl = env->hflags & HF_CPL_MASK;
     rpl = new_cs & 3;
     if (rpl < cpl) {
-        raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
     }
     dpl = (e2 >> DESC_DPL_SHIFT) & 3;
     if (e2 & DESC_C_MASK) {
         if (dpl > rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
         }
     } else {
         if (dpl != rpl) {
-            raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
+            raise_exception_err_ra(env, EXCP0D_GPF, new_cs & 0xfffc, retaddr);
         }
     }
     if (!(e2 & DESC_P_MASK)) {
-        raise_exception_err(env, EXCP0B_NOSEG, new_cs & 0xfffc);
+        raise_exception_err_ra(env, EXCP0B_NOSEG, new_cs & 0xfffc, retaddr);
     }
 
     sp += addend;
@@ -2135,21 +2113,21 @@
         /* return to different privilege level */
 #ifdef TARGET_X86_64
         if (shift == 2) {
-            POPQ(sp, new_esp);
-            POPQ(sp, new_ss);
+            POPQ_RA(sp, new_esp, retaddr);
+            POPQ_RA(sp, new_ss, retaddr);
             new_ss &= 0xffff;
         } else
 #endif
         {
             if (shift == 1) {
                 /* 32 bits */
-                POPL(ssp, sp, sp_mask, new_esp);
-                POPL(ssp, sp, sp_mask, new_ss);
+                POPL_RA(ssp, sp, sp_mask, new_esp, retaddr);
+                POPL_RA(ssp, sp, sp_mask, new_ss, retaddr);
                 new_ss &= 0xffff;
             } else {
                 /* 16 bits */
-                POPW(ssp, sp, sp_mask, new_esp);
-                POPW(ssp, sp, sp_mask, new_ss);
+                POPW_RA(ssp, sp, sp_mask, new_esp, retaddr);
+                POPW_RA(ssp, sp, sp_mask, new_ss, retaddr);
             }
         }
         LOG_PCALL("new ss:esp=%04x:" TARGET_FMT_lx "\n",
@@ -2168,26 +2146,26 @@
             } else
 #endif
             {
-                raise_exception_err(env, EXCP0D_GPF, 0);
+                raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
             }
         } else {
             if ((new_ss & 3) != rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
-            if (load_segment(env, &ss_e1, &ss_e2, new_ss) != 0) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+            if (load_segment_ra(env, &ss_e1, &ss_e2, new_ss, retaddr) != 0) {
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             if (!(ss_e2 & DESC_S_MASK) ||
                 (ss_e2 & DESC_CS_MASK) ||
                 !(ss_e2 & DESC_W_MASK)) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             dpl = (ss_e2 >> DESC_DPL_SHIFT) & 3;
             if (dpl != rpl) {
-                raise_exception_err(env, EXCP0D_GPF, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0D_GPF, new_ss & 0xfffc, retaddr);
             }
             if (!(ss_e2 & DESC_P_MASK)) {
-                raise_exception_err(env, EXCP0B_NOSEG, new_ss & 0xfffc);
+                raise_exception_err_ra(env, EXCP0B_NOSEG, new_ss & 0xfffc, retaddr);
             }
             cpu_x86_load_seg_cache(env, R_SS, new_ss,
                                    get_seg_base(ss_e1, ss_e2),
@@ -2237,12 +2215,12 @@
     return;
 
  return_to_vm86:
-    POPL(ssp, sp, sp_mask, new_esp);
-    POPL(ssp, sp, sp_mask, new_ss);
-    POPL(ssp, sp, sp_mask, new_es);
-    POPL(ssp, sp, sp_mask, new_ds);
-    POPL(ssp, sp, sp_mask, new_fs);
-    POPL(ssp, sp, sp_mask, new_gs);
+    POPL_RA(ssp, sp, sp_mask, new_esp, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_ss, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_es, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_ds, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_fs, retaddr);
+    POPL_RA(ssp, sp, sp_mask, new_gs, retaddr);
 
     /* modify processor state */
     cpu_load_eflags(env, new_eflags, TF_MASK | AC_MASK | ID_MASK |
@@ -2268,37 +2246,37 @@
     if (env->eflags & NT_MASK) {
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
-            raise_exception_err(env, EXCP0D_GPF, 0);
+            raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
         }
 #endif
-        tss_selector = cpu_lduw_kernel(env, env->tr.base + 0);
+        tss_selector = cpu_lduw_kernel_ra(env, env->tr.base + 0, GETPC());
         if (tss_selector & 4) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
-        if (load_segment(env, &e1, &e2, tss_selector) != 0) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+        if (load_segment_ra(env, &e1, &e2, tss_selector, GETPC()) != 0) {
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
         type = (e2 >> DESC_TYPE_SHIFT) & 0x17;
         /* NOTE: we check both segment and busy TSS */
         if (type != 3) {
-            raise_exception_err(env, EXCP0A_TSS, tss_selector & 0xfffc);
+            raise_exception_err_ra(env, EXCP0A_TSS, tss_selector & 0xfffc, GETPC());
         }
-        switch_tss(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip);
+        switch_tss_ra(env, tss_selector, e1, e2, SWITCH_TSS_IRET, next_eip, GETPC());
     } else {
-        helper_ret_protected(env, shift, 1, 0);
+        helper_ret_protected(env, shift, 1, 0, GETPC());
     }
     env->hflags2 &= ~HF2_NMI_MASK;
 }
 
 void helper_lret_protected(CPUX86State *env, int shift, int addend)
 {
-    helper_ret_protected(env, shift, 0, addend);
+    helper_ret_protected(env, shift, 0, addend, GETPC());
 }
 
 void helper_sysenter(CPUX86State *env)
 {
     if (env->sysenter_cs == 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
     env->eflags &= ~(VM_MASK | IF_MASK | RF_MASK);
 
@@ -2334,7 +2312,7 @@
 
     cpl = env->hflags & HF_CPL_MASK;
     if (env->sysenter_cs == 0 || cpl != 0) {
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, GETPC());
     }
 #ifdef TARGET_X86_64
     if (dflag == 2) {
@@ -2378,7 +2356,7 @@
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     rpl = selector & 3;
@@ -2425,7 +2403,7 @@
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     rpl = selector & 3;
@@ -2474,7 +2452,7 @@
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     if (!(e2 & DESC_S_MASK)) {
@@ -2512,7 +2490,7 @@
     if ((selector & 0xfffc) == 0) {
         goto fail;
     }
-    if (load_segment(env, &e1, &e2, selector) != 0) {
+    if (load_segment_ra(env, &e1, &e2, selector, GETPC()) != 0) {
         goto fail;
     }
     if (!(e2 & DESC_S_MASK)) {
@@ -2553,7 +2531,8 @@
 #endif
 
 /* check if Port I/O is allowed in TSS */
-static inline void check_io(CPUX86State *env, int addr, int size)
+static inline void check_io(CPUX86State *env, int addr, int size,
+                            uintptr_t retaddr)
 {
     int io_offset, val, mask;
 
@@ -2563,33 +2542,33 @@
         env->tr.limit < 103) {
         goto fail;
     }
-    io_offset = cpu_lduw_kernel(env, env->tr.base + 0x66);
+    io_offset = cpu_lduw_kernel_ra(env, env->tr.base + 0x66, retaddr);
     io_offset += (addr >> 3);
     /* Note: the check needs two bytes */
     if ((io_offset + 1) > env->tr.limit) {
         goto fail;
     }
-    val = cpu_lduw_kernel(env, env->tr.base + io_offset);
+    val = cpu_lduw_kernel_ra(env, env->tr.base + io_offset, retaddr);
     val >>= (addr & 7);
     mask = (1 << size) - 1;
     /* all bits must be zero to allow the I/O */
     if ((val & mask) != 0) {
     fail:
-        raise_exception_err(env, EXCP0D_GPF, 0);
+        raise_exception_err_ra(env, EXCP0D_GPF, 0, retaddr);
     }
 }
 
 void helper_check_iob(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 1);
+    check_io(env, t0, 1, GETPC());
 }
 
 void helper_check_iow(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 2);
+    check_io(env, t0, 2, GETPC());
 }
 
 void helper_check_iol(CPUX86State *env, uint32_t t0)
 {
-    check_io(env, t0, 4);
+    check_io(env, t0, 4, GETPC());
 }
diff --git a/target-i386/smm_helper.c b/target-i386/smm_helper.c
index 58051d3..4dd6a2c 100644
--- a/target-i386/smm_helper.c
+++ b/target-i386/smm_helper.c
@@ -17,8 +17,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/log.h"
 
 /* SMM support */
 
@@ -40,6 +42,16 @@
 #define SMM_REVISION_ID 0x00020000
 #endif
 
+void cpu_smm_update(X86CPU *cpu)
+{
+    CPUX86State *env = &cpu->env;
+    bool smm_enabled = (env->hflags & HF_SMM_MASK);
+
+    if (cpu->smram) {
+        memory_region_set_enabled(cpu->smram, smm_enabled);
+    }
+}
+
 void do_smm_enter(X86CPU *cpu)
 {
     CPUX86State *env = &cpu->env;
@@ -52,7 +64,12 @@
     log_cpu_state_mask(CPU_LOG_INT, CPU(cpu), CPU_DUMP_CCOP);
 
     env->hflags |= HF_SMM_MASK;
-    cpu_smm_update(env);
+    if (env->hflags2 & HF2_NMI_MASK) {
+        env->hflags2 |= HF2_SMM_INSIDE_NMI_MASK;
+    } else {
+        env->hflags2 |= HF2_NMI_MASK;
+    }
+    cpu_smm_update(cpu);
 
     sm_state = env->smbase + 0x8000;
 
@@ -60,83 +77,87 @@
     for (i = 0; i < 6; i++) {
         dt = &env->segs[i];
         offset = 0x7e00 + i * 16;
-        stw_phys(cs->as, sm_state + offset, dt->selector);
-        stw_phys(cs->as, sm_state + offset + 2, (dt->flags >> 8) & 0xf0ff);
-        stl_phys(cs->as, sm_state + offset + 4, dt->limit);
-        stq_phys(cs->as, sm_state + offset + 8, dt->base);
+        x86_stw_phys(cs, sm_state + offset, dt->selector);
+        x86_stw_phys(cs, sm_state + offset + 2, (dt->flags >> 8) & 0xf0ff);
+        x86_stl_phys(cs, sm_state + offset + 4, dt->limit);
+        x86_stq_phys(cs, sm_state + offset + 8, dt->base);
     }
 
-    stq_phys(cs->as, sm_state + 0x7e68, env->gdt.base);
-    stl_phys(cs->as, sm_state + 0x7e64, env->gdt.limit);
+    x86_stq_phys(cs, sm_state + 0x7e68, env->gdt.base);
+    x86_stl_phys(cs, sm_state + 0x7e64, env->gdt.limit);
 
-    stw_phys(cs->as, sm_state + 0x7e70, env->ldt.selector);
-    stq_phys(cs->as, sm_state + 0x7e78, env->ldt.base);
-    stl_phys(cs->as, sm_state + 0x7e74, env->ldt.limit);
-    stw_phys(cs->as, sm_state + 0x7e72, (env->ldt.flags >> 8) & 0xf0ff);
+    x86_stw_phys(cs, sm_state + 0x7e70, env->ldt.selector);
+    x86_stq_phys(cs, sm_state + 0x7e78, env->ldt.base);
+    x86_stl_phys(cs, sm_state + 0x7e74, env->ldt.limit);
+    x86_stw_phys(cs, sm_state + 0x7e72, (env->ldt.flags >> 8) & 0xf0ff);
 
-    stq_phys(cs->as, sm_state + 0x7e88, env->idt.base);
-    stl_phys(cs->as, sm_state + 0x7e84, env->idt.limit);
+    x86_stq_phys(cs, sm_state + 0x7e88, env->idt.base);
+    x86_stl_phys(cs, sm_state + 0x7e84, env->idt.limit);
 
-    stw_phys(cs->as, sm_state + 0x7e90, env->tr.selector);
-    stq_phys(cs->as, sm_state + 0x7e98, env->tr.base);
-    stl_phys(cs->as, sm_state + 0x7e94, env->tr.limit);
-    stw_phys(cs->as, sm_state + 0x7e92, (env->tr.flags >> 8) & 0xf0ff);
+    x86_stw_phys(cs, sm_state + 0x7e90, env->tr.selector);
+    x86_stq_phys(cs, sm_state + 0x7e98, env->tr.base);
+    x86_stl_phys(cs, sm_state + 0x7e94, env->tr.limit);
+    x86_stw_phys(cs, sm_state + 0x7e92, (env->tr.flags >> 8) & 0xf0ff);
 
-    stq_phys(cs->as, sm_state + 0x7ed0, env->efer);
+    /* ??? Vol 1, 16.5.6 Intel MPX and SMM says that IA32_BNDCFGS
+       is saved at offset 7ED0.  Vol 3, 34.4.1.1, Table 32-2, has
+       7EA0-7ED7 as "reserved".  What's this, and what's really
+       supposed to happen?  */
+    x86_stq_phys(cs, sm_state + 0x7ed0, env->efer);
 
-    stq_phys(cs->as, sm_state + 0x7ff8, env->regs[R_EAX]);
-    stq_phys(cs->as, sm_state + 0x7ff0, env->regs[R_ECX]);
-    stq_phys(cs->as, sm_state + 0x7fe8, env->regs[R_EDX]);
-    stq_phys(cs->as, sm_state + 0x7fe0, env->regs[R_EBX]);
-    stq_phys(cs->as, sm_state + 0x7fd8, env->regs[R_ESP]);
-    stq_phys(cs->as, sm_state + 0x7fd0, env->regs[R_EBP]);
-    stq_phys(cs->as, sm_state + 0x7fc8, env->regs[R_ESI]);
-    stq_phys(cs->as, sm_state + 0x7fc0, env->regs[R_EDI]);
+    x86_stq_phys(cs, sm_state + 0x7ff8, env->regs[R_EAX]);
+    x86_stq_phys(cs, sm_state + 0x7ff0, env->regs[R_ECX]);
+    x86_stq_phys(cs, sm_state + 0x7fe8, env->regs[R_EDX]);
+    x86_stq_phys(cs, sm_state + 0x7fe0, env->regs[R_EBX]);
+    x86_stq_phys(cs, sm_state + 0x7fd8, env->regs[R_ESP]);
+    x86_stq_phys(cs, sm_state + 0x7fd0, env->regs[R_EBP]);
+    x86_stq_phys(cs, sm_state + 0x7fc8, env->regs[R_ESI]);
+    x86_stq_phys(cs, sm_state + 0x7fc0, env->regs[R_EDI]);
     for (i = 8; i < 16; i++) {
-        stq_phys(cs->as, sm_state + 0x7ff8 - i * 8, env->regs[i]);
+        x86_stq_phys(cs, sm_state + 0x7ff8 - i * 8, env->regs[i]);
     }
-    stq_phys(cs->as, sm_state + 0x7f78, env->eip);
-    stl_phys(cs->as, sm_state + 0x7f70, cpu_compute_eflags(env));
-    stl_phys(cs->as, sm_state + 0x7f68, env->dr[6]);
-    stl_phys(cs->as, sm_state + 0x7f60, env->dr[7]);
+    x86_stq_phys(cs, sm_state + 0x7f78, env->eip);
+    x86_stl_phys(cs, sm_state + 0x7f70, cpu_compute_eflags(env));
+    x86_stl_phys(cs, sm_state + 0x7f68, env->dr[6]);
+    x86_stl_phys(cs, sm_state + 0x7f60, env->dr[7]);
 
-    stl_phys(cs->as, sm_state + 0x7f48, env->cr[4]);
-    stl_phys(cs->as, sm_state + 0x7f50, env->cr[3]);
-    stl_phys(cs->as, sm_state + 0x7f58, env->cr[0]);
+    x86_stl_phys(cs, sm_state + 0x7f48, env->cr[4]);
+    x86_stq_phys(cs, sm_state + 0x7f50, env->cr[3]);
+    x86_stl_phys(cs, sm_state + 0x7f58, env->cr[0]);
 
-    stl_phys(cs->as, sm_state + 0x7efc, SMM_REVISION_ID);
-    stl_phys(cs->as, sm_state + 0x7f00, env->smbase);
+    x86_stl_phys(cs, sm_state + 0x7efc, SMM_REVISION_ID);
+    x86_stl_phys(cs, sm_state + 0x7f00, env->smbase);
 #else
-    stl_phys(cs->as, sm_state + 0x7ffc, env->cr[0]);
-    stl_phys(cs->as, sm_state + 0x7ff8, env->cr[3]);
-    stl_phys(cs->as, sm_state + 0x7ff4, cpu_compute_eflags(env));
-    stl_phys(cs->as, sm_state + 0x7ff0, env->eip);
-    stl_phys(cs->as, sm_state + 0x7fec, env->regs[R_EDI]);
-    stl_phys(cs->as, sm_state + 0x7fe8, env->regs[R_ESI]);
-    stl_phys(cs->as, sm_state + 0x7fe4, env->regs[R_EBP]);
-    stl_phys(cs->as, sm_state + 0x7fe0, env->regs[R_ESP]);
-    stl_phys(cs->as, sm_state + 0x7fdc, env->regs[R_EBX]);
-    stl_phys(cs->as, sm_state + 0x7fd8, env->regs[R_EDX]);
-    stl_phys(cs->as, sm_state + 0x7fd4, env->regs[R_ECX]);
-    stl_phys(cs->as, sm_state + 0x7fd0, env->regs[R_EAX]);
-    stl_phys(cs->as, sm_state + 0x7fcc, env->dr[6]);
-    stl_phys(cs->as, sm_state + 0x7fc8, env->dr[7]);
+    x86_stl_phys(cs, sm_state + 0x7ffc, env->cr[0]);
+    x86_stl_phys(cs, sm_state + 0x7ff8, env->cr[3]);
+    x86_stl_phys(cs, sm_state + 0x7ff4, cpu_compute_eflags(env));
+    x86_stl_phys(cs, sm_state + 0x7ff0, env->eip);
+    x86_stl_phys(cs, sm_state + 0x7fec, env->regs[R_EDI]);
+    x86_stl_phys(cs, sm_state + 0x7fe8, env->regs[R_ESI]);
+    x86_stl_phys(cs, sm_state + 0x7fe4, env->regs[R_EBP]);
+    x86_stl_phys(cs, sm_state + 0x7fe0, env->regs[R_ESP]);
+    x86_stl_phys(cs, sm_state + 0x7fdc, env->regs[R_EBX]);
+    x86_stl_phys(cs, sm_state + 0x7fd8, env->regs[R_EDX]);
+    x86_stl_phys(cs, sm_state + 0x7fd4, env->regs[R_ECX]);
+    x86_stl_phys(cs, sm_state + 0x7fd0, env->regs[R_EAX]);
+    x86_stl_phys(cs, sm_state + 0x7fcc, env->dr[6]);
+    x86_stl_phys(cs, sm_state + 0x7fc8, env->dr[7]);
 
-    stl_phys(cs->as, sm_state + 0x7fc4, env->tr.selector);
-    stl_phys(cs->as, sm_state + 0x7f64, env->tr.base);
-    stl_phys(cs->as, sm_state + 0x7f60, env->tr.limit);
-    stl_phys(cs->as, sm_state + 0x7f5c, (env->tr.flags >> 8) & 0xf0ff);
+    x86_stl_phys(cs, sm_state + 0x7fc4, env->tr.selector);
+    x86_stl_phys(cs, sm_state + 0x7f64, env->tr.base);
+    x86_stl_phys(cs, sm_state + 0x7f60, env->tr.limit);
+    x86_stl_phys(cs, sm_state + 0x7f5c, (env->tr.flags >> 8) & 0xf0ff);
 
-    stl_phys(cs->as, sm_state + 0x7fc0, env->ldt.selector);
-    stl_phys(cs->as, sm_state + 0x7f80, env->ldt.base);
-    stl_phys(cs->as, sm_state + 0x7f7c, env->ldt.limit);
-    stl_phys(cs->as, sm_state + 0x7f78, (env->ldt.flags >> 8) & 0xf0ff);
+    x86_stl_phys(cs, sm_state + 0x7fc0, env->ldt.selector);
+    x86_stl_phys(cs, sm_state + 0x7f80, env->ldt.base);
+    x86_stl_phys(cs, sm_state + 0x7f7c, env->ldt.limit);
+    x86_stl_phys(cs, sm_state + 0x7f78, (env->ldt.flags >> 8) & 0xf0ff);
 
-    stl_phys(cs->as, sm_state + 0x7f74, env->gdt.base);
-    stl_phys(cs->as, sm_state + 0x7f70, env->gdt.limit);
+    x86_stl_phys(cs, sm_state + 0x7f74, env->gdt.base);
+    x86_stl_phys(cs, sm_state + 0x7f70, env->gdt.limit);
 
-    stl_phys(cs->as, sm_state + 0x7f58, env->idt.base);
-    stl_phys(cs->as, sm_state + 0x7f54, env->idt.limit);
+    x86_stl_phys(cs, sm_state + 0x7f58, env->idt.base);
+    x86_stl_phys(cs, sm_state + 0x7f54, env->idt.limit);
 
     for (i = 0; i < 6; i++) {
         dt = &env->segs[i];
@@ -145,15 +166,15 @@
         } else {
             offset = 0x7f2c + (i - 3) * 12;
         }
-        stl_phys(cs->as, sm_state + 0x7fa8 + i * 4, dt->selector);
-        stl_phys(cs->as, sm_state + offset + 8, dt->base);
-        stl_phys(cs->as, sm_state + offset + 4, dt->limit);
-        stl_phys(cs->as, sm_state + offset, (dt->flags >> 8) & 0xf0ff);
+        x86_stl_phys(cs, sm_state + 0x7fa8 + i * 4, dt->selector);
+        x86_stl_phys(cs, sm_state + offset + 8, dt->base);
+        x86_stl_phys(cs, sm_state + offset + 4, dt->limit);
+        x86_stl_phys(cs, sm_state + offset, (dt->flags >> 8) & 0xf0ff);
     }
-    stl_phys(cs->as, sm_state + 0x7f14, env->cr[4]);
+    x86_stl_phys(cs, sm_state + 0x7f14, env->cr[4]);
 
-    stl_phys(cs->as, sm_state + 0x7efc, SMM_REVISION_ID);
-    stl_phys(cs->as, sm_state + 0x7ef8, env->smbase);
+    x86_stl_phys(cs, sm_state + 0x7efc, SMM_REVISION_ID);
+    x86_stl_phys(cs, sm_state + 0x7ef8, env->smbase);
 #endif
     /* init SMM cpu state */
 
@@ -172,22 +193,22 @@
     cpu_x86_load_seg_cache(env, R_CS, (env->smbase >> 4) & 0xffff, env->smbase,
                            0xffffffff,
                            DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
-                           DESC_A_MASK);
+                           DESC_G_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0xffffffff,
                            DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
-                           DESC_A_MASK);
+                           DESC_G_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0xffffffff,
                            DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
-                           DESC_A_MASK);
+                           DESC_G_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_SS, 0, 0, 0xffffffff,
                            DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
-                           DESC_A_MASK);
+                           DESC_G_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0xffffffff,
                            DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
-                           DESC_A_MASK);
+                           DESC_G_MASK | DESC_A_MASK);
     cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0xffffffff,
                            DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
-                           DESC_A_MASK);
+                           DESC_G_MASK | DESC_A_MASK);
 }
 
 void helper_rsm(CPUX86State *env)
@@ -200,91 +221,91 @@
 
     sm_state = env->smbase + 0x8000;
 #ifdef TARGET_X86_64
-    cpu_load_efer(env, ldq_phys(cs->as, sm_state + 0x7ed0));
+    cpu_load_efer(env, x86_ldq_phys(cs, sm_state + 0x7ed0));
 
-    env->gdt.base = ldq_phys(cs->as, sm_state + 0x7e68);
-    env->gdt.limit = ldl_phys(cs->as, sm_state + 0x7e64);
+    env->gdt.base = x86_ldq_phys(cs, sm_state + 0x7e68);
+    env->gdt.limit = x86_ldl_phys(cs, sm_state + 0x7e64);
 
-    env->ldt.selector = lduw_phys(cs->as, sm_state + 0x7e70);
-    env->ldt.base = ldq_phys(cs->as, sm_state + 0x7e78);
-    env->ldt.limit = ldl_phys(cs->as, sm_state + 0x7e74);
-    env->ldt.flags = (lduw_phys(cs->as, sm_state + 0x7e72) & 0xf0ff) << 8;
+    env->ldt.selector = x86_lduw_phys(cs, sm_state + 0x7e70);
+    env->ldt.base = x86_ldq_phys(cs, sm_state + 0x7e78);
+    env->ldt.limit = x86_ldl_phys(cs, sm_state + 0x7e74);
+    env->ldt.flags = (x86_lduw_phys(cs, sm_state + 0x7e72) & 0xf0ff) << 8;
 
-    env->idt.base = ldq_phys(cs->as, sm_state + 0x7e88);
-    env->idt.limit = ldl_phys(cs->as, sm_state + 0x7e84);
+    env->idt.base = x86_ldq_phys(cs, sm_state + 0x7e88);
+    env->idt.limit = x86_ldl_phys(cs, sm_state + 0x7e84);
 
-    env->tr.selector = lduw_phys(cs->as, sm_state + 0x7e90);
-    env->tr.base = ldq_phys(cs->as, sm_state + 0x7e98);
-    env->tr.limit = ldl_phys(cs->as, sm_state + 0x7e94);
-    env->tr.flags = (lduw_phys(cs->as, sm_state + 0x7e92) & 0xf0ff) << 8;
+    env->tr.selector = x86_lduw_phys(cs, sm_state + 0x7e90);
+    env->tr.base = x86_ldq_phys(cs, sm_state + 0x7e98);
+    env->tr.limit = x86_ldl_phys(cs, sm_state + 0x7e94);
+    env->tr.flags = (x86_lduw_phys(cs, sm_state + 0x7e92) & 0xf0ff) << 8;
 
-    env->regs[R_EAX] = ldq_phys(cs->as, sm_state + 0x7ff8);
-    env->regs[R_ECX] = ldq_phys(cs->as, sm_state + 0x7ff0);
-    env->regs[R_EDX] = ldq_phys(cs->as, sm_state + 0x7fe8);
-    env->regs[R_EBX] = ldq_phys(cs->as, sm_state + 0x7fe0);
-    env->regs[R_ESP] = ldq_phys(cs->as, sm_state + 0x7fd8);
-    env->regs[R_EBP] = ldq_phys(cs->as, sm_state + 0x7fd0);
-    env->regs[R_ESI] = ldq_phys(cs->as, sm_state + 0x7fc8);
-    env->regs[R_EDI] = ldq_phys(cs->as, sm_state + 0x7fc0);
+    env->regs[R_EAX] = x86_ldq_phys(cs, sm_state + 0x7ff8);
+    env->regs[R_ECX] = x86_ldq_phys(cs, sm_state + 0x7ff0);
+    env->regs[R_EDX] = x86_ldq_phys(cs, sm_state + 0x7fe8);
+    env->regs[R_EBX] = x86_ldq_phys(cs, sm_state + 0x7fe0);
+    env->regs[R_ESP] = x86_ldq_phys(cs, sm_state + 0x7fd8);
+    env->regs[R_EBP] = x86_ldq_phys(cs, sm_state + 0x7fd0);
+    env->regs[R_ESI] = x86_ldq_phys(cs, sm_state + 0x7fc8);
+    env->regs[R_EDI] = x86_ldq_phys(cs, sm_state + 0x7fc0);
     for (i = 8; i < 16; i++) {
-        env->regs[i] = ldq_phys(cs->as, sm_state + 0x7ff8 - i * 8);
+        env->regs[i] = x86_ldq_phys(cs, sm_state + 0x7ff8 - i * 8);
     }
-    env->eip = ldq_phys(cs->as, sm_state + 0x7f78);
-    cpu_load_eflags(env, ldl_phys(cs->as, sm_state + 0x7f70),
+    env->eip = x86_ldq_phys(cs, sm_state + 0x7f78);
+    cpu_load_eflags(env, x86_ldl_phys(cs, sm_state + 0x7f70),
                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
-    env->dr[6] = ldl_phys(cs->as, sm_state + 0x7f68);
-    env->dr[7] = ldl_phys(cs->as, sm_state + 0x7f60);
+    env->dr[6] = x86_ldl_phys(cs, sm_state + 0x7f68);
+    env->dr[7] = x86_ldl_phys(cs, sm_state + 0x7f60);
 
-    cpu_x86_update_cr4(env, ldl_phys(cs->as, sm_state + 0x7f48));
-    cpu_x86_update_cr3(env, ldl_phys(cs->as, sm_state + 0x7f50));
-    cpu_x86_update_cr0(env, ldl_phys(cs->as, sm_state + 0x7f58));
+    cpu_x86_update_cr4(env, x86_ldl_phys(cs, sm_state + 0x7f48));
+    cpu_x86_update_cr3(env, x86_ldq_phys(cs, sm_state + 0x7f50));
+    cpu_x86_update_cr0(env, x86_ldl_phys(cs, sm_state + 0x7f58));
 
     for (i = 0; i < 6; i++) {
         offset = 0x7e00 + i * 16;
         cpu_x86_load_seg_cache(env, i,
-                               lduw_phys(cs->as, sm_state + offset),
-                               ldq_phys(cs->as, sm_state + offset + 8),
-                               ldl_phys(cs->as, sm_state + offset + 4),
-                               (lduw_phys(cs->as, sm_state + offset + 2) &
+                               x86_lduw_phys(cs, sm_state + offset),
+                               x86_ldq_phys(cs, sm_state + offset + 8),
+                               x86_ldl_phys(cs, sm_state + offset + 4),
+                               (x86_lduw_phys(cs, sm_state + offset + 2) &
                                 0xf0ff) << 8);
     }
 
-    val = ldl_phys(cs->as, sm_state + 0x7efc); /* revision ID */
+    val = x86_ldl_phys(cs, sm_state + 0x7efc); /* revision ID */
     if (val & 0x20000) {
-        env->smbase = ldl_phys(cs->as, sm_state + 0x7f00) & ~0x7fff;
+        env->smbase = x86_ldl_phys(cs, sm_state + 0x7f00);
     }
 #else
-    cpu_x86_update_cr0(env, ldl_phys(cs->as, sm_state + 0x7ffc));
-    cpu_x86_update_cr3(env, ldl_phys(cs->as, sm_state + 0x7ff8));
-    cpu_load_eflags(env, ldl_phys(cs->as, sm_state + 0x7ff4),
+    cpu_x86_update_cr0(env, x86_ldl_phys(cs, sm_state + 0x7ffc));
+    cpu_x86_update_cr3(env, x86_ldl_phys(cs, sm_state + 0x7ff8));
+    cpu_load_eflags(env, x86_ldl_phys(cs, sm_state + 0x7ff4),
                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
-    env->eip = ldl_phys(cs->as, sm_state + 0x7ff0);
-    env->regs[R_EDI] = ldl_phys(cs->as, sm_state + 0x7fec);
-    env->regs[R_ESI] = ldl_phys(cs->as, sm_state + 0x7fe8);
-    env->regs[R_EBP] = ldl_phys(cs->as, sm_state + 0x7fe4);
-    env->regs[R_ESP] = ldl_phys(cs->as, sm_state + 0x7fe0);
-    env->regs[R_EBX] = ldl_phys(cs->as, sm_state + 0x7fdc);
-    env->regs[R_EDX] = ldl_phys(cs->as, sm_state + 0x7fd8);
-    env->regs[R_ECX] = ldl_phys(cs->as, sm_state + 0x7fd4);
-    env->regs[R_EAX] = ldl_phys(cs->as, sm_state + 0x7fd0);
-    env->dr[6] = ldl_phys(cs->as, sm_state + 0x7fcc);
-    env->dr[7] = ldl_phys(cs->as, sm_state + 0x7fc8);
+    env->eip = x86_ldl_phys(cs, sm_state + 0x7ff0);
+    env->regs[R_EDI] = x86_ldl_phys(cs, sm_state + 0x7fec);
+    env->regs[R_ESI] = x86_ldl_phys(cs, sm_state + 0x7fe8);
+    env->regs[R_EBP] = x86_ldl_phys(cs, sm_state + 0x7fe4);
+    env->regs[R_ESP] = x86_ldl_phys(cs, sm_state + 0x7fe0);
+    env->regs[R_EBX] = x86_ldl_phys(cs, sm_state + 0x7fdc);
+    env->regs[R_EDX] = x86_ldl_phys(cs, sm_state + 0x7fd8);
+    env->regs[R_ECX] = x86_ldl_phys(cs, sm_state + 0x7fd4);
+    env->regs[R_EAX] = x86_ldl_phys(cs, sm_state + 0x7fd0);
+    env->dr[6] = x86_ldl_phys(cs, sm_state + 0x7fcc);
+    env->dr[7] = x86_ldl_phys(cs, sm_state + 0x7fc8);
 
-    env->tr.selector = ldl_phys(cs->as, sm_state + 0x7fc4) & 0xffff;
-    env->tr.base = ldl_phys(cs->as, sm_state + 0x7f64);
-    env->tr.limit = ldl_phys(cs->as, sm_state + 0x7f60);
-    env->tr.flags = (ldl_phys(cs->as, sm_state + 0x7f5c) & 0xf0ff) << 8;
+    env->tr.selector = x86_ldl_phys(cs, sm_state + 0x7fc4) & 0xffff;
+    env->tr.base = x86_ldl_phys(cs, sm_state + 0x7f64);
+    env->tr.limit = x86_ldl_phys(cs, sm_state + 0x7f60);
+    env->tr.flags = (x86_ldl_phys(cs, sm_state + 0x7f5c) & 0xf0ff) << 8;
 
-    env->ldt.selector = ldl_phys(cs->as, sm_state + 0x7fc0) & 0xffff;
-    env->ldt.base = ldl_phys(cs->as, sm_state + 0x7f80);
-    env->ldt.limit = ldl_phys(cs->as, sm_state + 0x7f7c);
-    env->ldt.flags = (ldl_phys(cs->as, sm_state + 0x7f78) & 0xf0ff) << 8;
+    env->ldt.selector = x86_ldl_phys(cs, sm_state + 0x7fc0) & 0xffff;
+    env->ldt.base = x86_ldl_phys(cs, sm_state + 0x7f80);
+    env->ldt.limit = x86_ldl_phys(cs, sm_state + 0x7f7c);
+    env->ldt.flags = (x86_ldl_phys(cs, sm_state + 0x7f78) & 0xf0ff) << 8;
 
-    env->gdt.base = ldl_phys(cs->as, sm_state + 0x7f74);
-    env->gdt.limit = ldl_phys(cs->as, sm_state + 0x7f70);
+    env->gdt.base = x86_ldl_phys(cs, sm_state + 0x7f74);
+    env->gdt.limit = x86_ldl_phys(cs, sm_state + 0x7f70);
 
-    env->idt.base = ldl_phys(cs->as, sm_state + 0x7f58);
-    env->idt.limit = ldl_phys(cs->as, sm_state + 0x7f54);
+    env->idt.base = x86_ldl_phys(cs, sm_state + 0x7f58);
+    env->idt.limit = x86_ldl_phys(cs, sm_state + 0x7f54);
 
     for (i = 0; i < 6; i++) {
         if (i < 3) {
@@ -293,22 +314,26 @@
             offset = 0x7f2c + (i - 3) * 12;
         }
         cpu_x86_load_seg_cache(env, i,
-                               ldl_phys(cs->as,
+                               x86_ldl_phys(cs,
                                         sm_state + 0x7fa8 + i * 4) & 0xffff,
-                               ldl_phys(cs->as, sm_state + offset + 8),
-                               ldl_phys(cs->as, sm_state + offset + 4),
-                               (ldl_phys(cs->as,
+                               x86_ldl_phys(cs, sm_state + offset + 8),
+                               x86_ldl_phys(cs, sm_state + offset + 4),
+                               (x86_ldl_phys(cs,
                                          sm_state + offset) & 0xf0ff) << 8);
     }
-    cpu_x86_update_cr4(env, ldl_phys(cs->as, sm_state + 0x7f14));
+    cpu_x86_update_cr4(env, x86_ldl_phys(cs, sm_state + 0x7f14));
 
-    val = ldl_phys(cs->as, sm_state + 0x7efc); /* revision ID */
+    val = x86_ldl_phys(cs, sm_state + 0x7efc); /* revision ID */
     if (val & 0x20000) {
-        env->smbase = ldl_phys(cs->as, sm_state + 0x7ef8) & ~0x7fff;
+        env->smbase = x86_ldl_phys(cs, sm_state + 0x7ef8);
     }
 #endif
+    if ((env->hflags2 & HF2_SMM_INSIDE_NMI_MASK) == 0) {
+        env->hflags2 &= ~HF2_NMI_MASK;
+    }
+    env->hflags2 &= ~HF2_SMM_INSIDE_NMI_MASK;
     env->hflags &= ~HF_SMM_MASK;
-    cpu_smm_update(env);
+    cpu_smm_update(cpu);
 
     qemu_log_mask(CPU_LOG_INT, "SMM: after RSM\n");
     log_cpu_state_mask(CPU_LOG_INT, CPU(cpu), CPU_DUMP_CCOP);
diff --git a/target-i386/svm.h b/target-i386/svm.h
index 04193ed..922c8fd 100644
--- a/target-i386/svm.h
+++ b/target-i386/svm.h
@@ -1,5 +1,5 @@
-#ifndef __SVM_H
-#define __SVM_H
+#ifndef SVM_H
+#define SVM_H
 
 #define TLB_CONTROL_DO_NOTHING 0
 #define TLB_CONTROL_FLUSH_ALL_ASID 1
diff --git a/target-i386/svm_helper.c b/target-i386/svm_helper.c
index 429d029..782b3f1 100644
--- a/target-i386/svm_helper.c
+++ b/target-i386/svm_helper.c
@@ -17,9 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/cpu-all.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 /* Secure Virtual Machine helpers */
@@ -87,13 +89,13 @@
 {
     CPUState *cs = CPU(x86_env_get_cpu(env));
 
-    stw_phys(cs->as, addr + offsetof(struct vmcb_seg, selector),
+    x86_stw_phys(cs, addr + offsetof(struct vmcb_seg, selector),
              sc->selector);
-    stq_phys(cs->as, addr + offsetof(struct vmcb_seg, base),
+    x86_stq_phys(cs, addr + offsetof(struct vmcb_seg, base),
              sc->base);
-    stl_phys(cs->as, addr + offsetof(struct vmcb_seg, limit),
+    x86_stl_phys(cs, addr + offsetof(struct vmcb_seg, limit),
              sc->limit);
-    stw_phys(cs->as, addr + offsetof(struct vmcb_seg, attrib),
+    x86_stw_phys(cs, addr + offsetof(struct vmcb_seg, attrib),
              ((sc->flags >> 8) & 0xff) | ((sc->flags >> 12) & 0x0f00));
 }
 
@@ -103,11 +105,11 @@
     CPUState *cs = CPU(x86_env_get_cpu(env));
     unsigned int flags;
 
-    sc->selector = lduw_phys(cs->as,
+    sc->selector = x86_lduw_phys(cs,
                              addr + offsetof(struct vmcb_seg, selector));
-    sc->base = ldq_phys(cs->as, addr + offsetof(struct vmcb_seg, base));
-    sc->limit = ldl_phys(cs->as, addr + offsetof(struct vmcb_seg, limit));
-    flags = lduw_phys(cs->as, addr + offsetof(struct vmcb_seg, attrib));
+    sc->base = x86_ldq_phys(cs, addr + offsetof(struct vmcb_seg, base));
+    sc->limit = x86_ldl_phys(cs, addr + offsetof(struct vmcb_seg, limit));
+    flags = x86_lduw_phys(cs, addr + offsetof(struct vmcb_seg, attrib));
     sc->flags = ((flags & 0xff) << 8) | ((flags & 0x0f00) << 12);
 }
 
@@ -141,32 +143,32 @@
     env->vm_vmcb = addr;
 
     /* save the current CPU state in the hsave page */
-    stq_phys(cs->as, env->vm_hsave + offsetof(struct vmcb, save.gdtr.base),
+    x86_stq_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.gdtr.base),
              env->gdt.base);
-    stl_phys(cs->as, env->vm_hsave + offsetof(struct vmcb, save.gdtr.limit),
+    x86_stl_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.gdtr.limit),
              env->gdt.limit);
 
-    stq_phys(cs->as, env->vm_hsave + offsetof(struct vmcb, save.idtr.base),
+    x86_stq_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.idtr.base),
              env->idt.base);
-    stl_phys(cs->as, env->vm_hsave + offsetof(struct vmcb, save.idtr.limit),
+    x86_stl_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.idtr.limit),
              env->idt.limit);
 
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.cr0), env->cr[0]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.cr2), env->cr[2]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.cr3), env->cr[3]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.cr4), env->cr[4]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.dr6), env->dr[6]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.dr7), env->dr[7]);
 
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.efer), env->efer);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.rflags),
              cpu_compute_eflags(env));
 
@@ -179,30 +181,30 @@
     svm_save_seg(env, env->vm_hsave + offsetof(struct vmcb, save.ds),
                  &env->segs[R_DS]);
 
-    stq_phys(cs->as, env->vm_hsave + offsetof(struct vmcb, save.rip),
+    x86_stq_phys(cs, env->vm_hsave + offsetof(struct vmcb, save.rip),
              env->eip + next_eip_addend);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.rsp), env->regs[R_ESP]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_hsave + offsetof(struct vmcb, save.rax), env->regs[R_EAX]);
 
     /* load the interception bitmaps so we do not need to access the
        vmcb in svm mode */
-    env->intercept = ldq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    env->intercept = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                       control.intercept));
-    env->intercept_cr_read = lduw_phys(cs->as, env->vm_vmcb +
+    env->intercept_cr_read = x86_lduw_phys(cs, env->vm_vmcb +
                                        offsetof(struct vmcb,
                                                 control.intercept_cr_read));
-    env->intercept_cr_write = lduw_phys(cs->as, env->vm_vmcb +
+    env->intercept_cr_write = x86_lduw_phys(cs, env->vm_vmcb +
                                         offsetof(struct vmcb,
                                                  control.intercept_cr_write));
-    env->intercept_dr_read = lduw_phys(cs->as, env->vm_vmcb +
+    env->intercept_dr_read = x86_lduw_phys(cs, env->vm_vmcb +
                                        offsetof(struct vmcb,
                                                 control.intercept_dr_read));
-    env->intercept_dr_write = lduw_phys(cs->as, env->vm_vmcb +
+    env->intercept_dr_write = x86_lduw_phys(cs, env->vm_vmcb +
                                         offsetof(struct vmcb,
                                                  control.intercept_dr_write));
-    env->intercept_exceptions = ldl_phys(cs->as, env->vm_vmcb +
+    env->intercept_exceptions = x86_ldl_phys(cs, env->vm_vmcb +
                                          offsetof(struct vmcb,
                                                   control.intercept_exceptions
                                                   ));
@@ -210,35 +212,35 @@
     /* enable intercepts */
     env->hflags |= HF_SVMI_MASK;
 
-    env->tsc_offset = ldq_phys(cs->as, env->vm_vmcb +
+    env->tsc_offset = x86_ldq_phys(cs, env->vm_vmcb +
                                offsetof(struct vmcb, control.tsc_offset));
 
-    env->gdt.base  = ldq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    env->gdt.base  = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                       save.gdtr.base));
-    env->gdt.limit = ldl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    env->gdt.limit = x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                       save.gdtr.limit));
 
-    env->idt.base  = ldq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    env->idt.base  = x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                       save.idtr.base));
-    env->idt.limit = ldl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    env->idt.limit = x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                       save.idtr.limit));
 
     /* clear exit_info_2 so we behave like the real hardware */
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2), 0);
 
-    cpu_x86_update_cr0(env, ldq_phys(cs->as,
+    cpu_x86_update_cr0(env, x86_ldq_phys(cs,
                                      env->vm_vmcb + offsetof(struct vmcb,
                                                              save.cr0)));
-    cpu_x86_update_cr4(env, ldq_phys(cs->as,
+    cpu_x86_update_cr4(env, x86_ldq_phys(cs,
                                      env->vm_vmcb + offsetof(struct vmcb,
                                                              save.cr4)));
-    cpu_x86_update_cr3(env, ldq_phys(cs->as,
+    cpu_x86_update_cr3(env, x86_ldq_phys(cs,
                                      env->vm_vmcb + offsetof(struct vmcb,
                                                              save.cr3)));
-    env->cr[2] = ldq_phys(cs->as,
+    env->cr[2] = x86_ldq_phys(cs,
                           env->vm_vmcb + offsetof(struct vmcb, save.cr2));
-    int_ctl = ldl_phys(cs->as,
+    int_ctl = x86_ldl_phys(cs,
                        env->vm_vmcb + offsetof(struct vmcb, control.int_ctl));
     env->hflags2 &= ~(HF2_HIF_MASK | HF2_VINTR_MASK);
     if (int_ctl & V_INTR_MASKING_MASK) {
@@ -250,10 +252,10 @@
     }
 
     cpu_load_efer(env,
-                  ldq_phys(cs->as,
+                  x86_ldq_phys(cs,
                            env->vm_vmcb + offsetof(struct vmcb, save.efer)));
     env->eflags = 0;
-    cpu_load_eflags(env, ldq_phys(cs->as,
+    cpu_load_eflags(env, x86_ldq_phys(cs,
                                   env->vm_vmcb + offsetof(struct vmcb,
                                                           save.rflags)),
                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
@@ -267,21 +269,21 @@
     svm_load_seg_cache(env, env->vm_vmcb + offsetof(struct vmcb, save.ds),
                        R_DS);
 
-    env->eip = ldq_phys(cs->as,
+    env->eip = x86_ldq_phys(cs,
                         env->vm_vmcb + offsetof(struct vmcb, save.rip));
 
-    env->regs[R_ESP] = ldq_phys(cs->as,
+    env->regs[R_ESP] = x86_ldq_phys(cs,
                                 env->vm_vmcb + offsetof(struct vmcb, save.rsp));
-    env->regs[R_EAX] = ldq_phys(cs->as,
+    env->regs[R_EAX] = x86_ldq_phys(cs,
                                 env->vm_vmcb + offsetof(struct vmcb, save.rax));
-    env->dr[7] = ldq_phys(cs->as,
+    env->dr[7] = x86_ldq_phys(cs,
                           env->vm_vmcb + offsetof(struct vmcb, save.dr7));
-    env->dr[6] = ldq_phys(cs->as,
+    env->dr[6] = x86_ldq_phys(cs,
                           env->vm_vmcb + offsetof(struct vmcb, save.dr6));
 
     /* FIXME: guest state consistency checks */
 
-    switch (ldub_phys(cs->as,
+    switch (x86_ldub_phys(cs,
                       env->vm_vmcb + offsetof(struct vmcb, control.tlb_ctl))) {
     case TLB_CONTROL_DO_NOTHING:
         break;
@@ -300,12 +302,12 @@
     }
 
     /* maybe we need to inject an event */
-    event_inj = ldl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+    event_inj = x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                  control.event_inj));
     if (event_inj & SVM_EVTINJ_VALID) {
         uint8_t vector = event_inj & SVM_EVTINJ_VEC_MASK;
         uint16_t valid_err = event_inj & SVM_EVTINJ_VALID_ERR;
-        uint32_t event_inj_err = ldl_phys(cs->as, env->vm_vmcb +
+        uint32_t event_inj_err = x86_ldl_phys(cs, env->vm_vmcb +
                                           offsetof(struct vmcb,
                                                    control.event_inj_err));
 
@@ -372,7 +374,7 @@
 
     qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmload! " TARGET_FMT_lx
                   "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
-                  addr, ldq_phys(cs->as, addr + offsetof(struct vmcb,
+                  addr, x86_ldq_phys(cs, addr + offsetof(struct vmcb,
                                                           save.fs.base)),
                   env->segs[R_FS].base);
 
@@ -382,18 +384,18 @@
     svm_load_seg(env, addr + offsetof(struct vmcb, save.ldtr), &env->ldt);
 
 #ifdef TARGET_X86_64
-    env->kernelgsbase = ldq_phys(cs->as, addr + offsetof(struct vmcb,
+    env->kernelgsbase = x86_ldq_phys(cs, addr + offsetof(struct vmcb,
                                                  save.kernel_gs_base));
-    env->lstar = ldq_phys(cs->as, addr + offsetof(struct vmcb, save.lstar));
-    env->cstar = ldq_phys(cs->as, addr + offsetof(struct vmcb, save.cstar));
-    env->fmask = ldq_phys(cs->as, addr + offsetof(struct vmcb, save.sfmask));
+    env->lstar = x86_ldq_phys(cs, addr + offsetof(struct vmcb, save.lstar));
+    env->cstar = x86_ldq_phys(cs, addr + offsetof(struct vmcb, save.cstar));
+    env->fmask = x86_ldq_phys(cs, addr + offsetof(struct vmcb, save.sfmask));
 #endif
-    env->star = ldq_phys(cs->as, addr + offsetof(struct vmcb, save.star));
-    env->sysenter_cs = ldq_phys(cs->as,
+    env->star = x86_ldq_phys(cs, addr + offsetof(struct vmcb, save.star));
+    env->sysenter_cs = x86_ldq_phys(cs,
                                 addr + offsetof(struct vmcb, save.sysenter_cs));
-    env->sysenter_esp = ldq_phys(cs->as, addr + offsetof(struct vmcb,
+    env->sysenter_esp = x86_ldq_phys(cs, addr + offsetof(struct vmcb,
                                                  save.sysenter_esp));
-    env->sysenter_eip = ldq_phys(cs->as, addr + offsetof(struct vmcb,
+    env->sysenter_eip = x86_ldq_phys(cs, addr + offsetof(struct vmcb,
                                                  save.sysenter_eip));
 }
 
@@ -412,7 +414,7 @@
 
     qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmsave! " TARGET_FMT_lx
                   "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
-                  addr, ldq_phys(cs->as,
+                  addr, x86_ldq_phys(cs,
                                  addr + offsetof(struct vmcb, save.fs.base)),
                   env->segs[R_FS].base);
 
@@ -426,18 +428,18 @@
                  &env->ldt);
 
 #ifdef TARGET_X86_64
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.kernel_gs_base),
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.kernel_gs_base),
              env->kernelgsbase);
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.lstar), env->lstar);
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.cstar), env->cstar);
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.sfmask), env->fmask);
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.lstar), env->lstar);
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.cstar), env->cstar);
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.sfmask), env->fmask);
 #endif
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.star), env->star);
-    stq_phys(cs->as,
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.star), env->star);
+    x86_stq_phys(cs,
              addr + offsetof(struct vmcb, save.sysenter_cs), env->sysenter_cs);
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.sysenter_esp),
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.sysenter_esp),
              env->sysenter_esp);
-    stq_phys(cs->as, addr + offsetof(struct vmcb, save.sysenter_eip),
+    x86_stq_phys(cs, addr + offsetof(struct vmcb, save.sysenter_eip),
              env->sysenter_eip);
 }
 
@@ -515,7 +517,7 @@
     case SVM_EXIT_MSR:
         if (env->intercept & (1ULL << (SVM_EXIT_MSR - SVM_EXIT_INTR))) {
             /* FIXME: this should be read in at vmrun (faster this way?) */
-            uint64_t addr = ldq_phys(cs->as, env->vm_vmcb +
+            uint64_t addr = x86_ldq_phys(cs, env->vm_vmcb +
                                      offsetof(struct vmcb,
                                               control.msrpm_base_pa));
             uint32_t t0, t1;
@@ -541,7 +543,7 @@
                 t1 = 0;
                 break;
             }
-            if (ldub_phys(cs->as, addr + t1) & ((1 << param) << t0)) {
+            if (x86_ldub_phys(cs, addr + t1) & ((1 << param) << t0)) {
                 helper_vmexit(env, type, param);
             }
         }
@@ -567,13 +569,13 @@
 
     if (env->intercept & (1ULL << (SVM_EXIT_IOIO - SVM_EXIT_INTR))) {
         /* FIXME: this should be read in at vmrun (faster this way?) */
-        uint64_t addr = ldq_phys(cs->as, env->vm_vmcb +
+        uint64_t addr = x86_ldq_phys(cs, env->vm_vmcb +
                                  offsetof(struct vmcb, control.iopm_base_pa));
         uint16_t mask = (1 << ((param >> 4) & 7)) - 1;
 
-        if (lduw_phys(cs->as, addr + port / 8) & (mask << (port & 7))) {
+        if (x86_lduw_phys(cs, addr + port / 8) & (mask << (port & 7))) {
             /* next env->eip */
-            stq_phys(cs->as,
+            x86_stq_phys(cs,
                      env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2),
                      env->eip + next_eip_addend);
             helper_vmexit(env, SVM_EXIT_IOIO, param | (port << 16));
@@ -590,17 +592,17 @@
     qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmexit(%08x, %016" PRIx64 ", %016"
                   PRIx64 ", " TARGET_FMT_lx ")!\n",
                   exit_code, exit_info_1,
-                  ldq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+                  x86_ldq_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                                    control.exit_info_2)),
                   env->eip);
 
     if (env->hflags & HF_INHIBIT_IRQ_MASK) {
-        stl_phys(cs->as,
+        x86_stl_phys(cs,
                  env->vm_vmcb + offsetof(struct vmcb, control.int_state),
                  SVM_INTERRUPT_SHADOW_MASK);
         env->hflags &= ~HF_INHIBIT_IRQ_MASK;
     } else {
-        stl_phys(cs->as,
+        x86_stl_phys(cs,
                  env->vm_vmcb + offsetof(struct vmcb, control.int_state), 0);
     }
 
@@ -614,50 +616,50 @@
     svm_save_seg(env, env->vm_vmcb + offsetof(struct vmcb, save.ds),
                  &env->segs[R_DS]);
 
-    stq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base),
+    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base),
              env->gdt.base);
-    stl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit),
+    x86_stl_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit),
              env->gdt.limit);
 
-    stq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.idtr.base),
+    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.idtr.base),
              env->idt.base);
-    stl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.idtr.limit),
+    x86_stl_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.idtr.limit),
              env->idt.limit);
 
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.efer), env->efer);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.cr0), env->cr[0]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.cr2), env->cr[2]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.cr3), env->cr[3]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.cr4), env->cr[4]);
 
-    int_ctl = ldl_phys(cs->as,
+    int_ctl = x86_ldl_phys(cs,
                        env->vm_vmcb + offsetof(struct vmcb, control.int_ctl));
     int_ctl &= ~(V_TPR_MASK | V_IRQ_MASK);
     int_ctl |= env->v_tpr & V_TPR_MASK;
     if (cs->interrupt_request & CPU_INTERRUPT_VIRQ) {
         int_ctl |= V_IRQ_MASK;
     }
-    stl_phys(cs->as,
+    x86_stl_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, control.int_ctl), int_ctl);
 
-    stq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.rflags),
+    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.rflags),
              cpu_compute_eflags(env));
-    stq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.rip),
+    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.rip),
              env->eip);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.rsp), env->regs[R_ESP]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.rax), env->regs[R_EAX]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.dr7), env->dr[7]);
-    stq_phys(cs->as,
+    x86_stq_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, save.dr6), env->dr[6]);
-    stb_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, save.cpl),
+    x86_stb_phys(cs, env->vm_vmcb + offsetof(struct vmcb, save.cpl),
              env->hflags & HF_CPL_MASK);
 
     /* Reload the host state from vm_hsave */
@@ -668,32 +670,32 @@
     cs->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
     env->tsc_offset = 0;
 
-    env->gdt.base  = ldq_phys(cs->as, env->vm_hsave + offsetof(struct vmcb,
+    env->gdt.base  = x86_ldq_phys(cs, env->vm_hsave + offsetof(struct vmcb,
                                                        save.gdtr.base));
-    env->gdt.limit = ldl_phys(cs->as, env->vm_hsave + offsetof(struct vmcb,
+    env->gdt.limit = x86_ldl_phys(cs, env->vm_hsave + offsetof(struct vmcb,
                                                        save.gdtr.limit));
 
-    env->idt.base  = ldq_phys(cs->as, env->vm_hsave + offsetof(struct vmcb,
+    env->idt.base  = x86_ldq_phys(cs, env->vm_hsave + offsetof(struct vmcb,
                                                        save.idtr.base));
-    env->idt.limit = ldl_phys(cs->as, env->vm_hsave + offsetof(struct vmcb,
+    env->idt.limit = x86_ldl_phys(cs, env->vm_hsave + offsetof(struct vmcb,
                                                        save.idtr.limit));
 
-    cpu_x86_update_cr0(env, ldq_phys(cs->as,
+    cpu_x86_update_cr0(env, x86_ldq_phys(cs,
                                      env->vm_hsave + offsetof(struct vmcb,
                                                               save.cr0)) |
                        CR0_PE_MASK);
-    cpu_x86_update_cr4(env, ldq_phys(cs->as,
+    cpu_x86_update_cr4(env, x86_ldq_phys(cs,
                                      env->vm_hsave + offsetof(struct vmcb,
                                                               save.cr4)));
-    cpu_x86_update_cr3(env, ldq_phys(cs->as,
+    cpu_x86_update_cr3(env, x86_ldq_phys(cs,
                                      env->vm_hsave + offsetof(struct vmcb,
                                                               save.cr3)));
     /* we need to set the efer after the crs so the hidden flags get
        set properly */
-    cpu_load_efer(env, ldq_phys(cs->as, env->vm_hsave + offsetof(struct vmcb,
+    cpu_load_efer(env, x86_ldq_phys(cs, env->vm_hsave + offsetof(struct vmcb,
                                                          save.efer)));
     env->eflags = 0;
-    cpu_load_eflags(env, ldq_phys(cs->as,
+    cpu_load_eflags(env, x86_ldq_phys(cs,
                                   env->vm_hsave + offsetof(struct vmcb,
                                                            save.rflags)),
                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK |
@@ -708,33 +710,33 @@
     svm_load_seg_cache(env, env->vm_hsave + offsetof(struct vmcb, save.ds),
                        R_DS);
 
-    env->eip = ldq_phys(cs->as,
+    env->eip = x86_ldq_phys(cs,
                         env->vm_hsave + offsetof(struct vmcb, save.rip));
-    env->regs[R_ESP] = ldq_phys(cs->as, env->vm_hsave +
+    env->regs[R_ESP] = x86_ldq_phys(cs, env->vm_hsave +
                                 offsetof(struct vmcb, save.rsp));
-    env->regs[R_EAX] = ldq_phys(cs->as, env->vm_hsave +
+    env->regs[R_EAX] = x86_ldq_phys(cs, env->vm_hsave +
                                 offsetof(struct vmcb, save.rax));
 
-    env->dr[6] = ldq_phys(cs->as,
+    env->dr[6] = x86_ldq_phys(cs,
                           env->vm_hsave + offsetof(struct vmcb, save.dr6));
-    env->dr[7] = ldq_phys(cs->as,
+    env->dr[7] = x86_ldq_phys(cs,
                           env->vm_hsave + offsetof(struct vmcb, save.dr7));
 
     /* other setups */
-    stq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, control.exit_code),
+    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, control.exit_code),
              exit_code);
-    stq_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb, control.exit_info_1),
+    x86_stq_phys(cs, env->vm_vmcb + offsetof(struct vmcb, control.exit_info_1),
              exit_info_1);
 
-    stl_phys(cs->as,
+    x86_stl_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info),
-             ldl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+             x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                               control.event_inj)));
-    stl_phys(cs->as,
+    x86_stl_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, control.exit_int_info_err),
-             ldl_phys(cs->as, env->vm_vmcb + offsetof(struct vmcb,
+             x86_ldl_phys(cs, env->vm_vmcb + offsetof(struct vmcb,
                                               control.event_inj_err)));
-    stl_phys(cs->as,
+    x86_stl_phys(cs,
              env->vm_vmcb + offsetof(struct vmcb, control.event_inj), 0);
 
     env->hflags2 &= ~HF2_GIF_MASK;
diff --git a/target-i386/topology.h b/target-i386/topology.h
deleted file mode 100644
index 07a6c5f..0000000
--- a/target-i386/topology.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- *  x86 CPU topology data structures and functions
- *
- *  Copyright (c) 2012 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef TARGET_I386_TOPOLOGY_H
-#define TARGET_I386_TOPOLOGY_H
-
-/* This file implements the APIC-ID-based CPU topology enumeration logic,
- * documented at the following document:
- *   Intel® 64 Architecture Processor Topology Enumeration
- *   http://software.intel.com/en-us/articles/intel-64-architecture-processor-topology-enumeration/
- *
- * This code should be compatible with AMD's "Extended Method" described at:
- *   AMD CPUID Specification (Publication #25481)
- *   Section 3: Multiple Core Calcuation
- * as long as:
- *  nr_threads is set to 1;
- *  OFFSET_IDX is assumed to be 0;
- *  CPUID Fn8000_0008_ECX[ApicIdCoreIdSize[3:0]] is set to apicid_core_width().
- */
-
-#include <stdint.h>
-#include <string.h>
-
-#include "qemu/bitops.h"
-
-/* APIC IDs can be 32-bit, but beware: APIC IDs > 255 require x2APIC support
- */
-typedef uint32_t apic_id_t;
-
-/* Return the bit width needed for 'count' IDs
- */
-static unsigned apicid_bitwidth_for_count(unsigned count)
-{
-    g_assert(count >= 1);
-    count -= 1;
-    return count ? 32 - clz32(count) : 0;
-}
-
-/* Bit width of the SMT_ID (thread ID) field on the APIC ID
- */
-static inline unsigned apicid_smt_width(unsigned nr_cores, unsigned nr_threads)
-{
-    return apicid_bitwidth_for_count(nr_threads);
-}
-
-/* Bit width of the Core_ID field
- */
-static inline unsigned apicid_core_width(unsigned nr_cores, unsigned nr_threads)
-{
-    return apicid_bitwidth_for_count(nr_cores);
-}
-
-/* Bit offset of the Core_ID field
- */
-static inline unsigned apicid_core_offset(unsigned nr_cores,
-                                          unsigned nr_threads)
-{
-    return apicid_smt_width(nr_cores, nr_threads);
-}
-
-/* Bit offset of the Pkg_ID (socket ID) field
- */
-static inline unsigned apicid_pkg_offset(unsigned nr_cores, unsigned nr_threads)
-{
-    return apicid_core_offset(nr_cores, nr_threads) +
-           apicid_core_width(nr_cores, nr_threads);
-}
-
-/* Make APIC ID for the CPU based on Pkg_ID, Core_ID, SMT_ID
- *
- * The caller must make sure core_id < nr_cores and smt_id < nr_threads.
- */
-static inline apic_id_t apicid_from_topo_ids(unsigned nr_cores,
-                                             unsigned nr_threads,
-                                             unsigned pkg_id,
-                                             unsigned core_id,
-                                             unsigned smt_id)
-{
-    return (pkg_id  << apicid_pkg_offset(nr_cores, nr_threads)) |
-           (core_id << apicid_core_offset(nr_cores, nr_threads)) |
-           smt_id;
-}
-
-/* Calculate thread/core/package IDs for a specific topology,
- * based on (contiguous) CPU index
- */
-static inline void x86_topo_ids_from_idx(unsigned nr_cores,
-                                         unsigned nr_threads,
-                                         unsigned cpu_index,
-                                         unsigned *pkg_id,
-                                         unsigned *core_id,
-                                         unsigned *smt_id)
-{
-    unsigned core_index = cpu_index / nr_threads;
-    *smt_id = cpu_index % nr_threads;
-    *core_id = core_index % nr_cores;
-    *pkg_id = core_index / nr_cores;
-}
-
-/* Make APIC ID for the CPU 'cpu_index'
- *
- * 'cpu_index' is a sequential, contiguous ID for the CPU.
- */
-static inline apic_id_t x86_apicid_from_cpu_idx(unsigned nr_cores,
-                                                unsigned nr_threads,
-                                                unsigned cpu_index)
-{
-    unsigned pkg_id, core_id, smt_id;
-    x86_topo_ids_from_idx(nr_cores, nr_threads, cpu_index,
-                          &pkg_id, &core_id, &smt_id);
-    return apicid_from_topo_ids(nr_cores, nr_threads, pkg_id, core_id, smt_id);
-}
-
-#endif /* TARGET_I386_TOPOLOGY_H */
diff --git a/target-i386/trace-events b/target-i386/trace-events
new file mode 100644
index 0000000..05c5453
--- /dev/null
+++ b/target-i386/trace-events
@@ -0,0 +1,7 @@
+# See docs/tracing.txt for syntax documentation.
+
+# target-i386/kvm.c
+kvm_x86_fixup_msi_error(uint32_t gsi) "VT-d failed to remap interrupt for GSI %" PRIu32
+kvm_x86_add_msi_route(int virq) "Adding route entry for virq %d"
+kvm_x86_remove_msi_route(int virq) "Removing route entry for virq %d"
+kvm_x86_update_msi_routes(int num) "Updated %d MSI routes"
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 899ca51..121527c 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -16,16 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "qemu/host-utils.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/cpu_ldst.h"
 
@@ -33,7 +29,11 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
+
+#ifdef CONFIG_HAX
 #include "sysemu/hax.h"
+#endif
 
 #define PREFIX_REPZ   0x01
 #define PREFIX_REPNZ  0x02
@@ -60,24 +60,37 @@
 # define clztl  clz32
 #endif
 
+/* For a switch indexed by MODRM, match all memory operands for a given OP.  */
+#define CASE_MODRM_MEM_OP(OP) \
+    case (0 << 6) | (OP << 3) | 0 ... (0 << 6) | (OP << 3) | 7: \
+    case (1 << 6) | (OP << 3) | 0 ... (1 << 6) | (OP << 3) | 7: \
+    case (2 << 6) | (OP << 3) | 0 ... (2 << 6) | (OP << 3) | 7
+
+#define CASE_MODRM_OP(OP) \
+    case (0 << 6) | (OP << 3) | 0 ... (0 << 6) | (OP << 3) | 7: \
+    case (1 << 6) | (OP << 3) | 0 ... (1 << 6) | (OP << 3) | 7: \
+    case (2 << 6) | (OP << 3) | 0 ... (2 << 6) | (OP << 3) | 7: \
+    case (3 << 6) | (OP << 3) | 0 ... (3 << 6) | (OP << 3) | 7
+
 //#define MACRO_TEST   1
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_A0;
 static TCGv cpu_cc_dst, cpu_cc_src, cpu_cc_src2, cpu_cc_srcT;
 static TCGv_i32 cpu_cc_op;
 static TCGv cpu_regs[CPU_NB_REGS];
+static TCGv cpu_seg_base[6];
+static TCGv_i64 cpu_bndl[4];
+static TCGv_i64 cpu_bndu[4];
 /* local temps */
-static TCGv cpu_T[2];
+static TCGv cpu_T0, cpu_T1;
 /* local register indexes (only used inside old micro ops) */
 static TCGv cpu_tmp0, cpu_tmp4;
 static TCGv_ptr cpu_ptr0, cpu_ptr1;
 static TCGv_i32 cpu_tmp2_i32, cpu_tmp3_i32;
 static TCGv_i64 cpu_tmp1_i64;
 
-static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 #ifdef TARGET_X86_64
@@ -90,6 +103,7 @@
     int prefix;
     TCGMemOp aflag;
     TCGMemOp dflag;
+    target_ulong pc_start;
     target_ulong pc; /* pc = eip + cs_base */
     int is_jmp; /* 1 = means jump (stop translation), 2 means CPU
                    static state change (stop translation) */
@@ -115,6 +129,7 @@
     int tf;     /* TF cpu flag */
     int singlestep_enabled; /* "hardware" single step enabled */
     int jmp_opt; /* use direct block chaining for direct jumps */
+    int repz_opt; /* optimize jumps within repz instructions */
     int mem_index; /* select memory access functions */
     uint64_t flags; /* all execution flags */
     struct TranslationBlock *tb;
@@ -125,6 +140,7 @@
     int cpuid_ext2_features;
     int cpuid_ext3_features;
     int cpuid_7_0_ebx_features;
+    int cpuid_xsave_features;
 } DisasContext;
 
 static void gen_eob(DisasContext *s);
@@ -309,6 +325,12 @@
     }
 }
 
+/* Select the size of the stack pointer.  */
+static inline TCGMemOp mo_stacksize(DisasContext *s)
+{
+    return CODE64(s) ? MO_64 : s->ss32 ? MO_32 : MO_16;
+}
+
 /* Select only size 64 else 32.  Used for SSE operand sizes.  */
 static inline TCGMemOp mo_64_32(TCGMemOp ot)
 {
@@ -371,34 +393,12 @@
     }
 }
 
-static inline void gen_op_movl_A0_reg(int reg)
-{
-    tcg_gen_mov_tl(cpu_A0, cpu_regs[reg]);
-}
-
-static inline void gen_op_addl_A0_im(int32_t val)
-{
-    tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
-#ifdef TARGET_X86_64
-    tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
-#endif
-}
-
-#ifdef TARGET_X86_64
-static inline void gen_op_addq_A0_im(int64_t val)
-{
-    tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
-}
-#endif
-    
 static void gen_add_A0_im(DisasContext *s, int val)
 {
-#ifdef TARGET_X86_64
-    if (CODE64(s))
-        gen_op_addq_A0_im(val);
-    else
-#endif
-        gen_op_addl_A0_im(val);
+    tcg_gen_addi_tl(cpu_A0, cpu_A0, val);
+    if (!CODE64(s)) {
+        tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+    }
 }
 
 static inline void gen_op_jmp_v(TCGv dest)
@@ -414,68 +414,10 @@
 
 static inline void gen_op_add_reg_T0(TCGMemOp size, int reg)
 {
-    tcg_gen_add_tl(cpu_tmp0, cpu_regs[reg], cpu_T[0]);
+    tcg_gen_add_tl(cpu_tmp0, cpu_regs[reg], cpu_T0);
     gen_op_mov_reg_v(size, reg, cpu_tmp0);
 }
 
-static inline void gen_op_addl_A0_reg_sN(int shift, int reg)
-{
-    tcg_gen_mov_tl(cpu_tmp0, cpu_regs[reg]);
-    if (shift != 0)
-        tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, shift);
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-    /* For x86_64, this sets the higher half of register to zero.
-       For i386, this is equivalent to a nop. */
-    tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
-}
-
-static inline void gen_op_movl_A0_seg(int reg)
-{
-    tcg_gen_ld32u_tl(cpu_A0, cpu_env, offsetof(CPUX86State, segs[reg].base) + REG_L_OFFSET);
-}
-
-static inline void gen_op_addl_A0_seg(DisasContext *s, int reg)
-{
-    tcg_gen_ld_tl(cpu_tmp0, cpu_env, offsetof(CPUX86State, segs[reg].base));
-#ifdef TARGET_X86_64
-    if (CODE64(s)) {
-        tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
-        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-    } else {
-        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-        tcg_gen_andi_tl(cpu_A0, cpu_A0, 0xffffffff);
-    }
-#else
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-#endif
-}
-
-#ifdef TARGET_X86_64
-static inline void gen_op_movq_A0_seg(int reg)
-{
-    tcg_gen_ld_tl(cpu_A0, cpu_env, offsetof(CPUX86State, segs[reg].base));
-}
-
-static inline void gen_op_addq_A0_seg(int reg)
-{
-    tcg_gen_ld_tl(cpu_tmp0, cpu_env, offsetof(CPUX86State, segs[reg].base));
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-}
-
-static inline void gen_op_movq_A0_reg(int reg)
-{
-    tcg_gen_mov_tl(cpu_A0, cpu_regs[reg]);
-}
-
-static inline void gen_op_addq_A0_reg_sN(int shift, int reg)
-{
-    tcg_gen_mov_tl(cpu_tmp0, cpu_regs[reg]);
-    if (shift != 0)
-        tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, shift);
-    tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-}
-#endif
-
 static inline void gen_op_ld_v(DisasContext *s, int idx, TCGv t0, TCGv a0)
 {
     tcg_gen_qemu_ld_tl(t0, a0, s->mem_index, idx | MO_LE);
@@ -489,9 +431,9 @@
 static inline void gen_op_st_rm_T0_A0(DisasContext *s, int idx, int d)
 {
     if (d == OR_TMP0) {
-        gen_op_st_v(s, idx, cpu_T[0], cpu_A0);
+        gen_op_st_v(s, idx, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_reg_v(idx, d, cpu_T[0]);
+        gen_op_mov_reg_v(idx, d, cpu_T0);
     }
 }
 
@@ -501,74 +443,77 @@
     gen_op_jmp_v(cpu_tmp0);
 }
 
-static inline void gen_string_movl_A0_ESI(DisasContext *s)
+/* Compute SEG:REG into A0.  SEG is selected from the override segment
+   (OVR_SEG) and the default segment (DEF_SEG).  OVR_SEG may be -1 to
+   indicate no override.  */
+static void gen_lea_v_seg(DisasContext *s, TCGMemOp aflag, TCGv a0,
+                          int def_seg, int ovr_seg)
 {
-    int override;
-
-    override = s->override;
-    switch (s->aflag) {
+    switch (aflag) {
 #ifdef TARGET_X86_64
     case MO_64:
-        if (override >= 0) {
-            gen_op_movq_A0_seg(override);
-            gen_op_addq_A0_reg_sN(0, R_ESI);
-        } else {
-            gen_op_movq_A0_reg(R_ESI);
+        if (ovr_seg < 0) {
+            tcg_gen_mov_tl(cpu_A0, a0);
+            return;
         }
         break;
 #endif
     case MO_32:
         /* 32 bit address */
-        if (s->addseg && override < 0)
-            override = R_DS;
-        if (override >= 0) {
-            gen_op_movl_A0_seg(override);
-            gen_op_addl_A0_reg_sN(0, R_ESI);
-        } else {
-            gen_op_movl_A0_reg(R_ESI);
+        if (ovr_seg < 0) {
+            if (s->addseg) {
+                ovr_seg = def_seg;
+            } else {
+                tcg_gen_ext32u_tl(cpu_A0, a0);
+                return;
+            }
         }
         break;
     case MO_16:
-        /* 16 address, always override */
-        if (override < 0)
-            override = R_DS;
-        tcg_gen_ext16u_tl(cpu_A0, cpu_regs[R_ESI]);
-        gen_op_addl_A0_seg(s, override);
+        /* 16 bit address */
+        tcg_gen_ext16u_tl(cpu_A0, a0);
+        a0 = cpu_A0;
+        if (ovr_seg < 0) {
+            if (s->addseg) {
+                ovr_seg = def_seg;
+            } else {
+                return;
+            }
+        }
         break;
     default:
         tcg_abort();
     }
+
+    if (ovr_seg >= 0) {
+        TCGv seg = cpu_seg_base[ovr_seg];
+
+        if (aflag == MO_64) {
+            tcg_gen_add_tl(cpu_A0, a0, seg);
+        } else if (CODE64(s)) {
+            tcg_gen_ext32u_tl(cpu_A0, a0);
+            tcg_gen_add_tl(cpu_A0, cpu_A0, seg);
+        } else {
+            tcg_gen_add_tl(cpu_A0, a0, seg);
+            tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+        }
+    }
+}
+
+static inline void gen_string_movl_A0_ESI(DisasContext *s)
+{
+    gen_lea_v_seg(s, s->aflag, cpu_regs[R_ESI], R_DS, s->override);
 }
 
 static inline void gen_string_movl_A0_EDI(DisasContext *s)
 {
-    switch (s->aflag) {
-#ifdef TARGET_X86_64
-    case MO_64:
-        gen_op_movq_A0_reg(R_EDI);
-        break;
-#endif
-    case MO_32:
-        if (s->addseg) {
-            gen_op_movl_A0_seg(R_ES);
-            gen_op_addl_A0_reg_sN(0, R_EDI);
-        } else {
-            gen_op_movl_A0_reg(R_EDI);
-        }
-        break;
-    case MO_16:
-        tcg_gen_ext16u_tl(cpu_A0, cpu_regs[R_EDI]);
-        gen_op_addl_A0_seg(s, R_ES);
-        break;
-    default:
-        tcg_abort();
-    }
+    gen_lea_v_seg(s, s->aflag, cpu_regs[R_EDI], R_ES, -1);
 }
 
 static inline void gen_op_movl_T0_Dshift(TCGMemOp ot)
 {
-    tcg_gen_ld32s_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, df));
-    tcg_gen_shli_tl(cpu_T[0], cpu_T[0], ot);
+    tcg_gen_ld32s_tl(cpu_T0, cpu_env, offsetof(CPUX86State, df));
+    tcg_gen_shli_tl(cpu_T0, cpu_T0, ot);
 };
 
 static TCGv gen_ext_tl(TCGv dst, TCGv src, TCGMemOp size, bool sign)
@@ -612,14 +557,14 @@
     gen_ext_tl(reg, reg, ot, true);
 }
 
-static inline void gen_op_jnz_ecx(TCGMemOp size, int label1)
+static inline void gen_op_jnz_ecx(TCGMemOp size, TCGLabel *label1)
 {
     tcg_gen_mov_tl(cpu_tmp0, cpu_regs[R_ECX]);
     gen_extu(size, cpu_tmp0);
     tcg_gen_brcondi_tl(TCG_COND_NE, cpu_tmp0, 0, label1);
 }
 
-static inline void gen_op_jz_ecx(TCGMemOp size, int label1)
+static inline void gen_op_jz_ecx(TCGMemOp size, TCGLabel *label1)
 {
     tcg_gen_mov_tl(cpu_tmp0, cpu_regs[R_ECX]);
     gen_extu(size, cpu_tmp0);
@@ -630,13 +575,13 @@
 {
     switch (ot) {
     case MO_8:
-        gen_helper_inb(v, n);
+        gen_helper_inb(v, cpu_env, n);
         break;
     case MO_16:
-        gen_helper_inw(v, n);
+        gen_helper_inw(v, cpu_env, n);
         break;
     case MO_32:
-        gen_helper_inl(v, n);
+        gen_helper_inl(v, cpu_env, n);
         break;
     default:
         tcg_abort();
@@ -647,13 +592,13 @@
 {
     switch (ot) {
     case MO_8:
-        gen_helper_outb(v, n);
+        gen_helper_outb(cpu_env, v, n);
         break;
     case MO_16:
-        gen_helper_outw(v, n);
+        gen_helper_outw(cpu_env, v, n);
         break;
     case MO_32:
-        gen_helper_outl(v, n);
+        gen_helper_outl(cpu_env, v, n);
         break;
     default:
         tcg_abort();
@@ -663,15 +608,10 @@
 static void gen_check_io(DisasContext *s, TCGMemOp ot, target_ulong cur_eip,
                          uint32_t svm_flags)
 {
-    int state_saved;
     target_ulong next_eip;
 
-    state_saved = 0;
     if (s->pe && (s->cpl > s->iopl || s->vm86)) {
-        gen_update_cc_op(s);
-        gen_jmp_im(cur_eip);
-        state_saved = 1;
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         switch (ot) {
         case MO_8:
             gen_helper_check_iob(cpu_env, cpu_tmp2_i32);
@@ -687,13 +627,11 @@
         }
     }
     if(s->flags & HF_SVMI_MASK) {
-        if (!state_saved) {
-            gen_update_cc_op(s);
-            gen_jmp_im(cur_eip);
-        }
+        gen_update_cc_op(s);
+        gen_jmp_im(cur_eip);
         svm_flags |= (1 << (4 + ot));
         next_eip = s->pc - s->cs_base;
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         gen_helper_svm_check_io(cpu_env, cpu_tmp2_i32,
                                 tcg_const_i32(svm_flags),
                                 tcg_const_i32(next_eip - cur_eip));
@@ -703,9 +641,9 @@
 static inline void gen_movs(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_ESI(s);
-    gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     gen_string_movl_A0_EDI(s);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
     gen_op_add_reg_T0(s->aflag, R_EDI);
@@ -713,31 +651,31 @@
 
 static void gen_op_update1_cc(void)
 {
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static void gen_op_update2_cc(void)
 {
-    tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_src, cpu_T1);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static void gen_op_update3_cc(TCGv reg)
 {
     tcg_gen_mov_tl(cpu_cc_src2, reg);
-    tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_src, cpu_T1);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static inline void gen_op_testl_T0_T1_cc(void)
 {
-    tcg_gen_and_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]);
+    tcg_gen_and_tl(cpu_cc_dst, cpu_T0, cpu_T1);
 }
 
 static void gen_op_update_neg_cc(void)
 {
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-    tcg_gen_neg_tl(cpu_cc_src, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+    tcg_gen_neg_tl(cpu_cc_src, cpu_T0);
     tcg_gen_movi_tl(cpu_cc_srcT, 0);
 }
 
@@ -1077,13 +1015,13 @@
 
 /* generate a conditional jump to label 'l1' according to jump opcode
    value 'b'. In the fast case, T0 is guaranted not to be used. */
-static inline void gen_jcc1_noeob(DisasContext *s, int b, int l1)
+static inline void gen_jcc1_noeob(DisasContext *s, int b, TCGLabel *l1)
 {
-    CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0]);
+    CCPrepare cc = gen_prepare_cc(s, b, cpu_T0);
 
     if (cc.mask != -1) {
-        tcg_gen_andi_tl(cpu_T[0], cc.reg, cc.mask);
-        cc.reg = cpu_T[0];
+        tcg_gen_andi_tl(cpu_T0, cc.reg, cc.mask);
+        cc.reg = cpu_T0;
     }
     if (cc.use_reg2) {
         tcg_gen_brcond_tl(cc.cond, cc.reg, cc.reg2, l1);
@@ -1095,14 +1033,14 @@
 /* Generate a conditional jump to label 'l1' according to jump opcode
    value 'b'. In the fast case, T0 is guaranted not to be used.
    A translation block must end soon.  */
-static inline void gen_jcc1(DisasContext *s, int b, int l1)
+static inline void gen_jcc1(DisasContext *s, int b, TCGLabel *l1)
 {
-    CCPrepare cc = gen_prepare_cc(s, b, cpu_T[0]);
+    CCPrepare cc = gen_prepare_cc(s, b, cpu_T0);
 
     gen_update_cc_op(s);
     if (cc.mask != -1) {
-        tcg_gen_andi_tl(cpu_T[0], cc.reg, cc.mask);
-        cc.reg = cpu_T[0];
+        tcg_gen_andi_tl(cpu_T0, cc.reg, cc.mask);
+        cc.reg = cpu_T0;
     }
     set_cc_op(s, CC_OP_DYNAMIC);
     if (cc.use_reg2) {
@@ -1114,12 +1052,10 @@
 
 /* XXX: does not work with gdbstub "ice" single step - not a
    serious problem */
-static int gen_jz_ecx_string(DisasContext *s, target_ulong next_eip)
+static TCGLabel *gen_jz_ecx_string(DisasContext *s, target_ulong next_eip)
 {
-    int l1, l2;
-
-    l1 = gen_new_label();
-    l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     gen_op_jnz_ecx(s->aflag, l1);
     gen_set_label(l2);
     gen_jmp_tb(s, next_eip, 1);
@@ -1129,9 +1065,9 @@
 
 static inline void gen_stos(DisasContext *s, TCGMemOp ot)
 {
-    gen_op_mov_v_reg(MO_32, cpu_T[0], R_EAX);
+    gen_op_mov_v_reg(MO_32, cpu_T0, R_EAX);
     gen_string_movl_A0_EDI(s);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
 }
@@ -1139,8 +1075,8 @@
 static inline void gen_lods(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_ESI(s);
-    gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
-    gen_op_mov_reg_v(ot, R_EAX, cpu_T[0]);
+    gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
+    gen_op_mov_reg_v(ot, R_EAX, cpu_T0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
 }
@@ -1148,7 +1084,7 @@
 static inline void gen_scas(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_EDI(s);
-    gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
     gen_op(s, OP_CMPL, ot, R_EAX);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
@@ -1157,7 +1093,7 @@
 static inline void gen_cmps(DisasContext *s, TCGMemOp ot)
 {
     gen_string_movl_A0_EDI(s);
-    gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
     gen_string_movl_A0_ESI(s);
     gen_op(s, OP_CMPL, ot, OR_TMP0);
     gen_op_movl_T0_Dshift(ot);
@@ -1165,41 +1101,59 @@
     gen_op_add_reg_T0(s->aflag, R_EDI);
 }
 
+static void gen_bpt_io(DisasContext *s, TCGv_i32 t_port, int ot)
+{
+    if (s->flags & HF_IOBPT_MASK) {
+        TCGv_i32 t_size = tcg_const_i32(1 << ot);
+        TCGv t_next = tcg_const_tl(s->pc - s->cs_base);
+
+        gen_helper_bpt_io(cpu_env, t_port, t_size, t_next);
+        tcg_temp_free_i32(t_size);
+        tcg_temp_free(t_next);
+    }
+}
+
+
 static inline void gen_ins(DisasContext *s, TCGMemOp ot)
 {
-    if (use_icount)
+    if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
+    }
     gen_string_movl_A0_EDI(s);
     /* Note: we must do this dummy write first to be restartable in
        case of page fault. */
-    tcg_gen_movi_tl(cpu_T[0], 0);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    tcg_gen_movi_tl(cpu_T0, 0);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_EDX]);
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
-    gen_helper_in_func(ot, cpu_T[0], cpu_tmp2_i32);
-    gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+    gen_helper_in_func(ot, cpu_T0, cpu_tmp2_i32);
+    gen_op_st_v(s, ot, cpu_T0, cpu_A0);
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_EDI);
-    if (use_icount)
+    gen_bpt_io(s, cpu_tmp2_i32, ot);
+    if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
+    }
 }
 
 static inline void gen_outs(DisasContext *s, TCGMemOp ot)
 {
-    if (use_icount)
+    if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
+    }
     gen_string_movl_A0_ESI(s);
-    gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+    gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
 
     tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_EDX]);
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
-    tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[0]);
+    tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T0);
     gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
-
     gen_op_movl_T0_Dshift(ot);
     gen_op_add_reg_T0(s->aflag, R_ESI);
-    if (use_icount)
+    gen_bpt_io(s, cpu_tmp2_i32, ot);
+    if (s->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
+    }
 }
 
 /* same method as Valgrind : we generate jumps to current or next
@@ -1208,14 +1162,14 @@
 static inline void gen_repz_ ## op(DisasContext *s, TCGMemOp ot,              \
                                  target_ulong cur_eip, target_ulong next_eip) \
 {                                                                             \
-    int l2;\
+    TCGLabel *l2;                                                             \
     gen_update_cc_op(s);                                                      \
     l2 = gen_jz_ecx_string(s, next_eip);                                      \
     gen_ ## op(s, ot);                                                        \
     gen_op_add_reg_im(s->aflag, R_ECX, -1);                                   \
     /* a loop would cause two single step exceptions if ECX = 1               \
        before rep string_insn */                                              \
-    if (!s->jmp_opt)                                                          \
+    if (s->repz_opt)                                                          \
         gen_op_jz_ecx(s->aflag, l2);                                          \
     gen_jmp(s, cur_eip);                                                      \
 }
@@ -1226,14 +1180,14 @@
                                    target_ulong next_eip,                     \
                                    int nz)                                    \
 {                                                                             \
-    int l2;\
+    TCGLabel *l2;                                                             \
     gen_update_cc_op(s);                                                      \
     l2 = gen_jz_ecx_string(s, next_eip);                                      \
     gen_ ## op(s, ot);                                                        \
     gen_op_add_reg_im(s->aflag, R_ECX, -1);                                   \
     gen_update_cc_op(s);                                                      \
     gen_jcc1(s, (JCC_Z << 1) | (nz ^ 1), l2);                                 \
-    if (!s->jmp_opt)                                                          \
+    if (s->repz_opt)                                                          \
         gen_op_jz_ecx(s->aflag, l2);                                          \
     gen_jmp(s, cur_eip);                                                      \
 }
@@ -1306,63 +1260,63 @@
 static void gen_op(DisasContext *s1, int op, TCGMemOp ot, int d)
 {
     if (d != OR_TMP0) {
-        gen_op_mov_v_reg(ot, cpu_T[0], d);
+        gen_op_mov_v_reg(ot, cpu_T0, d);
     } else {
-        gen_op_ld_v(s1, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s1, ot, cpu_T0, cpu_A0);
     }
     switch(op) {
     case OP_ADCL:
         gen_compute_eflags_c(s1, cpu_tmp4);
-        tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-        tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
+        tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
+        tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_tmp4);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update3_cc(cpu_tmp4);
         set_cc_op(s1, CC_OP_ADCB + ot);
         break;
     case OP_SBBL:
         gen_compute_eflags_c(s1, cpu_tmp4);
-        tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-        tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_tmp4);
+        tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1);
+        tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_tmp4);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update3_cc(cpu_tmp4);
         set_cc_op(s1, CC_OP_SBBB + ot);
         break;
     case OP_ADDL:
-        tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update2_cc();
         set_cc_op(s1, CC_OP_ADDB + ot);
         break;
     case OP_SUBL:
-        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T[0]);
-        tcg_gen_sub_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0);
+        tcg_gen_sub_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update2_cc();
         set_cc_op(s1, CC_OP_SUBB + ot);
         break;
     default:
     case OP_ANDL:
-        tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update1_cc();
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_ORL:
-        tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update1_cc();
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_XORL:
-        tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_T1);
         gen_op_st_rm_T0_A0(s1, ot, d);
         gen_op_update1_cc();
         set_cc_op(s1, CC_OP_LOGICB + ot);
         break;
     case OP_CMPL:
-        tcg_gen_mov_tl(cpu_cc_src, cpu_T[1]);
-        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T[0]);
-        tcg_gen_sub_tl(cpu_cc_dst, cpu_T[0], cpu_T[1]);
+        tcg_gen_mov_tl(cpu_cc_src, cpu_T1);
+        tcg_gen_mov_tl(cpu_cc_srcT, cpu_T0);
+        tcg_gen_sub_tl(cpu_cc_dst, cpu_T0, cpu_T1);
         set_cc_op(s1, CC_OP_SUBB + ot);
         break;
     }
@@ -1372,20 +1326,20 @@
 static void gen_inc(DisasContext *s1, TCGMemOp ot, int d, int c)
 {
     if (d != OR_TMP0) {
-        gen_op_mov_v_reg(ot, cpu_T[0], d);
+        gen_op_mov_v_reg(ot, cpu_T0, d);
     } else {
-        gen_op_ld_v(s1, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s1, ot, cpu_T0, cpu_A0);
     }
     gen_compute_eflags_c(s1, cpu_cc_src);
     if (c > 0) {
-        tcg_gen_addi_tl(cpu_T[0], cpu_T[0], 1);
+        tcg_gen_addi_tl(cpu_T0, cpu_T0, 1);
         set_cc_op(s1, CC_OP_INCB + ot);
     } else {
-        tcg_gen_addi_tl(cpu_T[0], cpu_T[0], -1);
+        tcg_gen_addi_tl(cpu_T0, cpu_T0, -1);
         set_cc_op(s1, CC_OP_DECB + ot);
     }
     gen_op_st_rm_T0_A0(s1, ot, d);
-    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
 }
 
 static void gen_shift_flags(DisasContext *s, TCGMemOp ot, TCGv result,
@@ -1440,33 +1394,33 @@
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
-    tcg_gen_andi_tl(cpu_T[1], cpu_T[1], mask);
-    tcg_gen_subi_tl(cpu_tmp0, cpu_T[1], 1);
+    tcg_gen_andi_tl(cpu_T1, cpu_T1, mask);
+    tcg_gen_subi_tl(cpu_tmp0, cpu_T1, 1);
 
     if (is_right) {
         if (is_arith) {
-            gen_exts(ot, cpu_T[0]);
-            tcg_gen_sar_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_sar_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            gen_exts(ot, cpu_T0);
+            tcg_gen_sar_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_sar_tl(cpu_T0, cpu_T0, cpu_T1);
         } else {
-            gen_extu(ot, cpu_T[0]);
-            tcg_gen_shr_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            gen_extu(ot, cpu_T0);
+            tcg_gen_shr_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_shr_tl(cpu_T0, cpu_T0, cpu_T1);
         }
     } else {
-        tcg_gen_shl_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
-        tcg_gen_shl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_shl_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
+        tcg_gen_shl_tl(cpu_T0, cpu_T0, cpu_T1);
     }
 
     /* store */
     gen_op_st_rm_T0_A0(s, ot, op1);
 
-    gen_shift_flags(s, ot, cpu_T[0], cpu_tmp0, cpu_T[1], is_right);
+    gen_shift_flags(s, ot, cpu_T0, cpu_tmp0, cpu_T1, is_right);
 }
 
 static void gen_shift_rm_im(DisasContext *s, TCGMemOp ot, int op1, int op2,
@@ -1476,25 +1430,25 @@
 
     /* load */
     if (op1 == OR_TMP0)
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     else
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
 
     op2 &= mask;
     if (op2 != 0) {
         if (is_right) {
             if (is_arith) {
-                gen_exts(ot, cpu_T[0]);
-                tcg_gen_sari_tl(cpu_tmp4, cpu_T[0], op2 - 1);
-                tcg_gen_sari_tl(cpu_T[0], cpu_T[0], op2);
+                gen_exts(ot, cpu_T0);
+                tcg_gen_sari_tl(cpu_tmp4, cpu_T0, op2 - 1);
+                tcg_gen_sari_tl(cpu_T0, cpu_T0, op2);
             } else {
-                gen_extu(ot, cpu_T[0]);
-                tcg_gen_shri_tl(cpu_tmp4, cpu_T[0], op2 - 1);
-                tcg_gen_shri_tl(cpu_T[0], cpu_T[0], op2);
+                gen_extu(ot, cpu_T0);
+                tcg_gen_shri_tl(cpu_tmp4, cpu_T0, op2 - 1);
+                tcg_gen_shri_tl(cpu_T0, cpu_T0, op2);
             }
         } else {
-            tcg_gen_shli_tl(cpu_tmp4, cpu_T[0], op2 - 1);
-            tcg_gen_shli_tl(cpu_T[0], cpu_T[0], op2);
+            tcg_gen_shli_tl(cpu_tmp4, cpu_T0, op2 - 1);
+            tcg_gen_shli_tl(cpu_T0, cpu_T0, op2);
         }
     }
 
@@ -1504,7 +1458,7 @@
     /* update eflags if non zero shift */
     if (op2 != 0) {
         tcg_gen_mov_tl(cpu_cc_src, cpu_tmp4);
-        tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+        tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
         set_cc_op(s, (is_right ? CC_OP_SARB : CC_OP_SHLB) + ot);
     }
 }
@@ -1516,41 +1470,41 @@
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
-    tcg_gen_andi_tl(cpu_T[1], cpu_T[1], mask);
+    tcg_gen_andi_tl(cpu_T1, cpu_T1, mask);
 
     switch (ot) {
     case MO_8:
         /* Replicate the 8-bit input so that a 32-bit rotate works.  */
-        tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
-        tcg_gen_muli_tl(cpu_T[0], cpu_T[0], 0x01010101);
+        tcg_gen_ext8u_tl(cpu_T0, cpu_T0);
+        tcg_gen_muli_tl(cpu_T0, cpu_T0, 0x01010101);
         goto do_long;
     case MO_16:
         /* Replicate the 16-bit input so that a 32-bit rotate works.  */
-        tcg_gen_deposit_tl(cpu_T[0], cpu_T[0], cpu_T[0], 16, 16);
+        tcg_gen_deposit_tl(cpu_T0, cpu_T0, cpu_T0, 16, 16);
         goto do_long;
     do_long:
 #ifdef TARGET_X86_64
     case MO_32:
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
         if (is_right) {
             tcg_gen_rotr_i32(cpu_tmp2_i32, cpu_tmp2_i32, cpu_tmp3_i32);
         } else {
             tcg_gen_rotl_i32(cpu_tmp2_i32, cpu_tmp2_i32, cpu_tmp3_i32);
         }
-        tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+        tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
         break;
 #endif
     default:
         if (is_right) {
-            tcg_gen_rotr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            tcg_gen_rotr_tl(cpu_T0, cpu_T0, cpu_T1);
         } else {
-            tcg_gen_rotl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+            tcg_gen_rotl_tl(cpu_T0, cpu_T0, cpu_T1);
         }
         break;
     }
@@ -1566,12 +1520,12 @@
        since we've computed the flags into CC_SRC, these variables are
        currently dead.  */
     if (is_right) {
-        tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask - 1);
-        tcg_gen_shri_tl(cpu_cc_dst, cpu_T[0], mask);
+        tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask - 1);
+        tcg_gen_shri_tl(cpu_cc_dst, cpu_T0, mask);
         tcg_gen_andi_tl(cpu_cc_dst, cpu_cc_dst, 1);
     } else {
-        tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask);
-        tcg_gen_andi_tl(cpu_cc_dst, cpu_T[0], 1);
+        tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask);
+        tcg_gen_andi_tl(cpu_cc_dst, cpu_T0, 1);
     }
     tcg_gen_andi_tl(cpu_cc_src2, cpu_cc_src2, 1);
     tcg_gen_xor_tl(cpu_cc_src2, cpu_cc_src2, cpu_cc_dst);
@@ -1582,7 +1536,7 @@
        exactly as we computed above.  */
     t0 = tcg_const_i32(0);
     t1 = tcg_temp_new_i32();
-    tcg_gen_trunc_tl_i32(t1, cpu_T[1]);
+    tcg_gen_trunc_tl_i32(t1, cpu_T1);
     tcg_gen_movi_i32(cpu_tmp2_i32, CC_OP_ADCOX); 
     tcg_gen_movi_i32(cpu_tmp3_i32, CC_OP_EFLAGS);
     tcg_gen_movcond_i32(TCG_COND_NE, cpu_cc_op, t1, t0,
@@ -1602,9 +1556,9 @@
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
     op2 &= mask;
@@ -1612,20 +1566,20 @@
         switch (ot) {
 #ifdef TARGET_X86_64
         case MO_32:
-            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
             if (is_right) {
                 tcg_gen_rotri_i32(cpu_tmp2_i32, cpu_tmp2_i32, op2);
             } else {
                 tcg_gen_rotli_i32(cpu_tmp2_i32, cpu_tmp2_i32, op2);
             }
-            tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+            tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
             break;
 #endif
         default:
             if (is_right) {
-                tcg_gen_rotri_tl(cpu_T[0], cpu_T[0], op2);
+                tcg_gen_rotri_tl(cpu_T0, cpu_T0, op2);
             } else {
-                tcg_gen_rotli_tl(cpu_T[0], cpu_T[0], op2);
+                tcg_gen_rotli_tl(cpu_T0, cpu_T0, op2);
             }
             break;
         case MO_8:
@@ -1638,10 +1592,10 @@
             if (is_right) {
                 shift = mask + 1 - shift;
             }
-            gen_extu(ot, cpu_T[0]);
-            tcg_gen_shli_tl(cpu_tmp0, cpu_T[0], shift);
-            tcg_gen_shri_tl(cpu_T[0], cpu_T[0], mask + 1 - shift);
-            tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            gen_extu(ot, cpu_T0);
+            tcg_gen_shli_tl(cpu_tmp0, cpu_T0, shift);
+            tcg_gen_shri_tl(cpu_T0, cpu_T0, mask + 1 - shift);
+            tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         }
     }
@@ -1658,12 +1612,12 @@
            since we've computed the flags into CC_SRC, these variables are
            currently dead.  */
         if (is_right) {
-            tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask - 1);
-            tcg_gen_shri_tl(cpu_cc_dst, cpu_T[0], mask);
+            tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask - 1);
+            tcg_gen_shri_tl(cpu_cc_dst, cpu_T0, mask);
             tcg_gen_andi_tl(cpu_cc_dst, cpu_cc_dst, 1);
         } else {
-            tcg_gen_shri_tl(cpu_cc_src2, cpu_T[0], mask);
-            tcg_gen_andi_tl(cpu_cc_dst, cpu_T[0], 1);
+            tcg_gen_shri_tl(cpu_cc_src2, cpu_T0, mask);
+            tcg_gen_andi_tl(cpu_cc_dst, cpu_T0, 1);
         }
         tcg_gen_andi_tl(cpu_cc_src2, cpu_cc_src2, 1);
         tcg_gen_xor_tl(cpu_cc_src2, cpu_cc_src2, cpu_cc_dst);
@@ -1680,24 +1634,24 @@
 
     /* load */
     if (op1 == OR_TMP0)
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     else
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     
     if (is_right) {
         switch (ot) {
         case MO_8:
-            gen_helper_rcrb(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrb(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_16:
-            gen_helper_rcrw(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrw(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_32:
-            gen_helper_rcrl(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrl(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_helper_rcrq(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcrq(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #endif
         default:
@@ -1706,17 +1660,17 @@
     } else {
         switch (ot) {
         case MO_8:
-            gen_helper_rclb(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rclb(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_16:
-            gen_helper_rclw(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rclw(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
         case MO_32:
-            gen_helper_rcll(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rcll(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_helper_rclq(cpu_T[0], cpu_env, cpu_T[0], cpu_T[1]);
+            gen_helper_rclq(cpu_T0, cpu_env, cpu_T0, cpu_T1);
             break;
 #endif
         default:
@@ -1736,9 +1690,9 @@
 
     /* load */
     if (op1 == OR_TMP0) {
-        gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
     } else {
-        gen_op_mov_v_reg(ot, cpu_T[0], op1);
+        gen_op_mov_v_reg(ot, cpu_T0, op1);
     }
 
     count = tcg_temp_new();
@@ -1750,11 +1704,11 @@
            This means "shrdw C, B, A" shifts A:B:A >> C.  Build the B:A
            portion by constructing it as a 32-bit value.  */
         if (is_right) {
-            tcg_gen_deposit_tl(cpu_tmp0, cpu_T[0], cpu_T[1], 16, 16);
-            tcg_gen_mov_tl(cpu_T[1], cpu_T[0]);
-            tcg_gen_mov_tl(cpu_T[0], cpu_tmp0);
+            tcg_gen_deposit_tl(cpu_tmp0, cpu_T0, cpu_T1, 16, 16);
+            tcg_gen_mov_tl(cpu_T1, cpu_T0);
+            tcg_gen_mov_tl(cpu_T0, cpu_tmp0);
         } else {
-            tcg_gen_deposit_tl(cpu_T[1], cpu_T[0], cpu_T[1], 16, 16);
+            tcg_gen_deposit_tl(cpu_T1, cpu_T0, cpu_T1, 16, 16);
         }
         /* FALLTHRU */
 #ifdef TARGET_X86_64
@@ -1762,57 +1716,57 @@
         /* Concatenate the two 32-bit values and use a 64-bit shift.  */
         tcg_gen_subi_tl(cpu_tmp0, count, 1);
         if (is_right) {
-            tcg_gen_concat_tl_i64(cpu_T[0], cpu_T[0], cpu_T[1]);
-            tcg_gen_shr_i64(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_shr_i64(cpu_T[0], cpu_T[0], count);
+            tcg_gen_concat_tl_i64(cpu_T0, cpu_T0, cpu_T1);
+            tcg_gen_shr_i64(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_shr_i64(cpu_T0, cpu_T0, count);
         } else {
-            tcg_gen_concat_tl_i64(cpu_T[0], cpu_T[1], cpu_T[0]);
-            tcg_gen_shl_i64(cpu_tmp0, cpu_T[0], cpu_tmp0);
-            tcg_gen_shl_i64(cpu_T[0], cpu_T[0], count);
+            tcg_gen_concat_tl_i64(cpu_T0, cpu_T1, cpu_T0);
+            tcg_gen_shl_i64(cpu_tmp0, cpu_T0, cpu_tmp0);
+            tcg_gen_shl_i64(cpu_T0, cpu_T0, count);
             tcg_gen_shri_i64(cpu_tmp0, cpu_tmp0, 32);
-            tcg_gen_shri_i64(cpu_T[0], cpu_T[0], 32);
+            tcg_gen_shri_i64(cpu_T0, cpu_T0, 32);
         }
         break;
 #endif
     default:
         tcg_gen_subi_tl(cpu_tmp0, count, 1);
         if (is_right) {
-            tcg_gen_shr_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
+            tcg_gen_shr_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
 
             tcg_gen_subfi_tl(cpu_tmp4, mask + 1, count);
-            tcg_gen_shr_tl(cpu_T[0], cpu_T[0], count);
-            tcg_gen_shl_tl(cpu_T[1], cpu_T[1], cpu_tmp4);
+            tcg_gen_shr_tl(cpu_T0, cpu_T0, count);
+            tcg_gen_shl_tl(cpu_T1, cpu_T1, cpu_tmp4);
         } else {
-            tcg_gen_shl_tl(cpu_tmp0, cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_T0, cpu_tmp0);
             if (ot == MO_16) {
                 /* Only needed if count > 16, for Intel behaviour.  */
                 tcg_gen_subfi_tl(cpu_tmp4, 33, count);
-                tcg_gen_shr_tl(cpu_tmp4, cpu_T[1], cpu_tmp4);
+                tcg_gen_shr_tl(cpu_tmp4, cpu_T1, cpu_tmp4);
                 tcg_gen_or_tl(cpu_tmp0, cpu_tmp0, cpu_tmp4);
             }
 
             tcg_gen_subfi_tl(cpu_tmp4, mask + 1, count);
-            tcg_gen_shl_tl(cpu_T[0], cpu_T[0], count);
-            tcg_gen_shr_tl(cpu_T[1], cpu_T[1], cpu_tmp4);
+            tcg_gen_shl_tl(cpu_T0, cpu_T0, count);
+            tcg_gen_shr_tl(cpu_T1, cpu_T1, cpu_tmp4);
         }
         tcg_gen_movi_tl(cpu_tmp4, 0);
-        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T[1], count, cpu_tmp4,
-                           cpu_tmp4, cpu_T[1]);
-        tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T1, count, cpu_tmp4,
+                           cpu_tmp4, cpu_T1);
+        tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_T1);
         break;
     }
 
     /* store */
     gen_op_st_rm_T0_A0(s, ot, op1);
 
-    gen_shift_flags(s, ot, cpu_T[0], cpu_tmp0, count, is_right);
+    gen_shift_flags(s, ot, cpu_T0, cpu_tmp0, count, is_right);
     tcg_temp_free(count);
 }
 
 static void gen_shift(DisasContext *s1, int op, TCGMemOp ot, int d, int s)
 {
     if (s != OR_TMP1)
-        gen_op_mov_v_reg(ot, cpu_T[1], s);
+        gen_op_mov_v_reg(ot, cpu_T1, s);
     switch(op) {
     case OP_ROL:
         gen_rot_rm_T1(s1, ot, d, 0);
@@ -1860,48 +1814,58 @@
         break;
     default:
         /* currently not optimized */
-        tcg_gen_movi_tl(cpu_T[1], c);
+        tcg_gen_movi_tl(cpu_T1, c);
         gen_shift(s1, op, ot, d, OR_TMP1);
         break;
     }
 }
 
-static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
-{
-    target_long disp;
-    int havesib;
+/* Decompose an address.  */
+
+typedef struct AddressParts {
+    int def_seg;
     int base;
     int index;
     int scale;
-    int mod, rm, code, override, must_add_seg;
-    TCGv sum;
+    target_long disp;
+} AddressParts;
 
-    override = s->override;
-    must_add_seg = s->addseg;
-    if (override >= 0)
-        must_add_seg = 1;
+static AddressParts gen_lea_modrm_0(CPUX86State *env, DisasContext *s,
+                                    int modrm)
+{
+    int def_seg, base, index, scale, mod, rm;
+    target_long disp;
+    bool havesib;
+
+    def_seg = R_DS;
+    index = -1;
+    scale = 0;
+    disp = 0;
+
     mod = (modrm >> 6) & 3;
     rm = modrm & 7;
+    base = rm | REX_B(s);
+
+    if (mod == 3) {
+        /* Normally filtered out earlier, but including this path
+           simplifies multi-byte nop, as well as bndcl, bndcu, bndcn.  */
+        goto done;
+    }
 
     switch (s->aflag) {
     case MO_64:
     case MO_32:
         havesib = 0;
-        base = rm;
-        index = -1;
-        scale = 0;
-
-        if (base == 4) {
-            havesib = 1;
-            code = cpu_ldub_code(env, s->pc++);
+        if (rm == 4) {
+            int code = cpu_ldub_code(env, s->pc++);
             scale = (code >> 6) & 3;
             index = ((code >> 3) & 7) | REX_X(s);
             if (index == 4) {
                 index = -1;  /* no index */
             }
-            base = (code & 7);
+            base = (code & 7) | REX_B(s);
+            havesib = 1;
         }
-        base |= REX_B(s);
 
         switch (mod) {
         case 0:
@@ -1910,10 +1874,9 @@
                 disp = (int32_t)cpu_ldl_code(env, s->pc);
                 s->pc += 4;
                 if (CODE64(s) && !havesib) {
+                    base = -2;
                     disp += s->pc + s->rip_offset;
                 }
-            } else {
-                disp = 0;
             }
             break;
         case 1:
@@ -1930,204 +1893,132 @@
         if (base == R_ESP && s->popl_esp_hack) {
             disp += s->popl_esp_hack;
         }
-
-        /* Compute the address, with a minimum number of TCG ops.  */
-        TCGV_UNUSED(sum);
-        if (index >= 0) {
-            if (scale == 0) {
-                sum = cpu_regs[index];
-            } else {
-                tcg_gen_shli_tl(cpu_A0, cpu_regs[index], scale);
-                sum = cpu_A0;
-            }
-            if (base >= 0) {
-                tcg_gen_add_tl(cpu_A0, sum, cpu_regs[base]);
-                sum = cpu_A0;
-            }
-        } else if (base >= 0) {
-            sum = cpu_regs[base];
-        }
-        if (TCGV_IS_UNUSED(sum)) {
-            tcg_gen_movi_tl(cpu_A0, disp);
-        } else {
-            tcg_gen_addi_tl(cpu_A0, sum, disp);
-        }
-
-        if (must_add_seg) {
-            if (override < 0) {
-                if (base == R_EBP || base == R_ESP) {
-                    override = R_SS;
-                } else {
-                    override = R_DS;
-                }
-            }
-
-            tcg_gen_ld_tl(cpu_tmp0, cpu_env,
-                          offsetof(CPUX86State, segs[override].base));
-            if (CODE64(s)) {
-                if (s->aflag == MO_32) {
-                    tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
-                }
-                tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-                return;
-            }
-
-            tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-        }
-
-        if (s->aflag == MO_32) {
-            tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+        if (base == R_EBP || base == R_ESP) {
+            def_seg = R_SS;
         }
         break;
 
     case MO_16:
-        switch (mod) {
-        case 0:
+        if (mod == 0) {
             if (rm == 6) {
+                base = -1;
                 disp = cpu_lduw_code(env, s->pc);
                 s->pc += 2;
-                tcg_gen_movi_tl(cpu_A0, disp);
-                rm = 0; /* avoid SS override */
-                goto no_rm;
-            } else {
-                disp = 0;
+                break;
             }
-            break;
-        case 1:
+        } else if (mod == 1) {
             disp = (int8_t)cpu_ldub_code(env, s->pc++);
-            break;
-        default:
-        case 2:
+        } else {
             disp = (int16_t)cpu_lduw_code(env, s->pc);
             s->pc += 2;
-            break;
         }
 
-        sum = cpu_A0;
         switch (rm) {
         case 0:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBX], cpu_regs[R_ESI]);
+            base = R_EBX;
+            index = R_ESI;
             break;
         case 1:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBX], cpu_regs[R_EDI]);
+            base = R_EBX;
+            index = R_EDI;
             break;
         case 2:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBP], cpu_regs[R_ESI]);
+            base = R_EBP;
+            index = R_ESI;
+            def_seg = R_SS;
             break;
         case 3:
-            tcg_gen_add_tl(cpu_A0, cpu_regs[R_EBP], cpu_regs[R_EDI]);
+            base = R_EBP;
+            index = R_EDI;
+            def_seg = R_SS;
             break;
         case 4:
-            sum = cpu_regs[R_ESI];
+            base = R_ESI;
             break;
         case 5:
-            sum = cpu_regs[R_EDI];
+            base = R_EDI;
             break;
         case 6:
-            sum = cpu_regs[R_EBP];
+            base = R_EBP;
+            def_seg = R_SS;
             break;
         default:
         case 7:
-            sum = cpu_regs[R_EBX];
+            base = R_EBX;
             break;
         }
-        tcg_gen_addi_tl(cpu_A0, sum, disp);
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    no_rm:
-        if (must_add_seg) {
-            if (override < 0) {
-                if (rm == 2 || rm == 3 || rm == 6) {
-                    override = R_SS;
-                } else {
-                    override = R_DS;
-                }
-            }
-            gen_op_addl_A0_seg(s, override);
-        }
         break;
 
     default:
         tcg_abort();
     }
+
+ done:
+    return (AddressParts){ def_seg, base, index, scale, disp };
+}
+
+/* Compute the address, with a minimum number of TCG ops.  */
+static TCGv gen_lea_modrm_1(AddressParts a)
+{
+    TCGv ea;
+
+    TCGV_UNUSED(ea);
+    if (a.index >= 0) {
+        if (a.scale == 0) {
+            ea = cpu_regs[a.index];
+        } else {
+            tcg_gen_shli_tl(cpu_A0, cpu_regs[a.index], a.scale);
+            ea = cpu_A0;
+        }
+        if (a.base >= 0) {
+            tcg_gen_add_tl(cpu_A0, ea, cpu_regs[a.base]);
+            ea = cpu_A0;
+        }
+    } else if (a.base >= 0) {
+        ea = cpu_regs[a.base];
+    }
+    if (TCGV_IS_UNUSED(ea)) {
+        tcg_gen_movi_tl(cpu_A0, a.disp);
+        ea = cpu_A0;
+    } else if (a.disp != 0) {
+        tcg_gen_addi_tl(cpu_A0, ea, a.disp);
+        ea = cpu_A0;
+    }
+
+    return ea;
+}
+
+static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm)
+{
+    AddressParts a = gen_lea_modrm_0(env, s, modrm);
+    TCGv ea = gen_lea_modrm_1(a);
+    gen_lea_v_seg(s, s->aflag, ea, a.def_seg, s->override);
 }
 
 static void gen_nop_modrm(CPUX86State *env, DisasContext *s, int modrm)
 {
-    int mod, rm, base, code;
+    (void)gen_lea_modrm_0(env, s, modrm);
+}
 
-    mod = (modrm >> 6) & 3;
-    if (mod == 3)
-        return;
-    rm = modrm & 7;
+/* Used for BNDCL, BNDCU, BNDCN.  */
+static void gen_bndck(CPUX86State *env, DisasContext *s, int modrm,
+                      TCGCond cond, TCGv_i64 bndv)
+{
+    TCGv ea = gen_lea_modrm_1(gen_lea_modrm_0(env, s, modrm));
 
-    switch (s->aflag) {
-    case MO_64:
-    case MO_32:
-        base = rm;
-
-        if (base == 4) {
-            code = cpu_ldub_code(env, s->pc++);
-            base = (code & 7);
-        }
-
-        switch (mod) {
-        case 0:
-            if (base == 5) {
-                s->pc += 4;
-            }
-            break;
-        case 1:
-            s->pc++;
-            break;
-        default:
-        case 2:
-            s->pc += 4;
-            break;
-        }
-        break;
-
-    case MO_16:
-        switch (mod) {
-        case 0:
-            if (rm == 6) {
-                s->pc += 2;
-            }
-            break;
-        case 1:
-            s->pc++;
-            break;
-        default:
-        case 2:
-            s->pc += 2;
-            break;
-        }
-        break;
-
-    default:
-        tcg_abort();
+    tcg_gen_extu_tl_i64(cpu_tmp1_i64, ea);
+    if (!CODE64(s)) {
+        tcg_gen_ext32u_i64(cpu_tmp1_i64, cpu_tmp1_i64);
     }
+    tcg_gen_setcond_i64(cond, cpu_tmp1_i64, cpu_tmp1_i64, bndv);
+    tcg_gen_extrl_i64_i32(cpu_tmp2_i32, cpu_tmp1_i64);
+    gen_helper_bndck(cpu_env, cpu_tmp2_i32);
 }
 
 /* used for LEA and MOV AX, mem */
 static void gen_add_A0_ds_seg(DisasContext *s)
 {
-    int override, must_add_seg;
-    must_add_seg = s->addseg;
-    override = R_DS;
-    if (s->override >= 0) {
-        override = s->override;
-        must_add_seg = 1;
-    }
-    if (must_add_seg) {
-#ifdef TARGET_X86_64
-        if (CODE64(s)) {
-            gen_op_addq_A0_seg(override);
-        } else
-#endif
-        {
-            gen_op_addl_A0_seg(s, override);
-        }
-    }
+    gen_lea_v_seg(s, s->aflag, cpu_A0, R_DS, s->override);
 }
 
 /* generate modrm memory load or store of 'reg'. TMP0 is used if reg ==
@@ -2142,23 +2033,23 @@
     if (mod == 3) {
         if (is_store) {
             if (reg != OR_TMP0)
-                gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
             if (reg != OR_TMP0)
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
         }
     } else {
         gen_lea_modrm(env, s, modrm);
         if (is_store) {
             if (reg != OR_TMP0)
-                gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
             if (reg != OR_TMP0)
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
         }
     }
 }
@@ -2198,20 +2089,25 @@
     }
 }
 
+static inline bool use_goto_tb(DisasContext *s, target_ulong pc)
+{
+#ifndef CONFIG_USER_ONLY
+    return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) ||
+           (pc & TARGET_PAGE_MASK) == (s->pc_start & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(DisasContext *s, int tb_num, target_ulong eip)
 {
-    TranslationBlock *tb;
-    target_ulong pc;
+    target_ulong pc = s->cs_base + eip;
 
-    pc = s->cs_base + eip;
-    tb = s->tb;
-    /* NOTE: we handle the case where the TB spans two pages here */
-    if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) ||
-        (pc & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))  {
+    if (use_goto_tb(s, pc))  {
         /* jump to same page: we can use a direct jump */
         tcg_gen_goto_tb(tb_num);
         gen_jmp_im(eip);
-        tcg_gen_exit_tb((uintptr_t)tb + tb_num);
+        tcg_gen_exit_tb((uintptr_t)s->tb + tb_num);
     } else {
         /* jump to another page: currently not optimized */
         gen_jmp_im(eip);
@@ -2222,7 +2118,7 @@
 static inline void gen_jcc(DisasContext *s, int b,
                            target_ulong val, target_ulong next_eip)
 {
-    int l1, l2;
+    TCGLabel *l1, *l2;
 
     if (s->jmp_opt) {
         l1 = gen_new_label();
@@ -2255,7 +2151,7 @@
 
     gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
 
-    cc = gen_prepare_cc(s, b, cpu_T[1]);
+    cc = gen_prepare_cc(s, b, cpu_T1);
     if (cc.mask != -1) {
         TCGv t0 = tcg_temp_new();
         tcg_gen_andi_tl(t0, cc.reg, cc.mask);
@@ -2265,9 +2161,9 @@
         cc.reg2 = tcg_const_tl(cc.imm);
     }
 
-    tcg_gen_movcond_tl(cc.cond, cpu_T[0], cc.reg, cc.reg2,
-                       cpu_T[0], cpu_regs[reg]);
-    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+    tcg_gen_movcond_tl(cc.cond, cpu_T0, cc.reg, cc.reg2,
+                       cpu_T0, cpu_regs[reg]);
+    gen_op_mov_reg_v(ot, reg, cpu_T0);
 
     if (cc.mask != -1) {
         tcg_temp_free(cc.reg);
@@ -2279,29 +2175,24 @@
 
 static inline void gen_op_movl_T0_seg(int seg_reg)
 {
-    tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
+    tcg_gen_ld32u_tl(cpu_T0, cpu_env,
                      offsetof(CPUX86State,segs[seg_reg].selector));
 }
 
 static inline void gen_op_movl_seg_T0_vm(int seg_reg)
 {
-    tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffff);
-    tcg_gen_st32_tl(cpu_T[0], cpu_env, 
+    tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
+    tcg_gen_st32_tl(cpu_T0, cpu_env,
                     offsetof(CPUX86State,segs[seg_reg].selector));
-    tcg_gen_shli_tl(cpu_T[0], cpu_T[0], 4);
-    tcg_gen_st_tl(cpu_T[0], cpu_env, 
-                  offsetof(CPUX86State,segs[seg_reg].base));
+    tcg_gen_shli_tl(cpu_seg_base[seg_reg], cpu_T0, 4);
 }
 
 /* move T0 to seg_reg and compute if the CPU state may change. Never
    call this function with seg_reg == R_CS */
-static void gen_movl_seg_T0(DisasContext *s, int seg_reg, target_ulong cur_eip)
+static void gen_movl_seg_T0(DisasContext *s, int seg_reg)
 {
     if (s->pe && !s->vm86) {
-        /* XXX: optimize by finding processor state dynamically */
-        gen_update_cc_op(s);
-        gen_jmp_im(cur_eip);
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         gen_helper_load_seg(cpu_env, tcg_const_i32(seg_reg), cpu_tmp2_i32);
         /* abort translation because the addseg value may change or
            because ss32 may change. For R_SS, translation must always
@@ -2342,44 +2233,25 @@
 
 static inline void gen_stack_update(DisasContext *s, int addend)
 {
-#ifdef TARGET_X86_64
-    if (CODE64(s)) {
-        gen_op_add_reg_im(MO_64, R_ESP, addend);
-    } else
-#endif
-    if (s->ss32) {
-        gen_op_add_reg_im(MO_32, R_ESP, addend);
-    } else {
-        gen_op_add_reg_im(MO_16, R_ESP, addend);
-    }
+    gen_op_add_reg_im(mo_stacksize(s), R_ESP, addend);
 }
 
 /* Generate a push. It depends on ss32, addseg and dflag.  */
 static void gen_push_v(DisasContext *s, TCGv val)
 {
-    TCGMemOp a_ot, d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp a_ot = mo_stacksize(s);
     int size = 1 << d_ot;
     TCGv new_esp = cpu_A0;
 
     tcg_gen_subi_tl(cpu_A0, cpu_regs[R_ESP], size);
 
-    if (CODE64(s)) {
-        a_ot = MO_64;
-    } else if (s->ss32) {
-        a_ot = MO_32;
+    if (!CODE64(s)) {
         if (s->addseg) {
             new_esp = cpu_tmp4;
             tcg_gen_mov_tl(new_esp, cpu_A0);
-            gen_op_addl_A0_seg(s, R_SS);
-        } else {
-            tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
         }
-    } else {
-        a_ot = MO_16;
-        new_esp = cpu_tmp4;
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-        tcg_gen_mov_tl(new_esp, cpu_A0);
-        gen_op_addl_A0_seg(s, R_SS);
+        gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
     }
 
     gen_op_st_v(s, d_ot, val, cpu_A0);
@@ -2390,127 +2262,112 @@
 static TCGMemOp gen_pop_T0(DisasContext *s)
 {
     TCGMemOp d_ot = mo_pushpop(s, s->dflag);
-    TCGv addr = cpu_A0;
 
-    if (CODE64(s)) {
-        addr = cpu_regs[R_ESP];
-    } else if (!s->ss32) {
-        tcg_gen_ext16u_tl(cpu_A0, cpu_regs[R_ESP]);
-        gen_op_addl_A0_seg(s, R_SS);
-    } else if (s->addseg) {
-        tcg_gen_mov_tl(cpu_A0, cpu_regs[R_ESP]);
-        gen_op_addl_A0_seg(s, R_SS);
-    } else {
-        tcg_gen_ext32u_tl(cpu_A0, cpu_regs[R_ESP]);
-    }
+    gen_lea_v_seg(s, mo_stacksize(s), cpu_regs[R_ESP], R_SS, -1);
+    gen_op_ld_v(s, d_ot, cpu_T0, cpu_A0);
 
-    gen_op_ld_v(s, d_ot, cpu_T[0], addr);
     return d_ot;
 }
 
-static void gen_pop_update(DisasContext *s, TCGMemOp ot)
+static inline void gen_pop_update(DisasContext *s, TCGMemOp ot)
 {
     gen_stack_update(s, 1 << ot);
 }
 
-static void gen_stack_A0(DisasContext *s)
+static inline void gen_stack_A0(DisasContext *s)
 {
-    gen_op_movl_A0_reg(R_ESP);
-    if (!s->ss32)
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-    if (s->addseg)
-        gen_op_addl_A0_seg(s, R_SS);
+    gen_lea_v_seg(s, s->ss32 ? MO_32 : MO_16, cpu_regs[R_ESP], R_SS, -1);
 }
 
-/* NOTE: wrap around in 16 bit not fully handled */
 static void gen_pusha(DisasContext *s)
 {
+    TCGMemOp s_ot = s->ss32 ? MO_32 : MO_16;
+    TCGMemOp d_ot = s->dflag;
+    int size = 1 << d_ot;
     int i;
-    gen_op_movl_A0_reg(R_ESP);
-    gen_op_addl_A0_im(-8 << s->dflag);
-    if (!s->ss32)
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-    if (s->addseg)
-        gen_op_addl_A0_seg(s, R_SS);
-    for(i = 0;i < 8; i++) {
-        gen_op_mov_v_reg(MO_32, cpu_T[0], 7 - i);
-        gen_op_st_v(s, s->dflag, cpu_T[0], cpu_A0);
-        gen_op_addl_A0_im(1 << s->dflag);
+
+    for (i = 0; i < 8; i++) {
+        tcg_gen_addi_tl(cpu_A0, cpu_regs[R_ESP], (i - 8) * size);
+        gen_lea_v_seg(s, s_ot, cpu_A0, R_SS, -1);
+        gen_op_st_v(s, d_ot, cpu_regs[7 - i], cpu_A0);
     }
-    gen_op_mov_reg_v(MO_16 + s->ss32, R_ESP, cpu_T[1]);
+
+    gen_stack_update(s, -8 * size);
 }
 
-/* NOTE: wrap around in 16 bit not fully handled */
 static void gen_popa(DisasContext *s)
 {
+    TCGMemOp s_ot = s->ss32 ? MO_32 : MO_16;
+    TCGMemOp d_ot = s->dflag;
+    int size = 1 << d_ot;
     int i;
-    gen_op_movl_A0_reg(R_ESP);
-    if (!s->ss32)
-        tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-    tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-    tcg_gen_addi_tl(cpu_T[1], cpu_T[1], 8 << s->dflag);
-    if (s->addseg)
-        gen_op_addl_A0_seg(s, R_SS);
-    for(i = 0;i < 8; i++) {
+
+    for (i = 0; i < 8; i++) {
         /* ESP is not reloaded */
-        if (i != 3) {
-            gen_op_ld_v(s, s->dflag, cpu_T[0], cpu_A0);
-            gen_op_mov_reg_v(s->dflag, 7 - i, cpu_T[0]);
+        if (7 - i == R_ESP) {
+            continue;
         }
-        gen_op_addl_A0_im(1 << s->dflag);
+        tcg_gen_addi_tl(cpu_A0, cpu_regs[R_ESP], i * size);
+        gen_lea_v_seg(s, s_ot, cpu_A0, R_SS, -1);
+        gen_op_ld_v(s, d_ot, cpu_T0, cpu_A0);
+        gen_op_mov_reg_v(d_ot, 7 - i, cpu_T0);
     }
-    gen_op_mov_reg_v(MO_16 + s->ss32, R_ESP, cpu_T[1]);
+
+    gen_stack_update(s, 8 * size);
 }
 
 static void gen_enter(DisasContext *s, int esp_addend, int level)
 {
-    TCGMemOp ot = mo_pushpop(s, s->dflag);
-    int opsize = 1 << ot;
+    TCGMemOp d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp a_ot = CODE64(s) ? MO_64 : s->ss32 ? MO_32 : MO_16;
+    int size = 1 << d_ot;
 
-    level &= 0x1f;
-#ifdef TARGET_X86_64
-    if (CODE64(s)) {
-        gen_op_movl_A0_reg(R_ESP);
-        gen_op_addq_A0_im(-opsize);
-        tcg_gen_mov_tl(cpu_T[1], cpu_A0);
+    /* Push BP; compute FrameTemp into T1.  */
+    tcg_gen_subi_tl(cpu_T1, cpu_regs[R_ESP], size);
+    gen_lea_v_seg(s, a_ot, cpu_T1, R_SS, -1);
+    gen_op_st_v(s, d_ot, cpu_regs[R_EBP], cpu_A0);
 
-        /* push bp */
-        gen_op_mov_v_reg(MO_32, cpu_T[0], R_EBP);
-        gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
-        if (level) {
-            /* XXX: must save state */
-            gen_helper_enter64_level(cpu_env, tcg_const_i32(level),
-                                     tcg_const_i32((ot == MO_64)),
-                                     cpu_T[1]);
+    level &= 31;
+    if (level != 0) {
+        int i;
+
+        /* Copy level-1 pointers from the previous frame.  */
+        for (i = 1; i < level; ++i) {
+            tcg_gen_subi_tl(cpu_A0, cpu_regs[R_EBP], size * i);
+            gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
+            gen_op_ld_v(s, d_ot, cpu_tmp0, cpu_A0);
+
+            tcg_gen_subi_tl(cpu_A0, cpu_T1, size * i);
+            gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
+            gen_op_st_v(s, d_ot, cpu_tmp0, cpu_A0);
         }
-        gen_op_mov_reg_v(ot, R_EBP, cpu_T[1]);
-        tcg_gen_addi_tl(cpu_T[1], cpu_T[1], -esp_addend + (-opsize * level));
-        gen_op_mov_reg_v(MO_64, R_ESP, cpu_T[1]);
-    } else
-#endif
-    {
-        gen_op_movl_A0_reg(R_ESP);
-        gen_op_addl_A0_im(-opsize);
-        if (!s->ss32)
-            tcg_gen_ext16u_tl(cpu_A0, cpu_A0);
-        tcg_gen_mov_tl(cpu_T[1], cpu_A0);
-        if (s->addseg)
-            gen_op_addl_A0_seg(s, R_SS);
-        /* push bp */
-        gen_op_mov_v_reg(MO_32, cpu_T[0], R_EBP);
-        gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
-        if (level) {
-            /* XXX: must save state */
-            gen_helper_enter_level(cpu_env, tcg_const_i32(level),
-                                   tcg_const_i32(s->dflag - 1),
-                                   cpu_T[1]);
-        }
-        gen_op_mov_reg_v(ot, R_EBP, cpu_T[1]);
-        tcg_gen_addi_tl(cpu_T[1], cpu_T[1], -esp_addend + (-opsize * level));
-        gen_op_mov_reg_v(MO_16 + s->ss32, R_ESP, cpu_T[1]);
+
+        /* Push the current FrameTemp as the last level.  */
+        tcg_gen_subi_tl(cpu_A0, cpu_T1, size * level);
+        gen_lea_v_seg(s, a_ot, cpu_A0, R_SS, -1);
+        gen_op_st_v(s, d_ot, cpu_T1, cpu_A0);
     }
+
+    /* Copy the FrameTemp value to EBP.  */
+    gen_op_mov_reg_v(a_ot, R_EBP, cpu_T1);
+
+    /* Compute the final value of ESP.  */
+    tcg_gen_subi_tl(cpu_T1, cpu_T1, esp_addend + size * level);
+    gen_op_mov_reg_v(a_ot, R_ESP, cpu_T1);
+}
+
+static void gen_leave(DisasContext *s)
+{
+    TCGMemOp d_ot = mo_pushpop(s, s->dflag);
+    TCGMemOp a_ot = mo_stacksize(s);
+
+    gen_lea_v_seg(s, a_ot, cpu_regs[R_EBP], R_SS, -1);
+    gen_op_ld_v(s, d_ot, cpu_T0, cpu_A0);
+
+    tcg_gen_addi_tl(cpu_T1, cpu_regs[R_EBP], 1 << d_ot);
+
+    gen_op_mov_reg_v(d_ot, R_EBP, cpu_T0);
+    gen_op_mov_reg_v(a_ot, R_ESP, cpu_T1);
 }
 
 static void gen_exception(DisasContext *s, int trapno, target_ulong cur_eip)
@@ -2521,6 +2378,30 @@
     s->is_jmp = DISAS_TB_JUMP;
 }
 
+/* Generate #UD for the current instruction.  The assumption here is that
+   the instruction is known, but it isn't allowed in the current cpu mode.  */
+static void gen_illegal_opcode(DisasContext *s)
+{
+    gen_exception(s, EXCP06_ILLOP, s->pc_start - s->cs_base);
+}
+
+/* Similarly, except that the assumption here is that we don't decode
+   the instruction at all -- either a missing opcode, an unimplemented
+   feature, or just a bogus instruction stream.  */
+static void gen_unknown_opcode(CPUX86State *env, DisasContext *s)
+{
+    gen_illegal_opcode(s);
+
+    if (qemu_loglevel_mask(LOG_UNIMP)) {
+        target_ulong pc = s->pc_start, end = s->pc;
+        qemu_log("ILLOPC: " TARGET_FMT_lx ":", pc);
+        for (; pc < end; ++pc) {
+            qemu_log(" %02x", cpu_ldub_code(env, pc));
+        }
+        qemu_log("\n");
+    }
+}
+
 /* an interrupt is different from an exception because of the
    privilege checks */
 static void gen_interrupt(DisasContext *s, int intno,
@@ -2541,14 +2422,56 @@
     s->is_jmp = DISAS_TB_JUMP;
 }
 
-/* generate a generic end of block. Trace exception is also generated
-   if needed */
-static void gen_eob(DisasContext *s)
+static void gen_set_hflag(DisasContext *s, uint32_t mask)
+{
+    if ((s->flags & mask) == 0) {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_ld_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_ori_i32(t, t, mask);
+        tcg_gen_st_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_temp_free_i32(t);
+        s->flags |= mask;
+    }
+}
+
+static void gen_reset_hflag(DisasContext *s, uint32_t mask)
+{
+    if (s->flags & mask) {
+        TCGv_i32 t = tcg_temp_new_i32();
+        tcg_gen_ld_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_gen_andi_i32(t, t, ~mask);
+        tcg_gen_st_i32(t, cpu_env, offsetof(CPUX86State, hflags));
+        tcg_temp_free_i32(t);
+        s->flags &= ~mask;
+    }
+}
+
+/* Clear BND registers during legacy branches.  */
+static void gen_bnd_jmp(DisasContext *s)
+{
+    /* Clear the registers only if BND prefix is missing, MPX is enabled,
+       and if the BNDREGs are known to be in use (non-zero) already.
+       The helper itself will check BNDPRESERVE at runtime.  */
+    if ((s->prefix & PREFIX_REPNZ) == 0
+        && (s->flags & HF_MPX_EN_MASK) != 0
+        && (s->flags & HF_MPX_IU_MASK) != 0) {
+        gen_helper_bnd_jmp(cpu_env);
+    }
+}
+
+/* Generate an end of block. Trace exception is also generated if needed.
+   If IIM, set HF_INHIBIT_IRQ_MASK if it isn't already set.  */
+static void gen_eob_inhibit_irq(DisasContext *s, bool inhibit)
 {
     gen_update_cc_op(s);
-    if (s->tb->flags & HF_INHIBIT_IRQ_MASK) {
-        gen_helper_reset_inhibit_irq(cpu_env);
+
+    /* If several instructions disable interrupts, only the first does it.  */
+    if (inhibit && !(s->flags & HF_INHIBIT_IRQ_MASK)) {
+        gen_set_hflag(s, HF_INHIBIT_IRQ_MASK);
+    } else {
+        gen_reset_hflag(s, HF_INHIBIT_IRQ_MASK);
     }
+
     if (s->tb->flags & HF_RF_MASK) {
         gen_helper_reset_rf(cpu_env);
     }
@@ -2562,6 +2485,12 @@
     s->is_jmp = DISAS_TB_JUMP;
 }
 
+/* End of block, resetting the inhibit irq flag.  */
+static void gen_eob(DisasContext *s)
+{
+    gen_eob_inhibit_irq(s, false);
+}
+
 /* generate a jump to eip. No segment change must happen before as a
    direct call to the next block may occur */
 static void gen_jmp_tb(DisasContext *s, target_ulong eip, int tb_num)
@@ -2598,28 +2527,28 @@
 {
     int mem_index = s->mem_index;
     tcg_gen_qemu_ld_i64(cpu_tmp1_i64, cpu_A0, mem_index, MO_LEQ);
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(0)));
     tcg_gen_addi_tl(cpu_tmp0, cpu_A0, 8);
     tcg_gen_qemu_ld_i64(cpu_tmp1_i64, cpu_tmp0, mem_index, MO_LEQ);
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(1)));
 }
 
 static inline void gen_sto_env_A0(DisasContext *s, int offset)
 {
     int mem_index = s->mem_index;
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(0)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(0)));
     tcg_gen_qemu_st_i64(cpu_tmp1_i64, cpu_A0, mem_index, MO_LEQ);
     tcg_gen_addi_tl(cpu_tmp0, cpu_A0, 8);
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(XMMReg, XMM_Q(1)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, offset + offsetof(ZMMReg, ZMM_Q(1)));
     tcg_gen_qemu_st_i64(cpu_tmp1_i64, cpu_tmp0, mem_index, MO_LEQ);
 }
 
 static inline void gen_op_movo(int d_offset, int s_offset)
 {
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset);
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset);
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + 8);
-    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + 8);
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + offsetof(ZMMReg, ZMM_Q(0)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + offsetof(ZMMReg, ZMM_Q(0)));
+    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env, s_offset + offsetof(ZMMReg, ZMM_Q(1)));
+    tcg_gen_st_i64(cpu_tmp1_i64, cpu_env, d_offset + offsetof(ZMMReg, ZMM_Q(1)));
 }
 
 static inline void gen_op_movq(int d_offset, int s_offset)
@@ -2992,7 +2921,7 @@
         b1 = 0;
     sse_fn_epp = sse_op_table1[b][b1];
     if (!sse_fn_epp) {
-        goto illegal_op;
+        goto unknown_op;
     }
     if ((b <= 0x5f && b >= 0x10) || b == 0xc6 || b == 0xc2) {
         is_xmm = 1;
@@ -3011,15 +2940,19 @@
     }
     if (s->flags & HF_EM_MASK) {
     illegal_op:
-        gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
+        gen_illegal_opcode(s);
         return;
     }
-    if (is_xmm && !(s->flags & HF_OSFXSR_MASK))
-        if ((b != 0x38 && b != 0x3a) || (s->prefix & PREFIX_DATA))
-            goto illegal_op;
+    if (is_xmm
+        && !(s->flags & HF_OSFXSR_MASK)
+        && ((b != 0x38 && b != 0x3a) || (s->prefix & PREFIX_DATA))) {
+        goto unknown_op;
+    }
     if (b == 0x0e) {
-        if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
-            goto illegal_op;
+        if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW)) {
+            /* If we were fully decoding this we might use illegal_op.  */
+            goto unknown_op;
+        }
         /* femms */
         gen_helper_emms(cpu_env);
         return;
@@ -3044,8 +2977,9 @@
         b |= (b1 << 8);
         switch(b) {
         case 0x0e7: /* movntq */
-            if (mod == 3)
+            if (mod == 3) {
                 goto illegal_op;
+            }
             gen_lea_modrm(env, s, modrm);
             gen_stq_env_A0(s, offsetof(CPUX86State, fpregs[reg].mmx));
             break;
@@ -3069,25 +3003,26 @@
                 goto illegal_op;
             gen_lea_modrm(env, s, modrm);
             if (b1 & 1) {
-                gen_stq_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
+                gen_stq_env_A0(s, offsetof(CPUX86State,
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                    xmm_regs[reg].XMM_L(0)));
-                gen_op_st_v(s, MO_32, cpu_T[0], cpu_A0);
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                    xmm_regs[reg].ZMM_L(0)));
+                gen_op_st_v(s, MO_32, cpu_T0, cpu_A0);
             }
             break;
         case 0x6e: /* movd mm, ea */
 #ifdef TARGET_X86_64
             if (s->dflag == MO_64) {
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0);
-                tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
+                tcg_gen_st_tl(cpu_T0, cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
             } else
 #endif
             {
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,fpregs[reg].mmx));
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_movl_mm_T0_mmx(cpu_ptr0, cpu_tmp2_i32);
             }
             break;
@@ -3097,14 +3032,14 @@
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
-                gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]);
+                gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T0);
             } else
 #endif
             {
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_movl_mm_T0_xmm(cpu_ptr0, cpu_tmp2_i32);
             }
             break;
@@ -3138,30 +3073,30 @@
         case 0x210: /* movss xmm, ea */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_32, cpu_T[0], cpu_A0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)));
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
+                gen_op_ld_v(s, MO_32, cpu_T0, cpu_A0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)));
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)));
             }
             break;
         case 0x310: /* movsd xmm, ea */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
+                                           xmm_regs[reg].ZMM_Q(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
             break;
         case 0x012: /* movlps */
@@ -3169,12 +3104,12 @@
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 /* movhlps */
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(1)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(1)));
             }
             break;
         case 0x212: /* movsldup */
@@ -3183,40 +3118,40 @@
                 gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)));
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(2)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(2)));
             }
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)));
             break;
         case 0x312: /* movddup */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
-            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
+            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
             break;
         case 0x016: /* movhps */
         case 0x116: /* movhpd */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(1)));
+                                           xmm_regs[reg].ZMM_Q(1)));
             } else {
                 /* movlhps */
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
             break;
         case 0x216: /* movshdup */
@@ -3225,15 +3160,15 @@
                 gen_ldo_env_A0(s, offsetof(CPUX86State, xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(1)));
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_L(3)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(1)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_L(3)));
             }
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(1)));
-            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)),
-                        offsetof(CPUX86State,xmm_regs[reg].XMM_L(3)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(1)));
+            gen_op_movl(offsetof(CPUX86State,xmm_regs[reg].ZMM_L(2)),
+                        offsetof(CPUX86State,xmm_regs[reg].ZMM_L(3)));
             break;
         case 0x178:
         case 0x378:
@@ -3259,13 +3194,13 @@
         case 0x7e: /* movd ea, mm */
 #ifdef TARGET_X86_64
             if (s->dflag == MO_64) {
-                tcg_gen_ld_i64(cpu_T[0], cpu_env, 
+                tcg_gen_ld_i64(cpu_T0, cpu_env,
                                offsetof(CPUX86State,fpregs[reg].mmx));
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1);
             } else
 #endif
             {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env,
                                  offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0)));
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1);
             }
@@ -3273,14 +3208,14 @@
         case 0x17e: /* movd ea, xmm */
 #ifdef TARGET_X86_64
             if (s->dflag == MO_64) {
-                tcg_gen_ld_i64(cpu_T[0], cpu_env, 
-                               offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
+                tcg_gen_ld_i64(cpu_T0, cpu_env,
+                               offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
                 gen_ldst_modrm(env, s, modrm, MO_64, OR_TMP0, 1);
             } else
 #endif
             {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
-                                 offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env,
+                                 offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
                 gen_ldst_modrm(env, s, modrm, MO_32, OR_TMP0, 1);
             }
             break;
@@ -3288,13 +3223,13 @@
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_ldq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             }
-            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
+            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)));
             break;
         case 0x7f: /* movq ea, mm */
             if (mod != 3) {
@@ -3324,23 +3259,23 @@
         case 0x211: /* movss ea, xmm */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-                gen_op_st_v(s, MO_32, cpu_T[0], cpu_A0);
+                tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
+                gen_op_st_v(s, MO_32, cpu_T0, cpu_A0);
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movl(offsetof(CPUX86State,xmm_regs[rm].XMM_L(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
+                gen_op_movl(offsetof(CPUX86State,xmm_regs[rm].ZMM_L(0)),
+                            offsetof(CPUX86State,xmm_regs[reg].ZMM_L(0)));
             }
             break;
         case 0x311: /* movsd ea, xmm */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
             }
             break;
         case 0x013: /* movlps */
@@ -3348,7 +3283,7 @@
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 goto illegal_op;
             }
@@ -3358,7 +3293,7 @@
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(1)));
+                                           xmm_regs[reg].ZMM_Q(1)));
             } else {
                 goto illegal_op;
             }
@@ -3370,26 +3305,26 @@
         case 0x172:
         case 0x173:
             if (b1 >= 2) {
-	        goto illegal_op;
+	        goto unknown_op;
             }
             val = cpu_ldub_code(env, s->pc++);
             if (is_xmm) {
-                tcg_gen_movi_tl(cpu_T[0], val);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(1)));
+                tcg_gen_movi_tl(cpu_T0, val);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_t0.ZMM_L(1)));
                 op1_offset = offsetof(CPUX86State,xmm_t0);
             } else {
-                tcg_gen_movi_tl(cpu_T[0], val);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(0)));
-                tcg_gen_movi_tl(cpu_T[0], 0);
-                tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(1)));
+                tcg_gen_movi_tl(cpu_T0, val);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(0)));
+                tcg_gen_movi_tl(cpu_T0, 0);
+                tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,mmx_t0.MMX_L(1)));
                 op1_offset = offsetof(CPUX86State,mmx_t0);
             }
             sse_fn_epp = sse_op_table2[((b - 1) & 3) * 8 +
                                        (((modrm >> 3)) & 7)][b1];
             if (!sse_fn_epp) {
-                goto illegal_op;
+                goto unknown_op;
             }
             if (is_xmm) {
                 rm = (modrm & 7) | REX_B(s);
@@ -3448,12 +3383,12 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             if (ot == MO_32) {
                 SSEFunc_0_epi sse_fn_epi = sse_op_table3ai[(b >> 8) & 1];
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 sse_fn_epi(cpu_env, cpu_ptr0, cpu_tmp2_i32);
             } else {
 #ifdef TARGET_X86_64
                 SSEFunc_0_epl sse_fn_epl = sse_op_table3aq[(b >> 8) & 1];
-                sse_fn_epl(cpu_env, cpu_ptr0, cpu_T[0]);
+                sse_fn_epl(cpu_env, cpu_ptr0, cpu_T0);
 #else
                 goto illegal_op;
 #endif
@@ -3498,10 +3433,10 @@
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 if ((b >> 8) & 1) {
-                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.XMM_Q(0)));
+                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.ZMM_Q(0)));
                 } else {
-                    gen_op_ld_v(s, MO_32, cpu_T[0], cpu_A0);
-                    tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
+                    gen_op_ld_v(s, MO_32, cpu_T0, cpu_A0);
+                    tcg_gen_st32_tl(cpu_T0, cpu_env, offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
                 }
                 op2_offset = offsetof(CPUX86State,xmm_t0);
             } else {
@@ -3513,17 +3448,17 @@
                 SSEFunc_i_ep sse_fn_i_ep =
                     sse_op_table3bi[((b >> 7) & 2) | (b & 1)];
                 sse_fn_i_ep(cpu_tmp2_i32, cpu_env, cpu_ptr0);
-                tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+                tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
             } else {
 #ifdef TARGET_X86_64
                 SSEFunc_l_ep sse_fn_l_ep =
                     sse_op_table3bq[((b >> 7) & 2) | (b & 1)];
-                sse_fn_l_ep(cpu_T[0], cpu_env, cpu_ptr0);
+                sse_fn_l_ep(cpu_T0, cpu_env, cpu_ptr0);
 #else
                 goto illegal_op;
 #endif
             }
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
             break;
         case 0xc4: /* pinsrw */
         case 0x1c4:
@@ -3532,11 +3467,11 @@
             val = cpu_ldub_code(env, s->pc++);
             if (b1) {
                 val &= 7;
-                tcg_gen_st16_tl(cpu_T[0], cpu_env,
-                                offsetof(CPUX86State,xmm_regs[reg].XMM_W(val)));
+                tcg_gen_st16_tl(cpu_T0, cpu_env,
+                                offsetof(CPUX86State,xmm_regs[reg].ZMM_W(val)));
             } else {
                 val &= 3;
-                tcg_gen_st16_tl(cpu_T[0], cpu_env,
+                tcg_gen_st16_tl(cpu_T0, cpu_env,
                                 offsetof(CPUX86State,fpregs[reg].mmx.MMX_W(val)));
             }
             break;
@@ -3549,41 +3484,41 @@
             if (b1) {
                 val &= 7;
                 rm = (modrm & 7) | REX_B(s);
-                tcg_gen_ld16u_tl(cpu_T[0], cpu_env,
-                                 offsetof(CPUX86State,xmm_regs[rm].XMM_W(val)));
+                tcg_gen_ld16u_tl(cpu_T0, cpu_env,
+                                 offsetof(CPUX86State,xmm_regs[rm].ZMM_W(val)));
             } else {
                 val &= 3;
                 rm = (modrm & 7);
-                tcg_gen_ld16u_tl(cpu_T[0], cpu_env,
+                tcg_gen_ld16u_tl(cpu_T0, cpu_env,
                                 offsetof(CPUX86State,fpregs[rm].mmx.MMX_W(val)));
             }
             reg = ((modrm >> 3) & 7) | rex_r;
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
             break;
         case 0x1d6: /* movq ea, xmm */
             if (mod != 3) {
                 gen_lea_modrm(env, s, modrm);
                 gen_stq_env_A0(s, offsetof(CPUX86State,
-                                           xmm_regs[reg].XMM_Q(0)));
+                                           xmm_regs[reg].ZMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
-                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)),
-                            offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
-                gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[rm].XMM_Q(1)));
+                gen_op_movq(offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)),
+                            offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)));
+                gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(1)));
             }
             break;
         case 0x2d6: /* movq2dq */
             gen_helper_enter_mmx(cpu_env);
             rm = (modrm & 7);
-            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
+            gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(0)),
                         offsetof(CPUX86State,fpregs[rm].mmx));
-            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
+            gen_op_movq_env_0(offsetof(CPUX86State,xmm_regs[reg].ZMM_Q(1)));
             break;
         case 0x3d6: /* movdq2q */
             gen_helper_enter_mmx(cpu_env);
             rm = (modrm & 7) | REX_B(s);
             gen_op_movq(offsetof(CPUX86State,fpregs[reg & 7].mmx),
-                        offsetof(CPUX86State,xmm_regs[rm].XMM_Q(0)));
+                        offsetof(CPUX86State,xmm_regs[rm].ZMM_Q(0)));
             break;
         case 0xd7: /* pmovmskb */
         case 0x1d7:
@@ -3613,12 +3548,12 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             if (b1 >= 2) {
-                goto illegal_op;
+                goto unknown_op;
             }
 
             sse_fn_epp = sse_op_table6[b].op[b1];
             if (!sse_fn_epp) {
-                goto illegal_op;
+                goto unknown_op;
             }
             if (!(s->cpuid_ext_features & sse_op_table6[b].ext_mask))
                 goto illegal_op;
@@ -3635,20 +3570,20 @@
                     case 0x23: case 0x33: /* pmovsxwd, pmovzxwd */
                     case 0x25: case 0x35: /* pmovsxdq, pmovzxdq */
                         gen_ldq_env_A0(s, op2_offset +
-                                        offsetof(XMMReg, XMM_Q(0)));
+                                        offsetof(ZMMReg, ZMM_Q(0)));
                         break;
                     case 0x21: case 0x31: /* pmovsxbd, pmovzxbd */
                     case 0x24: case 0x34: /* pmovsxwq, pmovzxwq */
                         tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0,
                                             s->mem_index, MO_LEUL);
                         tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, op2_offset +
-                                        offsetof(XMMReg, XMM_L(0)));
+                                        offsetof(ZMMReg, ZMM_L(0)));
                         break;
                     case 0x22: case 0x32: /* pmovsxbq, pmovzxbq */
                         tcg_gen_qemu_ld_tl(cpu_tmp0, cpu_A0,
                                            s->mem_index, MO_LEUW);
                         tcg_gen_st16_tl(cpu_tmp0, cpu_env, op2_offset +
-                                        offsetof(XMMReg, XMM_W(0)));
+                                        offsetof(ZMMReg, ZMM_W(0)));
                         break;
                     case 0x2a:            /* movntqda */
                         gen_ldo_env_A0(s, op1_offset);
@@ -3668,7 +3603,7 @@
                 }
             }
             if (sse_fn_epp == SSE_SPECIAL) {
-                goto illegal_op;
+                goto unknown_op;
             }
 
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
@@ -3705,11 +3640,11 @@
 
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[reg]);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                gen_helper_crc32(cpu_T[0], cpu_tmp2_i32,
-                                 cpu_T[0], tcg_const_i32(8 << ot));
+                gen_helper_crc32(cpu_T0, cpu_tmp2_i32,
+                                 cpu_T0, tcg_const_i32(8 << ot));
 
                 ot = mo_64_32(s->dflag);
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 break;
 
             case 0x1f0: /* crc32 or movbe */
@@ -3734,9 +3669,9 @@
 
                 gen_lea_modrm(env, s, modrm);
                 if ((b & 1) == 0) {
-                    tcg_gen_qemu_ld_tl(cpu_T[0], cpu_A0,
+                    tcg_gen_qemu_ld_tl(cpu_T0, cpu_A0,
                                        s->mem_index, ot | MO_BE);
-                    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                    gen_op_mov_reg_v(ot, reg, cpu_T0);
                 } else {
                     tcg_gen_qemu_st_tl(cpu_regs[reg], cpu_A0,
                                        s->mem_index, ot | MO_BE);
@@ -3751,8 +3686,8 @@
                 }
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                tcg_gen_andc_tl(cpu_T[0], cpu_regs[s->vex_v], cpu_T[0]);
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                tcg_gen_andc_tl(cpu_T0, cpu_regs[s->vex_v], cpu_T0);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 gen_op_update1_cc();
                 set_cc_op(s, CC_OP_LOGICB + ot);
                 break;
@@ -3771,12 +3706,12 @@
                     /* Extract START, and shift the operand.
                        Shifts larger than operand size get zeros.  */
                     tcg_gen_ext8u_tl(cpu_A0, cpu_regs[s->vex_v]);
-                    tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_A0);
+                    tcg_gen_shr_tl(cpu_T0, cpu_T0, cpu_A0);
 
                     bound = tcg_const_tl(ot == MO_64 ? 63 : 31);
                     zero = tcg_const_tl(0);
-                    tcg_gen_movcond_tl(TCG_COND_LEU, cpu_T[0], cpu_A0, bound,
-                                       cpu_T[0], zero);
+                    tcg_gen_movcond_tl(TCG_COND_LEU, cpu_T0, cpu_A0, bound,
+                                       cpu_T0, zero);
                     tcg_temp_free(zero);
 
                     /* Extract the LEN into a mask.  Lengths larger than
@@ -3786,12 +3721,12 @@
                     tcg_gen_movcond_tl(TCG_COND_LEU, cpu_A0, cpu_A0, bound,
                                        cpu_A0, bound);
                     tcg_temp_free(bound);
-                    tcg_gen_movi_tl(cpu_T[1], 1);
-                    tcg_gen_shl_tl(cpu_T[1], cpu_T[1], cpu_A0);
-                    tcg_gen_subi_tl(cpu_T[1], cpu_T[1], 1);
-                    tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_movi_tl(cpu_T1, 1);
+                    tcg_gen_shl_tl(cpu_T1, cpu_T1, cpu_A0);
+                    tcg_gen_subi_tl(cpu_T1, cpu_T1, 1);
+                    tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
 
-                    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                    gen_op_mov_reg_v(ot, reg, cpu_T0);
                     gen_op_update1_cc();
                     set_cc_op(s, CC_OP_LOGICB + ot);
                 }
@@ -3805,21 +3740,21 @@
                 }
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-                tcg_gen_ext8u_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                tcg_gen_ext8u_tl(cpu_T1, cpu_regs[s->vex_v]);
                 {
                     TCGv bound = tcg_const_tl(ot == MO_64 ? 63 : 31);
                     /* Note that since we're using BMILG (in order to get O
                        cleared) we need to store the inverse into C.  */
                     tcg_gen_setcond_tl(TCG_COND_LT, cpu_cc_src,
-                                       cpu_T[1], bound);
-                    tcg_gen_movcond_tl(TCG_COND_GT, cpu_T[1], cpu_T[1],
-                                       bound, bound, cpu_T[1]);
+                                       cpu_T1, bound);
+                    tcg_gen_movcond_tl(TCG_COND_GT, cpu_T1, cpu_T1,
+                                       bound, bound, cpu_T1);
                     tcg_temp_free(bound);
                 }
                 tcg_gen_movi_tl(cpu_A0, -1);
-                tcg_gen_shl_tl(cpu_A0, cpu_A0, cpu_T[1]);
-                tcg_gen_andc_tl(cpu_T[0], cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                tcg_gen_shl_tl(cpu_A0, cpu_A0, cpu_T1);
+                tcg_gen_andc_tl(cpu_T0, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 gen_op_update1_cc();
                 set_cc_op(s, CC_OP_BMILGB + ot);
                 break;
@@ -3834,7 +3769,7 @@
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 switch (ot) {
                 default:
-                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                     tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_regs[R_EDX]);
                     tcg_gen_mulu2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                                       cpu_tmp2_i32, cpu_tmp3_i32);
@@ -3843,8 +3778,10 @@
                     break;
 #ifdef TARGET_X86_64
                 case MO_64:
-                    tcg_gen_mulu2_i64(cpu_regs[s->vex_v], cpu_regs[reg],
-                                      cpu_T[0], cpu_regs[R_EDX]);
+                    tcg_gen_mulu2_i64(cpu_T0, cpu_T1,
+                                      cpu_T0, cpu_regs[R_EDX]);
+                    tcg_gen_mov_i64(cpu_regs[s->vex_v], cpu_T0);
+                    tcg_gen_mov_i64(cpu_regs[reg], cpu_T1);
                     break;
 #endif
                 }
@@ -3861,11 +3798,11 @@
                 /* Note that by zero-extending the mask operand, we
                    automatically handle zero-extending the result.  */
                 if (ot == MO_64) {
-                    tcg_gen_mov_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_mov_tl(cpu_T1, cpu_regs[s->vex_v]);
                 } else {
-                    tcg_gen_ext32u_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_ext32u_tl(cpu_T1, cpu_regs[s->vex_v]);
                 }
-                gen_helper_pdep(cpu_regs[reg], cpu_T[0], cpu_T[1]);
+                gen_helper_pdep(cpu_regs[reg], cpu_T0, cpu_T1);
                 break;
 
             case 0x2f5: /* pext Gy, By, Ey */
@@ -3879,11 +3816,11 @@
                 /* Note that by zero-extending the mask operand, we
                    automatically handle zero-extending the result.  */
                 if (ot == MO_64) {
-                    tcg_gen_mov_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_mov_tl(cpu_T1, cpu_regs[s->vex_v]);
                 } else {
-                    tcg_gen_ext32u_tl(cpu_T[1], cpu_regs[s->vex_v]);
+                    tcg_gen_ext32u_tl(cpu_T1, cpu_regs[s->vex_v]);
                 }
-                gen_helper_pext(cpu_regs[reg], cpu_T[0], cpu_T[1]);
+                gen_helper_pext(cpu_regs[reg], cpu_T0, cpu_T1);
                 break;
 
             case 0x1f6: /* adcx Gy, Ey */
@@ -3942,22 +3879,22 @@
                         /* If we know TL is 64-bit, and we want a 32-bit
                            result, just do everything in 64-bit arithmetic.  */
                         tcg_gen_ext32u_i64(cpu_regs[reg], cpu_regs[reg]);
-                        tcg_gen_ext32u_i64(cpu_T[0], cpu_T[0]);
-                        tcg_gen_add_i64(cpu_T[0], cpu_T[0], cpu_regs[reg]);
-                        tcg_gen_add_i64(cpu_T[0], cpu_T[0], carry_in);
-                        tcg_gen_ext32u_i64(cpu_regs[reg], cpu_T[0]);
-                        tcg_gen_shri_i64(carry_out, cpu_T[0], 32);
+                        tcg_gen_ext32u_i64(cpu_T0, cpu_T0);
+                        tcg_gen_add_i64(cpu_T0, cpu_T0, cpu_regs[reg]);
+                        tcg_gen_add_i64(cpu_T0, cpu_T0, carry_in);
+                        tcg_gen_ext32u_i64(cpu_regs[reg], cpu_T0);
+                        tcg_gen_shri_i64(carry_out, cpu_T0, 32);
                         break;
 #endif
                     default:
                         /* Otherwise compute the carry-out in two steps.  */
                         zero = tcg_const_tl(0);
-                        tcg_gen_add2_tl(cpu_T[0], carry_out,
-                                        cpu_T[0], zero,
+                        tcg_gen_add2_tl(cpu_T0, carry_out,
+                                        cpu_T0, zero,
                                         carry_in, zero);
                         tcg_gen_add2_tl(cpu_regs[reg], carry_out,
                                         cpu_regs[reg], carry_out,
-                                        cpu_T[0], zero);
+                                        cpu_T0, zero);
                         tcg_temp_free(zero);
                         break;
                     }
@@ -3976,24 +3913,24 @@
                 ot = mo_64_32(s->dflag);
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 if (ot == MO_64) {
-                    tcg_gen_andi_tl(cpu_T[1], cpu_regs[s->vex_v], 63);
+                    tcg_gen_andi_tl(cpu_T1, cpu_regs[s->vex_v], 63);
                 } else {
-                    tcg_gen_andi_tl(cpu_T[1], cpu_regs[s->vex_v], 31);
+                    tcg_gen_andi_tl(cpu_T1, cpu_regs[s->vex_v], 31);
                 }
                 if (b == 0x1f7) {
-                    tcg_gen_shl_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_shl_tl(cpu_T0, cpu_T0, cpu_T1);
                 } else if (b == 0x2f7) {
                     if (ot != MO_64) {
-                        tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
+                        tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
                     }
-                    tcg_gen_sar_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_sar_tl(cpu_T0, cpu_T0, cpu_T1);
                 } else {
                     if (ot != MO_64) {
-                        tcg_gen_ext32u_tl(cpu_T[0], cpu_T[0]);
+                        tcg_gen_ext32u_tl(cpu_T0, cpu_T0);
                     }
-                    tcg_gen_shr_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
+                    tcg_gen_shr_tl(cpu_T0, cpu_T0, cpu_T1);
                 }
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 break;
 
             case 0x0f3:
@@ -4010,36 +3947,36 @@
 
                 switch (reg & 7) {
                 case 1: /* blsr By,Ey */
-                    tcg_gen_neg_tl(cpu_T[1], cpu_T[0]);
-                    tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                    gen_op_mov_reg_v(ot, s->vex_v, cpu_T[0]);
+                    tcg_gen_neg_tl(cpu_T1, cpu_T0);
+                    tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_T1);
+                    gen_op_mov_reg_v(ot, s->vex_v, cpu_T0);
                     gen_op_update2_cc();
                     set_cc_op(s, CC_OP_BMILGB + ot);
                     break;
 
                 case 2: /* blsmsk By,Ey */
-                    tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
-                    tcg_gen_subi_tl(cpu_T[0], cpu_T[0], 1);
-                    tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_cc_src);
-                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+                    tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
+                    tcg_gen_subi_tl(cpu_T0, cpu_T0, 1);
+                    tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_cc_src);
+                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
                     set_cc_op(s, CC_OP_BMILGB + ot);
                     break;
 
                 case 3: /* blsi By, Ey */
-                    tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
-                    tcg_gen_subi_tl(cpu_T[0], cpu_T[0], 1);
-                    tcg_gen_and_tl(cpu_T[0], cpu_T[0], cpu_cc_src);
-                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+                    tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
+                    tcg_gen_subi_tl(cpu_T0, cpu_T0, 1);
+                    tcg_gen_and_tl(cpu_T0, cpu_T0, cpu_cc_src);
+                    tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
                     set_cc_op(s, CC_OP_BMILGB + ot);
                     break;
 
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
 
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
             break;
 
@@ -4051,12 +3988,12 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             if (b1 >= 2) {
-                goto illegal_op;
+                goto unknown_op;
             }
 
             sse_fn_eppi = sse_op_table7[b].op[b1];
             if (!sse_fn_eppi) {
-                goto illegal_op;
+                goto unknown_op;
             }
             if (!(s->cpuid_ext_features & sse_op_table7[b].ext_mask))
                 goto illegal_op;
@@ -4070,22 +4007,22 @@
                 val = cpu_ldub_code(env, s->pc++);
                 switch (b) {
                 case 0x14: /* pextrb */
-                    tcg_gen_ld8u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_B(val & 15)));
+                    tcg_gen_ld8u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_B(val & 15)));
                     if (mod == 3) {
-                        gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                        gen_op_mov_reg_v(ot, rm, cpu_T0);
                     } else {
-                        tcg_gen_qemu_st_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_st_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_UB);
                     }
                     break;
                 case 0x15: /* pextrw */
-                    tcg_gen_ld16u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_W(val & 7)));
+                    tcg_gen_ld16u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_W(val & 7)));
                     if (mod == 3) {
-                        gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                        gen_op_mov_reg_v(ot, rm, cpu_T0);
                     } else {
-                        tcg_gen_qemu_st_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_st_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_LEUW);
                     }
                     break;
@@ -4093,7 +4030,7 @@
                     if (ot == MO_32) { /* pextrd */
                         tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(val & 3)));
+                                                xmm_regs[reg].ZMM_L(val & 3)));
                         if (mod == 3) {
                             tcg_gen_extu_i32_tl(cpu_regs[rm], cpu_tmp2_i32);
                         } else {
@@ -4104,7 +4041,7 @@
 #ifdef TARGET_X86_64
                         tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_Q(val & 1)));
+                                                xmm_regs[reg].ZMM_Q(val & 1)));
                         if (mod == 3) {
                             tcg_gen_mov_i64(cpu_regs[rm], cpu_tmp1_i64);
                         } else {
@@ -4117,53 +4054,53 @@
                     }
                     break;
                 case 0x17: /* extractps */
-                    tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_L(val & 3)));
+                    tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_L(val & 3)));
                     if (mod == 3) {
-                        gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                        gen_op_mov_reg_v(ot, rm, cpu_T0);
                     } else {
-                        tcg_gen_qemu_st_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_st_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_LEUL);
                     }
                     break;
                 case 0x20: /* pinsrb */
                     if (mod == 3) {
-                        gen_op_mov_v_reg(MO_32, cpu_T[0], rm);
+                        gen_op_mov_v_reg(MO_32, cpu_T0, rm);
                     } else {
-                        tcg_gen_qemu_ld_tl(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_ld_tl(cpu_T0, cpu_A0,
                                            s->mem_index, MO_UB);
                     }
-                    tcg_gen_st8_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
-                                            xmm_regs[reg].XMM_B(val & 15)));
+                    tcg_gen_st8_tl(cpu_T0, cpu_env, offsetof(CPUX86State,
+                                            xmm_regs[reg].ZMM_B(val & 15)));
                     break;
                 case 0x21: /* insertps */
                     if (mod == 3) {
                         tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,xmm_regs[rm]
-                                                .XMM_L((val >> 6) & 3)));
+                                                .ZMM_L((val >> 6) & 3)));
                     } else {
                         tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0,
                                             s->mem_index, MO_LEUL);
                     }
                     tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
                                     offsetof(CPUX86State,xmm_regs[reg]
-                                            .XMM_L((val >> 4) & 3)));
+                                            .ZMM_L((val >> 4) & 3)));
                     if ((val >> 0) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(0)));
+                                                xmm_regs[reg].ZMM_L(0)));
                     if ((val >> 1) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(1)));
+                                                xmm_regs[reg].ZMM_L(1)));
                     if ((val >> 2) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(2)));
+                                                xmm_regs[reg].ZMM_L(2)));
                     if ((val >> 3) & 1)
                         tcg_gen_st_i32(tcg_const_i32(0 /*float32_zero*/),
                                         cpu_env, offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(3)));
+                                                xmm_regs[reg].ZMM_L(3)));
                     break;
                 case 0x22:
                     if (ot == MO_32) { /* pinsrd */
@@ -4175,7 +4112,7 @@
                         }
                         tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_L(val & 3)));
+                                                xmm_regs[reg].ZMM_L(val & 3)));
                     } else { /* pinsrq */
 #ifdef TARGET_X86_64
                         if (mod == 3) {
@@ -4186,7 +4123,7 @@
                         }
                         tcg_gen_st_i64(cpu_tmp1_i64, cpu_env,
                                         offsetof(CPUX86State,
-                                                xmm_regs[reg].XMM_Q(val & 1)));
+                                                xmm_regs[reg].ZMM_Q(val & 1)));
 #else
                         goto illegal_op;
 #endif
@@ -4248,22 +4185,24 @@
                 gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
                 b = cpu_ldub_code(env, s->pc++);
                 if (ot == MO_64) {
-                    tcg_gen_rotri_tl(cpu_T[0], cpu_T[0], b & 63);
+                    tcg_gen_rotri_tl(cpu_T0, cpu_T0, b & 63);
                 } else {
-                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                    tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                     tcg_gen_rotri_i32(cpu_tmp2_i32, cpu_tmp2_i32, b & 31);
-                    tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
+                    tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
                 }
-                gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(ot, reg, cpu_T0);
                 break;
 
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
             break;
 
         default:
-            goto illegal_op;
+        unknown_op:
+            gen_unknown_opcode(env, s);
+            return;
         }
     } else {
         /* generic MMX or SSE operation */
@@ -4309,13 +4248,13 @@
                 switch (sz) {
                 case 2:
                     /* 32 bit access */
-                    gen_op_ld_v(s, MO_32, cpu_T[0], cpu_A0);
-                    tcg_gen_st32_tl(cpu_T[0], cpu_env,
-                                    offsetof(CPUX86State,xmm_t0.XMM_L(0)));
+                    gen_op_ld_v(s, MO_32, cpu_T0, cpu_A0);
+                    tcg_gen_st32_tl(cpu_T0, cpu_env,
+                                    offsetof(CPUX86State,xmm_t0.ZMM_L(0)));
                     break;
                 case 3:
                     /* 64 bit access */
-                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.XMM_D(0)));
+                    gen_ldq_env_A0(s, offsetof(CPUX86State, xmm_t0.ZMM_D(0)));
                     break;
                 default:
                     /* 128 bit access */
@@ -4339,11 +4278,12 @@
         }
         switch(b) {
         case 0x0f: /* 3DNow! data insns */
-            if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
-                goto illegal_op;
             val = cpu_ldub_code(env, s->pc++);
             sse_fn_epp = sse_op_table5[val];
             if (!sse_fn_epp) {
+                goto unknown_op;
+            }
+            if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW)) {
                 goto illegal_op;
             }
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
@@ -4363,7 +4303,7 @@
             /* compare insns */
             val = cpu_ldub_code(env, s->pc++);
             if (val >= 8)
-                goto illegal_op;
+                goto unknown_op;
             sse_fn_epp = sse_op_table4[val][b1];
 
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
@@ -4408,10 +4348,7 @@
     target_ulong next_eip, tval;
     int rex_w, rex_r;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(pc_start);
-    }
-    s->pc = pc_start;
+    s->pc_start = s->pc = pc_start;
     prefixes = 0;
     s->override = -1;
     rex_w = -1;
@@ -4524,7 +4461,7 @@
                     b = 0x13a;
                     break;
                 default:   /* Reserved for future use.  */
-                    goto illegal_op;
+                    goto unknown_op;
                 }
             }
             s->vex_v = (~vex3 >> 3) & 0xf;
@@ -4604,13 +4541,13 @@
                 xor_zero:
                     /* xor reg, reg optimisation */
                     set_cc_op(s, CC_OP_CLR);
-                    tcg_gen_movi_tl(cpu_T[0], 0);
-                    gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+                    tcg_gen_movi_tl(cpu_T0, 0);
+                    gen_op_mov_reg_v(ot, reg, cpu_T0);
                     break;
                 } else {
                     opreg = rm;
                 }
-                gen_op_mov_v_reg(ot, cpu_T[1], reg);
+                gen_op_mov_v_reg(ot, cpu_T1, reg);
                 gen_op(s, op, ot, opreg);
                 break;
             case 1: /* OP Gv, Ev */
@@ -4620,17 +4557,17 @@
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
                     gen_lea_modrm(env, s, modrm);
-                    gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+                    gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
                 } else if (op == OP_XORL && rm == reg) {
                     goto xor_zero;
                 } else {
-                    gen_op_mov_v_reg(ot, cpu_T[1], rm);
+                    gen_op_mov_v_reg(ot, cpu_T1, rm);
                 }
                 gen_op(s, op, ot, reg);
                 break;
             case 2: /* OP A, Iv */
                 val = insn_get(env, s, ot);
-                tcg_gen_movi_tl(cpu_T[1], val);
+                tcg_gen_movi_tl(cpu_T1, val);
                 gen_op(s, op, ot, OR_EAX);
                 break;
             }
@@ -4675,7 +4612,7 @@
                 val = (int8_t)insn_get(env, s, MO_8);
                 break;
             }
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
             gen_op(s, op, ot, opreg);
         }
         break;
@@ -4702,32 +4639,32 @@
             if (op == 0)
                 s->rip_offset = insn_const_size(ot);
             gen_lea_modrm(env, s, modrm);
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
 
         switch(op) {
         case 0: /* test */
             val = insn_get(env, s, ot);
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
             gen_op_testl_T0_T1_cc();
             set_cc_op(s, CC_OP_LOGICB + ot);
             break;
         case 2: /* not */
-            tcg_gen_not_tl(cpu_T[0], cpu_T[0]);
+            tcg_gen_not_tl(cpu_T0, cpu_T0);
             if (mod != 3) {
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             } else {
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
             break;
         case 3: /* neg */
-            tcg_gen_neg_tl(cpu_T[0], cpu_T[0]);
+            tcg_gen_neg_tl(cpu_T0, cpu_T0);
             if (mod != 3) {
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             } else {
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
             gen_op_update_neg_cc();
             set_cc_op(s, CC_OP_SUBB + ot);
@@ -4735,32 +4672,32 @@
         case 4: /* mul */
             switch(ot) {
             case MO_8:
-                gen_op_mov_v_reg(MO_8, cpu_T[1], R_EAX);
-                tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext8u_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_8, cpu_T1, R_EAX);
+                tcg_gen_ext8u_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext8u_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_andi_tl(cpu_cc_src, cpu_T[0], 0xff00);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_andi_tl(cpu_cc_src, cpu_T0, 0xff00);
                 set_cc_op(s, CC_OP_MULB);
                 break;
             case MO_16:
-                gen_op_mov_v_reg(MO_16, cpu_T[1], R_EAX);
-                tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext16u_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_16, cpu_T1, R_EAX);
+                tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext16u_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 16);
-                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_shri_tl(cpu_T0, cpu_T0, 16);
+                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
                 set_cc_op(s, CC_OP_MULW);
                 break;
             default:
             case MO_32:
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_regs[R_EAX]);
                 tcg_gen_mulu2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                                   cpu_tmp2_i32, cpu_tmp3_i32);
@@ -4773,7 +4710,7 @@
 #ifdef TARGET_X86_64
             case MO_64:
                 tcg_gen_mulu2_i64(cpu_regs[R_EAX], cpu_regs[R_EDX],
-                                  cpu_T[0], cpu_regs[R_EAX]);
+                                  cpu_T0, cpu_regs[R_EAX]);
                 tcg_gen_mov_tl(cpu_cc_dst, cpu_regs[R_EAX]);
                 tcg_gen_mov_tl(cpu_cc_src, cpu_regs[R_EDX]);
                 set_cc_op(s, CC_OP_MULQ);
@@ -4784,34 +4721,34 @@
         case 5: /* imul */
             switch(ot) {
             case MO_8:
-                gen_op_mov_v_reg(MO_8, cpu_T[1], R_EAX);
-                tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext8s_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_8, cpu_T1, R_EAX);
+                tcg_gen_ext8s_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext8s_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_ext8s_tl(cpu_tmp0, cpu_T[0]);
-                tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_ext8s_tl(cpu_tmp0, cpu_T0);
+                tcg_gen_sub_tl(cpu_cc_src, cpu_T0, cpu_tmp0);
                 set_cc_op(s, CC_OP_MULB);
                 break;
             case MO_16:
-                gen_op_mov_v_reg(MO_16, cpu_T[1], R_EAX);
-                tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-                tcg_gen_ext16s_tl(cpu_T[1], cpu_T[1]);
+                gen_op_mov_v_reg(MO_16, cpu_T1, R_EAX);
+                tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+                tcg_gen_ext16s_tl(cpu_T1, cpu_T1);
                 /* XXX: use 32 bit mul which could be faster */
-                tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
-                tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-                tcg_gen_ext16s_tl(cpu_tmp0, cpu_T[0]);
-                tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
-                tcg_gen_shri_tl(cpu_T[0], cpu_T[0], 16);
-                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T[0]);
+                tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+                gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
+                tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+                tcg_gen_ext16s_tl(cpu_tmp0, cpu_T0);
+                tcg_gen_sub_tl(cpu_cc_src, cpu_T0, cpu_tmp0);
+                tcg_gen_shri_tl(cpu_T0, cpu_T0, 16);
+                gen_op_mov_reg_v(MO_16, R_EDX, cpu_T0);
                 set_cc_op(s, CC_OP_MULW);
                 break;
             default:
             case MO_32:
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_regs[R_EAX]);
                 tcg_gen_muls2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                                   cpu_tmp2_i32, cpu_tmp3_i32);
@@ -4826,7 +4763,7 @@
 #ifdef TARGET_X86_64
             case MO_64:
                 tcg_gen_muls2_i64(cpu_regs[R_EAX], cpu_regs[R_EDX],
-                                  cpu_T[0], cpu_regs[R_EAX]);
+                                  cpu_T0, cpu_regs[R_EAX]);
                 tcg_gen_mov_tl(cpu_cc_dst, cpu_regs[R_EAX]);
                 tcg_gen_sari_tl(cpu_cc_src, cpu_regs[R_EAX], 63);
                 tcg_gen_sub_tl(cpu_cc_src, cpu_cc_src, cpu_regs[R_EDX]);
@@ -4838,22 +4775,18 @@
         case 6: /* div */
             switch(ot) {
             case MO_8:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divb_AL(cpu_env, cpu_T[0]);
+                gen_helper_divb_AL(cpu_env, cpu_T0);
                 break;
             case MO_16:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divw_AX(cpu_env, cpu_T[0]);
+                gen_helper_divw_AX(cpu_env, cpu_T0);
                 break;
             default:
             case MO_32:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divl_EAX(cpu_env, cpu_T[0]);
+                gen_helper_divl_EAX(cpu_env, cpu_T0);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_divq_EAX(cpu_env, cpu_T[0]);
+                gen_helper_divq_EAX(cpu_env, cpu_T0);
                 break;
 #endif
             }
@@ -4861,28 +4794,24 @@
         case 7: /* idiv */
             switch(ot) {
             case MO_8:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivb_AL(cpu_env, cpu_T[0]);
+                gen_helper_idivb_AL(cpu_env, cpu_T0);
                 break;
             case MO_16:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivw_AX(cpu_env, cpu_T[0]);
+                gen_helper_idivw_AX(cpu_env, cpu_T0);
                 break;
             default:
             case MO_32:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivl_EAX(cpu_env, cpu_T[0]);
+                gen_helper_idivl_EAX(cpu_env, cpu_T0);
                 break;
 #ifdef TARGET_X86_64
             case MO_64:
-                gen_jmp_im(pc_start - s->cs_base);
-                gen_helper_idivq_EAX(cpu_env, cpu_T[0]);
+                gen_helper_idivq_EAX(cpu_env, cpu_T0);
                 break;
 #endif
             }
             break;
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
 
@@ -4895,7 +4824,7 @@
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
         if (op >= 2 && b == 0xfe) {
-            goto illegal_op;
+            goto unknown_op;
         }
         if (CODE64(s)) {
             if (op == 2 || op == 4) {
@@ -4911,9 +4840,9 @@
         if (mod != 3) {
             gen_lea_modrm(env, s, modrm);
             if (op >= 2 && op != 3 && op != 5)
-                gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
 
         switch(op) {
@@ -4934,29 +4863,28 @@
         case 2: /* call Ev */
             /* XXX: optimize if memory (no 'and' is necessary) */
             if (dflag == MO_16) {
-                tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
+                tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
             }
             next_eip = s->pc - s->cs_base;
-            tcg_gen_movi_tl(cpu_T[1], next_eip);
-            gen_push_v(s, cpu_T[1]);
-            gen_op_jmp_v(cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T1, next_eip);
+            gen_push_v(s, cpu_T1);
+            gen_op_jmp_v(cpu_T0);
+            gen_bnd_jmp(s);
             gen_eob(s);
             break;
         case 3: /* lcall Ev */
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
             gen_add_A0_im(s, 1 << ot);
-            gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, MO_16, cpu_T0, cpu_A0);
         do_lcall:
             if (s->pe && !s->vm86) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                gen_helper_lcall_protected(cpu_env, cpu_tmp2_i32, cpu_T[1],
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+                gen_helper_lcall_protected(cpu_env, cpu_tmp2_i32, cpu_T1,
                                            tcg_const_i32(dflag - 1),
-                                           tcg_const_i32(s->pc - pc_start));
+                                           tcg_const_tl(s->pc - s->cs_base));
             } else {
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                gen_helper_lcall_real(cpu_env, cpu_tmp2_i32, cpu_T[1],
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+                gen_helper_lcall_real(cpu_env, cpu_tmp2_i32, cpu_T1,
                                       tcg_const_i32(dflag - 1),
                                       tcg_const_i32(s->pc - s->cs_base));
             }
@@ -4964,33 +4892,32 @@
             break;
         case 4: /* jmp Ev */
             if (dflag == MO_16) {
-                tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
+                tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
             }
-            gen_op_jmp_v(cpu_T[0]);
+            gen_op_jmp_v(cpu_T0);
+            gen_bnd_jmp(s);
             gen_eob(s);
             break;
         case 5: /* ljmp Ev */
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
             gen_add_A0_im(s, 1 << ot);
-            gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, MO_16, cpu_T0, cpu_A0);
         do_ljmp:
             if (s->pe && !s->vm86) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                gen_helper_ljmp_protected(cpu_env, cpu_tmp2_i32, cpu_T[1],
-                                          tcg_const_i32(s->pc - pc_start));
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+                gen_helper_ljmp_protected(cpu_env, cpu_tmp2_i32, cpu_T1,
+                                          tcg_const_tl(s->pc - s->cs_base));
             } else {
                 gen_op_movl_seg_T0_vm(R_CS);
-                gen_op_jmp_v(cpu_T[1]);
+                gen_op_jmp_v(cpu_T1);
             }
             gen_eob(s);
             break;
         case 6: /* push Ev */
-            gen_push_v(s, cpu_T[0]);
+            gen_push_v(s, cpu_T0);
             break;
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
 
@@ -5002,7 +4929,7 @@
         reg = ((modrm >> 3) & 7) | rex_r;
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_op_mov_v_reg(ot, cpu_T[1], reg);
+        gen_op_mov_v_reg(ot, cpu_T1, reg);
         gen_op_testl_T0_T1_cc();
         set_cc_op(s, CC_OP_LOGICB + ot);
         break;
@@ -5012,8 +4939,8 @@
         ot = mo_b_d(b, dflag);
         val = insn_get(env, s, ot);
 
-        gen_op_mov_v_reg(ot, cpu_T[0], OR_EAX);
-        tcg_gen_movi_tl(cpu_T[1], val);
+        gen_op_mov_v_reg(ot, cpu_T0, OR_EAX);
+        tcg_gen_movi_tl(cpu_T1, val);
         gen_op_testl_T0_T1_cc();
         set_cc_op(s, CC_OP_LOGICB + ot);
         break;
@@ -5022,20 +4949,20 @@
         switch (dflag) {
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_op_mov_v_reg(MO_32, cpu_T[0], R_EAX);
-            tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_64, R_EAX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_32, cpu_T0, R_EAX);
+            tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_64, R_EAX, cpu_T0);
             break;
 #endif
         case MO_32:
-            gen_op_mov_v_reg(MO_16, cpu_T[0], R_EAX);
-            tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_32, R_EAX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_16, cpu_T0, R_EAX);
+            tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_32, R_EAX, cpu_T0);
             break;
         case MO_16:
-            gen_op_mov_v_reg(MO_8, cpu_T[0], R_EAX);
-            tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_8, cpu_T0, R_EAX);
+            tcg_gen_ext8s_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
             break;
         default:
             tcg_abort();
@@ -5045,22 +4972,22 @@
         switch (dflag) {
 #ifdef TARGET_X86_64
         case MO_64:
-            gen_op_mov_v_reg(MO_64, cpu_T[0], R_EAX);
-            tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 63);
-            gen_op_mov_reg_v(MO_64, R_EDX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_64, cpu_T0, R_EAX);
+            tcg_gen_sari_tl(cpu_T0, cpu_T0, 63);
+            gen_op_mov_reg_v(MO_64, R_EDX, cpu_T0);
             break;
 #endif
         case MO_32:
-            gen_op_mov_v_reg(MO_32, cpu_T[0], R_EAX);
-            tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 31);
-            gen_op_mov_reg_v(MO_32, R_EDX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_32, cpu_T0, R_EAX);
+            tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
+            tcg_gen_sari_tl(cpu_T0, cpu_T0, 31);
+            gen_op_mov_reg_v(MO_32, R_EDX, cpu_T0);
             break;
         case MO_16:
-            gen_op_mov_v_reg(MO_16, cpu_T[0], R_EAX);
-            tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_sari_tl(cpu_T[0], cpu_T[0], 15);
-            gen_op_mov_reg_v(MO_16, R_EDX, cpu_T[0]);
+            gen_op_mov_v_reg(MO_16, cpu_T0, R_EAX);
+            tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+            tcg_gen_sari_tl(cpu_T0, cpu_T0, 15);
+            gen_op_mov_reg_v(MO_16, R_EDX, cpu_T0);
             break;
         default:
             tcg_abort();
@@ -5079,25 +5006,25 @@
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         if (b == 0x69) {
             val = insn_get(env, s, ot);
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
         } else if (b == 0x6b) {
             val = (int8_t)insn_get(env, s, MO_8);
-            tcg_gen_movi_tl(cpu_T[1], val);
+            tcg_gen_movi_tl(cpu_T1, val);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[1], reg);
+            gen_op_mov_v_reg(ot, cpu_T1, reg);
         }
         switch (ot) {
 #ifdef TARGET_X86_64
         case MO_64:
-            tcg_gen_muls2_i64(cpu_regs[reg], cpu_T[1], cpu_T[0], cpu_T[1]);
+            tcg_gen_muls2_i64(cpu_regs[reg], cpu_T1, cpu_T0, cpu_T1);
             tcg_gen_mov_tl(cpu_cc_dst, cpu_regs[reg]);
             tcg_gen_sari_tl(cpu_cc_src, cpu_cc_dst, 63);
-            tcg_gen_sub_tl(cpu_cc_src, cpu_cc_src, cpu_T[1]);
+            tcg_gen_sub_tl(cpu_cc_src, cpu_cc_src, cpu_T1);
             break;
 #endif
         case MO_32:
-            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-            tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+            tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
             tcg_gen_muls2_i32(cpu_tmp2_i32, cpu_tmp3_i32,
                               cpu_tmp2_i32, cpu_tmp3_i32);
             tcg_gen_extu_i32_tl(cpu_regs[reg], cpu_tmp2_i32);
@@ -5107,14 +5034,14 @@
             tcg_gen_extu_i32_tl(cpu_cc_src, cpu_tmp2_i32);
             break;
         default:
-            tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_ext16s_tl(cpu_T[1], cpu_T[1]);
+            tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
+            tcg_gen_ext16s_tl(cpu_T1, cpu_T1);
             /* XXX: use 32 bit mul which could be faster */
-            tcg_gen_mul_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-            tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
-            tcg_gen_ext16s_tl(cpu_tmp0, cpu_T[0]);
-            tcg_gen_sub_tl(cpu_cc_src, cpu_T[0], cpu_tmp0);
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            tcg_gen_mul_tl(cpu_T0, cpu_T0, cpu_T1);
+            tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
+            tcg_gen_ext16s_tl(cpu_tmp0, cpu_T0);
+            tcg_gen_sub_tl(cpu_cc_src, cpu_T0, cpu_tmp0);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
             break;
         }
         set_cc_op(s, CC_OP_MULB + ot);
@@ -5127,18 +5054,18 @@
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
             rm = (modrm & 7) | REX_B(s);
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_mov_v_reg(ot, cpu_T[1], rm);
-            tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_mov_v_reg(ot, cpu_T1, rm);
+            tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
         } else {
             gen_lea_modrm(env, s, modrm);
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
-            tcg_gen_add_tl(cpu_T[0], cpu_T[0], cpu_T[1]);
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
+            tcg_gen_add_tl(cpu_T0, cpu_T0, cpu_T1);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
         }
         gen_op_update2_cc();
         set_cc_op(s, CC_OP_ADDB + ot);
@@ -5146,7 +5073,7 @@
     case 0x1b0:
     case 0x1b1: /* cmpxchg Ev, Gv */
         {
-            int label1, label2;
+            TCGLabel *label1, *label2;
             TCGv t0, t1, t2, a0;
 
             ot = mo_b_d(b, dflag);
@@ -5208,8 +5135,6 @@
         if (dflag == MO_64) {
             if (!(s->cpuid_ext_features & CPUID_EXT_CX16))
                 goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_update_cc_op(s);
             gen_lea_modrm(env, s, modrm);
             gen_helper_cmpxchg16b(cpu_env, cpu_A0);
         } else
@@ -5217,8 +5142,6 @@
         {
             if (!(s->cpuid_features & CPUID_CX8))
                 goto illegal_op;
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_update_cc_op(s);
             gen_lea_modrm(env, s, modrm);
             gen_helper_cmpxchg8b(cpu_env, cpu_A0);
         }
@@ -5228,14 +5151,14 @@
         /**************************/
         /* push/pop */
     case 0x50 ... 0x57: /* push */
-        gen_op_mov_v_reg(MO_32, cpu_T[0], (b & 7) | REX_B(s));
-        gen_push_v(s, cpu_T[0]);
+        gen_op_mov_v_reg(MO_32, cpu_T0, (b & 7) | REX_B(s));
+        gen_push_v(s, cpu_T0);
         break;
     case 0x58 ... 0x5f: /* pop */
         ot = gen_pop_T0(s);
         /* NOTE: order is important for pop %sp */
         gen_pop_update(s, ot);
-        gen_op_mov_reg_v(ot, (b & 7) | REX_B(s), cpu_T[0]);
+        gen_op_mov_reg_v(ot, (b & 7) | REX_B(s), cpu_T0);
         break;
     case 0x60: /* pusha */
         if (CODE64(s))
@@ -5254,8 +5177,8 @@
             val = insn_get(env, s, ot);
         else
             val = (int8_t)insn_get(env, s, MO_8);
-        tcg_gen_movi_tl(cpu_T[0], val);
-        gen_push_v(s, cpu_T[0]);
+        tcg_gen_movi_tl(cpu_T0, val);
+        gen_push_v(s, cpu_T0);
         break;
     case 0x8f: /* pop Ev */
         modrm = cpu_ldub_code(env, s->pc++);
@@ -5265,7 +5188,7 @@
             /* NOTE: order is important for pop %sp */
             gen_pop_update(s, ot);
             rm = (modrm & 7) | REX_B(s);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
         } else {
             /* NOTE: order is important too for MMU exceptions */
             s->popl_esp_hack = 1 << ot;
@@ -5284,20 +5207,7 @@
         }
         break;
     case 0xc9: /* leave */
-        /* XXX: exception not precise (ESP is updated before potential exception) */
-        if (CODE64(s)) {
-            gen_op_mov_v_reg(MO_64, cpu_T[0], R_EBP);
-            gen_op_mov_reg_v(MO_64, R_ESP, cpu_T[0]);
-        } else if (s->ss32) {
-            gen_op_mov_v_reg(MO_32, cpu_T[0], R_EBP);
-            gen_op_mov_reg_v(MO_32, R_ESP, cpu_T[0]);
-        } else {
-            gen_op_mov_v_reg(MO_16, cpu_T[0], R_EBP);
-            gen_op_mov_reg_v(MO_16, R_ESP, cpu_T[0]);
-        }
-        ot = gen_pop_T0(s);
-        gen_op_mov_reg_v(ot, R_EBP, cpu_T[0]);
-        gen_pop_update(s, ot);
+        gen_leave(s);
         break;
     case 0x06: /* push es */
     case 0x0e: /* push cs */
@@ -5306,12 +5216,12 @@
         if (CODE64(s))
             goto illegal_op;
         gen_op_movl_T0_seg(b >> 3);
-        gen_push_v(s, cpu_T[0]);
+        gen_push_v(s, cpu_T0);
         break;
     case 0x1a0: /* push fs */
     case 0x1a8: /* push gs */
         gen_op_movl_T0_seg((b >> 3) & 7);
-        gen_push_v(s, cpu_T[0]);
+        gen_push_v(s, cpu_T0);
         break;
     case 0x07: /* pop es */
     case 0x17: /* pop ss */
@@ -5320,25 +5230,23 @@
             goto illegal_op;
         reg = b >> 3;
         ot = gen_pop_T0(s);
-        gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, reg);
         gen_pop_update(s, ot);
-        if (reg == R_SS) {
-            /* if reg == SS, inhibit interrupts/trace. */
-            /* If several instructions disable interrupts, only the
-               _first_ does it */
-            if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                gen_helper_set_inhibit_irq(cpu_env);
-            s->tf = 0;
-        }
+        /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp.  */
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
-            gen_eob(s);
+            if (reg == R_SS) {
+                s->tf = 0;
+                gen_eob_inhibit_irq(s, true);
+            } else {
+                gen_eob(s);
+            }
         }
         break;
     case 0x1a1: /* pop fs */
     case 0x1a9: /* pop gs */
         ot = gen_pop_T0(s);
-        gen_movl_seg_T0(s, (b >> 3) & 7, pc_start - s->cs_base);
+        gen_movl_seg_T0(s, (b >> 3) & 7);
         gen_pop_update(s, ot);
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
@@ -5367,11 +5275,11 @@
             gen_lea_modrm(env, s, modrm);
         }
         val = insn_get(env, s, ot);
-        tcg_gen_movi_tl(cpu_T[0], val);
+        tcg_gen_movi_tl(cpu_T0, val);
         if (mod != 3) {
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_reg_v(ot, (modrm & 7) | REX_B(s), cpu_T[0]);
+            gen_op_mov_reg_v(ot, (modrm & 7) | REX_B(s), cpu_T0);
         }
         break;
     case 0x8a:
@@ -5381,7 +5289,7 @@
         reg = ((modrm >> 3) & 7) | rex_r;
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+        gen_op_mov_reg_v(ot, reg, cpu_T0);
         break;
     case 0x8e: /* mov seg, Gv */
         modrm = cpu_ldub_code(env, s->pc++);
@@ -5389,18 +5297,16 @@
         if (reg >= 6 || reg == R_CS)
             goto illegal_op;
         gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-        gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
-        if (reg == R_SS) {
-            /* if reg == SS, inhibit interrupts/trace */
-            /* If several instructions disable interrupts, only the
-               _first_ does it */
-            if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                gen_helper_set_inhibit_irq(cpu_env);
-            s->tf = 0;
-        }
+        gen_movl_seg_T0(s, reg);
+        /* Note that reg == R_SS in gen_movl_seg_T0 always sets is_jmp.  */
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
-            gen_eob(s);
+            if (reg == R_SS) {
+                s->tf = 0;
+                gen_eob_inhibit_irq(s, true);
+            } else {
+                gen_eob(s);
+            }
         }
         break;
     case 0x8c: /* mov Gv, seg */
@@ -5435,45 +5341,42 @@
             rm = (modrm & 7) | REX_B(s);
 
             if (mod == 3) {
-                gen_op_mov_v_reg(ot, cpu_T[0], rm);
+                gen_op_mov_v_reg(ot, cpu_T0, rm);
                 switch (s_ot) {
                 case MO_UB:
-                    tcg_gen_ext8u_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext8u_tl(cpu_T0, cpu_T0);
                     break;
                 case MO_SB:
-                    tcg_gen_ext8s_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext8s_tl(cpu_T0, cpu_T0);
                     break;
                 case MO_UW:
-                    tcg_gen_ext16u_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext16u_tl(cpu_T0, cpu_T0);
                     break;
                 default:
                 case MO_SW:
-                    tcg_gen_ext16s_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext16s_tl(cpu_T0, cpu_T0);
                     break;
                 }
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             } else {
                 gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, s_ot, cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_ld_v(s, s_ot, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             }
         }
         break;
 
     case 0x8d: /* lea */
-        ot = dflag;
         modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
         reg = ((modrm >> 3) & 7) | rex_r;
-        /* we must ensure that no segment is added */
-        s->override = -1;
-        val = s->addseg;
-        s->addseg = 0;
-        gen_lea_modrm(env, s, modrm);
-        s->addseg = val;
-        gen_op_mov_reg_v(ot, reg, cpu_A0);
+        {
+            AddressParts a = gen_lea_modrm_0(env, s, modrm);
+            TCGv ea = gen_lea_modrm_1(a);
+            gen_op_mov_reg_v(dflag, reg, ea);
+        }
         break;
 
     case 0xa0: /* mov EAX, Ov */
@@ -5498,27 +5401,27 @@
             tcg_gen_movi_tl(cpu_A0, offset_addr);
             gen_add_A0_ds_seg(s);
             if ((b & 2) == 0) {
-                gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(ot, R_EAX, cpu_T[0]);
+                gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(ot, R_EAX, cpu_T0);
             } else {
-                gen_op_mov_v_reg(ot, cpu_T[0], R_EAX);
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_mov_v_reg(ot, cpu_T0, R_EAX);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             }
         }
         break;
     case 0xd7: /* xlat */
         tcg_gen_mov_tl(cpu_A0, cpu_regs[R_EBX]);
-        tcg_gen_ext8u_tl(cpu_T[0], cpu_regs[R_EAX]);
-        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_T[0]);
+        tcg_gen_ext8u_tl(cpu_T0, cpu_regs[R_EAX]);
+        tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_T0);
         gen_extu(s->aflag, cpu_A0);
         gen_add_A0_ds_seg(s);
-        gen_op_ld_v(s, MO_8, cpu_T[0], cpu_A0);
-        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T[0]);
+        gen_op_ld_v(s, MO_8, cpu_T0, cpu_A0);
+        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T0);
         break;
     case 0xb0 ... 0xb7: /* mov R, Ib */
         val = insn_get(env, s, MO_8);
-        tcg_gen_movi_tl(cpu_T[0], val);
-        gen_op_mov_reg_v(MO_8, (b & 7) | REX_B(s), cpu_T[0]);
+        tcg_gen_movi_tl(cpu_T0, val);
+        gen_op_mov_reg_v(MO_8, (b & 7) | REX_B(s), cpu_T0);
         break;
     case 0xb8 ... 0xbf: /* mov R, Iv */
 #ifdef TARGET_X86_64
@@ -5528,16 +5431,16 @@
             tmp = cpu_ldq_code(env, s->pc);
             s->pc += 8;
             reg = (b & 7) | REX_B(s);
-            tcg_gen_movi_tl(cpu_T[0], tmp);
-            gen_op_mov_reg_v(MO_64, reg, cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T0, tmp);
+            gen_op_mov_reg_v(MO_64, reg, cpu_T0);
         } else
 #endif
         {
             ot = dflag;
             val = insn_get(env, s, ot);
             reg = (b & 7) | REX_B(s);
-            tcg_gen_movi_tl(cpu_T[0], val);
-            gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T0, val);
+            gen_op_mov_reg_v(ot, reg, cpu_T0);
         }
         break;
 
@@ -5556,21 +5459,21 @@
         if (mod == 3) {
             rm = (modrm & 7) | REX_B(s);
         do_xchg_reg:
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
-            gen_op_mov_v_reg(ot, cpu_T[1], rm);
-            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
+            gen_op_mov_v_reg(ot, cpu_T1, rm);
+            gen_op_mov_reg_v(ot, rm, cpu_T0);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
         } else {
             gen_lea_modrm(env, s, modrm);
-            gen_op_mov_v_reg(ot, cpu_T[0], reg);
+            gen_op_mov_v_reg(ot, cpu_T0, reg);
             /* for xchg, lock is implicit */
             if (!(prefixes & PREFIX_LOCK))
                 gen_helper_lock();
-            gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
-            gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
+            gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             if (!(prefixes & PREFIX_LOCK))
                 gen_helper_unlock();
-            gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+            gen_op_mov_reg_v(ot, reg, cpu_T1);
         }
         break;
     case 0xc4: /* les Gv */
@@ -5597,13 +5500,13 @@
         if (mod == 3)
             goto illegal_op;
         gen_lea_modrm(env, s, modrm);
-        gen_op_ld_v(s, ot, cpu_T[1], cpu_A0);
+        gen_op_ld_v(s, ot, cpu_T1, cpu_A0);
         gen_add_A0_im(s, 1 << ot);
         /* load the segment first to handle exceptions properly */
-        gen_op_ld_v(s, MO_16, cpu_T[0], cpu_A0);
-        gen_movl_seg_T0(s, op, pc_start - s->cs_base);
+        gen_op_ld_v(s, MO_16, cpu_T0, cpu_A0);
+        gen_movl_seg_T0(s, op);
         /* then put the data */
-        gen_op_mov_reg_v(ot, reg, cpu_T[1]);
+        gen_op_mov_reg_v(ot, reg, cpu_T1);
         if (s->is_jmp) {
             gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
@@ -5682,7 +5585,7 @@
         } else {
             opreg = rm;
         }
-        gen_op_mov_v_reg(ot, cpu_T[1], reg);
+        gen_op_mov_v_reg(ot, cpu_T1, reg);
 
         if (shift) {
             TCGv imm = tcg_const_tl(cpu_ldub_code(env, s->pc++));
@@ -5833,8 +5736,6 @@
                 }
                 break;
             case 0x0c: /* fldenv mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fldenv(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x0d: /* fldcw mem */
@@ -5843,8 +5744,6 @@
                 gen_helper_fldcw(cpu_env, cpu_tmp2_i32);
                 break;
             case 0x0e: /* fnstenv mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fstenv(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x0f: /* fnstcw mem */
@@ -5853,24 +5752,16 @@
                                     s->mem_index, MO_LEUW);
                 break;
             case 0x1d: /* fldt mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fldt_ST0(cpu_env, cpu_A0);
                 break;
             case 0x1f: /* fstpt mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fstt_ST0(cpu_env, cpu_A0);
                 gen_helper_fpop(cpu_env);
                 break;
             case 0x2c: /* frstor mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_frstor(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x2e: /* fnsave mem */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fsave(cpu_env, cpu_A0, tcg_const_i32(dflag - 1));
                 break;
             case 0x2f: /* fnstsw mem */
@@ -5879,13 +5770,9 @@
                                     s->mem_index, MO_LEUW);
                 break;
             case 0x3c: /* fbld */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fbld_ST0(cpu_env, cpu_A0);
                 break;
             case 0x3e: /* fbstp */
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
                 gen_helper_fbst_ST0(cpu_env, cpu_A0);
                 gen_helper_fpop(cpu_env);
                 break;
@@ -5899,7 +5786,7 @@
                 gen_helper_fpop(cpu_env);
                 break;
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
         } else {
             /* register float ops */
@@ -5920,12 +5807,10 @@
                 switch(rm) {
                 case 0: /* fnop */
                     /* check exceptions (FreeBSD FPU probe) */
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
                     gen_helper_fwait(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x0c: /* grp d9/4 */
@@ -5944,7 +5829,7 @@
                     gen_helper_fxam_ST0(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x0d: /* grp d9/5 */
@@ -5979,7 +5864,7 @@
                         gen_helper_fldz_ST0(cpu_env);
                         break;
                     default:
-                        goto illegal_op;
+                        goto unknown_op;
                     }
                 }
                 break;
@@ -6079,7 +5964,7 @@
                     gen_helper_fpop(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x1c:
@@ -6097,7 +5982,7 @@
                 case 4: /* fsetpm (287 only, just do nop here) */
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x1d: /* fucomi */
@@ -6149,7 +6034,7 @@
                     gen_helper_fpop(cpu_env);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x38: /* ffreep sti, undocumented op */
@@ -6160,11 +6045,11 @@
                 switch(rm) {
                 case 0:
                     gen_helper_fnstsw(cpu_tmp2_i32, cpu_env);
-                    tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
-                    gen_op_mov_reg_v(MO_16, R_EAX, cpu_T[0]);
+                    tcg_gen_extu_i32_tl(cpu_T0, cpu_tmp2_i32);
+                    gen_op_mov_reg_v(MO_16, R_EAX, cpu_T0);
                     break;
                 default:
-                    goto illegal_op;
+                    goto unknown_op;
                 }
                 break;
             case 0x3d: /* fucomip */
@@ -6190,7 +6075,8 @@
             case 0x10 ... 0x13: /* fcmovxx */
             case 0x18 ... 0x1b:
                 {
-                    int op1, l1;
+                    int op1;
+                    TCGLabel *l1;
                     static const uint8_t fcmov_cc[8] = {
                         (JCC_B << 1),
                         (JCC_Z << 1),
@@ -6209,7 +6095,7 @@
                 }
                 break;
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
         }
         break;
@@ -6270,14 +6156,14 @@
     case 0x6c: /* insS */
     case 0x6d:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base, 
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes) | 4);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
             gen_repz_ins(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
         } else {
             gen_ins(s, ot);
-            if (use_icount) {
+            if (s->tb->cflags & CF_USE_ICOUNT) {
                 gen_jmp(s, s->pc - s->cs_base);
             }
         }
@@ -6285,14 +6171,14 @@
     case 0x6e: /* outsS */
     case 0x6f:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes) | 4);
         if (prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) {
             gen_repz_outs(s, ot, pc_start - s->cs_base, s->pc - s->cs_base);
         } else {
             gen_outs(s, ot);
-            if (use_icount) {
+            if (s->tb->cflags & CF_USE_ICOUNT) {
                 gen_jmp(s, s->pc - s->cs_base);
             }
         }
@@ -6305,15 +6191,17 @@
     case 0xe5:
         ot = mo_b_d32(b, dflag);
         val = cpu_ldub_code(env, s->pc++);
-        tcg_gen_movi_tl(cpu_T[0], val);
+        tcg_gen_movi_tl(cpu_T0, val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
-        if (use_icount)
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
+	}
         tcg_gen_movi_i32(cpu_tmp2_i32, val);
-        gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
-        gen_op_mov_reg_v(ot, R_EAX, cpu_T[1]);
-        if (use_icount) {
+        gen_helper_in_func(ot, cpu_T1, cpu_tmp2_i32);
+        gen_op_mov_reg_v(ot, R_EAX, cpu_T1);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
         }
@@ -6322,17 +6210,19 @@
     case 0xe7:
         ot = mo_b_d32(b, dflag);
         val = cpu_ldub_code(env, s->pc++);
-        tcg_gen_movi_tl(cpu_T[0], val);
+        tcg_gen_movi_tl(cpu_T0, val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
-        gen_op_mov_v_reg(ot, cpu_T[1], R_EAX);
+        gen_op_mov_v_reg(ot, cpu_T1, R_EAX);
 
-        if (use_icount)
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
+	}
         tcg_gen_movi_i32(cpu_tmp2_i32, val);
-        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
         gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
-        if (use_icount) {
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
         }
@@ -6340,15 +6230,17 @@
     case 0xec:
     case 0xed:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
-        if (use_icount)
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
-        gen_op_mov_reg_v(ot, R_EAX, cpu_T[1]);
-        if (use_icount) {
+	}
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+        gen_helper_in_func(ot, cpu_T1, cpu_tmp2_i32);
+        gen_op_mov_reg_v(ot, R_EAX, cpu_T1);
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
         }
@@ -6356,17 +6248,19 @@
     case 0xee:
     case 0xef:
         ot = mo_b_d32(b, dflag);
-        tcg_gen_ext16u_tl(cpu_T[0], cpu_regs[R_EDX]);
+        tcg_gen_ext16u_tl(cpu_T0, cpu_regs[R_EDX]);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
-        gen_op_mov_v_reg(ot, cpu_T[1], R_EAX);
+        gen_op_mov_v_reg(ot, cpu_T1, R_EAX);
 
-        if (use_icount)
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
+	}
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
+        tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T1);
         gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
-        if (use_icount) {
+        gen_bpt_io(s, cpu_tmp2_i32, ot);
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
         }
@@ -6380,14 +6274,16 @@
         ot = gen_pop_T0(s);
         gen_stack_update(s, val + (1 << ot));
         /* Note that gen_pop_T0 uses a zero-extending load.  */
-        gen_op_jmp_v(cpu_T[0]);
+        gen_op_jmp_v(cpu_T0);
+        gen_bnd_jmp(s);
         gen_eob(s);
         break;
     case 0xc3: /* ret */
         ot = gen_pop_T0(s);
         gen_pop_update(s, ot);
         /* Note that gen_pop_T0 uses a zero-extending load.  */
-        gen_op_jmp_v(cpu_T[0]);
+        gen_op_jmp_v(cpu_T0);
+        gen_bnd_jmp(s);
         gen_eob(s);
         break;
     case 0xca: /* lret im */
@@ -6402,13 +6298,13 @@
         } else {
             gen_stack_A0(s);
             /* pop offset */
-            gen_op_ld_v(s, dflag, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, dflag, cpu_T0, cpu_A0);
             /* NOTE: keeping EIP updated is not a problem in case of
                exception */
-            gen_op_jmp_v(cpu_T[0]);
+            gen_op_jmp_v(cpu_T0);
             /* pop selector */
-            gen_op_addl_A0_im(1 << dflag);
-            gen_op_ld_v(s, dflag, cpu_T[0], cpu_A0);
+            gen_add_A0_im(s, 1 << dflag);
+            gen_op_ld_v(s, dflag, cpu_T0, cpu_A0);
             gen_op_movl_seg_T0_vm(R_CS);
             /* add stack offset */
             gen_stack_update(s, val + (2 << dflag));
@@ -6432,8 +6328,6 @@
                 set_cc_op(s, CC_OP_EFLAGS);
             }
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_iret_protected(cpu_env, tcg_const_i32(dflag - 1),
                                       tcg_const_i32(s->pc - s->cs_base));
             set_cc_op(s, CC_OP_EFLAGS);
@@ -6454,8 +6348,9 @@
             } else if (!CODE64(s)) {
                 tval &= 0xffffffff;
             }
-            tcg_gen_movi_tl(cpu_T[0], next_eip);
-            gen_push_v(s, cpu_T[0]);
+            tcg_gen_movi_tl(cpu_T0, next_eip);
+            gen_push_v(s, cpu_T0);
+            gen_bnd_jmp(s);
             gen_jmp(s, tval);
         }
         break;
@@ -6469,8 +6364,8 @@
             offset = insn_get(env, s, ot);
             selector = insn_get(env, s, MO_16);
 
-            tcg_gen_movi_tl(cpu_T[0], selector);
-            tcg_gen_movi_tl(cpu_T[1], offset);
+            tcg_gen_movi_tl(cpu_T0, selector);
+            tcg_gen_movi_tl(cpu_T1, offset);
         }
         goto do_lcall;
     case 0xe9: /* jmp im */
@@ -6485,6 +6380,7 @@
         } else if (!CODE64(s)) {
             tval &= 0xffffffff;
         }
+        gen_bnd_jmp(s);
         gen_jmp(s, tval);
         break;
     case 0xea: /* ljmp im */
@@ -6497,8 +6393,8 @@
             offset = insn_get(env, s, ot);
             selector = insn_get(env, s, MO_16);
 
-            tcg_gen_movi_tl(cpu_T[0], selector);
-            tcg_gen_movi_tl(cpu_T[1], offset);
+            tcg_gen_movi_tl(cpu_T0, selector);
+            tcg_gen_movi_tl(cpu_T1, offset);
         }
         goto do_ljmp;
     case 0xeb: /* jmp Jb */
@@ -6524,12 +6420,13 @@
         if (dflag == MO_16) {
             tval &= 0xffff;
         }
+        gen_bnd_jmp(s);
         gen_jcc(s, b, tval, next_eip);
         break;
 
     case 0x190 ... 0x19f: /* setcc Gv */
         modrm = cpu_ldub_code(env, s->pc++);
-        gen_setcc1(s, b, cpu_T[0]);
+        gen_setcc1(s, b, cpu_T0);
         gen_ldst_modrm(env, s, modrm, MO_8, OR_TMP0, 1);
         break;
     case 0x140 ... 0x14f: /* cmov Gv, Ev */
@@ -6550,8 +6447,8 @@
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
             gen_update_cc_op(s);
-            gen_helper_read_eflags(cpu_T[0], cpu_env);
-            gen_push_v(s, cpu_T[0]);
+            gen_helper_read_eflags(cpu_T0, cpu_env);
+            gen_push_v(s, cpu_T0);
         }
         break;
     case 0x9d: /* popf */
@@ -6562,13 +6459,13 @@
             ot = gen_pop_T0(s);
             if (s->cpl == 0) {
                 if (dflag != MO_16) {
-                    gen_helper_write_eflags(cpu_env, cpu_T[0],
+                    gen_helper_write_eflags(cpu_env, cpu_T0,
                                             tcg_const_i32((TF_MASK | AC_MASK |
                                                            ID_MASK | NT_MASK |
                                                            IF_MASK |
                                                            IOPL_MASK)));
                 } else {
-                    gen_helper_write_eflags(cpu_env, cpu_T[0],
+                    gen_helper_write_eflags(cpu_env, cpu_T0,
                                             tcg_const_i32((TF_MASK | AC_MASK |
                                                            ID_MASK | NT_MASK |
                                                            IF_MASK | IOPL_MASK)
@@ -6577,14 +6474,14 @@
             } else {
                 if (s->cpl <= s->iopl) {
                     if (dflag != MO_16) {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                                 tcg_const_i32((TF_MASK |
                                                                AC_MASK |
                                                                ID_MASK |
                                                                NT_MASK |
                                                                IF_MASK)));
                     } else {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                                 tcg_const_i32((TF_MASK |
                                                                AC_MASK |
                                                                ID_MASK |
@@ -6594,11 +6491,11 @@
                     }
                 } else {
                     if (dflag != MO_16) {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                            tcg_const_i32((TF_MASK | AC_MASK |
                                                           ID_MASK | NT_MASK)));
                     } else {
-                        gen_helper_write_eflags(cpu_env, cpu_T[0],
+                        gen_helper_write_eflags(cpu_env, cpu_T0,
                                            tcg_const_i32((TF_MASK | AC_MASK |
                                                           ID_MASK | NT_MASK)
                                                          & 0xffff));
@@ -6615,19 +6512,19 @@
     case 0x9e: /* sahf */
         if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
             goto illegal_op;
-        gen_op_mov_v_reg(MO_8, cpu_T[0], R_AH);
+        gen_op_mov_v_reg(MO_8, cpu_T0, R_AH);
         gen_compute_eflags(s);
         tcg_gen_andi_tl(cpu_cc_src, cpu_cc_src, CC_O);
-        tcg_gen_andi_tl(cpu_T[0], cpu_T[0], CC_S | CC_Z | CC_A | CC_P | CC_C);
-        tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_T[0]);
+        tcg_gen_andi_tl(cpu_T0, cpu_T0, CC_S | CC_Z | CC_A | CC_P | CC_C);
+        tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_T0);
         break;
     case 0x9f: /* lahf */
         if (CODE64(s) && !(s->cpuid_ext3_features & CPUID_EXT3_LAHF_LM))
             goto illegal_op;
         gen_compute_eflags(s);
         /* Note: gen_compute_eflags() only gives the condition codes */
-        tcg_gen_ori_tl(cpu_T[0], cpu_cc_src, 0x02);
-        gen_op_mov_reg_v(MO_8, R_AH, cpu_T[0]);
+        tcg_gen_ori_tl(cpu_T0, cpu_cc_src, 0x02);
+        gen_op_mov_reg_v(MO_8, R_AH, cpu_T0);
         break;
     case 0xf5: /* cmc */
         gen_compute_eflags(s);
@@ -6661,15 +6558,15 @@
         if (mod != 3) {
             s->rip_offset = 1;
             gen_lea_modrm(env, s, modrm);
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
         /* load shift */
         val = cpu_ldub_code(env, s->pc++);
-        tcg_gen_movi_tl(cpu_T[1], val);
+        tcg_gen_movi_tl(cpu_T1, val);
         if (op < 4)
-            goto illegal_op;
+            goto unknown_op;
         op -= 4;
         goto bt_op;
     case 0x1a3: /* bt Gv, Ev */
@@ -6689,46 +6586,46 @@
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
-        gen_op_mov_v_reg(MO_32, cpu_T[1], reg);
+        gen_op_mov_v_reg(MO_32, cpu_T1, reg);
         if (mod != 3) {
             gen_lea_modrm(env, s, modrm);
             /* specific case: we need to add a displacement */
-            gen_exts(ot, cpu_T[1]);
-            tcg_gen_sari_tl(cpu_tmp0, cpu_T[1], 3 + ot);
+            gen_exts(ot, cpu_T1);
+            tcg_gen_sari_tl(cpu_tmp0, cpu_T1, 3 + ot);
             tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, ot);
             tcg_gen_add_tl(cpu_A0, cpu_A0, cpu_tmp0);
-            gen_op_ld_v(s, ot, cpu_T[0], cpu_A0);
+            gen_op_ld_v(s, ot, cpu_T0, cpu_A0);
         } else {
-            gen_op_mov_v_reg(ot, cpu_T[0], rm);
+            gen_op_mov_v_reg(ot, cpu_T0, rm);
         }
     bt_op:
-        tcg_gen_andi_tl(cpu_T[1], cpu_T[1], (1 << (3 + ot)) - 1);
-        tcg_gen_shr_tl(cpu_tmp4, cpu_T[0], cpu_T[1]);
+        tcg_gen_andi_tl(cpu_T1, cpu_T1, (1 << (3 + ot)) - 1);
+        tcg_gen_shr_tl(cpu_tmp4, cpu_T0, cpu_T1);
         switch(op) {
         case 0:
             break;
         case 1:
             tcg_gen_movi_tl(cpu_tmp0, 1);
-            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
-            tcg_gen_or_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T1);
+            tcg_gen_or_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         case 2:
             tcg_gen_movi_tl(cpu_tmp0, 1);
-            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
-            tcg_gen_andc_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T1);
+            tcg_gen_andc_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         default:
         case 3:
             tcg_gen_movi_tl(cpu_tmp0, 1);
-            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T[1]);
-            tcg_gen_xor_tl(cpu_T[0], cpu_T[0], cpu_tmp0);
+            tcg_gen_shl_tl(cpu_tmp0, cpu_tmp0, cpu_T1);
+            tcg_gen_xor_tl(cpu_T0, cpu_T0, cpu_tmp0);
             break;
         }
         if (op != 0) {
             if (mod != 3) {
-                gen_op_st_v(s, ot, cpu_T[0], cpu_A0);
+                gen_op_st_v(s, ot, cpu_T0, cpu_A0);
             } else {
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
         }
 
@@ -6768,7 +6665,7 @@
         modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_extu(ot, cpu_T[0]);
+        gen_extu(ot, cpu_T0);
 
         /* Note that lzcnt and tzcnt are in different extensions.  */
         if ((prefixes & PREFIX_REPZ)
@@ -6776,18 +6673,18 @@
                 ? s->cpuid_ext3_features & CPUID_EXT3_ABM
                 : s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_BMI1)) {
             int size = 8 << ot;
-            tcg_gen_mov_tl(cpu_cc_src, cpu_T[0]);
+            tcg_gen_mov_tl(cpu_cc_src, cpu_T0);
             if (b & 1) {
                 /* For lzcnt, reduce the target_ulong result by the
                    number of zeros that we expect to find at the top.  */
-                gen_helper_clz(cpu_T[0], cpu_T[0]);
-                tcg_gen_subi_tl(cpu_T[0], cpu_T[0], TARGET_LONG_BITS - size);
+                gen_helper_clz(cpu_T0, cpu_T0);
+                tcg_gen_subi_tl(cpu_T0, cpu_T0, TARGET_LONG_BITS - size);
             } else {
                 /* For tzcnt, a zero input must return the operand size:
                    force all bits outside the operand size to 1.  */
                 target_ulong mask = (target_ulong)-2 << (size - 1);
-                tcg_gen_ori_tl(cpu_T[0], cpu_T[0], mask);
-                gen_helper_ctz(cpu_T[0], cpu_T[0]);
+                tcg_gen_ori_tl(cpu_T0, cpu_T0, mask);
+                gen_helper_ctz(cpu_T0, cpu_T0);
             }
             /* For lzcnt/tzcnt, C and Z bits are defined and are
                related to the result.  */
@@ -6796,24 +6693,24 @@
         } else {
             /* For bsr/bsf, only the Z bit is defined and it is related
                to the input and not the result.  */
-            tcg_gen_mov_tl(cpu_cc_dst, cpu_T[0]);
+            tcg_gen_mov_tl(cpu_cc_dst, cpu_T0);
             set_cc_op(s, CC_OP_LOGICB + ot);
             if (b & 1) {
                 /* For bsr, return the bit index of the first 1 bit,
                    not the count of leading zeros.  */
-                gen_helper_clz(cpu_T[0], cpu_T[0]);
-                tcg_gen_xori_tl(cpu_T[0], cpu_T[0], TARGET_LONG_BITS - 1);
+                gen_helper_clz(cpu_T0, cpu_T0);
+                tcg_gen_xori_tl(cpu_T0, cpu_T0, TARGET_LONG_BITS - 1);
             } else {
-                gen_helper_ctz(cpu_T[0], cpu_T[0]);
+                gen_helper_ctz(cpu_T0, cpu_T0);
             }
             /* ??? The manual says that the output is undefined when the
                input is zero, but real hardware leaves it unchanged, and
                real programs appear to depend on that.  */
             tcg_gen_movi_tl(cpu_tmp0, 0);
-            tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T[0], cpu_cc_dst, cpu_tmp0,
-                               cpu_regs[reg], cpu_T[0]);
+            tcg_gen_movcond_tl(TCG_COND_EQ, cpu_T0, cpu_cc_dst, cpu_tmp0,
+                               cpu_regs[reg], cpu_T0);
         }
-        gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+        gen_op_mov_reg_v(ot, reg, cpu_T0);
         break;
         /************************/
         /* bcd */
@@ -6886,8 +6783,6 @@
             (HF_MP_MASK | HF_TS_MASK)) {
             gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_fwait(cpu_env);
         }
         break;
@@ -6916,7 +6811,7 @@
         gen_debug(s, pc_start - s->cs_base);
 #else
         /* start debug */
-        tb_flush(env);
+        tb_flush(CPU(x86_env_get_cpu(env)));
         qemu_set_log(CPU_LOG_INT | CPU_LOG_TB_IN_ASM);
 #endif
         break;
@@ -6937,27 +6832,13 @@
         }
         break;
     case 0xfb: /* sti */
-        if (!s->vm86) {
-            if (s->cpl <= s->iopl) {
-            gen_sti:
-                gen_helper_sti(cpu_env);
-                /* interruptions are enabled only the first insn after sti */
-                /* If several instructions disable interrupts, only the
-                   _first_ does it */
-                if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                    gen_helper_set_inhibit_irq(cpu_env);
-                /* give a chance to handle pending irqs */
-                gen_jmp_im(s->pc - s->cs_base);
-                gen_eob(s);
-            } else {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            }
+        if (s->vm86 ? s->iopl == 3 : s->cpl <= s->iopl) {
+            gen_helper_sti(cpu_env);
+            /* interruptions are enabled only the first insn after sti */
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob_inhibit_irq(s, true);
         } else {
-            if (s->iopl == 3) {
-                goto gen_sti;
-            } else {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            }
+            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         }
         break;
     case 0x62: /* bound */
@@ -6969,10 +6850,9 @@
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_op_mov_v_reg(ot, cpu_T[0], reg);
+        gen_op_mov_v_reg(ot, cpu_T0, reg);
         gen_lea_modrm(env, s, modrm);
-        gen_jmp_im(pc_start - s->cs_base);
-        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
         if (ot == MO_16) {
             gen_helper_boundw(cpu_env, cpu_A0, cpu_tmp2_i32);
         } else {
@@ -6983,31 +6863,31 @@
         reg = (b & 7) | REX_B(s);
 #ifdef TARGET_X86_64
         if (dflag == MO_64) {
-            gen_op_mov_v_reg(MO_64, cpu_T[0], reg);
-            tcg_gen_bswap64_i64(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_64, reg, cpu_T[0]);
+            gen_op_mov_v_reg(MO_64, cpu_T0, reg);
+            tcg_gen_bswap64_i64(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_64, reg, cpu_T0);
         } else
 #endif
         {
-            gen_op_mov_v_reg(MO_32, cpu_T[0], reg);
-            tcg_gen_ext32u_tl(cpu_T[0], cpu_T[0]);
-            tcg_gen_bswap32_tl(cpu_T[0], cpu_T[0]);
-            gen_op_mov_reg_v(MO_32, reg, cpu_T[0]);
+            gen_op_mov_v_reg(MO_32, cpu_T0, reg);
+            tcg_gen_ext32u_tl(cpu_T0, cpu_T0);
+            tcg_gen_bswap32_tl(cpu_T0, cpu_T0);
+            gen_op_mov_reg_v(MO_32, reg, cpu_T0);
         }
         break;
     case 0xd6: /* salc */
         if (CODE64(s))
             goto illegal_op;
-        gen_compute_eflags_c(s, cpu_T[0]);
-        tcg_gen_neg_tl(cpu_T[0], cpu_T[0]);
-        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T[0]);
+        gen_compute_eflags_c(s, cpu_T0);
+        tcg_gen_neg_tl(cpu_T0, cpu_T0);
+        gen_op_mov_reg_v(MO_8, R_EAX, cpu_T0);
         break;
     case 0xe0: /* loopnz */
     case 0xe1: /* loopz */
     case 0xe2: /* loop */
     case 0xe3: /* jecxz */
         {
-            int l1, l2, l3;
+            TCGLabel *l1, *l2, *l3;
 
             tval = (int8_t)insn_get(env, s, MO_8);
             next_eip = s->pc - s->cs_base;
@@ -7064,10 +6944,11 @@
     case 0x131: /* rdtsc */
         gen_update_cc_op(s);
         gen_jmp_im(pc_start - s->cs_base);
-        if (use_icount)
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
+	}
         gen_helper_rdtsc(cpu_env);
-        if (use_icount) {
+        if (s->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
             gen_jmp(s, s->pc - s->cs_base);
         }
@@ -7084,8 +6965,6 @@
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysenter(cpu_env);
             gen_eob(s);
         }
@@ -7097,8 +6976,6 @@
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysexit(cpu_env, tcg_const_i32(dflag - 1));
             gen_eob(s);
         }
@@ -7115,8 +6992,6 @@
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
             gen_helper_sysret(cpu_env, tcg_const_i32(dflag - 1));
             /* condition codes are modified only in long mode */
             if (s->lma) {
@@ -7150,7 +7025,8 @@
             if (!s->pe || s->vm86)
                 goto illegal_op;
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_READ);
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,ldt.selector));
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env,
+                             offsetof(CPUX86State, ldt.selector));
             ot = mod == 3 ? dflag : MO_16;
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
@@ -7162,8 +7038,7 @@
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_lldt(cpu_env, cpu_tmp2_i32);
             }
             break;
@@ -7171,7 +7046,8 @@
             if (!s->pe || s->vm86)
                 goto illegal_op;
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_READ);
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,tr.selector));
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env,
+                             offsetof(CPUX86State, tr.selector));
             ot = mod == 3 ? dflag : MO_16;
             gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
@@ -7183,8 +7059,7 @@
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
                 gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T0);
                 gen_helper_ltr(cpu_env, cpu_tmp2_i32);
             }
             break;
@@ -7195,278 +7070,347 @@
             gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
             gen_update_cc_op(s);
             if (op == 4) {
-                gen_helper_verr(cpu_env, cpu_T[0]);
+                gen_helper_verr(cpu_env, cpu_T0);
             } else {
-                gen_helper_verw(cpu_env, cpu_T[0]);
+                gen_helper_verw(cpu_env, cpu_T0);
             }
             set_cc_op(s, CC_OP_EFLAGS);
             break;
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
+
     case 0x101:
         modrm = cpu_ldub_code(env, s->pc++);
-        mod = (modrm >> 6) & 3;
-        op = (modrm >> 3) & 7;
-        rm = modrm & 7;
-        switch(op) {
-        case 0: /* sgdt */
-            if (mod == 3)
-                goto illegal_op;
+        switch (modrm) {
+        CASE_MODRM_MEM_OP(0): /* sgdt */
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
             gen_lea_modrm(env, s, modrm);
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.limit));
-            gen_op_st_v(s, MO_16, cpu_T[0], cpu_A0);
+            tcg_gen_ld32u_tl(cpu_T0,
+                             cpu_env, offsetof(CPUX86State, gdt.limit));
+            gen_op_st_v(s, MO_16, cpu_T0, cpu_A0);
             gen_add_A0_im(s, 2);
-            tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.base));
+            tcg_gen_ld_tl(cpu_T0, cpu_env, offsetof(CPUX86State, gdt.base));
             if (dflag == MO_16) {
-                tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffffff);
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
             }
-            gen_op_st_v(s, CODE64(s) + MO_32, cpu_T[0], cpu_A0);
+            gen_op_st_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
             break;
-        case 1:
-            if (mod == 3) {
-                switch (rm) {
-                case 0: /* monitor */
-                    if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) ||
-                        s->cpl != 0)
-                        goto illegal_op;
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    tcg_gen_mov_tl(cpu_A0, cpu_regs[R_EAX]);
-                    gen_extu(s->aflag, cpu_A0);
-                    gen_add_A0_ds_seg(s);
-                    gen_helper_monitor(cpu_env, cpu_A0);
-                    break;
-                case 1: /* mwait */
-                    if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) ||
-                        s->cpl != 0)
-                        goto illegal_op;
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start));
-                    gen_eob(s);
-                    break;
-                case 2: /* clac */
-                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
-                        s->cpl != 0) {
-                        goto illegal_op;
-                    }
-                    gen_helper_clac(cpu_env);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    gen_eob(s);
-                    break;
-                case 3: /* stac */
-                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
-                        s->cpl != 0) {
-                        goto illegal_op;
-                    }
-                    gen_helper_stac(cpu_env);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    gen_eob(s);
-                    break;
-                default:
-                    goto illegal_op;
-                }
-            } else { /* sidt */
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
-                gen_lea_modrm(env, s, modrm);
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.limit));
-                gen_op_st_v(s, MO_16, cpu_T[0], cpu_A0);
-                gen_add_A0_im(s, 2);
-                tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.base));
-                if (dflag == MO_16) {
-                    tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffffff);
-                }
-                gen_op_st_v(s, CODE64(s) + MO_32, cpu_T[0], cpu_A0);
+
+        case 0xc8: /* monitor */
+            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || s->cpl != 0) {
+                goto illegal_op;
             }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            tcg_gen_mov_tl(cpu_A0, cpu_regs[R_EAX]);
+            gen_extu(s->aflag, cpu_A0);
+            gen_add_A0_ds_seg(s);
+            gen_helper_monitor(cpu_env, cpu_A0);
             break;
-        case 2: /* lgdt */
-        case 3: /* lidt */
-            if (mod == 3) {
-                gen_update_cc_op(s);
-                gen_jmp_im(pc_start - s->cs_base);
-                switch(rm) {
-                case 0: /* VMRUN */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1),
-                                         tcg_const_i32(s->pc - pc_start));
-                        tcg_gen_exit_tb(0);
-                        s->is_jmp = DISAS_TB_JUMP;
-                    }
-                    break;
-                case 1: /* VMMCALL */
-                    if (!(s->flags & HF_SVME_MASK))
-                        goto illegal_op;
-                    gen_helper_vmmcall(cpu_env);
-                    break;
-                case 2: /* VMLOAD */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1));
-                    }
-                    break;
-                case 3: /* VMSAVE */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1));
-                    }
-                    break;
-                case 4: /* STGI */
-                    if ((!(s->flags & HF_SVME_MASK) &&
-                         !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || 
-                        !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_stgi(cpu_env);
-                    }
-                    break;
-                case 5: /* CLGI */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_clgi(cpu_env);
-                    }
-                    break;
-                case 6: /* SKINIT */
-                    if ((!(s->flags & HF_SVME_MASK) && 
-                         !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || 
-                        !s->pe)
-                        goto illegal_op;
-                    gen_helper_skinit(cpu_env);
-                    break;
-                case 7: /* INVLPGA */
-                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
-                        goto illegal_op;
-                    if (s->cpl != 0) {
-                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        break;
-                    } else {
-                        gen_helper_invlpga(cpu_env,
-                                           tcg_const_i32(s->aflag - 1));
-                    }
-                    break;
-                default:
-                    goto illegal_op;
-                }
-            } else if (s->cpl != 0) {
-                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            } else {
-                gen_svm_check_intercept(s, pc_start,
-                                        op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE);
-                gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_16, cpu_T[1], cpu_A0);
-                gen_add_A0_im(s, 2);
-                gen_op_ld_v(s, CODE64(s) + MO_32, cpu_T[0], cpu_A0);
-                if (dflag == MO_16) {
-                    tcg_gen_andi_tl(cpu_T[0], cpu_T[0], 0xffffff);
-                }
-                if (op == 2) {
-                    tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,gdt.base));
-                    tcg_gen_st32_tl(cpu_T[1], cpu_env, offsetof(CPUX86State,gdt.limit));
-                } else {
-                    tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,idt.base));
-                    tcg_gen_st32_tl(cpu_T[1], cpu_env, offsetof(CPUX86State,idt.limit));
-                }
+
+        case 0xc9: /* mwait */
+            if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) || s->cpl != 0) {
+                goto illegal_op;
             }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start));
+            gen_eob(s);
             break;
-        case 4: /* smsw */
-            gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
-#if defined TARGET_X86_64 && defined HOST_WORDS_BIGENDIAN
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]) + 4);
-#else
-            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
-#endif
-            gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 1);
+
+        case 0xca: /* clac */
+            if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
+                || s->cpl != 0) {
+                goto illegal_op;
+            }
+            gen_helper_clac(cpu_env);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
             break;
-        case 6: /* lmsw */
+
+        case 0xcb: /* stac */
+            if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)
+                || s->cpl != 0) {
+                goto illegal_op;
+            }
+            gen_helper_stac(cpu_env);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        CASE_MODRM_MEM_OP(1): /* sidt */
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State, idt.limit));
+            gen_op_st_v(s, MO_16, cpu_T0, cpu_A0);
+            gen_add_A0_im(s, 2);
+            tcg_gen_ld_tl(cpu_T0, cpu_env, offsetof(CPUX86State, idt.base));
+            if (dflag == MO_16) {
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
+            }
+            gen_op_st_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
+            break;
+
+        case 0xd0: /* xgetbv */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (s->prefix & (PREFIX_LOCK | PREFIX_DATA
+                                 | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
+            }
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_xgetbv(cpu_tmp1_i64, cpu_env, cpu_tmp2_i32);
+            tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], cpu_tmp1_i64);
+            break;
+
+        case 0xd1: /* xsetbv */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (s->prefix & (PREFIX_LOCK | PREFIX_DATA
+                                 | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
+            }
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-            } else {
-                gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
-                gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
-                gen_helper_lmsw(cpu_env, cpu_T[0]);
-                gen_jmp_im(s->pc - s->cs_base);
-                gen_eob(s);
+                break;
             }
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_xsetbv(cpu_env, cpu_tmp2_i32, cpu_tmp1_i64);
+            /* End TB because translation flags may change.  */
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
             break;
-        case 7:
-            if (mod != 3) { /* invlpg */
+
+        case 0xd8: /* VMRUN */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmrun(cpu_env, tcg_const_i32(s->aflag - 1),
+                             tcg_const_i32(s->pc - pc_start));
+            tcg_gen_exit_tb(0);
+            s->is_jmp = DISAS_TB_JUMP;
+            break;
+
+        case 0xd9: /* VMMCALL */
+            if (!(s->flags & HF_SVME_MASK)) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmmcall(cpu_env);
+            break;
+
+        case 0xda: /* VMLOAD */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmload(cpu_env, tcg_const_i32(s->aflag - 1));
+            break;
+
+        case 0xdb: /* VMSAVE */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_vmsave(cpu_env, tcg_const_i32(s->aflag - 1));
+            break;
+
+        case 0xdc: /* STGI */
+            if ((!(s->flags & HF_SVME_MASK)
+                   && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
+                || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_stgi(cpu_env);
+            break;
+
+        case 0xdd: /* CLGI */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_clgi(cpu_env);
+            break;
+
+        case 0xde: /* SKINIT */
+            if ((!(s->flags & HF_SVME_MASK)
+                 && !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT))
+                || !s->pe) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_skinit(cpu_env);
+            break;
+
+        case 0xdf: /* INVLPGA */
+            if (!(s->flags & HF_SVME_MASK) || !s->pe) {
+                goto illegal_op;
+            }
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_helper_invlpga(cpu_env, tcg_const_i32(s->aflag - 1));
+            break;
+
+        CASE_MODRM_MEM_OP(2): /* lgdt */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_WRITE);
+            gen_lea_modrm(env, s, modrm);
+            gen_op_ld_v(s, MO_16, cpu_T1, cpu_A0);
+            gen_add_A0_im(s, 2);
+            gen_op_ld_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
+            if (dflag == MO_16) {
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
+            }
+            tcg_gen_st_tl(cpu_T0, cpu_env, offsetof(CPUX86State, gdt.base));
+            tcg_gen_st32_tl(cpu_T1, cpu_env, offsetof(CPUX86State, gdt.limit));
+            break;
+
+        CASE_MODRM_MEM_OP(3): /* lidt */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_WRITE);
+            gen_lea_modrm(env, s, modrm);
+            gen_op_ld_v(s, MO_16, cpu_T1, cpu_A0);
+            gen_add_A0_im(s, 2);
+            gen_op_ld_v(s, CODE64(s) + MO_32, cpu_T0, cpu_A0);
+            if (dflag == MO_16) {
+                tcg_gen_andi_tl(cpu_T0, cpu_T0, 0xffffff);
+            }
+            tcg_gen_st_tl(cpu_T0, cpu_env, offsetof(CPUX86State, idt.base));
+            tcg_gen_st32_tl(cpu_T1, cpu_env, offsetof(CPUX86State, idt.limit));
+            break;
+
+        CASE_MODRM_OP(4): /* smsw */
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
+            tcg_gen_ld_tl(cpu_T0, cpu_env, offsetof(CPUX86State, cr[0]));
+            if (CODE64(s)) {
+                mod = (modrm >> 6) & 3;
+                ot = (mod != 3 ? MO_16 : s->dflag);
+            } else {
+                ot = MO_16;
+            }
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
+            break;
+        case 0xee: /* rdpkru */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_rdpkru(cpu_tmp1_i64, cpu_env, cpu_tmp2_i32);
+            tcg_gen_extr_i64_tl(cpu_regs[R_EAX], cpu_regs[R_EDX], cpu_tmp1_i64);
+            break;
+        case 0xef: /* wrpkru */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_regs[R_ECX]);
+            gen_helper_wrpkru(cpu_env, cpu_tmp2_i32, cpu_tmp1_i64);
+            break;
+        CASE_MODRM_OP(6): /* lmsw */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
+            gen_ldst_modrm(env, s, modrm, MO_16, OR_TMP0, 0);
+            gen_helper_lmsw(cpu_env, cpu_T0);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        CASE_MODRM_MEM_OP(7): /* invlpg */
+            if (s->cpl != 0) {
+                gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                break;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            gen_lea_modrm(env, s, modrm);
+            gen_helper_invlpg(cpu_env, cpu_A0);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        case 0xf8: /* swapgs */
+#ifdef TARGET_X86_64
+            if (CODE64(s)) {
                 if (s->cpl != 0) {
                     gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
                 } else {
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    gen_lea_modrm(env, s, modrm);
-                    gen_helper_invlpg(cpu_env, cpu_A0);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    gen_eob(s);
+                    tcg_gen_mov_tl(cpu_T0, cpu_seg_base[R_GS]);
+                    tcg_gen_ld_tl(cpu_seg_base[R_GS], cpu_env,
+                                  offsetof(CPUX86State, kernelgsbase));
+                    tcg_gen_st_tl(cpu_T0, cpu_env,
+                                  offsetof(CPUX86State, kernelgsbase));
                 }
-            } else {
-                switch (rm) {
-                case 0: /* swapgs */
-#ifdef TARGET_X86_64
-                    if (CODE64(s)) {
-                        if (s->cpl != 0) {
-                            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
-                        } else {
-                            tcg_gen_ld_tl(cpu_T[0], cpu_env,
-                                offsetof(CPUX86State,segs[R_GS].base));
-                            tcg_gen_ld_tl(cpu_T[1], cpu_env,
-                                offsetof(CPUX86State,kernelgsbase));
-                            tcg_gen_st_tl(cpu_T[1], cpu_env,
-                                offsetof(CPUX86State,segs[R_GS].base));
-                            tcg_gen_st_tl(cpu_T[0], cpu_env,
-                                offsetof(CPUX86State,kernelgsbase));
-                        }
-                    } else
+                break;
+            }
 #endif
-                    {
-                        goto illegal_op;
-                    }
-                    break;
-                case 1: /* rdtscp */
-                    if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP))
-                        goto illegal_op;
-                    gen_update_cc_op(s);
-                    gen_jmp_im(pc_start - s->cs_base);
-                    if (use_icount)
-                        gen_io_start();
-                    gen_helper_rdtscp(cpu_env);
-                    if (use_icount) {
-                        gen_io_end();
-                        gen_jmp(s, s->pc - s->cs_base);
-                    }
-                    break;
-                default:
-                    goto illegal_op;
-                }
+            goto illegal_op;
+
+        case 0xf9: /* rdtscp */
+            if (!(s->cpuid_ext2_features & CPUID_EXT2_RDTSCP)) {
+                goto illegal_op;
+            }
+            gen_update_cc_op(s);
+            gen_jmp_im(pc_start - s->cs_base);
+            if (s->tb->cflags & CF_USE_ICOUNT) {
+                gen_io_start();
+            }
+            gen_helper_rdtscp(cpu_env);
+            if (s->tb->cflags & CF_USE_ICOUNT) {
+                gen_io_end();
+                gen_jmp(s, s->pc - s->cs_base);
             }
             break;
+
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
+
     case 0x108: /* invd */
     case 0x109: /* wbinvd */
         if (s->cpl != 0) {
@@ -7489,21 +7433,21 @@
             rm = (modrm & 7) | REX_B(s);
 
             if (mod == 3) {
-                gen_op_mov_v_reg(MO_32, cpu_T[0], rm);
+                gen_op_mov_v_reg(MO_32, cpu_T0, rm);
                 /* sign extend */
                 if (d_ot == MO_64) {
-                    tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
+                    tcg_gen_ext32s_tl(cpu_T0, cpu_T0);
                 }
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             } else {
                 gen_lea_modrm(env, s, modrm);
-                gen_op_ld_v(s, MO_32 | MO_SIGN, cpu_T[0], cpu_A0);
-                gen_op_mov_reg_v(d_ot, reg, cpu_T[0]);
+                gen_op_ld_v(s, MO_32 | MO_SIGN, cpu_T0, cpu_A0);
+                gen_op_mov_reg_v(d_ot, reg, cpu_T0);
             }
         } else
 #endif
         {
-            int label1;
+            TCGLabel *label1;
             TCGv t0, t1, t2, a0;
 
             if (!s->pe || s->vm86)
@@ -7552,7 +7496,7 @@
     case 0x102: /* lar */
     case 0x103: /* lsl */
         {
-            int label1;
+            TCGLabel *label1;
             TCGv t0;
             if (!s->pe || s->vm86)
                 goto illegal_op;
@@ -7563,9 +7507,9 @@
             t0 = tcg_temp_local_new();
             gen_update_cc_op(s);
             if (b == 0x102) {
-                gen_helper_lar(t0, cpu_env, cpu_T[0]);
+                gen_helper_lar(t0, cpu_env, cpu_T0);
             } else {
-                gen_helper_lsl(t0, cpu_env, cpu_T[0]);
+                gen_helper_lsl(t0, cpu_env, cpu_T0);
             }
             tcg_gen_andi_tl(cpu_tmp0, cpu_cc_src, CC_Z);
             label1 = gen_new_label();
@@ -7587,7 +7531,7 @@
         case 3: /* prefetchnt0 */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(env, s, modrm);
+            gen_nop_modrm(env, s, modrm);
             /* nothing more to do */
             break;
         default: /* nop (multi byte) */
@@ -7595,7 +7539,199 @@
             break;
         }
         break;
-    case 0x119 ... 0x11f: /* nop (multi byte) */
+    case 0x11a:
+        modrm = cpu_ldub_code(env, s->pc++);
+        if (s->flags & HF_MPX_EN_MASK) {
+            mod = (modrm >> 6) & 3;
+            reg = ((modrm >> 3) & 7) | rex_r;
+            if (prefixes & PREFIX_REPZ) {
+                /* bndcl */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                gen_bndck(env, s, modrm, TCG_COND_LTU, cpu_bndl[reg]);
+            } else if (prefixes & PREFIX_REPNZ) {
+                /* bndcu */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                TCGv_i64 notu = tcg_temp_new_i64();
+                tcg_gen_not_i64(notu, cpu_bndu[reg]);
+                gen_bndck(env, s, modrm, TCG_COND_GTU, notu);
+                tcg_temp_free_i64(notu);
+            } else if (prefixes & PREFIX_DATA) {
+                /* bndmov -- from reg/mem */
+                if (reg >= 4 || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                if (mod == 3) {
+                    int reg2 = (modrm & 7) | REX_B(s);
+                    if (reg2 >= 4 || (prefixes & PREFIX_LOCK)) {
+                        goto illegal_op;
+                    }
+                    if (s->flags & HF_MPX_IU_MASK) {
+                        tcg_gen_mov_i64(cpu_bndl[reg], cpu_bndl[reg2]);
+                        tcg_gen_mov_i64(cpu_bndu[reg], cpu_bndu[reg2]);
+                    }
+                } else {
+                    gen_lea_modrm(env, s, modrm);
+                    if (CODE64(s)) {
+                        tcg_gen_qemu_ld_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 8);
+                        tcg_gen_qemu_ld_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                    } else {
+                        tcg_gen_qemu_ld_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 4);
+                        tcg_gen_qemu_ld_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                    }
+                    /* bnd registers are now in-use */
+                    gen_set_hflag(s, HF_MPX_IU_MASK);
+                }
+            } else if (mod != 3) {
+                /* bndldx */
+                AddressParts a = gen_lea_modrm_0(env, s, modrm);
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16
+                    || a.base < -1) {
+                    goto illegal_op;
+                }
+                if (a.base >= 0) {
+                    tcg_gen_addi_tl(cpu_A0, cpu_regs[a.base], a.disp);
+                } else {
+                    tcg_gen_movi_tl(cpu_A0, 0);
+                }
+                gen_lea_v_seg(s, s->aflag, cpu_A0, a.def_seg, s->override);
+                if (a.index >= 0) {
+                    tcg_gen_mov_tl(cpu_T0, cpu_regs[a.index]);
+                } else {
+                    tcg_gen_movi_tl(cpu_T0, 0);
+                }
+                if (CODE64(s)) {
+                    gen_helper_bndldx64(cpu_bndl[reg], cpu_env, cpu_A0, cpu_T0);
+                    tcg_gen_ld_i64(cpu_bndu[reg], cpu_env,
+                                   offsetof(CPUX86State, mmx_t0.MMX_Q(0)));
+                } else {
+                    gen_helper_bndldx32(cpu_bndu[reg], cpu_env, cpu_A0, cpu_T0);
+                    tcg_gen_ext32u_i64(cpu_bndl[reg], cpu_bndu[reg]);
+                    tcg_gen_shri_i64(cpu_bndu[reg], cpu_bndu[reg], 32);
+                }
+                gen_set_hflag(s, HF_MPX_IU_MASK);
+            }
+        }
+        gen_nop_modrm(env, s, modrm);
+        break;
+    case 0x11b:
+        modrm = cpu_ldub_code(env, s->pc++);
+        if (s->flags & HF_MPX_EN_MASK) {
+            mod = (modrm >> 6) & 3;
+            reg = ((modrm >> 3) & 7) | rex_r;
+            if (mod != 3 && (prefixes & PREFIX_REPZ)) {
+                /* bndmk */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                AddressParts a = gen_lea_modrm_0(env, s, modrm);
+                if (a.base >= 0) {
+                    tcg_gen_extu_tl_i64(cpu_bndl[reg], cpu_regs[a.base]);
+                    if (!CODE64(s)) {
+                        tcg_gen_ext32u_i64(cpu_bndl[reg], cpu_bndl[reg]);
+                    }
+                } else if (a.base == -1) {
+                    /* no base register has lower bound of 0 */
+                    tcg_gen_movi_i64(cpu_bndl[reg], 0);
+                } else {
+                    /* rip-relative generates #ud */
+                    goto illegal_op;
+                }
+                tcg_gen_not_tl(cpu_A0, gen_lea_modrm_1(a));
+                if (!CODE64(s)) {
+                    tcg_gen_ext32u_tl(cpu_A0, cpu_A0);
+                }
+                tcg_gen_extu_tl_i64(cpu_bndu[reg], cpu_A0);
+                /* bnd registers are now in-use */
+                gen_set_hflag(s, HF_MPX_IU_MASK);
+                break;
+            } else if (prefixes & PREFIX_REPNZ) {
+                /* bndcn */
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                gen_bndck(env, s, modrm, TCG_COND_GTU, cpu_bndu[reg]);
+            } else if (prefixes & PREFIX_DATA) {
+                /* bndmov -- to reg/mem */
+                if (reg >= 4 || s->aflag == MO_16) {
+                    goto illegal_op;
+                }
+                if (mod == 3) {
+                    int reg2 = (modrm & 7) | REX_B(s);
+                    if (reg2 >= 4 || (prefixes & PREFIX_LOCK)) {
+                        goto illegal_op;
+                    }
+                    if (s->flags & HF_MPX_IU_MASK) {
+                        tcg_gen_mov_i64(cpu_bndl[reg2], cpu_bndl[reg]);
+                        tcg_gen_mov_i64(cpu_bndu[reg2], cpu_bndu[reg]);
+                    }
+                } else {
+                    gen_lea_modrm(env, s, modrm);
+                    if (CODE64(s)) {
+                        tcg_gen_qemu_st_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 8);
+                        tcg_gen_qemu_st_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEQ);
+                    } else {
+                        tcg_gen_qemu_st_i64(cpu_bndl[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                        tcg_gen_addi_tl(cpu_A0, cpu_A0, 4);
+                        tcg_gen_qemu_st_i64(cpu_bndu[reg], cpu_A0,
+                                            s->mem_index, MO_LEUL);
+                    }
+                }
+            } else if (mod != 3) {
+                /* bndstx */
+                AddressParts a = gen_lea_modrm_0(env, s, modrm);
+                if (reg >= 4
+                    || (prefixes & PREFIX_LOCK)
+                    || s->aflag == MO_16
+                    || a.base < -1) {
+                    goto illegal_op;
+                }
+                if (a.base >= 0) {
+                    tcg_gen_addi_tl(cpu_A0, cpu_regs[a.base], a.disp);
+                } else {
+                    tcg_gen_movi_tl(cpu_A0, 0);
+                }
+                gen_lea_v_seg(s, s->aflag, cpu_A0, a.def_seg, s->override);
+                if (a.index >= 0) {
+                    tcg_gen_mov_tl(cpu_T0, cpu_regs[a.index]);
+                } else {
+                    tcg_gen_movi_tl(cpu_T0, 0);
+                }
+                if (CODE64(s)) {
+                    gen_helper_bndstx64(cpu_env, cpu_A0, cpu_T0,
+                                        cpu_bndl[reg], cpu_bndu[reg]);
+                } else {
+                    gen_helper_bndstx32(cpu_env, cpu_A0, cpu_T0,
+                                        cpu_bndl[reg], cpu_bndu[reg]);
+                }
+            }
+        }
+        gen_nop_modrm(env, s, modrm);
+        break;
+    case 0x119: case 0x11c ... 0x11f: /* nop (multi byte) */
         modrm = cpu_ldub_code(env, s->pc++);
         gen_nop_modrm(env, s, modrm);
         break;
@@ -7629,18 +7765,18 @@
                 gen_update_cc_op(s);
                 gen_jmp_im(pc_start - s->cs_base);
                 if (b & 2) {
-                    gen_op_mov_v_reg(ot, cpu_T[0], rm);
+                    gen_op_mov_v_reg(ot, cpu_T0, rm);
                     gen_helper_write_crN(cpu_env, tcg_const_i32(reg),
-                                         cpu_T[0]);
+                                         cpu_T0);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
                 } else {
-                    gen_helper_read_crN(cpu_T[0], cpu_env, tcg_const_i32(reg));
-                    gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                    gen_helper_read_crN(cpu_T0, cpu_env, tcg_const_i32(reg));
+                    gen_op_mov_reg_v(ot, rm, cpu_T0);
                 }
                 break;
             default:
-                goto illegal_op;
+                goto unknown_op;
             }
         }
         break;
@@ -7661,19 +7797,21 @@
                 ot = MO_64;
             else
                 ot = MO_32;
-            /* XXX: do it dynamically with CR4.DE bit */
-            if (reg == 4 || reg == 5 || reg >= 8)
+            if (reg >= 8) {
                 goto illegal_op;
+            }
             if (b & 2) {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_DR0 + reg);
-                gen_op_mov_v_reg(ot, cpu_T[0], rm);
-                gen_helper_movl_drN_T0(cpu_env, tcg_const_i32(reg), cpu_T[0]);
+                gen_op_mov_v_reg(ot, cpu_T0, rm);
+                tcg_gen_movi_i32(cpu_tmp2_i32, reg);
+                gen_helper_set_dr(cpu_env, cpu_tmp2_i32, cpu_T0);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_DR0 + reg);
-                tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,dr[reg]));
-                gen_op_mov_reg_v(ot, rm, cpu_T[0]);
+                tcg_gen_movi_i32(cpu_tmp2_i32, reg);
+                gen_helper_get_dr(cpu_T0, cpu_env, cpu_tmp2_i32);
+                gen_op_mov_reg_v(ot, rm, cpu_T0);
             }
         }
         break;
@@ -7703,83 +7841,200 @@
         break;
     case 0x1ae:
         modrm = cpu_ldub_code(env, s->pc++);
-        mod = (modrm >> 6) & 3;
-        op = (modrm >> 3) & 7;
-        switch(op) {
-        case 0: /* fxsave */
-            if (mod == 3 || !(s->cpuid_features & CPUID_FXSR) ||
-                (s->prefix & PREFIX_LOCK))
+        switch (modrm) {
+        CASE_MODRM_MEM_OP(0): /* fxsave */
+            if (!(s->cpuid_features & CPUID_FXSR)
+                || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
+            }
             if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) {
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_helper_fxsave(cpu_env, cpu_A0, tcg_const_i32(dflag == MO_64));
+            gen_helper_fxsave(cpu_env, cpu_A0);
             break;
-        case 1: /* fxrstor */
-            if (mod == 3 || !(s->cpuid_features & CPUID_FXSR) ||
-                (s->prefix & PREFIX_LOCK))
+
+        CASE_MODRM_MEM_OP(1): /* fxrstor */
+            if (!(s->cpuid_features & CPUID_FXSR)
+                || (prefixes & PREFIX_LOCK)) {
                 goto illegal_op;
+            }
             if ((s->flags & HF_EM_MASK) || (s->flags & HF_TS_MASK)) {
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
             gen_lea_modrm(env, s, modrm);
-            gen_update_cc_op(s);
-            gen_jmp_im(pc_start - s->cs_base);
-            gen_helper_fxrstor(cpu_env, cpu_A0, tcg_const_i32(dflag == MO_64));
+            gen_helper_fxrstor(cpu_env, cpu_A0);
             break;
-        case 2: /* ldmxcsr */
-        case 3: /* stmxcsr */
+
+        CASE_MODRM_MEM_OP(2): /* ldmxcsr */
+            if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK)) {
+                goto illegal_op;
+            }
             if (s->flags & HF_TS_MASK) {
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
-            if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK) ||
-                mod == 3)
-                goto illegal_op;
             gen_lea_modrm(env, s, modrm);
-            if (op == 2) {
-                tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0,
-                                    s->mem_index, MO_LEUL);
-                gen_helper_ldmxcsr(cpu_env, cpu_tmp2_i32);
+            tcg_gen_qemu_ld_i32(cpu_tmp2_i32, cpu_A0, s->mem_index, MO_LEUL);
+            gen_helper_ldmxcsr(cpu_env, cpu_tmp2_i32);
+            break;
+
+        CASE_MODRM_MEM_OP(3): /* stmxcsr */
+            if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK)) {
+                goto illegal_op;
+            }
+            if (s->flags & HF_TS_MASK) {
+                gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
+                break;
+            }
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_ld32u_tl(cpu_T0, cpu_env, offsetof(CPUX86State, mxcsr));
+            gen_op_st_v(s, MO_32, cpu_T0, cpu_A0);
+            break;
+
+        CASE_MODRM_MEM_OP(4): /* xsave */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (prefixes & (PREFIX_LOCK | PREFIX_DATA
+                                | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
+            }
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            gen_helper_xsave(cpu_env, cpu_A0, cpu_tmp1_i64);
+            break;
+
+        CASE_MODRM_MEM_OP(5): /* xrstor */
+            if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                || (prefixes & (PREFIX_LOCK | PREFIX_DATA
+                                | PREFIX_REPZ | PREFIX_REPNZ))) {
+                goto illegal_op;
+            }
+            gen_lea_modrm(env, s, modrm);
+            tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                  cpu_regs[R_EDX]);
+            gen_helper_xrstor(cpu_env, cpu_A0, cpu_tmp1_i64);
+            /* XRSTOR is how MPX is enabled, which changes how
+               we translate.  Thus we need to end the TB.  */
+            gen_update_cc_op(s);
+            gen_jmp_im(s->pc - s->cs_base);
+            gen_eob(s);
+            break;
+
+        CASE_MODRM_MEM_OP(6): /* xsaveopt / clwb */
+            if (prefixes & PREFIX_LOCK) {
+                goto illegal_op;
+            }
+            if (prefixes & PREFIX_DATA) {
+                /* clwb */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_CLWB)) {
+                    goto illegal_op;
+                }
+                gen_nop_modrm(env, s, modrm);
             } else {
-                tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, mxcsr));
-                gen_op_st_v(s, MO_32, cpu_T[0], cpu_A0);
+                /* xsaveopt */
+                if ((s->cpuid_ext_features & CPUID_EXT_XSAVE) == 0
+                    || (s->cpuid_xsave_features & CPUID_XSAVE_XSAVEOPT) == 0
+                    || (prefixes & (PREFIX_REPZ | PREFIX_REPNZ))) {
+                    goto illegal_op;
+                }
+                gen_lea_modrm(env, s, modrm);
+                tcg_gen_concat_tl_i64(cpu_tmp1_i64, cpu_regs[R_EAX],
+                                      cpu_regs[R_EDX]);
+                gen_helper_xsaveopt(cpu_env, cpu_A0, cpu_tmp1_i64);
             }
             break;
-        case 5: /* lfence */
-        case 6: /* mfence */
-            if ((modrm & 0xc7) != 0xc0 || !(s->cpuid_features & CPUID_SSE2))
+
+        CASE_MODRM_MEM_OP(7): /* clflush / clflushopt */
+            if (prefixes & PREFIX_LOCK) {
                 goto illegal_op;
-            break;
-        case 7: /* sfence / clflush */
-            if ((modrm & 0xc7) == 0xc0) {
-                /* sfence */
-                /* XXX: also check for cpuid_ext2_features & CPUID_EXT2_EMMX */
-                if (!(s->cpuid_features & CPUID_SSE))
+            }
+            if (prefixes & PREFIX_DATA) {
+                /* clflushopt */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_CLFLUSHOPT)) {
                     goto illegal_op;
+                }
             } else {
                 /* clflush */
-                if (!(s->cpuid_features & CPUID_CLFLUSH))
+                if ((s->prefix & (PREFIX_REPZ | PREFIX_REPNZ))
+                    || !(s->cpuid_features & CPUID_CLFLUSH)) {
                     goto illegal_op;
-                gen_lea_modrm(env, s, modrm);
+                }
+            }
+            gen_nop_modrm(env, s, modrm);
+            break;
+
+        case 0xc0 ... 0xc7: /* rdfsbase (f3 0f ae /0) */
+        case 0xc8 ... 0xc8: /* rdgsbase (f3 0f ae /1) */
+        case 0xd0 ... 0xd7: /* wrfsbase (f3 0f ae /2) */
+        case 0xd8 ... 0xd8: /* wrgsbase (f3 0f ae /3) */
+            if (CODE64(s)
+                && (prefixes & PREFIX_REPZ)
+                && !(prefixes & PREFIX_LOCK)
+                && (s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_FSGSBASE)) {
+                TCGv base, treg, src, dst;
+
+                /* Preserve hflags bits by testing CR4 at runtime.  */
+                tcg_gen_movi_i32(cpu_tmp2_i32, CR4_FSGSBASE_MASK);
+                gen_helper_cr4_testbit(cpu_env, cpu_tmp2_i32);
+
+                base = cpu_seg_base[modrm & 8 ? R_GS : R_FS];
+                treg = cpu_regs[(modrm & 7) | REX_B(s)];
+
+                if (modrm & 0x10) {
+                    /* wr*base */
+                    dst = base, src = treg;
+                } else {
+                    /* rd*base */
+                    dst = treg, src = base;
+                }
+
+                if (s->dflag == MO_32) {
+                    tcg_gen_ext32u_tl(dst, src);
+                } else {
+                    tcg_gen_mov_tl(dst, src);
+                }
+                break;
+            }
+            goto unknown_op;
+
+        case 0xf8: /* sfence / pcommit */
+            if (prefixes & PREFIX_DATA) {
+                /* pcommit */
+                if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_PCOMMIT)
+                    || (prefixes & PREFIX_LOCK)) {
+                    goto illegal_op;
+                }
+                break;
+            }
+            /* fallthru */
+        case 0xf9 ... 0xff: /* sfence */
+            if (!(s->cpuid_features & CPUID_SSE)
+                || (prefixes & PREFIX_LOCK)) {
+                goto illegal_op;
             }
             break;
+        case 0xe8 ... 0xef: /* lfence */
+        case 0xf0 ... 0xf7: /* mfence */
+            if (!(s->cpuid_features & CPUID_SSE2)
+                || (prefixes & PREFIX_LOCK)) {
+                goto illegal_op;
+            }
+            break;
+
         default:
-            goto illegal_op;
+            goto unknown_op;
         }
         break;
+
     case 0x10d: /* 3DNow! prefetch(w) */
         modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_lea_modrm(env, s, modrm);
-        /* ignore for now */
+        gen_nop_modrm(env, s, modrm);
         break;
     case 0x1aa: /* rsm */
         gen_svm_check_intercept(s, pc_start, SVM_EXIT_RSM);
@@ -7807,8 +8062,8 @@
         }
 
         gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
-        gen_helper_popcnt(cpu_T[0], cpu_env, cpu_T[0], tcg_const_i32(ot));
-        gen_op_mov_reg_v(ot, reg, cpu_T[0]);
+        gen_helper_popcnt(cpu_T0, cpu_env, cpu_T0, tcg_const_i32(ot));
+        gen_op_mov_reg_v(ot, reg, cpu_T0);
 
         set_cc_op(s, CC_OP_EFLAGS);
         break;
@@ -7826,7 +8081,7 @@
         gen_sse(env, s, b, pc_start, rex_r);
         break;
     default:
-        goto illegal_op;
+        goto unknown_op;
     }
     /* lock generation */
     if (s->prefix & PREFIX_LOCK)
@@ -7836,11 +8091,17 @@
     if (s->prefix & PREFIX_LOCK)
         gen_helper_unlock();
     /* XXX: ensure that no lock was generated */
-    gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
+    gen_illegal_opcode(s);
+    return s->pc;
+ unknown_op:
+    if (s->prefix & PREFIX_LOCK)
+        gen_helper_unlock();
+    /* XXX: ensure that no lock was generated */
+    gen_unknown_opcode(env, s);
     return s->pc;
 }
 
-void optimize_flags_init(void)
+void tcg_x86_init(void)
 {
     static const char reg_names[CPU_NB_REGS][4] = {
 #ifdef TARGET_X86_64
@@ -7871,39 +8132,74 @@
         [R_ESP] = "esp",
 #endif
     };
+    static const char seg_base_names[6][8] = {
+        [R_CS] = "cs_base",
+        [R_DS] = "ds_base",
+        [R_ES] = "es_base",
+        [R_FS] = "fs_base",
+        [R_GS] = "gs_base",
+        [R_SS] = "ss_base",
+    };
+    static const char bnd_regl_names[4][8] = {
+        "bnd0_lb", "bnd1_lb", "bnd2_lb", "bnd3_lb"
+    };
+    static const char bnd_regu_names[4][8] = {
+        "bnd0_ub", "bnd1_ub", "bnd2_ub", "bnd3_ub"
+    };
     int i;
+    static bool initialized;
+
+    if (initialized) {
+        return;
+    }
+    initialized = true;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_cc_op = tcg_global_mem_new_i32(TCG_AREG0,
+    tcg_ctx.tcg_env = cpu_env;
+    cpu_cc_op = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUX86State, cc_op), "cc_op");
-    cpu_cc_dst = tcg_global_mem_new(TCG_AREG0, offsetof(CPUX86State, cc_dst),
+    cpu_cc_dst = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_dst),
                                     "cc_dst");
-    cpu_cc_src = tcg_global_mem_new(TCG_AREG0, offsetof(CPUX86State, cc_src),
+    cpu_cc_src = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_src),
                                     "cc_src");
-    cpu_cc_src2 = tcg_global_mem_new(TCG_AREG0, offsetof(CPUX86State, cc_src2),
+    cpu_cc_src2 = tcg_global_mem_new(cpu_env, offsetof(CPUX86State, cc_src2),
                                      "cc_src2");
 
     for (i = 0; i < CPU_NB_REGS; ++i) {
-        cpu_regs[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_regs[i] = tcg_global_mem_new(cpu_env,
                                          offsetof(CPUX86State, regs[i]),
                                          reg_names[i]);
     }
+
+    for (i = 0; i < 6; ++i) {
+        cpu_seg_base[i]
+            = tcg_global_mem_new(cpu_env,
+                                 offsetof(CPUX86State, segs[i].base),
+                                 seg_base_names[i]);
+    }
+
+    for (i = 0; i < 4; ++i) {
+        cpu_bndl[i]
+            = tcg_global_mem_new_i64(cpu_env,
+                                     offsetof(CPUX86State, bnd_regs[i].lb),
+                                     bnd_regl_names[i]);
+        cpu_bndu[i]
+            = tcg_global_mem_new_i64(cpu_env,
+                                     offsetof(CPUX86State, bnd_regs[i].ub),
+                                     bnd_regu_names[i]);
+    }
+
+    helper_lock_init();
 }
 
-/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(X86CPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+/* generate intermediate code for basic block 'tb'.  */
+void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUX86State *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
     target_ulong pc_ptr;
-    CPUBreakpoint *bp;
-    int j, lj;
-    uint64_t flags;
+    uint32_t flags;
     target_ulong pc_start;
     target_ulong cs_base;
     int num_insns;
@@ -7931,33 +8227,41 @@
     dc->popl_esp_hack = 0;
     /* select memory access functions */
     dc->mem_index = 0;
-    if (flags & HF_SOFTMMU_MASK) {
-        dc->mem_index = cpu_mmu_index(env);
-    }
+#ifdef CONFIG_SOFTMMU
+    dc->mem_index = cpu_mmu_index(env, false);
+#endif
     dc->cpuid_features = env->features[FEAT_1_EDX];
     dc->cpuid_ext_features = env->features[FEAT_1_ECX];
     dc->cpuid_ext2_features = env->features[FEAT_8000_0001_EDX];
     dc->cpuid_ext3_features = env->features[FEAT_8000_0001_ECX];
     dc->cpuid_7_0_ebx_features = env->features[FEAT_7_0_EBX];
+    dc->cpuid_xsave_features = env->features[FEAT_XSAVE];
 #ifdef TARGET_X86_64
     dc->lma = (flags >> HF_LMA_SHIFT) & 1;
     dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
 #endif
     dc->flags = flags;
     dc->jmp_opt = !(dc->tf || cs->singlestep_enabled ||
-                    (flags & HF_INHIBIT_IRQ_MASK)
-#ifndef CONFIG_SOFTMMU
-                    || (flags & HF_SOFTMMU_MASK)
-#endif
-                    );
+                    (flags & HF_INHIBIT_IRQ_MASK));
+    /* Do not optimize repz jumps at all in icount mode, because
+       rep movsS instructions are execured with different paths
+       in !repz_opt and repz_opt modes. The first one was used
+       always except single step mode. And this setting
+       disables jumps optimization and control paths become
+       equivalent in run and single step modes.
+       Now there will be no jump optimization for repz in
+       record/replay modes and there will always be an
+       additional step for ecx=0 when icount is enabled.
+     */
+    dc->repz_opt = !dc->jmp_opt && !(tb->cflags & CF_USE_ICOUNT);
 #if 0
     /* check addseg logic */
     if (!dc->addseg && (dc->vm86 || !dc->pe || !dc->code32))
         printf("ERROR addseg\n");
 #endif
 
-    cpu_T[0] = tcg_temp_new();
-    cpu_T[1] = tcg_temp_new();
+    cpu_T0 = tcg_temp_new();
+    cpu_T1 = tcg_temp_new();
     cpu_A0 = tcg_temp_new();
 
     cpu_tmp0 = tcg_temp_new();
@@ -7971,47 +8275,50 @@
 
     dc->is_jmp = DISAS_NEXT;
     pc_ptr = pc_start;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     for(;;) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == pc_ptr &&
-                    !((bp->flags & BP_CPU) && (tb->flags & HF_RF_MASK))) {
-                    gen_debug(dc, pc_ptr - dc->cs_base);
-                    goto done_generating;
-                }
-            }
+        tcg_gen_insn_start(pc_ptr, dc->cc_op);
+        num_insns++;
+
+        /* If RF is set, suppress an internally generated breakpoint.  */
+        if (unlikely(cpu_breakpoint_test(cs, pc_ptr,
+                                         tb->flags & HF_RF_MASK
+                                         ? BP_GDB : BP_ANY))) {
+            gen_debug(dc, pc_ptr - dc->cs_base);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            pc_ptr += 1;
+            goto done_generating;
         }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = pc_ptr;
-            gen_opc_cc_op[lj] = dc->cc_op;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         pc_ptr = disas_insn(env, dc, pc_ptr);
-        num_insns++;
+        
 #ifdef CONFIG_HAX
         if (hax_enabled() && hax_stop_translate(cs)) {
+            /* When the host CPU doesn't support VMX "unrestricted guest" mode,
+             * TCG is used to execute MMIO instructions. This code path is used
+             * to limit the translation to a single machine instruction, in order
+             * to try to return to HAX execution as soon as possible. */
             gen_jmp_im(pc_ptr - dc->cs_base);
             gen_eob(dc);
             break;
         }
-#endif
+#endif /* CONFIG_HAX */
+
         /* stop translation if indicated */
         if (dc->is_jmp)
             break;
@@ -8026,6 +8333,20 @@
             gen_eob(dc);
             break;
         }
+        /* Do not cross the boundary of the pages in icount mode,
+           it can cause an exception. Do it only when boundary is
+           crossed by the first instruction in the block.
+           If current instruction already crossed the bound - it's ok,
+           because an exception hasn't stopped this code.
+         */
+        if ((tb->cflags & CF_USE_ICOUNT)
+            && ((pc_ptr & TARGET_PAGE_MASK)
+                != ((pc_ptr + TARGET_MAX_INSN_SIZE - 1) & TARGET_PAGE_MASK)
+                || (pc_ptr & ~TARGET_PAGE_MASK) == 0)) {
+            gen_jmp_im(pc_ptr - dc->cs_base);
+            gen_eob(dc);
+            break;
+        }
         /* if too long translation, stop generation too */
         if (tcg_op_buf_full() ||
             (pc_ptr - pc_start) >= (TARGET_PAGE_SIZE - 32) ||
@@ -8045,16 +8366,9 @@
 done_generating:
     gen_tb_end(tb, num_insns);
 
-    /* we don't forget to fill the last values */
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    }
-
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         int disas_flags;
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
@@ -8064,47 +8378,21 @@
         else
 #endif
             disas_flags = !dc->code32;
-        log_target_disas(env, pc_start, pc_ptr - pc_start, disas_flags);
+        log_target_disas(cs, pc_start, pc_ptr - pc_start, disas_flags);
         qemu_log("\n");
     }
 #endif
 
-    if (!search_pc) {
-        tb->size = pc_ptr - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = pc_ptr - pc_start;
+    tb->icount = num_insns;
 }
 
-void gen_intermediate_code(CPUX86State *env, TranslationBlock *tb)
+void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(x86_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUX86State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(x86_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUX86State *env, TranslationBlock *tb, int pc_pos)
-{
-    int cc_op;
-#ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_OP)) {
-        int i;
-        qemu_log("RESTORE:\n");
-        for(i = 0;i <= pc_pos; i++) {
-            if (tcg_ctx.gen_opc_instr_start[i]) {
-                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i,
-                        tcg_ctx.gen_opc_pc[i]);
-            }
-        }
-        qemu_log("pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
-                pc_pos, tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base,
-                (uint32_t)tb->cs_base);
-    }
-#endif
-    env->eip = tcg_ctx.gen_opc_pc[pc_pos] - tb->cs_base;
-    cc_op = gen_opc_cc_op[pc_pos];
-    if (cc_op != CC_OP_DYNAMIC)
+    int cc_op = data[1];
+    env->eip = data[0] - tb->cs_base;
+    if (cc_op != CC_OP_DYNAMIC) {
         env->cc_op = cc_op;
+    }
 }
diff --git a/target-lm32/cpu-qom.h b/target-lm32/cpu-qom.h
index 77bc7b2..b423d25 100644
--- a/target-lm32/cpu-qom.h
+++ b/target-lm32/cpu-qom.h
@@ -21,7 +21,6 @@
 #define QEMU_LM32_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #define TYPE_LM32_CPU "lm32-cpu"
 
@@ -48,45 +47,6 @@
     void (*parent_reset)(CPUState *cpu);
 } LM32CPUClass;
 
-/**
- * LM32CPU:
- * @env: #CPULM32State
- *
- * A LatticeMico32 CPU.
- */
-typedef struct LM32CPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPULM32State env;
-
-    uint32_t revision;
-    uint8_t num_interrupts;
-    uint8_t num_breakpoints;
-    uint8_t num_watchpoints;
-    uint32_t features;
-} LM32CPU;
-
-static inline LM32CPU *lm32_env_get_cpu(CPULM32State *env)
-{
-    return container_of(env, LM32CPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(lm32_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(LM32CPU, env)
-
-#ifndef CONFIG_USER_ONLY
-extern const struct VMStateDescription vmstate_lm32_cpu;
-#endif
-
-void lm32_cpu_do_interrupt(CPUState *cpu);
-bool lm32_cpu_exec_interrupt(CPUState *cs, int int_req);
-void lm32_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
-                         int flags);
-hwaddr lm32_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int lm32_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int lm32_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+typedef struct LM32CPU LM32CPU;
 
 #endif
diff --git a/target-lm32/cpu.c b/target-lm32/cpu.c
index f8081f5..a783d46 100644
--- a/target-lm32/cpu.c
+++ b/target-lm32/cpu.c
@@ -18,8 +18,11 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
+#include "exec/exec-all.h"
 
 
 static void lm32_cpu_set_pc(CPUState *cs, vaddr value)
@@ -131,6 +134,12 @@
     tlb_flush(s, 1);
 }
 
+static void lm32_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_lm32;
+    info->print_insn = print_insn_lm32;
+}
+
 static void lm32_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -151,7 +160,7 @@
     static bool tcg_initialized;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     env->flags = 0;
 
@@ -275,6 +284,14 @@
     cc->gdb_num_core_regs = 32 + 7;
     cc->gdb_stop_before_watchpoint = true;
     cc->debug_excp_handler = lm32_debug_excp_handler;
+    cc->disas_set_info = lm32_cpu_disas_set_info;
+
+    /*
+     * Reason: lm32_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void lm32_register_cpu_type(const LM32CPUInfo *info)
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index 0dab6e8..d8a3515 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -17,26 +17,22 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef CPU_LM32_H
-#define CPU_LM32_H
+#ifndef LM32_CPU_H
+#define LM32_CPU_H
 
 #define TARGET_LONG_BITS 32
 
 #define CPUArchState struct CPULM32State
 
-#include "config.h"
 #include "qemu-common.h"
+#include "cpu-qom.h"
 #include "exec/cpu-defs.h"
 struct CPULM32State;
 typedef struct CPULM32State CPULM32State;
 
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE EM_LATTICEMICO32
-
 #define NB_MMU_MODES 1
 #define TARGET_PAGE_BITS 12
-static inline int cpu_mmu_index(CPULM32State *env)
+static inline int cpu_mmu_index(CPULM32State *env, bool ifetch)
 {
     return 0;
 }
@@ -185,6 +181,47 @@
 
 };
 
+/**
+ * LM32CPU:
+ * @env: #CPULM32State
+ *
+ * A LatticeMico32 CPU.
+ */
+struct LM32CPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPULM32State env;
+
+    uint32_t revision;
+    uint8_t num_interrupts;
+    uint8_t num_breakpoints;
+    uint8_t num_watchpoints;
+    uint32_t features;
+};
+
+static inline LM32CPU *lm32_env_get_cpu(CPULM32State *env)
+{
+    return container_of(env, LM32CPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(lm32_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(LM32CPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_lm32_cpu;
+#endif
+
+void lm32_cpu_do_interrupt(CPUState *cpu);
+bool lm32_cpu_exec_interrupt(CPUState *cs, int int_req);
+void lm32_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
+                         int flags);
+hwaddr lm32_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int lm32_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int lm32_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+
 typedef enum {
     LM32_WP_DISABLED = 0,
     LM32_WP_READ,
@@ -198,10 +235,7 @@
     return (dc >> (idx+1)*2) & 0x3;
 }
 
-#include "cpu-qom.h"
-
 LM32CPU *cpu_lm32_init(const char *cpu_model);
-int cpu_lm32_exec(CPULM32State *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
@@ -219,18 +253,9 @@
 void lm32_watchpoint_remove(CPULM32State *env, int index);
 bool lm32_cpu_do_semihosting(CPUState *cs);
 
-static inline CPULM32State *cpu_init(const char *cpu_model)
-{
-    LM32CPU *cpu = cpu_lm32_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_lm32_init(cpu_model))
 
 #define cpu_list lm32_cpu_list
-#define cpu_exec cpu_lm32_exec
-#define cpu_gen_code cpu_lm32_gen_code
 #define cpu_signal_handler cpu_lm32_signal_handler
 
 int lm32_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
@@ -239,13 +264,11 @@
 #include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPULM32State *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->pc;
     *cs_base = 0;
     *flags = 0;
 }
 
-#include "exec/exec-all.h"
-
 #endif
diff --git a/target-lm32/gdbstub.c b/target-lm32/gdbstub.c
index 4979a98..cf929dd 100644
--- a/target-lm32/gdbstub.c
+++ b/target-lm32/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 #include "hw/lm32/lm32_pic.h"
 
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index 7a41f29..891da18 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -17,9 +17,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
+#include "exec/semihost.h"
+#include "exec/log.h"
 
 int lm32_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
                               int mmu_idx)
@@ -80,7 +84,7 @@
 
     switch (wp_type) {
     case LM32_WP_DISABLED:
-        /* nothing to to */
+        /* nothing to do */
         break;
     case LM32_WP_READ:
         flags = BP_CPU | BP_STOP_BEFORE_ACCESS | BP_MEM_READ;
@@ -137,7 +141,7 @@
             if (check_watchpoints(env)) {
                 raise_exception(env, EXCP_WATCHPOINT);
             } else {
-                cpu_resume_from_signal(cs, NULL);
+                cpu_loop_exit_noexc(cs);
             }
         }
     } else {
@@ -162,7 +166,7 @@
 
     switch (cs->exception_index) {
     case EXCP_SYSTEMCALL:
-        if (unlikely(semihosting_enabled)) {
+        if (unlikely(semihosting_enabled())) {
             /* do_semicall() returns true if call was handled. Otherwise
              * do the normal exception handling. */
             if (lm32_cpu_do_semihosting(cs)) {
diff --git a/target-lm32/lm32-semi.c b/target-lm32/lm32-semi.c
index ec6524f..70f8738 100644
--- a/target-lm32/lm32-semi.c
+++ b/target-lm32/lm32-semi.c
@@ -10,10 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <stddef.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/log.h"
@@ -187,7 +184,7 @@
         if (!p) {
             ret = -1;
         } else {
-            ret = stat(p, &s);
+            ret = qemu_stat(p, &s);
             unlock_user(p, arg0, 0);
             if (translate_stat(env, arg1, &s) == 0) {
                 ret = -1;
diff --git a/target-lm32/machine.c b/target-lm32/machine.c
index 8327c6d..3c258a4 100644
--- a/target-lm32/machine.c
+++ b/target-lm32/machine.c
@@ -1,5 +1,9 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "migration/cpu.h"
 
 static const VMStateDescription vmstate_env = {
     .name = "env",
diff --git a/target-lm32/op_helper.c b/target-lm32/op_helper.c
index 61209c1..2177c8a 100644
--- a/target-lm32/op_helper.c
+++ b/target-lm32/op_helper.c
@@ -1,4 +1,4 @@
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -6,6 +6,7 @@
 #include "hw/lm32/lm32_pic.h"
 #include "hw/char/lm32_juart.h"
 
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
@@ -143,12 +144,12 @@
  * NULL, it means that the function was called in C code (i.e. not
  * from generated code or from helper.c)
  */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = lm32_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = lm32_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index 79b67d1..2d8caeb 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -17,9 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 
 #include "exec/cpu_ldst.h"
@@ -28,6 +30,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define DISAS_LM32 1
@@ -42,7 +45,7 @@
 
 #define MEM_INDEX 0
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_R[32];
 static TCGv cpu_pc;
 static TCGv cpu_ie;
@@ -131,16 +134,25 @@
     gen_helper_ill(cpu_env);
 }
 
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+    if (unlikely(dc->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-
-    tb = dc->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-            likely(!dc->singlestep_enabled)) {
+    if (use_goto_tb(dc, dest)) {
         tcg_gen_goto_tb(n);
         tcg_gen_movi_tl(cpu_pc, dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)dc->tb + n);
     } else {
         tcg_gen_movi_tl(cpu_pc, dest);
         if (dc->singlestep_enabled) {
@@ -219,7 +231,7 @@
     /* restore IE.IE in case of an eret */
     if (dc->r0 == R_EA) {
         TCGv t0 = tcg_temp_new();
-        int l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
         tcg_gen_andi_tl(t0, cpu_ie, IE_EIE);
         tcg_gen_ori_tl(cpu_ie, cpu_ie, IE_IE);
         tcg_gen_brcondi_tl(TCG_COND_EQ, t0, IE_EIE, l1);
@@ -228,7 +240,7 @@
         tcg_temp_free(t0);
     } else if (dc->r0 == R_BA) {
         TCGv t0 = tcg_temp_new();
-        int l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
         tcg_gen_andi_tl(t0, cpu_ie, IE_BIE);
         tcg_gen_ori_tl(cpu_ie, cpu_ie, IE_IE);
         tcg_gen_brcondi_tl(TCG_COND_EQ, t0, IE_BIE, l1);
@@ -252,9 +264,7 @@
 
 static inline void gen_cond_branch(DisasContext *dc, int cond)
 {
-    int l1;
-
-    l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     tcg_gen_brcond_tl(cond, cpu_R[dc->r0], cpu_R[dc->r1], l1);
     gen_goto_tb(dc, 0, dc->pc + 4);
     gen_set_label(l1);
@@ -428,7 +438,7 @@
 
 static void dec_divu(DisasContext *dc)
 {
-    int l1;
+    TCGLabel *l1;
 
     LOG_DIS("divu r%d, r%d, r%d\n", dc->r2, dc->r0, dc->r1);
 
@@ -508,7 +518,7 @@
 
 static void dec_modu(DisasContext *dc)
 {
-    int l1;
+    TCGLabel *l1;
 
     LOG_DIS("modu r%d, r%d, %d\n", dc->r2, dc->r0, dc->r1);
 
@@ -769,8 +779,8 @@
         }
         tcg_gen_sari_tl(cpu_R[dc->r1], cpu_R[dc->r0], dc->imm5);
     } else {
-        int l1 = gen_new_label();
-        int l2 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
+        TCGLabel *l2 = gen_new_label();
         TCGv t0 = tcg_temp_local_new();
         tcg_gen_andi_tl(t0, cpu_R[dc->r1], 0x1f);
 
@@ -805,8 +815,8 @@
         }
         tcg_gen_shri_tl(cpu_R[dc->r1], cpu_R[dc->r0], dc->imm5);
     } else {
-        int l1 = gen_new_label();
-        int l2 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
+        TCGLabel *l2 = gen_new_label();
         TCGv t0 = tcg_temp_local_new();
         tcg_gen_andi_tl(t0, cpu_R[dc->r1], 0x1f);
 
@@ -865,24 +875,24 @@
         break;
     case CSR_IM:
         /* mark as an io operation because it could cause an interrupt */
-        if (use_icount) {
+        if (dc->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
         }
         gen_helper_wcsr_im(cpu_env, cpu_R[dc->r1]);
         tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
-        if (use_icount) {
+        if (dc->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
         }
         dc->is_jmp = DISAS_UPDATE;
         break;
     case CSR_IP:
         /* mark as an io operation because it could cause an interrupt */
-        if (use_icount) {
+        if (dc->tb->cflags & CF_USE_ICOUNT) {
             gen_io_start();
         }
         gen_helper_wcsr_ip(cpu_env, cpu_R[dc->r1]);
         tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
-        if (use_icount) {
+        if (dc->tb->cflags & CF_USE_ICOUNT) {
             gen_io_end();
         }
         dc->is_jmp = DISAS_UPDATE;
@@ -1007,10 +1017,6 @@
 
 static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
@@ -1038,32 +1044,13 @@
     decinfo[dc->opcode](dc);
 }
 
-static void check_breakpoint(CPULM32State *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(lm32_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_tl(cpu_pc, dc->pc);
-                t_gen_raise_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 /* generate intermediate code for basic block 'tb'.  */
-static inline
-void gen_intermediate_code_internal(LM32CPU *cpu,
-                                    TranslationBlock *tb, bool search_pc)
+void gen_intermediate_code(CPULM32State *env, struct TranslationBlock *tb)
 {
+    LM32CPU *cpu = lm32_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPULM32State *env = &cpu->env;
     struct DisasContext ctx, *dc = &ctx;
     uint32_t pc_start;
-    int j, lj;
     uint32_t next_page_start;
     int num_insns;
     int max_insns;
@@ -1085,41 +1072,41 @@
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do {
-        check_breakpoint(env, dc);
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            t_gen_raise_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
         decode(dc, cpu_ldl_code(env, dc->pc));
         dc->pc += 4;
-        num_insns++;
-
     } while (!dc->is_jmp
          && !tcg_op_buf_full()
          && !cs->singlestep_enabled
@@ -1156,37 +1143,20 @@
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("\n");
-        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
+        log_target_disas(cs, pc_start, dc->pc - pc_start, 0);
         qemu_log("\nisize=%d osize=%d\n",
                  dc->pc - pc_start, tcg_op_buf_count());
     }
 #endif
 }
 
-void gen_intermediate_code(CPULM32State *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(lm32_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPULM32State *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(lm32_env_get_cpu(env), tb, true);
-}
-
 void lm32_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -1221,9 +1191,10 @@
     cpu_fprintf(f, "\n\n");
 }
 
-void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPULM32State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
 
 void lm32_translate_init(void)
@@ -1231,50 +1202,51 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
     for (i = 0; i < ARRAY_SIZE(cpu_R); i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPULM32State, regs[i]),
                           regnames[i]);
     }
 
     for (i = 0; i < ARRAY_SIZE(cpu_bp); i++) {
-        cpu_bp[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_bp[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPULM32State, bp[i]),
                           regnames[32+i]);
     }
 
     for (i = 0; i < ARRAY_SIZE(cpu_wp); i++) {
-        cpu_wp[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_wp[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPULM32State, wp[i]),
                           regnames[36+i]);
     }
 
-    cpu_pc = tcg_global_mem_new(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, pc),
                     "pc");
-    cpu_ie = tcg_global_mem_new(TCG_AREG0,
+    cpu_ie = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, ie),
                     "ie");
-    cpu_icc = tcg_global_mem_new(TCG_AREG0,
+    cpu_icc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, icc),
                     "icc");
-    cpu_dcc = tcg_global_mem_new(TCG_AREG0,
+    cpu_dcc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, dcc),
                     "dcc");
-    cpu_cc = tcg_global_mem_new(TCG_AREG0,
+    cpu_cc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, cc),
                     "cc");
-    cpu_cfg = tcg_global_mem_new(TCG_AREG0,
+    cpu_cfg = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, cfg),
                     "cfg");
-    cpu_eba = tcg_global_mem_new(TCG_AREG0,
+    cpu_eba = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, eba),
                     "eba");
-    cpu_dc = tcg_global_mem_new(TCG_AREG0,
+    cpu_dc = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, dc),
                     "dc");
-    cpu_deba = tcg_global_mem_new(TCG_AREG0,
+    cpu_deba = tcg_global_mem_new(cpu_env,
                     offsetof(CPULM32State, deba),
                     "deba");
 }
diff --git a/target-m68k/cpu-qom.h b/target-m68k/cpu-qom.h
index c28e55d..9885bba 100644
--- a/target-m68k/cpu-qom.h
+++ b/target-m68k/cpu-qom.h
@@ -47,38 +47,6 @@
     void (*parent_reset)(CPUState *cpu);
 } M68kCPUClass;
 
-/**
- * M68kCPU:
- * @env: #CPUM68KState
- *
- * A Motorola 68k CPU.
- */
-typedef struct M68kCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUM68KState env;
-} M68kCPU;
-
-static inline M68kCPU *m68k_env_get_cpu(CPUM68KState *env)
-{
-    return container_of(env, M68kCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(m68k_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(M68kCPU, env)
-
-void m68k_cpu_do_interrupt(CPUState *cpu);
-bool m68k_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void m68k_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
-                         int flags);
-hwaddr m68k_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int m68k_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int m68k_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-
-void m68k_cpu_exec_enter(CPUState *cs);
-void m68k_cpu_exec_exit(CPUState *cs);
+typedef struct M68kCPU M68kCPU;
 
 #endif
diff --git a/target-m68k/cpu.c b/target-m68k/cpu.c
index 4cfb725..116b784 100644
--- a/target-m68k/cpu.c
+++ b/target-m68k/cpu.c
@@ -18,9 +18,12 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
+#include "exec/exec-all.h"
 
 
 static void m68k_cpu_set_pc(CPUState *cs, vaddr value)
@@ -61,6 +64,11 @@
     tlb_flush(s, 1);
 }
 
+static void m68k_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->print_insn = print_insn_m68k;
+}
+
 /* CPU models */
 
 static ObjectClass *m68k_cpu_class_by_name(const char *cpu_model)
@@ -168,7 +176,7 @@
     static bool inited;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     if (tcg_enabled() && !inited) {
         inited = true;
@@ -208,10 +216,19 @@
 #endif
     cc->cpu_exec_enter = m68k_cpu_exec_enter;
     cc->cpu_exec_exit = m68k_cpu_exec_exit;
+    cc->disas_set_info = m68k_cpu_disas_set_info;
 
-    dc->vmsd = &vmstate_m68k_cpu;
     cc->gdb_num_core_regs = 18;
     cc->gdb_core_xml_file = "cf-core.xml";
+
+    dc->vmsd = &vmstate_m68k_cpu;
+
+    /*
+     * Reason: m68k_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void register_cpu_type(const M68kCPUInfo *info)
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index f67bbcc..b2faa6b 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -17,25 +17,21 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef CPU_M68K_H
-#define CPU_M68K_H
+
+#ifndef M68K_CPU_H
+#define M68K_CPU_H
 
 #define TARGET_LONG_BITS 32
 
 #define CPUArchState struct CPUM68KState
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
-
+#include "cpu-qom.h"
 #include "fpu/softfloat.h"
 
 #define MAX_QREGS 32
 
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE	EM_68K
-
 #define EXCP_ACCESS         2   /* Access (MMU) error.  */
 #define EXCP_ADDRESS        3   /* Address error.  */
 #define EXCP_ILLEGAL        4   /* Illegal instruction.  */
@@ -114,12 +110,43 @@
     uint32_t features;
 } CPUM68KState;
 
-#include "cpu-qom.h"
+/**
+ * M68kCPU:
+ * @env: #CPUM68KState
+ *
+ * A Motorola 68k CPU.
+ */
+struct M68kCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUM68KState env;
+};
+
+static inline M68kCPU *m68k_env_get_cpu(CPUM68KState *env)
+{
+    return container_of(env, M68kCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(m68k_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(M68kCPU, env)
+
+void m68k_cpu_do_interrupt(CPUState *cpu);
+bool m68k_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void m68k_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
+                         int flags);
+hwaddr m68k_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int m68k_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int m68k_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+
+void m68k_cpu_exec_enter(CPUState *cs);
+void m68k_cpu_exec_exit(CPUState *cs);
 
 void m68k_tcg_init(void);
 void m68k_cpu_init_gdb(M68kCPU *cpu);
 M68kCPU *cpu_m68k_init(const char *cpu_model);
-int cpu_m68k_exec(CPUM68KState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
@@ -214,17 +241,8 @@
 #define TARGET_PHYS_ADDR_SPACE_BITS 32
 #define TARGET_VIRT_ADDR_SPACE_BITS 32
 
-static inline CPUM68KState *cpu_init(const char *cpu_model)
-{
-    M68kCPU *cpu = cpu_m68k_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_m68k_init(cpu_model))
 
-#define cpu_exec cpu_m68k_exec
-#define cpu_gen_code cpu_m68k_gen_code
 #define cpu_signal_handler cpu_m68k_signal_handler
 #define cpu_list m68k_cpu_list
 
@@ -232,7 +250,7 @@
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUM68KState *env)
+static inline int cpu_mmu_index (CPUM68KState *env, bool ifetch)
 {
     return (env->sr & SR_S) == 0 ? 1 : 0;
 }
@@ -243,7 +261,7 @@
 #include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->pc;
     *cs_base = 0;
@@ -252,6 +270,4 @@
             | ((env->macsr >> 4) & 0xf);        /* Bits 0-3 */
 }
 
-#include "exec/exec-all.h"
-
 #endif
diff --git a/target-m68k/gdbstub.c b/target-m68k/gdbstub.c
index ae8179c..c7f44c9 100644
--- a/target-m68k/gdbstub.c
+++ b/target-m68k/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int m68k_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index 77225a2..f52d0e3 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/gdbstub.h"
 
 #include "exec/helper-proto.h"
@@ -556,10 +558,10 @@
     /* ??? Should flush denormals to zero.  */
     float64 res;
     res = float64_sub(a, b, &env->fp_status);
-    if (float64_is_quiet_nan(res)) {
+    if (float64_is_quiet_nan(res, &env->fp_status)) {
         /* +/-inf compares equal against itself, but sub returns nan.  */
-        if (!float64_is_quiet_nan(a)
-            && !float64_is_quiet_nan(b)) {
+        if (!float64_is_quiet_nan(a, &env->fp_status)
+            && !float64_is_quiet_nan(b, &env->fp_status)) {
             res = float64_zero;
             if (float64_lt_quiet(a, res, &env->fp_status))
                 res = float64_chs(res);
diff --git a/target-m68k/m68k-qreg.h b/target-m68k/m68k-qreg.h
deleted file mode 100644
index c224d5e..0000000
--- a/target-m68k/m68k-qreg.h
+++ /dev/null
@@ -1,11 +0,0 @@
-enum {
-#define DEFO32(name, offset) QREG_##name,
-#define DEFR(name, reg, mode) QREG_##name,
-#define DEFF64(name, offset) QREG_##name,
-    QREG_NULL,
-#include "qregs.def"
-    TARGET_NUM_QREGS = 0x100
-#undef DEFO32
-#undef DEFR
-#undef DEFF64
-};
diff --git a/target-m68k/m68k-semi.c b/target-m68k/m68k-semi.c
index 9dffe8d..14912a6 100644
--- a/target-m68k/m68k-semi.c
+++ b/target-m68k/m68k-semi.c
@@ -17,15 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <sys/time.h>
-#include <time.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #if defined(CONFIG_USER_ONLY)
@@ -36,6 +28,7 @@
 #include "exec/gdbstub.h"
 #include "exec/softmmu-semi.h"
 #endif
+#include "qemu/log.h"
 #include "sysemu/sysemu.h"
 
 #define HOSTED_EXIT  0
@@ -351,7 +344,7 @@
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = stat(p, &s);
+                result = qemu_stat(p, &s);
                 unlock_user(p, arg0, 0);
             }
             if (result == 0) {
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index 06661f5..e41ae46 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -16,9 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "exec/semihost.h"
 
 #if defined(CONFIG_USER_ONLY)
 
@@ -33,17 +36,15 @@
 
 #else
 
-extern int semihosting_enabled;
-
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = m68k_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = m68k_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
@@ -63,8 +64,8 @@
     env->pc = cpu_ldl_kernel(env, sp + 4);
     sp |= (fmt >> 28) & 3;
     env->sr = fmt & 0xffff;
-    m68k_switch_sp(env);
     env->aregs[7] = sp + 8;
+    m68k_switch_sp(env);
 }
 
 static void do_interrupt_all(CPUM68KState *env, int is_hw)
@@ -85,7 +86,7 @@
             do_rte(env);
             return;
         case EXCP_HALT_INSN:
-            if (semihosting_enabled
+            if (semihosting_enabled()
                     && (env->sr & SR_S) != 0
                     && (env->pc & 3) == 0
                     && cpu_lduw_code(env, env->pc - 4) == 0x4e71
@@ -108,10 +109,7 @@
 
     vector = cs->exception_index << 2;
 
-    sp = env->aregs[7];
-
     fmt |= 0x40000000;
-    fmt |= (sp & 3) << 28;
     fmt |= vector << 16;
     fmt |= env->sr;
 
@@ -121,6 +119,8 @@
         env->sr &= ~SR_M;
     }
     m68k_switch_sp(env);
+    sp = env->aregs[7];
+    fmt |= (sp & 3) << 28;
 
     /* ??? This could cause MMU faults.  */
     sp &= ~3;
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index 5c2f4d0..ecd5e5c 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -18,8 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "exec/cpu_ldst.h"
@@ -28,6 +30,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 //#define DEBUG_DISPATCH 1
@@ -48,7 +51,7 @@
 static TCGv_i32 cpu_halted;
 static TCGv_i32 cpu_exception_index;
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 
 static char cpu_reg_names[3*8*3 + 5*4];
 static TCGv cpu_dregs[8];
@@ -74,48 +77,53 @@
     char *p;
     int i;
 
-#define DEFO32(name,  offset) QREG_##name = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUM68KState, offset), #name);
-#define DEFO64(name,  offset) QREG_##name = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUM68KState, offset), #name);
-#define DEFF64(name,  offset) DEFO64(name, offset)
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
+
+#define DEFO32(name, offset) \
+    QREG_##name = tcg_global_mem_new_i32(cpu_env, \
+        offsetof(CPUM68KState, offset), #name);
+#define DEFO64(name, offset) \
+    QREG_##name = tcg_global_mem_new_i64(cpu_env, \
+        offsetof(CPUM68KState, offset), #name);
+#define DEFF64(name, offset) DEFO64(name, offset)
 #include "qregs.def"
 #undef DEFO32
 #undef DEFO64
 #undef DEFF64
 
-    cpu_halted = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_halted = tcg_global_mem_new_i32(cpu_env,
                                         -offsetof(M68kCPU, env) +
                                         offsetof(CPUState, halted), "HALTED");
-    cpu_exception_index = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_exception_index = tcg_global_mem_new_i32(cpu_env,
                                                  -offsetof(M68kCPU, env) +
                                                  offsetof(CPUState, exception_index),
                                                  "EXCEPTION");
 
-    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-
     p = cpu_reg_names;
     for (i = 0; i < 8; i++) {
         sprintf(p, "D%d", i);
-        cpu_dregs[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_dregs[i] = tcg_global_mem_new(cpu_env,
                                           offsetof(CPUM68KState, dregs[i]), p);
         p += 3;
         sprintf(p, "A%d", i);
-        cpu_aregs[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_aregs[i] = tcg_global_mem_new(cpu_env,
                                           offsetof(CPUM68KState, aregs[i]), p);
         p += 3;
         sprintf(p, "F%d", i);
-        cpu_fregs[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_fregs[i] = tcg_global_mem_new_i64(cpu_env,
                                           offsetof(CPUM68KState, fregs[i]), p);
         p += 3;
     }
     for (i = 0; i < 4; i++) {
         sprintf(p, "ACC%d", i);
-        cpu_macc[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_macc[i] = tcg_global_mem_new_i64(cpu_env,
                                          offsetof(CPUM68KState, macc[i]), p);
         p += 5;
     }
 
-    NULL_QREG = tcg_global_mem_new(TCG_AREG0, -4, "NULL");
-    store_dummy = tcg_global_mem_new(TCG_AREG0, -8, "NULL");
+    NULL_QREG = tcg_global_mem_new(cpu_env, -4, "NULL");
+    store_dummy = tcg_global_mem_new(cpu_env, -8, "NULL");
 }
 
 /* internal defines */
@@ -129,7 +137,6 @@
     uint32_t fpcr;
     struct TranslationBlock *tb;
     int singlestep_enabled;
-    int is_mem;
     TCGv_i64 mactmp;
     int done_mac;
 } DisasContext;
@@ -179,7 +186,6 @@
 {
     TCGv tmp;
     int index = IS_USER(s);
-    s->is_mem = 1;
     tmp = tcg_temp_new_i32();
     switch(opsize) {
     case OS_BYTE:
@@ -209,7 +215,6 @@
 {
     TCGv_i64 tmp;
     int index = IS_USER(s);
-    s->is_mem = 1;
     tmp = tcg_temp_new_i64();
     tcg_gen_qemu_ldf64(tmp, addr, index);
     gen_throws_exception = gen_last_qop;
@@ -220,7 +225,6 @@
 static inline void gen_store(DisasContext *s, int opsize, TCGv addr, TCGv val)
 {
     int index = IS_USER(s);
-    s->is_mem = 1;
     switch(opsize) {
     case OS_BYTE:
         tcg_gen_qemu_st8(val, addr, index);
@@ -241,7 +245,6 @@
 static inline void gen_store64(DisasContext *s, TCGv addr, TCGv_i64 val)
 {
     int index = IS_USER(s);
-    s->is_mem = 1;
     tcg_gen_qemu_stf64(val, addr, index);
     gen_throws_exception = gen_last_qop;
 }
@@ -297,8 +300,7 @@
 
 /* Handle a base + index + displacement effective addresss.
    A NULL_QREG base means pc-relative.  */
-static TCGv gen_lea_indexed(CPUM68KState *env, DisasContext *s, int opsize,
-                            TCGv base)
+static TCGv gen_lea_indexed(CPUM68KState *env, DisasContext *s, TCGv base)
 {
     uint32_t offset;
     uint16_t ext;
@@ -529,7 +531,7 @@
         return tmp;
     case 6: /* Indirect index + displacement.  */
         reg = AREG(insn, 0);
-        return gen_lea_indexed(env, s, opsize, reg);
+        return gen_lea_indexed(env, s, reg);
     case 7: /* Other */
         switch (insn & 7) {
         case 0: /* Absolute short.  */
@@ -545,7 +547,7 @@
             s->pc += 2;
             return tcg_const_i32(offset);
         case 3: /* pc index+displacement.  */
-            return gen_lea_indexed(env, s, opsize, NULL_QREG);
+            return gen_lea_indexed(env, s, NULL_QREG);
         case 4: /* Immediate.  */
         default:
             return NULL_QREG;
@@ -679,7 +681,7 @@
 }
 
 /* This generates a conditional branch, clobbering all temporaries.  */
-static void gen_jmpcc(DisasContext *s, int cond, int l1)
+static void gen_jmpcc(DisasContext *s, int cond, TCGLabel *l1)
 {
     TCGv tmp;
 
@@ -784,7 +786,7 @@
 
 DISAS_INSN(scc)
 {
-    int l1;
+    TCGLabel *l1;
     int cond;
     TCGv reg;
 
@@ -852,19 +854,25 @@
         }                                                               \
     } while (0)
 
+static inline bool use_goto_tb(DisasContext *s, uint32_t dest)
+{
+#ifndef CONFIG_USER_ONLY
+    return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
+           (s->insn_pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 /* Generate a jump to an immediate address.  */
 static void gen_jmp_tb(DisasContext *s, int n, uint32_t dest)
 {
-    TranslationBlock *tb;
-
-    tb = s->tb;
     if (unlikely(s->singlestep_enabled)) {
         gen_exception(s, dest, EXCP_DEBUG);
-    } else if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) ||
-               (s->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+    } else if (use_goto_tb(s, dest)) {
         tcg_gen_goto_tb(n);
         tcg_gen_movi_i32(QREG_PC, dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)s->tb + n);
     } else {
         gen_jmp_im(s, dest);
         tcg_gen_exit_tb(0);
@@ -1658,7 +1666,7 @@
     int32_t offset;
     uint32_t base;
     int op;
-    int l1;
+    TCGLabel *l1;
 
     base = s->pc;
     op = (insn >> 8) & 0xf;
@@ -1995,8 +2003,8 @@
         gen_exception(s, s->pc - 2, EXCP_PRIVILEGE);
         return;
     }
-    /* TODO: Implement USP.  */
-    gen_exception(s, s->pc - 2, EXCP_ILLEGAL);
+    tcg_gen_ld_i32(AREG(insn, 0), cpu_env,
+                   offsetof(CPUM68KState, sp[M68K_USP]));
 }
 
 DISAS_INSN(move_to_usp)
@@ -2005,8 +2013,8 @@
         gen_exception(s, s->pc - 2, EXCP_PRIVILEGE);
         return;
     }
-    /* TODO: Implement USP.  */
-    gen_exception(s, s->pc - 2, EXCP_ILLEGAL);
+    tcg_gen_st_i32(AREG(insn, 0), cpu_env,
+                   offsetof(CPUM68KState, sp[M68K_USP]));
 }
 
 DISAS_INSN(halt)
@@ -2227,7 +2235,6 @@
             mask = 0x80;
             for (i = 0; i < 8; i++) {
                 if (ext & mask) {
-                    s->is_mem = 1;
                     dest = FREG(i, 0);
                     if (ext & (1 << 13)) {
                         /* store */
@@ -2395,7 +2402,7 @@
     uint32_t offset;
     uint32_t addr;
     TCGv flag;
-    int l1;
+    TCGLabel *l1;
 
     addr = s->pc;
     offset = cpu_ldsw_code(env, s->pc);
@@ -2687,7 +2694,7 @@
     if (s->env->macsr & MACSR_FI) {
         gen_helper_get_macf(rx, cpu_env, acc);
     } else if ((s->env->macsr & MACSR_OMC) == 0) {
-        tcg_gen_trunc_i64_i32(rx, acc);
+        tcg_gen_extrl_i64_i32(rx, acc);
     } else if (s->env->macsr & MACSR_SU) {
         gen_helper_get_macs(rx, acc);
     } else {
@@ -2962,10 +2969,6 @@
 {
     uint16_t insn;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(s->pc);
-    }
-
     insn = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
@@ -2973,15 +2976,11 @@
 }
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(M68kCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUM68KState *env, TranslationBlock *tb)
 {
+    M68kCPU *cpu = m68k_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUM68KState *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     int pc_offset;
     int num_insns;
@@ -2999,45 +2998,40 @@
     dc->singlestep_enabled = cs->singlestep_enabled;
     dc->fpcr = env->fpcr;
     dc->user = (env->sr & SR_S) == 0;
-    dc->is_mem = 0;
     dc->done_mac = 0;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do {
         pc_offset = dc->pc - pc_start;
         gen_throws_exception = NULL;
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    gen_exception(dc, dc->pc, EXCP_DEBUG);
-                    dc->is_jmp = DISAS_JUMP;
-                    break;
-                }
-            }
-            if (dc->is_jmp)
-                break;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            gen_exception(dc, dc->pc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_JUMP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 2;
+            break;
         }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
+
         dc->insn_pc = dc->pc;
 	disas_m68k_insn(env, dc);
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -3074,35 +3068,16 @@
     gen_tb_end(tb, num_insns);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
+        log_target_disas(cs, pc_start, dc->pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-
-    //optimize_flags();
-    //expand_target_qops();
-}
-
-void gen_intermediate_code(CPUM68KState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(m68k_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUM68KState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(m68k_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 void m68k_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
@@ -3128,7 +3103,8 @@
     cpu_fprintf (f, "FPRESULT = %12g\n", *(double *)&env->fp_result);
 }
 
-void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUM68KState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-microblaze/cpu-qom.h b/target-microblaze/cpu-qom.h
index e3e0701..1a61db7 100644
--- a/target-microblaze/cpu-qom.h
+++ b/target-microblaze/cpu-qom.h
@@ -47,36 +47,6 @@
     void (*parent_reset)(CPUState *cpu);
 } MicroBlazeCPUClass;
 
-/**
- * MicroBlazeCPU:
- * @env: #CPUMBState
- *
- * A MicroBlaze CPU.
- */
-typedef struct MicroBlazeCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    uint32_t base_vectors;
-    /*< public >*/
-
-    CPUMBState env;
-} MicroBlazeCPU;
-
-static inline MicroBlazeCPU *mb_env_get_cpu(CPUMBState *env)
-{
-    return container_of(env, MicroBlazeCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(mb_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(MicroBlazeCPU, env)
-
-void mb_cpu_do_interrupt(CPUState *cs);
-bool mb_cpu_exec_interrupt(CPUState *cs, int int_req);
-void mb_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
-                       int flags);
-hwaddr mb_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int mb_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int mb_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+typedef struct MicroBlazeCPU MicroBlazeCPU;
 
 #endif
diff --git a/target-microblaze/cpu.c b/target-microblaze/cpu.c
index 67e3182..8edc00a 100644
--- a/target-microblaze/cpu.c
+++ b/target-microblaze/cpu.c
@@ -21,11 +21,51 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "hw/qdev-properties.h"
 #include "migration/vmstate.h"
+#include "exec/exec-all.h"
 
+static const struct {
+    const char *name;
+    uint8_t version_id;
+} mb_cpu_lookup[] = {
+    /* These key value are as per MBV field in PVR0 */
+    {"5.00.a", 0x01},
+    {"5.00.b", 0x02},
+    {"5.00.c", 0x03},
+    {"6.00.a", 0x04},
+    {"6.00.b", 0x06},
+    {"7.00.a", 0x05},
+    {"7.00.b", 0x07},
+    {"7.10.a", 0x08},
+    {"7.10.b", 0x09},
+    {"7.10.c", 0x0a},
+    {"7.10.d", 0x0b},
+    {"7.20.a", 0x0c},
+    {"7.20.b", 0x0d},
+    {"7.20.c", 0x0e},
+    {"7.20.d", 0x0f},
+    {"7.30.a", 0x10},
+    {"7.30.b", 0x11},
+    {"8.00.a", 0x12},
+    {"8.00.b", 0x13},
+    {"8.10.a", 0x14},
+    {"8.20.a", 0x15},
+    {"8.20.b", 0x16},
+    {"8.30.a", 0x17},
+    {"8.40.a", 0x18},
+    {"8.40.b", 0x19},
+    {"8.50.a", 0x1A},
+    {"9.0", 0x1B},
+    {"9.1", 0x1D},
+    {"9.2", 0x1F},
+    {"9.3", 0x20},
+    {NULL, 0},
+};
 
 static void mb_cpu_set_pc(CPUState *cs, vaddr value)
 {
@@ -63,21 +103,50 @@
 
     mcc->parent_reset(s);
 
-    memset(env, 0, sizeof(CPUMBState));
+    memset(env, 0, offsetof(CPUMBState, pvr));
     env->res_addr = RES_ADDR_NONE;
     tlb_flush(s, 1);
 
     /* Disable stack protector.  */
     env->shr = ~0;
 
-    env->pvr.regs[0] = PVR0_PVR_FULL_MASK \
-                       | PVR0_USE_BARREL_MASK \
+    env->sregs[SR_PC] = cpu->cfg.base_vectors;
+
+#if defined(CONFIG_USER_ONLY)
+    /* start in user mode with interrupts enabled.  */
+    env->sregs[SR_MSR] = MSR_EE | MSR_IE | MSR_VM | MSR_UM;
+#else
+    env->sregs[SR_MSR] = 0;
+    mmu_init(&env->mmu);
+    env->mmu.c_mmu = 3;
+    env->mmu.c_mmu_tlb_access = 3;
+    env->mmu.c_mmu_zones = 16;
+#endif
+}
+
+static void mb_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_arch_microblaze;
+    info->print_insn = print_insn_microblaze;
+}
+
+static void mb_cpu_realizefn(DeviceState *dev, Error **errp)
+{
+    CPUState *cs = CPU(dev);
+    MicroBlazeCPUClass *mcc = MICROBLAZE_CPU_GET_CLASS(dev);
+    MicroBlazeCPU *cpu = MICROBLAZE_CPU(cs);
+    CPUMBState *env = &cpu->env;
+    uint8_t version_code = 0;
+    int i = 0;
+
+    qemu_init_vcpu(cs);
+
+    env->pvr.regs[0] = PVR0_USE_BARREL_MASK \
                        | PVR0_USE_DIV_MASK \
                        | PVR0_USE_HW_MUL_MASK \
                        | PVR0_USE_EXC_MASK \
                        | PVR0_USE_ICACHE_MASK \
                        | PVR0_USE_DCACHE_MASK \
-                       | PVR0_USE_MMU \
                        | (0xb << 8);
     env->pvr.regs[2] = PVR2_D_OPB_MASK \
                         | PVR2_D_LMB_MASK \
@@ -89,36 +158,36 @@
                         | PVR2_USE_DIV_MASK \
                         | PVR2_USE_HW_MUL_MASK \
                         | PVR2_USE_MUL64_MASK \
-                        | PVR2_USE_FPU_MASK \
-                        | PVR2_USE_FPU2_MASK \
                         | PVR2_FPU_EXC_MASK \
                         | 0;
+
+    for (i = 0; mb_cpu_lookup[i].name && cpu->cfg.version; i++) {
+        if (strcmp(mb_cpu_lookup[i].name, cpu->cfg.version) == 0) {
+            version_code = mb_cpu_lookup[i].version_id;
+            break;
+        }
+    }
+
+    if (!version_code) {
+        qemu_log("Invalid MicroBlaze version number: %s\n", cpu->cfg.version);
+    }
+
+    env->pvr.regs[0] |= (cpu->cfg.stackprot ? PVR0_SPROT_MASK : 0) |
+                        (cpu->cfg.use_fpu ? PVR0_USE_FPU_MASK : 0) |
+                        (cpu->cfg.use_mmu ? PVR0_USE_MMU_MASK : 0) |
+                        (cpu->cfg.endi ? PVR0_ENDI_MASK : 0) |
+                        (version_code << 16) |
+                        (cpu->cfg.pvr == C_PVR_FULL ? PVR0_PVR_FULL_MASK : 0);
+
+    env->pvr.regs[2] |= (cpu->cfg.use_fpu ? PVR2_USE_FPU_MASK : 0) |
+                        (cpu->cfg.use_fpu > 1 ? PVR2_USE_FPU2_MASK : 0);
+
+    env->pvr.regs[5] |= cpu->cfg.dcache_writeback ?
+                                        PVR5_DCACHE_WRITEBACK_MASK : 0;
+
     env->pvr.regs[10] = 0x0c000000; /* Default to spartan 3a dsp family.  */
     env->pvr.regs[11] = PVR11_USE_MMU | (16 << 17);
 
-    env->sregs[SR_PC] = cpu->base_vectors;
-
-#if defined(CONFIG_USER_ONLY)
-    /* start in user mode with interrupts enabled.  */
-    env->sregs[SR_MSR] = MSR_EE | MSR_IE | MSR_VM | MSR_UM;
-    env->pvr.regs[10] = 0x0c000000; /* Spartan 3a dsp.  */
-#else
-    env->sregs[SR_MSR] = 0;
-    mmu_init(&env->mmu);
-    env->mmu.c_mmu = 3;
-    env->mmu.c_mmu_tlb_access = 3;
-    env->mmu.c_mmu_zones = 16;
-#endif
-}
-
-static void mb_cpu_realizefn(DeviceState *dev, Error **errp)
-{
-    CPUState *cs = CPU(dev);
-    MicroBlazeCPUClass *mcc = MICROBLAZE_CPU_GET_CLASS(dev);
-
-    cpu_reset(cs);
-    qemu_init_vcpu(cs);
-
     mcc->parent_realize(dev, errp);
 }
 
@@ -130,7 +199,7 @@
     static bool tcg_initialized;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     set_float_rounding_mode(float_round_nearest_even, &env->fp_status);
 
@@ -151,7 +220,20 @@
 };
 
 static Property mb_properties[] = {
-    DEFINE_PROP_UINT32("xlnx.base-vectors", MicroBlazeCPU, base_vectors, 0),
+    DEFINE_PROP_UINT32("base-vectors", MicroBlazeCPU, cfg.base_vectors, 0),
+    DEFINE_PROP_BOOL("use-stack-protection", MicroBlazeCPU, cfg.stackprot,
+                     false),
+    /* If use-fpu > 0 - FPU is enabled
+     * If use-fpu = 2 - Floating point conversion and square root instructions
+     *                  are enabled
+     */
+    DEFINE_PROP_UINT8("use-fpu", MicroBlazeCPU, cfg.use_fpu, 2),
+    DEFINE_PROP_BOOL("use-mmu", MicroBlazeCPU, cfg.use_mmu, true),
+    DEFINE_PROP_BOOL("dcache-writeback", MicroBlazeCPU, cfg.dcache_writeback,
+                     false),
+    DEFINE_PROP_BOOL("endianness", MicroBlazeCPU, cfg.endi, false),
+    DEFINE_PROP_STRING("version", MicroBlazeCPU, cfg.version),
+    DEFINE_PROP_UINT8("pvr", MicroBlazeCPU, cfg.pvr, C_PVR_FULL),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -183,6 +265,14 @@
     dc->vmsd = &vmstate_mb_cpu;
     dc->props = mb_properties;
     cc->gdb_num_core_regs = 32 + 5;
+
+    cc->disas_set_info = mb_disas_set_info;
+
+    /*
+     * Reason: mb_cpu_initfn() calls cpu_exec_init(), which saves the
+     * object in cpus -> dangling pointer after final object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo mb_cpu_type_info = {
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 6ccd060..beb75ff 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -16,11 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef CPU_MICROBLAZE_H
-#define CPU_MICROBLAZE_H
 
-#include "config.h"
+#ifndef MICROBLAZE_CPU_H
+#define MICROBLAZE_CPU_H
+
 #include "qemu-common.h"
+#include "cpu-qom.h"
 
 #define TARGET_LONG_BITS 32
 
@@ -34,16 +35,11 @@
 #include "mmu.h"
 #endif
 
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE	EM_MICROBLAZE
-
-#define EXCP_NMI        1
-#define EXCP_MMU        2
-#define EXCP_IRQ        3
-#define EXCP_BREAK      4
-#define EXCP_HW_BREAK   5
-#define EXCP_HW_EXCP    6
+#define EXCP_MMU        1
+#define EXCP_IRQ        2
+#define EXCP_BREAK      3
+#define EXCP_HW_BREAK   4
+#define EXCP_HW_EXCP    5
 
 /* MicroBlaze-specific interrupt pending bits.  */
 #define CPU_INTERRUPT_NMI       CPU_INTERRUPT_TGT_EXT_3
@@ -125,12 +121,13 @@
 #define PVR0_USE_EXC_MASK               0x04000000
 #define PVR0_USE_ICACHE_MASK            0x02000000
 #define PVR0_USE_DCACHE_MASK            0x01000000
-#define PVR0_USE_MMU                    0x00800000      /* new */
+#define PVR0_USE_MMU_MASK               0x00800000
 #define PVR0_USE_BTC			0x00400000
-#define PVR0_ENDI			0x00200000
+#define PVR0_ENDI_MASK                  0x00200000
 #define PVR0_FAULT			0x00100000
 #define PVR0_VERSION_MASK               0x0000FF00
 #define PVR0_USER1_MASK                 0x000000FF
+#define PVR0_SPROT_MASK                 0x00000001
 
 /* User 2 PVR mask */
 #define PVR1_USER2_MASK                 0xFFFFFFFF
@@ -214,7 +211,9 @@
 /* MSR Reset value PVR mask */
 #define PVR11_MSR_RESET_VALUE_MASK      0x000007FF
 
-
+#define C_PVR_NONE                      0
+#define C_PVR_BASIC                     1
+#define C_PVR_FULL                      2
 
 /* CPU flags.  */
 
@@ -263,53 +262,80 @@
 #define IFLAGS_TB_MASK  (D_FLAG | IMM_FLAG | DRTI_FLAG | DRTE_FLAG | DRTB_FLAG)
     uint32_t iflags;
 
-    struct {
-        uint32_t regs[16];
-    } pvr;
-
 #if !defined(CONFIG_USER_ONLY)
     /* Unified MMU.  */
     struct microblaze_mmu mmu;
 #endif
 
     CPU_COMMON
+
+    /* These fields are preserved on reset.  */
+
+    struct {
+        uint32_t regs[16];
+    } pvr;
 };
 
-#include "cpu-qom.h"
+/**
+ * MicroBlazeCPU:
+ * @env: #CPUMBState
+ *
+ * A MicroBlaze CPU.
+ */
+struct MicroBlazeCPU {
+    /*< private >*/
+    CPUState parent_obj;
+
+    /*< public >*/
+
+    /* Microblaze Configuration Settings */
+    struct {
+        bool stackprot;
+        uint32_t base_vectors;
+        uint8_t use_fpu;
+        bool use_mmu;
+        bool dcache_writeback;
+        bool endi;
+        char *version;
+        uint8_t pvr;
+    } cfg;
+
+    CPUMBState env;
+};
+
+static inline MicroBlazeCPU *mb_env_get_cpu(CPUMBState *env)
+{
+    return container_of(env, MicroBlazeCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(mb_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(MicroBlazeCPU, env)
+
+void mb_cpu_do_interrupt(CPUState *cs);
+bool mb_cpu_exec_interrupt(CPUState *cs, int int_req);
+void mb_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
+                       int flags);
+hwaddr mb_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int mb_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int mb_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
 void mb_tcg_init(void);
 MicroBlazeCPU *cpu_mb_init(const char *cpu_model);
-int cpu_mb_exec(CPUMBState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
 int cpu_mb_signal_handler(int host_signum, void *pinfo,
                           void *puc);
 
-enum {
-    CC_OP_DYNAMIC, /* Use env->cc_op  */
-    CC_OP_FLAGS,
-    CC_OP_CMP,
-};
-
 /* FIXME: MB uses variable pages down to 1K but linux only uses 4k.  */
 #define TARGET_PAGE_BITS 12
-#define MMAP_SHIFT TARGET_PAGE_BITS
 
 #define TARGET_PHYS_ADDR_SPACE_BITS 32
 #define TARGET_VIRT_ADDR_SPACE_BITS 32
 
-static inline CPUMBState *cpu_init(const char *cpu_model)
-{
-    MicroBlazeCPU *cpu = cpu_mb_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_mb_init(cpu_model))
 
-#define cpu_exec cpu_mb_exec
-#define cpu_gen_code cpu_mb_gen_code
 #define cpu_signal_handler cpu_mb_signal_handler
 
 /* MMU modes definitions */
@@ -321,7 +347,7 @@
 #define MMU_USER_IDX    2
 /* See NB_MMU_MODES further up the file.  */
 
-static inline int cpu_mmu_index (CPUMBState *env)
+static inline int cpu_mmu_index (CPUMBState *env, bool ifetch)
 {
         /* Are we in nommu mode?.  */
         if (!(env->sregs[SR_MSR] & MSR_VM))
@@ -335,20 +361,10 @@
 int mb_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
                             int mmu_idx);
 
-static inline int cpu_interrupts_enabled(CPUMBState *env)
-{
-    return env->sregs[SR_MSR] & MSR_IE;
-}
-
 #include "exec/cpu-all.h"
 
-static inline target_ulong cpu_get_pc(CPUMBState *env)
-{
-    return env->sregs[SR_PC];
-}
-
 static inline void cpu_get_tb_cpu_state(CPUMBState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->sregs[SR_PC];
     *cs_base = 0;
@@ -362,6 +378,4 @@
                               unsigned size);
 #endif
 
-#include "exec/exec-all.h"
-
 #endif
diff --git a/target-microblaze/gdbstub.c b/target-microblaze/gdbstub.c
index a70e2ee..7fb076c 100644
--- a/target-microblaze/gdbstub.c
+++ b/target-microblaze/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int mb_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
diff --git a/target-microblaze/helper.c b/target-microblaze/helper.c
index 59466c9..da394d1 100644
--- a/target-microblaze/helper.c
+++ b/target-microblaze/helper.c
@@ -18,11 +18,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
+#include "exec/log.h"
 
 #define D(x)
-#define DMMU(x)
 
 #if defined(CONFIG_USER_ONLY)
 
@@ -57,10 +59,10 @@
     int prot;
 
     mmu_available = 0;
-    if (env->pvr.regs[0] & PVR0_USE_MMU) {
+    if (cpu->cfg.use_mmu) {
         mmu_available = 1;
-        if ((env->pvr.regs[0] & PVR0_PVR_FULL_MASK)
-            && (env->pvr.regs[11] & PVR11_USE_MMU) != PVR11_USE_MMU) {
+        if ((cpu->cfg.pvr == C_PVR_FULL) &&
+            (env->pvr.regs[11] & PVR11_USE_MMU) != PVR11_USE_MMU) {
             mmu_available = 0;
         }
     }
@@ -75,13 +77,14 @@
             vaddr = address & TARGET_PAGE_MASK;
             paddr = lu.paddr + vaddr - lu.vaddr;
 
-            DMMU(qemu_log("MMU map mmu=%d v=%x p=%x prot=%x\n",
-                     mmu_idx, vaddr, paddr, lu.prot));
+            qemu_log_mask(CPU_LOG_MMU, "MMU map mmu=%d v=%x p=%x prot=%x\n",
+                    mmu_idx, vaddr, paddr, lu.prot);
             tlb_set_page(cs, vaddr, paddr, lu.prot, mmu_idx, TARGET_PAGE_SIZE);
             r = 0;
         } else {
             env->sregs[SR_EAR] = address;
-            DMMU(qemu_log("mmu=%d miss v=%x\n", mmu_idx, address));
+            qemu_log_mask(CPU_LOG_MMU, "mmu=%d miss v=%" VADDR_PRIx "\n",
+                                        mmu_idx, address);
 
             switch (lu.err) {
                 case ERR_PROT:
@@ -128,7 +131,7 @@
     switch (cs->exception_index) {
         case EXCP_HW_EXCP:
             if (!(env->pvr.regs[0] & PVR0_USE_EXC_MASK)) {
-                qemu_log("Exception raised on system without exceptions!\n");
+                qemu_log_mask(LOG_GUEST_ERROR, "Exception raised on system without exceptions!\n");
                 return;
             }
 
@@ -154,7 +157,7 @@
                           env->sregs[SR_ESR], env->iflags);
             log_cpu_state_mask(CPU_LOG_INT, cs, 0);
             env->iflags &= ~(IMM_FLAG | D_FLAG);
-            env->sregs[SR_PC] = cpu->base_vectors + 0x20;
+            env->sregs[SR_PC] = cpu->cfg.base_vectors + 0x20;
             break;
 
         case EXCP_MMU:
@@ -194,7 +197,7 @@
                           env->sregs[SR_PC], env->sregs[SR_EAR], env->iflags);
             log_cpu_state_mask(CPU_LOG_INT, cs, 0);
             env->iflags &= ~(IMM_FLAG | D_FLAG);
-            env->sregs[SR_PC] = cpu->base_vectors + 0x20;
+            env->sregs[SR_PC] = cpu->cfg.base_vectors + 0x20;
             break;
 
         case EXCP_IRQ:
@@ -235,7 +238,7 @@
             env->sregs[SR_MSR] |= t;
 
             env->regs[14] = env->sregs[SR_PC];
-            env->sregs[SR_PC] = cpu->base_vectors + 0x10;
+            env->sregs[SR_PC] = cpu->cfg.base_vectors + 0x10;
             //log_cpu_state_mask(CPU_LOG_INT, cs, 0);
             break;
 
@@ -254,7 +257,7 @@
             if (cs->exception_index == EXCP_HW_BREAK) {
                 env->regs[16] = env->sregs[SR_PC];
                 env->sregs[SR_MSR] |= MSR_BIP;
-                env->sregs[SR_PC] = cpu->base_vectors + 0x18;
+                env->sregs[SR_PC] = cpu->cfg.base_vectors + 0x18;
             } else
                 env->sregs[SR_PC] = env->btarget;
             break;
diff --git a/target-microblaze/mmu.c b/target-microblaze/mmu.c
index 728da13..a22a496 100644
--- a/target-microblaze/mmu.c
+++ b/target-microblaze/mmu.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 
 #define D(x)
 
@@ -60,7 +62,7 @@
     uint32_t t;
 
     if (newpid & ~0xff)
-        qemu_log("Illegal rpid=%x\n", newpid);
+        qemu_log_mask(LOG_GUEST_ERROR, "Illegal rpid=%x\n", newpid);
 
     for (i = 0; i < ARRAY_SIZE(mmu->rams[RAM_TAG]); i++) {
         /* Lookup and decode.  */
@@ -121,7 +123,7 @@
             t0 &= 0x3;
 
             if (tlb_zsel > mmu->c_mmu_zones) {
-                qemu_log("tlb zone select out of range! %d\n", tlb_zsel);
+                qemu_log_mask(LOG_GUEST_ERROR, "tlb zone select out of range! %d\n", tlb_zsel);
                 t0 = 1; /* Ignore.  */
             }
 
@@ -183,7 +185,7 @@
     uint32_t r;
 
     if (env->mmu.c_mmu < 2 || !env->mmu.c_mmu_tlb_access) {
-        qemu_log("MMU access on MMU-less system\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "MMU access on MMU-less system\n");
         return 0;
     }
 
@@ -192,7 +194,7 @@
         case MMU_R_TLBLO:
         case MMU_R_TLBHI:
             if (!(env->mmu.c_mmu_tlb_access & 1)) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return 0;
             }
 
@@ -204,7 +206,7 @@
         case MMU_R_PID:
         case MMU_R_ZPR:
             if (!(env->mmu.c_mmu_tlb_access & 1)) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return 0;
             }
             r = env->mmu.regs[rn];
@@ -224,7 +226,7 @@
     D(qemu_log("%s rn=%d=%x old=%x\n", __func__, rn, v, env->mmu.regs[rn]));
 
     if (env->mmu.c_mmu < 2 || !env->mmu.c_mmu_tlb_access) {
-        qemu_log("MMU access on MMU-less system\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "MMU access on MMU-less system\n");
         return;
     }
 
@@ -235,7 +237,7 @@
             i = env->mmu.regs[MMU_R_TLBX] & 0xff;
             if (rn == MMU_R_TLBHI) {
                 if (i < 3 && !(v & TLB_VALID) && qemu_loglevel_mask(~0))
-                    qemu_log("invalidating index %x at pc=%x\n",
+                    qemu_log_mask(LOG_GUEST_ERROR, "invalidating index %x at pc=%x\n",
                              i, env->sregs[SR_PC]);
                 env->mmu.tids[i] = env->mmu.regs[MMU_R_PID] & 0xff;
                 mmu_flush_idx(env, i);
@@ -246,7 +248,7 @@
             break;
         case MMU_R_ZPR:
             if (env->mmu.c_mmu_tlb_access <= 1) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return;
             }
 
@@ -259,7 +261,7 @@
             break;
         case MMU_R_PID:
             if (env->mmu.c_mmu_tlb_access <= 1) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return;
             }
 
@@ -274,12 +276,12 @@
             int hit;
 
             if (env->mmu.c_mmu_tlb_access <= 1) {
-                qemu_log("Invalid access to MMU reg %d\n", rn);
+                qemu_log_mask(LOG_GUEST_ERROR, "Invalid access to MMU reg %d\n", rn);
                 return;
             }
 
             hit = mmu_translate(&env->mmu, &lu,
-                                v & TLB_EPN_MASK, 0, cpu_mmu_index(env));
+                                v & TLB_EPN_MASK, 0, cpu_mmu_index(env, false));
             if (hit) {
                 env->mmu.regs[MMU_R_TLBX] = lu.idx;
             } else
diff --git a/target-microblaze/mmu.h b/target-microblaze/mmu.h
index 3f74dda..3b7a998 100644
--- a/target-microblaze/mmu.h
+++ b/target-microblaze/mmu.h
@@ -82,7 +82,6 @@
     } err;
 };
 
-void mmu_flip_um(CPUMBState *env, unsigned int um);
 unsigned int mmu_translate(struct microblaze_mmu *mmu,
                            struct microblaze_mmu_lookup *lu,
                            target_ulong vaddr, int rw, int mmu_idx);
diff --git a/target-microblaze/op_helper.c b/target-microblaze/op_helper.c
index a4c8f04..4a856e6 100644
--- a/target-microblaze/op_helper.c
+++ b/target-microblaze/op_helper.c
@@ -18,10 +18,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #define D(x)
@@ -32,12 +33,12 @@
  * NULL, it means that the function was called in C code (i.e. not
  * from generated code or from helper.c)
  */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = mb_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = mb_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
@@ -56,7 +57,7 @@
     int nonblock = ctrl & STREAM_NONBLOCK;
     int exception = ctrl & STREAM_EXCEPTION;
 
-    qemu_log("Unhandled stream put to stream-id=%d data=%x %s%s%s%s%s\n",
+    qemu_log_mask(LOG_UNIMP, "Unhandled stream put to stream-id=%d data=%x %s%s%s%s%s\n",
              id, data,
              test ? "t" : "",
              nonblock ? "n" : "",
@@ -73,7 +74,7 @@
     int nonblock = ctrl & STREAM_NONBLOCK;
     int exception = ctrl & STREAM_EXCEPTION;
 
-    qemu_log("Unhandled stream get from stream-id=%d %s%s%s%s%s\n",
+    qemu_log_mask(LOG_UNIMP, "Unhandled stream get from stream-id=%d %s%s%s%s%s\n",
              id,
              test ? "t" : "",
              nonblock ? "n" : "",
@@ -151,9 +152,7 @@
 
 uint32_t helper_carry(uint32_t a, uint32_t b, uint32_t cf)
 {
-    uint32_t ncf;
-    ncf = compute_carry(a, b, cf);
-    return ncf;
+    return compute_carry(a, b, cf);
 }
 
 static inline int div_prepare(CPUMBState *env, uint32_t a, uint32_t b)
@@ -289,12 +288,14 @@
     fa.l = a;
     fb.l = b;
 
-    if (float32_is_signaling_nan(fa.f) || float32_is_signaling_nan(fb.f)) {
+    if (float32_is_signaling_nan(fa.f, &env->fp_status) ||
+        float32_is_signaling_nan(fb.f, &env->fp_status)) {
         update_fpu_flags(env, float_flag_invalid);
         r = 1;
     }
 
-    if (float32_is_quiet_nan(fa.f) || float32_is_quiet_nan(fb.f)) {
+    if (float32_is_quiet_nan(fa.f, &env->fp_status) ||
+        float32_is_quiet_nan(fb.f, &env->fp_status)) {
         r = 1;
     }
 
@@ -468,11 +469,11 @@
 void helper_stackprot(CPUMBState *env, uint32_t addr)
 {
     if (addr < env->slr || addr > env->shr) {
-            qemu_log("Stack protector violation at %x %x %x\n",
-                     addr, env->slr, env->shr);
-            env->sregs[SR_EAR] = addr;
-            env->sregs[SR_ESR] = ESR_EC_STACKPROT;
-            helper_raise_exception(env, EXCP_HW_EXCP);
+        qemu_log_mask(CPU_LOG_INT, "Stack protector violation at %x %x %x\n",
+                      addr, env->slr, env->shr);
+        env->sregs[SR_EAR] = addr;
+        env->sregs[SR_ESR] = ESR_EC_STACKPROT;
+        helper_raise_exception(env, EXCP_HW_EXCP);
     }
 }
 
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index d37f235..80098ec 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -18,8 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/helper-proto.h"
 #include "microblaze-decode.h"
@@ -27,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define SIM_COMPAT 0
@@ -44,7 +47,7 @@
             (((src) >> start) & ((1 << (end - start + 1)) - 1))
 
 static TCGv env_debug;
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_R[32];
 static TCGv cpu_SR[18];
 static TCGv env_imm;
@@ -122,14 +125,21 @@
     dc->is_jmp = DISAS_UPDATE;
 }
 
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+#ifndef CONFIG_USER_ONLY
+    return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = dc->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+    if (use_goto_tb(dc, dest)) {
         tcg_gen_goto_tb(n);
         tcg_gen_movi_tl(cpu_SR[SR_PC], dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)dc->tb + n);
     } else {
         tcg_gen_movi_tl(cpu_SR[SR_PC], dest);
         tcg_gen_exit_tb(0);
@@ -313,7 +323,6 @@
 static void dec_pattern(DisasContext *dc)
 {
     unsigned int mode;
-    int l1;
 
     if ((dc->tb_flags & MSR_EE_FLAG)
           && (dc->cpu->env.pvr.regs[2] & PVR2_ILL_OPCODE_EXC_MASK)
@@ -333,29 +342,15 @@
         case 2:
             LOG_DIS("pcmpeq r%d r%d r%d\n", dc->rd, dc->ra, dc->rb);
             if (dc->rd) {
-                TCGv t0 = tcg_temp_local_new();
-                l1 = gen_new_label();
-                tcg_gen_movi_tl(t0, 1);
-                tcg_gen_brcond_tl(TCG_COND_EQ,
-                                  cpu_R[dc->ra], cpu_R[dc->rb], l1);
-                tcg_gen_movi_tl(t0, 0);
-                gen_set_label(l1);
-                tcg_gen_mov_tl(cpu_R[dc->rd], t0);
-                tcg_temp_free(t0);
+                tcg_gen_setcond_tl(TCG_COND_EQ, cpu_R[dc->rd],
+                                   cpu_R[dc->ra], cpu_R[dc->rb]);
             }
             break;
         case 3:
             LOG_DIS("pcmpne r%d r%d r%d\n", dc->rd, dc->ra, dc->rb);
-            l1 = gen_new_label();
             if (dc->rd) {
-                TCGv t0 = tcg_temp_local_new();
-                tcg_gen_movi_tl(t0, 1);
-                tcg_gen_brcond_tl(TCG_COND_NE,
-                                  cpu_R[dc->ra], cpu_R[dc->rb], l1);
-                tcg_gen_movi_tl(t0, 0);
-                gen_set_label(l1);
-                tcg_gen_mov_tl(cpu_R[dc->rd], t0);
-                tcg_temp_free(t0);
+                tcg_gen_setcond_tl(TCG_COND_NE, cpu_R[dc->rd],
+                                   cpu_R[dc->ra], cpu_R[dc->rb]);
             }
             break;
         default:
@@ -433,7 +428,7 @@
     CPUState *cs = CPU(dc->cpu);
     TCGv t0, t1;
     unsigned int sr, to, rn;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     sr = dc->imm & ((1 << 14) - 1);
     to = dc->imm & (1 << 14);
@@ -598,9 +593,9 @@
     tcg_gen_ext_i32_i64(t1, b);
     tcg_gen_mul_i64(t0, t0, t1);
 
-    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_extrl_i64_i32(d, t0);
     tcg_gen_shri_i64(t0, t0, 32);
-    tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_extrl_i64_i32(d2, t0);
 
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
@@ -618,9 +613,9 @@
     tcg_gen_extu_i32_i64(t1, b);
     tcg_gen_mul_i64(t0, t0, t1);
 
-    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_extrl_i64_i32(d, t0);
     tcg_gen_shri_i64(t0, t0, 32);
-    tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_extrl_i64_i32(d2, t0);
 
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
@@ -745,7 +740,7 @@
     CPUState *cs = CPU(dc->cpu);
     TCGv t0;
     unsigned int op;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     op = dc->ir & ((1 << 9) - 1);
     switch (op) {
@@ -862,7 +857,7 @@
     int stackprot = 0;
 
     /* All load/stores use ra.  */
-    if (dc->ra == 1) {
+    if (dc->ra == 1 && dc->cpu->cfg.stackprot) {
         stackprot = 1;
     }
 
@@ -875,7 +870,7 @@
             return &cpu_R[dc->ra];
         }
 
-        if (dc->rb == 1) {
+        if (dc->rb == 1 && dc->cpu->cfg.stackprot) {
             stackprot = 1;
         }
 
@@ -1009,7 +1004,7 @@
      * address and if that succeeds we write into the destination reg.
      */
     v = tcg_temp_new();
-    tcg_gen_qemu_ld_tl(v, *addr, cpu_mmu_index(&dc->cpu->env), mop);
+    tcg_gen_qemu_ld_tl(v, *addr, cpu_mmu_index(&dc->cpu->env, false), mop);
 
     if ((dc->cpu->env.pvr.regs[2] & PVR2_UNALIGNED_EXC_MASK) && size > 1) {
         tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
@@ -1027,7 +1022,7 @@
     tcg_temp_free(v);
 
     if (ex) { /* lwx */
-        /* no support for for AXI exclusive so always clear C */
+        /* no support for AXI exclusive so always clear C */
         write_carryi(dc, 0);
     }
 
@@ -1038,7 +1033,7 @@
 static void dec_store(DisasContext *dc)
 {
     TCGv t, *addr, swx_addr;
-    int swx_skip = 0;
+    TCGLabel *swx_skip = NULL;
     unsigned int size, rev = 0, ex = 0;
     TCGMemOp mop;
 
@@ -1087,7 +1082,7 @@
            this compare and the following write to be atomic. For user
            emulation we need to add atomicity between threads.  */
         tval = tcg_temp_new();
-        tcg_gen_qemu_ld_tl(tval, swx_addr, cpu_mmu_index(&dc->cpu->env),
+        tcg_gen_qemu_ld_tl(tval, swx_addr, cpu_mmu_index(&dc->cpu->env, false),
                            MO_TEUL);
         tcg_gen_brcond_tl(TCG_COND_NE, env_res_val, tval, swx_skip);
         write_carryi(dc, 0);
@@ -1138,7 +1133,7 @@
                 break;
         }
     }
-    tcg_gen_qemu_st_tl(cpu_R[dc->rd], *addr, cpu_mmu_index(&dc->cpu->env), mop);
+    tcg_gen_qemu_st_tl(cpu_R[dc->rd], *addr, cpu_mmu_index(&dc->cpu->env, false), mop);
 
     /* Verify alignment if needed.  */
     if ((dc->cpu->env.pvr.regs[2] & PVR2_UNALIGNED_EXC_MASK) && size > 1) {
@@ -1192,9 +1187,7 @@
 
 static void eval_cond_jmp(DisasContext *dc, TCGv pc_true, TCGv pc_false)
 {
-    int l1;
-
-    l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     /* Conditional jmp.  */
     tcg_gen_mov_tl(cpu_SR[SR_PC], pc_false);
     tcg_gen_brcondi_tl(TCG_COND_EQ, env_btaken, 0, l1);
@@ -1236,7 +1229,7 @@
 static void dec_br(DisasContext *dc)
 {
     unsigned int dslot, link, abs, mbar;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     dslot = dc->ir & (1 << 20);
     abs = dc->ir & (1 << 19);
@@ -1368,7 +1361,7 @@
 static void dec_rts(DisasContext *dc)
 {
     unsigned int b_bit, i_bit, e_bit;
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
 
     i_bit = dc->ir & (1 << 21);
     b_bit = dc->ir & (1 << 22);
@@ -1413,15 +1406,11 @@
 
 static int dec_check_fpuv2(DisasContext *dc)
 {
-    int r;
-
-    r = dc->cpu->env.pvr.regs[2] & PVR2_USE_FPU2_MASK;
-
-    if (!r && (dc->tb_flags & MSR_EE_FLAG)) {
+    if ((dc->cpu->cfg.use_fpu != 2) && (dc->tb_flags & MSR_EE_FLAG)) {
         tcg_gen_movi_tl(cpu_SR[SR_ESR], ESR_EC_FPU);
         t_gen_raise_exception(dc, EXCP_HW_EXCP);
     }
-    return r;
+    return (dc->cpu->cfg.use_fpu == 2) ? 0 : PVR2_USE_FPU2_MASK;
 }
 
 static void dec_fpu(DisasContext *dc)
@@ -1430,7 +1419,7 @@
 
     if ((dc->tb_flags & MSR_EE_FLAG)
           && (dc->cpu->env.pvr.regs[2] & PVR2_ILL_OPCODE_EXC_MASK)
-          && !((dc->cpu->env.pvr.regs[2] & PVR2_USE_FPU_MASK))) {
+          && (dc->cpu->cfg.use_fpu != 1)) {
         tcg_gen_movi_tl(cpu_SR[SR_ESR], ESR_EC_ILLEGAL_OP);
         t_gen_raise_exception(dc, EXCP_HW_EXCP);
         return;
@@ -1537,14 +1526,14 @@
         t_gen_raise_exception(dc, EXCP_HW_EXCP);
         return;
     }
-    qemu_log ("unknown insn pc=%x opc=%x\n", dc->pc, dc->opcode);
+    qemu_log_mask(LOG_GUEST_ERROR, "unknown insn pc=%x opc=%x\n", dc->pc, dc->opcode);
     dc->abort_at_next_insn = 1;
 }
 
 /* Insns connected to FSL or AXI stream attached devices.  */
 static void dec_stream(DisasContext *dc)
 {
-    int mem_index = cpu_mmu_index(&dc->cpu->env);
+    int mem_index = cpu_mmu_index(&dc->cpu->env, false);
     TCGv_i32 t_id, t_ctrl;
     int ctrl;
 
@@ -1609,10 +1598,6 @@
 {
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
@@ -1651,30 +1636,12 @@
     }
 }
 
-static void check_breakpoint(CPUMBState *env, DisasContext *dc)
-{
-    CPUState *cs = CPU(mb_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                t_gen_raise_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
-}
-
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(MicroBlazeCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMBState *env, struct TranslationBlock *tb)
 {
+    MicroBlazeCPU *cpu = mb_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUMBState *env = &cpu->env;
     uint32_t pc_start;
-    int j, lj;
     struct DisasContext ctx;
     struct DisasContext *dc = &ctx;
     uint32_t next_page_start, org_flags;
@@ -1711,47 +1678,51 @@
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do
     {
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
 #if SIM_COMPAT
         if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
             tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
             gen_helper_debug();
         }
 #endif
-        check_breakpoint(env, dc);
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-                        tcg_ctx.gen_opc_icount[lj] = num_insns;
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            t_gen_raise_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
         /* Pretty disas.  */
         LOG_DIS("%8.8x:\t", dc->pc);
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         dc->clear_imm = 1;
         decode(dc, cpu_ldl_code(env, dc->pc));
         if (dc->clear_imm)
             dc->tb_flags &= ~IMM_FLAG;
         dc->pc += 4;
-        num_insns++;
 
         if (dc->delayed_branch) {
             dc->delayed_branch--;
@@ -1773,10 +1744,8 @@
                     gen_goto_tb(dc, 0, dc->jmp_pc);
                     dc->is_jmp = DISAS_TB_JUMP;
                 } else if (dc->jmp == JMP_DIRECT_CC) {
-                    int l1;
-
+                    TCGLabel *l1 = gen_new_label();
                     t_sync_flags(dc);
-                    l1 = gen_new_label();
                     /* Conditional jmp.  */
                     tcg_gen_brcondi_tl(TCG_COND_NE, env_btaken, 0, l1);
                     gen_goto_tb(dc, 1, dc->pc);
@@ -1844,22 +1813,16 @@
     }
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = dc->pc - pc_start;
-                tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
 #if !SIM_COMPAT
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("\n");
 #if DISAS_GNU
-        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
+        log_target_disas(cs, pc_start, dc->pc - pc_start, 0);
 #endif
         qemu_log("\nisize=%d osize=%d\n",
                  dc->pc - pc_start, tcg_op_buf_count());
@@ -1869,16 +1832,6 @@
     assert(!dc->abort_at_next_insn);
 }
 
-void gen_intermediate_code (CPUMBState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mb_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUMBState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mb_env_get_cpu(env), tb, true);
-}
-
 void mb_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                        int flags)
 {
@@ -1925,41 +1878,43 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
-    env_debug = tcg_global_mem_new(TCG_AREG0, 
+    env_debug = tcg_global_mem_new(cpu_env,
                     offsetof(CPUMBState, debug),
                     "debug0");
-    env_iflags = tcg_global_mem_new(TCG_AREG0, 
+    env_iflags = tcg_global_mem_new(cpu_env,
                     offsetof(CPUMBState, iflags),
                     "iflags");
-    env_imm = tcg_global_mem_new(TCG_AREG0, 
+    env_imm = tcg_global_mem_new(cpu_env,
                     offsetof(CPUMBState, imm),
                     "imm");
-    env_btarget = tcg_global_mem_new(TCG_AREG0,
+    env_btarget = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, btarget),
                      "btarget");
-    env_btaken = tcg_global_mem_new(TCG_AREG0,
+    env_btaken = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, btaken),
                      "btaken");
-    env_res_addr = tcg_global_mem_new(TCG_AREG0,
+    env_res_addr = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, res_addr),
                      "res_addr");
-    env_res_val = tcg_global_mem_new(TCG_AREG0,
+    env_res_val = tcg_global_mem_new(cpu_env,
                      offsetof(CPUMBState, res_val),
                      "res_val");
     for (i = 0; i < ARRAY_SIZE(cpu_R); i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPUMBState, regs[i]),
                           regnames[i]);
     }
     for (i = 0; i < ARRAY_SIZE(cpu_SR); i++) {
-        cpu_SR[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_SR[i] = tcg_global_mem_new(cpu_env,
                           offsetof(CPUMBState, sregs[i]),
                           special_regnames[i]);
     }
 }
 
-void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMBState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->sregs[SR_PC] = tcg_ctx.gen_opc_pc[pc_pos];
+    env->sregs[SR_PC] = data[0];
 }
diff --git a/target-mips/Makefile.objs b/target-mips/Makefile.objs
index 108fd9b..bc5ed85 100644
--- a/target-mips/Makefile.objs
+++ b/target-mips/Makefile.objs
@@ -1,4 +1,4 @@
 obj-y += translate.o dsp_helper.o op_helper.o lmi_helper.o helper.o cpu.o
-obj-y += gdbstub.o msa_helper.o
+obj-y += gdbstub.o msa_helper.o mips-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/target-mips/cpu-qom.h b/target-mips/cpu-qom.h
index 2ffc1bf..3f5bf23 100644
--- a/target-mips/cpu-qom.h
+++ b/target-mips/cpu-qom.h
@@ -51,37 +51,6 @@
     void (*parent_reset)(CPUState *cpu);
 } MIPSCPUClass;
 
-/**
- * MIPSCPU:
- * @env: #CPUMIPSState
- *
- * A MIPS CPU.
- */
-typedef struct MIPSCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUMIPSState env;
-} MIPSCPU;
-
-static inline MIPSCPU *mips_env_get_cpu(CPUMIPSState *env)
-{
-    return container_of(env, MIPSCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(mips_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(MIPSCPU, env)
-
-void mips_cpu_do_interrupt(CPUState *cpu);
-bool mips_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void mips_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
-                         int flags);
-hwaddr mips_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int mips_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int mips_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-void mips_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
-                                  int is_write, int is_user, uintptr_t retaddr);
+typedef struct MIPSCPU MIPSCPU;
 
 #endif
diff --git a/target-mips/cpu.c b/target-mips/cpu.c
index 98dc94e..64ad112 100644
--- a/target-mips/cpu.c
+++ b/target-mips/cpu.c
@@ -18,10 +18,13 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "kvm_mips.h"
 #include "qemu-common.h"
 #include "sysemu/kvm.h"
+#include "exec/exec-all.h"
 
 
 static void mips_cpu_set_pc(CPUState *cs, vaddr value)
@@ -53,12 +56,15 @@
     CPUMIPSState *env = &cpu->env;
     bool has_work = false;
 
-    /* It is implementation dependent if non-enabled interrupts
-       wake-up the CPU, however most of the implementations only
+    /* Prior to MIPS Release 6 it is implementation dependent if non-enabled
+       interrupts wake-up the CPU, however most of the implementations only
        check for interrupts that can be taken. */
     if ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
         cpu_mips_hw_interrupts_pending(env)) {
-        has_work = true;
+        if (cpu_mips_hw_interrupts_enabled(env) ||
+            (env->insn_flags & ISA_MIPS32R6)) {
+            has_work = true;
+        }
     }
 
     /* MIPS-MT has the ability to halt the CPU.  */
@@ -73,6 +79,15 @@
             has_work = false;
         }
     }
+    /* MIPS Release 6 has the ability to halt the CPU.  */
+    if (env->CP0_Config5 & (1 << CP0C5_VP)) {
+        if (cs->interrupt_request & CPU_INTERRUPT_WAKE) {
+            has_work = true;
+        }
+        if (!mips_vp_active(env)) {
+            has_work = false;
+        }
+    }
     return has_work;
 }
 
@@ -97,6 +112,14 @@
 #endif
 }
 
+static void mips_cpu_disas_set_info(CPUState *s, disassemble_info *info) {
+#ifdef TARGET_WORDS_BIGENDIAN
+    info->print_insn = print_insn_big_mips;
+#else
+    info->print_insn = print_insn_little_mips;
+#endif
+}
+
 static void mips_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -115,7 +138,7 @@
     CPUMIPSState *env = &cpu->env;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     if (tcg_enabled()) {
         mips_tcg_init();
@@ -148,10 +171,19 @@
     cc->do_unassigned_access = mips_cpu_unassigned_access;
     cc->do_unaligned_access = mips_cpu_do_unaligned_access;
     cc->get_phys_page_debug = mips_cpu_get_phys_page_debug;
+    cc->vmsd = &vmstate_mips_cpu;
 #endif
+    cc->disas_set_info = mips_cpu_disas_set_info;
 
     cc->gdb_num_core_regs = 73;
     cc->gdb_stop_before_watchpoint = true;
+
+    /*
+     * Reason: mips_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo mips_cpu_type_info = {
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index 8ab5df2..efc9424 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -1,17 +1,14 @@
-#if !defined (__MIPS_CPU_H__)
-#define __MIPS_CPU_H__
+#ifndef MIPS_CPU_H
+#define MIPS_CPU_H
 
 //#define DEBUG_OP
 
 #define ALIGNED_ONLY
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE	EM_MIPS
 
 #define CPUArchState struct CPUMIPSState
 
-#include "config.h"
 #include "qemu-common.h"
+#include "cpu-qom.h"
 #include "mips-defs.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
@@ -22,20 +19,20 @@
 struct r4k_tlb_t {
     target_ulong VPN;
     uint32_t PageMask;
-    uint_fast8_t ASID;
-    uint_fast16_t G:1;
-    uint_fast16_t C0:3;
-    uint_fast16_t C1:3;
-    uint_fast16_t V0:1;
-    uint_fast16_t V1:1;
-    uint_fast16_t D0:1;
-    uint_fast16_t D1:1;
-    uint_fast16_t XI0:1;
-    uint_fast16_t XI1:1;
-    uint_fast16_t RI0:1;
-    uint_fast16_t RI1:1;
-    uint_fast16_t EHINV:1;
-    target_ulong PFN[2];
+    uint16_t ASID;
+    unsigned int G:1;
+    unsigned int C0:3;
+    unsigned int C1:3;
+    unsigned int V0:1;
+    unsigned int V1:1;
+    unsigned int D0:1;
+    unsigned int D1:1;
+    unsigned int XI0:1;
+    unsigned int XI1:1;
+    unsigned int RI0:1;
+    unsigned int RI1:1;
+    unsigned int EHINV:1;
+    uint64_t PFN[2];
 };
 
 #if !defined(CONFIG_USER_ONLY)
@@ -103,6 +100,7 @@
     uint32_t fcr0;
 #define FCR0_FREP 29
 #define FCR0_UFRP 28
+#define FCR0_HAS2008 23
 #define FCR0_F64 22
 #define FCR0_L 21
 #define FCR0_W 20
@@ -113,7 +111,11 @@
 #define FCR0_PRID 8
 #define FCR0_REV 0
     /* fcsr */
+    uint32_t fcr31_rw_bitmask;
     uint32_t fcr31;
+#define FCR31_FS 24
+#define FCR31_ABS2008 19
+#define FCR31_NAN2008 18
 #define SET_FP_COND(num,env)     do { ((env).fcr31) |= ((num) ? (1 << ((num) + 24)) : (1 << 23)); } while(0)
 #define CLEAR_FP_COND(num,env)   do { ((env).fcr31) &= ~((num) ? (1 << ((num) + 24)) : (1 << 23)); } while(0)
 #define GET_FP_COND(env)         ((((env).fcr31 >> 24) & 0xfe) | (((env).fcr31 >> 23) & 0x1))
@@ -133,6 +135,7 @@
 };
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 2
 
 typedef struct CPUMIPSMVPContext CPUMIPSMVPContext;
 struct CPUMIPSMVPContext {
@@ -165,6 +168,7 @@
 #define MIPS_FPU_MAX 1
 #define MIPS_DSP_ACC 4
 #define MIPS_KSCRATCH_NUM 6
+#define MIPS_MAAR_MAX 16 /* Must be an even number. */
 
 typedef struct TCState TCState;
 struct TCState {
@@ -225,8 +229,14 @@
 
     uint32_t SEGBITS;
     uint32_t PABITS;
+#if defined(TARGET_MIPS64)
+# define PABITS_BASE 36
+#else
+# define PABITS_BASE 32
+#endif
     target_ulong SEGMask;
-    target_ulong PAMask;
+    uint64_t PAMask;
+#define PAMASK_BASE ((1ULL << PABITS_BASE) - 1)
 
     int32_t msair;
 #define MSAIR_ProcID    8
@@ -234,6 +244,8 @@
 
     int32_t CP0_Index;
     /* CP0_MVP* are per MVP registers. */
+    int32_t CP0_VPControl;
+#define CP0VPCtl_DIS    0
     int32_t CP0_Random;
     int32_t CP0_VPEControl;
 #define CP0VPECo_YSI	21
@@ -274,8 +286,8 @@
 #define CP0VPEOpt_DWX2	2
 #define CP0VPEOpt_DWX1	1
 #define CP0VPEOpt_DWX0	0
-    target_ulong CP0_EntryLo0;
-    target_ulong CP0_EntryLo1;
+    uint64_t CP0_EntryLo0;
+    uint64_t CP0_EntryLo1;
 #if defined(TARGET_MIPS64)
 # define CP0EnLo_RI 63
 # define CP0EnLo_XI 62
@@ -288,6 +300,8 @@
 #define CP0EnLo_D    2
 #define CP0EnLo_V    1
 #define CP0EnLo_G    0
+    int32_t CP0_GlobalNumber;
+#define CP0GN_VPId 0
     target_ulong CP0_Context;
     target_ulong CP0_KScratch[MIPS_KSCRATCH_NUM];
     int32_t CP0_PageMask;
@@ -295,6 +309,7 @@
     int32_t CP0_PageGrain;
 #define CP0PG_RIE 31
 #define CP0PG_XIE 30
+#define CP0PG_ELPA 29
 #define CP0PG_IEC 27
     int32_t CP0_Wired;
     int32_t CP0_SRSConf0_rw_bitmask;
@@ -333,6 +348,7 @@
     int32_t CP0_Count;
     target_ulong CP0_EntryHi;
 #define CP0EnHi_EHINV 10
+    target_ulong CP0_EntryHi_ASID_mask;
     int32_t CP0_Compare;
     int32_t CP0_Status;
 #define CP0St_CU3   31
@@ -358,7 +374,7 @@
 #define CP0St_IE    0
     int32_t CP0_IntCtl;
 #define CP0IntCtl_IPTI 29
-#define CP0IntCtl_IPPC1 26
+#define CP0IntCtl_IPPCI 26
 #define CP0IntCtl_VS 5
     int32_t CP0_SRSCtl;
 #define CP0SRSCtl_HSS 26
@@ -389,12 +405,13 @@
     target_ulong CP0_EPC;
     int32_t CP0_PRid;
     int32_t CP0_EBase;
+    target_ulong CP0_CMGCRBase;
     int32_t CP0_Config0;
 #define CP0C0_M    31
 #define CP0C0_K23  28
 #define CP0C0_KU   25
 #define CP0C0_MDU  20
-#define CP0C0_MM   17
+#define CP0C0_MM   18
 #define CP0C0_BM   16
 #define CP0C0_BE   15
 #define CP0C0_AT   13
@@ -431,7 +448,7 @@
     int32_t CP0_Config3;
 #define CP0C3_M    31
 #define CP0C3_BPG  30
-#define CP0C3_CMCGR 29
+#define CP0C3_CMGCR 29
 #define CP0C3_MSAP  28
 #define CP0C3_BP 27
 #define CP0C3_BI 26
@@ -452,40 +469,48 @@
 #define CP0C3_MT   2
 #define CP0C3_SM   1
 #define CP0C3_TL   0
-    uint32_t CP0_Config4;
-    uint32_t CP0_Config4_rw_bitmask;
+    int32_t CP0_Config4;
+    int32_t CP0_Config4_rw_bitmask;
 #define CP0C4_M    31
 #define CP0C4_IE   29
+#define CP0C4_AE   28
 #define CP0C4_KScrExist 16
 #define CP0C4_MMUExtDef 14
 #define CP0C4_FTLBPageSize 8
 #define CP0C4_FTLBWays 4
 #define CP0C4_FTLBSets 0
 #define CP0C4_MMUSizeExt 0
-    uint32_t CP0_Config5;
-    uint32_t CP0_Config5_rw_bitmask;
+    int32_t CP0_Config5;
+    int32_t CP0_Config5_rw_bitmask;
 #define CP0C5_M          31
 #define CP0C5_K          30
 #define CP0C5_CV         29
 #define CP0C5_EVA        28
 #define CP0C5_MSAEn      27
+#define CP0C5_XNP        13
 #define CP0C5_UFE        9
 #define CP0C5_FRE        8
+#define CP0C5_VP         7
 #define CP0C5_SBRI       6
+#define CP0C5_MVH        5
 #define CP0C5_LLB        4
+#define CP0C5_MRP        3
 #define CP0C5_UFR        2
 #define CP0C5_NFExists   0
     int32_t CP0_Config6;
     int32_t CP0_Config7;
+    uint64_t CP0_MAAR[MIPS_MAAR_MAX];
+    int32_t CP0_MAARI;
     /* XXX: Maybe make LLAddr per-TC? */
-    target_ulong lladdr;
+    uint64_t lladdr;
     target_ulong llval;
     target_ulong llnewval;
     target_ulong llreg;
-    target_ulong CP0_LLAddr_rw_bitmask;
+    uint64_t CP0_LLAddr_rw_bitmask;
     int CP0_LLAddr_shift;
     target_ulong CP0_WatchLo[8];
     int32_t CP0_WatchHi[8];
+#define CP0WH_ASID 16
     target_ulong CP0_XContext;
     int32_t CP0_Framemask;
     int32_t CP0_Debug;
@@ -509,7 +534,11 @@
 #define CP0DB_DSS  0
     target_ulong CP0_DEPC;
     int32_t CP0_Performance0;
-    int32_t CP0_TagLo;
+    int32_t CP0_ErrCtl;
+#define CP0EC_WST 29
+#define CP0EC_SPR 28
+#define CP0EC_ITC 26
+    uint64_t CP0_TagLo;
     int32_t CP0_DataLo;
     int32_t CP0_TagHi;
     int32_t CP0_DataHi;
@@ -524,7 +553,7 @@
 #define EXCP_INST_NOTAVAIL 0x2 /* No valid instruction word for BadInstr */
     uint32_t hflags;    /* CPU State */
     /* TMASK defines different execution modes */
-#define MIPS_HFLAG_TMASK  0x75807FF
+#define MIPS_HFLAG_TMASK  0xF5807FF
 #define MIPS_HFLAG_MODE   0x00007 /* execution modes                    */
     /* The KSU flags must be the lowest bits in hflags. The flag order
        must be the same as defined for CP0 Status. This allows to use
@@ -571,8 +600,9 @@
 #define MIPS_HFLAG_SBRI  0x400000 /* R6 SDBBP causes RI excpt. in user mode */
 #define MIPS_HFLAG_FBNSLOT 0x800000 /* Forbidden slot                   */
 #define MIPS_HFLAG_MSA   0x1000000
-#define MIPS_HFLAG_LLBIT 0x2000000 /* In this mode we snoop all stores */
-#define MIPS_HFLAG_FRE   0x4000000 /* FRE enabled */
+#define MIPS_HFLAG_FRE   0x2000000 /* FRE enabled */
+#define MIPS_HFLAG_ELPA  0x4000000
+#define MIPS_HFLAG_ITC_CACHE  0x8000000 /* CACHE instr. operates on ITC tag */
     target_ulong btarget;        /* Jump / branch target               */
     target_ulong bcond;          /* Branch condition (if needed)       */
 
@@ -593,9 +623,48 @@
     const mips_def_t *cpu_model;
     void *irq[8];
     QEMUTimer *timer; /* Internal timer */
+    MemoryRegion *itc_tag; /* ITC Configuration Tags */
+    target_ulong exception_base; /* ExceptionBase input to the core */
 };
 
-#include "cpu-qom.h"
+/**
+ * MIPSCPU:
+ * @env: #CPUMIPSState
+ *
+ * A MIPS CPU.
+ */
+struct MIPSCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUMIPSState env;
+};
+
+static inline MIPSCPU *mips_env_get_cpu(CPUMIPSState *env)
+{
+    return container_of(env, MIPSCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(mips_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(MIPSCPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_mips_cpu;
+#endif
+
+void mips_cpu_do_interrupt(CPUState *cpu);
+bool mips_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void mips_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
+                         int flags);
+hwaddr mips_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int mips_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int mips_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void mips_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                  MMUAccessType access_type,
+                                  int mmu_idx, uintptr_t retaddr,
+                                  unsigned size);
 
 #if !defined(CONFIG_USER_ONLY)
 
@@ -620,7 +689,9 @@
 
 extern CPUInterpreterContext CPU;
 
-int tlb_exception_interpreter(CPUMIPSState *env, uint32_t *handler, uint32_t size);
+int tlb_exception_interpreter(CPUMIPSState *env, uint32_t *handler,
+                              uint32_t size);
+
 int no_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                         target_ulong address, int rw, int access_type);
 int fixed_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
@@ -641,44 +712,41 @@
 
 void mips_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 
-#define cpu_exec cpu_mips_exec
-#define cpu_gen_code cpu_mips_gen_code
 #define cpu_signal_handler cpu_mips_signal_handler
 #define cpu_list mips_cpu_list
 
 extern void cpu_wrdsp(uint32_t rs, uint32_t mask_num, CPUMIPSState *env);
 extern uint32_t cpu_rddsp(uint32_t mask_num, CPUMIPSState *env);
 
-#define CPU_SAVE_VERSION 5
-
 /* MMU modes definitions. We carefully match the indices with our
    hflags layout. */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _super
 #define MMU_MODE2_SUFFIX _user
 #define MMU_USER_IDX 2
-static inline int cpu_mmu_index (CPUMIPSState *env)
+static inline int cpu_mmu_index (CPUMIPSState *env, bool ifetch)
 {
     return env->hflags & MIPS_HFLAG_KSU;
 }
 
-static inline int cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
+static inline bool cpu_mips_hw_interrupts_enabled(CPUMIPSState *env)
 {
-    int32_t pending;
-    int32_t status;
-    int r;
-
-    if (!(env->CP0_Status & (1 << CP0St_IE)) ||
-        (env->CP0_Status & (1 << CP0St_EXL)) ||
-        (env->CP0_Status & (1 << CP0St_ERL)) ||
+    return (env->CP0_Status & (1 << CP0St_IE)) &&
+        !(env->CP0_Status & (1 << CP0St_EXL)) &&
+        !(env->CP0_Status & (1 << CP0St_ERL)) &&
+        !(env->hflags & MIPS_HFLAG_DM) &&
         /* Note that the TCStatus IXMT field is initialized to zero,
            and only MT capable cores can set it to one. So we don't
            need to check for MT capabilities here.  */
-        (env->active_tc.CP0_TCStatus & (1 << CP0TCSt_IXMT)) ||
-        (env->hflags & MIPS_HFLAG_DM)) {
-        /* Interrupts are disabled */
-        return 0;
-    }
+        !(env->active_tc.CP0_TCStatus & (1 << CP0TCSt_IXMT));
+}
+
+/* Check if there is pending and not masked out interrupt */
+static inline bool cpu_mips_hw_interrupts_pending(CPUMIPSState *env)
+{
+    int32_t pending;
+    int32_t status;
+    bool r;
 
     pending = env->CP0_Cause & CP0Ca_IP_mask;
     status = env->CP0_Status & CP0Ca_IP_mask;
@@ -692,7 +760,7 @@
         /* A MIPS configured with compatibility or VInt (Vectored Interrupts)
            treats the pending lines as individual interrupt lines, the status
            lines are individual masks.  */
-        r = pending & status;
+        r = (pending & status) != 0;
     }
     return r;
 }
@@ -769,19 +837,13 @@
  */
 #define CPU_INTERRUPT_WAKE CPU_INTERRUPT_TGT_INT_0
 
-int cpu_mips_exec(CPUMIPSState *s);
 void mips_tcg_init(void);
 MIPSCPU *cpu_mips_init(const char *cpu_model);
 int cpu_mips_signal_handler(int host_signum, void *pinfo, void *puc);
 
-static inline CPUMIPSState *cpu_init(const char *cpu_model)
-{
-    MIPSCPU *cpu = cpu_mips_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_mips_init(cpu_model))
+bool cpu_supports_cps_smp(const char *cpu_model);
+void cpu_set_exception_base(int vp_index, target_ulong address);
 
 /* TODO QOM'ify CPU reset and remove */
 void cpu_state_reset(CPUMIPSState *s);
@@ -800,6 +862,11 @@
 /* helper.c */
 int mips_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
                               int mmu_idx);
+
+/* op_helper.c */
+uint32_t float_class_s(uint32_t arg, float_status *fst);
+uint64_t float_class_d(uint64_t arg, float_status *fst);
+
 #if !defined(CONFIG_USER_ONLY)
 void r4k_invalidate_tlb (CPUMIPSState *env, int idx, int use_extra);
 hwaddr cpu_mips_translate_address (CPUMIPSState *env, target_ulong address,
@@ -811,8 +878,53 @@
 extern unsigned int ieee_rm[];
 int ieee_ex_to_mips(int xcpt);
 
+static inline void restore_rounding_mode(CPUMIPSState *env)
+{
+    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3],
+                            &env->active_fpu.fp_status);
+}
+
+static inline void restore_flush_mode(CPUMIPSState *env)
+{
+    set_flush_to_zero((env->active_fpu.fcr31 & (1 << FCR31_FS)) != 0,
+                      &env->active_fpu.fp_status);
+}
+
+static inline void restore_snan_bit_mode(CPUMIPSState *env)
+{
+    set_snan_bit_is_one((env->active_fpu.fcr31 & (1 << FCR31_NAN2008)) == 0,
+                        &env->active_fpu.fp_status);
+}
+
+static inline void restore_fp_status(CPUMIPSState *env)
+{
+    restore_rounding_mode(env);
+    restore_flush_mode(env);
+    restore_snan_bit_mode(env);
+}
+
+static inline void restore_msa_fp_status(CPUMIPSState *env)
+{
+    float_status *status = &env->active_tc.msa_fp_status;
+    int rounding_mode = (env->active_tc.msacsr & MSACSR_RM_MASK) >> MSACSR_RM;
+    bool flush_to_zero = (env->active_tc.msacsr & MSACSR_FS_MASK) != 0;
+
+    set_float_rounding_mode(ieee_rm[rounding_mode], status);
+    set_flush_to_zero(flush_to_zero, status);
+    set_flush_inputs_to_zero(flush_to_zero, status);
+}
+
+static inline void restore_pamask(CPUMIPSState *env)
+{
+    if (env->hflags & MIPS_HFLAG_ELPA) {
+        env->PAMask = (1ULL << env->PABITS) - 1;
+    } else {
+        env->PAMask = PAMASK_BASE;
+    }
+}
+
 static inline void cpu_get_tb_cpu_state(CPUMIPSState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->active_tc.PC;
     *cs_base = 0;
@@ -851,30 +963,52 @@
     return active;
 }
 
-#include "exec/exec-all.h"
+static inline int mips_vp_active(CPUMIPSState *env)
+{
+    CPUState *other_cs = first_cpu;
+
+    /* Check if the VP disabled other VPs (which means the VP is enabled) */
+    if ((env->CP0_VPControl >> CP0VPCtl_DIS) & 1) {
+        return 1;
+    }
+
+    /* Check if the virtual processor is disabled due to a DVP */
+    CPU_FOREACH(other_cs) {
+        MIPSCPU *other_cpu = MIPS_CPU(other_cs);
+        if ((&other_cpu->env != env) &&
+            ((other_cpu->env.CP0_VPControl >> CP0VPCtl_DIS) & 1)) {
+            return 0;
+        }
+    }
+    return 1;
+}
 
 static inline void compute_hflags(CPUMIPSState *env)
 {
     env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
                      MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
                      MIPS_HFLAG_AWRAP | MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2 |
-                     MIPS_HFLAG_SBRI | MIPS_HFLAG_MSA | MIPS_HFLAG_FRE);
+                     MIPS_HFLAG_SBRI | MIPS_HFLAG_MSA | MIPS_HFLAG_FRE |
+                     MIPS_HFLAG_ELPA);
     if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
         !(env->CP0_Status & (1 << CP0St_ERL)) &&
         !(env->hflags & MIPS_HFLAG_DM)) {
         env->hflags |= (env->CP0_Status >> CP0St_KSU) & MIPS_HFLAG_KSU;
     }
 #if defined(TARGET_MIPS64)
-    if (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_UM) ||
-        (env->CP0_Status & (1 << CP0St_PX)) ||
-        (env->CP0_Status & (1 << CP0St_UX))) {
+    if ((env->insn_flags & ISA_MIPS3) &&
+        (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_UM) ||
+         (env->CP0_Status & (1 << CP0St_PX)) ||
+         (env->CP0_Status & (1 << CP0St_UX)))) {
         env->hflags |= MIPS_HFLAG_64;
     }
 
-    if (((env->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_UM) &&
-        !(env->CP0_Status & (1 << CP0St_UX))) {
+    if (!(env->insn_flags & ISA_MIPS3)) {
         env->hflags |= MIPS_HFLAG_AWRAP;
-    } else if (env->insn_flags & ISA_MIPS32R6) {
+    } else if (((env->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_UM) &&
+               !(env->CP0_Status & (1 << CP0St_UX))) {
+        env->hflags |= MIPS_HFLAG_AWRAP;
+    } else if (env->insn_flags & ISA_MIPS64R6) {
         /* Address wrapping for Supervisor and Kernel is specified in R6 */
         if ((((env->hflags & MIPS_HFLAG_KSU) == MIPS_HFLAG_SM) &&
              !(env->CP0_Status & (1 << CP0St_SX))) ||
@@ -941,5 +1075,26 @@
             env->hflags |= MIPS_HFLAG_FRE;
         }
     }
+    if (env->CP0_Config3 & (1 << CP0C3_LPA)) {
+        if (env->CP0_PageGrain & (1 << CP0PG_ELPA)) {
+            env->hflags |= MIPS_HFLAG_ELPA;
+        }
+    }
 }
-#endif /* !defined (__MIPS_CPU_H__) */
+
+void cpu_mips_tlb_flush(CPUMIPSState *env, int flush_global);
+void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc);
+void cpu_mips_store_status(CPUMIPSState *env, target_ulong val);
+void cpu_mips_store_cause(CPUMIPSState *env, target_ulong val);
+
+void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env, uint32_t exception,
+                                          int error_code, uintptr_t pc);
+
+static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
+                                                    uint32_t exception,
+                                                    uintptr_t pc)
+{
+    do_raise_exception_err(env, exception, 0, pc);
+}
+
+#endif /* MIPS_CPU_H */
diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
index 349f2a0..df7d220 100644
--- a/target-mips/dsp_helper.c
+++ b/target-mips/dsp_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/bitops.h"
@@ -3678,7 +3679,7 @@
 
 void helper_wrdsp(target_ulong rs, target_ulong mask_num, CPUMIPSState *env)
 {
-    return cpu_wrdsp(rs, mask_num, env);
+    cpu_wrdsp(rs, mask_num, env);
 }
 
 uint32_t cpu_rddsp(uint32_t mask_num, CPUMIPSState *env)
diff --git a/target-mips/gdbstub.c b/target-mips/gdbstub.c
index f65fec2..7c68228 100644
--- a/target-mips/gdbstub.c
+++ b/target-mips/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int mips_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
@@ -29,8 +30,13 @@
     if (n < 32) {
         return gdb_get_regl(mem_buf, env->active_tc.gpr[n]);
     }
-    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
-        if (n >= 38 && n < 70) {
+    if (env->CP0_Config1 & (1 << CP0C1_FP) && n >= 38 && n < 72) {
+        switch (n) {
+        case 70:
+            return gdb_get_regl(mem_buf, (int32_t)env->active_fpu.fcr31);
+        case 71:
+            return gdb_get_regl(mem_buf, (int32_t)env->active_fpu.fcr0);
+        default:
             if (env->CP0_Status & (1 << CP0St_FR)) {
                 return gdb_get_regl(mem_buf,
                     env->active_fpu.fpr[n - 38].d);
@@ -39,12 +45,6 @@
                     env->active_fpu.fpr[n - 38].w[FP_ENDIAN_IDX]);
             }
         }
-        switch (n) {
-        case 70:
-            return gdb_get_regl(mem_buf, (int32_t)env->active_fpu.fcr31);
-        case 71:
-            return gdb_get_regl(mem_buf, (int32_t)env->active_fpu.fcr0);
-        }
     }
     switch (n) {
     case 32:
@@ -64,8 +64,10 @@
         return gdb_get_regl(mem_buf, 0); /* fp */
     case 89:
         return gdb_get_regl(mem_buf, (int32_t)env->CP0_PRid);
-    }
-    if (n >= 73 && n <= 88) {
+    default:
+        if (n > 89) {
+            return 0;
+        }
         /* 16 embedded regs.  */
         return gdb_get_regl(mem_buf, 0);
     }
@@ -73,10 +75,6 @@
     return 0;
 }
 
-#define RESTORE_ROUNDING_MODE \
-    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3], \
-                            &env->active_fpu.fp_status)
-
 int mips_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n)
 {
     MIPSCPU *cpu = MIPS_CPU(cs);
@@ -89,30 +87,31 @@
         env->active_tc.gpr[n] = tmp;
         return sizeof(target_ulong);
     }
-    if (env->CP0_Config1 & (1 << CP0C1_FP)
-            && n >= 38 && n < 73) {
-        if (n < 70) {
+    if (env->CP0_Config1 & (1 << CP0C1_FP) && n >= 38 && n < 72) {
+        switch (n) {
+        case 70:
+            env->active_fpu.fcr31 = (tmp & env->active_fpu.fcr31_rw_bitmask) |
+                  (env->active_fpu.fcr31 & ~(env->active_fpu.fcr31_rw_bitmask));
+            restore_fp_status(env);
+            break;
+        case 71:
+            /* FIR is read-only.  Ignore writes.  */
+            break;
+        default:
             if (env->CP0_Status & (1 << CP0St_FR)) {
                 env->active_fpu.fpr[n - 38].d = tmp;
             } else {
                 env->active_fpu.fpr[n - 38].w[FP_ENDIAN_IDX] = tmp;
             }
-        }
-        switch (n) {
-        case 70:
-            env->active_fpu.fcr31 = tmp & 0xFF83FFFF;
-            /* set rounding mode */
-            RESTORE_ROUNDING_MODE;
-            break;
-        case 71:
-            env->active_fpu.fcr0 = tmp;
             break;
         }
         return sizeof(target_ulong);
     }
     switch (n) {
     case 32:
-        env->CP0_Status = tmp;
+#ifndef CONFIG_USER_ONLY
+        cpu_mips_store_status(env, tmp);
+#endif
         break;
     case 33:
         env->active_tc.LO[0] = tmp;
@@ -124,7 +123,9 @@
         env->CP0_BadVAddr = tmp;
         break;
     case 36:
-        env->CP0_Cause = tmp;
+#ifndef CONFIG_USER_ONLY
+        cpu_mips_store_cause(env, tmp);
+#endif
         break;
     case 37:
         env->active_tc.PC = tmp & ~(target_ulong)1;
diff --git a/target-mips/helper.c b/target-mips/helper.c
index ba198c5..b1e7b39 100644
--- a/target-mips/helper.c
+++ b/target-mips/helper.c
@@ -16,16 +16,13 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "sysemu/kvm.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "exec/log.h"
 
 enum {
     TLBRET_XI = -6,
@@ -70,7 +67,7 @@
 int r4k_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                      target_ulong address, int rw, int access_type)
 {
-    uint8_t ASID = env->CP0_EntryHi & 0xFF;
+    uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
     int i;
 
     for (i = 0; i < env->tlb->tlb_in_use; i++) {
@@ -127,10 +124,6 @@
     /* effective address (modified for KVM T&E kernel segments) */
     target_ulong address = real_address;
 
-#if 0
-    qemu_log("user mode %d h %08x\n", user_mode, env->hflags);
-#endif
-
 #define USEG_LIMIT      0x7FFFFFFFUL
 #define KSEG0_BASE      0x80000000UL
 #define KSEG1_BASE      0xA0000000UL
@@ -227,13 +220,116 @@
             ret = TLBRET_BADADDR;
         }
     }
-#if 0
-    qemu_log(TARGET_FMT_lx " %d %d => %" HWADDR_PRIx " %d (%d)\n",
-            address, rw, access_type, *physical, *prot, ret);
-#endif
-
     return ret;
 }
+
+void cpu_mips_tlb_flush(CPUMIPSState *env, int flush_global)
+{
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+
+    /* Flush qemu's TLB and discard all shadowed entries.  */
+    tlb_flush(CPU(cpu), flush_global);
+    env->tlb->tlb_in_use = env->tlb->nb_tlb;
+}
+
+/* Called for updates to CP0_Status.  */
+void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
+{
+    int32_t tcstatus, *tcst;
+    uint32_t v = cpu->CP0_Status;
+    uint32_t cu, mx, asid, ksu;
+    uint32_t mask = ((1 << CP0TCSt_TCU3)
+                       | (1 << CP0TCSt_TCU2)
+                       | (1 << CP0TCSt_TCU1)
+                       | (1 << CP0TCSt_TCU0)
+                       | (1 << CP0TCSt_TMX)
+                       | (3 << CP0TCSt_TKSU)
+                       | (0xff << CP0TCSt_TASID));
+
+    cu = (v >> CP0St_CU0) & 0xf;
+    mx = (v >> CP0St_MX) & 0x1;
+    ksu = (v >> CP0St_KSU) & 0x3;
+    asid = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
+
+    tcstatus = cu << CP0TCSt_TCU0;
+    tcstatus |= mx << CP0TCSt_TMX;
+    tcstatus |= ksu << CP0TCSt_TKSU;
+    tcstatus |= asid;
+
+    if (tc == cpu->current_tc) {
+        tcst = &cpu->active_tc.CP0_TCStatus;
+    } else {
+        tcst = &cpu->tcs[tc].CP0_TCStatus;
+    }
+
+    *tcst &= ~mask;
+    *tcst |= tcstatus;
+    compute_hflags(cpu);
+}
+
+void cpu_mips_store_status(CPUMIPSState *env, target_ulong val)
+{
+    uint32_t mask = env->CP0_Status_rw_bitmask;
+    target_ulong old = env->CP0_Status;
+
+    if (env->insn_flags & ISA_MIPS32R6) {
+        bool has_supervisor = extract32(mask, CP0St_KSU, 2) == 0x3;
+#if defined(TARGET_MIPS64)
+        uint32_t ksux = (1 << CP0St_KX) & val;
+        ksux |= (ksux >> 1) & val; /* KX = 0 forces SX to be 0 */
+        ksux |= (ksux >> 1) & val; /* SX = 0 forces UX to be 0 */
+        val = (val & ~(7 << CP0St_UX)) | ksux;
+#endif
+        if (has_supervisor && extract32(val, CP0St_KSU, 2) == 0x3) {
+            mask &= ~(3 << CP0St_KSU);
+        }
+        mask &= ~(((1 << CP0St_SR) | (1 << CP0St_NMI)) & val);
+    }
+
+    env->CP0_Status = (old & ~mask) | (val & mask);
+#if defined(TARGET_MIPS64)
+    if ((env->CP0_Status ^ old) & (old & (7 << CP0St_UX))) {
+        /* Access to at least one of the 64-bit segments has been disabled */
+        cpu_mips_tlb_flush(env, 1);
+    }
+#endif
+    if (env->CP0_Config3 & (1 << CP0C3_MT)) {
+        sync_c0_status(env, env, env->current_tc);
+    } else {
+        compute_hflags(env);
+    }
+}
+
+void cpu_mips_store_cause(CPUMIPSState *env, target_ulong val)
+{
+    uint32_t mask = 0x00C00300;
+    uint32_t old = env->CP0_Cause;
+    int i;
+
+    if (env->insn_flags & ISA_MIPS32R2) {
+        mask |= 1 << CP0Ca_DC;
+    }
+    if (env->insn_flags & ISA_MIPS32R6) {
+        mask &= ~((1 << CP0Ca_WP) & val);
+    }
+
+    env->CP0_Cause = (env->CP0_Cause & ~mask) | (val & mask);
+
+    if ((old ^ env->CP0_Cause) & (1 << CP0Ca_DC)) {
+        if (env->CP0_Cause & (1 << CP0Ca_DC)) {
+            cpu_mips_stop_count(env);
+        } else {
+            cpu_mips_start_count(env);
+        }
+    }
+
+    /* Set/reset software interrupts */
+    for (i = 0 ; i < 2 ; i++) {
+        if ((old ^ env->CP0_Cause) & (1 << (CP0Ca_IP + i))) {
+            cpu_mips_soft_irq(env, i, env->CP0_Cause & (1 << (CP0Ca_IP + i)));
+        }
+    }
+}
 #endif
 
 static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
@@ -299,13 +395,15 @@
     env->CP0_BadVAddr = address;
     env->CP0_Context = (env->CP0_Context & ~0x007fffff) |
                        ((address >> 9) & 0x007ffff0);
-    env->CP0_EntryHi =
-        (env->CP0_EntryHi & 0xFF) | (address & (TARGET_PAGE_MASK << 1));
+    env->CP0_EntryHi = (env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask) |
+                       (env->CP0_EntryHi & (1 << CP0EnHi_EHINV)) |
+                       (address & (TARGET_PAGE_MASK << 1));
 #if defined(TARGET_MIPS64)
     env->CP0_EntryHi &= env->SEGMask;
-    env->CP0_XContext = (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) |
-                        ((address & 0xC00000000000ULL) >> (55 - env->SEGBITS)) |
-                        ((address & ((1ULL << env->SEGBITS) - 1) & 0xFFFFFFFFFFFFE000ULL) >> 9);
+    env->CP0_XContext =
+        /* PTEBase */   (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) |
+        /* R */         (extract64(address, 62, 2) << (env->SEGBITS - 9)) |
+        /* BadVPN2 */   (extract64(address, 13, env->SEGBITS - 13) << 4);
 #endif
     cs->exception_index = exception;
     env->error_code = error_code;
@@ -568,7 +666,8 @@
 #if 0
     log_cpu_state(cs, 0);
 #endif
-    qemu_log("%s pc " TARGET_FMT_lx " ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
+    qemu_log_mask(CPU_LOG_MMU,
+              "%s pc " TARGET_FMT_lx " ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
               __func__, env->active_tc.PC, address, rw, mmu_idx);
 
     /* data access */
@@ -578,7 +677,8 @@
     access_type = ACCESS_INT;
     ret = get_physical_address(env, &physical, &prot,
                                address, rw, access_type);
-    qemu_log("%s address=%" VADDR_PRIx " ret %d physical " TARGET_FMT_plx
+    qemu_log_mask(CPU_LOG_MMU,
+             "%s address=%" VADDR_PRIx " ret %d physical " TARGET_FMT_plx
              " prot %d\n",
              __func__, address, ret, physical, prot);
     if (ret == TLBRET_MATCH) {
@@ -618,7 +718,6 @@
         return physical;
     }
 }
-#endif
 
 static const char * const excp_names[EXCP_LAST + 1] = {
     [EXCP_RESET] = "reset",
@@ -659,6 +758,7 @@
     [EXCP_MSADIS] = "MSA disabled",
     [EXCP_MSAFPE] = "MSA floating point",
 };
+#endif
 
 target_ulong exception_resume_pc (CPUMIPSState *env)
 {
@@ -715,14 +815,16 @@
     int cause = -1;
     const char *name;
 
-    if (qemu_log_enabled() && cs->exception_index != EXCP_EXT_INTERRUPT) {
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && cs->exception_index != EXCP_EXT_INTERRUPT) {
         if (cs->exception_index < 0 || cs->exception_index > EXCP_LAST) {
             name = "unknown";
         } else {
             name = excp_names[cs->exception_index];
         }
 
-        qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " %s exception\n",
+        qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx
+                 " %s exception\n",
                  __func__, env->active_tc.PC, env->CP0_EPC, name);
     }
     if (cs->exception_index == EXCP_EXT_INTERRUPT &&
@@ -757,12 +859,19 @@
         env->CP0_DEPC = exception_resume_pc(env);
         env->hflags &= ~MIPS_HFLAG_BMASK;
  enter_debug_mode:
-        env->hflags |= MIPS_HFLAG_DM | MIPS_HFLAG_64 | MIPS_HFLAG_CP0;
+        if (env->insn_flags & ISA_MIPS3) {
+            env->hflags |= MIPS_HFLAG_64;
+            if (!(env->insn_flags & ISA_MIPS64R6) ||
+                env->CP0_Status & (1 << CP0St_KX)) {
+                env->hflags &= ~MIPS_HFLAG_AWRAP;
+            }
+        }
+        env->hflags |= MIPS_HFLAG_DM | MIPS_HFLAG_CP0;
         env->hflags &= ~(MIPS_HFLAG_KSU);
         /* EJTAG probe trap enable is not implemented... */
         if (!(env->CP0_Status & (1 << CP0St_EXL)))
             env->CP0_Cause &= ~(1U << CP0Ca_BD);
-        env->active_tc.PC = (int32_t)0xBFC00480;
+        env->active_tc.PC = env->exception_base + 0x480;
         set_hflags_for_handler(env);
         break;
     case EXCP_RESET:
@@ -770,7 +879,7 @@
         break;
     case EXCP_SRESET:
         env->CP0_Status |= (1 << CP0St_SR);
-        memset(env->CP0_WatchLo, 0, sizeof(*env->CP0_WatchLo));
+        memset(env->CP0_WatchLo, 0, sizeof(env->CP0_WatchLo));
         goto set_error_EPC;
     case EXCP_NMI:
         env->CP0_Status |= (1 << CP0St_NMI);
@@ -778,43 +887,46 @@
         env->CP0_ErrorEPC = exception_resume_pc(env);
         env->hflags &= ~MIPS_HFLAG_BMASK;
         env->CP0_Status |= (1 << CP0St_ERL) | (1 << CP0St_BEV);
-        env->hflags |= MIPS_HFLAG_64 | MIPS_HFLAG_CP0;
+        if (env->insn_flags & ISA_MIPS3) {
+            env->hflags |= MIPS_HFLAG_64;
+            if (!(env->insn_flags & ISA_MIPS64R6) ||
+                env->CP0_Status & (1 << CP0St_KX)) {
+                env->hflags &= ~MIPS_HFLAG_AWRAP;
+            }
+        }
+        env->hflags |= MIPS_HFLAG_CP0;
         env->hflags &= ~(MIPS_HFLAG_KSU);
         if (!(env->CP0_Status & (1 << CP0St_EXL)))
             env->CP0_Cause &= ~(1U << CP0Ca_BD);
-        env->active_tc.PC = (int32_t)0xBFC00000;
+        env->active_tc.PC = env->exception_base;
         set_hflags_for_handler(env);
         break;
     case EXCP_EXT_INTERRUPT:
         cause = 0;
-        if (env->CP0_Cause & (1 << CP0Ca_IV))
-            offset = 0x200;
+        if (env->CP0_Cause & (1 << CP0Ca_IV)) {
+            uint32_t spacing = (env->CP0_IntCtl >> CP0IntCtl_VS) & 0x1f;
 
-        if (env->CP0_Config3 & ((1 << CP0C3_VInt) | (1 << CP0C3_VEIC))) {
-            /* Vectored Interrupts.  */
-            unsigned int spacing;
-            unsigned int vector;
-            unsigned int pending = (env->CP0_Cause & CP0Ca_IP_mask) >> 8;
+            if ((env->CP0_Status & (1 << CP0St_BEV)) || spacing == 0) {
+                offset = 0x200;
+            } else {
+                uint32_t vector = 0;
+                uint32_t pending = (env->CP0_Cause & CP0Ca_IP_mask) >> CP0Ca_IP;
 
-            pending &= env->CP0_Status >> 8;
-            /* Compute the Vector Spacing.  */
-            spacing = (env->CP0_IntCtl >> CP0IntCtl_VS) & ((1 << 6) - 1);
-            spacing <<= 5;
-
-            if (env->CP0_Config3 & (1 << CP0C3_VInt)) {
-                /* For VInt mode, the MIPS computes the vector internally.  */
-                for (vector = 7; vector > 0; vector--) {
-                    if (pending & (1 << vector)) {
-                        /* Found it.  */
-                        break;
+                if (env->CP0_Config3 & (1 << CP0C3_VEIC)) {
+                    /* For VEIC mode, the external interrupt controller feeds
+                     * the vector through the CP0Cause IP lines.  */
+                    vector = pending;
+                } else {
+                    /* Vectored Interrupts
+                     * Mask with Status.IM7-IM0 to get enabled interrupts. */
+                    pending &= (env->CP0_Status >> CP0St_IM) & 0xff;
+                    /* Find the highest-priority interrupt. */
+                    while (pending >>= 1) {
+                        vector++;
                     }
                 }
-            } else {
-                /* For VEIC mode, the external interrupt controller feeds the
-                   vector through the CP0Cause IP lines.  */
-                vector = pending;
+                offset = 0x200 + (vector * (spacing << 5));
             }
-            offset = 0x200 + vector * spacing;
         }
         goto set_EPC;
     case EXCP_LTLBL:
@@ -926,7 +1038,7 @@
         goto set_EPC;
     case EXCP_DWATCH:
         cause = 23;
-        /* XXX: TODO: manage defered watch exceptions */
+        /* XXX: TODO: manage deferred watch exceptions */
         goto set_EPC;
     case EXCP_MCHECK:
         cause = 24;
@@ -956,12 +1068,19 @@
                 env->CP0_Cause &= ~(1U << CP0Ca_BD);
             }
             env->CP0_Status |= (1 << CP0St_EXL);
-            env->hflags |= MIPS_HFLAG_64 | MIPS_HFLAG_CP0;
+            if (env->insn_flags & ISA_MIPS3) {
+                env->hflags |= MIPS_HFLAG_64;
+                if (!(env->insn_flags & ISA_MIPS64R6) ||
+                    env->CP0_Status & (1 << CP0St_KX)) {
+                    env->hflags &= ~MIPS_HFLAG_AWRAP;
+                }
+            }
+            env->hflags |= MIPS_HFLAG_CP0;
             env->hflags &= ~(MIPS_HFLAG_KSU);
         }
         env->hflags &= ~MIPS_HFLAG_BMASK;
         if (env->CP0_Status & (1 << CP0St_BEV)) {
-            env->active_tc.PC = (int32_t)0xBFC00200;
+            env->active_tc.PC = env->exception_base + 0x200;
         } else {
             env->active_tc.PC = (int32_t)(env->CP0_EBase & ~0x3ff);
         }
@@ -970,16 +1089,15 @@
         env->CP0_Cause = (env->CP0_Cause & ~(0x1f << CP0Ca_EC)) | (cause << CP0Ca_EC);
         break;
     default:
-        qemu_log("Invalid MIPS exception %d. Exiting\n", cs->exception_index);
-        printf("Invalid MIPS exception %d. Exiting\n", cs->exception_index);
-        exit(1);
+        abort();
     }
-    if (qemu_log_enabled() && cs->exception_index != EXCP_EXT_INTERRUPT) {
+    if (qemu_loglevel_mask(CPU_LOG_INT)
+        && cs->exception_index != EXCP_EXT_INTERRUPT) {
         qemu_log("%s: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " cause %d\n"
-                "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
-                __func__, env->active_tc.PC, env->CP0_EPC, cause,
-                env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
-                env->CP0_DEPC);
+                 "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
+                 __func__, env->active_tc.PC, env->CP0_EPC, cause,
+                 env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
+                 env->CP0_DEPC);
     }
 #endif
     cs->exception_index = EXCP_NONE;
@@ -991,7 +1109,8 @@
         MIPSCPU *cpu = MIPS_CPU(cs);
         CPUMIPSState *env = &cpu->env;
 
-        if (cpu_mips_hw_interrupts_pending(env)) {
+        if (cpu_mips_hw_interrupts_enabled(env) &&
+            cpu_mips_hw_interrupts_pending(env)) {
             /* Raise it */
             cs->exception_index = EXCP_EXT_INTERRUPT;
             env->error_code = 0;
@@ -1010,7 +1129,7 @@
     r4k_tlb_t *tlb;
     target_ulong addr;
     target_ulong end;
-    uint8_t ASID = env->CP0_EntryHi & 0xFF;
+    uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
     target_ulong mask;
 
     tlb = &env->tlb->mmu.r4k.tlb[idx];
@@ -1061,3 +1180,20 @@
     }
 }
 #endif
+
+void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
+                                          uint32_t exception,
+                                          int error_code,
+                                          uintptr_t pc)
+{
+    CPUState *cs = CPU(mips_env_get_cpu(env));
+
+    if (exception < EXCP_SC) {
+        qemu_log_mask(CPU_LOG_INT, "%s: %d %d\n",
+                      __func__, exception, error_code);
+    }
+    cs->exception_index = exception;
+    env->error_code = error_code;
+
+    cpu_loop_exit_restore(cs, pc);
+}
diff --git a/target-mips/helper.h b/target-mips/helper.h
index b8716de..666936c 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -1,5 +1,8 @@
 DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
 DEF_HELPER_2(raise_exception, noreturn, env, i32)
+DEF_HELPER_1(raise_exception_debug, noreturn, env)
+
+DEF_HELPER_1(do_semihosting, void, env)
 
 #ifdef TARGET_MIPS64
 DEF_HELPER_4(sdl, void, env, tl, tl, int)
@@ -74,6 +77,8 @@
 DEF_HELPER_1(mftc0_ebase, tl, env)
 DEF_HELPER_2(mftc0_configx, tl, env, tl)
 DEF_HELPER_1(mfc0_lladdr, tl, env)
+DEF_HELPER_1(mfc0_maar, tl, env)
+DEF_HELPER_1(mfhc0_maar, tl, env)
 DEF_HELPER_2(mfc0_watchlo, tl, env, i32)
 DEF_HELPER_2(mfc0_watchhi, tl, env, i32)
 DEF_HELPER_1(mfc0_debug, tl, env)
@@ -85,6 +90,7 @@
 DEF_HELPER_1(dmfc0_tcschedule, tl, env)
 DEF_HELPER_1(dmfc0_tcschefback, tl, env)
 DEF_HELPER_1(dmfc0_lladdr, tl, env)
+DEF_HELPER_1(dmfc0_maar, tl, env)
 DEF_HELPER_2(dmfc0_watchlo, tl, env, i32)
 #endif /* TARGET_MIPS64 */
 
@@ -137,9 +143,13 @@
 DEF_HELPER_2(mttc0_ebase, void, env, tl)
 DEF_HELPER_2(mtc0_config0, void, env, tl)
 DEF_HELPER_2(mtc0_config2, void, env, tl)
+DEF_HELPER_2(mtc0_config3, void, env, tl)
 DEF_HELPER_2(mtc0_config4, void, env, tl)
 DEF_HELPER_2(mtc0_config5, void, env, tl)
 DEF_HELPER_2(mtc0_lladdr, void, env, tl)
+DEF_HELPER_2(mtc0_maar, void, env, tl)
+DEF_HELPER_2(mthc0_maar, void, env, tl)
+DEF_HELPER_2(mtc0_maari, void, env, tl)
 DEF_HELPER_3(mtc0_watchlo, void, env, tl, i32)
 DEF_HELPER_3(mtc0_watchhi, void, env, tl, i32)
 DEF_HELPER_2(mtc0_xcontext, void, env, tl)
@@ -147,6 +157,7 @@
 DEF_HELPER_2(mtc0_debug, void, env, tl)
 DEF_HELPER_2(mttc0_debug, void, env, tl)
 DEF_HELPER_2(mtc0_performance0, void, env, tl)
+DEF_HELPER_2(mtc0_errctl, void, env, tl)
 DEF_HELPER_2(mtc0_taglo, void, env, tl)
 DEF_HELPER_2(mtc0_datalo, void, env, tl)
 DEF_HELPER_2(mtc0_taghi, void, env, tl)
@@ -172,6 +183,10 @@
 DEF_HELPER_0(emt, tl)
 DEF_HELPER_1(dvpe, tl, env)
 DEF_HELPER_1(evpe, tl, env)
+
+/* R6 Multi-threading */
+DEF_HELPER_1(dvp, tl, env)
+DEF_HELPER_1(evp, tl, env)
 #endif /* !CONFIG_USER_ONLY */
 
 /* microMIPS functions */
@@ -192,8 +207,6 @@
 DEF_HELPER_2(float_cvtd_s, i64, env, i32)
 DEF_HELPER_2(float_cvtd_w, i64, env, i32)
 DEF_HELPER_2(float_cvtd_l, i64, env, i64)
-DEF_HELPER_2(float_cvtl_d, i64, env, i64)
-DEF_HELPER_2(float_cvtl_s, i64, env, i32)
 DEF_HELPER_2(float_cvtps_pw, i64, env, i64)
 DEF_HELPER_2(float_cvtpw_ps, i64, env, i64)
 DEF_HELPER_2(float_cvts_d, i32, env, i64)
@@ -201,14 +214,12 @@
 DEF_HELPER_2(float_cvts_l, i32, env, i64)
 DEF_HELPER_2(float_cvts_pl, i32, env, i32)
 DEF_HELPER_2(float_cvts_pu, i32, env, i32)
-DEF_HELPER_2(float_cvtw_s, i32, env, i32)
-DEF_HELPER_2(float_cvtw_d, i32, env, i64)
 
 DEF_HELPER_3(float_addr_ps, i64, env, i64, i64)
 DEF_HELPER_3(float_mulr_ps, i64, env, i64, i64)
 
-DEF_HELPER_FLAGS_1(float_class_s, TCG_CALL_NO_RWG_SE, i32, i32)
-DEF_HELPER_FLAGS_1(float_class_d, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_2(float_class_s, TCG_CALL_NO_RWG_SE, i32, env, i32)
+DEF_HELPER_FLAGS_2(float_class_d, TCG_CALL_NO_RWG_SE, i64, env, i64)
 
 #define FOP_PROTO(op)                                     \
 DEF_HELPER_4(float_ ## op ## _s, i32, env, i32, i32, i32) \
@@ -227,14 +238,20 @@
 #undef FOP_PROTO
 
 #define FOP_PROTO(op)                            \
-DEF_HELPER_2(float_ ## op ## l_s, i64, env, i32) \
-DEF_HELPER_2(float_ ## op ## l_d, i64, env, i64) \
-DEF_HELPER_2(float_ ## op ## w_s, i32, env, i32) \
-DEF_HELPER_2(float_ ## op ## w_d, i32, env, i64)
+DEF_HELPER_2(float_ ## op ## _l_s, i64, env, i32) \
+DEF_HELPER_2(float_ ## op ## _l_d, i64, env, i64) \
+DEF_HELPER_2(float_ ## op ## _w_s, i32, env, i32) \
+DEF_HELPER_2(float_ ## op ## _w_d, i32, env, i64)
+FOP_PROTO(cvt)
 FOP_PROTO(round)
 FOP_PROTO(trunc)
 FOP_PROTO(ceil)
 FOP_PROTO(floor)
+FOP_PROTO(cvt_2008)
+FOP_PROTO(round_2008)
+FOP_PROTO(trunc_2008)
+FOP_PROTO(ceil_2008)
+FOP_PROTO(floor_2008)
 #undef FOP_PROTO
 
 #define FOP_PROTO(op)                            \
@@ -354,6 +371,8 @@
 DEF_HELPER_1(rdhwr_synci_step, tl, env)
 DEF_HELPER_1(rdhwr_cc, tl, env)
 DEF_HELPER_1(rdhwr_ccres, tl, env)
+DEF_HELPER_1(rdhwr_performance, tl, env)
+DEF_HELPER_1(rdhwr_xnp, tl, env)
 DEF_HELPER_2(pmon, void, env, int)
 DEF_HELPER_1(wait, void, env)
 
@@ -939,3 +958,5 @@
 MSALDST_PROTO(w)
 MSALDST_PROTO(d)
 #undef MSALDST_PROTO
+
+DEF_HELPER_3(cache, void, env, tl, i32)
diff --git a/target-mips/kvm.c b/target-mips/kvm.c
index 97fd51a..dcf5fbb 100644
--- a/target-mips/kvm.c
+++ b/target-mips/kvm.c
@@ -9,26 +9,29 @@
  * Authors: Sanjay Lal <sanjayl@kymasys.com>
 */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
+#include "cpu.h"
 #include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
-#include "cpu.h"
 #include "sysemu/cpus.h"
 #include "kvm_mips.h"
+#include "exec/memattrs.h"
 
 #define DEBUG_KVM 0
 
 #define DPRINTF(fmt, ...) \
     do { if (DEBUG_KVM) { fprintf(stderr, fmt, ## __VA_ARGS__); } } while (0)
 
+static int kvm_mips_fpu_cap;
+static int kvm_mips_msa_cap;
+
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
@@ -40,21 +43,44 @@
     return cs->cpu_index;
 }
 
-int kvm_arch_init(KVMState *s)
+int kvm_arch_init(MachineState *ms, KVMState *s)
 {
     /* MIPS has 128 signals */
     kvm_set_sigmask_len(s, 16);
 
+    kvm_mips_fpu_cap = kvm_check_extension(s, KVM_CAP_MIPS_FPU);
+    kvm_mips_msa_cap = kvm_check_extension(s, KVM_CAP_MIPS_MSA);
+
     DPRINTF("%s\n", __func__);
     return 0;
 }
 
 int kvm_arch_init_vcpu(CPUState *cs)
 {
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
     int ret = 0;
 
     qemu_add_vm_change_state_handler(kvm_mips_update_state, cs);
 
+    if (kvm_mips_fpu_cap && env->CP0_Config1 & (1 << CP0C1_FP)) {
+        ret = kvm_vcpu_enable_cap(cs, KVM_CAP_MIPS_FPU, 0, 0);
+        if (ret < 0) {
+            /* mark unsupported so it gets disabled on reset */
+            kvm_mips_fpu_cap = 0;
+            ret = 0;
+        }
+    }
+
+    if (kvm_mips_msa_cap && env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        ret = kvm_vcpu_enable_cap(cs, KVM_CAP_MIPS_MSA, 0, 0);
+        if (ret < 0) {
+            /* mark unsupported so it gets disabled on reset */
+            kvm_mips_msa_cap = 0;
+            ret = 0;
+        }
+    }
+
     DPRINTF("%s\n", __func__);
     return ret;
 }
@@ -63,10 +89,14 @@
 {
     CPUMIPSState *env = &cpu->env;
 
-    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
-        fprintf(stderr, "Warning: FPU not supported with KVM, disabling\n");
+    if (!kvm_mips_fpu_cap && env->CP0_Config1 & (1 << CP0C1_FP)) {
+        fprintf(stderr, "Warning: KVM does not support FPU, disabling\n");
         env->CP0_Config1 &= ~(1 << CP0C1_FP);
     }
+    if (!kvm_mips_msa_cap && env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        fprintf(stderr, "Warning: KVM does not support MSA, disabling\n");
+        env->CP0_Config3 &= ~(1 << CP0C3_MSAP);
+    }
 
     DPRINTF("%s\n", __func__);
 }
@@ -87,7 +117,6 @@
 {
     CPUMIPSState *env = &cpu->env;
 
-    DPRINTF("%s: %#x\n", __func__, env->CP0_Cause & (1 << (2 + CP0Ca_IP)));
     return env->CP0_Cause & (0x1 << (2 + CP0Ca_IP));
 }
 
@@ -98,6 +127,8 @@
     int r;
     struct kvm_mips_interrupt intr;
 
+    qemu_mutex_lock_iothread();
+
     if ((cs->interrupt_request & CPU_INTERRUPT_HARD) &&
             cpu_mips_io_interrupts_pending(cpu)) {
         intr.cpu = -1;
@@ -108,11 +139,13 @@
                          __func__, cs->cpu_index, intr.irq);
         }
     }
+
+    qemu_mutex_unlock_iothread();
 }
 
-void kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
+MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
-    DPRINTF("%s\n", __func__);
+    return MEMTXATTRS_UNSPECIFIED;
 }
 
 int kvm_arch_process_async_events(CPUState *cs)
@@ -206,10 +239,10 @@
 }
 
 #define MIPS_CP0_32(_R, _S)                                     \
-    (KVM_REG_MIPS | KVM_REG_SIZE_U32 | 0x10000 | (8 * (_R) + (_S)))
+    (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U32 | (8 * (_R) + (_S)))
 
 #define MIPS_CP0_64(_R, _S)                                     \
-    (KVM_REG_MIPS | KVM_REG_SIZE_U64 | 0x10000 | (8 * (_R) + (_S)))
+    (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U64 | (8 * (_R) + (_S)))
 
 #define KVM_REG_MIPS_CP0_INDEX          MIPS_CP0_32(0, 0)
 #define KVM_REG_MIPS_CP0_CONTEXT        MIPS_CP0_64(4, 0)
@@ -224,26 +257,32 @@
 #define KVM_REG_MIPS_CP0_STATUS         MIPS_CP0_32(12, 0)
 #define KVM_REG_MIPS_CP0_CAUSE          MIPS_CP0_32(13, 0)
 #define KVM_REG_MIPS_CP0_EPC            MIPS_CP0_64(14, 0)
+#define KVM_REG_MIPS_CP0_PRID           MIPS_CP0_32(15, 0)
+#define KVM_REG_MIPS_CP0_CONFIG         MIPS_CP0_32(16, 0)
+#define KVM_REG_MIPS_CP0_CONFIG1        MIPS_CP0_32(16, 1)
+#define KVM_REG_MIPS_CP0_CONFIG2        MIPS_CP0_32(16, 2)
+#define KVM_REG_MIPS_CP0_CONFIG3        MIPS_CP0_32(16, 3)
+#define KVM_REG_MIPS_CP0_CONFIG4        MIPS_CP0_32(16, 4)
+#define KVM_REG_MIPS_CP0_CONFIG5        MIPS_CP0_32(16, 5)
 #define KVM_REG_MIPS_CP0_ERROREPC       MIPS_CP0_64(30, 0)
 
-/* CP0_Count control */
-#define KVM_REG_MIPS_COUNT_CTL          (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-                                         0x20000 | 0)
-#define KVM_REG_MIPS_COUNT_CTL_DC       0x00000001      /* master disable */
-/* CP0_Count resume monotonic nanoseconds */
-#define KVM_REG_MIPS_COUNT_RESUME       (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-                                         0x20000 | 1)
-/* CP0_Count rate in Hz */
-#define KVM_REG_MIPS_COUNT_HZ           (KVM_REG_MIPS | KVM_REG_SIZE_U64 | \
-                                         0x20000 | 2)
-
 static inline int kvm_mips_put_one_reg(CPUState *cs, uint64_t reg_id,
                                        int32_t *addr)
 {
-    uint64_t val64 = *addr;
     struct kvm_one_reg cp0reg = {
         .id = reg_id,
-        .addr = (uintptr_t)&val64
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &cp0reg);
+}
+
+static inline int kvm_mips_put_one_ureg(CPUState *cs, uint64_t reg_id,
+                                        uint32_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
     };
 
     return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &cp0reg);
@@ -262,7 +301,18 @@
 }
 
 static inline int kvm_mips_put_one_reg64(CPUState *cs, uint64_t reg_id,
-                                         uint64_t *addr)
+                                         int64_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &cp0reg);
+}
+
+static inline int kvm_mips_put_one_ureg64(CPUState *cs, uint64_t reg_id,
+                                          uint64_t *addr)
 {
     struct kvm_one_reg cp0reg = {
         .id = reg_id,
@@ -275,21 +325,26 @@
 static inline int kvm_mips_get_one_reg(CPUState *cs, uint64_t reg_id,
                                        int32_t *addr)
 {
-    int ret;
-    uint64_t val64 = 0;
     struct kvm_one_reg cp0reg = {
         .id = reg_id,
-        .addr = (uintptr_t)&val64
+        .addr = (uintptr_t)addr
     };
 
-    ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
-    if (ret >= 0) {
-        *addr = val64;
-    }
-    return ret;
+    return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
 }
 
-static inline int kvm_mips_get_one_ulreg(CPUState *cs, uint64 reg_id,
+static inline int kvm_mips_get_one_ureg(CPUState *cs, uint64_t reg_id,
+                                        uint32_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
+}
+
+static inline int kvm_mips_get_one_ulreg(CPUState *cs, uint64_t reg_id,
                                          target_ulong *addr)
 {
     int ret;
@@ -306,8 +361,8 @@
     return ret;
 }
 
-static inline int kvm_mips_get_one_reg64(CPUState *cs, uint64 reg_id,
-                                         uint64_t *addr)
+static inline int kvm_mips_get_one_reg64(CPUState *cs, uint64_t reg_id,
+                                         int64_t *addr)
 {
     struct kvm_one_reg cp0reg = {
         .id = reg_id,
@@ -317,6 +372,50 @@
     return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
 }
 
+static inline int kvm_mips_get_one_ureg64(CPUState *cs, uint64_t reg_id,
+                                          uint64_t *addr)
+{
+    struct kvm_one_reg cp0reg = {
+        .id = reg_id,
+        .addr = (uintptr_t)addr
+    };
+
+    return kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &cp0reg);
+}
+
+#define KVM_REG_MIPS_CP0_CONFIG_MASK    (1U << CP0C0_M)
+#define KVM_REG_MIPS_CP0_CONFIG1_MASK   ((1U << CP0C1_M) | \
+                                         (1U << CP0C1_FP))
+#define KVM_REG_MIPS_CP0_CONFIG2_MASK   (1U << CP0C2_M)
+#define KVM_REG_MIPS_CP0_CONFIG3_MASK   ((1U << CP0C3_M) | \
+                                         (1U << CP0C3_MSAP))
+#define KVM_REG_MIPS_CP0_CONFIG4_MASK   (1U << CP0C4_M)
+#define KVM_REG_MIPS_CP0_CONFIG5_MASK   ((1U << CP0C5_MSAEn) | \
+                                         (1U << CP0C5_UFE) | \
+                                         (1U << CP0C5_FRE) | \
+                                         (1U << CP0C5_UFR))
+
+static inline int kvm_mips_change_one_reg(CPUState *cs, uint64_t reg_id,
+                                          int32_t *addr, int32_t mask)
+{
+    int err;
+    int32_t tmp, change;
+
+    err = kvm_mips_get_one_reg(cs, reg_id, &tmp);
+    if (err < 0) {
+        return err;
+    }
+
+    /* only change bits in mask */
+    change = (*addr ^ tmp) & mask;
+    if (!change) {
+        return 0;
+    }
+
+    tmp = tmp ^ change;
+    return kvm_mips_put_one_reg(cs, reg_id, &tmp);
+}
+
 /*
  * We freeze the KVM timer when either the VM clock is stopped or the state is
  * saved (the state is dirty).
@@ -334,13 +433,13 @@
     int err, ret = 0;
 
     /* freeze KVM timer */
-    err = kvm_mips_get_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+    err = kvm_mips_get_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
     if (err < 0) {
         DPRINTF("%s: Failed to get COUNT_CTL (%d)\n", __func__, err);
         ret = err;
     } else if (!(count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) {
         count_ctl |= KVM_REG_MIPS_COUNT_CTL_DC;
-        err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+        err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
         if (err < 0) {
             DPRINTF("%s: Failed to set COUNT_CTL.DC=1 (%d)\n", __func__, err);
             ret = err;
@@ -376,14 +475,14 @@
     int err_dc, err, ret = 0;
 
     /* check the timer is frozen */
-    err_dc = kvm_mips_get_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+    err_dc = kvm_mips_get_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
     if (err_dc < 0) {
         DPRINTF("%s: Failed to get COUNT_CTL (%d)\n", __func__, err_dc);
         ret = err_dc;
     } else if (!(count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) {
         /* freeze timer (sets COUNT_RESUME for us) */
         count_ctl |= KVM_REG_MIPS_COUNT_CTL_DC;
-        err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+        err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
         if (err < 0) {
             DPRINTF("%s: Failed to set COUNT_CTL.DC=1 (%d)\n", __func__, err);
             ret = err;
@@ -407,7 +506,7 @@
     /* resume KVM timer */
     if (err_dc >= 0) {
         count_ctl &= ~KVM_REG_MIPS_COUNT_CTL_DC;
-        err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
+        err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_CTL, &count_ctl);
         if (err < 0) {
             DPRINTF("%s: Failed to set COUNT_CTL.DC=0 (%d)\n", __func__, err);
             ret = err;
@@ -439,9 +538,9 @@
         }
     } else {
         /* Set clock restore time to now */
-        count_resume = get_clock();
-        ret = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_COUNT_RESUME,
-                                     &count_resume);
+        count_resume = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+        ret = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_COUNT_RESUME,
+                                      &count_resume);
         if (ret < 0) {
             fprintf(stderr, "Failed setting COUNT_RESUME\n");
             return;
@@ -456,6 +555,167 @@
     }
 }
 
+static int kvm_mips_put_fpu_registers(CPUState *cs, int level)
+{
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
+    int err, ret = 0;
+    unsigned int i;
+
+    /* Only put FPU state if we're emulating a CPU with an FPU */
+    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
+        /* FPU Control Registers */
+        if (level == KVM_PUT_FULL_STATE) {
+            err = kvm_mips_put_one_ureg(cs, KVM_REG_MIPS_FCR_IR,
+                                        &env->active_fpu.fcr0);
+            if (err < 0) {
+                DPRINTF("%s: Failed to put FCR_IR (%d)\n", __func__, err);
+                ret = err;
+            }
+        }
+        err = kvm_mips_put_one_ureg(cs, KVM_REG_MIPS_FCR_CSR,
+                                    &env->active_fpu.fcr31);
+        if (err < 0) {
+            DPRINTF("%s: Failed to put FCR_CSR (%d)\n", __func__, err);
+            ret = err;
+        }
+
+        /*
+         * FPU register state is a subset of MSA vector state, so don't put FPU
+         * registers if we're emulating a CPU with MSA.
+         */
+        if (!(env->CP0_Config3 & (1 << CP0C3_MSAP))) {
+            /* Floating point registers */
+            for (i = 0; i < 32; ++i) {
+                if (env->CP0_Status & (1 << CP0St_FR)) {
+                    err = kvm_mips_put_one_ureg64(cs, KVM_REG_MIPS_FPR_64(i),
+                                                  &env->active_fpu.fpr[i].d);
+                } else {
+                    err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FPR_32(i),
+                                    &env->active_fpu.fpr[i].w[FP_ENDIAN_IDX]);
+                }
+                if (err < 0) {
+                    DPRINTF("%s: Failed to put FPR%u (%d)\n", __func__, i, err);
+                    ret = err;
+                }
+            }
+        }
+    }
+
+    /* Only put MSA state if we're emulating a CPU with MSA */
+    if (env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        /* MSA Control Registers */
+        if (level == KVM_PUT_FULL_STATE) {
+            err = kvm_mips_put_one_reg(cs, KVM_REG_MIPS_MSA_IR,
+                                       &env->msair);
+            if (err < 0) {
+                DPRINTF("%s: Failed to put MSA_IR (%d)\n", __func__, err);
+                ret = err;
+            }
+        }
+        err = kvm_mips_put_one_reg(cs, KVM_REG_MIPS_MSA_CSR,
+                                   &env->active_tc.msacsr);
+        if (err < 0) {
+            DPRINTF("%s: Failed to put MSA_CSR (%d)\n", __func__, err);
+            ret = err;
+        }
+
+        /* Vector registers (includes FP registers) */
+        for (i = 0; i < 32; ++i) {
+            /* Big endian MSA not supported by QEMU yet anyway */
+            err = kvm_mips_put_one_reg64(cs, KVM_REG_MIPS_VEC_128(i),
+                                         env->active_fpu.fpr[i].wr.d);
+            if (err < 0) {
+                DPRINTF("%s: Failed to put VEC%u (%d)\n", __func__, i, err);
+                ret = err;
+            }
+        }
+    }
+
+    return ret;
+}
+
+static int kvm_mips_get_fpu_registers(CPUState *cs)
+{
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
+    int err, ret = 0;
+    unsigned int i;
+
+    /* Only get FPU state if we're emulating a CPU with an FPU */
+    if (env->CP0_Config1 & (1 << CP0C1_FP)) {
+        /* FPU Control Registers */
+        err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FCR_IR,
+                                    &env->active_fpu.fcr0);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get FCR_IR (%d)\n", __func__, err);
+            ret = err;
+        }
+        err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FCR_CSR,
+                                    &env->active_fpu.fcr31);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get FCR_CSR (%d)\n", __func__, err);
+            ret = err;
+        } else {
+            restore_fp_status(env);
+        }
+
+        /*
+         * FPU register state is a subset of MSA vector state, so don't save FPU
+         * registers if we're emulating a CPU with MSA.
+         */
+        if (!(env->CP0_Config3 & (1 << CP0C3_MSAP))) {
+            /* Floating point registers */
+            for (i = 0; i < 32; ++i) {
+                if (env->CP0_Status & (1 << CP0St_FR)) {
+                    err = kvm_mips_get_one_ureg64(cs, KVM_REG_MIPS_FPR_64(i),
+                                                  &env->active_fpu.fpr[i].d);
+                } else {
+                    err = kvm_mips_get_one_ureg(cs, KVM_REG_MIPS_FPR_32(i),
+                                    &env->active_fpu.fpr[i].w[FP_ENDIAN_IDX]);
+                }
+                if (err < 0) {
+                    DPRINTF("%s: Failed to get FPR%u (%d)\n", __func__, i, err);
+                    ret = err;
+                }
+            }
+        }
+    }
+
+    /* Only get MSA state if we're emulating a CPU with MSA */
+    if (env->CP0_Config3 & (1 << CP0C3_MSAP)) {
+        /* MSA Control Registers */
+        err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_MSA_IR,
+                                   &env->msair);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get MSA_IR (%d)\n", __func__, err);
+            ret = err;
+        }
+        err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_MSA_CSR,
+                                   &env->active_tc.msacsr);
+        if (err < 0) {
+            DPRINTF("%s: Failed to get MSA_CSR (%d)\n", __func__, err);
+            ret = err;
+        } else {
+            restore_msa_fp_status(env);
+        }
+
+        /* Vector registers (includes FP registers) */
+        for (i = 0; i < 32; ++i) {
+            /* Big endian MSA not supported by QEMU yet anyway */
+            err = kvm_mips_get_one_reg64(cs, KVM_REG_MIPS_VEC_128(i),
+                                         env->active_fpu.fpr[i].wr.d);
+            if (err < 0) {
+                DPRINTF("%s: Failed to get VEC%u (%d)\n", __func__, i, err);
+                ret = err;
+            }
+        }
+    }
+
+    return ret;
+}
+
+
 static int kvm_mips_put_cp0_registers(CPUState *cs, int level)
 {
     MIPSCPU *cpu = MIPS_CPU(cs);
@@ -534,6 +794,53 @@
         DPRINTF("%s: Failed to put CP0_EPC (%d)\n", __func__, err);
         ret = err;
     }
+    err = kvm_mips_put_one_reg(cs, KVM_REG_MIPS_CP0_PRID, &env->CP0_PRid);
+    if (err < 0) {
+        DPRINTF("%s: Failed to put CP0_PRID (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG,
+                                  &env->CP0_Config0,
+                                  KVM_REG_MIPS_CP0_CONFIG_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG1,
+                                  &env->CP0_Config1,
+                                  KVM_REG_MIPS_CP0_CONFIG1_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG1 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG2,
+                                  &env->CP0_Config2,
+                                  KVM_REG_MIPS_CP0_CONFIG2_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG2 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG3,
+                                  &env->CP0_Config3,
+                                  KVM_REG_MIPS_CP0_CONFIG3_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG3 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG4,
+                                  &env->CP0_Config4,
+                                  KVM_REG_MIPS_CP0_CONFIG4_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG4 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_change_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG5,
+                                  &env->CP0_Config5,
+                                  KVM_REG_MIPS_CP0_CONFIG5_MASK);
+    if (err < 0) {
+        DPRINTF("%s: Failed to change CP0_CONFIG5 (%d)\n", __func__, err);
+        ret = err;
+    }
     err = kvm_mips_put_one_ulreg(cs, KVM_REG_MIPS_CP0_ERROREPC,
                                  &env->CP0_ErrorEPC);
     if (err < 0) {
@@ -620,6 +927,41 @@
         DPRINTF("%s: Failed to get CP0_EPC (%d)\n", __func__, err);
         ret = err;
     }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_PRID, &env->CP0_PRid);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_PRID (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG, &env->CP0_Config0);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG1, &env->CP0_Config1);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG1 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG2, &env->CP0_Config2);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG2 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG3, &env->CP0_Config3);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG3 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG4, &env->CP0_Config4);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG4 (%d)\n", __func__, err);
+        ret = err;
+    }
+    err = kvm_mips_get_one_reg(cs, KVM_REG_MIPS_CP0_CONFIG5, &env->CP0_Config5);
+    if (err < 0) {
+        DPRINTF("%s: Failed to get CP0_CONFIG5 (%d)\n", __func__, err);
+        ret = err;
+    }
     err = kvm_mips_get_one_ulreg(cs, KVM_REG_MIPS_CP0_ERROREPC,
                                  &env->CP0_ErrorEPC);
     if (err < 0) {
@@ -640,12 +982,12 @@
 
     /* Set the registers based on QEMU's view of things */
     for (i = 0; i < 32; i++) {
-        regs.gpr[i] = env->active_tc.gpr[i];
+        regs.gpr[i] = (int64_t)(target_long)env->active_tc.gpr[i];
     }
 
-    regs.hi = env->active_tc.HI[0];
-    regs.lo = env->active_tc.LO[0];
-    regs.pc = env->active_tc.PC;
+    regs.hi = (int64_t)(target_long)env->active_tc.HI[0];
+    regs.lo = (int64_t)(target_long)env->active_tc.LO[0];
+    regs.pc = (int64_t)(target_long)env->active_tc.PC;
 
     ret = kvm_vcpu_ioctl(cs, KVM_SET_REGS, &regs);
 
@@ -658,6 +1000,11 @@
         return ret;
     }
 
+    ret = kvm_mips_put_fpu_registers(cs, level);
+    if (ret < 0) {
+        return ret;
+    }
+
     return ret;
 }
 
@@ -685,6 +1032,29 @@
     env->active_tc.PC = regs.pc;
 
     kvm_mips_get_cp0_registers(cs);
+    kvm_mips_get_fpu_registers(cs);
 
     return ret;
 }
+
+int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
+                             uint64_t address, uint32_t data, PCIDevice *dev)
+{
+    return 0;
+}
+
+int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
+                                int vector, PCIDevice *dev)
+{
+    return 0;
+}
+
+int kvm_arch_release_virq_post(int virq)
+{
+    return 0;
+}
+
+int kvm_arch_msi_data_to_gsi(uint32_t data)
+{
+    abort();
+}
diff --git a/target-mips/kvm_mips.h b/target-mips/kvm_mips.h
index 54f5965..ae957f3 100644
--- a/target-mips/kvm_mips.h
+++ b/target-mips/kvm_mips.h
@@ -9,8 +9,8 @@
  * Authors: Sanjay Lal <sanjayl@kymasys.com>
 */
 
-#ifndef __KVM_MIPS_H__
-#define __KVM_MIPS_H__
+#ifndef KVM_MIPS_H
+#define KVM_MIPS_H
 
 /**
  * kvm_mips_reset_vcpu:
@@ -23,4 +23,4 @@
 int kvm_mips_set_interrupt(MIPSCPU *cpu, int irq, int level);
 int kvm_mips_set_ipi_interrupt(MIPSCPU *cpu, int irq, int level);
 
-#endif /* __KVM_MIPS_H__ */
+#endif /* KVM_MIPS_H */
diff --git a/target-mips/lmi_helper.c b/target-mips/lmi_helper.c
index bbfcd59..fb1245b 100644
--- a/target-mips/lmi_helper.c
+++ b/target-mips/lmi_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/target-mips/machine.c b/target-mips/machine.c
index 0ba7d73..a27f2f1 100644
--- a/target-mips/machine.c
+++ b/target-mips/machine.c
@@ -1,339 +1,303 @@
-#include "hw/hw.h"
-#include "hw/boards.h"
-
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "cpu.h"
+#include "hw/hw.h"
+#include "cpu.h"
+#include "migration/cpu.h"
 
-static void save_tc(QEMUFile *f, TCState *tc)
+static int cpu_post_load(void *opaque, int version_id)
 {
-    int i;
+    MIPSCPU *cpu = opaque;
+    CPUMIPSState *env = &cpu->env;
 
-    /* Save active TC */
-    for(i = 0; i < 32; i++)
-        qemu_put_betls(f, &tc->gpr[i]);
-    qemu_put_betls(f, &tc->PC);
-    for(i = 0; i < MIPS_DSP_ACC; i++)
-        qemu_put_betls(f, &tc->HI[i]);
-    for(i = 0; i < MIPS_DSP_ACC; i++)
-        qemu_put_betls(f, &tc->LO[i]);
-    for(i = 0; i < MIPS_DSP_ACC; i++)
-        qemu_put_betls(f, &tc->ACX[i]);
-    qemu_put_betls(f, &tc->DSPControl);
-    qemu_put_sbe32s(f, &tc->CP0_TCStatus);
-    qemu_put_sbe32s(f, &tc->CP0_TCBind);
-    qemu_put_betls(f, &tc->CP0_TCHalt);
-    qemu_put_betls(f, &tc->CP0_TCContext);
-    qemu_put_betls(f, &tc->CP0_TCSchedule);
-    qemu_put_betls(f, &tc->CP0_TCScheFBack);
-    qemu_put_sbe32s(f, &tc->CP0_Debug_tcstatus);
-    qemu_put_betls(f, &tc->CP0_UserLocal);
-}
+    restore_fp_status(env);
+    restore_msa_fp_status(env);
+    compute_hflags(env);
+    restore_pamask(env);
 
-static void save_fpu(QEMUFile *f, CPUMIPSFPUContext *fpu)
-{
-    int i;
-
-    for(i = 0; i < 32; i++)
-        qemu_put_be64s(f, &fpu->fpr[i].d);
-    qemu_put_s8s(f, &fpu->fp_status.float_detect_tininess);
-    qemu_put_s8s(f, &fpu->fp_status.float_rounding_mode);
-    qemu_put_s8s(f, &fpu->fp_status.float_exception_flags);
-    qemu_put_be32s(f, &fpu->fcr0);
-    qemu_put_be32s(f, &fpu->fcr31);
-}
-
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    CPUMIPSState *env = opaque;
-    int i;
-
-    /* Save active TC */
-    save_tc(f, &env->active_tc);
-
-    /* Save active FPU */
-    save_fpu(f, &env->active_fpu);
-
-    /* Save MVP */
-    qemu_put_sbe32s(f, &env->mvp->CP0_MVPControl);
-    qemu_put_sbe32s(f, &env->mvp->CP0_MVPConf0);
-    qemu_put_sbe32s(f, &env->mvp->CP0_MVPConf1);
-
-    /* Save TLB */
-    qemu_put_be32s(f, &env->tlb->nb_tlb);
-    qemu_put_be32s(f, &env->tlb->tlb_in_use);
-    for(i = 0; i < MIPS_TLB_MAX; i++) {
-        uint16_t flags = ((env->tlb->mmu.r4k.tlb[i].EHINV << 15) |
-                          (env->tlb->mmu.r4k.tlb[i].RI1 << 14) |
-                          (env->tlb->mmu.r4k.tlb[i].RI0 << 13) |
-                          (env->tlb->mmu.r4k.tlb[i].XI1 << 12) |
-                          (env->tlb->mmu.r4k.tlb[i].XI0 << 11) |
-                          (env->tlb->mmu.r4k.tlb[i].G << 10) |
-                          (env->tlb->mmu.r4k.tlb[i].C0 << 7) |
-                          (env->tlb->mmu.r4k.tlb[i].C1 << 4) |
-                          (env->tlb->mmu.r4k.tlb[i].V0 << 3) |
-                          (env->tlb->mmu.r4k.tlb[i].V1 << 2) |
-                          (env->tlb->mmu.r4k.tlb[i].D0 << 1) |
-                          (env->tlb->mmu.r4k.tlb[i].D1 << 0));
-        uint8_t asid;
-
-        qemu_put_betls(f, &env->tlb->mmu.r4k.tlb[i].VPN);
-        qemu_put_be32s(f, &env->tlb->mmu.r4k.tlb[i].PageMask);
-        asid = env->tlb->mmu.r4k.tlb[i].ASID;
-        qemu_put_8s(f, &asid);
-        qemu_put_be16s(f, &flags);
-        qemu_put_betls(f, &env->tlb->mmu.r4k.tlb[i].PFN[0]);
-        qemu_put_betls(f, &env->tlb->mmu.r4k.tlb[i].PFN[1]);
-    }
-
-    /* Save CPU metastate */
-    qemu_put_be32s(f, &env->current_tc);
-    qemu_put_be32s(f, &env->current_fpu);
-    qemu_put_sbe32s(f, &env->error_code);
-    qemu_put_be32s(f, &env->hflags);
-    qemu_put_betls(f, &env->btarget);
-    i = env->bcond;
-    qemu_put_sbe32s(f, &i);
-
-    /* Save remaining CP1 registers */
-    qemu_put_sbe32s(f, &env->CP0_Index);
-    qemu_put_sbe32s(f, &env->CP0_Random);
-    qemu_put_sbe32s(f, &env->CP0_VPEControl);
-    qemu_put_sbe32s(f, &env->CP0_VPEConf0);
-    qemu_put_sbe32s(f, &env->CP0_VPEConf1);
-    qemu_put_betls(f, &env->CP0_YQMask);
-    qemu_put_betls(f, &env->CP0_VPESchedule);
-    qemu_put_betls(f, &env->CP0_VPEScheFBack);
-    qemu_put_sbe32s(f, &env->CP0_VPEOpt);
-    qemu_put_betls(f, &env->CP0_EntryLo0);
-    qemu_put_betls(f, &env->CP0_EntryLo1);
-    qemu_put_betls(f, &env->CP0_Context);
-    qemu_put_sbe32s(f, &env->CP0_PageMask);
-    qemu_put_sbe32s(f, &env->CP0_PageGrain);
-    qemu_put_sbe32s(f, &env->CP0_Wired);
-    qemu_put_sbe32s(f, &env->CP0_SRSConf0);
-    qemu_put_sbe32s(f, &env->CP0_SRSConf1);
-    qemu_put_sbe32s(f, &env->CP0_SRSConf2);
-    qemu_put_sbe32s(f, &env->CP0_SRSConf3);
-    qemu_put_sbe32s(f, &env->CP0_SRSConf4);
-    qemu_put_sbe32s(f, &env->CP0_HWREna);
-    qemu_put_betls(f, &env->CP0_BadVAddr);
-    qemu_put_be32s(f, &env->CP0_BadInstr);
-    qemu_put_be32s(f, &env->CP0_BadInstrP);
-    qemu_put_sbe32s(f, &env->CP0_Count);
-    qemu_put_betls(f, &env->CP0_EntryHi);
-    qemu_put_sbe32s(f, &env->CP0_Compare);
-    qemu_put_sbe32s(f, &env->CP0_Status);
-    qemu_put_sbe32s(f, &env->CP0_IntCtl);
-    qemu_put_sbe32s(f, &env->CP0_SRSCtl);
-    qemu_put_sbe32s(f, &env->CP0_SRSMap);
-    qemu_put_sbe32s(f, &env->CP0_Cause);
-    qemu_put_betls(f, &env->CP0_EPC);
-    qemu_put_sbe32s(f, &env->CP0_PRid);
-    qemu_put_sbe32s(f, &env->CP0_EBase);
-    qemu_put_sbe32s(f, &env->CP0_Config0);
-    qemu_put_sbe32s(f, &env->CP0_Config1);
-    qemu_put_sbe32s(f, &env->CP0_Config2);
-    qemu_put_sbe32s(f, &env->CP0_Config3);
-    qemu_put_sbe32s(f, &env->CP0_Config6);
-    qemu_put_sbe32s(f, &env->CP0_Config7);
-    qemu_put_betls(f, &env->lladdr);
-    for(i = 0; i < 8; i++)
-        qemu_put_betls(f, &env->CP0_WatchLo[i]);
-    for(i = 0; i < 8; i++)
-        qemu_put_sbe32s(f, &env->CP0_WatchHi[i]);
-    qemu_put_betls(f, &env->CP0_XContext);
-    qemu_put_sbe32s(f, &env->CP0_Framemask);
-    qemu_put_sbe32s(f, &env->CP0_Debug);
-    qemu_put_betls(f, &env->CP0_DEPC);
-    qemu_put_sbe32s(f, &env->CP0_Performance0);
-    qemu_put_sbe32s(f, &env->CP0_TagLo);
-    qemu_put_sbe32s(f, &env->CP0_DataLo);
-    qemu_put_sbe32s(f, &env->CP0_TagHi);
-    qemu_put_sbe32s(f, &env->CP0_DataHi);
-    qemu_put_betls(f, &env->CP0_ErrorEPC);
-    qemu_put_sbe32s(f, &env->CP0_DESAVE);
-    for (i = 0; i < MIPS_KSCRATCH_NUM; i++) {
-        qemu_put_betls(f, &env->CP0_KScratch[i]);
-    }
-
-    /* Save inactive TC state */
-    for (i = 0; i < MIPS_SHADOW_SET_MAX; i++)
-        save_tc(f, &env->tcs[i]);
-    for (i = 0; i < MIPS_FPU_MAX; i++)
-        save_fpu(f, &env->fpus[i]);
-}
-
-static void load_tc(QEMUFile *f, TCState *tc, int version_id)
-{
-    int i;
-
-    /* Save active TC */
-    for(i = 0; i < 32; i++)
-        qemu_get_betls(f, &tc->gpr[i]);
-    qemu_get_betls(f, &tc->PC);
-    for(i = 0; i < MIPS_DSP_ACC; i++)
-        qemu_get_betls(f, &tc->HI[i]);
-    for(i = 0; i < MIPS_DSP_ACC; i++)
-        qemu_get_betls(f, &tc->LO[i]);
-    for(i = 0; i < MIPS_DSP_ACC; i++)
-        qemu_get_betls(f, &tc->ACX[i]);
-    qemu_get_betls(f, &tc->DSPControl);
-    qemu_get_sbe32s(f, &tc->CP0_TCStatus);
-    qemu_get_sbe32s(f, &tc->CP0_TCBind);
-    qemu_get_betls(f, &tc->CP0_TCHalt);
-    qemu_get_betls(f, &tc->CP0_TCContext);
-    qemu_get_betls(f, &tc->CP0_TCSchedule);
-    qemu_get_betls(f, &tc->CP0_TCScheFBack);
-    qemu_get_sbe32s(f, &tc->CP0_Debug_tcstatus);
-    if (version_id >= 4) {
-        qemu_get_betls(f, &tc->CP0_UserLocal);
-    }
-}
-
-static void load_fpu(QEMUFile *f, CPUMIPSFPUContext *fpu)
-{
-    int i;
-
-    for(i = 0; i < 32; i++)
-        qemu_get_be64s(f, &fpu->fpr[i].d);
-    qemu_get_s8s(f, &fpu->fp_status.float_detect_tininess);
-    qemu_get_s8s(f, &fpu->fp_status.float_rounding_mode);
-    qemu_get_s8s(f, &fpu->fp_status.float_exception_flags);
-    qemu_get_be32s(f, &fpu->fcr0);
-    qemu_get_be32s(f, &fpu->fcr31);
-}
-
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
-{
-    CPUMIPSState *env = opaque;
-    MIPSCPU *cpu = mips_env_get_cpu(env);
-    int i;
-
-    if (version_id < 3) {
-        return -EINVAL;
-    }
-
-    /* Load active TC */
-    load_tc(f, &env->active_tc, version_id);
-
-    /* Load active FPU */
-    load_fpu(f, &env->active_fpu);
-
-    /* Load MVP */
-    qemu_get_sbe32s(f, &env->mvp->CP0_MVPControl);
-    qemu_get_sbe32s(f, &env->mvp->CP0_MVPConf0);
-    qemu_get_sbe32s(f, &env->mvp->CP0_MVPConf1);
-
-    /* Load TLB */
-    qemu_get_be32s(f, &env->tlb->nb_tlb);
-    qemu_get_be32s(f, &env->tlb->tlb_in_use);
-    for(i = 0; i < MIPS_TLB_MAX; i++) {
-        uint16_t flags;
-        uint8_t asid;
-
-        qemu_get_betls(f, &env->tlb->mmu.r4k.tlb[i].VPN);
-        qemu_get_be32s(f, &env->tlb->mmu.r4k.tlb[i].PageMask);
-        qemu_get_8s(f, &asid);
-        env->tlb->mmu.r4k.tlb[i].ASID = asid;
-        qemu_get_be16s(f, &flags);
-        env->tlb->mmu.r4k.tlb[i].G = (flags >> 10) & 1;
-        env->tlb->mmu.r4k.tlb[i].C0 = (flags >> 7) & 3;
-        env->tlb->mmu.r4k.tlb[i].C1 = (flags >> 4) & 3;
-        env->tlb->mmu.r4k.tlb[i].V0 = (flags >> 3) & 1;
-        env->tlb->mmu.r4k.tlb[i].V1 = (flags >> 2) & 1;
-        env->tlb->mmu.r4k.tlb[i].D0 = (flags >> 1) & 1;
-        env->tlb->mmu.r4k.tlb[i].D1 = (flags >> 0) & 1;
-        if (version_id >= 5) {
-            env->tlb->mmu.r4k.tlb[i].EHINV = (flags >> 15) & 1;
-            env->tlb->mmu.r4k.tlb[i].RI1 = (flags >> 14) & 1;
-            env->tlb->mmu.r4k.tlb[i].RI0 = (flags >> 13) & 1;
-            env->tlb->mmu.r4k.tlb[i].XI1 = (flags >> 12) & 1;
-            env->tlb->mmu.r4k.tlb[i].XI0 = (flags >> 11) & 1;
-        }
-        qemu_get_betls(f, &env->tlb->mmu.r4k.tlb[i].PFN[0]);
-        qemu_get_betls(f, &env->tlb->mmu.r4k.tlb[i].PFN[1]);
-    }
-
-    /* Load CPU metastate */
-    qemu_get_be32s(f, &env->current_tc);
-    qemu_get_be32s(f, &env->current_fpu);
-    qemu_get_sbe32s(f, &env->error_code);
-    qemu_get_be32s(f, &env->hflags);
-    qemu_get_betls(f, &env->btarget);
-    qemu_get_sbe32s(f, &i);
-    env->bcond = i;
-
-    /* Load remaining CP1 registers */
-    qemu_get_sbe32s(f, &env->CP0_Index);
-    qemu_get_sbe32s(f, &env->CP0_Random);
-    qemu_get_sbe32s(f, &env->CP0_VPEControl);
-    qemu_get_sbe32s(f, &env->CP0_VPEConf0);
-    qemu_get_sbe32s(f, &env->CP0_VPEConf1);
-    qemu_get_betls(f, &env->CP0_YQMask);
-    qemu_get_betls(f, &env->CP0_VPESchedule);
-    qemu_get_betls(f, &env->CP0_VPEScheFBack);
-    qemu_get_sbe32s(f, &env->CP0_VPEOpt);
-    qemu_get_betls(f, &env->CP0_EntryLo0);
-    qemu_get_betls(f, &env->CP0_EntryLo1);
-    qemu_get_betls(f, &env->CP0_Context);
-    qemu_get_sbe32s(f, &env->CP0_PageMask);
-    qemu_get_sbe32s(f, &env->CP0_PageGrain);
-    qemu_get_sbe32s(f, &env->CP0_Wired);
-    qemu_get_sbe32s(f, &env->CP0_SRSConf0);
-    qemu_get_sbe32s(f, &env->CP0_SRSConf1);
-    qemu_get_sbe32s(f, &env->CP0_SRSConf2);
-    qemu_get_sbe32s(f, &env->CP0_SRSConf3);
-    qemu_get_sbe32s(f, &env->CP0_SRSConf4);
-    qemu_get_sbe32s(f, &env->CP0_HWREna);
-    qemu_get_betls(f, &env->CP0_BadVAddr);
-    qemu_get_sbe32s(f, &env->CP0_Count);
-    qemu_get_betls(f, &env->CP0_EntryHi);
-    qemu_get_sbe32s(f, &env->CP0_Compare);
-    qemu_get_sbe32s(f, &env->CP0_Status);
-    qemu_get_sbe32s(f, &env->CP0_IntCtl);
-    qemu_get_sbe32s(f, &env->CP0_SRSCtl);
-    qemu_get_sbe32s(f, &env->CP0_SRSMap);
-    qemu_get_sbe32s(f, &env->CP0_Cause);
-    qemu_get_betls(f, &env->CP0_EPC);
-    qemu_get_sbe32s(f, &env->CP0_PRid);
-    qemu_get_sbe32s(f, &env->CP0_EBase);
-    qemu_get_sbe32s(f, &env->CP0_Config0);
-    qemu_get_sbe32s(f, &env->CP0_Config1);
-    qemu_get_sbe32s(f, &env->CP0_Config2);
-    qemu_get_sbe32s(f, &env->CP0_Config3);
-    qemu_get_sbe32s(f, &env->CP0_Config6);
-    qemu_get_sbe32s(f, &env->CP0_Config7);
-    qemu_get_betls(f, &env->lladdr);
-    for(i = 0; i < 8; i++)
-        qemu_get_betls(f, &env->CP0_WatchLo[i]);
-    for(i = 0; i < 8; i++)
-        qemu_get_sbe32s(f, &env->CP0_WatchHi[i]);
-    qemu_get_betls(f, &env->CP0_XContext);
-    qemu_get_sbe32s(f, &env->CP0_Framemask);
-    qemu_get_sbe32s(f, &env->CP0_Debug);
-    qemu_get_betls(f, &env->CP0_DEPC);
-    qemu_get_sbe32s(f, &env->CP0_Performance0);
-    qemu_get_sbe32s(f, &env->CP0_TagLo);
-    qemu_get_sbe32s(f, &env->CP0_DataLo);
-    qemu_get_sbe32s(f, &env->CP0_TagHi);
-    qemu_get_sbe32s(f, &env->CP0_DataHi);
-    qemu_get_betls(f, &env->CP0_ErrorEPC);
-    qemu_get_sbe32s(f, &env->CP0_DESAVE);
-    if (version_id >= 5) {
-        qemu_get_be32s(f, &env->CP0_BadInstr);
-        qemu_get_be32s(f, &env->CP0_BadInstrP);
-        for (i = 0; i < MIPS_KSCRATCH_NUM; i++) {
-            qemu_get_betls(f, &env->CP0_KScratch[i]);
-        }
-    }
-
-    /* Load inactive TC state */
-    for (i = 0; i < MIPS_SHADOW_SET_MAX; i++) {
-        load_tc(f, &env->tcs[i], version_id);
-    }
-    for (i = 0; i < MIPS_FPU_MAX; i++)
-        load_fpu(f, &env->fpus[i]);
-
-    /* XXX: ensure compatibility for halted bit ? */
-    tlb_flush(CPU(cpu), 1);
     return 0;
 }
+
+/* FPU state */
+
+static int get_fpr(QEMUFile *f, void *pv, size_t size)
+{
+    int i;
+    fpr_t *v = pv;
+    /* Restore entire MSA vector register */
+    for (i = 0; i < MSA_WRLEN/64; i++) {
+        qemu_get_sbe64s(f, &v->wr.d[i]);
+    }
+    return 0;
+}
+
+static void put_fpr(QEMUFile *f, void *pv, size_t size)
+{
+    int i;
+    fpr_t *v = pv;
+    /* Save entire MSA vector register */
+    for (i = 0; i < MSA_WRLEN/64; i++) {
+        qemu_put_sbe64s(f, &v->wr.d[i]);
+    }
+}
+
+const VMStateInfo vmstate_info_fpr = {
+    .name = "fpr",
+    .get  = get_fpr,
+    .put  = put_fpr,
+};
+
+#define VMSTATE_FPR_ARRAY_V(_f, _s, _n, _v)                     \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_fpr, fpr_t)
+
+#define VMSTATE_FPR_ARRAY(_f, _s, _n)                           \
+    VMSTATE_FPR_ARRAY_V(_f, _s, _n, 0)
+
+static VMStateField vmstate_fpu_fields[] = {
+    VMSTATE_FPR_ARRAY(fpr, CPUMIPSFPUContext, 32),
+    VMSTATE_UINT32(fcr0, CPUMIPSFPUContext),
+    VMSTATE_UINT32(fcr31, CPUMIPSFPUContext),
+    VMSTATE_END_OF_LIST()
+};
+
+const VMStateDescription vmstate_fpu = {
+    .name = "cpu/fpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = vmstate_fpu_fields
+};
+
+const VMStateDescription vmstate_inactive_fpu = {
+    .name = "cpu/inactive_fpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = vmstate_fpu_fields
+};
+
+/* TC state */
+
+static VMStateField vmstate_tc_fields[] = {
+    VMSTATE_UINTTL_ARRAY(gpr, TCState, 32),
+    VMSTATE_UINTTL(PC, TCState),
+    VMSTATE_UINTTL_ARRAY(HI, TCState, MIPS_DSP_ACC),
+    VMSTATE_UINTTL_ARRAY(LO, TCState, MIPS_DSP_ACC),
+    VMSTATE_UINTTL_ARRAY(ACX, TCState, MIPS_DSP_ACC),
+    VMSTATE_UINTTL(DSPControl, TCState),
+    VMSTATE_INT32(CP0_TCStatus, TCState),
+    VMSTATE_INT32(CP0_TCBind, TCState),
+    VMSTATE_UINTTL(CP0_TCHalt, TCState),
+    VMSTATE_UINTTL(CP0_TCContext, TCState),
+    VMSTATE_UINTTL(CP0_TCSchedule, TCState),
+    VMSTATE_UINTTL(CP0_TCScheFBack, TCState),
+    VMSTATE_INT32(CP0_Debug_tcstatus, TCState),
+    VMSTATE_UINTTL(CP0_UserLocal, TCState),
+    VMSTATE_INT32(msacsr, TCState),
+    VMSTATE_END_OF_LIST()
+};
+
+const VMStateDescription vmstate_tc = {
+    .name = "cpu/tc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = vmstate_tc_fields
+};
+
+const VMStateDescription vmstate_inactive_tc = {
+    .name = "cpu/inactive_tc",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = vmstate_tc_fields
+};
+
+/* MVP state */
+
+const VMStateDescription vmstate_mvp = {
+    .name = "cpu/mvp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(CP0_MVPControl, CPUMIPSMVPContext),
+        VMSTATE_INT32(CP0_MVPConf0, CPUMIPSMVPContext),
+        VMSTATE_INT32(CP0_MVPConf1, CPUMIPSMVPContext),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* TLB state */
+
+static int get_tlb(QEMUFile *f, void *pv, size_t size)
+{
+    r4k_tlb_t *v = pv;
+    uint16_t flags;
+
+    qemu_get_betls(f, &v->VPN);
+    qemu_get_be32s(f, &v->PageMask);
+    qemu_get_be16s(f, &v->ASID);
+    qemu_get_be16s(f, &flags);
+    v->G = (flags >> 10) & 1;
+    v->C0 = (flags >> 7) & 3;
+    v->C1 = (flags >> 4) & 3;
+    v->V0 = (flags >> 3) & 1;
+    v->V1 = (flags >> 2) & 1;
+    v->D0 = (flags >> 1) & 1;
+    v->D1 = (flags >> 0) & 1;
+    v->EHINV = (flags >> 15) & 1;
+    v->RI1 = (flags >> 14) & 1;
+    v->RI0 = (flags >> 13) & 1;
+    v->XI1 = (flags >> 12) & 1;
+    v->XI0 = (flags >> 11) & 1;
+    qemu_get_be64s(f, &v->PFN[0]);
+    qemu_get_be64s(f, &v->PFN[1]);
+
+    return 0;
+}
+
+static void put_tlb(QEMUFile *f, void *pv, size_t size)
+{
+    r4k_tlb_t *v = pv;
+
+    uint16_t asid = v->ASID;
+    uint16_t flags = ((v->EHINV << 15) |
+                      (v->RI1 << 14) |
+                      (v->RI0 << 13) |
+                      (v->XI1 << 12) |
+                      (v->XI0 << 11) |
+                      (v->G << 10) |
+                      (v->C0 << 7) |
+                      (v->C1 << 4) |
+                      (v->V0 << 3) |
+                      (v->V1 << 2) |
+                      (v->D0 << 1) |
+                      (v->D1 << 0));
+
+    qemu_put_betls(f, &v->VPN);
+    qemu_put_be32s(f, &v->PageMask);
+    qemu_put_be16s(f, &asid);
+    qemu_put_be16s(f, &flags);
+    qemu_put_be64s(f, &v->PFN[0]);
+    qemu_put_be64s(f, &v->PFN[1]);
+}
+
+const VMStateInfo vmstate_info_tlb = {
+    .name = "tlb_entry",
+    .get  = get_tlb,
+    .put  = put_tlb,
+};
+
+#define VMSTATE_TLB_ARRAY_V(_f, _s, _n, _v)                     \
+    VMSTATE_ARRAY(_f, _s, _n, _v, vmstate_info_tlb, r4k_tlb_t)
+
+#define VMSTATE_TLB_ARRAY(_f, _s, _n)                           \
+    VMSTATE_TLB_ARRAY_V(_f, _s, _n, 0)
+
+const VMStateDescription vmstate_tlb = {
+    .name = "cpu/tlb",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(nb_tlb, CPUMIPSTLBContext),
+        VMSTATE_UINT32(tlb_in_use, CPUMIPSTLBContext),
+        VMSTATE_TLB_ARRAY(mmu.r4k.tlb, CPUMIPSTLBContext, MIPS_TLB_MAX),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+/* MIPS CPU state */
+
+const VMStateDescription vmstate_mips_cpu = {
+    .name = "cpu",
+    .version_id = 8,
+    .minimum_version_id = 8,
+    .post_load = cpu_post_load,
+    .fields = (VMStateField[]) {
+        /* Active TC */
+        VMSTATE_STRUCT(env.active_tc, MIPSCPU, 1, vmstate_tc, TCState),
+
+        /* Active FPU */
+        VMSTATE_STRUCT(env.active_fpu, MIPSCPU, 1, vmstate_fpu,
+                       CPUMIPSFPUContext),
+
+        /* MVP */
+        VMSTATE_STRUCT_POINTER(env.mvp, MIPSCPU, vmstate_mvp,
+                               CPUMIPSMVPContext),
+
+        /* TLB */
+        VMSTATE_STRUCT_POINTER(env.tlb, MIPSCPU, vmstate_tlb,
+                               CPUMIPSTLBContext),
+
+        /* CPU metastate */
+        VMSTATE_UINT32(env.current_tc, MIPSCPU),
+        VMSTATE_UINT32(env.current_fpu, MIPSCPU),
+        VMSTATE_INT32(env.error_code, MIPSCPU),
+        VMSTATE_UINTTL(env.btarget, MIPSCPU),
+        VMSTATE_UINTTL(env.bcond, MIPSCPU),
+
+        /* Remaining CP0 registers */
+        VMSTATE_INT32(env.CP0_Index, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Random, MIPSCPU),
+        VMSTATE_INT32(env.CP0_VPEControl, MIPSCPU),
+        VMSTATE_INT32(env.CP0_VPEConf0, MIPSCPU),
+        VMSTATE_INT32(env.CP0_VPEConf1, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_YQMask, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_VPESchedule, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_VPEScheFBack, MIPSCPU),
+        VMSTATE_INT32(env.CP0_VPEOpt, MIPSCPU),
+        VMSTATE_UINT64(env.CP0_EntryLo0, MIPSCPU),
+        VMSTATE_UINT64(env.CP0_EntryLo1, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_Context, MIPSCPU),
+        VMSTATE_INT32(env.CP0_PageMask, MIPSCPU),
+        VMSTATE_INT32(env.CP0_PageGrain, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Wired, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSConf0, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSConf1, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSConf2, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSConf3, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSConf4, MIPSCPU),
+        VMSTATE_INT32(env.CP0_HWREna, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_BadVAddr, MIPSCPU),
+        VMSTATE_UINT32(env.CP0_BadInstr, MIPSCPU),
+        VMSTATE_UINT32(env.CP0_BadInstrP, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Count, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_EntryHi, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Compare, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Status, MIPSCPU),
+        VMSTATE_INT32(env.CP0_IntCtl, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSCtl, MIPSCPU),
+        VMSTATE_INT32(env.CP0_SRSMap, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Cause, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_EPC, MIPSCPU),
+        VMSTATE_INT32(env.CP0_PRid, MIPSCPU),
+        VMSTATE_INT32(env.CP0_EBase, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Config0, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Config1, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Config2, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Config3, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Config6, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Config7, MIPSCPU),
+        VMSTATE_UINT64_ARRAY(env.CP0_MAAR, MIPSCPU, MIPS_MAAR_MAX),
+        VMSTATE_INT32(env.CP0_MAARI, MIPSCPU),
+        VMSTATE_UINT64(env.lladdr, MIPSCPU),
+        VMSTATE_UINTTL_ARRAY(env.CP0_WatchLo, MIPSCPU, 8),
+        VMSTATE_INT32_ARRAY(env.CP0_WatchHi, MIPSCPU, 8),
+        VMSTATE_UINTTL(env.CP0_XContext, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Framemask, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Debug, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_DEPC, MIPSCPU),
+        VMSTATE_INT32(env.CP0_Performance0, MIPSCPU),
+        VMSTATE_UINT64(env.CP0_TagLo, MIPSCPU),
+        VMSTATE_INT32(env.CP0_DataLo, MIPSCPU),
+        VMSTATE_INT32(env.CP0_TagHi, MIPSCPU),
+        VMSTATE_INT32(env.CP0_DataHi, MIPSCPU),
+        VMSTATE_UINTTL(env.CP0_ErrorEPC, MIPSCPU),
+        VMSTATE_INT32(env.CP0_DESAVE, MIPSCPU),
+        VMSTATE_UINTTL_ARRAY(env.CP0_KScratch, MIPSCPU, MIPS_KSCRATCH_NUM),
+
+        /* Inactive TC */
+        VMSTATE_STRUCT_ARRAY(env.tcs, MIPSCPU, MIPS_SHADOW_SET_MAX, 1,
+                             vmstate_inactive_tc, TCState),
+        VMSTATE_STRUCT_ARRAY(env.fpus, MIPSCPU, MIPS_FPU_MAX, 1,
+                             vmstate_inactive_fpu, CPUMIPSFPUContext),
+
+        VMSTATE_END_OF_LIST()
+    },
+};
diff --git a/target-mips/mips-defs.h b/target-mips/mips-defs.h
index 1784227..047554e 100644
--- a/target-mips/mips-defs.h
+++ b/target-mips/mips-defs.h
@@ -1,5 +1,5 @@
-#if !defined (__QEMU_MIPS_DEFS_H__)
-#define __QEMU_MIPS_DEFS_H__
+#ifndef QEMU_MIPS_DEFS_H
+#define QEMU_MIPS_DEFS_H
 
 /* If we want to use host float regs... */
 //#define USE_HOST_FLOAT_REGS
@@ -10,11 +10,11 @@
 
 #if defined(TARGET_MIPS64)
 #define TARGET_LONG_BITS 64
-#define TARGET_PHYS_ADDR_SPACE_BITS 36
-#define TARGET_VIRT_ADDR_SPACE_BITS 42
+#define TARGET_PHYS_ADDR_SPACE_BITS 48
+#define TARGET_VIRT_ADDR_SPACE_BITS 48
 #else
 #define TARGET_LONG_BITS 32
-#define TARGET_PHYS_ADDR_SPACE_BITS 36
+#define TARGET_PHYS_ADDR_SPACE_BITS 40
 #define TARGET_VIRT_ADDR_SPACE_BITS 32
 #endif
 
@@ -88,4 +88,4 @@
    Note that we still maintain Count/Compare to match the host clock. */
 //#define MIPS_STRICT_STANDARD 1
 
-#endif /* !defined (__QEMU_MIPS_DEFS_H__) */
+#endif /* QEMU_MIPS_DEFS_H */
diff --git a/target-mips/mips-semi.c b/target-mips/mips-semi.c
new file mode 100644
index 0000000..a7aefba
--- /dev/null
+++ b/target-mips/mips-semi.c
@@ -0,0 +1,374 @@
+/*
+ * Unified Hosting Interface syscalls.
+ *
+ * Copyright (c) 2015 Imagination Technologies
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/log.h"
+#include "exec/helper-proto.h"
+#include "exec/softmmu-semi.h"
+#include "exec/semihost.h"
+
+typedef enum UHIOp {
+    UHI_exit = 1,
+    UHI_open = 2,
+    UHI_close = 3,
+    UHI_read = 4,
+    UHI_write = 5,
+    UHI_lseek = 6,
+    UHI_unlink = 7,
+    UHI_fstat = 8,
+    UHI_argc = 9,
+    UHI_argnlen = 10,
+    UHI_argn = 11,
+    UHI_plog = 13,
+    UHI_assert = 14,
+    UHI_pread = 19,
+    UHI_pwrite = 20,
+    UHI_link = 22
+} UHIOp;
+
+typedef struct UHIStat {
+    int16_t uhi_st_dev;
+    uint16_t uhi_st_ino;
+    uint32_t uhi_st_mode;
+    uint16_t uhi_st_nlink;
+    uint16_t uhi_st_uid;
+    uint16_t uhi_st_gid;
+    int16_t uhi_st_rdev;
+    uint64_t uhi_st_size;
+    uint64_t uhi_st_atime;
+    uint64_t uhi_st_spare1;
+    uint64_t uhi_st_mtime;
+    uint64_t uhi_st_spare2;
+    uint64_t uhi_st_ctime;
+    uint64_t uhi_st_spare3;
+    uint64_t uhi_st_blksize;
+    uint64_t uhi_st_blocks;
+    uint64_t uhi_st_spare4[2];
+} UHIStat;
+
+enum UHIOpenFlags {
+    UHIOpen_RDONLY = 0x0,
+    UHIOpen_WRONLY = 0x1,
+    UHIOpen_RDWR   = 0x2,
+    UHIOpen_APPEND = 0x8,
+    UHIOpen_CREAT  = 0x200,
+    UHIOpen_TRUNC  = 0x400,
+    UHIOpen_EXCL   = 0x800
+};
+
+/* Errno values taken from asm-mips/errno.h */
+static uint16_t host_to_mips_errno[] = {
+    [ENAMETOOLONG] = 78,
+#ifdef EOVERFLOW
+    [EOVERFLOW]    = 79,
+#endif
+#ifdef ELOOP
+    [ELOOP]        = 90,
+#endif
+};
+
+static int errno_mips(int err)
+{
+    if (err < 0 || err >= ARRAY_SIZE(host_to_mips_errno)) {
+        return EINVAL;
+    } else if (host_to_mips_errno[err]) {
+        return host_to_mips_errno[err];
+    } else {
+        return err;
+    }
+}
+
+static int copy_stat_to_target(CPUMIPSState *env, const struct stat *src,
+                               target_ulong vaddr)
+{
+    hwaddr len = sizeof(struct UHIStat);
+    UHIStat *dst = lock_user(VERIFY_WRITE, vaddr, len, 0);
+    if (!dst) {
+        errno = EFAULT;
+        return -1;
+    }
+
+    dst->uhi_st_dev = tswap16(src->st_dev);
+    dst->uhi_st_ino = tswap16(src->st_ino);
+    dst->uhi_st_mode = tswap32(src->st_mode);
+    dst->uhi_st_nlink = tswap16(src->st_nlink);
+    dst->uhi_st_uid = tswap16(src->st_uid);
+    dst->uhi_st_gid = tswap16(src->st_gid);
+    dst->uhi_st_rdev = tswap16(src->st_rdev);
+    dst->uhi_st_size = tswap64(src->st_size);
+    dst->uhi_st_atime = tswap64(src->st_atime);
+    dst->uhi_st_mtime = tswap64(src->st_mtime);
+    dst->uhi_st_ctime = tswap64(src->st_ctime);
+#ifdef _WIN32
+    dst->uhi_st_blksize = 0;
+    dst->uhi_st_blocks = 0;
+#else
+    dst->uhi_st_blksize = tswap64(src->st_blksize);
+    dst->uhi_st_blocks = tswap64(src->st_blocks);
+#endif
+    unlock_user(dst, vaddr, len);
+    return 0;
+}
+
+static int get_open_flags(target_ulong target_flags)
+{
+    int open_flags = 0;
+
+    if (target_flags & UHIOpen_RDWR) {
+        open_flags |= O_RDWR;
+    } else if (target_flags & UHIOpen_WRONLY) {
+        open_flags |= O_WRONLY;
+    } else {
+        open_flags |= O_RDONLY;
+    }
+
+    open_flags |= (target_flags & UHIOpen_APPEND) ? O_APPEND : 0;
+    open_flags |= (target_flags & UHIOpen_CREAT)  ? O_CREAT  : 0;
+    open_flags |= (target_flags & UHIOpen_TRUNC)  ? O_TRUNC  : 0;
+    open_flags |= (target_flags & UHIOpen_EXCL)   ? O_EXCL   : 0;
+
+    return open_flags;
+}
+
+static int write_to_file(CPUMIPSState *env, target_ulong fd, target_ulong vaddr,
+                         target_ulong len, target_ulong offset)
+{
+    int num_of_bytes;
+    void *dst = lock_user(VERIFY_READ, vaddr, len, 1);
+    if (!dst) {
+        errno = EFAULT;
+        return -1;
+    }
+
+    if (offset) {
+#ifdef _WIN32
+        num_of_bytes = 0;
+#else
+        num_of_bytes = pwrite(fd, dst, len, offset);
+#endif
+    } else {
+        num_of_bytes = write(fd, dst, len);
+    }
+
+    unlock_user(dst, vaddr, 0);
+    return num_of_bytes;
+}
+
+static int read_from_file(CPUMIPSState *env, target_ulong fd,
+                          target_ulong vaddr, target_ulong len,
+                          target_ulong offset)
+{
+    int num_of_bytes;
+    void *dst = lock_user(VERIFY_WRITE, vaddr, len, 0);
+    if (!dst) {
+        errno = EFAULT;
+        return -1;
+    }
+
+    if (offset) {
+#ifdef _WIN32
+        num_of_bytes = 0;
+#else
+        num_of_bytes = pread(fd, dst, len, offset);
+#endif
+    } else {
+        num_of_bytes = read(fd, dst, len);
+    }
+
+    unlock_user(dst, vaddr, len);
+    return num_of_bytes;
+}
+
+static int copy_argn_to_target(CPUMIPSState *env, int arg_num,
+                               target_ulong vaddr)
+{
+    int strsize = strlen(semihosting_get_arg(arg_num)) + 1;
+    char *dst = lock_user(VERIFY_WRITE, vaddr, strsize, 0);
+    if (!dst) {
+        return -1;
+    }
+
+    strcpy(dst, semihosting_get_arg(arg_num));
+
+    unlock_user(dst, vaddr, strsize);
+    return 0;
+}
+
+#define GET_TARGET_STRING(p, addr)              \
+    do {                                        \
+        p = lock_user_string(addr);             \
+        if (!p) {                               \
+            gpr[2] = -1;                        \
+            gpr[3] = EFAULT;                    \
+            goto uhi_done;                      \
+        }                                       \
+    } while (0)
+
+#define GET_TARGET_STRINGS_2(p, addr, p2, addr2)        \
+    do {                                                \
+        p = lock_user_string(addr);                     \
+        if (!p) {                                       \
+            gpr[2] = -1;                                \
+            gpr[3] = EFAULT;                            \
+            goto uhi_done;                              \
+        }                                               \
+        p2 = lock_user_string(addr2);                   \
+        if (!p2) {                                      \
+            unlock_user(p, addr, 0);                    \
+            gpr[2] = -1;                                \
+            gpr[3] = EFAULT;                            \
+            goto uhi_done;                              \
+        }                                               \
+    } while (0)
+
+#define FREE_TARGET_STRING(p, gpr)              \
+    do {                                        \
+        unlock_user(p, gpr, 0);                 \
+    } while (0)
+
+void helper_do_semihosting(CPUMIPSState *env)
+{
+    target_ulong *gpr = env->active_tc.gpr;
+    const UHIOp op = gpr[25];
+    char *p, *p2;
+
+    switch (op) {
+    case UHI_exit:
+        qemu_log("UHI(%d): exit(%d)\n", op, (int)gpr[4]);
+        exit(gpr[4]);
+    case UHI_open:
+        GET_TARGET_STRING(p, gpr[4]);
+        if (!strcmp("/dev/stdin", p)) {
+            gpr[2] = 0;
+        } else if (!strcmp("/dev/stdout", p)) {
+            gpr[2] = 1;
+        } else if (!strcmp("/dev/stderr", p)) {
+            gpr[2] = 2;
+        } else {
+            gpr[2] = open(p, get_open_flags(gpr[5]), gpr[6]);
+            gpr[3] = errno_mips(errno);
+        }
+        FREE_TARGET_STRING(p, gpr[4]);
+        break;
+    case UHI_close:
+        if (gpr[4] < 3) {
+            /* ignore closing stdin/stdout/stderr */
+            gpr[2] = 0;
+            goto uhi_done;
+        }
+        gpr[2] = close(gpr[4]);
+        gpr[3] = errno_mips(errno);
+        break;
+    case UHI_read:
+        gpr[2] = read_from_file(env, gpr[4], gpr[5], gpr[6], 0);
+        gpr[3] = errno_mips(errno);
+        break;
+    case UHI_write:
+        gpr[2] = write_to_file(env, gpr[4], gpr[5], gpr[6], 0);
+        gpr[3] = errno_mips(errno);
+        break;
+    case UHI_lseek:
+        gpr[2] = lseek(gpr[4], gpr[5], gpr[6]);
+        gpr[3] = errno_mips(errno);
+        break;
+    case UHI_unlink:
+        GET_TARGET_STRING(p, gpr[4]);
+        gpr[2] = remove(p);
+        gpr[3] = errno_mips(errno);
+        FREE_TARGET_STRING(p, gpr[4]);
+        break;
+    case UHI_fstat:
+        {
+            struct stat sbuf;
+            memset(&sbuf, 0, sizeof(sbuf));
+            gpr[2] = fstat(gpr[4], &sbuf);
+            gpr[3] = errno_mips(errno);
+            if (gpr[2]) {
+                goto uhi_done;
+            }
+            gpr[2] = copy_stat_to_target(env, &sbuf, gpr[5]);
+            gpr[3] = errno_mips(errno);
+        }
+        break;
+    case UHI_argc:
+        gpr[2] = semihosting_get_argc();
+        break;
+    case UHI_argnlen:
+        if (gpr[4] >= semihosting_get_argc()) {
+            gpr[2] = -1;
+            goto uhi_done;
+        }
+        gpr[2] = strlen(semihosting_get_arg(gpr[4]));
+        break;
+    case UHI_argn:
+        if (gpr[4] >= semihosting_get_argc()) {
+            gpr[2] = -1;
+            goto uhi_done;
+        }
+        gpr[2] = copy_argn_to_target(env, gpr[4], gpr[5]);
+        break;
+    case UHI_plog:
+        GET_TARGET_STRING(p, gpr[4]);
+        p2 = strstr(p, "%d");
+        if (p2) {
+            int char_num = p2 - p;
+            char *buf = g_malloc(char_num + 1);
+            strncpy(buf, p, char_num);
+            buf[char_num] = '\0';
+            gpr[2] = printf("%s%d%s", buf, (int)gpr[5], p2 + 2);
+            g_free(buf);
+        } else {
+            gpr[2] = printf("%s", p);
+        }
+        FREE_TARGET_STRING(p, gpr[4]);
+        break;
+    case UHI_assert:
+        GET_TARGET_STRINGS_2(p, gpr[4], p2, gpr[5]);
+        printf("assertion '");
+        printf("\"%s\"", p);
+        printf("': file \"%s\", line %d\n", p2, (int)gpr[6]);
+        FREE_TARGET_STRING(p2, gpr[5]);
+        FREE_TARGET_STRING(p, gpr[4]);
+        abort();
+        break;
+    case UHI_pread:
+        gpr[2] = read_from_file(env, gpr[4], gpr[5], gpr[6], gpr[7]);
+        gpr[3] = errno_mips(errno);
+        break;
+    case UHI_pwrite:
+        gpr[2] = write_to_file(env, gpr[4], gpr[5], gpr[6], gpr[7]);
+        gpr[3] = errno_mips(errno);
+        break;
+#ifndef _WIN32
+    case UHI_link:
+        GET_TARGET_STRINGS_2(p, gpr[4], p2, gpr[5]);
+        gpr[2] = link(p, p2);
+        gpr[3] = errno_mips(errno);
+        FREE_TARGET_STRING(p2, gpr[5]);
+        FREE_TARGET_STRING(p, gpr[4]);
+        break;
+#endif
+    default:
+        fprintf(stderr, "Unknown UHI operation %d\n", op);
+        abort();
+    }
+uhi_done:
+    return;
+}
diff --git a/target-mips/msa_helper.c b/target-mips/msa_helper.c
index 4176378..a8d8d09 100644
--- a/target-mips/msa_helper.c
+++ b/target-mips/msa_helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 
 /* Data format min and max values */
@@ -1348,21 +1350,11 @@
         break;
     case 1:
         env->active_tc.msacsr = (int32_t)elm & MSACSR_MASK;
-        /* set float_status rounding mode */
-        set_float_rounding_mode(
-            ieee_rm[(env->active_tc.msacsr & MSACSR_RM_MASK) >> MSACSR_RM],
-            &env->active_tc.msa_fp_status);
-        /* set float_status flush modes */
-        set_flush_to_zero(
-          (env->active_tc.msacsr & MSACSR_FS_MASK) != 0 ? 1 : 0,
-          &env->active_tc.msa_fp_status);
-        set_flush_inputs_to_zero(
-          (env->active_tc.msacsr & MSACSR_FS_MASK) != 0 ? 1 : 0,
-          &env->active_tc.msa_fp_status);
+        restore_msa_fp_status(env);
         /* check exception */
         if ((GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)
             & GET_FP_CAUSE(env->active_tc.msacsr)) {
-            helper_raise_exception(env, EXCP_MSAFPE);
+            do_raise_exception(env, EXCP_MSAFPE, GETPC());
         }
         break;
     }
@@ -1503,11 +1495,11 @@
 #define FLOAT_ONE32 make_float32(0x3f8 << 20)
 #define FLOAT_ONE64 make_float64(0x3ffULL << 52)
 
-#define FLOAT_SNAN16 (float16_default_nan ^ 0x0220)
+#define FLOAT_SNAN16(s) (float16_default_nan(s) ^ 0x0220)
         /* 0x7c20 */
-#define FLOAT_SNAN32 (float32_default_nan ^ 0x00400020)
+#define FLOAT_SNAN32(s) (float32_default_nan(s) ^ 0x00400020)
         /* 0x7f800020 */
-#define FLOAT_SNAN64 (float64_default_nan ^ 0x0008000000000020ULL)
+#define FLOAT_SNAN64(s) (float64_default_nan(s) ^ 0x0008000000000020ULL)
         /* 0x7ff0000000000020 */
 
 static inline void clear_msacsr_cause(CPUMIPSState *env)
@@ -1515,14 +1507,14 @@
     SET_FP_CAUSE(env->active_tc.msacsr, 0);
 }
 
-static inline void check_msacsr_cause(CPUMIPSState *env)
+static inline void check_msacsr_cause(CPUMIPSState *env, uintptr_t retaddr)
 {
     if ((GET_FP_CAUSE(env->active_tc.msacsr) &
             (GET_FP_ENABLE(env->active_tc.msacsr) | FP_UNIMPLEMENTED)) == 0) {
         UPDATE_FP_FLAGS(env->active_tc.msacsr,
                 GET_FP_CAUSE(env->active_tc.msacsr));
     } else {
-        helper_raise_exception(env, EXCP_MSAFPE);
+        do_raise_exception(env, EXCP_MSAFPE, retaddr);
     }
 }
 
@@ -1614,189 +1606,190 @@
     return c & enable;
 }
 
-static inline float16 float16_from_float32(int32 a, flag ieee STATUS_PARAM)
+static inline float16 float16_from_float32(int32_t a, flag ieee,
+                                           float_status *status)
 {
       float16 f_val;
 
-      f_val = float32_to_float16((float32)a, ieee  STATUS_VAR);
-      f_val = float16_maybe_silence_nan(f_val);
+      f_val = float32_to_float16((float32)a, ieee, status);
+      f_val = float16_maybe_silence_nan(f_val, status);
 
       return a < 0 ? (f_val | (1 << 15)) : f_val;
 }
 
-static inline float32 float32_from_float64(int64 a STATUS_PARAM)
+static inline float32 float32_from_float64(int64_t a, float_status *status)
 {
       float32 f_val;
 
-      f_val = float64_to_float32((float64)a STATUS_VAR);
-      f_val = float32_maybe_silence_nan(f_val);
+      f_val = float64_to_float32((float64)a, status);
+      f_val = float32_maybe_silence_nan(f_val, status);
 
       return a < 0 ? (f_val | (1 << 31)) : f_val;
 }
 
-static inline float32 float32_from_float16(int16_t a, flag ieee STATUS_PARAM)
+static inline float32 float32_from_float16(int16_t a, flag ieee,
+                                           float_status *status)
 {
       float32 f_val;
 
-      f_val = float16_to_float32((float16)a, ieee STATUS_VAR);
-      f_val = float32_maybe_silence_nan(f_val);
+      f_val = float16_to_float32((float16)a, ieee, status);
+      f_val = float32_maybe_silence_nan(f_val, status);
 
       return a < 0 ? (f_val | (1 << 31)) : f_val;
 }
 
-static inline float64 float64_from_float32(int32 a STATUS_PARAM)
+static inline float64 float64_from_float32(int32_t a, float_status *status)
 {
       float64 f_val;
 
-      f_val = float32_to_float64((float64)a STATUS_VAR);
-      f_val = float64_maybe_silence_nan(f_val);
+      f_val = float32_to_float64((float64)a, status);
+      f_val = float64_maybe_silence_nan(f_val, status);
 
       return a < 0 ? (f_val | (1ULL << 63)) : f_val;
 }
 
-static inline float32 float32_from_q16(int16_t a STATUS_PARAM)
+static inline float32 float32_from_q16(int16_t a, float_status *status)
 {
     float32 f_val;
 
     /* conversion as integer and scaling */
-    f_val = int32_to_float32(a STATUS_VAR);
-    f_val = float32_scalbn(f_val, -15 STATUS_VAR);
+    f_val = int32_to_float32(a, status);
+    f_val = float32_scalbn(f_val, -15, status);
 
     return f_val;
 }
 
-static inline float64 float64_from_q32(int32 a STATUS_PARAM)
+static inline float64 float64_from_q32(int32_t a, float_status *status)
 {
     float64 f_val;
 
     /* conversion as integer and scaling */
-    f_val = int32_to_float64(a STATUS_VAR);
-    f_val = float64_scalbn(f_val, -31 STATUS_VAR);
+    f_val = int32_to_float64(a, status);
+    f_val = float64_scalbn(f_val, -31, status);
 
     return f_val;
 }
 
-static inline int16_t float32_to_q16(float32 a STATUS_PARAM)
+static inline int16_t float32_to_q16(float32 a, float_status *status)
 {
-    int32 q_val;
-    int32 q_min = 0xffff8000;
-    int32 q_max = 0x00007fff;
+    int32_t q_val;
+    int32_t q_min = 0xffff8000;
+    int32_t q_max = 0x00007fff;
 
     int ieee_ex;
 
     if (float32_is_any_nan(a)) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
 
     /* scaling */
-    a = float32_scalbn(a, 15 STATUS_VAR);
+    a = float32_scalbn(a, 15, status);
 
     ieee_ex = get_float_exception_flags(status);
     set_float_exception_flags(ieee_ex & (~float_flag_underflow)
-                              STATUS_VAR);
+                             , status);
 
     if (ieee_ex & float_flag_overflow) {
-        float_raise(float_flag_inexact STATUS_VAR);
-        return (int32)a < 0 ? q_min : q_max;
+        float_raise(float_flag_inexact, status);
+        return (int32_t)a < 0 ? q_min : q_max;
     }
 
     /* conversion to int */
-    q_val = float32_to_int32(a STATUS_VAR);
+    q_val = float32_to_int32(a, status);
 
     ieee_ex = get_float_exception_flags(status);
     set_float_exception_flags(ieee_ex & (~float_flag_underflow)
-                              STATUS_VAR);
+                             , status);
 
     if (ieee_ex & float_flag_invalid) {
         set_float_exception_flags(ieee_ex & (~float_flag_invalid)
-                                STATUS_VAR);
-        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
-        return (int32)a < 0 ? q_min : q_max;
+                               , status);
+        float_raise(float_flag_overflow | float_flag_inexact, status);
+        return (int32_t)a < 0 ? q_min : q_max;
     }
 
     if (q_val < q_min) {
-        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        float_raise(float_flag_overflow | float_flag_inexact, status);
         return (int16_t)q_min;
     }
 
     if (q_max < q_val) {
-        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
+        float_raise(float_flag_overflow | float_flag_inexact, status);
         return (int16_t)q_max;
     }
 
     return (int16_t)q_val;
 }
 
-static inline int32 float64_to_q32(float64 a STATUS_PARAM)
+static inline int32_t float64_to_q32(float64 a, float_status *status)
 {
-    int64 q_val;
-    int64 q_min = 0xffffffff80000000LL;
-    int64 q_max = 0x000000007fffffffLL;
+    int64_t q_val;
+    int64_t q_min = 0xffffffff80000000LL;
+    int64_t q_max = 0x000000007fffffffLL;
 
     int ieee_ex;
 
     if (float64_is_any_nan(a)) {
-        float_raise(float_flag_invalid STATUS_VAR);
+        float_raise(float_flag_invalid, status);
         return 0;
     }
 
     /* scaling */
-    a = float64_scalbn(a, 31 STATUS_VAR);
+    a = float64_scalbn(a, 31, status);
 
     ieee_ex = get_float_exception_flags(status);
     set_float_exception_flags(ieee_ex & (~float_flag_underflow)
-            STATUS_VAR);
+           , status);
 
     if (ieee_ex & float_flag_overflow) {
-        float_raise(float_flag_inexact STATUS_VAR);
-        return (int64)a < 0 ? q_min : q_max;
+        float_raise(float_flag_inexact, status);
+        return (int64_t)a < 0 ? q_min : q_max;
     }
 
     /* conversion to integer */
-    q_val = float64_to_int64(a STATUS_VAR);
+    q_val = float64_to_int64(a, status);
 
     ieee_ex = get_float_exception_flags(status);
     set_float_exception_flags(ieee_ex & (~float_flag_underflow)
-            STATUS_VAR);
+           , status);
 
     if (ieee_ex & float_flag_invalid) {
         set_float_exception_flags(ieee_ex & (~float_flag_invalid)
-                STATUS_VAR);
-        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
-        return (int64)a < 0 ? q_min : q_max;
+               , status);
+        float_raise(float_flag_overflow | float_flag_inexact, status);
+        return (int64_t)a < 0 ? q_min : q_max;
     }
 
     if (q_val < q_min) {
-        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
-        return (int32)q_min;
+        float_raise(float_flag_overflow | float_flag_inexact, status);
+        return (int32_t)q_min;
     }
 
     if (q_max < q_val) {
-        float_raise(float_flag_overflow | float_flag_inexact STATUS_VAR);
-        return (int32)q_max;
+        float_raise(float_flag_overflow | float_flag_inexact, status);
+        return (int32_t)q_max;
     }
 
-    return (int32)q_val;
+    return (int32_t)q_val;
 }
 
 #define MSA_FLOAT_COND(DEST, OP, ARG1, ARG2, BITS, QUIET)                   \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
         int64_t cond;                                                       \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
+        set_float_exception_flags(0, status);                               \
         if (!QUIET) {                                                       \
-            cond = float ## BITS ## _ ## OP(ARG1, ARG2,                     \
-                                          &env->active_tc.msa_fp_status);   \
+            cond = float ## BITS ## _ ## OP(ARG1, ARG2, status);            \
         } else {                                                            \
-            cond = float ## BITS ## _ ## OP ## _quiet(ARG1, ARG2,           \
-                                          &env->active_tc.msa_fp_status);   \
+            cond = float ## BITS ## _ ## OP ## _quiet(ARG1, ARG2, status);  \
         }                                                                   \
         DEST = cond ? M_MAX_UINT(BITS) : 0;                                 \
         c = update_msacsr(env, CLEAR_IS_INEXACT, 0);                        \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
@@ -1860,7 +1853,8 @@
     } while (0)
 
 static inline void compare_af(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1882,13 +1876,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_un(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1912,13 +1907,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_eq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1940,13 +1936,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ueq(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1968,13 +1965,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_lt(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -1996,13 +1994,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ult(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2024,13 +2023,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_le(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2052,13 +2052,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ule(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2080,13 +2081,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_or(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet)
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2108,13 +2110,14 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_une(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                               wr_t *pwt, uint32_t df, int quiet)
+                               wr_t *pwt, uint32_t df, int quiet,
+                               uintptr_t retaddr)
 {
     wr_t wx, *pwx = &wx;
     uint32_t i;
@@ -2136,13 +2139,15 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
 
 static inline void compare_ne(CPUMIPSState *env, wr_t *pwd, wr_t *pws,
-                              wr_t *pwt, uint32_t df, int quiet) {
+                              wr_t *pwt, uint32_t df, int quiet,
+                              uintptr_t retaddr)
+{
     wr_t wx, *pwx = &wx;
     uint32_t i;
 
@@ -2163,7 +2168,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, retaddr);
 
     msa_move_v(pwd, pwx);
 }
@@ -2174,7 +2179,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_af(env, pwd, pws, pwt, df, 1);
+    compare_af(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2183,7 +2188,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_un(env, pwd, pws, pwt, df, 1);
+    compare_un(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fceq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2192,7 +2197,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_eq(env, pwd, pws, pwt, df, 1);
+    compare_eq(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2201,7 +2206,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ueq(env, pwd, pws, pwt, df, 1);
+    compare_ueq(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fclt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2210,7 +2215,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_lt(env, pwd, pws, pwt, df, 1);
+    compare_lt(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2219,7 +2224,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ult(env, pwd, pws, pwt, df, 1);
+    compare_ult(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2228,7 +2233,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_le(env, pwd, pws, pwt, df, 1);
+    compare_le(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2237,7 +2242,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ule(env, pwd, pws, pwt, df, 1);
+    compare_ule(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fsaf_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2246,7 +2251,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_af(env, pwd, pws, pwt, df, 0);
+    compare_af(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsun_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2255,7 +2260,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_un(env, pwd, pws, pwt, df, 0);
+    compare_un(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fseq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2264,7 +2269,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_eq(env, pwd, pws, pwt, df, 0);
+    compare_eq(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsueq_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2273,7 +2278,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ueq(env, pwd, pws, pwt, df, 0);
+    compare_ueq(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fslt_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2282,7 +2287,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_lt(env, pwd, pws, pwt, df, 0);
+    compare_lt(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsult_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2291,7 +2296,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ult(env, pwd, pws, pwt, df, 0);
+    compare_ult(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsle_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2300,7 +2305,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_le(env, pwd, pws, pwt, df, 0);
+    compare_le(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsule_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2309,7 +2314,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ule(env, pwd, pws, pwt, df, 0);
+    compare_ule(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fcor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2318,7 +2323,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_or(env, pwd, pws, pwt, df, 1);
+    compare_or(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2327,7 +2332,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_une(env, pwd, pws, pwt, df, 1);
+    compare_une(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fcne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2336,7 +2341,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ne(env, pwd, pws, pwt, df, 1);
+    compare_ne(env, pwd, pws, pwt, df, 1, GETPC());
 }
 
 void helper_msa_fsor_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2345,7 +2350,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_or(env, pwd, pws, pwt, df, 0);
+    compare_or(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsune_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2354,7 +2359,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_une(env, pwd, pws, pwt, df, 0);
+    compare_une(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 void helper_msa_fsne_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
@@ -2363,7 +2368,7 @@
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     wr_t *pwt = &(env->active_fpu.fpr[wt].wr);
-    compare_ne(env, pwd, pws, pwt, df, 0);
+    compare_ne(env, pwd, pws, pwt, df, 0, GETPC());
 }
 
 #define float16_is_zero(ARG) 0
@@ -2375,15 +2380,15 @@
 
 #define MSA_FLOAT_BINOP(DEST, OP, ARG1, ARG2, BITS)                         \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _ ## OP(ARG1, ARG2,                         \
-                                        &env->active_tc.msa_fp_status);     \
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _ ## OP(ARG1, ARG2, status);                \
         c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
@@ -2413,7 +2418,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2443,7 +2448,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -2473,7 +2478,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2504,22 +2509,22 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
 
 #define MSA_FLOAT_MULADD(DEST, ARG1, ARG2, ARG3, NEGATE, BITS)              \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _muladd(ARG2, ARG3, ARG1, NEGATE,           \
-                                        &env->active_tc.msa_fp_status);     \
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _muladd(ARG2, ARG3, ARG1, NEGATE, status);  \
         c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
@@ -2551,7 +2556,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2586,7 +2591,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2623,21 +2628,22 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
 
 #define MSA_FLOAT_UNOP(DEST, OP, ARG, BITS)                                 \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _ ## OP(ARG, &env->active_tc.msa_fp_status);\
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _ ## OP(ARG, status);                       \
         c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
@@ -2674,20 +2680,21 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
 #define MSA_FLOAT_UNOP_XD(DEST, OP, ARG, BITS, XBITS)                       \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _ ## OP(ARG, &env->active_tc.msa_fp_status);\
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _ ## OP(ARG, status);                       \
         c = update_msacsr(env, CLEAR_FS_UNDERFLOW, 0);                      \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## XBITS >> 6) << 6) | c;                   \
+            DEST = ((FLOAT_SNAN ## XBITS(status) >> 6) << 6) | c;           \
         }                                                                   \
     } while (0)
 
@@ -2719,37 +2726,37 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
 
-#define NUMBER_QNAN_PAIR(ARG1, ARG2, BITS)      \
-    !float ## BITS ## _is_any_nan(ARG1)         \
-    && float ## BITS ## _is_quiet_nan(ARG2)
+#define NUMBER_QNAN_PAIR(ARG1, ARG2, BITS, STATUS)      \
+    !float ## BITS ## _is_any_nan(ARG1)                 \
+    && float ## BITS ## _is_quiet_nan(ARG2, STATUS)
 
 #define MSA_FLOAT_MAXOP(DEST, OP, ARG1, ARG2, BITS)                         \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _ ## OP(ARG1, ARG2,                         \
-                                        &env->active_tc.msa_fp_status);     \
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _ ## OP(ARG1, ARG2, status);                \
         c = update_msacsr(env, 0, 0);                                       \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
-#define FMAXMIN_A(F, G, X, _S, _T, BITS)                            \
+#define FMAXMIN_A(F, G, X, _S, _T, BITS, STATUS)                    \
     do {                                                            \
         uint## BITS ##_t S = _S, T = _T;                            \
         uint## BITS ##_t as, at, xs, xt, xd;                        \
-        if (NUMBER_QNAN_PAIR(S, T, BITS)) {                         \
+        if (NUMBER_QNAN_PAIR(S, T, BITS, STATUS)) {                 \
             T = S;                                                  \
         }                                                           \
-        else if (NUMBER_QNAN_PAIR(T, S, BITS)) {                    \
+        else if (NUMBER_QNAN_PAIR(T, S, BITS, STATUS)) {            \
             S = T;                                                  \
         }                                                           \
         as = float## BITS ##_abs(S);                                \
@@ -2763,6 +2770,7 @@
 void helper_msa_fmin_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         uint32_t ws, uint32_t wt)
 {
+    float_status *status = &env->active_tc.msa_fp_status;
     wr_t wx, *pwx = &wx;
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
@@ -2774,9 +2782,9 @@
     switch (df) {
     case DF_WORD:
         for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
-            if (NUMBER_QNAN_PAIR(pws->w[i], pwt->w[i], 32)) {
+            if (NUMBER_QNAN_PAIR(pws->w[i], pwt->w[i], 32, status)) {
                 MSA_FLOAT_MAXOP(pwx->w[i], min, pws->w[i], pws->w[i], 32);
-            } else if (NUMBER_QNAN_PAIR(pwt->w[i], pws->w[i], 32)) {
+            } else if (NUMBER_QNAN_PAIR(pwt->w[i], pws->w[i], 32, status)) {
                 MSA_FLOAT_MAXOP(pwx->w[i], min, pwt->w[i], pwt->w[i], 32);
             } else {
                 MSA_FLOAT_MAXOP(pwx->w[i], min, pws->w[i], pwt->w[i], 32);
@@ -2785,9 +2793,9 @@
         break;
     case DF_DOUBLE:
         for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
-            if (NUMBER_QNAN_PAIR(pws->d[i], pwt->d[i], 64)) {
+            if (NUMBER_QNAN_PAIR(pws->d[i], pwt->d[i], 64, status)) {
                 MSA_FLOAT_MAXOP(pwx->d[i], min, pws->d[i], pws->d[i], 64);
-            } else if (NUMBER_QNAN_PAIR(pwt->d[i], pws->d[i], 64)) {
+            } else if (NUMBER_QNAN_PAIR(pwt->d[i], pws->d[i], 64, status)) {
                 MSA_FLOAT_MAXOP(pwx->d[i], min, pwt->d[i], pwt->d[i], 64);
             } else {
                 MSA_FLOAT_MAXOP(pwx->d[i], min, pws->d[i], pwt->d[i], 64);
@@ -2798,7 +2806,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2806,6 +2814,7 @@
 void helper_msa_fmin_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         uint32_t ws, uint32_t wt)
 {
+    float_status *status = &env->active_tc.msa_fp_status;
     wr_t wx, *pwx = &wx;
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
@@ -2817,19 +2826,19 @@
     switch (df) {
     case DF_WORD:
         for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
-            FMAXMIN_A(min, max, pwx->w[i], pws->w[i], pwt->w[i], 32);
+            FMAXMIN_A(min, max, pwx->w[i], pws->w[i], pwt->w[i], 32, status);
         }
         break;
     case DF_DOUBLE:
         for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
-            FMAXMIN_A(min, max, pwx->d[i], pws->d[i], pwt->d[i], 64);
+            FMAXMIN_A(min, max, pwx->d[i], pws->d[i], pwt->d[i], 64, status);
         }
         break;
     default:
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2837,6 +2846,7 @@
 void helper_msa_fmax_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         uint32_t ws, uint32_t wt)
 {
+    float_status *status = &env->active_tc.msa_fp_status;
     wr_t wx, *pwx = &wx;
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
@@ -2848,9 +2858,9 @@
     switch (df) {
     case DF_WORD:
         for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
-            if (NUMBER_QNAN_PAIR(pws->w[i], pwt->w[i], 32)) {
+            if (NUMBER_QNAN_PAIR(pws->w[i], pwt->w[i], 32, status)) {
                 MSA_FLOAT_MAXOP(pwx->w[i], max, pws->w[i], pws->w[i], 32);
-            } else if (NUMBER_QNAN_PAIR(pwt->w[i], pws->w[i], 32)) {
+            } else if (NUMBER_QNAN_PAIR(pwt->w[i], pws->w[i], 32, status)) {
                 MSA_FLOAT_MAXOP(pwx->w[i], max, pwt->w[i], pwt->w[i], 32);
             } else {
                 MSA_FLOAT_MAXOP(pwx->w[i], max, pws->w[i], pwt->w[i], 32);
@@ -2859,9 +2869,9 @@
         break;
     case DF_DOUBLE:
         for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
-            if (NUMBER_QNAN_PAIR(pws->d[i], pwt->d[i], 64)) {
+            if (NUMBER_QNAN_PAIR(pws->d[i], pwt->d[i], 64, status)) {
                 MSA_FLOAT_MAXOP(pwx->d[i], max, pws->d[i], pws->d[i], 64);
-            } else if (NUMBER_QNAN_PAIR(pwt->d[i], pws->d[i], 64)) {
+            } else if (NUMBER_QNAN_PAIR(pwt->d[i], pws->d[i], 64, status)) {
                 MSA_FLOAT_MAXOP(pwx->d[i], max, pwt->d[i], pwt->d[i], 64);
             } else {
                 MSA_FLOAT_MAXOP(pwx->d[i], max, pws->d[i], pwt->d[i], 64);
@@ -2872,7 +2882,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2880,6 +2890,7 @@
 void helper_msa_fmax_a_df(CPUMIPSState *env, uint32_t df, uint32_t wd,
         uint32_t ws, uint32_t wt)
 {
+    float_status *status = &env->active_tc.msa_fp_status;
     wr_t wx, *pwx = &wx;
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
@@ -2891,19 +2902,19 @@
     switch (df) {
     case DF_WORD:
         for (i = 0; i < DF_ELEMENTS(DF_WORD); i++) {
-            FMAXMIN_A(max, min, pwx->w[i], pws->w[i], pwt->w[i], 32);
+            FMAXMIN_A(max, min, pwx->w[i], pws->w[i], pwt->w[i], 32, status);
         }
         break;
     case DF_DOUBLE:
         for (i = 0; i < DF_ELEMENTS(DF_DOUBLE); i++) {
-            FMAXMIN_A(max, min, pwx->d[i], pws->d[i], pwt->d[i], 64);
+            FMAXMIN_A(max, min, pwx->d[i], pws->d[i], pwt->d[i], 64, status);
         }
         break;
     default:
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2911,29 +2922,32 @@
 void helper_msa_fclass_df(CPUMIPSState *env, uint32_t df,
         uint32_t wd, uint32_t ws)
 {
+    float_status* status = &env->active_tc.msa_fp_status;
+
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);
     wr_t *pws = &(env->active_fpu.fpr[ws].wr);
     if (df == DF_WORD) {
-        pwd->w[0] = helper_float_class_s(pws->w[0]);
-        pwd->w[1] = helper_float_class_s(pws->w[1]);
-        pwd->w[2] = helper_float_class_s(pws->w[2]);
-        pwd->w[3] = helper_float_class_s(pws->w[3]);
+        pwd->w[0] = float_class_s(pws->w[0], status);
+        pwd->w[1] = float_class_s(pws->w[1], status);
+        pwd->w[2] = float_class_s(pws->w[2], status);
+        pwd->w[3] = float_class_s(pws->w[3], status);
     } else {
-        pwd->d[0] = helper_float_class_d(pws->d[0]);
-        pwd->d[1] = helper_float_class_d(pws->d[1]);
+        pwd->d[0] = float_class_d(pws->d[0], status);
+        pwd->d[1] = float_class_d(pws->d[1], status);
     }
 }
 
 #define MSA_FLOAT_UNOP0(DEST, OP, ARG, BITS)                                \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _ ## OP(ARG, &env->active_tc.msa_fp_status);\
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _ ## OP(ARG, status);                       \
         c = update_msacsr(env, CLEAR_FS_UNDERFLOW, 0);                      \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         } else if (float ## BITS ## _is_any_nan(ARG)) {                     \
             DEST = 0;                                                       \
         }                                                                   \
@@ -2964,7 +2978,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -2994,7 +3008,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3024,25 +3038,25 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
 
 #define MSA_FLOAT_RECIPROCAL(DEST, ARG, BITS)                               \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        DEST = float ## BITS ## _ ## div(FLOAT_ONE ## BITS, ARG,            \
-                                         &env->active_tc.msa_fp_status);    \
+        set_float_exception_flags(0, status);                               \
+        DEST = float ## BITS ## _ ## div(FLOAT_ONE ## BITS, ARG, status);   \
         c = update_msacsr(env, float ## BITS ## _is_infinity(ARG) ||        \
-                          float ## BITS ## _is_quiet_nan(DEST) ?            \
+                          float ## BITS ## _is_quiet_nan(DEST, status) ?    \
                           0 : RECIPROCAL_INEXACT,                           \
                           IS_DENORMAL(DEST, BITS));                         \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
@@ -3073,7 +3087,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3103,7 +3117,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3133,35 +3147,32 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
 
 #define MSA_FLOAT_LOGB(DEST, ARG, BITS)                                     \
     do {                                                                    \
+        float_status *status = &env->active_tc.msa_fp_status;               \
         int c;                                                              \
                                                                             \
-        set_float_exception_flags(0, &env->active_tc.msa_fp_status);        \
-        set_float_rounding_mode(float_round_down,                           \
-                                &env->active_tc.msa_fp_status);             \
-        DEST = float ## BITS ## _ ## log2(ARG,                              \
-                                          &env->active_tc.msa_fp_status);   \
-        DEST = float ## BITS ## _ ## round_to_int(DEST,                     \
-                                          &env->active_tc.msa_fp_status);   \
+        set_float_exception_flags(0, status);                               \
+        set_float_rounding_mode(float_round_down, status);                  \
+        DEST = float ## BITS ## _ ## log2(ARG, status);                     \
+        DEST = float ## BITS ## _ ## round_to_int(DEST, status);            \
         set_float_rounding_mode(ieee_rm[(env->active_tc.msacsr &            \
                                          MSACSR_RM_MASK) >> MSACSR_RM],     \
-                                &env->active_tc.msa_fp_status);             \
+                                status);                                    \
                                                                             \
-        set_float_exception_flags(                                          \
-            get_float_exception_flags(&env->active_tc.msa_fp_status)        \
-                                                & (~float_flag_inexact),    \
-            &env->active_tc.msa_fp_status);                                 \
+        set_float_exception_flags(get_float_exception_flags(status) &       \
+                                  (~float_flag_inexact),                    \
+                                  status);                                  \
                                                                             \
         c = update_msacsr(env, 0, IS_DENORMAL(DEST, BITS));                 \
                                                                             \
         if (get_enabled_exceptions(env, c)) {                               \
-            DEST = ((FLOAT_SNAN ## BITS >> 6) << 6) | c;                    \
+            DEST = ((FLOAT_SNAN ## BITS(status) >> 6) << 6) | c;            \
         }                                                                   \
     } while (0)
 
@@ -3190,7 +3201,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3225,7 +3236,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -3259,7 +3270,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
     msa_move_v(pwd, pwx);
 }
 
@@ -3340,7 +3351,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3370,7 +3381,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3406,7 +3417,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
@@ -3436,7 +3447,7 @@
         assert(0);
     }
 
-    check_msacsr_cause(env);
+    check_msacsr_cause(env, GETPC());
 
     msa_move_v(pwd, pwx);
 }
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index 26cabac..501ee29 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -16,48 +16,17 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "sysemu/kvm.h"
 
-#ifndef CONFIG_USER_ONLY
-static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
-#endif
-
 /*****************************************************************************/
 /* Exceptions processing helpers */
 
-static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
-                                                        uint32_t exception,
-                                                        int error_code,
-                                                        uintptr_t pc)
-{
-    CPUState *cs = CPU(mips_env_get_cpu(env));
-
-    if (exception < EXCP_SC) {
-        qemu_log("%s: %d %d\n", __func__, exception, error_code);
-    }
-    cs->exception_index = exception;
-    env->error_code = error_code;
-
-    if (pc) {
-        /* now we have a real cpu fault */
-        cpu_restore_state(cs, pc);
-    }
-
-    cpu_loop_exit(cs);
-}
-
-static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
-                                                    uint32_t exception,
-                                                    uintptr_t pc)
-{
-    do_raise_exception_err(env, exception, 0, pc);
-}
-
 void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
                                 int error_code)
 {
@@ -66,27 +35,37 @@
 
 void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
 {
+    do_raise_exception(env, exception, GETPC());
+}
+
+void helper_raise_exception_debug(CPUMIPSState *env)
+{
+    do_raise_exception(env, EXCP_DEBUG, 0);
+}
+
+static void raise_exception(CPUMIPSState *env, uint32_t exception)
+{
     do_raise_exception(env, exception, 0);
 }
 
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_LD(name, insn, type)                                     \
 static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             int mem_idx)                               \
+                             int mem_idx, uintptr_t retaddr)            \
 {                                                                       \
-    return (type) insn##_raw(addr);                                     \
+    return (type) cpu_##insn##_data_ra(env, addr, retaddr);             \
 }
 #else
 #define HELPER_LD(name, insn, type)                                     \
 static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             int mem_idx)                               \
+                             int mem_idx, uintptr_t retaddr)            \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: return (type) cpu_##insn##_kernel(env, addr); break;        \
-    case 1: return (type) cpu_##insn##_super(env, addr); break;         \
+    case 0: return (type) cpu_##insn##_kernel_ra(env, addr, retaddr);   \
+    case 1: return (type) cpu_##insn##_super_ra(env, addr, retaddr);    \
     default:                                                            \
-    case 2: return (type) cpu_##insn##_user(env, addr); break;          \
+    case 2: return (type) cpu_##insn##_user_ra(env, addr, retaddr);     \
     }                                                                   \
 }
 #endif
@@ -99,21 +78,21 @@
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_ST(name, insn, type)                                     \
 static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             type val, int mem_idx)                     \
+                             type val, int mem_idx, uintptr_t retaddr)  \
 {                                                                       \
-    insn##_raw(addr, val);                                              \
+    cpu_##insn##_data_ra(env, addr, val, retaddr);                      \
 }
 #else
 #define HELPER_ST(name, insn, type)                                     \
 static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
-                             type val, int mem_idx)                     \
+                             type val, int mem_idx, uintptr_t retaddr)  \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: cpu_##insn##_kernel(env, addr, val); break;                 \
-    case 1: cpu_##insn##_super(env, addr, val); break;                  \
+    case 0: cpu_##insn##_kernel_ra(env, addr, val, retaddr); break;     \
+    case 1: cpu_##insn##_super_ra(env, addr, val, retaddr); break;      \
     default:                                                            \
-    case 2: cpu_##insn##_user(env, addr, val); break;                   \
+    case 2: cpu_##insn##_user_ra(env, addr, val, retaddr); break;       \
     }                                                                   \
 }
 #endif
@@ -154,10 +133,8 @@
 
 static inline target_ulong set_HIT0_LO(CPUMIPSState *env, uint64_t HILO)
 {
-    target_ulong tmp;
     env->active_tc.LO[0] = (int32_t)(HILO & 0xFFFFFFFF);
-    tmp = env->active_tc.HI[0] = (int32_t)(HILO >> 32);
-    return tmp;
+    return env->active_tc.HI[0] = (int32_t)(HILO >> 32);
 }
 
 static inline target_ulong set_HI_LOT0(CPUMIPSState *env, uint64_t HILO)
@@ -292,29 +269,34 @@
 
 static inline hwaddr do_translate_address(CPUMIPSState *env,
                                                       target_ulong address,
-                                                      int rw)
+                                                      int rw, uintptr_t retaddr)
 {
     hwaddr lladdr;
+    CPUState *cs = CPU(mips_env_get_cpu(env));
 
     lladdr = cpu_mips_translate_address(env, address, rw);
 
     if (lladdr == -1LL) {
-        cpu_loop_exit(CPU(mips_env_get_cpu(env)));
+        cpu_loop_exit_restore(cs, retaddr);
     } else {
         return lladdr;
     }
 }
 
-#define HELPER_LD_ATOMIC(name, insn)                                          \
+#define HELPER_LD_ATOMIC(name, insn, almask)                                  \
 target_ulong helper_##name(CPUMIPSState *env, target_ulong arg, int mem_idx)  \
 {                                                                             \
-    env->lladdr = do_translate_address(env, arg, 0);                          \
-    env->llval = do_##insn(env, arg, mem_idx);                                \
+    if (arg & almask) {                                                       \
+        env->CP0_BadVAddr = arg;                                              \
+        do_raise_exception(env, EXCP_AdEL, GETPC());                          \
+    }                                                                         \
+    env->lladdr = do_translate_address(env, arg, 0, GETPC());                 \
+    env->llval = do_##insn(env, arg, mem_idx, GETPC());                       \
     return env->llval;                                                        \
 }
-HELPER_LD_ATOMIC(ll, lw)
+HELPER_LD_ATOMIC(ll, lw, 0x3)
 #ifdef TARGET_MIPS64
-HELPER_LD_ATOMIC(lld, ld)
+HELPER_LD_ATOMIC(lld, ld, 0x7)
 #endif
 #undef HELPER_LD_ATOMIC
 
@@ -326,12 +308,12 @@
                                                                               \
     if (arg2 & almask) {                                                      \
         env->CP0_BadVAddr = arg2;                                             \
-        helper_raise_exception(env, EXCP_AdES);                               \
+        do_raise_exception(env, EXCP_AdES, GETPC());                          \
     }                                                                         \
-    if (do_translate_address(env, arg2, 1) == env->lladdr) {                  \
-        tmp = do_##ld_insn(env, arg2, mem_idx);                               \
+    if (do_translate_address(env, arg2, 1, GETPC()) == env->lladdr) {         \
+        tmp = do_##ld_insn(env, arg2, mem_idx, GETPC());                      \
         if (tmp == env->llval) {                                              \
-            do_##st_insn(env, arg2, arg1, mem_idx);                           \
+            do_##st_insn(env, arg2, arg1, mem_idx, GETPC());                  \
             return 1;                                                         \
         }                                                                     \
     }                                                                         \
@@ -355,31 +337,43 @@
 void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) <= 2)
-        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK(arg2) <= 2) {
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) <= 1)
-        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK(arg2) <= 1) {
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) == 0)
-        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
+    if (GET_LMASK(arg2) == 0) {
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx,
+              GETPC());
+    }
 }
 
 void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK(arg2) >= 1)
-        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK(arg2) >= 1) {
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) >= 2)
-        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK(arg2) >= 2) {
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK(arg2) == 3)
-        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK(arg2) == 3) {
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 }
 
 #if defined(TARGET_MIPS64)
@@ -395,55 +389,83 @@
 void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) <= 6)
-        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
+    if (GET_LMASK64(arg2) <= 6) {
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 5)
-        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
+    if (GET_LMASK64(arg2) <= 5) {
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 4)
-        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
+    if (GET_LMASK64(arg2) <= 4) {
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 3)
-        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK64(arg2) <= 3) {
+        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 2)
-        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK64(arg2) <= 2) {
+        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 1)
-        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK64(arg2) <= 1) {
+        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) <= 0)
-        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
+    if (GET_LMASK64(arg2) <= 0) {
+        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx,
+              GETPC());
+    }
 }
 
 void helper_sdr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
                 int mem_idx)
 {
-    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx, GETPC());
 
-    if (GET_LMASK64(arg2) >= 1)
-        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+    if (GET_LMASK64(arg2) >= 1) {
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 2)
-        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+    if (GET_LMASK64(arg2) >= 2) {
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 3)
-        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+    if (GET_LMASK64(arg2) >= 3) {
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 4)
-        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
+    if (GET_LMASK64(arg2) >= 4) {
+        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 5)
-        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
+    if (GET_LMASK64(arg2) >= 5) {
+        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) >= 6)
-        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
+    if (GET_LMASK64(arg2) >= 6) {
+        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx,
+              GETPC());
+    }
 
-    if (GET_LMASK64(arg2) == 7)
-        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
+    if (GET_LMASK64(arg2) == 7) {
+        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx,
+              GETPC());
+    }
 }
 #endif /* TARGET_MIPS64 */
 
@@ -460,13 +482,14 @@
 
         for (i = 0; i < base_reglist; i++) {
             env->active_tc.gpr[multiple_regs[i]] =
-                (target_long)do_lw(env, addr, mem_idx);
+                (target_long)do_lw(env, addr, mem_idx, GETPC());
             addr += 4;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx);
+        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx,
+                                                    GETPC());
     }
 }
 
@@ -480,13 +503,14 @@
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
+            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx,
+                  GETPC());
             addr += 4;
         }
     }
 
     if (do_r31) {
-        do_sw(env, addr, env->active_tc.gpr[31], mem_idx);
+        do_sw(env, addr, env->active_tc.gpr[31], mem_idx, GETPC());
     }
 }
 
@@ -501,13 +525,14 @@
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx);
+            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx,
+                                                         GETPC());
             addr += 8;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx);
+        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx, GETPC());
     }
 }
 
@@ -521,13 +546,14 @@
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
+            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx,
+                  GETPC());
             addr += 8;
         }
     }
 
     if (do_r31) {
-        do_sd(env, addr, env->active_tc.gpr[31], mem_idx);
+        do_sd(env, addr, env->active_tc.gpr[31], mem_idx, GETPC());
     }
 }
 #endif
@@ -544,9 +570,17 @@
     return cpu->halted && mips_vpe_active(env);
 }
 
+static bool mips_vp_is_wfi(MIPSCPU *c)
+{
+    CPUState *cpu = CPU(c);
+    CPUMIPSState *env = &c->env;
+
+    return cpu->halted && mips_vp_active(env);
+}
+
 static inline void mips_vpe_wake(MIPSCPU *c)
 {
-    /* Dont set ->halted = 0 directly, let it be done via cpu_has_work
+    /* Don't set ->halted = 0 directly, let it be done via cpu_has_work
        because there might be other conditions that state that c should
        be sleeping.  */
     cpu_interrupt(CPU(c), CPU_INTERRUPT_WAKE);
@@ -626,40 +660,9 @@
 
    These helper call synchronizes the regs for a given cpu.  */
 
-/* Called for updates to CP0_Status.  */
-static void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
-{
-    int32_t tcstatus, *tcst;
-    uint32_t v = cpu->CP0_Status;
-    uint32_t cu, mx, asid, ksu;
-    uint32_t mask = ((1 << CP0TCSt_TCU3)
-                       | (1 << CP0TCSt_TCU2)
-                       | (1 << CP0TCSt_TCU1)
-                       | (1 << CP0TCSt_TCU0)
-                       | (1 << CP0TCSt_TMX)
-                       | (3 << CP0TCSt_TKSU)
-                       | (0xff << CP0TCSt_TASID));
-
-    cu = (v >> CP0St_CU0) & 0xf;
-    mx = (v >> CP0St_MX) & 0x1;
-    ksu = (v >> CP0St_KSU) & 0x3;
-    asid = env->CP0_EntryHi & 0xff;
-
-    tcstatus = cu << CP0TCSt_TCU0;
-    tcstatus |= mx << CP0TCSt_TMX;
-    tcstatus |= ksu << CP0TCSt_TKSU;
-    tcstatus |= asid;
-
-    if (tc == cpu->current_tc) {
-        tcst = &cpu->active_tc.CP0_TCStatus;
-    } else {
-        tcst = &cpu->tcs[tc].CP0_TCStatus;
-    }
-
-    *tcst &= ~mask;
-    *tcst |= tcstatus;
-    compute_hflags(cpu);
-}
+/* Called for updates to CP0_Status.  Defined in "cpu.h" for gdbstub.c.  */
+/* static inline void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu,
+                                     int tc);  */
 
 /* Called for updates to CP0_TCStatus.  */
 static void sync_c0_tcstatus(CPUMIPSState *cpu, int tc,
@@ -676,7 +679,7 @@
 
     tcu = (v >> CP0TCSt_TCU0) & 0xf;
     tmx = (v >> CP0TCSt_TMX) & 0x1;
-    tasid = v & 0xff;
+    tasid = v & cpu->CP0_EntryHi_ASID_mask;
     tksu = (v >> CP0TCSt_TKSU) & 0x3;
 
     status = tcu << CP0St_CU0;
@@ -687,8 +690,8 @@
     cpu->CP0_Status |= status;
 
     /* Sync the TASID with EntryHi.  */
-    cpu->CP0_EntryHi &= ~0xff;
-    cpu->CP0_EntryHi = tasid;
+    cpu->CP0_EntryHi &= ~cpu->CP0_EntryHi_ASID_mask;
+    cpu->CP0_EntryHi |= tasid;
 
     compute_hflags(cpu);
 }
@@ -699,7 +702,7 @@
     int32_t *tcst;
     uint32_t asid, v = cpu->CP0_EntryHi;
 
-    asid = v & 0xff;
+    asid = v & cpu->CP0_EntryHi_ASID_mask;
 
     if (tc == cpu->current_tc) {
         tcst = &cpu->active_tc.CP0_TCStatus;
@@ -707,7 +710,7 @@
         tcst = &cpu->tcs[tc].CP0_TCStatus;
     }
 
-    *tcst &= ~0xff;
+    *tcst &= ~cpu->CP0_EntryHi_ASID_mask;
     *tcst |= asid;
 }
 
@@ -885,6 +888,16 @@
     return (int32_t)(env->lladdr >> env->CP0_LLAddr_shift);
 }
 
+target_ulong helper_mfc0_maar(CPUMIPSState *env)
+{
+    return (int32_t) env->CP0_MAAR[env->CP0_MAARI];
+}
+
+target_ulong helper_mfhc0_maar(CPUMIPSState *env)
+{
+    return env->CP0_MAAR[env->CP0_MAARI] >> 32;
+}
+
 target_ulong helper_mfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return (int32_t)env->CP0_WatchLo[sel];
@@ -951,6 +964,11 @@
     return env->lladdr >> env->CP0_LLAddr_shift;
 }
 
+target_ulong helper_dmfc0_maar(CPUMIPSState *env)
+{
+    return env->CP0_MAAR[env->CP0_MAARI];
+}
+
 target_ulong helper_dmfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return env->CP0_WatchLo[sel];
@@ -1095,19 +1113,23 @@
     env->CP0_VPEOpt = arg1 & 0x0000ffff;
 }
 
+#define MTC0_ENTRYLO_MASK(env) ((env->PAMask >> 6) & 0x3FFFFFFF)
+
 void helper_mtc0_entrylo0(CPUMIPSState *env, target_ulong arg1)
 {
-    /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     target_ulong rxi = arg1 & (env->CP0_PageGrain & (3u << CP0PG_XIE));
-    env->CP0_EntryLo0 = (arg1 & 0x3FFFFFFF) | (rxi << (CP0EnLo_XI - 30));
+    env->CP0_EntryLo0 = (arg1 & MTC0_ENTRYLO_MASK(env))
+                        | (rxi << (CP0EnLo_XI - 30));
 }
 
 #if defined(TARGET_MIPS64)
+#define DMTC0_ENTRYLO_MASK(env) (env->PAMask >> 6)
+
 void helper_dmtc0_entrylo0(CPUMIPSState *env, uint64_t arg1)
 {
     uint64_t rxi = arg1 & ((env->CP0_PageGrain & (3ull << CP0PG_XIE)) << 32);
-    env->CP0_EntryLo0 = (arg1 & 0x3FFFFFFF) | rxi;
+    env->CP0_EntryLo0 = (arg1 & DMTC0_ENTRYLO_MASK(env)) | rxi;
 }
 #endif
 
@@ -1273,17 +1295,17 @@
 
 void helper_mtc0_entrylo1(CPUMIPSState *env, target_ulong arg1)
 {
-    /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     target_ulong rxi = arg1 & (env->CP0_PageGrain & (3u << CP0PG_XIE));
-    env->CP0_EntryLo1 = (arg1 & 0x3FFFFFFF) | (rxi << (CP0EnLo_XI - 30));
+    env->CP0_EntryLo1 = (arg1 & MTC0_ENTRYLO_MASK(env))
+                        | (rxi << (CP0EnLo_XI - 30));
 }
 
 #if defined(TARGET_MIPS64)
 void helper_dmtc0_entrylo1(CPUMIPSState *env, uint64_t arg1)
 {
     uint64_t rxi = arg1 & ((env->CP0_PageGrain & (3ull << CP0PG_XIE)) << 32);
-    env->CP0_EntryLo1 = (arg1 & 0x3FFFFFFF) | rxi;
+    env->CP0_EntryLo1 = (arg1 & DMTC0_ENTRYLO_MASK(env)) | rxi;
 }
 #endif
 
@@ -1306,10 +1328,11 @@
 void helper_mtc0_pagegrain(CPUMIPSState *env, target_ulong arg1)
 {
     /* SmartMIPS not implemented */
-    /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     env->CP0_PageGrain = (arg1 & env->CP0_PageGrain_rw_bitmask) |
                          (env->CP0_PageGrain & ~env->CP0_PageGrain_rw_bitmask);
+    compute_hflags(env);
+    restore_pamask(env);
 }
 
 void helper_mtc0_wired(CPUMIPSState *env, target_ulong arg1)
@@ -1352,6 +1375,13 @@
 {
     uint32_t mask = 0x0000000F;
 
+    if ((env->CP0_Config1 & (1 << CP0C1_PC)) &&
+        (env->insn_flags & ISA_MIPS32R6)) {
+        mask |= (1 << 4);
+    }
+    if (env->insn_flags & ISA_MIPS32R6) {
+        mask |= (1 << 5);
+    }
     if (env->CP0_Config3 & (1 << CP0C3_ULRI)) {
         mask |= (1 << 29);
 
@@ -1373,7 +1403,7 @@
 void helper_mtc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 {
     target_ulong old, val, mask;
-    mask = (TARGET_PAGE_MASK << 1) | 0xFF;
+    mask = (TARGET_PAGE_MASK << 1) | env->CP0_EntryHi_ASID_mask;
     if (((env->CP0_Config4 >> CP0C4_IE) & 0x3) >= 2) {
         mask |= 1 << CP0EnHi_EHINV;
     }
@@ -1399,8 +1429,10 @@
         sync_c0_entryhi(env, env->current_tc);
     }
     /* If the ASID changes, flush qemu's TLB.  */
-    if ((old & 0xFF) != (val & 0xFF))
+    if ((old & env->CP0_EntryHi_ASID_mask) !=
+        (val & env->CP0_EntryHi_ASID_mask)) {
         cpu_mips_tlb_flush(env, 1);
+    }
 }
 
 void helper_mttc0_entryhi(CPUMIPSState *env, target_ulong arg1)
@@ -1421,25 +1453,10 @@
 {
     MIPSCPU *cpu = mips_env_get_cpu(env);
     uint32_t val, old;
-    uint32_t mask = env->CP0_Status_rw_bitmask;
 
-    if (env->insn_flags & ISA_MIPS32R6) {
-        bool has_supervisor = extract32(mask, CP0St_KSU, 2) == 0x3;
-
-        if (has_supervisor && extract32(arg1, CP0St_KSU, 2) == 0x3) {
-            mask &= ~(3 << CP0St_KSU);
-        }
-        mask &= ~(((1 << CP0St_SR) | (1 << CP0St_NMI)) & arg1);
-    }
-
-    val = arg1 & mask;
     old = env->CP0_Status;
-    env->CP0_Status = (env->CP0_Status & ~mask) | val;
-    if (env->CP0_Config3 & (1 << CP0C3_MT)) {
-        sync_c0_status(env, env, env->current_tc);
-    } else {
-        compute_hflags(env);
-    }
+    cpu_mips_store_status(env, arg1);
+    val = env->CP0_Status;
 
     if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
         qemu_log("Status %08x (%08x) => %08x (%08x) Cause %08x",
@@ -1460,15 +1477,15 @@
 void helper_mttc0_status(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
+    uint32_t mask = env->CP0_Status_rw_bitmask & ~0xf1000018;
     CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
-    other->CP0_Status = arg1 & ~0xf1000018;
+    other->CP0_Status = (other->CP0_Status & ~mask) | (arg1 & mask);
     sync_c0_status(env, other, other_tc);
 }
 
 void helper_mtc0_intctl(CPUMIPSState *env, target_ulong arg1)
 {
-    /* vectored interrupts not implemented, no performance counters. */
     env->CP0_IntCtl = (env->CP0_IntCtl & ~0x000003e0) | (arg1 & 0x000003e0);
 }
 
@@ -1478,40 +1495,9 @@
     env->CP0_SRSCtl = (env->CP0_SRSCtl & ~mask) | (arg1 & mask);
 }
 
-static void mtc0_cause(CPUMIPSState *cpu, target_ulong arg1)
-{
-    uint32_t mask = 0x00C00300;
-    uint32_t old = cpu->CP0_Cause;
-    int i;
-
-    if (cpu->insn_flags & ISA_MIPS32R2) {
-        mask |= 1 << CP0Ca_DC;
-    }
-    if (cpu->insn_flags & ISA_MIPS32R6) {
-        mask &= ~((1 << CP0Ca_WP) & arg1);
-    }
-
-    cpu->CP0_Cause = (cpu->CP0_Cause & ~mask) | (arg1 & mask);
-
-    if ((old ^ cpu->CP0_Cause) & (1 << CP0Ca_DC)) {
-        if (cpu->CP0_Cause & (1 << CP0Ca_DC)) {
-            cpu_mips_stop_count(cpu);
-        } else {
-            cpu_mips_start_count(cpu);
-        }
-    }
-
-    /* Set/reset software interrupts */
-    for (i = 0 ; i < 2 ; i++) {
-        if ((old ^ cpu->CP0_Cause) & (1 << (CP0Ca_IP + i))) {
-            cpu_mips_soft_irq(cpu, i, cpu->CP0_Cause & (1 << (CP0Ca_IP + i)));
-        }
-    }
-}
-
 void helper_mtc0_cause(CPUMIPSState *env, target_ulong arg1)
 {
-    mtc0_cause(env, arg1);
+    cpu_mips_store_cause(env, arg1);
 }
 
 void helper_mttc0_cause(CPUMIPSState *env, target_ulong arg1)
@@ -1519,7 +1505,7 @@
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
-    mtc0_cause(other, arg1);
+    cpu_mips_store_cause(other, arg1);
 }
 
 target_ulong helper_mftc0_epc(CPUMIPSState *env)
@@ -1540,7 +1526,6 @@
 
 void helper_mtc0_ebase(CPUMIPSState *env, target_ulong arg1)
 {
-    /* vectored interrupts not implemented */
     env->CP0_EBase = (env->CP0_EBase & ~0x3FFFF000) | (arg1 & 0x3FFFF000);
 }
 
@@ -1581,6 +1566,14 @@
     env->CP0_Config2 = (env->CP0_Config2 & 0x8FFF0FFF);
 }
 
+void helper_mtc0_config3(CPUMIPSState *env, target_ulong arg1)
+{
+    if (env->insn_flags & ASE_MICROMIPS) {
+        env->CP0_Config3 = (env->CP0_Config3 & ~(1 << CP0C3_ISA_ON_EXC)) |
+                           (arg1 & (1 << CP0C3_ISA_ON_EXC));
+    }
+}
+
 void helper_mtc0_config4(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Config4 = (env->CP0_Config4 & (~env->CP0_Config4_rw_bitmask)) |
@@ -1601,6 +1594,36 @@
     env->lladdr = (env->lladdr & ~mask) | (arg1 & mask);
 }
 
+#define MTC0_MAAR_MASK(env) \
+        ((0x1ULL << 63) | ((env->PAMask >> 4) & ~0xFFFull) | 0x3)
+
+void helper_mtc0_maar(CPUMIPSState *env, target_ulong arg1)
+{
+    env->CP0_MAAR[env->CP0_MAARI] = arg1 & MTC0_MAAR_MASK(env);
+}
+
+void helper_mthc0_maar(CPUMIPSState *env, target_ulong arg1)
+{
+    env->CP0_MAAR[env->CP0_MAARI] =
+        (((uint64_t) arg1 << 32) & MTC0_MAAR_MASK(env)) |
+        (env->CP0_MAAR[env->CP0_MAARI] & 0x00000000ffffffffULL);
+}
+
+void helper_mtc0_maari(CPUMIPSState *env, target_ulong arg1)
+{
+    int index = arg1 & 0x3f;
+    if (index == 0x3f) {
+        /* Software may write all ones to INDEX to determine the
+           maximum value supported. */
+        env->CP0_MAARI = MIPS_MAAR_MAX - 1;
+    } else if (index < MIPS_MAAR_MAX) {
+        env->CP0_MAARI = index;
+    }
+    /* Other than the all ones, if the
+       value written is not supported, then INDEX is unchanged
+       from its previous value. */
+}
+
 void helper_mtc0_watchlo(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     /* Watch exceptions for instructions, data loads, data stores
@@ -1610,7 +1633,8 @@
 
 void helper_mtc0_watchhi(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
-    env->CP0_WatchHi[sel] = (arg1 & 0x40FF0FF8);
+    int mask = 0x40000FF8 | (env->CP0_EntryHi_ASID_mask << CP0WH_ASID);
+    env->CP0_WatchHi[sel] = arg1 & mask;
     env->CP0_WatchHi[sel] &= ~(env->CP0_WatchHi[sel] & arg1 & 0x7);
 }
 
@@ -1655,9 +1679,31 @@
     env->CP0_Performance0 = arg1 & 0x000007ff;
 }
 
+void helper_mtc0_errctl(CPUMIPSState *env, target_ulong arg1)
+{
+    int32_t wst = arg1 & (1 << CP0EC_WST);
+    int32_t spr = arg1 & (1 << CP0EC_SPR);
+    int32_t itc = env->itc_tag ? (arg1 & (1 << CP0EC_ITC)) : 0;
+
+    env->CP0_ErrCtl = wst | spr | itc;
+
+    if (itc && !wst && !spr) {
+        env->hflags |= MIPS_HFLAG_ITC_CACHE;
+    } else {
+        env->hflags &= ~MIPS_HFLAG_ITC_CACHE;
+    }
+}
+
 void helper_mtc0_taglo(CPUMIPSState *env, target_ulong arg1)
 {
-    env->CP0_TagLo = arg1 & 0xFFFFFCF6;
+    if (env->hflags & MIPS_HFLAG_ITC_CACHE) {
+        /* If CACHE instruction is configured for ITC tags then make all
+           CP0.TagLo bits writable. The actual write to ITC Configuration
+           Tag will take care of the read-only bits. */
+        env->CP0_TagLo = arg1;
+    } else {
+        env->CP0_TagLo = arg1 & 0xFFFFFCF6;
+    }
 }
 
 void helper_mtc0_datalo(CPUMIPSState *env, target_ulong arg1)
@@ -1852,13 +1898,13 @@
                 env->active_tc.CP0_TCStatus & (1 << CP0TCSt_DT)) {
                 env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
                 env->CP0_VPEControl |= 4 << CP0VPECo_EXCPT;
-                helper_raise_exception(env, EXCP_THREAD);
+                do_raise_exception(env, EXCP_THREAD, GETPC());
             }
         }
     } else if (arg1 == 0) {
         if (0 /* TODO: TC underflow */) {
             env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
-            helper_raise_exception(env, EXCP_THREAD);
+            do_raise_exception(env, EXCP_THREAD, GETPC());
         } else {
             // TODO: Deallocate TC
         }
@@ -1866,22 +1912,53 @@
         /* Yield qualifier inputs not implemented. */
         env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
         env->CP0_VPEControl |= 2 << CP0VPECo_EXCPT;
-        helper_raise_exception(env, EXCP_THREAD);
+        do_raise_exception(env, EXCP_THREAD, GETPC());
     }
     return env->CP0_YQMask;
 }
 
+/* R6 Multi-threading */
 #ifndef CONFIG_USER_ONLY
-/* TLB management */
-static void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global)
+target_ulong helper_dvp(CPUMIPSState *env)
 {
-    MIPSCPU *cpu = mips_env_get_cpu(env);
+    CPUState *other_cs = first_cpu;
+    target_ulong prev = env->CP0_VPControl;
 
-    /* Flush qemu's TLB and discard all shadowed entries.  */
-    tlb_flush(CPU(cpu), flush_global);
-    env->tlb->tlb_in_use = env->tlb->nb_tlb;
+    if (!((env->CP0_VPControl >> CP0VPCtl_DIS) & 1)) {
+        CPU_FOREACH(other_cs) {
+            MIPSCPU *other_cpu = MIPS_CPU(other_cs);
+            /* Turn off all VPs except the one executing the dvp. */
+            if (&other_cpu->env != env) {
+                mips_vpe_sleep(other_cpu);
+            }
+        }
+        env->CP0_VPControl |= (1 << CP0VPCtl_DIS);
+    }
+    return prev;
 }
 
+target_ulong helper_evp(CPUMIPSState *env)
+{
+    CPUState *other_cs = first_cpu;
+    target_ulong prev = env->CP0_VPControl;
+
+    if ((env->CP0_VPControl >> CP0VPCtl_DIS) & 1) {
+        CPU_FOREACH(other_cs) {
+            MIPSCPU *other_cpu = MIPS_CPU(other_cs);
+            if ((&other_cpu->env != env) && !mips_vp_is_wfi(other_cpu)) {
+                /* If the VP is WFI, don't disturb its sleep.
+                 * Otherwise, wake it up. */
+                mips_vpe_wake(other_cpu);
+            }
+        }
+        env->CP0_VPControl &= ~(1 << CP0VPCtl_DIS);
+    }
+    return prev;
+}
+#endif /* !CONFIG_USER_ONLY */
+
+#ifndef CONFIG_USER_ONLY
+/* TLB management */
 static void r4k_mips_tlb_flush_extra (CPUMIPSState *env, int first)
 {
     /* Discard entries from env->tlb[first] onwards.  */
@@ -1890,6 +1967,16 @@
     }
 }
 
+static inline uint64_t get_tlb_pfn_from_entrylo(uint64_t entrylo)
+{
+#if defined(TARGET_MIPS64)
+    return extract64(entrylo, 6, 54);
+#else
+    return extract64(entrylo, 6, 24) | /* PFN */
+           (extract64(entrylo, 32, 32) << 24); /* PFNX */
+#endif
+}
+
 static void r4k_fill_tlb(CPUMIPSState *env, int idx)
 {
     r4k_tlb_t *tlb;
@@ -1905,7 +1992,7 @@
 #if defined(TARGET_MIPS64)
     tlb->VPN &= env->SEGMask;
 #endif
-    tlb->ASID = env->CP0_EntryHi & 0xFF;
+    tlb->ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
     tlb->PageMask = env->CP0_PageMask;
     tlb->G = env->CP0_EntryLo0 & env->CP0_EntryLo1 & 1;
     tlb->V0 = (env->CP0_EntryLo0 & 2) != 0;
@@ -1913,20 +2000,20 @@
     tlb->C0 = (env->CP0_EntryLo0 >> 3) & 0x7;
     tlb->XI0 = (env->CP0_EntryLo0 >> CP0EnLo_XI) & 1;
     tlb->RI0 = (env->CP0_EntryLo0 >> CP0EnLo_RI) & 1;
-    tlb->PFN[0] = (env->CP0_EntryLo0 >> 6) << 12;
+    tlb->PFN[0] = get_tlb_pfn_from_entrylo(env->CP0_EntryLo0) << 12;
     tlb->V1 = (env->CP0_EntryLo1 & 2) != 0;
     tlb->D1 = (env->CP0_EntryLo1 & 4) != 0;
     tlb->C1 = (env->CP0_EntryLo1 >> 3) & 0x7;
     tlb->XI1 = (env->CP0_EntryLo1 >> CP0EnLo_XI) & 1;
     tlb->RI1 = (env->CP0_EntryLo1 >> CP0EnLo_RI) & 1;
-    tlb->PFN[1] = (env->CP0_EntryLo1 >> 6) << 12;
+    tlb->PFN[1] = get_tlb_pfn_from_entrylo(env->CP0_EntryLo1) << 12;
 }
 
 void r4k_helper_tlbinv(CPUMIPSState *env)
 {
     int idx;
     r4k_tlb_t *tlb;
-    uint8_t ASID = env->CP0_EntryHi & 0xFF;
+    uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
 
     for (idx = 0; idx < env->tlb->nb_tlb; idx++) {
         tlb = &env->tlb->mmu.r4k.tlb[idx];
@@ -1952,7 +2039,7 @@
     r4k_tlb_t *tlb;
     int idx;
     target_ulong VPN;
-    uint8_t ASID;
+    uint16_t ASID;
     bool G, V0, D0, V1, D1;
 
     idx = (env->CP0_Index & ~0x80000000) % env->tlb->nb_tlb;
@@ -1961,7 +2048,7 @@
 #if defined(TARGET_MIPS64)
     VPN &= env->SEGMask;
 #endif
-    ASID = env->CP0_EntryHi & 0xff;
+    ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
     G = env->CP0_EntryLo0 & env->CP0_EntryLo1 & 1;
     V0 = (env->CP0_EntryLo0 & 2) != 0;
     D0 = (env->CP0_EntryLo0 & 4) != 0;
@@ -1994,10 +2081,10 @@
     target_ulong mask;
     target_ulong tag;
     target_ulong VPN;
-    uint8_t ASID;
+    uint16_t ASID;
     int i;
 
-    ASID = env->CP0_EntryHi & 0xFF;
+    ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
     for (i = 0; i < env->tlb->nb_tlb; i++) {
         tlb = &env->tlb->mmu.r4k.tlb[i];
         /* 1k pages are not supported. */
@@ -2036,13 +2123,23 @@
     }
 }
 
+static inline uint64_t get_entrylo_pfn_from_tlb(uint64_t tlb_pfn)
+{
+#if defined(TARGET_MIPS64)
+    return tlb_pfn << 6;
+#else
+    return (extract64(tlb_pfn, 0, 24) << 6) | /* PFN */
+           (extract64(tlb_pfn, 24, 32) << 32); /* PFNX */
+#endif
+}
+
 void r4k_helper_tlbr(CPUMIPSState *env)
 {
     r4k_tlb_t *tlb;
-    uint8_t ASID;
+    uint16_t ASID;
     int idx;
 
-    ASID = env->CP0_EntryHi & 0xFF;
+    ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
     idx = (env->CP0_Index & ~0x80000000) % env->tlb->nb_tlb;
     tlb = &env->tlb->mmu.r4k.tlb[idx];
 
@@ -2061,13 +2158,13 @@
         env->CP0_EntryHi = tlb->VPN | tlb->ASID;
         env->CP0_PageMask = tlb->PageMask;
         env->CP0_EntryLo0 = tlb->G | (tlb->V0 << 1) | (tlb->D0 << 2) |
-                        ((target_ulong)tlb->RI0 << CP0EnLo_RI) |
-                        ((target_ulong)tlb->XI0 << CP0EnLo_XI) |
-                        (tlb->C0 << 3) | (tlb->PFN[0] >> 6);
+                        ((uint64_t)tlb->RI0 << CP0EnLo_RI) |
+                        ((uint64_t)tlb->XI0 << CP0EnLo_XI) | (tlb->C0 << 3) |
+                        get_entrylo_pfn_from_tlb(tlb->PFN[0] >> 12);
         env->CP0_EntryLo1 = tlb->G | (tlb->V1 << 1) | (tlb->D1 << 2) |
-                        ((target_ulong)tlb->RI1 << CP0EnLo_RI) |
-                        ((target_ulong)tlb->XI1 << CP0EnLo_XI) |
-                        (tlb->C1 << 3) | (tlb->PFN[1] >> 6);
+                        ((uint64_t)tlb->RI1 << CP0EnLo_RI) |
+                        ((uint64_t)tlb->XI1 << CP0EnLo_XI) | (tlb->C1 << 3) |
+                        get_entrylo_pfn_from_tlb(tlb->PFN[1] >> 12);
     }
 }
 
@@ -2193,60 +2290,58 @@
     debug_pre_eret(env);
     set_pc(env, env->CP0_DEPC);
 
-    env->hflags &= MIPS_HFLAG_DM;
+    env->hflags &= ~MIPS_HFLAG_DM;
     compute_hflags(env);
     debug_post_eret(env);
-    env->lladdr = 1;
 }
 #endif /* !CONFIG_USER_ONLY */
 
+static inline void check_hwrena(CPUMIPSState *env, int reg, uintptr_t pc)
+{
+    if ((env->hflags & MIPS_HFLAG_CP0) || (env->CP0_HWREna & (1 << reg))) {
+        return;
+    }
+    do_raise_exception(env, EXCP_RI, pc);
+}
+
 target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 0)))
-        return env->CP0_EBase & 0x3ff;
-    else
-        helper_raise_exception(env, EXCP_RI);
-
-    return 0;
+    check_hwrena(env, 0, GETPC());
+    return env->CP0_EBase & 0x3ff;
 }
 
 target_ulong helper_rdhwr_synci_step(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 1)))
-        return env->SYNCI_Step;
-    else
-        helper_raise_exception(env, EXCP_RI);
-
-    return 0;
+    check_hwrena(env, 1, GETPC());
+    return env->SYNCI_Step;
 }
 
 target_ulong helper_rdhwr_cc(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 2))) {
+    check_hwrena(env, 2, GETPC());
 #ifdef CONFIG_USER_ONLY
-        return env->CP0_Count;
+    return env->CP0_Count;
 #else
-        return (int32_t)cpu_mips_get_count(env);
+    return (int32_t)cpu_mips_get_count(env);
 #endif
-    } else {
-         helper_raise_exception(env, EXCP_RI);
-    }
-
-    return 0;
 }
 
 target_ulong helper_rdhwr_ccres(CPUMIPSState *env)
 {
-    if ((env->hflags & MIPS_HFLAG_CP0) ||
-        (env->CP0_HWREna & (1 << 3)))
-        return env->CCRes;
-    else
-        helper_raise_exception(env, EXCP_RI);
+    check_hwrena(env, 3, GETPC());
+    return env->CCRes;
+}
 
-    return 0;
+target_ulong helper_rdhwr_performance(CPUMIPSState *env)
+{
+    check_hwrena(env, 4, GETPC());
+    return env->CP0_Performance0;
+}
+
+target_ulong helper_rdhwr_xnp(CPUMIPSState *env)
+{
+    check_hwrena(env, 5, GETPC());
+    return (env->CP0_Config5 >> CP0C5_XNP) & 1;
 }
 
 void helper_pmon(CPUMIPSState *env, int function)
@@ -2281,20 +2376,25 @@
 
     cs->halted = 1;
     cpu_reset_interrupt(cs, CPU_INTERRUPT_WAKE);
-    helper_raise_exception(env, EXCP_HLT);
+    /* Last instruction in the block, PC was updated before
+       - no need to recover PC and icount */
+    raise_exception(env, EXCP_HLT);
 }
 
 #if !defined(CONFIG_USER_ONLY)
 
 void mips_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
-                                  int access_type, int is_user,
-                                  uintptr_t retaddr)
+                                  MMUAccessType access_type,
+                                  int mmu_idx, uintptr_t retaddr,
+                                  unsigned size)
 {
     MIPSCPU *cpu = MIPS_CPU(cs);
     CPUMIPSState *env = &cpu->env;
     int error_code = 0;
     int excp;
 
+    (void)size;
+
     env->CP0_BadVAddr = addr;
 
     if (access_type == MMU_DATA_STORE) {
@@ -2309,12 +2409,12 @@
     do_raise_exception_err(env, excp, error_code, retaddr);
 }
 
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = mips_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = mips_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (ret) {
         MIPSCPU *cpu = MIPS_CPU(cs);
         CPUMIPSState *env = &cpu->env;
@@ -2342,9 +2442,9 @@
     }
 
     if (is_exec) {
-        helper_raise_exception(env, EXCP_IBE);
+        raise_exception(env, EXCP_IBE);
     } else {
-        helper_raise_exception(env, EXCP_DBE);
+        raise_exception(env, EXCP_DBE);
     }
 }
 #endif /* !CONFIG_USER_ONLY */
@@ -2353,6 +2453,7 @@
 
 #define FLOAT_TWO32 make_float32(1 << 30)
 #define FLOAT_TWO64 make_float64(1ULL << 62)
+
 #define FP_TO_INT32_OVERFLOW 0x7fffffff
 #define FP_TO_INT64_OVERFLOW 0x7fffffffffffffffULL
 
@@ -2364,18 +2465,6 @@
     float_round_down
 };
 
-static inline void restore_rounding_mode(CPUMIPSState *env)
-{
-    set_float_rounding_mode(ieee_rm[env->active_fpu.fcr31 & 3],
-                            &env->active_fpu.fp_status);
-}
-
-static inline void restore_flush_mode(CPUMIPSState *env)
-{
-    set_flush_to_zero((env->active_fpu.fcr31 & (1 << 24)) != 0,
-                      &env->active_fpu.fp_status);
-}
-
 target_ulong helper_cfc1(CPUMIPSState *env, uint32_t reg)
 {
     target_ulong arg1 = 0;
@@ -2391,12 +2480,12 @@
                 arg1 = (int32_t)
                        ((env->CP0_Status & (1  << CP0St_FR)) >> CP0St_FR);
             } else {
-                helper_raise_exception(env, EXCP_RI);
+                do_raise_exception(env, EXCP_RI, GETPC());
             }
         }
         break;
     case 5:
-        /* FRE support - read Config5.FRE bit */
+        /* FRE Support - read Config5.FRE bit */
         if (env->active_fpu.fcr0 & (1 << FCR0_FREP)) {
             if (env->CP0_Config5 & (1 << CP0C5_UFE)) {
                 arg1 = (env->CP0_Config5 >> CP0C5_FRE) & 1;
@@ -2434,7 +2523,7 @@
             env->CP0_Status &= ~(1 << CP0St_FR);
             compute_hflags(env);
         } else {
-            helper_raise_exception(env, EXCP_RI);
+            do_raise_exception(env, EXCP_RI, GETPC());
         }
         break;
     case 4:
@@ -2446,7 +2535,7 @@
             env->CP0_Status |= (1 << CP0St_FR);
             compute_hflags(env);
         } else {
-            helper_raise_exception(env, EXCP_RI);
+            do_raise_exception(env, EXCP_RI, GETPC());
         }
         break;
     case 5:
@@ -2462,7 +2551,7 @@
         }
         break;
     case 6:
-        /* FRE support - set Config5.FRE bit */
+        /* FRE Support - set Config5.FRE bit */
         if (!((env->active_fpu.fcr0 & (1 << FCR0_FREP)) && (rt == 0))) {
             return;
         }
@@ -2492,21 +2581,13 @@
                      ((arg1 & 0x4) << 22);
         break;
     case 31:
-        if (env->insn_flags & ISA_MIPS32R6) {
-            uint32_t mask = 0xfefc0000;
-            env->active_fpu.fcr31 = (arg1 & ~mask) |
-                (env->active_fpu.fcr31 & mask);
-        } else if (!(arg1 & 0x007c0000)) {
-            env->active_fpu.fcr31 = arg1;
-        }
+        env->active_fpu.fcr31 = (arg1 & env->active_fpu.fcr31_rw_bitmask) |
+               (env->active_fpu.fcr31 & ~(env->active_fpu.fcr31_rw_bitmask));
         break;
     default:
         return;
     }
-    /* set rounding mode */
-    restore_rounding_mode(env);
-    /* set flush-to-zero mode */
-    restore_flush_mode(env);
+    restore_fp_status(env);
     set_float_exception_flags(0, &env->active_fpu.fp_status);
     if ((GET_FP_ENABLE(env->active_fpu.fcr31) | 0x20) & GET_FP_CAUSE(env->active_fpu.fcr31))
         do_raise_exception(env, EXCP_FPE, GETPC());
@@ -2577,6 +2658,7 @@
     uint64_t fdt2;
 
     fdt2 = float32_to_float64(fst0, &env->active_fpu.fp_status);
+    fdt2 = float64_maybe_silence_nan(fdt2, &env->active_fpu.fp_status);
     update_fcr31(env, GETPC());
     return fdt2;
 }
@@ -2599,7 +2681,7 @@
     return fdt2;
 }
 
-uint64_t helper_float_cvtl_d(CPUMIPSState *env, uint64_t fdt0)
+uint64_t helper_float_cvt_l_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
@@ -2612,7 +2694,7 @@
     return dt2;
 }
 
-uint64_t helper_float_cvtl_s(CPUMIPSState *env, uint32_t fst0)
+uint64_t helper_float_cvt_l_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
@@ -2666,6 +2748,7 @@
     uint32_t fst2;
 
     fst2 = float64_to_float32(fdt0, &env->active_fpu.fp_status);
+    fst2 = float32_maybe_silence_nan(fst2, &env->active_fpu.fp_status);
     update_fcr31(env, GETPC());
     return fst2;
 }
@@ -2706,20 +2789,20 @@
     return wt2;
 }
 
-uint32_t helper_float_cvtw_s(CPUMIPSState *env, uint32_t fst0)
+uint32_t helper_float_cvt_w_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    update_fcr31(env, GETPC());
     if (get_float_exception_flags(&env->active_fpu.fp_status)
         & (float_flag_invalid | float_flag_overflow)) {
         wt2 = FP_TO_INT32_OVERFLOW;
     }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_cvtw_d(CPUMIPSState *env, uint64_t fdt0)
+uint32_t helper_float_cvt_w_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
@@ -2732,7 +2815,7 @@
     return wt2;
 }
 
-uint64_t helper_float_roundl_d(CPUMIPSState *env, uint64_t fdt0)
+uint64_t helper_float_round_l_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
@@ -2747,7 +2830,7 @@
     return dt2;
 }
 
-uint64_t helper_float_roundl_s(CPUMIPSState *env, uint32_t fst0)
+uint64_t helper_float_round_l_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
@@ -2762,7 +2845,7 @@
     return dt2;
 }
 
-uint32_t helper_float_roundw_d(CPUMIPSState *env, uint64_t fdt0)
+uint32_t helper_float_round_w_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
@@ -2777,7 +2860,7 @@
     return wt2;
 }
 
-uint32_t helper_float_roundw_s(CPUMIPSState *env, uint32_t fst0)
+uint32_t helper_float_round_w_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
@@ -2792,7 +2875,7 @@
     return wt2;
 }
 
-uint64_t helper_float_truncl_d(CPUMIPSState *env, uint64_t fdt0)
+uint64_t helper_float_trunc_l_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
@@ -2805,7 +2888,7 @@
     return dt2;
 }
 
-uint64_t helper_float_truncl_s(CPUMIPSState *env, uint32_t fst0)
+uint64_t helper_float_trunc_l_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
@@ -2818,7 +2901,7 @@
     return dt2;
 }
 
-uint32_t helper_float_truncw_d(CPUMIPSState *env, uint64_t fdt0)
+uint32_t helper_float_trunc_w_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
@@ -2831,7 +2914,7 @@
     return wt2;
 }
 
-uint32_t helper_float_truncw_s(CPUMIPSState *env, uint32_t fst0)
+uint32_t helper_float_trunc_w_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
@@ -2844,7 +2927,7 @@
     return wt2;
 }
 
-uint64_t helper_float_ceill_d(CPUMIPSState *env, uint64_t fdt0)
+uint64_t helper_float_ceil_l_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
@@ -2859,7 +2942,7 @@
     return dt2;
 }
 
-uint64_t helper_float_ceill_s(CPUMIPSState *env, uint32_t fst0)
+uint64_t helper_float_ceil_l_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
@@ -2874,7 +2957,7 @@
     return dt2;
 }
 
-uint32_t helper_float_ceilw_d(CPUMIPSState *env, uint64_t fdt0)
+uint32_t helper_float_ceil_w_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
@@ -2889,7 +2972,7 @@
     return wt2;
 }
 
-uint32_t helper_float_ceilw_s(CPUMIPSState *env, uint32_t fst0)
+uint32_t helper_float_ceil_w_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
@@ -2904,7 +2987,7 @@
     return wt2;
 }
 
-uint64_t helper_float_floorl_d(CPUMIPSState *env, uint64_t fdt0)
+uint64_t helper_float_floor_l_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
@@ -2919,7 +3002,7 @@
     return dt2;
 }
 
-uint64_t helper_float_floorl_s(CPUMIPSState *env, uint32_t fst0)
+uint64_t helper_float_floor_l_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
@@ -2934,7 +3017,7 @@
     return dt2;
 }
 
-uint32_t helper_float_floorw_d(CPUMIPSState *env, uint64_t fdt0)
+uint32_t helper_float_floor_w_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
@@ -2949,7 +3032,7 @@
     return wt2;
 }
 
-uint32_t helper_float_floorw_s(CPUMIPSState *env, uint32_t fst0)
+uint32_t helper_float_floor_w_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
@@ -2964,6 +3047,334 @@
     return wt2;
 }
 
+uint64_t helper_float_cvt_2008_l_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint64_t dt2;
+
+    dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint64_t helper_float_cvt_2008_l_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint64_t dt2;
+
+    dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint32_t helper_float_cvt_2008_w_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint32_t wt2;
+
+    wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint32_t helper_float_cvt_2008_w_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint32_t wt2;
+
+    wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint64_t helper_float_round_2008_l_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint64_t dt2;
+
+    set_float_rounding_mode(float_round_nearest_even,
+            &env->active_fpu.fp_status);
+    dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint64_t helper_float_round_2008_l_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint64_t dt2;
+
+    set_float_rounding_mode(float_round_nearest_even,
+            &env->active_fpu.fp_status);
+    dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint32_t helper_float_round_2008_w_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint32_t wt2;
+
+    set_float_rounding_mode(float_round_nearest_even,
+            &env->active_fpu.fp_status);
+    wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint32_t helper_float_round_2008_w_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint32_t wt2;
+
+    set_float_rounding_mode(float_round_nearest_even,
+            &env->active_fpu.fp_status);
+    wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint64_t helper_float_trunc_2008_l_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint64_t dt2;
+
+    dt2 = float64_to_int64_round_to_zero(fdt0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint64_t helper_float_trunc_2008_l_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint64_t dt2;
+
+    dt2 = float32_to_int64_round_to_zero(fst0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint32_t helper_float_trunc_2008_w_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint32_t wt2;
+
+    wt2 = float64_to_int32_round_to_zero(fdt0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint32_t helper_float_trunc_2008_w_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint32_t wt2;
+
+    wt2 = float32_to_int32_round_to_zero(fst0, &env->active_fpu.fp_status);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint64_t helper_float_ceil_2008_l_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint64_t dt2;
+
+    set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
+    dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint64_t helper_float_ceil_2008_l_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint64_t dt2;
+
+    set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
+    dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint32_t helper_float_ceil_2008_w_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint32_t wt2;
+
+    set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
+    wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint32_t helper_float_ceil_2008_w_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint32_t wt2;
+
+    set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
+    wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint64_t helper_float_floor_2008_l_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint64_t dt2;
+
+    set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
+    dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint64_t helper_float_floor_2008_l_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint64_t dt2;
+
+    set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
+    dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            dt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return dt2;
+}
+
+uint32_t helper_float_floor_2008_w_d(CPUMIPSState *env, uint64_t fdt0)
+{
+    uint32_t wt2;
+
+    set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
+    wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float64_is_any_nan(fdt0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
+uint32_t helper_float_floor_2008_w_s(CPUMIPSState *env, uint32_t fst0)
+{
+    uint32_t wt2;
+
+    set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
+    wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
+    restore_rounding_mode(env);
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+            & float_flag_invalid) {
+        if (float32_is_any_nan(fst0)) {
+            wt2 = 0;
+        }
+    }
+    update_fcr31(env, GETPC());
+    return wt2;
+}
+
 /* unary operations, not modifying fp status  */
 #define FLOAT_UNOP(name)                                       \
 uint64_t helper_float_ ## name ## _d(uint64_t fdt0)                \
@@ -2987,110 +3398,6 @@
 FLOAT_UNOP(chs)
 #undef FLOAT_UNOP
 
-#define FLOAT_FMADDSUB(name, bits, muladd_arg)                          \
-uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,            \
-                                          uint ## bits ## _t fs,        \
-                                          uint ## bits ## _t ft,        \
-                                          uint ## bits ## _t fd)        \
-{                                                                       \
-    uint ## bits ## _t fdret;                                           \
-                                                                        \
-    fdret = float ## bits ## _muladd(fs, ft, fd, muladd_arg,            \
-                                     &env->active_fpu.fp_status);       \
-    update_fcr31(env, GETPC());                                         \
-    return fdret;                                                       \
-}
-
-FLOAT_FMADDSUB(maddf_s, 32, 0)
-FLOAT_FMADDSUB(maddf_d, 64, 0)
-FLOAT_FMADDSUB(msubf_s, 32, float_muladd_negate_product)
-FLOAT_FMADDSUB(msubf_d, 64, float_muladd_negate_product)
-#undef FLOAT_FMADDSUB
-
-#define FLOAT_MINMAX(name, bits, minmaxfunc)                            \
-uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,            \
-                                          uint ## bits ## _t fs,        \
-                                          uint ## bits ## _t ft)        \
-{                                                                       \
-    uint ## bits ## _t fdret;                                           \
-                                                                        \
-    fdret = float ## bits ## _ ## minmaxfunc(fs, ft,                    \
-                                           &env->active_fpu.fp_status); \
-    update_fcr31(env, GETPC());                                         \
-    return fdret;                                                       \
-}
-
-FLOAT_MINMAX(max_s, 32, maxnum)
-FLOAT_MINMAX(max_d, 64, maxnum)
-FLOAT_MINMAX(maxa_s, 32, maxnummag)
-FLOAT_MINMAX(maxa_d, 64, maxnummag)
-
-FLOAT_MINMAX(min_s, 32, minnum)
-FLOAT_MINMAX(min_d, 64, minnum)
-FLOAT_MINMAX(mina_s, 32, minnummag)
-FLOAT_MINMAX(mina_d, 64, minnummag)
-#undef FLOAT_MINMAX
-
-#define FLOAT_RINT(name, bits)                                              \
-uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,                \
-                                          uint ## bits ## _t fs)            \
-{                                                                           \
-    uint ## bits ## _t fdret;                                               \
-                                                                            \
-    fdret = float ## bits ## _round_to_int(fs, &env->active_fpu.fp_status); \
-    update_fcr31(env, GETPC());                                             \
-    return fdret;                                                           \
-}
-
-FLOAT_RINT(rint_s, 32)
-FLOAT_RINT(rint_d, 64)
-#undef FLOAT_RINT
-
-#define FLOAT_CLASS_SIGNALING_NAN      0x001
-#define FLOAT_CLASS_QUIET_NAN          0x002
-#define FLOAT_CLASS_NEGATIVE_INFINITY  0x004
-#define FLOAT_CLASS_NEGATIVE_NORMAL    0x008
-#define FLOAT_CLASS_NEGATIVE_SUBNORMAL 0x010
-#define FLOAT_CLASS_NEGATIVE_ZERO      0x020
-#define FLOAT_CLASS_POSITIVE_INFINITY  0x040
-#define FLOAT_CLASS_POSITIVE_NORMAL    0x080
-#define FLOAT_CLASS_POSITIVE_SUBNORMAL 0x100
-#define FLOAT_CLASS_POSITIVE_ZERO      0x200
-
-#define FLOAT_CLASS(name, bits)                                      \
-uint ## bits ## _t helper_float_ ## name (uint ## bits ## _t arg)    \
-{                                                                    \
-    if (float ## bits ## _is_signaling_nan(arg)) {                   \
-        return FLOAT_CLASS_SIGNALING_NAN;                            \
-    } else if (float ## bits ## _is_quiet_nan(arg)) {                \
-        return FLOAT_CLASS_QUIET_NAN;                                \
-    } else if (float ## bits ## _is_neg(arg)) {                      \
-        if (float ## bits ## _is_infinity(arg)) {                    \
-            return FLOAT_CLASS_NEGATIVE_INFINITY;                    \
-        } else if (float ## bits ## _is_zero(arg)) {                 \
-            return FLOAT_CLASS_NEGATIVE_ZERO;                        \
-        } else if (float ## bits ## _is_zero_or_denormal(arg)) {     \
-            return FLOAT_CLASS_NEGATIVE_SUBNORMAL;                   \
-        } else {                                                     \
-            return FLOAT_CLASS_NEGATIVE_NORMAL;                      \
-        }                                                            \
-    } else {                                                         \
-        if (float ## bits ## _is_infinity(arg)) {                    \
-            return FLOAT_CLASS_POSITIVE_INFINITY;                    \
-        } else if (float ## bits ## _is_zero(arg)) {                 \
-            return FLOAT_CLASS_POSITIVE_ZERO;                        \
-        } else if (float ## bits ## _is_zero_or_denormal(arg)) {     \
-            return FLOAT_CLASS_POSITIVE_SUBNORMAL;                   \
-        } else {                                                     \
-            return FLOAT_CLASS_POSITIVE_NORMAL;                      \
-        }                                                            \
-    }                                                                \
-}
-
-FLOAT_CLASS(class_s, 32)
-FLOAT_CLASS(class_d, 64)
-#undef FLOAT_CLASS
-
 /* MIPS specific unary operations */
 uint64_t helper_float_recip_d(CPUMIPSState *env, uint64_t fdt0)
 {
@@ -3192,7 +3499,72 @@
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-#define FLOAT_OP(name, p) void helper_float_##name##_##p(CPUMIPSState *env)
+#define FLOAT_RINT(name, bits)                                              \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,                \
+                                          uint ## bits ## _t fs)            \
+{                                                                           \
+    uint ## bits ## _t fdret;                                               \
+                                                                            \
+    fdret = float ## bits ## _round_to_int(fs, &env->active_fpu.fp_status); \
+    update_fcr31(env, GETPC());                                             \
+    return fdret;                                                           \
+}
+
+FLOAT_RINT(rint_s, 32)
+FLOAT_RINT(rint_d, 64)
+#undef FLOAT_RINT
+
+#define FLOAT_CLASS_SIGNALING_NAN      0x001
+#define FLOAT_CLASS_QUIET_NAN          0x002
+#define FLOAT_CLASS_NEGATIVE_INFINITY  0x004
+#define FLOAT_CLASS_NEGATIVE_NORMAL    0x008
+#define FLOAT_CLASS_NEGATIVE_SUBNORMAL 0x010
+#define FLOAT_CLASS_NEGATIVE_ZERO      0x020
+#define FLOAT_CLASS_POSITIVE_INFINITY  0x040
+#define FLOAT_CLASS_POSITIVE_NORMAL    0x080
+#define FLOAT_CLASS_POSITIVE_SUBNORMAL 0x100
+#define FLOAT_CLASS_POSITIVE_ZERO      0x200
+
+#define FLOAT_CLASS(name, bits)                                      \
+uint ## bits ## _t float_ ## name (uint ## bits ## _t arg,           \
+                                   float_status *status)             \
+{                                                                    \
+    if (float ## bits ## _is_signaling_nan(arg, status)) {           \
+        return FLOAT_CLASS_SIGNALING_NAN;                            \
+    } else if (float ## bits ## _is_quiet_nan(arg, status)) {        \
+        return FLOAT_CLASS_QUIET_NAN;                                \
+    } else if (float ## bits ## _is_neg(arg)) {                      \
+        if (float ## bits ## _is_infinity(arg)) {                    \
+            return FLOAT_CLASS_NEGATIVE_INFINITY;                    \
+        } else if (float ## bits ## _is_zero(arg)) {                 \
+            return FLOAT_CLASS_NEGATIVE_ZERO;                        \
+        } else if (float ## bits ## _is_zero_or_denormal(arg)) {     \
+            return FLOAT_CLASS_NEGATIVE_SUBNORMAL;                   \
+        } else {                                                     \
+            return FLOAT_CLASS_NEGATIVE_NORMAL;                      \
+        }                                                            \
+    } else {                                                         \
+        if (float ## bits ## _is_infinity(arg)) {                    \
+            return FLOAT_CLASS_POSITIVE_INFINITY;                    \
+        } else if (float ## bits ## _is_zero(arg)) {                 \
+            return FLOAT_CLASS_POSITIVE_ZERO;                        \
+        } else if (float ## bits ## _is_zero_or_denormal(arg)) {     \
+            return FLOAT_CLASS_POSITIVE_SUBNORMAL;                   \
+        } else {                                                     \
+            return FLOAT_CLASS_POSITIVE_NORMAL;                      \
+        }                                                            \
+    }                                                                \
+}                                                                    \
+                                                                     \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,         \
+                                          uint ## bits ## _t arg)    \
+{                                                                    \
+    return float_ ## name(arg, &env->active_fpu.fp_status);          \
+}
+
+FLOAT_CLASS(class_s, 32)
+FLOAT_CLASS(class_d, 64)
+#undef FLOAT_CLASS
 
 /* binary operations */
 #define FLOAT_BINOP(name)                                          \
@@ -3239,61 +3611,6 @@
 FLOAT_BINOP(div)
 #undef FLOAT_BINOP
 
-#define UNFUSED_FMA(prefix, a, b, c, flags)                          \
-{                                                                    \
-    a = prefix##_mul(a, b, &env->active_fpu.fp_status);              \
-    if ((flags) & float_muladd_negate_c) {                           \
-        a = prefix##_sub(a, c, &env->active_fpu.fp_status);          \
-    } else {                                                         \
-        a = prefix##_add(a, c, &env->active_fpu.fp_status);          \
-    }                                                                \
-    if ((flags) & float_muladd_negate_result) {                      \
-        a = prefix##_chs(a);                                         \
-    }                                                                \
-}
-
-/* FMA based operations */
-#define FLOAT_FMA(name, type)                                        \
-uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,              \
-                                     uint64_t fdt0, uint64_t fdt1,   \
-                                     uint64_t fdt2)                  \
-{                                                                    \
-    UNFUSED_FMA(float64, fdt0, fdt1, fdt2, type);                    \
-    update_fcr31(env, GETPC());                                      \
-    return fdt0;                                                     \
-}                                                                    \
-                                                                     \
-uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,              \
-                                     uint32_t fst0, uint32_t fst1,   \
-                                     uint32_t fst2)                  \
-{                                                                    \
-    UNFUSED_FMA(float32, fst0, fst1, fst2, type);                    \
-    update_fcr31(env, GETPC());                                      \
-    return fst0;                                                     \
-}                                                                    \
-                                                                     \
-uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,             \
-                                      uint64_t fdt0, uint64_t fdt1,  \
-                                      uint64_t fdt2)                 \
-{                                                                    \
-    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                               \
-    uint32_t fsth0 = fdt0 >> 32;                                     \
-    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                               \
-    uint32_t fsth1 = fdt1 >> 32;                                     \
-    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                               \
-    uint32_t fsth2 = fdt2 >> 32;                                     \
-                                                                     \
-    UNFUSED_FMA(float32, fst0, fst1, fst2, type);                    \
-    UNFUSED_FMA(float32, fsth0, fsth1, fsth2, type);                 \
-    update_fcr31(env, GETPC());                                      \
-    return ((uint64_t)fsth0 << 32) | fst0;                           \
-}
-FLOAT_FMA(madd, 0)
-FLOAT_FMA(msub, float_muladd_negate_c)
-FLOAT_FMA(nmadd, float_muladd_negate_result)
-FLOAT_FMA(nmsub, float_muladd_negate_result | float_muladd_negate_c)
-#undef FLOAT_FMA
-
 /* MIPS specific binary operations */
 uint64_t helper_float_recip2_d(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
@@ -3391,6 +3708,106 @@
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
+#define FLOAT_MINMAX(name, bits, minmaxfunc)                            \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,            \
+                                          uint ## bits ## _t fs,        \
+                                          uint ## bits ## _t ft)        \
+{                                                                       \
+    uint ## bits ## _t fdret;                                           \
+                                                                        \
+    fdret = float ## bits ## _ ## minmaxfunc(fs, ft,                    \
+                                           &env->active_fpu.fp_status); \
+    update_fcr31(env, GETPC());                                         \
+    return fdret;                                                       \
+}
+
+FLOAT_MINMAX(max_s, 32, maxnum)
+FLOAT_MINMAX(max_d, 64, maxnum)
+FLOAT_MINMAX(maxa_s, 32, maxnummag)
+FLOAT_MINMAX(maxa_d, 64, maxnummag)
+
+FLOAT_MINMAX(min_s, 32, minnum)
+FLOAT_MINMAX(min_d, 64, minnum)
+FLOAT_MINMAX(mina_s, 32, minnummag)
+FLOAT_MINMAX(mina_d, 64, minnummag)
+#undef FLOAT_MINMAX
+
+/* ternary operations */
+#define UNFUSED_FMA(prefix, a, b, c, flags)                          \
+{                                                                    \
+    a = prefix##_mul(a, b, &env->active_fpu.fp_status);              \
+    if ((flags) & float_muladd_negate_c) {                           \
+        a = prefix##_sub(a, c, &env->active_fpu.fp_status);          \
+    } else {                                                         \
+        a = prefix##_add(a, c, &env->active_fpu.fp_status);          \
+    }                                                                \
+    if ((flags) & float_muladd_negate_result) {                      \
+        a = prefix##_chs(a);                                         \
+    }                                                                \
+}
+
+/* FMA based operations */
+#define FLOAT_FMA(name, type)                                        \
+uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,              \
+                                     uint64_t fdt0, uint64_t fdt1,   \
+                                     uint64_t fdt2)                  \
+{                                                                    \
+    UNFUSED_FMA(float64, fdt0, fdt1, fdt2, type);                    \
+    update_fcr31(env, GETPC());                                      \
+    return fdt0;                                                     \
+}                                                                    \
+                                                                     \
+uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,              \
+                                     uint32_t fst0, uint32_t fst1,   \
+                                     uint32_t fst2)                  \
+{                                                                    \
+    UNFUSED_FMA(float32, fst0, fst1, fst2, type);                    \
+    update_fcr31(env, GETPC());                                      \
+    return fst0;                                                     \
+}                                                                    \
+                                                                     \
+uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,             \
+                                      uint64_t fdt0, uint64_t fdt1,  \
+                                      uint64_t fdt2)                 \
+{                                                                    \
+    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                               \
+    uint32_t fsth0 = fdt0 >> 32;                                     \
+    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                               \
+    uint32_t fsth1 = fdt1 >> 32;                                     \
+    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                               \
+    uint32_t fsth2 = fdt2 >> 32;                                     \
+                                                                     \
+    UNFUSED_FMA(float32, fst0, fst1, fst2, type);                    \
+    UNFUSED_FMA(float32, fsth0, fsth1, fsth2, type);                 \
+    update_fcr31(env, GETPC());                                      \
+    return ((uint64_t)fsth0 << 32) | fst0;                           \
+}
+FLOAT_FMA(madd, 0)
+FLOAT_FMA(msub, float_muladd_negate_c)
+FLOAT_FMA(nmadd, float_muladd_negate_result)
+FLOAT_FMA(nmsub, float_muladd_negate_result | float_muladd_negate_c)
+#undef FLOAT_FMA
+
+#define FLOAT_FMADDSUB(name, bits, muladd_arg)                          \
+uint ## bits ## _t helper_float_ ## name (CPUMIPSState *env,            \
+                                          uint ## bits ## _t fs,        \
+                                          uint ## bits ## _t ft,        \
+                                          uint ## bits ## _t fd)        \
+{                                                                       \
+    uint ## bits ## _t fdret;                                           \
+                                                                        \
+    fdret = float ## bits ## _muladd(fs, ft, fd, muladd_arg,            \
+                                     &env->active_fpu.fp_status);       \
+    update_fcr31(env, GETPC());                                         \
+    return fdret;                                                       \
+}
+
+FLOAT_FMADDSUB(maddf_s, 32, 0)
+FLOAT_FMADDSUB(maddf_d, 64, 0)
+FLOAT_FMADDSUB(msubf_s, 32, float_muladd_negate_product)
+FLOAT_FMADDSUB(msubf_d, 64, float_muladd_negate_product)
+#undef FLOAT_FMADDSUB
+
 /* compare operations */
 #define FOP_COND_D(op, cond)                                   \
 void helper_cmp_d_ ## op(CPUMIPSState *env, uint64_t fdt0,     \
@@ -3688,7 +4105,7 @@
 #if !defined(CONFIG_USER_ONLY)
 #define MEMOP_IDX(DF)                                           \
         TCGMemOpIdx oi = make_memop_idx(MO_TE | DF | MO_UNALN,  \
-                                        cpu_mmu_index(env));
+                                        cpu_mmu_index(env, false));
 #else
 #define MEMOP_IDX(DF)
 #endif
@@ -3744,7 +4161,7 @@
                             target_ulong addr)                          \
 {                                                                       \
     wr_t *pwd = &(env->active_fpu.fpr[wd].wr);                          \
-    int mmu_idx = cpu_mmu_index(env);                                   \
+    int mmu_idx = cpu_mmu_index(env, false);				\
     int i;                                                              \
     MEMOP_IDX(DF)                                                       \
     ensure_writable_pages(env, addr, mmu_idx, GETRA());                 \
@@ -3764,3 +4181,19 @@
 MSA_ST_DF(DF_WORD,   w, cpu_stl_data)
 MSA_ST_DF(DF_DOUBLE, d, cpu_stq_data)
 #endif
+
+void helper_cache(CPUMIPSState *env, target_ulong addr, uint32_t op)
+{
+#ifndef CONFIG_USER_ONLY
+    target_ulong index = addr & 0x1fffffff;
+    if (op == 9) {
+        /* Index Store Tag */
+        memory_region_dispatch_write(env->itc_tag, index, env->CP0_TagLo,
+                                     8, MEMTXATTRS_UNSPECIFIED);
+    } else if (op == 5) {
+        /* Index Load Tag */
+        memory_region_dispatch_read(env->itc_tag, index, &env->CP0_TagLo,
+                                    8, MEMTXATTRS_UNSPECIFIED);
+    }
+#endif
+}
diff --git a/target-mips/translate.c b/target-mips/translate.c
index 44b5fcc..eec9b2f 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -21,20 +21,22 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/cpu_ldst.h"
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 #include "sysemu/kvm.h"
+#include "exec/semihost.h"
 
 #include "trace-tcg.h"
-
+#include "exec/log.h"
 
 #define MIPS_DEBUG_DISAS 0
-//#define MIPS_DEBUG_SIGN_EXTENSIONS
 
 /* MIPS major opcodes */
 #define MASK_OP_MAJOR(op)  (op & (0x3F << 26))
@@ -324,6 +326,7 @@
     OPC_TLTIU    = (0x0B << 16) | OPC_REGIMM,
     OPC_TEQI     = (0x0C << 16) | OPC_REGIMM,
     OPC_TNEI     = (0x0E << 16) | OPC_REGIMM,
+    OPC_SIGRIE   = (0x17 << 16) | OPC_REGIMM,
     OPC_SYNCI    = (0x1F << 16) | OPC_REGIMM,
 
     OPC_DAHI     = (0x06 << 16) | OPC_REGIMM,
@@ -868,8 +871,10 @@
 enum {
     OPC_MFC0     = (0x00 << 21) | OPC_CP0,
     OPC_DMFC0    = (0x01 << 21) | OPC_CP0,
+    OPC_MFHC0    = (0x02 << 21) | OPC_CP0,
     OPC_MTC0     = (0x04 << 21) | OPC_CP0,
     OPC_DMTC0    = (0x05 << 21) | OPC_CP0,
+    OPC_MTHC0    = (0x06 << 21) | OPC_CP0,
     OPC_MFTR     = (0x08 << 21) | OPC_CP0,
     OPC_RDPGPR   = (0x0A << 21) | OPC_CP0,
     OPC_MFMC0    = (0x0B << 21) | OPC_CP0,
@@ -890,6 +895,8 @@
     OPC_EVPE     = 0x01 | (1 << 5) | OPC_MFMC0,
     OPC_DI       = (0 << 5) | (0x0C << 11) | OPC_MFMC0,
     OPC_EI       = (1 << 5) | (0x0C << 11) | OPC_MFMC0,
+    OPC_DVP      = 0x04 | (0 << 3) | (1 << 5) | (0 << 11) | OPC_MFMC0,
+    OPC_EVP      = 0x04 | (0 << 3) | (0 << 5) | (0 << 11) | OPC_MFMC0,
 };
 
 /* Coprocessor 0 (with rs == C0) */
@@ -2105,8 +2112,10 @@
 }
 #endif
 
+
+
 /* global register indices */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_gpr[32], cpu_PC;
 static TCGv cpu_HI[MIPS_DSP_ACC], cpu_LO[MIPS_DSP_ACC];
 static TCGv cpu_dspctrl, btarget, bcond;
@@ -2115,9 +2124,6 @@
 static TCGv_i64 fpu_f64[32];
 static TCGv_i64 msa_wr_d[64];
 
-static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 #define gen_helper_0e0i(name, arg) do {                           \
@@ -2181,6 +2187,15 @@
     int ie;
     bool bi;
     bool bp;
+    uint64_t PAMask;
+    bool mvh;
+    int CP0_LLAddr_shift;
+    bool ps;
+    bool vp;
+    bool cmgcr;
+    bool mrp;
+    bool nan2008;
+    bool abs2008;
 } DisasContext;
 
 enum {
@@ -2232,15 +2247,6 @@
     "w30.d0", "w30.d1", "w31.d0", "w31.d1",
 };
 
-#define MIPS_DEBUG(fmt, ...)                                                  \
-    do {                                                                      \
-        if (MIPS_DEBUG_DISAS) {                                               \
-            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
-                          TARGET_FMT_lx ": %08x " fmt "\n",                   \
-                          ctx->pc, ctx->opcode , ## __VA_ARGS__);             \
-        }                                                                     \
-    } while (0)
-
 #define LOG_DISAS(...)                                                        \
     do {                                                                      \
         if (MIPS_DEBUG_DISAS) {                                               \
@@ -2249,8 +2255,14 @@
     } while (0)
 
 #define MIPS_INVAL(op)                                                        \
-    MIPS_DEBUG("Invalid %s %03x %03x %03x", op, ctx->opcode >> 26,            \
-               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F))
+    do {                                                                      \
+        if (MIPS_DEBUG_DISAS) {                                               \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
+                          TARGET_FMT_lx ": %08x Invalid %s %03x %03x %03x\n", \
+                          ctx->pc, ctx->opcode, op, ctx->opcode >> 26,        \
+                          ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F));  \
+        }                                                                     \
+    } while (0)
 
 /* General purpose registers moves. */
 static inline void gen_load_gpr (TCGv t, int reg)
@@ -2315,7 +2327,68 @@
     }
 }
 
-static inline void generate_exception (DisasContext *ctx, int excp);
+/* Tests */
+static inline void gen_save_pc(target_ulong pc)
+{
+    tcg_gen_movi_tl(cpu_PC, pc);
+}
+
+static inline void save_cpu_state(DisasContext *ctx, int do_save_pc)
+{
+    LOG_DISAS("hflags %08x saved %08x\n", ctx->hflags, ctx->saved_hflags);
+    if (do_save_pc && ctx->pc != ctx->saved_pc) {
+        gen_save_pc(ctx->pc);
+        ctx->saved_pc = ctx->pc;
+    }
+    if (ctx->hflags != ctx->saved_hflags) {
+        tcg_gen_movi_i32(hflags, ctx->hflags);
+        ctx->saved_hflags = ctx->hflags;
+        switch (ctx->hflags & MIPS_HFLAG_BMASK_BASE) {
+        case MIPS_HFLAG_BR:
+            break;
+        case MIPS_HFLAG_BC:
+        case MIPS_HFLAG_BL:
+        case MIPS_HFLAG_B:
+            tcg_gen_movi_tl(btarget, ctx->btarget);
+            break;
+        }
+    }
+}
+
+static inline void restore_cpu_state(CPUMIPSState *env, DisasContext *ctx)
+{
+    ctx->saved_hflags = ctx->hflags;
+    switch (ctx->hflags & MIPS_HFLAG_BMASK_BASE) {
+    case MIPS_HFLAG_BR:
+        break;
+    case MIPS_HFLAG_BC:
+    case MIPS_HFLAG_BL:
+    case MIPS_HFLAG_B:
+        ctx->btarget = env->btarget;
+        break;
+    }
+}
+
+static inline void generate_exception_err(DisasContext *ctx, int excp, int err)
+{
+    TCGv_i32 texcp = tcg_const_i32(excp);
+    TCGv_i32 terr = tcg_const_i32(err);
+    save_cpu_state(ctx, 1);
+    gen_helper_raise_exception_err(cpu_env, texcp, terr);
+    tcg_temp_free_i32(terr);
+    tcg_temp_free_i32(texcp);
+    ctx->bstate = BS_EXCP;
+}
+
+static inline void generate_exception(DisasContext *ctx, int excp)
+{
+    gen_helper_0e0i(raise_exception, excp);
+}
+
+static inline void generate_exception_end(DisasContext *ctx, int excp)
+{
+    generate_exception_err(ctx, excp, 0);
+}
 
 /* Floating point register moves. */
 static void gen_load_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
@@ -2323,15 +2396,16 @@
     if (ctx->hflags & MIPS_HFLAG_FRE) {
         generate_exception(ctx, EXCP_RI);
     }
-    tcg_gen_trunc_i64_i32(t, fpu_f64[reg]);
+    tcg_gen_extrl_i64_i32(t, fpu_f64[reg]);
 }
 
 static void gen_store_fpr32(DisasContext *ctx, TCGv_i32 t, int reg)
 {
+    TCGv_i64 t64;
     if (ctx->hflags & MIPS_HFLAG_FRE) {
         generate_exception(ctx, EXCP_RI);
     }
-    TCGv_i64 t64 = tcg_temp_new_i64();
+    t64 = tcg_temp_new_i64();
     tcg_gen_extu_i32_i64(t64, t);
     tcg_gen_deposit_i64(fpu_f64[reg], fpu_f64[reg], t64, 0, 32);
     tcg_temp_free_i64(t64);
@@ -2340,10 +2414,7 @@
 static void gen_load_fpr32h(DisasContext *ctx, TCGv_i32 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        TCGv_i64 t64 = tcg_temp_new_i64();
-        tcg_gen_shri_i64(t64, fpu_f64[reg], 32);
-        tcg_gen_trunc_i64_i32(t, t64);
-        tcg_temp_free_i64(t64);
+        tcg_gen_extrh_i64_i32(t, fpu_f64[reg]);
     } else {
         gen_load_fpr32(ctx, t, reg | 1);
     }
@@ -2392,66 +2463,6 @@
         return 23;
 }
 
-/* Tests */
-static inline void gen_save_pc(target_ulong pc)
-{
-    tcg_gen_movi_tl(cpu_PC, pc);
-}
-
-static inline void save_cpu_state (DisasContext *ctx, int do_save_pc)
-{
-    LOG_DISAS("hflags %08x saved %08x\n", ctx->hflags, ctx->saved_hflags);
-    if (do_save_pc && ctx->pc != ctx->saved_pc) {
-        gen_save_pc(ctx->pc);
-        ctx->saved_pc = ctx->pc;
-    }
-    if (ctx->hflags != ctx->saved_hflags) {
-        tcg_gen_movi_i32(hflags, ctx->hflags);
-        ctx->saved_hflags = ctx->hflags;
-        switch (ctx->hflags & MIPS_HFLAG_BMASK_BASE) {
-        case MIPS_HFLAG_BR:
-            break;
-        case MIPS_HFLAG_BC:
-        case MIPS_HFLAG_BL:
-        case MIPS_HFLAG_B:
-            tcg_gen_movi_tl(btarget, ctx->btarget);
-            break;
-        }
-    }
-}
-
-static inline void restore_cpu_state (CPUMIPSState *env, DisasContext *ctx)
-{
-    ctx->saved_hflags = ctx->hflags;
-    switch (ctx->hflags & MIPS_HFLAG_BMASK_BASE) {
-    case MIPS_HFLAG_BR:
-        break;
-    case MIPS_HFLAG_BC:
-    case MIPS_HFLAG_BL:
-    case MIPS_HFLAG_B:
-        ctx->btarget = env->btarget;
-        break;
-    }
-}
-
-static inline void
-generate_exception_err (DisasContext *ctx, int excp, int err)
-{
-    TCGv_i32 texcp = tcg_const_i32(excp);
-    TCGv_i32 terr = tcg_const_i32(err);
-    save_cpu_state(ctx, 1);
-    gen_helper_raise_exception_err(cpu_env, texcp, terr);
-    tcg_temp_free_i32(terr);
-    tcg_temp_free_i32(texcp);
-}
-
-static inline void
-generate_exception (DisasContext *ctx, int excp)
-{
-    save_cpu_state(ctx, 1);
-    gen_helper_0e0i(raise_exception, excp);
-}
-
 /* Addresses computation */
 static inline void gen_op_addr_add (DisasContext *ctx, TCGv ret, TCGv arg0, TCGv arg1)
 {
@@ -2478,6 +2489,26 @@
     return sum;
 }
 
+/* Sign-extract the low 32-bits to a target_long.  */
+static inline void gen_move_low32(TCGv ret, TCGv_i64 arg)
+{
+#if defined(TARGET_MIPS64)
+    tcg_gen_ext32s_i64(ret, arg);
+#else
+    tcg_gen_extrl_i64_i32(ret, arg);
+#endif
+}
+
+/* Sign-extract the high 32-bits to a target_long.  */
+static inline void gen_move_high32(TCGv ret, TCGv_i64 arg)
+{
+#if defined(TARGET_MIPS64)
+    tcg_gen_sari_i64(ret, arg, 32);
+#else
+    tcg_gen_extrh_i64_i32(ret, arg);
+#endif
+}
+
 static inline void check_cp0_enabled(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_CP0)))
@@ -2497,7 +2528,7 @@
 static inline void check_cop1x(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_COP1X)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /* Verify that the processor is running with 64-bit floating-point
@@ -2506,7 +2537,7 @@
 static inline void check_cp1_64bitmode(DisasContext *ctx)
 {
     if (unlikely(~ctx->hflags & (MIPS_HFLAG_F64 | MIPS_HFLAG_COP1X)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /*
@@ -2523,7 +2554,7 @@
 static inline void check_cp1_registers(DisasContext *ctx, int regs)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_F64) && (regs & 1)))
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 }
 
 /* Verify that the processor is running with DSP instructions enabled.
@@ -2534,9 +2565,9 @@
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSP))) {
         if (ctx->insn_flags & ASE_DSP) {
-            generate_exception(ctx, EXCP_DSPDIS);
+            generate_exception_end(ctx, EXCP_DSPDIS);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
     }
 }
@@ -2545,9 +2576,9 @@
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSPR2))) {
         if (ctx->insn_flags & ASE_DSP) {
-            generate_exception(ctx, EXCP_DSPDIS);
+            generate_exception_end(ctx, EXCP_DSPDIS);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
     }
 }
@@ -2557,7 +2588,7 @@
 static inline void check_insn(DisasContext *ctx, int flags)
 {
     if (unlikely(!(ctx->insn_flags & flags))) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
 }
 
@@ -2567,8 +2598,18 @@
 static inline void check_insn_opc_removed(DisasContext *ctx, int flags)
 {
     if (unlikely(ctx->insn_flags & flags)) {
+        generate_exception_end(ctx, EXCP_RI);
+    }
+}
+
+/* This code generates a "reserved instruction" exception if the
+   CPU does not support 64-bit paired-single (PS) floating point data type */
+static inline void check_ps(DisasContext *ctx)
+{
+    if (unlikely(!ctx->ps)) {
         generate_exception(ctx, EXCP_RI);
     }
+    check_cp1_64bitmode(ctx);
 }
 
 #ifdef TARGET_MIPS64
@@ -2577,7 +2618,16 @@
 static inline void check_mips_64(DisasContext *ctx)
 {
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_64)))
+        generate_exception_end(ctx, EXCP_RI);
+}
+#endif
+
+#ifndef CONFIG_USER_ONLY
+static inline void check_mvh(DisasContext *ctx)
+{
+    if (unlikely(!ctx->mvh)) {
         generate_exception(ctx, EXCP_RI);
+    }
 }
 #endif
 
@@ -2595,7 +2645,7 @@
     TCGv_i##bits fp1 = tcg_temp_new_i##bits ();                               \
     switch (ifmt) {                                                           \
     case FMT_PS:                                                              \
-        check_cp1_64bitmode(ctx);                                             \
+        check_ps(ctx);                                                        \
         break;                                                                \
     case FMT_D:                                                               \
         if (abs) {                                                            \
@@ -2648,10 +2698,8 @@
 {                                                                       \
     TCGv_i ## bits fp0 = tcg_temp_new_i ## bits();                      \
     TCGv_i ## bits fp1 = tcg_temp_new_i ## bits();                      \
-    switch (ifmt) {                                                     \
-    case FMT_D:                                                         \
+    if (ifmt == FMT_D) {                                                \
         check_cp1_registers(ctx, fs | ft | fd);                         \
-        break;                                                          \
     }                                                                   \
     gen_ldcmp_fpr ## bits(ctx, fp0, fs);                                \
     gen_ldcmp_fpr ## bits(ctx, fp1, ft);                                \
@@ -2766,8 +2814,8 @@
 static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx) \
 {                                                                            \
     TCGv t0 = tcg_temp_new();                                                \
-    int l1 = gen_new_label();                                                \
-    int l2 = gen_new_label();                                                \
+    TCGLabel *l1 = gen_new_label();                                          \
+    TCGLabel *l2 = gen_new_label();                                          \
                                                                              \
     tcg_gen_andi_tl(t0, arg2, almask);                                       \
     tcg_gen_brcondi_tl(TCG_COND_EQ, t0, 0, l1);                              \
@@ -2779,7 +2827,7 @@
     tcg_gen_movi_tl(t0, rt | ((almask << 3) & 0x20));                        \
     tcg_gen_st_tl(t0, cpu_env, offsetof(CPUMIPSState, llreg));                   \
     tcg_gen_st_tl(arg1, cpu_env, offsetof(CPUMIPSState, llnewval));              \
-    gen_helper_0e0i(raise_exception, EXCP_SC);                               \
+    generate_exception_end(ctx, EXCP_SC);                                    \
     gen_set_label(l2);                                                       \
     tcg_gen_movi_tl(t0, 0);                                                  \
     gen_store_gpr(t0, rt);                                                   \
@@ -2832,14 +2880,12 @@
 static void gen_ld(DisasContext *ctx, uint32_t opc,
                    int rt, int base, int16_t offset)
 {
-    const char *opn = "ld";
     TCGv t0, t1, t2;
 
     if (rt == 0 && ctx->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
         /* Loongson CPU uses a load to zero register for prefetch.
            We emulate it as a NOP. On other CPU we must perform the
            actual memory access. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -2852,23 +2898,22 @@
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUL |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lwu";
         break;
     case OPC_LD:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "ld";
         break;
     case OPC_LLD:
     case R6_OPC_LLD:
-        save_cpu_state(ctx, 1);
         op_ld_lld(t0, t0, ctx);
         gen_store_gpr(t0, rt);
-        opn = "lld";
         break;
     case OPC_LDL:
         t1 = tcg_temp_new();
+        /* Do a byte access to possibly trigger a page
+           fault with the unaligned address.  */
+        tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_UB);
         tcg_gen_andi_tl(t1, t0, 7);
 #ifndef TARGET_WORDS_BIGENDIAN
         tcg_gen_xori_tl(t1, t1, 7);
@@ -2877,19 +2922,20 @@
         tcg_gen_andi_tl(t0, t0, ~7);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         tcg_gen_shl_tl(t0, t0, t1);
-        tcg_gen_xori_tl(t1, t1, 63);
-        t2 = tcg_const_tl(0x7fffffffffffffffull);
-        tcg_gen_shr_tl(t2, t2, t1);
+        t2 = tcg_const_tl(-1);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        tcg_gen_and_tl(t1, t1, t2);
+        tcg_gen_andc_tl(t1, t1, t2);
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         gen_store_gpr(t0, rt);
-        opn = "ldl";
         break;
     case OPC_LDR:
         t1 = tcg_temp_new();
+        /* Do a byte access to possibly trigger a page
+           fault with the unaligned address.  */
+        tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_UB);
         tcg_gen_andi_tl(t1, t0, 7);
 #ifdef TARGET_WORDS_BIGENDIAN
         tcg_gen_xori_tl(t1, t1, 7);
@@ -2907,7 +2953,6 @@
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         gen_store_gpr(t0, rt);
-        opn = "ldr";
         break;
     case OPC_LDPC:
         t1 = tcg_const_tl(pc_relative_pc(ctx));
@@ -2915,7 +2960,6 @@
         tcg_temp_free(t1);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t0, rt);
-        opn = "ldpc";
         break;
 #endif
     case OPC_LWPC:
@@ -2924,38 +2968,35 @@
         tcg_temp_free(t1);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t0, rt);
-        opn = "lwpc";
         break;
     case OPC_LW:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lw";
         break;
     case OPC_LH:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESW |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lh";
         break;
     case OPC_LHU:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUW |
                            ctx->default_tcg_memop_mask);
         gen_store_gpr(t0, rt);
-        opn = "lhu";
         break;
     case OPC_LB:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_SB);
         gen_store_gpr(t0, rt);
-        opn = "lb";
         break;
     case OPC_LBU:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_UB);
         gen_store_gpr(t0, rt);
-        opn = "lbu";
         break;
     case OPC_LWL:
         t1 = tcg_temp_new();
+        /* Do a byte access to possibly trigger a page
+           fault with the unaligned address.  */
+        tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_UB);
         tcg_gen_andi_tl(t1, t0, 3);
 #ifndef TARGET_WORDS_BIGENDIAN
         tcg_gen_xori_tl(t1, t1, 3);
@@ -2964,20 +3005,21 @@
         tcg_gen_andi_tl(t0, t0, ~3);
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEUL);
         tcg_gen_shl_tl(t0, t0, t1);
-        tcg_gen_xori_tl(t1, t1, 31);
-        t2 = tcg_const_tl(0x7fffffffull);
-        tcg_gen_shr_tl(t2, t2, t1);
+        t2 = tcg_const_tl(-1);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        tcg_gen_and_tl(t1, t1, t2);
+        tcg_gen_andc_tl(t1, t1, t2);
         tcg_temp_free(t2);
         tcg_gen_or_tl(t0, t0, t1);
         tcg_temp_free(t1);
         tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
-        opn = "lwl";
         break;
     case OPC_LWR:
         t1 = tcg_temp_new();
+        /* Do a byte access to possibly trigger a page
+           fault with the unaligned address.  */
+        tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_UB);
         tcg_gen_andi_tl(t1, t0, 3);
 #ifdef TARGET_WORDS_BIGENDIAN
         tcg_gen_xori_tl(t1, t1, 3);
@@ -2996,18 +3038,13 @@
         tcg_temp_free(t1);
         tcg_gen_ext32s_tl(t0, t0);
         gen_store_gpr(t0, rt);
-        opn = "lwr";
         break;
     case OPC_LL:
     case R6_OPC_LL:
-        save_cpu_state(ctx, 1);
         op_ld_ll(t0, t0, ctx);
         gen_store_gpr(t0, rt);
-        opn = "ll";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
 }
 
@@ -3015,7 +3052,6 @@
 static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
                     int base, int16_t offset)
 {
-    const char *opn = "st";
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
 
@@ -3026,46 +3062,32 @@
     case OPC_SD:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEQ |
                            ctx->default_tcg_memop_mask);
-        opn = "sd";
         break;
     case OPC_SDL:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(sdl, t1, t0, ctx->mem_idx);
-        opn = "sdl";
         break;
     case OPC_SDR:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(sdr, t1, t0, ctx->mem_idx);
-        opn = "sdr";
         break;
 #endif
     case OPC_SW:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL |
                            ctx->default_tcg_memop_mask);
-        opn = "sw";
         break;
     case OPC_SH:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUW |
                            ctx->default_tcg_memop_mask);
-        opn = "sh";
         break;
     case OPC_SB:
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_8);
-        opn = "sb";
         break;
     case OPC_SWL:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(swl, t1, t0, ctx->mem_idx);
-        opn = "swl";
         break;
     case OPC_SWR:
-        save_cpu_state(ctx, 1);
         gen_helper_0e2i(swr, t1, t0, ctx->mem_idx);
-        opn = "swr";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3075,7 +3097,6 @@
 static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
                          int base, int16_t offset)
 {
-    const char *opn = "st_cond";
     TCGv t0, t1;
 
 #ifdef CONFIG_USER_ONLY
@@ -3091,20 +3112,14 @@
 #if defined(TARGET_MIPS64)
     case OPC_SCD:
     case R6_OPC_SCD:
-        save_cpu_state(ctx, 1);
         op_st_scd(t1, t0, rt, ctx);
-        opn = "scd";
         break;
 #endif
     case OPC_SC:
     case R6_OPC_SC:
-        save_cpu_state(ctx, 1);
         op_st_sc(t1, t0, rt, ctx);
-        opn = "sc";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t1);
     tcg_temp_free(t0);
 }
@@ -3113,7 +3128,6 @@
 static void gen_flt_ldst (DisasContext *ctx, uint32_t opc, int ft,
                           int base, int16_t offset)
 {
-    const char *opn = "flt_ldst";
     TCGv t0 = tcg_temp_new();
 
     gen_base_offset_addr(ctx, t0, base, offset);
@@ -3128,7 +3142,6 @@
             gen_store_fpr32(ctx, fp0, ft);
             tcg_temp_free_i32(fp0);
         }
-        opn = "lwc1";
         break;
     case OPC_SWC1:
         {
@@ -3138,7 +3151,6 @@
                                 ctx->default_tcg_memop_mask);
             tcg_temp_free_i32(fp0);
         }
-        opn = "swc1";
         break;
     case OPC_LDC1:
         {
@@ -3148,7 +3160,6 @@
             gen_store_fpr64(ctx, fp0, ft);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ldc1";
         break;
     case OPC_SDC1:
         {
@@ -3158,15 +3169,12 @@
                                 ctx->default_tcg_memop_mask);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sdc1";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("flt_ldst");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %d(%s)", opn, fregnames[ft], offset, regnames[base]);
  out:
     tcg_temp_free(t0);
 }
@@ -3176,7 +3184,14 @@
 {
     if (ctx->CP0_Config1 & (1 << CP0C1_FP)) {
         check_cp1_enabled(ctx);
-        gen_flt_ldst(ctx, op, rt, rs, imm);
+        switch (op) {
+        case OPC_LDC1:
+        case OPC_SDC1:
+            check_insn(ctx, ISA_MIPS2);
+            /* Fallthrough */
+        default:
+            gen_flt_ldst(ctx, op, rt, rs, imm);
+        }
     } else {
         generate_exception_err(ctx, EXCP_CpU, 1);
     }
@@ -3187,12 +3202,10 @@
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
-    const char *opn = "imm arith";
 
     if (rt == 0 && opc != OPC_ADDI && opc != OPC_DADDI) {
         /* If no destination, treat it as a NOP.
            For addi, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
     switch (opc) {
@@ -3201,7 +3214,7 @@
             TCGv t0 = tcg_temp_local_new();
             TCGv t1 = tcg_temp_new();
             TCGv t2 = tcg_temp_new();
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             gen_load_gpr(t1, rs);
             tcg_gen_addi_tl(t0, t1, uimm);
@@ -3220,7 +3233,6 @@
             gen_store_gpr(t0, rt);
             tcg_temp_free(t0);
         }
-        opn = "addi";
         break;
     case OPC_ADDIU:
         if (rs != 0) {
@@ -3229,7 +3241,6 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
         }
-        opn = "addiu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DADDI:
@@ -3237,7 +3248,7 @@
             TCGv t0 = tcg_temp_local_new();
             TCGv t1 = tcg_temp_new();
             TCGv t2 = tcg_temp_new();
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             gen_load_gpr(t1, rs);
             tcg_gen_addi_tl(t0, t1, uimm);
@@ -3254,7 +3265,6 @@
             gen_store_gpr(t0, rt);
             tcg_temp_free(t0);
         }
-        opn = "daddi";
         break;
     case OPC_DADDIU:
         if (rs != 0) {
@@ -3262,12 +3272,9 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
         }
-        opn = "daddiu";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
 }
 
 /* Logic with immediate operand */
@@ -3278,7 +3285,6 @@
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     uimm = (uint16_t)imm;
@@ -3288,39 +3294,30 @@
             tcg_gen_andi_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], 0);
-        MIPS_DEBUG("andi %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_ORI:
         if (rs != 0)
             tcg_gen_ori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        MIPS_DEBUG("ori %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_XORI:
         if (likely(rs != 0))
             tcg_gen_xori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        MIPS_DEBUG("xori %s, %s, " TARGET_FMT_lx, regnames[rt],
-                   regnames[rs], uimm);
         break;
     case OPC_LUI:
         if (rs != 0 && (ctx->insn_flags & ISA_MIPS32R6)) {
             /* OPC_AUI */
             tcg_gen_addi_tl(cpu_gpr[rt], cpu_gpr[rs], imm << 16);
             tcg_gen_ext32s_tl(cpu_gpr[rt], cpu_gpr[rt]);
-            MIPS_DEBUG("aui %s, %s, %04x", regnames[rt], regnames[rs], imm);
         } else {
             tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
-            MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
         }
         break;
 
     default:
-        MIPS_DEBUG("Unknown logical immediate opcode %08x", opc);
         break;
     }
 }
@@ -3330,12 +3327,10 @@
                         int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
-    const char *opn = "imm arith";
     TCGv t0;
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     t0 = tcg_temp_new();
@@ -3343,15 +3338,11 @@
     switch (opc) {
     case OPC_SLTI:
         tcg_gen_setcondi_tl(TCG_COND_LT, cpu_gpr[rt], t0, uimm);
-        opn = "slti";
         break;
     case OPC_SLTIU:
         tcg_gen_setcondi_tl(TCG_COND_LTU, cpu_gpr[rt], t0, uimm);
-        opn = "sltiu";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
     tcg_temp_free(t0);
 }
 
@@ -3360,12 +3351,10 @@
                           int rt, int rs, int16_t imm)
 {
     target_ulong uimm = ((uint16_t)imm) & 0x1f;
-    const char *opn = "imm shift";
     TCGv t0;
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3375,11 +3364,9 @@
     case OPC_SLL:
         tcg_gen_shli_tl(t0, t0, uimm);
         tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
-        opn = "sll";
         break;
     case OPC_SRA:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm);
-        opn = "sra";
         break;
     case OPC_SRL:
         if (uimm != 0) {
@@ -3388,7 +3375,6 @@
         } else {
             tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
         }
-        opn = "srl";
         break;
     case OPC_ROTR:
         if (uimm != 0) {
@@ -3401,20 +3387,16 @@
         } else {
             tcg_gen_ext32s_tl(cpu_gpr[rt], t0);
         }
-        opn = "rotr";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DSLL:
         tcg_gen_shli_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsll";
         break;
     case OPC_DSRA:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsra";
         break;
     case OPC_DSRL:
         tcg_gen_shri_tl(cpu_gpr[rt], t0, uimm);
-        opn = "dsrl";
         break;
     case OPC_DROTR:
         if (uimm != 0) {
@@ -3422,28 +3404,21 @@
         } else {
             tcg_gen_mov_tl(cpu_gpr[rt], t0);
         }
-        opn = "drotr";
         break;
     case OPC_DSLL32:
         tcg_gen_shli_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsll32";
         break;
     case OPC_DSRA32:
         tcg_gen_sari_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsra32";
         break;
     case OPC_DSRL32:
         tcg_gen_shri_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "dsrl32";
         break;
     case OPC_DROTR32:
         tcg_gen_rotri_tl(cpu_gpr[rt], t0, uimm + 32);
-        opn = "drotr32";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
     tcg_temp_free(t0);
 }
 
@@ -3451,13 +3426,10 @@
 static void gen_arith(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "arith";
-
     if (rd == 0 && opc != OPC_ADD && opc != OPC_SUB
        && opc != OPC_DADD && opc != OPC_DSUB) {
         /* If no destination, treat it as a NOP.
            For add & sub, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3467,7 +3439,7 @@
             TCGv t0 = tcg_temp_local_new();
             TCGv t1 = tcg_temp_new();
             TCGv t2 = tcg_temp_new();
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             gen_load_gpr(t1, rs);
             gen_load_gpr(t2, rt);
@@ -3485,7 +3457,6 @@
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "add";
         break;
     case OPC_ADDU:
         if (rs != 0 && rt != 0) {
@@ -3498,14 +3469,13 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "addu";
         break;
     case OPC_SUB:
         {
             TCGv t0 = tcg_temp_local_new();
             TCGv t1 = tcg_temp_new();
             TCGv t2 = tcg_temp_new();
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             gen_load_gpr(t1, rs);
             gen_load_gpr(t2, rt);
@@ -3523,7 +3493,6 @@
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "sub";
         break;
     case OPC_SUBU:
         if (rs != 0 && rt != 0) {
@@ -3537,7 +3506,6 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "subu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DADD:
@@ -3545,7 +3513,7 @@
             TCGv t0 = tcg_temp_local_new();
             TCGv t1 = tcg_temp_new();
             TCGv t2 = tcg_temp_new();
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             gen_load_gpr(t1, rs);
             gen_load_gpr(t2, rt);
@@ -3562,7 +3530,6 @@
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "dadd";
         break;
     case OPC_DADDU:
         if (rs != 0 && rt != 0) {
@@ -3574,14 +3541,13 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "daddu";
         break;
     case OPC_DSUB:
         {
             TCGv t0 = tcg_temp_local_new();
             TCGv t1 = tcg_temp_new();
             TCGv t2 = tcg_temp_new();
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             gen_load_gpr(t1, rs);
             gen_load_gpr(t2, rt);
@@ -3598,7 +3564,6 @@
             gen_store_gpr(t0, rd);
             tcg_temp_free(t0);
         }
-        opn = "dsub";
         break;
     case OPC_DSUBU:
         if (rs != 0 && rt != 0) {
@@ -3610,7 +3575,6 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "dsubu";
         break;
 #endif
     case OPC_MUL:
@@ -3620,23 +3584,18 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "mul";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Conditional move */
 static void gen_cond_move(DisasContext *ctx, uint32_t opc,
                           int rd, int rs, int rt)
 {
-    const char *opn = "cond move";
     TCGv t0, t1, t2;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3648,38 +3607,28 @@
     switch (opc) {
     case OPC_MOVN:
         tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
-        opn = "movn";
         break;
     case OPC_MOVZ:
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
-        opn = "movz";
         break;
     case OPC_SELNEZ:
         tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, t1);
-        opn = "selnez";
         break;
     case OPC_SELEQZ:
         tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, t1);
-        opn = "seleqz";
         break;
     }
     tcg_temp_free(t2);
     tcg_temp_free(t1);
     tcg_temp_free(t0);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Logic */
 static void gen_logic(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "logic";
-
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3690,7 +3639,6 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "and";
         break;
     case OPC_NOR:
         if (rs != 0 && rt != 0) {
@@ -3702,7 +3650,6 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], ~((target_ulong)0));
         }
-        opn = "nor";
         break;
     case OPC_OR:
         if (likely(rs != 0 && rt != 0)) {
@@ -3714,7 +3661,6 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "or";
         break;
     case OPC_XOR:
         if (likely(rs != 0 && rt != 0)) {
@@ -3726,23 +3672,18 @@
         } else {
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
         }
-        opn = "xor";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Set on lower than */
 static void gen_slt(DisasContext *ctx, uint32_t opc,
                     int rd, int rs, int rt)
 {
-    const char *opn = "slt";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3753,15 +3694,11 @@
     switch (opc) {
     case OPC_SLT:
         tcg_gen_setcond_tl(TCG_COND_LT, cpu_gpr[rd], t0, t1);
-        opn = "slt";
         break;
     case OPC_SLTU:
         tcg_gen_setcond_tl(TCG_COND_LTU, cpu_gpr[rd], t0, t1);
-        opn = "sltu";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3770,13 +3707,11 @@
 static void gen_shift(DisasContext *ctx, uint32_t opc,
                       int rd, int rs, int rt)
 {
-    const char *opn = "shifts";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP.
            For add & sub, we must generate the overflow exception when needed. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3789,19 +3724,16 @@
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_shl_tl(t0, t1, t0);
         tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-        opn = "sllv";
         break;
     case OPC_SRAV:
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_sar_tl(cpu_gpr[rd], t1, t0);
-        opn = "srav";
         break;
     case OPC_SRLV:
         tcg_gen_ext32u_tl(t1, t1);
         tcg_gen_andi_tl(t0, t0, 0x1f);
         tcg_gen_shr_tl(t0, t1, t0);
         tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-        opn = "srlv";
         break;
     case OPC_ROTRV:
         {
@@ -3815,34 +3747,27 @@
             tcg_gen_ext_i32_tl(cpu_gpr[rd], t2);
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
-            opn = "rotrv";
         }
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DSLLV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_shl_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsllv";
         break;
     case OPC_DSRAV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_sar_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsrav";
         break;
     case OPC_DSRLV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_shr_tl(cpu_gpr[rd], t1, t0);
-        opn = "dsrlv";
         break;
     case OPC_DROTRV:
         tcg_gen_andi_tl(t0, t0, 0x3f);
         tcg_gen_rotr_tl(cpu_gpr[rd], t1, t0);
-        opn = "drotrv";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -3850,11 +3775,8 @@
 /* Arithmetic on HI/LO registers */
 static void gen_HILO(DisasContext *ctx, uint32_t opc, int acc, int reg)
 {
-    const char *opn = "hilo";
-
     if (reg == 0 && (opc == OPC_MFHI || opc == OPC_MFLO)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -3872,7 +3794,6 @@
         {
             tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[acc]);
         }
-        opn = "mfhi";
         break;
     case OPC_MFLO:
 #if defined(TARGET_MIPS64)
@@ -3883,7 +3804,6 @@
         {
             tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[acc]);
         }
-        opn = "mflo";
         break;
     case OPC_MTHI:
         if (reg != 0) {
@@ -3898,7 +3818,6 @@
         } else {
             tcg_gen_movi_tl(cpu_HI[acc], 0);
         }
-        opn = "mthi";
         break;
     case OPC_MTLO:
         if (reg != 0) {
@@ -3913,11 +3832,8 @@
         } else {
             tcg_gen_movi_tl(cpu_LO[acc], 0);
         }
-        opn = "mtlo";
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s", opn, regnames[reg]);
 }
 
 static inline void gen_r6_ld(target_long addr, int reg, int memidx,
@@ -3929,45 +3845,46 @@
     tcg_temp_free(t0);
 }
 
-static inline void gen_pcrel(DisasContext *ctx, int rs, int16_t imm)
+static inline void gen_pcrel(DisasContext *ctx, int opc, target_ulong pc,
+                             int rs)
 {
     target_long offset;
     target_long addr;
 
-    switch (MASK_OPC_PCREL_TOP2BITS(ctx->opcode)) {
+    switch (MASK_OPC_PCREL_TOP2BITS(opc)) {
     case OPC_ADDIUPC:
         if (rs != 0) {
             offset = sextract32(ctx->opcode << 2, 0, 21);
-            addr = addr_add(ctx, ctx->pc, offset);
+            addr = addr_add(ctx, pc, offset);
             tcg_gen_movi_tl(cpu_gpr[rs], addr);
         }
         break;
     case R6_OPC_LWPC:
         offset = sextract32(ctx->opcode << 2, 0, 21);
-        addr = addr_add(ctx, ctx->pc, offset);
+        addr = addr_add(ctx, pc, offset);
         gen_r6_ld(addr, rs, ctx->mem_idx, MO_TESL);
         break;
 #if defined(TARGET_MIPS64)
     case OPC_LWUPC:
         check_mips_64(ctx);
         offset = sextract32(ctx->opcode << 2, 0, 21);
-        addr = addr_add(ctx, ctx->pc, offset);
+        addr = addr_add(ctx, pc, offset);
         gen_r6_ld(addr, rs, ctx->mem_idx, MO_TEUL);
         break;
 #endif
     default:
-        switch (MASK_OPC_PCREL_TOP5BITS(ctx->opcode)) {
+        switch (MASK_OPC_PCREL_TOP5BITS(opc)) {
         case OPC_AUIPC:
             if (rs != 0) {
-                offset = imm << 16;
-                addr = addr_add(ctx, ctx->pc, offset);
+                offset = sextract32(ctx->opcode, 0, 16) << 16;
+                addr = addr_add(ctx, pc, offset);
                 tcg_gen_movi_tl(cpu_gpr[rs], addr);
             }
             break;
         case OPC_ALUIPC:
             if (rs != 0) {
-                offset = imm << 16;
-                addr = ~0xFFFF & addr_add(ctx, ctx->pc, offset);
+                offset = sextract32(ctx->opcode, 0, 16) << 16;
+                addr = ~0xFFFF & addr_add(ctx, pc, offset);
                 tcg_gen_movi_tl(cpu_gpr[rs], addr);
             }
             break;
@@ -3978,13 +3895,13 @@
         case R6_OPC_LDPC + (3 << 16):
             check_mips_64(ctx);
             offset = sextract32(ctx->opcode << 3, 0, 21);
-            addr = addr_add(ctx, (ctx->pc & ~0x7), offset);
+            addr = addr_add(ctx, (pc & ~0x7), offset);
             gen_r6_ld(addr, rs, ctx->mem_idx, MO_TEQ);
             break;
 #endif
         default:
             MIPS_INVAL("OPC_PCREL");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -3993,12 +3910,10 @@
 
 static void gen_r6_muldiv(DisasContext *ctx, int opc, int rd, int rs, int rt)
 {
-    const char *opn = "r6 mul/div";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -4027,7 +3942,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "div";
         break;
     case R6_OPC_MOD:
         {
@@ -4047,7 +3961,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "mod";
         break;
     case R6_OPC_DIVU:
         {
@@ -4061,7 +3974,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "divu";
         break;
     case R6_OPC_MODU:
         {
@@ -4075,7 +3987,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "modu";
         break;
     case R6_OPC_MUL:
         {
@@ -4088,7 +3999,6 @@
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mul";
         break;
     case R6_OPC_MUH:
         {
@@ -4101,7 +4011,6 @@
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "muh";
         break;
     case R6_OPC_MULU:
         {
@@ -4114,7 +4023,6 @@
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mulu";
         break;
     case R6_OPC_MUHU:
         {
@@ -4127,7 +4035,6 @@
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "muhu";
         break;
 #if defined(TARGET_MIPS64)
     case R6_OPC_DDIV:
@@ -4145,7 +4052,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddiv";
         break;
     case R6_OPC_DMOD:
         {
@@ -4162,7 +4068,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "dmod";
         break;
     case R6_OPC_DDIVU:
         {
@@ -4173,7 +4078,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddivu";
         break;
     case R6_OPC_DMODU:
         {
@@ -4184,11 +4088,9 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "dmodu";
         break;
     case R6_OPC_DMUL:
         tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
-        opn = "dmul";
         break;
     case R6_OPC_DMUH:
         {
@@ -4196,11 +4098,9 @@
             tcg_gen_muls2_i64(t2, cpu_gpr[rd], t0, t1);
             tcg_temp_free(t2);
         }
-        opn = "dmuh";
         break;
     case R6_OPC_DMULU:
         tcg_gen_mul_i64(cpu_gpr[rd], t0, t1);
-        opn = "dmulu";
         break;
     case R6_OPC_DMUHU:
         {
@@ -4208,16 +4108,13 @@
             tcg_gen_mulu2_i64(t2, cpu_gpr[rd], t0, t1);
             tcg_temp_free(t2);
         }
-        opn = "dmuhu";
         break;
 #endif
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("r6 mul/div");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
  out:
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -4226,7 +4123,6 @@
 static void gen_muldiv(DisasContext *ctx, uint32_t opc,
                        int acc, int rs, int rt)
 {
-    const char *opn = "mul/div";
     TCGv t0, t1;
 
     t0 = tcg_temp_new();
@@ -4260,7 +4156,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "div";
         break;
     case OPC_DIVU:
         {
@@ -4276,7 +4171,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "divu";
         break;
     case OPC_MULT:
         {
@@ -4290,7 +4184,6 @@
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "mult";
         break;
     case OPC_MULTU:
         {
@@ -4304,7 +4197,6 @@
             tcg_temp_free_i32(t2);
             tcg_temp_free_i32(t3);
         }
-        opn = "multu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DDIV:
@@ -4323,7 +4215,6 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddiv";
         break;
     case OPC_DDIVU:
         {
@@ -4335,15 +4226,12 @@
             tcg_temp_free(t3);
             tcg_temp_free(t2);
         }
-        opn = "ddivu";
         break;
     case OPC_DMULT:
         tcg_gen_muls2_i64(cpu_LO[acc], cpu_HI[acc], t0, t1);
-        opn = "dmult";
         break;
     case OPC_DMULTU:
         tcg_gen_mulu2_i64(cpu_LO[acc], cpu_HI[acc], t0, t1);
-        opn = "dmultu";
         break;
 #endif
     case OPC_MADD:
@@ -4357,14 +4245,10 @@
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "madd";
         break;
     case OPC_MADDU:
         {
@@ -4379,14 +4263,10 @@
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "maddu";
         break;
     case OPC_MSUB:
         {
@@ -4399,14 +4279,10 @@
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "msub";
         break;
     case OPC_MSUBU:
         {
@@ -4421,22 +4297,16 @@
             tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
-            tcg_gen_trunc_i64_tl(t0, t2);
-            tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_tl(t1, t2);
+            gen_move_low32(cpu_LO[acc], t2);
+            gen_move_high32(cpu_HI[acc], t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
-            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
-        opn = "msubu";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("mul/div");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rs], regnames[rt]);
  out:
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -4445,7 +4315,6 @@
 static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
                             int rd, int rs, int rt)
 {
-    const char *opn = "mul vr54xx";
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
 
@@ -4455,68 +4324,52 @@
     switch (opc) {
     case OPC_VR54XX_MULS:
         gen_helper_muls(t0, cpu_env, t0, t1);
-        opn = "muls";
         break;
     case OPC_VR54XX_MULSU:
         gen_helper_mulsu(t0, cpu_env, t0, t1);
-        opn = "mulsu";
         break;
     case OPC_VR54XX_MACC:
         gen_helper_macc(t0, cpu_env, t0, t1);
-        opn = "macc";
         break;
     case OPC_VR54XX_MACCU:
         gen_helper_maccu(t0, cpu_env, t0, t1);
-        opn = "maccu";
         break;
     case OPC_VR54XX_MSAC:
         gen_helper_msac(t0, cpu_env, t0, t1);
-        opn = "msac";
         break;
     case OPC_VR54XX_MSACU:
         gen_helper_msacu(t0, cpu_env, t0, t1);
-        opn = "msacu";
         break;
     case OPC_VR54XX_MULHI:
         gen_helper_mulhi(t0, cpu_env, t0, t1);
-        opn = "mulhi";
         break;
     case OPC_VR54XX_MULHIU:
         gen_helper_mulhiu(t0, cpu_env, t0, t1);
-        opn = "mulhiu";
         break;
     case OPC_VR54XX_MULSHI:
         gen_helper_mulshi(t0, cpu_env, t0, t1);
-        opn = "mulshi";
         break;
     case OPC_VR54XX_MULSHIU:
         gen_helper_mulshiu(t0, cpu_env, t0, t1);
-        opn = "mulshiu";
         break;
     case OPC_VR54XX_MACCHI:
         gen_helper_macchi(t0, cpu_env, t0, t1);
-        opn = "macchi";
         break;
     case OPC_VR54XX_MACCHIU:
         gen_helper_macchiu(t0, cpu_env, t0, t1);
-        opn = "macchiu";
         break;
     case OPC_VR54XX_MSACHI:
         gen_helper_msachi(t0, cpu_env, t0, t1);
-        opn = "msachi";
         break;
     case OPC_VR54XX_MSACHIU:
         gen_helper_msachiu(t0, cpu_env, t0, t1);
-        opn = "msachiu";
         break;
     default:
         MIPS_INVAL("mul vr54xx");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
     gen_store_gpr(t0, rd);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 
  out:
     tcg_temp_free(t0);
@@ -4526,12 +4379,10 @@
 static void gen_cl (DisasContext *ctx, uint32_t opc,
                     int rd, int rs)
 {
-    const char *opn = "CLx";
     TCGv t0;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
     t0 = tcg_temp_new();
@@ -4540,28 +4391,22 @@
     case OPC_CLO:
     case R6_OPC_CLO:
         gen_helper_clo(cpu_gpr[rd], t0);
-        opn = "clo";
         break;
     case OPC_CLZ:
     case R6_OPC_CLZ:
         gen_helper_clz(cpu_gpr[rd], t0);
-        opn = "clz";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DCLO:
     case R6_OPC_DCLO:
         gen_helper_dclo(cpu_gpr[rd], t0);
-        opn = "dclo";
         break;
     case OPC_DCLZ:
     case R6_OPC_DCLZ:
         gen_helper_dclz(cpu_gpr[rd], t0);
-        opn = "dclz";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s", opn, regnames[rd], regnames[rs]);
     tcg_temp_free(t0);
 }
 
@@ -4569,12 +4414,10 @@
 static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
                                  int rd, int rs, int rt)
 {
-    const char *opn = "loongson";
     TCGv t0, t1;
 
     if (rd == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -4606,7 +4449,6 @@
     case OPC_MULT_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
         tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
-        opn = "mult.g";
         break;
     case OPC_MULTU_G_2E:
     case OPC_MULTU_G_2F:
@@ -4614,14 +4456,13 @@
         tcg_gen_ext32u_tl(t1, t1);
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
         tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
-        opn = "multu.g";
         break;
     case OPC_DIV_G_2E:
     case OPC_DIV_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-            int l3 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
+            TCGLabel *l3 = gen_new_label();
             tcg_gen_ext32s_tl(t0, t0);
             tcg_gen_ext32s_tl(t1, t1);
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1);
@@ -4637,13 +4478,12 @@
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l3);
         }
-        opn = "div.g";
         break;
     case OPC_DIVU_G_2E:
     case OPC_DIVU_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1);
@@ -4654,14 +4494,13 @@
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l2);
         }
-        opn = "divu.g";
         break;
     case OPC_MOD_G_2E:
     case OPC_MOD_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-            int l3 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
+            TCGLabel *l3 = gen_new_label();
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
@@ -4675,13 +4514,12 @@
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l3);
         }
-        opn = "mod.g";
         break;
     case OPC_MODU_G_2E:
     case OPC_MODU_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1);
@@ -4692,25 +4530,22 @@
             tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
             gen_set_label(l2);
         }
-        opn = "modu.g";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMULT_G_2E:
     case OPC_DMULT_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
-        opn = "dmult.g";
         break;
     case OPC_DMULTU_G_2E:
     case OPC_DMULTU_G_2F:
         tcg_gen_mul_tl(cpu_gpr[rd], t0, t1);
-        opn = "dmultu.g";
         break;
     case OPC_DDIV_G_2E:
     case OPC_DDIV_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-            int l3 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
+            TCGLabel *l3 = gen_new_label();
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1);
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
             tcg_gen_br(l3);
@@ -4723,13 +4558,12 @@
             tcg_gen_div_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l3);
         }
-        opn = "ddiv.g";
         break;
     case OPC_DDIVU_G_2E:
     case OPC_DDIVU_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1);
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
             tcg_gen_br(l2);
@@ -4737,14 +4571,13 @@
             tcg_gen_divu_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l2);
         }
-        opn = "ddivu.g";
         break;
     case OPC_DMOD_G_2E:
     case OPC_DMOD_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-            int l3 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
+            TCGLabel *l3 = gen_new_label();
             tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
             tcg_gen_brcondi_tl(TCG_COND_NE, t0, -1LL << 63, l2);
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, -1LL, l2);
@@ -4755,13 +4588,12 @@
             tcg_gen_rem_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l3);
         }
-        opn = "dmod.g";
         break;
     case OPC_DMODU_G_2E:
     case OPC_DMODU_G_2F:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
             tcg_gen_brcondi_tl(TCG_COND_NE, t1, 0, l1);
             tcg_gen_movi_tl(cpu_gpr[rd], 0);
             tcg_gen_br(l2);
@@ -4769,13 +4601,10 @@
             tcg_gen_remu_tl(cpu_gpr[rd], t0, t1);
             gen_set_label(l2);
         }
-        opn = "dmodu.g";
         break;
 #endif
     }
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s", opn, regnames[rd], regnames[rs]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -4783,7 +4612,6 @@
 /* Loongson multimedia instructions */
 static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
 {
-    const char *opn = "loongson_cp2";
     uint32_t opc, shift_max;
     TCGv_i64 t0, t1;
 
@@ -4806,11 +4634,11 @@
     gen_load_fpr64(ctx, t1, rt);
 
 #define LMI_HELPER(UP, LO) \
-    case OPC_##UP: gen_helper_##LO(t0, t0, t1); opn = #LO; break
+    case OPC_##UP: gen_helper_##LO(t0, t0, t1); break
 #define LMI_HELPER_1(UP, LO) \
-    case OPC_##UP: gen_helper_##LO(t0, t0); opn = #LO; break
+    case OPC_##UP: gen_helper_##LO(t0, t0); break
 #define LMI_DIRECT(UP, LO, OP) \
-    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); opn = #LO; break
+    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); break
 
     switch (opc) {
     LMI_HELPER(PADDSH, paddsh);
@@ -4881,19 +4709,15 @@
 
     case OPC_PINSRH_0:
         tcg_gen_deposit_i64(t0, t0, t1, 0, 16);
-        opn = "pinsrh_0";
         break;
     case OPC_PINSRH_1:
         tcg_gen_deposit_i64(t0, t0, t1, 16, 16);
-        opn = "pinsrh_1";
         break;
     case OPC_PINSRH_2:
         tcg_gen_deposit_i64(t0, t0, t1, 32, 16);
-        opn = "pinsrh_2";
         break;
     case OPC_PINSRH_3:
         tcg_gen_deposit_i64(t0, t0, t1, 48, 16);
-        opn = "pinsrh_3";
         break;
 
     case OPC_PEXTRH:
@@ -4901,42 +4725,33 @@
         tcg_gen_shli_i64(t1, t1, 4);
         tcg_gen_shr_i64(t0, t0, t1);
         tcg_gen_ext16u_i64(t0, t0);
-        opn = "pextrh";
         break;
 
     case OPC_ADDU_CP2:
         tcg_gen_add_i64(t0, t0, t1);
         tcg_gen_ext32s_i64(t0, t0);
-        opn = "addu";
         break;
     case OPC_SUBU_CP2:
         tcg_gen_sub_i64(t0, t0, t1);
         tcg_gen_ext32s_i64(t0, t0);
-        opn = "addu";
         break;
 
     case OPC_SLL_CP2:
-        opn = "sll";
         shift_max = 32;
         goto do_shift;
     case OPC_SRL_CP2:
-        opn = "srl";
         shift_max = 32;
         goto do_shift;
     case OPC_SRA_CP2:
-        opn = "sra";
         shift_max = 32;
         goto do_shift;
     case OPC_DSLL_CP2:
-        opn = "dsll";
         shift_max = 64;
         goto do_shift;
     case OPC_DSRL_CP2:
-        opn = "dsrl";
         shift_max = 64;
         goto do_shift;
     case OPC_DSRA_CP2:
-        opn = "dsra";
         shift_max = 64;
         goto do_shift;
     do_shift:
@@ -4977,7 +4792,7 @@
     case OPC_DADD_CP2:
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
-            int lab = gen_new_label();
+            TCGLabel *lab = gen_new_label();
 
             tcg_gen_mov_i64(t2, t0);
             tcg_gen_add_i64(t0, t1, t2);
@@ -4991,8 +4806,6 @@
             tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
             generate_exception(ctx, EXCP_OVERFLOW);
             gen_set_label(lab);
-
-            opn = (opc == OPC_ADD_CP2 ? "add" : "dadd");
             break;
         }
 
@@ -5000,7 +4813,7 @@
     case OPC_DSUB_CP2:
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
-            int lab = gen_new_label();
+            TCGLabel *lab = gen_new_label();
 
             tcg_gen_mov_i64(t2, t0);
             tcg_gen_sub_i64(t0, t1, t2);
@@ -5014,8 +4827,6 @@
             tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
             generate_exception(ctx, EXCP_OVERFLOW);
             gen_set_label(lab);
-
-            opn = (opc == OPC_SUB_CP2 ? "sub" : "dsub");
             break;
         }
 
@@ -5023,7 +4834,6 @@
         tcg_gen_ext32u_i64(t0, t0);
         tcg_gen_ext32u_i64(t1, t1);
         tcg_gen_mul_i64(t0, t0, t1);
-        opn = "pmuluw";
         break;
 
     case OPC_SEQU_CP2:
@@ -5035,8 +4845,8 @@
         /* ??? Document is unclear: Set FCC[CC].  Does that mean the
            FD field is the CC field?  */
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("loongson_cp2");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -5045,9 +4855,6 @@
 
     gen_store_fpr64(ctx, t0, rd);
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s", opn,
-               fregnames[rd], fregnames[rs], fregnames[rt]);
     tcg_temp_free_i64(t0);
     tcg_temp_free_i64(t1);
 }
@@ -5099,7 +4906,7 @@
         case OPC_TGEU:  /* rs >= rs unsigned */
         case OPC_TGEIU: /* r0 >= 0  unsigned */
             /* Always trap */
-            generate_exception(ctx, EXCP_TRAP);
+            generate_exception_end(ctx, EXCP_TRAP);
             break;
         case OPC_TLT:   /* rs < rs           */
         case OPC_TLTI:  /* r0 < 0            */
@@ -5111,7 +4918,7 @@
             break;
         }
     } else {
-        int l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
 
         switch (opc) {
         case OPC_TEQ:
@@ -5146,20 +4953,30 @@
     tcg_temp_free(t1);
 }
 
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+    if (unlikely(ctx->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = ctx->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-        likely(!ctx->singlestep_enabled)) {
+    if (use_goto_tb(ctx, dest)) {
         tcg_gen_goto_tb(n);
         gen_save_pc(dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
     } else {
         gen_save_pc(dest);
         if (ctx->singlestep_enabled) {
             save_cpu_state(ctx, 0);
-            gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+            gen_helper_raise_exception_debug(cpu_env);
         }
         tcg_gen_exit_tb(0);
     }
@@ -5182,7 +4999,7 @@
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x"
                   TARGET_FMT_lx "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -5242,14 +5059,14 @@
             /* Hint = 0 is JR/JALR, hint 16 is JR.HB/JALR.HB, the
                others are reserved. */
             MIPS_INVAL("jump hint");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
         gen_load_gpr(btarget, rs);
         break;
     default:
         MIPS_INVAL("branch/jump");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
     if (bcond_compute == 0) {
@@ -5263,20 +5080,17 @@
         case OPC_BLEZL:   /* 0 <= 0 likely   */
             /* Always take */
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("balways");
             break;
         case OPC_BGEZAL:  /* 0 >= 0          */
         case OPC_BGEZALL: /* 0 >= 0 likely   */
             /* Always take and link */
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("balways and link");
             break;
         case OPC_BNE:     /* rx != rx        */
         case OPC_BGTZ:    /* 0 > 0           */
         case OPC_BLTZ:    /* 0 < 0           */
             /* Treat as NOP. */
-            MIPS_DEBUG("bnever (NOP)");
             goto out;
         case OPC_BLTZAL:  /* 0 < 0           */
             /* Handle as an unconditional branch to get correct delay
@@ -5284,24 +5098,20 @@
             blink = 31;
             btgt = ctx->pc + insn_bytes + delayslot_size;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("bnever and link");
             break;
         case OPC_BLTZALL: /* 0 < 0 likely */
             tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 8);
             /* Skip the instruction in the delay slot */
-            MIPS_DEBUG("bnever, link and skip");
             ctx->pc += 4;
             goto out;
         case OPC_BNEL:    /* rx != rx likely */
         case OPC_BGTZL:   /* 0 > 0 likely */
         case OPC_BLTZL:   /* 0 < 0 likely */
             /* Skip the instruction in the delay slot */
-            MIPS_DEBUG("bnever and skip");
             ctx->pc += 4;
             goto out;
         case OPC_J:
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("j " TARGET_FMT_lx, btgt);
             break;
         case OPC_JALX:
             ctx->hflags |= MIPS_HFLAG_BX;
@@ -5309,118 +5119,91 @@
         case OPC_JAL:
             blink = 31;
             ctx->hflags |= MIPS_HFLAG_B;
-            MIPS_DEBUG("jal " TARGET_FMT_lx, btgt);
             break;
         case OPC_JR:
             ctx->hflags |= MIPS_HFLAG_BR;
-            MIPS_DEBUG("jr %s", regnames[rs]);
             break;
         case OPC_JALR:
             blink = rt;
             ctx->hflags |= MIPS_HFLAG_BR;
-            MIPS_DEBUG("jalr %s, %s", regnames[rt], regnames[rs]);
             break;
         default:
             MIPS_INVAL("branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
     } else {
         switch (opc) {
         case OPC_BEQ:
             tcg_gen_setcond_tl(TCG_COND_EQ, bcond, t0, t1);
-            MIPS_DEBUG("beq %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto not_likely;
         case OPC_BEQL:
             tcg_gen_setcond_tl(TCG_COND_EQ, bcond, t0, t1);
-            MIPS_DEBUG("beql %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto likely;
         case OPC_BNE:
             tcg_gen_setcond_tl(TCG_COND_NE, bcond, t0, t1);
-            MIPS_DEBUG("bne %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto not_likely;
         case OPC_BNEL:
             tcg_gen_setcond_tl(TCG_COND_NE, bcond, t0, t1);
-            MIPS_DEBUG("bnel %s, %s, " TARGET_FMT_lx,
-                       regnames[rs], regnames[rt], btgt);
             goto likely;
         case OPC_BGEZ:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgez %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BGEZL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BGEZAL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
-            MIPS_DEBUG("bgezal %s, " TARGET_FMT_lx, regnames[rs], btgt);
             blink = 31;
             goto not_likely;
         case OPC_BGEZALL:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bgezall %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BGTZ:
             tcg_gen_setcondi_tl(TCG_COND_GT, bcond, t0, 0);
-            MIPS_DEBUG("bgtz %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BGTZL:
             tcg_gen_setcondi_tl(TCG_COND_GT, bcond, t0, 0);
-            MIPS_DEBUG("bgtzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BLEZ:
             tcg_gen_setcondi_tl(TCG_COND_LE, bcond, t0, 0);
-            MIPS_DEBUG("blez %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BLEZL:
             tcg_gen_setcondi_tl(TCG_COND_LE, bcond, t0, 0);
-            MIPS_DEBUG("blezl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BLTZ:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
-            MIPS_DEBUG("bltz %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto not_likely;
         case OPC_BLTZL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
-            MIPS_DEBUG("bltzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
         case OPC_BPOSGE32:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 32);
-            MIPS_DEBUG("bposge32 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #if defined(TARGET_MIPS64)
         case OPC_BPOSGE64:
             tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 64);
-            MIPS_DEBUG("bposge64 " TARGET_FMT_lx, btgt);
             goto not_likely;
 #endif
         case OPC_BLTZAL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bltzal %s, " TARGET_FMT_lx, regnames[rs], btgt);
         not_likely:
             ctx->hflags |= MIPS_HFLAG_BC;
             break;
         case OPC_BLTZALL:
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             blink = 31;
-            MIPS_DEBUG("bltzall %s, " TARGET_FMT_lx, regnames[rs], btgt);
         likely:
             ctx->hflags |= MIPS_HFLAG_BL;
             break;
         default:
             MIPS_INVAL("conditional branch/jump");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             goto out;
         }
     }
-    MIPS_DEBUG("enter ds: link %d cond %02x target " TARGET_FMT_lx,
-               blink, ctx->hflags, btgt);
 
     ctx->btarget = btgt;
 
@@ -5457,48 +5240,53 @@
     gen_load_gpr(t1, rs);
     switch (opc) {
     case OPC_EXT:
-        if (lsb + msb > 31)
+        if (lsb + msb > 31) {
             goto fail;
+        }
         tcg_gen_shri_tl(t0, t1, lsb);
         if (msb != 31) {
-            tcg_gen_andi_tl(t0, t0, (1 << (msb + 1)) - 1);
+            tcg_gen_andi_tl(t0, t0, (1U << (msb + 1)) - 1);
         } else {
             tcg_gen_ext32s_tl(t0, t0);
         }
         break;
 #if defined(TARGET_MIPS64)
-    case OPC_DEXTM:
-        tcg_gen_shri_tl(t0, t1, lsb);
-        if (msb != 31) {
-            tcg_gen_andi_tl(t0, t0, (1ULL << (msb + 1 + 32)) - 1);
-        }
-        break;
     case OPC_DEXTU:
-        tcg_gen_shri_tl(t0, t1, lsb + 32);
-        tcg_gen_andi_tl(t0, t0, (1ULL << (msb + 1)) - 1);
-        break;
+        lsb += 32;
+        goto do_dext;
+    case OPC_DEXTM:
+        msb += 32;
+        goto do_dext;
     case OPC_DEXT:
+    do_dext:
+        if (lsb + msb > 63) {
+            goto fail;
+        }
         tcg_gen_shri_tl(t0, t1, lsb);
-        tcg_gen_andi_tl(t0, t0, (1ULL << (msb + 1)) - 1);
+        if (msb != 63) {
+            tcg_gen_andi_tl(t0, t0, (1ULL << (msb + 1)) - 1);
+        }
         break;
 #endif
     case OPC_INS:
-        if (lsb > msb)
+        if (lsb > msb) {
             goto fail;
+        }
         gen_load_gpr(t0, rt);
         tcg_gen_deposit_tl(t0, t0, t1, lsb, msb - lsb + 1);
         tcg_gen_ext32s_tl(t0, t0);
         break;
 #if defined(TARGET_MIPS64)
-    case OPC_DINSM:
-        gen_load_gpr(t0, rt);
-        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb + 32 - lsb + 1);
-        break;
     case OPC_DINSU:
-        gen_load_gpr(t0, rt);
-        tcg_gen_deposit_tl(t0, t0, t1, lsb + 32, msb - lsb + 1);
-        break;
+        lsb += 32;
+        /* FALLTHRU */
+    case OPC_DINSM:
+        msb += 32;
+        /* FALLTHRU */
     case OPC_DINS:
+        if (lsb > msb) {
+            goto fail;
+        }
         gen_load_gpr(t0, rt);
         tcg_gen_deposit_tl(t0, t0, t1, lsb, msb - lsb + 1);
         break;
@@ -5506,7 +5294,7 @@
     default:
 fail:
         MIPS_INVAL("bitops");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         tcg_temp_free(t0);
         tcg_temp_free(t1);
         return;
@@ -5522,7 +5310,6 @@
 
     if (rd == 0) {
         /* If no destination, treat it as a NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -5579,15 +5366,164 @@
 #endif
     default:
         MIPS_INVAL("bsfhl");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         tcg_temp_free(t0);
         return;
     }
     tcg_temp_free(t0);
 }
 
+static void gen_lsa(DisasContext *ctx, int opc, int rd, int rs, int rt,
+                    int imm2)
+{
+    TCGv t0;
+    TCGv t1;
+    if (rd == 0) {
+        /* Treat as NOP. */
+        return;
+    }
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
+    gen_load_gpr(t0, rs);
+    gen_load_gpr(t1, rt);
+    tcg_gen_shli_tl(t0, t0, imm2 + 1);
+    tcg_gen_add_tl(cpu_gpr[rd], t0, t1);
+    if (opc == OPC_LSA) {
+        tcg_gen_ext32s_tl(cpu_gpr[rd], cpu_gpr[rd]);
+    }
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+
+    return;
+}
+
+static void gen_align(DisasContext *ctx, int opc, int rd, int rs, int rt,
+                      int bp)
+{
+    TCGv t0;
+    if (rd == 0) {
+        /* Treat as NOP. */
+        return;
+    }
+    t0 = tcg_temp_new();
+    gen_load_gpr(t0, rt);
+    if (bp == 0) {
+        switch (opc) {
+        case OPC_ALIGN:
+            tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
+            break;
+#if defined(TARGET_MIPS64)
+        case OPC_DALIGN:
+            tcg_gen_mov_tl(cpu_gpr[rd], t0);
+            break;
+#endif
+        }
+    } else {
+        TCGv t1 = tcg_temp_new();
+        gen_load_gpr(t1, rs);
+        switch (opc) {
+        case OPC_ALIGN:
+            {
+                TCGv_i64 t2 = tcg_temp_new_i64();
+                tcg_gen_concat_tl_i64(t2, t1, t0);
+                tcg_gen_shri_i64(t2, t2, 8 * (4 - bp));
+                gen_move_low32(cpu_gpr[rd], t2);
+                tcg_temp_free_i64(t2);
+            }
+            break;
+#if defined(TARGET_MIPS64)
+        case OPC_DALIGN:
+            tcg_gen_shli_tl(t0, t0, 8 * bp);
+            tcg_gen_shri_tl(t1, t1, 8 * (8 - bp));
+            tcg_gen_or_tl(cpu_gpr[rd], t1, t0);
+            break;
+#endif
+        }
+        tcg_temp_free(t1);
+    }
+
+    tcg_temp_free(t0);
+}
+
+static void gen_bitswap(DisasContext *ctx, int opc, int rd, int rt)
+{
+    TCGv t0;
+    if (rd == 0) {
+        /* Treat as NOP. */
+        return;
+    }
+    t0 = tcg_temp_new();
+    gen_load_gpr(t0, rt);
+    switch (opc) {
+    case OPC_BITSWAP:
+        gen_helper_bitswap(cpu_gpr[rd], t0);
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_DBITSWAP:
+        gen_helper_dbitswap(cpu_gpr[rd], t0);
+        break;
+#endif
+    }
+    tcg_temp_free(t0);
+}
+
 #ifndef CONFIG_USER_ONLY
 /* CP0 (MMU and control) */
+static inline void gen_mthc0_entrylo(TCGv arg, target_ulong off)
+{
+    TCGv_i64 t0 = tcg_temp_new_i64();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+
+    tcg_gen_ext_tl_i64(t0, arg);
+    tcg_gen_ld_i64(t1, cpu_env, off);
+#if defined(TARGET_MIPS64)
+    tcg_gen_deposit_i64(t1, t1, t0, 30, 32);
+#else
+    tcg_gen_concat32_i64(t1, t1, t0);
+#endif
+    tcg_gen_st_i64(t1, cpu_env, off);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void gen_mthc0_store64(TCGv arg, target_ulong off)
+{
+    TCGv_i64 t0 = tcg_temp_new_i64();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+
+    tcg_gen_ext_tl_i64(t0, arg);
+    tcg_gen_ld_i64(t1, cpu_env, off);
+    tcg_gen_concat32_i64(t1, t1, t0);
+    tcg_gen_st_i64(t1, cpu_env, off);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void gen_mfhc0_entrylo(TCGv arg, target_ulong off)
+{
+    TCGv_i64 t0 = tcg_temp_new_i64();
+
+    tcg_gen_ld_i64(t0, cpu_env, off);
+#if defined(TARGET_MIPS64)
+    tcg_gen_shri_i64(t0, t0, 30);
+#else
+    tcg_gen_shri_i64(t0, t0, 32);
+#endif
+    gen_move_low32(arg, t0);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void gen_mfhc0_load64(TCGv arg, target_ulong off, int shift)
+{
+    TCGv_i64 t0 = tcg_temp_new_i64();
+
+    tcg_gen_ld_i64(t0, cpu_env, off);
+    tcg_gen_shri_i64(t0, t0, 32 + shift);
+    gen_move_low32(arg, t0);
+    tcg_temp_free_i64(t0);
+}
+
 static inline void gen_mfc0_load32 (TCGv arg, target_ulong off)
 {
     TCGv_i32 t0 = tcg_temp_new_i32();
@@ -5612,10 +5548,151 @@
     tcg_temp_free_i32(t0);
 }
 
-static inline void gen_mtc0_store64 (TCGv arg, target_ulong off)
+#define CP0_CHECK(c)                            \
+    do {                                        \
+        if (!(c)) {                             \
+            goto cp0_unimplemented;             \
+        }                                       \
+    } while (0)
+
+static void gen_mfhc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
-    tcg_gen_ext32s_tl(arg, arg);
-    tcg_gen_st_tl(arg, cpu_env, off);
+    const char *rn = "invalid";
+
+    CP0_CHECK(ctx->hflags & MIPS_HFLAG_ELPA);
+
+    switch (reg) {
+    case 2:
+        switch (sel) {
+        case 0:
+            gen_mfhc0_entrylo(arg, offsetof(CPUMIPSState, CP0_EntryLo0));
+            rn = "EntryLo0";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    case 3:
+        switch (sel) {
+        case 0:
+            gen_mfhc0_entrylo(arg, offsetof(CPUMIPSState, CP0_EntryLo1));
+            rn = "EntryLo1";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    case 17:
+        switch (sel) {
+        case 0:
+            gen_mfhc0_load64(arg, offsetof(CPUMIPSState, lladdr),
+                             ctx->CP0_LLAddr_shift);
+            rn = "LLAddr";
+            break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mfhc0_maar(arg, cpu_env);
+            rn = "MAAR";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    case 28:
+        switch (sel) {
+        case 0:
+        case 2:
+        case 4:
+        case 6:
+            gen_mfhc0_load64(arg, offsetof(CPUMIPSState, CP0_TagLo), 0);
+            rn = "TagLo";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    default:
+        goto cp0_unimplemented;
+    }
+
+    (void)rn; /* avoid a compiler warning */
+    LOG_DISAS("mfhc0 %s (reg %d sel %d)\n", rn, reg, sel);
+    return;
+
+cp0_unimplemented:
+    LOG_DISAS("mfhc0 %s (reg %d sel %d)\n", rn, reg, sel);
+    tcg_gen_movi_tl(arg, 0);
+}
+
+static void gen_mthc0(DisasContext *ctx, TCGv arg, int reg, int sel)
+{
+    const char *rn = "invalid";
+    uint64_t mask = ctx->PAMask >> 36;
+
+    CP0_CHECK(ctx->hflags & MIPS_HFLAG_ELPA);
+
+    switch (reg) {
+    case 2:
+        switch (sel) {
+        case 0:
+            tcg_gen_andi_tl(arg, arg, mask);
+            gen_mthc0_entrylo(arg, offsetof(CPUMIPSState, CP0_EntryLo0));
+            rn = "EntryLo0";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    case 3:
+        switch (sel) {
+        case 0:
+            tcg_gen_andi_tl(arg, arg, mask);
+            gen_mthc0_entrylo(arg, offsetof(CPUMIPSState, CP0_EntryLo1));
+            rn = "EntryLo1";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    case 17:
+        switch (sel) {
+        case 0:
+            /* LLAddr is read-only (the only exception is bit 0 if LLB is
+               supported); the CP0_LLAddr_rw_bitmask does not seem to be
+               relevant for modern MIPS cores supporting MTHC0, therefore
+               treating MTHC0 to LLAddr as NOP. */
+            rn = "LLAddr";
+            break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mthc0_maar(cpu_env, arg);
+            rn = "MAAR";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    case 28:
+        switch (sel) {
+        case 0:
+        case 2:
+        case 4:
+        case 6:
+            tcg_gen_andi_tl(arg, arg, mask);
+            gen_mthc0_store64(arg, offsetof(CPUMIPSState, CP0_TagLo));
+            rn = "TagLo";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
+        break;
+    default:
+        goto cp0_unimplemented;
+    }
+
+    (void)rn; /* avoid a compiler warning */
+cp0_unimplemented:
+    LOG_DISAS("mthc0 %s (reg %d sel %d)\n", rn, reg, sel);
 }
 
 static inline void gen_mfc0_unimplemented(DisasContext *ctx, TCGv arg)
@@ -5627,13 +5704,6 @@
     }
 }
 
-#define CP0_CHECK(c)                            \
-    do {                                        \
-        if (!(c)) {                             \
-            goto cp0_unimplemented;             \
-        }                                       \
-    } while (0)
-
 static void gen_mfc0(DisasContext *ctx, TCGv arg, int reg, int sel)
 {
     const char *rn = "invalid";
@@ -5663,6 +5733,11 @@
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPControl));
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -5716,17 +5791,20 @@
     case 2:
         switch (sel) {
         case 0:
-            tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EntryLo0));
+            {
+                TCGv_i64 tmp = tcg_temp_new_i64();
+                tcg_gen_ld_i64(tmp, cpu_env,
+                               offsetof(CPUMIPSState, CP0_EntryLo0));
 #if defined(TARGET_MIPS64)
-            if (ctx->rxi) {
-                TCGv tmp = tcg_temp_new();
-                tcg_gen_andi_tl(tmp, arg, (3ull << 62));
-                tcg_gen_shri_tl(tmp, tmp, 32);
-                tcg_gen_or_tl(arg, arg, tmp);
-                tcg_temp_free(tmp);
-            }
+                if (ctx->rxi) {
+                    /* Move RI/XI fields to bits 31:30 */
+                    tcg_gen_shri_tl(arg, tmp, CP0EnLo_XI);
+                    tcg_gen_deposit_tl(tmp, tmp, arg, 30, 2);
+                }
 #endif
-            tcg_gen_ext32s_tl(arg, arg);
+                gen_move_low32(arg, tmp);
+                tcg_temp_free_i64(tmp);
+            }
             rn = "EntryLo0";
             break;
         case 1:
@@ -5771,19 +5849,27 @@
     case 3:
         switch (sel) {
         case 0:
-            tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EntryLo1));
+            {
+                TCGv_i64 tmp = tcg_temp_new_i64();
+                tcg_gen_ld_i64(tmp, cpu_env,
+                               offsetof(CPUMIPSState, CP0_EntryLo1));
 #if defined(TARGET_MIPS64)
-            if (ctx->rxi) {
-                TCGv tmp = tcg_temp_new();
-                tcg_gen_andi_tl(tmp, arg, (3ull << 62));
-                tcg_gen_shri_tl(tmp, tmp, 32);
-                tcg_gen_or_tl(arg, arg, tmp);
-                tcg_temp_free(tmp);
-            }
+                if (ctx->rxi) {
+                    /* Move RI/XI fields to bits 31:30 */
+                    tcg_gen_shri_tl(arg, tmp, CP0EnLo_XI);
+                    tcg_gen_deposit_tl(tmp, tmp, arg, 30, 2);
+                }
 #endif
-            tcg_gen_ext32s_tl(arg, arg);
+                gen_move_low32(arg, tmp);
+                tcg_temp_free_i64(tmp);
+            }
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_GlobalNumber));
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -5896,10 +5982,11 @@
         switch (sel) {
         case 0:
             /* Mark as an IO operation because we read the time.  */
-            if (use_icount)
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_start();
+	    }
             gen_helper_mfc0_count(arg, cpu_env);
-            if (use_icount) {
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_end();
             }
             /* Break the TB to be able to take timer interrupts immediately
@@ -5991,6 +6078,13 @@
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_EBase));
             rn = "EBase";
             break;
+        case 3:
+            check_insn(ctx, ISA_MIPS32R2);
+            CP0_CHECK(ctx->cmgcr);
+            tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_CMGCRBase));
+            tcg_gen_ext32s_tl(arg, arg);
+            rn = "CMGCRBase";
+            break;
         default:
             goto cp0_unimplemented;
        }
@@ -6040,6 +6134,16 @@
             gen_helper_mfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mfc0_maar(arg, cpu_env);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_MAARI));
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6171,8 +6275,14 @@
         }
         break;
     case 26:
-        tcg_gen_movi_tl(arg, 0); /* unimplemented */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_ErrCtl));
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -6190,7 +6300,12 @@
         case 2:
         case 4:
         case 6:
-            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_TagLo));
+            {
+                TCGv_i64 tmp = tcg_temp_new_i64();
+                tcg_gen_ld_i64(tmp, cpu_env, offsetof(CPUMIPSState, CP0_TagLo));
+                gen_move_low32(arg, tmp);
+                tcg_temp_free_i64(tmp);
+            }
             rn = "TagLo";
             break;
         case 1:
@@ -6272,8 +6387,9 @@
     if (sel != 0)
         check_insn(ctx, ISA_MIPS32);
 
-    if (use_icount)
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
+    }
 
     switch (reg) {
     case 0:
@@ -6297,6 +6413,11 @@
             /* ignored */
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6329,12 +6450,14 @@
             break;
         case 5:
             CP0_CHECK(ctx->insn_flags & ASE_MT);
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPESchedule));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_VPESchedule));
             rn = "VPESchedule";
             break;
         case 6:
             CP0_CHECK(ctx->insn_flags & ASE_MT);
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_VPEScheFBack));
+            tcg_gen_st_tl(arg, cpu_env,
+                          offsetof(CPUMIPSState, CP0_VPEScheFBack));
             rn = "VPEScheFBack";
             break;
         case 7:
@@ -6397,6 +6520,11 @@
             gen_helper_mtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6432,6 +6560,7 @@
             check_insn(ctx, ISA_MIPS32R2);
             gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
+            ctx->bstate = BS_STOP;
             break;
         default:
             goto cp0_unimplemented;
@@ -6583,7 +6712,7 @@
     case 14:
         switch (sel) {
         case 0:
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_EPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EPC));
             rn = "EPC";
             break;
         default:
@@ -6624,8 +6753,10 @@
             ctx->bstate = BS_STOP;
             break;
         case 3:
-            /* ignored, read only */
+            gen_helper_mtc0_config3(cpu_env, arg);
             rn = "Config3";
+            /* Stop translation as we may have switched the execution mode */
+            ctx->bstate = BS_STOP;
             break;
         case 4:
             gen_helper_mtc0_config4(cpu_env, arg);
@@ -6658,6 +6789,16 @@
             gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maar(cpu_env, arg);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maari(cpu_env, arg);
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -6754,7 +6895,7 @@
         switch (sel) {
         case 0:
             /* EJTAG support */
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_DEPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_DEPC));
             rn = "DEPC";
             break;
         default:
@@ -6800,8 +6941,15 @@
         }
        break;
     case 26:
-        /* ignored */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_helper_mtc0_errctl(cpu_env, arg);
+            ctx->bstate = BS_STOP;
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -6857,7 +7005,7 @@
     case 30:
         switch (sel) {
         case 0:
-            gen_mtc0_store64(arg, offsetof(CPUMIPSState, CP0_ErrorEPC));
+            tcg_gen_st_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_ErrorEPC));
             rn = "ErrorEPC";
             break;
         default:
@@ -6889,7 +7037,7 @@
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("mtc0 %s (reg %d sel %d)\n", rn, reg, sel);
     /* For simplicity assume that all writes can cause interrupts.  */
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
         ctx->bstate = BS_STOP;
     }
@@ -6929,6 +7077,11 @@
             gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_VPControl));
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7030,6 +7183,11 @@
             tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_EntryLo1));
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_GlobalNumber));
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7140,10 +7298,11 @@
         switch (sel) {
         case 0:
             /* Mark as an IO operation because we read the time.  */
-            if (use_icount)
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_start();
+            }
             gen_helper_mfc0_count(arg, cpu_env);
-            if (use_icount) {
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_end();
             }
             /* Break the TB to be able to take timer interrupts immediately
@@ -7233,6 +7392,12 @@
             gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_EBase));
             rn = "EBase";
             break;
+        case 3:
+            check_insn(ctx, ISA_MIPS32R2);
+            CP0_CHECK(ctx->cmgcr);
+            tcg_gen_ld_tl(arg, cpu_env, offsetof(CPUMIPSState, CP0_CMGCRBase));
+            rn = "CMGCRBase";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7282,6 +7447,16 @@
             gen_helper_dmfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_dmfc0_maar(arg, cpu_env);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_MAARI));
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7409,8 +7584,14 @@
         }
         break;
     case 26:
-        tcg_gen_movi_tl(arg, 0); /* unimplemented */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_mfc0_load32(arg, offsetof(CPUMIPSState, CP0_ErrCtl));
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -7509,8 +7690,9 @@
     if (sel != 0)
         check_insn(ctx, ISA_MIPS64);
 
-    if (use_icount)
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
+    }
 
     switch (reg) {
     case 0:
@@ -7534,6 +7716,11 @@
             /* ignored */
             rn = "MVPConf1";
             break;
+        case 4:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "VPControl";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7634,6 +7821,11 @@
             gen_helper_dmtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
+        case 1:
+            CP0_CHECK(ctx->vp);
+            /* ignored */
+            rn = "GlobalNumber";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -7816,11 +8008,11 @@
             save_cpu_state(ctx, 1);
             /* Mark as an IO operation because we may trigger a software
                interrupt.  */
-            if (use_icount) {
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_start();
             }
             gen_helper_mtc0_cause(cpu_env, arg);
-            if (use_icount) {
+            if (ctx->tb->cflags & CF_USE_ICOUNT) {
                 gen_io_end();
             }
             /* Stop translation as we may have triggered an intetrupt */
@@ -7875,8 +8067,10 @@
             ctx->bstate = BS_STOP;
             break;
         case 3:
-            /* ignored */
+            gen_helper_mtc0_config3(cpu_env, arg);
             rn = "Config3";
+            /* Stop translation as we may have switched the execution mode */
+            ctx->bstate = BS_STOP;
             break;
         case 4:
             /* currently ignored */
@@ -7900,6 +8094,16 @@
             gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
+        case 1:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maar(cpu_env, arg);
+            rn = "MAAR";
+            break;
+        case 2:
+            CP0_CHECK(ctx->mrp);
+            gen_helper_mtc0_maari(cpu_env, arg);
+            rn = "MAARI";
+            break;
         default:
             goto cp0_unimplemented;
         }
@@ -8038,8 +8242,15 @@
         }
         break;
     case 26:
-        /* ignored */
-        rn = "ECC";
+        switch (sel) {
+        case 0:
+            gen_helper_mtc0_errctl(cpu_env, arg);
+            ctx->bstate = BS_STOP;
+            rn = "ErrCtl";
+            break;
+        default:
+            goto cp0_unimplemented;
+        }
         break;
     case 27:
         switch (sel) {
@@ -8127,7 +8338,7 @@
     (void)rn; /* avoid a compiler warning */
     LOG_DISAS("dmtc0 %s (reg %d sel %d)\n", rn, reg, sel);
     /* For simplicity assume that all writes can cause interrupts.  */
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
         ctx->bstate = BS_STOP;
     }
@@ -8352,7 +8563,7 @@
 die:
     tcg_temp_free(t0);
     LOG_DISAS("mftr (reg %d u %d sel %d h %d)\n", rt, u, sel, h);
-    generate_exception(ctx, EXCP_RI);
+    generate_exception_end(ctx, EXCP_RI);
 }
 
 static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
@@ -8533,7 +8744,6 @@
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        save_cpu_state(ctx, 1);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(rd);
 
@@ -8557,7 +8767,7 @@
 die:
     tcg_temp_free(t0);
     LOG_DISAS("mttr (reg %d u %d sel %d h %d)\n", rd, u, sel, h);
-    generate_exception(ctx, EXCP_RI);
+    generate_exception_end(ctx, EXCP_RI);
 }
 
 static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt, int rd)
@@ -8606,6 +8816,25 @@
         opn = "dmtc0";
         break;
 #endif
+    case OPC_MFHC0:
+        check_mvh(ctx);
+        if (rt == 0) {
+            /* Treat as NOP. */
+            return;
+        }
+        gen_mfhc0(ctx, cpu_gpr[rt], rd, ctx->opcode & 0x7);
+        opn = "mfhc0";
+        break;
+    case OPC_MTHC0:
+        check_mvh(ctx);
+        {
+            TCGv t0 = tcg_temp_new();
+            gen_load_gpr(t0, rt);
+            gen_mthc0(ctx, t0, rd, ctx->opcode & 0x7);
+            tcg_temp_free(t0);
+        }
+        opn = "mthc0";
+        break;
     case OPC_MFTR:
         check_insn(ctx, ASE_MT);
         if (rd == 0) {
@@ -8667,33 +8896,33 @@
     case OPC_ERET: /* OPC_ERETNC */
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
-        }
-        if (ctx->opcode & (1 << 6)) {
-            /* OPC_ERETNC */
-            opn = "eretnc";
-            check_insn(ctx, ISA_MIPS32R5);
-            gen_helper_eretnc(cpu_env);
         } else {
-            /* OPC_ERET */
-            opn = "eret";
-            check_insn(ctx, ISA_MIPS2);
-            gen_helper_eret(cpu_env);
+            int bit_shift = (ctx->hflags & MIPS_HFLAG_M16) ? 16 : 6;
+            if (ctx->opcode & (1 << bit_shift)) {
+                /* OPC_ERETNC */
+                opn = "eretnc";
+                check_insn(ctx, ISA_MIPS32R5);
+                gen_helper_eretnc(cpu_env);
+            } else {
+                /* OPC_ERET */
+                opn = "eret";
+                check_insn(ctx, ISA_MIPS2);
+                gen_helper_eret(cpu_env);
+            }
+            ctx->bstate = BS_EXCP;
         }
-        ctx->bstate = BS_EXCP;
         break;
     case OPC_DERET:
         opn = "deret";
         check_insn(ctx, ISA_MIPS32);
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         }
         if (!(ctx->hflags & MIPS_HFLAG_DM)) {
             MIPS_INVAL(opn);
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             gen_helper_deret(cpu_env);
             ctx->bstate = BS_EXCP;
@@ -8704,7 +8933,6 @@
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
         if ((ctx->insn_flags & ISA_MIPS32R6) &&
             (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            MIPS_DEBUG("CTI in delay / forbidden slot");
             goto die;
         }
         /* If we get an exception, we want to restart at next instruction */
@@ -8717,11 +8945,10 @@
     default:
  die:
         MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
     (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %d", opn, regnames[rt], rd);
 }
 #endif /* !CONFIG_USER_ONLY */
 
@@ -8730,12 +8957,10 @@
                                 int32_t cc, int32_t offset)
 {
     target_ulong btarget;
-    const char *opn = "cp1 cond branch";
     TCGv_i32 t0 = tcg_temp_new_i32();
 
     if ((ctx->insn_flags & ISA_MIPS32R6) && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-        MIPS_DEBUG("CTI in delay / forbidden slot");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -8750,26 +8975,22 @@
         tcg_gen_not_i32(t0, t0);
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1f";
         goto not_likely;
     case OPC_BC1FL:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_not_i32(t0, t0);
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1fl";
         goto likely;
     case OPC_BC1T:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1t";
         goto not_likely;
     case OPC_BC1TL:
         tcg_gen_shri_i32(t0, fpu_fcr31, get_fp_bit(cc));
         tcg_gen_andi_i32(t0, t0, 1);
         tcg_gen_extu_i32_tl(bcond, t0);
-        opn = "bc1tl";
     likely:
         ctx->hflags |= MIPS_HFLAG_BL;
         break;
@@ -8783,7 +9004,6 @@
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any2f";
         goto not_likely;
     case OPC_BC1TANY2:
         {
@@ -8795,7 +9015,6 @@
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any2t";
         goto not_likely;
     case OPC_BC1FANY4:
         {
@@ -8811,7 +9030,6 @@
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any4f";
         goto not_likely;
     case OPC_BC1TANY4:
         {
@@ -8827,18 +9045,14 @@
             tcg_gen_andi_i32(t0, t0, 1);
             tcg_gen_extu_i32_tl(bcond, t0);
         }
-        opn = "bc1any4t";
     not_likely:
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("cp1 cond branch");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
-               ctx->hflags, btarget);
     ctx->btarget = btarget;
     ctx->hflags |= MIPS_HFLAG_BDS32;
  out:
@@ -8847,10 +9061,10 @@
 
 /* R6 CP1 Branches */
 static void gen_compute_branch1_r6(DisasContext *ctx, uint32_t op,
-                                   int32_t ft, int32_t offset)
+                                   int32_t ft, int32_t offset,
+                                   int delayslot_size)
 {
     target_ulong btarget;
-    const char *opn = "cp1 cond branch";
     TCGv_i64 t0 = tcg_temp_new_i64();
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
@@ -8858,7 +9072,7 @@
         LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
                   "\n", ctx->pc);
 #endif
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
@@ -8870,27 +9084,30 @@
     switch (op) {
     case OPC_BC1EQZ:
         tcg_gen_xori_i64(t0, t0, 1);
-        opn = "bc1eqz";
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     case OPC_BC1NEZ:
         /* t0 already set */
-        opn = "bc1nez";
         ctx->hflags |= MIPS_HFLAG_BC;
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception(ctx, EXCP_RI);
+        MIPS_INVAL("cp1 cond branch");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
 
     tcg_gen_trunc_i64_tl(bcond, t0);
 
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s: cond %02x target " TARGET_FMT_lx, opn,
-               ctx->hflags, btarget);
     ctx->btarget = btarget;
-    ctx->hflags |= MIPS_HFLAG_BDS32;
+
+    switch (delayslot_size) {
+    case 2:
+        ctx->hflags |= MIPS_HFLAG_BDS16;
+        break;
+    case 4:
+        ctx->hflags |= MIPS_HFLAG_BDS32;
+        break;
+    }
 
 out:
     tcg_temp_free_i64(t0);
@@ -9111,7 +9328,6 @@
 };
 static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
 {
-    const char *opn = "cp1 move";
     TCGv t0 = tcg_temp_new();
 
     switch (opc) {
@@ -9124,7 +9340,6 @@
             tcg_temp_free_i32(fp0);
         }
         gen_store_gpr(t0, rt);
-        opn = "mfc1";
         break;
     case OPC_MTC1:
         gen_load_gpr(t0, rt);
@@ -9135,16 +9350,14 @@
             gen_store_fpr32(ctx, fp0, fs);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mtc1";
         break;
     case OPC_CFC1:
         gen_helper_1e0i(cfc1, t0, fs);
         gen_store_gpr(t0, rt);
-        opn = "cfc1";
         break;
     case OPC_CTC1:
         gen_load_gpr(t0, rt);
-        save_cpu_state(ctx, 1);
+        save_cpu_state(ctx, 0);
         {
             TCGv_i32 fs_tmp = tcg_const_i32(fs);
 
@@ -9153,18 +9366,15 @@
         }
         /* Stop translation as we may have changed hflags */
         ctx->bstate = BS_STOP;
-        opn = "ctc1";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DMFC1:
         gen_load_fpr64(ctx, t0, fs);
         gen_store_gpr(t0, rt);
-        opn = "dmfc1";
         break;
     case OPC_DMTC1:
         gen_load_gpr(t0, rt);
         gen_store_fpr64(ctx, t0, fs);
-        opn = "dmtc1";
         break;
 #endif
     case OPC_MFHC1:
@@ -9176,7 +9386,6 @@
             tcg_temp_free_i32(fp0);
         }
         gen_store_gpr(t0, rt);
-        opn = "mfhc1";
         break;
     case OPC_MTHC1:
         gen_load_gpr(t0, rt);
@@ -9187,15 +9396,12 @@
             gen_store_fpr32h(ctx, fp0, fs);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mthc1";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("cp1 move");
+        generate_exception_end(ctx, EXCP_RI);
         goto out;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s %s", opn, regnames[rt], fregnames[fs]);
 
  out:
     tcg_temp_free(t0);
@@ -9203,7 +9409,7 @@
 
 static void gen_movci (DisasContext *ctx, int rd, int rs, int cc, int tf)
 {
-    int l1;
+    TCGLabel *l1;
     TCGCond cond;
     TCGv_i32 t0;
 
@@ -9230,11 +9436,12 @@
     gen_set_label(l1);
 }
 
-static inline void gen_movcf_s (DisasContext *ctx, int fs, int fd, int cc, int tf)
+static inline void gen_movcf_s(DisasContext *ctx, int fs, int fd, int cc,
+                               int tf)
 {
     int cond;
     TCGv_i32 t0 = tcg_temp_new_i32();
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
 
     if (tf)
         cond = TCG_COND_EQ;
@@ -9254,7 +9461,7 @@
     int cond;
     TCGv_i32 t0 = tcg_temp_new_i32();
     TCGv_i64 fp0;
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
 
     if (tf)
         cond = TCG_COND_EQ;
@@ -9276,8 +9483,8 @@
 {
     int cond;
     TCGv_i32 t0 = tcg_temp_new_i32();
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
 
     if (tf)
         cond = TCG_COND_EQ;
@@ -9324,7 +9531,7 @@
         break;
     default:
         MIPS_INVAL("gen_sel_s");
-        generate_exception (ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -9361,7 +9568,7 @@
         break;
     default:
         MIPS_INVAL("gen_sel_d");
-        generate_exception (ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -9375,46 +9582,7 @@
 static void gen_farith (DisasContext *ctx, enum fopcode op1,
                         int ft, int fs, int fd, int cc)
 {
-    const char *opn = "farith";
-    const char *condnames[] = {
-            "c.f",
-            "c.un",
-            "c.eq",
-            "c.ueq",
-            "c.olt",
-            "c.ult",
-            "c.ole",
-            "c.ule",
-            "c.sf",
-            "c.ngle",
-            "c.seq",
-            "c.ngl",
-            "c.lt",
-            "c.nge",
-            "c.le",
-            "c.ngt",
-    };
-    const char *condnames_abs[] = {
-            "cabs.f",
-            "cabs.un",
-            "cabs.eq",
-            "cabs.ueq",
-            "cabs.olt",
-            "cabs.ult",
-            "cabs.ole",
-            "cabs.ule",
-            "cabs.sf",
-            "cabs.ngle",
-            "cabs.seq",
-            "cabs.ngl",
-            "cabs.lt",
-            "cabs.nge",
-            "cabs.le",
-            "cabs.ngt",
-    };
-    enum { BINOP, CMPOP, OTHEROP } optype = OTHEROP;
     uint32_t func = ctx->opcode & 0x3f;
-
     switch (op1) {
     case OPC_ADD_S:
         {
@@ -9428,8 +9596,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "add.s";
-        optype = BINOP;
         break;
     case OPC_SUB_S:
         {
@@ -9443,8 +9609,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "sub.s";
-        optype = BINOP;
         break;
     case OPC_MUL_S:
         {
@@ -9458,8 +9622,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mul.s";
-        optype = BINOP;
         break;
     case OPC_DIV_S:
         {
@@ -9473,8 +9635,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "div.s";
-        optype = BINOP;
         break;
     case OPC_SQRT_S:
         {
@@ -9485,18 +9645,20 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "sqrt.s";
         break;
     case OPC_ABS_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_abs_s(fp0, fp0);
+            if (ctx->abs2008) {
+                tcg_gen_andi_i32(fp0, fp0, 0x7fffffffUL);
+            } else {
+                gen_helper_float_abs_s(fp0, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "abs.s";
         break;
     case OPC_MOV_S:
         {
@@ -9506,18 +9668,20 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "mov.s";
         break;
     case OPC_NEG_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_chs_s(fp0, fp0);
+            if (ctx->abs2008) {
+                tcg_gen_xori_i32(fp0, fp0, 1UL << 31);
+            } else {
+                gen_helper_float_chs_s(fp0, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "neg.s";
         break;
     case OPC_ROUND_L_S:
         check_cp1_64bitmode(ctx);
@@ -9526,12 +9690,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(ctx, fp32, fs);
-            gen_helper_float_roundl_s(fp64, cpu_env, fp32);
+            if (ctx->nan2008) {
+                gen_helper_float_round_2008_l_s(fp64, cpu_env, fp32);
+            } else {
+                gen_helper_float_round_l_s(fp64, cpu_env, fp32);
+            }
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "round.l.s";
         break;
     case OPC_TRUNC_L_S:
         check_cp1_64bitmode(ctx);
@@ -9540,12 +9707,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(ctx, fp32, fs);
-            gen_helper_float_truncl_s(fp64, cpu_env, fp32);
+            if (ctx->nan2008) {
+                gen_helper_float_trunc_2008_l_s(fp64, cpu_env, fp32);
+            } else {
+                gen_helper_float_trunc_l_s(fp64, cpu_env, fp32);
+            }
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "trunc.l.s";
         break;
     case OPC_CEIL_L_S:
         check_cp1_64bitmode(ctx);
@@ -9554,12 +9724,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(ctx, fp32, fs);
-            gen_helper_float_ceill_s(fp64, cpu_env, fp32);
+            if (ctx->nan2008) {
+                gen_helper_float_ceil_2008_l_s(fp64, cpu_env, fp32);
+            } else {
+                gen_helper_float_ceil_l_s(fp64, cpu_env, fp32);
+            }
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "ceil.l.s";
         break;
     case OPC_FLOOR_L_S:
         check_cp1_64bitmode(ctx);
@@ -9568,81 +9741,92 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(ctx, fp32, fs);
-            gen_helper_float_floorl_s(fp64, cpu_env, fp32);
+            if (ctx->nan2008) {
+                gen_helper_float_floor_2008_l_s(fp64, cpu_env, fp32);
+            } else {
+                gen_helper_float_floor_l_s(fp64, cpu_env, fp32);
+            }
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "floor.l.s";
         break;
     case OPC_ROUND_W_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_roundw_s(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_round_2008_w_s(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_round_w_s(fp0, cpu_env, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "round.w.s";
         break;
     case OPC_TRUNC_W_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_truncw_s(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_trunc_2008_w_s(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_trunc_w_s(fp0, cpu_env, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "trunc.w.s";
         break;
     case OPC_CEIL_W_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_ceilw_s(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_ceil_2008_w_s(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_ceil_w_s(fp0, cpu_env, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "ceil.w.s";
         break;
     case OPC_FLOOR_W_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_floorw_s(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_floor_2008_w_s(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_floor_w_s(fp0, cpu_env, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "floor.w.s";
         break;
     case OPC_SEL_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "sel.s";
         break;
     case OPC_SELEQZ_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "seleqz.s";
         break;
     case OPC_SELNEZ_S:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_s(ctx, op1, fd, ft, fs);
-        opn = "selnez.s";
         break;
     case OPC_MOVCF_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_s(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.s";
         break;
     case OPC_MOVZ_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
             TCGv_i32 fp0;
 
             if (ft != 0) {
@@ -9654,12 +9838,11 @@
             tcg_temp_free_i32(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.s";
         break;
     case OPC_MOVN_S:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
             TCGv_i32 fp0;
 
             if (ft != 0) {
@@ -9671,7 +9854,6 @@
                 gen_set_label(l1);
             }
         }
-        opn = "movn.s";
         break;
     case OPC_RECIP_S:
         {
@@ -9682,7 +9864,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "recip.s";
         break;
     case OPC_RSQRT_S:
         {
@@ -9693,7 +9874,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "rsqrt.s";
         break;
     case OPC_MADDF_S:
         check_insn(ctx, ISA_MIPS32R6);
@@ -9709,7 +9889,6 @@
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "maddf.s";
         }
         break;
     case OPC_MSUBF_S:
@@ -9726,7 +9905,6 @@
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "msubf.s";
         }
         break;
     case OPC_RINT_S:
@@ -9737,7 +9915,6 @@
             gen_helper_float_rint_s(fp0, cpu_env, fp0);
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
-            opn = "rint.s";
         }
         break;
     case OPC_CLASS_S:
@@ -9745,10 +9922,9 @@
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_class_s(fp0, fp0);
+            gen_helper_float_class_s(fp0, cpu_env, fp0);
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
-            opn = "class.s";
         }
         break;
     case OPC_MIN_S: /* OPC_RECIP2_S */
@@ -9764,7 +9940,6 @@
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "min.s";
         } else {
             /* OPC_RECIP2_S */
             check_cp1_64bitmode(ctx);
@@ -9779,7 +9954,6 @@
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "recip2.s";
         }
         break;
     case OPC_MINA_S: /* OPC_RECIP1_S */
@@ -9795,7 +9969,6 @@
             tcg_temp_free_i32(fp2);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "mina.s";
         } else {
             /* OPC_RECIP1_S */
             check_cp1_64bitmode(ctx);
@@ -9807,7 +9980,6 @@
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "recip1.s";
         }
         break;
     case OPC_MAX_S: /* OPC_RSQRT1_S */
@@ -9821,7 +9993,6 @@
             gen_store_fpr32(ctx, fp1, fd);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "max.s";
         } else {
             /* OPC_RSQRT1_S */
             check_cp1_64bitmode(ctx);
@@ -9833,7 +10004,6 @@
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "rsqrt1.s";
         }
         break;
     case OPC_MAXA_S: /* OPC_RSQRT2_S */
@@ -9847,7 +10017,6 @@
             gen_store_fpr32(ctx, fp1, fd);
             tcg_temp_free_i32(fp1);
             tcg_temp_free_i32(fp0);
-            opn = "maxa.s";
         } else {
             /* OPC_RSQRT2_S */
             check_cp1_64bitmode(ctx);
@@ -9862,7 +10031,6 @@
                 gen_store_fpr32(ctx, fp0, fd);
                 tcg_temp_free_i32(fp0);
             }
-            opn = "rsqrt2.s";
         }
         break;
     case OPC_CVT_D_S:
@@ -9877,18 +10045,20 @@
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.d.s";
         break;
     case OPC_CVT_W_S:
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(ctx, fp0, fs);
-            gen_helper_float_cvtw_s(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_cvt_2008_w_s(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_cvt_w_s(fp0, cpu_env, fp0);
+            }
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.w.s";
         break;
     case OPC_CVT_L_S:
         check_cp1_64bitmode(ctx);
@@ -9897,16 +10067,18 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(ctx, fp32, fs);
-            gen_helper_float_cvtl_s(fp64, cpu_env, fp32);
+            if (ctx->nan2008) {
+                gen_helper_float_cvt_2008_l_s(fp64, cpu_env, fp32);
+            } else {
+                gen_helper_float_cvt_l_s(fp64, cpu_env, fp32);
+            }
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.l.s";
         break;
     case OPC_CVT_PS_S:
-        check_insn_opc_removed(ctx, ISA_MIPS32R6);
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp64 = tcg_temp_new_i64();
             TCGv_i32 fp32_0 = tcg_temp_new_i32();
@@ -9920,7 +10092,6 @@
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.ps.s";
         break;
     case OPC_CMP_F_S:
     case OPC_CMP_UN_S:
@@ -9941,10 +10112,8 @@
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_s(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_s(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
         break;
     case OPC_ADD_D:
@@ -9960,8 +10129,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "add.d";
-        optype = BINOP;
         break;
     case OPC_SUB_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9976,8 +10143,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sub.d";
-        optype = BINOP;
         break;
     case OPC_MUL_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -9992,8 +10157,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mul.d";
-        optype = BINOP;
         break;
     case OPC_DIV_D:
         check_cp1_registers(ctx, fs | ft | fd);
@@ -10008,8 +10171,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "div.d";
-        optype = BINOP;
         break;
     case OPC_SQRT_D:
         check_cp1_registers(ctx, fs | fd);
@@ -10021,7 +10182,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sqrt.d";
         break;
     case OPC_ABS_D:
         check_cp1_registers(ctx, fs | fd);
@@ -10029,11 +10189,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_abs_d(fp0, fp0);
+            if (ctx->abs2008) {
+                tcg_gen_andi_i64(fp0, fp0, 0x7fffffffffffffffULL);
+            } else {
+                gen_helper_float_abs_d(fp0, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "abs.d";
         break;
     case OPC_MOV_D:
         check_cp1_registers(ctx, fs | fd);
@@ -10044,7 +10207,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mov.d";
         break;
     case OPC_NEG_D:
         check_cp1_registers(ctx, fs | fd);
@@ -10052,11 +10214,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_chs_d(fp0, fp0);
+            if (ctx->abs2008) {
+                tcg_gen_xori_i64(fp0, fp0, 1ULL << 63);
+            } else {
+                gen_helper_float_chs_d(fp0, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "neg.d";
         break;
     case OPC_ROUND_L_D:
         check_cp1_64bitmode(ctx);
@@ -10064,11 +10229,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_roundl_d(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_round_2008_l_d(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_round_l_d(fp0, cpu_env, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "round.l.d";
         break;
     case OPC_TRUNC_L_D:
         check_cp1_64bitmode(ctx);
@@ -10076,11 +10244,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_truncl_d(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_trunc_2008_l_d(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_trunc_l_d(fp0, cpu_env, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "trunc.l.d";
         break;
     case OPC_CEIL_L_D:
         check_cp1_64bitmode(ctx);
@@ -10088,11 +10259,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_ceill_d(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_ceil_2008_l_d(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_ceil_l_d(fp0, cpu_env, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ceil.l.d";
         break;
     case OPC_FLOOR_L_D:
         check_cp1_64bitmode(ctx);
@@ -10100,11 +10274,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_floorl_d(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_floor_2008_l_d(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_floor_l_d(fp0, cpu_env, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "floor.l.d";
         break;
     case OPC_ROUND_W_D:
         check_cp1_registers(ctx, fs);
@@ -10113,12 +10290,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_roundw_d(fp32, cpu_env, fp64);
+            if (ctx->nan2008) {
+                gen_helper_float_round_2008_w_d(fp32, cpu_env, fp64);
+            } else {
+                gen_helper_float_round_w_d(fp32, cpu_env, fp64);
+            }
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "round.w.d";
         break;
     case OPC_TRUNC_W_D:
         check_cp1_registers(ctx, fs);
@@ -10127,12 +10307,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_truncw_d(fp32, cpu_env, fp64);
+            if (ctx->nan2008) {
+                gen_helper_float_trunc_2008_w_d(fp32, cpu_env, fp64);
+            } else {
+                gen_helper_float_trunc_w_d(fp32, cpu_env, fp64);
+            }
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "trunc.w.d";
         break;
     case OPC_CEIL_W_D:
         check_cp1_registers(ctx, fs);
@@ -10141,12 +10324,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_ceilw_d(fp32, cpu_env, fp64);
+            if (ctx->nan2008) {
+                gen_helper_float_ceil_2008_w_d(fp32, cpu_env, fp64);
+            } else {
+                gen_helper_float_ceil_w_d(fp32, cpu_env, fp64);
+            }
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "ceil.w.d";
         break;
     case OPC_FLOOR_W_D:
         check_cp1_registers(ctx, fs);
@@ -10155,37 +10341,36 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_floorw_d(fp32, cpu_env, fp64);
+            if (ctx->nan2008) {
+                gen_helper_float_floor_2008_w_d(fp32, cpu_env, fp64);
+            } else {
+                gen_helper_float_floor_w_d(fp32, cpu_env, fp64);
+            }
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "floor.w.d";
         break;
     case OPC_SEL_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "sel.d";
         break;
     case OPC_SELEQZ_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "seleqz.d";
         break;
     case OPC_SELNEZ_D:
         check_insn(ctx, ISA_MIPS32R6);
         gen_sel_d(ctx, op1, fd, ft, fs);
-        opn = "selnez.d";
         break;
     case OPC_MOVCF_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movcf_d(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.d";
         break;
     case OPC_MOVZ_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
             TCGv_i64 fp0;
 
             if (ft != 0) {
@@ -10197,12 +10382,11 @@
             tcg_temp_free_i64(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.d";
         break;
     case OPC_MOVN_D:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         {
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
             TCGv_i64 fp0;
 
             if (ft != 0) {
@@ -10214,9 +10398,9 @@
                 gen_set_label(l1);
             }
         }
-        opn = "movn.d";
         break;
     case OPC_RECIP_D:
+        check_cp1_registers(ctx, fs | fd);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10225,9 +10409,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip.d";
         break;
     case OPC_RSQRT_D:
+        check_cp1_registers(ctx, fs | fd);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10236,7 +10420,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt.d";
         break;
     case OPC_MADDF_D:
         check_insn(ctx, ISA_MIPS32R6);
@@ -10252,7 +10435,6 @@
             tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "maddf.d";
         }
         break;
     case OPC_MSUBF_D:
@@ -10269,7 +10451,6 @@
             tcg_temp_free_i64(fp2);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "msubf.d";
         }
         break;
     case OPC_RINT_D:
@@ -10280,7 +10461,6 @@
             gen_helper_float_rint_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
-            opn = "rint.d";
         }
         break;
     case OPC_CLASS_D:
@@ -10288,10 +10468,9 @@
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_class_d(fp0, fp0);
+            gen_helper_float_class_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
-            opn = "class.d";
         }
         break;
     case OPC_MIN_D: /* OPC_RECIP2_D */
@@ -10305,7 +10484,6 @@
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "min.d";
         } else {
             /* OPC_RECIP2_D */
             check_cp1_64bitmode(ctx);
@@ -10320,7 +10498,6 @@
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "recip2.d";
         }
         break;
     case OPC_MINA_D: /* OPC_RECIP1_D */
@@ -10334,7 +10511,6 @@
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "mina.d";
         } else {
             /* OPC_RECIP1_D */
             check_cp1_64bitmode(ctx);
@@ -10346,7 +10522,6 @@
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "recip1.d";
         }
         break;
     case OPC_MAX_D: /*  OPC_RSQRT1_D */
@@ -10360,7 +10535,6 @@
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "max.d";
         } else {
             /* OPC_RSQRT1_D */
             check_cp1_64bitmode(ctx);
@@ -10372,7 +10546,6 @@
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "rsqrt1.d";
         }
         break;
     case OPC_MAXA_D: /* OPC_RSQRT2_D */
@@ -10386,7 +10559,6 @@
             gen_store_fpr64(ctx, fp1, fd);
             tcg_temp_free_i64(fp1);
             tcg_temp_free_i64(fp0);
-            opn = "maxa.d";
         } else {
             /* OPC_RSQRT2_D */
             check_cp1_64bitmode(ctx);
@@ -10401,7 +10573,6 @@
                 gen_store_fpr64(ctx, fp0, fd);
                 tcg_temp_free_i64(fp0);
             }
-            opn = "rsqrt2.d";
         }
         break;
     case OPC_CMP_F_D:
@@ -10423,10 +10594,8 @@
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_d(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_d(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
         break;
     case OPC_CVT_S_D:
@@ -10441,7 +10610,6 @@
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.s.d";
         break;
     case OPC_CVT_W_D:
         check_cp1_registers(ctx, fs);
@@ -10450,12 +10618,15 @@
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvtw_d(fp32, cpu_env, fp64);
+            if (ctx->nan2008) {
+                gen_helper_float_cvt_2008_w_d(fp32, cpu_env, fp64);
+            } else {
+                gen_helper_float_cvt_w_d(fp32, cpu_env, fp64);
+            }
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.w.d";
         break;
     case OPC_CVT_L_D:
         check_cp1_64bitmode(ctx);
@@ -10463,11 +10634,14 @@
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtl_d(fp0, cpu_env, fp0);
+            if (ctx->nan2008) {
+                gen_helper_float_cvt_2008_l_d(fp0, cpu_env, fp0);
+            } else {
+                gen_helper_float_cvt_l_d(fp0, cpu_env, fp0);
+            }
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.l.d";
         break;
     case OPC_CVT_S_W:
         {
@@ -10478,7 +10652,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.w";
         break;
     case OPC_CVT_D_W:
         check_cp1_registers(ctx, fd);
@@ -10492,7 +10665,6 @@
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
         }
-        opn = "cvt.d.w";
         break;
     case OPC_CVT_S_L:
         check_cp1_64bitmode(ctx);
@@ -10506,7 +10678,6 @@
             gen_store_fpr32(ctx, fp32, fd);
             tcg_temp_free_i32(fp32);
         }
-        opn = "cvt.s.l";
         break;
     case OPC_CVT_D_L:
         check_cp1_64bitmode(ctx);
@@ -10518,11 +10689,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.d.l";
         break;
     case OPC_CVT_PS_PW:
-        check_insn_opc_removed(ctx, ISA_MIPS32R6);
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10531,10 +10700,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.ps.pw";
         break;
     case OPC_ADD_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10546,10 +10714,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "add.ps";
         break;
     case OPC_SUB_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10561,10 +10728,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sub.ps";
         break;
     case OPC_MUL_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10576,10 +10742,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mul.ps";
         break;
     case OPC_ABS_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10588,10 +10753,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "abs.ps";
         break;
     case OPC_MOV_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10599,10 +10763,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mov.ps";
         break;
     case OPC_NEG_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10611,17 +10774,15 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "neg.ps";
         break;
     case OPC_MOVCF_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         gen_movcf_ps(ctx, fs, fd, (ft >> 2) & 0x7, ft & 0x1);
-        opn = "movcf.ps";
         break;
     case OPC_MOVZ_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
             TCGv_i64 fp0;
 
             if (ft != 0)
@@ -10632,12 +10793,11 @@
             tcg_temp_free_i64(fp0);
             gen_set_label(l1);
         }
-        opn = "movz.ps";
         break;
     case OPC_MOVN_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
             TCGv_i64 fp0;
 
             if (ft != 0) {
@@ -10649,10 +10809,9 @@
                 gen_set_label(l1);
             }
         }
-        opn = "movn.ps";
         break;
     case OPC_ADDR_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10664,10 +10823,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "addr.ps";
         break;
     case OPC_MULR_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10679,10 +10837,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "mulr.ps";
         break;
     case OPC_RECIP2_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10694,10 +10851,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip2.ps";
         break;
     case OPC_RECIP1_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10706,10 +10862,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "recip1.ps";
         break;
     case OPC_RSQRT1_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10718,10 +10873,9 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt1.ps";
         break;
     case OPC_RSQRT2_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -10733,7 +10887,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "rsqrt2.ps";
         break;
     case OPC_CVT_S_PU:
         check_cp1_64bitmode(ctx);
@@ -10745,10 +10898,9 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.pu";
         break;
     case OPC_CVT_PW_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
@@ -10757,7 +10909,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "cvt.pw.ps";
         break;
     case OPC_CVT_S_PL:
         check_cp1_64bitmode(ctx);
@@ -10769,10 +10920,9 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "cvt.s.pl";
         break;
     case OPC_PLL_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
             TCGv_i32 fp1 = tcg_temp_new_i32();
@@ -10784,10 +10934,9 @@
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "pll.ps";
         break;
     case OPC_PLU_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
             TCGv_i32 fp1 = tcg_temp_new_i32();
@@ -10799,10 +10948,9 @@
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "plu.ps";
         break;
     case OPC_PUL_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
             TCGv_i32 fp1 = tcg_temp_new_i32();
@@ -10814,10 +10962,9 @@
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "pul.ps";
         break;
     case OPC_PUU_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i32 fp0 = tcg_temp_new_i32();
             TCGv_i32 fp1 = tcg_temp_new_i32();
@@ -10829,7 +10976,6 @@
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
         }
-        opn = "puu.ps";
         break;
     case OPC_CMP_F_PS:
     case OPC_CMP_UN_PS:
@@ -10849,37 +10995,21 @@
     case OPC_CMP_NGT_PS:
         if (ctx->opcode & (1 << 6)) {
             gen_cmpabs_ps(ctx, func-48, ft, fs, cc);
-            opn = condnames_abs[func-48];
         } else {
             gen_cmp_ps(ctx, func-48, ft, fs, cc);
-            opn = condnames[func-48];
         }
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("farith");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
-    (void)opn; /* avoid a compiler warning */
-    switch (optype) {
-    case BINOP:
-        MIPS_DEBUG("%s %s, %s, %s", opn, fregnames[fd], fregnames[fs], fregnames[ft]);
-        break;
-    case CMPOP:
-        MIPS_DEBUG("%s %s,%s", opn, fregnames[fs], fregnames[ft]);
-        break;
-    default:
-        MIPS_DEBUG("%s %s,%s", opn, fregnames[fd], fregnames[fs]);
-        break;
-    }
 }
 
 /* Coprocessor 3 (FPU) */
 static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
                            int fd, int fs, int base, int index)
 {
-    const char *opn = "extended float load/store";
-    int store = 0;
     TCGv t0 = tcg_temp_new();
 
     if (base == 0) {
@@ -10902,7 +11032,6 @@
             gen_store_fpr32(ctx, fp0, fd);
             tcg_temp_free_i32(fp0);
         }
-        opn = "lwxc1";
         break;
     case OPC_LDXC1:
         check_cop1x(ctx);
@@ -10913,7 +11042,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "ldxc1";
         break;
     case OPC_LUXC1:
         check_cp1_64bitmode(ctx);
@@ -10925,7 +11053,6 @@
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
-        opn = "luxc1";
         break;
     case OPC_SWXC1:
         check_cop1x(ctx);
@@ -10935,8 +11062,6 @@
             tcg_gen_qemu_st_i32(fp0, t0, ctx->mem_idx, MO_TEUL);
             tcg_temp_free_i32(fp0);
         }
-        opn = "swxc1";
-        store = 1;
         break;
     case OPC_SDXC1:
         check_cop1x(ctx);
@@ -10947,8 +11072,6 @@
             tcg_gen_qemu_st_i64(fp0, t0, ctx->mem_idx, MO_TEQ);
             tcg_temp_free_i64(fp0);
         }
-        opn = "sdxc1";
-        store = 1;
         break;
     case OPC_SUXC1:
         check_cp1_64bitmode(ctx);
@@ -10959,30 +11082,23 @@
             tcg_gen_qemu_st_i64(fp0, t0, ctx->mem_idx, MO_TEQ);
             tcg_temp_free_i64(fp0);
         }
-        opn = "suxc1";
-        store = 1;
         break;
     }
     tcg_temp_free(t0);
-    (void)opn; (void)store; /* avoid compiler warnings */
-    MIPS_DEBUG("%s %s, %s(%s)", opn, fregnames[store ? fs : fd],
-               regnames[index], regnames[base]);
 }
 
 static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
                             int fd, int fr, int fs, int ft)
 {
-    const char *opn = "flt3_arith";
-
     switch (opc) {
     case OPC_ALNV_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv t0 = tcg_temp_local_new();
             TCGv_i32 fp = tcg_temp_new_i32();
             TCGv_i32 fph = tcg_temp_new_i32();
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
+            TCGLabel *l2 = gen_new_label();
 
             gen_load_gpr(t0, fr);
             tcg_gen_andi_tl(t0, t0, 0x7);
@@ -11011,7 +11127,6 @@
             tcg_temp_free_i32(fp);
             tcg_temp_free_i32(fph);
         }
-        opn = "alnv.ps";
         break;
     case OPC_MADD_S:
         check_cop1x(ctx);
@@ -11029,7 +11144,6 @@
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "madd.s";
         break;
     case OPC_MADD_D:
         check_cop1x(ctx);
@@ -11048,10 +11162,9 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "madd.d";
         break;
     case OPC_MADD_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -11066,7 +11179,6 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "madd.ps";
         break;
     case OPC_MSUB_S:
         check_cop1x(ctx);
@@ -11084,7 +11196,6 @@
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "msub.s";
         break;
     case OPC_MSUB_D:
         check_cop1x(ctx);
@@ -11103,10 +11214,9 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "msub.d";
         break;
     case OPC_MSUB_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -11121,7 +11231,6 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "msub.ps";
         break;
     case OPC_NMADD_S:
         check_cop1x(ctx);
@@ -11139,7 +11248,6 @@
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "nmadd.s";
         break;
     case OPC_NMADD_D:
         check_cop1x(ctx);
@@ -11158,10 +11266,9 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmadd.d";
         break;
     case OPC_NMADD_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -11176,7 +11283,6 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmadd.ps";
         break;
     case OPC_NMSUB_S:
         check_cop1x(ctx);
@@ -11194,7 +11300,6 @@
             gen_store_fpr32(ctx, fp2, fd);
             tcg_temp_free_i32(fp2);
         }
-        opn = "nmsub.s";
         break;
     case OPC_NMSUB_D:
         check_cop1x(ctx);
@@ -11213,10 +11318,9 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmsub.d";
         break;
     case OPC_NMSUB_PS:
-        check_cp1_64bitmode(ctx);
+        check_ps(ctx);
         {
             TCGv_i64 fp0 = tcg_temp_new_i64();
             TCGv_i64 fp1 = tcg_temp_new_i64();
@@ -11231,19 +11335,15 @@
             gen_store_fpr64(ctx, fp2, fd);
             tcg_temp_free_i64(fp2);
         }
-        opn = "nmsub.ps";
         break;
     default:
-        MIPS_INVAL(opn);
-        generate_exception (ctx, EXCP_RI);
+        MIPS_INVAL("flt3_arith");
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, %s, %s", opn, fregnames[fd], fregnames[fr],
-               fregnames[fs], fregnames[ft]);
 }
 
-static void gen_rdhwr(DisasContext *ctx, int rt, int rd)
+static void gen_rdhwr(DisasContext *ctx, int rt, int rd, int sel)
 {
     TCGv t0;
 
@@ -11256,25 +11356,37 @@
 
     switch (rd) {
     case 0:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_cpunum(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 1:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_synci_step(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 2:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_cc(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 3:
-        save_cpu_state(ctx, 1);
         gen_helper_rdhwr_ccres(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
+    case 4:
+        check_insn(ctx, ISA_MIPS32R6);
+        if (sel != 0) {
+            /* Performance counter registers are not implemented other than
+             * control register 0.
+             */
+            generate_exception(ctx, EXCP_RI);
+        }
+        gen_helper_rdhwr_performance(t0, cpu_env);
+        gen_store_gpr(t0, rt);
+        break;
+    case 5:
+        check_insn(ctx, ISA_MIPS32R6);
+        gen_helper_rdhwr_xnp(t0, cpu_env);
+        gen_store_gpr(t0, rt);
+        break;
     case 29:
 #if defined(CONFIG_USER_ONLY)
         tcg_gen_ld_tl(t0, cpu_env,
@@ -11288,13 +11400,13 @@
                           offsetof(CPUMIPSState, active_tc.CP0_UserLocal));
             gen_store_gpr(t0, rt);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("rdhwr");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
     tcg_temp_free(t0);
@@ -11322,12 +11434,10 @@
         /* FIXME: Need to clear can_do_io.  */
         switch (proc_hflags & MIPS_HFLAG_BMASK_BASE) {
         case MIPS_HFLAG_FBNSLOT:
-            MIPS_DEBUG("forbidden slot");
             gen_goto_tb(ctx, 0, ctx->pc + insn_bytes);
             break;
         case MIPS_HFLAG_B:
             /* unconditional branch */
-            MIPS_DEBUG("unconditional branch");
             if (proc_hflags & MIPS_HFLAG_BX) {
                 tcg_gen_xori_i32(hflags, hflags, MIPS_HFLAG_M16);
             }
@@ -11335,14 +11445,12 @@
             break;
         case MIPS_HFLAG_BL:
             /* blikely taken case */
-            MIPS_DEBUG("blikely branch taken");
             gen_goto_tb(ctx, 0, ctx->btarget);
             break;
         case MIPS_HFLAG_BC:
             /* Conditional branch */
-            MIPS_DEBUG("conditional branch");
             {
-                int l1 = gen_new_label();
+                TCGLabel *l1 = gen_new_label();
 
                 tcg_gen_brcondi_tl(TCG_COND_NE, bcond, 0, l1);
                 gen_goto_tb(ctx, 1, ctx->pc + insn_bytes);
@@ -11352,7 +11460,6 @@
             break;
         case MIPS_HFLAG_BR:
             /* unconditional branch to register */
-            MIPS_DEBUG("branch to register");
             if (ctx->insn_flags & (ASE_MIPS16 | ASE_MICROMIPS)) {
                 TCGv t0 = tcg_temp_new();
                 TCGv_i32 t1 = tcg_temp_new_i32();
@@ -11371,7 +11478,7 @@
             }
             if (ctx->singlestep_enabled) {
                 save_cpu_state(ctx, 0);
-                gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+                gen_helper_raise_exception_debug(cpu_env);
             }
             tcg_gen_exit_tb(0);
             break;
@@ -11382,6 +11489,242 @@
     }
 }
 
+/* Compact Branches */
+static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
+                                       int rs, int rt, int32_t offset)
+{
+    int bcond_compute = 0;
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    int m16_lowbit = (ctx->hflags & MIPS_HFLAG_M16) != 0;
+
+    if (ctx->hflags & MIPS_HFLAG_BMASK) {
+#ifdef MIPS_DEBUG_DISAS
+        LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
+                  "\n", ctx->pc);
+#endif
+        generate_exception_end(ctx, EXCP_RI);
+        goto out;
+    }
+
+    /* Load needed operands and calculate btarget */
+    switch (opc) {
+    /* compact branch */
+    case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC */
+    case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
+        gen_load_gpr(t0, rs);
+        gen_load_gpr(t1, rt);
+        bcond_compute = 1;
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        if (rs <= rt && rs == 0) {
+            /* OPC_BEQZALC, OPC_BNEZALC */
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4 + m16_lowbit);
+        }
+        break;
+    case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC */
+    case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC */
+        gen_load_gpr(t0, rs);
+        gen_load_gpr(t1, rt);
+        bcond_compute = 1;
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        break;
+    case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC */
+    case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC */
+        if (rs == 0 || rs == rt) {
+            /* OPC_BLEZALC, OPC_BGEZALC */
+            /* OPC_BGTZALC, OPC_BLTZALC */
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4 + m16_lowbit);
+        }
+        gen_load_gpr(t0, rs);
+        gen_load_gpr(t1, rt);
+        bcond_compute = 1;
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        break;
+    case OPC_BC:
+    case OPC_BALC:
+        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        break;
+    case OPC_BEQZC:
+    case OPC_BNEZC:
+        if (rs != 0) {
+            /* OPC_BEQZC, OPC_BNEZC */
+            gen_load_gpr(t0, rs);
+            bcond_compute = 1;
+            ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
+        } else {
+            /* OPC_JIC, OPC_JIALC */
+            TCGv tbase = tcg_temp_new();
+            TCGv toffset = tcg_temp_new();
+
+            gen_load_gpr(tbase, rt);
+            tcg_gen_movi_tl(toffset, offset);
+            gen_op_addr_add(ctx, btarget, tbase, toffset);
+            tcg_temp_free(tbase);
+            tcg_temp_free(toffset);
+        }
+        break;
+    default:
+        MIPS_INVAL("Compact branch/jump");
+        generate_exception_end(ctx, EXCP_RI);
+        goto out;
+    }
+
+    if (bcond_compute == 0) {
+        /* Uncoditional compact branch */
+        switch (opc) {
+        case OPC_JIALC:
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4 + m16_lowbit);
+            /* Fallthrough */
+        case OPC_JIC:
+            ctx->hflags |= MIPS_HFLAG_BR;
+            break;
+        case OPC_BALC:
+            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4 + m16_lowbit);
+            /* Fallthrough */
+        case OPC_BC:
+            ctx->hflags |= MIPS_HFLAG_B;
+            break;
+        default:
+            MIPS_INVAL("Compact branch/jump");
+            generate_exception_end(ctx, EXCP_RI);
+            goto out;
+        }
+
+        /* Generating branch here as compact branches don't have delay slot */
+        gen_branch(ctx, 4);
+    } else {
+        /* Conditional compact branch */
+        TCGLabel *fs = gen_new_label();
+        save_cpu_state(ctx, 0);
+
+        switch (opc) {
+        case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BLEZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LE), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BGEZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GE), t1, 0, fs);
+            } else {
+                /* OPC_BGEUC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_GEU), t0, t1, fs);
+            }
+            break;
+        case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BGTZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GT), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BLTZALC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LT), t1, 0, fs);
+            } else {
+                /* OPC_BLTUC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_LTU), t0, t1, fs);
+            }
+            break;
+        case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BLEZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LE), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BGEZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GE), t1, 0, fs);
+            } else {
+                /* OPC_BGEC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_GE), t0, t1, fs);
+            }
+            break;
+        case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC */
+            if (rs == 0 && rt != 0) {
+                /* OPC_BGTZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GT), t1, 0, fs);
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* OPC_BLTZC */
+                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LT), t1, 0, fs);
+            } else {
+                /* OPC_BLTC */
+                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_LT), t0, t1, fs);
+            }
+            break;
+        case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC */
+        case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
+            if (rs >= rt) {
+                /* OPC_BOVC, OPC_BNVC */
+                TCGv t2 = tcg_temp_new();
+                TCGv t3 = tcg_temp_new();
+                TCGv t4 = tcg_temp_new();
+                TCGv input_overflow = tcg_temp_new();
+
+                gen_load_gpr(t0, rs);
+                gen_load_gpr(t1, rt);
+                tcg_gen_ext32s_tl(t2, t0);
+                tcg_gen_setcond_tl(TCG_COND_NE, input_overflow, t2, t0);
+                tcg_gen_ext32s_tl(t3, t1);
+                tcg_gen_setcond_tl(TCG_COND_NE, t4, t3, t1);
+                tcg_gen_or_tl(input_overflow, input_overflow, t4);
+
+                tcg_gen_add_tl(t4, t2, t3);
+                tcg_gen_ext32s_tl(t4, t4);
+                tcg_gen_xor_tl(t2, t2, t3);
+                tcg_gen_xor_tl(t3, t4, t3);
+                tcg_gen_andc_tl(t2, t3, t2);
+                tcg_gen_setcondi_tl(TCG_COND_LT, t4, t2, 0);
+                tcg_gen_or_tl(t4, t4, input_overflow);
+                if (opc == OPC_BOVC) {
+                    /* OPC_BOVC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t4, 0, fs);
+                } else {
+                    /* OPC_BNVC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t4, 0, fs);
+                }
+                tcg_temp_free(input_overflow);
+                tcg_temp_free(t4);
+                tcg_temp_free(t3);
+                tcg_temp_free(t2);
+            } else if (rs < rt && rs == 0) {
+                /* OPC_BEQZALC, OPC_BNEZALC */
+                if (opc == OPC_BEQZALC) {
+                    /* OPC_BEQZALC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t1, 0, fs);
+                } else {
+                    /* OPC_BNEZALC */
+                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t1, 0, fs);
+                }
+            } else {
+                /* OPC_BEQC, OPC_BNEC */
+                if (opc == OPC_BEQC) {
+                    /* OPC_BEQC */
+                    tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_EQ), t0, t1, fs);
+                } else {
+                    /* OPC_BNEC */
+                    tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_NE), t0, t1, fs);
+                }
+            }
+            break;
+        case OPC_BEQZC:
+            tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t0, 0, fs);
+            break;
+        case OPC_BNEZC:
+            tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t0, 0, fs);
+            break;
+        default:
+            MIPS_INVAL("Compact conditional branch/jump");
+            generate_exception_end(ctx, EXCP_RI);
+            goto out;
+        }
+
+        /* Generating branch here as compact branches don't have delay slot */
+        gen_goto_tb(ctx, 1, ctx->btarget);
+        gen_set_label(fs);
+
+        ctx->hflags |= MIPS_HFLAG_FBNSLOT;
+    }
+
+out:
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+}
+
 /* ISA extensions (ASEs) */
 /* MIPS16 extension to MIPS32 */
 
@@ -11510,6 +11853,7 @@
 {
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
     int args, astatic;
 
     switch (aregs) {
@@ -11539,7 +11883,7 @@
         args = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11568,7 +11912,8 @@
     gen_load_gpr(t0, 29);
 
 #define DECR_AND_STORE(reg) do {                                 \
-        tcg_gen_subi_tl(t0, t0, 4);                              \
+        tcg_gen_movi_tl(t2, -4);                                 \
+        gen_op_addr_add(ctx, t0, t0, t2);                        \
         gen_load_gpr(t1, reg);                                   \
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL); \
     } while (0)
@@ -11634,7 +11979,7 @@
         astatic = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11652,9 +11997,11 @@
     }
 #undef DECR_AND_STORE
 
-    tcg_gen_subi_tl(cpu_gpr[29], cpu_gpr[29], framesize);
+    tcg_gen_movi_tl(t2, -framesize);
+    gen_op_addr_add(ctx, cpu_gpr[29], cpu_gpr[29], t2);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
+    tcg_temp_free(t2);
 }
 
 static void gen_mips16_restore (DisasContext *ctx,
@@ -11665,11 +12012,14 @@
     int astatic;
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
 
-    tcg_gen_addi_tl(t0, cpu_gpr[29], framesize);
+    tcg_gen_movi_tl(t2, framesize);
+    gen_op_addr_add(ctx, t0, cpu_gpr[29], t2);
 
 #define DECR_AND_LOAD(reg) do {                            \
-        tcg_gen_subi_tl(t0, t0, 4);                        \
+        tcg_gen_movi_tl(t2, -4);                           \
+        gen_op_addr_add(ctx, t0, t0, t2);                  \
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL); \
         gen_store_gpr(t1, reg);                            \
     } while (0)
@@ -11735,7 +12085,7 @@
         astatic = 4;
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11753,9 +12103,11 @@
     }
 #undef DECR_AND_LOAD
 
-    tcg_gen_addi_tl(cpu_gpr[29], cpu_gpr[29], framesize);
+    tcg_gen_movi_tl(t2, framesize);
+    gen_op_addr_add(ctx, cpu_gpr[29], cpu_gpr[29], t2);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
+    tcg_temp_free(t2);
 }
 
 static void gen_addiupc (DisasContext *ctx, int rx, int imm,
@@ -11764,7 +12116,7 @@
     TCGv t0;
 
     if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -11779,6 +12131,15 @@
     tcg_temp_free(t0);
 }
 
+static void gen_cache_operation(DisasContext *ctx, uint32_t op, int base,
+                                int16_t offset)
+{
+    TCGv_i32 t0 = tcg_const_i32(op);
+    TCGv t1 = tcg_temp_new();
+    gen_base_offset_addr(ctx, t1, base, offset);
+    gen_helper_cache(cpu_env, t1, t0);
+}
+
 #if defined(TARGET_MIPS64)
 static void decode_i64_mips16 (DisasContext *ctx,
                                int ry, int funct, int16_t offset,
@@ -11786,44 +12147,53 @@
 {
     switch (funct) {
     case I64_LDSP:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : offset << 3;
         gen_ld(ctx, OPC_LD, ry, 29, offset);
         break;
     case I64_SDSP:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : offset << 3;
         gen_st(ctx, OPC_SD, ry, 29, offset);
         break;
     case I64_SDRASP:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : (ctx->opcode & 0xff) << 3;
         gen_st(ctx, OPC_SD, 31, 29, offset);
         break;
     case I64_DADJSP:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : ((int8_t)ctx->opcode) << 3;
         gen_arith_imm(ctx, OPC_DADDIU, 29, 29, offset);
         break;
     case I64_LDPC:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
         if (extended && (ctx->hflags & MIPS_HFLAG_BMASK)) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             offset = extended ? offset : offset << 3;
             gen_ld(ctx, OPC_LDPC, ry, 0, offset);
         }
         break;
     case I64_DADDIU5:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : ((int8_t)(offset << 3)) >> 3;
         gen_arith_imm(ctx, OPC_DADDIU, ry, ry, offset);
         break;
     case I64_DADDIUPC:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : offset << 2;
         gen_addiupc(ctx, ry, offset, 1, extended);
         break;
     case I64_DADDIUSP:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         offset = extended ? offset : offset << 2;
         gen_arith_imm(ctx, OPC_DADDIU, ry, 29, offset);
@@ -11879,7 +12249,7 @@
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
@@ -11892,7 +12262,8 @@
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_LD:
-            check_mips_64(ctx);
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
         gen_ld(ctx, OPC_LD, ry, rx, offset);
         break;
 #endif
@@ -11906,7 +12277,7 @@
             check_mips_64(ctx);
             gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
         } else {
             gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
@@ -11936,6 +12307,7 @@
             gen_arith_imm(ctx, OPC_ADDIU, 29, 29, imm);
             break;
         case I8_SVRS:
+            check_insn(ctx, ISA_MIPS32);
             {
                 int xsregs = (ctx->opcode >> 24) & 0x7;
                 int aregs = (ctx->opcode >> 16) & 0xf;
@@ -11957,7 +12329,7 @@
             }
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -11969,6 +12341,8 @@
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_SD:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
         gen_st(ctx, OPC_SD, ry, rx, offset);
         break;
 #endif
@@ -11995,6 +12369,8 @@
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_LWU:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
         gen_ld(ctx, OPC_LWU, ry, rx, offset);
         break;
 #endif
@@ -12016,13 +12392,22 @@
         break;
 #endif
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
     return 4;
 }
 
+static inline bool is_uhi(int sdbbp_code)
+{
+#ifdef CONFIG_USER_ONLY
+    return false;
+#else
+    return semihosting_enabled() && sdbbp_code == 1;
+#endif
+}
+
 static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx)
 {
     int rx, ry;
@@ -12084,10 +12469,11 @@
             break;
         case 0x1:
 #if defined(TARGET_MIPS64)
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSLL, rx, ry, sa);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #endif
             break;
         case 0x2:
@@ -12100,6 +12486,7 @@
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_LD:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_ld(ctx, OPC_LD, ry, rx, offset << 3);
         break;
@@ -12110,10 +12497,11 @@
 
             if ((ctx->opcode >> 4) & 1) {
 #if defined(TARGET_MIPS64)
+                check_insn(ctx, ISA_MIPS3);
                 check_mips_64(ctx);
                 gen_arith_imm(ctx, OPC_DADDIU, ry, rx, imm);
 #else
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
 #endif
             } else {
                 gen_arith_imm(ctx, OPC_ADDIU, ry, rx, imm);
@@ -12161,6 +12549,7 @@
                               ((int8_t)ctx->opcode) << 3);
                 break;
             case I8_SVRS:
+                check_insn(ctx, ISA_MIPS32);
                 {
                     int do_ra = ctx->opcode & (1 << 6);
                     int do_s0 = ctx->opcode & (1 << 5);
@@ -12196,7 +12585,7 @@
                 gen_arith(ctx, OPC_ADDU, ry, reg32, 0);
                 break;
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         }
@@ -12216,6 +12605,7 @@
         break;
 #if defined(TARGET_MIPS64)
     case M16_OPC_SD:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_st(ctx, OPC_SD, ry, rx, offset << 3);
         break;
@@ -12243,6 +12633,7 @@
         break;
 #if defined (TARGET_MIPS64)
     case M16_OPC_LWU:
+        check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
         gen_ld(ctx, OPC_LWU, ry, rx, offset << 2);
         break;
@@ -12274,15 +12665,17 @@
 #if defined(TARGET_MIPS64)
             case RRR_DADDU:
                 mips32_op = OPC_DADDU;
+                check_insn(ctx, ISA_MIPS3);
                 check_mips_64(ctx);
                 break;
             case RRR_DSUBU:
                 mips32_op = OPC_DSUBU;
+                check_insn(ctx, ISA_MIPS3);
                 check_mips_64(ctx);
                 break;
 #endif
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 goto done;
             }
 
@@ -12299,6 +12692,10 @@
                 int link = (ctx->opcode >> 6) & 0x1;
                 int ra = (ctx->opcode >> 5) & 0x1;
 
+                if (nd) {
+                    check_insn(ctx, ISA_MIPS32);
+                }
+
                 if (link) {
                     op = OPC_JALR;
                 } else {
@@ -12310,14 +12707,14 @@
             }
             break;
         case RR_SDBBP:
-            /* XXX: not clear which exception should be raised
-             *      when in debug mode...
-             */
-            check_insn(ctx, ISA_MIPS32);
-            if (!(ctx->hflags & MIPS_HFLAG_DM)) {
-                generate_exception(ctx, EXCP_DBp);
+            if (is_uhi(extract32(ctx->opcode, 5, 6))) {
+                gen_helper_do_semihosting(cpu_env);
             } else {
-                generate_exception(ctx, EXCP_DBp);
+                /* XXX: not clear which exception should be raised
+                 *      when in debug mode...
+                 */
+                check_insn(ctx, ISA_MIPS32);
+                generate_exception_end(ctx, EXCP_DBp);
             }
             break;
         case RR_SLT:
@@ -12327,7 +12724,7 @@
             gen_slt(ctx, OPC_SLTU, 24, rx, ry);
             break;
         case RR_BREAK:
-            generate_exception(ctx, EXCP_BREAK);
+            generate_exception_end(ctx, EXCP_BREAK);
             break;
         case RR_SLLV:
             gen_shift(ctx, OPC_SLLV, ry, rx, ry);
@@ -12340,6 +12737,7 @@
             break;
 #if defined (TARGET_MIPS64)
         case RR_DSRL:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSRL, ry, ry, sa);
             break;
@@ -12366,6 +12764,7 @@
             gen_HILO(ctx, OPC_MFHI, 0, rx);
             break;
         case RR_CNVT:
+            check_insn(ctx, ISA_MIPS32);
             switch (cnvt_op) {
             case RR_RY_CNVT_ZEB:
                 tcg_gen_ext8u_tl(cpu_gpr[rx], cpu_gpr[rx]);
@@ -12381,16 +12780,18 @@
                 break;
 #if defined (TARGET_MIPS64)
             case RR_RY_CNVT_ZEW:
+                check_insn(ctx, ISA_MIPS64);
                 check_mips_64(ctx);
                 tcg_gen_ext32u_tl(cpu_gpr[rx], cpu_gpr[rx]);
                 break;
             case RR_RY_CNVT_SEW:
+                check_insn(ctx, ISA_MIPS64);
                 check_mips_64(ctx);
                 tcg_gen_ext32s_tl(cpu_gpr[rx], cpu_gpr[rx]);
                 break;
 #endif
             default:
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             break;
@@ -12399,18 +12800,22 @@
             break;
 #if defined (TARGET_MIPS64)
         case RR_DSRA:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_shift_imm(ctx, OPC_DSRA, ry, ry, sa);
             break;
         case RR_DSLLV:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_shift(ctx, OPC_DSLLV, ry, rx, ry);
             break;
         case RR_DSRLV:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_shift(ctx, OPC_DSRLV, ry, rx, ry);
             break;
         case RR_DSRAV:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_shift(ctx, OPC_DSRAV, ry, rx, ry);
             break;
@@ -12429,24 +12834,28 @@
             break;
 #if defined (TARGET_MIPS64)
         case RR_DMULT:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_muldiv(ctx, OPC_DMULT, 0, rx, ry);
             break;
         case RR_DMULTU:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_muldiv(ctx, OPC_DMULTU, 0, rx, ry);
             break;
         case RR_DDIV:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_muldiv(ctx, OPC_DDIV, 0, rx, ry);
             break;
         case RR_DDIVU:
+            check_insn(ctx, ISA_MIPS3);
             check_mips_64(ctx);
             gen_muldiv(ctx, OPC_DDIVU, 0, rx, ry);
             break;
 #endif
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -12461,7 +12870,7 @@
         break;
 #endif
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -12488,6 +12897,8 @@
     LBU16 = 0x02,
     MOVE16 = 0x03,
     ADDI32 = 0x04,
+    R6_LUI = 0x04,
+    AUI = 0x04,
     LBU32 = 0x05,
     SB32 = 0x06,
     LB32 = 0x07,
@@ -12510,56 +12921,88 @@
     POOL32S = 0x16,  /* MIPS64 */
     DADDIU32 = 0x17, /* MIPS64 */
 
-    /* 0x1f is reserved */
     POOL32C = 0x18,
     LWGP16 = 0x19,
     LW16 = 0x1a,
     POOL16E = 0x1b,
     XORI32 = 0x1c,
     JALS32 = 0x1d,
+    BOVC = 0x1d,
+    BEQC = 0x1d,
+    BEQZALC = 0x1d,
     ADDIUPC = 0x1e,
+    PCREL = 0x1e,
+    BNVC = 0x1f,
+    BNEC = 0x1f,
+    BNEZALC = 0x1f,
 
-    /* 0x20 is reserved */
-    RES_20 = 0x20,
+    R6_BEQZC = 0x20,
+    JIC = 0x20,
     POOL16F = 0x21,
     SB16 = 0x22,
     BEQZ16 = 0x23,
+    BEQZC16 = 0x23,
     SLTI32 = 0x24,
     BEQ32 = 0x25,
+    BC = 0x25,
     SWC132 = 0x26,
     LWC132 = 0x27,
 
-    /* 0x28 and 0x29 are reserved */
-    RES_28 = 0x28,
+    /* 0x29 is reserved */
     RES_29 = 0x29,
+    R6_BNEZC = 0x28,
+    JIALC = 0x28,
     SH16 = 0x2a,
     BNEZ16 = 0x2b,
+    BNEZC16 = 0x2b,
     SLTIU32 = 0x2c,
     BNE32 = 0x2d,
+    BALC = 0x2d,
     SDC132 = 0x2e,
     LDC132 = 0x2f,
 
-    /* 0x30 and 0x31 are reserved */
-    RES_30 = 0x30,
+    /* 0x31 is reserved */
     RES_31 = 0x31,
+    BLEZALC = 0x30,
+    BGEZALC = 0x30,
+    BGEUC = 0x30,
     SWSP16 = 0x32,
     B16 = 0x33,
+    BC16 = 0x33,
     ANDI32 = 0x34,
     J32 = 0x35,
+    BGTZC = 0x35,
+    BLTZC = 0x35,
+    BLTC = 0x35,
     SD32 = 0x36, /* MIPS64 */
     LD32 = 0x37, /* MIPS64 */
 
-    /* 0x38 and 0x39 are reserved */
-    RES_38 = 0x38,
+    /* 0x39 is reserved */
     RES_39 = 0x39,
+    BGTZALC = 0x38,
+    BLTZALC = 0x38,
+    BLTUC = 0x38,
     SW16 = 0x3a,
     LI16 = 0x3b,
     JALX32 = 0x3c,
     JAL32 = 0x3d,
+    BLEZC = 0x3d,
+    BGEZC = 0x3d,
+    BGEC = 0x3d,
     SW32 = 0x3e,
     LW32 = 0x3f
 };
 
+/* PCREL Instructions perform PC-Relative address calculation. bits 20..16 */
+enum {
+    ADDIUPC_00 = 0x00,
+    ADDIUPC_07 = 0x07,
+    AUIPC = 0x1e,
+    ALUIPC = 0x1f,
+    LWPC_08 = 0x08,
+    LWPC_0F = 0x0F,
+};
+
 /* POOL32A encoding of minor opcode field */
 
 enum {
@@ -12569,6 +13012,9 @@
     SRL32 = 0x1,
     SRA = 0x2,
     ROTR = 0x3,
+    SELEQZ = 0x5,
+    SELNEZ = 0x6,
+    R6_RDHWR = 0x7,
 
     SLLV = 0x0,
     SRLV = 0x1,
@@ -12587,13 +13033,25 @@
     SLTU = 0xe,
 
     MOVN = 0x0,
+    R6_MUL  = 0x0,
     MOVZ = 0x1,
+    MUH  = 0x1,
+    MULU = 0x2,
+    MUHU = 0x3,
     LWXS = 0x4,
+    R6_DIV  = 0x4,
+    MOD  = 0x5,
+    R6_DIVU = 0x6,
+    MODU = 0x7,
 
     /* The following can be distinguished by their lower 6 bits. */
+    BREAK32 = 0x07,
     INS = 0x0c,
+    LSA = 0x0f,
+    ALIGN = 0x1f,
     EXT = 0x2c,
-    POOL32AXF = 0x3c
+    POOL32AXF = 0x3c,
+    SIGRIE = 0x3f
 };
 
 /* POOL32AXF encoding of minor opcode field extension */
@@ -12644,6 +13102,7 @@
     /* end of microMIPS32 DSP */
 
     /* bits 15..12 for 0x2c */
+    BITSWAP = 0x0,
     SEB = 0x2,
     SEH = 0x3,
     CLO = 0x4,
@@ -12670,7 +13129,10 @@
     /* bits 15..12 for 0x3c */
     JALR = 0x0,
     JR = 0x0,                   /* alias */
+    JALRC = 0x0,
+    JRC = 0x0,
     JALR_HB = 0x1,
+    JALRC_HB = 0x1,
     JALRS = 0x4,
     JALRS_HB = 0x5,
 
@@ -12683,6 +13145,8 @@
     TLBR = 0x1,
     TLBWI = 0x2,
     TLBWR = 0x3,
+    TLBINV = 0x4,
+    TLBINVF = 0x5,
     WAIT = 0x9,
     IRET = 0xd,
     DERET = 0xe,
@@ -12752,32 +13216,39 @@
 enum {
     /* These are the bit 7..6 values */
     ADD_FMT = 0x0,
-    MOVN_FMT = 0x0,
 
     SUB_FMT = 0x1,
-    MOVZ_FMT = 0x1,
 
     MUL_FMT = 0x2,
 
     DIV_FMT = 0x3,
 
     /* These are the bit 8..6 values */
+    MOVN_FMT = 0x0,
     RSQRT2_FMT = 0x0,
     MOVF_FMT = 0x0,
+    RINT_FMT = 0x0,
+    SELNEZ_FMT = 0x0,
 
+    MOVZ_FMT = 0x1,
     LWXC1 = 0x1,
     MOVT_FMT = 0x1,
+    CLASS_FMT = 0x1,
+    SELEQZ_FMT = 0x1,
 
     PLL_PS = 0x2,
     SWXC1 = 0x2,
+    SEL_FMT = 0x2,
 
     PLU_PS = 0x3,
     LDXC1 = 0x3,
 
+    MOVN_FMT_04 = 0x4,
     PUL_PS = 0x4,
     SDXC1 = 0x4,
     RECIP2_FMT = 0x4,
 
+    MOVZ_FMT_05 = 0x05,
     PUU_PS = 0x5,
     LUXC1 = 0x5,
 
@@ -12785,8 +13256,10 @@
     SUXC1 = 0x6,
     ADDR_PS = 0x6,
     PREFX = 0x6,
+    MADDF_FMT = 0x6,
 
     MULR_PS = 0x7,
+    MSUBF_FMT = 0x7,
 
     MADD_S = 0x01,
     MADD_D = 0x09,
@@ -12803,10 +13276,17 @@
     NMSUB_D = 0x2a,
     NMSUB_PS = 0x32,
 
+    MIN_FMT = 0x3,
+    MAX_FMT = 0xb,
+    MINA_FMT = 0x23,
+    MAXA_FMT = 0x2b,
     POOL32FXF = 0x3b,
 
     CABS_COND_FMT = 0x1c,              /* MIPS3D */
-    C_COND_FMT = 0x3c
+    C_COND_FMT = 0x3c,
+
+    CMP_CONDN_S = 0x5,
+    CMP_CONDN_D = 0x15
 };
 
 /* POOL32Fxf encoding of minor opcode extension field */
@@ -12859,10 +13339,15 @@
     BGTZ = 0x06,
     BEQZC = 0x07,
     TLTI = 0x08,
+    BC1EQZC = 0x08,
     TGEI = 0x09,
+    BC1NEZC = 0x09,
     TLTIU = 0x0a,
+    BC2EQZC = 0x0a,
     TGEIU = 0x0b,
+    BC2NEZC = 0x0a,
     TNEI = 0x0c,
+    R6_SYNCI = 0x0c,
     LUI = 0x0d,
     TEQI = 0x0e,
     SYNCI = 0x10,
@@ -12915,6 +13400,26 @@
     JRADDIUSP = 0x30
 };
 
+/* R6 POOL16C encoding of minor opcode field (bits 0..5) */
+
+enum {
+    R6_NOT16    = 0x00,
+    R6_AND16    = 0x01,
+    R6_LWM16    = 0x02,
+    R6_JRC16    = 0x03,
+    MOVEP       = 0x04,
+    MOVEP_07    = 0x07,
+    R6_XOR16    = 0x08,
+    R6_OR16     = 0x09,
+    R6_SWM16    = 0x0a,
+    JALRC16     = 0x0b,
+    MOVEP_0C    = 0x0c,
+    MOVEP_0F    = 0x0f,
+    JRCADDIUSP  = 0x13,
+    R6_BREAK16  = 0x1b,
+    R6_SDBBP16  = 0x3b
+};
+
 /* POOL16D encoding of minor opcode field */
 
 enum {
@@ -13015,12 +13520,11 @@
 static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
                                int base, int16_t offset)
 {
-    const char *opn = "ldst_multiple";
     TCGv t0, t1;
     TCGv_i32 t2;
 
     if (ctx->hflags & MIPS_HFLAG_BMASK) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -13035,25 +13539,19 @@
     switch (opc) {
     case LWM32:
         gen_helper_lwm(cpu_env, t0, t1, t2);
-        opn = "lwm";
         break;
     case SWM32:
         gen_helper_swm(cpu_env, t0, t1, t2);
-        opn = "swm";
         break;
 #ifdef TARGET_MIPS64
     case LDM:
         gen_helper_ldm(cpu_env, t0, t1, t2);
-        opn = "ldm";
         break;
     case SDM:
         gen_helper_sdm(cpu_env, t0, t1, t2);
-        opn = "sdm";
         break;
 #endif
     }
-    (void)opn;
-    MIPS_DEBUG("%s, %x, %d(%s)", opn, reglist, offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
     tcg_temp_free_i32(t2);
@@ -13150,17 +13648,17 @@
         gen_HILO(ctx, OPC_MFLO, 0, uMIPS_RS5(ctx->opcode));
         break;
     case BREAK16:
-        generate_exception(ctx, EXCP_BREAK);
+        generate_exception_end(ctx, EXCP_BREAK);
         break;
     case SDBBP16:
-        /* XXX: not clear which exception should be raised
-         *      when in debug mode...
-         */
-        check_insn(ctx, ISA_MIPS32);
-        if (!(ctx->hflags & MIPS_HFLAG_DM)) {
-            generate_exception(ctx, EXCP_DBp);
+        if (is_uhi(extract32(ctx->opcode, 0, 4))) {
+            gen_helper_do_semihosting(cpu_env);
         } else {
-            generate_exception(ctx, EXCP_DBp);
+            /* XXX: not clear which exception should be raised
+             *      when in debug mode...
+             */
+            check_insn(ctx, ISA_MIPS32);
+            generate_exception_end(ctx, EXCP_DBp);
         }
         break;
     case JRADDIUSP + 0:
@@ -13174,6 +13672,114 @@
         }
         break;
     default:
+        generate_exception_end(ctx, EXCP_RI);
+        break;
+    }
+}
+
+static inline void gen_movep(DisasContext *ctx, int enc_dest, int enc_rt,
+                             int enc_rs)
+{
+    int rd, rs, re, rt;
+    static const int rd_enc[] = { 5, 5, 6, 4, 4, 4, 4, 4 };
+    static const int re_enc[] = { 6, 7, 7, 21, 22, 5, 6, 7 };
+    static const int rs_rt_enc[] = { 0, 17, 2, 3, 16, 18, 19, 20 };
+    rd = rd_enc[enc_dest];
+    re = re_enc[enc_dest];
+    rs = rs_rt_enc[enc_rs];
+    rt = rs_rt_enc[enc_rt];
+    if (rs) {
+        tcg_gen_mov_tl(cpu_gpr[rd], cpu_gpr[rs]);
+    } else {
+        tcg_gen_movi_tl(cpu_gpr[rd], 0);
+    }
+    if (rt) {
+        tcg_gen_mov_tl(cpu_gpr[re], cpu_gpr[rt]);
+    } else {
+        tcg_gen_movi_tl(cpu_gpr[re], 0);
+    }
+}
+
+static void gen_pool16c_r6_insn(DisasContext *ctx)
+{
+    int rt = mmreg((ctx->opcode >> 7) & 0x7);
+    int rs = mmreg((ctx->opcode >> 4) & 0x7);
+
+    switch (ctx->opcode & 0xf) {
+    case R6_NOT16:
+        gen_logic(ctx, OPC_NOR, rt, rs, 0);
+        break;
+    case R6_AND16:
+        gen_logic(ctx, OPC_AND, rt, rt, rs);
+        break;
+    case R6_LWM16:
+        {
+            int lwm_converted = 0x11 + extract32(ctx->opcode, 8, 2);
+            int offset = extract32(ctx->opcode, 4, 4);
+            gen_ldst_multiple(ctx, LWM32, lwm_converted, 29, offset << 2);
+        }
+        break;
+    case R6_JRC16: /* JRCADDIUSP */
+        if ((ctx->opcode >> 4) & 1) {
+            /* JRCADDIUSP */
+            int imm = extract32(ctx->opcode, 5, 5);
+            gen_compute_branch(ctx, OPC_JR, 2, 31, 0, 0, 0);
+            gen_arith_imm(ctx, OPC_ADDIU, 29, 29, imm << 2);
+        } else {
+            /* JRC16 */
+            int rs = extract32(ctx->opcode, 5, 5);
+            gen_compute_branch(ctx, OPC_JR, 2, rs, 0, 0, 0);
+        }
+        break;
+    case MOVEP ... MOVEP_07:
+    case MOVEP_0C ... MOVEP_0F:
+        {
+            int enc_dest = uMIPS_RD(ctx->opcode);
+            int enc_rt = uMIPS_RS2(ctx->opcode);
+            int enc_rs = (ctx->opcode & 3) | ((ctx->opcode >> 1) & 4);
+            gen_movep(ctx, enc_dest, enc_rt, enc_rs);
+        }
+        break;
+    case R6_XOR16:
+        gen_logic(ctx, OPC_XOR, rt, rt, rs);
+        break;
+    case R6_OR16:
+        gen_logic(ctx, OPC_OR, rt, rt, rs);
+        break;
+    case R6_SWM16:
+        {
+            int swm_converted = 0x11 + extract32(ctx->opcode, 8, 2);
+            int offset = extract32(ctx->opcode, 4, 4);
+            gen_ldst_multiple(ctx, SWM32, swm_converted, 29, offset << 2);
+        }
+        break;
+    case JALRC16: /* BREAK16, SDBBP16 */
+        switch (ctx->opcode & 0x3f) {
+        case JALRC16:
+        case JALRC16 + 0x20:
+            /* JALRC16 */
+            gen_compute_branch(ctx, OPC_JALR, 2, (ctx->opcode >> 5) & 0x1f,
+                               31, 0, 0);
+            break;
+        case R6_BREAK16:
+            /* BREAK16 */
+            generate_exception(ctx, EXCP_BREAK);
+            break;
+        case R6_SDBBP16:
+            /* SDBBP16 */
+            if (is_uhi(extract32(ctx->opcode, 6, 4))) {
+                gen_helper_do_semihosting(cpu_env);
+            } else {
+                if (ctx->hflags & MIPS_HFLAG_SBRI) {
+                    generate_exception(ctx, EXCP_RI);
+                } else {
+                    generate_exception(ctx, EXCP_DBp);
+                }
+            }
+            break;
+        }
+        break;
+    default:
         generate_exception(ctx, EXCP_RI);
         break;
     }
@@ -13202,11 +13808,10 @@
 static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
                            int base, int16_t offset)
 {
-    const char *opn = "ldst_pair";
     TCGv t0, t1;
 
     if (ctx->hflags & MIPS_HFLAG_BMASK || rd == 31) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -13218,7 +13823,7 @@
     switch (opc) {
     case LWP:
         if (rd == base) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return;
         }
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL);
@@ -13227,7 +13832,6 @@
         gen_op_addr_add(ctx, t0, t0, t1);
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t1, rd+1);
-        opn = "lwp";
         break;
     case SWP:
         gen_load_gpr(t1, rd);
@@ -13236,12 +13840,11 @@
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEUL);
-        opn = "swp";
         break;
 #ifdef TARGET_MIPS64
     case LDP:
         if (rd == base) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return;
         }
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TEQ);
@@ -13250,7 +13853,6 @@
         gen_op_addr_add(ctx, t0, t0, t1);
         tcg_gen_qemu_ld_tl(t1, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t1, rd+1);
-        opn = "ldp";
         break;
     case SDP:
         gen_load_gpr(t1, rd);
@@ -13259,12 +13861,9 @@
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
         tcg_gen_qemu_st_tl(t1, t0, ctx->mem_idx, MO_TEQ);
-        opn = "sdp";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s, %s, %d(%s)", opn, regnames[rd], offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
 }
@@ -13350,6 +13949,10 @@
         break;
     case 0x2c:
         switch (minor) {
+        case BITSWAP:
+            check_insn(ctx, ISA_MIPS32R6);
+            gen_bitswap(ctx, OPC_BITSWAP, rs, rt);
+            break;
         case SEB:
             gen_bshfl(ctx, OPC_SEB, rs, rt);
             break;
@@ -13366,21 +13969,26 @@
             gen_cl(ctx, mips32_op, rt, rs);
             break;
         case RDHWR:
-            gen_rdhwr(ctx, rt, rs);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            gen_rdhwr(ctx, rt, rs, 0);
             break;
         case WSBH:
             gen_bshfl(ctx, OPC_WSBH, rs, rt);
             break;
         case MULT:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_MULT;
             goto do_mul;
         case MULTU:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_MULTU;
             goto do_mul;
         case DIV:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_DIV;
             goto do_div;
         case DIVU:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_DIVU;
             goto do_div;
         do_div:
@@ -13388,15 +13996,19 @@
             gen_muldiv(ctx, mips32_op, 0, rs, rt);
             break;
         case MADD:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_MADD;
             goto do_mul;
         case MADDU:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_MADDU;
             goto do_mul;
         case MSUB:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_MSUB;
             goto do_mul;
         case MSUBU:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_MSUBU;
         do_mul:
             check_insn(ctx, ISA_MIPS32);
@@ -13422,13 +14034,20 @@
         break;
     case 0x3c:
         switch (minor) {
-        case JALR:
-        case JALR_HB:
-            gen_compute_branch(ctx, OPC_JALR, 4, rs, rt, 0, 4);
-            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+        case JALR:    /* JALRC */
+        case JALR_HB: /* JALRC_HB */
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                /* JALRC, JALRC_HB */
+                gen_compute_branch(ctx, OPC_JALR, 4, rs, rt, 0, 0);
+            } else {
+                /* JALR, JALR_HB */
+                gen_compute_branch(ctx, OPC_JALR, 4, rs, rt, 0, 4);
+                ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+            }
             break;
         case JALRS:
         case JALRS_HB:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_JALR, 4, rs, rt, 0, 2);
             ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
@@ -13441,12 +14060,12 @@
         case RDPGPR:
             check_cp0_enabled(ctx);
             check_insn(ctx, ISA_MIPS32R2);
-            gen_load_srsgpr(rt, rs);
+            gen_load_srsgpr(rs, rt);
             break;
         case WRPGPR:
             check_cp0_enabled(ctx);
             check_insn(ctx, ISA_MIPS32R2);
-            gen_store_srsgpr(rt, rs);
+            gen_store_srsgpr(rs, rt);
             break;
         default:
             goto pool32axf_invalid;
@@ -13467,6 +14086,12 @@
         case TLBWR:
             mips32_op = OPC_TLBWR;
             goto do_cp0;
+        case TLBINV:
+            mips32_op = OPC_TLBINV;
+            goto do_cp0;
+        case TLBINVF:
+            mips32_op = OPC_TLBINVF;
+            goto do_cp0;
         case WAIT:
             mips32_op = OPC_WAIT;
             goto do_cp0;
@@ -13521,15 +14146,18 @@
             /* NOP */
             break;
         case SYSCALL:
-            generate_exception(ctx, EXCP_SYSCALL);
-            ctx->bstate = BS_STOP;
+            generate_exception_end(ctx, EXCP_SYSCALL);
             break;
         case SDBBP:
-            check_insn(ctx, ISA_MIPS32);
-            if (!(ctx->hflags & MIPS_HFLAG_DM)) {
-                generate_exception(ctx, EXCP_DBp);
+            if (is_uhi(extract32(ctx->opcode, 16, 10))) {
+                gen_helper_do_semihosting(cpu_env);
             } else {
-                generate_exception(ctx, EXCP_DBp);
+                check_insn(ctx, ISA_MIPS32);
+                if (ctx->hflags & MIPS_HFLAG_SBRI) {
+                    generate_exception_end(ctx, EXCP_RI);
+                } else {
+                    generate_exception_end(ctx, EXCP_DBp);
+                }
             }
             break;
         default:
@@ -13555,6 +14183,7 @@
         }
         break;
     case 0x35:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         switch (minor) {
         case MFHI32:
             gen_HILO(ctx, OPC_MFHI, 0, rs);
@@ -13575,7 +14204,7 @@
     default:
     pool32axf_invalid:
         MIPS_INVAL("pool32axf");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -13827,6 +14456,7 @@
     case COND_FLOAT_MOV(MOVT, 5):
     case COND_FLOAT_MOV(MOVT, 6):
     case COND_FLOAT_MOV(MOVT, 7):
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movci(ctx, rt, rs, (ctx->opcode >> 13) & 0x7, 1);
         break;
     case COND_FLOAT_MOV(MOVF, 0):
@@ -13837,17 +14467,17 @@
     case COND_FLOAT_MOV(MOVF, 5):
     case COND_FLOAT_MOV(MOVF, 6):
     case COND_FLOAT_MOV(MOVF, 7):
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         gen_movci(ctx, rt, rs, (ctx->opcode >> 13) & 0x7, 0);
         break;
     default:
         MIPS_INVAL("pool32fxf");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
 
-static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
-                                    uint16_t insn_hw1)
+static void decode_micromips32_opc(CPUMIPSState *env, DisasContext *ctx)
 {
     int32_t offset;
     uint16_t insn;
@@ -13887,6 +14517,18 @@
             do_shifti:
                 gen_shift_imm(ctx, mips32_op, rt, rs, rd);
                 break;
+            case SELEQZ:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_cond_move(ctx, OPC_SELEQZ, rd, rs, rt);
+                break;
+            case SELNEZ:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_cond_move(ctx, OPC_SELNEZ, rd, rs, rt);
+                break;
+            case R6_RDHWR:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_rdhwr(ctx, rt, rs, extract32(ctx->opcode, 11, 3));
+                break;
             default:
                 goto pool32a_invalid;
             }
@@ -13908,6 +14550,7 @@
                 mips32_op = OPC_SUBU;
                 goto do_arith;
             case MUL:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MUL;
             do_arith:
                 gen_arith(ctx, mips32_op, rd, rs, rt);
@@ -13959,16 +14602,52 @@
             minor = (ctx->opcode >> 6) & 0xf;
             switch (minor) {
                 /* Conditional moves */
-            case MOVN:
-                mips32_op = OPC_MOVN;
-                goto do_cmov;
-            case MOVZ:
-                mips32_op = OPC_MOVZ;
-            do_cmov:
-                gen_cond_move(ctx, mips32_op, rd, rs, rt);
+            case MOVN: /* MUL */
+                if (ctx->insn_flags & ISA_MIPS32R6) {
+                    /* MUL */
+                    gen_r6_muldiv(ctx, R6_OPC_MUL, rd, rs, rt);
+                } else {
+                    /* MOVN */
+                    gen_cond_move(ctx, OPC_MOVN, rd, rs, rt);
+                }
                 break;
-            case LWXS:
-                gen_ldxs(ctx, rs, rt, rd);
+            case MOVZ: /* MUH */
+                if (ctx->insn_flags & ISA_MIPS32R6) {
+                    /* MUH */
+                    gen_r6_muldiv(ctx, R6_OPC_MUH, rd, rs, rt);
+                } else {
+                    /* MOVZ */
+                    gen_cond_move(ctx, OPC_MOVZ, rd, rs, rt);
+                }
+                break;
+            case MULU:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_muldiv(ctx, R6_OPC_MULU, rd, rs, rt);
+                break;
+            case MUHU:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_muldiv(ctx, R6_OPC_MUHU, rd, rs, rt);
+                break;
+            case LWXS: /* DIV */
+                if (ctx->insn_flags & ISA_MIPS32R6) {
+                    /* DIV */
+                    gen_r6_muldiv(ctx, R6_OPC_DIV, rd, rs, rt);
+                } else {
+                    /* LWXS */
+                    gen_ldxs(ctx, rs, rt, rd);
+                }
+                break;
+            case MOD:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_muldiv(ctx, R6_OPC_MOD, rd, rs, rt);
+                break;
+            case R6_DIVU:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_muldiv(ctx, R6_OPC_DIVU, rd, rs, rt);
+                break;
+            case MODU:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_muldiv(ctx, R6_OPC_MODU, rd, rs, rt);
                 break;
             default:
                 goto pool32a_invalid;
@@ -13977,19 +14656,33 @@
         case INS:
             gen_bitops(ctx, OPC_INS, rt, rs, rr, rd);
             return;
+        case LSA:
+            check_insn(ctx, ISA_MIPS32R6);
+            gen_lsa(ctx, OPC_LSA, rd, rs, rt,
+                    extract32(ctx->opcode, 9, 2));
+            break;
+        case ALIGN:
+            check_insn(ctx, ISA_MIPS32R6);
+            gen_align(ctx, OPC_ALIGN, rd, rs, rt,
+                      extract32(ctx->opcode, 9, 2));
+            break;
         case EXT:
             gen_bitops(ctx, OPC_EXT, rt, rs, rr, rd);
             return;
         case POOL32AXF:
             gen_pool32axf(env, ctx, rt, rs);
             break;
-        case 0x07:
-            generate_exception(ctx, EXCP_BREAK);
+        case BREAK32:
+            generate_exception_end(ctx, EXCP_BREAK);
+            break;
+        case SIGRIE:
+            check_insn(ctx, ISA_MIPS32R6);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         default:
         pool32a_invalid:
                 MIPS_INVAL("pool32a");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
         }
         break;
@@ -13998,32 +14691,40 @@
         switch (minor) {
         case CACHE:
             check_cp0_enabled(ctx);
-            /* Treat as no-op. */
+            if (ctx->hflags & MIPS_HFLAG_ITC_CACHE) {
+                gen_cache_operation(ctx, rt, rs, imm);
+            }
             break;
         case LWC2:
         case SWC2:
             /* COP2: Not implemented. */
             generate_exception_err(ctx, EXCP_CpU, 2);
             break;
-        case LWP:
-        case SWP:
 #ifdef TARGET_MIPS64
         case LDP:
         case SDP:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            /* Fallthrough */
 #endif
+        case LWP:
+        case SWP:
             gen_ldst_pair(ctx, minor, rt, rs, SIMM(ctx->opcode, 0, 12));
             break;
-        case LWM32:
-        case SWM32:
 #ifdef TARGET_MIPS64
         case LDM:
         case SDM:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            /* Fallthrough */
 #endif
+        case LWM32:
+        case SWM32:
             gen_ldst_multiple(ctx, minor, rt, rs, SIMM(ctx->opcode, 0, 12));
             break;
         default:
             MIPS_INVAL("pool32b");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -14033,47 +14734,61 @@
             check_cp1_enabled(ctx);
             switch (minor) {
             case ALNV_PS:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_ALNV_PS;
                 goto do_madd;
             case MADD_S:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MADD_S;
                 goto do_madd;
             case MADD_D:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MADD_D;
                 goto do_madd;
             case MADD_PS:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MADD_PS;
                 goto do_madd;
             case MSUB_S:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MSUB_S;
                 goto do_madd;
             case MSUB_D:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MSUB_D;
                 goto do_madd;
             case MSUB_PS:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_MSUB_PS;
                 goto do_madd;
             case NMADD_S:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_NMADD_S;
                 goto do_madd;
             case NMADD_D:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_NMADD_D;
                 goto do_madd;
             case NMADD_PS:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_NMADD_PS;
                 goto do_madd;
             case NMSUB_S:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_NMSUB_S;
                 goto do_madd;
             case NMSUB_D:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_NMSUB_D;
                 goto do_madd;
             case NMSUB_PS:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 mips32_op = OPC_NMSUB_PS;
             do_madd:
                 gen_flt3_arith(ctx, mips32_op, rd, rr, rs, rt);
                 break;
             case CABS_COND_FMT:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 cond = (ctx->opcode >> 6) & 0xf;
                 cc = (ctx->opcode >> 13) & 0x7;
                 fmt = (ctx->opcode >> 10) & 0x3;
@@ -14092,6 +14807,7 @@
                 }
                 break;
             case C_COND_FMT:
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 cond = (ctx->opcode >> 6) & 0xf;
                 cc = (ctx->opcode >> 13) & 0x7;
                 fmt = (ctx->opcode >> 10) & 0x3;
@@ -14109,6 +14825,14 @@
                     goto pool32f_invalid;
                 }
                 break;
+            case CMP_CONDN_S:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_cmp_s(ctx, (ctx->opcode >> 6) & 0x1f, rt, rs, rd);
+                break;
+            case CMP_CONDN_D:
+                check_insn(ctx, ISA_MIPS32R6);
+                gen_r6_cmp_d(ctx, (ctx->opcode >> 6) & 0x1f, rt, rs, rd);
+                break;
             case POOL32FXF:
                 gen_pool32fxf(ctx, rt, rs);
                 break;
@@ -14128,6 +14852,7 @@
                     mips32_op = OPC_PUU_PS;
                     goto do_ps;
                 case CVT_PS_S:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_CVT_PS_S;
                 do_ps:
                     gen_farith(ctx, mips32_op, rt, rs, rd, 0);
@@ -14136,25 +14861,44 @@
                     goto pool32f_invalid;
                 }
                 break;
+            case MIN_FMT:
+                check_insn(ctx, ISA_MIPS32R6);
+                switch ((ctx->opcode >> 9) & 0x3) {
+                case FMT_SDPS_S:
+                    gen_farith(ctx, OPC_MIN_S, rt, rs, rd, 0);
+                    break;
+                case FMT_SDPS_D:
+                    gen_farith(ctx, OPC_MIN_D, rt, rs, rd, 0);
+                    break;
+                default:
+                    goto pool32f_invalid;
+                }
+                break;
             case 0x08:
                 /* [LS][WDU]XC1 */
                 switch ((ctx->opcode >> 6) & 0x7) {
                 case LWXC1:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_LWXC1;
                     goto do_ldst_cp1;
                 case SWXC1:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_SWXC1;
                     goto do_ldst_cp1;
                 case LDXC1:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_LDXC1;
                     goto do_ldst_cp1;
                 case SDXC1:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_SDXC1;
                     goto do_ldst_cp1;
                 case LUXC1:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_LUXC1;
                     goto do_ldst_cp1;
                 case SUXC1:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     mips32_op = OPC_SUXC1;
                 do_ldst_cp1:
                     gen_flt3_ldst(ctx, mips32_op, rd, rd, rt, rs);
@@ -14163,8 +14907,22 @@
                     goto pool32f_invalid;
                 }
                 break;
+            case MAX_FMT:
+                check_insn(ctx, ISA_MIPS32R6);
+                switch ((ctx->opcode >> 9) & 0x3) {
+                case FMT_SDPS_S:
+                    gen_farith(ctx, OPC_MAX_S, rt, rs, rd, 0);
+                    break;
+                case FMT_SDPS_D:
+                    gen_farith(ctx, OPC_MAX_D, rt, rs, rd, 0);
+                    break;
+                default:
+                    goto pool32f_invalid;
+                }
+                break;
             case 0x18:
                 /* 3D insns */
+                check_insn_opc_removed(ctx, ISA_MIPS32R6);
                 fmt = (ctx->opcode >> 9) & 0x3;
                 switch ((ctx->opcode >> 6) & 0x7) {
                 case RSQRT2_FMT:
@@ -14210,41 +14968,74 @@
                 }
                 break;
             case 0x20:
-                /* MOV[FT].fmt and PREFX */
+                /* MOV[FT].fmt, PREFX, RINT.fmt, CLASS.fmt*/
                 cc = (ctx->opcode >> 13) & 0x7;
                 fmt = (ctx->opcode >> 9) & 0x3;
                 switch ((ctx->opcode >> 6) & 0x7) {
-                case MOVF_FMT:
-                    switch (fmt) {
-                    case FMT_SDPS_S:
-                        gen_movcf_s(ctx, rs, rt, cc, 0);
-                        break;
-                    case FMT_SDPS_D:
-                        gen_movcf_d(ctx, rs, rt, cc, 0);
-                        break;
-                    case FMT_SDPS_PS:
-                        gen_movcf_ps(ctx, rs, rt, cc, 0);
-                        break;
-                    default:
-                        goto pool32f_invalid;
+                case MOVF_FMT: /* RINT_FMT */
+                    if (ctx->insn_flags & ISA_MIPS32R6) {
+                        /* RINT_FMT */
+                        switch (fmt) {
+                        case FMT_SDPS_S:
+                            gen_farith(ctx, OPC_RINT_S, 0, rt, rs, 0);
+                            break;
+                        case FMT_SDPS_D:
+                            gen_farith(ctx, OPC_RINT_D, 0, rt, rs, 0);
+                            break;
+                        default:
+                            goto pool32f_invalid;
+                        }
+                    } else {
+                        /* MOVF_FMT */
+                        switch (fmt) {
+                        case FMT_SDPS_S:
+                            gen_movcf_s(ctx, rs, rt, cc, 0);
+                            break;
+                        case FMT_SDPS_D:
+                            gen_movcf_d(ctx, rs, rt, cc, 0);
+                            break;
+                        case FMT_SDPS_PS:
+                            check_ps(ctx);
+                            gen_movcf_ps(ctx, rs, rt, cc, 0);
+                            break;
+                        default:
+                            goto pool32f_invalid;
+                        }
                     }
                     break;
-                case MOVT_FMT:
-                    switch (fmt) {
-                    case FMT_SDPS_S:
-                        gen_movcf_s(ctx, rs, rt, cc, 1);
-                        break;
-                    case FMT_SDPS_D:
-                        gen_movcf_d(ctx, rs, rt, cc, 1);
-                        break;
-                    case FMT_SDPS_PS:
-                        gen_movcf_ps(ctx, rs, rt, cc, 1);
-                        break;
-                    default:
-                        goto pool32f_invalid;
+                case MOVT_FMT: /* CLASS_FMT */
+                    if (ctx->insn_flags & ISA_MIPS32R6) {
+                        /* CLASS_FMT */
+                        switch (fmt) {
+                        case FMT_SDPS_S:
+                            gen_farith(ctx, OPC_CLASS_S, 0, rt, rs, 0);
+                            break;
+                        case FMT_SDPS_D:
+                            gen_farith(ctx, OPC_CLASS_D, 0, rt, rs, 0);
+                            break;
+                        default:
+                            goto pool32f_invalid;
+                        }
+                    } else {
+                        /* MOVT_FMT */
+                        switch (fmt) {
+                        case FMT_SDPS_S:
+                            gen_movcf_s(ctx, rs, rt, cc, 1);
+                            break;
+                        case FMT_SDPS_D:
+                            gen_movcf_d(ctx, rs, rt, cc, 1);
+                            break;
+                        case FMT_SDPS_PS:
+                            check_ps(ctx);
+                            gen_movcf_ps(ctx, rs, rt, cc, 1);
+                            break;
+                        default:
+                            goto pool32f_invalid;
+                        }
                     }
                     break;
                 case PREFX:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     break;
                 default:
                     goto pool32f_invalid;
@@ -14259,11 +15050,38 @@
                     mips32_op = OPC_##prfx##_D;         \
                     goto do_fpop;                       \
                 case FMT_SDPS_PS:                       \
+                    check_ps(ctx);                      \
                     mips32_op = OPC_##prfx##_PS;        \
                     goto do_fpop;                       \
                 default:                                \
                     goto pool32f_invalid;               \
                 }
+            case MINA_FMT:
+                check_insn(ctx, ISA_MIPS32R6);
+                switch ((ctx->opcode >> 9) & 0x3) {
+                case FMT_SDPS_S:
+                    gen_farith(ctx, OPC_MINA_S, rt, rs, rd, 0);
+                    break;
+                case FMT_SDPS_D:
+                    gen_farith(ctx, OPC_MINA_D, rt, rs, rd, 0);
+                    break;
+                default:
+                    goto pool32f_invalid;
+                }
+                break;
+            case MAXA_FMT:
+                check_insn(ctx, ISA_MIPS32R6);
+                switch ((ctx->opcode >> 9) & 0x3) {
+                case FMT_SDPS_S:
+                    gen_farith(ctx, OPC_MAXA_S, rt, rs, rd, 0);
+                    break;
+                case FMT_SDPS_D:
+                    gen_farith(ctx, OPC_MAXA_D, rt, rs, rd, 0);
+                    break;
+                default:
+                    goto pool32f_invalid;
+                }
+                break;
             case 0x30:
                 /* regular FP ops */
                 switch ((ctx->opcode >> 6) & 0x3) {
@@ -14292,13 +15110,90 @@
                 break;
             case 0x38:
                 /* cmovs */
-                switch ((ctx->opcode >> 6) & 0x3) {
-                case MOVN_FMT:
+                switch ((ctx->opcode >> 6) & 0x7) {
+                case MOVN_FMT: /* SELNEZ_FMT */
+                    if (ctx->insn_flags & ISA_MIPS32R6) {
+                        /* SELNEZ_FMT */
+                        switch ((ctx->opcode >> 9) & 0x3) {
+                        case FMT_SDPS_S:
+                            gen_sel_s(ctx, OPC_SELNEZ_S, rd, rt, rs);
+                            break;
+                        case FMT_SDPS_D:
+                            gen_sel_d(ctx, OPC_SELNEZ_D, rd, rt, rs);
+                            break;
+                        default:
+                            goto pool32f_invalid;
+                        }
+                    } else {
+                        /* MOVN_FMT */
+                        FINSN_3ARG_SDPS(MOVN);
+                    }
+                    break;
+                case MOVN_FMT_04:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     FINSN_3ARG_SDPS(MOVN);
                     break;
-                case MOVZ_FMT:
+                case MOVZ_FMT: /* SELEQZ_FMT */
+                    if (ctx->insn_flags & ISA_MIPS32R6) {
+                        /* SELEQZ_FMT */
+                        switch ((ctx->opcode >> 9) & 0x3) {
+                        case FMT_SDPS_S:
+                            gen_sel_s(ctx, OPC_SELEQZ_S, rd, rt, rs);
+                            break;
+                        case FMT_SDPS_D:
+                            gen_sel_d(ctx, OPC_SELEQZ_D, rd, rt, rs);
+                            break;
+                        default:
+                            goto pool32f_invalid;
+                        }
+                    } else {
+                        /* MOVZ_FMT */
+                        FINSN_3ARG_SDPS(MOVZ);
+                    }
+                    break;
+                case MOVZ_FMT_05:
+                    check_insn_opc_removed(ctx, ISA_MIPS32R6);
                     FINSN_3ARG_SDPS(MOVZ);
                     break;
+                case SEL_FMT:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    switch ((ctx->opcode >> 9) & 0x3) {
+                    case FMT_SDPS_S:
+                        gen_sel_s(ctx, OPC_SEL_S, rd, rt, rs);
+                        break;
+                    case FMT_SDPS_D:
+                        gen_sel_d(ctx, OPC_SEL_D, rd, rt, rs);
+                        break;
+                    default:
+                        goto pool32f_invalid;
+                    }
+                    break;
+                case MADDF_FMT:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    switch ((ctx->opcode >> 9) & 0x3) {
+                    case FMT_SDPS_S:
+                        mips32_op = OPC_MADDF_S;
+                        goto do_fpop;
+                    case FMT_SDPS_D:
+                        mips32_op = OPC_MADDF_D;
+                        goto do_fpop;
+                    default:
+                        goto pool32f_invalid;
+                    }
+                    break;
+                case MSUBF_FMT:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    switch ((ctx->opcode >> 9) & 0x3) {
+                    case FMT_SDPS_S:
+                        mips32_op = OPC_MSUBF_S;
+                        goto do_fpop;
+                    case FMT_SDPS_D:
+                        mips32_op = OPC_MSUBF_D;
+                        goto do_fpop;
+                    default:
+                        goto pool32f_invalid;
+                    }
+                    break;
                 default:
                     goto pool32f_invalid;
                 }
@@ -14309,7 +15204,7 @@
             default:
             pool32f_invalid:
                 MIPS_INVAL("pool32f");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else {
@@ -14320,51 +15215,87 @@
         minor = (ctx->opcode >> 21) & 0x1f;
         switch (minor) {
         case BLTZ:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BLTZ, 4, rs, -1, imm << 1, 4);
             break;
         case BLTZAL:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BLTZAL, 4, rs, -1, imm << 1, 4);
             ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
         case BLTZALS:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BLTZAL, 4, rs, -1, imm << 1, 2);
             ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
         case BGEZ:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BGEZ, 4, rs, -1, imm << 1, 4);
             break;
         case BGEZAL:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BGEZAL, 4, rs, -1, imm << 1, 4);
             ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
         case BGEZALS:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BGEZAL, 4, rs, -1, imm << 1, 2);
             ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
             break;
         case BLEZ:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BLEZ, 4, rs, -1, imm << 1, 4);
             break;
         case BGTZ:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, OPC_BGTZ, 4, rs, -1, imm << 1, 4);
             break;
 
             /* Traps */
-        case TLTI:
-            mips32_op = OPC_TLTI;
-            goto do_trapi;
-        case TGEI:
-            mips32_op = OPC_TGEI;
-            goto do_trapi;
+        case TLTI: /* BC1EQZC */
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                /* BC1EQZC */
+                check_cp1_enabled(ctx);
+                gen_compute_branch1_r6(ctx, OPC_BC1EQZ, rs, imm << 1, 0);
+            } else {
+                /* TLTI */
+                mips32_op = OPC_TLTI;
+                goto do_trapi;
+            }
+            break;
+        case TGEI: /* BC1NEZC */
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                /* BC1NEZC */
+                check_cp1_enabled(ctx);
+                gen_compute_branch1_r6(ctx, OPC_BC1NEZ, rs, imm << 1, 0);
+            } else {
+                /* TGEI */
+                mips32_op = OPC_TGEI;
+                goto do_trapi;
+            }
+            break;
         case TLTIU:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_TLTIU;
             goto do_trapi;
         case TGEIU:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_TGEIU;
             goto do_trapi;
-        case TNEI:
-            mips32_op = OPC_TNEI;
-            goto do_trapi;
+        case TNEI: /* SYNCI */
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                /* SYNCI */
+                /* Break the TB to be able to sync copied instructions
+                   immediately */
+                ctx->bstate = BS_STOP;
+            } else {
+                /* TNEI */
+                mips32_op = OPC_TNEI;
+                goto do_trapi;
+            }
+            break;
         case TEQI:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_TEQI;
         do_trapi:
             gen_trap(ctx, mips32_op, rs, -1, imm);
@@ -14372,6 +15303,7 @@
 
         case BNEZC:
         case BEQZC:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_compute_branch(ctx, minor == BNEZC ? OPC_BNE : OPC_BEQ,
                                4, rs, 0, imm << 1, 0);
             /* Compact branches don't have a delay slot, so just let
@@ -14379,28 +15311,35 @@
                target. */
             break;
         case LUI:
-            gen_logic_imm(ctx, OPC_LUI, rs, -1, imm);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            gen_logic_imm(ctx, OPC_LUI, rs, 0, imm);
             break;
         case SYNCI:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             /* Break the TB to be able to sync copied instructions
                immediately */
             ctx->bstate = BS_STOP;
             break;
         case BC2F:
         case BC2T:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             /* COP2: Not implemented. */
             generate_exception_err(ctx, EXCP_CpU, 2);
             break;
         case BC1F:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = (ctx->opcode & (1 << 16)) ? OPC_BC1FANY2 : OPC_BC1F;
             goto do_cp1branch;
         case BC1T:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = (ctx->opcode & (1 << 16)) ? OPC_BC1TANY2 : OPC_BC1T;
             goto do_cp1branch;
         case BC1ANY4F:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_BC1FANY4;
             goto do_cp1mips3d;
         case BC1ANY4T:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_BC1TANY4;
         do_cp1mips3d:
             check_cop1x(ctx);
@@ -14421,42 +15360,64 @@
             /* Fall through */
         default:
             MIPS_INVAL("pool32i");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
     case POOL32C:
         minor = (ctx->opcode >> 12) & 0xf;
+        offset = sextract32(ctx->opcode, 0,
+                            (ctx->insn_flags & ISA_MIPS32R6) ? 9 : 12);
         switch (minor) {
         case LWL:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_LWL;
             goto do_ld_lr;
         case SWL:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_SWL;
             goto do_st_lr;
         case LWR:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_LWR;
             goto do_ld_lr;
         case SWR:
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_SWR;
             goto do_st_lr;
 #if defined(TARGET_MIPS64)
         case LDL:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_LDL;
             goto do_ld_lr;
         case SDL:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_SDL;
             goto do_st_lr;
         case LDR:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_LDR;
             goto do_ld_lr;
         case SDR:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            check_insn_opc_removed(ctx, ISA_MIPS32R6);
             mips32_op = OPC_SDR;
             goto do_st_lr;
         case LWU:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
             mips32_op = OPC_LWU;
             goto do_ld_lr;
         case LLD:
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
             mips32_op = OPC_LLD;
             goto do_ld_lr;
 #endif
@@ -14464,31 +15425,44 @@
             mips32_op = OPC_LL;
             goto do_ld_lr;
         do_ld_lr:
-            gen_ld(ctx, mips32_op, rt, rs, SIMM(ctx->opcode, 0, 12));
+            gen_ld(ctx, mips32_op, rt, rs, offset);
             break;
         do_st_lr:
             gen_st(ctx, mips32_op, rt, rs, SIMM(ctx->opcode, 0, 12));
             break;
         case SC:
-            gen_st_cond(ctx, OPC_SC, rt, rs, SIMM(ctx->opcode, 0, 12));
+            gen_st_cond(ctx, OPC_SC, rt, rs, offset);
             break;
 #if defined(TARGET_MIPS64)
         case SCD:
-            gen_st_cond(ctx, OPC_SCD, rt, rs, SIMM(ctx->opcode, 0, 12));
+            check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
+            gen_st_cond(ctx, OPC_SCD, rt, rs, offset);
             break;
 #endif
         case PREF:
             /* Treat as no-op */
+            if ((ctx->insn_flags & ISA_MIPS32R6) && (rt >= 24)) {
+                /* hint codes 24-31 are reserved and signal RI */
+                generate_exception(ctx, EXCP_RI);
+            }
             break;
         default:
             MIPS_INVAL("pool32c");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
-    case ADDI32:
-        mips32_op = OPC_ADDI;
-        goto do_addi;
+    case ADDI32: /* AUI, LUI */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* AUI, LUI */
+            gen_logic_imm(ctx, OPC_LUI, rt, rs, imm);
+        } else {
+            /* ADDI32 */
+            mips32_op = OPC_ADDI;
+            goto do_addi;
+        }
+        break;
     case ADDIU32:
         mips32_op = OPC_ADDIU;
     do_addi:
@@ -14518,29 +15492,89 @@
         gen_slt_imm(ctx, mips32_op, rt, rs, imm);
         break;
     case JALX32:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
         gen_compute_branch(ctx, OPC_JALX, 4, rt, rs, offset, 4);
         ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
         break;
-    case JALS32:
-        offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 1;
-        gen_compute_branch(ctx, OPC_JAL, 4, rt, rs, offset, 2);
-        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+    case JALS32: /* BOVC, BEQC, BEQZALC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            if (rs >= rt) {
+                /* BOVC */
+                mips32_op = OPC_BOVC;
+            } else if (rs < rt && rs == 0) {
+                /* BEQZALC */
+                mips32_op = OPC_BEQZALC;
+            } else {
+                /* BEQC */
+                mips32_op = OPC_BEQC;
+            }
+            gen_compute_compact_branch(ctx, mips32_op, rs, rt, imm << 1);
+        } else {
+            /* JALS32 */
+            offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 1;
+            gen_compute_branch(ctx, OPC_JAL, 4, rt, rs, offset, 2);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+        }
         break;
-    case BEQ32:
-        gen_compute_branch(ctx, OPC_BEQ, 4, rt, rs, imm << 1, 4);
+    case BEQ32: /* BC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* BC */
+            gen_compute_compact_branch(ctx, OPC_BC, 0, 0,
+                                       sextract32(ctx->opcode << 1, 0, 27));
+        } else {
+            /* BEQ32 */
+            gen_compute_branch(ctx, OPC_BEQ, 4, rt, rs, imm << 1, 4);
+        }
         break;
-    case BNE32:
-        gen_compute_branch(ctx, OPC_BNE, 4, rt, rs, imm << 1, 4);
+    case BNE32: /* BALC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* BALC */
+            gen_compute_compact_branch(ctx, OPC_BALC, 0, 0,
+                                       sextract32(ctx->opcode << 1, 0, 27));
+        } else {
+            /* BNE32 */
+            gen_compute_branch(ctx, OPC_BNE, 4, rt, rs, imm << 1, 4);
+        }
         break;
-    case J32:
-        gen_compute_branch(ctx, OPC_J, 4, rt, rs,
-                           (int32_t)(ctx->opcode & 0x3FFFFFF) << 1, 4);
+    case J32: /* BGTZC, BLTZC, BLTC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            if (rs == 0 && rt != 0) {
+                /* BGTZC */
+                mips32_op = OPC_BGTZC;
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* BLTZC */
+                mips32_op = OPC_BLTZC;
+            } else {
+                /* BLTC */
+                mips32_op = OPC_BLTC;
+            }
+            gen_compute_compact_branch(ctx, mips32_op, rs, rt, imm << 1);
+        } else {
+            /* J32 */
+            gen_compute_branch(ctx, OPC_J, 4, rt, rs,
+                               (int32_t)(ctx->opcode & 0x3FFFFFF) << 1, 4);
+        }
         break;
-    case JAL32:
-        gen_compute_branch(ctx, OPC_JAL, 4, rt, rs,
-                           (int32_t)(ctx->opcode & 0x3FFFFFF) << 1, 4);
-        ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+    case JAL32: /* BLEZC, BGEZC, BGEC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            if (rs == 0 && rt != 0) {
+                /* BLEZC */
+                mips32_op = OPC_BLEZC;
+            } else if (rs != 0 && rt != 0 && rs == rt) {
+                /* BGEZC */
+                mips32_op = OPC_BGEZC;
+            } else {
+                /* BGEC */
+                mips32_op = OPC_BGEC;
+            }
+            gen_compute_compact_branch(ctx, mips32_op, rs, rt, imm << 1);
+        } else {
+            /* JAL32 */
+            gen_compute_branch(ctx, OPC_JAL, 4, rt, rs,
+                               (int32_t)(ctx->opcode & 0x3FFFFFF) << 1, 4);
+            ctx->hflags |= MIPS_HFLAG_BDS_STRICT;
+        }
         break;
         /* Floating point (COP1) */
     case LWC132:
@@ -14557,14 +15591,98 @@
     do_cop1:
         gen_cop1_ldst(ctx, mips32_op, rt, rs, imm);
         break;
-    case ADDIUPC:
-        {
+    case ADDIUPC: /* PCREL: ADDIUPC, AUIPC, ALUIPC, LWPC */
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            /* PCREL: ADDIUPC, AUIPC, ALUIPC, LWPC */
+            switch ((ctx->opcode >> 16) & 0x1f) {
+            case ADDIUPC_00 ... ADDIUPC_07:
+                gen_pcrel(ctx, OPC_ADDIUPC, ctx->pc & ~0x3, rt);
+                break;
+            case AUIPC:
+                gen_pcrel(ctx, OPC_AUIPC, ctx->pc, rt);
+                break;
+            case ALUIPC:
+                gen_pcrel(ctx, OPC_ALUIPC, ctx->pc, rt);
+                break;
+            case LWPC_08 ... LWPC_0F:
+                gen_pcrel(ctx, R6_OPC_LWPC, ctx->pc & ~0x3, rt);
+                break;
+            default:
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+        } else {
+            /* ADDIUPC */
             int reg = mmreg(ZIMM(ctx->opcode, 23, 3));
             int offset = SIMM(ctx->opcode, 0, 23) << 2;
 
             gen_addiupc(ctx, reg, offset, 0, 0);
         }
         break;
+    case BNVC: /* BNEC, BNEZALC */
+        check_insn(ctx, ISA_MIPS32R6);
+        if (rs >= rt) {
+            /* BNVC */
+            mips32_op = OPC_BNVC;
+        } else if (rs < rt && rs == 0) {
+            /* BNEZALC */
+            mips32_op = OPC_BNEZALC;
+        } else {
+            /* BNEC */
+            mips32_op = OPC_BNEC;
+        }
+        gen_compute_compact_branch(ctx, mips32_op, rs, rt, imm << 1);
+        break;
+    case R6_BNEZC: /* JIALC */
+        check_insn(ctx, ISA_MIPS32R6);
+        if (rt != 0) {
+            /* BNEZC */
+            gen_compute_compact_branch(ctx, OPC_BNEZC, rt, 0,
+                                       sextract32(ctx->opcode << 1, 0, 22));
+        } else {
+            /* JIALC */
+            gen_compute_compact_branch(ctx, OPC_JIALC, 0, rs, imm);
+        }
+        break;
+    case R6_BEQZC: /* JIC */
+        check_insn(ctx, ISA_MIPS32R6);
+        if (rt != 0) {
+            /* BEQZC */
+            gen_compute_compact_branch(ctx, OPC_BEQZC, rt, 0,
+                                       sextract32(ctx->opcode << 1, 0, 22));
+        } else {
+            /* JIC */
+            gen_compute_compact_branch(ctx, OPC_JIC, 0, rs, imm);
+        }
+        break;
+    case BLEZALC: /* BGEZALC, BGEUC */
+        check_insn(ctx, ISA_MIPS32R6);
+        if (rs == 0 && rt != 0) {
+            /* BLEZALC */
+            mips32_op = OPC_BLEZALC;
+        } else if (rs != 0 && rt != 0 && rs == rt) {
+            /* BGEZALC */
+            mips32_op = OPC_BGEZALC;
+        } else {
+            /* BGEUC */
+            mips32_op = OPC_BGEUC;
+        }
+        gen_compute_compact_branch(ctx, mips32_op, rs, rt, imm << 1);
+        break;
+    case BGTZALC: /* BLTZALC, BLTUC */
+        check_insn(ctx, ISA_MIPS32R6);
+        if (rs == 0 && rt != 0) {
+            /* BGTZALC */
+            mips32_op = OPC_BGTZALC;
+        } else if (rs != 0 && rt != 0 && rs == rt) {
+            /* BLTZALC */
+            mips32_op = OPC_BLTZALC;
+        } else {
+            /* BLTUC */
+            mips32_op = OPC_BLTUC;
+        }
+        gen_compute_compact_branch(ctx, mips32_op, rs, rt, imm << 1);
+        break;
         /* Loads and stores */
     case LB32:
         mips32_op = OPC_LB;
@@ -14583,9 +15701,13 @@
         goto do_ld;
 #ifdef TARGET_MIPS64
     case LD32:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
         mips32_op = OPC_LD;
         goto do_ld;
     case SD32:
+        check_insn(ctx, ISA_MIPS3);
+        check_mips_64(ctx);
         mips32_op = OPC_SD;
         goto do_st;
 #endif
@@ -14605,7 +15727,7 @@
         gen_st(ctx, mips32_op, rt, rs, imm);
         break;
     default:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -14617,8 +15739,7 @@
     /* make sure instructions are on a halfword boundary */
     if (ctx->pc & 0x1) {
         env->CP0_BadVAddr = ctx->pc;
-        generate_exception(ctx, EXCP_AdEL);
-        ctx->bstate = BS_STOP;
+        generate_exception_end(ctx, EXCP_AdEL);
         return 2;
     }
 
@@ -14637,9 +15758,7 @@
         case 7:
         /* LB32, LH32, LWC132, LDC132, LW32 */
             if (ctx->hflags & MIPS_HFLAG_BDS16) {
-                generate_exception(ctx, EXCP_RI);
-                /* Just stop translation; the user is confused.  */
-                ctx->bstate = BS_STOP;
+                generate_exception_end(ctx, EXCP_RI);
                 return 2;
             }
             break;
@@ -14650,9 +15769,7 @@
         case 3:
         /* MOVE16, ANDI16, POOL16D, POOL16E, BEQZ16, BNEZ16, B16, LI16 */
             if (ctx->hflags & MIPS_HFLAG_BDS32) {
-                generate_exception(ctx, EXCP_RI);
-                /* Just stop translation; the user is confused.  */
-                ctx->bstate = BS_STOP;
+                generate_exception_end(ctx, EXCP_RI);
                 return 2;
             }
             break;
@@ -14675,8 +15792,14 @@
                 opc = OPC_SUBU;
                 break;
             }
-
-            gen_arith(ctx, opc, rd, rs1, rs2);
+            if (ctx->insn_flags & ISA_MIPS32R6) {
+                /* In the Release 6 the register number location in
+                 * the instruction encoding has changed.
+                 */
+                gen_arith(ctx, opc, rs1, rd, rs2);
+            } else {
+                gen_arith(ctx, opc, rd, rs1, rs2);
+            }
         }
         break;
     case POOL16B:
@@ -14700,7 +15823,11 @@
         }
         break;
     case POOL16C:
-        gen_pool16c_insn(ctx);
+        if (ctx->insn_flags & ISA_MIPS32R6) {
+            gen_pool16c_r6_insn(ctx);
+        } else {
+            gen_pool16c_insn(ctx);
+        }
         break;
     case LWGP16:
         {
@@ -14712,25 +15839,15 @@
         }
         break;
     case POOL16F:
+        check_insn_opc_removed(ctx, ISA_MIPS32R6);
         if (ctx->opcode & 1) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         } else {
             /* MOVEP */
             int enc_dest = uMIPS_RD(ctx->opcode);
             int enc_rt = uMIPS_RS2(ctx->opcode);
             int enc_rs = uMIPS_RS1(ctx->opcode);
-            int rd, rs, re, rt;
-            static const int rd_enc[] = { 5, 5, 6, 4, 4, 4, 4, 4 };
-            static const int re_enc[] = { 6, 7, 7, 21, 22, 5, 6, 7 };
-            static const int rs_rt_enc[] = { 0, 17, 2, 3, 16, 18, 19, 20 };
-
-            rd = rd_enc[enc_dest];
-            re = re_enc[enc_dest];
-            rs = rs_rt_enc[enc_rs];
-            rt = rs_rt_enc[enc_rt];
-
-            gen_arith_imm(ctx, OPC_ADDIU, rd, rs, 0);
-            gen_arith_imm(ctx, OPC_ADDIU, re, rt, 0);
+            gen_movep(ctx, enc_dest, enc_rt, enc_rs);
         }
         break;
     case LBU16:
@@ -14811,7 +15928,7 @@
             int rd = uMIPS_RD5(ctx->opcode);
             int rs = uMIPS_RS5(ctx->opcode);
 
-            gen_arith_imm(ctx, OPC_ADDIU, rd, rs, 0);
+            gen_arith(ctx, OPC_ADDU, rd, rs, 0);
         }
         break;
     case ANDI16:
@@ -14837,15 +15954,18 @@
             break;
         }
         break;
-    case B16:
+    case B16: /* BC16 */
         gen_compute_branch(ctx, OPC_BEQ, 2, 0, 0,
-                           SIMM(ctx->opcode, 0, 10) << 1, 4);
+                           sextract32(ctx->opcode, 0, 10) << 1,
+                           (ctx->insn_flags & ISA_MIPS32R6) ? 0 : 4);
         break;
-    case BNEZ16:
-    case BEQZ16:
+    case BNEZ16: /* BNEZC16 */
+    case BEQZ16: /* BEQZC16 */
         gen_compute_branch(ctx, op == BNEZ16 ? OPC_BNE : OPC_BEQ, 2,
                            mmreg(uMIPS_RD(ctx->opcode)),
-                           0, SIMM(ctx->opcode, 0, 7) << 1, 4);
+                           0, sextract32(ctx->opcode, 0, 7) << 1,
+                           (ctx->insn_flags & ISA_MIPS32R6) ? 0 : 4);
+
         break;
     case LI16:
         {
@@ -14856,17 +15976,13 @@
             tcg_gen_movi_tl(cpu_gpr[reg], imm);
         }
         break;
-    case RES_20:
-    case RES_28:
     case RES_29:
-    case RES_30:
     case RES_31:
-    case RES_38:
     case RES_39:
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     default:
-        decode_micromips32_opc (env, ctx, op);
+        decode_micromips32_opc(env, ctx);
         return 4;
     }
 
@@ -14885,7 +16001,6 @@
 static void gen_mipsdsp_ld(DisasContext *ctx, uint32_t opc,
                            int rd, int base, int offset)
 {
-    const char *opn = "ldx";
     TCGv t0;
 
     check_dsp(ctx);
@@ -14903,42 +16018,33 @@
     case OPC_LBUX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_UB);
         gen_store_gpr(t0, rd);
-        opn = "lbux";
         break;
     case OPC_LHX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESW);
         gen_store_gpr(t0, rd);
-        opn = "lhx";
         break;
     case OPC_LWX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TESL);
         gen_store_gpr(t0, rd);
-        opn = "lwx";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_LDX:
         tcg_gen_qemu_ld_tl(t0, t0, ctx->mem_idx, MO_TEQ);
         gen_store_gpr(t0, rd);
-        opn = "ldx";
         break;
 #endif
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s(%s)", opn,
-               regnames[rd], regnames[offset], regnames[base]);
     tcg_temp_free(t0);
 }
 
 static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
                               int ret, int v1, int v2)
 {
-    const char *opn = "mipsdsp arith";
     TCGv v1_t;
     TCGv v2_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -15378,23 +16484,18 @@
 
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
                               int ret, int v1, int v2)
 {
     uint32_t op2;
-    const char *opn = "mipsdsp shift";
     TCGv t0;
     TCGv v1_t;
     TCGv v2_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -15501,7 +16602,7 @@
                 break;
             default:            /* Invalid */
                 MIPS_INVAL("MASK SHLL.QB");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             break;
@@ -15616,7 +16717,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK SHLL.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -15626,21 +16727,17 @@
     tcg_temp_free(t0);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                  int ret, int v1, int v2, int check_ret)
 {
-    const char *opn = "mipsdsp multiply";
     TCGv_i32 t0;
     TCGv v1_t;
     TCGv v2_t;
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -15938,23 +17035,17 @@
     tcg_temp_free_i32(t0);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
-
 }
 
 static void gen_mipsdsp_bitinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int val)
 {
-    const char *opn = "mipsdsp Bit/ Manipulation";
     int16_t imm;
     TCGv t0;
     TCGv val_t;
 
     if (ret == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16082,23 +17173,18 @@
     }
     tcg_temp_free(t0);
     tcg_temp_free(val_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
                                      uint32_t op1, uint32_t op2,
                                      int ret, int v1, int v2, int check_ret)
 {
-    const char *opn = "mipsdsp add compare pick";
     TCGv t1;
     TCGv v1_t;
     TCGv v2_t;
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16273,22 +17359,17 @@
     tcg_temp_free(t1);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_append(CPUMIPSState *env, DisasContext *ctx,
                                uint32_t op1, int rt, int rs, int sa)
 {
-    const char *opn = "mipsdsp append/dappend";
     TCGv t0;
 
     check_dspr2(ctx);
 
     if (rt == 0) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16325,7 +17406,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK APPEND");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -16359,22 +17440,19 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DAPPEND");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
 #endif
     }
     tcg_temp_free(t0);
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
                                 int ret, int v1, int v2, int check_ret)
 
 {
-    const char *opn = "mipsdsp accumulator";
     TCGv t0;
     TCGv t1;
     TCGv v1_t;
@@ -16383,7 +17461,6 @@
 
     if ((ret == 0) && (check_ret == 1)) {
         /* Treat as NOP. */
-        MIPS_DEBUG("NOP");
         return;
     }
 
@@ -16595,249 +17672,10 @@
     tcg_temp_free(t1);
     tcg_temp_free(v1_t);
     tcg_temp_free(v2_t);
-
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s", opn);
 }
 
 /* End MIPSDSP functions. */
 
-/* Compact Branches */
-static void gen_compute_compact_branch(DisasContext *ctx, uint32_t opc,
-                                       int rs, int rt, int32_t offset)
-{
-    int bcond_compute = 0;
-    TCGv t0 = tcg_temp_new();
-    TCGv t1 = tcg_temp_new();
-
-    if (ctx->hflags & MIPS_HFLAG_BMASK) {
-#ifdef MIPS_DEBUG_DISAS
-        LOG_DISAS("Branch in delay / forbidden slot at PC 0x" TARGET_FMT_lx
-                  "\n", ctx->pc);
-#endif
-        generate_exception(ctx, EXCP_RI);
-        goto out;
-    }
-
-    /* Load needed operands and calculate btarget */
-    switch (opc) {
-    /* compact branch */
-    case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC */
-    case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
-        gen_load_gpr(t0, rs);
-        gen_load_gpr(t1, rt);
-        bcond_compute = 1;
-        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
-        if (rs <= rt && rs == 0) {
-            /* OPC_BEQZALC, OPC_BNEZALC */
-            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
-        }
-        break;
-    case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC */
-    case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC */
-        gen_load_gpr(t0, rs);
-        gen_load_gpr(t1, rt);
-        bcond_compute = 1;
-        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
-        break;
-    case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC */
-    case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC */
-        if (rs == 0 || rs == rt) {
-            /* OPC_BLEZALC, OPC_BGEZALC */
-            /* OPC_BGTZALC, OPC_BLTZALC */
-            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
-        }
-        gen_load_gpr(t0, rs);
-        gen_load_gpr(t1, rt);
-        bcond_compute = 1;
-        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
-        break;
-    case OPC_BC:
-    case OPC_BALC:
-        ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
-        break;
-    case OPC_BEQZC:
-    case OPC_BNEZC:
-        if (rs != 0) {
-            /* OPC_BEQZC, OPC_BNEZC */
-            gen_load_gpr(t0, rs);
-            bcond_compute = 1;
-            ctx->btarget = addr_add(ctx, ctx->pc + 4, offset);
-        } else {
-            /* OPC_JIC, OPC_JIALC */
-            TCGv tbase = tcg_temp_new();
-            TCGv toffset = tcg_temp_new();
-
-            gen_load_gpr(tbase, rt);
-            tcg_gen_movi_tl(toffset, offset);
-            gen_op_addr_add(ctx, btarget, tbase, toffset);
-            tcg_temp_free(tbase);
-            tcg_temp_free(toffset);
-        }
-        break;
-    default:
-        MIPS_INVAL("Compact branch/jump");
-        generate_exception(ctx, EXCP_RI);
-        goto out;
-    }
-
-    if (bcond_compute == 0) {
-        /* Uncoditional compact branch */
-        switch (opc) {
-        case OPC_JIALC:
-            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
-            /* Fallthrough */
-        case OPC_JIC:
-            ctx->hflags |= MIPS_HFLAG_BR;
-            break;
-        case OPC_BALC:
-            tcg_gen_movi_tl(cpu_gpr[31], ctx->pc + 4);
-            /* Fallthrough */
-        case OPC_BC:
-            ctx->hflags |= MIPS_HFLAG_B;
-            break;
-        default:
-            MIPS_INVAL("Compact branch/jump");
-            generate_exception(ctx, EXCP_RI);
-            goto out;
-        }
-
-        /* Generating branch here as compact branches don't have delay slot */
-        gen_branch(ctx, 4);
-    } else {
-        /* Conditional compact branch */
-        int fs = gen_new_label();
-        save_cpu_state(ctx, 0);
-
-        switch (opc) {
-        case OPC_BLEZALC: /* OPC_BGEZALC, OPC_BGEUC */
-            if (rs == 0 && rt != 0) {
-                /* OPC_BLEZALC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LE), t1, 0, fs);
-            } else if (rs != 0 && rt != 0 && rs == rt) {
-                /* OPC_BGEZALC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GE), t1, 0, fs);
-            } else {
-                /* OPC_BGEUC */
-                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_GEU), t0, t1, fs);
-            }
-            break;
-        case OPC_BGTZALC: /* OPC_BLTZALC, OPC_BLTUC */
-            if (rs == 0 && rt != 0) {
-                /* OPC_BGTZALC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GT), t1, 0, fs);
-            } else if (rs != 0 && rt != 0 && rs == rt) {
-                /* OPC_BLTZALC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LT), t1, 0, fs);
-            } else {
-                /* OPC_BLTUC */
-                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_LTU), t0, t1, fs);
-            }
-            break;
-        case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC */
-            if (rs == 0 && rt != 0) {
-                /* OPC_BLEZC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LE), t1, 0, fs);
-            } else if (rs != 0 && rt != 0 && rs == rt) {
-                /* OPC_BGEZC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GE), t1, 0, fs);
-            } else {
-                /* OPC_BGEC */
-                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_GE), t0, t1, fs);
-            }
-            break;
-        case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC */
-            if (rs == 0 && rt != 0) {
-                /* OPC_BGTZC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_GT), t1, 0, fs);
-            } else if (rs != 0 && rt != 0 && rs == rt) {
-                /* OPC_BLTZC */
-                tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_LT), t1, 0, fs);
-            } else {
-                /* OPC_BLTC */
-                tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_LT), t0, t1, fs);
-            }
-            break;
-        case OPC_BOVC: /* OPC_BEQZALC, OPC_BEQC */
-        case OPC_BNVC: /* OPC_BNEZALC, OPC_BNEC */
-            if (rs >= rt) {
-                /* OPC_BOVC, OPC_BNVC */
-                TCGv t2 = tcg_temp_new();
-                TCGv t3 = tcg_temp_new();
-                TCGv t4 = tcg_temp_new();
-                TCGv input_overflow = tcg_temp_new();
-
-                gen_load_gpr(t0, rs);
-                gen_load_gpr(t1, rt);
-                tcg_gen_ext32s_tl(t2, t0);
-                tcg_gen_setcond_tl(TCG_COND_NE, input_overflow, t2, t0);
-                tcg_gen_ext32s_tl(t3, t1);
-                tcg_gen_setcond_tl(TCG_COND_NE, t4, t3, t1);
-                tcg_gen_or_tl(input_overflow, input_overflow, t4);
-
-                tcg_gen_add_tl(t4, t2, t3);
-                tcg_gen_ext32s_tl(t4, t4);
-                tcg_gen_xor_tl(t2, t2, t3);
-                tcg_gen_xor_tl(t3, t4, t3);
-                tcg_gen_andc_tl(t2, t3, t2);
-                tcg_gen_setcondi_tl(TCG_COND_LT, t4, t2, 0);
-                tcg_gen_or_tl(t4, t4, input_overflow);
-                if (opc == OPC_BOVC) {
-                    /* OPC_BOVC */
-                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t4, 0, fs);
-                } else {
-                    /* OPC_BNVC */
-                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t4, 0, fs);
-                }
-                tcg_temp_free(input_overflow);
-                tcg_temp_free(t4);
-                tcg_temp_free(t3);
-                tcg_temp_free(t2);
-            } else if (rs < rt && rs == 0) {
-                /* OPC_BEQZALC, OPC_BNEZALC */
-                if (opc == OPC_BEQZALC) {
-                    /* OPC_BEQZALC */
-                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t1, 0, fs);
-                } else {
-                    /* OPC_BNEZALC */
-                    tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t1, 0, fs);
-                }
-            } else {
-                /* OPC_BEQC, OPC_BNEC */
-                if (opc == OPC_BEQC) {
-                    /* OPC_BEQC */
-                    tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_EQ), t0, t1, fs);
-                } else {
-                    /* OPC_BNEC */
-                    tcg_gen_brcond_tl(tcg_invert_cond(TCG_COND_NE), t0, t1, fs);
-                }
-            }
-            break;
-        case OPC_BEQZC:
-            tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_EQ), t0, 0, fs);
-            break;
-        case OPC_BNEZC:
-            tcg_gen_brcondi_tl(tcg_invert_cond(TCG_COND_NE), t0, 0, fs);
-            break;
-        default:
-            MIPS_INVAL("Compact conditional branch/jump");
-            generate_exception(ctx, EXCP_RI);
-            goto out;
-        }
-
-        /* Generating branch here as compact branches don't have delay slot */
-        gen_goto_tb(ctx, 1, ctx->btarget);
-        gen_set_label(fs);
-
-        ctx->hflags |= MIPS_HFLAG_FBNSLOT;
-        MIPS_DEBUG("Compact conditional branch");
-    }
-
-out:
-    tcg_temp_free(t0);
-    tcg_temp_free(t1);
-}
-
 static void decode_opc_special_r6(CPUMIPSState *env, DisasContext *ctx)
 {
     int rs, rt, rd, sa;
@@ -16851,18 +17689,7 @@
     op1 = MASK_SPECIAL(ctx->opcode);
     switch (op1) {
     case OPC_LSA:
-        if (rd != 0) {
-            int imm2 = extract32(ctx->opcode, 6, 3);
-            TCGv t0 = tcg_temp_new();
-            TCGv t1 = tcg_temp_new();
-            gen_load_gpr(t0, rs);
-            gen_load_gpr(t1, rt);
-            tcg_gen_shli_tl(t0, t0, imm2 + 1);
-            tcg_gen_add_tl(t0, t0, t1);
-            tcg_gen_ext32s_tl(cpu_gpr[rd], t0);
-            tcg_temp_free(t1);
-            tcg_temp_free(t0);
-        }
+        gen_lsa(ctx, op1, rd, rs, rt, extract32(ctx->opcode, 6, 2));
         break;
     case OPC_MULT ... OPC_DIVU:
         op2 = MASK_R6_MULDIV(ctx->opcode);
@@ -16879,7 +17706,7 @@
             break;
         default:
             MIPS_INVAL("special_r6 muldiv");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -16894,30 +17721,24 @@
                We need additionally to check other fields */
             gen_cl(ctx, op1, rd, rs);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case R6_OPC_SDBBP:
-        if (ctx->hflags & MIPS_HFLAG_SBRI) {
-            generate_exception(ctx, EXCP_RI);
+        if (is_uhi(extract32(ctx->opcode, 6, 20))) {
+            gen_helper_do_semihosting(cpu_env);
         } else {
-            generate_exception(ctx, EXCP_DBp);
+            if (ctx->hflags & MIPS_HFLAG_SBRI) {
+                generate_exception_end(ctx, EXCP_RI);
+            } else {
+                generate_exception_end(ctx, EXCP_DBp);
+            }
         }
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DLSA:
         check_mips_64(ctx);
-        if (rd != 0) {
-            int imm2 = extract32(ctx->opcode, 6, 3);
-            TCGv t0 = tcg_temp_new();
-            TCGv t1 = tcg_temp_new();
-            gen_load_gpr(t0, rs);
-            gen_load_gpr(t1, rt);
-            tcg_gen_shli_tl(t0, t0, imm2 + 1);
-            tcg_gen_add_tl(cpu_gpr[rd], t0, t1);
-            tcg_temp_free(t1);
-            tcg_temp_free(t0);
-        }
+        gen_lsa(ctx, op1, rd, rs, rt, extract32(ctx->opcode, 6, 2));
         break;
     case R6_OPC_DCLO:
     case R6_OPC_DCLZ:
@@ -16927,7 +17748,7 @@
             check_mips_64(ctx);
             gen_cl(ctx, op1, rd, rs);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case OPC_DMULT ... OPC_DDIVU:
@@ -16946,14 +17767,14 @@
             break;
         default:
             MIPS_INVAL("special_r6 muldiv");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special_r6");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17021,16 +17842,16 @@
     case OPC_SPIM:
 #ifdef MIPS_STRICT_STANDARD
         MIPS_INVAL("SPIM");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 #else
         /* Implemented as RI exception for now. */
         MIPS_INVAL("spim (unofficial)");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
 #endif
         break;
     default:            /* Invalid */
         MIPS_INVAL("special_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17052,8 +17873,7 @@
             rs == 0 && rt == 0) { /* PAUSE */
             if ((ctx->insn_flags & ISA_MIPS32R6) &&
                 (ctx->hflags & MIPS_HFLAG_BMASK)) {
-                MIPS_DEBUG("CTI in delay / forbidden slot");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         }
@@ -17073,7 +17893,7 @@
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17096,7 +17916,7 @@
             gen_shift(ctx, op1, rd, rs, rt);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17115,6 +17935,7 @@
         break;
     case OPC_TGE ... OPC_TEQ: /* Traps */
     case OPC_TNE:
+        check_insn(ctx, ISA_MIPS2);
         gen_trap(ctx, op1, rs, rt, -1);
         break;
     case OPC_LSA: /* OPC_PMON */
@@ -17125,20 +17946,20 @@
             /* Pmon entry point, also R4010 selsl */
 #ifdef MIPS_STRICT_STANDARD
             MIPS_INVAL("PMON / selsl");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
 #else
             gen_helper_0e0i(pmon, sa);
 #endif
         }
         break;
     case OPC_SYSCALL:
-        generate_exception(ctx, EXCP_SYSCALL);
-        ctx->bstate = BS_STOP;
+        generate_exception_end(ctx, EXCP_SYSCALL);
         break;
     case OPC_BREAK:
-        generate_exception(ctx, EXCP_BREAK);
+        generate_exception_end(ctx, EXCP_BREAK);
         break;
     case OPC_SYNC:
+        check_insn(ctx, ISA_MIPS2);
         /* Treat as NOP. */
         break;
 
@@ -17166,7 +17987,7 @@
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17184,7 +18005,7 @@
             gen_shift_imm(ctx, op1, rd, rt, sa);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17213,7 +18034,7 @@
             gen_shift(ctx, op1, rd, rs, rt);
             break;
         default:
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17269,16 +18090,15 @@
         gen_cl(ctx, op1, rd, rs);
         break;
     case OPC_SDBBP:
-        /* XXX: not clear which exception should be raised
-         *      when in debug mode...
-         */
-        check_insn(ctx, ISA_MIPS32);
-        if (!(ctx->hflags & MIPS_HFLAG_DM)) {
-            generate_exception(ctx, EXCP_DBp);
+        if (is_uhi(extract32(ctx->opcode, 6, 20))) {
+            gen_helper_do_semihosting(cpu_env);
         } else {
-            generate_exception(ctx, EXCP_DBp);
+            /* XXX: not clear which exception should be raised
+             *      when in debug mode...
+             */
+            check_insn(ctx, ISA_MIPS32);
+            generate_exception_end(ctx, EXCP_DBp);
         }
-        /* Treat as NOP. */
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DCLO:
@@ -17299,7 +18119,7 @@
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special2_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17321,12 +18141,15 @@
     case R6_OPC_PREF:
         if (rt >= 24) {
             /* hint codes 24-31 are reserved and signal RI */
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         /* Treat as NOP. */
         break;
     case R6_OPC_CACHE:
-        /* Treat as NOP. */
+        check_cp0_enabled(ctx);
+        if (ctx->hflags & MIPS_HFLAG_ITC_CACHE) {
+            gen_cache_operation(ctx, rt, rs, imm);
+        }
         break;
     case R6_OPC_SC:
         gen_st_cond(ctx, op1, rt, rs, imm);
@@ -17340,35 +18163,15 @@
                 /* Treat as NOP. */
                 break;
             }
-            TCGv t0 = tcg_temp_new();
-            gen_load_gpr(t0, rt);
-
             op2 = MASK_BSHFL(ctx->opcode);
             switch (op2) {
             case OPC_ALIGN ... OPC_ALIGN_END:
-                sa &= 3;
-                if (sa == 0) {
-                    tcg_gen_mov_tl(cpu_gpr[rd], t0);
-                } else {
-                    TCGv t1 = tcg_temp_new();
-                    TCGv_i64 t2 = tcg_temp_new_i64();
-                    gen_load_gpr(t1, rs);
-                    tcg_gen_concat_tl_i64(t2, t1, t0);
-                    tcg_gen_shri_i64(t2, t2, 8 * (4 - sa));
-#if defined(TARGET_MIPS64)
-                    tcg_gen_ext32s_i64(cpu_gpr[rd], t2);
-#else
-                    tcg_gen_trunc_i64_i32(cpu_gpr[rd], t2);
-#endif
-                    tcg_temp_free_i64(t2);
-                    tcg_temp_free(t1);
-                }
+                gen_align(ctx, OPC_ALIGN, rd, rs, rt, sa & 3);
                 break;
             case OPC_BITSWAP:
-                gen_helper_bitswap(cpu_gpr[rd], t0);
+                gen_bitswap(ctx, op2, rd, rt);
                 break;
             }
-            tcg_temp_free(t0);
         }
         break;
 #if defined(TARGET_MIPS64)
@@ -17385,35 +18188,22 @@
                 /* Treat as NOP. */
                 break;
             }
-            TCGv t0 = tcg_temp_new();
-            gen_load_gpr(t0, rt);
-
             op2 = MASK_DBSHFL(ctx->opcode);
             switch (op2) {
             case OPC_DALIGN ... OPC_DALIGN_END:
-                sa &= 7;
-                if (sa == 0) {
-                    tcg_gen_mov_tl(cpu_gpr[rd], t0);
-                } else {
-                    TCGv t1 = tcg_temp_new();
-                    gen_load_gpr(t1, rs);
-                    tcg_gen_shli_tl(t0, t0, 8 * sa);
-                    tcg_gen_shri_tl(t1, t1, 8 * (8 - sa));
-                    tcg_gen_or_tl(cpu_gpr[rd], t1, t0);
-                    tcg_temp_free(t1);
-                }
+                gen_align(ctx, OPC_DALIGN, rd, rs, rt, sa & 7);
                 break;
             case OPC_DBITSWAP:
-                gen_helper_dbitswap(cpu_gpr[rd], t0);
+                gen_bitswap(ctx, op2, rd, rt);
                 break;
             }
-            tcg_temp_free(t0);
+
         }
         break;
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special3_r6");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17459,13 +18249,13 @@
                 break;
             default:
                 MIPS_INVAL("MASK ADDUH.QB");
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else if (ctx->insn_flags & INSN_LOONGSON2E) {
             gen_loongson_integer(ctx, op1, rd, rs, rt);
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
     case OPC_LX_DSP:
@@ -17481,7 +18271,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK LX");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17512,7 +18302,7 @@
             break;
         default:
             MIPS_INVAL("MASK ABSQ_S.PH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17549,7 +18339,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ADDU.QB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
 
         }
@@ -17589,7 +18379,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK CMPU.EQ.QB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17625,7 +18415,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DPAW.PH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17638,7 +18428,6 @@
                 TCGv t0, t1;
 
                 if (rt == 0) {
-                    MIPS_DEBUG("NOP");
                     break;
                 }
 
@@ -17656,7 +18445,7 @@
             }
         default:            /* Invalid */
             MIPS_INVAL("MASK INSV");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17691,7 +18480,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK EXTR.W");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17734,7 +18523,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ABSQ_S.QH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17773,7 +18562,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK ADDU.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17818,7 +18607,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK CMPU_EQ.OB");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17855,7 +18644,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK EXTR.W");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17894,7 +18683,7 @@
             break;
         default:            /* Invalid */
             MIPS_INVAL("MASK DPAQ.W.QH");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17906,7 +18695,6 @@
             TCGv t0, t1;
 
             if (rt == 0) {
-                MIPS_DEBUG("NOP");
                 break;
             }
             check_dsp(ctx);
@@ -17925,7 +18713,7 @@
         }
         default:            /* Invalid */
             MIPS_INVAL("MASK DINSV");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -17935,7 +18723,7 @@
 #endif
     default:            /* Invalid */
         MIPS_INVAL("special3_legacy");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -17996,7 +18784,7 @@
         break;
 #endif
     case OPC_RDHWR:
-        gen_rdhwr(ctx, rt, rd);
+        gen_rdhwr(ctx, rt, rd, extract32(ctx->opcode, 6, 3));
         break;
     case OPC_FORK:
         check_insn(ctx, ASE_MT);
@@ -18016,7 +18804,6 @@
         {
             TCGv t0 = tcg_temp_new();
 
-            save_cpu_state(ctx, 1);
             gen_load_gpr(t0, rs);
             gen_helper_yield(t0, cpu_env, t0);
             gen_store_gpr(t0, rd);
@@ -18037,16 +18824,16 @@
 {
     if (unlikely((ctx->hflags & MIPS_HFLAG_FPU) &&
                  !(ctx->hflags & MIPS_HFLAG_F64))) {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return 0;
     }
 
     if (unlikely(!(ctx->hflags & MIPS_HFLAG_MSA))) {
         if (ctx->insn_flags & ASE_MSA) {
-            generate_exception(ctx, EXCP_MSADIS);
+            generate_exception_end(ctx, EXCP_MSADIS);
             return 0;
         } else {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             return 0;
         }
     }
@@ -18103,8 +18890,7 @@
     check_msa_access(ctx);
 
     if (ctx->insn_flags & ISA_MIPS32R6 && ctx->hflags & MIPS_HFLAG_BMASK) {
-        MIPS_DEBUG("CTI in delay / forbidden slot");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
     switch (op1) {
@@ -18179,7 +18965,7 @@
         {
             uint8_t df = (ctx->opcode >> 24) & 0x3;
             if (df == DF_DOUBLE) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
             } else {
                 TCGv_i32 tdf = tcg_const_i32(df);
                 gen_helper_msa_shf_df(cpu_env, tdf, twd, tws, ti8);
@@ -18189,7 +18975,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18261,7 +19047,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18297,7 +19083,7 @@
         m = dfm & 0x7;
         df = DF_BYTE;
     } else {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -18345,7 +19131,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18540,7 +19326,8 @@
     case OPC_HSUB_S_df:
     case OPC_HSUB_U_df:
         if (df == DF_BYTE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
+            break;
         }
         switch (MASK_MSA_3R(ctx->opcode)) {
         case OPC_DOTP_S_df:
@@ -18577,7 +19364,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
     tcg_temp_free_i32(twd);
@@ -18609,7 +19396,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18646,7 +19433,7 @@
 #if !defined(TARGET_MIPS64)
         /* Double format valid only for MIPS64 */
         if (df == DF_DOUBLE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
 #endif
@@ -18664,7 +19451,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
     }
     tcg_temp_free_i32(twd);
     tcg_temp_free_i32(tws);
@@ -18694,7 +19481,7 @@
         gen_msa_elm_3e(env, ctx);
         return;
     } else {
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         return;
     }
 
@@ -18849,7 +19636,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -18877,7 +19664,7 @@
 #if !defined(TARGET_MIPS64)
         /* Double format valid only for MIPS64 */
         if (df == DF_DOUBLE) {
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
 #endif
@@ -18894,7 +19681,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19009,7 +19796,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
@@ -19038,7 +19825,7 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
@@ -19136,13 +19923,13 @@
         break;
     default:
         MIPS_INVAL("MSA instruction");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 
 }
 
-static void decode_opc (CPUMIPSState *env, DisasContext *ctx)
+static void decode_opc(CPUMIPSState *env, DisasContext *ctx)
 {
     int32_t offset;
     int rs, rt, rd, sa;
@@ -19158,19 +19945,14 @@
 
     /* Handle blikely not taken case */
     if ((ctx->hflags & MIPS_HFLAG_BMASK_BASE) == MIPS_HFLAG_BL) {
-        int l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
 
-        MIPS_DEBUG("blikely condition (" TARGET_FMT_lx ")", ctx->pc + 4);
         tcg_gen_brcondi_tl(TCG_COND_NE, bcond, 0, l1);
         tcg_gen_movi_i32(hflags, ctx->hflags & ~MIPS_HFLAG_BMASK);
         gen_goto_tb(ctx, 1, ctx->pc + 4);
         gen_set_label(l1);
     }
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     op = MASK_OP_MAJOR(ctx->opcode);
     rs = (ctx->opcode >> 21) & 0x1f;
     rt = (ctx->opcode >> 16) & 0x1f;
@@ -19194,7 +19976,9 @@
         case OPC_BGEZL:
         case OPC_BLTZALL:
         case OPC_BGEZALL:
+            check_insn(ctx, ISA_MIPS2);
             check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            /* Fallthrough */
         case OPC_BLTZ:
         case OPC_BGEZ:
             gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2, 4);
@@ -19206,7 +19990,7 @@
                     /* OPC_NAL, OPC_BAL */
                     gen_compute_branch(ctx, op1, 4, 0, -1, imm << 2, 4);
                 } else {
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                 }
             } else {
                 gen_compute_branch(ctx, op1, 4, rs, -1, imm << 2, 4);
@@ -19214,9 +19998,14 @@
             break;
         case OPC_TGEI ... OPC_TEQI: /* REGIMM traps */
         case OPC_TNEI:
+            check_insn(ctx, ISA_MIPS2);
             check_insn_opc_removed(ctx, ISA_MIPS32R6);
             gen_trap(ctx, op1, rs, -1, imm);
             break;
+        case OPC_SIGRIE:
+            check_insn(ctx, ISA_MIPS32R6);
+            generate_exception_end(ctx, EXCP_RI);
+            break;
         case OPC_SYNCI:
             check_insn(ctx, ISA_MIPS32R2);
             /* Break the TB to be able to sync copied instructions
@@ -19237,7 +20026,6 @@
             if (rs != 0) {
                 tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 32);
             }
-            MIPS_DEBUG("dahi %s, %04x", regnames[rs], imm);
             break;
         case OPC_DATI:
             check_insn(ctx, ISA_MIPS32R6);
@@ -19245,12 +20033,11 @@
             if (rs != 0) {
                 tcg_gen_addi_tl(cpu_gpr[rs], cpu_gpr[rs], (int64_t)imm << 48);
             }
-            MIPS_DEBUG("dati %s, %04x", regnames[rs], imm);
             break;
 #endif
         default:            /* Invalid */
             MIPS_INVAL("regimm");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19262,6 +20049,8 @@
         case OPC_MTC0:
         case OPC_MFTR:
         case OPC_MTTR:
+        case OPC_MFHC0:
+        case OPC_MTHC0:
 #if defined(TARGET_MIPS64)
         case OPC_DMFC0:
         case OPC_DMTC0:
@@ -19303,12 +20092,27 @@
                     gen_helper_evpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
+                case OPC_DVP:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    if (ctx->vp) {
+                        gen_helper_dvp(t0, cpu_env);
+                        gen_store_gpr(t0, rt);
+                    }
+                    break;
+                case OPC_EVP:
+                    check_insn(ctx, ISA_MIPS32R6);
+                    if (ctx->vp) {
+                        gen_helper_evp(t0, cpu_env);
+                        gen_store_gpr(t0, rt);
+                    }
+                    break;
                 case OPC_DI:
                     check_insn(ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
                     gen_helper_di(t0, cpu_env);
                     gen_store_gpr(t0, rt);
-                    /* Stop translation as we may have switched the execution mode */
+                    /* Stop translation as we may have switched
+                       the execution mode.  */
                     ctx->bstate = BS_STOP;
                     break;
                 case OPC_EI:
@@ -19316,12 +20120,13 @@
                     save_cpu_state(ctx, 1);
                     gen_helper_ei(t0, cpu_env);
                     gen_store_gpr(t0, rt);
-                    /* Stop translation as we may have switched the execution mode */
+                    /* Stop translation as we may have switched
+                       the execution mode.  */
                     ctx->bstate = BS_STOP;
                     break;
                 default:            /* Invalid */
                     MIPS_INVAL("mfmc0");
-                    generate_exception(ctx, EXCP_RI);
+                    generate_exception_end(ctx, EXCP_RI);
                     break;
                 }
                 tcg_temp_free(t0);
@@ -19338,7 +20143,7 @@
             break;
         default:
             MIPS_INVAL("cp0");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19373,7 +20178,7 @@
     case OPC_BLEZC: /* OPC_BGEZC, OPC_BGEC, OPC_BLEZL */
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rt == 0) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             /* OPC_BLEZC, OPC_BGEZC, OPC_BGEC */
@@ -19386,7 +20191,7 @@
     case OPC_BGTZC: /* OPC_BLTZC, OPC_BLTC, OPC_BGTZL */
         if (ctx->insn_flags & ISA_MIPS32R6) {
             if (rt == 0) {
-                generate_exception(ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
             /* OPC_BGTZC, OPC_BLTZC, OPC_BLTC */
@@ -19418,15 +20223,20 @@
         break;
     case OPC_BEQL:
     case OPC_BNEL:
+        check_insn(ctx, ISA_MIPS2);
          check_insn_opc_removed(ctx, ISA_MIPS32R6);
+        /* Fallthrough */
     case OPC_BEQ:
     case OPC_BNE:
          gen_compute_branch(ctx, op, 4, rs, rt, imm << 2, 4);
          break;
-    case OPC_LWL: /* Load and stores */
+    case OPC_LL: /* Load and stores */
+        check_insn(ctx, ISA_MIPS2);
+        /* Fallthrough */
+    case OPC_LWL:
     case OPC_LWR:
-    case OPC_LL:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
+         /* Fallthrough */
     case OPC_LB ... OPC_LH:
     case OPC_LW ... OPC_LHU:
          gen_ld(ctx, op, rt, rs, imm);
@@ -19434,11 +20244,13 @@
     case OPC_SWL:
     case OPC_SWR:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
+        /* fall through */
     case OPC_SB ... OPC_SH:
     case OPC_SW:
          gen_st(ctx, op, rt, rs, imm);
          break;
     case OPC_SC:
+        check_insn(ctx, ISA_MIPS2);
          check_insn_opc_removed(ctx, ISA_MIPS32R6);
          gen_st_cond(ctx, op, rt, rs, imm);
          break;
@@ -19446,6 +20258,9 @@
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
         check_cp0_enabled(ctx);
         check_insn(ctx, ISA_MIPS3 | ISA_MIPS32);
+        if (ctx->hflags & MIPS_HFLAG_ITC_CACHE) {
+            gen_cache_operation(ctx, rt, rs, imm);
+        }
         /* Treat as NOP. */
         break;
     case OPC_PREF:
@@ -19482,6 +20297,7 @@
         case OPC_DMTC1:
             check_cp1_enabled(ctx);
             check_insn(ctx, ISA_MIPS3);
+            check_mips_64(ctx);
             gen_cp1(ctx, op1, rt, rd);
             break;
 #endif
@@ -19490,7 +20306,7 @@
             if (ctx->insn_flags & ISA_MIPS32R6) {
                 /* OPC_BC1EQZ */
                 gen_compute_branch1_r6(ctx, MASK_CP1(ctx->opcode),
-                                rt, imm << 2);
+                                       rt, imm << 2, 4);
             } else {
                 /* OPC_BC1ANY2 */
                 check_cop1x(ctx);
@@ -19503,7 +20319,7 @@
             check_cp1_enabled(ctx);
             check_insn(ctx, ISA_MIPS32R6);
             gen_compute_branch1_r6(ctx, MASK_CP1(ctx->opcode),
-                            rt, imm << 2);
+                                   rt, imm << 2, 4);
             break;
         case OPC_BC1ANY4:
             check_cp1_enabled(ctx);
@@ -19518,8 +20334,8 @@
                                 (rt >> 2) & 0x7, imm << 2);
             break;
         case OPC_PS_FMT:
-            check_cp1_enabled(ctx);
-            check_insn_opc_removed(ctx, ISA_MIPS32R6);
+            check_ps(ctx);
+            /* fall through */
         case OPC_S_FMT:
         case OPC_D_FMT:
             check_cp1_enabled(ctx);
@@ -19582,8 +20398,9 @@
                     gen_r6_cmp_d(ctx, ctx->opcode & 0x1f, rt, rd, sa);
                     break;
                 default:
-                    gen_farith(ctx, ctx->opcode & FOP(0x3f, 0x1f), rt, rd, sa,
-                               (imm >> 8) & 0x7);
+                    gen_farith(ctx, ctx->opcode & FOP(0x3f, 0x1f),
+                               rt, rd, sa, (imm >> 8) & 0x7);
+
                     break;
                 }
             } else {
@@ -19607,7 +20424,7 @@
             break;
         default:
             MIPS_INVAL("cp1");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             break;
         }
         break;
@@ -19654,18 +20471,24 @@
             check_cp1_enabled(ctx);
             op1 = MASK_CP3(ctx->opcode);
             switch (op1) {
+            case OPC_LUXC1:
+            case OPC_SUXC1:
+                check_insn(ctx, ISA_MIPS5 | ISA_MIPS32R2);
+                /* Fallthrough */
             case OPC_LWXC1:
             case OPC_LDXC1:
-            case OPC_LUXC1:
             case OPC_SWXC1:
             case OPC_SDXC1:
-            case OPC_SUXC1:
+                check_insn(ctx, ISA_MIPS4 | ISA_MIPS32R2);
                 gen_flt3_ldst(ctx, op1, sa, rd, rs, rt);
                 break;
             case OPC_PREFX:
+                check_insn(ctx, ISA_MIPS4 | ISA_MIPS32R2);
                 /* Treat as NOP. */
                 break;
             case OPC_ALNV_PS:
+                check_insn(ctx, ISA_MIPS5 | ISA_MIPS32R2);
+                /* Fallthrough */
             case OPC_MADD_S:
             case OPC_MADD_D:
             case OPC_MADD_PS:
@@ -19678,11 +20501,12 @@
             case OPC_NMSUB_S:
             case OPC_NMSUB_D:
             case OPC_NMSUB_PS:
+                check_insn(ctx, ISA_MIPS4 | ISA_MIPS32R2);
                 gen_flt3_arith(ctx, op1, sa, rs, rd, rt);
                 break;
             default:
                 MIPS_INVAL("cp3");
-                generate_exception (ctx, EXCP_RI);
+                generate_exception_end(ctx, EXCP_RI);
                 break;
             }
         } else {
@@ -19695,6 +20519,7 @@
     case OPC_LDL ... OPC_LDR:
     case OPC_LLD:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
+        /* fall through */
     case OPC_LWU:
     case OPC_LD:
         check_insn(ctx, ISA_MIPS3);
@@ -19703,6 +20528,7 @@
         break;
     case OPC_SDL ... OPC_SDR:
         check_insn_opc_removed(ctx, ISA_MIPS32R6);
+        /* fall through */
     case OPC_SD:
         check_insn(ctx, ISA_MIPS3);
         check_mips_64(ctx);
@@ -19736,7 +20562,7 @@
             gen_compute_compact_branch(ctx, op, rs, rt, imm << 2);
         } else {
             MIPS_INVAL("major opcode");
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
         }
         break;
 #endif
@@ -19745,15 +20571,16 @@
 #if defined(TARGET_MIPS64)
             /* OPC_DAUI */
             check_mips_64(ctx);
-            if (rt != 0) {
+            if (rs == 0) {
+                generate_exception(ctx, EXCP_RI);
+            } else if (rt != 0) {
                 TCGv t0 = tcg_temp_new();
                 gen_load_gpr(t0, rs);
                 tcg_gen_addi_tl(cpu_gpr[rt], t0, imm << 16);
                 tcg_temp_free(t0);
             }
-            MIPS_DEBUG("daui %s, %s, %04x", regnames[rt], regnames[rs], imm);
 #else
-            generate_exception(ctx, EXCP_RI);
+            generate_exception_end(ctx, EXCP_RI);
             MIPS_INVAL("major opcode");
 #endif
         } else {
@@ -19769,34 +20596,29 @@
         break;
     case OPC_PCREL:
         check_insn(ctx, ISA_MIPS32R6);
-        gen_pcrel(ctx, rs, imm);
+        gen_pcrel(ctx, ctx->opcode, ctx->pc, rs);
         break;
     default:            /* Invalid */
         MIPS_INVAL("major opcode");
-        generate_exception(ctx, EXCP_RI);
+        generate_exception_end(ctx, EXCP_RI);
         break;
     }
 }
 
-static inline void
-gen_intermediate_code_internal(MIPSCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMIPSState *env, struct TranslationBlock *tb)
 {
+    MIPSCPU *cpu = mips_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUMIPSState *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
+    target_ulong next_page_start;
     int num_insns;
     int max_insns;
     int insn_bytes;
     int is_slot;
 
-    if (search_pc)
-        qemu_log("search pc %d\n", search_pc);
-
     pc_start = tb->pc;
+    next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.singlestep_enabled = cs->singlestep_enabled;
@@ -19804,14 +20626,25 @@
     ctx.CP0_Config1 = env->CP0_Config1;
     ctx.tb = tb;
     ctx.bstate = BS_NONE;
+    ctx.btarget = 0;
     ctx.kscrexist = (env->CP0_Config4 >> CP0C4_KScrExist) & 0xff;
     ctx.rxi = (env->CP0_Config3 >> CP0C3_RXI) & 1;
     ctx.ie = (env->CP0_Config4 >> CP0C4_IE) & 3;
     ctx.bi = (env->CP0_Config3 >> CP0C3_BI) & 1;
     ctx.bp = (env->CP0_Config3 >> CP0C3_BP) & 1;
+    ctx.PAMask = env->PAMask;
+    ctx.mvh = (env->CP0_Config5 >> CP0C5_MVH) & 1;
+    ctx.CP0_LLAddr_shift = env->CP0_LLAddr_shift;
+    ctx.cmgcr = (env->CP0_Config3 >> CP0C3_CMGCR) & 1;
     /* Restore delay slot state from the tb context.  */
     ctx.hflags = (uint32_t)tb->flags; /* FIXME: maybe use 64 bits here? */
-    ctx.ulri = env->CP0_Config3 & (1 << CP0C3_ULRI);
+    ctx.ulri = (env->CP0_Config3 >> CP0C3_ULRI) & 1;
+    ctx.ps = ((env->active_fpu.fcr0 >> FCR0_PS) & 1) ||
+             (env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F));
+    ctx.vp = (env->CP0_Config5 >> CP0C5_VP) & 1;
+    ctx.mrp = (env->CP0_Config5 >> CP0C5_MRP) & 1;
+    ctx.nan2008 = (env->active_fpu.fcr31 >> FCR31_NAN2008) & 1;
+    ctx.abs2008 = (env->active_fpu.fcr31 >> FCR31_ABS2008) & 1;
     restore_cpu_state(env, &ctx);
 #ifdef CONFIG_USER_ONLY
         ctx.mem_idx = MIPS_HFLAG_UM;
@@ -19822,40 +20655,34 @@
                                  MO_UNALN : MO_ALIGN;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     LOG_DISAS("\ntb %p idx %d hflags %04x\n", tb, ctx.mem_idx, ctx.hflags);
-    gen_tb_start();
+    gen_tb_start(tb);
     while (ctx.bstate == BS_NONE) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.pc) {
-                    save_cpu_state(&ctx, 1);
-                    ctx.bstate = BS_BRANCH;
-                    gen_helper_0e0i(raise_exception, EXCP_DEBUG);
-                    /* Include the breakpoint location or the tb won't
-                     * be flushed when it must be.  */
-                    ctx.pc += 4;
-                    goto done_generating;
-                }
-            }
+        tcg_gen_insn_start(ctx.pc, ctx.hflags & MIPS_HFLAG_BMASK, ctx.btarget);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            save_cpu_state(&ctx, 1);
+            ctx.bstate = BS_BRANCH;
+            gen_helper_raise_exception_debug(cpu_env);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 4;
+            goto done_generating;
         }
 
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            gen_opc_hflags[lj] = ctx.hflags & MIPS_HFLAG_BMASK;
-            gen_opc_btarget[lj] = ctx.btarget;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
+        }
 
         is_slot = ctx.hflags & MIPS_HFLAG_BMASK;
         if (!(ctx.hflags & MIPS_HFLAG_M16)) {
@@ -19869,8 +20696,7 @@
             ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_mips16_opc(env, &ctx);
         } else {
-            generate_exception(&ctx, EXCP_RI);
-            ctx.bstate = BS_STOP;
+            generate_exception_end(&ctx, EXCP_RI);
             break;
         }
 
@@ -19881,14 +20707,18 @@
                    forbidden slot */
                 is_slot = 1;
             }
+            if ((ctx.hflags & MIPS_HFLAG_M16) &&
+                (ctx.hflags & MIPS_HFLAG_FBNSLOT)) {
+                /* Force to generate branch as microMIPS R6 doesn't restrict
+                   branches in the forbidden slot. */
+                is_slot = 1;
+            }
         }
         if (is_slot) {
             gen_branch(&ctx, insn_bytes);
         }
         ctx.pc += insn_bytes;
 
-        num_insns++;
-
         /* Execute a branch and its delay slot as a single instruction.
            This is what GDB expects and is consistent with what the
            hardware does (e.g. if a delay slot instruction faults, the
@@ -19897,8 +20727,9 @@
             break;
         }
 
-        if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0)
+        if (ctx.pc >= next_page_start) {
             break;
+        }
 
         if (tcg_op_buf_full()) {
             break;
@@ -19915,7 +20746,7 @@
     }
     if (cs->singlestep_enabled && ctx.bstate != BS_BRANCH) {
         save_cpu_state(&ctx, ctx.bstate != BS_EXCP);
-        gen_helper_0e0i(raise_exception, EXCP_DEBUG);
+        gen_helper_raise_exception_debug(cpu_env);
     } else {
         switch (ctx.bstate) {
         case BS_STOP:
@@ -19936,35 +20767,20 @@
 done_generating:
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
+
 #ifdef DEBUG_DISAS
     LOG_DISAS("\n");
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(cs, pc_start, ctx.pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
 }
 
-void gen_intermediate_code (CPUMIPSState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mips_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUMIPSState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(mips_env_get_cpu(env), tb, true);
-}
-
 static void fpu_dump_state(CPUMIPSState *env, FILE *f, fprintf_function fpu_fprintf,
                            int flags)
 {
@@ -20005,40 +20821,6 @@
 #undef printfpr
 }
 
-#if defined(TARGET_MIPS64) && defined(MIPS_DEBUG_SIGN_EXTENSIONS)
-/* Debug help: The architecture requires 32bit code to maintain proper
-   sign-extended values on 64bit machines.  */
-
-#define SIGN_EXT_P(val) ((((val) & ~0x7fffffff) == 0) || (((val) & ~0x7fffffff) == ~0x7fffffff))
-
-static void
-cpu_mips_check_sign_extensions (CPUMIPSState *env, FILE *f,
-                                fprintf_function cpu_fprintf,
-                                int flags)
-{
-    int i;
-
-    if (!SIGN_EXT_P(env->active_tc.PC))
-        cpu_fprintf(f, "BROKEN: pc=0x" TARGET_FMT_lx "\n", env->active_tc.PC);
-    if (!SIGN_EXT_P(env->active_tc.HI[0]))
-        cpu_fprintf(f, "BROKEN: HI=0x" TARGET_FMT_lx "\n", env->active_tc.HI[0]);
-    if (!SIGN_EXT_P(env->active_tc.LO[0]))
-        cpu_fprintf(f, "BROKEN: LO=0x" TARGET_FMT_lx "\n", env->active_tc.LO[0]);
-    if (!SIGN_EXT_P(env->btarget))
-        cpu_fprintf(f, "BROKEN: btarget=0x" TARGET_FMT_lx "\n", env->btarget);
-
-    for (i = 0; i < 32; i++) {
-        if (!SIGN_EXT_P(env->active_tc.gpr[i]))
-            cpu_fprintf(f, "BROKEN: %s=0x" TARGET_FMT_lx "\n", regnames[i], env->active_tc.gpr[i]);
-    }
-
-    if (!SIGN_EXT_P(env->CP0_EPC))
-        cpu_fprintf(f, "BROKEN: EPC=0x" TARGET_FMT_lx "\n", env->CP0_EPC);
-    if (!SIGN_EXT_P(env->lladdr))
-        cpu_fprintf(f, "BROKEN: LLAddr=0x" TARGET_FMT_lx "\n", env->lladdr);
-}
-#endif
-
 void mips_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
                          int flags)
 {
@@ -20061,13 +20843,15 @@
 
     cpu_fprintf(f, "CP0 Status  0x%08x Cause   0x%08x EPC    0x" TARGET_FMT_lx "\n",
                 env->CP0_Status, env->CP0_Cause, env->CP0_EPC);
-    cpu_fprintf(f, "    Config0 0x%08x Config1 0x%08x LLAddr 0x" TARGET_FMT_lx "\n",
+    cpu_fprintf(f, "    Config0 0x%08x Config1 0x%08x LLAddr 0x%016"
+                PRIx64 "\n",
                 env->CP0_Config0, env->CP0_Config1, env->lladdr);
+    cpu_fprintf(f, "    Config2 0x%08x Config3 0x%08x\n",
+                env->CP0_Config2, env->CP0_Config3);
+    cpu_fprintf(f, "    Config4 0x%08x Config5 0x%08x\n",
+                env->CP0_Config4, env->CP0_Config5);
     if (env->hflags & MIPS_HFLAG_FPU)
         fpu_dump_state(env, f, cpu_fprintf, flags);
-#if defined(TARGET_MIPS64) && defined(MIPS_DEBUG_SIGN_EXTENSIONS)
-    cpu_mips_check_sign_extensions(env, f, cpu_fprintf, flags);
-#endif
 }
 
 void mips_tcg_init(void)
@@ -20080,48 +20864,50 @@
         return;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
+
     TCGV_UNUSED(cpu_gpr[0]);
     for (i = 1; i < 32; i++)
-        cpu_gpr[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr[i] = tcg_global_mem_new(cpu_env,
                                         offsetof(CPUMIPSState, active_tc.gpr[i]),
                                         regnames[i]);
 
     for (i = 0; i < 32; i++) {
         int off = offsetof(CPUMIPSState, active_fpu.fpr[i].wr.d[0]);
         msa_wr_d[i * 2] =
-                tcg_global_mem_new_i64(TCG_AREG0, off, msaregnames[i * 2]);
+                tcg_global_mem_new_i64(cpu_env, off, msaregnames[i * 2]);
         /* The scalar floating-point unit (FPU) registers are mapped on
          * the MSA vector registers. */
         fpu_f64[i] = msa_wr_d[i * 2];
         off = offsetof(CPUMIPSState, active_fpu.fpr[i].wr.d[1]);
         msa_wr_d[i * 2 + 1] =
-                tcg_global_mem_new_i64(TCG_AREG0, off, msaregnames[i * 2 + 1]);
+                tcg_global_mem_new_i64(cpu_env, off, msaregnames[i * 2 + 1]);
     }
 
-    cpu_PC = tcg_global_mem_new(TCG_AREG0,
+    cpu_PC = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUMIPSState, active_tc.PC), "PC");
     for (i = 0; i < MIPS_DSP_ACC; i++) {
-        cpu_HI[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_HI[i] = tcg_global_mem_new(cpu_env,
                                        offsetof(CPUMIPSState, active_tc.HI[i]),
                                        regnames_HI[i]);
-        cpu_LO[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_LO[i] = tcg_global_mem_new(cpu_env,
                                        offsetof(CPUMIPSState, active_tc.LO[i]),
                                        regnames_LO[i]);
     }
-    cpu_dspctrl = tcg_global_mem_new(TCG_AREG0,
+    cpu_dspctrl = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUMIPSState, active_tc.DSPControl),
                                      "DSPControl");
-    bcond = tcg_global_mem_new(TCG_AREG0,
+    bcond = tcg_global_mem_new(cpu_env,
                                offsetof(CPUMIPSState, bcond), "bcond");
-    btarget = tcg_global_mem_new(TCG_AREG0,
+    btarget = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUMIPSState, btarget), "btarget");
-    hflags = tcg_global_mem_new_i32(TCG_AREG0,
+    hflags = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUMIPSState, hflags), "hflags");
 
-    fpu_fcr0 = tcg_global_mem_new_i32(TCG_AREG0,
+    fpu_fcr0 = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUMIPSState, active_fpu.fcr0),
                                       "fcr0");
-    fpu_fcr31 = tcg_global_mem_new_i32(TCG_AREG0,
+    fpu_fcr31 = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUMIPSState, active_fpu.fcr31),
                                        "fcr31");
 
@@ -20142,6 +20928,7 @@
     cpu = MIPS_CPU(object_new(TYPE_MIPS_CPU));
     env = &cpu->env;
     env->cpu_model = def;
+    env->exception_base = (int32_t)0xBFC00000;
 
 #ifndef CONFIG_USER_ONLY
     mmu_init(env, def);
@@ -20154,6 +20941,22 @@
     return cpu;
 }
 
+bool cpu_supports_cps_smp(const char *cpu_model)
+{
+    const mips_def_t *def = cpu_mips_find_by_name(cpu_model);
+    if (!def) {
+        return false;
+    }
+
+    return (def->CP0_Config3 & (1 << CP0C3_CMGCR)) != 0;
+}
+
+void cpu_set_exception_base(int vp_index, target_ulong address)
+{
+    MIPSCPU *vp = MIPS_CPU(qemu_get_cpu(vp_index));
+    vp->env.exception_base = address;
+}
+
 void cpu_state_reset(CPUMIPSState *env)
 {
     MIPSCPU *cpu = mips_env_get_cpu(env);
@@ -20191,7 +20994,6 @@
     }
 #endif
     env->PABITS = env->cpu_model->PABITS;
-    env->PAMask = (target_ulong)((1ULL << env->cpu_model->PABITS) - 1);
     env->CP0_SRSConf0_rw_bitmask = env->cpu_model->CP0_SRSConf0_rw_bitmask;
     env->CP0_SRSConf0 = env->cpu_model->CP0_SRSConf0;
     env->CP0_SRSConf1_rw_bitmask = env->cpu_model->CP0_SRSConf1_rw_bitmask;
@@ -20205,6 +21007,8 @@
     env->CP0_PageGrain_rw_bitmask = env->cpu_model->CP0_PageGrain_rw_bitmask;
     env->CP0_PageGrain = env->cpu_model->CP0_PageGrain;
     env->active_fpu.fcr0 = env->cpu_model->CP1_fcr0;
+    env->active_fpu.fcr31_rw_bitmask = env->cpu_model->CP1_fcr31_rw_bitmask;
+    env->active_fpu.fcr31 = env->cpu_model->CP1_fcr31;
     env->msair = env->cpu_model->MSAIR;
     env->insn_flags = env->cpu_model->insn_flags;
 
@@ -20238,20 +21042,27 @@
     if (env->hflags & MIPS_HFLAG_BMASK) {
         /* If the exception was raised from a delay slot,
            come back to the jump.  */
-        env->CP0_ErrorEPC = env->active_tc.PC - 4;
+        env->CP0_ErrorEPC = (env->active_tc.PC
+                             - (env->hflags & MIPS_HFLAG_B16 ? 2 : 4));
     } else {
         env->CP0_ErrorEPC = env->active_tc.PC;
     }
-    env->active_tc.PC = (int32_t)0xBFC00000;
+    env->active_tc.PC = env->exception_base;
     env->CP0_Random = env->tlb->nb_tlb - 1;
     env->tlb->tlb_in_use = env->tlb->nb_tlb;
     env->CP0_Wired = 0;
+    env->CP0_GlobalNumber = (cs->cpu_index & 0xFF) << CP0GN_VPId;
     env->CP0_EBase = (cs->cpu_index & 0x3FF);
     if (kvm_enabled()) {
         env->CP0_EBase |= 0x40000000;
     } else {
         env->CP0_EBase |= 0x80000000;
     }
+    if (env->CP0_Config3 & (1 << CP0C3_CMGCR)) {
+        env->CP0_CMGCRBase = 0x1fbf8000 >> 4;
+    }
+    env->CP0_EntryHi_ASID_mask = (env->CP0_Config4 & (1 << CP0C4_AE)) ?
+                                 0x3ff : 0xff;
     env->CP0_Status = (1 << CP0St_BEV) | (1 << CP0St_ERL);
     /* vectored interrupts not implemented, timer on int 7,
        no performance counters. */
@@ -20309,21 +21120,29 @@
     }
 
     compute_hflags(env);
+    restore_fp_status(env);
+    restore_pamask(env);
     cs->exception_index = EXCP_NONE;
+
+    if (semihosting_get_argc()) {
+        /* UHI interface can be used to obtain argc and argv */
+        env->active_tc.gpr[4] = -1;
+    }
 }
 
-void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->active_tc.PC = tcg_ctx.gen_opc_pc[pc_pos];
+    env->active_tc.PC = data[0];
     env->hflags &= ~MIPS_HFLAG_BMASK;
-    env->hflags |= gen_opc_hflags[pc_pos];
+    env->hflags |= data[1];
     switch (env->hflags & MIPS_HFLAG_BMASK_BASE) {
     case MIPS_HFLAG_BR:
         break;
     case MIPS_HFLAG_BC:
     case MIPS_HFLAG_BL:
     case MIPS_HFLAG_B:
-        env->btarget = gen_opc_btarget[pc_pos];
+        env->btarget = data[2];
         break;
     }
 }
diff --git a/target-mips/translate_init.c b/target-mips/translate_init.c
index eae4f0e..39ed5c4 100644
--- a/target-mips/translate_init.c
+++ b/target-mips/translate_init.c
@@ -84,6 +84,8 @@
     int32_t CP0_TCStatus_rw_bitmask;
     int32_t CP0_SRSCtl;
     int32_t CP1_fcr0;
+    int32_t CP1_fcr31_rw_bitmask;
+    int32_t CP1_fcr31;
     int32_t MSAIR;
     int32_t SEGBITS;
     int32_t PABITS;
@@ -272,6 +274,8 @@
         .CP0_Status_rw_bitmask = 0x3678FF1F,
         .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
                     (1 << FCR0_D) | (1 << FCR0_S) | (0x93 << FCR0_PRID),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 32,
         .PABITS = 32,
         .insn_flags = CPU_MIPS32R2 | ASE_MIPS16,
@@ -302,6 +306,8 @@
                     (0xff << CP0TCSt_TASID),
         .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
                     (1 << FCR0_D) | (1 << FCR0_S) | (0x95 << FCR0_PRID),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .CP0_SRSCtl = (0xf << CP0SRSCtl_HSS),
         .CP0_SRSConf0_rw_bitmask = 0x3fffffff,
         .CP0_SRSConf0 = (1U << CP0SRSC0_M) | (0x3fe << CP0SRSC0_SRS3) |
@@ -334,7 +340,7 @@
                        (1 << CP0C1_CA),
         .CP0_Config2 = MIPS_CONFIG2,
         .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_DSP2P) | (1 << CP0C3_DSPP) |
-                       (0 << CP0C3_VInt),
+                       (1 << CP0C3_VInt),
         .CP0_LLAddr_rw_bitmask = 0,
         .CP0_LLAddr_shift = 4,
         .SYNCI_Step = 32,
@@ -342,42 +348,96 @@
         .CP0_Status_rw_bitmask = 0x3778FF1F,
         .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
                     (1 << FCR0_D) | (1 << FCR0_S) | (0x93 << FCR0_PRID),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 32,
         .PABITS = 32,
         .insn_flags = CPU_MIPS32R2 | ASE_MIPS16 | ASE_DSP | ASE_DSPR2,
         .mmu_type = MMU_TYPE_R4000,
     },
     {
-        /* A generic CPU providing MIPS32 Release 5 features.
-           FIXME: Eventually this should be replaced by a real CPU model. */
-        .name = "mips32r5-generic",
-        .CP0_PRid = 0x00019700,
-        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
-                    (MMU_TYPE_R4000 << CP0C0_MT),
-        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (15 << CP0C1_MMU) |
-                       (0 << CP0C1_IS) | (3 << CP0C1_IL) | (1 << CP0C1_IA) |
-                       (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
-                       (1 << CP0C1_CA),
+        .name = "M14K",
+        .CP0_PRid = 0x00019b00,
+        /* Config1 implemented, fixed mapping MMU,
+           no virtual icache, uncached coherency. */
+        .CP0_Config0 = MIPS_CONFIG0 | (0x2 << CP0C0_KU) | (0x2 << CP0C0_K23) |
+                       (0x1 << CP0C0_AR) | (MMU_TYPE_FMT << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1,
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP),
-        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M),
-        .CP0_Config4_rw_bitmask = 0,
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_UFR) | (1 << CP0C5_LLB),
-        .CP0_Config5_rw_bitmask = (0 << CP0C5_M) | (1 << CP0C5_K) |
-                                  (1 << CP0C5_CV) | (0 << CP0C5_EVA) |
-                                  (1 << CP0C5_MSAEn) | (1 << CP0C5_UFR) |
-                                  (0 << CP0C5_NFExists),
+        .CP0_Config3 = MIPS_CONFIG3 | (0x2 << CP0C3_ISA) | (1 << CP0C3_VInt),
         .CP0_LLAddr_rw_bitmask = 0,
         .CP0_LLAddr_shift = 4,
         .SYNCI_Step = 32,
         .CCRes = 2,
-        .CP0_Status_rw_bitmask = 0x3778FF1F,
-        .CP1_fcr0 = (1 << FCR0_UFRP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x93 << FCR0_PRID),
+        .CP0_Status_rw_bitmask = 0x1258FF17,
         .SEGBITS = 32,
         .PABITS = 32,
-        .insn_flags = CPU_MIPS32R5 | ASE_MIPS16 | ASE_MSA,
+        .insn_flags = CPU_MIPS32R2 | ASE_MICROMIPS,
+        .mmu_type = MMU_TYPE_FMT,
+    },
+    {
+        .name = "M14Kc",
+        /* This is the TLB-based MMU core.  */
+        .CP0_PRid = 0x00019c00,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (15 << CP0C1_MMU) |
+                       (0 << CP0C1_IS) | (3 << CP0C1_IL) | (1 << CP0C1_IA) |
+                       (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (0x2 << CP0C3_ISA) | (0 << CP0C3_VInt),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 4,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x1278FF17,
+        .SEGBITS = 32,
+        .PABITS = 32,
+        .insn_flags = CPU_MIPS32R2 | ASE_MICROMIPS,
+        .mmu_type = MMU_TYPE_R4000,
+    },
+    {
+        /* FIXME:
+         * Config3: CMGCR, SC, PW, VZ, CTXTC, CDMM, TL
+         * Config4: MMUExtDef
+         * Config5: EVA, MRP
+         * FIR(FCR0): Has2008
+         * */
+        .name = "P5600",
+        .CP0_PRid = 0x0001A800,
+        .CP0_Config0 = MIPS_CONFIG0 | (1 << CP0C0_MM) | (1 << CP0C0_AR) |
+                    (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (0x3F << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_FP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP) |
+                       (1 << CP0C3_BP) | (1 << CP0C3_BI) | (1 << CP0C3_ULRI) |
+                       (1 << CP0C3_RXI) | (1 << CP0C3_LPA) | (1 << CP0C3_VInt),
+        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) | (2 << CP0C4_IE) |
+                       (0x1c << CP0C4_KScrExist),
+        .CP0_Config4_rw_bitmask = 0,
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_MVH) | (1 << CP0C5_LLB) |
+                       (1 << CP0C5_MRP),
+        .CP0_Config5_rw_bitmask = (1 << CP0C5_K) | (1 << CP0C5_CV) |
+                                  (1 << CP0C5_MSAEn) | (1 << CP0C5_UFE) |
+                                  (1 << CP0C5_FRE) | (1 << CP0C5_UFR),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 0,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x3C68FF1F,
+        .CP0_PageGrain_rw_bitmask = (1U << CP0PG_RIE) | (1 << CP0PG_XIE) |
+                    (1 << CP0PG_ELPA) | (1 << CP0PG_IEC),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_UFRP) | (1 << FCR0_HAS2008) |
+                    (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
+                    (1 << FCR0_D) | (1 << FCR0_S) | (0x03 << FCR0_PRID),
+        .CP1_fcr31 = (1 << FCR31_ABS2008) | (1 << FCR31_NAN2008),
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
+        .SEGBITS = 32,
+        .PABITS = 40,
+        .insn_flags = CPU_MIPS32R5 | ASE_MSA,
         .mmu_type = MMU_TYPE_R4000,
     },
     {
@@ -389,18 +449,18 @@
         .CP0_Config0 = MIPS_CONFIG0 | (0x2 << CP0C0_AR) |
                        (MMU_TYPE_R4000 << CP0C0_MT),
         .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (31 << CP0C1_MMU) |
-                       (2 << CP0C1_IS) | (3 << CP0C1_IL) | (3 << CP0C1_IA) |
-                       (2 << CP0C1_DS) | (3 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
                        (0 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_RXI) | (1 << CP0C3_BP) |
-                       (1 << CP0C3_BI) | (1 << CP0C3_ULRI) | (1 << CP0C3_MSAP) |
-                       (1U << CP0C3_M),
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_BP) | (1 << CP0C3_BI) |
+                       (2 << CP0C3_ISA) | (1 << CP0C3_ULRI) |
+                       (1 << CP0C3_RXI) | (1U << CP0C3_M),
         .CP0_Config4 = MIPS_CONFIG4 | (0xfc << CP0C4_KScrExist) |
                        (3 << CP0C4_IE) | (1U << CP0C4_M),
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_LLB),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_XNP) | (1 << CP0C5_LLB),
         .CP0_Config5_rw_bitmask = (1 << CP0C5_SBRI) | (1 << CP0C5_FRE) |
-                                  (1 << CP0C5_UFE) | (1 << CP0C5_MSAEn),
+                                  (1 << CP0C5_UFE),
         .CP0_LLAddr_rw_bitmask = 0,
         .CP0_LLAddr_shift = 0,
         .SYNCI_Step = 32,
@@ -409,12 +469,14 @@
         .CP0_PageGrain = (1 << CP0PG_IEC) | (1 << CP0PG_XIE) |
                          (1U << CP0PG_RIE),
         .CP0_PageGrain_rw_bitmask = 0,
-        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_HAS2008) | (1 << FCR0_F64) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = (1 << FCR31_ABS2008) | (1 << FCR31_NAN2008),
+        .CP1_fcr31_rw_bitmask = 0x0103FFFF,
         .SEGBITS = 32,
         .PABITS = 32,
-        .insn_flags = CPU_MIPS32R6 | ASE_MSA,
+        .insn_flags = CPU_MIPS32R6 | ASE_MICROMIPS,
         .mmu_type = MMU_TYPE_R4000,
     },
 #if defined(TARGET_MIPS64)
@@ -432,6 +494,8 @@
         .CP0_Status_rw_bitmask = 0x3678FFFF,
         /* The R4000 has a full 64bit FPU but doesn't use the fcr0 bits. */
         .CP1_fcr0 = (0x5 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0x0183FFFF,
         .SEGBITS = 40,
         .PABITS = 36,
         .insn_flags = CPU_MIPS3,
@@ -450,6 +514,8 @@
         .CP0_Status_rw_bitmask = 0x3678FFFF,
         /* The VR5432 has a full 64bit FPU but doesn't use the fcr0 bits. */
         .CP1_fcr0 = (0x54 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 40,
         .PABITS = 32,
         .insn_flags = CPU_VR54XX,
@@ -470,7 +536,7 @@
         .CP0_LLAddr_shift = 4,
         .SYNCI_Step = 32,
         .CCRes = 2,
-        .CP0_Status_rw_bitmask = 0x32F8FFFF,
+        .CP0_Status_rw_bitmask = 0x12F8FFFF,
         .SEGBITS = 42,
         .PABITS = 36,
         .insn_flags = CPU_MIPS64,
@@ -495,6 +561,8 @@
         /* The 5Kf has F64 / L / W but doesn't use the fcr0 bits. */
         .CP1_fcr0 = (1 << FCR0_D) | (1 << FCR0_S) |
                     (0x81 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 42,
         .PABITS = 36,
         .insn_flags = CPU_MIPS64,
@@ -522,6 +590,8 @@
         .CP1_fcr0 = (1 << FCR0_3D) | (1 << FCR0_PS) |
                     (1 << FCR0_D) | (1 << FCR0_S) |
                     (0x82 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 40,
         .PABITS = 36,
         .insn_flags = CPU_MIPS64 | ASE_MIPS3D,
@@ -548,33 +618,76 @@
         .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_3D) | (1 << FCR0_PS) |
                     (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
                     (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 42,
-        /* The architectural limit is 59, but we have hardcoded 36 bit
-           in some places...
-        .PABITS = 59, */ /* the architectural limit */
         .PABITS = 36,
         .insn_flags = CPU_MIPS64R2 | ASE_MIPS3D,
         .mmu_type = MMU_TYPE_R4000,
     },
     {
-        /* A generic CPU supporting MIPS64 Release 6 ISA.
-           FIXME: Support IEEE 754-2008 FP.
-                  Eventually this should be replaced by a real CPU model. */
-        .name = "MIPS64R6-generic",
-        .CP0_PRid = 0x00010000,
+        .name = "5KEc",
+        .CP0_PRid = 0x00018900,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) | (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (31 << CP0C1_MMU) |
+                       (1 << CP0C1_IS) | (4 << CP0C1_IL) | (1 << CP0C1_IA) |
+                       (1 << CP0C1_DS) | (4 << CP0C1_DL) | (1 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3,
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 4,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x12F8FFFF,
+        .SEGBITS = 42,
+        .PABITS = 36,
+        .insn_flags = CPU_MIPS64R2,
+        .mmu_type = MMU_TYPE_R4000,
+    },
+    {
+        .name = "5KEf",
+        .CP0_PRid = 0x00018900,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) | (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (31 << CP0C1_MMU) |
+                       (1 << CP0C1_IS) | (4 << CP0C1_IL) | (1 << CP0C1_IA) |
+                       (1 << CP0C1_DS) | (4 << CP0C1_DL) | (1 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3,
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 4,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x36F8FFFF,
+        .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
+                    (1 << FCR0_D) | (1 << FCR0_S) |
+                    (0x89 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .SEGBITS = 42,
+        .PABITS = 36,
+        .insn_flags = CPU_MIPS64R2,
+        .mmu_type = MMU_TYPE_R4000,
+    },
+    {
+        .name = "I6400",
+        .CP0_PRid = 0x1A900,
         .CP0_Config0 = MIPS_CONFIG0 | (0x2 << CP0C0_AR) | (0x2 << CP0C0_AT) |
                        (MMU_TYPE_R4000 << CP0C0_MT),
-        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (0x1F << CP0C1_MMU) |
-                       (2 << CP0C1_IS) | (3 << CP0C1_IL) | (3 << CP0C1_IA) |
-                       (2 << CP0C1_DS) | (3 << CP0C1_DL) | (3 << CP0C1_DA) |
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (15 << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (5 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (5 << CP0C1_DL) | (3 << CP0C1_DA) |
                        (0 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
         .CP0_Config2 = MIPS_CONFIG2,
-        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) | (1 << CP0C3_MSAP) |
+        .CP0_Config3 = MIPS_CONFIG3 | (1U << CP0C3_M) |
+                       (1 << CP0C3_CMGCR) | (1 << CP0C3_MSAP) |
                        (1 << CP0C3_BP) | (1 << CP0C3_BI) | (1 << CP0C3_ULRI) |
-                       (1 << CP0C3_RXI) | (1 << CP0C3_LPA),
-        .CP0_Config4 = MIPS_CONFIG4 | (0xfc << CP0C4_KScrExist) |
-                       (3 << CP0C4_IE) | (1U << CP0C4_M),
-        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_LLB),
+                       (1 << CP0C3_RXI) | (1 << CP0C3_LPA) | (1 << CP0C3_VInt),
+        .CP0_Config4 = MIPS_CONFIG4 | (1U << CP0C4_M) | (3 << CP0C4_IE) |
+                       (1 << CP0C4_AE) | (0xfc << CP0C4_KScrExist),
+        .CP0_Config5 = MIPS_CONFIG5 | (1 << CP0C5_XNP) | (1 << CP0C5_VP) |
+                       (1 << CP0C5_LLB) | (1 << CP0C5_MRP),
         .CP0_Config5_rw_bitmask = (1 << CP0C5_MSAEn) | (1 << CP0C5_SBRI) |
                                   (1 << CP0C5_FRE) | (1 << CP0C5_UFE),
         .CP0_LLAddr_rw_bitmask = 0,
@@ -584,12 +697,14 @@
         .CP0_Status_rw_bitmask = 0x30D8FFFF,
         .CP0_PageGrain = (1 << CP0PG_IEC) | (1 << CP0PG_XIE) |
                          (1U << CP0PG_RIE),
-        .CP0_PageGrain_rw_bitmask = 0,
-        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_F64) | (1 << FCR0_L) |
-                    (1 << FCR0_W) | (1 << FCR0_D) | (1 << FCR0_S) |
-                    (0x03 << FCR0_PRID) | (0x20 << FCR0_REV),
-        .MSAIR = 0x03 << MSAIR_ProcID | 0x20 << MSAIR_Rev,
-        .SEGBITS = 42,
+        .CP0_PageGrain_rw_bitmask = (1 << CP0PG_ELPA),
+        .CP1_fcr0 = (1 << FCR0_FREP) | (1 << FCR0_HAS2008) | (1 << FCR0_F64) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x03 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = (1 << FCR31_ABS2008) | (1 << FCR31_NAN2008),
+        .CP1_fcr31_rw_bitmask = 0x0103FFFF,
+        .MSAIR = 0x03 << MSAIR_ProcID,
+        .SEGBITS = 48,
         .PABITS = 48,
         .insn_flags = CPU_MIPS64R6 | ASE_MSA,
         .mmu_type = MMU_TYPE_R4000,
@@ -597,36 +712,42 @@
     {
         .name = "Loongson-2E",
         .CP0_PRid = 0x6302,
-        /*64KB I-cache and d-cache. 4 way with 32 bit cache line size*/
-        .CP0_Config0 = (0x1<<17) | (0x1<<16) | (0x1<<11) | (0x1<<8) | (0x1<<5) |
-                       (0x1<<4) | (0x1<<1),
-        /* Note: Config1 is only used internally, Loongson-2E has only Config0. */
+        /* 64KB I-cache and d-cache. 4 way with 32 bit cache line size.  */
+        .CP0_Config0 = (0x1<<17) | (0x1<<16) | (0x1<<11) | (0x1<<8) |
+                       (0x1<<5) | (0x1<<4) | (0x1<<1),
+        /* Note: Config1 is only used internally,
+           Loongson-2E has only Config0.  */
         .CP0_Config1 = (1 << CP0C1_FP) | (47 << CP0C1_MMU),
         .SYNCI_Step = 16,
         .CCRes = 2,
         .CP0_Status_rw_bitmask = 0x35D0FFFF,
         .CP1_fcr0 = (0x5 << FCR0_PRID) | (0x1 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 40,
         .PABITS = 40,
         .insn_flags = CPU_LOONGSON2E,
         .mmu_type = MMU_TYPE_R4000,
     },
     {
-      .name = "Loongson-2F",
-      .CP0_PRid = 0x6303,
-      /*64KB I-cache and d-cache. 4 way with 32 bit cache line size*/
-      .CP0_Config0 = (0x1<<17) | (0x1<<16) | (0x1<<11) | (0x1<<8) | (0x1<<5) |
-                     (0x1<<4) | (0x1<<1),
-      /* Note: Config1 is only used internally, Loongson-2F has only Config0. */
-      .CP0_Config1 = (1 << CP0C1_FP) | (47 << CP0C1_MMU),
-      .SYNCI_Step = 16,
-      .CCRes = 2,
-      .CP0_Status_rw_bitmask = 0xF5D0FF1F,   /*bit5:7 not writable*/
-      .CP1_fcr0 = (0x5 << FCR0_PRID) | (0x1 << FCR0_REV),
-      .SEGBITS = 40,
-      .PABITS = 40,
-      .insn_flags = CPU_LOONGSON2F,
-      .mmu_type = MMU_TYPE_R4000,
+        .name = "Loongson-2F",
+        .CP0_PRid = 0x6303,
+        /* 64KB I-cache and d-cache. 4 way with 32 bit cache line size.  */
+        .CP0_Config0 = (0x1<<17) | (0x1<<16) | (0x1<<11) | (0x1<<8) |
+                       (0x1<<5) | (0x1<<4) | (0x1<<1),
+        /* Note: Config1 is only used internally,
+           Loongson-2F has only Config0.  */
+        .CP0_Config1 = (1 << CP0C1_FP) | (47 << CP0C1_MMU),
+        .SYNCI_Step = 16,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0xF5D0FF1F,   /* Bits 7:5 not writable.  */
+        .CP1_fcr0 = (0x5 << FCR0_PRID) | (0x1 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
+        .SEGBITS = 40,
+        .PABITS = 40,
+        .insn_flags = CPU_LOONGSON2F,
+        .mmu_type = MMU_TYPE_R4000,
     },
     {
         /* A generic CPU providing MIPS64 ASE DSP 2 features.
@@ -650,10 +771,9 @@
         .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_3D) | (1 << FCR0_PS) |
                     (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
                     (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .CP1_fcr31 = 0,
+        .CP1_fcr31_rw_bitmask = 0xFF83FFFF,
         .SEGBITS = 42,
-        /* The architectural limit is 59, but we have hardcoded 36 bit
-           in some places...
-        .PABITS = 59, */ /* the architectural limit */
         .PABITS = 36,
         .insn_flags = CPU_MIPS64R2 | ASE_DSP | ASE_DSPR2,
         .mmu_type = MMU_TYPE_R4000,
@@ -785,6 +905,8 @@
        - round to nearest / ties to even (RM bits are 0) */
     env->active_tc.msacsr = 0;
 
+    restore_msa_fp_status(env);
+
     /* tininess detected after rounding.*/
     set_float_detect_tininess(float_tininess_after_rounding,
                               &env->active_tc.msa_fp_status);
@@ -792,14 +914,9 @@
     /* clear float_status exception flags */
     set_float_exception_flags(0, &env->active_tc.msa_fp_status);
 
-    /* set float_status rounding mode */
-    set_float_rounding_mode(float_round_nearest_even,
-                            &env->active_tc.msa_fp_status);
-
-    /* set float_status flush modes */
-    set_flush_to_zero(0, &env->active_tc.msa_fp_status);
-    set_flush_inputs_to_zero(0, &env->active_tc.msa_fp_status);
-
     /* clear float_status nan mode */
     set_default_nan_mode(0, &env->active_tc.msa_fp_status);
+
+    /* set proper signanling bit meaning ("1" means "quiet") */
+    set_snan_bit_is_one(0, &env->active_tc.msa_fp_status);
 }
diff --git a/target-moxie/cpu.c b/target-moxie/cpu.c
index 47b617f..50a0899 100644
--- a/target-moxie/cpu.c
+++ b/target-moxie/cpu.c
@@ -17,10 +17,13 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
 #include "machine.h"
+#include "exec/exec-all.h"
 
 static void moxie_cpu_set_pc(CPUState *cs, vaddr value)
 {
@@ -48,6 +51,12 @@
     tlb_flush(s, 1);
 }
 
+static void moxie_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_arch_moxie;
+    info->print_insn = print_insn_moxie;
+}
+
 static void moxie_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
@@ -66,7 +75,7 @@
     static int inited;
 
     cs->env_ptr = &cpu->env;
-    cpu_exec_init(&cpu->env);
+    cpu_exec_init(cs, &error_abort);
 
     if (tcg_enabled() && !inited) {
         inited = 1;
@@ -114,6 +123,14 @@
     cc->get_phys_page_debug = moxie_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_moxie_cpu;
 #endif
+    cc->disas_set_info = moxie_cpu_disas_set_info;
+
+    /*
+     * Reason: moxie_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void moxielite_initfn(Object *obj)
diff --git a/target-moxie/cpu.h b/target-moxie/cpu.h
index c5b12a5..3e880fa 100644
--- a/target-moxie/cpu.h
+++ b/target-moxie/cpu.h
@@ -16,20 +16,16 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef _CPU_MOXIE_H
-#define _CPU_MOXIE_H
 
-#include "config.h"
+#ifndef MOXIE_CPU_H
+#define MOXIE_CPU_H
+
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 32
 
 #define CPUArchState struct CPUMoxieState
 
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE     0xFEED /* EM_MOXIE */
-
 #define MOXIE_EX_DIV0        0
 #define MOXIE_EX_BAD         1
 #define MOXIE_EX_IRQ         2
@@ -114,7 +110,6 @@
 #define ENV_OFFSET offsetof(MoxieCPU, env)
 
 MoxieCPU *cpu_moxie_init(const char *cpu_model);
-int cpu_moxie_exec(CPUMoxieState *s);
 void moxie_cpu_do_interrupt(CPUState *cs);
 void moxie_cpu_dump_state(CPUState *cpu, FILE *f,
                           fprintf_function cpu_fprintf, int flags);
@@ -123,29 +118,19 @@
 int cpu_moxie_signal_handler(int host_signum, void *pinfo,
                              void *puc);
 
-static inline CPUMoxieState *cpu_init(const char *cpu_model)
-{
-    MoxieCPU *cpu = cpu_moxie_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_moxie_init(cpu_model))
 
-#define cpu_exec cpu_moxie_exec
-#define cpu_gen_code cpu_moxie_gen_code
 #define cpu_signal_handler cpu_moxie_signal_handler
 
-static inline int cpu_mmu_index(CPUMoxieState *env)
+static inline int cpu_mmu_index(CPUMoxieState *env, bool ifetch)
 {
     return 0;
 }
 
 #include "exec/cpu-all.h"
-#include "exec/exec-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUMoxieState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->pc;
     *cs_base = 0;
@@ -155,4 +140,4 @@
 int moxie_cpu_handle_mmu_fault(CPUState *cpu, vaddr address,
                                int rw, int mmu_idx);
 
-#endif /* _CPU_MOXIE_H */
+#endif /* MOXIE_CPU_H */
diff --git a/target-moxie/helper.c b/target-moxie/helper.c
index 6c98965..330299f 100644
--- a/target-moxie/helper.c
+++ b/target-moxie/helper.c
@@ -17,11 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/exec-all.h"
@@ -32,12 +29,12 @@
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = moxie_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = moxie_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             cpu_restore_state(cs, retaddr);
@@ -56,7 +53,7 @@
     /* Stash the address where the exception occurred.  */
     cpu_restore_state(cs, GETPC());
     env->sregs[5] = env->pc;
-    /* Jump the the exception handline routine.  */
+    /* Jump to the exception handline routine.  */
     env->pc = env->sregs[1];
     cpu_loop_exit(cs);
 }
diff --git a/target-moxie/machine.c b/target-moxie/machine.c
index da1a857..282dcd8 100644
--- a/target-moxie/machine.c
+++ b/target-moxie/machine.c
@@ -1,5 +1,10 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "machine.h"
+#include "migration/cpu.h"
 
 const VMStateDescription vmstate_moxie_cpu = {
     .name = "cpu",
diff --git a/target-moxie/mmu.c b/target-moxie/mmu.c
index 5217eed..9203330 100644
--- a/target-moxie/mmu.c
+++ b/target-moxie/mmu.c
@@ -17,11 +17,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/exec-all.h"
diff --git a/target-moxie/mmu.h b/target-moxie/mmu.h
index e01ffc2..284a44d 100644
--- a/target-moxie/mmu.h
+++ b/target-moxie/mmu.h
@@ -6,11 +6,6 @@
 typedef struct {
     uint32_t phy;
     uint32_t pfn;
-    int g:1;
-    int v:1;
-    int k:1;
-    int w:1;
-    int e:1;
     int cause_op;
 } MoxieMMUResult;
 
diff --git a/target-moxie/translate.c b/target-moxie/translate.c
index cc9af66..0660b44 100644
--- a/target-moxie/translate.c
+++ b/target-moxie/translate.c
@@ -21,12 +21,7 @@
  *    http://moxielogic.org/wiki
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -36,6 +31,7 @@
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
+#include "exec/log.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
@@ -60,7 +56,7 @@
 
 static TCGv cpu_pc;
 static TCGv cpu_gregs[16];
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cc_a, cc_b;
 
 #include "exec/gen-icount.h"
@@ -110,32 +106,42 @@
         return;
     }
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    tcg_ctx.tcg_env = cpu_env;
+    cpu_pc = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUMoxieState, pc), "$pc");
     for (i = 0; i < 16; i++)
-        cpu_gregs[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_gregs[i] = tcg_global_mem_new_i32(cpu_env,
                                               offsetof(CPUMoxieState, gregs[i]),
                                               gregnames[i]);
 
-    cc_a = tcg_global_mem_new_i32(TCG_AREG0,
+    cc_a = tcg_global_mem_new_i32(cpu_env,
                                   offsetof(CPUMoxieState, cc_a), "cc_a");
-    cc_b = tcg_global_mem_new_i32(TCG_AREG0,
+    cc_b = tcg_global_mem_new_i32(cpu_env,
                                   offsetof(CPUMoxieState, cc_b), "cc_b");
 
     done_init = 1;
 }
 
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+    if (unlikely(ctx->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(CPUMoxieState *env, DisasContext *ctx,
                                int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = ctx->tb;
-
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-        !ctx->singlestep_enabled) {
+    if (use_goto_tb(ctx, dest)) {
         tcg_gen_goto_tb(n);
         tcg_gen_movi_i32(cpu_pc, dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
     } else {
         tcg_gen_movi_i32(cpu_pc, dest);
         if (ctx->singlestep_enabled) {
@@ -154,10 +160,6 @@
     /* Set the default instruction length.  */
     int length = 2;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     /* Examine the 16-bit opcode.  */
     opcode = ctx->opcode;
 
@@ -169,7 +171,7 @@
 
 #define BRANCH(cond)                                                         \
     do {                                                                     \
-        int l1 = gen_new_label();                                            \
+        TCGLabel *l1 = gen_new_label();                                      \
         tcg_gen_brcond_i32(cond, cc_a, cc_b, l1);                            \
         gen_goto_tb(env, ctx, 1, ctx->pc+2);                                 \
         gen_set_label(l1);                                                   \
@@ -820,17 +822,13 @@
 }
 
 /* generate intermediate code for basic block 'tb'.  */
-static inline void
-gen_intermediate_code_internal(MoxieCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUMoxieState *env, struct TranslationBlock *tb)
 {
+    MoxieCPU *cpu = moxie_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
-    CPUMoxieState *env = &cpu->env;
-    int num_insns;
+    int num_insns, max_insns;
 
     pc_start = tb->pc;
     ctx.pc = pc_start;
@@ -840,40 +838,40 @@
     ctx.singlestep_enabled = 0;
     ctx.bstate = BS_NONE;
     num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (ctx.pc == bp->pc) {
-                    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-                    gen_helper_debug(cpu_env);
-                    ctx.bstate = BS_EXCP;
-                    goto done_generating;
-                }
-            }
-        }
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
-        ctx.opcode = cpu_lduw_code(env, ctx.pc);
-        ctx.pc += decode_opc(cpu, &ctx);
+        tcg_gen_insn_start(ctx.pc);
         num_insns++;
 
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            tcg_gen_movi_i32(cpu_pc, ctx.pc);
+            gen_helper_debug(cpu_env);
+            ctx.bstate = BS_EXCP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 2;
+            goto done_generating;
+        }
+
+        ctx.opcode = cpu_lduw_code(env, ctx.pc);
+        ctx.pc += decode_opc(cpu, &ctx);
+
+        if (num_insns >= max_insns) {
+            break;
+        }
         if (cs->singlestep_enabled) {
             break;
         }
-
         if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0) {
             break;
         }
@@ -899,29 +897,12 @@
  done_generating:
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 }
 
-void gen_intermediate_code(CPUMoxieState *env, struct TranslationBlock *tb)
+void restore_state_to_opc(CPUMoxieState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(moxie_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUMoxieState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(moxie_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUMoxieState *env, TranslationBlock *tb, int pc_pos)
-{
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-openrisc/cpu.c b/target-openrisc/cpu.c
index 39bedc1..155913f 100644
--- a/target-openrisc/cpu.c
+++ b/target-openrisc/cpu.c
@@ -17,8 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
+#include "exec/exec-all.h"
 
 static void openrisc_cpu_set_pc(CPUState *cs, vaddr value)
 {
@@ -92,7 +95,7 @@
     static int inited;
 
     cs->env_ptr = &cpu->env;
-    cpu_exec_init(&cpu->env);
+    cpu_exec_init(cs, &error_abort);
 
 #ifndef CONFIG_USER_ONLY
     cpu_openrisc_mmu_init(cpu);
@@ -177,6 +180,13 @@
     dc->vmsd = &vmstate_openrisc_cpu;
 #endif
     cc->gdb_num_core_regs = 32 + 3;
+
+    /*
+     * Reason: openrisc_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void cpu_register(const OpenRISCCPUInfo *info)
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index 69b96c6..aaf1535 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -17,18 +17,16 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef CPU_OPENRISC_H
-#define CPU_OPENRISC_H
+#ifndef OPENRISC_CPU_H
+#define OPENRISC_CPU_H
 
 #define TARGET_LONG_BITS 32
-#define ELF_MACHINE    EM_OPENRISC
 
 #define CPUArchState struct CPUOpenRISCState
 
 /* cpu_openrisc_map_address_* in CPUOpenRISCTLBContext need this decl.  */
 struct OpenRISCCPU;
 
-#include "config.h"
 #include "qemu-common.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
@@ -346,7 +344,6 @@
 OpenRISCCPU *cpu_openrisc_init(const char *cpu_model);
 
 void cpu_openrisc_list(FILE *f, fprintf_function cpu_fprintf);
-int cpu_openrisc_exec(CPUOpenRISCState *s);
 void openrisc_cpu_do_interrupt(CPUState *cpu);
 bool openrisc_cpu_exec_interrupt(CPUState *cpu, int int_req);
 void openrisc_cpu_dump_state(CPUState *cpu, FILE *f,
@@ -360,8 +357,6 @@
 int cpu_openrisc_signal_handler(int host_signum, void *pinfo, void *puc);
 
 #define cpu_list cpu_openrisc_list
-#define cpu_exec cpu_openrisc_exec
-#define cpu_gen_code cpu_openrisc_gen_code
 #define cpu_signal_handler cpu_openrisc_signal_handler
 
 #ifndef CONFIG_USER_ONLY
@@ -389,20 +384,13 @@
                                int *prot, target_ulong address, int rw);
 #endif
 
-static inline CPUOpenRISCState *cpu_init(const char *cpu_model)
-{
-    OpenRISCCPU *cpu = cpu_openrisc_init(cpu_model);
-    if (cpu) {
-        return &cpu->env;
-    }
-    return NULL;
-}
+#define cpu_init(cpu_model) CPU(cpu_openrisc_init(cpu_model))
 
 #include "exec/cpu-all.h"
 
 static inline void cpu_get_tb_cpu_state(CPUOpenRISCState *env,
                                         target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->pc;
     *cs_base = 0;
@@ -410,7 +398,7 @@
     *flags = (env->flags & D_FLAG);
 }
 
-static inline int cpu_mmu_index(CPUOpenRISCState *env)
+static inline int cpu_mmu_index(CPUOpenRISCState *env, bool ifetch)
 {
     if (!(env->sr & SR_IME)) {
         return MMU_NOMMU_IDX;
@@ -420,11 +408,4 @@
 
 #define CPU_INTERRUPT_TIMER   CPU_INTERRUPT_TGT_INT_0
 
-#include "exec/exec-all.h"
-
-static inline target_ulong cpu_get_pc(CPUOpenRISCState *env)
-{
-    return env->pc;
-}
-
-#endif /* CPU_OPENRISC_H */
+#endif /* OPENRISC_CPU_H */
diff --git a/target-openrisc/exception.c b/target-openrisc/exception.c
index 74652a5..49470be 100644
--- a/target-openrisc/exception.c
+++ b/target-openrisc/exception.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exception.h"
 
 void QEMU_NORETURN raise_exception(OpenRISCCPU *cpu, uint32_t excp)
diff --git a/target-openrisc/exception.h b/target-openrisc/exception.h
index 4b64430..4ec56b4 100644
--- a/target-openrisc/exception.h
+++ b/target-openrisc/exception.h
@@ -17,12 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef QEMU_OPENRISC_EXCP_H
-#define QEMU_OPENRISC_EXCP_H
+#ifndef TARGET_OPENRISC_EXCEPTION_H
+#define TARGET_OPENRISC_EXCEPTION_H
 
 #include "cpu.h"
 #include "qemu-common.h"
 
 void QEMU_NORETURN raise_exception(OpenRISCCPU *cpu, uint32_t excp);
 
-#endif /* QEMU_OPENRISC_EXCP_H */
+#endif /* TARGET_OPENRISC_EXCEPTION_H */
diff --git a/target-openrisc/exception_helper.c b/target-openrisc/exception_helper.c
index 6093953..329a9e4 100644
--- a/target-openrisc/exception_helper.c
+++ b/target-openrisc/exception_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exception.h"
diff --git a/target-openrisc/fpu_helper.c b/target-openrisc/fpu_helper.c
index c94ed35..c54404b 100644
--- a/target-openrisc/fpu_helper.c
+++ b/target-openrisc/fpu_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exception.h"
diff --git a/target-openrisc/gdbstub.c b/target-openrisc/gdbstub.c
index 18bcc46..cb16e76 100644
--- a/target-openrisc/gdbstub.c
+++ b/target-openrisc/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 int openrisc_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
diff --git a/target-openrisc/int_helper.c b/target-openrisc/int_helper.c
index 6e27aeb..4d1f958 100644
--- a/target-openrisc/int_helper.c
+++ b/target-openrisc/int_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "exception.h"
diff --git a/target-openrisc/interrupt.c b/target-openrisc/interrupt.c
index e480cfd..5fe3f11 100644
--- a/target-openrisc/interrupt.c
+++ b/target-openrisc/interrupt.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 #include "qemu/host-utils.h"
diff --git a/target-openrisc/interrupt_helper.c b/target-openrisc/interrupt_helper.c
index 55a780c..116f9109 100644
--- a/target-openrisc/interrupt_helper.c
+++ b/target-openrisc/interrupt_helper.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 
 void HELPER(rfe)(CPUOpenRISCState *env)
diff --git a/target-openrisc/machine.c b/target-openrisc/machine.c
index 9f66a9c..17b0c77 100644
--- a/target-openrisc/machine.c
+++ b/target-openrisc/machine.c
@@ -17,8 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
+#include "migration/cpu.h"
 
 static const VMStateDescription vmstate_env = {
     .name = "env",
diff --git a/target-openrisc/mmu.c b/target-openrisc/mmu.c
index 750a936..505dcdc 100644
--- a/target-openrisc/mmu.c
+++ b/target-openrisc/mmu.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu-common.h"
 #include "exec/gdbstub.h"
 #include "qemu/host-utils.h"
diff --git a/target-openrisc/mmu_helper.c b/target-openrisc/mmu_helper.c
index ee1c6f6..a44d0aa 100644
--- a/target-openrisc/mmu_helper.c
+++ b/target-openrisc/mmu_helper.c
@@ -18,17 +18,19 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
 
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write,
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
               int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = openrisc_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = openrisc_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
 
     if (ret) {
         if (retaddr) {
diff --git a/target-openrisc/sys_helper.c b/target-openrisc/sys_helper.c
index 53ca6bc..a719e45 100644
--- a/target-openrisc/sys_helper.c
+++ b/target-openrisc/sys_helper.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 
 #define TO_SPR(group, number) (((group) << 11) + (number))
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index fdc1a16..28c9446 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -18,13 +18,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu-common.h"
 #include "qemu/log.h"
-#include "config.h"
 #include "qemu/bitops.h"
 #include "exec/cpu_ldst.h"
 
@@ -32,6 +32,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define OPENRISC_DISAS
@@ -52,7 +53,7 @@
     uint32_t delayed_branch;
 } DisasContext;
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_sr;
 static TCGv cpu_R[32];
 static TCGv cpu_pc;
@@ -77,39 +78,40 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_sr = tcg_global_mem_new(TCG_AREG0,
+    tcg_ctx.tcg_env = cpu_env;
+    cpu_sr = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUOpenRISCState, sr), "sr");
-    env_flags = tcg_global_mem_new_i32(TCG_AREG0,
+    env_flags = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUOpenRISCState, flags),
                                        "flags");
-    cpu_pc = tcg_global_mem_new(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUOpenRISCState, pc), "pc");
-    cpu_npc = tcg_global_mem_new(TCG_AREG0,
+    cpu_npc = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUOpenRISCState, npc), "npc");
-    cpu_ppc = tcg_global_mem_new(TCG_AREG0,
+    cpu_ppc = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUOpenRISCState, ppc), "ppc");
-    jmp_pc = tcg_global_mem_new(TCG_AREG0,
+    jmp_pc = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUOpenRISCState, jmp_pc), "jmp_pc");
-    env_btaken = tcg_global_mem_new_i32(TCG_AREG0,
+    env_btaken = tcg_global_mem_new_i32(cpu_env,
                                         offsetof(CPUOpenRISCState, btaken),
                                         "btaken");
-    fpcsr = tcg_global_mem_new_i32(TCG_AREG0,
+    fpcsr = tcg_global_mem_new_i32(cpu_env,
                                    offsetof(CPUOpenRISCState, fpcsr),
                                    "fpcsr");
-    machi = tcg_global_mem_new(TCG_AREG0,
+    machi = tcg_global_mem_new(cpu_env,
                                offsetof(CPUOpenRISCState, machi),
                                "machi");
-    maclo = tcg_global_mem_new(TCG_AREG0,
+    maclo = tcg_global_mem_new(cpu_env,
                                offsetof(CPUOpenRISCState, maclo),
                                "maclo");
-    fpmaddhi = tcg_global_mem_new(TCG_AREG0,
+    fpmaddhi = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUOpenRISCState, fpmaddhi),
                                   "fpmaddhi");
-    fpmaddlo = tcg_global_mem_new(TCG_AREG0,
+    fpmaddlo = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUOpenRISCState, fpmaddlo),
                                   "fpmaddlo");
     for (i = 0; i < 32; i++) {
-        cpu_R[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new(cpu_env,
                                       offsetof(CPUOpenRISCState, gpr[i]),
                                       regnames[i]);
     }
@@ -118,9 +120,7 @@
 /* Writeback SR_F translation space to execution space.  */
 static inline void wb_SR_F(void)
 {
-    int label;
-
-    label = gen_new_label();
+    TCGLabel *label = gen_new_label();
     tcg_gen_andi_tl(cpu_sr, cpu_sr, ~SR_F);
     tcg_gen_brcondi_tl(TCG_COND_EQ, env_btaken, 0, label);
     tcg_gen_ori_tl(cpu_sr, cpu_sr, SR_F);
@@ -191,15 +191,25 @@
 }
 #endif*/
 
+static inline bool use_goto_tb(DisasContext *dc, target_ulong dest)
+{
+    if (unlikely(dc->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (dc->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = dc->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-                                       likely(!dc->singlestep_enabled)) {
+    if (use_goto_tb(dc, dest)) {
         tcg_gen_movi_tl(cpu_pc, dest);
         tcg_gen_goto_tb(n);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)dc->tb + n);
     } else {
         tcg_gen_movi_tl(cpu_pc, dest);
         if (dc->singlestep_enabled) {
@@ -226,7 +236,7 @@
     case 0x03:     /* l.bnf */
     case 0x04:     /* l.bf  */
         {
-            int lab = gen_new_label();
+            TCGLabel *lab = gen_new_label();
             TCGv sr_f = tcg_temp_new();
             tcg_gen_movi_tl(jmp_pc, dc->pc+8);
             tcg_gen_andi_tl(sr_f, cpu_sr, SR_F);
@@ -272,7 +282,7 @@
         case 0x00:    /* l.add */
             LOG_DIS("l.add r%d, r%d, r%d\n", rd, ra, rb);
             {
-                int lab = gen_new_label();
+                TCGLabel *lab = gen_new_label();
                 TCGv_i64 ta = tcg_temp_new_i64();
                 TCGv_i64 tb = tcg_temp_new_i64();
                 TCGv_i64 td = tcg_temp_local_new_i64();
@@ -281,7 +291,7 @@
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(tb, cpu_R[rb]);
                 tcg_gen_add_i64(td, ta, tb);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 31);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -311,7 +321,7 @@
         case 0x00:
             LOG_DIS("l.addc r%d, r%d, r%d\n", rd, ra, rb);
             {
-                int lab = gen_new_label();
+                TCGLabel *lab = gen_new_label();
                 TCGv_i64 ta = tcg_temp_new_i64();
                 TCGv_i64 tb = tcg_temp_new_i64();
                 TCGv_i64 tcy = tcg_temp_local_new_i64();
@@ -326,7 +336,7 @@
                 tcg_gen_shri_i64(tcy, tcy, 10);
                 tcg_gen_add_i64(td, ta, tb);
                 tcg_gen_add_i64(td, td, tcy);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 32);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -358,7 +368,7 @@
         case 0x00:
             LOG_DIS("l.sub r%d, r%d, r%d\n", rd, ra, rb);
             {
-                int lab = gen_new_label();
+                TCGLabel *lab = gen_new_label();
                 TCGv_i64 ta = tcg_temp_new_i64();
                 TCGv_i64 tb = tcg_temp_new_i64();
                 TCGv_i64 td = tcg_temp_local_new_i64();
@@ -368,7 +378,7 @@
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(tb, cpu_R[rb]);
                 tcg_gen_sub_i64(td, ta, tb);
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 31);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -450,10 +460,10 @@
         case 0x03:    /* l.div */
             LOG_DIS("l.div r%d, r%d, r%d\n", rd, ra, rb);
             {
-                int lab0 = gen_new_label();
-                int lab1 = gen_new_label();
-                int lab2 = gen_new_label();
-                int lab3 = gen_new_label();
+                TCGLabel *lab0 = gen_new_label();
+                TCGLabel *lab1 = gen_new_label();
+                TCGLabel *lab2 = gen_new_label();
+                TCGLabel *lab3 = gen_new_label();
                 TCGv_i32 sr_ove = tcg_temp_local_new_i32();
                 if (rb == 0) {
                     tcg_gen_ori_tl(cpu_sr, cpu_sr, (SR_OV | SR_CY));
@@ -492,9 +502,9 @@
         case 0x03:    /* l.divu */
             LOG_DIS("l.divu r%d, r%d, r%d\n", rd, ra, rb);
             {
-                int lab0 = gen_new_label();
-                int lab1 = gen_new_label();
-                int lab2 = gen_new_label();
+                TCGLabel *lab0 = gen_new_label();
+                TCGLabel *lab1 = gen_new_label();
+                TCGLabel *lab2 = gen_new_label();
                 TCGv_i32 sr_ove = tcg_temp_local_new_i32();
                 if (rb == 0) {
                     tcg_gen_ori_tl(cpu_sr, cpu_sr, (SR_OV | SR_CY));
@@ -533,7 +543,7 @@
                 TCGv_i64 trb = tcg_temp_local_new_i64();
                 TCGv_i64 high = tcg_temp_new_i64();
                 TCGv_i32 sr_ove = tcg_temp_local_new_i32();
-                int lab = gen_new_label();
+                TCGLabel *lab = gen_new_label();
                 /* Calculate each result. */
                 tcg_gen_extu_i32_i64(tra, cpu_R[ra]);
                 tcg_gen_extu_i32_i64(trb, cpu_R[rb]);
@@ -568,7 +578,7 @@
         case 0x00:    /* l.cmov */
             LOG_DIS("l.cmov r%d, r%d, r%d\n", rd, ra, rb);
             {
-                int lab = gen_new_label();
+                TCGLabel *lab = gen_new_label();
                 TCGv res = tcg_temp_local_new();
                 TCGv sr_f = tcg_temp_new();
                 tcg_gen_andi_tl(sr_f, cpu_sr, SR_F);
@@ -781,9 +791,9 @@
             tcg_gen_ext_i32_i64(t1, dst);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_add_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(dst);
             tcg_temp_free(ttmp);
             tcg_temp_free_i64(t1);
@@ -893,14 +903,14 @@
             if (I16 == 0) {
                 tcg_gen_mov_tl(cpu_R[rd], cpu_R[ra]);
             } else {
-                int lab = gen_new_label();
+                TCGLabel *lab = gen_new_label();
                 TCGv_i64 ta = tcg_temp_new_i64();
                 TCGv_i64 td = tcg_temp_local_new_i64();
                 TCGv_i32 res = tcg_temp_local_new_i32();
                 TCGv_i32 sr_ove = tcg_temp_local_new_i32();
                 tcg_gen_extu_i32_i64(ta, cpu_R[ra]);
                 tcg_gen_addi_i64(td, ta, sign_extend(I16, 16));
-                tcg_gen_trunc_i64_i32(res, td);
+                tcg_gen_extrl_i64_i32(res, td);
                 tcg_gen_shri_i64(td, td, 32);
                 tcg_gen_andi_i64(td, td, 0x3);
                 /* Jump to lab when no overflow.  */
@@ -923,7 +933,7 @@
     case 0x28:    /* l.addic */
         LOG_DIS("l.addic r%d, r%d, %d\n", rd, ra, I16);
         {
-            int lab = gen_new_label();
+            TCGLabel *lab = gen_new_label();
             TCGv_i64 ta = tcg_temp_new_i64();
             TCGv_i64 td = tcg_temp_local_new_i64();
             TCGv_i64 tcy = tcg_temp_local_new_i64();
@@ -936,7 +946,7 @@
             tcg_gen_extu_i32_i64(tcy, sr_cy);
             tcg_gen_addi_i64(td, ta, sign_extend(I16, 16));
             tcg_gen_add_i64(td, td, tcy);
-            tcg_gen_trunc_i64_i32(res, td);
+            tcg_gen_extrl_i64_i32(res, td);
             tcg_gen_shri_i64(td, td, 32);
             tcg_gen_andi_i64(td, td, 0x3);
             /* Jump to lab when no overflow.  */
@@ -1075,9 +1085,9 @@
             tcg_gen_ext_i32_i64(t1, t0);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_add_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(t0);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(t2);
@@ -1094,9 +1104,9 @@
             tcg_gen_ext_i32_i64(t1, t0);
             tcg_gen_concat_i32_i64(t2, maclo, machi);
             tcg_gen_sub_i64(t2, t2, t1);
-            tcg_gen_trunc_i64_i32(maclo, t2);
+            tcg_gen_extrl_i64_i32(maclo, t2);
             tcg_gen_shri_i64(t2, t2, 32);
-            tcg_gen_trunc_i64_i32(machi, t2);
+            tcg_gen_extrl_i64_i32(machi, t2);
             tcg_temp_free_i32(t0);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(t2);
@@ -1320,7 +1330,7 @@
 #ifdef OPENRISC_DISAS
     uint32_t K16;
 #endif
-    op0 = extract32(insn, 16, 8);
+    op0 = extract32(insn, 16, 10);
 #ifdef OPENRISC_DISAS
     K16 = extract32(insn, 0, 16);
 #endif
@@ -1620,30 +1630,12 @@
     }
 }
 
-static void check_breakpoint(OpenRISCCPU *cpu, DisasContext *dc)
+void gen_intermediate_code(CPUOpenRISCState *env, struct TranslationBlock *tb)
 {
-    CPUState *cs = CPU(cpu);
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_tl(cpu_pc, dc->pc);
-                gen_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-            }
-        }
-    }
-}
-
-static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  int search_pc)
-{
+    OpenRISCCPU *cpu = openrisc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
     struct DisasContext ctx, *dc = &ctx;
     uint32_t pc_start;
-    int j, k;
     uint32_t next_page_start;
     int num_insns;
     int max_insns;
@@ -1655,7 +1647,7 @@
     dc->ppc = pc_start;
     dc->pc = pc_start;
     dc->flags = cpu->env.cpucfgr;
-    dc->mem_idx = cpu_mmu_index(&cpu->env);
+    dc->mem_idx = cpu_mmu_index(&cpu->env, false);
     dc->synced_flags = dc->tb_flags = tb->flags;
     dc->delayed_branch = !!(dc->tb_flags & D_FLAG);
     dc->singlestep_enabled = cs->singlestep_enabled;
@@ -1665,36 +1657,35 @@
     }
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    k = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
 
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
 
     do {
-        check_breakpoint(cpu, dc);
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (k < j) {
-                k++;
-                while (k < j) {
-                    tcg_ctx.gen_opc_instr_start[k++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[k] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[k] = 1;
-            tcg_ctx.gen_opc_icount[k] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            gen_exception(dc, EXCP_DEBUG);
+            dc->is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            break;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc->pc);
-        }
-
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
         dc->ppc = dc->pc - 4;
@@ -1703,7 +1694,6 @@
         tcg_gen_movi_tl(cpu_npc, dc->npc);
         disas_openrisc_insn(dc, cpu);
         dc->pc = dc->npc;
-        num_insns++;
         /* delay slot */
         if (dc->delayed_branch) {
             dc->delayed_branch--;
@@ -1758,38 +1748,20 @@
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        k++;
-        while (k <= j) {
-            tcg_ctx.gen_opc_instr_start[k++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("\n");
-        log_target_disas(&cpu->env, pc_start, dc->pc - pc_start, 0);
+        log_target_disas(cs, pc_start, dc->pc - pc_start, 0);
         qemu_log("\nisize=%d osize=%d\n",
                  dc->pc - pc_start, tcg_op_buf_count());
     }
 #endif
 }
 
-void gen_intermediate_code(CPUOpenRISCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(openrisc_env_get_cpu(env), tb, 0);
-}
-
-void gen_intermediate_code_pc(CPUOpenRISCState *env,
-                              struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(openrisc_env_get_cpu(env), tb, 1);
-}
-
 void openrisc_cpu_dump_state(CPUState *cs, FILE *f,
                              fprintf_function cpu_fprintf,
                              int flags)
@@ -1806,7 +1778,7 @@
 }
 
 void restore_state_to_opc(CPUOpenRISCState *env, TranslationBlock *tb,
-                          int pc_pos)
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-ppc/Makefile.objs b/target-ppc/Makefile.objs
index a7ae392..e667e69 100644
--- a/target-ppc/Makefile.objs
+++ b/target-ppc/Makefile.objs
@@ -1,10 +1,10 @@
 obj-y += cpu-models.o
 obj-y += translate.o
 ifeq ($(CONFIG_SOFTMMU),y)
-obj-y += machine.o mmu_helper.o mmu-hash32.o
+obj-y += machine.o mmu_helper.o mmu-hash32.o monitor.o
 obj-$(TARGET_PPC64) += mmu-hash64.o arch_dump.o
 endif
-obj-$(CONFIG_KVM) += kvm.o kvm_ppc.o
+obj-$(CONFIG_KVM) += kvm.o
 obj-$(call lnot,$(CONFIG_KVM)) += kvm-stub.o
 obj-y += dfp_helper.o
 obj-y += excp_helper.o
diff --git a/target-ppc/arch_dump.c b/target-ppc/arch_dump.c
index 5acafc6..40282a1 100644
--- a/target-ppc/arch_dump.c
+++ b/target-ppc/arch_dump.c
@@ -12,6 +12,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "elf.h"
 #include "exec/cpu-all.h"
@@ -219,6 +220,11 @@
     } else {
         info->d_endian = ELFDATA2LSB;
     }
+    /* 64KB is the max page size for pseries kernel */
+    if (strncmp(object_get_typename(qdev_get_machine()),
+                "pseries-", 8) == 0) {
+        info->page_size = (1U << 16);
+    }
 
     return 0;
 }
@@ -278,9 +284,3 @@
     PowerPCCPU *cpu = POWERPC_CPU(cs);
     return ppc64_write_all_elf64_notes("CORE", f, cpu, cpuid, opaque);
 }
-
-int ppc64_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                   CPUState *cpu, void *opaque)
-{
-    return 0;
-}
diff --git a/target-ppc/cpu-models.c b/target-ppc/cpu-models.c
index 3f18996..5209e63 100644
--- a/target-ppc/cpu-models.c
+++ b/target-ppc/cpu-models.c
@@ -24,6 +24,7 @@
  * inside "#if defined(TODO) ... #endif" statements to make tests easier.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "cpu-models.h"
 
@@ -1124,8 +1125,6 @@
     POWERPC_DEF("POWER5",        CPU_POWERPC_POWER5,                 POWER5,
                 "POWER5")
 #endif
-    POWERPC_DEF("POWER5+",       CPU_POWERPC_POWER5P,                POWER5P,
-                "POWER5+")
     POWERPC_DEF("POWER5+_v2.1",  CPU_POWERPC_POWER5P_v21,            POWER5P,
                 "POWER5+ v2.1")
 #if defined(TODO)
@@ -1140,12 +1139,14 @@
                 "POWER7 v2.3")
     POWERPC_DEF("POWER7+_v2.1",  CPU_POWERPC_POWER7P_v21,            POWER7,
                 "POWER7+ v2.1")
-    POWERPC_DEF("POWER8E_v1.0",  CPU_POWERPC_POWER8E_v10,            POWER8,
-                "POWER8E v1.0")
-    POWERPC_DEF("POWER8_v1.0",   CPU_POWERPC_POWER8_v10,             POWER8,
-                "POWER8 v1.0")
-    POWERPC_DEF("970",           CPU_POWERPC_970,                    970,
-                "PowerPC 970")
+    POWERPC_DEF("POWER8E_v2.1",  CPU_POWERPC_POWER8E_v21,            POWER8,
+                "POWER8E v2.1")
+    POWERPC_DEF("POWER8_v2.0",   CPU_POWERPC_POWER8_v20,             POWER8,
+                "POWER8 v2.0")
+    POWERPC_DEF("POWER8NVL_v1.0",CPU_POWERPC_POWER8NVL_v10,          POWER8,
+                "POWER8NVL v1.0")
+    POWERPC_DEF("970_v2.2",      CPU_POWERPC_970_v22,                970,
+                "PowerPC 970 v2.2")
     POWERPC_DEF("970fx_v1.0",    CPU_POWERPC_970FX_v10,              970,
                 "PowerPC 970FX v1.0 (G5)")
     POWERPC_DEF("970fx_v2.0",    CPU_POWERPC_970FX_v20,              970,
@@ -1387,11 +1388,14 @@
     { "Dino",  "POWER3" },
     { "POWER3+", "631" },
     { "POWER5gr", "POWER5" },
-    { "POWER5gs", "POWER5+" },
+    { "POWER5+", "POWER5+_v2.1" },
+    { "POWER5gs", "POWER5+_v2.1" },
     { "POWER7", "POWER7_v2.3" },
     { "POWER7+", "POWER7+_v2.1" },
-    { "POWER8E", "POWER8E_v1.0" },
-    { "POWER8", "POWER8_v1.0" },
+    { "POWER8E", "POWER8E_v2.1" },
+    { "POWER8", "POWER8_v2.0" },
+    { "POWER8NVL", "POWER8NVL_v1.0" },
+    { "970", "970_v2.2" },
     { "970fx", "970fx_v3.1" },
     { "970mp", "970mp_v1.1" },
     { "Apache", "RS64" },
diff --git a/target-ppc/cpu-models.h b/target-ppc/cpu-models.h
index 290a759..f21a44c 100644
--- a/target-ppc/cpu-models.h
+++ b/target-ppc/cpu-models.h
@@ -547,7 +547,6 @@
     CPU_POWERPC_POWER4P            = 0x00380000,
      /* XXX: missing 0x003A0201 */
     CPU_POWERPC_POWER5             = 0x003A0203,
-    CPU_POWERPC_POWER5P            = 0x003B0000,
     CPU_POWERPC_POWER5P_v21        = 0x003B0201,
     CPU_POWERPC_POWER6             = 0x003E0000,
     CPU_POWERPC_POWER6_5           = 0x0F000001, /* POWER6 in POWER5 mode */
@@ -558,10 +557,12 @@
     CPU_POWERPC_POWER7P_BASE       = 0x004A0000,
     CPU_POWERPC_POWER7P_v21        = 0x004A0201,
     CPU_POWERPC_POWER8E_BASE       = 0x004B0000,
-    CPU_POWERPC_POWER8E_v10        = 0x004B0100,
+    CPU_POWERPC_POWER8E_v21        = 0x004B0201,
     CPU_POWERPC_POWER8_BASE        = 0x004D0000,
-    CPU_POWERPC_POWER8_v10         = 0x004D0100,
-    CPU_POWERPC_970                = 0x00390202,
+    CPU_POWERPC_POWER8_v20         = 0x004D0200,
+    CPU_POWERPC_POWER8NVL_BASE     = 0x004C0000,
+    CPU_POWERPC_POWER8NVL_v10      = 0x004C0100,
+    CPU_POWERPC_970_v22            = 0x00390202,
     CPU_POWERPC_970FX_v10          = 0x00391100,
     CPU_POWERPC_970FX_v20          = 0x003C0200,
     CPU_POWERPC_970FX_v21          = 0x003C0201,
diff --git a/target-ppc/cpu-qom.h b/target-ppc/cpu-qom.h
index 6967a80..2864105 100644
--- a/target-ppc/cpu-qom.h
+++ b/target-ppc/cpu-qom.h
@@ -21,7 +21,6 @@
 #define QEMU_PPC_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #ifdef TARGET_PPC64
 #define TYPE_POWERPC_CPU "powerpc64-cpu"
@@ -39,6 +38,127 @@
     OBJECT_GET_CLASS(PowerPCCPUClass, (obj), TYPE_POWERPC_CPU)
 
 typedef struct PowerPCCPU PowerPCCPU;
+typedef struct CPUPPCState CPUPPCState;
+typedef struct ppc_tb_t ppc_tb_t;
+typedef struct ppc_dcr_t ppc_dcr_t;
+
+/*****************************************************************************/
+/* MMU model                                                                 */
+typedef enum powerpc_mmu_t powerpc_mmu_t;
+enum powerpc_mmu_t {
+    POWERPC_MMU_UNKNOWN    = 0x00000000,
+    /* Standard 32 bits PowerPC MMU                            */
+    POWERPC_MMU_32B        = 0x00000001,
+    /* PowerPC 6xx MMU with software TLB                       */
+    POWERPC_MMU_SOFT_6xx   = 0x00000002,
+    /* PowerPC 74xx MMU with software TLB                      */
+    POWERPC_MMU_SOFT_74xx  = 0x00000003,
+    /* PowerPC 4xx MMU with software TLB                       */
+    POWERPC_MMU_SOFT_4xx   = 0x00000004,
+    /* PowerPC 4xx MMU with software TLB and zones protections */
+    POWERPC_MMU_SOFT_4xx_Z = 0x00000005,
+    /* PowerPC MMU in real mode only                           */
+    POWERPC_MMU_REAL       = 0x00000006,
+    /* Freescale MPC8xx MMU model                              */
+    POWERPC_MMU_MPC8xx     = 0x00000007,
+    /* BookE MMU model                                         */
+    POWERPC_MMU_BOOKE      = 0x00000008,
+    /* BookE 2.06 MMU model                                    */
+    POWERPC_MMU_BOOKE206   = 0x00000009,
+    /* PowerPC 601 MMU model (specific BATs format)            */
+    POWERPC_MMU_601        = 0x0000000A,
+#define POWERPC_MMU_64       0x00010000
+#define POWERPC_MMU_1TSEG    0x00020000
+#define POWERPC_MMU_AMR      0x00040000
+#define POWERPC_MMU_64K      0x00080000
+    /* 64 bits PowerPC MMU                                     */
+    POWERPC_MMU_64B        = POWERPC_MMU_64 | 0x00000001,
+    /* Architecture 2.03 and later (has LPCR) */
+    POWERPC_MMU_2_03       = POWERPC_MMU_64 | 0x00000002,
+    /* Architecture 2.06 variant                               */
+    POWERPC_MMU_2_06       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
+                             | POWERPC_MMU_64K
+                             | POWERPC_MMU_AMR | 0x00000003,
+    /* Architecture 2.06 "degraded" (no 1T segments)           */
+    POWERPC_MMU_2_06a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
+                             | 0x00000003,
+    /* Architecture 2.07 variant                               */
+    POWERPC_MMU_2_07       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
+                             | POWERPC_MMU_64K
+                             | POWERPC_MMU_AMR | 0x00000004,
+    /* Architecture 2.07 "degraded" (no 1T segments)           */
+    POWERPC_MMU_2_07a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
+                             | 0x00000004,
+};
+
+/*****************************************************************************/
+/* Exception model                                                           */
+typedef enum powerpc_excp_t powerpc_excp_t;
+enum powerpc_excp_t {
+    POWERPC_EXCP_UNKNOWN   = 0,
+    /* Standard PowerPC exception model */
+    POWERPC_EXCP_STD,
+    /* PowerPC 40x exception model      */
+    POWERPC_EXCP_40x,
+    /* PowerPC 601 exception model      */
+    POWERPC_EXCP_601,
+    /* PowerPC 602 exception model      */
+    POWERPC_EXCP_602,
+    /* PowerPC 603 exception model      */
+    POWERPC_EXCP_603,
+    /* PowerPC 603e exception model     */
+    POWERPC_EXCP_603E,
+    /* PowerPC G2 exception model       */
+    POWERPC_EXCP_G2,
+    /* PowerPC 604 exception model      */
+    POWERPC_EXCP_604,
+    /* PowerPC 7x0 exception model      */
+    POWERPC_EXCP_7x0,
+    /* PowerPC 7x5 exception model      */
+    POWERPC_EXCP_7x5,
+    /* PowerPC 74xx exception model     */
+    POWERPC_EXCP_74xx,
+    /* BookE exception model            */
+    POWERPC_EXCP_BOOKE,
+    /* PowerPC 970 exception model      */
+    POWERPC_EXCP_970,
+    /* POWER7 exception model           */
+    POWERPC_EXCP_POWER7,
+    /* POWER8 exception model           */
+    POWERPC_EXCP_POWER8,
+};
+
+/*****************************************************************************/
+/* PM instructions */
+typedef enum {
+    PPC_PM_DOZE,
+    PPC_PM_NAP,
+    PPC_PM_SLEEP,
+    PPC_PM_RVWINKLE,
+} powerpc_pm_insn_t;
+
+/*****************************************************************************/
+/* Input pins model                                                          */
+typedef enum powerpc_input_t powerpc_input_t;
+enum powerpc_input_t {
+    PPC_FLAGS_INPUT_UNKNOWN = 0,
+    /* PowerPC 6xx bus                  */
+    PPC_FLAGS_INPUT_6xx,
+    /* BookE bus                        */
+    PPC_FLAGS_INPUT_BookE,
+    /* PowerPC 405 bus                  */
+    PPC_FLAGS_INPUT_405,
+    /* PowerPC 970 bus                  */
+    PPC_FLAGS_INPUT_970,
+    /* PowerPC POWER7 bus               */
+    PPC_FLAGS_INPUT_POWER7,
+    /* PowerPC 401 bus                  */
+    PPC_FLAGS_INPUT_401,
+    /* Freescale RCPU bus               */
+    PPC_FLAGS_INPUT_RCPU,
+};
+
+struct ppc_segment_page_sizes;
 
 /**
  * PowerPCCPUClass:
@@ -57,7 +177,8 @@
 
     uint32_t pvr;
     bool (*pvr_match)(struct PowerPCCPUClass *pcc, uint32_t pvr);
-    uint64_t pcr_mask;
+    uint64_t pcr_mask;          /* Available bits in PCR register */
+    uint64_t pcr_supported;     /* Bits for supported PowerISA versions */
     uint32_t svr;
     uint64_t insns_flags;
     uint64_t insns_flags2;
@@ -68,69 +189,14 @@
     uint32_t flags;
     int bfd_mach;
     uint32_t l1_dcache_size, l1_icache_size;
-#if defined(TARGET_PPC64)
     const struct ppc_segment_page_sizes *sps;
-#endif
     void (*init_proc)(CPUPPCState *env);
     int  (*check_pow)(CPUPPCState *env);
-#if defined(CONFIG_SOFTMMU)
-    int (*handle_mmu_fault)(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
-                            int mmu_idx);
-#endif
+    int (*handle_mmu_fault)(PowerPCCPU *cpu, vaddr eaddr, int rwx, int mmu_idx);
     bool (*interrupts_big_endian)(PowerPCCPU *cpu);
 } PowerPCCPUClass;
 
-/**
- * PowerPCCPU:
- * @env: #CPUPPCState
- * @cpu_dt_id: CPU index used in the device tree. KVM uses this index too
- * @max_compat: Maximal supported logical PVR from the command line
- * @cpu_version: Current logical PVR, zero if in "raw" mode
- *
- * A PowerPC CPU.
- */
-struct PowerPCCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUPPCState env;
-    int cpu_dt_id;
-    uint32_t max_compat;
-    uint32_t cpu_version;
-};
-
-static inline PowerPCCPU *ppc_env_get_cpu(CPUPPCState *env)
-{
-    return container_of(env, PowerPCCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(ppc_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(PowerPCCPU, env)
-
-PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr);
-PowerPCCPUClass *ppc_cpu_class_by_pvr_mask(uint32_t pvr);
-
-void ppc_cpu_do_interrupt(CPUState *cpu);
-bool ppc_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void ppc_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
-                        int flags);
-void ppc_cpu_dump_statistics(CPUState *cpu, FILE *f,
-                             fprintf_function cpu_fprintf, int flags);
-hwaddr ppc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int ppc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int ppc_cpu_gdb_read_register_apple(CPUState *cpu, uint8_t *buf, int reg);
-int ppc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-int ppc_cpu_gdb_write_register_apple(CPUState *cpu, uint8_t *buf, int reg);
-int ppc64_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                   CPUState *cpu, void *opaque);
-int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
-                               int cpuid, void *opaque);
 #ifndef CONFIG_USER_ONLY
-void ppc_cpu_do_system_reset(CPUState *cs);
-extern const struct VMStateDescription vmstate_ppc_cpu;
-
 typedef struct PPCTimebase {
     uint64_t guest_timebase;
     int64_t time_of_the_day_ns;
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index 068fcb2..786ab5c 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -16,10 +16,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#if !defined (__CPU_PPC_H__)
-#define __CPU_PPC_H__
 
-#include "config.h"
+#ifndef PPC_CPU_H
+#define PPC_CPU_H
+
 #include "qemu-common.h"
 
 //#define PPC_EMULATE_32BITS_HYPV
@@ -29,8 +29,6 @@
 #define TARGET_LONG_BITS 64
 #define TARGET_PAGE_BITS 12
 
-#define TARGET_IS_BIENDIAN 1
-
 /* Note that the official physical address space bits is 62-M where M
    is implementation dependent.  I've not looked up M for the set of
    cpus we emulate at the system level.  */
@@ -45,6 +43,7 @@
 # define TARGET_VIRT_ADDR_SPACE_BITS 64
 #endif
 
+#define TARGET_PAGE_BITS_64K 16
 #define TARGET_PAGE_BITS_16M 24
 
 #else /* defined (TARGET_PPC64) */
@@ -76,97 +75,16 @@
 #define CPUArchState struct CPUPPCState
 
 #include "exec/cpu-defs.h"
-
+#include "cpu-qom.h"
 #include "fpu/softfloat.h"
 
-#define TARGET_HAS_ICE 1
-
 #if defined (TARGET_PPC64)
-#define ELF_MACHINE     EM_PPC64
+#define PPC_ELF_MACHINE     EM_PPC64
 #else
-#define ELF_MACHINE     EM_PPC
+#define PPC_ELF_MACHINE     EM_PPC
 #endif
 
 /*****************************************************************************/
-/* MMU model                                                                 */
-typedef enum powerpc_mmu_t powerpc_mmu_t;
-enum powerpc_mmu_t {
-    POWERPC_MMU_UNKNOWN    = 0x00000000,
-    /* Standard 32 bits PowerPC MMU                            */
-    POWERPC_MMU_32B        = 0x00000001,
-    /* PowerPC 6xx MMU with software TLB                       */
-    POWERPC_MMU_SOFT_6xx   = 0x00000002,
-    /* PowerPC 74xx MMU with software TLB                      */
-    POWERPC_MMU_SOFT_74xx  = 0x00000003,
-    /* PowerPC 4xx MMU with software TLB                       */
-    POWERPC_MMU_SOFT_4xx   = 0x00000004,
-    /* PowerPC 4xx MMU with software TLB and zones protections */
-    POWERPC_MMU_SOFT_4xx_Z = 0x00000005,
-    /* PowerPC MMU in real mode only                           */
-    POWERPC_MMU_REAL       = 0x00000006,
-    /* Freescale MPC8xx MMU model                              */
-    POWERPC_MMU_MPC8xx     = 0x00000007,
-    /* BookE MMU model                                         */
-    POWERPC_MMU_BOOKE      = 0x00000008,
-    /* BookE 2.06 MMU model                                    */
-    POWERPC_MMU_BOOKE206   = 0x00000009,
-    /* PowerPC 601 MMU model (specific BATs format)            */
-    POWERPC_MMU_601        = 0x0000000A,
-#if defined(TARGET_PPC64)
-#define POWERPC_MMU_64       0x00010000
-#define POWERPC_MMU_1TSEG    0x00020000
-#define POWERPC_MMU_AMR      0x00040000
-    /* 64 bits PowerPC MMU                                     */
-    POWERPC_MMU_64B        = POWERPC_MMU_64 | 0x00000001,
-    /* Architecture 2.06 variant                               */
-    POWERPC_MMU_2_06       = POWERPC_MMU_64 | POWERPC_MMU_1TSEG
-                             | POWERPC_MMU_AMR | 0x00000003,
-    /* Architecture 2.06 "degraded" (no 1T segments)           */
-    POWERPC_MMU_2_06a      = POWERPC_MMU_64 | POWERPC_MMU_AMR
-                             | 0x00000003,
-    /* Architecture 2.06 "degraded" (no 1T segments or AMR)    */
-    POWERPC_MMU_2_06d      = POWERPC_MMU_64 | 0x00000003,
-#endif /* defined(TARGET_PPC64) */
-};
-
-/*****************************************************************************/
-/* Exception model                                                           */
-typedef enum powerpc_excp_t powerpc_excp_t;
-enum powerpc_excp_t {
-    POWERPC_EXCP_UNKNOWN   = 0,
-    /* Standard PowerPC exception model */
-    POWERPC_EXCP_STD,
-    /* PowerPC 40x exception model      */
-    POWERPC_EXCP_40x,
-    /* PowerPC 601 exception model      */
-    POWERPC_EXCP_601,
-    /* PowerPC 602 exception model      */
-    POWERPC_EXCP_602,
-    /* PowerPC 603 exception model      */
-    POWERPC_EXCP_603,
-    /* PowerPC 603e exception model     */
-    POWERPC_EXCP_603E,
-    /* PowerPC G2 exception model       */
-    POWERPC_EXCP_G2,
-    /* PowerPC 604 exception model      */
-    POWERPC_EXCP_604,
-    /* PowerPC 7x0 exception model      */
-    POWERPC_EXCP_7x0,
-    /* PowerPC 7x5 exception model      */
-    POWERPC_EXCP_7x5,
-    /* PowerPC 74xx exception model     */
-    POWERPC_EXCP_74xx,
-    /* BookE exception model            */
-    POWERPC_EXCP_BOOKE,
-#if defined(TARGET_PPC64)
-    /* PowerPC 970 exception model      */
-    POWERPC_EXCP_970,
-    /* POWER7 exception model           */
-    POWERPC_EXCP_POWER7,
-#endif /* defined(TARGET_PPC64) */
-};
-
-/*****************************************************************************/
 /* Exception vectors definitions                                             */
 enum {
     POWERPC_EXCP_NONE    = -1,
@@ -199,6 +117,9 @@
     POWERPC_EXCP_HYPPRIV  = 41, /* Embedded hypervisor priv instruction      */
     /* Vectors 42 to 63 are reserved                                         */
     /* Exceptions defined in the PowerPC server specification                */
+    /* Server doorbell variants */
+#define POWERPC_EXCP_SDOOR      POWERPC_EXCP_GDOORI
+#define POWERPC_EXCP_SDOOR_HV   POWERPC_EXCP_DOORI
     POWERPC_EXCP_RESET    = 64, /* System reset exception                    */
     POWERPC_EXCP_DSEG     = 65, /* Data segment exception                    */
     POWERPC_EXCP_ISEG     = 66, /* Instruction segment exception             */
@@ -241,8 +162,12 @@
     /* VSX Unavailable (Power ISA 2.06 and later)                            */
     POWERPC_EXCP_VSXU     = 94, /* VSX Unavailable                           */
     POWERPC_EXCP_FU       = 95, /* Facility Unavailable                      */
+    /* Additional ISA 2.06 and later server exceptions                       */
+    POWERPC_EXCP_HV_EMU   = 96, /* HV emulation assistance                   */
+    POWERPC_EXCP_HV_MAINT = 97, /* HMI                                       */
+    POWERPC_EXCP_HV_FU    = 98, /* Hypervisor Facility unavailable           */
     /* EOL                                                                   */
-    POWERPC_EXCP_NB       = 96,
+    POWERPC_EXCP_NB       = 99,
     /* QEMU exceptions: used internally during code translation              */
     POWERPC_EXCP_STOP         = 0x200, /* stop translation                   */
     POWERPC_EXCP_BRANCH       = 0x201, /* branch instruction                 */
@@ -291,27 +216,6 @@
     POWERPC_EXCP_TRAP          = 0x40,
 };
 
-/*****************************************************************************/
-/* Input pins model                                                          */
-typedef enum powerpc_input_t powerpc_input_t;
-enum powerpc_input_t {
-    PPC_FLAGS_INPUT_UNKNOWN = 0,
-    /* PowerPC 6xx bus                  */
-    PPC_FLAGS_INPUT_6xx,
-    /* BookE bus                        */
-    PPC_FLAGS_INPUT_BookE,
-    /* PowerPC 405 bus                  */
-    PPC_FLAGS_INPUT_405,
-    /* PowerPC 970 bus                  */
-    PPC_FLAGS_INPUT_970,
-    /* PowerPC POWER7 bus               */
-    PPC_FLAGS_INPUT_POWER7,
-    /* PowerPC 401 bus                  */
-    PPC_FLAGS_INPUT_401,
-    /* Freescale RCPU bus               */
-    PPC_FLAGS_INPUT_RCPU,
-};
-
 #define PPC_INPUT(env) (env->bus_model)
 
 /*****************************************************************************/
@@ -319,22 +223,20 @@
 
 /*****************************************************************************/
 /* Types used to describe some PowerPC registers */
-typedef struct CPUPPCState CPUPPCState;
-typedef struct ppc_tb_t ppc_tb_t;
+typedef struct DisasContext DisasContext;
 typedef struct ppc_spr_t ppc_spr_t;
-typedef struct ppc_dcr_t ppc_dcr_t;
 typedef union ppc_avr_t ppc_avr_t;
 typedef union ppc_tlb_t ppc_tlb_t;
 
 /* SPR access micro-ops generations callbacks */
 struct ppc_spr_t {
-    void (*uea_read)(void *opaque, int gpr_num, int spr_num);
-    void (*uea_write)(void *opaque, int spr_num, int gpr_num);
+    void (*uea_read)(DisasContext *ctx, int gpr_num, int spr_num);
+    void (*uea_write)(DisasContext *ctx, int spr_num, int gpr_num);
 #if !defined(CONFIG_USER_ONLY)
-    void (*oea_read)(void *opaque, int gpr_num, int spr_num);
-    void (*oea_write)(void *opaque, int spr_num, int gpr_num);
-    void (*hea_read)(void *opaque, int gpr_num, int spr_num);
-    void (*hea_write)(void *opaque, int spr_num, int gpr_num);
+    void (*oea_read)(DisasContext *ctx, int gpr_num, int spr_num);
+    void (*oea_write)(DisasContext *ctx, int spr_num, int gpr_num);
+    void (*hea_read)(DisasContext *ctx, int gpr_num, int spr_num);
+    void (*hea_write)(DisasContext *ctx, int spr_num, int gpr_num);
 #endif
     const char *name;
     target_ulong default_value;
@@ -413,6 +315,7 @@
 struct ppc_slb_t {
     uint64_t esid;
     uint64_t vsid;
+    const struct ppc_one_seg_page_size *sps;
 };
 
 #define MAX_SLB_ENTRIES         64
@@ -462,15 +365,43 @@
 #define MSR_EP   6  /* Exception prefix on 601                               */
 #define MSR_IR   5  /* Instruction relocate                                  */
 #define MSR_DR   4  /* Data relocate                                         */
+#define MSR_IS   5  /* Instruction address space (BookE)                     */
+#define MSR_DS   4  /* Data address space (BookE)                            */
 #define MSR_PE   3  /* Protection enable on 403                              */
 #define MSR_PX   2  /* Protection exclusive on 403                  x        */
 #define MSR_PMM  2  /* Performance monitor mark on POWER            x        */
 #define MSR_RI   1  /* Recoverable interrupt                        1        */
 #define MSR_LE   0  /* Little-endian mode                           1 hflags */
 
-#define LPCR_ILE (1 << (63-38))
-#define LPCR_AIL_SHIFT (63-40)      /* Alternate interrupt location */
-#define LPCR_AIL (3 << LPCR_AIL_SHIFT)
+/* LPCR bits */
+#define LPCR_VPM0         (1ull << (63 - 0))
+#define LPCR_VPM1         (1ull << (63 - 1))
+#define LPCR_ISL          (1ull << (63 - 2))
+#define LPCR_KBV          (1ull << (63 - 3))
+#define LPCR_DPFD_SHIFT   (63 - 11)
+#define LPCR_DPFD         (0x3ull << LPCR_DPFD_SHIFT)
+#define LPCR_VRMASD_SHIFT (63 - 16)
+#define LPCR_VRMASD       (0x1full << LPCR_VRMASD_SHIFT)
+#define LPCR_RMLS_SHIFT   (63 - 37)
+#define LPCR_RMLS         (0xfull << LPCR_RMLS_SHIFT)
+#define LPCR_ILE          (1ull << (63 - 38))
+#define LPCR_AIL_SHIFT    (63 - 40)      /* Alternate interrupt location */
+#define LPCR_AIL          (3ull << LPCR_AIL_SHIFT)
+#define LPCR_ONL          (1ull << (63 - 45))
+#define LPCR_P7_PECE0     (1ull << (63 - 49))
+#define LPCR_P7_PECE1     (1ull << (63 - 50))
+#define LPCR_P7_PECE2     (1ull << (63 - 51))
+#define LPCR_P8_PECE0     (1ull << (63 - 47))
+#define LPCR_P8_PECE1     (1ull << (63 - 48))
+#define LPCR_P8_PECE2     (1ull << (63 - 49))
+#define LPCR_P8_PECE3     (1ull << (63 - 50))
+#define LPCR_P8_PECE4     (1ull << (63 - 51))
+#define LPCR_MER          (1ull << (63 - 52))
+#define LPCR_TC           (1ull << (63 - 54))
+#define LPCR_LPES0        (1ull << (63 - 60))
+#define LPCR_LPES1        (1ull << (63 - 61))
+#define LPCR_RMI          (1ull << (63 - 62))
+#define LPCR_HDICE        (1ull << (63 - 63))
 
 #define msr_sf   ((env->msr >> MSR_SF)   & 1)
 #define msr_isf  ((env->msr >> MSR_ISF)  & 1)
@@ -505,6 +436,8 @@
 #define msr_ep   ((env->msr >> MSR_EP)   & 1)
 #define msr_ir   ((env->msr >> MSR_IR)   & 1)
 #define msr_dr   ((env->msr >> MSR_DR)   & 1)
+#define msr_is   ((env->msr >> MSR_IS)   & 1)
+#define msr_ds   ((env->msr >> MSR_DS)   & 1)
 #define msr_pe   ((env->msr >> MSR_PE)   & 1)
 #define msr_px   ((env->msr >> MSR_PX)   & 1)
 #define msr_pmm  ((env->msr >> MSR_PMM)  & 1)
@@ -558,6 +491,26 @@
 #define ESR_VLEMI (1 << (63 - 58)) /* VLE operation                          */
 #define ESR_MIF   (1 << (63 - 62)) /* Misaligned instruction (VLE)           */
 
+/* Transaction EXception And Summary Register bits                           */
+#define TEXASR_FAILURE_PERSISTENT                (63 - 7)
+#define TEXASR_DISALLOWED                        (63 - 8)
+#define TEXASR_NESTING_OVERFLOW                  (63 - 9)
+#define TEXASR_FOOTPRINT_OVERFLOW                (63 - 10)
+#define TEXASR_SELF_INDUCED_CONFLICT             (63 - 11)
+#define TEXASR_NON_TRANSACTIONAL_CONFLICT        (63 - 12)
+#define TEXASR_TRANSACTION_CONFLICT              (63 - 13)
+#define TEXASR_TRANSLATION_INVALIDATION_CONFLICT (63 - 14)
+#define TEXASR_IMPLEMENTATION_SPECIFIC           (63 - 15)
+#define TEXASR_INSTRUCTION_FETCH_CONFLICT        (63 - 16)
+#define TEXASR_ABORT                             (63 - 31)
+#define TEXASR_SUSPENDED                         (63 - 32)
+#define TEXASR_PRIVILEGE_HV                      (63 - 34)
+#define TEXASR_PRIVILEGE_PR                      (63 - 35)
+#define TEXASR_FAILURE_SUMMARY                   (63 - 36)
+#define TEXASR_TFIAR_EXACT                       (63 - 37)
+#define TEXASR_ROT                               (63 - 38)
+#define TEXASR_TRANSACTION_LEVEL                 (63 - 52) /* 12 bits */
+
 enum {
     POWERPC_FLAG_NONE     = 0x00000000,
     /* Flag for MSR bit 25 signification (VRE/SPE)                           */
@@ -584,6 +537,8 @@
     POWERPC_FLAG_CFAR     = 0x00040000,
     /* Has VSX                                                               */
     POWERPC_FLAG_VSX      = 0x00080000,
+    /* Has Transaction Memory (ISA 2.07)                                     */
+    POWERPC_FLAG_TM       = 0x00100000,
 };
 
 /*****************************************************************************/
@@ -656,6 +611,46 @@
 #define fpscr_eex (((env->fpscr) >> FPSCR_XX) & ((env->fpscr) >> FPSCR_XE) &  \
                    0x1F)
 
+#define FP_FX		(1ull << FPSCR_FX)
+#define FP_FEX		(1ull << FPSCR_FEX)
+#define FP_VX		(1ull << FPSCR_VX)
+#define FP_OX		(1ull << FPSCR_OX)
+#define FP_UX		(1ull << FPSCR_UX)
+#define FP_ZX		(1ull << FPSCR_ZX)
+#define FP_XX		(1ull << FPSCR_XX)
+#define FP_VXSNAN	(1ull << FPSCR_VXSNAN)
+#define FP_VXISI	(1ull << FPSCR_VXISI)
+#define FP_VXIDI	(1ull << FPSCR_VXIDI)
+#define FP_VXZDZ	(1ull << FPSCR_VXZDZ)
+#define FP_VXIMZ	(1ull << FPSCR_VXIMZ)
+#define FP_VXVC		(1ull << FPSCR_VXVC)
+#define FP_FR		(1ull << FSPCR_FR)
+#define FP_FI		(1ull << FPSCR_FI)
+#define FP_C		(1ull << FPSCR_C)
+#define FP_FL		(1ull << FPSCR_FL)
+#define FP_FG		(1ull << FPSCR_FG)
+#define FP_FE		(1ull << FPSCR_FE)
+#define FP_FU		(1ull << FPSCR_FU)
+#define FP_FPCC		(FP_FL | FP_FG | FP_FE | FP_FU)
+#define FP_FPRF		(FP_C  | FP_FL | FP_FG | FP_FE | FP_FU)
+#define FP_VXSOFT	(1ull << FPSCR_VXSOFT)
+#define FP_VXSQRT	(1ull << FPSCR_VXSQRT)
+#define FP_VXCVI	(1ull << FPSCR_VXCVI)
+#define FP_VE		(1ull << FPSCR_VE)
+#define FP_OE		(1ull << FPSCR_OE)
+#define FP_UE		(1ull << FPSCR_UE)
+#define FP_ZE		(1ull << FPSCR_ZE)
+#define FP_XE		(1ull << FPSCR_XE)
+#define FP_NI		(1ull << FPSCR_NI)
+#define FP_RN1		(1ull << FPSCR_RN1)
+#define FP_RN		(1ull << FPSCR_RN)
+
+/* the exception bits which can be cleared by mcrfs - includes FX */
+#define FP_EX_CLEAR_BITS (FP_FX     | FP_OX     | FP_UX     | FP_ZX     | \
+                          FP_XX     | FP_VXSNAN | FP_VXISI  | FP_VXIDI  | \
+                          FP_VXZDZ  | FP_VXIMZ  | FP_VXVC   | FP_VXSOFT | \
+                          FP_VXSQRT | FP_VXCVI)
+
 /*****************************************************************************/
 /* Vector status and control register */
 #define VSCR_NJ		16 /* Vector non-java */
@@ -922,7 +917,7 @@
 
 /*****************************************************************************/
 /* The whole PowerPC CPU context */
-#define NB_MMU_MODES 3
+#define NB_MMU_MODES    8
 
 #define PPC_CPU_OPCODES_LEN          0x40
 #define PPC_CPU_INDIRECT_OPCODES_LEN 0x20
@@ -987,6 +982,7 @@
     /* PowerPC 64 SLB area */
     ppc_slb_t slb[MAX_SLB_ENTRIES];
     int32_t slb_nr;
+    /* tcg TLB needs flush (deferred slb inval instruction typically) */
 #endif
     /* segment registers */
     hwaddr htab_base;
@@ -1012,6 +1008,7 @@
     target_ulong pb[4];
     bool tlb_dirty;   /* Set to non-zero when modifying TLB                  */
     bool kvm_sw_tlb;  /* non-zero if KVM SW TLB API is active                */
+    uint32_t tlb_need_flush; /* Delayed flush needed */
 #endif
 
     /* Other registers */
@@ -1051,6 +1048,9 @@
     uint64_t insns_flags2;
 #if defined(TARGET_PPC64)
     struct ppc_segment_page_sizes sps;
+    ppc_slb_t vrma_slb;
+    target_ulong rmls;
+    bool ci_large_pages;
 #endif
 
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
@@ -1076,6 +1076,15 @@
     hwaddr mpic_iack;
     /* true when the external proxy facility mode is enabled */
     bool mpic_proxy;
+    /* set when the processor has an HV mode, thus HV priv
+     * instructions and SPRs are diallowed if MSR:HV is 0
+     */
+    bool has_hv_mode;
+    /* On P7/P8, set when in PM state, we need to handle resume
+     * in a special way (such as routing some resume causes to
+     * 0x100), so flag this here.
+     */
+    bool in_pm_state;
 #endif
 
     /* Those resources are used only during code translation */
@@ -1085,7 +1094,8 @@
     /* Those resources are used only in QEMU core */
     target_ulong hflags;      /* hflags is a MSR & HFLAGS_MASK         */
     target_ulong hflags_nmsr; /* specific hflags, not coming from MSR */
-    int mmu_idx;         /* precomputed MMU index to speed up mem accesses */
+    int immu_idx;         /* precomputed MMU index to speed up insn access */
+    int dmmu_idx;         /* precomputed MMU index to speed up data accesses */
 
     /* Power management */
     int (*check_pow)(CPUPPCState *env);
@@ -1136,13 +1146,63 @@
     env->wdt_period[3] = (d_);                  \
  } while (0)
 
-#include "cpu-qom.h"
+/**
+ * PowerPCCPU:
+ * @env: #CPUPPCState
+ * @cpu_dt_id: CPU index used in the device tree. KVM uses this index too
+ * @max_compat: Maximal supported logical PVR from the command line
+ * @cpu_version: Current logical PVR, zero if in "raw" mode
+ *
+ * A PowerPC CPU.
+ */
+struct PowerPCCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUPPCState env;
+    int cpu_dt_id;
+    uint32_t max_compat;
+    uint32_t cpu_version;
+};
+
+static inline PowerPCCPU *ppc_env_get_cpu(CPUPPCState *env)
+{
+    return container_of(env, PowerPCCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(ppc_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(PowerPCCPU, env)
+
+PowerPCCPUClass *ppc_cpu_class_by_pvr(uint32_t pvr);
+PowerPCCPUClass *ppc_cpu_class_by_pvr_mask(uint32_t pvr);
+
+void ppc_cpu_do_interrupt(CPUState *cpu);
+bool ppc_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void ppc_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
+                        int flags);
+void ppc_cpu_dump_statistics(CPUState *cpu, FILE *f,
+                             fprintf_function cpu_fprintf, int flags);
+int ppc_cpu_get_monitor_def(CPUState *cs, const char *name,
+                            uint64_t *pval);
+hwaddr ppc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int ppc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int ppc_cpu_gdb_read_register_apple(CPUState *cpu, uint8_t *buf, int reg);
+int ppc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+int ppc_cpu_gdb_write_register_apple(CPUState *cpu, uint8_t *buf, int reg);
+int ppc64_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
+                               int cpuid, void *opaque);
+#ifndef CONFIG_USER_ONLY
+void ppc_cpu_do_system_reset(CPUState *cs);
+extern const struct VMStateDescription vmstate_ppc_cpu;
+#endif
 
 /*****************************************************************************/
 PowerPCCPU *cpu_ppc_init(const char *cpu_model);
 void ppc_translate_init(void);
+const char *ppc_cpu_lookup_alias(const char *alias);
 void gen_update_current_nip(void *opaque);
-int cpu_ppc_exec (CPUPPCState *s);
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
    is returned if the signal was handled by the virtual CPU.  */
@@ -1160,7 +1220,9 @@
 
 void ppc_cpu_list (FILE *f, fprintf_function cpu_fprintf);
 int ppc_get_compat_smt_threads(PowerPCCPU *cpu);
-int ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version);
+#if defined(TARGET_PPC64)
+void ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version, Error **errp);
+#endif
 
 /* Time-base and decrementer management */
 #ifndef NO_CPU_IO_DEFS
@@ -1191,6 +1253,7 @@
 void store_booke_tsr (CPUPPCState *env, target_ulong val);
 void ppc_tlb_invalidate_all (CPUPPCState *env);
 void ppc_tlb_invalidate_one (CPUPPCState *env, target_ulong addr);
+void cpu_ppc_set_papr(PowerPCCPU *cpu);
 #endif
 #endif
 
@@ -1216,28 +1279,16 @@
 int ppc_dcr_read (ppc_dcr_t *dcr_env, int dcrn, uint32_t *valp);
 int ppc_dcr_write (ppc_dcr_t *dcr_env, int dcrn, uint32_t val);
 
-static inline CPUPPCState *cpu_init(const char *cpu_model)
-{
-    PowerPCCPU *cpu = cpu_ppc_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_ppc_init(cpu_model))
 
-#define cpu_exec cpu_ppc_exec
-#define cpu_gen_code cpu_ppc_gen_code
 #define cpu_signal_handler cpu_ppc_signal_handler
 #define cpu_list ppc_cpu_list
 
 /* MMU modes definitions */
-#define MMU_MODE0_SUFFIX _user
-#define MMU_MODE1_SUFFIX _kernel
-#define MMU_MODE2_SUFFIX _hypv
 #define MMU_USER_IDX 0
-static inline int cpu_mmu_index (CPUPPCState *env)
+static inline int cpu_mmu_index (CPUPPCState *env, bool ifetch)
 {
-    return env->mmu_idx;
+    return ifetch ? env->immu_idx : env->dmmu_idx;
 }
 
 #include "exec/cpu-all.h"
@@ -1285,11 +1336,14 @@
 #define SPR_SRR1              (0x01B)
 #define SPR_CFAR              (0x01C)
 #define SPR_AMR               (0x01D)
+#define SPR_ACOP              (0x01F)
 #define SPR_BOOKE_PID         (0x030)
+#define SPR_BOOKS_PID         (0x030)
 #define SPR_BOOKE_DECAR       (0x036)
 #define SPR_BOOKE_CSRR0       (0x03A)
 #define SPR_BOOKE_CSRR1       (0x03B)
 #define SPR_BOOKE_DEAR        (0x03D)
+#define SPR_IAMR              (0x03D)
 #define SPR_BOOKE_ESR         (0x03E)
 #define SPR_BOOKE_IVPR        (0x03F)
 #define SPR_MPC_EIE           (0x050)
@@ -1319,6 +1373,12 @@
 #define SPR_UAMOR             (0x09D)
 #define SPR_MPC_ICTRL         (0x09E)
 #define SPR_MPC_BAR           (0x09F)
+#define SPR_PSPB              (0x09F)
+#define SPR_DAWR              (0x0B4)
+#define SPR_RPR               (0x0BA)
+#define SPR_CIABR             (0x0BB)
+#define SPR_DAWRX             (0x0BC)
+#define SPR_HFSCR             (0x0BE)
 #define SPR_VRSAVE            (0x100)
 #define SPR_USPRG0            (0x100)
 #define SPR_USPRG1            (0x101)
@@ -1373,19 +1433,25 @@
 #define SPR_HSRR1             (0x13B)
 #define SPR_BOOKE_IAC4        (0x13B)
 #define SPR_BOOKE_DAC1        (0x13C)
-#define SPR_LPIDR             (0x13D)
+#define SPR_MMCRH             (0x13C)
 #define SPR_DABR2             (0x13D)
 #define SPR_BOOKE_DAC2        (0x13D)
+#define SPR_TFMR              (0x13D)
 #define SPR_BOOKE_DVC1        (0x13E)
 #define SPR_LPCR              (0x13E)
 #define SPR_BOOKE_DVC2        (0x13F)
+#define SPR_LPIDR             (0x13F)
 #define SPR_BOOKE_TSR         (0x150)
+#define SPR_HMER              (0x150)
+#define SPR_HMEER             (0x151)
 #define SPR_PCR               (0x152)
+#define SPR_BOOKE_LPIDR       (0x152)
 #define SPR_BOOKE_TCR         (0x154)
 #define SPR_BOOKE_TLB0PS      (0x158)
 #define SPR_BOOKE_TLB1PS      (0x159)
 #define SPR_BOOKE_TLB2PS      (0x15A)
 #define SPR_BOOKE_TLB3PS      (0x15B)
+#define SPR_AMOR              (0x15D)
 #define SPR_BOOKE_MAS7_MAS3   (0x174)
 #define SPR_BOOKE_IVOR0       (0x190)
 #define SPR_BOOKE_IVOR1       (0x191)
@@ -1502,6 +1568,7 @@
 #define SPR_PERF0             (0x300)
 #define SPR_RCPU_MI_RBA0      (0x300)
 #define SPR_MPC_MI_CTR        (0x300)
+#define SPR_POWER_USIER       (0x300)
 #define SPR_PERF1             (0x301)
 #define SPR_RCPU_MI_RBA1      (0x301)
 #define SPR_POWER_UMMCR2      (0x301)
@@ -1551,6 +1618,7 @@
 #define SPR_PERFF             (0x30F)
 #define SPR_MPC_MD_TW         (0x30F)
 #define SPR_UPERF0            (0x310)
+#define SPR_POWER_SIER        (0x310)
 #define SPR_UPERF1            (0x311)
 #define SPR_POWER_MMCR2       (0x311)
 #define SPR_UPERF2            (0x312)
@@ -1602,6 +1670,9 @@
 #define SPR_MPC_MD_DBRAM1     (0x32A)
 #define SPR_RCPU_L2U_RA3      (0x32B)
 #define SPR_TAR               (0x32F)
+#define SPR_IC                (0x350)
+#define SPR_VTB               (0x351)
+#define SPR_MMCRC             (0x353)
 #define SPR_440_INV0          (0x370)
 #define SPR_440_INV1          (0x371)
 #define SPR_440_INV2          (0x372)
@@ -1611,8 +1682,14 @@
 #define SPR_440_ITV2          (0x376)
 #define SPR_440_ITV3          (0x377)
 #define SPR_440_CCR1          (0x378)
+#define SPR_TACR              (0x378)
+#define SPR_TCSCR             (0x379)
+#define SPR_CSIGR             (0x37a)
 #define SPR_DCRIPR            (0x37B)
+#define SPR_POWER_SPMC1       (0x37C)
+#define SPR_POWER_SPMC2       (0x37D)
 #define SPR_POWER_MMCRS       (0x37E)
+#define SPR_WORT              (0x37F)
 #define SPR_PPR               (0x380)
 #define SPR_750_GQR0          (0x390)
 #define SPR_440_DNV0          (0x390)
@@ -1635,6 +1712,7 @@
 #define SPR_440_DVLIM         (0x398)
 #define SPR_750_WPAR          (0x399)
 #define SPR_440_IVLIM         (0x399)
+#define SPR_TSCR              (0x399)
 #define SPR_750_DMAU          (0x39A)
 #define SPR_750_DMAL          (0x39B)
 #define SPR_440_RSTCFG        (0x39B)
@@ -1809,9 +1887,10 @@
 #define   L1CSR1_ICE		0x00000001	/* Instruction Cache Enable */
 
 /* HID0 bits */
-#define HID0_DEEPNAP        (1 << 24)
-#define HID0_DOZE           (1 << 23)
-#define HID0_NAP            (1 << 22)
+#define HID0_DEEPNAP        (1 << 24)           /* pre-2.06 */
+#define HID0_DOZE           (1 << 23)           /* pre-2.06 */
+#define HID0_NAP            (1 << 22)           /* pre-2.06 */
+#define HID0_HILE           (1ull << (63 - 19)) /* POWER8 */
 
 /*****************************************************************************/
 /* PowerPC Instructions types definitions                                    */
@@ -1858,6 +1937,8 @@
     PPC_POPCNTB        = 0x0000000000001000ULL,
     /*   string load / store                                                 */
     PPC_STRING         = 0x0000000000002000ULL,
+    /*   real mode cache inhibited load / store                              */
+    PPC_CILDST         = 0x0000000000004000ULL,
 
     /* Floating-point unit extensions                                        */
     /*   Optional floating point instructions                                */
@@ -1972,7 +2053,7 @@
                         | PPC_MFAPIDI | PPC_TLBIVA | PPC_TLBIVAX \
                         | PPC_4xx_COMMON | PPC_40x_ICBT | PPC_RFMCI \
                         | PPC_RFDI | PPC_DCR | PPC_DCRX | PPC_DCRUX \
-                        | PPC_POPCNTWD)
+                        | PPC_POPCNTWD | PPC_CILDST)
 
     /* extended type values */
 
@@ -2010,6 +2091,10 @@
     PPC2_ISA207S       = 0x0000000000008000ULL,
     /* Double precision floating point conversion for signed integer 64      */
     PPC2_FP_CVT_S64    = 0x0000000000010000ULL,
+    /* Transactional Memory (ISA 2.07, Book II)                              */
+    PPC2_TM            = 0x0000000000020000ULL,
+    /* Server PM instructgions (ISA 2.06, Book III)                          */
+    PPC2_PM_ISA206     = 0x0000000000040000ULL,
 
 #define PPC_TCG_INSNS2 (PPC2_BOOKE206 | PPC2_VSX | PPC2_PRCNTL | PPC2_DBRX | \
                         PPC2_ISA205 | PPC2_VSX207 | PPC2_PERM_ISA206 | \
@@ -2017,7 +2102,7 @@
                         PPC2_FP_CVT_ISA206 | PPC2_FP_TST_ISA206 | \
                         PPC2_BCTAR_ISA207 | PPC2_LSQ_ISA207 | \
                         PPC2_ALTIVEC_207 | PPC2_ISA207S | PPC2_DFP | \
-                        PPC2_FP_CVT_S64)
+                        PPC2_FP_CVT_S64 | PPC2_TM | PPC2_PM_ISA206)
 };
 
 /*****************************************************************************/
@@ -2147,17 +2232,47 @@
     PPC_INTERRUPT_CDOORBELL,      /* Critical doorbell interrupt          */
     PPC_INTERRUPT_DOORBELL,       /* Doorbell interrupt                   */
     PPC_INTERRUPT_PERFM,          /* Performance monitor interrupt        */
+    PPC_INTERRUPT_HMI,            /* Hypervisor Maintainance interrupt    */
+    PPC_INTERRUPT_HDOORBELL,      /* Hypervisor Doorbell interrupt        */
 };
 
 /* Processor Compatibility mask (PCR) */
 enum {
     PCR_COMPAT_2_05     = 1ull << (63-62),
     PCR_COMPAT_2_06     = 1ull << (63-61),
+    PCR_COMPAT_2_07     = 1ull << (63-60),
     PCR_VEC_DIS         = 1ull << (63-0), /* Vec. disable (bit NA since POWER8) */
     PCR_VSX_DIS         = 1ull << (63-1), /* VSX disable (bit NA since POWER8) */
     PCR_TM_DIS          = 1ull << (63-2), /* Trans. memory disable (POWER8) */
 };
 
+/* HMER/HMEER */
+enum {
+    HMER_MALFUNCTION_ALERT      = 1ull << (63 - 0),
+    HMER_PROC_RECV_DONE         = 1ull << (63 - 2),
+    HMER_PROC_RECV_ERROR_MASKED = 1ull << (63 - 3),
+    HMER_TFAC_ERROR             = 1ull << (63 - 4),
+    HMER_TFMR_PARITY_ERROR      = 1ull << (63 - 5),
+    HMER_XSCOM_FAIL             = 1ull << (63 - 8),
+    HMER_XSCOM_DONE             = 1ull << (63 - 9),
+    HMER_PROC_RECV_AGAIN        = 1ull << (63 - 11),
+    HMER_WARN_RISE              = 1ull << (63 - 14),
+    HMER_WARN_FALL              = 1ull << (63 - 15),
+    HMER_SCOM_FIR_HMI           = 1ull << (63 - 16),
+    HMER_TRIG_FIR_HMI           = 1ull << (63 - 17),
+    HMER_HYP_RESOURCE_ERR       = 1ull << (63 - 20),
+    HMER_XSCOM_STATUS_MASK      = 7ull << (63 - 23),
+    HMER_XSCOM_STATUS_LSH       = (63 - 23),
+};
+
+/* Alternate Interrupt Location (AIL) */
+enum {
+    AIL_NONE                = 0,
+    AIL_RESERVED            = 1,
+    AIL_0001_8000           = 2,
+    AIL_C000_0000_0000_4000 = 3,
+};
+
 /*****************************************************************************/
 
 static inline target_ulong cpu_read_xer(CPUPPCState *env)
@@ -2174,7 +2289,7 @@
 }
 
 static inline void cpu_get_tb_cpu_state(CPUPPCState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->nip;
     *cs_base = 0;
@@ -2233,8 +2348,8 @@
 {
     int r;
     uint32_t ways = booke206_tlb_ways(env, tlbn);
-    int ways_bits = ffs(ways) - 1;
-    int tlb_bits = ffs(booke206_tlb_size(env, tlbn)) - 1;
+    int ways_bits = ctz32(ways);
+    int tlb_bits = ctz32(booke206_tlb_size(env, tlbn));
     int i;
 
     way &= ways - 1;
@@ -2286,9 +2401,17 @@
     return msr & (1ULL << MSR_SF);
 }
 
-extern void (*cpu_ppc_hypercall)(PowerPCCPU *);
+/**
+ * Check whether register rx is in the range between start and
+ * start + nregs (as needed by the LSWX and LSWI instructions)
+ */
+static inline bool lsw_reg_in_range(int start, int nregs, int rx)
+{
+    return (start + nregs <= 32 && rx >= start && rx < start + nregs) ||
+           (start + nregs > 32 && (rx >= start || rx < start + nregs - 32));
+}
 
-#include "exec/exec-all.h"
+extern void (*cpu_ppc_hypercall)(PowerPCCPU *);
 
 void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env);
 
@@ -2310,4 +2433,5 @@
  */
 PowerPCCPU *ppc_get_vcpu_by_dt_id(int cpu_dt_id);
 
-#endif /* !defined (__CPU_PPC_H__) */
+void ppc_maybe_bswap_register(CPUPPCState *env, uint8_t *mem_buf, int len);
+#endif /* PPC_CPU_H */
diff --git a/target-ppc/dfp_helper.c b/target-ppc/dfp_helper.c
index 49820bf..db0ede6 100644
--- a/target-ppc/dfp_helper.c
+++ b/target-ppc/dfp_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -170,27 +171,6 @@
     }
 }
 
-#define FP_FX       (1ull << FPSCR_FX)
-#define FP_FEX      (1ull << FPSCR_FEX)
-#define FP_OX       (1ull << FPSCR_OX)
-#define FP_OE       (1ull << FPSCR_OE)
-#define FP_UX       (1ull << FPSCR_UX)
-#define FP_UE       (1ull << FPSCR_UE)
-#define FP_XX       (1ull << FPSCR_XX)
-#define FP_XE       (1ull << FPSCR_XE)
-#define FP_ZX       (1ull << FPSCR_ZX)
-#define FP_ZE       (1ull << FPSCR_ZE)
-#define FP_VX       (1ull << FPSCR_VX)
-#define FP_VXSNAN   (1ull << FPSCR_VXSNAN)
-#define FP_VXISI    (1ull << FPSCR_VXISI)
-#define FP_VXIMZ    (1ull << FPSCR_VXIMZ)
-#define FP_VXZDZ    (1ull << FPSCR_VXZDZ)
-#define FP_VXIDI    (1ull << FPSCR_VXIDI)
-#define FP_VXVC     (1ull << FPSCR_VXVC)
-#define FP_VXCVI    (1ull << FPSCR_VXCVI)
-#define FP_VE       (1ull << FPSCR_VE)
-#define FP_FI       (1ull << FPSCR_FI)
-
 static void dfp_set_FPSCR_flag(struct PPC_DFP *dfp, uint64_t flag,
                 uint64_t enabled)
 {
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index b803475..d6e1678 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -16,13 +16,16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #include "helper_regs.h"
 
 //#define DEBUG_OP
+//#define DEBUG_SOFTWARE_TLB
 //#define DEBUG_EXCEPTIONS
 
 #ifdef DEBUG_EXCEPTIONS
@@ -74,18 +77,8 @@
     CPUState *cs = CPU(cpu);
     CPUPPCState *env = &cpu->env;
     target_ulong msr, new_msr, vector;
-    int srr0, srr1, asrr0, asrr1;
-    int lpes0, lpes1, lev;
-
-    if (0) {
-        /* XXX: find a suitable condition to enable the hypervisor mode */
-        lpes0 = (env->spr[SPR_LPCR] >> 1) & 1;
-        lpes1 = (env->spr[SPR_LPCR] >> 2) & 1;
-    } else {
-        /* Those values ensure we won't enter the hypervisor mode */
-        lpes0 = 0;
-        lpes1 = 1;
-    }
+    int srr0, srr1, asrr0, asrr1, lev, ail;
+    bool lpes0;
 
     qemu_log_mask(CPU_LOG_INT, "Raise exception at " TARGET_FMT_lx
                   " => %08x (%02x)\n", env->nip, excp, env->error_code);
@@ -97,8 +90,10 @@
         msr = env->msr & ~0x783f0000ULL;
     }
 
-    /* new interrupt handler msr */
-    new_msr = env->msr & ((target_ulong)1 << MSR_ME);
+    /* new interrupt handler msr preserves existing HV and ME unless
+     * explicitly overriden
+     */
+    new_msr = env->msr & (((target_ulong)1 << MSR_ME) | MSR_HVB);
 
     /* target registers */
     srr0 = SPR_SRR0;
@@ -106,6 +101,84 @@
     asrr0 = -1;
     asrr1 = -1;
 
+    /* check for special resume at 0x100 from doze/nap/sleep/winkle on P7/P8 */
+    if (env->in_pm_state) {
+        env->in_pm_state = false;
+
+        /* Pretend to be returning from doze always as we don't lose state */
+        msr |= (0x1ull << (63 - 47));
+
+        /* Non-machine check are routed to 0x100 with a wakeup cause
+         * encoded in SRR1
+         */
+        if (excp != POWERPC_EXCP_MCHECK) {
+            switch (excp) {
+            case POWERPC_EXCP_RESET:
+                msr |= 0x4ull << (63 - 45);
+                break;
+            case POWERPC_EXCP_EXTERNAL:
+                msr |= 0x8ull << (63 - 45);
+                break;
+            case POWERPC_EXCP_DECR:
+                msr |= 0x6ull << (63 - 45);
+                break;
+            case POWERPC_EXCP_SDOOR:
+                msr |= 0x5ull << (63 - 45);
+                break;
+            case POWERPC_EXCP_SDOOR_HV:
+                msr |= 0x3ull << (63 - 45);
+                break;
+            case POWERPC_EXCP_HV_MAINT:
+                msr |= 0xaull << (63 - 45);
+                break;
+            default:
+                cpu_abort(cs, "Unsupported exception %d in Power Save mode\n",
+                          excp);
+            }
+            excp = POWERPC_EXCP_RESET;
+        }
+    }
+
+    /* Exception targetting modifiers
+     *
+     * LPES0 is supported on POWER7/8
+     * LPES1 is not supported (old iSeries mode)
+     *
+     * On anything else, we behave as if LPES0 is 1
+     * (externals don't alter MSR:HV)
+     *
+     * AIL is initialized here but can be cleared by
+     * selected exceptions
+     */
+#if defined(TARGET_PPC64)
+    if (excp_model == POWERPC_EXCP_POWER7 ||
+        excp_model == POWERPC_EXCP_POWER8) {
+        lpes0 = !!(env->spr[SPR_LPCR] & LPCR_LPES0);
+        if (excp_model == POWERPC_EXCP_POWER8) {
+            ail = (env->spr[SPR_LPCR] & LPCR_AIL) >> LPCR_AIL_SHIFT;
+        } else {
+            ail = 0;
+        }
+    } else
+#endif /* defined(TARGET_PPC64) */
+    {
+        lpes0 = true;
+        ail = 0;
+    }
+
+    /* Hypervisor emulation assistance interrupt only exists on server
+     * arch 2.05 server or later. We also don't want to generate it if
+     * we don't have HVB in msr_mask (PAPR mode).
+     */
+    if (excp == POWERPC_EXCP_HV_EMU
+#if defined(TARGET_PPC64)
+        && !((env->mmu_model & POWERPC_MMU_64) && (env->msr_mask & MSR_HVB))
+#endif /* defined(TARGET_PPC64) */
+
+    ) {
+        excp = POWERPC_EXCP_PROGRAM;
+    }
+
     switch (excp) {
     case POWERPC_EXCP_NONE:
         /* Should never happen */
@@ -131,20 +204,17 @@
             /* Machine check exception is not enabled.
              * Enter checkstop state.
              */
-            if (qemu_log_enabled()) {
+            fprintf(stderr, "Machine check while not allowed. "
+                    "Entering checkstop state\n");
+            if (qemu_log_separate()) {
                 qemu_log("Machine check while not allowed. "
                         "Entering checkstop state\n");
-            } else {
-                fprintf(stderr, "Machine check while not allowed. "
-                        "Entering checkstop state\n");
             }
             cs->halted = 1;
             cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
         }
-        if (0) {
-            /* XXX: find a suitable condition to enable the hypervisor mode */
-            new_msr |= (target_ulong)MSR_HVB;
-        }
+        new_msr |= (target_ulong)MSR_HVB;
+        ail = 0;
 
         /* machine check exceptions don't have ME set */
         new_msr &= ~((target_ulong)1 << MSR_ME);
@@ -169,23 +239,20 @@
     case POWERPC_EXCP_DSI:       /* Data storage exception                   */
         LOG_EXCP("DSI exception: DSISR=" TARGET_FMT_lx" DAR=" TARGET_FMT_lx
                  "\n", env->spr[SPR_DSISR], env->spr[SPR_DAR]);
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_next;
     case POWERPC_EXCP_ISI:       /* Instruction storage exception            */
         LOG_EXCP("ISI exception: msr=" TARGET_FMT_lx ", nip=" TARGET_FMT_lx
                  "\n", msr, env->nip);
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         msr |= env->error_code;
         goto store_next;
     case POWERPC_EXCP_EXTERNAL:  /* External input                           */
         cs = CPU(cpu);
 
-        if (lpes0 == 1) {
+        if (!lpes0) {
             new_msr |= (target_ulong)MSR_HVB;
+            new_msr |= env->msr & ((target_ulong)1 << MSR_RI);
+            srr0 = SPR_HSRR0;
+            srr1 = SPR_HSRR1;
         }
         if (env->mpic_proxy) {
             /* IACK the IRQ on delivery */
@@ -193,14 +260,11 @@
         }
         goto store_next;
     case POWERPC_EXCP_ALIGN:     /* Alignment exception                      */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         /* XXX: this is false */
         /* Get rS/rD and rA from faulting opcode */
         env->spr[SPR_DSISR] |= (cpu_ldl_code(env, (env->nip - 4))
                                 & 0x03FF0000) >> 16;
-        goto store_current;
+        goto store_next;
     case POWERPC_EXCP_PROGRAM:   /* Program exception                        */
         switch (env->error_code & ~0xF) {
         case POWERPC_EXCP_FP:
@@ -210,9 +274,6 @@
                 env->error_code = 0;
                 return;
             }
-            if (lpes1 == 0) {
-                new_msr |= (target_ulong)MSR_HVB;
-            }
             msr |= 0x00100000;
             if (msr_fe0 == msr_fe1) {
                 goto store_next;
@@ -221,23 +282,14 @@
             break;
         case POWERPC_EXCP_INVAL:
             LOG_EXCP("Invalid instruction at " TARGET_FMT_lx "\n", env->nip);
-            if (lpes1 == 0) {
-                new_msr |= (target_ulong)MSR_HVB;
-            }
             msr |= 0x00080000;
             env->spr[SPR_BOOKE_ESR] = ESR_PIL;
             break;
         case POWERPC_EXCP_PRIV:
-            if (lpes1 == 0) {
-                new_msr |= (target_ulong)MSR_HVB;
-            }
             msr |= 0x00040000;
             env->spr[SPR_BOOKE_ESR] = ESR_PPR;
             break;
         case POWERPC_EXCP_TRAP:
-            if (lpes1 == 0) {
-                new_msr |= (target_ulong)MSR_HVB;
-            }
             msr |= 0x00020000;
             env->spr[SPR_BOOKE_ESR] = ESR_PTR;
             break;
@@ -248,28 +300,30 @@
             break;
         }
         goto store_current;
+    case POWERPC_EXCP_HV_EMU:
+        srr0 = SPR_HSRR0;
+        srr1 = SPR_HSRR1;
+        new_msr |= (target_ulong)MSR_HVB;
+        new_msr |= env->msr & ((target_ulong)1 << MSR_RI);
+        goto store_current;
     case POWERPC_EXCP_FPU:       /* Floating-point unavailable exception     */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_current;
     case POWERPC_EXCP_SYSCALL:   /* System call exception                    */
         dump_syscall(env);
         lev = env->error_code;
+
+        /* "PAPR mode" built-in hypercall emulation */
         if ((lev == 1) && cpu_ppc_hypercall) {
             cpu_ppc_hypercall(cpu);
             return;
         }
-        if (lev == 1 || (lpes0 == 0 && lpes1 == 0)) {
+        if (lev == 1) {
             new_msr |= (target_ulong)MSR_HVB;
         }
         goto store_next;
     case POWERPC_EXCP_APU:       /* Auxiliary processor unavailable          */
         goto store_current;
     case POWERPC_EXCP_DECR:      /* Decrementer exception                    */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_next;
     case POWERPC_EXCP_FIT:       /* Fixed-interval timer interrupt           */
         /* FIT on 4xx */
@@ -339,20 +393,12 @@
             new_msr &= ~((target_ulong)1 << MSR_ME);
         }
 
-        if (0) {
-            /* XXX: find a suitable condition to enable the hypervisor mode */
-            new_msr |= (target_ulong)MSR_HVB;
-        }
+        new_msr |= (target_ulong)MSR_HVB;
+        ail = 0;
         goto store_next;
     case POWERPC_EXCP_DSEG:      /* Data segment exception                   */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_next;
     case POWERPC_EXCP_ISEG:      /* Instruction segment exception            */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_next;
     case POWERPC_EXCP_HDECR:     /* Hypervisor decrementer exception         */
         srr0 = SPR_HSRR0;
@@ -361,9 +407,6 @@
         new_msr |= env->msr & ((target_ulong)1 << MSR_RI);
         goto store_next;
     case POWERPC_EXCP_TRACE:     /* Trace exception                          */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_next;
     case POWERPC_EXCP_HDSI:      /* Hypervisor data storage exception        */
         srr0 = SPR_HSRR0;
@@ -390,19 +433,10 @@
         new_msr |= env->msr & ((target_ulong)1 << MSR_RI);
         goto store_next;
     case POWERPC_EXCP_VPU:       /* Vector unavailable exception             */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_current;
     case POWERPC_EXCP_VSXU:       /* VSX unavailable exception               */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_current;
     case POWERPC_EXCP_FU:         /* Facility unavailable exception          */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         goto store_current;
     case POWERPC_EXCP_PIT:       /* Programmable interval timer interrupt    */
         LOG_EXCP("PIT exception\n");
@@ -421,9 +455,6 @@
                   "is not implemented yet !\n");
         goto store_next;
     case POWERPC_EXCP_IFTLB:     /* Instruction fetch TLB error              */
-        if (lpes1 == 0) { /* XXX: check this */
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         switch (excp_model) {
         case POWERPC_EXCP_602:
         case POWERPC_EXCP_603:
@@ -440,9 +471,6 @@
         }
         break;
     case POWERPC_EXCP_DLTLB:     /* Data load TLB miss                       */
-        if (lpes1 == 0) { /* XXX: check this */
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         switch (excp_model) {
         case POWERPC_EXCP_602:
         case POWERPC_EXCP_603:
@@ -459,9 +487,6 @@
         }
         break;
     case POWERPC_EXCP_DSTLB:     /* Data store TLB miss                      */
-        if (lpes1 == 0) { /* XXX: check this */
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         switch (excp_model) {
         case POWERPC_EXCP_602:
         case POWERPC_EXCP_603:
@@ -567,9 +592,6 @@
                   "is not implemented yet !\n");
         goto store_next;
     case POWERPC_EXCP_PERFM:     /* Embedded performance monitor interrupt   */
-        if (lpes1 == 0) {
-            new_msr |= (target_ulong)MSR_HVB;
-        }
         /* XXX: TODO */
         cpu_abort(cs,
                   "Performance counter exception is not implemented yet !\n");
@@ -613,6 +635,13 @@
     }
     /* Save MSR */
     env->spr[srr1] = msr;
+
+    /* Sanity check */
+    if (!(env->msr_mask & MSR_HVB) && (srr0 == SPR_HSRR0)) {
+        cpu_abort(cs, "Trying to deliver HV exception %d with "
+                  "no HV support\n", excp);
+    }
+
     /* If any alternate SRR register are defined, duplicate saved values */
     if (asrr0 != -1) {
         env->spr[asrr0] = env->spr[srr0];
@@ -621,16 +650,20 @@
         env->spr[asrr1] = env->spr[srr1];
     }
 
-    if (env->spr[SPR_LPCR] & LPCR_AIL) {
-        new_msr |= (1 << MSR_IR) | (1 << MSR_DR);
-    } else if (msr & ((1 << MSR_IR) | (1 << MSR_DR))) {
-        /* If we disactivated any translation, flush TLBs */
-        tlb_flush(cs, 1);
-    }
-
+    /* Sort out endianness of interrupt, this differs depending on the
+     * CPU, the HV mode, etc...
+     */
 #ifdef TARGET_PPC64
     if (excp_model == POWERPC_EXCP_POWER7) {
-        if (env->spr[SPR_LPCR] & LPCR_ILE) {
+        if (!(new_msr & MSR_HVB) && (env->spr[SPR_LPCR] & LPCR_ILE)) {
+            new_msr |= (target_ulong)1 << MSR_LE;
+        }
+    } else if (excp_model == POWERPC_EXCP_POWER8) {
+        if (new_msr & MSR_HVB) {
+            if (env->spr[SPR_HID0] & HID0_HILE) {
+                new_msr |= (target_ulong)1 << MSR_LE;
+            }
+        } else if (env->spr[SPR_LPCR] & LPCR_ILE) {
             new_msr |= (target_ulong)1 << MSR_LE;
         }
     } else if (msr_ile) {
@@ -649,6 +682,30 @@
                   excp);
     }
     vector |= env->excp_prefix;
+
+    /* AIL only works if there is no HV transition and we are running with
+     * translations enabled
+     */
+    if (!((msr >> MSR_IR) & 1) || !((msr >> MSR_DR) & 1) ||
+        ((new_msr & MSR_HVB) && !(msr & MSR_HVB))) {
+        ail = 0;
+    }
+    /* Handle AIL */
+    if (ail) {
+        new_msr |= (1 << MSR_IR) | (1 << MSR_DR);
+        switch(ail) {
+        case AIL_0001_8000:
+            vector |= 0x18000;
+            break;
+        case AIL_C000_0000_0000_4000:
+            vector |= 0xc000000000004000ull;
+            break;
+        default:
+            cpu_abort(cs, "Invalid AIL combination %d\n", ail);
+            break;
+        }
+    }
+
 #if defined(TARGET_PPC64)
     if (excp_model == POWERPC_EXCP_BOOKE) {
         if (env->spr[SPR_BOOKE_EPCR] & EPCR_ICM) {
@@ -665,8 +722,12 @@
         }
     }
 #endif
-    /* XXX: we don't use hreg_store_msr here as already have treated
-     *      any special case that could occur. Just store MSR and update hflags
+    /* We don't use hreg_store_msr here as already have treated
+     * any special case that could occur. Just store MSR and update hflags
+     *
+     * Note: We *MUST* not use hreg_store_msr() as-is anyway because it
+     * will prevent setting of the HV bit which some exceptions might need
+     * to do.
      */
     env->msr = new_msr & env->msr_mask;
     hreg_compute_hflags(env);
@@ -675,13 +736,10 @@
     cs->exception_index = POWERPC_EXCP_NONE;
     env->error_code = 0;
 
-    if ((env->mmu_model == POWERPC_MMU_BOOKE) ||
-        (env->mmu_model == POWERPC_MMU_BOOKE206)) {
-        /* XXX: The BookE changes address space when switching modes,
-                we should probably implement that as different MMU indexes,
-                but for the moment we do it the slow way and flush all.  */
-        tlb_flush(cs, 1);
-    }
+    /* Any interrupt is context synchronizing, check if TCG TLB
+     * needs a delayed flush on ppc64
+     */
+    check_tlb_flush(env);
 }
 
 void ppc_cpu_do_interrupt(CPUState *cs)
@@ -695,7 +753,6 @@
 static void ppc_hw_interrupt(CPUPPCState *env)
 {
     PowerPCCPU *cpu = ppc_env_get_cpu(env);
-    int hdice;
 #if 0
     CPUState *cs = CPU(cpu);
 
@@ -723,19 +780,25 @@
         return;
     }
 #endif
-    if (0) {
-        /* XXX: find a suitable condition to enable the hypervisor mode */
-        hdice = env->spr[SPR_LPCR] & 1;
-    } else {
-        hdice = 0;
-    }
-    if ((msr_ee != 0 || msr_hv == 0 || msr_pr != 0) && hdice != 0) {
-        /* Hypervisor decrementer exception */
-        if (env->pending_interrupts & (1 << PPC_INTERRUPT_HDECR)) {
+    /* Hypervisor decrementer exception */
+    if (env->pending_interrupts & (1 << PPC_INTERRUPT_HDECR)) {
+        /* LPCR will be clear when not supported so this will work */
+        bool hdice = !!(env->spr[SPR_LPCR] & LPCR_HDICE);
+        if ((msr_ee != 0 || msr_hv == 0) && hdice) {
+            /* HDEC clears on delivery */
+            env->pending_interrupts &= ~(1 << PPC_INTERRUPT_HDECR);
             powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_HDECR);
             return;
         }
     }
+    /* Extermal interrupt can ignore MSR:EE under some circumstances */
+    if (env->pending_interrupts & (1 << PPC_INTERRUPT_EXT)) {
+        bool lpes0 = !!(env->spr[SPR_LPCR] & LPCR_LPES0);
+        if (msr_ee != 0 || (env->has_hv_mode && msr_hv == 0 && !lpes0)) {
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_EXTERNAL);
+            return;
+        }
+    }
     if (msr_ce != 0) {
         /* External critical interrupt */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_CEXT)) {
@@ -781,17 +844,6 @@
             powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DECR);
             return;
         }
-        /* External interrupt */
-        if (env->pending_interrupts & (1 << PPC_INTERRUPT_EXT)) {
-            /* Taking an external interrupt does not clear the external
-             * interrupt status
-             */
-#if 0
-            env->pending_interrupts &= ~(1 << PPC_INTERRUPT_EXT);
-#endif
-            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_EXTERNAL);
-            return;
-        }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_DOORBELL)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DOORBELL);
             powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DOORI);
@@ -877,25 +929,46 @@
     }
 }
 
-static inline void do_rfi(CPUPPCState *env, target_ulong nip, target_ulong msr,
-                          target_ulong msrm, int keep_msrh)
+#if defined(TARGET_PPC64)
+void helper_pminsn(CPUPPCState *env, powerpc_pm_insn_t insn)
+{
+    CPUState *cs;
+
+    cs = CPU(ppc_env_get_cpu(env));
+    cs->halted = 1;
+    env->in_pm_state = true;
+
+    /* The architecture specifies that HDEC interrupts are
+     * discarded in PM states
+     */
+    env->pending_interrupts &= ~(1 << PPC_INTERRUPT_HDECR);
+
+    /* Technically, nap doesn't set EE, but if we don't set it
+     * then ppc_hw_interrupt() won't deliver. We could add some
+     * other tests there based on LPCR but it's simpler to just
+     * whack EE in. It will be cleared by the 0x100 at wakeup
+     * anyway. It will still be observable by the guest in SRR1
+     * but this doesn't seem to be a problem.
+     */
+    env->msr |= (1ull << MSR_EE);
+    helper_raise_exception(env, EXCP_HLT);
+}
+#endif /* defined(TARGET_PPC64) */
+
+static inline void do_rfi(CPUPPCState *env, target_ulong nip, target_ulong msr)
 {
     CPUState *cs = CPU(ppc_env_get_cpu(env));
 
+    /* MSR:POW cannot be set by any form of rfi */
+    msr &= ~(1ULL << MSR_POW);
+
 #if defined(TARGET_PPC64)
-    if (msr_is_64bit(env, msr)) {
-        nip = (uint64_t)nip;
-        msr &= (uint64_t)msrm;
-    } else {
+    /* Switching to 32-bit ? Crop the nip */
+    if (!msr_is_64bit(env, msr)) {
         nip = (uint32_t)nip;
-        msr = (uint32_t)(msr & msrm);
-        if (keep_msrh) {
-            msr |= env->msr & ~((uint64_t)0xFFFFFFFF);
-        }
     }
 #else
     nip = (uint32_t)nip;
-    msr &= (uint32_t)msrm;
 #endif
     /* XXX: beware: this is false if VLE is supported */
     env->nip = nip & ~((target_ulong)0x00000003);
@@ -907,30 +980,31 @@
      * as rfi is always the last insn of a TB
      */
     cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
+
+    /* Context synchronizing: check if TCG TLB needs flush */
+    check_tlb_flush(env);
 }
 
 void helper_rfi(CPUPPCState *env)
 {
-    if (env->excp_model == POWERPC_EXCP_BOOKE) {
-        do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
-               ~((target_ulong)0), 0);
-    } else {
-        do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
-               ~((target_ulong)0x783F0000), 1);
-    }
+    do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1] & 0xfffffffful);
 }
 
+#define MSR_BOOK3S_MASK
 #if defined(TARGET_PPC64)
 void helper_rfid(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1],
-           ~((target_ulong)0x783F0000), 0);
+    /* The architeture defines a number of rules for which bits
+     * can change but in practice, we handle this in hreg_store_msr()
+     * which will be called by do_rfi(), so there is no need to filter
+     * here
+     */
+    do_rfi(env, env->spr[SPR_SRR0], env->spr[SPR_SRR1]);
 }
 
 void helper_hrfid(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_HSRR0], env->spr[SPR_HSRR1],
-           ~((target_ulong)0x783F0000), 0);
+    do_rfi(env, env->spr[SPR_HSRR0], env->spr[SPR_HSRR1]);
 }
 #endif
 
@@ -938,28 +1012,24 @@
 /* Embedded PowerPC specific helpers */
 void helper_40x_rfci(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_40x_SRR2], env->spr[SPR_40x_SRR3],
-           ~((target_ulong)0xFFFF0000), 0);
+    do_rfi(env, env->spr[SPR_40x_SRR2], env->spr[SPR_40x_SRR3]);
 }
 
 void helper_rfci(CPUPPCState *env)
 {
-    do_rfi(env, env->spr[SPR_BOOKE_CSRR0], env->spr[SPR_BOOKE_CSRR1],
-           ~((target_ulong)0), 0);
+    do_rfi(env, env->spr[SPR_BOOKE_CSRR0], env->spr[SPR_BOOKE_CSRR1]);
 }
 
 void helper_rfdi(CPUPPCState *env)
 {
     /* FIXME: choose CSRR1 or DSRR1 based on cpu type */
-    do_rfi(env, env->spr[SPR_BOOKE_DSRR0], env->spr[SPR_BOOKE_DSRR1],
-           ~((target_ulong)0), 0);
+    do_rfi(env, env->spr[SPR_BOOKE_DSRR0], env->spr[SPR_BOOKE_DSRR1]);
 }
 
 void helper_rfmci(CPUPPCState *env)
 {
     /* FIXME: choose CSRR1 or MCSRR1 based on cpu type */
-    do_rfi(env, env->spr[SPR_BOOKE_MCSRR0], env->spr[SPR_BOOKE_MCSRR1],
-           ~((target_ulong)0), 0);
+    do_rfi(env, env->spr[SPR_BOOKE_MCSRR0], env->spr[SPR_BOOKE_MCSRR1]);
 }
 #endif
 
@@ -997,7 +1067,7 @@
 
 void helper_rfsvc(CPUPPCState *env)
 {
-    do_rfi(env, env->lr, env->ctr, 0x0000FFFF, 0);
+    do_rfi(env, env->lr, env->ctr & 0x0000FFFF);
 }
 
 /* Embedded.Processor Control */
diff --git a/target-ppc/fpu_helper.c b/target-ppc/fpu_helper.c
index 7f74466..d9795d0 100644
--- a/target-ppc/fpu_helper.c
+++ b/target-ppc/fpu_helper.c
@@ -16,9 +16,13 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
+#define float64_snan_to_qnan(x) ((x) | 0x0008000000000000ULL)
+#define float32_snan_to_qnan(x) ((x) | 0x00400000)
+
 /*****************************************************************************/
 /* Floating point operations helpers */
 uint64_t helper_float32_to_float64(CPUPPCState *env, uint32_t arg)
@@ -60,59 +64,55 @@
     return ((f >> 52) & 0x7FF) - 1023;
 }
 
-uint32_t helper_compute_fprf(CPUPPCState *env, uint64_t arg, uint32_t set_fprf)
+void helper_compute_fprf(CPUPPCState *env, uint64_t arg)
 {
     CPU_DoubleU farg;
     int isneg;
-    int ret;
+    int fprf;
 
     farg.ll = arg;
     isneg = float64_is_neg(farg.d);
     if (unlikely(float64_is_any_nan(farg.d))) {
-        if (float64_is_signaling_nan(farg.d)) {
+        if (float64_is_signaling_nan(farg.d, &env->fp_status)) {
             /* Signaling NaN: flags are undefined */
-            ret = 0x00;
+            fprf = 0x00;
         } else {
             /* Quiet NaN */
-            ret = 0x11;
+            fprf = 0x11;
         }
     } else if (unlikely(float64_is_infinity(farg.d))) {
         /* +/- infinity */
         if (isneg) {
-            ret = 0x09;
+            fprf = 0x09;
         } else {
-            ret = 0x05;
+            fprf = 0x05;
         }
     } else {
         if (float64_is_zero(farg.d)) {
             /* +/- zero */
             if (isneg) {
-                ret = 0x12;
+                fprf = 0x12;
             } else {
-                ret = 0x02;
+                fprf = 0x02;
             }
         } else {
             if (isden(farg.d)) {
                 /* Denormalized numbers */
-                ret = 0x10;
+                fprf = 0x10;
             } else {
                 /* Normalized numbers */
-                ret = 0x00;
+                fprf = 0x00;
             }
             if (isneg) {
-                ret |= 0x08;
+                fprf |= 0x08;
             } else {
-                ret |= 0x04;
+                fprf |= 0x04;
             }
         }
     }
-    if (set_fprf) {
-        /* We update FPSCR_FPRF */
-        env->fpscr &= ~(0x1F << FPSCR_FPRF);
-        env->fpscr |= ret << FPSCR_FPRF;
-    }
-    /* We just need fpcc to update Rc1 */
-    return ret & 0xF;
+    /* We update FPSCR_FPRF */
+    env->fpscr &= ~(0x1F << FPSCR_FPRF);
+    env->fpscr |= fprf << FPSCR_FPRF;
 }
 
 /* Floating-point invalid operations exception */
@@ -195,7 +195,7 @@
     /* Update the floating-point invalid operation summary */
     env->fpscr |= 1 << FPSCR_VX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (ve != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -212,7 +212,7 @@
     env->fpscr |= 1 << FPSCR_ZX;
     env->fpscr &= ~((1 << FPSCR_FR) | (1 << FPSCR_FI));
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_ze != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -229,7 +229,7 @@
 
     env->fpscr |= 1 << FPSCR_OX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_oe != 0) {
         /* XXX: should adjust the result */
         /* Update the floating-point enabled exception summary */
@@ -249,7 +249,7 @@
 
     env->fpscr |= 1 << FPSCR_UX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_ue != 0) {
         /* XXX: should adjust the result */
         /* Update the floating-point enabled exception summary */
@@ -266,7 +266,7 @@
 
     env->fpscr |= 1 << FPSCR_XX;
     /* Update the floating-point exception summary */
-    env->fpscr |= 1 << FPSCR_FX;
+    env->fpscr |= FP_FX;
     if (fpscr_xe != 0) {
         /* Update the floating-point enabled exception summary */
         env->fpscr |= 1 << FPSCR_FEX;
@@ -331,31 +331,31 @@
     if (prev == 0) {
         switch (bit) {
         case FPSCR_VX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ve) {
                 goto raise_ve;
             }
             break;
         case FPSCR_OX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_oe) {
                 goto raise_oe;
             }
             break;
         case FPSCR_UX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ue) {
                 goto raise_ue;
             }
             break;
         case FPSCR_ZX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ze) {
                 goto raise_ze;
             }
             break;
         case FPSCR_XX:
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_xe) {
                 goto raise_xe;
             }
@@ -370,7 +370,7 @@
         case FPSCR_VXSQRT:
         case FPSCR_VXCVI:
             env->fpscr |= 1 << FPSCR_VX;
-            env->fpscr |= 1 << FPSCR_FX;
+            env->fpscr |= FP_FX;
             if (fpscr_ve != 0) {
                 goto raise_ve;
             }
@@ -534,8 +534,8 @@
         /* Magnitude subtraction of infinities */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXISI, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status))) {
             /* sNaN addition */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -558,8 +558,8 @@
         /* Magnitude subtraction of infinities */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXISI, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status))) {
             /* sNaN subtraction */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -582,8 +582,8 @@
         /* Multiplication of zero by infinity */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXIMZ, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status))) {
             /* sNaN multiplication */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -609,8 +609,8 @@
         /* Division of zero by zero */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXZDZ, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status))) {
             /* sNaN division */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -632,7 +632,7 @@
     if (unlikely(env->fp_status.float_exception_flags)) {              \
         if (float64_is_any_nan(arg)) {                                 \
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXCVI, 1);      \
-            if (float64_is_signaling_nan(arg)) {                       \
+            if (float64_is_signaling_nan(arg, &env->fp_status)) {      \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1); \
             }                                                          \
             farg.ll = nanval;                                          \
@@ -681,7 +681,7 @@
 
     farg.ll = arg;
 
-    if (unlikely(float64_is_signaling_nan(farg.d))) {
+    if (unlikely(float64_is_signaling_nan(farg.d, &env->fp_status))) {
         /* sNaN round */
         fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         farg.ll = arg | 0x0008000000000000ULL;
@@ -737,9 +737,9 @@
         /* Multiplication of zero by infinity */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXIMZ, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d) ||
-                     float64_is_signaling_nan(farg3.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg3.d, &env->fp_status))) {
             /* sNaN operation */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -780,9 +780,9 @@
         /* Multiplication of zero by infinity */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXIMZ, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d) ||
-                     float64_is_signaling_nan(farg3.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg3.d, &env->fp_status))) {
             /* sNaN operation */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -821,9 +821,9 @@
         /* Multiplication of zero by infinity */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXIMZ, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d) ||
-                     float64_is_signaling_nan(farg3.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg3.d, &env->fp_status))) {
             /* sNaN operation */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -866,9 +866,9 @@
         /* Multiplication of zero by infinity */
         farg1.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXIMZ, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d) ||
-                     float64_is_signaling_nan(farg3.d))) {
+        if (unlikely(float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg3.d, &env->fp_status))) {
             /* sNaN operation */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
         }
@@ -903,7 +903,7 @@
 
     farg.ll = arg;
 
-    if (unlikely(float64_is_signaling_nan(farg.d))) {
+    if (unlikely(float64_is_signaling_nan(farg.d, &env->fp_status))) {
         /* sNaN square root */
         fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
     }
@@ -920,14 +920,16 @@
 
     farg.ll = arg;
 
-    if (unlikely(float64_is_neg(farg.d) && !float64_is_zero(farg.d))) {
+    if (unlikely(float64_is_any_nan(farg.d))) {
+        if (unlikely(float64_is_signaling_nan(farg.d, &env->fp_status))) {
+            /* sNaN reciprocal square root */
+            fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
+            farg.ll = float64_snan_to_qnan(farg.ll);
+        }
+    } else if (unlikely(float64_is_neg(farg.d) && !float64_is_zero(farg.d))) {
         /* Square root of a negative nonzero number */
         farg.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSQRT, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg.d))) {
-            /* sNaN square root */
-            fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
-        }
         farg.d = float64_sqrt(farg.d, &env->fp_status);
     }
     return farg.ll;
@@ -940,7 +942,7 @@
 
     farg.ll = arg;
 
-    if (unlikely(float64_is_signaling_nan(farg.d))) {
+    if (unlikely(float64_is_signaling_nan(farg.d, &env->fp_status))) {
         /* sNaN reciprocal */
         fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
     }
@@ -956,7 +958,7 @@
 
     farg.ll = arg;
 
-    if (unlikely(float64_is_signaling_nan(farg.d))) {
+    if (unlikely(float64_is_signaling_nan(farg.d, &env->fp_status))) {
         /* sNaN reciprocal */
         fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
     }
@@ -974,17 +976,20 @@
 
     farg.ll = arg;
 
-    if (unlikely(float64_is_neg(farg.d) && !float64_is_zero(farg.d))) {
+    if (unlikely(float64_is_any_nan(farg.d))) {
+        if (unlikely(float64_is_signaling_nan(farg.d, &env->fp_status))) {
+            /* sNaN reciprocal square root */
+            fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
+            farg.ll = float64_snan_to_qnan(farg.ll);
+        }
+    } else if (unlikely(float64_is_neg(farg.d) && !float64_is_zero(farg.d))) {
         /* Reciprocal square root of a negative nonzero number */
         farg.ll = fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSQRT, 1);
     } else {
-        if (unlikely(float64_is_signaling_nan(farg.d))) {
-            /* sNaN reciprocal square root */
-            fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
-        }
         farg.d = float64_sqrt(farg.d, &env->fp_status);
         farg.d = float64_div(float64_one, farg.d, &env->fp_status);
     }
+
     return farg.ll;
 }
 
@@ -1095,8 +1100,8 @@
     env->fpscr |= ret << FPSCR_FPRF;
     env->crf[crfD] = ret;
     if (unlikely(ret == 0x01UL
-                 && (float64_is_signaling_nan(farg1.d) ||
-                     float64_is_signaling_nan(farg2.d)))) {
+                 && (float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+                     float64_is_signaling_nan(farg2.d, &env->fp_status)))) {
         /* sNaN comparison */
         fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 1);
     }
@@ -1126,8 +1131,8 @@
     env->fpscr |= ret << FPSCR_FPRF;
     env->crf[crfD] = ret;
     if (unlikely(ret == 0x01UL)) {
-        if (float64_is_signaling_nan(farg1.d) ||
-            float64_is_signaling_nan(farg2.d)) {
+        if (float64_is_signaling_nan(farg1.d, &env->fp_status) ||
+            float64_is_signaling_nan(farg2.d, &env->fp_status)) {
             /* sNaN comparison */
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN |
                                   POWERPC_EXCP_FP_VXVC, 1);
@@ -1163,7 +1168,7 @@
 
     u.l = val;
     /* NaN are not treated the same way IEEE 754 does */
-    if (unlikely(float32_is_quiet_nan(u.f))) {
+    if (unlikely(float32_is_quiet_nan(u.f, &env->vec_status))) {
         return 0;
     }
 
@@ -1176,7 +1181,7 @@
 
     u.l = val;
     /* NaN are not treated the same way IEEE 754 does */
-    if (unlikely(float32_is_quiet_nan(u.f))) {
+    if (unlikely(float32_is_quiet_nan(u.f, &env->vec_status))) {
         return 0;
     }
 
@@ -1189,7 +1194,7 @@
 
     u.l = val;
     /* NaN are not treated the same way IEEE 754 does */
-    if (unlikely(float32_is_quiet_nan(u.f))) {
+    if (unlikely(float32_is_quiet_nan(u.f, &env->vec_status))) {
         return 0;
     }
 
@@ -1202,7 +1207,7 @@
 
     u.l = val;
     /* NaN are not treated the same way IEEE 754 does */
-    if (unlikely(float32_is_quiet_nan(u.f))) {
+    if (unlikely(float32_is_quiet_nan(u.f, &env->vec_status))) {
         return 0;
     }
 
@@ -1240,7 +1245,7 @@
 
     u.l = val;
     /* NaN are not treated the same way IEEE 754 does */
-    if (unlikely(float32_is_quiet_nan(u.f))) {
+    if (unlikely(float32_is_quiet_nan(u.f, &env->vec_status))) {
         return 0;
     }
     tmp = uint64_to_float32(1ULL << 32, &env->vec_status);
@@ -1256,7 +1261,7 @@
 
     u.l = val;
     /* NaN are not treated the same way IEEE 754 does */
-    if (unlikely(float32_is_quiet_nan(u.f))) {
+    if (unlikely(float32_is_quiet_nan(u.f, &env->vec_status))) {
         return 0;
     }
     tmp = uint64_to_float32(1ULL << 32, &env->vec_status);
@@ -1437,7 +1442,7 @@
 #define HELPER_SINGLE_SPE_CMP(name)                                     \
     uint32_t helper_e##name(CPUPPCState *env, uint32_t op1, uint32_t op2) \
     {                                                                   \
-        return e##name(env, op1, op2) << 2;                             \
+        return e##name(env, op1, op2);                                  \
     }
 /* efststlt */
 HELPER_SINGLE_SPE_CMP(fststlt);
@@ -1834,8 +1839,8 @@
         if (unlikely(tstat.float_exception_flags & float_flag_invalid)) {    \
             if (tp##_is_infinity(xa.fld) && tp##_is_infinity(xb.fld)) {      \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXISI, sfprf);    \
-            } else if (tp##_is_signaling_nan(xa.fld) ||                      \
-                       tp##_is_signaling_nan(xb.fld)) {                      \
+            } else if (tp##_is_signaling_nan(xa.fld, &tstat) ||              \
+                       tp##_is_signaling_nan(xb.fld, &tstat)) {              \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);   \
             }                                                                \
         }                                                                    \
@@ -1845,7 +1850,7 @@
         }                                                                    \
                                                                              \
         if (sfprf) {                                                         \
-            helper_compute_fprf(env, xt.fld, sfprf);                         \
+            helper_compute_fprf(env, xt.fld);                                \
         }                                                                    \
     }                                                                        \
     putVSR(xT(opcode), &xt, env);                                            \
@@ -1889,8 +1894,8 @@
             if ((tp##_is_infinity(xa.fld) && tp##_is_zero(xb.fld)) ||        \
                 (tp##_is_infinity(xb.fld) && tp##_is_zero(xa.fld))) {        \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXIMZ, sfprf);    \
-            } else if (tp##_is_signaling_nan(xa.fld) ||                      \
-                       tp##_is_signaling_nan(xb.fld)) {                      \
+            } else if (tp##_is_signaling_nan(xa.fld, &tstat) ||              \
+                       tp##_is_signaling_nan(xb.fld, &tstat)) {              \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);   \
             }                                                                \
         }                                                                    \
@@ -1900,7 +1905,7 @@
         }                                                                    \
                                                                              \
         if (sfprf) {                                                         \
-            helper_compute_fprf(env, xt.fld, sfprf);                         \
+            helper_compute_fprf(env, xt.fld);                                \
         }                                                                    \
     }                                                                        \
                                                                              \
@@ -1943,8 +1948,8 @@
             } else if (tp##_is_zero(xa.fld) &&                                \
                 tp##_is_zero(xb.fld)) {                                       \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXZDZ, sfprf);     \
-            } else if (tp##_is_signaling_nan(xa.fld) ||                       \
-                tp##_is_signaling_nan(xb.fld)) {                              \
+            } else if (tp##_is_signaling_nan(xa.fld, &tstat) ||               \
+                tp##_is_signaling_nan(xb.fld, &tstat)) {                      \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);    \
             }                                                                 \
         }                                                                     \
@@ -1954,7 +1959,7 @@
         }                                                                     \
                                                                               \
         if (sfprf) {                                                          \
-            helper_compute_fprf(env, xt.fld, sfprf);                          \
+            helper_compute_fprf(env, xt.fld);                                 \
         }                                                                     \
     }                                                                         \
                                                                               \
@@ -1985,7 +1990,7 @@
     helper_reset_fpstatus(env);                                               \
                                                                               \
     for (i = 0; i < nels; i++) {                                              \
-        if (unlikely(tp##_is_signaling_nan(xb.fld))) {                        \
+        if (unlikely(tp##_is_signaling_nan(xb.fld, &env->fp_status))) {       \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);    \
         }                                                                     \
         xt.fld = tp##_div(tp##_one, xb.fld, &env->fp_status);                 \
@@ -1995,7 +2000,7 @@
         }                                                                     \
                                                                               \
         if (sfprf) {                                                          \
-            helper_compute_fprf(env, xt.fld, sfprf);                          \
+            helper_compute_fprf(env, xt.fld);                                 \
         }                                                                     \
     }                                                                         \
                                                                               \
@@ -2034,7 +2039,7 @@
         if (unlikely(tstat.float_exception_flags & float_flag_invalid)) {    \
             if (tp##_is_neg(xb.fld) && !tp##_is_zero(xb.fld)) {              \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSQRT, sfprf);   \
-            } else if (tp##_is_signaling_nan(xb.fld)) {                      \
+            } else if (tp##_is_signaling_nan(xb.fld, &tstat)) {              \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);   \
             }                                                                \
         }                                                                    \
@@ -2044,7 +2049,7 @@
         }                                                                    \
                                                                              \
         if (sfprf) {                                                         \
-            helper_compute_fprf(env, xt.fld, sfprf);                         \
+            helper_compute_fprf(env, xt.fld);                                \
         }                                                                    \
     }                                                                        \
                                                                              \
@@ -2084,7 +2089,7 @@
         if (unlikely(tstat.float_exception_flags & float_flag_invalid)) {    \
             if (tp##_is_neg(xb.fld) && !tp##_is_zero(xb.fld)) {              \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSQRT, sfprf);   \
-            } else if (tp##_is_signaling_nan(xb.fld)) {                      \
+            } else if (tp##_is_signaling_nan(xb.fld, &tstat)) {              \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);   \
             }                                                                \
         }                                                                    \
@@ -2094,7 +2099,7 @@
         }                                                                    \
                                                                              \
         if (sfprf) {                                                         \
-            helper_compute_fprf(env, xt.fld, sfprf);                         \
+            helper_compute_fprf(env, xt.fld);                                \
         }                                                                    \
     }                                                                        \
                                                                              \
@@ -2269,9 +2274,9 @@
         env->fp_status.float_exception_flags |= tstat.float_exception_flags;  \
                                                                               \
         if (unlikely(tstat.float_exception_flags & float_flag_invalid)) {     \
-            if (tp##_is_signaling_nan(xa.fld) ||                              \
-                tp##_is_signaling_nan(b->fld) ||                              \
-                tp##_is_signaling_nan(c->fld)) {                              \
+            if (tp##_is_signaling_nan(xa.fld, &tstat) ||                      \
+                tp##_is_signaling_nan(b->fld, &tstat) ||                      \
+                tp##_is_signaling_nan(c->fld, &tstat)) {                      \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, sfprf);    \
                 tstat.float_exception_flags &= ~float_flag_invalid;           \
             }                                                                 \
@@ -2294,7 +2299,7 @@
         }                                                                     \
                                                                               \
         if (sfprf) {                                                          \
-            helper_compute_fprf(env, xt_out.fld, sfprf);                      \
+            helper_compute_fprf(env, xt_out.fld);                             \
         }                                                                     \
     }                                                                         \
     putVSR(xT(opcode), &xt_out, env);                                         \
@@ -2353,8 +2358,8 @@
                                                                          \
     if (unlikely(float64_is_any_nan(xa.VsrD(0)) ||                       \
                  float64_is_any_nan(xb.VsrD(0)))) {                      \
-        if (float64_is_signaling_nan(xa.VsrD(0)) ||                      \
-            float64_is_signaling_nan(xb.VsrD(0))) {                      \
+        if (float64_is_signaling_nan(xa.VsrD(0), &env->fp_status) ||     \
+            float64_is_signaling_nan(xb.VsrD(0), &env->fp_status)) {     \
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 0);       \
         }                                                                \
         if (ordered) {                                                   \
@@ -2382,9 +2387,6 @@
 VSX_SCALAR_CMP(xscmpodp, 1)
 VSX_SCALAR_CMP(xscmpudp, 0)
 
-#define float64_snan_to_qnan(x) ((x) | 0x0008000000000000ULL)
-#define float32_snan_to_qnan(x) ((x) | 0x00400000)
-
 /* VSX_MAX_MIN - VSX floating point maximum/minimum
  *   name  - instruction mnemonic
  *   op    - operation (max or min)
@@ -2404,8 +2406,8 @@
                                                                               \
     for (i = 0; i < nels; i++) {                                              \
         xt.fld = tp##_##op(xa.fld, xb.fld, &env->fp_status);                  \
-        if (unlikely(tp##_is_signaling_nan(xa.fld) ||                         \
-                     tp##_is_signaling_nan(xb.fld))) {                        \
+        if (unlikely(tp##_is_signaling_nan(xa.fld, &env->fp_status) ||        \
+                     tp##_is_signaling_nan(xb.fld, &env->fp_status))) {       \
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 0);            \
         }                                                                     \
     }                                                                         \
@@ -2444,8 +2446,8 @@
     for (i = 0; i < nels; i++) {                                          \
         if (unlikely(tp##_is_any_nan(xa.fld) ||                           \
                      tp##_is_any_nan(xb.fld))) {                          \
-            if (tp##_is_signaling_nan(xa.fld) ||                          \
-                tp##_is_signaling_nan(xb.fld)) {                          \
+            if (tp##_is_signaling_nan(xa.fld, &env->fp_status) ||         \
+                tp##_is_signaling_nan(xb.fld, &env->fp_status)) {         \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 0);    \
             }                                                             \
             if (svxvc) {                                                  \
@@ -2498,13 +2500,14 @@
                                                                    \
     for (i = 0; i < nels; i++) {                                   \
         xt.tfld = stp##_to_##ttp(xb.sfld, &env->fp_status);        \
-        if (unlikely(stp##_is_signaling_nan(xb.sfld))) {           \
+        if (unlikely(stp##_is_signaling_nan(xb.sfld,               \
+                                            &env->fp_status))) {   \
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 0); \
             xt.tfld = ttp##_snan_to_qnan(xt.tfld);                 \
         }                                                          \
         if (sfprf) {                                               \
             helper_compute_fprf(env, ttp##_to_float64(xt.tfld,     \
-                                &env->fp_status), sfprf);          \
+                                &env->fp_status));                 \
         }                                                          \
     }                                                              \
                                                                    \
@@ -2553,7 +2556,7 @@
                                                                              \
     for (i = 0; i < nels; i++) {                                             \
         if (unlikely(stp##_is_any_nan(xb.sfld))) {                           \
-            if (stp##_is_signaling_nan(xb.sfld)) {                           \
+            if (stp##_is_signaling_nan(xb.sfld, &env->fp_status)) {          \
                 fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 0);       \
             }                                                                \
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXCVI, 0);            \
@@ -2614,7 +2617,7 @@
             xt.tfld = helper_frsp(env, xt.tfld);                        \
         }                                                               \
         if (sfprf) {                                                    \
-            helper_compute_fprf(env, xt.tfld, sfprf);                   \
+            helper_compute_fprf(env, xt.tfld);                          \
         }                                                               \
     }                                                                   \
                                                                         \
@@ -2662,14 +2665,15 @@
     }                                                                  \
                                                                        \
     for (i = 0; i < nels; i++) {                                       \
-        if (unlikely(tp##_is_signaling_nan(xb.fld))) {                 \
+        if (unlikely(tp##_is_signaling_nan(xb.fld,                     \
+                                           &env->fp_status))) {        \
             fload_invalid_op_excp(env, POWERPC_EXCP_FP_VXSNAN, 0);     \
             xt.fld = tp##_snan_to_qnan(xb.fld);                        \
         } else {                                                       \
             xt.fld = tp##_round_to_int(xb.fld, &env->fp_status);       \
         }                                                              \
         if (sfprf) {                                                   \
-            helper_compute_fprf(env, xt.fld, sfprf);                   \
+            helper_compute_fprf(env, xt.fld);                          \
         }                                                              \
     }                                                                  \
                                                                        \
@@ -2685,19 +2689,19 @@
     helper_float_check_status(env);                                    \
 }
 
-VSX_ROUND(xsrdpi, 1, float64, VsrD(0), float_round_nearest_even, 1)
+VSX_ROUND(xsrdpi, 1, float64, VsrD(0), float_round_ties_away, 1)
 VSX_ROUND(xsrdpic, 1, float64, VsrD(0), FLOAT_ROUND_CURRENT, 1)
 VSX_ROUND(xsrdpim, 1, float64, VsrD(0), float_round_down, 1)
 VSX_ROUND(xsrdpip, 1, float64, VsrD(0), float_round_up, 1)
 VSX_ROUND(xsrdpiz, 1, float64, VsrD(0), float_round_to_zero, 1)
 
-VSX_ROUND(xvrdpi, 2, float64, VsrD(i), float_round_nearest_even, 0)
+VSX_ROUND(xvrdpi, 2, float64, VsrD(i), float_round_ties_away, 0)
 VSX_ROUND(xvrdpic, 2, float64, VsrD(i), FLOAT_ROUND_CURRENT, 0)
 VSX_ROUND(xvrdpim, 2, float64, VsrD(i), float_round_down, 0)
 VSX_ROUND(xvrdpip, 2, float64, VsrD(i), float_round_up, 0)
 VSX_ROUND(xvrdpiz, 2, float64, VsrD(i), float_round_to_zero, 0)
 
-VSX_ROUND(xvrspi, 4, float32, VsrW(i), float_round_nearest_even, 0)
+VSX_ROUND(xvrspi, 4, float32, VsrW(i), float_round_ties_away, 0)
 VSX_ROUND(xvrspic, 4, float32, VsrW(i), FLOAT_ROUND_CURRENT, 0)
 VSX_ROUND(xvrspim, 4, float32, VsrW(i), float_round_down, 0)
 VSX_ROUND(xvrspip, 4, float32, VsrW(i), float_round_up, 0)
@@ -2709,7 +2713,7 @@
 
     uint64_t xt = helper_frsp(env, xb);
 
-    helper_compute_fprf(env, xt, 1);
+    helper_compute_fprf(env, xt);
     helper_float_check_status(env);
     return xt;
 }
diff --git a/target-ppc/gdbstub.c b/target-ppc/gdbstub.c
index 14675f4..7a33813 100644
--- a/target-ppc/gdbstub.c
+++ b/target-ppc/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 static int ppc_gdb_register_len_apple(int n)
@@ -88,7 +89,7 @@
    the proper ordering for the binary, and cannot be changed.
    For system mode, TARGET_WORDS_BIGENDIAN is always set, and we must check
    the current mode of the chip to see if we're running in little-endian.  */
-static void maybe_bswap_register(CPUPPCState *env, uint8_t *mem_buf, int len)
+void ppc_maybe_bswap_register(CPUPPCState *env, uint8_t *mem_buf, int len)
 {
 #ifndef CONFIG_USER_ONLY
     if (!msr_le) {
@@ -158,7 +159,7 @@
             break;
         }
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     return r;
 }
 
@@ -214,7 +215,7 @@
             break;
         }
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     return r;
 }
 
@@ -227,7 +228,7 @@
     if (!r) {
         return r;
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     if (n < 32) {
         /* gprs */
         env->gpr[n] = ldtul_p(mem_buf);
@@ -277,7 +278,7 @@
     if (!r) {
         return r;
     }
-    maybe_bswap_register(env, mem_buf, r);
+    ppc_maybe_bswap_register(env, mem_buf, r);
     if (n < 32) {
         /* gprs */
         env->gpr[n] = ldq_p(mem_buf);
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index 210fd97..1f5cfd0 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -13,9 +13,12 @@
 DEF_HELPER_1(rfdi, void, env)
 DEF_HELPER_1(rfmci, void, env)
 #if defined(TARGET_PPC64)
+DEF_HELPER_2(pminsn, void, env, i32)
 DEF_HELPER_1(rfid, void, env)
 DEF_HELPER_1(hrfid, void, env)
+DEF_HELPER_2(store_lpcr, void, env, tl)
 #endif
+DEF_HELPER_1(check_tlb_flush, void, env)
 #endif
 
 DEF_HELPER_3(lmw, void, env, tl, i32)
@@ -52,7 +55,7 @@
 
 DEF_HELPER_1(float_check_status, void, env)
 DEF_HELPER_1(reset_fpstatus, void, env)
-DEF_HELPER_3(compute_fprf, i32, env, i64, i32)
+DEF_HELPER_2(compute_fprf, void, env, i64)
 DEF_HELPER_3(store_fpscr, void, env, i64, i32)
 DEF_HELPER_2(fpscr_clrbit, void, env, i32)
 DEF_HELPER_2(fpscr_setbit, void, env, i32)
@@ -544,10 +547,12 @@
 DEF_HELPER_2(74xx_tlbi, void, env, tl)
 DEF_HELPER_FLAGS_1(tlbia, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_FLAGS_2(tlbie, TCG_CALL_NO_RWG, void, env, tl)
+DEF_HELPER_FLAGS_2(tlbiva, TCG_CALL_NO_RWG, void, env, tl)
 #if defined(TARGET_PPC64)
 DEF_HELPER_FLAGS_3(store_slb, TCG_CALL_NO_RWG, void, env, tl, tl)
 DEF_HELPER_2(load_slb_esid, tl, env, tl)
 DEF_HELPER_2(load_slb_vsid, tl, env, tl)
+DEF_HELPER_2(find_slb_vsid, tl, env, tl)
 DEF_HELPER_FLAGS_1(slbia, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_FLAGS_2(slbie, TCG_CALL_NO_RWG, void, env, tl)
 #endif
@@ -595,6 +600,8 @@
 DEF_HELPER_2(store_601_rtcu, void, env, tl)
 DEF_HELPER_1(load_decr, tl, env)
 DEF_HELPER_2(store_decr, void, env, tl)
+DEF_HELPER_1(load_hdecr, tl, env)
+DEF_HELPER_2(store_hdecr, void, env, tl)
 DEF_HELPER_2(store_hid0_601, void, env, tl)
 DEF_HELPER_3(store_403_pbr, void, env, i32, tl)
 DEF_HELPER_1(load_40x_pit, tl, env)
@@ -665,3 +672,6 @@
 DEF_HELPER_4(dscriq, void, env, fprp, fprp, i32)
 DEF_HELPER_4(dscli, void, env, fprp, fprp, i32)
 DEF_HELPER_4(dscliq, void, env, fprp, fprp, i32)
+
+DEF_HELPER_1(tbegin, void, env)
+DEF_HELPER_1(fixup_thrm, void, env)
diff --git a/target-ppc/helper_regs.h b/target-ppc/helper_regs.h
index 271fddf..3d279f1 100644
--- a/target-ppc/helper_regs.h
+++ b/target-ppc/helper_regs.h
@@ -17,8 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#if !defined(__HELPER_REGS_H__)
-#define __HELPER_REGS_H__
+#ifndef HELPER_REGS_H
+#define HELPER_REGS_H
 
 /* Swap temporary saved registers with GPRs */
 static inline void hreg_swap_gpr_tgpr(CPUPPCState *env)
@@ -41,11 +41,43 @@
 
 static inline void hreg_compute_mem_idx(CPUPPCState *env)
 {
-    /* Precompute MMU index */
-    if (msr_pr == 0 && msr_hv != 0) {
-        env->mmu_idx = 2;
+    /* This is our encoding for server processors. The architecture
+     * specifies that there is no such thing as userspace with
+     * translation off, however it appears that MacOS does it and
+     * some 32-bit CPUs support it. Weird...
+     *
+     *   0 = Guest User space virtual mode
+     *   1 = Guest Kernel space virtual mode
+     *   2 = Guest User space real mode
+     *   3 = Guest Kernel space real mode
+     *   4 = HV User space virtual mode
+     *   5 = HV Kernel space virtual mode
+     *   6 = HV User space real mode
+     *   7 = HV Kernel space real mode
+     *
+     * For BookE, we need 8 MMU modes as follow:
+     *
+     *  0 = AS 0 HV User space
+     *  1 = AS 0 HV Kernel space
+     *  2 = AS 1 HV User space
+     *  3 = AS 1 HV Kernel space
+     *  4 = AS 0 Guest User space
+     *  5 = AS 0 Guest Kernel space
+     *  6 = AS 1 Guest User space
+     *  7 = AS 1 Guest Kernel space
+     */
+    if (env->mmu_model & POWERPC_MMU_BOOKE) {
+        env->immu_idx = env->dmmu_idx = msr_pr ? 0 : 1;
+        env->immu_idx += msr_is ? 2 : 0;
+        env->dmmu_idx += msr_ds ? 2 : 0;
+        env->immu_idx += msr_gs ? 4 : 0;
+        env->dmmu_idx += msr_gs ? 4 : 0;
     } else {
-        env->mmu_idx = 1 - msr_pr;
+        env->immu_idx = env->dmmu_idx = msr_pr ? 0 : 1;
+        env->immu_idx += msr_ir ? 0 : 2;
+        env->dmmu_idx += msr_dr ? 0 : 2;
+        env->immu_idx += msr_hv ? 4 : 0;
+        env->dmmu_idx += msr_hv ? 4 : 0;
     }
 }
 
@@ -56,7 +88,7 @@
     /* We 'forget' FE0 & FE1: we'll never generate imprecise exceptions */
     hflags_mask = (1 << MSR_VR) | (1 << MSR_AP) | (1 << MSR_SA) |
         (1 << MSR_PR) | (1 << MSR_FP) | (1 << MSR_SE) | (1 << MSR_BE) |
-        (1 << MSR_LE) | (1 << MSR_VSX);
+        (1 << MSR_LE) | (1 << MSR_VSX) | (1 << MSR_IR) | (1 << MSR_DR);
     hflags_mask |= (1ULL << MSR_CM) | (1ULL << MSR_SF) | MSR_HVB;
     hreg_compute_mem_idx(env);
     env->hflags = env->msr & hflags_mask;
@@ -75,16 +107,17 @@
     excp = 0;
     value &= env->msr_mask;
 #if !defined(CONFIG_USER_ONLY)
-    if (!alter_hv) {
-        /* mtmsr cannot alter the hypervisor state */
+    /* Neither mtmsr nor guest state can alter HV */
+    if (!alter_hv || !(env->msr & MSR_HVB)) {
         value &= ~MSR_HVB;
         value |= env->msr & MSR_HVB;
     }
     if (((value >> MSR_IR) & 1) != msr_ir ||
         ((value >> MSR_DR) & 1) != msr_dr) {
-        /* Flush all tlb when changing translation mode */
-        tlb_flush(cs, 1);
-        excp = POWERPC_EXCP_NONE;
+        cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
+    }
+    if ((env->mmu_model & POWERPC_MMU_BOOKE) &&
+        ((value >> MSR_GS) & 1) != msr_gs) {
         cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
     }
     if (unlikely((env->flags & POWERPC_FLAG_TGPR) &&
@@ -96,6 +129,15 @@
         /* Change the exception prefix on PowerPC 601 */
         env->excp_prefix = ((value >> MSR_EP) & 1) * 0xFFF00000;
     }
+    /* If PR=1 then EE, IR and DR must be 1
+     *
+     * Note: We only enforce this on 64-bit processors. It appears that
+     * 32-bit implementations supports PR=1 and EE/DR/IR=0 and MacOS
+     * exploits it.
+     */
+    if ((env->insns_flags & PPC_64B) && ((value >> MSR_PR) & 1)) {
+        value |= (1 << MSR_EE) | (1 << MSR_DR) | (1 << MSR_IR);
+    }
 #endif
     env->msr = value;
     hreg_compute_hflags(env);
@@ -111,4 +153,17 @@
     return excp;
 }
 
-#endif /* !defined(__HELPER_REGS_H__) */
+#if !defined(CONFIG_USER_ONLY)
+static inline void check_tlb_flush(CPUPPCState *env)
+{
+    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    if (env->tlb_need_flush) {
+        env->tlb_need_flush = 0;
+        tlb_flush(cs, 1);
+    }
+}
+#else
+static inline void check_tlb_flush(CPUPPCState *env) { }
+#endif
+
+#endif /* HELPER_REGS_H */
diff --git a/target-ppc/int_helper.c b/target-ppc/int_helper.c
index 4c2b71c..7445376 100644
--- a/target-ppc/int_helper.c
+++ b/target-ppc/int_helper.c
@@ -16,10 +16,12 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
-#include "qemu/aes.h"
+#include "crypto/aes.h"
 
 #include "helper_regs.h"
 /*****************************************************************************/
@@ -2327,24 +2329,28 @@
 
 void helper_vcipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 {
+    ppc_avr_t result;
     int i;
 
     VECTOR_FOR_INORDER_I(i, u32) {
-        r->AVRW(i) = b->AVRW(i) ^
+        result.AVRW(i) = b->AVRW(i) ^
             (AES_Te0[a->AVRB(AES_shifts[4*i + 0])] ^
              AES_Te1[a->AVRB(AES_shifts[4*i + 1])] ^
              AES_Te2[a->AVRB(AES_shifts[4*i + 2])] ^
              AES_Te3[a->AVRB(AES_shifts[4*i + 3])]);
     }
+    *r = result;
 }
 
 void helper_vcipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 {
+    ppc_avr_t result;
     int i;
 
     VECTOR_FOR_INORDER_I(i, u8) {
-        r->AVRB(i) = b->AVRB(i) ^ (AES_sbox[a->AVRB(AES_shifts[i])]);
+        result.AVRB(i) = b->AVRB(i) ^ (AES_sbox[a->AVRB(AES_shifts[i])]);
     }
+    *r = result;
 }
 
 void helper_vncipher(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
@@ -2369,11 +2375,13 @@
 
 void helper_vncipherlast(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b)
 {
+    ppc_avr_t result;
     int i;
 
     VECTOR_FOR_INORDER_I(i, u8) {
-        r->AVRB(i) = b->AVRB(i) ^ (AES_isbox[a->AVRB(AES_ishifts[i])]);
+        result.AVRB(i) = b->AVRB(i) ^ (AES_isbox[a->AVRB(AES_ishifts[i])]);
     }
+    *r = result;
 }
 
 #define ROTRu32(v, n) (((v) >> (n)) | ((v) << (32-n)))
@@ -2460,16 +2468,19 @@
 
 void helper_vpermxor(ppc_avr_t *r,  ppc_avr_t *a, ppc_avr_t *b, ppc_avr_t *c)
 {
+    ppc_avr_t result;
     int i;
+
     VECTOR_FOR_INORDER_I(i, u8) {
         int indexA = c->u8[i] >> 4;
         int indexB = c->u8[i] & 0xF;
 #if defined(HOST_WORDS_BIGENDIAN)
-        r->u8[i] = a->u8[indexA] ^ b->u8[indexB];
+        result.u8[i] = a->u8[indexA] ^ b->u8[indexB];
 #else
-        r->u8[i] = a->u8[15-indexA] ^ b->u8[15-indexB];
+        result.u8[i] = a->u8[15-indexA] ^ b->u8[15-indexB];
 #endif
     }
+    *r = result;
 }
 
 #undef VECTOR_FOR_INORDER_I
diff --git a/target-ppc/kvm-stub.c b/target-ppc/kvm-stub.c
index ee3f5d2..efeafca 100644
--- a/target-ppc/kvm-stub.c
+++ b/target-ppc/kvm-stub.c
@@ -9,7 +9,9 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "hw/ppc/openpic.h"
 
 int kvm_openpic_connect_vcpu(DeviceState *d, CPUState *cs)
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index 6843fa0..dcb68b9 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -14,20 +14,21 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include <dirent.h>
-#include <sys/types.h>
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 #include <sys/vfs.h>
 
 #include <linux/kvm.h>
 
 #include "qemu-common.h"
+#include "qemu/error-report.h"
+#include "cpu.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
+#include "sysemu/numa.h"
 #include "kvm_ppc.h"
-#include "cpu.h"
 #include "sysemu/cpus.h"
 #include "sysemu/device_tree.h"
 #include "mmu-hash64.h"
@@ -39,6 +40,12 @@
 #include "sysemu/watchdog.h"
 #include "trace.h"
 #include "exec/gdbstub.h"
+#include "exec/memattrs.h"
+#include "sysemu/hostmem.h"
+#include "qemu/cutils.h"
+#if defined(TARGET_PPC64)
+#include "hw/ppc/spapr_cpu_core.h"
+#endif
 
 //#define DEBUG_KVM
 
@@ -95,7 +102,7 @@
 
 static int kvm_ppc_register_host_cpu_type(void);
 
-int kvm_arch_init(KVMState *s)
+int kvm_arch_init(MachineState *ms, KVMState *s)
 {
     cap_interrupt_unset = kvm_check_extension(s, KVM_CAP_PPC_UNSET_IRQ);
     cap_interrupt_level = kvm_check_extension(s, KVM_CAP_PPC_IRQ_LEVEL);
@@ -257,7 +264,8 @@
             info->flags |= KVM_PPC_1T_SEGMENTS;
         }
 
-        if (env->mmu_model == POWERPC_MMU_2_06) {
+        if (env->mmu_model == POWERPC_MMU_2_06 ||
+            env->mmu_model == POWERPC_MMU_2_07) {
             info->slb_size = 32;
         } else {
             info->slb_size = 64;
@@ -270,8 +278,9 @@
         info->sps[i].enc[0].pte_enc = 0;
         i++;
 
-        /* 64K on MMU 2.06 */
-        if (env->mmu_model == POWERPC_MMU_2_06) {
+        /* 64K on MMU 2.06 and later */
+        if (env->mmu_model == POWERPC_MMU_2_06 ||
+            env->mmu_model == POWERPC_MMU_2_07) {
             info->sps[i].page_shift = 16;
             info->sps[i].slb_enc = 0x110;
             info->sps[i].enc[0].page_shift = 16;
@@ -302,16 +311,11 @@
     kvm_get_fallback_smmu_info(cpu, info);
 }
 
-static long getrampagesize(void)
+static long gethugepagesize(const char *mem_path)
 {
     struct statfs fs;
     int ret;
 
-    if (!mem_path) {
-        /* guest RAM is backed by normal anonymous pages */
-        return getpagesize();
-    }
-
     do {
         ret = statfs(mem_path, &fs);
     } while (ret != 0 && errno == EINTR);
@@ -333,6 +337,80 @@
     return fs.f_bsize;
 }
 
+/*
+ * FIXME TOCTTOU: this iterates over memory backends' mem-path, which
+ * may or may not name the same files / on the same filesystem now as
+ * when we actually open and map them.  Iterate over the file
+ * descriptors instead, and use qemu_fd_getpagesize().
+ */
+static int find_max_supported_pagesize(Object *obj, void *opaque)
+{
+    char *mem_path;
+    long *hpsize_min = opaque;
+
+    if (object_dynamic_cast(obj, TYPE_MEMORY_BACKEND)) {
+        mem_path = object_property_get_str(obj, "mem-path", NULL);
+        if (mem_path) {
+            long hpsize = gethugepagesize(mem_path);
+            if (hpsize < *hpsize_min) {
+                *hpsize_min = hpsize;
+            }
+        } else {
+            *hpsize_min = getpagesize();
+        }
+    }
+
+    return 0;
+}
+
+static long getrampagesize(void)
+{
+    long hpsize = LONG_MAX;
+    long mainrampagesize;
+    Object *memdev_root;
+
+    if (mem_path) {
+        mainrampagesize = gethugepagesize(mem_path);
+    } else {
+        mainrampagesize = getpagesize();
+    }
+
+    /* it's possible we have memory-backend objects with
+     * hugepage-backed RAM. these may get mapped into system
+     * address space via -numa parameters or memory hotplug
+     * hooks. we want to take these into account, but we
+     * also want to make sure these supported hugepage
+     * sizes are applicable across the entire range of memory
+     * we may boot from, so we take the min across all
+     * backends, and assume normal pages in cases where a
+     * backend isn't backed by hugepages.
+     */
+    memdev_root = object_resolve_path("/objects", NULL);
+    if (memdev_root) {
+        object_child_foreach(memdev_root, find_max_supported_pagesize, &hpsize);
+    }
+    if (hpsize == LONG_MAX) {
+        /* No additional memory regions found ==> Report main RAM page size */
+        return mainrampagesize;
+    }
+
+    /* If NUMA is disabled or the NUMA nodes are not backed with a
+     * memory-backend, then there is at least one node using "normal" RAM,
+     * so if its page size is smaller we have got to report that size instead.
+     */
+    if (hpsize > mainrampagesize &&
+        (nb_numa_nodes == 0 || numa_info[0].node_memdev == NULL)) {
+        static bool warned;
+        if (!warned) {
+            error_report("Huge page support disabled (n/a for main memory).");
+            warned = true;
+        }
+        return mainrampagesize;
+    }
+
+    return hpsize;
+}
+
 static bool kvm_valid_page_size(uint32_t flags, long rampgsize, uint32_t shift)
 {
     if (!(flags & KVM_PPC_PAGE_SIZES_REAL)) {
@@ -366,6 +444,13 @@
     /* Convert to QEMU form */
     memset(&env->sps, 0, sizeof(env->sps));
 
+    /* If we have HV KVM, we need to forbid CI large pages if our
+     * host page size is smaller than 64K.
+     */
+    if (smmu_info.flags & KVM_PPC_PAGE_SIZES_REAL) {
+        env->ci_large_pages = getpagesize() >= 0x10000;
+    }
+
     /*
      * XXX This loop should be an entry wide AND of the capabilities that
      *     the selected CPU has with the capabilities that KVM supports.
@@ -457,6 +542,10 @@
     /* Synchronize sregs with kvm */
     ret = kvm_arch_sync_sregs(cpu);
     if (ret) {
+        if (ret == -EINVAL) {
+            error_report("Register sync failed... If you're using kvm-hv.ko,"
+                         " only \"-cpu host\" is possible");
+        }
         return ret;
     }
 
@@ -595,8 +684,13 @@
         for (i = 0; i < 32; i++) {
             uint64_t vsr[2];
 
+#ifdef HOST_WORDS_BIGENDIAN
             vsr[0] = float64_val(env->fpr[i]);
             vsr[1] = env->vsr[i];
+#else
+            vsr[0] = env->vsr[i];
+            vsr[1] = float64_val(env->fpr[i]);
+#endif
             reg.addr = (uintptr_t) &vsr;
             reg.id = vsx ? KVM_REG_PPC_VSR(i) : KVM_REG_PPC_FPR(i);
 
@@ -666,10 +760,17 @@
                         vsx ? "VSR" : "FPR", i, strerror(errno));
                 return ret;
             } else {
+#ifdef HOST_WORDS_BIGENDIAN
                 env->fpr[i] = vsr[0];
                 if (vsx) {
                     env->vsr[i] = vsr[1];
                 }
+#else
+                env->fpr[i] = vsr[1];
+                if (vsx) {
+                    env->vsr[i] = vsr[0];
+                }
+#endif
             }
         }
     }
@@ -795,6 +896,44 @@
 }
 #endif /* TARGET_PPC64 */
 
+int kvmppc_put_books_sregs(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    struct kvm_sregs sregs;
+    int i;
+
+    sregs.pvr = env->spr[SPR_PVR];
+
+    sregs.u.s.sdr1 = env->spr[SPR_SDR1];
+
+    /* Sync SLB */
+#ifdef TARGET_PPC64
+    for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
+        sregs.u.s.ppc64.slb[i].slbe = env->slb[i].esid;
+        if (env->slb[i].esid & SLB_ESID_V) {
+            sregs.u.s.ppc64.slb[i].slbe |= i;
+        }
+        sregs.u.s.ppc64.slb[i].slbv = env->slb[i].vsid;
+    }
+#endif
+
+    /* Sync SRs */
+    for (i = 0; i < 16; i++) {
+        sregs.u.s.ppc32.sr[i] = env->sr[i];
+    }
+
+    /* Sync BATs */
+    for (i = 0; i < 8; i++) {
+        /* Beware. We have to swap upper and lower bits here */
+        sregs.u.s.ppc32.dbat[i] = ((uint64_t)env->DBAT[0][i] << 32)
+            | env->DBAT[1][i];
+        sregs.u.s.ppc32.ibat[i] = ((uint64_t)env->IBAT[0][i] << 32)
+            | env->IBAT[1][i];
+    }
+
+    return kvm_vcpu_ioctl(CPU(cpu), KVM_SET_SREGS, &sregs);
+}
+
 int kvm_arch_put_registers(CPUState *cs, int level)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
@@ -848,39 +987,8 @@
     }
 
     if (cap_segstate && (level >= KVM_PUT_RESET_STATE)) {
-        struct kvm_sregs sregs;
-
-        sregs.pvr = env->spr[SPR_PVR];
-
-        sregs.u.s.sdr1 = env->spr[SPR_SDR1];
-
-        /* Sync SLB */
-#ifdef TARGET_PPC64
-        for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
-            sregs.u.s.ppc64.slb[i].slbe = env->slb[i].esid;
-            if (env->slb[i].esid & SLB_ESID_V) {
-                sregs.u.s.ppc64.slb[i].slbe |= i;
-            }
-            sregs.u.s.ppc64.slb[i].slbv = env->slb[i].vsid;
-        }
-#endif
-
-        /* Sync SRs */
-        for (i = 0; i < 16; i++) {
-            sregs.u.s.ppc32.sr[i] = env->sr[i];
-        }
-
-        /* Sync BATs */
-        for (i = 0; i < 8; i++) {
-            /* Beware. We have to swap upper and lower bits here */
-            sregs.u.s.ppc32.dbat[i] = ((uint64_t)env->DBAT[0][i] << 32)
-                | env->DBAT[1][i];
-            sregs.u.s.ppc32.ibat[i] = ((uint64_t)env->IBAT[0][i] << 32)
-                | env->IBAT[1][i];
-        }
-
-        ret = kvm_vcpu_ioctl(cs, KVM_SET_SREGS, &sregs);
-        if (ret) {
+        ret = kvmppc_put_books_sregs(cpu);
+        if (ret < 0) {
             return ret;
         }
     }
@@ -942,12 +1050,197 @@
      env->excp_vectors[vector] = env->spr[ivor] + env->spr[SPR_BOOKE_IVPR];
 }
 
+static int kvmppc_get_booke_sregs(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    struct kvm_sregs sregs;
+    int ret;
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_BASE) {
+        env->spr[SPR_BOOKE_CSRR0] = sregs.u.e.csrr0;
+        env->spr[SPR_BOOKE_CSRR1] = sregs.u.e.csrr1;
+        env->spr[SPR_BOOKE_ESR] = sregs.u.e.esr;
+        env->spr[SPR_BOOKE_DEAR] = sregs.u.e.dear;
+        env->spr[SPR_BOOKE_MCSR] = sregs.u.e.mcsr;
+        env->spr[SPR_BOOKE_TSR] = sregs.u.e.tsr;
+        env->spr[SPR_BOOKE_TCR] = sregs.u.e.tcr;
+        env->spr[SPR_DECR] = sregs.u.e.dec;
+        env->spr[SPR_TBL] = sregs.u.e.tb & 0xffffffff;
+        env->spr[SPR_TBU] = sregs.u.e.tb >> 32;
+        env->spr[SPR_VRSAVE] = sregs.u.e.vrsave;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_ARCH206) {
+        env->spr[SPR_BOOKE_PIR] = sregs.u.e.pir;
+        env->spr[SPR_BOOKE_MCSRR0] = sregs.u.e.mcsrr0;
+        env->spr[SPR_BOOKE_MCSRR1] = sregs.u.e.mcsrr1;
+        env->spr[SPR_BOOKE_DECAR] = sregs.u.e.decar;
+        env->spr[SPR_BOOKE_IVPR] = sregs.u.e.ivpr;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_64) {
+        env->spr[SPR_BOOKE_EPCR] = sregs.u.e.epcr;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_SPRG8) {
+        env->spr[SPR_BOOKE_SPRG8] = sregs.u.e.sprg8;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_IVOR) {
+        env->spr[SPR_BOOKE_IVOR0] = sregs.u.e.ivor_low[0];
+        kvm_sync_excp(env, POWERPC_EXCP_CRITICAL,  SPR_BOOKE_IVOR0);
+        env->spr[SPR_BOOKE_IVOR1] = sregs.u.e.ivor_low[1];
+        kvm_sync_excp(env, POWERPC_EXCP_MCHECK,  SPR_BOOKE_IVOR1);
+        env->spr[SPR_BOOKE_IVOR2] = sregs.u.e.ivor_low[2];
+        kvm_sync_excp(env, POWERPC_EXCP_DSI,  SPR_BOOKE_IVOR2);
+        env->spr[SPR_BOOKE_IVOR3] = sregs.u.e.ivor_low[3];
+        kvm_sync_excp(env, POWERPC_EXCP_ISI,  SPR_BOOKE_IVOR3);
+        env->spr[SPR_BOOKE_IVOR4] = sregs.u.e.ivor_low[4];
+        kvm_sync_excp(env, POWERPC_EXCP_EXTERNAL,  SPR_BOOKE_IVOR4);
+        env->spr[SPR_BOOKE_IVOR5] = sregs.u.e.ivor_low[5];
+        kvm_sync_excp(env, POWERPC_EXCP_ALIGN,  SPR_BOOKE_IVOR5);
+        env->spr[SPR_BOOKE_IVOR6] = sregs.u.e.ivor_low[6];
+        kvm_sync_excp(env, POWERPC_EXCP_PROGRAM,  SPR_BOOKE_IVOR6);
+        env->spr[SPR_BOOKE_IVOR7] = sregs.u.e.ivor_low[7];
+        kvm_sync_excp(env, POWERPC_EXCP_FPU,  SPR_BOOKE_IVOR7);
+        env->spr[SPR_BOOKE_IVOR8] = sregs.u.e.ivor_low[8];
+        kvm_sync_excp(env, POWERPC_EXCP_SYSCALL,  SPR_BOOKE_IVOR8);
+        env->spr[SPR_BOOKE_IVOR9] = sregs.u.e.ivor_low[9];
+        kvm_sync_excp(env, POWERPC_EXCP_APU,  SPR_BOOKE_IVOR9);
+        env->spr[SPR_BOOKE_IVOR10] = sregs.u.e.ivor_low[10];
+        kvm_sync_excp(env, POWERPC_EXCP_DECR,  SPR_BOOKE_IVOR10);
+        env->spr[SPR_BOOKE_IVOR11] = sregs.u.e.ivor_low[11];
+        kvm_sync_excp(env, POWERPC_EXCP_FIT,  SPR_BOOKE_IVOR11);
+        env->spr[SPR_BOOKE_IVOR12] = sregs.u.e.ivor_low[12];
+        kvm_sync_excp(env, POWERPC_EXCP_WDT,  SPR_BOOKE_IVOR12);
+        env->spr[SPR_BOOKE_IVOR13] = sregs.u.e.ivor_low[13];
+        kvm_sync_excp(env, POWERPC_EXCP_DTLB,  SPR_BOOKE_IVOR13);
+        env->spr[SPR_BOOKE_IVOR14] = sregs.u.e.ivor_low[14];
+        kvm_sync_excp(env, POWERPC_EXCP_ITLB,  SPR_BOOKE_IVOR14);
+        env->spr[SPR_BOOKE_IVOR15] = sregs.u.e.ivor_low[15];
+        kvm_sync_excp(env, POWERPC_EXCP_DEBUG,  SPR_BOOKE_IVOR15);
+
+        if (sregs.u.e.features & KVM_SREGS_E_SPE) {
+            env->spr[SPR_BOOKE_IVOR32] = sregs.u.e.ivor_high[0];
+            kvm_sync_excp(env, POWERPC_EXCP_SPEU,  SPR_BOOKE_IVOR32);
+            env->spr[SPR_BOOKE_IVOR33] = sregs.u.e.ivor_high[1];
+            kvm_sync_excp(env, POWERPC_EXCP_EFPDI,  SPR_BOOKE_IVOR33);
+            env->spr[SPR_BOOKE_IVOR34] = sregs.u.e.ivor_high[2];
+            kvm_sync_excp(env, POWERPC_EXCP_EFPRI,  SPR_BOOKE_IVOR34);
+        }
+
+        if (sregs.u.e.features & KVM_SREGS_E_PM) {
+            env->spr[SPR_BOOKE_IVOR35] = sregs.u.e.ivor_high[3];
+            kvm_sync_excp(env, POWERPC_EXCP_EPERFM,  SPR_BOOKE_IVOR35);
+        }
+
+        if (sregs.u.e.features & KVM_SREGS_E_PC) {
+            env->spr[SPR_BOOKE_IVOR36] = sregs.u.e.ivor_high[4];
+            kvm_sync_excp(env, POWERPC_EXCP_DOORI,  SPR_BOOKE_IVOR36);
+            env->spr[SPR_BOOKE_IVOR37] = sregs.u.e.ivor_high[5];
+            kvm_sync_excp(env, POWERPC_EXCP_DOORCI, SPR_BOOKE_IVOR37);
+        }
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_ARCH206_MMU) {
+        env->spr[SPR_BOOKE_MAS0] = sregs.u.e.mas0;
+        env->spr[SPR_BOOKE_MAS1] = sregs.u.e.mas1;
+        env->spr[SPR_BOOKE_MAS2] = sregs.u.e.mas2;
+        env->spr[SPR_BOOKE_MAS3] = sregs.u.e.mas7_3 & 0xffffffff;
+        env->spr[SPR_BOOKE_MAS4] = sregs.u.e.mas4;
+        env->spr[SPR_BOOKE_MAS6] = sregs.u.e.mas6;
+        env->spr[SPR_BOOKE_MAS7] = sregs.u.e.mas7_3 >> 32;
+        env->spr[SPR_MMUCFG] = sregs.u.e.mmucfg;
+        env->spr[SPR_BOOKE_TLB0CFG] = sregs.u.e.tlbcfg[0];
+        env->spr[SPR_BOOKE_TLB1CFG] = sregs.u.e.tlbcfg[1];
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_EXP) {
+        env->spr[SPR_BOOKE_EPR] = sregs.u.e.epr;
+    }
+
+    if (sregs.u.e.features & KVM_SREGS_E_PD) {
+        env->spr[SPR_BOOKE_EPLC] = sregs.u.e.eplc;
+        env->spr[SPR_BOOKE_EPSC] = sregs.u.e.epsc;
+    }
+
+    if (sregs.u.e.impl_id == KVM_SREGS_E_IMPL_FSL) {
+        env->spr[SPR_E500_SVR] = sregs.u.e.impl.fsl.svr;
+        env->spr[SPR_Exxx_MCAR] = sregs.u.e.impl.fsl.mcar;
+        env->spr[SPR_HID0] = sregs.u.e.impl.fsl.hid0;
+
+        if (sregs.u.e.impl.fsl.features & KVM_SREGS_E_FSL_PIDn) {
+            env->spr[SPR_BOOKE_PID1] = sregs.u.e.impl.fsl.pid1;
+            env->spr[SPR_BOOKE_PID2] = sregs.u.e.impl.fsl.pid2;
+        }
+    }
+
+    return 0;
+}
+
+static int kvmppc_get_books_sregs(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    struct kvm_sregs sregs;
+    int ret;
+    int i;
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_GET_SREGS, &sregs);
+    if (ret < 0) {
+        return ret;
+    }
+
+    if (!env->external_htab) {
+        ppc_store_sdr1(env, sregs.u.s.sdr1);
+    }
+
+    /* Sync SLB */
+#ifdef TARGET_PPC64
+    /*
+     * The packed SLB array we get from KVM_GET_SREGS only contains
+     * information about valid entries. So we flush our internal copy
+     * to get rid of stale ones, then put all valid SLB entries back
+     * in.
+     */
+    memset(env->slb, 0, sizeof(env->slb));
+    for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
+        target_ulong rb = sregs.u.s.ppc64.slb[i].slbe;
+        target_ulong rs = sregs.u.s.ppc64.slb[i].slbv;
+        /*
+         * Only restore valid entries
+         */
+        if (rb & SLB_ESID_V) {
+            ppc_store_slb(cpu, rb & 0xfff, rb & ~0xfffULL, rs);
+        }
+    }
+#endif
+
+    /* Sync SRs */
+    for (i = 0; i < 16; i++) {
+        env->sr[i] = sregs.u.s.ppc32.sr[i];
+    }
+
+    /* Sync BATs */
+    for (i = 0; i < 8; i++) {
+        env->DBAT[0][i] = sregs.u.s.ppc32.dbat[i] & 0xffffffff;
+        env->DBAT[1][i] = sregs.u.s.ppc32.dbat[i] >> 32;
+        env->IBAT[0][i] = sregs.u.s.ppc32.ibat[i] & 0xffffffff;
+        env->IBAT[1][i] = sregs.u.s.ppc32.ibat[i] >> 32;
+    }
+
+    return 0;
+}
+
 int kvm_arch_get_registers(CPUState *cs)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
     CPUPPCState *env = &cpu->env;
     struct kvm_regs regs;
-    struct kvm_sregs sregs;
     uint32_t cr;
     int i, ret;
 
@@ -987,174 +1280,17 @@
     kvm_get_fp(cs);
 
     if (cap_booke_sregs) {
-        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
+        ret = kvmppc_get_booke_sregs(cpu);
         if (ret < 0) {
             return ret;
         }
-
-        if (sregs.u.e.features & KVM_SREGS_E_BASE) {
-            env->spr[SPR_BOOKE_CSRR0] = sregs.u.e.csrr0;
-            env->spr[SPR_BOOKE_CSRR1] = sregs.u.e.csrr1;
-            env->spr[SPR_BOOKE_ESR] = sregs.u.e.esr;
-            env->spr[SPR_BOOKE_DEAR] = sregs.u.e.dear;
-            env->spr[SPR_BOOKE_MCSR] = sregs.u.e.mcsr;
-            env->spr[SPR_BOOKE_TSR] = sregs.u.e.tsr;
-            env->spr[SPR_BOOKE_TCR] = sregs.u.e.tcr;
-            env->spr[SPR_DECR] = sregs.u.e.dec;
-            env->spr[SPR_TBL] = sregs.u.e.tb & 0xffffffff;
-            env->spr[SPR_TBU] = sregs.u.e.tb >> 32;
-            env->spr[SPR_VRSAVE] = sregs.u.e.vrsave;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_ARCH206) {
-            env->spr[SPR_BOOKE_PIR] = sregs.u.e.pir;
-            env->spr[SPR_BOOKE_MCSRR0] = sregs.u.e.mcsrr0;
-            env->spr[SPR_BOOKE_MCSRR1] = sregs.u.e.mcsrr1;
-            env->spr[SPR_BOOKE_DECAR] = sregs.u.e.decar;
-            env->spr[SPR_BOOKE_IVPR] = sregs.u.e.ivpr;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_64) {
-            env->spr[SPR_BOOKE_EPCR] = sregs.u.e.epcr;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_SPRG8) {
-            env->spr[SPR_BOOKE_SPRG8] = sregs.u.e.sprg8;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_IVOR) {
-            env->spr[SPR_BOOKE_IVOR0] = sregs.u.e.ivor_low[0];
-            kvm_sync_excp(env, POWERPC_EXCP_CRITICAL,  SPR_BOOKE_IVOR0);
-            env->spr[SPR_BOOKE_IVOR1] = sregs.u.e.ivor_low[1];
-            kvm_sync_excp(env, POWERPC_EXCP_MCHECK,  SPR_BOOKE_IVOR1);
-            env->spr[SPR_BOOKE_IVOR2] = sregs.u.e.ivor_low[2];
-            kvm_sync_excp(env, POWERPC_EXCP_DSI,  SPR_BOOKE_IVOR2);
-            env->spr[SPR_BOOKE_IVOR3] = sregs.u.e.ivor_low[3];
-            kvm_sync_excp(env, POWERPC_EXCP_ISI,  SPR_BOOKE_IVOR3);
-            env->spr[SPR_BOOKE_IVOR4] = sregs.u.e.ivor_low[4];
-            kvm_sync_excp(env, POWERPC_EXCP_EXTERNAL,  SPR_BOOKE_IVOR4);
-            env->spr[SPR_BOOKE_IVOR5] = sregs.u.e.ivor_low[5];
-            kvm_sync_excp(env, POWERPC_EXCP_ALIGN,  SPR_BOOKE_IVOR5);
-            env->spr[SPR_BOOKE_IVOR6] = sregs.u.e.ivor_low[6];
-            kvm_sync_excp(env, POWERPC_EXCP_PROGRAM,  SPR_BOOKE_IVOR6);
-            env->spr[SPR_BOOKE_IVOR7] = sregs.u.e.ivor_low[7];
-            kvm_sync_excp(env, POWERPC_EXCP_FPU,  SPR_BOOKE_IVOR7);
-            env->spr[SPR_BOOKE_IVOR8] = sregs.u.e.ivor_low[8];
-            kvm_sync_excp(env, POWERPC_EXCP_SYSCALL,  SPR_BOOKE_IVOR8);
-            env->spr[SPR_BOOKE_IVOR9] = sregs.u.e.ivor_low[9];
-            kvm_sync_excp(env, POWERPC_EXCP_APU,  SPR_BOOKE_IVOR9);
-            env->spr[SPR_BOOKE_IVOR10] = sregs.u.e.ivor_low[10];
-            kvm_sync_excp(env, POWERPC_EXCP_DECR,  SPR_BOOKE_IVOR10);
-            env->spr[SPR_BOOKE_IVOR11] = sregs.u.e.ivor_low[11];
-            kvm_sync_excp(env, POWERPC_EXCP_FIT,  SPR_BOOKE_IVOR11);
-            env->spr[SPR_BOOKE_IVOR12] = sregs.u.e.ivor_low[12];
-            kvm_sync_excp(env, POWERPC_EXCP_WDT,  SPR_BOOKE_IVOR12);
-            env->spr[SPR_BOOKE_IVOR13] = sregs.u.e.ivor_low[13];
-            kvm_sync_excp(env, POWERPC_EXCP_DTLB,  SPR_BOOKE_IVOR13);
-            env->spr[SPR_BOOKE_IVOR14] = sregs.u.e.ivor_low[14];
-            kvm_sync_excp(env, POWERPC_EXCP_ITLB,  SPR_BOOKE_IVOR14);
-            env->spr[SPR_BOOKE_IVOR15] = sregs.u.e.ivor_low[15];
-            kvm_sync_excp(env, POWERPC_EXCP_DEBUG,  SPR_BOOKE_IVOR15);
-
-            if (sregs.u.e.features & KVM_SREGS_E_SPE) {
-                env->spr[SPR_BOOKE_IVOR32] = sregs.u.e.ivor_high[0];
-                kvm_sync_excp(env, POWERPC_EXCP_SPEU,  SPR_BOOKE_IVOR32);
-                env->spr[SPR_BOOKE_IVOR33] = sregs.u.e.ivor_high[1];
-                kvm_sync_excp(env, POWERPC_EXCP_EFPDI,  SPR_BOOKE_IVOR33);
-                env->spr[SPR_BOOKE_IVOR34] = sregs.u.e.ivor_high[2];
-                kvm_sync_excp(env, POWERPC_EXCP_EFPRI,  SPR_BOOKE_IVOR34);
-            }
-
-            if (sregs.u.e.features & KVM_SREGS_E_PM) {
-                env->spr[SPR_BOOKE_IVOR35] = sregs.u.e.ivor_high[3];
-                kvm_sync_excp(env, POWERPC_EXCP_EPERFM,  SPR_BOOKE_IVOR35);
-            }
-
-            if (sregs.u.e.features & KVM_SREGS_E_PC) {
-                env->spr[SPR_BOOKE_IVOR36] = sregs.u.e.ivor_high[4];
-                kvm_sync_excp(env, POWERPC_EXCP_DOORI,  SPR_BOOKE_IVOR36);
-                env->spr[SPR_BOOKE_IVOR37] = sregs.u.e.ivor_high[5];
-                kvm_sync_excp(env, POWERPC_EXCP_DOORCI, SPR_BOOKE_IVOR37);
-            }
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_ARCH206_MMU) {
-            env->spr[SPR_BOOKE_MAS0] = sregs.u.e.mas0;
-            env->spr[SPR_BOOKE_MAS1] = sregs.u.e.mas1;
-            env->spr[SPR_BOOKE_MAS2] = sregs.u.e.mas2;
-            env->spr[SPR_BOOKE_MAS3] = sregs.u.e.mas7_3 & 0xffffffff;
-            env->spr[SPR_BOOKE_MAS4] = sregs.u.e.mas4;
-            env->spr[SPR_BOOKE_MAS6] = sregs.u.e.mas6;
-            env->spr[SPR_BOOKE_MAS7] = sregs.u.e.mas7_3 >> 32;
-            env->spr[SPR_MMUCFG] = sregs.u.e.mmucfg;
-            env->spr[SPR_BOOKE_TLB0CFG] = sregs.u.e.tlbcfg[0];
-            env->spr[SPR_BOOKE_TLB1CFG] = sregs.u.e.tlbcfg[1];
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_EXP) {
-            env->spr[SPR_BOOKE_EPR] = sregs.u.e.epr;
-        }
-
-        if (sregs.u.e.features & KVM_SREGS_E_PD) {
-            env->spr[SPR_BOOKE_EPLC] = sregs.u.e.eplc;
-            env->spr[SPR_BOOKE_EPSC] = sregs.u.e.epsc;
-        }
-
-        if (sregs.u.e.impl_id == KVM_SREGS_E_IMPL_FSL) {
-            env->spr[SPR_E500_SVR] = sregs.u.e.impl.fsl.svr;
-            env->spr[SPR_Exxx_MCAR] = sregs.u.e.impl.fsl.mcar;
-            env->spr[SPR_HID0] = sregs.u.e.impl.fsl.hid0;
-
-            if (sregs.u.e.impl.fsl.features & KVM_SREGS_E_FSL_PIDn) {
-                env->spr[SPR_BOOKE_PID1] = sregs.u.e.impl.fsl.pid1;
-                env->spr[SPR_BOOKE_PID2] = sregs.u.e.impl.fsl.pid2;
-            }
-        }
     }
 
     if (cap_segstate) {
-        ret = kvm_vcpu_ioctl(cs, KVM_GET_SREGS, &sregs);
+        ret = kvmppc_get_books_sregs(cpu);
         if (ret < 0) {
             return ret;
         }
-
-        if (!env->external_htab) {
-            ppc_store_sdr1(env, sregs.u.s.sdr1);
-        }
-
-        /* Sync SLB */
-#ifdef TARGET_PPC64
-        /*
-         * The packed SLB array we get from KVM_GET_SREGS only contains
-         * information about valid entries. So we flush our internal
-         * copy to get rid of stale ones, then put all valid SLB entries
-         * back in.
-         */
-        memset(env->slb, 0, sizeof(env->slb));
-        for (i = 0; i < ARRAY_SIZE(env->slb); i++) {
-            target_ulong rb = sregs.u.s.ppc64.slb[i].slbe;
-            target_ulong rs = sregs.u.s.ppc64.slb[i].slbv;
-            /*
-             * Only restore valid entries
-             */
-            if (rb & SLB_ESID_V) {
-                ppc_store_slb(env, rb, rs);
-            }
-        }
-#endif
-
-        /* Sync SRs */
-        for (i = 0; i < 16; i++) {
-            env->sr[i] = sregs.u.s.ppc32.sr[i];
-        }
-
-        /* Sync BATs */
-        for (i = 0; i < 8; i++) {
-            env->DBAT[0][i] = sregs.u.s.ppc32.dbat[i] & 0xffffffff;
-            env->DBAT[1][i] = sregs.u.s.ppc32.dbat[i] >> 32;
-            env->IBAT[0][i] = sregs.u.s.ppc32.ibat[i] & 0xffffffff;
-            env->IBAT[1][i] = sregs.u.s.ppc32.ibat[i] >> 32;
-        }
     }
 
     if (cap_hior) {
@@ -1241,6 +1377,8 @@
     int r;
     unsigned irq;
 
+    qemu_mutex_lock_iothread();
+
     /* PowerPC QEMU tracks the various core input pins (interrupt, critical
      * interrupt, reset, etc) in PPC-specific env->irq_input_state. */
     if (!cap_interrupt_level &&
@@ -1262,16 +1400,19 @@
 
         /* Always wake up soon in case the interrupt was level based */
         timer_mod(idle_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-                       (get_ticks_per_sec() / 50));
+                       (NANOSECONDS_PER_SECOND / 50));
     }
 
     /* We don't know if there are more interrupts pending after this. However,
      * the guest will return to userspace in the course of handling this one
      * anyways, so we will get a chance to deliver the rest. */
+
+    qemu_mutex_unlock_iothread();
 }
 
-void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
+MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
+    return MEMTXATTRS_UNSPECIFIED;
 }
 
 int kvm_arch_process_async_events(CPUState *cs)
@@ -1568,6 +1709,8 @@
     CPUPPCState *env = &cpu->env;
     int ret;
 
+    qemu_mutex_lock_iothread();
+
     switch (run->exit_reason) {
     case KVM_EXIT_DCR:
         if (run->dcr.is_write) {
@@ -1618,6 +1761,7 @@
         break;
     }
 
+    qemu_mutex_unlock_iothread();
     return ret;
 }
 
@@ -1716,7 +1860,7 @@
 {
     char line[512];
     char *ns;
-    uint32_t retval = get_ticks_per_sec();
+    uint32_t retval = NANOSECONDS_PER_SECOND;
 
     if (read_cpuinfo("timebase", line, sizeof(line))) {
         return retval;
@@ -1728,8 +1872,7 @@
 
     ns++;
 
-    retval = atoi(ns);
-    return retval;
+    return atoi(ns);
 }
 
 bool kvmppc_get_host_serial(char **value)
@@ -1776,13 +1919,8 @@
     return 0;
 }
 
-/* Read a CPU node property from the host device tree that's a single
- * integer (32-bit or 64-bit).  Returns 0 if anything goes wrong
- * (can't find or open the property, or doesn't understand the
- * format) */
-static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
+static uint64_t kvmppc_read_int_dt(const char *filename)
 {
-    char buf[PATH_MAX], *tmp;
     union {
         uint32_t v32;
         uint64_t v64;
@@ -1790,14 +1928,7 @@
     FILE *f;
     int len;
 
-    if (kvmppc_find_cpu_dt(buf, sizeof(buf))) {
-        return -1;
-    }
-
-    tmp = g_strdup_printf("%s/%s", buf, propname);
-
-    f = fopen(tmp, "rb");
-    g_free(tmp);
+    f = fopen(filename, "rb");
     if (!f) {
         return -1;
     }
@@ -1815,6 +1946,26 @@
     return 0;
 }
 
+/* Read a CPU node property from the host device tree that's a single
+ * integer (32-bit or 64-bit).  Returns 0 if anything goes wrong
+ * (can't find or open the property, or doesn't understand the
+ * format) */
+static uint64_t kvmppc_read_int_cpu_dt(const char *propname)
+{
+    char buf[PATH_MAX], *tmp;
+    uint64_t val;
+
+    if (kvmppc_find_cpu_dt(buf, sizeof(buf))) {
+        return -1;
+    }
+
+    tmp = g_strdup_printf("%s/%s", buf, propname);
+    val = kvmppc_read_int_dt(tmp);
+    g_free(tmp);
+
+    return val;
+}
+
 uint64_t kvmppc_get_clockfreq(void)
 {
     return kvmppc_read_int_cpu_dt("clock-frequency");
@@ -1879,7 +2030,29 @@
     hc[2] = cpu_to_be32(0x48000008);
     hc[3] = cpu_to_be32(bswap32(0x3860ffff));
 
-    return 0;
+    return 1;
+}
+
+static inline int kvmppc_enable_hcall(KVMState *s, target_ulong hcall)
+{
+    return kvm_vm_enable_cap(s, KVM_CAP_PPC_ENABLE_HCALL, 0, hcall, 1);
+}
+
+void kvmppc_enable_logical_ci_hcalls(void)
+{
+    /*
+     * FIXME: it would be nice if we could detect the cases where
+     * we're using a device which requires the in kernel
+     * implementation of these hcalls, but the kernel lacks them and
+     * produce a warning.
+     */
+    kvmppc_enable_hcall(kvm_state, H_LOGICAL_CI_LOAD);
+    kvmppc_enable_hcall(kvm_state, H_LOGICAL_CI_STORE);
+}
+
+void kvmppc_enable_set_mode_hcall(void)
+{
+    kvmppc_enable_hcall(kvm_state, H_SET_MODE);
 }
 
 void kvmppc_set_papr(PowerPCCPU *cpu)
@@ -1889,7 +2062,8 @@
 
     ret = kvm_vcpu_enable_cap(cs, KVM_CAP_PPC_PAPR, 0);
     if (ret) {
-        cpu_abort(cs, "This KVM version does not support PAPR\n");
+        error_report("This vCPU type or KVM version does not support PAPR");
+        exit(1);
     }
 
     /* Update the capability flag so we sync the right information
@@ -1909,7 +2083,8 @@
 
     ret = kvm_vcpu_enable_cap(cs, KVM_CAP_PPC_EPR, 0, mpic_proxy);
     if (ret && mpic_proxy) {
-        cpu_abort(cs, "This KVM version does not support EPR\n");
+        error_report("This KVM version does not support EPR");
+        exit(1);
     }
 }
 
@@ -1995,7 +2170,7 @@
 }
 
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
-                              bool vfio_accel)
+                              bool need_vfio)
 {
     struct kvm_create_spapr_tce args = {
         .liobn = liobn,
@@ -2009,7 +2184,7 @@
      * destroying the table, which the upper layers -will- do
      */
     *pfd = -1;
-    if (!cap_spapr_tce || (vfio_accel && !cap_spapr_vfio)) {
+    if (!cap_spapr_tce || (need_vfio && !cap_spapr_vfio)) {
         return NULL;
     }
 
@@ -2117,6 +2292,7 @@
 
 static void kvmppc_host_cpu_class_init(ObjectClass *oc, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(oc);
     PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
     uint32_t vmx = kvmppc_get_vmx();
     uint32_t dfp = kvmppc_get_dfp();
@@ -2143,6 +2319,9 @@
     if (icache_size != -1) {
         pcc->l1_icache_size = icache_size;
     }
+
+    /* Reason: kvmppc_host_cpu_initfn() dies when !kvm_enabled() */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 bool kvmppc_has_cap_epr(void)
@@ -2172,6 +2351,32 @@
     return POWERPC_CPU_CLASS(oc);
 }
 
+PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void)
+{
+    uint32_t host_pvr = mfpvr();
+    PowerPCCPUClass *pvr_pcc;
+
+    pvr_pcc = ppc_cpu_class_by_pvr(host_pvr);
+    if (pvr_pcc == NULL) {
+        pvr_pcc = ppc_cpu_class_by_pvr_mask(host_pvr);
+    }
+
+    return pvr_pcc;
+}
+
+#if defined(TARGET_PPC64)
+static void spapr_cpu_core_host_initfn(Object *obj)
+{
+    sPAPRCPUCore *core = SPAPR_CPU_CORE(obj);
+    char *name = g_strdup_printf("%s-" TYPE_POWERPC_CPU, "host");
+    ObjectClass *oc = object_class_by_name(name);
+
+    g_assert(oc);
+    g_free((void *)name);
+    core->cpu_class = oc;
+}
+#endif
+
 static int kvm_ppc_register_host_cpu_type(void)
 {
     TypeInfo type_info = {
@@ -2179,14 +2384,10 @@
         .instance_init = kvmppc_host_cpu_initfn,
         .class_init = kvmppc_host_cpu_class_init,
     };
-    uint32_t host_pvr = mfpvr();
     PowerPCCPUClass *pvr_pcc;
     DeviceClass *dc;
 
-    pvr_pcc = ppc_cpu_class_by_pvr(host_pvr);
-    if (pvr_pcc == NULL) {
-        pvr_pcc = ppc_cpu_class_by_pvr_mask(host_pvr);
-    }
+    pvr_pcc = kvm_ppc_get_host_cpu_class();
     if (pvr_pcc == NULL) {
         return -1;
     }
@@ -2200,6 +2401,21 @@
     type_info.name = g_strdup_printf("%s-"TYPE_POWERPC_CPU, dc->desc);
     type_register(&type_info);
 
+#if defined(TARGET_PPC64)
+    type_info.name = g_strdup_printf("%s-"TYPE_SPAPR_CPU_CORE, "host");
+    type_info.parent = TYPE_SPAPR_CPU_CORE,
+    type_info.instance_size = sizeof(sPAPRCPUCore),
+    type_info.instance_init = spapr_cpu_core_host_initfn,
+    type_info.class_init = NULL;
+    type_register(&type_info);
+    g_free((void *)type_info.name);
+
+    /* Register generic spapr CPU family class for current host CPU type */
+    type_info.name = g_strdup_printf("%s-"TYPE_SPAPR_CPU_CORE, dc->desc);
+    type_register(&type_info);
+    g_free((void *)type_info.name);
+#endif
+
     return 0;
 }
 
@@ -2246,8 +2462,23 @@
                     strerror(errno));
             return rc;
         } else if (rc) {
-            /* Kernel already retuns data in BE format for the file */
-            qemu_put_buffer(f, buf, rc);
+            uint8_t *buffer = buf;
+            ssize_t n = rc;
+            while (n) {
+                struct kvm_get_htab_header *head =
+                    (struct kvm_get_htab_header *) buffer;
+                size_t chunksize = sizeof(*head) +
+                     HASH_PTE_SIZE_64 * head->n_valid;
+
+                qemu_put_be32(f, head->index);
+                qemu_put_be16(f, head->n_valid);
+                qemu_put_be16(f, head->n_invalid);
+                qemu_put_buffer(f, (void *)(head + 1),
+                                HASH_PTE_SIZE_64 * head->n_valid);
+
+                buffer += chunksize;
+                n -= chunksize;
+            }
         }
     } while ((rc != 0)
              && ((max_ns < 0)
@@ -2264,7 +2495,6 @@
     ssize_t rc;
 
     buf = alloca(chunksize);
-    /* This is KVM on ppc, so this is all big-endian */
     buf->index = index;
     buf->n_valid = n_valid;
     buf->n_invalid = n_invalid;
@@ -2388,3 +2618,34 @@
 error_out:
     return;
 }
+
+int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
+                             uint64_t address, uint32_t data, PCIDevice *dev)
+{
+    return 0;
+}
+
+int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
+                                int vector, PCIDevice *dev)
+{
+    return 0;
+}
+
+int kvm_arch_release_virq_post(int virq)
+{
+    return 0;
+}
+
+int kvm_arch_msi_data_to_gsi(uint32_t data)
+{
+    return data & 0xffff;
+}
+
+int kvmppc_enable_hwrng(void)
+{
+    if (!kvm_enabled() || !kvm_check_extension(kvm_state, KVM_CAP_PPC_HWRNG)) {
+        return -1;
+    }
+
+    return kvmppc_enable_hcall(kvm_state, H_RANDOM);
+}
diff --git a/target-ppc/kvm_ppc.c b/target-ppc/kvm_ppc.c
deleted file mode 100644
index f769acd..0000000
--- a/target-ppc/kvm_ppc.c
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * PowerPC KVM support
- *
- * Copyright IBM Corp. 2008
- *
- * Authors:
- *  Hollis Blanchard <hollisb@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-
-#include "qemu-common.h"
-#include "qemu/timer.h"
-#include "kvm_ppc.h"
-#include "sysemu/device_tree.h"
-#include "qemu/main-loop.h"
-
-#define PROC_DEVTREE_PATH "/proc/device-tree"
-
-static QEMUTimer *kvmppc_timer;
-static unsigned int kvmppc_timer_rate;
-
-static void kvmppc_timer_hack(void *opaque)
-{
-    qemu_notify_event();
-    timer_mod(kvmppc_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + kvmppc_timer_rate);
-}
-
-void kvmppc_init(void)
-{
-    /* XXX The only reason KVM yields control back to qemu is device IO. Since
-     * an idle guest does no IO, qemu's device model will never get a chance to
-     * run. So, until QEMU gains IO threads, we create this timer to ensure
-     * that the device model gets a chance to run. */
-    kvmppc_timer_rate = get_ticks_per_sec() / 10;
-    kvmppc_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, &kvmppc_timer_hack, NULL);
-    timer_mod(kvmppc_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + kvmppc_timer_rate);
-}
-
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index 2e0224c..5461d10 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -6,13 +6,11 @@
  *
  */
 
-#ifndef __KVM_PPC_H__
-#define __KVM_PPC_H__
+#ifndef KVM_PPC_H
+#define KVM_PPC_H
 
 #define TYPE_HOST_POWERPC_CPU "host-" TYPE_POWERPC_CPU
 
-void kvmppc_init(void);
-
 #ifdef CONFIG_KVM
 
 uint32_t kvmppc_get_tbfreq(void);
@@ -24,6 +22,8 @@
 int kvmppc_get_hasidle(CPUPPCState *env);
 int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len);
 int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level);
+void kvmppc_enable_logical_ci_hcalls(void);
+void kvmppc_enable_set_mode_hcall(void);
 void kvmppc_set_papr(PowerPCCPU *cpu);
 int kvmppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version);
 void kvmppc_set_mpic_proxy(PowerPCCPU *cpu, int mpic_proxy);
@@ -36,7 +36,7 @@
 off_t kvmppc_alloc_rma(void **rma);
 bool kvmppc_spapr_use_multitce(void);
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd,
-                              bool vfio_accel);
+                              bool need_vfio);
 int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
 int kvmppc_reset_htab(int shift_hint);
 uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
@@ -54,6 +54,9 @@
 void kvmppc_hash64_write_pte(CPUPPCState *env, target_ulong pte_index,
                              target_ulong pte0, target_ulong pte1);
 bool kvmppc_has_cap_fixup_hcalls(void);
+int kvmppc_enable_hwrng(void);
+int kvmppc_put_books_sregs(PowerPCCPU *cpu);
+PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void);
 
 #else
 
@@ -97,14 +100,17 @@
     return -1;
 }
 
-static inline int kvmppc_read_segment_page_sizes(uint32_t *prop, int maxcells)
+static inline int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
 {
     return -1;
 }
 
-static inline int kvmppc_set_interrupt(PowerPCCPU *cpu, int irq, int level)
+static inline void kvmppc_enable_logical_ci_hcalls(void)
 {
-    return -1;
+}
+
+static inline void kvmppc_enable_set_mode_hcall(void)
+{
 }
 
 static inline void kvmppc_set_papr(PowerPCCPU *cpu)
@@ -158,7 +164,7 @@
 
 static inline void *kvmppc_create_spapr_tce(uint32_t liobn,
                                             uint32_t window_size, int *fd,
-                                            bool vfio_accel)
+                                            bool need_vfio)
 {
     return NULL;
 }
@@ -171,7 +177,7 @@
 
 static inline int kvmppc_reset_htab(int shift_hint)
 {
-    return -1;
+    return 0;
 }
 
 static inline uint64_t kvmppc_rma_size(uint64_t current_size,
@@ -180,11 +186,6 @@
     return ram_size;
 }
 
-static inline int kvmppc_update_sdr1(CPUPPCState *env)
-{
-    return 0;
-}
-
 #endif /* !CONFIG_USER_ONLY */
 
 static inline bool kvmppc_has_cap_epr(void)
@@ -243,18 +244,65 @@
     abort();
 }
 
+static inline int kvmppc_enable_hwrng(void)
+{
+    return -1;
+}
+
+static inline int kvmppc_put_books_sregs(PowerPCCPU *cpu)
+{
+    abort();
+}
+
+static inline PowerPCCPUClass *kvm_ppc_get_host_cpu_class(void)
+{
+    return NULL;
+}
+
 #endif
 
 #ifndef CONFIG_KVM
+
 #define kvmppc_eieio() do { } while (0)
-#else
+
+static inline void kvmppc_dcbst_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+}
+
+static inline void kvmppc_icbi_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+}
+
+#else   /* CONFIG_KVM */
+
 #define kvmppc_eieio() \
     do {                                          \
         if (kvm_enabled()) {                          \
             asm volatile("eieio" : : : "memory"); \
         } \
     } while (0)
-#endif
+
+/* Store data cache blocks back to memory */
+static inline void kvmppc_dcbst_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+    uint8_t *p;
+
+    for (p = addr; p < addr + len; p += cpu->env.dcache_line_size) {
+        asm volatile("dcbst 0,%0" : : "r"(p) : "memory");
+    }
+}
+
+/* Invalidate instruction cache blocks */
+static inline void kvmppc_icbi_range(PowerPCCPU *cpu, uint8_t *addr, int len)
+{
+    uint8_t *p;
+
+    for (p = addr; p < addr + len; p += cpu->env.icache_line_size) {
+        asm volatile("icbi 0,%0" : : "r"(p));
+    }
+}
+
+#endif  /* CONFIG_KVM */
 
 #ifndef KVM_INTERRUPT_SET
 #define KVM_INTERRUPT_SET -1
@@ -268,4 +316,4 @@
 #define KVM_INTERRUPT_SET_LEVEL -3
 #endif
 
-#endif /* __KVM_PPC_H__ */
+#endif /* KVM_PPC_H */
diff --git a/target-ppc/machine.c b/target-ppc/machine.c
index c801b82..4820f22 100644
--- a/target-ppc/machine.c
+++ b/target-ppc/machine.c
@@ -1,7 +1,14 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "sysemu/kvm.h"
 #include "helper_regs.h"
+#include "mmu-hash64.h"
+#include "migration/cpu.h"
+#include "exec/exec-all.h"
 
 static int cpu_load_old(QEMUFile *f, void *opaque, int version_id)
 {
@@ -90,9 +97,12 @@
     qemu_get_betls(f, &env->nip);
     qemu_get_betls(f, &env->hflags);
     qemu_get_betls(f, &env->hflags_nmsr);
-    qemu_get_sbe32s(f, &env->mmu_idx);
+    qemu_get_sbe32(f); /* Discard unused mmu_idx */
     qemu_get_sbe32(f); /* Discard unused power_mode */
 
+    /* Recompute mmu indices */
+    hreg_compute_mem_idx(env);
+
     return 0;
 }
 
@@ -134,7 +144,7 @@
 
     env->spr[SPR_LR] = env->lr;
     env->spr[SPR_CTR] = env->ctr;
-    env->spr[SPR_XER] = env->xer;
+    env->spr[SPR_XER] = cpu_read_xer(env);
 #if defined(TARGET_PPC64)
     env->spr[SPR_CFAR] = env->cfar;
 #endif
@@ -159,6 +169,7 @@
     PowerPCCPU *cpu = opaque;
     CPUPPCState *env = &cpu->env;
     int i;
+    target_ulong msr;
 
     /*
      * We always ignore the source PVR. The user or management
@@ -167,7 +178,7 @@
     env->spr[SPR_PVR] = env->spr_cb[SPR_PVR].default_value;
     env->lr = env->spr[SPR_LR];
     env->ctr = env->spr[SPR_CTR];
-    env->xer = env->spr[SPR_XER];
+    cpu_write_xer(env, env->spr[SPR_XER]);
 #if defined(TARGET_PPC64)
     env->cfar = env->spr[SPR_CFAR];
 #endif
@@ -190,7 +201,12 @@
         /* Restore htab_base and htab_mask variables */
         ppc_store_sdr1(env, env->spr[SPR_SDR1]);
     }
-    hreg_compute_hflags(env);
+
+    /* Invalidate all msr bits except MSR_TGPR/MSR_HVB before restoring */
+    msr = env->msr;
+    env->msr ^= ~((1ULL << MSR_TGPR) | MSR_HVB);
+    ppc_store_msr(env, msr);
+
     hreg_compute_mem_idx(env);
 
     return 0;
@@ -207,6 +223,7 @@
     .name = "cpu/fpu",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = fpu_needed,
     .fields = (VMStateField[]) {
         VMSTATE_FLOAT64_ARRAY(env.fpr, PowerPCCPU, 32),
         VMSTATE_UINTTL(env.fpscr, PowerPCCPU),
@@ -225,6 +242,7 @@
     .name = "cpu/altivec",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = altivec_needed,
     .fields = (VMStateField[]) {
         VMSTATE_AVR_ARRAY(env.avr, PowerPCCPU, 32),
         VMSTATE_UINT32(env.vscr, PowerPCCPU),
@@ -243,6 +261,7 @@
     .name = "cpu/vsx",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = vsx_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINT64_ARRAY(env.vsr, PowerPCCPU, 32),
         VMSTATE_END_OF_LIST()
@@ -263,6 +282,7 @@
     .version_id = 1,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
+    .needed = tm_needed,
     .fields      = (VMStateField []) {
         VMSTATE_UINTTL_ARRAY(env.tm_gpr, PowerPCCPU, 32),
         VMSTATE_AVR_ARRAY(env.tm_vsr, PowerPCCPU, 64),
@@ -296,6 +316,7 @@
     .name = "cpu/sr",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = sr_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINTTL_ARRAY(env.sr, PowerPCCPU, 32),
         VMSTATE_END_OF_LIST()
@@ -341,10 +362,30 @@
     return (cpu->env.mmu_model & POWERPC_MMU_64);
 }
 
+static int slb_post_load(void *opaque, int version_id)
+{
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
+    int i;
+
+    /* We've pulled in the raw esid and vsid values from the migration
+     * stream, but we need to recompute the page size pointers */
+    for (i = 0; i < env->slb_nr; i++) {
+        if (ppc_store_slb(cpu, i, env->slb[i].esid, env->slb[i].vsid) < 0) {
+            /* Migration source had bad values in its SLB */
+            return -1;
+        }
+    }
+
+    return 0;
+}
+
 static const VMStateDescription vmstate_slb = {
     .name = "cpu/slb",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = slb_needed,
+    .post_load = slb_post_load,
     .fields = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(env.slb_nr, PowerPCCPU),
         VMSTATE_SLB_ARRAY(env.slb, PowerPCCPU, MAX_SLB_ENTRIES),
@@ -377,6 +418,7 @@
     .name = "cpu/tlb6xx",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = tlb6xx_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(env.nb_tlb, PowerPCCPU),
         VMSTATE_STRUCT_VARRAY_POINTER_INT32(env.tlb.tlb6, PowerPCCPU,
@@ -423,6 +465,7 @@
     .name = "cpu/pbr403",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = pbr403_needed,
     .fields = (VMStateField[]) {
         VMSTATE_UINTTL_ARRAY(env.pb, PowerPCCPU, 4),
         VMSTATE_END_OF_LIST()
@@ -433,6 +476,7 @@
     .name = "cpu/tlb6xx",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = tlbemb_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(env.nb_tlb, PowerPCCPU),
         VMSTATE_STRUCT_VARRAY_POINTER_INT32(env.tlb.tlbe, PowerPCCPU,
@@ -442,13 +486,9 @@
         /* 403 protection registers */
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_pbr403,
-            .needed = pbr403_needed,
-        } , {
-            /* empty */
-        }
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_pbr403,
+        NULL
     }
 };
 
@@ -477,6 +517,7 @@
     .name = "cpu/tlbmas",
     .version_id = 1,
     .minimum_version_id = 1,
+    .needed = tlbmas_needed,
     .fields = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(env.nb_tlb, PowerPCCPU),
         VMSTATE_STRUCT_VARRAY_POINTER_INT32(env.tlb.tlbm, PowerPCCPU,
@@ -527,38 +568,18 @@
         VMSTATE_UINT32_EQUAL(env.nb_BATs, PowerPCCPU),
         VMSTATE_END_OF_LIST()
     },
-    .subsections = (VMStateSubsection []) {
-        {
-            .vmsd = &vmstate_fpu,
-            .needed = fpu_needed,
-        } , {
-            .vmsd = &vmstate_altivec,
-            .needed = altivec_needed,
-        } , {
-            .vmsd = &vmstate_vsx,
-            .needed = vsx_needed,
-        } , {
-            .vmsd = &vmstate_sr,
-            .needed = sr_needed,
-        } , {
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fpu,
+        &vmstate_altivec,
+        &vmstate_vsx,
+        &vmstate_sr,
 #ifdef TARGET_PPC64
-            .vmsd = &vmstate_tm,
-            .needed = tm_needed,
-        } , {
-            .vmsd = &vmstate_slb,
-            .needed = slb_needed,
-        } , {
+        &vmstate_tm,
+        &vmstate_slb,
 #endif /* TARGET_PPC64 */
-            .vmsd = &vmstate_tlb6xx,
-            .needed = tlb6xx_needed,
-        } , {
-            .vmsd = &vmstate_tlbemb,
-            .needed = tlbemb_needed,
-        } , {
-            .vmsd = &vmstate_tlbmas,
-            .needed = tlbmas_needed,
-        } , {
-            /* empty */
-        }
+        &vmstate_tlb6xx,
+        &vmstate_tlbemb,
+        &vmstate_tlbmas,
+        NULL
     }
 };
diff --git a/target-ppc/mem_helper.c b/target-ppc/mem_helper.c
index 50344b8..e4ed377 100644
--- a/target-ppc/mem_helper.c
+++ b/target-ppc/mem_helper.c
@@ -16,11 +16,14 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 
 #include "helper_regs.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 //#define DEBUG_OP
@@ -100,8 +103,10 @@
                  uint32_t ra, uint32_t rb)
 {
     if (likely(xer_bc != 0)) {
-        if (unlikely((ra != 0 && reg < ra && (reg + xer_bc) > ra) ||
-                     (reg < rb && (reg + xer_bc) > rb))) {
+        int num_used_regs = (xer_bc + 3) / 4;
+        if (unlikely((ra != 0 && lsw_reg_in_range(reg, num_used_regs, ra)) ||
+                     lsw_reg_in_range(reg, num_used_regs, rb))) {
+            env->nip += 4;     /* Compensate the "nip - 4" from gen_lswx() */
             helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                        POWERPC_EXCP_INVAL |
                                        POWERPC_EXCP_INVAL_LSWX);
@@ -227,16 +232,16 @@
                                                                 \
         if (needs_byteswap(env)) {                              \
             r->element[LO_IDX ? index : (adjust - index)] =     \
-                swap(access(env, addr));                        \
+                swap(access(env, addr, GETPC()));               \
         } else {                                                \
             r->element[LO_IDX ? index : (adjust - index)] =     \
-                access(env, addr);                              \
+                access(env, addr, GETPC());                     \
         }                                                       \
     }
 #define I(x) (x)
-LVE(lvebx, cpu_ldub_data, I, u8)
-LVE(lvehx, cpu_lduw_data, bswap16, u16)
-LVE(lvewx, cpu_ldl_data, bswap32, u32)
+LVE(lvebx, cpu_ldub_data_ra, I, u8)
+LVE(lvehx, cpu_lduw_data_ra, bswap16, u16)
+LVE(lvewx, cpu_ldl_data_ra, bswap32, u32)
 #undef I
 #undef LVE
 
@@ -254,18 +259,41 @@
                                                                         \
         if (needs_byteswap(env)) {                                      \
             access(env, addr, swap(r->element[LO_IDX ? index :          \
-                                              (adjust - index)]));      \
+                                              (adjust - index)]),       \
+                        GETPC());                                       \
         } else {                                                        \
             access(env, addr, r->element[LO_IDX ? index :               \
-                                         (adjust - index)]);            \
+                                         (adjust - index)], GETPC());   \
         }                                                               \
     }
 #define I(x) (x)
-STVE(stvebx, cpu_stb_data, I, u8)
-STVE(stvehx, cpu_stw_data, bswap16, u16)
-STVE(stvewx, cpu_stl_data, bswap32, u32)
+STVE(stvebx, cpu_stb_data_ra, I, u8)
+STVE(stvehx, cpu_stw_data_ra, bswap16, u16)
+STVE(stvewx, cpu_stl_data_ra, bswap32, u32)
 #undef I
 #undef LVE
 
 #undef HI_IDX
 #undef LO_IDX
+
+void helper_tbegin(CPUPPCState *env)
+{
+    /* As a degenerate implementation, always fail tbegin.  The reason
+     * given is "Nesting overflow".  The "persistent" bit is set,
+     * providing a hint to the error handler to not retry.  The TFIAR
+     * captures the address of the failure, which is this tbegin
+     * instruction.  Instruction execution will continue with the
+     * next instruction in memory, which is precisely what we want.
+     */
+
+    env->spr[SPR_TEXASR] =
+        (1ULL << TEXASR_FAILURE_PERSISTENT) |
+        (1ULL << TEXASR_NESTING_OVERFLOW) |
+        (msr_hv << TEXASR_PRIVILEGE_HV) |
+        (msr_pr << TEXASR_PRIVILEGE_PR) |
+        (1ULL << TEXASR_FAILURE_SUMMARY) |
+        (1ULL << TEXASR_TFIAR_EXACT);
+    env->spr[SPR_TFIAR] = env->nip | (msr_hv << 1) | msr_pr;
+    env->spr[SPR_TFHAR] = env->nip + 4;
+    env->crf[0] = 0xB; /* 0b1010 = transaction failure */
+}
diff --git a/target-ppc/mfrom_table_gen.c b/target-ppc/mfrom_table_gen.c
index a140ded..6317918 100644
--- a/target-ppc/mfrom_table_gen.c
+++ b/target-ppc/mfrom_table_gen.c
@@ -1,6 +1,5 @@
 #define _GNU_SOURCE
-#include <stdint.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <math.h>
 
 int main (void)
diff --git a/target-ppc/misc_helper.c b/target-ppc/misc_helper.c
index a577b3a..cb5ebf5 100644
--- a/target-ppc/misc_helper.c
+++ b/target-ppc/misc_helper.c
@@ -16,7 +16,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 
 #include "helper_regs.h"
@@ -77,8 +79,13 @@
 
 void helper_store_sdr1(CPUPPCState *env, target_ulong val)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     if (!env->external_htab) {
-        ppc_store_sdr1(env, val);
+        if (env->spr[SPR_SDR1] != val) {
+            ppc_store_sdr1(env, val);
+            tlb_flush(CPU(cpu), 1);
+        }
     }
 }
 
@@ -159,3 +166,44 @@
 {
     hreg_store_msr(env, value, 0);
 }
+
+/* This code is lifted from MacOnLinux. It is called whenever
+ * THRM1,2 or 3 is read an fixes up the values in such a way
+ * that will make MacOS not hang. These registers exist on some
+ * 75x and 74xx processors.
+ */
+void helper_fixup_thrm(CPUPPCState *env)
+{
+    target_ulong v, t;
+    int i;
+
+#define THRM1_TIN       (1 << 31)
+#define THRM1_TIV       (1 << 30)
+#define THRM1_THRES(x)  (((x) & 0x7f) << 23)
+#define THRM1_TID       (1 << 2)
+#define THRM1_TIE       (1 << 1)
+#define THRM1_V         (1 << 0)
+#define THRM3_E         (1 << 0)
+
+    if (!(env->spr[SPR_THRM3] & THRM3_E)) {
+        return;
+    }
+
+    /* Note: Thermal interrupts are unimplemented */
+    for (i = SPR_THRM1; i <= SPR_THRM2; i++) {
+        v = env->spr[i];
+        if (!(v & THRM1_V)) {
+            continue;
+        }
+        v |= THRM1_TIV;
+        v &= ~THRM1_TIN;
+        t = v & THRM1_THRES(127);
+        if ((v & THRM1_TID) && t < THRM1_THRES(24)) {
+            v |= THRM1_TIN;
+        }
+        if (!(v & THRM1_TID) && t > THRM1_THRES(24)) {
+            v |= THRM1_TIN;
+        }
+        env->spr[i] = v;
+    }
+}
diff --git a/target-ppc/mmu-hash32.c b/target-ppc/mmu-hash32.c
index 0a13a81..29bace6 100644
--- a/target-ppc/mmu-hash32.c
+++ b/target-ppc/mmu-hash32.c
@@ -18,25 +18,19 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash32.h"
+#include "exec/log.h"
 
-//#define DEBUG_MMU
 //#define DEBUG_BAT
 
-#ifdef DEBUG_MMU
-#  define LOG_MMU(...) qemu_log(__VA_ARGS__)
-#  define LOG_MMU_STATE(cpu) log_cpu_state((cpu), 0)
-#else
-#  define LOG_MMU(...) do { } while (0)
-#  define LOG_MMU_STATE(cpu) do { } while (0)
-#endif
-
 #ifdef DEBUG_BATS
-#  define LOG_BATS(...) qemu_log(__VA_ARGS__)
+#  define LOG_BATS(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_BATS(...) do { } while (0)
 #endif
@@ -92,9 +86,10 @@
     return prot;
 }
 
-static int ppc_hash32_pte_prot(CPUPPCState *env,
+static int ppc_hash32_pte_prot(PowerPCCPU *cpu,
                                target_ulong sr, ppc_hash_pte32_t pte)
 {
+    CPUPPCState *env = &cpu->env;
     unsigned pp, key;
 
     key = !!(msr_pr ? (sr & SR32_KP) : (sr & SR32_KS));
@@ -103,9 +98,11 @@
     return ppc_hash32_pp_prot(key, pp, !!(sr & SR32_NX));
 }
 
-static target_ulong hash32_bat_size(CPUPPCState *env,
+static target_ulong hash32_bat_size(PowerPCCPU *cpu,
                                     target_ulong batu, target_ulong batl)
 {
+    CPUPPCState *env = &cpu->env;
+
     if ((msr_pr && !(batu & BATU32_VP))
         || (!msr_pr && !(batu & BATU32_VS))) {
         return 0;
@@ -114,7 +111,7 @@
     return BATU32_BEPI & ~((batu & BATU32_BL) << 15);
 }
 
-static int hash32_bat_prot(CPUPPCState *env,
+static int hash32_bat_prot(PowerPCCPU *cpu,
                            target_ulong batu, target_ulong batl)
 {
     int pp, prot;
@@ -130,7 +127,7 @@
     return prot;
 }
 
-static target_ulong hash32_bat_601_size(CPUPPCState *env,
+static target_ulong hash32_bat_601_size(PowerPCCPU *cpu,
                                 target_ulong batu, target_ulong batl)
 {
     if (!(batl & BATL32_601_V)) {
@@ -140,9 +137,10 @@
     return BATU32_BEPI & ~((batl & BATL32_601_BL) << 17);
 }
 
-static int hash32_bat_601_prot(CPUPPCState *env,
+static int hash32_bat_601_prot(PowerPCCPU *cpu,
                                target_ulong batu, target_ulong batl)
 {
+    CPUPPCState *env = &cpu->env;
     int key, pp;
 
     pp = batu & BATU32_601_PP;
@@ -154,9 +152,10 @@
     return ppc_hash32_pp_prot(key, pp, 0);
 }
 
-static hwaddr ppc_hash32_bat_lookup(CPUPPCState *env, target_ulong ea, int rwx,
+static hwaddr ppc_hash32_bat_lookup(PowerPCCPU *cpu, target_ulong ea, int rwx,
                                     int *prot)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong *BATlt, *BATut;
     int i;
 
@@ -175,9 +174,9 @@
         target_ulong mask;
 
         if (unlikely(env->mmu_model == POWERPC_MMU_601)) {
-            mask = hash32_bat_601_size(env, batu, batl);
+            mask = hash32_bat_601_size(cpu, batu, batl);
         } else {
-            mask = hash32_bat_size(env, batu, batl);
+            mask = hash32_bat_size(cpu, batu, batl);
         }
         LOG_BATS("%s: %cBAT%d v " TARGET_FMT_lx " BATu " TARGET_FMT_lx
                  " BATl " TARGET_FMT_lx "\n", __func__,
@@ -187,9 +186,9 @@
             hwaddr raddr = (batl & mask) | (ea & ~mask);
 
             if (unlikely(env->mmu_model == POWERPC_MMU_601)) {
-                *prot = hash32_bat_601_prot(env, batu, batl);
+                *prot = hash32_bat_601_prot(cpu, batu, batl);
             } else {
-                *prot = hash32_bat_prot(env, batu, batl);
+                *prot = hash32_bat_prot(cpu, batu, batl);
             }
 
             return raddr & TARGET_PAGE_MASK;
@@ -218,14 +217,15 @@
     return -1;
 }
 
-static int ppc_hash32_direct_store(CPUPPCState *env, target_ulong sr,
+static int ppc_hash32_direct_store(PowerPCCPU *cpu, target_ulong sr,
                                    target_ulong eaddr, int rwx,
                                    hwaddr *raddr, int *prot)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUState *cs = CPU(cpu);
+    CPUPPCState *env = &cpu->env;
     int key = !!(msr_pr ? (sr & SR32_KP) : (sr & SR32_KS));
 
-    LOG_MMU("direct store...\n");
+    qemu_log_mask(CPU_LOG_MMU, "direct store...\n");
 
     if ((sr & 0x1FF00000) >> 20 == 0x07f) {
         /* Memory-forced I/O controller interface access */
@@ -283,9 +283,8 @@
         }
         return 1;
     default:
-        qemu_log("ERROR: instruction should not need "
+        cpu_abort(cs, "ERROR: instruction should not need "
                  "address translation\n");
-        abort();
     }
     if ((rwx == 1 || key != 1) && (rwx == 0 || key != 0)) {
         *raddr = eaddr;
@@ -303,12 +302,14 @@
     }
 }
 
-hwaddr get_pteg_offset32(CPUPPCState *env, hwaddr hash)
+hwaddr get_pteg_offset32(PowerPCCPU *cpu, hwaddr hash)
 {
+    CPUPPCState *env = &cpu->env;
+
     return (hash * HASH_PTEG_SIZE_32) & env->htab_mask;
 }
 
-static hwaddr ppc_hash32_pteg_search(CPUPPCState *env, hwaddr pteg_off,
+static hwaddr ppc_hash32_pteg_search(PowerPCCPU *cpu, hwaddr pteg_off,
                                      bool secondary, target_ulong ptem,
                                      ppc_hash_pte32_t *pte)
 {
@@ -317,8 +318,8 @@
     int i;
 
     for (i = 0; i < HPTES_PER_GROUP; i++) {
-        pte0 = ppc_hash32_load_hpte0(env, pte_offset);
-        pte1 = ppc_hash32_load_hpte1(env, pte_offset);
+        pte0 = ppc_hash32_load_hpte0(cpu, pte_offset);
+        pte1 = ppc_hash32_load_hpte1(cpu, pte_offset);
 
         if ((pte0 & HPTE32_V_VALID)
             && (secondary == !!(pte0 & HPTE32_V_SECONDARY))
@@ -334,10 +335,11 @@
     return -1;
 }
 
-static hwaddr ppc_hash32_htab_lookup(CPUPPCState *env,
+static hwaddr ppc_hash32_htab_lookup(PowerPCCPU *cpu,
                                      target_ulong sr, target_ulong eaddr,
                                      ppc_hash_pte32_t *pte)
 {
+    CPUPPCState *env = &cpu->env;
     hwaddr pteg_off, pte_offset;
     hwaddr hash;
     uint32_t vsid, pgidx, ptem;
@@ -348,25 +350,26 @@
     ptem = (vsid << 7) | (pgidx >> 10);
 
     /* Page address translation */
-    LOG_MMU("htab_base " TARGET_FMT_plx " htab_mask " TARGET_FMT_plx
+    qemu_log_mask(CPU_LOG_MMU, "htab_base " TARGET_FMT_plx
+            " htab_mask " TARGET_FMT_plx
             " hash " TARGET_FMT_plx "\n",
             env->htab_base, env->htab_mask, hash);
 
     /* Primary PTEG lookup */
-    LOG_MMU("0 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
+    qemu_log_mask(CPU_LOG_MMU, "0 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
             " vsid=%" PRIx32 " ptem=%" PRIx32
             " hash=" TARGET_FMT_plx "\n",
             env->htab_base, env->htab_mask, vsid, ptem, hash);
-    pteg_off = get_pteg_offset32(env, hash);
-    pte_offset = ppc_hash32_pteg_search(env, pteg_off, 0, ptem, pte);
+    pteg_off = get_pteg_offset32(cpu, hash);
+    pte_offset = ppc_hash32_pteg_search(cpu, pteg_off, 0, ptem, pte);
     if (pte_offset == -1) {
         /* Secondary PTEG lookup */
-        LOG_MMU("1 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
+        qemu_log_mask(CPU_LOG_MMU, "1 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
                 " vsid=%" PRIx32 " api=%" PRIx32
                 " hash=" TARGET_FMT_plx "\n", env->htab_base,
                 env->htab_mask, vsid, ptem, ~hash);
-        pteg_off = get_pteg_offset32(env, ~hash);
-        pte_offset = ppc_hash32_pteg_search(env, pteg_off, 1, ptem, pte);
+        pteg_off = get_pteg_offset32(cpu, ~hash);
+        pte_offset = ppc_hash32_pteg_search(cpu, pteg_off, 1, ptem, pte);
     }
 
     return pte_offset;
@@ -381,7 +384,7 @@
     return (rpn & ~mask) | (eaddr & mask);
 }
 
-int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr, int rwx,
+int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, vaddr eaddr, int rwx,
                                 int mmu_idx)
 {
     CPUState *cs = CPU(cpu);
@@ -408,7 +411,7 @@
 
     /* 2. Check Block Address Translation entries (BATs) */
     if (env->nb_BATs != 0) {
-        raddr = ppc_hash32_bat_lookup(env, eaddr, rwx, &prot);
+        raddr = ppc_hash32_bat_lookup(cpu, eaddr, rwx, &prot);
         if (raddr != -1) {
             if (need_prot[rwx] & ~prot) {
                 if (rwx == 2) {
@@ -439,7 +442,7 @@
 
     /* 4. Handle direct store segments */
     if (sr & SR32_T) {
-        if (ppc_hash32_direct_store(env, sr, eaddr, rwx,
+        if (ppc_hash32_direct_store(cpu, sr, eaddr, rwx,
                                     &raddr, &prot) == 0) {
             tlb_set_page(cs, eaddr & TARGET_PAGE_MASK,
                          raddr & TARGET_PAGE_MASK, prot, mmu_idx,
@@ -458,7 +461,7 @@
     }
 
     /* 6. Locate the PTE in the hash table */
-    pte_offset = ppc_hash32_htab_lookup(env, sr, eaddr, &pte);
+    pte_offset = ppc_hash32_htab_lookup(cpu, sr, eaddr, &pte);
     if (pte_offset == -1) {
         if (rwx == 2) {
             cs->exception_index = POWERPC_EXCP_ISI;
@@ -476,15 +479,16 @@
 
         return 1;
     }
-    LOG_MMU("found PTE at offset %08" HWADDR_PRIx "\n", pte_offset);
+    qemu_log_mask(CPU_LOG_MMU,
+                "found PTE at offset %08" HWADDR_PRIx "\n", pte_offset);
 
     /* 7. Check access permissions */
 
-    prot = ppc_hash32_pte_prot(env, sr, pte);
+    prot = ppc_hash32_pte_prot(cpu, sr, pte);
 
     if (need_prot[rwx] & ~prot) {
         /* Access right violation */
-        LOG_MMU("PTE access rejected\n");
+        qemu_log_mask(CPU_LOG_MMU, "PTE access rejected\n");
         if (rwx == 2) {
             cs->exception_index = POWERPC_EXCP_ISI;
             env->error_code = 0x08000000;
@@ -501,7 +505,7 @@
         return 1;
     }
 
-    LOG_MMU("PTE access granted !\n");
+    qemu_log_mask(CPU_LOG_MMU, "PTE access granted !\n");
 
     /* 8. Update PTE referenced and changed bits if necessary */
 
@@ -515,7 +519,7 @@
     }
 
     if (new_pte1 != pte.pte1) {
-        ppc_hash32_store_hpte1(env, pte_offset, new_pte1);
+        ppc_hash32_store_hpte1(cpu, pte_offset, new_pte1);
     }
 
     /* 9. Determine the real address from the PTE */
@@ -528,8 +532,9 @@
     return 0;
 }
 
-hwaddr ppc_hash32_get_phys_page_debug(CPUPPCState *env, target_ulong eaddr)
+hwaddr ppc_hash32_get_phys_page_debug(PowerPCCPU *cpu, target_ulong eaddr)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong sr;
     hwaddr pte_offset;
     ppc_hash_pte32_t pte;
@@ -541,7 +546,7 @@
     }
 
     if (env->nb_BATs != 0) {
-        hwaddr raddr = ppc_hash32_bat_lookup(env, eaddr, 0, &prot);
+        hwaddr raddr = ppc_hash32_bat_lookup(cpu, eaddr, 0, &prot);
         if (raddr != -1) {
             return raddr;
         }
@@ -554,7 +559,7 @@
         return -1;
     }
 
-    pte_offset = ppc_hash32_htab_lookup(env, sr, eaddr, &pte);
+    pte_offset = ppc_hash32_htab_lookup(cpu, sr, eaddr, &pte);
     if (pte_offset == -1) {
         return -1;
     }
diff --git a/target-ppc/mmu-hash32.h b/target-ppc/mmu-hash32.h
index d515d4f..5b9fb08 100644
--- a/target-ppc/mmu-hash32.h
+++ b/target-ppc/mmu-hash32.h
@@ -1,11 +1,11 @@
-#if !defined (__MMU_HASH32_H__)
-#define __MMU_HASH32_H__
+#ifndef MMU_HASH32_H
+#define MMU_HASH32_H
 
 #ifndef CONFIG_USER_ONLY
 
-hwaddr get_pteg_offset32(CPUPPCState *env, hwaddr hash);
-hwaddr ppc_hash32_get_phys_page_debug(CPUPPCState *env, target_ulong addr);
-int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, target_ulong address, int rw,
+hwaddr get_pteg_offset32(PowerPCCPU *cpu, hwaddr hash);
+hwaddr ppc_hash32_get_phys_page_debug(PowerPCCPU *cpu, target_ulong addr);
+int ppc_hash32_handle_mmu_fault(PowerPCCPU *cpu, vaddr address, int rw,
                                 int mmu_idx);
 
 /*
@@ -65,40 +65,42 @@
 #define HPTE32_R_WIMG           0x00000078
 #define HPTE32_R_PP             0x00000003
 
-static inline target_ulong ppc_hash32_load_hpte0(CPUPPCState *env,
+static inline target_ulong ppc_hash32_load_hpte0(PowerPCCPU *cpu,
                                                  hwaddr pte_offset)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    return ldl_phys(cs->as, env->htab_base + pte_offset);
+    return ldl_phys(CPU(cpu)->as, env->htab_base + pte_offset);
 }
 
-static inline target_ulong ppc_hash32_load_hpte1(CPUPPCState *env,
+static inline target_ulong ppc_hash32_load_hpte1(PowerPCCPU *cpu,
                                                  hwaddr pte_offset)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    return ldl_phys(cs->as, env->htab_base + pte_offset + HASH_PTE_SIZE_32/2);
+    return ldl_phys(CPU(cpu)->as,
+                    env->htab_base + pte_offset + HASH_PTE_SIZE_32 / 2);
 }
 
-static inline void ppc_hash32_store_hpte0(CPUPPCState *env,
+static inline void ppc_hash32_store_hpte0(PowerPCCPU *cpu,
                                           hwaddr pte_offset, target_ulong pte0)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    stl_phys(cs->as, env->htab_base + pte_offset, pte0);
+    stl_phys(CPU(cpu)->as, env->htab_base + pte_offset, pte0);
 }
 
-static inline void ppc_hash32_store_hpte1(CPUPPCState *env,
+static inline void ppc_hash32_store_hpte1(PowerPCCPU *cpu,
                                           hwaddr pte_offset, target_ulong pte1)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
     assert(!env->external_htab); /* Not supported on 32-bit for now */
-    stl_phys(cs->as, env->htab_base + pte_offset + HASH_PTE_SIZE_32/2, pte1);
+    stl_phys(CPU(cpu)->as,
+             env->htab_base + pte_offset + HASH_PTE_SIZE_32 / 2, pte1);
 }
 
 typedef struct {
@@ -107,4 +109,4 @@
 
 #endif /* CONFIG_USER_ONLY */
 
-#endif /* __MMU_HASH32_H__ */
+#endif /* MMU_HASH32_H */
diff --git a/target-ppc/mmu-hash64.c b/target-ppc/mmu-hash64.c
index c72198a..5de1358 100644
--- a/target-ppc/mmu-hash64.c
+++ b/target-ppc/mmu-hash64.c
@@ -17,40 +17,38 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
+#include "qemu/error-report.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash64.h"
+#include "exec/log.h"
 
-//#define DEBUG_MMU
 //#define DEBUG_SLB
 
-#ifdef DEBUG_MMU
-#  define LOG_MMU(...) qemu_log(__VA_ARGS__)
-#  define LOG_MMU_STATE(cpu) log_cpu_state((cpu), 0)
-#else
-#  define LOG_MMU(...) do { } while (0)
-#  define LOG_MMU_STATE(cpu) do { } while (0)
-#endif
-
 #ifdef DEBUG_SLB
-#  define LOG_SLB(...) qemu_log(__VA_ARGS__)
+#  define LOG_SLB(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_SLB(...) do { } while (0)
 #endif
 
 /*
- * Used to indicate whether we have allocated htab in the
- * host kernel
+ * Used to indicate that a CPU has its hash page table (HPT) managed
+ * within the host kernel
  */
-bool kvmppc_kern_htab;
+#define MMU_HASH64_KVM_MANAGED_HPT      ((void *)-1)
+
 /*
  * SLB handling
  */
 
-static ppc_slb_t *slb_lookup(CPUPPCState *env, target_ulong eaddr)
+static ppc_slb_t *slb_lookup(PowerPCCPU *cpu, target_ulong eaddr)
 {
+    CPUPPCState *env = &cpu->env;
     uint64_t esid_256M, esid_1T;
     int n;
 
@@ -78,12 +76,13 @@
     return NULL;
 }
 
-void dump_slb(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env)
+void dump_slb(FILE *f, fprintf_function cpu_fprintf, PowerPCCPU *cpu)
 {
+    CPUPPCState *env = &cpu->env;
     int i;
     uint64_t slbe, slbv;
 
-    cpu_synchronize_state(CPU(ppc_env_get_cpu(env)));
+    cpu_synchronize_state(CPU(cpu));
 
     cpu_fprintf(f, "SLB\tESID\t\t\tVSID\n");
     for (i = 0; i < env->slb_nr; i++) {
@@ -99,10 +98,8 @@
 
 void helper_slbia(CPUPPCState *env)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
-    int n, do_invalidate;
+    int n;
 
-    do_invalidate = 0;
     /* XXX: Warning: slbia never invalidates the first segment */
     for (n = 1; n < env->slb_nr; n++) {
         ppc_slb_t *slb = &env->slb[n];
@@ -113,12 +110,9 @@
              *      and we still don't have a tlb_flush_mask(env, n, mask)
              *      in QEMU, we just invalidate all TLBs
              */
-            do_invalidate = 1;
+            env->tlb_need_flush = 1;
         }
     }
-    if (do_invalidate) {
-        tlb_flush(CPU(cpu), 1);
-    }
 }
 
 void helper_slbie(CPUPPCState *env, target_ulong addr)
@@ -126,7 +120,7 @@
     PowerPCCPU *cpu = ppc_env_get_cpu(env);
     ppc_slb_t *slb;
 
-    slb = slb_lookup(env, addr);
+    slb = slb_lookup(cpu, addr);
     if (!slb) {
         return;
     }
@@ -138,39 +132,66 @@
          *      and we still don't have a tlb_flush_mask(env, n, mask)
          *      in QEMU, we just invalidate all TLBs
          */
-        tlb_flush(CPU(cpu), 1);
+        env->tlb_need_flush = 1;
     }
 }
 
-int ppc_store_slb(CPUPPCState *env, target_ulong rb, target_ulong rs)
+int ppc_store_slb(PowerPCCPU *cpu, target_ulong slot,
+                  target_ulong esid, target_ulong vsid)
 {
-    int slot = rb & 0xfff;
+    CPUPPCState *env = &cpu->env;
     ppc_slb_t *slb = &env->slb[slot];
+    const struct ppc_one_seg_page_size *sps = NULL;
+    int i;
 
-    if (rb & (0x1000 - env->slb_nr)) {
-        return -1; /* Reserved bits set or slot too high */
+    if (slot >= env->slb_nr) {
+        return -1; /* Bad slot number */
     }
-    if (rs & (SLB_VSID_B & ~SLB_VSID_B_1T)) {
+    if (esid & ~(SLB_ESID_ESID | SLB_ESID_V)) {
+        return -1; /* Reserved bits set */
+    }
+    if (vsid & (SLB_VSID_B & ~SLB_VSID_B_1T)) {
         return -1; /* Bad segment size */
     }
-    if ((rs & SLB_VSID_B) && !(env->mmu_model & POWERPC_MMU_1TSEG)) {
+    if ((vsid & SLB_VSID_B) && !(env->mmu_model & POWERPC_MMU_1TSEG)) {
         return -1; /* 1T segment on MMU that doesn't support it */
     }
 
-    /* Mask out the slot number as we store the entry */
-    slb->esid = rb & (SLB_ESID_ESID | SLB_ESID_V);
-    slb->vsid = rs;
+    for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_seg_page_size *sps1 = &env->sps.sps[i];
+
+        if (!sps1->page_shift) {
+            break;
+        }
+
+        if ((vsid & SLB_VSID_LLP_MASK) == sps1->slb_enc) {
+            sps = sps1;
+            break;
+        }
+    }
+
+    if (!sps) {
+        error_report("Bad page size encoding in SLB store: slot "TARGET_FMT_lu
+                     " esid 0x"TARGET_FMT_lx" vsid 0x"TARGET_FMT_lx,
+                     slot, esid, vsid);
+        return -1;
+    }
+
+    slb->esid = esid;
+    slb->vsid = vsid;
+    slb->sps = sps;
 
     LOG_SLB("%s: %d " TARGET_FMT_lx " - " TARGET_FMT_lx " => %016" PRIx64
-            " %016" PRIx64 "\n", __func__, slot, rb, rs,
+            " %016" PRIx64 "\n", __func__, slot, esid, vsid,
             slb->esid, slb->vsid);
 
     return 0;
 }
 
-static int ppc_load_slb_esid(CPUPPCState *env, target_ulong rb,
+static int ppc_load_slb_esid(PowerPCCPU *cpu, target_ulong rb,
                              target_ulong *rt)
 {
+    CPUPPCState *env = &cpu->env;
     int slot = rb & 0xfff;
     ppc_slb_t *slb = &env->slb[slot];
 
@@ -182,9 +203,10 @@
     return 0;
 }
 
-static int ppc_load_slb_vsid(CPUPPCState *env, target_ulong rb,
+static int ppc_load_slb_vsid(PowerPCCPU *cpu, target_ulong rb,
                              target_ulong *rt)
 {
+    CPUPPCState *env = &cpu->env;
     int slot = rb & 0xfff;
     ppc_slb_t *slb = &env->slb[slot];
 
@@ -196,9 +218,29 @@
     return 0;
 }
 
+static int ppc_find_slb_vsid(PowerPCCPU *cpu, target_ulong rb,
+                             target_ulong *rt)
+{
+    CPUPPCState *env = &cpu->env;
+    ppc_slb_t *slb;
+
+    if (!msr_is_64bit(env, env->msr)) {
+        rb &= 0xffffffff;
+    }
+    slb = slb_lookup(cpu, rb);
+    if (slb == NULL) {
+        *rt = (target_ulong)-1ul;
+    } else {
+        *rt = slb->vsid;
+    }
+    return 0;
+}
+
 void helper_store_slb(CPUPPCState *env, target_ulong rb, target_ulong rs)
 {
-    if (ppc_store_slb(env, rb, rs) < 0) {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    if (ppc_store_slb(cpu, rb & 0xfff, rb & ~0xfffULL, rs) < 0) {
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL);
     }
@@ -206,9 +248,22 @@
 
 target_ulong helper_load_slb_esid(CPUPPCState *env, target_ulong rb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     target_ulong rt = 0;
 
-    if (ppc_load_slb_esid(env, rb, &rt) < 0) {
+    if (ppc_load_slb_esid(cpu, rb, &rt) < 0) {
+        helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
+                                   POWERPC_EXCP_INVAL);
+    }
+    return rt;
+}
+
+target_ulong helper_find_slb_vsid(CPUPPCState *env, target_ulong rb)
+{
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+    target_ulong rt = 0;
+
+    if (ppc_find_slb_vsid(cpu, rb, &rt) < 0) {
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL);
     }
@@ -217,9 +272,10 @@
 
 target_ulong helper_load_slb_vsid(CPUPPCState *env, target_ulong rb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     target_ulong rt = 0;
 
-    if (ppc_load_slb_vsid(env, rb, &rt) < 0) {
+    if (ppc_load_slb_vsid(cpu, rb, &rt) < 0) {
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL);
     }
@@ -229,10 +285,56 @@
 /*
  * 64-bit hash table MMU handling
  */
+void ppc_hash64_set_sdr1(PowerPCCPU *cpu, target_ulong value,
+                         Error **errp)
+{
+    CPUPPCState *env = &cpu->env;
+    target_ulong htabsize = value & SDR_64_HTABSIZE;
 
-static int ppc_hash64_pte_prot(CPUPPCState *env,
+    env->spr[SPR_SDR1] = value;
+    if (htabsize > 28) {
+        error_setg(errp,
+                   "Invalid HTABSIZE 0x" TARGET_FMT_lx" stored in SDR1",
+                   htabsize);
+        htabsize = 28;
+    }
+    env->htab_mask = (1ULL << (htabsize + 18 - 7)) - 1;
+    env->htab_base = value & SDR_64_HTABORG;
+}
+
+void ppc_hash64_set_external_hpt(PowerPCCPU *cpu, void *hpt, int shift,
+                                 Error **errp)
+{
+    CPUPPCState *env = &cpu->env;
+    Error *local_err = NULL;
+
+    if (hpt) {
+        env->external_htab = hpt;
+    } else {
+        env->external_htab = MMU_HASH64_KVM_MANAGED_HPT;
+    }
+    ppc_hash64_set_sdr1(cpu, (target_ulong)(uintptr_t)hpt | (shift - 18),
+                        &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Not strictly necessary, but makes it clearer that an external
+     * htab is in use when debugging */
+    env->htab_base = -1;
+
+    if (kvm_enabled()) {
+        if (kvmppc_put_books_sregs(cpu) < 0) {
+            error_setg(errp, "Unable to update SDR1 in KVM");
+        }
+    }
+}
+
+static int ppc_hash64_pte_prot(PowerPCCPU *cpu,
                                ppc_slb_t *slb, ppc_hash_pte64_t pte)
 {
+    CPUPPCState *env = &cpu->env;
     unsigned pp, key;
     /* Some pp bit combinations have undefined behaviour, so default
      * to no access in those cases */
@@ -282,12 +384,12 @@
     return prot;
 }
 
-static int ppc_hash64_amr_prot(CPUPPCState *env, ppc_hash_pte64_t pte)
+static int ppc_hash64_amr_prot(PowerPCCPU *cpu, ppc_hash_pte64_t pte)
 {
+    CPUPPCState *env = &cpu->env;
     int key, amrbits;
     int prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 
-
     /* Only recent MMUs implement Virtual Page Class Key Protection */
     if (!(env->mmu_model & POWERPC_MMU_AMR)) {
         return prot;
@@ -323,25 +425,16 @@
     hwaddr pte_offset;
 
     pte_offset = pte_index * HASH_PTE_SIZE_64;
-    if (kvmppc_kern_htab) {
+    if (cpu->env.external_htab == MMU_HASH64_KVM_MANAGED_HPT) {
         /*
          * HTAB is controlled by KVM. Fetch the PTEG into a new buffer.
          */
         token = kvmppc_hash64_read_pteg(cpu, pte_index);
-        if (token) {
-            return token;
-        }
+    } else if (cpu->env.external_htab) {
         /*
-         * pteg read failed, even though we have allocated htab via
-         * kvmppc_reset_htab.
+         * HTAB is controlled by QEMU. Just point to the internally
+         * accessible PTEG.
          */
-        return 0;
-    }
-    /*
-     * HTAB is controlled by QEMU. Just point to the internally
-     * accessible PTEG.
-     */
-    if (cpu->env.external_htab) {
         token = (uint64_t)(uintptr_t) cpu->env.external_htab + pte_offset;
     } else if (cpu->env.htab_base) {
         token = cpu->env.htab_base + pte_offset;
@@ -349,133 +442,299 @@
     return token;
 }
 
-void ppc_hash64_stop_access(uint64_t token)
+void ppc_hash64_stop_access(PowerPCCPU *cpu, uint64_t token)
 {
-    if (kvmppc_kern_htab) {
-        return kvmppc_hash64_free_pteg(token);
+    if (cpu->env.external_htab == MMU_HASH64_KVM_MANAGED_HPT) {
+        kvmppc_hash64_free_pteg(token);
     }
 }
 
-static hwaddr ppc_hash64_pteg_search(CPUPPCState *env, hwaddr hash,
-                                     bool secondary, target_ulong ptem,
-                                     ppc_hash_pte64_t *pte)
+static unsigned hpte_page_shift(const struct ppc_one_seg_page_size *sps,
+    uint64_t pte0, uint64_t pte1)
 {
     int i;
+
+    if (!(pte0 & HPTE64_V_LARGE)) {
+        if (sps->page_shift != 12) {
+            /* 4kiB page in a non 4kiB segment */
+            return 0;
+        }
+        /* Normal 4kiB page */
+        return 12;
+    }
+
+    for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_page_size *ps = &sps->enc[i];
+        uint64_t mask;
+
+        if (!ps->page_shift) {
+            break;
+        }
+
+        if (ps->page_shift == 12) {
+            /* L bit is set so this can't be a 4kiB page */
+            continue;
+        }
+
+        mask = ((1ULL << ps->page_shift) - 1) & HPTE64_R_RPN;
+
+        if ((pte1 & mask) == ((uint64_t)ps->pte_enc << HPTE64_R_RPN_SHIFT)) {
+            return ps->page_shift;
+        }
+    }
+
+    return 0; /* Bad page size encoding */
+}
+
+static hwaddr ppc_hash64_pteg_search(PowerPCCPU *cpu, hwaddr hash,
+                                     const struct ppc_one_seg_page_size *sps,
+                                     target_ulong ptem,
+                                     ppc_hash_pte64_t *pte, unsigned *pshift)
+{
+    CPUPPCState *env = &cpu->env;
+    int i;
     uint64_t token;
     target_ulong pte0, pte1;
     target_ulong pte_index;
 
     pte_index = (hash & env->htab_mask) * HPTES_PER_GROUP;
-    token = ppc_hash64_start_access(ppc_env_get_cpu(env), pte_index);
+    token = ppc_hash64_start_access(cpu, pte_index);
     if (!token) {
         return -1;
     }
     for (i = 0; i < HPTES_PER_GROUP; i++) {
-        pte0 = ppc_hash64_load_hpte0(env, token, i);
-        pte1 = ppc_hash64_load_hpte1(env, token, i);
+        pte0 = ppc_hash64_load_hpte0(cpu, token, i);
+        pte1 = ppc_hash64_load_hpte1(cpu, token, i);
 
-        if ((pte0 & HPTE64_V_VALID)
-            && (secondary == !!(pte0 & HPTE64_V_SECONDARY))
-            && HPTE64_V_COMPARE(pte0, ptem)) {
+        /* This compares V, B, H (secondary) and the AVPN */
+        if (HPTE64_V_COMPARE(pte0, ptem)) {
+            *pshift = hpte_page_shift(sps, pte0, pte1);
+            /*
+             * If there is no match, ignore the PTE, it could simply
+             * be for a different segment size encoding and the
+             * architecture specifies we should not match. Linux will
+             * potentially leave behind PTEs for the wrong base page
+             * size when demoting segments.
+             */
+            if (*pshift == 0) {
+                continue;
+            }
+            /* We don't do anything with pshift yet as qemu TLB only deals
+             * with 4K pages anyway
+             */
             pte->pte0 = pte0;
             pte->pte1 = pte1;
-            ppc_hash64_stop_access(token);
+            ppc_hash64_stop_access(cpu, token);
             return (pte_index + i) * HASH_PTE_SIZE_64;
         }
     }
-    ppc_hash64_stop_access(token);
+    ppc_hash64_stop_access(cpu, token);
     /*
      * We didn't find a valid entry.
      */
     return -1;
 }
 
-static hwaddr ppc_hash64_htab_lookup(CPUPPCState *env,
+static hwaddr ppc_hash64_htab_lookup(PowerPCCPU *cpu,
                                      ppc_slb_t *slb, target_ulong eaddr,
-                                     ppc_hash_pte64_t *pte)
+                                     ppc_hash_pte64_t *pte, unsigned *pshift)
 {
+    CPUPPCState *env = &cpu->env;
     hwaddr pte_offset;
     hwaddr hash;
-    uint64_t vsid, epnshift, epnmask, epn, ptem;
+    uint64_t vsid, epnmask, epn, ptem;
+    const struct ppc_one_seg_page_size *sps = slb->sps;
 
-    /* Page size according to the SLB, which we use to generate the
-     * EPN for hash table lookup..  When we implement more recent MMU
-     * extensions this might be different from the actual page size
-     * encoded in the PTE */
-    epnshift = (slb->vsid & SLB_VSID_L)
-        ? TARGET_PAGE_BITS_16M : TARGET_PAGE_BITS;
-    epnmask = ~((1ULL << epnshift) - 1);
+    /* The SLB store path should prevent any bad page size encodings
+     * getting in there, so: */
+    assert(sps);
+
+    /* If ISL is set in LPCR we need to clamp the page size to 4K */
+    if (env->spr[SPR_LPCR] & LPCR_ISL) {
+        /* We assume that when using TCG, 4k is first entry of SPS */
+        sps = &env->sps.sps[0];
+        assert(sps->page_shift == 12);
+    }
+
+    epnmask = ~((1ULL << sps->page_shift) - 1);
 
     if (slb->vsid & SLB_VSID_B) {
         /* 1TB segment */
         vsid = (slb->vsid & SLB_VSID_VSID) >> SLB_VSID_SHIFT_1T;
         epn = (eaddr & ~SEGMENT_MASK_1T) & epnmask;
-        hash = vsid ^ (vsid << 25) ^ (epn >> epnshift);
+        hash = vsid ^ (vsid << 25) ^ (epn >> sps->page_shift);
     } else {
         /* 256M segment */
         vsid = (slb->vsid & SLB_VSID_VSID) >> SLB_VSID_SHIFT;
         epn = (eaddr & ~SEGMENT_MASK_256M) & epnmask;
-        hash = vsid ^ (epn >> epnshift);
+        hash = vsid ^ (epn >> sps->page_shift);
     }
     ptem = (slb->vsid & SLB_VSID_PTEM) | ((epn >> 16) & HPTE64_V_AVPN);
+    ptem |= HPTE64_V_VALID;
 
     /* Page address translation */
-    LOG_MMU("htab_base " TARGET_FMT_plx " htab_mask " TARGET_FMT_plx
+    qemu_log_mask(CPU_LOG_MMU,
+            "htab_base " TARGET_FMT_plx " htab_mask " TARGET_FMT_plx
             " hash " TARGET_FMT_plx "\n",
             env->htab_base, env->htab_mask, hash);
 
     /* Primary PTEG lookup */
-    LOG_MMU("0 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
+    qemu_log_mask(CPU_LOG_MMU,
+            "0 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
             " vsid=" TARGET_FMT_lx " ptem=" TARGET_FMT_lx
             " hash=" TARGET_FMT_plx "\n",
             env->htab_base, env->htab_mask, vsid, ptem,  hash);
-    pte_offset = ppc_hash64_pteg_search(env, hash, 0, ptem, pte);
+    pte_offset = ppc_hash64_pteg_search(cpu, hash, sps, ptem, pte, pshift);
 
     if (pte_offset == -1) {
         /* Secondary PTEG lookup */
-        LOG_MMU("1 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
+        ptem |= HPTE64_V_SECONDARY;
+        qemu_log_mask(CPU_LOG_MMU,
+                "1 htab=" TARGET_FMT_plx "/" TARGET_FMT_plx
                 " vsid=" TARGET_FMT_lx " api=" TARGET_FMT_lx
                 " hash=" TARGET_FMT_plx "\n", env->htab_base,
                 env->htab_mask, vsid, ptem, ~hash);
 
-        pte_offset = ppc_hash64_pteg_search(env, ~hash, 1, ptem, pte);
+        pte_offset = ppc_hash64_pteg_search(cpu, ~hash, sps, ptem, pte, pshift);
     }
 
     return pte_offset;
 }
 
-static hwaddr ppc_hash64_pte_raddr(ppc_slb_t *slb, ppc_hash_pte64_t pte,
-                                   target_ulong eaddr)
+unsigned ppc_hash64_hpte_page_shift_noslb(PowerPCCPU *cpu,
+                                          uint64_t pte0, uint64_t pte1)
 {
-    hwaddr rpn = pte.pte1 & HPTE64_R_RPN;
-    /* FIXME: Add support for SLLP extended page sizes */
-    int target_page_bits = (slb->vsid & SLB_VSID_L)
-        ? TARGET_PAGE_BITS_16M : TARGET_PAGE_BITS;
-    hwaddr mask = (1ULL << target_page_bits) - 1;
+    CPUPPCState *env = &cpu->env;
+    int i;
 
-    return (rpn & ~mask) | (eaddr & mask);
+    if (!(pte0 & HPTE64_V_LARGE)) {
+        return 12;
+    }
+
+    /*
+     * The encodings in env->sps need to be carefully chosen so that
+     * this gives an unambiguous result.
+     */
+    for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_seg_page_size *sps = &env->sps.sps[i];
+        unsigned shift;
+
+        if (!sps->page_shift) {
+            break;
+        }
+
+        shift = hpte_page_shift(sps, pte0, pte1);
+        if (shift) {
+            return shift;
+        }
+    }
+
+    return 0;
 }
 
-int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong eaddr,
+static void ppc_hash64_set_isi(CPUState *cs, CPUPPCState *env,
+                               uint64_t error_code)
+{
+    bool vpm;
+
+    if (msr_ir) {
+        vpm = !!(env->spr[SPR_LPCR] & LPCR_VPM1);
+    } else {
+        vpm = !!(env->spr[SPR_LPCR] & LPCR_VPM0);
+    }
+    if (vpm && !msr_hv) {
+        cs->exception_index = POWERPC_EXCP_HISI;
+    } else {
+        cs->exception_index = POWERPC_EXCP_ISI;
+    }
+    env->error_code = error_code;
+}
+
+static void ppc_hash64_set_dsi(CPUState *cs, CPUPPCState *env, uint64_t dar,
+                               uint64_t dsisr)
+{
+    bool vpm;
+
+    if (msr_dr) {
+        vpm = !!(env->spr[SPR_LPCR] & LPCR_VPM1);
+    } else {
+        vpm = !!(env->spr[SPR_LPCR] & LPCR_VPM0);
+    }
+    if (vpm && !msr_hv) {
+        cs->exception_index = POWERPC_EXCP_HDSI;
+        env->spr[SPR_HDAR] = dar;
+        env->spr[SPR_HDSISR] = dsisr;
+    } else {
+        cs->exception_index = POWERPC_EXCP_DSI;
+        env->spr[SPR_DAR] = dar;
+        env->spr[SPR_DSISR] = dsisr;
+   }
+    env->error_code = 0;
+}
+
+
+int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, vaddr eaddr,
                                 int rwx, int mmu_idx)
 {
     CPUState *cs = CPU(cpu);
     CPUPPCState *env = &cpu->env;
     ppc_slb_t *slb;
+    unsigned apshift;
     hwaddr pte_offset;
     ppc_hash_pte64_t pte;
     int pp_prot, amr_prot, prot;
-    uint64_t new_pte1;
+    uint64_t new_pte1, dsisr;
     const int need_prot[] = {PAGE_READ, PAGE_WRITE, PAGE_EXEC};
     hwaddr raddr;
 
     assert((rwx == 0) || (rwx == 1) || (rwx == 2));
 
+    /* Note on LPCR usage: 970 uses HID4, but our special variant
+     * of store_spr copies relevant fields into env->spr[SPR_LPCR].
+     * Similarily we filter unimplemented bits when storing into
+     * LPCR depending on the MMU version. This code can thus just
+     * use the LPCR "as-is".
+     */
+
     /* 1. Handle real mode accesses */
     if (((rwx == 2) && (msr_ir == 0)) || ((rwx != 2) && (msr_dr == 0))) {
-        /* Translation is off */
-        /* In real mode the top 4 effective address bits are ignored */
+        /* Translation is supposedly "off"  */
+        /* In real mode the top 4 effective address bits are (mostly) ignored */
         raddr = eaddr & 0x0FFFFFFFFFFFFFFFULL;
+
+        /* In HV mode, add HRMOR if top EA bit is clear */
+        if (msr_hv || !env->has_hv_mode) {
+            if (!(eaddr >> 63)) {
+                raddr |= env->spr[SPR_HRMOR];
+            }
+        } else {
+            /* Otherwise, check VPM for RMA vs VRMA */
+            if (env->spr[SPR_LPCR] & LPCR_VPM0) {
+                slb = &env->vrma_slb;
+                if (slb->sps) {
+                    goto skip_slb_search;
+                }
+                /* Not much else to do here */
+                cs->exception_index = POWERPC_EXCP_MCHECK;
+                env->error_code = 0;
+                return 1;
+            } else if (raddr < env->rmls) {
+                /* RMA. Check bounds in RMLS */
+                raddr |= env->spr[SPR_RMOR];
+            } else {
+                /* The access failed, generate the approriate interrupt */
+                if (rwx == 2) {
+                    ppc_hash64_set_isi(cs, env, 0x08000000);
+                } else {
+                    dsisr = 0x08000000;
+                    if (rwx == 1) {
+                        dsisr |= 0x02000000;
+                    }
+                    ppc_hash64_set_dsi(cs, env, eaddr, dsisr);
+                }
+                return 1;
+            }
+        }
         tlb_set_page(cs, eaddr & TARGET_PAGE_MASK, raddr & TARGET_PAGE_MASK,
                      PAGE_READ | PAGE_WRITE | PAGE_EXEC, mmu_idx,
                      TARGET_PAGE_SIZE);
@@ -483,8 +742,7 @@
     }
 
     /* 2. Translation is on, so look up the SLB */
-    slb = slb_lookup(env, eaddr);
-
+    slb = slb_lookup(cpu, eaddr);
     if (!slb) {
         if (rwx == 2) {
             cs->exception_index = POWERPC_EXCP_ISEG;
@@ -497,51 +755,44 @@
         return 1;
     }
 
+skip_slb_search:
+
     /* 3. Check for segment level no-execute violation */
     if ((rwx == 2) && (slb->vsid & SLB_VSID_N)) {
-        cs->exception_index = POWERPC_EXCP_ISI;
-        env->error_code = 0x10000000;
+        ppc_hash64_set_isi(cs, env, 0x10000000);
         return 1;
     }
 
     /* 4. Locate the PTE in the hash table */
-    pte_offset = ppc_hash64_htab_lookup(env, slb, eaddr, &pte);
+    pte_offset = ppc_hash64_htab_lookup(cpu, slb, eaddr, &pte, &apshift);
     if (pte_offset == -1) {
+        dsisr = 0x40000000;
         if (rwx == 2) {
-            cs->exception_index = POWERPC_EXCP_ISI;
-            env->error_code = 0x40000000;
+            ppc_hash64_set_isi(cs, env, dsisr);
         } else {
-            cs->exception_index = POWERPC_EXCP_DSI;
-            env->error_code = 0;
-            env->spr[SPR_DAR] = eaddr;
             if (rwx == 1) {
-                env->spr[SPR_DSISR] = 0x42000000;
-            } else {
-                env->spr[SPR_DSISR] = 0x40000000;
+                dsisr |= 0x02000000;
             }
+            ppc_hash64_set_dsi(cs, env, eaddr, dsisr);
         }
         return 1;
     }
-    LOG_MMU("found PTE at offset %08" HWADDR_PRIx "\n", pte_offset);
+    qemu_log_mask(CPU_LOG_MMU,
+                "found PTE at offset %08" HWADDR_PRIx "\n", pte_offset);
 
     /* 5. Check access permissions */
 
-    pp_prot = ppc_hash64_pte_prot(env, slb, pte);
-    amr_prot = ppc_hash64_amr_prot(env, pte);
+    pp_prot = ppc_hash64_pte_prot(cpu, slb, pte);
+    amr_prot = ppc_hash64_amr_prot(cpu, pte);
     prot = pp_prot & amr_prot;
 
     if ((need_prot[rwx] & ~prot) != 0) {
         /* Access right violation */
-        LOG_MMU("PTE access rejected\n");
+        qemu_log_mask(CPU_LOG_MMU, "PTE access rejected\n");
         if (rwx == 2) {
-            cs->exception_index = POWERPC_EXCP_ISI;
-            env->error_code = 0x08000000;
+            ppc_hash64_set_isi(cs, env, 0x08000000);
         } else {
-            target_ulong dsisr = 0;
-
-            cs->exception_index = POWERPC_EXCP_DSI;
-            env->error_code = 0;
-            env->spr[SPR_DAR] = eaddr;
+            dsisr = 0;
             if (need_prot[rwx] & ~pp_prot) {
                 dsisr |= 0x08000000;
             }
@@ -551,12 +802,12 @@
             if (need_prot[rwx] & ~amr_prot) {
                 dsisr |= 0x00200000;
             }
-            env->spr[SPR_DSISR] = dsisr;
+            ppc_hash64_set_dsi(cs, env, eaddr, dsisr);
         }
         return 1;
     }
 
-    LOG_MMU("PTE access granted !\n");
+    qemu_log_mask(CPU_LOG_MMU, "PTE access granted !\n");
 
     /* 6. Update PTE referenced and changed bits if necessary */
 
@@ -570,60 +821,239 @@
     }
 
     if (new_pte1 != pte.pte1) {
-        ppc_hash64_store_hpte(env, pte_offset / HASH_PTE_SIZE_64,
+        ppc_hash64_store_hpte(cpu, pte_offset / HASH_PTE_SIZE_64,
                               pte.pte0, new_pte1);
     }
 
     /* 7. Determine the real address from the PTE */
 
-    raddr = ppc_hash64_pte_raddr(slb, pte, eaddr);
+    raddr = deposit64(pte.pte1 & HPTE64_R_RPN, 0, apshift, eaddr);
 
     tlb_set_page(cs, eaddr & TARGET_PAGE_MASK, raddr & TARGET_PAGE_MASK,
-                 prot, mmu_idx, TARGET_PAGE_SIZE);
+                 prot, mmu_idx, 1ULL << apshift);
 
     return 0;
 }
 
-hwaddr ppc_hash64_get_phys_page_debug(CPUPPCState *env, target_ulong addr)
+hwaddr ppc_hash64_get_phys_page_debug(PowerPCCPU *cpu, target_ulong addr)
 {
+    CPUPPCState *env = &cpu->env;
     ppc_slb_t *slb;
-    hwaddr pte_offset;
+    hwaddr pte_offset, raddr;
     ppc_hash_pte64_t pte;
+    unsigned apshift;
 
+    /* Handle real mode */
     if (msr_dr == 0) {
         /* In real mode the top 4 effective address bits are ignored */
-        return addr & 0x0FFFFFFFFFFFFFFFULL;
+        raddr = addr & 0x0FFFFFFFFFFFFFFFULL;
+
+        /* In HV mode, add HRMOR if top EA bit is clear */
+        if ((msr_hv || !env->has_hv_mode) && !(addr >> 63)) {
+            return raddr | env->spr[SPR_HRMOR];
+        }
+
+        /* Otherwise, check VPM for RMA vs VRMA */
+        if (env->spr[SPR_LPCR] & LPCR_VPM0) {
+            slb = &env->vrma_slb;
+            if (!slb->sps) {
+                return -1;
+            }
+        } else if (raddr < env->rmls) {
+            /* RMA. Check bounds in RMLS */
+            return raddr | env->spr[SPR_RMOR];
+        } else {
+            return -1;
+        }
+    } else {
+        slb = slb_lookup(cpu, addr);
+        if (!slb) {
+            return -1;
+        }
     }
 
-    slb = slb_lookup(env, addr);
-    if (!slb) {
-        return -1;
-    }
-
-    pte_offset = ppc_hash64_htab_lookup(env, slb, addr, &pte);
+    pte_offset = ppc_hash64_htab_lookup(cpu, slb, addr, &pte, &apshift);
     if (pte_offset == -1) {
         return -1;
     }
 
-    return ppc_hash64_pte_raddr(slb, pte, addr) & TARGET_PAGE_MASK;
+    return deposit64(pte.pte1 & HPTE64_R_RPN, 0, apshift, addr)
+        & TARGET_PAGE_MASK;
 }
 
-void ppc_hash64_store_hpte(CPUPPCState *env,
+void ppc_hash64_store_hpte(PowerPCCPU *cpu,
                            target_ulong pte_index,
                            target_ulong pte0, target_ulong pte1)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
 
-    if (kvmppc_kern_htab) {
-        return kvmppc_hash64_write_pte(env, pte_index, pte0, pte1);
+    if (env->external_htab == MMU_HASH64_KVM_MANAGED_HPT) {
+        kvmppc_hash64_write_pte(env, pte_index, pte0, pte1);
+        return;
     }
 
     pte_index *= HASH_PTE_SIZE_64;
     if (env->external_htab) {
         stq_p(env->external_htab + pte_index, pte0);
-        stq_p(env->external_htab + pte_index + HASH_PTE_SIZE_64/2, pte1);
+        stq_p(env->external_htab + pte_index + HASH_PTE_SIZE_64 / 2, pte1);
     } else {
-        stq_phys(cs->as, env->htab_base + pte_index, pte0);
-        stq_phys(cs->as, env->htab_base + pte_index + HASH_PTE_SIZE_64/2, pte1);
+        stq_phys(CPU(cpu)->as, env->htab_base + pte_index, pte0);
+        stq_phys(CPU(cpu)->as,
+                 env->htab_base + pte_index + HASH_PTE_SIZE_64 / 2, pte1);
     }
 }
+
+void ppc_hash64_tlb_flush_hpte(PowerPCCPU *cpu,
+                               target_ulong pte_index,
+                               target_ulong pte0, target_ulong pte1)
+{
+    /*
+     * XXX: given the fact that there are too many segments to
+     * invalidate, and we still don't have a tlb_flush_mask(env, n,
+     * mask) in QEMU, we just invalidate all TLBs
+     */
+    tlb_flush(CPU(cpu), 1);
+}
+
+void ppc_hash64_update_rmls(CPUPPCState *env)
+{
+    uint64_t lpcr = env->spr[SPR_LPCR];
+
+    /*
+     * This is the full 4 bits encoding of POWER8. Previous
+     * CPUs only support a subset of these but the filtering
+     * is done when writing LPCR
+     */
+    switch ((lpcr & LPCR_RMLS) >> LPCR_RMLS_SHIFT) {
+    case 0x8: /* 32MB */
+        env->rmls = 0x2000000ull;
+        break;
+    case 0x3: /* 64MB */
+        env->rmls = 0x4000000ull;
+        break;
+    case 0x7: /* 128MB */
+        env->rmls = 0x8000000ull;
+        break;
+    case 0x4: /* 256MB */
+        env->rmls = 0x10000000ull;
+        break;
+    case 0x2: /* 1GB */
+        env->rmls = 0x40000000ull;
+        break;
+    case 0x1: /* 16GB */
+        env->rmls = 0x400000000ull;
+        break;
+    default:
+        /* What to do here ??? */
+        env->rmls = 0;
+    }
+}
+
+void ppc_hash64_update_vrma(CPUPPCState *env)
+{
+    const struct ppc_one_seg_page_size *sps = NULL;
+    target_ulong esid, vsid, lpcr;
+    ppc_slb_t *slb = &env->vrma_slb;
+    uint32_t vrmasd;
+    int i;
+
+    /* First clear it */
+    slb->esid = slb->vsid = 0;
+    slb->sps = NULL;
+
+    /* Is VRMA enabled ? */
+    lpcr = env->spr[SPR_LPCR];
+    if (!(lpcr & LPCR_VPM0)) {
+        return;
+    }
+
+    /* Make one up. Mostly ignore the ESID which will not be
+     * needed for translation
+     */
+    vsid = SLB_VSID_VRMA;
+    vrmasd = (lpcr & LPCR_VRMASD) >> LPCR_VRMASD_SHIFT;
+    vsid |= (vrmasd << 4) & (SLB_VSID_L | SLB_VSID_LP);
+    esid = SLB_ESID_V;
+
+   for (i = 0; i < PPC_PAGE_SIZES_MAX_SZ; i++) {
+        const struct ppc_one_seg_page_size *sps1 = &env->sps.sps[i];
+
+        if (!sps1->page_shift) {
+            break;
+        }
+
+        if ((vsid & SLB_VSID_LLP_MASK) == sps1->slb_enc) {
+            sps = sps1;
+            break;
+        }
+    }
+
+    if (!sps) {
+        error_report("Bad page size encoding esid 0x"TARGET_FMT_lx
+                     " vsid 0x"TARGET_FMT_lx, esid, vsid);
+        return;
+    }
+
+    slb->vsid = vsid;
+    slb->esid = esid;
+    slb->sps = sps;
+}
+
+void helper_store_lpcr(CPUPPCState *env, target_ulong val)
+{
+    uint64_t lpcr = 0;
+
+    /* Filter out bits */
+    switch (env->mmu_model) {
+    case POWERPC_MMU_64B: /* 970 */
+        if (val & 0x40) {
+            lpcr |= LPCR_LPES0;
+        }
+        if (val & 0x8000000000000000ull) {
+            lpcr |= LPCR_LPES1;
+        }
+        if (val & 0x20) {
+            lpcr |= (0x4ull << LPCR_RMLS_SHIFT);
+        }
+        if (val & 0x4000000000000000ull) {
+            lpcr |= (0x2ull << LPCR_RMLS_SHIFT);
+        }
+        if (val & 0x2000000000000000ull) {
+            lpcr |= (0x1ull << LPCR_RMLS_SHIFT);
+        }
+        env->spr[SPR_RMOR] = ((lpcr >> 41) & 0xffffull) << 26;
+
+        /* XXX We could also write LPID from HID4 here
+         * but since we don't tag any translation on it
+         * it doesn't actually matter
+         */
+        /* XXX For proper emulation of 970 we also need
+         * to dig HRMOR out of HID5
+         */
+        break;
+    case POWERPC_MMU_2_03: /* P5p */
+        lpcr = val & (LPCR_RMLS | LPCR_ILE |
+                      LPCR_LPES0 | LPCR_LPES1 |
+                      LPCR_RMI | LPCR_HDICE);
+        break;
+    case POWERPC_MMU_2_06: /* P7 */
+        lpcr = val & (LPCR_VPM0 | LPCR_VPM1 | LPCR_ISL | LPCR_DPFD |
+                      LPCR_VRMASD | LPCR_RMLS | LPCR_ILE |
+                      LPCR_P7_PECE0 | LPCR_P7_PECE1 | LPCR_P7_PECE2 |
+                      LPCR_MER | LPCR_TC |
+                      LPCR_LPES0 | LPCR_LPES1 | LPCR_HDICE);
+        break;
+    case POWERPC_MMU_2_07: /* P8 */
+        lpcr = val & (LPCR_VPM0 | LPCR_VPM1 | LPCR_ISL | LPCR_KBV |
+                      LPCR_DPFD | LPCR_VRMASD | LPCR_RMLS | LPCR_ILE |
+                      LPCR_AIL | LPCR_ONL | LPCR_P8_PECE0 | LPCR_P8_PECE1 |
+                      LPCR_P8_PECE2 | LPCR_P8_PECE3 | LPCR_P8_PECE4 |
+                      LPCR_MER | LPCR_TC | LPCR_LPES0 | LPCR_HDICE);
+        break;
+    default:
+        ;
+    }
+    env->spr[SPR_LPCR] = lpcr;
+    ppc_hash64_update_rmls(env);
+    ppc_hash64_update_vrma(env);
+}
diff --git a/target-ppc/mmu-hash64.h b/target-ppc/mmu-hash64.h
index 49e385d..db265e3 100644
--- a/target-ppc/mmu-hash64.h
+++ b/target-ppc/mmu-hash64.h
@@ -1,16 +1,25 @@
-#if !defined (__MMU_HASH64_H__)
-#define __MMU_HASH64_H__
+#ifndef MMU_HASH64_H
+#define MMU_HASH64_H
 
 #ifndef CONFIG_USER_ONLY
 
 #ifdef TARGET_PPC64
-void dump_slb(FILE *f, fprintf_function cpu_fprintf, CPUPPCState *env);
-int ppc_store_slb (CPUPPCState *env, target_ulong rb, target_ulong rs);
-hwaddr ppc_hash64_get_phys_page_debug(CPUPPCState *env, target_ulong addr);
-int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, target_ulong address, int rw,
+void ppc_hash64_check_page_sizes(PowerPCCPU *cpu, Error **errp);
+void dump_slb(FILE *f, fprintf_function cpu_fprintf, PowerPCCPU *cpu);
+int ppc_store_slb(PowerPCCPU *cpu, target_ulong slot,
+                  target_ulong esid, target_ulong vsid);
+hwaddr ppc_hash64_get_phys_page_debug(PowerPCCPU *cpu, target_ulong addr);
+int ppc_hash64_handle_mmu_fault(PowerPCCPU *cpu, vaddr address, int rw,
                                 int mmu_idx);
-void ppc_hash64_store_hpte(CPUPPCState *env, target_ulong index,
+void ppc_hash64_store_hpte(PowerPCCPU *cpu, target_ulong index,
                            target_ulong pte0, target_ulong pte1);
+void ppc_hash64_tlb_flush_hpte(PowerPCCPU *cpu,
+                               target_ulong pte_index,
+                               target_ulong pte0, target_ulong pte1);
+unsigned ppc_hash64_hpte_page_shift_noslb(PowerPCCPU *cpu,
+                                          uint64_t pte0, uint64_t pte1);
+void ppc_hash64_update_vrma(CPUPPCState *env);
+void ppc_hash64_update_rmls(CPUPPCState *env);
 #endif
 
 /*
@@ -29,6 +38,7 @@
 #define SLB_VSID_B_256M         0x0000000000000000ULL
 #define SLB_VSID_B_1T           0x4000000000000000ULL
 #define SLB_VSID_VSID           0x3FFFFFFFFFFFF000ULL
+#define SLB_VSID_VRMA           (0x0001FFFFFF000000ULL | SLB_VSID_B_1T)
 #define SLB_VSID_PTEM           (SLB_VSID_B | SLB_VSID_VSID)
 #define SLB_VSID_KS             0x0000000000000800ULL
 #define SLB_VSID_KP             0x0000000000000400ULL
@@ -37,6 +47,11 @@
 #define SLB_VSID_C              0x0000000000000080ULL /* class */
 #define SLB_VSID_LP             0x0000000000000030ULL
 #define SLB_VSID_ATTR           0x0000000000000FFFULL
+#define SLB_VSID_LLP_MASK       (SLB_VSID_L | SLB_VSID_LP)
+#define SLB_VSID_4K             0x0000000000000000ULL
+#define SLB_VSID_64K            0x0000000000000110ULL
+#define SLB_VSID_16M            0x0000000000000100ULL
+#define SLB_VSID_16G            0x0000000000000120ULL
 
 /*
  * Hash page table definitions
@@ -50,7 +65,7 @@
 #define HPTE64_V_AVPN_SHIFT     7
 #define HPTE64_V_AVPN           0x3fffffffffffff80ULL
 #define HPTE64_V_AVPN_VAL(x)    (((x) & HPTE64_V_AVPN) >> HPTE64_V_AVPN_SHIFT)
-#define HPTE64_V_COMPARE(x, y)  (!(((x) ^ (y)) & 0xffffffffffffff80ULL))
+#define HPTE64_V_COMPARE(x, y)  (!(((x) ^ (y)) & 0xffffffffffffff83ULL))
 #define HPTE64_V_LARGE          0x0000000000000004ULL
 #define HPTE64_V_SECONDARY      0x0000000000000002ULL
 #define HPTE64_V_VALID          0x0000000000000001ULL
@@ -77,36 +92,39 @@
 #define HPTE64_V_1TB_SEG        0x4000000000000000ULL
 #define HPTE64_V_VRMA_MASK      0x4001ffffff000000ULL
 
+void ppc_hash64_set_sdr1(PowerPCCPU *cpu, target_ulong value,
+                         Error **errp);
+void ppc_hash64_set_external_hpt(PowerPCCPU *cpu, void *hpt, int shift,
+                                 Error **errp);
 
-extern bool kvmppc_kern_htab;
 uint64_t ppc_hash64_start_access(PowerPCCPU *cpu, target_ulong pte_index);
-void ppc_hash64_stop_access(uint64_t token);
+void ppc_hash64_stop_access(PowerPCCPU *cpu, uint64_t token);
 
-static inline target_ulong ppc_hash64_load_hpte0(CPUPPCState *env,
+static inline target_ulong ppc_hash64_load_hpte0(PowerPCCPU *cpu,
                                                  uint64_t token, int index)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
     uint64_t addr;
 
     addr = token + (index * HASH_PTE_SIZE_64);
     if (env->external_htab) {
         return  ldq_p((const void *)(uintptr_t)addr);
     } else {
-        return ldq_phys(cs->as, addr);
+        return ldq_phys(CPU(cpu)->as, addr);
     }
 }
 
-static inline target_ulong ppc_hash64_load_hpte1(CPUPPCState *env,
+static inline target_ulong ppc_hash64_load_hpte1(PowerPCCPU *cpu,
                                                  uint64_t token, int index)
 {
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
+    CPUPPCState *env = &cpu->env;
     uint64_t addr;
 
     addr = token + (index * HASH_PTE_SIZE_64) + HASH_PTE_SIZE_64/2;
     if (env->external_htab) {
         return  ldq_p((const void *)(uintptr_t)addr);
     } else {
-        return ldq_phys(cs->as, addr);
+        return ldq_phys(CPU(cpu)->as, addr);
     }
 }
 
@@ -116,4 +134,4 @@
 
 #endif /* CONFIG_USER_ONLY */
 
-#endif /* !defined (__MMU_HASH64_H__) */
+#endif /* MMU_HASH64_H */
diff --git a/target-ppc/mmu_helper.c b/target-ppc/mmu_helper.c
index 4a34a73..3eb3cd7 100644
--- a/target-ppc/mmu_helper.c
+++ b/target-ppc/mmu_helper.c
@@ -16,37 +16,39 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash64.h"
 #include "mmu-hash32.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "exec/log.h"
+#include "helper_regs.h"
 
 //#define DEBUG_MMU
 //#define DEBUG_BATS
 //#define DEBUG_SOFTWARE_TLB
 //#define DUMP_PAGE_TABLES
-//#define DEBUG_SOFTWARE_TLB
 //#define FLUSH_ALL_TLBS
 
 #ifdef DEBUG_MMU
-#  define LOG_MMU(...) qemu_log(__VA_ARGS__)
-#  define LOG_MMU_STATE(cpu) log_cpu_state((cpu), 0)
+#  define LOG_MMU_STATE(cpu) log_cpu_state_mask(CPU_LOG_MMU, (cpu), 0)
 #else
-#  define LOG_MMU(...) do { } while (0)
 #  define LOG_MMU_STATE(cpu) do { } while (0)
 #endif
 
 #ifdef DEBUG_SOFTWARE_TLB
-#  define LOG_SWTLB(...) qemu_log(__VA_ARGS__)
+#  define LOG_SWTLB(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_SWTLB(...) do { } while (0)
 #endif
 
 #ifdef DEBUG_BATS
-#  define LOG_BATS(...) qemu_log(__VA_ARGS__)
+#  define LOG_BATS(...) qemu_log_mask(CPU_LOG_MMU, __VA_ARGS__)
 #else
 #  define LOG_BATS(...) do { } while (0)
 #endif
@@ -164,7 +166,7 @@
             if (ctx->raddr != (hwaddr)-1ULL) {
                 /* all matches should have equal RPN, WIMG & PP */
                 if ((ctx->raddr & mmask) != (pte1 & mmask)) {
-                    qemu_log("Bad RPN/WIMG/PP\n");
+                    qemu_log_mask(CPU_LOG_MMU, "Bad RPN/WIMG/PP\n");
                     return -3;
                 }
             }
@@ -176,10 +178,10 @@
             ret = check_prot(ctx->prot, rw, type);
             if (ret == 0) {
                 /* Access granted */
-                LOG_MMU("PTE access granted !\n");
+                qemu_log_mask(CPU_LOG_MMU, "PTE access granted !\n");
             } else {
                 /* Access right violation */
-                LOG_MMU("PTE access rejected\n");
+                qemu_log_mask(CPU_LOG_MMU, "PTE access rejected\n");
             }
         }
     }
@@ -480,8 +482,9 @@
     ctx->nx = sr & 0x10000000 ? 1 : 0;
     vsid = sr & 0x00FFFFFF;
     target_page_bits = TARGET_PAGE_BITS;
-    LOG_MMU("Check segment v=" TARGET_FMT_lx " %d " TARGET_FMT_lx " nip="
-            TARGET_FMT_lx " lr=" TARGET_FMT_lx
+    qemu_log_mask(CPU_LOG_MMU,
+            "Check segment v=" TARGET_FMT_lx " %d " TARGET_FMT_lx
+            " nip=" TARGET_FMT_lx " lr=" TARGET_FMT_lx
             " ir=%d dr=%d pr=%d %d t=%d\n",
             eaddr, (int)(eaddr >> 28), sr, env->nip, env->lr, (int)msr_ir,
             (int)msr_dr, pr != 0 ? 1 : 0, rw, type);
@@ -489,14 +492,16 @@
     hash = vsid ^ pgidx;
     ctx->ptem = (vsid << 7) | (pgidx >> 10);
 
-    LOG_MMU("pte segment: key=%d ds %d nx %d vsid " TARGET_FMT_lx "\n",
+    qemu_log_mask(CPU_LOG_MMU,
+            "pte segment: key=%d ds %d nx %d vsid " TARGET_FMT_lx "\n",
             ctx->key, ds, ctx->nx, vsid);
     ret = -1;
     if (!ds) {
         /* Check if instruction fetch is allowed, if needed */
         if (type != ACCESS_CODE || ctx->nx == 0) {
             /* Page address translation */
-            LOG_MMU("htab_base " TARGET_FMT_plx " htab_mask " TARGET_FMT_plx
+            qemu_log_mask(CPU_LOG_MMU, "htab_base " TARGET_FMT_plx
+                    " htab_mask " TARGET_FMT_plx
                     " hash " TARGET_FMT_plx "\n",
                     env->htab_base, env->htab_mask, hash);
             ctx->hash[0] = hash;
@@ -507,18 +512,20 @@
             /* Software TLB search */
             ret = ppc6xx_tlb_check(env, ctx, eaddr, rw, type);
 #if defined(DUMP_PAGE_TABLES)
-            if (qemu_log_enabled()) {
+            if (qemu_loglevel_mask(CPU_LOG_MMU)) {
+                CPUState *cs = ENV_GET_CPU(env);
                 hwaddr curaddr;
                 uint32_t a0, a1, a2, a3;
 
                 qemu_log("Page table: " TARGET_FMT_plx " len " TARGET_FMT_plx
-                         "\n", sdr, mask + 0x80);
-                for (curaddr = sdr; curaddr < (sdr + mask + 0x80);
+                         "\n", env->htab_base, env->htab_mask + 0x80);
+                for (curaddr = env->htab_base;
+                     curaddr < (env->htab_base + env->htab_mask + 0x80);
                      curaddr += 16) {
-                    a0 = ldl_phys(curaddr);
-                    a1 = ldl_phys(curaddr + 4);
-                    a2 = ldl_phys(curaddr + 8);
-                    a3 = ldl_phys(curaddr + 12);
+                    a0 = ldl_phys(cs->as, curaddr);
+                    a1 = ldl_phys(cs->as, curaddr + 4);
+                    a2 = ldl_phys(cs->as, curaddr + 8);
+                    a3 = ldl_phys(cs->as, curaddr + 12);
                     if (a0 != 0 || a1 != 0 || a2 != 0 || a3 != 0) {
                         qemu_log(TARGET_FMT_plx ": %08x %08x %08x %08x\n",
                                  curaddr, a0, a1, a2, a3);
@@ -527,13 +534,13 @@
             }
 #endif
         } else {
-            LOG_MMU("No access allowed\n");
+            qemu_log_mask(CPU_LOG_MMU, "No access allowed\n");
             ret = -3;
         }
     } else {
         target_ulong sr;
 
-        LOG_MMU("direct store...\n");
+        qemu_log_mask(CPU_LOG_MMU, "direct store...\n");
         /* Direct-store segment : absolutely *BUGGY* for now */
 
         /* Direct-store implies a 32-bit MMU.
@@ -574,8 +581,8 @@
             /* eciwx or ecowx */
             return -4;
         default:
-            qemu_log("ERROR: instruction should not need "
-                        "address translation\n");
+            qemu_log_mask(CPU_LOG_MMU, "ERROR: instruction should not need "
+                          "address translation\n");
             return -4;
         }
         if ((rw == 1 || ctx->key != 1) && (rw == 0 || ctx->key != 0)) {
@@ -657,32 +664,6 @@
     tlb_flush(CPU(cpu), 1);
 }
 
-static inline void ppc4xx_tlb_invalidate_virt(CPUPPCState *env,
-                                              target_ulong eaddr, uint32_t pid)
-{
-#if !defined(FLUSH_ALL_TLBS)
-    CPUState *cs = CPU(ppc_env_get_cpu(env));
-    ppcemb_tlb_t *tlb;
-    hwaddr raddr;
-    target_ulong page, end;
-    int i;
-
-    for (i = 0; i < env->nb_tlb; i++) {
-        tlb = &env->tlb.tlbe[i];
-        if (ppcemb_tlb_check(env, tlb, &raddr, eaddr, pid, 0, i) == 0) {
-            end = tlb->EPN + tlb->size;
-            for (page = tlb->EPN; page < end; page += TARGET_PAGE_SIZE) {
-                tlb_flush_page(cs, page);
-            }
-            tlb->prot &= ~PAGE_VALID;
-            break;
-        }
-    }
-#else
-    ppc4xx_tlb_invalidate_all(env);
-#endif
-}
-
 static int mmu40x_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                        target_ulong address, int rw,
                                        int access_type)
@@ -915,9 +896,9 @@
 
     mask = ~(booke206_tlb_to_page_size(env, tlb) - 1);
     LOG_SWTLB("%s: TLB ADDR=0x" TARGET_FMT_lx " PID=0x%x MAS1=0x%x MAS2=0x%"
-              PRIx64 " mask=0x" TARGET_FMT_lx " MAS7_3=0x%" PRIx64 " MAS8=%x\n",
-              __func__, address, pid, tlb->mas1, tlb->mas2, mask, tlb->mas7_3,
-              tlb->mas8);
+              PRIx64 " mask=0x%" HWADDR_PRIx " MAS7_3=0x%" PRIx64 " MAS8=0x%"
+              PRIx32 "\n", __func__, address, pid, tlb->mas1, tlb->mas2, mask,
+              tlb->mas7_3, tlb->mas8);
 
     /* Check PID */
     tlb_pid = (tlb->mas1 & MAS1_TID_MASK) >> MAS1_TID_SHIFT;
@@ -1292,10 +1273,12 @@
         break;
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
-        dump_slb(f, cpu_fprintf, env);
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
+        dump_slb(f, cpu_fprintf, ppc_env_get_cpu(env));
         break;
 #endif
     default:
@@ -1432,15 +1415,17 @@
     switch (env->mmu_model) {
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
-        return ppc_hash64_get_phys_page_debug(env, addr);
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
+        return ppc_hash64_get_phys_page_debug(cpu, addr);
 #endif
 
     case POWERPC_MMU_32B:
     case POWERPC_MMU_601:
-        return ppc_hash32_get_phys_page_debug(env, addr);
+        return ppc_hash32_get_phys_page_debug(cpu, addr);
 
     default:
         ;
@@ -1506,6 +1491,7 @@
                                     int rw, int mmu_idx)
 {
     CPUState *cs = CPU(ppc_env_get_cpu(env));
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
     mmu_ctx_t ctx;
     int access_type;
     int ret = 0;
@@ -1607,9 +1593,9 @@
                 tlb_miss:
                     env->error_code |= ctx.key << 19;
                     env->spr[SPR_HASH1] = env->htab_base +
-                        get_pteg_offset32(env, ctx.hash[0]);
+                        get_pteg_offset32(cpu, ctx.hash[0]);
                     env->spr[SPR_HASH2] = env->htab_base +
-                        get_pteg_offset32(env, ctx.hash[1]);
+                        get_pteg_offset32(cpu, ctx.hash[1]);
                     break;
                 case POWERPC_MMU_SOFT_74xx:
                     if (rw == 1) {
@@ -1762,6 +1748,9 @@
 void helper_store_ibatu(CPUPPCState *env, uint32_t nr, target_ulong value)
 {
     target_ulong mask;
+#if defined(FLUSH_ALL_TLBS)
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+#endif
 
     dump_store_bat(env, 'I', 0, nr, value);
     if (env->IBAT[0][nr] != value) {
@@ -1780,7 +1769,7 @@
 #if !defined(FLUSH_ALL_TLBS)
         do_invalidate_BAT(env, env->IBAT[0][nr], mask);
 #else
-        tlb_flush(env, 1);
+        tlb_flush(CPU(cpu), 1);
 #endif
     }
 }
@@ -1794,6 +1783,9 @@
 void helper_store_dbatu(CPUPPCState *env, uint32_t nr, target_ulong value)
 {
     target_ulong mask;
+#if defined(FLUSH_ALL_TLBS)
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+#endif
 
     dump_store_bat(env, 'D', 0, nr, value);
     if (env->DBAT[0][nr] != value) {
@@ -1812,7 +1804,7 @@
 #if !defined(FLUSH_ALL_TLBS)
         do_invalidate_BAT(env, env->DBAT[0][nr], mask);
 #else
-        tlb_flush(env, 1);
+        tlb_flush(CPU(cpu), 1);
 #endif
     }
 }
@@ -1827,6 +1819,7 @@
 {
     target_ulong mask;
 #if defined(FLUSH_ALL_TLBS)
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     int do_inval;
 #endif
 
@@ -1859,7 +1852,7 @@
         }
 #if defined(FLUSH_ALL_TLBS)
         if (do_inval) {
-            tlb_flush(env, 1);
+            tlb_flush(CPU(cpu), 1);
         }
 #endif
     }
@@ -1870,6 +1863,7 @@
 #if !defined(FLUSH_ALL_TLBS)
     target_ulong mask;
 #else
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     int do_inval;
 #endif
 
@@ -1898,7 +1892,7 @@
         env->DBAT[1][nr] = value;
 #if defined(FLUSH_ALL_TLBS)
         if (do_inval) {
-            tlb_flush(env, 1);
+            tlb_flush(CPU(cpu), 1);
         }
 #endif
     }
@@ -1936,10 +1930,13 @@
     case POWERPC_MMU_601:
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
 #endif /* defined(TARGET_PPC64) */
+        env->tlb_need_flush = 0;
         tlb_flush(CPU(cpu), 1);
         break;
     default:
@@ -1952,9 +1949,6 @@
 void ppc_tlb_invalidate_one(CPUPPCState *env, target_ulong addr)
 {
 #if !defined(FLUSH_ALL_TLBS)
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
-    CPUState *cs;
-
     addr &= TARGET_PAGE_MASK;
     switch (env->mmu_model) {
     case POWERPC_MMU_SOFT_6xx:
@@ -1964,67 +1958,33 @@
             ppc6xx_tlb_invalidate_virt(env, addr, 1);
         }
         break;
-    case POWERPC_MMU_SOFT_4xx:
-    case POWERPC_MMU_SOFT_4xx_Z:
-        ppc4xx_tlb_invalidate_virt(env, addr, env->spr[SPR_40x_PID]);
-        break;
-    case POWERPC_MMU_REAL:
-        cpu_abort(CPU(cpu), "No TLB for PowerPC 4xx in real mode\n");
-        break;
-    case POWERPC_MMU_MPC8xx:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "MPC8xx MMU model is not implemented\n");
-        break;
-    case POWERPC_MMU_BOOKE:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "BookE MMU model is not implemented\n");
-        break;
-    case POWERPC_MMU_BOOKE206:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "BookE 2.06 MMU model is not implemented\n");
-        break;
     case POWERPC_MMU_32B:
     case POWERPC_MMU_601:
-        /* tlbie invalidate TLBs for all segments */
-        addr &= ~((target_ulong)-1ULL << 28);
-        cs = CPU(cpu);
-        /* XXX: this case should be optimized,
-         * giving a mask to tlb_flush_page
+        /* Actual CPUs invalidate entire congruence classes based on the
+         * geometry of their TLBs and some OSes take that into account,
+         * we just mark the TLB to be flushed later (context synchronizing
+         * event or sync instruction on 32-bit).
          */
-        tlb_flush_page(cs, addr | (0x0 << 28));
-        tlb_flush_page(cs, addr | (0x1 << 28));
-        tlb_flush_page(cs, addr | (0x2 << 28));
-        tlb_flush_page(cs, addr | (0x3 << 28));
-        tlb_flush_page(cs, addr | (0x4 << 28));
-        tlb_flush_page(cs, addr | (0x5 << 28));
-        tlb_flush_page(cs, addr | (0x6 << 28));
-        tlb_flush_page(cs, addr | (0x7 << 28));
-        tlb_flush_page(cs, addr | (0x8 << 28));
-        tlb_flush_page(cs, addr | (0x9 << 28));
-        tlb_flush_page(cs, addr | (0xA << 28));
-        tlb_flush_page(cs, addr | (0xB << 28));
-        tlb_flush_page(cs, addr | (0xC << 28));
-        tlb_flush_page(cs, addr | (0xD << 28));
-        tlb_flush_page(cs, addr | (0xE << 28));
-        tlb_flush_page(cs, addr | (0xF << 28));
+        env->tlb_need_flush = 1;
         break;
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
         /* tlbie invalidate TLBs for all segments */
         /* XXX: given the fact that there are too many segments to invalidate,
          *      and we still don't have a tlb_flush_mask(env, n, mask) in QEMU,
          *      we just invalidate all TLBs
          */
-        tlb_flush(CPU(cpu), 1);
+        env->tlb_need_flush = 1;
         break;
 #endif /* defined(TARGET_PPC64) */
     default:
-        /* XXX: TODO */
-        cpu_abort(CPU(cpu), "Unknown MMU model\n");
-        break;
+        /* Should never reach here with other MMU models */
+        assert(0);
     }
 #else
     ppc_tlb_invalidate_all(env);
@@ -2035,31 +1995,25 @@
 /* Special registers manipulation */
 void ppc_store_sdr1(CPUPPCState *env, target_ulong value)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
-
-    LOG_MMU("%s: " TARGET_FMT_lx "\n", __func__, value);
+    qemu_log_mask(CPU_LOG_MMU, "%s: " TARGET_FMT_lx "\n", __func__, value);
     assert(!env->external_htab);
-    if (env->spr[SPR_SDR1] != value) {
-        env->spr[SPR_SDR1] = value;
+    env->spr[SPR_SDR1] = value;
 #if defined(TARGET_PPC64)
-        if (env->mmu_model & POWERPC_MMU_64) {
-            target_ulong htabsize = value & SDR_64_HTABSIZE;
+    if (env->mmu_model & POWERPC_MMU_64) {
+        PowerPCCPU *cpu = ppc_env_get_cpu(env);
+        Error *local_err = NULL;
 
-            if (htabsize > 28) {
-                fprintf(stderr, "Invalid HTABSIZE 0x" TARGET_FMT_lx
-                        " stored in SDR1\n", htabsize);
-                htabsize = 28;
-            }
-            env->htab_mask = (1ULL << (htabsize + 18 - 7)) - 1;
-            env->htab_base = value & SDR_64_HTABORG;
-        } else
-#endif /* defined(TARGET_PPC64) */
-        {
-            /* FIXME: Should check for valid HTABMASK values */
-            env->htab_mask = ((value & SDR_32_HTABMASK) << 16) | 0xFFFF;
-            env->htab_base = value & SDR_32_HTABORG;
+        ppc_hash64_set_sdr1(cpu, value, &local_err);
+        if (local_err) {
+            error_report_err(local_err);
+            error_free(local_err);
         }
-        tlb_flush(CPU(cpu), 1);
+    } else
+#endif /* defined(TARGET_PPC64) */
+    {
+        /* FIXME: Should check for valid HTABMASK values */
+        env->htab_mask = ((value & SDR_32_HTABMASK) << 16) | 0xFFFF;
+        env->htab_base = value & SDR_32_HTABORG;
     }
 }
 
@@ -2077,27 +2031,23 @@
 
 void helper_store_sr(CPUPPCState *env, target_ulong srnum, target_ulong value)
 {
-    PowerPCCPU *cpu = ppc_env_get_cpu(env);
-
-    LOG_MMU("%s: reg=%d " TARGET_FMT_lx " " TARGET_FMT_lx "\n", __func__,
+    qemu_log_mask(CPU_LOG_MMU,
+            "%s: reg=%d " TARGET_FMT_lx " " TARGET_FMT_lx "\n", __func__,
             (int)srnum, value, env->sr[srnum]);
 #if defined(TARGET_PPC64)
     if (env->mmu_model & POWERPC_MMU_64) {
-        uint64_t rb = 0, rs = 0;
+        PowerPCCPU *cpu = ppc_env_get_cpu(env);
+        uint64_t esid, vsid;
 
         /* ESID = srnum */
-        rb |= ((uint32_t)srnum & 0xf) << 28;
-        /* Set the valid bit */
-        rb |= SLB_ESID_V;
-        /* Index = ESID */
-        rb |= (uint32_t)srnum;
+        esid = ((uint64_t)(srnum & 0xf) << 28) | SLB_ESID_V;
 
         /* VSID = VSID */
-        rs |= (value & 0xfffffff) << 12;
+        vsid = (value & 0xfffffff) << 12;
         /* flags = flags */
-        rs |= ((value >> 27) & 0xf) << 8;
+        vsid |= ((value >> 27) & 0xf) << 8;
 
-        ppc_store_slb(env, rb, rs);
+        ppc_store_slb(cpu, srnum, esid, vsid);
     } else
 #endif
     if (env->sr[srnum] != value) {
@@ -2115,7 +2065,7 @@
             }
         }
 #else
-        tlb_flush(CPU(cpu), 1);
+        env->tlb_need_flush = 1;
 #endif
     }
 }
@@ -2131,6 +2081,16 @@
     ppc_tlb_invalidate_one(env, addr);
 }
 
+void helper_tlbiva(CPUPPCState *env, target_ulong addr)
+{
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    /* tlbiva instruction only exists on BookE */
+    assert(env->mmu_model == POWERPC_MMU_BOOKE);
+    /* XXX: TODO */
+    cpu_abort(CPU(cpu), "BookE MMU model is not implemented\n");
+}
+
 /* Software driven TLBs management */
 /* PowerPC 602/603 software TLB load instructions helpers */
 static void do_6xx_tlb(CPUPPCState *env, target_ulong new_EPN, int is_code)
@@ -2907,14 +2867,19 @@
 }
 
 
+void helper_check_tlb_flush(CPUPPCState *env)
+{
+    check_tlb_flush(env);
+}
+
 /*****************************************************************************/
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
 /* XXX: fix it to restore all registers */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     PowerPCCPU *cpu = POWERPC_CPU(cs);
     PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cs);
@@ -2922,9 +2887,9 @@
     int ret;
 
     if (pcc->handle_mmu_fault) {
-        ret = pcc->handle_mmu_fault(cpu, addr, is_write, mmu_idx);
+        ret = pcc->handle_mmu_fault(cpu, addr, access_type, mmu_idx);
     } else {
-        ret = cpu_ppc_handle_mmu_fault(env, addr, is_write, mmu_idx);
+        ret = cpu_ppc_handle_mmu_fault(env, addr, access_type, mmu_idx);
     }
     if (unlikely(ret != 0)) {
         if (likely(retaddr)) {
diff --git a/target-ppc/monitor.c b/target-ppc/monitor.c
new file mode 100644
index 0000000..c2d0806
--- /dev/null
+++ b/target-ppc/monitor.c
@@ -0,0 +1,147 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+static target_long monitor_get_ccr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    unsigned int u;
+    int i;
+
+    u = 0;
+    for (i = 0; i < 8; i++)
+        u |= env->crf[i] << (32 - (4 * (i + 1)));
+
+    return u;
+}
+
+static target_long monitor_get_decr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_decr(env);
+}
+
+static target_long monitor_get_tbu (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_tbu(env);
+}
+
+static target_long monitor_get_tbl (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return cpu_ppc_load_tbl(env);
+}
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
+
+const MonitorDef monitor_defs[] = {
+    { "fpscr", offsetof(CPUPPCState, fpscr) },
+    /* Next instruction pointer */
+    { "nip|pc", offsetof(CPUPPCState, nip) },
+    { "lr", offsetof(CPUPPCState, lr) },
+    { "ctr", offsetof(CPUPPCState, ctr) },
+    { "decr", 0, &monitor_get_decr, },
+    { "ccr|cr", 0, &monitor_get_ccr, },
+    /* Machine state register */
+    { "xer", offsetof(CPUPPCState, xer) },
+    { "msr", offsetof(CPUPPCState, msr) },
+    { "tbu", 0, &monitor_get_tbu, },
+    { "tbl", 0, &monitor_get_tbl, },
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
+
+static int ppc_cpu_get_reg_num(const char *numstr, int maxnum, int *pregnum)
+{
+    int regnum;
+    char *endptr = NULL;
+
+    if (!*numstr) {
+        return false;
+    }
+
+    regnum = strtoul(numstr, &endptr, 10);
+    if (*endptr || (regnum >= maxnum)) {
+        return false;
+    }
+    *pregnum = regnum;
+
+    return true;
+}
+
+int target_get_monitor_def(CPUState *cs, const char *name, uint64_t *pval)
+{
+    int i, regnum;
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    /* General purpose registers */
+    if ((tolower(name[0]) == 'r') &&
+        ppc_cpu_get_reg_num(name + 1, ARRAY_SIZE(env->gpr), &regnum)) {
+        *pval = env->gpr[regnum];
+        return 0;
+    }
+
+    /* Floating point registers */
+    if ((tolower(name[0]) == 'f') &&
+        ppc_cpu_get_reg_num(name + 1, ARRAY_SIZE(env->fpr), &regnum)) {
+        *pval = env->fpr[regnum];
+        return 0;
+    }
+
+    /* Special purpose registers */
+    for (i = 0; i < ARRAY_SIZE(env->spr_cb); ++i) {
+        ppc_spr_t *spr = &env->spr_cb[i];
+
+        if (spr->name && (strcasecmp(name, spr->name) == 0)) {
+            *pval = env->spr[i];
+            return 0;
+        }
+    }
+
+    /* Segment registers */
+#if !defined(CONFIG_USER_ONLY)
+    if ((strncasecmp(name, "sr", 2) == 0) &&
+        ppc_cpu_get_reg_num(name + 2, ARRAY_SIZE(env->sr), &regnum)) {
+        *pval = env->sr[regnum];
+        return 0;
+    }
+#endif
+
+    return -EINVAL;
+}
diff --git a/target-ppc/timebase_helper.c b/target-ppc/timebase_helper.c
index 865dcbe..a07faa4 100644
--- a/target-ppc/timebase_helper.c
+++ b/target-ppc/timebase_helper.c
@@ -16,8 +16,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "qemu/log.h"
 
 /*****************************************************************************/
 /* SPR accesses */
@@ -100,6 +102,16 @@
     cpu_ppc_store_decr(env, val);
 }
 
+target_ulong helper_load_hdecr(CPUPPCState *env)
+{
+    return cpu_ppc_load_hdecr(env);
+}
+
+void helper_store_hdecr(CPUPPCState *env, target_ulong val)
+{
+    cpu_ppc_store_hdecr(env, val);
+}
+
 target_ulong helper_load_40x_pit(CPUPPCState *env)
 {
     return load_40x_pit(env);
@@ -130,13 +142,14 @@
     uint32_t val = 0;
 
     if (unlikely(env->dcr_env == NULL)) {
-        qemu_log("No DCR environment\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "No DCR environment\n");
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL |
                                    POWERPC_EXCP_INVAL_INVAL);
     } else if (unlikely(ppc_dcr_read(env->dcr_env,
                                      (uint32_t)dcrn, &val) != 0)) {
-        qemu_log("DCR read error %d %03x\n", (uint32_t)dcrn, (uint32_t)dcrn);
+        qemu_log_mask(LOG_GUEST_ERROR, "DCR read error %d %03x\n",
+                      (uint32_t)dcrn, (uint32_t)dcrn);
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL | POWERPC_EXCP_PRIV_REG);
     }
@@ -146,13 +159,14 @@
 void helper_store_dcr(CPUPPCState *env, target_ulong dcrn, target_ulong val)
 {
     if (unlikely(env->dcr_env == NULL)) {
-        qemu_log("No DCR environment\n");
+        qemu_log_mask(LOG_GUEST_ERROR, "No DCR environment\n");
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL |
                                    POWERPC_EXCP_INVAL_INVAL);
     } else if (unlikely(ppc_dcr_write(env->dcr_env, (uint32_t)dcrn,
                                       (uint32_t)val) != 0)) {
-        qemu_log("DCR write error %d %03x\n", (uint32_t)dcrn, (uint32_t)dcrn);
+        qemu_log_mask(LOG_GUEST_ERROR, "DCR write error %d %03x\n",
+                      (uint32_t)dcrn, (uint32_t)dcrn);
         helper_raise_exception_err(env, POWERPC_EXCP_PROGRAM,
                                    POWERPC_EXCP_INVAL | POWERPC_EXCP_PRIV_REG);
     }
diff --git a/target-ppc/trace-events b/target-ppc/trace-events
new file mode 100644
index 0000000..8fcc3ce
--- /dev/null
+++ b/target-ppc/trace-events
@@ -0,0 +1,5 @@
+# See docs/tracing.txt for syntax documentation.
+
+# target-ppc/kvm.c
+kvm_failed_spr_set(int str, const char *msg) "Warning: Unable to set SPR %d to KVM: %s"
+kvm_failed_spr_get(int str, const char *msg) "Warning: Unable to retrieve SPR %d from KVM: %s"
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index 344cb5d..92030b6 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -18,8 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "qemu/host-utils.h"
 #include "exec/cpu_ldst.h"
@@ -28,6 +30,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 #define CPU_SINGLE_STEP 0x1
@@ -47,7 +50,7 @@
 /* Code translation helpers                                                  */
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static char cpu_reg_names[10*3 + 22*4 /* GPR */
     + 10*4 + 22*5 /* SPE GPRh */
     + 10*4 + 22*5 /* FPR */
@@ -85,13 +88,14 @@
         return;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
     p = cpu_reg_names;
     cpu_reg_names_size = sizeof(cpu_reg_names);
 
     for (i = 0; i < 8; i++) {
         snprintf(p, cpu_reg_names_size, "crf%d", i);
-        cpu_crf[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_crf[i] = tcg_global_mem_new_i32(cpu_env,
                                             offsetof(CPUPPCState, crf[i]), p);
         p += 5;
         cpu_reg_names_size -= 5;
@@ -99,28 +103,28 @@
 
     for (i = 0; i < 32; i++) {
         snprintf(p, cpu_reg_names_size, "r%d", i);
-        cpu_gpr[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr[i] = tcg_global_mem_new(cpu_env,
                                         offsetof(CPUPPCState, gpr[i]), p);
         p += (i < 10) ? 3 : 4;
         cpu_reg_names_size -= (i < 10) ? 3 : 4;
         snprintf(p, cpu_reg_names_size, "r%dH", i);
-        cpu_gprh[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gprh[i] = tcg_global_mem_new(cpu_env,
                                          offsetof(CPUPPCState, gprh[i]), p);
         p += (i < 10) ? 4 : 5;
         cpu_reg_names_size -= (i < 10) ? 4 : 5;
 
         snprintf(p, cpu_reg_names_size, "fp%d", i);
-        cpu_fpr[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_fpr[i] = tcg_global_mem_new_i64(cpu_env,
                                             offsetof(CPUPPCState, fpr[i]), p);
         p += (i < 10) ? 4 : 5;
         cpu_reg_names_size -= (i < 10) ? 4 : 5;
 
         snprintf(p, cpu_reg_names_size, "avr%dH", i);
 #ifdef HOST_WORDS_BIGENDIAN
-        cpu_avrh[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrh[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[0]), p);
 #else
-        cpu_avrh[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrh[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[1]), p);
 #endif
         p += (i < 10) ? 6 : 7;
@@ -128,86 +132,87 @@
 
         snprintf(p, cpu_reg_names_size, "avr%dL", i);
 #ifdef HOST_WORDS_BIGENDIAN
-        cpu_avrl[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrl[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[1]), p);
 #else
-        cpu_avrl[i] = tcg_global_mem_new_i64(TCG_AREG0,
+        cpu_avrl[i] = tcg_global_mem_new_i64(cpu_env,
                                              offsetof(CPUPPCState, avr[i].u64[0]), p);
 #endif
         p += (i < 10) ? 6 : 7;
         cpu_reg_names_size -= (i < 10) ? 6 : 7;
         snprintf(p, cpu_reg_names_size, "vsr%d", i);
-        cpu_vsr[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                             offsetof(CPUPPCState, vsr[i]), p);
+        cpu_vsr[i] = tcg_global_mem_new_i64(cpu_env,
+                                            offsetof(CPUPPCState, vsr[i]), p);
         p += (i < 10) ? 5 : 6;
         cpu_reg_names_size -= (i < 10) ? 5 : 6;
     }
 
-    cpu_nip = tcg_global_mem_new(TCG_AREG0,
+    cpu_nip = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, nip), "nip");
 
-    cpu_msr = tcg_global_mem_new(TCG_AREG0,
+    cpu_msr = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, msr), "msr");
 
-    cpu_ctr = tcg_global_mem_new(TCG_AREG0,
+    cpu_ctr = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, ctr), "ctr");
 
-    cpu_lr = tcg_global_mem_new(TCG_AREG0,
+    cpu_lr = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, lr), "lr");
 
 #if defined(TARGET_PPC64)
-    cpu_cfar = tcg_global_mem_new(TCG_AREG0,
+    cpu_cfar = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUPPCState, cfar), "cfar");
 #endif
 
-    cpu_xer = tcg_global_mem_new(TCG_AREG0,
+    cpu_xer = tcg_global_mem_new(cpu_env,
                                  offsetof(CPUPPCState, xer), "xer");
-    cpu_so = tcg_global_mem_new(TCG_AREG0,
+    cpu_so = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, so), "SO");
-    cpu_ov = tcg_global_mem_new(TCG_AREG0,
+    cpu_ov = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, ov), "OV");
-    cpu_ca = tcg_global_mem_new(TCG_AREG0,
+    cpu_ca = tcg_global_mem_new(cpu_env,
                                 offsetof(CPUPPCState, ca), "CA");
 
-    cpu_reserve = tcg_global_mem_new(TCG_AREG0,
+    cpu_reserve = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUPPCState, reserve_addr),
                                      "reserve_addr");
 
-    cpu_fpscr = tcg_global_mem_new(TCG_AREG0,
+    cpu_fpscr = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUPPCState, fpscr), "fpscr");
 
-    cpu_access_type = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_access_type = tcg_global_mem_new_i32(cpu_env,
                                              offsetof(CPUPPCState, access_type), "access_type");
 
     done_init = 1;
 }
 
 /* internal defines */
-typedef struct DisasContext {
+struct DisasContext {
     struct TranslationBlock *tb;
     target_ulong nip;
     uint32_t opcode;
     uint32_t exception;
     /* Routine used to access memory */
-    bool pr, hv;
+    bool pr, hv, dr, le_mode;
+    bool lazy_tlb_flush;
     int mem_idx;
     int access_type;
     /* Translation flags */
-    int le_mode;
     TCGMemOp default_tcg_memop_mask;
 #if defined(TARGET_PPC64)
-    int sf_mode;
-    int has_cfar;
+    bool sf_mode;
+    bool has_cfar;
 #endif
-    int fpu_enabled;
-    int altivec_enabled;
-    int vsx_enabled;
-    int spe_enabled;
+    bool fpu_enabled;
+    bool altivec_enabled;
+    bool vsx_enabled;
+    bool spe_enabled;
+    bool tm_enabled;
     ppc_spr_t *spr_cb; /* Needed to check rights for mfspr/mtspr */
     int singlestep_enabled;
     uint64_t insns_flags;
     uint64_t insns_flags2;
-} DisasContext;
+};
 
 /* Return true iff byteswap is needed in a scalar memop */
 static inline bool need_byteswap(const DisasContext *ctx)
@@ -250,26 +255,10 @@
     gen_helper_reset_fpstatus(cpu_env);
 }
 
-static inline void gen_compute_fprf(TCGv_i64 arg, int set_fprf, int set_rc)
+static inline void gen_compute_fprf(TCGv_i64 arg)
 {
-    TCGv_i32 t0 = tcg_temp_new_i32();
-
-    if (set_fprf != 0) {
-        /* This case might be optimized later */
-        tcg_gen_movi_i32(t0, 1);
-        gen_helper_compute_fprf(t0, cpu_env, arg, t0);
-        if (unlikely(set_rc)) {
-            tcg_gen_mov_i32(cpu_crf[1], t0);
-        }
-        gen_helper_float_check_status(cpu_env);
-    } else if (unlikely(set_rc)) {
-        /* We always need to compute fpcc */
-        tcg_gen_movi_i32(t0, 0);
-        gen_helper_compute_fprf(t0, cpu_env, arg, t0);
-        tcg_gen_mov_i32(cpu_crf[1], t0);
-    }
-
-    tcg_temp_free_i32(t0);
+    gen_helper_compute_fprf(cpu_env, arg);
+    gen_helper_float_check_status(cpu_env);
 }
 
 static inline void gen_set_access_type(DisasContext *ctx, int access_type)
@@ -295,7 +284,7 @@
     tcg_gen_movi_tl(cpu_nip, ctx->nip);
 }
 
-static inline void gen_exception_err(DisasContext *ctx, uint32_t excp, uint32_t error)
+static void gen_exception_err(DisasContext *ctx, uint32_t excp, uint32_t error)
 {
     TCGv_i32 t0, t1;
     if (ctx->exception == POWERPC_EXCP_NONE) {
@@ -309,7 +298,7 @@
     ctx->exception = (excp);
 }
 
-static inline void gen_exception(DisasContext *ctx, uint32_t excp)
+static void gen_exception(DisasContext *ctx, uint32_t excp)
 {
     TCGv_i32 t0;
     if (ctx->exception == POWERPC_EXCP_NONE) {
@@ -321,7 +310,7 @@
     ctx->exception = (excp);
 }
 
-static inline void gen_debug_exception(DisasContext *ctx)
+static void gen_debug_exception(DisasContext *ctx)
 {
     TCGv_i32 t0;
 
@@ -336,7 +325,19 @@
 
 static inline void gen_inval_exception(DisasContext *ctx, uint32_t error)
 {
-    gen_exception_err(ctx, POWERPC_EXCP_PROGRAM, POWERPC_EXCP_INVAL | error);
+    /* Will be converted to program check if needed */
+    gen_exception_err(ctx, POWERPC_EXCP_HV_EMU, POWERPC_EXCP_INVAL | error);
+}
+
+static inline void gen_priv_exception(DisasContext *ctx, uint32_t error)
+{
+    gen_exception_err(ctx, POWERPC_EXCP_PROGRAM, POWERPC_EXCP_PRIV | error);
+}
+
+static inline void gen_hvpriv_exception(DisasContext *ctx, uint32_t error)
+{
+    /* Will be converted to program check if needed */
+    gen_exception_err(ctx, POWERPC_EXCP_HV_EMU, POWERPC_EXCP_PRIV | error);
 }
 
 /* Stop translation */
@@ -346,11 +347,13 @@
     ctx->exception = POWERPC_EXCP_STOP;
 }
 
+#ifndef CONFIG_USER_ONLY
 /* No need to update nip here, as execution flow will change */
 static inline void gen_sync_exception(DisasContext *ctx)
 {
     ctx->exception = POWERPC_EXCP_SYNC;
 }
+#endif
 
 #define GEN_HANDLER(name, opc1, opc2, opc3, inval, type)                      \
 GEN_OPCODE(name, opc1, opc2, opc3, inval, type, PPC_NONE)
@@ -375,6 +378,40 @@
     const char *oname;
 } opcode_t;
 
+/* Helpers for priv. check */
+#define GEN_PRIV                                                \
+    do {                                                        \
+        gen_priv_exception(ctx, POWERPC_EXCP_PRIV_OPC); return; \
+    } while (0)
+
+#if defined(CONFIG_USER_ONLY)
+#define CHK_HV GEN_PRIV
+#define CHK_SV GEN_PRIV
+#define CHK_HVRM GEN_PRIV
+#else
+#define CHK_HV                                                          \
+    do {                                                                \
+        if (unlikely(ctx->pr || !ctx->hv)) {                            \
+            GEN_PRIV;                                                   \
+        }                                                               \
+    } while (0)
+#define CHK_SV                   \
+    do {                         \
+        if (unlikely(ctx->pr)) { \
+            GEN_PRIV;            \
+        }                        \
+    } while (0)
+#define CHK_HVRM                                            \
+    do {                                                    \
+        if (unlikely(ctx->pr || !ctx->hv || ctx->dr)) {     \
+            GEN_PRIV;                                       \
+        }                                                   \
+    } while (0)
+#endif
+
+#define CHK_NONE
+
+
 /*****************************************************************************/
 /***                           Instruction decoding                        ***/
 #define EXTRACT_HELPER(name, shift, nb)                                       \
@@ -452,7 +489,10 @@
 EXTRACT_HELPER(TO, 21, 5);
 
 EXTRACT_HELPER(CRM, 12, 8);
+
+#ifndef CONFIG_USER_ONLY
 EXTRACT_HELPER(SR, 16, 4);
+#endif
 
 /* mtfsf/mtfsfi */
 EXTRACT_HELPER(FPBF, 23, 3);
@@ -763,27 +803,20 @@
 /* isel (PowerPC 2.03 specification) */
 static void gen_isel(DisasContext *ctx)
 {
-    int l1, l2;
     uint32_t bi = rC(ctx->opcode);
-    uint32_t mask;
-    TCGv_i32 t0;
+    uint32_t mask = 0x08 >> (bi & 0x03);
+    TCGv t0 = tcg_temp_new();
+    TCGv zr;
 
-    l1 = gen_new_label();
-    l2 = gen_new_label();
+    tcg_gen_extu_i32_tl(t0, cpu_crf[bi >> 2]);
+    tcg_gen_andi_tl(t0, t0, mask);
 
-    mask = 0x08 >> (bi & 0x03);
-    t0 = tcg_temp_new_i32();
-    tcg_gen_andi_i32(t0, cpu_crf[bi >> 2], mask);
-    tcg_gen_brcondi_i32(TCG_COND_EQ, t0, 0, l1);
-    if (rA(ctx->opcode) == 0)
-        tcg_gen_movi_tl(cpu_gpr[rD(ctx->opcode)], 0);
-    else
-        tcg_gen_mov_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);
-    tcg_gen_br(l2);
-    gen_set_label(l1);
-    tcg_gen_mov_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rB(ctx->opcode)]);
-    gen_set_label(l2);
-    tcg_temp_free_i32(t0);
+    zr = tcg_const_tl(0);
+    tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rD(ctx->opcode)], t0, zr,
+                       rA(ctx->opcode) ? cpu_gpr[rA(ctx->opcode)] : zr,
+                       cpu_gpr[rB(ctx->opcode)]);
+    tcg_temp_free(zr);
+    tcg_temp_free(t0);
 }
 
 /* cmpb: PowerPC 2.05 specification */
@@ -954,8 +987,8 @@
 static inline void gen_op_arith_divw(DisasContext *ctx, TCGv ret, TCGv arg1,
                                      TCGv arg2, int sign, int compute_ov)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     TCGv_i32 t0 = tcg_temp_local_new_i32();
     TCGv_i32 t1 = tcg_temp_local_new_i32();
 
@@ -963,7 +996,7 @@
     tcg_gen_trunc_tl_i32(t1, arg2);
     tcg_gen_brcondi_i32(TCG_COND_EQ, t1, 0, l1);
     if (sign) {
-        int l3 = gen_new_label();
+        TCGLabel *l3 = gen_new_label();
         tcg_gen_brcondi_i32(TCG_COND_NE, t1, -1, l3);
         tcg_gen_brcondi_i32(TCG_COND_EQ, t0, INT32_MIN, l1);
         gen_set_label(l3);
@@ -1029,12 +1062,12 @@
 static inline void gen_op_arith_divd(DisasContext *ctx, TCGv ret, TCGv arg1,
                                      TCGv arg2, int sign, int compute_ov)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
 
     tcg_gen_brcondi_i64(TCG_COND_EQ, arg2, 0, l1);
     if (sign) {
-        int l3 = gen_new_label();
+        TCGLabel *l3 = gen_new_label();
         tcg_gen_brcondi_i64(TCG_COND_NE, arg2, -1, l3);
         tcg_gen_brcondi_i64(TCG_COND_EQ, arg1, INT64_MIN, l1);
         gen_set_label(l3);
@@ -1406,6 +1439,19 @@
 /* nor & nor. */
 GEN_LOGICAL2(nor, tcg_gen_nor_tl, 0x03, PPC_INTEGER);
 
+#if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
+static void gen_pause(DisasContext *ctx)
+{
+    TCGv_i32 t0 = tcg_const_i32(0);
+    tcg_gen_st_i32(t0, cpu_env,
+                   -offsetof(PowerPCCPU, env) + offsetof(CPUState, halted));
+    tcg_temp_free_i32(t0);
+
+    /* Stop translation, this gives other CPUs a chance to run */
+    gen_exception_err(ctx, EXCP_HLT, 1);
+}
+#endif /* defined(TARGET_PPC64) */
+
 /* or & or. */
 static void gen_or(DisasContext *ctx)
 {
@@ -1425,7 +1471,7 @@
     } else if (unlikely(Rc(ctx->opcode) != 0)) {
         gen_set_Rc0(ctx, cpu_gpr[rs]);
 #if defined(TARGET_PPC64)
-    } else {
+    } else if (rs != 0) { /* 0 is nop */
         int prio = 0;
 
         switch (rs) {
@@ -1461,14 +1507,13 @@
             }
             break;
         case 7:
-            if (ctx->hv) {
+            if (ctx->hv && !ctx->pr) {
                 /* Set process priority to very high */
                 prio = 7;
             }
             break;
 #endif
         default:
-            /* nop */
             break;
         }
         if (prio) {
@@ -1479,6 +1524,14 @@
             gen_store_spr(SPR_PPR, t0);
             tcg_temp_free(t0);
         }
+#if !defined(CONFIG_USER_ONLY)
+        /* Pause out of TCG otherwise spin loops with smt_low eat too much
+         * CPU and the kernel hangs.  This applies to all encodings other
+         * than no-op, e.g., miso(rs=26), yield(27), mdoio(29), mdoom(30),
+         * and all currently undefined.
+         */
+        gen_pause(ctx);
+#endif
 #endif
     }
 }
@@ -1503,8 +1556,6 @@
     target_ulong uimm = UIMM(ctx->opcode);
 
     if (rS(ctx->opcode) == rA(ctx->opcode) && uimm == 0) {
-        /* NOP */
-        /* XXX: should handle special NOPs for POWER series */
         return;
     }
     tcg_gen_ori_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rS(ctx->opcode)], uimm);
@@ -1624,141 +1675,138 @@
 /* rlwimi & rlwimi. */
 static void gen_rlwimi(DisasContext *ctx)
 {
-    uint32_t mb, me, sh;
+    TCGv t_ra = cpu_gpr[rA(ctx->opcode)];
+    TCGv t_rs = cpu_gpr[rS(ctx->opcode)];
+    uint32_t sh = SH(ctx->opcode);
+    uint32_t mb = MB(ctx->opcode);
+    uint32_t me = ME(ctx->opcode);
 
-    mb = MB(ctx->opcode);
-    me = ME(ctx->opcode);
-    sh = SH(ctx->opcode);
-    if (likely(sh == (31-me) && mb <= me)) {
-        tcg_gen_deposit_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rA(ctx->opcode)],
-                           cpu_gpr[rS(ctx->opcode)], sh, me - mb + 1);
+    if (sh == (31-me) && mb <= me) {
+        tcg_gen_deposit_tl(t_ra, t_ra, t_rs, sh, me - mb + 1);
     } else {
         target_ulong mask;
         TCGv t1;
-        TCGv t0 = tcg_temp_new();
-#if defined(TARGET_PPC64)
-        tcg_gen_deposit_i64(t0, cpu_gpr[rS(ctx->opcode)],
-            cpu_gpr[rS(ctx->opcode)], 32, 32);
-        tcg_gen_rotli_i64(t0, t0, sh);
-#else
-        tcg_gen_rotli_i32(t0, cpu_gpr[rS(ctx->opcode)], sh);
-#endif
+
 #if defined(TARGET_PPC64)
         mb += 32;
         me += 32;
 #endif
         mask = MASK(mb, me);
+
         t1 = tcg_temp_new();
-        tcg_gen_andi_tl(t0, t0, mask);
-        tcg_gen_andi_tl(t1, cpu_gpr[rA(ctx->opcode)], ~mask);
-        tcg_gen_or_tl(cpu_gpr[rA(ctx->opcode)], t0, t1);
-        tcg_temp_free(t0);
+        if (mask <= 0xffffffffu) {
+            TCGv_i32 t0 = tcg_temp_new_i32();
+            tcg_gen_trunc_tl_i32(t0, t_rs);
+            tcg_gen_rotli_i32(t0, t0, sh);
+            tcg_gen_extu_i32_tl(t1, t0);
+            tcg_temp_free_i32(t0);
+        } else {
+#if defined(TARGET_PPC64)
+            tcg_gen_deposit_i64(t1, t_rs, t_rs, 32, 32);
+            tcg_gen_rotli_i64(t1, t1, sh);
+#else
+            g_assert_not_reached();
+#endif
+        }
+
+        tcg_gen_andi_tl(t1, t1, mask);
+        tcg_gen_andi_tl(t_ra, t_ra, ~mask);
+        tcg_gen_or_tl(t_ra, t_ra, t1);
         tcg_temp_free(t1);
     }
-    if (unlikely(Rc(ctx->opcode) != 0))
-        gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_Rc0(ctx, t_ra);
+    }
 }
 
 /* rlwinm & rlwinm. */
 static void gen_rlwinm(DisasContext *ctx)
 {
-    uint32_t mb, me, sh;
+    TCGv t_ra = cpu_gpr[rA(ctx->opcode)];
+    TCGv t_rs = cpu_gpr[rS(ctx->opcode)];
+    uint32_t sh = SH(ctx->opcode);
+    uint32_t mb = MB(ctx->opcode);
+    uint32_t me = ME(ctx->opcode);
 
-    sh = SH(ctx->opcode);
-    mb = MB(ctx->opcode);
-    me = ME(ctx->opcode);
-
-    if (likely(mb == 0 && me == (31 - sh))) {
-        if (likely(sh == 0)) {
-            tcg_gen_ext32u_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rS(ctx->opcode)]);
-        } else {
-            TCGv t0 = tcg_temp_new();
-            tcg_gen_ext32u_tl(t0, cpu_gpr[rS(ctx->opcode)]);
-            tcg_gen_shli_tl(t0, t0, sh);
-            tcg_gen_ext32u_tl(cpu_gpr[rA(ctx->opcode)], t0);
-            tcg_temp_free(t0);
-        }
-    } else if (likely(sh != 0 && me == 31 && sh == (32 - mb))) {
-        TCGv t0 = tcg_temp_new();
-        tcg_gen_ext32u_tl(t0, cpu_gpr[rS(ctx->opcode)]);
-        tcg_gen_shri_tl(t0, t0, mb);
-        tcg_gen_ext32u_tl(cpu_gpr[rA(ctx->opcode)], t0);
-        tcg_temp_free(t0);
-    } else if (likely(mb == 0 && me == 31)) {
-        TCGv_i32 t0 = tcg_temp_new_i32();
-        tcg_gen_trunc_tl_i32(t0, cpu_gpr[rS(ctx->opcode)]);
-        tcg_gen_rotli_i32(t0, t0, sh);
-        tcg_gen_extu_i32_tl(cpu_gpr[rA(ctx->opcode)], t0);
-        tcg_temp_free_i32(t0);
+    if (mb == 0 && me == (31 - sh)) {
+        tcg_gen_shli_tl(t_ra, t_rs, sh);
+        tcg_gen_ext32u_tl(t_ra, t_ra);
+    } else if (sh != 0 && me == 31 && sh == (32 - mb)) {
+        tcg_gen_ext32u_tl(t_ra, t_rs);
+        tcg_gen_shri_tl(t_ra, t_ra, mb);
     } else {
-        TCGv t0 = tcg_temp_new();
-#if defined(TARGET_PPC64)
-        tcg_gen_deposit_i64(t0, cpu_gpr[rS(ctx->opcode)],
-            cpu_gpr[rS(ctx->opcode)], 32, 32);
-        tcg_gen_rotli_i64(t0, t0, sh);
-#else
-        tcg_gen_rotli_i32(t0, cpu_gpr[rS(ctx->opcode)], sh);
-#endif
+        target_ulong mask;
 #if defined(TARGET_PPC64)
         mb += 32;
         me += 32;
 #endif
-        tcg_gen_andi_tl(cpu_gpr[rA(ctx->opcode)], t0, MASK(mb, me));
-        tcg_temp_free(t0);
+        mask = MASK(mb, me);
+
+        if (mask <= 0xffffffffu) {
+            TCGv_i32 t0 = tcg_temp_new_i32();
+            tcg_gen_trunc_tl_i32(t0, t_rs);
+            tcg_gen_rotli_i32(t0, t0, sh);
+            tcg_gen_andi_i32(t0, t0, mask);
+            tcg_gen_extu_i32_tl(t_ra, t0);
+            tcg_temp_free_i32(t0);
+        } else {
+#if defined(TARGET_PPC64)
+            tcg_gen_deposit_i64(t_ra, t_rs, t_rs, 32, 32);
+            tcg_gen_rotli_i64(t_ra, t_ra, sh);
+            tcg_gen_andi_i64(t_ra, t_ra, mask);
+#else
+            g_assert_not_reached();
+#endif
+        }
     }
-    if (unlikely(Rc(ctx->opcode) != 0))
-        gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_Rc0(ctx, t_ra);
+    }
 }
 
 /* rlwnm & rlwnm. */
 static void gen_rlwnm(DisasContext *ctx)
 {
-    uint32_t mb, me;
-    mb = MB(ctx->opcode);
-    me = ME(ctx->opcode);
+    TCGv t_ra = cpu_gpr[rA(ctx->opcode)];
+    TCGv t_rs = cpu_gpr[rS(ctx->opcode)];
+    TCGv t_rb = cpu_gpr[rB(ctx->opcode)];
+    uint32_t mb = MB(ctx->opcode);
+    uint32_t me = ME(ctx->opcode);
+    target_ulong mask;
 
-    if (likely(mb == 0 && me == 31)) {
-        TCGv_i32 t0, t1;
-        t0 = tcg_temp_new_i32();
-        t1 = tcg_temp_new_i32();
-        tcg_gen_trunc_tl_i32(t0, cpu_gpr[rB(ctx->opcode)]);
-        tcg_gen_trunc_tl_i32(t1, cpu_gpr[rS(ctx->opcode)]);
+#if defined(TARGET_PPC64)
+    mb += 32;
+    me += 32;
+#endif
+    mask = MASK(mb, me);
+
+    if (mask <= 0xffffffffu) {
+        TCGv_i32 t0 = tcg_temp_new_i32();
+        TCGv_i32 t1 = tcg_temp_new_i32();
+        tcg_gen_trunc_tl_i32(t0, t_rb);
+        tcg_gen_trunc_tl_i32(t1, t_rs);
         tcg_gen_andi_i32(t0, t0, 0x1f);
         tcg_gen_rotl_i32(t1, t1, t0);
-        tcg_gen_extu_i32_tl(cpu_gpr[rA(ctx->opcode)], t1);
+        tcg_gen_extu_i32_tl(t_ra, t1);
         tcg_temp_free_i32(t0);
         tcg_temp_free_i32(t1);
     } else {
-        TCGv t0;
 #if defined(TARGET_PPC64)
-        TCGv t1;
-#endif
-
-        t0 = tcg_temp_new();
-        tcg_gen_andi_tl(t0, cpu_gpr[rB(ctx->opcode)], 0x1f);
-#if defined(TARGET_PPC64)
-        t1 = tcg_temp_new_i64();
-        tcg_gen_deposit_i64(t1, cpu_gpr[rS(ctx->opcode)],
-                            cpu_gpr[rS(ctx->opcode)], 32, 32);
-        tcg_gen_rotl_i64(t0, t1, t0);
-        tcg_temp_free_i64(t1);
+        TCGv_i64 t0 = tcg_temp_new_i64();
+        tcg_gen_andi_i64(t0, t_rb, 0x1f);
+        tcg_gen_deposit_i64(t_ra, t_rs, t_rs, 32, 32);
+        tcg_gen_rotl_i64(t_ra, t_ra, t0);
+        tcg_temp_free_i64(t0);
 #else
-        tcg_gen_rotl_i32(t0, cpu_gpr[rS(ctx->opcode)], t0);
+        g_assert_not_reached();
 #endif
-        if (unlikely(mb != 0 || me != 31)) {
-#if defined(TARGET_PPC64)
-            mb += 32;
-            me += 32;
-#endif
-            tcg_gen_andi_tl(cpu_gpr[rA(ctx->opcode)], t0, MASK(mb, me));
-        } else {
-            tcg_gen_andi_tl(t0, t0, MASK(32, 63));
-            tcg_gen_mov_tl(cpu_gpr[rA(ctx->opcode)], t0);
-        }
-        tcg_temp_free(t0);
     }
-    if (unlikely(Rc(ctx->opcode) != 0))
-        gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
+
+    tcg_gen_andi_tl(t_ra, t_ra, mask);
+
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_Rc0(ctx, t_ra);
+    }
 }
 
 #if defined(TARGET_PPC64)
@@ -1793,26 +1841,24 @@
     gen_##name(ctx, 1, 1);                                                    \
 }
 
-static inline void gen_rldinm(DisasContext *ctx, uint32_t mb, uint32_t me,
-                              uint32_t sh)
+static void gen_rldinm(DisasContext *ctx, int mb, int me, int sh)
 {
-    if (likely(sh != 0 && mb == 0 && me == (63 - sh))) {
-        tcg_gen_shli_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rS(ctx->opcode)], sh);
-    } else if (likely(sh != 0 && me == 63 && sh == (64 - mb))) {
-        tcg_gen_shri_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rS(ctx->opcode)], mb);
+    TCGv t_ra = cpu_gpr[rA(ctx->opcode)];
+    TCGv t_rs = cpu_gpr[rS(ctx->opcode)];
+
+    if (sh != 0 && mb == 0 && me == (63 - sh)) {
+        tcg_gen_shli_tl(t_ra, t_rs, sh);
+    } else if (sh != 0 && me == 63 && sh == (64 - mb)) {
+        tcg_gen_shri_tl(t_ra, t_rs, mb);
     } else {
-        TCGv t0 = tcg_temp_new();
-        tcg_gen_rotli_tl(t0, cpu_gpr[rS(ctx->opcode)], sh);
-        if (likely(mb == 0 && me == 63)) {
-            tcg_gen_mov_tl(cpu_gpr[rA(ctx->opcode)], t0);
-        } else {
-            tcg_gen_andi_tl(cpu_gpr[rA(ctx->opcode)], t0, MASK(mb, me));
-        }
-        tcg_temp_free(t0);
+        tcg_gen_rotli_tl(t_ra, t_rs, sh);
+        tcg_gen_andi_tl(t_ra, t_ra, MASK(mb, me));
     }
-    if (unlikely(Rc(ctx->opcode) != 0))
-        gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_Rc0(ctx, t_ra);
+    }
 }
+
 /* rldicl - rldicl. */
 static inline void gen_rldicl(DisasContext *ctx, int mbn, int shn)
 {
@@ -1823,6 +1869,7 @@
     gen_rldinm(ctx, mb, 63, sh);
 }
 GEN_PPC64_R4(rldicl, 0x1E, 0x00);
+
 /* rldicr - rldicr. */
 static inline void gen_rldicr(DisasContext *ctx, int men, int shn)
 {
@@ -1833,6 +1880,7 @@
     gen_rldinm(ctx, 0, me, sh);
 }
 GEN_PPC64_R4(rldicr, 0x1E, 0x02);
+
 /* rldic - rldic. */
 static inline void gen_rldic(DisasContext *ctx, int mbn, int shn)
 {
@@ -1844,21 +1892,22 @@
 }
 GEN_PPC64_R4(rldic, 0x1E, 0x04);
 
-static inline void gen_rldnm(DisasContext *ctx, uint32_t mb, uint32_t me)
+static void gen_rldnm(DisasContext *ctx, int mb, int me)
 {
+    TCGv t_ra = cpu_gpr[rA(ctx->opcode)];
+    TCGv t_rs = cpu_gpr[rS(ctx->opcode)];
+    TCGv t_rb = cpu_gpr[rB(ctx->opcode)];
     TCGv t0;
 
     t0 = tcg_temp_new();
-    tcg_gen_andi_tl(t0, cpu_gpr[rB(ctx->opcode)], 0x3f);
-    tcg_gen_rotl_tl(t0, cpu_gpr[rS(ctx->opcode)], t0);
-    if (unlikely(mb != 0 || me != 63)) {
-        tcg_gen_andi_tl(cpu_gpr[rA(ctx->opcode)], t0, MASK(mb, me));
-    } else {
-        tcg_gen_mov_tl(cpu_gpr[rA(ctx->opcode)], t0);
-    }
+    tcg_gen_andi_tl(t0, t_rb, 0x3f);
+    tcg_gen_rotl_tl(t_ra, t_rs, t0);
     tcg_temp_free(t0);
-    if (unlikely(Rc(ctx->opcode) != 0))
-        gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
+
+    tcg_gen_andi_tl(t_ra, t_ra, MASK(mb, me));
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_Rc0(ctx, t_ra);
+    }
 }
 
 /* rldcl - rldcl. */
@@ -1870,6 +1919,7 @@
     gen_rldnm(ctx, mb, 63);
 }
 GEN_PPC64_R2(rldcl, 0x1E, 0x08);
+
 /* rldcr - rldcr. */
 static inline void gen_rldcr(DisasContext *ctx, int men)
 {
@@ -1879,32 +1929,31 @@
     gen_rldnm(ctx, 0, me);
 }
 GEN_PPC64_R2(rldcr, 0x1E, 0x09);
+
 /* rldimi - rldimi. */
-static inline void gen_rldimi(DisasContext *ctx, int mbn, int shn)
+static void gen_rldimi(DisasContext *ctx, int mbn, int shn)
 {
-    uint32_t sh, mb, me;
+    TCGv t_ra = cpu_gpr[rA(ctx->opcode)];
+    TCGv t_rs = cpu_gpr[rS(ctx->opcode)];
+    uint32_t sh = SH(ctx->opcode) | (shn << 5);
+    uint32_t mb = MB(ctx->opcode) | (mbn << 5);
+    uint32_t me = 63 - sh;
 
-    sh = SH(ctx->opcode) | (shn << 5);
-    mb = MB(ctx->opcode) | (mbn << 5);
-    me = 63 - sh;
-    if (unlikely(sh == 0 && mb == 0)) {
-        tcg_gen_mov_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rS(ctx->opcode)]);
+    if (mb <= me) {
+        tcg_gen_deposit_tl(t_ra, t_ra, t_rs, sh, me - mb + 1);
     } else {
-        TCGv t0, t1;
-        target_ulong mask;
+        target_ulong mask = MASK(mb, me);
+        TCGv t1 = tcg_temp_new();
 
-        t0 = tcg_temp_new();
-        tcg_gen_rotli_tl(t0, cpu_gpr[rS(ctx->opcode)], sh);
-        t1 = tcg_temp_new();
-        mask = MASK(mb, me);
-        tcg_gen_andi_tl(t0, t0, mask);
-        tcg_gen_andi_tl(t1, cpu_gpr[rA(ctx->opcode)], ~mask);
-        tcg_gen_or_tl(cpu_gpr[rA(ctx->opcode)], t0, t1);
-        tcg_temp_free(t0);
+        tcg_gen_rotli_tl(t1, t_rs, sh);
+        tcg_gen_andi_tl(t1, t1, mask);
+        tcg_gen_andi_tl(t_ra, t_ra, ~mask);
+        tcg_gen_or_tl(t_ra, t_ra, t1);
         tcg_temp_free(t1);
     }
-    if (unlikely(Rc(ctx->opcode) != 0))
-        gen_set_Rc0(ctx, cpu_gpr[rA(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_Rc0(ctx, t_ra);
+    }
 }
 GEN_PPC64_R4(rldimi, 0x1E, 0x06);
 #endif
@@ -2077,6 +2126,21 @@
 }
 #endif
 
+#if defined(TARGET_PPC64)
+static void gen_set_cr1_from_fpscr(DisasContext *ctx)
+{
+    TCGv_i32 tmp = tcg_temp_new_i32();
+    tcg_gen_trunc_tl_i32(tmp, cpu_fpscr);
+    tcg_gen_shri_i32(cpu_crf[1], tmp, 28);
+    tcg_temp_free_i32(tmp);
+}
+#else
+static void gen_set_cr1_from_fpscr(DisasContext *ctx)
+{
+    tcg_gen_shri_tl(cpu_crf[1], cpu_fpscr, 28);
+}
+#endif
+
 /***                       Floating-Point arithmetic                       ***/
 #define _GEN_FLOAT_ACB(name, op, op1, op2, isfloat, set_fprf, type)           \
 static void gen_f##name(DisasContext *ctx)                                    \
@@ -2095,8 +2159,12 @@
         gen_helper_frsp(cpu_fpr[rD(ctx->opcode)], cpu_env,                    \
                         cpu_fpr[rD(ctx->opcode)]);                            \
     }                                                                         \
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], set_fprf,                      \
-                     Rc(ctx->opcode) != 0);                                   \
+    if (set_fprf) {                                                           \
+        gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);                           \
+    }                                                                         \
+    if (unlikely(Rc(ctx->opcode) != 0)) {                                     \
+        gen_set_cr1_from_fpscr(ctx);                                          \
+    }                                                                         \
 }
 
 #define GEN_FLOAT_ACB(name, op2, set_fprf, type)                              \
@@ -2120,8 +2188,12 @@
         gen_helper_frsp(cpu_fpr[rD(ctx->opcode)], cpu_env,                    \
                         cpu_fpr[rD(ctx->opcode)]);                            \
     }                                                                         \
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)],                                \
-                     set_fprf, Rc(ctx->opcode) != 0);                         \
+    if (set_fprf) {                                                           \
+        gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);                           \
+    }                                                                         \
+    if (unlikely(Rc(ctx->opcode) != 0)) {                                     \
+        gen_set_cr1_from_fpscr(ctx);                                          \
+    }                                                                         \
 }
 #define GEN_FLOAT_AB(name, op2, inval, set_fprf, type)                        \
 _GEN_FLOAT_AB(name, name, 0x3F, op2, inval, 0, set_fprf, type);               \
@@ -2144,8 +2216,12 @@
         gen_helper_frsp(cpu_fpr[rD(ctx->opcode)], cpu_env,                    \
                         cpu_fpr[rD(ctx->opcode)]);                            \
     }                                                                         \
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)],                                \
-                     set_fprf, Rc(ctx->opcode) != 0);                         \
+    if (set_fprf) {                                                           \
+        gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);                           \
+    }                                                                         \
+    if (unlikely(Rc(ctx->opcode) != 0)) {                                     \
+        gen_set_cr1_from_fpscr(ctx);                                          \
+    }                                                                         \
 }
 #define GEN_FLOAT_AC(name, op2, inval, set_fprf, type)                        \
 _GEN_FLOAT_AC(name, name, 0x3F, op2, inval, 0, set_fprf, type);               \
@@ -2163,8 +2239,12 @@
     gen_reset_fpstatus();                                                     \
     gen_helper_f##name(cpu_fpr[rD(ctx->opcode)], cpu_env,                     \
                        cpu_fpr[rB(ctx->opcode)]);                             \
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)],                                \
-                     set_fprf, Rc(ctx->opcode) != 0);                         \
+    if (set_fprf) {                                                           \
+        gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);                           \
+    }                                                                         \
+    if (unlikely(Rc(ctx->opcode) != 0)) {                                     \
+        gen_set_cr1_from_fpscr(ctx);                                          \
+    }                                                                         \
 }
 
 #define GEN_FLOAT_BS(name, op1, op2, set_fprf, type)                          \
@@ -2179,8 +2259,12 @@
     gen_reset_fpstatus();                                                     \
     gen_helper_f##name(cpu_fpr[rD(ctx->opcode)], cpu_env,                     \
                        cpu_fpr[rB(ctx->opcode)]);                             \
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)],                                \
-                     set_fprf, Rc(ctx->opcode) != 0);                         \
+    if (set_fprf) {                                                           \
+        gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);                           \
+    }                                                                         \
+    if (unlikely(Rc(ctx->opcode) != 0)) {                                     \
+        gen_set_cr1_from_fpscr(ctx);                                          \
+    }                                                                         \
 }
 
 /* fadd - fadds */
@@ -2213,7 +2297,10 @@
                        cpu_fpr[rB(ctx->opcode)]);
     gen_helper_frsp(cpu_fpr[rD(ctx->opcode)], cpu_env,
                     cpu_fpr[rD(ctx->opcode)]);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 1, Rc(ctx->opcode) != 0);
+    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /* fsel */
@@ -2234,7 +2321,10 @@
     gen_reset_fpstatus();
     gen_helper_fsqrt(cpu_fpr[rD(ctx->opcode)], cpu_env,
                      cpu_fpr[rB(ctx->opcode)]);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 1, Rc(ctx->opcode) != 0);
+    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 static void gen_fsqrts(DisasContext *ctx)
@@ -2250,7 +2340,10 @@
                      cpu_fpr[rB(ctx->opcode)]);
     gen_helper_frsp(cpu_fpr[rD(ctx->opcode)], cpu_env,
                     cpu_fpr[rD(ctx->opcode)]);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 1, Rc(ctx->opcode) != 0);
+    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)]);
+    if (unlikely(Rc(ctx->opcode) != 0)) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /***                     Floating-Point multiply-and-add                   ***/
@@ -2370,7 +2463,9 @@
     }
     tcg_gen_andi_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpr[rB(ctx->opcode)],
                      ~(1ULL << 63));
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
+    if (unlikely(Rc(ctx->opcode))) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /* fmr  - fmr. */
@@ -2382,7 +2477,9 @@
         return;
     }
     tcg_gen_mov_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpr[rB(ctx->opcode)]);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
+    if (unlikely(Rc(ctx->opcode))) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /* fnabs */
@@ -2395,7 +2492,9 @@
     }
     tcg_gen_ori_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpr[rB(ctx->opcode)],
                     1ULL << 63);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
+    if (unlikely(Rc(ctx->opcode))) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /* fneg */
@@ -2408,7 +2507,9 @@
     }
     tcg_gen_xori_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpr[rB(ctx->opcode)],
                      1ULL << 63);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
+    if (unlikely(Rc(ctx->opcode))) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /* fcpsgn: PowerPC 2.05 specification */
@@ -2421,7 +2522,9 @@
     }
     tcg_gen_deposit_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpr[rA(ctx->opcode)],
                         cpu_fpr[rB(ctx->opcode)], 0, 63);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
+    if (unlikely(Rc(ctx->opcode))) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 static void gen_fmrgew(DisasContext *ctx)
@@ -2456,18 +2559,31 @@
 static void gen_mcrfs(DisasContext *ctx)
 {
     TCGv tmp = tcg_temp_new();
+    TCGv_i32 tmask;
+    TCGv_i64 tnew_fpscr = tcg_temp_new_i64();
     int bfa;
+    int nibble;
+    int shift;
 
     if (unlikely(!ctx->fpu_enabled)) {
         gen_exception(ctx, POWERPC_EXCP_FPU);
         return;
     }
-    bfa = 4 * (7 - crfS(ctx->opcode));
-    tcg_gen_shri_tl(tmp, cpu_fpscr, bfa);
+    bfa = crfS(ctx->opcode);
+    nibble = 7 - bfa;
+    shift = 4 * nibble;
+    tcg_gen_shri_tl(tmp, cpu_fpscr, shift);
     tcg_gen_trunc_tl_i32(cpu_crf[crfD(ctx->opcode)], tmp);
-    tcg_temp_free(tmp);
     tcg_gen_andi_i32(cpu_crf[crfD(ctx->opcode)], cpu_crf[crfD(ctx->opcode)], 0xf);
-    tcg_gen_andi_tl(cpu_fpscr, cpu_fpscr, ~(0xF << bfa));
+    tcg_temp_free(tmp);
+    tcg_gen_extu_tl_i64(tnew_fpscr, cpu_fpscr);
+    /* Only the exception bits (including FX) should be cleared if read */
+    tcg_gen_andi_i64(tnew_fpscr, tnew_fpscr, ~((0xF << shift) & FP_EX_CLEAR_BITS));
+    /* FEX and VX need to be updated, so don't set fpscr directly */
+    tmask = tcg_const_i32(1 << nibble);
+    gen_helper_store_fpscr(cpu_env, tnew_fpscr, tmask);
+    tcg_temp_free_i32(tmask);
+    tcg_temp_free_i64(tnew_fpscr);
 }
 
 /* mffs */
@@ -2479,7 +2595,9 @@
     }
     gen_reset_fpstatus();
     tcg_gen_extu_tl_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpscr);
-    gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
+    if (unlikely(Rc(ctx->opcode))) {
+        gen_set_cr1_from_fpscr(ctx);
+    }
 }
 
 /* mtfsb0 */
@@ -2669,7 +2787,7 @@
 
 static inline void gen_check_align(DisasContext *ctx, TCGv EA, int mask)
 {
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv_i32 t1, t2;
     /* NIP cannot be restored if the memory exception comes from an helper */
@@ -2816,18 +2934,23 @@
     tcg_temp_free(EA);                                                        \
 }
 
-#define GEN_LDX_E(name, ldop, opc2, opc3, type, type2)                        \
+#define GEN_LDX_E(name, ldop, opc2, opc3, type, type2, chk)                   \
 static void glue(gen_, name##x)(DisasContext *ctx)                            \
 {                                                                             \
     TCGv EA;                                                                  \
+    chk;                                                                      \
     gen_set_access_type(ctx, ACCESS_INT);                                     \
     EA = tcg_temp_new();                                                      \
     gen_addr_reg_index(ctx, EA);                                              \
     gen_qemu_##ldop(ctx, cpu_gpr[rD(ctx->opcode)], EA);                       \
     tcg_temp_free(EA);                                                        \
 }
+
 #define GEN_LDX(name, ldop, opc2, opc3, type)                                 \
-    GEN_LDX_E(name, ldop, opc2, opc3, type, PPC_NONE)
+    GEN_LDX_E(name, ldop, opc2, opc3, type, PPC_NONE, CHK_NONE)
+
+#define GEN_LDX_HVRM(name, ldop, opc2, opc3, type)                            \
+    GEN_LDX_E(name, ldop, opc2, opc3, type, PPC_NONE, CHK_HVRM)
 
 #define GEN_LDS(name, ldop, op, type)                                         \
 GEN_LD(name, ldop, op | 0x20, type);                                          \
@@ -2853,6 +2976,12 @@
 /* ldx */
 GEN_LDX(ld, ld64, 0x15, 0x00, PPC_64B);
 
+/* CI load/store variants */
+GEN_LDX_HVRM(ldcix, ld64, 0x15, 0x1b, PPC_CILDST)
+GEN_LDX_HVRM(lwzcix, ld32u, 0x15, 0x15, PPC_CILDST)
+GEN_LDX_HVRM(lhzcix, ld16u, 0x15, 0x19, PPC_CILDST)
+GEN_LDX_HVRM(lbzcix, ld8u, 0x15, 0x1a, PPC_CILDST)
+
 static void gen_ld(DisasContext *ctx)
 {
     TCGv EA;
@@ -2889,7 +3018,7 @@
     bool le_is_supported = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
 
     if (!legal_in_user_mode && ctx->pr) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+        gen_priv_exception(ctx, POWERPC_EXCP_PRIV_OPC);
         return;
     }
 
@@ -2971,10 +3100,11 @@
     tcg_temp_free(EA);                                                        \
 }
 
-#define GEN_STX_E(name, stop, opc2, opc3, type, type2)                        \
+#define GEN_STX_E(name, stop, opc2, opc3, type, type2, chk)                   \
 static void glue(gen_, name##x)(DisasContext *ctx)                            \
 {                                                                             \
     TCGv EA;                                                                  \
+    chk;                                                                      \
     gen_set_access_type(ctx, ACCESS_INT);                                     \
     EA = tcg_temp_new();                                                      \
     gen_addr_reg_index(ctx, EA);                                              \
@@ -2982,7 +3112,10 @@
     tcg_temp_free(EA);                                                        \
 }
 #define GEN_STX(name, stop, opc2, opc3, type)                                 \
-    GEN_STX_E(name, stop, opc2, opc3, type, PPC_NONE)
+    GEN_STX_E(name, stop, opc2, opc3, type, PPC_NONE, CHK_NONE)
+
+#define GEN_STX_HVRM(name, stop, opc2, opc3, type)                            \
+    GEN_STX_E(name, stop, opc2, opc3, type, PPC_NONE, CHK_HVRM)
 
 #define GEN_STS(name, stop, op, type)                                         \
 GEN_ST(name, stop, op | 0x20, type);                                          \
@@ -2999,6 +3132,10 @@
 #if defined(TARGET_PPC64)
 GEN_STUX(std, st64, 0x15, 0x05, PPC_64B);
 GEN_STX(std, st64, 0x15, 0x04, PPC_64B);
+GEN_STX_HVRM(stdcix, st64, 0x15, 0x1f, PPC_CILDST)
+GEN_STX_HVRM(stwcix, st32, 0x15, 0x1c, PPC_CILDST)
+GEN_STX_HVRM(sthcix, st16, 0x15, 0x1d, PPC_CILDST)
+GEN_STX_HVRM(stbcix, st8, 0x15, 0x1e, PPC_CILDST)
 
 static void gen_std(DisasContext *ctx)
 {
@@ -3007,12 +3144,15 @@
 
     rs = rS(ctx->opcode);
     if ((ctx->opcode & 0x3) == 0x2) { /* stq */
-
         bool legal_in_user_mode = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
         bool le_is_supported = (ctx->insns_flags2 & PPC2_LSQ_ISA207) != 0;
 
+        if (!(ctx->insns_flags & PPC_64BX)) {
+            gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
+        }
+
         if (!legal_in_user_mode && ctx->pr) {
-            gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+            gen_priv_exception(ctx, POWERPC_EXCP_PRIV_OPC);
             return;
         }
 
@@ -3084,7 +3224,7 @@
     TCGMemOp op = MO_Q | (ctx->default_tcg_memop_mask ^ MO_BSWAP);
     tcg_gen_qemu_ld_i64(arg1, arg2, ctx->mem_idx, op);
 }
-GEN_LDX_E(ldbr, ld64ur, 0x14, 0x10, PPC_NONE, PPC2_DBRX);
+GEN_LDX_E(ldbr, ld64ur, 0x14, 0x10, PPC_NONE, PPC2_DBRX, CHK_NONE);
 #endif  /* TARGET_PPC64 */
 
 /* sthbrx */
@@ -3110,7 +3250,7 @@
     TCGMemOp op = MO_Q | (ctx->default_tcg_memop_mask ^ MO_BSWAP);
     tcg_gen_qemu_st_i64(arg1, arg2, ctx->mem_idx, op);
 }
-GEN_STX_E(stdbr, st64r, 0x14, 0x14, PPC_NONE, PPC2_DBRX);
+GEN_STX_E(stdbr, st64r, 0x14, 0x14, PPC_NONE, PPC2_DBRX, CHK_NONE);
 #endif  /* TARGET_PPC64 */
 
 /***                    Integer load and store multiple                    ***/
@@ -3166,10 +3306,8 @@
 
     if (nb == 0)
         nb = 32;
-    nr = nb / 4;
-    if (unlikely(((start + nr) > 32  &&
-                  start <= ra && (start + nr - 32) > ra) ||
-                 ((start + nr) <= 32 && start <= ra && (start + nr) > ra))) {
+    nr = (nb + 3) / 4;
+    if (unlikely(lsw_reg_in_range(start, nr, ra))) {
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_LSWX);
         return;
     }
@@ -3253,9 +3391,37 @@
 {
 }
 
+#if !defined(CONFIG_USER_ONLY)
+static inline void gen_check_tlb_flush(DisasContext *ctx)
+{
+    TCGv_i32 t;
+    TCGLabel *l;
+
+    if (!ctx->lazy_tlb_flush) {
+        return;
+    }
+    l = gen_new_label();
+    t = tcg_temp_new_i32();
+    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUPPCState, tlb_need_flush));
+    tcg_gen_brcondi_i32(TCG_COND_EQ, t, 0, l);
+    gen_helper_check_tlb_flush(cpu_env);
+    gen_set_label(l);
+    tcg_temp_free_i32(t);
+}
+#else
+static inline void gen_check_tlb_flush(DisasContext *ctx) { }
+#endif
+
 /* isync */
 static void gen_isync(DisasContext *ctx)
 {
+    /*
+     * We need to check for a pending TLB flush. This can only happen in
+     * kernel mode however so check MSR_PR
+     */
+    if (!ctx->pr) {
+        gen_check_tlb_flush(ctx);
+    }
     gen_stop_exception(ctx);
 }
 
@@ -3302,7 +3468,7 @@
 static void gen_conditional_store(DisasContext *ctx, TCGv EA,
                                   int reg, int size)
 {
-    int l1;
+    TCGLabel *l1;
 
     tcg_gen_trunc_tl_i32(cpu_crf[0], cpu_so);
     l1 = gen_new_label();
@@ -3412,12 +3578,25 @@
 /* sync */
 static void gen_sync(DisasContext *ctx)
 {
+    uint32_t l = (ctx->opcode >> 21) & 3;
+
+    /*
+     * We may need to check for a pending TLB flush.
+     *
+     * We do this on ptesync (l == 2) on ppc64 and any sync pn ppc32.
+     *
+     * Additionally, this can only happen in kernel mode however so
+     * check MSR_PR as well.
+     */
+    if (((l == 2) || !(ctx->insns_flags & PPC_64B)) && !ctx->pr) {
+        gen_check_tlb_flush(ctx);
+    }
 }
 
 /* wait */
 static void gen_wait(DisasContext *ctx)
 {
-    TCGv_i32 t0 = tcg_temp_new_i32();
+    TCGv_i32 t0 = tcg_const_i32(1);
     tcg_gen_st_i32(t0, cpu_env,
                    -offsetof(PowerPCCPU, env) + offsetof(CPUState, halted));
     tcg_temp_free_i32(t0);
@@ -3425,6 +3604,68 @@
     gen_exception_err(ctx, EXCP_HLT, 1);
 }
 
+#if defined(TARGET_PPC64)
+static void gen_doze(DisasContext *ctx)
+{
+#if defined(CONFIG_USER_ONLY)
+    GEN_PRIV;
+#else
+    TCGv_i32 t;
+
+    CHK_HV;
+    t = tcg_const_i32(PPC_PM_DOZE);
+    gen_helper_pminsn(cpu_env, t);
+    tcg_temp_free_i32(t);
+    gen_stop_exception(ctx);
+#endif /* defined(CONFIG_USER_ONLY) */
+}
+
+static void gen_nap(DisasContext *ctx)
+{
+#if defined(CONFIG_USER_ONLY)
+    GEN_PRIV;
+#else
+    TCGv_i32 t;
+
+    CHK_HV;
+    t = tcg_const_i32(PPC_PM_NAP);
+    gen_helper_pminsn(cpu_env, t);
+    tcg_temp_free_i32(t);
+    gen_stop_exception(ctx);
+#endif /* defined(CONFIG_USER_ONLY) */
+}
+
+static void gen_sleep(DisasContext *ctx)
+{
+#if defined(CONFIG_USER_ONLY)
+    GEN_PRIV;
+#else
+    TCGv_i32 t;
+
+    CHK_HV;
+    t = tcg_const_i32(PPC_PM_SLEEP);
+    gen_helper_pminsn(cpu_env, t);
+    tcg_temp_free_i32(t);
+    gen_stop_exception(ctx);
+#endif /* defined(CONFIG_USER_ONLY) */
+}
+
+static void gen_rvwinkle(DisasContext *ctx)
+{
+#if defined(CONFIG_USER_ONLY)
+    GEN_PRIV;
+#else
+    TCGv_i32 t;
+
+    CHK_HV;
+    t = tcg_const_i32(PPC_PM_RVWINKLE);
+    gen_helper_pminsn(cpu_env, t);
+    tcg_temp_free_i32(t);
+    gen_stop_exception(ctx);
+#endif /* defined(CONFIG_USER_ONLY) */
+}
+#endif /* #if defined(TARGET_PPC64) */
+
 /***                         Floating-point load                           ***/
 #define GEN_LDF(name, ldop, opc, type)                                        \
 static void glue(gen_, name)(DisasContext *ctx)                                       \
@@ -3763,19 +4004,29 @@
 #endif
 }
 
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+    if (unlikely(ctx->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 /***                                Branch                                 ***/
 static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = ctx->tb;
     if (NARROW_MODE(ctx)) {
         dest = (uint32_t) dest;
     }
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-        likely(!ctx->singlestep_enabled)) {
+    if (use_goto_tb(ctx, dest)) {
         tcg_gen_goto_tb(n);
         tcg_gen_movi_tl(cpu_nip, dest & ~3);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
     } else {
         tcg_gen_movi_tl(cpu_nip, dest & ~3);
         if (unlikely(ctx->singlestep_enabled)) {
@@ -3833,7 +4084,7 @@
 static inline void gen_bcond(DisasContext *ctx, int type)
 {
     uint32_t bo = BO(ctx->opcode);
-    int l1;
+    TCGLabel *l1;
     TCGv target;
 
     ctx->exception = POWERPC_EXCP_BRANCH;
@@ -3993,13 +4244,14 @@
 static void gen_rfi(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
+    /* FIXME: This instruction doesn't exist anymore on 64-bit server
+     * processors compliant with arch 2.x, we should remove it there,
+     * but we need to fix OpenBIOS not to use it on 970 first
+     */
     /* Restore CPU state */
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_update_cfar(ctx, ctx->nip);
     gen_helper_rfi(cpu_env);
     gen_sync_exception(ctx);
@@ -4010,13 +4262,10 @@
 static void gen_rfid(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     /* Restore CPU state */
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_update_cfar(ctx, ctx->nip);
     gen_helper_rfid(cpu_env);
     gen_sync_exception(ctx);
@@ -4026,13 +4275,10 @@
 static void gen_hrfid(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     /* Restore CPU state */
-    if (unlikely(!ctx->hv)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_HV;
     gen_helper_hrfid(cpu_env);
     gen_sync_exception(ctx);
 #endif
@@ -4195,18 +4441,11 @@
 /* mfmsr */
 static void gen_mfmsr(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-#else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
     tcg_gen_mov_tl(cpu_gpr[rD(ctx->opcode)], cpu_msr);
-#endif
 }
 
-static void spr_noaccess(void *opaque, int gprn, int sprn)
+static void spr_noaccess(DisasContext *ctx, int gprn, int sprn)
 {
 #if 0
     sprn = ((sprn >> 5) & 0x1F) | ((sprn & 0x1F) << 5);
@@ -4218,17 +4457,20 @@
 /* mfspr */
 static inline void gen_op_mfspr(DisasContext *ctx)
 {
-    void (*read_cb)(void *opaque, int gprn, int sprn);
+    void (*read_cb)(DisasContext *ctx, int gprn, int sprn);
     uint32_t sprn = SPR(ctx->opcode);
 
-#if !defined(CONFIG_USER_ONLY)
-    if (ctx->hv)
-        read_cb = ctx->spr_cb[sprn].hea_read;
-    else if (!ctx->pr)
-        read_cb = ctx->spr_cb[sprn].oea_read;
-    else
-#endif
+#if defined(CONFIG_USER_ONLY)
+    read_cb = ctx->spr_cb[sprn].uea_read;
+#else
+    if (ctx->pr) {
         read_cb = ctx->spr_cb[sprn].uea_read;
+    } else if (ctx->hv) {
+        read_cb = ctx->spr_cb[sprn].hea_read;
+    } else {
+        read_cb = ctx->spr_cb[sprn].oea_read;
+    }
+#endif
     if (likely(read_cb != NULL)) {
         if (likely(read_cb != SPR_NOACCESS)) {
             (*read_cb)(ctx, rD(ctx->opcode), sprn);
@@ -4239,20 +4481,42 @@
              * allowing userland application to read the PVR
              */
             if (sprn != SPR_PVR) {
-                qemu_log("Trying to read privileged spr %d (0x%03x) at "
-                         TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-                printf("Trying to read privileged spr %d (0x%03x) at "
-                       TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+                fprintf(stderr, "Trying to read privileged spr %d (0x%03x) at "
+                        TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+                if (qemu_log_separate()) {
+                    qemu_log("Trying to read privileged spr %d (0x%03x) at "
+                             TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+                }
             }
-            gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+            gen_priv_exception(ctx, POWERPC_EXCP_PRIV_REG);
         }
     } else {
+        /* ISA 2.07 defines these as no-ops */
+        if ((ctx->insns_flags2 & PPC2_ISA207S) &&
+            (sprn >= 808 && sprn <= 811)) {
+            /* This is a nop */
+            return;
+        }
         /* Not defined */
-        qemu_log("Trying to read invalid spr %d (0x%03x) at "
-                 TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-        printf("Trying to read invalid spr %d (0x%03x) at "
-               TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_SPR);
+        fprintf(stderr, "Trying to read invalid spr %d (0x%03x) at "
+                TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        if (qemu_log_separate()) {
+            qemu_log("Trying to read invalid spr %d (0x%03x) at "
+                     TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        }
+
+        /* The behaviour depends on MSR:PR and SPR# bit 0x10,
+         * it can generate a priv, a hv emu or a no-op
+         */
+        if (sprn & 0x10) {
+            if (ctx->pr) {
+                gen_priv_exception(ctx, POWERPC_EXCP_INVAL_SPR);
+            }
+        } else {
+            if (ctx->pr || sprn == 0 || sprn == 4 || sprn == 5 || sprn == 6) {
+                gen_hvpriv_exception(ctx, POWERPC_EXCP_INVAL_SPR);
+            }
+        }
     }
 }
 
@@ -4299,18 +4563,14 @@
 #if defined(TARGET_PPC64)
 static void gen_mtmsrd(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-#else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
+
+#if !defined(CONFIG_USER_ONLY)
     if (ctx->opcode & 0x00010000) {
         /* Special form that does not need any synchronisation */
         TCGv t0 = tcg_temp_new();
         tcg_gen_andi_tl(t0, cpu_gpr[rS(ctx->opcode)], (1 << MSR_RI) | (1 << MSR_EE));
-        tcg_gen_andi_tl(cpu_msr, cpu_msr, ~((1 << MSR_RI) | (1 << MSR_EE)));
+        tcg_gen_andi_tl(cpu_msr, cpu_msr, ~(target_ulong)((1 << MSR_RI) | (1 << MSR_EE)));
         tcg_gen_or_tl(cpu_msr, cpu_msr, t0);
         tcg_temp_free(t0);
     } else {
@@ -4324,24 +4584,20 @@
         /* Note that mtmsr is not always defined as context-synchronizing */
         gen_stop_exception(ctx);
     }
-#endif
+#endif /* !defined(CONFIG_USER_ONLY) */
 }
-#endif
+#endif /* defined(TARGET_PPC64) */
 
 static void gen_mtmsr(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-#else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
-    if (ctx->opcode & 0x00010000) {
+    CHK_SV;
+
+#if !defined(CONFIG_USER_ONLY)
+   if (ctx->opcode & 0x00010000) {
         /* Special form that does not need any synchronisation */
         TCGv t0 = tcg_temp_new();
         tcg_gen_andi_tl(t0, cpu_gpr[rS(ctx->opcode)], (1 << MSR_RI) | (1 << MSR_EE));
-        tcg_gen_andi_tl(cpu_msr, cpu_msr, ~((1 << MSR_RI) | (1 << MSR_EE)));
+        tcg_gen_andi_tl(cpu_msr, cpu_msr, ~(target_ulong)((1 << MSR_RI) | (1 << MSR_EE)));
         tcg_gen_or_tl(cpu_msr, cpu_msr, t0);
         tcg_temp_free(t0);
     } else {
@@ -4369,35 +4625,62 @@
 /* mtspr */
 static void gen_mtspr(DisasContext *ctx)
 {
-    void (*write_cb)(void *opaque, int sprn, int gprn);
+    void (*write_cb)(DisasContext *ctx, int sprn, int gprn);
     uint32_t sprn = SPR(ctx->opcode);
 
-#if !defined(CONFIG_USER_ONLY)
-    if (ctx->hv)
-        write_cb = ctx->spr_cb[sprn].hea_write;
-    else if (!ctx->pr)
-        write_cb = ctx->spr_cb[sprn].oea_write;
-    else
-#endif
+#if defined(CONFIG_USER_ONLY)
+    write_cb = ctx->spr_cb[sprn].uea_write;
+#else
+    if (ctx->pr) {
         write_cb = ctx->spr_cb[sprn].uea_write;
+    } else if (ctx->hv) {
+        write_cb = ctx->spr_cb[sprn].hea_write;
+    } else {
+        write_cb = ctx->spr_cb[sprn].oea_write;
+    }
+#endif
     if (likely(write_cb != NULL)) {
         if (likely(write_cb != SPR_NOACCESS)) {
             (*write_cb)(ctx, sprn, rS(ctx->opcode));
         } else {
             /* Privilege exception */
-            qemu_log("Trying to write privileged spr %d (0x%03x) at "
-                     TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-            printf("Trying to write privileged spr %d (0x%03x) at "
-                   TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-            gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+            fprintf(stderr, "Trying to write privileged spr %d (0x%03x) at "
+                    TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+            if (qemu_log_separate()) {
+                qemu_log("Trying to write privileged spr %d (0x%03x) at "
+                         TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+            }
+            gen_priv_exception(ctx, POWERPC_EXCP_PRIV_REG);
         }
     } else {
+        /* ISA 2.07 defines these as no-ops */
+        if ((ctx->insns_flags2 & PPC2_ISA207S) &&
+            (sprn >= 808 && sprn <= 811)) {
+            /* This is a nop */
+            return;
+        }
+
         /* Not defined */
-        qemu_log("Trying to write invalid spr %d (0x%03x) at "
-                 TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-        printf("Trying to write invalid spr %d (0x%03x) at "
-               TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
-        gen_inval_exception(ctx, POWERPC_EXCP_INVAL_SPR);
+        if (qemu_log_separate()) {
+            qemu_log("Trying to write invalid spr %d (0x%03x) at "
+                     TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+        }
+        fprintf(stderr, "Trying to write invalid spr %d (0x%03x) at "
+                TARGET_FMT_lx "\n", sprn, sprn, ctx->nip - 4);
+
+
+        /* The behaviour depends on MSR:PR and SPR# bit 0x10,
+         * it can generate a priv, a hv emu or a no-op
+         */
+        if (sprn & 0x10) {
+            if (ctx->pr) {
+                gen_priv_exception(ctx, POWERPC_EXCP_INVAL_SPR);
+            }
+        } else {
+            if (ctx->pr || sprn == 0) {
+                gen_hvpriv_exception(ctx, POWERPC_EXCP_INVAL_SPR);
+            }
+        }
     }
 }
 
@@ -4419,13 +4702,11 @@
 static void gen_dcbi(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv EA, val;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     EA = tcg_temp_new();
     gen_set_access_type(ctx, ACCESS_CACHE);
     gen_addr_reg_index(ctx, EA);
@@ -4435,7 +4716,7 @@
     gen_qemu_st8(ctx, val, EA);
     tcg_temp_free(val);
     tcg_temp_free(EA);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* dcdst */
@@ -4556,72 +4837,64 @@
 static void gen_mfsr(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_const_tl(SR(ctx->opcode));
     gen_helper_load_sr(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mfsrin */
 static void gen_mfsrin(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     tcg_gen_shri_tl(t0, cpu_gpr[rB(ctx->opcode)], 28);
     tcg_gen_andi_tl(t0, t0, 0xF);
     gen_helper_load_sr(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mtsr */
 static void gen_mtsr(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_const_tl(SR(ctx->opcode));
     gen_helper_store_sr(cpu_env, t0, cpu_gpr[rS(ctx->opcode)]);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mtsrin */
 static void gen_mtsrin(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
+
     t0 = tcg_temp_new();
     tcg_gen_shri_tl(t0, cpu_gpr[rB(ctx->opcode)], 28);
     tcg_gen_andi_tl(t0, t0, 0xF);
     gen_helper_store_sr(cpu_env, t0, cpu_gpr[rD(ctx->opcode)]);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 #if defined(TARGET_PPC64)
@@ -4631,114 +4904,125 @@
 static void gen_mfsr_64b(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_const_tl(SR(ctx->opcode));
     gen_helper_load_sr(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mfsrin */
 static void gen_mfsrin_64b(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     tcg_gen_shri_tl(t0, cpu_gpr[rB(ctx->opcode)], 28);
     tcg_gen_andi_tl(t0, t0, 0xF);
     gen_helper_load_sr(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mtsr */
 static void gen_mtsr_64b(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_const_tl(SR(ctx->opcode));
     gen_helper_store_sr(cpu_env, t0, cpu_gpr[rS(ctx->opcode)]);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mtsrin */
 static void gen_mtsrin_64b(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     tcg_gen_shri_tl(t0, cpu_gpr[rB(ctx->opcode)], 28);
     tcg_gen_andi_tl(t0, t0, 0xF);
     gen_helper_store_sr(cpu_env, t0, cpu_gpr[rS(ctx->opcode)]);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* slbmte */
 static void gen_slbmte(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
+
     gen_helper_store_slb(cpu_env, cpu_gpr[rB(ctx->opcode)],
                          cpu_gpr[rS(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_slbmfee(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
+
     gen_helper_load_slb_esid(cpu_gpr[rS(ctx->opcode)], cpu_env,
                              cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_slbmfev(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
+    GEN_PRIV;
+#else
+    CHK_SV;
+
+    gen_helper_load_slb_vsid(cpu_gpr[rS(ctx->opcode)], cpu_env,
+                             cpu_gpr[rB(ctx->opcode)]);
+#endif /* defined(CONFIG_USER_ONLY) */
+}
+
+static void gen_slbfee_(DisasContext *ctx)
+{
+#if defined(CONFIG_USER_ONLY)
     gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
 #else
+    TCGLabel *l1, *l2;
+
     if (unlikely(ctx->pr)) {
         gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
         return;
     }
-    gen_helper_load_slb_vsid(cpu_gpr[rS(ctx->opcode)], cpu_env,
+    gen_helper_find_slb_vsid(cpu_gpr[rS(ctx->opcode)], cpu_env,
                              cpu_gpr[rB(ctx->opcode)]);
+    l1 = gen_new_label();
+    l2 = gen_new_label();
+    tcg_gen_trunc_tl_i32(cpu_crf[0], cpu_so);
+    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr[rS(ctx->opcode)], -1, l1);
+    tcg_gen_ori_i32(cpu_crf[0], cpu_crf[0], 1 << CRF_EQ);
+    tcg_gen_br(l2);
+    gen_set_label(l1);
+    tcg_gen_movi_tl(cpu_gpr[rS(ctx->opcode)], 0);
+    gen_set_label(l2);
 #endif
 }
 #endif /* defined(TARGET_PPC64) */
@@ -4750,40 +5034,34 @@
 static void gen_tlbia(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_HV;
+
     gen_helper_tlbia(cpu_env);
-#endif
+#endif  /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbiel */
 static void gen_tlbiel(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
+
     gen_helper_tlbie(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbie */
 static void gen_tlbie(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_HV;
+
     if (NARROW_MODE(ctx)) {
         TCGv t0 = tcg_temp_new();
         tcg_gen_ext32u_tl(t0, cpu_gpr[rB(ctx->opcode)]);
@@ -4792,24 +5070,23 @@
     } else {
         gen_helper_tlbie(cpu_env, cpu_gpr[rB(ctx->opcode)]);
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbsync */
 static void gen_tlbsync(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
-    /* This has no effect: it should ensure that all previous
-     * tlbie have completed
+    CHK_HV;
+
+    /* tlbsync is a nop for server, ptesync handles delayed tlb flush,
+     * embedded however needs to deal with tlbsync. We don't try to be
+     * fancy and swallow the overhead of checking for both.
      */
-    gen_stop_exception(ctx);
-#endif
+    gen_check_tlb_flush(ctx);
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 #if defined(TARGET_PPC64)
@@ -4817,30 +5094,26 @@
 static void gen_slbia(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
+
     gen_helper_slbia(cpu_env);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* slbie */
 static void gen_slbie(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
+
     gen_helper_slbie(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
-#endif
+#endif  /* defined(TARGET_PPC64) */
 
 /***                              External control                         ***/
 /* Optional: */
@@ -4876,8 +5149,8 @@
 /* abs - abs. */
 static void gen_abs(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     tcg_gen_brcondi_tl(TCG_COND_GE, cpu_gpr[rA(ctx->opcode)], 0, l1);
     tcg_gen_neg_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);
     tcg_gen_br(l2);
@@ -4891,9 +5164,9 @@
 /* abso - abso. */
 static void gen_abso(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
-    int l3 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
+    TCGLabel *l3 = gen_new_label();
     /* Start with XER OV disabled, the most likely case */
     tcg_gen_movi_tl(cpu_ov, 0);
     tcg_gen_brcondi_tl(TCG_COND_GE, cpu_gpr[rA(ctx->opcode)], 0, l2);
@@ -4959,8 +5232,8 @@
 /* doz - doz. */
 static void gen_doz(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     tcg_gen_brcond_tl(TCG_COND_GE, cpu_gpr[rB(ctx->opcode)], cpu_gpr[rA(ctx->opcode)], l1);
     tcg_gen_sub_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rB(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);
     tcg_gen_br(l2);
@@ -4974,8 +5247,8 @@
 /* dozo - dozo. */
 static void gen_dozo(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
     TCGv t2 = tcg_temp_new();
@@ -5005,8 +5278,8 @@
 static void gen_dozi(DisasContext *ctx)
 {
     target_long simm = SIMM(ctx->opcode);
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     tcg_gen_brcondi_tl(TCG_COND_LT, cpu_gpr[rA(ctx->opcode)], simm, l1);
     tcg_gen_subfi_tl(cpu_gpr[rD(ctx->opcode)], simm, cpu_gpr[rA(ctx->opcode)]);
     tcg_gen_br(l2);
@@ -5042,7 +5315,7 @@
 /* maskg - maskg. */
 static void gen_maskg(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
     TCGv t2 = tcg_temp_new();
@@ -5102,7 +5375,7 @@
 /* mulo - mulo. */
 static void gen_mulo(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv_i64 t0 = tcg_temp_new_i64();
     TCGv_i64 t1 = tcg_temp_new_i64();
     TCGv t2 = tcg_temp_new();
@@ -5130,8 +5403,8 @@
 /* nabs - nabs. */
 static void gen_nabs(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     tcg_gen_brcondi_tl(TCG_COND_GT, cpu_gpr[rA(ctx->opcode)], 0, l1);
     tcg_gen_mov_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);
     tcg_gen_br(l2);
@@ -5145,8 +5418,8 @@
 /* nabso - nabso. */
 static void gen_nabso(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     tcg_gen_brcondi_tl(TCG_COND_GT, cpu_gpr[rA(ctx->opcode)], 0, l1);
     tcg_gen_mov_tl(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);
     tcg_gen_br(l2);
@@ -5271,8 +5544,8 @@
 /* sllq - sllq. */
 static void gen_sllq(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     TCGv t0 = tcg_temp_local_new();
     TCGv t1 = tcg_temp_local_new();
     TCGv t2 = tcg_temp_local_new();
@@ -5300,7 +5573,7 @@
 /* slq - slq. */
 static void gen_slq(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
     tcg_gen_andi_tl(t1, cpu_gpr[rB(ctx->opcode)], 0x1F);
@@ -5324,7 +5597,7 @@
 static void gen_sraiq(DisasContext *ctx)
 {
     int sh = SH(ctx->opcode);
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
     tcg_gen_shri_tl(t0, cpu_gpr[rS(ctx->opcode)], sh);
@@ -5346,8 +5619,8 @@
 /* sraq - sraq. */
 static void gen_sraq(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_local_new();
     TCGv t2 = tcg_temp_local_new();
@@ -5469,8 +5742,8 @@
 /* srlq */
 static void gen_srlq(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
     TCGv t0 = tcg_temp_local_new();
     TCGv t1 = tcg_temp_local_new();
     TCGv t2 = tcg_temp_local_new();
@@ -5499,7 +5772,7 @@
 /* srq */
 static void gen_srq(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
     tcg_gen_andi_tl(t1, cpu_gpr[rB(ctx->opcode)], 0x1F);
@@ -5539,14 +5812,11 @@
 static void gen_mfrom(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_helper_602_mfrom(cpu_gpr[rD(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* 602 - 603 - G2 TLB management */
@@ -5555,28 +5825,22 @@
 static void gen_tlbld_6xx(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_helper_6xx_tlbd(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbli */
 static void gen_tlbli_6xx(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_helper_6xx_tlbi(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* 74xx TLB management */
@@ -5585,28 +5849,22 @@
 static void gen_tlbld_74xx(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_helper_74xx_tlbd(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbli */
 static void gen_tlbli_74xx(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_helper_74xx_tlbi(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* POWER instructions not in PowerPC 601 */
@@ -5620,15 +5878,12 @@
 /* cli */
 static void gen_cli(DisasContext *ctx)
 {
-    /* Cache line invalidate: privileged and treated as no-op */
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
-#endif
+    /* Cache line invalidate: privileged and treated as no-op */
+    CHK_SV;
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* dclst */
@@ -5640,15 +5895,13 @@
 static void gen_mfsri(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     int ra = rA(ctx->opcode);
     int rd = rD(ctx->opcode);
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
     tcg_gen_shri_tl(t0, t0, 28);
@@ -5657,38 +5910,34 @@
     tcg_temp_free(t0);
     if (ra != 0 && ra != rd)
         tcg_gen_mov_tl(cpu_gpr[ra], cpu_gpr[rd]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_rac(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
     gen_helper_rac(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_rfsvc(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
+
     gen_helper_rfsvc(cpu_env);
     gen_sync_exception(ctx);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* svc is not implemented for now */
@@ -5841,18 +6090,16 @@
 static void gen_tlbiva(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
-    gen_helper_tlbie(cpu_env, cpu_gpr[rB(ctx->opcode)]);
+    gen_helper_tlbiva(cpu_env, cpu_gpr[rB(ctx->opcode)]);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* All 405 MAC instructions are translated here */
@@ -5933,7 +6180,7 @@
 
         if (opc3 & 0x12) {
             /* Check overflow and/or saturate */
-            int l1 = gen_new_label();
+            TCGLabel *l1 = gen_new_label();
 
             if (opc3 & 0x10) {
                 /* Start with XER OV disabled, the most likely case */
@@ -6074,38 +6321,34 @@
 static void gen_mfdcr(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv dcrn;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     /* NIP cannot be restored if the memory exception comes from an helper */
     gen_update_nip(ctx, ctx->nip - 4);
     dcrn = tcg_const_tl(SPR(ctx->opcode));
     gen_helper_load_dcr(cpu_gpr[rD(ctx->opcode)], cpu_env, dcrn);
     tcg_temp_free(dcrn);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mtdcr */
 static void gen_mtdcr(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
     TCGv dcrn;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+
+    CHK_SV;
     /* NIP cannot be restored if the memory exception comes from an helper */
     gen_update_nip(ctx, ctx->nip - 4);
     dcrn = tcg_const_tl(SPR(ctx->opcode));
     gen_helper_store_dcr(cpu_env, dcrn, cpu_gpr[rS(ctx->opcode)]);
     tcg_temp_free(dcrn);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mfdcrx */
@@ -6113,18 +6356,15 @@
 static void gen_mfdcrx(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
     /* NIP cannot be restored if the memory exception comes from an helper */
     gen_update_nip(ctx, ctx->nip - 4);
     gen_helper_load_dcr(cpu_gpr[rD(ctx->opcode)], cpu_env,
                         cpu_gpr[rA(ctx->opcode)]);
     /* Note: Rc update flag set leads to undefined state of Rc0 */
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mtdcrx */
@@ -6132,18 +6372,15 @@
 static void gen_mtdcrx(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_REG);
-        return;
-    }
+    CHK_SV;
     /* NIP cannot be restored if the memory exception comes from an helper */
     gen_update_nip(ctx, ctx->nip - 4);
     gen_helper_store_dcr(cpu_env, cpu_gpr[rA(ctx->opcode)],
                          cpu_gpr[rS(ctx->opcode)]);
     /* Note: Rc update flag set leads to undefined state of Rc0 */
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* mfdcrux (PPC 460) : user-mode access to DCR */
@@ -6169,28 +6406,19 @@
 /* dccci */
 static void gen_dccci(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-#else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* interpreted as no-op */
-#endif
 }
 
 /* dcread */
 static void gen_dcread(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv EA, val;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     gen_set_access_type(ctx, ACCESS_CACHE);
     EA = tcg_temp_new();
     gen_addr_reg_index(ctx, EA);
@@ -6199,7 +6427,7 @@
     tcg_temp_free(val);
     tcg_gen_mov_tl(cpu_gpr[rD(ctx->opcode)], EA);
     tcg_temp_free(EA);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* icbt */
@@ -6214,60 +6442,40 @@
 /* iccci */
 static void gen_iccci(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-#else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* interpreted as no-op */
-#endif
 }
 
 /* icread */
 static void gen_icread(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-#else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* interpreted as no-op */
-#endif
 }
 
 /* rfci (supervisor only) */
 static void gen_rfci_40x(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* Restore CPU state */
     gen_helper_40x_rfci(cpu_env);
     gen_sync_exception(ctx);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_rfci(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* Restore CPU state */
     gen_helper_rfci(cpu_env);
     gen_sync_exception(ctx);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* BookE specific */
@@ -6276,32 +6484,26 @@
 static void gen_rfdi(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* Restore CPU state */
     gen_helper_rfdi(cpu_env);
     gen_sync_exception(ctx);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* XXX: not implemented on 440 ? */
 static void gen_rfmci(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     /* Restore CPU state */
     gen_helper_rfmci(cpu_env);
     gen_sync_exception(ctx);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* TLB management - PowerPC 405 implementation */
@@ -6310,12 +6512,9 @@
 static void gen_tlbre_40x(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     switch (rB(ctx->opcode)) {
     case 0:
         gen_helper_4xx_tlbre_hi(cpu_gpr[rD(ctx->opcode)], cpu_env,
@@ -6329,44 +6528,40 @@
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
         break;
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbsx - tlbsx. */
 static void gen_tlbsx_40x(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
     gen_helper_4xx_tlbsx(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
     if (Rc(ctx->opcode)) {
-        int l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
         tcg_gen_trunc_tl_i32(cpu_crf[0], cpu_so);
         tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr[rD(ctx->opcode)], -1, l1);
         tcg_gen_ori_i32(cpu_crf[0], cpu_crf[0], 0x02);
         gen_set_label(l1);
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbwe */
 static void gen_tlbwe_40x(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
+
     switch (rB(ctx->opcode)) {
     case 0:
         gen_helper_4xx_tlbwe_hi(cpu_env, cpu_gpr[rA(ctx->opcode)],
@@ -6380,7 +6575,7 @@
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
         break;
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* TLB management - PowerPC 440 implementation */
@@ -6389,12 +6584,10 @@
 static void gen_tlbre_440(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
+
     switch (rB(ctx->opcode)) {
     case 0:
     case 1:
@@ -6410,44 +6603,39 @@
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
         break;
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbsx - tlbsx. */
 static void gen_tlbsx_440(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
     gen_helper_440_tlbsx(cpu_gpr[rD(ctx->opcode)], cpu_env, t0);
     tcg_temp_free(t0);
     if (Rc(ctx->opcode)) {
-        int l1 = gen_new_label();
+        TCGLabel *l1 = gen_new_label();
         tcg_gen_trunc_tl_i32(cpu_crf[0], cpu_so);
         tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr[rD(ctx->opcode)], -1, l1);
         tcg_gen_ori_i32(cpu_crf[0], cpu_crf[0], 0x02);
         gen_set_label(l1);
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbwe */
 static void gen_tlbwe_440(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     switch (rB(ctx->opcode)) {
     case 0:
     case 1:
@@ -6463,7 +6651,7 @@
         gen_inval_exception(ctx, POWERPC_EXCP_INVAL_INVAL);
         break;
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* TLB management - PowerPC BookE 2.06 implementation */
@@ -6471,30 +6659,23 @@
 /* tlbre */
 static void gen_tlbre_booke206(DisasContext *ctx)
 {
-#if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+ #if defined(CONFIG_USER_ONLY)
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
-
+   CHK_SV;
     gen_helper_booke206_tlbre(cpu_env);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbsx - tlbsx. */
 static void gen_tlbsx_booke206(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
 
+    CHK_SV;
     if (rA(ctx->opcode)) {
         t0 = tcg_temp_new();
         tcg_gen_mov_tl(t0, cpu_gpr[rD(ctx->opcode)]);
@@ -6505,54 +6686,44 @@
     tcg_gen_add_tl(t0, t0, cpu_gpr[rB(ctx->opcode)]);
     gen_helper_booke206_tlbsx(cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* tlbwe */
 static void gen_tlbwe_booke206(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     gen_update_nip(ctx, ctx->nip - 4);
     gen_helper_booke206_tlbwe(cpu_env);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_tlbivax_booke206(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
 
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
-
     gen_helper_booke206_tlbivax(cpu_env, t0);
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_tlbilx_booke206(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
 
+    CHK_SV;
     t0 = tcg_temp_new();
     gen_addr_reg_index(ctx, t0);
 
@@ -6572,7 +6743,7 @@
     }
 
     tcg_temp_free(t0);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 
@@ -6580,13 +6751,11 @@
 static void gen_wrtee(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
     TCGv t0;
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+
+    CHK_SV;
     t0 = tcg_temp_new();
     tcg_gen_andi_tl(t0, cpu_gpr[rD(ctx->opcode)], (1 << MSR_EE));
     tcg_gen_andi_tl(cpu_msr, cpu_msr, ~(1 << MSR_EE));
@@ -6596,19 +6765,16 @@
      * if we just set msr_ee to 1
      */
     gen_stop_exception(ctx);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* wrteei */
 static void gen_wrteei(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
+    CHK_SV;
     if (ctx->opcode & 0x00008000) {
         tcg_gen_ori_tl(cpu_msr, cpu_msr, (1 << MSR_EE));
         /* Stop translation to have a chance to raise an exception */
@@ -6616,7 +6782,7 @@
     } else {
         tcg_gen_andi_tl(cpu_msr, cpu_msr, ~(1 << MSR_EE));
     }
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /* PowerPC 440 specific instructions */
@@ -6656,29 +6822,21 @@
 static void gen_msgclr(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
-
+    CHK_SV;
     gen_helper_msgclr(cpu_env, cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 static void gen_msgsnd(DisasContext *ctx)
 {
 #if defined(CONFIG_USER_ONLY)
-    gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
+    GEN_PRIV;
 #else
-    if (unlikely(ctx->pr)) {
-        gen_inval_exception(ctx, POWERPC_EXCP_PRIV_OPC);
-        return;
-    }
-
+    CHK_SV;
     gen_helper_msgsnd(cpu_gpr[rB(ctx->opcode)]);
-#endif
+#endif /* defined(CONFIG_USER_ONLY) */
 }
 
 /***                      Altivec vector extension                         ***/
@@ -6743,7 +6901,7 @@
     tcg_temp_free(EA);                                                        \
 }
 
-#define GEN_VR_LVE(name, opc2, opc3)                                    \
+#define GEN_VR_LVE(name, opc2, opc3, size)                              \
 static void gen_lve##name(DisasContext *ctx)                            \
     {                                                                   \
         TCGv EA;                                                        \
@@ -6755,13 +6913,16 @@
         gen_set_access_type(ctx, ACCESS_INT);                           \
         EA = tcg_temp_new();                                            \
         gen_addr_reg_index(ctx, EA);                                    \
+        if (size > 1) {                                                 \
+            tcg_gen_andi_tl(EA, EA, ~(size - 1));                       \
+        }                                                               \
         rs = gen_avr_ptr(rS(ctx->opcode));                              \
         gen_helper_lve##name(cpu_env, rs, EA);                          \
         tcg_temp_free(EA);                                              \
         tcg_temp_free_ptr(rs);                                          \
     }
 
-#define GEN_VR_STVE(name, opc2, opc3)                                   \
+#define GEN_VR_STVE(name, opc2, opc3, size)                             \
 static void gen_stve##name(DisasContext *ctx)                           \
     {                                                                   \
         TCGv EA;                                                        \
@@ -6773,6 +6934,9 @@
         gen_set_access_type(ctx, ACCESS_INT);                           \
         EA = tcg_temp_new();                                            \
         gen_addr_reg_index(ctx, EA);                                    \
+        if (size > 1) {                                                 \
+            tcg_gen_andi_tl(EA, EA, ~(size - 1));                       \
+        }                                                               \
         rs = gen_avr_ptr(rS(ctx->opcode));                              \
         gen_helper_stve##name(cpu_env, rs, EA);                         \
         tcg_temp_free(EA);                                              \
@@ -6783,17 +6947,17 @@
 /* As we don't emulate the cache, lvxl is stricly equivalent to lvx */
 GEN_VR_LDX(lvxl, 0x07, 0x0B);
 
-GEN_VR_LVE(bx, 0x07, 0x00);
-GEN_VR_LVE(hx, 0x07, 0x01);
-GEN_VR_LVE(wx, 0x07, 0x02);
+GEN_VR_LVE(bx, 0x07, 0x00, 1);
+GEN_VR_LVE(hx, 0x07, 0x01, 2);
+GEN_VR_LVE(wx, 0x07, 0x02, 4);
 
 GEN_VR_STX(svx, 0x07, 0x07);
 /* As we don't emulate the cache, stvxl is stricly equivalent to stvx */
 GEN_VR_STX(svxl, 0x07, 0x0F);
 
-GEN_VR_STVE(bx, 0x07, 0x04);
-GEN_VR_STVE(hx, 0x07, 0x05);
-GEN_VR_STVE(wx, 0x07, 0x06);
+GEN_VR_STVE(bx, 0x07, 0x04, 1);
+GEN_VR_STVE(hx, 0x07, 0x05, 2);
+GEN_VR_STVE(wx, 0x07, 0x06, 4);
 
 static void gen_lvsl(DisasContext *ctx)
 {
@@ -8205,21 +8369,6 @@
     return r;
 }
 
-#if defined(TARGET_PPC64)
-static void gen_set_cr1_from_fpscr(DisasContext *ctx)
-{
-    TCGv_i32 tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_tl_i32(tmp, cpu_fpscr);
-    tcg_gen_shri_i32(cpu_crf[1], tmp, 28);
-    tcg_temp_free_i32(tmp);
-}
-#else
-static void gen_set_cr1_from_fpscr(DisasContext *ctx)
-{
-        tcg_gen_shri_tl(cpu_crf[1], cpu_fpscr, 28);
-}
-#endif
-
 #define GEN_DFP_T_A_B_Rc(name)                   \
 static void gen_##name(DisasContext *ctx)        \
 {                                                \
@@ -8539,8 +8688,8 @@
 
 static inline void gen_op_evabs(TCGv_i32 ret, TCGv_i32 arg1)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
 
     tcg_gen_brcondi_i32(TCG_COND_GE, arg1, 0, l1);
     tcg_gen_neg_i32(ret, arg1);
@@ -8589,12 +8738,10 @@
 
 static inline void gen_op_evsrwu(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    TCGv_i32 t0;
-    int l1, l2;
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
+    TCGv_i32 t0 = tcg_temp_local_new_i32();
 
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-    t0 = tcg_temp_local_new_i32();
     /* No error here: 6 bits are used */
     tcg_gen_andi_i32(t0, arg2, 0x3F);
     tcg_gen_brcondi_i32(TCG_COND_GE, t0, 32, l1);
@@ -8608,12 +8755,10 @@
 GEN_SPEOP_ARITH2(evsrwu, gen_op_evsrwu);
 static inline void gen_op_evsrws(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    TCGv_i32 t0;
-    int l1, l2;
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
+    TCGv_i32 t0 = tcg_temp_local_new_i32();
 
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-    t0 = tcg_temp_local_new_i32();
     /* No error here: 6 bits are used */
     tcg_gen_andi_i32(t0, arg2, 0x3F);
     tcg_gen_brcondi_i32(TCG_COND_GE, t0, 32, l1);
@@ -8627,12 +8772,10 @@
 GEN_SPEOP_ARITH2(evsrws, gen_op_evsrws);
 static inline void gen_op_evslw(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    TCGv_i32 t0;
-    int l1, l2;
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
+    TCGv_i32 t0 = tcg_temp_local_new_i32();
 
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-    t0 = tcg_temp_local_new_i32();
     /* No error here: 6 bits are used */
     tcg_gen_andi_i32(t0, arg2, 0x3F);
     tcg_gen_brcondi_i32(TCG_COND_GE, t0, 32, l1);
@@ -8700,10 +8843,10 @@
         gen_exception(ctx, POWERPC_EXCP_SPEU);                                \
         return;                                                               \
     }                                                                         \
-    int l1 = gen_new_label();                                                 \
-    int l2 = gen_new_label();                                                 \
-    int l3 = gen_new_label();                                                 \
-    int l4 = gen_new_label();                                                 \
+    TCGLabel *l1 = gen_new_label();                                           \
+    TCGLabel *l2 = gen_new_label();                                           \
+    TCGLabel *l3 = gen_new_label();                                           \
+    TCGLabel *l4 = gen_new_label();                                           \
                                                                               \
     tcg_gen_ext32s_tl(cpu_gpr[rA(ctx->opcode)], cpu_gpr[rA(ctx->opcode)]);    \
     tcg_gen_ext32s_tl(cpu_gpr[rB(ctx->opcode)], cpu_gpr[rB(ctx->opcode)]);    \
@@ -8793,11 +8936,12 @@
 
 static inline void gen_evsel(DisasContext *ctx)
 {
-    int l1 = gen_new_label();
-    int l2 = gen_new_label();
-    int l3 = gen_new_label();
-    int l4 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
+    TCGLabel *l2 = gen_new_label();
+    TCGLabel *l3 = gen_new_label();
+    TCGLabel *l4 = gen_new_label();
     TCGv_i32 t0 = tcg_temp_local_new_i32();
+
     tcg_gen_andi_i32(t0, cpu_crf[ctx->opcode & 0x07], 1 << 3);
     tcg_gen_brcondi_i32(TCG_COND_EQ, t0, 0, l1);
     tcg_gen_mov_tl(cpu_gprh[rD(ctx->opcode)], cpu_gprh[rA(ctx->opcode)]);
@@ -9642,6 +9786,85 @@
 GEN_SPE(efdtstgt,  efdtstlt,  0x1E, 0x0B, 0x00600000, 0x00600000, PPC_SPE_DOUBLE); //
 GEN_SPE(efdtsteq,  speundef,  0x1F, 0x0B, 0x00600000, 0xFFFFFFFF, PPC_SPE_DOUBLE); //
 
+static void gen_tbegin(DisasContext *ctx)
+{
+    if (unlikely(!ctx->tm_enabled)) {
+        gen_exception_err(ctx, POWERPC_EXCP_FU, FSCR_IC_TM);
+        return;
+    }
+    gen_helper_tbegin(cpu_env);
+}
+
+#define GEN_TM_NOOP(name)                                      \
+static inline void gen_##name(DisasContext *ctx)               \
+{                                                              \
+    if (unlikely(!ctx->tm_enabled)) {                          \
+        gen_exception_err(ctx, POWERPC_EXCP_FU, FSCR_IC_TM);   \
+        return;                                                \
+    }                                                          \
+    /* Because tbegin always fails in QEMU, these user         \
+     * space instructions all have a simple implementation:    \
+     *                                                         \
+     *     CR[0] = 0b0 || MSR[TS] || 0b0                       \
+     *           = 0b0 || 0b00    || 0b0                       \
+     */                                                        \
+    tcg_gen_movi_i32(cpu_crf[0], 0);                           \
+}
+
+GEN_TM_NOOP(tend);
+GEN_TM_NOOP(tabort);
+GEN_TM_NOOP(tabortwc);
+GEN_TM_NOOP(tabortwci);
+GEN_TM_NOOP(tabortdc);
+GEN_TM_NOOP(tabortdci);
+GEN_TM_NOOP(tsr);
+
+static void gen_tcheck(DisasContext *ctx)
+{
+    if (unlikely(!ctx->tm_enabled)) {
+        gen_exception_err(ctx, POWERPC_EXCP_FU, FSCR_IC_TM);
+        return;
+    }
+    /* Because tbegin always fails, the tcheck implementation
+     * is simple:
+     *
+     * CR[CRF] = TDOOMED || MSR[TS] || 0b0
+     *         = 0b1 || 0b00 || 0b0
+     */
+    tcg_gen_movi_i32(cpu_crf[crfD(ctx->opcode)], 0x8);
+}
+
+#if defined(CONFIG_USER_ONLY)
+#define GEN_TM_PRIV_NOOP(name)                                 \
+static inline void gen_##name(DisasContext *ctx)               \
+{                                                              \
+    gen_priv_exception(ctx, POWERPC_EXCP_PRIV_OPC);           \
+}
+
+#else
+
+#define GEN_TM_PRIV_NOOP(name)                                 \
+static inline void gen_##name(DisasContext *ctx)               \
+{                                                              \
+    CHK_SV;                                                    \
+    if (unlikely(!ctx->tm_enabled)) {                          \
+        gen_exception_err(ctx, POWERPC_EXCP_FU, FSCR_IC_TM);   \
+        return;                                                \
+    }                                                          \
+    /* Because tbegin always fails, the implementation is      \
+     * simple:                                                 \
+     *                                                         \
+     *   CR[0] = 0b0 || MSR[TS] || 0b0                         \
+     *         = 0b0 || 0b00 | 0b0                             \
+     */                                                        \
+    tcg_gen_movi_i32(cpu_crf[0], 0);                           \
+}
+
+#endif
+
+GEN_TM_PRIV_NOOP(treclaim);
+GEN_TM_PRIV_NOOP(trechkpt);
+
 static opcode_t opcodes[] = {
 GEN_HANDLER(invalid, 0x00, 0x00, 0x00, 0xFFFFFFFF, PPC_NONE),
 GEN_HANDLER(cmp, 0x1F, 0x00, 0x00, 0x00400000, PPC_INTEGER),
@@ -9751,6 +9974,10 @@
 GEN_HANDLER(rfi, 0x13, 0x12, 0x01, 0x03FF8001, PPC_FLOW),
 #if defined(TARGET_PPC64)
 GEN_HANDLER(rfid, 0x13, 0x12, 0x00, 0x03FF8001, PPC_64B),
+GEN_HANDLER_E(doze, 0x13, 0x12, 0x0c, 0x03FFF801, PPC_NONE, PPC2_PM_ISA206),
+GEN_HANDLER_E(nap, 0x13, 0x12, 0x0d, 0x03FFF801, PPC_NONE, PPC2_PM_ISA206),
+GEN_HANDLER_E(sleep, 0x13, 0x12, 0x0e, 0x03FFF801, PPC_NONE, PPC2_PM_ISA206),
+GEN_HANDLER_E(rvwinkle, 0x13, 0x12, 0x0f, 0x03FFF801, PPC_NONE, PPC2_PM_ISA206),
 GEN_HANDLER(hrfid, 0x13, 0x12, 0x08, 0x03FF8001, PPC_64H),
 #endif
 GEN_HANDLER(sc, 0x11, 0xFF, 0xFF, 0x03FFF01D, PPC_FLOW),
@@ -9769,8 +9996,8 @@
 #if defined(TARGET_PPC64)
 GEN_HANDLER(mtmsrd, 0x1F, 0x12, 0x05, 0x001EF801, PPC_64B),
 #endif
-GEN_HANDLER(mtmsr, 0x1F, 0x12, 0x04, 0x001FF801, PPC_MISC),
-GEN_HANDLER(mtspr, 0x1F, 0x13, 0x0E, 0x00000001, PPC_MISC),
+GEN_HANDLER(mtmsr, 0x1F, 0x12, 0x04, 0x001EF801, PPC_MISC),
+GEN_HANDLER(mtspr, 0x1F, 0x13, 0x0E, 0x00000000, PPC_MISC),
 GEN_HANDLER(dcbf, 0x1F, 0x16, 0x02, 0x03C00001, PPC_CACHE),
 GEN_HANDLER(dcbi, 0x1F, 0x16, 0x0E, 0x03E00001, PPC_CACHE),
 GEN_HANDLER(dcbst, 0x1F, 0x16, 0x01, 0x03E00001, PPC_CACHE),
@@ -9797,13 +10024,16 @@
 GEN_HANDLER2(slbmte, "slbmte", 0x1F, 0x12, 0x0C, 0x001F0001, PPC_SEGMENT_64B),
 GEN_HANDLER2(slbmfee, "slbmfee", 0x1F, 0x13, 0x1C, 0x001F0001, PPC_SEGMENT_64B),
 GEN_HANDLER2(slbmfev, "slbmfev", 0x1F, 0x13, 0x1A, 0x001F0001, PPC_SEGMENT_64B),
+GEN_HANDLER2(slbfee_, "slbfee.", 0x1F, 0x13, 0x1E, 0x001F0000, PPC_SEGMENT_64B),
 #endif
 GEN_HANDLER(tlbia, 0x1F, 0x12, 0x0B, 0x03FFFC01, PPC_MEM_TLBIA),
-GEN_HANDLER(tlbiel, 0x1F, 0x12, 0x08, 0x03FF0001, PPC_MEM_TLBIE),
-GEN_HANDLER(tlbie, 0x1F, 0x12, 0x09, 0x03FF0001, PPC_MEM_TLBIE),
+/* XXX Those instructions will need to be handled differently for
+ * different ISA versions */
+GEN_HANDLER(tlbiel, 0x1F, 0x12, 0x08, 0x001F0001, PPC_MEM_TLBIE),
+GEN_HANDLER(tlbie, 0x1F, 0x12, 0x09, 0x001F0001, PPC_MEM_TLBIE),
 GEN_HANDLER(tlbsync, 0x1F, 0x16, 0x11, 0x03FFF801, PPC_MEM_TLBSYNC),
 #if defined(TARGET_PPC64)
-GEN_HANDLER(slbia, 0x1F, 0x12, 0x0F, 0x03FFFC01, PPC_SLBI),
+GEN_HANDLER(slbia, 0x1F, 0x12, 0x0F, 0x031FFC01, PPC_SLBI),
 GEN_HANDLER(slbie, 0x1F, 0x12, 0x0D, 0x03FF0001, PPC_SLBI),
 #endif
 GEN_HANDLER(eciwx, 0x1F, 0x16, 0x0D, 0x00000001, PPC_EXTERN),
@@ -10100,7 +10330,7 @@
 GEN_HANDLER(name##u, opc, 0xFF, 0xFF, 0x00000000, type),
 #define GEN_LDUX(name, ldop, opc2, opc3, type)                                \
 GEN_HANDLER(name##ux, 0x1F, opc2, opc3, 0x00000001, type),
-#define GEN_LDX_E(name, ldop, opc2, opc3, type, type2)                        \
+#define GEN_LDX_E(name, ldop, opc2, opc3, type, type2, chk)                   \
 GEN_HANDLER_E(name##x, 0x1F, opc2, opc3, 0x00000001, type, type2),
 #define GEN_LDS(name, ldop, op, type)                                         \
 GEN_LD(name, ldop, op | 0x20, type)                                           \
@@ -10117,7 +10347,13 @@
 GEN_LDX(lwa, ld32s, 0x15, 0x0A, PPC_64B)
 GEN_LDUX(ld, ld64, 0x15, 0x01, PPC_64B)
 GEN_LDX(ld, ld64, 0x15, 0x00, PPC_64B)
-GEN_LDX_E(ldbr, ld64ur, 0x14, 0x10, PPC_NONE, PPC2_DBRX)
+GEN_LDX_E(ldbr, ld64ur, 0x14, 0x10, PPC_NONE, PPC2_DBRX, CHK_NONE)
+
+/* HV/P7 and later only */
+GEN_LDX_HVRM(ldcix, ld64, 0x15, 0x1b, PPC_CILDST)
+GEN_LDX_HVRM(lwzcix, ld32u, 0x15, 0x18, PPC_CILDST)
+GEN_LDX_HVRM(lhzcix, ld16u, 0x15, 0x19, PPC_CILDST)
+GEN_LDX_HVRM(lbzcix, ld8u, 0x15, 0x1a, PPC_CILDST)
 #endif
 GEN_LDX(lhbr, ld16ur, 0x16, 0x18, PPC_INTEGER)
 GEN_LDX(lwbr, ld32ur, 0x16, 0x10, PPC_INTEGER)
@@ -10133,7 +10369,7 @@
 GEN_HANDLER(stop##u, opc, 0xFF, 0xFF, 0x00000000, type),
 #define GEN_STUX(name, stop, opc2, opc3, type)                                \
 GEN_HANDLER(name##ux, 0x1F, opc2, opc3, 0x00000001, type),
-#define GEN_STX_E(name, stop, opc2, opc3, type, type2)                        \
+#define GEN_STX_E(name, stop, opc2, opc3, type, type2, chk)                   \
 GEN_HANDLER_E(name##x, 0x1F, opc2, opc3, 0x00000001, type, type2),
 #define GEN_STS(name, stop, op, type)                                         \
 GEN_ST(name, stop, op | 0x20, type)                                           \
@@ -10147,7 +10383,11 @@
 #if defined(TARGET_PPC64)
 GEN_STUX(std, st64, 0x15, 0x05, PPC_64B)
 GEN_STX(std, st64, 0x15, 0x04, PPC_64B)
-GEN_STX_E(stdbr, st64r, 0x14, 0x14, PPC_NONE, PPC2_DBRX)
+GEN_STX_E(stdbr, st64r, 0x14, 0x14, PPC_NONE, PPC2_DBRX, CHK_NONE)
+GEN_STX_HVRM(stdcix, st64, 0x15, 0x1f, PPC_CILDST)
+GEN_STX_HVRM(stwcix, st32, 0x15, 0x1c, PPC_CILDST)
+GEN_STX_HVRM(sthcix, st16, 0x15, 0x1d, PPC_CILDST)
+GEN_STX_HVRM(stbcix, st8, 0x15, 0x1e, PPC_CILDST)
 #endif
 GEN_STX(sthbr, st16r, 0x16, 0x1C, PPC_INTEGER)
 GEN_STX(stwbr, st32r, 0x16, 0x14, PPC_INTEGER)
@@ -10556,6 +10796,13 @@
 GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 2, opc3, 0, PPC_NONE, fl2), \
 GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 3, opc3, 0, PPC_NONE, fl2)
 
+#undef GEN_XX2IFORM
+#define GEN_XX2IFORM(name, opc2, opc3, fl2)                           \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 0, opc3, 1, PPC_NONE, fl2), \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 1, opc3, 1, PPC_NONE, fl2), \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 2, opc3, 1, PPC_NONE, fl2), \
+GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 3, opc3, 1, PPC_NONE, fl2)
+
 #undef GEN_XX3_RC_FORM
 #define GEN_XX3_RC_FORM(name, opc2, opc3, fl2)                          \
 GEN_HANDLER2_E(name, #name, 0x3C, opc2 | 0x00, opc3 | 0x00, 0, PPC_NONE, fl2), \
@@ -10617,8 +10864,8 @@
 GEN_XX3FORM(xsnmaddmdp, 0x04, 0x15, PPC2_VSX),
 GEN_XX3FORM(xsnmsubadp, 0x04, 0x16, PPC2_VSX),
 GEN_XX3FORM(xsnmsubmdp, 0x04, 0x17, PPC2_VSX),
-GEN_XX2FORM(xscmpodp,  0x0C, 0x05, PPC2_VSX),
-GEN_XX2FORM(xscmpudp,  0x0C, 0x04, PPC2_VSX),
+GEN_XX2IFORM(xscmpodp,  0x0C, 0x05, PPC2_VSX),
+GEN_XX2IFORM(xscmpudp,  0x0C, 0x04, PPC2_VSX),
 GEN_XX3FORM(xsmaxdp, 0x00, 0x14, PPC2_VSX),
 GEN_XX3FORM(xsmindp, 0x00, 0x15, PPC2_VSX),
 GEN_XX2FORM(xscvdpsp, 0x12, 0x10, PPC2_VSX),
@@ -11054,6 +11301,29 @@
 GEN_SPEOP_LDST(evstwho, 0x1A, 2),
 GEN_SPEOP_LDST(evstwwe, 0x1C, 2),
 GEN_SPEOP_LDST(evstwwo, 0x1E, 2),
+
+GEN_HANDLER2_E(tbegin, "tbegin", 0x1F, 0x0E, 0x14, 0x01DFF800, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tend,   "tend",   0x1F, 0x0E, 0x15, 0x01FFF800, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tabort, "tabort", 0x1F, 0x0E, 0x1C, 0x03E0F800, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tabortwc, "tabortwc", 0x1F, 0x0E, 0x18, 0x00000000, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tabortwci, "tabortwci", 0x1F, 0x0E, 0x1A, 0x00000000, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tabortdc, "tabortdc", 0x1F, 0x0E, 0x19, 0x00000000, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tabortdci, "tabortdci", 0x1F, 0x0E, 0x1B, 0x00000000, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tsr, "tsr", 0x1F, 0x0E, 0x17, 0x03DFF800, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(tcheck, "tcheck", 0x1F, 0x0E, 0x16, 0x007FF800, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(treclaim, "treclaim", 0x1F, 0x0E, 0x1D, 0x03E0F800, \
+               PPC_NONE, PPC2_TM),
+GEN_HANDLER2_E(trechkpt, "trechkpt", 0x1F, 0x0E, 0x1F, 0x03FFF800, \
+               PPC_NONE, PPC2_TM),
 };
 
 #include "helper_regs.h"
@@ -11072,11 +11342,13 @@
     int i;
 
     cpu_fprintf(f, "NIP " TARGET_FMT_lx "   LR " TARGET_FMT_lx " CTR "
-                TARGET_FMT_lx " XER " TARGET_FMT_lx "\n",
-                env->nip, env->lr, env->ctr, cpu_read_xer(env));
+                TARGET_FMT_lx " XER " TARGET_FMT_lx " CPU#%d\n",
+                env->nip, env->lr, env->ctr, cpu_read_xer(env),
+                cs->cpu_index);
     cpu_fprintf(f, "MSR " TARGET_FMT_lx " HID0 " TARGET_FMT_lx "  HF "
-                TARGET_FMT_lx " idx %d\n", env->msr, env->spr[SPR_HID0],
-                env->hflags, env->mmu_idx);
+                TARGET_FMT_lx " iidx %d didx %d\n",
+                env->msr, env->spr[SPR_HID0],
+                env->hflags, env->immu_idx, env->dmmu_idx);
 #if !defined(NO_TIMER_DUMP)
     cpu_fprintf(f, "TB %08" PRIu32 " %08" PRIu64
 #if !defined(CONFIG_USER_ONLY)
@@ -11136,6 +11408,13 @@
                 env->spr[SPR_SPRG4], env->spr[SPR_SPRG5],
                 env->spr[SPR_SPRG6], env->spr[SPR_SPRG7]);
 
+#if defined(TARGET_PPC64)
+    if (env->excp_model == POWERPC_EXCP_POWER7 ||
+        env->excp_model == POWERPC_EXCP_POWER8) {
+        cpu_fprintf(f, "HSRR0 " TARGET_FMT_lx " HSRR1 " TARGET_FMT_lx "\n",
+                    env->spr[SPR_HSRR0], env->spr[SPR_HSRR1]);
+    }
+#endif
     if (env->excp_model == POWERPC_EXCP_BOOKE) {
         cpu_fprintf(f, "CSRR0 " TARGET_FMT_lx " CSRR1 " TARGET_FMT_lx
                        " MCSRR0 " TARGET_FMT_lx " MCSRR1 " TARGET_FMT_lx "\n",
@@ -11182,9 +11461,11 @@
     case POWERPC_MMU_SOFT_74xx:
 #if defined(TARGET_PPC64)
     case POWERPC_MMU_64B:
+    case POWERPC_MMU_2_03:
     case POWERPC_MMU_2_06:
     case POWERPC_MMU_2_06a:
-    case POWERPC_MMU_2_06d:
+    case POWERPC_MMU_2_07:
+    case POWERPC_MMU_2_07a:
 #endif
         cpu_fprintf(f, " SDR1 " TARGET_FMT_lx "   DAR " TARGET_FMT_lx
                        "  DSISR " TARGET_FMT_lx "\n", env->spr[SPR_SDR1],
@@ -11264,17 +11545,13 @@
 }
 
 /*****************************************************************************/
-static inline void gen_intermediate_code_internal(PowerPCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUPPCState *env, struct TranslationBlock *tb)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUPPCState *env = &cpu->env;
     DisasContext ctx, *ctxp = &ctx;
     opc_handler_t **table, *handler;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
 
@@ -11284,31 +11561,46 @@
     ctx.exception = POWERPC_EXCP_NONE;
     ctx.spr_cb = env->spr_cb;
     ctx.pr = msr_pr;
-    ctx.hv = !msr_pr && msr_hv;
-    ctx.mem_idx = env->mmu_idx;
+    ctx.mem_idx = env->dmmu_idx;
+    ctx.dr = msr_dr;
+#if !defined(CONFIG_USER_ONLY)
+    ctx.hv = msr_hv || !env->has_hv_mode;
+#endif
     ctx.insns_flags = env->insns_flags;
     ctx.insns_flags2 = env->insns_flags2;
     ctx.access_type = -1;
-    ctx.le_mode = env->hflags & (1 << MSR_LE) ? 1 : 0;
+    ctx.le_mode = !!(env->hflags & (1 << MSR_LE));
     ctx.default_tcg_memop_mask = ctx.le_mode ? MO_LE : MO_BE;
 #if defined(TARGET_PPC64)
     ctx.sf_mode = msr_is_64bit(env, env->msr);
     ctx.has_cfar = !!(env->flags & POWERPC_FLAG_CFAR);
 #endif
-    ctx.fpu_enabled = msr_fp;
+    if (env->mmu_model == POWERPC_MMU_32B ||
+        env->mmu_model == POWERPC_MMU_601 ||
+        (env->mmu_model & POWERPC_MMU_64B))
+            ctx.lazy_tlb_flush = true;
+
+    ctx.fpu_enabled = !!msr_fp;
     if ((env->flags & POWERPC_FLAG_SPE) && msr_spe)
-        ctx.spe_enabled = msr_spe;
+        ctx.spe_enabled = !!msr_spe;
     else
-        ctx.spe_enabled = 0;
+        ctx.spe_enabled = false;
     if ((env->flags & POWERPC_FLAG_VRE) && msr_vr)
-        ctx.altivec_enabled = msr_vr;
+        ctx.altivec_enabled = !!msr_vr;
     else
-        ctx.altivec_enabled = 0;
+        ctx.altivec_enabled = false;
     if ((env->flags & POWERPC_FLAG_VSX) && msr_vsx) {
-        ctx.vsx_enabled = msr_vsx;
+        ctx.vsx_enabled = !!msr_vsx;
     } else {
-        ctx.vsx_enabled = 0;
+        ctx.vsx_enabled = false;
     }
+#if defined(TARGET_PPC64)
+    if ((env->flags & POWERPC_FLAG_TM) && msr_tm) {
+        ctx.tm_enabled = !!msr_tm;
+    } else {
+        ctx.tm_enabled = false;
+    }
+#endif
     if ((env->flags & POWERPC_FLAG_SE) && msr_se)
         ctx.singlestep_enabled = CPU_SINGLE_STEP;
     else
@@ -11324,36 +11616,34 @@
 #endif
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
     tcg_clear_temp_count();
     /* Set env in case of segfault during code fetch */
     while (ctx.exception == POWERPC_EXCP_NONE && !tcg_op_buf_full()) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == ctx.nip) {
-                    gen_debug_exception(ctxp);
-                    break;
-                }
-            }
+        tcg_gen_insn_start(ctx.nip);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.nip, BP_ANY))) {
+            gen_debug_exception(ctxp);
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.nip += 4;
+            break;
         }
-        if (unlikely(search_pc)) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[lj] = ctx.nip;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
-        }
+
         LOG_DISAS("----------------\n");
         LOG_DISAS("nip=" TARGET_FMT_lx " super=%d ir=%d\n",
                   ctx.nip, ctx.mem_idx, (int)msr_ir);
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
         if (unlikely(need_byteswap(&ctx))) {
             ctx.opcode = bswap32(cpu_ldl_code(env, ctx.nip));
@@ -11363,12 +11653,8 @@
         LOG_DISAS("translate opcode %08x (%02x %02x %02x) (%s)\n",
                     ctx.opcode, opc1(ctx.opcode), opc2(ctx.opcode),
                     opc3(ctx.opcode), ctx.le_mode ? "little" : "big");
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(ctx.nip);
-        }
         ctx.nip += 4;
         table = env->opcodes;
-        num_insns++;
         handler = table[opc1(ctx.opcode)];
         if (is_indirect_opcode(handler)) {
             table = ind_table(handler);
@@ -11380,12 +11666,10 @@
         }
         /* Is opcode *REALLY* valid ? */
         if (unlikely(handler->handler == &gen_invalid)) {
-            if (qemu_log_enabled()) {
-                qemu_log("invalid/unsupported opcode: "
-                         "%02x - %02x - %02x (%08x) " TARGET_FMT_lx " %d\n",
-                         opc1(ctx.opcode), opc2(ctx.opcode),
-                         opc3(ctx.opcode), ctx.opcode, ctx.nip - 4, (int)msr_ir);
-            }
+            qemu_log_mask(LOG_GUEST_ERROR, "invalid/unsupported opcode: "
+                          "%02x - %02x - %02x (%08x) " TARGET_FMT_lx " %d\n",
+                          opc1(ctx.opcode), opc2(ctx.opcode),
+                          opc3(ctx.opcode), ctx.opcode, ctx.nip - 4, (int)msr_ir);
         } else {
             uint32_t inval;
 
@@ -11396,13 +11680,11 @@
             }
 
             if (unlikely((ctx.opcode & inval) != 0)) {
-                if (qemu_log_enabled()) {
-                    qemu_log("invalid bits: %08x for opcode: "
-                             "%02x - %02x - %02x (%08x) " TARGET_FMT_lx "\n",
-                             ctx.opcode & inval, opc1(ctx.opcode),
-                             opc2(ctx.opcode), opc3(ctx.opcode),
-                             ctx.opcode, ctx.nip - 4);
-                }
+                qemu_log_mask(LOG_GUEST_ERROR, "invalid bits: %08x for opcode: "
+                              "%02x - %02x - %02x (%08x) " TARGET_FMT_lx "\n",
+                              ctx.opcode & inval, opc1(ctx.opcode),
+                              opc2(ctx.opcode), opc3(ctx.opcode),
+                              ctx.opcode, ctx.nip - 4);
                 gen_inval_exception(ctxp, POWERPC_EXCP_INVAL_INVAL);
                 break;
             }
@@ -11447,38 +11729,24 @@
     }
     gen_tb_end(tb, num_insns);
 
-    if (unlikely(search_pc)) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-    } else {
-        tb->size = ctx.nip - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.nip - pc_start;
+    tb->icount = num_insns;
+
 #if defined(DEBUG_DISAS)
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         int flags;
         flags = env->bfd_mach;
         flags |= ctx.le_mode << 16;
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, ctx.nip - pc_start, flags);
+        log_target_disas(cs, pc_start, ctx.nip - pc_start, flags);
         qemu_log("\n");
     }
 #endif
 }
 
-void gen_intermediate_code (CPUPPCState *env, struct TranslationBlock *tb)
+void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(ppc_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUPPCState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(ppc_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUPPCState *env, TranslationBlock *tb, int pc_pos)
-{
-    env->nip = tcg_ctx.gen_opc_pc[pc_pos];
+    env->nip = data[0];
 }
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index a35de72..7a9b15e 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -18,9 +18,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "disas/bfd.h"
 #include "exec/gdbstub.h"
-#include <sysemu/kvm.h>
+#include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/cpus.h"
@@ -30,29 +31,13 @@
 #include "qemu/error-report.h"
 #include "qapi/visitor.h"
 #include "hw/qdev-properties.h"
+#include "hw/ppc/ppc.h"
 
 //#define PPC_DUMP_CPU
 //#define PPC_DEBUG_SPR
 //#define PPC_DUMP_SPR_ACCESSES
 /* #define USE_APPLE_GDB */
 
-/* For user-mode emulation, we don't emulate any IRQ controller */
-#if defined(CONFIG_USER_ONLY)
-#define PPC_IRQ_INIT_FN(name)                                                 \
-static inline void glue(glue(ppc, name),_irq_init) (CPUPPCState *env)         \
-{                                                                             \
-}
-#else
-#define PPC_IRQ_INIT_FN(name)                                                 \
-void glue(glue(ppc, name),_irq_init) (CPUPPCState *env);
-#endif
-
-PPC_IRQ_INIT_FN(40x);
-PPC_IRQ_INIT_FN(6xx);
-PPC_IRQ_INIT_FN(970);
-PPC_IRQ_INIT_FN(POWER7);
-PPC_IRQ_INIT_FN(e500);
-
 /* Generic callbacks:
  * do nothing but store/retrieve spr value
  */
@@ -65,7 +50,7 @@
 #endif
 }
 
-static void spr_read_generic (void *opaque, int gprn, int sprn)
+static void spr_read_generic (DisasContext *ctx, int gprn, int sprn)
 {
     gen_load_spr(cpu_gpr[gprn], sprn);
     spr_load_dump_spr(sprn);
@@ -80,14 +65,14 @@
 #endif
 }
 
-static void spr_write_generic (void *opaque, int sprn, int gprn)
+static void spr_write_generic (DisasContext *ctx, int sprn, int gprn)
 {
     gen_store_spr(sprn, cpu_gpr[gprn]);
     spr_store_dump_spr(sprn);
 }
 
 #if !defined(CONFIG_USER_ONLY)
-static void spr_write_generic32(void *opaque, int sprn, int gprn)
+static void spr_write_generic32(DisasContext *ctx, int sprn, int gprn)
 {
 #ifdef TARGET_PPC64
     TCGv t0 = tcg_temp_new();
@@ -96,11 +81,11 @@
     tcg_temp_free(t0);
     spr_store_dump_spr(sprn);
 #else
-    spr_write_generic(opaque, sprn, gprn);
+    spr_write_generic(ctx, sprn, gprn);
 #endif
 }
 
-static void spr_write_clear (void *opaque, int sprn, int gprn)
+static void spr_write_clear (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
@@ -112,7 +97,7 @@
     tcg_temp_free(t1);
 }
 
-static void spr_access_nop(void *opaque, int sprn, int gprn)
+static void spr_access_nop(DisasContext *ctx, int sprn, int gprn)
 {
 }
 
@@ -120,47 +105,47 @@
 
 /* SPR common to all PowerPC */
 /* XER */
-static void spr_read_xer (void *opaque, int gprn, int sprn)
+static void spr_read_xer (DisasContext *ctx, int gprn, int sprn)
 {
     gen_read_xer(cpu_gpr[gprn]);
 }
 
-static void spr_write_xer (void *opaque, int sprn, int gprn)
+static void spr_write_xer (DisasContext *ctx, int sprn, int gprn)
 {
     gen_write_xer(cpu_gpr[gprn]);
 }
 
 /* LR */
-static void spr_read_lr (void *opaque, int gprn, int sprn)
+static void spr_read_lr (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_mov_tl(cpu_gpr[gprn], cpu_lr);
 }
 
-static void spr_write_lr (void *opaque, int sprn, int gprn)
+static void spr_write_lr (DisasContext *ctx, int sprn, int gprn)
 {
     tcg_gen_mov_tl(cpu_lr, cpu_gpr[gprn]);
 }
 
 /* CFAR */
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
-static void spr_read_cfar (void *opaque, int gprn, int sprn)
+static void spr_read_cfar (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_mov_tl(cpu_gpr[gprn], cpu_cfar);
 }
 
-static void spr_write_cfar (void *opaque, int sprn, int gprn)
+static void spr_write_cfar (DisasContext *ctx, int sprn, int gprn)
 {
     tcg_gen_mov_tl(cpu_cfar, cpu_gpr[gprn]);
 }
 #endif /* defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY) */
 
 /* CTR */
-static void spr_read_ctr (void *opaque, int gprn, int sprn)
+static void spr_read_ctr (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_mov_tl(cpu_gpr[gprn], cpu_ctr);
 }
 
-static void spr_write_ctr (void *opaque, int sprn, int gprn)
+static void spr_write_ctr (DisasContext *ctx, int sprn, int gprn)
 {
     tcg_gen_mov_tl(cpu_ctr, cpu_gpr[gprn]);
 }
@@ -171,13 +156,13 @@
 /* UPMCx */
 /* USIA */
 /* UDECR */
-static void spr_read_ureg (void *opaque, int gprn, int sprn)
+static void spr_read_ureg (DisasContext *ctx, int gprn, int sprn)
 {
     gen_load_spr(cpu_gpr[gprn], sprn + 0x10);
 }
 
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
-static void spr_write_ureg(void *opaque, int sprn, int gprn)
+static void spr_write_ureg(DisasContext *ctx, int sprn, int gprn)
 {
     gen_store_spr(sprn + 0x10, cpu_gpr[gprn]);
 }
@@ -186,150 +171,176 @@
 /* SPR common to all non-embedded PowerPC */
 /* DECR */
 #if !defined(CONFIG_USER_ONLY)
-static void spr_read_decr (void *opaque, int gprn, int sprn)
+static void spr_read_decr (DisasContext *ctx, int gprn, int sprn)
 {
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
     }
     gen_helper_load_decr(cpu_gpr[gprn], cpu_env);
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
-        gen_stop_exception(opaque);
+        gen_stop_exception(ctx);
     }
 }
 
-static void spr_write_decr (void *opaque, int sprn, int gprn)
+static void spr_write_decr (DisasContext *ctx, int sprn, int gprn)
 {
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
     }
     gen_helper_store_decr(cpu_env, cpu_gpr[gprn]);
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
-        gen_stop_exception(opaque);
+        gen_stop_exception(ctx);
     }
 }
 #endif
 
 /* SPR common to all non-embedded PowerPC, except 601 */
 /* Time base */
-static void spr_read_tbl (void *opaque, int gprn, int sprn)
+static void spr_read_tbl (DisasContext *ctx, int gprn, int sprn)
 {
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
     }
     gen_helper_load_tbl(cpu_gpr[gprn], cpu_env);
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
-        gen_stop_exception(opaque);
+        gen_stop_exception(ctx);
     }
 }
 
-static void spr_read_tbu (void *opaque, int gprn, int sprn)
+static void spr_read_tbu (DisasContext *ctx, int gprn, int sprn)
 {
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
     }
     gen_helper_load_tbu(cpu_gpr[gprn], cpu_env);
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
-        gen_stop_exception(opaque);
+        gen_stop_exception(ctx);
     }
 }
 
 __attribute__ (( unused ))
-static void spr_read_atbl (void *opaque, int gprn, int sprn)
+static void spr_read_atbl (DisasContext *ctx, int gprn, int sprn)
 {
     gen_helper_load_atbl(cpu_gpr[gprn], cpu_env);
 }
 
 __attribute__ (( unused ))
-static void spr_read_atbu (void *opaque, int gprn, int sprn)
+static void spr_read_atbu (DisasContext *ctx, int gprn, int sprn)
 {
     gen_helper_load_atbu(cpu_gpr[gprn], cpu_env);
 }
 
 #if !defined(CONFIG_USER_ONLY)
-static void spr_write_tbl (void *opaque, int sprn, int gprn)
+static void spr_write_tbl (DisasContext *ctx, int sprn, int gprn)
 {
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
     }
     gen_helper_store_tbl(cpu_env, cpu_gpr[gprn]);
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
-        gen_stop_exception(opaque);
+        gen_stop_exception(ctx);
     }
 }
 
-static void spr_write_tbu (void *opaque, int sprn, int gprn)
+static void spr_write_tbu (DisasContext *ctx, int sprn, int gprn)
 {
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_start();
     }
     gen_helper_store_tbu(cpu_env, cpu_gpr[gprn]);
-    if (use_icount) {
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
         gen_io_end();
-        gen_stop_exception(opaque);
+        gen_stop_exception(ctx);
     }
 }
 
 __attribute__ (( unused ))
-static void spr_write_atbl (void *opaque, int sprn, int gprn)
+static void spr_write_atbl (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_atbl(cpu_env, cpu_gpr[gprn]);
 }
 
 __attribute__ (( unused ))
-static void spr_write_atbu (void *opaque, int sprn, int gprn)
+static void spr_write_atbu (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_atbu(cpu_env, cpu_gpr[gprn]);
 }
 
 #if defined(TARGET_PPC64)
 __attribute__ (( unused ))
-static void spr_read_purr (void *opaque, int gprn, int sprn)
+static void spr_read_purr (DisasContext *ctx, int gprn, int sprn)
 {
     gen_helper_load_purr(cpu_gpr[gprn], cpu_env);
 }
+
+/* HDECR */
+static void spr_read_hdecr(DisasContext *ctx, int gprn, int sprn)
+{
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
+        gen_io_start();
+    }
+    gen_helper_load_hdecr(cpu_gpr[gprn], cpu_env);
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
+        gen_io_end();
+        gen_stop_exception(ctx);
+    }
+}
+
+static void spr_write_hdecr(DisasContext *ctx, int sprn, int gprn)
+{
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
+        gen_io_start();
+    }
+    gen_helper_store_hdecr(cpu_env, cpu_gpr[gprn]);
+    if (ctx->tb->cflags & CF_USE_ICOUNT) {
+        gen_io_end();
+        gen_stop_exception(ctx);
+    }
+}
+
 #endif
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
 /* IBAT0U...IBAT0U */
 /* IBAT0L...IBAT7L */
-static void spr_read_ibat (void *opaque, int gprn, int sprn)
+static void spr_read_ibat (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][(sprn - SPR_IBAT0U) / 2]));
 }
 
-static void spr_read_ibat_h (void *opaque, int gprn, int sprn)
+static void spr_read_ibat_h (DisasContext *ctx, int gprn, int sprn)
 {
-    tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][(sprn - SPR_IBAT4U) / 2]));
+    tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][((sprn - SPR_IBAT4U) / 2) + 4]));
 }
 
-static void spr_write_ibatu (void *opaque, int sprn, int gprn)
+static void spr_write_ibatu (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32((sprn - SPR_IBAT0U) / 2);
     gen_helper_store_ibatu(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_ibatu_h (void *opaque, int sprn, int gprn)
+static void spr_write_ibatu_h (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32(((sprn - SPR_IBAT4U) / 2) + 4);
     gen_helper_store_ibatu(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_ibatl (void *opaque, int sprn, int gprn)
+static void spr_write_ibatl (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32((sprn - SPR_IBAT0L) / 2);
     gen_helper_store_ibatl(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_ibatl_h (void *opaque, int sprn, int gprn)
+static void spr_write_ibatl_h (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32(((sprn - SPR_IBAT4L) / 2) + 4);
     gen_helper_store_ibatl(cpu_env, t0, cpu_gpr[gprn]);
@@ -338,38 +349,38 @@
 
 /* DBAT0U...DBAT7U */
 /* DBAT0L...DBAT7L */
-static void spr_read_dbat (void *opaque, int gprn, int sprn)
+static void spr_read_dbat (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, DBAT[sprn & 1][(sprn - SPR_DBAT0U) / 2]));
 }
 
-static void spr_read_dbat_h (void *opaque, int gprn, int sprn)
+static void spr_read_dbat_h (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, DBAT[sprn & 1][((sprn - SPR_DBAT4U) / 2) + 4]));
 }
 
-static void spr_write_dbatu (void *opaque, int sprn, int gprn)
+static void spr_write_dbatu (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32((sprn - SPR_DBAT0U) / 2);
     gen_helper_store_dbatu(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_dbatu_h (void *opaque, int sprn, int gprn)
+static void spr_write_dbatu_h (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32(((sprn - SPR_DBAT4U) / 2) + 4);
     gen_helper_store_dbatu(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_dbatl (void *opaque, int sprn, int gprn)
+static void spr_write_dbatl (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32((sprn - SPR_DBAT0L) / 2);
     gen_helper_store_dbatl(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_dbatl_h (void *opaque, int sprn, int gprn)
+static void spr_write_dbatl_h (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32(((sprn - SPR_DBAT4L) / 2) + 4);
     gen_helper_store_dbatl(cpu_env, t0, cpu_gpr[gprn]);
@@ -377,19 +388,19 @@
 }
 
 /* SDR1 */
-static void spr_write_sdr1 (void *opaque, int sprn, int gprn)
+static void spr_write_sdr1 (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_sdr1(cpu_env, cpu_gpr[gprn]);
 }
 
 /* 64 bits PowerPC specific SPRs */
 #if defined(TARGET_PPC64)
-static void spr_read_hior (void *opaque, int gprn, int sprn)
+static void spr_read_hior (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, excp_prefix));
 }
 
-static void spr_write_hior (void *opaque, int sprn, int gprn)
+static void spr_write_hior (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
     tcg_gen_andi_tl(t0, cpu_gpr[gprn], 0x3FFFFF00000ULL);
@@ -401,31 +412,29 @@
 
 /* PowerPC 601 specific registers */
 /* RTC */
-static void spr_read_601_rtcl (void *opaque, int gprn, int sprn)
+static void spr_read_601_rtcl (DisasContext *ctx, int gprn, int sprn)
 {
     gen_helper_load_601_rtcl(cpu_gpr[gprn], cpu_env);
 }
 
-static void spr_read_601_rtcu (void *opaque, int gprn, int sprn)
+static void spr_read_601_rtcu (DisasContext *ctx, int gprn, int sprn)
 {
     gen_helper_load_601_rtcu(cpu_gpr[gprn], cpu_env);
 }
 
 #if !defined(CONFIG_USER_ONLY)
-static void spr_write_601_rtcu (void *opaque, int sprn, int gprn)
+static void spr_write_601_rtcu (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_601_rtcu(cpu_env, cpu_gpr[gprn]);
 }
 
-static void spr_write_601_rtcl (void *opaque, int sprn, int gprn)
+static void spr_write_601_rtcl (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_601_rtcl(cpu_env, cpu_gpr[gprn]);
 }
 
-static void spr_write_hid0_601 (void *opaque, int sprn, int gprn)
+static void spr_write_hid0_601 (DisasContext *ctx, int sprn, int gprn)
 {
-    DisasContext *ctx = opaque;
-
     gen_helper_store_hid0_601(cpu_env, cpu_gpr[gprn]);
     /* Must stop the translation as endianness may have changed */
     gen_stop_exception(ctx);
@@ -434,19 +443,19 @@
 
 /* Unified bats */
 #if !defined(CONFIG_USER_ONLY)
-static void spr_read_601_ubat (void *opaque, int gprn, int sprn)
+static void spr_read_601_ubat (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, IBAT[sprn & 1][(sprn - SPR_IBAT0U) / 2]));
 }
 
-static void spr_write_601_ubatu (void *opaque, int sprn, int gprn)
+static void spr_write_601_ubatu (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32((sprn - SPR_IBAT0U) / 2);
     gen_helper_store_601_batl(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_601_ubatl (void *opaque, int sprn, int gprn)
+static void spr_write_601_ubatl (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32((sprn - SPR_IBAT0U) / 2);
     gen_helper_store_601_batu(cpu_env, t0, cpu_gpr[gprn]);
@@ -456,36 +465,34 @@
 
 /* PowerPC 40x specific registers */
 #if !defined(CONFIG_USER_ONLY)
-static void spr_read_40x_pit (void *opaque, int gprn, int sprn)
+static void spr_read_40x_pit (DisasContext *ctx, int gprn, int sprn)
 {
     gen_helper_load_40x_pit(cpu_gpr[gprn], cpu_env);
 }
 
-static void spr_write_40x_pit (void *opaque, int sprn, int gprn)
+static void spr_write_40x_pit (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_40x_pit(cpu_env, cpu_gpr[gprn]);
 }
 
-static void spr_write_40x_dbcr0 (void *opaque, int sprn, int gprn)
+static void spr_write_40x_dbcr0 (DisasContext *ctx, int sprn, int gprn)
 {
-    DisasContext *ctx = opaque;
-
     gen_helper_store_40x_dbcr0(cpu_env, cpu_gpr[gprn]);
     /* We must stop translation as we may have rebooted */
     gen_stop_exception(ctx);
 }
 
-static void spr_write_40x_sler (void *opaque, int sprn, int gprn)
+static void spr_write_40x_sler (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_40x_sler(cpu_env, cpu_gpr[gprn]);
 }
 
-static void spr_write_booke_tcr (void *opaque, int sprn, int gprn)
+static void spr_write_booke_tcr (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_booke_tcr(cpu_env, cpu_gpr[gprn]);
 }
 
-static void spr_write_booke_tsr (void *opaque, int sprn, int gprn)
+static void spr_write_booke_tsr (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_store_booke_tsr(cpu_env, cpu_gpr[gprn]);
 }
@@ -494,19 +501,19 @@
 /* PowerPC 403 specific registers */
 /* PBL1 / PBU1 / PBL2 / PBU2 */
 #if !defined(CONFIG_USER_ONLY)
-static void spr_read_403_pbr (void *opaque, int gprn, int sprn)
+static void spr_read_403_pbr (DisasContext *ctx, int gprn, int sprn)
 {
     tcg_gen_ld_tl(cpu_gpr[gprn], cpu_env, offsetof(CPUPPCState, pb[sprn - SPR_403_PBL1]));
 }
 
-static void spr_write_403_pbr (void *opaque, int sprn, int gprn)
+static void spr_write_403_pbr (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32(sprn - SPR_403_PBL1);
     gen_helper_store_403_pbr(cpu_env, t0, cpu_gpr[gprn]);
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_pir (void *opaque, int sprn, int gprn)
+static void spr_write_pir (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
     tcg_gen_andi_tl(t0, cpu_gpr[gprn], 0xF);
@@ -516,7 +523,7 @@
 #endif
 
 /* SPE specific registers */
-static void spr_read_spefscr (void *opaque, int gprn, int sprn)
+static void spr_read_spefscr (DisasContext *ctx, int gprn, int sprn)
 {
     TCGv_i32 t0 = tcg_temp_new_i32();
     tcg_gen_ld_i32(t0, cpu_env, offsetof(CPUPPCState, spe_fscr));
@@ -524,7 +531,7 @@
     tcg_temp_free_i32(t0);
 }
 
-static void spr_write_spefscr (void *opaque, int sprn, int gprn)
+static void spr_write_spefscr (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_temp_new_i32();
     tcg_gen_trunc_tl_i32(t0, cpu_gpr[gprn]);
@@ -534,7 +541,7 @@
 
 #if !defined(CONFIG_USER_ONLY)
 /* Callback used to write the exception vector base */
-static void spr_write_excp_prefix (void *opaque, int sprn, int gprn)
+static void spr_write_excp_prefix (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
     tcg_gen_ld_tl(t0, cpu_env, offsetof(CPUPPCState, ivpr_mask));
@@ -544,9 +551,8 @@
     tcg_temp_free(t0);
 }
 
-static void spr_write_excp_vector (void *opaque, int sprn, int gprn)
+static void spr_write_excp_vector (DisasContext *ctx, int sprn, int gprn)
 {
-    DisasContext *ctx = opaque;
     int sprn_offs;
 
     if (sprn >= SPR_BOOKE_IVOR0 && sprn <= SPR_BOOKE_IVOR15) {
@@ -583,17 +589,33 @@
 #define spr_register_kvm(env, num, name, uea_read, uea_write,                  \
                          oea_read, oea_write, one_reg_id, initial_value)       \
     _spr_register(env, num, name, uea_read, uea_write, initial_value)
+#define spr_register_kvm_hv(env, num, name, uea_read, uea_write,               \
+                            oea_read, oea_write, hea_read, hea_write,          \
+                            one_reg_id, initial_value)                         \
+    _spr_register(env, num, name, uea_read, uea_write, initial_value)
 #else
 #if !defined(CONFIG_KVM)
 #define spr_register_kvm(env, num, name, uea_read, uea_write,                  \
-                         oea_read, oea_write, one_reg_id, initial_value) \
+                         oea_read, oea_write, one_reg_id, initial_value)       \
     _spr_register(env, num, name, uea_read, uea_write,                         \
-                  oea_read, oea_write, initial_value)
+                  oea_read, oea_write, oea_read, oea_write, initial_value)
+#define spr_register_kvm_hv(env, num, name, uea_read, uea_write,               \
+                            oea_read, oea_write, hea_read, hea_write,          \
+                            one_reg_id, initial_value)                         \
+    _spr_register(env, num, name, uea_read, uea_write,                         \
+                  oea_read, oea_write, hea_read, hea_write, initial_value)
 #else
 #define spr_register_kvm(env, num, name, uea_read, uea_write,                  \
-                         oea_read, oea_write, one_reg_id, initial_value) \
+                         oea_read, oea_write, one_reg_id, initial_value)       \
     _spr_register(env, num, name, uea_read, uea_write,                         \
-                  oea_read, oea_write, one_reg_id, initial_value)
+                  oea_read, oea_write, oea_read, oea_write,                    \
+                  one_reg_id, initial_value)
+#define spr_register_kvm_hv(env, num, name, uea_read, uea_write,               \
+                            oea_read, oea_write, hea_read, hea_write,          \
+                            one_reg_id, initial_value)                         \
+    _spr_register(env, num, name, uea_read, uea_write,                         \
+                  oea_read, oea_write, hea_read, hea_write,                    \
+                  one_reg_id, initial_value)
 #endif
 #endif
 
@@ -602,14 +624,23 @@
     spr_register_kvm(env, num, name, uea_read, uea_write,                      \
                      oea_read, oea_write, 0, initial_value)
 
+#define spr_register_hv(env, num, name, uea_read, uea_write,                   \
+                        oea_read, oea_write, hea_read, hea_write,              \
+                        initial_value)                                         \
+    spr_register_kvm_hv(env, num, name, uea_read, uea_write,                   \
+                        oea_read, oea_write, hea_read, hea_write,              \
+                        0, initial_value)
+
 static inline void _spr_register(CPUPPCState *env, int num,
                                  const char *name,
-                                 void (*uea_read)(void *opaque, int gprn, int sprn),
-                                 void (*uea_write)(void *opaque, int sprn, int gprn),
+                                 void (*uea_read)(DisasContext *ctx, int gprn, int sprn),
+                                 void (*uea_write)(DisasContext *ctx, int sprn, int gprn),
 #if !defined(CONFIG_USER_ONLY)
 
-                                 void (*oea_read)(void *opaque, int gprn, int sprn),
-                                 void (*oea_write)(void *opaque, int sprn, int gprn),
+                                 void (*oea_read)(DisasContext *ctx, int gprn, int sprn),
+                                 void (*oea_write)(DisasContext *ctx, int sprn, int gprn),
+                                 void (*hea_read)(DisasContext *opaque, int gprn, int sprn),
+                                 void (*hea_write)(DisasContext *opaque, int sprn, int gprn),
 #endif
 #if defined(CONFIG_KVM)
                                  uint64_t one_reg_id,
@@ -637,6 +668,8 @@
 #if !defined(CONFIG_USER_ONLY)
     spr->oea_read = oea_read;
     spr->oea_write = oea_write;
+    spr->hea_read = hea_read;
+    spr->hea_write = hea_write;
 #endif
 #if defined(CONFIG_KVM)
     spr->one_reg_id = one_reg_id,
@@ -1040,30 +1073,102 @@
 
 #ifdef TARGET_PPC64
 #ifndef CONFIG_USER_ONLY
-static void spr_read_uamr (void *opaque, int gprn, int sprn)
-{
-    gen_load_spr(cpu_gpr[gprn], SPR_AMR);
-    spr_load_dump_spr(SPR_AMR);
-}
-
-static void spr_write_uamr (void *opaque, int sprn, int gprn)
-{
-    gen_store_spr(SPR_AMR, cpu_gpr[gprn]);
-    spr_store_dump_spr(SPR_AMR);
-}
-
-static void spr_write_uamr_pr (void *opaque, int sprn, int gprn)
+static void spr_write_amr(DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
 
-    gen_load_spr(t0, SPR_UAMOR);
-    tcg_gen_and_tl(t0, t0, cpu_gpr[gprn]);
+    /* Note, the HV=1 PR=0 case is handled earlier by simply using
+     * spr_write_generic for HV mode in the SPR table
+     */
+
+    /* Build insertion mask into t1 based on context */
+    if (ctx->pr) {
+        gen_load_spr(t1, SPR_UAMOR);
+    } else {
+        gen_load_spr(t1, SPR_AMOR);
+    }
+
+    /* Mask new bits into t2 */
+    tcg_gen_and_tl(t2, t1, cpu_gpr[gprn]);
+
+    /* Load AMR and clear new bits in t0 */
+    gen_load_spr(t0, SPR_AMR);
+    tcg_gen_andc_tl(t0, t0, t1);
+
+    /* Or'in new bits and write it out */
+    tcg_gen_or_tl(t0, t0, t2);
     gen_store_spr(SPR_AMR, t0);
     spr_store_dump_spr(SPR_AMR);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
+}
+
+static void spr_write_uamor(DisasContext *ctx, int sprn, int gprn)
+{
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
+
+    /* Note, the HV=1 case is handled earlier by simply using
+     * spr_write_generic for HV mode in the SPR table
+     */
+
+    /* Build insertion mask into t1 based on context */
+    gen_load_spr(t1, SPR_AMOR);
+
+    /* Mask new bits into t2 */
+    tcg_gen_and_tl(t2, t1, cpu_gpr[gprn]);
+
+    /* Load AMR and clear new bits in t0 */
+    gen_load_spr(t0, SPR_UAMOR);
+    tcg_gen_andc_tl(t0, t0, t1);
+
+    /* Or'in new bits and write it out */
+    tcg_gen_or_tl(t0, t0, t2);
+    gen_store_spr(SPR_UAMOR, t0);
+    spr_store_dump_spr(SPR_UAMOR);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
+}
+
+static void spr_write_iamr(DisasContext *ctx, int sprn, int gprn)
+{
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
+
+    /* Note, the HV=1 case is handled earlier by simply using
+     * spr_write_generic for HV mode in the SPR table
+     */
+
+    /* Build insertion mask into t1 based on context */
+    gen_load_spr(t1, SPR_AMOR);
+
+    /* Mask new bits into t2 */
+    tcg_gen_and_tl(t2, t1, cpu_gpr[gprn]);
+
+    /* Load AMR and clear new bits in t0 */
+    gen_load_spr(t0, SPR_IAMR);
+    tcg_gen_andc_tl(t0, t0, t1);
+
+    /* Or'in new bits and write it out */
+    tcg_gen_or_tl(t0, t0, t2);
+    gen_store_spr(SPR_IAMR, t0);
+    spr_store_dump_spr(SPR_IAMR);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
 }
 #endif /* CONFIG_USER_ONLY */
 
-static void gen_spr_amr (CPUPPCState *env)
+static void gen_spr_amr(CPUPPCState *env, bool has_iamr)
 {
 #ifndef CONFIG_USER_ONLY
     /* Virtual Page Class Key protection */
@@ -1071,38 +1176,61 @@
      * userspace accessible, 29 is privileged.  So we only need to set
      * the kvm ONE_REG id on one of them, we use 29 */
     spr_register(env, SPR_UAMR, "UAMR",
-                 &spr_read_uamr, &spr_write_uamr_pr,
-                 &spr_read_uamr, &spr_write_uamr,
+                 &spr_read_generic, &spr_write_amr,
+                 &spr_read_generic, &spr_write_amr,
                  0);
-    spr_register_kvm(env, SPR_AMR, "AMR",
+    spr_register_kvm_hv(env, SPR_AMR, "AMR",
                      SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_amr,
                      &spr_read_generic, &spr_write_generic,
                      KVM_REG_PPC_AMR, 0);
-    spr_register_kvm(env, SPR_UAMOR, "UAMOR",
+    spr_register_kvm_hv(env, SPR_UAMOR, "UAMOR",
                      SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_uamor,
                      &spr_read_generic, &spr_write_generic,
                      KVM_REG_PPC_UAMOR, 0);
+    spr_register_hv(env, SPR_AMOR, "AMOR",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    &spr_read_generic, &spr_write_generic,
+                    0);
+    if (has_iamr) {
+        spr_register_kvm_hv(env, SPR_IAMR, "IAMR",
+                            SPR_NOACCESS, SPR_NOACCESS,
+                            &spr_read_generic, &spr_write_iamr,
+                            &spr_read_generic, &spr_write_generic,
+                            KVM_REG_PPC_IAMR, 0);
+    }
 #endif /* !CONFIG_USER_ONLY */
 }
 #endif /* TARGET_PPC64 */
 
+#ifndef CONFIG_USER_ONLY
+static void spr_read_thrm(DisasContext *ctx, int gprn, int sprn)
+{
+    gen_helper_fixup_thrm(cpu_env);
+    gen_load_spr(cpu_gpr[gprn], sprn);
+    spr_load_dump_spr(sprn);
+}
+#endif /* !CONFIG_USER_ONLY */
+
 static void gen_spr_thrm (CPUPPCState *env)
 {
     /* Thermal management */
     /* XXX : not implemented */
     spr_register(env, SPR_THRM1, "THRM1",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
+                 &spr_read_thrm, &spr_write_generic,
                  0x00000000);
     /* XXX : not implemented */
     spr_register(env, SPR_THRM2, "THRM2",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
+                 &spr_read_thrm, &spr_write_generic,
                  0x00000000);
     /* XXX : not implemented */
     spr_register(env, SPR_THRM3, "THRM3",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
+                 &spr_read_thrm, &spr_write_generic,
                  0x00000000);
 }
 
@@ -1454,7 +1582,7 @@
 }
 
 #if !defined(CONFIG_USER_ONLY)
-static void spr_write_e500_l1csr0 (void *opaque, int sprn, int gprn)
+static void spr_write_e500_l1csr0 (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
 
@@ -1463,7 +1591,7 @@
     tcg_temp_free(t0);
 }
 
-static void spr_write_e500_l1csr1(void *opaque, int sprn, int gprn)
+static void spr_write_e500_l1csr1(DisasContext *ctx, int sprn, int gprn)
 {
     TCGv t0 = tcg_temp_new();
 
@@ -1472,12 +1600,12 @@
     tcg_temp_free(t0);
 }
 
-static void spr_write_booke206_mmucsr0 (void *opaque, int sprn, int gprn)
+static void spr_write_booke206_mmucsr0 (DisasContext *ctx, int sprn, int gprn)
 {
     gen_helper_booke206_tlbflush(cpu_env, cpu_gpr[gprn]);
 }
 
-static void spr_write_booke_pid (void *opaque, int sprn, int gprn)
+static void spr_write_booke_pid (DisasContext *ctx, int sprn, int gprn)
 {
     TCGv_i32 t0 = tcg_const_i32(sprn);
     gen_helper_booke_setpid(cpu_env, t0, cpu_gpr[gprn]);
@@ -1693,7 +1821,7 @@
     /* TLB assist registers */
     /* XXX : not implemented */
     for (i = 0; i < 8; i++) {
-        void (*uea_write)(void *o, int sprn, int gprn) = &spr_write_generic32;
+        void (*uea_write)(DisasContext *ctx, int sprn, int gprn) = &spr_write_generic32;
         if (i == 2 && (mas_mask & (1 << i)) && (env->insns_flags & PPC_64B)) {
             uea_write = &spr_write_generic;
         }
@@ -3078,18 +3206,30 @@
     env->excp_vectors[POWERPC_EXCP_HDECR]    = 0x00000980;
     env->excp_vectors[POWERPC_EXCP_SYSCALL]  = 0x00000C00;
     env->excp_vectors[POWERPC_EXCP_TRACE]    = 0x00000D00;
+    env->excp_vectors[POWERPC_EXCP_HDSI]     = 0x00000E00;
+    env->excp_vectors[POWERPC_EXCP_HISI]     = 0x00000E20;
+    env->excp_vectors[POWERPC_EXCP_HV_EMU]   = 0x00000E40;
+    env->excp_vectors[POWERPC_EXCP_HV_MAINT] = 0x00000E60;
     env->excp_vectors[POWERPC_EXCP_PERFM]    = 0x00000F00;
     env->excp_vectors[POWERPC_EXCP_VPU]      = 0x00000F20;
     env->excp_vectors[POWERPC_EXCP_VSXU]     = 0x00000F40;
-    env->excp_vectors[POWERPC_EXCP_FU]       = 0x00000F60;
-    env->excp_vectors[POWERPC_EXCP_IABR]     = 0x00001300;
-    env->excp_vectors[POWERPC_EXCP_MAINT]    = 0x00001600;
-    env->excp_vectors[POWERPC_EXCP_VPUA]     = 0x00001700;
-    env->excp_vectors[POWERPC_EXCP_THERM]    = 0x00001800;
     /* Hardware reset vector */
     env->hreset_vector = 0x0000000000000100ULL;
 #endif
 }
+
+static void init_excp_POWER8(CPUPPCState *env)
+{
+    init_excp_POWER7(env);
+
+#if !defined(CONFIG_USER_ONLY)
+    env->excp_vectors[POWERPC_EXCP_SDOOR]    = 0x00000A00;
+    env->excp_vectors[POWERPC_EXCP_FU]       = 0x00000F60;
+    env->excp_vectors[POWERPC_EXCP_HV_FU]    = 0x00000F80;
+    env->excp_vectors[POWERPC_EXCP_SDOOR_HV] = 0x00000E80;
+#endif
+}
+
 #endif
 
 /*****************************************************************************/
@@ -3166,7 +3306,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(12, 16, 20, 24);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3220,7 +3360,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(12, 16, 20, 24);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3272,7 +3412,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(12, 16, 20, 24);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3331,7 +3471,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(8, 12, 16, 20);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3382,7 +3522,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(8, 12, 16, 20);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3448,7 +3588,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(8, 12, 16, 20);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3514,7 +3654,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(8, 12, 16, 20);
     SET_WDT_PERIOD(16, 20, 24, 28);
@@ -3612,7 +3752,7 @@
     init_excp_BookE(env);
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(12, 16, 20, 24);
     SET_WDT_PERIOD(20, 24, 28, 32);
@@ -3882,7 +4022,7 @@
     init_excp_BookE(env);
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
-    ppc40x_irq_init(env);
+    ppc40x_irq_init(ppc_env_get_cpu(env));
 
     SET_FIT_PERIOD(12, 16, 20, 24);
     SET_WDT_PERIOD(20, 24, 28, 32);
@@ -4304,7 +4444,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(G2)(ObjectClass *oc, void *data)
@@ -4383,7 +4523,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(G2LE)(ObjectClass *oc, void *data)
@@ -4636,7 +4776,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(e300)(ObjectClass *oc, void *data)
@@ -4680,7 +4820,7 @@
 }
 
 #if !defined(CONFIG_USER_ONLY)
-static void spr_write_mas73(void *opaque, int sprn, int gprn)
+static void spr_write_mas73(DisasContext *ctx, int sprn, int gprn)
 {
     TCGv val = tcg_temp_new();
     tcg_gen_ext32u_tl(val, cpu_gpr[gprn]);
@@ -4690,7 +4830,7 @@
     tcg_temp_free(val);
 }
 
-static void spr_read_mas73(void *opaque, int gprn, int sprn)
+static void spr_read_mas73(DisasContext *ctx, int gprn, int sprn)
 {
     TCGv mas7 = tcg_temp_new();
     TCGv mas3 = tcg_temp_new();
@@ -4891,7 +5031,7 @@
 
     init_excp_e200(env, ivpr_mask);
     /* Allocate hardware IRQ controller */
-    ppce500_irq_init(env);
+    ppce500_irq_init(ppc_env_get_cpu(env));
 }
 
 static void init_proc_e500v1(CPUPPCState *env)
@@ -4993,7 +5133,7 @@
     dc->desc = "e500mc core";
     pcc->init_proc = init_proc_e500mc;
     pcc->check_pow = check_pow_none;
-    pcc->insns_flags = PPC_INSNS_BASE | PPC_ISEL |
+    pcc->insns_flags = PPC_INSNS_BASE | PPC_ISEL | PPC_MFTB |
                        PPC_WRTEE | PPC_RFDI | PPC_RFMCI |
                        PPC_CACHE | PPC_CACHE_LOCK | PPC_CACHE_ICBI |
                        PPC_CACHE_DCBZ | PPC_CACHE_DCBA |
@@ -5039,7 +5179,7 @@
     dc->desc = "e5500 core";
     pcc->init_proc = init_proc_e5500;
     pcc->check_pow = check_pow_none;
-    pcc->insns_flags = PPC_INSNS_BASE | PPC_ISEL |
+    pcc->insns_flags = PPC_INSNS_BASE | PPC_ISEL | PPC_MFTB |
                        PPC_WRTEE | PPC_RFDI | PPC_RFMCI |
                        PPC_CACHE | PPC_CACHE_LOCK | PPC_CACHE_ICBI |
                        PPC_CACHE_DCBZ | PPC_CACHE_DCBA |
@@ -5135,7 +5275,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 64;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(601)(ObjectClass *oc, void *data)
@@ -5239,7 +5379,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(602)(ObjectClass *oc, void *data)
@@ -5308,7 +5448,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(603)(ObjectClass *oc, void *data)
@@ -5374,7 +5514,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(603E)(ObjectClass *oc, void *data)
@@ -5434,7 +5574,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(604)(ObjectClass *oc, void *data)
@@ -5517,7 +5657,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(604E)(ObjectClass *oc, void *data)
@@ -5587,7 +5727,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(740)(ObjectClass *oc, void *data)
@@ -5665,7 +5805,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(750)(ObjectClass *oc, void *data)
@@ -5828,7 +5968,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(750cl)(ObjectClass *oc, void *data)
@@ -5948,7 +6088,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(750cx)(ObjectClass *oc, void *data)
@@ -6035,7 +6175,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(750fx)(ObjectClass *oc, void *data)
@@ -6122,7 +6262,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(750gx)(ObjectClass *oc, void *data)
@@ -6200,7 +6340,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(745)(ObjectClass *oc, void *data)
@@ -6286,7 +6426,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(755)(ObjectClass *oc, void *data)
@@ -6355,7 +6495,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7400)(ObjectClass *oc, void *data)
@@ -6439,7 +6579,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7410)(ObjectClass *oc, void *data)
@@ -6549,7 +6689,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7440)(ObjectClass *oc, void *data)
@@ -6682,7 +6822,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7450)(ObjectClass *oc, void *data)
@@ -6818,7 +6958,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7445)(ObjectClass *oc, void *data)
@@ -6956,7 +7096,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7455)(ObjectClass *oc, void *data)
@@ -7118,7 +7258,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(7457)(ObjectClass *oc, void *data)
@@ -7255,7 +7395,7 @@
     env->dcache_line_size = 32;
     env->icache_line_size = 32;
     /* Allocate hardware IRQ controller */
-    ppc6xx_irq_init(env);
+    ppc6xx_irq_init(ppc_env_get_cpu(env));
 }
 
 POWERPC_FAMILY(e600)(ObjectClass *oc, void *data)
@@ -7322,14 +7462,14 @@
     BOOK3S_CPU_POWER8
 };
 
-static void gen_fscr_facility_check(void *opaque, int facility_sprn, int bit,
-                                    int sprn, int cause)
+static void gen_fscr_facility_check(DisasContext *ctx, int facility_sprn,
+                                    int bit, int sprn, int cause)
 {
     TCGv_i32 t1 = tcg_const_i32(bit);
     TCGv_i32 t2 = tcg_const_i32(sprn);
     TCGv_i32 t3 = tcg_const_i32(cause);
 
-    gen_update_current_nip(opaque);
+    gen_update_current_nip(ctx);
     gen_helper_fscr_facility_check(cpu_env, t1, t2, t3);
 
     tcg_temp_free_i32(t3);
@@ -7337,14 +7477,14 @@
     tcg_temp_free_i32(t1);
 }
 
-static void gen_msr_facility_check(void *opaque, int facility_sprn, int bit,
-                                   int sprn, int cause)
+static void gen_msr_facility_check(DisasContext *ctx, int facility_sprn,
+                                   int bit, int sprn, int cause)
 {
     TCGv_i32 t1 = tcg_const_i32(bit);
     TCGv_i32 t2 = tcg_const_i32(sprn);
     TCGv_i32 t3 = tcg_const_i32(cause);
 
-    gen_update_current_nip(opaque);
+    gen_update_current_nip(ctx);
     gen_helper_msr_facility_check(cpu_env, t1, t2, t3);
 
     tcg_temp_free_i32(t3);
@@ -7352,7 +7492,7 @@
     tcg_temp_free_i32(t1);
 }
 
-static void spr_read_prev_upper32(void *opaque, int gprn, int sprn)
+static void spr_read_prev_upper32(DisasContext *ctx, int gprn, int sprn)
 {
     TCGv spr_up = tcg_temp_new();
     TCGv spr = tcg_temp_new();
@@ -7365,7 +7505,7 @@
     tcg_temp_free(spr_up);
 }
 
-static void spr_write_prev_upper32(void *opaque, int sprn, int gprn)
+static void spr_write_prev_upper32(DisasContext *ctx, int sprn, int gprn)
 {
     TCGv spr = tcg_temp_new();
 
@@ -7411,16 +7551,6 @@
                  0x00000000);
 }
 
-static void gen_spr_970_lpar(CPUPPCState *env)
-{
-    /* Logical partitionning */
-    /* PPC970: HID4 is effectively the LPCR */
-    spr_register(env, SPR_970_HID4, "HID4",
-                 SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
-}
-
 static void gen_spr_book3s_common(CPUPPCState *env)
 {
     spr_register(env, SPR_CTRL, "SPR_CTRL",
@@ -7468,6 +7598,25 @@
                      KVM_REG_PPC_DABRX, 0x00000000);
 }
 
+static void gen_spr_book3s_207_dbg(CPUPPCState *env)
+{
+    spr_register_kvm_hv(env, SPR_DAWR, "DAWR",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_generic,
+                        KVM_REG_PPC_DAWR, 0x00000000);
+    spr_register_kvm_hv(env, SPR_DAWRX, "DAWRX",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_generic,
+                        KVM_REG_PPC_DAWRX, 0x00000000);
+    spr_register_kvm_hv(env, SPR_CIABR, "CIABR",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_generic,
+                        KVM_REG_PPC_CIABR, 0x00000000);
+}
+
 static void gen_spr_970_dbg(CPUPPCState *env)
 {
     /* Breakpoints */
@@ -7607,6 +7756,30 @@
                      SPR_NOACCESS, SPR_NOACCESS,
                      &spr_read_generic, &spr_write_generic,
                      KVM_REG_PPC_MMCRS, 0x00000000);
+    spr_register_kvm(env, SPR_POWER_SIER, "SIER",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_SIER, 0x00000000);
+    spr_register_kvm(env, SPR_POWER_SPMC1, "SPMC1",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_SPMC1, 0x00000000);
+    spr_register_kvm(env, SPR_POWER_SPMC2, "SPMC2",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_SPMC2, 0x00000000);
+    spr_register_kvm(env, SPR_TACR, "TACR",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_TACR, 0x00000000);
+    spr_register_kvm(env, SPR_TCSCR, "TCSCR",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_TCSCR, 0x00000000);
+    spr_register_kvm(env, SPR_CSIGR, "CSIGR",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_CSIGR, 0x00000000);
 }
 
 static void gen_spr_power8_pmu_user(CPUPPCState *env)
@@ -7615,6 +7788,10 @@
                  &spr_read_ureg, SPR_NOACCESS,
                  &spr_read_ureg, &spr_write_ureg,
                  0x00000000);
+    spr_register(env, SPR_POWER_USIER, "USIER",
+                 &spr_read_generic, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
 }
 
 static void gen_spr_power5p_ear(CPUPPCState *env)
@@ -7626,21 +7803,155 @@
                  0x00000000);
 }
 
+#if !defined(CONFIG_USER_ONLY)
+static void spr_write_hmer(DisasContext *ctx, int sprn, int gprn)
+{
+    TCGv hmer = tcg_temp_new();
+
+    gen_load_spr(hmer, sprn);
+    tcg_gen_and_tl(hmer, cpu_gpr[gprn], hmer);
+    gen_store_spr(sprn, hmer);
+    spr_store_dump_spr(sprn);
+    tcg_temp_free(hmer);
+}
+
+static void spr_write_lpcr(DisasContext *ctx, int sprn, int gprn)
+{
+    gen_helper_store_lpcr(cpu_env, cpu_gpr[gprn]);
+}
+
+static void spr_write_970_hid4(DisasContext *ctx, int sprn, int gprn)
+{
+#if defined(TARGET_PPC64)
+    spr_write_generic(ctx, sprn, gprn);
+    gen_helper_store_lpcr(cpu_env, cpu_gpr[gprn]);
+#endif
+}
+
+#endif /* !defined(CONFIG_USER_ONLY) */
+
+static void gen_spr_970_lpar(CPUPPCState *env)
+{
+#if !defined(CONFIG_USER_ONLY)
+    /* Logical partitionning */
+    /* PPC970: HID4 is effectively the LPCR */
+    spr_register(env, SPR_970_HID4, "HID4",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_970_hid4,
+                 0x00000000);
+#endif
+}
+
 static void gen_spr_power5p_lpar(CPUPPCState *env)
 {
+#if !defined(CONFIG_USER_ONLY)
     /* Logical partitionning */
-    spr_register_kvm(env, SPR_LPCR, "LPCR",
-                     SPR_NOACCESS, SPR_NOACCESS,
-                     &spr_read_generic, &spr_write_generic,
-                     KVM_REG_PPC_LPCR, 0x00000000);
+    spr_register_kvm_hv(env, SPR_LPCR, "LPCR",
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        SPR_NOACCESS, SPR_NOACCESS,
+                        &spr_read_generic, &spr_write_lpcr,
+                        KVM_REG_PPC_LPCR, LPCR_LPES0 | LPCR_LPES1);
+    spr_register_hv(env, SPR_HDEC, "HDEC",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    &spr_read_hdecr, &spr_write_hdecr, 0);
+#endif
 }
 
 static void gen_spr_book3s_ids(CPUPPCState *env)
 {
+    /* FIXME: Will need to deal with thread vs core only SPRs */
+
     /* Processor identification */
-    spr_register(env, SPR_PIR, "PIR",
+    spr_register_hv(env, SPR_PIR, "PIR",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_pir,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, NULL,
+                 0x00000000);
+    spr_register_hv(env, SPR_HID0, "HID0",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_TSCR, "TSCR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HMER, "HMER",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_hmer,
+                 0x00000000);
+    spr_register_hv(env, SPR_HMEER, "HMEER",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_TFMR, "TFMR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_LPIDR, "LPIDR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HFSCR, "HFSCR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_MMCRC, "MMCRC",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_MMCRH, "MMCRH",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HSPRG0, "HSPRG0",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HSPRG1, "HSPRG1",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HSRR0, "HSRR0",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HSRR1, "HSRR1",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HDAR, "HDAR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HDSISR, "HDSISR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_RMOR, "RMOR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
+                 0x00000000);
+    spr_register_hv(env, SPR_HRMOR, "HRMOR",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_generic, &spr_write_generic,
                  0x00000000);
 }
 
@@ -7704,48 +8015,48 @@
                  0x00000000);
 }
 
-static void spr_read_tar(void *opaque, int gprn, int sprn)
+static void spr_read_tar(DisasContext *ctx, int gprn, int sprn)
 {
-    gen_fscr_facility_check(opaque, SPR_FSCR, FSCR_TAR, sprn, FSCR_IC_TAR);
-    spr_read_generic(opaque, gprn, sprn);
+    gen_fscr_facility_check(ctx, SPR_FSCR, FSCR_TAR, sprn, FSCR_IC_TAR);
+    spr_read_generic(ctx, gprn, sprn);
 }
 
-static void spr_write_tar(void *opaque, int sprn, int gprn)
+static void spr_write_tar(DisasContext *ctx, int sprn, int gprn)
 {
-    gen_fscr_facility_check(opaque, SPR_FSCR, FSCR_TAR, sprn, FSCR_IC_TAR);
-    spr_write_generic(opaque, sprn, gprn);
+    gen_fscr_facility_check(ctx, SPR_FSCR, FSCR_TAR, sprn, FSCR_IC_TAR);
+    spr_write_generic(ctx, sprn, gprn);
 }
 
 static void gen_spr_power8_tce_address_control(CPUPPCState *env)
 {
-    spr_register(env, SPR_TAR, "TAR",
-                 &spr_read_tar, &spr_write_tar,
-                 &spr_read_generic, &spr_write_generic,
-                 0x00000000);
+    spr_register_kvm(env, SPR_TAR, "TAR",
+                     &spr_read_tar, &spr_write_tar,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_TAR, 0x00000000);
 }
 
-static void spr_read_tm(void *opaque, int gprn, int sprn)
+static void spr_read_tm(DisasContext *ctx, int gprn, int sprn)
 {
-    gen_msr_facility_check(opaque, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
-    spr_read_generic(opaque, gprn, sprn);
+    gen_msr_facility_check(ctx, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
+    spr_read_generic(ctx, gprn, sprn);
 }
 
-static void spr_write_tm(void *opaque, int sprn, int gprn)
+static void spr_write_tm(DisasContext *ctx, int sprn, int gprn)
 {
-    gen_msr_facility_check(opaque, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
-    spr_write_generic(opaque, sprn, gprn);
+    gen_msr_facility_check(ctx, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
+    spr_write_generic(ctx, sprn, gprn);
 }
 
-static void spr_read_tm_upper32(void *opaque, int gprn, int sprn)
+static void spr_read_tm_upper32(DisasContext *ctx, int gprn, int sprn)
 {
-    gen_msr_facility_check(opaque, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
-    spr_read_prev_upper32(opaque, gprn, sprn);
+    gen_msr_facility_check(ctx, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
+    spr_read_prev_upper32(ctx, gprn, sprn);
 }
 
-static void spr_write_tm_upper32(void *opaque, int sprn, int gprn)
+static void spr_write_tm_upper32(DisasContext *ctx, int sprn, int gprn)
 {
-    gen_msr_facility_check(opaque, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
-    spr_write_prev_upper32(opaque, sprn, gprn);
+    gen_msr_facility_check(ctx, SPR_FSCR, MSR_TM, sprn, FSCR_IC_TM);
+    spr_write_prev_upper32(ctx, sprn, gprn);
 }
 
 static void gen_spr_power8_tm(CPUPPCState *env)
@@ -7768,28 +8079,28 @@
                  0x00000000);
 }
 
-static void spr_read_ebb(void *opaque, int gprn, int sprn)
+static void spr_read_ebb(DisasContext *ctx, int gprn, int sprn)
 {
-    gen_fscr_facility_check(opaque, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
-    spr_read_generic(opaque, gprn, sprn);
+    gen_fscr_facility_check(ctx, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
+    spr_read_generic(ctx, gprn, sprn);
 }
 
-static void spr_write_ebb(void *opaque, int sprn, int gprn)
+static void spr_write_ebb(DisasContext *ctx, int sprn, int gprn)
 {
-    gen_fscr_facility_check(opaque, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
-    spr_write_generic(opaque, sprn, gprn);
+    gen_fscr_facility_check(ctx, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
+    spr_write_generic(ctx, sprn, gprn);
 }
 
-static void spr_read_ebb_upper32(void *opaque, int gprn, int sprn)
+static void spr_read_ebb_upper32(DisasContext *ctx, int gprn, int sprn)
 {
-    gen_fscr_facility_check(opaque, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
-    spr_read_prev_upper32(opaque, gprn, sprn);
+    gen_fscr_facility_check(ctx, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
+    spr_read_prev_upper32(ctx, gprn, sprn);
 }
 
-static void spr_write_ebb_upper32(void *opaque, int sprn, int gprn)
+static void spr_write_ebb_upper32(DisasContext *ctx, int sprn, int gprn)
 {
-    gen_fscr_facility_check(opaque, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
-    spr_write_prev_upper32(opaque, sprn, gprn);
+    gen_fscr_facility_check(ctx, SPR_FSCR, FSCR_EBB, sprn, FSCR_IC_EBB);
+    spr_write_prev_upper32(ctx, sprn, gprn);
 }
 
 static void gen_spr_power8_ebb(CPUPPCState *env)
@@ -7824,6 +8135,15 @@
                      KVM_REG_PPC_BESCR, 0x00000000);
 }
 
+/* Virtual Time Base */
+static void gen_spr_vtb(CPUPPCState *env)
+{
+    spr_register(env, SPR_VTB, "VTB",
+                 SPR_NOACCESS, SPR_NOACCESS,
+                 &spr_read_tbl, SPR_NOACCESS,
+                 0x00000000);
+}
+
 static void gen_spr_power8_fscr(CPUPPCState *env)
 {
 #if defined(CONFIG_USER_ONLY)
@@ -7837,6 +8157,70 @@
                      KVM_REG_PPC_FSCR, initval);
 }
 
+static void gen_spr_power8_pspb(CPUPPCState *env)
+{
+    spr_register_kvm(env, SPR_PSPB, "PSPB",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic32,
+                     KVM_REG_PPC_PSPB, 0);
+}
+
+static void gen_spr_power8_ic(CPUPPCState *env)
+{
+#if !defined(CONFIG_USER_ONLY)
+    spr_register_hv(env, SPR_IC, "IC",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    &spr_read_generic, SPR_NOACCESS,
+                    &spr_read_generic, &spr_write_generic,
+                    0);
+#endif
+}
+
+static void gen_spr_power8_book4(CPUPPCState *env)
+{
+    /* Add a number of P8 book4 registers */
+#if !defined(CONFIG_USER_ONLY)
+    spr_register_kvm(env, SPR_ACOP, "ACOP",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_ACOP, 0);
+    spr_register_kvm(env, SPR_BOOKS_PID, "PID",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_PID, 0);
+    spr_register_kvm(env, SPR_WORT, "WORT",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_WORT, 0);
+#endif
+}
+
+static void gen_spr_power7_book4(CPUPPCState *env)
+{
+    /* Add a number of P7 book4 registers */
+#if !defined(CONFIG_USER_ONLY)
+    spr_register_kvm(env, SPR_ACOP, "ACOP",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_ACOP, 0);
+    spr_register_kvm(env, SPR_BOOKS_PID, "PID",
+                     SPR_NOACCESS, SPR_NOACCESS,
+                     &spr_read_generic, &spr_write_generic,
+                     KVM_REG_PPC_PID, 0);
+#endif
+}
+
+static void gen_spr_power8_rpr(CPUPPCState *env)
+{
+#if !defined(CONFIG_USER_ONLY)
+    spr_register_hv(env, SPR_RPR, "RPR",
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    SPR_NOACCESS, SPR_NOACCESS,
+                    &spr_read_generic, &spr_write_generic,
+                    0x00000103070F1F3F);
+#endif
+}
+
 static void init_proc_book3s_64(CPUPPCState *env, int version)
 {
     gen_spr_ne_601(env);
@@ -7858,8 +8242,9 @@
     case BOOK3S_CPU_POWER7:
     case BOOK3S_CPU_POWER8:
         gen_spr_book3s_ids(env);
-        gen_spr_amr(env);
+        gen_spr_amr(env, version >= BOOK3S_CPU_POWER8);
         gen_spr_book3s_purr(env);
+        env->ci_large_pages = true;
         break;
     default:
         g_assert_not_reached();
@@ -7878,6 +8263,9 @@
         gen_spr_power6_common(env);
         gen_spr_power6_dbg(env);
     }
+    if (version == BOOK3S_CPU_POWER7) {
+        gen_spr_power7_book4(env);
+    }
     if (version >= BOOK3S_CPU_POWER8) {
         gen_spr_power8_tce_address_control(env);
         gen_spr_power8_ids(env);
@@ -7886,9 +8274,16 @@
         gen_spr_power8_pmu_sup(env);
         gen_spr_power8_pmu_user(env);
         gen_spr_power8_tm(env);
+        gen_spr_power8_pspb(env);
+        gen_spr_vtb(env);
+        gen_spr_power8_ic(env);
+        gen_spr_power8_book4(env);
+        gen_spr_power8_rpr(env);
     }
     if (version < BOOK3S_CPU_POWER8) {
         gen_spr_book3s_dbg(env);
+    } else {
+        gen_spr_book3s_207_dbg(env);
     }
 #if !defined(CONFIG_USER_ONLY)
     switch (version) {
@@ -7908,12 +8303,15 @@
     case BOOK3S_CPU_970:
     case BOOK3S_CPU_POWER5PLUS:
         init_excp_970(env);
-        ppc970_irq_init(env);
+        ppc970_irq_init(ppc_env_get_cpu(env));
         break;
     case BOOK3S_CPU_POWER7:
-    case BOOK3S_CPU_POWER8:
         init_excp_POWER7(env);
-        ppcPOWER7_irq_init(env);
+        ppcPOWER7_irq_init(ppc_env_get_cpu(env));
+        break;
+    case BOOK3S_CPU_POWER8:
+        init_excp_POWER8(env);
+        ppcPOWER7_irq_init(ppc_env_get_cpu(env));
         break;
     default:
         g_assert_not_reached();
@@ -8014,7 +8412,7 @@
                     (1ull << MSR_DR) |
                     (1ull << MSR_PMM) |
                     (1ull << MSR_RI);
-    pcc->mmu_model = POWERPC_MMU_64B;
+    pcc->mmu_model = POWERPC_MMU_2_03;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
 #endif
@@ -8028,8 +8426,8 @@
     pcc->l1_icache_size = 0x10000;
 }
 
-static void powerpc_get_compat(Object *obj, Visitor *v,
-                               void *opaque, const char *name, Error **errp)
+static void powerpc_get_compat(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     char *value = (char *)"";
     Property *prop = opaque;
@@ -8048,23 +8446,23 @@
     case 0:
         break;
     default:
-        error_setg(errp, "Internal error: compat is set to %x",
-                   max_compat ? *max_compat : -1);
+        error_report("Internal error: compat is set to %x", *max_compat);
+        abort();
         break;
     }
 
-    visit_type_str(v, &value, name, errp);
+    visit_type_str(v, name, &value, errp);
 }
 
-static void powerpc_set_compat(Object *obj, Visitor *v,
-                               void *opaque, const char *name, Error **errp)
+static void powerpc_set_compat(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
 {
     Error *error = NULL;
     char *value = NULL;
     Property *prop = opaque;
     uint32_t *max_compat = qdev_get_prop_ptr(DEVICE(obj), prop);
 
-    visit_type_str(v, &value, name, &error);
+    visit_type_str(v, name, &value, &error);
     if (error) {
         error_propagate(errp, error);
         return;
@@ -8098,6 +8496,36 @@
     DEFINE_PROP_END_OF_LIST(),
 };
 
+#ifdef CONFIG_SOFTMMU
+static const struct ppc_segment_page_sizes POWER7_POWER8_sps = {
+    .sps = {
+        {
+            .page_shift = 12, /* 4K */
+            .slb_enc = 0,
+            .enc = { { .page_shift = 12, .pte_enc = 0 },
+                     { .page_shift = 16, .pte_enc = 0x7 },
+                     { .page_shift = 24, .pte_enc = 0x38 }, },
+        },
+        {
+            .page_shift = 16, /* 64K */
+            .slb_enc = SLB_VSID_64K,
+            .enc = { { .page_shift = 16, .pte_enc = 0x1 },
+                     { .page_shift = 24, .pte_enc = 0x8 }, },
+        },
+        {
+            .page_shift = 24, /* 16M */
+            .slb_enc = SLB_VSID_16M,
+            .enc = { { .page_shift = 24, .pte_enc = 0 }, },
+        },
+        {
+            .page_shift = 34, /* 16G */
+            .slb_enc = SLB_VSID_16G,
+            .enc = { { .page_shift = 34, .pte_enc = 0x3 }, },
+        },
+    }
+};
+#endif /* CONFIG_SOFTMMU */
+
 static void init_proc_POWER7 (CPUPPCState *env)
 {
     init_proc_book3s_64(env, BOOK3S_CPU_POWER7);
@@ -8114,18 +8542,55 @@
     return false;
 }
 
+static bool cpu_has_work_POWER7(CPUState *cs)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    if (cs->halted) {
+        if (!(cs->interrupt_request & CPU_INTERRUPT_HARD)) {
+            return false;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_EXT)) &&
+            (env->spr[SPR_LPCR] & LPCR_P7_PECE0)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_DECR)) &&
+            (env->spr[SPR_LPCR] & LPCR_P7_PECE1)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_MCK)) &&
+            (env->spr[SPR_LPCR] & LPCR_P7_PECE2)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_HMI)) &&
+            (env->spr[SPR_LPCR] & LPCR_P7_PECE2)) {
+            return true;
+        }
+        if (env->pending_interrupts & (1u << PPC_INTERRUPT_RESET)) {
+            return true;
+        }
+        return false;
+    } else {
+        return msr_ee && (cs->interrupt_request & CPU_INTERRUPT_HARD);
+    }
+}
+
 POWERPC_FAMILY(POWER7)(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
     PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+    CPUClass *cc = CPU_CLASS(oc);
 
     dc->fw_name = "PowerPC,POWER7";
     dc->desc = "POWER7";
     dc->props = powerpc_servercpu_properties;
     pcc->pvr_match = ppc_pvr_match_power7;
-    pcc->pcr_mask = PCR_COMPAT_2_05 | PCR_COMPAT_2_06;
+    pcc->pcr_mask = PCR_VEC_DIS | PCR_VSX_DIS | PCR_COMPAT_2_05;
+    pcc->pcr_supported = PCR_COMPAT_2_06 | PCR_COMPAT_2_05;
     pcc->init_proc = init_proc_POWER7;
     pcc->check_pow = check_pow_nocheck;
+    cc->has_work = cpu_has_work_POWER7;
     pcc->insns_flags = PPC_INSNS_BASE | PPC_ISEL | PPC_STRING | PPC_MFTB |
                        PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |
                        PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |
@@ -8135,13 +8600,15 @@
                        PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |
                        PPC_MEM_SYNC | PPC_MEM_EIEIO |
                        PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |
-                       PPC_64B | PPC_ALTIVEC |
+                       PPC_64B | PPC_64H | PPC_64BX | PPC_ALTIVEC |
                        PPC_SEGMENT_64B | PPC_SLBI |
-                       PPC_POPCNTB | PPC_POPCNTWD;
+                       PPC_POPCNTB | PPC_POPCNTWD |
+                       PPC_CILDST;
     pcc->insns_flags2 = PPC2_VSX | PPC2_DFP | PPC2_DBRX | PPC2_ISA205 |
                         PPC2_PERM_ISA206 | PPC2_DIVE_ISA206 |
                         PPC2_ATOMIC_ISA206 | PPC2_FP_CVT_ISA206 |
-                        PPC2_FP_TST_ISA206 | PPC2_FP_CVT_S64;
+                        PPC2_FP_TST_ISA206 | PPC2_FP_CVT_S64 |
+                        PPC2_PM_ISA206;
     pcc->msr_mask = (1ull << MSR_SF) |
                     (1ull << MSR_VR) |
                     (1ull << MSR_VSX) |
@@ -8161,6 +8628,7 @@
     pcc->mmu_model = POWERPC_MMU_2_06;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
+    pcc->sps = &POWER7_POWER8_sps;
 #endif
     pcc->excp_model = POWERPC_EXCP_POWER7;
     pcc->bus_model = PPC_FLAGS_INPUT_POWER7;
@@ -8181,6 +8649,9 @@
 
 static bool ppc_pvr_match_power8(PowerPCCPUClass *pcc, uint32_t pvr)
 {
+    if ((pvr & CPU_POWERPC_POWER_SERVER_MASK) == CPU_POWERPC_POWER8NVL_BASE) {
+        return true;
+    }
     if ((pvr & CPU_POWERPC_POWER_SERVER_MASK) == CPU_POWERPC_POWER8E_BASE) {
         return true;
     }
@@ -8190,18 +8661,63 @@
     return false;
 }
 
+static bool cpu_has_work_POWER8(CPUState *cs)
+{
+    PowerPCCPU *cpu = POWERPC_CPU(cs);
+    CPUPPCState *env = &cpu->env;
+
+    if (cs->halted) {
+        if (!(cs->interrupt_request & CPU_INTERRUPT_HARD)) {
+            return false;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_EXT)) &&
+            (env->spr[SPR_LPCR] & LPCR_P8_PECE2)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_DECR)) &&
+            (env->spr[SPR_LPCR] & LPCR_P8_PECE3)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_MCK)) &&
+            (env->spr[SPR_LPCR] & LPCR_P8_PECE4)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_HMI)) &&
+            (env->spr[SPR_LPCR] & LPCR_P8_PECE4)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_DOORBELL)) &&
+            (env->spr[SPR_LPCR] & LPCR_P8_PECE0)) {
+            return true;
+        }
+        if ((env->pending_interrupts & (1u << PPC_INTERRUPT_HDOORBELL)) &&
+            (env->spr[SPR_LPCR] & LPCR_P8_PECE1)) {
+            return true;
+        }
+        if (env->pending_interrupts & (1u << PPC_INTERRUPT_RESET)) {
+            return true;
+        }
+        return false;
+    } else {
+        return msr_ee && (cs->interrupt_request & CPU_INTERRUPT_HARD);
+    }
+}
+
 POWERPC_FAMILY(POWER8)(ObjectClass *oc, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(oc);
     PowerPCCPUClass *pcc = POWERPC_CPU_CLASS(oc);
+    CPUClass *cc = CPU_CLASS(oc);
 
     dc->fw_name = "PowerPC,POWER8";
     dc->desc = "POWER8";
     dc->props = powerpc_servercpu_properties;
     pcc->pvr_match = ppc_pvr_match_power8;
-    pcc->pcr_mask = PCR_COMPAT_2_05 | PCR_COMPAT_2_06;
+    pcc->pcr_mask = PCR_TM_DIS | PCR_COMPAT_2_06 | PCR_COMPAT_2_05;
+    pcc->pcr_supported = PCR_COMPAT_2_07 | PCR_COMPAT_2_06 | PCR_COMPAT_2_05;
     pcc->init_proc = init_proc_POWER8;
     pcc->check_pow = check_pow_nocheck;
+    cc->has_work = cpu_has_work_POWER8;
     pcc->insns_flags = PPC_INSNS_BASE | PPC_ISEL | PPC_STRING | PPC_MFTB |
                        PPC_FLOAT | PPC_FLOAT_FSEL | PPC_FLOAT_FRES |
                        PPC_FLOAT_FSQRT | PPC_FLOAT_FRSQRTE |
@@ -8211,16 +8727,19 @@
                        PPC_CACHE | PPC_CACHE_ICBI | PPC_CACHE_DCBZ |
                        PPC_MEM_SYNC | PPC_MEM_EIEIO |
                        PPC_MEM_TLBIE | PPC_MEM_TLBSYNC |
-                       PPC_64B | PPC_64BX | PPC_ALTIVEC |
+                       PPC_64B | PPC_64H | PPC_64BX | PPC_ALTIVEC |
                        PPC_SEGMENT_64B | PPC_SLBI |
-                       PPC_POPCNTB | PPC_POPCNTWD;
+                       PPC_POPCNTB | PPC_POPCNTWD |
+                       PPC_CILDST;
     pcc->insns_flags2 = PPC2_VSX | PPC2_VSX207 | PPC2_DFP | PPC2_DBRX |
                         PPC2_PERM_ISA206 | PPC2_DIVE_ISA206 |
                         PPC2_ATOMIC_ISA206 | PPC2_FP_CVT_ISA206 |
                         PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 |
                         PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 |
-                        PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64;
+                        PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 |
+                        PPC2_TM | PPC2_PM_ISA206;
     pcc->msr_mask = (1ull << MSR_SF) |
+                    (1ull << MSR_SHV) |
                     (1ull << MSR_TM) |
                     (1ull << MSR_VR) |
                     (1ull << MSR_VSX) |
@@ -8237,23 +8756,82 @@
                     (1ull << MSR_PMM) |
                     (1ull << MSR_RI) |
                     (1ull << MSR_LE);
-    pcc->mmu_model = POWERPC_MMU_2_06;
+    pcc->mmu_model = POWERPC_MMU_2_07;
 #if defined(CONFIG_SOFTMMU)
     pcc->handle_mmu_fault = ppc_hash64_handle_mmu_fault;
+    pcc->sps = &POWER7_POWER8_sps;
 #endif
-    pcc->excp_model = POWERPC_EXCP_POWER7;
+    pcc->excp_model = POWERPC_EXCP_POWER8;
     pcc->bus_model = PPC_FLAGS_INPUT_POWER7;
     pcc->bfd_mach = bfd_mach_ppc64;
     pcc->flags = POWERPC_FLAG_VRE | POWERPC_FLAG_SE |
                  POWERPC_FLAG_BE | POWERPC_FLAG_PMM |
                  POWERPC_FLAG_BUS_CLK | POWERPC_FLAG_CFAR |
-                 POWERPC_FLAG_VSX;
+                 POWERPC_FLAG_VSX | POWERPC_FLAG_TM;
     pcc->l1_dcache_size = 0x8000;
     pcc->l1_icache_size = 0x8000;
     pcc->interrupts_big_endian = ppc_cpu_interrupts_big_endian_lpcr;
 }
-#endif /* defined (TARGET_PPC64) */
 
+#if !defined(CONFIG_USER_ONLY)
+
+void cpu_ppc_set_papr(PowerPCCPU *cpu)
+{
+    CPUPPCState *env = &cpu->env;
+    ppc_spr_t *lpcr = &env->spr_cb[SPR_LPCR];
+    ppc_spr_t *amor = &env->spr_cb[SPR_AMOR];
+
+    /* PAPR always has exception vectors in RAM not ROM. To ensure this,
+     * MSR[IP] should never be set.
+     *
+     * We also disallow setting of MSR_HV
+     */
+    env->msr_mask &= ~((1ull << MSR_EP) | MSR_HVB);
+
+    /* Set emulated LPCR to not send interrupts to hypervisor. Note that
+     * under KVM, the actual HW LPCR will be set differently by KVM itself,
+     * the settings below ensure proper operations with TCG in absence of
+     * a real hypervisor.
+     *
+     * Clearing VPM0 will also cause us to use RMOR in mmu-hash64.c for
+     * real mode accesses, which thankfully defaults to 0 and isn't
+     * accessible in guest mode.
+     */
+    lpcr->default_value &= ~(LPCR_VPM0 | LPCR_VPM1 | LPCR_ISL | LPCR_KBV);
+    lpcr->default_value |= LPCR_LPES0 | LPCR_LPES1;
+
+    /* Set RMLS to the max (ie, 16G) */
+    lpcr->default_value &= ~LPCR_RMLS;
+    lpcr->default_value |= 1ull << LPCR_RMLS_SHIFT;
+
+    /* P7 and P8 has slightly different PECE bits, mostly because P8 adds
+     * bit 47 and 48 which are reserved on P7. Here we set them all, which
+     * will work as expected for both implementations
+     */
+    lpcr->default_value |= LPCR_P8_PECE0 | LPCR_P8_PECE1 | LPCR_P8_PECE2 |
+                           LPCR_P8_PECE3 | LPCR_P8_PECE4;
+
+    /* We should be followed by a CPU reset but update the active value
+     * just in case...
+     */
+    env->spr[SPR_LPCR] = lpcr->default_value;
+
+    /* Set a full AMOR so guest can use the AMR as it sees fit */
+    env->spr[SPR_AMOR] = amor->default_value = 0xffffffffffffffffull;
+
+    /* Update some env bits based on new LPCR value */
+    ppc_hash64_update_rmls(env);
+    ppc_hash64_update_vrma(env);
+
+    /* Tell KVM that we're in PAPR mode */
+    if (kvm_enabled()) {
+        kvmppc_set_papr(cpu);
+    }
+}
+
+#endif /* !defined(CONFIG_USER_ONLY) */
+
+#endif /* defined (TARGET_PPC64) */
 
 /*****************************************************************************/
 /* Generic CPU instantiation routine                                         */
@@ -8464,8 +9042,6 @@
 #endif
 
 /*****************************************************************************/
-#include <stdlib.h>
-#include <string.h>
 
 /* Opcode types */
 enum {
@@ -8571,11 +9147,7 @@
                               unsigned char idx1, unsigned char idx2,
                               opc_handler_t *handler)
 {
-    int ret;
-
-    ret = register_ind_in_table(ppc_opcodes, idx1, idx2, handler);
-
-    return ret;
+    return register_ind_in_table(ppc_opcodes, idx1, idx2, handler);
 }
 
 static int register_dblind_insn (opc_handler_t **ppc_opcodes,
@@ -8748,14 +9320,25 @@
 }
 #endif
 
+static bool avr_need_swap(CPUPPCState *env)
+{
+#ifdef HOST_WORDS_BIGENDIAN
+    return msr_le;
+#else
+    return !msr_le;
+#endif
+}
+
 static int gdb_get_float_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
         stfq_p(mem_buf, env->fpr[n]);
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         return 8;
     }
     if (n == 32) {
         stl_p(mem_buf, env->fpscr);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     return 0;
@@ -8764,10 +9347,12 @@
 static int gdb_set_float_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         env->fpr[n] = ldfq_p(mem_buf);
         return 8;
     }
     if (n == 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         helper_store_fpscr(env, ldl_p(mem_buf), 0xffffffff);
         return 4;
     }
@@ -8777,21 +9362,25 @@
 static int gdb_get_avr_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
-#ifdef HOST_WORDS_BIGENDIAN
-        stq_p(mem_buf, env->avr[n].u64[0]);
-        stq_p(mem_buf+8, env->avr[n].u64[1]);
-#else
-        stq_p(mem_buf, env->avr[n].u64[1]);
-        stq_p(mem_buf+8, env->avr[n].u64[0]);
-#endif
+        if (!avr_need_swap(env)) {
+            stq_p(mem_buf, env->avr[n].u64[0]);
+            stq_p(mem_buf+8, env->avr[n].u64[1]);
+        } else {
+            stq_p(mem_buf, env->avr[n].u64[1]);
+            stq_p(mem_buf+8, env->avr[n].u64[0]);
+        }
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        ppc_maybe_bswap_register(env, mem_buf + 8, 8);
         return 16;
     }
     if (n == 32) {
         stl_p(mem_buf, env->vscr);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     if (n == 33) {
         stl_p(mem_buf, (uint32_t)env->spr[SPR_VRSAVE]);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     return 0;
@@ -8800,20 +9389,24 @@
 static int gdb_set_avr_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
 {
     if (n < 32) {
-#ifdef HOST_WORDS_BIGENDIAN
-        env->avr[n].u64[0] = ldq_p(mem_buf);
-        env->avr[n].u64[1] = ldq_p(mem_buf+8);
-#else
-        env->avr[n].u64[1] = ldq_p(mem_buf);
-        env->avr[n].u64[0] = ldq_p(mem_buf+8);
-#endif
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        ppc_maybe_bswap_register(env, mem_buf + 8, 8);
+        if (!avr_need_swap(env)) {
+            env->avr[n].u64[0] = ldq_p(mem_buf);
+            env->avr[n].u64[1] = ldq_p(mem_buf+8);
+        } else {
+            env->avr[n].u64[1] = ldq_p(mem_buf);
+            env->avr[n].u64[0] = ldq_p(mem_buf+8);
+        }
         return 16;
     }
     if (n == 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         env->vscr = ldl_p(mem_buf);
         return 4;
     }
     if (n == 33) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         env->spr[SPR_VRSAVE] = (target_ulong)ldl_p(mem_buf);
         return 4;
     }
@@ -8825,6 +9418,7 @@
     if (n < 32) {
 #if defined(TARGET_PPC64)
         stl_p(mem_buf, env->gpr[n] >> 32);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
 #else
         stl_p(mem_buf, env->gprh[n]);
 #endif
@@ -8832,10 +9426,12 @@
     }
     if (n == 32) {
         stq_p(mem_buf, env->spe_acc);
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         return 8;
     }
     if (n == 33) {
         stl_p(mem_buf, env->spe_fscr);
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         return 4;
     }
     return 0;
@@ -8846,7 +9442,11 @@
     if (n < 32) {
 #if defined(TARGET_PPC64)
         target_ulong lo = (uint32_t)env->gpr[n];
-        target_ulong hi = (target_ulong)ldl_p(mem_buf) << 32;
+        target_ulong hi;
+
+        ppc_maybe_bswap_register(env, mem_buf, 4);
+
+        hi = (target_ulong)ldl_p(mem_buf) << 32;
         env->gpr[n] = lo | hi;
 #else
         env->gprh[n] = ldl_p(mem_buf);
@@ -8854,16 +9454,38 @@
         return 4;
     }
     if (n == 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 8);
         env->spe_acc = ldq_p(mem_buf);
         return 8;
     }
     if (n == 33) {
+        ppc_maybe_bswap_register(env, mem_buf, 4);
         env->spe_fscr = ldl_p(mem_buf);
         return 4;
     }
     return 0;
 }
 
+static int gdb_get_vsx_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
+{
+    if (n < 32) {
+        stq_p(mem_buf, env->vsr[n]);
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        return 8;
+    }
+    return 0;
+}
+
+static int gdb_set_vsx_reg(CPUPPCState *env, uint8_t *mem_buf, int n)
+{
+    if (n < 32) {
+        ppc_maybe_bswap_register(env, mem_buf, 8);
+        env->vsr[n] = ldq_p(mem_buf);
+        return 8;
+    }
+    return 0;
+}
+
 static int ppc_fixup_cpu(PowerPCCPU *cpu)
 {
     CPUPPCState *env = &cpu->env;
@@ -8906,7 +9528,7 @@
     PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
     Error *local_err = NULL;
 #if !defined(CONFIG_USER_ONLY)
-    int max_smt = kvm_enabled() ? kvmppc_smt_threads() : 1;
+    int max_smt = kvmppc_smt_threads();
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
@@ -8921,9 +9543,25 @@
                    smp_threads, kvm_enabled() ? "KVM" : "TCG");
         return;
     }
+#endif
 
+    cpu_exec_init(cs, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+#if !defined(CONFIG_USER_ONLY)
     cpu->cpu_dt_id = (cs->cpu_index / smp_threads) * max_smt
         + (cs->cpu_index % smp_threads);
+
+    if (kvm_enabled() && !kvm_vcpu_id_is_valid(cpu->cpu_dt_id)) {
+        error_setg(errp, "Can't create CPU with id %d in KVM", cpu->cpu_dt_id);
+        error_append_hint(errp, "Adjust the number of cpus to %d "
+                          "or try to raise the number of threads per core\n",
+                          cpu->cpu_dt_id * smp_threads / max_smt);
+        return;
+    }
 #endif
 
     if (tcg_enabled()) {
@@ -8961,6 +9599,10 @@
         gdb_register_coprocessor(cs, gdb_get_spe_reg, gdb_set_spe_reg,
                                  34, "power-spe.xml", 0);
     }
+    if (pcc->insns_flags2 & PPC2_VSX) {
+        gdb_register_coprocessor(cs, gdb_get_vsx_reg, gdb_set_vsx_reg,
+                                 32, "power-vsx.xml", 0);
+    }
 
     qemu_init_vcpu(cs);
 
@@ -9135,6 +9777,8 @@
     opc_handler_t **table;
     int i, j;
 
+    cpu_exec_exit(CPU(dev));
+
     for (i = 0; i < PPC_CPU_OPCODES_LEN; i++) {
         if (env->opcodes[i] == &invalid_handler) {
             continue;
@@ -9173,35 +9817,46 @@
     return ret;
 }
 
-int ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version)
+#ifdef TARGET_PPC64
+void ppc_set_compat(PowerPCCPU *cpu, uint32_t cpu_version, Error **errp)
 {
     int ret = 0;
     CPUPPCState *env = &cpu->env;
+    PowerPCCPUClass *host_pcc;
 
     cpu->cpu_version = cpu_version;
 
     switch (cpu_version) {
     case CPU_POWERPC_LOGICAL_2_05:
-        env->spr[SPR_PCR] = PCR_COMPAT_2_05;
+        env->spr[SPR_PCR] = PCR_TM_DIS | PCR_VSX_DIS | PCR_COMPAT_2_07 |
+                            PCR_COMPAT_2_06 | PCR_COMPAT_2_05;
         break;
     case CPU_POWERPC_LOGICAL_2_06:
-        env->spr[SPR_PCR] = PCR_COMPAT_2_06;
-        break;
     case CPU_POWERPC_LOGICAL_2_06_PLUS:
-        env->spr[SPR_PCR] = PCR_COMPAT_2_06;
+        env->spr[SPR_PCR] = PCR_TM_DIS | PCR_COMPAT_2_07 | PCR_COMPAT_2_06;
+        break;
+    case CPU_POWERPC_LOGICAL_2_07:
+        env->spr[SPR_PCR] = PCR_COMPAT_2_07;
         break;
     default:
         env->spr[SPR_PCR] = 0;
         break;
     }
 
-    if (kvm_enabled() && kvmppc_set_compat(cpu, cpu->cpu_version) < 0) {
-        error_report("Unable to set compatibility mode in KVM");
-        ret = -1;
+    host_pcc = kvm_ppc_get_host_cpu_class();
+    if (host_pcc) {
+        env->spr[SPR_PCR] &= host_pcc->pcr_mask;
     }
 
-    return ret;
+    if (kvm_enabled()) {
+        ret = kvmppc_set_compat(cpu, cpu->cpu_version);
+        if (ret < 0) {
+            error_setg_errno(errp, -ret,
+                             "Unable to set CPU compatibility mode in KVM");
+        }
+    }
 }
+#endif
 
 static gint ppc_cpu_compare_class_pvr(gconstpointer a, gconstpointer b)
 {
@@ -9290,7 +9945,6 @@
     return -1;
 }
 
-#include <ctype.h>
 
 static ObjectClass *ppc_cpu_class_by_name(const char *name);
 
@@ -9334,8 +9988,7 @@
                 break;
         }
         if (i == 8) {
-            ret = OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
-            return ret;
+            return OBJECT_CLASS(ppc_cpu_class_by_pvr(strtoul(name, NULL, 16)));
         }
     }
 
@@ -9359,6 +10012,19 @@
     return NULL;
 }
 
+const char *ppc_cpu_lookup_alias(const char *alias)
+{
+    int ai;
+
+    for (ai = 0; ppc_cpu_aliases[ai].alias != NULL; ai++) {
+        if (strcmp(ppc_cpu_aliases[ai].alias, alias) == 0) {
+            return ppc_cpu_aliases[ai].model;
+        }
+    }
+
+    return NULL;
+}
+
 PowerPCCPU *cpu_ppc_init(const char *cpu_model)
 {
     return POWERPC_CPU(cpu_generic_init(TYPE_POWERPC_CPU, cpu_model));
@@ -9534,10 +10200,7 @@
     pcc->parent_reset(s);
 
     msr = (target_ulong)0;
-    if (0) {
-        /* XXX: find a suitable condition to enable the hypervisor mode */
-        msr |= (target_ulong)MSR_HVB;
-    }
+    msr |= (target_ulong)MSR_HVB;
     msr |= (target_ulong)0 << MSR_AP; /* TO BE CHECKED */
     msr |= (target_ulong)0 << MSR_SA; /* TO BE CHECKED */
     msr |= (target_ulong)1 << MSR_EP;
@@ -9566,7 +10229,7 @@
 
 #if defined(TARGET_PPC64)
     if (env->mmu_model & POWERPC_MMU_64) {
-        env->msr |= (1ULL << MSR_SF);
+        msr |= (1ULL << MSR_SF);
     }
 #endif
 
@@ -9627,8 +10290,6 @@
     CPUPPCState *env = &cpu->env;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
-    cpu->cpu_dt_id = cs->cpu_index;
 
     env->msr_mask = pcc->msr_mask;
     env->mmu_model = pcc->mmu_model;
@@ -9640,12 +10301,25 @@
     env->bfd_mach = pcc->bfd_mach;
     env->check_pow = pcc->check_pow;
 
+    /* Mark HV mode as supported if the CPU has an MSR_HV bit
+     * in the msr_mask. The mask can later be cleared by PAPR
+     * mode but the hv mode support will remain, thus enforcing
+     * that we cannot use priv. instructions in guest in PAPR
+     * mode. For 970 we currently simply don't set HV in msr_mask
+     * thus simulating an "Apple mode" 970. If we ever want to
+     * support 970 HV mode, we'll have to add a processor attribute
+     * of some sort.
+     */
+#if !defined(CONFIG_USER_ONLY)
+    env->has_hv_mode = !!(env->msr_mask & MSR_HVB);
+#endif
+
 #if defined(TARGET_PPC64)
     if (pcc->sps) {
         env->sps = *pcc->sps;
     } else if (env->mmu_model & POWERPC_MMU_64) {
-        /* Use default sets of page sizes */
-        static const struct ppc_segment_page_sizes defsps = {
+        /* Use default sets of page sizes. We don't support MPSS */
+        static const struct ppc_segment_page_sizes defsps_4k = {
             .sps = {
                 { .page_shift = 12, /* 4K */
                   .slb_enc = 0,
@@ -9657,7 +10331,23 @@
                 },
             },
         };
-        env->sps = defsps;
+        static const struct ppc_segment_page_sizes defsps_64k = {
+            .sps = {
+                { .page_shift = 12, /* 4K */
+                  .slb_enc = 0,
+                  .enc = { { .page_shift = 12, .pte_enc = 0 } }
+                },
+                { .page_shift = 16, /* 64K */
+                  .slb_enc = 0x110,
+                  .enc = { { .page_shift = 16, .pte_enc = 1 } }
+                },
+                { .page_shift = 24, /* 16M */
+                  .slb_enc = 0x100,
+                  .enc = { { .page_shift = 24, .pte_enc = 0 } }
+                },
+            },
+        };
+        env->sps = (env->mmu_model & POWERPC_MMU_64K) ? defsps_64k : defsps_4k;
     }
 #endif /* defined(TARGET_PPC64) */
 
@@ -9711,7 +10401,6 @@
     cc->vmsd = &vmstate_ppc_cpu;
 #if defined(TARGET_PPC64)
     cc->write_elf64_note = ppc64_cpu_write_elf64_note;
-    cc->write_elf64_qemunote = ppc64_cpu_write_elf64_qemunote;
 #endif
 #endif
     cc->cpu_exec_enter = ppc_cpu_exec_enter;
diff --git a/target-ppc/user_only_helper.c b/target-ppc/user_only_helper.c
index 829f66f..6aff347 100644
--- a/target-ppc/user_only_helper.c
+++ b/target-ppc/user_only_helper.c
@@ -18,6 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 
 int ppc_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
diff --git a/target-s390x/Makefile.objs b/target-s390x/Makefile.objs
index 2c57494..dd62cbd 100644
--- a/target-s390x/Makefile.objs
+++ b/target-s390x/Makefile.objs
@@ -1,5 +1,5 @@
 obj-y += translate.o helper.o cpu.o interrupt.o
 obj-y += int_helper.o fpu_helper.o cc_helper.o mem_helper.o misc_helper.o
 obj-y += gdbstub.o
-obj-$(CONFIG_SOFTMMU) += machine.o ioinst.o arch_dump.o
+obj-$(CONFIG_SOFTMMU) += machine.o ioinst.o arch_dump.o mmu_helper.o
 obj-$(CONFIG_KVM) += kvm.o
diff --git a/target-s390x/arch_dump.c b/target-s390x/arch_dump.c
index a1554f5..4731869 100644
--- a/target-s390x/arch_dump.c
+++ b/target-s390x/arch_dump.c
@@ -11,6 +11,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "elf.h"
 #include "exec/cpu-all.h"
@@ -44,6 +45,18 @@
 
 typedef struct S390xElfFpregsetStruct S390xElfFpregset;
 
+struct S390xElfVregsLoStruct {
+    uint64_t vregs[16];
+} QEMU_PACKED;
+
+typedef struct S390xElfVregsLoStruct S390xElfVregsLo;
+
+struct S390xElfVregsHiStruct {
+    uint64_t vregs[16][2];
+} QEMU_PACKED;
+
+typedef struct S390xElfVregsHiStruct S390xElfVregsHi;
+
 typedef struct noteStruct {
     Elf64_Nhdr hdr;
     char name[5];
@@ -51,6 +64,8 @@
     union {
         S390xElfPrstatus prstatus;
         S390xElfFpregset fpregset;
+        S390xElfVregsLo vregslo;
+        S390xElfVregsHi vregshi;
         uint32_t prefix;
         uint64_t timer;
         uint64_t todcmp;
@@ -78,14 +93,38 @@
 static void s390x_write_elf64_fpregset(Note *note, S390CPU *cpu)
 {
     int i;
+    CPUS390XState *cs = &cpu->env;
 
     note->hdr.n_type = cpu_to_be32(NT_FPREGSET);
     note->contents.fpregset.fpc = cpu_to_be32(cpu->env.fpc);
     for (i = 0; i <= 15; i++) {
-        note->contents.fpregset.fprs[i] = cpu_to_be64(cpu->env.fregs[i].ll);
+        note->contents.fpregset.fprs[i] = cpu_to_be64(get_freg(cs, i)->ll);
     }
 }
 
+static void s390x_write_elf64_vregslo(Note *note, S390CPU *cpu)
+{
+    int i;
+
+    note->hdr.n_type = cpu_to_be32(NT_S390_VXRS_LOW);
+    for (i = 0; i <= 15; i++) {
+        note->contents.vregslo.vregs[i] = cpu_to_be64(cpu->env.vregs[i][1].ll);
+    }
+}
+
+static void s390x_write_elf64_vregshi(Note *note, S390CPU *cpu)
+{
+    int i;
+    S390xElfVregsHi *temp_vregshi;
+
+    temp_vregshi = &note->contents.vregshi;
+
+    note->hdr.n_type = cpu_to_be32(NT_S390_VXRS_HIGH);
+    for (i = 0; i <= 15; i++) {
+        temp_vregshi->vregs[i][0] = cpu_to_be64(cpu->env.vregs[i + 16][0].ll);
+        temp_vregshi->vregs[i][1] = cpu_to_be64(cpu->env.vregs[i + 16][1].ll);
+    }
+}
 
 static void s390x_write_elf64_timer(Note *note, S390CPU *cpu)
 {
@@ -134,6 +173,8 @@
     {sizeof(((Note *)0)->contents.timer),    s390x_write_elf64_timer},
     {sizeof(((Note *)0)->contents.todcmp),   s390x_write_elf64_todcmp},
     {sizeof(((Note *)0)->contents.todpreg),  s390x_write_elf64_todpreg},
+    {sizeof(((Note *)0)->contents.vregslo),  s390x_write_elf64_vregslo},
+    {sizeof(((Note *)0)->contents.vregshi),  s390x_write_elf64_vregshi},
     { 0, NULL}
 };
 
@@ -206,9 +247,3 @@
 
     return (elf_note_size) * nr_cpus;
 }
-
-int s390_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                  CPUState *cpu, void *opaque)
-{
-    return 0;
-}
diff --git a/target-s390x/cc_helper.c b/target-s390x/cc_helper.c
index 373eb17..1cf8551 100644
--- a/target-s390x/cc_helper.c
+++ b/target-s390x/cc_helper.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 
@@ -179,16 +181,11 @@
 
 static uint32_t cc_calc_subb_64(uint64_t a1, uint64_t a2, uint64_t ar)
 {
-    /* We had borrow-in if normal subtraction isn't equal.  */
-    int borrow_in = ar - (a1 - a2);
     int borrow_out;
 
-    /* If a2 was ULONG_MAX, and borrow_in, then a2 is logically 65 bits,
-       and we must have had borrow out.  */
-    if (borrow_in && a2 == (uint64_t)-1) {
-        borrow_out = 1;
+    if (ar != a1 - a2) {	/* difference means borrow-in */
+        borrow_out = (a2 >= a1);
     } else {
-        a2 += borrow_in;
         borrow_out = (a2 > a1);
     }
 
@@ -200,7 +197,7 @@
     if ((uint64_t)dst == 0x8000000000000000ULL) {
         return 3;
     } else if (dst) {
-        return 1;
+        return 2;
     } else {
         return 0;
     }
@@ -285,16 +282,11 @@
 
 static uint32_t cc_calc_subb_32(uint32_t a1, uint32_t a2, uint32_t ar)
 {
-    /* We had borrow-in if normal subtraction isn't equal.  */
-    int borrow_in = ar - (a1 - a2);
     int borrow_out;
 
-    /* If a2 was UINT_MAX, and borrow_in, then a2 is logically 65 bits,
-       and we must have had borrow out.  */
-    if (borrow_in && a2 == (uint32_t)-1) {
-        borrow_out = 1;
+    if (ar != a1 - a2) {	/* difference means borrow-in */
+        borrow_out = (a2 >= a1);
     } else {
-        a2 += borrow_in;
         borrow_out = (a2 > a1);
     }
 
@@ -306,7 +298,7 @@
     if ((uint32_t)dst == 0x80000000UL) {
         return 3;
     } else if (dst) {
-        return 1;
+        return 2;
     } else {
         return 0;
     }
@@ -570,7 +562,7 @@
         env->psw.mask |= PSW_ASC_HOME;
         break;
     default:
-        qemu_log("unknown sacf mode: %" PRIx64 "\n", a1);
+        HELPER_LOG("unknown sacf mode: %" PRIx64 "\n", a1);
         program_interrupt(env, PGM_SPECIFICATION, 2);
         break;
     }
diff --git a/target-s390x/cpu-qom.h b/target-s390x/cpu-qom.h
index 8b376df..66b5d18 100644
--- a/target-s390x/cpu-qom.h
+++ b/target-s390x/cpu-qom.h
@@ -21,7 +21,6 @@
 #define QEMU_S390_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #define TYPE_S390_CPU "s390-cpu"
 
@@ -47,6 +46,8 @@
     CPUClass parent_class;
     /*< public >*/
 
+    int64_t next_cpu_id;
+
     DeviceRealize parent_realize;
     void (*parent_reset)(CPUState *cpu);
     void (*load_normal)(CPUState *cpu);
@@ -54,46 +55,6 @@
     void (*initial_cpu_reset)(CPUState *cpu);
 } S390CPUClass;
 
-/**
- * S390CPU:
- * @env: #CPUS390XState.
- *
- * An S/390 CPU.
- */
-typedef struct S390CPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUS390XState env;
-} S390CPU;
-
-static inline S390CPU *s390_env_get_cpu(CPUS390XState *env)
-{
-    return container_of(env, S390CPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(s390_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(S390CPU, env)
-
-#ifndef CONFIG_USER_ONLY
-extern const struct VMStateDescription vmstate_s390_cpu;
-#endif
-
-void s390_cpu_do_interrupt(CPUState *cpu);
-bool s390_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void s390_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
-                         int flags);
-int s390_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
-                              int cpuid, void *opaque);
-
-int s390_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
-                                  CPUState *cpu, void *opaque);
-hwaddr s390_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-hwaddr s390_cpu_get_phys_addr_debug(CPUState *cpu, vaddr addr);
-int s390_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int s390_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-void s390_cpu_gdb_init(CPUState *cs);
+typedef struct S390CPU S390CPU;
 
 #endif
diff --git a/target-s390x/cpu.c b/target-s390x/cpu.c
index 498f9e5..e43e2d6 100644
--- a/target-s390x/cpu.c
+++ b/target-s390x/cpu.c
@@ -23,14 +23,22 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/timer.h"
 #include "qemu/error-report.h"
-#include "hw/hw.h"
 #include "trace.h"
+#include "qapi/visitor.h"
+#include "migration/vmstate.h"
+#include "exec/exec-all.h"
 #ifndef CONFIG_USER_ONLY
+#include "hw/hw.h"
 #include "sysemu/arch_init.h"
+#include "sysemu/sysemu.h"
+#include "hw/s390x/sclp.h"
 #endif
 
 #define CR0_RESET       0xE0UL
@@ -96,6 +104,7 @@
 
     env->pfault_token = -1UL;
     scc->parent_reset(s);
+    cpu->env.sigp_order = 0;
     s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
     tlb_flush(s, 1);
 }
@@ -105,6 +114,7 @@
 {
     S390CPU *cpu = S390_CPU(s);
     CPUS390XState *env = &cpu->env;
+    int i;
 
     s390_cpu_reset(s);
     /* initial reset does not touch regs,fregs and aregs */
@@ -115,12 +125,24 @@
     env->cregs[0] = CR0_RESET;
     env->cregs[14] = CR14_RESET;
 
+    /* architectured initial value for Breaking-Event-Address register */
+    env->gbea = 1;
+
     env->pfault_token = -1UL;
+    env->ext_index = -1;
+    for (i = 0; i < ARRAY_SIZE(env->io_index); i++) {
+        env->io_index[i] = -1;
+    }
+
+    /* tininess for underflow is detected before rounding */
+    set_float_detect_tininess(float_tininess_before_rounding,
+                              &env->fpu_status);
 
     /* Reset state inside the kernel that we cannot access yet from QEMU. */
     if (kvm_enabled()) {
         kvm_s390_reset_vcpu(cpu);
     }
+    tlb_flush(s, 1);
 }
 
 /* CPUClass:reset() */
@@ -129,8 +151,10 @@
     S390CPU *cpu = S390_CPU(s);
     S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
     CPUS390XState *env = &cpu->env;
+    int i;
 
     scc->parent_reset(s);
+    cpu->env.sigp_order = 0;
     s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
 
     memset(env, 0, offsetof(CPUS390XState, cpu_num));
@@ -139,7 +163,18 @@
     env->cregs[0] = CR0_RESET;
     env->cregs[14] = CR14_RESET;
 
+    /* architectured initial value for Breaking-Event-Address register */
+    env->gbea = 1;
+
     env->pfault_token = -1UL;
+    env->ext_index = -1;
+    for (i = 0; i < ARRAY_SIZE(env->io_index); i++) {
+        env->io_index[i] = -1;
+    }
+
+    /* tininess for underflow is detected before rounding */
+    set_float_detect_tininess(float_tininess_before_rounding,
+                              &env->fpu_status);
 
     /* Reset state inside the kernel that we cannot access yet from QEMU. */
     if (kvm_enabled()) {
@@ -157,11 +192,49 @@
 }
 #endif
 
+static void s390_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_s390_64;
+    info->print_insn = print_insn_s390;
+}
+
 static void s390_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
     S390CPUClass *scc = S390_CPU_GET_CLASS(dev);
+    S390CPU *cpu = S390_CPU(dev);
+    CPUS390XState *env = &cpu->env;
+    Error *err = NULL;
 
+#if !defined(CONFIG_USER_ONLY)
+    if (cpu->id >= max_cpus) {
+        error_setg(&err, "Unable to add CPU: %" PRIi64
+                   ", max allowed: %d", cpu->id, max_cpus - 1);
+        goto out;
+    }
+#endif
+    if (cpu_exists(cpu->id)) {
+        error_setg(&err, "Unable to add CPU: %" PRIi64
+                   ", it already exists", cpu->id);
+        goto out;
+    }
+    if (cpu->id != scc->next_cpu_id) {
+        error_setg(&err, "Unable to add CPU: %" PRIi64
+                   ", The next available id is %" PRIi64, cpu->id,
+                   scc->next_cpu_id);
+        goto out;
+    }
+
+    cpu_exec_init(cs, &err);
+    if (err != NULL) {
+        goto out;
+    }
+    scc->next_cpu_id++;
+
+#if !defined(CONFIG_USER_ONLY)
+    qemu_register_reset(s390_cpu_machine_reset_cb, cpu);
+#endif
+    env->cpu_num = cpu->id;
     s390_cpu_gdb_init(cs);
     qemu_init_vcpu(cs);
 #if !defined(CONFIG_USER_ONLY)
@@ -170,7 +243,55 @@
     cpu_reset(cs);
 #endif
 
-    scc->parent_realize(dev, errp);
+    scc->parent_realize(dev, &err);
+
+#if !defined(CONFIG_USER_ONLY)
+    if (dev->hotplugged) {
+        raise_irq_cpu_hotplug();
+    }
+#endif
+
+out:
+    error_propagate(errp, err);
+}
+
+static void s390x_cpu_get_id(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
+{
+    S390CPU *cpu = S390_CPU(obj);
+    int64_t value = cpu->id;
+
+    visit_type_int(v, name, &value, errp);
+}
+
+static void s390x_cpu_set_id(Object *obj, Visitor *v, const char *name,
+                             void *opaque, Error **errp)
+{
+    S390CPU *cpu = S390_CPU(obj);
+    DeviceState *dev = DEVICE(obj);
+    const int64_t min = 0;
+    const int64_t max = UINT32_MAX;
+    Error *err = NULL;
+    int64_t value;
+
+    if (dev->realized) {
+        error_setg(errp, "Attempt to set property '%s' on '%s' after "
+                   "it was realized", name, object_get_typename(obj));
+        return;
+    }
+
+    visit_type_int(v, name, &value, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return;
+    }
+    if (value < min || value > max) {
+        error_setg(errp, "Property %s.%s doesn't take value %" PRId64
+                   " (minimum: %" PRId64 ", maximum: %" PRId64 ")" ,
+                   object_get_typename(obj), name, value, min, max);
+        return;
+    }
+    cpu->id = value;
 }
 
 static void s390_cpu_initfn(Object *obj)
@@ -179,15 +300,16 @@
     S390CPU *cpu = S390_CPU(obj);
     CPUS390XState *env = &cpu->env;
     static bool inited;
-    static int cpu_num = 0;
 #if !defined(CONFIG_USER_ONLY)
     struct tm tm;
 #endif
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cs->halted = 1;
+    cs->exception_index = EXCP_HLT;
+    object_property_add(OBJECT(cpu), "id", "int64_t", s390x_cpu_get_id,
+                        s390x_cpu_set_id, NULL, NULL, NULL);
 #if !defined(CONFIG_USER_ONLY)
-    qemu_register_reset(s390_cpu_machine_reset_cb, cpu);
     qemu_get_timedate(&tm, 0);
     env->tod_offset = TOD_UNIX_EPOCH +
                       (time2tod(mktimegm(&tm)) * 1000000000ULL);
@@ -196,8 +318,6 @@
     env->cpu_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, s390x_cpu_timer, cpu);
     s390_cpu_set_state(CPU_STATE_STOPPED, cpu);
 #endif
-    env->cpu_num = cpu_num++;
-    env->ext_index = -1;
 
     if (tcg_enabled() && !inited) {
         inited = true;
@@ -211,6 +331,7 @@
     S390CPU *cpu = S390_CPU(obj);
 
     qemu_unregister_reset(s390_cpu_machine_reset_cb, cpu);
+    g_free(cpu->irqstate);
 #endif
 }
 
@@ -303,6 +424,7 @@
     CPUClass *cc = CPU_CLASS(scc);
     DeviceClass *dc = DEVICE_CLASS(oc);
 
+    scc->next_cpu_id = 0;
     scc->parent_realize = dc->realize;
     dc->realize = s390_cpu_realizefn;
 
@@ -325,12 +447,21 @@
     cc->get_phys_page_debug = s390_cpu_get_phys_page_debug;
     cc->vmsd = &vmstate_s390_cpu;
     cc->write_elf64_note = s390_cpu_write_elf64_note;
-    cc->write_elf64_qemunote = s390_cpu_write_elf64_qemunote;
     cc->cpu_exec_interrupt = s390_cpu_exec_interrupt;
+    cc->debug_excp_handler = s390x_cpu_debug_excp_handler;
 #endif
+    cc->disas_set_info = s390_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = S390_NUM_CORE_REGS;
     cc->gdb_core_xml_file = "s390x-core64.xml";
     cc->gdb_arch_name = s390_gdb_arch_name;
+
+    /*
+     * Reason: s390_cpu_realizefn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo s390_cpu_type_info = {
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index fe2f95d..c216bda 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -19,15 +19,15 @@
  * You should have received a copy of the GNU (Lesser) General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef CPU_S390X_H
-#define CPU_S390X_H
 
-#include "config.h"
+#ifndef S390X_CPU_H
+#define S390X_CPU_H
+
 #include "qemu-common.h"
+#include "cpu-qom.h"
 
 #define TARGET_LONG_BITS 64
 
-#define ELF_MACHINE	EM_S390
 #define ELF_MACHINE_UNAME "S390X"
 
 #define CPUArchState struct CPUS390XState
@@ -43,12 +43,13 @@
 #include "fpu/softfloat.h"
 
 #define NB_MMU_MODES 3
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 #define MMU_MODE0_SUFFIX _primary
 #define MMU_MODE1_SUFFIX _secondary
 #define MMU_MODE2_SUFFIX _home
 
-#define MMU_USER_IDX 1
+#define MMU_USER_IDX 0
 
 #define MAX_EXT_QUEUE 16
 #define MAX_IO_QUEUE 16
@@ -81,7 +82,11 @@
 
 typedef struct CPUS390XState {
     uint64_t regs[16];     /* GP registers */
-    CPU_DoubleU fregs[16]; /* FP registers */
+    /*
+     * The floating point registers are part of the vector registers.
+     * vregs[0][0] -> vregs[15][0] are 16 floating point registers
+     */
+    CPU_DoubleU vregs[32][2];  /* vector registers */
     uint32_t aregs[16];    /* access registers */
 
     uint32_t fpc;          /* floating-point control register */
@@ -107,6 +112,9 @@
     uint32_t int_svc_code;
     uint32_t int_svc_ilen;
 
+    uint64_t per_address;
+    uint16_t per_perc_atmid;
+
     uint64_t cregs[16]; /* control registers */
 
     ExtQueue ext_queue[MAX_EXT_QUEUE];
@@ -129,12 +137,14 @@
     uint64_t gbea;
     uint64_t pp;
 
+    uint8_t riccb[64];
+
     CPU_COMMON
 
     /* reset does memset(0) up to here */
 
-    int cpu_num;
-    uint8_t *storage_keys;
+    uint32_t cpu_num;
+    uint32_t machine_type;
 
     uint64_t tod_offset;
     uint64_t tod_basetime;
@@ -155,10 +165,62 @@
 #define CPU_STATE_LOAD                 0x04
     uint8_t cpu_state;
 
+    /* currently processed sigp order */
+    uint8_t sigp_order;
+
 } CPUS390XState;
 
-#include "cpu-qom.h"
-#include <sysemu/kvm.h>
+static inline CPU_DoubleU *get_freg(CPUS390XState *cs, int nr)
+{
+    return &cs->vregs[nr][0];
+}
+
+/**
+ * S390CPU:
+ * @env: #CPUS390XState.
+ *
+ * An S/390 CPU.
+ */
+struct S390CPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUS390XState env;
+    int64_t id;
+    /* needed for live migration */
+    void *irqstate;
+    uint32_t irqstate_saved_size;
+};
+
+static inline S390CPU *s390_env_get_cpu(CPUS390XState *env)
+{
+    return container_of(env, S390CPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(s390_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(S390CPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_s390_cpu;
+#endif
+
+void s390_cpu_do_interrupt(CPUState *cpu);
+bool s390_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void s390_cpu_dump_state(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
+                         int flags);
+int s390_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
+                              int cpuid, void *opaque);
+
+hwaddr s390_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+hwaddr s390_cpu_get_phys_addr_debug(CPUState *cpu, vaddr addr);
+int s390_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int s390_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void s390_cpu_gdb_init(CPUState *cs);
+void s390x_cpu_debug_excp_handler(CPUState *cs);
+
+#include "sysemu/kvm.h"
 
 /* distinguish between 24 bit and 31 bit addressing */
 #define HIGH_ORDER_BIT 0x80000000
@@ -285,19 +347,46 @@
 #define FLAG_MASK_32            0x00001000
 
 /* Control register 0 bits */
+#define CR0_LOWPROT             0x0000000010000000ULL
 #define CR0_EDAT                0x0000000000800000ULL
 
-static inline int cpu_mmu_index (CPUS390XState *env)
-{
-    if (env->psw.mask & PSW_MASK_PSTATE) {
-        return 1;
-    }
+/* MMU */
+#define MMU_PRIMARY_IDX         0
+#define MMU_SECONDARY_IDX       1
+#define MMU_HOME_IDX            2
 
-    return 0;
+static inline int cpu_mmu_index (CPUS390XState *env, bool ifetch)
+{
+    switch (env->psw.mask & PSW_MASK_ASC) {
+    case PSW_ASC_PRIMARY:
+        return MMU_PRIMARY_IDX;
+    case PSW_ASC_SECONDARY:
+        return MMU_SECONDARY_IDX;
+    case PSW_ASC_HOME:
+        return MMU_HOME_IDX;
+    case PSW_ASC_ACCREG:
+        /* Fallthrough: access register mode is not yet supported */
+    default:
+        abort();
+    }
+}
+
+static inline uint64_t cpu_mmu_idx_to_asc(int mmu_idx)
+{
+    switch (mmu_idx) {
+    case MMU_PRIMARY_IDX:
+        return PSW_ASC_PRIMARY;
+    case MMU_SECONDARY_IDX:
+        return PSW_ASC_SECONDARY;
+    case MMU_HOME_IDX:
+        return PSW_ASC_HOME;
+    default:
+        abort();
+    }
 }
 
 static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->psw.addr;
     *cs_base = 0;
@@ -323,17 +412,58 @@
     }
 }
 
+/* PER bits from control register 9 */
+#define PER_CR9_EVENT_BRANCH           0x80000000
+#define PER_CR9_EVENT_IFETCH           0x40000000
+#define PER_CR9_EVENT_STORE            0x20000000
+#define PER_CR9_EVENT_STORE_REAL       0x08000000
+#define PER_CR9_EVENT_NULLIFICATION    0x01000000
+#define PER_CR9_CONTROL_BRANCH_ADDRESS 0x00800000
+#define PER_CR9_CONTROL_ALTERATION     0x00200000
+
+/* PER bits from the PER CODE/ATMID/AI in lowcore */
+#define PER_CODE_EVENT_BRANCH          0x8000
+#define PER_CODE_EVENT_IFETCH          0x4000
+#define PER_CODE_EVENT_STORE           0x2000
+#define PER_CODE_EVENT_STORE_REAL      0x0800
+#define PER_CODE_EVENT_NULLIFICATION   0x0100
+
+/* Compute the ATMID field that is stored in the per_perc_atmid lowcore
+   entry when a PER exception is triggered.  */
+static inline uint8_t get_per_atmid(CPUS390XState *env)
+{
+    return ((env->psw.mask & PSW_MASK_64) ?      (1 << 7) : 0) |
+           (                                     (1 << 6)    ) |
+           ((env->psw.mask & PSW_MASK_32) ?      (1 << 5) : 0) |
+           ((env->psw.mask & PSW_MASK_DAT)?      (1 << 4) : 0) |
+           ((env->psw.mask & PSW_ASC_SECONDARY)? (1 << 3) : 0) |
+           ((env->psw.mask & PSW_ASC_ACCREG)?    (1 << 2) : 0);
+}
+
+/* Check if an address is within the PER starting address and the PER
+   ending address.  The address range might loop.  */
+static inline bool get_per_in_range(CPUS390XState *env, uint64_t addr)
+{
+    if (env->cregs[10] <= env->cregs[11]) {
+        return env->cregs[10] <= addr && addr <= env->cregs[11];
+    } else {
+        return env->cregs[10] <= addr || addr <= env->cregs[11];
+    }
+}
+
 #ifndef CONFIG_USER_ONLY
 /* In several cases of runtime exceptions, we havn't recorded the true
    instruction length.  Use these codes when raising exceptions in order
    to re-compute the length by examining the insn in memory.  */
 #define ILEN_LATER       0x20
 #define ILEN_LATER_INC   0x21
+void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilen);
 #endif
 
 S390CPU *cpu_s390x_init(const char *cpu_model);
+S390CPU *s390x_new_cpu(const char *cpu_model, int64_t id, Error **errp);
+S390CPU *cpu_s390x_create(const char *cpu_model, Error **errp);
 void s390x_translate_init(void);
-int cpu_s390x_exec(CPUS390XState *s);
 
 /* you can call this signal handler from your SIGBUS and SIGSEGV
    signal handlers to inform the virtual CPU of exceptions. non zero
@@ -343,14 +473,12 @@
 int s390_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
                               int mmu_idx);
 
-#include "ioinst.h"
 
 #ifndef CONFIG_USER_ONLY
-void *s390_cpu_physical_memory_map(CPUS390XState *env, hwaddr addr, hwaddr *len,
-                                   int is_write);
-void s390_cpu_physical_memory_unmap(CPUS390XState *env, void *addr, hwaddr len,
-                                    int is_write);
-static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb)
+void do_restart_interrupt(CPUS390XState *env);
+
+static inline hwaddr decode_basedisp_s(CPUS390XState *env, uint32_t ipb,
+                                       uint8_t *ar)
 {
     hwaddr addr = 0;
     uint8_t reg;
@@ -360,6 +488,9 @@
         addr = env->regs[reg];
     }
     addr += (ipb >> 16) & 0xfff;
+    if (ar) {
+        *ar = reg;
+    }
 
     return addr;
 }
@@ -386,33 +517,95 @@
 void s390x_cpu_timer(void *opaque);
 
 int s390_virtio_hypercall(CPUS390XState *env);
-void s390_virtio_irq(int config_change, uint64_t token);
 
 #ifdef CONFIG_KVM
-void kvm_s390_virtio_irq(int config_change, uint64_t token);
 void kvm_s390_service_interrupt(uint32_t parm);
 void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
 void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq);
 int kvm_s390_inject_flic(struct kvm_s390_irq *irq);
+void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
+int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
+                    int len, bool is_write);
+int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_clock);
+int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_clock);
 #else
-static inline void kvm_s390_virtio_irq(int config_change, uint64_t token)
-{
-}
 static inline void kvm_s390_service_interrupt(uint32_t parm)
 {
 }
+static inline int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    return -ENOSYS;
+}
+static inline int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    return -ENOSYS;
+}
+static inline int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar,
+                                  void *hostbuf, int len, bool is_write)
+{
+    return -ENOSYS;
+}
+static inline void kvm_s390_access_exception(S390CPU *cpu, uint16_t code,
+                                             uint64_t te_code)
+{
+}
 #endif
+
+static inline int s390_get_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_get_clock(tod_high, tod_low);
+    }
+    /* Fixme TCG */
+    *tod_high = 0;
+    *tod_low = 0;
+    return 0;
+}
+
+static inline int s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    if (kvm_enabled()) {
+        return kvm_s390_set_clock(tod_high, tod_low);
+    }
+    /* Fixme TCG */
+    return 0;
+}
+
 S390CPU *s390_cpu_addr2state(uint16_t cpu_addr);
 unsigned int s390_cpu_halt(S390CPU *cpu);
 void s390_cpu_unhalt(S390CPU *cpu);
 unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu);
+static inline uint8_t s390_cpu_get_state(S390CPU *cpu)
+{
+    return cpu->env.cpu_state;
+}
+
+void gtod_save(QEMUFile *f, void *opaque);
+int gtod_load(QEMUFile *f, void *opaque, int version_id);
+
+void cpu_inject_ext(S390CPU *cpu, uint32_t code, uint32_t param,
+                    uint64_t param64);
+
+/* ioinst.c */
+void ioinst_handle_xsch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_csch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_hsch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
+void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
+void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb);
+void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
+int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
+void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb);
+int ioinst_handle_tpi(S390CPU *cpu, uint32_t ipb);
+void ioinst_handle_schm(S390CPU *cpu, uint64_t reg1, uint64_t reg2,
+                        uint32_t ipb);
+void ioinst_handle_rsch(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_rchp(S390CPU *cpu, uint64_t reg1);
+void ioinst_handle_sal(S390CPU *cpu, uint64_t reg1);
 
 /* service interrupts are floating therefore we must not pass an cpustate */
 void s390_sclp_extint(uint32_t parm);
 
-/* from s390-virtio-bus */
-extern const hwaddr virtio_size;
-
 #else
 static inline unsigned int s390_cpu_halt(S390CPU *cpu)
 {
@@ -431,125 +624,14 @@
 void cpu_lock(void);
 void cpu_unlock(void);
 
-typedef struct SubchDev SubchDev;
+extern void subsystem_reset(void);
 
-#ifndef CONFIG_USER_ONLY
-extern void io_subsystem_reset(void);
-SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
-                         uint16_t schid);
-bool css_subch_visible(SubchDev *sch);
-void css_conditional_io_interrupt(SubchDev *sch);
-int css_do_stsch(SubchDev *sch, SCHIB *schib);
-bool css_schid_final(int m, uint8_t cssid, uint8_t ssid, uint16_t schid);
-int css_do_msch(SubchDev *sch, SCHIB *schib);
-int css_do_xsch(SubchDev *sch);
-int css_do_csch(SubchDev *sch);
-int css_do_hsch(SubchDev *sch);
-int css_do_ssch(SubchDev *sch, ORB *orb);
-int css_do_tsch(SubchDev *sch, IRB *irb);
-int css_do_stcrw(CRW *crw);
-int css_do_tpi(IOIntCode *int_code, int lowcore);
-int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid, uint8_t l_chpid,
-                         int rfmt, void *buf);
-void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo);
-int css_enable_mcsse(void);
-int css_enable_mss(void);
-int css_do_rsch(SubchDev *sch);
-int css_do_rchp(uint8_t cssid, uint8_t chpid);
-bool css_present(uint8_t cssid);
-#else
-static inline SubchDev *css_find_subch(uint8_t m, uint8_t cssid, uint8_t ssid,
-                                       uint16_t schid)
-{
-    return NULL;
-}
-static inline bool css_subch_visible(SubchDev *sch)
-{
-    return false;
-}
-static inline void css_conditional_io_interrupt(SubchDev *sch)
-{
-}
-static inline int css_do_stsch(SubchDev *sch, SCHIB *schib)
-{
-    return -ENODEV;
-}
-static inline bool css_schid_final(uint8_t cssid, uint8_t ssid, uint16_t schid)
-{
-    return true;
-}
-static inline int css_do_msch(SubchDev *sch, SCHIB *schib)
-{
-    return -ENODEV;
-}
-static inline int css_do_xsch(SubchDev *sch)
-{
-    return -ENODEV;
-}
-static inline int css_do_csch(SubchDev *sch)
-{
-    return -ENODEV;
-}
-static inline int css_do_hsch(SubchDev *sch)
-{
-    return -ENODEV;
-}
-static inline int css_do_ssch(SubchDev *sch, ORB *orb)
-{
-    return -ENODEV;
-}
-static inline int css_do_tsch(SubchDev *sch, IRB *irb)
-{
-    return -ENODEV;
-}
-static inline int css_do_stcrw(CRW *crw)
-{
-    return 1;
-}
-static inline int css_do_tpi(IOIntCode *int_code, int lowcore)
-{
-    return 0;
-}
-static inline int css_collect_chp_desc(int m, uint8_t cssid, uint8_t f_chpid,
-                                       int rfmt, uint8_t l_chpid, void *buf)
-{
-    return 0;
-}
-static inline void css_do_schm(uint8_t mbk, int update, int dct, uint64_t mbo)
-{
-}
-static inline int css_enable_mss(void)
-{
-    return -EINVAL;
-}
-static inline int css_enable_mcsse(void)
-{
-    return -EINVAL;
-}
-static inline int css_do_rsch(SubchDev *sch)
-{
-    return -ENODEV;
-}
-static inline int css_do_rchp(uint8_t cssid, uint8_t chpid)
-{
-    return -ENODEV;
-}
-static inline bool css_present(uint8_t cssid)
-{
-    return false;
-}
-#endif
-
-#define cpu_init(model) (&cpu_s390x_init(model)->env)
-#define cpu_exec cpu_s390x_exec
-#define cpu_gen_code cpu_s390x_gen_code
+#define cpu_init(model) CPU(cpu_s390x_init(model))
 #define cpu_signal_handler cpu_s390x_signal_handler
 
 void s390_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 #define cpu_list s390_cpu_list
 
-#include "exec/exec-all.h"
-
 #define EXCP_EXT 1 /* external interrupt */
 #define EXCP_SVC 2 /* supervisor call (syscall) */
 #define EXCP_PGM 3 /* program interruption */
@@ -693,6 +775,7 @@
 
     env->psw.mask &= ~(3ull << 44);
     env->psw.mask |= (cc & 3) << 44;
+    env->cc_op = cc;
 }
 
 typedef struct LowCore
@@ -730,15 +813,17 @@
     uint8_t         pad5[0xf4-0xf0];          /* 0x0f0 */
     uint32_t        external_damage_code;     /* 0x0f4 */
     uint64_t        failing_storage_address;  /* 0x0f8 */
-    uint8_t         pad6[0x120-0x100];        /* 0x100 */
+    uint8_t         pad6[0x110-0x100];        /* 0x100 */
+    uint64_t        per_breaking_event_addr;  /* 0x110 */
+    uint8_t         pad7[0x120-0x118];        /* 0x118 */
     PSW             restart_old_psw;          /* 0x120 */
     PSW             external_old_psw;         /* 0x130 */
     PSW             svc_old_psw;              /* 0x140 */
     PSW             program_old_psw;          /* 0x150 */
     PSW             mcck_old_psw;             /* 0x160 */
     PSW             io_old_psw;               /* 0x170 */
-    uint8_t         pad7[0x1a0-0x180];        /* 0x180 */
-    PSW             restart_psw;              /* 0x1a0 */
+    uint8_t         pad8[0x1a0-0x180];        /* 0x180 */
+    PSW             restart_new_psw;          /* 0x1a0 */
     PSW             external_new_psw;         /* 0x1b0 */
     PSW             svc_new_psw;              /* 0x1c0 */
     PSW             program_new_psw;          /* 0x1d0 */
@@ -755,10 +840,10 @@
     uint64_t        last_update_clock;        /* 0x280 */
     uint64_t        steal_clock;              /* 0x288 */
     PSW             return_mcck_psw;          /* 0x290 */
-    uint8_t         pad8[0xc00-0x2a0];        /* 0x2a0 */
+    uint8_t         pad9[0xc00-0x2a0];        /* 0x2a0 */
     /* System info area */
     uint64_t        save_area[16];            /* 0xc00 */
-    uint8_t         pad9[0xd40-0xc80];        /* 0xc80 */
+    uint8_t         pad10[0xd40-0xc80];       /* 0xc80 */
     uint64_t        kernel_stack;             /* 0xd40 */
     uint64_t        thread_info;              /* 0xd48 */
     uint64_t        async_stack;              /* 0xd50 */
@@ -766,7 +851,7 @@
     uint64_t        user_asce;                /* 0xd60 */
     uint64_t        panic_stack;              /* 0xd68 */
     uint64_t        user_exec_asce;           /* 0xd70 */
-    uint8_t         pad10[0xdc0-0xd78];       /* 0xd78 */
+    uint8_t         pad11[0xdc0-0xd78];       /* 0xd78 */
 
     /* SMP info area: defined by DJB */
     uint64_t        clock_comparator;         /* 0xdc0 */
@@ -895,9 +980,13 @@
         uint8_t  name[8];
         uint32_t caf;
         uint8_t  cpi[16];
-        uint8_t  res3[24];
+        uint8_t res5[3];
+        uint8_t ext_name_encoding;
+        uint32_t res3;
+        uint8_t uuid[16];
     } vm[8];
-    uint8_t res4[3552];
+    uint8_t res4[1504];
+    uint8_t ext_names[8][256];
 };
 
 /* MMU defines */
@@ -915,6 +1004,8 @@
 #define _ASCE_TABLE_LENGTH      0x03      /* region table length              */
 
 #define _REGION_ENTRY_ORIGIN    ~0xfffULL /* region/segment table origin      */
+#define _REGION_ENTRY_RO        0x200     /* region/segment protection bit    */
+#define _REGION_ENTRY_TF        0xc0      /* region/segment table offset      */
 #define _REGION_ENTRY_INV       0x20      /* invalid region table entry       */
 #define _REGION_ENTRY_TYPE_MASK 0x0c      /* region/segment table type mask   */
 #define _REGION_ENTRY_TYPE_R1   0x0c      /* region first table type          */
@@ -929,12 +1020,14 @@
 
 #define _PAGE_RO        0x200            /* HW read-only bit  */
 #define _PAGE_INVALID   0x400            /* HW invalid bit    */
+#define _PAGE_RES0      0x800            /* bit must be zero  */
 
 #define SK_C                    (0x1 << 1)
 #define SK_R                    (0x1 << 2)
 #define SK_F                    (0x1 << 3)
 #define SK_ACC_MASK             (0xf << 4)
 
+/* SIGP order codes */
 #define SIGP_SENSE             0x01
 #define SIGP_EXTERNAL_CALL     0x02
 #define SIGP_EMERGENCY         0x03
@@ -947,8 +1040,15 @@
 #define SIGP_SET_PREFIX        0x0d
 #define SIGP_STORE_STATUS_ADDR 0x0e
 #define SIGP_SET_ARCH          0x12
+#define SIGP_STORE_ADTL_STATUS 0x17
 
-/* cpu status bits */
+/* SIGP condition codes */
+#define SIGP_CC_ORDER_CODE_ACCEPTED 0
+#define SIGP_CC_STATUS_STORED       1
+#define SIGP_CC_BUSY                2
+#define SIGP_CC_NOT_OPERATIONAL     3
+
+/* SIGP status bits */
 #define SIGP_STAT_EQUIPMENT_CHECK   0x80000000UL
 #define SIGP_STAT_INCORRECT_STATE   0x00000200UL
 #define SIGP_STAT_INVALID_PARAMETER 0x00000100UL
@@ -960,14 +1060,28 @@
 #define SIGP_STAT_INVALID_ORDER     0x00000002UL
 #define SIGP_STAT_RECEIVER_CHECK    0x00000001UL
 
+/* SIGP SET ARCHITECTURE modes */
+#define SIGP_MODE_ESA_S390 0
+#define SIGP_MODE_Z_ARCH_TRANS_ALL_PSW 1
+#define SIGP_MODE_Z_ARCH_TRANS_CUR_PSW 2
+
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr);
 int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
-                  target_ulong *raddr, int *flags);
+                  target_ulong *raddr, int *flags, bool exc);
 int sclp_service_call(CPUS390XState *env, uint64_t sccb, uint32_t code);
 uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
                  uint64_t vr);
+void s390_cpu_recompute_watchpoints(CPUState *cs);
 
-#define TARGET_HAS_ICE 1
+int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
+                         int len, bool is_write);
+
+#define s390_cpu_virt_mem_read(cpu, laddr, ar, dest, len)    \
+        s390_cpu_virt_mem_rw(cpu, laddr, ar, dest, len, false)
+#define s390_cpu_virt_mem_write(cpu, laddr, ar, dest, len)       \
+        s390_cpu_virt_mem_rw(cpu, laddr, ar, dest, len, true)
+#define s390_cpu_virt_mem_check_write(cpu, laddr, ar, len)   \
+        s390_cpu_virt_mem_rw(cpu, laddr, ar, NULL, len, true)
 
 /* The value of the TOD clock for 1.1.1970. */
 #define TOD_UNIX_EPOCH 0x7d91048bca000000ULL
@@ -977,67 +1091,9 @@
     return (ns << 9) / 125;
 }
 
-static inline void cpu_inject_ext(S390CPU *cpu, uint32_t code, uint32_t param,
-                                  uint64_t param64)
-{
-    CPUS390XState *env = &cpu->env;
-
-    if (env->ext_index == MAX_EXT_QUEUE - 1) {
-        /* ugh - can't queue anymore. Let's drop. */
-        return;
-    }
-
-    env->ext_index++;
-    assert(env->ext_index < MAX_EXT_QUEUE);
-
-    env->ext_queue[env->ext_index].code = code;
-    env->ext_queue[env->ext_index].param = param;
-    env->ext_queue[env->ext_index].param64 = param64;
-
-    env->pending_int |= INTERRUPT_EXT;
-    cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
-}
-
-static inline void cpu_inject_io(S390CPU *cpu, uint16_t subchannel_id,
-                                 uint16_t subchannel_number,
-                                 uint32_t io_int_parm, uint32_t io_int_word)
-{
-    CPUS390XState *env = &cpu->env;
-    int isc = IO_INT_WORD_ISC(io_int_word);
-
-    if (env->io_index[isc] == MAX_IO_QUEUE - 1) {
-        /* ugh - can't queue anymore. Let's drop. */
-        return;
-    }
-
-    env->io_index[isc]++;
-    assert(env->io_index[isc] < MAX_IO_QUEUE);
-
-    env->io_queue[env->io_index[isc]][isc].id = subchannel_id;
-    env->io_queue[env->io_index[isc]][isc].nr = subchannel_number;
-    env->io_queue[env->io_index[isc]][isc].parm = io_int_parm;
-    env->io_queue[env->io_index[isc]][isc].word = io_int_word;
-
-    env->pending_int |= INTERRUPT_IO;
-    cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
-}
-
-static inline void cpu_inject_crw_mchk(S390CPU *cpu)
-{
-    CPUS390XState *env = &cpu->env;
-
-    if (env->mchk_index == MAX_MCHK_QUEUE - 1) {
-        /* ugh - can't queue anymore. Let's drop. */
-        return;
-    }
-
-    env->mchk_index++;
-    assert(env->mchk_index < MAX_MCHK_QUEUE);
-
-    env->mchk_queue[env->mchk_index].type = 1;
-
-    env->pending_int |= INTERRUPT_MCHK;
-    cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
+/* Converts s390's clock format to ns */
+static inline uint64_t tod2time(uint64_t t) {
+    return (t * 125) >> 9;
 }
 
 /* from s390-virtio-ccw */
@@ -1051,6 +1107,7 @@
 
 /* misc_helper.c */
 #ifndef CONFIG_USER_ONLY
+int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3);
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3);
 #endif
 void program_interrupt(CPUS390XState *env, uint32_t code, int ilen);
@@ -1067,9 +1124,14 @@
                                     int vq, bool assign);
 int kvm_s390_cpu_restart(S390CPU *cpu);
 int kvm_s390_get_memslot_count(KVMState *s);
-void kvm_s390_clear_cmma_callback(void *opaque);
+void kvm_s390_cmma_reset(void);
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
 void kvm_s390_reset_vcpu(S390CPU *cpu);
+int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit);
+void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
+int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
+int kvm_s390_get_ri(void);
+void kvm_s390_crypto_reset(void);
 #else
 static inline void kvm_s390_io_interrupt(uint16_t subchannel_id,
                                         uint16_t subchannel_nr,
@@ -1093,7 +1155,7 @@
 {
     return -ENOSYS;
 }
-static inline void kvm_s390_clear_cmma_callback(void *opaque)
+static inline void kvm_s390_cmma_reset(void)
 {
 }
 static inline int kvm_s390_get_memslot_count(KVMState *s)
@@ -1107,13 +1169,39 @@
 static inline void kvm_s390_reset_vcpu(S390CPU *cpu)
 {
 }
+static inline int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit,
+                                         uint64_t *hw_limit)
+{
+    return 0;
+}
+static inline void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu)
+{
+}
+static inline int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
+{
+    return 0;
+}
+static inline int kvm_s390_get_ri(void)
+{
+    return 0;
+}
+static inline void kvm_s390_crypto_reset(void)
+{
+}
 #endif
 
-static inline void cmma_reset(S390CPU *cpu)
+static inline int s390_set_memory_limit(uint64_t new_limit, uint64_t *hw_limit)
 {
     if (kvm_enabled()) {
-        CPUState *cs = CPU(cpu);
-        kvm_s390_clear_cmma_callback(cs->kvm_state);
+        return kvm_s390_set_mem_limit(kvm_state, new_limit, hw_limit);
+    }
+    return 0;
+}
+
+static inline void s390_cmma_reset(void)
+{
+    if (kvm_enabled()) {
+        kvm_s390_cmma_reset();
     }
 }
 
@@ -1142,11 +1230,58 @@
                                               uint32_t sch_id, int vq,
                                               bool assign)
 {
+    return kvm_s390_assign_subch_ioeventfd(notifier, sch_id, vq, assign);
+}
+
+static inline void s390_crypto_reset(void)
+{
     if (kvm_enabled()) {
-        return kvm_s390_assign_subch_ioeventfd(notifier, sch_id, vq, assign);
-    } else {
-        return -ENOSYS;
+        kvm_s390_crypto_reset();
     }
 }
 
+/* machine check interruption code */
+
+/* subclasses */
+#define MCIC_SC_SD 0x8000000000000000ULL
+#define MCIC_SC_PD 0x4000000000000000ULL
+#define MCIC_SC_SR 0x2000000000000000ULL
+#define MCIC_SC_CD 0x0800000000000000ULL
+#define MCIC_SC_ED 0x0400000000000000ULL
+#define MCIC_SC_DG 0x0100000000000000ULL
+#define MCIC_SC_W  0x0080000000000000ULL
+#define MCIC_SC_CP 0x0040000000000000ULL
+#define MCIC_SC_SP 0x0020000000000000ULL
+#define MCIC_SC_CK 0x0010000000000000ULL
+
+/* subclass modifiers */
+#define MCIC_SCM_B  0x0002000000000000ULL
+#define MCIC_SCM_DA 0x0000000020000000ULL
+#define MCIC_SCM_AP 0x0000000000080000ULL
+
+/* storage errors */
+#define MCIC_SE_SE 0x0000800000000000ULL
+#define MCIC_SE_SC 0x0000400000000000ULL
+#define MCIC_SE_KE 0x0000200000000000ULL
+#define MCIC_SE_DS 0x0000100000000000ULL
+#define MCIC_SE_IE 0x0000000080000000ULL
+
+/* validity bits */
+#define MCIC_VB_WP 0x0000080000000000ULL
+#define MCIC_VB_MS 0x0000040000000000ULL
+#define MCIC_VB_PM 0x0000020000000000ULL
+#define MCIC_VB_IA 0x0000010000000000ULL
+#define MCIC_VB_FA 0x0000008000000000ULL
+#define MCIC_VB_VR 0x0000004000000000ULL
+#define MCIC_VB_EC 0x0000002000000000ULL
+#define MCIC_VB_FP 0x0000001000000000ULL
+#define MCIC_VB_GR 0x0000000800000000ULL
+#define MCIC_VB_CR 0x0000000400000000ULL
+#define MCIC_VB_ST 0x0000000100000000ULL
+#define MCIC_VB_AR 0x0000000040000000ULL
+#define MCIC_VB_PR 0x0000000000200000ULL
+#define MCIC_VB_FC 0x0000000000100000ULL
+#define MCIC_VB_CT 0x0000000000020000ULL
+#define MCIC_VB_CC 0x0000000000010000ULL
+
 #endif
diff --git a/target-s390x/fpu_helper.c b/target-s390x/fpu_helper.c
index b946ec1..e604e9f 100644
--- a/target-s390x/fpu_helper.c
+++ b/target-s390x/fpu_helper.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "exec/helper-proto.h"
 
@@ -265,7 +267,7 @@
 {
     float64 ret = float32_to_float64(f2, &env->fpu_status);
     handle_exceptions(env, GETPC());
-    return ret;
+    return float64_maybe_silence_nan(ret, &env->fpu_status);
 }
 
 /* convert 128-bit float to 64-bit float */
@@ -273,7 +275,7 @@
 {
     float64 ret = float128_to_float64(make_float128(ah, al), &env->fpu_status);
     handle_exceptions(env, GETPC());
-    return ret;
+    return float64_maybe_silence_nan(ret, &env->fpu_status);
 }
 
 /* convert 64-bit float to 128-bit float */
@@ -281,7 +283,7 @@
 {
     float128 ret = float64_to_float128(f2, &env->fpu_status);
     handle_exceptions(env, GETPC());
-    return RET128(ret);
+    return RET128(float128_maybe_silence_nan(ret, &env->fpu_status));
 }
 
 /* convert 32-bit float to 128-bit float */
@@ -289,7 +291,7 @@
 {
     float128 ret = float32_to_float128(f2, &env->fpu_status);
     handle_exceptions(env, GETPC());
-    return RET128(ret);
+    return RET128(float128_maybe_silence_nan(ret, &env->fpu_status));
 }
 
 /* convert 64-bit float to 32-bit float */
@@ -297,7 +299,7 @@
 {
     float32 ret = float64_to_float32(f2, &env->fpu_status);
     handle_exceptions(env, GETPC());
-    return ret;
+    return float32_maybe_silence_nan(ret, &env->fpu_status);
 }
 
 /* convert 128-bit float to 32-bit float */
@@ -305,7 +307,7 @@
 {
     float32 ret = float128_to_float32(make_float128(ah, al), &env->fpu_status);
     handle_exceptions(env, GETPC());
-    return ret;
+    return float32_maybe_silence_nan(ret, &env->fpu_status);
 }
 
 /* 32-bit FP compare */
@@ -552,6 +554,37 @@
     return ret;
 }
 
+/* round to integer 32-bit */
+uint64_t HELPER(fieb)(CPUS390XState *env, uint64_t f2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float32 ret = float32_round_to_int(f2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* round to integer 64-bit */
+uint64_t HELPER(fidb)(CPUS390XState *env, uint64_t f2, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float64 ret = float64_round_to_int(f2, &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return ret;
+}
+
+/* round to integer 128-bit */
+uint64_t HELPER(fixb)(CPUS390XState *env, uint64_t ah, uint64_t al, uint32_t m3)
+{
+    int hold = swap_round_mode(env, m3);
+    float128 ret = float128_round_to_int(make_float128(ah, al),
+                                         &env->fpu_status);
+    set_float_rounding_mode(hold, &env->fpu_status);
+    handle_exceptions(env, GETPC());
+    return RET128(ret);
+}
+
 /* 32-bit FP multiply and add */
 uint64_t HELPER(maeb)(CPUS390XState *env, uint64_t f1,
                       uint64_t f2, uint64_t f3)
@@ -591,7 +624,7 @@
 }
 
 /* test data class 32-bit */
-uint32_t HELPER(tceb)(uint64_t f1, uint64_t m2)
+uint32_t HELPER(tceb)(CPUS390XState *env, uint64_t f1, uint64_t m2)
 {
     float32 v1 = f1;
     int neg = float32_is_neg(v1);
@@ -600,7 +633,8 @@
     if ((float32_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
         (float32_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
         (float32_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float32_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        (float32_is_signaling_nan(v1, &env->fpu_status) &&
+         (m2 & (1 << (1-neg))))) {
         cc = 1;
     } else if (m2 & (1 << (9-neg))) {
         /* assume normalized number */
@@ -611,7 +645,7 @@
 }
 
 /* test data class 64-bit */
-uint32_t HELPER(tcdb)(uint64_t v1, uint64_t m2)
+uint32_t HELPER(tcdb)(CPUS390XState *env, uint64_t v1, uint64_t m2)
 {
     int neg = float64_is_neg(v1);
     uint32_t cc = 0;
@@ -619,7 +653,8 @@
     if ((float64_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
         (float64_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
         (float64_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float64_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        (float64_is_signaling_nan(v1, &env->fpu_status) &&
+         (m2 & (1 << (1-neg))))) {
         cc = 1;
     } else if (m2 & (1 << (9-neg))) {
         /* assume normalized number */
@@ -630,7 +665,8 @@
 }
 
 /* test data class 128-bit */
-uint32_t HELPER(tcxb)(uint64_t ah, uint64_t al, uint64_t m2)
+uint32_t HELPER(tcxb)(CPUS390XState *env, uint64_t ah,
+                      uint64_t al, uint64_t m2)
 {
     float128 v1 = make_float128(ah, al);
     int neg = float128_is_neg(v1);
@@ -639,7 +675,8 @@
     if ((float128_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
         (float128_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
         (float128_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float128_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        (float128_is_signaling_nan(v1, &env->fpu_status) &&
+         (m2 & (1 << (1-neg))))) {
         cc = 1;
     } else if (m2 & (1 << (9-neg))) {
         /* assume normalized number */
diff --git a/target-s390x/gdbstub.c b/target-s390x/gdbstub.c
index 8945f02..3d223de 100644
--- a/target-s390x/gdbstub.c
+++ b/target-s390x/gdbstub.c
@@ -17,8 +17,10 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/gdbstub.h"
 #include "qemu/bitops.h"
 
@@ -92,6 +94,7 @@
     switch (n) {
     case S390_A0_REGNUM ... S390_A15_REGNUM:
         env->aregs[n] = ldl_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
         return 4;
     default:
         return 0;
@@ -111,7 +114,7 @@
     case S390_FPC_REGNUM:
         return gdb_get_reg32(mem_buf, env->fpc);
     case S390_F0_REGNUM ... S390_F15_REGNUM:
-        return gdb_get_reg64(mem_buf, env->fregs[n - S390_F0_REGNUM].ll);
+        return gdb_get_reg64(mem_buf, get_freg(env, n - S390_F0_REGNUM)->ll);
     default:
         return 0;
     }
@@ -124,13 +127,164 @@
         env->fpc = ldl_p(mem_buf);
         return 4;
     case S390_F0_REGNUM ... S390_F15_REGNUM:
-        env->fregs[n - S390_F0_REGNUM].ll = ldtul_p(mem_buf);
+        get_freg(env, n - S390_F0_REGNUM)->ll = ldtul_p(mem_buf);
         return 8;
     default:
         return 0;
     }
 }
 
+/* the values represent the positions in s390-vx.xml */
+#define S390_V0L_REGNUM 0
+#define S390_V15L_REGNUM 15
+#define S390_V16_REGNUM 16
+#define S390_V31_REGNUM 31
+/* total number of registers in s390-vx.xml */
+#define S390_NUM_VREGS 32
+
+static int cpu_read_vreg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    int ret;
+
+    switch (n) {
+    case S390_V0L_REGNUM ... S390_V15L_REGNUM:
+        ret = gdb_get_reg64(mem_buf, env->vregs[n][1].ll);
+        break;
+    case S390_V16_REGNUM ... S390_V31_REGNUM:
+        ret = gdb_get_reg64(mem_buf, env->vregs[n][0].ll);
+        ret += gdb_get_reg64(mem_buf + 8, env->vregs[n][1].ll);
+        break;
+    default:
+        ret = 0;
+    }
+
+    return ret;
+}
+
+static int cpu_write_vreg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_V0L_REGNUM ... S390_V15L_REGNUM:
+        env->vregs[n][1].ll = ldtul_p(mem_buf + 8);
+        return 8;
+    case S390_V16_REGNUM ... S390_V31_REGNUM:
+        env->vregs[n][0].ll = ldtul_p(mem_buf);
+        env->vregs[n][1].ll = ldtul_p(mem_buf + 8);
+        return 16;
+    default:
+        return 0;
+    }
+}
+
+/* the values represent the positions in s390-cr.xml */
+#define S390_C0_REGNUM 0
+#define S390_C15_REGNUM 15
+/* total number of registers in s390-cr.xml */
+#define S390_NUM_C_REGS 16
+
+#ifndef CONFIG_USER_ONLY
+static int cpu_read_c_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_C0_REGNUM ... S390_C15_REGNUM:
+        return gdb_get_regl(mem_buf, env->cregs[n]);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_c_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_C0_REGNUM ... S390_C15_REGNUM:
+        env->cregs[n] = ldtul_p(mem_buf);
+        if (tcg_enabled()) {
+            tlb_flush(ENV_GET_CPU(env), 1);
+        }
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    default:
+        return 0;
+    }
+}
+
+/* the values represent the positions in s390-virt.xml */
+#define S390_VIRT_CKC_REGNUM    0
+#define S390_VIRT_CPUTM_REGNUM  1
+#define S390_VIRT_BEA_REGNUM    2
+#define S390_VIRT_PREFIX_REGNUM 3
+#define S390_VIRT_PP_REGNUM     4
+#define S390_VIRT_PFT_REGNUM    5
+#define S390_VIRT_PFS_REGNUM    6
+#define S390_VIRT_PFC_REGNUM    7
+/* total number of registers in s390-virt.xml */
+#define S390_NUM_VIRT_REGS 8
+
+static int cpu_read_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_VIRT_CKC_REGNUM:
+        return gdb_get_regl(mem_buf, env->ckc);
+    case S390_VIRT_CPUTM_REGNUM:
+        return gdb_get_regl(mem_buf, env->cputm);
+    case S390_VIRT_BEA_REGNUM:
+        return gdb_get_regl(mem_buf, env->gbea);
+    case S390_VIRT_PREFIX_REGNUM:
+        return gdb_get_regl(mem_buf, env->psa);
+    case S390_VIRT_PP_REGNUM:
+        return gdb_get_regl(mem_buf, env->pp);
+    case S390_VIRT_PFT_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_token);
+    case S390_VIRT_PFS_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_select);
+    case S390_VIRT_PFC_REGNUM:
+        return gdb_get_regl(mem_buf, env->pfault_compare);
+    default:
+        return 0;
+    }
+}
+
+static int cpu_write_virt_reg(CPUS390XState *env, uint8_t *mem_buf, int n)
+{
+    switch (n) {
+    case S390_VIRT_CKC_REGNUM:
+        env->ckc = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_CPUTM_REGNUM:
+        env->cputm = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_BEA_REGNUM:
+        env->gbea = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PREFIX_REGNUM:
+        env->psa = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PP_REGNUM:
+        env->pp = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFT_REGNUM:
+        env->pfault_token = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFS_REGNUM:
+        env->pfault_select = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    case S390_VIRT_PFC_REGNUM:
+        env->pfault_compare = ldtul_p(mem_buf);
+        cpu_synchronize_post_init(ENV_GET_CPU(env));
+        return 8;
+    default:
+        return 0;
+    }
+}
+#endif
+
 void s390_cpu_gdb_init(CPUState *cs)
 {
     gdb_register_coprocessor(cs, cpu_read_ac_reg,
@@ -140,4 +294,20 @@
     gdb_register_coprocessor(cs, cpu_read_fp_reg,
                              cpu_write_fp_reg,
                              S390_NUM_FP_REGS, "s390-fpr.xml", 0);
+
+    gdb_register_coprocessor(cs, cpu_read_vreg,
+                             cpu_write_vreg,
+                             S390_NUM_VREGS, "s390-vx.xml", 0);
+
+#ifndef CONFIG_USER_ONLY
+    gdb_register_coprocessor(cs, cpu_read_c_reg,
+                             cpu_write_c_reg,
+                             S390_NUM_C_REGS, "s390-cr.xml", 0);
+
+    if (kvm_enabled()) {
+        gdb_register_coprocessor(cs, cpu_read_virt_reg,
+                                 cpu_write_virt_reg,
+                                 S390_NUM_VIRT_REGS, "s390-virt.xml", 0);
+    }
+#endif
 }
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index 09aec7b..54a5177 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -18,23 +18,26 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "exec/gdbstub.h"
 #include "qemu/timer.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "hw/s390x/ioinst.h"
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/sysemu.h"
 #endif
 
 //#define DEBUG_S390
-//#define DEBUG_S390_PTE
 //#define DEBUG_S390_STDOUT
 
 #ifdef DEBUG_S390
 #ifdef DEBUG_S390_STDOUT
 #define DPRINTF(fmt, ...) \
     do { fprintf(stderr, fmt, ## __VA_ARGS__); \
-         qemu_log(fmt, ##__VA_ARGS__); } while (0)
+         if (qemu_log_separate()) qemu_log(fmt, ##__VA_ARGS__); } while (0)
 #else
 #define DPRINTF(fmt, ...) \
     do { qemu_log(fmt, ## __VA_ARGS__); } while (0)
@@ -44,12 +47,6 @@
     do { } while (0)
 #endif
 
-#ifdef DEBUG_S390_PTE
-#define PTE_DPRINTF DPRINTF
-#else
-#define PTE_DPRINTF(fmt, ...) \
-    do { } while (0)
-#endif
 
 #ifndef CONFIG_USER_ONLY
 void s390x_tod_timer(void *opaque)
@@ -71,14 +68,47 @@
 }
 #endif
 
-S390CPU *cpu_s390x_init(const char *cpu_model)
+S390CPU *cpu_s390x_create(const char *cpu_model, Error **errp)
+{
+    return S390_CPU(object_new(TYPE_S390_CPU));
+}
+
+S390CPU *s390x_new_cpu(const char *cpu_model, int64_t id, Error **errp)
 {
     S390CPU *cpu;
+    Error *err = NULL;
 
-    cpu = S390_CPU(object_new(TYPE_S390_CPU));
+    cpu = cpu_s390x_create(cpu_model, &err);
+    if (err != NULL) {
+        goto out;
+    }
 
-    object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
+    object_property_set_int(OBJECT(cpu), id, "id", &err);
+    if (err != NULL) {
+        goto out;
+    }
+    object_property_set_bool(OBJECT(cpu), true, "realized", &err);
 
+out:
+    if (err) {
+        error_propagate(errp, err);
+        object_unref(OBJECT(cpu));
+        cpu = NULL;
+    }
+    return cpu;
+}
+
+S390CPU *cpu_s390x_init(const char *cpu_model)
+{
+    Error *err = NULL;
+    S390CPU *cpu;
+    /* Use to track CPU ID for linux-user only */
+    static int64_t next_cpu_id;
+
+    cpu = s390x_new_cpu(cpu_model, next_cpu_id++, &err);
+    if (err) {
+        error_report_err(err);
+    }
     return cpu;
 }
 
@@ -105,8 +135,7 @@
 #else /* !CONFIG_USER_ONLY */
 
 /* Ensure to exit the TB after this call! */
-static void trigger_pgm_exception(CPUS390XState *env, uint32_t code,
-                                  uint32_t ilen)
+void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilen)
 {
     CPUState *cs = CPU(s390_env_get_cpu(env));
 
@@ -115,325 +144,12 @@
     env->int_pgm_ilen = ilen;
 }
 
-static int trans_bits(CPUS390XState *env, uint64_t mode)
-{
-    S390CPU *cpu = s390_env_get_cpu(env);
-    int bits = 0;
-
-    switch (mode) {
-    case PSW_ASC_PRIMARY:
-        bits = 1;
-        break;
-    case PSW_ASC_SECONDARY:
-        bits = 2;
-        break;
-    case PSW_ASC_HOME:
-        bits = 3;
-        break;
-    default:
-        cpu_abort(CPU(cpu), "unknown asc mode\n");
-        break;
-    }
-
-    return bits;
-}
-
-static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr,
-                               uint64_t mode)
-{
-    CPUState *cs = CPU(s390_env_get_cpu(env));
-    int ilen = ILEN_LATER_INC;
-    int bits = trans_bits(env, mode) | 4;
-
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
-
-    stq_phys(cs->as,
-             env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
-    trigger_pgm_exception(env, PGM_PROTECTION, ilen);
-}
-
-static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr,
-                               uint32_t type, uint64_t asc, int rw)
-{
-    CPUState *cs = CPU(s390_env_get_cpu(env));
-    int ilen = ILEN_LATER;
-    int bits = trans_bits(env, asc);
-
-    /* Code accesses have an undefined ilc.  */
-    if (rw == 2) {
-        ilen = 2;
-    }
-
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
-
-    stq_phys(cs->as,
-             env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
-    trigger_pgm_exception(env, type, ilen);
-}
-
-/**
- * Translate real address to absolute (= physical)
- * address by taking care of the prefix mapping.
- */
-static target_ulong mmu_real2abs(CPUS390XState *env, target_ulong raddr)
-{
-    if (raddr < 0x2000) {
-        return raddr + env->psa;    /* Map the lowcore. */
-    } else if (raddr >= env->psa && raddr < env->psa + 0x2000) {
-        return raddr - env->psa;    /* Map the 0 page. */
-    }
-    return raddr;
-}
-
-/* Decode page table entry (normal 4KB page) */
-static int mmu_translate_pte(CPUS390XState *env, target_ulong vaddr,
-                             uint64_t asc, uint64_t asce,
-                             target_ulong *raddr, int *flags, int rw)
-{
-    if (asce & _PAGE_INVALID) {
-        DPRINTF("%s: PTE=0x%" PRIx64 " invalid\n", __func__, asce);
-        trigger_page_fault(env, vaddr, PGM_PAGE_TRANS, asc, rw);
-        return -1;
-    }
-
-    if (asce & _PAGE_RO) {
-        *flags &= ~PAGE_WRITE;
-    }
-
-    *raddr = asce & _ASCE_ORIGIN;
-
-    PTE_DPRINTF("%s: PTE=0x%" PRIx64 "\n", __func__, asce);
-
-    return 0;
-}
-
-/* Decode EDAT1 segment frame absolute address (1MB page) */
-static int mmu_translate_sfaa(CPUS390XState *env, target_ulong vaddr,
-                              uint64_t asc, uint64_t asce, target_ulong *raddr,
-                              int *flags, int rw)
-{
-    if (asce & _SEGMENT_ENTRY_INV) {
-        DPRINTF("%s: SEG=0x%" PRIx64 " invalid\n", __func__, asce);
-        trigger_page_fault(env, vaddr, PGM_SEGMENT_TRANS, asc, rw);
-        return -1;
-    }
-
-    if (asce & _SEGMENT_ENTRY_RO) {
-        *flags &= ~PAGE_WRITE;
-    }
-
-    *raddr = (asce & 0xfffffffffff00000ULL) | (vaddr & 0xfffff);
-
-    PTE_DPRINTF("%s: SEG=0x%" PRIx64 "\n", __func__, asce);
-
-    return 0;
-}
-
-static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
-                              uint64_t asc, uint64_t asce, int level,
-                              target_ulong *raddr, int *flags, int rw)
-{
-    CPUState *cs = CPU(s390_env_get_cpu(env));
-    uint64_t offs = 0;
-    uint64_t origin;
-    uint64_t new_asce;
-
-    PTE_DPRINTF("%s: 0x%" PRIx64 "\n", __func__, asce);
-
-    if (((level != _ASCE_TYPE_SEGMENT) && (asce & _REGION_ENTRY_INV)) ||
-        ((level == _ASCE_TYPE_SEGMENT) && (asce & _SEGMENT_ENTRY_INV))) {
-        /* XXX different regions have different faults */
-        DPRINTF("%s: invalid region\n", __func__);
-        trigger_page_fault(env, vaddr, PGM_SEGMENT_TRANS, asc, rw);
-        return -1;
-    }
-
-    if ((level <= _ASCE_TYPE_MASK) && ((asce & _ASCE_TYPE_MASK) != level)) {
-        trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
-        return -1;
-    }
-
-    if (asce & _ASCE_REAL_SPACE) {
-        /* direct mapping */
-
-        *raddr = vaddr;
-        return 0;
-    }
-
-    origin = asce & _ASCE_ORIGIN;
-
-    switch (level) {
-    case _ASCE_TYPE_REGION1 + 4:
-        offs = (vaddr >> 50) & 0x3ff8;
-        break;
-    case _ASCE_TYPE_REGION1:
-        offs = (vaddr >> 39) & 0x3ff8;
-        break;
-    case _ASCE_TYPE_REGION2:
-        offs = (vaddr >> 28) & 0x3ff8;
-        break;
-    case _ASCE_TYPE_REGION3:
-        offs = (vaddr >> 17) & 0x3ff8;
-        break;
-    case _ASCE_TYPE_SEGMENT:
-        offs = (vaddr >> 9) & 0x07f8;
-        origin = asce & _SEGMENT_ENTRY_ORIGIN;
-        break;
-    }
-
-    /* XXX region protection flags */
-    /* *flags &= ~PAGE_WRITE */
-
-    new_asce = ldq_phys(cs->as, origin + offs);
-    PTE_DPRINTF("%s: 0x%" PRIx64 " + 0x%" PRIx64 " => 0x%016" PRIx64 "\n",
-                __func__, origin, offs, new_asce);
-
-    if (level == _ASCE_TYPE_SEGMENT) {
-        /* 4KB page */
-        return mmu_translate_pte(env, vaddr, asc, new_asce, raddr, flags, rw);
-    } else if (level - 4 == _ASCE_TYPE_SEGMENT &&
-               (new_asce & _SEGMENT_ENTRY_FC) && (env->cregs[0] & CR0_EDAT)) {
-        /* 1MB page */
-        return mmu_translate_sfaa(env, vaddr, asc, new_asce, raddr, flags, rw);
-    } else {
-        /* yet another region */
-        return mmu_translate_asce(env, vaddr, asc, new_asce, level - 4, raddr,
-                                  flags, rw);
-    }
-}
-
-static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr,
-                             uint64_t asc, target_ulong *raddr, int *flags,
-                             int rw)
-{
-    uint64_t asce = 0;
-    int level, new_level;
-    int r;
-
-    switch (asc) {
-    case PSW_ASC_PRIMARY:
-        PTE_DPRINTF("%s: asc=primary\n", __func__);
-        asce = env->cregs[1];
-        break;
-    case PSW_ASC_SECONDARY:
-        PTE_DPRINTF("%s: asc=secondary\n", __func__);
-        asce = env->cregs[7];
-        break;
-    case PSW_ASC_HOME:
-        PTE_DPRINTF("%s: asc=home\n", __func__);
-        asce = env->cregs[13];
-        break;
-    }
-
-    switch (asce & _ASCE_TYPE_MASK) {
-    case _ASCE_TYPE_REGION1:
-        break;
-    case _ASCE_TYPE_REGION2:
-        if (vaddr & 0xffe0000000000000ULL) {
-            DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                    " 0xffe0000000000000ULL\n", __func__, vaddr);
-            trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
-            return -1;
-        }
-        break;
-    case _ASCE_TYPE_REGION3:
-        if (vaddr & 0xfffffc0000000000ULL) {
-            DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                    " 0xfffffc0000000000ULL\n", __func__, vaddr);
-            trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
-            return -1;
-        }
-        break;
-    case _ASCE_TYPE_SEGMENT:
-        if (vaddr & 0xffffffff80000000ULL) {
-            DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                    " 0xffffffff80000000ULL\n", __func__, vaddr);
-            trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
-            return -1;
-        }
-        break;
-    }
-
-    /* fake level above current */
-    level = asce & _ASCE_TYPE_MASK;
-    new_level = level + 4;
-    asce = (asce & ~_ASCE_TYPE_MASK) | (new_level & _ASCE_TYPE_MASK);
-
-    r = mmu_translate_asce(env, vaddr, asc, asce, new_level, raddr, flags, rw);
-
-    if ((rw == 1) && !(*flags & PAGE_WRITE)) {
-        trigger_prot_fault(env, vaddr, asc);
-        return -1;
-    }
-
-    return r;
-}
-
-int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
-                  target_ulong *raddr, int *flags)
-{
-    int r = -1;
-    uint8_t *sk;
-
-    *flags = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
-    vaddr &= TARGET_PAGE_MASK;
-
-    if (!(env->psw.mask & PSW_MASK_DAT)) {
-        *raddr = vaddr;
-        r = 0;
-        goto out;
-    }
-
-    switch (asc) {
-    case PSW_ASC_PRIMARY:
-    case PSW_ASC_HOME:
-        r = mmu_translate_asc(env, vaddr, asc, raddr, flags, rw);
-        break;
-    case PSW_ASC_SECONDARY:
-        /*
-         * Instruction: Primary
-         * Data: Secondary
-         */
-        if (rw == 2) {
-            r = mmu_translate_asc(env, vaddr, PSW_ASC_PRIMARY, raddr, flags,
-                                  rw);
-            *flags &= ~(PAGE_READ | PAGE_WRITE);
-        } else {
-            r = mmu_translate_asc(env, vaddr, PSW_ASC_SECONDARY, raddr, flags,
-                                  rw);
-            *flags &= ~(PAGE_EXEC);
-        }
-        break;
-    case PSW_ASC_ACCREG:
-    default:
-        hw_error("guest switched to unknown asc mode\n");
-        break;
-    }
-
- out:
-    /* Convert real address -> absolute address */
-    *raddr = mmu_real2abs(env, *raddr);
-
-    if (*raddr <= ram_size) {
-        sk = &env->storage_keys[*raddr / TARGET_PAGE_SIZE];
-        if (*flags & PAGE_READ) {
-            *sk |= SK_R;
-        }
-
-        if (*flags & PAGE_WRITE) {
-            *sk |= SK_C;
-        }
-    }
-
-    return r;
-}
-
 int s390_cpu_handle_mmu_fault(CPUState *cs, vaddr orig_vaddr,
                               int rw, int mmu_idx)
 {
     S390CPU *cpu = S390_CPU(cs);
     CPUS390XState *env = &cpu->env;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    uint64_t asc = cpu_mmu_idx_to_asc(mmu_idx);
     target_ulong vaddr, raddr;
     int prot;
 
@@ -448,21 +164,21 @@
         vaddr &= 0x7fffffff;
     }
 
-    if (mmu_translate(env, vaddr, rw, asc, &raddr, &prot)) {
+    if (mmu_translate(env, vaddr, rw, asc, &raddr, &prot, true)) {
         /* Translation ended in exception */
         return 1;
     }
 
     /* check out of RAM access */
-    if (raddr > (ram_size + virtio_size)) {
+    if (raddr > ram_size) {
         DPRINTF("%s: raddr %" PRIx64 " > ram_size %" PRIx64 "\n", __func__,
                 (uint64_t)raddr, (uint64_t)ram_size);
         trigger_pgm_exception(env, PGM_ADDRESSING, ILEN_LATER);
         return 1;
     }
 
-    DPRINTF("%s: set tlb %" PRIx64 " -> %" PRIx64 " (%x)\n", __func__,
-            (uint64_t)vaddr, (uint64_t)raddr, prot);
+    qemu_log_mask(CPU_LOG_MMU, "%s: set tlb %" PRIx64 " -> %" PRIx64 " (%x)\n",
+            __func__, (uint64_t)vaddr, (uint64_t)raddr, prot);
 
     tlb_set_page(cs, orig_vaddr, raddr, prot,
                  mmu_idx, TARGET_PAGE_SIZE);
@@ -475,8 +191,7 @@
     S390CPU *cpu = S390_CPU(cs);
     CPUS390XState *env = &cpu->env;
     target_ulong raddr;
-    int prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
-    int old_exc = cs->exception_index;
+    int prot;
     uint64_t asc = env->psw.mask & PSW_MASK_ASC;
 
     /* 31-Bit mode */
@@ -484,9 +199,9 @@
         vaddr &= 0x7fffffff;
     }
 
-    mmu_translate(env, vaddr, 2, asc, &raddr, &prot);
-    cs->exception_index = old_exc;
-
+    if (mmu_translate(env, vaddr, MMU_INST_FETCH, asc, &raddr, &prot, false)) {
+        return -1;
+    }
     return raddr;
 }
 
@@ -504,9 +219,17 @@
 
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr)
 {
+    uint64_t old_mask = env->psw.mask;
+
     env->psw.addr = addr;
     env->psw.mask = mask;
-    env->cc_op = (mask >> 44) & 3;
+    if (tcg_enabled()) {
+        env->cc_op = (mask >> 44) & 3;
+    }
+
+    if ((old_mask ^ mask) & PSW_MASK_PER) {
+        s390_cpu_recompute_watchpoints(CPU(s390_env_get_cpu(env)));
+    }
 
     if (mask & PSW_MASK_WAIT) {
         S390CPU *cpu = s390_env_get_cpu(env);
@@ -520,14 +243,16 @@
 
 static uint64_t get_psw_mask(CPUS390XState *env)
 {
-    uint64_t r;
+    uint64_t r = env->psw.mask;
 
-    env->cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst, env->cc_vr);
+    if (tcg_enabled()) {
+        env->cc_op = calc_cc(env, env->cc_op, env->cc_src, env->cc_dst,
+                             env->cc_vr);
 
-    r = env->psw.mask;
-    r &= ~PSW_MASK_CC;
-    assert(!(env->cc_op & ~3));
-    r |= (uint64_t)env->cc_op << 44;
+        r &= ~PSW_MASK_CC;
+        assert(!(env->cc_op & ~3));
+        r |= (uint64_t)env->cc_op << 44;
+    }
 
     return r;
 }
@@ -552,44 +277,17 @@
     cpu_physical_memory_unmap(lowcore, sizeof(LowCore), 1, sizeof(LowCore));
 }
 
-void *s390_cpu_physical_memory_map(CPUS390XState *env, hwaddr addr, hwaddr *len,
-                                   int is_write)
-{
-    hwaddr start = addr;
-
-    /* Mind the prefix area. */
-    if (addr < 8192) {
-        /* Map the lowcore. */
-        start += env->psa;
-        *len = MIN(*len, 8192 - addr);
-    } else if ((addr >= env->psa) && (addr < env->psa + 8192)) {
-        /* Map the 0 page. */
-        start -= env->psa;
-        *len = MIN(*len, 8192 - start);
-    }
-
-    return cpu_physical_memory_map(start, len, is_write);
-}
-
-void s390_cpu_physical_memory_unmap(CPUS390XState *env, void *addr, hwaddr len,
-                                    int is_write)
-{
-    cpu_physical_memory_unmap(addr, len, is_write, len);
-}
-
-static void do_svc_interrupt(CPUS390XState *env)
+void do_restart_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
 
     lowcore = cpu_map_lowcore(env);
 
-    lowcore->svc_code = cpu_to_be16(env->int_svc_code);
-    lowcore->svc_ilen = cpu_to_be16(env->int_svc_ilen);
-    lowcore->svc_old_psw.mask = cpu_to_be64(get_psw_mask(env));
-    lowcore->svc_old_psw.addr = cpu_to_be64(env->psw.addr + env->int_svc_ilen);
-    mask = be64_to_cpu(lowcore->svc_new_psw.mask);
-    addr = be64_to_cpu(lowcore->svc_new_psw.addr);
+    lowcore->restart_old_psw.mask = cpu_to_be64(get_psw_mask(env));
+    lowcore->restart_old_psw.addr = cpu_to_be64(env->psw.addr);
+    mask = be64_to_cpu(lowcore->restart_new_psw.mask);
+    addr = be64_to_cpu(lowcore->restart_new_psw.addr);
 
     cpu_unmap_lowcore(lowcore);
 
@@ -619,12 +317,21 @@
 
     lowcore = cpu_map_lowcore(env);
 
+    /* Signal PER events with the exception.  */
+    if (env->per_perc_atmid) {
+        env->int_pgm_code |= PGM_PER;
+        lowcore->per_address = cpu_to_be64(env->per_address);
+        lowcore->per_perc_atmid = cpu_to_be16(env->per_perc_atmid);
+        env->per_perc_atmid = 0;
+    }
+
     lowcore->pgm_ilen = cpu_to_be16(ilen);
     lowcore->pgm_code = cpu_to_be16(env->int_pgm_code);
     lowcore->program_old_psw.mask = cpu_to_be64(get_psw_mask(env));
     lowcore->program_old_psw.addr = cpu_to_be64(env->psw.addr);
     mask = be64_to_cpu(lowcore->program_new_psw.mask);
     addr = be64_to_cpu(lowcore->program_new_psw.addr);
+    lowcore->per_breaking_event_addr = cpu_to_be64(env->gbea);
 
     cpu_unmap_lowcore(lowcore);
 
@@ -635,6 +342,33 @@
     load_psw(env, mask, addr);
 }
 
+static void do_svc_interrupt(CPUS390XState *env)
+{
+    uint64_t mask, addr;
+    LowCore *lowcore;
+
+    lowcore = cpu_map_lowcore(env);
+
+    lowcore->svc_code = cpu_to_be16(env->int_svc_code);
+    lowcore->svc_ilen = cpu_to_be16(env->int_svc_ilen);
+    lowcore->svc_old_psw.mask = cpu_to_be64(get_psw_mask(env));
+    lowcore->svc_old_psw.addr = cpu_to_be64(env->psw.addr + env->int_svc_ilen);
+    mask = be64_to_cpu(lowcore->svc_new_psw.mask);
+    addr = be64_to_cpu(lowcore->svc_new_psw.addr);
+
+    cpu_unmap_lowcore(lowcore);
+
+    load_psw(env, mask, addr);
+
+    /* When a PER event is pending, the PER exception has to happen
+       immediately after the SERVICE CALL one.  */
+    if (env->per_perc_atmid) {
+        env->int_pgm_code = PGM_PER;
+        env->int_pgm_ilen = env->int_svc_ilen;
+        do_program_interrupt(env);
+    }
+}
+
 #define VIRTIO_SUBCODE_64 0x0D00
 
 static void do_ext_interrupt(CPUS390XState *env)
@@ -648,7 +382,7 @@
         cpu_abort(CPU(cpu), "Ext int w/o ext mask\n");
     }
 
-    if (env->ext_index < 0 || env->ext_index > MAX_EXT_QUEUE) {
+    if (env->ext_index < 0 || env->ext_index >= MAX_EXT_QUEUE) {
         cpu_abort(CPU(cpu), "Ext queue overrun: %d\n", env->ext_index);
     }
 
@@ -696,7 +430,7 @@
         if (env->io_index[isc] < 0) {
             continue;
         }
-        if (env->io_index[isc] > MAX_IO_QUEUE) {
+        if (env->io_index[isc] >= MAX_IO_QUEUE) {
             cpu_abort(CPU(cpu), "I/O queue overrun for isc %d: %d\n",
                       isc, env->io_index[isc]);
         }
@@ -754,7 +488,7 @@
         cpu_abort(CPU(cpu), "Machine check w/o mchk mask\n");
     }
 
-    if (env->mchk_index < 0 || env->mchk_index > MAX_MCHK_QUEUE) {
+    if (env->mchk_index < 0 || env->mchk_index >= MAX_MCHK_QUEUE) {
         cpu_abort(CPU(cpu), "Mchk queue overrun: %d\n", env->mchk_index);
     }
 
@@ -772,7 +506,7 @@
     lowcore = cpu_map_lowcore(env);
 
     for (i = 0; i < 16; i++) {
-        lowcore->floating_pt_save_area[i] = cpu_to_be64(env->fregs[i].ll);
+        lowcore->floating_pt_save_area[i] = cpu_to_be64(get_freg(env, i)->ll);
         lowcore->gpregs_save_area[i] = cpu_to_be64(env->regs[i]);
         lowcore->access_regs_save_area[i] = cpu_to_be32(env->aregs[i]);
         lowcore->cregs_save_area[i] = cpu_to_be64(env->cregs[i]);
@@ -884,4 +618,73 @@
     }
     return false;
 }
+
+void s390_cpu_recompute_watchpoints(CPUState *cs)
+{
+    const int wp_flags = BP_CPU | BP_MEM_WRITE | BP_STOP_BEFORE_ACCESS;
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
+
+    /* We are called when the watchpoints have changed. First
+       remove them all.  */
+    cpu_watchpoint_remove_all(cs, BP_CPU);
+
+    /* Return if PER is not enabled */
+    if (!(env->psw.mask & PSW_MASK_PER)) {
+        return;
+    }
+
+    /* Return if storage-alteration event is not enabled.  */
+    if (!(env->cregs[9] & PER_CR9_EVENT_STORE)) {
+        return;
+    }
+
+    if (env->cregs[10] == 0 && env->cregs[11] == -1LL) {
+        /* We can't create a watchoint spanning the whole memory range, so
+           split it in two parts.   */
+        cpu_watchpoint_insert(cs, 0, 1ULL << 63, wp_flags, NULL);
+        cpu_watchpoint_insert(cs, 1ULL << 63, 1ULL << 63, wp_flags, NULL);
+    } else if (env->cregs[10] > env->cregs[11]) {
+        /* The address range loops, create two watchpoints.  */
+        cpu_watchpoint_insert(cs, env->cregs[10], -env->cregs[10],
+                              wp_flags, NULL);
+        cpu_watchpoint_insert(cs, 0, env->cregs[11] + 1, wp_flags, NULL);
+
+    } else {
+        /* Default case, create a single watchpoint.  */
+        cpu_watchpoint_insert(cs, env->cregs[10],
+                              env->cregs[11] - env->cregs[10] + 1,
+                              wp_flags, NULL);
+    }
+}
+
+void s390x_cpu_debug_excp_handler(CPUState *cs)
+{
+    S390CPU *cpu = S390_CPU(cs);
+    CPUS390XState *env = &cpu->env;
+    CPUWatchpoint *wp_hit = cs->watchpoint_hit;
+
+    if (wp_hit && wp_hit->flags & BP_CPU) {
+        /* FIXME: When the storage-alteration-space control bit is set,
+           the exception should only be triggered if the memory access
+           is done using an address space with the storage-alteration-event
+           bit set.  We have no way to detect that with the current
+           watchpoint code.  */
+        cs->watchpoint_hit = NULL;
+
+        env->per_address = env->psw.addr;
+        env->per_perc_atmid |= PER_CODE_EVENT_STORE | get_per_atmid(env);
+        /* FIXME: We currently no way to detect the address space used
+           to trigger the watchpoint.  For now just consider it is the
+           current default ASC. This turn to be true except when MVCP
+           and MVCS instrutions are not used.  */
+        env->per_perc_atmid |= env->psw.mask & (PSW_MASK_ASC) >> 46;
+
+        /* Remove all watchpoints to re-execute the code.  A PER exception
+           will be triggered, it will call load_psw which will recompute
+           the watchpoints.  */
+        cpu_watchpoint_remove_all(cs, BP_CPU);
+        cpu_loop_exit_noexc(cs);
+    }
+}
 #endif /* CONFIG_USER_ONLY */
diff --git a/target-s390x/helper.h b/target-s390x/helper.h
index faebfd9..207a6e7 100644
--- a/target-s390x/helper.h
+++ b/target-s390x/helper.h
@@ -15,10 +15,6 @@
 DEF_HELPER_4(mvpg, void, env, i64, i64, i64)
 DEF_HELPER_4(mvst, i64, env, i64, i64, i64)
 DEF_HELPER_5(ex, i32, env, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_NO_RWG_SE, i32, s32)
-DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_NO_RWG_SE, s32, s32)
-DEF_HELPER_FLAGS_1(abs_i64, TCG_CALL_NO_RWG_SE, i64, s64)
-DEF_HELPER_FLAGS_1(nabs_i64, TCG_CALL_NO_RWG_SE, s64, s64)
 DEF_HELPER_FLAGS_4(stam, TCG_CALL_NO_WG, void, env, i32, i64, i32)
 DEF_HELPER_FLAGS_4(lam, TCG_CALL_NO_WG, void, env, i32, i64, i32)
 DEF_HELPER_4(mvcle, i32, env, i32, i64, i32)
@@ -64,13 +60,16 @@
 DEF_HELPER_FLAGS_3(clfeb, TCG_CALL_NO_WG, i64, env, i64, i32)
 DEF_HELPER_FLAGS_3(clfdb, TCG_CALL_NO_WG, i64, env, i64, i32)
 DEF_HELPER_FLAGS_4(clfxb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
+DEF_HELPER_FLAGS_3(fieb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_3(fidb, TCG_CALL_NO_WG, i64, env, i64, i32)
+DEF_HELPER_FLAGS_4(fixb, TCG_CALL_NO_WG, i64, env, i64, i64, i32)
 DEF_HELPER_FLAGS_4(maeb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(madb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(mseb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_4(msdb, TCG_CALL_NO_WG, i64, env, i64, i64, i64)
-DEF_HELPER_FLAGS_2(tceb, TCG_CALL_NO_RWG_SE, i32, i64, i64)
-DEF_HELPER_FLAGS_2(tcdb, TCG_CALL_NO_RWG_SE, i32, i64, i64)
-DEF_HELPER_FLAGS_3(tcxb, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_3(tceb, TCG_CALL_NO_RWG_SE, i32, env, i64, i64)
+DEF_HELPER_FLAGS_3(tcdb, TCG_CALL_NO_RWG_SE, i32, env, i64, i64)
+DEF_HELPER_FLAGS_4(tcxb, TCG_CALL_NO_RWG_SE, i32, env, i64, i64, i64)
 DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i64, i64)
 DEF_HELPER_FLAGS_2(sqeb, TCG_CALL_NO_WG, i64, env, i64)
 DEF_HELPER_FLAGS_2(sqdb, TCG_CALL_NO_WG, i64, env, i64)
@@ -78,6 +77,8 @@
 DEF_HELPER_FLAGS_1(cvd, TCG_CALL_NO_RWG_SE, i64, s32)
 DEF_HELPER_FLAGS_4(unpk, TCG_CALL_NO_WG, void, env, i32, i64, i64)
 DEF_HELPER_FLAGS_4(tr, TCG_CALL_NO_WG, void, env, i32, i64, i64)
+DEF_HELPER_4(tre, i64, env, i64, i64, i64)
+DEF_HELPER_4(trt, i32, env, i32, i64, i64)
 DEF_HELPER_4(cksm, i64, env, i64, i64, i64)
 DEF_HELPER_FLAGS_5(calc_cc, TCG_CALL_NO_RWG_SE, i32, env, i32, i64, i64, i64)
 DEF_HELPER_FLAGS_2(sfpc, TCG_CALL_NO_RWG, void, env, i64)
@@ -86,7 +87,7 @@
 
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_3(servc, i32, env, i64, i64)
-DEF_HELPER_4(diag, i64, env, i32, i64, i64)
+DEF_HELPER_4(diag, void, env, i32, i32, i32)
 DEF_HELPER_3(load_psw, noreturn, env, i64, i64)
 DEF_HELPER_FLAGS_2(spx, TCG_CALL_NO_RWG, void, env, i64)
 DEF_HELPER_FLAGS_1(stck, TCG_CALL_NO_RWG_SE, i64, env)
@@ -111,5 +112,22 @@
 DEF_HELPER_FLAGS_3(ipte, TCG_CALL_NO_RWG, void, env, i64, i64)
 DEF_HELPER_FLAGS_1(ptlb, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_2(lra, i64, env, i64)
+DEF_HELPER_FLAGS_2(lura, TCG_CALL_NO_WG, i64, env, i64)
+DEF_HELPER_FLAGS_2(lurag, TCG_CALL_NO_WG, i64, env, i64)
 DEF_HELPER_FLAGS_3(stura, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_FLAGS_3(sturg, TCG_CALL_NO_WG, void, env, i64, i64)
+DEF_HELPER_1(per_check_exception, void, env)
+DEF_HELPER_FLAGS_3(per_branch, TCG_CALL_NO_RWG, void, env, i64, i64)
+DEF_HELPER_FLAGS_2(per_ifetch, TCG_CALL_NO_RWG, void, env, i64)
+
+DEF_HELPER_2(xsch, void, env, i64)
+DEF_HELPER_2(csch, void, env, i64)
+DEF_HELPER_2(hsch, void, env, i64)
+DEF_HELPER_3(msch, void, env, i64, i64)
+DEF_HELPER_2(rchp, void, env, i64)
+DEF_HELPER_2(rsch, void, env, i64)
+DEF_HELPER_3(ssch, void, env, i64, i64)
+DEF_HELPER_3(stsch, void, env, i64, i64)
+DEF_HELPER_3(tsch, void, env, i64, i64)
+DEF_HELPER_2(chsc, void, env, i64)
 #endif
diff --git a/target-s390x/insn-data.def b/target-s390x/insn-data.def
index 4d2feb6..075ff59 100644
--- a/target-s390x/insn-data.def
+++ b/target-s390x/insn-data.def
@@ -1,3 +1,24 @@
+/*
+ *  Arguments to the opcode prototypes
+ *
+ *  C(OPC,    NAME,    FMT,   FAC, I1, I2, P, W, OP, CC)
+ *  D(OPC,    NAME,    FMT,   FAC, I1, I2, P, W, OP, CC, DATA)
+ *
+ *  OPC  = (op << 8) | op2 where op is the major, op2 the minor opcode
+ *  NAME = name of the opcode, used internally
+ *  FMT  = format of the opcode (defined in insn-format.def)
+ *  FAC  = facility the opcode is available in (defined in DisasFacility)
+ *  I1   = func in1_xx fills o->in1
+ *  I2   = func in2_xx fills o->in2
+ *  P    = func prep_xx initializes o->*out*
+ *  W    = func wout_xx writes o->*out* somewhere
+ *  OP   = func op_xx does the bulk of the operation
+ *  CC   = func cout_xx defines how cc should get set
+ *  DATA = immediate argument to op_xx function
+ *
+ *  The helpers get called in order: I1, I2, P, OP, W, CC
+ */
+
 /* ADD */
     C(0x1a00, AR,      RR_a,  Z,   r1, r2, new, r1_32, add, adds32)
     C(0xb9f8, ARK,     RRF_a, DO,  r2, r3, new, r1_32, add, adds32)
@@ -13,6 +34,9 @@
     C(0xb34a, AXBR,    RRE,   Z,   0, x2_o, x1, 0, axb, f128)
     C(0xed0a, AEB,     RXE,   Z,   e1, m2_32u, new, e1, aeb, f32)
     C(0xed1a, ADB,     RXE,   Z,   f1_o, m2_64, f1, 0, adb, f64)
+/* ADD HIGH */
+    C(0xb9c8, AHHHR,   RRF_a, HW,  r2_sr32, r3_sr32, new, r1_32h, add, adds32)
+    C(0xb9d8, AHHLR,   RRF_a, HW,  r2_sr32, r3, new, r1_32h, add, adds32)
 /* ADD IMMEDIATE */
     C(0xc209, AFI,     RIL_a, EI,  r1, i2, new, r1_32, add, adds32)
     C(0xeb6a, ASI,     SIY,   GIE, m1_32s, i2, new, m1_32, add, adds32)
@@ -20,6 +44,8 @@
     C(0xc208, AGFI,    RIL_a, EI,  r1, i2, r1, 0, add, adds64)
     C(0xeb7a, AGSI,    SIY,   GIE, m1_64, i2, new, m1_64, add, adds64)
     C(0xecd9, AGHIK,   RIE_d, DO,  r3, i2, r1, 0, add, adds64)
+/* ADD IMMEDIATE HIGH */
+    C(0xcc08, AIH,     RIL_a, HW,  r1_sr32, i2, new, r1_32h, add, adds32)
 /* ADD HALFWORD */
     C(0x4a00, AH,      RX_a,  Z,   r1, m2_16s, new, r1_32, add, adds32)
     C(0xe37a, AHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, add, adds32)
@@ -37,6 +63,9 @@
     C(0xb9ea, ALGRK,   RRF_a, DO,  r2, r3, r1, 0, add, addu64)
     C(0xe30a, ALG,     RXY_a, Z,   r1, m2_64, r1, 0, add, addu64)
     C(0xe31a, ALGF,    RXY_a, Z,   r1, m2_32u, r1, 0, add, addu64)
+/* ADD LOGICAL HIGH */
+    C(0xb9ca, ALHHHR,  RRF_a, HW,  r2_sr32, r3_sr32, new, r1_32h, add, addu32)
+    C(0xb9da, ALHHLR,  RRF_a, HW,  r2_sr32, r3, new, r1_32h, add, addu32)
 /* ADD LOGICAL IMMEDIATE */
     C(0xc20b, ALFI,    RIL_a, EI,  r1, i2_32u, new, r1_32, add, addu32)
     C(0xc20a, ALGFI,   RIL_a, EI,  r1, i2_32u, r1, 0, add, addu64)
@@ -45,6 +74,9 @@
     C(0xecda, ALHSIK,  RIE_d, DO,  r3, i2, new, r1_32, add, addu32)
     C(0xeb7e, ALGSI,   SIY,   GIE, m1_64, i2, new, m1_64, add, addu64)
     C(0xecdb, ALGHSIK, RIE_d, DO,  r3, i2, r1, 0, add, addu64)
+/* ADD LOGICAL WITH SIGNED IMMEDIATE HIGH */
+    C(0xcc0a, ALSIH,   RIL_a, HW,  r1_sr32, i2, new, r1_32h, add, addu32)
+    C(0xcc0b, ALSIHN,  RIL_a, HW,  r1_sr32, i2, new, r1_32h, add, 0)
 /* ADD LOGICAL WITH CARRY */
     C(0xb998, ALCR,    RRE,   Z,   r1, r2, new, r1_32, addc, addc32)
     C(0xb988, ALCGR,   RRE,   Z,   r1, r2, r1, 0, addc, addc64)
@@ -90,6 +122,8 @@
 /* BRANCH RELATIVE ON COUNT */
     C(0xa706, BRCT,    RI_b,  Z,   0, 0, 0, 0, bct32, 0)
     C(0xa707, BRCTG,   RI_b,  Z,   0, 0, 0, 0, bct64, 0)
+/* BRANCH RELATIVE ON COUNT HIGH */
+    C(0xcc06, BRCTH,   RIL_b, HW,  0, 0, 0, 0, bcth, 0)
 /* BRANCH ON INDEX */
     D(0x8600, BXH,     RS_a,  Z,   0, a2, 0, 0, bx32, 0, 0)
     D(0x8700, BXLE,    RS_a,  Z,   0, a2, 0, 0, bx32, 0, 1)
@@ -138,8 +172,14 @@
     C(0xe55c, CHSI,    SIL,   GIE, m1_32s, i2, 0, 0, 0, cmps64)
     C(0xe558, CGHSI,   SIL,   GIE, m1_64, i2, 0, 0, 0, cmps64)
 /* COMPARE HALFWORD RELATIVE LONG */
-    C(0xc605, CHRL,    RIL_a, GIE, r1_o, mri2_32s, 0, 0, 0, cmps32)
-    C(0xc604, CGHRL,   RIL_a, GIE, r1_o, mri2_64, 0, 0, 0, cmps64)
+    C(0xc605, CHRL,    RIL_b, GIE, r1_o, mri2_32s, 0, 0, 0, cmps32)
+    C(0xc604, CGHRL,   RIL_b, GIE, r1_o, mri2_64, 0, 0, 0, cmps64)
+/* COMPARE HIGH */
+    C(0xb9cd, CHHR,    RRE,   HW,  r1_sr32, r2_sr32, 0, 0, 0, cmps32)
+    C(0xb9dd, CHLR,    RRE,   HW,  r1_sr32, r2_o, 0, 0, 0, cmps32)
+    C(0xe3cd, CHF,     RXY_a, HW,  r1_sr32, m2_32s, 0, 0, 0, cmps32)
+/* COMPARE IMMEDIATE HIGH */
+    C(0xcc0d, CIH,     RIL_a, HW,  r1_sr32, i2, 0, 0, 0, cmps32)
 
 /* COMPARE LOGICAL */
     C(0x1500, CLR,     RR_a,  Z,   r1, r2, 0, 0, 0, cmpu32)
@@ -150,6 +190,10 @@
     C(0xe321, CLG,     RXY_a, Z,   r1, m2_64, 0, 0, 0, cmpu64)
     C(0xe331, CLGF,    RXY_a, Z,   r1, m2_32u, 0, 0, 0, cmpu64)
     C(0xd500, CLC,     SS_a,  Z,   la1, a2, 0, 0, clc, 0)
+/* COMPARE LOGICAL HIGH */
+    C(0xb9cf, CLHHR,   RRE,   HW,  r1_sr32, r2_sr32, 0, 0, 0, cmpu32)
+    C(0xb9df, CLHLR,   RRE,   HW,  r1_sr32, r2_o, 0, 0, 0, cmpu32)
+    C(0xe3cf, CLHF,    RXY_a, HW,  r1_sr32, m2_32s, 0, 0, 0, cmpu32)
 /* COMPARE LOGICAL IMMEDIATE */
     C(0xc20f, CLFI,    RIL_a, EI,  r1, i2, 0, 0, 0, cmpu32)
     C(0xc20e, CLGFI,   RIL_a, EI,  r1, i2_32u, 0, 0, 0, cmpu64)
@@ -158,6 +202,8 @@
     C(0xe555, CLHHSI,  SIL,   GIE, m1_16u, i2_16u, 0, 0, 0, cmpu64)
     C(0xe55d, CLFHSI,  SIL,   GIE, m1_32u, i2_16u, 0, 0, 0, cmpu64)
     C(0xe559, CLGHSI,  SIL,   GIE, m1_64, i2_16u, 0, 0, 0, cmpu64)
+/* COMPARE LOGICAL IMMEDIATE HIGH */
+    C(0xcc0f, CLIH,    RIL_a, HW,  r1_sr32, i2, 0, 0, 0, cmpu32)
 /* COMPARE LOGICAL RELATIVE LONG */
     C(0xc60f, CLRL,    RIL_b, GIE, r1_o, mri2_32u, 0, 0, 0, cmpu32)
     C(0xc60a, CLGRL,   RIL_b, GIE, r1_o, mri2_64, 0, 0, 0, cmpu64)
@@ -209,8 +255,10 @@
 /* COMPARE LOGICAL AND TRAP */
     D(0xb973, CLRT,    RRF_c, GIE, r1_32u, r2_32u, 0, 0, ct, 0, 1)
     D(0xb961, CLGRT,   RRF_c, GIE, r1_o, r2_o, 0, 0, ct, 0, 1)
+    D(0xeb23, CLT,     RSY_b, MIE, r1_32u, m2_32u, 0, 0, ct, 0, 1)
+    D(0xeb2b, CLGT,    RSY_b, MIE, r1_o, m2_64, 0, 0, ct, 0, 1)
     D(0xec73, CLFIT,   RIE_a, GIE, r1_32u, i2_32u, 0, 0, ct, 0, 1)
-    D(0xec71, CLGIT,   RIE_a, GIE, r1_o, i2_32u, 0, 0, ct, 0, 0)
+    D(0xec71, CLGIT,   RIE_a, GIE, r1_o, i2_32u, 0, 0, ct, 0, 1)
 
 /* CONVERT TO DECIMAL */
     C(0x4e00, CVD,     RX_a,  Z,   r1_o, a2, 0, 0, cvd, 0)
@@ -285,8 +333,12 @@
 
 /* EXTRACT ACCESS */
     C(0xb24f, EAR,     RRE,   Z,   0, 0, new, r1_32, ear, 0)
+/* EXTRACT CPU ATTRIBUTE */
+    C(0xeb4c, ECAG,    RSY_a, GIE, 0, a2, r1, 0, ecag, 0)
 /* EXTRACT FPC */
     C(0xb38c, EFPC,    RRE,   Z,   0, 0, new, r1_32, efpc, 0)
+/* EXTRACT PSW */
+    C(0xb98d, EPSW,    RRE,   Z,   0, 0, 0, 0, epsw, 0)
 
 /* FIND LEFTMOST ONE */
     C(0xb983, FLOGR,   RRE,   EI,  0, r2_o, r1_P, 0, flogr, 0)
@@ -311,7 +363,7 @@
 /* LOAD */
     C(0x1800, LR,      RR_a,  Z,   0, r2_o, 0, cond_r1r2_32, mov2, 0)
     C(0x5800, L,       RX_a,  Z,   0, a2, new, r1_32, ld32s, 0)
-    C(0xe358, LY,      RXY_a, Z,   0, a2, new, r1_32, ld32s, 0)
+    C(0xe358, LY,      RXY_a, LD,  0, a2, new, r1_32, ld32s, 0)
     C(0xb904, LGR,     RRE,   Z,   0, r2_o, 0, r1, mov2, 0)
     C(0xb914, LGFR,    RRE,   Z,   0, r2_32s, 0, r1, mov2, 0)
     C(0xe304, LG,      RXY_a, Z,   0, a2, r1, 0, ld64, 0)
@@ -332,8 +384,26 @@
 /* LOAD ADDRESS */
     C(0x4100, LA,      RX_a,  Z,   0, a2, 0, r1, mov2, 0)
     C(0xe371, LAY,     RXY_a, LD,  0, a2, 0, r1, mov2, 0)
+/* LOAD ADDRESS EXTENDED */
+    C(0x5100, LAE,     RX_a,  Z,   0, a2, 0, r1, mov2e, 0)
+    C(0xe375, LAEY,    RXY_a, GIE, 0, a2, 0, r1, mov2e, 0)
 /* LOAD ADDRESS RELATIVE LONG */
     C(0xc000, LARL,    RIL_b, Z,   0, ri2, 0, r1, mov2, 0)
+/* LOAD AND ADD */
+    C(0xebf8, LAA,     RSY_a, ILA, r3_32s, m2_32s_atomic, new, m2_32_r1_atomic, add, adds32)
+    C(0xebe8, LAAG,    RSY_a, ILA, r3, m2_64_atomic, new, m2_64_r1_atomic, add, adds64)
+/* LOAD AND ADD LOGICAL */
+    C(0xebfa, LAAL,    RSY_a, ILA, r3_32s, m2_32s_atomic, new, m2_32_r1_atomic, add, addu32)
+    C(0xebea, LAALG,   RSY_a, ILA, r3, m2_64_atomic, new, m2_64_r1_atomic, add, addu64)
+/* LOAD AND AND */
+    C(0xebf4, LAN,     RSY_a, ILA, r3_32s, m2_32s_atomic, new, m2_32_r1_atomic, and, nz32)
+    C(0xebe4, LANG,    RSY_a, ILA, r3, m2_64_atomic, new, m2_64_r1_atomic, and, nz64)
+/* LOAD AND EXCLUSIVE OR */
+    C(0xebf7, LAX,     RSY_a, ILA, r3_32s, m2_32s_atomic, new, m2_32_r1_atomic, xor, nz32)
+    C(0xebe7, LAXG,    RSY_a, ILA, r3, m2_64_atomic, new, m2_64_r1_atomic, xor, nz64)
+/* LOAD AND OR */
+    C(0xebf6, LAO,     RSY_a, ILA, r3_32s, m2_32s_atomic, new, m2_32_r1_atomic, or, nz32)
+    C(0xebe6, LAOG,    RSY_a, ILA, r3, m2_64_atomic, new, m2_64_r1_atomic, or, nz64)
 /* LOAD AND TEST */
     C(0x1200, LTR,     RR_a,  Z,   0, r2_o, 0, cond_r1r2_32, mov2, s32)
     C(0xb902, LTGR,    RRE,   Z,   0, r2_o, 0, r1, mov2, s64)
@@ -344,11 +414,16 @@
     C(0xb302, LTEBR,   RRE,   Z,   0, e2, 0, cond_e1e2, mov2, f32)
     C(0xb312, LTDBR,   RRE,   Z,   0, f2_o, 0, f1, mov2, f64)
     C(0xb342, LTXBR,   RRE,   Z,   0, x2_o, 0, x1, movx, f128)
+/* LOAD AND TRAP */
+    C(0xe39f, LAT,     RXY_a, LAT, 0, m2_32u, r1, 0, lat, 0)
+    C(0xe385, LGAT,    RXY_a, LAT, 0, a2, r1, 0, lgat, 0)
 /* LOAD BYTE */
     C(0xb926, LBR,     RRE,   EI,  0, r2_8s, 0, r1_32, mov2, 0)
     C(0xb906, LGBR,    RRE,   EI,  0, r2_8s, 0, r1, mov2, 0)
     C(0xe376, LB,      RXY_a, LD,  0, a2, new, r1_32, ld8s, 0)
     C(0xe377, LGB,     RXY_a, LD,  0, a2, r1, 0, ld8s, 0)
+/* LOAD BYTE HIGH */
+    C(0xe3c0, LBH,     RXY_a, HW,  0, a2, new, r1_32h, ld8s, 0)
 /* LOAD COMPLEMENT */
     C(0x1300, LCR,     RR_a,  Z,   0, r2, new, r1_32, neg, neg32)
     C(0xb903, LCGR,    RRE,   Z,   0, r2, r1, 0, neg, neg64)
@@ -363,15 +438,23 @@
     C(0x4800, LH,      RX_a,  Z,   0, a2, new, r1_32, ld16s, 0)
     C(0xe378, LHY,     RXY_a, LD,  0, a2, new, r1_32, ld16s, 0)
     C(0xe315, LGH,     RXY_a, Z,   0, a2, r1, 0, ld16s, 0)
+/* LOAD HALFWORD HIGH */
+    C(0xe3c4, LHH,     RXY_a, HW,  0, a2, new, r1_32h, ld16s, 0)
 /* LOAD HALFWORD IMMEDIATE */
     C(0xa708, LHI,     RI_a,  Z,   0, i2, 0, r1_32, mov2, 0)
     C(0xa709, LGHI,    RI_a,  Z,   0, i2, 0, r1, mov2, 0)
 /* LOAD HALFWORD RELATIVE LONG */
     C(0xc405, LHRL,    RIL_b, GIE, 0, ri2, new, r1_32, ld16s, 0)
     C(0xc404, LGHRL,   RIL_b, GIE, 0, ri2, r1, 0, ld16s, 0)
+/* LOAD HIGH */
+    C(0xe3ca, LFH,     RXY_a, HW,  0, a2, new, r1_32h, ld32u, 0)
+/* LOAG HIGH AND TRAP */
+    C(0xe3c8, LFHAT,   RXY_a, LAT, 0, m2_32u, r1, 0, lfhat, 0)
 /* LOAD LOGICAL */
     C(0xb916, LLGFR,   RRE,   Z,   0, r2_32u, 0, r1, mov2, 0)
     C(0xe316, LLGF,    RXY_a, Z,   0, a2, r1, 0, ld32u, 0)
+/* LOAD LOGICAL AND TRAP */
+    C(0xe39d, LLGFAT,  RXY_a, LAT, 0, a2, r1, 0, llgfat, 0)
 /* LOAD LOGICAL RELATIVE LONG */
     C(0xc40e, LLGFRL,  RIL_b, GIE, 0, ri2, r1, 0, ld32u, 0)
 /* LOAD LOGICAL CHARACTER */
@@ -379,11 +462,15 @@
     C(0xb984, LLGCR,   RRE,   EI,  0, r2_8u, 0, r1, mov2, 0)
     C(0xe394, LLC,     RXY_a, EI,  0, a2, new, r1_32, ld8u, 0)
     C(0xe390, LLGC,    RXY_a, Z,   0, a2, r1, 0, ld8u, 0)
+/* LOAD LOGICAL CHARACTER HIGH */
+    C(0xe3c2, LLCH,    RXY_a, HW,  0, a2, new, r1_32h, ld8u, 0)
 /* LOAD LOGICAL HALFWORD */
     C(0xb995, LLHR,    RRE,   EI,  0, r2_16u, 0, r1_32, mov2, 0)
     C(0xb985, LLGHR,   RRE,   EI,  0, r2_16u, 0, r1, mov2, 0)
     C(0xe395, LLH,     RXY_a, EI,  0, a2, new, r1_32, ld16u, 0)
     C(0xe391, LLGH,    RXY_a, Z,   0, a2, r1, 0, ld16u, 0)
+/* LOAD LOGICAL HALFWORD HIGH */
+    C(0xe3c6, LLHH,    RXY_a, HW,  0, a2, new, r1_32h, ld16u, 0)
 /* LOAD LOGICAL HALFWORD RELATIVE LONG */
     C(0xc402, LLHRL,   RIL_b, GIE, 0, ri2, new, r1_32, ld16u, 0)
     C(0xc406, LLGHRL,  RIL_b, GIE, 0, ri2, r1, 0, ld16u, 0)
@@ -397,6 +484,9 @@
 /* LOAD LOGICAL THIRTY ONE BITS */
     C(0xb917, LLGTR,   RRE,   Z,  0, r2_o, r1, 0, llgt, 0)
     C(0xe317, LLGT,    RXY_a, Z,  0, m2_32u, r1, 0, llgt, 0)
+/* LOAD LOGICAL THIRTY ONE BITS AND TRAP */
+    C(0xe39c, LLGTAT,  RXY_a, LAT, 0, m2_32u, r1, 0, llgtat, 0)
+
 /* LOAD FPR FROM GR */
     C(0xb3c1, LDGR,    RRE,   FPRGR, 0, r2_o, 0, f1, mov2, 0)
 /* LOAD GR FROM FPR */
@@ -408,11 +498,13 @@
     C(0xb301, LNEBR,   RRE,   Z,   0, e2, new, e1, nabsf32, f32)
     C(0xb311, LNDBR,   RRE,   Z,   0, f2_o, f1, 0, nabsf64, f64)
     C(0xb341, LNXBR,   RRE,   Z,   0, x2_o, x1, 0, nabsf128, f128)
+    C(0xb371, LNDFR,   RRE,   FPSSH, 0, f2_o, f1, 0, nabsf64, 0)
 /* LOAD ON CONDITION */
     C(0xb9f2, LOCR,    RRF_c, LOC, r1, r2, new, r1_32, loc, 0)
     C(0xb9e2, LOCGR,   RRF_c, LOC, r1, r2, r1, 0, loc, 0)
     C(0xebf2, LOC,     RSY_b, LOC, r1, m2_32u, new, r1_32, loc, 0)
     C(0xebe2, LOCG,    RSY_b, LOC, r1, m2_64, r1, 0, loc, 0)
+/* LOAD PAIR DISJOINT TODO */
 /* LOAD POSITIVE */
     C(0x1000, LPR,     RR_a,  Z,   0, r2_32s, new, r1_32, abs, abs32)
     C(0xb900, LPGR,    RRE,   Z,   0, r2, r1, 0, abs, abs64)
@@ -420,6 +512,7 @@
     C(0xb300, LPEBR,   RRE,   Z,   0, e2, new, e1, absf32, f32)
     C(0xb310, LPDBR,   RRE,   Z,   0, f2_o, f1, 0, absf64, f64)
     C(0xb340, LPXBR,   RRE,   Z,   0, x2_o, x1, 0, absf128, f128)
+    C(0xb370, LPDFR,   RRE,   FPSSH, 0, f2_o, f1, 0, absf64, 0)
 /* LOAD REVERSED */
     C(0xb91f, LRVR,    RRE,   Z,   0, r2_32u, new, r1_32, rev32, 0)
     C(0xb90f, LRVGR,   RRE,   Z,   0, r2_o, r1, 0, rev64, 0)
@@ -435,6 +528,10 @@
     C(0xb29d, LFPC,    S,     Z,   0, m2_32u, 0, 0, sfpc, 0)
 /* LOAD FPC AND SIGNAL */
     C(0xb2bd, LFAS,    S,     IEEEE_SIM, 0, m2_32u, 0, 0, sfas, 0)
+/* LOAD FP INTEGER */
+    C(0xb357, FIEBR,   RRF_e, Z,   0, e2, new, e1, fieb, 0)
+    C(0xb35f, FIDBR,   RRF_e, Z,   0, f2_o, f1, 0, fidb, 0)
+    C(0xb347, FIXBR,   RRF_e, Z,   0, x2_o, x1, 0, fixb, 0)
 
 /* LOAD LENGTHENED */
     C(0xb304, LDEBR,   RRE,   Z,   0, e2, f1, 0, ldeb, 0)
@@ -554,8 +651,9 @@
 
 /* ROTATE THEN INSERT SELECTED BITS */
     C(0xec55, RISBG,   RIE_f, GIE, 0, r2, r1, 0, risbg, s64)
-    C(0xec5d, RISBHG,  RIE_f, GIE, 0, r2, r1, 0, risbg, 0)
-    C(0xec51, RISBLG,  RIE_f, GIE, 0, r2, r1, 0, risbg, 0)
+    C(0xec59, RISBGN,  RIE_f, MIE, 0, r2, r1, 0, risbg, 0)
+    C(0xec5d, RISBHG,  RIE_f, HW,  0, r2, r1, 0, risbg, 0)
+    C(0xec51, RISBLG,  RIE_f, HW,  0, r2, r1, 0, risbg, 0)
 /* ROTATE_THEN <OP> SELECTED BITS */
     C(0xec54, RNSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
     C(0xec56, ROSBG,   RIE_f, GIE, 0, r2, r1, 0, rosbg, 0)
@@ -566,6 +664,10 @@
 
 /* SET ACCESS */
     C(0xb24e, SAR,     RRE,   Z,   0, r2_o, 0, 0, sar, 0)
+/* SET ADDRESSING MODE */
+    D(0x010c, SAM24,   E,     Z,   0, 0, 0, 0, sam, 0, 0)
+    D(0x010d, SAM31,   E,     Z,   0, 0, 0, 0, sam, 0, 1)
+    D(0x010e, SAM64,   E,     Z,   0, 0, 0, 0, sam, 0, 3)
 /* SET FPC */
     C(0xb384, SFPC,    RRE,   Z,   0, r1_o, 0, 0, sfpc, 0)
 /* SET FPC AND SIGNAL */
@@ -574,7 +676,7 @@
     C(0xb299, SRNM,    S,     Z,   0, 0, 0, 0, srnm, 0)
     C(0xb2b8, SRNMB,   S,     FPE, 0, 0, 0, 0, srnm, 0)
 /* SET DFP ROUNDING MODE */
-    C(0xb2b9, SRNMT,   S,     DFP, 0, 0, 0, 0, srnm, 0)
+    C(0xb2b9, SRNMT,   S,     DFPR, 0, 0, 0, 0, srnm, 0)
 
 /* SHIFT LEFT SINGLE */
     D(0x8b00, SLA,     RS_a,  Z,   r1, sh32, new, r1_32, sla, 0, 31)
@@ -622,15 +724,21 @@
 /* STORE CHARACTER */
     C(0x4200, STC,     RX_a,  Z,   r1_o, a2, 0, 0, st8, 0)
     C(0xe372, STCY,    RXY_a, LD,  r1_o, a2, 0, 0, st8, 0)
+/* STORE CHARACTER HIGH */
+    C(0xe3c3, STCH,    RXY_a, HW,  r1_sr32, a2, 0, 0, st8, 0)
 /* STORE CHARACTERS UNDER MASK */
     D(0xbe00, STCM,    RS_b,  Z,   r1_o, a2, 0, 0, stcm, 0, 0)
     D(0xeb2d, STCMY,   RSY_b, LD,  r1_o, a2, 0, 0, stcm, 0, 0)
-    D(0xeb2c, STCMH,   RSY_b, LD,  r1_o, a2, 0, 0, stcm, 0, 32)
+    D(0xeb2c, STCMH,   RSY_b, Z,   r1_o, a2, 0, 0, stcm, 0, 32)
 /* STORE HALFWORD */
     C(0x4000, STH,     RX_a,  Z,   r1_o, a2, 0, 0, st16, 0)
     C(0xe370, STHY,    RXY_a, LD,  r1_o, a2, 0, 0, st16, 0)
+/* STORE HALFWORD HIGH */
+    C(0xe3c7, STHH,    RXY_a, HW,  r1_sr32, a2, 0, 0, st16, 0)
 /* STORE HALFWORD RELATIVE LONG */
     C(0xc407, STHRL,   RIL_b, GIE, r1_o, ri2, 0, 0, st16, 0)
+/* STORE HIGH */
+    C(0xe3cb, STFH,    RXY_a, HW,  r1_sr32, a2, 0, 0, st32, 0)
 /* STORE ON CONDITION */
     D(0xebf3, STOC,    RSY_b, LOC, 0, 0, 0, 0, soc, 0, 0)
     D(0xebe3, STOCG,   RSY_b, LOC, 0, 0, 0, 0, soc, 0, 1)
@@ -670,6 +778,9 @@
 /* SUBTRACT HALFWORD */
     C(0x4b00, SH,      RX_a,  Z,   r1, m2_16s, new, r1_32, sub, subs32)
     C(0xe37b, SHY,     RXY_a, LD,  r1, m2_16s, new, r1_32, sub, subs32)
+/* SUBTRACT HIGH */
+    C(0xb9c9, SHHHR,   RRF_a, HW,  r2_sr32, r3_sr32, new, r1_32h, sub, subs32)
+    C(0xb9d9, SHHLR,   RRF_a, HW,  r2_sr32, r3, new, r1_32h, sub, subs32)
 /* SUBTRACT LOGICAL */
     C(0x1f00, SLR,     RR_a,  Z,   r1, r2, new, r1_32, sub, subu32)
     C(0xb9fb, SLRK,    RRF_a, DO,  r2, r3, new, r1_32, sub, subu32)
@@ -680,6 +791,9 @@
     C(0xb9eb, SLGRK,   RRF_a, DO,  r2, r3, r1, 0, sub, subu64)
     C(0xe30b, SLG,     RXY_a, Z,   r1, m2_64, r1, 0, sub, subu64)
     C(0xe31b, SLGF,    RXY_a, Z,   r1, m2_32u, r1, 0, sub, subu64)
+/* SUBTRACT LOCICAL HIGH */
+    C(0xb9cb, SLHHHR,  RRF_a, HW,  r2_sr32, r3_sr32, new, r1_32h, sub, subu32)
+    C(0xb9db, SLHHLR,  RRF_a, HW,  r2_sr32, r3, new, r1_32h, sub, subu32)
 /* SUBTRACT LOGICAL IMMEDIATE */
     C(0xc205, SLFI,    RIL_a, EI,  r1, i2_32u, new, r1_32, sub, subu32)
     C(0xc204, SLGFI,   RIL_a, EI,  r1, i2_32u, r1, 0, sub, subu64)
@@ -707,6 +821,10 @@
 
 /* TRANSLATE */
     C(0xdc00, TR,      SS_a,  Z,   la1, a2, 0, 0, tr, 0)
+/* TRANSLATE AND TEST */
+    C(0xdd00, TRT,     SS_a,  Z,   la1, a2, 0, 0, trt, 0)
+/* TRANSLATE EXTENDED */
+    C(0xb2a5, TRE,     RRE,   Z,   0, r2, r1_P, 0, tre, 0)
 
 /* UNPACK */
     /* Really format SS_b, but we pack both lengths into one argument
@@ -717,7 +835,7 @@
 /* COMPARE AND SWAP AND PURGE */
     C(0xb250, CSP,     RRE,   Z,   0, ra2, 0, 0, csp, 0)
 /* DIAGNOSE (KVM hypercall) */
-    C(0x8300, DIAG,    RX_a,  Z,   0, 0, 0, 0, diag, 0)
+    C(0x8300, DIAG,    RSI,   Z,   0, 0, 0, 0, diag, 0)
 /* INSERT STORAGE KEY EXTENDED */
     C(0xb229, ISKE,    RRE,   Z,   0, r2_o, new, r1_8, iske, 0)
 /* INVALIDATE PAGE TABLE ENTRY */
@@ -733,6 +851,9 @@
     C(0xb100, LRA,     RX_a,  Z,   0, a2, r1, 0, lra, 0)
     C(0xe313, LRAY,    RXY_a, LD,  0, a2, r1, 0, lra, 0)
     C(0xe303, LRAG,    RXY_a, Z,   0, a2, r1, 0, lra, 0)
+/* LOAD USING REAL ADDRESS */
+    C(0xb24b, LURA,    RRE,   Z,   0, r2, new, r1_32, lura, 0)
+    C(0xb905, LURAG,   RRE,   Z,   0, r2, r1, 0, lurag, 0)
 /* MOVE TO PRIMARY */
     C(0xda00, MVCP,    SS_d,  Z,   la1, a2, 0, 0, mvcp, 0)
 /* MOVE TO SECONDARY */
@@ -743,10 +864,6 @@
     C(0xb22a, RRBE,    RRE,   Z,   0, r2_o, 0, 0, rrbe, 0)
 /* SERVICE CALL LOGICAL PROCESSOR (PV hypercall) */
     C(0xb220, SERVC,   RRE,   Z,   r1_o, r2_o, 0, 0, servc, 0)
-/* SET ADDRESSING MODE */
-    D(0x010c, SAM24,   E,     Z,   0, 0, 0, 0, sam, 0, 0)
-    D(0x010d, SAM31,   E,     Z,   0, 0, 0, 0, sam, 0, 1)
-    D(0x010e, SAM64,   E,     Z,   0, 0, 0, 0, sam, 0, 3)
 /* SET ADDRESS SPACE CONTROL FAST */
     C(0xb279, SACF,    S,     Z,   0, a2, 0, 0, sacf, 0)
 /* SET CLOCK */
@@ -768,7 +885,7 @@
     C(0xae00, SIGP,    RS_a,  Z,   r3_o, a2, 0, 0, sigp, 0)
 /* STORE CLOCK */
     C(0xb205, STCK,    S,     Z,   la2, 0, new, m1_64, stck, 0)
-    C(0xb27c, STCKF,   S,     Z,   la2, 0, new, m1_64, stck, 0)
+    C(0xb27c, STCKF,   S,     SCF, la2, 0, new, m1_64, stck, 0)
 /* STORE CLOCK EXTENDED */
     C(0xb278, STCKE,   S,     Z,   0, a2, 0, 0, stcke, 0)
 /* STORE CLOCK COMPARATOR */
@@ -794,20 +911,21 @@
     C(0xad00, STOSM,   SI,    Z,   la1, 0, 0, 0, stnosm, 0)
 /* STORE USING REAL ADDRESS */
     C(0xb246, STURA,   RRE,   Z,   r1_o, r2_o, 0, 0, stura, 0)
+    C(0xb925, STURG,   RRE,   Z,   r1_o, r2_o, 0, 0, sturg, 0)
 /* TEST PROTECTION */
     C(0xe501, TPROT,   SSE,   Z,   la1, a2, 0, 0, tprot, 0)
 
-/* I/O Instructions.  For each we simply indicate non-operation.  */
-    C(0xb276, XSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb230, CSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb231, HSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb232, MSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb23b, RCHP,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb238, RSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb233, SSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb234, STSCH,   S,     Z,   0, 0, 0, 0, subchannel, 0)
-    C(0xb235, TSCH,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+/* CCW I/O Instructions */
+    C(0xb276, XSCH,    S,     Z,   0, 0, 0, 0, xsch, 0)
+    C(0xb230, CSCH,    S,     Z,   0, 0, 0, 0, csch, 0)
+    C(0xb231, HSCH,    S,     Z,   0, 0, 0, 0, hsch, 0)
+    C(0xb232, MSCH,    S,     Z,   0, insn, 0, 0, msch, 0)
+    C(0xb23b, RCHP,    S,     Z,   0, 0, 0, 0, rchp, 0)
+    C(0xb238, RSCH,    S,     Z,   0, 0, 0, 0, rsch, 0)
+    C(0xb233, SSCH,    S,     Z,   0, insn, 0, 0, ssch, 0)
+    C(0xb234, STSCH,   S,     Z,   0, insn, 0, 0, stsch, 0)
+    C(0xb235, TSCH,    S,     Z,   0, insn, 0, 0, tsch, 0)
     /* ??? Not listed in PoO ninth edition, but there's a linux driver that
        uses it: "A CHSC subchannel is usually present on LPAR only."  */
-    C(0xb25f, CHSC,    S,     Z,   0, 0, 0, 0, subchannel, 0)
+    C(0xb25f, CHSC,  RRE,     Z,   0, insn, 0, 0, chsc, 0)
 #endif /* CONFIG_USER_ONLY */
diff --git a/target-s390x/int_helper.c b/target-s390x/int_helper.c
index cb8dd98..370c94d 100644
--- a/target-s390x/int_helper.c
+++ b/target-s390x/int_helper.c
@@ -18,7 +18,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 
@@ -115,59 +117,18 @@
     return ret;
 }
 
-/* absolute value 32-bit */
-uint32_t HELPER(abs_i32)(int32_t val)
-{
-    if (val < 0) {
-        return -val;
-    } else {
-        return val;
-    }
-}
-
-/* negative absolute value 32-bit */
-int32_t HELPER(nabs_i32)(int32_t val)
-{
-    if (val < 0) {
-        return val;
-    } else {
-        return -val;
-    }
-}
-
-/* absolute value 64-bit */
-uint64_t HELPER(abs_i64)(int64_t val)
-{
-    HELPER_LOG("%s: val 0x%" PRIx64 "\n", __func__, val);
-
-    if (val < 0) {
-        return -val;
-    } else {
-        return val;
-    }
-}
-
-/* negative absolute value 64-bit */
-int64_t HELPER(nabs_i64)(int64_t val)
-{
-    if (val < 0) {
-        return val;
-    } else {
-        return -val;
-    }
-}
-
 /* count leading zeros, for find leftmost one */
 uint64_t HELPER(clz)(uint64_t v)
 {
     return clz64(v);
 }
 
-uint64_t HELPER(cvd)(int32_t bin)
+uint64_t HELPER(cvd)(int32_t reg)
 {
     /* positive 0 */
     uint64_t dec = 0x0c;
-    int shift = 4;
+    int64_t bin = reg;
+    int shift;
 
     if (bin < 0) {
         bin = -bin;
@@ -175,9 +136,7 @@
     }
 
     for (shift = 4; (shift < 64) && bin; shift += 4) {
-        int current_number = bin % 10;
-
-        dec |= (current_number) << shift;
+        dec |= (bin % 10) << shift;
         bin /= 10;
     }
 
diff --git a/target-s390x/interrupt.c b/target-s390x/interrupt.c
index 1404d0a..9edef96 100644
--- a/target-s390x/interrupt.c
+++ b/target-s390x/interrupt.c
@@ -7,15 +7,80 @@
  * option) any later version.  See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
+#include "hw/s390x/ioinst.h"
+
+#if !defined(CONFIG_USER_ONLY)
+void cpu_inject_ext(S390CPU *cpu, uint32_t code, uint32_t param,
+                    uint64_t param64)
+{
+    CPUS390XState *env = &cpu->env;
+
+    if (env->ext_index == MAX_EXT_QUEUE - 1) {
+        /* ugh - can't queue anymore. Let's drop. */
+        return;
+    }
+
+    env->ext_index++;
+    assert(env->ext_index < MAX_EXT_QUEUE);
+
+    env->ext_queue[env->ext_index].code = code;
+    env->ext_queue[env->ext_index].param = param;
+    env->ext_queue[env->ext_index].param64 = param64;
+
+    env->pending_int |= INTERRUPT_EXT;
+    cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
+}
+
+static void cpu_inject_io(S390CPU *cpu, uint16_t subchannel_id,
+                          uint16_t subchannel_number,
+                          uint32_t io_int_parm, uint32_t io_int_word)
+{
+    CPUS390XState *env = &cpu->env;
+    int isc = IO_INT_WORD_ISC(io_int_word);
+
+    if (env->io_index[isc] == MAX_IO_QUEUE - 1) {
+        /* ugh - can't queue anymore. Let's drop. */
+        return;
+    }
+
+    env->io_index[isc]++;
+    assert(env->io_index[isc] < MAX_IO_QUEUE);
+
+    env->io_queue[env->io_index[isc]][isc].id = subchannel_id;
+    env->io_queue[env->io_index[isc]][isc].nr = subchannel_number;
+    env->io_queue[env->io_index[isc]][isc].parm = io_int_parm;
+    env->io_queue[env->io_index[isc]][isc].word = io_int_word;
+
+    env->pending_int |= INTERRUPT_IO;
+    cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
+}
+
+static void cpu_inject_crw_mchk(S390CPU *cpu)
+{
+    CPUS390XState *env = &cpu->env;
+
+    if (env->mchk_index == MAX_MCHK_QUEUE - 1) {
+        /* ugh - can't queue anymore. Let's drop. */
+        return;
+    }
+
+    env->mchk_index++;
+    assert(env->mchk_index < MAX_MCHK_QUEUE);
+
+    env->mchk_queue[env->mchk_index].type = 1;
+
+    env->pending_int |= INTERRUPT_MCHK;
+    cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
+}
 
 /*
  * All of the following interrupts are floating, i.e. not per-vcpu.
  * We just need a dummy cpustate in order to be able to inject in the
  * non-kvm case.
  */
-#if !defined(CONFIG_USER_ONLY)
 void s390_sclp_extint(uint32_t parm)
 {
     if (kvm_enabled()) {
@@ -27,17 +92,6 @@
     }
 }
 
-void s390_virtio_irq(int config_change, uint64_t token)
-{
-    if (kvm_enabled()) {
-        kvm_s390_virtio_irq(config_change, token);
-    } else {
-        S390CPU *dummy_cpu = s390_cpu_addr2state(0);
-
-        cpu_inject_ext(dummy_cpu, EXT_VIRTIO, config_change, token);
-    }
-}
-
 void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
                        uint32_t io_int_parm, uint32_t io_int_word)
 {
diff --git a/target-s390x/ioinst.c b/target-s390x/ioinst.c
index b8a6486..a5a288b 100644
--- a/target-s390x/ioinst.c
+++ b/target-s390x/ioinst.c
@@ -1,7 +1,7 @@
 /*
  * I/O instructions for S/390
  *
- * Copyright 2012 IBM Corp.
+ * Copyright 2012, 2015 IBM Corp.
  * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or (at
@@ -9,11 +9,12 @@
  * directory.
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
-#include "ioinst.h"
+#include "hw/s390x/ioinst.h"
 #include "trace.h"
+#include "hw/s390x/s390-pci-bus.h"
 
 int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
                                  int *schid)
@@ -128,12 +129,12 @@
 
 static int ioinst_schib_valid(SCHIB *schib)
 {
-    if ((schib->pmcw.flags & PMCW_FLAGS_MASK_INVALID) ||
-        (schib->pmcw.chars & PMCW_CHARS_MASK_INVALID)) {
+    if ((be16_to_cpu(schib->pmcw.flags) & PMCW_FLAGS_MASK_INVALID) ||
+        (be32_to_cpu(schib->pmcw.chars) & PMCW_CHARS_MASK_INVALID)) {
         return 0;
     }
     /* Disallow extended measurements for now. */
-    if (schib->pmcw.chars & PMCW_CHARS_MASK_XMWME) {
+    if (be32_to_cpu(schib->pmcw.chars) & PMCW_CHARS_MASK_XMWME) {
         return 0;
     }
     return 1;
@@ -143,32 +144,30 @@
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
-    SCHIB *schib;
+    SCHIB schib;
     uint64_t addr;
     int ret = -ENODEV;
     int cc;
-    hwaddr len = sizeof(*schib);
     CPUS390XState *env = &cpu->env;
+    uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb);
+    addr = decode_basedisp_s(env, ipb, &ar);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return;
     }
-    schib = s390_cpu_physical_memory_map(env, addr, &len, 0);
-    if (!schib || len != sizeof(*schib)) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        goto out;
+    if (s390_cpu_virt_mem_read(cpu, addr, ar, &schib, sizeof(schib))) {
+        return;
     }
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
-        !ioinst_schib_valid(schib)) {
+        !ioinst_schib_valid(&schib)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        goto out;
+        return;
     }
     trace_ioinst_sch_id("msch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
     if (sch && css_subch_visible(sch)) {
-        ret = css_do_msch(sch, schib);
+        ret = css_do_msch(sch, &schib);
     }
     switch (ret) {
     case -ENODEV:
@@ -185,9 +184,6 @@
         break;
     }
     setcc(cpu, cc);
-
-out:
-    s390_cpu_physical_memory_unmap(env, schib, len, 0);
 }
 
 static void copy_orb_from_guest(ORB *dest, const ORB *src)
@@ -215,28 +211,26 @@
 {
     int cssid, ssid, schid, m;
     SubchDev *sch;
-    ORB *orig_orb, orb;
+    ORB orig_orb, orb;
     uint64_t addr;
     int ret = -ENODEV;
     int cc;
-    hwaddr len = sizeof(*orig_orb);
     CPUS390XState *env = &cpu->env;
+    uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb);
+    addr = decode_basedisp_s(env, ipb, &ar);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return;
     }
-    orig_orb = s390_cpu_physical_memory_map(env, addr, &len, 0);
-    if (!orig_orb || len != sizeof(*orig_orb)) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        goto out;
+    if (s390_cpu_virt_mem_read(cpu, addr, ar, &orig_orb, sizeof(orb))) {
+        return;
     }
-    copy_orb_from_guest(&orb, orig_orb);
+    copy_orb_from_guest(&orb, &orig_orb);
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid) ||
         !ioinst_orb_valid(&orb)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        goto out;
+        return;
     }
     trace_ioinst_sch_id("ssch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
@@ -258,35 +252,31 @@
         break;
     }
     setcc(cpu, cc);
-
-out:
-    s390_cpu_physical_memory_unmap(env, orig_orb, len, 0);
 }
 
 void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb)
 {
-    CRW *crw;
+    CRW crw;
     uint64_t addr;
     int cc;
-    hwaddr len = sizeof(*crw);
     CPUS390XState *env = &cpu->env;
+    uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb);
+    addr = decode_basedisp_s(env, ipb, &ar);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return;
     }
-    crw = s390_cpu_physical_memory_map(env, addr, &len, 1);
-    if (!crw || len != sizeof(*crw)) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        goto out;
-    }
-    cc = css_do_stcrw(crw);
-    /* 0 - crw stored, 1 - zeroes stored */
-    setcc(cpu, cc);
 
-out:
-    s390_cpu_physical_memory_unmap(env, crw, len, 1);
+    cc = css_do_stcrw(&crw);
+    /* 0 - crw stored, 1 - zeroes stored */
+
+    if (s390_cpu_virt_mem_write(cpu, addr, ar, &crw, sizeof(crw)) == 0) {
+        setcc(cpu, cc);
+    } else if (cc == 0) {
+        /* Write failed: requeue CRW since STCRW is a suppressing instruction */
+        css_undo_stcrw(&crw);
+    }
 }
 
 void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb)
@@ -295,30 +285,32 @@
     SubchDev *sch;
     uint64_t addr;
     int cc;
-    SCHIB *schib;
-    hwaddr len = sizeof(*schib);
+    SCHIB schib;
     CPUS390XState *env = &cpu->env;
+    uint8_t ar;
 
-    addr = decode_basedisp_s(env, ipb);
+    addr = decode_basedisp_s(env, ipb, &ar);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return;
     }
-    schib = s390_cpu_physical_memory_map(env, addr, &len, 1);
-    if (!schib || len != sizeof(*schib)) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        goto out;
-    }
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
-        program_interrupt(env, PGM_OPERAND, 2);
-        goto out;
+        /*
+         * As operand exceptions have a lower priority than access exceptions,
+         * we check whether the memory area is writeable (injecting the
+         * access execption if it is not) first.
+         */
+        if (!s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib))) {
+            program_interrupt(env, PGM_OPERAND, 2);
+        }
+        return;
     }
     trace_ioinst_sch_id("stsch", cssid, ssid, schid);
     sch = css_find_subch(m, cssid, ssid, schid);
     if (sch) {
         if (css_subch_visible(sch)) {
-            css_do_stsch(sch, schib);
+            css_do_stsch(sch, &schib);
             cc = 0;
         } else {
             /* Indicate no more subchannels in this css/ss */
@@ -329,53 +321,67 @@
             cc = 3; /* No more subchannels in this css/ss */
         } else {
             /* Store an empty schib. */
-            memset(schib, 0, sizeof(*schib));
+            memset(&schib, 0, sizeof(schib));
             cc = 0;
         }
     }
+    if (cc != 3) {
+        if (s390_cpu_virt_mem_write(cpu, addr, ar, &schib,
+                                    sizeof(schib)) != 0) {
+            return;
+        }
+    } else {
+        /* Access exceptions have a higher priority than cc3 */
+        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib)) != 0) {
+            return;
+        }
+    }
     setcc(cpu, cc);
-
-out:
-    s390_cpu_physical_memory_unmap(env, schib, len, 1);
 }
 
-int ioinst_handle_tsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb)
+int ioinst_handle_tsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb)
 {
+    CPUS390XState *env = &cpu->env;
     int cssid, ssid, schid, m;
     SubchDev *sch;
-    IRB *irb;
+    IRB irb;
     uint64_t addr;
-    int ret = -ENODEV;
-    int cc;
-    hwaddr len = sizeof(*irb);
+    int cc, irb_len;
+    uint8_t ar;
 
     if (ioinst_disassemble_sch_ident(reg1, &m, &cssid, &ssid, &schid)) {
         program_interrupt(env, PGM_OPERAND, 2);
         return -EIO;
     }
     trace_ioinst_sch_id("tsch", cssid, ssid, schid);
-    addr = decode_basedisp_s(env, ipb);
+    addr = decode_basedisp_s(env, ipb, &ar);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return -EIO;
     }
-    irb = s390_cpu_physical_memory_map(env, addr, &len, 1);
-    if (!irb || len != sizeof(*irb)) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        cc = -EIO;
-        goto out;
-    }
+
     sch = css_find_subch(m, cssid, ssid, schid);
     if (sch && css_subch_visible(sch)) {
-        ret = css_do_tsch(sch, irb);
-        /* 0 - status pending, 1 - not status pending */
-        cc = ret;
+        cc = css_do_tsch_get_irb(sch, &irb, &irb_len);
     } else {
         cc = 3;
     }
-out:
-    s390_cpu_physical_memory_unmap(env, irb, sizeof(*irb), 1);
-    return cc;
+    /* 0 - status pending, 1 - not status pending, 3 - not operational */
+    if (cc != 3) {
+        if (s390_cpu_virt_mem_write(cpu, addr, ar, &irb, irb_len) != 0) {
+            return -EFAULT;
+        }
+        css_do_tsch_update_subch(sch);
+    } else {
+        irb_len = sizeof(irb) - sizeof(irb.emw);
+        /* Access exceptions have a higher priority than cc3 */
+        if (s390_cpu_virt_mem_check_write(cpu, addr, ar, irb_len) != 0) {
+            return -EFAULT;
+        }
+    }
+
+    setcc(cpu, cc);
+    return 0;
 }
 
 typedef struct ChscReq {
@@ -398,6 +404,7 @@
 #define CHSC_SCPD 0x0002
 #define CHSC_SCSC 0x0010
 #define CHSC_SDA  0x0031
+#define CHSC_SEI  0x000e
 
 #define CHSC_SCPD_0_M 0x20000000
 #define CHSC_SCPD_0_C 0x10000000
@@ -502,6 +509,7 @@
 
     general_chars[0] = cpu_to_be32(0x03000000);
     general_chars[1] = cpu_to_be32(0x00059000);
+    general_chars[3] = cpu_to_be32(0x00080000);
 
     chsc_chars[0] = cpu_to_be32(0x40000000);
     chsc_chars[3] = cpu_to_be32(0x00040000);
@@ -566,6 +574,55 @@
     res->param = 0;
 }
 
+static int chsc_sei_nt0_get_event(void *res)
+{
+    /* no events yet */
+    return 1;
+}
+
+static int chsc_sei_nt0_have_event(void)
+{
+    /* no events yet */
+    return 0;
+}
+
+#define CHSC_SEI_NT0    (1ULL << 63)
+#define CHSC_SEI_NT2    (1ULL << 61)
+static void ioinst_handle_chsc_sei(ChscReq *req, ChscResp *res)
+{
+    uint64_t selection_mask = ldq_p(&req->param1);
+    uint8_t *res_flags = (uint8_t *)res->data;
+    int have_event = 0;
+    int have_more = 0;
+
+    /* regarding architecture nt0 can not be masked */
+    have_event = !chsc_sei_nt0_get_event(res);
+    have_more = chsc_sei_nt0_have_event();
+
+    if (selection_mask & CHSC_SEI_NT2) {
+        if (!have_event) {
+            have_event = !chsc_sei_nt2_get_event(res);
+        }
+
+        if (!have_more) {
+            have_more = chsc_sei_nt2_have_event();
+        }
+    }
+
+    if (have_event) {
+        res->code = cpu_to_be16(0x0001);
+        if (have_more) {
+            (*res_flags) |= 0x80;
+        } else {
+            (*res_flags) &= ~0x80;
+            css_clear_sei_pending();
+        }
+    } else {
+        res->code = cpu_to_be16(0x0005);
+        res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
+    }
+}
+
 static void ioinst_handle_chsc_unimplemented(ChscResp *res)
 {
     res->len = cpu_to_be16(CHSC_MIN_RESP_LEN);
@@ -581,8 +638,8 @@
     int reg;
     uint16_t len;
     uint16_t command;
-    hwaddr map_size = TARGET_PAGE_SIZE;
     CPUS390XState *env = &cpu->env;
+    uint8_t buf[TARGET_PAGE_SIZE];
 
     trace_ioinst("chsc");
     reg = (ipb >> 20) & 0x00f;
@@ -592,16 +649,20 @@
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return;
     }
-    req = s390_cpu_physical_memory_map(env, addr, &map_size, 1);
-    if (!req || map_size != TARGET_PAGE_SIZE) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        goto out;
+    /*
+     * Reading sizeof(ChscReq) bytes is currently enough for all of our
+     * present CHSC sub-handlers ... if we ever need more, we should take
+     * care of req->len here first.
+     */
+    if (s390_cpu_virt_mem_read(cpu, addr, reg, buf, sizeof(ChscReq))) {
+        return;
     }
+    req = (ChscReq *)buf;
     len = be16_to_cpu(req->len);
     /* Length field valid? */
     if ((len < 16) || (len > 4088) || (len & 7)) {
         program_interrupt(env, PGM_OPERAND, 2);
-        goto out;
+        return;
     }
     memset((char *)req + len, 0, TARGET_PAGE_SIZE - len);
     res = (void *)((char *)req + len);
@@ -617,26 +678,32 @@
     case CHSC_SDA:
         ioinst_handle_chsc_sda(req, res);
         break;
+    case CHSC_SEI:
+        ioinst_handle_chsc_sei(req, res);
+        break;
     default:
         ioinst_handle_chsc_unimplemented(res);
         break;
     }
 
-    setcc(cpu, 0);    /* Command execution complete */
-out:
-    s390_cpu_physical_memory_unmap(env, req, map_size, 1);
+    if (!s390_cpu_virt_mem_write(cpu, addr + len, reg, res,
+                                 be16_to_cpu(res->len))) {
+        setcc(cpu, 0);    /* Command execution complete */
+    }
 }
 
-int ioinst_handle_tpi(CPUS390XState *env, uint32_t ipb)
+int ioinst_handle_tpi(S390CPU *cpu, uint32_t ipb)
 {
+    CPUS390XState *env = &cpu->env;
     uint64_t addr;
     int lowcore;
-    IOIntCode *int_code;
-    hwaddr len, orig_len;
+    IOIntCode int_code;
+    hwaddr len;
     int ret;
+    uint8_t ar;
 
     trace_ioinst("tpi");
-    addr = decode_basedisp_s(env, ipb);
+    addr = decode_basedisp_s(env, ipb, &ar);
     if (addr & 3) {
         program_interrupt(env, PGM_SPECIFICATION, 2);
         return -EIO;
@@ -644,16 +711,10 @@
 
     lowcore = addr ? 0 : 1;
     len = lowcore ? 8 /* two words */ : 12 /* three words */;
-    orig_len = len;
-    int_code = s390_cpu_physical_memory_map(env, addr, &len, 1);
-    if (!int_code || (len != orig_len)) {
-        program_interrupt(env, PGM_ADDRESSING, 2);
-        ret = -EIO;
-        goto out;
+    ret = css_do_tpi(&int_code, lowcore);
+    if (ret == 1) {
+        s390_cpu_virt_mem_write(cpu, lowcore ? 184 : addr, ar, &int_code, len);
     }
-    ret = css_do_tpi(int_code, lowcore);
-out:
-    s390_cpu_physical_memory_unmap(env, int_code, len, 1);
     return ret;
 }
 
diff --git a/target-s390x/ioinst.h b/target-s390x/ioinst.h
deleted file mode 100644
index 29f6423..0000000
--- a/target-s390x/ioinst.h
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * S/390 channel I/O instructions
- *
- * Copyright 2012 IBM Corp.
- * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or (at
- * your option) any later version. See the COPYING file in the top-level
- * directory.
-*/
-
-#ifndef IOINST_S390X_H
-#define IOINST_S390X_H
-/*
- * Channel I/O related definitions, as defined in the Principles
- * Of Operation (and taken from the Linux implementation).
- */
-
-/* subchannel status word (command mode only) */
-typedef struct SCSW {
-    uint16_t flags;
-    uint16_t ctrl;
-    uint32_t cpa;
-    uint8_t dstat;
-    uint8_t cstat;
-    uint16_t count;
-} QEMU_PACKED SCSW;
-
-#define SCSW_FLAGS_MASK_KEY 0xf000
-#define SCSW_FLAGS_MASK_SCTL 0x0800
-#define SCSW_FLAGS_MASK_ESWF 0x0400
-#define SCSW_FLAGS_MASK_CC 0x0300
-#define SCSW_FLAGS_MASK_FMT 0x0080
-#define SCSW_FLAGS_MASK_PFCH 0x0040
-#define SCSW_FLAGS_MASK_ISIC 0x0020
-#define SCSW_FLAGS_MASK_ALCC 0x0010
-#define SCSW_FLAGS_MASK_SSI 0x0008
-#define SCSW_FLAGS_MASK_ZCC 0x0004
-#define SCSW_FLAGS_MASK_ECTL 0x0002
-#define SCSW_FLAGS_MASK_PNO 0x0001
-
-#define SCSW_CTRL_MASK_FCTL 0x7000
-#define SCSW_CTRL_MASK_ACTL 0x0fe0
-#define SCSW_CTRL_MASK_STCTL 0x001f
-
-#define SCSW_FCTL_CLEAR_FUNC 0x1000
-#define SCSW_FCTL_HALT_FUNC 0x2000
-#define SCSW_FCTL_START_FUNC 0x4000
-
-#define SCSW_ACTL_SUSP 0x0020
-#define SCSW_ACTL_DEVICE_ACTIVE 0x0040
-#define SCSW_ACTL_SUBCH_ACTIVE 0x0080
-#define SCSW_ACTL_CLEAR_PEND 0x0100
-#define SCSW_ACTL_HALT_PEND  0x0200
-#define SCSW_ACTL_START_PEND 0x0400
-#define SCSW_ACTL_RESUME_PEND 0x0800
-
-#define SCSW_STCTL_STATUS_PEND 0x0001
-#define SCSW_STCTL_SECONDARY 0x0002
-#define SCSW_STCTL_PRIMARY 0x0004
-#define SCSW_STCTL_INTERMEDIATE 0x0008
-#define SCSW_STCTL_ALERT 0x0010
-
-#define SCSW_DSTAT_ATTENTION     0x80
-#define SCSW_DSTAT_STAT_MOD      0x40
-#define SCSW_DSTAT_CU_END        0x20
-#define SCSW_DSTAT_BUSY          0x10
-#define SCSW_DSTAT_CHANNEL_END   0x08
-#define SCSW_DSTAT_DEVICE_END    0x04
-#define SCSW_DSTAT_UNIT_CHECK    0x02
-#define SCSW_DSTAT_UNIT_EXCEP    0x01
-
-#define SCSW_CSTAT_PCI           0x80
-#define SCSW_CSTAT_INCORR_LEN    0x40
-#define SCSW_CSTAT_PROG_CHECK    0x20
-#define SCSW_CSTAT_PROT_CHECK    0x10
-#define SCSW_CSTAT_DATA_CHECK    0x08
-#define SCSW_CSTAT_CHN_CTRL_CHK  0x04
-#define SCSW_CSTAT_INTF_CTRL_CHK 0x02
-#define SCSW_CSTAT_CHAIN_CHECK   0x01
-
-/* path management control word */
-typedef struct PMCW {
-    uint32_t intparm;
-    uint16_t flags;
-    uint16_t devno;
-    uint8_t  lpm;
-    uint8_t  pnom;
-    uint8_t  lpum;
-    uint8_t  pim;
-    uint16_t mbi;
-    uint8_t  pom;
-    uint8_t  pam;
-    uint8_t  chpid[8];
-    uint32_t chars;
-} QEMU_PACKED PMCW;
-
-#define PMCW_FLAGS_MASK_QF 0x8000
-#define PMCW_FLAGS_MASK_W 0x4000
-#define PMCW_FLAGS_MASK_ISC 0x3800
-#define PMCW_FLAGS_MASK_ENA 0x0080
-#define PMCW_FLAGS_MASK_LM 0x0060
-#define PMCW_FLAGS_MASK_MME 0x0018
-#define PMCW_FLAGS_MASK_MP 0x0004
-#define PMCW_FLAGS_MASK_TF 0x0002
-#define PMCW_FLAGS_MASK_DNV 0x0001
-#define PMCW_FLAGS_MASK_INVALID 0x0700
-
-#define PMCW_CHARS_MASK_ST 0x00e00000
-#define PMCW_CHARS_MASK_MBFC 0x00000004
-#define PMCW_CHARS_MASK_XMWME 0x00000002
-#define PMCW_CHARS_MASK_CSENSE 0x00000001
-#define PMCW_CHARS_MASK_INVALID 0xff1ffff8
-
-/* subchannel information block */
-typedef struct SCHIB {
-    PMCW pmcw;
-    SCSW scsw;
-    uint64_t mba;
-    uint8_t mda[4];
-} QEMU_PACKED SCHIB;
-
-/* interruption response block */
-typedef struct IRB {
-    SCSW scsw;
-    uint32_t esw[5];
-    uint32_t ecw[8];
-    uint32_t emw[8];
-} QEMU_PACKED IRB;
-
-/* operation request block */
-typedef struct ORB {
-    uint32_t intparm;
-    uint16_t ctrl0;
-    uint8_t lpm;
-    uint8_t ctrl1;
-    uint32_t cpa;
-} QEMU_PACKED ORB;
-
-#define ORB_CTRL0_MASK_KEY 0xf000
-#define ORB_CTRL0_MASK_SPND 0x0800
-#define ORB_CTRL0_MASK_STR 0x0400
-#define ORB_CTRL0_MASK_MOD 0x0200
-#define ORB_CTRL0_MASK_SYNC 0x0100
-#define ORB_CTRL0_MASK_FMT 0x0080
-#define ORB_CTRL0_MASK_PFCH 0x0040
-#define ORB_CTRL0_MASK_ISIC 0x0020
-#define ORB_CTRL0_MASK_ALCC 0x0010
-#define ORB_CTRL0_MASK_SSIC 0x0008
-#define ORB_CTRL0_MASK_C64 0x0002
-#define ORB_CTRL0_MASK_I2K 0x0001
-#define ORB_CTRL0_MASK_INVALID 0x0004
-
-#define ORB_CTRL1_MASK_ILS 0x80
-#define ORB_CTRL1_MASK_MIDAW 0x40
-#define ORB_CTRL1_MASK_ORBX 0x01
-#define ORB_CTRL1_MASK_INVALID 0x3e
-
-/* channel command word (type 0) */
-typedef struct CCW0 {
-        uint8_t cmd_code;
-        uint8_t cda0;
-        uint16_t cda1;
-        uint8_t flags;
-        uint8_t reserved;
-        uint16_t count;
-} QEMU_PACKED CCW0;
-
-/* channel command word (type 1) */
-typedef struct CCW1 {
-    uint8_t cmd_code;
-    uint8_t flags;
-    uint16_t count;
-    uint32_t cda;
-} QEMU_PACKED CCW1;
-
-#define CCW_FLAG_DC              0x80
-#define CCW_FLAG_CC              0x40
-#define CCW_FLAG_SLI             0x20
-#define CCW_FLAG_SKIP            0x10
-#define CCW_FLAG_PCI             0x08
-#define CCW_FLAG_IDA             0x04
-#define CCW_FLAG_SUSPEND         0x02
-
-#define CCW_CMD_NOOP             0x03
-#define CCW_CMD_BASIC_SENSE      0x04
-#define CCW_CMD_TIC              0x08
-#define CCW_CMD_SENSE_ID         0xe4
-
-typedef struct CRW {
-    uint16_t flags;
-    uint16_t rsid;
-} QEMU_PACKED CRW;
-
-#define CRW_FLAGS_MASK_S 0x4000
-#define CRW_FLAGS_MASK_R 0x2000
-#define CRW_FLAGS_MASK_C 0x1000
-#define CRW_FLAGS_MASK_RSC 0x0f00
-#define CRW_FLAGS_MASK_A 0x0080
-#define CRW_FLAGS_MASK_ERC 0x003f
-
-#define CRW_ERC_INIT 0x02
-#define CRW_ERC_IPI  0x04
-
-#define CRW_RSC_SUBCH 0x3
-#define CRW_RSC_CHP   0x4
-
-/* I/O interruption code */
-typedef struct IOIntCode {
-    uint32_t subsys_id;
-    uint32_t intparm;
-    uint32_t interrupt_id;
-} QEMU_PACKED IOIntCode;
-
-/* schid disintegration */
-#define IOINST_SCHID_ONE(_schid)   ((_schid & 0x00010000) >> 16)
-#define IOINST_SCHID_M(_schid)     ((_schid & 0x00080000) >> 19)
-#define IOINST_SCHID_CSSID(_schid) ((_schid & 0xff000000) >> 24)
-#define IOINST_SCHID_SSID(_schid)  ((_schid & 0x00060000) >> 17)
-#define IOINST_SCHID_NR(_schid)    (_schid & 0x0000ffff)
-
-#define IO_INT_WORD_ISC(_int_word) ((_int_word & 0x38000000) >> 24)
-#define ISC_TO_ISC_BITS(_isc)      ((0x80 >> _isc) << 24)
-
-#define IO_INT_WORD_AI 0x80000000
-
-int ioinst_disassemble_sch_ident(uint32_t value, int *m, int *cssid, int *ssid,
-                                 int *schid);
-void ioinst_handle_xsch(S390CPU *cpu, uint64_t reg1);
-void ioinst_handle_csch(S390CPU *cpu, uint64_t reg1);
-void ioinst_handle_hsch(S390CPU *cpu, uint64_t reg1);
-void ioinst_handle_msch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
-void ioinst_handle_ssch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
-void ioinst_handle_stcrw(S390CPU *cpu, uint32_t ipb);
-void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb);
-int ioinst_handle_tsch(CPUS390XState *env, uint64_t reg1, uint32_t ipb);
-void ioinst_handle_chsc(S390CPU *cpu, uint32_t ipb);
-int ioinst_handle_tpi(CPUS390XState *env, uint32_t ipb);
-void ioinst_handle_schm(S390CPU *cpu, uint64_t reg1, uint64_t reg2,
-                        uint32_t ipb);
-void ioinst_handle_rsch(S390CPU *cpu, uint64_t reg1);
-void ioinst_handle_rchp(S390CPU *cpu, uint64_t reg1);
-void ioinst_handle_sal(S390CPU *cpu, uint64_t reg1);
-
-#endif
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 2c638ab..80ac621 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -21,25 +21,31 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/ioctl.h>
-#include <sys/mman.h>
 
 #include <linux/kvm.h>
 #include <asm/ptrace.h>
 
 #include "qemu-common.h"
+#include "cpu.h"
+#include "qemu/error-report.h"
 #include "qemu/timer.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
 #include "hw/hw.h"
-#include "cpu.h"
 #include "sysemu/device_tree.h"
 #include "qapi/qmp/qjson.h"
-#include "monitor/monitor.h"
 #include "exec/gdbstub.h"
+#include "exec/address-spaces.h"
 #include "trace.h"
 #include "qapi-event.h"
+#include "hw/s390x/s390-pci-inst.h"
+#include "hw/s390x/s390-pci-bus.h"
+#include "hw/s390x/ipl.h"
+#include "hw/s390x/ebcdic.h"
+#include "exec/memattrs.h"
+#include "hw/s390x/s390-virtio-ccw.h"
 
 /* #define DEBUG_KVM */
 
@@ -51,11 +57,15 @@
     do { } while (0)
 #endif
 
+#define kvm_vm_check_mem_attr(s, attr) \
+    kvm_vm_check_attr(s, KVM_S390_VM_MEM_CTRL, attr)
+
 #define IPA0_DIAG                       0x8300
 #define IPA0_SIGP                       0xae00
 #define IPA0_B2                         0xb200
 #define IPA0_B9                         0xb900
 #define IPA0_EB                         0xeb00
+#define IPA0_E3                         0xe300
 
 #define PRIV_B2_SCLP_CALL               0x20
 #define PRIV_B2_CSCH                    0x30
@@ -76,9 +86,19 @@
 #define PRIV_B2_XSCH                    0x76
 
 #define PRIV_EB_SQBS                    0x8a
+#define PRIV_EB_PCISTB                  0xd0
+#define PRIV_EB_SIC                     0xd1
 
 #define PRIV_B9_EQBS                    0x9c
+#define PRIV_B9_CLP                     0xa0
+#define PRIV_B9_PCISTG                  0xd0
+#define PRIV_B9_PCILG                   0xd2
+#define PRIV_B9_RPCIT                   0xd3
 
+#define PRIV_E3_MPCIFC                  0xd0
+#define PRIV_E3_STPCIFC                 0xd4
+
+#define DIAG_TIMEREVENT                 0x288
 #define DIAG_IPL                        0x308
 #define DIAG_KVM_HYPERCALL              0x500
 #define DIAG_KVM_BREAKPOINT             0x501
@@ -91,6 +111,14 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_IO                         0x40
 
+#define NR_LOCAL_IRQS 32
+/*
+ * Needs to be big enough to contain max_cpus emergency signals
+ * and in addition NR_LOCAL_IRQS interrupts
+ */
+#define VCPU_IRQ_BUF_SIZE (sizeof(struct kvm_s390_irq) * \
+                           (max_cpus + NR_LOCAL_IRQS))
+
 static CPUWatchpoint hw_watchpoint;
 /*
  * We don't use a list because this structure is also used to transmit the
@@ -105,39 +133,56 @@
 
 static int cap_sync_regs;
 static int cap_async_pf;
+static int cap_mem_op;
+static int cap_s390_irq;
+static int cap_ri;
 
 static void *legacy_s390_alloc(size_t size, uint64_t *align);
 
-static int kvm_s390_check_clear_cmma(KVMState *s)
+static int kvm_s390_query_mem_limit(KVMState *s, uint64_t *memory_limit)
 {
     struct kvm_device_attr attr = {
         .group = KVM_S390_VM_MEM_CTRL,
-        .attr = KVM_S390_VM_MEM_CLR_CMMA,
+        .attr = KVM_S390_VM_MEM_LIMIT_SIZE,
+        .addr = (uint64_t) memory_limit,
     };
 
-    return kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attr);
+    return kvm_vm_ioctl(s, KVM_GET_DEVICE_ATTR, &attr);
 }
 
-static int kvm_s390_check_enable_cmma(KVMState *s)
-{
-    struct kvm_device_attr attr = {
-        .group = KVM_S390_VM_MEM_CTRL,
-        .attr = KVM_S390_VM_MEM_ENABLE_CMMA,
-    };
-
-    return kvm_vm_ioctl(s, KVM_HAS_DEVICE_ATTR, &attr);
-}
-
-void kvm_s390_clear_cmma_callback(void *opaque)
+int kvm_s390_set_mem_limit(KVMState *s, uint64_t new_limit, uint64_t *hw_limit)
 {
     int rc;
-    KVMState *s = opaque;
+
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_MEM_CTRL,
+        .attr = KVM_S390_VM_MEM_LIMIT_SIZE,
+        .addr = (uint64_t) &new_limit,
+    };
+
+    if (!kvm_vm_check_mem_attr(s, KVM_S390_VM_MEM_LIMIT_SIZE)) {
+        return 0;
+    }
+
+    rc = kvm_s390_query_mem_limit(s, hw_limit);
+    if (rc) {
+        return rc;
+    } else if (*hw_limit < new_limit) {
+        return -E2BIG;
+    }
+
+    return kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+void kvm_s390_cmma_reset(void)
+{
+    int rc;
     struct kvm_device_attr attr = {
         .group = KVM_S390_VM_MEM_CTRL,
         .attr = KVM_S390_VM_MEM_CLR_CMMA,
     };
 
-    rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
+    rc = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
     trace_kvm_clear_cmma(rc);
 }
 
@@ -149,23 +194,72 @@
         .attr = KVM_S390_VM_MEM_ENABLE_CMMA,
     };
 
-    if (kvm_s390_check_enable_cmma(s) || kvm_s390_check_clear_cmma(s)) {
+    if (!kvm_vm_check_mem_attr(s, KVM_S390_VM_MEM_ENABLE_CMMA) ||
+        !kvm_vm_check_mem_attr(s, KVM_S390_VM_MEM_CLR_CMMA)) {
         return;
     }
 
     rc = kvm_vm_ioctl(s, KVM_SET_DEVICE_ATTR, &attr);
-    if (!rc) {
-        qemu_register_reset(kvm_s390_clear_cmma_callback, s);
-    }
     trace_kvm_enable_cmma(rc);
 }
 
-int kvm_arch_init(KVMState *s)
+static void kvm_s390_set_attr(uint64_t attr)
+{
+    struct kvm_device_attr attribute = {
+        .group = KVM_S390_VM_CRYPTO,
+        .attr  = attr,
+    };
+
+    int ret = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attribute);
+
+    if (ret) {
+        error_report("Failed to set crypto device attribute %lu: %s",
+                     attr, strerror(-ret));
+    }
+}
+
+static void kvm_s390_init_aes_kw(void)
+{
+    uint64_t attr = KVM_S390_VM_CRYPTO_DISABLE_AES_KW;
+
+    if (object_property_get_bool(OBJECT(qdev_get_machine()), "aes-key-wrap",
+                                 NULL)) {
+            attr = KVM_S390_VM_CRYPTO_ENABLE_AES_KW;
+    }
+
+    if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+            kvm_s390_set_attr(attr);
+    }
+}
+
+static void kvm_s390_init_dea_kw(void)
+{
+    uint64_t attr = KVM_S390_VM_CRYPTO_DISABLE_DEA_KW;
+
+    if (object_property_get_bool(OBJECT(qdev_get_machine()), "dea-key-wrap",
+                                 NULL)) {
+            attr = KVM_S390_VM_CRYPTO_ENABLE_DEA_KW;
+    }
+
+    if (kvm_vm_check_attr(kvm_state, KVM_S390_VM_CRYPTO, attr)) {
+            kvm_s390_set_attr(attr);
+    }
+}
+
+void kvm_s390_crypto_reset(void)
+{
+    kvm_s390_init_aes_kw();
+    kvm_s390_init_dea_kw();
+}
+
+int kvm_arch_init(MachineState *ms, KVMState *s)
 {
     cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
     cap_async_pf = kvm_check_extension(s, KVM_CAP_ASYNC_PF);
+    cap_mem_op = kvm_check_extension(s, KVM_CAP_S390_MEM_OP);
+    cap_s390_irq = kvm_check_extension(s, KVM_CAP_S390_INJECT_IRQ);
 
-    if (kvm_check_extension(s, KVM_CAP_VM_ATTRIBUTES)) {
+    if (!mem_path) {
         kvm_s390_enable_cmma(s);
     }
 
@@ -173,6 +267,16 @@
         || !kvm_check_extension(s, KVM_CAP_S390_COW)) {
         phys_mem_set_alloc(legacy_s390_alloc);
     }
+
+    kvm_vm_enable_cap(s, KVM_CAP_S390_USER_SIGP, 0);
+    kvm_vm_enable_cap(s, KVM_CAP_S390_VECTOR_REGISTERS, 0);
+    kvm_vm_enable_cap(s, KVM_CAP_S390_USER_STSI, 0);
+    if (ri_allowed()) {
+        if (kvm_vm_enable_cap(s, KVM_CAP_S390_RI, 0) == 0) {
+            cap_ri = 1;
+        }
+    }
+
     return 0;
 }
 
@@ -185,6 +289,7 @@
 {
     S390CPU *cpu = S390_CPU(cs);
     kvm_s390_set_cpu_state(cpu, cpu->env.cpu_state);
+    cpu->irqstate = g_malloc0(VCPU_IRQ_BUF_SIZE);
     return 0;
 }
 
@@ -198,17 +303,22 @@
      * Before this ioctl cpu_synchronize_state() is called in common kvm
      * code (kvm-all) */
     if (kvm_vcpu_ioctl(cs, KVM_S390_INITIAL_RESET, NULL)) {
-        error_report("Initial CPU reset failed on CPU %i\n", cs->cpu_index);
+        error_report("Initial CPU reset failed on CPU %i", cs->cpu_index);
     }
 }
 
+static int can_sync_regs(CPUState *cs, int regs)
+{
+    return cap_sync_regs && (cs->kvm_run->kvm_valid_regs & regs) == regs;
+}
+
 int kvm_arch_put_registers(CPUState *cs, int level)
 {
     S390CPU *cpu = S390_CPU(cs);
     CPUS390XState *env = &cpu->env;
     struct kvm_sregs sregs;
     struct kvm_regs regs;
-    struct kvm_fpu fpu;
+    struct kvm_fpu fpu = {};
     int r;
     int i;
 
@@ -216,7 +326,7 @@
     cs->kvm_run->psw_addr = env->psw.addr;
     cs->kvm_run->psw_mask = env->psw.mask;
 
-    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+    if (can_sync_regs(cs, KVM_SYNC_GPRS)) {
         for (i = 0; i < 16; i++) {
             cs->kvm_run->s.regs.gprs[i] = env->regs[i];
             cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
@@ -231,15 +341,30 @@
         }
     }
 
-    /* Floating point */
-    for (i = 0; i < 16; i++) {
-        fpu.fprs[i] = env->fregs[i].ll;
-    }
-    fpu.fpc = env->fpc;
+    if (can_sync_regs(cs, KVM_SYNC_VRS)) {
+        for (i = 0; i < 32; i++) {
+            cs->kvm_run->s.regs.vrs[i][0] = env->vregs[i][0].ll;
+            cs->kvm_run->s.regs.vrs[i][1] = env->vregs[i][1].ll;
+        }
+        cs->kvm_run->s.regs.fpc = env->fpc;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_VRS;
+    } else if (can_sync_regs(cs, KVM_SYNC_FPRS)) {
+        for (i = 0; i < 16; i++) {
+            cs->kvm_run->s.regs.fprs[i] = get_freg(env, i)->ll;
+        }
+        cs->kvm_run->s.regs.fpc = env->fpc;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_FPRS;
+    } else {
+        /* Floating point */
+        for (i = 0; i < 16; i++) {
+            fpu.fprs[i] = get_freg(env, i)->ll;
+        }
+        fpu.fpc = env->fpc;
 
-    r = kvm_vcpu_ioctl(cs, KVM_SET_FPU, &fpu);
-    if (r < 0) {
-        return r;
+        r = kvm_vcpu_ioctl(cs, KVM_SET_FPU, &fpu);
+        if (r < 0) {
+            return r;
+        }
     }
 
     /* Do we need to save more than that? */
@@ -247,18 +372,38 @@
         return 0;
     }
 
-    /*
-     * These ONE_REGS are not protected by a capability. As they are only
-     * necessary for migration we just trace a possible error, but don't
-     * return with an error return code.
-     */
-    kvm_set_one_reg(cs, KVM_REG_S390_CPU_TIMER, &env->cputm);
-    kvm_set_one_reg(cs, KVM_REG_S390_CLOCK_COMP, &env->ckc);
-    kvm_set_one_reg(cs, KVM_REG_S390_TODPR, &env->todpr);
-    kvm_set_one_reg(cs, KVM_REG_S390_GBEA, &env->gbea);
-    kvm_set_one_reg(cs, KVM_REG_S390_PP, &env->pp);
+    if (can_sync_regs(cs, KVM_SYNC_ARCH0)) {
+        cs->kvm_run->s.regs.cputm = env->cputm;
+        cs->kvm_run->s.regs.ckc = env->ckc;
+        cs->kvm_run->s.regs.todpr = env->todpr;
+        cs->kvm_run->s.regs.gbea = env->gbea;
+        cs->kvm_run->s.regs.pp = env->pp;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_ARCH0;
+    } else {
+        /*
+         * These ONE_REGS are not protected by a capability. As they are only
+         * necessary for migration we just trace a possible error, but don't
+         * return with an error return code.
+         */
+        kvm_set_one_reg(cs, KVM_REG_S390_CPU_TIMER, &env->cputm);
+        kvm_set_one_reg(cs, KVM_REG_S390_CLOCK_COMP, &env->ckc);
+        kvm_set_one_reg(cs, KVM_REG_S390_TODPR, &env->todpr);
+        kvm_set_one_reg(cs, KVM_REG_S390_GBEA, &env->gbea);
+        kvm_set_one_reg(cs, KVM_REG_S390_PP, &env->pp);
+    }
 
-    if (cap_async_pf) {
+    if (can_sync_regs(cs, KVM_SYNC_RICCB)) {
+        memcpy(cs->kvm_run->s.regs.riccb, env->riccb, 64);
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_RICCB;
+    }
+
+    /* pfault parameters */
+    if (can_sync_regs(cs, KVM_SYNC_PFAULT)) {
+        cs->kvm_run->s.regs.pft = env->pfault_token;
+        cs->kvm_run->s.regs.pfs = env->pfault_select;
+        cs->kvm_run->s.regs.pfc = env->pfault_compare;
+        cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_PFAULT;
+    } else if (cap_async_pf) {
         r = kvm_set_one_reg(cs, KVM_REG_S390_PFTOKEN, &env->pfault_token);
         if (r < 0) {
             return r;
@@ -273,9 +418,8 @@
         }
     }
 
-    if (cap_sync_regs &&
-        cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
-        cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+    /* access registers and control registers*/
+    if (can_sync_regs(cs, KVM_SYNC_ACRS | KVM_SYNC_CRS)) {
         for (i = 0; i < 16; i++) {
             cs->kvm_run->s.regs.acrs[i] = env->aregs[i];
             cs->kvm_run->s.regs.crs[i] = env->cregs[i];
@@ -294,7 +438,7 @@
     }
 
     /* Finally the prefix */
-    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+    if (can_sync_regs(cs, KVM_SYNC_PREFIX)) {
         cs->kvm_run->s.regs.prefix = env->psa;
         cs->kvm_run->kvm_dirty_regs |= KVM_SYNC_PREFIX;
     } else {
@@ -317,7 +461,7 @@
     env->psw.mask = cs->kvm_run->psw_mask;
 
     /* the GPRS */
-    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+    if (can_sync_regs(cs, KVM_SYNC_GPRS)) {
         for (i = 0; i < 16; i++) {
             env->regs[i] = cs->kvm_run->s.regs.gprs[i];
         }
@@ -332,9 +476,7 @@
     }
 
     /* The ACRS and CRS */
-    if (cap_sync_regs &&
-        cs->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
-        cs->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+    if (can_sync_regs(cs, KVM_SYNC_ACRS | KVM_SYNC_CRS)) {
         for (i = 0; i < 16; i++) {
             env->aregs[i] = cs->kvm_run->s.regs.acrs[i];
             env->cregs[i] = cs->kvm_run->s.regs.crs[i];
@@ -350,33 +492,63 @@
         }
     }
 
-    /* Floating point */
-    r = kvm_vcpu_ioctl(cs, KVM_GET_FPU, &fpu);
-    if (r < 0) {
-        return r;
+    /* Floating point and vector registers */
+    if (can_sync_regs(cs, KVM_SYNC_VRS)) {
+        for (i = 0; i < 32; i++) {
+            env->vregs[i][0].ll = cs->kvm_run->s.regs.vrs[i][0];
+            env->vregs[i][1].ll = cs->kvm_run->s.regs.vrs[i][1];
+        }
+        env->fpc = cs->kvm_run->s.regs.fpc;
+    } else if (can_sync_regs(cs, KVM_SYNC_FPRS)) {
+        for (i = 0; i < 16; i++) {
+            get_freg(env, i)->ll = cs->kvm_run->s.regs.fprs[i];
+        }
+        env->fpc = cs->kvm_run->s.regs.fpc;
+    } else {
+        r = kvm_vcpu_ioctl(cs, KVM_GET_FPU, &fpu);
+        if (r < 0) {
+            return r;
+        }
+        for (i = 0; i < 16; i++) {
+            get_freg(env, i)->ll = fpu.fprs[i];
+        }
+        env->fpc = fpu.fpc;
     }
-    for (i = 0; i < 16; i++) {
-        env->fregs[i].ll = fpu.fprs[i];
-    }
-    env->fpc = fpu.fpc;
 
     /* The prefix */
-    if (cap_sync_regs && cs->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+    if (can_sync_regs(cs, KVM_SYNC_PREFIX)) {
         env->psa = cs->kvm_run->s.regs.prefix;
     }
 
-    /*
-     * These ONE_REGS are not protected by a capability. As they are only
-     * necessary for migration we just trace a possible error, but don't
-     * return with an error return code.
-     */
-    kvm_get_one_reg(cs, KVM_REG_S390_CPU_TIMER, &env->cputm);
-    kvm_get_one_reg(cs, KVM_REG_S390_CLOCK_COMP, &env->ckc);
-    kvm_get_one_reg(cs, KVM_REG_S390_TODPR, &env->todpr);
-    kvm_get_one_reg(cs, KVM_REG_S390_GBEA, &env->gbea);
-    kvm_get_one_reg(cs, KVM_REG_S390_PP, &env->pp);
+    if (can_sync_regs(cs, KVM_SYNC_ARCH0)) {
+        env->cputm = cs->kvm_run->s.regs.cputm;
+        env->ckc = cs->kvm_run->s.regs.ckc;
+        env->todpr = cs->kvm_run->s.regs.todpr;
+        env->gbea = cs->kvm_run->s.regs.gbea;
+        env->pp = cs->kvm_run->s.regs.pp;
+    } else {
+        /*
+         * These ONE_REGS are not protected by a capability. As they are only
+         * necessary for migration we just trace a possible error, but don't
+         * return with an error return code.
+         */
+        kvm_get_one_reg(cs, KVM_REG_S390_CPU_TIMER, &env->cputm);
+        kvm_get_one_reg(cs, KVM_REG_S390_CLOCK_COMP, &env->ckc);
+        kvm_get_one_reg(cs, KVM_REG_S390_TODPR, &env->todpr);
+        kvm_get_one_reg(cs, KVM_REG_S390_GBEA, &env->gbea);
+        kvm_get_one_reg(cs, KVM_REG_S390_PP, &env->pp);
+    }
 
-    if (cap_async_pf) {
+    if (can_sync_regs(cs, KVM_SYNC_RICCB)) {
+        memcpy(env->riccb, cs->kvm_run->s.regs.riccb, 64);
+    }
+
+    /* pfault parameters */
+    if (can_sync_regs(cs, KVM_SYNC_PFAULT)) {
+        env->pfault_token = cs->kvm_run->s.regs.pft;
+        env->pfault_select = cs->kvm_run->s.regs.pfs;
+        env->pfault_compare = cs->kvm_run->s.regs.pfc;
+    } else if (cap_async_pf) {
         r = kvm_get_one_reg(cs, KVM_REG_S390_PFTOKEN, &env->pfault_token);
         if (r < 0) {
             return r;
@@ -394,6 +566,85 @@
     return 0;
 }
 
+int kvm_s390_get_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    int r;
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_TOD,
+        .attr = KVM_S390_VM_TOD_LOW,
+        .addr = (uint64_t)tod_low,
+    };
+
+    r = kvm_vm_ioctl(kvm_state, KVM_GET_DEVICE_ATTR, &attr);
+    if (r) {
+        return r;
+    }
+
+    attr.attr = KVM_S390_VM_TOD_HIGH;
+    attr.addr = (uint64_t)tod_high;
+    return kvm_vm_ioctl(kvm_state, KVM_GET_DEVICE_ATTR, &attr);
+}
+
+int kvm_s390_set_clock(uint8_t *tod_high, uint64_t *tod_low)
+{
+    int r;
+
+    struct kvm_device_attr attr = {
+        .group = KVM_S390_VM_TOD,
+        .attr = KVM_S390_VM_TOD_LOW,
+        .addr = (uint64_t)tod_low,
+    };
+
+    r = kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+    if (r) {
+        return r;
+    }
+
+    attr.attr = KVM_S390_VM_TOD_HIGH;
+    attr.addr = (uint64_t)tod_high;
+    return kvm_vm_ioctl(kvm_state, KVM_SET_DEVICE_ATTR, &attr);
+}
+
+/**
+ * kvm_s390_mem_op:
+ * @addr:      the logical start address in guest memory
+ * @ar:        the access register number
+ * @hostbuf:   buffer in host memory. NULL = do only checks w/o copying
+ * @len:       length that should be transferred
+ * @is_write:  true = write, false = read
+ * Returns:    0 on success, non-zero if an exception or error occurred
+ *
+ * Use KVM ioctl to read/write from/to guest memory. An access exception
+ * is injected into the vCPU in case of translation errors.
+ */
+int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
+                    int len, bool is_write)
+{
+    struct kvm_s390_mem_op mem_op = {
+        .gaddr = addr,
+        .flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION,
+        .size = len,
+        .op = is_write ? KVM_S390_MEMOP_LOGICAL_WRITE
+                       : KVM_S390_MEMOP_LOGICAL_READ,
+        .buf = (uint64_t)hostbuf,
+        .ar = ar,
+    };
+    int ret;
+
+    if (!cap_mem_op) {
+        return -ENOSYS;
+    }
+    if (!hostbuf) {
+        mem_op.flags |= KVM_S390_MEMOP_F_CHECK_ONLY;
+    }
+
+    ret = kvm_vcpu_ioctl(CPU(cpu), KVM_S390_MEM_OP, &mem_op);
+    if (ret < 0) {
+        error_printf("KVM_S390_MEM_OP failed: %s\n", strerror(-ret));
+    }
+    return ret;
+}
+
 /*
  * Legacy layout for s390:
  * Older S390 KVM requires the topmost vma of the RAM to be
@@ -571,8 +822,9 @@
 {
 }
 
-void kvm_arch_post_run(CPUState *cpu, struct kvm_run *run)
+MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run)
 {
+    return MEMTXATTRS_UNSPECIFIED;
 }
 
 int kvm_arch_process_async_events(CPUState *cs)
@@ -629,10 +881,9 @@
     return r;
 }
 
-void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
+static void inject_vcpu_irq_legacy(CPUState *cs, struct kvm_s390_irq *irq)
 {
     struct kvm_s390_interrupt kvmint = {};
-    CPUState *cs = CPU(cpu);
     int r;
 
     r = s390_kvm_irq_to_interrupt(irq, &kvmint);
@@ -648,6 +899,23 @@
     }
 }
 
+void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
+{
+    CPUState *cs = CPU(cpu);
+    int r;
+
+    if (cap_s390_irq) {
+        r = kvm_vcpu_ioctl(cs, KVM_S390_IRQ, irq);
+        if (!r) {
+            return;
+        }
+        error_report("KVM failed to inject interrupt %llx", irq->type);
+        exit(1);
+    }
+
+    inject_vcpu_irq_legacy(cs, irq);
+}
+
 static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
 {
     struct kvm_s390_interrupt kvmint = {};
@@ -683,17 +951,6 @@
     __kvm_s390_floating_interrupt(irq);
 }
 
-void kvm_s390_virtio_irq(int config_change, uint64_t token)
-{
-    struct kvm_s390_irq irq = {
-        .type = KVM_S390_INT_VIRTIO,
-        .u.ext.ext_params = config_change,
-        .u.ext.ext_params2 = token,
-    };
-
-    kvm_s390_floating_interrupt(&irq);
-}
-
 void kvm_s390_service_interrupt(uint32_t parm)
 {
     struct kvm_s390_irq irq = {
@@ -714,6 +971,18 @@
     kvm_s390_vcpu_interrupt(cpu, &irq);
 }
 
+void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code)
+{
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_PROGRAM_INT,
+        .u.pgm.code = code,
+        .u.pgm.trans_exc_code = te_code,
+        .u.pgm.exc_access_id = te_code & 3,
+    };
+
+    kvm_s390_vcpu_interrupt(cpu, &irq);
+}
+
 static int kvm_sclp_service_call(S390CPU *cpu, struct kvm_run *run,
                                  uint16_t ipbh0)
 {
@@ -809,11 +1078,135 @@
     return rc;
 }
 
+static uint64_t get_base_disp_rxy(S390CPU *cpu, struct kvm_run *run,
+                                  uint8_t *ar)
+{
+    CPUS390XState *env = &cpu->env;
+    uint32_t x2 = (run->s390_sieic.ipa & 0x000f);
+    uint32_t base2 = run->s390_sieic.ipb >> 28;
+    uint32_t disp2 = ((run->s390_sieic.ipb & 0x0fff0000) >> 16) +
+                     ((run->s390_sieic.ipb & 0xff00) << 4);
+
+    if (disp2 & 0x80000) {
+        disp2 += 0xfff00000;
+    }
+    if (ar) {
+        *ar = base2;
+    }
+
+    return (base2 ? env->regs[base2] : 0) +
+           (x2 ? env->regs[x2] : 0) + (long)(int)disp2;
+}
+
+static uint64_t get_base_disp_rsy(S390CPU *cpu, struct kvm_run *run,
+                                  uint8_t *ar)
+{
+    CPUS390XState *env = &cpu->env;
+    uint32_t base2 = run->s390_sieic.ipb >> 28;
+    uint32_t disp2 = ((run->s390_sieic.ipb & 0x0fff0000) >> 16) +
+                     ((run->s390_sieic.ipb & 0xff00) << 4);
+
+    if (disp2 & 0x80000) {
+        disp2 += 0xfff00000;
+    }
+    if (ar) {
+        *ar = base2;
+    }
+
+    return (base2 ? env->regs[base2] : 0) + (long)(int)disp2;
+}
+
+static int kvm_clp_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r2 = (run->s390_sieic.ipb & 0x000f0000) >> 16;
+
+    return clp_service_call(cpu, r2);
+}
+
+static int kvm_pcilg_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r1 = (run->s390_sieic.ipb & 0x00f00000) >> 20;
+    uint8_t r2 = (run->s390_sieic.ipb & 0x000f0000) >> 16;
+
+    return pcilg_service_call(cpu, r1, r2);
+}
+
+static int kvm_pcistg_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r1 = (run->s390_sieic.ipb & 0x00f00000) >> 20;
+    uint8_t r2 = (run->s390_sieic.ipb & 0x000f0000) >> 16;
+
+    return pcistg_service_call(cpu, r1, r2);
+}
+
+static int kvm_stpcifc_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    uint64_t fiba;
+    uint8_t ar;
+
+    cpu_synchronize_state(CPU(cpu));
+    fiba = get_base_disp_rxy(cpu, run, &ar);
+
+    return stpcifc_service_call(cpu, r1, fiba, ar);
+}
+
+static int kvm_sic_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    /* NOOP */
+    return 0;
+}
+
+static int kvm_rpcit_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r1 = (run->s390_sieic.ipb & 0x00f00000) >> 20;
+    uint8_t r2 = (run->s390_sieic.ipb & 0x000f0000) >> 16;
+
+    return rpcit_service_call(cpu, r1, r2);
+}
+
+static int kvm_pcistb_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    uint8_t r3 = run->s390_sieic.ipa & 0x000f;
+    uint64_t gaddr;
+    uint8_t ar;
+
+    cpu_synchronize_state(CPU(cpu));
+    gaddr = get_base_disp_rsy(cpu, run, &ar);
+
+    return pcistb_service_call(cpu, r1, r3, gaddr, ar);
+}
+
+static int kvm_mpcifc_service_call(S390CPU *cpu, struct kvm_run *run)
+{
+    uint8_t r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    uint64_t fiba;
+    uint8_t ar;
+
+    cpu_synchronize_state(CPU(cpu));
+    fiba = get_base_disp_rxy(cpu, run, &ar);
+
+    return mpcifc_service_call(cpu, r1, fiba, ar);
+}
+
 static int handle_b9(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
 {
     int r = 0;
 
     switch (ipa1) {
+    case PRIV_B9_CLP:
+        r = kvm_clp_service_call(cpu, run);
+        break;
+    case PRIV_B9_PCISTG:
+        r = kvm_pcistg_service_call(cpu, run);
+        break;
+    case PRIV_B9_PCILG:
+        r = kvm_pcilg_service_call(cpu, run);
+        break;
+    case PRIV_B9_RPCIT:
+        r = kvm_rpcit_service_call(cpu, run);
+        break;
     case PRIV_B9_EQBS:
         /* just inject exception */
         r = -1;
@@ -832,6 +1225,12 @@
     int r = 0;
 
     switch (ipbl) {
+    case PRIV_EB_PCISTB:
+        r = kvm_pcistb_service_call(cpu, run);
+        break;
+    case PRIV_EB_SIC:
+        r = kvm_sic_service_call(cpu, run);
+        break;
     case PRIV_EB_SQBS:
         /* just inject exception */
         r = -1;
@@ -845,6 +1244,26 @@
     return r;
 }
 
+static int handle_e3(S390CPU *cpu, struct kvm_run *run, uint8_t ipbl)
+{
+    int r = 0;
+
+    switch (ipbl) {
+    case PRIV_E3_MPCIFC:
+        r = kvm_mpcifc_service_call(cpu, run);
+        break;
+    case PRIV_E3_STPCIFC:
+        r = kvm_stpcifc_service_call(cpu, run);
+        break;
+    default:
+        r = -1;
+        DPRINTF("KVM: unhandled PRIV: 0xe3%x\n", ipbl);
+        break;
+    }
+
+    return r;
+}
+
 static int handle_hypercall(S390CPU *cpu, struct kvm_run *run)
 {
     CPUS390XState *env = &cpu->env;
@@ -860,12 +1279,26 @@
     return ret;
 }
 
+static void kvm_handle_diag_288(S390CPU *cpu, struct kvm_run *run)
+{
+    uint64_t r1, r3;
+    int rc;
+
+    cpu_synchronize_state(CPU(cpu));
+    r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
+    r3 = run->s390_sieic.ipa & 0x000f;
+    rc = handle_diag_288(&cpu->env, r1, r3);
+    if (rc) {
+        enter_pgmcheck(cpu, PGM_SPECIFICATION);
+    }
+}
+
 static void kvm_handle_diag_308(S390CPU *cpu, struct kvm_run *run)
 {
     uint64_t r1, r3;
 
     cpu_synchronize_state(CPU(cpu));
-    r1 = (run->s390_sieic.ipa & 0x00f0) >> 8;
+    r1 = (run->s390_sieic.ipa & 0x00f0) >> 4;
     r3 = run->s390_sieic.ipa & 0x000f;
     handle_diag_308(&cpu->env, r1, r3);
 }
@@ -897,8 +1330,11 @@
      * For any diagnose call we support, bits 48-63 of the resulting
      * address specify the function code; the remainder is ignored.
      */
-    func_code = decode_basedisp_rs(&cpu->env, ipb) & DIAG_KVM_CODE_MASK;
+    func_code = decode_basedisp_rs(&cpu->env, ipb, NULL) & DIAG_KVM_CODE_MASK;
     switch (func_code) {
+    case DIAG_TIMEREVENT:
+        kvm_handle_diag_288(cpu, run);
+        break;
     case DIAG_IPL:
         kvm_handle_diag_308(cpu, run);
         break;
@@ -910,59 +1346,380 @@
         break;
     default:
         DPRINTF("KVM: unknown DIAG: 0x%x\n", func_code);
-        r = -1;
+        enter_pgmcheck(cpu, PGM_SPECIFICATION);
         break;
     }
 
     return r;
 }
 
-static void sigp_cpu_start(void *arg)
-{
-    CPUState *cs = arg;
-    S390CPU *cpu = S390_CPU(cs);
+typedef struct SigpInfo {
+    S390CPU *cpu;
+    uint64_t param;
+    int cc;
+    uint64_t *status_reg;
+} SigpInfo;
 
-    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
-    DPRINTF("DONE: KVM cpu start: %p\n", &cpu->env);
+static void set_sigp_status(SigpInfo *si, uint64_t status)
+{
+    *si->status_reg &= 0xffffffff00000000ULL;
+    *si->status_reg |= status;
+    si->cc = SIGP_CC_STATUS_STORED;
 }
 
-static void sigp_cpu_restart(void *arg)
+static void sigp_start(void *arg)
 {
-    CPUState *cs = arg;
-    S390CPU *cpu = S390_CPU(cs);
+    SigpInfo *si = arg;
+
+    if (s390_cpu_get_state(si->cpu) != CPU_STATE_STOPPED) {
+        si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+        return;
+    }
+
+    s390_cpu_set_state(CPU_STATE_OPERATING, si->cpu);
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+static void sigp_stop(void *arg)
+{
+    SigpInfo *si = arg;
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_SIGP_STOP,
+    };
+
+    if (s390_cpu_get_state(si->cpu) != CPU_STATE_OPERATING) {
+        si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+        return;
+    }
+
+    /* disabled wait - sleeping in user space */
+    if (CPU(si->cpu)->halted) {
+        s390_cpu_set_state(CPU_STATE_STOPPED, si->cpu);
+    } else {
+        /* execute the stop function */
+        si->cpu->env.sigp_order = SIGP_STOP;
+        kvm_s390_vcpu_interrupt(si->cpu, &irq);
+    }
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+#define ADTL_SAVE_AREA_SIZE 1024
+static int kvm_s390_store_adtl_status(S390CPU *cpu, hwaddr addr)
+{
+    void *mem;
+    hwaddr len = ADTL_SAVE_AREA_SIZE;
+
+    mem = cpu_physical_memory_map(addr, &len, 1);
+    if (!mem) {
+        return -EFAULT;
+    }
+    if (len != ADTL_SAVE_AREA_SIZE) {
+        cpu_physical_memory_unmap(mem, len, 1, 0);
+        return -EFAULT;
+    }
+
+    memcpy(mem, &cpu->env.vregs, 512);
+
+    cpu_physical_memory_unmap(mem, len, 1, len);
+
+    return 0;
+}
+
+#define KVM_S390_STORE_STATUS_DEF_ADDR offsetof(LowCore, floating_pt_save_area)
+#define SAVE_AREA_SIZE 512
+static int kvm_s390_store_status(S390CPU *cpu, hwaddr addr, bool store_arch)
+{
+    static const uint8_t ar_id = 1;
+    uint64_t ckc = cpu->env.ckc >> 8;
+    void *mem;
+    int i;
+    hwaddr len = SAVE_AREA_SIZE;
+
+    mem = cpu_physical_memory_map(addr, &len, 1);
+    if (!mem) {
+        return -EFAULT;
+    }
+    if (len != SAVE_AREA_SIZE) {
+        cpu_physical_memory_unmap(mem, len, 1, 0);
+        return -EFAULT;
+    }
+
+    if (store_arch) {
+        cpu_physical_memory_write(offsetof(LowCore, ar_access_id), &ar_id, 1);
+    }
+    for (i = 0; i < 16; ++i) {
+        *((uint64_t *)mem + i) = get_freg(&cpu->env, i)->ll;
+    }
+    memcpy(mem + 128, &cpu->env.regs, 128);
+    memcpy(mem + 256, &cpu->env.psw, 16);
+    memcpy(mem + 280, &cpu->env.psa, 4);
+    memcpy(mem + 284, &cpu->env.fpc, 4);
+    memcpy(mem + 292, &cpu->env.todpr, 4);
+    memcpy(mem + 296, &cpu->env.cputm, 8);
+    memcpy(mem + 304, &ckc, 8);
+    memcpy(mem + 320, &cpu->env.aregs, 64);
+    memcpy(mem + 384, &cpu->env.cregs, 128);
+
+    cpu_physical_memory_unmap(mem, len, 1, len);
+
+    return 0;
+}
+
+static void sigp_stop_and_store_status(void *arg)
+{
+    SigpInfo *si = arg;
+    struct kvm_s390_irq irq = {
+        .type = KVM_S390_SIGP_STOP,
+    };
+
+    /* disabled wait - sleeping in user space */
+    if (s390_cpu_get_state(si->cpu) == CPU_STATE_OPERATING &&
+        CPU(si->cpu)->halted) {
+        s390_cpu_set_state(CPU_STATE_STOPPED, si->cpu);
+    }
+
+    switch (s390_cpu_get_state(si->cpu)) {
+    case CPU_STATE_OPERATING:
+        si->cpu->env.sigp_order = SIGP_STOP_STORE_STATUS;
+        kvm_s390_vcpu_interrupt(si->cpu, &irq);
+        /* store will be performed when handling the stop intercept */
+        break;
+    case CPU_STATE_STOPPED:
+        /* already stopped, just store the status */
+        cpu_synchronize_state(CPU(si->cpu));
+        kvm_s390_store_status(si->cpu, KVM_S390_STORE_STATUS_DEF_ADDR, true);
+        break;
+    }
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+static void sigp_store_status_at_address(void *arg)
+{
+    SigpInfo *si = arg;
+    uint32_t address = si->param & 0x7ffffe00u;
+
+    /* cpu has to be stopped */
+    if (s390_cpu_get_state(si->cpu) != CPU_STATE_STOPPED) {
+        set_sigp_status(si, SIGP_STAT_INCORRECT_STATE);
+        return;
+    }
+
+    cpu_synchronize_state(CPU(si->cpu));
+
+    if (kvm_s390_store_status(si->cpu, address, false)) {
+        set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
+        return;
+    }
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+static void sigp_store_adtl_status(void *arg)
+{
+    SigpInfo *si = arg;
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_S390_VECTOR_REGISTERS)) {
+        set_sigp_status(si, SIGP_STAT_INVALID_ORDER);
+        return;
+    }
+
+    /* cpu has to be stopped */
+    if (s390_cpu_get_state(si->cpu) != CPU_STATE_STOPPED) {
+        set_sigp_status(si, SIGP_STAT_INCORRECT_STATE);
+        return;
+    }
+
+    /* parameter must be aligned to 1024-byte boundary */
+    if (si->param & 0x3ff) {
+        set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
+        return;
+    }
+
+    cpu_synchronize_state(CPU(si->cpu));
+
+    if (kvm_s390_store_adtl_status(si->cpu, si->param)) {
+        set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
+        return;
+    }
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+static void sigp_restart(void *arg)
+{
+    SigpInfo *si = arg;
     struct kvm_s390_irq irq = {
         .type = KVM_S390_RESTART,
     };
 
-    kvm_s390_vcpu_interrupt(cpu, &irq);
-    s390_cpu_set_state(CPU_STATE_OPERATING, cpu);
+    switch (s390_cpu_get_state(si->cpu)) {
+    case CPU_STATE_STOPPED:
+        /* the restart irq has to be delivered prior to any other pending irq */
+        cpu_synchronize_state(CPU(si->cpu));
+        do_restart_interrupt(&si->cpu->env);
+        s390_cpu_set_state(CPU_STATE_OPERATING, si->cpu);
+        break;
+    case CPU_STATE_OPERATING:
+        kvm_s390_vcpu_interrupt(si->cpu, &irq);
+        break;
+    }
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
 }
 
 int kvm_s390_cpu_restart(S390CPU *cpu)
 {
-    run_on_cpu(CPU(cpu), sigp_cpu_restart, CPU(cpu));
+    SigpInfo si = {
+        .cpu = cpu,
+    };
+
+    run_on_cpu(CPU(cpu), sigp_restart, &si);
     DPRINTF("DONE: KVM cpu restart: %p\n", &cpu->env);
     return 0;
 }
 
 static void sigp_initial_cpu_reset(void *arg)
 {
-    CPUState *cpu = arg;
-    S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
+    SigpInfo *si = arg;
+    CPUState *cs = CPU(si->cpu);
+    S390CPUClass *scc = S390_CPU_GET_CLASS(si->cpu);
 
-    cpu_synchronize_state(cpu);
-    scc->initial_cpu_reset(cpu);
-    cpu_synchronize_post_reset(cpu);
+    cpu_synchronize_state(cs);
+    scc->initial_cpu_reset(cs);
+    cpu_synchronize_post_reset(cs);
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
 }
 
 static void sigp_cpu_reset(void *arg)
 {
-    CPUState *cpu = arg;
-    S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
+    SigpInfo *si = arg;
+    CPUState *cs = CPU(si->cpu);
+    S390CPUClass *scc = S390_CPU_GET_CLASS(si->cpu);
 
-    cpu_synchronize_state(cpu);
-    scc->cpu_reset(cpu);
-    cpu_synchronize_post_reset(cpu);
+    cpu_synchronize_state(cs);
+    scc->cpu_reset(cs);
+    cpu_synchronize_post_reset(cs);
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+static void sigp_set_prefix(void *arg)
+{
+    SigpInfo *si = arg;
+    uint32_t addr = si->param & 0x7fffe000u;
+
+    cpu_synchronize_state(CPU(si->cpu));
+
+    if (!address_space_access_valid(&address_space_memory, addr,
+                                    sizeof(struct LowCore), false)) {
+        set_sigp_status(si, SIGP_STAT_INVALID_PARAMETER);
+        return;
+    }
+
+    /* cpu has to be stopped */
+    if (s390_cpu_get_state(si->cpu) != CPU_STATE_STOPPED) {
+        set_sigp_status(si, SIGP_STAT_INCORRECT_STATE);
+        return;
+    }
+
+    si->cpu->env.psa = addr;
+    cpu_synchronize_post_init(CPU(si->cpu));
+    si->cc = SIGP_CC_ORDER_CODE_ACCEPTED;
+}
+
+static int handle_sigp_single_dst(S390CPU *dst_cpu, uint8_t order,
+                                  uint64_t param, uint64_t *status_reg)
+{
+    SigpInfo si = {
+        .cpu = dst_cpu,
+        .param = param,
+        .status_reg = status_reg,
+    };
+
+    /* cpu available? */
+    if (dst_cpu == NULL) {
+        return SIGP_CC_NOT_OPERATIONAL;
+    }
+
+    /* only resets can break pending orders */
+    if (dst_cpu->env.sigp_order != 0 &&
+        order != SIGP_CPU_RESET &&
+        order != SIGP_INITIAL_CPU_RESET) {
+        return SIGP_CC_BUSY;
+    }
+
+    switch (order) {
+    case SIGP_START:
+        run_on_cpu(CPU(dst_cpu), sigp_start, &si);
+        break;
+    case SIGP_STOP:
+        run_on_cpu(CPU(dst_cpu), sigp_stop, &si);
+        break;
+    case SIGP_RESTART:
+        run_on_cpu(CPU(dst_cpu), sigp_restart, &si);
+        break;
+    case SIGP_STOP_STORE_STATUS:
+        run_on_cpu(CPU(dst_cpu), sigp_stop_and_store_status, &si);
+        break;
+    case SIGP_STORE_STATUS_ADDR:
+        run_on_cpu(CPU(dst_cpu), sigp_store_status_at_address, &si);
+        break;
+    case SIGP_STORE_ADTL_STATUS:
+        run_on_cpu(CPU(dst_cpu), sigp_store_adtl_status, &si);
+        break;
+    case SIGP_SET_PREFIX:
+        run_on_cpu(CPU(dst_cpu), sigp_set_prefix, &si);
+        break;
+    case SIGP_INITIAL_CPU_RESET:
+        run_on_cpu(CPU(dst_cpu), sigp_initial_cpu_reset, &si);
+        break;
+    case SIGP_CPU_RESET:
+        run_on_cpu(CPU(dst_cpu), sigp_cpu_reset, &si);
+        break;
+    default:
+        DPRINTF("KVM: unknown SIGP: 0x%x\n", order);
+        set_sigp_status(&si, SIGP_STAT_INVALID_ORDER);
+    }
+
+    return si.cc;
+}
+
+static int sigp_set_architecture(S390CPU *cpu, uint32_t param,
+                                 uint64_t *status_reg)
+{
+    CPUState *cur_cs;
+    S390CPU *cur_cpu;
+
+    /* due to the BQL, we are the only active cpu */
+    CPU_FOREACH(cur_cs) {
+        cur_cpu = S390_CPU(cur_cs);
+        if (cur_cpu->env.sigp_order != 0) {
+            return SIGP_CC_BUSY;
+        }
+        cpu_synchronize_state(cur_cs);
+        /* all but the current one have to be stopped */
+        if (cur_cpu != cpu &&
+            s390_cpu_get_state(cur_cpu) != CPU_STATE_STOPPED) {
+            *status_reg &= 0xffffffff00000000ULL;
+            *status_reg |= SIGP_STAT_INCORRECT_STATE;
+            return SIGP_CC_STATUS_STORED;
+        }
+    }
+
+    switch (param & 0xff) {
+    case SIGP_MODE_ESA_S390:
+        /* not supported */
+        return SIGP_CC_NOT_OPERATIONAL;
+    case SIGP_MODE_Z_ARCH_TRANS_ALL_PSW:
+    case SIGP_MODE_Z_ARCH_TRANS_CUR_PSW:
+        CPU_FOREACH(cur_cs) {
+            cur_cpu = S390_CPU(cur_cs);
+            cur_cpu->env.pfault_token = -1UL;
+        }
+        break;
+    default:
+        *status_reg &= 0xffffffff00000000ULL;
+        *status_reg |= SIGP_STAT_INVALID_PARAMETER;
+        return SIGP_CC_STATUS_STORED;
+    }
+
+    return SIGP_CC_ORDER_CODE_ACCEPTED;
 }
 
 #define SIGP_ORDER_MASK 0x000000ff
@@ -970,57 +1727,41 @@
 static int handle_sigp(S390CPU *cpu, struct kvm_run *run, uint8_t ipa1)
 {
     CPUS390XState *env = &cpu->env;
-    uint8_t order_code;
-    uint16_t cpu_addr;
-    S390CPU *target_cpu;
-    uint64_t *statusreg = &env->regs[ipa1 >> 4];
-    int cc;
+    const uint8_t r1 = ipa1 >> 4;
+    const uint8_t r3 = ipa1 & 0x0f;
+    int ret;
+    uint8_t order;
+    uint64_t *status_reg;
+    uint64_t param;
+    S390CPU *dst_cpu = NULL;
 
     cpu_synchronize_state(CPU(cpu));
 
     /* get order code */
-    order_code = decode_basedisp_rs(env, run->s390_sieic.ipb) & SIGP_ORDER_MASK;
+    order = decode_basedisp_rs(env, run->s390_sieic.ipb, NULL)
+        & SIGP_ORDER_MASK;
+    status_reg = &env->regs[r1];
+    param = (r1 % 2) ? env->regs[r1] : env->regs[r1 + 1];
 
-    cpu_addr = env->regs[ipa1 & 0x0f];
-    target_cpu = s390_cpu_addr2state(cpu_addr);
-    if (target_cpu == NULL) {
-        cc = 3;    /* not operational */
-        goto out;
-    }
-
-    switch (order_code) {
-    case SIGP_START:
-        run_on_cpu(CPU(target_cpu), sigp_cpu_start, CPU(target_cpu));
-        cc = 0;
-        break;
-    case SIGP_RESTART:
-        run_on_cpu(CPU(target_cpu), sigp_cpu_restart, CPU(target_cpu));
-        cc = 0;
-        break;
+    switch (order) {
     case SIGP_SET_ARCH:
-        *statusreg &= 0xffffffff00000000UL;
-        *statusreg |= SIGP_STAT_INVALID_PARAMETER;
-        cc = 1;   /* status stored */
-        break;
-    case SIGP_INITIAL_CPU_RESET:
-        run_on_cpu(CPU(target_cpu), sigp_initial_cpu_reset, CPU(target_cpu));
-        cc = 0;
-        break;
-    case SIGP_CPU_RESET:
-        run_on_cpu(CPU(target_cpu), sigp_cpu_reset, CPU(target_cpu));
-        cc = 0;
+        ret = sigp_set_architecture(cpu, param, status_reg);
         break;
     default:
-        DPRINTF("KVM: unknown SIGP: 0x%x\n", order_code);
-        *statusreg &= 0xffffffff00000000UL;
-        *statusreg |= SIGP_STAT_INVALID_ORDER;
-        cc = 1;   /* status stored */
-        break;
+        /* all other sigp orders target a single vcpu */
+        dst_cpu = s390_cpu_addr2state(env->regs[r3]);
+        ret = handle_sigp_single_dst(dst_cpu, order, param, status_reg);
     }
 
-out:
-    setcc(cpu, cc);
-    return 0;
+    trace_kvm_sigp_finished(order, CPU(cpu)->cpu_index,
+                            dst_cpu ? CPU(dst_cpu)->cpu_index : -1, ret);
+
+    if (ret >= 0) {
+        setcc(cpu, ret);
+        return 0;
+    }
+
+    return ret;
 }
 
 static int handle_instruction(S390CPU *cpu, struct kvm_run *run)
@@ -1041,6 +1782,9 @@
     case IPA0_EB:
         r = handle_eb(cpu, run, run->s390_sieic.ipb & 0xff);
         break;
+    case IPA0_E3:
+        r = handle_e3(cpu, run, run->s390_sieic.ipb & 0xff);
+        break;
     case IPA0_DIAG:
         r = handle_diag(cpu, run, run->s390_sieic.ipb);
         break;
@@ -1063,13 +1807,6 @@
     return cs->kvm_run->psw_addr == 0xfffUL;
 }
 
-static void guest_panicked(void)
-{
-    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE,
-                                   &error_abort);
-    vm_stop(RUN_STATE_GUEST_PANICKED);
-}
-
 static void unmanageable_intercept(S390CPU *cpu, const char *str, int pswoffset)
 {
     CPUState *cs = CPU(cpu);
@@ -1078,7 +1815,7 @@
                  str, cs->cpu_index, ldq_phys(cs->as, cpu->env.psa + pswoffset),
                  ldq_phys(cs->as, cpu->env.psa + pswoffset + 8));
     s390_cpu_halt(cpu);
-    guest_panicked();
+    qemu_system_guest_panicked();
 }
 
 static int handle_intercept(S390CPU *cpu)
@@ -1111,7 +1848,7 @@
                 if (is_special_wait_psw(cs)) {
                     qemu_system_shutdown_request();
                 } else {
-                    guest_panicked();
+                    qemu_system_guest_panicked();
                 }
             }
             r = EXCP_HALTED;
@@ -1120,6 +1857,11 @@
             if (s390_cpu_set_state(CPU_STATE_STOPPED, cpu) == 0) {
                 qemu_system_shutdown_request();
             }
+            if (cpu->env.sigp_order == SIGP_STOP_STORE_STATUS) {
+                kvm_s390_store_status(cpu, KVM_S390_STORE_STATUS_DEF_ADDR,
+                                      true);
+            }
+            cpu->env.sigp_order = 0;
             r = EXCP_HALTED;
             break;
         case ICPT_SOFT_INTERCEPT:
@@ -1141,19 +1883,14 @@
 
 static int handle_tsch(S390CPU *cpu)
 {
-    CPUS390XState *env = &cpu->env;
     CPUState *cs = CPU(cpu);
     struct kvm_run *run = cs->kvm_run;
     int ret;
 
     cpu_synchronize_state(cs);
 
-    ret = ioinst_handle_tsch(env, env->regs[1], run->s390_tsch.ipb);
-    if (ret >= 0) {
-        /* Success; set condition code. */
-        setcc(cpu, ret);
-        ret = 0;
-    } else if (ret < -1) {
+    ret = ioinst_handle_tsch(cpu, cpu->env.regs[1], run->s390_tsch.ipb);
+    if (ret < 0) {
         /*
          * Failure.
          * If an I/O interrupt had been dequeued, we have to reinject it.
@@ -1169,6 +1906,72 @@
     return ret;
 }
 
+static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
+{
+    struct sysib_322 sysib;
+    int del;
+
+    if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
+        return;
+    }
+    /* Shift the stack of Extended Names to prepare for our own data */
+    memmove(&sysib.ext_names[1], &sysib.ext_names[0],
+            sizeof(sysib.ext_names[0]) * (sysib.count - 1));
+    /* First virt level, that doesn't provide Ext Names delimits stack. It is
+     * assumed it's not capable of managing Extended Names for lower levels.
+     */
+    for (del = 1; del < sysib.count; del++) {
+        if (!sysib.vm[del].ext_name_encoding || !sysib.ext_names[del][0]) {
+            break;
+        }
+    }
+    if (del < sysib.count) {
+        memset(sysib.ext_names[del], 0,
+               sizeof(sysib.ext_names[0]) * (sysib.count - del));
+    }
+    /* Insert short machine name in EBCDIC, padded with blanks */
+    if (qemu_name) {
+        memset(sysib.vm[0].name, 0x40, sizeof(sysib.vm[0].name));
+        ebcdic_put(sysib.vm[0].name, qemu_name, MIN(sizeof(sysib.vm[0].name),
+                                                    strlen(qemu_name)));
+    }
+    sysib.vm[0].ext_name_encoding = 2; /* 2 = UTF-8 */
+    memset(sysib.ext_names[0], 0, sizeof(sysib.ext_names[0]));
+    /* If hypervisor specifies zero Extended Name in STSI322 SYSIB, it's
+     * considered by s390 as not capable of providing any Extended Name.
+     * Therefore if no name was specified on qemu invocation, we go with the
+     * same "KVMguest" default, which KVM has filled into short name field.
+     */
+    if (qemu_name) {
+        strncpy((char *)sysib.ext_names[0], qemu_name,
+                sizeof(sysib.ext_names[0]));
+    } else {
+        strcpy((char *)sysib.ext_names[0], "KVMguest");
+    }
+    /* Insert UUID */
+    memcpy(sysib.vm[0].uuid, qemu_uuid, sizeof(sysib.vm[0].uuid));
+
+    s390_cpu_virt_mem_write(cpu, addr, ar, &sysib, sizeof(sysib));
+}
+
+static int handle_stsi(S390CPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
+
+    switch (run->s390_stsi.fc) {
+    case 3:
+        if (run->s390_stsi.sel1 != 2 || run->s390_stsi.sel2 != 2) {
+            return 0;
+        }
+        /* Only sysib 3.2.2 needs post-handling for now. */
+        insert_stsi_3_2_2(cpu, run->s390_stsi.addr, run->s390_stsi.ar);
+        return 0;
+    default:
+        return 0;
+    }
+}
+
 static int kvm_arch_handle_debug_exit(S390CPU *cpu)
 {
     CPUState *cs = CPU(cpu);
@@ -1208,16 +2011,21 @@
     S390CPU *cpu = S390_CPU(cs);
     int ret = 0;
 
+    qemu_mutex_lock_iothread();
+
     switch (run->exit_reason) {
         case KVM_EXIT_S390_SIEIC:
             ret = handle_intercept(cpu);
             break;
         case KVM_EXIT_S390_RESET:
-            qemu_system_reset_request();
+            s390_reipl_request();
             break;
         case KVM_EXIT_S390_TSCH:
             ret = handle_tsch(cpu);
             break;
+        case KVM_EXIT_S390_STSI:
+            ret = handle_stsi(cpu);
+            break;
         case KVM_EXIT_DEBUG:
             ret = kvm_arch_handle_debug_exit(cpu);
             break;
@@ -1225,6 +2033,7 @@
             fprintf(stderr, "Unknown KVM exit: %d\n", run->exit_reason);
             break;
     }
+    qemu_mutex_unlock_iothread();
 
     if (ret == 0) {
         ret = EXCP_INTERRUPT;
@@ -1261,18 +2070,37 @@
     if (io_int_word & IO_INT_WORD_AI) {
         irq.type = KVM_S390_INT_IO(1, 0, 0, 0);
     } else {
-        irq.type = ((subchannel_id & 0xff00) << 24) |
-            ((subchannel_id & 0x00060) << 22) | (subchannel_nr << 16);
+        irq.type = KVM_S390_INT_IO(0, (subchannel_id & 0xff00) >> 8,
+                                      (subchannel_id & 0x0006),
+                                      subchannel_nr);
     }
     kvm_s390_floating_interrupt(&irq);
 }
 
+static uint64_t build_channel_report_mcic(void)
+{
+    uint64_t mcic;
+
+    /* subclass: indicate channel report pending */
+    mcic = MCIC_SC_CP |
+    /* subclass modifiers: none */
+    /* storage errors: none */
+    /* validity bits: no damage */
+        MCIC_VB_WP | MCIC_VB_MS | MCIC_VB_PM | MCIC_VB_IA | MCIC_VB_FP |
+        MCIC_VB_GR | MCIC_VB_CR | MCIC_VB_ST | MCIC_VB_AR | MCIC_VB_PR |
+        MCIC_VB_FC | MCIC_VB_CT | MCIC_VB_CC;
+    if (kvm_check_extension(kvm_state, KVM_CAP_S390_VECTOR_REGISTERS)) {
+        mcic |= MCIC_VB_VR;
+    }
+    return mcic;
+}
+
 void kvm_s390_crw_mchk(void)
 {
     struct kvm_s390_irq irq = {
         .type = KVM_S390_MCHK,
         .u.mchk.cr14 = 1 << 28,
-        .u.mchk.mcic = 0x00400f1d40330000ULL,
+        .u.mchk.mcic = build_channel_report_mcic(),
     };
     kvm_s390_floating_interrupt(&irq);
 }
@@ -1294,7 +2122,6 @@
      * have to override the common code kvm_halt_in_kernel_allowed setting.
      */
     if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
-        kvm_irqfds_allowed = true;
         kvm_gsi_routing_allowed = true;
         kvm_halt_in_kernel_allowed = false;
     }
@@ -1325,6 +2152,11 @@
     return kvm_check_extension(s, KVM_CAP_NR_MEMSLOTS);
 }
 
+int kvm_s390_get_ri(void)
+{
+    return cap_ri;
+}
+
 int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state)
 {
     struct kvm_mp_state mp_state = {};
@@ -1362,3 +2194,91 @@
 
     return ret;
 }
+
+void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu)
+{
+    struct kvm_s390_irq_state irq_state;
+    CPUState *cs = CPU(cpu);
+    int32_t bytes;
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_S390_IRQ_STATE)) {
+        return;
+    }
+
+    irq_state.buf = (uint64_t) cpu->irqstate;
+    irq_state.len = VCPU_IRQ_BUF_SIZE;
+
+    bytes = kvm_vcpu_ioctl(cs, KVM_S390_GET_IRQ_STATE, &irq_state);
+    if (bytes < 0) {
+        cpu->irqstate_saved_size = 0;
+        error_report("Migration of interrupt state failed");
+        return;
+    }
+
+    cpu->irqstate_saved_size = bytes;
+}
+
+int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu)
+{
+    CPUState *cs = CPU(cpu);
+    struct kvm_s390_irq_state irq_state;
+    int r;
+
+    if (cpu->irqstate_saved_size == 0) {
+        return 0;
+    }
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_S390_IRQ_STATE)) {
+        return -ENOSYS;
+    }
+
+    irq_state.buf = (uint64_t) cpu->irqstate;
+    irq_state.len = cpu->irqstate_saved_size;
+
+    r = kvm_vcpu_ioctl(cs, KVM_S390_SET_IRQ_STATE, &irq_state);
+    if (r) {
+        error_report("Setting interrupt state failed %d", r);
+    }
+    return r;
+}
+
+int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
+                             uint64_t address, uint32_t data, PCIDevice *dev)
+{
+    S390PCIBusDevice *pbdev;
+    uint32_t idx = data >> ZPCI_MSI_VEC_BITS;
+    uint32_t vec = data & ZPCI_MSI_VEC_MASK;
+
+    pbdev = s390_pci_find_dev_by_idx(idx);
+    if (!pbdev) {
+        DPRINTF("add_msi_route no dev\n");
+        return -ENODEV;
+    }
+
+    pbdev->routes.adapter.ind_offset = vec;
+
+    route->type = KVM_IRQ_ROUTING_S390_ADAPTER;
+    route->flags = 0;
+    route->u.adapter.summary_addr = pbdev->routes.adapter.summary_addr;
+    route->u.adapter.ind_addr = pbdev->routes.adapter.ind_addr;
+    route->u.adapter.summary_offset = pbdev->routes.adapter.summary_offset;
+    route->u.adapter.ind_offset = pbdev->routes.adapter.ind_offset;
+    route->u.adapter.adapter_id = pbdev->routes.adapter.adapter_id;
+    return 0;
+}
+
+int kvm_arch_add_msi_route_post(struct kvm_irq_routing_entry *route,
+                                int vector, PCIDevice *dev)
+{
+    return 0;
+}
+
+int kvm_arch_release_virq_post(int virq)
+{
+    return 0;
+}
+
+int kvm_arch_msi_data_to_gsi(uint32_t data)
+{
+    abort();
+}
diff --git a/target-s390x/machine.c b/target-s390x/machine.c
index fbcb0d0..aa39e5d 100644
--- a/target-s390x/machine.c
+++ b/target-s390x/machine.c
@@ -14,6 +14,7 @@
  * or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/hw.h"
 #include "cpu.h"
 #include "sysemu/kvm.h"
@@ -28,38 +29,154 @@
      */
     if (kvm_enabled()) {
         kvm_s390_set_cpu_state(cpu, cpu->env.cpu_state);
+        return kvm_s390_vcpu_interrupt_post_load(cpu);
     }
 
     return 0;
 }
+static void cpu_pre_save(void *opaque)
+{
+    S390CPU *cpu = opaque;
+
+    if (kvm_enabled()) {
+        kvm_s390_vcpu_interrupt_pre_save(cpu);
+    }
+}
+
+static inline bool fpu_needed(void *opaque)
+{
+    /* This looks odd, but we might want to NOT transfer fprs in the future */
+    return true;
+}
+
+static const VMStateDescription vmstate_fpu = {
+    .name = "cpu/fpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = fpu_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(env.vregs[0][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[1][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[2][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[3][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[4][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[5][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[6][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[7][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[8][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[9][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[10][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[11][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[12][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[13][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[14][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[15][0].ll, S390CPU),
+        VMSTATE_UINT32(env.fpc, S390CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool vregs_needed(void *opaque)
+{
+#ifdef CONFIG_KVM
+    if (kvm_enabled()) {
+        return kvm_check_extension(kvm_state, KVM_CAP_S390_VECTOR_REGISTERS);
+    }
+#endif
+    return 0;
+}
+
+static const VMStateDescription vmstate_vregs = {
+    .name = "cpu/vregs",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vregs_needed,
+    .fields = (VMStateField[]) {
+        /* vregs[0][0] -> vregs[15][0] and fregs are overlays */
+        VMSTATE_UINT64(env.vregs[16][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[17][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[18][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[19][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[20][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[21][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[22][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[23][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[24][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[25][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[26][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[27][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[28][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[29][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[30][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[31][0].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[0][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[1][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[2][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[3][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[4][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[5][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[6][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[7][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[8][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[9][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[10][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[11][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[12][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[13][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[14][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[15][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[16][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[17][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[18][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[19][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[20][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[21][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[22][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[23][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[24][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[25][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[26][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[27][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[28][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[29][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[30][1].ll, S390CPU),
+        VMSTATE_UINT64(env.vregs[31][1].ll, S390CPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool riccb_needed(void *opaque)
+{
+#ifdef CONFIG_KVM
+    if (kvm_enabled()) {
+        return kvm_s390_get_ri();
+    }
+#endif
+    return 0;
+}
+
+const VMStateDescription vmstate_riccb = {
+    .name = "cpu/riccb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = riccb_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8_ARRAY(env.riccb, S390CPU, 64),
+        VMSTATE_END_OF_LIST()
+    }
+};
 
 const VMStateDescription vmstate_s390_cpu = {
     .name = "cpu",
     .post_load = cpu_post_load,
-    .version_id = 1,
-    .minimum_version_id = 1,
+    .pre_save = cpu_pre_save,
+    .version_id = 4,
+    .minimum_version_id = 3,
     .fields      = (VMStateField[]) {
-        VMSTATE_UINT64(env.fregs[0].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[1].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[2].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[3].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[4].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[5].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[6].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[7].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[8].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[9].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[10].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[11].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[12].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[13].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[14].ll, S390CPU),
-        VMSTATE_UINT64(env.fregs[15].ll, S390CPU),
         VMSTATE_UINT64_ARRAY(env.regs, S390CPU, 16),
         VMSTATE_UINT64(env.psw.mask, S390CPU),
         VMSTATE_UINT64(env.psw.addr, S390CPU),
         VMSTATE_UINT64(env.psa, S390CPU),
-        VMSTATE_UINT32(env.fpc, S390CPU),
         VMSTATE_UINT32(env.todpr, S390CPU),
         VMSTATE_UINT64(env.pfault_token, S390CPU),
         VMSTATE_UINT64(env.pfault_compare, S390CPU),
@@ -71,6 +188,16 @@
         VMSTATE_UINT32_ARRAY(env.aregs, S390CPU, 16),
         VMSTATE_UINT64_ARRAY(env.cregs, S390CPU, 16),
         VMSTATE_UINT8(env.cpu_state, S390CPU),
+        VMSTATE_UINT8(env.sigp_order, S390CPU),
+        VMSTATE_UINT32_V(irqstate_saved_size, S390CPU, 4),
+        VMSTATE_VBUFFER_UINT32(irqstate, S390CPU, 4, NULL, 0,
+                               irqstate_saved_size),
         VMSTATE_END_OF_LIST()
-     },
+    },
+    .subsections = (const VMStateDescription*[]) {
+        &vmstate_fpu,
+        &vmstate_vregs,
+        &vmstate_riccb,
+        NULL
+    },
 };
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
index 5a55de8..99bc5e2 100644
--- a/target-s390x/mem_helper.c
+++ b/target-s390x/mem_helper.c
@@ -18,10 +18,16 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
+#if !defined(CONFIG_USER_ONLY)
+#include "hw/s390x/storage-keys.h"
+#endif
+
 /*****************************************************************************/
 /* Softmmu support */
 #if !defined(CONFIG_USER_ONLY)
@@ -30,12 +36,12 @@
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
 /* XXX: fix it to restore all registers */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = s390_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = s390_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret != 0)) {
         if (likely(retaddr)) {
             /* now we have a real cpu fault */
@@ -54,63 +60,67 @@
 #define HELPER_LOG(x...)
 #endif
 
+/* Reduce the length so that addr + len doesn't cross a page boundary.  */
+static inline uint64_t adj_len_to_page(uint64_t len, uint64_t addr)
+{
 #ifndef CONFIG_USER_ONLY
-static void mvc_fast_memset(CPUS390XState *env, uint32_t l, uint64_t dest,
-                            uint8_t byte)
-{
-    S390CPU *cpu = s390_env_get_cpu(env);
-    hwaddr dest_phys;
-    hwaddr len = l;
-    void *dest_p;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    int flags;
-
-    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
-        cpu_stb_data(env, dest, byte);
-        cpu_abort(CPU(cpu), "should never reach here");
+    if ((addr & ~TARGET_PAGE_MASK) + len - 1 >= TARGET_PAGE_SIZE) {
+        return -addr & ~TARGET_PAGE_MASK;
     }
-    dest_phys |= dest & ~TARGET_PAGE_MASK;
-
-    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
-
-    memset(dest_p, byte, len);
-
-    cpu_physical_memory_unmap(dest_p, 1, len, len);
-}
-
-static void mvc_fast_memmove(CPUS390XState *env, uint32_t l, uint64_t dest,
-                             uint64_t src)
-{
-    S390CPU *cpu = s390_env_get_cpu(env);
-    hwaddr dest_phys;
-    hwaddr src_phys;
-    hwaddr len = l;
-    void *dest_p;
-    void *src_p;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    int flags;
-
-    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
-        cpu_stb_data(env, dest, 0);
-        cpu_abort(CPU(cpu), "should never reach here");
-    }
-    dest_phys |= dest & ~TARGET_PAGE_MASK;
-
-    if (mmu_translate(env, src, 0, asc, &src_phys, &flags)) {
-        cpu_ldub_data(env, src);
-        cpu_abort(CPU(cpu), "should never reach here");
-    }
-    src_phys |= src & ~TARGET_PAGE_MASK;
-
-    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
-    src_p = cpu_physical_memory_map(src_phys, &len, 0);
-
-    memmove(dest_p, src_p, len);
-
-    cpu_physical_memory_unmap(dest_p, 1, len, len);
-    cpu_physical_memory_unmap(src_p, 0, len, len);
-}
 #endif
+    return len;
+}
+
+static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte,
+                        uint32_t l)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+
+    while (l > 0) {
+        void *p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
+        if (p) {
+            /* Access to the whole page in write mode granted.  */
+            int l_adj = adj_len_to_page(l, dest);
+            memset(p, byte, l_adj);
+            dest += l_adj;
+            l -= l_adj;
+        } else {
+            /* We failed to get access to the whole page. The next write
+               access will likely fill the QEMU TLB for the next iteration.  */
+            cpu_stb_data(env, dest, byte);
+            dest++;
+            l--;
+        }
+    }
+}
+
+static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src,
+                         uint32_t l)
+{
+    int mmu_idx = cpu_mmu_index(env, false);
+
+    while (l > 0) {
+        void *src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, mmu_idx);
+        void *dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, mmu_idx);
+        if (src_p && dest_p) {
+            /* Access to both whole pages granted.  */
+            int l_adj = adj_len_to_page(l, src);
+            l_adj = adj_len_to_page(l_adj, dest);
+            memmove(dest_p, src_p, l_adj);
+            src += l_adj;
+            dest += l_adj;
+            l -= l_adj;
+        } else {
+            /* We failed to get access to one or both whole pages. The next
+               read or write access will likely fill the QEMU TLB for the
+               next iteration.  */
+            cpu_stb_data(env, dest, cpu_ldub_data(env, src));
+            src++;
+            dest++;
+            l--;
+        }
+    }
+}
 
 /* and on array */
 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
@@ -143,19 +153,11 @@
     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
                __func__, l, dest, src);
 
-#ifndef CONFIG_USER_ONLY
     /* xor with itself is the same as memset(0) */
-    if ((l > 32) && (src == dest) &&
-        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK)) {
-        mvc_fast_memset(env, l + 1, dest, 0);
-        return 0;
-    }
-#else
     if (src == dest) {
-        memset(g2h(dest), 0, l + 1);
+        fast_memset(env, dest, 0, l + 1);
         return 0;
     }
-#endif
 
     for (i = 0; i <= l; i++) {
         x = cpu_ldub_data(env, dest + i) ^ cpu_ldub_data(env, src + i);
@@ -191,44 +193,25 @@
 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
 {
     int i = 0;
-    int x = 0;
-    uint32_t l_64 = (l + 1) / 8;
 
     HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
                __func__, l, dest, src);
 
-#ifndef CONFIG_USER_ONLY
-    if ((l > 32) &&
-        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK) &&
-        (dest & TARGET_PAGE_MASK) == ((dest + l) & TARGET_PAGE_MASK)) {
-        if (dest == (src + 1)) {
-            mvc_fast_memset(env, l + 1, dest, cpu_ldub_data(env, src));
-            return;
-        } else if ((src & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
-            mvc_fast_memmove(env, l + 1, dest, src);
-            return;
-        }
-    }
-#else
+    /* mvc with source pointing to the byte after the destination is the
+       same as memset with the first source byte */
     if (dest == (src + 1)) {
-        memset(g2h(dest), cpu_ldub_data(env, src), l + 1);
-        return;
-    } else {
-        memmove(g2h(dest), g2h(src), l + 1);
+        fast_memset(env, dest, cpu_ldub_data(env, src), l + 1);
         return;
     }
-#endif
 
-    /* handle the parts that fit into 8-byte loads/stores */
-    if (dest != (src + 1)) {
-        for (i = 0; i < l_64; i++) {
-            cpu_stq_data(env, dest + x, cpu_ldq_data(env, src + x));
-            x += 8;
-        }
+    /* mvc and memmove do not behave the same when areas overlap! */
+    if ((dest < src) || (src + l < dest)) {
+        fast_memmove(env, dest, src, l + 1);
+        return;
     }
 
-    /* slow version crossing pages with byte accesses */
-    for (i = x; i <= l; i++) {
+    /* slow version with byte accesses which always work */
+    for (i = 0; i <= l; i++) {
         cpu_stb_data(env, dest + i, cpu_ldub_data(env, src + i));
     }
 }
@@ -395,11 +378,7 @@
 {
     /* XXX missing r0 handling */
     env->cc_op = 0;
-#ifdef CONFIG_USER_ONLY
-    memmove(g2h(r1), g2h(r2), TARGET_PAGE_SIZE);
-#else
-    mvc_fast_memmove(env, TARGET_PAGE_SIZE, r1, r2);
-#endif
+    fast_memmove(env, r1, r2, TARGET_PAGE_SIZE);
 }
 
 /* string copy (c is string terminator) */
@@ -490,10 +469,18 @@
             helper_mvc(env, l, get_address(env, 0, b1, d1),
                        get_address(env, 0, b2, d2));
             break;
+        case 0x400:
+            cc = helper_nc(env, l, get_address(env, 0, b1, d1),
+                            get_address(env, 0, b2, d2));
+            break;
         case 0x500:
             cc = helper_clc(env, l, get_address(env, 0, b1, d1),
                             get_address(env, 0, b2, d2));
             break;
+        case 0x600:
+            cc = helper_oc(env, l, get_address(env, 0, b1, d1),
+                            get_address(env, 0, b2, d2));
+            break;
         case 0x700:
             cc = helper_xc(env, l, get_address(env, 0, b1, d1),
                            get_address(env, 0, b2, d2));
@@ -502,6 +489,10 @@
             helper_tr(env, l, get_address(env, 0, b1, d1),
                       get_address(env, 0, b2, d2));
             break;
+        case 0xd00:
+            cc = helper_trt(env, l, get_address(env, 0, b1, d1),
+                            get_address(env, 0, b2, d2));
+            break;
         default:
             goto abort;
         }
@@ -566,7 +557,7 @@
     uint64_t dest = get_address_31fix(env, r1);
     uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
     uint64_t src = get_address_31fix(env, r2);
-    uint8_t pad = src >> 24;
+    uint8_t pad = env->regs[r2 + 1] >> 24;
     uint8_t v;
     uint32_t cc;
 
@@ -793,15 +784,81 @@
     }
 }
 
+uint64_t HELPER(tre)(CPUS390XState *env, uint64_t array,
+                     uint64_t len, uint64_t trans)
+{
+    uint8_t end = env->regs[0] & 0xff;
+    uint64_t l = len;
+    uint64_t i;
+
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        array &= 0x7fffffff;
+        l = (uint32_t)l;
+    }
+
+    /* Lest we fail to service interrupts in a timely manner, limit the
+       amount of work we're willing to do.  For now, let's cap at 8k.  */
+    if (l > 0x2000) {
+        l = 0x2000;
+        env->cc_op = 3;
+    } else {
+        env->cc_op = 0;
+    }
+
+    for (i = 0; i < l; i++) {
+        uint8_t byte, new_byte;
+
+        byte = cpu_ldub_data(env, array + i);
+
+        if (byte == end) {
+            env->cc_op = 1;
+            break;
+        }
+
+        new_byte = cpu_ldub_data(env, trans + byte);
+        cpu_stb_data(env, array + i, new_byte);
+    }
+
+    env->retxl = len - i;
+    return array + i;
+}
+
+uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
+                     uint64_t trans)
+{
+    uint32_t cc = 0;
+    int i;
+
+    for (i = 0; i <= len; i++) {
+        uint8_t byte = cpu_ldub_data(env, array + i);
+        uint8_t sbyte = cpu_ldub_data(env, trans + byte);
+
+        if (sbyte != 0) {
+            env->regs[1] = array + i;
+            env->regs[2] = (env->regs[2] & ~0xff) | sbyte;
+            cc = (i == len) ? 2 : 1;
+            break;
+        }
+    }
+
+    return cc;
+}
+
 #if !defined(CONFIG_USER_ONLY)
 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 {
     S390CPU *cpu = s390_env_get_cpu(env);
+    bool PERchanged = false;
     int i;
     uint64_t src = a2;
+    uint64_t val;
 
     for (i = r1;; i = (i + 1) % 16) {
-        env->cregs[i] = cpu_ldq_data(env, src);
+        val = cpu_ldq_data(env, src);
+        if (env->cregs[i] != val && i >= 9 && i <= 11) {
+            PERchanged = true;
+        }
+        env->cregs[i] = val;
         HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
                    i, src, env->cregs[i]);
         src += sizeof(uint64_t);
@@ -811,18 +868,27 @@
         }
     }
 
+    if (PERchanged && env->psw.mask & PSW_MASK_PER) {
+        s390_cpu_recompute_watchpoints(CPU(cpu));
+    }
+
     tlb_flush(CPU(cpu), 1);
 }
 
 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
 {
     S390CPU *cpu = s390_env_get_cpu(env);
+    bool PERchanged = false;
     int i;
     uint64_t src = a2;
+    uint32_t val;
 
     for (i = r1;; i = (i + 1) % 16) {
-        env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) |
-            cpu_ldl_data(env, src);
+        val = cpu_ldl_data(env, src);
+        if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
+            PERchanged = true;
+        }
+        env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) | val;
         src += sizeof(uint32_t);
 
         if (i == r3) {
@@ -830,6 +896,10 @@
         }
     }
 
+    if (PERchanged && env->psw.mask & PSW_MASK_PER) {
+        s390_cpu_recompute_watchpoints(CPU(cpu));
+    }
+
     tlb_flush(CPU(cpu), 1);
 }
 
@@ -873,40 +943,73 @@
 /* insert storage key extended */
 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     uint64_t addr = get_address(env, 0, 0, r2);
+    uint8_t key;
 
     if (addr > ram_size) {
         return 0;
     }
 
-    return env->storage_keys[addr / TARGET_PAGE_SIZE];
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    if (skeyclass->get_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
+    return key;
 }
 
 /* set storage key extended */
 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
 {
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
     uint64_t addr = get_address(env, 0, 0, r2);
+    uint8_t key;
 
     if (addr > ram_size) {
         return;
     }
 
-    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    key = (uint8_t) r1;
+    skeyclass->set_skeys(ss, addr / TARGET_PAGE_SIZE, 1, &key);
 }
 
 /* reset reference bit extended */
 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
 {
-    uint8_t re;
-    uint8_t key;
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
+    uint8_t re, key;
 
     if (r2 > ram_size) {
         return 0;
     }
 
-    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    if (skeyclass->get_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
+
     re = key & (SK_R | SK_C);
-    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
+    key &= ~SK_R;
+
+    if (skeyclass->set_skeys(ss, r2 / TARGET_PAGE_SIZE, 1, &key)) {
+        return 0;
+    }
 
     /*
      * cc
@@ -944,59 +1047,46 @@
     return cc;
 }
 
-static uint32_t mvc_asc(CPUS390XState *env, int64_t l, uint64_t a1,
-                        uint64_t mode1, uint64_t a2, uint64_t mode2)
+uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
 {
-    CPUState *cs = CPU(s390_env_get_cpu(env));
-    target_ulong src, dest;
-    int flags, cc = 0, i;
+    int cc = 0, i;
 
-    if (!l) {
-        return 0;
-    } else if (l > 256) {
+    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
+               __func__, l, a1, a2);
+
+    if (l > 256) {
         /* max 256 */
         l = 256;
         cc = 3;
     }
 
-    if (mmu_translate(env, a1 & TARGET_PAGE_MASK, 1, mode1, &dest, &flags)) {
-        cpu_loop_exit(CPU(s390_env_get_cpu(env)));
-    }
-    dest |= a1 & ~TARGET_PAGE_MASK;
-
-    if (mmu_translate(env, a2 & TARGET_PAGE_MASK, 0, mode2, &src, &flags)) {
-        cpu_loop_exit(CPU(s390_env_get_cpu(env)));
-    }
-    src |= a2 & ~TARGET_PAGE_MASK;
-
     /* XXX replace w/ memcpy */
     for (i = 0; i < l; i++) {
-        /* XXX be more clever */
-        if ((((dest + i) & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) ||
-            (((src + i) & TARGET_PAGE_MASK) != (src & TARGET_PAGE_MASK))) {
-            mvc_asc(env, l - i, a1 + i, mode1, a2 + i, mode2);
-            break;
-        }
-        stb_phys(cs->as, dest + i, ldub_phys(cs->as, src + i));
+        cpu_stb_secondary(env, a1 + i, cpu_ldub_primary(env, a2 + i));
     }
 
     return cc;
 }
 
-uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
-{
-    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
-               __func__, l, a1, a2);
-
-    return mvc_asc(env, l, a1, PSW_ASC_SECONDARY, a2, PSW_ASC_PRIMARY);
-}
-
 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
 {
+    int cc = 0, i;
+
     HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
                __func__, l, a1, a2);
 
-    return mvc_asc(env, l, a1, PSW_ASC_PRIMARY, a2, PSW_ASC_SECONDARY);
+    if (l > 256) {
+        /* max 256 */
+        l = 256;
+        cc = 3;
+    }
+
+    /* XXX replace w/ memcpy */
+    for (i = 0; i < l; i++) {
+        cpu_stb_primary(env, a1 + i, cpu_ldub_secondary(env, a2 + i));
+    }
+
+    return cc;
 }
 
 /* invalidate pte */
@@ -1034,12 +1124,50 @@
     tlb_flush(CPU(cpu), 1);
 }
 
+/* load using real address */
+uint64_t HELPER(lura)(CPUS390XState *env, uint64_t addr)
+{
+    CPUState *cs = CPU(s390_env_get_cpu(env));
+
+    return (uint32_t)ldl_phys(cs->as, get_address(env, 0, 0, addr));
+}
+
+uint64_t HELPER(lurag)(CPUS390XState *env, uint64_t addr)
+{
+    CPUState *cs = CPU(s390_env_get_cpu(env));
+
+    return ldq_phys(cs->as, get_address(env, 0, 0, addr));
+}
+
 /* store using real address */
 void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1)
 {
     CPUState *cs = CPU(s390_env_get_cpu(env));
 
-    stw_phys(cs->as, get_address(env, 0, 0, addr), (uint32_t)v1);
+    stl_phys(cs->as, get_address(env, 0, 0, addr), (uint32_t)v1);
+
+    if ((env->psw.mask & PSW_MASK_PER) &&
+        (env->cregs[9] & PER_CR9_EVENT_STORE) &&
+        (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
+        /* PSW is saved just before calling the helper.  */
+        env->per_address = env->psw.addr;
+        env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
+    }
+}
+
+void HELPER(sturg)(CPUS390XState *env, uint64_t addr, uint64_t v1)
+{
+    CPUState *cs = CPU(s390_env_get_cpu(env));
+
+    stq_phys(cs->as, get_address(env, 0, 0, addr), v1);
+
+    if ((env->psw.mask & PSW_MASK_PER) &&
+        (env->cregs[9] & PER_CR9_EVENT_STORE) &&
+        (env->cregs[9] & PER_CR9_EVENT_STORE_REAL)) {
+        /* PSW is saved just before calling the helper.  */
+        env->per_address = env->psw.addr;
+        env->per_perc_atmid = PER_CODE_EVENT_STORE_REAL | get_per_atmid(env);
+    }
 }
 
 /* load real address */
@@ -1058,7 +1186,7 @@
     }
 
     cs->exception_index = old_exc;
-    if (mmu_translate(env, addr, 0, asc, &ret, &flags)) {
+    if (mmu_translate(env, addr, 0, asc, &ret, &flags, true)) {
         cc = 3;
     }
     if (cs->exception_index == EXCP_PGM) {
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index ef9758a..86da194 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -18,22 +18,26 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/memory.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
-#include <string.h>
 #include "sysemu/kvm.h"
 #include "qemu/timer.h"
+#include "exec/address-spaces.h"
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
 #endif
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #if !defined(CONFIG_USER_ONLY)
+#include "hw/watchdog/wdt_diag288.h"
 #include "sysemu/cpus.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/ebcdic.h"
+#include "hw/s390x/ipl.h"
 #endif
 
 /* #define DEBUG_HELPER */
@@ -59,7 +63,7 @@
     /* Advance past the insn.  */
     t = cpu_ldub_code(env, env->psw.addr);
     env->int_pgm_ilen = t = get_ilen(t);
-    env->psw.addr += 2 * t;
+    env->psw.addr += t;
 
     cpu_loop_exit(cs);
 }
@@ -124,8 +128,9 @@
     CPU_FOREACH(t) {
         run_on_cpu(t, s390_do_cpu_full_reset, t);
     }
-    cmma_reset(cpu);
-    io_subsystem_reset();
+    s390_cmma_reset();
+    subsystem_reset();
+    s390_crypto_reset();
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
     resume_all_vcpus();
@@ -142,8 +147,8 @@
     CPU_FOREACH(t) {
         run_on_cpu(t, s390_do_cpu_reset, t);
     }
-    cmma_reset(cpu);
-    io_subsystem_reset();
+    s390_cmma_reset();
+    subsystem_reset();
     scc->initial_cpu_reset(CPU(cpu));
     scc->load_normal(CPU(cpu));
     cpu_synchronize_all_post_reset();
@@ -151,12 +156,43 @@
     return 0;
 }
 
+int handle_diag_288(CPUS390XState *env, uint64_t r1, uint64_t r3)
+{
+    uint64_t func = env->regs[r1];
+    uint64_t timeout = env->regs[r1 + 1];
+    uint64_t action = env->regs[r3];
+    Object *obj;
+    DIAG288State *diag288;
+    DIAG288Class *diag288_class;
+
+    if (r1 % 2 || action != 0) {
+        return -1;
+    }
+
+    /* Timeout must be more than 15 seconds except for timer deletion */
+    if (func != WDT_DIAG288_CANCEL && timeout < 15) {
+        return -1;
+    }
+
+    obj = object_resolve_path_type("", TYPE_WDT_DIAG288, NULL);
+    if (!obj) {
+        return -1;
+    }
+
+    diag288 = DIAG288(obj);
+    diag288_class = DIAG288_GET_CLASS(diag288);
+    return diag288_class->handle_timer(diag288, func, timeout);
+}
+
+#define DIAG_308_RC_OK              0x0001
 #define DIAG_308_RC_NO_CONF         0x0102
 #define DIAG_308_RC_INVALID         0x0402
+
 void handle_diag_308(CPUS390XState *env, uint64_t r1, uint64_t r3)
 {
     uint64_t addr =  env->regs[r1];
     uint64_t subcode = env->regs[r3];
+    IplParameterBlock *iplb;
 
     if (env->psw.mask & PSW_MASK_PSTATE) {
         program_interrupt(env, PGM_PRIVILEGED, ILEN_LATER_INC);
@@ -171,23 +207,68 @@
     switch (subcode) {
     case 0:
         modified_clear_reset(s390_env_get_cpu(env));
+        if (tcg_enabled()) {
+            cpu_loop_exit(CPU(s390_env_get_cpu(env)));
+        }
         break;
     case 1:
         load_normal_reset(s390_env_get_cpu(env));
+        if (tcg_enabled()) {
+            cpu_loop_exit(CPU(s390_env_get_cpu(env)));
+        }
+        break;
+    case 3:
+        s390_reipl_request();
+        if (tcg_enabled()) {
+            cpu_loop_exit(CPU(s390_env_get_cpu(env)));
+        }
         break;
     case 5:
         if ((r1 & 1) || (addr & 0x0fffULL)) {
             program_interrupt(env, PGM_SPECIFICATION, ILEN_LATER_INC);
             return;
         }
-        env->regs[r1+1] = DIAG_308_RC_INVALID;
+        if (!address_space_access_valid(&address_space_memory, addr,
+                                        sizeof(IplParameterBlock), false)) {
+            program_interrupt(env, PGM_ADDRESSING, ILEN_LATER_INC);
+            return;
+        }
+        iplb = g_malloc0(sizeof(IplParameterBlock));
+        cpu_physical_memory_read(addr, iplb, sizeof(iplb->len));
+        if (!iplb_valid_len(iplb)) {
+            env->regs[r1 + 1] = DIAG_308_RC_INVALID;
+            goto out;
+        }
+
+        cpu_physical_memory_read(addr, iplb, be32_to_cpu(iplb->len));
+
+        if (!iplb_valid_ccw(iplb) && !iplb_valid_fcp(iplb)) {
+            env->regs[r1 + 1] = DIAG_308_RC_INVALID;
+            goto out;
+        }
+
+        s390_ipl_update_diag308(iplb);
+        env->regs[r1 + 1] = DIAG_308_RC_OK;
+out:
+        g_free(iplb);
         return;
     case 6:
         if ((r1 & 1) || (addr & 0x0fffULL)) {
             program_interrupt(env, PGM_SPECIFICATION, ILEN_LATER_INC);
             return;
         }
-        env->regs[r1+1] = DIAG_308_RC_NO_CONF;
+        if (!address_space_access_valid(&address_space_memory, addr,
+                                        sizeof(IplParameterBlock), true)) {
+            program_interrupt(env, PGM_ADDRESSING, ILEN_LATER_INC);
+            return;
+        }
+        iplb = s390_ipl_get_iplb();
+        if (iplb) {
+            cpu_physical_memory_write(addr, iplb, be32_to_cpu(iplb->len));
+            env->regs[r1 + 1] = DIAG_308_RC_OK;
+        } else {
+            env->regs[r1 + 1] = DIAG_308_RC_NO_CONF;
+        }
         return;
     default:
         hw_error("Unhandled diag308 subcode %" PRIx64, subcode);
@@ -196,9 +277,7 @@
 }
 #endif
 
-/* DIAG */
-uint64_t HELPER(diag)(CPUS390XState *env, uint32_t num, uint64_t mem,
-                      uint64_t code)
+void HELPER(diag)(CPUS390XState *env, uint32_t r1, uint32_t r3, uint32_t num)
 {
     uint64_t r;
 
@@ -213,6 +292,7 @@
         break;
     case 0x308:
         /* ipl */
+        handle_diag_308(env, r1, r3);
         r = 0;
         break;
     default:
@@ -223,8 +303,6 @@
     if (r) {
         program_interrupt(env, PGM_OPERATION, ILEN_LATER_INC);
     }
-
-    return r;
 }
 
 /* Set Prefix */
@@ -234,12 +312,13 @@
     uint32_t prefix = a1 & 0x7fffe000;
 
     env->psa = prefix;
-    qemu_log("prefix: %#x\n", prefix);
+    HELPER_LOG("prefix: %#x\n", prefix);
     tlb_flush_page(cs, 0);
     tlb_flush_page(cs, TARGET_PAGE_SIZE);
 }
 
-static inline uint64_t clock_value(CPUS390XState *env)
+/* Store Clock */
+uint64_t HELPER(stck)(CPUS390XState *env)
 {
     uint64_t time;
 
@@ -249,12 +328,6 @@
     return time;
 }
 
-/* Store Clock */
-uint64_t HELPER(stck)(CPUS390XState *env)
-{
-    return clock_value(env);
-}
-
 /* Set Clock Comparator */
 void HELPER(sckc)(CPUS390XState *env, uint64_t time)
 {
@@ -262,19 +335,21 @@
         return;
     }
 
-    /* difference between now and then */
-    time -= clock_value(env);
-    /* nanoseconds */
-    time = (time * 125) >> 9;
+    env->ckc = time;
 
-    timer_mod(env->tod_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + time);
+    /* difference between origins */
+    time -= env->tod_offset;
+
+    /* nanoseconds */
+    time = tod2time(time);
+
+    timer_mod(env->tod_timer, env->tod_basetime + time);
 }
 
 /* Store Clock Comparator */
 uint64_t HELPER(stckc)(CPUS390XState *env)
 {
-    /* XXX implement */
-    return 0;
+    return env->ckc;
 }
 
 /* Set CPU Timer */
@@ -285,16 +360,17 @@
     }
 
     /* nanoseconds */
-    time = (time * 125) >> 9;
+    time = tod2time(time);
 
-    timer_mod(env->cpu_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + time);
+    env->cputm = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + time;
+
+    timer_mod(env->cpu_timer, env->cputm);
 }
 
 /* Store CPU Timer */
 uint64_t HELPER(stpt)(CPUS390XState *env)
 {
-    /* XXX implement */
-    return 0;
+    return time2tod(env->cputm - qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
 }
 
 /* Store System Information */
@@ -427,7 +503,7 @@
 uint32_t HELPER(sigp)(CPUS390XState *env, uint64_t order_code, uint32_t r1,
                       uint64_t cpu_addr)
 {
-    int cc = 0;
+    int cc = SIGP_CC_ORDER_CODE_ACCEPTED;
 
     HELPER_LOG("%s: %016" PRIx64 " %08x %016" PRIx64 "\n",
                __func__, order_code, r1, cpu_addr);
@@ -461,9 +537,117 @@
     default:
         /* unknown sigp */
         fprintf(stderr, "XXX unknown sigp: 0x%" PRIx64 "\n", order_code);
-        cc = 3;
+        cc = SIGP_CC_NOT_OPERATIONAL;
     }
 
     return cc;
 }
 #endif
+
+#ifndef CONFIG_USER_ONLY
+void HELPER(xsch)(CPUS390XState *env, uint64_t r1)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_xsch(cpu, r1);
+}
+
+void HELPER(csch)(CPUS390XState *env, uint64_t r1)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_csch(cpu, r1);
+}
+
+void HELPER(hsch)(CPUS390XState *env, uint64_t r1)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_hsch(cpu, r1);
+}
+
+void HELPER(msch)(CPUS390XState *env, uint64_t r1, uint64_t inst)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_msch(cpu, r1, inst >> 16);
+}
+
+void HELPER(rchp)(CPUS390XState *env, uint64_t r1)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_rchp(cpu, r1);
+}
+
+void HELPER(rsch)(CPUS390XState *env, uint64_t r1)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_rsch(cpu, r1);
+}
+
+void HELPER(ssch)(CPUS390XState *env, uint64_t r1, uint64_t inst)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_ssch(cpu, r1, inst >> 16);
+}
+
+void HELPER(stsch)(CPUS390XState *env, uint64_t r1, uint64_t inst)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_stsch(cpu, r1, inst >> 16);
+}
+
+void HELPER(tsch)(CPUS390XState *env, uint64_t r1, uint64_t inst)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_tsch(cpu, r1, inst >> 16);
+}
+
+void HELPER(chsc)(CPUS390XState *env, uint64_t inst)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+    ioinst_handle_chsc(cpu, inst >> 16);
+}
+#endif
+
+#ifndef CONFIG_USER_ONLY
+void HELPER(per_check_exception)(CPUS390XState *env)
+{
+    CPUState *cs = CPU(s390_env_get_cpu(env));
+
+    if (env->per_perc_atmid) {
+        env->int_pgm_code = PGM_PER;
+        env->int_pgm_ilen = get_ilen(cpu_ldub_code(env, env->per_address));
+
+        cs->exception_index = EXCP_PGM;
+        cpu_loop_exit(cs);
+    }
+}
+
+void HELPER(per_branch)(CPUS390XState *env, uint64_t from, uint64_t to)
+{
+    if ((env->cregs[9] & PER_CR9_EVENT_BRANCH)) {
+        if (!(env->cregs[9] & PER_CR9_CONTROL_BRANCH_ADDRESS)
+            || get_per_in_range(env, to)) {
+            env->per_address = from;
+            env->per_perc_atmid = PER_CODE_EVENT_BRANCH | get_per_atmid(env);
+        }
+    }
+}
+
+void HELPER(per_ifetch)(CPUS390XState *env, uint64_t addr)
+{
+    if ((env->cregs[9] & PER_CR9_EVENT_IFETCH) && get_per_in_range(env, addr)) {
+        env->per_address = addr;
+        env->per_perc_atmid = PER_CODE_EVENT_IFETCH | get_per_atmid(env);
+
+        /* If the instruction has to be nullified, trigger the
+           exception immediately. */
+        if (env->cregs[9] & PER_CR9_EVENT_NULLIFICATION) {
+            CPUState *cs = CPU(s390_env_get_cpu(env));
+
+            env->int_pgm_code = PGM_PER;
+            env->int_pgm_ilen = get_ilen(cpu_ldub_code(env, addr));
+
+            cs->exception_index = EXCP_PGM;
+            cpu_loop_exit(cs);
+        }
+    }
+}
+#endif
diff --git a/target-s390x/mmu_helper.c b/target-s390x/mmu_helper.c
new file mode 100644
index 0000000..b11a027
--- /dev/null
+++ b/target-s390x/mmu_helper.c
@@ -0,0 +1,499 @@
+/*
+ * S390x MMU related functions
+ *
+ * Copyright (c) 2011 Alexander Graf
+ * Copyright (c) 2015 Thomas Huth, IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/error-report.h"
+#include "exec/address-spaces.h"
+#include "cpu.h"
+#include "sysemu/kvm.h"
+#include "trace.h"
+#include "hw/s390x/storage-keys.h"
+
+/* #define DEBUG_S390 */
+/* #define DEBUG_S390_PTE */
+/* #define DEBUG_S390_STDOUT */
+
+#ifdef DEBUG_S390
+#ifdef DEBUG_S390_STDOUT
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, fmt, ## __VA_ARGS__); \
+         if (qemu_log_separate()) qemu_log(fmt, ##__VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { qemu_log(fmt, ## __VA_ARGS__); } while (0)
+#endif
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+#ifdef DEBUG_S390_PTE
+#define PTE_DPRINTF DPRINTF
+#else
+#define PTE_DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+/* Fetch/store bits in the translation exception code: */
+#define FS_READ  0x800
+#define FS_WRITE 0x400
+
+static void trigger_access_exception(CPUS390XState *env, uint32_t type,
+                                     uint32_t ilen, uint64_t tec)
+{
+    S390CPU *cpu = s390_env_get_cpu(env);
+
+    if (kvm_enabled()) {
+        kvm_s390_access_exception(cpu, type, tec);
+    } else {
+        CPUState *cs = CPU(cpu);
+        stq_phys(cs->as, env->psa + offsetof(LowCore, trans_exc_code), tec);
+        trigger_pgm_exception(env, type, ilen);
+    }
+}
+
+static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr,
+                               uint64_t asc, int rw, bool exc)
+{
+    uint64_t tec;
+
+    tec = vaddr | (rw == MMU_DATA_STORE ? FS_WRITE : FS_READ) | 4 | asc >> 46;
+
+    DPRINTF("%s: trans_exc_code=%016" PRIx64 "\n", __func__, tec);
+
+    if (!exc) {
+        return;
+    }
+
+    trigger_access_exception(env, PGM_PROTECTION, ILEN_LATER_INC, tec);
+}
+
+static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr,
+                               uint32_t type, uint64_t asc, int rw, bool exc)
+{
+    int ilen = ILEN_LATER;
+    uint64_t tec;
+
+    tec = vaddr | (rw == MMU_DATA_STORE ? FS_WRITE : FS_READ) | asc >> 46;
+
+    DPRINTF("%s: trans_exc_code=%016" PRIx64 "\n", __func__, tec);
+
+    if (!exc) {
+        return;
+    }
+
+    /* Code accesses have an undefined ilc.  */
+    if (rw == MMU_INST_FETCH) {
+        ilen = 2;
+    }
+
+    trigger_access_exception(env, type, ilen, tec);
+}
+
+/**
+ * Translate real address to absolute (= physical)
+ * address by taking care of the prefix mapping.
+ */
+static target_ulong mmu_real2abs(CPUS390XState *env, target_ulong raddr)
+{
+    if (raddr < 0x2000) {
+        return raddr + env->psa;    /* Map the lowcore. */
+    } else if (raddr >= env->psa && raddr < env->psa + 0x2000) {
+        return raddr - env->psa;    /* Map the 0 page. */
+    }
+    return raddr;
+}
+
+/* Decode page table entry (normal 4KB page) */
+static int mmu_translate_pte(CPUS390XState *env, target_ulong vaddr,
+                             uint64_t asc, uint64_t pt_entry,
+                             target_ulong *raddr, int *flags, int rw, bool exc)
+{
+    if (pt_entry & _PAGE_INVALID) {
+        DPRINTF("%s: PTE=0x%" PRIx64 " invalid\n", __func__, pt_entry);
+        trigger_page_fault(env, vaddr, PGM_PAGE_TRANS, asc, rw, exc);
+        return -1;
+    }
+    if (pt_entry & _PAGE_RES0) {
+        trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw, exc);
+        return -1;
+    }
+    if (pt_entry & _PAGE_RO) {
+        *flags &= ~PAGE_WRITE;
+    }
+
+    *raddr = pt_entry & _ASCE_ORIGIN;
+
+    PTE_DPRINTF("%s: PTE=0x%" PRIx64 "\n", __func__, pt_entry);
+
+    return 0;
+}
+
+#define VADDR_PX    0xff000         /* Page index bits */
+
+/* Decode segment table entry */
+static int mmu_translate_segment(CPUS390XState *env, target_ulong vaddr,
+                                 uint64_t asc, uint64_t st_entry,
+                                 target_ulong *raddr, int *flags, int rw,
+                                 bool exc)
+{
+    CPUState *cs = CPU(s390_env_get_cpu(env));
+    uint64_t origin, offs, pt_entry;
+
+    if (st_entry & _SEGMENT_ENTRY_RO) {
+        *flags &= ~PAGE_WRITE;
+    }
+
+    if ((st_entry & _SEGMENT_ENTRY_FC) && (env->cregs[0] & CR0_EDAT)) {
+        /* Decode EDAT1 segment frame absolute address (1MB page) */
+        *raddr = (st_entry & 0xfffffffffff00000ULL) | (vaddr & 0xfffff);
+        PTE_DPRINTF("%s: SEG=0x%" PRIx64 "\n", __func__, st_entry);
+        return 0;
+    }
+
+    /* Look up 4KB page entry */
+    origin = st_entry & _SEGMENT_ENTRY_ORIGIN;
+    offs  = (vaddr & VADDR_PX) >> 9;
+    pt_entry = ldq_phys(cs->as, origin + offs);
+    PTE_DPRINTF("%s: 0x%" PRIx64 " + 0x%" PRIx64 " => 0x%016" PRIx64 "\n",
+                __func__, origin, offs, pt_entry);
+    return mmu_translate_pte(env, vaddr, asc, pt_entry, raddr, flags, rw, exc);
+}
+
+/* Decode region table entries */
+static int mmu_translate_region(CPUS390XState *env, target_ulong vaddr,
+                                uint64_t asc, uint64_t entry, int level,
+                                target_ulong *raddr, int *flags, int rw,
+                                bool exc)
+{
+    CPUState *cs = CPU(s390_env_get_cpu(env));
+    uint64_t origin, offs, new_entry;
+    const int pchks[4] = {
+        PGM_SEGMENT_TRANS, PGM_REG_THIRD_TRANS,
+        PGM_REG_SEC_TRANS, PGM_REG_FIRST_TRANS
+    };
+
+    PTE_DPRINTF("%s: 0x%" PRIx64 "\n", __func__, entry);
+
+    origin = entry & _REGION_ENTRY_ORIGIN;
+    offs = (vaddr >> (17 + 11 * level / 4)) & 0x3ff8;
+
+    new_entry = ldq_phys(cs->as, origin + offs);
+    PTE_DPRINTF("%s: 0x%" PRIx64 " + 0x%" PRIx64 " => 0x%016" PRIx64 "\n",
+                __func__, origin, offs, new_entry);
+
+    if ((new_entry & _REGION_ENTRY_INV) != 0) {
+        DPRINTF("%s: invalid region\n", __func__);
+        trigger_page_fault(env, vaddr, pchks[level / 4], asc, rw, exc);
+        return -1;
+    }
+
+    if ((new_entry & _REGION_ENTRY_TYPE_MASK) != level) {
+        trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw, exc);
+        return -1;
+    }
+
+    if (level == _ASCE_TYPE_SEGMENT) {
+        return mmu_translate_segment(env, vaddr, asc, new_entry, raddr, flags,
+                                     rw, exc);
+    }
+
+    /* Check region table offset and length */
+    offs = (vaddr >> (28 + 11 * (level - 4) / 4)) & 3;
+    if (offs < ((new_entry & _REGION_ENTRY_TF) >> 6)
+        || offs > (new_entry & _REGION_ENTRY_LENGTH)) {
+        DPRINTF("%s: invalid offset or len (%lx)\n", __func__, new_entry);
+        trigger_page_fault(env, vaddr, pchks[level / 4 - 1], asc, rw, exc);
+        return -1;
+    }
+
+    if ((env->cregs[0] & CR0_EDAT) && (new_entry & _REGION_ENTRY_RO)) {
+        *flags &= ~PAGE_WRITE;
+    }
+
+    /* yet another region */
+    return mmu_translate_region(env, vaddr, asc, new_entry, level - 4,
+                                raddr, flags, rw, exc);
+}
+
+static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
+                              uint64_t asc, uint64_t asce, target_ulong *raddr,
+                              int *flags, int rw, bool exc)
+{
+    int level;
+    int r;
+
+    if (asce & _ASCE_REAL_SPACE) {
+        /* direct mapping */
+        *raddr = vaddr;
+        return 0;
+    }
+
+    level = asce & _ASCE_TYPE_MASK;
+    switch (level) {
+    case _ASCE_TYPE_REGION1:
+        if ((vaddr >> 62) > (asce & _ASCE_TABLE_LENGTH)) {
+            trigger_page_fault(env, vaddr, PGM_REG_FIRST_TRANS, asc, rw, exc);
+            return -1;
+        }
+        break;
+    case _ASCE_TYPE_REGION2:
+        if (vaddr & 0xffe0000000000000ULL) {
+            DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
+                    " 0xffe0000000000000ULL\n", __func__, vaddr);
+            trigger_page_fault(env, vaddr, PGM_ASCE_TYPE, asc, rw, exc);
+            return -1;
+        }
+        if ((vaddr >> 51 & 3) > (asce & _ASCE_TABLE_LENGTH)) {
+            trigger_page_fault(env, vaddr, PGM_REG_SEC_TRANS, asc, rw, exc);
+            return -1;
+        }
+        break;
+    case _ASCE_TYPE_REGION3:
+        if (vaddr & 0xfffffc0000000000ULL) {
+            DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
+                    " 0xfffffc0000000000ULL\n", __func__, vaddr);
+            trigger_page_fault(env, vaddr, PGM_ASCE_TYPE, asc, rw, exc);
+            return -1;
+        }
+        if ((vaddr >> 40 & 3) > (asce & _ASCE_TABLE_LENGTH)) {
+            trigger_page_fault(env, vaddr, PGM_REG_THIRD_TRANS, asc, rw, exc);
+            return -1;
+        }
+        break;
+    case _ASCE_TYPE_SEGMENT:
+        if (vaddr & 0xffffffff80000000ULL) {
+            DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
+                    " 0xffffffff80000000ULL\n", __func__, vaddr);
+            trigger_page_fault(env, vaddr, PGM_ASCE_TYPE, asc, rw, exc);
+            return -1;
+        }
+        if ((vaddr >> 29 & 3) > (asce & _ASCE_TABLE_LENGTH)) {
+            trigger_page_fault(env, vaddr, PGM_SEGMENT_TRANS, asc, rw, exc);
+            return -1;
+        }
+        break;
+    }
+
+    r = mmu_translate_region(env, vaddr, asc, asce, level, raddr, flags, rw,
+                             exc);
+    if (rw == MMU_DATA_STORE && !(*flags & PAGE_WRITE)) {
+        trigger_prot_fault(env, vaddr, asc, rw, exc);
+        return -1;
+    }
+
+    return r;
+}
+
+/**
+ * Translate a virtual (logical) address into a physical (absolute) address.
+ * @param vaddr  the virtual address
+ * @param rw     0 = read, 1 = write, 2 = code fetch
+ * @param asc    address space control (one of the PSW_ASC_* modes)
+ * @param raddr  the translated address is stored to this pointer
+ * @param flags  the PAGE_READ/WRITE/EXEC flags are stored to this pointer
+ * @param exc    true = inject a program check if a fault occurred
+ * @return       0 if the translation was successful, -1 if a fault occurred
+ */
+int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
+                  target_ulong *raddr, int *flags, bool exc)
+{
+    static S390SKeysState *ss;
+    static S390SKeysClass *skeyclass;
+    int r = -1;
+    uint8_t key;
+
+    if (unlikely(!ss)) {
+        ss = s390_get_skeys_device();
+        skeyclass = S390_SKEYS_GET_CLASS(ss);
+    }
+
+    *flags = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
+    vaddr &= TARGET_PAGE_MASK;
+
+    if (!(env->psw.mask & PSW_MASK_DAT)) {
+        *raddr = vaddr;
+        r = 0;
+        goto out;
+    }
+
+    switch (asc) {
+    case PSW_ASC_PRIMARY:
+        PTE_DPRINTF("%s: asc=primary\n", __func__);
+        r = mmu_translate_asce(env, vaddr, asc, env->cregs[1], raddr, flags,
+                               rw, exc);
+        break;
+    case PSW_ASC_HOME:
+        PTE_DPRINTF("%s: asc=home\n", __func__);
+        r = mmu_translate_asce(env, vaddr, asc, env->cregs[13], raddr, flags,
+                               rw, exc);
+        break;
+    case PSW_ASC_SECONDARY:
+        PTE_DPRINTF("%s: asc=secondary\n", __func__);
+        /*
+         * Instruction: Primary
+         * Data: Secondary
+         */
+        if (rw == MMU_INST_FETCH) {
+            r = mmu_translate_asce(env, vaddr, PSW_ASC_PRIMARY, env->cregs[1],
+                                   raddr, flags, rw, exc);
+            *flags &= ~(PAGE_READ | PAGE_WRITE);
+        } else {
+            r = mmu_translate_asce(env, vaddr, PSW_ASC_SECONDARY, env->cregs[7],
+                                   raddr, flags, rw, exc);
+            *flags &= ~(PAGE_EXEC);
+        }
+        break;
+    case PSW_ASC_ACCREG:
+    default:
+        hw_error("guest switched to unknown asc mode\n");
+        break;
+    }
+
+ out:
+    /* Convert real address -> absolute address */
+    *raddr = mmu_real2abs(env, *raddr);
+
+    if (r == 0 && *raddr < ram_size) {
+        if (skeyclass->get_skeys(ss, *raddr / TARGET_PAGE_SIZE, 1, &key)) {
+            trace_get_skeys_nonzero(r);
+            return 0;
+        }
+
+        if (*flags & PAGE_READ) {
+            key |= SK_R;
+        }
+
+        if (*flags & PAGE_WRITE) {
+            key |= SK_C;
+        }
+
+        if (skeyclass->set_skeys(ss, *raddr / TARGET_PAGE_SIZE, 1, &key)) {
+            trace_set_skeys_nonzero(r);
+            return 0;
+        }
+    }
+
+    return r;
+}
+
+/**
+ * lowprot_enabled: Check whether low-address protection is enabled
+ */
+static bool lowprot_enabled(const CPUS390XState *env)
+{
+    if (!(env->cregs[0] & CR0_LOWPROT)) {
+        return false;
+    }
+    if (!(env->psw.mask & PSW_MASK_DAT)) {
+        return true;
+    }
+
+    /* Check the private-space control bit */
+    switch (env->psw.mask & PSW_MASK_ASC) {
+    case PSW_ASC_PRIMARY:
+        return !(env->cregs[1] & _ASCE_PRIVATE_SPACE);
+    case PSW_ASC_SECONDARY:
+        return !(env->cregs[7] & _ASCE_PRIVATE_SPACE);
+    case PSW_ASC_HOME:
+        return !(env->cregs[13] & _ASCE_PRIVATE_SPACE);
+    default:
+        /* We don't support access register mode */
+        error_report("unsupported addressing mode");
+        exit(1);
+    }
+}
+
+/**
+ * translate_pages: Translate a set of consecutive logical page addresses
+ * to absolute addresses
+ */
+static int translate_pages(S390CPU *cpu, vaddr addr, int nr_pages,
+                           target_ulong *pages, bool is_write)
+{
+    bool lowprot = is_write && lowprot_enabled(&cpu->env);
+    uint64_t asc = cpu->env.psw.mask & PSW_MASK_ASC;
+    CPUS390XState *env = &cpu->env;
+    int ret, i, pflags;
+
+    for (i = 0; i < nr_pages; i++) {
+        /* Low-address protection? */
+        if (lowprot && (addr < 512 || (addr >= 4096 && addr < 4096 + 512))) {
+            trigger_access_exception(env, PGM_PROTECTION, ILEN_LATER_INC, 0);
+            return -EACCES;
+        }
+        ret = mmu_translate(env, addr, is_write, asc, &pages[i], &pflags, true);
+        if (ret) {
+            return ret;
+        }
+        if (!address_space_access_valid(&address_space_memory, pages[i],
+                                        TARGET_PAGE_SIZE, is_write)) {
+            program_interrupt(env, PGM_ADDRESSING, 0);
+            return -EFAULT;
+        }
+        addr += TARGET_PAGE_SIZE;
+    }
+
+    return 0;
+}
+
+/**
+ * s390_cpu_virt_mem_rw:
+ * @laddr:     the logical start address
+ * @ar:        the access register number
+ * @hostbuf:   buffer in host memory. NULL = do only checks w/o copying
+ * @len:       length that should be transferred
+ * @is_write:  true = write, false = read
+ * Returns:    0 on success, non-zero if an exception occurred
+ *
+ * Copy from/to guest memory using logical addresses. Note that we inject a
+ * program interrupt in case there is an error while accessing the memory.
+ */
+int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
+                         int len, bool is_write)
+{
+    int currlen, nr_pages, i;
+    target_ulong *pages;
+    int ret;
+
+    if (kvm_enabled()) {
+        ret = kvm_s390_mem_op(cpu, laddr, ar, hostbuf, len, is_write);
+        if (ret >= 0) {
+            return ret;
+        }
+    }
+
+    nr_pages = (((laddr & ~TARGET_PAGE_MASK) + len - 1) >> TARGET_PAGE_BITS)
+               + 1;
+    pages = g_malloc(nr_pages * sizeof(*pages));
+
+    ret = translate_pages(cpu, laddr, nr_pages, pages, is_write);
+    if (ret == 0 && hostbuf != NULL) {
+        /* Copy data by stepping through the area page by page */
+        for (i = 0; i < nr_pages; i++) {
+            currlen = MIN(len, TARGET_PAGE_SIZE - (laddr % TARGET_PAGE_SIZE));
+            cpu_physical_memory_rw(pages[i] | (laddr & ~TARGET_PAGE_MASK),
+                                   hostbuf, currlen, is_write);
+            laddr += currlen;
+            hostbuf += currlen;
+            len -= currlen;
+        }
+    }
+
+    g_free(pages);
+    return ret;
+}
diff --git a/target-s390x/trace-events b/target-s390x/trace-events
new file mode 100644
index 0000000..df59f5f
--- /dev/null
+++ b/target-s390x/trace-events
@@ -0,0 +1,22 @@
+# See docs/tracing.txt for syntax documentation.
+
+# target-s390x/mmu_helper.c
+get_skeys_nonzero(int rc) "SKEY: Call to get_skeys unexpectedly returned %d"
+set_skeys_nonzero(int rc) "SKEY: Call to set_skeys unexpectedly returned %d"
+
+# target-s390x/ioinst.c
+ioinst(const char *insn) "IOINST: %s"
+ioinst_sch_id(const char *insn, int cssid, int ssid, int schid) "IOINST: %s (%x.%x.%04x)"
+ioinst_chp_id(const char *insn, int cssid, int chpid) "IOINST: %s (%x.%02x)"
+ioinst_chsc_cmd(uint16_t cmd, uint16_t len) "IOINST: chsc command %04x, len %04x"
+
+# target-s390x/kvm.c
+kvm_enable_cmma(int rc) "CMMA: enabling with result code %d"
+kvm_clear_cmma(int rc) "CMMA: clearing with result code %d"
+kvm_failed_cpu_state_set(int cpu_index, uint8_t state, const char *msg) "Warning: Unable to set cpu %d state %" PRIu8 " to KVM: %s"
+kvm_sigp_finished(uint8_t order, int cpu_index, int dst_index, int cc) "SIGP: Finished order %u on cpu %d -> cpu %d with cc=%d"
+
+# target-s390x/cpu.c
+cpu_set_state(int cpu_index, uint8_t state) "setting cpu %d state to %" PRIu8
+cpu_halt(int cpu_index) "halting cpu %d"
+cpu_unhalt(int cpu_index) "unhalting cpu %d"
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 215ef2e..1a07d70 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -28,21 +28,24 @@
 #  define LOG_DISAS(...) do { } while (0)
 #endif
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "qemu/host-utils.h"
 #include "exec/cpu_ldst.h"
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 
 #include "exec/gen-icount.h"
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 /* Information that (most) every instruction needs to manipulate.  */
@@ -113,7 +116,7 @@
     }
 
     for (i = 0; i < 16; i++) {
-        cpu_fprintf(f, "F%02d=%016" PRIx64, i, env->fregs[i].ll);
+        cpu_fprintf(f, "F%02d=%016" PRIx64, i, get_freg(env, i)->ll);
         if ((i % 4) == 3) {
             cpu_fprintf(f, "\n");
         } else {
@@ -121,6 +124,12 @@
         }
     }
 
+    for (i = 0; i < 32; i++) {
+        cpu_fprintf(f, "V%02d=%016" PRIx64 "%016" PRIx64, i,
+                    env->vregs[i][0].ll, env->vregs[i][1].ll);
+        cpu_fprintf(f, (i % 2) ? "\n" : " ");
+    }
+
 #ifndef CONFIG_USER_ONLY
     for (i = 0; i < 16; i++) {
         cpu_fprintf(f, "C%02d=%016" PRIx64, i, env->cregs[i]);
@@ -144,6 +153,7 @@
 
 static TCGv_i64 psw_addr;
 static TCGv_i64 psw_mask;
+static TCGv_i64 gbea;
 
 static TCGv_i32 cc_op;
 static TCGv_i64 cc_src;
@@ -154,40 +164,42 @@
 static TCGv_i64 regs[16];
 static TCGv_i64 fregs[16];
 
-static uint8_t gen_opc_cc_op[OPC_BUF_SIZE];
-
 void s390x_translate_init(void)
 {
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    psw_addr = tcg_global_mem_new_i64(TCG_AREG0,
+    tcg_ctx.tcg_env = cpu_env;
+    psw_addr = tcg_global_mem_new_i64(cpu_env,
                                       offsetof(CPUS390XState, psw.addr),
                                       "psw_addr");
-    psw_mask = tcg_global_mem_new_i64(TCG_AREG0,
+    psw_mask = tcg_global_mem_new_i64(cpu_env,
                                       offsetof(CPUS390XState, psw.mask),
                                       "psw_mask");
+    gbea = tcg_global_mem_new_i64(cpu_env,
+                                  offsetof(CPUS390XState, gbea),
+                                  "gbea");
 
-    cc_op = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUS390XState, cc_op),
+    cc_op = tcg_global_mem_new_i32(cpu_env, offsetof(CPUS390XState, cc_op),
                                    "cc_op");
-    cc_src = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_src),
+    cc_src = tcg_global_mem_new_i64(cpu_env, offsetof(CPUS390XState, cc_src),
                                     "cc_src");
-    cc_dst = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_dst),
+    cc_dst = tcg_global_mem_new_i64(cpu_env, offsetof(CPUS390XState, cc_dst),
                                     "cc_dst");
-    cc_vr = tcg_global_mem_new_i64(TCG_AREG0, offsetof(CPUS390XState, cc_vr),
+    cc_vr = tcg_global_mem_new_i64(cpu_env, offsetof(CPUS390XState, cc_vr),
                                    "cc_vr");
 
     for (i = 0; i < 16; i++) {
         snprintf(cpu_reg_names[i], sizeof(cpu_reg_names[0]), "r%d", i);
-        regs[i] = tcg_global_mem_new(TCG_AREG0,
+        regs[i] = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUS390XState, regs[i]),
                                      cpu_reg_names[i]);
     }
 
     for (i = 0; i < 16; i++) {
         snprintf(cpu_reg_names[i + 16], sizeof(cpu_reg_names[0]), "f%d", i);
-        fregs[i] = tcg_global_mem_new(TCG_AREG0,
-                                      offsetof(CPUS390XState, fregs[i].d),
+        fregs[i] = tcg_global_mem_new(cpu_env,
+                                      offsetof(CPUS390XState, vregs[i][0].d),
                                       cpu_reg_names[i + 16]);
     }
 }
@@ -243,6 +255,46 @@
     tcg_gen_movi_i64(psw_addr, s->pc);
 }
 
+static void per_branch(DisasContext *s, bool to_next)
+{
+#ifndef CONFIG_USER_ONLY
+    tcg_gen_movi_i64(gbea, s->pc);
+
+    if (s->tb->flags & FLAG_MASK_PER) {
+        TCGv_i64 next_pc = to_next ? tcg_const_i64(s->next_pc) : psw_addr;
+        gen_helper_per_branch(cpu_env, gbea, next_pc);
+        if (to_next) {
+            tcg_temp_free_i64(next_pc);
+        }
+    }
+#endif
+}
+
+static void per_branch_cond(DisasContext *s, TCGCond cond,
+                            TCGv_i64 arg1, TCGv_i64 arg2)
+{
+#ifndef CONFIG_USER_ONLY
+    if (s->tb->flags & FLAG_MASK_PER) {
+        TCGLabel *lab = gen_new_label();
+        tcg_gen_brcond_i64(tcg_invert_cond(cond), arg1, arg2, lab);
+
+        tcg_gen_movi_i64(gbea, s->pc);
+        gen_helper_per_branch(cpu_env, gbea, psw_addr);
+
+        gen_set_label(lab);
+    } else {
+        TCGv_i64 pc = tcg_const_i64(s->pc);
+        tcg_gen_movcond_i64(cond, gbea, arg1, arg2, gbea, pc);
+        tcg_temp_free_i64(pc);
+    }
+#endif
+}
+
+static void per_breaking_event(DisasContext *s)
+{
+    tcg_gen_movi_i64(gbea, s->pc);
+}
+
 static void update_cc_op(DisasContext *s)
 {
     if (s->cc_op != CC_OP_DYNAMIC && s->cc_op != CC_OP_STATIC) {
@@ -314,15 +366,31 @@
 
 static inline void gen_illegal_opcode(DisasContext *s)
 {
-    gen_program_exception(s, PGM_SPECIFICATION);
+    gen_program_exception(s, PGM_OPERATION);
 }
 
-static inline void check_privileged(DisasContext *s)
+static inline void gen_trap(DisasContext *s)
+{
+    TCGv_i32 t;
+
+    /* Set DXC to 0xff.  */
+    t = tcg_temp_new_i32();
+    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_gen_ori_i32(t, t, 0xff00);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
+    tcg_temp_free_i32(t);
+
+    gen_program_exception(s, PGM_DATA);
+}
+
+#ifndef CONFIG_USER_ONLY
+static void check_privileged(DisasContext *s)
 {
     if (s->tb->flags & (PSW_MASK_PSTATE >> 32)) {
         gen_program_exception(s, PGM_PRIVILEGED);
     }
 }
+#endif
 
 static TCGv_i64 get_address(DisasContext *s, int x2, int b2, int d2)
 {
@@ -542,11 +610,17 @@
 
 static int use_goto_tb(DisasContext *s, uint64_t dest)
 {
-    /* NOTE: we handle the case where the TB spans two pages here */
-    return (((dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK)
-             || (dest & TARGET_PAGE_MASK) == ((s->pc - 1) & TARGET_PAGE_MASK))
-            && !s->singlestep_enabled
-            && !(s->tb->cflags & CF_LAST_IO));
+    if (unlikely(s->singlestep_enabled) ||
+        (s->tb->cflags & CF_LAST_IO) ||
+        (s->tb->flags & FLAG_MASK_PER)) {
+        return false;
+    }
+#ifndef CONFIG_USER_ONLY
+    return (dest & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) ||
+           (dest & TARGET_PAGE_MASK) == (s->pc & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
 }
 
 static void account_noninline_branch(DisasContext *s, int cc_op)
@@ -744,7 +818,7 @@
     case CC_OP_LTGT0_32:
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_dst);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_dst);
         c->u.s32.b = tcg_const_i32(0);
         break;
     case CC_OP_LTGT_32:
@@ -752,9 +826,9 @@
     case CC_OP_SUBU_32:
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_src);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_src);
         c->u.s32.b = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.b, cc_dst);
+        tcg_gen_extrl_i64_i32(c->u.s32.b, cc_dst);
         break;
 
     case CC_OP_LTGT0_64:
@@ -784,11 +858,11 @@
         c->is_64 = false;
         c->u.s32.a = tcg_temp_new_i32();
         c->u.s32.b = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(c->u.s32.a, cc_vr);
+        tcg_gen_extrl_i64_i32(c->u.s32.a, cc_vr);
         if (cond == TCG_COND_EQ || cond == TCG_COND_NE) {
             tcg_gen_movi_i32(c->u.s32.b, 0);
         } else {
-            tcg_gen_trunc_i64_i32(c->u.s32.b, cc_src);
+            tcg_gen_extrl_i64_i32(c->u.s32.b, cc_src);
         }
         break;
 
@@ -979,6 +1053,7 @@
 };
 
 struct DisasFields {
+    uint64_t raw_insn;
     unsigned op:8;
     unsigned op2:8;
     unsigned presentC:16;
@@ -1111,11 +1186,14 @@
     FAC_HFP_MA,             /* HFP multiply-and-add/subtract */
     FAC_HW,                 /* high-word */
     FAC_IEEEE_SIM,          /* IEEE exception sumilation */
+    FAC_MIE,                /* miscellaneous-instruction-extensions */
+    FAC_LAT,                /* load-and-trap */
     FAC_LOC,                /* load/store on condition */
     FAC_LD,                 /* long displacement */
     FAC_PC,                 /* population count */
     FAC_SCF,                /* store clock fast */
     FAC_SFLE,               /* store facility list extended */
+    FAC_ILA,                /* interlocked access facility 1 */
 } DisasFacility;
 
 struct DisasInsn {
@@ -1156,16 +1234,19 @@
 static ExitStatus help_goto_direct(DisasContext *s, uint64_t dest)
 {
     if (dest == s->next_pc) {
+        per_branch(s, true);
         return NO_EXIT;
     }
     if (use_goto_tb(s, dest)) {
         update_cc_op(s);
+        per_breaking_event(s);
         tcg_gen_goto_tb(0);
         tcg_gen_movi_i64(psw_addr, dest);
         tcg_gen_exit_tb((uintptr_t)s->tb);
         return EXIT_GOTO_TB;
     } else {
         tcg_gen_movi_i64(psw_addr, dest);
+        per_branch(s, false);
         return EXIT_PC_UPDATED;
     }
 }
@@ -1175,7 +1256,7 @@
 {
     ExitStatus ret;
     uint64_t dest = s->pc + 2 * imm;
-    int lab;
+    TCGLabel *lab;
 
     /* Take care of the special cases first.  */
     if (c->cond == TCG_COND_NEVER) {
@@ -1185,6 +1266,7 @@
     if (is_imm) {
         if (dest == s->next_pc) {
             /* Branch to next.  */
+            per_branch(s, true);
             ret = NO_EXIT;
             goto egress;
         }
@@ -1200,6 +1282,7 @@
         }
         if (c->cond == TCG_COND_ALWAYS) {
             tcg_gen_mov_i64(psw_addr, cdest);
+            per_branch(s, false);
             ret = EXIT_PC_UPDATED;
             goto egress;
         }
@@ -1224,6 +1307,7 @@
 
             /* Branch taken.  */
             gen_set_label(lab);
+            per_breaking_event(s);
             tcg_gen_goto_tb(1);
             tcg_gen_movi_i64(psw_addr, dest);
             tcg_gen_exit_tb((uintptr_t)s->tb + 1);
@@ -1255,6 +1339,7 @@
             if (is_imm) {
                 tcg_gen_movi_i64(psw_addr, dest);
             }
+            per_breaking_event(s);
             ret = EXIT_PC_UPDATED;
         }
     } else {
@@ -1270,6 +1355,7 @@
         if (c->is_64) {
             tcg_gen_movcond_i64(c->cond, psw_addr, c->u.s64.a, c->u.s64.b,
                                 cdest, next);
+            per_branch_cond(s, c->cond, c->u.s64.a, c->u.s64.b);
         } else {
             TCGv_i32 t0 = tcg_temp_new_i32();
             TCGv_i64 t1 = tcg_temp_new_i64();
@@ -1278,6 +1364,7 @@
             tcg_gen_extu_i32_i64(t1, t0);
             tcg_temp_free_i32(t0);
             tcg_gen_movcond_i64(TCG_COND_NE, psw_addr, t1, z, cdest, next);
+            per_branch_cond(s, TCG_COND_NE, t1, z);
             tcg_temp_free_i64(t1);
             tcg_temp_free_i64(z);
         }
@@ -1301,7 +1388,13 @@
 
 static ExitStatus op_abs(DisasContext *s, DisasOps *o)
 {
-    gen_helper_abs_i64(o->out, o->in2);
+    TCGv_i64 z, n;
+    z = tcg_const_i64(0);
+    n = tcg_temp_new_i64();
+    tcg_gen_neg_i64(n, o->in2);
+    tcg_gen_movcond_i64(TCG_COND_LT, o->out, o->in2, z, n, o->in2);
+    tcg_temp_free_i64(n);
+    tcg_temp_free_i64(z);
     return NO_EXIT;
 }
 
@@ -1404,6 +1497,7 @@
     tcg_gen_movi_i64(o->out, pc_to_link_info(s, s->next_pc));
     if (!TCGV_IS_UNUSED_I64(o->in2)) {
         tcg_gen_mov_i64(psw_addr, o->in2);
+        per_branch(s, false);
         return EXIT_PC_UPDATED;
     } else {
         return NO_EXIT;
@@ -1445,12 +1539,36 @@
     store_reg32_i64(r1, t);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_const_i32(0);
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
     tcg_temp_free_i64(t);
 
     return help_branch(s, &c, is_imm, imm, o->in2);
 }
 
+static ExitStatus op_bcth(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int imm = get_field(s->fields, i2);
+    DisasCompare c;
+    TCGv_i64 t;
+
+    c.cond = TCG_COND_NE;
+    c.is_64 = false;
+    c.g1 = false;
+    c.g2 = false;
+
+    t = tcg_temp_new_i64();
+    tcg_gen_shri_i64(t, regs[r1], 32);
+    tcg_gen_subi_i64(t, t, 1);
+    store_reg32h_i64(r1, t);
+    c.u.s32.a = tcg_temp_new_i32();
+    c.u.s32.b = tcg_const_i32(0);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
+    tcg_temp_free_i64(t);
+
+    return help_branch(s, &c, 1, imm, o->in2);
+}
+
 static ExitStatus op_bct64(DisasContext *s, DisasOps *o)
 {
     int r1 = get_field(s->fields, r1);
@@ -1488,8 +1606,8 @@
     tcg_gen_add_i64(t, regs[r1], regs[r3]);
     c.u.s32.a = tcg_temp_new_i32();
     c.u.s32.b = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(c.u.s32.a, t);
-    tcg_gen_trunc_i64_i32(c.u.s32.b, regs[r3 | 1]);
+    tcg_gen_extrl_i64_i32(c.u.s32.a, t);
+    tcg_gen_extrl_i64_i32(c.u.s32.b, regs[r3 | 1]);
     store_reg32_i64(r1, t);
     tcg_temp_free_i64(t);
 
@@ -1794,7 +1912,7 @@
 {
     TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
     TCGv_i32 t1 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, o->in1);
+    tcg_gen_extrl_i64_i32(t1, o->in1);
     potential_page_fault(s);
     gen_helper_clm(cc_op, cpu_env, t1, m3, o->in2);
     set_cc_static(s);
@@ -1866,7 +1984,7 @@
 
     /* Store CC back to cc_op.  Wait until after the store so that any
        exception gets the old cc_op value.  */
-    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_gen_extrl_i64_i32(cc_op, cc);
     tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
@@ -1916,7 +2034,7 @@
     /* Save back state now that we've passed all exceptions.  */
     tcg_gen_mov_i64(regs[r1], outh);
     tcg_gen_mov_i64(regs[r1 + 1], outl);
-    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_gen_extrl_i64_i32(cc_op, cc);
     tcg_temp_free_i64(outh);
     tcg_temp_free_i64(outl);
     tcg_temp_free_i64(cc);
@@ -1940,7 +2058,7 @@
 {
     TCGv_i64 t1 = tcg_temp_new_i64();
     TCGv_i32 t2 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t2, o->in1);
+    tcg_gen_extrl_i64_i32(t2, o->in1);
     gen_helper_cvd(t1, t2);
     tcg_temp_free_i32(t2);
     tcg_gen_qemu_st64(t1, o->in2, get_mem_index(s));
@@ -1951,8 +2069,7 @@
 static ExitStatus op_ct(DisasContext *s, DisasOps *o)
 {
     int m3 = get_field(s->fields, m3);
-    int lab = gen_new_label();
-    TCGv_i32 t;
+    TCGLabel *lab = gen_new_label();
     TCGCond c;
 
     c = tcg_invert_cond(ltgt_cond[m3]);
@@ -1961,15 +2078,8 @@
     }
     tcg_gen_brcond_i64(c, o->in1, o->in2, lab);
 
-    /* Set DXC to 0xff.  */
-    t = tcg_temp_new_i32();
-    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
-    tcg_gen_ori_i32(t, t, 0xff00);
-    tcg_gen_st_i32(t, cpu_env, offsetof(CPUS390XState, fpc));
-    tcg_temp_free_i32(t);
-
     /* Trap.  */
-    gen_program_exception(s, PGM_DATA);
+    gen_trap(s);
 
     gen_set_label(lab);
     return NO_EXIT;
@@ -1978,15 +2088,19 @@
 #ifndef CONFIG_USER_ONLY
 static ExitStatus op_diag(DisasContext *s, DisasOps *o)
 {
-    TCGv_i32 tmp;
+    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+    TCGv_i32 func_code = tcg_const_i32(get_field(s->fields, i2));
 
     check_privileged(s);
-    potential_page_fault(s);
+    update_psw_addr(s);
+    gen_op_calc_cc(s);
 
-    /* We pretend the format is RX_a so that D2 is the field we want.  */
-    tmp = tcg_const_i32(get_field(s->fields, d2) & 0xfff);
-    gen_helper_diag(regs[2], cpu_env, tmp, regs[2], regs[1]);
-    tcg_temp_free_i32(tmp);
+    gen_helper_diag(cpu_env, r1, r3, func_code);
+
+    tcg_temp_free_i32(func_code);
+    tcg_temp_free_i32(r3);
+    tcg_temp_free_i32(r1);
     return NO_EXIT;
 }
 #endif
@@ -2045,12 +2159,37 @@
     return NO_EXIT;
 }
 
+static ExitStatus op_ecag(DisasContext *s, DisasOps *o)
+{
+    /* No cache information provided.  */
+    tcg_gen_movi_i64(o->out, -1);
+    return NO_EXIT;
+}
+
 static ExitStatus op_efpc(DisasContext *s, DisasOps *o)
 {
     tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, fpc));
     return NO_EXIT;
 }
 
+static ExitStatus op_epsw(DisasContext *s, DisasOps *o)
+{
+    int r1 = get_field(s->fields, r1);
+    int r2 = get_field(s->fields, r2);
+    TCGv_i64 t = tcg_temp_new_i64();
+
+    /* Note the "subsequently" in the PoO, which implies a defined result
+       if r1 == r2.  Thus we cannot defer these writes to an output hook.  */
+    tcg_gen_shri_i64(t, psw_mask, 32);
+    store_reg32_i64(r1, t);
+    if (r2 != 0) {
+        store_reg32_i64(r2, psw_mask);
+    }
+
+    tcg_temp_free_i64(t);
+    return NO_EXIT;
+}
+
 static ExitStatus op_ex(DisasContext *s, DisasOps *o)
 {
     /* ??? Perhaps a better way to implement EXECUTE is to set a bit in
@@ -2067,13 +2206,37 @@
     TCGv_i64 tmp;
 
     update_psw_addr(s);
-    update_cc_op(s);
+    gen_op_calc_cc(s);
 
     tmp = tcg_const_i64(s->next_pc);
     gen_helper_ex(cc_op, cpu_env, cc_op, o->in1, o->in2, tmp);
     tcg_temp_free_i64(tmp);
 
-    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_fieb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_fieb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_fidb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_fidb(o->out, cpu_env, o->in2, m3);
+    tcg_temp_free_i32(m3);
+    return NO_EXIT;
+}
+
+static ExitStatus op_fixb(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 m3 = tcg_const_i32(get_field(s->fields, m3));
+    gen_helper_fixb(o->out, cpu_env, o->in1, o->in2, m3);
+    return_low128(o->out2);
+    tcg_temp_free_i32(m3);
     return NO_EXIT;
 }
 
@@ -2286,6 +2449,61 @@
     return NO_EXIT;
 }
 
+static ExitStatus op_lat(DisasContext *s, DisasOps *o)
+{
+    TCGLabel *lab = gen_new_label();
+    store_reg32_i64(get_field(s->fields, r1), o->in2);
+    /* The value is stored even in case of trap. */
+    tcg_gen_brcondi_i64(TCG_COND_NE, o->in2, 0, lab);
+    gen_trap(s);
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lgat(DisasContext *s, DisasOps *o)
+{
+    TCGLabel *lab = gen_new_label();
+    tcg_gen_qemu_ld64(o->out, o->in2, get_mem_index(s));
+    /* The value is stored even in case of trap. */
+    tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab);
+    gen_trap(s);
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lfhat(DisasContext *s, DisasOps *o)
+{
+    TCGLabel *lab = gen_new_label();
+    store_reg32h_i64(get_field(s->fields, r1), o->in2);
+    /* The value is stored even in case of trap. */
+    tcg_gen_brcondi_i64(TCG_COND_NE, o->in2, 0, lab);
+    gen_trap(s);
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
+static ExitStatus op_llgfat(DisasContext *s, DisasOps *o)
+{
+    TCGLabel *lab = gen_new_label();
+    tcg_gen_qemu_ld32u(o->out, o->in2, get_mem_index(s));
+    /* The value is stored even in case of trap. */
+    tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab);
+    gen_trap(s);
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
+static ExitStatus op_llgtat(DisasContext *s, DisasOps *o)
+{
+    TCGLabel *lab = gen_new_label();
+    tcg_gen_andi_i64(o->out, o->in2, 0x7fffffff);
+    /* The value is stored even in case of trap. */
+    tcg_gen_brcondi_i64(TCG_COND_NE, o->out, 0, lab);
+    gen_trap(s);
+    gen_set_label(lab);
+    return NO_EXIT;
+}
+
 static ExitStatus op_loc(DisasContext *s, DisasOps *o)
 {
     DisasCompare c;
@@ -2354,6 +2572,7 @@
     TCGv_i64 t1, t2;
 
     check_privileged(s);
+    per_breaking_event(s);
 
     t1 = tcg_temp_new_i64();
     t2 = tcg_temp_new_i64();
@@ -2373,6 +2592,7 @@
     TCGv_i64 t1, t2;
 
     check_privileged(s);
+    per_breaking_event(s);
 
     t1 = tcg_temp_new_i64();
     t2 = tcg_temp_new_i64();
@@ -2401,21 +2621,45 @@
 {
     int r1 = get_field(s->fields, r1);
     int r3 = get_field(s->fields, r3);
-    TCGv_i64 t = tcg_temp_new_i64();
-    TCGv_i64 t4 = tcg_const_i64(4);
+    TCGv_i64 t1, t2;
 
-    while (1) {
-        tcg_gen_qemu_ld32u(t, o->in2, get_mem_index(s));
-        store_reg32_i64(r1, t);
-        if (r1 == r3) {
-            break;
-        }
-        tcg_gen_add_i64(o->in2, o->in2, t4);
-        r1 = (r1 + 1) & 15;
+    /* Only one register to read. */
+    t1 = tcg_temp_new_i64();
+    if (unlikely(r1 == r3)) {
+        tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+        store_reg32_i64(r1, t1);
+        tcg_temp_free(t1);
+        return NO_EXIT;
     }
 
-    tcg_temp_free_i64(t);
-    tcg_temp_free_i64(t4);
+    /* First load the values of the first and last registers to trigger
+       possible page faults. */
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(t2, o->in2, 4 * ((r3 - r1) & 15));
+    tcg_gen_qemu_ld32u(t2, t2, get_mem_index(s));
+    store_reg32_i64(r1, t1);
+    store_reg32_i64(r3, t2);
+
+    /* Only two registers to read. */
+    if (((r1 + 1) & 15) == r3) {
+        tcg_temp_free(t2);
+        tcg_temp_free(t1);
+        return NO_EXIT;
+    }
+
+    /* Then load the remaining registers. Page fault can't occur. */
+    r3 = (r3 - 1) & 15;
+    tcg_gen_movi_i64(t2, 4);
+    while (r1 != r3) {
+        r1 = (r1 + 1) & 15;
+        tcg_gen_add_i64(o->in2, o->in2, t2);
+        tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+        store_reg32_i64(r1, t1);
+    }
+    tcg_temp_free(t2);
+    tcg_temp_free(t1);
+
     return NO_EXIT;
 }
 
@@ -2423,21 +2667,45 @@
 {
     int r1 = get_field(s->fields, r1);
     int r3 = get_field(s->fields, r3);
-    TCGv_i64 t = tcg_temp_new_i64();
-    TCGv_i64 t4 = tcg_const_i64(4);
+    TCGv_i64 t1, t2;
 
-    while (1) {
-        tcg_gen_qemu_ld32u(t, o->in2, get_mem_index(s));
-        store_reg32h_i64(r1, t);
-        if (r1 == r3) {
-            break;
-        }
-        tcg_gen_add_i64(o->in2, o->in2, t4);
-        r1 = (r1 + 1) & 15;
+    /* Only one register to read. */
+    t1 = tcg_temp_new_i64();
+    if (unlikely(r1 == r3)) {
+        tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+        store_reg32h_i64(r1, t1);
+        tcg_temp_free(t1);
+        return NO_EXIT;
     }
 
-    tcg_temp_free_i64(t);
-    tcg_temp_free_i64(t4);
+    /* First load the values of the first and last registers to trigger
+       possible page faults. */
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(t2, o->in2, 4 * ((r3 - r1) & 15));
+    tcg_gen_qemu_ld32u(t2, t2, get_mem_index(s));
+    store_reg32h_i64(r1, t1);
+    store_reg32h_i64(r3, t2);
+
+    /* Only two registers to read. */
+    if (((r1 + 1) & 15) == r3) {
+        tcg_temp_free(t2);
+        tcg_temp_free(t1);
+        return NO_EXIT;
+    }
+
+    /* Then load the remaining registers. Page fault can't occur. */
+    r3 = (r3 - 1) & 15;
+    tcg_gen_movi_i64(t2, 4);
+    while (r1 != r3) {
+        r1 = (r1 + 1) & 15;
+        tcg_gen_add_i64(o->in2, o->in2, t2);
+        tcg_gen_qemu_ld32u(t1, o->in2, get_mem_index(s));
+        store_reg32h_i64(r1, t1);
+    }
+    tcg_temp_free(t2);
+    tcg_temp_free(t1);
+
     return NO_EXIT;
 }
 
@@ -2445,21 +2713,61 @@
 {
     int r1 = get_field(s->fields, r1);
     int r3 = get_field(s->fields, r3);
-    TCGv_i64 t8 = tcg_const_i64(8);
+    TCGv_i64 t1, t2;
 
-    while (1) {
+    /* Only one register to read. */
+    if (unlikely(r1 == r3)) {
         tcg_gen_qemu_ld64(regs[r1], o->in2, get_mem_index(s));
-        if (r1 == r3) {
-            break;
-        }
-        tcg_gen_add_i64(o->in2, o->in2, t8);
-        r1 = (r1 + 1) & 15;
+        return NO_EXIT;
     }
 
-    tcg_temp_free_i64(t8);
+    /* First load the values of the first and last registers to trigger
+       possible page faults. */
+    t1 = tcg_temp_new_i64();
+    t2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld64(t1, o->in2, get_mem_index(s));
+    tcg_gen_addi_i64(t2, o->in2, 8 * ((r3 - r1) & 15));
+    tcg_gen_qemu_ld64(regs[r3], t2, get_mem_index(s));
+    tcg_gen_mov_i64(regs[r1], t1);
+    tcg_temp_free(t2);
+
+    /* Only two registers to read. */
+    if (((r1 + 1) & 15) == r3) {
+        tcg_temp_free(t1);
+        return NO_EXIT;
+    }
+
+    /* Then load the remaining registers. Page fault can't occur. */
+    r3 = (r3 - 1) & 15;
+    tcg_gen_movi_i64(t1, 8);
+    while (r1 != r3) {
+        r1 = (r1 + 1) & 15;
+        tcg_gen_add_i64(o->in2, o->in2, t1);
+        tcg_gen_qemu_ld64(regs[r1], o->in2, get_mem_index(s));
+    }
+    tcg_temp_free(t1);
+
     return NO_EXIT;
 }
 
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_lura(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lura(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+
+static ExitStatus op_lurag(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_lurag(o->out, cpu_env, o->in2);
+    return NO_EXIT;
+}
+#endif
+
 static ExitStatus op_mov2(DisasContext *s, DisasOps *o)
 {
     o->out = o->in2;
@@ -2469,6 +2777,41 @@
     return NO_EXIT;
 }
 
+static ExitStatus op_mov2e(DisasContext *s, DisasOps *o)
+{
+    int b2 = get_field(s->fields, b2);
+    TCGv ar1 = tcg_temp_new_i64();
+
+    o->out = o->in2;
+    o->g_out = o->g_in2;
+    TCGV_UNUSED_I64(o->in2);
+    o->g_in2 = false;
+
+    switch (s->tb->flags & FLAG_MASK_ASC) {
+    case PSW_ASC_PRIMARY >> 32:
+        tcg_gen_movi_i64(ar1, 0);
+        break;
+    case PSW_ASC_ACCREG >> 32:
+        tcg_gen_movi_i64(ar1, 1);
+        break;
+    case PSW_ASC_SECONDARY >> 32:
+        if (b2) {
+            tcg_gen_ld32u_i64(ar1, cpu_env, offsetof(CPUS390XState, aregs[b2]));
+        } else {
+            tcg_gen_movi_i64(ar1, 0);
+        }
+        break;
+    case PSW_ASC_HOME >> 32:
+        tcg_gen_movi_i64(ar1, 2);
+        break;
+    }
+
+    tcg_gen_st32_i64(ar1, cpu_env, offsetof(CPUS390XState, aregs[1]));
+    tcg_temp_free_i64(ar1);
+
+    return NO_EXIT;
+}
+
 static ExitStatus op_movx(DisasContext *s, DisasOps *o)
 {
     o->out = o->in1;
@@ -2629,7 +2972,13 @@
 
 static ExitStatus op_nabs(DisasContext *s, DisasOps *o)
 {
-    gen_helper_nabs_i64(o->out, o->in2);
+    TCGv_i64 z, n;
+    z = tcg_const_i64(0);
+    n = tcg_temp_new_i64();
+    tcg_gen_neg_i64(n, o->in2);
+    tcg_gen_movcond_i64(TCG_COND_GE, o->out, o->in2, z, n, o->in2);
+    tcg_temp_free_i64(n);
+    tcg_temp_free_i64(z);
     return NO_EXIT;
 }
 
@@ -2893,8 +3242,8 @@
     TCGv_i32 t1 = tcg_temp_new_i32();
     TCGv_i32 t2 = tcg_temp_new_i32();
     TCGv_i32 to = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(t1, o->in1);
-    tcg_gen_trunc_i64_i32(t2, o->in2);
+    tcg_gen_extrl_i64_i32(t1, o->in1);
+    tcg_gen_extrl_i64_i32(t2, o->in2);
     tcg_gen_rotl_i32(to, t1, t2);
     tcg_gen_extu_i32_i64(o->out, to);
     tcg_temp_free_i32(t1);
@@ -2925,19 +3274,42 @@
     /* Addressing mode has changed, so end the block.  */
     return EXIT_PC_STALE;
 }
+#endif
 
 static ExitStatus op_sam(DisasContext *s, DisasOps *o)
 {
     int sam = s->insn->data;
-    TCGv_i64 tsam = tcg_const_i64(sam);
+    TCGv_i64 tsam;
+    uint64_t mask;
 
-    /* Overwrite PSW_MASK_64 and PSW_MASK_32 */
+    switch (sam) {
+    case 0:
+        mask = 0xffffff;
+        break;
+    case 1:
+        mask = 0x7fffffff;
+        break;
+    default:
+        mask = -1;
+        break;
+    }
+
+    /* Bizarre but true, we check the address of the current insn for the
+       specification exception, not the next to be executed.  Thus the PoO
+       documents that Bad Things Happen two bytes before the end.  */
+    if (s->pc & ~mask) {
+        gen_program_exception(s, PGM_SPECIFICATION);
+        return EXIT_NORETURN;
+    }
+    s->next_pc &= mask;
+
+    tsam = tcg_const_i64(sam);
     tcg_gen_deposit_i64(psw_mask, psw_mask, tsam, 31, 2);
-
     tcg_temp_free_i64(tsam);
+
+    /* Always exit the TB, since we (may have) changed execution mode.  */
     return EXIT_PC_STALE;
 }
-#endif
 
 static ExitStatus op_sar(DisasContext *s, DisasOps *o)
 {
@@ -3009,10 +3381,15 @@
 {
     DisasCompare c;
     TCGv_i64 a;
-    int lab, r1;
+    TCGLabel *lab;
+    int r1;
 
     disas_jcc(s, &c, get_field(s->fields, m3));
 
+    /* We want to store when the condition is fulfilled, so branch
+       out when it's not */
+    c.cond = tcg_invert_cond(c.cond);
+
     lab = gen_new_label();
     if (c.is_64) {
         tcg_gen_brcond_i64(c.cond, c.u.s64.a, c.u.s64.b, lab);
@@ -3221,8 +3598,14 @@
 
 static ExitStatus op_stidp(DisasContext *s, DisasOps *o)
 {
+    TCGv_i64 t1 = tcg_temp_new_i64();
+
     check_privileged(s);
     tcg_gen_ld32u_i64(o->out, cpu_env, offsetof(CPUS390XState, cpu_num));
+    tcg_gen_ld32u_i64(t1, cpu_env, offsetof(CPUS390XState, machine_type));
+    tcg_gen_deposit_i64(o->out, o->out, t1, 32, 32);
+    tcg_temp_free_i64(t1);
+
     return NO_EXIT;
 }
 
@@ -3270,11 +3653,93 @@
     return NO_EXIT;
 }
 
-static ExitStatus op_subchannel(DisasContext *s, DisasOps *o)
+static ExitStatus op_xsch(DisasContext *s, DisasOps *o)
 {
     check_privileged(s);
-    /* Not operational.  */
-    gen_op_movi_cc(s, 3);
+    potential_page_fault(s);
+    gen_helper_xsch(cpu_env, regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_csch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_csch(cpu_env, regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_hsch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_hsch(cpu_env, regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_msch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_msch(cpu_env, regs[1], o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rchp(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_rchp(cpu_env, regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_rsch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_rsch(cpu_env, regs[1]);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_ssch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_ssch(cpu_env, regs[1], o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_stsch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_stsch(cpu_env, regs[1], o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_tsch(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_tsch(cpu_env, regs[1], o->in2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_chsc(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_chsc(cpu_env, o->in2);
+    set_cc_static(s);
     return NO_EXIT;
 }
 
@@ -3317,6 +3782,14 @@
     gen_helper_stura(cpu_env, o->in2, o->in1);
     return NO_EXIT;
 }
+
+static ExitStatus op_sturg(DisasContext *s, DisasOps *o)
+{
+    check_privileged(s);
+    potential_page_fault(s);
+    gen_helper_sturg(cpu_env, o->in2, o->in1);
+    return NO_EXIT;
+}
 #endif
 
 static ExitStatus op_st8(DisasContext *s, DisasOps *o)
@@ -3513,21 +3986,21 @@
 
 static ExitStatus op_tceb(DisasContext *s, DisasOps *o)
 {
-    gen_helper_tceb(cc_op, o->in1, o->in2);
+    gen_helper_tceb(cc_op, cpu_env, o->in1, o->in2);
     set_cc_static(s);
     return NO_EXIT;
 }
 
 static ExitStatus op_tcdb(DisasContext *s, DisasOps *o)
 {
-    gen_helper_tcdb(cc_op, o->in1, o->in2);
+    gen_helper_tcdb(cc_op, cpu_env, o->in1, o->in2);
     set_cc_static(s);
     return NO_EXIT;
 }
 
 static ExitStatus op_tcxb(DisasContext *s, DisasOps *o)
 {
-    gen_helper_tcxb(cc_op, o->out, o->out2, o->in2);
+    gen_helper_tcxb(cc_op, cpu_env, o->out, o->out2, o->in2);
     set_cc_static(s);
     return NO_EXIT;
 }
@@ -3552,6 +4025,25 @@
     return NO_EXIT;
 }
 
+static ExitStatus op_tre(DisasContext *s, DisasOps *o)
+{
+    potential_page_fault(s);
+    gen_helper_tre(o->out, cpu_env, o->out, o->out2, o->in2);
+    return_low128(o->out2);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
+static ExitStatus op_trt(DisasContext *s, DisasOps *o)
+{
+    TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
+    potential_page_fault(s);
+    gen_helper_trt(cc_op, cpu_env, l, o->addr1, o->in2);
+    tcg_temp_free_i32(l);
+    set_cc_static(s);
+    return NO_EXIT;
+}
+
 static ExitStatus op_unpk(DisasContext *s, DisasOps *o)
 {
     TCGv_i32 l = tcg_const_i32(get_field(s->fields, l1));
@@ -3895,6 +4387,12 @@
 }
 #define SPEC_wout_r1_32 0
 
+static void wout_r1_32h(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    store_reg32h_i64(get_field(f, r1), o->out);
+}
+#define SPEC_wout_r1_32h 0
+
 static void wout_r1_P32(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     int r1 = get_field(f, r1);
@@ -3978,6 +4476,22 @@
 }
 #define SPEC_wout_m2_32 0
 
+static void wout_m2_32_r1_atomic(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    /* XXX release reservation */
+    tcg_gen_qemu_st32(o->out, o->addr1, get_mem_index(s));
+    store_reg32_i64(get_field(f, r1), o->in2);
+}
+#define SPEC_wout_m2_32_r1_atomic 0
+
+static void wout_m2_64_r1_atomic(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    /* XXX release reservation */
+    tcg_gen_qemu_st64(o->out, o->addr1, get_mem_index(s));
+    store_reg(get_field(f, r1), o->in2);
+}
+#define SPEC_wout_m2_64_r1_atomic 0
+
 /* ====================================================================== */
 /* The "INput 1" generators.  These load the first operand to an insn.  */
 
@@ -4049,6 +4563,13 @@
 }
 #define SPEC_in1_r2 0
 
+static void in1_r2_sr32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(o->in1, regs[get_field(f, r2)], 32);
+}
+#define SPEC_in1_r2_sr32 0
+
 static void in1_r3(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in1 = load_reg(get_field(f, r3));
@@ -4262,6 +4783,13 @@
 }
 #define SPEC_in2_r3 0
 
+static void in2_r3_sr32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(o->in2, regs[get_field(f, r3)], 32);
+}
+#define SPEC_in2_r3_sr32 0
+
 static void in2_r2_32s(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in2 = tcg_temp_new_i64();
@@ -4276,6 +4804,13 @@
 }
 #define SPEC_in2_r2_32u 0
 
+static void in2_r2_sr32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(o->in2, regs[get_field(f, r2)], 32);
+}
+#define SPEC_in2_r2_sr32 0
+
 static void in2_e2(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in2 = load_freg32_i64(get_field(f, r2));
@@ -4399,6 +4934,24 @@
 }
 #define SPEC_in2_mri2_64 0
 
+static void in2_m2_32s_atomic(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    /* XXX should reserve the address */
+    in1_la2(s, f, o);
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld32s(o->in2, o->addr1, get_mem_index(s));
+}
+#define SPEC_in2_m2_32s_atomic 0
+
+static void in2_m2_64_atomic(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    /* XXX should reserve the address */
+    in1_la2(s, f, o);
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_qemu_ld64(o->in2, o->addr1, get_mem_index(s));
+}
+#define SPEC_in2_m2_64_atomic 0
+
 static void in2_i2(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in2 = tcg_const_i64(get_field(f, i2));
@@ -4437,6 +4990,14 @@
 }
 #define SPEC_in2_i2_32u_shl 0
 
+#ifndef CONFIG_USER_ONLY
+static void in2_insn(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    o->in2 = tcg_const_i64(s->fields->raw_insn);
+}
+#define SPEC_in2_insn 0
+#endif
+
 /* ====================================================================== */
 
 /* Find opc within the table of insns.  This is formulated as a switch
@@ -4613,6 +5174,7 @@
     }
 
     memset(f, 0, sizeof(*f));
+    f->raw_insn = insn;
     f->op = op;
     f->op2 = op2;
 
@@ -4649,6 +5211,14 @@
         return EXIT_NORETURN;
     }
 
+#ifndef CONFIG_USER_ONLY
+    if (s->tb->flags & FLAG_MASK_PER) {
+        TCGv_i64 addr = tcg_const_i64(s->pc);
+        gen_helper_per_ifetch(cpu_env, addr);
+        tcg_temp_free_i64(addr);
+    }
+#endif
+
     /* Check for insn specification exceptions.  */
     if (insn->spec) {
         int spec = insn->spec, excp = 0, r;
@@ -4736,23 +5306,34 @@
         tcg_temp_free_i64(o.addr1);
     }
 
+#ifndef CONFIG_USER_ONLY
+    if (s->tb->flags & FLAG_MASK_PER) {
+        /* An exception might be triggered, save PSW if not already done.  */
+        if (ret == NO_EXIT || ret == EXIT_PC_STALE) {
+            tcg_gen_movi_i64(psw_addr, s->next_pc);
+        }
+
+        /* Save off cc.  */
+        update_cc_op(s);
+
+        /* Call the helper to check for a possible PER exception.  */
+        gen_helper_per_check_exception(cpu_env);
+    }
+#endif
+
     /* Advance to the next instruction.  */
     s->pc = s->next_pc;
     return ret;
 }
 
-static inline void gen_intermediate_code_internal(S390CPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool search_pc)
+void gen_intermediate_code(CPUS390XState *env, struct TranslationBlock *tb)
 {
+    S390CPU *cpu = s390_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUS390XState *env = &cpu->env;
     DisasContext dc;
     target_ulong pc_start;
     uint64_t next_page_start;
-    int j, lj = -1;
     int num_insns, max_insns;
-    CPUBreakpoint *bp;
     ExitStatus status;
     bool do_debug;
 
@@ -4775,41 +5356,32 @@
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
-    gen_tb_start();
+    gen_tb_start(tb);
 
     do {
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc.pc;
-            gen_opc_cc_op[lj] = dc.cc_op;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc.pc, dc.cc_op);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc.pc, BP_ANY))) {
+            status = EXIT_PC_STALE;
+            do_debug = true;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc.pc += 2;
+            break;
         }
-        if (++num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc.pc);
-        }
-
         status = NO_EXIT;
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc.pc) {
-                    status = EXIT_PC_STALE;
-                    do_debug = true;
-                    break;
-                }
-            }
-        }
         if (status == NO_EXIT) {
             status = translate_one(env, &dc);
         }
@@ -4854,41 +5426,24 @@
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = dc.pc - pc_start;
+    tb->icount = num_insns;
 
 #if defined(S390X_DEBUG_DISAS)
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, dc.pc - pc_start, 1);
+        log_target_disas(cs, pc_start, dc.pc - pc_start, 1);
         qemu_log("\n");
     }
 #endif
 }
 
-void gen_intermediate_code (CPUS390XState *env, struct TranslationBlock *tb)
+void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(s390_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc (CPUS390XState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(s390_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUS390XState *env, TranslationBlock *tb, int pc_pos)
-{
-    int cc_op;
-    env->psw.addr = tcg_ctx.gen_opc_pc[pc_pos];
-    cc_op = gen_opc_cc_op[pc_pos];
+    int cc_op = data[1];
+    env->psw.addr = data[0];
     if ((cc_op != CC_OP_DYNAMIC) && (cc_op != CC_OP_STATIC)) {
         env->cc_op = cc_op;
     }
diff --git a/target-sh4/Makefile.objs b/target-sh4/Makefile.objs
index a285358..2c25d96 100644
--- a/target-sh4/Makefile.objs
+++ b/target-sh4/Makefile.objs
@@ -1,2 +1,3 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
+obj-$(CONFIG_SOFTMMU) += monitor.o
 obj-y += gdbstub.o
diff --git a/target-sh4/cpu-qom.h b/target-sh4/cpu-qom.h
index 6341238..01abb20 100644
--- a/target-sh4/cpu-qom.h
+++ b/target-sh4/cpu-qom.h
@@ -60,35 +60,6 @@
     uint32_t cvr;
 } SuperHCPUClass;
 
-/**
- * SuperHCPU:
- * @env: #CPUSH4State
- *
- * A SuperH CPU.
- */
-typedef struct SuperHCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUSH4State env;
-} SuperHCPU;
-
-static inline SuperHCPU *sh_env_get_cpu(CPUSH4State *env)
-{
-    return container_of(env, SuperHCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(sh_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(SuperHCPU, env)
-
-void superh_cpu_do_interrupt(CPUState *cpu);
-bool superh_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void superh_cpu_dump_state(CPUState *cpu, FILE *f,
-                           fprintf_function cpu_fprintf, int flags);
-hwaddr superh_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int superh_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int superh_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+typedef struct SuperHCPU SuperHCPU;
 
 #endif
diff --git a/target-sh4/cpu.c b/target-sh4/cpu.c
index d187a2b..f589532 100644
--- a/target-sh4/cpu.c
+++ b/target-sh4/cpu.c
@@ -19,9 +19,12 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
+#include "exec/exec-all.h"
 
 
 static void superh_cpu_set_pc(CPUState *cs, vaddr value)
@@ -61,12 +64,20 @@
     env->fpscr = FPSCR_PR; /* value for userspace according to the kernel */
     set_float_rounding_mode(float_round_nearest_even, &env->fp_status); /* ?! */
 #else
-    env->sr = SR_MD | SR_RB | SR_BL | SR_I3 | SR_I2 | SR_I1 | SR_I0;
+    env->sr = (1u << SR_MD) | (1u << SR_RB) | (1u << SR_BL) |
+              (1u << SR_I3) | (1u << SR_I2) | (1u << SR_I1) | (1u << SR_I0);
     env->fpscr = FPSCR_DN | FPSCR_RM_ZERO; /* CPU reset value according to SH4 manual */
     set_float_rounding_mode(float_round_to_zero, &env->fp_status);
     set_flush_to_zero(1, &env->fp_status);
 #endif
     set_default_nan_mode(1, &env->fp_status);
+    set_snan_bit_is_one(1, &env->fp_status);
+}
+
+static void superh_cpu_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->mach = bfd_mach_sh4;
+    info->print_insn = print_insn_sh;
 }
 
 typedef struct SuperHCPUListState {
@@ -247,7 +258,7 @@
     CPUSH4State *env = &cpu->env;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     env->movcal_backup_tail = &(env->movcal_backup);
 
@@ -287,8 +298,18 @@
 #else
     cc->get_phys_page_debug = superh_cpu_get_phys_page_debug;
 #endif
-    dc->vmsd = &vmstate_sh_cpu;
+    cc->disas_set_info = superh_cpu_disas_set_info;
+
     cc->gdb_num_core_regs = 59;
+
+    dc->vmsd = &vmstate_sh_cpu;
+
+    /*
+     * Reason: superh_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo superh_cpu_type_info = {
diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h
index a2e9e2c..478ab55 100644
--- a/target-sh4/cpu.h
+++ b/target-sh4/cpu.h
@@ -16,16 +16,14 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef _CPU_SH4_H
-#define _CPU_SH4_H
 
-#include "config.h"
+#ifndef SH4_CPU_H
+#define SH4_CPU_H
+
 #include "qemu-common.h"
+#include "cpu-qom.h"
 
 #define TARGET_LONG_BITS 32
-#define TARGET_HAS_ICE 1
-
-#define ELF_MACHINE	EM_SH
 
 /* CPU Subtypes */
 #define SH_CPU_SH7750  (1 << 0)
@@ -48,18 +46,18 @@
 #define TARGET_PHYS_ADDR_SPACE_BITS 32
 #define TARGET_VIRT_ADDR_SPACE_BITS 32
 
-#define SR_MD (1 << 30)
-#define SR_RB (1 << 29)
-#define SR_BL (1 << 28)
-#define SR_FD (1 << 15)
-#define SR_M  (1 << 9)
-#define SR_Q  (1 << 8)
-#define SR_I3 (1 << 7)
-#define SR_I2 (1 << 6)
-#define SR_I1 (1 << 5)
-#define SR_I0 (1 << 4)
-#define SR_S  (1 << 1)
-#define SR_T  (1 << 0)
+#define SR_MD 30
+#define SR_RB 29
+#define SR_BL 28
+#define SR_FD 15
+#define SR_M  9
+#define SR_Q  8
+#define SR_I3 7
+#define SR_I2 6
+#define SR_I1 5
+#define SR_I0 4
+#define SR_S  1
+#define SR_T  0
 
 #define FPSCR_MASK             (0x003fffff)
 #define FPSCR_FR               (1 << 21)
@@ -123,6 +121,7 @@
 #define ITLB_SIZE 4
 
 #define NB_MMU_MODES 2
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 enum sh_features {
     SH_FEATURE_SH4A = 1,
@@ -139,7 +138,10 @@
     uint32_t flags;		/* general execution flags */
     uint32_t gregs[24];		/* general registers */
     float32 fregs[32];		/* floating point registers */
-    uint32_t sr;		/* status register */
+    uint32_t sr;                /* status register (with T split out) */
+    uint32_t sr_m;              /* M bit of status register */
+    uint32_t sr_q;              /* Q bit of status register */
+    uint32_t sr_t;              /* T bit of status register */
     uint32_t ssr;		/* saved status register */
     uint32_t spc;		/* saved program counter */
     uint32_t gbr;		/* global base register */
@@ -187,11 +189,39 @@
     memory_content **movcal_backup_tail;
 } CPUSH4State;
 
-#include "cpu-qom.h"
+/**
+ * SuperHCPU:
+ * @env: #CPUSH4State
+ *
+ * A SuperH CPU.
+ */
+struct SuperHCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUSH4State env;
+};
+
+static inline SuperHCPU *sh_env_get_cpu(CPUSH4State *env)
+{
+    return container_of(env, SuperHCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(sh_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(SuperHCPU, env)
+
+void superh_cpu_do_interrupt(CPUState *cpu);
+bool superh_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void superh_cpu_dump_state(CPUState *cpu, FILE *f,
+                           fprintf_function cpu_fprintf, int flags);
+hwaddr superh_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int superh_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int superh_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
 void sh4_translate_init(void);
 SuperHCPU *cpu_sh4_init(const char *cpu_model);
-int cpu_sh4_exec(CPUSH4State * s);
 int cpu_sh4_signal_handler(int host_signum, void *pinfo,
                            void *puc);
 int superh_cpu_handle_mmu_fault(CPUState *cpu, vaddr address, int rw,
@@ -222,17 +252,8 @@
 
 void cpu_load_tlb(CPUSH4State * env);
 
-static inline CPUSH4State *cpu_init(const char *cpu_model)
-{
-    SuperHCPU *cpu = cpu_sh4_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_sh4_init(cpu_model))
 
-#define cpu_exec cpu_sh4_exec
-#define cpu_gen_code cpu_sh4_gen_code
 #define cpu_signal_handler cpu_sh4_signal_handler
 #define cpu_list sh4_cpu_list
 
@@ -240,9 +261,9 @@
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index (CPUSH4State *env)
+static inline int cpu_mmu_index (CPUSH4State *env, bool ifetch)
 {
-    return (env->sr & SR_MD) == 0 ? 1 : 0;
+    return (env->sr & (1u << SR_MD)) == 0 ? 1 : 0;
 }
 
 #include "exec/cpu-all.h"
@@ -339,19 +360,32 @@
 
 #define TB_FLAG_PENDING_MOVCA  (1 << 4)
 
+static inline target_ulong cpu_read_sr(CPUSH4State *env)
+{
+    return env->sr | (env->sr_m << SR_M) |
+                     (env->sr_q << SR_Q) |
+                     (env->sr_t << SR_T);
+}
+
+static inline void cpu_write_sr(CPUSH4State *env, target_ulong sr)
+{
+    env->sr_m = (sr >> SR_M) & 1;
+    env->sr_q = (sr >> SR_Q) & 1;
+    env->sr_t = (sr >> SR_T) & 1;
+    env->sr = sr & ~((1u << SR_M) | (1u << SR_Q) | (1u << SR_T));
+}
+
 static inline void cpu_get_tb_cpu_state(CPUSH4State *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->pc;
     *cs_base = 0;
     *flags = (env->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL
                     | DELAY_SLOT_TRUE | DELAY_SLOT_CLEARME))   /* Bits  0- 3 */
             | (env->fpscr & (FPSCR_FR | FPSCR_SZ | FPSCR_PR))  /* Bits 19-21 */
-            | (env->sr & (SR_MD | SR_RB))                      /* Bits 29-30 */
-            | (env->sr & SR_FD)                                /* Bit 15 */
+            | (env->sr & ((1u << SR_MD) | (1u << SR_RB)))      /* Bits 29-30 */
+            | (env->sr & (1u << SR_FD))                        /* Bit 15 */
             | (env->movcal_backup ? TB_FLAG_PENDING_MOVCA : 0); /* Bit 4 */
 }
 
-#include "exec/exec-all.h"
-
-#endif				/* _CPU_SH4_H */
+#endif /* SH4_CPU_H */
diff --git a/target-sh4/gdbstub.c b/target-sh4/gdbstub.c
index df4fa2a..13bea00 100644
--- a/target-sh4/gdbstub.c
+++ b/target-sh4/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 /* Hint: Use "set architecture sh4" in GDB to see fpu registers */
@@ -31,7 +32,7 @@
 
     switch (n) {
     case 0 ... 7:
-        if ((env->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB)) {
+        if ((env->sr & (1u << SR_MD)) && (env->sr & (1u << SR_RB))) {
             return gdb_get_regl(mem_buf, env->gregs[n + 16]);
         } else {
             return gdb_get_regl(mem_buf, env->gregs[n]);
@@ -51,7 +52,7 @@
     case 21:
         return gdb_get_regl(mem_buf, env->macl);
     case 22:
-        return gdb_get_regl(mem_buf, env->sr);
+        return gdb_get_regl(mem_buf, cpu_read_sr(env));
     case 23:
         return gdb_get_regl(mem_buf, env->fpul);
     case 24:
@@ -83,7 +84,7 @@
 
     switch (n) {
     case 0 ... 7:
-        if ((env->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB)) {
+        if ((env->sr & (1u << SR_MD)) && (env->sr & (1u << SR_RB))) {
             env->gregs[n + 16] = ldl_p(mem_buf);
         } else {
             env->gregs[n] = ldl_p(mem_buf);
@@ -111,7 +112,7 @@
         env->macl = ldl_p(mem_buf);
         break;
     case 22:
-        env->sr = ldl_p(mem_buf);
+        cpu_write_sr(env, ldl_p(mem_buf));
         break;
     case 23:
         env->fpul = ldl_p(mem_buf);
diff --git a/target-sh4/helper.c b/target-sh4/helper.c
index 5811360..a33ac69 100644
--- a/target-sh4/helper.c
+++ b/target-sh4/helper.c
@@ -16,14 +16,11 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
+#include "exec/log.h"
 
 #if !defined(CONFIG_USER_ONLY)
 #include "hw/sh4/sh_intc.h"
@@ -60,7 +57,7 @@
 
 int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 {
-    /* For user mode, only U0 area is cachable. */
+    /* For user mode, only U0 area is cacheable. */
     return !(addr & 0x80000000);
 }
 
@@ -93,7 +90,7 @@
     do_exp = cs->exception_index != -1;
     do_irq = do_irq && (cs->exception_index == -1);
 
-    if (env->sr & SR_BL) {
+    if (env->sr & (1u << SR_BL)) {
         if (do_exp && cs->exception_index != 0x1e0) {
             cs->exception_index = 0x000; /* masked exception -> reset */
         }
@@ -162,10 +159,10 @@
         log_cpu_state(cs, 0);
     }
 
-    env->ssr = env->sr;
+    env->ssr = cpu_read_sr(env);
     env->spc = env->pc;
     env->sgr = env->gregs[15];
-    env->sr |= SR_BL | SR_MD | SR_RB;
+    env->sr |= (1u << SR_BL) | (1u << SR_MD) | (1u << SR_RB);
 
     if (env->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) {
         /* Branch instruction should be executed again before delay slot. */
@@ -182,7 +179,7 @@
         case 0x000:
         case 0x020:
         case 0x140:
-            env->sr &= ~SR_FD;
+            env->sr &= ~(1u << SR_FD);
             env->sr |= 0xf << 4; /* IMASK */
             env->pc = 0xa0000000;
             break;
@@ -355,23 +352,24 @@
     int use_asid, n;
     tlb_t *matching = NULL;
 
-    use_asid = (env->mmucr & MMUCR_SV) == 0 || (env->sr & SR_MD) == 0;
+    use_asid = !(env->mmucr & MMUCR_SV) || !(env->sr & (1u << SR_MD));
 
     if (rw == 2) {
         n = find_itlb_entry(env, address, use_asid);
 	if (n >= 0) {
 	    matching = &env->itlb[n];
-	    if (!(env->sr & SR_MD) && !(matching->pr & 2))
+            if (!(env->sr & (1u << SR_MD)) && !(matching->pr & 2)) {
 		n = MMU_ITLB_VIOLATION;
-	    else
+            } else {
 		*prot = PAGE_EXEC;
+            }
         } else {
             n = find_utlb_entry(env, address, use_asid);
             if (n >= 0) {
                 n = copy_utlb_entry_itlb(env, n);
                 matching = &env->itlb[n];
-                if (!(env->sr & SR_MD) && !(matching->pr & 2)) {
-                      n = MMU_ITLB_VIOLATION;
+                if (!(env->sr & (1u << SR_MD)) && !(matching->pr & 2)) {
+                    n = MMU_ITLB_VIOLATION;
                 } else {
                     *prot = PAGE_READ | PAGE_EXEC;
                     if ((matching->pr & 1) && matching->d) {
@@ -388,7 +386,7 @@
 	n = find_utlb_entry(env, address, use_asid);
 	if (n >= 0) {
 	    matching = &env->utlb[n];
-            if (!(env->sr & SR_MD) && !(matching->pr & 2)) {
+            if (!(env->sr & (1u << SR_MD)) && !(matching->pr & 2)) {
                 n = (rw == 1) ? MMU_DTLB_VIOLATION_WRITE :
                     MMU_DTLB_VIOLATION_READ;
             } else if ((rw == 1) && !(matching->pr & 1)) {
@@ -421,7 +419,7 @@
     /* P1, P2 and P4 areas do not use translation */
     if ((address >= 0x80000000 && address < 0xc0000000) ||
 	address >= 0xe0000000) {
-	if (!(env->sr & SR_MD)
+        if (!(env->sr & (1u << SR_MD))
 	    && (address < 0xe0000000 || address >= 0xe4000000)) {
 	    /* Unauthorized access in user mode (only store queues are available) */
 	    fprintf(stderr, "Unauthorized access\n");
@@ -690,7 +688,7 @@
     uint8_t d = (uint8_t)((mem_value & 0x00000200) >> 9);
     uint8_t v = (uint8_t)((mem_value & 0x00000100) >> 8);
     uint8_t asid = (uint8_t)(mem_value & 0x000000ff);
-    int use_asid = (s->mmucr & MMUCR_SV) == 0 || (s->sr & SR_MD) == 0;
+    int use_asid = !(s->mmucr & MMUCR_SV) || !(s->sr & (1u << SR_MD));
 
     if (associate) {
         int i;
@@ -821,15 +819,15 @@
 int cpu_sh4_is_cached(CPUSH4State * env, target_ulong addr)
 {
     int n;
-    int use_asid = (env->mmucr & MMUCR_SV) == 0 || (env->sr & SR_MD) == 0;
+    int use_asid = !(env->mmucr & MMUCR_SV) || !(env->sr & (1u << SR_MD));
 
     /* check area */
-    if (env->sr & SR_MD) {
-        /* For previledged mode, P2 and P4 area is not cachable. */
+    if (env->sr & (1u << SR_MD)) {
+        /* For privileged mode, P2 and P4 area is not cacheable. */
         if ((0xA0000000 <= addr && addr < 0xC0000000) || 0xE0000000 <= addr)
             return 0;
     } else {
-        /* For user mode, only U0 area is cachable. */
+        /* For user mode, only U0 area is cacheable. */
         if (0x80000000 <= addr)
             return 0;
     }
diff --git a/target-sh4/helper.h b/target-sh4/helper.h
index 3b5c436..dce859c 100644
--- a/target-sh4/helper.h
+++ b/target-sh4/helper.h
@@ -11,7 +11,6 @@
 DEF_HELPER_1(discard_movcal_backup, void, env)
 DEF_HELPER_2(ocbi, void, env, i32)
 
-DEF_HELPER_3(div1, i32, env, i32, i32)
 DEF_HELPER_3(macl, void, env, i32, i32)
 DEF_HELPER_3(macw, void, env, i32, i32)
 
@@ -19,28 +18,28 @@
 
 DEF_HELPER_FLAGS_1(fabs_FT, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_FLAGS_1(fabs_DT, TCG_CALL_NO_RWG_SE, f64, f64)
-DEF_HELPER_3(fadd_FT, f32, env, f32, f32)
-DEF_HELPER_3(fadd_DT, f64, env, f64, f64)
-DEF_HELPER_2(fcnvsd_FT_DT, f64, env, f32)
-DEF_HELPER_2(fcnvds_DT_FT, f32, env, f64)
+DEF_HELPER_FLAGS_3(fadd_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fadd_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(fcnvsd_FT_DT, TCG_CALL_NO_WG, f64, env, f32)
+DEF_HELPER_FLAGS_2(fcnvds_DT_FT, TCG_CALL_NO_WG, f32, env, f64)
 
 DEF_HELPER_3(fcmp_eq_FT, void, env, f32, f32)
 DEF_HELPER_3(fcmp_eq_DT, void, env, f64, f64)
 DEF_HELPER_3(fcmp_gt_FT, void, env, f32, f32)
 DEF_HELPER_3(fcmp_gt_DT, void, env, f64, f64)
-DEF_HELPER_3(fdiv_FT, f32, env, f32, f32)
-DEF_HELPER_3(fdiv_DT, f64, env, f64, f64)
-DEF_HELPER_2(float_FT, f32, env, i32)
-DEF_HELPER_2(float_DT, f64, env, i32)
-DEF_HELPER_4(fmac_FT, f32, env, f32, f32, f32)
-DEF_HELPER_3(fmul_FT, f32, env, f32, f32)
-DEF_HELPER_3(fmul_DT, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(fdiv_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fdiv_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(float_FT, TCG_CALL_NO_WG, f32, env, i32)
+DEF_HELPER_FLAGS_2(float_DT, TCG_CALL_NO_WG, f64, env, i32)
+DEF_HELPER_FLAGS_4(fmac_FT, TCG_CALL_NO_WG, f32, env, f32, f32, f32)
+DEF_HELPER_FLAGS_3(fmul_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fmul_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
 DEF_HELPER_FLAGS_1(fneg_T, TCG_CALL_NO_RWG_SE, f32, f32)
-DEF_HELPER_3(fsub_FT, f32, env, f32, f32)
-DEF_HELPER_3(fsub_DT, f64, env, f64, f64)
-DEF_HELPER_2(fsqrt_FT, f32, env, f32)
-DEF_HELPER_2(fsqrt_DT, f64, env, f64)
-DEF_HELPER_2(ftrc_FT, i32, env, f32)
-DEF_HELPER_2(ftrc_DT, i32, env, f64)
+DEF_HELPER_FLAGS_3(fsub_FT, TCG_CALL_NO_WG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fsub_DT, TCG_CALL_NO_WG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_2(fsqrt_FT, TCG_CALL_NO_WG, f32, env, f32)
+DEF_HELPER_FLAGS_2(fsqrt_DT, TCG_CALL_NO_WG, f64, env, f64)
+DEF_HELPER_FLAGS_2(ftrc_FT, TCG_CALL_NO_WG, i32, env, f32)
+DEF_HELPER_FLAGS_2(ftrc_DT, TCG_CALL_NO_WG, i32, env, f64)
 DEF_HELPER_3(fipr, void, env, i32, i32)
 DEF_HELPER_2(ftrv, void, env, i32)
diff --git a/target-sh4/monitor.c b/target-sh4/monitor.c
new file mode 100644
index 0000000..426e5d4
--- /dev/null
+++ b/target-sh4/monitor.c
@@ -0,0 +1,53 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+static void print_tlb(Monitor *mon, int idx, tlb_t *tlb)
+{
+    monitor_printf(mon, " tlb%i:\t"
+                   "asid=%hhu vpn=%x\tppn=%x\tsz=%hhu size=%u\t"
+                   "v=%hhu shared=%hhu cached=%hhu prot=%hhu "
+                   "dirty=%hhu writethrough=%hhu\n",
+                   idx,
+                   tlb->asid, tlb->vpn, tlb->ppn, tlb->sz, tlb->size,
+                   tlb->v, tlb->sh, tlb->c, tlb->pr,
+                   tlb->d, tlb->wt);
+}
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    int i;
+
+    monitor_printf (mon, "ITLB:\n");
+    for (i = 0 ; i < ITLB_SIZE ; i++)
+        print_tlb (mon, i, &env->itlb[i]);
+    monitor_printf (mon, "UTLB:\n");
+    for (i = 0 ; i < UTLB_SIZE ; i++)
+        print_tlb (mon, i, &env->utlb[i]);
+}
diff --git a/target-sh4/op_helper.c b/target-sh4/op_helper.c
index 74a5c4e..684d3f3 100644
--- a/target-sh4/op_helper.c
+++ b/target-sh4/op_helper.c
@@ -16,20 +16,20 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <assert.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
 
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = superh_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = superh_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (ret) {
         /* now we have a real cpu fault */
         if (retaddr) {
@@ -109,7 +109,8 @@
 {
     if (cpu_sh4_is_cached (env, address))
     {
-	memory_content *r = malloc (sizeof(memory_content));
+        memory_content *r = g_new(memory_content, 1);
+
 	r->address = address;
 	r->value = value;
 	r->next = NULL;
@@ -126,7 +127,7 @@
     while(current)
     {
 	memory_content *next = current->next;
-	free (current);
+        g_free(current);
 	env->movcal_backup = current = next;
 	if (current == NULL)
 	    env->movcal_backup_tail = &(env->movcal_backup);
@@ -149,131 +150,13 @@
 		env->movcal_backup_tail = current;
 	    }
 
-	    free (*current);
+            g_free(*current);
 	    *current = next;
 	    break;
 	}
     }
 }
 
-#define T (env->sr & SR_T)
-#define Q (env->sr & SR_Q ? 1 : 0)
-#define M (env->sr & SR_M ? 1 : 0)
-#define SETT env->sr |= SR_T
-#define CLRT env->sr &= ~SR_T
-#define SETQ env->sr |= SR_Q
-#define CLRQ env->sr &= ~SR_Q
-#define SETM env->sr |= SR_M
-#define CLRM env->sr &= ~SR_M
-
-uint32_t helper_div1(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
-{
-    uint32_t tmp0, tmp2;
-    uint8_t old_q, tmp1 = 0xff;
-
-    //printf("div1 arg0=0x%08x arg1=0x%08x M=%d Q=%d T=%d\n", arg0, arg1, M, Q, T);
-    old_q = Q;
-    if ((0x80000000 & arg1) != 0)
-	SETQ;
-    else
-	CLRQ;
-    tmp2 = arg0;
-    arg1 <<= 1;
-    arg1 |= T;
-    switch (old_q) {
-    case 0:
-	switch (M) {
-	case 0:
-	    tmp0 = arg1;
-	    arg1 -= tmp2;
-	    tmp1 = arg1 > tmp0;
-	    switch (Q) {
-	    case 0:
-		if (tmp1)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    case 1:
-		if (tmp1 == 0)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    }
-	    break;
-	case 1:
-	    tmp0 = arg1;
-	    arg1 += tmp2;
-	    tmp1 = arg1 < tmp0;
-	    switch (Q) {
-	    case 0:
-		if (tmp1 == 0)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    case 1:
-		if (tmp1)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    }
-	    break;
-	}
-	break;
-    case 1:
-	switch (M) {
-	case 0:
-	    tmp0 = arg1;
-	    arg1 += tmp2;
-	    tmp1 = arg1 < tmp0;
-	    switch (Q) {
-	    case 0:
-		if (tmp1)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    case 1:
-		if (tmp1 == 0)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    }
-	    break;
-	case 1:
-	    tmp0 = arg1;
-	    arg1 -= tmp2;
-	    tmp1 = arg1 > tmp0;
-	    switch (Q) {
-	    case 0:
-		if (tmp1 == 0)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    case 1:
-		if (tmp1)
-		    SETQ;
-		else
-		    CLRQ;
-		break;
-	    }
-	    break;
-	}
-	break;
-    }
-    if (Q == M)
-	SETT;
-    else
-	CLRT;
-    //printf("Output: arg1=0x%08x M=%d Q=%d T=%d\n", arg1, M, Q, T);
-    return arg1;
-}
-
 void helper_macl(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     int64_t res;
@@ -282,7 +165,7 @@
     res += (int64_t) (int32_t) arg0 *(int64_t) (int32_t) arg1;
     env->mach = (res >> 32) & 0xffffffff;
     env->macl = res & 0xffffffff;
-    if (env->sr & SR_S) {
+    if (env->sr & (1u << SR_S)) {
 	if (res < 0)
 	    env->mach |= 0xffff0000;
 	else
@@ -298,7 +181,7 @@
     res += (int64_t) (int16_t) arg0 *(int64_t) (int16_t) arg1;
     env->mach = (res >> 32) & 0xffffffff;
     env->macl = res & 0xffffffff;
-    if (env->sr & SR_S) {
+    if (env->sr & (1u << SR_S)) {
 	if (res < -0x80000000) {
 	    env->mach = 1;
 	    env->macl = 0x80000000;
@@ -309,16 +192,6 @@
     }
 }
 
-static inline void set_t(CPUSH4State *env)
-{
-    env->sr |= SR_T;
-}
-
-static inline void clr_t(CPUSH4State *env)
-{
-    env->sr &= ~SR_T;
-}
-
 void helper_ld_fpscr(CPUSH4State *env, uint32_t val)
 {
     env->fpscr = val & FPSCR_MASK;
@@ -403,10 +276,8 @@
     relation = float32_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
         update_fpscr(env, GETPC());
-    } else if (relation == float_relation_equal) {
-        set_t(env);
     } else {
-        clr_t(env);
+        env->sr_t = (relation == float_relation_equal);
     }
 }
 
@@ -418,10 +289,8 @@
     relation = float64_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
         update_fpscr(env, GETPC());
-    } else if (relation == float_relation_equal) {
-        set_t(env);
     } else {
-        clr_t(env);
+        env->sr_t = (relation == float_relation_equal);
     }
 }
 
@@ -433,10 +302,8 @@
     relation = float32_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
         update_fpscr(env, GETPC());
-    } else if (relation == float_relation_greater) {
-        set_t(env);
     } else {
-        clr_t(env);
+        env->sr_t = (relation == float_relation_greater);
     }
 }
 
@@ -448,10 +315,8 @@
     relation = float64_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
         update_fpscr(env, GETPC());
-    } else if (relation == float_relation_greater) {
-        set_t(env);
     } else {
-        clr_t(env);
+        env->sr_t = (relation == float_relation_greater);
     }
 }
 
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index bc82044..ca80cf7 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -18,10 +18,11 @@
  */
 
 #define DEBUG_DISAS
-//#define SH4_SINGLE_STEP
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/cpu_ldst.h"
 
@@ -29,6 +30,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 typedef struct DisasContext {
@@ -47,7 +49,7 @@
 #if defined(CONFIG_USER_ONLY)
 #define IS_USER(ctx) 1
 #else
-#define IS_USER(ctx) (!(ctx->flags & SR_MD))
+#define IS_USER(ctx) (!(ctx->flags & (1u << SR_MD)))
 #endif
 
 enum {
@@ -60,9 +62,10 @@
 };
 
 /* global register indexes */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv cpu_gregs[24];
-static TCGv cpu_pc, cpu_sr, cpu_ssr, cpu_spc, cpu_gbr;
+static TCGv cpu_sr, cpu_sr_m, cpu_sr_q, cpu_sr_t;
+static TCGv cpu_pc, cpu_ssr, cpu_spc, cpu_gbr;
 static TCGv cpu_vbr, cpu_sgr, cpu_dbr, cpu_mach, cpu_macl;
 static TCGv cpu_pr, cpu_fpscr, cpu_fpul, cpu_ldst;
 static TCGv cpu_fregs[32];
@@ -70,8 +73,6 @@
 /* internal register indexes */
 static TCGv cpu_flags, cpu_delayed_pc;
 
-static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-
 #include "exec/gen-icount.h"
 
 void sh4_translate_init(void)
@@ -100,49 +101,56 @@
         return;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
     for (i = 0; i < 24; i++)
-        cpu_gregs[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_gregs[i] = tcg_global_mem_new_i32(cpu_env,
                                               offsetof(CPUSH4State, gregs[i]),
                                               gregnames[i]);
 
-    cpu_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_pc = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUSH4State, pc), "PC");
-    cpu_sr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_sr = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUSH4State, sr), "SR");
-    cpu_ssr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_sr_m = tcg_global_mem_new_i32(cpu_env,
+                                      offsetof(CPUSH4State, sr_m), "SR_M");
+    cpu_sr_q = tcg_global_mem_new_i32(cpu_env,
+                                      offsetof(CPUSH4State, sr_q), "SR_Q");
+    cpu_sr_t = tcg_global_mem_new_i32(cpu_env,
+                                      offsetof(CPUSH4State, sr_t), "SR_T");
+    cpu_ssr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, ssr), "SSR");
-    cpu_spc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_spc = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, spc), "SPC");
-    cpu_gbr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_gbr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, gbr), "GBR");
-    cpu_vbr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_vbr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, vbr), "VBR");
-    cpu_sgr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_sgr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, sgr), "SGR");
-    cpu_dbr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_dbr = tcg_global_mem_new_i32(cpu_env,
                                      offsetof(CPUSH4State, dbr), "DBR");
-    cpu_mach = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_mach = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUSH4State, mach), "MACH");
-    cpu_macl = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_macl = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUSH4State, macl), "MACL");
-    cpu_pr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_pr = tcg_global_mem_new_i32(cpu_env,
                                     offsetof(CPUSH4State, pr), "PR");
-    cpu_fpscr = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_fpscr = tcg_global_mem_new_i32(cpu_env,
                                        offsetof(CPUSH4State, fpscr), "FPSCR");
-    cpu_fpul = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_fpul = tcg_global_mem_new_i32(cpu_env,
                                       offsetof(CPUSH4State, fpul), "FPUL");
 
-    cpu_flags = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_flags = tcg_global_mem_new_i32(cpu_env,
 				       offsetof(CPUSH4State, flags), "_flags_");
-    cpu_delayed_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_delayed_pc = tcg_global_mem_new_i32(cpu_env,
 					    offsetof(CPUSH4State, delayed_pc),
 					    "_delayed_pc_");
-    cpu_ldst = tcg_global_mem_new_i32(TCG_AREG0,
+    cpu_ldst = tcg_global_mem_new_i32(cpu_env,
 				      offsetof(CPUSH4State, ldst), "_ldst_");
 
     for (i = 0; i < 32; i++)
-        cpu_fregs[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_fregs[i] = tcg_global_mem_new_i32(cpu_env,
                                               offsetof(CPUSH4State, fregs[i]),
                                               fregnames[i]);
 
@@ -156,7 +164,7 @@
     CPUSH4State *env = &cpu->env;
     int i;
     cpu_fprintf(f, "pc=0x%08x sr=0x%08x pr=0x%08x fpscr=0x%08x\n",
-		env->pc, env->sr, env->pr, env->fpscr);
+                env->pc, cpu_read_sr(env), env->pr, env->fpscr);
     cpu_fprintf(f, "spc=0x%08x ssr=0x%08x gbr=0x%08x vbr=0x%08x\n",
 		env->spc, env->ssr, env->gbr, env->vbr);
     cpu_fprintf(f, "sgr=0x%08x dbr=0x%08x delayed_pc=0x%08x fpul=0x%08x\n",
@@ -175,17 +183,50 @@
     }
 }
 
-static void gen_goto_tb(DisasContext * ctx, int n, target_ulong dest)
+static void gen_read_sr(TCGv dst)
 {
-    TranslationBlock *tb;
-    tb = ctx->tb;
+    TCGv t0 = tcg_temp_new();
+    tcg_gen_shli_i32(t0, cpu_sr_q, SR_Q);
+    tcg_gen_or_i32(dst, dst, t0);
+    tcg_gen_shli_i32(t0, cpu_sr_m, SR_M);
+    tcg_gen_or_i32(dst, dst, t0);
+    tcg_gen_shli_i32(t0, cpu_sr_t, SR_T);
+    tcg_gen_or_i32(dst, cpu_sr, t0);
+    tcg_temp_free_i32(t0);
+}
 
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-	!ctx->singlestep_enabled) {
+static void gen_write_sr(TCGv src)
+{
+    tcg_gen_andi_i32(cpu_sr, src,
+                     ~((1u << SR_Q) | (1u << SR_M) | (1u << SR_T)));
+    tcg_gen_shri_i32(cpu_sr_q, src, SR_Q);
+    tcg_gen_andi_i32(cpu_sr_q, cpu_sr_q, 1);
+    tcg_gen_shri_i32(cpu_sr_m, src, SR_M);
+    tcg_gen_andi_i32(cpu_sr_m, cpu_sr_m, 1);
+    tcg_gen_shri_i32(cpu_sr_t, src, SR_T);
+    tcg_gen_andi_i32(cpu_sr_t, cpu_sr_t, 1);
+}
+
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+    if (unlikely(ctx->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
+static void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
+{
+    if (use_goto_tb(ctx, dest)) {
 	/* Use a direct jump if in same page and singlestep not enabled */
         tcg_gen_goto_tb(n);
         tcg_gen_movi_i32(cpu_pc, dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
     } else {
         tcg_gen_movi_i32(cpu_pc, dest);
         if (ctx->singlestep_enabled)
@@ -210,12 +251,9 @@
 
 static inline void gen_branch_slot(uint32_t delayed_pc, int t)
 {
-    TCGv sr;
-    int label = gen_new_label();
+    TCGLabel *label = gen_new_label();
     tcg_gen_movi_i32(cpu_delayed_pc, delayed_pc);
-    sr = tcg_temp_new();
-    tcg_gen_andi_i32(sr, cpu_sr, SR_T);
-    tcg_gen_brcondi_i32(t ? TCG_COND_EQ:TCG_COND_NE, sr, 0, label);
+    tcg_gen_brcondi_i32(t ? TCG_COND_EQ : TCG_COND_NE, cpu_sr_t, 0, label);
     tcg_gen_ori_i32(cpu_flags, cpu_flags, DELAY_SLOT_TRUE);
     gen_set_label(label);
 }
@@ -224,13 +262,8 @@
 static void gen_conditional_jump(DisasContext * ctx,
 				 target_ulong ift, target_ulong ifnott)
 {
-    int l1;
-    TCGv sr;
-
-    l1 = gen_new_label();
-    sr = tcg_temp_new();
-    tcg_gen_andi_i32(sr, cpu_sr, SR_T);
-    tcg_gen_brcondi_i32(TCG_COND_NE, sr, 0, l1);
+    TCGLabel *l1 = gen_new_label();
+    tcg_gen_brcondi_i32(TCG_COND_NE, cpu_sr_t, 0, l1);
     gen_goto_tb(ctx, 0, ifnott);
     gen_set_label(l1);
     gen_goto_tb(ctx, 1, ift);
@@ -239,7 +272,7 @@
 /* Delayed conditional jump (bt or bf) */
 static void gen_delayed_conditional_jump(DisasContext * ctx)
 {
-    int l1;
+    TCGLabel *l1;
     TCGv ds;
 
     l1 = gen_new_label();
@@ -252,54 +285,12 @@
     gen_jump(ctx);
 }
 
-static inline void gen_cmp(int cond, TCGv t0, TCGv t1)
-{
-    TCGv t;
-
-    t = tcg_temp_new();
-    tcg_gen_setcond_i32(cond, t, t1, t0);
-    tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-    tcg_gen_or_i32(cpu_sr, cpu_sr, t);
-
-    tcg_temp_free(t);
-}
-
-static inline void gen_cmp_imm(int cond, TCGv t0, int32_t imm)
-{
-    TCGv t;
-
-    t = tcg_temp_new();
-    tcg_gen_setcondi_i32(cond, t, t0, imm);
-    tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-    tcg_gen_or_i32(cpu_sr, cpu_sr, t);
-
-    tcg_temp_free(t);
-}
-
 static inline void gen_store_flags(uint32_t flags)
 {
     tcg_gen_andi_i32(cpu_flags, cpu_flags, DELAY_SLOT_TRUE);
     tcg_gen_ori_i32(cpu_flags, cpu_flags, flags);
 }
 
-static inline void gen_copy_bit_i32(TCGv t0, int p0, TCGv t1, int p1)
-{
-    TCGv tmp = tcg_temp_new();
-
-    p0 &= 0x1f;
-    p1 &= 0x1f;
-
-    tcg_gen_andi_i32(tmp, t1, (1 << p1));
-    tcg_gen_andi_i32(t0, t0, ~(1 << p0));
-    if (p0 < p1)
-        tcg_gen_shri_i32(tmp, tmp, p1 - p0);
-    else if (p0 > p1)
-        tcg_gen_shli_i32(tmp, tmp, p0 - p1);
-    tcg_gen_or_i32(t0, t0, tmp);
-
-    tcg_temp_free(tmp);
-}
-
 static inline void gen_load_fpr64(TCGv_i64 t, int reg)
 {
     tcg_gen_concat_i32_i64(t, cpu_fregs[reg + 1], cpu_fregs[reg]);
@@ -308,10 +299,10 @@
 static inline void gen_store_fpr64 (TCGv_i64 t, int reg)
 {
     TCGv_i32 tmp = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(tmp, t);
+    tcg_gen_extrl_i64_i32(tmp, t);
     tcg_gen_mov_i32(cpu_fregs[reg + 1], tmp);
     tcg_gen_shri_i64(t, t, 32);
-    tcg_gen_trunc_i64_i32(tmp, t);
+    tcg_gen_extrl_i64_i32(tmp, t);
     tcg_gen_mov_i32(cpu_fregs[reg], tmp);
     tcg_temp_free_i32(tmp);
 }
@@ -326,10 +317,12 @@
 #define B11_8 ((ctx->opcode >> 8) & 0xf)
 #define B15_12 ((ctx->opcode >> 12) & 0xf)
 
-#define REG(x) ((x) < 8 && (ctx->flags & (SR_MD | SR_RB)) == (SR_MD | SR_RB) \
+#define REG(x) ((x) < 8 && (ctx->flags & (1u << SR_MD))\
+                        && (ctx->flags & (1u << SR_RB))\
                 ? (cpu_gregs[x + 16]) : (cpu_gregs[x]))
 
-#define ALTREG(x) ((x) < 8 && (ctx->flags & (SR_MD | SR_RB)) != (SR_MD | SR_RB)\
+#define ALTREG(x) ((x) < 8 && (!(ctx->flags & (1u << SR_MD))\
+                               || !(ctx->flags & (1u << SR_RB)))\
 		? (cpu_gregs[x + 16]) : (cpu_gregs[x]))
 
 #define FREG(x) (ctx->flags & FPSCR_FR ? (x) ^ 0x10 : (x))
@@ -359,7 +352,7 @@
   }
 
 #define CHECK_FPU_ENABLED                                       \
-  if (ctx->flags & SR_FD) {                                     \
+  if (ctx->flags & (1u << SR_FD)) {                             \
       tcg_gen_movi_i32(cpu_pc, ctx->pc);                        \
       if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) { \
           gen_helper_raise_slot_fpu_disable(cpu_env);           \
@@ -409,7 +402,9 @@
 
     switch (ctx->opcode) {
     case 0x0019:		/* div0u */
-	tcg_gen_andi_i32(cpu_sr, cpu_sr, ~(SR_M | SR_Q | SR_T));
+        tcg_gen_movi_i32(cpu_sr_m, 0);
+        tcg_gen_movi_i32(cpu_sr_q, 0);
+        tcg_gen_movi_i32(cpu_sr_t, 0);
 	return;
     case 0x000b:		/* rts */
 	CHECK_NOT_DELAY_SLOT
@@ -422,10 +417,10 @@
 	tcg_gen_movi_i32(cpu_macl, 0);
 	return;
     case 0x0048:		/* clrs */
-	tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_S);
+        tcg_gen_andi_i32(cpu_sr, cpu_sr, ~(1u << SR_S));
 	return;
     case 0x0008:		/* clrt */
-        tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+        tcg_gen_movi_i32(cpu_sr_t, 0);
 	return;
     case 0x0038:		/* ldtlb */
 	CHECK_PRIVILEGED
@@ -434,16 +429,16 @@
     case 0x002b:		/* rte */
 	CHECK_PRIVILEGED
 	CHECK_NOT_DELAY_SLOT
-	tcg_gen_mov_i32(cpu_sr, cpu_ssr);
+        gen_write_sr(cpu_ssr);
 	tcg_gen_mov_i32(cpu_delayed_pc, cpu_spc);
 	ctx->flags |= DELAY_SLOT;
 	ctx->delayed_pc = (uint32_t) - 1;
 	return;
     case 0x0058:		/* sets */
-	tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_S);
+        tcg_gen_ori_i32(cpu_sr, cpu_sr, (1u << SR_S));
 	return;
     case 0x0018:		/* sett */
-        tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_T);
+        tcg_gen_movi_i32(cpu_sr_t, 1);
 	return;
     case 0xfbfd:		/* frchg */
 	tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_FR);
@@ -628,15 +623,11 @@
 	return;
     case 0x6008:		/* swap.b Rm,Rn */
 	{
-	    TCGv high, low;
-	    high = tcg_temp_new();
-	    tcg_gen_andi_i32(high, REG(B7_4), 0xffff0000);
-	    low = tcg_temp_new();
+            TCGv low = tcg_temp_new();;
 	    tcg_gen_ext16u_i32(low, REG(B7_4));
 	    tcg_gen_bswap16_i32(low, low);
-	    tcg_gen_or_i32(REG(B11_8), high, low);
+            tcg_gen_deposit_i32(REG(B11_8), REG(B7_4), low, 0, 16);
 	    tcg_temp_free(low);
-	    tcg_temp_free(high);
 	}
 	return;
     case 0x6009:		/* swap.w Rm,Rn */
@@ -659,22 +650,14 @@
 	return;
     case 0x300e:		/* addc Rm,Rn */
         {
-            TCGv t0, t1, t2;
-            t0 = tcg_temp_new();
-            tcg_gen_andi_i32(t0, cpu_sr, SR_T);
+            TCGv t0, t1;
+            t0 = tcg_const_tl(0);
             t1 = tcg_temp_new();
-            tcg_gen_add_i32(t1, REG(B7_4), REG(B11_8));
-            tcg_gen_add_i32(t0, t0, t1);
-            t2 = tcg_temp_new();
-            tcg_gen_setcond_i32(TCG_COND_GTU, t2, REG(B11_8), t1);
-            tcg_gen_setcond_i32(TCG_COND_GTU, t1, t1, t0);
-            tcg_gen_or_i32(t1, t1, t2);
-            tcg_temp_free(t2);
-            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
-            tcg_temp_free(t1);
-            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_gen_add2_i32(t1, cpu_sr_t, cpu_sr_t, t0, REG(B7_4), t0);
+            tcg_gen_add2_i32(REG(B11_8), cpu_sr_t,
+                             REG(B11_8), t0, t1, cpu_sr_t);
             tcg_temp_free(t0);
+            tcg_temp_free(t1);
         }
 	return;
     case 0x300f:		/* addv Rm,Rn */
@@ -686,11 +669,9 @@
             tcg_gen_xor_i32(t1, t0, REG(B11_8));
             t2 = tcg_temp_new();
             tcg_gen_xor_i32(t2, REG(B7_4), REG(B11_8));
-            tcg_gen_andc_i32(t1, t1, t2);
+            tcg_gen_andc_i32(cpu_sr_t, t1, t2);
             tcg_temp_free(t2);
-            tcg_gen_shri_i32(t1, t1, 31);
-            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_gen_shri_i32(cpu_sr_t, cpu_sr_t, 31);
             tcg_temp_free(t1);
             tcg_gen_mov_i32(REG(B7_4), t0);
             tcg_temp_free(t0);
@@ -700,54 +681,72 @@
 	tcg_gen_and_i32(REG(B11_8), REG(B11_8), REG(B7_4));
 	return;
     case 0x3000:		/* cmp/eq Rm,Rn */
-	gen_cmp(TCG_COND_EQ, REG(B7_4), REG(B11_8));
+        tcg_gen_setcond_i32(TCG_COND_EQ, cpu_sr_t, REG(B11_8), REG(B7_4));
 	return;
     case 0x3003:		/* cmp/ge Rm,Rn */
-	gen_cmp(TCG_COND_GE, REG(B7_4), REG(B11_8));
+        tcg_gen_setcond_i32(TCG_COND_GE, cpu_sr_t, REG(B11_8), REG(B7_4));
 	return;
     case 0x3007:		/* cmp/gt Rm,Rn */
-	gen_cmp(TCG_COND_GT, REG(B7_4), REG(B11_8));
+        tcg_gen_setcond_i32(TCG_COND_GT, cpu_sr_t, REG(B11_8), REG(B7_4));
 	return;
     case 0x3006:		/* cmp/hi Rm,Rn */
-	gen_cmp(TCG_COND_GTU, REG(B7_4), REG(B11_8));
+        tcg_gen_setcond_i32(TCG_COND_GTU, cpu_sr_t, REG(B11_8), REG(B7_4));
 	return;
     case 0x3002:		/* cmp/hs Rm,Rn */
-	gen_cmp(TCG_COND_GEU, REG(B7_4), REG(B11_8));
+        tcg_gen_setcond_i32(TCG_COND_GEU, cpu_sr_t, REG(B11_8), REG(B7_4));
 	return;
     case 0x200c:		/* cmp/str Rm,Rn */
 	{
 	    TCGv cmp1 = tcg_temp_new();
 	    TCGv cmp2 = tcg_temp_new();
-	    tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-	    tcg_gen_xor_i32(cmp1, REG(B7_4), REG(B11_8));
-	    tcg_gen_andi_i32(cmp2, cmp1, 0xff000000);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-	    tcg_gen_or_i32(cpu_sr, cpu_sr, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x00ff0000);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-	    tcg_gen_or_i32(cpu_sr, cpu_sr, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x0000ff00);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-	    tcg_gen_or_i32(cpu_sr, cpu_sr, cmp2);
-	    tcg_gen_andi_i32(cmp2, cmp1, 0x000000ff);
-	    tcg_gen_setcondi_i32(TCG_COND_EQ, cmp2, cmp2, 0);
-	    tcg_gen_or_i32(cpu_sr, cpu_sr, cmp2);
+            tcg_gen_xor_i32(cmp2, REG(B7_4), REG(B11_8));
+            tcg_gen_subi_i32(cmp1, cmp2, 0x01010101);
+            tcg_gen_andc_i32(cmp1, cmp1, cmp2);
+            tcg_gen_andi_i32(cmp1, cmp1, 0x80808080);
+            tcg_gen_setcondi_i32(TCG_COND_NE, cpu_sr_t, cmp1, 0);
 	    tcg_temp_free(cmp2);
 	    tcg_temp_free(cmp1);
 	}
 	return;
     case 0x2007:		/* div0s Rm,Rn */
-	{
-	    gen_copy_bit_i32(cpu_sr, 8, REG(B11_8), 31);	/* SR_Q */
-	    gen_copy_bit_i32(cpu_sr, 9, REG(B7_4), 31);		/* SR_M */
-	    TCGv val = tcg_temp_new();
-	    tcg_gen_xor_i32(val, REG(B7_4), REG(B11_8));
-	    gen_copy_bit_i32(cpu_sr, 0, val, 31);		/* SR_T */
-	    tcg_temp_free(val);
-	}
+        tcg_gen_shri_i32(cpu_sr_q, REG(B11_8), 31);         /* SR_Q */
+        tcg_gen_shri_i32(cpu_sr_m, REG(B7_4), 31);          /* SR_M */
+        tcg_gen_xor_i32(cpu_sr_t, cpu_sr_q, cpu_sr_m);      /* SR_T */
 	return;
     case 0x3004:		/* div1 Rm,Rn */
-        gen_helper_div1(REG(B11_8), cpu_env, REG(B7_4), REG(B11_8));
+        {
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+            TCGv zero = tcg_const_i32(0);
+
+            /* shift left arg1, saving the bit being pushed out and inserting
+               T on the right */
+            tcg_gen_shri_i32(t0, REG(B11_8), 31);
+            tcg_gen_shli_i32(REG(B11_8), REG(B11_8), 1);
+            tcg_gen_or_i32(REG(B11_8), REG(B11_8), cpu_sr_t);
+
+            /* Add or subtract arg0 from arg1 depending if Q == M. To avoid
+               using 64-bit temps, we compute arg0's high part from q ^ m, so
+               that it is 0x00000000 when adding the value or 0xffffffff when
+               subtracting it. */
+            tcg_gen_xor_i32(t1, cpu_sr_q, cpu_sr_m);
+            tcg_gen_subi_i32(t1, t1, 1);
+            tcg_gen_neg_i32(t2, REG(B7_4));
+            tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, zero, REG(B7_4), t2);
+            tcg_gen_add2_i32(REG(B11_8), t1, REG(B11_8), zero, t2, t1);
+
+            /* compute T and Q depending on carry */
+            tcg_gen_andi_i32(t1, t1, 1);
+            tcg_gen_xor_i32(t1, t1, t0);
+            tcg_gen_xori_i32(cpu_sr_t, t1, 1);
+            tcg_gen_xor_i32(cpu_sr_q, cpu_sr_m, t1);
+
+            tcg_temp_free(zero);
+            tcg_temp_free(t2);
+            tcg_temp_free(t1);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x300d:		/* dmuls.l Rm,Rn */
         tcg_gen_muls2_i32(cpu_macl, cpu_mach, REG(B7_4), REG(B11_8));
@@ -827,19 +826,13 @@
 	return;
     case 0x600a:		/* negc Rm,Rn */
         {
-	    TCGv t0, t1;
-            t0 = tcg_temp_new();
-            tcg_gen_neg_i32(t0, REG(B7_4));
-            t1 = tcg_temp_new();
-            tcg_gen_andi_i32(t1, cpu_sr, SR_T);
-            tcg_gen_sub_i32(REG(B11_8), t0, t1);
-            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-            tcg_gen_setcondi_i32(TCG_COND_GTU, t1, t0, 0);
-            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
-            tcg_gen_setcond_i32(TCG_COND_GTU, t1, REG(B11_8), t0);
-            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            TCGv t0 = tcg_const_i32(0);
+            tcg_gen_add2_i32(REG(B11_8), cpu_sr_t,
+                             REG(B7_4), t0, cpu_sr_t, t0);
+            tcg_gen_sub2_i32(REG(B11_8), cpu_sr_t,
+                             t0, t0, REG(B11_8), cpu_sr_t);
+            tcg_gen_andi_i32(cpu_sr_t, cpu_sr_t, 1);
             tcg_temp_free(t0);
-            tcg_temp_free(t1);
         }
 	return;
     case 0x6007:		/* not Rm,Rn */
@@ -850,67 +843,54 @@
 	return;
     case 0x400c:		/* shad Rm,Rn */
 	{
-	    int label1 = gen_new_label();
-	    int label2 = gen_new_label();
-	    int label3 = gen_new_label();
-	    int label4 = gen_new_label();
-	    TCGv shift;
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B7_4), 0, label1);
-	    /* Rm positive, shift to the left */
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_shl_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label4);
-	    /* Rm negative, shift to the right */
-	    gen_set_label(label1);
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_brcondi_i32(TCG_COND_EQ, shift, 0, label2);
-	    tcg_gen_not_i32(shift, REG(B7_4));
-	    tcg_gen_andi_i32(shift, shift, 0x1f);
-	    tcg_gen_addi_i32(shift, shift, 1);
-	    tcg_gen_sar_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label4);
-	    /* Rm = -32 */
-	    gen_set_label(label2);
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B11_8), 0, label3);
-	    tcg_gen_movi_i32(REG(B11_8), 0);
-	    tcg_gen_br(label4);
-	    gen_set_label(label3);
-	    tcg_gen_movi_i32(REG(B11_8), 0xffffffff);
-	    gen_set_label(label4);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+
+            tcg_gen_andi_i32(t0, REG(B7_4), 0x1f);
+
+            /* positive case: shift to the left */
+            tcg_gen_shl_i32(t1, REG(B11_8), t0);
+
+            /* negative case: shift to the right in two steps to
+               correctly handle the -32 case */
+            tcg_gen_xori_i32(t0, t0, 0x1f);
+            tcg_gen_sar_i32(t2, REG(B11_8), t0);
+            tcg_gen_sari_i32(t2, t2, 1);
+
+            /* select between the two cases */
+            tcg_gen_movi_i32(t0, 0);
+            tcg_gen_movcond_i32(TCG_COND_GE, REG(B11_8), REG(B7_4), t0, t1, t2);
+
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t2);
 	}
 	return;
     case 0x400d:		/* shld Rm,Rn */
 	{
-	    int label1 = gen_new_label();
-	    int label2 = gen_new_label();
-	    int label3 = gen_new_label();
-	    TCGv shift;
-	    tcg_gen_brcondi_i32(TCG_COND_LT, REG(B7_4), 0, label1);
-	    /* Rm positive, shift to the left */
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_shl_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label3);
-	    /* Rm negative, shift to the right */
-	    gen_set_label(label1);
-            shift = tcg_temp_new();
-	    tcg_gen_andi_i32(shift, REG(B7_4), 0x1f);
-	    tcg_gen_brcondi_i32(TCG_COND_EQ, shift, 0, label2);
-	    tcg_gen_not_i32(shift, REG(B7_4));
-	    tcg_gen_andi_i32(shift, shift, 0x1f);
-	    tcg_gen_addi_i32(shift, shift, 1);
-	    tcg_gen_shr_i32(REG(B11_8), REG(B11_8), shift);
-	    tcg_temp_free(shift);
-	    tcg_gen_br(label3);
-	    /* Rm = -32 */
-	    gen_set_label(label2);
-	    tcg_gen_movi_i32(REG(B11_8), 0);
-	    gen_set_label(label3);
+            TCGv t0 = tcg_temp_new();
+            TCGv t1 = tcg_temp_new();
+            TCGv t2 = tcg_temp_new();
+
+            tcg_gen_andi_i32(t0, REG(B7_4), 0x1f);
+
+            /* positive case: shift to the left */
+            tcg_gen_shl_i32(t1, REG(B11_8), t0);
+
+            /* negative case: shift to the right in two steps to
+               correctly handle the -32 case */
+            tcg_gen_xori_i32(t0, t0, 0x1f);
+            tcg_gen_shr_i32(t2, REG(B11_8), t0);
+            tcg_gen_shri_i32(t2, t2, 1);
+
+            /* select between the two cases */
+            tcg_gen_movi_i32(t0, 0);
+            tcg_gen_movcond_i32(TCG_COND_GE, REG(B11_8), REG(B7_4), t0, t1, t2);
+
+            tcg_temp_free(t0);
+            tcg_temp_free(t1);
+            tcg_temp_free(t2);
 	}
 	return;
     case 0x3008:		/* sub Rm,Rn */
@@ -918,22 +898,15 @@
 	return;
     case 0x300a:		/* subc Rm,Rn */
         {
-            TCGv t0, t1, t2;
-            t0 = tcg_temp_new();
-            tcg_gen_andi_i32(t0, cpu_sr, SR_T);
+            TCGv t0, t1;
+            t0 = tcg_const_tl(0);
             t1 = tcg_temp_new();
-            tcg_gen_sub_i32(t1, REG(B11_8), REG(B7_4));
-            tcg_gen_sub_i32(t0, t1, t0);
-            t2 = tcg_temp_new();
-            tcg_gen_setcond_i32(TCG_COND_LTU, t2, REG(B11_8), t1);
-            tcg_gen_setcond_i32(TCG_COND_LTU, t1, t1, t0);
-            tcg_gen_or_i32(t1, t1, t2);
-            tcg_temp_free(t2);
-            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
-            tcg_temp_free(t1);
-            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_gen_add2_i32(t1, cpu_sr_t, cpu_sr_t, t0, REG(B7_4), t0);
+            tcg_gen_sub2_i32(REG(B11_8), cpu_sr_t,
+                             REG(B11_8), t0, t1, cpu_sr_t);
+            tcg_gen_andi_i32(cpu_sr_t, cpu_sr_t, 1);
             tcg_temp_free(t0);
+            tcg_temp_free(t1);
         }
 	return;
     case 0x300b:		/* subv Rm,Rn */
@@ -947,9 +920,7 @@
             tcg_gen_xor_i32(t2, REG(B11_8), REG(B7_4));
             tcg_gen_and_i32(t1, t1, t2);
             tcg_temp_free(t2);
-            tcg_gen_shri_i32(t1, t1, 31);
-            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_gen_shri_i32(cpu_sr_t, t1, 31);
             tcg_temp_free(t1);
             tcg_gen_mov_i32(REG(B11_8), t0);
             tcg_temp_free(t0);
@@ -959,7 +930,7 @@
 	{
 	    TCGv val = tcg_temp_new();
 	    tcg_gen_and_i32(val, REG(B7_4), REG(B11_8));
-	    gen_cmp_imm(TCG_COND_EQ, val, 0);
+            tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, val, 0);
 	    tcg_temp_free(val);
 	}
 	return;
@@ -1025,24 +996,19 @@
 	return;
     case 0xf00b: /* fmov {F,D,X}Rm,@-Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
+        TCGv addr = tcg_temp_new_i32();
+        tcg_gen_subi_i32(addr, REG(B11_8), 4);
         if (ctx->flags & FPSCR_SZ) {
-	    TCGv addr = tcg_temp_new_i32();
 	    int fr = XREG(B7_4);
-	    tcg_gen_subi_i32(addr, REG(B11_8), 4);
             tcg_gen_qemu_st_i32(cpu_fregs[fr+1], addr, ctx->memidx, MO_TEUL);
 	    tcg_gen_subi_i32(addr, addr, 4);
             tcg_gen_qemu_st_i32(cpu_fregs[fr], addr, ctx->memidx, MO_TEUL);
-	    tcg_gen_mov_i32(REG(B11_8), addr);
-	    tcg_temp_free(addr);
 	} else {
-	    TCGv addr;
-	    addr = tcg_temp_new_i32();
-	    tcg_gen_subi_i32(addr, REG(B11_8), 4);
             tcg_gen_qemu_st_i32(cpu_fregs[FREG(B7_4)], addr,
                                 ctx->memidx, MO_TEUL);
-	    tcg_gen_mov_i32(REG(B11_8), addr);
-	    tcg_temp_free(addr);
 	}
+        tcg_gen_mov_i32(REG(B11_8), addr);
+        tcg_temp_free(addr);
 	return;
     case 0xf006: /* fmov @(R0,Rm),{F,D,X}Rm - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
@@ -1210,7 +1176,7 @@
 	ctx->flags |= DELAY_SLOT_CONDITIONAL;
 	return;
     case 0x8800:		/* cmp/eq #imm,R0 */
-	gen_cmp_imm(TCG_COND_EQ, REG(0), B7_0s);
+        tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, REG(0), B7_0s);
 	return;
     case 0xc400:		/* mov.b @(disp,GBR),R0 */
 	{
@@ -1326,7 +1292,7 @@
 	{
 	    TCGv val = tcg_temp_new();
 	    tcg_gen_andi_i32(val, REG(0), B7_0);
-	    gen_cmp_imm(TCG_COND_EQ, val, 0);
+            tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, val, 0);
 	    tcg_temp_free(val);
 	}
 	return;
@@ -1336,7 +1302,7 @@
 	    tcg_gen_add_i32(val, REG(0), cpu_gbr);
             tcg_gen_qemu_ld_i32(val, val, ctx->memidx, MO_UB);
 	    tcg_gen_andi_i32(val, val, B7_0);
-	    gen_cmp_imm(TCG_COND_EQ, val, 0);
+            tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, val, 0);
 	    tcg_temp_free(val);
 	}
 	return;
@@ -1399,14 +1365,14 @@
 	ctx->delayed_pc = (uint32_t) - 1;
 	return;
     case 0x4015:		/* cmp/pl Rn */
-	gen_cmp_imm(TCG_COND_GT, REG(B11_8), 0);
+        tcg_gen_setcondi_i32(TCG_COND_GT, cpu_sr_t, REG(B11_8), 0);
 	return;
     case 0x4011:		/* cmp/pz Rn */
-	gen_cmp_imm(TCG_COND_GE, REG(B11_8), 0);
+        tcg_gen_setcondi_i32(TCG_COND_GE, cpu_sr_t, REG(B11_8), 0);
 	return;
     case 0x4010:		/* dt Rn */
 	tcg_gen_subi_i32(REG(B11_8), REG(B11_8), 1);
-	gen_cmp_imm(TCG_COND_EQ, REG(B11_8), 0);
+        tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, REG(B11_8), 0);
 	return;
     case 0x402b:		/* jmp @Rn */
 	CHECK_NOT_DELAY_SLOT
@@ -1423,15 +1389,21 @@
 	return;
     case 0x400e:		/* ldc Rm,SR */
 	CHECK_PRIVILEGED
-	tcg_gen_andi_i32(cpu_sr, REG(B11_8), 0x700083f3);
-	ctx->bstate = BS_STOP;
+        {
+            TCGv val = tcg_temp_new();
+            tcg_gen_andi_i32(val, REG(B11_8), 0x700083f3);
+            gen_write_sr(val);
+            tcg_temp_free(val);
+            ctx->bstate = BS_STOP;
+        }
 	return;
     case 0x4007:		/* ldc.l @Rm+,SR */
 	CHECK_PRIVILEGED
 	{
 	    TCGv val = tcg_temp_new();
             tcg_gen_qemu_ld_i32(val, REG(B11_8), ctx->memidx, MO_TESL);
-	    tcg_gen_andi_i32(cpu_sr, val, 0x700083f3);
+            tcg_gen_andi_i32(val, val, 0x700083f3);
+            gen_write_sr(val);
 	    tcg_temp_free(val);
 	    tcg_gen_addi_i32(REG(B11_8), REG(B11_8), 4);
 	    ctx->bstate = BS_STOP;
@@ -1439,15 +1411,18 @@
 	return;
     case 0x0002:		/* stc SR,Rn */
 	CHECK_PRIVILEGED
-	tcg_gen_mov_i32(REG(B11_8), cpu_sr);
+        gen_read_sr(REG(B11_8));
 	return;
     case 0x4003:		/* stc SR,@-Rn */
 	CHECK_PRIVILEGED
 	{
 	    TCGv addr = tcg_temp_new();
+            TCGv val = tcg_temp_new();
 	    tcg_gen_subi_i32(addr, REG(B11_8), 4);
-            tcg_gen_qemu_st_i32(cpu_sr, addr, ctx->memidx, MO_TEUL);
+            gen_read_sr(val);
+            tcg_gen_qemu_st_i32(val, addr, ctx->memidx, MO_TEUL);
 	    tcg_gen_mov_i32(REG(B11_8), addr);
+            tcg_temp_free(val);
 	    tcg_temp_free(addr);
 	}
 	return;
@@ -1545,7 +1520,7 @@
 	tcg_gen_addi_i32(REG(B11_8), REG(B11_8), 4);
 	return;
     case 0x0029:		/* movt Rn */
-	tcg_gen_andi_i32(REG(B11_8), cpu_sr, SR_T);
+        tcg_gen_mov_i32(REG(B11_8), cpu_sr_t);
 	return;
     case 0x0073:
         /* MOVCO.L
@@ -1554,9 +1529,8 @@
                0 -> LDST
         */
         if (ctx->features & SH_FEATURE_SH4A) {
-	    int label = gen_new_label();
-            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-	    tcg_gen_or_i32(cpu_sr, cpu_sr, cpu_ldst);
+            TCGLabel *label = gen_new_label();
+            tcg_gen_mov_i32(cpu_sr_t, cpu_ldst);
 	    tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ldst, 0, label);
             tcg_gen_qemu_st_i32(REG(0), REG(B11_8), ctx->memidx, MO_TEUL);
 	    gen_set_label(label);
@@ -1609,42 +1583,42 @@
     case 0x4024:		/* rotcl Rn */
 	{
 	    TCGv tmp = tcg_temp_new();
-	    tcg_gen_mov_i32(tmp, cpu_sr);
-	    gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 31);
+            tcg_gen_mov_i32(tmp, cpu_sr_t);
+            tcg_gen_shri_i32(cpu_sr_t, REG(B11_8), 31);
 	    tcg_gen_shli_i32(REG(B11_8), REG(B11_8), 1);
-	    gen_copy_bit_i32(REG(B11_8), 0, tmp, 0);
+            tcg_gen_or_i32(REG(B11_8), REG(B11_8), tmp);
 	    tcg_temp_free(tmp);
 	}
 	return;
     case 0x4025:		/* rotcr Rn */
 	{
 	    TCGv tmp = tcg_temp_new();
-	    tcg_gen_mov_i32(tmp, cpu_sr);
-	    gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 0);
+            tcg_gen_shli_i32(tmp, cpu_sr_t, 31);
+            tcg_gen_andi_i32(cpu_sr_t, REG(B11_8), 1);
 	    tcg_gen_shri_i32(REG(B11_8), REG(B11_8), 1);
-	    gen_copy_bit_i32(REG(B11_8), 31, tmp, 0);
+            tcg_gen_or_i32(REG(B11_8), REG(B11_8), tmp);
 	    tcg_temp_free(tmp);
 	}
 	return;
     case 0x4004:		/* rotl Rn */
 	tcg_gen_rotli_i32(REG(B11_8), REG(B11_8), 1);
-	gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 0);
+        tcg_gen_andi_i32(cpu_sr_t, REG(B11_8), 0);
 	return;
     case 0x4005:		/* rotr Rn */
-	gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 0);
+        tcg_gen_andi_i32(cpu_sr_t, REG(B11_8), 0);
 	tcg_gen_rotri_i32(REG(B11_8), REG(B11_8), 1);
 	return;
     case 0x4000:		/* shll Rn */
     case 0x4020:		/* shal Rn */
-	gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 31);
+        tcg_gen_shri_i32(cpu_sr_t, REG(B11_8), 31);
 	tcg_gen_shli_i32(REG(B11_8), REG(B11_8), 1);
 	return;
     case 0x4021:		/* shar Rn */
-	gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 0);
+        tcg_gen_andi_i32(cpu_sr_t, REG(B11_8), 1);
 	tcg_gen_sari_i32(REG(B11_8), REG(B11_8), 1);
 	return;
     case 0x4001:		/* shlr Rn */
-	gen_copy_bit_i32(cpu_sr, 0, REG(B11_8), 0);
+        tcg_gen_andi_i32(cpu_sr_t, REG(B11_8), 1);
 	tcg_gen_shri_i32(REG(B11_8), REG(B11_8), 1);
 	return;
     case 0x4008:		/* shll2 Rn */
@@ -1672,7 +1646,7 @@
 	    tcg_gen_mov_i32(addr, REG(B11_8));
 	    val = tcg_temp_local_new();
             tcg_gen_qemu_ld_i32(val, addr, ctx->memidx, MO_UB);
-	    gen_cmp_imm(TCG_COND_EQ, val, 0);
+            tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_sr_t, val, 0);
 	    tcg_gen_ori_i32(val, val, 0x80);
             tcg_gen_qemu_st_i32(val, addr, ctx->memidx, MO_UB);
 	    tcg_temp_free(val);
@@ -1827,10 +1801,6 @@
 {
     uint32_t old_flags = ctx->flags;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(ctx->pc);
-    }
-
     _decode_opc(ctx);
 
     if (old_flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) {
@@ -1857,16 +1827,12 @@
         gen_store_flags(ctx->flags);
 }
 
-static inline void
-gen_intermediate_code_internal(SuperHCPU *cpu, TranslationBlock *tb,
-                               bool search_pc)
+void gen_intermediate_code(CPUSH4State * env, struct TranslationBlock *tb)
 {
+    SuperHCPU *cpu = sh_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUSH4State *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    CPUBreakpoint *bp;
-    int i, ii;
     int num_insns;
     int max_insns;
 
@@ -1874,7 +1840,7 @@
     ctx.pc = pc_start;
     ctx.flags = (uint32_t)tb->flags;
     ctx.bstate = BS_NONE;
-    ctx.memidx = (ctx.flags & SR_MD) == 0 ? 1 : 0;
+    ctx.memidx = (ctx.flags & (1u << SR_MD)) == 0 ? 1 : 0;
     /* We don't know if the delayed pc came from a dynamic or static branch,
        so assume it is a dynamic branch.  */
     ctx.delayed_pc = -1; /* use delayed pc from env pointer */
@@ -1883,45 +1849,39 @@
     ctx.features = env->features;
     ctx.has_movcal = (ctx.flags & TB_FLAG_PENDING_MOVCA);
 
-    ii = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
-    gen_tb_start();
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
+    gen_tb_start(tb);
     while (ctx.bstate == BS_NONE && !tcg_op_buf_full()) {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (ctx.pc == bp->pc) {
-		    /* We have hit a breakpoint - make sure PC is up-to-date */
-		    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-                    gen_helper_debug(cpu_env);
-                    ctx.bstate = BS_BRANCH;
-		    break;
-		}
-	    }
-	}
-        if (search_pc) {
-            i = tcg_op_buf_count();
-            if (ii < i) {
-                ii++;
-                while (ii < i)
-                    tcg_ctx.gen_opc_instr_start[ii++] = 0;
-            }
-            tcg_ctx.gen_opc_pc[ii] = ctx.pc;
-            gen_opc_hflags[ii] = ctx.flags;
-            tcg_ctx.gen_opc_instr_start[ii] = 1;
-            tcg_ctx.gen_opc_icount[ii] = num_insns;
+        tcg_gen_insn_start(ctx.pc, ctx.flags);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, ctx.pc, BP_ANY))) {
+            /* We have hit a breakpoint - make sure PC is up-to-date */
+            tcg_gen_movi_i32(cpu_pc, ctx.pc);
+            gen_helper_debug(cpu_env);
+            ctx.bstate = BS_BRANCH;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            ctx.pc += 2;
+            break;
         }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
-#if 0
-	fprintf(stderr, "Loading opcode at address 0x%08x\n", ctx.pc);
-	fflush(stderr);
-#endif
+        }
+
         ctx.opcode = cpu_lduw_code(env, ctx.pc);
 	decode_opc(&ctx);
-        num_insns++;
 	ctx.pc += 2;
 	if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0)
 	    break;
@@ -1961,37 +1921,22 @@
 
     gen_tb_end(tb, num_insns);
 
-    if (search_pc) {
-        i = tcg_op_buf_count();
-        ii++;
-        while (ii <= i)
-            tcg_ctx.gen_opc_instr_start[ii++] = 0;
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
 	qemu_log("IN:\n");	/* , lookup_symbol(pc_start)); */
-        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(cs, pc_start, ctx.pc - pc_start, 0);
 	qemu_log("\n");
     }
 #endif
 }
 
-void gen_intermediate_code(CPUSH4State * env, struct TranslationBlock *tb)
+void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    gen_intermediate_code_internal(sh_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUSH4State * env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(sh_env_get_cpu(env), tb, true);
-}
-
-void restore_state_to_opc(CPUSH4State *env, TranslationBlock *tb, int pc_pos)
-{
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
-    env->flags = gen_opc_hflags[pc_pos];
+    env->pc = data[0];
+    env->flags = data[1];
 }
diff --git a/target-sparc/Makefile.objs b/target-sparc/Makefile.objs
index 1cd81cc..ec90569 100644
--- a/target-sparc/Makefile.objs
+++ b/target-sparc/Makefile.objs
@@ -1,4 +1,4 @@
-obj-$(CONFIG_SOFTMMU) += machine.o
+obj-$(CONFIG_SOFTMMU) += machine.o monitor.o
 obj-y += translate.o helper.o cpu.o
 obj-y += fop_helper.o cc_helper.o win_helper.o mmu_helper.o ldst_helper.o
 obj-$(TARGET_SPARC) += int32_helper.o
diff --git a/target-sparc/asi.h b/target-sparc/asi.h
new file mode 100644
index 0000000..c9a1849
--- /dev/null
+++ b/target-sparc/asi.h
@@ -0,0 +1,311 @@
+#ifndef _SPARC_ASI_H
+#define _SPARC_ASI_H
+
+/* asi.h:  Address Space Identifier values for the sparc.
+ *
+ * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu)
+ *
+ * Pioneer work for sun4m: Paul Hatchman (paul@sfe.com.au)
+ * Joint edition for sun4c+sun4m: Pete A. Zaitcev <zaitcev@ipmce.su>
+ */
+
+/* The first batch are for the sun4c. */
+
+#define ASI_NULL1           0x00
+#define ASI_NULL2           0x01
+
+/* sun4c and sun4 control registers and mmu/vac ops */
+#define ASI_CONTROL         0x02
+#define ASI_SEGMAP          0x03
+#define ASI_PTE             0x04
+#define ASI_HWFLUSHSEG      0x05
+#define ASI_HWFLUSHPAGE     0x06
+#define ASI_REGMAP          0x06
+#define ASI_HWFLUSHCONTEXT  0x07
+
+#define ASI_USERTXT         0x08
+#define ASI_KERNELTXT       0x09
+#define ASI_USERDATA        0x0a
+#define ASI_KERNELDATA      0x0b
+
+/* VAC Cache flushing on sun4c and sun4 */
+#define ASI_FLUSHSEG        0x0c
+#define ASI_FLUSHPG         0x0d
+#define ASI_FLUSHCTX        0x0e
+
+/* SPARCstation-5: only 6 bits are decoded. */
+/* wo = Write Only, rw = Read Write;        */
+/* ss = Single Size, as = All Sizes;        */
+#define ASI_M_RES00         0x00   /* Don't touch... */
+#define ASI_M_UNA01         0x01   /* Same here... */
+#define ASI_M_MXCC          0x02   /* Access to TI VIKING MXCC registers */
+#define ASI_M_FLUSH_PROBE   0x03   /* Reference MMU Flush/Probe; rw, ss */
+#define ASI_M_MMUREGS       0x04   /* MMU Registers; rw, ss */
+#define ASI_M_TLBDIAG       0x05   /* MMU TLB only Diagnostics */
+#define ASI_M_DIAGS         0x06   /* Reference MMU Diagnostics */
+#define ASI_M_IODIAG        0x07   /* MMU I/O TLB only Diagnostics */
+#define ASI_M_USERTXT       0x08   /* Same as ASI_USERTXT; rw, as */
+#define ASI_M_KERNELTXT     0x09   /* Same as ASI_KERNELTXT; rw, as */
+#define ASI_M_USERDATA      0x0A   /* Same as ASI_USERDATA; rw, as */
+#define ASI_M_KERNELDATA    0x0B   /* Same as ASI_KERNELDATA; rw, as */
+#define ASI_M_TXTC_TAG      0x0C   /* Instruction Cache Tag; rw, ss */
+#define ASI_M_TXTC_DATA     0x0D   /* Instruction Cache Data; rw, ss */
+#define ASI_M_DATAC_TAG     0x0E   /* Data Cache Tag; rw, ss */
+#define ASI_M_DATAC_DATA    0x0F   /* Data Cache Data; rw, ss */
+
+/* The following cache flushing ASIs work only with the 'sta'
+ * instruction. Results are unpredictable for 'swap' and 'ldstuba',
+ * so don't do it.
+ */
+
+/* These ASI flushes affect external caches too. */
+#define ASI_M_FLUSH_PAGE    0x10   /* Flush I&D Cache Line (page); wo, ss */
+#define ASI_M_FLUSH_SEG     0x11   /* Flush I&D Cache Line (seg); wo, ss */
+#define ASI_M_FLUSH_REGION  0x12   /* Flush I&D Cache Line (region); wo, ss */
+#define ASI_M_FLUSH_CTX     0x13   /* Flush I&D Cache Line (context); wo, ss */
+#define ASI_M_FLUSH_USER    0x14   /* Flush I&D Cache Line (user); wo, ss */
+
+/* Block-copy operations are available only on certain V8 cpus. */
+#define ASI_M_BCOPY         0x17   /* Block copy */
+
+/* These affect only the ICACHE and are Ross HyperSparc and TurboSparc specific. */
+#define ASI_M_IFLUSH_PAGE   0x18   /* Flush I Cache Line (page); wo, ss */
+#define ASI_M_IFLUSH_SEG    0x19   /* Flush I Cache Line (seg); wo, ss */
+#define ASI_M_IFLUSH_REGION 0x1A   /* Flush I Cache Line (region); wo, ss */
+#define ASI_M_IFLUSH_CTX    0x1B   /* Flush I Cache Line (context); wo, ss */
+#define ASI_M_IFLUSH_USER   0x1C   /* Flush I Cache Line (user); wo, ss */
+
+/* Block-fill operations are available on certain V8 cpus */
+#define ASI_M_BFILL         0x1F
+
+/* This allows direct access to main memory, actually 0x20 to 0x2f are
+ * the available ASI's for physical ram pass-through, but I don't have
+ * any idea what the other ones do....
+ */
+
+#define ASI_M_BYPASS       0x20   /* Reference MMU bypass; rw, as */
+#define ASI_M_FBMEM        0x29   /* Graphics card frame buffer access */
+#define ASI_M_VMEUS        0x2A   /* VME user 16-bit access */
+#define ASI_M_VMEPS        0x2B   /* VME priv 16-bit access */
+#define ASI_M_VMEUT        0x2C   /* VME user 32-bit access */
+#define ASI_M_VMEPT        0x2D   /* VME priv 32-bit access */
+#define ASI_M_SBUS         0x2E   /* Direct SBus access */
+#define ASI_M_CTL          0x2F   /* Control Space (ECC and MXCC are here) */
+
+
+/* This is ROSS HyperSparc only. */
+#define ASI_M_FLUSH_IWHOLE 0x31   /* Flush entire ICACHE; wo, ss */
+
+/* Tsunami/Viking/TurboSparc i/d cache flash clear. */
+#define ASI_M_IC_FLCLEAR   0x36
+#define ASI_M_DC_FLCLEAR   0x37
+
+#define ASI_M_DCDR         0x39   /* Data Cache Diagnostics Register rw, ss */
+
+#define ASI_M_VIKING_TMP1  0x40	  /* Emulation temporary 1 on Viking */
+/* only available on SuperSparc I */
+/* #define ASI_M_VIKING_TMP2  0x41 */  /* Emulation temporary 2 on Viking */
+
+#define ASI_M_ACTION       0x4c   /* Breakpoint Action Register (GNU/Viking) */
+
+/* LEON ASI */
+#define ASI_LEON_NOCACHE        0x01
+
+#define ASI_LEON_DCACHE_MISS    0x01
+
+#define ASI_LEON_CACHEREGS      0x02
+#define ASI_LEON_IFLUSH         0x10
+#define ASI_LEON_DFLUSH         0x11
+
+#define ASI_LEON_MMUFLUSH       0x18
+#define ASI_LEON_MMUREGS        0x19
+#define ASI_LEON_BYPASS         0x1c
+#define ASI_LEON_FLUSH_PAGE     0x10
+
+/* V9 Architecture mandary ASIs. */
+#define ASI_N			0x04 /* Nucleus				*/
+#define ASI_NL			0x0c /* Nucleus, little endian		*/
+#define ASI_AIUP		0x10 /* Primary, user			*/
+#define ASI_AIUS		0x11 /* Secondary, user			*/
+#define ASI_AIUPL		0x18 /* Primary, user, little endian	*/
+#define ASI_AIUSL		0x19 /* Secondary, user, little endian	*/
+#define ASI_P			0x80 /* Primary, implicit		*/
+#define ASI_S			0x81 /* Secondary, implicit		*/
+#define ASI_PNF			0x82 /* Primary, no fault		*/
+#define ASI_SNF			0x83 /* Secondary, no fault		*/
+#define ASI_PL			0x88 /* Primary, implicit, l-endian	*/
+#define ASI_SL			0x89 /* Secondary, implicit, l-endian	*/
+#define ASI_PNFL		0x8a /* Primary, no fault, l-endian	*/
+#define ASI_SNFL		0x8b /* Secondary, no fault, l-endian	*/
+
+/* SpitFire and later extended ASIs.  The "(III)" marker designates
+ * UltraSparc-III and later specific ASIs.  The "(CMT)" marker designates
+ * Chip Multi Threading specific ASIs.  "(NG)" designates Niagara specific
+ * ASIs, "(4V)" designates SUN4V specific ASIs.  "(NG4)" designates SPARC-T4
+ * and later ASIs.
+ */
+#define ASI_REAL                0x14 /* Real address, cachable          */
+#define ASI_PHYS_USE_EC		0x14 /* PADDR, E-cachable		*/
+#define ASI_REAL_IO             0x15 /* Real address, non-cachable      */
+#define ASI_PHYS_BYPASS_EC_E	0x15 /* PADDR, E-bit			*/
+#define ASI_BLK_AIUP_4V		0x16 /* (4V) Prim, user, block ld/st	*/
+#define ASI_BLK_AIUS_4V		0x17 /* (4V) Sec, user, block ld/st	*/
+#define ASI_REAL_L              0x1c /* Real address, cachable, LE      */
+#define ASI_PHYS_USE_EC_L	0x1c /* PADDR, E-cachable, little endian*/
+#define ASI_REAL_IO_L           0x1d /* Real address, non-cachable, LE  */
+#define ASI_PHYS_BYPASS_EC_E_L	0x1d /* PADDR, E-bit, little endian	*/
+#define ASI_BLK_AIUP_L_4V	0x1e /* (4V) Prim, user, block, l-endian*/
+#define ASI_BLK_AIUS_L_4V	0x1f /* (4V) Sec, user, block, l-endian	*/
+#define ASI_SCRATCHPAD		0x20 /* (4V) Scratch Pad Registers	*/
+#define ASI_MMU			0x21 /* (4V) MMU Context Registers	*/
+#define ASI_TWINX_AIUP          0x22 /* twin load, primary user         */
+#define ASI_TWINX_AIUS          0x23 /* twin load, secondary user       */
+#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 /* (NG) init-store, twin load,
+					 * secondary, user
+					 */
+#define ASI_NUCLEUS_QUAD_LDD	0x24 /* Cachable, qword load		*/
+#define ASI_QUEUE		0x25 /* (4V) Interrupt Queue Registers	*/
+#define ASI_TWINX_REAL          0x26 /* twin load, real, cachable       */
+#define ASI_QUAD_LDD_PHYS_4V	0x26 /* (4V) Physical, qword load	*/
+#define ASI_TWINX_N             0x27 /* twin load, nucleus              */
+#define ASI_TWINX_AIUP_L        0x2a /* twin load, primary user, LE     */
+#define ASI_TWINX_AIUS_L        0x2b /* twin load, secondary user, LE   */
+#define ASI_NUCLEUS_QUAD_LDD_L	0x2c /* Cachable, qword load, l-endian 	*/
+#define ASI_TWINX_REAL_L        0x2e /* twin load, real, cachable, LE   */
+#define ASI_QUAD_LDD_PHYS_L_4V	0x2e /* (4V) Phys, qword load, l-endian	*/
+#define ASI_TWINX_NL            0x2f /* twin load, nucleus, LE          */
+#define ASI_PCACHE_DATA_STATUS	0x30 /* (III) PCache data stat RAM diag	*/
+#define ASI_PCACHE_DATA		0x31 /* (III) PCache data RAM diag	*/
+#define ASI_PCACHE_TAG		0x32 /* (III) PCache tag RAM diag	*/
+#define ASI_PCACHE_SNOOP_TAG	0x33 /* (III) PCache snoop tag RAM diag	*/
+#define ASI_QUAD_LDD_PHYS	0x34 /* (III+) PADDR, qword load	*/
+#define ASI_WCACHE_VALID_BITS	0x38 /* (III) WCache Valid Bits diag	*/
+#define ASI_WCACHE_DATA		0x39 /* (III) WCache data RAM diag	*/
+#define ASI_WCACHE_TAG		0x3a /* (III) WCache tag RAM diag	*/
+#define ASI_WCACHE_SNOOP_TAG	0x3b /* (III) WCache snoop tag RAM diag	*/
+#define ASI_QUAD_LDD_PHYS_L	0x3c /* (III+) PADDR, qw-load, l-endian	*/
+#define ASI_SRAM_FAST_INIT	0x40 /* (III+) Fast SRAM init		*/
+#define ASI_CORE_AVAILABLE	0x41 /* (CMT) LP Available		*/
+#define ASI_CORE_ENABLE_STAT	0x41 /* (CMT) LP Enable Status		*/
+#define ASI_CORE_ENABLE		0x41 /* (CMT) LP Enable RW		*/
+#define ASI_XIR_STEERING	0x41 /* (CMT) XIR Steering RW		*/
+#define ASI_CORE_RUNNING_RW	0x41 /* (CMT) LP Running RW		*/
+#define ASI_CORE_RUNNING_W1S	0x41 /* (CMT) LP Running Write-One Set	*/
+#define ASI_CORE_RUNNING_W1C	0x41 /* (CMT) LP Running Write-One Clr	*/
+#define ASI_CORE_RUNNING_STAT	0x41 /* (CMT) LP Running Status		*/
+#define ASI_CMT_ERROR_STEERING	0x41 /* (CMT) Error Steering RW		*/
+#define ASI_DCACHE_INVALIDATE	0x42 /* (III) DCache Invalidate diag	*/
+#define ASI_DCACHE_UTAG		0x43 /* (III) DCache uTag diag		*/
+#define ASI_DCACHE_SNOOP_TAG	0x44 /* (III) DCache snoop tag RAM diag	*/
+#define ASI_LSU_CONTROL		0x45 /* Load-store control unit		*/
+#define ASI_DCU_CONTROL_REG	0x45 /* (III) DCache Unit Control reg	*/
+#define ASI_DCACHE_DATA		0x46 /* DCache data-ram diag access	*/
+#define ASI_DCACHE_TAG		0x47 /* Dcache tag/valid ram diag access*/
+#define ASI_INTR_DISPATCH_STAT	0x48 /* IRQ vector dispatch status	*/
+#define ASI_INTR_RECEIVE	0x49 /* IRQ vector receive status	*/
+#define ASI_UPA_CONFIG		0x4a /* UPA config space		*/
+#define ASI_JBUS_CONFIG		0x4a /* (IIIi) JBUS Config Register	*/
+#define ASI_SAFARI_CONFIG	0x4a /* (III) Safari Config Register	*/
+#define ASI_SAFARI_ADDRESS	0x4a /* (III) Safari Address Register	*/
+#define ASI_ESTATE_ERROR_EN	0x4b /* E-cache error enable space	*/
+#define ASI_AFSR		0x4c /* Async fault status register	*/
+#define ASI_AFAR		0x4d /* Async fault address register	*/
+#define ASI_EC_TAG_DATA		0x4e /* E-cache tag/valid ram diag acc	*/
+#define ASI_IMMU		0x50 /* Insn-MMU main register space	*/
+#define ASI_IMMU_TSB_8KB_PTR	0x51 /* Insn-MMU 8KB TSB pointer reg	*/
+#define ASI_IMMU_TSB_64KB_PTR	0x52 /* Insn-MMU 64KB TSB pointer reg	*/
+#define ASI_ITLB_DATA_IN	0x54 /* Insn-MMU TLB data in reg	*/
+#define ASI_ITLB_DATA_ACCESS	0x55 /* Insn-MMU TLB data access reg	*/
+#define ASI_ITLB_TAG_READ	0x56 /* Insn-MMU TLB tag read reg	*/
+#define ASI_IMMU_DEMAP		0x57 /* Insn-MMU TLB demap		*/
+#define ASI_DMMU		0x58 /* Data-MMU main register space	*/
+#define ASI_DMMU_TSB_8KB_PTR	0x59 /* Data-MMU 8KB TSB pointer reg	*/
+#define ASI_DMMU_TSB_64KB_PTR	0x5a /* Data-MMU 16KB TSB pointer reg	*/
+#define ASI_DMMU_TSB_DIRECT_PTR	0x5b /* Data-MMU TSB direct pointer reg	*/
+#define ASI_DTLB_DATA_IN	0x5c /* Data-MMU TLB data in reg	*/
+#define ASI_DTLB_DATA_ACCESS	0x5d /* Data-MMU TLB data access reg	*/
+#define ASI_DTLB_TAG_READ	0x5e /* Data-MMU TLB tag read reg	*/
+#define ASI_DMMU_DEMAP		0x5f /* Data-MMU TLB demap		*/
+#define ASI_IIU_INST_TRAP	0x60 /* (III) Instruction Breakpoint	*/
+#define ASI_INTR_ID		0x63 /* (CMT) Interrupt ID register	*/
+#define ASI_CORE_ID		0x63 /* (CMT) LP ID register		*/
+#define ASI_CESR_ID		0x63 /* (CMT) CESR ID register		*/
+#define ASI_IC_INSTR		0x66 /* Insn cache instrucion ram diag	*/
+#define ASI_IC_TAG		0x67 /* Insn cache tag/valid ram diag 	*/
+#define ASI_IC_STAG		0x68 /* (III) Insn cache snoop tag ram	*/
+#define ASI_IC_PRE_DECODE	0x6e /* Insn cache pre-decode ram diag	*/
+#define ASI_IC_NEXT_FIELD	0x6f /* Insn cache next-field ram diag	*/
+#define ASI_BRPRED_ARRAY	0x6f /* (III) Branch Prediction RAM diag*/
+#define ASI_BLK_AIUP		0x70 /* Primary, user, block load/store	*/
+#define ASI_BLK_AIUS		0x71 /* Secondary, user, block ld/st	*/
+#define ASI_MCU_CTRL_REG	0x72 /* (III) Memory controller regs	*/
+#define ASI_EC_DATA		0x74 /* (III) E-cache data staging reg	*/
+#define ASI_EC_CTRL		0x75 /* (III) E-cache control reg	*/
+#define ASI_EC_W		0x76 /* E-cache diag write access	*/
+#define ASI_UDB_ERROR_W		0x77 /* External UDB error regs W	*/
+#define ASI_UDB_CONTROL_W	0x77 /* External UDB control regs W	*/
+#define ASI_INTR_W		0x77 /* IRQ vector dispatch write	*/
+#define ASI_INTR_DATAN_W	0x77 /* (III) Out irq vector data reg N	*/
+#define ASI_INTR_DISPATCH_W	0x77 /* (III) Interrupt vector dispatch	*/
+#define ASI_BLK_AIUPL		0x78 /* Primary, user, little, blk ld/st*/
+#define ASI_BLK_AIUSL		0x79 /* Secondary, user, little, blk ld/st*/
+#define ASI_EC_R		0x7e /* E-cache diag read access	*/
+#define ASI_UDBH_ERROR_R	0x7f /* External UDB error regs rd hi	*/
+#define ASI_UDBL_ERROR_R	0x7f /* External UDB error regs rd low	*/
+#define ASI_UDBH_CONTROL_R	0x7f /* External UDB control regs rd hi	*/
+#define ASI_UDBL_CONTROL_R	0x7f /* External UDB control regs rd low*/
+#define ASI_INTR_R		0x7f /* IRQ vector dispatch read	*/
+#define ASI_INTR_DATAN_R	0x7f /* (III) In irq vector data reg N	*/
+#define ASI_PIC			0xb0 /* (NG4) PIC registers		*/
+#define ASI_PST8_P		0xc0 /* Primary, 8 8-bit, partial	*/
+#define ASI_PST8_S		0xc1 /* Secondary, 8 8-bit, partial	*/
+#define ASI_PST16_P		0xc2 /* Primary, 4 16-bit, partial	*/
+#define ASI_PST16_S		0xc3 /* Secondary, 4 16-bit, partial	*/
+#define ASI_PST32_P		0xc4 /* Primary, 2 32-bit, partial	*/
+#define ASI_PST32_S		0xc5 /* Secondary, 2 32-bit, partial	*/
+#define ASI_PST8_PL		0xc8 /* Primary, 8 8-bit, partial, L	*/
+#define ASI_PST8_SL		0xc9 /* Secondary, 8 8-bit, partial, L	*/
+#define ASI_PST16_PL		0xca /* Primary, 4 16-bit, partial, L	*/
+#define ASI_PST16_SL		0xcb /* Secondary, 4 16-bit, partial, L	*/
+#define ASI_PST32_PL		0xcc /* Primary, 2 32-bit, partial, L	*/
+#define ASI_PST32_SL		0xcd /* Secondary, 2 32-bit, partial, L	*/
+#define ASI_FL8_P		0xd0 /* Primary, 1 8-bit, fpu ld/st	*/
+#define ASI_FL8_S		0xd1 /* Secondary, 1 8-bit, fpu ld/st	*/
+#define ASI_FL16_P		0xd2 /* Primary, 1 16-bit, fpu ld/st	*/
+#define ASI_FL16_S		0xd3 /* Secondary, 1 16-bit, fpu ld/st	*/
+#define ASI_FL8_PL		0xd8 /* Primary, 1 8-bit, fpu ld/st, L	*/
+#define ASI_FL8_SL		0xd9 /* Secondary, 1 8-bit, fpu ld/st, L*/
+#define ASI_FL16_PL		0xda /* Primary, 1 16-bit, fpu ld/st, L	*/
+#define ASI_FL16_SL		0xdb /* Secondary, 1 16-bit, fpu ld/st,L*/
+#define ASI_BLK_COMMIT_P	0xe0 /* Primary, blk store commit	*/
+#define ASI_BLK_COMMIT_S	0xe1 /* Secondary, blk store commit	*/
+#define ASI_TWINX_P             0xe2 /* twin load, primary implicit     */
+#define ASI_BLK_INIT_QUAD_LDD_P	0xe2 /* (NG) init-store, twin load,
+				      * primary, implicit */
+#define ASI_TWINX_S             0xe3 /* twin load, secondary implicit   */
+#define ASI_BLK_INIT_QUAD_LDD_S	0xe3 /* (NG) init-store, twin load,
+				      * secondary, implicit */
+#define ASI_TWINX_PL            0xea /* twin load, primary implicit, LE */
+#define ASI_TWINX_SL            0xeb /* twin load, secondary implicit, LE */
+#define ASI_BLK_P		0xf0 /* Primary, blk ld/st		*/
+#define ASI_BLK_S		0xf1 /* Secondary, blk ld/st		*/
+#define ASI_ST_BLKINIT_MRU_P	0xf2 /* (NG4) init-store, twin load,
+				      * Most-Recently-Used, primary,
+				      * implicit
+				      */
+#define ASI_ST_BLKINIT_MRU_S	0xf2 /* (NG4) init-store, twin load,
+				      * Most-Recently-Used, secondary,
+				      * implicit
+				      */
+#define ASI_BLK_PL		0xf8 /* Primary, blk ld/st, little	*/
+#define ASI_BLK_SL		0xf9 /* Secondary, blk ld/st, little	*/
+#define ASI_ST_BLKINIT_MRU_PL	0xfa /* (NG4) init-store, twin load,
+				      * Most-Recently-Used, primary,
+				      * implicit, little-endian
+				      */
+#define ASI_ST_BLKINIT_MRU_SL	0xfb /* (NG4) init-store, twin load,
+				      * Most-Recently-Used, secondary,
+				      * implicit, little-endian
+				      */
+
+#endif /* _SPARC_ASI_H */
diff --git a/target-sparc/cc_helper.c b/target-sparc/cc_helper.c
index 35dab73..a410a0b 100644
--- a/target-sparc/cc_helper.c
+++ b/target-sparc/cc_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -199,10 +200,7 @@
 
 static uint32_t compute_C_addx_xcc(CPUSPARCState *env)
 {
-    uint32_t ret;
-
-    ret = get_C_addx_xcc(CC_DST, CC_SRC, CC_SRC2);
-    return ret;
+    return get_C_addx_xcc(CC_DST, CC_SRC, CC_SRC2);
 }
 #endif
 
@@ -218,10 +216,7 @@
 
 static uint32_t compute_C_addx(CPUSPARCState *env)
 {
-    uint32_t ret;
-
-    ret = get_C_addx_icc(CC_DST, CC_SRC, CC_SRC2);
-    return ret;
+    return get_C_addx_icc(CC_DST, CC_SRC, CC_SRC2);
 }
 
 static inline uint32_t get_V_tag_icc(target_ulong src1, target_ulong src2)
@@ -364,10 +359,7 @@
 
 static uint32_t compute_C_subx_xcc(CPUSPARCState *env)
 {
-    uint32_t ret;
-
-    ret = get_C_subx_xcc(CC_DST, CC_SRC, CC_SRC2);
-    return ret;
+    return get_C_subx_xcc(CC_DST, CC_SRC, CC_SRC2);
 }
 #endif
 
@@ -383,10 +375,7 @@
 
 static uint32_t compute_C_subx(CPUSPARCState *env)
 {
-    uint32_t ret;
-
-    ret = get_C_subx_icc(CC_DST, CC_SRC, CC_SRC2);
-    return ret;
+    return get_C_subx_icc(CC_DST, CC_SRC, CC_SRC2);
 }
 
 static uint32_t compute_all_tsub(CPUSPARCState *env)
@@ -478,8 +467,5 @@
 
 uint32_t helper_compute_C_icc(CPUSPARCState *env)
 {
-    uint32_t ret;
-
-    ret = icc_table[CC_OP].compute_c(env) >> PSR_CARRY_SHIFT;
-    return ret;
+    return icc_table[CC_OP].compute_c(env) >> PSR_CARRY_SHIFT;
 }
diff --git a/target-sparc/cpu-qom.h b/target-sparc/cpu-qom.h
index 477c4d5..f63af72 100644
--- a/target-sparc/cpu-qom.h
+++ b/target-sparc/cpu-qom.h
@@ -21,7 +21,6 @@
 #define QEMU_SPARC_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #ifdef TARGET_SPARC64
 #define TYPE_SPARC_CPU "sparc64-cpu"
@@ -52,37 +51,6 @@
     void (*parent_reset)(CPUState *cpu);
 } SPARCCPUClass;
 
-/**
- * SPARCCPU:
- * @env: #CPUSPARCState
- *
- * A SPARC CPU.
- */
-typedef struct SPARCCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUSPARCState env;
-} SPARCCPU;
-
-static inline SPARCCPU *sparc_env_get_cpu(CPUSPARCState *env)
-{
-    return container_of(env, SPARCCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(sparc_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(SPARCCPU, env)
-
-void sparc_cpu_do_interrupt(CPUState *cpu);
-void sparc_cpu_dump_state(CPUState *cpu, FILE *f,
-                          fprintf_function cpu_fprintf, int flags);
-hwaddr sparc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int sparc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int sparc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-void QEMU_NORETURN sparc_cpu_do_unaligned_access(CPUState *cpu,
-                                                 vaddr addr, int is_write,
-                                                 int is_user, uintptr_t retaddr);
+typedef struct SPARCCPU SPARCCPU;
 
 #endif
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index aa7626c..e4089f2 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -17,8 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu/error-report.h"
+#include "exec/exec-all.h"
 
 //#define DEBUG_FEATURES
 
@@ -90,9 +93,19 @@
     return false;
 }
 
+static void cpu_sparc_disas_set_info(CPUState *cpu, disassemble_info *info)
+{
+    info->print_insn = print_insn_sparc;
+#ifdef TARGET_SPARC64
+    info->mach = bfd_mach_sparc_v9b;
+#endif
+}
+
+static void sparc_cpu_parse_features(CPUState *cs, char *features,
+                                     Error **errp);
+
 static int cpu_sparc_register(SPARCCPU *cpu, const char *cpu_model)
 {
-    CPUClass *cc = CPU_GET_CLASS(cpu);
     CPUSPARCState *env = &cpu->env;
     char *s = g_strdup(cpu_model);
     char *featurestr, *name = strtok(s, ",");
@@ -108,11 +121,10 @@
     memcpy(env->def, def, sizeof(*def));
 
     featurestr = strtok(NULL, ",");
-    cc->parse_features(CPU(cpu), featurestr, &err);
+    sparc_cpu_parse_features(CPU(cpu), featurestr, &err);
     g_free(s);
     if (err) {
-        error_report("%s", error_get_pretty(err));
-        error_free(err);
+        error_report_err(err);
         return -1;
     }
 
@@ -803,7 +815,7 @@
     CPUSPARCState *env = &cpu->env;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     if (tcg_enabled()) {
         gen_intermediate_code_init(env);
@@ -830,7 +842,6 @@
     scc->parent_reset = cc->reset;
     cc->reset = sparc_cpu_reset;
 
-    cc->parse_features = sparc_cpu_parse_features;
     cc->has_work = sparc_cpu_has_work;
     cc->do_interrupt = sparc_cpu_do_interrupt;
     cc->cpu_exec_interrupt = sparc_cpu_exec_interrupt;
@@ -848,13 +859,22 @@
     cc->do_unassigned_access = sparc_cpu_unassigned_access;
     cc->do_unaligned_access = sparc_cpu_do_unaligned_access;
     cc->get_phys_page_debug = sparc_cpu_get_phys_page_debug;
+    cc->vmsd = &vmstate_sparc_cpu;
 #endif
+    cc->disas_set_info = cpu_sparc_disas_set_info;
 
 #if defined(TARGET_SPARC64) && !defined(TARGET_ABI32)
     cc->gdb_num_core_regs = 86;
 #else
     cc->gdb_num_core_regs = 72;
 #endif
+
+    /*
+     * Reason: sparc_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo sparc_cpu_type_info = {
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index 836f87f..3c4a850 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -1,9 +1,9 @@
-#ifndef CPU_SPARC_H
-#define CPU_SPARC_H
+#ifndef SPARC_CPU_H
+#define SPARC_CPU_H
 
-#include "config.h"
 #include "qemu-common.h"
 #include "qemu/bswap.h"
+#include "cpu-qom.h"
 
 #define ALIGNED_ONLY
 
@@ -31,14 +31,6 @@
 
 #include "fpu/softfloat.h"
 
-#define TARGET_HAS_ICE 1
-
-#if !defined(TARGET_SPARC64)
-#define ELF_MACHINE     EM_SPARC
-#else
-#define ELF_MACHINE     EM_SPARCV9
-#endif
-
 /*#define EXCP_INTERRUPT 0x100*/
 
 /* trap definitions */
@@ -238,6 +230,7 @@
     uint32_t tt;
 } trap_state;
 #endif
+#define TARGET_INSN_START_EXTRA_WORDS 1
 
 typedef struct sparc_def_t {
     const char *name;
@@ -373,16 +366,14 @@
     uint32_t    frequency;
     uint32_t    disabled;
     uint64_t    disabled_mask;
+    uint32_t    npt;
+    uint64_t    npt_mask;
     int64_t     clock_offset;
     QEMUTimer  *qtimer;
 };
 
 typedef struct CPUTimer CPUTimer;
 
-struct QEMUFile;
-void cpu_put_timer(struct QEMUFile *f, CPUTimer *s);
-void cpu_get_timer(struct QEMUFile *f, CPUTimer *s);
-
 typedef struct CPUSPARCState CPUSPARCState;
 
 struct CPUSPARCState {
@@ -516,7 +507,44 @@
     uint32_t cache_control;
 };
 
-#include "cpu-qom.h"
+/**
+ * SPARCCPU:
+ * @env: #CPUSPARCState
+ *
+ * A SPARC CPU.
+ */
+struct SPARCCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUSPARCState env;
+};
+
+static inline SPARCCPU *sparc_env_get_cpu(CPUSPARCState *env)
+{
+    return container_of(env, SPARCCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(sparc_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(SPARCCPU, env)
+
+#ifndef CONFIG_USER_ONLY
+extern const struct VMStateDescription vmstate_sparc_cpu;
+#endif
+
+void sparc_cpu_do_interrupt(CPUState *cpu);
+void sparc_cpu_dump_state(CPUState *cpu, FILE *f,
+                          fprintf_function cpu_fprintf, int flags);
+hwaddr sparc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int sparc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int sparc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void QEMU_NORETURN sparc_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                                 MMUAccessType access_type,
+                                                 int mmu_idx,
+                                                 uintptr_t retaddr,
+                                                 unsigned size);
 
 #ifndef NO_CPU_IO_DEFS
 /* cpu_init.c */
@@ -539,11 +567,11 @@
 void gen_intermediate_code_init(CPUSPARCState *env);
 
 /* cpu-exec.c */
-int cpu_sparc_exec(CPUSPARCState *s);
 
 /* win_helper.c */
 target_ulong cpu_get_psr(CPUSPARCState *env1);
 void cpu_put_psr(CPUSPARCState *env1, target_ulong val);
+void cpu_put_psr_raw(CPUSPARCState *env1, target_ulong val);
 #ifdef TARGET_SPARC64
 target_ulong cpu_get_ccr(CPUSPARCState *env1);
 void cpu_put_ccr(CPUSPARCState *env1, target_ulong val);
@@ -596,23 +624,12 @@
 int cpu_sparc_signal_handler(int host_signum, void *pinfo, void *puc);
 
 #ifndef NO_CPU_IO_DEFS
-static inline CPUSPARCState *cpu_init(const char *cpu_model)
-{
-    SPARCCPU *cpu = cpu_sparc_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_sparc_init(cpu_model))
 #endif
 
-#define cpu_exec cpu_sparc_exec
-#define cpu_gen_code cpu_sparc_gen_code
 #define cpu_signal_handler cpu_sparc_signal_handler
 #define cpu_list sparc_cpu_list
 
-#define CPU_SAVE_VERSION 7
-
 /* MMU modes definitions */
 #if defined (TARGET_SPARC64)
 #define MMU_USER_IDX   0
@@ -651,7 +668,7 @@
 }
 #endif
 
-static inline int cpu_mmu_index(CPUSPARCState *env1)
+static inline int cpu_mmu_index(CPUSPARCState *env1, bool ifetch)
 {
 #if defined(CONFIG_USER_ONLY)
     return MMU_USER_IDX;
@@ -703,34 +720,34 @@
 trap_state* cpu_tsptr(CPUSPARCState* env);
 #endif
 
-#define TB_FLAG_FPU_ENABLED (1 << 4)
-#define TB_FLAG_AM_ENABLED (1 << 5)
+#define TB_FLAG_MMU_MASK     7
+#define TB_FLAG_FPU_ENABLED  (1 << 4)
+#define TB_FLAG_AM_ENABLED   (1 << 5)
+#define TB_FLAG_ASI_SHIFT    24
 
 static inline void cpu_get_tb_cpu_state(CPUSPARCState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *pflags)
 {
+    uint32_t flags;
     *pc = env->pc;
     *cs_base = env->npc;
+    flags = cpu_mmu_index(env, false);
 #ifdef TARGET_SPARC64
-    // AM . Combined FPU enable bits . PRIV . DMMU enabled . IMMU enabled
-    *flags = (env->pstate & PS_PRIV)               /* 2 */
-        | ((env->lsu & (DMMU_E | IMMU_E)) >> 2)    /* 1, 0 */
-        | ((env->tl & 0xff) << 8)
-        | (env->dmmu.mmu_primary_context << 16);   /* 16... */
     if (env->pstate & PS_AM) {
-        *flags |= TB_FLAG_AM_ENABLED;
+        flags |= TB_FLAG_AM_ENABLED;
     }
-    if ((env->def->features & CPU_FEATURE_FLOAT) && (env->pstate & PS_PEF)
+    if ((env->def->features & CPU_FEATURE_FLOAT)
+        && (env->pstate & PS_PEF)
         && (env->fprs & FPRS_FEF)) {
-        *flags |= TB_FLAG_FPU_ENABLED;
+        flags |= TB_FLAG_FPU_ENABLED;
     }
+    flags |= env->asi << TB_FLAG_ASI_SHIFT;
 #else
-    // FPU enable . Supervisor
-    *flags = env->psrs;
     if ((env->def->features & CPU_FEATURE_FLOAT) && env->psref) {
-        *flags |= TB_FLAG_FPU_ENABLED;
+        flags |= TB_FLAG_FPU_ENABLED;
     }
 #endif
+    *pflags = flags;
 }
 
 static inline bool tb_fpu_enabled(int tb_flags)
@@ -751,6 +768,4 @@
 #endif
 }
 
-#include "exec/exec-all.h"
-
 #endif
diff --git a/target-sparc/fop_helper.c b/target-sparc/fop_helper.c
index ee4592e..c7fb176 100644
--- a/target-sparc/fop_helper.c
+++ b/target-sparc/fop_helper.c
@@ -17,49 +17,62 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/helper-proto.h"
 
 #define QT0 (env->qt0)
 #define QT1 (env->qt1)
 
-static void check_ieee_exceptions(CPUSPARCState *env)
+static target_ulong do_check_ieee_exceptions(CPUSPARCState *env, uintptr_t ra)
 {
-    target_ulong status;
+    target_ulong status = get_float_exception_flags(&env->fp_status);
+    target_ulong fsr = env->fsr;
 
-    status = get_float_exception_flags(&env->fp_status);
-    if (status) {
+    if (unlikely(status)) {
+        /* Keep exception flags clear for next time.  */
+        set_float_exception_flags(0, &env->fp_status);
+
         /* Copy IEEE 754 flags into FSR */
         if (status & float_flag_invalid) {
-            env->fsr |= FSR_NVC;
+            fsr |= FSR_NVC;
         }
         if (status & float_flag_overflow) {
-            env->fsr |= FSR_OFC;
+            fsr |= FSR_OFC;
         }
         if (status & float_flag_underflow) {
-            env->fsr |= FSR_UFC;
+            fsr |= FSR_UFC;
         }
         if (status & float_flag_divbyzero) {
-            env->fsr |= FSR_DZC;
+            fsr |= FSR_DZC;
         }
         if (status & float_flag_inexact) {
-            env->fsr |= FSR_NXC;
+            fsr |= FSR_NXC;
         }
 
-        if ((env->fsr & FSR_CEXC_MASK) & ((env->fsr & FSR_TEM_MASK) >> 23)) {
-            /* Unmasked exception, generate a trap */
-            env->fsr |= FSR_FTT_IEEE_EXCP;
-            helper_raise_exception(env, TT_FP_EXCP);
+        if ((fsr & FSR_CEXC_MASK) & ((fsr & FSR_TEM_MASK) >> 23)) {
+            CPUState *cs = CPU(sparc_env_get_cpu(env));
+
+            /* Unmasked exception, generate a trap.  Note that while
+               the helper is marked as NO_WG, we can get away with
+               writing to cpu state along the exception path, since
+               TCG generated code will never see the write.  */
+            env->fsr = fsr | FSR_FTT_IEEE_EXCP;
+            cs->exception_index = TT_FP_EXCP;
+            cpu_loop_exit_restore(cs, ra);
         } else {
             /* Accumulate exceptions */
-            env->fsr |= (env->fsr & FSR_CEXC_MASK) << 5;
+            fsr |= (fsr & FSR_CEXC_MASK) << 5;
         }
     }
+
+    return fsr;
 }
 
-static inline void clear_float_exceptions(CPUSPARCState *env)
+target_ulong helper_check_ieee_exceptions(CPUSPARCState *env)
 {
-    set_float_exception_flags(0, &env->fp_status);
+    return do_check_ieee_exceptions(env, GETPC());
 }
 
 #define F_HELPER(name, p) void helper_f##name##p(CPUSPARCState *env)
@@ -68,26 +81,16 @@
     float32 helper_f ## name ## s (CPUSPARCState *env, float32 src1, \
                                    float32 src2)                \
     {                                                           \
-        float32 ret;                                            \
-        clear_float_exceptions(env);                            \
-        ret = float32_ ## name (src1, src2, &env->fp_status);   \
-        check_ieee_exceptions(env);                             \
-        return ret;                                             \
+        return float32_ ## name (src1, src2, &env->fp_status);  \
     }                                                           \
     float64 helper_f ## name ## d (CPUSPARCState * env, float64 src1,\
                                    float64 src2)                \
     {                                                           \
-        float64 ret;                                            \
-        clear_float_exceptions(env);                            \
-        ret = float64_ ## name (src1, src2, &env->fp_status);   \
-        check_ieee_exceptions(env);                             \
-        return ret;                                             \
+        return float64_ ## name (src1, src2, &env->fp_status);  \
     }                                                           \
     F_HELPER(name, q)                                           \
     {                                                           \
-        clear_float_exceptions(env);                            \
         QT0 = float128_ ## name (QT0, QT1, &env->fp_status);    \
-        check_ieee_exceptions(env);                             \
     }
 
 F_BINOP(add);
@@ -98,22 +101,16 @@
 
 float64 helper_fsmuld(CPUSPARCState *env, float32 src1, float32 src2)
 {
-    float64 ret;
-    clear_float_exceptions(env);
-    ret = float64_mul(float32_to_float64(src1, &env->fp_status),
-                      float32_to_float64(src2, &env->fp_status),
-                      &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float64_mul(float32_to_float64(src1, &env->fp_status),
+                       float32_to_float64(src2, &env->fp_status),
+                       &env->fp_status);
 }
 
 void helper_fdmulq(CPUSPARCState *env, float64 src1, float64 src2)
 {
-    clear_float_exceptions(env);
     QT0 = float128_mul(float64_to_float128(src1, &env->fp_status),
                        float64_to_float128(src2, &env->fp_status),
                        &env->fp_status);
-    check_ieee_exceptions(env);
 }
 
 float32 helper_fnegs(float32 src)
@@ -136,48 +133,32 @@
 /* Integer to float conversion.  */
 float32 helper_fitos(CPUSPARCState *env, int32_t src)
 {
-    /* Inexact error possible converting int to float.  */
-    float32 ret;
-    clear_float_exceptions(env);
-    ret = int32_to_float32(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return int32_to_float32(src, &env->fp_status);
 }
 
 float64 helper_fitod(CPUSPARCState *env, int32_t src)
 {
-    /* No possible exceptions converting int to double.  */
     return int32_to_float64(src, &env->fp_status);
 }
 
 void helper_fitoq(CPUSPARCState *env, int32_t src)
 {
-    /* No possible exceptions converting int to long double.  */
     QT0 = int32_to_float128(src, &env->fp_status);
 }
 
 #ifdef TARGET_SPARC64
 float32 helper_fxtos(CPUSPARCState *env, int64_t src)
 {
-    float32 ret;
-    clear_float_exceptions(env);
-    ret = int64_to_float32(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return int64_to_float32(src, &env->fp_status);
 }
 
 float64 helper_fxtod(CPUSPARCState *env, int64_t src)
 {
-    float64 ret;
-    clear_float_exceptions(env);
-    ret = int64_to_float64(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return int64_to_float64(src, &env->fp_status);
 }
 
 void helper_fxtoq(CPUSPARCState *env, int64_t src)
 {
-    /* No possible exceptions converting long long to long double.  */
     QT0 = int64_to_float128(src, &env->fp_status);
 }
 #endif
@@ -186,108 +167,64 @@
 /* floating point conversion */
 float32 helper_fdtos(CPUSPARCState *env, float64 src)
 {
-    float32 ret;
-    clear_float_exceptions(env);
-    ret = float64_to_float32(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float64_to_float32(src, &env->fp_status);
 }
 
 float64 helper_fstod(CPUSPARCState *env, float32 src)
 {
-    float64 ret;
-    clear_float_exceptions(env);
-    ret = float32_to_float64(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float32_to_float64(src, &env->fp_status);
 }
 
 float32 helper_fqtos(CPUSPARCState *env)
 {
-    float32 ret;
-    clear_float_exceptions(env);
-    ret = float128_to_float32(QT1, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float128_to_float32(QT1, &env->fp_status);
 }
 
 void helper_fstoq(CPUSPARCState *env, float32 src)
 {
-    clear_float_exceptions(env);
     QT0 = float32_to_float128(src, &env->fp_status);
-    check_ieee_exceptions(env);
 }
 
 float64 helper_fqtod(CPUSPARCState *env)
 {
-    float64 ret;
-    clear_float_exceptions(env);
-    ret = float128_to_float64(QT1, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float128_to_float64(QT1, &env->fp_status);
 }
 
 void helper_fdtoq(CPUSPARCState *env, float64 src)
 {
-    clear_float_exceptions(env);
     QT0 = float64_to_float128(src, &env->fp_status);
-    check_ieee_exceptions(env);
 }
 
 /* Float to integer conversion.  */
 int32_t helper_fstoi(CPUSPARCState *env, float32 src)
 {
-    int32_t ret;
-    clear_float_exceptions(env);
-    ret = float32_to_int32_round_to_zero(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float32_to_int32_round_to_zero(src, &env->fp_status);
 }
 
 int32_t helper_fdtoi(CPUSPARCState *env, float64 src)
 {
-    int32_t ret;
-    clear_float_exceptions(env);
-    ret = float64_to_int32_round_to_zero(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float64_to_int32_round_to_zero(src, &env->fp_status);
 }
 
 int32_t helper_fqtoi(CPUSPARCState *env)
 {
-    int32_t ret;
-    clear_float_exceptions(env);
-    ret = float128_to_int32_round_to_zero(QT1, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float128_to_int32_round_to_zero(QT1, &env->fp_status);
 }
 
 #ifdef TARGET_SPARC64
 int64_t helper_fstox(CPUSPARCState *env, float32 src)
 {
-    int64_t ret;
-    clear_float_exceptions(env);
-    ret = float32_to_int64_round_to_zero(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float32_to_int64_round_to_zero(src, &env->fp_status);
 }
 
 int64_t helper_fdtox(CPUSPARCState *env, float64 src)
 {
-    int64_t ret;
-    clear_float_exceptions(env);
-    ret = float64_to_int64_round_to_zero(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float64_to_int64_round_to_zero(src, &env->fp_status);
 }
 
 int64_t helper_fqtox(CPUSPARCState *env)
 {
-    int64_t ret;
-    clear_float_exceptions(env);
-    ret = float128_to_int64_round_to_zero(QT1, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float128_to_int64_round_to_zero(QT1, &env->fp_status);
 }
 #endif
 
@@ -310,87 +247,79 @@
 
 float32 helper_fsqrts(CPUSPARCState *env, float32 src)
 {
-    float32 ret;
-    clear_float_exceptions(env);
-    ret = float32_sqrt(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float32_sqrt(src, &env->fp_status);
 }
 
 float64 helper_fsqrtd(CPUSPARCState *env, float64 src)
 {
-    float64 ret;
-    clear_float_exceptions(env);
-    ret = float64_sqrt(src, &env->fp_status);
-    check_ieee_exceptions(env);
-    return ret;
+    return float64_sqrt(src, &env->fp_status);
 }
 
 void helper_fsqrtq(CPUSPARCState *env)
 {
-    clear_float_exceptions(env);
     QT0 = float128_sqrt(QT1, &env->fp_status);
-    check_ieee_exceptions(env);
 }
 
 #define GEN_FCMP(name, size, reg1, reg2, FS, E)                         \
-    void glue(helper_, name) (CPUSPARCState *env)                       \
+    target_ulong glue(helper_, name) (CPUSPARCState *env)               \
     {                                                                   \
         int ret;                                                        \
-        clear_float_exceptions(env);                                    \
+        target_ulong fsr;                                               \
         if (E) {                                                        \
             ret = glue(size, _compare)(reg1, reg2, &env->fp_status);    \
         } else {                                                        \
             ret = glue(size, _compare_quiet)(reg1, reg2,                \
                                              &env->fp_status);          \
         }                                                               \
-        check_ieee_exceptions(env);                                     \
+        fsr = do_check_ieee_exceptions(env, GETPC());                   \
         switch (ret) {                                                  \
         case float_relation_unordered:                                  \
-            env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                    \
-            env->fsr |= FSR_NVA;                                        \
+            fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                         \
+            fsr |= FSR_NVA;                                             \
             break;                                                      \
         case float_relation_less:                                       \
-            env->fsr &= ~(FSR_FCC1) << FS;                              \
-            env->fsr |= FSR_FCC0 << FS;                                 \
+            fsr &= ~(FSR_FCC1) << FS;                                   \
+            fsr |= FSR_FCC0 << FS;                                      \
             break;                                                      \
         case float_relation_greater:                                    \
-            env->fsr &= ~(FSR_FCC0) << FS;                              \
-            env->fsr |= FSR_FCC1 << FS;                                 \
+            fsr &= ~(FSR_FCC0) << FS;                                   \
+            fsr |= FSR_FCC1 << FS;                                      \
             break;                                                      \
         default:                                                        \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                      \
             break;                                                      \
         }                                                               \
+        return fsr;                                                     \
     }
 #define GEN_FCMP_T(name, size, FS, E)                                   \
-    void glue(helper_, name)(CPUSPARCState *env, size src1, size src2)  \
+    target_ulong glue(helper_, name)(CPUSPARCState *env, size src1, size src2)\
     {                                                                   \
         int ret;                                                        \
-        clear_float_exceptions(env);                                    \
+        target_ulong fsr;                                               \
         if (E) {                                                        \
             ret = glue(size, _compare)(src1, src2, &env->fp_status);    \
         } else {                                                        \
             ret = glue(size, _compare_quiet)(src1, src2,                \
                                              &env->fp_status);          \
         }                                                               \
-        check_ieee_exceptions(env);                                     \
+        fsr = do_check_ieee_exceptions(env, GETPC());                   \
         switch (ret) {                                                  \
         case float_relation_unordered:                                  \
-            env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                    \
+            fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                         \
             break;                                                      \
         case float_relation_less:                                       \
-            env->fsr &= ~(FSR_FCC1 << FS);                              \
-            env->fsr |= FSR_FCC0 << FS;                                 \
+            fsr &= ~(FSR_FCC1 << FS);                                   \
+            fsr |= FSR_FCC0 << FS;                                      \
             break;                                                      \
         case float_relation_greater:                                    \
-            env->fsr &= ~(FSR_FCC0 << FS);                              \
-            env->fsr |= FSR_FCC1 << FS;                                 \
+            fsr &= ~(FSR_FCC0 << FS);                                   \
+            fsr |= FSR_FCC1 << FS;                                      \
             break;                                                      \
         default:                                                        \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                      \
             break;                                                      \
         }                                                               \
+        return fsr;                                                     \
     }
 
 GEN_FCMP_T(fcmps, float32, 0, 0);
@@ -430,11 +359,11 @@
 #undef GEN_FCMP_T
 #undef GEN_FCMP
 
-static inline void set_fsr(CPUSPARCState *env)
+static void set_fsr(CPUSPARCState *env, target_ulong fsr)
 {
     int rnd_mode;
 
-    switch (env->fsr & FSR_RD_MASK) {
+    switch (fsr & FSR_RD_MASK) {
     case FSR_RD_NEAREST:
         rnd_mode = float_round_nearest_even;
         break;
@@ -452,16 +381,20 @@
     set_float_rounding_mode(rnd_mode, &env->fp_status);
 }
 
-void helper_ldfsr(CPUSPARCState *env, uint32_t new_fsr)
+target_ulong helper_ldfsr(CPUSPARCState *env, target_ulong old_fsr,
+                          uint32_t new_fsr)
 {
-    env->fsr = (new_fsr & FSR_LDFSR_MASK) | (env->fsr & FSR_LDFSR_OLDMASK);
-    set_fsr(env);
+    old_fsr = (new_fsr & FSR_LDFSR_MASK) | (old_fsr & FSR_LDFSR_OLDMASK);
+    set_fsr(env, old_fsr);
+    return old_fsr;
 }
 
 #ifdef TARGET_SPARC64
-void helper_ldxfsr(CPUSPARCState *env, uint64_t new_fsr)
+target_ulong helper_ldxfsr(CPUSPARCState *env, target_ulong old_fsr,
+                           uint64_t new_fsr)
 {
-    env->fsr = (new_fsr & FSR_LDXFSR_MASK) | (env->fsr & FSR_LDXFSR_OLDMASK);
-    set_fsr(env);
+    old_fsr = (new_fsr & FSR_LDXFSR_MASK) | (old_fsr & FSR_LDXFSR_OLDMASK);
+    set_fsr(env, old_fsr);
+    return old_fsr;
 }
 #endif
diff --git a/target-sparc/gdbstub.c b/target-sparc/gdbstub.c
index 3de3242..ffc2baa 100644
--- a/target-sparc/gdbstub.c
+++ b/target-sparc/gdbstub.c
@@ -17,8 +17,9 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
 
 #ifdef TARGET_ABI32
diff --git a/target-sparc/helper.c b/target-sparc/helper.c
index 4850c7c..bedc672 100644
--- a/target-sparc/helper.c
+++ b/target-sparc/helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
 #include "sysemu/sysemu.h"
@@ -51,10 +53,16 @@
 #endif
 }
 
-uint64_t helper_tick_get_count(void *opaque)
+uint64_t helper_tick_get_count(CPUSPARCState *env, void *opaque, int mem_idx)
 {
 #if !defined(CONFIG_USER_ONLY)
-    return cpu_tick_get_count(opaque);
+    CPUTimer *timer = opaque;
+
+    if (timer->npt && mem_idx < MMU_KERNEL_IDX) {
+        helper_raise_exception(env, TT_PRIV_INSN);
+    }
+
+    return cpu_tick_get_count(timer);
 #else
     return 0;
 #endif
diff --git a/target-sparc/helper.h b/target-sparc/helper.h
index 1ad23e8..caa2a89 100644
--- a/target-sparc/helper.h
+++ b/target-sparc/helper.h
@@ -4,34 +4,32 @@
 DEF_HELPER_1(rdpsr, tl, env)
 DEF_HELPER_1(power_down, void, env)
 #else
-DEF_HELPER_2(wrpil, void, env, tl)
+DEF_HELPER_FLAGS_2(wrpil, TCG_CALL_NO_RWG, void, env, tl)
 DEF_HELPER_2(wrpstate, void, env, tl)
 DEF_HELPER_1(done, void, env)
 DEF_HELPER_1(retry, void, env)
-DEF_HELPER_1(flushw, void, env)
-DEF_HELPER_1(saved, void, env)
-DEF_HELPER_1(restored, void, env)
+DEF_HELPER_FLAGS_1(flushw, TCG_CALL_NO_WG, void, env)
+DEF_HELPER_FLAGS_1(saved, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_1(restored, TCG_CALL_NO_RWG, void, env)
 DEF_HELPER_1(rdccr, tl, env)
 DEF_HELPER_2(wrccr, void, env, tl)
 DEF_HELPER_1(rdcwp, tl, env)
 DEF_HELPER_2(wrcwp, void, env, tl)
 DEF_HELPER_FLAGS_2(array8, TCG_CALL_NO_RWG_SE, tl, tl, tl)
-DEF_HELPER_1(popc, tl, tl)
-DEF_HELPER_4(ldda_asi, void, env, tl, int, int)
-DEF_HELPER_5(ldf_asi, void, env, tl, int, int, int)
-DEF_HELPER_5(stf_asi, void, env, tl, int, int, int)
-DEF_HELPER_5(casx_asi, tl, env, tl, tl, tl, i32)
-DEF_HELPER_2(set_softint, void, env, i64)
-DEF_HELPER_2(clear_softint, void, env, i64)
-DEF_HELPER_2(write_softint, void, env, i64)
-DEF_HELPER_2(tick_set_count, void, ptr, i64)
-DEF_HELPER_1(tick_get_count, i64, ptr)
-DEF_HELPER_2(tick_set_limit, void, ptr, i64)
+DEF_HELPER_FLAGS_1(popc, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_3(ldda_asi, TCG_CALL_NO_WG, void, env, tl, int)
+DEF_HELPER_FLAGS_5(casx_asi, TCG_CALL_NO_WG, tl, env, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_2(set_softint, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(clear_softint, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(write_softint, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(tick_set_count, TCG_CALL_NO_RWG, void, ptr, i64)
+DEF_HELPER_FLAGS_3(tick_get_count, TCG_CALL_NO_WG, i64, env, ptr, int)
+DEF_HELPER_FLAGS_2(tick_set_limit, TCG_CALL_NO_RWG, void, ptr, i64)
 #endif
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
-DEF_HELPER_5(cas_asi, tl, env, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_5(cas_asi, TCG_CALL_NO_WG, tl, env, tl, tl, tl, i32)
 #endif
-DEF_HELPER_3(check_align, void, env, tl, i32)
+DEF_HELPER_FLAGS_3(check_align, TCG_CALL_NO_WG, void, env, tl, i32)
 DEF_HELPER_1(debug, void, env)
 DEF_HELPER_1(save, void, env)
 DEF_HELPER_1(restore, void, env)
@@ -42,95 +40,97 @@
 DEF_HELPER_3(taddcctv, tl, env, tl, tl)
 DEF_HELPER_3(tsubcctv, tl, env, tl, tl)
 #ifdef TARGET_SPARC64
-DEF_HELPER_3(sdivx, s64, env, s64, s64)
-DEF_HELPER_3(udivx, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(sdivx, TCG_CALL_NO_WG, s64, env, s64, s64)
+DEF_HELPER_FLAGS_3(udivx, TCG_CALL_NO_WG, i64, env, i64, i64)
 #endif
-DEF_HELPER_3(ldqf, void, env, tl, int)
-DEF_HELPER_3(stqf, void, env, tl, int)
+DEF_HELPER_FLAGS_3(ldqf, TCG_CALL_NO_WG, void, env, tl, int)
+DEF_HELPER_FLAGS_3(stqf, TCG_CALL_NO_WG, void, env, tl, int)
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
-DEF_HELPER_5(ld_asi, i64, env, tl, int, int, int)
-DEF_HELPER_5(st_asi, void, env, tl, i64, int, int)
+DEF_HELPER_FLAGS_4(ld_asi, TCG_CALL_NO_WG, i64, env, tl, int, i32)
+DEF_HELPER_FLAGS_5(st_asi, TCG_CALL_NO_WG, void, env, tl, i64, int, i32)
 #endif
-DEF_HELPER_2(ldfsr, void, env, i32)
+DEF_HELPER_FLAGS_1(check_ieee_exceptions, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_3(ldfsr, TCG_CALL_NO_RWG, tl, env, tl, i32)
 DEF_HELPER_FLAGS_1(fabss, TCG_CALL_NO_RWG_SE, f32, f32)
-DEF_HELPER_2(fsqrts, f32, env, f32)
-DEF_HELPER_2(fsqrtd, f64, env, f64)
-DEF_HELPER_3(fcmps, void, env, f32, f32)
-DEF_HELPER_3(fcmpd, void, env, f64, f64)
-DEF_HELPER_3(fcmpes, void, env, f32, f32)
-DEF_HELPER_3(fcmped, void, env, f64, f64)
-DEF_HELPER_1(fsqrtq, void, env)
-DEF_HELPER_1(fcmpq, void, env)
-DEF_HELPER_1(fcmpeq, void, env)
+DEF_HELPER_FLAGS_2(fsqrts, TCG_CALL_NO_RWG, f32, env, f32)
+DEF_HELPER_FLAGS_2(fsqrtd, TCG_CALL_NO_RWG, f64, env, f64)
+DEF_HELPER_FLAGS_3(fcmps, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmpd, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_3(fcmpes, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmped, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_1(fsqrtq, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_1(fcmpq, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_1(fcmpeq, TCG_CALL_NO_WG, tl, env)
 #ifdef TARGET_SPARC64
-DEF_HELPER_2(ldxfsr, void, env, i64)
+DEF_HELPER_FLAGS_3(ldxfsr, TCG_CALL_NO_RWG, tl, env, tl, i64)
 DEF_HELPER_FLAGS_1(fabsd, TCG_CALL_NO_RWG_SE, f64, f64)
-DEF_HELPER_3(fcmps_fcc1, void, env, f32, f32)
-DEF_HELPER_3(fcmps_fcc2, void, env, f32, f32)
-DEF_HELPER_3(fcmps_fcc3, void, env, f32, f32)
-DEF_HELPER_3(fcmpd_fcc1, void, env, f64, f64)
-DEF_HELPER_3(fcmpd_fcc2, void, env, f64, f64)
-DEF_HELPER_3(fcmpd_fcc3, void, env, f64, f64)
-DEF_HELPER_3(fcmpes_fcc1, void, env, f32, f32)
-DEF_HELPER_3(fcmpes_fcc2, void, env, f32, f32)
-DEF_HELPER_3(fcmpes_fcc3, void, env, f32, f32)
-DEF_HELPER_3(fcmped_fcc1, void, env, f64, f64)
-DEF_HELPER_3(fcmped_fcc2, void, env, f64, f64)
-DEF_HELPER_3(fcmped_fcc3, void, env, f64, f64)
-DEF_HELPER_1(fabsq, void, env)
-DEF_HELPER_1(fcmpq_fcc1, void, env)
-DEF_HELPER_1(fcmpq_fcc2, void, env)
-DEF_HELPER_1(fcmpq_fcc3, void, env)
-DEF_HELPER_1(fcmpeq_fcc1, void, env)
-DEF_HELPER_1(fcmpeq_fcc2, void, env)
-DEF_HELPER_1(fcmpeq_fcc3, void, env)
+DEF_HELPER_FLAGS_3(fcmps_fcc1, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmps_fcc2, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmps_fcc3, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmpd_fcc1, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_3(fcmpd_fcc2, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_3(fcmpd_fcc3, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_3(fcmpes_fcc1, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmpes_fcc2, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmpes_fcc3, TCG_CALL_NO_WG, tl, env, f32, f32)
+DEF_HELPER_FLAGS_3(fcmped_fcc1, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_3(fcmped_fcc2, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_3(fcmped_fcc3, TCG_CALL_NO_WG, tl, env, f64, f64)
+DEF_HELPER_FLAGS_1(fabsq, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_1(fcmpq_fcc1, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_1(fcmpq_fcc2, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_1(fcmpq_fcc3, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_1(fcmpeq_fcc1, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_1(fcmpeq_fcc2, TCG_CALL_NO_WG, tl, env)
+DEF_HELPER_FLAGS_1(fcmpeq_fcc3, TCG_CALL_NO_WG, tl, env)
 #endif
 DEF_HELPER_2(raise_exception, noreturn, env, int)
-#define F_HELPER_0_1(name) DEF_HELPER_1(f ## name, void, env)
+#define F_HELPER_0_1(name) \
+  DEF_HELPER_FLAGS_1(f ## name, TCG_CALL_NO_RWG, void, env)
 
-DEF_HELPER_3(faddd, f64, env, f64, f64)
-DEF_HELPER_3(fsubd, f64, env, f64, f64)
-DEF_HELPER_3(fmuld, f64, env, f64, f64)
-DEF_HELPER_3(fdivd, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(faddd, TCG_CALL_NO_RWG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(fsubd, TCG_CALL_NO_RWG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(fmuld, TCG_CALL_NO_RWG, f64, env, f64, f64)
+DEF_HELPER_FLAGS_3(fdivd, TCG_CALL_NO_RWG, f64, env, f64, f64)
 F_HELPER_0_1(addq)
 F_HELPER_0_1(subq)
 F_HELPER_0_1(mulq)
 F_HELPER_0_1(divq)
 
-DEF_HELPER_3(fadds, f32, env, f32, f32)
-DEF_HELPER_3(fsubs, f32, env, f32, f32)
-DEF_HELPER_3(fmuls, f32, env, f32, f32)
-DEF_HELPER_3(fdivs, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fadds, TCG_CALL_NO_RWG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fsubs, TCG_CALL_NO_RWG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fmuls, TCG_CALL_NO_RWG, f32, env, f32, f32)
+DEF_HELPER_FLAGS_3(fdivs, TCG_CALL_NO_RWG, f32, env, f32, f32)
 
-DEF_HELPER_3(fsmuld, f64, env, f32, f32)
-DEF_HELPER_3(fdmulq, void, env, f64, f64)
+DEF_HELPER_FLAGS_3(fsmuld, TCG_CALL_NO_RWG, f64, env, f32, f32)
+DEF_HELPER_FLAGS_3(fdmulq, TCG_CALL_NO_RWG, void, env, f64, f64)
 
 DEF_HELPER_FLAGS_1(fnegs, TCG_CALL_NO_RWG_SE, f32, f32)
-DEF_HELPER_2(fitod, f64, env, s32)
-DEF_HELPER_2(fitoq, void, env, s32)
+DEF_HELPER_FLAGS_2(fitod, TCG_CALL_NO_RWG_SE, f64, env, s32)
+DEF_HELPER_FLAGS_2(fitoq, TCG_CALL_NO_RWG, void, env, s32)
 
-DEF_HELPER_2(fitos, f32, env, s32)
+DEF_HELPER_FLAGS_2(fitos, TCG_CALL_NO_RWG, f32, env, s32)
 
 #ifdef TARGET_SPARC64
 DEF_HELPER_FLAGS_1(fnegd, TCG_CALL_NO_RWG_SE, f64, f64)
-DEF_HELPER_1(fnegq, void, env)
-DEF_HELPER_2(fxtos, f32, env, s64)
-DEF_HELPER_2(fxtod, f64, env, s64)
-DEF_HELPER_2(fxtoq, void, env, s64)
+DEF_HELPER_FLAGS_1(fnegq, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(fxtos, TCG_CALL_NO_RWG, f32, env, s64)
+DEF_HELPER_FLAGS_2(fxtod, TCG_CALL_NO_RWG, f64, env, s64)
+DEF_HELPER_FLAGS_2(fxtoq, TCG_CALL_NO_RWG, void, env, s64)
 #endif
-DEF_HELPER_2(fdtos, f32, env, f64)
-DEF_HELPER_2(fstod, f64, env, f32)
-DEF_HELPER_1(fqtos, f32, env)
-DEF_HELPER_2(fstoq, void, env, f32)
-DEF_HELPER_1(fqtod, f64, env)
-DEF_HELPER_2(fdtoq, void, env, f64)
-DEF_HELPER_2(fstoi, s32, env, f32)
-DEF_HELPER_2(fdtoi, s32, env, f64)
-DEF_HELPER_1(fqtoi, s32, env)
+DEF_HELPER_FLAGS_2(fdtos, TCG_CALL_NO_RWG, f32, env, f64)
+DEF_HELPER_FLAGS_2(fstod, TCG_CALL_NO_RWG, f64, env, f32)
+DEF_HELPER_FLAGS_1(fqtos, TCG_CALL_NO_RWG, f32, env)
+DEF_HELPER_FLAGS_2(fstoq, TCG_CALL_NO_RWG, void, env, f32)
+DEF_HELPER_FLAGS_1(fqtod, TCG_CALL_NO_RWG, f64, env)
+DEF_HELPER_FLAGS_2(fdtoq, TCG_CALL_NO_RWG, void, env, f64)
+DEF_HELPER_FLAGS_2(fstoi, TCG_CALL_NO_RWG, s32, env, f32)
+DEF_HELPER_FLAGS_2(fdtoi, TCG_CALL_NO_RWG, s32, env, f64)
+DEF_HELPER_FLAGS_1(fqtoi, TCG_CALL_NO_RWG, s32, env)
 #ifdef TARGET_SPARC64
-DEF_HELPER_2(fstox, s64, env, f32)
-DEF_HELPER_2(fdtox, s64, env, f64)
-DEF_HELPER_1(fqtox, s64, env)
+DEF_HELPER_FLAGS_2(fstox, TCG_CALL_NO_RWG, s64, env, f32)
+DEF_HELPER_FLAGS_2(fdtox, TCG_CALL_NO_RWG, s64, env, f64)
+DEF_HELPER_FLAGS_1(fqtox, TCG_CALL_NO_RWG, s64, env)
 
 DEF_HELPER_FLAGS_2(fpmerge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 DEF_HELPER_FLAGS_2(fmul8x16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
@@ -172,4 +172,4 @@
 #undef VIS_HELPER
 #undef VIS_CMPHELPER
 DEF_HELPER_1(compute_psr, void, env)
-DEF_HELPER_1(compute_C_icc, i32, env)
+DEF_HELPER_FLAGS_1(compute_C_icc, TCG_CALL_NO_WG_SE, i32, env)
diff --git a/target-sparc/int32_helper.c b/target-sparc/int32_helper.c
index 7c380ba..09afe13 100644
--- a/target-sparc/int32_helper.c
+++ b/target-sparc/int32_helper.c
@@ -17,9 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "trace.h"
 #include "sysemu/sysemu.h"
+#include "exec/log.h"
 
 #define DEBUG_PCALL
 
diff --git a/target-sparc/int64_helper.c b/target-sparc/int64_helper.c
index b02d22b..29360fa 100644
--- a/target-sparc/int64_helper.c
+++ b/target-sparc/int64_helper.c
@@ -17,8 +17,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/log.h"
 #include "trace.h"
 
 #define DEBUG_PCALL
@@ -156,9 +158,8 @@
     } else if ((intno & 0x1c0) == TT_FILL) {
         cpu_set_cwp(env, cpu_cwp_inc(env, env->cwp + 1));
     }
-    env->tbr &= ~0x7fffULL;
-    env->tbr |= ((env->tl > 1) ? 1 << 14 : 0) | (intno << 5);
-    env->pc = env->tbr;
+    env->pc = env->tbr  & ~0x7fffULL;
+    env->pc |= ((env->tl > 1) ? 1 << 14 : 0) | (intno << 5);
     env->npc = env->pc + 4;
     cs->exception_index = -1;
 }
diff --git a/target-sparc/ldst_helper.c b/target-sparc/ldst_helper.c
index 1a62e19..66ee0fe 100644
--- a/target-sparc/ldst_helper.c
+++ b/target-sparc/ldst_helper.c
@@ -17,9 +17,13 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "tcg.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "asi.h"
 
 //#define DEBUG_MMU
 //#define DEBUG_MXCC
@@ -250,6 +254,7 @@
 
 #endif
 
+#if defined(TARGET_SPARC64) || defined(CONFIG_USER_ONLY)
 static inline target_ulong address_mask(CPUSPARCState *env1, target_ulong addr)
 {
 #ifdef TARGET_SPARC64
@@ -259,12 +264,14 @@
 #endif
     return addr;
 }
+#endif
 
+#ifdef TARGET_SPARC64
 /* returns true if access using this ASI is to have address translated by MMU
    otherwise access is to raw physical address */
+/* TODO: check sparc32 bits */
 static inline int is_translating_asi(int asi)
 {
-#ifdef TARGET_SPARC64
     /* Ultrasparc IIi translating asi
        - note this list is defined by cpu implementation
     */
@@ -281,10 +288,6 @@
     default:
         return 0;
     }
-#else
-    /* TODO: check sparc32 bits */
-    return 0;
-#endif
 }
 
 static inline target_ulong asi_address_mask(CPUSPARCState *env,
@@ -296,6 +299,7 @@
         return addr;
     }
 }
+#endif
 
 void helper_check_align(CPUSPARCState *env, target_ulong addr, uint32_t align)
 {
@@ -425,9 +429,11 @@
     return ret;
 }
 
-uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
-                       int sign)
+uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr,
+                       int asi, uint32_t memop)
 {
+    int size = 1 << (memop & MO_SIZE);
+    int sign = memop & MO_SIGN;
     CPUState *cs = CPU(sparc_env_get_cpu(env));
     uint64_t ret = 0;
 #if defined(DEBUG_MXCC) || defined(DEBUG_ASI)
@@ -436,7 +442,8 @@
 
     helper_check_align(env, addr, size - 1);
     switch (asi) {
-    case 2: /* SuperSparc MXCC registers and Leon3 cache control */
+    case ASI_M_MXCC: /* SuperSparc MXCC registers, or... */
+    /* case ASI_LEON_CACHEREGS:  Leon3 cache control */
         switch (addr) {
         case 0x00:          /* Leon3 Cache Control */
         case 0x08:          /* Leon3 Instruction Cache config */
@@ -495,8 +502,8 @@
         dump_mxcc(env);
 #endif
         break;
-    case 3: /* MMU probe */
-    case 0x18: /* LEON3 MMU probe */
+    case ASI_M_FLUSH_PROBE: /* SuperSparc MMU probe */
+    case ASI_LEON_MMUFLUSH: /* LEON3 MMU probe */
         {
             int mmulev;
 
@@ -510,8 +517,8 @@
                         addr, mmulev, ret);
         }
         break;
-    case 4: /* read MMU regs */
-    case 0x19: /* LEON3 read MMU regs */
+    case ASI_M_MMUREGS: /* SuperSparc MMU regs */
+    case ASI_LEON_MMUREGS: /* LEON3 MMU regs */
         {
             int reg = (addr >> 8) & 0x1f;
 
@@ -526,11 +533,11 @@
             DPRINTF_MMU("mmu_read: reg[%d] = 0x%08" PRIx64 "\n", reg, ret);
         }
         break;
-    case 5: /* Turbosparc ITLB Diagnostic */
-    case 6: /* Turbosparc DTLB Diagnostic */
-    case 7: /* Turbosparc IOTLB Diagnostic */
+    case ASI_M_TLBDIAG: /* Turbosparc ITLB Diagnostic */
+    case ASI_M_DIAGS:   /* Turbosparc DTLB Diagnostic */
+    case ASI_M_IODIAG:  /* Turbosparc IOTLB Diagnostic */
         break;
-    case 9: /* Supervisor code access */
+    case ASI_KERNELTXT: /* Supervisor code access */
         switch (size) {
         case 1:
             ret = cpu_ldub_code(env, addr);
@@ -547,7 +554,7 @@
             break;
         }
         break;
-    case 0xa: /* User data access */
+    case ASI_USERDATA: /* User data access */
         switch (size) {
         case 1:
             ret = cpu_ldub_user(env, addr);
@@ -564,8 +571,8 @@
             break;
         }
         break;
-    case 0xb: /* Supervisor data access */
-    case 0x80:
+    case ASI_KERNELDATA: /* Supervisor data access */
+    case ASI_P: /* Implicit primary context data access (v9 only?) */
         switch (size) {
         case 1:
             ret = cpu_ldub_kernel(env, addr);
@@ -582,13 +589,13 @@
             break;
         }
         break;
-    case 0xc: /* I-cache tag */
-    case 0xd: /* I-cache data */
-    case 0xe: /* D-cache tag */
-    case 0xf: /* D-cache data */
+    case ASI_M_TXTC_TAG:   /* SparcStation 5 I-cache tag */
+    case ASI_M_TXTC_DATA:  /* SparcStation 5 I-cache data */
+    case ASI_M_DATAC_TAG:  /* SparcStation 5 D-cache tag */
+    case ASI_M_DATAC_DATA: /* SparcStation 5 D-cache data */
         break;
-    case 0x20: /* MMU passthrough */
-    case 0x1c: /* LEON MMU passthrough */
+    case ASI_M_BYPASS:    /* MMU passthrough */
+    case ASI_LEON_BYPASS: /* LEON MMU passthrough */
         switch (size) {
         case 1:
             ret = ldub_phys(cs->as, addr);
@@ -667,7 +674,7 @@
     case 0x4c: /* SuperSPARC MMU Breakpoint Action */
         ret = env->mmubpaction;
         break;
-    case 8: /* User code access, XXX */
+    case ASI_USERTXT: /* User code access, XXX */
     default:
         cpu_unassigned_access(cs, addr, false, false, asi, size);
         ret = 0;
@@ -694,15 +701,17 @@
     return ret;
 }
 
-void helper_st_asi(CPUSPARCState *env, target_ulong addr, uint64_t val, int asi,
-                   int size)
+void helper_st_asi(CPUSPARCState *env, target_ulong addr, uint64_t val,
+                   int asi, uint32_t memop)
 {
+    int size = 1 << (memop & MO_SIZE);
     SPARCCPU *cpu = sparc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
 
     helper_check_align(env, addr, size - 1);
     switch (asi) {
-    case 2: /* SuperSparc MXCC registers and Leon3 cache control */
+    case ASI_M_MXCC: /* SuperSparc MXCC registers, or... */
+    /* case ASI_LEON_CACHEREGS:  Leon3 cache control */
         switch (addr) {
         case 0x00:          /* Leon3 Cache Control */
         case 0x08:          /* Leon3 Instruction Cache config */
@@ -836,8 +845,8 @@
         dump_mxcc(env);
 #endif
         break;
-    case 3: /* MMU flush */
-    case 0x18: /* LEON3 MMU flush */
+    case ASI_M_FLUSH_PROBE: /* SuperSparc MMU flush */
+    case ASI_LEON_MMUFLUSH: /* LEON3 MMU flush */
         {
             int mmulev;
 
@@ -861,8 +870,8 @@
 #endif
         }
         break;
-    case 4: /* write MMU regs */
-    case 0x19: /* LEON3 write MMU regs */
+    case ASI_M_MMUREGS: /* write MMU regs */
+    case ASI_LEON_MMUREGS: /* LEON3 write MMU regs */
         {
             int reg = (addr >> 8) & 0x1f;
             uint32_t oldreg;
@@ -916,11 +925,11 @@
 #endif
         }
         break;
-    case 5: /* Turbosparc ITLB Diagnostic */
-    case 6: /* Turbosparc DTLB Diagnostic */
-    case 7: /* Turbosparc IOTLB Diagnostic */
+    case ASI_M_TLBDIAG: /* Turbosparc ITLB Diagnostic */
+    case ASI_M_DIAGS:   /* Turbosparc DTLB Diagnostic */
+    case ASI_M_IODIAG:  /* Turbosparc IOTLB Diagnostic */
         break;
-    case 0xa: /* User data access */
+    case ASI_USERDATA: /* User data access */
         switch (size) {
         case 1:
             cpu_stb_user(env, addr, val);
@@ -937,8 +946,8 @@
             break;
         }
         break;
-    case 0xb: /* Supervisor data access */
-    case 0x80:
+    case ASI_KERNELDATA: /* Supervisor data access */
+    case ASI_P:
         switch (size) {
         case 1:
             cpu_stb_kernel(env, addr, val);
@@ -955,17 +964,17 @@
             break;
         }
         break;
-    case 0xc: /* I-cache tag */
-    case 0xd: /* I-cache data */
-    case 0xe: /* D-cache tag */
-    case 0xf: /* D-cache data */
-    case 0x10: /* I/D-cache flush page */
-    case 0x11: /* I/D-cache flush segment */
-    case 0x12: /* I/D-cache flush region */
-    case 0x13: /* I/D-cache flush context */
-    case 0x14: /* I/D-cache flush user */
+    case ASI_M_TXTC_TAG:   /* I-cache tag */
+    case ASI_M_TXTC_DATA:  /* I-cache data */
+    case ASI_M_DATAC_TAG:  /* D-cache tag */
+    case ASI_M_DATAC_DATA: /* D-cache data */
+    case ASI_M_FLUSH_PAGE:   /* I/D-cache flush page */
+    case ASI_M_FLUSH_SEG:    /* I/D-cache flush segment */
+    case ASI_M_FLUSH_REGION: /* I/D-cache flush region */
+    case ASI_M_FLUSH_CTX:    /* I/D-cache flush context */
+    case ASI_M_FLUSH_USER:   /* I/D-cache flush user */
         break;
-    case 0x17: /* Block copy, sta access */
+    case ASI_M_BCOPY: /* Block copy, sta access */
         {
             /* val = src
                addr = dst
@@ -979,20 +988,20 @@
             }
         }
         break;
-    case 0x1f: /* Block fill, stda access */
+    case ASI_M_BFILL: /* Block fill, stda access */
         {
             /* addr = dst
                fill 32 bytes with val */
             unsigned int i;
-            uint32_t dst = addr & 7;
+            uint32_t dst = addr & ~7;
 
             for (i = 0; i < 32; i += 8, dst += 8) {
                 cpu_stq_kernel(env, dst, val);
             }
         }
         break;
-    case 0x20: /* MMU passthrough */
-    case 0x1c: /* LEON MMU passthrough */
+    case ASI_M_BYPASS:    /* MMU passthrough */
+    case ASI_LEON_BYPASS: /* LEON MMU passthrough */
         {
             switch (size) {
             case 1:
@@ -1076,8 +1085,8 @@
     case 0x4c: /* SuperSPARC MMU Breakpoint Action */
         env->mmubpaction = val & 0x1fff;
         break;
-    case 8: /* User code access, XXX */
-    case 9: /* Supervisor code access, XXX */
+    case ASI_USERTXT: /* User code access, XXX */
+    case ASI_KERNELTXT: /* Supervisor code access, XXX */
     default:
         cpu_unassigned_access(CPU(sparc_env_get_cpu(env)),
                               addr, true, false, asi, size);
@@ -1092,9 +1101,11 @@
 #else /* TARGET_SPARC64 */
 
 #ifdef CONFIG_USER_ONLY
-uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
-                       int sign)
+uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr,
+                       int asi, uint32_t memop)
 {
+    int size = 1 << (memop & MO_SIZE);
+    int sign = memop & MO_SIGN;
     uint64_t ret = 0;
 #if defined(DEBUG_ASI)
     target_ulong last_addr = addr;
@@ -1108,8 +1119,8 @@
     addr = asi_address_mask(env, asi, addr);
 
     switch (asi) {
-    case 0x82: /* Primary no-fault */
-    case 0x8a: /* Primary no-fault LE */
+    case ASI_PNF:  /* Primary no-fault */
+    case ASI_PNFL: /* Primary no-fault LE */
         if (page_check_range(addr, size, PAGE_READ) == -1) {
 #ifdef DEBUG_ASI
             dump_asi("read ", last_addr, asi, size, ret);
@@ -1117,28 +1128,28 @@
             return 0;
         }
         /* Fall through */
-    case 0x80: /* Primary */
-    case 0x88: /* Primary LE */
+    case ASI_P: /* Primary */
+    case ASI_PL: /* Primary LE */
         {
             switch (size) {
             case 1:
-                ret = ldub_raw(addr);
+                ret = cpu_ldub_data(env, addr);
                 break;
             case 2:
-                ret = lduw_raw(addr);
+                ret = cpu_lduw_data(env, addr);
                 break;
             case 4:
-                ret = ldl_raw(addr);
+                ret = cpu_ldl_data(env, addr);
                 break;
             default:
             case 8:
-                ret = ldq_raw(addr);
+                ret = cpu_ldq_data(env, addr);
                 break;
             }
         }
         break;
-    case 0x83: /* Secondary no-fault */
-    case 0x8b: /* Secondary no-fault LE */
+    case ASI_SNF:  /* Secondary no-fault */
+    case ASI_SNFL: /* Secondary no-fault LE */
         if (page_check_range(addr, size, PAGE_READ) == -1) {
 #ifdef DEBUG_ASI
             dump_asi("read ", last_addr, asi, size, ret);
@@ -1146,8 +1157,8 @@
             return 0;
         }
         /* Fall through */
-    case 0x81: /* Secondary */
-    case 0x89: /* Secondary LE */
+    case ASI_S:  /* Secondary */
+    case ASI_SL: /* Secondary LE */
         /* XXX */
         break;
     default:
@@ -1156,10 +1167,10 @@
 
     /* Convert from little endian */
     switch (asi) {
-    case 0x88: /* Primary LE */
-    case 0x89: /* Secondary LE */
-    case 0x8a: /* Primary no-fault LE */
-    case 0x8b: /* Secondary no-fault LE */
+    case ASI_PL:   /* Primary LE */
+    case ASI_SL:   /* Secondary LE */
+    case ASI_PNFL: /* Primary no-fault LE */
+    case ASI_SNFL: /* Secondary no-fault LE */
         switch (size) {
         case 2:
             ret = bswap16(ret);
@@ -1200,8 +1211,9 @@
 }
 
 void helper_st_asi(CPUSPARCState *env, target_ulong addr, target_ulong val,
-                   int asi, int size)
+                   int asi, uint32_t memop)
 {
+    int size = 1 << (memop & MO_SIZE);
 #ifdef DEBUG_ASI
     dump_asi("write", addr, asi, size, val);
 #endif
@@ -1214,8 +1226,8 @@
 
     /* Convert to little endian */
     switch (asi) {
-    case 0x88: /* Primary LE */
-    case 0x89: /* Secondary LE */
+    case ASI_PL: /* Primary LE */
+    case ASI_SL: /* Secondary LE */
         switch (size) {
         case 2:
             val = bswap16(val);
@@ -1234,35 +1246,35 @@
     }
 
     switch (asi) {
-    case 0x80: /* Primary */
-    case 0x88: /* Primary LE */
+    case ASI_P:  /* Primary */
+    case ASI_PL: /* Primary LE */
         {
             switch (size) {
             case 1:
-                stb_raw(addr, val);
+                cpu_stb_data(env, addr, val);
                 break;
             case 2:
-                stw_raw(addr, val);
+                cpu_stw_data(env, addr, val);
                 break;
             case 4:
-                stl_raw(addr, val);
+                cpu_stl_data(env, addr, val);
                 break;
             case 8:
             default:
-                stq_raw(addr, val);
+                cpu_stq_data(env, addr, val);
                 break;
             }
         }
         break;
-    case 0x81: /* Secondary */
-    case 0x89: /* Secondary LE */
+    case ASI_S:  /* Secondary */
+    case ASI_SL: /* Secondary LE */
         /* XXX */
         return;
 
-    case 0x82: /* Primary no-fault, RO */
-    case 0x83: /* Secondary no-fault, RO */
-    case 0x8a: /* Primary no-fault LE, RO */
-    case 0x8b: /* Secondary no-fault LE, RO */
+    case ASI_PNF:  /* Primary no-fault, RO */
+    case ASI_SNF:  /* Secondary no-fault, RO */
+    case ASI_PNFL: /* Primary no-fault LE, RO */
+    case ASI_SNFL: /* Secondary no-fault LE, RO */
     default:
         helper_raise_exception(env, TT_DATA_ACCESS);
         return;
@@ -1271,9 +1283,11 @@
 
 #else /* CONFIG_USER_ONLY */
 
-uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
-                       int sign)
+uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr,
+                       int asi, uint32_t memop)
 {
+    int size = 1 << (memop & MO_SIZE);
+    int sign = memop & MO_SIGN;
     CPUState *cs = CPU(sparc_env_get_cpu(env));
     uint64_t ret = 0;
 #if defined(DEBUG_ASI)
@@ -1316,16 +1330,14 @@
     }
 
     switch (asi) {
-    case 0x10: /* As if user primary */
-    case 0x11: /* As if user secondary */
-    case 0x18: /* As if user primary LE */
-    case 0x19: /* As if user secondary LE */
-    case 0x80: /* Primary */
-    case 0x81: /* Secondary */
-    case 0x88: /* Primary LE */
-    case 0x89: /* Secondary LE */
-    case 0xe2: /* UA2007 Primary block init */
-    case 0xe3: /* UA2007 Secondary block init */
+    case ASI_AIUP:  /* As if user primary */
+    case ASI_AIUS:  /* As if user secondary */
+    case ASI_AIUPL: /* As if user primary LE */
+    case ASI_AIUSL: /* As if user secondary LE */
+    case ASI_P:  /* Primary */
+    case ASI_S:  /* Secondary */
+    case ASI_PL: /* Primary LE */
+    case ASI_SL: /* Secondary LE */
         if ((asi & 0x80) && (env->pstate & PS_PRIV)) {
             if (cpu_hypervisor_mode(env)) {
                 switch (size) {
@@ -1416,10 +1428,10 @@
             }
         }
         break;
-    case 0x14: /* Bypass */
-    case 0x15: /* Bypass, non-cacheable */
-    case 0x1c: /* Bypass LE */
-    case 0x1d: /* Bypass, non-cacheable LE */
+    case ASI_REAL:        /* Bypass */
+    case ASI_REAL_IO:   /* Bypass, non-cacheable */
+    case ASI_REAL_L:      /* Bypass LE */
+    case ASI_REAL_IO_L: /* Bypass, non-cacheable LE */
         {
             switch (size) {
             case 1:
@@ -1438,13 +1450,8 @@
             }
             break;
         }
-    case 0x24: /* Nucleus quad LDD 128 bit atomic */
-    case 0x2c: /* Nucleus quad LDD 128 bit atomic LE
-                  Only ldda allowed */
-        helper_raise_exception(env, TT_ILL_INSN);
-        return 0;
-    case 0x04: /* Nucleus */
-    case 0x0c: /* Nucleus Little Endian (LE) */
+    case ASI_N:  /* Nucleus */
+    case ASI_NL: /* Nucleus Little Endian (LE) */
         {
             switch (size) {
             case 1:
@@ -1463,13 +1470,13 @@
             }
             break;
         }
-    case 0x4a: /* UPA config */
+    case ASI_UPA_CONFIG: /* UPA config */
         /* XXX */
         break;
-    case 0x45: /* LSU */
+    case ASI_LSU_CONTROL: /* LSU */
         ret = env->lsu;
         break;
-    case 0x50: /* I-MMU regs */
+    case ASI_IMMU: /* I-MMU regs */
         {
             int reg = (addr >> 3) & 0xf;
 
@@ -1482,7 +1489,7 @@
 
             break;
         }
-    case 0x51: /* I-MMU 8k TSB pointer */
+    case ASI_IMMU_TSB_8KB_PTR: /* I-MMU 8k TSB pointer */
         {
             /* env->immuregs[5] holds I-MMU TSB register value
                env->immuregs[6] holds I-MMU Tag Access register value */
@@ -1490,7 +1497,7 @@
                                          8*1024);
             break;
         }
-    case 0x52: /* I-MMU 64k TSB pointer */
+    case ASI_IMMU_TSB_64KB_PTR: /* I-MMU 64k TSB pointer */
         {
             /* env->immuregs[5] holds I-MMU TSB register value
                env->immuregs[6] holds I-MMU Tag Access register value */
@@ -1498,21 +1505,21 @@
                                          64*1024);
             break;
         }
-    case 0x55: /* I-MMU data access */
+    case ASI_ITLB_DATA_ACCESS: /* I-MMU data access */
         {
             int reg = (addr >> 3) & 0x3f;
 
             ret = env->itlb[reg].tte;
             break;
         }
-    case 0x56: /* I-MMU tag read */
+    case ASI_ITLB_TAG_READ: /* I-MMU tag read */
         {
             int reg = (addr >> 3) & 0x3f;
 
             ret = env->itlb[reg].tag;
             break;
         }
-    case 0x58: /* D-MMU regs */
+    case ASI_DMMU: /* D-MMU regs */
         {
             int reg = (addr >> 3) & 0xf;
 
@@ -1524,7 +1531,7 @@
             }
             break;
         }
-    case 0x59: /* D-MMU 8k TSB pointer */
+    case ASI_DMMU_TSB_8KB_PTR: /* D-MMU 8k TSB pointer */
         {
             /* env->dmmuregs[5] holds D-MMU TSB register value
                env->dmmuregs[6] holds D-MMU Tag Access register value */
@@ -1532,7 +1539,7 @@
                                          8*1024);
             break;
         }
-    case 0x5a: /* D-MMU 64k TSB pointer */
+    case ASI_DMMU_TSB_64KB_PTR: /* D-MMU 64k TSB pointer */
         {
             /* env->dmmuregs[5] holds D-MMU TSB register value
                env->dmmuregs[6] holds D-MMU Tag Access register value */
@@ -1540,26 +1547,26 @@
                                          64*1024);
             break;
         }
-    case 0x5d: /* D-MMU data access */
+    case ASI_DTLB_DATA_ACCESS: /* D-MMU data access */
         {
             int reg = (addr >> 3) & 0x3f;
 
             ret = env->dtlb[reg].tte;
             break;
         }
-    case 0x5e: /* D-MMU tag read */
+    case ASI_DTLB_TAG_READ: /* D-MMU tag read */
         {
             int reg = (addr >> 3) & 0x3f;
 
             ret = env->dtlb[reg].tag;
             break;
         }
-    case 0x48: /* Interrupt dispatch, RO */
+    case ASI_INTR_DISPATCH_STAT: /* Interrupt dispatch, RO */
         break;
-    case 0x49: /* Interrupt data receive */
+    case ASI_INTR_RECEIVE: /* Interrupt data receive */
         ret = env->ivec_status;
         break;
-    case 0x7f: /* Incoming interrupt vector, RO */
+    case ASI_INTR_R: /* Incoming interrupt vector, RO */
         {
             int reg = (addr >> 4) & 0x3;
             if (reg < 3) {
@@ -1567,40 +1574,59 @@
             }
             break;
         }
-    case 0x46: /* D-cache data */
-    case 0x47: /* D-cache tag access */
-    case 0x4b: /* E-cache error enable */
-    case 0x4c: /* E-cache asynchronous fault status */
-    case 0x4d: /* E-cache asynchronous fault address */
-    case 0x4e: /* E-cache tag data */
-    case 0x66: /* I-cache instruction access */
-    case 0x67: /* I-cache tag access */
-    case 0x6e: /* I-cache predecode */
-    case 0x6f: /* I-cache LRU etc. */
-    case 0x76: /* E-cache tag */
-    case 0x7e: /* E-cache tag */
+    case ASI_DCACHE_DATA:     /* D-cache data */
+    case ASI_DCACHE_TAG:      /* D-cache tag access */
+    case ASI_ESTATE_ERROR_EN: /* E-cache error enable */
+    case ASI_AFSR:            /* E-cache asynchronous fault status */
+    case ASI_AFAR:            /* E-cache asynchronous fault address */
+    case ASI_EC_TAG_DATA:     /* E-cache tag data */
+    case ASI_IC_INSTR:        /* I-cache instruction access */
+    case ASI_IC_TAG:          /* I-cache tag access */
+    case ASI_IC_PRE_DECODE:   /* I-cache predecode */
+    case ASI_IC_NEXT_FIELD:   /* I-cache LRU etc. */
+    case ASI_EC_W:            /* E-cache tag */
+    case ASI_EC_R:            /* E-cache tag */
         break;
-    case 0x5b: /* D-MMU data pointer */
-    case 0x54: /* I-MMU data in, WO */
-    case 0x57: /* I-MMU demap, WO */
-    case 0x5c: /* D-MMU data in, WO */
-    case 0x5f: /* D-MMU demap, WO */
-    case 0x77: /* Interrupt vector, WO */
+    case ASI_DMMU_TSB_DIRECT_PTR: /* D-MMU data pointer */
+    case ASI_ITLB_DATA_IN:        /* I-MMU data in, WO */
+    case ASI_IMMU_DEMAP:          /* I-MMU demap, WO */
+    case ASI_DTLB_DATA_IN:        /* D-MMU data in, WO */
+    case ASI_DMMU_DEMAP:          /* D-MMU demap, WO */
+    case ASI_INTR_W:              /* Interrupt vector, WO */
     default:
         cpu_unassigned_access(cs, addr, false, false, 1, size);
         ret = 0;
         break;
+
+    case ASI_NUCLEUS_QUAD_LDD:   /* Nucleus quad LDD 128 bit atomic */
+    case ASI_NUCLEUS_QUAD_LDD_L: /* Nucleus quad LDD 128 bit atomic LE */
+    case ASI_TWINX_AIUP:   /* As if user primary, twinx */
+    case ASI_TWINX_AIUS:   /* As if user secondary, twinx */
+    case ASI_TWINX_REAL:   /* Real address, twinx */
+    case ASI_TWINX_AIUP_L: /* As if user primary, twinx, LE */
+    case ASI_TWINX_AIUS_L: /* As if user secondary, twinx, LE */
+    case ASI_TWINX_REAL_L: /* Real address, twinx, LE */
+    case ASI_TWINX_N:  /* Nucleus, twinx */
+    case ASI_TWINX_NL: /* Nucleus, twinx, LE */
+    /* ??? From the UA2011 document; overlaps BLK_INIT_QUAD_LDD_* */
+    case ASI_TWINX_P:  /* Primary, twinx */
+    case ASI_TWINX_PL: /* Primary, twinx, LE */
+    case ASI_TWINX_S:  /* Secondary, twinx */
+    case ASI_TWINX_SL: /* Secondary, twinx, LE */
+        /* These are all 128-bit atomic; only ldda (now ldtxa) allowed */
+        helper_raise_exception(env, TT_ILL_INSN);
+        return 0;
     }
 
     /* Convert from little endian */
     switch (asi) {
-    case 0x0c: /* Nucleus Little Endian (LE) */
-    case 0x18: /* As if user primary LE */
-    case 0x19: /* As if user secondary LE */
-    case 0x1c: /* Bypass LE */
-    case 0x1d: /* Bypass, non-cacheable LE */
-    case 0x88: /* Primary LE */
-    case 0x89: /* Secondary LE */
+    case ASI_NL: /* Nucleus Little Endian (LE) */
+    case ASI_AIUPL: /* As if user primary LE */
+    case ASI_AIUSL: /* As if user secondary LE */
+    case ASI_REAL_L:      /* Bypass LE */
+    case ASI_REAL_IO_L: /* Bypass, non-cacheable LE */
+    case ASI_PL: /* Primary LE */
+    case ASI_SL: /* Secondary LE */
         switch(size) {
         case 2:
             ret = bswap16(ret);
@@ -1641,8 +1667,9 @@
 }
 
 void helper_st_asi(CPUSPARCState *env, target_ulong addr, target_ulong val,
-                   int asi, int size)
+                   int asi, uint32_t memop)
 {
+    int size = 1 << (memop & MO_SIZE);
     SPARCCPU *cpu = sparc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
 
@@ -1664,13 +1691,13 @@
 
     /* Convert to little endian */
     switch (asi) {
-    case 0x0c: /* Nucleus Little Endian (LE) */
-    case 0x18: /* As if user primary LE */
-    case 0x19: /* As if user secondary LE */
-    case 0x1c: /* Bypass LE */
-    case 0x1d: /* Bypass, non-cacheable LE */
-    case 0x88: /* Primary LE */
-    case 0x89: /* Secondary LE */
+    case ASI_NL: /* Nucleus Little Endian (LE) */
+    case ASI_AIUPL: /* As if user primary LE */
+    case ASI_AIUSL: /* As if user secondary LE */
+    case ASI_REAL_L: /* Bypass LE */
+    case ASI_REAL_IO_L: /* Bypass, non-cacheable LE */
+    case ASI_PL: /* Primary LE */
+    case ASI_SL: /* Secondary LE */
         switch (size) {
         case 2:
             val = bswap16(val);
@@ -1689,16 +1716,14 @@
     }
 
     switch (asi) {
-    case 0x10: /* As if user primary */
-    case 0x11: /* As if user secondary */
-    case 0x18: /* As if user primary LE */
-    case 0x19: /* As if user secondary LE */
-    case 0x80: /* Primary */
-    case 0x81: /* Secondary */
-    case 0x88: /* Primary LE */
-    case 0x89: /* Secondary LE */
-    case 0xe2: /* UA2007 Primary block init */
-    case 0xe3: /* UA2007 Secondary block init */
+    case ASI_AIUP:  /* As if user primary */
+    case ASI_AIUS:  /* As if user secondary */
+    case ASI_AIUPL: /* As if user primary LE */
+    case ASI_AIUSL: /* As if user secondary LE */
+    case ASI_P:  /* Primary */
+    case ASI_S:  /* Secondary */
+    case ASI_PL: /* Primary LE */
+    case ASI_SL: /* Secondary LE */
         if ((asi & 0x80) && (env->pstate & PS_PRIV)) {
             if (cpu_hypervisor_mode(env)) {
                 switch (size) {
@@ -1789,10 +1814,10 @@
             }
         }
         break;
-    case 0x14: /* Bypass */
-    case 0x15: /* Bypass, non-cacheable */
-    case 0x1c: /* Bypass LE */
-    case 0x1d: /* Bypass, non-cacheable LE */
+    case ASI_REAL:      /* Bypass */
+    case ASI_REAL_IO:   /* Bypass, non-cacheable */
+    case ASI_REAL_L:    /* Bypass LE */
+    case ASI_REAL_IO_L: /* Bypass, non-cacheable LE */
         {
             switch (size) {
             case 1:
@@ -1811,13 +1836,8 @@
             }
         }
         return;
-    case 0x24: /* Nucleus quad LDD 128 bit atomic */
-    case 0x2c: /* Nucleus quad LDD 128 bit atomic LE
-                  Only ldda allowed */
-        helper_raise_exception(env, TT_ILL_INSN);
-        return;
-    case 0x04: /* Nucleus */
-    case 0x0c: /* Nucleus Little Endian (LE) */
+    case ASI_N:  /* Nucleus */
+    case ASI_NL: /* Nucleus Little Endian (LE) */
         {
             switch (size) {
             case 1:
@@ -1837,10 +1857,10 @@
             break;
         }
 
-    case 0x4a: /* UPA config */
+    case ASI_UPA_CONFIG: /* UPA config */
         /* XXX */
         return;
-    case 0x45: /* LSU */
+    case ASI_LSU_CONTROL: /* LSU */
         {
             uint64_t oldreg;
 
@@ -1858,7 +1878,7 @@
             }
             return;
         }
-    case 0x50: /* I-MMU regs */
+    case ASI_IMMU: /* I-MMU regs */
         {
             int reg = (addr >> 3) & 0xf;
             uint64_t oldreg;
@@ -1902,10 +1922,10 @@
 #endif
             return;
         }
-    case 0x54: /* I-MMU data in */
+    case ASI_ITLB_DATA_IN: /* I-MMU data in */
         replace_tlb_1bit_lru(env->itlb, env->immu.tag_access, val, "immu", env);
         return;
-    case 0x55: /* I-MMU data access */
+    case ASI_ITLB_DATA_ACCESS: /* I-MMU data access */
         {
             /* TODO: auto demap */
 
@@ -1919,10 +1939,10 @@
 #endif
             return;
         }
-    case 0x57: /* I-MMU demap */
+    case ASI_IMMU_DEMAP: /* I-MMU demap */
         demap_tlb(env->itlb, addr, "immu", env);
         return;
-    case 0x58: /* D-MMU regs */
+    case ASI_DMMU: /* D-MMU regs */
         {
             int reg = (addr >> 3) & 0xf;
             uint64_t oldreg;
@@ -1975,10 +1995,10 @@
 #endif
             return;
         }
-    case 0x5c: /* D-MMU data in */
+    case ASI_DTLB_DATA_IN: /* D-MMU data in */
         replace_tlb_1bit_lru(env->dtlb, env->dmmu.tag_access, val, "dmmu", env);
         return;
-    case 0x5d: /* D-MMU data access */
+    case ASI_DTLB_DATA_ACCESS: /* D-MMU data access */
         {
             unsigned int i = (addr >> 3) & 0x3f;
 
@@ -1990,38 +2010,56 @@
 #endif
             return;
         }
-    case 0x5f: /* D-MMU demap */
+    case ASI_DMMU_DEMAP: /* D-MMU demap */
         demap_tlb(env->dtlb, addr, "dmmu", env);
         return;
-    case 0x49: /* Interrupt data receive */
+    case ASI_INTR_RECEIVE: /* Interrupt data receive */
         env->ivec_status = val & 0x20;
         return;
-    case 0x46: /* D-cache data */
-    case 0x47: /* D-cache tag access */
-    case 0x4b: /* E-cache error enable */
-    case 0x4c: /* E-cache asynchronous fault status */
-    case 0x4d: /* E-cache asynchronous fault address */
-    case 0x4e: /* E-cache tag data */
-    case 0x66: /* I-cache instruction access */
-    case 0x67: /* I-cache tag access */
-    case 0x6e: /* I-cache predecode */
-    case 0x6f: /* I-cache LRU etc. */
-    case 0x76: /* E-cache tag */
-    case 0x7e: /* E-cache tag */
+    case ASI_NUCLEUS_QUAD_LDD:   /* Nucleus quad LDD 128 bit atomic */
+    case ASI_NUCLEUS_QUAD_LDD_L: /* Nucleus quad LDD 128 bit atomic LE */
+    case ASI_TWINX_AIUP:   /* As if user primary, twinx */
+    case ASI_TWINX_AIUS:   /* As if user secondary, twinx */
+    case ASI_TWINX_REAL:   /* Real address, twinx */
+    case ASI_TWINX_AIUP_L: /* As if user primary, twinx, LE */
+    case ASI_TWINX_AIUS_L: /* As if user secondary, twinx, LE */
+    case ASI_TWINX_REAL_L: /* Real address, twinx, LE */
+    case ASI_TWINX_N:  /* Nucleus, twinx */
+    case ASI_TWINX_NL: /* Nucleus, twinx, LE */
+    /* ??? From the UA2011 document; overlaps BLK_INIT_QUAD_LDD_* */
+    case ASI_TWINX_P:  /* Primary, twinx */
+    case ASI_TWINX_PL: /* Primary, twinx, LE */
+    case ASI_TWINX_S:  /* Secondary, twinx */
+    case ASI_TWINX_SL: /* Secondary, twinx, LE */
+        /* Only stda allowed */
+        helper_raise_exception(env, TT_ILL_INSN);
         return;
-    case 0x51: /* I-MMU 8k TSB pointer, RO */
-    case 0x52: /* I-MMU 64k TSB pointer, RO */
-    case 0x56: /* I-MMU tag read, RO */
-    case 0x59: /* D-MMU 8k TSB pointer, RO */
-    case 0x5a: /* D-MMU 64k TSB pointer, RO */
-    case 0x5b: /* D-MMU data pointer, RO */
-    case 0x5e: /* D-MMU tag read, RO */
-    case 0x48: /* Interrupt dispatch, RO */
-    case 0x7f: /* Incoming interrupt vector, RO */
-    case 0x82: /* Primary no-fault, RO */
-    case 0x83: /* Secondary no-fault, RO */
-    case 0x8a: /* Primary no-fault LE, RO */
-    case 0x8b: /* Secondary no-fault LE, RO */
+    case ASI_DCACHE_DATA: /* D-cache data */
+    case ASI_DCACHE_TAG: /* D-cache tag access */
+    case ASI_ESTATE_ERROR_EN: /* E-cache error enable */
+    case ASI_AFSR: /* E-cache asynchronous fault status */
+    case ASI_AFAR: /* E-cache asynchronous fault address */
+    case ASI_EC_TAG_DATA: /* E-cache tag data */
+    case ASI_IC_INSTR: /* I-cache instruction access */
+    case ASI_IC_TAG: /* I-cache tag access */
+    case ASI_IC_PRE_DECODE: /* I-cache predecode */
+    case ASI_IC_NEXT_FIELD: /* I-cache LRU etc. */
+    case ASI_EC_W: /* E-cache tag */
+    case ASI_EC_R: /* E-cache tag */
+        return;
+    case ASI_IMMU_TSB_8KB_PTR: /* I-MMU 8k TSB pointer, RO */
+    case ASI_IMMU_TSB_64KB_PTR: /* I-MMU 64k TSB pointer, RO */
+    case ASI_ITLB_TAG_READ: /* I-MMU tag read, RO */
+    case ASI_DMMU_TSB_8KB_PTR: /* D-MMU 8k TSB pointer, RO */
+    case ASI_DMMU_TSB_64KB_PTR: /* D-MMU 64k TSB pointer, RO */
+    case ASI_DMMU_TSB_DIRECT_PTR: /* D-MMU data pointer, RO */
+    case ASI_DTLB_TAG_READ: /* D-MMU tag read, RO */
+    case ASI_INTR_DISPATCH_STAT: /* Interrupt dispatch, RO */
+    case ASI_INTR_R: /* Incoming interrupt vector, RO */
+    case ASI_PNF: /* Primary no-fault, RO */
+    case ASI_SNF: /* Secondary no-fault, RO */
+    case ASI_PNFL: /* Primary no-fault LE, RO */
+    case ASI_SNFL: /* Secondary no-fault LE, RO */
     default:
         cpu_unassigned_access(cs, addr, true, false, 1, size);
         return;
@@ -2029,8 +2067,11 @@
 }
 #endif /* CONFIG_USER_ONLY */
 
-void helper_ldda_asi(CPUSPARCState *env, target_ulong addr, int asi, int rd)
+/* 128-bit LDDA; result returned in QT0.  */
+void helper_ldda_asi(CPUSPARCState *env, target_ulong addr, int asi)
 {
+    uint64_t h, l;
+
     if ((asi < 0x80 && (env->pstate & PS_PRIV) == 0)
         || (cpu_has_hypervisor(env)
             && asi >= 0x30 && asi < 0x80
@@ -2042,191 +2083,82 @@
 
     switch (asi) {
 #if !defined(CONFIG_USER_ONLY)
-    case 0x24: /* Nucleus quad LDD 128 bit atomic */
-    case 0x2c: /* Nucleus quad LDD 128 bit atomic LE */
+    case ASI_TWINX_AIUP:   /* As if user primary, twinx */
+    case ASI_TWINX_AIUP_L: /* As if user primary, twinx, LE */
         helper_check_align(env, addr, 0xf);
-        if (rd == 0) {
-            env->gregs[1] = cpu_ldq_nucleus(env, addr + 8);
-            if (asi == 0x2c) {
-                bswap64s(&env->gregs[1]);
-            }
-        } else if (rd < 8) {
-            env->gregs[rd] = cpu_ldq_nucleus(env, addr);
-            env->gregs[rd + 1] = cpu_ldq_nucleus(env, addr + 8);
-            if (asi == 0x2c) {
-                bswap64s(&env->gregs[rd]);
-                bswap64s(&env->gregs[rd + 1]);
-            }
-        } else {
-            env->regwptr[rd] = cpu_ldq_nucleus(env, addr);
-            env->regwptr[rd + 1] = cpu_ldq_nucleus(env, addr + 8);
-            if (asi == 0x2c) {
-                bswap64s(&env->regwptr[rd]);
-                bswap64s(&env->regwptr[rd + 1]);
-            }
+        h = cpu_ldq_user(env, addr);
+        l = cpu_ldq_user(env, addr + 8);
+        break;
+    case ASI_TWINX_AIUS:   /* As if user secondary, twinx */
+    case ASI_TWINX_AIUS_L: /* As if user secondary, twinx, LE */
+        helper_check_align(env, addr, 0xf);
+        h = cpu_ldq_user_secondary(env, addr);
+        l = cpu_ldq_user_secondary(env, addr + 8);
+        break;
+    case ASI_TWINX_REAL:   /* Real address, twinx */
+    case ASI_TWINX_REAL_L: /* Real address, twinx, LE */
+        helper_check_align(env, addr, 0xf);
+        {
+            CPUState *cs = CPU(sparc_env_get_cpu(env));
+            h = ldq_phys(cs->as, addr);
+            l = ldq_phys(cs->as, addr + 8);
         }
         break;
+    case ASI_NUCLEUS_QUAD_LDD:
+    case ASI_NUCLEUS_QUAD_LDD_L:
+    case ASI_TWINX_N:  /* Nucleus, twinx */
+    case ASI_TWINX_NL: /* Nucleus, twinx, LE */
+        helper_check_align(env, addr, 0xf);
+        h = cpu_ldq_nucleus(env, addr);
+        l = cpu_ldq_nucleus(env, addr + 8);
+        break;
+    case ASI_TWINX_S: /* Secondary, twinx */
+    case ASI_TWINX_SL: /* Secondary, twinx, LE */
+        if (!cpu_hypervisor_mode(env)) {
+            helper_check_align(env, addr, 0xf);
+            if (env->pstate & PS_PRIV) {
+                h = cpu_ldq_kernel_secondary(env, addr);
+                l = cpu_ldq_kernel_secondary(env, addr + 8);
+            } else {
+                h = cpu_ldq_user_secondary(env, addr);
+                l = cpu_ldq_user_secondary(env, addr + 8);
+            }
+            break;
+        }
+        /* fallthru */
+    case ASI_TWINX_P:  /* Primary, twinx */
+    case ASI_TWINX_PL: /* Primary, twinx, LE */
+        helper_check_align(env, addr, 0xf);
+        h = cpu_ldq_data(env, addr);
+        l = cpu_ldq_data(env, addr + 8);
+        break;
+#else
+    case ASI_TWINX_P:  /* Primary, twinx */
+    case ASI_TWINX_PL: /* Primary, twinx, LE */
+    case ASI_TWINX_S:  /* Primary, twinx */
+    case ASI_TWINX_SL: /* Primary, twinx, LE */
+        /* ??? Should be available, but we need to implement
+           an atomic 128-bit load.  */
+        helper_raise_exception(env, TT_PRIV_ACT);
 #endif
     default:
-        helper_check_align(env, addr, 0x3);
-        if (rd == 0) {
-            env->gregs[1] = helper_ld_asi(env, addr + 4, asi, 4, 0);
-        } else if (rd < 8) {
-            env->gregs[rd] = helper_ld_asi(env, addr, asi, 4, 0);
-            env->gregs[rd + 1] = helper_ld_asi(env, addr + 4, asi, 4, 0);
-        } else {
-            env->regwptr[rd] = helper_ld_asi(env, addr, asi, 4, 0);
-            env->regwptr[rd + 1] = helper_ld_asi(env, addr + 4, asi, 4, 0);
-        }
-        break;
-    }
-}
-
-void helper_ldf_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
-                    int rd)
-{
-    unsigned int i;
-    target_ulong val;
-
-    helper_check_align(env, addr, 3);
-    addr = asi_address_mask(env, asi, addr);
-
-    switch (asi) {
-    case 0xf0: /* UA2007/JPS1 Block load primary */
-    case 0xf1: /* UA2007/JPS1 Block load secondary */
-    case 0xf8: /* UA2007/JPS1 Block load primary LE */
-    case 0xf9: /* UA2007/JPS1 Block load secondary LE */
-        if (rd & 7) {
-            helper_raise_exception(env, TT_ILL_INSN);
-            return;
-        }
-        helper_check_align(env, addr, 0x3f);
-        for (i = 0; i < 8; i++, rd += 2, addr += 8) {
-            env->fpr[rd / 2].ll = helper_ld_asi(env, addr, asi & 0x8f, 8, 0);
-        }
+        /* Non-twinx asi, so this is the legacy ldda insn, which
+           performs two word sized operations.  */
+        /* ??? The UA2011 manual recommends emulating this with
+           a single 64-bit load.  However, LE asis *are* treated
+           as two 32-bit loads individually byte swapped.  */
+        helper_check_align(env, addr, 0x7);
+        QT0.high = (uint32_t)helper_ld_asi(env, addr, asi, MO_UL);
+        QT0.low = (uint32_t)helper_ld_asi(env, addr + 4, asi, MO_UL);
         return;
-
-    case 0x16: /* UA2007 Block load primary, user privilege */
-    case 0x17: /* UA2007 Block load secondary, user privilege */
-    case 0x1e: /* UA2007 Block load primary LE, user privilege */
-    case 0x1f: /* UA2007 Block load secondary LE, user privilege */
-    case 0x70: /* JPS1 Block load primary, user privilege */
-    case 0x71: /* JPS1 Block load secondary, user privilege */
-    case 0x78: /* JPS1 Block load primary LE, user privilege */
-    case 0x79: /* JPS1 Block load secondary LE, user privilege */
-        if (rd & 7) {
-            helper_raise_exception(env, TT_ILL_INSN);
-            return;
-        }
-        helper_check_align(env, addr, 0x3f);
-        for (i = 0; i < 8; i++, rd += 2, addr += 8) {
-            env->fpr[rd / 2].ll = helper_ld_asi(env, addr, asi & 0x19, 8, 0);
-        }
-        return;
-
-    default:
-        break;
     }
 
-    switch (size) {
-    default:
-    case 4:
-        val = helper_ld_asi(env, addr, asi, size, 0);
-        if (rd & 1) {
-            env->fpr[rd / 2].l.lower = val;
-        } else {
-            env->fpr[rd / 2].l.upper = val;
-        }
-        break;
-    case 8:
-        env->fpr[rd / 2].ll = helper_ld_asi(env, addr, asi, size, 0);
-        break;
-    case 16:
-        env->fpr[rd / 2].ll = helper_ld_asi(env, addr, asi, 8, 0);
-        env->fpr[rd / 2 + 1].ll = helper_ld_asi(env, addr + 8, asi, 8, 0);
-        break;
+    if (asi & 8) {
+        h = bswap64(h);
+        l = bswap64(l);
     }
-}
-
-void helper_stf_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
-                    int rd)
-{
-    unsigned int i;
-    target_ulong val;
-
-    addr = asi_address_mask(env, asi, addr);
-
-    switch (asi) {
-    case 0xe0: /* UA2007/JPS1 Block commit store primary (cache flush) */
-    case 0xe1: /* UA2007/JPS1 Block commit store secondary (cache flush) */
-    case 0xf0: /* UA2007/JPS1 Block store primary */
-    case 0xf1: /* UA2007/JPS1 Block store secondary */
-    case 0xf8: /* UA2007/JPS1 Block store primary LE */
-    case 0xf9: /* UA2007/JPS1 Block store secondary LE */
-        if (rd & 7) {
-            helper_raise_exception(env, TT_ILL_INSN);
-            return;
-        }
-        helper_check_align(env, addr, 0x3f);
-        for (i = 0; i < 8; i++, rd += 2, addr += 8) {
-            helper_st_asi(env, addr, env->fpr[rd / 2].ll, asi & 0x8f, 8);
-        }
-
-        return;
-    case 0x16: /* UA2007 Block load primary, user privilege */
-    case 0x17: /* UA2007 Block load secondary, user privilege */
-    case 0x1e: /* UA2007 Block load primary LE, user privilege */
-    case 0x1f: /* UA2007 Block load secondary LE, user privilege */
-    case 0x70: /* JPS1 Block store primary, user privilege */
-    case 0x71: /* JPS1 Block store secondary, user privilege */
-    case 0x78: /* JPS1 Block load primary LE, user privilege */
-    case 0x79: /* JPS1 Block load secondary LE, user privilege */
-        if (rd & 7) {
-            helper_raise_exception(env, TT_ILL_INSN);
-            return;
-        }
-        helper_check_align(env, addr, 0x3f);
-        for (i = 0; i < 8; i++, rd += 2, addr += 8) {
-            helper_st_asi(env, addr, env->fpr[rd / 2].ll, asi & 0x19, 8);
-        }
-
-        return;
-    case 0xd2: /* 16-bit floating point load primary */
-    case 0xd3: /* 16-bit floating point load secondary */
-    case 0xda: /* 16-bit floating point load primary, LE */
-    case 0xdb: /* 16-bit floating point load secondary, LE */
-        helper_check_align(env, addr, 1);
-        /* Fall through */
-    case 0xd0: /* 8-bit floating point load primary */
-    case 0xd1: /* 8-bit floating point load secondary */
-    case 0xd8: /* 8-bit floating point load primary, LE */
-    case 0xd9: /* 8-bit floating point load secondary, LE */
-        val = env->fpr[rd / 2].l.lower;
-        helper_st_asi(env, addr, val, asi & 0x8d, ((asi & 2) >> 1) + 1);
-        return;
-    default:
-        helper_check_align(env, addr, 3);
-        break;
-    }
-
-    switch (size) {
-    default:
-    case 4:
-        if (rd & 1) {
-            val = env->fpr[rd / 2].l.lower;
-        } else {
-            val = env->fpr[rd / 2].l.upper;
-        }
-        helper_st_asi(env, addr, val, asi, size);
-        break;
-    case 8:
-        helper_st_asi(env, addr, env->fpr[rd / 2].ll, asi, size);
-        break;
-    case 16:
-        helper_st_asi(env, addr, env->fpr[rd / 2].ll, asi, 8);
-        helper_st_asi(env, addr + 8, env->fpr[rd / 2 + 1].ll, asi, 8);
-        break;
-    }
+    QT0.high = h;
+    QT0.low = l;
 }
 
 target_ulong helper_casx_asi(CPUSPARCState *env, target_ulong addr,
@@ -2235,9 +2167,9 @@
 {
     target_ulong ret;
 
-    ret = helper_ld_asi(env, addr, asi, 8, 0);
+    ret = helper_ld_asi(env, addr, asi, MO_Q);
     if (val2 == ret) {
-        helper_st_asi(env, addr, val1, asi, 8);
+        helper_st_asi(env, addr, val1, asi, MO_Q);
     }
     return ret;
 }
@@ -2250,10 +2182,10 @@
     target_ulong ret;
 
     val2 &= 0xffffffffUL;
-    ret = helper_ld_asi(env, addr, asi, 4, 0);
+    ret = helper_ld_asi(env, addr, asi, MO_UL);
     ret &= 0xffffffffUL;
     if (val2 == ret) {
-        helper_st_asi(env, addr, val1 & 0xffffffffUL, asi, 4);
+        helper_st_asi(env, addr, val1 & 0xffffffffUL, asi, MO_UL);
     }
     return ret;
 }
@@ -2289,8 +2221,8 @@
         break;
     }
 #else
-    u.ll.upper = ldq_raw(address_mask(env, addr));
-    u.ll.lower = ldq_raw(address_mask(env, addr + 8));
+    u.ll.upper = cpu_ldq_data(env, address_mask(env, addr));
+    u.ll.lower = cpu_ldq_data(env, address_mask(env, addr + 8));
     QT0 = u.q;
 #endif
 }
@@ -2326,8 +2258,8 @@
     }
 #else
     u.q = QT0;
-    stq_raw(address_mask(env, addr), u.ll.upper);
-    stq_raw(address_mask(env, addr + 8), u.ll.lower);
+    cpu_stq_data(env, address_mask(env, addr), u.ll.upper);
+    cpu_stq_data(env, address_mask(env, addr + 8), u.ll.lower);
 #endif
 }
 
@@ -2418,9 +2350,11 @@
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-void QEMU_NORETURN sparc_cpu_do_unaligned_access(CPUState *cs,
-                                                 vaddr addr, int is_write,
-                                                 int is_user, uintptr_t retaddr)
+void QEMU_NORETURN sparc_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
+                                                 MMUAccessType access_type,
+                                                 int mmu_idx,
+                                                 uintptr_t retaddr,
+                                                 unsigned size)
 {
     SPARCCPU *cpu = SPARC_CPU(cs);
     CPUSPARCState *env = &cpu->env;
@@ -2439,12 +2373,12 @@
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
 /* XXX: fix it to restore all registers */
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = sparc_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = sparc_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (ret) {
         if (retaddr) {
             cpu_restore_state(cs, retaddr);
diff --git a/target-sparc/machine.c b/target-sparc/machine.c
index 3f3de4c..59c92f7 100644
--- a/target-sparc/machine.c
+++ b/target-sparc/machine.c
@@ -1,218 +1,197 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
 #include "qemu/timer.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
+#include "migration/cpu.h"
+#include "exec/exec-all.h"
 
-void cpu_save(QEMUFile *f, void *opaque)
-{
-    CPUSPARCState *env = opaque;
-    int i;
-    uint32_t tmp;
+#ifdef TARGET_SPARC64
+static const VMStateDescription vmstate_cpu_timer = {
+    .name = "cpu_timer",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(frequency, CPUTimer),
+        VMSTATE_UINT32(disabled, CPUTimer),
+        VMSTATE_UINT64(disabled_mask, CPUTimer),
+        VMSTATE_UINT32(npt, CPUTimer),
+        VMSTATE_UINT64(npt_mask, CPUTimer),
+        VMSTATE_INT64(clock_offset, CPUTimer),
+        VMSTATE_TIMER_PTR(qtimer, CPUTimer),
+        VMSTATE_END_OF_LIST()
+    }
+};
 
-    // if env->cwp == env->nwindows - 1, this will set the ins of the last
-    // window as the outs of the first window
-    cpu_set_cwp(env, env->cwp);
+#define VMSTATE_CPU_TIMER(_f, _s)                             \
+    VMSTATE_STRUCT_POINTER(_f, _s, vmstate_cpu_timer, CPUTimer)
 
-    for(i = 0; i < 8; i++)
-        qemu_put_betls(f, &env->gregs[i]);
-    qemu_put_be32s(f, &env->nwindows);
-    for(i = 0; i < env->nwindows * 16; i++)
-        qemu_put_betls(f, &env->regbase[i]);
+static const VMStateDescription vmstate_trap_state = {
+    .name = "trap_state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(tpc, trap_state),
+        VMSTATE_UINT64(tnpc, trap_state),
+        VMSTATE_UINT64(tstate, trap_state),
+        VMSTATE_UINT32(tt, trap_state),
+        VMSTATE_END_OF_LIST()
+    }
+};
 
-    /* FPU */
-    for (i = 0; i < TARGET_DPREGS; i++) {
-        qemu_put_be32(f, env->fpr[i].l.upper);
-        qemu_put_be32(f, env->fpr[i].l.lower);
+static const VMStateDescription vmstate_tlb_entry = {
+    .name = "tlb_entry",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(tag, SparcTLBEntry),
+        VMSTATE_UINT64(tte, SparcTLBEntry),
+        VMSTATE_END_OF_LIST()
     }
-
-    qemu_put_betls(f, &env->pc);
-    qemu_put_betls(f, &env->npc);
-    qemu_put_betls(f, &env->y);
-    tmp = cpu_get_psr(env);
-    qemu_put_be32(f, tmp);
-    qemu_put_betls(f, &env->fsr);
-    qemu_put_betls(f, &env->tbr);
-    tmp = env->interrupt_index;
-    qemu_put_be32(f, tmp);
-    qemu_put_be32s(f, &env->pil_in);
-#ifndef TARGET_SPARC64
-    qemu_put_be32s(f, &env->wim);
-    /* MMU */
-    for (i = 0; i < 32; i++)
-        qemu_put_be32s(f, &env->mmuregs[i]);
-    for (i = 0; i < 4; i++) {
-        qemu_put_be64s(f, &env->mxccdata[i]);
-    }
-    for (i = 0; i < 8; i++) {
-        qemu_put_be64s(f, &env->mxccregs[i]);
-    }
-    qemu_put_be32s(f, &env->mmubpctrv);
-    qemu_put_be32s(f, &env->mmubpctrc);
-    qemu_put_be32s(f, &env->mmubpctrs);
-    qemu_put_be64s(f, &env->mmubpaction);
-    for (i = 0; i < 4; i++) {
-        qemu_put_be64s(f, &env->mmubpregs[i]);
-    }
-#else
-    qemu_put_be64s(f, &env->lsu);
-    for (i = 0; i < 16; i++) {
-        qemu_put_be64s(f, &env->immuregs[i]);
-        qemu_put_be64s(f, &env->dmmuregs[i]);
-    }
-    for (i = 0; i < 64; i++) {
-        qemu_put_be64s(f, &env->itlb[i].tag);
-        qemu_put_be64s(f, &env->itlb[i].tte);
-        qemu_put_be64s(f, &env->dtlb[i].tag);
-        qemu_put_be64s(f, &env->dtlb[i].tte);
-    }
-    qemu_put_be32s(f, &env->mmu_version);
-    for (i = 0; i < MAXTL_MAX; i++) {
-        qemu_put_be64s(f, &env->ts[i].tpc);
-        qemu_put_be64s(f, &env->ts[i].tnpc);
-        qemu_put_be64s(f, &env->ts[i].tstate);
-        qemu_put_be32s(f, &env->ts[i].tt);
-    }
-    qemu_put_be32s(f, &env->xcc);
-    qemu_put_be32s(f, &env->asi);
-    qemu_put_be32s(f, &env->pstate);
-    qemu_put_be32s(f, &env->tl);
-    qemu_put_be32s(f, &env->cansave);
-    qemu_put_be32s(f, &env->canrestore);
-    qemu_put_be32s(f, &env->otherwin);
-    qemu_put_be32s(f, &env->wstate);
-    qemu_put_be32s(f, &env->cleanwin);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->agregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->bgregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->igregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_put_be64s(f, &env->mgregs[i]);
-    qemu_put_be64s(f, &env->fprs);
-    qemu_put_be64s(f, &env->tick_cmpr);
-    qemu_put_be64s(f, &env->stick_cmpr);
-    cpu_put_timer(f, env->tick);
-    cpu_put_timer(f, env->stick);
-    qemu_put_be64s(f, &env->gsr);
-    qemu_put_be32s(f, &env->gl);
-    qemu_put_be64s(f, &env->hpstate);
-    for (i = 0; i < MAXTL_MAX; i++)
-        qemu_put_be64s(f, &env->htstate[i]);
-    qemu_put_be64s(f, &env->hintp);
-    qemu_put_be64s(f, &env->htba);
-    qemu_put_be64s(f, &env->hver);
-    qemu_put_be64s(f, &env->hstick_cmpr);
-    qemu_put_be64s(f, &env->ssr);
-    cpu_put_timer(f, env->hstick);
+};
 #endif
-}
 
-int cpu_load(QEMUFile *f, void *opaque, int version_id)
+static int get_psr(QEMUFile *f, void *opaque, size_t size)
 {
-    CPUSPARCState *env = opaque;
-    SPARCCPU *cpu = sparc_env_get_cpu(env);
-    int i;
-    uint32_t tmp;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
+    uint32_t val = qemu_get_be32(f);
 
-    if (version_id < 6)
-        return -EINVAL;
-    for(i = 0; i < 8; i++)
-        qemu_get_betls(f, &env->gregs[i]);
-    qemu_get_be32s(f, &env->nwindows);
-    for(i = 0; i < env->nwindows * 16; i++)
-        qemu_get_betls(f, &env->regbase[i]);
+    /* needed to ensure that the wrapping registers are correctly updated */
+    env->cwp = 0;
+    cpu_put_psr_raw(env, val);
 
-    /* FPU */
-    for (i = 0; i < TARGET_DPREGS; i++) {
-        env->fpr[i].l.upper = qemu_get_be32(f);
-        env->fpr[i].l.lower = qemu_get_be32(f);
-    }
-
-    qemu_get_betls(f, &env->pc);
-    qemu_get_betls(f, &env->npc);
-    qemu_get_betls(f, &env->y);
-    tmp = qemu_get_be32(f);
-    env->cwp = 0; /* needed to ensure that the wrapping registers are
-                     correctly updated */
-    cpu_put_psr(env, tmp);
-    qemu_get_betls(f, &env->fsr);
-    qemu_get_betls(f, &env->tbr);
-    tmp = qemu_get_be32(f);
-    env->interrupt_index = tmp;
-    qemu_get_be32s(f, &env->pil_in);
-#ifndef TARGET_SPARC64
-    qemu_get_be32s(f, &env->wim);
-    /* MMU */
-    for (i = 0; i < 32; i++)
-        qemu_get_be32s(f, &env->mmuregs[i]);
-    for (i = 0; i < 4; i++) {
-        qemu_get_be64s(f, &env->mxccdata[i]);
-    }
-    for (i = 0; i < 8; i++) {
-        qemu_get_be64s(f, &env->mxccregs[i]);
-    }
-    qemu_get_be32s(f, &env->mmubpctrv);
-    qemu_get_be32s(f, &env->mmubpctrc);
-    qemu_get_be32s(f, &env->mmubpctrs);
-    qemu_get_be64s(f, &env->mmubpaction);
-    for (i = 0; i < 4; i++) {
-        qemu_get_be64s(f, &env->mmubpregs[i]);
-    }
-#else
-    qemu_get_be64s(f, &env->lsu);
-    for (i = 0; i < 16; i++) {
-        qemu_get_be64s(f, &env->immuregs[i]);
-        qemu_get_be64s(f, &env->dmmuregs[i]);
-    }
-    for (i = 0; i < 64; i++) {
-        qemu_get_be64s(f, &env->itlb[i].tag);
-        qemu_get_be64s(f, &env->itlb[i].tte);
-        qemu_get_be64s(f, &env->dtlb[i].tag);
-        qemu_get_be64s(f, &env->dtlb[i].tte);
-    }
-    qemu_get_be32s(f, &env->mmu_version);
-    for (i = 0; i < MAXTL_MAX; i++) {
-        qemu_get_be64s(f, &env->ts[i].tpc);
-        qemu_get_be64s(f, &env->ts[i].tnpc);
-        qemu_get_be64s(f, &env->ts[i].tstate);
-        qemu_get_be32s(f, &env->ts[i].tt);
-    }
-    qemu_get_be32s(f, &env->xcc);
-    qemu_get_be32s(f, &env->asi);
-    qemu_get_be32s(f, &env->pstate);
-    qemu_get_be32s(f, &env->tl);
-    qemu_get_be32s(f, &env->cansave);
-    qemu_get_be32s(f, &env->canrestore);
-    qemu_get_be32s(f, &env->otherwin);
-    qemu_get_be32s(f, &env->wstate);
-    qemu_get_be32s(f, &env->cleanwin);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->agregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->bgregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->igregs[i]);
-    for (i = 0; i < 8; i++)
-        qemu_get_be64s(f, &env->mgregs[i]);
-    qemu_get_be64s(f, &env->fprs);
-    qemu_get_be64s(f, &env->tick_cmpr);
-    qemu_get_be64s(f, &env->stick_cmpr);
-    cpu_get_timer(f, env->tick);
-    cpu_get_timer(f, env->stick);
-    qemu_get_be64s(f, &env->gsr);
-    qemu_get_be32s(f, &env->gl);
-    qemu_get_be64s(f, &env->hpstate);
-    for (i = 0; i < MAXTL_MAX; i++)
-        qemu_get_be64s(f, &env->htstate[i]);
-    qemu_get_be64s(f, &env->hintp);
-    qemu_get_be64s(f, &env->htba);
-    qemu_get_be64s(f, &env->hver);
-    qemu_get_be64s(f, &env->hstick_cmpr);
-    qemu_get_be64s(f, &env->ssr);
-    cpu_get_timer(f, env->hstick);
-#endif
-    tlb_flush(CPU(cpu), 1);
     return 0;
 }
+
+static void put_psr(QEMUFile *f, void *opaque, size_t size)
+{
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
+    uint32_t val;
+
+    val = cpu_get_psr(env);
+
+    qemu_put_be32(f, val);
+}
+
+static const VMStateInfo vmstate_psr = {
+    .name = "psr",
+    .get = get_psr,
+    .put = put_psr,
+};
+
+static void cpu_pre_save(void *opaque)
+{
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
+
+    /* if env->cwp == env->nwindows - 1, this will set the ins of the last
+     * window as the outs of the first window
+     */
+    cpu_set_cwp(env, env->cwp);
+}
+
+/* 32-bit SPARC retains migration compatibility with older versions
+ * of QEMU; 64-bit SPARC has had a migration break since then, so the
+ * versions are different.
+ */
+#ifndef TARGET_SPARC64
+#define SPARC_VMSTATE_VER 7
+#else
+#define SPARC_VMSTATE_VER 9
+#endif
+
+const VMStateDescription vmstate_sparc_cpu = {
+    .name = "cpu",
+    .version_id = SPARC_VMSTATE_VER,
+    .minimum_version_id = SPARC_VMSTATE_VER,
+    .minimum_version_id_old = SPARC_VMSTATE_VER,
+    .pre_save = cpu_pre_save,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL_ARRAY(env.gregs, SPARCCPU, 8),
+        VMSTATE_UINT32(env.nwindows, SPARCCPU),
+        VMSTATE_VARRAY_MULTIPLY(env.regbase, SPARCCPU, env.nwindows, 16,
+                                vmstate_info_uinttl, target_ulong),
+        VMSTATE_CPUDOUBLE_ARRAY(env.fpr, SPARCCPU, TARGET_DPREGS),
+        VMSTATE_UINTTL(env.pc, SPARCCPU),
+        VMSTATE_UINTTL(env.npc, SPARCCPU),
+        VMSTATE_UINTTL(env.y, SPARCCPU),
+        {
+
+            .name = "psr",
+            .version_id = 0,
+            .size = sizeof(uint32_t),
+            .info = &vmstate_psr,
+            .flags = VMS_SINGLE,
+            .offset = 0,
+        },
+        VMSTATE_UINTTL(env.fsr, SPARCCPU),
+        VMSTATE_UINTTL(env.tbr, SPARCCPU),
+        VMSTATE_INT32(env.interrupt_index, SPARCCPU),
+        VMSTATE_UINT32(env.pil_in, SPARCCPU),
+#ifndef TARGET_SPARC64
+        /* MMU */
+        VMSTATE_UINT32(env.wim, SPARCCPU),
+        VMSTATE_UINT32_ARRAY(env.mmuregs, SPARCCPU, 32),
+        VMSTATE_UINT64_ARRAY(env.mxccdata, SPARCCPU, 4),
+        VMSTATE_UINT64_ARRAY(env.mxccregs, SPARCCPU, 8),
+        VMSTATE_UINT32(env.mmubpctrv, SPARCCPU),
+        VMSTATE_UINT32(env.mmubpctrc, SPARCCPU),
+        VMSTATE_UINT32(env.mmubpctrs, SPARCCPU),
+        VMSTATE_UINT64(env.mmubpaction, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.mmubpregs, SPARCCPU, 4),
+#else
+        VMSTATE_UINT64(env.lsu, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.immuregs, SPARCCPU, 16),
+        VMSTATE_UINT64_ARRAY(env.dmmuregs, SPARCCPU, 16),
+        VMSTATE_STRUCT_ARRAY(env.itlb, SPARCCPU, 64, 0,
+                             vmstate_tlb_entry, SparcTLBEntry),
+        VMSTATE_STRUCT_ARRAY(env.dtlb, SPARCCPU, 64, 0,
+                             vmstate_tlb_entry, SparcTLBEntry),
+        VMSTATE_UINT32(env.mmu_version, SPARCCPU),
+        VMSTATE_STRUCT_ARRAY(env.ts, SPARCCPU, MAXTL_MAX, 0,
+                             vmstate_trap_state, trap_state),
+        VMSTATE_UINT32(env.xcc, SPARCCPU),
+        VMSTATE_UINT32(env.asi, SPARCCPU),
+        VMSTATE_UINT32(env.pstate, SPARCCPU),
+        VMSTATE_UINT32(env.tl, SPARCCPU),
+        VMSTATE_UINT32(env.cansave, SPARCCPU),
+        VMSTATE_UINT32(env.canrestore, SPARCCPU),
+        VMSTATE_UINT32(env.otherwin, SPARCCPU),
+        VMSTATE_UINT32(env.wstate, SPARCCPU),
+        VMSTATE_UINT32(env.cleanwin, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.agregs, SPARCCPU, 8),
+        VMSTATE_UINT64_ARRAY(env.bgregs, SPARCCPU, 8),
+        VMSTATE_UINT64_ARRAY(env.igregs, SPARCCPU, 8),
+        VMSTATE_UINT64_ARRAY(env.mgregs, SPARCCPU, 8),
+        VMSTATE_UINT64(env.fprs, SPARCCPU),
+        VMSTATE_UINT64(env.tick_cmpr, SPARCCPU),
+        VMSTATE_UINT64(env.stick_cmpr, SPARCCPU),
+        VMSTATE_CPU_TIMER(env.tick, SPARCCPU),
+        VMSTATE_CPU_TIMER(env.stick, SPARCCPU),
+        VMSTATE_UINT64(env.gsr, SPARCCPU),
+        VMSTATE_UINT32(env.gl, SPARCCPU),
+        VMSTATE_UINT64(env.hpstate, SPARCCPU),
+        VMSTATE_UINT64_ARRAY(env.htstate, SPARCCPU, MAXTL_MAX),
+        VMSTATE_UINT64(env.hintp, SPARCCPU),
+        VMSTATE_UINT64(env.htba, SPARCCPU),
+        VMSTATE_UINT64(env.hver, SPARCCPU),
+        VMSTATE_UINT64(env.hstick_cmpr, SPARCCPU),
+        VMSTATE_UINT64(env.ssr, SPARCCPU),
+        VMSTATE_CPU_TIMER(env.hstick, SPARCCPU),
+        /* On SPARC32 env.psrpil and env.cwp are migrated as part of the PSR */
+        VMSTATE_UINT32(env.psrpil, SPARCCPU),
+        VMSTATE_UINT32(env.cwp, SPARCCPU),
+#endif
+        VMSTATE_END_OF_LIST()
+    },
+};
diff --git a/target-sparc/mmu_helper.c b/target-sparc/mmu_helper.c
index 61afbcf..32b629f 100644
--- a/target-sparc/mmu_helper.c
+++ b/target-sparc/mmu_helper.c
@@ -17,7 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "trace.h"
 #include "exec/address-spaces.h"
 
@@ -213,10 +215,9 @@
                                       address, rw, mmu_idx, &page_size);
     vaddr = address;
     if (error_code == 0) {
-#ifdef DEBUG_MMU
-        printf("Translate at %" VADDR_PRIx " -> " TARGET_FMT_plx ", vaddr "
-               TARGET_FMT_lx "\n", address, paddr, vaddr);
-#endif
+        qemu_log_mask(CPU_LOG_MMU,
+                "Translate at %" VADDR_PRIx " -> " TARGET_FMT_plx ", vaddr "
+                TARGET_FMT_lx "\n", address, paddr, vaddr);
         tlb_set_page(cs, vaddr, paddr, prot, mmu_idx, page_size);
         return 0;
     }
@@ -850,7 +851,7 @@
     SPARCCPU *cpu = SPARC_CPU(cs);
     CPUSPARCState *env = &cpu->env;
     hwaddr phys_addr;
-    int mmu_idx = cpu_mmu_index(env);
+    int mmu_idx = cpu_mmu_index(env, false);
     MemoryRegionSection section;
 
     if (cpu_sparc_get_phys_page(env, &phys_addr, addr, 2, mmu_idx) != 0) {
diff --git a/target-sparc/monitor.c b/target-sparc/monitor.c
new file mode 100644
index 0000000..7cc1b0f
--- /dev/null
+++ b/target-sparc/monitor.c
@@ -0,0 +1,159 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
+
+#ifndef TARGET_SPARC64
+static target_long monitor_get_psr (const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+
+    return cpu_get_psr(env);
+}
+#endif
+
+static target_long monitor_get_reg(const struct MonitorDef *md, int val)
+{
+    CPUArchState *env = mon_get_cpu_env();
+    return env->regwptr[val];
+}
+
+const MonitorDef monitor_defs[] = {
+    { "g0", offsetof(CPUSPARCState, gregs[0]) },
+    { "g1", offsetof(CPUSPARCState, gregs[1]) },
+    { "g2", offsetof(CPUSPARCState, gregs[2]) },
+    { "g3", offsetof(CPUSPARCState, gregs[3]) },
+    { "g4", offsetof(CPUSPARCState, gregs[4]) },
+    { "g5", offsetof(CPUSPARCState, gregs[5]) },
+    { "g6", offsetof(CPUSPARCState, gregs[6]) },
+    { "g7", offsetof(CPUSPARCState, gregs[7]) },
+    { "o0", 0, monitor_get_reg },
+    { "o1", 1, monitor_get_reg },
+    { "o2", 2, monitor_get_reg },
+    { "o3", 3, monitor_get_reg },
+    { "o4", 4, monitor_get_reg },
+    { "o5", 5, monitor_get_reg },
+    { "o6", 6, monitor_get_reg },
+    { "o7", 7, monitor_get_reg },
+    { "l0", 8, monitor_get_reg },
+    { "l1", 9, monitor_get_reg },
+    { "l2", 10, monitor_get_reg },
+    { "l3", 11, monitor_get_reg },
+    { "l4", 12, monitor_get_reg },
+    { "l5", 13, monitor_get_reg },
+    { "l6", 14, monitor_get_reg },
+    { "l7", 15, monitor_get_reg },
+    { "i0", 16, monitor_get_reg },
+    { "i1", 17, monitor_get_reg },
+    { "i2", 18, monitor_get_reg },
+    { "i3", 19, monitor_get_reg },
+    { "i4", 20, monitor_get_reg },
+    { "i5", 21, monitor_get_reg },
+    { "i6", 22, monitor_get_reg },
+    { "i7", 23, monitor_get_reg },
+    { "pc", offsetof(CPUSPARCState, pc) },
+    { "npc", offsetof(CPUSPARCState, npc) },
+    { "y", offsetof(CPUSPARCState, y) },
+#ifndef TARGET_SPARC64
+    { "psr", 0, &monitor_get_psr, },
+    { "wim", offsetof(CPUSPARCState, wim) },
+#endif
+    { "tbr", offsetof(CPUSPARCState, tbr) },
+    { "fsr", offsetof(CPUSPARCState, fsr) },
+    { "f0", offsetof(CPUSPARCState, fpr[0].l.upper) },
+    { "f1", offsetof(CPUSPARCState, fpr[0].l.lower) },
+    { "f2", offsetof(CPUSPARCState, fpr[1].l.upper) },
+    { "f3", offsetof(CPUSPARCState, fpr[1].l.lower) },
+    { "f4", offsetof(CPUSPARCState, fpr[2].l.upper) },
+    { "f5", offsetof(CPUSPARCState, fpr[2].l.lower) },
+    { "f6", offsetof(CPUSPARCState, fpr[3].l.upper) },
+    { "f7", offsetof(CPUSPARCState, fpr[3].l.lower) },
+    { "f8", offsetof(CPUSPARCState, fpr[4].l.upper) },
+    { "f9", offsetof(CPUSPARCState, fpr[4].l.lower) },
+    { "f10", offsetof(CPUSPARCState, fpr[5].l.upper) },
+    { "f11", offsetof(CPUSPARCState, fpr[5].l.lower) },
+    { "f12", offsetof(CPUSPARCState, fpr[6].l.upper) },
+    { "f13", offsetof(CPUSPARCState, fpr[6].l.lower) },
+    { "f14", offsetof(CPUSPARCState, fpr[7].l.upper) },
+    { "f15", offsetof(CPUSPARCState, fpr[7].l.lower) },
+    { "f16", offsetof(CPUSPARCState, fpr[8].l.upper) },
+    { "f17", offsetof(CPUSPARCState, fpr[8].l.lower) },
+    { "f18", offsetof(CPUSPARCState, fpr[9].l.upper) },
+    { "f19", offsetof(CPUSPARCState, fpr[9].l.lower) },
+    { "f20", offsetof(CPUSPARCState, fpr[10].l.upper) },
+    { "f21", offsetof(CPUSPARCState, fpr[10].l.lower) },
+    { "f22", offsetof(CPUSPARCState, fpr[11].l.upper) },
+    { "f23", offsetof(CPUSPARCState, fpr[11].l.lower) },
+    { "f24", offsetof(CPUSPARCState, fpr[12].l.upper) },
+    { "f25", offsetof(CPUSPARCState, fpr[12].l.lower) },
+    { "f26", offsetof(CPUSPARCState, fpr[13].l.upper) },
+    { "f27", offsetof(CPUSPARCState, fpr[13].l.lower) },
+    { "f28", offsetof(CPUSPARCState, fpr[14].l.upper) },
+    { "f29", offsetof(CPUSPARCState, fpr[14].l.lower) },
+    { "f30", offsetof(CPUSPARCState, fpr[15].l.upper) },
+    { "f31", offsetof(CPUSPARCState, fpr[15].l.lower) },
+#ifdef TARGET_SPARC64
+    { "f32", offsetof(CPUSPARCState, fpr[16]) },
+    { "f34", offsetof(CPUSPARCState, fpr[17]) },
+    { "f36", offsetof(CPUSPARCState, fpr[18]) },
+    { "f38", offsetof(CPUSPARCState, fpr[19]) },
+    { "f40", offsetof(CPUSPARCState, fpr[20]) },
+    { "f42", offsetof(CPUSPARCState, fpr[21]) },
+    { "f44", offsetof(CPUSPARCState, fpr[22]) },
+    { "f46", offsetof(CPUSPARCState, fpr[23]) },
+    { "f48", offsetof(CPUSPARCState, fpr[24]) },
+    { "f50", offsetof(CPUSPARCState, fpr[25]) },
+    { "f52", offsetof(CPUSPARCState, fpr[26]) },
+    { "f54", offsetof(CPUSPARCState, fpr[27]) },
+    { "f56", offsetof(CPUSPARCState, fpr[28]) },
+    { "f58", offsetof(CPUSPARCState, fpr[29]) },
+    { "f60", offsetof(CPUSPARCState, fpr[30]) },
+    { "f62", offsetof(CPUSPARCState, fpr[31]) },
+    { "asi", offsetof(CPUSPARCState, asi) },
+    { "pstate", offsetof(CPUSPARCState, pstate) },
+    { "cansave", offsetof(CPUSPARCState, cansave) },
+    { "canrestore", offsetof(CPUSPARCState, canrestore) },
+    { "otherwin", offsetof(CPUSPARCState, otherwin) },
+    { "wstate", offsetof(CPUSPARCState, wstate) },
+    { "cleanwin", offsetof(CPUSPARCState, cleanwin) },
+    { "fprs", offsetof(CPUSPARCState, fprs) },
+#endif
+    { NULL },
+};
+
+const MonitorDef *target_monitor_defs(void)
+{
+    return monitor_defs;
+}
diff --git a/target-sparc/trace-events b/target-sparc/trace-events
new file mode 100644
index 0000000..bf52d97
--- /dev/null
+++ b/target-sparc/trace-events
@@ -0,0 +1,28 @@
+# See docs/tracing.txt for syntax documentation.
+
+# target-sparc/mmu_helper.c
+mmu_helper_dfault(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl) "DFAULT at %"PRIx64" context %"PRIx64" mmu_idx=%d tl=%d"
+mmu_helper_dprot(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl) "DPROT at %"PRIx64" context %"PRIx64" mmu_idx=%d tl=%d"
+mmu_helper_dmiss(uint64_t address, uint64_t context) "DMISS at %"PRIx64" context %"PRIx64
+mmu_helper_tfault(uint64_t address, uint64_t context) "TFAULT at %"PRIx64" context %"PRIx64
+mmu_helper_tmiss(uint64_t address, uint64_t context) "TMISS at %"PRIx64" context %"PRIx64
+mmu_helper_get_phys_addr_code(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64
+mmu_helper_get_phys_addr_data(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64
+mmu_helper_mmu_fault(uint64_t address, uint64_t paddr, int mmu_idx, uint32_t tl, uint64_t prim_context, uint64_t sec_context) "Translate at %"PRIx64" -> %"PRIx64", mmu_idx=%d tl=%d primary context=%"PRIx64" secondary context=%"PRIx64
+
+# target-sparc/int64_helper.c
+int_helper_set_softint(uint32_t softint) "new %08x"
+int_helper_clear_softint(uint32_t softint) "new %08x"
+int_helper_write_softint(uint32_t softint) "new %08x"
+
+# target-sparc/int32_helper.c
+int_helper_icache_freeze(void) "Instruction cache: freeze"
+int_helper_dcache_freeze(void) "Data cache: freeze"
+
+# target-sparc/win_helper.c
+win_helper_gregset_error(uint32_t pstate) "ERROR in get_gregset: active pstate bits=%x"
+win_helper_switch_pstate(uint32_t pstate_regs, uint32_t new_pstate_regs) "change_pstate: switching regs old=%x new=%x"
+win_helper_no_switch_pstate(uint32_t new_pstate_regs) "change_pstate: regs new=%x (unchanged)"
+win_helper_wrpil(uint32_t psrpil, uint32_t new_pil) "old=%x new=%x"
+win_helper_done(uint32_t tl) "tl=%d"
+win_helper_retry(uint32_t tl) "tl=%d"
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index 99b4f53..e7691e4 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -18,21 +18,20 @@
    License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "disas/disas.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/cpu_ldst.h"
 
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
+#include "asi.h"
 
 
 #define DEBUG_DISAS
@@ -42,31 +41,29 @@
                          according to jump_pc[T2] */
 
 /* global register indexes */
-static TCGv_ptr cpu_env, cpu_regwptr;
+static TCGv_env cpu_env;
+static TCGv_ptr cpu_regwptr;
 static TCGv cpu_cc_src, cpu_cc_src2, cpu_cc_dst;
 static TCGv_i32 cpu_cc_op;
 static TCGv_i32 cpu_psr;
-static TCGv cpu_fsr, cpu_pc, cpu_npc, cpu_gregs[8];
+static TCGv cpu_fsr, cpu_pc, cpu_npc;
+static TCGv cpu_regs[32];
 static TCGv cpu_y;
 #ifndef CONFIG_USER_ONLY
 static TCGv cpu_tbr;
 #endif
 static TCGv cpu_cond;
 #ifdef TARGET_SPARC64
-static TCGv_i32 cpu_xcc, cpu_asi, cpu_fprs;
+static TCGv_i32 cpu_xcc, cpu_fprs;
 static TCGv cpu_gsr;
 static TCGv cpu_tick_cmpr, cpu_stick_cmpr, cpu_hstick_cmpr;
 static TCGv cpu_hintp, cpu_htba, cpu_hver, cpu_ssr, cpu_ver;
-static TCGv_i32 cpu_softint;
 #else
 static TCGv cpu_wim;
 #endif
 /* Floating point registers */
 static TCGv_i64 cpu_fpr[TARGET_DPREGS];
 
-static target_ulong gen_opc_npc[OPC_BUF_SIZE];
-static target_ulong gen_opc_jump_pc[2];
-
 #include "exec/gen-icount.h"
 
 typedef struct DisasContext {
@@ -85,6 +82,10 @@
     TCGv ttl[5];
     int n_t32;
     int n_ttl;
+#ifdef TARGET_SPARC64
+    int fprs_dirty;
+    int asi;
+#endif
 } DisasContext;
 
 typedef struct {
@@ -140,10 +141,16 @@
     return t;
 }
 
-static inline void gen_update_fprs_dirty(int rd)
+static inline void gen_update_fprs_dirty(DisasContext *dc, int rd)
 {
 #if defined(TARGET_SPARC64)
-    tcg_gen_ori_i32(cpu_fprs, cpu_fprs, (rd < 32) ? 1 : 2);
+    int bit = (rd < 32) ? 1 : 2;
+    /* If we know we've already set this bit within the TB,
+       we can avoid setting it again.  */
+    if (!(dc->fprs_dirty & bit)) {
+        dc->fprs_dirty |= bit;
+        tcg_gen_ori_i32(cpu_fprs, cpu_fprs, bit);
+    }
 #endif
 }
 
@@ -164,7 +171,7 @@
         TCGv_i64 t = tcg_temp_new_i64();
 
         tcg_gen_shri_i64(t, cpu_fpr[src / 2], 32);
-        tcg_gen_trunc_i64_i32(ret, t);
+        tcg_gen_extrl_i64_i32(ret, t);
         tcg_temp_free_i64(t);
 
         return ret;
@@ -185,7 +192,7 @@
     tcg_gen_deposit_i64(cpu_fpr[dst / 2], cpu_fpr[dst / 2], t,
                         (dst & 1 ? 0 : 32), 32);
 #endif
-    gen_update_fprs_dirty(dst);
+    gen_update_fprs_dirty(dc, dst);
 }
 
 static TCGv_i32 gen_dest_fpr_F(DisasContext *dc)
@@ -203,7 +210,7 @@
 {
     dst = DFPREG(dst);
     tcg_gen_mov_i64(cpu_fpr[dst / 2], v);
-    gen_update_fprs_dirty(dst);
+    gen_update_fprs_dirty(dc, dst);
 }
 
 static TCGv_i64 gen_dest_fpr_D(DisasContext *dc, unsigned int dst)
@@ -236,14 +243,14 @@
 }
 
 #ifdef TARGET_SPARC64
-static void gen_move_Q(unsigned int rd, unsigned int rs)
+static void gen_move_Q(DisasContext *dc, unsigned int rd, unsigned int rs)
 {
     rd = QFPREG(rd);
     rs = QFPREG(rs);
 
     tcg_gen_mov_i64(cpu_fpr[rd / 2], cpu_fpr[rs / 2]);
     tcg_gen_mov_i64(cpu_fpr[rd / 2 + 1], cpu_fpr[rs / 2 + 1]);
-    gen_update_fprs_dirty(rd);
+    gen_update_fprs_dirty(dc, rd);
 }
 #endif
 
@@ -279,53 +286,58 @@
 
 static inline TCGv gen_load_gpr(DisasContext *dc, int reg)
 {
-    if (reg == 0 || reg >= 8) {
-        TCGv t = get_temp_tl(dc);
-        if (reg == 0) {
-            tcg_gen_movi_tl(t, 0);
-        } else {
-            tcg_gen_ld_tl(t, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
-        }
-        return t;
+    if (reg > 0) {
+        assert(reg < 32);
+        return cpu_regs[reg];
     } else {
-        return cpu_gregs[reg];
+        TCGv t = get_temp_tl(dc);
+        tcg_gen_movi_tl(t, 0);
+        return t;
     }
 }
 
 static inline void gen_store_gpr(DisasContext *dc, int reg, TCGv v)
 {
     if (reg > 0) {
-        if (reg < 8) {
-            tcg_gen_mov_tl(cpu_gregs[reg], v);
-        } else {
-            tcg_gen_st_tl(v, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
-        }
+        assert(reg < 32);
+        tcg_gen_mov_tl(cpu_regs[reg], v);
     }
 }
 
 static inline TCGv gen_dest_gpr(DisasContext *dc, int reg)
 {
-    if (reg == 0 || reg >= 8) {
-        return get_temp_tl(dc);
+    if (reg > 0) {
+        assert(reg < 32);
+        return cpu_regs[reg];
     } else {
-        return cpu_gregs[reg];
+        return get_temp_tl(dc);
     }
 }
 
+static inline bool use_goto_tb(DisasContext *s, target_ulong pc,
+                               target_ulong npc)
+{
+    if (unlikely(s->singlestep)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (pc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK) &&
+           (npc & TARGET_PAGE_MASK) == (s->tb->pc & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(DisasContext *s, int tb_num,
                                target_ulong pc, target_ulong npc)
 {
-    TranslationBlock *tb;
-
-    tb = s->tb;
-    if ((pc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) &&
-        (npc & TARGET_PAGE_MASK) == (tb->pc & TARGET_PAGE_MASK) &&
-        !s->singlestep)  {
+    if (use_goto_tb(s, pc, npc))  {
         /* jump to same page: we can use a direct jump */
         tcg_gen_goto_tb(tb_num);
         tcg_gen_movi_tl(cpu_pc, pc);
         tcg_gen_movi_tl(cpu_npc, npc);
-        tcg_gen_exit_tb((uintptr_t)tb + tb_num);
+        tcg_gen_exit_tb((uintptr_t)s->tb + tb_num);
     } else {
         /* jump to another page: currently not optimized */
         tcg_gen_movi_tl(cpu_pc, pc);
@@ -363,14 +375,6 @@
     tcg_gen_andi_tl(reg, reg, 0x1);
 }
 
-static inline void gen_op_addi_cc(TCGv dst, TCGv src1, target_long src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_movi_tl(cpu_cc_src2, src2);
-    tcg_gen_addi_tl(cpu_cc_dst, cpu_cc_src, src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
 static inline void gen_op_add_cc(TCGv dst, TCGv src1, TCGv src2)
 {
     tcg_gen_mov_tl(cpu_cc_src, src1);
@@ -387,8 +391,8 @@
 #if TARGET_LONG_BITS == 64
     cc_src1_32 = tcg_temp_new_i32();
     cc_src2_32 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(cc_src1_32, cpu_cc_dst);
-    tcg_gen_trunc_i64_i32(cc_src2_32, cpu_cc_src);
+    tcg_gen_extrl_i64_i32(cc_src1_32, cpu_cc_dst);
+    tcg_gen_extrl_i64_i32(cc_src2_32, cpu_cc_src);
 #else
     cc_src1_32 = cpu_cc_dst;
     cc_src2_32 = cpu_cc_src;
@@ -413,8 +417,8 @@
 #if TARGET_LONG_BITS == 64
     cc_src1_32 = tcg_temp_new_i32();
     cc_src2_32 = tcg_temp_new_i32();
-    tcg_gen_trunc_i64_i32(cc_src1_32, cpu_cc_src);
-    tcg_gen_trunc_i64_i32(cc_src2_32, cpu_cc_src2);
+    tcg_gen_extrl_i64_i32(cc_src1_32, cpu_cc_src);
+    tcg_gen_extrl_i64_i32(cc_src2_32, cpu_cc_src2);
 #else
     cc_src1_32 = cpu_cc_src;
     cc_src2_32 = cpu_cc_src2;
@@ -502,22 +506,6 @@
     }
 }
 
-static inline void gen_op_subi_cc(TCGv dst, TCGv src1, target_long src2, DisasContext *dc)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_movi_tl(cpu_cc_src2, src2);
-    if (src2 == 0) {
-        tcg_gen_mov_tl(cpu_cc_dst, src1);
-        tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
-        dc->cc_op = CC_OP_LOGIC;
-    } else {
-        tcg_gen_subi_tl(cpu_cc_dst, cpu_cc_src, src2);
-        tcg_gen_movi_i32(cpu_cc_op, CC_OP_SUB);
-        dc->cc_op = CC_OP_SUB;
-    }
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
 static inline void gen_op_sub_cc(TCGv dst, TCGv src1, TCGv src2)
 {
     tcg_gen_mov_tl(cpu_cc_src, src1);
@@ -969,9 +957,7 @@
 static inline void gen_branch2(DisasContext *dc, target_ulong pc1,
                                target_ulong pc2, TCGv r_cond)
 {
-    int l1;
-
-    l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
 
     tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
 
@@ -981,19 +967,44 @@
     gen_goto_tb(dc, 1, pc2, pc2 + 4);
 }
 
-static inline void gen_branch_a(DisasContext *dc, target_ulong pc1,
-                                target_ulong pc2, TCGv r_cond)
+static void gen_branch_a(DisasContext *dc, target_ulong pc1)
 {
-    int l1;
+    TCGLabel *l1 = gen_new_label();
+    target_ulong npc = dc->npc;
 
-    l1 = gen_new_label();
+    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_cond, 0, l1);
 
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
-
-    gen_goto_tb(dc, 0, pc2, pc1);
+    gen_goto_tb(dc, 0, npc, pc1);
 
     gen_set_label(l1);
-    gen_goto_tb(dc, 1, pc2 + 4, pc2 + 8);
+    gen_goto_tb(dc, 1, npc + 4, npc + 8);
+
+    dc->is_br = 1;
+}
+
+static void gen_branch_n(DisasContext *dc, target_ulong pc1)
+{
+    target_ulong npc = dc->npc;
+
+    if (likely(npc != DYNAMIC_PC)) {
+        dc->pc = npc;
+        dc->jump_pc[0] = pc1;
+        dc->jump_pc[1] = npc + 4;
+        dc->npc = JUMP_PC;
+    } else {
+        TCGv t, z;
+
+        tcg_gen_mov_tl(cpu_pc, cpu_npc);
+
+        tcg_gen_addi_tl(cpu_npc, cpu_npc, 4);
+        t = tcg_const_tl(pc1);
+        z = tcg_const_tl(0);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_npc, cpu_cond, z, t, cpu_npc);
+        tcg_temp_free(t);
+        tcg_temp_free(z);
+
+        dc->pc = DYNAMIC_PC;
+    }
 }
 
 static inline void gen_generic_branch(DisasContext *dc)
@@ -1043,6 +1054,24 @@
     save_npc(dc);
 }
 
+static void gen_exception(DisasContext *dc, int which)
+{
+    TCGv_i32 t;
+
+    save_state(dc);
+    t = tcg_const_i32(which);
+    gen_helper_raise_exception(cpu_env, t);
+    tcg_temp_free_i32(t);
+    dc->is_br = 1;
+}
+
+static void gen_check_align(TCGv addr, int mask)
+{
+    TCGv_i32 r_mask = tcg_const_i32(mask);
+    gen_helper_check_align(cpu_env, addr, r_mask);
+    tcg_temp_free_i32(r_mask);
+}
+
 static inline void gen_mov_pc_npc(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
@@ -1426,18 +1455,9 @@
         flush_cond(dc);
         gen_cond(cpu_cond, cc, cond, dc);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, cpu_cond);
-            dc->is_br = 1;
+            gen_branch_a(dc, target);
         } else {
-            dc->pc = dc->npc;
-            dc->jump_pc[0] = target;
-            if (unlikely(dc->npc == DYNAMIC_PC)) {
-                dc->jump_pc[1] = DYNAMIC_PC;
-                tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-            } else {
-                dc->jump_pc[1] = dc->npc + 4;
-                dc->npc = JUMP_PC;
-            }
+            gen_branch_n(dc, target);
         }
     }
 }
@@ -1475,18 +1495,9 @@
         flush_cond(dc);
         gen_fcond(cpu_cond, cc, cond);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, cpu_cond);
-            dc->is_br = 1;
+            gen_branch_a(dc, target);
         } else {
-            dc->pc = dc->npc;
-            dc->jump_pc[0] = target;
-            if (unlikely(dc->npc == DYNAMIC_PC)) {
-                dc->jump_pc[1] = DYNAMIC_PC;
-                tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-            } else {
-                dc->jump_pc[1] = dc->npc + 4;
-                dc->npc = JUMP_PC;
-            }
+            gen_branch_n(dc, target);
         }
     }
 }
@@ -1504,18 +1515,9 @@
     flush_cond(dc);
     gen_cond_reg(cpu_cond, cond, r_reg);
     if (a) {
-        gen_branch_a(dc, target, dc->npc, cpu_cond);
-        dc->is_br = 1;
+        gen_branch_a(dc, target);
     } else {
-        dc->pc = dc->npc;
-        dc->jump_pc[0] = target;
-        if (unlikely(dc->npc == DYNAMIC_PC)) {
-            dc->jump_pc[1] = DYNAMIC_PC;
-            tcg_gen_addi_tl(cpu_pc, cpu_npc, 4);
-        } else {
-            dc->jump_pc[1] = dc->npc + 4;
-            dc->npc = JUMP_PC;
-        }
+        gen_branch_n(dc, target);
     }
 }
 
@@ -1523,16 +1525,16 @@
 {
     switch (fccno) {
     case 0:
-        gen_helper_fcmps(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmps(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 1:
-        gen_helper_fcmps_fcc1(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmps_fcc1(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 2:
-        gen_helper_fcmps_fcc2(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmps_fcc2(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 3:
-        gen_helper_fcmps_fcc3(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmps_fcc3(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     }
 }
@@ -1541,16 +1543,16 @@
 {
     switch (fccno) {
     case 0:
-        gen_helper_fcmpd(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpd(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 1:
-        gen_helper_fcmpd_fcc1(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpd_fcc1(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 2:
-        gen_helper_fcmpd_fcc2(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpd_fcc2(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 3:
-        gen_helper_fcmpd_fcc3(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpd_fcc3(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     }
 }
@@ -1559,16 +1561,16 @@
 {
     switch (fccno) {
     case 0:
-        gen_helper_fcmpq(cpu_env);
+        gen_helper_fcmpq(cpu_fsr, cpu_env);
         break;
     case 1:
-        gen_helper_fcmpq_fcc1(cpu_env);
+        gen_helper_fcmpq_fcc1(cpu_fsr, cpu_env);
         break;
     case 2:
-        gen_helper_fcmpq_fcc2(cpu_env);
+        gen_helper_fcmpq_fcc2(cpu_fsr, cpu_env);
         break;
     case 3:
-        gen_helper_fcmpq_fcc3(cpu_env);
+        gen_helper_fcmpq_fcc3(cpu_fsr, cpu_env);
         break;
     }
 }
@@ -1577,16 +1579,16 @@
 {
     switch (fccno) {
     case 0:
-        gen_helper_fcmpes(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpes(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 1:
-        gen_helper_fcmpes_fcc1(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpes_fcc1(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 2:
-        gen_helper_fcmpes_fcc2(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpes_fcc2(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 3:
-        gen_helper_fcmpes_fcc3(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmpes_fcc3(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     }
 }
@@ -1595,16 +1597,16 @@
 {
     switch (fccno) {
     case 0:
-        gen_helper_fcmped(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmped(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 1:
-        gen_helper_fcmped_fcc1(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmped_fcc1(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 2:
-        gen_helper_fcmped_fcc2(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmped_fcc2(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     case 3:
-        gen_helper_fcmped_fcc3(cpu_env, r_rs1, r_rs2);
+        gen_helper_fcmped_fcc3(cpu_fsr, cpu_env, r_rs1, r_rs2);
         break;
     }
 }
@@ -1613,16 +1615,16 @@
 {
     switch (fccno) {
     case 0:
-        gen_helper_fcmpeq(cpu_env);
+        gen_helper_fcmpeq(cpu_fsr, cpu_env);
         break;
     case 1:
-        gen_helper_fcmpeq_fcc1(cpu_env);
+        gen_helper_fcmpeq_fcc1(cpu_fsr, cpu_env);
         break;
     case 2:
-        gen_helper_fcmpeq_fcc2(cpu_env);
+        gen_helper_fcmpeq_fcc2(cpu_fsr, cpu_env);
         break;
     case 3:
-        gen_helper_fcmpeq_fcc3(cpu_env);
+        gen_helper_fcmpeq_fcc3(cpu_fsr, cpu_env);
         break;
     }
 }
@@ -1631,57 +1633,47 @@
 
 static inline void gen_op_fcmps(int fccno, TCGv r_rs1, TCGv r_rs2)
 {
-    gen_helper_fcmps(cpu_env, r_rs1, r_rs2);
+    gen_helper_fcmps(cpu_fsr, cpu_env, r_rs1, r_rs2);
 }
 
 static inline void gen_op_fcmpd(int fccno, TCGv_i64 r_rs1, TCGv_i64 r_rs2)
 {
-    gen_helper_fcmpd(cpu_env, r_rs1, r_rs2);
+    gen_helper_fcmpd(cpu_fsr, cpu_env, r_rs1, r_rs2);
 }
 
 static inline void gen_op_fcmpq(int fccno)
 {
-    gen_helper_fcmpq(cpu_env);
+    gen_helper_fcmpq(cpu_fsr, cpu_env);
 }
 
 static inline void gen_op_fcmpes(int fccno, TCGv r_rs1, TCGv r_rs2)
 {
-    gen_helper_fcmpes(cpu_env, r_rs1, r_rs2);
+    gen_helper_fcmpes(cpu_fsr, cpu_env, r_rs1, r_rs2);
 }
 
 static inline void gen_op_fcmped(int fccno, TCGv_i64 r_rs1, TCGv_i64 r_rs2)
 {
-    gen_helper_fcmped(cpu_env, r_rs1, r_rs2);
+    gen_helper_fcmped(cpu_fsr, cpu_env, r_rs1, r_rs2);
 }
 
 static inline void gen_op_fcmpeq(int fccno)
 {
-    gen_helper_fcmpeq(cpu_env);
+    gen_helper_fcmpeq(cpu_fsr, cpu_env);
 }
 #endif
 
-static inline void gen_op_fpexception_im(int fsr_flags)
+static void gen_op_fpexception_im(DisasContext *dc, int fsr_flags)
 {
-    TCGv_i32 r_const;
-
     tcg_gen_andi_tl(cpu_fsr, cpu_fsr, FSR_FTT_NMASK);
     tcg_gen_ori_tl(cpu_fsr, cpu_fsr, fsr_flags);
-    r_const = tcg_const_i32(TT_FP_EXCP);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
+    gen_exception(dc, TT_FP_EXCP);
 }
 
 static int gen_trap_ifnofpu(DisasContext *dc)
 {
 #if !defined(CONFIG_USER_ONLY)
     if (!dc->fpu_enabled) {
-        TCGv_i32 r_const;
-
-        save_state(dc);
-        r_const = tcg_const_i32(TT_NFPU_INSN);
-        gen_helper_raise_exception(cpu_env, r_const);
-        tcg_temp_free_i32(r_const);
-        dc->is_br = 1;
+        gen_exception(dc, TT_NFPU_INSN);
         return 1;
     }
 #endif
@@ -1702,6 +1694,7 @@
     dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_F(dc, rd, dst);
 }
@@ -1729,6 +1722,7 @@
     dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src1, src2);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_F(dc, rd, dst);
 }
@@ -1758,6 +1752,7 @@
     dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_D(dc, rd, dst);
 }
@@ -1787,6 +1782,7 @@
     dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src1, src2);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_D(dc, rd, dst);
 }
@@ -1842,9 +1838,10 @@
     gen_op_load_fpr_QT1(QFPREG(rs));
 
     gen(cpu_env);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_op_store_QT0_fpr(QFPREG(rd));
-    gen_update_fprs_dirty(QFPREG(rd));
+    gen_update_fprs_dirty(dc, QFPREG(rd));
 }
 
 #ifdef TARGET_SPARC64
@@ -1856,7 +1853,7 @@
     gen(cpu_env);
 
     gen_op_store_QT0_fpr(QFPREG(rd));
-    gen_update_fprs_dirty(QFPREG(rd));
+    gen_update_fprs_dirty(dc, QFPREG(rd));
 }
 #endif
 
@@ -1867,9 +1864,10 @@
     gen_op_load_fpr_QT1(QFPREG(rs2));
 
     gen(cpu_env);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_op_store_QT0_fpr(QFPREG(rd));
-    gen_update_fprs_dirty(QFPREG(rd));
+    gen_update_fprs_dirty(dc, QFPREG(rd));
 }
 
 static inline void gen_fop_DFF(DisasContext *dc, int rd, int rs1, int rs2,
@@ -1883,6 +1881,7 @@
     dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src1, src2);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_D(dc, rd, dst);
 }
@@ -1896,9 +1895,10 @@
     src2 = gen_load_fpr_D(dc, rs2);
 
     gen(cpu_env, src1, src2);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_op_store_QT0_fpr(QFPREG(rd));
-    gen_update_fprs_dirty(QFPREG(rd));
+    gen_update_fprs_dirty(dc, QFPREG(rd));
 }
 
 #ifdef TARGET_SPARC64
@@ -1912,6 +1912,7 @@
     dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_D(dc, rd, dst);
 }
@@ -1941,6 +1942,7 @@
     dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_F(dc, rd, dst);
 }
@@ -1954,6 +1956,7 @@
     dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_F(dc, rd, dst);
 }
@@ -1967,6 +1970,7 @@
     dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env);
+    gen_helper_check_ieee_exceptions(cpu_fsr, cpu_env);
 
     gen_store_fpr_D(dc, rd, dst);
 }
@@ -1981,7 +1985,7 @@
     gen(cpu_env, src);
 
     gen_op_store_QT0_fpr(QFPREG(rd));
-    gen_update_fprs_dirty(QFPREG(rd));
+    gen_update_fprs_dirty(dc, QFPREG(rd));
 }
 
 static inline void gen_ne_fop_QD(DisasContext *dc, int rd, int rs,
@@ -1994,265 +1998,737 @@
     gen(cpu_env, src);
 
     gen_op_store_QT0_fpr(QFPREG(rd));
-    gen_update_fprs_dirty(QFPREG(rd));
+    gen_update_fprs_dirty(dc, QFPREG(rd));
 }
 
 /* asi moves */
-#ifdef TARGET_SPARC64
-static inline TCGv_i32 gen_get_asi(int insn, TCGv r_addr)
-{
+#if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
+typedef enum {
+    GET_ASI_HELPER,
+    GET_ASI_EXCP,
+    GET_ASI_DIRECT,
+    GET_ASI_DTWINX,
+    GET_ASI_BLOCK,
+    GET_ASI_SHORT,
+} ASIType;
+
+typedef struct {
+    ASIType type;
     int asi;
+    int mem_idx;
+    TCGMemOp memop;
+} DisasASI;
+
+static DisasASI get_asi(DisasContext *dc, int insn, TCGMemOp memop)
+{
+    int asi = GET_FIELD(insn, 19, 26);
+    ASIType type = GET_ASI_HELPER;
+    int mem_idx = dc->mem_idx;
+
+#ifndef TARGET_SPARC64
+    /* Before v9, all asis are immediate and privileged.  */
+    if (IS_IMM) {
+        gen_exception(dc, TT_ILL_INSN);
+        type = GET_ASI_EXCP;
+    } else if (supervisor(dc)
+               /* Note that LEON accepts ASI_USERDATA in user mode, for
+                  use with CASA.  Also note that previous versions of
+                  QEMU allowed (and old versions of gcc emitted) ASI_P
+                  for LEON, which is incorrect.  */
+               || (asi == ASI_USERDATA
+                   && (dc->def->features & CPU_FEATURE_CASA))) {
+        switch (asi) {
+        case ASI_USERDATA:   /* User data access */
+            mem_idx = MMU_USER_IDX;
+            type = GET_ASI_DIRECT;
+            break;
+        case ASI_KERNELDATA: /* Supervisor data access */
+            mem_idx = MMU_KERNEL_IDX;
+            type = GET_ASI_DIRECT;
+            break;
+        }
+    } else {
+        gen_exception(dc, TT_PRIV_INSN);
+        type = GET_ASI_EXCP;
+    }
+#else
+    if (IS_IMM) {
+        asi = dc->asi;
+    }
+    /* With v9, all asis below 0x80 are privileged.  */
+    /* ??? We ought to check cpu_has_hypervisor, but we didn't copy
+       down that bit into DisasContext.  For the moment that's ok,
+       since the direct implementations below doesn't have any ASIs
+       in the restricted [0x30, 0x7f] range, and the check will be
+       done properly in the helper.  */
+    if (!supervisor(dc) && asi < 0x80) {
+        gen_exception(dc, TT_PRIV_ACT);
+        type = GET_ASI_EXCP;
+    } else {
+        switch (asi) {
+        case ASI_N:  /* Nucleus */
+        case ASI_NL: /* Nucleus LE */
+        case ASI_TWINX_N:
+        case ASI_TWINX_NL:
+            mem_idx = MMU_NUCLEUS_IDX;
+            break;
+        case ASI_AIUP:  /* As if user primary */
+        case ASI_AIUPL: /* As if user primary LE */
+        case ASI_TWINX_AIUP:
+        case ASI_TWINX_AIUP_L:
+        case ASI_BLK_AIUP_4V:
+        case ASI_BLK_AIUP_L_4V:
+        case ASI_BLK_AIUP:
+        case ASI_BLK_AIUPL:
+            mem_idx = MMU_USER_IDX;
+            break;
+        case ASI_AIUS:  /* As if user secondary */
+        case ASI_AIUSL: /* As if user secondary LE */
+        case ASI_TWINX_AIUS:
+        case ASI_TWINX_AIUS_L:
+        case ASI_BLK_AIUS_4V:
+        case ASI_BLK_AIUS_L_4V:
+        case ASI_BLK_AIUS:
+        case ASI_BLK_AIUSL:
+            mem_idx = MMU_USER_SECONDARY_IDX;
+            break;
+        case ASI_S:  /* Secondary */
+        case ASI_SL: /* Secondary LE */
+        case ASI_TWINX_S:
+        case ASI_TWINX_SL:
+        case ASI_BLK_COMMIT_S:
+        case ASI_BLK_S:
+        case ASI_BLK_SL:
+        case ASI_FL8_S:
+        case ASI_FL8_SL:
+        case ASI_FL16_S:
+        case ASI_FL16_SL:
+            if (mem_idx == MMU_USER_IDX) {
+                mem_idx = MMU_USER_SECONDARY_IDX;
+            } else if (mem_idx == MMU_KERNEL_IDX) {
+                mem_idx = MMU_KERNEL_SECONDARY_IDX;
+            }
+            break;
+        case ASI_P:  /* Primary */
+        case ASI_PL: /* Primary LE */
+        case ASI_TWINX_P:
+        case ASI_TWINX_PL:
+        case ASI_BLK_COMMIT_P:
+        case ASI_BLK_P:
+        case ASI_BLK_PL:
+        case ASI_FL8_P:
+        case ASI_FL8_PL:
+        case ASI_FL16_P:
+        case ASI_FL16_PL:
+            break;
+        }
+        switch (asi) {
+        case ASI_N:
+        case ASI_NL:
+        case ASI_AIUP:
+        case ASI_AIUPL:
+        case ASI_AIUS:
+        case ASI_AIUSL:
+        case ASI_S:
+        case ASI_SL:
+        case ASI_P:
+        case ASI_PL:
+            type = GET_ASI_DIRECT;
+            break;
+        case ASI_TWINX_N:
+        case ASI_TWINX_NL:
+        case ASI_TWINX_AIUP:
+        case ASI_TWINX_AIUP_L:
+        case ASI_TWINX_AIUS:
+        case ASI_TWINX_AIUS_L:
+        case ASI_TWINX_P:
+        case ASI_TWINX_PL:
+        case ASI_TWINX_S:
+        case ASI_TWINX_SL:
+            type = GET_ASI_DTWINX;
+            break;
+        case ASI_BLK_COMMIT_P:
+        case ASI_BLK_COMMIT_S:
+        case ASI_BLK_AIUP_4V:
+        case ASI_BLK_AIUP_L_4V:
+        case ASI_BLK_AIUP:
+        case ASI_BLK_AIUPL:
+        case ASI_BLK_AIUS_4V:
+        case ASI_BLK_AIUS_L_4V:
+        case ASI_BLK_AIUS:
+        case ASI_BLK_AIUSL:
+        case ASI_BLK_S:
+        case ASI_BLK_SL:
+        case ASI_BLK_P:
+        case ASI_BLK_PL:
+            type = GET_ASI_BLOCK;
+            break;
+        case ASI_FL8_S:
+        case ASI_FL8_SL:
+        case ASI_FL8_P:
+        case ASI_FL8_PL:
+            memop = MO_UB;
+            type = GET_ASI_SHORT;
+            break;
+        case ASI_FL16_S:
+        case ASI_FL16_SL:
+        case ASI_FL16_P:
+        case ASI_FL16_PL:
+            memop = MO_TEUW;
+            type = GET_ASI_SHORT;
+            break;
+        }
+        /* The little-endian asis all have bit 3 set.  */
+        if (asi & 8) {
+            memop ^= MO_BSWAP;
+        }
+    }
+#endif
+
+    return (DisasASI){ type, asi, mem_idx, memop };
+}
+
+static void gen_ld_asi(DisasContext *dc, TCGv dst, TCGv addr,
+                       int insn, TCGMemOp memop)
+{
+    DisasASI da = get_asi(dc, insn, memop);
+
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+    case GET_ASI_DTWINX: /* Reserved for ldda.  */
+        gen_exception(dc, TT_ILL_INSN);
+        break;
+    case GET_ASI_DIRECT:
+        gen_address_mask(dc, addr);
+        tcg_gen_qemu_ld_tl(dst, addr, da.mem_idx, da.memop);
+        break;
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(memop);
+
+            save_state(dc);
+#ifdef TARGET_SPARC64
+            gen_helper_ld_asi(dst, cpu_env, addr, r_asi, r_mop);
+#else
+            {
+                TCGv_i64 t64 = tcg_temp_new_i64();
+                gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_mop);
+                tcg_gen_trunc_i64_tl(dst, t64);
+                tcg_temp_free_i64(t64);
+            }
+#endif
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+        }
+        break;
+    }
+}
+
+static void gen_st_asi(DisasContext *dc, TCGv src, TCGv addr,
+                       int insn, TCGMemOp memop)
+{
+    DisasASI da = get_asi(dc, insn, memop);
+
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+    case GET_ASI_DTWINX: /* Reserved for stda.  */
+        gen_exception(dc, TT_ILL_INSN);
+        break;
+    case GET_ASI_DIRECT:
+        gen_address_mask(dc, addr);
+        tcg_gen_qemu_st_tl(src, addr, da.mem_idx, da.memop);
+        break;
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(memop & MO_SIZE);
+
+            save_state(dc);
+#ifdef TARGET_SPARC64
+            gen_helper_st_asi(cpu_env, addr, src, r_asi, r_mop);
+#else
+            {
+                TCGv_i64 t64 = tcg_temp_new_i64();
+                tcg_gen_extu_tl_i64(t64, src);
+                gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_mop);
+                tcg_temp_free_i64(t64);
+            }
+#endif
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+
+            /* A write to a TLB register may alter page maps.  End the TB. */
+            dc->npc = DYNAMIC_PC;
+        }
+        break;
+    }
+}
+
+static void gen_swap_asi(DisasContext *dc, TCGv dst, TCGv src,
+                         TCGv addr, int insn)
+{
+    DisasASI da = get_asi(dc, insn, MO_TEUL);
+
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(MO_UL);
+            TCGv_i64 s64, t64;
+
+            save_state(dc);
+            t64 = tcg_temp_new_i64();
+            gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_mop);
+
+            s64 = tcg_temp_new_i64();
+            tcg_gen_extu_tl_i64(s64, src);
+            gen_helper_st_asi(cpu_env, addr, s64, r_asi, r_mop);
+            tcg_temp_free_i64(s64);
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+
+            tcg_gen_trunc_i64_tl(dst, t64);
+            tcg_temp_free_i64(t64);
+        }
+        break;
+    }
+}
+
+static void gen_cas_asi(DisasContext *dc, TCGv addr, TCGv val2,
+                        int insn, int rd)
+{
+    DisasASI da = get_asi(dc, insn, MO_TEUL);
+    TCGv val1, dst;
     TCGv_i32 r_asi;
 
-    if (IS_IMM) {
-        r_asi = tcg_temp_new_i32();
-        tcg_gen_mov_i32(r_asi, cpu_asi);
-    } else {
-        asi = GET_FIELD(insn, 19, 26);
-        r_asi = tcg_const_i32(asi);
+    if (da.type == GET_ASI_EXCP) {
+        return;
     }
-    return r_asi;
-}
 
-static inline void gen_ld_asi(TCGv dst, TCGv addr, int insn, int size,
-                              int sign)
-{
-    TCGv_i32 r_asi, r_size, r_sign;
-
-    r_asi = gen_get_asi(insn, addr);
-    r_size = tcg_const_i32(size);
-    r_sign = tcg_const_i32(sign);
-    gen_helper_ld_asi(dst, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free_i32(r_sign);
-    tcg_temp_free_i32(r_size);
+    save_state(dc);
+    val1 = gen_load_gpr(dc, rd);
+    dst = gen_dest_gpr(dc, rd);
+    r_asi = tcg_const_i32(da.asi);
+    gen_helper_cas_asi(dst, cpu_env, addr, val1, val2, r_asi);
     tcg_temp_free_i32(r_asi);
+    gen_store_gpr(dc, rd, dst);
 }
 
-static inline void gen_st_asi(TCGv src, TCGv addr, int insn, int size)
+static void gen_ldstub_asi(DisasContext *dc, TCGv dst, TCGv addr, int insn)
 {
-    TCGv_i32 r_asi, r_size;
+    DisasASI da = get_asi(dc, insn, MO_UB);
 
-    r_asi = gen_get_asi(insn, addr);
-    r_size = tcg_const_i32(size);
-    gen_helper_st_asi(cpu_env, addr, src, r_asi, r_size);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(MO_UB);
+            TCGv_i64 s64, t64;
+
+            save_state(dc);
+            t64 = tcg_temp_new_i64();
+            gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_mop);
+
+            s64 = tcg_const_i64(0xff);
+            gen_helper_st_asi(cpu_env, addr, s64, r_asi, r_mop);
+            tcg_temp_free_i64(s64);
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+
+            tcg_gen_trunc_i64_tl(dst, t64);
+            tcg_temp_free_i64(t64);
+        }
+        break;
+    }
+}
+#endif
+
+#ifdef TARGET_SPARC64
+static void gen_ldf_asi(DisasContext *dc, TCGv addr,
+                        int insn, int size, int rd)
+{
+    DisasASI da = get_asi(dc, insn, (size == 4 ? MO_TEUL : MO_TEQ));
+    TCGv_i32 d32;
+
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+
+    case GET_ASI_DIRECT:
+        gen_address_mask(dc, addr);
+        switch (size) {
+        case 4:
+            d32 = gen_dest_fpr_F(dc);
+            tcg_gen_qemu_ld_i32(d32, addr, da.mem_idx, da.memop);
+            gen_store_fpr_F(dc, rd, d32);
+            break;
+        case 8:
+            tcg_gen_qemu_ld_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop);
+            break;
+        case 16:
+            tcg_gen_qemu_ld_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop);
+            tcg_gen_addi_tl(addr, addr, 8);
+            tcg_gen_qemu_ld_i64(cpu_fpr[rd/2+1], addr, da.mem_idx, da.memop);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+
+    case GET_ASI_BLOCK:
+        /* Valid for lddfa on aligned registers only.  */
+        if (size == 8 && (rd & 7) == 0) {
+            TCGv eight;
+            int i;
+
+            gen_check_align(addr, 0x3f);
+            gen_address_mask(dc, addr);
+
+            eight = tcg_const_tl(8);
+            for (i = 0; ; ++i) {
+                tcg_gen_qemu_ld_i64(cpu_fpr[rd / 2 + i], addr,
+                                    da.mem_idx, da.memop);
+                if (i == 7) {
+                    break;
+                }
+                tcg_gen_add_tl(addr, addr, eight);
+            }
+            tcg_temp_free(eight);
+        } else {
+            gen_exception(dc, TT_ILL_INSN);
+        }
+        break;
+
+    case GET_ASI_SHORT:
+        /* Valid for lddfa only.  */
+        if (size == 8) {
+            gen_address_mask(dc, addr);
+            tcg_gen_qemu_ld_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop);
+        } else {
+            gen_exception(dc, TT_ILL_INSN);
+        }
+        break;
+
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(da.memop);
+
+            save_state(dc);
+            /* According to the table in the UA2011 manual, the only
+               other asis that are valid for ldfa/lddfa/ldqfa are
+               the NO_FAULT asis.  We still need a helper for these,
+               but we can just use the integer asi helper for them.  */
+            switch (size) {
+            case 4:
+                {
+                    TCGv d64 = tcg_temp_new_i64();
+                    gen_helper_ld_asi(d64, cpu_env, addr, r_asi, r_mop);
+                    d32 = gen_dest_fpr_F(dc);
+                    tcg_gen_extrl_i64_i32(d32, d64);
+                    tcg_temp_free_i64(d64);
+                    gen_store_fpr_F(dc, rd, d32);
+                }
+                break;
+            case 8:
+                gen_helper_ld_asi(cpu_fpr[rd / 2], cpu_env, addr, r_asi, r_mop);
+                break;
+            case 16:
+                gen_helper_ld_asi(cpu_fpr[rd / 2], cpu_env, addr, r_asi, r_mop);
+                tcg_gen_addi_tl(addr, addr, 8);
+                gen_helper_ld_asi(cpu_fpr[rd/2+1], cpu_env, addr, r_asi, r_mop);
+                break;
+            default:
+                g_assert_not_reached();
+            }
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+        }
+        break;
+    }
 }
 
-static inline void gen_ldf_asi(TCGv addr, int insn, int size, int rd)
+static void gen_stf_asi(DisasContext *dc, TCGv addr,
+                        int insn, int size, int rd)
 {
-    TCGv_i32 r_asi, r_size, r_rd;
+    DisasASI da = get_asi(dc, insn, (size == 4 ? MO_TEUL : MO_TEQ));
+    TCGv_i32 d32;
 
-    r_asi = gen_get_asi(insn, addr);
-    r_size = tcg_const_i32(size);
-    r_rd = tcg_const_i32(rd);
-    gen_helper_ldf_asi(cpu_env, addr, r_asi, r_size, r_rd);
-    tcg_temp_free_i32(r_rd);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+
+    case GET_ASI_DIRECT:
+        gen_address_mask(dc, addr);
+        switch (size) {
+        case 4:
+            d32 = gen_load_fpr_F(dc, rd);
+            tcg_gen_qemu_st_i32(d32, addr, da.mem_idx, da.memop);
+            break;
+        case 8:
+            tcg_gen_qemu_st_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop);
+            break;
+        case 16:
+            tcg_gen_qemu_st_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop);
+            tcg_gen_addi_tl(addr, addr, 8);
+            tcg_gen_qemu_st_i64(cpu_fpr[rd/2+1], addr, da.mem_idx, da.memop);
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        break;
+
+    case GET_ASI_BLOCK:
+        /* Valid for stdfa on aligned registers only.  */
+        if (size == 8 && (rd & 7) == 0) {
+            TCGv eight;
+            int i;
+
+            gen_check_align(addr, 0x3f);
+            gen_address_mask(dc, addr);
+
+            eight = tcg_const_tl(8);
+            for (i = 0; ; ++i) {
+                tcg_gen_qemu_st_i64(cpu_fpr[rd / 2 + i], addr,
+                                    da.mem_idx, da.memop);
+                if (i == 7) {
+                    break;
+                }
+                tcg_gen_add_tl(addr, addr, eight);
+            }
+            tcg_temp_free(eight);
+        } else {
+            gen_exception(dc, TT_ILL_INSN);
+        }
+        break;
+
+    case GET_ASI_SHORT:
+        /* Valid for stdfa only.  */
+        if (size == 8) {
+            gen_address_mask(dc, addr);
+            tcg_gen_qemu_st_i64(cpu_fpr[rd / 2], addr, da.mem_idx, da.memop);
+        } else {
+            gen_exception(dc, TT_ILL_INSN);
+        }
+        break;
+
+    default:
+        /* According to the table in the UA2011 manual, the only
+           other asis that are valid for ldfa/lddfa/ldqfa are
+           the PST* asis, which aren't currently handled.  */
+        gen_exception(dc, TT_ILL_INSN);
+        break;
+    }
 }
 
-static inline void gen_stf_asi(TCGv addr, int insn, int size, int rd)
+static void gen_ldda_asi(DisasContext *dc, TCGv addr, int insn, int rd)
 {
-    TCGv_i32 r_asi, r_size, r_rd;
+    DisasASI da = get_asi(dc, insn, MO_TEQ);
+    TCGv_i64 hi = gen_dest_gpr(dc, rd);
+    TCGv_i64 lo = gen_dest_gpr(dc, rd + 1);
 
-    r_asi = gen_get_asi(insn, addr);
-    r_size = tcg_const_i32(size);
-    r_rd = tcg_const_i32(rd);
-    gen_helper_stf_asi(cpu_env, addr, r_asi, r_size, r_rd);
-    tcg_temp_free_i32(r_rd);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        return;
+
+    case GET_ASI_DTWINX:
+        gen_check_align(addr, 15);
+        gen_address_mask(dc, addr);
+        tcg_gen_qemu_ld_i64(hi, addr, da.mem_idx, da.memop);
+        tcg_gen_addi_tl(addr, addr, 8);
+        tcg_gen_qemu_ld_i64(lo, addr, da.mem_idx, da.memop);
+        break;
+
+    case GET_ASI_DIRECT:
+        {
+            TCGv_i64 tmp = tcg_temp_new_i64();
+
+            gen_address_mask(dc, addr);
+            tcg_gen_qemu_ld_i64(tmp, addr, da.mem_idx, da.memop);
+
+            /* Note that LE ldda acts as if each 32-bit register
+               result is byte swapped.  Having just performed one
+               64-bit bswap, we need now to swap the writebacks.  */
+            if ((da.memop & MO_BSWAP) == MO_TE) {
+                tcg_gen_extr32_i64(lo, hi, tmp);
+            } else {
+                tcg_gen_extr32_i64(hi, lo, tmp);
+            }
+            tcg_temp_free_i64(tmp);
+        }
+        break;
+
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+
+            save_state(dc);
+            gen_helper_ldda_asi(cpu_env, addr, r_asi);
+            tcg_temp_free_i32(r_asi);
+
+            tcg_gen_ld_i64(hi, cpu_env, offsetof(CPUSPARCState, qt0.high));
+            tcg_gen_ld_i64(lo, cpu_env, offsetof(CPUSPARCState, qt0.low));
+        }
+        break;
+    }
+
+    gen_store_gpr(dc, rd, hi);
+    gen_store_gpr(dc, rd + 1, lo);
 }
 
-static inline void gen_swap_asi(TCGv dst, TCGv src, TCGv addr, int insn)
+static void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                         int insn, int rd)
 {
-    TCGv_i32 r_asi, r_size, r_sign;
-    TCGv_i64 t64 = tcg_temp_new_i64();
-
-    r_asi = gen_get_asi(insn, addr);
-    r_size = tcg_const_i32(4);
-    r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free_i32(r_sign);
-    gen_helper_st_asi(cpu_env, addr, src, r_asi, r_size);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-    tcg_gen_trunc_i64_tl(dst, t64);
-    tcg_temp_free_i64(t64);
-}
-
-static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
-                                int insn, int rd)
-{
-    TCGv_i32 r_asi, r_rd;
-
-    r_asi = gen_get_asi(insn, addr);
-    r_rd = tcg_const_i32(rd);
-    gen_helper_ldda_asi(cpu_env, addr, r_asi, r_rd);
-    tcg_temp_free_i32(r_rd);
-    tcg_temp_free_i32(r_asi);
-}
-
-static inline void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
-                                int insn, int rd)
-{
-    TCGv_i32 r_asi, r_size;
+    DisasASI da = get_asi(dc, insn, MO_TEQ);
     TCGv lo = gen_load_gpr(dc, rd + 1);
-    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    tcg_gen_concat_tl_i64(t64, lo, hi);
-    r_asi = gen_get_asi(insn, addr);
-    r_size = tcg_const_i32(8);
-    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-    tcg_temp_free_i64(t64);
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+
+    case GET_ASI_DTWINX:
+        gen_check_align(addr, 15);
+        gen_address_mask(dc, addr);
+        tcg_gen_qemu_st_i64(hi, addr, da.mem_idx, da.memop);
+        tcg_gen_addi_tl(addr, addr, 8);
+        tcg_gen_qemu_st_i64(lo, addr, da.mem_idx, da.memop);
+        break;
+
+    case GET_ASI_DIRECT:
+        {
+            TCGv_i64 t64 = tcg_temp_new_i64();
+
+            /* Note that LE stda acts as if each 32-bit register result is
+               byte swapped.  We will perform one 64-bit LE store, so now
+               we must swap the order of the construction.  */
+            if ((da.memop & MO_BSWAP) == MO_TE) {
+                tcg_gen_concat32_i64(t64, lo, hi);
+            } else {
+                tcg_gen_concat32_i64(t64, hi, lo);
+            }
+            gen_address_mask(dc, addr);
+            tcg_gen_qemu_st_i64(t64, addr, da.mem_idx, da.memop);
+            tcg_temp_free_i64(t64);
+        }
+        break;
+
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(MO_Q);
+            TCGv_i64 t64;
+
+            save_state(dc);
+
+            t64 = tcg_temp_new_i64();
+            tcg_gen_concat_tl_i64(t64, lo, hi);
+            gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_mop);
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+            tcg_temp_free_i64(t64);
+        }
+        break;
+    }
 }
 
-static inline void gen_casx_asi(DisasContext *dc, TCGv addr,
-                                TCGv val2, int insn, int rd)
+static void gen_casx_asi(DisasContext *dc, TCGv addr, TCGv val2,
+                         int insn, int rd)
 {
+    DisasASI da = get_asi(dc, insn, MO_TEQ);
     TCGv val1 = gen_load_gpr(dc, rd);
     TCGv dst = gen_dest_gpr(dc, rd);
-    TCGv_i32 r_asi = gen_get_asi(insn, addr);
+    TCGv_i32 r_asi;
 
+    if (da.type == GET_ASI_EXCP) {
+        return;
+    }
+
+    save_state(dc);
+    r_asi = tcg_const_i32(da.asi);
     gen_helper_casx_asi(dst, cpu_env, addr, val1, val2, r_asi);
     tcg_temp_free_i32(r_asi);
     gen_store_gpr(dc, rd, dst);
 }
 
 #elif !defined(CONFIG_USER_ONLY)
-
-static inline void gen_ld_asi(TCGv dst, TCGv addr, int insn, int size,
-                              int sign)
+static void gen_ldda_asi(DisasContext *dc, TCGv addr, int insn, int rd)
 {
-    TCGv_i32 r_asi, r_size, r_sign;
+    /* ??? Work around an apparent bug in Ubuntu gcc 4.8.2-10ubuntu2+12,
+       whereby "rd + 1" elicits "error: array subscript is above array".
+       Since we have already asserted that rd is even, the semantics
+       are unchanged.  */
+    TCGv lo = gen_dest_gpr(dc, rd | 1);
+    TCGv hi = gen_dest_gpr(dc, rd);
     TCGv_i64 t64 = tcg_temp_new_i64();
+    DisasASI da = get_asi(dc, insn, MO_TEQ);
 
-    r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-    r_size = tcg_const_i32(size);
-    r_sign = tcg_const_i32(sign);
-    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free_i32(r_sign);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-    tcg_gen_trunc_i64_tl(dst, t64);
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        tcg_temp_free_i64(t64);
+        return;
+    case GET_ASI_DIRECT:
+        gen_address_mask(dc, addr);
+        tcg_gen_qemu_ld_i64(t64, addr, da.mem_idx, da.memop);
+        break;
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(MO_Q);
+
+            save_state(dc);
+            gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_mop);
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+        }
+        break;
+    }
+
+    tcg_gen_extr_i64_i32(lo, hi, t64);
     tcg_temp_free_i64(t64);
-}
-
-static inline void gen_st_asi(TCGv src, TCGv addr, int insn, int size)
-{
-    TCGv_i32 r_asi, r_size;
-    TCGv_i64 t64 = tcg_temp_new_i64();
-
-    tcg_gen_extu_tl_i64(t64, src);
-    r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-    r_size = tcg_const_i32(size);
-    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-    tcg_temp_free_i64(t64);
-}
-
-static inline void gen_swap_asi(TCGv dst, TCGv src, TCGv addr, int insn)
-{
-    TCGv_i32 r_asi, r_size, r_sign;
-    TCGv_i64 r_val, t64;
-
-    r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-    r_size = tcg_const_i32(4);
-    r_sign = tcg_const_i32(0);
-    t64 = tcg_temp_new_i64();
-    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free(r_sign);
-    r_val = tcg_temp_new_i64();
-    tcg_gen_extu_tl_i64(r_val, src);
-    gen_helper_st_asi(cpu_env, addr, r_val, r_asi, r_size);
-    tcg_temp_free_i64(r_val);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-    tcg_gen_trunc_i64_tl(dst, t64);
-    tcg_temp_free_i64(t64);
-}
-
-static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
-                                int insn, int rd)
-{
-    TCGv_i32 r_asi, r_size, r_sign;
-    TCGv t;
-    TCGv_i64 t64;
-
-    r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-    r_size = tcg_const_i32(8);
-    r_sign = tcg_const_i32(0);
-    t64 = tcg_temp_new_i64();
-    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free_i32(r_sign);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-
-    t = gen_dest_gpr(dc, rd + 1);
-    tcg_gen_trunc_i64_tl(t, t64);
-    gen_store_gpr(dc, rd + 1, t);
-
-    tcg_gen_shri_i64(t64, t64, 32);
-    tcg_gen_trunc_i64_tl(hi, t64);
-    tcg_temp_free_i64(t64);
+    gen_store_gpr(dc, rd | 1, lo);
     gen_store_gpr(dc, rd, hi);
 }
 
-static inline void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
-                                int insn, int rd)
+static void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                         int insn, int rd)
 {
-    TCGv_i32 r_asi, r_size;
+    DisasASI da = get_asi(dc, insn, MO_TEQ);
     TCGv lo = gen_load_gpr(dc, rd + 1);
     TCGv_i64 t64 = tcg_temp_new_i64();
 
     tcg_gen_concat_tl_i64(t64, lo, hi);
-    r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-    r_size = tcg_const_i32(8);
-    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
+
+    switch (da.type) {
+    case GET_ASI_EXCP:
+        break;
+    case GET_ASI_DIRECT:
+        gen_address_mask(dc, addr);
+        tcg_gen_qemu_st_i64(t64, addr, da.mem_idx, da.memop);
+        break;
+    default:
+        {
+            TCGv_i32 r_asi = tcg_const_i32(da.asi);
+            TCGv_i32 r_mop = tcg_const_i32(MO_Q);
+
+            save_state(dc);
+            gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_mop);
+            tcg_temp_free_i32(r_mop);
+            tcg_temp_free_i32(r_asi);
+        }
+        break;
+    }
+
     tcg_temp_free_i64(t64);
 }
 #endif
 
-#if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
-static inline void gen_cas_asi(DisasContext *dc, TCGv addr,
-                               TCGv val2, int insn, int rd)
-{
-    TCGv val1 = gen_load_gpr(dc, rd);
-    TCGv dst = gen_dest_gpr(dc, rd);
-#ifdef TARGET_SPARC64
-    TCGv_i32 r_asi = gen_get_asi(insn, addr);
-#else
-    TCGv_i32 r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-#endif
-
-    gen_helper_cas_asi(dst, cpu_env, addr, val1, val2, r_asi);
-    tcg_temp_free_i32(r_asi);
-    gen_store_gpr(dc, rd, dst);
-}
-
-static inline void gen_ldstub_asi(TCGv dst, TCGv addr, int insn)
-{
-    TCGv_i64 r_val;
-    TCGv_i32 r_asi, r_size;
-
-    gen_ld_asi(dst, addr, insn, 1, 0);
-
-    r_val = tcg_const_i64(0xffULL);
-    r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
-    r_size = tcg_const_i32(1);
-    gen_helper_st_asi(cpu_env, addr, r_val, r_asi, r_size);
-    tcg_temp_free_i32(r_size);
-    tcg_temp_free_i32(r_asi);
-    tcg_temp_free_i64(r_val);
-}
-#endif
-
 static TCGv get_src1(DisasContext *dc, unsigned int insn)
 {
     unsigned int rs1 = GET_FIELD(insn, 13, 17);
@@ -2282,11 +2758,11 @@
        the later.  */
     c32 = tcg_temp_new_i32();
     if (cmp->is_bool) {
-        tcg_gen_trunc_i64_i32(c32, cmp->c1);
+        tcg_gen_extrl_i64_i32(c32, cmp->c1);
     } else {
         TCGv_i64 c64 = tcg_temp_new_i64();
         tcg_gen_setcond_i64(cmp->cond, c64, cmp->c1, cmp->c2);
-        tcg_gen_trunc_i64_i32(c32, c64);
+        tcg_gen_extrl_i64_i32(c32, c64);
         tcg_temp_free_i64(c64);
     }
 
@@ -2321,10 +2797,11 @@
     tcg_gen_movcond_i64(cmp->cond, cpu_fpr[qd / 2 + 1], cmp->c1, cmp->c2,
                         cpu_fpr[qs / 2 + 1], cpu_fpr[qd / 2 + 1]);
 
-    gen_update_fprs_dirty(qd);
+    gen_update_fprs_dirty(dc, qd);
 }
 
-static inline void gen_load_trap_state_at_tl(TCGv_ptr r_tsptr, TCGv_ptr cpu_env)
+#ifndef CONFIG_USER_ONLY
+static inline void gen_load_trap_state_at_tl(TCGv_ptr r_tsptr, TCGv_env cpu_env)
 {
     TCGv_i32 r_tl = tcg_temp_new_i32();
 
@@ -2348,6 +2825,7 @@
 
     tcg_temp_free_i32(r_tl);
 }
+#endif
 
 static void gen_edge(DisasContext *dc, TCGv dst, TCGv s1, TCGv s2,
                      int width, bool cc, bool left)
@@ -2508,10 +2986,6 @@
     TCGv_i64 cpu_src1_64, cpu_src2_64, cpu_dst_64;
     target_long simm;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(dc->pc);
-    }
-
     opc = GET_FIELD(insn, 0, 1);
     rd = GET_FIELD(insn, 2, 6);
 
@@ -2627,7 +3101,8 @@
             if (xop == 0x3a) {  /* generate trap */
                 int cond = GET_FIELD(insn, 3, 6);
                 TCGv_i32 trap;
-                int l1 = -1, mask;
+                TCGLabel *l1 = NULL;
+                int mask;
 
                 if (cond == 0) {
                     /* Trap never.  */
@@ -2734,18 +3209,22 @@
                     gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x3: /* V9 rdasi */
-                    tcg_gen_ext_i32_tl(cpu_dst, cpu_asi);
+                    tcg_gen_movi_tl(cpu_dst, dc->asi);
                     gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x4: /* V9 rdtick */
                     {
                         TCGv_ptr r_tickptr;
+                        TCGv_i32 r_const;
 
                         r_tickptr = tcg_temp_new_ptr();
+                        r_const = tcg_const_i32(dc->mem_idx);
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, tick));
-                        gen_helper_tick_get_count(cpu_dst, r_tickptr);
+                        gen_helper_tick_get_count(cpu_dst, cpu_env, r_tickptr,
+                                                  r_const);
                         tcg_temp_free_ptr(r_tickptr);
+                        tcg_temp_free_i32(r_const);
                         gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
@@ -2773,7 +3252,8 @@
                     gen_store_gpr(dc, rd, cpu_gsr);
                     break;
                 case 0x16: /* Softint */
-                    tcg_gen_ext_i32_tl(cpu_dst, cpu_softint);
+                    tcg_gen_ld32s_tl(cpu_dst, cpu_env,
+                                     offsetof(CPUSPARCState, softint));
                     gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x17: /* Tick compare */
@@ -2782,12 +3262,16 @@
                 case 0x18: /* System tick */
                     {
                         TCGv_ptr r_tickptr;
+                        TCGv_i32 r_const;
 
                         r_tickptr = tcg_temp_new_ptr();
+                        r_const = tcg_const_i32(dc->mem_idx);
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, stick));
-                        gen_helper_tick_get_count(cpu_dst, r_tickptr);
+                        gen_helper_tick_get_count(cpu_dst, cpu_env, r_tickptr,
+                                                  r_const);
                         tcg_temp_free_ptr(r_tickptr);
+                        tcg_temp_free_i32(r_const);
                         gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
@@ -2895,12 +3379,16 @@
                 case 4: // tick
                     {
                         TCGv_ptr r_tickptr;
+                        TCGv_i32 r_const;
 
                         r_tickptr = tcg_temp_new_ptr();
+                        r_const = tcg_const_i32(dc->mem_idx);
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, tick));
-                        gen_helper_tick_get_count(cpu_tmp0, r_tickptr);
+                        gen_helper_tick_get_count(cpu_tmp0, cpu_env,
+                                                  r_tickptr, r_const);
                         tcg_temp_free_ptr(r_tickptr);
+                        tcg_temp_free_i32(r_const);
                     }
                     break;
                 case 5: // tba
@@ -2983,7 +3471,7 @@
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
                 xop = GET_FIELD(insn, 18, 26);
-                save_state(dc);
+
                 switch (xop) {
                 case 0x1: /* fmovs */
                     cpu_src1_32 = gen_load_fpr_F(dc, rs2);
@@ -3107,7 +3595,7 @@
                     break;
                 case 0x3: /* V9 fmovq */
                     CHECK_FPU_FEATURE(dc, FLOAT128);
-                    gen_move_Q(rd, rs2);
+                    gen_move_Q(dc, rd, rs2);
                     break;
                 case 0x6: /* V9 fnegd */
                     gen_ne_fop_DD(dc, rd, rs2, gen_helper_fnegd);
@@ -3158,7 +3646,6 @@
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
                 xop = GET_FIELD(insn, 18, 26);
-                save_state(dc);
 
 #ifdef TARGET_SPARC64
 #define FMOVR(sz)                                                  \
@@ -3647,11 +4134,18 @@
                             case 0x3: /* V9 wrasi */
                                 tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
                                 tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0xff);
-                                tcg_gen_trunc_tl_i32(cpu_asi, cpu_tmp0);
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState, asi));
+                                /* End TB to notice changed ASI.  */
+                                save_state(dc);
+                                gen_op_next_insn();
+                                tcg_gen_exit_tb(0);
+                                dc->is_br = 1;
                                 break;
                             case 0x6: /* V9 wrfprs */
                                 tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
                                 tcg_gen_trunc_tl_i32(cpu_fprs, cpu_tmp0);
+                                dc->fprs_dirty = 0;
                                 save_state(dc);
                                 gen_op_next_insn();
                                 tcg_gen_exit_tb(0);
@@ -4494,8 +4988,6 @@
 #endif
 #ifdef TARGET_SPARC64
             } else if (xop == 0x39) { /* V9 return */
-                TCGv_i32 r_const;
-
                 save_state(dc);
                 cpu_src1 = get_src1(dc, insn);
                 cpu_tmp0 = get_temp_tl(dc);
@@ -4513,9 +5005,7 @@
                 }
                 gen_helper_restore(cpu_env);
                 gen_mov_pc_npc(dc);
-                r_const = tcg_const_i32(3);
-                gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
-                tcg_temp_free_i32(r_const);
+                gen_check_align(cpu_tmp0, 3);
                 tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                 dc->npc = DYNAMIC_PC;
                 goto jmp_insn;
@@ -4538,16 +5028,12 @@
                 switch (xop) {
                 case 0x38:      /* jmpl */
                     {
-                        TCGv t;
-                        TCGv_i32 r_const;
-
-                        t = gen_dest_gpr(dc, rd);
+                        TCGv t = gen_dest_gpr(dc, rd);
                         tcg_gen_movi_tl(t, dc->pc);
                         gen_store_gpr(dc, rd, t);
+
                         gen_mov_pc_npc(dc);
-                        r_const = tcg_const_i32(3);
-                        gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
-                        tcg_temp_free_i32(r_const);
+                        gen_check_align(cpu_tmp0, 3);
                         gen_address_mask(dc, cpu_tmp0);
                         tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                         dc->npc = DYNAMIC_PC;
@@ -4556,14 +5042,10 @@
 #if !defined(CONFIG_USER_ONLY) && !defined(TARGET_SPARC64)
                 case 0x39:      /* rett, V9 return */
                     {
-                        TCGv_i32 r_const;
-
                         if (!supervisor(dc))
                             goto priv_insn;
                         gen_mov_pc_npc(dc);
-                        r_const = tcg_const_i32(3);
-                        gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
-                        tcg_temp_free_i32(r_const);
+                        gen_check_align(cpu_tmp0, 3);
                         tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                         dc->npc = DYNAMIC_PC;
                         gen_helper_rett(cpu_env);
@@ -4659,14 +5141,8 @@
                     if (rd & 1)
                         goto illegal_insn;
                     else {
-                        TCGv_i32 r_const;
                         TCGv_i64 t64;
 
-                        save_state(dc);
-                        r_const = tcg_const_i32(7);
-                        /* XXX remove alignment check */
-                        gen_helper_check_align(cpu_env, cpu_addr, r_const);
-                        tcg_temp_free_i32(r_const);
                         gen_address_mask(dc, cpu_addr);
                         t64 = tcg_temp_new_i64();
                         tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx);
@@ -4690,12 +5166,15 @@
                 case 0xd:       /* ldstub -- XXX: should be atomically */
                     {
                         TCGv r_const;
+                        TCGv tmp = tcg_temp_new();
 
                         gen_address_mask(dc, cpu_addr);
-                        tcg_gen_qemu_ld8s(cpu_val, cpu_addr, dc->mem_idx);
+                        tcg_gen_qemu_ld8u(tmp, cpu_addr, dc->mem_idx);
                         r_const = tcg_const_tl(0xff);
                         tcg_gen_qemu_st8(r_const, cpu_addr, dc->mem_idx);
+                        tcg_gen_mov_tl(cpu_val, tmp);
                         tcg_temp_free(r_const);
+                        tcg_temp_free(tmp);
                     }
                     break;
                 case 0x0f:
@@ -4712,89 +5191,34 @@
                     break;
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
                 case 0x10:      /* lda, V9 lduwa, load word alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 4, 0);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TEUL);
                     break;
                 case 0x11:      /* lduba, load unsigned byte alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 1, 0);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_UB);
                     break;
                 case 0x12:      /* lduha, load unsigned halfword alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 2, 0);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TEUW);
                     break;
                 case 0x13:      /* ldda, load double word alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
+                    if (rd & 1) {
                         goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    if (rd & 1)
-                        goto illegal_insn;
-                    save_state(dc);
-                    gen_ldda_asi(dc, cpu_val, cpu_addr, insn, rd);
+                    }
+                    gen_ldda_asi(dc, cpu_addr, insn, rd);
                     goto skip_move;
                 case 0x19:      /* ldsba, load signed byte alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 1, 1);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_SB);
                     break;
                 case 0x1a:      /* ldsha, load signed halfword alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 2, 1);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TESW);
                     break;
                 case 0x1d:      /* ldstuba -- XXX: should be atomically */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_ldstub_asi(cpu_val, cpu_addr, insn);
+                    gen_ldstub_asi(dc, cpu_val, cpu_addr, insn);
                     break;
                 case 0x1f:      /* swapa, swap reg with alt. memory. Also
                                    atomically */
                     CHECK_IU_FEATURE(dc, SWAP);
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
                     cpu_src1 = gen_load_gpr(dc, rd);
-                    gen_swap_asi(cpu_val, cpu_src1, cpu_addr, insn);
+                    gen_swap_asi(dc, cpu_val, cpu_src1, cpu_addr, insn);
                     break;
 
 #ifndef TARGET_SPARC64
@@ -4814,12 +5238,10 @@
                     tcg_gen_qemu_ld64(cpu_val, cpu_addr, dc->mem_idx);
                     break;
                 case 0x18: /* V9 ldswa */
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 4, 1);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TESL);
                     break;
                 case 0x1b: /* V9 ldxa */
-                    save_state(dc);
-                    gen_ld_asi(cpu_val, cpu_addr, insn, 8, 0);
+                    gen_ld_asi(dc, cpu_val, cpu_addr, insn, MO_TEQ);
                     break;
                 case 0x2d: /* V9 prefetch, no effect */
                     goto skip_move;
@@ -4827,17 +5249,15 @@
                     if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc);
-                    gen_ldf_asi(cpu_addr, insn, 4, rd);
-                    gen_update_fprs_dirty(rd);
+                    gen_ldf_asi(dc, cpu_addr, insn, 4, rd);
+                    gen_update_fprs_dirty(dc, rd);
                     goto skip_move;
                 case 0x33: /* V9 lddfa */
                     if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc);
-                    gen_ldf_asi(cpu_addr, insn, 8, DFPREG(rd));
-                    gen_update_fprs_dirty(DFPREG(rd));
+                    gen_ldf_asi(dc, cpu_addr, insn, 8, DFPREG(rd));
+                    gen_update_fprs_dirty(dc, DFPREG(rd));
                     goto skip_move;
                 case 0x3d: /* V9 prefetcha, no effect */
                     goto skip_move;
@@ -4846,9 +5266,8 @@
                     if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc);
-                    gen_ldf_asi(cpu_addr, insn, 16, QFPREG(rd));
-                    gen_update_fprs_dirty(QFPREG(rd));
+                    gen_ldf_asi(dc, cpu_addr, insn, 16, QFPREG(rd));
+                    gen_update_fprs_dirty(dc, QFPREG(rd));
                     goto skip_move;
 #endif
                 default:
@@ -4864,7 +5283,6 @@
                 if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
                 }
-                save_state(dc);
                 switch (xop) {
                 case 0x20:      /* ldf, load fpreg */
                     gen_address_mask(dc, cpu_addr);
@@ -4880,7 +5298,7 @@
                     if (rd == 1) {
                         TCGv_i64 t64 = tcg_temp_new_i64();
                         tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx);
-                        gen_helper_ldxfsr(cpu_env, t64);
+                        gen_helper_ldxfsr(cpu_fsr, cpu_env, cpu_fsr, t64);
                         tcg_temp_free_i64(t64);
                         break;
                     }
@@ -4889,7 +5307,7 @@
                     t0 = get_temp_tl(dc);
                     tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
                     tcg_gen_trunc_tl_i32(cpu_dst_32, t0);
-                    gen_helper_ldfsr(cpu_env, cpu_dst_32);
+                    gen_helper_ldfsr(cpu_fsr, cpu_env, cpu_fsr, cpu_dst_32);
                     break;
                 case 0x22:      /* ldqf, load quad fpreg */
                     {
@@ -4901,7 +5319,7 @@
                         gen_helper_ldqf(cpu_env, cpu_addr, r_const);
                         tcg_temp_free_i32(r_const);
                         gen_op_store_QT0_fpr(QFPREG(rd));
-                        gen_update_fprs_dirty(QFPREG(rd));
+                        gen_update_fprs_dirty(dc, QFPREG(rd));
                     }
                     break;
                 case 0x23:      /* lddf, load double fpreg */
@@ -4934,18 +5352,11 @@
                     if (rd & 1)
                         goto illegal_insn;
                     else {
-                        TCGv_i32 r_const;
                         TCGv_i64 t64;
                         TCGv lo;
 
-                        save_state(dc);
                         gen_address_mask(dc, cpu_addr);
-                        r_const = tcg_const_i32(7);
-                        /* XXX remove alignment check */
-                        gen_helper_check_align(cpu_env, cpu_addr, r_const);
-                        tcg_temp_free_i32(r_const);
                         lo = gen_load_gpr(dc, rd + 1);
-
                         t64 = tcg_temp_new_i64();
                         tcg_gen_concat_tl_i64(t64, lo, cpu_val);
                         tcg_gen_qemu_st64(t64, cpu_addr, dc->mem_idx);
@@ -4954,51 +5365,19 @@
                     break;
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
                 case 0x14: /* sta, V9 stwa, store word alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_st_asi(cpu_val, cpu_addr, insn, 4);
-                    dc->npc = DYNAMIC_PC;
+                    gen_st_asi(dc, cpu_val, cpu_addr, insn, MO_TEUL);
                     break;
                 case 0x15: /* stba, store byte alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_st_asi(cpu_val, cpu_addr, insn, 1);
-                    dc->npc = DYNAMIC_PC;
+                    gen_st_asi(dc, cpu_val, cpu_addr, insn, MO_UB);
                     break;
                 case 0x16: /* stha, store halfword alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
-                        goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    save_state(dc);
-                    gen_st_asi(cpu_val, cpu_addr, insn, 2);
-                    dc->npc = DYNAMIC_PC;
+                    gen_st_asi(dc, cpu_val, cpu_addr, insn, MO_TEUW);
                     break;
                 case 0x17: /* stda, store double word alternate */
-#ifndef TARGET_SPARC64
-                    if (IS_IMM)
+                    if (rd & 1) {
                         goto illegal_insn;
-                    if (!supervisor(dc))
-                        goto priv_insn;
-#endif
-                    if (rd & 1)
-                        goto illegal_insn;
-                    else {
-                        save_state(dc);
-                        gen_stda_asi(dc, cpu_val, cpu_addr, insn, rd);
                     }
+                    gen_stda_asi(dc, cpu_val, cpu_addr, insn, rd);
                     break;
 #endif
 #ifdef TARGET_SPARC64
@@ -5007,9 +5386,7 @@
                     tcg_gen_qemu_st64(cpu_val, cpu_addr, dc->mem_idx);
                     break;
                 case 0x1e: /* V9 stxa */
-                    save_state(dc);
-                    gen_st_asi(cpu_val, cpu_addr, insn, 8);
-                    dc->npc = DYNAMIC_PC;
+                    gen_st_asi(dc, cpu_val, cpu_addr, insn, MO_TEQ);
                     break;
 #endif
                 default:
@@ -5019,7 +5396,6 @@
                 if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
                 }
-                save_state(dc);
                 switch (xop) {
                 case 0x24: /* stf, store fpreg */
                     {
@@ -5032,17 +5408,14 @@
                     break;
                 case 0x25: /* stfsr, V9 stxfsr */
                     {
-                        TCGv t = get_temp_tl(dc);
-
-                        tcg_gen_ld_tl(t, cpu_env, offsetof(CPUSPARCState, fsr));
 #ifdef TARGET_SPARC64
                         gen_address_mask(dc, cpu_addr);
                         if (rd == 1) {
-                            tcg_gen_qemu_st64(t, cpu_addr, dc->mem_idx);
+                            tcg_gen_qemu_st64(cpu_fsr, cpu_addr, dc->mem_idx);
                             break;
                         }
 #endif
-                        tcg_gen_qemu_st32(t, cpu_addr, dc->mem_idx);
+                        tcg_gen_qemu_st32(cpu_fsr, cpu_addr, dc->mem_idx);
                     }
                     break;
                 case 0x26:
@@ -5081,34 +5454,29 @@
                     goto illegal_insn;
                 }
             } else if (xop > 0x33 && xop < 0x3f) {
-                save_state(dc);
                 switch (xop) {
 #ifdef TARGET_SPARC64
                 case 0x34: /* V9 stfa */
                     if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    gen_stf_asi(cpu_addr, insn, 4, rd);
+                    gen_stf_asi(dc, cpu_addr, insn, 4, rd);
                     break;
                 case 0x36: /* V9 stqfa */
                     {
-                        TCGv_i32 r_const;
-
                         CHECK_FPU_FEATURE(dc, FLOAT128);
                         if (gen_trap_ifnofpu(dc)) {
                             goto jmp_insn;
                         }
-                        r_const = tcg_const_i32(7);
-                        gen_helper_check_align(cpu_env, cpu_addr, r_const);
-                        tcg_temp_free_i32(r_const);
-                        gen_stf_asi(cpu_addr, insn, 16, QFPREG(rd));
+                        gen_check_align(cpu_addr, 7);
+                        gen_stf_asi(dc, cpu_addr, insn, 16, QFPREG(rd));
                     }
                     break;
                 case 0x37: /* V9 stdfa */
                     if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    gen_stf_asi(cpu_addr, insn, 8, DFPREG(rd));
+                    gen_stf_asi(dc, cpu_addr, insn, 8, DFPREG(rd));
                     break;
                 case 0x3e: /* V9 casxa */
                     rs2 = GET_FIELD(insn, 27, 31);
@@ -5126,12 +5494,6 @@
                 case 0x3c: /* V9 or LEON3 casa */
 #ifndef TARGET_SPARC64
                     CHECK_IU_FEATURE(dc, CASA);
-                    if (IS_IMM) {
-                        goto illegal_insn;
-                    }
-                    if (!supervisor(dc)) {
-                        goto priv_insn;
-                    }
 #endif
                     rs2 = GET_FIELD(insn, 27, 31);
                     cpu_src2 = gen_load_gpr(dc, rs2);
@@ -5162,63 +5524,27 @@
  jmp_insn:
     goto egress;
  illegal_insn:
-    {
-        TCGv_i32 r_const;
-
-        save_state(dc);
-        r_const = tcg_const_i32(TT_ILL_INSN);
-        gen_helper_raise_exception(cpu_env, r_const);
-        tcg_temp_free_i32(r_const);
-        dc->is_br = 1;
-    }
+    gen_exception(dc, TT_ILL_INSN);
     goto egress;
  unimp_flush:
-    {
-        TCGv_i32 r_const;
-
-        save_state(dc);
-        r_const = tcg_const_i32(TT_UNIMP_FLUSH);
-        gen_helper_raise_exception(cpu_env, r_const);
-        tcg_temp_free_i32(r_const);
-        dc->is_br = 1;
-    }
+    gen_exception(dc, TT_UNIMP_FLUSH);
     goto egress;
 #if !defined(CONFIG_USER_ONLY)
  priv_insn:
-    {
-        TCGv_i32 r_const;
-
-        save_state(dc);
-        r_const = tcg_const_i32(TT_PRIV_INSN);
-        gen_helper_raise_exception(cpu_env, r_const);
-        tcg_temp_free_i32(r_const);
-        dc->is_br = 1;
-    }
+    gen_exception(dc, TT_PRIV_INSN);
     goto egress;
 #endif
  nfpu_insn:
-    save_state(dc);
-    gen_op_fpexception_im(FSR_FTT_UNIMPFPOP);
-    dc->is_br = 1;
+    gen_op_fpexception_im(dc, FSR_FTT_UNIMPFPOP);
     goto egress;
 #if !defined(CONFIG_USER_ONLY) && !defined(TARGET_SPARC64)
  nfq_insn:
-    save_state(dc);
-    gen_op_fpexception_im(FSR_FTT_SEQ_ERROR);
-    dc->is_br = 1;
+    gen_op_fpexception_im(dc, FSR_FTT_SEQ_ERROR);
     goto egress;
 #endif
 #ifndef TARGET_SPARC64
  ncp_insn:
-    {
-        TCGv r_const;
-
-        save_state(dc);
-        r_const = tcg_const_i32(TT_NCP_INSN);
-        gen_helper_raise_exception(cpu_env, r_const);
-        tcg_temp_free(r_const);
-        dc->is_br = 1;
-    }
+    gen_exception(dc, TT_NCP_INSN);
     goto egress;
 #endif
  egress:
@@ -5238,16 +5564,12 @@
     }
 }
 
-static inline void gen_intermediate_code_internal(SPARCCPU *cpu,
-                                                  TranslationBlock *tb,
-                                                  bool spc)
+void gen_intermediate_code(CPUSPARCState * env, TranslationBlock * tb)
 {
+    SPARCCPU *cpu = sparc_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUSPARCState *env = &cpu->env;
     target_ulong pc_start, last_pc;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj = -1;
     int num_insns;
     int max_insns;
     unsigned int insn;
@@ -5259,50 +5581,53 @@
     last_pc = dc->pc;
     dc->npc = (target_ulong) tb->cs_base;
     dc->cc_op = CC_OP_DYNAMIC;
-    dc->mem_idx = cpu_mmu_index(env);
+    dc->mem_idx = tb->flags & TB_FLAG_MMU_MASK;
     dc->def = env->def;
     dc->fpu_enabled = tb_fpu_enabled(tb->flags);
     dc->address_mask_32bit = tb_am_enabled(tb->flags);
     dc->singlestep = (cs->singlestep_enabled || singlestep);
+#ifdef TARGET_SPARC64
+    dc->fprs_dirty = 0;
+    dc->asi = (tb->flags >> TB_FLAG_ASI_SHIFT) & 0xff;
+#endif
 
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
-    if (max_insns == 0)
+    if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
-    gen_tb_start();
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
+    gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    if (dc->pc != pc_start)
-                        save_state(dc);
-                    gen_helper_debug(cpu_env);
-                    tcg_gen_exit_tb(0);
-                    dc->is_br = 1;
-                    goto exit_gen_loop;
-                }
-            }
+        if (dc->npc & JUMP_PC) {
+            assert(dc->jump_pc[1] == dc->pc + 4);
+            tcg_gen_insn_start(dc->pc, dc->jump_pc[0] | JUMP_PC);
+        } else {
+            tcg_gen_insn_start(dc->pc, dc->npc);
         }
-        if (spc) {
-            qemu_log("Search PC...\n");
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j)
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                tcg_ctx.gen_opc_pc[lj] = dc->pc;
-                gen_opc_npc[lj] = dc->npc;
-                tcg_ctx.gen_opc_instr_start[lj] = 1;
-                tcg_ctx.gen_opc_icount[lj] = num_insns;
-            }
-        }
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
-            gen_io_start();
+        num_insns++;
         last_pc = dc->pc;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            if (dc->pc != pc_start) {
+                save_state(dc);
+            }
+            gen_helper_debug(cpu_env);
+            tcg_gen_exit_tb(0);
+            dc->is_br = 1;
+            goto exit_gen_loop;
+        }
+
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
+            gen_io_start();
+        }
+
         insn = cpu_ldl_code(env, dc->pc);
 
         disas_sparc_insn(dc, insn);
-        num_insns++;
 
         if (dc->is_br)
             break;
@@ -5341,154 +5666,131 @@
     }
     gen_tb_end(tb, num_insns);
 
-    if (spc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j)
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-#if 0
-        log_page_dump();
-#endif
-        gen_opc_jump_pc[0] = dc->jump_pc[0];
-        gen_opc_jump_pc[1] = dc->jump_pc[1];
-    } else {
-        tb->size = last_pc + 4 - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = last_pc + 4 - pc_start;
+    tb->icount = num_insns;
+
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("--------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, last_pc + 4 - pc_start, 0);
+        log_target_disas(cs, pc_start, last_pc + 4 - pc_start, 0);
         qemu_log("\n");
     }
 #endif
 }
 
-void gen_intermediate_code(CPUSPARCState * env, TranslationBlock * tb)
-{
-    gen_intermediate_code_internal(sparc_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUSPARCState * env, TranslationBlock * tb)
-{
-    gen_intermediate_code_internal(sparc_env_get_cpu(env), tb, true);
-}
-
 void gen_intermediate_code_init(CPUSPARCState *env)
 {
-    unsigned int i;
     static int inited;
-    static const char * const gregnames[8] = {
-        NULL, // g0 not used
-        "g1",
-        "g2",
-        "g3",
-        "g4",
-        "g5",
-        "g6",
-        "g7",
+    static const char gregnames[32][4] = {
+        "g0", "g1", "g2", "g3", "g4", "g5", "g6", "g7",
+        "o0", "o1", "o2", "o3", "o4", "o5", "o6", "o7",
+        "l0", "l1", "l2", "l3", "l4", "l5", "l6", "l7",
+        "i0", "i1", "i2", "i3", "i4", "i5", "i6", "i7",
     };
-    static const char * const fregnames[32] = {
+    static const char fregnames[32][4] = {
         "f0", "f2", "f4", "f6", "f8", "f10", "f12", "f14",
         "f16", "f18", "f20", "f22", "f24", "f26", "f28", "f30",
         "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46",
         "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",
     };
 
-    /* init various static tables */
-    if (!inited) {
-        inited = 1;
-
-        cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-        cpu_regwptr = tcg_global_mem_new_ptr(TCG_AREG0,
-                                             offsetof(CPUSPARCState, regwptr),
-                                             "regwptr");
+    static const struct { TCGv_i32 *ptr; int off; const char *name; } r32[] = {
 #ifdef TARGET_SPARC64
-        cpu_xcc = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, xcc),
-                                         "xcc");
-        cpu_asi = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, asi),
-                                         "asi");
-        cpu_fprs = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, fprs),
-                                          "fprs");
-        cpu_gsr = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, gsr),
-                                     "gsr");
-        cpu_tick_cmpr = tcg_global_mem_new(TCG_AREG0,
-                                           offsetof(CPUSPARCState, tick_cmpr),
-                                           "tick_cmpr");
-        cpu_stick_cmpr = tcg_global_mem_new(TCG_AREG0,
-                                            offsetof(CPUSPARCState, stick_cmpr),
-                                            "stick_cmpr");
-        cpu_hstick_cmpr = tcg_global_mem_new(TCG_AREG0,
-                                             offsetof(CPUSPARCState, hstick_cmpr),
-                                             "hstick_cmpr");
-        cpu_hintp = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, hintp),
-                                       "hintp");
-        cpu_htba = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, htba),
-                                      "htba");
-        cpu_hver = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, hver),
-                                      "hver");
-        cpu_ssr = tcg_global_mem_new(TCG_AREG0,
-                                     offsetof(CPUSPARCState, ssr), "ssr");
-        cpu_ver = tcg_global_mem_new(TCG_AREG0,
-                                     offsetof(CPUSPARCState, version), "ver");
-        cpu_softint = tcg_global_mem_new_i32(TCG_AREG0,
-                                             offsetof(CPUSPARCState, softint),
-                                             "softint");
+        { &cpu_xcc, offsetof(CPUSPARCState, xcc), "xcc" },
+        { &cpu_fprs, offsetof(CPUSPARCState, fprs), "fprs" },
 #else
-        cpu_wim = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, wim),
-                                     "wim");
+        { &cpu_wim, offsetof(CPUSPARCState, wim), "wim" },
 #endif
-        cpu_cond = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, cond),
-                                      "cond");
-        cpu_cc_src = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, cc_src),
-                                        "cc_src");
-        cpu_cc_src2 = tcg_global_mem_new(TCG_AREG0,
-                                         offsetof(CPUSPARCState, cc_src2),
-                                         "cc_src2");
-        cpu_cc_dst = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, cc_dst),
-                                        "cc_dst");
-        cpu_cc_op = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, cc_op),
-                                           "cc_op");
-        cpu_psr = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUSPARCState, psr),
-                                         "psr");
-        cpu_fsr = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, fsr),
-                                     "fsr");
-        cpu_pc = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, pc),
-                                    "pc");
-        cpu_npc = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, npc),
-                                     "npc");
-        cpu_y = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, y), "y");
+        { &cpu_cc_op, offsetof(CPUSPARCState, cc_op), "cc_op" },
+        { &cpu_psr, offsetof(CPUSPARCState, psr), "psr" },
+    };
+
+    static const struct { TCGv *ptr; int off; const char *name; } rtl[] = {
+#ifdef TARGET_SPARC64
+        { &cpu_gsr, offsetof(CPUSPARCState, gsr), "gsr" },
+        { &cpu_tick_cmpr, offsetof(CPUSPARCState, tick_cmpr), "tick_cmpr" },
+        { &cpu_stick_cmpr, offsetof(CPUSPARCState, stick_cmpr), "stick_cmpr" },
+        { &cpu_hstick_cmpr, offsetof(CPUSPARCState, hstick_cmpr),
+          "hstick_cmpr" },
+        { &cpu_hintp, offsetof(CPUSPARCState, hintp), "hintp" },
+        { &cpu_htba, offsetof(CPUSPARCState, htba), "htba" },
+        { &cpu_hver, offsetof(CPUSPARCState, hver), "hver" },
+        { &cpu_ssr, offsetof(CPUSPARCState, ssr), "ssr" },
+        { &cpu_ver, offsetof(CPUSPARCState, version), "ver" },
+#endif
+        { &cpu_cond, offsetof(CPUSPARCState, cond), "cond" },
+        { &cpu_cc_src, offsetof(CPUSPARCState, cc_src), "cc_src" },
+        { &cpu_cc_src2, offsetof(CPUSPARCState, cc_src2), "cc_src2" },
+        { &cpu_cc_dst, offsetof(CPUSPARCState, cc_dst), "cc_dst" },
+        { &cpu_fsr, offsetof(CPUSPARCState, fsr), "fsr" },
+        { &cpu_pc, offsetof(CPUSPARCState, pc), "pc" },
+        { &cpu_npc, offsetof(CPUSPARCState, npc), "npc" },
+        { &cpu_y, offsetof(CPUSPARCState, y), "y" },
 #ifndef CONFIG_USER_ONLY
-        cpu_tbr = tcg_global_mem_new(TCG_AREG0, offsetof(CPUSPARCState, tbr),
-                                     "tbr");
+        { &cpu_tbr, offsetof(CPUSPARCState, tbr), "tbr" },
 #endif
-        for (i = 1; i < 8; i++) {
-            cpu_gregs[i] = tcg_global_mem_new(TCG_AREG0,
-                                              offsetof(CPUSPARCState, gregs[i]),
-                                              gregnames[i]);
-        }
-        for (i = 0; i < TARGET_DPREGS; i++) {
-            cpu_fpr[i] = tcg_global_mem_new_i64(TCG_AREG0,
-                                                offsetof(CPUSPARCState, fpr[i]),
-                                                fregnames[i]);
-        }
+    };
+
+    unsigned int i;
+
+    /* init various static tables */
+    if (inited) {
+        return;
+    }
+    inited = 1;
+
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
+
+    cpu_regwptr = tcg_global_mem_new_ptr(cpu_env,
+                                         offsetof(CPUSPARCState, regwptr),
+                                         "regwptr");
+
+    for (i = 0; i < ARRAY_SIZE(r32); ++i) {
+        *r32[i].ptr = tcg_global_mem_new_i32(cpu_env, r32[i].off, r32[i].name);
+    }
+
+    for (i = 0; i < ARRAY_SIZE(rtl); ++i) {
+        *rtl[i].ptr = tcg_global_mem_new(cpu_env, rtl[i].off, rtl[i].name);
+    }
+
+    TCGV_UNUSED(cpu_regs[0]);
+    for (i = 1; i < 8; ++i) {
+        cpu_regs[i] = tcg_global_mem_new(cpu_env,
+                                         offsetof(CPUSPARCState, gregs[i]),
+                                         gregnames[i]);
+    }
+
+    for (i = 8; i < 32; ++i) {
+        cpu_regs[i] = tcg_global_mem_new(cpu_regwptr,
+                                         (i - 8) * sizeof(target_ulong),
+                                         gregnames[i]);
+    }
+
+    for (i = 0; i < TARGET_DPREGS; i++) {
+        cpu_fpr[i] = tcg_global_mem_new_i64(cpu_env,
+                                            offsetof(CPUSPARCState, fpr[i]),
+                                            fregnames[i]);
     }
 }
 
-void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    target_ulong npc;
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
-    npc = gen_opc_npc[pc_pos];
-    if (npc == 1) {
+    target_ulong pc = data[0];
+    target_ulong npc = data[1];
+
+    env->pc = pc;
+    if (npc == DYNAMIC_PC) {
         /* dynamic NPC: already stored */
-    } else if (npc == 2) {
+    } else if (npc & JUMP_PC) {
         /* jump PC: use 'cond' and the jump targets of the translation */
         if (env->cond) {
-            env->npc = gen_opc_jump_pc[0];
+            env->npc = npc & ~3;
         } else {
-            env->npc = gen_opc_jump_pc[1];
+            env->npc = pc + 4;
         }
     } else {
         env->npc = npc;
diff --git a/target-sparc/vis_helper.c b/target-sparc/vis_helper.c
index 383cc8b..8a9b763 100644
--- a/target-sparc/vis_helper.c
+++ b/target-sparc/vis_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
@@ -447,7 +448,7 @@
     for (word = 0; word < 2; word++) {
         uint32_t val;
         int32_t src = rs2 >> (word * 32);
-        int64_t scaled = src << scale;
+        int64_t scaled = (int64_t)src << scale;
         int64_t from_fixed = scaled >> 16;
 
         val = (from_fixed < -32768 ? -32768 :
diff --git a/target-sparc/win_helper.c b/target-sparc/win_helper.c
index f01ae08..a8a6c0c 100644
--- a/target-sparc/win_helper.c
+++ b/target-sparc/win_helper.c
@@ -17,6 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "trace.h"
@@ -64,23 +65,28 @@
 #endif
 }
 
-void cpu_put_psr(CPUSPARCState *env, target_ulong val)
+void cpu_put_psr_raw(CPUSPARCState *env, target_ulong val)
 {
     env->psr = val & PSR_ICC;
 #if !defined(TARGET_SPARC64)
     env->psref = (val & PSR_EF) ? 1 : 0;
     env->psrpil = (val & PSR_PIL) >> 8;
-#endif
-#if ((!defined(TARGET_SPARC64)) && !defined(CONFIG_USER_ONLY))
-    cpu_check_irqs(env);
-#endif
-#if !defined(TARGET_SPARC64)
     env->psrs = (val & PSR_S) ? 1 : 0;
     env->psrps = (val & PSR_PS) ? 1 : 0;
     env->psret = (val & PSR_ET) ? 1 : 0;
-    cpu_set_cwp(env, val & PSR_CWP);
 #endif
     env->cc_op = CC_OP_FLAGS;
+#if !defined(TARGET_SPARC64)
+    cpu_set_cwp(env, val & PSR_CWP);
+#endif
+}
+
+void cpu_put_psr(CPUSPARCState *env, target_ulong val)
+{
+    cpu_put_psr_raw(env, val);
+#if ((!defined(TARGET_SPARC64)) && !defined(CONFIG_USER_ONLY))
+    cpu_check_irqs(env);
+#endif
 }
 
 int cpu_cwp_inc(CPUSPARCState *env, int cwp)
diff --git a/target-tilegx/Makefile.objs b/target-tilegx/Makefile.objs
new file mode 100644
index 0000000..0db778f
--- /dev/null
+++ b/target-tilegx/Makefile.objs
@@ -0,0 +1 @@
+obj-y += cpu.o translate.o helper.o simd_helper.o
diff --git a/target-tilegx/cpu.c b/target-tilegx/cpu.c
new file mode 100644
index 0000000..7017cb6
--- /dev/null
+++ b/target-tilegx/cpu.c
@@ -0,0 +1,188 @@
+/*
+ * QEMU TILE-Gx CPU
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "cpu.h"
+#include "qemu-common.h"
+#include "hw/qdev-properties.h"
+#include "migration/vmstate.h"
+#include "linux-user/syscall_defs.h"
+#include "exec/exec-all.h"
+
+static void tilegx_cpu_dump_state(CPUState *cs, FILE *f,
+                                  fprintf_function cpu_fprintf, int flags)
+{
+    static const char * const reg_names[TILEGX_R_COUNT] = {
+         "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+         "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+        "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+        "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+        "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+        "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+        "r48", "r49", "r50", "r51",  "bp",  "tp",  "sp",  "lr"
+    };
+
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+    CPUTLGState *env = &cpu->env;
+    int i;
+
+    for (i = 0; i < TILEGX_R_COUNT; i++) {
+        cpu_fprintf(f, "%-4s" TARGET_FMT_lx "%s",
+                    reg_names[i], env->regs[i],
+                    (i % 4) == 3 ? "\n" : " ");
+    }
+    cpu_fprintf(f, "PC  " TARGET_FMT_lx " CEX " TARGET_FMT_lx "\n\n",
+                env->pc, env->spregs[TILEGX_SPR_CMPEXCH]);
+}
+
+TileGXCPU *cpu_tilegx_init(const char *cpu_model)
+{
+    TileGXCPU *cpu;
+
+    cpu = TILEGX_CPU(object_new(TYPE_TILEGX_CPU));
+
+    object_property_set_bool(OBJECT(cpu), true, "realized", NULL);
+
+    return cpu;
+}
+
+static void tilegx_cpu_set_pc(CPUState *cs, vaddr value)
+{
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+
+    cpu->env.pc = value;
+}
+
+static bool tilegx_cpu_has_work(CPUState *cs)
+{
+    return true;
+}
+
+static void tilegx_cpu_reset(CPUState *s)
+{
+    TileGXCPU *cpu = TILEGX_CPU(s);
+    TileGXCPUClass *tcc = TILEGX_CPU_GET_CLASS(cpu);
+    CPUTLGState *env = &cpu->env;
+
+    tcc->parent_reset(s);
+
+    memset(env, 0, sizeof(CPUTLGState));
+    tlb_flush(s, 1);
+}
+
+static void tilegx_cpu_realizefn(DeviceState *dev, Error **errp)
+{
+    CPUState *cs = CPU(dev);
+    TileGXCPUClass *tcc = TILEGX_CPU_GET_CLASS(dev);
+
+    cpu_reset(cs);
+    qemu_init_vcpu(cs);
+
+    tcc->parent_realize(dev, errp);
+}
+
+static void tilegx_cpu_initfn(Object *obj)
+{
+    CPUState *cs = CPU(obj);
+    TileGXCPU *cpu = TILEGX_CPU(obj);
+    CPUTLGState *env = &cpu->env;
+    static bool tcg_initialized;
+
+    cs->env_ptr = env;
+    cpu_exec_init(cs, &error_abort);
+
+    if (tcg_enabled() && !tcg_initialized) {
+        tcg_initialized = true;
+        tilegx_tcg_init();
+    }
+}
+
+static void tilegx_cpu_do_interrupt(CPUState *cs)
+{
+    cs->exception_index = -1;
+}
+
+static int tilegx_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
+                                       int mmu_idx)
+{
+    TileGXCPU *cpu = TILEGX_CPU(cs);
+
+    /* The sigcode field will be filled in by do_signal in main.c.  */
+    cs->exception_index = TILEGX_EXCP_SIGNAL;
+    cpu->env.excaddr = address;
+    cpu->env.signo = TARGET_SIGSEGV;
+    cpu->env.sigcode = 0;
+
+    return 1;
+}
+
+static bool tilegx_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
+{
+    if (interrupt_request & CPU_INTERRUPT_HARD) {
+        tilegx_cpu_do_interrupt(cs);
+        return true;
+    }
+    return false;
+}
+
+static void tilegx_cpu_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    CPUClass *cc = CPU_CLASS(oc);
+    TileGXCPUClass *tcc = TILEGX_CPU_CLASS(oc);
+
+    tcc->parent_realize = dc->realize;
+    dc->realize = tilegx_cpu_realizefn;
+
+    tcc->parent_reset = cc->reset;
+    cc->reset = tilegx_cpu_reset;
+
+    cc->has_work = tilegx_cpu_has_work;
+    cc->do_interrupt = tilegx_cpu_do_interrupt;
+    cc->cpu_exec_interrupt = tilegx_cpu_exec_interrupt;
+    cc->dump_state = tilegx_cpu_dump_state;
+    cc->set_pc = tilegx_cpu_set_pc;
+    cc->handle_mmu_fault = tilegx_cpu_handle_mmu_fault;
+    cc->gdb_num_core_regs = 0;
+
+    /*
+     * Reason: tilegx_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
+}
+
+static const TypeInfo tilegx_cpu_type_info = {
+    .name = TYPE_TILEGX_CPU,
+    .parent = TYPE_CPU,
+    .instance_size = sizeof(TileGXCPU),
+    .instance_init = tilegx_cpu_initfn,
+    .class_size = sizeof(TileGXCPUClass),
+    .class_init = tilegx_cpu_class_init,
+};
+
+static void tilegx_cpu_register_types(void)
+{
+    type_register_static(&tilegx_cpu_type_info);
+}
+
+type_init(tilegx_cpu_register_types)
diff --git a/target-tilegx/cpu.h b/target-tilegx/cpu.h
new file mode 100644
index 0000000..1735427
--- /dev/null
+++ b/target-tilegx/cpu.h
@@ -0,0 +1,178 @@
+/*
+ *  TILE-Gx virtual CPU header
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TILEGX_CPU_H
+#define TILEGX_CPU_H
+
+#include "qemu-common.h"
+
+#define TARGET_LONG_BITS 64
+
+#define CPUArchState struct CPUTLGState
+
+#include "exec/cpu-defs.h"
+
+
+/* TILE-Gx common register alias */
+#define TILEGX_R_RE    0   /*  0 register, for function/syscall return value */
+#define TILEGX_R_ERR   1   /*  1 register, for syscall errno flag */
+#define TILEGX_R_NR    10  /* 10 register, for syscall number */
+#define TILEGX_R_BP    52  /* 52 register, optional frame pointer */
+#define TILEGX_R_TP    53  /* TP register, thread local storage data */
+#define TILEGX_R_SP    54  /* SP register, stack pointer */
+#define TILEGX_R_LR    55  /* LR register, may save pc, but it is not pc */
+#define TILEGX_R_COUNT 56  /* Only 56 registers are really useful */
+#define TILEGX_R_SN    56  /* SN register, obsoleted, it likes zero register */
+#define TILEGX_R_IDN0  57  /* IDN0 register, cause IDN_ACCESS exception */
+#define TILEGX_R_IDN1  58  /* IDN1 register, cause IDN_ACCESS exception */
+#define TILEGX_R_UDN0  59  /* UDN0 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN1  60  /* UDN1 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN2  61  /* UDN2 register, cause UDN_ACCESS exception */
+#define TILEGX_R_UDN3  62  /* UDN3 register, cause UDN_ACCESS exception */
+#define TILEGX_R_ZERO  63  /* Zero register, always zero */
+#define TILEGX_R_NOREG 255 /* Invalid register value */
+
+/* TILE-Gx special registers used by outside */
+enum {
+    TILEGX_SPR_CMPEXCH = 0,
+    TILEGX_SPR_CRITICAL_SEC = 1,
+    TILEGX_SPR_SIM_CONTROL = 2,
+    TILEGX_SPR_EX_CONTEXT_0_0 = 3,
+    TILEGX_SPR_EX_CONTEXT_0_1 = 4,
+    TILEGX_SPR_COUNT
+};
+
+/* Exception numbers */
+typedef enum {
+    TILEGX_EXCP_NONE = 0,
+    TILEGX_EXCP_SYSCALL = 1,
+    TILEGX_EXCP_SIGNAL = 2,
+    TILEGX_EXCP_OPCODE_UNKNOWN = 0x101,
+    TILEGX_EXCP_OPCODE_UNIMPLEMENTED = 0x102,
+    TILEGX_EXCP_OPCODE_CMPEXCH = 0x103,
+    TILEGX_EXCP_OPCODE_CMPEXCH4 = 0x104,
+    TILEGX_EXCP_OPCODE_EXCH = 0x105,
+    TILEGX_EXCP_OPCODE_EXCH4 = 0x106,
+    TILEGX_EXCP_OPCODE_FETCHADD = 0x107,
+    TILEGX_EXCP_OPCODE_FETCHADD4 = 0x108,
+    TILEGX_EXCP_OPCODE_FETCHADDGEZ = 0x109,
+    TILEGX_EXCP_OPCODE_FETCHADDGEZ4 = 0x10a,
+    TILEGX_EXCP_OPCODE_FETCHAND = 0x10b,
+    TILEGX_EXCP_OPCODE_FETCHAND4 = 0x10c,
+    TILEGX_EXCP_OPCODE_FETCHOR = 0x10d,
+    TILEGX_EXCP_OPCODE_FETCHOR4 = 0x10e,
+    TILEGX_EXCP_REG_IDN_ACCESS = 0x181,
+    TILEGX_EXCP_REG_UDN_ACCESS = 0x182,
+    TILEGX_EXCP_UNALIGNMENT = 0x201,
+    TILEGX_EXCP_DBUG_BREAK = 0x301
+} TileExcp;
+
+typedef struct CPUTLGState {
+    uint64_t regs[TILEGX_R_COUNT];     /* Common used registers by outside */
+    uint64_t spregs[TILEGX_SPR_COUNT]; /* Special used registers by outside */
+    uint64_t pc;                       /* Current pc */
+
+#if defined(CONFIG_USER_ONLY)
+    uint64_t excaddr;                  /* exception address */
+    uint64_t atomic_srca;              /* Arguments to atomic "exceptions" */
+    uint64_t atomic_srcb;
+    uint32_t atomic_dstr;
+    uint32_t signo;                    /* Signal number */
+    uint32_t sigcode;                  /* Signal code */
+#endif
+
+    CPU_COMMON
+} CPUTLGState;
+
+#include "qom/cpu.h"
+
+#define TYPE_TILEGX_CPU "tilegx-cpu"
+
+#define TILEGX_CPU_CLASS(klass) \
+    OBJECT_CLASS_CHECK(TileGXCPUClass, (klass), TYPE_TILEGX_CPU)
+#define TILEGX_CPU(obj) \
+    OBJECT_CHECK(TileGXCPU, (obj), TYPE_TILEGX_CPU)
+#define TILEGX_CPU_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(TileGXCPUClass, (obj), TYPE_TILEGX_CPU)
+
+/**
+ * TileGXCPUClass:
+ * @parent_realize: The parent class' realize handler.
+ * @parent_reset: The parent class' reset handler.
+ *
+ * A Tile-Gx CPU model.
+ */
+typedef struct TileGXCPUClass {
+    /*< private >*/
+    CPUClass parent_class;
+    /*< public >*/
+
+    DeviceRealize parent_realize;
+    void (*parent_reset)(CPUState *cpu);
+} TileGXCPUClass;
+
+/**
+ * TileGXCPU:
+ * @env: #CPUTLGState
+ *
+ * A Tile-GX CPU.
+ */
+typedef struct TileGXCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUTLGState env;
+} TileGXCPU;
+
+static inline TileGXCPU *tilegx_env_get_cpu(CPUTLGState *env)
+{
+    return container_of(env, TileGXCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(tilegx_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(TileGXCPU, env)
+
+/* TILE-Gx memory attributes */
+#define TARGET_PAGE_BITS 16  /* TILE-Gx uses 64KB page size */
+#define TARGET_PHYS_ADDR_SPACE_BITS 42
+#define TARGET_VIRT_ADDR_SPACE_BITS 64
+#define MMU_USER_IDX    0  /* Current memory operation is in user mode */
+
+#include "exec/cpu-all.h"
+
+void tilegx_tcg_init(void);
+int cpu_tilegx_signal_handler(int host_signum, void *pinfo, void *puc);
+
+TileGXCPU *cpu_tilegx_init(const char *cpu_model);
+
+#define cpu_init(cpu_model) CPU(cpu_tilegx_init(cpu_model))
+
+#define cpu_signal_handler cpu_tilegx_signal_handler
+
+static inline void cpu_get_tb_cpu_state(CPUTLGState *env, target_ulong *pc,
+                                        target_ulong *cs_base, uint32_t *flags)
+{
+    *pc = env->pc;
+    *cs_base = 0;
+    *flags = 0;
+}
+
+#endif
diff --git a/target-tilegx/helper.c b/target-tilegx/helper.c
new file mode 100644
index 0000000..b4fba9c
--- /dev/null
+++ b/target-tilegx/helper.c
@@ -0,0 +1,163 @@
+/*
+ * QEMU TILE-Gx helpers
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
+#include "qemu-common.h"
+#include "exec/helper-proto.h"
+#include <zlib.h> /* For crc32 */
+#include "syscall_defs.h"
+
+void helper_exception(CPUTLGState *env, uint32_t excp)
+{
+    CPUState *cs = CPU(tilegx_env_get_cpu(env));
+
+    cs->exception_index = excp;
+    cpu_loop_exit(cs);
+}
+
+void helper_ext01_ics(CPUTLGState *env)
+{
+    uint64_t val = env->spregs[TILEGX_SPR_EX_CONTEXT_0_1];
+
+    switch (val) {
+    case 0:
+    case 1:
+        env->spregs[TILEGX_SPR_CRITICAL_SEC] = val;
+        break;
+    default:
+#if defined(CONFIG_USER_ONLY)
+        env->signo = TARGET_SIGILL;
+        env->sigcode = TARGET_ILL_ILLOPC;
+        helper_exception(env, TILEGX_EXCP_SIGNAL);
+#else
+        helper_exception(env, TILEGX_EXCP_OPCODE_UNIMPLEMENTED);
+#endif
+        break;
+    }
+}
+
+uint64_t helper_cntlz(uint64_t arg)
+{
+    return clz64(arg);
+}
+
+uint64_t helper_cnttz(uint64_t arg)
+{
+    return ctz64(arg);
+}
+
+uint64_t helper_pcnt(uint64_t arg)
+{
+    return ctpop64(arg);
+}
+
+uint64_t helper_revbits(uint64_t arg)
+{
+    return revbit64(arg);
+}
+
+/*
+ * Functional Description
+ *     uint64_t a = rf[SrcA];
+ *     uint64_t b = rf[SrcB];
+ *     uint64_t d = rf[Dest];
+ *     uint64_t output = 0;
+ *     unsigned int counter;
+ *     for (counter = 0; counter < (WORD_SIZE / BYTE_SIZE); counter++)
+ *     {
+ *         int sel = getByte (b, counter) & 0xf;
+ *         uint8_t byte = (sel < 8) ? getByte (d, sel) : getByte (a, (sel - 8));
+ *         output = setByte (output, counter, byte);
+ *     }
+ *     rf[Dest] = output;
+ */
+uint64_t helper_shufflebytes(uint64_t dest, uint64_t srca, uint64_t srcb)
+{
+    uint64_t vdst = 0;
+    int count;
+
+    for (count = 0; count < 64; count += 8) {
+        uint64_t sel = srcb >> count;
+        uint64_t src = (sel & 8) ? srca : dest;
+        vdst |= extract64(src, (sel & 7) * 8, 8) << count;
+    }
+
+    return vdst;
+}
+
+uint64_t helper_crc32_8(uint64_t accum, uint64_t input)
+{
+    uint8_t buf = input;
+
+    /* zlib crc32 converts the accumulator and output to one's complement.  */
+    return crc32(accum ^ 0xffffffff, &buf, 1) ^ 0xffffffff;
+}
+
+uint64_t helper_crc32_32(uint64_t accum, uint64_t input)
+{
+    uint8_t buf[4];
+
+    stl_le_p(buf, input);
+
+    /* zlib crc32 converts the accumulator and output to one's complement.  */
+    return crc32(accum ^ 0xffffffff, buf, 4) ^ 0xffffffff;
+}
+
+uint64_t helper_cmula(uint64_t srcd, uint64_t srca, uint64_t srcb)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    uint32_t reald = srcd;
+    uint32_t imagd = srcd >> 32;
+    uint32_t realr = reala * realb - imaga * imagb + reald;
+    uint32_t imagr = reala * imagb + imaga * realb + imagd;
+
+    return deposit64(realr, 32, 32, imagr);
+}
+
+uint64_t helper_cmulaf(uint64_t srcd, uint64_t srca, uint64_t srcb)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    uint32_t reald = (int16_t)srcd;
+    uint32_t imagd = (int16_t)(srcd >> 16);
+    int32_t realr = reala * realb - imaga * imagb;
+    int32_t imagr = reala * imagb + imaga * realb;
+
+    return deposit32((realr >> 15) + reald, 16, 16, (imagr >> 15) + imagd);
+}
+
+uint64_t helper_cmul2(uint64_t srca, uint64_t srcb, int shift, int round)
+{
+    uint32_t reala = (int16_t)srca;
+    uint32_t imaga = (int16_t)(srca >> 16);
+    uint32_t realb = (int16_t)srcb;
+    uint32_t imagb = (int16_t)(srcb >> 16);
+    int32_t realr = reala * realb - imaga * imagb + round;
+    int32_t imagr = reala * imagb + imaga * realb + round;
+
+    return deposit32(realr >> shift, 16, 16, imagr >> shift);
+}
diff --git a/target-tilegx/helper.h b/target-tilegx/helper.h
new file mode 100644
index 0000000..9281d0f
--- /dev/null
+++ b/target-tilegx/helper.h
@@ -0,0 +1,26 @@
+DEF_HELPER_2(exception, noreturn, env, i32)
+DEF_HELPER_1(ext01_ics, void, env)
+DEF_HELPER_FLAGS_1(cntlz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(cnttz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pcnt, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(revbits, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_3(shufflebytes, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(crc32_8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(crc32_32, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cmula, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_3(cmulaf, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_4(cmul2, TCG_CALL_NO_RWG_SE, i64, i64, i64, int, int)
+
+DEF_HELPER_FLAGS_2(v1int_h, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1int_l, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2int_h, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2int_l, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(v1multu, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2mults, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shl, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shru, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v1shrs, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shl, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shru, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(v2shrs, TCG_CALL_NO_RWG_SE, i64, i64, i64)
diff --git a/target-tilegx/opcode_tilegx.h b/target-tilegx/opcode_tilegx.h
new file mode 100644
index 0000000..55376be
--- /dev/null
+++ b/target-tilegx/opcode_tilegx.h
@@ -0,0 +1,1406 @@
+/* TILE-Gx opcode information.
+ *
+ * Copyright 2011 Tilera Corporation. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   as published by the Free Software Foundation, version 2.
+ *
+ *   This program is distributed in the hope that it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *   NON INFRINGEMENT.  See the GNU General Public License for
+ *   more details.
+ *
+ *
+ *
+ *
+ *
+ */
+
+#ifndef OPCODE_TILEGX_H
+#define OPCODE_TILEGX_H
+
+#ifndef __ASSEMBLER__
+
+typedef uint64_t tilegx_bundle_bits;
+
+/* These are the bits that determine if a bundle is in the X encoding. */
+#define TILEGX_BUNDLE_MODE_MASK ((tilegx_bundle_bits)3 << 62)
+
+enum
+{
+  /* Maximum number of instructions in a bundle (2 for X, 3 for Y). */
+  TILEGX_MAX_INSTRUCTIONS_PER_BUNDLE = 3,
+
+  /* How many different pipeline encodings are there? X0, X1, Y0, Y1, Y2. */
+  TILEGX_NUM_PIPELINE_ENCODINGS = 5,
+
+  /* Log base 2 of TILEGX_BUNDLE_SIZE_IN_BYTES. */
+  TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES = 3,
+
+  /* Instructions take this many bytes. */
+  TILEGX_BUNDLE_SIZE_IN_BYTES = 1 << TILEGX_LOG2_BUNDLE_SIZE_IN_BYTES,
+
+  /* Log base 2 of TILEGX_BUNDLE_ALIGNMENT_IN_BYTES. */
+  TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES = 3,
+
+  /* Bundles should be aligned modulo this number of bytes. */
+  TILEGX_BUNDLE_ALIGNMENT_IN_BYTES =
+    (1 << TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES),
+
+  /* Number of registers (some are magic, such as network I/O). */
+  TILEGX_NUM_REGISTERS = 64,
+};
+
+/* Make a few "tile_" variables to simplify common code between
+   architectures.  */
+
+typedef tilegx_bundle_bits tile_bundle_bits;
+#define TILE_BUNDLE_SIZE_IN_BYTES TILEGX_BUNDLE_SIZE_IN_BYTES
+#define TILE_BUNDLE_ALIGNMENT_IN_BYTES TILEGX_BUNDLE_ALIGNMENT_IN_BYTES
+#define TILE_LOG2_BUNDLE_ALIGNMENT_IN_BYTES \
+  TILEGX_LOG2_BUNDLE_ALIGNMENT_IN_BYTES
+#define TILE_BPT_BUNDLE TILEGX_BPT_BUNDLE
+
+/* 64-bit pattern for a { bpt ; nop } bundle. */
+#define TILEGX_BPT_BUNDLE 0x286a44ae51485000ULL
+
+static inline unsigned int
+get_BFEnd_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_BFOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 24)) & 0xf);
+}
+
+static inline unsigned int
+get_BFStart_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3f);
+}
+
+static inline unsigned int
+get_BrOff_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 37)) & 0x0001ffc0);
+}
+
+static inline unsigned int
+get_BrType_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 54)) & 0x1f);
+}
+
+static inline unsigned int
+get_Dest_Imm8_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 43)) & 0x000000c0);
+}
+
+static inline unsigned int
+get_Dest_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 0)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 0)) & 0x3f);
+}
+
+static inline unsigned int
+get_Dest_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x3f);
+}
+
+static inline unsigned int
+get_Imm16_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xffff);
+}
+
+static inline unsigned int
+get_Imm16_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xffff);
+}
+
+static inline unsigned int
+get_Imm8OpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 20)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8OpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 51)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0xff);
+}
+
+static inline unsigned int
+get_Imm8_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0xff);
+}
+
+static inline unsigned int
+get_JumpOff_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x7ffffff);
+}
+
+static inline unsigned int
+get_JumpOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 58)) & 0x1);
+}
+
+static inline unsigned int
+get_MF_Imm14_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3fff);
+}
+
+static inline unsigned int
+get_MT_Imm14_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 31)) & 0x0000003f) |
+         (((unsigned int)(n >> 37)) & 0x00003fc0);
+}
+
+static inline unsigned int
+get_Mode(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 62)) & 0x3);
+}
+
+static inline unsigned int
+get_Opcode_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 28)) & 0x7);
+}
+
+static inline unsigned int
+get_Opcode_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 59)) & 0x7);
+}
+
+static inline unsigned int
+get_Opcode_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 27)) & 0xf);
+}
+
+static inline unsigned int
+get_Opcode_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 58)) & 0xf);
+}
+
+static inline unsigned int
+get_Opcode_Y2(tilegx_bundle_bits n)
+{
+  return (((n >> 26)) & 0x00000001) |
+         (((unsigned int)(n >> 56)) & 0x00000002);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3ff);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3ff);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3);
+}
+
+static inline unsigned int
+get_RRROpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3);
+}
+
+static inline unsigned int
+get_ShAmt_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShAmt_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3ff);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3ff);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 18)) & 0x3);
+}
+
+static inline unsigned int
+get_ShiftOpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 49)) & 0x3);
+}
+
+static inline unsigned int
+get_SrcA_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 6)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 6)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 37)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcA_Y2(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 20)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcBDest_Y2(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 51)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_SrcB_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_X0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_X1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_Y0(tilegx_bundle_bits num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((n >> 12)) & 0x3f);
+}
+
+static inline unsigned int
+get_UnaryOpcodeExtension_Y1(tilegx_bundle_bits n)
+{
+  return (((unsigned int)(n >> 43)) & 0x3f);
+}
+
+
+static inline int
+sign_extend(int n, int num_bits)
+{
+  int shift = (int)(sizeof(int) * 8 - num_bits);
+  return (n << shift) >> shift;
+}
+
+
+
+static inline tilegx_bundle_bits
+create_BFEnd_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_BFOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xf) << 24);
+}
+
+static inline tilegx_bundle_bits
+create_BFStart_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_BrOff_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x0001ffc0)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_BrType_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x1f)) << 54);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Imm8_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x000000c0)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 0);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 0);
+}
+
+static inline tilegx_bundle_bits
+create_Dest_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_Imm16_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xffff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm16_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xffff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8OpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 20);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8OpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 51);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xff) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_Imm8_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xff)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_JumpOff_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x7ffffff)) << 31);
+}
+
+static inline tilegx_bundle_bits
+create_JumpOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x1)) << 58);
+}
+
+static inline tilegx_bundle_bits
+create_MF_Imm14_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3fff)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_MT_Imm14_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x0000003f)) << 31) |
+         (((tilegx_bundle_bits)(n & 0x00003fc0)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_Mode(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 62);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x7) << 28);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x7)) << 59);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0xf) << 27);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0xf)) << 58);
+}
+
+static inline tilegx_bundle_bits
+create_Opcode_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x00000001) << 26) |
+         (((tilegx_bundle_bits)(n & 0x00000002)) << 56);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3ff) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3ff)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_RRROpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_ShAmt_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3ff) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3ff)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3) << 18);
+}
+
+static inline tilegx_bundle_bits
+create_ShiftOpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3)) << 49);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 6);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 6);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 37);
+}
+
+static inline tilegx_bundle_bits
+create_SrcA_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 20);
+}
+
+static inline tilegx_bundle_bits
+create_SrcBDest_Y2(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 51);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_SrcB_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_X0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_X1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_Y0(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return ((n & 0x3f) << 12);
+}
+
+static inline tilegx_bundle_bits
+create_UnaryOpcodeExtension_Y1(int num)
+{
+  const unsigned int n = (unsigned int)num;
+  return (((tilegx_bundle_bits)(n & 0x3f)) << 43);
+}
+
+
+enum
+{
+  ADDI_IMM8_OPCODE_X0 = 1,
+  ADDI_IMM8_OPCODE_X1 = 1,
+  ADDI_OPCODE_Y0 = 0,
+  ADDI_OPCODE_Y1 = 1,
+  ADDLI_OPCODE_X0 = 1,
+  ADDLI_OPCODE_X1 = 0,
+  ADDXI_IMM8_OPCODE_X0 = 2,
+  ADDXI_IMM8_OPCODE_X1 = 2,
+  ADDXI_OPCODE_Y0 = 1,
+  ADDXI_OPCODE_Y1 = 2,
+  ADDXLI_OPCODE_X0 = 2,
+  ADDXLI_OPCODE_X1 = 1,
+  ADDXSC_RRR_0_OPCODE_X0 = 1,
+  ADDXSC_RRR_0_OPCODE_X1 = 1,
+  ADDX_RRR_0_OPCODE_X0 = 2,
+  ADDX_RRR_0_OPCODE_X1 = 2,
+  ADDX_RRR_0_OPCODE_Y0 = 0,
+  ADDX_RRR_0_OPCODE_Y1 = 0,
+  ADD_RRR_0_OPCODE_X0 = 3,
+  ADD_RRR_0_OPCODE_X1 = 3,
+  ADD_RRR_0_OPCODE_Y0 = 1,
+  ADD_RRR_0_OPCODE_Y1 = 1,
+  ANDI_IMM8_OPCODE_X0 = 3,
+  ANDI_IMM8_OPCODE_X1 = 3,
+  ANDI_OPCODE_Y0 = 2,
+  ANDI_OPCODE_Y1 = 3,
+  AND_RRR_0_OPCODE_X0 = 4,
+  AND_RRR_0_OPCODE_X1 = 4,
+  AND_RRR_5_OPCODE_Y0 = 0,
+  AND_RRR_5_OPCODE_Y1 = 0,
+  BEQZT_BRANCH_OPCODE_X1 = 16,
+  BEQZ_BRANCH_OPCODE_X1 = 17,
+  BFEXTS_BF_OPCODE_X0 = 4,
+  BFEXTU_BF_OPCODE_X0 = 5,
+  BFINS_BF_OPCODE_X0 = 6,
+  BF_OPCODE_X0 = 3,
+  BGEZT_BRANCH_OPCODE_X1 = 18,
+  BGEZ_BRANCH_OPCODE_X1 = 19,
+  BGTZT_BRANCH_OPCODE_X1 = 20,
+  BGTZ_BRANCH_OPCODE_X1 = 21,
+  BLBCT_BRANCH_OPCODE_X1 = 22,
+  BLBC_BRANCH_OPCODE_X1 = 23,
+  BLBST_BRANCH_OPCODE_X1 = 24,
+  BLBS_BRANCH_OPCODE_X1 = 25,
+  BLEZT_BRANCH_OPCODE_X1 = 26,
+  BLEZ_BRANCH_OPCODE_X1 = 27,
+  BLTZT_BRANCH_OPCODE_X1 = 28,
+  BLTZ_BRANCH_OPCODE_X1 = 29,
+  BNEZT_BRANCH_OPCODE_X1 = 30,
+  BNEZ_BRANCH_OPCODE_X1 = 31,
+  BRANCH_OPCODE_X1 = 2,
+  CMOVEQZ_RRR_0_OPCODE_X0 = 5,
+  CMOVEQZ_RRR_4_OPCODE_Y0 = 0,
+  CMOVNEZ_RRR_0_OPCODE_X0 = 6,
+  CMOVNEZ_RRR_4_OPCODE_Y0 = 1,
+  CMPEQI_IMM8_OPCODE_X0 = 4,
+  CMPEQI_IMM8_OPCODE_X1 = 4,
+  CMPEQI_OPCODE_Y0 = 3,
+  CMPEQI_OPCODE_Y1 = 4,
+  CMPEQ_RRR_0_OPCODE_X0 = 7,
+  CMPEQ_RRR_0_OPCODE_X1 = 5,
+  CMPEQ_RRR_3_OPCODE_Y0 = 0,
+  CMPEQ_RRR_3_OPCODE_Y1 = 2,
+  CMPEXCH4_RRR_0_OPCODE_X1 = 6,
+  CMPEXCH_RRR_0_OPCODE_X1 = 7,
+  CMPLES_RRR_0_OPCODE_X0 = 8,
+  CMPLES_RRR_0_OPCODE_X1 = 8,
+  CMPLES_RRR_2_OPCODE_Y0 = 0,
+  CMPLES_RRR_2_OPCODE_Y1 = 0,
+  CMPLEU_RRR_0_OPCODE_X0 = 9,
+  CMPLEU_RRR_0_OPCODE_X1 = 9,
+  CMPLEU_RRR_2_OPCODE_Y0 = 1,
+  CMPLEU_RRR_2_OPCODE_Y1 = 1,
+  CMPLTSI_IMM8_OPCODE_X0 = 5,
+  CMPLTSI_IMM8_OPCODE_X1 = 5,
+  CMPLTSI_OPCODE_Y0 = 4,
+  CMPLTSI_OPCODE_Y1 = 5,
+  CMPLTS_RRR_0_OPCODE_X0 = 10,
+  CMPLTS_RRR_0_OPCODE_X1 = 10,
+  CMPLTS_RRR_2_OPCODE_Y0 = 2,
+  CMPLTS_RRR_2_OPCODE_Y1 = 2,
+  CMPLTUI_IMM8_OPCODE_X0 = 6,
+  CMPLTUI_IMM8_OPCODE_X1 = 6,
+  CMPLTU_RRR_0_OPCODE_X0 = 11,
+  CMPLTU_RRR_0_OPCODE_X1 = 11,
+  CMPLTU_RRR_2_OPCODE_Y0 = 3,
+  CMPLTU_RRR_2_OPCODE_Y1 = 3,
+  CMPNE_RRR_0_OPCODE_X0 = 12,
+  CMPNE_RRR_0_OPCODE_X1 = 12,
+  CMPNE_RRR_3_OPCODE_Y0 = 1,
+  CMPNE_RRR_3_OPCODE_Y1 = 3,
+  CMULAF_RRR_0_OPCODE_X0 = 13,
+  CMULA_RRR_0_OPCODE_X0 = 14,
+  CMULFR_RRR_0_OPCODE_X0 = 15,
+  CMULF_RRR_0_OPCODE_X0 = 16,
+  CMULHR_RRR_0_OPCODE_X0 = 17,
+  CMULH_RRR_0_OPCODE_X0 = 18,
+  CMUL_RRR_0_OPCODE_X0 = 19,
+  CNTLZ_UNARY_OPCODE_X0 = 1,
+  CNTLZ_UNARY_OPCODE_Y0 = 1,
+  CNTTZ_UNARY_OPCODE_X0 = 2,
+  CNTTZ_UNARY_OPCODE_Y0 = 2,
+  CRC32_32_RRR_0_OPCODE_X0 = 20,
+  CRC32_8_RRR_0_OPCODE_X0 = 21,
+  DBLALIGN2_RRR_0_OPCODE_X0 = 22,
+  DBLALIGN2_RRR_0_OPCODE_X1 = 13,
+  DBLALIGN4_RRR_0_OPCODE_X0 = 23,
+  DBLALIGN4_RRR_0_OPCODE_X1 = 14,
+  DBLALIGN6_RRR_0_OPCODE_X0 = 24,
+  DBLALIGN6_RRR_0_OPCODE_X1 = 15,
+  DBLALIGN_RRR_0_OPCODE_X0 = 25,
+  DRAIN_UNARY_OPCODE_X1 = 1,
+  DTLBPR_UNARY_OPCODE_X1 = 2,
+  EXCH4_RRR_0_OPCODE_X1 = 16,
+  EXCH_RRR_0_OPCODE_X1 = 17,
+  FDOUBLE_ADDSUB_RRR_0_OPCODE_X0 = 26,
+  FDOUBLE_ADD_FLAGS_RRR_0_OPCODE_X0 = 27,
+  FDOUBLE_MUL_FLAGS_RRR_0_OPCODE_X0 = 28,
+  FDOUBLE_PACK1_RRR_0_OPCODE_X0 = 29,
+  FDOUBLE_PACK2_RRR_0_OPCODE_X0 = 30,
+  FDOUBLE_SUB_FLAGS_RRR_0_OPCODE_X0 = 31,
+  FDOUBLE_UNPACK_MAX_RRR_0_OPCODE_X0 = 32,
+  FDOUBLE_UNPACK_MIN_RRR_0_OPCODE_X0 = 33,
+  FETCHADD4_RRR_0_OPCODE_X1 = 18,
+  FETCHADDGEZ4_RRR_0_OPCODE_X1 = 19,
+  FETCHADDGEZ_RRR_0_OPCODE_X1 = 20,
+  FETCHADD_RRR_0_OPCODE_X1 = 21,
+  FETCHAND4_RRR_0_OPCODE_X1 = 22,
+  FETCHAND_RRR_0_OPCODE_X1 = 23,
+  FETCHOR4_RRR_0_OPCODE_X1 = 24,
+  FETCHOR_RRR_0_OPCODE_X1 = 25,
+  FINV_UNARY_OPCODE_X1 = 3,
+  FLUSHWB_UNARY_OPCODE_X1 = 4,
+  FLUSH_UNARY_OPCODE_X1 = 5,
+  FNOP_UNARY_OPCODE_X0 = 3,
+  FNOP_UNARY_OPCODE_X1 = 6,
+  FNOP_UNARY_OPCODE_Y0 = 3,
+  FNOP_UNARY_OPCODE_Y1 = 8,
+  FSINGLE_ADD1_RRR_0_OPCODE_X0 = 34,
+  FSINGLE_ADDSUB2_RRR_0_OPCODE_X0 = 35,
+  FSINGLE_MUL1_RRR_0_OPCODE_X0 = 36,
+  FSINGLE_MUL2_RRR_0_OPCODE_X0 = 37,
+  FSINGLE_PACK1_UNARY_OPCODE_X0 = 4,
+  FSINGLE_PACK1_UNARY_OPCODE_Y0 = 4,
+  FSINGLE_PACK2_RRR_0_OPCODE_X0 = 38,
+  FSINGLE_SUB1_RRR_0_OPCODE_X0 = 39,
+  ICOH_UNARY_OPCODE_X1 = 7,
+  ILL_UNARY_OPCODE_X1 = 8,
+  ILL_UNARY_OPCODE_Y1 = 9,
+  IMM8_OPCODE_X0 = 4,
+  IMM8_OPCODE_X1 = 3,
+  INV_UNARY_OPCODE_X1 = 9,
+  IRET_UNARY_OPCODE_X1 = 10,
+  JALRP_UNARY_OPCODE_X1 = 11,
+  JALRP_UNARY_OPCODE_Y1 = 10,
+  JALR_UNARY_OPCODE_X1 = 12,
+  JALR_UNARY_OPCODE_Y1 = 11,
+  JAL_JUMP_OPCODE_X1 = 0,
+  JRP_UNARY_OPCODE_X1 = 13,
+  JRP_UNARY_OPCODE_Y1 = 12,
+  JR_UNARY_OPCODE_X1 = 14,
+  JR_UNARY_OPCODE_Y1 = 13,
+  JUMP_OPCODE_X1 = 4,
+  J_JUMP_OPCODE_X1 = 1,
+  LD1S_ADD_IMM8_OPCODE_X1 = 7,
+  LD1S_OPCODE_Y2 = 0,
+  LD1S_UNARY_OPCODE_X1 = 15,
+  LD1U_ADD_IMM8_OPCODE_X1 = 8,
+  LD1U_OPCODE_Y2 = 1,
+  LD1U_UNARY_OPCODE_X1 = 16,
+  LD2S_ADD_IMM8_OPCODE_X1 = 9,
+  LD2S_OPCODE_Y2 = 2,
+  LD2S_UNARY_OPCODE_X1 = 17,
+  LD2U_ADD_IMM8_OPCODE_X1 = 10,
+  LD2U_OPCODE_Y2 = 3,
+  LD2U_UNARY_OPCODE_X1 = 18,
+  LD4S_ADD_IMM8_OPCODE_X1 = 11,
+  LD4S_OPCODE_Y2 = 1,
+  LD4S_UNARY_OPCODE_X1 = 19,
+  LD4U_ADD_IMM8_OPCODE_X1 = 12,
+  LD4U_OPCODE_Y2 = 2,
+  LD4U_UNARY_OPCODE_X1 = 20,
+  LDNA_UNARY_OPCODE_X1 = 21,
+  LDNT1S_ADD_IMM8_OPCODE_X1 = 13,
+  LDNT1S_UNARY_OPCODE_X1 = 22,
+  LDNT1U_ADD_IMM8_OPCODE_X1 = 14,
+  LDNT1U_UNARY_OPCODE_X1 = 23,
+  LDNT2S_ADD_IMM8_OPCODE_X1 = 15,
+  LDNT2S_UNARY_OPCODE_X1 = 24,
+  LDNT2U_ADD_IMM8_OPCODE_X1 = 16,
+  LDNT2U_UNARY_OPCODE_X1 = 25,
+  LDNT4S_ADD_IMM8_OPCODE_X1 = 17,
+  LDNT4S_UNARY_OPCODE_X1 = 26,
+  LDNT4U_ADD_IMM8_OPCODE_X1 = 18,
+  LDNT4U_UNARY_OPCODE_X1 = 27,
+  LDNT_ADD_IMM8_OPCODE_X1 = 19,
+  LDNT_UNARY_OPCODE_X1 = 28,
+  LD_ADD_IMM8_OPCODE_X1 = 20,
+  LD_OPCODE_Y2 = 3,
+  LD_UNARY_OPCODE_X1 = 29,
+  LNK_UNARY_OPCODE_X1 = 30,
+  LNK_UNARY_OPCODE_Y1 = 14,
+  LDNA_ADD_IMM8_OPCODE_X1 = 21,
+  MFSPR_IMM8_OPCODE_X1 = 22,
+  MF_UNARY_OPCODE_X1 = 31,
+  MM_BF_OPCODE_X0 = 7,
+  MNZ_RRR_0_OPCODE_X0 = 40,
+  MNZ_RRR_0_OPCODE_X1 = 26,
+  MNZ_RRR_4_OPCODE_Y0 = 2,
+  MNZ_RRR_4_OPCODE_Y1 = 2,
+  MODE_OPCODE_YA2 = 1,
+  MODE_OPCODE_YB2 = 2,
+  MODE_OPCODE_YC2 = 3,
+  MTSPR_IMM8_OPCODE_X1 = 23,
+  MULAX_RRR_0_OPCODE_X0 = 41,
+  MULAX_RRR_3_OPCODE_Y0 = 2,
+  MULA_HS_HS_RRR_0_OPCODE_X0 = 42,
+  MULA_HS_HS_RRR_9_OPCODE_Y0 = 0,
+  MULA_HS_HU_RRR_0_OPCODE_X0 = 43,
+  MULA_HS_LS_RRR_0_OPCODE_X0 = 44,
+  MULA_HS_LU_RRR_0_OPCODE_X0 = 45,
+  MULA_HU_HU_RRR_0_OPCODE_X0 = 46,
+  MULA_HU_HU_RRR_9_OPCODE_Y0 = 1,
+  MULA_HU_LS_RRR_0_OPCODE_X0 = 47,
+  MULA_HU_LU_RRR_0_OPCODE_X0 = 48,
+  MULA_LS_LS_RRR_0_OPCODE_X0 = 49,
+  MULA_LS_LS_RRR_9_OPCODE_Y0 = 2,
+  MULA_LS_LU_RRR_0_OPCODE_X0 = 50,
+  MULA_LU_LU_RRR_0_OPCODE_X0 = 51,
+  MULA_LU_LU_RRR_9_OPCODE_Y0 = 3,
+  MULX_RRR_0_OPCODE_X0 = 52,
+  MULX_RRR_3_OPCODE_Y0 = 3,
+  MUL_HS_HS_RRR_0_OPCODE_X0 = 53,
+  MUL_HS_HS_RRR_8_OPCODE_Y0 = 0,
+  MUL_HS_HU_RRR_0_OPCODE_X0 = 54,
+  MUL_HS_LS_RRR_0_OPCODE_X0 = 55,
+  MUL_HS_LU_RRR_0_OPCODE_X0 = 56,
+  MUL_HU_HU_RRR_0_OPCODE_X0 = 57,
+  MUL_HU_HU_RRR_8_OPCODE_Y0 = 1,
+  MUL_HU_LS_RRR_0_OPCODE_X0 = 58,
+  MUL_HU_LU_RRR_0_OPCODE_X0 = 59,
+  MUL_LS_LS_RRR_0_OPCODE_X0 = 60,
+  MUL_LS_LS_RRR_8_OPCODE_Y0 = 2,
+  MUL_LS_LU_RRR_0_OPCODE_X0 = 61,
+  MUL_LU_LU_RRR_0_OPCODE_X0 = 62,
+  MUL_LU_LU_RRR_8_OPCODE_Y0 = 3,
+  MZ_RRR_0_OPCODE_X0 = 63,
+  MZ_RRR_0_OPCODE_X1 = 27,
+  MZ_RRR_4_OPCODE_Y0 = 3,
+  MZ_RRR_4_OPCODE_Y1 = 3,
+  NAP_UNARY_OPCODE_X1 = 32,
+  NOP_UNARY_OPCODE_X0 = 5,
+  NOP_UNARY_OPCODE_X1 = 33,
+  NOP_UNARY_OPCODE_Y0 = 5,
+  NOP_UNARY_OPCODE_Y1 = 15,
+  NOR_RRR_0_OPCODE_X0 = 64,
+  NOR_RRR_0_OPCODE_X1 = 28,
+  NOR_RRR_5_OPCODE_Y0 = 1,
+  NOR_RRR_5_OPCODE_Y1 = 1,
+  ORI_IMM8_OPCODE_X0 = 7,
+  ORI_IMM8_OPCODE_X1 = 24,
+  OR_RRR_0_OPCODE_X0 = 65,
+  OR_RRR_0_OPCODE_X1 = 29,
+  OR_RRR_5_OPCODE_Y0 = 2,
+  OR_RRR_5_OPCODE_Y1 = 2,
+  PCNT_UNARY_OPCODE_X0 = 6,
+  PCNT_UNARY_OPCODE_Y0 = 6,
+  REVBITS_UNARY_OPCODE_X0 = 7,
+  REVBITS_UNARY_OPCODE_Y0 = 7,
+  REVBYTES_UNARY_OPCODE_X0 = 8,
+  REVBYTES_UNARY_OPCODE_Y0 = 8,
+  ROTLI_SHIFT_OPCODE_X0 = 1,
+  ROTLI_SHIFT_OPCODE_X1 = 1,
+  ROTLI_SHIFT_OPCODE_Y0 = 0,
+  ROTLI_SHIFT_OPCODE_Y1 = 0,
+  ROTL_RRR_0_OPCODE_X0 = 66,
+  ROTL_RRR_0_OPCODE_X1 = 30,
+  ROTL_RRR_6_OPCODE_Y0 = 0,
+  ROTL_RRR_6_OPCODE_Y1 = 0,
+  RRR_0_OPCODE_X0 = 5,
+  RRR_0_OPCODE_X1 = 5,
+  RRR_0_OPCODE_Y0 = 5,
+  RRR_0_OPCODE_Y1 = 6,
+  RRR_1_OPCODE_Y0 = 6,
+  RRR_1_OPCODE_Y1 = 7,
+  RRR_2_OPCODE_Y0 = 7,
+  RRR_2_OPCODE_Y1 = 8,
+  RRR_3_OPCODE_Y0 = 8,
+  RRR_3_OPCODE_Y1 = 9,
+  RRR_4_OPCODE_Y0 = 9,
+  RRR_4_OPCODE_Y1 = 10,
+  RRR_5_OPCODE_Y0 = 10,
+  RRR_5_OPCODE_Y1 = 11,
+  RRR_6_OPCODE_Y0 = 11,
+  RRR_6_OPCODE_Y1 = 12,
+  RRR_7_OPCODE_Y0 = 12,
+  RRR_7_OPCODE_Y1 = 13,
+  RRR_8_OPCODE_Y0 = 13,
+  RRR_9_OPCODE_Y0 = 14,
+  SHIFT_OPCODE_X0 = 6,
+  SHIFT_OPCODE_X1 = 6,
+  SHIFT_OPCODE_Y0 = 15,
+  SHIFT_OPCODE_Y1 = 14,
+  SHL16INSLI_OPCODE_X0 = 7,
+  SHL16INSLI_OPCODE_X1 = 7,
+  SHL1ADDX_RRR_0_OPCODE_X0 = 67,
+  SHL1ADDX_RRR_0_OPCODE_X1 = 31,
+  SHL1ADDX_RRR_7_OPCODE_Y0 = 1,
+  SHL1ADDX_RRR_7_OPCODE_Y1 = 1,
+  SHL1ADD_RRR_0_OPCODE_X0 = 68,
+  SHL1ADD_RRR_0_OPCODE_X1 = 32,
+  SHL1ADD_RRR_1_OPCODE_Y0 = 0,
+  SHL1ADD_RRR_1_OPCODE_Y1 = 0,
+  SHL2ADDX_RRR_0_OPCODE_X0 = 69,
+  SHL2ADDX_RRR_0_OPCODE_X1 = 33,
+  SHL2ADDX_RRR_7_OPCODE_Y0 = 2,
+  SHL2ADDX_RRR_7_OPCODE_Y1 = 2,
+  SHL2ADD_RRR_0_OPCODE_X0 = 70,
+  SHL2ADD_RRR_0_OPCODE_X1 = 34,
+  SHL2ADD_RRR_1_OPCODE_Y0 = 1,
+  SHL2ADD_RRR_1_OPCODE_Y1 = 1,
+  SHL3ADDX_RRR_0_OPCODE_X0 = 71,
+  SHL3ADDX_RRR_0_OPCODE_X1 = 35,
+  SHL3ADDX_RRR_7_OPCODE_Y0 = 3,
+  SHL3ADDX_RRR_7_OPCODE_Y1 = 3,
+  SHL3ADD_RRR_0_OPCODE_X0 = 72,
+  SHL3ADD_RRR_0_OPCODE_X1 = 36,
+  SHL3ADD_RRR_1_OPCODE_Y0 = 2,
+  SHL3ADD_RRR_1_OPCODE_Y1 = 2,
+  SHLI_SHIFT_OPCODE_X0 = 2,
+  SHLI_SHIFT_OPCODE_X1 = 2,
+  SHLI_SHIFT_OPCODE_Y0 = 1,
+  SHLI_SHIFT_OPCODE_Y1 = 1,
+  SHLXI_SHIFT_OPCODE_X0 = 3,
+  SHLXI_SHIFT_OPCODE_X1 = 3,
+  SHLX_RRR_0_OPCODE_X0 = 73,
+  SHLX_RRR_0_OPCODE_X1 = 37,
+  SHL_RRR_0_OPCODE_X0 = 74,
+  SHL_RRR_0_OPCODE_X1 = 38,
+  SHL_RRR_6_OPCODE_Y0 = 1,
+  SHL_RRR_6_OPCODE_Y1 = 1,
+  SHRSI_SHIFT_OPCODE_X0 = 4,
+  SHRSI_SHIFT_OPCODE_X1 = 4,
+  SHRSI_SHIFT_OPCODE_Y0 = 2,
+  SHRSI_SHIFT_OPCODE_Y1 = 2,
+  SHRS_RRR_0_OPCODE_X0 = 75,
+  SHRS_RRR_0_OPCODE_X1 = 39,
+  SHRS_RRR_6_OPCODE_Y0 = 2,
+  SHRS_RRR_6_OPCODE_Y1 = 2,
+  SHRUI_SHIFT_OPCODE_X0 = 5,
+  SHRUI_SHIFT_OPCODE_X1 = 5,
+  SHRUI_SHIFT_OPCODE_Y0 = 3,
+  SHRUI_SHIFT_OPCODE_Y1 = 3,
+  SHRUXI_SHIFT_OPCODE_X0 = 6,
+  SHRUXI_SHIFT_OPCODE_X1 = 6,
+  SHRUX_RRR_0_OPCODE_X0 = 76,
+  SHRUX_RRR_0_OPCODE_X1 = 40,
+  SHRU_RRR_0_OPCODE_X0 = 77,
+  SHRU_RRR_0_OPCODE_X1 = 41,
+  SHRU_RRR_6_OPCODE_Y0 = 3,
+  SHRU_RRR_6_OPCODE_Y1 = 3,
+  SHUFFLEBYTES_RRR_0_OPCODE_X0 = 78,
+  ST1_ADD_IMM8_OPCODE_X1 = 25,
+  ST1_OPCODE_Y2 = 0,
+  ST1_RRR_0_OPCODE_X1 = 42,
+  ST2_ADD_IMM8_OPCODE_X1 = 26,
+  ST2_OPCODE_Y2 = 1,
+  ST2_RRR_0_OPCODE_X1 = 43,
+  ST4_ADD_IMM8_OPCODE_X1 = 27,
+  ST4_OPCODE_Y2 = 2,
+  ST4_RRR_0_OPCODE_X1 = 44,
+  STNT1_ADD_IMM8_OPCODE_X1 = 28,
+  STNT1_RRR_0_OPCODE_X1 = 45,
+  STNT2_ADD_IMM8_OPCODE_X1 = 29,
+  STNT2_RRR_0_OPCODE_X1 = 46,
+  STNT4_ADD_IMM8_OPCODE_X1 = 30,
+  STNT4_RRR_0_OPCODE_X1 = 47,
+  STNT_ADD_IMM8_OPCODE_X1 = 31,
+  STNT_RRR_0_OPCODE_X1 = 48,
+  ST_ADD_IMM8_OPCODE_X1 = 32,
+  ST_OPCODE_Y2 = 3,
+  ST_RRR_0_OPCODE_X1 = 49,
+  SUBXSC_RRR_0_OPCODE_X0 = 79,
+  SUBXSC_RRR_0_OPCODE_X1 = 50,
+  SUBX_RRR_0_OPCODE_X0 = 80,
+  SUBX_RRR_0_OPCODE_X1 = 51,
+  SUBX_RRR_0_OPCODE_Y0 = 2,
+  SUBX_RRR_0_OPCODE_Y1 = 2,
+  SUB_RRR_0_OPCODE_X0 = 81,
+  SUB_RRR_0_OPCODE_X1 = 52,
+  SUB_RRR_0_OPCODE_Y0 = 3,
+  SUB_RRR_0_OPCODE_Y1 = 3,
+  SWINT0_UNARY_OPCODE_X1 = 34,
+  SWINT1_UNARY_OPCODE_X1 = 35,
+  SWINT2_UNARY_OPCODE_X1 = 36,
+  SWINT3_UNARY_OPCODE_X1 = 37,
+  TBLIDXB0_UNARY_OPCODE_X0 = 9,
+  TBLIDXB0_UNARY_OPCODE_Y0 = 9,
+  TBLIDXB1_UNARY_OPCODE_X0 = 10,
+  TBLIDXB1_UNARY_OPCODE_Y0 = 10,
+  TBLIDXB2_UNARY_OPCODE_X0 = 11,
+  TBLIDXB2_UNARY_OPCODE_Y0 = 11,
+  TBLIDXB3_UNARY_OPCODE_X0 = 12,
+  TBLIDXB3_UNARY_OPCODE_Y0 = 12,
+  UNARY_RRR_0_OPCODE_X0 = 82,
+  UNARY_RRR_0_OPCODE_X1 = 53,
+  UNARY_RRR_1_OPCODE_Y0 = 3,
+  UNARY_RRR_1_OPCODE_Y1 = 3,
+  V1ADDI_IMM8_OPCODE_X0 = 8,
+  V1ADDI_IMM8_OPCODE_X1 = 33,
+  V1ADDUC_RRR_0_OPCODE_X0 = 83,
+  V1ADDUC_RRR_0_OPCODE_X1 = 54,
+  V1ADD_RRR_0_OPCODE_X0 = 84,
+  V1ADD_RRR_0_OPCODE_X1 = 55,
+  V1ADIFFU_RRR_0_OPCODE_X0 = 85,
+  V1AVGU_RRR_0_OPCODE_X0 = 86,
+  V1CMPEQI_IMM8_OPCODE_X0 = 9,
+  V1CMPEQI_IMM8_OPCODE_X1 = 34,
+  V1CMPEQ_RRR_0_OPCODE_X0 = 87,
+  V1CMPEQ_RRR_0_OPCODE_X1 = 56,
+  V1CMPLES_RRR_0_OPCODE_X0 = 88,
+  V1CMPLES_RRR_0_OPCODE_X1 = 57,
+  V1CMPLEU_RRR_0_OPCODE_X0 = 89,
+  V1CMPLEU_RRR_0_OPCODE_X1 = 58,
+  V1CMPLTSI_IMM8_OPCODE_X0 = 10,
+  V1CMPLTSI_IMM8_OPCODE_X1 = 35,
+  V1CMPLTS_RRR_0_OPCODE_X0 = 90,
+  V1CMPLTS_RRR_0_OPCODE_X1 = 59,
+  V1CMPLTUI_IMM8_OPCODE_X0 = 11,
+  V1CMPLTUI_IMM8_OPCODE_X1 = 36,
+  V1CMPLTU_RRR_0_OPCODE_X0 = 91,
+  V1CMPLTU_RRR_0_OPCODE_X1 = 60,
+  V1CMPNE_RRR_0_OPCODE_X0 = 92,
+  V1CMPNE_RRR_0_OPCODE_X1 = 61,
+  V1DDOTPUA_RRR_0_OPCODE_X0 = 161,
+  V1DDOTPUSA_RRR_0_OPCODE_X0 = 93,
+  V1DDOTPUS_RRR_0_OPCODE_X0 = 94,
+  V1DDOTPU_RRR_0_OPCODE_X0 = 162,
+  V1DOTPA_RRR_0_OPCODE_X0 = 95,
+  V1DOTPUA_RRR_0_OPCODE_X0 = 163,
+  V1DOTPUSA_RRR_0_OPCODE_X0 = 96,
+  V1DOTPUS_RRR_0_OPCODE_X0 = 97,
+  V1DOTPU_RRR_0_OPCODE_X0 = 164,
+  V1DOTP_RRR_0_OPCODE_X0 = 98,
+  V1INT_H_RRR_0_OPCODE_X0 = 99,
+  V1INT_H_RRR_0_OPCODE_X1 = 62,
+  V1INT_L_RRR_0_OPCODE_X0 = 100,
+  V1INT_L_RRR_0_OPCODE_X1 = 63,
+  V1MAXUI_IMM8_OPCODE_X0 = 12,
+  V1MAXUI_IMM8_OPCODE_X1 = 37,
+  V1MAXU_RRR_0_OPCODE_X0 = 101,
+  V1MAXU_RRR_0_OPCODE_X1 = 64,
+  V1MINUI_IMM8_OPCODE_X0 = 13,
+  V1MINUI_IMM8_OPCODE_X1 = 38,
+  V1MINU_RRR_0_OPCODE_X0 = 102,
+  V1MINU_RRR_0_OPCODE_X1 = 65,
+  V1MNZ_RRR_0_OPCODE_X0 = 103,
+  V1MNZ_RRR_0_OPCODE_X1 = 66,
+  V1MULTU_RRR_0_OPCODE_X0 = 104,
+  V1MULUS_RRR_0_OPCODE_X0 = 105,
+  V1MULU_RRR_0_OPCODE_X0 = 106,
+  V1MZ_RRR_0_OPCODE_X0 = 107,
+  V1MZ_RRR_0_OPCODE_X1 = 67,
+  V1SADAU_RRR_0_OPCODE_X0 = 108,
+  V1SADU_RRR_0_OPCODE_X0 = 109,
+  V1SHLI_SHIFT_OPCODE_X0 = 7,
+  V1SHLI_SHIFT_OPCODE_X1 = 7,
+  V1SHL_RRR_0_OPCODE_X0 = 110,
+  V1SHL_RRR_0_OPCODE_X1 = 68,
+  V1SHRSI_SHIFT_OPCODE_X0 = 8,
+  V1SHRSI_SHIFT_OPCODE_X1 = 8,
+  V1SHRS_RRR_0_OPCODE_X0 = 111,
+  V1SHRS_RRR_0_OPCODE_X1 = 69,
+  V1SHRUI_SHIFT_OPCODE_X0 = 9,
+  V1SHRUI_SHIFT_OPCODE_X1 = 9,
+  V1SHRU_RRR_0_OPCODE_X0 = 112,
+  V1SHRU_RRR_0_OPCODE_X1 = 70,
+  V1SUBUC_RRR_0_OPCODE_X0 = 113,
+  V1SUBUC_RRR_0_OPCODE_X1 = 71,
+  V1SUB_RRR_0_OPCODE_X0 = 114,
+  V1SUB_RRR_0_OPCODE_X1 = 72,
+  V2ADDI_IMM8_OPCODE_X0 = 14,
+  V2ADDI_IMM8_OPCODE_X1 = 39,
+  V2ADDSC_RRR_0_OPCODE_X0 = 115,
+  V2ADDSC_RRR_0_OPCODE_X1 = 73,
+  V2ADD_RRR_0_OPCODE_X0 = 116,
+  V2ADD_RRR_0_OPCODE_X1 = 74,
+  V2ADIFFS_RRR_0_OPCODE_X0 = 117,
+  V2AVGS_RRR_0_OPCODE_X0 = 118,
+  V2CMPEQI_IMM8_OPCODE_X0 = 15,
+  V2CMPEQI_IMM8_OPCODE_X1 = 40,
+  V2CMPEQ_RRR_0_OPCODE_X0 = 119,
+  V2CMPEQ_RRR_0_OPCODE_X1 = 75,
+  V2CMPLES_RRR_0_OPCODE_X0 = 120,
+  V2CMPLES_RRR_0_OPCODE_X1 = 76,
+  V2CMPLEU_RRR_0_OPCODE_X0 = 121,
+  V2CMPLEU_RRR_0_OPCODE_X1 = 77,
+  V2CMPLTSI_IMM8_OPCODE_X0 = 16,
+  V2CMPLTSI_IMM8_OPCODE_X1 = 41,
+  V2CMPLTS_RRR_0_OPCODE_X0 = 122,
+  V2CMPLTS_RRR_0_OPCODE_X1 = 78,
+  V2CMPLTUI_IMM8_OPCODE_X0 = 17,
+  V2CMPLTUI_IMM8_OPCODE_X1 = 42,
+  V2CMPLTU_RRR_0_OPCODE_X0 = 123,
+  V2CMPLTU_RRR_0_OPCODE_X1 = 79,
+  V2CMPNE_RRR_0_OPCODE_X0 = 124,
+  V2CMPNE_RRR_0_OPCODE_X1 = 80,
+  V2DOTPA_RRR_0_OPCODE_X0 = 125,
+  V2DOTP_RRR_0_OPCODE_X0 = 126,
+  V2INT_H_RRR_0_OPCODE_X0 = 127,
+  V2INT_H_RRR_0_OPCODE_X1 = 81,
+  V2INT_L_RRR_0_OPCODE_X0 = 128,
+  V2INT_L_RRR_0_OPCODE_X1 = 82,
+  V2MAXSI_IMM8_OPCODE_X0 = 18,
+  V2MAXSI_IMM8_OPCODE_X1 = 43,
+  V2MAXS_RRR_0_OPCODE_X0 = 129,
+  V2MAXS_RRR_0_OPCODE_X1 = 83,
+  V2MINSI_IMM8_OPCODE_X0 = 19,
+  V2MINSI_IMM8_OPCODE_X1 = 44,
+  V2MINS_RRR_0_OPCODE_X0 = 130,
+  V2MINS_RRR_0_OPCODE_X1 = 84,
+  V2MNZ_RRR_0_OPCODE_X0 = 131,
+  V2MNZ_RRR_0_OPCODE_X1 = 85,
+  V2MULFSC_RRR_0_OPCODE_X0 = 132,
+  V2MULS_RRR_0_OPCODE_X0 = 133,
+  V2MULTS_RRR_0_OPCODE_X0 = 134,
+  V2MZ_RRR_0_OPCODE_X0 = 135,
+  V2MZ_RRR_0_OPCODE_X1 = 86,
+  V2PACKH_RRR_0_OPCODE_X0 = 136,
+  V2PACKH_RRR_0_OPCODE_X1 = 87,
+  V2PACKL_RRR_0_OPCODE_X0 = 137,
+  V2PACKL_RRR_0_OPCODE_X1 = 88,
+  V2PACKUC_RRR_0_OPCODE_X0 = 138,
+  V2PACKUC_RRR_0_OPCODE_X1 = 89,
+  V2SADAS_RRR_0_OPCODE_X0 = 139,
+  V2SADAU_RRR_0_OPCODE_X0 = 140,
+  V2SADS_RRR_0_OPCODE_X0 = 141,
+  V2SADU_RRR_0_OPCODE_X0 = 142,
+  V2SHLI_SHIFT_OPCODE_X0 = 10,
+  V2SHLI_SHIFT_OPCODE_X1 = 10,
+  V2SHLSC_RRR_0_OPCODE_X0 = 143,
+  V2SHLSC_RRR_0_OPCODE_X1 = 90,
+  V2SHL_RRR_0_OPCODE_X0 = 144,
+  V2SHL_RRR_0_OPCODE_X1 = 91,
+  V2SHRSI_SHIFT_OPCODE_X0 = 11,
+  V2SHRSI_SHIFT_OPCODE_X1 = 11,
+  V2SHRS_RRR_0_OPCODE_X0 = 145,
+  V2SHRS_RRR_0_OPCODE_X1 = 92,
+  V2SHRUI_SHIFT_OPCODE_X0 = 12,
+  V2SHRUI_SHIFT_OPCODE_X1 = 12,
+  V2SHRU_RRR_0_OPCODE_X0 = 146,
+  V2SHRU_RRR_0_OPCODE_X1 = 93,
+  V2SUBSC_RRR_0_OPCODE_X0 = 147,
+  V2SUBSC_RRR_0_OPCODE_X1 = 94,
+  V2SUB_RRR_0_OPCODE_X0 = 148,
+  V2SUB_RRR_0_OPCODE_X1 = 95,
+  V4ADDSC_RRR_0_OPCODE_X0 = 149,
+  V4ADDSC_RRR_0_OPCODE_X1 = 96,
+  V4ADD_RRR_0_OPCODE_X0 = 150,
+  V4ADD_RRR_0_OPCODE_X1 = 97,
+  V4INT_H_RRR_0_OPCODE_X0 = 151,
+  V4INT_H_RRR_0_OPCODE_X1 = 98,
+  V4INT_L_RRR_0_OPCODE_X0 = 152,
+  V4INT_L_RRR_0_OPCODE_X1 = 99,
+  V4PACKSC_RRR_0_OPCODE_X0 = 153,
+  V4PACKSC_RRR_0_OPCODE_X1 = 100,
+  V4SHLSC_RRR_0_OPCODE_X0 = 154,
+  V4SHLSC_RRR_0_OPCODE_X1 = 101,
+  V4SHL_RRR_0_OPCODE_X0 = 155,
+  V4SHL_RRR_0_OPCODE_X1 = 102,
+  V4SHRS_RRR_0_OPCODE_X0 = 156,
+  V4SHRS_RRR_0_OPCODE_X1 = 103,
+  V4SHRU_RRR_0_OPCODE_X0 = 157,
+  V4SHRU_RRR_0_OPCODE_X1 = 104,
+  V4SUBSC_RRR_0_OPCODE_X0 = 158,
+  V4SUBSC_RRR_0_OPCODE_X1 = 105,
+  V4SUB_RRR_0_OPCODE_X0 = 159,
+  V4SUB_RRR_0_OPCODE_X1 = 106,
+  WH64_UNARY_OPCODE_X1 = 38,
+  XORI_IMM8_OPCODE_X0 = 20,
+  XORI_IMM8_OPCODE_X1 = 45,
+  XOR_RRR_0_OPCODE_X0 = 160,
+  XOR_RRR_0_OPCODE_X1 = 107,
+  XOR_RRR_5_OPCODE_Y0 = 3,
+  XOR_RRR_5_OPCODE_Y1 = 3
+};
+
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* OPCODE_TILEGX_H */
diff --git a/target-tilegx/simd_helper.c b/target-tilegx/simd_helper.c
new file mode 100644
index 0000000..2d40ddb
--- /dev/null
+++ b/target-tilegx/simd_helper.c
@@ -0,0 +1,166 @@
+/*
+ * QEMU TILE-Gx helpers
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu-common.h"
+#include "exec/helper-proto.h"
+
+
+/* Broadcast a value to all elements of a vector.  */
+#define V1(X)      (((X) & 0xff) * 0x0101010101010101ull)
+#define V2(X)      (((X) & 0xffff) * 0x0001000100010001ull)
+
+
+uint64_t helper_v1multu(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 64; i += 8) {
+        unsigned ae = extract64(a, i, 8);
+        unsigned be = extract64(b, i, 8);
+        r = deposit64(r, i, 8, ae * be);
+    }
+    return r;
+}
+
+uint64_t helper_v2mults(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    /* While the instruction talks about signed inputs, with a
+       truncated result the sign of the inputs doesn't matter.  */
+    for (i = 0; i < 64; i += 16) {
+        unsigned ae = extract64(a, i, 16);
+        unsigned be = extract64(b, i, 16);
+        r = deposit64(r, i, 16, ae * be);
+    }
+    return r;
+}
+
+uint64_t helper_v1shl(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 7;
+    m = V1(0xff >> b);
+    return (a & m) << b;
+}
+
+uint64_t helper_v2shl(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 15;
+    m = V2(0xffff >> b);
+    return (a & m) << b;
+}
+
+uint64_t helper_v1shru(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 7;
+    m = V1(0xff << b);
+    return (a & m) >> b;
+}
+
+uint64_t helper_v2shru(uint64_t a, uint64_t b)
+{
+    uint64_t m;
+
+    b &= 15;
+    m = V2(0xffff << b);
+    return (a & m) >> b;
+}
+
+uint64_t helper_v1shrs(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    b &= 7;
+    for (i = 0; i < 64; i += 8) {
+        r = deposit64(r, i, 8, sextract64(a, i + b, 8 - b));
+    }
+    return r;
+}
+
+uint64_t helper_v2shrs(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    b &= 15;
+    for (i = 0; i < 64; i += 16) {
+        r = deposit64(r, i, 16, sextract64(a, i + b, 16 - b));
+    }
+    return r;
+}
+
+uint64_t helper_v1int_h(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 8) {
+        r = deposit64(r, 2 * i + 8, 8, extract64(a, i + 32, 8));
+        r = deposit64(r, 2 * i, 8, extract64(b, i + 32, 8));
+    }
+    return r;
+}
+
+uint64_t helper_v1int_l(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 8) {
+        r = deposit64(r, 2 * i + 8, 8, extract64(a, i, 8));
+        r = deposit64(r, 2 * i, 8, extract64(b, i, 8));
+    }
+    return r;
+}
+
+uint64_t helper_v2int_h(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 16) {
+        r = deposit64(r, 2 * i + 16, 16, extract64(a, i + 32, 16));
+        r = deposit64(r, 2 * i, 16, extract64(b, i + 32, 16));
+    }
+    return r;
+}
+
+uint64_t helper_v2int_l(uint64_t a, uint64_t b)
+{
+    uint64_t r = 0;
+    int i;
+
+    for (i = 0; i < 32; i += 16) {
+        r = deposit64(r, 2 * i + 16, 16, extract64(a, i, 16));
+        r = deposit64(r, 2 * i, 16, extract64(b, i, 16));
+    }
+    return r;
+}
diff --git a/target-tilegx/spr_def_64.h b/target-tilegx/spr_def_64.h
new file mode 100644
index 0000000..67a6c17
--- /dev/null
+++ b/target-tilegx/spr_def_64.h
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2011 Tilera Corporation. All Rights Reserved.
+ *
+ *   This program is free software; you can redistribute it and/or
+ *   modify it under the terms of the GNU General Public License
+ *   as published by the Free Software Foundation, version 2.
+ *
+ *   This program is distributed in the hope that it will be useful, but
+ *   WITHOUT ANY WARRANTY; without even the implied warranty of
+ *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ *   NON INFRINGEMENT.  See the GNU General Public License for
+ *   more details.
+ */
+
+#ifndef __DOXYGEN__
+
+#ifndef __ARCH_SPR_DEF_64_H__
+#define __ARCH_SPR_DEF_64_H__
+
+#define SPR_AUX_PERF_COUNT_0 0x2105
+#define SPR_AUX_PERF_COUNT_1 0x2106
+#define SPR_AUX_PERF_COUNT_CTL 0x2107
+#define SPR_AUX_PERF_COUNT_STS 0x2108
+#define SPR_CMPEXCH_VALUE 0x2780
+#define SPR_CYCLE 0x2781
+#define SPR_DONE 0x2705
+#define SPR_DSTREAM_PF 0x2706
+#define SPR_EVENT_BEGIN 0x2782
+#define SPR_EVENT_END 0x2783
+#define SPR_EX_CONTEXT_0_0 0x2580
+#define SPR_EX_CONTEXT_0_1 0x2581
+#define SPR_EX_CONTEXT_0_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_0_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_0_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_0_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_0_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_0_1__ICS_MASK  0x4
+#define SPR_EX_CONTEXT_1_0 0x2480
+#define SPR_EX_CONTEXT_1_1 0x2481
+#define SPR_EX_CONTEXT_1_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_1_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_1_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_1_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_1_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_1_1__ICS_MASK  0x4
+#define SPR_EX_CONTEXT_2_0 0x2380
+#define SPR_EX_CONTEXT_2_1 0x2381
+#define SPR_EX_CONTEXT_2_1__PL_SHIFT 0
+#define SPR_EX_CONTEXT_2_1__PL_RMASK 0x3
+#define SPR_EX_CONTEXT_2_1__PL_MASK  0x3
+#define SPR_EX_CONTEXT_2_1__ICS_SHIFT 2
+#define SPR_EX_CONTEXT_2_1__ICS_RMASK 0x1
+#define SPR_EX_CONTEXT_2_1__ICS_MASK  0x4
+#define SPR_FAIL 0x2707
+#define SPR_IDN_AVAIL_EN 0x1a05
+#define SPR_IDN_DATA_AVAIL 0x0a80
+#define SPR_IDN_DEADLOCK_TIMEOUT 0x1806
+#define SPR_IDN_DEMUX_COUNT_0 0x0a05
+#define SPR_IDN_DEMUX_COUNT_1 0x0a06
+#define SPR_IDN_DIRECTION_PROTECT 0x1405
+#define SPR_IDN_PENDING 0x0a08
+#define SPR_ILL_TRANS_REASON__I_STREAM_VA_RMASK 0x1
+#define SPR_INTCTRL_0_STATUS 0x2505
+#define SPR_INTCTRL_1_STATUS 0x2405
+#define SPR_INTCTRL_2_STATUS 0x2305
+#define SPR_INTERRUPT_CRITICAL_SECTION 0x2708
+#define SPR_INTERRUPT_MASK_0 0x2506
+#define SPR_INTERRUPT_MASK_1 0x2406
+#define SPR_INTERRUPT_MASK_2 0x2306
+#define SPR_INTERRUPT_MASK_RESET_0 0x2507
+#define SPR_INTERRUPT_MASK_RESET_1 0x2407
+#define SPR_INTERRUPT_MASK_RESET_2 0x2307
+#define SPR_INTERRUPT_MASK_SET_0 0x2508
+#define SPR_INTERRUPT_MASK_SET_1 0x2408
+#define SPR_INTERRUPT_MASK_SET_2 0x2308
+#define SPR_INTERRUPT_VECTOR_BASE_0 0x2509
+#define SPR_INTERRUPT_VECTOR_BASE_1 0x2409
+#define SPR_INTERRUPT_VECTOR_BASE_2 0x2309
+#define SPR_INTERRUPT_VECTOR_BASE_3 0x2209
+#define SPR_IPI_EVENT_0 0x1f05
+#define SPR_IPI_EVENT_1 0x1e05
+#define SPR_IPI_EVENT_2 0x1d05
+#define SPR_IPI_EVENT_RESET_0 0x1f06
+#define SPR_IPI_EVENT_RESET_1 0x1e06
+#define SPR_IPI_EVENT_RESET_2 0x1d06
+#define SPR_IPI_EVENT_SET_0 0x1f07
+#define SPR_IPI_EVENT_SET_1 0x1e07
+#define SPR_IPI_EVENT_SET_2 0x1d07
+#define SPR_IPI_MASK_0 0x1f08
+#define SPR_IPI_MASK_1 0x1e08
+#define SPR_IPI_MASK_2 0x1d08
+#define SPR_IPI_MASK_RESET_0 0x1f09
+#define SPR_IPI_MASK_RESET_1 0x1e09
+#define SPR_IPI_MASK_RESET_2 0x1d09
+#define SPR_IPI_MASK_SET_0 0x1f0a
+#define SPR_IPI_MASK_SET_1 0x1e0a
+#define SPR_IPI_MASK_SET_2 0x1d0a
+#define SPR_MPL_AUX_PERF_COUNT_SET_0 0x2100
+#define SPR_MPL_AUX_PERF_COUNT_SET_1 0x2101
+#define SPR_MPL_AUX_PERF_COUNT_SET_2 0x2102
+#define SPR_MPL_AUX_TILE_TIMER_SET_0 0x1700
+#define SPR_MPL_AUX_TILE_TIMER_SET_1 0x1701
+#define SPR_MPL_AUX_TILE_TIMER_SET_2 0x1702
+#define SPR_MPL_IDN_ACCESS_SET_0 0x0a00
+#define SPR_MPL_IDN_ACCESS_SET_1 0x0a01
+#define SPR_MPL_IDN_ACCESS_SET_2 0x0a02
+#define SPR_MPL_IDN_AVAIL_SET_0 0x1a00
+#define SPR_MPL_IDN_AVAIL_SET_1 0x1a01
+#define SPR_MPL_IDN_AVAIL_SET_2 0x1a02
+#define SPR_MPL_IDN_COMPLETE_SET_0 0x0500
+#define SPR_MPL_IDN_COMPLETE_SET_1 0x0501
+#define SPR_MPL_IDN_COMPLETE_SET_2 0x0502
+#define SPR_MPL_IDN_FIREWALL_SET_0 0x1400
+#define SPR_MPL_IDN_FIREWALL_SET_1 0x1401
+#define SPR_MPL_IDN_FIREWALL_SET_2 0x1402
+#define SPR_MPL_IDN_TIMER_SET_0 0x1800
+#define SPR_MPL_IDN_TIMER_SET_1 0x1801
+#define SPR_MPL_IDN_TIMER_SET_2 0x1802
+#define SPR_MPL_INTCTRL_0_SET_0 0x2500
+#define SPR_MPL_INTCTRL_0_SET_1 0x2501
+#define SPR_MPL_INTCTRL_0_SET_2 0x2502
+#define SPR_MPL_INTCTRL_1_SET_0 0x2400
+#define SPR_MPL_INTCTRL_1_SET_1 0x2401
+#define SPR_MPL_INTCTRL_1_SET_2 0x2402
+#define SPR_MPL_INTCTRL_2_SET_0 0x2300
+#define SPR_MPL_INTCTRL_2_SET_1 0x2301
+#define SPR_MPL_INTCTRL_2_SET_2 0x2302
+#define SPR_MPL_IPI_0 0x1f04
+#define SPR_MPL_IPI_0_SET_0 0x1f00
+#define SPR_MPL_IPI_0_SET_1 0x1f01
+#define SPR_MPL_IPI_0_SET_2 0x1f02
+#define SPR_MPL_IPI_1 0x1e04
+#define SPR_MPL_IPI_1_SET_0 0x1e00
+#define SPR_MPL_IPI_1_SET_1 0x1e01
+#define SPR_MPL_IPI_1_SET_2 0x1e02
+#define SPR_MPL_IPI_2 0x1d04
+#define SPR_MPL_IPI_2_SET_0 0x1d00
+#define SPR_MPL_IPI_2_SET_1 0x1d01
+#define SPR_MPL_IPI_2_SET_2 0x1d02
+#define SPR_MPL_PERF_COUNT_SET_0 0x2000
+#define SPR_MPL_PERF_COUNT_SET_1 0x2001
+#define SPR_MPL_PERF_COUNT_SET_2 0x2002
+#define SPR_MPL_UDN_ACCESS_SET_0 0x0b00
+#define SPR_MPL_UDN_ACCESS_SET_1 0x0b01
+#define SPR_MPL_UDN_ACCESS_SET_2 0x0b02
+#define SPR_MPL_UDN_AVAIL_SET_0 0x1b00
+#define SPR_MPL_UDN_AVAIL_SET_1 0x1b01
+#define SPR_MPL_UDN_AVAIL_SET_2 0x1b02
+#define SPR_MPL_UDN_COMPLETE_SET_0 0x0600
+#define SPR_MPL_UDN_COMPLETE_SET_1 0x0601
+#define SPR_MPL_UDN_COMPLETE_SET_2 0x0602
+#define SPR_MPL_UDN_FIREWALL_SET_0 0x1500
+#define SPR_MPL_UDN_FIREWALL_SET_1 0x1501
+#define SPR_MPL_UDN_FIREWALL_SET_2 0x1502
+#define SPR_MPL_UDN_TIMER_SET_0 0x1900
+#define SPR_MPL_UDN_TIMER_SET_1 0x1901
+#define SPR_MPL_UDN_TIMER_SET_2 0x1902
+#define SPR_MPL_WORLD_ACCESS_SET_0 0x2700
+#define SPR_MPL_WORLD_ACCESS_SET_1 0x2701
+#define SPR_MPL_WORLD_ACCESS_SET_2 0x2702
+#define SPR_PASS 0x2709
+#define SPR_PERF_COUNT_0 0x2005
+#define SPR_PERF_COUNT_1 0x2006
+#define SPR_PERF_COUNT_CTL 0x2007
+#define SPR_PERF_COUNT_DN_CTL 0x2008
+#define SPR_PERF_COUNT_STS 0x2009
+#define SPR_PROC_STATUS 0x2784
+#define SPR_SIM_CONTROL 0x2785
+#define SPR_SINGLE_STEP_CONTROL_0 0x0405
+#define SPR_SINGLE_STEP_CONTROL_0__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_0__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_CONTROL_1 0x0305
+#define SPR_SINGLE_STEP_CONTROL_1__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_1__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_CONTROL_2 0x0205
+#define SPR_SINGLE_STEP_CONTROL_2__CANCELED_MASK  0x1
+#define SPR_SINGLE_STEP_CONTROL_2__INHIBIT_MASK  0x2
+#define SPR_SINGLE_STEP_EN_0_0 0x250a
+#define SPR_SINGLE_STEP_EN_0_1 0x240a
+#define SPR_SINGLE_STEP_EN_0_2 0x230a
+#define SPR_SINGLE_STEP_EN_1_0 0x250b
+#define SPR_SINGLE_STEP_EN_1_1 0x240b
+#define SPR_SINGLE_STEP_EN_1_2 0x230b
+#define SPR_SINGLE_STEP_EN_2_0 0x250c
+#define SPR_SINGLE_STEP_EN_2_1 0x240c
+#define SPR_SINGLE_STEP_EN_2_2 0x230c
+#define SPR_SYSTEM_SAVE_0_0 0x2582
+#define SPR_SYSTEM_SAVE_0_1 0x2583
+#define SPR_SYSTEM_SAVE_0_2 0x2584
+#define SPR_SYSTEM_SAVE_0_3 0x2585
+#define SPR_SYSTEM_SAVE_1_0 0x2482
+#define SPR_SYSTEM_SAVE_1_1 0x2483
+#define SPR_SYSTEM_SAVE_1_2 0x2484
+#define SPR_SYSTEM_SAVE_1_3 0x2485
+#define SPR_SYSTEM_SAVE_2_0 0x2382
+#define SPR_SYSTEM_SAVE_2_1 0x2383
+#define SPR_SYSTEM_SAVE_2_2 0x2384
+#define SPR_SYSTEM_SAVE_2_3 0x2385
+#define SPR_TILE_COORD 0x270b
+#define SPR_TILE_RTF_HWM 0x270c
+#define SPR_TILE_TIMER_CONTROL 0x1605
+#define SPR_UDN_AVAIL_EN 0x1b05
+#define SPR_UDN_DATA_AVAIL 0x0b80
+#define SPR_UDN_DEADLOCK_TIMEOUT 0x1906
+#define SPR_UDN_DEMUX_COUNT_0 0x0b05
+#define SPR_UDN_DEMUX_COUNT_1 0x0b06
+#define SPR_UDN_DEMUX_COUNT_2 0x0b07
+#define SPR_UDN_DEMUX_COUNT_3 0x0b08
+#define SPR_UDN_DIRECTION_PROTECT 0x1505
+#define SPR_UDN_PENDING 0x0b0a
+#define SPR_WATCH_MASK 0x200a
+#define SPR_WATCH_VAL 0x200b
+
+#endif /* !defined(__ARCH_SPR_DEF_64_H__) */
+
+#endif /* !defined(__DOXYGEN__) */
diff --git a/target-tilegx/translate.c b/target-tilegx/translate.c
new file mode 100644
index 0000000..11c9732
--- /dev/null
+++ b/target-tilegx/translate.c
@@ -0,0 +1,2453 @@
+/*
+ * QEMU TILE-Gx CPU
+ *
+ *  Copyright (c) 2015 Chen Gang
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see
+ * <http://www.gnu.org/licenses/lgpl-2.1.html>
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/log.h"
+#include "exec/log.h"
+#include "disas/disas.h"
+#include "exec/exec-all.h"
+#include "tcg-op.h"
+#include "exec/cpu_ldst.h"
+#include "linux-user/syscall_defs.h"
+
+#include "opcode_tilegx.h"
+#include "spr_def_64.h"
+
+#define FMT64X                          "%016" PRIx64
+
+static TCGv_env cpu_env;
+static TCGv cpu_pc;
+static TCGv cpu_regs[TILEGX_R_COUNT];
+
+static const char * const reg_names[64] = {
+     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+     "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+    "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+    "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+    "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+    "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+    "r48", "r49", "r50", "r51",  "bp",  "tp",  "sp",  "lr",
+    "sn", "idn0", "idn1", "udn0", "udn1", "udn2", "udn2", "zero"
+};
+
+/* Modified registers are cached in temporaries until the end of the bundle. */
+typedef struct {
+    unsigned reg;
+    TCGv val;
+} DisasContextTemp;
+
+#define MAX_WRITEBACK 4
+
+/* This is the state at translation time.  */
+typedef struct {
+    uint64_t pc;		/* Current pc */
+
+    TCGv zero;                  /* For zero register */
+
+    DisasContextTemp wb[MAX_WRITEBACK];
+    int num_wb;
+    int mmuidx;
+    bool exit_tb;
+    TileExcp atomic_excp;
+
+    struct {
+        TCGCond cond;    /* branch condition */
+        TCGv dest;       /* branch destination */
+        TCGv val1;       /* value to be compared against zero, for cond */
+    } jmp;               /* Jump object, only once in each TB block */
+} DisasContext;
+
+#include "exec/gen-icount.h"
+
+/* Differentiate the various pipe encodings.  */
+#define TY_X0  0
+#define TY_X1  1
+#define TY_Y0  2
+#define TY_Y1  3
+
+/* Remerge the base opcode and extension fields for switching.
+   The X opcode fields are 3 bits; Y0/Y1 opcode fields are 4 bits;
+   Y2 opcode field is 2 bits.  */
+#define OE(OP, EXT, XY) (TY_##XY + OP * 4 + EXT * 64)
+
+/* Similar, but for Y2 only.  */
+#define OEY2(OP, MODE) (OP + MODE * 4)
+
+/* Similar, but make sure opcode names match up.  */
+#define OE_RR_X0(E)    OE(RRR_0_OPCODE_X0, E##_UNARY_OPCODE_X0, X0)
+#define OE_RR_X1(E)    OE(RRR_0_OPCODE_X1, E##_UNARY_OPCODE_X1, X1)
+#define OE_RR_Y0(E)    OE(RRR_1_OPCODE_Y0, E##_UNARY_OPCODE_Y0, Y0)
+#define OE_RR_Y1(E)    OE(RRR_1_OPCODE_Y1, E##_UNARY_OPCODE_Y1, Y1)
+#define OE_RRR(E,N,XY) OE(RRR_##N##_OPCODE_##XY, E##_RRR_##N##_OPCODE_##XY, XY)
+#define OE_IM(E,XY)    OE(IMM8_OPCODE_##XY, E##_IMM8_OPCODE_##XY, XY)
+#define OE_SH(E,XY)    OE(SHIFT_OPCODE_##XY, E##_SHIFT_OPCODE_##XY, XY)
+
+#define V1_IMM(X)      (((X) & 0xff) * 0x0101010101010101ull)
+#define V2_IMM(X)      (((X) & 0xffff) * 0x0001000100010001ull)
+
+
+static void gen_exception(DisasContext *dc, TileExcp num)
+{
+    TCGv_i32 tmp;
+
+    tcg_gen_movi_tl(cpu_pc, dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+
+    tmp = tcg_const_i32(num);
+    gen_helper_exception(cpu_env, tmp);
+    tcg_temp_free_i32(tmp);
+    dc->exit_tb = true;
+}
+
+static bool check_gr(DisasContext *dc, uint8_t reg)
+{
+    if (likely(reg < TILEGX_R_COUNT)) {
+        return true;
+    }
+
+    switch (reg) {
+    case TILEGX_R_SN:
+    case TILEGX_R_ZERO:
+        break;
+    case TILEGX_R_IDN0:
+    case TILEGX_R_IDN1:
+        gen_exception(dc, TILEGX_EXCP_REG_IDN_ACCESS);
+        break;
+    case TILEGX_R_UDN0:
+    case TILEGX_R_UDN1:
+    case TILEGX_R_UDN2:
+    case TILEGX_R_UDN3:
+        gen_exception(dc, TILEGX_EXCP_REG_UDN_ACCESS);
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    return false;
+}
+
+static TCGv load_zero(DisasContext *dc)
+{
+    if (TCGV_IS_UNUSED_I64(dc->zero)) {
+        dc->zero = tcg_const_i64(0);
+    }
+    return dc->zero;
+}
+
+static TCGv load_gr(DisasContext *dc, unsigned reg)
+{
+    if (check_gr(dc, reg)) {
+        return cpu_regs[reg];
+    }
+    return load_zero(dc);
+}
+
+static TCGv dest_gr(DisasContext *dc, unsigned reg)
+{
+    int n;
+
+    /* Skip the result, mark the exception if necessary, and continue */
+    check_gr(dc, reg);
+
+    n = dc->num_wb++;
+    dc->wb[n].reg = reg;
+    return dc->wb[n].val = tcg_temp_new_i64();
+}
+
+static void gen_saturate_op(TCGv tdest, TCGv tsrca, TCGv tsrcb,
+                            void (*operate)(TCGv, TCGv, TCGv))
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_ext32s_tl(tdest, tsrca);
+    tcg_gen_ext32s_tl(t0, tsrcb);
+    operate(tdest, tdest, t0);
+
+    tcg_gen_movi_tl(t0, 0x7fffffff);
+    tcg_gen_movcond_tl(TCG_COND_GT, tdest, tdest, t0, t0, tdest);
+    tcg_gen_movi_tl(t0, -0x80000000LL);
+    tcg_gen_movcond_tl(TCG_COND_LT, tdest, tdest, t0, t0, tdest);
+
+    tcg_temp_free(t0);
+}
+
+static void gen_atomic_excp(DisasContext *dc, unsigned dest, TCGv tdest,
+                            TCGv tsrca, TCGv tsrcb, TileExcp excp)
+{
+#ifdef CONFIG_USER_ONLY
+    TCGv_i32 t;
+
+    tcg_gen_st_tl(tsrca, cpu_env, offsetof(CPUTLGState, atomic_srca));
+    tcg_gen_st_tl(tsrcb, cpu_env, offsetof(CPUTLGState, atomic_srcb));
+    t = tcg_const_i32(dest);
+    tcg_gen_st_i32(t, cpu_env, offsetof(CPUTLGState, atomic_dstr));
+    tcg_temp_free_i32(t);
+
+    /* We're going to write the real result in the exception.  But in
+       the meantime we've already created a writeback register, and
+       we don't want that to remain uninitialized.  */
+    tcg_gen_movi_tl(tdest, 0);
+
+    /* Note that we need to delay issuing the exception that implements
+       the atomic operation until after writing back the results of the
+       instruction occupying the X0 pipe.  */
+    dc->atomic_excp = excp;
+#else
+    gen_exception(dc, TILEGX_EXCP_OPCODE_UNIMPLEMENTED);
+#endif
+}
+
+/* Shift the 128-bit value TSRCA:TSRCD right by the number of bytes
+   specified by the bottom 3 bits of TSRCB, and set TDEST to the
+   low 64 bits of the resulting value.  */
+static void gen_dblalign(TCGv tdest, TCGv tsrcd, TCGv tsrca, TCGv tsrcb)
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_andi_tl(t0, tsrcb, 7);
+    tcg_gen_shli_tl(t0, t0, 3);
+    tcg_gen_shr_tl(tdest, tsrcd, t0);
+
+    /* We want to do "t0 = tsrca << (64 - t0)".  Two's complement
+       arithmetic on a 6-bit field tells us that 64 - t0 is equal
+       to (t0 ^ 63) + 1.  So we can do the shift in two parts,
+       neither of which will be an invalid shift by 64.  */
+    tcg_gen_xori_tl(t0, t0, 63);
+    tcg_gen_shl_tl(t0, tsrca, t0);
+    tcg_gen_shli_tl(t0, t0, 1);
+    tcg_gen_or_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t0);
+}
+
+/* Similarly, except that the 128-bit value is TSRCA:TSRCB, and the
+   right shift is an immediate.  */
+static void gen_dblaligni(TCGv tdest, TCGv tsrca, TCGv tsrcb, int shr)
+{
+    TCGv t0 = tcg_temp_new();
+
+    tcg_gen_shri_tl(t0, tsrcb, shr);
+    tcg_gen_shli_tl(tdest, tsrca, 64 - shr);
+    tcg_gen_or_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t0);
+}
+
+typedef enum {
+    LU, LS, HU, HS
+} MulHalf;
+
+static void gen_ext_half(TCGv d, TCGv s, MulHalf h)
+{
+    switch (h) {
+    case LU:
+        tcg_gen_ext32u_tl(d, s);
+        break;
+    case LS:
+        tcg_gen_ext32s_tl(d, s);
+        break;
+    case HU:
+        tcg_gen_shri_tl(d, s, 32);
+        break;
+    case HS:
+        tcg_gen_sari_tl(d, s, 32);
+        break;
+    }
+}
+
+static void gen_mul_half(TCGv tdest, TCGv tsrca, TCGv tsrcb,
+                         MulHalf ha, MulHalf hb)
+{
+    TCGv t = tcg_temp_new();
+    gen_ext_half(t, tsrca, ha);
+    gen_ext_half(tdest, tsrcb, hb);
+    tcg_gen_mul_tl(tdest, tdest, t);
+    tcg_temp_free(t);
+}
+
+static void gen_cmul2(TCGv tdest, TCGv tsrca, TCGv tsrcb, int sh, int rd)
+{
+    TCGv_i32 tsh = tcg_const_i32(sh);
+    TCGv_i32 trd = tcg_const_i32(rd);
+    gen_helper_cmul2(tdest, tsrca, tsrcb, tsh, trd);
+    tcg_temp_free_i32(tsh);
+    tcg_temp_free_i32(trd);
+}
+
+static TileExcp gen_st_opcode(DisasContext *dc, unsigned dest, unsigned srca,
+                              unsigned srcb, TCGMemOp memop, const char *name)
+{
+    if (dest) {
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    tcg_gen_qemu_st_tl(load_gr(dc, srcb), load_gr(dc, srca),
+		       dc->mmuidx, memop);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", name,
+                  reg_names[srca], reg_names[srcb]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_st_add_opcode(DisasContext *dc, unsigned srca, unsigned srcb,
+                                  int imm, TCGMemOp memop, const char *name)
+{
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcb = load_gr(dc, srcb);
+
+    tcg_gen_qemu_st_tl(tsrcb, tsrca, dc->mmuidx, memop);
+    tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %d", name,
+                  reg_names[srca], reg_names[srcb], imm);
+    return TILEGX_EXCP_NONE;
+}
+
+/* Equality comparison with zero can be done quickly and efficiently.  */
+static void gen_v1cmpeq0(TCGv v)
+{
+    TCGv m = tcg_const_tl(V1_IMM(0x7f));
+    TCGv c = tcg_temp_new();
+
+    /* ~(((v & m) + m) | m | v).  Sets the msb for each byte == 0.  */
+    tcg_gen_and_tl(c, v, m);
+    tcg_gen_add_tl(c, c, m);
+    tcg_gen_or_tl(c, c, m);
+    tcg_gen_nor_tl(c, c, v);
+    tcg_temp_free(m);
+
+    /* Shift the msb down to form the lsb boolean result.  */
+    tcg_gen_shri_tl(v, c, 7);
+    tcg_temp_free(c);
+}
+
+static void gen_v1cmpne0(TCGv v)
+{
+    TCGv m = tcg_const_tl(V1_IMM(0x7f));
+    TCGv c = tcg_temp_new();
+
+    /* (((v & m) + m) | v) & ~m.  Sets the msb for each byte != 0.  */
+    tcg_gen_and_tl(c, v, m);
+    tcg_gen_add_tl(c, c, m);
+    tcg_gen_or_tl(c, c, v);
+    tcg_gen_andc_tl(c, c, m);
+    tcg_temp_free(m);
+
+    /* Shift the msb down to form the lsb boolean result.  */
+    tcg_gen_shri_tl(v, c, 7);
+    tcg_temp_free(c);
+}
+
+/* Vector addition can be performed via arithmetic plus masking.  It is
+   efficient this way only for 4 or more elements.  */
+static void gen_v12add(TCGv tdest, TCGv tsrca, TCGv tsrcb, uint64_t sign)
+{
+    TCGv tmask = tcg_const_tl(~sign);
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+
+    /* ((a & ~sign) + (b & ~sign)) ^ ((a ^ b) & sign).  */
+    tcg_gen_and_tl(t0, tsrca, tmask);
+    tcg_gen_and_tl(t1, tsrcb, tmask);
+    tcg_gen_add_tl(tdest, t0, t1);
+    tcg_gen_xor_tl(t0, tsrca, tsrcb);
+    tcg_gen_andc_tl(t0, t0, tmask);
+    tcg_gen_xor_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(tmask);
+}
+
+/* Similarly for vector subtraction.  */
+static void gen_v12sub(TCGv tdest, TCGv tsrca, TCGv tsrcb, uint64_t sign)
+{
+    TCGv tsign = tcg_const_tl(sign);
+    TCGv t0 = tcg_temp_new();
+    TCGv t1 = tcg_temp_new();
+
+    /* ((a | sign) - (b & ~sign)) ^ ((a ^ ~b) & sign).  */
+    tcg_gen_or_tl(t0, tsrca, tsign);
+    tcg_gen_andc_tl(t1, tsrcb, tsign);
+    tcg_gen_sub_tl(tdest, t0, t1);
+    tcg_gen_eqv_tl(t0, tsrca, tsrcb);
+    tcg_gen_and_tl(t0, t0, tsign);
+    tcg_gen_xor_tl(tdest, tdest, t0);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
+    tcg_temp_free(tsign);
+}
+
+static void gen_v4sh(TCGv d64, TCGv a64, TCGv b64,
+                     void (*generate)(TCGv_i32, TCGv_i32, TCGv_i32))
+{
+    TCGv_i32 al = tcg_temp_new_i32();
+    TCGv_i32 ah = tcg_temp_new_i32();
+    TCGv_i32 bl = tcg_temp_new_i32();
+
+    tcg_gen_extr_i64_i32(al, ah, a64);
+    tcg_gen_extrl_i64_i32(bl, b64);
+    tcg_gen_andi_i32(bl, bl, 31);
+    generate(al, al, bl);
+    generate(ah, ah, bl);
+    tcg_gen_concat_i32_i64(d64, al, ah);
+
+    tcg_temp_free_i32(al);
+    tcg_temp_free_i32(ah);
+    tcg_temp_free_i32(bl);
+}
+
+static void gen_v4op(TCGv d64, TCGv a64, TCGv b64,
+                     void (*generate)(TCGv_i32, TCGv_i32, TCGv_i32))
+{
+    TCGv_i32 al = tcg_temp_new_i32();
+    TCGv_i32 ah = tcg_temp_new_i32();
+    TCGv_i32 bl = tcg_temp_new_i32();
+    TCGv_i32 bh = tcg_temp_new_i32();
+
+    tcg_gen_extr_i64_i32(al, ah, a64);
+    tcg_gen_extr_i64_i32(bl, bh, b64);
+    generate(al, al, bl);
+    generate(ah, ah, bh);
+    tcg_gen_concat_i32_i64(d64, al, ah);
+
+    tcg_temp_free_i32(al);
+    tcg_temp_free_i32(ah);
+    tcg_temp_free_i32(bl);
+    tcg_temp_free_i32(bh);
+}
+
+static TileExcp gen_signal(DisasContext *dc, int signo, int sigcode,
+                           const char *mnemonic)
+{
+    TCGv_i32 t0 = tcg_const_i32(signo);
+    TCGv_i32 t1 = tcg_const_i32(sigcode);
+
+    tcg_gen_st_i32(t0, cpu_env, offsetof(CPUTLGState, signo));
+    tcg_gen_st_i32(t1, cpu_env, offsetof(CPUTLGState, sigcode));
+
+    tcg_temp_free_i32(t1);
+    tcg_temp_free_i32(t0);
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s", mnemonic);
+    return TILEGX_EXCP_SIGNAL;
+}
+
+static bool parse_from_addli(uint64_t bundle, int *signo, int *sigcode)
+{
+    int imm;
+
+    if ((get_Opcode_X0(bundle) != ADDLI_OPCODE_X0)
+        || (get_Dest_X0(bundle) != TILEGX_R_ZERO)
+        || (get_SrcA_X0(bundle) != TILEGX_R_ZERO)) {
+        return false;
+    }
+
+    imm = get_Imm16_X0(bundle);
+    *signo = imm & 0x3f;
+    *sigcode = (imm >> 6) & 0xf;
+
+    /* ??? The linux kernel validates both signo and the sigcode vs the
+       known max for each signal.  Don't bother here.  */
+    return true;
+}
+
+static TileExcp gen_specill(DisasContext *dc, unsigned dest, unsigned srca,
+                            uint64_t bundle)
+{
+    const char *mnemonic;
+    int signo;
+    int sigcode;
+
+    if (dest == 0x1c && srca == 0x25) {
+        signo = TARGET_SIGTRAP;
+        sigcode = TARGET_TRAP_BRKPT;
+        mnemonic = "bpt";
+    } else if (dest == 0x1d && srca == 0x25
+               && parse_from_addli(bundle, &signo, &sigcode)) {
+        mnemonic = "raise";
+    } else {
+        signo = TARGET_SIGILL;
+        sigcode = TARGET_ILL_ILLOPC;
+        mnemonic = "ill";
+    }
+
+    return gen_signal(dc, signo, sigcode, mnemonic);
+}
+
+static TileExcp gen_rr_opcode(DisasContext *dc, unsigned opext,
+                              unsigned dest, unsigned srca, uint64_t bundle)
+{
+    TCGv tdest, tsrca;
+    const char *mnemonic;
+    TCGMemOp memop;
+    TileExcp ret = TILEGX_EXCP_NONE;
+    bool prefetch_nofault = false;
+
+    /* Eliminate instructions with no output before doing anything else.  */
+    switch (opext) {
+    case OE_RR_Y0(NOP):
+    case OE_RR_Y1(NOP):
+    case OE_RR_X0(NOP):
+    case OE_RR_X1(NOP):
+        mnemonic = "nop";
+        goto done0;
+    case OE_RR_Y0(FNOP):
+    case OE_RR_Y1(FNOP):
+    case OE_RR_X0(FNOP):
+    case OE_RR_X1(FNOP):
+        mnemonic = "fnop";
+        goto done0;
+    case OE_RR_X1(DRAIN):
+        mnemonic = "drain";
+        goto done0;
+    case OE_RR_X1(FLUSHWB):
+        mnemonic = "flushwb";
+        goto done0;
+    case OE_RR_X1(ILL):
+        return gen_specill(dc, dest, srca, bundle);
+    case OE_RR_Y1(ILL):
+        return gen_signal(dc, TARGET_SIGILL, TARGET_ILL_ILLOPC, "ill");
+    case OE_RR_X1(MF):
+        mnemonic = "mf";
+        goto done0;
+    case OE_RR_X1(NAP):
+        /* ??? This should yield, especially in system mode.  */
+        mnemonic = "nap";
+        goto done0;
+    case OE_RR_X1(IRET):
+        gen_helper_ext01_ics(cpu_env);
+        dc->jmp.cond = TCG_COND_ALWAYS;
+        dc->jmp.dest = tcg_temp_new();
+        tcg_gen_ld_tl(dc->jmp.dest, cpu_env,
+                      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_0]));
+        tcg_gen_andi_tl(dc->jmp.dest, dc->jmp.dest, ~7);
+        mnemonic = "iret";
+        goto done0;
+    case OE_RR_X1(SWINT0):
+    case OE_RR_X1(SWINT2):
+    case OE_RR_X1(SWINT3):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(SWINT1):
+        ret = TILEGX_EXCP_SYSCALL;
+        mnemonic = "swint1";
+    done0:
+        if (srca || dest) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s", mnemonic);
+        return ret;
+
+    case OE_RR_X1(DTLBPR):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(FINV):
+        mnemonic = "finv";
+        goto done1;
+    case OE_RR_X1(FLUSH):
+        mnemonic = "flush";
+        goto done1;
+    case OE_RR_X1(ICOH):
+        mnemonic = "icoh";
+        goto done1;
+    case OE_RR_X1(INV):
+        mnemonic = "inv";
+        goto done1;
+    case OE_RR_X1(WH64):
+        mnemonic = "wh64";
+        goto done1;
+    case OE_RR_X1(JRP):
+    case OE_RR_Y1(JRP):
+        mnemonic = "jrp";
+        goto do_jr;
+    case OE_RR_X1(JR):
+    case OE_RR_Y1(JR):
+        mnemonic = "jr";
+        goto do_jr;
+    case OE_RR_X1(JALRP):
+    case OE_RR_Y1(JALRP):
+        mnemonic = "jalrp";
+        goto do_jalr;
+    case OE_RR_X1(JALR):
+    case OE_RR_Y1(JALR):
+        mnemonic = "jalr";
+    do_jalr:
+        tcg_gen_movi_tl(dest_gr(dc, TILEGX_R_LR),
+                        dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+    do_jr:
+        dc->jmp.cond = TCG_COND_ALWAYS;
+        dc->jmp.dest = tcg_temp_new();
+        tcg_gen_andi_tl(dc->jmp.dest, load_gr(dc, srca), ~7);
+    done1:
+        if (dest) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s", mnemonic, reg_names[srca]);
+        return ret;
+    }
+
+    tdest = dest_gr(dc, dest);
+    tsrca = load_gr(dc, srca);
+
+    switch (opext) {
+    case OE_RR_X0(CNTLZ):
+    case OE_RR_Y0(CNTLZ):
+        gen_helper_cntlz(tdest, tsrca);
+        mnemonic = "cntlz";
+        break;
+    case OE_RR_X0(CNTTZ):
+    case OE_RR_Y0(CNTTZ):
+        gen_helper_cnttz(tdest, tsrca);
+        mnemonic = "cnttz";
+        break;
+    case OE_RR_X0(FSINGLE_PACK1):
+    case OE_RR_Y0(FSINGLE_PACK1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RR_X1(LD1S):
+        memop = MO_SB;
+        mnemonic = "ld1s"; /* prefetch_l1_fault */
+        goto do_load;
+    case OE_RR_X1(LD1U):
+        memop = MO_UB;
+        mnemonic = "ld1u"; /* prefetch, prefetch_l1 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LD2S):
+        memop = MO_TESW;
+        mnemonic = "ld2s"; /* prefetch_l2_fault */
+        goto do_load;
+    case OE_RR_X1(LD2U):
+        memop = MO_TEUW;
+        mnemonic = "ld2u"; /* prefetch_l2 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LD4S):
+        memop = MO_TESL;
+        mnemonic = "ld4s"; /* prefetch_l3_fault */
+        goto do_load;
+    case OE_RR_X1(LD4U):
+        memop = MO_TEUL;
+        mnemonic = "ld4u"; /* prefetch_l3 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load;
+    case OE_RR_X1(LDNT1S):
+        memop = MO_SB;
+        mnemonic = "ldnt1s";
+        goto do_load;
+    case OE_RR_X1(LDNT1U):
+        memop = MO_UB;
+        mnemonic = "ldnt1u";
+        goto do_load;
+    case OE_RR_X1(LDNT2S):
+        memop = MO_TESW;
+        mnemonic = "ldnt2s";
+        goto do_load;
+    case OE_RR_X1(LDNT2U):
+        memop = MO_TEUW;
+        mnemonic = "ldnt2u";
+        goto do_load;
+    case OE_RR_X1(LDNT4S):
+        memop = MO_TESL;
+        mnemonic = "ldnt4s";
+        goto do_load;
+    case OE_RR_X1(LDNT4U):
+        memop = MO_TEUL;
+        mnemonic = "ldnt4u";
+        goto do_load;
+    case OE_RR_X1(LDNT):
+        memop = MO_TEQ;
+        mnemonic = "ldnt";
+        goto do_load;
+    case OE_RR_X1(LD):
+        memop = MO_TEQ;
+        mnemonic = "ld";
+    do_load:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(tdest, tsrca, dc->mmuidx, memop);
+        }
+        break;
+    case OE_RR_X1(LDNA):
+        tcg_gen_andi_tl(tdest, tsrca, ~7);
+        tcg_gen_qemu_ld_tl(tdest, tdest, dc->mmuidx, MO_TEQ);
+        mnemonic = "ldna";
+        break;
+    case OE_RR_X1(LNK):
+    case OE_RR_Y1(LNK):
+        if (srca) {
+            return TILEGX_EXCP_OPCODE_UNKNOWN;
+        }
+        tcg_gen_movi_tl(tdest, dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+        mnemonic = "lnk";
+        break;
+    case OE_RR_X0(PCNT):
+    case OE_RR_Y0(PCNT):
+        gen_helper_pcnt(tdest, tsrca);
+        mnemonic = "pcnt";
+        break;
+    case OE_RR_X0(REVBITS):
+    case OE_RR_Y0(REVBITS):
+        gen_helper_revbits(tdest, tsrca);
+        mnemonic = "revbits";
+        break;
+    case OE_RR_X0(REVBYTES):
+    case OE_RR_Y0(REVBYTES):
+        tcg_gen_bswap64_tl(tdest, tsrca);
+        mnemonic = "revbytes";
+        break;
+    case OE_RR_X0(TBLIDXB0):
+    case OE_RR_Y0(TBLIDXB0):
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tsrca, 2, 8);
+        mnemonic = "tblidxb0";
+        break;
+    case OE_RR_X0(TBLIDXB1):
+    case OE_RR_Y0(TBLIDXB1):
+        tcg_gen_shri_tl(tdest, tsrca, 8);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb1";
+        break;
+    case OE_RR_X0(TBLIDXB2):
+    case OE_RR_Y0(TBLIDXB2):
+        tcg_gen_shri_tl(tdest, tsrca, 16);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb2";
+        break;
+    case OE_RR_X0(TBLIDXB3):
+    case OE_RR_Y0(TBLIDXB3):
+        tcg_gen_shri_tl(tdest, tsrca, 24);
+        tcg_gen_deposit_tl(tdest, load_gr(dc, dest), tdest, 2, 8);
+        mnemonic = "tblidxb3";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", mnemonic,
+                  reg_names[dest], reg_names[srca]);
+    return ret;
+}
+
+static TileExcp gen_rrr_opcode(DisasContext *dc, unsigned opext,
+                               unsigned dest, unsigned srca, unsigned srcb)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcb = load_gr(dc, srcb);
+    TCGv t0;
+    const char *mnemonic;
+
+    switch (opext) {
+    case OE_RRR(ADDXSC, 0, X0):
+    case OE_RRR(ADDXSC, 0, X1):
+        gen_saturate_op(tdest, tsrca, tsrcb, tcg_gen_add_tl);
+        mnemonic = "addxsc";
+        break;
+    case OE_RRR(ADDX, 0, X0):
+    case OE_RRR(ADDX, 0, X1):
+    case OE_RRR(ADDX, 0, Y0):
+    case OE_RRR(ADDX, 0, Y1):
+        tcg_gen_add_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addx";
+        break;
+    case OE_RRR(ADD, 0, X0):
+    case OE_RRR(ADD, 0, X1):
+    case OE_RRR(ADD, 0, Y0):
+    case OE_RRR(ADD, 0, Y1):
+        tcg_gen_add_tl(tdest, tsrca, tsrcb);
+        mnemonic = "add";
+        break;
+    case OE_RRR(AND, 0, X0):
+    case OE_RRR(AND, 0, X1):
+    case OE_RRR(AND, 5, Y0):
+    case OE_RRR(AND, 5, Y1):
+        tcg_gen_and_tl(tdest, tsrca, tsrcb);
+        mnemonic = "and";
+        break;
+    case OE_RRR(CMOVEQZ, 0, X0):
+    case OE_RRR(CMOVEQZ, 4, Y0):
+        tcg_gen_movcond_tl(TCG_COND_EQ, tdest, tsrca, load_zero(dc),
+                           tsrcb, load_gr(dc, dest));
+        mnemonic = "cmoveqz";
+        break;
+    case OE_RRR(CMOVNEZ, 0, X0):
+    case OE_RRR(CMOVNEZ, 4, Y0):
+        tcg_gen_movcond_tl(TCG_COND_NE, tdest, tsrca, load_zero(dc),
+                           tsrcb, load_gr(dc, dest));
+        mnemonic = "cmovnez";
+        break;
+    case OE_RRR(CMPEQ, 0, X0):
+    case OE_RRR(CMPEQ, 0, X1):
+    case OE_RRR(CMPEQ, 3, Y0):
+    case OE_RRR(CMPEQ, 3, Y1):
+        tcg_gen_setcond_tl(TCG_COND_EQ, tdest, tsrca, tsrcb);
+        mnemonic = "cmpeq";
+        break;
+    case OE_RRR(CMPEXCH4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_CMPEXCH4);
+        mnemonic = "cmpexch4";
+        break;
+    case OE_RRR(CMPEXCH, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_CMPEXCH);
+        mnemonic = "cmpexch";
+        break;
+    case OE_RRR(CMPLES, 0, X0):
+    case OE_RRR(CMPLES, 0, X1):
+    case OE_RRR(CMPLES, 2, Y0):
+    case OE_RRR(CMPLES, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LE, tdest, tsrca, tsrcb);
+        mnemonic = "cmples";
+        break;
+    case OE_RRR(CMPLEU, 0, X0):
+    case OE_RRR(CMPLEU, 0, X1):
+    case OE_RRR(CMPLEU, 2, Y0):
+    case OE_RRR(CMPLEU, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LEU, tdest, tsrca, tsrcb);
+        mnemonic = "cmpleu";
+        break;
+    case OE_RRR(CMPLTS, 0, X0):
+    case OE_RRR(CMPLTS, 0, X1):
+    case OE_RRR(CMPLTS, 2, Y0):
+    case OE_RRR(CMPLTS, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LT, tdest, tsrca, tsrcb);
+        mnemonic = "cmplts";
+        break;
+    case OE_RRR(CMPLTU, 0, X0):
+    case OE_RRR(CMPLTU, 0, X1):
+    case OE_RRR(CMPLTU, 2, Y0):
+    case OE_RRR(CMPLTU, 2, Y1):
+        tcg_gen_setcond_tl(TCG_COND_LTU, tdest, tsrca, tsrcb);
+        mnemonic = "cmpltu";
+        break;
+    case OE_RRR(CMPNE, 0, X0):
+    case OE_RRR(CMPNE, 0, X1):
+    case OE_RRR(CMPNE, 3, Y0):
+    case OE_RRR(CMPNE, 3, Y1):
+        tcg_gen_setcond_tl(TCG_COND_NE, tdest, tsrca, tsrcb);
+        mnemonic = "cmpne";
+        break;
+    case OE_RRR(CMULAF, 0, X0):
+        gen_helper_cmulaf(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "cmulaf";
+        break;
+    case OE_RRR(CMULA, 0, X0):
+        gen_helper_cmula(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "cmula";
+        break;
+    case OE_RRR(CMULFR, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 15, 1 << 14);
+        mnemonic = "cmulfr";
+        break;
+    case OE_RRR(CMULF, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 15, 0);
+        mnemonic = "cmulf";
+        break;
+    case OE_RRR(CMULHR, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 16, 1 << 15);
+        mnemonic = "cmulhr";
+        break;
+    case OE_RRR(CMULH, 0, X0):
+        gen_cmul2(tdest, tsrca, tsrcb, 16, 0);
+        mnemonic = "cmulh";
+        break;
+    case OE_RRR(CMUL, 0, X0):
+        gen_helper_cmula(tdest, load_zero(dc), tsrca, tsrcb);
+        mnemonic = "cmul";
+        break;
+    case OE_RRR(CRC32_32, 0, X0):
+        gen_helper_crc32_32(tdest, tsrca, tsrcb);
+        mnemonic = "crc32_32";
+        break;
+    case OE_RRR(CRC32_8, 0, X0):
+        gen_helper_crc32_8(tdest, tsrca, tsrcb);
+        mnemonic = "crc32_8";
+        break;
+    case OE_RRR(DBLALIGN2, 0, X0):
+    case OE_RRR(DBLALIGN2, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 16);
+        mnemonic = "dblalign2";
+        break;
+    case OE_RRR(DBLALIGN4, 0, X0):
+    case OE_RRR(DBLALIGN4, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 32);
+        mnemonic = "dblalign4";
+        break;
+    case OE_RRR(DBLALIGN6, 0, X0):
+    case OE_RRR(DBLALIGN6, 0, X1):
+        gen_dblaligni(tdest, tsrca, tsrcb, 48);
+        mnemonic = "dblalign6";
+        break;
+    case OE_RRR(DBLALIGN, 0, X0):
+        gen_dblalign(tdest, load_gr(dc, dest), tsrca, tsrcb);
+        mnemonic = "dblalign";
+        break;
+    case OE_RRR(EXCH4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_EXCH4);
+        mnemonic = "exch4";
+        break;
+    case OE_RRR(EXCH, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_EXCH);
+        mnemonic = "exch";
+        break;
+    case OE_RRR(FDOUBLE_ADDSUB, 0, X0):
+    case OE_RRR(FDOUBLE_ADD_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_MUL_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_PACK1, 0, X0):
+    case OE_RRR(FDOUBLE_PACK2, 0, X0):
+    case OE_RRR(FDOUBLE_SUB_FLAGS, 0, X0):
+    case OE_RRR(FDOUBLE_UNPACK_MAX, 0, X0):
+    case OE_RRR(FDOUBLE_UNPACK_MIN, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(FETCHADD4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADD4);
+        mnemonic = "fetchadd4";
+        break;
+    case OE_RRR(FETCHADDGEZ4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADDGEZ4);
+        mnemonic = "fetchaddgez4";
+        break;
+    case OE_RRR(FETCHADDGEZ, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADDGEZ);
+        mnemonic = "fetchaddgez";
+        break;
+    case OE_RRR(FETCHADD, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHADD);
+        mnemonic = "fetchadd";
+        break;
+    case OE_RRR(FETCHAND4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHAND4);
+        mnemonic = "fetchand4";
+        break;
+    case OE_RRR(FETCHAND, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHAND);
+        mnemonic = "fetchand";
+        break;
+    case OE_RRR(FETCHOR4, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHOR4);
+        mnemonic = "fetchor4";
+        break;
+    case OE_RRR(FETCHOR, 0, X1):
+        gen_atomic_excp(dc, dest, tdest, tsrca, tsrcb,
+                        TILEGX_EXCP_OPCODE_FETCHOR);
+        mnemonic = "fetchor";
+        break;
+    case OE_RRR(FSINGLE_ADD1, 0, X0):
+    case OE_RRR(FSINGLE_ADDSUB2, 0, X0):
+    case OE_RRR(FSINGLE_MUL1, 0, X0):
+    case OE_RRR(FSINGLE_MUL2, 0, X0):
+    case OE_RRR(FSINGLE_PACK2, 0, X0):
+    case OE_RRR(FSINGLE_SUB1, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(MNZ, 0, X0):
+    case OE_RRR(MNZ, 0, X1):
+    case OE_RRR(MNZ, 4, Y0):
+    case OE_RRR(MNZ, 4, Y1):
+        t0 = load_zero(dc);
+        tcg_gen_movcond_tl(TCG_COND_NE, tdest, tsrca, t0, tsrcb, t0);
+        mnemonic = "mnz";
+        break;
+    case OE_RRR(MULAX, 0, X0):
+    case OE_RRR(MULAX, 3, Y0):
+        tcg_gen_mul_tl(tdest, tsrca, tsrcb);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "mulax";
+        break;
+    case OE_RRR(MULA_HS_HS, 0, X0):
+    case OE_RRR(MULA_HS_HS, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_hs";
+        break;
+    case OE_RRR(MULA_HS_HU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_hu";
+        break;
+    case OE_RRR(MULA_HS_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_ls";
+        break;
+    case OE_RRR(MULA_HS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hs_lu";
+        break;
+    case OE_RRR(MULA_HU_HU, 0, X0):
+    case OE_RRR(MULA_HU_HU, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, HU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_hu";
+        break;
+    case OE_RRR(MULA_HU_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_ls";
+        break;
+    case OE_RRR(MULA_HU_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_hu_lu";
+        break;
+    case OE_RRR(MULA_LS_LS, 0, X0):
+    case OE_RRR(MULA_LS_LS, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LS);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_ls_ls";
+        break;
+    case OE_RRR(MULA_LS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_ls_lu";
+        break;
+    case OE_RRR(MULA_LU_LU, 0, X0):
+    case OE_RRR(MULA_LU_LU, 9, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LU, LU);
+        tcg_gen_add_tl(tdest, tdest, load_gr(dc, dest));
+        mnemonic = "mula_lu_lu";
+        break;
+    case OE_RRR(MULX, 0, X0):
+    case OE_RRR(MULX, 3, Y0):
+        tcg_gen_mul_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "mulx";
+        break;
+    case OE_RRR(MUL_HS_HS, 0, X0):
+    case OE_RRR(MUL_HS_HS, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HS);
+        mnemonic = "mul_hs_hs";
+        break;
+    case OE_RRR(MUL_HS_HU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, HU);
+        mnemonic = "mul_hs_hu";
+        break;
+    case OE_RRR(MUL_HS_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LS);
+        mnemonic = "mul_hs_ls";
+        break;
+    case OE_RRR(MUL_HS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HS, LU);
+        mnemonic = "mul_hs_lu";
+        break;
+    case OE_RRR(MUL_HU_HU, 0, X0):
+    case OE_RRR(MUL_HU_HU, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, HU);
+        mnemonic = "mul_hu_hu";
+        break;
+    case OE_RRR(MUL_HU_LS, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LS);
+        mnemonic = "mul_hu_ls";
+        break;
+    case OE_RRR(MUL_HU_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, HU, LU);
+        mnemonic = "mul_hu_lu";
+        break;
+    case OE_RRR(MUL_LS_LS, 0, X0):
+    case OE_RRR(MUL_LS_LS, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LS);
+        mnemonic = "mul_ls_ls";
+        break;
+    case OE_RRR(MUL_LS_LU, 0, X0):
+        gen_mul_half(tdest, tsrca, tsrcb, LS, LU);
+        mnemonic = "mul_ls_lu";
+        break;
+    case OE_RRR(MUL_LU_LU, 0, X0):
+    case OE_RRR(MUL_LU_LU, 8, Y0):
+        gen_mul_half(tdest, tsrca, tsrcb, LU, LU);
+        mnemonic = "mul_lu_lu";
+        break;
+    case OE_RRR(MZ, 0, X0):
+    case OE_RRR(MZ, 0, X1):
+    case OE_RRR(MZ, 4, Y0):
+    case OE_RRR(MZ, 4, Y1):
+        t0 = load_zero(dc);
+        tcg_gen_movcond_tl(TCG_COND_EQ, tdest, tsrca, t0, tsrcb, t0);
+        mnemonic = "mz";
+        break;
+    case OE_RRR(NOR, 0, X0):
+    case OE_RRR(NOR, 0, X1):
+    case OE_RRR(NOR, 5, Y0):
+    case OE_RRR(NOR, 5, Y1):
+        tcg_gen_nor_tl(tdest, tsrca, tsrcb);
+        mnemonic = "nor";
+        break;
+    case OE_RRR(OR, 0, X0):
+    case OE_RRR(OR, 0, X1):
+    case OE_RRR(OR, 5, Y0):
+    case OE_RRR(OR, 5, Y1):
+        tcg_gen_or_tl(tdest, tsrca, tsrcb);
+        mnemonic = "or";
+        break;
+    case OE_RRR(ROTL, 0, X0):
+    case OE_RRR(ROTL, 0, X1):
+    case OE_RRR(ROTL, 6, Y0):
+    case OE_RRR(ROTL, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_rotl_tl(tdest, tsrca, tdest);
+        mnemonic = "rotl";
+        break;
+    case OE_RRR(SHL1ADDX, 0, X0):
+    case OE_RRR(SHL1ADDX, 0, X1):
+    case OE_RRR(SHL1ADDX, 7, Y0):
+    case OE_RRR(SHL1ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 1);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl1addx";
+        break;
+    case OE_RRR(SHL1ADD, 0, X0):
+    case OE_RRR(SHL1ADD, 0, X1):
+    case OE_RRR(SHL1ADD, 1, Y0):
+    case OE_RRR(SHL1ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 1);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl1add";
+        break;
+    case OE_RRR(SHL2ADDX, 0, X0):
+    case OE_RRR(SHL2ADDX, 0, X1):
+    case OE_RRR(SHL2ADDX, 7, Y0):
+    case OE_RRR(SHL2ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 2);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl2addx";
+        break;
+    case OE_RRR(SHL2ADD, 0, X0):
+    case OE_RRR(SHL2ADD, 0, X1):
+    case OE_RRR(SHL2ADD, 1, Y0):
+    case OE_RRR(SHL2ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 2);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl2add";
+        break;
+    case OE_RRR(SHL3ADDX, 0, X0):
+    case OE_RRR(SHL3ADDX, 0, X1):
+    case OE_RRR(SHL3ADDX, 7, Y0):
+    case OE_RRR(SHL3ADDX, 7, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 3);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shl3addx";
+        break;
+    case OE_RRR(SHL3ADD, 0, X0):
+    case OE_RRR(SHL3ADD, 0, X1):
+    case OE_RRR(SHL3ADD, 1, Y0):
+    case OE_RRR(SHL3ADD, 1, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, 3);
+        tcg_gen_add_tl(tdest, tdest, tsrcb);
+        mnemonic = "shl3add";
+        break;
+    case OE_RRR(SHLX, 0, X0):
+    case OE_RRR(SHLX, 0, X1):
+        tcg_gen_andi_tl(tdest, tsrcb, 31);
+        tcg_gen_shl_tl(tdest, tsrca, tdest);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shlx";
+        break;
+    case OE_RRR(SHL, 0, X0):
+    case OE_RRR(SHL, 0, X1):
+    case OE_RRR(SHL, 6, Y0):
+    case OE_RRR(SHL, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_shl_tl(tdest, tsrca, tdest);
+        mnemonic = "shl";
+        break;
+    case OE_RRR(SHRS, 0, X0):
+    case OE_RRR(SHRS, 0, X1):
+    case OE_RRR(SHRS, 6, Y0):
+    case OE_RRR(SHRS, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_sar_tl(tdest, tsrca, tdest);
+        mnemonic = "shrs";
+        break;
+    case OE_RRR(SHRUX, 0, X0):
+    case OE_RRR(SHRUX, 0, X1):
+        t0 = tcg_temp_new();
+        tcg_gen_andi_tl(t0, tsrcb, 31);
+        tcg_gen_ext32u_tl(tdest, tsrca);
+        tcg_gen_shr_tl(tdest, tdest, t0);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        tcg_temp_free(t0);
+        mnemonic = "shrux";
+        break;
+    case OE_RRR(SHRU, 0, X0):
+    case OE_RRR(SHRU, 0, X1):
+    case OE_RRR(SHRU, 6, Y0):
+    case OE_RRR(SHRU, 6, Y1):
+        tcg_gen_andi_tl(tdest, tsrcb, 63);
+        tcg_gen_shr_tl(tdest, tsrca, tdest);
+        mnemonic = "shru";
+        break;
+    case OE_RRR(SHUFFLEBYTES, 0, X0):
+        gen_helper_shufflebytes(tdest, load_gr(dc, dest), tsrca, tsrca);
+        mnemonic = "shufflebytes";
+        break;
+    case OE_RRR(SUBXSC, 0, X0):
+    case OE_RRR(SUBXSC, 0, X1):
+        gen_saturate_op(tdest, tsrca, tsrcb, tcg_gen_sub_tl);
+        mnemonic = "subxsc";
+        break;
+    case OE_RRR(SUBX, 0, X0):
+    case OE_RRR(SUBX, 0, X1):
+    case OE_RRR(SUBX, 0, Y0):
+    case OE_RRR(SUBX, 0, Y1):
+        tcg_gen_sub_tl(tdest, tsrca, tsrcb);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "subx";
+        break;
+    case OE_RRR(SUB, 0, X0):
+    case OE_RRR(SUB, 0, X1):
+    case OE_RRR(SUB, 0, Y0):
+    case OE_RRR(SUB, 0, Y1):
+        tcg_gen_sub_tl(tdest, tsrca, tsrcb);
+        mnemonic = "sub";
+        break;
+    case OE_RRR(V1ADDUC, 0, X0):
+    case OE_RRR(V1ADDUC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1ADD, 0, X0):
+    case OE_RRR(V1ADD, 0, X1):
+        gen_v12add(tdest, tsrca, tsrcb, V1_IMM(0x80));
+        mnemonic = "v1add";
+        break;
+    case OE_RRR(V1ADIFFU, 0, X0):
+    case OE_RRR(V1AVGU, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1CMPEQ, 0, X0):
+    case OE_RRR(V1CMPEQ, 0, X1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        gen_v1cmpeq0(tdest);
+        mnemonic = "v1cmpeq";
+        break;
+    case OE_RRR(V1CMPLES, 0, X0):
+    case OE_RRR(V1CMPLES, 0, X1):
+    case OE_RRR(V1CMPLEU, 0, X0):
+    case OE_RRR(V1CMPLEU, 0, X1):
+    case OE_RRR(V1CMPLTS, 0, X0):
+    case OE_RRR(V1CMPLTS, 0, X1):
+    case OE_RRR(V1CMPLTU, 0, X0):
+    case OE_RRR(V1CMPLTU, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1CMPNE, 0, X0):
+    case OE_RRR(V1CMPNE, 0, X1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        gen_v1cmpne0(tdest);
+        mnemonic = "v1cmpne";
+        break;
+    case OE_RRR(V1DDOTPUA, 0, X0):
+    case OE_RRR(V1DDOTPUSA, 0, X0):
+    case OE_RRR(V1DDOTPUS, 0, X0):
+    case OE_RRR(V1DDOTPU, 0, X0):
+    case OE_RRR(V1DOTPA, 0, X0):
+    case OE_RRR(V1DOTPUA, 0, X0):
+    case OE_RRR(V1DOTPUSA, 0, X0):
+    case OE_RRR(V1DOTPUS, 0, X0):
+    case OE_RRR(V1DOTPU, 0, X0):
+    case OE_RRR(V1DOTP, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1INT_H, 0, X0):
+    case OE_RRR(V1INT_H, 0, X1):
+        gen_helper_v1int_h(tdest, tsrca, tsrcb);
+        mnemonic = "v1int_h";
+        break;
+    case OE_RRR(V1INT_L, 0, X0):
+    case OE_RRR(V1INT_L, 0, X1):
+        gen_helper_v1int_l(tdest, tsrca, tsrcb);
+        mnemonic = "v1int_l";
+        break;
+    case OE_RRR(V1MAXU, 0, X0):
+    case OE_RRR(V1MAXU, 0, X1):
+    case OE_RRR(V1MINU, 0, X0):
+    case OE_RRR(V1MINU, 0, X1):
+    case OE_RRR(V1MNZ, 0, X0):
+    case OE_RRR(V1MNZ, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1MULTU, 0, X0):
+        gen_helper_v1multu(tdest, tsrca, tsrcb);
+        mnemonic = "v1multu";
+        break;
+    case OE_RRR(V1MULUS, 0, X0):
+    case OE_RRR(V1MULU, 0, X0):
+    case OE_RRR(V1MZ, 0, X0):
+    case OE_RRR(V1MZ, 0, X1):
+    case OE_RRR(V1SADAU, 0, X0):
+    case OE_RRR(V1SADU, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1SHL, 0, X0):
+    case OE_RRR(V1SHL, 0, X1):
+        gen_helper_v1shl(tdest, tsrca, tsrcb);
+        mnemonic = "v1shl";
+        break;
+    case OE_RRR(V1SHRS, 0, X0):
+    case OE_RRR(V1SHRS, 0, X1):
+        gen_helper_v1shrs(tdest, tsrca, tsrcb);
+        mnemonic = "v1shrs";
+        break;
+    case OE_RRR(V1SHRU, 0, X0):
+    case OE_RRR(V1SHRU, 0, X1):
+        gen_helper_v1shru(tdest, tsrca, tsrcb);
+        mnemonic = "v1shru";
+        break;
+    case OE_RRR(V1SUBUC, 0, X0):
+    case OE_RRR(V1SUBUC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V1SUB, 0, X0):
+    case OE_RRR(V1SUB, 0, X1):
+        gen_v12sub(tdest, tsrca, tsrcb, V1_IMM(0x80));
+        mnemonic = "v1sub";
+        break;
+    case OE_RRR(V2ADDSC, 0, X0):
+    case OE_RRR(V2ADDSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2ADD, 0, X0):
+    case OE_RRR(V2ADD, 0, X1):
+        gen_v12add(tdest, tsrca, tsrcb, V2_IMM(0x8000));
+        mnemonic = "v2add";
+        break;
+    case OE_RRR(V2ADIFFS, 0, X0):
+    case OE_RRR(V2AVGS, 0, X0):
+    case OE_RRR(V2CMPEQ, 0, X0):
+    case OE_RRR(V2CMPEQ, 0, X1):
+    case OE_RRR(V2CMPLES, 0, X0):
+    case OE_RRR(V2CMPLES, 0, X1):
+    case OE_RRR(V2CMPLEU, 0, X0):
+    case OE_RRR(V2CMPLEU, 0, X1):
+    case OE_RRR(V2CMPLTS, 0, X0):
+    case OE_RRR(V2CMPLTS, 0, X1):
+    case OE_RRR(V2CMPLTU, 0, X0):
+    case OE_RRR(V2CMPLTU, 0, X1):
+    case OE_RRR(V2CMPNE, 0, X0):
+    case OE_RRR(V2CMPNE, 0, X1):
+    case OE_RRR(V2DOTPA, 0, X0):
+    case OE_RRR(V2DOTP, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2INT_H, 0, X0):
+    case OE_RRR(V2INT_H, 0, X1):
+        gen_helper_v2int_h(tdest, tsrca, tsrcb);
+        mnemonic = "v2int_h";
+        break;
+    case OE_RRR(V2INT_L, 0, X0):
+    case OE_RRR(V2INT_L, 0, X1):
+        gen_helper_v2int_l(tdest, tsrca, tsrcb);
+        mnemonic = "v2int_l";
+        break;
+    case OE_RRR(V2MAXS, 0, X0):
+    case OE_RRR(V2MAXS, 0, X1):
+    case OE_RRR(V2MINS, 0, X0):
+    case OE_RRR(V2MINS, 0, X1):
+    case OE_RRR(V2MNZ, 0, X0):
+    case OE_RRR(V2MNZ, 0, X1):
+    case OE_RRR(V2MULFSC, 0, X0):
+    case OE_RRR(V2MULS, 0, X0):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2MULTS, 0, X0):
+        gen_helper_v2mults(tdest, tsrca, tsrcb);
+        mnemonic = "v2mults";
+        break;
+    case OE_RRR(V2MZ, 0, X0):
+    case OE_RRR(V2MZ, 0, X1):
+    case OE_RRR(V2PACKH, 0, X0):
+    case OE_RRR(V2PACKH, 0, X1):
+    case OE_RRR(V2PACKL, 0, X0):
+    case OE_RRR(V2PACKL, 0, X1):
+    case OE_RRR(V2PACKUC, 0, X0):
+    case OE_RRR(V2PACKUC, 0, X1):
+    case OE_RRR(V2SADAS, 0, X0):
+    case OE_RRR(V2SADAU, 0, X0):
+    case OE_RRR(V2SADS, 0, X0):
+    case OE_RRR(V2SADU, 0, X0):
+    case OE_RRR(V2SHLSC, 0, X0):
+    case OE_RRR(V2SHLSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2SHL, 0, X0):
+    case OE_RRR(V2SHL, 0, X1):
+        gen_helper_v2shl(tdest, tsrca, tsrcb);
+        mnemonic = "v2shl";
+        break;
+    case OE_RRR(V2SHRS, 0, X0):
+    case OE_RRR(V2SHRS, 0, X1):
+        gen_helper_v2shrs(tdest, tsrca, tsrcb);
+        mnemonic = "v2shrs";
+        break;
+    case OE_RRR(V2SHRU, 0, X0):
+    case OE_RRR(V2SHRU, 0, X1):
+        gen_helper_v2shru(tdest, tsrca, tsrcb);
+        mnemonic = "v2shru";
+        break;
+    case OE_RRR(V2SUBSC, 0, X0):
+    case OE_RRR(V2SUBSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V2SUB, 0, X0):
+    case OE_RRR(V2SUB, 0, X1):
+        gen_v12sub(tdest, tsrca, tsrcb, V2_IMM(0x8000));
+        mnemonic = "v2sub";
+        break;
+    case OE_RRR(V4ADDSC, 0, X0):
+    case OE_RRR(V4ADDSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4ADD, 0, X0):
+    case OE_RRR(V4ADD, 0, X1):
+        gen_v4op(tdest, tsrca, tsrcb, tcg_gen_add_i32);
+        mnemonic = "v4add";
+        break;
+    case OE_RRR(V4INT_H, 0, X0):
+    case OE_RRR(V4INT_H, 0, X1):
+        tcg_gen_shri_tl(tdest, tsrcb, 32);
+        tcg_gen_deposit_tl(tdest, tsrca, tdest, 0, 32);
+        mnemonic = "v4int_h";
+        break;
+    case OE_RRR(V4INT_L, 0, X0):
+    case OE_RRR(V4INT_L, 0, X1):
+        tcg_gen_deposit_tl(tdest, tsrcb, tsrca, 32, 32);
+        mnemonic = "v4int_l";
+        break;
+    case OE_RRR(V4PACKSC, 0, X0):
+    case OE_RRR(V4PACKSC, 0, X1):
+    case OE_RRR(V4SHLSC, 0, X0):
+    case OE_RRR(V4SHLSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4SHL, 0, X0):
+    case OE_RRR(V4SHL, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_shl_i32);
+        mnemonic = "v4shl";
+        break;
+    case OE_RRR(V4SHRS, 0, X0):
+    case OE_RRR(V4SHRS, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_sar_i32);
+        mnemonic = "v4shrs";
+        break;
+    case OE_RRR(V4SHRU, 0, X0):
+    case OE_RRR(V4SHRU, 0, X1):
+        gen_v4sh(tdest, tsrca, tsrcb, tcg_gen_shr_i32);
+        mnemonic = "v4shru";
+        break;
+    case OE_RRR(V4SUBSC, 0, X0):
+    case OE_RRR(V4SUBSC, 0, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_RRR(V4SUB, 0, X0):
+    case OE_RRR(V4SUB, 0, X1):
+        gen_v4op(tdest, tsrca, tsrcb, tcg_gen_sub_i32);
+        mnemonic = "v2sub";
+        break;
+    case OE_RRR(XOR, 0, X0):
+    case OE_RRR(XOR, 0, X1):
+    case OE_RRR(XOR, 5, Y0):
+    case OE_RRR(XOR, 5, Y1):
+        tcg_gen_xor_tl(tdest, tsrca, tsrcb);
+        mnemonic = "xor";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %s", mnemonic,
+                  reg_names[dest], reg_names[srca], reg_names[srcb]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_rri_opcode(DisasContext *dc, unsigned opext,
+                               unsigned dest, unsigned srca, int imm)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    bool prefetch_nofault = false;
+    const char *mnemonic;
+    TCGMemOp memop;
+    int i2, i3;
+    TCGv t0;
+
+    switch (opext) {
+    case OE(ADDI_OPCODE_Y0, 0, Y0):
+    case OE(ADDI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ADDI, X0):
+    case OE_IM(ADDI, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        mnemonic = "addi";
+        break;
+    case OE(ADDXI_OPCODE_Y0, 0, Y0):
+    case OE(ADDXI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ADDXI, X0):
+    case OE_IM(ADDXI, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addxi";
+        break;
+    case OE(ANDI_OPCODE_Y0, 0, Y0):
+    case OE(ANDI_OPCODE_Y1, 0, Y1):
+    case OE_IM(ANDI, X0):
+    case OE_IM(ANDI, X1):
+        tcg_gen_andi_tl(tdest, tsrca, imm);
+        mnemonic = "andi";
+        break;
+    case OE(CMPEQI_OPCODE_Y0, 0, Y0):
+    case OE(CMPEQI_OPCODE_Y1, 0, Y1):
+    case OE_IM(CMPEQI, X0):
+    case OE_IM(CMPEQI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_EQ, tdest, tsrca, imm);
+        mnemonic = "cmpeqi";
+        break;
+    case OE(CMPLTSI_OPCODE_Y0, 0, Y0):
+    case OE(CMPLTSI_OPCODE_Y1, 0, Y1):
+    case OE_IM(CMPLTSI, X0):
+    case OE_IM(CMPLTSI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_LT, tdest, tsrca, imm);
+        mnemonic = "cmpltsi";
+        break;
+    case OE_IM(CMPLTUI, X0):
+    case OE_IM(CMPLTUI, X1):
+        tcg_gen_setcondi_tl(TCG_COND_LTU, tdest, tsrca, imm);
+        mnemonic = "cmpltui";
+        break;
+    case OE_IM(LD1S_ADD, X1):
+        memop = MO_SB;
+        mnemonic = "ld1s_add"; /* prefetch_add_l1_fault */
+        goto do_load_add;
+    case OE_IM(LD1U_ADD, X1):
+        memop = MO_UB;
+        mnemonic = "ld1u_add"; /* prefetch_add_l1 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LD2S_ADD, X1):
+        memop = MO_TESW;
+        mnemonic = "ld2s_add"; /* prefetch_add_l2_fault */
+        goto do_load_add;
+    case OE_IM(LD2U_ADD, X1):
+        memop = MO_TEUW;
+        mnemonic = "ld2u_add"; /* prefetch_add_l2 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LD4S_ADD, X1):
+        memop = MO_TESL;
+        mnemonic = "ld4s_add"; /* prefetch_add_l3_fault */
+        goto do_load_add;
+    case OE_IM(LD4U_ADD, X1):
+        memop = MO_TEUL;
+        mnemonic = "ld4u_add"; /* prefetch_add_l3 */
+        prefetch_nofault = (dest == TILEGX_R_ZERO);
+        goto do_load_add;
+    case OE_IM(LDNT1S_ADD, X1):
+        memop = MO_SB;
+        mnemonic = "ldnt1s_add";
+        goto do_load_add;
+    case OE_IM(LDNT1U_ADD, X1):
+        memop = MO_UB;
+        mnemonic = "ldnt1u_add";
+        goto do_load_add;
+    case OE_IM(LDNT2S_ADD, X1):
+        memop = MO_TESW;
+        mnemonic = "ldnt2s_add";
+        goto do_load_add;
+    case OE_IM(LDNT2U_ADD, X1):
+        memop = MO_TEUW;
+        mnemonic = "ldnt2u_add";
+        goto do_load_add;
+    case OE_IM(LDNT4S_ADD, X1):
+        memop = MO_TESL;
+        mnemonic = "ldnt4s_add";
+        goto do_load_add;
+    case OE_IM(LDNT4U_ADD, X1):
+        memop = MO_TEUL;
+        mnemonic = "ldnt4u_add";
+        goto do_load_add;
+    case OE_IM(LDNT_ADD, X1):
+        memop = MO_TEQ;
+        mnemonic = "ldnt_add";
+        goto do_load_add;
+    case OE_IM(LD_ADD, X1):
+        memop = MO_TEQ;
+        mnemonic = "ld_add";
+    do_load_add:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(tdest, tsrca, dc->mmuidx, memop);
+        }
+        tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+        break;
+    case OE_IM(LDNA_ADD, X1):
+        tcg_gen_andi_tl(tdest, tsrca, ~7);
+        tcg_gen_qemu_ld_tl(tdest, tdest, dc->mmuidx, MO_TEQ);
+        tcg_gen_addi_tl(dest_gr(dc, srca), tsrca, imm);
+        mnemonic = "ldna_add";
+        break;
+    case OE_IM(ORI, X0):
+    case OE_IM(ORI, X1):
+        tcg_gen_ori_tl(tdest, tsrca, imm);
+        mnemonic = "ori";
+        break;
+    case OE_IM(V1ADDI, X0):
+    case OE_IM(V1ADDI, X1):
+        t0 = tcg_const_tl(V1_IMM(imm));
+        gen_v12add(tdest, tsrca, t0, V1_IMM(0x80));
+        tcg_temp_free(t0);
+        mnemonic = "v1addi";
+        break;
+    case OE_IM(V1CMPEQI, X0):
+    case OE_IM(V1CMPEQI, X1):
+        tcg_gen_xori_tl(tdest, tsrca, V1_IMM(imm));
+        gen_v1cmpeq0(tdest);
+        mnemonic = "v1cmpeqi";
+        break;
+    case OE_IM(V1CMPLTSI, X0):
+    case OE_IM(V1CMPLTSI, X1):
+    case OE_IM(V1CMPLTUI, X0):
+    case OE_IM(V1CMPLTUI, X1):
+    case OE_IM(V1MAXUI, X0):
+    case OE_IM(V1MAXUI, X1):
+    case OE_IM(V1MINUI, X0):
+    case OE_IM(V1MINUI, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_IM(V2ADDI, X0):
+    case OE_IM(V2ADDI, X1):
+        t0 = tcg_const_tl(V2_IMM(imm));
+        gen_v12add(tdest, tsrca, t0, V2_IMM(0x8000));
+        tcg_temp_free(t0);
+        mnemonic = "v2addi";
+        break;
+    case OE_IM(V2CMPEQI, X0):
+    case OE_IM(V2CMPEQI, X1):
+    case OE_IM(V2CMPLTSI, X0):
+    case OE_IM(V2CMPLTSI, X1):
+    case OE_IM(V2CMPLTUI, X0):
+    case OE_IM(V2CMPLTUI, X1):
+    case OE_IM(V2MAXSI, X0):
+    case OE_IM(V2MAXSI, X1):
+    case OE_IM(V2MINSI, X0):
+    case OE_IM(V2MINSI, X1):
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    case OE_IM(XORI, X0):
+    case OE_IM(XORI, X1):
+        tcg_gen_xori_tl(tdest, tsrca, imm);
+        mnemonic = "xori";
+        break;
+
+    case OE_SH(ROTLI, X0):
+    case OE_SH(ROTLI, X1):
+    case OE_SH(ROTLI, Y0):
+    case OE_SH(ROTLI, Y1):
+        tcg_gen_rotli_tl(tdest, tsrca, imm);
+        mnemonic = "rotli";
+        break;
+    case OE_SH(SHLI, X0):
+    case OE_SH(SHLI, X1):
+    case OE_SH(SHLI, Y0):
+    case OE_SH(SHLI, Y1):
+        tcg_gen_shli_tl(tdest, tsrca, imm);
+        mnemonic = "shli";
+        break;
+    case OE_SH(SHLXI, X0):
+    case OE_SH(SHLXI, X1):
+        tcg_gen_shli_tl(tdest, tsrca, imm & 31);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "shlxi";
+        break;
+    case OE_SH(SHRSI, X0):
+    case OE_SH(SHRSI, X1):
+    case OE_SH(SHRSI, Y0):
+    case OE_SH(SHRSI, Y1):
+        tcg_gen_sari_tl(tdest, tsrca, imm);
+        mnemonic = "shrsi";
+        break;
+    case OE_SH(SHRUI, X0):
+    case OE_SH(SHRUI, X1):
+    case OE_SH(SHRUI, Y0):
+    case OE_SH(SHRUI, Y1):
+        tcg_gen_shri_tl(tdest, tsrca, imm);
+        mnemonic = "shrui";
+        break;
+    case OE_SH(SHRUXI, X0):
+    case OE_SH(SHRUXI, X1):
+        if ((imm & 31) == 0) {
+            tcg_gen_ext32s_tl(tdest, tsrca);
+        } else {
+            tcg_gen_ext32u_tl(tdest, tsrca);
+            tcg_gen_shri_tl(tdest, tdest, imm & 31);
+        }
+        mnemonic = "shlxi";
+        break;
+    case OE_SH(V1SHLI, X0):
+    case OE_SH(V1SHLI, X1):
+        i2 = imm & 7;
+        i3 = 0xff >> i2;
+        tcg_gen_andi_tl(tdest, tsrca, V1_IMM(i3));
+        tcg_gen_shli_tl(tdest, tdest, i2);
+        mnemonic = "v1shli";
+        break;
+    case OE_SH(V1SHRSI, X0):
+    case OE_SH(V1SHRSI, X1):
+        t0 = tcg_const_tl(imm & 7);
+        gen_helper_v1shrs(tdest, tsrca, t0);
+        tcg_temp_free(t0);
+        mnemonic = "v1shrsi";
+        break;
+    case OE_SH(V1SHRUI, X0):
+    case OE_SH(V1SHRUI, X1):
+        i2 = imm & 7;
+        i3 = (0xff << i2) & 0xff;
+        tcg_gen_andi_tl(tdest, tsrca, V1_IMM(i3));
+        tcg_gen_shri_tl(tdest, tdest, i2);
+        mnemonic = "v1shrui";
+        break;
+    case OE_SH(V2SHLI, X0):
+    case OE_SH(V2SHLI, X1):
+        i2 = imm & 15;
+        i3 = 0xffff >> i2;
+        tcg_gen_andi_tl(tdest, tsrca, V2_IMM(i3));
+        tcg_gen_shli_tl(tdest, tdest, i2);
+        mnemonic = "v2shli";
+        break;
+    case OE_SH(V2SHRSI, X0):
+    case OE_SH(V2SHRSI, X1):
+        t0 = tcg_const_tl(imm & 15);
+        gen_helper_v2shrs(tdest, tsrca, t0);
+        tcg_temp_free(t0);
+        mnemonic = "v2shrsi";
+        break;
+    case OE_SH(V2SHRUI, X0):
+    case OE_SH(V2SHRUI, X1):
+        i2 = imm & 15;
+        i3 = (0xffff << i2) & 0xffff;
+        tcg_gen_andi_tl(tdest, tsrca, V2_IMM(i3));
+        tcg_gen_shri_tl(tdest, tdest, i2);
+        mnemonic = "v2shrui";
+        break;
+
+    case OE(ADDLI_OPCODE_X0, 0, X0):
+    case OE(ADDLI_OPCODE_X1, 0, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        mnemonic = "addli";
+        break;
+    case OE(ADDXLI_OPCODE_X0, 0, X0):
+    case OE(ADDXLI_OPCODE_X1, 0, X1):
+        tcg_gen_addi_tl(tdest, tsrca, imm);
+        tcg_gen_ext32s_tl(tdest, tdest);
+        mnemonic = "addxli";
+        break;
+    case OE(SHL16INSLI_OPCODE_X0, 0, X0):
+    case OE(SHL16INSLI_OPCODE_X1, 0, X1):
+        tcg_gen_shli_tl(tdest, tsrca, 16);
+        tcg_gen_ori_tl(tdest, tdest, imm & 0xffff);
+        mnemonic = "shl16insli";
+        break;
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %d", mnemonic,
+                  reg_names[dest], reg_names[srca], imm);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_bf_opcode_x0(DisasContext *dc, unsigned ext,
+                                 unsigned dest, unsigned srca,
+                                 unsigned bfs, unsigned bfe)
+{
+    TCGv tdest = dest_gr(dc, dest);
+    TCGv tsrca = load_gr(dc, srca);
+    TCGv tsrcd;
+    int len;
+    const char *mnemonic;
+
+    /* The bitfield is either between E and S inclusive,
+       or up from S and down from E inclusive.  */
+    if (bfs <= bfe) {
+        len = bfe - bfs + 1;
+    } else {
+        len = (64 - bfs) + (bfe + 1);
+    }
+
+    switch (ext) {
+    case BFEXTU_BF_OPCODE_X0:
+        if (bfs == 0 && bfe == 7) {
+            tcg_gen_ext8u_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 15) {
+            tcg_gen_ext16u_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 31) {
+            tcg_gen_ext32u_tl(tdest, tsrca);
+        } else {
+            int rol = 63 - bfe;
+            if (bfs <= bfe) {
+                tcg_gen_shli_tl(tdest, tsrca, rol);
+            } else {
+                tcg_gen_rotli_tl(tdest, tsrca, rol);
+            }
+            tcg_gen_shri_tl(tdest, tdest, (bfs + rol) & 63);
+        }
+        mnemonic = "bfextu";
+        break;
+
+    case BFEXTS_BF_OPCODE_X0:
+        if (bfs == 0 && bfe == 7) {
+            tcg_gen_ext8s_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 15) {
+            tcg_gen_ext16s_tl(tdest, tsrca);
+        } else if (bfs == 0 && bfe == 31) {
+            tcg_gen_ext32s_tl(tdest, tsrca);
+        } else {
+            int rol = 63 - bfe;
+            if (bfs <= bfe) {
+                tcg_gen_shli_tl(tdest, tsrca, rol);
+            } else {
+                tcg_gen_rotli_tl(tdest, tsrca, rol);
+            }
+            tcg_gen_sari_tl(tdest, tdest, (bfs + rol) & 63);
+        }
+        mnemonic = "bfexts";
+        break;
+
+    case BFINS_BF_OPCODE_X0:
+        tsrcd = load_gr(dc, dest);
+        if (bfs <= bfe) {
+            tcg_gen_deposit_tl(tdest, tsrcd, tsrca, bfs, len);
+        } else {
+            tcg_gen_rotri_tl(tdest, tsrcd, bfs);
+            tcg_gen_deposit_tl(tdest, tdest, tsrca, 0, len);
+            tcg_gen_rotli_tl(tdest, tdest, bfs);
+        }
+        mnemonic = "bfins";
+        break;
+
+    case MM_BF_OPCODE_X0:
+        tsrcd = load_gr(dc, dest);
+        if (bfs == 0) {
+            tcg_gen_deposit_tl(tdest, tsrca, tsrcd, 0, len);
+        } else {
+            uint64_t mask = len == 64 ? -1 : rol64((1ULL << len) - 1, bfs);
+            TCGv tmp = tcg_const_tl(mask);
+
+            tcg_gen_and_tl(tdest, tsrcd, tmp);
+            tcg_gen_andc_tl(tmp, tsrca, tmp);
+            tcg_gen_or_tl(tdest, tdest, tmp);
+            tcg_temp_free(tmp);
+        }
+        mnemonic = "mm";
+        break;
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s, %u, %u", mnemonic,
+                  reg_names[dest], reg_names[srca], bfs, bfe);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_branch_opcode_x1(DisasContext *dc, unsigned ext,
+                                     unsigned srca, int off)
+{
+    target_ulong tgt = dc->pc + off * TILEGX_BUNDLE_SIZE_IN_BYTES;
+    const char *mnemonic;
+
+    dc->jmp.dest = tcg_const_tl(tgt);
+    dc->jmp.val1 = tcg_temp_new();
+    tcg_gen_mov_tl(dc->jmp.val1, load_gr(dc, srca));
+
+    /* Note that the "predict taken" opcodes have bit 0 clear.
+       Therefore, fold the two cases together by setting bit 0.  */
+    switch (ext | 1) {
+    case BEQZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_EQ;
+        mnemonic = "beqz";
+        break;
+    case BNEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_NE;
+        mnemonic = "bnez";
+        break;
+    case BGEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_GE;
+        mnemonic = "bgez";
+        break;
+    case BGTZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_GT;
+        mnemonic = "bgtz";
+        break;
+    case BLEZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_LE;
+        mnemonic = "blez";
+        break;
+    case BLTZ_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_LT;
+        mnemonic = "bltz";
+        break;
+    case BLBC_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_EQ;
+        tcg_gen_andi_tl(dc->jmp.val1, dc->jmp.val1, 1);
+        mnemonic = "blbc";
+        break;
+    case BLBS_BRANCH_OPCODE_X1:
+        dc->jmp.cond = TCG_COND_NE;
+        tcg_gen_andi_tl(dc->jmp.val1, dc->jmp.val1, 1);
+        mnemonic = "blbs";
+        break;
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("%s%s %s, " TARGET_FMT_lx " <%s>",
+                 mnemonic, ext & 1 ? "" : "t",
+                 reg_names[srca], tgt, lookup_symbol(tgt));
+    }
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_jump_opcode_x1(DisasContext *dc, unsigned ext, int off)
+{
+    target_ulong tgt = dc->pc + off * TILEGX_BUNDLE_SIZE_IN_BYTES;
+    const char *mnemonic = "j";
+
+    /* The extension field is 1 bit, therefore we only have JAL and J.  */
+    if (ext == JAL_JUMP_OPCODE_X1) {
+        tcg_gen_movi_tl(dest_gr(dc, TILEGX_R_LR),
+                        dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+        mnemonic = "jal";
+    }
+    dc->jmp.cond = TCG_COND_ALWAYS;
+    dc->jmp.dest = tcg_const_tl(tgt);
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("%s " TARGET_FMT_lx " <%s>",
+                 mnemonic, tgt, lookup_symbol(tgt));
+    }
+    return TILEGX_EXCP_NONE;
+}
+
+typedef struct {
+    const char *name;
+    intptr_t offset;
+    void (*get)(TCGv, TCGv_ptr);
+    void (*put)(TCGv_ptr, TCGv);
+} TileSPR;
+
+static const TileSPR *find_spr(unsigned spr)
+{
+    /* Allow the compiler to construct the binary search tree.  */
+#define D(N, O, G, P) \
+    case SPR_##N: { static const TileSPR x = { #N, O, G, P }; return &x; }
+
+    switch (spr) {
+    D(CMPEXCH_VALUE,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_CMPEXCH]), 0, 0)
+    D(INTERRUPT_CRITICAL_SECTION,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_CRITICAL_SEC]), 0, 0)
+    D(SIM_CONTROL,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_SIM_CONTROL]), 0, 0)
+    D(EX_CONTEXT_0_0,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_0]), 0, 0)
+    D(EX_CONTEXT_0_1,
+      offsetof(CPUTLGState, spregs[TILEGX_SPR_EX_CONTEXT_0_1]), 0, 0)
+    }
+
+#undef D
+
+    qemu_log_mask(LOG_UNIMP, "UNIMP SPR %u\n", spr);
+    return NULL;
+}
+
+static TileExcp gen_mtspr_x1(DisasContext *dc, unsigned spr, unsigned srca)
+{
+    const TileSPR *def = find_spr(spr);
+    TCGv tsrca;
+
+    if (def == NULL) {
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr spr[%u], %s", spr, reg_names[srca]);
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    }
+
+    tsrca = load_gr(dc, srca);
+    if (def->put) {
+        def->put(cpu_env, tsrca);
+    } else {
+        tcg_gen_st_tl(tsrca, cpu_env, def->offset);
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr %s, %s", def->name, reg_names[srca]);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp gen_mfspr_x1(DisasContext *dc, unsigned dest, unsigned spr)
+{
+    const TileSPR *def = find_spr(spr);
+    TCGv tdest;
+
+    if (def == NULL) {
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "mtspr %s, spr[%u]", reg_names[dest], spr);
+        return TILEGX_EXCP_OPCODE_UNIMPLEMENTED;
+    }
+
+    tdest = dest_gr(dc, dest);
+    if (def->get) {
+        def->get(tdest, cpu_env);
+    } else {
+        tcg_gen_ld_tl(tdest, cpu_env, def->offset);
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "mfspr %s, %s", reg_names[dest], def->name);
+    return TILEGX_EXCP_NONE;
+}
+
+static TileExcp decode_y0(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_Y0(bundle);
+    unsigned ext = get_RRROpcodeExtension_Y0(bundle);
+    unsigned dest = get_Dest_Y0(bundle);
+    unsigned srca = get_SrcA_Y0(bundle);
+    unsigned srcb;
+    int imm;
+
+    switch (opc) {
+    case RRR_1_OPCODE_Y0:
+        if (ext == UNARY_RRR_1_OPCODE_Y0) {
+            ext = get_UnaryOpcodeExtension_Y0(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, Y0), dest, srca, bundle);
+        }
+        /* fallthru */
+    case RRR_0_OPCODE_Y0:
+    case RRR_2_OPCODE_Y0:
+    case RRR_3_OPCODE_Y0:
+    case RRR_4_OPCODE_Y0:
+    case RRR_5_OPCODE_Y0:
+    case RRR_6_OPCODE_Y0:
+    case RRR_7_OPCODE_Y0:
+    case RRR_8_OPCODE_Y0:
+    case RRR_9_OPCODE_Y0:
+        srcb = get_SrcB_Y0(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, Y0), dest, srca, srcb);
+
+    case SHIFT_OPCODE_Y0:
+        ext = get_ShiftOpcodeExtension_Y0(bundle);
+        imm = get_ShAmt_Y0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, Y0), dest, srca, imm);
+
+    case ADDI_OPCODE_Y0:
+    case ADDXI_OPCODE_Y0:
+    case ANDI_OPCODE_Y0:
+    case CMPEQI_OPCODE_Y0:
+    case CMPLTSI_OPCODE_Y0:
+        imm = (int8_t)get_Imm8_Y0(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, Y0), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_y1(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_Y1(bundle);
+    unsigned ext = get_RRROpcodeExtension_Y1(bundle);
+    unsigned dest = get_Dest_Y1(bundle);
+    unsigned srca = get_SrcA_Y1(bundle);
+    unsigned srcb;
+    int imm;
+
+    switch (get_Opcode_Y1(bundle)) {
+    case RRR_1_OPCODE_Y1:
+        if (ext == UNARY_RRR_1_OPCODE_Y0) {
+            ext = get_UnaryOpcodeExtension_Y1(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, Y1), dest, srca, bundle);
+        }
+        /* fallthru */
+    case RRR_0_OPCODE_Y1:
+    case RRR_2_OPCODE_Y1:
+    case RRR_3_OPCODE_Y1:
+    case RRR_4_OPCODE_Y1:
+    case RRR_5_OPCODE_Y1:
+    case RRR_6_OPCODE_Y1:
+    case RRR_7_OPCODE_Y1:
+        srcb = get_SrcB_Y1(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, Y1), dest, srca, srcb);
+
+    case SHIFT_OPCODE_Y1:
+        ext = get_ShiftOpcodeExtension_Y1(bundle);
+        imm = get_ShAmt_Y1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, Y1), dest, srca, imm);
+
+    case ADDI_OPCODE_Y1:
+    case ADDXI_OPCODE_Y1:
+    case ANDI_OPCODE_Y1:
+    case CMPEQI_OPCODE_Y1:
+    case CMPLTSI_OPCODE_Y1:
+        imm = (int8_t)get_Imm8_Y1(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, Y1), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_y2(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned mode = get_Mode(bundle);
+    unsigned opc = get_Opcode_Y2(bundle);
+    unsigned srca = get_SrcA_Y2(bundle);
+    unsigned srcbdest = get_SrcBDest_Y2(bundle);
+    const char *mnemonic;
+    TCGMemOp memop;
+    bool prefetch_nofault = false;
+
+    switch (OEY2(opc, mode)) {
+    case OEY2(LD1S_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_SB;
+        mnemonic = "ld1s"; /* prefetch_l1_fault */
+        goto do_load;
+    case OEY2(LD1U_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_UB;
+        mnemonic = "ld1u"; /* prefetch, prefetch_l1 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD2S_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_TESW;
+        mnemonic = "ld2s"; /* prefetch_l2_fault */
+        goto do_load;
+    case OEY2(LD2U_OPCODE_Y2, MODE_OPCODE_YA2):
+        memop = MO_TEUW;
+        mnemonic = "ld2u"; /* prefetch_l2 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD4S_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TESL;
+        mnemonic = "ld4s"; /* prefetch_l3_fault */
+        goto do_load;
+    case OEY2(LD4U_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TEUL;
+        mnemonic = "ld4u"; /* prefetch_l3 */
+        prefetch_nofault = (srcbdest == TILEGX_R_ZERO);
+        goto do_load;
+    case OEY2(LD_OPCODE_Y2, MODE_OPCODE_YB2):
+        memop = MO_TEQ;
+        mnemonic = "ld";
+    do_load:
+        if (!prefetch_nofault) {
+            tcg_gen_qemu_ld_tl(dest_gr(dc, srcbdest), load_gr(dc, srca),
+                               dc->mmuidx, memop);
+        }
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "%s %s, %s", mnemonic,
+                      reg_names[srcbdest], reg_names[srca]);
+        return TILEGX_EXCP_NONE;
+
+    case OEY2(ST1_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_UB, "st1");
+    case OEY2(ST2_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEUW, "st2");
+    case OEY2(ST4_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEUL, "st4");
+    case OEY2(ST_OPCODE_Y2, MODE_OPCODE_YC2):
+        return gen_st_opcode(dc, 0, srca, srcbdest, MO_TEQ, "st");
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_x0(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_X0(bundle);
+    unsigned dest = get_Dest_X0(bundle);
+    unsigned srca = get_SrcA_X0(bundle);
+    unsigned ext, srcb, bfs, bfe;
+    int imm;
+
+    switch (opc) {
+    case RRR_0_OPCODE_X0:
+        ext = get_RRROpcodeExtension_X0(bundle);
+        if (ext == UNARY_RRR_0_OPCODE_X0) {
+            ext = get_UnaryOpcodeExtension_X0(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, X0), dest, srca, bundle);
+        }
+        srcb = get_SrcB_X0(bundle);
+        return gen_rrr_opcode(dc, OE(opc, ext, X0), dest, srca, srcb);
+
+    case SHIFT_OPCODE_X0:
+        ext = get_ShiftOpcodeExtension_X0(bundle);
+        imm = get_ShAmt_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X0), dest, srca, imm);
+
+    case IMM8_OPCODE_X0:
+        ext = get_Imm8OpcodeExtension_X0(bundle);
+        imm = (int8_t)get_Imm8_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X0), dest, srca, imm);
+
+    case BF_OPCODE_X0:
+        ext = get_BFOpcodeExtension_X0(bundle);
+        bfs = get_BFStart_X0(bundle);
+        bfe = get_BFEnd_X0(bundle);
+        return gen_bf_opcode_x0(dc, ext, dest, srca, bfs, bfe);
+
+    case ADDLI_OPCODE_X0:
+    case SHL16INSLI_OPCODE_X0:
+    case ADDXLI_OPCODE_X0:
+        imm = (int16_t)get_Imm16_X0(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, X0), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static TileExcp decode_x1(DisasContext *dc, tilegx_bundle_bits bundle)
+{
+    unsigned opc = get_Opcode_X1(bundle);
+    unsigned dest = get_Dest_X1(bundle);
+    unsigned srca = get_SrcA_X1(bundle);
+    unsigned ext, srcb;
+    int imm;
+
+    switch (opc) {
+    case RRR_0_OPCODE_X1:
+        ext = get_RRROpcodeExtension_X1(bundle);
+        srcb = get_SrcB_X1(bundle);
+        switch (ext) {
+        case UNARY_RRR_0_OPCODE_X1:
+            ext = get_UnaryOpcodeExtension_X1(bundle);
+            return gen_rr_opcode(dc, OE(opc, ext, X1), dest, srca, bundle);
+        case ST1_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_UB, "st1");
+        case ST2_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUW, "st2");
+        case ST4_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUL, "st4");
+        case STNT1_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_UB, "stnt1");
+        case STNT2_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUW, "stnt2");
+        case STNT4_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEUL, "stnt4");
+        case STNT_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEQ, "stnt");
+        case ST_RRR_0_OPCODE_X1:
+            return gen_st_opcode(dc, dest, srca, srcb, MO_TEQ, "st");
+        }
+        return gen_rrr_opcode(dc, OE(opc, ext, X1), dest, srca, srcb);
+
+    case SHIFT_OPCODE_X1:
+        ext = get_ShiftOpcodeExtension_X1(bundle);
+        imm = get_ShAmt_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X1), dest, srca, imm);
+
+    case IMM8_OPCODE_X1:
+        ext = get_Imm8OpcodeExtension_X1(bundle);
+        imm = (int8_t)get_Dest_Imm8_X1(bundle);
+        srcb = get_SrcB_X1(bundle);
+        switch (ext) {
+        case ST1_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_UB, "st1_add");
+        case ST2_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUW, "st2_add");
+        case ST4_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUL, "st4_add");
+        case STNT1_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_UB, "stnt1_add");
+        case STNT2_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUW, "stnt2_add");
+        case STNT4_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEUL, "stnt4_add");
+        case STNT_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEQ, "stnt_add");
+        case ST_ADD_IMM8_OPCODE_X1:
+            return gen_st_add_opcode(dc, srca, srcb, imm, MO_TEQ, "st_add");
+        case MFSPR_IMM8_OPCODE_X1:
+            return gen_mfspr_x1(dc, dest, get_MF_Imm14_X1(bundle));
+        case MTSPR_IMM8_OPCODE_X1:
+            return gen_mtspr_x1(dc, get_MT_Imm14_X1(bundle), srca);
+        }
+        imm = (int8_t)get_Imm8_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, ext, X1), dest, srca, imm);
+
+    case BRANCH_OPCODE_X1:
+        ext = get_BrType_X1(bundle);
+        imm = sextract32(get_BrOff_X1(bundle), 0, 17);
+        return gen_branch_opcode_x1(dc, ext, srca, imm);
+
+    case JUMP_OPCODE_X1:
+        ext = get_JumpOpcodeExtension_X1(bundle);
+        imm = sextract32(get_JumpOff_X1(bundle), 0, 27);
+        return gen_jump_opcode_x1(dc, ext, imm);
+
+    case ADDLI_OPCODE_X1:
+    case SHL16INSLI_OPCODE_X1:
+    case ADDXLI_OPCODE_X1:
+        imm = (int16_t)get_Imm16_X1(bundle);
+        return gen_rri_opcode(dc, OE(opc, 0, X1), dest, srca, imm);
+
+    default:
+        return TILEGX_EXCP_OPCODE_UNKNOWN;
+    }
+}
+
+static void notice_excp(DisasContext *dc, uint64_t bundle,
+                        const char *type, TileExcp excp)
+{
+    if (likely(excp == TILEGX_EXCP_NONE)) {
+        return;
+    }
+    gen_exception(dc, excp);
+    switch (excp) {
+    case TILEGX_EXCP_OPCODE_UNIMPLEMENTED:
+        qemu_log_mask(LOG_UNIMP, "UNIMP %s, [" FMT64X "]\n", type, bundle);
+        break;
+    case TILEGX_EXCP_OPCODE_UNKNOWN:
+        qemu_log_mask(LOG_UNIMP, "UNKNOWN %s, [" FMT64X "]\n", type, bundle);
+        break;
+    default:
+        break;
+    }
+}
+
+static void translate_one_bundle(DisasContext *dc, uint64_t bundle)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(dc->wb); i++) {
+        DisasContextTemp *wb = &dc->wb[i];
+        wb->reg = TILEGX_R_NOREG;
+        TCGV_UNUSED_I64(wb->val);
+    }
+    dc->num_wb = 0;
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "  %" PRIx64 ":  { ", dc->pc);
+    if (get_Mode(bundle)) {
+        notice_excp(dc, bundle, "y0", decode_y0(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "y1", decode_y1(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "y2", decode_y2(dc, bundle));
+    } else {
+        notice_excp(dc, bundle, "x0", decode_x0(dc, bundle));
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " ; ");
+        notice_excp(dc, bundle, "x1", decode_x1(dc, bundle));
+    }
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, " }\n");
+
+    for (i = dc->num_wb - 1; i >= 0; --i) {
+        DisasContextTemp *wb = &dc->wb[i];
+        if (wb->reg < TILEGX_R_COUNT) {
+            tcg_gen_mov_i64(cpu_regs[wb->reg], wb->val);
+        }
+        tcg_temp_free_i64(wb->val);
+    }
+
+    if (dc->jmp.cond != TCG_COND_NEVER) {
+        if (dc->jmp.cond == TCG_COND_ALWAYS) {
+            tcg_gen_mov_i64(cpu_pc, dc->jmp.dest);
+        } else {
+            TCGv next = tcg_const_i64(dc->pc + TILEGX_BUNDLE_SIZE_IN_BYTES);
+            tcg_gen_movcond_i64(dc->jmp.cond, cpu_pc,
+                                dc->jmp.val1, load_zero(dc),
+                                dc->jmp.dest, next);
+            tcg_temp_free_i64(dc->jmp.val1);
+            tcg_temp_free_i64(next);
+        }
+        tcg_temp_free_i64(dc->jmp.dest);
+        tcg_gen_exit_tb(0);
+        dc->exit_tb = true;
+    } else if (dc->atomic_excp != TILEGX_EXCP_NONE) {
+        gen_exception(dc, dc->atomic_excp);
+    }
+}
+
+void gen_intermediate_code(CPUTLGState *env, struct TranslationBlock *tb)
+{
+    TileGXCPU *cpu = tilegx_env_get_cpu(env);
+    DisasContext ctx;
+    DisasContext *dc = &ctx;
+    CPUState *cs = CPU(cpu);
+    uint64_t pc_start = tb->pc;
+    uint64_t next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+    int num_insns = 0;
+    int max_insns = tb->cflags & CF_COUNT_MASK;
+
+    dc->pc = pc_start;
+    dc->mmuidx = 0;
+    dc->exit_tb = false;
+    dc->atomic_excp = TILEGX_EXCP_NONE;
+    dc->jmp.cond = TCG_COND_NEVER;
+    TCGV_UNUSED_I64(dc->jmp.dest);
+    TCGV_UNUSED_I64(dc->jmp.val1);
+    TCGV_UNUSED_I64(dc->zero);
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+    }
+    if (!max_insns) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (cs->singlestep_enabled || singlestep) {
+        max_insns = 1;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+    gen_tb_start(tb);
+
+    while (1) {
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        translate_one_bundle(dc, cpu_ldq_data(env, dc->pc));
+
+        if (dc->exit_tb) {
+            /* PC updated and EXIT_TB/GOTO_TB/exception emitted.  */
+            break;
+        }
+        dc->pc += TILEGX_BUNDLE_SIZE_IN_BYTES;
+        if (num_insns >= max_insns
+            || dc->pc >= next_page_start
+            || tcg_op_buf_full()) {
+            /* Ending the TB due to TB size or page boundary.  Set PC.  */
+            tcg_gen_movi_tl(cpu_pc, dc->pc);
+            tcg_gen_exit_tb(0);
+            break;
+        }
+    }
+
+    gen_tb_end(tb, num_insns);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
+
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "\n");
+}
+
+void restore_state_to_opc(CPUTLGState *env, TranslationBlock *tb,
+                          target_ulong *data)
+{
+    env->pc = data[0];
+}
+
+void tilegx_tcg_init(void)
+{
+    int i;
+
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
+    cpu_pc = tcg_global_mem_new_i64(cpu_env, offsetof(CPUTLGState, pc), "pc");
+    for (i = 0; i < TILEGX_R_COUNT; i++) {
+        cpu_regs[i] = tcg_global_mem_new_i64(cpu_env,
+                                             offsetof(CPUTLGState, regs[i]),
+                                             reg_names[i]);
+    }
+}
diff --git a/target-tricore/Makefile.objs b/target-tricore/Makefile.objs
index 21e820d..7a05670 100644
--- a/target-tricore/Makefile.objs
+++ b/target-tricore/Makefile.objs
@@ -1 +1 @@
-obj-y += translate.o helper.o cpu.o op_helper.o
+obj-y += translate.o helper.o cpu.o op_helper.o fpu_helper.o
diff --git a/target-tricore/cpu-qom.h b/target-tricore/cpu-qom.h
index 66c9664..6a69756 100644
--- a/target-tricore/cpu-qom.h
+++ b/target-tricore/cpu-qom.h
@@ -39,32 +39,6 @@
     void (*parent_reset)(CPUState *cpu);
 } TriCoreCPUClass;
 
-/**
- * TriCoreCPU:
- * @env: #CPUTriCoreState
- *
- * A TriCore CPU.
- */
-typedef struct TriCoreCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
+typedef struct TriCoreCPU TriCoreCPU;
 
-    CPUTriCoreState env;
-} TriCoreCPU;
-
-static inline TriCoreCPU *tricore_env_get_cpu(CPUTriCoreState *env)
-{
-    return TRICORE_CPU(container_of(env, TriCoreCPU, env));
-}
-
-#define ENV_GET_CPU(e) CPU(tricore_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(TriCoreCPU, env)
-
-hwaddr tricore_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-void tricore_cpu_dump_state(CPUState *cpu, FILE *f,
-                            fprintf_function cpu_fprintf, int flags);
-
-
-#endif /*QEMU_TRICORE_CPU_QOM_H */
+#endif /* QEMU_TRICORE_CPU_QOM_H */
diff --git a/target-tricore/cpu.c b/target-tricore/cpu.c
index 7bf041a..35d4ee4 100644
--- a/target-tricore/cpu.c
+++ b/target-tricore/cpu.c
@@ -17,8 +17,11 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
+#include "exec/exec-all.h"
 
 static inline void set_feature(CPUTriCoreState *env, int feature)
 {
@@ -63,8 +66,21 @@
 static void tricore_cpu_realizefn(DeviceState *dev, Error **errp)
 {
     CPUState *cs = CPU(dev);
+    TriCoreCPU *cpu = TRICORE_CPU(dev);
     TriCoreCPUClass *tcc = TRICORE_CPU_GET_CLASS(dev);
+    CPUTriCoreState *env = &cpu->env;
 
+    /* Some features automatically imply others */
+    if (tricore_feature(env, TRICORE_FEATURE_161)) {
+        set_feature(env, TRICORE_FEATURE_16);
+    }
+
+    if (tricore_feature(env, TRICORE_FEATURE_16)) {
+        set_feature(env, TRICORE_FEATURE_131);
+    }
+    if (tricore_feature(env, TRICORE_FEATURE_131)) {
+        set_feature(env, TRICORE_FEATURE_13);
+    }
     cpu_reset(cs);
     qemu_init_vcpu(cs);
 
@@ -79,7 +95,7 @@
     CPUTriCoreState *env = &cpu->env;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     if (tcg_enabled()) {
         tricore_tcg_init();
@@ -112,11 +128,18 @@
     set_feature(&cpu->env, TRICORE_FEATURE_13);
 }
 
-static void aurix_initfn(Object *obj)
+static void tc1797_initfn(Object *obj)
 {
     TriCoreCPU *cpu = TRICORE_CPU(obj);
 
-    set_feature(&cpu->env, TRICORE_FEATURE_16);
+    set_feature(&cpu->env, TRICORE_FEATURE_131);
+}
+
+static void tc27x_initfn(Object *obj)
+{
+    TriCoreCPU *cpu = TRICORE_CPU(obj);
+
+    set_feature(&cpu->env, TRICORE_FEATURE_161);
 }
 
 typedef struct TriCoreCPUInfo {
@@ -127,7 +150,8 @@
 
 static const TriCoreCPUInfo tricore_cpus[] = {
     { .name = "tc1796",      .initfn = tc1796_initfn },
-    { .name = "aurix",       .initfn = aurix_initfn },
+    { .name = "tc1797",      .initfn = tc1797_initfn },
+    { .name = "tc27x",       .initfn = tc27x_initfn },
     { .name = NULL }
 };
 
@@ -149,6 +173,12 @@
     cc->set_pc = tricore_cpu_set_pc;
     cc->synchronize_from_tb = tricore_cpu_synchronize_from_tb;
 
+    /*
+     * Reason: tricore_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void cpu_register(const TriCoreCPUInfo *info)
diff --git a/target-tricore/cpu.h b/target-tricore/cpu.h
index 7555b70..a3493a1 100644
--- a/target-tricore/cpu.h
+++ b/target-tricore/cpu.h
@@ -16,17 +16,16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#if !defined(__TRICORE_CPU_H__)
-#define __TRICORE_CPU_H__
+
+#ifndef TRICORE_CPU_H
+#define TRICORE_CPU_H
 
 #include "tricore-defs.h"
-#include "config.h"
 #include "qemu-common.h"
+#include "cpu-qom.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
 
-#define ELF_MACHINE     EM_TRICORE
-
 #define CPUArchState struct CPUTriCoreState
 
 struct CPUTriCoreState;
@@ -186,8 +185,7 @@
     uint32_t M2CNT;
     uint32_t M3CNT;
     /* Floating Point Registers */
-    /* XXX: */
-
+    float_status fp_status;
     /* QEMU */
     int error_code;
     uint32_t hflags;    /* CPU State */
@@ -202,6 +200,34 @@
     struct QEMUTimer *timer; /* Internal timer */
 };
 
+/**
+ * TriCoreCPU:
+ * @env: #CPUTriCoreState
+ *
+ * A TriCore CPU.
+ */
+struct TriCoreCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUTriCoreState env;
+};
+
+static inline TriCoreCPU *tricore_env_get_cpu(CPUTriCoreState *env)
+{
+    return TRICORE_CPU(container_of(env, TriCoreCPU, env));
+}
+
+#define ENV_GET_CPU(e) CPU(tricore_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(TriCoreCPU, env)
+
+hwaddr tricore_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+void tricore_cpu_dump_state(CPUState *cpu, FILE *f,
+                            fprintf_function cpu_fprintf, int flags);
+
+
 #define MASK_PCXI_PCPN 0xff000000
 #define MASK_PCXI_PIE  0x00800000
 #define MASK_PCXI_UL   0x00400000
@@ -220,6 +246,7 @@
 #define MASK_PSW_GW  0x00000100
 #define MASK_PSW_CDE 0x00000080
 #define MASK_PSW_CDC 0x0000007f
+#define MASK_PSW_FPU_RM 0x3000000
 
 #define MASK_SYSCON_PRO_TEN 0x2
 #define MASK_SYSCON_FCD_SF  0x1
@@ -238,6 +265,14 @@
 #define MASK_LCX_LCXS 0x000f0000
 #define MASK_LCX_LCX0 0x0000ffff
 
+#define MASK_DBGSR_DE 0x1
+#define MASK_DBGSR_HALT 0x6
+#define MASK_DBGSR_SUSP 0x10
+#define MASK_DBGSR_PREVSUSP 0x20
+#define MASK_DBGSR_PEVT 0x40
+#define MASK_DBGSR_EVTSRC 0x1f00
+
+#define TRICORE_HFLAG_KUU     0x3
 #define TRICORE_HFLAG_UM0     0x00002 /* user mode-0 flag          */
 #define TRICORE_HFLAG_UM1     0x00001 /* user mode-1 flag          */
 #define TRICORE_HFLAG_SM      0x00000 /* kernel mode flag          */
@@ -246,6 +281,7 @@
     TRICORE_FEATURE_13,
     TRICORE_FEATURE_131,
     TRICORE_FEATURE_16,
+    TRICORE_FEATURE_161,
 };
 
 static inline int tricore_feature(CPUTriCoreState *env, int feature)
@@ -264,6 +300,7 @@
     TRAPC_ASSERT   = 5,
     TRAPC_SYSCALL  = 6,
     TRAPC_NMI      = 7,
+    TRAPC_IRQ      = 8
 };
 
 /* Class 0 TIN */
@@ -332,16 +369,16 @@
 uint32_t psw_read(CPUTriCoreState *env);
 void psw_write(CPUTriCoreState *env, uint32_t val);
 
-#include "cpu-qom.h"
+void fpu_set_state(CPUTriCoreState *env);
+
 #define MMU_USER_IDX 2
 
 void tricore_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 
-#define cpu_exec cpu_tricore_exec
 #define cpu_signal_handler cpu_tricore_signal_handler
 #define cpu_list tricore_cpu_list
 
-static inline int cpu_mmu_index(CPUTriCoreState *env)
+static inline int cpu_mmu_index(CPUTriCoreState *env, bool ifetch)
 {
     return 0;
 }
@@ -363,12 +400,11 @@
 };
 
 void cpu_state_reset(CPUTriCoreState *s);
-int cpu_tricore_exec(CPUTriCoreState *s);
 void tricore_tcg_init(void);
 int cpu_tricore_signal_handler(int host_signum, void *pinfo, void *puc);
 
 static inline void cpu_get_tb_cpu_state(CPUTriCoreState *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->PC;
     *cs_base = 0;
@@ -377,15 +413,7 @@
 
 TriCoreCPU *cpu_tricore_init(const char *cpu_model);
 
-static inline CPUTriCoreState *cpu_init(const char *cpu_model)
-{
-    TriCoreCPU *cpu = cpu_tricore_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-
-}
+#define cpu_init(cpu_model) CPU(cpu_tricore_init(cpu_model))
 
 
 /* helpers.c */
@@ -393,11 +421,4 @@
                                  int rw, int mmu_idx);
 #define cpu_handle_mmu_fault cpu_tricore_handle_mmu_fault
 
-#include "exec/exec-all.h"
-
-static inline void cpu_pc_from_tb(CPUTriCoreState *env, TranslationBlock *tb)
-{
-    env->PC = tb->pc;
-}
-
-#endif /*__TRICORE_CPU_H__ */
+#endif /* TRICORE_CPU_H */
diff --git a/target-tricore/csfr.def b/target-tricore/csfr.def
new file mode 100644
index 0000000..05c45dd
--- /dev/null
+++ b/target-tricore/csfr.def
@@ -0,0 +1,124 @@
+/* A(ll) access permited
+   R(ead only) access
+   E(nd init protected) access
+
+   A|R|E(offset, register, feature introducing reg)
+
+   NOTE: PSW is handled as a special case in gen_mtcr/mfcr */
+
+A(0xfe00, PCXI, TRICORE_FEATURE_13)
+A(0xfe08, PC, TRICORE_FEATURE_13)
+A(0xfe14, SYSCON, TRICORE_FEATURE_13)
+R(0xfe18, CPU_ID, TRICORE_FEATURE_13)
+E(0xfe20, BIV, TRICORE_FEATURE_13)
+E(0xfe24, BTV, TRICORE_FEATURE_13)
+E(0xfe28, ISP, TRICORE_FEATURE_13)
+A(0xfe2c, ICR, TRICORE_FEATURE_13)
+A(0xfe38, FCX, TRICORE_FEATURE_13)
+A(0xfe3c, LCX, TRICORE_FEATURE_13)
+E(0x9400, COMPAT, TRICORE_FEATURE_131)
+/* memory protection register */
+A(0xC000, DPR0_0L, TRICORE_FEATURE_13)
+A(0xC004, DPR0_0U, TRICORE_FEATURE_13)
+A(0xC008, DPR0_1L, TRICORE_FEATURE_13)
+A(0xC00C, DPR0_1U, TRICORE_FEATURE_13)
+A(0xC010, DPR0_2L, TRICORE_FEATURE_13)
+A(0xC014, DPR0_2U, TRICORE_FEATURE_13)
+A(0xC018, DPR0_3L, TRICORE_FEATURE_13)
+A(0xC01C, DPR0_3U, TRICORE_FEATURE_13)
+A(0xC400, DPR1_0L, TRICORE_FEATURE_13)
+A(0xC404, DPR1_0U, TRICORE_FEATURE_13)
+A(0xC408, DPR1_1L, TRICORE_FEATURE_13)
+A(0xC40C, DPR1_1U, TRICORE_FEATURE_13)
+A(0xC410, DPR1_2L, TRICORE_FEATURE_13)
+A(0xC414, DPR1_2U, TRICORE_FEATURE_13)
+A(0xC418, DPR1_3L, TRICORE_FEATURE_13)
+A(0xC41C, DPR1_3U, TRICORE_FEATURE_13)
+A(0xC800, DPR2_0L, TRICORE_FEATURE_13)
+A(0xC804, DPR2_0U, TRICORE_FEATURE_13)
+A(0xC808, DPR2_1L, TRICORE_FEATURE_13)
+A(0xC80C, DPR2_1U, TRICORE_FEATURE_13)
+A(0xC810, DPR2_2L, TRICORE_FEATURE_13)
+A(0xC814, DPR2_2U, TRICORE_FEATURE_13)
+A(0xC818, DPR2_3L, TRICORE_FEATURE_13)
+A(0xC81C, DPR2_3U, TRICORE_FEATURE_13)
+A(0xCC00, DPR3_0L, TRICORE_FEATURE_13)
+A(0xCC04, DPR3_0U, TRICORE_FEATURE_13)
+A(0xCC08, DPR3_1L, TRICORE_FEATURE_13)
+A(0xCC0C, DPR3_1U, TRICORE_FEATURE_13)
+A(0xCC10, DPR3_2L, TRICORE_FEATURE_13)
+A(0xCC14, DPR3_2U, TRICORE_FEATURE_13)
+A(0xCC18, DPR3_3L, TRICORE_FEATURE_13)
+A(0xCC1C, DPR3_3U, TRICORE_FEATURE_13)
+A(0xD000, CPR0_0L, TRICORE_FEATURE_13)
+A(0xD004, CPR0_0U, TRICORE_FEATURE_13)
+A(0xD008, CPR0_1L, TRICORE_FEATURE_13)
+A(0xD00C, CPR0_1U, TRICORE_FEATURE_13)
+A(0xD010, CPR0_2L, TRICORE_FEATURE_13)
+A(0xD014, CPR0_2U, TRICORE_FEATURE_13)
+A(0xD018, CPR0_3L, TRICORE_FEATURE_13)
+A(0xD01C, CPR0_3U, TRICORE_FEATURE_13)
+A(0xD400, CPR1_0L, TRICORE_FEATURE_13)
+A(0xD404, CPR1_0U, TRICORE_FEATURE_13)
+A(0xD408, CPR1_1L, TRICORE_FEATURE_13)
+A(0xD40C, CPR1_1U, TRICORE_FEATURE_13)
+A(0xD410, CPR1_2L, TRICORE_FEATURE_13)
+A(0xD414, CPR1_2U, TRICORE_FEATURE_13)
+A(0xD418, CPR1_3L, TRICORE_FEATURE_13)
+A(0xD41C, CPR1_3U, TRICORE_FEATURE_13)
+A(0xD800, CPR2_0L, TRICORE_FEATURE_13)
+A(0xD804, CPR2_0U, TRICORE_FEATURE_13)
+A(0xD808, CPR2_1L, TRICORE_FEATURE_13)
+A(0xD80C, CPR2_1U, TRICORE_FEATURE_13)
+A(0xD810, CPR2_2L, TRICORE_FEATURE_13)
+A(0xD814, CPR2_2U, TRICORE_FEATURE_13)
+A(0xD818, CPR2_3L, TRICORE_FEATURE_13)
+A(0xD81C, CPR2_3U, TRICORE_FEATURE_13)
+A(0xDC00, CPR3_0L, TRICORE_FEATURE_13)
+A(0xDC04, CPR3_0U, TRICORE_FEATURE_13)
+A(0xDC08, CPR3_1L, TRICORE_FEATURE_13)
+A(0xDC0C, CPR3_1U, TRICORE_FEATURE_13)
+A(0xDC10, CPR3_2L, TRICORE_FEATURE_13)
+A(0xDC14, CPR3_2U, TRICORE_FEATURE_13)
+A(0xDC18, CPR3_3L, TRICORE_FEATURE_13)
+A(0xDC1C, CPR3_3U, TRICORE_FEATURE_13)
+A(0xE000, DPM0, TRICORE_FEATURE_13)
+A(0xE080, DPM1, TRICORE_FEATURE_13)
+A(0xE100, DPM2, TRICORE_FEATURE_13)
+A(0xE180, DPM3, TRICORE_FEATURE_13)
+A(0xE200, CPM0, TRICORE_FEATURE_13)
+A(0xE280, CPM1, TRICORE_FEATURE_13)
+A(0xE300, CPM2, TRICORE_FEATURE_13)
+A(0xE380, CPM3, TRICORE_FEATURE_13)
+/* memory management registers */
+A(0x8000, MMU_CON, TRICORE_FEATURE_13)
+A(0x8004, MMU_ASI, TRICORE_FEATURE_13)
+A(0x800C, MMU_TVA, TRICORE_FEATURE_13)
+A(0x8010, MMU_TPA, TRICORE_FEATURE_13)
+A(0x8014, MMU_TPX, TRICORE_FEATURE_13)
+A(0x8018, MMU_TFA, TRICORE_FEATURE_13)
+E(0x9004, BMACON, TRICORE_FEATURE_131)
+E(0x900C, SMACON, TRICORE_FEATURE_131)
+A(0x9020, DIEAR, TRICORE_FEATURE_131)
+A(0x9024, DIETR, TRICORE_FEATURE_131)
+A(0x9028, CCDIER, TRICORE_FEATURE_131)
+E(0x9044, MIECON, TRICORE_FEATURE_131)
+A(0x9210, PIEAR, TRICORE_FEATURE_131)
+A(0x9214, PIETR, TRICORE_FEATURE_131)
+A(0x9218, CCPIER, TRICORE_FEATURE_131)
+/* debug registers */
+A(0xFD00, DBGSR, TRICORE_FEATURE_13)
+A(0xFD08, EXEVT, TRICORE_FEATURE_13)
+A(0xFD0C, CREVT, TRICORE_FEATURE_13)
+A(0xFD10, SWEVT, TRICORE_FEATURE_13)
+A(0xFD20, TR0EVT, TRICORE_FEATURE_13)
+A(0xFD24, TR1EVT, TRICORE_FEATURE_13)
+A(0xFD40, DMS, TRICORE_FEATURE_13)
+A(0xFD44, DCX, TRICORE_FEATURE_13)
+A(0xFD48, DBGTCR, TRICORE_FEATURE_131)
+A(0xFC00, CCTRL, TRICORE_FEATURE_131)
+A(0xFC04, CCNT, TRICORE_FEATURE_131)
+A(0xFC08, ICNT, TRICORE_FEATURE_131)
+A(0xFC0C, M1CNT, TRICORE_FEATURE_131)
+A(0xFC10, M2CNT, TRICORE_FEATURE_131)
+A(0xFC14, M3CNT, TRICORE_FEATURE_131)
diff --git a/target-tricore/fpu_helper.c b/target-tricore/fpu_helper.c
new file mode 100644
index 0000000..98fe947
--- /dev/null
+++ b/target-tricore/fpu_helper.c
@@ -0,0 +1,217 @@
+/*
+ *  TriCore emulation for qemu: fpu helper.
+ *
+ *  Copyright (c) 2016 Bastian Koppelmann University of Paderborn
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/helper-proto.h"
+
+#define ADD_NAN   0x7cf00001
+#define DIV_NAN   0x7fc00008
+#define MUL_NAN   0x7fc00002
+#define FPU_FS PSW_USB_C
+#define FPU_FI PSW_USB_V
+#define FPU_FV PSW_USB_SV
+#define FPU_FZ PSW_USB_AV
+#define FPU_FU PSW_USB_SAV
+
+/* we don't care about input_denormal */
+static inline uint8_t f_get_excp_flags(CPUTriCoreState *env)
+{
+    return get_float_exception_flags(&env->fp_status)
+           & (float_flag_invalid
+              | float_flag_overflow
+              | float_flag_underflow
+              | float_flag_output_denormal
+              | float_flag_divbyzero
+              | float_flag_inexact);
+}
+
+static inline bool f_is_denormal(float32 arg)
+{
+    return float32_is_zero_or_denormal(arg) && !float32_is_zero(arg);
+}
+
+static void f_update_psw_flags(CPUTriCoreState *env, uint8_t flags)
+{
+    uint8_t some_excp = 0;
+    set_float_exception_flags(0, &env->fp_status);
+
+    if (flags & float_flag_invalid) {
+        env->FPU_FI = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_overflow) {
+        env->FPU_FV = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_underflow || flags & float_flag_output_denormal) {
+        env->FPU_FU = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_divbyzero) {
+        env->FPU_FZ = 1 << 31;
+        some_excp = 1;
+    }
+
+    if (flags & float_flag_inexact || flags & float_flag_output_denormal) {
+        env->PSW |= 1 << 26;
+        some_excp = 1;
+    }
+
+    env->FPU_FS = some_excp;
+}
+
+#define FADD_SUB(op)                                                           \
+uint32_t helper_f##op(CPUTriCoreState *env, uint32_t r1, uint32_t r2)          \
+{                                                                              \
+    float32 arg1 = make_float32(r1);                                           \
+    float32 arg2 = make_float32(r2);                                           \
+    uint32_t flags;                                                            \
+    float32 f_result;                                                          \
+                                                                               \
+    f_result = float32_##op(arg2, arg1, &env->fp_status);                      \
+    flags = f_get_excp_flags(env);                                             \
+    if (flags) {                                                               \
+        /* If the output is a NaN, but the inputs aren't,                      \
+           we return a unique value.  */                                       \
+        if ((flags & float_flag_invalid)                                       \
+            && !float32_is_any_nan(arg1)                                       \
+            && !float32_is_any_nan(arg2)) {                                    \
+            f_result = ADD_NAN;                                                \
+        }                                                                      \
+        f_update_psw_flags(env, flags);                                        \
+    } else {                                                                   \
+        env->FPU_FS = 0;                                                       \
+    }                                                                          \
+    return (uint32_t)f_result;                                                 \
+}
+FADD_SUB(add)
+FADD_SUB(sub)
+
+uint32_t helper_fmul(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t flags;
+    float32 arg1 = make_float32(r1);
+    float32 arg2 = make_float32(r2);
+    float32 f_result;
+
+    f_result = float32_mul(arg1, arg2, &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        /* If the output is a NaN, but the inputs aren't,
+           we return a unique value.  */
+        if ((flags & float_flag_invalid)
+            && !float32_is_any_nan(arg1)
+            && !float32_is_any_nan(arg2)) {
+                f_result = MUL_NAN;
+        }
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+    return (uint32_t)f_result;
+
+}
+
+uint32_t helper_fdiv(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t flags;
+    float32 arg1 = make_float32(r1);
+    float32 arg2 = make_float32(r2);
+    float32 f_result;
+
+    f_result = float32_div(arg1, arg2 , &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        /* If the output is a NaN, but the inputs aren't,
+           we return a unique value.  */
+        if ((flags & float_flag_invalid)
+            && !float32_is_any_nan(arg1)
+            && !float32_is_any_nan(arg2)) {
+                f_result = DIV_NAN;
+        }
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+
+    return (uint32_t)f_result;
+}
+
+uint32_t helper_fcmp(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t result, flags;
+    float32 arg1 = make_float32(r1);
+    float32 arg2 = make_float32(r2);
+
+    set_flush_inputs_to_zero(0, &env->fp_status);
+
+    result = 1 << (float32_compare_quiet(arg1, arg2, &env->fp_status) + 1);
+    result |= f_is_denormal(arg1) << 4;
+    result |= f_is_denormal(arg2) << 5;
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+
+    set_flush_inputs_to_zero(1, &env->fp_status);
+    return result;
+}
+
+uint32_t helper_ftoi(CPUTriCoreState *env, uint32_t arg)
+{
+    float32 f_arg = make_float32(arg);
+    int32_t result, flags;
+
+    result = float32_to_int32(f_arg, &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        if (float32_is_any_nan(f_arg)) {
+            result = 0;
+        }
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+    return (uint32_t)result;
+}
+
+uint32_t helper_itof(CPUTriCoreState *env, uint32_t arg)
+{
+    float32 f_result;
+    uint32_t flags;
+    f_result = int32_to_float32(arg, &env->fp_status);
+
+    flags = f_get_excp_flags(env);
+    if (flags) {
+        f_update_psw_flags(env, flags);
+    } else {
+        env->FPU_FS = 0;
+    }
+    return (uint32_t)f_result;
+}
diff --git a/target-tricore/helper.c b/target-tricore/helper.c
index f52504c..3118905 100644
--- a/target-tricore/helper.c
+++ b/target-tricore/helper.c
@@ -15,14 +15,10 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <signal.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
+#include "exec/exec-all.h"
 
 enum {
     TLBRET_DIRTY = -4,
@@ -66,8 +62,8 @@
     access_type = ACCESS_INT;
     ret = get_physical_address(env, &physical, &prot,
                                address, rw, access_type);
-    qemu_log("%s address=" TARGET_FMT_lx " ret %d physical " TARGET_FMT_plx
-             " prot %d\n", __func__, address, ret, physical, prot);
+    qemu_log_mask(CPU_LOG_MMU, "%s address=" TARGET_FMT_lx " ret %d physical " TARGET_FMT_plx
+                  " prot %d\n", __func__, address, ret, physical, prot);
 
     if (ret == TLBRET_MATCH) {
         tlb_set_page(cs, address & TARGET_PAGE_MASK,
@@ -115,10 +111,18 @@
     g_slist_free(list);
 }
 
+void fpu_set_state(CPUTriCoreState *env)
+{
+    set_float_rounding_mode(env->PSW & MASK_PSW_FPU_RM, &env->fp_status);
+    set_flush_inputs_to_zero(1, &env->fp_status);
+    set_flush_to_zero(1, &env->fp_status);
+    set_default_nan_mode(1, &env->fp_status);
+}
+
 uint32_t psw_read(CPUTriCoreState *env)
 {
     /* clear all USB bits */
-    env->PSW &= 0xffffff;
+    env->PSW &= 0x6ffffff;
     /* now set them from the cache */
     env->PSW |= ((env->PSW_USB_C != 0) << 31);
     env->PSW |= ((env->PSW_USB_V   & (1 << 31))  >> 1);
@@ -132,9 +136,11 @@
 void psw_write(CPUTriCoreState *env, uint32_t val)
 {
     env->PSW_USB_C = (val & MASK_USB_C);
-    env->PSW_USB_V = (val & MASK_USB_V << 1);
-    env->PSW_USB_SV = (val & MASK_USB_SV << 2);
-    env->PSW_USB_AV = ((val & MASK_USB_AV) << 3);
-    env->PSW_USB_SAV = ((val & MASK_USB_SAV) << 4);
+    env->PSW_USB_V = (val & MASK_USB_V) << 1;
+    env->PSW_USB_SV = (val & MASK_USB_SV) << 2;
+    env->PSW_USB_AV = (val & MASK_USB_AV) << 3;
+    env->PSW_USB_SAV = (val & MASK_USB_SAV) << 4;
     env->PSW = val;
+
+    fpu_set_state(env);
 }
diff --git a/target-tricore/helper.h b/target-tricore/helper.h
index b3fc33c..9333e16 100644
--- a/target-tricore/helper.h
+++ b/target-tricore/helper.h
@@ -17,16 +17,135 @@
 
 /* Arithmetic */
 DEF_HELPER_3(add_ssov, i32, env, i32, i32)
+DEF_HELPER_3(add64_ssov, i64, env, i64, i64)
+DEF_HELPER_3(add_suov, i32, env, i32, i32)
+DEF_HELPER_3(add_h_ssov, i32, env, i32, i32)
+DEF_HELPER_3(add_h_suov, i32, env, i32, i32)
+DEF_HELPER_4(addr_h_ssov, i32, env, i64, i32, i32)
+DEF_HELPER_4(addsur_h_ssov, i32, env, i64, i32, i32)
 DEF_HELPER_3(sub_ssov, i32, env, i32, i32)
+DEF_HELPER_3(sub64_ssov, i64, env, i64, i64)
+DEF_HELPER_3(sub_suov, i32, env, i32, i32)
+DEF_HELPER_3(sub_h_ssov, i32, env, i32, i32)
+DEF_HELPER_3(sub_h_suov, i32, env, i32, i32)
+DEF_HELPER_4(subr_h_ssov, i32, env, i64, i32, i32)
+DEF_HELPER_4(subadr_h_ssov, i32, env, i64, i32, i32)
+DEF_HELPER_3(mul_ssov, i32, env, i32, i32)
+DEF_HELPER_3(mul_suov, i32, env, i32, i32)
+DEF_HELPER_3(sha_ssov, i32, env, i32, i32)
+DEF_HELPER_3(absdif_ssov, i32, env, i32, i32)
+DEF_HELPER_4(madd32_ssov, i32, env, i32, i32, i32)
+DEF_HELPER_4(madd32_suov, i32, env, i32, i32, i32)
+DEF_HELPER_4(madd64_ssov, i64, env, i32, i64, i32)
+DEF_HELPER_5(madd64_q_ssov, i64, env, i64, i32, i32, i32)
+DEF_HELPER_3(madd32_q_add_ssov, i32, env, i64, i64)
+DEF_HELPER_5(maddr_q_ssov, i32, env, i32, i32, i32, i32)
+DEF_HELPER_4(madd64_suov, i64, env, i32, i64, i32)
+DEF_HELPER_4(msub32_ssov, i32, env, i32, i32, i32)
+DEF_HELPER_4(msub32_suov, i32, env, i32, i32, i32)
+DEF_HELPER_4(msub64_ssov, i64, env, i32, i64, i32)
+DEF_HELPER_5(msub64_q_ssov, i64, env, i64, i32, i32, i32)
+DEF_HELPER_3(msub32_q_sub_ssov, i32, env, i64, i64)
+DEF_HELPER_5(msubr_q_ssov, i32, env, i32, i32, i32, i32)
+DEF_HELPER_4(msub64_suov, i64, env, i32, i64, i32)
+DEF_HELPER_3(absdif_h_ssov, i32, env, i32, i32)
+DEF_HELPER_2(abs_ssov, i32, env, i32)
+DEF_HELPER_2(abs_h_ssov, i32, env, i32)
+/* hword/byte arithmetic */
+DEF_HELPER_2(abs_b, i32, env, i32)
+DEF_HELPER_2(abs_h, i32, env, i32)
+DEF_HELPER_3(absdif_b, i32, env, i32, i32)
+DEF_HELPER_3(absdif_h, i32, env, i32, i32)
+DEF_HELPER_4(addr_h, i32, env, i64, i32, i32)
+DEF_HELPER_4(addsur_h, i32, env, i64, i32, i32)
+DEF_HELPER_5(maddr_q, i32, env, i32, i32, i32, i32)
+DEF_HELPER_3(add_b, i32, env, i32, i32)
+DEF_HELPER_3(add_h, i32, env, i32, i32)
+DEF_HELPER_3(sub_b, i32, env, i32, i32)
+DEF_HELPER_3(sub_h, i32, env, i32, i32)
+DEF_HELPER_4(subr_h, i32, env, i64, i32, i32)
+DEF_HELPER_4(subadr_h, i32, env, i64, i32, i32)
+DEF_HELPER_5(msubr_q, i32, env, i32, i32, i32, i32)
+DEF_HELPER_FLAGS_2(eq_b, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(eq_h, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(eqany_b, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(eqany_h, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(lt_b, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(lt_bu, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(lt_h, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(lt_hu, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(max_b, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(max_bu, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(max_h, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(max_hu, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(ixmax, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+DEF_HELPER_FLAGS_2(ixmax_u, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+DEF_HELPER_FLAGS_2(min_b, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(min_bu, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(min_h, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(min_hu, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(ixmin, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+DEF_HELPER_FLAGS_2(ixmin_u, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+/* count leading ... */
+DEF_HELPER_FLAGS_1(clo, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(clo_h, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(clz_h, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(cls, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(cls_h, TCG_CALL_NO_RWG_SE, i32, i32)
+/* sh */
+DEF_HELPER_FLAGS_2(sh, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(sh_h, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_3(sha, i32, env, i32, i32)
+DEF_HELPER_2(sha_h, i32, i32, i32)
+/* merge/split/parity */
+DEF_HELPER_FLAGS_2(bmerge, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_1(bsplit, TCG_CALL_NO_RWG_SE, i64, i32)
+DEF_HELPER_FLAGS_1(parity, TCG_CALL_NO_RWG_SE, i32, i32)
+/* float */
+DEF_HELPER_FLAGS_4(pack, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32, i32)
+DEF_HELPER_1(unpack, i64, i32)
+DEF_HELPER_3(fadd, i32, env, i32, i32)
+DEF_HELPER_3(fsub, i32, env, i32, i32)
+DEF_HELPER_3(fmul, i32, env, i32, i32)
+DEF_HELPER_3(fdiv, i32, env, i32, i32)
+DEF_HELPER_3(fcmp, i32, env, i32, i32)
+DEF_HELPER_2(ftoi, i32, env, i32)
+DEF_HELPER_2(itof, i32, env, i32)
+/* dvinit */
+DEF_HELPER_3(dvinit_b_13, i64, env, i32, i32)
+DEF_HELPER_3(dvinit_b_131, i64, env, i32, i32)
+DEF_HELPER_3(dvinit_h_13, i64, env, i32, i32)
+DEF_HELPER_3(dvinit_h_131, i64, env, i32, i32)
+DEF_HELPER_FLAGS_2(dvadj, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+DEF_HELPER_FLAGS_2(dvstep, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+DEF_HELPER_FLAGS_2(dvstep_u, TCG_CALL_NO_RWG_SE, i64, i64, i32)
+DEF_HELPER_3(divide, i64, env, i32, i32)
+DEF_HELPER_3(divide_u, i64, env, i32, i32)
+/* mulh */
+DEF_HELPER_FLAGS_5(mul_h, TCG_CALL_NO_RWG_SE, i64, i32, i32, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mulm_h, TCG_CALL_NO_RWG_SE, i64, i32, i32, i32, i32, i32)
+DEF_HELPER_FLAGS_5(mulr_h, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32, i32, i32)
+/* crc32 */
+DEF_HELPER_FLAGS_2(crc32, TCG_CALL_NO_RWG_SE, i32, i32, i32)
 /* CSA */
 DEF_HELPER_2(call, void, env, i32)
 DEF_HELPER_1(ret, void, env)
 DEF_HELPER_2(bisr, void, env, i32)
 DEF_HELPER_1(rfe, void, env)
+DEF_HELPER_1(rfm, void, env)
 DEF_HELPER_2(ldlcx, void, env, i32)
 DEF_HELPER_2(lducx, void, env, i32)
 DEF_HELPER_2(stlcx, void, env, i32)
 DEF_HELPER_2(stucx, void, env, i32)
+DEF_HELPER_1(svlcx, void, env)
+DEF_HELPER_1(svucx, void, env)
+DEF_HELPER_1(rslcx, void, env)
 /* Address mode helper */
 DEF_HELPER_1(br_update, i32, i32)
 DEF_HELPER_2(circ_update, i32, i32, i32)
+/* PSW cache helper */
+DEF_HELPER_2(psw_write, void, env, i32)
+DEF_HELPER_1(psw_read, i32, env)
+/* Exceptions */
+DEF_HELPER_3(raise_exception_sync, noreturn, env, i32, i32)
diff --git a/target-tricore/op_helper.c b/target-tricore/op_helper.c
index a36988a..ac02e0a 100644
--- a/target-tricore/op_helper.c
+++ b/target-tricore/op_helper.c
@@ -14,11 +14,100 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "qemu/host-utils.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include <zlib.h> /* for crc32 */
+
+
+/* Exception helpers */
+
+static void QEMU_NORETURN
+raise_exception_sync_internal(CPUTriCoreState *env, uint32_t class, int tin,
+                              uintptr_t pc, uint32_t fcd_pc)
+{
+    CPUState *cs = CPU(tricore_env_get_cpu(env));
+    /* in case we come from a helper-call we need to restore the PC */
+    if (pc) {
+        cpu_restore_state(cs, pc);
+    }
+
+    /* Tin is loaded into d[15] */
+    env->gpr_d[15] = tin;
+
+    if (class == TRAPC_CTX_MNG && tin == TIN3_FCU) {
+        /* upper context cannot be saved, if the context list is empty */
+    } else {
+        helper_svucx(env);
+    }
+
+    /* The return address in a[11] is updated */
+    if (class == TRAPC_CTX_MNG && tin == TIN3_FCD) {
+        env->SYSCON |= MASK_SYSCON_FCD_SF;
+        /* when we run out of CSAs after saving a context a FCD trap is taken
+           and the return address is the start of the trap handler which used
+           the last CSA */
+        env->gpr_a[11] = fcd_pc;
+    } else if (class == TRAPC_SYSCALL) {
+        env->gpr_a[11] = env->PC + 4;
+    } else {
+        env->gpr_a[11] = env->PC;
+    }
+    /* The stack pointer in A[10] is set to the Interrupt Stack Pointer (ISP)
+       when the processor was not previously using the interrupt stack
+       (in case of PSW.IS = 0). The stack pointer bit is set for using the
+       interrupt stack: PSW.IS = 1. */
+    if ((env->PSW & MASK_PSW_IS) == 0) {
+        env->gpr_a[10] = env->ISP;
+    }
+    env->PSW |= MASK_PSW_IS;
+    /* The I/O mode is set to Supervisor mode, which means all permissions
+       are enabled: PSW.IO = 10 B .*/
+    env->PSW |= (2 << 10);
+
+    /*The current Protection Register Set is set to 0: PSW.PRS = 00 B .*/
+    env->PSW &= ~MASK_PSW_PRS;
+
+    /* The Call Depth Counter (CDC) is cleared, and the call depth limit is
+       set for 64: PSW.CDC = 0000000 B .*/
+    env->PSW &= ~MASK_PSW_CDC;
+
+    /* Call Depth Counter is enabled, PSW.CDE = 1. */
+    env->PSW |= MASK_PSW_CDE;
+
+    /* Write permission to global registers A[0], A[1], A[8], A[9] is
+       disabled: PSW.GW = 0. */
+    env->PSW &= ~MASK_PSW_GW;
+
+    /*The interrupt system is globally disabled: ICR.IE = 0. The ‘old’
+      ICR.IE and ICR.CCPN are saved */
+
+    /* PCXI.PIE = ICR.IE */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.PCPN = ICR.CCPN */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* Update PC using the trap vector table */
+    env->PC = env->BTV | (class << 5);
+
+    cpu_loop_exit(cs);
+}
+
+void helper_raise_exception_sync(CPUTriCoreState *env, uint32_t class,
+                                 uint32_t tin)
+{
+    raise_exception_sync_internal(env, class, tin, 0, 0);
+}
+
+static void raise_exception_sync_helper(CPUTriCoreState *env, uint32_t class,
+                                        uint32_t tin, uintptr_t pc)
+{
+    raise_exception_sync_internal(env, class, tin, pc, 0);
+}
 
 /* Addressing mode helper */
 
@@ -56,49 +145,2171 @@
     return reg - index + new_index;
 }
 
-#define SSOV(env, ret, arg, len) do {               \
-    int64_t max_pos = INT##len ##_MAX;              \
-    int64_t max_neg = INT##len ##_MIN;              \
-    if (arg > max_pos) {                            \
-        env->PSW_USB_V = (1 << 31);                 \
-        env->PSW_USB_SV = (1 << 31);                \
-        ret = (target_ulong)max_pos;                \
-    } else {                                        \
-        if (arg < max_neg) {                        \
-            env->PSW_USB_V = (1 << 31);             \
-            env->PSW_USB_SV = (1 << 31);            \
-            ret = (target_ulong)max_neg;            \
-        } else {                                    \
-            env->PSW_USB_V = 0;                     \
-            ret = (target_ulong)arg;                \
-        }                                           \
-    }                                               \
-    env->PSW_USB_AV = arg ^ arg * 2u;               \
-    env->PSW_USB_SAV |= env->PSW_USB_AV;            \
-} while (0)
+static uint32_t ssov32(CPUTriCoreState *env, int64_t arg)
+{
+    uint32_t ret;
+    int64_t max_pos = INT32_MAX;
+    int64_t max_neg = INT32_MIN;
+    if (arg > max_pos) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        ret = (target_ulong)max_pos;
+    } else {
+        if (arg < max_neg) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            ret = (target_ulong)max_neg;
+        } else {
+            env->PSW_USB_V = 0;
+            ret = (target_ulong)arg;
+        }
+    }
+    env->PSW_USB_AV = arg ^ arg * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    return ret;
+}
+
+static uint32_t suov32_pos(CPUTriCoreState *env, uint64_t arg)
+{
+    uint32_t ret;
+    uint64_t max_pos = UINT32_MAX;
+    if (arg > max_pos) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        ret = (target_ulong)max_pos;
+    } else {
+        env->PSW_USB_V = 0;
+        ret = (target_ulong)arg;
+     }
+    env->PSW_USB_AV = arg ^ arg * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    return ret;
+}
+
+static uint32_t suov32_neg(CPUTriCoreState *env, int64_t arg)
+{
+    uint32_t ret;
+
+    if (arg < 0) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        ret = 0;
+    } else {
+        env->PSW_USB_V = 0;
+        ret = (target_ulong)arg;
+    }
+    env->PSW_USB_AV = arg ^ arg * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    return ret;
+}
+
+static uint32_t ssov16(CPUTriCoreState *env, int32_t hw0, int32_t hw1)
+{
+    int32_t max_pos = INT16_MAX;
+    int32_t max_neg = INT16_MIN;
+    int32_t av0, av1;
+
+    env->PSW_USB_V = 0;
+    av0 = hw0 ^ hw0 * 2u;
+    if (hw0 > max_pos) {
+        env->PSW_USB_V = (1 << 31);
+        hw0 = max_pos;
+    } else if (hw0 < max_neg) {
+        env->PSW_USB_V = (1 << 31);
+        hw0 = max_neg;
+    }
+
+    av1 = hw1 ^ hw1 * 2u;
+    if (hw1 > max_pos) {
+        env->PSW_USB_V = (1 << 31);
+        hw1 = max_pos;
+    } else if (hw1 < max_neg) {
+        env->PSW_USB_V = (1 << 31);
+        hw1 = max_neg;
+    }
+
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = (av0 | av1) << 16;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    return (hw0 & 0xffff) | (hw1 << 16);
+}
+
+static uint32_t suov16(CPUTriCoreState *env, int32_t hw0, int32_t hw1)
+{
+    int32_t max_pos = UINT16_MAX;
+    int32_t av0, av1;
+
+    env->PSW_USB_V = 0;
+    av0 = hw0 ^ hw0 * 2u;
+    if (hw0 > max_pos) {
+        env->PSW_USB_V = (1 << 31);
+        hw0 = max_pos;
+    } else if (hw0 < 0) {
+        env->PSW_USB_V = (1 << 31);
+        hw0 = 0;
+    }
+
+    av1 = hw1 ^ hw1 * 2u;
+    if (hw1 > max_pos) {
+        env->PSW_USB_V = (1 << 31);
+        hw1 = max_pos;
+    } else if (hw1 < 0) {
+        env->PSW_USB_V = (1 << 31);
+        hw1 = 0;
+    }
+
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = (av0 | av1) << 16;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    return (hw0 & 0xffff) | (hw1 << 16);
+}
 
 target_ulong helper_add_ssov(CPUTriCoreState *env, target_ulong r1,
                              target_ulong r2)
 {
-    target_ulong ret;
     int64_t t1 = sextract64(r1, 0, 32);
     int64_t t2 = sextract64(r2, 0, 32);
     int64_t result = t1 + t2;
-    SSOV(env, ret, result, 32);
-    return ret;
+    return ssov32(env, result);
+}
+
+uint64_t helper_add64_ssov(CPUTriCoreState *env, uint64_t r1, uint64_t r2)
+{
+    uint64_t result;
+    int64_t ovf;
+
+    result = r1 + r2;
+    ovf = (result ^ r1) & ~(r1 ^ r2);
+    env->PSW_USB_AV = (result ^ result * 2u) >> 32;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    if (ovf < 0) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        /* ext_ret > MAX_INT */
+        if ((int64_t)r1 >= 0) {
+            result = INT64_MAX;
+        /* ext_ret < MIN_INT */
+        } else {
+            result = INT64_MIN;
+        }
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return result;
+}
+
+target_ulong helper_add_h_ssov(CPUTriCoreState *env, target_ulong r1,
+                               target_ulong r2)
+{
+    int32_t ret_hw0, ret_hw1;
+
+    ret_hw0 = sextract32(r1, 0, 16) + sextract32(r2, 0, 16);
+    ret_hw1 = sextract32(r1, 16, 16) + sextract32(r2, 16, 16);
+    return ssov16(env, ret_hw0, ret_hw1);
+}
+
+uint32_t helper_addr_h_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                            uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low + mul_res0 + 0x8000;
+    result1 = r2_high + mul_res1 + 0x8000;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    if (result0 > INT32_MAX) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MAX;
+    } else if (result0 < INT32_MIN) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MIN;
+    }
+
+    if (result1 > INT32_MAX) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MAX;
+    } else if (result1 < INT32_MIN) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MIN;
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+uint32_t helper_addsur_h_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                              uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low - mul_res0 + 0x8000;
+    result1 = r2_high + mul_res1 + 0x8000;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    if (result0 > INT32_MAX) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MAX;
+    } else if (result0 < INT32_MIN) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MIN;
+    }
+
+    if (result1 > INT32_MAX) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MAX;
+    } else if (result1 < INT32_MIN) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MIN;
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+
+target_ulong helper_add_suov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    int64_t t1 = extract64(r1, 0, 32);
+    int64_t t2 = extract64(r2, 0, 32);
+    int64_t result = t1 + t2;
+    return suov32_pos(env, result);
+}
+
+target_ulong helper_add_h_suov(CPUTriCoreState *env, target_ulong r1,
+                               target_ulong r2)
+{
+    int32_t ret_hw0, ret_hw1;
+
+    ret_hw0 = extract32(r1, 0, 16) + extract32(r2, 0, 16);
+    ret_hw1 = extract32(r1, 16, 16) + extract32(r2, 16, 16);
+    return suov16(env, ret_hw0, ret_hw1);
 }
 
 target_ulong helper_sub_ssov(CPUTriCoreState *env, target_ulong r1,
                              target_ulong r2)
 {
-    target_ulong ret;
     int64_t t1 = sextract64(r1, 0, 32);
     int64_t t2 = sextract64(r2, 0, 32);
     int64_t result = t1 - t2;
-    SSOV(env, ret, result, 32);
+    return ssov32(env, result);
+}
+
+uint64_t helper_sub64_ssov(CPUTriCoreState *env, uint64_t r1, uint64_t r2)
+{
+    uint64_t result;
+    int64_t ovf;
+
+    result = r1 - r2;
+    ovf = (result ^ r1) & (r1 ^ r2);
+    env->PSW_USB_AV = (result ^ result * 2u) >> 32;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    if (ovf < 0) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        /* ext_ret > MAX_INT */
+        if ((int64_t)r1 >= 0) {
+            result = INT64_MAX;
+        /* ext_ret < MIN_INT */
+        } else {
+            result = INT64_MIN;
+        }
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return result;
+}
+
+target_ulong helper_sub_h_ssov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    int32_t ret_hw0, ret_hw1;
+
+    ret_hw0 = sextract32(r1, 0, 16) - sextract32(r2, 0, 16);
+    ret_hw1 = sextract32(r1, 16, 16) - sextract32(r2, 16, 16);
+    return ssov16(env, ret_hw0, ret_hw1);
+}
+
+uint32_t helper_subr_h_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                            uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low - mul_res0 + 0x8000;
+    result1 = r2_high - mul_res1 + 0x8000;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    if (result0 > INT32_MAX) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MAX;
+    } else if (result0 < INT32_MIN) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MIN;
+    }
+
+    if (result1 > INT32_MAX) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MAX;
+    } else if (result1 < INT32_MIN) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MIN;
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+uint32_t helper_subadr_h_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                              uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low + mul_res0 + 0x8000;
+    result1 = r2_high - mul_res1 + 0x8000;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    if (result0 > INT32_MAX) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MAX;
+    } else if (result0 < INT32_MIN) {
+        ovf0 = (1 << 31);
+        result0 = INT32_MIN;
+    }
+
+    if (result1 > INT32_MAX) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MAX;
+    } else if (result1 < INT32_MIN) {
+        ovf1 = (1 << 31);
+        result1 = INT32_MIN;
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+target_ulong helper_sub_suov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    int64_t t1 = extract64(r1, 0, 32);
+    int64_t t2 = extract64(r2, 0, 32);
+    int64_t result = t1 - t2;
+    return suov32_neg(env, result);
+}
+
+target_ulong helper_sub_h_suov(CPUTriCoreState *env, target_ulong r1,
+                               target_ulong r2)
+{
+    int32_t ret_hw0, ret_hw1;
+
+    ret_hw0 = extract32(r1, 0, 16) - extract32(r2, 0, 16);
+    ret_hw1 = extract32(r1, 16, 16) - extract32(r2, 16, 16);
+    return suov16(env, ret_hw0, ret_hw1);
+}
+
+target_ulong helper_mul_ssov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t result = t1 * t2;
+    return ssov32(env, result);
+}
+
+target_ulong helper_mul_suov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    int64_t t1 = extract64(r1, 0, 32);
+    int64_t t2 = extract64(r2, 0, 32);
+    int64_t result = t1 * t2;
+
+    return suov32_pos(env, result);
+}
+
+target_ulong helper_sha_ssov(CPUTriCoreState *env, target_ulong r1,
+                             target_ulong r2)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int32_t t2 = sextract64(r2, 0, 6);
+    int64_t result;
+    if (t2 == 0) {
+        result = t1;
+    } else if (t2 > 0) {
+        result = t1 << t2;
+    } else {
+        result = t1 >> -t2;
+    }
+    return ssov32(env, result);
+}
+
+uint32_t helper_abs_ssov(CPUTriCoreState *env, target_ulong r1)
+{
+    target_ulong result;
+    result = ((int32_t)r1 >= 0) ? r1 : (0 - r1);
+    return ssov32(env, result);
+}
+
+uint32_t helper_abs_h_ssov(CPUTriCoreState *env, target_ulong r1)
+{
+    int32_t ret_h0, ret_h1;
+
+    ret_h0 = sextract32(r1, 0, 16);
+    ret_h0 = (ret_h0 >= 0) ? ret_h0 : (0 - ret_h0);
+
+    ret_h1 = sextract32(r1, 16, 16);
+    ret_h1 = (ret_h1 >= 0) ? ret_h1 : (0 - ret_h1);
+
+    return ssov16(env, ret_h0, ret_h1);
+}
+
+target_ulong helper_absdif_ssov(CPUTriCoreState *env, target_ulong r1,
+                                target_ulong r2)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t result;
+
+    if (t1 > t2) {
+        result = t1 - t2;
+    } else {
+        result = t2 - t1;
+    }
+    return ssov32(env, result);
+}
+
+uint32_t helper_absdif_h_ssov(CPUTriCoreState *env, target_ulong r1,
+                              target_ulong r2)
+{
+    int32_t t1, t2;
+    int32_t ret_h0, ret_h1;
+
+    t1 = sextract32(r1, 0, 16);
+    t2 = sextract32(r2, 0, 16);
+    if (t1 > t2) {
+        ret_h0 = t1 - t2;
+    } else {
+        ret_h0 = t2 - t1;
+    }
+
+    t1 = sextract32(r1, 16, 16);
+    t2 = sextract32(r2, 16, 16);
+    if (t1 > t2) {
+        ret_h1 = t1 - t2;
+    } else {
+        ret_h1 = t2 - t1;
+    }
+
+    return ssov16(env, ret_h0, ret_h1);
+}
+
+target_ulong helper_madd32_ssov(CPUTriCoreState *env, target_ulong r1,
+                                target_ulong r2, target_ulong r3)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t result;
+
+    result = t2 + (t1 * t3);
+    return ssov32(env, result);
+}
+
+target_ulong helper_madd32_suov(CPUTriCoreState *env, target_ulong r1,
+                                target_ulong r2, target_ulong r3)
+{
+    uint64_t t1 = extract64(r1, 0, 32);
+    uint64_t t2 = extract64(r2, 0, 32);
+    uint64_t t3 = extract64(r3, 0, 32);
+    int64_t result;
+
+    result = t2 + (t1 * t3);
+    return suov32_pos(env, result);
+}
+
+uint64_t helper_madd64_ssov(CPUTriCoreState *env, target_ulong r1,
+                            uint64_t r2, target_ulong r3)
+{
+    uint64_t ret, ovf;
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t mul;
+
+    mul = t1 * t3;
+    ret = mul + r2;
+    ovf = (ret ^ mul) & ~(mul ^ r2);
+
+    t1 = ret >> 32;
+    env->PSW_USB_AV = t1 ^ t1 * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    if ((int64_t)ovf < 0) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        /* ext_ret > MAX_INT */
+        if (mul >= 0) {
+            ret = INT64_MAX;
+        /* ext_ret < MIN_INT */
+        } else {
+            ret = INT64_MIN;
+        }
+    } else {
+        env->PSW_USB_V = 0;
+    }
+
     return ret;
 }
 
+uint32_t
+helper_madd32_q_add_ssov(CPUTriCoreState *env, uint64_t r1, uint64_t r2)
+{
+    int64_t result;
+
+    result = (r1 + r2);
+
+    env->PSW_USB_AV = (result ^ result * 2u);
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    /* we do the saturation by hand, since we produce an overflow on the host
+       if the mul before was (0x80000000 * 0x80000000) << 1). If this is the
+       case, we flip the saturated value. */
+    if (r2 == 0x8000000000000000LL) {
+        if (result > 0x7fffffffLL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MIN;
+        } else if (result < -0x80000000LL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MAX;
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    } else {
+        if (result > 0x7fffffffLL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MAX;
+        } else if (result < -0x80000000LL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MIN;
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    }
+    return (uint32_t)result;
+}
+
+uint64_t helper_madd64_q_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2,
+                              uint32_t r3, uint32_t n)
+{
+    int64_t t1 = (int64_t)r1;
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t result, mul;
+    int64_t ovf;
+
+    mul = (t2 * t3) << n;
+    result = mul + t1;
+
+    env->PSW_USB_AV = (result ^ result * 2u) >> 32;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    ovf = (result ^ mul) & ~(mul ^ t1);
+    /* we do the saturation by hand, since we produce an overflow on the host
+       if the mul was (0x80000000 * 0x80000000) << 1). If this is the
+       case, we flip the saturated value. */
+    if ((r2 == 0x80000000) && (r3 == 0x80000000) && (n == 1)) {
+        if (ovf >= 0) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            /* ext_ret > MAX_INT */
+            if (mul < 0) {
+                result = INT64_MAX;
+            /* ext_ret < MIN_INT */
+            } else {
+               result = INT64_MIN;
+            }
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    } else {
+        if (ovf < 0) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            /* ext_ret > MAX_INT */
+            if (mul >= 0) {
+                result = INT64_MAX;
+            /* ext_ret < MIN_INT */
+            } else {
+               result = INT64_MIN;
+            }
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    }
+    return (uint64_t)result;
+}
+
+uint32_t helper_maddr_q_ssov(CPUTriCoreState *env, uint32_t r1, uint32_t r2,
+                             uint32_t r3, uint32_t n)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t mul, ret;
+
+    if ((t2 == -0x8000ll) && (t3 == -0x8000ll) && (n == 1)) {
+        mul = 0x7fffffff;
+    } else {
+        mul = (t2 * t3) << n;
+    }
+
+    ret = t1 + mul + 0x8000;
+
+    env->PSW_USB_AV = ret ^ ret * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    if (ret > 0x7fffffffll) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV |= env->PSW_USB_V;
+        ret = INT32_MAX;
+    } else if (ret < -0x80000000ll) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV |= env->PSW_USB_V;
+        ret = INT32_MIN;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return ret & 0xffff0000ll;
+}
+
+uint64_t helper_madd64_suov(CPUTriCoreState *env, target_ulong r1,
+                            uint64_t r2, target_ulong r3)
+{
+    uint64_t ret, mul;
+    uint64_t t1 = extract64(r1, 0, 32);
+    uint64_t t3 = extract64(r3, 0, 32);
+
+    mul = t1 * t3;
+    ret = mul + r2;
+
+    t1 = ret >> 32;
+    env->PSW_USB_AV = t1 ^ t1 * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    if (ret < r2) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        /* saturate */
+        ret = UINT64_MAX;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return ret;
+}
+
+target_ulong helper_msub32_ssov(CPUTriCoreState *env, target_ulong r1,
+                                target_ulong r2, target_ulong r3)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t result;
+
+    result = t2 - (t1 * t3);
+    return ssov32(env, result);
+}
+
+target_ulong helper_msub32_suov(CPUTriCoreState *env, target_ulong r1,
+                                target_ulong r2, target_ulong r3)
+{
+    uint64_t t1 = extract64(r1, 0, 32);
+    uint64_t t2 = extract64(r2, 0, 32);
+    uint64_t t3 = extract64(r3, 0, 32);
+    uint64_t result;
+    uint64_t mul;
+
+    mul = (t1 * t3);
+    result = t2 - mul;
+
+    env->PSW_USB_AV = result ^ result * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    /* we calculate ovf by hand here, because the multiplication can overflow on
+       the host, which would give false results if we compare to less than
+       zero */
+    if (mul > t2) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        result = 0;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return result;
+}
+
+uint64_t helper_msub64_ssov(CPUTriCoreState *env, target_ulong r1,
+                            uint64_t r2, target_ulong r3)
+{
+    uint64_t ret, ovf;
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t mul;
+
+    mul = t1 * t3;
+    ret = r2 - mul;
+    ovf = (ret ^ r2) & (mul ^ r2);
+
+    t1 = ret >> 32;
+    env->PSW_USB_AV = t1 ^ t1 * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    if ((int64_t)ovf < 0) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        /* ext_ret > MAX_INT */
+        if (mul < 0) {
+            ret = INT64_MAX;
+        /* ext_ret < MIN_INT */
+        } else {
+            ret = INT64_MIN;
+        }
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return ret;
+}
+
+uint64_t helper_msub64_suov(CPUTriCoreState *env, target_ulong r1,
+                            uint64_t r2, target_ulong r3)
+{
+    uint64_t ret, mul;
+    uint64_t t1 = extract64(r1, 0, 32);
+    uint64_t t3 = extract64(r3, 0, 32);
+
+    mul = t1 * t3;
+    ret = r2 - mul;
+
+    t1 = ret >> 32;
+    env->PSW_USB_AV = t1 ^ t1 * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    if (ret > r2) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV = (1 << 31);
+        /* saturate */
+        ret = 0;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return ret;
+}
+
+uint32_t
+helper_msub32_q_sub_ssov(CPUTriCoreState *env, uint64_t r1, uint64_t r2)
+{
+    int64_t result;
+    int64_t t1 = (int64_t)r1;
+    int64_t t2 = (int64_t)r2;
+
+    result = t1 - t2;
+
+    env->PSW_USB_AV = (result ^ result * 2u);
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    /* we do the saturation by hand, since we produce an overflow on the host
+       if the mul before was (0x80000000 * 0x80000000) << 1). If this is the
+       case, we flip the saturated value. */
+    if (r2 == 0x8000000000000000LL) {
+        if (result > 0x7fffffffLL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MIN;
+        } else if (result < -0x80000000LL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MAX;
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    } else {
+        if (result > 0x7fffffffLL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MAX;
+        } else if (result < -0x80000000LL) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            result = INT32_MIN;
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    }
+    return (uint32_t)result;
+}
+
+uint64_t helper_msub64_q_ssov(CPUTriCoreState *env, uint64_t r1, uint32_t r2,
+                              uint32_t r3, uint32_t n)
+{
+    int64_t t1 = (int64_t)r1;
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t result, mul;
+    int64_t ovf;
+
+    mul = (t2 * t3) << n;
+    result = t1 - mul;
+
+    env->PSW_USB_AV = (result ^ result * 2u) >> 32;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    ovf = (result ^ t1) & (t1 ^ mul);
+    /* we do the saturation by hand, since we produce an overflow on the host
+       if the mul before was (0x80000000 * 0x80000000) << 1). If this is the
+       case, we flip the saturated value. */
+    if (mul == 0x8000000000000000LL) {
+        if (ovf >= 0) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            /* ext_ret > MAX_INT */
+            if (mul >= 0) {
+                result = INT64_MAX;
+            /* ext_ret < MIN_INT */
+            } else {
+               result = INT64_MIN;
+            }
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    } else {
+        if (ovf < 0) {
+            env->PSW_USB_V = (1 << 31);
+            env->PSW_USB_SV = (1 << 31);
+            /* ext_ret > MAX_INT */
+            if (mul < 0) {
+                result = INT64_MAX;
+            /* ext_ret < MIN_INT */
+            } else {
+               result = INT64_MIN;
+            }
+        } else {
+            env->PSW_USB_V = 0;
+        }
+    }
+
+    return (uint64_t)result;
+}
+
+uint32_t helper_msubr_q_ssov(CPUTriCoreState *env, uint32_t r1, uint32_t r2,
+                             uint32_t r3, uint32_t n)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t mul, ret;
+
+    if ((t2 == -0x8000ll) && (t3 == -0x8000ll) && (n == 1)) {
+        mul = 0x7fffffff;
+    } else {
+        mul = (t2 * t3) << n;
+    }
+
+    ret = t1 - mul + 0x8000;
+
+    env->PSW_USB_AV = ret ^ ret * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    if (ret > 0x7fffffffll) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV |= env->PSW_USB_V;
+        ret = INT32_MAX;
+    } else if (ret < -0x80000000ll) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV |= env->PSW_USB_V;
+        ret = INT32_MIN;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    return ret & 0xffff0000ll;
+}
+
+uint32_t helper_abs_b(CPUTriCoreState *env, target_ulong arg)
+{
+    int32_t b, i;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    int32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        b = sextract32(arg, i * 8, 8);
+        b = (b >= 0) ? b : (0 - b);
+        ovf |= (b > 0x7F) || (b < -0x80);
+        avf |= b ^ b * 2u;
+        ret |= (b & 0xff) << (i * 8);
+    }
+
+    env->PSW_USB_V = ovf << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 24;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_abs_h(CPUTriCoreState *env, target_ulong arg)
+{
+    int32_t h, i;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    int32_t ret = 0;
+
+    for (i = 0; i < 2; i++) {
+        h = sextract32(arg, i * 16, 16);
+        h = (h >= 0) ? h : (0 - h);
+        ovf |= (h > 0x7FFF) || (h < -0x8000);
+        avf |= h ^ h * 2u;
+        ret |= (h & 0xffff) << (i * 16);
+    }
+
+    env->PSW_USB_V = ovf << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 16;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_absdif_b(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t b, i;
+    int32_t extr_r2;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    int32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        extr_r2 = sextract32(r2, i * 8, 8);
+        b = sextract32(r1, i * 8, 8);
+        b = (b > extr_r2) ? (b - extr_r2) : (extr_r2 - b);
+        ovf |= (b > 0x7F) || (b < -0x80);
+        avf |= b ^ b * 2u;
+        ret |= (b & 0xff) << (i * 8);
+    }
+
+    env->PSW_USB_V = ovf << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 24;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+    return ret;
+}
+
+uint32_t helper_absdif_h(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t h, i;
+    int32_t extr_r2;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    int32_t ret = 0;
+
+    for (i = 0; i < 2; i++) {
+        extr_r2 = sextract32(r2, i * 16, 16);
+        h = sextract32(r1, i * 16, 16);
+        h = (h > extr_r2) ? (h - extr_r2) : (extr_r2 - h);
+        ovf |= (h > 0x7FFF) || (h < -0x8000);
+        avf |= h ^ h * 2u;
+        ret |= (h & 0xffff) << (i * 16);
+    }
+
+    env->PSW_USB_V = ovf << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 16;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_addr_h(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                       uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low + mul_res0 + 0x8000;
+    result1 = r2_high + mul_res1 + 0x8000;
+
+    if ((result0 > INT32_MAX) || (result0 < INT32_MIN)) {
+        ovf0 = (1 << 31);
+    }
+
+    if ((result1 > INT32_MAX) || (result1 < INT32_MIN)) {
+        ovf1 = (1 << 31);
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+uint32_t helper_addsur_h(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                         uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low - mul_res0 + 0x8000;
+    result1 = r2_high + mul_res1 + 0x8000;
+
+    if ((result0 > INT32_MAX) || (result0 < INT32_MIN)) {
+        ovf0 = (1 << 31);
+    }
+
+    if ((result1 > INT32_MAX) || (result1 < INT32_MIN)) {
+        ovf1 = (1 << 31);
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+uint32_t helper_maddr_q(CPUTriCoreState *env, uint32_t r1, uint32_t r2,
+                        uint32_t r3, uint32_t n)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t mul, ret;
+
+    if ((t2 == -0x8000ll) && (t3 == -0x8000ll) && (n == 1)) {
+        mul = 0x7fffffff;
+    } else {
+        mul = (t2 * t3) << n;
+    }
+
+    ret = t1 + mul + 0x8000;
+
+    if ((ret > 0x7fffffffll) || (ret < -0x80000000ll)) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV |= env->PSW_USB_V;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    env->PSW_USB_AV = ret ^ ret * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret & 0xffff0000ll;
+}
+
+uint32_t helper_add_b(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t b, i;
+    int32_t extr_r1, extr_r2;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    uint32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        extr_r1 = sextract32(r1, i * 8, 8);
+        extr_r2 = sextract32(r2, i * 8, 8);
+
+        b = extr_r1 + extr_r2;
+        ovf |= ((b > 0x7f) || (b < -0x80));
+        avf |= b ^ b * 2u;
+        ret |= ((b & 0xff) << (i*8));
+    }
+
+    env->PSW_USB_V = (ovf << 31);
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 24;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_add_h(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t h, i;
+    int32_t extr_r1, extr_r2;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    int32_t ret = 0;
+
+    for (i = 0; i < 2; i++) {
+        extr_r1 = sextract32(r1, i * 16, 16);
+        extr_r2 = sextract32(r2, i * 16, 16);
+        h = extr_r1 + extr_r2;
+        ovf |= ((h > 0x7fff) || (h < -0x8000));
+        avf |= h ^ h * 2u;
+        ret |= (h & 0xffff) << (i * 16);
+    }
+
+    env->PSW_USB_V = (ovf << 31);
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = (avf << 16);
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_subr_h(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                       uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low - mul_res0 + 0x8000;
+    result1 = r2_high - mul_res1 + 0x8000;
+
+    if ((result0 > INT32_MAX) || (result0 < INT32_MIN)) {
+        ovf0 = (1 << 31);
+    }
+
+    if ((result1 > INT32_MAX) || (result1 < INT32_MIN)) {
+        ovf1 = (1 << 31);
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+uint32_t helper_subadr_h(CPUTriCoreState *env, uint64_t r1, uint32_t r2_l,
+                         uint32_t r2_h)
+{
+    int64_t mul_res0 = sextract64(r1, 0, 32);
+    int64_t mul_res1 = sextract64(r1, 32, 32);
+    int64_t r2_low = sextract64(r2_l, 0, 32);
+    int64_t r2_high = sextract64(r2_h, 0, 32);
+    int64_t result0, result1;
+    uint32_t ovf0, ovf1;
+    uint32_t avf0, avf1;
+
+    ovf0 = ovf1 = 0;
+
+    result0 = r2_low + mul_res0 + 0x8000;
+    result1 = r2_high - mul_res1 + 0x8000;
+
+    if ((result0 > INT32_MAX) || (result0 < INT32_MIN)) {
+        ovf0 = (1 << 31);
+    }
+
+    if ((result1 > INT32_MAX) || (result1 < INT32_MIN)) {
+        ovf1 = (1 << 31);
+    }
+
+    env->PSW_USB_V = ovf0 | ovf1;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+
+    avf0 = result0 * 2u;
+    avf0 = result0 ^ avf0;
+    avf1 = result1 * 2u;
+    avf1 = result1 ^ avf1;
+
+    env->PSW_USB_AV = avf0 | avf1;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return (result1 & 0xffff0000ULL) | ((result0 >> 16) & 0xffffULL);
+}
+
+uint32_t helper_msubr_q(CPUTriCoreState *env, uint32_t r1, uint32_t r2,
+                        uint32_t r3, uint32_t n)
+{
+    int64_t t1 = sextract64(r1, 0, 32);
+    int64_t t2 = sextract64(r2, 0, 32);
+    int64_t t3 = sextract64(r3, 0, 32);
+    int64_t mul, ret;
+
+    if ((t2 == -0x8000ll) && (t3 == -0x8000ll) && (n == 1)) {
+        mul = 0x7fffffff;
+    } else {
+        mul = (t2 * t3) << n;
+    }
+
+    ret = t1 - mul + 0x8000;
+
+    if ((ret > 0x7fffffffll) || (ret < -0x80000000ll)) {
+        env->PSW_USB_V = (1 << 31);
+        env->PSW_USB_SV |= env->PSW_USB_V;
+    } else {
+        env->PSW_USB_V = 0;
+    }
+    env->PSW_USB_AV = ret ^ ret * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret & 0xffff0000ll;
+}
+
+uint32_t helper_sub_b(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t b, i;
+    int32_t extr_r1, extr_r2;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    uint32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        extr_r1 = sextract32(r1, i * 8, 8);
+        extr_r2 = sextract32(r2, i * 8, 8);
+
+        b = extr_r1 - extr_r2;
+        ovf |= ((b > 0x7f) || (b < -0x80));
+        avf |= b ^ b * 2u;
+        ret |= ((b & 0xff) << (i*8));
+    }
+
+    env->PSW_USB_V = (ovf << 31);
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 24;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_sub_h(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t h, i;
+    int32_t extr_r1, extr_r2;
+    int32_t ovf = 0;
+    int32_t avf = 0;
+    int32_t ret = 0;
+
+    for (i = 0; i < 2; i++) {
+        extr_r1 = sextract32(r1, i * 16, 16);
+        extr_r2 = sextract32(r2, i * 16, 16);
+        h = extr_r1 - extr_r2;
+        ovf |= ((h > 0x7fff) || (h < -0x8000));
+        avf |= h ^ h * 2u;
+        ret |= (h & 0xffff) << (i * 16);
+    }
+
+    env->PSW_USB_V = (ovf << 31);
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = avf << 16;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_eq_b(target_ulong r1, target_ulong r2)
+{
+    int32_t ret;
+    int32_t i, msk;
+
+    ret = 0;
+    msk = 0xff;
+    for (i = 0; i < 4; i++) {
+        if ((r1 & msk) == (r2 & msk)) {
+            ret |= msk;
+        }
+        msk = msk << 8;
+    }
+
+    return ret;
+}
+
+uint32_t helper_eq_h(target_ulong r1, target_ulong r2)
+{
+    int32_t ret = 0;
+
+    if ((r1 & 0xffff) == (r2 & 0xffff)) {
+        ret = 0xffff;
+    }
+
+    if ((r1 & 0xffff0000) == (r2 & 0xffff0000)) {
+        ret |= 0xffff0000;
+    }
+
+    return ret;
+}
+
+uint32_t helper_eqany_b(target_ulong r1, target_ulong r2)
+{
+    int32_t i;
+    uint32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        ret |= (sextract32(r1,  i * 8, 8) == sextract32(r2,  i * 8, 8));
+    }
+
+    return ret;
+}
+
+uint32_t helper_eqany_h(target_ulong r1, target_ulong r2)
+{
+    uint32_t ret;
+
+    ret = (sextract32(r1, 0, 16) == sextract32(r2,  0, 16));
+    ret |= (sextract32(r1, 16, 16) == sextract32(r2,  16, 16));
+
+    return ret;
+}
+
+uint32_t helper_lt_b(target_ulong r1, target_ulong r2)
+{
+    int32_t i;
+    uint32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        if (sextract32(r1,  i * 8, 8) < sextract32(r2,  i * 8, 8)) {
+            ret |= (0xff << (i * 8));
+        }
+    }
+
+    return ret;
+}
+
+uint32_t helper_lt_bu(target_ulong r1, target_ulong r2)
+{
+    int32_t i;
+    uint32_t ret = 0;
+
+    for (i = 0; i < 4; i++) {
+        if (extract32(r1,  i * 8, 8) < extract32(r2,  i * 8, 8)) {
+            ret |= (0xff << (i * 8));
+        }
+    }
+
+    return ret;
+}
+
+uint32_t helper_lt_h(target_ulong r1, target_ulong r2)
+{
+    uint32_t ret = 0;
+
+    if (sextract32(r1,  0, 16) < sextract32(r2,  0, 16)) {
+        ret |= 0xffff;
+    }
+
+    if (sextract32(r1,  16, 16) < sextract32(r2,  16, 16)) {
+        ret |= 0xffff0000;
+    }
+
+    return ret;
+}
+
+uint32_t helper_lt_hu(target_ulong r1, target_ulong r2)
+{
+    uint32_t ret = 0;
+
+    if (extract32(r1,  0, 16) < extract32(r2,  0, 16)) {
+        ret |= 0xffff;
+    }
+
+    if (extract32(r1,  16, 16) < extract32(r2,  16, 16)) {
+        ret |= 0xffff0000;
+    }
+
+    return ret;
+}
+
+#define EXTREMA_H_B(name, op)                                 \
+uint32_t helper_##name ##_b(target_ulong r1, target_ulong r2) \
+{                                                             \
+    int32_t i, extr_r1, extr_r2;                              \
+    uint32_t ret = 0;                                         \
+                                                              \
+    for (i = 0; i < 4; i++) {                                 \
+        extr_r1 = sextract32(r1, i * 8, 8);                   \
+        extr_r2 = sextract32(r2, i * 8, 8);                   \
+        extr_r1 = (extr_r1 op extr_r2) ? extr_r1 : extr_r2;   \
+        ret |= (extr_r1 & 0xff) << (i * 8);                   \
+    }                                                         \
+    return ret;                                               \
+}                                                             \
+                                                              \
+uint32_t helper_##name ##_bu(target_ulong r1, target_ulong r2)\
+{                                                             \
+    int32_t i;                                                \
+    uint32_t extr_r1, extr_r2;                                \
+    uint32_t ret = 0;                                         \
+                                                              \
+    for (i = 0; i < 4; i++) {                                 \
+        extr_r1 = extract32(r1, i * 8, 8);                    \
+        extr_r2 = extract32(r2, i * 8, 8);                    \
+        extr_r1 = (extr_r1 op extr_r2) ? extr_r1 : extr_r2;   \
+        ret |= (extr_r1 & 0xff) << (i * 8);                   \
+    }                                                         \
+    return ret;                                               \
+}                                                             \
+                                                              \
+uint32_t helper_##name ##_h(target_ulong r1, target_ulong r2) \
+{                                                             \
+    int32_t extr_r1, extr_r2;                                 \
+    uint32_t ret = 0;                                         \
+                                                              \
+    extr_r1 = sextract32(r1, 0, 16);                          \
+    extr_r2 = sextract32(r2, 0, 16);                          \
+    ret = (extr_r1 op extr_r2) ? extr_r1 : extr_r2;           \
+    ret = ret & 0xffff;                                       \
+                                                              \
+    extr_r1 = sextract32(r1, 16, 16);                         \
+    extr_r2 = sextract32(r2, 16, 16);                         \
+    extr_r1 = (extr_r1 op extr_r2) ? extr_r1 : extr_r2;       \
+    ret |= extr_r1 << 16;                                     \
+                                                              \
+    return ret;                                               \
+}                                                             \
+                                                              \
+uint32_t helper_##name ##_hu(target_ulong r1, target_ulong r2)\
+{                                                             \
+    uint32_t extr_r1, extr_r2;                                \
+    uint32_t ret = 0;                                         \
+                                                              \
+    extr_r1 = extract32(r1, 0, 16);                           \
+    extr_r2 = extract32(r2, 0, 16);                           \
+    ret = (extr_r1 op extr_r2) ? extr_r1 : extr_r2;           \
+    ret = ret & 0xffff;                                       \
+                                                              \
+    extr_r1 = extract32(r1, 16, 16);                          \
+    extr_r2 = extract32(r2, 16, 16);                          \
+    extr_r1 = (extr_r1 op extr_r2) ? extr_r1 : extr_r2;       \
+    ret |= extr_r1 << (16);                                   \
+                                                              \
+    return ret;                                               \
+}                                                             \
+                                                              \
+uint64_t helper_ix##name(uint64_t r1, uint32_t r2)            \
+{                                                             \
+    int64_t r2l, r2h, r1hl;                                   \
+    uint64_t ret = 0;                                         \
+                                                              \
+    ret = ((r1 + 2) & 0xffff);                                \
+    r2l = sextract64(r2, 0, 16);                              \
+    r2h = sextract64(r2, 16, 16);                             \
+    r1hl = sextract64(r1, 32, 16);                            \
+                                                              \
+    if ((r2l op ## = r2h) && (r2l op r1hl)) {                 \
+        ret |= (r2l & 0xffff) << 32;                          \
+        ret |= extract64(r1, 0, 16) << 16;                    \
+    } else if ((r2h op r2l) && (r2h op r1hl)) {               \
+        ret |= extract64(r2, 16, 16) << 32;                   \
+        ret |= extract64(r1 + 1, 0, 16) << 16;                \
+    } else {                                                  \
+        ret |= r1 & 0xffffffff0000ull;                        \
+    }                                                         \
+    return ret;                                               \
+}                                                             \
+                                                              \
+uint64_t helper_ix##name ##_u(uint64_t r1, uint32_t r2)       \
+{                                                             \
+    int64_t r2l, r2h, r1hl;                                   \
+    uint64_t ret = 0;                                         \
+                                                              \
+    ret = ((r1 + 2) & 0xffff);                                \
+    r2l = extract64(r2, 0, 16);                               \
+    r2h = extract64(r2, 16, 16);                              \
+    r1hl = extract64(r1, 32, 16);                             \
+                                                              \
+    if ((r2l op ## = r2h) && (r2l op r1hl)) {                 \
+        ret |= (r2l & 0xffff) << 32;                          \
+        ret |= extract64(r1, 0, 16) << 16;                    \
+    } else if ((r2h op r2l) && (r2h op r1hl)) {               \
+        ret |= extract64(r2, 16, 16) << 32;                   \
+        ret |= extract64(r1 + 1, 0, 16) << 16;                \
+    } else {                                                  \
+        ret |= r1 & 0xffffffff0000ull;                        \
+    }                                                         \
+    return ret;                                               \
+}
+
+EXTREMA_H_B(max, >)
+EXTREMA_H_B(min, <)
+
+#undef EXTREMA_H_B
+
+uint32_t helper_clo(target_ulong r1)
+{
+    return clo32(r1);
+}
+
+uint32_t helper_clo_h(target_ulong r1)
+{
+    uint32_t ret_hw0 = extract32(r1, 0, 16);
+    uint32_t ret_hw1 = extract32(r1, 16, 16);
+
+    ret_hw0 = clo32(ret_hw0 << 16);
+    ret_hw1 = clo32(ret_hw1 << 16);
+
+    if (ret_hw0 > 16) {
+        ret_hw0 = 16;
+    }
+    if (ret_hw1 > 16) {
+        ret_hw1 = 16;
+    }
+
+    return ret_hw0 | (ret_hw1 << 16);
+}
+
+uint32_t helper_clz(target_ulong r1)
+{
+    return clz32(r1);
+}
+
+uint32_t helper_clz_h(target_ulong r1)
+{
+    uint32_t ret_hw0 = extract32(r1, 0, 16);
+    uint32_t ret_hw1 = extract32(r1, 16, 16);
+
+    ret_hw0 = clz32(ret_hw0 << 16);
+    ret_hw1 = clz32(ret_hw1 << 16);
+
+    if (ret_hw0 > 16) {
+        ret_hw0 = 16;
+    }
+    if (ret_hw1 > 16) {
+        ret_hw1 = 16;
+    }
+
+    return ret_hw0 | (ret_hw1 << 16);
+}
+
+uint32_t helper_cls(target_ulong r1)
+{
+    return clrsb32(r1);
+}
+
+uint32_t helper_cls_h(target_ulong r1)
+{
+    uint32_t ret_hw0 = extract32(r1, 0, 16);
+    uint32_t ret_hw1 = extract32(r1, 16, 16);
+
+    ret_hw0 = clrsb32(ret_hw0 << 16);
+    ret_hw1 = clrsb32(ret_hw1 << 16);
+
+    if (ret_hw0 > 15) {
+        ret_hw0 = 15;
+    }
+    if (ret_hw1 > 15) {
+        ret_hw1 = 15;
+    }
+
+    return ret_hw0 | (ret_hw1 << 16);
+}
+
+uint32_t helper_sh(target_ulong r1, target_ulong r2)
+{
+    int32_t shift_count = sextract32(r2, 0, 6);
+
+    if (shift_count == -32) {
+        return 0;
+    } else if (shift_count < 0) {
+        return r1 >> -shift_count;
+    } else {
+        return r1 << shift_count;
+    }
+}
+
+uint32_t helper_sh_h(target_ulong r1, target_ulong r2)
+{
+    int32_t ret_hw0, ret_hw1;
+    int32_t shift_count;
+
+    shift_count = sextract32(r2, 0, 5);
+
+    if (shift_count == -16) {
+        return 0;
+    } else if (shift_count < 0) {
+        ret_hw0 = extract32(r1, 0, 16) >> -shift_count;
+        ret_hw1 = extract32(r1, 16, 16) >> -shift_count;
+        return (ret_hw0 & 0xffff) | (ret_hw1 << 16);
+    } else {
+        ret_hw0 = extract32(r1, 0, 16) << shift_count;
+        ret_hw1 = extract32(r1, 16, 16) << shift_count;
+        return (ret_hw0 & 0xffff) | (ret_hw1 << 16);
+    }
+}
+
+uint32_t helper_sha(CPUTriCoreState *env, target_ulong r1, target_ulong r2)
+{
+    int32_t shift_count;
+    int64_t result, t1;
+    uint32_t ret;
+
+    shift_count = sextract32(r2, 0, 6);
+    t1 = sextract32(r1, 0, 32);
+
+    if (shift_count == 0) {
+        env->PSW_USB_C = env->PSW_USB_V = 0;
+        ret = r1;
+    } else if (shift_count == -32) {
+        env->PSW_USB_C = r1;
+        env->PSW_USB_V = 0;
+        ret = t1 >> 31;
+    } else if (shift_count > 0) {
+        result = t1 << shift_count;
+        /* calc carry */
+        env->PSW_USB_C = ((result & 0xffffffff00000000ULL) != 0);
+        /* calc v */
+        env->PSW_USB_V = (((result > 0x7fffffffLL) ||
+                           (result < -0x80000000LL)) << 31);
+        /* calc sv */
+        env->PSW_USB_SV |= env->PSW_USB_V;
+        ret = (uint32_t)result;
+    } else {
+        env->PSW_USB_V = 0;
+        env->PSW_USB_C = (r1 & ((1 << -shift_count) - 1));
+        ret = t1 >> -shift_count;
+    }
+
+    env->PSW_USB_AV = ret ^ ret * 2u;
+    env->PSW_USB_SAV |= env->PSW_USB_AV;
+
+    return ret;
+}
+
+uint32_t helper_sha_h(target_ulong r1, target_ulong r2)
+{
+    int32_t shift_count;
+    int32_t ret_hw0, ret_hw1;
+
+    shift_count = sextract32(r2, 0, 5);
+
+    if (shift_count == 0) {
+        return r1;
+    } else if (shift_count < 0) {
+        ret_hw0 = sextract32(r1, 0, 16) >> -shift_count;
+        ret_hw1 = sextract32(r1, 16, 16) >> -shift_count;
+        return (ret_hw0 & 0xffff) | (ret_hw1 << 16);
+    } else {
+        ret_hw0 = sextract32(r1, 0, 16) << shift_count;
+        ret_hw1 = sextract32(r1, 16, 16) << shift_count;
+        return (ret_hw0 & 0xffff) | (ret_hw1 << 16);
+    }
+}
+
+uint32_t helper_bmerge(target_ulong r1, target_ulong r2)
+{
+    uint32_t i, ret;
+
+    ret = 0;
+    for (i = 0; i < 16; i++) {
+        ret |= (r1 & 1) << (2 * i + 1);
+        ret |= (r2 & 1) << (2 * i);
+        r1 = r1 >> 1;
+        r2 = r2 >> 1;
+    }
+    return ret;
+}
+
+uint64_t helper_bsplit(uint32_t r1)
+{
+    int32_t i;
+    uint64_t ret;
+
+    ret = 0;
+    for (i = 0; i < 32; i = i + 2) {
+        /* even */
+        ret |= (r1 & 1) << (i/2);
+        r1 = r1 >> 1;
+        /* odd */
+        ret |= (uint64_t)(r1 & 1) << (i/2 + 32);
+        r1 = r1 >> 1;
+    }
+    return ret;
+}
+
+uint32_t helper_parity(target_ulong r1)
+{
+    uint32_t ret;
+    uint32_t nOnes, i;
+
+    ret = 0;
+    nOnes = 0;
+    for (i = 0; i < 8; i++) {
+        ret ^= (r1 & 1);
+        r1 = r1 >> 1;
+    }
+    /* second byte */
+    nOnes = 0;
+    for (i = 0; i < 8; i++) {
+        nOnes ^= (r1 & 1);
+        r1 = r1 >> 1;
+    }
+    ret |= nOnes << 8;
+    /* third byte */
+    nOnes = 0;
+    for (i = 0; i < 8; i++) {
+        nOnes ^= (r1 & 1);
+        r1 = r1 >> 1;
+    }
+    ret |= nOnes << 16;
+    /* fourth byte */
+    nOnes = 0;
+    for (i = 0; i < 8; i++) {
+        nOnes ^= (r1 & 1);
+        r1 = r1 >> 1;
+    }
+    ret |= nOnes << 24;
+
+    return ret;
+}
+
+uint32_t helper_pack(uint32_t carry, uint32_t r1_low, uint32_t r1_high,
+                     target_ulong r2)
+{
+    uint32_t ret;
+    int32_t fp_exp, fp_frac, temp_exp, fp_exp_frac;
+    int32_t int_exp  = r1_high;
+    int32_t int_mant = r1_low;
+    uint32_t flag_rnd = (int_mant & (1 << 7)) && (
+                        (int_mant & (1 << 8)) ||
+                        (int_mant & 0x7f)     ||
+                        (carry != 0));
+    if (((int_mant & (1<<31)) == 0) && (int_exp == 255)) {
+        fp_exp = 255;
+        fp_frac = extract32(int_mant, 8, 23);
+    } else if ((int_mant & (1<<31)) && (int_exp >= 127)) {
+        fp_exp  = 255;
+        fp_frac = 0;
+    } else if ((int_mant & (1<<31)) && (int_exp <= -128)) {
+        fp_exp  = 0;
+        fp_frac = 0;
+    } else if (int_mant == 0) {
+        fp_exp  = 0;
+        fp_frac = 0;
+    } else {
+        if (((int_mant & (1 << 31)) == 0)) {
+            temp_exp = 0;
+        } else {
+            temp_exp = int_exp + 128;
+        }
+        fp_exp_frac = (((temp_exp & 0xff) << 23) |
+                      extract32(int_mant, 8, 23))
+                      + flag_rnd;
+        fp_exp  = extract32(fp_exp_frac, 23, 8);
+        fp_frac = extract32(fp_exp_frac, 0, 23);
+    }
+    ret = r2 & (1 << 31);
+    ret = ret + (fp_exp << 23);
+    ret = ret + (fp_frac & 0x7fffff);
+
+    return ret;
+}
+
+uint64_t helper_unpack(target_ulong arg1)
+{
+    int32_t fp_exp  = extract32(arg1, 23, 8);
+    int32_t fp_frac = extract32(arg1, 0, 23);
+    uint64_t ret;
+    int32_t int_exp, int_mant;
+
+    if (fp_exp == 255) {
+        int_exp = 255;
+        int_mant = (fp_frac << 7);
+    } else if ((fp_exp == 0) && (fp_frac == 0)) {
+        int_exp  = -127;
+        int_mant = 0;
+    } else if ((fp_exp == 0) && (fp_frac != 0)) {
+        int_exp  = -126;
+        int_mant = (fp_frac << 7);
+    } else {
+        int_exp  = fp_exp - 127;
+        int_mant = (fp_frac << 7);
+        int_mant |= (1 << 30);
+    }
+    ret = int_exp;
+    ret = ret << 32;
+    ret |= int_mant;
+
+    return ret;
+}
+
+uint64_t helper_dvinit_b_13(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t ret;
+    int32_t abs_sig_dividend, abs_divisor;
+
+    ret = sextract32(r1, 0, 32);
+    ret = ret << 24;
+    if (!((r1 & 0x80000000) == (r2 & 0x80000000))) {
+        ret |= 0xffffff;
+    }
+
+    abs_sig_dividend = abs((int32_t)r1) >> 8;
+    abs_divisor = abs((int32_t)r2);
+    /* calc overflow
+       ofv if (a/b >= 255) <=> (a/255 >= b) */
+    env->PSW_USB_V = (abs_sig_dividend >= abs_divisor) << 31;
+    env->PSW_USB_V = env->PSW_USB_V << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = 0;
+
+    return ret;
+}
+
+uint64_t helper_dvinit_b_131(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t ret = sextract32(r1, 0, 32);
+
+    ret = ret << 24;
+    if (!((r1 & 0x80000000) == (r2 & 0x80000000))) {
+        ret |= 0xffffff;
+    }
+    /* calc overflow */
+    env->PSW_USB_V = ((r2 == 0) || ((r2 == 0xffffffff) && (r1 == 0xffffff80)));
+    env->PSW_USB_V = env->PSW_USB_V << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = 0;
+
+    return ret;
+}
+
+uint64_t helper_dvinit_h_13(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t ret;
+    int32_t abs_sig_dividend, abs_divisor;
+
+    ret = sextract32(r1, 0, 32);
+    ret = ret << 16;
+    if (!((r1 & 0x80000000) == (r2 & 0x80000000))) {
+        ret |= 0xffff;
+    }
+
+    abs_sig_dividend = abs((int32_t)r1) >> 16;
+    abs_divisor = abs((int32_t)r2);
+    /* calc overflow
+       ofv if (a/b >= 0xffff) <=> (a/0xffff >= b) */
+    env->PSW_USB_V = (abs_sig_dividend >= abs_divisor) << 31;
+    env->PSW_USB_V = env->PSW_USB_V << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = 0;
+
+    return ret;
+}
+
+uint64_t helper_dvinit_h_131(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t ret = sextract32(r1, 0, 32);
+
+    ret = ret << 16;
+    if (!((r1 & 0x80000000) == (r2 & 0x80000000))) {
+        ret |= 0xffff;
+    }
+    /* calc overflow */
+    env->PSW_USB_V = ((r2 == 0) || ((r2 == 0xffffffff) && (r1 == 0xffff8000)));
+    env->PSW_USB_V = env->PSW_USB_V << 31;
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = 0;
+
+    return ret;
+}
+
+uint64_t helper_dvadj(uint64_t r1, uint32_t r2)
+{
+    int32_t x_sign = (r1 >> 63);
+    int32_t q_sign = x_sign ^ (r2 >> 31);
+    int32_t eq_pos = x_sign & ((r1 >> 32) == r2);
+    int32_t eq_neg = x_sign & ((r1 >> 32) == -r2);
+    uint32_t quotient;
+    uint64_t remainder;
+
+    if ((q_sign & ~eq_neg) | eq_pos) {
+        quotient = (r1 + 1) & 0xffffffff;
+    } else {
+        quotient = r1 & 0xffffffff;
+    }
+
+    if (eq_pos | eq_neg) {
+        remainder = 0;
+    } else {
+        remainder = (r1 & 0xffffffff00000000ull);
+    }
+    return remainder | quotient;
+}
+
+uint64_t helper_dvstep(uint64_t r1, uint32_t r2)
+{
+    int32_t dividend_sign = extract64(r1, 63, 1);
+    int32_t divisor_sign = extract32(r2, 31, 1);
+    int32_t quotient_sign = (dividend_sign != divisor_sign);
+    int32_t addend, dividend_quotient, remainder;
+    int32_t i, temp;
+
+    if (quotient_sign) {
+        addend = r2;
+    } else {
+        addend = -r2;
+    }
+    dividend_quotient = (int32_t)r1;
+    remainder = (int32_t)(r1 >> 32);
+
+    for (i = 0; i < 8; i++) {
+        remainder = (remainder << 1) | extract32(dividend_quotient, 31, 1);
+        dividend_quotient <<= 1;
+        temp = remainder + addend;
+        if ((temp < 0) == dividend_sign) {
+            remainder = temp;
+        }
+        if (((temp < 0) == dividend_sign)) {
+            dividend_quotient = dividend_quotient | !quotient_sign;
+        } else {
+            dividend_quotient = dividend_quotient | quotient_sign;
+        }
+    }
+    return ((uint64_t)remainder << 32) | (uint32_t)dividend_quotient;
+}
+
+uint64_t helper_dvstep_u(uint64_t r1, uint32_t r2)
+{
+    int32_t dividend_quotient = extract64(r1, 0, 32);
+    int64_t remainder = extract64(r1, 32, 32);
+    int32_t i;
+    int64_t temp;
+    for (i = 0; i < 8; i++) {
+        remainder = (remainder << 1) | extract32(dividend_quotient, 31, 1);
+        dividend_quotient <<= 1;
+        temp = (remainder & 0xffffffff) - r2;
+        if (temp >= 0) {
+            remainder = temp;
+        }
+        dividend_quotient = dividend_quotient | !(temp < 0);
+    }
+    return ((uint64_t)remainder << 32) | (uint32_t)dividend_quotient;
+}
+
+uint64_t helper_divide(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    int32_t quotient, remainder;
+    int32_t dividend = (int32_t)r1;
+    int32_t divisor = (int32_t)r2;
+
+    if (divisor == 0) {
+        if (dividend >= 0) {
+            quotient = 0x7fffffff;
+            remainder = 0;
+        } else {
+            quotient = 0x80000000;
+            remainder = 0;
+        }
+        env->PSW_USB_V = (1 << 31);
+    } else if ((divisor == 0xffffffff) && (dividend == 0x80000000)) {
+        quotient = 0x7fffffff;
+        remainder = 0;
+        env->PSW_USB_V = (1 << 31);
+    } else {
+        remainder = dividend % divisor;
+        quotient = (dividend - remainder)/divisor;
+        env->PSW_USB_V = 0;
+    }
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = 0;
+    return ((uint64_t)remainder << 32) | (uint32_t)quotient;
+}
+
+uint64_t helper_divide_u(CPUTriCoreState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t quotient, remainder;
+    uint32_t dividend = r1;
+    uint32_t divisor = r2;
+
+    if (divisor == 0) {
+        quotient = 0xffffffff;
+        remainder = 0;
+        env->PSW_USB_V = (1 << 31);
+    } else {
+        remainder = dividend % divisor;
+        quotient = (dividend - remainder)/divisor;
+        env->PSW_USB_V = 0;
+    }
+    env->PSW_USB_SV |= env->PSW_USB_V;
+    env->PSW_USB_AV = 0;
+    return ((uint64_t)remainder << 32) | quotient;
+}
+
+uint64_t helper_mul_h(uint32_t arg00, uint32_t arg01,
+                      uint32_t arg10, uint32_t arg11, uint32_t n)
+{
+    uint32_t result0, result1;
+
+    int32_t sc1 = ((arg00 & 0xffff) == 0x8000) &&
+                  ((arg10 & 0xffff) == 0x8000) && (n == 1);
+    int32_t sc0 = ((arg01 & 0xffff) == 0x8000) &&
+                  ((arg11 & 0xffff) == 0x8000) && (n == 1);
+    if (sc1) {
+        result1 = 0x7fffffff;
+    } else {
+        result1 = (((uint32_t)(arg00 * arg10)) << n);
+    }
+    if (sc0) {
+        result0 = 0x7fffffff;
+    } else {
+        result0 = (((uint32_t)(arg01 * arg11)) << n);
+    }
+    return (((uint64_t)result1 << 32)) | result0;
+}
+
+uint64_t helper_mulm_h(uint32_t arg00, uint32_t arg01,
+                       uint32_t arg10, uint32_t arg11, uint32_t n)
+{
+    uint64_t ret;
+    int64_t result0, result1;
+
+    int32_t sc1 = ((arg00 & 0xffff) == 0x8000) &&
+                  ((arg10 & 0xffff) == 0x8000) && (n == 1);
+    int32_t sc0 = ((arg01 & 0xffff) == 0x8000) &&
+                  ((arg11 & 0xffff) == 0x8000) && (n == 1);
+
+    if (sc1) {
+        result1 = 0x7fffffff;
+    } else {
+        result1 = (((int32_t)arg00 * (int32_t)arg10) << n);
+    }
+    if (sc0) {
+        result0 = 0x7fffffff;
+    } else {
+        result0 = (((int32_t)arg01 * (int32_t)arg11) << n);
+    }
+    ret = (result1 + result0);
+    ret = ret << 16;
+    return ret;
+}
+uint32_t helper_mulr_h(uint32_t arg00, uint32_t arg01,
+                       uint32_t arg10, uint32_t arg11, uint32_t n)
+{
+    uint32_t result0, result1;
+
+    int32_t sc1 = ((arg00 & 0xffff) == 0x8000) &&
+                  ((arg10 & 0xffff) == 0x8000) && (n == 1);
+    int32_t sc0 = ((arg01 & 0xffff) == 0x8000) &&
+                  ((arg11 & 0xffff) == 0x8000) && (n == 1);
+
+    if (sc1) {
+        result1 = 0x7fffffff;
+    } else {
+        result1 = ((arg00 * arg10) << n) + 0x8000;
+    }
+    if (sc0) {
+        result0 = 0x7fffffff;
+    } else {
+        result0 = ((arg01 * arg11) << n) + 0x8000;
+    }
+    return (result1 & 0xffff0000) | (result0 >> 16);
+}
+
+uint32_t helper_crc32(uint32_t arg0, uint32_t arg1)
+{
+    uint8_t buf[4];
+    stl_be_p(buf, arg0);
+
+    return crc32(arg1, buf, 4);
+}
+
 /* context save area (CSA) related helpers */
 
 static int cdc_increment(target_ulong *psw)
@@ -153,7 +2364,7 @@
 static void save_context_upper(CPUTriCoreState *env, int ea)
 {
     cpu_stl_data(env, ea, env->PCXI);
-    cpu_stl_data(env, ea+4, env->PSW);
+    cpu_stl_data(env, ea+4, psw_read(env));
     cpu_stl_data(env, ea+8, env->gpr_a[10]);
     cpu_stl_data(env, ea+12, env->gpr_a[11]);
     cpu_stl_data(env, ea+16, env->gpr_d[8]);
@@ -243,11 +2454,13 @@
     /* if (FCX == 0) trap(FCU); */
     if (env->FCX == 0) {
         /* FCU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
     }
     /* if (PSW.CDE) then if (cdc_increment()) then trap(CDO); */
     if (psw & MASK_PSW_CDE) {
         if (cdc_increment(&psw)) {
             /* CDO trap */
+            raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CDO, GETPC());
         }
     }
     /* PSW.CDE = 1;*/
@@ -283,6 +2496,7 @@
     /* if (tmp_FCX == LCX) trap(FCD);*/
     if (tmp_FCX == env->LCX) {
         /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
     }
     psw_write(env, psw);
 }
@@ -295,18 +2509,25 @@
 
     psw = psw_read(env);
      /* if (PSW.CDE) then if (cdc_decrement()) then trap(CDU);*/
-    if (env->PSW & MASK_PSW_CDE) {
-        if (cdc_decrement(&(env->PSW))) {
+    if (psw & MASK_PSW_CDE) {
+        if (cdc_decrement(&psw)) {
             /* CDU trap */
+            psw_write(env, psw);
+            raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CDU, GETPC());
         }
     }
     /*   if (PCXI[19: 0] == 0) then trap(CSU); */
     if ((env->PCXI & 0xfffff) == 0) {
         /* CSU trap */
+        psw_write(env, psw);
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CSU, GETPC());
     }
     /* if (PCXI.UL == 0) then trap(CTYP); */
     if ((env->PCXI & MASK_PCXI_UL) == 0) {
         /* CTYP trap */
+        cdc_increment(&psw); /* restore to the start of helper */
+        psw_write(env, psw);
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CTYP, GETPC());
     }
     /* PC = {A11 [31: 1], 1’b0}; */
     env->PC = env->gpr_a[11] & 0xfffffffe;
@@ -341,6 +2562,7 @@
 
     if (env->FCX == 0) {
         /* FCU trap */
+       raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
     }
 
     tmp_FCX = env->FCX;
@@ -372,6 +2594,7 @@
 
     if (tmp_FCX == env->LCX) {
         /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
     }
 }
 
@@ -383,15 +2606,19 @@
     /* if (PCXI[19: 0] == 0) then trap(CSU); */
     if ((env->PCXI & 0xfffff) == 0) {
         /* raise csu trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CSU, GETPC());
     }
     /* if (PCXI.UL == 0) then trap(CTYP); */
     if ((env->PCXI & MASK_PCXI_UL) == 0) {
         /* raise CTYP trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CTYP, GETPC());
     }
     /* if (!cdc_zero() AND PSW.CDE) then trap(NEST); */
     if (!cdc_zero(&(env->PSW)) && (env->PSW & MASK_PSW_CDE)) {
-        /* raise MNG trap */
+        /* raise NEST trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_NEST, GETPC());
     }
+    env->PC = env->gpr_a[11] & ~0x1;
     /* ICR.IE = PCXI.PIE; */
     env->ICR = (env->ICR & ~MASK_ICR_IE) + ((env->PCXI & MASK_PCXI_PIE) >> 15);
     /* ICR.CCPN = PCXI.PCPN; */
@@ -413,6 +2640,26 @@
     psw_write(env, new_PSW);
 }
 
+void helper_rfm(CPUTriCoreState *env)
+{
+    env->PC = (env->gpr_a[11] & ~0x1);
+    /* ICR.IE = PCXI.PIE; */
+    env->ICR = (env->ICR & ~MASK_ICR_IE) |
+               ((env->PCXI & MASK_PCXI_PIE) >> 15);
+    /* ICR.CCPN = PCXI.PCPN; */
+    env->ICR = (env->ICR & ~MASK_ICR_CCPN) |
+               ((env->PCXI & MASK_PCXI_PCPN) >> 24);
+    /* {PCXI, PSW, A[10], A[11]} = M(DCX, 4 * word); */
+    env->PCXI = cpu_ldl_data(env, env->DCX);
+    psw_write(env, cpu_ldl_data(env, env->DCX+4));
+    env->gpr_a[10] = cpu_ldl_data(env, env->DCX+8);
+    env->gpr_a[11] = cpu_ldl_data(env, env->DCX+12);
+
+    if (tricore_feature(env, TRICORE_FEATURE_131)) {
+        env->DBGTCR = 0;
+    }
+}
+
 void helper_ldlcx(CPUTriCoreState *env, uint32_t ea)
 {
     uint32_t dummy;
@@ -437,6 +2684,133 @@
     save_context_upper(env, ea);
 }
 
+void helper_svlcx(CPUTriCoreState *env)
+{
+    target_ulong tmp_FCX;
+    target_ulong ea;
+    target_ulong new_FCX;
+
+    if (env->FCX == 0) {
+        /* FCU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
+    }
+    /* tmp_FCX = FCX; */
+    tmp_FCX = env->FCX;
+    /* EA = {FCX.FCXS, 6'b0, FCX.FCXO, 6'b0}; */
+    ea = ((env->FCX & MASK_FCX_FCXS) << 12) +
+         ((env->FCX & MASK_FCX_FCXO) << 6);
+    /* new_FCX = M(EA, word); */
+    new_FCX = cpu_ldl_data(env, ea);
+    /* M(EA, 16 * word) = {PCXI, PSW, A[10], A[11], D[8], D[9], D[10], D[11],
+                           A[12], A[13], A[14], A[15], D[12], D[13], D[14],
+                           D[15]}; */
+    save_context_lower(env, ea);
+
+    /* PCXI.PCPN = ICR.CCPN; */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* PCXI.PIE = ICR.IE; */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.UL = 0; */
+    env->PCXI &= ~MASK_PCXI_UL;
+
+    /* PCXI[19: 0] = FCX[19: 0]; */
+    env->PCXI = (env->PCXI & 0xfff00000) + (env->FCX & 0xfffff);
+    /* FCX[19: 0] = new_FCX[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (new_FCX & 0xfffff);
+
+    /* if (tmp_FCX == LCX) trap(FCD);*/
+    if (tmp_FCX == env->LCX) {
+        /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
+    }
+}
+
+void helper_svucx(CPUTriCoreState *env)
+{
+    target_ulong tmp_FCX;
+    target_ulong ea;
+    target_ulong new_FCX;
+
+    if (env->FCX == 0) {
+        /* FCU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCU, GETPC());
+    }
+    /* tmp_FCX = FCX; */
+    tmp_FCX = env->FCX;
+    /* EA = {FCX.FCXS, 6'b0, FCX.FCXO, 6'b0}; */
+    ea = ((env->FCX & MASK_FCX_FCXS) << 12) +
+         ((env->FCX & MASK_FCX_FCXO) << 6);
+    /* new_FCX = M(EA, word); */
+    new_FCX = cpu_ldl_data(env, ea);
+    /* M(EA, 16 * word) = {PCXI, PSW, A[10], A[11], D[8], D[9], D[10], D[11],
+                           A[12], A[13], A[14], A[15], D[12], D[13], D[14],
+                           D[15]}; */
+    save_context_upper(env, ea);
+
+    /* PCXI.PCPN = ICR.CCPN; */
+    env->PCXI = (env->PCXI & 0xffffff) +
+                ((env->ICR & MASK_ICR_CCPN) << 24);
+    /* PCXI.PIE = ICR.IE; */
+    env->PCXI = ((env->PCXI & ~MASK_PCXI_PIE) +
+                ((env->ICR & MASK_ICR_IE) << 15));
+    /* PCXI.UL = 1; */
+    env->PCXI |= MASK_PCXI_UL;
+
+    /* PCXI[19: 0] = FCX[19: 0]; */
+    env->PCXI = (env->PCXI & 0xfff00000) + (env->FCX & 0xfffff);
+    /* FCX[19: 0] = new_FCX[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (new_FCX & 0xfffff);
+
+    /* if (tmp_FCX == LCX) trap(FCD);*/
+    if (tmp_FCX == env->LCX) {
+        /* FCD trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_FCD, GETPC());
+    }
+}
+
+void helper_rslcx(CPUTriCoreState *env)
+{
+    target_ulong ea;
+    target_ulong new_PCXI;
+    /*   if (PCXI[19: 0] == 0) then trap(CSU); */
+    if ((env->PCXI & 0xfffff) == 0) {
+        /* CSU trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CSU, GETPC());
+    }
+    /* if (PCXI.UL == 1) then trap(CTYP); */
+    if ((env->PCXI & MASK_PCXI_UL) != 0) {
+        /* CTYP trap */
+        raise_exception_sync_helper(env, TRAPC_CTX_MNG, TIN3_CTYP, GETPC());
+    }
+    /* EA = {PCXI.PCXS, 6'b0, PCXI.PCXO, 6'b0}; */
+    ea = ((env->PCXI & MASK_PCXI_PCXS) << 12) +
+         ((env->PCXI & MASK_PCXI_PCXO) << 6);
+    /* {new_PCXI, A[11], A[10], A[11], D[8], D[9], D[10], D[11], A[12],
+        A[13], A[14], A[15], D[12], D[13], D[14], D[15]} = M(EA, 16 * word); */
+    restore_context_lower(env, ea, &env->gpr_a[11], &new_PCXI);
+    /* M(EA, word) = FCX; */
+    cpu_stl_data(env, ea, env->FCX);
+    /* M(EA, word) = FCX; */
+    cpu_stl_data(env, ea, env->FCX);
+    /* FCX[19: 0] = PCXI[19: 0]; */
+    env->FCX = (env->FCX & 0xfff00000) + (env->PCXI & 0x000fffff);
+    /* PCXI = new_PCXI; */
+    env->PCXI = new_PCXI;
+}
+
+void helper_psw_write(CPUTriCoreState *env, uint32_t arg)
+{
+    psw_write(env, arg);
+}
+
+uint32_t helper_psw_read(CPUTriCoreState *env)
+{
+    return psw_read(env);
+}
+
+
 static inline void QEMU_NORETURN do_raise_exception_err(CPUTriCoreState *env,
                                                         uint32_t exception,
                                                         int error_code,
@@ -454,11 +2828,11 @@
     cpu_loop_exit(cs);
 }
 
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     int ret;
-    ret = cpu_tricore_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = cpu_tricore_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (ret) {
         TriCoreCPU *cpu = TRICORE_CPU(cs);
         CPUTriCoreState *env = &cpu->env;
diff --git a/target-tricore/translate.c b/target-tricore/translate.c
index 68e3c53..9a50df9 100644
--- a/target-tricore/translate.c
+++ b/target-tricore/translate.c
@@ -18,8 +18,10 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "exec/cpu_ldst.h"
 
@@ -27,6 +29,7 @@
 #include "exec/helper-gen.h"
 
 #include "tricore-opcodes.h"
+#include "exec/log.h"
 
 /*
  * TCG registers
@@ -45,7 +48,7 @@
 static TCGv cpu_PSW_AV;
 static TCGv cpu_PSW_SAV;
 /* CPU env */
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 
 #include "exec/gen-icount.h"
 
@@ -80,27 +83,43 @@
     BS_EXCP   = 3,
 };
 
+enum {
+    MODE_LL = 0,
+    MODE_LU = 1,
+    MODE_UL = 2,
+    MODE_UU = 3,
+};
+
 void tricore_cpu_dump_state(CPUState *cs, FILE *f,
                             fprintf_function cpu_fprintf, int flags)
 {
     TriCoreCPU *cpu = TRICORE_CPU(cs);
     CPUTriCoreState *env = &cpu->env;
+    uint32_t psw;
     int i;
 
-    cpu_fprintf(f, "PC=%08x\n", env->PC);
-    for (i = 0; i < 16; ++i) {
-        if ((i & 3) == 0) {
-            cpu_fprintf(f, "GPR A%02d:", i);
-        }
-        cpu_fprintf(f, " %s " TARGET_FMT_lx, regnames_a[i], env->gpr_a[i]);
-    }
-    for (i = 0; i < 16; ++i) {
-        if ((i & 3) == 0) {
-            cpu_fprintf(f, "GPR D%02d:", i);
-        }
-        cpu_fprintf(f, " %s " TARGET_FMT_lx, regnames_d[i], env->gpr_d[i]);
-    }
+    psw = psw_read(env);
 
+    cpu_fprintf(f, "PC: " TARGET_FMT_lx, env->PC);
+    cpu_fprintf(f, " PSW: " TARGET_FMT_lx, psw);
+    cpu_fprintf(f, " ICR: " TARGET_FMT_lx, env->ICR);
+    cpu_fprintf(f, "\nPCXI: " TARGET_FMT_lx, env->PCXI);
+    cpu_fprintf(f, " FCX: " TARGET_FMT_lx, env->FCX);
+    cpu_fprintf(f, " LCX: " TARGET_FMT_lx, env->LCX);
+
+    for (i = 0; i < 16; ++i) {
+        if ((i & 3) == 0) {
+            cpu_fprintf(f, "\nGPR A%02d:", i);
+        }
+        cpu_fprintf(f, " " TARGET_FMT_lx, env->gpr_a[i]);
+    }
+    for (i = 0; i < 16; ++i) {
+        if ((i & 3) == 0) {
+            cpu_fprintf(f, "\nGPR D%02d:", i);
+        }
+        cpu_fprintf(f, " " TARGET_FMT_lx, env->gpr_d[i]);
+    }
+    cpu_fprintf(f, "\n");
 }
 
 /*
@@ -115,10 +134,98 @@
     tcg_temp_free_i32(helper_tmp);                                \
     } while (0)
 
+#define GEN_HELPER_LL(name, ret, arg0, arg1, n) do {         \
+    TCGv arg00 = tcg_temp_new();                             \
+    TCGv arg01 = tcg_temp_new();                             \
+    TCGv arg11 = tcg_temp_new();                             \
+    tcg_gen_sari_tl(arg00, arg0, 16);                        \
+    tcg_gen_ext16s_tl(arg01, arg0);                          \
+    tcg_gen_ext16s_tl(arg11, arg1);                          \
+    gen_helper_##name(ret, arg00, arg01, arg11, arg11, n);   \
+    tcg_temp_free(arg00);                                    \
+    tcg_temp_free(arg01);                                    \
+    tcg_temp_free(arg11);                                    \
+} while (0)
+
+#define GEN_HELPER_LU(name, ret, arg0, arg1, n) do {         \
+    TCGv arg00 = tcg_temp_new();                             \
+    TCGv arg01 = tcg_temp_new();                             \
+    TCGv arg10 = tcg_temp_new();                             \
+    TCGv arg11 = tcg_temp_new();                             \
+    tcg_gen_sari_tl(arg00, arg0, 16);                        \
+    tcg_gen_ext16s_tl(arg01, arg0);                          \
+    tcg_gen_sari_tl(arg11, arg1, 16);                        \
+    tcg_gen_ext16s_tl(arg10, arg1);                          \
+    gen_helper_##name(ret, arg00, arg01, arg10, arg11, n);   \
+    tcg_temp_free(arg00);                                    \
+    tcg_temp_free(arg01);                                    \
+    tcg_temp_free(arg10);                                    \
+    tcg_temp_free(arg11);                                    \
+} while (0)
+
+#define GEN_HELPER_UL(name, ret, arg0, arg1, n) do {         \
+    TCGv arg00 = tcg_temp_new();                             \
+    TCGv arg01 = tcg_temp_new();                             \
+    TCGv arg10 = tcg_temp_new();                             \
+    TCGv arg11 = tcg_temp_new();                             \
+    tcg_gen_sari_tl(arg00, arg0, 16);                        \
+    tcg_gen_ext16s_tl(arg01, arg0);                          \
+    tcg_gen_sari_tl(arg10, arg1, 16);                        \
+    tcg_gen_ext16s_tl(arg11, arg1);                          \
+    gen_helper_##name(ret, arg00, arg01, arg10, arg11, n);   \
+    tcg_temp_free(arg00);                                    \
+    tcg_temp_free(arg01);                                    \
+    tcg_temp_free(arg10);                                    \
+    tcg_temp_free(arg11);                                    \
+} while (0)
+
+#define GEN_HELPER_UU(name, ret, arg0, arg1, n) do {         \
+    TCGv arg00 = tcg_temp_new();                             \
+    TCGv arg01 = tcg_temp_new();                             \
+    TCGv arg11 = tcg_temp_new();                             \
+    tcg_gen_sari_tl(arg01, arg0, 16);                        \
+    tcg_gen_ext16s_tl(arg00, arg0);                          \
+    tcg_gen_sari_tl(arg11, arg1, 16);                        \
+    gen_helper_##name(ret, arg00, arg01, arg11, arg11, n);   \
+    tcg_temp_free(arg00);                                    \
+    tcg_temp_free(arg01);                                    \
+    tcg_temp_free(arg11);                                    \
+} while (0)
+
+#define GEN_HELPER_RRR(name, rl, rh, al1, ah1, arg2) do {    \
+    TCGv_i64 ret = tcg_temp_new_i64();                       \
+    TCGv_i64 arg1 = tcg_temp_new_i64();                      \
+                                                             \
+    tcg_gen_concat_i32_i64(arg1, al1, ah1);                  \
+    gen_helper_##name(ret, arg1, arg2);                      \
+    tcg_gen_extr_i64_i32(rl, rh, ret);                       \
+                                                             \
+    tcg_temp_free_i64(ret);                                  \
+    tcg_temp_free_i64(arg1);                                 \
+} while (0)
+
+#define GEN_HELPER_RR(name, rl, rh, arg1, arg2) do {        \
+    TCGv_i64 ret = tcg_temp_new_i64();                      \
+                                                            \
+    gen_helper_##name(ret, cpu_env, arg1, arg2);            \
+    tcg_gen_extr_i64_i32(rl, rh, ret);                      \
+                                                            \
+    tcg_temp_free_i64(ret);                                 \
+} while (0)
+
 #define EA_ABS_FORMAT(con) (((con & 0x3C000) << 14) + (con & 0x3FFF))
 #define EA_B_ABSOLUT(con) (((offset & 0xf00000) << 8) | \
                            ((offset & 0x0fffff) << 1))
 
+/* For two 32-bit registers used a 64-bit register, the first
+   registernumber needs to be even. Otherwise we trap. */
+static inline void generate_trap(DisasContext *ctx, int class, int tin);
+#define CHECK_REG_PAIR(reg) do {                      \
+    if (reg & 0x1) {                                  \
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_OPD); \
+    }                                                 \
+} while (0)
+
 /* Functions for load/save to/from memory */
 
 static inline void gen_offset_ld(DisasContext *ctx, TCGv r1, TCGv r2,
@@ -204,6 +311,7 @@
     TCGv temp = tcg_temp_new();
     TCGv temp2 = tcg_temp_new();
 
+    CHECK_REG_PAIR(ereg);
     /* temp = (M(EA, word) */
     tcg_gen_qemu_ld_tl(temp, ea, ctx->mem_idx, MO_LEUL);
     /* temp = temp & ~E[a][63:32]) */
@@ -233,6 +341,96 @@
     tcg_temp_free(temp);
 }
 
+static void gen_cmpswap(DisasContext *ctx, int reg, TCGv ea)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    tcg_gen_qemu_ld_tl(temp, ea, ctx->mem_idx, MO_LEUL);
+    tcg_gen_movcond_tl(TCG_COND_EQ, temp2, cpu_gpr_d[reg+1], temp,
+                       cpu_gpr_d[reg], temp);
+    tcg_gen_qemu_st_tl(temp2, ea, ctx->mem_idx, MO_LEUL);
+    tcg_gen_mov_tl(cpu_gpr_d[reg], temp);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static void gen_swapmsk(DisasContext *ctx, int reg, TCGv ea)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+
+    tcg_gen_qemu_ld_tl(temp, ea, ctx->mem_idx, MO_LEUL);
+    tcg_gen_and_tl(temp2, cpu_gpr_d[reg], cpu_gpr_d[reg+1]);
+    tcg_gen_andc_tl(temp3, temp, cpu_gpr_d[reg+1]);
+    tcg_gen_or_tl(temp2, temp2, temp3);
+    tcg_gen_qemu_st_tl(temp2, ea, ctx->mem_idx, MO_LEUL);
+    tcg_gen_mov_tl(cpu_gpr_d[reg], temp);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+}
+
+
+/* We generate loads and store to core special function register (csfr) through
+   the function gen_mfcr and gen_mtcr. To handle access permissions, we use 3
+   makros R, A and E, which allow read-only, all and endinit protected access.
+   These makros also specify in which ISA version the csfr was introduced. */
+#define R(ADDRESS, REG, FEATURE)                                         \
+    case ADDRESS:                                                        \
+        if (tricore_feature(env, FEATURE)) {                             \
+            tcg_gen_ld_tl(ret, cpu_env, offsetof(CPUTriCoreState, REG)); \
+        }                                                                \
+        break;
+#define A(ADDRESS, REG, FEATURE) R(ADDRESS, REG, FEATURE)
+#define E(ADDRESS, REG, FEATURE) R(ADDRESS, REG, FEATURE)
+static inline void gen_mfcr(CPUTriCoreState *env, TCGv ret, int32_t offset)
+{
+    /* since we're caching PSW make this a special case */
+    if (offset == 0xfe04) {
+        gen_helper_psw_read(ret, cpu_env);
+    } else {
+        switch (offset) {
+#include "csfr.def"
+        }
+    }
+}
+#undef R
+#undef A
+#undef E
+
+#define R(ADDRESS, REG, FEATURE) /* don't gen writes to read-only reg,
+                                    since no execption occurs */
+#define A(ADDRESS, REG, FEATURE) R(ADDRESS, REG, FEATURE)                \
+    case ADDRESS:                                                        \
+        if (tricore_feature(env, FEATURE)) {                             \
+            tcg_gen_st_tl(r1, cpu_env, offsetof(CPUTriCoreState, REG));  \
+        }                                                                \
+        break;
+/* Endinit protected registers
+   TODO: Since the endinit bit is in a register of a not yet implemented
+         watchdog device, we handle endinit protected registers like
+         all-access registers for now. */
+#define E(ADDRESS, REG, FEATURE) A(ADDRESS, REG, FEATURE)
+static inline void gen_mtcr(CPUTriCoreState *env, DisasContext *ctx, TCGv r1,
+                            int32_t offset)
+{
+    if ((ctx->hflags & TRICORE_HFLAG_KUU) == TRICORE_HFLAG_SM) {
+        /* since we're caching PSW make this a special case */
+        if (offset == 0xfe04) {
+            gen_helper_psw_write(cpu_env, r1);
+        } else {
+            switch (offset) {
+#include "csfr.def"
+            }
+        }
+    } else {
+        /* generate privilege trap */
+    }
+}
+
 /* Functions for arithmetic instructions  */
 
 static inline void gen_add_d(TCGv ret, TCGv r1, TCGv r2)
@@ -259,12 +457,1107 @@
     tcg_temp_free(t0);
 }
 
+static inline void
+gen_add64_d(TCGv_i64 ret, TCGv_i64 r1, TCGv_i64 r2)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv_i64 t0 = tcg_temp_new_i64();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 result = tcg_temp_new_i64();
+
+    tcg_gen_add_i64(result, r1, r2);
+    /* calc v bit */
+    tcg_gen_xor_i64(t1, result, r1);
+    tcg_gen_xor_i64(t0, r1, r2);
+    tcg_gen_andc_i64(t1, t1, t0);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t1);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* calc AV/SAV bits */
+    tcg_gen_extrh_i64_i32(temp, result);
+    tcg_gen_add_tl(cpu_PSW_AV, temp, temp);
+    tcg_gen_xor_tl(cpu_PSW_AV, temp, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_i64(ret, result);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(result);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
+static inline void
+gen_addsub64_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+               TCGv r3, void(*op1)(TCGv, TCGv, TCGv),
+               void(*op2)(TCGv, TCGv, TCGv))
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv temp4 = tcg_temp_new();
+
+    (*op1)(temp, r1_low, r2);
+    /* calc V0 bit */
+    tcg_gen_xor_tl(temp2, temp, r1_low);
+    tcg_gen_xor_tl(temp3, r1_low, r2);
+    if (op1 == tcg_gen_add_tl) {
+        tcg_gen_andc_tl(temp2, temp2, temp3);
+    } else {
+        tcg_gen_and_tl(temp2, temp2, temp3);
+    }
+
+    (*op2)(temp3, r1_high, r3);
+    /* calc V1 bit */
+    tcg_gen_xor_tl(cpu_PSW_V, temp3, r1_high);
+    tcg_gen_xor_tl(temp4, r1_high, r3);
+    if (op2 == tcg_gen_add_tl) {
+        tcg_gen_andc_tl(cpu_PSW_V, cpu_PSW_V, temp4);
+    } else {
+        tcg_gen_and_tl(cpu_PSW_V, cpu_PSW_V, temp4);
+    }
+    /* combine V0/V1 bits */
+    tcg_gen_or_tl(cpu_PSW_V, cpu_PSW_V, temp2);
+    /* calc sv bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* write result */
+    tcg_gen_mov_tl(ret_low, temp);
+    tcg_gen_mov_tl(ret_high, temp3);
+    /* calc AV bit */
+    tcg_gen_add_tl(temp, ret_low, ret_low);
+    tcg_gen_xor_tl(temp, temp, ret_low);
+    tcg_gen_add_tl(cpu_PSW_AV, ret_high, ret_high);
+    tcg_gen_xor_tl(cpu_PSW_AV, cpu_PSW_AV, ret_high);
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free(temp4);
+}
+
+/* ret = r2 + (r1 * r3); */
+static inline void gen_madd32_d(TCGv ret, TCGv r1, TCGv r2, TCGv r3)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    tcg_gen_ext_i32_i64(t1, r1);
+    tcg_gen_ext_i32_i64(t2, r2);
+    tcg_gen_ext_i32_i64(t3, r3);
+
+    tcg_gen_mul_i64(t1, t1, t3);
+    tcg_gen_add_i64(t1, t2, t1);
+
+    tcg_gen_extrl_i64_i32(ret, t1);
+    /* calc V
+       t1 > 0x7fffffff */
+    tcg_gen_setcondi_i64(TCG_COND_GT, t3, t1, 0x7fffffffLL);
+    /* t1 < -0x80000000 */
+    tcg_gen_setcondi_i64(TCG_COND_LT, t2, t1, -0x80000000LL);
+    tcg_gen_or_i64(t2, t2, t3);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+}
+
+static inline void gen_maddi32_d(TCGv ret, TCGv r1, TCGv r2, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_madd32_d(ret, r1, r2, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_madd64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+             TCGv r3)
+{
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
+    TCGv t3 = tcg_temp_new();
+    TCGv t4 = tcg_temp_new();
+
+    tcg_gen_muls2_tl(t1, t2, r1, r3);
+    /* only the add can overflow */
+    tcg_gen_add2_tl(t3, t4, r2_low, r2_high, t1, t2);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, t4, r2_high);
+    tcg_gen_xor_tl(t1, r2_high, t2);
+    tcg_gen_andc_tl(cpu_PSW_V, cpu_PSW_V, t1);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, t4, t4);
+    tcg_gen_xor_tl(cpu_PSW_AV, t4, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back the result */
+    tcg_gen_mov_tl(ret_low, t3);
+    tcg_gen_mov_tl(ret_high, t4);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
+    tcg_temp_free(t3);
+    tcg_temp_free(t4);
+}
+
+static inline void
+gen_maddu64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+              TCGv r3)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    tcg_gen_extu_i32_i64(t1, r1);
+    tcg_gen_concat_i32_i64(t2, r2_low, r2_high);
+    tcg_gen_extu_i32_i64(t3, r3);
+
+    tcg_gen_mul_i64(t1, t1, t3);
+    tcg_gen_add_i64(t2, t2, t1);
+    /* write back result */
+    tcg_gen_extr_i64_i32(ret_low, ret_high, t2);
+    /* only the add overflows, if t2 < t1
+       calc V bit */
+    tcg_gen_setcond_i64(TCG_COND_LTU, t2, t2, t1);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, ret_high, ret_high);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret_high, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+}
+
+static inline void
+gen_maddi64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+              int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_madd64_d(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_maddui64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+               int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_maddu64_d(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_madd_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+           TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_addsub64_h(ret_low, ret_high, r1_low, r1_high, temp, temp2,
+                   tcg_gen_add_tl, tcg_gen_add_tl);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddsu_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+             TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_addsub64_h(ret_low, ret_high, r1_low, r1_high, temp, temp2,
+                   tcg_gen_sub_tl, tcg_gen_add_tl);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddsum_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+              TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+    TCGv_i64 temp64_3 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_concat_i32_i64(temp64_3, r1_low, r1_high);
+    tcg_gen_sari_i64(temp64_2, temp64, 32); /* high */
+    tcg_gen_ext32s_i64(temp64, temp64); /* low */
+    tcg_gen_sub_i64(temp64, temp64_2, temp64);
+    tcg_gen_shli_i64(temp64, temp64, 16);
+
+    gen_add64_d(temp64_2, temp64_3, temp64);
+    /* write back result */
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64_2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+    tcg_temp_free_i64(temp64_3);
+}
+
+static inline void gen_adds(TCGv ret, TCGv r1, TCGv r2);
+
+static inline void
+gen_madds_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+           TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_adds(ret_low, r1_low, temp);
+    tcg_gen_mov_tl(temp, cpu_PSW_V);
+    tcg_gen_mov_tl(temp3, cpu_PSW_AV);
+    gen_adds(ret_high, r1_high, temp2);
+    /* combine v bits */
+    tcg_gen_or_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* combine av bits */
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free_i64(temp64);
+
+}
+
+static inline void gen_subs(TCGv ret, TCGv r1, TCGv r2);
+
+static inline void
+gen_maddsus_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+              TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_subs(ret_low, r1_low, temp);
+    tcg_gen_mov_tl(temp, cpu_PSW_V);
+    tcg_gen_mov_tl(temp3, cpu_PSW_AV);
+    gen_adds(ret_high, r1_high, temp2);
+    /* combine v bits */
+    tcg_gen_or_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* combine av bits */
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free_i64(temp64);
+
+}
+
+static inline void
+gen_maddsums_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+               TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_sari_i64(temp64_2, temp64, 32); /* high */
+    tcg_gen_ext32s_i64(temp64, temp64); /* low */
+    tcg_gen_sub_i64(temp64, temp64_2, temp64);
+    tcg_gen_shli_i64(temp64, temp64, 16);
+    tcg_gen_concat_i32_i64(temp64_2, r1_low, r1_high);
+
+    gen_helper_add64_ssov(temp64, cpu_env, temp64_2, temp64);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+}
+
+
+static inline void
+gen_maddm_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+           TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+    TCGv_i64 temp64_3 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mulm_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_concat_i32_i64(temp64_2, r1_low, r1_high);
+    gen_add64_d(temp64_3, temp64_2, temp64);
+    /* write back result */
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64_3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+    tcg_temp_free_i64(temp64_3);
+}
+
+static inline void
+gen_maddms_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+           TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mulm_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_concat_i32_i64(temp64_2, r1_low, r1_high);
+    gen_helper_add64_ssov(temp64, cpu_env, temp64_2, temp64);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+}
+
+static inline void
+gen_maddr64_h(TCGv ret, TCGv r1_low, TCGv r1_high, TCGv r2, TCGv r3, uint32_t n,
+              uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    gen_helper_addr_h(ret, cpu_env, temp64, r1_low, r1_high);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddr32_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_maddr64_h(ret, temp, temp2, r2, r3, n, mode);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_maddsur32_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_helper_addsur_h(ret, cpu_env, temp64, temp, temp2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+
+static inline void
+gen_maddr64s_h(TCGv ret, TCGv r1_low, TCGv r1_high, TCGv r2, TCGv r3,
+               uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    gen_helper_addr_h_ssov(ret, cpu_env, temp64, r1_low, r1_high);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddr32s_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_maddr64s_h(ret, temp, temp2, r2, r3, n, mode);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_maddsur32s_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_helper_addsur_h_ssov(ret, cpu_env, temp64, temp, temp2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddr_q(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n)
+{
+    TCGv temp = tcg_const_i32(n);
+    gen_helper_maddr_q(ret, cpu_env, r1, r2, r3, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_maddrs_q(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n)
+{
+    TCGv temp = tcg_const_i32(n);
+    gen_helper_maddr_q_ssov(ret, cpu_env, r1, r2, r3, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_madd32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
+             uint32_t up_shift, CPUTriCoreState *env)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    tcg_gen_ext_i32_i64(t2, arg2);
+    tcg_gen_ext_i32_i64(t3, arg3);
+
+    tcg_gen_mul_i64(t2, t2, t3);
+    tcg_gen_shli_i64(t2, t2, n);
+
+    tcg_gen_ext_i32_i64(t1, arg1);
+    tcg_gen_sari_i64(t2, t2, up_shift);
+
+    tcg_gen_add_i64(t3, t1, t2);
+    tcg_gen_extrl_i64_i32(temp3, t3);
+    /* calc v bit */
+    tcg_gen_setcondi_i64(TCG_COND_GT, t1, t3, 0x7fffffffLL);
+    tcg_gen_setcondi_i64(TCG_COND_LT, t2, t3, -0x80000000LL);
+    tcg_gen_or_i64(t1, t1, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* We produce an overflow on the host if the mul before was
+       (0x80000000 * 0x80000000) << 1). If this is the
+       case, we negate the ovf. */
+    if (n == 1) {
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp, arg2, 0x80000000);
+        tcg_gen_setcond_tl(TCG_COND_EQ, temp2, arg2, arg3);
+        tcg_gen_and_tl(temp, temp, temp2);
+        tcg_gen_shli_tl(temp, temp, 31);
+        /* negate v bit, if special condition */
+        tcg_gen_xor_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    }
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, temp3, temp3);
+    tcg_gen_xor_tl(cpu_PSW_AV, temp3, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, temp3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+}
+
+static inline void
+gen_m16add32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    gen_add_d(ret, arg1, temp);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_m16adds32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    gen_adds(ret, arg1, temp);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_m16add64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+               TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    tcg_gen_ext_i32_i64(t2, temp);
+    tcg_gen_shli_i64(t2, t2, 16);
+    tcg_gen_concat_i32_i64(t1, arg1_low, arg1_high);
+    gen_add64_d(t3, t1, t2);
+    /* write back result */
+    tcg_gen_extr_i64_i32(rl, rh, t3);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_m16adds64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+               TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    tcg_gen_ext_i32_i64(t2, temp);
+    tcg_gen_shli_i64(t2, t2, 16);
+    tcg_gen_concat_i32_i64(t1, arg1_low, arg1_high);
+
+    gen_helper_add64_ssov(t1, cpu_env, t1, t2);
+    tcg_gen_extr_i64_i32(rl, rh, t1);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+}
+
+static inline void
+gen_madd64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+             TCGv arg3, uint32_t n, CPUTriCoreState *env)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_temp_new_i64();
+    TCGv temp, temp2;
+
+    tcg_gen_concat_i32_i64(t1, arg1_low, arg1_high);
+    tcg_gen_ext_i32_i64(t2, arg2);
+    tcg_gen_ext_i32_i64(t3, arg3);
+
+    tcg_gen_mul_i64(t2, t2, t3);
+    if (n != 0) {
+        tcg_gen_shli_i64(t2, t2, 1);
+    }
+    tcg_gen_add_i64(t4, t1, t2);
+    /* calc v bit */
+    tcg_gen_xor_i64(t3, t4, t1);
+    tcg_gen_xor_i64(t2, t1, t2);
+    tcg_gen_andc_i64(t3, t3, t2);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t3);
+    /* We produce an overflow on the host if the mul before was
+       (0x80000000 * 0x80000000) << 1). If this is the
+       case, we negate the ovf. */
+    if (n == 1) {
+        temp = tcg_temp_new();
+        temp2 = tcg_temp_new();
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp, arg2, 0x80000000);
+        tcg_gen_setcond_tl(TCG_COND_EQ, temp2, arg2, arg3);
+        tcg_gen_and_tl(temp, temp, temp2);
+        tcg_gen_shli_tl(temp, temp, 31);
+        /* negate v bit, if special condition */
+        tcg_gen_xor_tl(cpu_PSW_V, cpu_PSW_V, temp);
+
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+    }
+    /* write back result */
+    tcg_gen_extr_i64_i32(rl, rh, t4);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, rh, rh);
+    tcg_gen_xor_tl(cpu_PSW_AV, rh, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+    tcg_temp_free_i64(t4);
+}
+
+static inline void
+gen_madds32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
+              uint32_t up_shift)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    tcg_gen_ext_i32_i64(t1, arg1);
+    tcg_gen_ext_i32_i64(t2, arg2);
+    tcg_gen_ext_i32_i64(t3, arg3);
+
+    tcg_gen_mul_i64(t2, t2, t3);
+    tcg_gen_sari_i64(t2, t2, up_shift - n);
+
+    gen_helper_madd32_q_add_ssov(ret, cpu_env, t1, t2);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+}
+
+static inline void
+gen_madds64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+             TCGv arg3, uint32_t n)
+{
+    TCGv_i64 r1 = tcg_temp_new_i64();
+    TCGv temp = tcg_const_i32(n);
+
+    tcg_gen_concat_i32_i64(r1, arg1_low, arg1_high);
+    gen_helper_madd64_q_ssov(r1, cpu_env, r1, arg2, arg3, temp);
+    tcg_gen_extr_i64_i32(rl, rh, r1);
+
+    tcg_temp_free_i64(r1);
+    tcg_temp_free(temp);
+}
+/* ret = r2 - (r1 * r3); */
+static inline void gen_msub32_d(TCGv ret, TCGv r1, TCGv r2, TCGv r3)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    tcg_gen_ext_i32_i64(t1, r1);
+    tcg_gen_ext_i32_i64(t2, r2);
+    tcg_gen_ext_i32_i64(t3, r3);
+
+    tcg_gen_mul_i64(t1, t1, t3);
+    tcg_gen_sub_i64(t1, t2, t1);
+
+    tcg_gen_extrl_i64_i32(ret, t1);
+    /* calc V
+       t2 > 0x7fffffff */
+    tcg_gen_setcondi_i64(TCG_COND_GT, t3, t1, 0x7fffffffLL);
+    /* result < -0x80000000 */
+    tcg_gen_setcondi_i64(TCG_COND_LT, t2, t1, -0x80000000LL);
+    tcg_gen_or_i64(t2, t2, t3);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t2);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+}
+
+static inline void gen_msubi32_d(TCGv ret, TCGv r1, TCGv r2, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_msub32_d(ret, r1, r2, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msub64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+             TCGv r3)
+{
+    TCGv t1 = tcg_temp_new();
+    TCGv t2 = tcg_temp_new();
+    TCGv t3 = tcg_temp_new();
+    TCGv t4 = tcg_temp_new();
+
+    tcg_gen_muls2_tl(t1, t2, r1, r3);
+    /* only the sub can overflow */
+    tcg_gen_sub2_tl(t3, t4, r2_low, r2_high, t1, t2);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, t4, r2_high);
+    tcg_gen_xor_tl(t1, r2_high, t2);
+    tcg_gen_and_tl(cpu_PSW_V, cpu_PSW_V, t1);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, t4, t4);
+    tcg_gen_xor_tl(cpu_PSW_AV, t4, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back the result */
+    tcg_gen_mov_tl(ret_low, t3);
+    tcg_gen_mov_tl(ret_high, t4);
+
+    tcg_temp_free(t1);
+    tcg_temp_free(t2);
+    tcg_temp_free(t3);
+    tcg_temp_free(t4);
+}
+
+static inline void
+gen_msubi64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+              int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_msub64_d(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msubu64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+              TCGv r3)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    tcg_gen_extu_i32_i64(t1, r1);
+    tcg_gen_concat_i32_i64(t2, r2_low, r2_high);
+    tcg_gen_extu_i32_i64(t3, r3);
+
+    tcg_gen_mul_i64(t1, t1, t3);
+    tcg_gen_sub_i64(t3, t2, t1);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, t3);
+    /* calc V bit, only the sub can overflow, if t1 > t2 */
+    tcg_gen_setcond_i64(TCG_COND_GTU, t1, t1, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, ret_high, ret_high);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret_high, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+}
+
+static inline void
+gen_msubui64_d(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+               int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_msubu64_d(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
 static inline void gen_addi_d(TCGv ret, TCGv r1, target_ulong r2)
 {
     TCGv temp = tcg_const_i32(r2);
     gen_add_d(ret, r1, temp);
     tcg_temp_free(temp);
 }
+/* calculate the carry bit too */
+static inline void gen_add_CC(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv t0    = tcg_temp_new_i32();
+    TCGv result = tcg_temp_new_i32();
+
+    tcg_gen_movi_tl(t0, 0);
+    /* Addition and set C/V/SV bits */
+    tcg_gen_add2_i32(result, cpu_PSW_C, r1, t0, r2, t0);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, result, r1);
+    tcg_gen_xor_tl(t0, r1, r2);
+    tcg_gen_andc_tl(cpu_PSW_V, cpu_PSW_V, t0);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, result, result);
+    tcg_gen_xor_tl(cpu_PSW_AV, result, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, result);
+
+    tcg_temp_free(result);
+    tcg_temp_free(t0);
+}
+
+static inline void gen_addi_CC(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_add_CC(ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_addc_CC(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv carry = tcg_temp_new_i32();
+    TCGv t0    = tcg_temp_new_i32();
+    TCGv result = tcg_temp_new_i32();
+
+    tcg_gen_movi_tl(t0, 0);
+    tcg_gen_setcondi_tl(TCG_COND_NE, carry, cpu_PSW_C, 0);
+    /* Addition, carry and set C/V/SV bits */
+    tcg_gen_add2_i32(result, cpu_PSW_C, r1, t0, carry, t0);
+    tcg_gen_add2_i32(result, cpu_PSW_C, result, cpu_PSW_C, r2, t0);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, result, r1);
+    tcg_gen_xor_tl(t0, r1, r2);
+    tcg_gen_andc_tl(cpu_PSW_V, cpu_PSW_V, t0);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, result, result);
+    tcg_gen_xor_tl(cpu_PSW_AV, result, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, result);
+
+    tcg_temp_free(result);
+    tcg_temp_free(t0);
+    tcg_temp_free(carry);
+}
+
+static inline void gen_addci_CC(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_addc_CC(ret, r1, temp);
+    tcg_temp_free(temp);
+}
 
 static inline void gen_cond_add(TCGCond cond, TCGv r1, TCGv r2, TCGv r3,
                                 TCGv r4)
@@ -296,7 +1589,7 @@
     tcg_gen_and_tl(temp, temp, mask);
     tcg_gen_or_tl(cpu_PSW_SAV, temp, cpu_PSW_SAV);
     /* write back result */
-    tcg_gen_movcond_tl(cond, r3, r4, t0, result, r3);
+    tcg_gen_movcond_tl(cond, r3, r4, t0, result, r1);
 
     tcg_temp_free(t0);
     tcg_temp_free(temp);
@@ -337,6 +1630,856 @@
     tcg_temp_free(result);
 }
 
+static inline void
+gen_sub64_d(TCGv_i64 ret, TCGv_i64 r1, TCGv_i64 r2)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv_i64 t0 = tcg_temp_new_i64();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 result = tcg_temp_new_i64();
+
+    tcg_gen_sub_i64(result, r1, r2);
+    /* calc v bit */
+    tcg_gen_xor_i64(t1, result, r1);
+    tcg_gen_xor_i64(t0, r1, r2);
+    tcg_gen_and_i64(t1, t1, t0);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t1);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* calc AV/SAV bits */
+    tcg_gen_extrh_i64_i32(temp, result);
+    tcg_gen_add_tl(cpu_PSW_AV, temp, temp);
+    tcg_gen_xor_tl(cpu_PSW_AV, temp, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_i64(ret, result);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(result);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
+static inline void gen_sub_CC(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv result = tcg_temp_new();
+    TCGv temp = tcg_temp_new();
+
+    tcg_gen_sub_tl(result, r1, r2);
+    /* calc C bit */
+    tcg_gen_setcond_tl(TCG_COND_GEU, cpu_PSW_C, r1, r2);
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, result, r1);
+    tcg_gen_xor_tl(temp, r1, r2);
+    tcg_gen_and_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, result, result);
+    tcg_gen_xor_tl(cpu_PSW_AV, result, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, result);
+
+    tcg_temp_free(result);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_subc_CC(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv temp = tcg_temp_new();
+    tcg_gen_not_tl(temp, r2);
+    gen_addc_CC(ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_cond_sub(TCGCond cond, TCGv r1, TCGv r2, TCGv r3,
+                                TCGv r4)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv result = tcg_temp_new();
+    TCGv mask = tcg_temp_new();
+    TCGv t0 = tcg_const_i32(0);
+
+    /* create mask for sticky bits */
+    tcg_gen_setcond_tl(cond, mask, r4, t0);
+    tcg_gen_shli_tl(mask, mask, 31);
+
+    tcg_gen_sub_tl(result, r1, r2);
+    /* Calc PSW_V */
+    tcg_gen_xor_tl(temp, result, r1);
+    tcg_gen_xor_tl(temp2, r1, r2);
+    tcg_gen_and_tl(temp, temp, temp2);
+    tcg_gen_movcond_tl(cond, cpu_PSW_V, r4, t0, temp, cpu_PSW_V);
+    /* Set PSW_SV */
+    tcg_gen_and_tl(temp, temp, mask);
+    tcg_gen_or_tl(cpu_PSW_SV, temp, cpu_PSW_SV);
+    /* calc AV bit */
+    tcg_gen_add_tl(temp, result, result);
+    tcg_gen_xor_tl(temp, temp, result);
+    tcg_gen_movcond_tl(cond, cpu_PSW_AV, r4, t0, temp, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_and_tl(temp, temp, mask);
+    tcg_gen_or_tl(cpu_PSW_SAV, temp, cpu_PSW_SAV);
+    /* write back result */
+    tcg_gen_movcond_tl(cond, r3, r4, t0, result, r1);
+
+    tcg_temp_free(t0);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(result);
+    tcg_temp_free(mask);
+}
+
+static inline void
+gen_msub_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+           TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_addsub64_h(ret_low, ret_high, r1_low, r1_high, temp, temp2,
+                   tcg_gen_sub_tl, tcg_gen_sub_tl);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubs_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+            TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_subs(ret_low, r1_low, temp);
+    tcg_gen_mov_tl(temp, cpu_PSW_V);
+    tcg_gen_mov_tl(temp3, cpu_PSW_AV);
+    gen_subs(ret_high, r1_high, temp2);
+    /* combine v bits */
+    tcg_gen_or_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* combine av bits */
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubm_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+            TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+    TCGv_i64 temp64_3 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mulm_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_concat_i32_i64(temp64_2, r1_low, r1_high);
+    gen_sub64_d(temp64_3, temp64_2, temp64);
+    /* write back result */
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64_3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+    tcg_temp_free_i64(temp64_3);
+}
+
+static inline void
+gen_msubms_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+             TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mulm_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mulm_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_concat_i32_i64(temp64_2, r1_low, r1_high);
+    gen_helper_sub64_ssov(temp64, cpu_env, temp64_2, temp64);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+}
+
+static inline void
+gen_msubr64_h(TCGv ret, TCGv r1_low, TCGv r1_high, TCGv r2, TCGv r3, uint32_t n,
+              uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    gen_helper_subr_h(ret, cpu_env, temp64, r1_low, r1_high);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubr32_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_msubr64_h(ret, temp, temp2, r2, r3, n, mode);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_msubr64s_h(TCGv ret, TCGv r1_low, TCGv r1_high, TCGv r2, TCGv r3,
+               uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    gen_helper_subr_h_ssov(ret, cpu_env, temp64, r1_low, r1_high);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubr32s_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_msubr64s_h(ret, temp, temp2, r2, r3, n, mode);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_msubr_q(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n)
+{
+    TCGv temp = tcg_const_i32(n);
+    gen_helper_msubr_q(ret, cpu_env, r1, r2, r3, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msubrs_q(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n)
+{
+    TCGv temp = tcg_const_i32(n);
+    gen_helper_msubr_q_ssov(ret, cpu_env, r1, r2, r3, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msub32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
+             uint32_t up_shift, CPUTriCoreState *env)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_temp_new_i64();
+
+    tcg_gen_ext_i32_i64(t2, arg2);
+    tcg_gen_ext_i32_i64(t3, arg3);
+
+    tcg_gen_mul_i64(t2, t2, t3);
+
+    tcg_gen_ext_i32_i64(t1, arg1);
+    /* if we shift part of the fraction out, we need to round up */
+    tcg_gen_andi_i64(t4, t2, (1ll << (up_shift - n)) - 1);
+    tcg_gen_setcondi_i64(TCG_COND_NE, t4, t4, 0);
+    tcg_gen_sari_i64(t2, t2, up_shift - n);
+    tcg_gen_add_i64(t2, t2, t4);
+
+    tcg_gen_sub_i64(t3, t1, t2);
+    tcg_gen_extrl_i64_i32(temp3, t3);
+    /* calc v bit */
+    tcg_gen_setcondi_i64(TCG_COND_GT, t1, t3, 0x7fffffffLL);
+    tcg_gen_setcondi_i64(TCG_COND_LT, t2, t3, -0x80000000LL);
+    tcg_gen_or_i64(t1, t1, t2);
+    tcg_gen_extrl_i64_i32(cpu_PSW_V, t1);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, temp3, temp3);
+    tcg_gen_xor_tl(cpu_PSW_AV, temp3, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, temp3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+    tcg_temp_free_i64(t4);
+}
+
+static inline void
+gen_m16sub32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    gen_sub_d(ret, arg1, temp);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_m16subs32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    gen_subs(ret, arg1, temp);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_m16sub64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+               TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    tcg_gen_ext_i32_i64(t2, temp);
+    tcg_gen_shli_i64(t2, t2, 16);
+    tcg_gen_concat_i32_i64(t1, arg1_low, arg1_high);
+    gen_sub64_d(t3, t1, t2);
+    /* write back result */
+    tcg_gen_extr_i64_i32(rl, rh, t3);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_m16subs64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+               TCGv arg3, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+
+    if (n == 0) {
+        tcg_gen_mul_tl(temp, arg2, arg3);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(temp, arg2, arg3);
+        tcg_gen_shli_tl(temp, temp, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, temp, 0x80000000);
+        tcg_gen_sub_tl(temp, temp, temp2);
+    }
+    tcg_gen_ext_i32_i64(t2, temp);
+    tcg_gen_shli_i64(t2, t2, 16);
+    tcg_gen_concat_i32_i64(t1, arg1_low, arg1_high);
+
+    gen_helper_sub64_ssov(t1, cpu_env, t1, t2);
+    tcg_gen_extr_i64_i32(rl, rh, t1);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+}
+
+static inline void
+gen_msub64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+             TCGv arg3, uint32_t n, CPUTriCoreState *env)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_temp_new_i64();
+    TCGv temp, temp2;
+
+    tcg_gen_concat_i32_i64(t1, arg1_low, arg1_high);
+    tcg_gen_ext_i32_i64(t2, arg2);
+    tcg_gen_ext_i32_i64(t3, arg3);
+
+    tcg_gen_mul_i64(t2, t2, t3);
+    if (n != 0) {
+        tcg_gen_shli_i64(t2, t2, 1);
+    }
+    tcg_gen_sub_i64(t4, t1, t2);
+    /* calc v bit */
+    tcg_gen_xor_i64(t3, t4, t1);
+    tcg_gen_xor_i64(t2, t1, t2);
+    tcg_gen_and_i64(t3, t3, t2);
+    tcg_gen_extrh_i64_i32(cpu_PSW_V, t3);
+    /* We produce an overflow on the host if the mul before was
+       (0x80000000 * 0x80000000) << 1). If this is the
+       case, we negate the ovf. */
+    if (n == 1) {
+        temp = tcg_temp_new();
+        temp2 = tcg_temp_new();
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp, arg2, 0x80000000);
+        tcg_gen_setcond_tl(TCG_COND_EQ, temp2, arg2, arg3);
+        tcg_gen_and_tl(temp, temp, temp2);
+        tcg_gen_shli_tl(temp, temp, 31);
+        /* negate v bit, if special condition */
+        tcg_gen_xor_tl(cpu_PSW_V, cpu_PSW_V, temp);
+
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+    }
+    /* write back result */
+    tcg_gen_extr_i64_i32(rl, rh, t4);
+    /* Calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV/SAV bits */
+    tcg_gen_add_tl(cpu_PSW_AV, rh, rh);
+    tcg_gen_xor_tl(cpu_PSW_AV, rh, cpu_PSW_AV);
+    /* calc SAV */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+    tcg_temp_free_i64(t4);
+}
+
+static inline void
+gen_msubs32_q(TCGv ret, TCGv arg1, TCGv arg2, TCGv arg3, uint32_t n,
+              uint32_t up_shift)
+{
+    TCGv_i64 t1 = tcg_temp_new_i64();
+    TCGv_i64 t2 = tcg_temp_new_i64();
+    TCGv_i64 t3 = tcg_temp_new_i64();
+    TCGv_i64 t4 = tcg_temp_new_i64();
+
+    tcg_gen_ext_i32_i64(t1, arg1);
+    tcg_gen_ext_i32_i64(t2, arg2);
+    tcg_gen_ext_i32_i64(t3, arg3);
+
+    tcg_gen_mul_i64(t2, t2, t3);
+    /* if we shift part of the fraction out, we need to round up */
+    tcg_gen_andi_i64(t4, t2, (1ll << (up_shift - n)) - 1);
+    tcg_gen_setcondi_i64(TCG_COND_NE, t4, t4, 0);
+    tcg_gen_sari_i64(t3, t2, up_shift - n);
+    tcg_gen_add_i64(t3, t3, t4);
+
+    gen_helper_msub32_q_sub_ssov(ret, cpu_env, t1, t3);
+
+    tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t2);
+    tcg_temp_free_i64(t3);
+    tcg_temp_free_i64(t4);
+}
+
+static inline void
+gen_msubs64_q(TCGv rl, TCGv rh, TCGv arg1_low, TCGv arg1_high, TCGv arg2,
+             TCGv arg3, uint32_t n)
+{
+    TCGv_i64 r1 = tcg_temp_new_i64();
+    TCGv temp = tcg_const_i32(n);
+
+    tcg_gen_concat_i32_i64(r1, arg1_low, arg1_high);
+    gen_helper_msub64_q_ssov(r1, cpu_env, r1, arg2, arg3, temp);
+    tcg_gen_extr_i64_i32(rl, rh, r1);
+
+    tcg_temp_free_i64(r1);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msubad_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+             TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_addsub64_h(ret_low, ret_high, r1_low, r1_high, temp, temp2,
+                   tcg_gen_add_tl, tcg_gen_sub_tl);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubadm_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+              TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+    TCGv_i64 temp64_3 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_concat_i32_i64(temp64_3, r1_low, r1_high);
+    tcg_gen_sari_i64(temp64_2, temp64, 32); /* high */
+    tcg_gen_ext32s_i64(temp64, temp64); /* low */
+    tcg_gen_sub_i64(temp64, temp64_2, temp64);
+    tcg_gen_shli_i64(temp64, temp64, 16);
+
+    gen_sub64_d(temp64_2, temp64_3, temp64);
+    /* write back result */
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64_2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+    tcg_temp_free_i64(temp64_3);
+}
+
+static inline void
+gen_msubadr32_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_helper_subadr_h(ret, cpu_env, temp64, temp, temp2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubads_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+              TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv temp3 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_extr_i64_i32(temp, temp2, temp64);
+    gen_adds(ret_low, r1_low, temp);
+    tcg_gen_mov_tl(temp, cpu_PSW_V);
+    tcg_gen_mov_tl(temp3, cpu_PSW_AV);
+    gen_subs(ret_high, r1_high, temp2);
+    /* combine v bits */
+    tcg_gen_or_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* combine av bits */
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp3);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free(temp3);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubadms_h(TCGv ret_low, TCGv ret_high, TCGv r1_low, TCGv r1_high, TCGv r2,
+               TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    TCGv_i64 temp64_2 = tcg_temp_new_i64();
+
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_sari_i64(temp64_2, temp64, 32); /* high */
+    tcg_gen_ext32s_i64(temp64, temp64); /* low */
+    tcg_gen_sub_i64(temp64, temp64_2, temp64);
+    tcg_gen_shli_i64(temp64, temp64, 16);
+    tcg_gen_concat_i32_i64(temp64_2, r1_low, r1_high);
+
+    gen_helper_sub64_ssov(temp64, cpu_env, temp64_2, temp64);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp64);
+    tcg_temp_free_i64(temp64_2);
+}
+
+static inline void
+gen_msubadr32s_h(TCGv ret, TCGv r1, TCGv r2, TCGv r3, uint32_t n, uint32_t mode)
+{
+    TCGv temp = tcg_const_i32(n);
+    TCGv temp2 = tcg_temp_new();
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    switch (mode) {
+    case MODE_LL:
+        GEN_HELPER_LL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_LU:
+        GEN_HELPER_LU(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UL:
+        GEN_HELPER_UL(mul_h, temp64, r2, r3, temp);
+        break;
+    case MODE_UU:
+        GEN_HELPER_UU(mul_h, temp64, r2, r3, temp);
+        break;
+    }
+    tcg_gen_andi_tl(temp2, r1, 0xffff0000);
+    tcg_gen_shli_tl(temp, r1, 16);
+    gen_helper_subadr_h_ssov(ret, cpu_env, temp64, temp, temp2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void gen_abs(TCGv ret, TCGv r1)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv t0 = tcg_const_i32(0);
+
+    tcg_gen_neg_tl(temp, r1);
+    tcg_gen_movcond_tl(TCG_COND_GE, ret, r1, t0, r1, temp);
+    /* overflow can only happen, if r1 = 0x80000000 */
+    tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_PSW_V, r1, 0x80000000);
+    tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(t0);
+}
+
+static inline void gen_absdif(TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv temp = tcg_temp_new_i32();
+    TCGv result = tcg_temp_new_i32();
+
+    tcg_gen_sub_tl(result, r1, r2);
+    tcg_gen_sub_tl(temp, r2, r1);
+    tcg_gen_movcond_tl(TCG_COND_GT, result, r1, r2, result, temp);
+
+    /* calc V bit */
+    tcg_gen_xor_tl(cpu_PSW_V, result, r1);
+    tcg_gen_xor_tl(temp, result, r2);
+    tcg_gen_movcond_tl(TCG_COND_GT, cpu_PSW_V, r1, r2, cpu_PSW_V, temp);
+    tcg_gen_xor_tl(temp, r1, r2);
+    tcg_gen_and_tl(cpu_PSW_V, cpu_PSW_V, temp);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, result, result);
+    tcg_gen_xor_tl(cpu_PSW_AV, result, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* write back result */
+    tcg_gen_mov_tl(ret, result);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(result);
+}
+
+static inline void gen_absdifi(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_absdif(ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_absdifsi(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_absdif_ssov(ret, cpu_env, r1, temp);
+    tcg_temp_free(temp);
+}
+
 static inline void gen_mul_i32s(TCGv ret, TCGv r1, TCGv r2)
 {
     TCGv high = tcg_temp_new();
@@ -360,6 +2503,292 @@
     tcg_temp_free(low);
 }
 
+static inline void gen_muli_i32s(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_mul_i32s(ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_mul_i64s(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2)
+{
+    tcg_gen_muls2_tl(ret_low, ret_high, r1, r2);
+    /* clear V bit */
+    tcg_gen_movi_tl(cpu_PSW_V, 0);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret_high, ret_high);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret_high, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+}
+
+static inline void gen_muli_i64s(TCGv ret_low, TCGv ret_high, TCGv r1,
+                                int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_mul_i64s(ret_low, ret_high, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_mul_i64u(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2)
+{
+    tcg_gen_mulu2_tl(ret_low, ret_high, r1, r2);
+    /* clear V bit */
+    tcg_gen_movi_tl(cpu_PSW_V, 0);
+    /* calc SV bit */
+    tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    /* Calc AV bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret_high, ret_high);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret_high, cpu_PSW_AV);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+}
+
+static inline void gen_muli_i64u(TCGv ret_low, TCGv ret_high, TCGv r1,
+                                int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_mul_i64u(ret_low, ret_high, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_mulsi_i32(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_mul_ssov(ret, cpu_env, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_mulsui_i32(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_mul_suov(ret, cpu_env, r1, temp);
+    tcg_temp_free(temp);
+}
+/* gen_maddsi_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9); */
+static inline void gen_maddsi_32(TCGv ret, TCGv r1, TCGv r2, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_madd32_ssov(ret, cpu_env, r1, r2, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_maddsui_32(TCGv ret, TCGv r1, TCGv r2, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_madd32_suov(ret, cpu_env, r1, r2, temp);
+    tcg_temp_free(temp);
+}
+
+static void
+gen_mul_q(TCGv rl, TCGv rh, TCGv arg1, TCGv arg2, uint32_t n, uint32_t up_shift)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv_i64 temp_64 = tcg_temp_new_i64();
+    TCGv_i64 temp2_64 = tcg_temp_new_i64();
+
+    if (n == 0) {
+        if (up_shift == 32) {
+            tcg_gen_muls2_tl(rh, rl, arg1, arg2);
+        } else if (up_shift == 16) {
+            tcg_gen_ext_i32_i64(temp_64, arg1);
+            tcg_gen_ext_i32_i64(temp2_64, arg2);
+
+            tcg_gen_mul_i64(temp_64, temp_64, temp2_64);
+            tcg_gen_shri_i64(temp_64, temp_64, up_shift);
+            tcg_gen_extr_i64_i32(rl, rh, temp_64);
+        } else {
+            tcg_gen_muls2_tl(rl, rh, arg1, arg2);
+        }
+        /* reset v bit */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+    } else { /* n is expected to be 1 */
+        tcg_gen_ext_i32_i64(temp_64, arg1);
+        tcg_gen_ext_i32_i64(temp2_64, arg2);
+
+        tcg_gen_mul_i64(temp_64, temp_64, temp2_64);
+
+        if (up_shift == 0) {
+            tcg_gen_shli_i64(temp_64, temp_64, 1);
+        } else {
+            tcg_gen_shri_i64(temp_64, temp_64, up_shift - 1);
+        }
+        tcg_gen_extr_i64_i32(rl, rh, temp_64);
+        /* overflow only occurs if r1 = r2 = 0x8000 */
+        if (up_shift == 0) {/* result is 64 bit */
+            tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_PSW_V, rh,
+                                0x80000000);
+        } else { /* result is 32 bit */
+            tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_PSW_V, rl,
+                                0x80000000);
+        }
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* calc sv overflow bit */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+    }
+    /* calc av overflow bit */
+    if (up_shift == 0) {
+        tcg_gen_add_tl(cpu_PSW_AV, rh, rh);
+        tcg_gen_xor_tl(cpu_PSW_AV, rh, cpu_PSW_AV);
+    } else {
+        tcg_gen_add_tl(cpu_PSW_AV, rl, rl);
+        tcg_gen_xor_tl(cpu_PSW_AV, rl, cpu_PSW_AV);
+    }
+    /* calc sav overflow bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    tcg_temp_free(temp);
+    tcg_temp_free_i64(temp_64);
+    tcg_temp_free_i64(temp2_64);
+}
+
+static void
+gen_mul_q_16(TCGv ret, TCGv arg1, TCGv arg2, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    if (n == 0) {
+        tcg_gen_mul_tl(ret, arg1, arg2);
+    } else { /* n is expected to be 1 */
+        tcg_gen_mul_tl(ret, arg1, arg2);
+        tcg_gen_shli_tl(ret, ret, 1);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp, ret, 0x80000000);
+        tcg_gen_sub_tl(ret, ret, temp);
+    }
+    /* reset v bit */
+    tcg_gen_movi_tl(cpu_PSW_V, 0);
+    /* calc av overflow bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc sav overflow bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+
+    tcg_temp_free(temp);
+}
+
+static void gen_mulr_q(TCGv ret, TCGv arg1, TCGv arg2, uint32_t n)
+{
+    TCGv temp = tcg_temp_new();
+    if (n == 0) {
+        tcg_gen_mul_tl(ret, arg1, arg2);
+        tcg_gen_addi_tl(ret, ret, 0x8000);
+    } else {
+        tcg_gen_mul_tl(ret, arg1, arg2);
+        tcg_gen_shli_tl(ret, ret, 1);
+        tcg_gen_addi_tl(ret, ret, 0x8000);
+        /* catch special case r1 = r2 = 0x8000 */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp, ret, 0x80008000);
+        tcg_gen_muli_tl(temp, temp, 0x8001);
+        tcg_gen_sub_tl(ret, ret, temp);
+    }
+    /* reset v bit */
+    tcg_gen_movi_tl(cpu_PSW_V, 0);
+    /* calc av overflow bit */
+    tcg_gen_add_tl(cpu_PSW_AV, ret, ret);
+    tcg_gen_xor_tl(cpu_PSW_AV, ret, cpu_PSW_AV);
+    /* calc sav overflow bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* cut halfword off */
+    tcg_gen_andi_tl(ret, ret, 0xffff0000);
+
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_madds_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+             TCGv r3)
+{
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    tcg_gen_concat_i32_i64(temp64, r2_low, r2_high);
+    gen_helper_madd64_ssov(temp64, cpu_env, r1, temp64, r3);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddsi_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+              int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_madds_64(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_maddsu_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+             TCGv r3)
+{
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    tcg_gen_concat_i32_i64(temp64, r2_low, r2_high);
+    gen_helper_madd64_suov(temp64, cpu_env, r1, temp64, r3);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_maddsui_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+               int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_maddsu_64(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_msubsi_32(TCGv ret, TCGv r1, TCGv r2, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_msub32_ssov(ret, cpu_env, r1, r2, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_msubsui_32(TCGv ret, TCGv r1, TCGv r2, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_msub32_suov(ret, cpu_env, r1, r2, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msubs_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+             TCGv r3)
+{
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    tcg_gen_concat_i32_i64(temp64, r2_low, r2_high);
+    gen_helper_msub64_ssov(temp64, cpu_env, r1, temp64, r3);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubsi_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+              int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_msubs_64(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void
+gen_msubsu_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+             TCGv r3)
+{
+    TCGv_i64 temp64 = tcg_temp_new_i64();
+    tcg_gen_concat_i32_i64(temp64, r2_low, r2_high);
+    gen_helper_msub64_suov(temp64, cpu_env, r1, temp64, r3);
+    tcg_gen_extr_i64_i32(ret_low, ret_high, temp64);
+    tcg_temp_free_i64(temp64);
+}
+
+static inline void
+gen_msubsui_64(TCGv ret_low, TCGv ret_high, TCGv r1, TCGv r2_low, TCGv r2_high,
+               int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_msubsu_64(ret_low, ret_high, r1, r2_low, r2_high, temp);
+    tcg_temp_free(temp);
+}
+
 static void gen_saturate(TCGv ret, TCGv arg, int32_t up, int32_t low)
 {
     TCGv sat_neg = tcg_const_i32(low);
@@ -394,6 +2823,27 @@
     }
 }
 
+static void gen_sh_hi(TCGv ret, TCGv r1, int32_t shiftcount)
+{
+    TCGv temp_low, temp_high;
+
+    if (shiftcount == -16) {
+        tcg_gen_movi_tl(ret, 0);
+    } else {
+        temp_high = tcg_temp_new();
+        temp_low = tcg_temp_new();
+
+        tcg_gen_andi_tl(temp_low, r1, 0xffff);
+        tcg_gen_andi_tl(temp_high, r1, 0xffff0000);
+        gen_shi(temp_low, temp_low, shiftcount);
+        gen_shi(ret, temp_high, shiftcount);
+        tcg_gen_deposit_tl(ret, ret, temp_low, 0, 16);
+
+        tcg_temp_free(temp_low);
+        tcg_temp_free(temp_high);
+    }
+}
+
 static void gen_shaci(TCGv ret, TCGv r1, int32_t shift_count)
 {
     uint32_t msk, msk_start;
@@ -409,7 +2859,7 @@
     } else if (shift_count == -32) {
         /* set PSW.C */
         tcg_gen_mov_tl(cpu_PSW_C, r1);
-        /* fill ret completly with sign bit */
+        /* fill ret completely with sign bit */
         tcg_gen_sari_tl(ret, r1, 31);
         /* clear PSW.V */
         tcg_gen_movi_tl(cpu_PSW_V, 0);
@@ -453,16 +2903,100 @@
     tcg_temp_free(t_0);
 }
 
+static void gen_shas(TCGv ret, TCGv r1, TCGv r2)
+{
+    gen_helper_sha_ssov(ret, cpu_env, r1, r2);
+}
+
+static void gen_shasi(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_shas(ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static void gen_sha_hi(TCGv ret, TCGv r1, int32_t shift_count)
+{
+    TCGv low, high;
+
+    if (shift_count == 0) {
+        tcg_gen_mov_tl(ret, r1);
+    } else if (shift_count > 0) {
+        low = tcg_temp_new();
+        high = tcg_temp_new();
+
+        tcg_gen_andi_tl(high, r1, 0xffff0000);
+        tcg_gen_shli_tl(low, r1, shift_count);
+        tcg_gen_shli_tl(ret, high, shift_count);
+        tcg_gen_deposit_tl(ret, ret, low, 0, 16);
+
+        tcg_temp_free(low);
+        tcg_temp_free(high);
+    } else {
+        low = tcg_temp_new();
+        high = tcg_temp_new();
+
+        tcg_gen_ext16s_tl(low, r1);
+        tcg_gen_sari_tl(low, low, -shift_count);
+        tcg_gen_sari_tl(ret, r1, -shift_count);
+        tcg_gen_deposit_tl(ret, ret, low, 0, 16);
+
+        tcg_temp_free(low);
+        tcg_temp_free(high);
+    }
+
+}
+
+/* ret = {ret[30:0], (r1 cond r2)}; */
+static void gen_sh_cond(int cond, TCGv ret, TCGv r1, TCGv r2)
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    tcg_gen_shli_tl(temp, ret, 1);
+    tcg_gen_setcond_tl(cond, temp2, r1, r2);
+    tcg_gen_or_tl(ret, temp, temp2);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static void gen_sh_condi(int cond, TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_sh_cond(cond, ret, r1, temp);
+    tcg_temp_free(temp);
+}
+
 static inline void gen_adds(TCGv ret, TCGv r1, TCGv r2)
 {
     gen_helper_add_ssov(ret, cpu_env, r1, r2);
 }
 
+static inline void gen_addsi(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_add_ssov(ret, cpu_env, r1, temp);
+    tcg_temp_free(temp);
+}
+
+static inline void gen_addsui(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_helper_add_suov(ret, cpu_env, r1, temp);
+    tcg_temp_free(temp);
+}
+
 static inline void gen_subs(TCGv ret, TCGv r1, TCGv r2)
 {
     gen_helper_sub_ssov(ret, cpu_env, r1, r2);
 }
 
+static inline void gen_subsu(TCGv ret, TCGv r1, TCGv r2)
+{
+    gen_helper_sub_suov(ret, cpu_env, r1, r2);
+}
+
 static inline void gen_bit_2op(TCGv ret, TCGv r1, TCGv r2,
                                int pos1, int pos2,
                                void(*op1)(TCGv, TCGv, TCGv),
@@ -506,6 +3040,196 @@
     tcg_temp_free(temp2);
 }
 
+static inline void gen_accumulating_cond(int cond, TCGv ret, TCGv r1, TCGv r2,
+                                         void(*op)(TCGv, TCGv, TCGv))
+{
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+    /* temp = (arg1 cond arg2 )*/
+    tcg_gen_setcond_tl(cond, temp, r1, r2);
+    /* temp2 = ret[0]*/
+    tcg_gen_andi_tl(temp2, ret, 0x1);
+    /* temp = temp insn temp2 */
+    (*op)(temp, temp, temp2);
+    /* ret = {ret[31:1], temp} */
+    tcg_gen_deposit_tl(ret, ret, temp, 0, 1);
+
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void
+gen_accumulating_condi(int cond, TCGv ret, TCGv r1, int32_t con,
+                       void(*op)(TCGv, TCGv, TCGv))
+{
+    TCGv temp = tcg_const_i32(con);
+    gen_accumulating_cond(cond, ret, r1, temp, op);
+    tcg_temp_free(temp);
+}
+
+/* ret = (r1 cond r2) ? 0xFFFFFFFF ? 0x00000000;*/
+static inline void gen_cond_w(TCGCond cond, TCGv ret, TCGv r1, TCGv r2)
+{
+    tcg_gen_setcond_tl(cond, ret, r1, r2);
+    tcg_gen_neg_tl(ret, ret);
+}
+
+static inline void gen_eqany_bi(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv b0 = tcg_temp_new();
+    TCGv b1 = tcg_temp_new();
+    TCGv b2 = tcg_temp_new();
+    TCGv b3 = tcg_temp_new();
+
+    /* byte 0 */
+    tcg_gen_andi_tl(b0, r1, 0xff);
+    tcg_gen_setcondi_tl(TCG_COND_EQ, b0, b0, con & 0xff);
+
+    /* byte 1 */
+    tcg_gen_andi_tl(b1, r1, 0xff00);
+    tcg_gen_setcondi_tl(TCG_COND_EQ, b1, b1, con & 0xff00);
+
+    /* byte 2 */
+    tcg_gen_andi_tl(b2, r1, 0xff0000);
+    tcg_gen_setcondi_tl(TCG_COND_EQ, b2, b2, con & 0xff0000);
+
+    /* byte 3 */
+    tcg_gen_andi_tl(b3, r1, 0xff000000);
+    tcg_gen_setcondi_tl(TCG_COND_EQ, b3, b3, con & 0xff000000);
+
+    /* combine them */
+    tcg_gen_or_tl(ret, b0, b1);
+    tcg_gen_or_tl(ret, ret, b2);
+    tcg_gen_or_tl(ret, ret, b3);
+
+    tcg_temp_free(b0);
+    tcg_temp_free(b1);
+    tcg_temp_free(b2);
+    tcg_temp_free(b3);
+}
+
+static inline void gen_eqany_hi(TCGv ret, TCGv r1, int32_t con)
+{
+    TCGv h0 = tcg_temp_new();
+    TCGv h1 = tcg_temp_new();
+
+    /* halfword 0 */
+    tcg_gen_andi_tl(h0, r1, 0xffff);
+    tcg_gen_setcondi_tl(TCG_COND_EQ, h0, h0, con & 0xffff);
+
+    /* halfword 1 */
+    tcg_gen_andi_tl(h1, r1, 0xffff0000);
+    tcg_gen_setcondi_tl(TCG_COND_EQ, h1, h1, con & 0xffff0000);
+
+    /* combine them */
+    tcg_gen_or_tl(ret, h0, h1);
+
+    tcg_temp_free(h0);
+    tcg_temp_free(h1);
+}
+/* mask = ((1 << width) -1) << pos;
+   ret = (r1 & ~mask) | (r2 << pos) & mask); */
+static inline void gen_insert(TCGv ret, TCGv r1, TCGv r2, TCGv width, TCGv pos)
+{
+    TCGv mask = tcg_temp_new();
+    TCGv temp = tcg_temp_new();
+    TCGv temp2 = tcg_temp_new();
+
+    tcg_gen_movi_tl(mask, 1);
+    tcg_gen_shl_tl(mask, mask, width);
+    tcg_gen_subi_tl(mask, mask, 1);
+    tcg_gen_shl_tl(mask, mask, pos);
+
+    tcg_gen_shl_tl(temp, r2, pos);
+    tcg_gen_and_tl(temp, temp, mask);
+    tcg_gen_andc_tl(temp2, r1, mask);
+    tcg_gen_or_tl(ret, temp, temp2);
+
+    tcg_temp_free(mask);
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static inline void gen_bsplit(TCGv rl, TCGv rh, TCGv r1)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+
+    gen_helper_bsplit(temp, r1);
+    tcg_gen_extr_i64_i32(rl, rh, temp);
+
+    tcg_temp_free_i64(temp);
+}
+
+static inline void gen_unpack(TCGv rl, TCGv rh, TCGv r1)
+{
+    TCGv_i64 temp = tcg_temp_new_i64();
+
+    gen_helper_unpack(temp, r1);
+    tcg_gen_extr_i64_i32(rl, rh, temp);
+
+    tcg_temp_free_i64(temp);
+}
+
+static inline void
+gen_dvinit_b(CPUTriCoreState *env, TCGv rl, TCGv rh, TCGv r1, TCGv r2)
+{
+    TCGv_i64 ret = tcg_temp_new_i64();
+
+    if (!tricore_feature(env, TRICORE_FEATURE_131)) {
+        gen_helper_dvinit_b_13(ret, cpu_env, r1, r2);
+    } else {
+        gen_helper_dvinit_b_131(ret, cpu_env, r1, r2);
+    }
+    tcg_gen_extr_i64_i32(rl, rh, ret);
+
+    tcg_temp_free_i64(ret);
+}
+
+static inline void
+gen_dvinit_h(CPUTriCoreState *env, TCGv rl, TCGv rh, TCGv r1, TCGv r2)
+{
+    TCGv_i64 ret = tcg_temp_new_i64();
+
+    if (!tricore_feature(env, TRICORE_FEATURE_131)) {
+        gen_helper_dvinit_h_13(ret, cpu_env, r1, r2);
+    } else {
+        gen_helper_dvinit_h_131(ret, cpu_env, r1, r2);
+    }
+    tcg_gen_extr_i64_i32(rl, rh, ret);
+
+    tcg_temp_free_i64(ret);
+}
+
+static void gen_calc_usb_mul_h(TCGv arg_low, TCGv arg_high)
+{
+    TCGv temp = tcg_temp_new();
+    /* calc AV bit */
+    tcg_gen_add_tl(temp, arg_low, arg_low);
+    tcg_gen_xor_tl(temp, temp, arg_low);
+    tcg_gen_add_tl(cpu_PSW_AV, arg_high, arg_high);
+    tcg_gen_xor_tl(cpu_PSW_AV, cpu_PSW_AV, arg_high);
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    tcg_gen_movi_tl(cpu_PSW_V, 0);
+    tcg_temp_free(temp);
+}
+
+static void gen_calc_usb_mulr_h(TCGv arg)
+{
+    TCGv temp = tcg_temp_new();
+    /* calc AV bit */
+    tcg_gen_add_tl(temp, arg, arg);
+    tcg_gen_xor_tl(temp, temp, arg);
+    tcg_gen_shli_tl(cpu_PSW_AV, temp, 16);
+    tcg_gen_or_tl(cpu_PSW_AV, cpu_PSW_AV, temp);
+    /* calc SAV bit */
+    tcg_gen_or_tl(cpu_PSW_SAV, cpu_PSW_SAV, cpu_PSW_AV);
+    /* clear V bit */
+    tcg_gen_movi_tl(cpu_PSW_V, 0);
+    tcg_temp_free(temp);
+}
+
 /* helpers for generating program flow micro-ops */
 
 static inline void gen_save_pc(target_ulong pc)
@@ -513,15 +3237,25 @@
     tcg_gen_movi_tl(cpu_PC, pc);
 }
 
+static inline bool use_goto_tb(DisasContext *ctx, target_ulong dest)
+{
+    if (unlikely(ctx->singlestep_enabled)) {
+        return false;
+    }
+
+#ifndef CONFIG_USER_ONLY
+    return (ctx->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
 {
-    TranslationBlock *tb;
-    tb = ctx->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK) &&
-        likely(!ctx->singlestep_enabled)) {
+    if (use_goto_tb(ctx, dest)) {
         tcg_gen_goto_tb(n);
         gen_save_pc(dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)ctx->tb + n);
     } else {
         gen_save_pc(dest);
         if (ctx->singlestep_enabled) {
@@ -531,11 +3265,23 @@
     }
 }
 
+static void generate_trap(DisasContext *ctx, int class, int tin)
+{
+    TCGv_i32 classtemp = tcg_const_i32(class);
+    TCGv_i32 tintemp = tcg_const_i32(tin);
+
+    gen_save_pc(ctx->pc);
+    gen_helper_raise_exception_sync(cpu_env, classtemp, tintemp);
+    ctx->bstate = BS_EXCP;
+
+    tcg_temp_free(classtemp);
+    tcg_temp_free(tintemp);
+}
+
 static inline void gen_branch_cond(DisasContext *ctx, TCGCond cond, TCGv r1,
                                    TCGv r2, int16_t address)
 {
-    int jumpLabel;
-    jumpLabel = gen_new_label();
+    TCGLabel *jumpLabel = gen_new_label();
     tcg_gen_brcond_tl(cond, r1, r2, jumpLabel);
 
     gen_goto_tb(ctx, 1, ctx->next_pc);
@@ -554,8 +3300,7 @@
 
 static void gen_loop(DisasContext *ctx, int r1, int32_t offset)
 {
-    int l1;
-    l1 = gen_new_label();
+    TCGLabel *l1 = gen_new_label();
 
     tcg_gen_subi_tl(cpu_gpr_a[r1], cpu_gpr_a[r1], 1);
     tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr_a[r1], -1, l1);
@@ -564,10 +3309,37 @@
     gen_goto_tb(ctx, 0, ctx->next_pc);
 }
 
+static void gen_fcall_save_ctx(DisasContext *ctx)
+{
+    TCGv temp = tcg_temp_new();
+
+    tcg_gen_addi_tl(temp, cpu_gpr_a[10], -4);
+    tcg_gen_qemu_st_tl(cpu_gpr_a[11], temp, ctx->mem_idx, MO_LESL);
+    tcg_gen_movi_tl(cpu_gpr_a[11], ctx->next_pc);
+    tcg_gen_mov_tl(cpu_gpr_a[10], temp);
+
+    tcg_temp_free(temp);
+}
+
+static void gen_fret(DisasContext *ctx)
+{
+    TCGv temp = tcg_temp_new();
+
+    tcg_gen_andi_tl(temp, cpu_gpr_a[11], ~0x1);
+    tcg_gen_qemu_ld_tl(cpu_gpr_a[11], cpu_gpr_a[10], ctx->mem_idx, MO_LESL);
+    tcg_gen_addi_tl(cpu_gpr_a[10], cpu_gpr_a[10], 4);
+    tcg_gen_mov_tl(cpu_PC, temp);
+    tcg_gen_exit_tb(0);
+    ctx->bstate = BS_BRANCH;
+
+    tcg_temp_free(temp);
+}
+
 static void gen_compute_branch(DisasContext *ctx, uint32_t opc, int r1,
                                int r2 , int32_t constant , int32_t offset)
 {
-    TCGv temp;
+    TCGv temp, temp2;
+    int n;
 
     switch (opc) {
 /* SB-format jumps */
@@ -647,6 +3419,7 @@
         tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], 0xfffffffe);
         tcg_gen_exit_tb(0);
         break;
+    case OPC2_32_SYS_RET:
     case OPC2_16_SR_RET:
         gen_helper_ret(cpu_env);
         tcg_gen_exit_tb(0);
@@ -656,8 +3429,17 @@
         gen_helper_1arg(call, ctx->next_pc);
         gen_goto_tb(ctx, 0, EA_B_ABSOLUT(offset));
         break;
+    case OPC1_32_B_FCALL:
+        gen_fcall_save_ctx(ctx);
+        gen_goto_tb(ctx, 0, ctx->pc + offset * 2);
+        break;
+    case OPC1_32_B_FCALLA:
+        gen_fcall_save_ctx(ctx);
+        gen_goto_tb(ctx, 0, EA_B_ABSOLUT(offset));
+        break;
     case OPC1_32_B_JLA:
         tcg_gen_movi_tl(cpu_gpr_a[11], ctx->next_pc);
+        /* fall through */
     case OPC1_32_B_JA:
         gen_goto_tb(ctx, 0, EA_B_ABSOLUT(offset));
         break;
@@ -665,8 +3447,136 @@
         tcg_gen_movi_tl(cpu_gpr_a[11], ctx->next_pc);
         gen_goto_tb(ctx, 0, ctx->pc + offset * 2);
         break;
+/* BOL format */
+    case OPCM_32_BRC_EQ_NEQ:
+         if (MASK_OP_BRC_OP2(ctx->opcode) == OPC2_32_BRC_JEQ) {
+            gen_branch_condi(ctx, TCG_COND_EQ, cpu_gpr_d[r1], constant, offset);
+         } else {
+            gen_branch_condi(ctx, TCG_COND_NE, cpu_gpr_d[r1], constant, offset);
+         }
+         break;
+    case OPCM_32_BRC_GE:
+         if (MASK_OP_BRC_OP2(ctx->opcode) == OP2_32_BRC_JGE) {
+            gen_branch_condi(ctx, TCG_COND_GE, cpu_gpr_d[r1], constant, offset);
+         } else {
+            constant = MASK_OP_BRC_CONST4(ctx->opcode);
+            gen_branch_condi(ctx, TCG_COND_GEU, cpu_gpr_d[r1], constant,
+                             offset);
+         }
+         break;
+    case OPCM_32_BRC_JLT:
+         if (MASK_OP_BRC_OP2(ctx->opcode) == OPC2_32_BRC_JLT) {
+            gen_branch_condi(ctx, TCG_COND_LT, cpu_gpr_d[r1], constant, offset);
+         } else {
+            constant = MASK_OP_BRC_CONST4(ctx->opcode);
+            gen_branch_condi(ctx, TCG_COND_LTU, cpu_gpr_d[r1], constant,
+                             offset);
+         }
+         break;
+    case OPCM_32_BRC_JNE:
+        temp = tcg_temp_new();
+        if (MASK_OP_BRC_OP2(ctx->opcode) == OPC2_32_BRC_JNED) {
+            tcg_gen_mov_tl(temp, cpu_gpr_d[r1]);
+            /* subi is unconditional */
+            tcg_gen_subi_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 1);
+            gen_branch_condi(ctx, TCG_COND_NE, temp, constant, offset);
+        } else {
+            tcg_gen_mov_tl(temp, cpu_gpr_d[r1]);
+            /* addi is unconditional */
+            tcg_gen_addi_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 1);
+            gen_branch_condi(ctx, TCG_COND_NE, temp, constant, offset);
+        }
+        tcg_temp_free(temp);
+        break;
+/* BRN format */
+    case OPCM_32_BRN_JTT:
+        n = MASK_OP_BRN_N(ctx->opcode);
+
+        temp = tcg_temp_new();
+        tcg_gen_andi_tl(temp, cpu_gpr_d[r1], (1 << n));
+
+        if (MASK_OP_BRN_OP2(ctx->opcode) == OPC2_32_BRN_JNZ_T) {
+            gen_branch_condi(ctx, TCG_COND_NE, temp, 0, offset);
+        } else {
+            gen_branch_condi(ctx, TCG_COND_EQ, temp, 0, offset);
+        }
+        tcg_temp_free(temp);
+        break;
+/* BRR Format */
+    case OPCM_32_BRR_EQ_NEQ:
+        if (MASK_OP_BRR_OP2(ctx->opcode) == OPC2_32_BRR_JEQ) {
+            gen_branch_cond(ctx, TCG_COND_EQ, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                            offset);
+        } else {
+            gen_branch_cond(ctx, TCG_COND_NE, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                            offset);
+        }
+        break;
+    case OPCM_32_BRR_ADDR_EQ_NEQ:
+        if (MASK_OP_BRR_OP2(ctx->opcode) == OPC2_32_BRR_JEQ_A) {
+            gen_branch_cond(ctx, TCG_COND_EQ, cpu_gpr_a[r1], cpu_gpr_a[r2],
+                            offset);
+        } else {
+            gen_branch_cond(ctx, TCG_COND_NE, cpu_gpr_a[r1], cpu_gpr_a[r2],
+                            offset);
+        }
+        break;
+    case OPCM_32_BRR_GE:
+        if (MASK_OP_BRR_OP2(ctx->opcode) == OPC2_32_BRR_JGE) {
+            gen_branch_cond(ctx, TCG_COND_GE, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                            offset);
+        } else {
+            gen_branch_cond(ctx, TCG_COND_GEU, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                            offset);
+        }
+        break;
+    case OPCM_32_BRR_JLT:
+        if (MASK_OP_BRR_OP2(ctx->opcode) == OPC2_32_BRR_JLT) {
+            gen_branch_cond(ctx, TCG_COND_LT, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                            offset);
+        } else {
+            gen_branch_cond(ctx, TCG_COND_LTU, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                            offset);
+        }
+        break;
+    case OPCM_32_BRR_LOOP:
+        if (MASK_OP_BRR_OP2(ctx->opcode) == OPC2_32_BRR_LOOP) {
+            gen_loop(ctx, r2, offset * 2);
+        } else {
+            /* OPC2_32_BRR_LOOPU */
+            gen_goto_tb(ctx, 0, ctx->pc + offset * 2);
+        }
+        break;
+    case OPCM_32_BRR_JNE:
+        temp = tcg_temp_new();
+        temp2 = tcg_temp_new();
+        if (MASK_OP_BRC_OP2(ctx->opcode) == OPC2_32_BRR_JNED) {
+            tcg_gen_mov_tl(temp, cpu_gpr_d[r1]);
+            /* also save r2, in case of r1 == r2, so r2 is not decremented */
+            tcg_gen_mov_tl(temp2, cpu_gpr_d[r2]);
+            /* subi is unconditional */
+            tcg_gen_subi_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 1);
+            gen_branch_cond(ctx, TCG_COND_NE, temp, temp2, offset);
+        } else {
+            tcg_gen_mov_tl(temp, cpu_gpr_d[r1]);
+            /* also save r2, in case of r1 == r2, so r2 is not decremented */
+            tcg_gen_mov_tl(temp2, cpu_gpr_d[r2]);
+            /* addi is unconditional */
+            tcg_gen_addi_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], 1);
+            gen_branch_cond(ctx, TCG_COND_NE, temp, temp2, offset);
+        }
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+    case OPCM_32_BRR_JNZ:
+        if (MASK_OP_BRR_OP2(ctx->opcode) == OPC2_32_BRR_JNZ_A) {
+            gen_branch_condi(ctx, TCG_COND_NE, cpu_gpr_a[r1], 0, offset);
+        } else {
+            gen_branch_condi(ctx, TCG_COND_EQ, cpu_gpr_a[r1], 0, offset);
+        }
+        break;
     default:
-        printf("Branch Error at %x\n", ctx->pc);
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     ctx->bstate = BS_BRANCH;
 }
@@ -676,7 +3586,7 @@
  * Functions for decoding instructions
  */
 
-static void decode_src_opc(DisasContext *ctx, int op1)
+static void decode_src_opc(CPUTriCoreState *env, DisasContext *ctx, int op1)
 {
     int r1;
     int32_t const4;
@@ -737,12 +3647,22 @@
         const4 = MASK_OP_SRC_CONST4(ctx->opcode);
         tcg_gen_movi_tl(cpu_gpr_a[r1], const4);
         break;
+    case OPC1_16_SRC_MOV_E:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            tcg_gen_movi_tl(cpu_gpr_d[r1], const4);
+            tcg_gen_sari_tl(cpu_gpr_d[r1+1], cpu_gpr_d[r1], 31);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
     case OPC1_16_SRC_SH:
         gen_shi(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
         break;
     case OPC1_16_SRC_SHA:
         gen_shaci(cpu_gpr_d[r1], cpu_gpr_d[r1], const4);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -826,6 +3746,8 @@
     case OPC1_16_SRR_XOR:
         tcg_gen_xor_tl(cpu_gpr_d[r1], cpu_gpr_d[r1], cpu_gpr_d[r2]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -865,6 +3787,8 @@
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LEUL);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -902,6 +3826,8 @@
     case OPC1_16_SC_SUB_A:
         tcg_gen_subi_tl(cpu_gpr_a[10], cpu_gpr_a[10], const16);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -936,12 +3862,14 @@
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 2);
         break;
     case OPC1_16_SLR_LD_W:
-        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESW);
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESL);
         break;
     case OPC1_16_SLR_LD_W_POSTINC:
-        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESW);
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], cpu_gpr_a[r2], ctx->mem_idx, MO_LESL);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], 4);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -979,6 +3907,8 @@
     case OPC1_16_SRO_ST_W:
         gen_offset_st(ctx, cpu_gpr_d[15], cpu_gpr_a[r2], address * 4, MO_LESL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1001,6 +3931,11 @@
     case OPC2_16_SR_DEBUG:
         /* raise EXCP_DEBUG */
         break;
+    case OPC2_16_SR_FRET:
+        gen_fret(ctx);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1043,6 +3978,8 @@
     case OPC2_16_SR_SAT_HU:
         gen_saturate_u(cpu_gpr_d[r1], cpu_gpr_d[r1], 0xffff);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1074,9 +4011,10 @@
     case OPC1_16_SRC_LT:
     case OPC1_16_SRC_MOV:
     case OPC1_16_SRC_MOV_A:
+    case OPC1_16_SRC_MOV_E:
     case OPC1_16_SRC_SH:
     case OPC1_16_SRC_SHA:
-        decode_src_opc(ctx, op1);
+        decode_src_opc(env, ctx, op1);
         break;
 /* SRR-format */
     case OPC1_16_SRR_ADD:
@@ -1252,6 +4190,8 @@
         r1 = MASK_OP_SR_S1D(ctx->opcode);
         tcg_gen_not_tl(cpu_gpr_d[r1], cpu_gpr_d[r1]);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1278,14 +4218,18 @@
         tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LESL);
         break;
     case OPC2_32_ABS_LD_D:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
         break;
     case OPC2_32_ABS_LD_DA:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
         break;
     case OPC2_32_ABS_LD_W:
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -1317,6 +4261,8 @@
     case OPC2_32_ABS_LD_HU:
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -1342,6 +4288,8 @@
     case OPC2_32_ABS_SWAP_W:
         gen_swap(ctx, r1, temp);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 
     tcg_temp_free(temp);
@@ -1368,6 +4316,8 @@
     case OPC2_32_ABS_STUCX:
         gen_helper_1arg(stucx, EA_ABS_FORMAT(off18));
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1389,15 +4339,18 @@
         tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LESL);
         break;
     case OPC2_32_ABS_ST_D:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
         break;
     case OPC2_32_ABS_ST_DA:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
         break;
     case OPC2_32_ABS_ST_W:
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LESL);
         break;
-
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -1422,6 +4375,8 @@
     case OPC2_32_ABS_ST_H:
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUW);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
 }
@@ -1464,6 +4419,8 @@
         gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl, &tcg_gen_and_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1496,6 +4453,8 @@
         gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1553,6 +4512,8 @@
         gen_bit_1op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_xor_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1592,6 +4553,8 @@
         gen_bit_2op(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl, &tcg_gen_or_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1628,6 +4591,8 @@
         gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_or_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 1);
     tcg_gen_add_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
@@ -1667,6 +4632,8 @@
         gen_bit_1op(temp, cpu_gpr_d[r1], cpu_gpr_d[r2],
                     pos1, pos2, &tcg_gen_xor_tl);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 1);
     tcg_gen_add_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
@@ -1700,31 +4667,36 @@
     case OPC2_32_BO_CACHEA_I_POSTINC:
         /* instruction to access the cache, but we still need to handle
            the addressing mode */
-        tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_CACHEA_WI_PREINC:
     case OPC2_32_BO_CACHEA_W_PREINC:
     case OPC2_32_BO_CACHEA_I_PREINC:
         /* instruction to access the cache, but we still need to handle
            the addressing mode */
-        tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_CACHEI_WI_SHORTOFF:
     case OPC2_32_BO_CACHEI_W_SHORTOFF:
-        /* TODO: Raise illegal opcode trap,
-                 if tricore_feature(TRICORE_FEATURE_13) */
+        if (!tricore_feature(env, TRICORE_FEATURE_131)) {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_BO_CACHEI_W_POSTINC:
     case OPC2_32_BO_CACHEI_WI_POSTINC:
-        if (!tricore_feature(env, TRICORE_FEATURE_13)) {
-            tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
-        } /* TODO: else raise illegal opcode trap */
+        if (tricore_feature(env, TRICORE_FEATURE_131)) {
+            tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_BO_CACHEI_W_PREINC:
     case OPC2_32_BO_CACHEI_WI_PREINC:
-        if (!tricore_feature(env, TRICORE_FEATURE_13)) {
-            tcg_gen_addi_tl(cpu_gpr_d[r2], cpu_gpr_d[r2], off10);
-        } /* TODO: else raise illegal opcode trap */
+        if (tricore_feature(env, TRICORE_FEATURE_131)) {
+            tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
         break;
     case OPC2_32_BO_ST_A_SHORTOFF:
         gen_offset_st(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], off10, MO_LESL);
@@ -1749,14 +4721,17 @@
         gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_UB);
         break;
     case OPC2_32_BO_ST_D_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_st_2regs(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_ST_D_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_ST_D_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
@@ -1764,14 +4739,17 @@
         tcg_temp_free(temp);
         break;
     case OPC2_32_BO_ST_DA_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_st_2regs(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_ST_DA_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_ST_DA_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
@@ -1820,6 +4798,8 @@
     case OPC2_32_BO_ST_W_PREINC:
         gen_st_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -1839,7 +4819,7 @@
     temp = tcg_temp_new();
     temp2 = tcg_temp_new();
     temp3 = tcg_const_i32(off10);
-
+    CHECK_REG_PAIR(r2);
     tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
     tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
 
@@ -1871,10 +4851,12 @@
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_ST_D_BR:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_ST_D_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -1884,10 +4866,12 @@
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_ST_DA_BR:
+        CHECK_REG_PAIR(r1);
         gen_st_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_ST_DA_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_st_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -1922,6 +4906,8 @@
         tcg_gen_qemu_st_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -1976,14 +4962,17 @@
         gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_SB);
         break;
     case OPC2_32_BO_LD_D_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_ld_2regs(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_LD_D_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_LD_D_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp, ctx);
@@ -1991,14 +4980,17 @@
         tcg_temp_free(temp);
         break;
     case OPC2_32_BO_LD_DA_SHORTOFF:
+        CHECK_REG_PAIR(r1);
         gen_offset_ld_2regs(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2],
                             off10, ctx);
         break;
     case OPC2_32_BO_LD_DA_POSTINC:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], cpu_gpr_a[r2], ctx);
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         break;
     case OPC2_32_BO_LD_DA_PREINC:
+        CHECK_REG_PAIR(r1);
         temp = tcg_temp_new();
         tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp, ctx);
@@ -2052,6 +5044,8 @@
     case OPC2_32_BO_LD_W_PREINC:
         gen_ld_preincr(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], off10, MO_LEUL);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -2072,7 +5066,7 @@
     temp = tcg_temp_new();
     temp2 = tcg_temp_new();
     temp3 = tcg_const_i32(off10);
-
+    CHECK_REG_PAIR(r2);
     tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
     tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
 
@@ -2103,10 +5097,12 @@
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_LD_D_BR:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_d[r1+1], cpu_gpr_d[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_LD_D_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -2116,10 +5112,12 @@
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
     case OPC2_32_BO_LD_DA_BR:
+        CHECK_REG_PAIR(r1);
         gen_ld_2regs_64(cpu_gpr_a[r1+1], cpu_gpr_a[r1], temp2, ctx);
         gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
         break;
     case OPC2_32_BO_LD_DA_CIRC:
+        CHECK_REG_PAIR(r1);
         tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp2, ctx->mem_idx, MO_LEUL);
         tcg_gen_shri_tl(temp2, cpu_gpr_a[r2+1], 16);
         tcg_gen_addi_tl(temp, temp, 4);
@@ -2162,6 +5160,8 @@
         tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp2, ctx->mem_idx, MO_LEUL);
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -2230,6 +5230,32 @@
         tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
         gen_swap(ctx, r1, cpu_gpr_a[r2]);
         break;
+    case OPC2_32_BO_CMPSWAP_W_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_cmpswap(ctx, r1, temp);
+        break;
+    case OPC2_32_BO_CMPSWAP_W_POSTINC:
+        gen_cmpswap(ctx, r1, cpu_gpr_a[r2]);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_CMPSWAP_W_PREINC:
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        gen_cmpswap(ctx, r1, cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_BO_SWAPMSK_W_SHORTOFF:
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], off10);
+        gen_swapmsk(ctx, r1, temp);
+        break;
+    case OPC2_32_BO_SWAPMSK_W_POSTINC:
+        gen_swapmsk(ctx, r1, cpu_gpr_a[r2]);
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        break;
+    case OPC2_32_BO_SWAPMSK_W_PREINC:
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r2], off10);
+        gen_swapmsk(ctx, r1, cpu_gpr_a[r2]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
@@ -2252,7 +5278,7 @@
     temp = tcg_temp_new();
     temp2 = tcg_temp_new();
     temp3 = tcg_const_i32(off10);
-
+    CHECK_REG_PAIR(r2);
     tcg_gen_ext16u_tl(temp, cpu_gpr_a[r2+1]);
     tcg_gen_add_tl(temp2, cpu_gpr_a[r2], temp);
 
@@ -2273,23 +5299,3100 @@
         gen_swap(ctx, r1, temp2);
         gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
         break;
+    case OPC2_32_BO_CMPSWAP_W_BR:
+        gen_cmpswap(ctx, r1, temp2);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_CMPSWAP_W_CIRC:
+        gen_cmpswap(ctx, r1, temp2);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    case OPC2_32_BO_SWAPMSK_W_BR:
+        gen_swapmsk(ctx, r1, temp2);
+        gen_helper_br_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1]);
+        break;
+    case OPC2_32_BO_SWAPMSK_W_CIRC:
+        gen_swapmsk(ctx, r1, temp2);
+        gen_helper_circ_update(cpu_gpr_a[r2+1], cpu_gpr_a[r2+1], temp3);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
+
     tcg_temp_free(temp);
     tcg_temp_free(temp2);
     tcg_temp_free(temp3);
 }
 
+static void decode_bol_opc(CPUTriCoreState *env, DisasContext *ctx, int32_t op1)
+{
+    int r1, r2;
+    int32_t address;
+    TCGv temp;
+
+    r1 = MASK_OP_BOL_S1D(ctx->opcode);
+    r2 = MASK_OP_BOL_S2(ctx->opcode);
+    address = MASK_OP_BOL_OFF16_SEXT(ctx->opcode);
+
+    switch (op1) {
+    case OPC1_32_BOL_LD_A_LONGOFF:
+        temp = tcg_temp_new();
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], address);
+        tcg_gen_qemu_ld_tl(cpu_gpr_a[r1], temp, ctx->mem_idx, MO_LEUL);
+        tcg_temp_free(temp);
+        break;
+    case OPC1_32_BOL_LD_W_LONGOFF:
+        temp = tcg_temp_new();
+        tcg_gen_addi_tl(temp, cpu_gpr_a[r2], address);
+        tcg_gen_qemu_ld_tl(cpu_gpr_d[r1], temp, ctx->mem_idx, MO_LEUL);
+        tcg_temp_free(temp);
+        break;
+    case OPC1_32_BOL_LEA_LONGOFF:
+        tcg_gen_addi_tl(cpu_gpr_a[r1], cpu_gpr_a[r2], address);
+        break;
+    case OPC1_32_BOL_ST_A_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_st(ctx, cpu_gpr_a[r1], cpu_gpr_a[r2], address, MO_LEUL);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_BOL_ST_W_LONGOFF:
+        gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LEUL);
+        break;
+    case OPC1_32_BOL_LD_B_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_SB);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_BOL_LD_BU_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_UB);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_BOL_LD_H_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LESW);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_BOL_LD_HU_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_ld(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LEUW);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_BOL_ST_B_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_SB);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_BOL_ST_H_LONGOFF:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            gen_offset_st(ctx, cpu_gpr_d[r1], cpu_gpr_a[r2], address, MO_LESW);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RC format */
+static void decode_rc_logical_shift(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2;
+    int32_t const9;
+    TCGv temp;
+
+    r2 = MASK_OP_RC_D(ctx->opcode);
+    r1 = MASK_OP_RC_S1(ctx->opcode);
+    const9 = MASK_OP_RC_CONST9(ctx->opcode);
+    op2 = MASK_OP_RC_OP2(ctx->opcode);
+
+    temp = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RC_AND:
+        tcg_gen_andi_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ANDN:
+        tcg_gen_andi_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], ~const9);
+        break;
+    case OPC2_32_RC_NAND:
+        tcg_gen_movi_tl(temp, const9);
+        tcg_gen_nand_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], temp);
+        break;
+    case OPC2_32_RC_NOR:
+        tcg_gen_movi_tl(temp, const9);
+        tcg_gen_nor_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], temp);
+        break;
+    case OPC2_32_RC_OR:
+        tcg_gen_ori_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ORN:
+        tcg_gen_ori_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], ~const9);
+        break;
+    case OPC2_32_RC_SH:
+        const9 = sextract32(const9, 0, 6);
+        gen_shi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SH_H:
+        const9 = sextract32(const9, 0, 5);
+        gen_sh_hi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SHA:
+        const9 = sextract32(const9, 0, 6);
+        gen_shaci(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SHA_H:
+        const9 = sextract32(const9, 0, 5);
+        gen_sha_hi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SHAS:
+        gen_shasi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_XNOR:
+        tcg_gen_xori_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        tcg_gen_not_tl(cpu_gpr_d[r2], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RC_XOR:
+        tcg_gen_xori_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+}
+
+static void decode_rc_accumulator(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2;
+    int16_t const9;
+
+    TCGv temp;
+
+    r2 = MASK_OP_RC_D(ctx->opcode);
+    r1 = MASK_OP_RC_S1(ctx->opcode);
+    const9 = MASK_OP_RC_CONST9_SEXT(ctx->opcode);
+
+    op2 = MASK_OP_RC_OP2(ctx->opcode);
+
+    temp = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RC_ABSDIF:
+        gen_absdifi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ABSDIFS:
+        gen_absdifsi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ADD:
+        gen_addi_d(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ADDC:
+        gen_addci_CC(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ADDS:
+        gen_addsi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ADDS_U:
+        gen_addsui(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_ADDX:
+        gen_addi_CC(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_AND_EQ:
+        gen_accumulating_condi(TCG_COND_EQ, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RC_AND_GE:
+        gen_accumulating_condi(TCG_COND_GE, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RC_AND_GE_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_accumulating_condi(TCG_COND_GEU, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RC_AND_LT:
+        gen_accumulating_condi(TCG_COND_LT, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RC_AND_LT_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_accumulating_condi(TCG_COND_LTU, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RC_AND_NE:
+        gen_accumulating_condi(TCG_COND_NE, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RC_EQ:
+        tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_EQANY_B:
+        gen_eqany_bi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_EQANY_H:
+        gen_eqany_hi(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_GE:
+        tcg_gen_setcondi_tl(TCG_COND_GE, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_GE_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        tcg_gen_setcondi_tl(TCG_COND_GEU, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_LT:
+        tcg_gen_setcondi_tl(TCG_COND_LT, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_LT_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        tcg_gen_setcondi_tl(TCG_COND_LTU, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_MAX:
+        tcg_gen_movi_tl(temp, const9);
+        tcg_gen_movcond_tl(TCG_COND_GT, cpu_gpr_d[r2], cpu_gpr_d[r1], temp,
+                           cpu_gpr_d[r1], temp);
+        break;
+    case OPC2_32_RC_MAX_U:
+        tcg_gen_movi_tl(temp, MASK_OP_RC_CONST9(ctx->opcode));
+        tcg_gen_movcond_tl(TCG_COND_GTU, cpu_gpr_d[r2], cpu_gpr_d[r1], temp,
+                           cpu_gpr_d[r1], temp);
+        break;
+    case OPC2_32_RC_MIN:
+        tcg_gen_movi_tl(temp, const9);
+        tcg_gen_movcond_tl(TCG_COND_LT, cpu_gpr_d[r2], cpu_gpr_d[r1], temp,
+                           cpu_gpr_d[r1], temp);
+        break;
+    case OPC2_32_RC_MIN_U:
+        tcg_gen_movi_tl(temp, MASK_OP_RC_CONST9(ctx->opcode));
+        tcg_gen_movcond_tl(TCG_COND_LTU, cpu_gpr_d[r2], cpu_gpr_d[r1], temp,
+                           cpu_gpr_d[r1], temp);
+        break;
+    case OPC2_32_RC_NE:
+        tcg_gen_setcondi_tl(TCG_COND_NE, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_OR_EQ:
+        gen_accumulating_condi(TCG_COND_EQ, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RC_OR_GE:
+        gen_accumulating_condi(TCG_COND_GE, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RC_OR_GE_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_accumulating_condi(TCG_COND_GEU, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RC_OR_LT:
+        gen_accumulating_condi(TCG_COND_LT, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RC_OR_LT_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_accumulating_condi(TCG_COND_LTU, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RC_OR_NE:
+        gen_accumulating_condi(TCG_COND_NE, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RC_RSUB:
+        tcg_gen_movi_tl(temp, const9);
+        gen_sub_d(cpu_gpr_d[r2], temp, cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RC_RSUBS:
+        tcg_gen_movi_tl(temp, const9);
+        gen_subs(cpu_gpr_d[r2], temp, cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RC_RSUBS_U:
+        tcg_gen_movi_tl(temp, const9);
+        gen_subsu(cpu_gpr_d[r2], temp, cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RC_SH_EQ:
+        gen_sh_condi(TCG_COND_EQ, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SH_GE:
+        gen_sh_condi(TCG_COND_GE, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SH_GE_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_sh_condi(TCG_COND_GEU, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SH_LT:
+        gen_sh_condi(TCG_COND_LT, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SH_LT_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_sh_condi(TCG_COND_LTU, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_SH_NE:
+        gen_sh_condi(TCG_COND_NE, cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_XOR_EQ:
+        gen_accumulating_condi(TCG_COND_EQ, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RC_XOR_GE:
+        gen_accumulating_condi(TCG_COND_GE, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RC_XOR_GE_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_accumulating_condi(TCG_COND_GEU, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RC_XOR_LT:
+        gen_accumulating_condi(TCG_COND_LT, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RC_XOR_LT_U:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_accumulating_condi(TCG_COND_LTU, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RC_XOR_NE:
+        gen_accumulating_condi(TCG_COND_NE, cpu_gpr_d[r2], cpu_gpr_d[r1],
+                               const9, &tcg_gen_xor_tl);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+}
+
+static void decode_rc_serviceroutine(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t const9;
+
+    op2 = MASK_OP_RC_OP2(ctx->opcode);
+    const9 = MASK_OP_RC_CONST9(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RC_BISR:
+        gen_helper_1arg(bisr, const9);
+        break;
+    case OPC2_32_RC_SYSCALL:
+        /* TODO: Add exception generation */
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rc_mul(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2;
+    int16_t const9;
+
+    r2 = MASK_OP_RC_D(ctx->opcode);
+    r1 = MASK_OP_RC_S1(ctx->opcode);
+    const9 = MASK_OP_RC_CONST9_SEXT(ctx->opcode);
+
+    op2 = MASK_OP_RC_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RC_MUL_32:
+        gen_muli_i32s(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_MUL_64:
+        CHECK_REG_PAIR(r2);
+        gen_muli_i64s(cpu_gpr_d[r2], cpu_gpr_d[r2+1], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_MULS_32:
+        gen_mulsi_i32(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_MUL_U_64:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        CHECK_REG_PAIR(r2);
+        gen_muli_i64u(cpu_gpr_d[r2], cpu_gpr_d[r2+1], cpu_gpr_d[r1], const9);
+        break;
+    case OPC2_32_RC_MULS_U_32:
+        const9 = MASK_OP_RC_CONST9(ctx->opcode);
+        gen_mulsui_i32(cpu_gpr_d[r2], cpu_gpr_d[r1], const9);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RCPW format */
+static void decode_rcpw_insert(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2;
+    int32_t pos, width, const4;
+
+    TCGv temp;
+
+    op2    = MASK_OP_RCPW_OP2(ctx->opcode);
+    r1     = MASK_OP_RCPW_S1(ctx->opcode);
+    r2     = MASK_OP_RCPW_D(ctx->opcode);
+    const4 = MASK_OP_RCPW_CONST4(ctx->opcode);
+    width  = MASK_OP_RCPW_WIDTH(ctx->opcode);
+    pos    = MASK_OP_RCPW_POS(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RCPW_IMASK:
+        CHECK_REG_PAIR(r2);
+        /* if pos + width > 31 undefined result */
+        if (pos + width <= 31) {
+            tcg_gen_movi_tl(cpu_gpr_d[r2+1], ((1u << width) - 1) << pos);
+            tcg_gen_movi_tl(cpu_gpr_d[r2], (const4 << pos));
+        }
+        break;
+    case OPC2_32_RCPW_INSERT:
+        /* if pos + width > 32 undefined result */
+        if (pos + width <= 32) {
+            temp = tcg_const_i32(const4);
+            tcg_gen_deposit_tl(cpu_gpr_d[r2], cpu_gpr_d[r1], temp, pos, width);
+            tcg_temp_free(temp);
+        }
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RCRW format */
+
+static void decode_rcrw_insert(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r3, r4;
+    int32_t width, const4;
+
+    TCGv temp, temp2, temp3;
+
+    op2    = MASK_OP_RCRW_OP2(ctx->opcode);
+    r1     = MASK_OP_RCRW_S1(ctx->opcode);
+    r3     = MASK_OP_RCRW_S3(ctx->opcode);
+    r4     = MASK_OP_RCRW_D(ctx->opcode);
+    width  = MASK_OP_RCRW_WIDTH(ctx->opcode);
+    const4 = MASK_OP_RCRW_CONST4(ctx->opcode);
+
+    temp = tcg_temp_new();
+    temp2 = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RCRW_IMASK:
+        tcg_gen_andi_tl(temp, cpu_gpr_d[r4], 0x1f);
+        tcg_gen_movi_tl(temp2, (1 << width) - 1);
+        tcg_gen_shl_tl(cpu_gpr_d[r3 + 1], temp2, temp);
+        tcg_gen_movi_tl(temp2, const4);
+        tcg_gen_shl_tl(cpu_gpr_d[r3], temp2, temp);
+        break;
+    case OPC2_32_RCRW_INSERT:
+        temp3 = tcg_temp_new();
+
+        tcg_gen_movi_tl(temp, width);
+        tcg_gen_movi_tl(temp2, const4);
+        tcg_gen_andi_tl(temp3, cpu_gpr_d[r4], 0x1f);
+        gen_insert(cpu_gpr_d[r3], cpu_gpr_d[r1], temp2, temp, temp3);
+
+        tcg_temp_free(temp3);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+/* RCR format */
+
+static void decode_rcr_cond_select(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r3, r4;
+    int32_t const9;
+
+    TCGv temp, temp2;
+
+    op2 = MASK_OP_RCR_OP2(ctx->opcode);
+    r1 = MASK_OP_RCR_S1(ctx->opcode);
+    const9 = MASK_OP_RCR_CONST9_SEXT(ctx->opcode);
+    r3 = MASK_OP_RCR_S3(ctx->opcode);
+    r4 = MASK_OP_RCR_D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RCR_CADD:
+        gen_condi_add(TCG_COND_NE, cpu_gpr_d[r1], const9, cpu_gpr_d[r3],
+                      cpu_gpr_d[r4]);
+        break;
+    case OPC2_32_RCR_CADDN:
+        gen_condi_add(TCG_COND_EQ, cpu_gpr_d[r1], const9, cpu_gpr_d[r3],
+                      cpu_gpr_d[r4]);
+        break;
+    case OPC2_32_RCR_SEL:
+        temp = tcg_const_i32(0);
+        temp2 = tcg_const_i32(const9);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr_d[r4], cpu_gpr_d[r3], temp,
+                           cpu_gpr_d[r1], temp2);
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+    case OPC2_32_RCR_SELN:
+        temp = tcg_const_i32(0);
+        temp2 = tcg_const_i32(const9);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr_d[r4], cpu_gpr_d[r3], temp,
+                           cpu_gpr_d[r1], temp2);
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rcr_madd(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r3, r4;
+    int32_t const9;
+
+
+    op2 = MASK_OP_RCR_OP2(ctx->opcode);
+    r1 = MASK_OP_RCR_S1(ctx->opcode);
+    const9 = MASK_OP_RCR_CONST9_SEXT(ctx->opcode);
+    r3 = MASK_OP_RCR_S3(ctx->opcode);
+    r4 = MASK_OP_RCR_D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RCR_MADD_32:
+        gen_maddi32_d(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
+        break;
+    case OPC2_32_RCR_MADD_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddi64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    case OPC2_32_RCR_MADDS_32:
+        gen_maddsi_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
+        break;
+    case OPC2_32_RCR_MADDS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsi_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    case OPC2_32_RCR_MADD_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        const9 = MASK_OP_RCR_CONST9(ctx->opcode);
+        gen_maddui64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    case OPC2_32_RCR_MADDS_U_32:
+        const9 = MASK_OP_RCR_CONST9(ctx->opcode);
+        gen_maddsui_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
+        break;
+    case OPC2_32_RCR_MADDS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        const9 = MASK_OP_RCR_CONST9(ctx->opcode);
+        gen_maddsui_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rcr_msub(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r3, r4;
+    int32_t const9;
+
+
+    op2 = MASK_OP_RCR_OP2(ctx->opcode);
+    r1 = MASK_OP_RCR_S1(ctx->opcode);
+    const9 = MASK_OP_RCR_CONST9_SEXT(ctx->opcode);
+    r3 = MASK_OP_RCR_S3(ctx->opcode);
+    r4 = MASK_OP_RCR_D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RCR_MSUB_32:
+        gen_msubi32_d(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
+        break;
+    case OPC2_32_RCR_MSUB_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubi64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    case OPC2_32_RCR_MSUBS_32:
+        gen_msubsi_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
+        break;
+    case OPC2_32_RCR_MSUBS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubsi_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    case OPC2_32_RCR_MSUB_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        const9 = MASK_OP_RCR_CONST9(ctx->opcode);
+        gen_msubui64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    case OPC2_32_RCR_MSUBS_U_32:
+        const9 = MASK_OP_RCR_CONST9(ctx->opcode);
+        gen_msubsui_32(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3], const9);
+        break;
+    case OPC2_32_RCR_MSUBS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        const9 = MASK_OP_RCR_CONST9(ctx->opcode);
+        gen_msubsui_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                       cpu_gpr_d[r3], cpu_gpr_d[r3+1], const9);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RLC format */
+
+static void decode_rlc_opc(CPUTriCoreState *env, DisasContext *ctx,
+                           uint32_t op1)
+{
+    int32_t const16;
+    int r1, r2;
+
+    const16 = MASK_OP_RLC_CONST16_SEXT(ctx->opcode);
+    r1      = MASK_OP_RLC_S1(ctx->opcode);
+    r2      = MASK_OP_RLC_D(ctx->opcode);
+
+    switch (op1) {
+    case OPC1_32_RLC_ADDI:
+        gen_addi_d(cpu_gpr_d[r2], cpu_gpr_d[r1], const16);
+        break;
+    case OPC1_32_RLC_ADDIH:
+        gen_addi_d(cpu_gpr_d[r2], cpu_gpr_d[r1], const16 << 16);
+        break;
+    case OPC1_32_RLC_ADDIH_A:
+        tcg_gen_addi_tl(cpu_gpr_a[r2], cpu_gpr_a[r1], const16 << 16);
+        break;
+    case OPC1_32_RLC_MFCR:
+        const16 = MASK_OP_RLC_CONST16(ctx->opcode);
+        gen_mfcr(env, cpu_gpr_d[r2], const16);
+        break;
+    case OPC1_32_RLC_MOV:
+        tcg_gen_movi_tl(cpu_gpr_d[r2], const16);
+        break;
+    case OPC1_32_RLC_MOV_64:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            CHECK_REG_PAIR(r2);
+            tcg_gen_movi_tl(cpu_gpr_d[r2], const16);
+            tcg_gen_movi_tl(cpu_gpr_d[r2+1], const16 >> 15);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC1_32_RLC_MOV_U:
+        const16 = MASK_OP_RLC_CONST16(ctx->opcode);
+        tcg_gen_movi_tl(cpu_gpr_d[r2], const16);
+        break;
+    case OPC1_32_RLC_MOV_H:
+        tcg_gen_movi_tl(cpu_gpr_d[r2], const16 << 16);
+        break;
+    case OPC1_32_RLC_MOVH_A:
+        tcg_gen_movi_tl(cpu_gpr_a[r2], const16 << 16);
+        break;
+    case OPC1_32_RLC_MTCR:
+        const16 = MASK_OP_RLC_CONST16(ctx->opcode);
+        gen_mtcr(env, ctx, cpu_gpr_d[r1], const16);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RR format */
+static void decode_rr_accumulator(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r3, r2, r1;
+
+    r3 = MASK_OP_RR_D(ctx->opcode);
+    r2 = MASK_OP_RR_S2(ctx->opcode);
+    r1 = MASK_OP_RR_S1(ctx->opcode);
+    op2 = MASK_OP_RR_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RR_ABS:
+        gen_abs(cpu_gpr_d[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABS_B:
+        gen_helper_abs_b(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABS_H:
+        gen_helper_abs_h(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSDIF:
+        gen_absdif(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSDIF_B:
+        gen_helper_absdif_b(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                            cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSDIF_H:
+        gen_helper_absdif_h(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                            cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSDIFS:
+        gen_helper_absdif_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                               cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSDIFS_H:
+        gen_helper_absdif_h_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                                 cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSS:
+        gen_helper_abs_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ABSS_H:
+        gen_helper_abs_h_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADD:
+        gen_add_d(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADD_B:
+        gen_helper_add_b(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADD_H:
+        gen_helper_add_h(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADDC:
+        gen_addc_CC(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADDS:
+        gen_adds(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADDS_H:
+        gen_helper_add_h_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                              cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADDS_HU:
+        gen_helper_add_h_suov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                              cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADDS_U:
+        gen_helper_add_suov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                            cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ADDX:
+        gen_add_CC(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_AND_EQ:
+        gen_accumulating_cond(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RR_AND_GE:
+        gen_accumulating_cond(TCG_COND_GE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RR_AND_GE_U:
+        gen_accumulating_cond(TCG_COND_GEU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RR_AND_LT:
+        gen_accumulating_cond(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RR_AND_LT_U:
+        gen_accumulating_cond(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RR_AND_NE:
+        gen_accumulating_cond(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_and_tl);
+        break;
+    case OPC2_32_RR_EQ:
+        tcg_gen_setcond_tl(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_EQ_B:
+        gen_helper_eq_b(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_EQ_H:
+        gen_helper_eq_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_EQ_W:
+        gen_cond_w(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_EQANY_B:
+        gen_helper_eqany_b(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_EQANY_H:
+        gen_helper_eqany_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_GE:
+        tcg_gen_setcond_tl(TCG_COND_GE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_GE_U:
+        tcg_gen_setcond_tl(TCG_COND_GEU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT:
+        tcg_gen_setcond_tl(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_U:
+        tcg_gen_setcond_tl(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_B:
+        gen_helper_lt_b(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_BU:
+        gen_helper_lt_bu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_H:
+        gen_helper_lt_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_HU:
+        gen_helper_lt_hu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_W:
+        gen_cond_w(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_LT_WU:
+        gen_cond_w(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MAX:
+        tcg_gen_movcond_tl(TCG_COND_GT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MAX_U:
+        tcg_gen_movcond_tl(TCG_COND_GTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MAX_B:
+        gen_helper_max_b(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MAX_BU:
+        gen_helper_max_bu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MAX_H:
+        gen_helper_max_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MAX_HU:
+        gen_helper_max_hu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MIN:
+        tcg_gen_movcond_tl(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MIN_U:
+        tcg_gen_movcond_tl(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MIN_B:
+        gen_helper_min_b(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MIN_BU:
+        gen_helper_min_bu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MIN_H:
+        gen_helper_min_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MIN_HU:
+        gen_helper_min_hu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MOV:
+        tcg_gen_mov_tl(cpu_gpr_d[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_NE:
+        tcg_gen_setcond_tl(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                           cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_OR_EQ:
+        gen_accumulating_cond(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RR_OR_GE:
+        gen_accumulating_cond(TCG_COND_GE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RR_OR_GE_U:
+        gen_accumulating_cond(TCG_COND_GEU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RR_OR_LT:
+        gen_accumulating_cond(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RR_OR_LT_U:
+        gen_accumulating_cond(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RR_OR_NE:
+        gen_accumulating_cond(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_or_tl);
+        break;
+    case OPC2_32_RR_SAT_B:
+        gen_saturate(cpu_gpr_d[r3], cpu_gpr_d[r1], 0x7f, -0x80);
+        break;
+    case OPC2_32_RR_SAT_BU:
+        gen_saturate_u(cpu_gpr_d[r3], cpu_gpr_d[r1], 0xff);
+        break;
+    case OPC2_32_RR_SAT_H:
+        gen_saturate(cpu_gpr_d[r3], cpu_gpr_d[r1], 0x7fff, -0x8000);
+        break;
+    case OPC2_32_RR_SAT_HU:
+        gen_saturate_u(cpu_gpr_d[r3], cpu_gpr_d[r1], 0xffff);
+        break;
+    case OPC2_32_RR_SH_EQ:
+        gen_sh_cond(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                    cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH_GE:
+        gen_sh_cond(TCG_COND_GE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                    cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH_GE_U:
+        gen_sh_cond(TCG_COND_GEU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                    cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH_LT:
+        gen_sh_cond(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                    cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH_LT_U:
+        gen_sh_cond(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                    cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH_NE:
+        gen_sh_cond(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                    cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUB:
+        gen_sub_d(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUB_B:
+        gen_helper_sub_b(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUB_H:
+        gen_helper_sub_h(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUBC:
+        gen_subc_CC(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUBS:
+        gen_subs(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUBS_U:
+        gen_subsu(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUBS_H:
+        gen_helper_sub_h_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                              cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUBS_HU:
+        gen_helper_sub_h_suov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                              cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SUBX:
+        gen_sub_CC(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_XOR_EQ:
+        gen_accumulating_cond(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RR_XOR_GE:
+        gen_accumulating_cond(TCG_COND_GE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RR_XOR_GE_U:
+        gen_accumulating_cond(TCG_COND_GEU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RR_XOR_LT:
+        gen_accumulating_cond(TCG_COND_LT, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RR_XOR_LT_U:
+        gen_accumulating_cond(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_xor_tl);
+        break;
+    case OPC2_32_RR_XOR_NE:
+        gen_accumulating_cond(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_d[r1],
+                              cpu_gpr_d[r2], &tcg_gen_xor_tl);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rr_logical_shift(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r3, r2, r1;
+    TCGv temp;
+
+    r3 = MASK_OP_RR_D(ctx->opcode);
+    r2 = MASK_OP_RR_S2(ctx->opcode);
+    r1 = MASK_OP_RR_S1(ctx->opcode);
+
+    temp = tcg_temp_new();
+    op2 = MASK_OP_RR_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RR_AND:
+        tcg_gen_and_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ANDN:
+        tcg_gen_andc_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_CLO:
+        gen_helper_clo(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_CLO_H:
+        gen_helper_clo_h(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_CLS:
+        gen_helper_cls(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_CLS_H:
+        gen_helper_cls_h(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_CLZ:
+        gen_helper_clz(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_CLZ_H:
+        gen_helper_clz_h(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_NAND:
+        tcg_gen_nand_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_NOR:
+        tcg_gen_nor_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_OR:
+        tcg_gen_or_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_ORN:
+        tcg_gen_orc_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH:
+        gen_helper_sh(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SH_H:
+        gen_helper_sh_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SHA:
+        gen_helper_sha(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SHA_H:
+        gen_helper_sha_h(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_SHAS:
+        gen_shas(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_XNOR:
+        tcg_gen_eqv_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_XOR:
+        tcg_gen_xor_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+}
+
+static void decode_rr_address(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2, n;
+    int r1, r2, r3;
+    TCGv temp;
+
+    op2 = MASK_OP_RR_OP2(ctx->opcode);
+    r3 = MASK_OP_RR_D(ctx->opcode);
+    r2 = MASK_OP_RR_S2(ctx->opcode);
+    r1 = MASK_OP_RR_S1(ctx->opcode);
+    n = MASK_OP_RR_N(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RR_ADD_A:
+        tcg_gen_add_tl(cpu_gpr_a[r3], cpu_gpr_a[r1], cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_ADDSC_A:
+        temp = tcg_temp_new();
+        tcg_gen_shli_tl(temp, cpu_gpr_d[r1], n);
+        tcg_gen_add_tl(cpu_gpr_a[r3], cpu_gpr_a[r2], temp);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_RR_ADDSC_AT:
+        temp = tcg_temp_new();
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 3);
+        tcg_gen_add_tl(temp, cpu_gpr_a[r2], temp);
+        tcg_gen_andi_tl(cpu_gpr_a[r3], temp, 0xFFFFFFFC);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_RR_EQ_A:
+        tcg_gen_setcond_tl(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_a[r1],
+                           cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_EQZ:
+        tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_gpr_d[r3], cpu_gpr_a[r1], 0);
+        break;
+    case OPC2_32_RR_GE_A:
+        tcg_gen_setcond_tl(TCG_COND_GEU, cpu_gpr_d[r3], cpu_gpr_a[r1],
+                           cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_LT_A:
+        tcg_gen_setcond_tl(TCG_COND_LTU, cpu_gpr_d[r3], cpu_gpr_a[r1],
+                           cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_MOV_A:
+        tcg_gen_mov_tl(cpu_gpr_a[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_MOV_AA:
+        tcg_gen_mov_tl(cpu_gpr_a[r3], cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_MOV_D:
+        tcg_gen_mov_tl(cpu_gpr_d[r3], cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_NE_A:
+        tcg_gen_setcond_tl(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_a[r1],
+                           cpu_gpr_a[r2]);
+        break;
+    case OPC2_32_RR_NEZ_A:
+        tcg_gen_setcondi_tl(TCG_COND_NE, cpu_gpr_d[r3], cpu_gpr_a[r1], 0);
+        break;
+    case OPC2_32_RR_SUB_A:
+        tcg_gen_sub_tl(cpu_gpr_a[r3], cpu_gpr_a[r1], cpu_gpr_a[r2]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rr_idirect(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1;
+
+    op2 = MASK_OP_RR_OP2(ctx->opcode);
+    r1 = MASK_OP_RR_S1(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RR_JI:
+        tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], ~0x1);
+        break;
+    case OPC2_32_RR_JLI:
+        tcg_gen_movi_tl(cpu_gpr_a[11], ctx->next_pc);
+        tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], ~0x1);
+        break;
+    case OPC2_32_RR_CALLI:
+        gen_helper_1arg(call, ctx->next_pc);
+        tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], ~0x1);
+        break;
+    case OPC2_32_RR_FCALLI:
+        gen_fcall_save_ctx(ctx);
+        tcg_gen_andi_tl(cpu_PC, cpu_gpr_a[r1], ~0x1);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_gen_exit_tb(0);
+    ctx->bstate = BS_BRANCH;
+}
+
+static void decode_rr_divide(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+
+    TCGv temp, temp2, temp3;
+
+    op2 = MASK_OP_RR_OP2(ctx->opcode);
+    r3 = MASK_OP_RR_D(ctx->opcode);
+    r2 = MASK_OP_RR_S2(ctx->opcode);
+    r1 = MASK_OP_RR_S1(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RR_BMERGE:
+        gen_helper_bmerge(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_BSPLIT:
+        CHECK_REG_PAIR(r3);
+        gen_bsplit(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_DVINIT_B:
+        CHECK_REG_PAIR(r3);
+        gen_dvinit_b(env, cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_DVINIT_BU:
+        temp = tcg_temp_new();
+        temp2 = tcg_temp_new();
+        temp3 = tcg_temp_new();
+        CHECK_REG_PAIR(r3);
+        tcg_gen_shri_tl(temp3, cpu_gpr_d[r1], 8);
+        /* reset av */
+        tcg_gen_movi_tl(cpu_PSW_AV, 0);
+        if (!tricore_feature(env, TRICORE_FEATURE_131)) {
+            /* overflow = (abs(D[r3+1]) >= abs(D[r2])) */
+            tcg_gen_neg_tl(temp, temp3);
+            /* use cpu_PSW_AV to compare against 0 */
+            tcg_gen_movcond_tl(TCG_COND_LT, temp, temp3, cpu_PSW_AV,
+                               temp, temp3);
+            tcg_gen_neg_tl(temp2, cpu_gpr_d[r2]);
+            tcg_gen_movcond_tl(TCG_COND_LT, temp2, cpu_gpr_d[r2], cpu_PSW_AV,
+                               temp2, cpu_gpr_d[r2]);
+            tcg_gen_setcond_tl(TCG_COND_GE, cpu_PSW_V, temp, temp2);
+        } else {
+            /* overflow = (D[b] == 0) */
+            tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_PSW_V, cpu_gpr_d[r2], 0);
+        }
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* sv */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+        /* write result */
+        tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], 24);
+        tcg_gen_mov_tl(cpu_gpr_d[r3+1], temp3);
+
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        tcg_temp_free(temp3);
+        break;
+    case OPC2_32_RR_DVINIT_H:
+        CHECK_REG_PAIR(r3);
+        gen_dvinit_h(env, cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_DVINIT_HU:
+        temp = tcg_temp_new();
+        temp2 = tcg_temp_new();
+        temp3 = tcg_temp_new();
+        CHECK_REG_PAIR(r3);
+        tcg_gen_shri_tl(temp3, cpu_gpr_d[r1], 16);
+        /* reset av */
+        tcg_gen_movi_tl(cpu_PSW_AV, 0);
+        if (!tricore_feature(env, TRICORE_FEATURE_131)) {
+            /* overflow = (abs(D[r3+1]) >= abs(D[r2])) */
+            tcg_gen_neg_tl(temp, temp3);
+            /* use cpu_PSW_AV to compare against 0 */
+            tcg_gen_movcond_tl(TCG_COND_LT, temp, temp3, cpu_PSW_AV,
+                               temp, temp3);
+            tcg_gen_neg_tl(temp2, cpu_gpr_d[r2]);
+            tcg_gen_movcond_tl(TCG_COND_LT, temp2, cpu_gpr_d[r2], cpu_PSW_AV,
+                               temp2, cpu_gpr_d[r2]);
+            tcg_gen_setcond_tl(TCG_COND_GE, cpu_PSW_V, temp, temp2);
+        } else {
+            /* overflow = (D[b] == 0) */
+            tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_PSW_V, cpu_gpr_d[r2], 0);
+        }
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* sv */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+        /* write result */
+        tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], 16);
+        tcg_gen_mov_tl(cpu_gpr_d[r3+1], temp3);
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        tcg_temp_free(temp3);
+        break;
+    case OPC2_32_RR_DVINIT:
+        temp = tcg_temp_new();
+        temp2 = tcg_temp_new();
+        CHECK_REG_PAIR(r3);
+        /* overflow = ((D[b] == 0) ||
+                      ((D[b] == 0xFFFFFFFF) && (D[a] == 0x80000000))) */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp, cpu_gpr_d[r2], 0xffffffff);
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, cpu_gpr_d[r1], 0x80000000);
+        tcg_gen_and_tl(temp, temp, temp2);
+        tcg_gen_setcondi_tl(TCG_COND_EQ, temp2, cpu_gpr_d[r2], 0);
+        tcg_gen_or_tl(cpu_PSW_V, temp, temp2);
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* sv */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+        /* reset av */
+       tcg_gen_movi_tl(cpu_PSW_AV, 0);
+        /* write result */
+        tcg_gen_mov_tl(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        /* sign extend to high reg */
+        tcg_gen_sari_tl(cpu_gpr_d[r3+1], cpu_gpr_d[r1], 31);
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        break;
+    case OPC2_32_RR_DVINIT_U:
+        /* overflow = (D[b] == 0) */
+        tcg_gen_setcondi_tl(TCG_COND_EQ, cpu_PSW_V, cpu_gpr_d[r2], 0);
+        tcg_gen_shli_tl(cpu_PSW_V, cpu_PSW_V, 31);
+        /* sv */
+        tcg_gen_or_tl(cpu_PSW_SV, cpu_PSW_SV, cpu_PSW_V);
+        /* reset av */
+        tcg_gen_movi_tl(cpu_PSW_AV, 0);
+        /* write result */
+        tcg_gen_mov_tl(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        /* zero extend to high reg*/
+        tcg_gen_movi_tl(cpu_gpr_d[r3+1], 0);
+        break;
+    case OPC2_32_RR_PARITY:
+        gen_helper_parity(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_UNPACK:
+        CHECK_REG_PAIR(r3);
+        gen_unpack(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_CRC32:
+        if (tricore_feature(env, TRICORE_FEATURE_161)) {
+            gen_helper_crc32(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC2_32_RR_DIV:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            GEN_HELPER_RR(divide, cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
+                          cpu_gpr_d[r2]);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC2_32_RR_DIV_U:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            GEN_HELPER_RR(divide_u, cpu_gpr_d[r3], cpu_gpr_d[r3+1],
+                          cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC2_32_RR_MUL_F:
+        gen_helper_fmul(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_DIV_F:
+        gen_helper_fdiv(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_CMP_F:
+        gen_helper_fcmp(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR_FTOI:
+        gen_helper_ftoi(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RR_ITOF:
+        gen_helper_itof(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RR1 Format */
+static void decode_rr1_mul(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+
+    int r1, r2, r3;
+    TCGv n;
+    TCGv_i64 temp64;
+
+    r1 = MASK_OP_RR1_S1(ctx->opcode);
+    r2 = MASK_OP_RR1_S2(ctx->opcode);
+    r3 = MASK_OP_RR1_D(ctx->opcode);
+    n  = tcg_const_i32(MASK_OP_RR1_N(ctx->opcode));
+    op2 = MASK_OP_RR1_OP2(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RR1_MUL_H_32_LL:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_LL(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MUL_H_32_LU:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_LU(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MUL_H_32_UL:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_UL(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MUL_H_32_UU:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_UU(mul_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        gen_calc_usb_mul_h(cpu_gpr_d[r3], cpu_gpr_d[r3+1]);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MULM_H_64_LL:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_LL(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        /* reset V bit */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+        /* reset AV bit */
+        tcg_gen_mov_tl(cpu_PSW_AV, cpu_PSW_V);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MULM_H_64_LU:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_LU(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        /* reset V bit */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+        /* reset AV bit */
+        tcg_gen_mov_tl(cpu_PSW_AV, cpu_PSW_V);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MULM_H_64_UL:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_UL(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        /* reset V bit */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+        /* reset AV bit */
+        tcg_gen_mov_tl(cpu_PSW_AV, cpu_PSW_V);
+        tcg_temp_free_i64(temp64);
+        break;
+    case OPC2_32_RR1_MULM_H_64_UU:
+        temp64 = tcg_temp_new_i64();
+        CHECK_REG_PAIR(r3);
+        GEN_HELPER_UU(mulm_h, temp64, cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        tcg_gen_extr_i64_i32(cpu_gpr_d[r3], cpu_gpr_d[r3+1], temp64);
+        /* reset V bit */
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+        /* reset AV bit */
+        tcg_gen_mov_tl(cpu_PSW_AV, cpu_PSW_V);
+        tcg_temp_free_i64(temp64);
+
+        break;
+    case OPC2_32_RR1_MULR_H_16_LL:
+        GEN_HELPER_LL(mulr_h, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        gen_calc_usb_mulr_h(cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RR1_MULR_H_16_LU:
+        GEN_HELPER_LU(mulr_h, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        gen_calc_usb_mulr_h(cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RR1_MULR_H_16_UL:
+        GEN_HELPER_UL(mulr_h, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        gen_calc_usb_mulr_h(cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RR1_MULR_H_16_UU:
+        GEN_HELPER_UU(mulr_h, cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2], n);
+        gen_calc_usb_mulr_h(cpu_gpr_d[r3]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(n);
+}
+
+static void decode_rr1_mulq(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    uint32_t n;
+
+    TCGv temp, temp2;
+
+    r1 = MASK_OP_RR1_S1(ctx->opcode);
+    r2 = MASK_OP_RR1_S2(ctx->opcode);
+    r3 = MASK_OP_RR1_D(ctx->opcode);
+    n  = MASK_OP_RR1_N(ctx->opcode);
+    op2 = MASK_OP_RR1_OP2(ctx->opcode);
+
+    temp = tcg_temp_new();
+    temp2 = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RR1_MUL_Q_32:
+        gen_mul_q(cpu_gpr_d[r3], temp, cpu_gpr_d[r1], cpu_gpr_d[r2], n, 32);
+        break;
+    case OPC2_32_RR1_MUL_Q_64:
+        CHECK_REG_PAIR(r3);
+        gen_mul_q(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                  n, 0);
+        break;
+    case OPC2_32_RR1_MUL_Q_32_L:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_mul_q(cpu_gpr_d[r3], temp, cpu_gpr_d[r1], temp, n, 16);
+        break;
+    case OPC2_32_RR1_MUL_Q_64_L:
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_mul_q(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp, n, 0);
+        break;
+    case OPC2_32_RR1_MUL_Q_32_U:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_mul_q(cpu_gpr_d[r3], temp, cpu_gpr_d[r1], temp, n, 16);
+        break;
+    case OPC2_32_RR1_MUL_Q_64_U:
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_mul_q(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp, n, 0);
+        break;
+    case OPC2_32_RR1_MUL_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_mul_q_16(cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RR1_MUL_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_mul_q_16(cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RR1_MULR_Q_32_L:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_mulr_q(cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RR1_MULR_Q_32_U:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_mulr_q(cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+/* RR2 format */
+static void decode_rr2_mul(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+
+    op2 = MASK_OP_RR2_OP2(ctx->opcode);
+    r1  = MASK_OP_RR2_S1(ctx->opcode);
+    r2  = MASK_OP_RR2_S2(ctx->opcode);
+    r3  = MASK_OP_RR2_D(ctx->opcode);
+    switch (op2) {
+    case OPC2_32_RR2_MUL_32:
+        gen_mul_i32s(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR2_MUL_64:
+        CHECK_REG_PAIR(r3);
+        gen_mul_i64s(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR2_MULS_32:
+        gen_helper_mul_ssov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                            cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR2_MUL_U_64:
+        CHECK_REG_PAIR(r3);
+        gen_mul_i64u(cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RR2_MULS_U_32:
+        gen_helper_mul_suov(cpu_gpr_d[r3], cpu_env, cpu_gpr_d[r1],
+                            cpu_gpr_d[r2]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RRPW format */
+static void decode_rrpw_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3;
+    int32_t pos, width;
+
+    op2 = MASK_OP_RRPW_OP2(ctx->opcode);
+    r1 = MASK_OP_RRPW_S1(ctx->opcode);
+    r2 = MASK_OP_RRPW_S2(ctx->opcode);
+    r3 = MASK_OP_RRPW_D(ctx->opcode);
+    pos = MASK_OP_RRPW_POS(ctx->opcode);
+    width = MASK_OP_RRPW_WIDTH(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRPW_EXTR:
+        if (pos + width <= 31) {
+            /* optimize special cases */
+            if ((pos == 0) && (width == 8)) {
+                tcg_gen_ext8s_tl(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+            } else if ((pos == 0) && (width == 16)) {
+                tcg_gen_ext16s_tl(cpu_gpr_d[r3], cpu_gpr_d[r1]);
+            } else {
+                tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], 32 - pos - width);
+                tcg_gen_sari_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], 32 - width);
+            }
+        }
+        break;
+    case OPC2_32_RRPW_EXTR_U:
+        if (width == 0) {
+            tcg_gen_movi_tl(cpu_gpr_d[r3], 0);
+        } else {
+            tcg_gen_shri_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], pos);
+            tcg_gen_andi_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], ~0u >> (32-width));
+        }
+        break;
+    case OPC2_32_RRPW_IMASK:
+        CHECK_REG_PAIR(r3);
+        if (pos + width <= 31) {
+            tcg_gen_movi_tl(cpu_gpr_d[r3+1], ((1u << width) - 1) << pos);
+            tcg_gen_shli_tl(cpu_gpr_d[r3], cpu_gpr_d[r2], pos);
+        }
+        break;
+    case OPC2_32_RRPW_INSERT:
+        if (pos + width <= 31) {
+            tcg_gen_deposit_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                               width, pos);
+        }
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RRR format */
+static void decode_rrr_cond_select(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3, r4;
+    TCGv temp;
+
+    op2 = MASK_OP_RRR_OP2(ctx->opcode);
+    r1  = MASK_OP_RRR_S1(ctx->opcode);
+    r2  = MASK_OP_RRR_S2(ctx->opcode);
+    r3  = MASK_OP_RRR_S3(ctx->opcode);
+    r4  = MASK_OP_RRR_D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR_CADD:
+        gen_cond_add(TCG_COND_NE, cpu_gpr_d[r1], cpu_gpr_d[r2],
+                     cpu_gpr_d[r4], cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RRR_CADDN:
+        gen_cond_add(TCG_COND_EQ, cpu_gpr_d[r1], cpu_gpr_d[r2], cpu_gpr_d[r4],
+                     cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RRR_CSUB:
+        gen_cond_sub(TCG_COND_NE, cpu_gpr_d[r1], cpu_gpr_d[r2], cpu_gpr_d[r4],
+                     cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RRR_CSUBN:
+        gen_cond_sub(TCG_COND_EQ, cpu_gpr_d[r1], cpu_gpr_d[r2], cpu_gpr_d[r4],
+                     cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RRR_SEL:
+        temp = tcg_const_i32(0);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr_d[r4], cpu_gpr_d[r3], temp,
+                           cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        tcg_temp_free(temp);
+        break;
+    case OPC2_32_RRR_SELN:
+        temp = tcg_const_i32(0);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr_d[r4], cpu_gpr_d[r3], temp,
+                           cpu_gpr_d[r1], cpu_gpr_d[r2]);
+        tcg_temp_free(temp);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rrr_divide(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+
+    int r1, r2, r3, r4;
+
+    op2 = MASK_OP_RRR_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR_S1(ctx->opcode);
+    r2 = MASK_OP_RRR_S2(ctx->opcode);
+    r3 = MASK_OP_RRR_S3(ctx->opcode);
+    r4 = MASK_OP_RRR_D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR_DVADJ:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(dvadj, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_DVSTEP:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(dvstep, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_DVSTEP_U:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(dvstep_u, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_IXMAX:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(ixmax, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_IXMAX_U:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(ixmax_u, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_IXMIN:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(ixmin, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_IXMIN_U:
+        CHECK_REG_PAIR(r3);
+        CHECK_REG_PAIR(r4);
+        GEN_HELPER_RRR(ixmin_u, cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR_PACK:
+        CHECK_REG_PAIR(r3);
+        gen_helper_pack(cpu_gpr_d[r4], cpu_PSW_C, cpu_gpr_d[r3],
+                        cpu_gpr_d[r3+1], cpu_gpr_d[r1]);
+        break;
+    case OPC2_32_RRR_ADD_F:
+        gen_helper_fadd(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r3]);
+        break;
+    case OPC2_32_RRR_SUB_F:
+        gen_helper_fsub(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1], cpu_gpr_d[r3]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RRR2 format */
+static void decode_rrr2_madd(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4;
+
+    op2 = MASK_OP_RRR2_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR2_S1(ctx->opcode);
+    r2 = MASK_OP_RRR2_S2(ctx->opcode);
+    r3 = MASK_OP_RRR2_S3(ctx->opcode);
+    r4 = MASK_OP_RRR2_D(ctx->opcode);
+    switch (op2) {
+    case OPC2_32_RRR2_MADD_32:
+        gen_madd32_d(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MADD_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madd64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MADDS_32:
+        gen_helper_madd32_ssov(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1],
+                               cpu_gpr_d[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MADDS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madds_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MADD_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddu64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MADDS_U_32:
+        gen_helper_madd32_suov(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1],
+                               cpu_gpr_d[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MADDS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsu_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rrr2_msub(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4;
+
+    op2 = MASK_OP_RRR2_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR2_S1(ctx->opcode);
+    r2 = MASK_OP_RRR2_S2(ctx->opcode);
+    r3 = MASK_OP_RRR2_S3(ctx->opcode);
+    r4 = MASK_OP_RRR2_D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR2_MSUB_32:
+        gen_msub32_d(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MSUB_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msub64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MSUBS_32:
+        gen_helper_msub32_ssov(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1],
+                               cpu_gpr_d[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MSUBS_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubs_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                     cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MSUB_U_64:
+        gen_msubu64_d(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MSUBS_U_32:
+        gen_helper_msub32_suov(cpu_gpr_d[r4], cpu_env, cpu_gpr_d[r1],
+                               cpu_gpr_d[r3], cpu_gpr_d[r2]);
+        break;
+    case OPC2_32_RRR2_MSUBS_U_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubsu_64(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r1],
+                      cpu_gpr_d[r3], cpu_gpr_d[r3+1], cpu_gpr_d[r2]);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RRR1 format */
+static void decode_rrr1_madd(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4, n;
+
+    op2 = MASK_OP_RRR1_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR1_S1(ctx->opcode);
+    r2 = MASK_OP_RRR1_S2(ctx->opcode);
+    r3 = MASK_OP_RRR1_S3(ctx->opcode);
+    r4 = MASK_OP_RRR1_D(ctx->opcode);
+    n = MASK_OP_RRR1_N(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR1_MADD_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADD_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADD_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADD_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madd_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madds_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDM_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDM_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDM_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDM_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDMS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDMS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDMS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDMS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDR_H_LL:
+        gen_maddr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDR_H_LU:
+        gen_maddr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDR_H_UL:
+        gen_maddr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDR_H_UU:
+        gen_maddr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDRS_H_LL:
+        gen_maddr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDRS_H_LU:
+        gen_maddr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDRS_H_UL:
+        gen_maddr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDRS_H_UU:
+        gen_maddr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rrr1_maddq_h(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4, n;
+    TCGv temp, temp2;
+
+    op2 = MASK_OP_RRR1_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR1_S1(ctx->opcode);
+    r2 = MASK_OP_RRR1_S2(ctx->opcode);
+    r3 = MASK_OP_RRR1_S3(ctx->opcode);
+    r4 = MASK_OP_RRR1_D(ctx->opcode);
+    n = MASK_OP_RRR1_N(ctx->opcode);
+
+    temp = tcg_const_i32(n);
+    temp2 = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RRR1_MADD_Q_32:
+        gen_madd32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                     cpu_gpr_d[r2], n, 32, env);
+        break;
+    case OPC2_32_RRR1_MADD_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madd64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                     n, env);
+        break;
+    case OPC2_32_RRR1_MADD_Q_32_L:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_madd32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                     temp, n, 16, env);
+        break;
+    case OPC2_32_RRR1_MADD_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_madd64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                     n, env);
+        break;
+    case OPC2_32_RRR1_MADD_Q_32_U:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_madd32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                     temp, n, 16, env);
+        break;
+    case OPC2_32_RRR1_MADD_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_madd64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                     n, env);
+        break;
+    case OPC2_32_RRR1_MADD_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16add32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADD_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16add64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADD_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16add32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADD_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16add64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_32:
+        gen_madds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, 32);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_madds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_32_L:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_madds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      temp, n, 16);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_madds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                      n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_32_U:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_madds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      temp, n, 16);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_madds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                      n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16adds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16adds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                        cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16adds32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDS_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16adds64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                        cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDR_H_64_UL:
+        CHECK_REG_PAIR(r3);
+        gen_maddr64_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
+                      cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
+        break;
+    case OPC2_32_RRR1_MADDRS_H_64_UL:
+        CHECK_REG_PAIR(r3);
+        gen_maddr64s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
+                       cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
+        break;
+    case OPC2_32_RRR1_MADDR_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_maddr_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDR_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_maddr_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDRS_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_maddrs_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MADDRS_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_maddrs_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static void decode_rrr1_maddsu_h(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4, n;
+
+    op2 = MASK_OP_RRR1_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR1_S1(ctx->opcode);
+    r2 = MASK_OP_RRR1_S2(ctx->opcode);
+    r3 = MASK_OP_RRR1_S3(ctx->opcode);
+    r4 = MASK_OP_RRR1_D(ctx->opcode);
+    n = MASK_OP_RRR1_N(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR1_MADDSU_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDSU_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDSU_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDSU_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsu_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDSUS_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDSUS_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDSUS_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDSUS_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsus_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDSUM_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDSUM_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDSUM_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDSUM_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsum_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDSUMS_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDSUMS_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDSUMS_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDSUMS_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_maddsums_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDSUR_H_16_LL:
+        gen_maddsur32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDSUR_H_16_LU:
+        gen_maddsur32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDSUR_H_16_UL:
+        gen_maddsur32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDSUR_H_16_UU:
+        gen_maddsur32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MADDSURS_H_16_LL:
+        gen_maddsur32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MADDSURS_H_16_LU:
+        gen_maddsur32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MADDSURS_H_16_UL:
+        gen_maddsur32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MADDSURS_H_16_UU:
+        gen_maddsur32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rrr1_msub(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4, n;
+
+    op2 = MASK_OP_RRR1_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR1_S1(ctx->opcode);
+    r2 = MASK_OP_RRR1_S2(ctx->opcode);
+    r3 = MASK_OP_RRR1_S3(ctx->opcode);
+    r4 = MASK_OP_RRR1_D(ctx->opcode);
+    n = MASK_OP_RRR1_N(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR1_MSUB_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUB_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUB_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUB_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msub_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                   cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubs_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBM_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBM_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBM_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBM_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                    cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBMS_H_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBMS_H_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBMS_H_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBMS_H_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBR_H_LL:
+        gen_msubr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBR_H_LU:
+        gen_msubr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBR_H_UL:
+        gen_msubr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBR_H_UU:
+        gen_msubr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBRS_H_LL:
+        gen_msubr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBRS_H_LU:
+        gen_msubr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBRS_H_UL:
+        gen_msubr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBRS_H_UU:
+        gen_msubr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                       cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+static void decode_rrr1_msubq_h(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4, n;
+    TCGv temp, temp2;
+
+    op2 = MASK_OP_RRR1_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR1_S1(ctx->opcode);
+    r2 = MASK_OP_RRR1_S2(ctx->opcode);
+    r3 = MASK_OP_RRR1_S3(ctx->opcode);
+    r4 = MASK_OP_RRR1_D(ctx->opcode);
+    n = MASK_OP_RRR1_N(ctx->opcode);
+
+    temp = tcg_const_i32(n);
+    temp2 = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RRR1_MSUB_Q_32:
+        gen_msub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                     cpu_gpr_d[r2], n, 32, env);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                     n, env);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_32_L:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_msub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                     temp, n, 16, env);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_msub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                     n, env);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_32_U:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_msub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                     temp, n, 16, env);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_msub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                     n, env);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16sub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16sub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16sub32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUB_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16sub64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_32:
+        gen_msubs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      cpu_gpr_d[r2], n, 32);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_64:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_32_L:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_msubs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      temp, n, 16);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_64_L:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r2]);
+        gen_msubs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                      n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_32_U:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_msubs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                      temp, n, 16);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_64_U:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r2], 16);
+        gen_msubs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], temp,
+                      n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16subs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_m16subs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                        cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16subs32_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBS_Q_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_m16subs64_q(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                        cpu_gpr_d[r3+1], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBR_H_64_UL:
+        CHECK_REG_PAIR(r3);
+        gen_msubr64_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
+                      cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
+        break;
+    case OPC2_32_RRR1_MSUBRS_H_64_UL:
+        CHECK_REG_PAIR(r3);
+        gen_msubr64s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r3+1],
+                       cpu_gpr_d[r1], cpu_gpr_d[r2], n, 2);
+        break;
+    case OPC2_32_RRR1_MSUBR_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_msubr_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBR_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_msubr_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBRS_Q_32_LL:
+        tcg_gen_ext16s_tl(temp, cpu_gpr_d[r1]);
+        tcg_gen_ext16s_tl(temp2, cpu_gpr_d[r2]);
+        gen_msubrs_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    case OPC2_32_RRR1_MSUBRS_Q_32_UU:
+        tcg_gen_sari_tl(temp, cpu_gpr_d[r1], 16);
+        tcg_gen_sari_tl(temp2, cpu_gpr_d[r2], 16);
+        gen_msubrs_q(cpu_gpr_d[r4], cpu_gpr_d[r3], temp, temp2, n);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+    tcg_temp_free(temp2);
+}
+
+static void decode_rrr1_msubad_h(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1, r2, r3, r4, n;
+
+    op2 = MASK_OP_RRR1_OP2(ctx->opcode);
+    r1 = MASK_OP_RRR1_S1(ctx->opcode);
+    r2 = MASK_OP_RRR1_S2(ctx->opcode);
+    r3 = MASK_OP_RRR1_S3(ctx->opcode);
+    r4 = MASK_OP_RRR1_D(ctx->opcode);
+    n = MASK_OP_RRR1_N(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_RRR1_MSUBAD_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBAD_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBAD_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBAD_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubad_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                     cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBADS_H_32_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBADS_H_32_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBADS_H_32_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBADS_H_32_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubads_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBADM_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBADM_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBADM_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBADM_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadm_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                      cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                      n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBADMS_H_64_LL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBADMS_H_64_LU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBADMS_H_64_UL:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBADMS_H_64_UU:
+        CHECK_REG_PAIR(r4);
+        CHECK_REG_PAIR(r3);
+        gen_msubadms_h(cpu_gpr_d[r4], cpu_gpr_d[r4+1], cpu_gpr_d[r3],
+                       cpu_gpr_d[r3+1], cpu_gpr_d[r1], cpu_gpr_d[r2],
+                       n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBADR_H_16_LL:
+        gen_msubadr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBADR_H_16_LU:
+        gen_msubadr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBADR_H_16_UL:
+        gen_msubadr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBADR_H_16_UU:
+        gen_msubadr32_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                        cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    case OPC2_32_RRR1_MSUBADRS_H_16_LL:
+        gen_msubadr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_LL);
+        break;
+    case OPC2_32_RRR1_MSUBADRS_H_16_LU:
+        gen_msubadr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_LU);
+        break;
+    case OPC2_32_RRR1_MSUBADRS_H_16_UL:
+        gen_msubadr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_UL);
+        break;
+    case OPC2_32_RRR1_MSUBADRS_H_16_UU:
+        gen_msubadr32s_h(cpu_gpr_d[r4], cpu_gpr_d[r3], cpu_gpr_d[r1],
+                         cpu_gpr_d[r2], n, MODE_UU);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
+/* RRRR format */
+static void decode_rrrr_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3, r4;
+    TCGv tmp_width, tmp_pos;
+
+    r1 = MASK_OP_RRRR_S1(ctx->opcode);
+    r2 = MASK_OP_RRRR_S2(ctx->opcode);
+    r3 = MASK_OP_RRRR_S3(ctx->opcode);
+    r4 = MASK_OP_RRRR_D(ctx->opcode);
+    op2 = MASK_OP_RRRR_OP2(ctx->opcode);
+
+    tmp_pos = tcg_temp_new();
+    tmp_width = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RRRR_DEXTR:
+        tcg_gen_andi_tl(tmp_pos, cpu_gpr_d[r3], 0x1f);
+        if (r1 == r2) {
+            tcg_gen_rotl_tl(cpu_gpr_d[r4], cpu_gpr_d[r1], tmp_pos);
+        } else {
+            tcg_gen_shl_tl(tmp_width, cpu_gpr_d[r1], tmp_pos);
+            tcg_gen_subfi_tl(tmp_pos, 32, tmp_pos);
+            tcg_gen_shr_tl(tmp_pos, cpu_gpr_d[r2], tmp_pos);
+            tcg_gen_or_tl(cpu_gpr_d[r4], tmp_width, tmp_pos);
+        }
+        break;
+    case OPC2_32_RRRR_EXTR:
+    case OPC2_32_RRRR_EXTR_U:
+        CHECK_REG_PAIR(r3);
+        tcg_gen_andi_tl(tmp_width, cpu_gpr_d[r3+1], 0x1f);
+        tcg_gen_andi_tl(tmp_pos, cpu_gpr_d[r3], 0x1f);
+        tcg_gen_add_tl(tmp_pos, tmp_pos, tmp_width);
+        tcg_gen_subfi_tl(tmp_pos, 32, tmp_pos);
+        tcg_gen_shl_tl(cpu_gpr_d[r4], cpu_gpr_d[r1], tmp_pos);
+        tcg_gen_subfi_tl(tmp_width, 32, tmp_width);
+        if (op2 == OPC2_32_RRRR_EXTR) {
+            tcg_gen_sar_tl(cpu_gpr_d[r4], cpu_gpr_d[r4], tmp_width);
+        } else {
+            tcg_gen_shr_tl(cpu_gpr_d[r4], cpu_gpr_d[r4], tmp_width);
+        }
+        break;
+    case OPC2_32_RRRR_INSERT:
+        CHECK_REG_PAIR(r3);
+        tcg_gen_andi_tl(tmp_width, cpu_gpr_d[r3+1], 0x1f);
+        tcg_gen_andi_tl(tmp_pos, cpu_gpr_d[r3], 0x1f);
+        gen_insert(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r2], tmp_width,
+                   tmp_pos);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(tmp_pos);
+    tcg_temp_free(tmp_width);
+}
+
+/* RRRW format */
+static void decode_rrrw_extract_insert(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    int r1, r2, r3, r4;
+    int32_t width;
+
+    TCGv temp, temp2;
+
+    op2 = MASK_OP_RRRW_OP2(ctx->opcode);
+    r1  = MASK_OP_RRRW_S1(ctx->opcode);
+    r2  = MASK_OP_RRRW_S2(ctx->opcode);
+    r3  = MASK_OP_RRRW_S3(ctx->opcode);
+    r4  = MASK_OP_RRRW_D(ctx->opcode);
+    width = MASK_OP_RRRW_WIDTH(ctx->opcode);
+
+    temp = tcg_temp_new();
+
+    switch (op2) {
+    case OPC2_32_RRRW_EXTR:
+        tcg_gen_andi_tl(temp, cpu_gpr_d[r3], 0x1f);
+        tcg_gen_addi_tl(temp, temp, width);
+        tcg_gen_subfi_tl(temp, 32, temp);
+        tcg_gen_shl_tl(cpu_gpr_d[r4], cpu_gpr_d[r1], temp);
+        tcg_gen_sari_tl(cpu_gpr_d[r4], cpu_gpr_d[r4], 32 - width);
+        break;
+    case OPC2_32_RRRW_EXTR_U:
+        if (width == 0) {
+            tcg_gen_movi_tl(cpu_gpr_d[r4], 0);
+        } else {
+            tcg_gen_andi_tl(temp, cpu_gpr_d[r3], 0x1f);
+            tcg_gen_shr_tl(cpu_gpr_d[r4], cpu_gpr_d[r1], temp);
+            tcg_gen_andi_tl(cpu_gpr_d[r4], cpu_gpr_d[r4], ~0u >> (32-width));
+        }
+        break;
+    case OPC2_32_RRRW_IMASK:
+        temp2 = tcg_temp_new();
+
+        tcg_gen_andi_tl(temp, cpu_gpr_d[r3], 0x1f);
+        tcg_gen_movi_tl(temp2, (1 << width) - 1);
+        tcg_gen_shl_tl(temp2, temp2, temp);
+        tcg_gen_shl_tl(cpu_gpr_d[r4], cpu_gpr_d[r2], temp);
+        tcg_gen_mov_tl(cpu_gpr_d[r4+1], temp2);
+
+        tcg_temp_free(temp2);
+        break;
+    case OPC2_32_RRRW_INSERT:
+        temp2 = tcg_temp_new();
+
+        tcg_gen_movi_tl(temp, width);
+        tcg_gen_andi_tl(temp2, cpu_gpr_d[r3], 0x1f);
+        gen_insert(cpu_gpr_d[r4], cpu_gpr_d[r1], cpu_gpr_d[r2], temp, temp2);
+
+        tcg_temp_free(temp2);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+    tcg_temp_free(temp);
+}
+
+/* SYS Format*/
+static void decode_sys_interrupts(CPUTriCoreState *env, DisasContext *ctx)
+{
+    uint32_t op2;
+    uint32_t r1;
+    TCGLabel *l1;
+    TCGv tmp;
+
+    op2 = MASK_OP_SYS_OP2(ctx->opcode);
+    r1  = MASK_OP_SYS_S1D(ctx->opcode);
+
+    switch (op2) {
+    case OPC2_32_SYS_DEBUG:
+        /* raise EXCP_DEBUG */
+        break;
+    case OPC2_32_SYS_DISABLE:
+        tcg_gen_andi_tl(cpu_ICR, cpu_ICR, ~MASK_ICR_IE);
+        break;
+    case OPC2_32_SYS_DSYNC:
+        break;
+    case OPC2_32_SYS_ENABLE:
+        tcg_gen_ori_tl(cpu_ICR, cpu_ICR, MASK_ICR_IE);
+        break;
+    case OPC2_32_SYS_ISYNC:
+        break;
+    case OPC2_32_SYS_NOP:
+        break;
+    case OPC2_32_SYS_RET:
+        gen_compute_branch(ctx, op2, 0, 0, 0, 0);
+        break;
+    case OPC2_32_SYS_FRET:
+        gen_fret(ctx);
+        break;
+    case OPC2_32_SYS_RFE:
+        gen_helper_rfe(cpu_env);
+        tcg_gen_exit_tb(0);
+        ctx->bstate = BS_BRANCH;
+        break;
+    case OPC2_32_SYS_RFM:
+        if ((ctx->hflags & TRICORE_HFLAG_KUU) == TRICORE_HFLAG_SM) {
+            tmp = tcg_temp_new();
+            l1 = gen_new_label();
+
+            tcg_gen_ld32u_tl(tmp, cpu_env, offsetof(CPUTriCoreState, DBGSR));
+            tcg_gen_andi_tl(tmp, tmp, MASK_DBGSR_DE);
+            tcg_gen_brcondi_tl(TCG_COND_NE, tmp, 1, l1);
+            gen_helper_rfm(cpu_env);
+            gen_set_label(l1);
+            tcg_gen_exit_tb(0);
+            ctx->bstate = BS_BRANCH;
+            tcg_temp_free(tmp);
+        } else {
+            /* generate privilege trap */
+        }
+        break;
+    case OPC2_32_SYS_RSLCX:
+        gen_helper_rslcx(cpu_env);
+        break;
+    case OPC2_32_SYS_SVLCX:
+        gen_helper_svlcx(cpu_env);
+        break;
+    case OPC2_32_SYS_RESTORE:
+        if (tricore_feature(env, TRICORE_FEATURE_16)) {
+            if ((ctx->hflags & TRICORE_HFLAG_KUU) == TRICORE_HFLAG_SM ||
+                (ctx->hflags & TRICORE_HFLAG_KUU) == TRICORE_HFLAG_UM1) {
+                tcg_gen_deposit_tl(cpu_ICR, cpu_ICR, cpu_gpr_d[r1], 8, 1);
+            } /* else raise privilege trap */
+        } else {
+            generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+        }
+        break;
+    case OPC2_32_SYS_TRAPSV:
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_GE, cpu_PSW_SV, 0, l1);
+        generate_trap(ctx, TRAPC_ASSERT, TIN5_SOVF);
+        gen_set_label(l1);
+        break;
+    case OPC2_32_SYS_TRAPV:
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_GE, cpu_PSW_V, 0, l1);
+        generate_trap(ctx, TRAPC_ASSERT, TIN5_OVF);
+        gen_set_label(l1);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
+    }
+}
+
 static void decode_32Bit_opc(CPUTriCoreState *env, DisasContext *ctx)
 {
     int op1;
-    int32_t r1;
-    int32_t address;
-    int8_t b;
+    int32_t r1, r2, r3;
+    int32_t address, const16;
+    int8_t b, const4;
     int32_t bpos;
-    TCGv temp, temp2;
+    TCGv temp, temp2, temp3;
 
     op1 = MASK_OP_MAJOR(ctx->opcode);
 
+    /* handle JNZ.T opcode only being 7 bit long */
+    if (unlikely((op1 & 0x7f) == OPCM_32_BRN_JTT)) {
+        op1 = OPCM_32_BRN_JTT;
+    }
+
     switch (op1) {
 /* ABS-format */
     case OPCM_32_ABS_LDW:
@@ -2357,11 +8460,13 @@
 /* B-format */
     case OPC1_32_B_CALL:
     case OPC1_32_B_CALLA:
+    case OPC1_32_B_FCALL:
+    case OPC1_32_B_FCALLA:
     case OPC1_32_B_J:
     case OPC1_32_B_JA:
     case OPC1_32_B_JL:
     case OPC1_32_B_JLA:
-        address = MASK_OP_B_DISP24(ctx->opcode);
+        address = MASK_OP_B_DISP24_SEXT(ctx->opcode);
         gen_compute_branch(ctx, op1, 0, 0, 0, address);
         break;
 /* Bit-format */
@@ -2405,6 +8510,213 @@
     case OPCM_32_BO_ADDRMODE_LDMST_BITREVERSE_CIRCULAR:
         decode_bo_addrmode_ldmst_bitreverse_circular(env, ctx);
         break;
+/* BOL-format */
+    case OPC1_32_BOL_LD_A_LONGOFF:
+    case OPC1_32_BOL_LD_W_LONGOFF:
+    case OPC1_32_BOL_LEA_LONGOFF:
+    case OPC1_32_BOL_ST_W_LONGOFF:
+    case OPC1_32_BOL_ST_A_LONGOFF:
+    case OPC1_32_BOL_LD_B_LONGOFF:
+    case OPC1_32_BOL_LD_BU_LONGOFF:
+    case OPC1_32_BOL_LD_H_LONGOFF:
+    case OPC1_32_BOL_LD_HU_LONGOFF:
+    case OPC1_32_BOL_ST_B_LONGOFF:
+    case OPC1_32_BOL_ST_H_LONGOFF:
+        decode_bol_opc(env, ctx, op1);
+        break;
+/* BRC Format */
+    case OPCM_32_BRC_EQ_NEQ:
+    case OPCM_32_BRC_GE:
+    case OPCM_32_BRC_JLT:
+    case OPCM_32_BRC_JNE:
+        const4 = MASK_OP_BRC_CONST4_SEXT(ctx->opcode);
+        address = MASK_OP_BRC_DISP15_SEXT(ctx->opcode);
+        r1 = MASK_OP_BRC_S1(ctx->opcode);
+        gen_compute_branch(ctx, op1, r1, 0, const4, address);
+        break;
+/* BRN Format */
+    case OPCM_32_BRN_JTT:
+        address = MASK_OP_BRN_DISP15_SEXT(ctx->opcode);
+        r1 = MASK_OP_BRN_S1(ctx->opcode);
+        gen_compute_branch(ctx, op1, r1, 0, 0, address);
+        break;
+/* BRR Format */
+    case OPCM_32_BRR_EQ_NEQ:
+    case OPCM_32_BRR_ADDR_EQ_NEQ:
+    case OPCM_32_BRR_GE:
+    case OPCM_32_BRR_JLT:
+    case OPCM_32_BRR_JNE:
+    case OPCM_32_BRR_JNZ:
+    case OPCM_32_BRR_LOOP:
+        address = MASK_OP_BRR_DISP15_SEXT(ctx->opcode);
+        r2 = MASK_OP_BRR_S2(ctx->opcode);
+        r1 = MASK_OP_BRR_S1(ctx->opcode);
+        gen_compute_branch(ctx, op1, r1, r2, 0, address);
+        break;
+/* RC Format */
+    case OPCM_32_RC_LOGICAL_SHIFT:
+        decode_rc_logical_shift(env, ctx);
+        break;
+    case OPCM_32_RC_ACCUMULATOR:
+        decode_rc_accumulator(env, ctx);
+        break;
+    case OPCM_32_RC_SERVICEROUTINE:
+        decode_rc_serviceroutine(env, ctx);
+        break;
+    case OPCM_32_RC_MUL:
+        decode_rc_mul(env, ctx);
+        break;
+/* RCPW Format */
+    case OPCM_32_RCPW_MASK_INSERT:
+        decode_rcpw_insert(env, ctx);
+        break;
+/* RCRR Format */
+    case OPC1_32_RCRR_INSERT:
+        r1 = MASK_OP_RCRR_S1(ctx->opcode);
+        r2 = MASK_OP_RCRR_S3(ctx->opcode);
+        r3 = MASK_OP_RCRR_D(ctx->opcode);
+        const16 = MASK_OP_RCRR_CONST4(ctx->opcode);
+        temp = tcg_const_i32(const16);
+        temp2 = tcg_temp_new(); /* width*/
+        temp3 = tcg_temp_new(); /* pos */
+
+        CHECK_REG_PAIR(r3);
+
+        tcg_gen_andi_tl(temp2, cpu_gpr_d[r3+1], 0x1f);
+        tcg_gen_andi_tl(temp3, cpu_gpr_d[r3], 0x1f);
+
+        gen_insert(cpu_gpr_d[r2], cpu_gpr_d[r1], temp, temp2, temp3);
+
+        tcg_temp_free(temp);
+        tcg_temp_free(temp2);
+        tcg_temp_free(temp3);
+        break;
+/* RCRW Format */
+    case OPCM_32_RCRW_MASK_INSERT:
+        decode_rcrw_insert(env, ctx);
+        break;
+/* RCR Format */
+    case OPCM_32_RCR_COND_SELECT:
+        decode_rcr_cond_select(env, ctx);
+        break;
+    case OPCM_32_RCR_MADD:
+        decode_rcr_madd(env, ctx);
+        break;
+    case OPCM_32_RCR_MSUB:
+        decode_rcr_msub(env, ctx);
+        break;
+/* RLC Format */
+    case OPC1_32_RLC_ADDI:
+    case OPC1_32_RLC_ADDIH:
+    case OPC1_32_RLC_ADDIH_A:
+    case OPC1_32_RLC_MFCR:
+    case OPC1_32_RLC_MOV:
+    case OPC1_32_RLC_MOV_64:
+    case OPC1_32_RLC_MOV_U:
+    case OPC1_32_RLC_MOV_H:
+    case OPC1_32_RLC_MOVH_A:
+    case OPC1_32_RLC_MTCR:
+        decode_rlc_opc(env, ctx, op1);
+        break;
+/* RR Format */
+    case OPCM_32_RR_ACCUMULATOR:
+        decode_rr_accumulator(env, ctx);
+        break;
+    case OPCM_32_RR_LOGICAL_SHIFT:
+        decode_rr_logical_shift(env, ctx);
+        break;
+    case OPCM_32_RR_ADDRESS:
+        decode_rr_address(env, ctx);
+        break;
+    case OPCM_32_RR_IDIRECT:
+        decode_rr_idirect(env, ctx);
+        break;
+    case OPCM_32_RR_DIVIDE:
+        decode_rr_divide(env, ctx);
+        break;
+/* RR1 Format */
+    case OPCM_32_RR1_MUL:
+        decode_rr1_mul(env, ctx);
+        break;
+    case OPCM_32_RR1_MULQ:
+        decode_rr1_mulq(env, ctx);
+        break;
+/* RR2 format */
+    case OPCM_32_RR2_MUL:
+        decode_rr2_mul(env, ctx);
+        break;
+/* RRPW format */
+    case OPCM_32_RRPW_EXTRACT_INSERT:
+        decode_rrpw_extract_insert(env, ctx);
+        break;
+    case OPC1_32_RRPW_DEXTR:
+        r1 = MASK_OP_RRPW_S1(ctx->opcode);
+        r2 = MASK_OP_RRPW_S2(ctx->opcode);
+        r3 = MASK_OP_RRPW_D(ctx->opcode);
+        const16 = MASK_OP_RRPW_POS(ctx->opcode);
+        if (r1 == r2) {
+            tcg_gen_rotli_tl(cpu_gpr_d[r3], cpu_gpr_d[r1], const16);
+        } else {
+            temp = tcg_temp_new();
+            tcg_gen_shli_tl(temp, cpu_gpr_d[r1], const16);
+            tcg_gen_shri_tl(cpu_gpr_d[r3], cpu_gpr_d[r2], 32 - const16);
+            tcg_gen_or_tl(cpu_gpr_d[r3], cpu_gpr_d[r3], temp);
+            tcg_temp_free(temp);
+        }
+        break;
+/* RRR Format */
+    case OPCM_32_RRR_COND_SELECT:
+        decode_rrr_cond_select(env, ctx);
+        break;
+    case OPCM_32_RRR_DIVIDE:
+        decode_rrr_divide(env, ctx);
+        break;
+/* RRR2 Format */
+    case OPCM_32_RRR2_MADD:
+        decode_rrr2_madd(env, ctx);
+        break;
+    case OPCM_32_RRR2_MSUB:
+        decode_rrr2_msub(env, ctx);
+        break;
+/* RRR1 format */
+    case OPCM_32_RRR1_MADD:
+        decode_rrr1_madd(env, ctx);
+        break;
+    case OPCM_32_RRR1_MADDQ_H:
+        decode_rrr1_maddq_h(env, ctx);
+        break;
+    case OPCM_32_RRR1_MADDSU_H:
+        decode_rrr1_maddsu_h(env, ctx);
+        break;
+    case OPCM_32_RRR1_MSUB_H:
+        decode_rrr1_msub(env, ctx);
+        break;
+    case OPCM_32_RRR1_MSUB_Q:
+        decode_rrr1_msubq_h(env, ctx);
+        break;
+    case OPCM_32_RRR1_MSUBAD_H:
+        decode_rrr1_msubad_h(env, ctx);
+        break;
+/* RRRR format */
+    case OPCM_32_RRRR_EXTRACT_INSERT:
+        decode_rrrr_extract_insert(env, ctx);
+        break;
+/* RRRW format */
+    case OPCM_32_RRRW_EXTRACT_INSERT:
+        decode_rrrw_extract_insert(env, ctx);
+        break;
+/* SYS format */
+    case OPCM_32_SYS_INTERRUPTS:
+        decode_sys_interrupts(env, ctx);
+        break;
+    case OPC1_32_SYS_RSTV:
+        tcg_gen_movi_tl(cpu_PSW_V, 0);
+        tcg_gen_mov_tl(cpu_PSW_SV, cpu_PSW_V);
+        tcg_gen_mov_tl(cpu_PSW_AV, cpu_PSW_V);
+        tcg_gen_mov_tl(cpu_PSW_SAV, cpu_PSW_V);
+        break;
+    default:
+        generate_trap(ctx, TRAPC_INSN_ERR, TIN2_IOPC);
     }
 }
 
@@ -2421,43 +8733,44 @@
     }
 }
 
-static inline void
-gen_intermediate_code_internal(TriCoreCPU *cpu, struct TranslationBlock *tb,
-                              int search_pc)
+void gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
 {
+    TriCoreCPU *cpu = tricore_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUTriCoreState *env = &cpu->env;
     DisasContext ctx;
     target_ulong pc_start;
-    int num_insns;
-
-    if (search_pc) {
-        qemu_log("search pc %d\n", search_pc);
-    }
+    int num_insns, max_insns;
 
     num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+    if (singlestep) {
+        max_insns = 1;
+    }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
+
     pc_start = tb->pc;
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.tb = tb;
     ctx.singlestep_enabled = cs->singlestep_enabled;
     ctx.bstate = BS_NONE;
-    ctx.mem_idx = cpu_mmu_index(env);
+    ctx.mem_idx = cpu_mmu_index(env, false);
 
     tcg_clear_temp_count();
-    gen_tb_start();
+    gen_tb_start(tb);
     while (ctx.bstate == BS_NONE) {
+        tcg_gen_insn_start(ctx.pc);
+        num_insns++;
+
         ctx.opcode = cpu_ldl_code(env, ctx.pc);
         decode_opc(env, &ctx, 0);
 
-        num_insns++;
-
-        if (tcg_op_buf_full()) {
-            gen_save_pc(ctx.next_pc);
-            tcg_gen_exit_tb(0);
-            break;
-        }
-        if (singlestep) {
+        if (num_insns >= max_insns || tcg_op_buf_full()) {
             gen_save_pc(ctx.next_pc);
             tcg_gen_exit_tb(0);
             break;
@@ -2466,41 +8779,28 @@
     }
 
     gen_tb_end(tb, num_insns);
-    if (search_pc) {
-        printf("done_generating search pc\n");
-    } else {
-        tb->size = ctx.pc - pc_start;
-        tb->icount = num_insns;
-    }
+    tb->size = ctx.pc - pc_start;
+    tb->icount = num_insns;
+
     if (tcg_check_temp_count()) {
         printf("LEAK at %08x\n", env->PC);
     }
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(cs, pc_start, ctx.pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
 }
 
 void
-gen_intermediate_code(CPUTriCoreState *env, struct TranslationBlock *tb)
+restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb,
+                     target_ulong *data)
 {
-    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, false);
-}
-
-void
-gen_intermediate_code_pc(CPUTriCoreState *env, struct TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(tricore_env_get_cpu(env), tb, true);
-}
-
-void
-restore_state_to_opc(CPUTriCoreState *env, TranslationBlock *tb, int pc_pos)
-{
-    env->PC = tcg_ctx.gen_opc_pc[pc_pos];
+    env->PC = data[0];
 }
 /*
  *
@@ -2512,17 +8812,18 @@
 {
     /* Reset Regs to Default Value */
     env->PSW = 0xb80;
+    fpu_set_state(env);
 }
 
 static void tricore_tcg_init_csfr(void)
 {
-    cpu_PCXI = tcg_global_mem_new(TCG_AREG0,
+    cpu_PCXI = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, PCXI), "PCXI");
-    cpu_PSW = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, PSW), "PSW");
-    cpu_PC = tcg_global_mem_new(TCG_AREG0,
+    cpu_PC = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, PC), "PC");
-    cpu_ICR = tcg_global_mem_new(TCG_AREG0,
+    cpu_ICR = tcg_global_mem_new(cpu_env,
                           offsetof(CPUTriCoreState, ICR), "ICR");
 }
 
@@ -2534,32 +8835,33 @@
         return;
     }
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
     /* reg init */
     for (i = 0 ; i < 16 ; i++) {
-        cpu_gpr_a[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr_a[i] = tcg_global_mem_new(cpu_env,
                                           offsetof(CPUTriCoreState, gpr_a[i]),
                                           regnames_a[i]);
     }
     for (i = 0 ; i < 16 ; i++) {
-        cpu_gpr_d[i] = tcg_global_mem_new(TCG_AREG0,
+        cpu_gpr_d[i] = tcg_global_mem_new(cpu_env,
                                   offsetof(CPUTriCoreState, gpr_d[i]),
                                            regnames_d[i]);
     }
     tricore_tcg_init_csfr();
     /* init PSW flag cache */
-    cpu_PSW_C = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_C = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUTriCoreState, PSW_USB_C),
                                    "PSW_C");
-    cpu_PSW_V = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_V = tcg_global_mem_new(cpu_env,
                                    offsetof(CPUTriCoreState, PSW_USB_V),
                                    "PSW_V");
-    cpu_PSW_SV = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_SV = tcg_global_mem_new(cpu_env,
                                     offsetof(CPUTriCoreState, PSW_USB_SV),
                                     "PSW_SV");
-    cpu_PSW_AV = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_AV = tcg_global_mem_new(cpu_env,
                                     offsetof(CPUTriCoreState, PSW_USB_AV),
                                     "PSW_AV");
-    cpu_PSW_SAV = tcg_global_mem_new(TCG_AREG0,
+    cpu_PSW_SAV = tcg_global_mem_new(cpu_env,
                                      offsetof(CPUTriCoreState, PSW_USB_SAV),
                                      "PSW_SAV");
 }
diff --git a/target-tricore/tricore-defs.h b/target-tricore/tricore-defs.h
index 4350b03..40abfaa 100644
--- a/target-tricore/tricore-defs.h
+++ b/target-tricore/tricore-defs.h
@@ -15,8 +15,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#if !defined(__QEMU_TRICORE_DEFS_H__)
-#define __QEMU_TRICORE_DEFS_H__
+#ifndef QEMU_TRICORE_DEFS_H
+#define QEMU_TRICORE_DEFS_H
 
 #define TARGET_PAGE_BITS 14
 #define TARGET_LONG_BITS 32
@@ -25,4 +25,4 @@
 
 #define TRICORE_TLB_MAX 128
 
-#endif /* __QEMU_TRICORE_DEFS_H__ */
+#endif /* QEMU_TRICORE_DEFS_H */
diff --git a/target-tricore/tricore-opcodes.h b/target-tricore/tricore-opcodes.h
index 7e6f33b..df666b0 100644
--- a/target-tricore/tricore-opcodes.h
+++ b/target-tricore/tricore-opcodes.h
@@ -94,6 +94,8 @@
 /* B Format   */
 #define MASK_OP_B_DISP24(op)   (MASK_BITS_SHIFT(op, 16, 31) + \
                                (MASK_BITS_SHIFT(op, 8, 15) << 16))
+#define MASK_OP_B_DISP24_SEXT(op)   (MASK_BITS_SHIFT(op, 16, 31) + \
+                                    (MASK_BITS_SHIFT_SEXT(op, 8, 15) << 16))
 /* BIT Format */
 #define MASK_OP_BIT_D(op)      MASK_BITS_SHIFT(op, 28, 31)
 #define MASK_OP_BIT_POS2(op)   MASK_BITS_SHIFT(op, 23, 27)
@@ -105,7 +107,7 @@
 /* BO Format */
 #define MASK_OP_BO_OFF10(op)   (MASK_BITS_SHIFT(op, 16, 21) + \
                                (MASK_BITS_SHIFT(op, 28, 31) << 6))
-#define MASK_OP_BO_OFF10_SEXT(op)   (MASK_BITS_SHIFT_SEXT(op, 16, 21) + \
+#define MASK_OP_BO_OFF10_SEXT(op)   (MASK_BITS_SHIFT(op, 16, 21) + \
                                     (MASK_BITS_SHIFT_SEXT(op, 28, 31) << 6))
 #define MASK_OP_BO_OP2(op)     MASK_BITS_SHIFT(op, 22, 27)
 #define MASK_OP_BO_S2(op)      MASK_BITS_SHIFT(op, 12, 15)
@@ -114,26 +116,32 @@
 /* BOL Format */
 #define MASK_OP_BOL_OFF16(op)  ((MASK_BITS_SHIFT(op, 16, 21) +        \
                                (MASK_BITS_SHIFT(op, 28, 31) << 6)) + \
-                               (MASK_BITS_SHIFT(op, 22, 27) >> 10))
-
+                               (MASK_BITS_SHIFT(op, 22, 27) << 10))
+#define MASK_OP_BOL_OFF16_SEXT(op)  ((MASK_BITS_SHIFT(op, 16, 21) +        \
+                                    (MASK_BITS_SHIFT(op, 28, 31) << 6)) + \
+                                    (MASK_BITS_SHIFT_SEXT(op, 22, 27) << 10))
 #define MASK_OP_BOL_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
 #define MASK_OP_BOL_S1D(op)    MASK_BITS_SHIFT(op, 8, 11)
 
 /* BRC Format */
 #define MASK_OP_BRC_OP2(op)    MASK_BITS_SHIFT(op, 31, 31)
 #define MASK_OP_BRC_DISP15(op) MASK_BITS_SHIFT(op, 16, 30)
+#define MASK_OP_BRC_DISP15_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 16, 30)
 #define MASK_OP_BRC_CONST4(op) MASK_BITS_SHIFT(op, 12, 15)
+#define MASK_OP_BRC_CONST4_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 12, 15)
 #define MASK_OP_BRC_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
 
 /* BRN Format */
 #define MASK_OP_BRN_OP2(op)    MASK_BITS_SHIFT(op, 31, 31)
 #define MASK_OP_BRN_DISP15(op) MASK_BITS_SHIFT(op, 16, 30)
+#define MASK_OP_BRN_DISP15_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 16, 30)
 #define MASK_OP_BRN_N(op)      (MASK_BITS_SHIFT(op, 12, 15) + \
                                (MASK_BITS_SHIFT(op, 7, 7) << 4))
 #define MASK_OP_BRN_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
 /* BRR Format */
 #define MASK_OP_BRR_OP2(op)    MASK_BITS_SHIFT(op, 31, 31)
 #define MASK_OP_BRR_DISP15(op) MASK_BITS_SHIFT(op, 16, 30)
+#define MASK_OP_BRR_DISP15_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 16, 30)
 #define MASK_OP_BRR_S2(op)     MASK_BITS_SHIFT(op, 12, 15)
 #define MASK_OP_BRR_S1(op)     MASK_BITS_SHIFT(op, 8, 11)
 
@@ -145,6 +153,7 @@
 #define MASK_OP_RC_D(op)       MASK_OP_META_D(op)
 #define MASK_OP_RC_OP2(op)     MASK_BITS_SHIFT(op, 21, 27)
 #define MASK_OP_RC_CONST9(op)  MASK_BITS_SHIFT(op, 12, 20)
+#define MASK_OP_RC_CONST9_SEXT(op)  MASK_BITS_SHIFT_SEXT(op, 12, 20)
 #define MASK_OP_RC_S1(op)      MASK_OP_META_S1(op)
 
 /* RCPW Format */
@@ -162,6 +171,7 @@
 #define MASK_OP_RCR_S3(op)     MASK_BITS_SHIFT(op, 24, 27)
 #define MASK_OP_RCR_OP2(op)    MASK_BITS_SHIFT(op, 21, 23)
 #define MASK_OP_RCR_CONST9(op) MASK_BITS_SHIFT(op, 12, 20)
+#define MASK_OP_RCR_CONST9_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 12, 20)
 #define MASK_OP_RCR_S1(op)     MASK_OP_META_S1(op)
 
 /* RCRR Format */
@@ -185,6 +195,7 @@
 
 #define MASK_OP_RLC_D(op)       MASK_OP_META_D(op)
 #define MASK_OP_RLC_CONST16(op) MASK_BITS_SHIFT(op, 12, 27)
+#define MASK_OP_RLC_CONST16_SEXT(op) MASK_BITS_SHIFT_SEXT(op, 12, 27)
 #define MASK_OP_RLC_S1(op)      MASK_OP_META_S1(op)
 
 /* RR  Format */
@@ -388,6 +399,7 @@
     OPC2_16_SR_RET                                   = 0x09,
     OPC2_16_SR_RFE                                   = 0x08,
     OPC2_16_SR_DEBUG                                 = 0x0a,
+    OPC2_16_SR_FRET                                  = 0x07,
 };
 /* OPCM_16_SR_ACCU                                   */
 enum {
@@ -417,6 +429,8 @@
 /* B Format */
     OPC1_32_B_CALL                                   = 0x6d,
     OPC1_32_B_CALLA                                  = 0xed,
+    OPC1_32_B_FCALL                                  = 0x61,
+    OPC1_32_B_FCALLA                                 = 0xe1,
     OPC1_32_B_J                                      = 0x1d,
     OPC1_32_B_JA                                     = 0x9d,
     OPC1_32_B_JL                                     = 0x5d,
@@ -438,10 +452,16 @@
     OPCM_32_BO_ADDRMODE_LDMST_BITREVERSE_CIRCULAR    = 0x69,
 /* BOL Format */
     OPC1_32_BOL_LD_A_LONGOFF                         = 0x99,
-    OPC1_32_BOL_LD_W_LONFOFF                         = 0x19,
+    OPC1_32_BOL_LD_W_LONGOFF                         = 0x19,
     OPC1_32_BOL_LEA_LONGOFF                          = 0xd9,
     OPC1_32_BOL_ST_W_LONGOFF                         = 0x59,
     OPC1_32_BOL_ST_A_LONGOFF                         = 0xb5, /* 1.6 only */
+    OPC1_32_BOL_LD_B_LONGOFF                         = 0x79, /* 1.6 only */
+    OPC1_32_BOL_LD_BU_LONGOFF                        = 0x39, /* 1.6 only */
+    OPC1_32_BOL_LD_H_LONGOFF                         = 0xc9, /* 1.6 only */
+    OPC1_32_BOL_LD_HU_LONGOFF                        = 0xb9, /* 1.6 only */
+    OPC1_32_BOL_ST_B_LONGOFF                         = 0xe9, /* 1.6 only */
+    OPC1_32_BOL_ST_H_LONGOFF                         = 0xf9, /* 1.6 only */
 /* BRC Format */
     OPCM_32_BRC_EQ_NEQ                               = 0xdf,
     OPCM_32_BRC_GE                                   = 0xff,
@@ -478,6 +498,7 @@
     OPC1_32_RLC_ADDIH_A                              = 0x11,
     OPC1_32_RLC_MFCR                                 = 0x4d,
     OPC1_32_RLC_MOV                                  = 0x3b,
+    OPC1_32_RLC_MOV_64                               = 0xfb, /* 1.6 only */
     OPC1_32_RLC_MOV_U                                = 0xbb,
     OPC1_32_RLC_MOV_H                                = 0x7b,
     OPC1_32_RLC_MOVH_A                               = 0x91,
@@ -485,8 +506,8 @@
 /* RR Format */
     OPCM_32_RR_LOGICAL_SHIFT                         = 0x0f,
     OPCM_32_RR_ACCUMULATOR                           = 0x0b,
-    OPCM_32_RR_ADRESS                                = 0x01,
-    OPCM_32_RR_FLOAT                                 = 0x4b,
+    OPCM_32_RR_ADDRESS                               = 0x01,
+    OPCM_32_RR_DIVIDE                                = 0x4b,
     OPCM_32_RR_IDIRECT                               = 0x2d,
 /* RR1 Format */
     OPCM_32_RR1_MUL                                  = 0xb3,
@@ -498,14 +519,14 @@
     OPC1_32_RRPW_DEXTR                               = 0x77,
 /* RRR Format */
     OPCM_32_RRR_COND_SELECT                          = 0x2b,
-    OPCM_32_RRR_FLOAT                                = 0x6b,
+    OPCM_32_RRR_DIVIDE                               = 0x6b,
 /* RRR1 Format */
     OPCM_32_RRR1_MADD                                = 0x83,
     OPCM_32_RRR1_MADDQ_H                             = 0x43,
     OPCM_32_RRR1_MADDSU_H                            = 0xc3,
     OPCM_32_RRR1_MSUB_H                              = 0xa3,
     OPCM_32_RRR1_MSUB_Q                              = 0x63,
-    OPCM_32_RRR1_MSUBADS_H                           = 0xe3,
+    OPCM_32_RRR1_MSUBAD_H                            = 0xe3,
 /* RRR2 Format */
     OPCM_32_RRR2_MADD                                = 0x03,
     OPCM_32_RRR2_MSUB                                = 0x23,
@@ -745,6 +766,12 @@
     OPC2_32_BO_SWAP_W_SHORTOFF                   = 0x20,
     OPC2_32_BO_SWAP_W_POSTINC                    = 0x00,
     OPC2_32_BO_SWAP_W_PREINC                     = 0x10,
+    OPC2_32_BO_CMPSWAP_W_SHORTOFF                = 0x23,
+    OPC2_32_BO_CMPSWAP_W_POSTINC                 = 0x03,
+    OPC2_32_BO_CMPSWAP_W_PREINC                  = 0x13,
+    OPC2_32_BO_SWAPMSK_W_SHORTOFF                = 0x22,
+    OPC2_32_BO_SWAPMSK_W_POSTINC                 = 0x02,
+    OPC2_32_BO_SWAPMSK_W_PREINC                  = 0x12,
 };
 /*OPCM_32_BO_ADDRMODE_LDMST_BITREVERSE_CIRCULAR  */
 enum {
@@ -752,6 +779,10 @@
     OPC2_32_BO_LDMST_CIRC                        = 0x11,
     OPC2_32_BO_SWAP_W_BR                         = 0x00,
     OPC2_32_BO_SWAP_W_CIRC                       = 0x10,
+    OPC2_32_BO_CMPSWAP_W_BR                      = 0x03,
+    OPC2_32_BO_CMPSWAP_W_CIRC                    = 0x13,
+    OPC2_32_BO_SWAPMSK_W_BR                      = 0x02,
+    OPC2_32_BO_SWAPMSK_W_CIRC                    = 0x12,
 };
 /*
  * BRC Format
@@ -763,8 +794,8 @@
 };
 /* OPCM_32_BRC_GE                                   */
 enum {
-    OP2_BRC_JGE                                  = 0x00,
-    OPC_BRC_JGE_U                                = 0x01,
+    OP2_32_BRC_JGE                               = 0x00,
+    OPC_32_BRC_JGE_U                             = 0x01,
 };
 /* OPCM_32_BRC_JLT                                  */
 enum {
@@ -937,7 +968,7 @@
     OPC2_32_RCR_MSUB_64                          = 0x03,
     OPC2_32_RCR_MSUBS_32                         = 0x05,
     OPC2_32_RCR_MSUBS_64                         = 0x07,
-    OPC2_32_RCR_MSUB_U_32                        = 0x02,
+    OPC2_32_RCR_MSUB_U_64                        = 0x02,
     OPC2_32_RCR_MSUBS_U_32                       = 0x04,
     OPC2_32_RCR_MSUBS_U_64                       = 0x06,
 };
@@ -1024,8 +1055,8 @@
     OPC2_32_RR_MAX_BU                            = 0x5b,
     OPC2_32_RR_MAX_H                             = 0x7a,
     OPC2_32_RR_MAX_HU                            = 0x7b,
-    OPC2_32_RR_MIN                               = 0x19,
-    OPC2_32_RR_MIN_U                             = 0x18,
+    OPC2_32_RR_MIN                               = 0x18,
+    OPC2_32_RR_MIN_U                             = 0x19,
     OPC2_32_RR_MIN_B                             = 0x58,
     OPC2_32_RR_MIN_BU                            = 0x59,
     OPC2_32_RR_MIN_H                             = 0x78,
@@ -1064,7 +1095,7 @@
     OPC2_32_RR_XOR_LT_U                          = 0x32,
     OPC2_32_RR_XOR_NE                            = 0x30,
 };
-/* OPCM_32_RR_ADRESS                                */
+/* OPCM_32_RR_ADDRESS                               */
 enum {
     OPC2_32_RR_ADD_A                             = 0x01,
     OPC2_32_RR_ADDSC_A                           = 0x60,
@@ -1092,12 +1123,30 @@
     OPC2_32_RR_DVINIT_U                          = 0x0a,
     OPC2_32_RR_PARITY                            = 0x02,
     OPC2_32_RR_UNPACK                            = 0x08,
+    OPC2_32_RR_CRC32                             = 0x03,
+    OPC2_32_RR_DIV                               = 0x20,
+    OPC2_32_RR_DIV_U                             = 0x21,
+    OPC2_32_RR_MUL_F                             = 0x04,
+    OPC2_32_RR_DIV_F                             = 0x05,
+    OPC2_32_RR_FTOI                              = 0x10,
+    OPC2_32_RR_ITOF                              = 0x14,
+    OPC2_32_RR_CMP_F                             = 0x00,
+    OPC2_32_RR_FTOIZ                             = 0x13,
+    OPC2_32_RR_FTOQ31                            = 0x11,
+    OPC2_32_RR_FTOQ31Z                           = 0x18,
+    OPC2_32_RR_FTOU                              = 0x12,
+    OPC2_32_RR_FTOUZ                             = 0x17,
+    OPC2_32_RR_Q31TOF                            = 0x15,
+    OPC2_32_RR_QSEED_F                           = 0x19,
+    OPC2_32_RR_UPDFL                             = 0x0c,
+    OPC2_32_RR_UTOF                              = 0x16,
 };
 /* OPCM_32_RR_IDIRECT                               */
 enum {
     OPC2_32_RR_JI                                = 0x03,
     OPC2_32_RR_JLI                               = 0x02,
     OPC2_32_RR_CALLI                             = 0x00,
+    OPC2_32_RR_FCALLI                            = 0x01,
 };
 /*
  * RR1 Format
@@ -1174,6 +1223,10 @@
     OPC2_32_RRR_IXMIN                            = 0x08,
     OPC2_32_RRR_IXMIN_U                          = 0x09,
     OPC2_32_RRR_PACK                             = 0x00,
+    OPC2_32_RRR_ADD_F                            = 0x02,
+    OPC2_32_RRR_SUB_F                            = 0x03,
+    OPC2_32_RRR_MADD_F                           = 0x06,
+    OPC2_32_RRR_MSUB_F                           = 0x07,
 };
 /*
  * RRR1 Format
@@ -1227,10 +1280,10 @@
     OPC2_32_RRR1_MADDS_Q_64_LL                   = 0x3d,
     OPC2_32_RRR1_MADDS_Q_32_UU                   = 0x24,
     OPC2_32_RRR1_MADDS_Q_64_UU                   = 0x3c,
-    OPC2_32_RRR1_MADDR_H_16_UL                   = 0x1e,
-    OPC2_32_RRR1_MADDRS_H_16_UL                  = 0x3e,
-    OPC2_32_RRR1_MADDR_Q_32_L                    = 0x07,
-    OPC2_32_RRR1_MADDR_Q_32_U                    = 0x06,
+    OPC2_32_RRR1_MADDR_H_64_UL                   = 0x1e,
+    OPC2_32_RRR1_MADDRS_H_64_UL                  = 0x3e,
+    OPC2_32_RRR1_MADDR_Q_32_LL                   = 0x07,
+    OPC2_32_RRR1_MADDR_Q_32_UU                   = 0x06,
     OPC2_32_RRR1_MADDRS_Q_32_LL                  = 0x27,
     OPC2_32_RRR1_MADDRS_Q_32_UU                  = 0x26,
 };
@@ -1263,30 +1316,30 @@
 };
 /* OPCM_32_RRR1_MSUB_H                              */
 enum {
-    OPC2_32_RRR1_MSUB_H_32_LL                    = 0x1a,
-    OPC2_32_RRR1_MSUB_H_32_LU                    = 0x19,
-    OPC2_32_RRR1_MSUB_H_32_UL                    = 0x18,
-    OPC2_32_RRR1_MSUB_H_32_UU                    = 0x1b,
-    OPC2_32_RRR1_MSUBS_H_32_LL                   = 0x3a,
-    OPC2_32_RRR1_MSUBS_H_32_LU                   = 0x39,
-    OPC2_32_RRR1_MSUBS_H_32_UL                   = 0x38,
-    OPC2_32_RRR1_MSUBS_H_32_UU                   = 0x3b,
-    OPC2_32_RRR1_MSUBM_H_64_LL                   = 0x1e,
-    OPC2_32_RRR1_MSUBM_H_64_LU                   = 0x1d,
-    OPC2_32_RRR1_MSUBM_H_64_UL                   = 0x1c,
-    OPC2_32_RRR1_MSUBM_H_64_UU                   = 0x1f,
-    OPC2_32_RRR1_MSUBMS_H_64_LL                  = 0x3e,
-    OPC2_32_RRR1_MSUBMS_H_64_LU                  = 0x3d,
-    OPC2_32_RRR1_MSUBMS_H_64_UL                  = 0x3c,
-    OPC2_32_RRR1_MSUBMS_H_64_UU                  = 0x3f,
-    OPC2_32_RRR1_MSUBR_H_16_LL                   = 0x0e,
-    OPC2_32_RRR1_MSUBR_H_16_LU                   = 0x0d,
-    OPC2_32_RRR1_MSUBR_H_16_UL                   = 0x0c,
-    OPC2_32_RRR1_MSUBR_H_16_UU                   = 0x0f,
-    OPC2_32_RRR1_MSUBRS_H_16_LL                  = 0x2e,
-    OPC2_32_RRR1_MSUBRS_H_16_LU                  = 0x2d,
-    OPC2_32_RRR1_MSUBRS_H_16_UL                  = 0x2c,
-    OPC2_32_RRR1_MSUBRS_H_16_UU                  = 0x2f,
+    OPC2_32_RRR1_MSUB_H_LL                       = 0x1a,
+    OPC2_32_RRR1_MSUB_H_LU                       = 0x19,
+    OPC2_32_RRR1_MSUB_H_UL                       = 0x18,
+    OPC2_32_RRR1_MSUB_H_UU                       = 0x1b,
+    OPC2_32_RRR1_MSUBS_H_LL                      = 0x3a,
+    OPC2_32_RRR1_MSUBS_H_LU                      = 0x39,
+    OPC2_32_RRR1_MSUBS_H_UL                      = 0x38,
+    OPC2_32_RRR1_MSUBS_H_UU                      = 0x3b,
+    OPC2_32_RRR1_MSUBM_H_LL                      = 0x1e,
+    OPC2_32_RRR1_MSUBM_H_LU                      = 0x1d,
+    OPC2_32_RRR1_MSUBM_H_UL                      = 0x1c,
+    OPC2_32_RRR1_MSUBM_H_UU                      = 0x1f,
+    OPC2_32_RRR1_MSUBMS_H_LL                     = 0x3e,
+    OPC2_32_RRR1_MSUBMS_H_LU                     = 0x3d,
+    OPC2_32_RRR1_MSUBMS_H_UL                     = 0x3c,
+    OPC2_32_RRR1_MSUBMS_H_UU                     = 0x3f,
+    OPC2_32_RRR1_MSUBR_H_LL                      = 0x0e,
+    OPC2_32_RRR1_MSUBR_H_LU                      = 0x0d,
+    OPC2_32_RRR1_MSUBR_H_UL                      = 0x0c,
+    OPC2_32_RRR1_MSUBR_H_UU                      = 0x0f,
+    OPC2_32_RRR1_MSUBRS_H_LL                     = 0x2e,
+    OPC2_32_RRR1_MSUBRS_H_LU                     = 0x2d,
+    OPC2_32_RRR1_MSUBRS_H_UL                     = 0x2c,
+    OPC2_32_RRR1_MSUBRS_H_UU                     = 0x2f,
 };
 /* OPCM_32_RRR1_MSUB_Q                              */
 enum {
@@ -1310,8 +1363,8 @@
     OPC2_32_RRR1_MSUBS_Q_64_LL                   = 0x3d,
     OPC2_32_RRR1_MSUBS_Q_32_UU                   = 0x24,
     OPC2_32_RRR1_MSUBS_Q_64_UU                   = 0x3c,
-    OPC2_32_RRR1_MSUBR_H_32_UL                   = 0x1e,
-    OPC2_32_RRR1_MSUBRS_H_32_UL                  = 0x3e,
+    OPC2_32_RRR1_MSUBR_H_64_UL                   = 0x1e,
+    OPC2_32_RRR1_MSUBRS_H_64_UL                  = 0x3e,
     OPC2_32_RRR1_MSUBR_Q_32_LL                   = 0x07,
     OPC2_32_RRR1_MSUBR_Q_32_UU                   = 0x06,
     OPC2_32_RRR1_MSUBRS_Q_32_LL                  = 0x27,
@@ -1334,7 +1387,7 @@
     OPC2_32_RRR1_MSUBADMS_H_64_LL                = 0x3e,
     OPC2_32_RRR1_MSUBADMS_H_64_LU                = 0x3d,
     OPC2_32_RRR1_MSUBADMS_H_64_UL                = 0x3c,
-    OPC2_32_RRR1_MSUBADMS_H_16_UU                = 0x3f,
+    OPC2_32_RRR1_MSUBADMS_H_64_UU                = 0x3f,
     OPC2_32_RRR1_MSUBADR_H_16_LL                 = 0x0e,
     OPC2_32_RRR1_MSUBADR_H_16_LU                 = 0x0d,
     OPC2_32_RRR1_MSUBADR_H_16_UL                 = 0x0c,
@@ -1353,7 +1406,7 @@
     OPC2_32_RRR2_MADD_64                         = 0x6a,
     OPC2_32_RRR2_MADDS_32                        = 0x8a,
     OPC2_32_RRR2_MADDS_64                        = 0xea,
-    OPC2_32_RRR2_MADD_U_32                       = 0x68,
+    OPC2_32_RRR2_MADD_U_64                       = 0x68,
     OPC2_32_RRR2_MADDS_U_32                      = 0x88,
     OPC2_32_RRR2_MADDS_U_64                      = 0xe8,
 };
@@ -1405,4 +1458,6 @@
     OPC2_32_SYS_SVLCX                            = 0x08,
     OPC2_32_SYS_TRAPSV                           = 0x15,
     OPC2_32_SYS_TRAPV                            = 0x14,
+    OPC2_32_SYS_RESTORE                          = 0x0e,
+    OPC2_32_SYS_FRET                             = 0x03,
 };
diff --git a/target-unicore32/cpu-qom.h b/target-unicore32/cpu-qom.h
index ea65b83..bc68e78 100644
--- a/target-unicore32/cpu-qom.h
+++ b/target-unicore32/cpu-qom.h
@@ -12,7 +12,6 @@
 #define QEMU_UC32_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #define TYPE_UNICORE32_CPU "unicore32-cpu"
 
@@ -37,33 +36,6 @@
     DeviceRealize parent_realize;
 } UniCore32CPUClass;
 
-/**
- * UniCore32CPU:
- * @env: #CPUUniCore32State
- *
- * A UniCore32 CPU.
- */
-typedef struct UniCore32CPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUUniCore32State env;
-} UniCore32CPU;
-
-static inline UniCore32CPU *uc32_env_get_cpu(CPUUniCore32State *env)
-{
-    return container_of(env, UniCore32CPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(uc32_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(UniCore32CPU, env)
-
-void uc32_cpu_do_interrupt(CPUState *cpu);
-bool uc32_cpu_exec_interrupt(CPUState *cpu, int int_req);
-void uc32_cpu_dump_state(CPUState *cpu, FILE *f,
-                         fprintf_function cpu_fprintf, int flags);
-hwaddr uc32_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+typedef struct UniCore32CPU UniCore32CPU;
 
 #endif
diff --git a/target-unicore32/cpu.c b/target-unicore32/cpu.c
index 5b32987..e7a4984 100644
--- a/target-unicore32/cpu.c
+++ b/target-unicore32/cpu.c
@@ -12,9 +12,12 @@
  * or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
+#include "exec/exec-all.h"
 
 static void uc32_cpu_set_pc(CPUState *cs, vaddr value)
 {
@@ -75,6 +78,7 @@
 
     set_feature(env, UC32_HWCAP_CMOV);
     set_feature(env, UC32_HWCAP_UCF64);
+    set_snan_bit_is_one(1, &env->ucf64.fp_status);
 }
 
 static void uc32_any_cpu_initfn(Object *obj)
@@ -87,6 +91,7 @@
 
     set_feature(env, UC32_HWCAP_CMOV);
     set_feature(env, UC32_HWCAP_UCF64);
+    set_snan_bit_is_one(1, &env->ucf64.fp_status);
 }
 
 static const UniCore32CPUInfo uc32_cpus[] = {
@@ -111,7 +116,7 @@
     static bool inited;
 
     cs->env_ptr = env;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
 #ifdef CONFIG_USER_ONLY
     env->uncached_asr = ASR_MODE_USER;
@@ -155,6 +160,13 @@
     cc->get_phys_page_debug = uc32_cpu_get_phys_page_debug;
 #endif
     dc->vmsd = &vmstate_uc32_cpu;
+
+    /*
+     * Reason: uc32_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static void uc32_register_cpu_type(const UniCore32CPUInfo *info)
diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h
index 50972f9..7b5b405 100644
--- a/target-unicore32/cpu.h
+++ b/target-unicore32/cpu.h
@@ -8,8 +8,9 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#ifndef QEMU_UNICORE32_CPU_H
-#define QEMU_UNICORE32_CPU_H
+
+#ifndef UNICORE32_CPU_H
+#define UNICORE32_CPU_H
 
 #define TARGET_LONG_BITS                32
 #define TARGET_PAGE_BITS                12
@@ -17,12 +18,10 @@
 #define TARGET_PHYS_ADDR_SPACE_BITS     32
 #define TARGET_VIRT_ADDR_SPACE_BITS     32
 
-#define ELF_MACHINE             EM_UNICORE32
-
 #define CPUArchState                struct CPUUniCore32State
 
-#include "config.h"
 #include "qemu-common.h"
+#include "cpu-qom.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
 
@@ -74,6 +73,35 @@
 
 } CPUUniCore32State;
 
+/**
+ * UniCore32CPU:
+ * @env: #CPUUniCore32State
+ *
+ * A UniCore32 CPU.
+ */
+struct UniCore32CPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
+
+    CPUUniCore32State env;
+};
+
+static inline UniCore32CPU *uc32_env_get_cpu(CPUUniCore32State *env)
+{
+    return container_of(env, UniCore32CPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(uc32_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(UniCore32CPU, env)
+
+void uc32_cpu_do_interrupt(CPUState *cpu);
+bool uc32_cpu_exec_interrupt(CPUState *cpu, int int_req);
+void uc32_cpu_dump_state(CPUState *cpu, FILE *f,
+                         fprintf_function cpu_fprintf, int flags);
+hwaddr uc32_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+
 #define ASR_M                   (0x1f)
 #define ASR_MODE_USER           (0x10)
 #define ASR_MODE_INTR           (0x12)
@@ -122,29 +150,27 @@
 #define UC32_HWCAP_CMOV                 4 /* 1 << 2 */
 #define UC32_HWCAP_UCF64                8 /* 1 << 3 */
 
-#define cpu_init                        uc32_cpu_init
-#define cpu_exec                        uc32_cpu_exec
 #define cpu_signal_handler              uc32_cpu_signal_handler
 
-CPUUniCore32State *uc32_cpu_init(const char *cpu_model);
-int uc32_cpu_exec(CPUUniCore32State *s);
 int uc32_cpu_signal_handler(int host_signum, void *pinfo, void *puc);
 
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
 #define MMU_USER_IDX 1
-static inline int cpu_mmu_index(CPUUniCore32State *env)
+static inline int cpu_mmu_index(CPUUniCore32State *env, bool ifetch)
 {
     return (env->uncached_asr & ASR_M) == ASR_MODE_USER ? 1 : 0;
 }
 
 #include "exec/cpu-all.h"
-#include "cpu-qom.h"
-#include "exec/exec-all.h"
+
+UniCore32CPU *uc32_cpu_init(const char *cpu_model);
+
+#define cpu_init(cpu_model) CPU(uc32_cpu_init(cpu_model))
 
 static inline void cpu_get_tb_cpu_state(CPUUniCore32State *env, target_ulong *pc,
-                                        target_ulong *cs_base, int *flags)
+                                        target_ulong *cs_base, uint32_t *flags)
 {
     *pc = env->regs[31];
     *cs_base = 0;
@@ -159,4 +185,4 @@
 void uc32_translate_init(void);
 void switch_mode(CPUUniCore32State *, int);
 
-#endif /* QEMU_UNICORE32_CPU_H */
+#endif /* UNICORE32_CPU_H */
diff --git a/target-unicore32/helper.c b/target-unicore32/helper.c
index b4654fa..d603bde 100644
--- a/target-unicore32/helper.c
+++ b/target-unicore32/helper.c
@@ -9,7 +9,9 @@
  * or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
+#include "exec/exec-all.h"
 #include "exec/gdbstub.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
@@ -25,15 +27,9 @@
 #define DPRINTF(fmt, ...) do {} while (0)
 #endif
 
-CPUUniCore32State *uc32_cpu_init(const char *cpu_model)
+UniCore32CPU *uc32_cpu_init(const char *cpu_model)
 {
-    UniCore32CPU *cpu;
-
-    cpu = UNICORE32_CPU(cpu_generic_init(TYPE_UNICORE32_CPU, cpu_model));
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
+    return UNICORE32_CPU(cpu_generic_init(TYPE_UNICORE32_CPU, cpu_model));
 }
 
 uint32_t HELPER(clo)(uint32_t x)
diff --git a/target-unicore32/op_helper.c b/target-unicore32/op_helper.c
index 0266dbd..0872c29 100644
--- a/target-unicore32/op_helper.c
+++ b/target-unicore32/op_helper.c
@@ -8,8 +8,10 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 
 #define SIGNBIT (uint32_t)0x80000000
@@ -242,12 +244,12 @@
 }
 
 #ifndef CONFIG_USER_ONLY
-void tlb_fill(CPUState *cs, target_ulong addr, int is_write,
+void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
               int mmu_idx, uintptr_t retaddr)
 {
     int ret;
 
-    ret = uc32_cpu_handle_mmu_fault(cs, addr, is_write, mmu_idx);
+    ret = uc32_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
             /* now we have a real cpu fault */
diff --git a/target-unicore32/softmmu.c b/target-unicore32/softmmu.c
index 9a3786d..e7152e7 100644
--- a/target-unicore32/softmmu.c
+++ b/target-unicore32/softmmu.c
@@ -12,7 +12,9 @@
 #error This file only exist under softmmu circumstance
 #endif
 
-#include <cpu.h>
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 
 #undef DEBUG_UC32
 
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index e95c1e2..09354f9 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -8,14 +8,11 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "exec/cpu_ldst.h"
@@ -24,6 +21,7 @@
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 /* internal defines */
@@ -33,7 +31,7 @@
     /* Nonzero if this instruction has been conditionally skipped.  */
     int condjmp;
     /* The label that will be jumped to when the instruction is skipped.  */
-    int condlabel;
+    TCGLabel *condlabel;
     struct TranslationBlock *tb;
     int singlestep_enabled;
 #ifndef CONFIG_USER_ONLY
@@ -51,7 +49,7 @@
    conditional executions state has been updated.  */
 #define DISAS_SYSCALL 5
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv_i32 cpu_R[32];
 
 /* FIXME:  These should be removed.  */
@@ -72,9 +70,10 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    tcg_ctx.tcg_env = cpu_env;
 
     for (i = 0; i < 32; i++) {
-        cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new_i32(cpu_env,
                                 offsetof(CPUUniCore32State, regs[i]), regnames[i]);
     }
 }
@@ -419,11 +418,11 @@
     dead_tmp(shift);
 }
 
-static void gen_test_cc(int cc, int label)
+static void gen_test_cc(int cc, TCGLabel *label)
 {
     TCGv tmp;
     TCGv tmp2;
-    int inv;
+    TCGLabel *inv;
 
     switch (cc) {
     case 0: /* eq: Z */
@@ -1092,15 +1091,21 @@
     }
 }
 
+static inline bool use_goto_tb(DisasContext *s, uint32_t dest)
+{
+#ifndef CONFIG_USER_ONLY
+    return (s->tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK);
+#else
+    return true;
+#endif
+}
+
 static inline void gen_goto_tb(DisasContext *s, int n, uint32_t dest)
 {
-    TranslationBlock *tb;
-
-    tb = s->tb;
-    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+    if (use_goto_tb(s, dest)) {
         tcg_gen_goto_tb(n);
         gen_set_pc_im(dest);
-        tcg_gen_exit_tb((uintptr_t)tb + n);
+        tcg_gen_exit_tb((uintptr_t)s->tb + n);
     } else {
         gen_set_pc_im(dest);
         tcg_gen_exit_tb(0);
@@ -1794,10 +1799,6 @@
     UniCore32CPU *cpu = uc32_env_get_cpu(env);
     unsigned int insn;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-        tcg_gen_debug_insn_start(s->pc);
-    }
-
     insn = cpu_ldl_code(env, s->pc);
     s->pc += 4;
 
@@ -1867,17 +1868,12 @@
     }
 }
 
-/* generate intermediate code in gen_opc_buf and gen_opparam_buf for
-   basic block 'tb'. If search_pc is TRUE, also generate PC
-   information for each intermediate instruction. */
-static inline void gen_intermediate_code_internal(UniCore32CPU *cpu,
-        TranslationBlock *tb, bool search_pc)
+/* generate intermediate code for basic block 'tb'.  */
+void gen_intermediate_code(CPUUniCore32State *env, TranslationBlock *tb)
 {
+    UniCore32CPU *cpu = uc32_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUUniCore32State *env = &cpu->env;
     DisasContext dc1, *dc = &dc1;
-    CPUBreakpoint *bp;
-    int j, lj;
     target_ulong pc_start;
     uint32_t next_page_start;
     int num_insns;
@@ -1899,12 +1895,14 @@
     cpu_F0d = tcg_temp_new_i64();
     cpu_F1d = tcg_temp_new_i64();
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-    lj = -1;
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
 #ifndef CONFIG_USER_ONLY
     if ((env->uncached_asr & ASR_M) == ASR_MODE_USER) {
@@ -1914,35 +1912,24 @@
     }
 #endif
 
-    gen_tb_start();
+    gen_tb_start(tb);
     do {
-        if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-            QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-                if (bp->pc == dc->pc) {
-                    gen_set_pc_im(dc->pc);
-                    gen_exception(EXCP_DEBUG);
-                    dc->is_jmp = DISAS_JUMP;
-                    /* Advance PC so that clearing the breakpoint will
-                       invalidate this TB.  */
-                    dc->pc += 2; /* FIXME */
-                    goto done_generating;
-                }
-            }
-        }
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc->pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = num_insns;
+        tcg_gen_insn_start(dc->pc);
+        num_insns++;
+
+        if (unlikely(cpu_breakpoint_test(cs, dc->pc, BP_ANY))) {
+            gen_set_pc_im(dc->pc);
+            gen_exception(EXCP_DEBUG);
+            dc->is_jmp = DISAS_JUMP;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc->pc += 4;
+            goto done_generating;
         }
 
-        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (num_insns == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
@@ -1961,7 +1948,6 @@
          * Otherwise the subsequent code could get translated several times.
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
-        num_insns++;
     } while (!dc->is_jmp && !tcg_op_buf_full() &&
              !cs->singlestep_enabled &&
              !singlestep &&
@@ -2036,33 +2022,16 @@
     gen_tb_end(tb, num_insns);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
+        log_target_disas(cs, pc_start, dc->pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        lj++;
-        while (lj <= j) {
-            tcg_ctx.gen_opc_instr_start[lj++] = 0;
-        }
-    } else {
-        tb->size = dc->pc - pc_start;
-        tb->icount = num_insns;
-    }
-}
-
-void gen_intermediate_code(CPUUniCore32State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(uc32_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUUniCore32State *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(uc32_env_get_cpu(env), tb, true);
+    tb->size = dc->pc - pc_start;
+    tb->icount = num_insns;
 }
 
 static const char *cpu_mode_names[16] = {
@@ -2133,7 +2102,8 @@
     cpu_dump_state_ucf64(env, f, cpu_fprintf, flags);
 }
 
-void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUUniCore32State *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->regs[31] = tcg_ctx.gen_opc_pc[pc_pos];
+    env->regs[31] = data[0];
 }
diff --git a/target-unicore32/ucf64_helper.c b/target-unicore32/ucf64_helper.c
index 5af008f..6c91901 100644
--- a/target-unicore32/ucf64_helper.c
+++ b/target-unicore32/ucf64_helper.c
@@ -8,6 +8,7 @@
  * published by the Free Software Foundation, or any later version.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 
diff --git a/target-xtensa/Makefile.objs b/target-xtensa/Makefile.objs
index 5c150a8..481de91 100644
--- a/target-xtensa/Makefile.objs
+++ b/target-xtensa/Makefile.objs
@@ -2,5 +2,6 @@
 obj-y += core-dc232b.o
 obj-y += core-dc233c.o
 obj-y += core-fsf.o
+obj-$(CONFIG_SOFTMMU) += monitor.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += gdbstub.o
diff --git a/target-xtensa/core-dc232b.c b/target-xtensa/core-dc232b.c
index a3b914b..bb8ed41 100644
--- a/target-xtensa/core-dc232b.c
+++ b/target-xtensa/core-dc232b.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
@@ -33,7 +34,7 @@
 #include "core-dc232b/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig dc232b __attribute__((unused)) = {
+static XtensaConfig dc232b __attribute__((unused)) = {
     .name = "dc232b",
     .gdb_regmap = {
         .num_regs = 120,
diff --git a/target-xtensa/core-dc232b/core-isa.h b/target-xtensa/core-dc232b/core-isa.h
index 69f1065..a9935b8 100644
--- a/target-xtensa/core-dc232b/core-isa.h
+++ b/target-xtensa/core-dc232b/core-isa.h
@@ -8,9 +8,8 @@
  * Copyright (c) 1999-2007 Tensilica Inc.
  */
 
-#ifndef _XTENSA_CORE_CONFIGURATION_H
-#define _XTENSA_CORE_CONFIGURATION_H
-
+#ifndef XTENSA_DC232B_CORE_ISA_H
+#define XTENSA_DC232B_CORE_ISA_H
 
 /****************************************************************************
             Parameters Useful for Any Code, USER or PRIVILEGED
@@ -420,4 +419,4 @@
 #endif /* !XTENSA_HAL_NON_PRIVILEGED_ONLY */
 
 
-#endif /* _XTENSA_CORE_CONFIGURATION_H */
+#endif /* XTENSA_DC232B_CORE_ISA_H */
diff --git a/target-xtensa/core-dc233c.c b/target-xtensa/core-dc233c.c
index ac745d1..40475e5 100644
--- a/target-xtensa/core-dc233c.c
+++ b/target-xtensa/core-dc233c.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
@@ -34,7 +35,7 @@
 #include "core-dc233c/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig dc233c __attribute__((unused)) = {
+static XtensaConfig dc233c __attribute__((unused)) = {
     .name = "dc233c",
     .gdb_regmap = {
         .num_regs = 121,
diff --git a/target-xtensa/core-dc233c/core-isa.h b/target-xtensa/core-dc233c/core-isa.h
index e82c3cb..ff92b7f 100644
--- a/target-xtensa/core-dc233c/core-isa.h
+++ b/target-xtensa/core-dc233c/core-isa.h
@@ -28,9 +28,8 @@
    TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
    SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
 
-#ifndef _XTENSA_CORE_CONFIGURATION_H
-#define _XTENSA_CORE_CONFIGURATION_H
-
+#ifndef XTENSA_DC233C_CORE_ISA_H
+#define XTENSA_DC233C_CORE_ISA_H
 
 /****************************************************************************
             Parameters Useful for Any Code, USER or PRIVILEGED
@@ -471,4 +470,4 @@
 #endif /* !XTENSA_HAL_NON_PRIVILEGED_ONLY */
 
 
-#endif /* _XTENSA_CORE_CONFIGURATION_H */
+#endif /* XTENSA_DC233C_CORE_ISA_H */
diff --git a/target-xtensa/core-fsf.c b/target-xtensa/core-fsf.c
index cfcc840..15ef470 100644
--- a/target-xtensa/core-fsf.c
+++ b/target-xtensa/core-fsf.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
@@ -33,9 +34,14 @@
 #include "core-fsf/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig fsf __attribute__((unused)) = {
+static XtensaConfig fsf __attribute__((unused)) = {
     .name = "fsf",
+    .gdb_regmap = {
     /* GDB for this core is not supported currently */
+        .reg = {
+            XTREG_END
+        },
+    },
     .clock_freq_khz = 10000,
     DEFAULT_SECTIONS
 };
diff --git a/target-xtensa/core-fsf/core-isa.h b/target-xtensa/core-fsf/core-isa.h
index b519d6c..fb2bb8f 100644
--- a/target-xtensa/core-fsf/core-isa.h
+++ b/target-xtensa/core-fsf/core-isa.h
@@ -8,9 +8,8 @@
  * Copyright (C) 1999-2006 Tensilica Inc.
  */
 
-#ifndef _XTENSA_CORE_H
-#define _XTENSA_CORE_H
-
+#ifndef XTENSA_FSF_CORE_ISA_H
+#define XTENSA_FSF_CORE_ISA_H
 
 /****************************************************************************
             Parameters Useful for Any Code, USER or PRIVILEGED
@@ -358,4 +357,4 @@
 #endif /* !XTENSA_HAL_NON_PRIVILEGED_ONLY */
 
 
-#endif /* _XTENSA_CORE_CONFIGURATION_H */
+#endif /* XTENSA_FSF_CORE_ISA_H */
diff --git a/target-xtensa/cpu-qom.h b/target-xtensa/cpu-qom.h
index 9de5c6e..403bd95 100644
--- a/target-xtensa/cpu-qom.h
+++ b/target-xtensa/cpu-qom.h
@@ -30,7 +30,6 @@
 #define QEMU_XTENSA_CPU_QOM_H
 
 #include "qom/cpu.h"
-#include "cpu.h"
 
 #define TYPE_XTENSA_CPU "xtensa-cpu"
 
@@ -41,6 +40,8 @@
 #define XTENSA_CPU_GET_CLASS(obj) \
     OBJECT_GET_CLASS(XtensaCPUClass, (obj), TYPE_XTENSA_CPU)
 
+typedef struct XtensaConfig XtensaConfig;
+
 /**
  * XtensaCPUClass:
  * @parent_realize: The parent class' realize handler.
@@ -60,37 +61,6 @@
     const XtensaConfig *config;
 } XtensaCPUClass;
 
-/**
- * XtensaCPU:
- * @env: #CPUXtensaState
- *
- * An Xtensa CPU.
- */
-typedef struct XtensaCPU {
-    /*< private >*/
-    CPUState parent_obj;
-    /*< public >*/
-
-    CPUXtensaState env;
-} XtensaCPU;
-
-static inline XtensaCPU *xtensa_env_get_cpu(const CPUXtensaState *env)
-{
-    return container_of(env, XtensaCPU, env);
-}
-
-#define ENV_GET_CPU(e) CPU(xtensa_env_get_cpu(e))
-
-#define ENV_OFFSET offsetof(XtensaCPU, env)
-
-void xtensa_cpu_do_interrupt(CPUState *cpu);
-bool xtensa_cpu_exec_interrupt(CPUState *cpu, int interrupt_request);
-void xtensa_cpu_dump_state(CPUState *cpu, FILE *f,
-                           fprintf_function cpu_fprintf, int flags);
-hwaddr xtensa_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
-int xtensa_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
-int xtensa_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
-void xtensa_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
-                                    int is_write, int is_user, uintptr_t retaddr);
+typedef struct XtensaCPU XtensaCPU;
 
 #endif
diff --git a/target-xtensa/cpu.c b/target-xtensa/cpu.c
index 6a5414f..5ad08a2 100644
--- a/target-xtensa/cpu.c
+++ b/target-xtensa/cpu.c
@@ -28,9 +28,12 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "cpu.h"
 #include "qemu-common.h"
 #include "migration/vmstate.h"
+#include "exec/exec-all.h"
 
 
 static void xtensa_cpu_set_pc(CPUState *cs, vaddr value)
@@ -114,7 +117,7 @@
 
     cs->env_ptr = env;
     env->config = xcc->config;
-    cpu_exec_init(env);
+    cpu_exec_init(cs, &error_abort);
 
     if (tcg_enabled() && !tcg_inited) {
         tcg_inited = true;
@@ -151,9 +154,17 @@
 #ifndef CONFIG_USER_ONLY
     cc->do_unaligned_access = xtensa_cpu_do_unaligned_access;
     cc->get_phys_page_debug = xtensa_cpu_get_phys_page_debug;
+    cc->do_unassigned_access = xtensa_cpu_do_unassigned_access;
 #endif
     cc->debug_excp_handler = xtensa_breakpoint_handler;
     dc->vmsd = &vmstate_xtensa_cpu;
+
+    /*
+     * Reason: xtensa_cpu_initfn() calls cpu_exec_init(), which saves
+     * the object in cpus -> dangling pointer after final
+     * object_unref().
+     */
+    dc->cannot_destroy_with_object_finalize_yet = true;
 }
 
 static const TypeInfo xtensa_cpu_type_info = {
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index ac463f2..e3b350c 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -25,22 +25,19 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#ifndef CPU_XTENSA_H
-#define CPU_XTENSA_H
+#ifndef XTENSA_CPU_H
+#define XTENSA_CPU_H
 
 #define ALIGNED_ONLY
 #define TARGET_LONG_BITS 32
-#define ELF_MACHINE EM_XTENSA
 
 #define CPUArchState struct CPUXtensaState
 
-#include "config.h"
 #include "qemu-common.h"
+#include "cpu-qom.h"
 #include "exec/cpu-defs.h"
 #include "fpu/softfloat.h"
 
-#define TARGET_HAS_ICE 1
-
 #define NB_MMU_MODES 4
 
 #define TARGET_PHYS_ADDR_SPACE_BITS 32
@@ -67,6 +64,7 @@
     XTENSA_OPTION_MP_SYNCHRO,
     XTENSA_OPTION_CONDITIONAL_STORE,
     XTENSA_OPTION_ATOMCTL,
+    XTENSA_OPTION_DEPBITS,
 
     /* Interrupts and exceptions */
     XTENSA_OPTION_EXCEPTION,
@@ -289,6 +287,7 @@
     int targno;
     int type;
     int group;
+    unsigned size;
 } XtensaGdbReg;
 
 typedef struct XtensaGdbRegmap {
@@ -298,7 +297,7 @@
     XtensaGdbReg reg[1 + 16 + 64 + 256 + 256];
 } XtensaGdbRegmap;
 
-typedef struct XtensaConfig {
+struct XtensaConfig {
     const char *name;
     uint64_t options;
     XtensaGdbRegmap gdb_regmap;
@@ -331,13 +330,25 @@
 
     xtensa_tlb itlb;
     xtensa_tlb dtlb;
-} XtensaConfig;
+};
 
 typedef struct XtensaConfigList {
     const XtensaConfig *config;
     struct XtensaConfigList *next;
 } XtensaConfigList;
 
+#ifdef HOST_WORDS_BIGENDIAN
+enum {
+    FP_F32_HIGH,
+    FP_F32_LOW,
+};
+#else
+enum {
+    FP_F32_LOW,
+    FP_F32_HIGH,
+};
+#endif
+
 typedef struct CPUXtensaState {
     const XtensaConfig *config;
     uint32_t regs[16];
@@ -345,7 +356,10 @@
     uint32_t sregs[256];
     uint32_t uregs[256];
     uint32_t phys_regs[MAX_NAREG];
-    float32 fregs[16];
+    union {
+        float32 f32[2];
+        float64 f64;
+    } fregs[16];
     float_status fp_status;
 
     xtensa_tlb_entry itlb[7][MAX_TLB_WAY_SIZE];
@@ -366,10 +380,44 @@
     CPU_COMMON
 } CPUXtensaState;
 
-#include "cpu-qom.h"
+/**
+ * XtensaCPU:
+ * @env: #CPUXtensaState
+ *
+ * An Xtensa CPU.
+ */
+struct XtensaCPU {
+    /*< private >*/
+    CPUState parent_obj;
+    /*< public >*/
 
-#define cpu_exec cpu_xtensa_exec
-#define cpu_gen_code cpu_xtensa_gen_code
+    CPUXtensaState env;
+};
+
+static inline XtensaCPU *xtensa_env_get_cpu(const CPUXtensaState *env)
+{
+    return container_of(env, XtensaCPU, env);
+}
+
+#define ENV_GET_CPU(e) CPU(xtensa_env_get_cpu(e))
+
+#define ENV_OFFSET offsetof(XtensaCPU, env)
+
+void xtensa_cpu_do_interrupt(CPUState *cpu);
+bool xtensa_cpu_exec_interrupt(CPUState *cpu, int interrupt_request);
+void xtensa_cpu_do_unassigned_access(CPUState *cpu, hwaddr addr,
+                                     bool is_write, bool is_exec, int opaque,
+                                     unsigned size);
+void xtensa_cpu_dump_state(CPUState *cpu, FILE *f,
+                           fprintf_function cpu_fprintf, int flags);
+hwaddr xtensa_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+int xtensa_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
+int xtensa_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void xtensa_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                    MMUAccessType access_type,
+                                    int mmu_idx, uintptr_t retaddr,
+                                    unsigned size);
+
 #define cpu_signal_handler cpu_xtensa_signal_handler
 #define cpu_list xtensa_cpu_list
 
@@ -381,18 +429,11 @@
 
 XtensaCPU *cpu_xtensa_init(const char *cpu_model);
 
-static inline CPUXtensaState *cpu_init(const char *cpu_model)
-{
-    XtensaCPU *cpu = cpu_xtensa_init(cpu_model);
-    if (cpu == NULL) {
-        return NULL;
-    }
-    return &cpu->env;
-}
+#define cpu_init(cpu_model) CPU(cpu_xtensa_init(cpu_model))
 
 void xtensa_translate_init(void);
 void xtensa_breakpoint_handler(CPUState *cs);
-int cpu_xtensa_exec(CPUXtensaState *s);
+void xtensa_finalize_config(XtensaConfig *config);
 void xtensa_register_core(XtensaConfigList *node);
 void check_interrupts(CPUXtensaState *s);
 void xtensa_irq_init(CPUXtensaState *env);
@@ -484,7 +525,7 @@
 #define MMU_MODE2_SUFFIX _ring2
 #define MMU_MODE3_SUFFIX _ring3
 
-static inline int cpu_mmu_index(CPUXtensaState *env)
+static inline int cpu_mmu_index(CPUXtensaState *env, bool ifetch)
 {
     return xtensa_get_cring(env);
 }
@@ -497,9 +538,11 @@
 #define XTENSA_TBFLAG_CPENABLE_MASK 0x3fc0
 #define XTENSA_TBFLAG_CPENABLE_SHIFT 6
 #define XTENSA_TBFLAG_EXCEPTION 0x4000
+#define XTENSA_TBFLAG_WINDOW_MASK 0x18000
+#define XTENSA_TBFLAG_WINDOW_SHIFT 15
 
 static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
-        target_ulong *cs_base, int *flags)
+        target_ulong *cs_base, uint32_t *flags)
 {
     CPUState *cs = CPU(xtensa_env_get_cpu(env));
 
@@ -528,9 +571,18 @@
     if (cs->singlestep_enabled && env->exception_taken) {
         *flags |= XTENSA_TBFLAG_EXCEPTION;
     }
+    if (xtensa_option_enabled(env->config, XTENSA_OPTION_WINDOWED_REGISTER) &&
+        (env->sregs[PS] & (PS_WOE | PS_EXCM)) == PS_WOE) {
+        uint32_t windowstart = xtensa_replicate_windowstart(env) >>
+            (env->sregs[WINDOW_BASE] + 1);
+        uint32_t w = ctz32(windowstart | 0x8);
+
+        *flags |= w << XTENSA_TBFLAG_WINDOW_SHIFT;
+    } else {
+        *flags |= 3 << XTENSA_TBFLAG_WINDOW_SHIFT;
+    }
 }
 
 #include "exec/cpu-all.h"
-#include "exec/exec-all.h"
 
 #endif
diff --git a/target-xtensa/gdbstub.c b/target-xtensa/gdbstub.c
index 9e13b20..fa5469a 100644
--- a/target-xtensa/gdbstub.c
+++ b/target-xtensa/gdbstub.c
@@ -17,15 +17,18 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "cpu.h"
 #include "exec/gdbstub.h"
+#include "qemu/log.h"
 
 int xtensa_cpu_gdb_read_register(CPUState *cs, uint8_t *mem_buf, int n)
 {
     XtensaCPU *cpu = XTENSA_CPU(cs);
     CPUXtensaState *env = &cpu->env;
     const XtensaGdbReg *reg = env->config->gdb_regmap.reg + n;
+    unsigned i;
 
     if (n < 0 || n >= env->config->gdb_regmap.num_regs) {
         return 0;
@@ -47,15 +50,23 @@
         return gdb_get_reg32(mem_buf, env->uregs[reg->targno & 0xff]);
 
     case 4: /*f*/
-        return gdb_get_reg32(mem_buf, float32_val(env->fregs[reg->targno
-                                                             & 0x0f]));
+        i = reg->targno & 0x0f;
+        switch (reg->size) {
+        case 4:
+            return gdb_get_reg32(mem_buf,
+                                 float32_val(env->fregs[i].f32[FP_F32_LOW]));
+        case 8:
+            return gdb_get_reg64(mem_buf, float64_val(env->fregs[i].f64));
+        default:
+            return 0;
+        }
 
     case 8: /*a*/
         return gdb_get_reg32(mem_buf, env->regs[reg->targno & 0x0f]);
 
     default:
-        qemu_log("%s from reg %d of unsupported type %d\n",
-                 __func__, n, reg->type);
+        qemu_log_mask(LOG_UNIMP, "%s from reg %d of unsupported type %d\n",
+                      __func__, n, reg->type);
         return 0;
     }
 }
@@ -92,16 +103,24 @@
         break;
 
     case 4: /*f*/
-        env->fregs[reg->targno & 0x0f] = make_float32(tmp);
-        break;
+        switch (reg->size) {
+        case 4:
+            env->fregs[reg->targno & 0x0f].f32[FP_F32_LOW] = make_float32(tmp);
+            return 4;
+        case 8:
+            env->fregs[reg->targno & 0x0f].f64 = make_float64(tmp);
+            return 8;
+        default:
+            return 0;
+        }
 
     case 8: /*a*/
         env->regs[reg->targno & 0x0f] = tmp;
         break;
 
     default:
-        qemu_log("%s to reg %d of unsupported type %d\n",
-                 __func__, n, reg->type);
+        qemu_log_mask(LOG_UNIMP, "%s to reg %d of unsupported type %d\n",
+                      __func__, n, reg->type);
         return 0;
     }
 
diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c
index d84d259..768b32c 100644
--- a/target-xtensa/helper.c
+++ b/target-xtensa/helper.c
@@ -25,6 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/exec-all.h"
 #include "exec/gdbstub.h"
@@ -51,6 +52,20 @@
     cc->gdb_num_core_regs = config->gdb_regmap.num_regs;
 }
 
+void xtensa_finalize_config(XtensaConfig *config)
+{
+    unsigned i, n = 0;
+
+    if (config->gdb_regmap.num_regs) {
+        return;
+    }
+
+    for (i = 0; config->gdb_regmap.reg[i].targno >= 0; ++i) {
+        n += (config->gdb_regmap.reg[i].type != 6);
+    }
+    config->gdb_regmap.num_regs = n;
+}
+
 void xtensa_register_core(XtensaConfigList *node)
 {
     TypeInfo type = {
@@ -93,7 +108,7 @@
             if (cause) {
                 debug_exception_env(env, cause);
             }
-            cpu_resume_from_signal(cs, NULL);
+            cpu_loop_exit_noexc(cs);
         }
     }
 }
@@ -240,8 +255,8 @@
                     env->config->exception_vector[cs->exception_index]);
             env->exception_taken = 1;
         } else {
-            qemu_log("%s(pc = %08x) bad exception_index: %d\n",
-                    __func__, env->pc, cs->exception_index);
+            qemu_log_mask(CPU_LOG_INT, "%s(pc = %08x) bad exception_index: %d\n",
+                          __func__, env->pc, cs->exception_index);
         }
         break;
 
@@ -527,8 +542,8 @@
             wi = ++env->autorefill_idx & 0x3;
             xtensa_tlb_set_entry(env, dtlb, wi, ei, vpn, pte);
             env->sregs[EXCVADDR] = vaddr;
-            qemu_log("%s: autorefill(%08x): %08x -> %08x\n",
-                    __func__, vaddr, vpn, pte);
+            qemu_log_mask(CPU_LOG_MMU, "%s: autorefill(%08x): %08x -> %08x\n",
+                          __func__, vaddr, vpn, pte);
         } else {
             xtensa_tlb_set_entry_mmu(env, &tmp_entry, dtlb, wi, ei, vpn, pte);
             entry = &tmp_entry;
@@ -576,8 +591,8 @@
     int ret = get_physical_addr_mmu(env, false, pt_vaddr, 0, 0,
             &paddr, &page_size, &access, false);
 
-    qemu_log("%s: trying autorefill(%08x) -> %08x\n", __func__,
-            vaddr, ret ? ~0 : paddr);
+    qemu_log_mask(CPU_LOG_MMU, "%s: trying autorefill(%08x) -> %08x\n",
+                  __func__, vaddr, ret ? ~0 : paddr);
 
     if (ret == 0) {
         *pte = ldl_phys(cs->as, paddr);
diff --git a/target-xtensa/helper.h b/target-xtensa/helper.h
index ed3af0b..5ea9c5b 100644
--- a/target-xtensa/helper.h
+++ b/target-xtensa/helper.h
@@ -9,7 +9,7 @@
 DEF_HELPER_4(entry, void, env, i32, i32, i32)
 DEF_HELPER_2(retw, i32, env, i32)
 DEF_HELPER_2(rotw, void, env, i32)
-DEF_HELPER_3(window_check, void, env, i32, i32)
+DEF_HELPER_3(window_check, noreturn, env, i32, i32)
 DEF_HELPER_1(restore_owb, void, env)
 DEF_HELPER_2(movsp, void, env, i32)
 DEF_HELPER_2(wsr_lbeg, void, env, i32)
diff --git a/target-xtensa/import_core.sh b/target-xtensa/import_core.sh
index 73791ec..351bee4 100755
--- a/target-xtensa/import_core.sh
+++ b/target-xtensa/import_core.sh
@@ -22,8 +22,7 @@
 tar -xf "$OVERLAY" -C "$TARGET" --strip-components=1 \
     --xform='s/core/core-isa/' config/core.h
 tar -xf "$OVERLAY" -O gdb/xtensa-config.c | \
-    sed -n '1,/*\//p;/pc/,/a15/p' > "$TARGET"/gdb-config.c
-NUM_REGS=$(grep XTREG "$TARGET"/gdb-config.c | wc -l)
+    sed -n '1,/*\//p;/XTREG/,/XTREG_END/p' > "$TARGET"/gdb-config.c
 
 cat <<EOF > "${TARGET}.c"
 #include "cpu.h"
@@ -34,10 +33,9 @@
 #include "core-$NAME/core-isa.h"
 #include "overlay_tool.h"
 
-static const XtensaConfig $NAME __attribute__((unused)) = {
+static XtensaConfig $NAME __attribute__((unused)) = {
     .name = "$NAME",
     .gdb_regmap = {
-        .num_regs = $NUM_REGS,
         .reg = {
 #include "core-$NAME/gdb-config.c"
         }
diff --git a/target-xtensa/monitor.c b/target-xtensa/monitor.c
new file mode 100644
index 0000000..f3fa4cd
--- /dev/null
+++ b/target-xtensa/monitor.c
@@ -0,0 +1,35 @@
+/*
+ * QEMU monitor
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "monitor/monitor.h"
+#include "monitor/hmp-target.h"
+#include "hmp.h"
+
+void hmp_info_tlb(Monitor *mon, const QDict *qdict)
+{
+    CPUArchState *env1 = mon_get_cpu_env();
+
+    dump_mmu((FILE*)mon, (fprintf_function)monitor_printf, env1);
+}
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index 872e5a8..4f2a962 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -25,19 +25,24 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
 #include "exec/address-spaces.h"
 #include "qemu/timer.h"
 
 void xtensa_cpu_do_unaligned_access(CPUState *cs,
-        vaddr addr, int is_write, int is_user, uintptr_t retaddr)
+        vaddr addr, MMUAccessType access_type,
+        int mmu_idx, uintptr_t retaddr, unsigned size)
 {
     XtensaCPU *cpu = XTENSA_CPU(cs);
     CPUXtensaState *env = &cpu->env;
 
+    (void)size;
+
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_UNALIGNED_EXCEPTION) &&
             !xtensa_option_enabled(env->config, XTENSA_OPTION_HW_ALIGNMENT)) {
         cpu_restore_state(CPU(cpu), retaddr);
@@ -46,19 +51,19 @@
     }
 }
 
-void tlb_fill(CPUState *cs,
-              target_ulong vaddr, int is_write, int mmu_idx, uintptr_t retaddr)
+void tlb_fill(CPUState *cs, target_ulong vaddr, MMUAccessType access_type,
+              int mmu_idx, uintptr_t retaddr)
 {
     XtensaCPU *cpu = XTENSA_CPU(cs);
     CPUXtensaState *env = &cpu->env;
     uint32_t paddr;
     uint32_t page_size;
     unsigned access;
-    int ret = xtensa_get_physical_addr(env, true, vaddr, is_write, mmu_idx,
+    int ret = xtensa_get_physical_addr(env, true, vaddr, access_type, mmu_idx,
             &paddr, &page_size, &access);
 
-    qemu_log("%s(%08x, %d, %d) -> %08x, ret = %d\n", __func__,
-            vaddr, is_write, mmu_idx, paddr, ret);
+    qemu_log_mask(CPU_LOG_MMU, "%s(%08x, %d, %d) -> %08x, ret = %d\n",
+                  __func__, vaddr, access_type, mmu_idx, paddr, ret);
 
     if (ret == 0) {
         tlb_set_page(cs,
@@ -71,6 +76,20 @@
     }
 }
 
+void xtensa_cpu_do_unassigned_access(CPUState *cs, hwaddr addr,
+                                     bool is_write, bool is_exec, int opaque,
+                                     unsigned size)
+{
+    XtensaCPU *cpu = XTENSA_CPU(cs);
+    CPUXtensaState *env = &cpu->env;
+
+    HELPER(exception_cause_vaddr)(env, env->pc,
+                                  is_exec ?
+                                  INSTR_PIF_ADDR_ERROR_CAUSE :
+                                  LOAD_STORE_PIF_ADDR_ERROR_CAUSE,
+                                  is_exec ? addr : cs->mem_io_vaddr);
+}
+
 static void tb_invalidate_virtual_addr(CPUXtensaState *env, uint32_t vaddr)
 {
     uint32_t paddr;
@@ -231,8 +250,8 @@
 {
     int callinc = (env->sregs[PS] & PS_CALLINC) >> PS_CALLINC_SHIFT;
     if (s > 3 || ((env->sregs[PS] & (PS_WOE | PS_EXCM)) ^ PS_WOE) != 0) {
-        qemu_log("Illegal entry instruction(pc = %08x), PS = %08x\n",
-                pc, env->sregs[PS]);
+        qemu_log_mask(LOG_GUEST_ERROR, "Illegal entry instruction(pc = %08x), PS = %08x\n",
+                      pc, env->sregs[PS]);
         HELPER(exception_cause)(env, pc, ILLEGAL_INSTRUCTION_CAUSE);
     } else {
         uint32_t windowstart = xtensa_replicate_windowstart(env) >>
@@ -251,34 +270,27 @@
 void HELPER(window_check)(CPUXtensaState *env, uint32_t pc, uint32_t w)
 {
     uint32_t windowbase = windowbase_bound(env->sregs[WINDOW_BASE], env);
-    uint32_t windowstart = env->sregs[WINDOW_START];
-    uint32_t m, n;
+    uint32_t windowstart = xtensa_replicate_windowstart(env) >>
+        (env->sregs[WINDOW_BASE] + 1);
+    uint32_t n = ctz32(windowstart) + 1;
 
-    if ((env->sregs[PS] & (PS_WOE | PS_EXCM)) ^ PS_WOE) {
-        return;
-    }
+    assert(n <= w);
 
-    for (n = 1; ; ++n) {
-        if (n > w) {
-            return;
-        }
-        if (windowstart & windowstart_bit(windowbase + n, env)) {
-            break;
-        }
-    }
-
-    m = windowbase_bound(windowbase + n, env);
     rotate_window(env, n);
     env->sregs[PS] = (env->sregs[PS] & ~PS_OWB) |
         (windowbase << PS_OWB_SHIFT) | PS_EXCM;
     env->sregs[EPC1] = env->pc = pc;
 
-    if (windowstart & windowstart_bit(m + 1, env)) {
+    switch (ctz32(windowstart >> n)) {
+    case 0:
         HELPER(exception)(env, EXC_WINDOW_OVERFLOW4);
-    } else if (windowstart & windowstart_bit(m + 2, env)) {
+        break;
+    case 1:
         HELPER(exception)(env, EXC_WINDOW_OVERFLOW8);
-    } else {
+        break;
+    default:
         HELPER(exception)(env, EXC_WINDOW_OVERFLOW12);
+        break;
     }
 }
 
@@ -300,9 +312,9 @@
 
     if (n == 0 || (m != 0 && m != n) ||
             ((env->sregs[PS] & (PS_WOE | PS_EXCM)) ^ PS_WOE) != 0) {
-        qemu_log("Illegal retw instruction(pc = %08x), "
-                "PS = %08x, m = %d, n = %d\n",
-                pc, env->sregs[PS], m, n);
+        qemu_log_mask(LOG_GUEST_ERROR, "Illegal retw instruction(pc = %08x), "
+                      "PS = %08x, m = %d, n = %d\n",
+                      pc, env->sregs[PS], m, n);
         HELPER(exception_cause)(env, pc, ILLEGAL_INSTRUCTION_CAUSE);
     } else {
         int owb = windowbase;
@@ -736,8 +748,8 @@
             xtensa_tlb_set_entry_mmu(env, entry, dtlb, wi, ei, vpn, pte);
             tlb_flush_page(cs, entry->vaddr);
         } else {
-            qemu_log("%s %d, %d, %d trying to set immutable entry\n",
-                    __func__, dtlb, wi, ei);
+            qemu_log_mask(LOG_GUEST_ERROR, "%s %d, %d, %d trying to set immutable entry\n",
+                          __func__, dtlb, wi, ei);
         }
     } else {
         tlb_flush_page(cs, entry->vaddr);
@@ -799,15 +811,15 @@
     }
     /* contiguous mask after inversion is one less than some power of 2 */
     if ((~mask + 1) & ~mask) {
-        qemu_log("DBREAKC mask is not contiguous: 0x%08x\n", dbreakc);
+        qemu_log_mask(LOG_GUEST_ERROR, "DBREAKC mask is not contiguous: 0x%08x\n", dbreakc);
         /* cut mask after the first zero bit */
         mask = 0xffffffff << (32 - clo32(mask));
     }
     if (cpu_watchpoint_insert(cs, dbreaka & mask, ~mask + 1,
             flags, &env->cpu_watchpoint[i])) {
         env->cpu_watchpoint[i] = NULL;
-        qemu_log("Failed to set data breakpoint at 0x%08x/%d\n",
-                dbreaka & mask, ~mask + 1);
+        qemu_log_mask(LOG_GUEST_ERROR, "Failed to set data breakpoint at 0x%08x/%d\n",
+                      dbreaka & mask, ~mask + 1);
     }
 }
 
diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h
index 6105d4c..e8a7fda 100644
--- a/target-xtensa/overlay_tool.h
+++ b/target-xtensa/overlay_tool.h
@@ -27,7 +27,12 @@
 
 #define XTREG(idx, ofs, bi, sz, al, no, flags, cp, typ, grp, name, \
         a1, a2, a3, a4, a5, a6) \
-    { .targno = (no), .type = (typ), .group = (grp) },
+    { .targno = (no), .type = (typ), .group = (grp), .size = (sz) },
+#define XTREG_END { .targno = -1 },
+
+#ifndef XCHAL_HAVE_DEPBITS
+#define XCHAL_HAVE_DEPBITS 0
+#endif
 
 #ifndef XCHAL_HAVE_DIV32
 #define XCHAL_HAVE_DIV32 0
@@ -68,6 +73,7 @@
     XCHAL_OPTION(XCHAL_HAVE_S32C1I, XTENSA_OPTION_CONDITIONAL_STORE) | \
     XCHAL_OPTION(XCHAL_HAVE_S32C1I && XCHAL_HW_MIN_VERSION >= 230000, \
         XTENSA_OPTION_ATOMCTL) | \
+    XCHAL_OPTION(XCHAL_HAVE_DEPBITS, XTENSA_OPTION_DEPBITS) | \
     /* Interrupts and exceptions */ \
     XCHAL_OPTION(XCHAL_HAVE_EXCEPTIONS, XTENSA_OPTION_EXCEPTION) | \
     XCHAL_OPTION(XCHAL_HAVE_VECBASE, XTENSA_OPTION_RELOCATABLE_VECTOR) | \
@@ -316,6 +322,7 @@
         static XtensaConfigList node = { \
             .config = &core, \
         }; \
+        xtensa_finalize_config(&core); \
         xtensa_register_core(&node); \
     }
 #else
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 7ee565c..4c1e487 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -28,7 +28,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "cpu.h"
 #include "exec/exec-all.h"
@@ -36,12 +36,15 @@
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
+#include "exec/exec-all.h"
 #include "exec/cpu_ldst.h"
+#include "exec/semihost.h"
 
 #include "exec/helper-proto.h"
 #include "exec/helper-gen.h"
 
 #include "trace-tcg.h"
+#include "exec/log.h"
 
 
 typedef struct DisasContext {
@@ -63,7 +66,7 @@
     TCGv_i32 sar_m32;
 
     uint32_t ccount_delta;
-    unsigned used_window;
+    unsigned window;
 
     bool debug;
     bool icount;
@@ -72,7 +75,7 @@
     unsigned cpenable;
 } DisasContext;
 
-static TCGv_ptr cpu_env;
+static TCGv_env cpu_env;
 static TCGv_i32 cpu_pc;
 static TCGv_i32 cpu_R[16];
 static TCGv_i32 cpu_FR[16];
@@ -216,24 +219,25 @@
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
-    cpu_pc = tcg_global_mem_new_i32(TCG_AREG0,
+    tcg_ctx.tcg_env = cpu_env;
+    cpu_pc = tcg_global_mem_new_i32(cpu_env,
             offsetof(CPUXtensaState, pc), "pc");
 
     for (i = 0; i < 16; i++) {
-        cpu_R[i] = tcg_global_mem_new_i32(TCG_AREG0,
+        cpu_R[i] = tcg_global_mem_new_i32(cpu_env,
                 offsetof(CPUXtensaState, regs[i]),
                 regnames[i]);
     }
 
     for (i = 0; i < 16; i++) {
-        cpu_FR[i] = tcg_global_mem_new_i32(TCG_AREG0,
-                offsetof(CPUXtensaState, fregs[i]),
+        cpu_FR[i] = tcg_global_mem_new_i32(cpu_env,
+                offsetof(CPUXtensaState, fregs[i].f32[FP_F32_LOW]),
                 fregnames[i]);
     }
 
     for (i = 0; i < 256; ++i) {
         if (sregnames[i].name) {
-            cpu_SR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+            cpu_SR[i] = tcg_global_mem_new_i32(cpu_env,
                     offsetof(CPUXtensaState, sregs[i]),
                     sregnames[i].name);
         }
@@ -241,7 +245,7 @@
 
     for (i = 0; i < 256; ++i) {
         if (uregnames[i].name) {
-            cpu_UR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+            cpu_UR[i] = tcg_global_mem_new_i32(cpu_env,
                     offsetof(CPUXtensaState, uregs[i]),
                     uregnames[i].name);
         }
@@ -311,26 +315,16 @@
     tcg_temp_free(tmp);
 }
 
-static void gen_advance_ccount_cond(DisasContext *dc)
+static void gen_advance_ccount(DisasContext *dc)
 {
     if (dc->ccount_delta > 0) {
         TCGv_i32 tmp = tcg_const_i32(dc->ccount_delta);
         gen_helper_advance_ccount(cpu_env, tmp);
         tcg_temp_free(tmp);
     }
-}
-
-static void gen_advance_ccount(DisasContext *dc)
-{
-    gen_advance_ccount_cond(dc);
     dc->ccount_delta = 0;
 }
 
-static void reset_used_window(DisasContext *dc)
-{
-    dc->used_window = 0;
-}
-
 static void gen_exception(DisasContext *dc, int excp)
 {
     TCGv_i32 tmp = tcg_const_i32(excp);
@@ -377,21 +371,25 @@
     }
 }
 
-static void gen_check_privilege(DisasContext *dc)
+static bool gen_check_privilege(DisasContext *dc)
 {
     if (dc->cring) {
         gen_exception_cause(dc, PRIVILEGED_CAUSE);
         dc->is_jmp = DISAS_UPDATE;
+        return false;
     }
+    return true;
 }
 
-static void gen_check_cpenable(DisasContext *dc, unsigned cp)
+static bool gen_check_cpenable(DisasContext *dc, unsigned cp)
 {
     if (option_enabled(dc, XTENSA_OPTION_COPROCESSOR) &&
             !(dc->cpenable & (1 << cp))) {
         gen_exception_cause(dc, COPROCESSOR0_DISABLED + cp);
         dc->is_jmp = DISAS_UPDATE;
+        return false;
     }
+    return true;
 }
 
 static void gen_jump_slot(DisasContext *dc, TCGv dest, int slot)
@@ -422,9 +420,11 @@
 static void gen_jumpi(DisasContext *dc, uint32_t dest, int slot)
 {
     TCGv_i32 tmp = tcg_const_i32(dest);
+#ifndef CONFIG_USER_ONLY
     if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
         slot = -1;
     }
+#endif
     gen_jump_slot(dc, tmp, slot);
     tcg_temp_free(tmp);
 }
@@ -450,9 +450,11 @@
 static void gen_callwi(DisasContext *dc, int callinc, uint32_t dest, int slot)
 {
     TCGv_i32 tmp = tcg_const_i32(dest);
+#ifndef CONFIG_USER_ONLY
     if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
         slot = -1;
     }
+#endif
     gen_callw_slot(dc, callinc, tmp, slot);
     tcg_temp_free(tmp);
 }
@@ -462,7 +464,7 @@
     if (option_enabled(dc, XTENSA_OPTION_LOOP) &&
             !(dc->tb->flags & XTENSA_TBFLAG_EXCM) &&
             dc->next_pc == dc->lend) {
-        int label = gen_new_label();
+        TCGLabel *label = gen_new_label();
 
         gen_advance_ccount(dc);
         tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_SR[LCOUNT], 0, label);
@@ -485,7 +487,7 @@
 static void gen_brcond(DisasContext *dc, TCGCond cond,
         TCGv_i32 t0, TCGv_i32 t1, uint32_t offset)
 {
-    int label = gen_new_label();
+    TCGLabel *label = gen_new_label();
 
     gen_advance_ccount(dc);
     tcg_gen_brcond_i32(cond, t0, t1, label);
@@ -506,9 +508,9 @@
 {
     if (!xtensa_option_bits_enabled(dc->config, sregnames[sr].opt_bits)) {
         if (sregnames[sr].name) {
-            qemu_log("SR %s is not configured\n", sregnames[sr].name);
+            qemu_log_mask(LOG_GUEST_ERROR, "SR %s is not configured\n", sregnames[sr].name);
         } else {
-            qemu_log("SR %d is not implemented\n", sr);
+            qemu_log_mask(LOG_UNIMP, "SR %d is not implemented\n", sr);
         }
         gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
         return false;
@@ -519,8 +521,8 @@
             [SR_X] = "xsr",
         };
         assert(access < ARRAY_SIZE(access_text) && access_text[access]);
-        qemu_log("SR %s is not available for %s\n", sregnames[sr].name,
-                access_text[access]);
+        qemu_log_mask(LOG_GUEST_ERROR, "SR %s is not available for %s\n", sregnames[sr].name,
+                      access_text[access]);
         gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
         return false;
     }
@@ -597,13 +599,15 @@
 static void gen_wsr_windowbase(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     gen_helper_wsr_windowbase(cpu_env, v);
-    reset_used_window(dc);
+    /* This can change tb->flags, so exit tb */
+    gen_jumpi_check_loop_end(dc, -1);
 }
 
 static void gen_wsr_windowstart(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     tcg_gen_andi_i32(cpu_SR[sr], v, (1 << dc->config->nareg / 4) - 1);
-    reset_used_window(dc);
+    /* This can change tb->flags, so exit tb */
+    gen_jumpi_check_loop_end(dc, -1);
 }
 
 static void gen_wsr_ptevaddr(DisasContext *dc, uint32_t sr, TCGv_i32 v)
@@ -712,7 +716,6 @@
         mask |= PS_RING;
     }
     tcg_gen_andi_i32(cpu_SR[sr], v, mask);
-    reset_used_window(dc);
     gen_helper_check_interrupts(cpu_env);
     /* This can change mmu index and tb->flags, so exit tb */
     gen_jumpi_check_loop_end(dc, -1);
@@ -813,7 +816,7 @@
         tcg_gen_andi_i32(addr, addr, ~0 << shift);
     } else if (option_enabled(dc, XTENSA_OPTION_HW_ALIGNMENT) &&
             no_hw_alignment) {
-        int label = gen_new_label();
+        TCGLabel *label = gen_new_label();
         TCGv_i32 tmp = tcg_temp_new_i32();
         tcg_gen_andi_i32(tmp, addr, ~(~0 << shift));
         tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
@@ -833,46 +836,29 @@
     tcg_temp_free(intlevel);
 }
 
-static void gen_window_check1(DisasContext *dc, unsigned r1)
+static bool gen_window_check1(DisasContext *dc, unsigned r1)
 {
-    if (dc->tb->flags & XTENSA_TBFLAG_EXCM) {
-        return;
+    if (r1 / 4 > dc->window) {
+        TCGv_i32 pc = tcg_const_i32(dc->pc);
+        TCGv_i32 w = tcg_const_i32(r1 / 4);
+
+        gen_advance_ccount(dc);
+        gen_helper_window_check(cpu_env, pc, w);
+        dc->is_jmp = DISAS_UPDATE;
+        return false;
     }
-    if (option_enabled(dc, XTENSA_OPTION_WINDOWED_REGISTER) &&
-            r1 / 4 > dc->used_window) {
-        int label = gen_new_label();
-        TCGv_i32 ws = tcg_temp_new_i32();
-
-        dc->used_window = r1 / 4;
-        tcg_gen_deposit_i32(ws, cpu_SR[WINDOW_START], cpu_SR[WINDOW_START],
-                dc->config->nareg / 4, dc->config->nareg / 4);
-        tcg_gen_shr_i32(ws, ws, cpu_SR[WINDOW_BASE]);
-        tcg_gen_andi_i32(ws, ws, (2 << (r1 / 4)) - 2);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, ws, 0, label);
-        {
-            TCGv_i32 pc = tcg_const_i32(dc->pc);
-            TCGv_i32 w = tcg_const_i32(r1 / 4);
-
-            gen_advance_ccount_cond(dc);
-            gen_helper_window_check(cpu_env, pc, w);
-
-            tcg_temp_free(w);
-            tcg_temp_free(pc);
-        }
-        gen_set_label(label);
-        tcg_temp_free(ws);
-    }
+    return true;
 }
 
-static void gen_window_check2(DisasContext *dc, unsigned r1, unsigned r2)
+static bool gen_window_check2(DisasContext *dc, unsigned r1, unsigned r2)
 {
-    gen_window_check1(dc, r1 > r2 ? r1 : r2);
+    return gen_window_check1(dc, r1 > r2 ? r1 : r2);
 }
 
-static void gen_window_check3(DisasContext *dc, unsigned r1, unsigned r2,
+static bool gen_window_check3(DisasContext *dc, unsigned r1, unsigned r2,
         unsigned r3)
 {
-    gen_window_check2(dc, r1, r2 > r3 ? r2 : r3);
+    return gen_window_check2(dc, r1, r2 > r3 ? r2 : r3);
 }
 
 static TCGv_i32 gen_mac16_m(TCGv_i32 v, bool hi, bool is_unsigned)
@@ -887,22 +873,27 @@
     return m;
 }
 
+static inline unsigned xtensa_op0_insn_len(unsigned op0)
+{
+    return op0 >= 8 ? 2 : 3;
+}
+
 static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 {
 #define HAS_OPTION_BITS(opt) do { \
         if (!option_bits_enabled(dc, opt)) { \
-            qemu_log("Option is not enabled %s:%d\n", \
-                    __FILE__, __LINE__); \
+            qemu_log_mask(LOG_GUEST_ERROR, "Option is not enabled %s:%d\n", \
+                          __FILE__, __LINE__); \
             goto invalid_opcode; \
         } \
     } while (0)
 
 #define HAS_OPTION(opt) HAS_OPTION_BITS(XTENSA_OPTION_BIT(opt))
 
-#define TBD() qemu_log("TBD(pc = %08x): %s:%d\n", dc->pc, __FILE__, __LINE__)
+#define TBD() qemu_log_mask(LOG_UNIMP, "TBD(pc = %08x): %s:%d\n", dc->pc, __FILE__, __LINE__)
 #define RESERVED() do { \
-        qemu_log("RESERVED(pc = %08x, %02x%02x%02x): %s:%d\n", \
-                dc->pc, b0, b1, b2, __FILE__, __LINE__); \
+        qemu_log_mask(LOG_GUEST_ERROR, "RESERVED(pc = %08x, %02x%02x%02x): %s:%d\n", \
+                      dc->pc, b0, b1, b2, __FILE__, __LINE__); \
         goto invalid_opcode; \
     } while (0)
 
@@ -989,6 +980,7 @@
     uint8_t b0 = cpu_ldub_code(env, dc->pc);
     uint8_t b1 = cpu_ldub_code(env, dc->pc + 1);
     uint8_t b2 = 0;
+    unsigned len = xtensa_op0_insn_len(OP0);
 
     static const uint32_t B4CONST[] = {
         0xffffffff, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 16, 32, 64, 128, 256
@@ -998,13 +990,19 @@
         32768, 65536, 2, 3, 4, 5, 6, 7, 8, 10, 12, 16, 32, 64, 128, 256
     };
 
-    if (OP0 >= 8) {
-        dc->next_pc = dc->pc + 2;
+    switch (len) {
+    case 2:
         HAS_OPTION(XTENSA_OPTION_CODE_DENSITY);
-    } else {
-        dc->next_pc = dc->pc + 3;
+        break;
+
+    case 3:
         b2 = cpu_ldub_code(env, dc->pc + 2);
+        break;
+
+    default:
+        RESERVED();
     }
+    dc->next_pc = dc->pc + len;
 
     switch (OP0) {
     case 0: /*QRST*/
@@ -1031,8 +1029,9 @@
                         switch (CALLX_N) {
                         case 0: /*RET*/
                         case 2: /*JX*/
-                            gen_window_check1(dc, CALLX_S);
-                            gen_jump(dc, cpu_R[CALLX_S]);
+                            if (gen_window_check1(dc, CALLX_S)) {
+                                gen_jump(dc, cpu_R[CALLX_S]);
+                            }
                             break;
 
                         case 1: /*RETWw*/
@@ -1053,7 +1052,9 @@
                         break;
 
                     case 3: /*CALLX*/
-                        gen_window_check2(dc, CALLX_S, CALLX_N << 2);
+                        if (!gen_window_check2(dc, CALLX_S, CALLX_N << 2)) {
+                            break;
+                        }
                         switch (CALLX_N) {
                         case 0: /*CALLX0*/
                             {
@@ -1084,8 +1085,7 @@
 
                 case 1: /*MOVSPw*/
                     HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                    gen_window_check2(dc, RRR_T, RRR_S);
-                    {
+                    if (gen_window_check2(dc, RRR_T, RRR_S)) {
                         TCGv_i32 pc = tcg_const_i32(dc->pc);
                         gen_advance_ccount(dc);
                         gen_helper_movsp(cpu_env, pc);
@@ -1133,10 +1133,11 @@
                         HAS_OPTION(XTENSA_OPTION_EXCEPTION);
                         switch (RRR_S) {
                         case 0: /*RFEx*/
-                            gen_check_privilege(dc);
-                            tcg_gen_andi_i32(cpu_SR[PS], cpu_SR[PS], ~PS_EXCM);
-                            gen_helper_check_interrupts(cpu_env);
-                            gen_jump(dc, cpu_SR[EPC1]);
+                            if (gen_check_privilege(dc)) {
+                                tcg_gen_andi_i32(cpu_SR[PS], cpu_SR[PS], ~PS_EXCM);
+                                gen_helper_check_interrupts(cpu_env);
+                                gen_jump(dc, cpu_SR[EPC1]);
+                            }
                             break;
 
                         case 1: /*RFUEx*/
@@ -1144,16 +1145,16 @@
                             break;
 
                         case 2: /*RFDEx*/
-                            gen_check_privilege(dc);
-                            gen_jump(dc, cpu_SR[
-                                    dc->config->ndepc ? DEPC : EPC1]);
+                            if (gen_check_privilege(dc)) {
+                                gen_jump(dc, cpu_SR[
+                                         dc->config->ndepc ? DEPC : EPC1]);
+                            }
                             break;
 
                         case 4: /*RFWOw*/
                         case 5: /*RFWUw*/
                             HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                            gen_check_privilege(dc);
-                            {
+                            if (gen_check_privilege(dc)) {
                                 TCGv_i32 tmp = tcg_const_i32(1);
 
                                 tcg_gen_andi_i32(
@@ -1185,13 +1186,14 @@
                     case 1: /*RFIx*/
                         HAS_OPTION(XTENSA_OPTION_HIGH_PRIORITY_INTERRUPT);
                         if (RRR_S >= 2 && RRR_S <= dc->config->nlevel) {
-                            gen_check_privilege(dc);
-                            tcg_gen_mov_i32(cpu_SR[PS],
-                                    cpu_SR[EPS2 + RRR_S - 2]);
-                            gen_helper_check_interrupts(cpu_env);
-                            gen_jump(dc, cpu_SR[EPC1 + RRR_S - 1]);
+                            if (gen_check_privilege(dc)) {
+                                tcg_gen_mov_i32(cpu_SR[PS],
+                                                cpu_SR[EPS2 + RRR_S - 2]);
+                                gen_helper_check_interrupts(cpu_env);
+                                gen_jump(dc, cpu_SR[EPC1 + RRR_S - 1]);
+                            }
                         } else {
-                            qemu_log("RFI %d is illegal\n", RRR_S);
+                            qemu_log_mask(LOG_GUEST_ERROR, "RFI %d is illegal\n", RRR_S);
                             gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
                         }
                         break;
@@ -1222,11 +1224,12 @@
                         break;
 
                     case 1: /*SIMCALL*/
-                        if (semihosting_enabled) {
-                            gen_check_privilege(dc);
-                            gen_helper_simcall(cpu_env);
+                        if (semihosting_enabled()) {
+                            if (gen_check_privilege(dc)) {
+                                gen_helper_simcall(cpu_env);
+                            }
                         } else {
-                            qemu_log("SIMCALL but semihosting is disabled\n");
+                            qemu_log_mask(LOG_GUEST_ERROR, "SIMCALL but semihosting is disabled\n");
                             gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
                         }
                         break;
@@ -1239,19 +1242,21 @@
 
                 case 6: /*RSILx*/
                     HAS_OPTION(XTENSA_OPTION_INTERRUPT);
-                    gen_check_privilege(dc);
-                    gen_window_check1(dc, RRR_T);
-                    tcg_gen_mov_i32(cpu_R[RRR_T], cpu_SR[PS]);
-                    tcg_gen_andi_i32(cpu_SR[PS], cpu_SR[PS], ~PS_INTLEVEL);
-                    tcg_gen_ori_i32(cpu_SR[PS], cpu_SR[PS], RRR_S);
-                    gen_helper_check_interrupts(cpu_env);
-                    gen_jumpi_check_loop_end(dc, 0);
+                    if (gen_check_privilege(dc) &&
+                        gen_window_check1(dc, RRR_T)) {
+                        tcg_gen_mov_i32(cpu_R[RRR_T], cpu_SR[PS]);
+                        tcg_gen_andi_i32(cpu_SR[PS], cpu_SR[PS], ~PS_INTLEVEL);
+                        tcg_gen_ori_i32(cpu_SR[PS], cpu_SR[PS], RRR_S);
+                        gen_helper_check_interrupts(cpu_env);
+                        gen_jumpi_check_loop_end(dc, 0);
+                    }
                     break;
 
                 case 7: /*WAITIx*/
                     HAS_OPTION(XTENSA_OPTION_INTERRUPT);
-                    gen_check_privilege(dc);
-                    gen_waiti(dc, RRR_S);
+                    if (gen_check_privilege(dc)) {
+                        gen_waiti(dc, RRR_S);
+                    }
                     break;
 
                 case 8: /*ANY4p*/
@@ -1287,35 +1292,39 @@
                 break;
 
             case 1: /*AND*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                tcg_gen_and_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
+                    tcg_gen_and_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                }
                 break;
 
             case 2: /*OR*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                tcg_gen_or_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
+                    tcg_gen_or_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                }
                 break;
 
             case 3: /*XOR*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                tcg_gen_xor_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
+                    tcg_gen_xor_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                }
                 break;
 
             case 4: /*ST1*/
                 switch (RRR_R) {
                 case 0: /*SSR*/
-                    gen_window_check1(dc, RRR_S);
-                    gen_right_shift_sar(dc, cpu_R[RRR_S]);
+                    if (gen_window_check1(dc, RRR_S)) {
+                        gen_right_shift_sar(dc, cpu_R[RRR_S]);
+                    }
                     break;
 
                 case 1: /*SSL*/
-                    gen_window_check1(dc, RRR_S);
-                    gen_left_shift_sar(dc, cpu_R[RRR_S]);
+                    if (gen_window_check1(dc, RRR_S)) {
+                        gen_left_shift_sar(dc, cpu_R[RRR_S]);
+                    }
                     break;
 
                 case 2: /*SSA8L*/
-                    gen_window_check1(dc, RRR_S);
-                    {
+                    if (gen_window_check1(dc, RRR_S)) {
                         TCGv_i32 tmp = tcg_temp_new_i32();
                         tcg_gen_shli_i32(tmp, cpu_R[RRR_S], 3);
                         gen_right_shift_sar(dc, tmp);
@@ -1324,8 +1333,7 @@
                     break;
 
                 case 3: /*SSA8B*/
-                    gen_window_check1(dc, RRR_S);
-                    {
+                    if (gen_window_check1(dc, RRR_S)) {
                         TCGv_i32 tmp = tcg_temp_new_i32();
                         tcg_gen_shli_i32(tmp, cpu_R[RRR_S], 3);
                         gen_left_shift_sar(dc, tmp);
@@ -1352,26 +1360,28 @@
 
                 case 8: /*ROTWw*/
                     HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                    gen_check_privilege(dc);
-                    {
+                    if (gen_check_privilege(dc)) {
                         TCGv_i32 tmp = tcg_const_i32(
                                 RRR_T | ((RRR_T & 8) ? 0xfffffff0 : 0));
                         gen_helper_rotw(cpu_env, tmp);
                         tcg_temp_free(tmp);
-                        reset_used_window(dc);
+                        /* This can change tb->flags, so exit tb */
+                        gen_jumpi_check_loop_end(dc, -1);
                     }
                     break;
 
                 case 14: /*NSAu*/
                     HAS_OPTION(XTENSA_OPTION_MISC_OP_NSA);
-                    gen_window_check2(dc, RRR_S, RRR_T);
-                    gen_helper_nsa(cpu_R[RRR_T], cpu_R[RRR_S]);
+                    if (gen_window_check2(dc, RRR_S, RRR_T)) {
+                        gen_helper_nsa(cpu_R[RRR_T], cpu_R[RRR_S]);
+                    }
                     break;
 
                 case 15: /*NSAUu*/
                     HAS_OPTION(XTENSA_OPTION_MISC_OP_NSA);
-                    gen_window_check2(dc, RRR_S, RRR_T);
-                    gen_helper_nsau(cpu_R[RRR_T], cpu_R[RRR_S]);
+                    if (gen_window_check2(dc, RRR_S, RRR_T)) {
+                        gen_helper_nsau(cpu_R[RRR_T], cpu_R[RRR_S]);
+                    }
                     break;
 
                 default: /*reserved*/
@@ -1385,9 +1395,8 @@
                         XTENSA_OPTION_BIT(XTENSA_OPTION_MMU) |
                         XTENSA_OPTION_BIT(XTENSA_OPTION_REGION_PROTECTION) |
                         XTENSA_OPTION_BIT(XTENSA_OPTION_REGION_TRANSLATION));
-                gen_check_privilege(dc);
-                gen_window_check2(dc, RRR_S, RRR_T);
-                {
+                if (gen_check_privilege(dc) &&
+                    gen_window_check2(dc, RRR_S, RRR_T)) {
                     TCGv_i32 dtlb = tcg_const_i32((RRR_R & 8) != 0);
 
                     switch (RRR_R & 7) {
@@ -1430,7 +1439,9 @@
                 break;
 
             case 6: /*RT0*/
-                gen_window_check2(dc, RRR_R, RRR_T);
+                if (!gen_window_check2(dc, RRR_R, RRR_T)) {
+                    break;
+                }
                 switch (RRR_S) {
                 case 0: /*NEG*/
                     tcg_gen_neg_i32(cpu_R[RRR_R], cpu_R[RRR_T]);
@@ -1460,15 +1471,15 @@
                 break;
 
             case 8: /*ADD*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                tcg_gen_add_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
+                    tcg_gen_add_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                }
                 break;
 
             case 9: /*ADD**/
             case 10:
             case 11:
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     TCGv_i32 tmp = tcg_temp_new_i32();
                     tcg_gen_shli_i32(tmp, cpu_R[RRR_S], OP2 - 8);
                     tcg_gen_add_i32(cpu_R[RRR_R], tmp, cpu_R[RRR_T]);
@@ -1477,15 +1488,15 @@
                 break;
 
             case 12: /*SUB*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                tcg_gen_sub_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
+                    tcg_gen_sub_i32(cpu_R[RRR_R], cpu_R[RRR_S], cpu_R[RRR_T]);
+                }
                 break;
 
             case 13: /*SUB**/
             case 14:
             case 15:
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     TCGv_i32 tmp = tcg_temp_new_i32();
                     tcg_gen_shli_i32(tmp, cpu_R[RRR_S], OP2 - 12);
                     tcg_gen_sub_i32(cpu_R[RRR_R], tmp, cpu_R[RRR_T]);
@@ -1499,31 +1510,32 @@
             switch (OP2) {
             case 0: /*SLLI*/
             case 1:
-                gen_window_check2(dc, RRR_R, RRR_S);
-                tcg_gen_shli_i32(cpu_R[RRR_R], cpu_R[RRR_S],
-                        32 - (RRR_T | ((OP2 & 1) << 4)));
+                if (gen_window_check2(dc, RRR_R, RRR_S)) {
+                    tcg_gen_shli_i32(cpu_R[RRR_R], cpu_R[RRR_S],
+                                     32 - (RRR_T | ((OP2 & 1) << 4)));
+                }
                 break;
 
             case 2: /*SRAI*/
             case 3:
-                gen_window_check2(dc, RRR_R, RRR_T);
-                tcg_gen_sari_i32(cpu_R[RRR_R], cpu_R[RRR_T],
-                        RRR_S | ((OP2 & 1) << 4));
+                if (gen_window_check2(dc, RRR_R, RRR_T)) {
+                    tcg_gen_sari_i32(cpu_R[RRR_R], cpu_R[RRR_T],
+                                     RRR_S | ((OP2 & 1) << 4));
+                }
                 break;
 
             case 4: /*SRLI*/
-                gen_window_check2(dc, RRR_R, RRR_T);
-                tcg_gen_shri_i32(cpu_R[RRR_R], cpu_R[RRR_T], RRR_S);
+                if (gen_window_check2(dc, RRR_R, RRR_T)) {
+                    tcg_gen_shri_i32(cpu_R[RRR_R], cpu_R[RRR_T], RRR_S);
+                }
                 break;
 
             case 6: /*XSR*/
-                if (gen_check_sr(dc, RSR_SR, SR_X)) {
+                if (gen_check_sr(dc, RSR_SR, SR_X) &&
+                    (RSR_SR < 64 || gen_check_privilege(dc)) &&
+                    gen_window_check1(dc, RRR_T)) {
                     TCGv_i32 tmp = tcg_temp_new_i32();
 
-                    if (RSR_SR >= 64) {
-                        gen_check_privilege(dc);
-                    }
-                    gen_window_check1(dc, RRR_T);
                     tcg_gen_mov_i32(tmp, cpu_R[RRR_T]);
                     gen_rsr(dc, cpu_R[RRR_T], RSR_SR);
                     gen_wsr(dc, RSR_SR, tmp);
@@ -1539,7 +1551,7 @@
                     TCGv_i64 tmp = tcg_temp_new_i64(); \
                     tcg_gen_extu_i32_i64(tmp, reg); \
                     tcg_gen_##cmd##_i64(v, v, tmp); \
-                    tcg_gen_trunc_i64_i32(cpu_R[RRR_R], v); \
+                    tcg_gen_extrl_i64_i32(cpu_R[RRR_R], v); \
                     tcg_temp_free_i64(v); \
                     tcg_temp_free_i64(tmp); \
                 } while (0)
@@ -1547,8 +1559,7 @@
 #define gen_shift(cmd) gen_shift_reg(cmd, cpu_SR[SAR])
 
             case 8: /*SRC*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     TCGv_i64 v = tcg_temp_new_i64();
                     tcg_gen_concat_i32_i64(v, cpu_R[RRR_T], cpu_R[RRR_S]);
                     gen_shift(shr);
@@ -1556,7 +1567,9 @@
                 break;
 
             case 9: /*SRL*/
-                gen_window_check2(dc, RRR_R, RRR_T);
+                if (!gen_window_check2(dc, RRR_R, RRR_T)) {
+                    break;
+                }
                 if (dc->sar_5bit) {
                     tcg_gen_shr_i32(cpu_R[RRR_R], cpu_R[RRR_T], cpu_SR[SAR]);
                 } else {
@@ -1567,7 +1580,9 @@
                 break;
 
             case 10: /*SLL*/
-                gen_window_check2(dc, RRR_R, RRR_S);
+                if (!gen_window_check2(dc, RRR_R, RRR_S)) {
+                    break;
+                }
                 if (dc->sar_m32_5bit) {
                     tcg_gen_shl_i32(cpu_R[RRR_R], cpu_R[RRR_S], dc->sar_m32);
                 } else {
@@ -1582,7 +1597,9 @@
                 break;
 
             case 11: /*SRA*/
-                gen_window_check2(dc, RRR_R, RRR_T);
+                if (!gen_window_check2(dc, RRR_R, RRR_T)) {
+                    break;
+                }
                 if (dc->sar_5bit) {
                     tcg_gen_sar_i32(cpu_R[RRR_R], cpu_R[RRR_T], cpu_SR[SAR]);
                 } else {
@@ -1596,8 +1613,7 @@
 
             case 12: /*MUL16U*/
                 HAS_OPTION(XTENSA_OPTION_16_BIT_IMUL);
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     TCGv_i32 v1 = tcg_temp_new_i32();
                     TCGv_i32 v2 = tcg_temp_new_i32();
                     tcg_gen_ext16u_i32(v1, cpu_R[RRR_S]);
@@ -1610,8 +1626,7 @@
 
             case 13: /*MUL16S*/
                 HAS_OPTION(XTENSA_OPTION_16_BIT_IMUL);
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     TCGv_i32 v1 = tcg_temp_new_i32();
                     TCGv_i32 v2 = tcg_temp_new_i32();
                     tcg_gen_ext16s_i32(v1, cpu_R[RRR_S]);
@@ -1629,13 +1644,13 @@
             break;
 
         case 2: /*RST2*/
-            if (OP2 >= 8) {
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
+            if (OP2 >= 8 && !gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
+                break;
             }
 
             if (OP2 >= 12) {
                 HAS_OPTION(XTENSA_OPTION_32_BIT_IDIV);
-                int label = gen_new_label();
+                TCGLabel *label = gen_new_label();
                 tcg_gen_brcondi_i32(TCG_COND_NE, cpu_R[RRR_T], 0, label);
                 gen_exception_cause(dc, INTEGER_DIVIDE_BY_ZERO_CAUSE);
                 gen_set_label(label);
@@ -1707,8 +1722,8 @@
             case 13: /*QUOSi*/
             case 15: /*REMSi*/
                 {
-                    int label1 = gen_new_label();
-                    int label2 = gen_new_label();
+                    TCGLabel *label1 = gen_new_label();
+                    TCGLabel *label2 = gen_new_label();
 
                     tcg_gen_brcondi_i32(TCG_COND_NE, cpu_R[RRR_S], 0x80000000,
                             label1);
@@ -1742,29 +1757,24 @@
         case 3: /*RST3*/
             switch (OP2) {
             case 0: /*RSR*/
-                if (gen_check_sr(dc, RSR_SR, SR_R)) {
-                    if (RSR_SR >= 64) {
-                        gen_check_privilege(dc);
-                    }
-                    gen_window_check1(dc, RRR_T);
+                if (gen_check_sr(dc, RSR_SR, SR_R) &&
+                    (RSR_SR < 64 || gen_check_privilege(dc)) &&
+                    gen_window_check1(dc, RRR_T)) {
                     gen_rsr(dc, cpu_R[RRR_T], RSR_SR);
                 }
                 break;
 
             case 1: /*WSR*/
-                if (gen_check_sr(dc, RSR_SR, SR_W)) {
-                    if (RSR_SR >= 64) {
-                        gen_check_privilege(dc);
-                    }
-                    gen_window_check1(dc, RRR_T);
+                if (gen_check_sr(dc, RSR_SR, SR_W) &&
+                    (RSR_SR < 64 || gen_check_privilege(dc)) &&
+                    gen_window_check1(dc, RRR_T)) {
                     gen_wsr(dc, RSR_SR, cpu_R[RRR_T]);
                 }
                 break;
 
             case 2: /*SEXTu*/
                 HAS_OPTION(XTENSA_OPTION_MISC_OP_SEXT);
-                gen_window_check2(dc, RRR_R, RRR_S);
-                {
+                if (gen_window_check2(dc, RRR_R, RRR_S)) {
                     int shift = 24 - RRR_T;
 
                     if (shift == 24) {
@@ -1782,8 +1792,7 @@
 
             case 3: /*CLAMPSu*/
                 HAS_OPTION(XTENSA_OPTION_MISC_OP_CLAMPS);
-                gen_window_check2(dc, RRR_R, RRR_S);
-                {
+                if (gen_window_check2(dc, RRR_R, RRR_S)) {
                     TCGv_i32 tmp1 = tcg_temp_new_i32();
                     TCGv_i32 tmp2 = tcg_temp_new_i32();
                     TCGv_i32 zero = tcg_const_i32(0);
@@ -1808,8 +1817,7 @@
             case 6: /*MINUu*/
             case 7: /*MAXUu*/
                 HAS_OPTION(XTENSA_OPTION_MISC_OP_MINMAX);
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     static const TCGCond cond[] = {
                         TCG_COND_LE,
                         TCG_COND_GE,
@@ -1826,8 +1834,7 @@
             case 9: /*MOVNEZ*/
             case 10: /*MOVLTZ*/
             case 11: /*MOVGEZ*/
-                gen_window_check3(dc, RRR_R, RRR_S, RRR_T);
-                {
+                if (gen_window_check3(dc, RRR_R, RRR_S, RRR_T)) {
                     static const TCGCond cond[] = {
                         TCG_COND_EQ,
                         TCG_COND_NE,
@@ -1845,8 +1852,7 @@
             case 12: /*MOVFp*/
             case 13: /*MOVTp*/
                 HAS_OPTION(XTENSA_OPTION_BOOLEAN);
-                gen_window_check2(dc, RRR_R, RRR_S);
-                {
+                if (gen_window_check2(dc, RRR_R, RRR_S)) {
                     TCGv_i32 zero = tcg_const_i32(0);
                     TCGv_i32 tmp = tcg_temp_new_i32();
 
@@ -1861,25 +1867,25 @@
                 break;
 
             case 14: /*RUR*/
-                gen_window_check1(dc, RRR_R);
-                {
+                if (gen_window_check1(dc, RRR_R)) {
                     int st = (RRR_S << 4) + RRR_T;
                     if (uregnames[st].name) {
                         tcg_gen_mov_i32(cpu_R[RRR_R], cpu_UR[st]);
                     } else {
-                        qemu_log("RUR %d not implemented, ", st);
+                        qemu_log_mask(LOG_UNIMP, "RUR %d not implemented, ", st);
                         TBD();
                     }
                 }
                 break;
 
             case 15: /*WUR*/
-                gen_window_check1(dc, RRR_T);
-                if (uregnames[RSR_SR].name) {
-                    gen_wur(RSR_SR, cpu_R[RRR_T]);
-                } else {
-                    qemu_log("WUR %d not implemented, ", RSR_SR);
-                    TBD();
+                if (gen_window_check1(dc, RRR_T)) {
+                    if (uregnames[RSR_SR].name) {
+                        gen_wur(RSR_SR, cpu_R[RRR_T]);
+                    } else {
+                        qemu_log_mask(LOG_UNIMP, "WUR %d not implemented, ", RSR_SR);
+                        TBD();
+                    }
                 }
                 break;
 
@@ -1888,8 +1894,7 @@
 
         case 4: /*EXTUI*/
         case 5:
-            gen_window_check2(dc, RRR_R, RRR_T);
-            {
+            if (gen_window_check2(dc, RRR_R, RRR_T)) {
                 int shiftimm = RRR_S | ((OP1 & 1) << 4);
                 int maskimm = (1 << (OP2 + 1)) - 1;
 
@@ -1915,9 +1920,8 @@
             case 4: /*SSXf*/
             case 5: /*SSXUf*/
                 HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
-                gen_window_check2(dc, RRR_S, RRR_T);
-                gen_check_cpenable(dc, 0);
-                {
+                if (gen_window_check2(dc, RRR_S, RRR_T) &&
+                    gen_check_cpenable(dc, 0)) {
                     TCGv_i32 addr = tcg_temp_new_i32();
                     tcg_gen_add_i32(addr, cpu_R[RRR_S], cpu_R[RRR_T]);
                     gen_load_store_alignment(dc, 2, addr, false);
@@ -1940,12 +1944,14 @@
             break;
 
         case 9: /*LSC4*/
-            gen_window_check2(dc, RRR_S, RRR_T);
+            if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                break;
+            }
             switch (OP2) {
             case 0: /*L32E*/
                 HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                gen_check_privilege(dc);
-                {
+                if (gen_check_privilege(dc) &&
+                    gen_window_check2(dc, RRR_S, RRR_T)) {
                     TCGv_i32 addr = tcg_temp_new_i32();
                     tcg_gen_addi_i32(addr, cpu_R[RRR_S],
                             (0xffffffc0 | (RRR_R << 2)));
@@ -1956,8 +1962,8 @@
 
             case 4: /*S32E*/
                 HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-                gen_check_privilege(dc);
-                {
+                if (gen_check_privilege(dc) &&
+                    gen_window_check2(dc, RRR_S, RRR_T)) {
                     TCGv_i32 addr = tcg_temp_new_i32();
                     tcg_gen_addi_i32(addr, cpu_R[RRR_S],
                             (0xffffffc0 | (RRR_R << 2)));
@@ -1966,6 +1972,17 @@
                 }
                 break;
 
+            case 5: /*S32N*/
+                if (gen_window_check2(dc, RRI4_S, RRI4_T)) {
+                    TCGv_i32 addr = tcg_temp_new_i32();
+
+                    tcg_gen_addi_i32(addr, cpu_R[RRI4_S], RRI4_IMM4 << 2);
+                    gen_load_store_alignment(dc, 2, addr, false);
+                    tcg_gen_qemu_st32(cpu_R[RRI4_T], addr, dc->cring);
+                    tcg_temp_free(addr);
+                }
+                break;
+
             default:
                 RESERVED();
                 break;
@@ -1973,36 +1990,53 @@
             break;
 
         case 10: /*FP0*/
+            /*DEPBITS*/
+            if (option_enabled(dc, XTENSA_OPTION_DEPBITS)) {
+                if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                    break;
+                }
+                tcg_gen_deposit_i32(cpu_R[RRR_T], cpu_R[RRR_T], cpu_R[RRR_S],
+                                    OP2, RRR_R + 1);
+                break;
+            }
+
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
             switch (OP2) {
             case 0: /*ADD.Sf*/
-                gen_check_cpenable(dc, 0);
-                gen_helper_add_s(cpu_FR[RRR_R], cpu_env,
-                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                if (gen_check_cpenable(dc, 0)) {
+                    gen_helper_add_s(cpu_FR[RRR_R], cpu_env,
+                                     cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                }
                 break;
 
             case 1: /*SUB.Sf*/
-                gen_check_cpenable(dc, 0);
-                gen_helper_sub_s(cpu_FR[RRR_R], cpu_env,
-                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                if (gen_check_cpenable(dc, 0)) {
+                    gen_helper_sub_s(cpu_FR[RRR_R], cpu_env,
+                                     cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                }
                 break;
 
             case 2: /*MUL.Sf*/
-                gen_check_cpenable(dc, 0);
-                gen_helper_mul_s(cpu_FR[RRR_R], cpu_env,
-                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                if (gen_check_cpenable(dc, 0)) {
+                    gen_helper_mul_s(cpu_FR[RRR_R], cpu_env,
+                                     cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                }
                 break;
 
             case 4: /*MADD.Sf*/
-                gen_check_cpenable(dc, 0);
-                gen_helper_madd_s(cpu_FR[RRR_R], cpu_env,
-                        cpu_FR[RRR_R], cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                if (gen_check_cpenable(dc, 0)) {
+                    gen_helper_madd_s(cpu_FR[RRR_R], cpu_env,
+                                      cpu_FR[RRR_R], cpu_FR[RRR_S],
+                                      cpu_FR[RRR_T]);
+                }
                 break;
 
             case 5: /*MSUB.Sf*/
-                gen_check_cpenable(dc, 0);
-                gen_helper_msub_s(cpu_FR[RRR_R], cpu_env,
-                        cpu_FR[RRR_R], cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                if (gen_check_cpenable(dc, 0)) {
+                    gen_helper_msub_s(cpu_FR[RRR_R], cpu_env,
+                                      cpu_FR[RRR_R], cpu_FR[RRR_S],
+                                      cpu_FR[RRR_T]);
+                }
                 break;
 
             case 8: /*ROUND.Sf*/
@@ -2010,9 +2044,8 @@
             case 10: /*FLOOR.Sf*/
             case 11: /*CEIL.Sf*/
             case 14: /*UTRUNC.Sf*/
-                gen_window_check1(dc, RRR_R);
-                gen_check_cpenable(dc, 0);
-                {
+                if (gen_window_check1(dc, RRR_R) &&
+                    gen_check_cpenable(dc, 0)) {
                     static const unsigned rounding_mode_const[] = {
                         float_round_nearest_even,
                         float_round_to_zero,
@@ -2039,9 +2072,8 @@
 
             case 12: /*FLOAT.Sf*/
             case 13: /*UFLOAT.Sf*/
-                gen_window_check1(dc, RRR_S);
-                gen_check_cpenable(dc, 0);
-                {
+                if (gen_window_check1(dc, RRR_S) &&
+                    gen_check_cpenable(dc, 0)) {
                     TCGv_i32 scale = tcg_const_i32(-RRR_T);
 
                     if (OP2 == 13) {
@@ -2058,30 +2090,35 @@
             case 15: /*FP1OP*/
                 switch (RRR_T) {
                 case 0: /*MOV.Sf*/
-                    gen_check_cpenable(dc, 0);
-                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    if (gen_check_cpenable(dc, 0)) {
+                        tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    }
                     break;
 
                 case 1: /*ABS.Sf*/
-                    gen_check_cpenable(dc, 0);
-                    gen_helper_abs_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    if (gen_check_cpenable(dc, 0)) {
+                        gen_helper_abs_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    }
                     break;
 
                 case 4: /*RFRf*/
-                    gen_window_check1(dc, RRR_R);
-                    gen_check_cpenable(dc, 0);
-                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_FR[RRR_S]);
+                    if (gen_window_check1(dc, RRR_R) &&
+                        gen_check_cpenable(dc, 0)) {
+                        tcg_gen_mov_i32(cpu_R[RRR_R], cpu_FR[RRR_S]);
+                    }
                     break;
 
                 case 5: /*WFRf*/
-                    gen_window_check1(dc, RRR_S);
-                    gen_check_cpenable(dc, 0);
-                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_R[RRR_S]);
+                    if (gen_window_check1(dc, RRR_S) &&
+                        gen_check_cpenable(dc, 0)) {
+                        tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_R[RRR_S]);
+                    }
                     break;
 
                 case 6: /*NEG.Sf*/
-                    gen_check_cpenable(dc, 0);
-                    gen_helper_neg_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    if (gen_check_cpenable(dc, 0)) {
+                        gen_helper_neg_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    }
                     break;
 
                 default: /*reserved*/
@@ -2097,15 +2134,26 @@
             break;
 
         case 11: /*FP1*/
+            /*DEPBITS*/
+            if (option_enabled(dc, XTENSA_OPTION_DEPBITS)) {
+                if (!gen_window_check2(dc, RRR_S, RRR_T)) {
+                    break;
+                }
+                tcg_gen_deposit_i32(cpu_R[RRR_T], cpu_R[RRR_T], cpu_R[RRR_S],
+                                    OP2 + 16, RRR_R + 1);
+                break;
+            }
+
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
 
 #define gen_compare(rel, br, a, b) \
     do { \
-        TCGv_i32 bit = tcg_const_i32(1 << br); \
-        \
-        gen_check_cpenable(dc, 0); \
-        gen_helper_##rel(cpu_env, bit, cpu_FR[a], cpu_FR[b]); \
-        tcg_temp_free(bit); \
+        if (gen_check_cpenable(dc, 0)) { \
+            TCGv_i32 bit = tcg_const_i32(1 << br); \
+            \
+            gen_helper_##rel(cpu_env, bit, cpu_FR[a], cpu_FR[b]); \
+            tcg_temp_free(bit); \
+        } \
     } while (0)
 
             switch (OP2) {
@@ -2143,9 +2191,8 @@
             case 9: /*MOVNEZ.Sf*/
             case 10: /*MOVLTZ.Sf*/
             case 11: /*MOVGEZ.Sf*/
-                gen_window_check1(dc, RRR_T);
-                gen_check_cpenable(dc, 0);
-                {
+                if (gen_window_check1(dc, RRR_T) &&
+                    gen_check_cpenable(dc, 0)) {
                     static const TCGCond cond[] = {
                         TCG_COND_EQ,
                         TCG_COND_NE,
@@ -2163,8 +2210,7 @@
             case 12: /*MOVF.Sf*/
             case 13: /*MOVT.Sf*/
                 HAS_OPTION(XTENSA_OPTION_BOOLEAN);
-                gen_check_cpenable(dc, 0);
-                {
+                if (gen_check_cpenable(dc, 0)) {
                     TCGv_i32 zero = tcg_const_i32(0);
                     TCGv_i32 tmp = tcg_temp_new_i32();
 
@@ -2191,8 +2237,7 @@
         break;
 
     case 1: /*L32R*/
-        gen_window_check1(dc, RRR_T);
-        {
+        if (gen_window_check1(dc, RRR_T)) {
             TCGv_i32 tmp = tcg_const_i32(
                     ((dc->tb->flags & XTENSA_TBFLAG_LITBASE) ?
                      0 : ((dc->pc + 3) & ~3)) +
@@ -2208,14 +2253,16 @@
 
     case 2: /*LSAI*/
 #define gen_load_store(type, shift) do { \
-            TCGv_i32 addr = tcg_temp_new_i32(); \
-            gen_window_check2(dc, RRI8_S, RRI8_T); \
-            tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << shift); \
-            if (shift) { \
-                gen_load_store_alignment(dc, shift, addr, false); \
+            if (gen_window_check2(dc, RRI8_S, RRI8_T)) { \
+                TCGv_i32 addr = tcg_temp_new_i32(); \
+                \
+                tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << shift); \
+                if (shift) { \
+                    gen_load_store_alignment(dc, shift, addr, false); \
+                } \
+                tcg_gen_qemu_##type(cpu_R[RRI8_T], addr, dc->cring); \
+                tcg_temp_free(addr); \
             } \
-            tcg_gen_qemu_##type(cpu_R[RRI8_T], addr, dc->cring); \
-            tcg_temp_free(addr); \
         } while (0)
 
         switch (RRI8_R) {
@@ -2244,14 +2291,15 @@
             break;
 
 #define gen_dcache_hit_test(w, shift) do { \
-            TCGv_i32 addr = tcg_temp_new_i32(); \
-            TCGv_i32 res = tcg_temp_new_i32(); \
-            gen_window_check1(dc, RRI##w##_S); \
-            tcg_gen_addi_i32(addr, cpu_R[RRI##w##_S], \
-                             RRI##w##_IMM##w << shift); \
-            tcg_gen_qemu_ld8u(res, addr, dc->cring); \
-            tcg_temp_free(addr); \
-            tcg_temp_free(res); \
+            if (gen_window_check1(dc, RRI##w##_S)) { \
+                TCGv_i32 addr = tcg_temp_new_i32(); \
+                TCGv_i32 res = tcg_temp_new_i32(); \
+                tcg_gen_addi_i32(addr, cpu_R[RRI##w##_S], \
+                                 RRI##w##_IMM##w << shift); \
+                tcg_gen_qemu_ld8u(res, addr, dc->cring); \
+                tcg_temp_free(addr); \
+                tcg_temp_free(res); \
+            } \
         } while (0)
 
 #define gen_dcache_hit_test4() gen_dcache_hit_test(4, 4)
@@ -2288,45 +2336,52 @@
                 break;
 
             case 6: /*DHIc*/
-                gen_check_privilege(dc);
-                gen_dcache_hit_test8();
+                if (gen_check_privilege(dc)) {
+                    gen_dcache_hit_test8();
+                }
                 break;
 
             case 7: /*DIIc*/
-                gen_check_privilege(dc);
-                gen_window_check1(dc, RRI8_S);
+                if (gen_check_privilege(dc)) {
+                    gen_window_check1(dc, RRI8_S);
+                }
                 break;
 
             case 8: /*DCEc*/
                 switch (OP1) {
                 case 0: /*DPFLl*/
                     HAS_OPTION(XTENSA_OPTION_DCACHE_INDEX_LOCK);
-                    gen_check_privilege(dc);
-                    gen_dcache_hit_test4();
+                    if (gen_check_privilege(dc)) {
+                        gen_dcache_hit_test4();
+                    }
                     break;
 
                 case 2: /*DHUl*/
                     HAS_OPTION(XTENSA_OPTION_DCACHE_INDEX_LOCK);
-                    gen_check_privilege(dc);
-                    gen_dcache_hit_test4();
+                    if (gen_check_privilege(dc)) {
+                        gen_dcache_hit_test4();
+                    }
                     break;
 
                 case 3: /*DIUl*/
                     HAS_OPTION(XTENSA_OPTION_DCACHE_INDEX_LOCK);
-                    gen_check_privilege(dc);
-                    gen_window_check1(dc, RRI4_S);
+                    if (gen_check_privilege(dc)) {
+                        gen_window_check1(dc, RRI4_S);
+                    }
                     break;
 
                 case 4: /*DIWBc*/
                     HAS_OPTION(XTENSA_OPTION_DCACHE);
-                    gen_check_privilege(dc);
-                    gen_window_check1(dc, RRI4_S);
+                    if (gen_check_privilege(dc)) {
+                        gen_window_check1(dc, RRI4_S);
+                    }
                     break;
 
                 case 5: /*DIWBIc*/
                     HAS_OPTION(XTENSA_OPTION_DCACHE);
-                    gen_check_privilege(dc);
-                    gen_window_check1(dc, RRI4_S);
+                    if (gen_check_privilege(dc)) {
+                        gen_window_check1(dc, RRI4_S);
+                    }
                     break;
 
                 default: /*reserved*/
@@ -2341,13 +2396,14 @@
 #undef gen_dcache_hit_test8
 
 #define gen_icache_hit_test(w, shift) do { \
-            TCGv_i32 addr = tcg_temp_new_i32(); \
-            gen_window_check1(dc, RRI##w##_S); \
-            tcg_gen_movi_i32(cpu_pc, dc->pc); \
-            tcg_gen_addi_i32(addr, cpu_R[RRI##w##_S], \
-                             RRI##w##_IMM##w << shift); \
-            gen_helper_itlb_hit_test(cpu_env, addr); \
-            tcg_temp_free(addr); \
+            if (gen_window_check1(dc, RRI##w##_S)) { \
+                TCGv_i32 addr = tcg_temp_new_i32(); \
+                tcg_gen_movi_i32(cpu_pc, dc->pc); \
+                tcg_gen_addi_i32(addr, cpu_R[RRI##w##_S], \
+                                 RRI##w##_IMM##w << shift); \
+                gen_helper_itlb_hit_test(cpu_env, addr); \
+                tcg_temp_free(addr); \
+            }\
         } while (0)
 
 #define gen_icache_hit_test4() gen_icache_hit_test(4, 4)
@@ -2362,20 +2418,23 @@
                 switch (OP1) {
                 case 0: /*IPFLl*/
                     HAS_OPTION(XTENSA_OPTION_ICACHE_INDEX_LOCK);
-                    gen_check_privilege(dc);
-                    gen_icache_hit_test4();
+                    if (gen_check_privilege(dc)) {
+                        gen_icache_hit_test4();
+                    }
                     break;
 
                 case 2: /*IHUl*/
                     HAS_OPTION(XTENSA_OPTION_ICACHE_INDEX_LOCK);
-                    gen_check_privilege(dc);
-                    gen_icache_hit_test4();
+                    if (gen_check_privilege(dc)) {
+                        gen_icache_hit_test4();
+                    }
                     break;
 
                 case 3: /*IIUl*/
                     HAS_OPTION(XTENSA_OPTION_ICACHE_INDEX_LOCK);
-                    gen_check_privilege(dc);
-                    gen_window_check1(dc, RRI4_S);
+                    if (gen_check_privilege(dc)) {
+                        gen_window_check1(dc, RRI4_S);
+                    }
                     break;
 
                 default: /*reserved*/
@@ -2391,8 +2450,9 @@
 
             case 15: /*IIIc*/
                 HAS_OPTION(XTENSA_OPTION_ICACHE);
-                gen_check_privilege(dc);
-                gen_window_check1(dc, RRI8_S);
+                if (gen_check_privilege(dc)) {
+                    gen_window_check1(dc, RRI8_S);
+                }
                 break;
 
             default: /*reserved*/
@@ -2411,19 +2471,21 @@
 #undef gen_load_store
 
         case 10: /*MOVI*/
-            gen_window_check1(dc, RRI8_T);
-            tcg_gen_movi_i32(cpu_R[RRI8_T],
-                    RRI8_IMM8 | (RRI8_S << 8) |
-                    ((RRI8_S & 0x8) ? 0xfffff000 : 0));
+            if (gen_window_check1(dc, RRI8_T)) {
+                tcg_gen_movi_i32(cpu_R[RRI8_T],
+                                 RRI8_IMM8 | (RRI8_S << 8) |
+                                 ((RRI8_S & 0x8) ? 0xfffff000 : 0));
+            }
             break;
 
 #define gen_load_store_no_hw_align(type) do { \
-            TCGv_i32 addr = tcg_temp_local_new_i32(); \
-            gen_window_check2(dc, RRI8_S, RRI8_T); \
-            tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2); \
-            gen_load_store_alignment(dc, 2, addr, true); \
-            tcg_gen_qemu_##type(cpu_R[RRI8_T], addr, dc->cring); \
-            tcg_temp_free(addr); \
+            if (gen_window_check2(dc, RRI8_S, RRI8_T)) { \
+                TCGv_i32 addr = tcg_temp_local_new_i32(); \
+                tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2); \
+                gen_load_store_alignment(dc, 2, addr, true); \
+                tcg_gen_qemu_##type(cpu_R[RRI8_T], addr, dc->cring); \
+                tcg_temp_free(addr); \
+            } \
         } while (0)
 
         case 11: /*L32AIy*/
@@ -2432,20 +2494,22 @@
             break;
 
         case 12: /*ADDI*/
-            gen_window_check2(dc, RRI8_S, RRI8_T);
-            tcg_gen_addi_i32(cpu_R[RRI8_T], cpu_R[RRI8_S], RRI8_IMM8_SE);
+            if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
+                tcg_gen_addi_i32(cpu_R[RRI8_T], cpu_R[RRI8_S], RRI8_IMM8_SE);
+            }
             break;
 
         case 13: /*ADDMI*/
-            gen_window_check2(dc, RRI8_S, RRI8_T);
-            tcg_gen_addi_i32(cpu_R[RRI8_T], cpu_R[RRI8_S], RRI8_IMM8_SE << 8);
+            if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
+                tcg_gen_addi_i32(cpu_R[RRI8_T], cpu_R[RRI8_S],
+                                 RRI8_IMM8_SE << 8);
+            }
             break;
 
         case 14: /*S32C1Iy*/
             HAS_OPTION(XTENSA_OPTION_CONDITIONAL_STORE);
-            gen_window_check2(dc, RRI8_S, RRI8_T);
-            {
-                int label = gen_new_label();
+            if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
+                TCGLabel *label = gen_new_label();
                 TCGv_i32 tmp = tcg_temp_local_new_i32();
                 TCGv_i32 addr = tcg_temp_local_new_i32();
                 TCGv_i32 tpc;
@@ -2489,9 +2553,8 @@
         case 8: /*LSIUf*/
         case 12: /*SSIUf*/
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
-            gen_window_check1(dc, RRI8_S);
-            gen_check_cpenable(dc, 0);
-            {
+            if (gen_window_check1(dc, RRI8_S) &&
+                gen_check_cpenable(dc, 0)) {
                 TCGv_i32 addr = tcg_temp_new_i32();
                 tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2);
                 gen_load_store_alignment(dc, 2, addr, false);
@@ -2555,20 +2618,23 @@
             }
 
             if (op != MAC16_NONE) {
-                if (!is_m1_sr) {
-                    gen_window_check1(dc, RRR_S);
+                if (!is_m1_sr && !gen_window_check1(dc, RRR_S)) {
+                    break;
                 }
-                if (!is_m2_sr) {
-                    gen_window_check1(dc, RRR_T);
+                if (!is_m2_sr && !gen_window_check1(dc, RRR_T)) {
+                    break;
                 }
             }
 
+            if (ld_offset && !gen_window_check1(dc, RRR_S)) {
+                break;
+            }
+
             {
                 TCGv_i32 vaddr = tcg_temp_new_i32();
                 TCGv_i32 mem32 = tcg_temp_new_i32();
 
                 if (ld_offset) {
-                    gen_window_check1(dc, RRR_S);
                     tcg_gen_addi_i32(vaddr, cpu_R[RRR_S], ld_offset);
                     gen_load_store_alignment(dc, 2, vaddr, false);
                     tcg_gen_qemu_ld32u(mem32, vaddr, dc->cring);
@@ -2632,9 +2698,10 @@
         case 2: /*CALL8w*/
         case 3: /*CALL12w*/
             HAS_OPTION(XTENSA_OPTION_WINDOWED_REGISTER);
-            gen_window_check1(dc, CALL_N << 2);
-            gen_callwi(dc, CALL_N,
-                    (dc->pc & ~3) + (CALL_OFFSET_SE << 2) + 4, 0);
+            if (gen_window_check1(dc, CALL_N << 2)) {
+                gen_callwi(dc, CALL_N,
+                           (dc->pc & ~3) + (CALL_OFFSET_SE << 2) + 4, 0);
+            }
             break;
         }
         break;
@@ -2646,8 +2713,7 @@
             break;
 
         case 1: /*BZ*/
-            gen_window_check1(dc, BRI12_S);
-            {
+            if (gen_window_check1(dc, BRI12_S)) {
                 static const TCGCond cond[] = {
                     TCG_COND_EQ, /*BEQZ*/
                     TCG_COND_NE, /*BNEZ*/
@@ -2661,8 +2727,7 @@
             break;
 
         case 2: /*BI0*/
-            gen_window_check1(dc, BRI8_S);
-            {
+            if (gen_window_check1(dc, BRI8_S)) {
                 static const TCGCond cond[] = {
                     TCG_COND_EQ, /*BEQI*/
                     TCG_COND_NE, /*BNEI*/
@@ -2688,7 +2753,8 @@
                     tcg_temp_free(imm);
                     tcg_temp_free(s);
                     tcg_temp_free(pc);
-                    reset_used_window(dc);
+                    /* This can change tb->flags, so exit tb */
+                    gen_jumpi_check_loop_end(dc, -1);
                 }
                 break;
 
@@ -2711,8 +2777,7 @@
                 case 9: /*LOOPNEZ*/
                 case 10: /*LOOPGTZ*/
                     HAS_OPTION(XTENSA_OPTION_LOOP);
-                    gen_window_check1(dc, RRI8_S);
-                    {
+                    if (gen_window_check1(dc, RRI8_S)) {
                         uint32_t lend = dc->pc + RRI8_IMM8 + 4;
                         TCGv_i32 tmp = tcg_const_i32(lend);
 
@@ -2722,7 +2787,7 @@
                         tcg_temp_free(tmp);
 
                         if (BRI8_R > 8) {
-                            int label = gen_new_label();
+                            TCGLabel *label = gen_new_label();
                             tcg_gen_brcondi_i32(
                                     BRI8_R == 9 ? TCG_COND_NE : TCG_COND_GT,
                                     cpu_R[RRI8_S], 0, label);
@@ -2743,9 +2808,11 @@
 
             case 2: /*BLTUI*/
             case 3: /*BGEUI*/
-                gen_window_check1(dc, BRI8_S);
-                gen_brcondi(dc, BRI8_M == 2 ? TCG_COND_LTU : TCG_COND_GEU,
-                        cpu_R[BRI8_S], B4CONSTU[BRI8_R], 4 + BRI8_IMM8_SE);
+                if (gen_window_check1(dc, BRI8_S)) {
+                    gen_brcondi(dc, BRI8_M == 2 ? TCG_COND_LTU : TCG_COND_GEU,
+                                cpu_R[BRI8_S], B4CONSTU[BRI8_R],
+                                4 + BRI8_IMM8_SE);
+                }
                 break;
             }
             break;
@@ -2759,8 +2826,7 @@
 
             switch (RRI8_R & 7) {
             case 0: /*BNONE*/ /*BANY*/
-                gen_window_check2(dc, RRI8_S, RRI8_T);
-                {
+                if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
                     TCGv_i32 tmp = tcg_temp_new_i32();
                     tcg_gen_and_i32(tmp, cpu_R[RRI8_S], cpu_R[RRI8_T]);
                     gen_brcondi(dc, eq_ne, tmp, 0, 4 + RRI8_IMM8_SE);
@@ -2771,8 +2837,7 @@
             case 1: /*BEQ*/ /*BNE*/
             case 2: /*BLT*/ /*BGE*/
             case 3: /*BLTU*/ /*BGEU*/
-                gen_window_check2(dc, RRI8_S, RRI8_T);
-                {
+                if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
                     static const TCGCond cond[] = {
                         [1] = TCG_COND_EQ,
                         [2] = TCG_COND_LT,
@@ -2787,8 +2852,7 @@
                 break;
 
             case 4: /*BALL*/ /*BNALL*/
-                gen_window_check2(dc, RRI8_S, RRI8_T);
-                {
+                if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
                     TCGv_i32 tmp = tcg_temp_new_i32();
                     tcg_gen_and_i32(tmp, cpu_R[RRI8_S], cpu_R[RRI8_T]);
                     gen_brcond(dc, eq_ne, tmp, cpu_R[RRI8_T],
@@ -2798,8 +2862,7 @@
                 break;
 
             case 5: /*BBC*/ /*BBS*/
-                gen_window_check2(dc, RRI8_S, RRI8_T);
-                {
+                if (gen_window_check2(dc, RRI8_S, RRI8_T)) {
 #ifdef TARGET_WORDS_BIGENDIAN
                     TCGv_i32 bit = tcg_const_i32(0x80000000);
 #else
@@ -2821,8 +2884,7 @@
 
             case 6: /*BBCI*/ /*BBSI*/
             case 7:
-                gen_window_check1(dc, RRI8_S);
-                {
+                if (gen_window_check1(dc, RRI8_S)) {
                     TCGv_i32 tmp = tcg_temp_new_i32();
                     tcg_gen_andi_i32(tmp, cpu_R[RRI8_S],
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -2840,12 +2902,13 @@
         break;
 
 #define gen_narrow_load_store(type) do { \
-            TCGv_i32 addr = tcg_temp_new_i32(); \
-            gen_window_check2(dc, RRRN_S, RRRN_T); \
-            tcg_gen_addi_i32(addr, cpu_R[RRRN_S], RRRN_R << 2); \
-            gen_load_store_alignment(dc, 2, addr, false); \
-            tcg_gen_qemu_##type(cpu_R[RRRN_T], addr, dc->cring); \
-            tcg_temp_free(addr); \
+            if (gen_window_check2(dc, RRRN_S, RRRN_T)) { \
+                TCGv_i32 addr = tcg_temp_new_i32(); \
+                tcg_gen_addi_i32(addr, cpu_R[RRRN_S], RRRN_R << 2); \
+                gen_load_store_alignment(dc, 2, addr, false); \
+                tcg_gen_qemu_##type(cpu_R[RRRN_T], addr, dc->cring); \
+                tcg_temp_free(addr); \
+            } \
         } while (0)
 
     case 8: /*L32I.Nn*/
@@ -2858,17 +2921,22 @@
 #undef gen_narrow_load_store
 
     case 10: /*ADD.Nn*/
-        gen_window_check3(dc, RRRN_R, RRRN_S, RRRN_T);
-        tcg_gen_add_i32(cpu_R[RRRN_R], cpu_R[RRRN_S], cpu_R[RRRN_T]);
+        if (gen_window_check3(dc, RRRN_R, RRRN_S, RRRN_T)) {
+            tcg_gen_add_i32(cpu_R[RRRN_R], cpu_R[RRRN_S], cpu_R[RRRN_T]);
+        }
         break;
 
     case 11: /*ADDI.Nn*/
-        gen_window_check2(dc, RRRN_R, RRRN_S);
-        tcg_gen_addi_i32(cpu_R[RRRN_R], cpu_R[RRRN_S], RRRN_T ? RRRN_T : -1);
+        if (gen_window_check2(dc, RRRN_R, RRRN_S)) {
+            tcg_gen_addi_i32(cpu_R[RRRN_R], cpu_R[RRRN_S],
+                             RRRN_T ? RRRN_T : -1);
+        }
         break;
 
     case 12: /*ST2n*/
-        gen_window_check1(dc, RRRN_S);
+        if (!gen_window_check1(dc, RRRN_S)) {
+            break;
+        }
         if (RRRN_T < 8) { /*MOVI.Nn*/
             tcg_gen_movi_i32(cpu_R[RRRN_S],
                     RRRN_R | (RRRN_T << 4) |
@@ -2884,8 +2952,9 @@
     case 13: /*ST3n*/
         switch (RRRN_R) {
         case 0: /*MOV.Nn*/
-            gen_window_check2(dc, RRRN_S, RRRN_T);
-            tcg_gen_mov_i32(cpu_R[RRRN_T], cpu_R[RRRN_S]);
+            if (gen_window_check2(dc, RRRN_S, RRRN_T)) {
+                tcg_gen_mov_i32(cpu_R[RRRN_T], cpu_R[RRRN_S]);
+            }
             break;
 
         case 15: /*S3*/
@@ -2944,25 +3013,15 @@
     return;
 
 invalid_opcode:
-    qemu_log("INVALID(pc = %08x)\n", dc->pc);
+    qemu_log_mask(LOG_GUEST_ERROR, "INVALID(pc = %08x)\n", dc->pc);
     gen_exception_cause(dc, ILLEGAL_INSTRUCTION_CAUSE);
 #undef HAS_OPTION
 }
 
-static void check_breakpoint(CPUXtensaState *env, DisasContext *dc)
+static inline unsigned xtensa_insn_len(CPUXtensaState *env, DisasContext *dc)
 {
-    CPUState *cs = CPU(xtensa_env_get_cpu(env));
-    CPUBreakpoint *bp;
-
-    if (unlikely(!QTAILQ_EMPTY(&cs->breakpoints))) {
-        QTAILQ_FOREACH(bp, &cs->breakpoints, entry) {
-            if (bp->pc == dc->pc) {
-                tcg_gen_movi_i32(cpu_pc, dc->pc);
-                gen_exception(dc, EXCP_DEBUG);
-                dc->is_jmp = DISAS_UPDATE;
-             }
-        }
-    }
+    uint8_t b0 = cpu_ldub_code(env, dc->pc);
+    return xtensa_op0_insn_len(OP0);
 }
 
 static void gen_ibreak_check(CPUXtensaState *env, DisasContext *dc)
@@ -2978,15 +3037,12 @@
     }
 }
 
-static inline
-void gen_intermediate_code_internal(XtensaCPU *cpu,
-                                    TranslationBlock *tb, bool search_pc)
+void gen_intermediate_code(CPUXtensaState *env, TranslationBlock *tb)
 {
+    XtensaCPU *cpu = xtensa_env_get_cpu(env);
     CPUState *cs = CPU(cpu);
-    CPUXtensaState *env = &cpu->env;
     DisasContext dc;
     int insn_count = 0;
-    int j, lj = -1;
     int max_insns = tb->cflags & CF_COUNT_MASK;
     uint32_t pc_start = tb->pc;
     uint32_t next_page_start =
@@ -2995,6 +3051,9 @@
     if (max_insns == 0) {
         max_insns = CF_COUNT_MASK;
     }
+    if (max_insns > TCG_MAX_INSNS) {
+        max_insns = TCG_MAX_INSNS;
+    }
 
     dc.config = env->config;
     dc.singlestep_enabled = cs->singlestep_enabled;
@@ -3010,15 +3069,16 @@
     dc.icount = tb->flags & XTENSA_TBFLAG_ICOUNT;
     dc.cpenable = (tb->flags & XTENSA_TBFLAG_CPENABLE_MASK) >>
         XTENSA_TBFLAG_CPENABLE_SHIFT;
+    dc.window = ((tb->flags & XTENSA_TBFLAG_WINDOW_MASK) >>
+                 XTENSA_TBFLAG_WINDOW_SHIFT);
 
     init_litbase(&dc);
     init_sar_tracker(&dc);
-    reset_used_window(&dc);
     if (dc.icount) {
         dc.next_icount = tcg_temp_local_new_i32();
     }
 
-    gen_tb_start();
+    gen_tb_start(tb);
 
     if (tb->flags & XTENSA_TBFLAG_EXCEPTION) {
         tcg_gen_movi_i32(cpu_pc, dc.pc);
@@ -3026,33 +3086,29 @@
     }
 
     do {
-        check_breakpoint(env, &dc);
-
-        if (search_pc) {
-            j = tcg_op_buf_count();
-            if (lj < j) {
-                lj++;
-                while (lj < j) {
-                    tcg_ctx.gen_opc_instr_start[lj++] = 0;
-                }
-            }
-            tcg_ctx.gen_opc_pc[lj] = dc.pc;
-            tcg_ctx.gen_opc_instr_start[lj] = 1;
-            tcg_ctx.gen_opc_icount[lj] = insn_count;
-        }
-
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
-            tcg_gen_debug_insn_start(dc.pc);
-        }
+        tcg_gen_insn_start(dc.pc);
+        ++insn_count;
 
         ++dc.ccount_delta;
 
-        if (insn_count + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+        if (unlikely(cpu_breakpoint_test(cs, dc.pc, BP_ANY))) {
+            tcg_gen_movi_i32(cpu_pc, dc.pc);
+            gen_exception(&dc, EXCP_DEBUG);
+            dc.is_jmp = DISAS_UPDATE;
+            /* The address covered by the breakpoint must be included in
+               [tb->pc, tb->pc + tb->size) in order to for it to be
+               properly cleared -- thus we increment the PC here so that
+               the logic setting tb->size below does the right thing.  */
+            dc.pc += 2;
+            break;
+        }
+
+        if (insn_count == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
 
         if (dc.icount) {
-            int label = gen_new_label();
+            TCGLabel *label = gen_new_label();
 
             tcg_gen_addi_i32(dc.next_icount, cpu_SR[ICOUNT], 1);
             tcg_gen_brcondi_i32(TCG_COND_NE, dc.next_icount, 0, label);
@@ -3068,7 +3124,6 @@
         }
 
         disas_xtensa_insn(env, &dc);
-        ++insn_count;
         if (dc.icount) {
             tcg_gen_mov_i32(cpu_SR[ICOUNT], dc.next_icount);
         }
@@ -3080,6 +3135,7 @@
     } while (dc.is_jmp == DISAS_NEXT &&
             insn_count < max_insns &&
             dc.pc < next_page_start &&
+            dc.pc + xtensa_insn_len(env, &dc) <= next_page_start &&
             !tcg_op_buf_full());
 
     reset_litbase(&dc);
@@ -3098,31 +3154,16 @@
     gen_tb_end(tb, insn_count);
 
 #ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
+        && qemu_log_in_addr_range(pc_start)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(env, pc_start, dc.pc - pc_start, 0);
+        log_target_disas(cs, pc_start, dc.pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
-    if (search_pc) {
-        j = tcg_op_buf_count();
-        memset(tcg_ctx.gen_opc_instr_start + lj + 1, 0,
-                (j - lj) * sizeof(tcg_ctx.gen_opc_instr_start[0]));
-    } else {
-        tb->size = dc.pc - pc_start;
-        tb->icount = insn_count;
-    }
-}
-
-void gen_intermediate_code(CPUXtensaState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(xtensa_env_get_cpu(env), tb, false);
-}
-
-void gen_intermediate_code_pc(CPUXtensaState *env, TranslationBlock *tb)
-{
-    gen_intermediate_code_internal(xtensa_env_get_cpu(env), tb, true);
+    tb->size = dc.pc - pc_start;
+    tb->icount = insn_count;
 }
 
 void xtensa_cpu_dump_state(CPUState *cs, FILE *f,
@@ -3169,13 +3210,15 @@
 
         for (i = 0; i < 16; ++i) {
             cpu_fprintf(f, "F%02d=%08x (%+10.8e)%c", i,
-                    float32_val(env->fregs[i]),
-                    *(float *)&env->fregs[i], (i % 2) == 1 ? '\n' : ' ');
+                    float32_val(env->fregs[i].f32[FP_F32_LOW]),
+                    *(float *)(env->fregs[i].f32 + FP_F32_LOW),
+                    (i % 2) == 1 ? '\n' : ' ');
         }
     }
 }
 
-void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb, int pc_pos)
+void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb,
+                          target_ulong *data)
 {
-    env->pc = tcg_ctx.gen_opc_pc[pc_pos];
+    env->pc = data[0];
 }
diff --git a/target-xtensa/xtensa-semi.c b/target-xtensa/xtensa-semi.c
index 16e9d8c..370e365 100644
--- a/target-xtensa/xtensa-semi.c
+++ b/target-xtensa/xtensa-semi.c
@@ -25,10 +25,7 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-#include <errno.h>
-#include <unistd.h>
-#include <string.h>
-#include <stddef.h>
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "exec/helper-proto.h"
 #include "qemu/log.h"
@@ -313,7 +310,7 @@
         break;
 
     default:
-        qemu_log("%s(%d): not implemented\n", __func__, regs[2]);
+        qemu_log_mask(LOG_GUEST_ERROR, "%s(%d): not implemented\n", __func__, regs[2]);
         regs[2] = -1;
         regs[3] = TARGET_ENOSYS;
         break;
diff --git a/tcg-runtime.c b/tcg-runtime.c
index 9daba69..ea2ad64 100644
--- a/tcg-runtime.c
+++ b/tcg-runtime.c
@@ -21,7 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include "qemu/host-utils.h"
 
 /* This file is compiled once, and thus we can't include the standard
diff --git a/tcg/README b/tcg/README
index a550ff1..ce8beba 100644
--- a/tcg/README
+++ b/tcg/README
@@ -314,11 +314,17 @@
 
   dest = (t1 & ~0x0f00) | ((t2 << 8) & 0x0f00)
 
-* trunc_shr_i32 t0, t1, pos
+* extrl_i64_i32 t0, t1
 
-For 64-bit hosts only, right shift the 64-bit input T1 by POS and
-truncate to 32-bit output T0.  Depending on the host, this may be
-a simple mov/shift, or may require additional canonicalization.
+For 64-bit hosts only, extract the low 32-bits of input T1 and place it
+into 32-bit output T0.  Depending on the host, this may be a simple move,
+or may require additional canonicalization.
+
+* extrh_i64_i32 t0, t1
+
+For 64-bit hosts only, extract the high 32-bits of input T1 and place it
+into 32-bit output T0.  Depending on the host, this may be a simple shift,
+or may require additional canonicalization.
 
 ********* Conditional moves
 
@@ -454,8 +460,9 @@
 
 4) Backend
 
-tcg-target.h contains the target specific definitions. tcg-target.c
-contains the target specific code.
+tcg-target.h contains the target specific definitions. tcg-target.inc.c
+contains the target specific code; it is #included by tcg/tcg.c, rather
+than being a standalone C file.
 
 4.1) Assumptions
 
@@ -466,13 +473,25 @@
 few specific operations must be implemented to allow it (see add2_i32,
 sub2_i32, brcond2_i32).
 
+On a 64 bit target, the values are transferred between 32 and 64-bit
+registers using the following ops:
+- trunc_shr_i64_i32
+- ext_i32_i64
+- extu_i32_i64
+
+They ensure that the values are correctly truncated or extended when
+moved from a 32-bit to a 64-bit register or vice-versa. Note that the
+trunc_shr_i64_i32 is an optional op. It is not necessary to implement
+it if all the following conditions are met:
+- 64-bit registers can hold 32-bit values
+- 32-bit values in a 64-bit register do not need to stay zero or
+  sign extended
+- all 32-bit TCG ops ignore the high part of 64-bit registers
+
 Floating point operations are not supported in this version. A
 previous incarnation of the code generator had full support of them,
 but it is better to concentrate on integer operations first.
 
-On a 64 bit target, no assumption is made in TCG about the storage of
-the 32 bit values in 64 bit registers.
-
 4.2) Constraints
 
 GCC like constraints are used to define the constraints of every
diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
deleted file mode 100644
index 35920e1..0000000
--- a/tcg/aarch64/tcg-target.c
+++ /dev/null
@@ -1,1883 +0,0 @@
-/*
- * Initial TCG Implementation for aarch64
- *
- * Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
- * Written by Claudio Fontana
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or
- * (at your option) any later version.
- *
- * See the COPYING file in the top-level directory for details.
- */
-
-#include "tcg-be-ldst.h"
-#include "qemu/bitops.h"
-
-/* We're going to re-use TCGType in setting of the SF bit, which controls
-   the size of the operation performed.  If we know the values match, it
-   makes things much cleaner.  */
-QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "%x0", "%x1", "%x2", "%x3", "%x4", "%x5", "%x6", "%x7",
-    "%x8", "%x9", "%x10", "%x11", "%x12", "%x13", "%x14", "%x15",
-    "%x16", "%x17", "%x18", "%x19", "%x20", "%x21", "%x22", "%x23",
-    "%x24", "%x25", "%x26", "%x27", "%x28", "%fp", "%x30", "%sp",
-};
-#endif /* NDEBUG */
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
-    TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
-    TCG_REG_X28, /* we will reserve this for GUEST_BASE if configured */
-
-    TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
-    TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
-    TCG_REG_X16, TCG_REG_X17,
-
-    TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
-    TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
-
-    /* X18 reserved by system */
-    /* X19 reserved for AREG0 */
-    /* X29 reserved as fp */
-    /* X30 reserved as temporary */
-};
-
-static const int tcg_target_call_iarg_regs[8] = {
-    TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
-    TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
-};
-static const int tcg_target_call_oarg_regs[1] = {
-    TCG_REG_X0
-};
-
-#define TCG_REG_TMP TCG_REG_X30
-
-#ifndef CONFIG_SOFTMMU
-# ifdef CONFIG_USE_GUEST_BASE
-#  define TCG_REG_GUEST_BASE TCG_REG_X28
-# else
-#  define TCG_REG_GUEST_BASE TCG_REG_XZR
-# endif
-#endif
-
-static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
-{
-    ptrdiff_t offset = target - code_ptr;
-    assert(offset == sextract64(offset, 0, 26));
-    /* read instruction, mask away previous PC_REL26 parameter contents,
-       set the proper offset, then write back the instruction. */
-    *code_ptr = deposit32(*code_ptr, 0, 26, offset);
-}
-
-static inline void reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
-{
-    ptrdiff_t offset = target - code_ptr;
-    assert(offset == sextract64(offset, 0, 19));
-    *code_ptr = deposit32(*code_ptr, 5, 19, offset);
-}
-
-static inline void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                               intptr_t value, intptr_t addend)
-{
-    assert(addend == 0);
-    switch (type) {
-    case R_AARCH64_JUMP26:
-    case R_AARCH64_CALL26:
-        reloc_pc26(code_ptr, (tcg_insn_unit *)value);
-        break;
-    case R_AARCH64_CONDBR19:
-        reloc_pc19(code_ptr, (tcg_insn_unit *)value);
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-#define TCG_CT_CONST_AIMM 0x100
-#define TCG_CT_CONST_LIMM 0x200
-#define TCG_CT_CONST_ZERO 0x400
-#define TCG_CT_CONST_MONE 0x800
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct,
-                                   const char **pct_str)
-{
-    const char *ct_str = *pct_str;
-
-    switch (ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
-        break;
-    case 'l': /* qemu_ld / qemu_st address, data_reg */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
-#ifdef CONFIG_SOFTMMU
-        /* x0 and x1 will be overwritten when reading the tlb entry,
-           and x2, and x3 for helper args, better to avoid using them. */
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X1);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X2);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
-#endif
-        break;
-    case 'A': /* Valid for arithmetic immediate (positive or negative).  */
-        ct->ct |= TCG_CT_CONST_AIMM;
-        break;
-    case 'L': /* Valid for logical immediate.  */
-        ct->ct |= TCG_CT_CONST_LIMM;
-        break;
-    case 'M': /* minus one */
-        ct->ct |= TCG_CT_CONST_MONE;
-        break;
-    case 'Z': /* zero */
-        ct->ct |= TCG_CT_CONST_ZERO;
-        break;
-    default:
-        return -1;
-    }
-
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-static inline bool is_aimm(uint64_t val)
-{
-    return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
-}
-
-static inline bool is_limm(uint64_t val)
-{
-    /* Taking a simplified view of the logical immediates for now, ignoring
-       the replication that can happen across the field.  Match bit patterns
-       of the forms
-           0....01....1
-           0..01..10..0
-       and their inverses.  */
-
-    /* Make things easier below, by testing the form with msb clear. */
-    if ((int64_t)val < 0) {
-        val = ~val;
-    }
-    if (val == 0) {
-        return false;
-    }
-    val += val & -val;
-    return (val & (val - 1)) == 0;
-}
-
-static int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                  const TCGArgConstraint *arg_ct)
-{
-    int ct = arg_ct->ct;
-
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    }
-    if (type == TCG_TYPE_I32) {
-        val = (int32_t)val;
-    }
-    if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) || is_aimm(-val))) {
-        return 1;
-    }
-    if ((ct & TCG_CT_CONST_LIMM) && is_limm(val)) {
-        return 1;
-    }
-    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
-        return 1;
-    }
-    if ((ct & TCG_CT_CONST_MONE) && val == -1) {
-        return 1;
-    }
-
-    return 0;
-}
-
-enum aarch64_cond_code {
-    COND_EQ = 0x0,
-    COND_NE = 0x1,
-    COND_CS = 0x2,     /* Unsigned greater or equal */
-    COND_HS = COND_CS, /* ALIAS greater or equal */
-    COND_CC = 0x3,     /* Unsigned less than */
-    COND_LO = COND_CC, /* ALIAS Lower */
-    COND_MI = 0x4,     /* Negative */
-    COND_PL = 0x5,     /* Zero or greater */
-    COND_VS = 0x6,     /* Overflow */
-    COND_VC = 0x7,     /* No overflow */
-    COND_HI = 0x8,     /* Unsigned greater than */
-    COND_LS = 0x9,     /* Unsigned less or equal */
-    COND_GE = 0xa,
-    COND_LT = 0xb,
-    COND_GT = 0xc,
-    COND_LE = 0xd,
-    COND_AL = 0xe,
-    COND_NV = 0xf, /* behaves like COND_AL here */
-};
-
-static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
-    [TCG_COND_EQ] = COND_EQ,
-    [TCG_COND_NE] = COND_NE,
-    [TCG_COND_LT] = COND_LT,
-    [TCG_COND_GE] = COND_GE,
-    [TCG_COND_LE] = COND_LE,
-    [TCG_COND_GT] = COND_GT,
-    /* unsigned */
-    [TCG_COND_LTU] = COND_LO,
-    [TCG_COND_GTU] = COND_HI,
-    [TCG_COND_GEU] = COND_HS,
-    [TCG_COND_LEU] = COND_LS,
-};
-
-typedef enum {
-    LDST_ST = 0,    /* store */
-    LDST_LD = 1,    /* load */
-    LDST_LD_S_X = 2,  /* load and sign-extend into Xt */
-    LDST_LD_S_W = 3,  /* load and sign-extend into Wt */
-} AArch64LdstType;
-
-/* We encode the format of the insn into the beginning of the name, so that
-   we can have the preprocessor help "typecheck" the insn vs the output
-   function.  Arm didn't provide us with nice names for the formats, so we
-   use the section number of the architecture reference manual in which the
-   instruction group is described.  */
-typedef enum {
-    /* Compare and branch (immediate).  */
-    I3201_CBZ       = 0x34000000,
-    I3201_CBNZ      = 0x35000000,
-
-    /* Conditional branch (immediate).  */
-    I3202_B_C       = 0x54000000,
-
-    /* Unconditional branch (immediate).  */
-    I3206_B         = 0x14000000,
-    I3206_BL        = 0x94000000,
-
-    /* Unconditional branch (register).  */
-    I3207_BR        = 0xd61f0000,
-    I3207_BLR       = 0xd63f0000,
-    I3207_RET       = 0xd65f0000,
-
-    /* Load/store register.  Described here as 3.3.12, but the helper
-       that emits them can transform to 3.3.10 or 3.3.13.  */
-    I3312_STRB      = 0x38000000 | LDST_ST << 22 | MO_8 << 30,
-    I3312_STRH      = 0x38000000 | LDST_ST << 22 | MO_16 << 30,
-    I3312_STRW      = 0x38000000 | LDST_ST << 22 | MO_32 << 30,
-    I3312_STRX      = 0x38000000 | LDST_ST << 22 | MO_64 << 30,
-
-    I3312_LDRB      = 0x38000000 | LDST_LD << 22 | MO_8 << 30,
-    I3312_LDRH      = 0x38000000 | LDST_LD << 22 | MO_16 << 30,
-    I3312_LDRW      = 0x38000000 | LDST_LD << 22 | MO_32 << 30,
-    I3312_LDRX      = 0x38000000 | LDST_LD << 22 | MO_64 << 30,
-
-    I3312_LDRSBW    = 0x38000000 | LDST_LD_S_W << 22 | MO_8 << 30,
-    I3312_LDRSHW    = 0x38000000 | LDST_LD_S_W << 22 | MO_16 << 30,
-
-    I3312_LDRSBX    = 0x38000000 | LDST_LD_S_X << 22 | MO_8 << 30,
-    I3312_LDRSHX    = 0x38000000 | LDST_LD_S_X << 22 | MO_16 << 30,
-    I3312_LDRSWX    = 0x38000000 | LDST_LD_S_X << 22 | MO_32 << 30,
-
-    I3312_TO_I3310  = 0x00200800,
-    I3312_TO_I3313  = 0x01000000,
-
-    /* Load/store register pair instructions.  */
-    I3314_LDP       = 0x28400000,
-    I3314_STP       = 0x28000000,
-
-    /* Add/subtract immediate instructions.  */
-    I3401_ADDI      = 0x11000000,
-    I3401_ADDSI     = 0x31000000,
-    I3401_SUBI      = 0x51000000,
-    I3401_SUBSI     = 0x71000000,
-
-    /* Bitfield instructions.  */
-    I3402_BFM       = 0x33000000,
-    I3402_SBFM      = 0x13000000,
-    I3402_UBFM      = 0x53000000,
-
-    /* Extract instruction.  */
-    I3403_EXTR      = 0x13800000,
-
-    /* Logical immediate instructions.  */
-    I3404_ANDI      = 0x12000000,
-    I3404_ORRI      = 0x32000000,
-    I3404_EORI      = 0x52000000,
-
-    /* Move wide immediate instructions.  */
-    I3405_MOVN      = 0x12800000,
-    I3405_MOVZ      = 0x52800000,
-    I3405_MOVK      = 0x72800000,
-
-    /* PC relative addressing instructions.  */
-    I3406_ADR       = 0x10000000,
-    I3406_ADRP      = 0x90000000,
-
-    /* Add/subtract shifted register instructions (without a shift).  */
-    I3502_ADD       = 0x0b000000,
-    I3502_ADDS      = 0x2b000000,
-    I3502_SUB       = 0x4b000000,
-    I3502_SUBS      = 0x6b000000,
-
-    /* Add/subtract shifted register instructions (with a shift).  */
-    I3502S_ADD_LSL  = I3502_ADD,
-
-    /* Add/subtract with carry instructions.  */
-    I3503_ADC       = 0x1a000000,
-    I3503_SBC       = 0x5a000000,
-
-    /* Conditional select instructions.  */
-    I3506_CSEL      = 0x1a800000,
-    I3506_CSINC     = 0x1a800400,
-
-    /* Data-processing (1 source) instructions.  */
-    I3507_REV16     = 0x5ac00400,
-    I3507_REV32     = 0x5ac00800,
-    I3507_REV64     = 0x5ac00c00,
-
-    /* Data-processing (2 source) instructions.  */
-    I3508_LSLV      = 0x1ac02000,
-    I3508_LSRV      = 0x1ac02400,
-    I3508_ASRV      = 0x1ac02800,
-    I3508_RORV      = 0x1ac02c00,
-    I3508_SMULH     = 0x9b407c00,
-    I3508_UMULH     = 0x9bc07c00,
-    I3508_UDIV      = 0x1ac00800,
-    I3508_SDIV      = 0x1ac00c00,
-
-    /* Data-processing (3 source) instructions.  */
-    I3509_MADD      = 0x1b000000,
-    I3509_MSUB      = 0x1b008000,
-
-    /* Logical shifted register instructions (without a shift).  */
-    I3510_AND       = 0x0a000000,
-    I3510_BIC       = 0x0a200000,
-    I3510_ORR       = 0x2a000000,
-    I3510_ORN       = 0x2a200000,
-    I3510_EOR       = 0x4a000000,
-    I3510_EON       = 0x4a200000,
-    I3510_ANDS      = 0x6a000000,
-} AArch64Insn;
-
-static inline uint32_t tcg_in32(TCGContext *s)
-{
-    uint32_t v = *(uint32_t *)s->code_ptr;
-    return v;
-}
-
-/* Emit an opcode with "type-checking" of the format.  */
-#define tcg_out_insn(S, FMT, OP, ...) \
-    glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
-
-static void tcg_out_insn_3201(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rt, int imm19)
-{
-    tcg_out32(s, insn | ext << 31 | (imm19 & 0x7ffff) << 5 | rt);
-}
-
-static void tcg_out_insn_3202(TCGContext *s, AArch64Insn insn,
-                              TCGCond c, int imm19)
-{
-    tcg_out32(s, insn | tcg_cond_to_aarch64[c] | (imm19 & 0x7ffff) << 5);
-}
-
-static void tcg_out_insn_3206(TCGContext *s, AArch64Insn insn, int imm26)
-{
-    tcg_out32(s, insn | (imm26 & 0x03ffffff));
-}
-
-static void tcg_out_insn_3207(TCGContext *s, AArch64Insn insn, TCGReg rn)
-{
-    tcg_out32(s, insn | rn << 5);
-}
-
-static void tcg_out_insn_3314(TCGContext *s, AArch64Insn insn,
-                              TCGReg r1, TCGReg r2, TCGReg rn,
-                              tcg_target_long ofs, bool pre, bool w)
-{
-    insn |= 1u << 31; /* ext */
-    insn |= pre << 24;
-    insn |= w << 23;
-
-    assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
-    insn |= (ofs & (0x7f << 3)) << (15 - 3);
-
-    tcg_out32(s, insn | r2 << 10 | rn << 5 | r1);
-}
-
-static void tcg_out_insn_3401(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn, uint64_t aimm)
-{
-    if (aimm > 0xfff) {
-        assert((aimm & 0xfff) == 0);
-        aimm >>= 12;
-        assert(aimm <= 0xfff);
-        aimm |= 1 << 12;  /* apply LSL 12 */
-    }
-    tcg_out32(s, insn | ext << 31 | aimm << 10 | rn << 5 | rd);
-}
-
-/* This function can be used for both 3.4.2 (Bitfield) and 3.4.4
-   (Logical immediate).  Both insn groups have N, IMMR and IMMS fields
-   that feed the DecodeBitMasks pseudo function.  */
-static void tcg_out_insn_3402(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn, int n, int immr, int imms)
-{
-    tcg_out32(s, insn | ext << 31 | n << 22 | immr << 16 | imms << 10
-              | rn << 5 | rd);
-}
-
-#define tcg_out_insn_3404  tcg_out_insn_3402
-
-static void tcg_out_insn_3403(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn, TCGReg rm, int imms)
-{
-    tcg_out32(s, insn | ext << 31 | ext << 22 | rm << 16 | imms << 10
-              | rn << 5 | rd);
-}
-
-/* This function is used for the Move (wide immediate) instruction group.
-   Note that SHIFT is a full shift count, not the 2 bit HW field. */
-static void tcg_out_insn_3405(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, uint16_t half, unsigned shift)
-{
-    assert((shift & ~0x30) == 0);
-    tcg_out32(s, insn | ext << 31 | shift << (21 - 4) | half << 5 | rd);
-}
-
-static void tcg_out_insn_3406(TCGContext *s, AArch64Insn insn,
-                              TCGReg rd, int64_t disp)
-{
-    tcg_out32(s, insn | (disp & 3) << 29 | (disp & 0x1ffffc) << (5 - 2) | rd);
-}
-
-/* This function is for both 3.5.2 (Add/Subtract shifted register), for
-   the rare occasion when we actually want to supply a shift amount.  */
-static inline void tcg_out_insn_3502S(TCGContext *s, AArch64Insn insn,
-                                      TCGType ext, TCGReg rd, TCGReg rn,
-                                      TCGReg rm, int imm6)
-{
-    tcg_out32(s, insn | ext << 31 | rm << 16 | imm6 << 10 | rn << 5 | rd);
-}
-
-/* This function is for 3.5.2 (Add/subtract shifted register),
-   and 3.5.10 (Logical shifted register), for the vast majorty of cases
-   when we don't want to apply a shift.  Thus it can also be used for
-   3.5.3 (Add/subtract with carry) and 3.5.8 (Data processing 2 source).  */
-static void tcg_out_insn_3502(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn, TCGReg rm)
-{
-    tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd);
-}
-
-#define tcg_out_insn_3503  tcg_out_insn_3502
-#define tcg_out_insn_3508  tcg_out_insn_3502
-#define tcg_out_insn_3510  tcg_out_insn_3502
-
-static void tcg_out_insn_3506(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn, TCGReg rm, TCGCond c)
-{
-    tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd
-              | tcg_cond_to_aarch64[c] << 12);
-}
-
-static void tcg_out_insn_3507(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn)
-{
-    tcg_out32(s, insn | ext << 31 | rn << 5 | rd);
-}
-
-static void tcg_out_insn_3509(TCGContext *s, AArch64Insn insn, TCGType ext,
-                              TCGReg rd, TCGReg rn, TCGReg rm, TCGReg ra)
-{
-    tcg_out32(s, insn | ext << 31 | rm << 16 | ra << 10 | rn << 5 | rd);
-}
-
-static void tcg_out_insn_3310(TCGContext *s, AArch64Insn insn,
-                              TCGReg rd, TCGReg base, TCGType ext,
-                              TCGReg regoff)
-{
-    /* Note the AArch64Insn constants above are for C3.3.12.  Adjust.  */
-    tcg_out32(s, insn | I3312_TO_I3310 | regoff << 16 |
-              0x4000 | ext << 13 | base << 5 | rd);
-}
-
-static void tcg_out_insn_3312(TCGContext *s, AArch64Insn insn,
-                              TCGReg rd, TCGReg rn, intptr_t offset)
-{
-    tcg_out32(s, insn | (offset & 0x1ff) << 12 | rn << 5 | rd);
-}
-
-static void tcg_out_insn_3313(TCGContext *s, AArch64Insn insn,
-                              TCGReg rd, TCGReg rn, uintptr_t scaled_uimm)
-{
-    /* Note the AArch64Insn constants above are for C3.3.12.  Adjust.  */
-    tcg_out32(s, insn | I3312_TO_I3313 | scaled_uimm << 10 | rn << 5 | rd);
-}
-
-/* Register to register move using ORR (shifted register with no shift). */
-static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm)
-{
-    tcg_out_insn(s, 3510, ORR, ext, rd, TCG_REG_XZR, rm);
-}
-
-/* Register to register move using ADDI (move to/from SP).  */
-static void tcg_out_movr_sp(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rn)
-{
-    tcg_out_insn(s, 3401, ADDI, ext, rd, rn, 0);
-}
-
-/* This function is used for the Logical (immediate) instruction group.
-   The value of LIMM must satisfy IS_LIMM.  See the comment above about
-   only supporting simplified logical immediates.  */
-static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
-                             TCGReg rd, TCGReg rn, uint64_t limm)
-{
-    unsigned h, l, r, c;
-
-    assert(is_limm(limm));
-
-    h = clz64(limm);
-    l = ctz64(limm);
-    if (l == 0) {
-        r = 0;                  /* form 0....01....1 */
-        c = ctz64(~limm) - 1;
-        if (h == 0) {
-            r = clz64(~limm);   /* form 1..10..01..1 */
-            c += r;
-        }
-    } else {
-        r = 64 - l;             /* form 1....10....0 or 0..01..10..0 */
-        c = r - h - 1;
-    }
-    if (ext == TCG_TYPE_I32) {
-        r &= 31;
-        c &= 31;
-    }
-
-    tcg_out_insn_3404(s, insn, ext, rd, rn, ext, r, c);
-}
-
-static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
-                         tcg_target_long value)
-{
-    AArch64Insn insn;
-    int i, wantinv, shift;
-    tcg_target_long svalue = value;
-    tcg_target_long ivalue = ~value;
-    tcg_target_long imask;
-
-    /* For 32-bit values, discard potential garbage in value.  For 64-bit
-       values within [2**31, 2**32-1], we can create smaller sequences by
-       interpreting this as a negative 32-bit number, while ensuring that
-       the high 32 bits are cleared by setting SF=0.  */
-    if (type == TCG_TYPE_I32 || (value & ~0xffffffffull) == 0) {
-        svalue = (int32_t)value;
-        value = (uint32_t)value;
-        ivalue = (uint32_t)ivalue;
-        type = TCG_TYPE_I32;
-    }
-
-    /* Speed things up by handling the common case of small positive
-       and negative values specially.  */
-    if ((value & ~0xffffull) == 0) {
-        tcg_out_insn(s, 3405, MOVZ, type, rd, value, 0);
-        return;
-    } else if ((ivalue & ~0xffffull) == 0) {
-        tcg_out_insn(s, 3405, MOVN, type, rd, ivalue, 0);
-        return;
-    }
-
-    /* Check for bitfield immediates.  For the benefit of 32-bit quantities,
-       use the sign-extended value.  That lets us match rotated values such
-       as 0xff0000ff with the same 64-bit logic matching 0xffffffffff0000ff. */
-    if (is_limm(svalue)) {
-        tcg_out_logicali(s, I3404_ORRI, type, rd, TCG_REG_XZR, svalue);
-        return;
-    }
-
-    /* Look for host pointer values within 4G of the PC.  This happens
-       often when loading pointers to QEMU's own data structures.  */
-    if (type == TCG_TYPE_I64) {
-        tcg_target_long disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
-        if (disp == sextract64(disp, 0, 21)) {
-            tcg_out_insn(s, 3406, ADRP, rd, disp);
-            if (value & 0xfff) {
-                tcg_out_insn(s, 3401, ADDI, type, rd, rd, value & 0xfff);
-            }
-            return;
-        }
-    }
-
-    /* Would it take fewer insns to begin with MOVN?  For the value and its
-       inverse, count the number of 16-bit lanes that are 0.  */
-    for (i = wantinv = imask = 0; i < 64; i += 16) {
-        tcg_target_long mask = 0xffffull << i;
-        if ((value & mask) == 0) {
-            wantinv -= 1;
-        }
-        if ((ivalue & mask) == 0) {
-            wantinv += 1;
-            imask |= mask;
-        }
-    }
-
-    /* If we had more 0xffff than 0x0000, invert VALUE and use MOVN.  */
-    insn = I3405_MOVZ;
-    if (wantinv > 0) {
-        value = ivalue;
-        insn = I3405_MOVN;
-    }
-
-    /* Find the lowest lane that is not 0x0000.  */
-    shift = ctz64(value) & (63 & -16);
-    tcg_out_insn_3405(s, insn, type, rd, value >> shift, shift);
-
-    if (wantinv > 0) {
-        /* Re-invert the value, so MOVK sees non-inverted bits.  */
-        value = ~value;
-        /* Clear out all the 0xffff lanes.  */
-        value ^= imask;
-    }
-    /* Clear out the lane that we just set.  */
-    value &= ~(0xffffUL << shift);
-
-    /* Iterate until all lanes have been set, and thus cleared from VALUE.  */
-    while (value) {
-        shift = ctz64(value) & (63 & -16);
-        tcg_out_insn(s, 3405, MOVK, type, rd, value >> shift, shift);
-        value &= ~(0xffffUL << shift);
-    }
-}
-
-/* Define something more legible for general use.  */
-#define tcg_out_ldst_r  tcg_out_insn_3310
-
-static void tcg_out_ldst(TCGContext *s, AArch64Insn insn,
-                         TCGReg rd, TCGReg rn, intptr_t offset)
-{
-    TCGMemOp size = (uint32_t)insn >> 30;
-
-    /* If the offset is naturally aligned and in range, then we can
-       use the scaled uimm12 encoding */
-    if (offset >= 0 && !(offset & ((1 << size) - 1))) {
-        uintptr_t scaled_uimm = offset >> size;
-        if (scaled_uimm <= 0xfff) {
-            tcg_out_insn_3313(s, insn, rd, rn, scaled_uimm);
-            return;
-        }
-    }
-
-    /* Small signed offsets can use the unscaled encoding.  */
-    if (offset >= -256 && offset < 256) {
-        tcg_out_insn_3312(s, insn, rd, rn, offset);
-        return;
-    }
-
-    /* Worst-case scenario, move offset to temp register, use reg offset.  */
-    tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset);
-    tcg_out_ldst_r(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP);
-}
-
-static inline void tcg_out_mov(TCGContext *s,
-                               TCGType type, TCGReg ret, TCGReg arg)
-{
-    if (ret != arg) {
-        tcg_out_movr(s, type, ret, arg);
-    }
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, type == TCG_TYPE_I32 ? I3312_LDRW : I3312_LDRX,
-                 arg, arg1, arg2);
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, type == TCG_TYPE_I32 ? I3312_STRW : I3312_STRX,
-                 arg, arg1, arg2);
-}
-
-static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
-                               TCGReg rn, unsigned int a, unsigned int b)
-{
-    tcg_out_insn(s, 3402, BFM, ext, rd, rn, ext, a, b);
-}
-
-static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
-                                TCGReg rn, unsigned int a, unsigned int b)
-{
-    tcg_out_insn(s, 3402, UBFM, ext, rd, rn, ext, a, b);
-}
-
-static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
-                                TCGReg rn, unsigned int a, unsigned int b)
-{
-    tcg_out_insn(s, 3402, SBFM, ext, rd, rn, ext, a, b);
-}
-
-static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
-                                TCGReg rn, TCGReg rm, unsigned int a)
-{
-    tcg_out_insn(s, 3403, EXTR, ext, rd, rn, rm, a);
-}
-
-static inline void tcg_out_shl(TCGContext *s, TCGType ext,
-                               TCGReg rd, TCGReg rn, unsigned int m)
-{
-    int bits = ext ? 64 : 32;
-    int max = bits - 1;
-    tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
-}
-
-static inline void tcg_out_shr(TCGContext *s, TCGType ext,
-                               TCGReg rd, TCGReg rn, unsigned int m)
-{
-    int max = ext ? 63 : 31;
-    tcg_out_ubfm(s, ext, rd, rn, m & max, max);
-}
-
-static inline void tcg_out_sar(TCGContext *s, TCGType ext,
-                               TCGReg rd, TCGReg rn, unsigned int m)
-{
-    int max = ext ? 63 : 31;
-    tcg_out_sbfm(s, ext, rd, rn, m & max, max);
-}
-
-static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
-                                TCGReg rd, TCGReg rn, unsigned int m)
-{
-    int max = ext ? 63 : 31;
-    tcg_out_extr(s, ext, rd, rn, rn, m & max);
-}
-
-static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
-                                TCGReg rd, TCGReg rn, unsigned int m)
-{
-    int bits = ext ? 64 : 32;
-    int max = bits - 1;
-    tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
-}
-
-static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd,
-                               TCGReg rn, unsigned lsb, unsigned width)
-{
-    unsigned size = ext ? 64 : 32;
-    unsigned a = (size - lsb) & (size - 1);
-    unsigned b = width - 1;
-    tcg_out_bfm(s, ext, rd, rn, a, b);
-}
-
-static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
-                        tcg_target_long b, bool const_b)
-{
-    if (const_b) {
-        /* Using CMP or CMN aliases.  */
-        if (b >= 0) {
-            tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b);
-        } else {
-            tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b);
-        }
-    } else {
-        /* Using CMP alias SUBS wzr, Wn, Wm */
-        tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b);
-    }
-}
-
-static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
-{
-    ptrdiff_t offset = target - s->code_ptr;
-    assert(offset == sextract64(offset, 0, 26));
-    tcg_out_insn(s, 3206, B, offset);
-}
-
-static inline void tcg_out_goto_noaddr(TCGContext *s)
-{
-    /* We pay attention here to not modify the branch target by reading from
-       the buffer. This ensure that caches and memory are kept coherent during
-       retranslation.  Mask away possible garbage in the high bits for the
-       first translation, while keeping the offset bits for retranslation. */
-    uint32_t old = tcg_in32(s);
-    tcg_out_insn(s, 3206, B, old);
-}
-
-static inline void tcg_out_goto_cond_noaddr(TCGContext *s, TCGCond c)
-{
-    /* See comments in tcg_out_goto_noaddr.  */
-    uint32_t old = tcg_in32(s) >> 5;
-    tcg_out_insn(s, 3202, B_C, c, old);
-}
-
-static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
-{
-    tcg_out_insn(s, 3207, BLR, reg);
-}
-
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
-{
-    ptrdiff_t offset = target - s->code_ptr;
-    if (offset == sextract64(offset, 0, 26)) {
-        tcg_out_insn(s, 3206, BL, offset);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
-        tcg_out_callr(s, TCG_REG_TMP);
-    }
-}
-
-void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
-{
-    tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr;
-    tcg_insn_unit *target = (tcg_insn_unit *)addr;
-
-    reloc_pc26(code_ptr, target);
-    flush_icache_range(jmp_addr, jmp_addr + 4);
-}
-
-static inline void tcg_out_goto_label(TCGContext *s, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (!l->has_value) {
-        tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, label_index, 0);
-        tcg_out_goto_noaddr(s);
-    } else {
-        tcg_out_goto(s, l->u.value_ptr);
-    }
-}
-
-static void tcg_out_brcond(TCGContext *s, TCGMemOp ext, TCGCond c, TCGArg a,
-                           TCGArg b, bool b_const, int label)
-{
-    TCGLabel *l = &s->labels[label];
-    intptr_t offset;
-    bool need_cmp;
-
-    if (b_const && b == 0 && (c == TCG_COND_EQ || c == TCG_COND_NE)) {
-        need_cmp = false;
-    } else {
-        need_cmp = true;
-        tcg_out_cmp(s, ext, a, b, b_const);
-    }
-
-    if (!l->has_value) {
-        tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, label, 0);
-        offset = tcg_in32(s) >> 5;
-    } else {
-        offset = l->u.value_ptr - s->code_ptr;
-        assert(offset == sextract64(offset, 0, 19));
-    }
-
-    if (need_cmp) {
-        tcg_out_insn(s, 3202, B_C, c, offset);
-    } else if (c == TCG_COND_EQ) {
-        tcg_out_insn(s, 3201, CBZ, ext, a, offset);
-    } else {
-        tcg_out_insn(s, 3201, CBNZ, ext, a, offset);
-    }
-}
-
-static inline void tcg_out_rev64(TCGContext *s, TCGReg rd, TCGReg rn)
-{
-    tcg_out_insn(s, 3507, REV64, TCG_TYPE_I64, rd, rn);
-}
-
-static inline void tcg_out_rev32(TCGContext *s, TCGReg rd, TCGReg rn)
-{
-    tcg_out_insn(s, 3507, REV32, TCG_TYPE_I32, rd, rn);
-}
-
-static inline void tcg_out_rev16(TCGContext *s, TCGReg rd, TCGReg rn)
-{
-    tcg_out_insn(s, 3507, REV16, TCG_TYPE_I32, rd, rn);
-}
-
-static inline void tcg_out_sxt(TCGContext *s, TCGType ext, TCGMemOp s_bits,
-                               TCGReg rd, TCGReg rn)
-{
-    /* Using ALIASes SXTB, SXTH, SXTW, of SBFM Xd, Xn, #0, #7|15|31 */
-    int bits = (8 << s_bits) - 1;
-    tcg_out_sbfm(s, ext, rd, rn, 0, bits);
-}
-
-static inline void tcg_out_uxt(TCGContext *s, TCGMemOp s_bits,
-                               TCGReg rd, TCGReg rn)
-{
-    /* Using ALIASes UXTB, UXTH of UBFM Wd, Wn, #0, #7|15 */
-    int bits = (8 << s_bits) - 1;
-    tcg_out_ubfm(s, 0, rd, rn, 0, bits);
-}
-
-static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
-                            TCGReg rn, int64_t aimm)
-{
-    if (aimm >= 0) {
-        tcg_out_insn(s, 3401, ADDI, ext, rd, rn, aimm);
-    } else {
-        tcg_out_insn(s, 3401, SUBI, ext, rd, rn, -aimm);
-    }
-}
-
-static inline void tcg_out_addsub2(TCGContext *s, int ext, TCGReg rl,
-                                   TCGReg rh, TCGReg al, TCGReg ah,
-                                   tcg_target_long bl, tcg_target_long bh,
-                                   bool const_bl, bool const_bh, bool sub)
-{
-    TCGReg orig_rl = rl;
-    AArch64Insn insn;
-
-    if (rl == ah || (!const_bh && rl == bh)) {
-        rl = TCG_REG_TMP;
-    }
-
-    if (const_bl) {
-        insn = I3401_ADDSI;
-        if ((bl < 0) ^ sub) {
-            insn = I3401_SUBSI;
-            bl = -bl;
-        }
-        tcg_out_insn_3401(s, insn, ext, rl, al, bl);
-    } else {
-        tcg_out_insn_3502(s, sub ? I3502_SUBS : I3502_ADDS, ext, rl, al, bl);
-    }
-
-    insn = I3503_ADC;
-    if (const_bh) {
-        /* Note that the only two constants we support are 0 and -1, and
-           that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa.  */
-        if ((bh != 0) ^ sub) {
-            insn = I3503_SBC;
-        }
-        bh = TCG_REG_XZR;
-    } else if (sub) {
-        insn = I3503_SBC;
-    }
-    tcg_out_insn_3503(s, insn, ext, rh, ah, bh);
-
-    tcg_out_mov(s, ext, orig_rl, rl);
-}
-
-#ifdef CONFIG_SOFTMMU
-/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
- *                                     TCGMemOpIdx oi, uintptr_t ra)
- */
-static void * const qemu_ld_helpers[16] = {
-    [MO_UB]   = helper_ret_ldub_mmu,
-    [MO_LEUW] = helper_le_lduw_mmu,
-    [MO_LEUL] = helper_le_ldul_mmu,
-    [MO_LEQ]  = helper_le_ldq_mmu,
-    [MO_BEUW] = helper_be_lduw_mmu,
-    [MO_BEUL] = helper_be_ldul_mmu,
-    [MO_BEQ]  = helper_be_ldq_mmu,
-};
-
-/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
- *                                     uintxx_t val, TCGMemOpIdx oi,
- *                                     uintptr_t ra)
- */
-static void * const qemu_st_helpers[16] = {
-    [MO_UB]   = helper_ret_stb_mmu,
-    [MO_LEUW] = helper_le_stw_mmu,
-    [MO_LEUL] = helper_le_stl_mmu,
-    [MO_LEQ]  = helper_le_stq_mmu,
-    [MO_BEUW] = helper_be_stw_mmu,
-    [MO_BEUL] = helper_be_stl_mmu,
-    [MO_BEQ]  = helper_be_stq_mmu,
-};
-
-static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
-{
-    ptrdiff_t offset = tcg_pcrel_diff(s, target);
-    assert(offset == sextract64(offset, 0, 21));
-    tcg_out_insn(s, 3406, ADR, rd, offset);
-}
-
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGMemOp size = opc & MO_SIZE;
-
-    reloc_pc19(lb->label_ptr[0], s->code_ptr);
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
-    tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
-    tcg_out_adr(s, TCG_REG_X3, lb->raddr);
-    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-    if (opc & MO_SIGN) {
-        tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
-    } else {
-        tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
-    }
-
-    tcg_out_goto(s, lb->raddr);
-}
-
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGMemOp size = opc & MO_SIZE;
-
-    reloc_pc19(lb->label_ptr[0], s->code_ptr);
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
-    tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
-    tcg_out_mov(s, size == MO_64, TCG_REG_X2, lb->datalo_reg);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, oi);
-    tcg_out_adr(s, TCG_REG_X4, lb->raddr);
-    tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-    tcg_out_goto(s, lb->raddr);
-}
-
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
-                                TCGType ext, TCGReg data_reg, TCGReg addr_reg,
-                                tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
-{
-    TCGLabelQemuLdst *label = new_ldst_label(s);
-
-    label->is_ld = is_ld;
-    label->oi = oi;
-    label->type = ext;
-    label->datalo_reg = data_reg;
-    label->addrlo_reg = addr_reg;
-    label->raddr = raddr;
-    label->label_ptr[0] = label_ptr;
-}
-
-/* Load and compare a TLB entry, emitting the conditional jump to the
-   slow path for the failure case, which will be patched later when finalizing
-   the slow path. Generated code returns the host addend in X1,
-   clobbers X0,X2,X3,TMP. */
-static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc,
-                             tcg_insn_unit **label_ptr, int mem_index,
-                             bool is_read)
-{
-    int tlb_offset = is_read ?
-        offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
-        : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
-    int a_bits = get_alignment_bits(opc);
-    TCGReg base = TCG_AREG0, x3;
-    uint64_t tlb_mask;
-
-    /* For aligned accesses, we check the first byte and include the alignment
-       bits within the address.  For unaligned access, we check that we don't
-       cross pages using the address of the last byte of the access.  */
-    if (a_bits >= 0) {
-        /* A byte access or an alignment check required */
-        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);
-        x3 = addr_reg;
-    } else {
-        tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
-                     TCG_REG_X3, addr_reg, (1 << (opc & MO_SIZE)) - 1);
-        tlb_mask = TARGET_PAGE_MASK;
-        x3 = TCG_REG_X3;
-    }
-
-    /* Extract the TLB index from the address into X0.
-       X0<CPU_TLB_BITS:0> =
-       addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
-    tcg_out_ubfm(s, TARGET_LONG_BITS == 64, TCG_REG_X0, addr_reg,
-                 TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
-
-    /* Store the page mask part of the address into X3.  */
-    tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
-                     TCG_REG_X3, x3, tlb_mask);
-
-    /* Add any "high bits" from the tlb offset to the env address into X2,
-       to take advantage of the LSL12 form of the ADDI instruction.
-       X2 = env + (tlb_offset & 0xfff000) */
-    if (tlb_offset & 0xfff000) {
-        tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_X2, base,
-                     tlb_offset & 0xfff000);
-        base = TCG_REG_X2;
-    }
-
-    /* Merge the tlb index contribution into X2.
-       X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
-    tcg_out_insn(s, 3502S, ADD_LSL, TCG_TYPE_I64, TCG_REG_X2, base,
-                 TCG_REG_X0, CPU_TLB_ENTRY_BITS);
-
-    /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
-       X0 = load [X2 + (tlb_offset & 0x000fff)] */
-    tcg_out_ldst(s, TARGET_LONG_BITS == 32 ? I3312_LDRW : I3312_LDRX,
-                 TCG_REG_X0, TCG_REG_X2, tlb_offset & 0xfff);
-
-    /* Load the tlb addend. Do that early to avoid stalling.
-       X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
-    tcg_out_ldst(s, I3312_LDRX, TCG_REG_X1, TCG_REG_X2,
-                 (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
-                 (is_read ? offsetof(CPUTLBEntry, addr_read)
-                  : offsetof(CPUTLBEntry, addr_write)));
-
-    /* Perform the address comparison. */
-    tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
-
-    /* If not equal, we jump to the slow path. */
-    *label_ptr = s->code_ptr;
-    tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
-}
-
-#endif /* CONFIG_SOFTMMU */
-
-static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop, TCGType ext,
-                                   TCGReg data_r, TCGReg addr_r,
-                                   TCGType otype, TCGReg off_r)
-{
-    const TCGMemOp bswap = memop & MO_BSWAP;
-
-    switch (memop & MO_SSIZE) {
-    case MO_UB:
-        tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r);
-        break;
-    case MO_SB:
-        tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW,
-                       data_r, addr_r, otype, off_r);
-        break;
-    case MO_UW:
-        tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
-        if (bswap) {
-            tcg_out_rev16(s, data_r, data_r);
-        }
-        break;
-    case MO_SW:
-        if (bswap) {
-            tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
-            tcg_out_rev16(s, data_r, data_r);
-            tcg_out_sxt(s, ext, MO_16, data_r, data_r);
-        } else {
-            tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW),
-                           data_r, addr_r, otype, off_r);
-        }
-        break;
-    case MO_UL:
-        tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
-        if (bswap) {
-            tcg_out_rev32(s, data_r, data_r);
-        }
-        break;
-    case MO_SL:
-        if (bswap) {
-            tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
-            tcg_out_rev32(s, data_r, data_r);
-            tcg_out_sxt(s, TCG_TYPE_I64, MO_32, data_r, data_r);
-        } else {
-            tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r);
-        }
-        break;
-    case MO_Q:
-        tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r);
-        if (bswap) {
-            tcg_out_rev64(s, data_r, data_r);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp memop,
-                                   TCGReg data_r, TCGReg addr_r,
-                                   TCGType otype, TCGReg off_r)
-{
-    const TCGMemOp bswap = memop & MO_BSWAP;
-
-    switch (memop & MO_SIZE) {
-    case MO_8:
-        tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r);
-        break;
-    case MO_16:
-        if (bswap && data_r != TCG_REG_XZR) {
-            tcg_out_rev16(s, TCG_REG_TMP, data_r);
-            data_r = TCG_REG_TMP;
-        }
-        tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r);
-        break;
-    case MO_32:
-        if (bswap && data_r != TCG_REG_XZR) {
-            tcg_out_rev32(s, TCG_REG_TMP, data_r);
-            data_r = TCG_REG_TMP;
-        }
-        tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r);
-        break;
-    case MO_64:
-        if (bswap && data_r != TCG_REG_XZR) {
-            tcg_out_rev64(s, TCG_REG_TMP, data_r);
-            data_r = TCG_REG_TMP;
-        }
-        tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r);
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
-                            TCGMemOpIdx oi, TCGType ext)
-{
-    TCGMemOp memop = get_memop(oi);
-    const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
-#ifdef CONFIG_SOFTMMU
-    unsigned mem_index = get_mmuidx(oi);
-    tcg_insn_unit *label_ptr;
-
-    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1);
-    tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
-                           TCG_REG_X1, otype, addr_reg);
-    add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
-                        s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
-    tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
-                           GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR,
-                           otype, addr_reg);
-#endif /* CONFIG_SOFTMMU */
-}
-
-static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
-                            TCGMemOpIdx oi)
-{
-    TCGMemOp memop = get_memop(oi);
-    const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
-#ifdef CONFIG_SOFTMMU
-    unsigned mem_index = get_mmuidx(oi);
-    tcg_insn_unit *label_ptr;
-
-    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0);
-    tcg_out_qemu_st_direct(s, memop, data_reg,
-                           TCG_REG_X1, otype, addr_reg);
-    add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64,
-                        data_reg, addr_reg, s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
-    tcg_out_qemu_st_direct(s, memop, data_reg,
-                           GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR,
-                           otype, addr_reg);
-#endif /* CONFIG_SOFTMMU */
-}
-
-static tcg_insn_unit *tb_ret_addr;
-
-static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                       const TCGArg args[TCG_MAX_OP_ARGS],
-                       const int const_args[TCG_MAX_OP_ARGS])
-{
-    /* 99% of the time, we can signal the use of extension registers
-       by looking to see if the opcode handles 64-bit data.  */
-    TCGType ext = (tcg_op_defs[opc].flags & TCG_OPF_64BIT) != 0;
-
-    /* Hoist the loads of the most common arguments.  */
-    TCGArg a0 = args[0];
-    TCGArg a1 = args[1];
-    TCGArg a2 = args[2];
-    int c2 = const_args[2];
-
-    /* Some operands are defined with "rZ" constraint, a register or
-       the zero register.  These need not actually test args[I] == 0.  */
-#define REG0(I)  (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
-        tcg_out_goto(s, tb_ret_addr);
-        break;
-
-    case INDEX_op_goto_tb:
-#ifndef USE_DIRECT_JUMP
-#error "USE_DIRECT_JUMP required for aarch64"
-#endif
-        assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */
-        s->tb_jmp_offset[a0] = tcg_current_code_size(s);
-        /* actual branch destination will be patched by
-           aarch64_tb_set_jmp_target later, beware retranslation. */
-        tcg_out_goto_noaddr(s);
-        s->tb_next_offset[a0] = tcg_current_code_size(s);
-        break;
-
-    case INDEX_op_br:
-        tcg_out_goto_label(s, a0);
-        break;
-
-    case INDEX_op_ld8u_i32:
-    case INDEX_op_ld8u_i64:
-        tcg_out_ldst(s, I3312_LDRB, a0, a1, a2);
-        break;
-    case INDEX_op_ld8s_i32:
-        tcg_out_ldst(s, I3312_LDRSBW, a0, a1, a2);
-        break;
-    case INDEX_op_ld8s_i64:
-        tcg_out_ldst(s, I3312_LDRSBX, a0, a1, a2);
-        break;
-    case INDEX_op_ld16u_i32:
-    case INDEX_op_ld16u_i64:
-        tcg_out_ldst(s, I3312_LDRH, a0, a1, a2);
-        break;
-    case INDEX_op_ld16s_i32:
-        tcg_out_ldst(s, I3312_LDRSHW, a0, a1, a2);
-        break;
-    case INDEX_op_ld16s_i64:
-        tcg_out_ldst(s, I3312_LDRSHX, a0, a1, a2);
-        break;
-    case INDEX_op_ld_i32:
-    case INDEX_op_ld32u_i64:
-        tcg_out_ldst(s, I3312_LDRW, a0, a1, a2);
-        break;
-    case INDEX_op_ld32s_i64:
-        tcg_out_ldst(s, I3312_LDRSWX, a0, a1, a2);
-        break;
-    case INDEX_op_ld_i64:
-        tcg_out_ldst(s, I3312_LDRX, a0, a1, a2);
-        break;
-
-    case INDEX_op_st8_i32:
-    case INDEX_op_st8_i64:
-        tcg_out_ldst(s, I3312_STRB, REG0(0), a1, a2);
-        break;
-    case INDEX_op_st16_i32:
-    case INDEX_op_st16_i64:
-        tcg_out_ldst(s, I3312_STRH, REG0(0), a1, a2);
-        break;
-    case INDEX_op_st_i32:
-    case INDEX_op_st32_i64:
-        tcg_out_ldst(s, I3312_STRW, REG0(0), a1, a2);
-        break;
-    case INDEX_op_st_i64:
-        tcg_out_ldst(s, I3312_STRX, REG0(0), a1, a2);
-        break;
-
-    case INDEX_op_add_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_add_i64:
-        if (c2) {
-            tcg_out_addsubi(s, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3502, ADD, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_sub_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_sub_i64:
-        if (c2) {
-            tcg_out_addsubi(s, ext, a0, a1, -a2);
-        } else {
-            tcg_out_insn(s, 3502, SUB, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_neg_i64:
-    case INDEX_op_neg_i32:
-        tcg_out_insn(s, 3502, SUB, ext, a0, TCG_REG_XZR, a1);
-        break;
-
-    case INDEX_op_and_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_and_i64:
-        if (c2) {
-            tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3510, AND, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_andc_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_andc_i64:
-        if (c2) {
-            tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, ~a2);
-        } else {
-            tcg_out_insn(s, 3510, BIC, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_or_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_or_i64:
-        if (c2) {
-            tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3510, ORR, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_orc_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_orc_i64:
-        if (c2) {
-            tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, ~a2);
-        } else {
-            tcg_out_insn(s, 3510, ORN, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_xor_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_xor_i64:
-        if (c2) {
-            tcg_out_logicali(s, I3404_EORI, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3510, EOR, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_eqv_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_eqv_i64:
-        if (c2) {
-            tcg_out_logicali(s, I3404_EORI, ext, a0, a1, ~a2);
-        } else {
-            tcg_out_insn(s, 3510, EON, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_not_i64:
-    case INDEX_op_not_i32:
-        tcg_out_insn(s, 3510, ORN, ext, a0, TCG_REG_XZR, a1);
-        break;
-
-    case INDEX_op_mul_i64:
-    case INDEX_op_mul_i32:
-        tcg_out_insn(s, 3509, MADD, ext, a0, a1, a2, TCG_REG_XZR);
-        break;
-
-    case INDEX_op_div_i64:
-    case INDEX_op_div_i32:
-        tcg_out_insn(s, 3508, SDIV, ext, a0, a1, a2);
-        break;
-    case INDEX_op_divu_i64:
-    case INDEX_op_divu_i32:
-        tcg_out_insn(s, 3508, UDIV, ext, a0, a1, a2);
-        break;
-
-    case INDEX_op_rem_i64:
-    case INDEX_op_rem_i32:
-        tcg_out_insn(s, 3508, SDIV, ext, TCG_REG_TMP, a1, a2);
-        tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
-        break;
-    case INDEX_op_remu_i64:
-    case INDEX_op_remu_i32:
-        tcg_out_insn(s, 3508, UDIV, ext, TCG_REG_TMP, a1, a2);
-        tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
-        break;
-
-    case INDEX_op_shl_i64:
-    case INDEX_op_shl_i32:
-        if (c2) {
-            tcg_out_shl(s, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3508, LSLV, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_shr_i64:
-    case INDEX_op_shr_i32:
-        if (c2) {
-            tcg_out_shr(s, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3508, LSRV, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_sar_i64:
-    case INDEX_op_sar_i32:
-        if (c2) {
-            tcg_out_sar(s, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3508, ASRV, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_rotr_i64:
-    case INDEX_op_rotr_i32:
-        if (c2) {
-            tcg_out_rotr(s, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3508, RORV, ext, a0, a1, a2);
-        }
-        break;
-
-    case INDEX_op_rotl_i64:
-    case INDEX_op_rotl_i32:
-        if (c2) {
-            tcg_out_rotl(s, ext, a0, a1, a2);
-        } else {
-            tcg_out_insn(s, 3502, SUB, 0, TCG_REG_TMP, TCG_REG_XZR, a2);
-            tcg_out_insn(s, 3508, RORV, ext, a0, a1, TCG_REG_TMP);
-        }
-        break;
-
-    case INDEX_op_brcond_i32:
-        a1 = (int32_t)a1;
-        /* FALLTHRU */
-    case INDEX_op_brcond_i64:
-        tcg_out_brcond(s, ext, a2, a0, a1, const_args[1], args[3]);
-        break;
-
-    case INDEX_op_setcond_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_setcond_i64:
-        tcg_out_cmp(s, ext, a1, a2, c2);
-        /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond).  */
-        tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR,
-                     TCG_REG_XZR, tcg_invert_cond(args[3]));
-        break;
-
-    case INDEX_op_movcond_i32:
-        a2 = (int32_t)a2;
-        /* FALLTHRU */
-    case INDEX_op_movcond_i64:
-        tcg_out_cmp(s, ext, a1, a2, c2);
-        tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]);
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, a0, a1, a2, ext);
-        break;
-    case INDEX_op_qemu_st_i32:
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, REG0(0), a1, a2);
-        break;
-
-    case INDEX_op_bswap64_i64:
-        tcg_out_rev64(s, a0, a1);
-        break;
-    case INDEX_op_bswap32_i64:
-    case INDEX_op_bswap32_i32:
-        tcg_out_rev32(s, a0, a1);
-        break;
-    case INDEX_op_bswap16_i64:
-    case INDEX_op_bswap16_i32:
-        tcg_out_rev16(s, a0, a1);
-        break;
-
-    case INDEX_op_ext8s_i64:
-    case INDEX_op_ext8s_i32:
-        tcg_out_sxt(s, ext, MO_8, a0, a1);
-        break;
-    case INDEX_op_ext16s_i64:
-    case INDEX_op_ext16s_i32:
-        tcg_out_sxt(s, ext, MO_16, a0, a1);
-        break;
-    case INDEX_op_ext32s_i64:
-        tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
-        break;
-    case INDEX_op_ext8u_i64:
-    case INDEX_op_ext8u_i32:
-        tcg_out_uxt(s, MO_8, a0, a1);
-        break;
-    case INDEX_op_ext16u_i64:
-    case INDEX_op_ext16u_i32:
-        tcg_out_uxt(s, MO_16, a0, a1);
-        break;
-    case INDEX_op_ext32u_i64:
-        tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
-        break;
-
-    case INDEX_op_deposit_i64:
-    case INDEX_op_deposit_i32:
-        tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
-        break;
-
-    case INDEX_op_add2_i32:
-        tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
-                        (int32_t)args[4], args[5], const_args[4],
-                        const_args[5], false);
-        break;
-    case INDEX_op_add2_i64:
-        tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
-                        args[5], const_args[4], const_args[5], false);
-        break;
-    case INDEX_op_sub2_i32:
-        tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
-                        (int32_t)args[4], args[5], const_args[4],
-                        const_args[5], true);
-        break;
-    case INDEX_op_sub2_i64:
-        tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
-                        args[5], const_args[4], const_args[5], true);
-        break;
-
-    case INDEX_op_muluh_i64:
-        tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2);
-        break;
-    case INDEX_op_mulsh_i64:
-        tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2);
-        break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-
-#undef REG0
-}
-
-static const TCGTargetOpDef aarch64_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_ld8u_i64, { "r", "r" } },
-    { INDEX_op_ld8s_i64, { "r", "r" } },
-    { INDEX_op_ld16u_i64, { "r", "r" } },
-    { INDEX_op_ld16s_i64, { "r", "r" } },
-    { INDEX_op_ld32u_i64, { "r", "r" } },
-    { INDEX_op_ld32s_i64, { "r", "r" } },
-    { INDEX_op_ld_i64, { "r", "r" } },
-
-    { INDEX_op_st8_i32, { "rZ", "r" } },
-    { INDEX_op_st16_i32, { "rZ", "r" } },
-    { INDEX_op_st_i32, { "rZ", "r" } },
-    { INDEX_op_st8_i64, { "rZ", "r" } },
-    { INDEX_op_st16_i64, { "rZ", "r" } },
-    { INDEX_op_st32_i64, { "rZ", "r" } },
-    { INDEX_op_st_i64, { "rZ", "r" } },
-
-    { INDEX_op_add_i32, { "r", "r", "rA" } },
-    { INDEX_op_add_i64, { "r", "r", "rA" } },
-    { INDEX_op_sub_i32, { "r", "r", "rA" } },
-    { INDEX_op_sub_i64, { "r", "r", "rA" } },
-    { INDEX_op_mul_i32, { "r", "r", "r" } },
-    { INDEX_op_mul_i64, { "r", "r", "r" } },
-    { INDEX_op_div_i32, { "r", "r", "r" } },
-    { INDEX_op_div_i64, { "r", "r", "r" } },
-    { INDEX_op_divu_i32, { "r", "r", "r" } },
-    { INDEX_op_divu_i64, { "r", "r", "r" } },
-    { INDEX_op_rem_i32, { "r", "r", "r" } },
-    { INDEX_op_rem_i64, { "r", "r", "r" } },
-    { INDEX_op_remu_i32, { "r", "r", "r" } },
-    { INDEX_op_remu_i64, { "r", "r", "r" } },
-    { INDEX_op_and_i32, { "r", "r", "rL" } },
-    { INDEX_op_and_i64, { "r", "r", "rL" } },
-    { INDEX_op_or_i32, { "r", "r", "rL" } },
-    { INDEX_op_or_i64, { "r", "r", "rL" } },
-    { INDEX_op_xor_i32, { "r", "r", "rL" } },
-    { INDEX_op_xor_i64, { "r", "r", "rL" } },
-    { INDEX_op_andc_i32, { "r", "r", "rL" } },
-    { INDEX_op_andc_i64, { "r", "r", "rL" } },
-    { INDEX_op_orc_i32, { "r", "r", "rL" } },
-    { INDEX_op_orc_i64, { "r", "r", "rL" } },
-    { INDEX_op_eqv_i32, { "r", "r", "rL" } },
-    { INDEX_op_eqv_i64, { "r", "r", "rL" } },
-
-    { INDEX_op_neg_i32, { "r", "r" } },
-    { INDEX_op_neg_i64, { "r", "r" } },
-    { INDEX_op_not_i32, { "r", "r" } },
-    { INDEX_op_not_i64, { "r", "r" } },
-
-    { INDEX_op_shl_i32, { "r", "r", "ri" } },
-    { INDEX_op_shr_i32, { "r", "r", "ri" } },
-    { INDEX_op_sar_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
-    { INDEX_op_shl_i64, { "r", "r", "ri" } },
-    { INDEX_op_shr_i64, { "r", "r", "ri" } },
-    { INDEX_op_sar_i64, { "r", "r", "ri" } },
-    { INDEX_op_rotl_i64, { "r", "r", "ri" } },
-    { INDEX_op_rotr_i64, { "r", "r", "ri" } },
-
-    { INDEX_op_brcond_i32, { "r", "rA" } },
-    { INDEX_op_brcond_i64, { "r", "rA" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rA" } },
-    { INDEX_op_setcond_i64, { "r", "r", "rA" } },
-    { INDEX_op_movcond_i32, { "r", "r", "rA", "rZ", "rZ" } },
-    { INDEX_op_movcond_i64, { "r", "r", "rA", "rZ", "rZ" } },
-
-    { INDEX_op_qemu_ld_i32, { "r", "l" } },
-    { INDEX_op_qemu_ld_i64, { "r", "l" } },
-    { INDEX_op_qemu_st_i32, { "lZ", "l" } },
-    { INDEX_op_qemu_st_i64, { "lZ", "l" } },
-
-    { INDEX_op_bswap16_i32, { "r", "r" } },
-    { INDEX_op_bswap32_i32, { "r", "r" } },
-    { INDEX_op_bswap16_i64, { "r", "r" } },
-    { INDEX_op_bswap32_i64, { "r", "r" } },
-    { INDEX_op_bswap64_i64, { "r", "r" } },
-
-    { INDEX_op_ext8s_i32, { "r", "r" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-    { INDEX_op_ext8u_i32, { "r", "r" } },
-    { INDEX_op_ext16u_i32, { "r", "r" } },
-
-    { INDEX_op_ext8s_i64, { "r", "r" } },
-    { INDEX_op_ext16s_i64, { "r", "r" } },
-    { INDEX_op_ext32s_i64, { "r", "r" } },
-    { INDEX_op_ext8u_i64, { "r", "r" } },
-    { INDEX_op_ext16u_i64, { "r", "r" } },
-    { INDEX_op_ext32u_i64, { "r", "r" } },
-
-    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
-    { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
-
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
-    { INDEX_op_add2_i64, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
-    { INDEX_op_sub2_i64, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
-
-    { INDEX_op_muluh_i64, { "r", "r", "r" } },
-    { INDEX_op_mulsh_i64, { "r", "r", "r" } },
-
-    { -1 },
-};
-
-static void tcg_target_init(TCGContext *s)
-{
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
-
-    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
-                     (1 << TCG_REG_X0) | (1 << TCG_REG_X1) |
-                     (1 << TCG_REG_X2) | (1 << TCG_REG_X3) |
-                     (1 << TCG_REG_X4) | (1 << TCG_REG_X5) |
-                     (1 << TCG_REG_X6) | (1 << TCG_REG_X7) |
-                     (1 << TCG_REG_X8) | (1 << TCG_REG_X9) |
-                     (1 << TCG_REG_X10) | (1 << TCG_REG_X11) |
-                     (1 << TCG_REG_X12) | (1 << TCG_REG_X13) |
-                     (1 << TCG_REG_X14) | (1 << TCG_REG_X15) |
-                     (1 << TCG_REG_X16) | (1 << TCG_REG_X17) |
-                     (1 << TCG_REG_X18) | (1 << TCG_REG_X30));
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
-
-    tcg_add_target_add_op_defs(aarch64_op_defs);
-}
-
-/* Saving pairs: (X19, X20) .. (X27, X28), (X29(fp), X30(lr)).  */
-#define PUSH_SIZE  ((30 - 19 + 1) * 8)
-
-#define FRAME_SIZE \
-    ((PUSH_SIZE \
-      + TCG_STATIC_CALL_ARGS_SIZE \
-      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
-      + TCG_TARGET_STACK_ALIGN - 1) \
-     & ~(TCG_TARGET_STACK_ALIGN - 1))
-
-/* We're expecting a 2 byte uleb128 encoded value.  */
-QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
-
-/* We're expecting to use a single ADDI insn.  */
-QEMU_BUILD_BUG_ON(FRAME_SIZE - PUSH_SIZE > 0xfff);
-
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    TCGReg r;
-
-    /* Push (FP, LR) and allocate space for all saved registers.  */
-    tcg_out_insn(s, 3314, STP, TCG_REG_FP, TCG_REG_LR,
-                 TCG_REG_SP, -PUSH_SIZE, 1, 1);
-
-    /* Set up frame pointer for canonical unwinding.  */
-    tcg_out_movr_sp(s, TCG_TYPE_I64, TCG_REG_FP, TCG_REG_SP);
-
-    /* Store callee-preserved regs x19..x28.  */
-    for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
-        int ofs = (r - TCG_REG_X19 + 2) * 8;
-        tcg_out_insn(s, 3314, STP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
-    }
-
-    /* Make stack space for TCG locals.  */
-    tcg_out_insn(s, 3401, SUBI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
-                 FRAME_SIZE - PUSH_SIZE);
-
-    /* Inform TCG about how to find TCG locals with register, offset, size.  */
-    tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-#if defined(CONFIG_USE_GUEST_BASE)
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, GUEST_BASE);
-        tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
-    }
-#endif
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-    tcg_out_insn(s, 3207, BR, tcg_target_call_iarg_regs[1]);
-
-    tb_ret_addr = s->code_ptr;
-
-    /* Remove TCG locals stack space.  */
-    tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
-                 FRAME_SIZE - PUSH_SIZE);
-
-    /* Restore registers x19..x28.  */
-    for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
-        int ofs = (r - TCG_REG_X19 + 2) * 8;
-        tcg_out_insn(s, 3314, LDP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
-    }
-
-    /* Pop (FP, LR), restore SP to previous frame.  */
-    tcg_out_insn(s, 3314, LDP, TCG_REG_FP, TCG_REG_LR,
-                 TCG_REG_SP, PUSH_SIZE, 0, 1);
-    tcg_out_insn(s, 3207, RET, TCG_REG_LR);
-}
-
-typedef struct {
-    DebugFrameHeader h;
-    uint8_t fde_def_cfa[4];
-    uint8_t fde_reg_ofs[24];
-} DebugFrame;
-
-#define ELF_HOST_MACHINE EM_AARCH64
-
-static const DebugFrame debug_frame = {
-    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .h.cie.id = -1,
-    .h.cie.version = 1,
-    .h.cie.code_align = 1,
-    .h.cie.data_align = 0x78,             /* sleb128 -8 */
-    .h.cie.return_column = TCG_REG_LR,
-
-    /* Total FDE size does not include the "len" member.  */
-    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
-
-    .fde_def_cfa = {
-        12, TCG_REG_SP,                 /* DW_CFA_def_cfa sp, ... */
-        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
-        (FRAME_SIZE >> 7)
-    },
-    .fde_reg_ofs = {
-        0x80 + 28, 1,                   /* DW_CFA_offset, x28,  -8 */
-        0x80 + 27, 2,                   /* DW_CFA_offset, x27, -16 */
-        0x80 + 26, 3,                   /* DW_CFA_offset, x26, -24 */
-        0x80 + 25, 4,                   /* DW_CFA_offset, x25, -32 */
-        0x80 + 24, 5,                   /* DW_CFA_offset, x24, -40 */
-        0x80 + 23, 6,                   /* DW_CFA_offset, x23, -48 */
-        0x80 + 22, 7,                   /* DW_CFA_offset, x22, -56 */
-        0x80 + 21, 8,                   /* DW_CFA_offset, x21, -64 */
-        0x80 + 20, 9,                   /* DW_CFA_offset, x20, -72 */
-        0x80 + 19, 10,                  /* DW_CFA_offset, x1p, -80 */
-        0x80 + 30, 11,                  /* DW_CFA_offset,  lr, -88 */
-        0x80 + 29, 12,                  /* DW_CFA_offset,  fp, -96 */
-    }
-};
-
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index 60c7493..a1d101f 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -10,10 +10,11 @@
  * See the COPYING file in the top-level directory for details.
  */
 
-#ifndef TCG_TARGET_AARCH64
-#define TCG_TARGET_AARCH64 1
+#ifndef AARCH64_TCG_TARGET_H
+#define AARCH64_TCG_TARGET_H
 
 #define TCG_TARGET_INSN_UNIT_SIZE  4
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 24
 #undef TCG_TARGET_STACK_GROWSUP
 
 typedef enum {
@@ -69,7 +70,8 @@
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          1
@@ -104,4 +106,4 @@
     __builtin___clear_cache((char *)start, (char *)stop);
 }
 
-#endif /* TCG_TARGET_AARCH64 */
+#endif /* AARCH64_TCG_TARGET_H */
diff --git a/tcg/aarch64/tcg-target.inc.c b/tcg/aarch64/tcg-target.inc.c
new file mode 100644
index 0000000..08b2d03
--- /dev/null
+++ b/tcg/aarch64/tcg-target.inc.c
@@ -0,0 +1,1916 @@
+/*
+ * Initial TCG Implementation for aarch64
+ *
+ * Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
+ * Written by Claudio Fontana
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.
+ *
+ * See the COPYING file in the top-level directory for details.
+ */
+
+#include "tcg-be-ldst.h"
+#include "qemu/bitops.h"
+
+/* We're going to re-use TCGType in setting of the SF bit, which controls
+   the size of the operation performed.  If we know the values match, it
+   makes things much cleaner.  */
+QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "%x0", "%x1", "%x2", "%x3", "%x4", "%x5", "%x6", "%x7",
+    "%x8", "%x9", "%x10", "%x11", "%x12", "%x13", "%x14", "%x15",
+    "%x16", "%x17", "%x18", "%x19", "%x20", "%x21", "%x22", "%x23",
+    "%x24", "%x25", "%x26", "%x27", "%x28", "%fp", "%x30", "%sp",
+};
+#endif /* CONFIG_DEBUG_TCG */
+
+static const int tcg_target_reg_alloc_order[] = {
+    TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
+    TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
+    TCG_REG_X28, /* we will reserve this for guest_base if configured */
+
+    TCG_REG_X8, TCG_REG_X9, TCG_REG_X10, TCG_REG_X11,
+    TCG_REG_X12, TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
+    TCG_REG_X16, TCG_REG_X17,
+
+    TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
+    TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
+
+    /* X18 reserved by system */
+    /* X19 reserved for AREG0 */
+    /* X29 reserved as fp */
+    /* X30 reserved as temporary */
+};
+
+static const int tcg_target_call_iarg_regs[8] = {
+    TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
+    TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
+};
+static const int tcg_target_call_oarg_regs[1] = {
+    TCG_REG_X0
+};
+
+#define TCG_REG_TMP TCG_REG_X30
+
+#ifndef CONFIG_SOFTMMU
+/* Note that XZR cannot be encoded in the address base register slot,
+   as that actaully encodes SP.  So if we need to zero-extend the guest
+   address, via the address index register slot, we need to load even
+   a zero guest base into a register.  */
+#define USE_GUEST_BASE     (guest_base != 0 || TARGET_LONG_BITS == 32)
+#define TCG_REG_GUEST_BASE TCG_REG_X28
+#endif
+
+static inline void reloc_pc26(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+{
+    ptrdiff_t offset = target - code_ptr;
+    tcg_debug_assert(offset == sextract64(offset, 0, 26));
+    /* read instruction, mask away previous PC_REL26 parameter contents,
+       set the proper offset, then write back the instruction. */
+    *code_ptr = deposit32(*code_ptr, 0, 26, offset);
+}
+
+static inline void reloc_pc26_atomic(tcg_insn_unit *code_ptr,
+                                     tcg_insn_unit *target)
+{
+    ptrdiff_t offset = target - code_ptr;
+    tcg_insn_unit insn;
+    tcg_debug_assert(offset == sextract64(offset, 0, 26));
+    /* read instruction, mask away previous PC_REL26 parameter contents,
+       set the proper offset, then write back the instruction. */
+    insn = atomic_read(code_ptr);
+    atomic_set(code_ptr, deposit32(insn, 0, 26, offset));
+}
+
+static inline void reloc_pc19(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+{
+    ptrdiff_t offset = target - code_ptr;
+    tcg_debug_assert(offset == sextract64(offset, 0, 19));
+    *code_ptr = deposit32(*code_ptr, 5, 19, offset);
+}
+
+static inline void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                               intptr_t value, intptr_t addend)
+{
+    tcg_debug_assert(addend == 0);
+    switch (type) {
+    case R_AARCH64_JUMP26:
+    case R_AARCH64_CALL26:
+        reloc_pc26(code_ptr, (tcg_insn_unit *)value);
+        break;
+    case R_AARCH64_CONDBR19:
+        reloc_pc19(code_ptr, (tcg_insn_unit *)value);
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+#define TCG_CT_CONST_AIMM 0x100
+#define TCG_CT_CONST_LIMM 0x200
+#define TCG_CT_CONST_ZERO 0x400
+#define TCG_CT_CONST_MONE 0x800
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct,
+                                   const char **pct_str)
+{
+    const char *ct_str = *pct_str;
+
+    switch (ct_str[0]) {
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
+        break;
+    case 'l': /* qemu_ld / qemu_st address, data_reg */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
+#ifdef CONFIG_SOFTMMU
+        /* x0 and x1 will be overwritten when reading the tlb entry,
+           and x2, and x3 for helper args, better to avoid using them. */
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X1);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X2);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
+#endif
+        break;
+    case 'A': /* Valid for arithmetic immediate (positive or negative).  */
+        ct->ct |= TCG_CT_CONST_AIMM;
+        break;
+    case 'L': /* Valid for logical immediate.  */
+        ct->ct |= TCG_CT_CONST_LIMM;
+        break;
+    case 'M': /* minus one */
+        ct->ct |= TCG_CT_CONST_MONE;
+        break;
+    case 'Z': /* zero */
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+static inline bool is_aimm(uint64_t val)
+{
+    return (val & ~0xfff) == 0 || (val & ~0xfff000) == 0;
+}
+
+static inline bool is_limm(uint64_t val)
+{
+    /* Taking a simplified view of the logical immediates for now, ignoring
+       the replication that can happen across the field.  Match bit patterns
+       of the forms
+           0....01....1
+           0..01..10..0
+       and their inverses.  */
+
+    /* Make things easier below, by testing the form with msb clear. */
+    if ((int64_t)val < 0) {
+        val = ~val;
+    }
+    if (val == 0) {
+        return false;
+    }
+    val += val & -val;
+    return (val & (val - 1)) == 0;
+}
+
+static int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                  const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    }
+    if (type == TCG_TYPE_I32) {
+        val = (int32_t)val;
+    }
+    if ((ct & TCG_CT_CONST_AIMM) && (is_aimm(val) || is_aimm(-val))) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_LIMM) && is_limm(val)) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_MONE) && val == -1) {
+        return 1;
+    }
+
+    return 0;
+}
+
+enum aarch64_cond_code {
+    COND_EQ = 0x0,
+    COND_NE = 0x1,
+    COND_CS = 0x2,     /* Unsigned greater or equal */
+    COND_HS = COND_CS, /* ALIAS greater or equal */
+    COND_CC = 0x3,     /* Unsigned less than */
+    COND_LO = COND_CC, /* ALIAS Lower */
+    COND_MI = 0x4,     /* Negative */
+    COND_PL = 0x5,     /* Zero or greater */
+    COND_VS = 0x6,     /* Overflow */
+    COND_VC = 0x7,     /* No overflow */
+    COND_HI = 0x8,     /* Unsigned greater than */
+    COND_LS = 0x9,     /* Unsigned less or equal */
+    COND_GE = 0xa,
+    COND_LT = 0xb,
+    COND_GT = 0xc,
+    COND_LE = 0xd,
+    COND_AL = 0xe,
+    COND_NV = 0xf, /* behaves like COND_AL here */
+};
+
+static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
+    [TCG_COND_EQ] = COND_EQ,
+    [TCG_COND_NE] = COND_NE,
+    [TCG_COND_LT] = COND_LT,
+    [TCG_COND_GE] = COND_GE,
+    [TCG_COND_LE] = COND_LE,
+    [TCG_COND_GT] = COND_GT,
+    /* unsigned */
+    [TCG_COND_LTU] = COND_LO,
+    [TCG_COND_GTU] = COND_HI,
+    [TCG_COND_GEU] = COND_HS,
+    [TCG_COND_LEU] = COND_LS,
+};
+
+typedef enum {
+    LDST_ST = 0,    /* store */
+    LDST_LD = 1,    /* load */
+    LDST_LD_S_X = 2,  /* load and sign-extend into Xt */
+    LDST_LD_S_W = 3,  /* load and sign-extend into Wt */
+} AArch64LdstType;
+
+/* We encode the format of the insn into the beginning of the name, so that
+   we can have the preprocessor help "typecheck" the insn vs the output
+   function.  Arm didn't provide us with nice names for the formats, so we
+   use the section number of the architecture reference manual in which the
+   instruction group is described.  */
+typedef enum {
+    /* Compare and branch (immediate).  */
+    I3201_CBZ       = 0x34000000,
+    I3201_CBNZ      = 0x35000000,
+
+    /* Conditional branch (immediate).  */
+    I3202_B_C       = 0x54000000,
+
+    /* Unconditional branch (immediate).  */
+    I3206_B         = 0x14000000,
+    I3206_BL        = 0x94000000,
+
+    /* Unconditional branch (register).  */
+    I3207_BR        = 0xd61f0000,
+    I3207_BLR       = 0xd63f0000,
+    I3207_RET       = 0xd65f0000,
+
+    /* Load/store register.  Described here as 3.3.12, but the helper
+       that emits them can transform to 3.3.10 or 3.3.13.  */
+    I3312_STRB      = 0x38000000 | LDST_ST << 22 | MO_8 << 30,
+    I3312_STRH      = 0x38000000 | LDST_ST << 22 | MO_16 << 30,
+    I3312_STRW      = 0x38000000 | LDST_ST << 22 | MO_32 << 30,
+    I3312_STRX      = 0x38000000 | LDST_ST << 22 | MO_64 << 30,
+
+    I3312_LDRB      = 0x38000000 | LDST_LD << 22 | MO_8 << 30,
+    I3312_LDRH      = 0x38000000 | LDST_LD << 22 | MO_16 << 30,
+    I3312_LDRW      = 0x38000000 | LDST_LD << 22 | MO_32 << 30,
+    I3312_LDRX      = 0x38000000 | LDST_LD << 22 | MO_64 << 30,
+
+    I3312_LDRSBW    = 0x38000000 | LDST_LD_S_W << 22 | MO_8 << 30,
+    I3312_LDRSHW    = 0x38000000 | LDST_LD_S_W << 22 | MO_16 << 30,
+
+    I3312_LDRSBX    = 0x38000000 | LDST_LD_S_X << 22 | MO_8 << 30,
+    I3312_LDRSHX    = 0x38000000 | LDST_LD_S_X << 22 | MO_16 << 30,
+    I3312_LDRSWX    = 0x38000000 | LDST_LD_S_X << 22 | MO_32 << 30,
+
+    I3312_TO_I3310  = 0x00200800,
+    I3312_TO_I3313  = 0x01000000,
+
+    /* Load/store register pair instructions.  */
+    I3314_LDP       = 0x28400000,
+    I3314_STP       = 0x28000000,
+
+    /* Add/subtract immediate instructions.  */
+    I3401_ADDI      = 0x11000000,
+    I3401_ADDSI     = 0x31000000,
+    I3401_SUBI      = 0x51000000,
+    I3401_SUBSI     = 0x71000000,
+
+    /* Bitfield instructions.  */
+    I3402_BFM       = 0x33000000,
+    I3402_SBFM      = 0x13000000,
+    I3402_UBFM      = 0x53000000,
+
+    /* Extract instruction.  */
+    I3403_EXTR      = 0x13800000,
+
+    /* Logical immediate instructions.  */
+    I3404_ANDI      = 0x12000000,
+    I3404_ORRI      = 0x32000000,
+    I3404_EORI      = 0x52000000,
+
+    /* Move wide immediate instructions.  */
+    I3405_MOVN      = 0x12800000,
+    I3405_MOVZ      = 0x52800000,
+    I3405_MOVK      = 0x72800000,
+
+    /* PC relative addressing instructions.  */
+    I3406_ADR       = 0x10000000,
+    I3406_ADRP      = 0x90000000,
+
+    /* Add/subtract shifted register instructions (without a shift).  */
+    I3502_ADD       = 0x0b000000,
+    I3502_ADDS      = 0x2b000000,
+    I3502_SUB       = 0x4b000000,
+    I3502_SUBS      = 0x6b000000,
+
+    /* Add/subtract shifted register instructions (with a shift).  */
+    I3502S_ADD_LSL  = I3502_ADD,
+
+    /* Add/subtract with carry instructions.  */
+    I3503_ADC       = 0x1a000000,
+    I3503_SBC       = 0x5a000000,
+
+    /* Conditional select instructions.  */
+    I3506_CSEL      = 0x1a800000,
+    I3506_CSINC     = 0x1a800400,
+
+    /* Data-processing (1 source) instructions.  */
+    I3507_REV16     = 0x5ac00400,
+    I3507_REV32     = 0x5ac00800,
+    I3507_REV64     = 0x5ac00c00,
+
+    /* Data-processing (2 source) instructions.  */
+    I3508_LSLV      = 0x1ac02000,
+    I3508_LSRV      = 0x1ac02400,
+    I3508_ASRV      = 0x1ac02800,
+    I3508_RORV      = 0x1ac02c00,
+    I3508_SMULH     = 0x9b407c00,
+    I3508_UMULH     = 0x9bc07c00,
+    I3508_UDIV      = 0x1ac00800,
+    I3508_SDIV      = 0x1ac00c00,
+
+    /* Data-processing (3 source) instructions.  */
+    I3509_MADD      = 0x1b000000,
+    I3509_MSUB      = 0x1b008000,
+
+    /* Logical shifted register instructions (without a shift).  */
+    I3510_AND       = 0x0a000000,
+    I3510_BIC       = 0x0a200000,
+    I3510_ORR       = 0x2a000000,
+    I3510_ORN       = 0x2a200000,
+    I3510_EOR       = 0x4a000000,
+    I3510_EON       = 0x4a200000,
+    I3510_ANDS      = 0x6a000000,
+} AArch64Insn;
+
+static inline uint32_t tcg_in32(TCGContext *s)
+{
+    uint32_t v = *(uint32_t *)s->code_ptr;
+    return v;
+}
+
+/* Emit an opcode with "type-checking" of the format.  */
+#define tcg_out_insn(S, FMT, OP, ...) \
+    glue(tcg_out_insn_,FMT)(S, glue(glue(glue(I,FMT),_),OP), ## __VA_ARGS__)
+
+static void tcg_out_insn_3201(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rt, int imm19)
+{
+    tcg_out32(s, insn | ext << 31 | (imm19 & 0x7ffff) << 5 | rt);
+}
+
+static void tcg_out_insn_3202(TCGContext *s, AArch64Insn insn,
+                              TCGCond c, int imm19)
+{
+    tcg_out32(s, insn | tcg_cond_to_aarch64[c] | (imm19 & 0x7ffff) << 5);
+}
+
+static void tcg_out_insn_3206(TCGContext *s, AArch64Insn insn, int imm26)
+{
+    tcg_out32(s, insn | (imm26 & 0x03ffffff));
+}
+
+static void tcg_out_insn_3207(TCGContext *s, AArch64Insn insn, TCGReg rn)
+{
+    tcg_out32(s, insn | rn << 5);
+}
+
+static void tcg_out_insn_3314(TCGContext *s, AArch64Insn insn,
+                              TCGReg r1, TCGReg r2, TCGReg rn,
+                              tcg_target_long ofs, bool pre, bool w)
+{
+    insn |= 1u << 31; /* ext */
+    insn |= pre << 24;
+    insn |= w << 23;
+
+    tcg_debug_assert(ofs >= -0x200 && ofs < 0x200 && (ofs & 7) == 0);
+    insn |= (ofs & (0x7f << 3)) << (15 - 3);
+
+    tcg_out32(s, insn | r2 << 10 | rn << 5 | r1);
+}
+
+static void tcg_out_insn_3401(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn, uint64_t aimm)
+{
+    if (aimm > 0xfff) {
+        tcg_debug_assert((aimm & 0xfff) == 0);
+        aimm >>= 12;
+        tcg_debug_assert(aimm <= 0xfff);
+        aimm |= 1 << 12;  /* apply LSL 12 */
+    }
+    tcg_out32(s, insn | ext << 31 | aimm << 10 | rn << 5 | rd);
+}
+
+/* This function can be used for both 3.4.2 (Bitfield) and 3.4.4
+   (Logical immediate).  Both insn groups have N, IMMR and IMMS fields
+   that feed the DecodeBitMasks pseudo function.  */
+static void tcg_out_insn_3402(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn, int n, int immr, int imms)
+{
+    tcg_out32(s, insn | ext << 31 | n << 22 | immr << 16 | imms << 10
+              | rn << 5 | rd);
+}
+
+#define tcg_out_insn_3404  tcg_out_insn_3402
+
+static void tcg_out_insn_3403(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn, TCGReg rm, int imms)
+{
+    tcg_out32(s, insn | ext << 31 | ext << 22 | rm << 16 | imms << 10
+              | rn << 5 | rd);
+}
+
+/* This function is used for the Move (wide immediate) instruction group.
+   Note that SHIFT is a full shift count, not the 2 bit HW field. */
+static void tcg_out_insn_3405(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, uint16_t half, unsigned shift)
+{
+    tcg_debug_assert((shift & ~0x30) == 0);
+    tcg_out32(s, insn | ext << 31 | shift << (21 - 4) | half << 5 | rd);
+}
+
+static void tcg_out_insn_3406(TCGContext *s, AArch64Insn insn,
+                              TCGReg rd, int64_t disp)
+{
+    tcg_out32(s, insn | (disp & 3) << 29 | (disp & 0x1ffffc) << (5 - 2) | rd);
+}
+
+/* This function is for both 3.5.2 (Add/Subtract shifted register), for
+   the rare occasion when we actually want to supply a shift amount.  */
+static inline void tcg_out_insn_3502S(TCGContext *s, AArch64Insn insn,
+                                      TCGType ext, TCGReg rd, TCGReg rn,
+                                      TCGReg rm, int imm6)
+{
+    tcg_out32(s, insn | ext << 31 | rm << 16 | imm6 << 10 | rn << 5 | rd);
+}
+
+/* This function is for 3.5.2 (Add/subtract shifted register),
+   and 3.5.10 (Logical shifted register), for the vast majorty of cases
+   when we don't want to apply a shift.  Thus it can also be used for
+   3.5.3 (Add/subtract with carry) and 3.5.8 (Data processing 2 source).  */
+static void tcg_out_insn_3502(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn, TCGReg rm)
+{
+    tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd);
+}
+
+#define tcg_out_insn_3503  tcg_out_insn_3502
+#define tcg_out_insn_3508  tcg_out_insn_3502
+#define tcg_out_insn_3510  tcg_out_insn_3502
+
+static void tcg_out_insn_3506(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn, TCGReg rm, TCGCond c)
+{
+    tcg_out32(s, insn | ext << 31 | rm << 16 | rn << 5 | rd
+              | tcg_cond_to_aarch64[c] << 12);
+}
+
+static void tcg_out_insn_3507(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn)
+{
+    tcg_out32(s, insn | ext << 31 | rn << 5 | rd);
+}
+
+static void tcg_out_insn_3509(TCGContext *s, AArch64Insn insn, TCGType ext,
+                              TCGReg rd, TCGReg rn, TCGReg rm, TCGReg ra)
+{
+    tcg_out32(s, insn | ext << 31 | rm << 16 | ra << 10 | rn << 5 | rd);
+}
+
+static void tcg_out_insn_3310(TCGContext *s, AArch64Insn insn,
+                              TCGReg rd, TCGReg base, TCGType ext,
+                              TCGReg regoff)
+{
+    /* Note the AArch64Insn constants above are for C3.3.12.  Adjust.  */
+    tcg_out32(s, insn | I3312_TO_I3310 | regoff << 16 |
+              0x4000 | ext << 13 | base << 5 | rd);
+}
+
+static void tcg_out_insn_3312(TCGContext *s, AArch64Insn insn,
+                              TCGReg rd, TCGReg rn, intptr_t offset)
+{
+    tcg_out32(s, insn | (offset & 0x1ff) << 12 | rn << 5 | rd);
+}
+
+static void tcg_out_insn_3313(TCGContext *s, AArch64Insn insn,
+                              TCGReg rd, TCGReg rn, uintptr_t scaled_uimm)
+{
+    /* Note the AArch64Insn constants above are for C3.3.12.  Adjust.  */
+    tcg_out32(s, insn | I3312_TO_I3313 | scaled_uimm << 10 | rn << 5 | rd);
+}
+
+/* Register to register move using ORR (shifted register with no shift). */
+static void tcg_out_movr(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rm)
+{
+    tcg_out_insn(s, 3510, ORR, ext, rd, TCG_REG_XZR, rm);
+}
+
+/* Register to register move using ADDI (move to/from SP).  */
+static void tcg_out_movr_sp(TCGContext *s, TCGType ext, TCGReg rd, TCGReg rn)
+{
+    tcg_out_insn(s, 3401, ADDI, ext, rd, rn, 0);
+}
+
+/* This function is used for the Logical (immediate) instruction group.
+   The value of LIMM must satisfy IS_LIMM.  See the comment above about
+   only supporting simplified logical immediates.  */
+static void tcg_out_logicali(TCGContext *s, AArch64Insn insn, TCGType ext,
+                             TCGReg rd, TCGReg rn, uint64_t limm)
+{
+    unsigned h, l, r, c;
+
+    tcg_debug_assert(is_limm(limm));
+
+    h = clz64(limm);
+    l = ctz64(limm);
+    if (l == 0) {
+        r = 0;                  /* form 0....01....1 */
+        c = ctz64(~limm) - 1;
+        if (h == 0) {
+            r = clz64(~limm);   /* form 1..10..01..1 */
+            c += r;
+        }
+    } else {
+        r = 64 - l;             /* form 1....10....0 or 0..01..10..0 */
+        c = r - h - 1;
+    }
+    if (ext == TCG_TYPE_I32) {
+        r &= 31;
+        c &= 31;
+    }
+
+    tcg_out_insn_3404(s, insn, ext, rd, rn, ext, r, c);
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
+                         tcg_target_long value)
+{
+    AArch64Insn insn;
+    int i, wantinv, shift;
+    tcg_target_long svalue = value;
+    tcg_target_long ivalue = ~value;
+    tcg_target_long imask;
+
+    /* For 32-bit values, discard potential garbage in value.  For 64-bit
+       values within [2**31, 2**32-1], we can create smaller sequences by
+       interpreting this as a negative 32-bit number, while ensuring that
+       the high 32 bits are cleared by setting SF=0.  */
+    if (type == TCG_TYPE_I32 || (value & ~0xffffffffull) == 0) {
+        svalue = (int32_t)value;
+        value = (uint32_t)value;
+        ivalue = (uint32_t)ivalue;
+        type = TCG_TYPE_I32;
+    }
+
+    /* Speed things up by handling the common case of small positive
+       and negative values specially.  */
+    if ((value & ~0xffffull) == 0) {
+        tcg_out_insn(s, 3405, MOVZ, type, rd, value, 0);
+        return;
+    } else if ((ivalue & ~0xffffull) == 0) {
+        tcg_out_insn(s, 3405, MOVN, type, rd, ivalue, 0);
+        return;
+    }
+
+    /* Check for bitfield immediates.  For the benefit of 32-bit quantities,
+       use the sign-extended value.  That lets us match rotated values such
+       as 0xff0000ff with the same 64-bit logic matching 0xffffffffff0000ff. */
+    if (is_limm(svalue)) {
+        tcg_out_logicali(s, I3404_ORRI, type, rd, TCG_REG_XZR, svalue);
+        return;
+    }
+
+    /* Look for host pointer values within 4G of the PC.  This happens
+       often when loading pointers to QEMU's own data structures.  */
+    if (type == TCG_TYPE_I64) {
+        tcg_target_long disp = (value >> 12) - ((intptr_t)s->code_ptr >> 12);
+        if (disp == sextract64(disp, 0, 21)) {
+            tcg_out_insn(s, 3406, ADRP, rd, disp);
+            if (value & 0xfff) {
+                tcg_out_insn(s, 3401, ADDI, type, rd, rd, value & 0xfff);
+            }
+            return;
+        }
+    }
+
+    /* Would it take fewer insns to begin with MOVN?  For the value and its
+       inverse, count the number of 16-bit lanes that are 0.  */
+    for (i = wantinv = imask = 0; i < 64; i += 16) {
+        tcg_target_long mask = 0xffffull << i;
+        if ((value & mask) == 0) {
+            wantinv -= 1;
+        }
+        if ((ivalue & mask) == 0) {
+            wantinv += 1;
+            imask |= mask;
+        }
+    }
+
+    /* If we had more 0xffff than 0x0000, invert VALUE and use MOVN.  */
+    insn = I3405_MOVZ;
+    if (wantinv > 0) {
+        value = ivalue;
+        insn = I3405_MOVN;
+    }
+
+    /* Find the lowest lane that is not 0x0000.  */
+    shift = ctz64(value) & (63 & -16);
+    tcg_out_insn_3405(s, insn, type, rd, value >> shift, shift);
+
+    if (wantinv > 0) {
+        /* Re-invert the value, so MOVK sees non-inverted bits.  */
+        value = ~value;
+        /* Clear out all the 0xffff lanes.  */
+        value ^= imask;
+    }
+    /* Clear out the lane that we just set.  */
+    value &= ~(0xffffUL << shift);
+
+    /* Iterate until all lanes have been set, and thus cleared from VALUE.  */
+    while (value) {
+        shift = ctz64(value) & (63 & -16);
+        tcg_out_insn(s, 3405, MOVK, type, rd, value >> shift, shift);
+        value &= ~(0xffffUL << shift);
+    }
+}
+
+/* Define something more legible for general use.  */
+#define tcg_out_ldst_r  tcg_out_insn_3310
+
+static void tcg_out_ldst(TCGContext *s, AArch64Insn insn,
+                         TCGReg rd, TCGReg rn, intptr_t offset)
+{
+    TCGMemOp size = (uint32_t)insn >> 30;
+
+    /* If the offset is naturally aligned and in range, then we can
+       use the scaled uimm12 encoding */
+    if (offset >= 0 && !(offset & ((1 << size) - 1))) {
+        uintptr_t scaled_uimm = offset >> size;
+        if (scaled_uimm <= 0xfff) {
+            tcg_out_insn_3313(s, insn, rd, rn, scaled_uimm);
+            return;
+        }
+    }
+
+    /* Small signed offsets can use the unscaled encoding.  */
+    if (offset >= -256 && offset < 256) {
+        tcg_out_insn_3312(s, insn, rd, rn, offset);
+        return;
+    }
+
+    /* Worst-case scenario, move offset to temp register, use reg offset.  */
+    tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset);
+    tcg_out_ldst_r(s, insn, rd, rn, TCG_TYPE_I64, TCG_REG_TMP);
+}
+
+static inline void tcg_out_mov(TCGContext *s,
+                               TCGType type, TCGReg ret, TCGReg arg)
+{
+    if (ret != arg) {
+        tcg_out_movr(s, type, ret, arg);
+    }
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ldst(s, type == TCG_TYPE_I32 ? I3312_LDRW : I3312_LDRX,
+                 arg, arg1, arg2);
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ldst(s, type == TCG_TYPE_I32 ? I3312_STRW : I3312_STRX,
+                 arg, arg1, arg2);
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    if (val == 0) {
+        tcg_out_st(s, type, TCG_REG_XZR, base, ofs);
+        return true;
+    }
+    return false;
+}
+
+static inline void tcg_out_bfm(TCGContext *s, TCGType ext, TCGReg rd,
+                               TCGReg rn, unsigned int a, unsigned int b)
+{
+    tcg_out_insn(s, 3402, BFM, ext, rd, rn, ext, a, b);
+}
+
+static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
+                                TCGReg rn, unsigned int a, unsigned int b)
+{
+    tcg_out_insn(s, 3402, UBFM, ext, rd, rn, ext, a, b);
+}
+
+static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
+                                TCGReg rn, unsigned int a, unsigned int b)
+{
+    tcg_out_insn(s, 3402, SBFM, ext, rd, rn, ext, a, b);
+}
+
+static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
+                                TCGReg rn, TCGReg rm, unsigned int a)
+{
+    tcg_out_insn(s, 3403, EXTR, ext, rd, rn, rm, a);
+}
+
+static inline void tcg_out_shl(TCGContext *s, TCGType ext,
+                               TCGReg rd, TCGReg rn, unsigned int m)
+{
+    int bits = ext ? 64 : 32;
+    int max = bits - 1;
+    tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
+}
+
+static inline void tcg_out_shr(TCGContext *s, TCGType ext,
+                               TCGReg rd, TCGReg rn, unsigned int m)
+{
+    int max = ext ? 63 : 31;
+    tcg_out_ubfm(s, ext, rd, rn, m & max, max);
+}
+
+static inline void tcg_out_sar(TCGContext *s, TCGType ext,
+                               TCGReg rd, TCGReg rn, unsigned int m)
+{
+    int max = ext ? 63 : 31;
+    tcg_out_sbfm(s, ext, rd, rn, m & max, max);
+}
+
+static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
+                                TCGReg rd, TCGReg rn, unsigned int m)
+{
+    int max = ext ? 63 : 31;
+    tcg_out_extr(s, ext, rd, rn, rn, m & max);
+}
+
+static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
+                                TCGReg rd, TCGReg rn, unsigned int m)
+{
+    int bits = ext ? 64 : 32;
+    int max = bits - 1;
+    tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
+}
+
+static inline void tcg_out_dep(TCGContext *s, TCGType ext, TCGReg rd,
+                               TCGReg rn, unsigned lsb, unsigned width)
+{
+    unsigned size = ext ? 64 : 32;
+    unsigned a = (size - lsb) & (size - 1);
+    unsigned b = width - 1;
+    tcg_out_bfm(s, ext, rd, rn, a, b);
+}
+
+static void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg a,
+                        tcg_target_long b, bool const_b)
+{
+    if (const_b) {
+        /* Using CMP or CMN aliases.  */
+        if (b >= 0) {
+            tcg_out_insn(s, 3401, SUBSI, ext, TCG_REG_XZR, a, b);
+        } else {
+            tcg_out_insn(s, 3401, ADDSI, ext, TCG_REG_XZR, a, -b);
+        }
+    } else {
+        /* Using CMP alias SUBS wzr, Wn, Wm */
+        tcg_out_insn(s, 3502, SUBS, ext, TCG_REG_XZR, a, b);
+    }
+}
+
+static inline void tcg_out_goto(TCGContext *s, tcg_insn_unit *target)
+{
+    ptrdiff_t offset = target - s->code_ptr;
+    tcg_debug_assert(offset == sextract64(offset, 0, 26));
+    tcg_out_insn(s, 3206, B, offset);
+}
+
+static inline void tcg_out_goto_noaddr(TCGContext *s)
+{
+    /* We pay attention here to not modify the branch target by reading from
+       the buffer. This ensure that caches and memory are kept coherent during
+       retranslation.  Mask away possible garbage in the high bits for the
+       first translation, while keeping the offset bits for retranslation. */
+    uint32_t old = tcg_in32(s);
+    tcg_out_insn(s, 3206, B, old);
+}
+
+static inline void tcg_out_goto_cond_noaddr(TCGContext *s, TCGCond c)
+{
+    /* See comments in tcg_out_goto_noaddr.  */
+    uint32_t old = tcg_in32(s) >> 5;
+    tcg_out_insn(s, 3202, B_C, c, old);
+}
+
+static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
+{
+    tcg_out_insn(s, 3207, BLR, reg);
+}
+
+static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+{
+    ptrdiff_t offset = target - s->code_ptr;
+    if (offset == sextract64(offset, 0, 26)) {
+        tcg_out_insn(s, 3206, BL, offset);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target);
+        tcg_out_callr(s, TCG_REG_TMP);
+    }
+}
+
+void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
+{
+    tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr;
+    tcg_insn_unit *target = (tcg_insn_unit *)addr;
+
+    reloc_pc26_atomic(code_ptr, target);
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
+
+static inline void tcg_out_goto_label(TCGContext *s, TCGLabel *l)
+{
+    if (!l->has_value) {
+        tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, l, 0);
+        tcg_out_goto_noaddr(s);
+    } else {
+        tcg_out_goto(s, l->u.value_ptr);
+    }
+}
+
+static void tcg_out_brcond(TCGContext *s, TCGMemOp ext, TCGCond c, TCGArg a,
+                           TCGArg b, bool b_const, TCGLabel *l)
+{
+    intptr_t offset;
+    bool need_cmp;
+
+    if (b_const && b == 0 && (c == TCG_COND_EQ || c == TCG_COND_NE)) {
+        need_cmp = false;
+    } else {
+        need_cmp = true;
+        tcg_out_cmp(s, ext, a, b, b_const);
+    }
+
+    if (!l->has_value) {
+        tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, l, 0);
+        offset = tcg_in32(s) >> 5;
+    } else {
+        offset = l->u.value_ptr - s->code_ptr;
+        tcg_debug_assert(offset == sextract64(offset, 0, 19));
+    }
+
+    if (need_cmp) {
+        tcg_out_insn(s, 3202, B_C, c, offset);
+    } else if (c == TCG_COND_EQ) {
+        tcg_out_insn(s, 3201, CBZ, ext, a, offset);
+    } else {
+        tcg_out_insn(s, 3201, CBNZ, ext, a, offset);
+    }
+}
+
+static inline void tcg_out_rev64(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_insn(s, 3507, REV64, TCG_TYPE_I64, rd, rn);
+}
+
+static inline void tcg_out_rev32(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_insn(s, 3507, REV32, TCG_TYPE_I32, rd, rn);
+}
+
+static inline void tcg_out_rev16(TCGContext *s, TCGReg rd, TCGReg rn)
+{
+    tcg_out_insn(s, 3507, REV16, TCG_TYPE_I32, rd, rn);
+}
+
+static inline void tcg_out_sxt(TCGContext *s, TCGType ext, TCGMemOp s_bits,
+                               TCGReg rd, TCGReg rn)
+{
+    /* Using ALIASes SXTB, SXTH, SXTW, of SBFM Xd, Xn, #0, #7|15|31 */
+    int bits = (8 << s_bits) - 1;
+    tcg_out_sbfm(s, ext, rd, rn, 0, bits);
+}
+
+static inline void tcg_out_uxt(TCGContext *s, TCGMemOp s_bits,
+                               TCGReg rd, TCGReg rn)
+{
+    /* Using ALIASes UXTB, UXTH of UBFM Wd, Wn, #0, #7|15 */
+    int bits = (8 << s_bits) - 1;
+    tcg_out_ubfm(s, 0, rd, rn, 0, bits);
+}
+
+static void tcg_out_addsubi(TCGContext *s, int ext, TCGReg rd,
+                            TCGReg rn, int64_t aimm)
+{
+    if (aimm >= 0) {
+        tcg_out_insn(s, 3401, ADDI, ext, rd, rn, aimm);
+    } else {
+        tcg_out_insn(s, 3401, SUBI, ext, rd, rn, -aimm);
+    }
+}
+
+static inline void tcg_out_addsub2(TCGContext *s, int ext, TCGReg rl,
+                                   TCGReg rh, TCGReg al, TCGReg ah,
+                                   tcg_target_long bl, tcg_target_long bh,
+                                   bool const_bl, bool const_bh, bool sub)
+{
+    TCGReg orig_rl = rl;
+    AArch64Insn insn;
+
+    if (rl == ah || (!const_bh && rl == bh)) {
+        rl = TCG_REG_TMP;
+    }
+
+    if (const_bl) {
+        insn = I3401_ADDSI;
+        if ((bl < 0) ^ sub) {
+            insn = I3401_SUBSI;
+            bl = -bl;
+        }
+        tcg_out_insn_3401(s, insn, ext, rl, al, bl);
+    } else {
+        tcg_out_insn_3502(s, sub ? I3502_SUBS : I3502_ADDS, ext, rl, al, bl);
+    }
+
+    insn = I3503_ADC;
+    if (const_bh) {
+        /* Note that the only two constants we support are 0 and -1, and
+           that SBC = rn + ~rm + c, so adc -1 is sbc 0, and vice-versa.  */
+        if ((bh != 0) ^ sub) {
+            insn = I3503_SBC;
+        }
+        bh = TCG_REG_XZR;
+    } else if (sub) {
+        insn = I3503_SBC;
+    }
+    tcg_out_insn_3503(s, insn, ext, rh, ah, bh);
+
+    tcg_out_mov(s, ext, orig_rl, rl);
+}
+
+#ifdef CONFIG_SOFTMMU
+/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
+ *                                     TCGMemOpIdx oi, uintptr_t ra)
+ */
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+};
+
+/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
+ *                                     uintxx_t val, TCGMemOpIdx oi,
+ *                                     uintptr_t ra)
+ */
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
+};
+
+static inline void tcg_out_adr(TCGContext *s, TCGReg rd, void *target)
+{
+    ptrdiff_t offset = tcg_pcrel_diff(s, target);
+    tcg_debug_assert(offset == sextract64(offset, 0, 21));
+    tcg_out_insn(s, 3406, ADR, rd, offset);
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGMemOp size = opc & MO_SIZE;
+
+    reloc_pc19(lb->label_ptr[0], s->code_ptr);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
+    tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi);
+    tcg_out_adr(s, TCG_REG_X3, lb->raddr);
+    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+    if (opc & MO_SIGN) {
+        tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0);
+    } else {
+        tcg_out_mov(s, size == MO_64, lb->datalo_reg, TCG_REG_X0);
+    }
+
+    tcg_out_goto(s, lb->raddr);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGMemOp size = opc & MO_SIZE;
+
+    reloc_pc19(lb->label_ptr[0], s->code_ptr);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_X0, TCG_AREG0);
+    tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg);
+    tcg_out_mov(s, size == MO_64, TCG_REG_X2, lb->datalo_reg);
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, oi);
+    tcg_out_adr(s, TCG_REG_X4, lb->raddr);
+    tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+    tcg_out_goto(s, lb->raddr);
+}
+
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
+                                TCGType ext, TCGReg data_reg, TCGReg addr_reg,
+                                tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->oi = oi;
+    label->type = ext;
+    label->datalo_reg = data_reg;
+    label->addrlo_reg = addr_reg;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr;
+}
+
+/* Load and compare a TLB entry, emitting the conditional jump to the
+   slow path for the failure case, which will be patched later when finalizing
+   the slow path. Generated code returns the host addend in X1,
+   clobbers X0,X2,X3,TMP. */
+static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc,
+                             tcg_insn_unit **label_ptr, int mem_index,
+                             bool is_read)
+{
+    int tlb_offset = is_read ?
+        offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+        : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+    int a_bits = get_alignment_bits(opc);
+    TCGReg base = TCG_AREG0, x3;
+    uint64_t tlb_mask;
+
+    /* For aligned accesses, we check the first byte and include the alignment
+       bits within the address.  For unaligned access, we check that we don't
+       cross pages using the address of the last byte of the access.  */
+    if (a_bits >= 0) {
+        /* A byte access or an alignment check required */
+        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);
+        x3 = addr_reg;
+    } else {
+        tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64,
+                     TCG_REG_X3, addr_reg, (1 << (opc & MO_SIZE)) - 1);
+        tlb_mask = TARGET_PAGE_MASK;
+        x3 = TCG_REG_X3;
+    }
+
+    /* Extract the TLB index from the address into X0.
+       X0<CPU_TLB_BITS:0> =
+       addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
+    tcg_out_ubfm(s, TARGET_LONG_BITS == 64, TCG_REG_X0, addr_reg,
+                 TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
+
+    /* Store the page mask part of the address into X3.  */
+    tcg_out_logicali(s, I3404_ANDI, TARGET_LONG_BITS == 64,
+                     TCG_REG_X3, x3, tlb_mask);
+
+    /* Add any "high bits" from the tlb offset to the env address into X2,
+       to take advantage of the LSL12 form of the ADDI instruction.
+       X2 = env + (tlb_offset & 0xfff000) */
+    if (tlb_offset & 0xfff000) {
+        tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_X2, base,
+                     tlb_offset & 0xfff000);
+        base = TCG_REG_X2;
+    }
+
+    /* Merge the tlb index contribution into X2.
+       X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
+    tcg_out_insn(s, 3502S, ADD_LSL, TCG_TYPE_I64, TCG_REG_X2, base,
+                 TCG_REG_X0, CPU_TLB_ENTRY_BITS);
+
+    /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
+       X0 = load [X2 + (tlb_offset & 0x000fff)] */
+    tcg_out_ldst(s, TARGET_LONG_BITS == 32 ? I3312_LDRW : I3312_LDRX,
+                 TCG_REG_X0, TCG_REG_X2, tlb_offset & 0xfff);
+
+    /* Load the tlb addend. Do that early to avoid stalling.
+       X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
+    tcg_out_ldst(s, I3312_LDRX, TCG_REG_X1, TCG_REG_X2,
+                 (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
+                 (is_read ? offsetof(CPUTLBEntry, addr_read)
+                  : offsetof(CPUTLBEntry, addr_write)));
+
+    /* Perform the address comparison. */
+    tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
+
+    /* If not equal, we jump to the slow path. */
+    *label_ptr = s->code_ptr;
+    tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
+}
+
+#endif /* CONFIG_SOFTMMU */
+
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp memop, TCGType ext,
+                                   TCGReg data_r, TCGReg addr_r,
+                                   TCGType otype, TCGReg off_r)
+{
+    const TCGMemOp bswap = memop & MO_BSWAP;
+
+    switch (memop & MO_SSIZE) {
+    case MO_UB:
+        tcg_out_ldst_r(s, I3312_LDRB, data_r, addr_r, otype, off_r);
+        break;
+    case MO_SB:
+        tcg_out_ldst_r(s, ext ? I3312_LDRSBX : I3312_LDRSBW,
+                       data_r, addr_r, otype, off_r);
+        break;
+    case MO_UW:
+        tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
+        if (bswap) {
+            tcg_out_rev16(s, data_r, data_r);
+        }
+        break;
+    case MO_SW:
+        if (bswap) {
+            tcg_out_ldst_r(s, I3312_LDRH, data_r, addr_r, otype, off_r);
+            tcg_out_rev16(s, data_r, data_r);
+            tcg_out_sxt(s, ext, MO_16, data_r, data_r);
+        } else {
+            tcg_out_ldst_r(s, (ext ? I3312_LDRSHX : I3312_LDRSHW),
+                           data_r, addr_r, otype, off_r);
+        }
+        break;
+    case MO_UL:
+        tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
+        if (bswap) {
+            tcg_out_rev32(s, data_r, data_r);
+        }
+        break;
+    case MO_SL:
+        if (bswap) {
+            tcg_out_ldst_r(s, I3312_LDRW, data_r, addr_r, otype, off_r);
+            tcg_out_rev32(s, data_r, data_r);
+            tcg_out_sxt(s, TCG_TYPE_I64, MO_32, data_r, data_r);
+        } else {
+            tcg_out_ldst_r(s, I3312_LDRSWX, data_r, addr_r, otype, off_r);
+        }
+        break;
+    case MO_Q:
+        tcg_out_ldst_r(s, I3312_LDRX, data_r, addr_r, otype, off_r);
+        if (bswap) {
+            tcg_out_rev64(s, data_r, data_r);
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp memop,
+                                   TCGReg data_r, TCGReg addr_r,
+                                   TCGType otype, TCGReg off_r)
+{
+    const TCGMemOp bswap = memop & MO_BSWAP;
+
+    switch (memop & MO_SIZE) {
+    case MO_8:
+        tcg_out_ldst_r(s, I3312_STRB, data_r, addr_r, otype, off_r);
+        break;
+    case MO_16:
+        if (bswap && data_r != TCG_REG_XZR) {
+            tcg_out_rev16(s, TCG_REG_TMP, data_r);
+            data_r = TCG_REG_TMP;
+        }
+        tcg_out_ldst_r(s, I3312_STRH, data_r, addr_r, otype, off_r);
+        break;
+    case MO_32:
+        if (bswap && data_r != TCG_REG_XZR) {
+            tcg_out_rev32(s, TCG_REG_TMP, data_r);
+            data_r = TCG_REG_TMP;
+        }
+        tcg_out_ldst_r(s, I3312_STRW, data_r, addr_r, otype, off_r);
+        break;
+    case MO_64:
+        if (bswap && data_r != TCG_REG_XZR) {
+            tcg_out_rev64(s, TCG_REG_TMP, data_r);
+            data_r = TCG_REG_TMP;
+        }
+        tcg_out_ldst_r(s, I3312_STRX, data_r, addr_r, otype, off_r);
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_ld(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
+                            TCGMemOpIdx oi, TCGType ext)
+{
+    TCGMemOp memop = get_memop(oi);
+    const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
+#ifdef CONFIG_SOFTMMU
+    unsigned mem_index = get_mmuidx(oi);
+    tcg_insn_unit *label_ptr;
+
+    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 1);
+    tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
+                           TCG_REG_X1, otype, addr_reg);
+    add_qemu_ldst_label(s, true, oi, ext, data_reg, addr_reg,
+                        s->code_ptr, label_ptr);
+#else /* !CONFIG_SOFTMMU */
+    if (USE_GUEST_BASE) {
+        tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
+                               TCG_REG_GUEST_BASE, otype, addr_reg);
+    } else {
+        tcg_out_qemu_ld_direct(s, memop, ext, data_reg,
+                               addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
+    }
+#endif /* CONFIG_SOFTMMU */
+}
+
+static void tcg_out_qemu_st(TCGContext *s, TCGReg data_reg, TCGReg addr_reg,
+                            TCGMemOpIdx oi)
+{
+    TCGMemOp memop = get_memop(oi);
+    const TCGType otype = TARGET_LONG_BITS == 64 ? TCG_TYPE_I64 : TCG_TYPE_I32;
+#ifdef CONFIG_SOFTMMU
+    unsigned mem_index = get_mmuidx(oi);
+    tcg_insn_unit *label_ptr;
+
+    tcg_out_tlb_read(s, addr_reg, memop, &label_ptr, mem_index, 0);
+    tcg_out_qemu_st_direct(s, memop, data_reg,
+                           TCG_REG_X1, otype, addr_reg);
+    add_qemu_ldst_label(s, false, oi, (memop & MO_SIZE)== MO_64,
+                        data_reg, addr_reg, s->code_ptr, label_ptr);
+#else /* !CONFIG_SOFTMMU */
+    if (USE_GUEST_BASE) {
+        tcg_out_qemu_st_direct(s, memop, data_reg,
+                               TCG_REG_GUEST_BASE, otype, addr_reg);
+    } else {
+        tcg_out_qemu_st_direct(s, memop, data_reg,
+                               addr_reg, TCG_TYPE_I64, TCG_REG_XZR);
+    }
+#endif /* CONFIG_SOFTMMU */
+}
+
+static tcg_insn_unit *tb_ret_addr;
+
+static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                       const TCGArg args[TCG_MAX_OP_ARGS],
+                       const int const_args[TCG_MAX_OP_ARGS])
+{
+    /* 99% of the time, we can signal the use of extension registers
+       by looking to see if the opcode handles 64-bit data.  */
+    TCGType ext = (tcg_op_defs[opc].flags & TCG_OPF_64BIT) != 0;
+
+    /* Hoist the loads of the most common arguments.  */
+    TCGArg a0 = args[0];
+    TCGArg a1 = args[1];
+    TCGArg a2 = args[2];
+    int c2 = const_args[2];
+
+    /* Some operands are defined with "rZ" constraint, a register or
+       the zero register.  These need not actually test args[I] == 0.  */
+#define REG0(I)  (const_args[I] ? TCG_REG_XZR : (TCGReg)args[I])
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, a0);
+        tcg_out_goto(s, tb_ret_addr);
+        break;
+
+    case INDEX_op_goto_tb:
+#ifndef USE_DIRECT_JUMP
+#error "USE_DIRECT_JUMP required for aarch64"
+#endif
+        /* consistency for USE_DIRECT_JUMP */
+        tcg_debug_assert(s->tb_jmp_insn_offset != NULL);
+        s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+        /* actual branch destination will be patched by
+           aarch64_tb_set_jmp_target later, beware retranslation. */
+        tcg_out_goto_noaddr(s);
+        s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
+        break;
+
+    case INDEX_op_br:
+        tcg_out_goto_label(s, arg_label(a0));
+        break;
+
+    case INDEX_op_ld8u_i32:
+    case INDEX_op_ld8u_i64:
+        tcg_out_ldst(s, I3312_LDRB, a0, a1, a2);
+        break;
+    case INDEX_op_ld8s_i32:
+        tcg_out_ldst(s, I3312_LDRSBW, a0, a1, a2);
+        break;
+    case INDEX_op_ld8s_i64:
+        tcg_out_ldst(s, I3312_LDRSBX, a0, a1, a2);
+        break;
+    case INDEX_op_ld16u_i32:
+    case INDEX_op_ld16u_i64:
+        tcg_out_ldst(s, I3312_LDRH, a0, a1, a2);
+        break;
+    case INDEX_op_ld16s_i32:
+        tcg_out_ldst(s, I3312_LDRSHW, a0, a1, a2);
+        break;
+    case INDEX_op_ld16s_i64:
+        tcg_out_ldst(s, I3312_LDRSHX, a0, a1, a2);
+        break;
+    case INDEX_op_ld_i32:
+    case INDEX_op_ld32u_i64:
+        tcg_out_ldst(s, I3312_LDRW, a0, a1, a2);
+        break;
+    case INDEX_op_ld32s_i64:
+        tcg_out_ldst(s, I3312_LDRSWX, a0, a1, a2);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_ldst(s, I3312_LDRX, a0, a1, a2);
+        break;
+
+    case INDEX_op_st8_i32:
+    case INDEX_op_st8_i64:
+        tcg_out_ldst(s, I3312_STRB, REG0(0), a1, a2);
+        break;
+    case INDEX_op_st16_i32:
+    case INDEX_op_st16_i64:
+        tcg_out_ldst(s, I3312_STRH, REG0(0), a1, a2);
+        break;
+    case INDEX_op_st_i32:
+    case INDEX_op_st32_i64:
+        tcg_out_ldst(s, I3312_STRW, REG0(0), a1, a2);
+        break;
+    case INDEX_op_st_i64:
+        tcg_out_ldst(s, I3312_STRX, REG0(0), a1, a2);
+        break;
+
+    case INDEX_op_add_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_add_i64:
+        if (c2) {
+            tcg_out_addsubi(s, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3502, ADD, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_sub_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_sub_i64:
+        if (c2) {
+            tcg_out_addsubi(s, ext, a0, a1, -a2);
+        } else {
+            tcg_out_insn(s, 3502, SUB, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_neg_i64:
+    case INDEX_op_neg_i32:
+        tcg_out_insn(s, 3502, SUB, ext, a0, TCG_REG_XZR, a1);
+        break;
+
+    case INDEX_op_and_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_and_i64:
+        if (c2) {
+            tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3510, AND, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_andc_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_andc_i64:
+        if (c2) {
+            tcg_out_logicali(s, I3404_ANDI, ext, a0, a1, ~a2);
+        } else {
+            tcg_out_insn(s, 3510, BIC, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_or_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_or_i64:
+        if (c2) {
+            tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3510, ORR, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_orc_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_orc_i64:
+        if (c2) {
+            tcg_out_logicali(s, I3404_ORRI, ext, a0, a1, ~a2);
+        } else {
+            tcg_out_insn(s, 3510, ORN, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_xor_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_xor_i64:
+        if (c2) {
+            tcg_out_logicali(s, I3404_EORI, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3510, EOR, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_eqv_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_eqv_i64:
+        if (c2) {
+            tcg_out_logicali(s, I3404_EORI, ext, a0, a1, ~a2);
+        } else {
+            tcg_out_insn(s, 3510, EON, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_not_i64:
+    case INDEX_op_not_i32:
+        tcg_out_insn(s, 3510, ORN, ext, a0, TCG_REG_XZR, a1);
+        break;
+
+    case INDEX_op_mul_i64:
+    case INDEX_op_mul_i32:
+        tcg_out_insn(s, 3509, MADD, ext, a0, a1, a2, TCG_REG_XZR);
+        break;
+
+    case INDEX_op_div_i64:
+    case INDEX_op_div_i32:
+        tcg_out_insn(s, 3508, SDIV, ext, a0, a1, a2);
+        break;
+    case INDEX_op_divu_i64:
+    case INDEX_op_divu_i32:
+        tcg_out_insn(s, 3508, UDIV, ext, a0, a1, a2);
+        break;
+
+    case INDEX_op_rem_i64:
+    case INDEX_op_rem_i32:
+        tcg_out_insn(s, 3508, SDIV, ext, TCG_REG_TMP, a1, a2);
+        tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
+        break;
+    case INDEX_op_remu_i64:
+    case INDEX_op_remu_i32:
+        tcg_out_insn(s, 3508, UDIV, ext, TCG_REG_TMP, a1, a2);
+        tcg_out_insn(s, 3509, MSUB, ext, a0, TCG_REG_TMP, a2, a1);
+        break;
+
+    case INDEX_op_shl_i64:
+    case INDEX_op_shl_i32:
+        if (c2) {
+            tcg_out_shl(s, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3508, LSLV, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_shr_i64:
+    case INDEX_op_shr_i32:
+        if (c2) {
+            tcg_out_shr(s, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3508, LSRV, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_sar_i64:
+    case INDEX_op_sar_i32:
+        if (c2) {
+            tcg_out_sar(s, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3508, ASRV, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_rotr_i64:
+    case INDEX_op_rotr_i32:
+        if (c2) {
+            tcg_out_rotr(s, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3508, RORV, ext, a0, a1, a2);
+        }
+        break;
+
+    case INDEX_op_rotl_i64:
+    case INDEX_op_rotl_i32:
+        if (c2) {
+            tcg_out_rotl(s, ext, a0, a1, a2);
+        } else {
+            tcg_out_insn(s, 3502, SUB, 0, TCG_REG_TMP, TCG_REG_XZR, a2);
+            tcg_out_insn(s, 3508, RORV, ext, a0, a1, TCG_REG_TMP);
+        }
+        break;
+
+    case INDEX_op_brcond_i32:
+        a1 = (int32_t)a1;
+        /* FALLTHRU */
+    case INDEX_op_brcond_i64:
+        tcg_out_brcond(s, ext, a2, a0, a1, const_args[1], arg_label(args[3]));
+        break;
+
+    case INDEX_op_setcond_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_setcond_i64:
+        tcg_out_cmp(s, ext, a1, a2, c2);
+        /* Use CSET alias of CSINC Wd, WZR, WZR, invert(cond).  */
+        tcg_out_insn(s, 3506, CSINC, TCG_TYPE_I32, a0, TCG_REG_XZR,
+                     TCG_REG_XZR, tcg_invert_cond(args[3]));
+        break;
+
+    case INDEX_op_movcond_i32:
+        a2 = (int32_t)a2;
+        /* FALLTHRU */
+    case INDEX_op_movcond_i64:
+        tcg_out_cmp(s, ext, a1, a2, c2);
+        tcg_out_insn(s, 3506, CSEL, ext, a0, REG0(3), REG0(4), args[5]);
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, a0, a1, a2, ext);
+        break;
+    case INDEX_op_qemu_st_i32:
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, REG0(0), a1, a2);
+        break;
+
+    case INDEX_op_bswap64_i64:
+        tcg_out_rev64(s, a0, a1);
+        break;
+    case INDEX_op_bswap32_i64:
+    case INDEX_op_bswap32_i32:
+        tcg_out_rev32(s, a0, a1);
+        break;
+    case INDEX_op_bswap16_i64:
+    case INDEX_op_bswap16_i32:
+        tcg_out_rev16(s, a0, a1);
+        break;
+
+    case INDEX_op_ext8s_i64:
+    case INDEX_op_ext8s_i32:
+        tcg_out_sxt(s, ext, MO_8, a0, a1);
+        break;
+    case INDEX_op_ext16s_i64:
+    case INDEX_op_ext16s_i32:
+        tcg_out_sxt(s, ext, MO_16, a0, a1);
+        break;
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_ext32s_i64:
+        tcg_out_sxt(s, TCG_TYPE_I64, MO_32, a0, a1);
+        break;
+    case INDEX_op_ext8u_i64:
+    case INDEX_op_ext8u_i32:
+        tcg_out_uxt(s, MO_8, a0, a1);
+        break;
+    case INDEX_op_ext16u_i64:
+    case INDEX_op_ext16u_i32:
+        tcg_out_uxt(s, MO_16, a0, a1);
+        break;
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_ext32u_i64:
+        tcg_out_movr(s, TCG_TYPE_I32, a0, a1);
+        break;
+
+    case INDEX_op_deposit_i64:
+    case INDEX_op_deposit_i32:
+        tcg_out_dep(s, ext, a0, REG0(2), args[3], args[4]);
+        break;
+
+    case INDEX_op_add2_i32:
+        tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
+                        (int32_t)args[4], args[5], const_args[4],
+                        const_args[5], false);
+        break;
+    case INDEX_op_add2_i64:
+        tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
+                        args[5], const_args[4], const_args[5], false);
+        break;
+    case INDEX_op_sub2_i32:
+        tcg_out_addsub2(s, TCG_TYPE_I32, a0, a1, REG0(2), REG0(3),
+                        (int32_t)args[4], args[5], const_args[4],
+                        const_args[5], true);
+        break;
+    case INDEX_op_sub2_i64:
+        tcg_out_addsub2(s, TCG_TYPE_I64, a0, a1, REG0(2), REG0(3), args[4],
+                        args[5], const_args[4], const_args[5], true);
+        break;
+
+    case INDEX_op_muluh_i64:
+        tcg_out_insn(s, 3508, UMULH, TCG_TYPE_I64, a0, a1, a2);
+        break;
+    case INDEX_op_mulsh_i64:
+        tcg_out_insn(s, 3508, SMULH, TCG_TYPE_I64, a0, a1, a2);
+        break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+
+#undef REG0
+}
+
+static const TCGTargetOpDef aarch64_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_ld8u_i64, { "r", "r" } },
+    { INDEX_op_ld8s_i64, { "r", "r" } },
+    { INDEX_op_ld16u_i64, { "r", "r" } },
+    { INDEX_op_ld16s_i64, { "r", "r" } },
+    { INDEX_op_ld32u_i64, { "r", "r" } },
+    { INDEX_op_ld32s_i64, { "r", "r" } },
+    { INDEX_op_ld_i64, { "r", "r" } },
+
+    { INDEX_op_st8_i32, { "rZ", "r" } },
+    { INDEX_op_st16_i32, { "rZ", "r" } },
+    { INDEX_op_st_i32, { "rZ", "r" } },
+    { INDEX_op_st8_i64, { "rZ", "r" } },
+    { INDEX_op_st16_i64, { "rZ", "r" } },
+    { INDEX_op_st32_i64, { "rZ", "r" } },
+    { INDEX_op_st_i64, { "rZ", "r" } },
+
+    { INDEX_op_add_i32, { "r", "r", "rA" } },
+    { INDEX_op_add_i64, { "r", "r", "rA" } },
+    { INDEX_op_sub_i32, { "r", "r", "rA" } },
+    { INDEX_op_sub_i64, { "r", "r", "rA" } },
+    { INDEX_op_mul_i32, { "r", "r", "r" } },
+    { INDEX_op_mul_i64, { "r", "r", "r" } },
+    { INDEX_op_div_i32, { "r", "r", "r" } },
+    { INDEX_op_div_i64, { "r", "r", "r" } },
+    { INDEX_op_divu_i32, { "r", "r", "r" } },
+    { INDEX_op_divu_i64, { "r", "r", "r" } },
+    { INDEX_op_rem_i32, { "r", "r", "r" } },
+    { INDEX_op_rem_i64, { "r", "r", "r" } },
+    { INDEX_op_remu_i32, { "r", "r", "r" } },
+    { INDEX_op_remu_i64, { "r", "r", "r" } },
+    { INDEX_op_and_i32, { "r", "r", "rL" } },
+    { INDEX_op_and_i64, { "r", "r", "rL" } },
+    { INDEX_op_or_i32, { "r", "r", "rL" } },
+    { INDEX_op_or_i64, { "r", "r", "rL" } },
+    { INDEX_op_xor_i32, { "r", "r", "rL" } },
+    { INDEX_op_xor_i64, { "r", "r", "rL" } },
+    { INDEX_op_andc_i32, { "r", "r", "rL" } },
+    { INDEX_op_andc_i64, { "r", "r", "rL" } },
+    { INDEX_op_orc_i32, { "r", "r", "rL" } },
+    { INDEX_op_orc_i64, { "r", "r", "rL" } },
+    { INDEX_op_eqv_i32, { "r", "r", "rL" } },
+    { INDEX_op_eqv_i64, { "r", "r", "rL" } },
+
+    { INDEX_op_neg_i32, { "r", "r" } },
+    { INDEX_op_neg_i64, { "r", "r" } },
+    { INDEX_op_not_i32, { "r", "r" } },
+    { INDEX_op_not_i64, { "r", "r" } },
+
+    { INDEX_op_shl_i32, { "r", "r", "ri" } },
+    { INDEX_op_shr_i32, { "r", "r", "ri" } },
+    { INDEX_op_sar_i32, { "r", "r", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
+    { INDEX_op_shl_i64, { "r", "r", "ri" } },
+    { INDEX_op_shr_i64, { "r", "r", "ri" } },
+    { INDEX_op_sar_i64, { "r", "r", "ri" } },
+    { INDEX_op_rotl_i64, { "r", "r", "ri" } },
+    { INDEX_op_rotr_i64, { "r", "r", "ri" } },
+
+    { INDEX_op_brcond_i32, { "r", "rA" } },
+    { INDEX_op_brcond_i64, { "r", "rA" } },
+    { INDEX_op_setcond_i32, { "r", "r", "rA" } },
+    { INDEX_op_setcond_i64, { "r", "r", "rA" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rA", "rZ", "rZ" } },
+    { INDEX_op_movcond_i64, { "r", "r", "rA", "rZ", "rZ" } },
+
+    { INDEX_op_qemu_ld_i32, { "r", "l" } },
+    { INDEX_op_qemu_ld_i64, { "r", "l" } },
+    { INDEX_op_qemu_st_i32, { "lZ", "l" } },
+    { INDEX_op_qemu_st_i64, { "lZ", "l" } },
+
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
+    { INDEX_op_bswap16_i64, { "r", "r" } },
+    { INDEX_op_bswap32_i64, { "r", "r" } },
+    { INDEX_op_bswap64_i64, { "r", "r" } },
+
+    { INDEX_op_ext8s_i32, { "r", "r" } },
+    { INDEX_op_ext16s_i32, { "r", "r" } },
+    { INDEX_op_ext8u_i32, { "r", "r" } },
+    { INDEX_op_ext16u_i32, { "r", "r" } },
+
+    { INDEX_op_ext8s_i64, { "r", "r" } },
+    { INDEX_op_ext16s_i64, { "r", "r" } },
+    { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext8u_i64, { "r", "r" } },
+    { INDEX_op_ext16u_i64, { "r", "r" } },
+    { INDEX_op_ext32u_i64, { "r", "r" } },
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+    { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
+
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
+    { INDEX_op_add2_i64, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
+    { INDEX_op_sub2_i64, { "r", "r", "rZ", "rZ", "rA", "rMZ" } },
+
+    { INDEX_op_muluh_i64, { "r", "r", "r" } },
+    { INDEX_op_mulsh_i64, { "r", "r", "r" } },
+
+    { -1 },
+};
+
+static void tcg_target_init(TCGContext *s)
+{
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
+
+    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
+                     (1 << TCG_REG_X0) | (1 << TCG_REG_X1) |
+                     (1 << TCG_REG_X2) | (1 << TCG_REG_X3) |
+                     (1 << TCG_REG_X4) | (1 << TCG_REG_X5) |
+                     (1 << TCG_REG_X6) | (1 << TCG_REG_X7) |
+                     (1 << TCG_REG_X8) | (1 << TCG_REG_X9) |
+                     (1 << TCG_REG_X10) | (1 << TCG_REG_X11) |
+                     (1 << TCG_REG_X12) | (1 << TCG_REG_X13) |
+                     (1 << TCG_REG_X14) | (1 << TCG_REG_X15) |
+                     (1 << TCG_REG_X16) | (1 << TCG_REG_X17) |
+                     (1 << TCG_REG_X18) | (1 << TCG_REG_X30));
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
+
+    tcg_add_target_add_op_defs(aarch64_op_defs);
+}
+
+/* Saving pairs: (X19, X20) .. (X27, X28), (X29(fp), X30(lr)).  */
+#define PUSH_SIZE  ((30 - 19 + 1) * 8)
+
+#define FRAME_SIZE \
+    ((PUSH_SIZE \
+      + TCG_STATIC_CALL_ARGS_SIZE \
+      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
+      + TCG_TARGET_STACK_ALIGN - 1) \
+     & ~(TCG_TARGET_STACK_ALIGN - 1))
+
+/* We're expecting a 2 byte uleb128 encoded value.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
+
+/* We're expecting to use a single ADDI insn.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE - PUSH_SIZE > 0xfff);
+
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    TCGReg r;
+
+    /* Push (FP, LR) and allocate space for all saved registers.  */
+    tcg_out_insn(s, 3314, STP, TCG_REG_FP, TCG_REG_LR,
+                 TCG_REG_SP, -PUSH_SIZE, 1, 1);
+
+    /* Set up frame pointer for canonical unwinding.  */
+    tcg_out_movr_sp(s, TCG_TYPE_I64, TCG_REG_FP, TCG_REG_SP);
+
+    /* Store callee-preserved regs x19..x28.  */
+    for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
+        int ofs = (r - TCG_REG_X19 + 2) * 8;
+        tcg_out_insn(s, 3314, STP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
+    }
+
+    /* Make stack space for TCG locals.  */
+    tcg_out_insn(s, 3401, SUBI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
+                 FRAME_SIZE - PUSH_SIZE);
+
+    /* Inform TCG about how to find TCG locals with register, offset, size.  */
+    tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+
+#if !defined(CONFIG_SOFTMMU)
+    if (USE_GUEST_BASE) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, guest_base);
+        tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
+    }
+#endif
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+    tcg_out_insn(s, 3207, BR, tcg_target_call_iarg_regs[1]);
+
+    tb_ret_addr = s->code_ptr;
+
+    /* Remove TCG locals stack space.  */
+    tcg_out_insn(s, 3401, ADDI, TCG_TYPE_I64, TCG_REG_SP, TCG_REG_SP,
+                 FRAME_SIZE - PUSH_SIZE);
+
+    /* Restore registers x19..x28.  */
+    for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
+        int ofs = (r - TCG_REG_X19 + 2) * 8;
+        tcg_out_insn(s, 3314, LDP, r, r + 1, TCG_REG_SP, ofs, 1, 0);
+    }
+
+    /* Pop (FP, LR), restore SP to previous frame.  */
+    tcg_out_insn(s, 3314, LDP, TCG_REG_FP, TCG_REG_LR,
+                 TCG_REG_SP, PUSH_SIZE, 0, 1);
+    tcg_out_insn(s, 3207, RET, TCG_REG_LR);
+}
+
+typedef struct {
+    DebugFrameHeader h;
+    uint8_t fde_def_cfa[4];
+    uint8_t fde_reg_ofs[24];
+} DebugFrame;
+
+#define ELF_HOST_MACHINE EM_AARCH64
+
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x78,             /* sleb128 -8 */
+    .h.cie.return_column = TCG_REG_LR,
+
+    /* Total FDE size does not include the "len" member.  */
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, TCG_REG_SP,                 /* DW_CFA_def_cfa sp, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        0x80 + 28, 1,                   /* DW_CFA_offset, x28,  -8 */
+        0x80 + 27, 2,                   /* DW_CFA_offset, x27, -16 */
+        0x80 + 26, 3,                   /* DW_CFA_offset, x26, -24 */
+        0x80 + 25, 4,                   /* DW_CFA_offset, x25, -32 */
+        0x80 + 24, 5,                   /* DW_CFA_offset, x24, -40 */
+        0x80 + 23, 6,                   /* DW_CFA_offset, x23, -48 */
+        0x80 + 22, 7,                   /* DW_CFA_offset, x22, -56 */
+        0x80 + 21, 8,                   /* DW_CFA_offset, x21, -64 */
+        0x80 + 20, 9,                   /* DW_CFA_offset, x20, -72 */
+        0x80 + 19, 10,                  /* DW_CFA_offset, x1p, -80 */
+        0x80 + 30, 11,                  /* DW_CFA_offset,  lr, -88 */
+        0x80 + 29, 12,                  /* DW_CFA_offset,  fp, -96 */
+    }
+};
+
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
+}
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
deleted file mode 100644
index a078a22..0000000
--- a/tcg/arm/tcg-target.c
+++ /dev/null
@@ -1,2128 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008 Andrzej Zaborowski
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "elf.h"
-#include "tcg-be-ldst.h"
-
-/* The __ARM_ARCH define is provided by gcc 4.8.  Construct it otherwise.  */
-#ifndef __ARM_ARCH
-# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
-     || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
-     || defined(__ARM_ARCH_7EM__)
-#  define __ARM_ARCH 7
-# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
-       || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) \
-       || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6T2__)
-#  define __ARM_ARCH 6
-# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5E__) \
-       || defined(__ARM_ARCH_5T__) || defined(__ARM_ARCH_5TE__) \
-       || defined(__ARM_ARCH_5TEJ__)
-#  define __ARM_ARCH 5
-# else
-#  define __ARM_ARCH 4
-# endif
-#endif
-
-static int arm_arch = __ARM_ARCH;
-
-#if defined(__ARM_ARCH_5T__) \
-    || defined(__ARM_ARCH_5TE__) || defined(__ARM_ARCH_5TEJ__)
-# define use_armv5t_instructions 1
-#else
-# define use_armv5t_instructions use_armv6_instructions
-#endif
-
-#define use_armv6_instructions  (__ARM_ARCH >= 6 || arm_arch >= 6)
-#define use_armv7_instructions  (__ARM_ARCH >= 7 || arm_arch >= 7)
-
-#ifndef use_idiv_instructions
-bool use_idiv_instructions;
-#endif
-
-/* ??? Ought to think about changing CONFIG_SOFTMMU to always defined.  */
-#ifdef CONFIG_SOFTMMU
-# define USING_SOFTMMU 1
-#else
-# define USING_SOFTMMU 0
-#endif
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "%r0",
-    "%r1",
-    "%r2",
-    "%r3",
-    "%r4",
-    "%r5",
-    "%r6",
-    "%r7",
-    "%r8",
-    "%r9",
-    "%r10",
-    "%r11",
-    "%r12",
-    "%r13",
-    "%r14",
-    "%pc",
-};
-#endif
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-    TCG_REG_R11,
-    TCG_REG_R13,
-    TCG_REG_R0,
-    TCG_REG_R1,
-    TCG_REG_R2,
-    TCG_REG_R3,
-    TCG_REG_R12,
-    TCG_REG_R14,
-};
-
-static const int tcg_target_call_iarg_regs[4] = {
-    TCG_REG_R0, TCG_REG_R1, TCG_REG_R2, TCG_REG_R3
-};
-static const int tcg_target_call_oarg_regs[2] = {
-    TCG_REG_R0, TCG_REG_R1
-};
-
-#define TCG_REG_TMP  TCG_REG_R12
-
-static inline void reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
-{
-    ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
-    *code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff);
-}
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    assert(type == R_ARM_PC24);
-    assert(addend == 0);
-    reloc_pc24(code_ptr, (tcg_insn_unit *)value);
-}
-
-#define TCG_CT_CONST_ARM  0x100
-#define TCG_CT_CONST_INV  0x200
-#define TCG_CT_CONST_NEG  0x400
-#define TCG_CT_CONST_ZERO 0x800
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'I':
-        ct->ct |= TCG_CT_CONST_ARM;
-        break;
-    case 'K':
-        ct->ct |= TCG_CT_CONST_INV;
-        break;
-    case 'N': /* The gcc constraint letter is L, already used here.  */
-        ct->ct |= TCG_CT_CONST_NEG;
-        break;
-    case 'Z':
-        ct->ct |= TCG_CT_CONST_ZERO;
-        break;
-
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
-        break;
-
-    /* qemu_ld address */
-    case 'l':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
-#ifdef CONFIG_SOFTMMU
-        /* r0-r2,lr will be overwritten when reading the tlb entry,
-           so don't use these. */
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R14);
-#endif
-        break;
-
-    /* qemu_st address & data */
-    case 's':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
-        /* r0-r2 will be overwritten when reading the tlb entry (softmmu only)
-           and r0-r1 doing the byte swapping, so don't use these. */
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
-#if defined(CONFIG_SOFTMMU)
-        /* Avoid clashes with registers being used for helper args */
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
-#if TARGET_LONG_BITS == 64
-        /* Avoid clashes with registers being used for helper args */
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
-#endif
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R14);
-#endif
-        break;
-
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-
-    return 0;
-}
-
-static inline uint32_t rotl(uint32_t val, int n)
-{
-  return (val << n) | (val >> (32 - n));
-}
-
-/* ARM immediates for ALU instructions are made of an unsigned 8-bit
-   right-rotated by an even amount between 0 and 30. */
-static inline int encode_imm(uint32_t imm)
-{
-    int shift;
-
-    /* simple case, only lower bits */
-    if ((imm & ~0xff) == 0)
-        return 0;
-    /* then try a simple even shift */
-    shift = ctz32(imm) & ~1;
-    if (((imm >> shift) & ~0xff) == 0)
-        return 32 - shift;
-    /* now try harder with rotations */
-    if ((rotl(imm, 2) & ~0xff) == 0)
-        return 2;
-    if ((rotl(imm, 4) & ~0xff) == 0)
-        return 4;
-    if ((rotl(imm, 6) & ~0xff) == 0)
-        return 6;
-    /* imm can't be encoded */
-    return -1;
-}
-
-static inline int check_fit_imm(uint32_t imm)
-{
-    return encode_imm(imm) >= 0;
-}
-
-/* Test if a constant matches the constraint.
- * TODO: define constraints for:
- *
- * ldr/str offset:   between -0xfff and 0xfff
- * ldrh/strh offset: between -0xff and 0xff
- * mov operand2:     values represented with x << (2 * y), x < 0x100
- * add, sub, eor...: ditto
- */
-static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct;
-    ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_ARM) && check_fit_imm(val)) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_INV) && check_fit_imm(~val)) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_NEG) && check_fit_imm(-val)) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-#define TO_CPSR (1 << 20)
-
-typedef enum {
-    ARITH_AND = 0x0 << 21,
-    ARITH_EOR = 0x1 << 21,
-    ARITH_SUB = 0x2 << 21,
-    ARITH_RSB = 0x3 << 21,
-    ARITH_ADD = 0x4 << 21,
-    ARITH_ADC = 0x5 << 21,
-    ARITH_SBC = 0x6 << 21,
-    ARITH_RSC = 0x7 << 21,
-    ARITH_TST = 0x8 << 21 | TO_CPSR,
-    ARITH_CMP = 0xa << 21 | TO_CPSR,
-    ARITH_CMN = 0xb << 21 | TO_CPSR,
-    ARITH_ORR = 0xc << 21,
-    ARITH_MOV = 0xd << 21,
-    ARITH_BIC = 0xe << 21,
-    ARITH_MVN = 0xf << 21,
-
-    INSN_LDR_IMM   = 0x04100000,
-    INSN_LDR_REG   = 0x06100000,
-    INSN_STR_IMM   = 0x04000000,
-    INSN_STR_REG   = 0x06000000,
-
-    INSN_LDRH_IMM  = 0x005000b0,
-    INSN_LDRH_REG  = 0x001000b0,
-    INSN_LDRSH_IMM = 0x005000f0,
-    INSN_LDRSH_REG = 0x001000f0,
-    INSN_STRH_IMM  = 0x004000b0,
-    INSN_STRH_REG  = 0x000000b0,
-
-    INSN_LDRB_IMM  = 0x04500000,
-    INSN_LDRB_REG  = 0x06500000,
-    INSN_LDRSB_IMM = 0x005000d0,
-    INSN_LDRSB_REG = 0x001000d0,
-    INSN_STRB_IMM  = 0x04400000,
-    INSN_STRB_REG  = 0x06400000,
-
-    INSN_LDRD_IMM  = 0x004000d0,
-    INSN_LDRD_REG  = 0x000000d0,
-    INSN_STRD_IMM  = 0x004000f0,
-    INSN_STRD_REG  = 0x000000f0,
-} ARMInsn;
-
-#define SHIFT_IMM_LSL(im)	(((im) << 7) | 0x00)
-#define SHIFT_IMM_LSR(im)	(((im) << 7) | 0x20)
-#define SHIFT_IMM_ASR(im)	(((im) << 7) | 0x40)
-#define SHIFT_IMM_ROR(im)	(((im) << 7) | 0x60)
-#define SHIFT_REG_LSL(rs)	(((rs) << 8) | 0x10)
-#define SHIFT_REG_LSR(rs)	(((rs) << 8) | 0x30)
-#define SHIFT_REG_ASR(rs)	(((rs) << 8) | 0x50)
-#define SHIFT_REG_ROR(rs)	(((rs) << 8) | 0x70)
-
-enum arm_cond_code_e {
-    COND_EQ = 0x0,
-    COND_NE = 0x1,
-    COND_CS = 0x2,	/* Unsigned greater or equal */
-    COND_CC = 0x3,	/* Unsigned less than */
-    COND_MI = 0x4,	/* Negative */
-    COND_PL = 0x5,	/* Zero or greater */
-    COND_VS = 0x6,	/* Overflow */
-    COND_VC = 0x7,	/* No overflow */
-    COND_HI = 0x8,	/* Unsigned greater than */
-    COND_LS = 0x9,	/* Unsigned less or equal */
-    COND_GE = 0xa,
-    COND_LT = 0xb,
-    COND_GT = 0xc,
-    COND_LE = 0xd,
-    COND_AL = 0xe,
-};
-
-static const uint8_t tcg_cond_to_arm_cond[] = {
-    [TCG_COND_EQ] = COND_EQ,
-    [TCG_COND_NE] = COND_NE,
-    [TCG_COND_LT] = COND_LT,
-    [TCG_COND_GE] = COND_GE,
-    [TCG_COND_LE] = COND_LE,
-    [TCG_COND_GT] = COND_GT,
-    /* unsigned */
-    [TCG_COND_LTU] = COND_CC,
-    [TCG_COND_GEU] = COND_CS,
-    [TCG_COND_LEU] = COND_LS,
-    [TCG_COND_GTU] = COND_HI,
-};
-
-static inline void tcg_out_bx(TCGContext *s, int cond, int rn)
-{
-    tcg_out32(s, (cond << 28) | 0x012fff10 | rn);
-}
-
-static inline void tcg_out_b(TCGContext *s, int cond, int32_t offset)
-{
-    tcg_out32(s, (cond << 28) | 0x0a000000 |
-                    (((offset - 8) >> 2) & 0x00ffffff));
-}
-
-static inline void tcg_out_b_noaddr(TCGContext *s, int cond)
-{
-    /* We pay attention here to not modify the branch target by masking
-       the corresponding bytes.  This ensure that caches and memory are
-       kept coherent during retranslation. */
-    tcg_out32(s, deposit32(*s->code_ptr, 24, 8, (cond << 4) | 0x0a));
-}
-
-static inline void tcg_out_bl_noaddr(TCGContext *s, int cond)
-{
-    /* We pay attention here to not modify the branch target by masking
-       the corresponding bytes.  This ensure that caches and memory are
-       kept coherent during retranslation. */
-    tcg_out32(s, deposit32(*s->code_ptr, 24, 8, (cond << 4) | 0x0b));
-}
-
-static inline void tcg_out_bl(TCGContext *s, int cond, int32_t offset)
-{
-    tcg_out32(s, (cond << 28) | 0x0b000000 |
-                    (((offset - 8) >> 2) & 0x00ffffff));
-}
-
-static inline void tcg_out_blx(TCGContext *s, int cond, int rn)
-{
-    tcg_out32(s, (cond << 28) | 0x012fff30 | rn);
-}
-
-static inline void tcg_out_blx_imm(TCGContext *s, int32_t offset)
-{
-    tcg_out32(s, 0xfa000000 | ((offset & 2) << 23) |
-                (((offset - 8) >> 2) & 0x00ffffff));
-}
-
-static inline void tcg_out_dat_reg(TCGContext *s,
-                int cond, int opc, int rd, int rn, int rm, int shift)
-{
-    tcg_out32(s, (cond << 28) | (0 << 25) | opc |
-                    (rn << 16) | (rd << 12) | shift | rm);
-}
-
-static inline void tcg_out_nop(TCGContext *s)
-{
-    if (use_armv7_instructions) {
-        /* Architected nop introduced in v6k.  */
-        /* ??? This is an MSR (imm) 0,0,0 insn.  Anyone know if this
-           also Just So Happened to do nothing on pre-v6k so that we
-           don't need to conditionalize it?  */
-        tcg_out32(s, 0xe320f000);
-    } else {
-        /* Prior to that the assembler uses mov r0, r0.  */
-        tcg_out_dat_reg(s, COND_AL, ARITH_MOV, 0, 0, 0, SHIFT_IMM_LSL(0));
-    }
-}
-
-static inline void tcg_out_mov_reg(TCGContext *s, int cond, int rd, int rm)
-{
-    /* Simple reg-reg move, optimising out the 'do nothing' case */
-    if (rd != rm) {
-        tcg_out_dat_reg(s, cond, ARITH_MOV, rd, 0, rm, SHIFT_IMM_LSL(0));
-    }
-}
-
-static inline void tcg_out_dat_imm(TCGContext *s,
-                int cond, int opc, int rd, int rn, int im)
-{
-    tcg_out32(s, (cond << 28) | (1 << 25) | opc |
-                    (rn << 16) | (rd << 12) | im);
-}
-
-static void tcg_out_movi32(TCGContext *s, int cond, int rd, uint32_t arg)
-{
-    int rot, opc, rn;
-
-    /* For armv7, make sure not to use movw+movt when mov/mvn would do.
-       Speed things up by only checking when movt would be required.
-       Prior to armv7, have one go at fully rotated immediates before
-       doing the decomposition thing below.  */
-    if (!use_armv7_instructions || (arg & 0xffff0000)) {
-        rot = encode_imm(arg);
-        if (rot >= 0) {
-            tcg_out_dat_imm(s, cond, ARITH_MOV, rd, 0,
-                            rotl(arg, rot) | (rot << 7));
-            return;
-        }
-        rot = encode_imm(~arg);
-        if (rot >= 0) {
-            tcg_out_dat_imm(s, cond, ARITH_MVN, rd, 0,
-                            rotl(~arg, rot) | (rot << 7));
-            return;
-        }
-    }
-
-    /* Use movw + movt.  */
-    if (use_armv7_instructions) {
-        /* movw */
-        tcg_out32(s, (cond << 28) | 0x03000000 | (rd << 12)
-                  | ((arg << 4) & 0x000f0000) | (arg & 0xfff));
-        if (arg & 0xffff0000) {
-            /* movt */
-            tcg_out32(s, (cond << 28) | 0x03400000 | (rd << 12)
-                      | ((arg >> 12) & 0x000f0000) | ((arg >> 16) & 0xfff));
-        }
-        return;
-    }
-
-    /* TODO: This is very suboptimal, we can easily have a constant
-       pool somewhere after all the instructions.  */
-    opc = ARITH_MOV;
-    rn = 0;
-    /* If we have lots of leading 1's, we can shorten the sequence by
-       beginning with mvn and then clearing higher bits with eor.  */
-    if (clz32(~arg) > clz32(arg)) {
-        opc = ARITH_MVN, arg = ~arg;
-    }
-    do {
-        int i = ctz32(arg) & ~1;
-        rot = ((32 - i) << 7) & 0xf00;
-        tcg_out_dat_imm(s, cond, opc, rd, rn, ((arg >> i) & 0xff) | rot);
-        arg &= ~(0xff << i);
-
-        opc = ARITH_EOR;
-        rn = rd;
-    } while (arg);
-}
-
-static inline void tcg_out_dat_rI(TCGContext *s, int cond, int opc, TCGArg dst,
-                                  TCGArg lhs, TCGArg rhs, int rhs_is_const)
-{
-    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
-     * rhs must satisfy the "rI" constraint.
-     */
-    if (rhs_is_const) {
-        int rot = encode_imm(rhs);
-        assert(rot >= 0);
-        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
-    } else {
-        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
-    }
-}
-
-static void tcg_out_dat_rIK(TCGContext *s, int cond, int opc, int opinv,
-                            TCGReg dst, TCGReg lhs, TCGArg rhs,
-                            bool rhs_is_const)
-{
-    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
-     * rhs must satisfy the "rIK" constraint.
-     */
-    if (rhs_is_const) {
-        int rot = encode_imm(rhs);
-        if (rot < 0) {
-            rhs = ~rhs;
-            rot = encode_imm(rhs);
-            assert(rot >= 0);
-            opc = opinv;
-        }
-        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
-    } else {
-        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
-    }
-}
-
-static void tcg_out_dat_rIN(TCGContext *s, int cond, int opc, int opneg,
-                            TCGArg dst, TCGArg lhs, TCGArg rhs,
-                            bool rhs_is_const)
-{
-    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
-     * rhs must satisfy the "rIN" constraint.
-     */
-    if (rhs_is_const) {
-        int rot = encode_imm(rhs);
-        if (rot < 0) {
-            rhs = -rhs;
-            rot = encode_imm(rhs);
-            assert(rot >= 0);
-            opc = opneg;
-        }
-        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
-    } else {
-        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
-    }
-}
-
-static inline void tcg_out_mul32(TCGContext *s, int cond, TCGReg rd,
-                                 TCGReg rn, TCGReg rm)
-{
-    /* if ArchVersion() < 6 && d == n then UNPREDICTABLE;  */
-    if (!use_armv6_instructions && rd == rn) {
-        if (rd == rm) {
-            /* rd == rn == rm; copy an input to tmp first.  */
-            tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn);
-            rm = rn = TCG_REG_TMP;
-        } else {
-            rn = rm;
-            rm = rd;
-        }
-    }
-    /* mul */
-    tcg_out32(s, (cond << 28) | 0x90 | (rd << 16) | (rm << 8) | rn);
-}
-
-static inline void tcg_out_umull32(TCGContext *s, int cond, TCGReg rd0,
-                                   TCGReg rd1, TCGReg rn, TCGReg rm)
-{
-    /* if ArchVersion() < 6 && (dHi == n || dLo == n) then UNPREDICTABLE;  */
-    if (!use_armv6_instructions && (rd0 == rn || rd1 == rn)) {
-        if (rd0 == rm || rd1 == rm) {
-            tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn);
-            rn = TCG_REG_TMP;
-        } else {
-            TCGReg t = rn;
-            rn = rm;
-            rm = t;
-        }
-    }
-    /* umull */
-    tcg_out32(s, (cond << 28) | 0x00800090 |
-              (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn);
-}
-
-static inline void tcg_out_smull32(TCGContext *s, int cond, TCGReg rd0,
-                                   TCGReg rd1, TCGReg rn, TCGReg rm)
-{
-    /* if ArchVersion() < 6 && (dHi == n || dLo == n) then UNPREDICTABLE;  */
-    if (!use_armv6_instructions && (rd0 == rn || rd1 == rn)) {
-        if (rd0 == rm || rd1 == rm) {
-            tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn);
-            rn = TCG_REG_TMP;
-        } else {
-            TCGReg t = rn;
-            rn = rm;
-            rm = t;
-        }
-    }
-    /* smull */
-    tcg_out32(s, (cond << 28) | 0x00c00090 |
-              (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn);
-}
-
-static inline void tcg_out_sdiv(TCGContext *s, int cond, int rd, int rn, int rm)
-{
-    tcg_out32(s, 0x0710f010 | (cond << 28) | (rd << 16) | rn | (rm << 8));
-}
-
-static inline void tcg_out_udiv(TCGContext *s, int cond, int rd, int rn, int rm)
-{
-    tcg_out32(s, 0x0730f010 | (cond << 28) | (rd << 16) | rn | (rm << 8));
-}
-
-static inline void tcg_out_ext8s(TCGContext *s, int cond,
-                                 int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* sxtb */
-        tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rn, SHIFT_IMM_LSL(24));
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rd, SHIFT_IMM_ASR(24));
-    }
-}
-
-static inline void tcg_out_ext8u(TCGContext *s, int cond,
-                                 int rd, int rn)
-{
-    tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, 0xff);
-}
-
-static inline void tcg_out_ext16s(TCGContext *s, int cond,
-                                  int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* sxth */
-        tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rn, SHIFT_IMM_LSL(16));
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rd, SHIFT_IMM_ASR(16));
-    }
-}
-
-static inline void tcg_out_ext16u(TCGContext *s, int cond,
-                                  int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* uxth */
-        tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rn, SHIFT_IMM_LSL(16));
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rd, SHIFT_IMM_LSR(16));
-    }
-}
-
-static inline void tcg_out_bswap16s(TCGContext *s, int cond, int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* revsh */
-        tcg_out32(s, 0x06ff0fb0 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        TCG_REG_TMP, 0, rn, SHIFT_IMM_LSL(24));
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        TCG_REG_TMP, 0, TCG_REG_TMP, SHIFT_IMM_ASR(16));
-        tcg_out_dat_reg(s, cond, ARITH_ORR,
-                        rd, TCG_REG_TMP, rn, SHIFT_IMM_LSR(8));
-    }
-}
-
-static inline void tcg_out_bswap16(TCGContext *s, int cond, int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* rev16 */
-        tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        TCG_REG_TMP, 0, rn, SHIFT_IMM_LSL(24));
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        TCG_REG_TMP, 0, TCG_REG_TMP, SHIFT_IMM_LSR(16));
-        tcg_out_dat_reg(s, cond, ARITH_ORR,
-                        rd, TCG_REG_TMP, rn, SHIFT_IMM_LSR(8));
-    }
-}
-
-/* swap the two low bytes assuming that the two high input bytes and the
-   two high output bit can hold any value. */
-static inline void tcg_out_bswap16st(TCGContext *s, int cond, int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* rev16 */
-        tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        TCG_REG_TMP, 0, rn, SHIFT_IMM_LSR(8));
-        tcg_out_dat_imm(s, cond, ARITH_AND, TCG_REG_TMP, TCG_REG_TMP, 0xff);
-        tcg_out_dat_reg(s, cond, ARITH_ORR,
-                        rd, TCG_REG_TMP, rn, SHIFT_IMM_LSL(8));
-    }
-}
-
-static inline void tcg_out_bswap32(TCGContext *s, int cond, int rd, int rn)
-{
-    if (use_armv6_instructions) {
-        /* rev */
-        tcg_out32(s, 0x06bf0f30 | (cond << 28) | (rd << 12) | rn);
-    } else {
-        tcg_out_dat_reg(s, cond, ARITH_EOR,
-                        TCG_REG_TMP, rn, rn, SHIFT_IMM_ROR(16));
-        tcg_out_dat_imm(s, cond, ARITH_BIC,
-                        TCG_REG_TMP, TCG_REG_TMP, 0xff | 0x800);
-        tcg_out_dat_reg(s, cond, ARITH_MOV,
-                        rd, 0, rn, SHIFT_IMM_ROR(8));
-        tcg_out_dat_reg(s, cond, ARITH_EOR,
-                        rd, rd, TCG_REG_TMP, SHIFT_IMM_LSR(8));
-    }
-}
-
-bool tcg_target_deposit_valid(int ofs, int len)
-{
-    /* ??? Without bfi, we could improve over generic code by combining
-       the right-shift from a non-zero ofs with the orr.  We do run into
-       problems when rd == rs, and the mask generated from ofs+len doesn't
-       fit into an immediate.  We would have to be careful not to pessimize
-       wrt the optimizations performed on the expanded code.  */
-    return use_armv7_instructions;
-}
-
-static inline void tcg_out_deposit(TCGContext *s, int cond, TCGReg rd,
-                                   TCGArg a1, int ofs, int len, bool const_a1)
-{
-    if (const_a1) {
-        /* bfi becomes bfc with rn == 15.  */
-        a1 = 15;
-    }
-    /* bfi/bfc */
-    tcg_out32(s, 0x07c00010 | (cond << 28) | (rd << 12) | a1
-              | (ofs << 7) | ((ofs + len - 1) << 16));
-}
-
-/* Note that this routine is used for both LDR and LDRH formats, so we do
-   not wish to include an immediate shift at this point.  */
-static void tcg_out_memop_r(TCGContext *s, int cond, ARMInsn opc, TCGReg rt,
-                            TCGReg rn, TCGReg rm, bool u, bool p, bool w)
-{
-    tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24)
-              | (w << 21) | (rn << 16) | (rt << 12) | rm);
-}
-
-static void tcg_out_memop_8(TCGContext *s, int cond, ARMInsn opc, TCGReg rt,
-                            TCGReg rn, int imm8, bool p, bool w)
-{
-    bool u = 1;
-    if (imm8 < 0) {
-        imm8 = -imm8;
-        u = 0;
-    }
-    tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) | (w << 21) |
-              (rn << 16) | (rt << 12) | ((imm8 & 0xf0) << 4) | (imm8 & 0xf));
-}
-
-static void tcg_out_memop_12(TCGContext *s, int cond, ARMInsn opc, TCGReg rt,
-                             TCGReg rn, int imm12, bool p, bool w)
-{
-    bool u = 1;
-    if (imm12 < 0) {
-        imm12 = -imm12;
-        u = 0;
-    }
-    tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) | (w << 21) |
-              (rn << 16) | (rt << 12) | imm12);
-}
-
-static inline void tcg_out_ld32_12(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, int imm12)
-{
-    tcg_out_memop_12(s, cond, INSN_LDR_IMM, rt, rn, imm12, 1, 0);
-}
-
-static inline void tcg_out_st32_12(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, int imm12)
-{
-    tcg_out_memop_12(s, cond, INSN_STR_IMM, rt, rn, imm12, 1, 0);
-}
-
-static inline void tcg_out_ld32_r(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDR_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_st32_r(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_ldrd_8(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, int imm8)
-{
-    tcg_out_memop_8(s, cond, INSN_LDRD_IMM, rt, rn, imm8, 1, 0);
-}
-
-static inline void tcg_out_ldrd_r(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDRD_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_strd_8(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, int imm8)
-{
-    tcg_out_memop_8(s, cond, INSN_STRD_IMM, rt, rn, imm8, 1, 0);
-}
-
-static inline void tcg_out_strd_r(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_STRD_REG, rt, rn, rm, 1, 1, 0);
-}
-
-/* Register pre-increment with base writeback.  */
-static inline void tcg_out_ld32_rwb(TCGContext *s, int cond, TCGReg rt,
-                                    TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDR_REG, rt, rn, rm, 1, 1, 1);
-}
-
-static inline void tcg_out_st32_rwb(TCGContext *s, int cond, TCGReg rt,
-                                    TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 1);
-}
-
-static inline void tcg_out_ld16u_8(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, int imm8)
-{
-    tcg_out_memop_8(s, cond, INSN_LDRH_IMM, rt, rn, imm8, 1, 0);
-}
-
-static inline void tcg_out_st16_8(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, int imm8)
-{
-    tcg_out_memop_8(s, cond, INSN_STRH_IMM, rt, rn, imm8, 1, 0);
-}
-
-static inline void tcg_out_ld16u_r(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDRH_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_st16_r(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_STRH_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_ld16s_8(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, int imm8)
-{
-    tcg_out_memop_8(s, cond, INSN_LDRSH_IMM, rt, rn, imm8, 1, 0);
-}
-
-static inline void tcg_out_ld16s_r(TCGContext *s, int cond, TCGReg rt,
-                                   TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDRSH_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_ld8_12(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, int imm12)
-{
-    tcg_out_memop_12(s, cond, INSN_LDRB_IMM, rt, rn, imm12, 1, 0);
-}
-
-static inline void tcg_out_st8_12(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, int imm12)
-{
-    tcg_out_memop_12(s, cond, INSN_STRB_IMM, rt, rn, imm12, 1, 0);
-}
-
-static inline void tcg_out_ld8_r(TCGContext *s, int cond, TCGReg rt,
-                                 TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDRB_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_st8_r(TCGContext *s, int cond, TCGReg rt,
-                                 TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_STRB_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_ld8s_8(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, int imm8)
-{
-    tcg_out_memop_8(s, cond, INSN_LDRSB_IMM, rt, rn, imm8, 1, 0);
-}
-
-static inline void tcg_out_ld8s_r(TCGContext *s, int cond, TCGReg rt,
-                                  TCGReg rn, TCGReg rm)
-{
-    tcg_out_memop_r(s, cond, INSN_LDRSB_REG, rt, rn, rm, 1, 1, 0);
-}
-
-static inline void tcg_out_ld32u(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xfff || offset < -0xfff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_ld32_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_ld32_12(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_st32(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xfff || offset < -0xfff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_st32_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_st32_12(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_ld16u(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xff || offset < -0xff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_ld16u_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_ld16u_8(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_ld16s(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xff || offset < -0xff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_ld16s_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_ld16s_8(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_st16(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xff || offset < -0xff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_st16_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_st16_8(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_ld8u(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xfff || offset < -0xfff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_ld8_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_ld8_12(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_ld8s(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xff || offset < -0xff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_ld8s_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_ld8s_8(s, cond, rd, rn, offset);
-}
-
-static inline void tcg_out_st8(TCGContext *s, int cond,
-                int rd, int rn, int32_t offset)
-{
-    if (offset > 0xfff || offset < -0xfff) {
-        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
-        tcg_out_st8_r(s, cond, rd, rn, TCG_REG_TMP);
-    } else
-        tcg_out_st8_12(s, cond, rd, rn, offset);
-}
-
-/* The _goto case is normally between TBs within the same code buffer, and
- * with the code buffer limited to 16MB we wouldn't need the long case.
- * But we also use it for the tail-call to the qemu_ld/st helpers, which does.
- */
-static inline void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr)
-{
-    intptr_t addri = (intptr_t)addr;
-    ptrdiff_t disp = tcg_pcrel_diff(s, addr);
-
-    if ((addri & 1) == 0 && disp - 8 < 0x01fffffd && disp - 8 > -0x01fffffd) {
-        tcg_out_b(s, cond, disp);
-        return;
-    }
-
-    tcg_out_movi32(s, cond, TCG_REG_TMP, addri);
-    if (use_armv5t_instructions) {
-        tcg_out_bx(s, cond, TCG_REG_TMP);
-    } else {
-        if (addri & 1) {
-            tcg_abort();
-        }
-        tcg_out_mov_reg(s, cond, TCG_REG_PC, TCG_REG_TMP);
-    }
-}
-
-/* The call case is mostly used for helpers - so it's not unreasonable
- * for them to be beyond branch range */
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr)
-{
-    intptr_t addri = (intptr_t)addr;
-    ptrdiff_t disp = tcg_pcrel_diff(s, addr);
-
-    if (disp - 8 < 0x02000000 && disp - 8 >= -0x02000000) {
-        if (addri & 1) {
-            /* Use BLX if the target is in Thumb mode */
-            if (!use_armv5t_instructions) {
-                tcg_abort();
-            }
-            tcg_out_blx_imm(s, disp);
-        } else {
-            tcg_out_bl(s, COND_AL, disp);
-        }
-    } else if (use_armv7_instructions) {
-        tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri);
-        tcg_out_blx(s, COND_AL, TCG_REG_TMP);
-    } else {
-        tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R14, TCG_REG_PC, 4);
-        tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, -4);
-        tcg_out32(s, addri);
-    }
-}
-
-static inline void tcg_out_goto_label(TCGContext *s, int cond, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        tcg_out_goto(s, cond, l->u.value_ptr);
-    } else {
-        tcg_out_reloc(s, s->code_ptr, R_ARM_PC24, label_index, 0);
-        tcg_out_b_noaddr(s, cond);
-    }
-}
-
-#ifdef CONFIG_SOFTMMU
-/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
- *                                     int mmu_idx, uintptr_t ra)
- */
-static void * const qemu_ld_helpers[16] = {
-    [MO_UB]   = helper_ret_ldub_mmu,
-    [MO_SB]   = helper_ret_ldsb_mmu,
-
-    [MO_LEUW] = helper_le_lduw_mmu,
-    [MO_LEUL] = helper_le_ldul_mmu,
-    [MO_LEQ]  = helper_le_ldq_mmu,
-    [MO_LESW] = helper_le_ldsw_mmu,
-    [MO_LESL] = helper_le_ldul_mmu,
-
-    [MO_BEUW] = helper_be_lduw_mmu,
-    [MO_BEUL] = helper_be_ldul_mmu,
-    [MO_BEQ]  = helper_be_ldq_mmu,
-    [MO_BESW] = helper_be_ldsw_mmu,
-    [MO_BESL] = helper_be_ldul_mmu,
-};
-
-/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
- *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
- */
-static void * const qemu_st_helpers[16] = {
-    [MO_UB]   = helper_ret_stb_mmu,
-    [MO_LEUW] = helper_le_stw_mmu,
-    [MO_LEUL] = helper_le_stl_mmu,
-    [MO_LEQ]  = helper_le_stq_mmu,
-    [MO_BEUW] = helper_be_stw_mmu,
-    [MO_BEUL] = helper_be_stl_mmu,
-    [MO_BEQ]  = helper_be_stq_mmu,
-};
-
-/* Helper routines for marshalling helper function arguments into
- * the correct registers and stack.
- * argreg is where we want to put this argument, arg is the argument itself.
- * Return value is the updated argreg ready for the next call.
- * Note that argreg 0..3 is real registers, 4+ on stack.
- *
- * We provide routines for arguments which are: immediate, 32 bit
- * value in register, 16 and 8 bit values in register (which must be zero
- * extended before use) and 64 bit value in a lo:hi register pair.
- */
-#define DEFINE_TCG_OUT_ARG(NAME, ARGTYPE, MOV_ARG, EXT_ARG)                \
-static TCGReg NAME(TCGContext *s, TCGReg argreg, ARGTYPE arg)              \
-{                                                                          \
-    if (argreg < 4) {                                                      \
-        MOV_ARG(s, COND_AL, argreg, arg);                                  \
-    } else {                                                               \
-        int ofs = (argreg - 4) * 4;                                        \
-        EXT_ARG;                                                           \
-        assert(ofs + 4 <= TCG_STATIC_CALL_ARGS_SIZE);                      \
-        tcg_out_st32_12(s, COND_AL, arg, TCG_REG_CALL_STACK, ofs);         \
-    }                                                                      \
-    return argreg + 1;                                                     \
-}
-
-DEFINE_TCG_OUT_ARG(tcg_out_arg_imm32, uint32_t, tcg_out_movi32,
-    (tcg_out_movi32(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u,
-    (tcg_out_ext8u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u,
-    (tcg_out_ext16u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
-DEFINE_TCG_OUT_ARG(tcg_out_arg_reg32, TCGReg, tcg_out_mov_reg, )
-
-static TCGReg tcg_out_arg_reg64(TCGContext *s, TCGReg argreg,
-                                TCGReg arglo, TCGReg arghi)
-{
-    /* 64 bit arguments must go in even/odd register pairs
-     * and in 8-aligned stack slots.
-     */
-    if (argreg & 1) {
-        argreg++;
-    }
-    if (use_armv6_instructions && argreg >= 4
-        && (arglo & 1) == 0 && arghi == arglo + 1) {
-        tcg_out_strd_8(s, COND_AL, arglo,
-                       TCG_REG_CALL_STACK, (argreg - 4) * 4);
-        return argreg + 2;
-    } else {
-        argreg = tcg_out_arg_reg32(s, argreg, arglo);
-        argreg = tcg_out_arg_reg32(s, argreg, arghi);
-        return argreg;
-    }
-}
-
-#define TLB_SHIFT	(CPU_TLB_ENTRY_BITS + CPU_TLB_BITS)
-
-/* We're expecting to use an 8-bit immediate and to mask.  */
-QEMU_BUILD_BUG_ON(CPU_TLB_BITS > 8);
-
-/* We're expecting to use an 8-bit immediate add + 8-bit ldrd offset.
-   Using the offset of the second entry in the last tlb table ensures
-   that we can index all of the elements of the first entry.  */
-QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
-                  > 0xffff);
-
-/* Load and compare a TLB entry, leaving the flags set.  Returns the register
-   containing the addend of the tlb entry.  Clobbers R0, R1, R2, TMP.  */
-
-static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
-                               TCGMemOp s_bits, int mem_index, bool is_load)
-{
-    TCGReg base = TCG_AREG0;
-    int cmp_off =
-        (is_load
-         ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
-         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
-    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
-
-    /* Should generate something like the following:
-     *   shr    tmp, addrlo, #TARGET_PAGE_BITS                    (1)
-     *   add    r2, env, #high
-     *   and    r0, tmp, #(CPU_TLB_SIZE - 1)                      (2)
-     *   add    r2, r2, r0, lsl #CPU_TLB_ENTRY_BITS               (3)
-     *   ldr    r0, [r2, #cmp]                                    (4)
-     *   tst    addrlo, #s_mask
-     *   ldr    r2, [r2, #add]                                    (5)
-     *   cmpeq  r0, tmp, lsl #TARGET_PAGE_BITS
-     */
-    tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP,
-                    0, addrlo, SHIFT_IMM_LSR(TARGET_PAGE_BITS));
-
-    /* We checked that the offset is contained within 16 bits above.  */
-    if (add_off > 0xfff || (use_armv6_instructions && cmp_off > 0xff)) {
-        tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R2, base,
-                        (24 << 7) | (cmp_off >> 8));
-        base = TCG_REG_R2;
-        add_off -= cmp_off & 0xff00;
-        cmp_off &= 0xff;
-    }
-
-    tcg_out_dat_imm(s, COND_AL, ARITH_AND,
-                    TCG_REG_R0, TCG_REG_TMP, CPU_TLB_SIZE - 1);
-    tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R2, base,
-                    TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
-
-    /* Load the tlb comparator.  Use ldrd if needed and available,
-       but due to how the pointer needs setting up, ldm isn't useful.
-       Base arm5 doesn't have ldrd, but armv5te does.  */
-    if (use_armv6_instructions && TARGET_LONG_BITS == 64) {
-        tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_REG_R2, cmp_off);
-    } else {
-        tcg_out_ld32_12(s, COND_AL, TCG_REG_R0, TCG_REG_R2, cmp_off);
-        if (TARGET_LONG_BITS == 64) {
-            tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R2, cmp_off + 4);
-        }
-    }
-
-    /* Check alignment.  */
-    if (s_bits) {
-        tcg_out_dat_imm(s, COND_AL, ARITH_TST,
-                        0, addrlo, (1 << s_bits) - 1);
-    }
-
-    /* Load the tlb addend.  */
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R2, TCG_REG_R2, add_off);
-
-    tcg_out_dat_reg(s, (s_bits ? COND_EQ : COND_AL), ARITH_CMP, 0,
-                    TCG_REG_R0, TCG_REG_TMP, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
-
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
-                        TCG_REG_R1, addrhi, SHIFT_IMM_LSL(0));
-    }
-
-    return TCG_REG_R2;
-}
-
-/* Record the context of a call to the out of line helper code for the slow
-   path for a load or store, so that we can later generate the correct
-   helper code.  */
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
-                                TCGReg datalo, TCGReg datahi, TCGReg addrlo,
-                                TCGReg addrhi, tcg_insn_unit *raddr,
-                                tcg_insn_unit *label_ptr)
-{
-    TCGLabelQemuLdst *label = new_ldst_label(s);
-
-    label->is_ld = is_ld;
-    label->oi = oi;
-    label->datalo_reg = datalo;
-    label->datahi_reg = datahi;
-    label->addrlo_reg = addrlo;
-    label->addrhi_reg = addrhi;
-    label->raddr = raddr;
-    label->label_ptr[0] = label_ptr;
-}
-
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGReg argreg, datalo, datahi;
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    void *func;
-
-    reloc_pc24(lb->label_ptr[0], s->code_ptr);
-
-    argreg = tcg_out_arg_reg32(s, TCG_REG_R0, TCG_AREG0);
-    if (TARGET_LONG_BITS == 64) {
-        argreg = tcg_out_arg_reg64(s, argreg, lb->addrlo_reg, lb->addrhi_reg);
-    } else {
-        argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
-    }
-    argreg = tcg_out_arg_imm32(s, argreg, oi);
-    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);
-
-    /* For armv6 we can use the canonical unsigned helpers and minimize
-       icache usage.  For pre-armv6, use the signed helpers since we do
-       not have a single insn sign-extend.  */
-    if (use_armv6_instructions) {
-        func = qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)];
-    } else {
-        func = qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)];
-        if (opc & MO_SIGN) {
-            opc = MO_UL;
-        }
-    }
-    tcg_out_call(s, func);
-
-    datalo = lb->datalo_reg;
-    datahi = lb->datahi_reg;
-    switch (opc & MO_SSIZE) {
-    case MO_SB:
-        tcg_out_ext8s(s, COND_AL, datalo, TCG_REG_R0);
-        break;
-    case MO_SW:
-        tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
-        break;
-    default:
-        tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
-        break;
-    case MO_Q:
-        if (datalo != TCG_REG_R1) {
-            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
-            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
-        } else if (datahi != TCG_REG_R0) {
-            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
-            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
-        } else {
-            tcg_out_mov_reg(s, COND_AL, TCG_REG_TMP, TCG_REG_R0);
-            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
-            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_TMP);
-        }
-        break;
-    }
-
-    tcg_out_goto(s, COND_AL, lb->raddr);
-}
-
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGReg argreg, datalo, datahi;
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-
-    reloc_pc24(lb->label_ptr[0], s->code_ptr);
-
-    argreg = TCG_REG_R0;
-    argreg = tcg_out_arg_reg32(s, argreg, TCG_AREG0);
-    if (TARGET_LONG_BITS == 64) {
-        argreg = tcg_out_arg_reg64(s, argreg, lb->addrlo_reg, lb->addrhi_reg);
-    } else {
-        argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
-    }
-
-    datalo = lb->datalo_reg;
-    datahi = lb->datahi_reg;
-    switch (opc & MO_SIZE) {
-    case MO_8:
-        argreg = tcg_out_arg_reg8(s, argreg, datalo);
-        break;
-    case MO_16:
-        argreg = tcg_out_arg_reg16(s, argreg, datalo);
-        break;
-    case MO_32:
-    default:
-        argreg = tcg_out_arg_reg32(s, argreg, datalo);
-        break;
-    case MO_64:
-        argreg = tcg_out_arg_reg64(s, argreg, datalo, datahi);
-        break;
-    }
-
-    argreg = tcg_out_arg_imm32(s, argreg, oi);
-    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);
-
-    /* Tail-call to the helper, which will return to the fast path.  */
-    tcg_out_goto(s, COND_AL, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-}
-#endif /* SOFTMMU */
-
-static inline void tcg_out_qemu_ld_index(TCGContext *s, TCGMemOp opc,
-                                         TCGReg datalo, TCGReg datahi,
-                                         TCGReg addrlo, TCGReg addend)
-{
-    TCGMemOp bswap = opc & MO_BSWAP;
-
-    switch (opc & MO_SSIZE) {
-    case MO_UB:
-        tcg_out_ld8_r(s, COND_AL, datalo, addrlo, addend);
-        break;
-    case MO_SB:
-        tcg_out_ld8s_r(s, COND_AL, datalo, addrlo, addend);
-        break;
-    case MO_UW:
-        tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend);
-        if (bswap) {
-            tcg_out_bswap16(s, COND_AL, datalo, datalo);
-        }
-        break;
-    case MO_SW:
-        if (bswap) {
-            tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend);
-            tcg_out_bswap16s(s, COND_AL, datalo, datalo);
-        } else {
-            tcg_out_ld16s_r(s, COND_AL, datalo, addrlo, addend);
-        }
-        break;
-    case MO_UL:
-    default:
-        tcg_out_ld32_r(s, COND_AL, datalo, addrlo, addend);
-        if (bswap) {
-            tcg_out_bswap32(s, COND_AL, datalo, datalo);
-        }
-        break;
-    case MO_Q:
-        {
-            TCGReg dl = (bswap ? datahi : datalo);
-            TCGReg dh = (bswap ? datalo : datahi);
-
-            /* Avoid ldrd for user-only emulation, to handle unaligned.  */
-            if (USING_SOFTMMU && use_armv6_instructions
-                && (dl & 1) == 0 && dh == dl + 1) {
-                tcg_out_ldrd_r(s, COND_AL, dl, addrlo, addend);
-            } else if (dl != addend) {
-                tcg_out_ld32_rwb(s, COND_AL, dl, addend, addrlo);
-                tcg_out_ld32_12(s, COND_AL, dh, addend, 4);
-            } else {
-                tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_TMP,
-                                addend, addrlo, SHIFT_IMM_LSL(0));
-                tcg_out_ld32_12(s, COND_AL, dl, TCG_REG_TMP, 0);
-                tcg_out_ld32_12(s, COND_AL, dh, TCG_REG_TMP, 4);
-            }
-            if (bswap) {
-                tcg_out_bswap32(s, COND_AL, dl, dl);
-                tcg_out_bswap32(s, COND_AL, dh, dh);
-            }
-        }
-        break;
-    }
-}
-
-static inline void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp opc,
-                                          TCGReg datalo, TCGReg datahi,
-                                          TCGReg addrlo)
-{
-    TCGMemOp bswap = opc & MO_BSWAP;
-
-    switch (opc & MO_SSIZE) {
-    case MO_UB:
-        tcg_out_ld8_12(s, COND_AL, datalo, addrlo, 0);
-        break;
-    case MO_SB:
-        tcg_out_ld8s_8(s, COND_AL, datalo, addrlo, 0);
-        break;
-    case MO_UW:
-        tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0);
-        if (bswap) {
-            tcg_out_bswap16(s, COND_AL, datalo, datalo);
-        }
-        break;
-    case MO_SW:
-        if (bswap) {
-            tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0);
-            tcg_out_bswap16s(s, COND_AL, datalo, datalo);
-        } else {
-            tcg_out_ld16s_8(s, COND_AL, datalo, addrlo, 0);
-        }
-        break;
-    case MO_UL:
-    default:
-        tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0);
-        if (bswap) {
-            tcg_out_bswap32(s, COND_AL, datalo, datalo);
-        }
-        break;
-    case MO_Q:
-        {
-            TCGReg dl = (bswap ? datahi : datalo);
-            TCGReg dh = (bswap ? datalo : datahi);
-
-            /* Avoid ldrd for user-only emulation, to handle unaligned.  */
-            if (USING_SOFTMMU && use_armv6_instructions
-                && (dl & 1) == 0 && dh == dl + 1) {
-                tcg_out_ldrd_8(s, COND_AL, dl, addrlo, 0);
-            } else if (dl == addrlo) {
-                tcg_out_ld32_12(s, COND_AL, dh, addrlo, bswap ? 0 : 4);
-                tcg_out_ld32_12(s, COND_AL, dl, addrlo, bswap ? 4 : 0);
-            } else {
-                tcg_out_ld32_12(s, COND_AL, dl, addrlo, bswap ? 4 : 0);
-                tcg_out_ld32_12(s, COND_AL, dh, addrlo, bswap ? 0 : 4);
-            }
-            if (bswap) {
-                tcg_out_bswap32(s, COND_AL, dl, dl);
-                tcg_out_bswap32(s, COND_AL, dh, dh);
-            }
-        }
-        break;
-    }
-}
-
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
-{
-    TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused));
-    TCGMemOpIdx oi;
-    TCGMemOp opc;
-#ifdef CONFIG_SOFTMMU
-    int mem_index;
-    TCGReg addend;
-    tcg_insn_unit *label_ptr;
-#endif
-
-    datalo = *args++;
-    datahi = (is64 ? *args++ : 0);
-    addrlo = *args++;
-    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-
-#ifdef CONFIG_SOFTMMU
-    mem_index = get_mmuidx(oi);
-    addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 1);
-
-    /* This a conditional BL only to load a pointer within this opcode into LR
-       for the slow path.  We will not be using the value for a tail call.  */
-    label_ptr = s->code_ptr;
-    tcg_out_bl_noaddr(s, COND_NE);
-
-    tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend);
-
-    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
-                        s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
-        tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_TMP);
-    } else {
-        tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo);
-    }
-#endif
-}
-
-static inline void tcg_out_qemu_st_index(TCGContext *s, int cond, TCGMemOp opc,
-                                         TCGReg datalo, TCGReg datahi,
-                                         TCGReg addrlo, TCGReg addend)
-{
-    TCGMemOp bswap = opc & MO_BSWAP;
-
-    switch (opc & MO_SIZE) {
-    case MO_8:
-        tcg_out_st8_r(s, cond, datalo, addrlo, addend);
-        break;
-    case MO_16:
-        if (bswap) {
-            tcg_out_bswap16st(s, cond, TCG_REG_R0, datalo);
-            tcg_out_st16_r(s, cond, TCG_REG_R0, addrlo, addend);
-        } else {
-            tcg_out_st16_r(s, cond, datalo, addrlo, addend);
-        }
-        break;
-    case MO_32:
-    default:
-        if (bswap) {
-            tcg_out_bswap32(s, cond, TCG_REG_R0, datalo);
-            tcg_out_st32_r(s, cond, TCG_REG_R0, addrlo, addend);
-        } else {
-            tcg_out_st32_r(s, cond, datalo, addrlo, addend);
-        }
-        break;
-    case MO_64:
-        /* Avoid strd for user-only emulation, to handle unaligned.  */
-        if (bswap) {
-            tcg_out_bswap32(s, cond, TCG_REG_R0, datahi);
-            tcg_out_st32_rwb(s, cond, TCG_REG_R0, addend, addrlo);
-            tcg_out_bswap32(s, cond, TCG_REG_R0, datalo);
-            tcg_out_st32_12(s, cond, TCG_REG_R0, addend, 4);
-        } else if (USING_SOFTMMU && use_armv6_instructions
-                   && (datalo & 1) == 0 && datahi == datalo + 1) {
-            tcg_out_strd_r(s, cond, datalo, addrlo, addend);
-        } else {
-            tcg_out_st32_rwb(s, cond, datalo, addend, addrlo);
-            tcg_out_st32_12(s, cond, datahi, addend, 4);
-        }
-        break;
-    }
-}
-
-static inline void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp opc,
-                                          TCGReg datalo, TCGReg datahi,
-                                          TCGReg addrlo)
-{
-    TCGMemOp bswap = opc & MO_BSWAP;
-
-    switch (opc & MO_SIZE) {
-    case MO_8:
-        tcg_out_st8_12(s, COND_AL, datalo, addrlo, 0);
-        break;
-    case MO_16:
-        if (bswap) {
-            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, datalo);
-            tcg_out_st16_8(s, COND_AL, TCG_REG_R0, addrlo, 0);
-        } else {
-            tcg_out_st16_8(s, COND_AL, datalo, addrlo, 0);
-        }
-        break;
-    case MO_32:
-    default:
-        if (bswap) {
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datalo);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 0);
-        } else {
-            tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0);
-        }
-        break;
-    case MO_64:
-        /* Avoid strd for user-only emulation, to handle unaligned.  */
-        if (bswap) {
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datahi);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 0);
-            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datalo);
-            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 4);
-        } else if (USING_SOFTMMU && use_armv6_instructions
-                   && (datalo & 1) == 0 && datahi == datalo + 1) {
-            tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0);
-        } else {
-            tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0);
-            tcg_out_st32_12(s, COND_AL, datahi, addrlo, 4);
-        }
-        break;
-    }
-}
-
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
-{
-    TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused));
-    TCGMemOpIdx oi;
-    TCGMemOp opc;
-#ifdef CONFIG_SOFTMMU
-    int mem_index;
-    TCGReg addend;
-    tcg_insn_unit *label_ptr;
-#endif
-
-    datalo = *args++;
-    datahi = (is64 ? *args++ : 0);
-    addrlo = *args++;
-    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-
-#ifdef CONFIG_SOFTMMU
-    mem_index = get_mmuidx(oi);
-    addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 0);
-
-    tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, addrlo, addend);
-
-    /* The conditional call must come last, as we're going to return here.  */
-    label_ptr = s->code_ptr;
-    tcg_out_bl_noaddr(s, COND_NE);
-
-    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
-                        s->code_ptr, label_ptr);
-#else /* !CONFIG_SOFTMMU */
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, GUEST_BASE);
-        tcg_out_qemu_st_index(s, COND_AL, opc, datalo,
-                              datahi, addrlo, TCG_REG_TMP);
-    } else {
-        tcg_out_qemu_st_direct(s, opc, datalo, datahi, addrlo);
-    }
-#endif
-}
-
-static tcg_insn_unit *tb_ret_addr;
-
-static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                const TCGArg *args, const int *const_args)
-{
-    TCGArg a0, a1, a2, a3, a4, a5;
-    int c;
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_movi32(s, COND_AL, TCG_REG_R0, args[0]);
-        tcg_out_goto(s, COND_AL, tb_ret_addr);
-        break;
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* Direct jump method */
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            tcg_out_b_noaddr(s, COND_AL);
-        } else {
-            /* Indirect jump method */
-            intptr_t ptr = (intptr_t)(s->tb_next + args[0]);
-            tcg_out_movi32(s, COND_AL, TCG_REG_R0, ptr & ~0xfff);
-            tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, ptr & 0xfff);
-        }
-        s->tb_next_offset[args[0]] = tcg_current_code_size(s);
-        break;
-    case INDEX_op_br:
-        tcg_out_goto_label(s, COND_AL, args[0]);
-        break;
-
-    case INDEX_op_ld8u_i32:
-        tcg_out_ld8u(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld8s_i32:
-        tcg_out_ld8s(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld16u_i32:
-        tcg_out_ld16u(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld16s_i32:
-        tcg_out_ld16s(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld_i32:
-        tcg_out_ld32u(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st8_i32:
-        tcg_out_st8(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st16_i32:
-        tcg_out_st16(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i32:
-        tcg_out_st32(s, COND_AL, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_movcond_i32:
-        /* Constraints mean that v2 is always in the same register as dest,
-         * so we only need to do "if condition passed, move v1 to dest".
-         */
-        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
-                        args[1], args[2], const_args[2]);
-        tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[args[5]], ARITH_MOV,
-                        ARITH_MVN, args[0], 0, args[3], const_args[3]);
-        break;
-    case INDEX_op_add_i32:
-        tcg_out_dat_rIN(s, COND_AL, ARITH_ADD, ARITH_SUB,
-                        args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_sub_i32:
-        if (const_args[1]) {
-            if (const_args[2]) {
-                tcg_out_movi32(s, COND_AL, args[0], args[1] - args[2]);
-            } else {
-                tcg_out_dat_rI(s, COND_AL, ARITH_RSB,
-                               args[0], args[2], args[1], 1);
-            }
-        } else {
-            tcg_out_dat_rIN(s, COND_AL, ARITH_SUB, ARITH_ADD,
-                            args[0], args[1], args[2], const_args[2]);
-        }
-        break;
-    case INDEX_op_and_i32:
-        tcg_out_dat_rIK(s, COND_AL, ARITH_AND, ARITH_BIC,
-                        args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_andc_i32:
-        tcg_out_dat_rIK(s, COND_AL, ARITH_BIC, ARITH_AND,
-                        args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_or_i32:
-        c = ARITH_ORR;
-        goto gen_arith;
-    case INDEX_op_xor_i32:
-        c = ARITH_EOR;
-        /* Fall through.  */
-    gen_arith:
-        tcg_out_dat_rI(s, COND_AL, c, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_add2_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        a3 = args[3], a4 = args[4], a5 = args[5];
-        if (a0 == a3 || (a0 == a5 && !const_args[5])) {
-            a0 = TCG_REG_TMP;
-        }
-        tcg_out_dat_rIN(s, COND_AL, ARITH_ADD | TO_CPSR, ARITH_SUB | TO_CPSR,
-                        a0, a2, a4, const_args[4]);
-        tcg_out_dat_rIK(s, COND_AL, ARITH_ADC, ARITH_SBC,
-                        a1, a3, a5, const_args[5]);
-        tcg_out_mov_reg(s, COND_AL, args[0], a0);
-        break;
-    case INDEX_op_sub2_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        a3 = args[3], a4 = args[4], a5 = args[5];
-        if ((a0 == a3 && !const_args[3]) || (a0 == a5 && !const_args[5])) {
-            a0 = TCG_REG_TMP;
-        }
-        if (const_args[2]) {
-            if (const_args[4]) {
-                tcg_out_movi32(s, COND_AL, a0, a4);
-                a4 = a0;
-            }
-            tcg_out_dat_rI(s, COND_AL, ARITH_RSB | TO_CPSR, a0, a4, a2, 1);
-        } else {
-            tcg_out_dat_rIN(s, COND_AL, ARITH_SUB | TO_CPSR,
-                            ARITH_ADD | TO_CPSR, a0, a2, a4, const_args[4]);
-        }
-        if (const_args[3]) {
-            if (const_args[5]) {
-                tcg_out_movi32(s, COND_AL, a1, a5);
-                a5 = a1;
-            }
-            tcg_out_dat_rI(s, COND_AL, ARITH_RSC, a1, a5, a3, 1);
-        } else {
-            tcg_out_dat_rIK(s, COND_AL, ARITH_SBC, ARITH_ADC,
-                            a1, a3, a5, const_args[5]);
-        }
-        tcg_out_mov_reg(s, COND_AL, args[0], a0);
-        break;
-    case INDEX_op_neg_i32:
-        tcg_out_dat_imm(s, COND_AL, ARITH_RSB, args[0], args[1], 0);
-        break;
-    case INDEX_op_not_i32:
-        tcg_out_dat_reg(s, COND_AL,
-                        ARITH_MVN, args[0], 0, args[1], SHIFT_IMM_LSL(0));
-        break;
-    case INDEX_op_mul_i32:
-        tcg_out_mul32(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_mulu2_i32:
-        tcg_out_umull32(s, COND_AL, args[0], args[1], args[2], args[3]);
-        break;
-    case INDEX_op_muls2_i32:
-        tcg_out_smull32(s, COND_AL, args[0], args[1], args[2], args[3]);
-        break;
-    /* XXX: Perhaps args[2] & 0x1f is wrong */
-    case INDEX_op_shl_i32:
-        c = const_args[2] ?
-                SHIFT_IMM_LSL(args[2] & 0x1f) : SHIFT_REG_LSL(args[2]);
-        goto gen_shift32;
-    case INDEX_op_shr_i32:
-        c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_LSR(args[2] & 0x1f) :
-                SHIFT_IMM_LSL(0) : SHIFT_REG_LSR(args[2]);
-        goto gen_shift32;
-    case INDEX_op_sar_i32:
-        c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_ASR(args[2] & 0x1f) :
-                SHIFT_IMM_LSL(0) : SHIFT_REG_ASR(args[2]);
-        goto gen_shift32;
-    case INDEX_op_rotr_i32:
-        c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_ROR(args[2] & 0x1f) :
-                SHIFT_IMM_LSL(0) : SHIFT_REG_ROR(args[2]);
-        /* Fall through.  */
-    gen_shift32:
-        tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1], c);
-        break;
-
-    case INDEX_op_rotl_i32:
-        if (const_args[2]) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1],
-                            ((0x20 - args[2]) & 0x1f) ?
-                            SHIFT_IMM_ROR((0x20 - args[2]) & 0x1f) :
-                            SHIFT_IMM_LSL(0));
-        } else {
-            tcg_out_dat_imm(s, COND_AL, ARITH_RSB, TCG_REG_TMP, args[2], 0x20);
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1],
-                            SHIFT_REG_ROR(TCG_REG_TMP));
-        }
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
-                       args[0], args[1], const_args[1]);
-        tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[2]], args[3]);
-        break;
-    case INDEX_op_brcond2_i32:
-        /* The resulting conditions are:
-         * TCG_COND_EQ    -->  a0 == a2 && a1 == a3,
-         * TCG_COND_NE    --> (a0 != a2 && a1 == a3) ||  a1 != a3,
-         * TCG_COND_LT(U) --> (a0 <  a2 && a1 == a3) ||  a1 <  a3,
-         * TCG_COND_GE(U) --> (a0 >= a2 && a1 == a3) || (a1 >= a3 && a1 != a3),
-         * TCG_COND_LE(U) --> (a0 <= a2 && a1 == a3) || (a1 <= a3 && a1 != a3),
-         * TCG_COND_GT(U) --> (a0 >  a2 && a1 == a3) ||  a1 >  a3,
-         */
-        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
-                        args[1], args[3], const_args[3]);
-        tcg_out_dat_rIN(s, COND_EQ, ARITH_CMP, ARITH_CMN, 0,
-                        args[0], args[2], const_args[2]);
-        tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[4]], args[5]);
-        break;
-    case INDEX_op_setcond_i32:
-        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
-                        args[1], args[2], const_args[2]);
-        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[3]],
-                        ARITH_MOV, args[0], 0, 1);
-        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[3])],
-                        ARITH_MOV, args[0], 0, 0);
-        break;
-    case INDEX_op_setcond2_i32:
-        /* See brcond2_i32 comment */
-        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
-                        args[2], args[4], const_args[4]);
-        tcg_out_dat_rIN(s, COND_EQ, ARITH_CMP, ARITH_CMN, 0,
-                        args[1], args[3], const_args[3]);
-        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[5]],
-                        ARITH_MOV, args[0], 0, 1);
-        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[5])],
-                        ARITH_MOV, args[0], 0, 0);
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_qemu_ld(s, args, 0);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, args, 1);
-        break;
-    case INDEX_op_qemu_st_i32:
-        tcg_out_qemu_st(s, args, 0);
-        break;
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args, 1);
-        break;
-
-    case INDEX_op_bswap16_i32:
-        tcg_out_bswap16(s, COND_AL, args[0], args[1]);
-        break;
-    case INDEX_op_bswap32_i32:
-        tcg_out_bswap32(s, COND_AL, args[0], args[1]);
-        break;
-
-    case INDEX_op_ext8s_i32:
-        tcg_out_ext8s(s, COND_AL, args[0], args[1]);
-        break;
-    case INDEX_op_ext16s_i32:
-        tcg_out_ext16s(s, COND_AL, args[0], args[1]);
-        break;
-    case INDEX_op_ext16u_i32:
-        tcg_out_ext16u(s, COND_AL, args[0], args[1]);
-        break;
-
-    case INDEX_op_deposit_i32:
-        tcg_out_deposit(s, COND_AL, args[0], args[2],
-                        args[3], args[4], const_args[2]);
-        break;
-
-    case INDEX_op_div_i32:
-        tcg_out_sdiv(s, COND_AL, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_divu_i32:
-        tcg_out_udiv(s, COND_AL, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef arm_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "r", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
-
-    /* TODO: "r", "r", "ri" */
-    { INDEX_op_add_i32, { "r", "r", "rIN" } },
-    { INDEX_op_sub_i32, { "r", "rI", "rIN" } },
-    { INDEX_op_mul_i32, { "r", "r", "r" } },
-    { INDEX_op_mulu2_i32, { "r", "r", "r", "r" } },
-    { INDEX_op_muls2_i32, { "r", "r", "r", "r" } },
-    { INDEX_op_and_i32, { "r", "r", "rIK" } },
-    { INDEX_op_andc_i32, { "r", "r", "rIK" } },
-    { INDEX_op_or_i32, { "r", "r", "rI" } },
-    { INDEX_op_xor_i32, { "r", "r", "rI" } },
-    { INDEX_op_neg_i32, { "r", "r" } },
-    { INDEX_op_not_i32, { "r", "r" } },
-
-    { INDEX_op_shl_i32, { "r", "r", "ri" } },
-    { INDEX_op_shr_i32, { "r", "r", "ri" } },
-    { INDEX_op_sar_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
-
-    { INDEX_op_brcond_i32, { "r", "rIN" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rIN" } },
-    { INDEX_op_movcond_i32, { "r", "r", "rIN", "rIK", "0" } },
-
-    { INDEX_op_add2_i32, { "r", "r", "r", "r", "rIN", "rIK" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rI", "rI", "rIN", "rIK" } },
-    { INDEX_op_brcond2_i32, { "r", "r", "rIN", "rIN" } },
-    { INDEX_op_setcond2_i32, { "r", "r", "r", "rIN", "rIN" } },
-
-#if TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld_i32, { "r", "l" } },
-    { INDEX_op_qemu_ld_i64, { "r", "r", "l" } },
-    { INDEX_op_qemu_st_i32, { "s", "s" } },
-    { INDEX_op_qemu_st_i64, { "s", "s", "s" } },
-#else
-    { INDEX_op_qemu_ld_i32, { "r", "l", "l" } },
-    { INDEX_op_qemu_ld_i64, { "r", "r", "l", "l" } },
-    { INDEX_op_qemu_st_i32, { "s", "s", "s" } },
-    { INDEX_op_qemu_st_i64, { "s", "s", "s", "s" } },
-#endif
-
-    { INDEX_op_bswap16_i32, { "r", "r" } },
-    { INDEX_op_bswap32_i32, { "r", "r" } },
-
-    { INDEX_op_ext8s_i32, { "r", "r" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-    { INDEX_op_ext16u_i32, { "r", "r" } },
-
-    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
-
-    { INDEX_op_div_i32, { "r", "r", "r" } },
-    { INDEX_op_divu_i32, { "r", "r", "r" } },
-
-    { -1 },
-};
-
-static void tcg_target_init(TCGContext *s)
-{
-    /* Only probe for the platform and capabilities if we havn't already
-       determined maximum values at compile time.  */
-#ifndef use_idiv_instructions
-    {
-        unsigned long hwcap = qemu_getauxval(AT_HWCAP);
-        use_idiv_instructions = (hwcap & HWCAP_ARM_IDIVA) != 0;
-    }
-#endif
-    if (__ARM_ARCH < 7) {
-        const char *pl = (const char *)qemu_getauxval(AT_PLATFORM);
-        if (pl != NULL && pl[0] == 'v' && pl[1] >= '4' && pl[1] <= '9') {
-            arm_arch = pl[1] - '0';
-        }
-    }
-
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
-    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
-                     (1 << TCG_REG_R0) |
-                     (1 << TCG_REG_R1) |
-                     (1 << TCG_REG_R2) |
-                     (1 << TCG_REG_R3) |
-                     (1 << TCG_REG_R12) |
-                     (1 << TCG_REG_R14));
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_PC);
-
-    tcg_add_target_add_op_defs(arm_op_defs);
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ld32u(s, COND_AL, arg, arg1, arg2);
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_st32(s, COND_AL, arg, arg1, arg2);
-}
-
-static inline void tcg_out_mov(TCGContext *s, TCGType type,
-                               TCGReg ret, TCGReg arg)
-{
-    tcg_out_dat_reg(s, COND_AL, ARITH_MOV, ret, 0, arg, SHIFT_IMM_LSL(0));
-}
-
-static inline void tcg_out_movi(TCGContext *s, TCGType type,
-                                TCGReg ret, tcg_target_long arg)
-{
-    tcg_out_movi32(s, COND_AL, ret, arg);
-}
-
-/* Compute frame size via macros, to share between tcg_target_qemu_prologue
-   and tcg_register_jit.  */
-
-#define PUSH_SIZE  ((11 - 4 + 1 + 1) * sizeof(tcg_target_long))
-
-#define FRAME_SIZE \
-    ((PUSH_SIZE \
-      + TCG_STATIC_CALL_ARGS_SIZE \
-      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
-      + TCG_TARGET_STACK_ALIGN - 1) \
-     & -TCG_TARGET_STACK_ALIGN)
-
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int stack_addend;
-
-    /* Calling convention requires us to save r4-r11 and lr.  */
-    /* stmdb sp!, { r4 - r11, lr } */
-    tcg_out32(s, (COND_AL << 28) | 0x092d4ff0);
-
-    /* Reserve callee argument and tcg temp space.  */
-    stack_addend = FRAME_SIZE - PUSH_SIZE;
-
-    tcg_out_dat_rI(s, COND_AL, ARITH_SUB, TCG_REG_CALL_STACK,
-                   TCG_REG_CALL_STACK, stack_addend, 1);
-    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-
-    tcg_out_bx(s, COND_AL, tcg_target_call_iarg_regs[1]);
-    tb_ret_addr = s->code_ptr;
-
-    /* Epilogue.  We branch here via tb_ret_addr.  */
-    tcg_out_dat_rI(s, COND_AL, ARITH_ADD, TCG_REG_CALL_STACK,
-                   TCG_REG_CALL_STACK, stack_addend, 1);
-
-    /* ldmia sp!, { r4 - r11, pc } */
-    tcg_out32(s, (COND_AL << 28) | 0x08bd8ff0);
-}
-
-typedef struct {
-    DebugFrameHeader h;
-    uint8_t fde_def_cfa[4];
-    uint8_t fde_reg_ofs[18];
-} DebugFrame;
-
-#define ELF_HOST_MACHINE EM_ARM
-
-/* We're expecting a 2 byte uleb128 encoded value.  */
-QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
-
-static const DebugFrame debug_frame = {
-    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .h.cie.id = -1,
-    .h.cie.version = 1,
-    .h.cie.code_align = 1,
-    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
-    .h.cie.return_column = 14,
-
-    /* Total FDE size does not include the "len" member.  */
-    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
-
-    .fde_def_cfa = {
-        12, 13,                         /* DW_CFA_def_cfa sp, ... */
-        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
-        (FRAME_SIZE >> 7)
-    },
-    .fde_reg_ofs = {
-        /* The following must match the stmdb in the prologue.  */
-        0x8e, 1,                        /* DW_CFA_offset, lr, -4 */
-        0x8b, 2,                        /* DW_CFA_offset, r11, -8 */
-        0x8a, 3,                        /* DW_CFA_offset, r10, -12 */
-        0x89, 4,                        /* DW_CFA_offset, r9, -16 */
-        0x88, 5,                        /* DW_CFA_offset, r8, -20 */
-        0x87, 6,                        /* DW_CFA_offset, r7, -24 */
-        0x86, 7,                        /* DW_CFA_offset, r6, -28 */
-        0x85, 8,                        /* DW_CFA_offset, r5, -32 */
-        0x84, 9,                        /* DW_CFA_offset, r4, -36 */
-    }
-};
-
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index 1c719e2..a0e1acf 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -22,11 +22,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_ARM 
-#define TCG_TARGET_ARM 1
+
+#ifndef ARM_TCG_TARGET_H
+#define ARM_TCG_TARGET_H
 
 #undef TCG_TARGET_STACK_GROWSUP
 #define TCG_TARGET_INSN_UNIT_SIZE 4
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 16
 
 typedef enum {
     TCG_REG_R0 = 0,
diff --git a/tcg/arm/tcg-target.inc.c b/tcg/arm/tcg-target.inc.c
new file mode 100644
index 0000000..172feba
--- /dev/null
+++ b/tcg/arm/tcg-target.inc.c
@@ -0,0 +1,2152 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Andrzej Zaborowski
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "elf.h"
+#include "tcg-be-ldst.h"
+
+/* The __ARM_ARCH define is provided by gcc 4.8.  Construct it otherwise.  */
+#ifndef __ARM_ARCH
+# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
+     || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) \
+     || defined(__ARM_ARCH_7EM__)
+#  define __ARM_ARCH 7
+# elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \
+       || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) \
+       || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6T2__)
+#  define __ARM_ARCH 6
+# elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5E__) \
+       || defined(__ARM_ARCH_5T__) || defined(__ARM_ARCH_5TE__) \
+       || defined(__ARM_ARCH_5TEJ__)
+#  define __ARM_ARCH 5
+# else
+#  define __ARM_ARCH 4
+# endif
+#endif
+
+static int arm_arch = __ARM_ARCH;
+
+#if defined(__ARM_ARCH_5T__) \
+    || defined(__ARM_ARCH_5TE__) || defined(__ARM_ARCH_5TEJ__)
+# define use_armv5t_instructions 1
+#else
+# define use_armv5t_instructions use_armv6_instructions
+#endif
+
+#define use_armv6_instructions  (__ARM_ARCH >= 6 || arm_arch >= 6)
+#define use_armv7_instructions  (__ARM_ARCH >= 7 || arm_arch >= 7)
+
+#ifndef use_idiv_instructions
+bool use_idiv_instructions;
+#endif
+
+/* ??? Ought to think about changing CONFIG_SOFTMMU to always defined.  */
+#ifdef CONFIG_SOFTMMU
+# define USING_SOFTMMU 1
+#else
+# define USING_SOFTMMU 0
+#endif
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "%r0",
+    "%r1",
+    "%r2",
+    "%r3",
+    "%r4",
+    "%r5",
+    "%r6",
+    "%r7",
+    "%r8",
+    "%r9",
+    "%r10",
+    "%r11",
+    "%r12",
+    "%r13",
+    "%r14",
+    "%pc",
+};
+#endif
+
+static const int tcg_target_reg_alloc_order[] = {
+    TCG_REG_R4,
+    TCG_REG_R5,
+    TCG_REG_R6,
+    TCG_REG_R7,
+    TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
+    TCG_REG_R11,
+    TCG_REG_R13,
+    TCG_REG_R0,
+    TCG_REG_R1,
+    TCG_REG_R2,
+    TCG_REG_R3,
+    TCG_REG_R12,
+    TCG_REG_R14,
+};
+
+static const int tcg_target_call_iarg_regs[4] = {
+    TCG_REG_R0, TCG_REG_R1, TCG_REG_R2, TCG_REG_R3
+};
+static const int tcg_target_call_oarg_regs[2] = {
+    TCG_REG_R0, TCG_REG_R1
+};
+
+#define TCG_REG_TMP  TCG_REG_R12
+
+static inline void reloc_pc24(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+{
+    ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
+    *code_ptr = (*code_ptr & ~0xffffff) | (offset & 0xffffff);
+}
+
+static inline void reloc_pc24_atomic(tcg_insn_unit *code_ptr, tcg_insn_unit *target)
+{
+    ptrdiff_t offset = (tcg_ptr_byte_diff(target, code_ptr) - 8) >> 2;
+    tcg_insn_unit insn = atomic_read(code_ptr);
+    tcg_debug_assert(offset == sextract32(offset, 0, 24));
+    atomic_set(code_ptr, deposit32(insn, 0, 24, offset));
+}
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    tcg_debug_assert(type == R_ARM_PC24);
+    tcg_debug_assert(addend == 0);
+    reloc_pc24(code_ptr, (tcg_insn_unit *)value);
+}
+
+#define TCG_CT_CONST_ARM  0x100
+#define TCG_CT_CONST_INV  0x200
+#define TCG_CT_CONST_NEG  0x400
+#define TCG_CT_CONST_ZERO 0x800
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch (ct_str[0]) {
+    case 'I':
+        ct->ct |= TCG_CT_CONST_ARM;
+        break;
+    case 'K':
+        ct->ct |= TCG_CT_CONST_INV;
+        break;
+    case 'N': /* The gcc constraint letter is L, already used here.  */
+        ct->ct |= TCG_CT_CONST_NEG;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
+        break;
+
+    /* qemu_ld address */
+    case 'l':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
+#ifdef CONFIG_SOFTMMU
+        /* r0-r2,lr will be overwritten when reading the tlb entry,
+           so don't use these. */
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R14);
+#endif
+        break;
+
+    /* qemu_st address & data */
+    case 's':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, (1 << TCG_TARGET_NB_REGS) - 1);
+        /* r0-r2 will be overwritten when reading the tlb entry (softmmu only)
+           and r0-r1 doing the byte swapping, so don't use these. */
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
+#if defined(CONFIG_SOFTMMU)
+        /* Avoid clashes with registers being used for helper args */
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
+#if TARGET_LONG_BITS == 64
+        /* Avoid clashes with registers being used for helper args */
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
+#endif
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R14);
+#endif
+        break;
+
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+
+    return 0;
+}
+
+static inline uint32_t rotl(uint32_t val, int n)
+{
+  return (val << n) | (val >> (32 - n));
+}
+
+/* ARM immediates for ALU instructions are made of an unsigned 8-bit
+   right-rotated by an even amount between 0 and 30. */
+static inline int encode_imm(uint32_t imm)
+{
+    int shift;
+
+    /* simple case, only lower bits */
+    if ((imm & ~0xff) == 0)
+        return 0;
+    /* then try a simple even shift */
+    shift = ctz32(imm) & ~1;
+    if (((imm >> shift) & ~0xff) == 0)
+        return 32 - shift;
+    /* now try harder with rotations */
+    if ((rotl(imm, 2) & ~0xff) == 0)
+        return 2;
+    if ((rotl(imm, 4) & ~0xff) == 0)
+        return 4;
+    if ((rotl(imm, 6) & ~0xff) == 0)
+        return 6;
+    /* imm can't be encoded */
+    return -1;
+}
+
+static inline int check_fit_imm(uint32_t imm)
+{
+    return encode_imm(imm) >= 0;
+}
+
+/* Test if a constant matches the constraint.
+ * TODO: define constraints for:
+ *
+ * ldr/str offset:   between -0xfff and 0xfff
+ * ldrh/strh offset: between -0xff and 0xff
+ * mov operand2:     values represented with x << (2 * y), x < 0x100
+ * add, sub, eor...: ditto
+ */
+static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct;
+    ct = arg_ct->ct;
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_ARM) && check_fit_imm(val)) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_INV) && check_fit_imm(~val)) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_NEG) && check_fit_imm(-val)) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+#define TO_CPSR (1 << 20)
+
+typedef enum {
+    ARITH_AND = 0x0 << 21,
+    ARITH_EOR = 0x1 << 21,
+    ARITH_SUB = 0x2 << 21,
+    ARITH_RSB = 0x3 << 21,
+    ARITH_ADD = 0x4 << 21,
+    ARITH_ADC = 0x5 << 21,
+    ARITH_SBC = 0x6 << 21,
+    ARITH_RSC = 0x7 << 21,
+    ARITH_TST = 0x8 << 21 | TO_CPSR,
+    ARITH_CMP = 0xa << 21 | TO_CPSR,
+    ARITH_CMN = 0xb << 21 | TO_CPSR,
+    ARITH_ORR = 0xc << 21,
+    ARITH_MOV = 0xd << 21,
+    ARITH_BIC = 0xe << 21,
+    ARITH_MVN = 0xf << 21,
+
+    INSN_LDR_IMM   = 0x04100000,
+    INSN_LDR_REG   = 0x06100000,
+    INSN_STR_IMM   = 0x04000000,
+    INSN_STR_REG   = 0x06000000,
+
+    INSN_LDRH_IMM  = 0x005000b0,
+    INSN_LDRH_REG  = 0x001000b0,
+    INSN_LDRSH_IMM = 0x005000f0,
+    INSN_LDRSH_REG = 0x001000f0,
+    INSN_STRH_IMM  = 0x004000b0,
+    INSN_STRH_REG  = 0x000000b0,
+
+    INSN_LDRB_IMM  = 0x04500000,
+    INSN_LDRB_REG  = 0x06500000,
+    INSN_LDRSB_IMM = 0x005000d0,
+    INSN_LDRSB_REG = 0x001000d0,
+    INSN_STRB_IMM  = 0x04400000,
+    INSN_STRB_REG  = 0x06400000,
+
+    INSN_LDRD_IMM  = 0x004000d0,
+    INSN_LDRD_REG  = 0x000000d0,
+    INSN_STRD_IMM  = 0x004000f0,
+    INSN_STRD_REG  = 0x000000f0,
+} ARMInsn;
+
+#define SHIFT_IMM_LSL(im)	(((im) << 7) | 0x00)
+#define SHIFT_IMM_LSR(im)	(((im) << 7) | 0x20)
+#define SHIFT_IMM_ASR(im)	(((im) << 7) | 0x40)
+#define SHIFT_IMM_ROR(im)	(((im) << 7) | 0x60)
+#define SHIFT_REG_LSL(rs)	(((rs) << 8) | 0x10)
+#define SHIFT_REG_LSR(rs)	(((rs) << 8) | 0x30)
+#define SHIFT_REG_ASR(rs)	(((rs) << 8) | 0x50)
+#define SHIFT_REG_ROR(rs)	(((rs) << 8) | 0x70)
+
+enum arm_cond_code_e {
+    COND_EQ = 0x0,
+    COND_NE = 0x1,
+    COND_CS = 0x2,	/* Unsigned greater or equal */
+    COND_CC = 0x3,	/* Unsigned less than */
+    COND_MI = 0x4,	/* Negative */
+    COND_PL = 0x5,	/* Zero or greater */
+    COND_VS = 0x6,	/* Overflow */
+    COND_VC = 0x7,	/* No overflow */
+    COND_HI = 0x8,	/* Unsigned greater than */
+    COND_LS = 0x9,	/* Unsigned less or equal */
+    COND_GE = 0xa,
+    COND_LT = 0xb,
+    COND_GT = 0xc,
+    COND_LE = 0xd,
+    COND_AL = 0xe,
+};
+
+static const uint8_t tcg_cond_to_arm_cond[] = {
+    [TCG_COND_EQ] = COND_EQ,
+    [TCG_COND_NE] = COND_NE,
+    [TCG_COND_LT] = COND_LT,
+    [TCG_COND_GE] = COND_GE,
+    [TCG_COND_LE] = COND_LE,
+    [TCG_COND_GT] = COND_GT,
+    /* unsigned */
+    [TCG_COND_LTU] = COND_CC,
+    [TCG_COND_GEU] = COND_CS,
+    [TCG_COND_LEU] = COND_LS,
+    [TCG_COND_GTU] = COND_HI,
+};
+
+static inline void tcg_out_bx(TCGContext *s, int cond, int rn)
+{
+    tcg_out32(s, (cond << 28) | 0x012fff10 | rn);
+}
+
+static inline void tcg_out_b(TCGContext *s, int cond, int32_t offset)
+{
+    tcg_out32(s, (cond << 28) | 0x0a000000 |
+                    (((offset - 8) >> 2) & 0x00ffffff));
+}
+
+static inline void tcg_out_b_noaddr(TCGContext *s, int cond)
+{
+    /* We pay attention here to not modify the branch target by masking
+       the corresponding bytes.  This ensure that caches and memory are
+       kept coherent during retranslation. */
+    tcg_out32(s, deposit32(*s->code_ptr, 24, 8, (cond << 4) | 0x0a));
+}
+
+static inline void tcg_out_bl_noaddr(TCGContext *s, int cond)
+{
+    /* We pay attention here to not modify the branch target by masking
+       the corresponding bytes.  This ensure that caches and memory are
+       kept coherent during retranslation. */
+    tcg_out32(s, deposit32(*s->code_ptr, 24, 8, (cond << 4) | 0x0b));
+}
+
+static inline void tcg_out_bl(TCGContext *s, int cond, int32_t offset)
+{
+    tcg_out32(s, (cond << 28) | 0x0b000000 |
+                    (((offset - 8) >> 2) & 0x00ffffff));
+}
+
+static inline void tcg_out_blx(TCGContext *s, int cond, int rn)
+{
+    tcg_out32(s, (cond << 28) | 0x012fff30 | rn);
+}
+
+static inline void tcg_out_blx_imm(TCGContext *s, int32_t offset)
+{
+    tcg_out32(s, 0xfa000000 | ((offset & 2) << 23) |
+                (((offset - 8) >> 2) & 0x00ffffff));
+}
+
+static inline void tcg_out_dat_reg(TCGContext *s,
+                int cond, int opc, int rd, int rn, int rm, int shift)
+{
+    tcg_out32(s, (cond << 28) | (0 << 25) | opc |
+                    (rn << 16) | (rd << 12) | shift | rm);
+}
+
+static inline void tcg_out_nop(TCGContext *s)
+{
+    if (use_armv7_instructions) {
+        /* Architected nop introduced in v6k.  */
+        /* ??? This is an MSR (imm) 0,0,0 insn.  Anyone know if this
+           also Just So Happened to do nothing on pre-v6k so that we
+           don't need to conditionalize it?  */
+        tcg_out32(s, 0xe320f000);
+    } else {
+        /* Prior to that the assembler uses mov r0, r0.  */
+        tcg_out_dat_reg(s, COND_AL, ARITH_MOV, 0, 0, 0, SHIFT_IMM_LSL(0));
+    }
+}
+
+static inline void tcg_out_mov_reg(TCGContext *s, int cond, int rd, int rm)
+{
+    /* Simple reg-reg move, optimising out the 'do nothing' case */
+    if (rd != rm) {
+        tcg_out_dat_reg(s, cond, ARITH_MOV, rd, 0, rm, SHIFT_IMM_LSL(0));
+    }
+}
+
+static inline void tcg_out_dat_imm(TCGContext *s,
+                int cond, int opc, int rd, int rn, int im)
+{
+    tcg_out32(s, (cond << 28) | (1 << 25) | opc |
+                    (rn << 16) | (rd << 12) | im);
+}
+
+static void tcg_out_movi32(TCGContext *s, int cond, int rd, uint32_t arg)
+{
+    int rot, opc, rn;
+
+    /* For armv7, make sure not to use movw+movt when mov/mvn would do.
+       Speed things up by only checking when movt would be required.
+       Prior to armv7, have one go at fully rotated immediates before
+       doing the decomposition thing below.  */
+    if (!use_armv7_instructions || (arg & 0xffff0000)) {
+        rot = encode_imm(arg);
+        if (rot >= 0) {
+            tcg_out_dat_imm(s, cond, ARITH_MOV, rd, 0,
+                            rotl(arg, rot) | (rot << 7));
+            return;
+        }
+        rot = encode_imm(~arg);
+        if (rot >= 0) {
+            tcg_out_dat_imm(s, cond, ARITH_MVN, rd, 0,
+                            rotl(~arg, rot) | (rot << 7));
+            return;
+        }
+    }
+
+    /* Use movw + movt.  */
+    if (use_armv7_instructions) {
+        /* movw */
+        tcg_out32(s, (cond << 28) | 0x03000000 | (rd << 12)
+                  | ((arg << 4) & 0x000f0000) | (arg & 0xfff));
+        if (arg & 0xffff0000) {
+            /* movt */
+            tcg_out32(s, (cond << 28) | 0x03400000 | (rd << 12)
+                      | ((arg >> 12) & 0x000f0000) | ((arg >> 16) & 0xfff));
+        }
+        return;
+    }
+
+    /* TODO: This is very suboptimal, we can easily have a constant
+       pool somewhere after all the instructions.  */
+    opc = ARITH_MOV;
+    rn = 0;
+    /* If we have lots of leading 1's, we can shorten the sequence by
+       beginning with mvn and then clearing higher bits with eor.  */
+    if (clz32(~arg) > clz32(arg)) {
+        opc = ARITH_MVN, arg = ~arg;
+    }
+    do {
+        int i = ctz32(arg) & ~1;
+        rot = ((32 - i) << 7) & 0xf00;
+        tcg_out_dat_imm(s, cond, opc, rd, rn, ((arg >> i) & 0xff) | rot);
+        arg &= ~(0xff << i);
+
+        opc = ARITH_EOR;
+        rn = rd;
+    } while (arg);
+}
+
+static inline void tcg_out_dat_rI(TCGContext *s, int cond, int opc, TCGArg dst,
+                                  TCGArg lhs, TCGArg rhs, int rhs_is_const)
+{
+    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
+     * rhs must satisfy the "rI" constraint.
+     */
+    if (rhs_is_const) {
+        int rot = encode_imm(rhs);
+        tcg_debug_assert(rot >= 0);
+        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
+    } else {
+        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
+    }
+}
+
+static void tcg_out_dat_rIK(TCGContext *s, int cond, int opc, int opinv,
+                            TCGReg dst, TCGReg lhs, TCGArg rhs,
+                            bool rhs_is_const)
+{
+    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
+     * rhs must satisfy the "rIK" constraint.
+     */
+    if (rhs_is_const) {
+        int rot = encode_imm(rhs);
+        if (rot < 0) {
+            rhs = ~rhs;
+            rot = encode_imm(rhs);
+            tcg_debug_assert(rot >= 0);
+            opc = opinv;
+        }
+        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
+    } else {
+        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
+    }
+}
+
+static void tcg_out_dat_rIN(TCGContext *s, int cond, int opc, int opneg,
+                            TCGArg dst, TCGArg lhs, TCGArg rhs,
+                            bool rhs_is_const)
+{
+    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
+     * rhs must satisfy the "rIN" constraint.
+     */
+    if (rhs_is_const) {
+        int rot = encode_imm(rhs);
+        if (rot < 0) {
+            rhs = -rhs;
+            rot = encode_imm(rhs);
+            tcg_debug_assert(rot >= 0);
+            opc = opneg;
+        }
+        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
+    } else {
+        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
+    }
+}
+
+static inline void tcg_out_mul32(TCGContext *s, int cond, TCGReg rd,
+                                 TCGReg rn, TCGReg rm)
+{
+    /* if ArchVersion() < 6 && d == n then UNPREDICTABLE;  */
+    if (!use_armv6_instructions && rd == rn) {
+        if (rd == rm) {
+            /* rd == rn == rm; copy an input to tmp first.  */
+            tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn);
+            rm = rn = TCG_REG_TMP;
+        } else {
+            rn = rm;
+            rm = rd;
+        }
+    }
+    /* mul */
+    tcg_out32(s, (cond << 28) | 0x90 | (rd << 16) | (rm << 8) | rn);
+}
+
+static inline void tcg_out_umull32(TCGContext *s, int cond, TCGReg rd0,
+                                   TCGReg rd1, TCGReg rn, TCGReg rm)
+{
+    /* if ArchVersion() < 6 && (dHi == n || dLo == n) then UNPREDICTABLE;  */
+    if (!use_armv6_instructions && (rd0 == rn || rd1 == rn)) {
+        if (rd0 == rm || rd1 == rm) {
+            tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn);
+            rn = TCG_REG_TMP;
+        } else {
+            TCGReg t = rn;
+            rn = rm;
+            rm = t;
+        }
+    }
+    /* umull */
+    tcg_out32(s, (cond << 28) | 0x00800090 |
+              (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn);
+}
+
+static inline void tcg_out_smull32(TCGContext *s, int cond, TCGReg rd0,
+                                   TCGReg rd1, TCGReg rn, TCGReg rm)
+{
+    /* if ArchVersion() < 6 && (dHi == n || dLo == n) then UNPREDICTABLE;  */
+    if (!use_armv6_instructions && (rd0 == rn || rd1 == rn)) {
+        if (rd0 == rm || rd1 == rm) {
+            tcg_out_mov_reg(s, cond, TCG_REG_TMP, rn);
+            rn = TCG_REG_TMP;
+        } else {
+            TCGReg t = rn;
+            rn = rm;
+            rm = t;
+        }
+    }
+    /* smull */
+    tcg_out32(s, (cond << 28) | 0x00c00090 |
+              (rd1 << 16) | (rd0 << 12) | (rm << 8) | rn);
+}
+
+static inline void tcg_out_sdiv(TCGContext *s, int cond, int rd, int rn, int rm)
+{
+    tcg_out32(s, 0x0710f010 | (cond << 28) | (rd << 16) | rn | (rm << 8));
+}
+
+static inline void tcg_out_udiv(TCGContext *s, int cond, int rd, int rn, int rm)
+{
+    tcg_out32(s, 0x0730f010 | (cond << 28) | (rd << 16) | rn | (rm << 8));
+}
+
+static inline void tcg_out_ext8s(TCGContext *s, int cond,
+                                 int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* sxtb */
+        tcg_out32(s, 0x06af0070 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rn, SHIFT_IMM_LSL(24));
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rd, SHIFT_IMM_ASR(24));
+    }
+}
+
+static inline void tcg_out_ext8u(TCGContext *s, int cond,
+                                 int rd, int rn)
+{
+    tcg_out_dat_imm(s, cond, ARITH_AND, rd, rn, 0xff);
+}
+
+static inline void tcg_out_ext16s(TCGContext *s, int cond,
+                                  int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* sxth */
+        tcg_out32(s, 0x06bf0070 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rn, SHIFT_IMM_LSL(16));
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rd, SHIFT_IMM_ASR(16));
+    }
+}
+
+static inline void tcg_out_ext16u(TCGContext *s, int cond,
+                                  int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* uxth */
+        tcg_out32(s, 0x06ff0070 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rn, SHIFT_IMM_LSL(16));
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rd, SHIFT_IMM_LSR(16));
+    }
+}
+
+static inline void tcg_out_bswap16s(TCGContext *s, int cond, int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* revsh */
+        tcg_out32(s, 0x06ff0fb0 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_TMP, 0, rn, SHIFT_IMM_LSL(24));
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_TMP, 0, TCG_REG_TMP, SHIFT_IMM_ASR(16));
+        tcg_out_dat_reg(s, cond, ARITH_ORR,
+                        rd, TCG_REG_TMP, rn, SHIFT_IMM_LSR(8));
+    }
+}
+
+static inline void tcg_out_bswap16(TCGContext *s, int cond, int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* rev16 */
+        tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_TMP, 0, rn, SHIFT_IMM_LSL(24));
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_TMP, 0, TCG_REG_TMP, SHIFT_IMM_LSR(16));
+        tcg_out_dat_reg(s, cond, ARITH_ORR,
+                        rd, TCG_REG_TMP, rn, SHIFT_IMM_LSR(8));
+    }
+}
+
+/* swap the two low bytes assuming that the two high input bytes and the
+   two high output bit can hold any value. */
+static inline void tcg_out_bswap16st(TCGContext *s, int cond, int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* rev16 */
+        tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_TMP, 0, rn, SHIFT_IMM_LSR(8));
+        tcg_out_dat_imm(s, cond, ARITH_AND, TCG_REG_TMP, TCG_REG_TMP, 0xff);
+        tcg_out_dat_reg(s, cond, ARITH_ORR,
+                        rd, TCG_REG_TMP, rn, SHIFT_IMM_LSL(8));
+    }
+}
+
+static inline void tcg_out_bswap32(TCGContext *s, int cond, int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* rev */
+        tcg_out32(s, 0x06bf0f30 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_EOR,
+                        TCG_REG_TMP, rn, rn, SHIFT_IMM_ROR(16));
+        tcg_out_dat_imm(s, cond, ARITH_BIC,
+                        TCG_REG_TMP, TCG_REG_TMP, 0xff | 0x800);
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        rd, 0, rn, SHIFT_IMM_ROR(8));
+        tcg_out_dat_reg(s, cond, ARITH_EOR,
+                        rd, rd, TCG_REG_TMP, SHIFT_IMM_LSR(8));
+    }
+}
+
+bool tcg_target_deposit_valid(int ofs, int len)
+{
+    /* ??? Without bfi, we could improve over generic code by combining
+       the right-shift from a non-zero ofs with the orr.  We do run into
+       problems when rd == rs, and the mask generated from ofs+len doesn't
+       fit into an immediate.  We would have to be careful not to pessimize
+       wrt the optimizations performed on the expanded code.  */
+    return use_armv7_instructions;
+}
+
+static inline void tcg_out_deposit(TCGContext *s, int cond, TCGReg rd,
+                                   TCGArg a1, int ofs, int len, bool const_a1)
+{
+    if (const_a1) {
+        /* bfi becomes bfc with rn == 15.  */
+        a1 = 15;
+    }
+    /* bfi/bfc */
+    tcg_out32(s, 0x07c00010 | (cond << 28) | (rd << 12) | a1
+              | (ofs << 7) | ((ofs + len - 1) << 16));
+}
+
+/* Note that this routine is used for both LDR and LDRH formats, so we do
+   not wish to include an immediate shift at this point.  */
+static void tcg_out_memop_r(TCGContext *s, int cond, ARMInsn opc, TCGReg rt,
+                            TCGReg rn, TCGReg rm, bool u, bool p, bool w)
+{
+    tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24)
+              | (w << 21) | (rn << 16) | (rt << 12) | rm);
+}
+
+static void tcg_out_memop_8(TCGContext *s, int cond, ARMInsn opc, TCGReg rt,
+                            TCGReg rn, int imm8, bool p, bool w)
+{
+    bool u = 1;
+    if (imm8 < 0) {
+        imm8 = -imm8;
+        u = 0;
+    }
+    tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) | (w << 21) |
+              (rn << 16) | (rt << 12) | ((imm8 & 0xf0) << 4) | (imm8 & 0xf));
+}
+
+static void tcg_out_memop_12(TCGContext *s, int cond, ARMInsn opc, TCGReg rt,
+                             TCGReg rn, int imm12, bool p, bool w)
+{
+    bool u = 1;
+    if (imm12 < 0) {
+        imm12 = -imm12;
+        u = 0;
+    }
+    tcg_out32(s, (cond << 28) | opc | (u << 23) | (p << 24) | (w << 21) |
+              (rn << 16) | (rt << 12) | imm12);
+}
+
+static inline void tcg_out_ld32_12(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm12)
+{
+    tcg_out_memop_12(s, cond, INSN_LDR_IMM, rt, rn, imm12, 1, 0);
+}
+
+static inline void tcg_out_st32_12(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm12)
+{
+    tcg_out_memop_12(s, cond, INSN_STR_IMM, rt, rn, imm12, 1, 0);
+}
+
+static inline void tcg_out_ld32_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDR_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_st32_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_ldrd_8(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_LDRD_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_ldrd_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDRD_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_strd_8(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_STRD_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_strd_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_STRD_REG, rt, rn, rm, 1, 1, 0);
+}
+
+/* Register pre-increment with base writeback.  */
+static inline void tcg_out_ld32_rwb(TCGContext *s, int cond, TCGReg rt,
+                                    TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDR_REG, rt, rn, rm, 1, 1, 1);
+}
+
+static inline void tcg_out_st32_rwb(TCGContext *s, int cond, TCGReg rt,
+                                    TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_STR_REG, rt, rn, rm, 1, 1, 1);
+}
+
+static inline void tcg_out_ld16u_8(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_LDRH_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_st16_8(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_STRH_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_ld16u_r(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDRH_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_st16_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_STRH_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_ld16s_8(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_LDRSH_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_ld16s_r(TCGContext *s, int cond, TCGReg rt,
+                                   TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDRSH_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_ld8_12(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, int imm12)
+{
+    tcg_out_memop_12(s, cond, INSN_LDRB_IMM, rt, rn, imm12, 1, 0);
+}
+
+static inline void tcg_out_st8_12(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, int imm12)
+{
+    tcg_out_memop_12(s, cond, INSN_STRB_IMM, rt, rn, imm12, 1, 0);
+}
+
+static inline void tcg_out_ld8_r(TCGContext *s, int cond, TCGReg rt,
+                                 TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDRB_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_st8_r(TCGContext *s, int cond, TCGReg rt,
+                                 TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_STRB_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_ld8s_8(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, int imm8)
+{
+    tcg_out_memop_8(s, cond, INSN_LDRSB_IMM, rt, rn, imm8, 1, 0);
+}
+
+static inline void tcg_out_ld8s_r(TCGContext *s, int cond, TCGReg rt,
+                                  TCGReg rn, TCGReg rm)
+{
+    tcg_out_memop_r(s, cond, INSN_LDRSB_REG, rt, rn, rm, 1, 1, 0);
+}
+
+static inline void tcg_out_ld32u(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xfff || offset < -0xfff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_ld32_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_ld32_12(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_st32(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xfff || offset < -0xfff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_st32_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_st32_12(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_ld16u(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xff || offset < -0xff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_ld16u_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_ld16u_8(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_ld16s(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xff || offset < -0xff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_ld16s_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_ld16s_8(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_st16(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xff || offset < -0xff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_st16_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_st16_8(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_ld8u(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xfff || offset < -0xfff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_ld8_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_ld8_12(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_ld8s(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xff || offset < -0xff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_ld8s_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_ld8s_8(s, cond, rd, rn, offset);
+}
+
+static inline void tcg_out_st8(TCGContext *s, int cond,
+                int rd, int rn, int32_t offset)
+{
+    if (offset > 0xfff || offset < -0xfff) {
+        tcg_out_movi32(s, cond, TCG_REG_TMP, offset);
+        tcg_out_st8_r(s, cond, rd, rn, TCG_REG_TMP);
+    } else
+        tcg_out_st8_12(s, cond, rd, rn, offset);
+}
+
+/* The _goto case is normally between TBs within the same code buffer, and
+ * with the code buffer limited to 16MB we wouldn't need the long case.
+ * But we also use it for the tail-call to the qemu_ld/st helpers, which does.
+ */
+static inline void tcg_out_goto(TCGContext *s, int cond, tcg_insn_unit *addr)
+{
+    intptr_t addri = (intptr_t)addr;
+    ptrdiff_t disp = tcg_pcrel_diff(s, addr);
+
+    if ((addri & 1) == 0 && disp - 8 < 0x01fffffd && disp - 8 > -0x01fffffd) {
+        tcg_out_b(s, cond, disp);
+        return;
+    }
+
+    tcg_out_movi32(s, cond, TCG_REG_TMP, addri);
+    if (use_armv5t_instructions) {
+        tcg_out_bx(s, cond, TCG_REG_TMP);
+    } else {
+        if (addri & 1) {
+            tcg_abort();
+        }
+        tcg_out_mov_reg(s, cond, TCG_REG_PC, TCG_REG_TMP);
+    }
+}
+
+/* The call case is mostly used for helpers - so it's not unreasonable
+ * for them to be beyond branch range */
+static void tcg_out_call(TCGContext *s, tcg_insn_unit *addr)
+{
+    intptr_t addri = (intptr_t)addr;
+    ptrdiff_t disp = tcg_pcrel_diff(s, addr);
+
+    if (disp - 8 < 0x02000000 && disp - 8 >= -0x02000000) {
+        if (addri & 1) {
+            /* Use BLX if the target is in Thumb mode */
+            if (!use_armv5t_instructions) {
+                tcg_abort();
+            }
+            tcg_out_blx_imm(s, disp);
+        } else {
+            tcg_out_bl(s, COND_AL, disp);
+        }
+    } else if (use_armv7_instructions) {
+        tcg_out_movi32(s, COND_AL, TCG_REG_TMP, addri);
+        tcg_out_blx(s, COND_AL, TCG_REG_TMP);
+    } else {
+        tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R14, TCG_REG_PC, 4);
+        tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_PC, -4);
+        tcg_out32(s, addri);
+    }
+}
+
+void arm_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
+{
+    tcg_insn_unit *code_ptr = (tcg_insn_unit *)jmp_addr;
+    tcg_insn_unit *target = (tcg_insn_unit *)addr;
+
+    /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */
+    reloc_pc24_atomic(code_ptr, target);
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
+
+static inline void tcg_out_goto_label(TCGContext *s, int cond, TCGLabel *l)
+{
+    if (l->has_value) {
+        tcg_out_goto(s, cond, l->u.value_ptr);
+    } else {
+        tcg_out_reloc(s, s->code_ptr, R_ARM_PC24, l, 0);
+        tcg_out_b_noaddr(s, cond);
+    }
+}
+
+#ifdef CONFIG_SOFTMMU
+/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
+ *                                     int mmu_idx, uintptr_t ra)
+ */
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_SB]   = helper_ret_ldsb_mmu,
+
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_LESW] = helper_le_ldsw_mmu,
+    [MO_LESL] = helper_le_ldul_mmu,
+
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+    [MO_BESW] = helper_be_ldsw_mmu,
+    [MO_BESL] = helper_be_ldul_mmu,
+};
+
+/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
+ *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
+};
+
+/* Helper routines for marshalling helper function arguments into
+ * the correct registers and stack.
+ * argreg is where we want to put this argument, arg is the argument itself.
+ * Return value is the updated argreg ready for the next call.
+ * Note that argreg 0..3 is real registers, 4+ on stack.
+ *
+ * We provide routines for arguments which are: immediate, 32 bit
+ * value in register, 16 and 8 bit values in register (which must be zero
+ * extended before use) and 64 bit value in a lo:hi register pair.
+ */
+#define DEFINE_TCG_OUT_ARG(NAME, ARGTYPE, MOV_ARG, EXT_ARG)                \
+static TCGReg NAME(TCGContext *s, TCGReg argreg, ARGTYPE arg)              \
+{                                                                          \
+    if (argreg < 4) {                                                      \
+        MOV_ARG(s, COND_AL, argreg, arg);                                  \
+    } else {                                                               \
+        int ofs = (argreg - 4) * 4;                                        \
+        EXT_ARG;                                                           \
+        tcg_debug_assert(ofs + 4 <= TCG_STATIC_CALL_ARGS_SIZE);            \
+        tcg_out_st32_12(s, COND_AL, arg, TCG_REG_CALL_STACK, ofs);         \
+    }                                                                      \
+    return argreg + 1;                                                     \
+}
+
+DEFINE_TCG_OUT_ARG(tcg_out_arg_imm32, uint32_t, tcg_out_movi32,
+    (tcg_out_movi32(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
+DEFINE_TCG_OUT_ARG(tcg_out_arg_reg8, TCGReg, tcg_out_ext8u,
+    (tcg_out_ext8u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
+DEFINE_TCG_OUT_ARG(tcg_out_arg_reg16, TCGReg, tcg_out_ext16u,
+    (tcg_out_ext16u(s, COND_AL, TCG_REG_TMP, arg), arg = TCG_REG_TMP))
+DEFINE_TCG_OUT_ARG(tcg_out_arg_reg32, TCGReg, tcg_out_mov_reg, )
+
+static TCGReg tcg_out_arg_reg64(TCGContext *s, TCGReg argreg,
+                                TCGReg arglo, TCGReg arghi)
+{
+    /* 64 bit arguments must go in even/odd register pairs
+     * and in 8-aligned stack slots.
+     */
+    if (argreg & 1) {
+        argreg++;
+    }
+    if (use_armv6_instructions && argreg >= 4
+        && (arglo & 1) == 0 && arghi == arglo + 1) {
+        tcg_out_strd_8(s, COND_AL, arglo,
+                       TCG_REG_CALL_STACK, (argreg - 4) * 4);
+        return argreg + 2;
+    } else {
+        argreg = tcg_out_arg_reg32(s, argreg, arglo);
+        argreg = tcg_out_arg_reg32(s, argreg, arghi);
+        return argreg;
+    }
+}
+
+#define TLB_SHIFT	(CPU_TLB_ENTRY_BITS + CPU_TLB_BITS)
+
+/* We're expecting to use an 8-bit immediate and to mask.  */
+QEMU_BUILD_BUG_ON(CPU_TLB_BITS > 8);
+
+/* We're expecting to use an 8-bit immediate add + 8-bit ldrd offset.
+   Using the offset of the second entry in the last tlb table ensures
+   that we can index all of the elements of the first entry.  */
+QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
+                  > 0xffff);
+
+/* Load and compare a TLB entry, leaving the flags set.  Returns the register
+   containing the addend of the tlb entry.  Clobbers R0, R1, R2, TMP.  */
+
+static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
+                               TCGMemOp s_bits, int mem_index, bool is_load)
+{
+    TCGReg base = TCG_AREG0;
+    int cmp_off =
+        (is_load
+         ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+         : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
+
+    /* Should generate something like the following:
+     *   shr    tmp, addrlo, #TARGET_PAGE_BITS                    (1)
+     *   add    r2, env, #high
+     *   and    r0, tmp, #(CPU_TLB_SIZE - 1)                      (2)
+     *   add    r2, r2, r0, lsl #CPU_TLB_ENTRY_BITS               (3)
+     *   ldr    r0, [r2, #cmp]                                    (4)
+     *   tst    addrlo, #s_mask
+     *   ldr    r2, [r2, #add]                                    (5)
+     *   cmpeq  r0, tmp, lsl #TARGET_PAGE_BITS
+     */
+    tcg_out_dat_reg(s, COND_AL, ARITH_MOV, TCG_REG_TMP,
+                    0, addrlo, SHIFT_IMM_LSR(TARGET_PAGE_BITS));
+
+    /* We checked that the offset is contained within 16 bits above.  */
+    if (add_off > 0xfff || (use_armv6_instructions && cmp_off > 0xff)) {
+        tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R2, base,
+                        (24 << 7) | (cmp_off >> 8));
+        base = TCG_REG_R2;
+        add_off -= cmp_off & 0xff00;
+        cmp_off &= 0xff;
+    }
+
+    tcg_out_dat_imm(s, COND_AL, ARITH_AND,
+                    TCG_REG_R0, TCG_REG_TMP, CPU_TLB_SIZE - 1);
+    tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R2, base,
+                    TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
+
+    /* Load the tlb comparator.  Use ldrd if needed and available,
+       but due to how the pointer needs setting up, ldm isn't useful.
+       Base arm5 doesn't have ldrd, but armv5te does.  */
+    if (use_armv6_instructions && TARGET_LONG_BITS == 64) {
+        tcg_out_ldrd_8(s, COND_AL, TCG_REG_R0, TCG_REG_R2, cmp_off);
+    } else {
+        tcg_out_ld32_12(s, COND_AL, TCG_REG_R0, TCG_REG_R2, cmp_off);
+        if (TARGET_LONG_BITS == 64) {
+            tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R2, cmp_off + 4);
+        }
+    }
+
+    /* Check alignment.  */
+    if (s_bits) {
+        tcg_out_dat_imm(s, COND_AL, ARITH_TST,
+                        0, addrlo, (1 << s_bits) - 1);
+    }
+
+    /* Load the tlb addend.  */
+    tcg_out_ld32_12(s, COND_AL, TCG_REG_R2, TCG_REG_R2, add_off);
+
+    tcg_out_dat_reg(s, (s_bits ? COND_EQ : COND_AL), ARITH_CMP, 0,
+                    TCG_REG_R0, TCG_REG_TMP, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
+
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
+                        TCG_REG_R1, addrhi, SHIFT_IMM_LSL(0));
+    }
+
+    return TCG_REG_R2;
+}
+
+/* Record the context of a call to the out of line helper code for the slow
+   path for a load or store, so that we can later generate the correct
+   helper code.  */
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
+                                TCGReg datalo, TCGReg datahi, TCGReg addrlo,
+                                TCGReg addrhi, tcg_insn_unit *raddr,
+                                tcg_insn_unit *label_ptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->oi = oi;
+    label->datalo_reg = datalo;
+    label->datahi_reg = datahi;
+    label->addrlo_reg = addrlo;
+    label->addrhi_reg = addrhi;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr;
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGReg argreg, datalo, datahi;
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+    void *func;
+
+    reloc_pc24(lb->label_ptr[0], s->code_ptr);
+
+    argreg = tcg_out_arg_reg32(s, TCG_REG_R0, TCG_AREG0);
+    if (TARGET_LONG_BITS == 64) {
+        argreg = tcg_out_arg_reg64(s, argreg, lb->addrlo_reg, lb->addrhi_reg);
+    } else {
+        argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
+    }
+    argreg = tcg_out_arg_imm32(s, argreg, oi);
+    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);
+
+    /* For armv6 we can use the canonical unsigned helpers and minimize
+       icache usage.  For pre-armv6, use the signed helpers since we do
+       not have a single insn sign-extend.  */
+    if (use_armv6_instructions) {
+        func = qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)];
+    } else {
+        func = qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)];
+        if (opc & MO_SIGN) {
+            opc = MO_UL;
+        }
+    }
+    tcg_out_call(s, func);
+
+    datalo = lb->datalo_reg;
+    datahi = lb->datahi_reg;
+    switch (opc & MO_SSIZE) {
+    case MO_SB:
+        tcg_out_ext8s(s, COND_AL, datalo, TCG_REG_R0);
+        break;
+    case MO_SW:
+        tcg_out_ext16s(s, COND_AL, datalo, TCG_REG_R0);
+        break;
+    default:
+        tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
+        break;
+    case MO_Q:
+        if (datalo != TCG_REG_R1) {
+            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
+            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
+        } else if (datahi != TCG_REG_R0) {
+            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
+            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_R0);
+        } else {
+            tcg_out_mov_reg(s, COND_AL, TCG_REG_TMP, TCG_REG_R0);
+            tcg_out_mov_reg(s, COND_AL, datahi, TCG_REG_R1);
+            tcg_out_mov_reg(s, COND_AL, datalo, TCG_REG_TMP);
+        }
+        break;
+    }
+
+    tcg_out_goto(s, COND_AL, lb->raddr);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGReg argreg, datalo, datahi;
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+
+    reloc_pc24(lb->label_ptr[0], s->code_ptr);
+
+    argreg = TCG_REG_R0;
+    argreg = tcg_out_arg_reg32(s, argreg, TCG_AREG0);
+    if (TARGET_LONG_BITS == 64) {
+        argreg = tcg_out_arg_reg64(s, argreg, lb->addrlo_reg, lb->addrhi_reg);
+    } else {
+        argreg = tcg_out_arg_reg32(s, argreg, lb->addrlo_reg);
+    }
+
+    datalo = lb->datalo_reg;
+    datahi = lb->datahi_reg;
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        argreg = tcg_out_arg_reg8(s, argreg, datalo);
+        break;
+    case MO_16:
+        argreg = tcg_out_arg_reg16(s, argreg, datalo);
+        break;
+    case MO_32:
+    default:
+        argreg = tcg_out_arg_reg32(s, argreg, datalo);
+        break;
+    case MO_64:
+        argreg = tcg_out_arg_reg64(s, argreg, datalo, datahi);
+        break;
+    }
+
+    argreg = tcg_out_arg_imm32(s, argreg, oi);
+    argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14);
+
+    /* Tail-call to the helper, which will return to the fast path.  */
+    tcg_out_goto(s, COND_AL, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+}
+#endif /* SOFTMMU */
+
+static inline void tcg_out_qemu_ld_index(TCGContext *s, TCGMemOp opc,
+                                         TCGReg datalo, TCGReg datahi,
+                                         TCGReg addrlo, TCGReg addend)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
+
+    switch (opc & MO_SSIZE) {
+    case MO_UB:
+        tcg_out_ld8_r(s, COND_AL, datalo, addrlo, addend);
+        break;
+    case MO_SB:
+        tcg_out_ld8s_r(s, COND_AL, datalo, addrlo, addend);
+        break;
+    case MO_UW:
+        tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend);
+        if (bswap) {
+            tcg_out_bswap16(s, COND_AL, datalo, datalo);
+        }
+        break;
+    case MO_SW:
+        if (bswap) {
+            tcg_out_ld16u_r(s, COND_AL, datalo, addrlo, addend);
+            tcg_out_bswap16s(s, COND_AL, datalo, datalo);
+        } else {
+            tcg_out_ld16s_r(s, COND_AL, datalo, addrlo, addend);
+        }
+        break;
+    case MO_UL:
+    default:
+        tcg_out_ld32_r(s, COND_AL, datalo, addrlo, addend);
+        if (bswap) {
+            tcg_out_bswap32(s, COND_AL, datalo, datalo);
+        }
+        break;
+    case MO_Q:
+        {
+            TCGReg dl = (bswap ? datahi : datalo);
+            TCGReg dh = (bswap ? datalo : datahi);
+
+            /* Avoid ldrd for user-only emulation, to handle unaligned.  */
+            if (USING_SOFTMMU && use_armv6_instructions
+                && (dl & 1) == 0 && dh == dl + 1) {
+                tcg_out_ldrd_r(s, COND_AL, dl, addrlo, addend);
+            } else if (dl != addend) {
+                tcg_out_ld32_rwb(s, COND_AL, dl, addend, addrlo);
+                tcg_out_ld32_12(s, COND_AL, dh, addend, 4);
+            } else {
+                tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_TMP,
+                                addend, addrlo, SHIFT_IMM_LSL(0));
+                tcg_out_ld32_12(s, COND_AL, dl, TCG_REG_TMP, 0);
+                tcg_out_ld32_12(s, COND_AL, dh, TCG_REG_TMP, 4);
+            }
+            if (bswap) {
+                tcg_out_bswap32(s, COND_AL, dl, dl);
+                tcg_out_bswap32(s, COND_AL, dh, dh);
+            }
+        }
+        break;
+    }
+}
+
+static inline void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp opc,
+                                          TCGReg datalo, TCGReg datahi,
+                                          TCGReg addrlo)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
+
+    switch (opc & MO_SSIZE) {
+    case MO_UB:
+        tcg_out_ld8_12(s, COND_AL, datalo, addrlo, 0);
+        break;
+    case MO_SB:
+        tcg_out_ld8s_8(s, COND_AL, datalo, addrlo, 0);
+        break;
+    case MO_UW:
+        tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0);
+        if (bswap) {
+            tcg_out_bswap16(s, COND_AL, datalo, datalo);
+        }
+        break;
+    case MO_SW:
+        if (bswap) {
+            tcg_out_ld16u_8(s, COND_AL, datalo, addrlo, 0);
+            tcg_out_bswap16s(s, COND_AL, datalo, datalo);
+        } else {
+            tcg_out_ld16s_8(s, COND_AL, datalo, addrlo, 0);
+        }
+        break;
+    case MO_UL:
+    default:
+        tcg_out_ld32_12(s, COND_AL, datalo, addrlo, 0);
+        if (bswap) {
+            tcg_out_bswap32(s, COND_AL, datalo, datalo);
+        }
+        break;
+    case MO_Q:
+        {
+            TCGReg dl = (bswap ? datahi : datalo);
+            TCGReg dh = (bswap ? datalo : datahi);
+
+            /* Avoid ldrd for user-only emulation, to handle unaligned.  */
+            if (USING_SOFTMMU && use_armv6_instructions
+                && (dl & 1) == 0 && dh == dl + 1) {
+                tcg_out_ldrd_8(s, COND_AL, dl, addrlo, 0);
+            } else if (dl == addrlo) {
+                tcg_out_ld32_12(s, COND_AL, dh, addrlo, bswap ? 0 : 4);
+                tcg_out_ld32_12(s, COND_AL, dl, addrlo, bswap ? 4 : 0);
+            } else {
+                tcg_out_ld32_12(s, COND_AL, dl, addrlo, bswap ? 4 : 0);
+                tcg_out_ld32_12(s, COND_AL, dh, addrlo, bswap ? 0 : 4);
+            }
+            if (bswap) {
+                tcg_out_bswap32(s, COND_AL, dl, dl);
+                tcg_out_bswap32(s, COND_AL, dh, dh);
+            }
+        }
+        break;
+    }
+}
+
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
+{
+    TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused));
+    TCGMemOpIdx oi;
+    TCGMemOp opc;
+#ifdef CONFIG_SOFTMMU
+    int mem_index;
+    TCGReg addend;
+    tcg_insn_unit *label_ptr;
+#endif
+
+    datalo = *args++;
+    datahi = (is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+
+#ifdef CONFIG_SOFTMMU
+    mem_index = get_mmuidx(oi);
+    addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 1);
+
+    /* This a conditional BL only to load a pointer within this opcode into LR
+       for the slow path.  We will not be using the value for a tail call.  */
+    label_ptr = s->code_ptr;
+    tcg_out_bl_noaddr(s, COND_NE);
+
+    tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, addend);
+
+    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
+                        s->code_ptr, label_ptr);
+#else /* !CONFIG_SOFTMMU */
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base);
+        tcg_out_qemu_ld_index(s, opc, datalo, datahi, addrlo, TCG_REG_TMP);
+    } else {
+        tcg_out_qemu_ld_direct(s, opc, datalo, datahi, addrlo);
+    }
+#endif
+}
+
+static inline void tcg_out_qemu_st_index(TCGContext *s, int cond, TCGMemOp opc,
+                                         TCGReg datalo, TCGReg datahi,
+                                         TCGReg addrlo, TCGReg addend)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
+
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        tcg_out_st8_r(s, cond, datalo, addrlo, addend);
+        break;
+    case MO_16:
+        if (bswap) {
+            tcg_out_bswap16st(s, cond, TCG_REG_R0, datalo);
+            tcg_out_st16_r(s, cond, TCG_REG_R0, addrlo, addend);
+        } else {
+            tcg_out_st16_r(s, cond, datalo, addrlo, addend);
+        }
+        break;
+    case MO_32:
+    default:
+        if (bswap) {
+            tcg_out_bswap32(s, cond, TCG_REG_R0, datalo);
+            tcg_out_st32_r(s, cond, TCG_REG_R0, addrlo, addend);
+        } else {
+            tcg_out_st32_r(s, cond, datalo, addrlo, addend);
+        }
+        break;
+    case MO_64:
+        /* Avoid strd for user-only emulation, to handle unaligned.  */
+        if (bswap) {
+            tcg_out_bswap32(s, cond, TCG_REG_R0, datahi);
+            tcg_out_st32_rwb(s, cond, TCG_REG_R0, addend, addrlo);
+            tcg_out_bswap32(s, cond, TCG_REG_R0, datalo);
+            tcg_out_st32_12(s, cond, TCG_REG_R0, addend, 4);
+        } else if (USING_SOFTMMU && use_armv6_instructions
+                   && (datalo & 1) == 0 && datahi == datalo + 1) {
+            tcg_out_strd_r(s, cond, datalo, addrlo, addend);
+        } else {
+            tcg_out_st32_rwb(s, cond, datalo, addend, addrlo);
+            tcg_out_st32_12(s, cond, datahi, addend, 4);
+        }
+        break;
+    }
+}
+
+static inline void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp opc,
+                                          TCGReg datalo, TCGReg datahi,
+                                          TCGReg addrlo)
+{
+    TCGMemOp bswap = opc & MO_BSWAP;
+
+    switch (opc & MO_SIZE) {
+    case MO_8:
+        tcg_out_st8_12(s, COND_AL, datalo, addrlo, 0);
+        break;
+    case MO_16:
+        if (bswap) {
+            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, datalo);
+            tcg_out_st16_8(s, COND_AL, TCG_REG_R0, addrlo, 0);
+        } else {
+            tcg_out_st16_8(s, COND_AL, datalo, addrlo, 0);
+        }
+        break;
+    case MO_32:
+    default:
+        if (bswap) {
+            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datalo);
+            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 0);
+        } else {
+            tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0);
+        }
+        break;
+    case MO_64:
+        /* Avoid strd for user-only emulation, to handle unaligned.  */
+        if (bswap) {
+            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datahi);
+            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 0);
+            tcg_out_bswap32(s, COND_AL, TCG_REG_R0, datalo);
+            tcg_out_st32_12(s, COND_AL, TCG_REG_R0, addrlo, 4);
+        } else if (USING_SOFTMMU && use_armv6_instructions
+                   && (datalo & 1) == 0 && datahi == datalo + 1) {
+            tcg_out_strd_8(s, COND_AL, datalo, addrlo, 0);
+        } else {
+            tcg_out_st32_12(s, COND_AL, datalo, addrlo, 0);
+            tcg_out_st32_12(s, COND_AL, datahi, addrlo, 4);
+        }
+        break;
+    }
+}
+
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
+{
+    TCGReg addrlo, datalo, datahi, addrhi __attribute__((unused));
+    TCGMemOpIdx oi;
+    TCGMemOp opc;
+#ifdef CONFIG_SOFTMMU
+    int mem_index;
+    TCGReg addend;
+    tcg_insn_unit *label_ptr;
+#endif
+
+    datalo = *args++;
+    datahi = (is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+
+#ifdef CONFIG_SOFTMMU
+    mem_index = get_mmuidx(oi);
+    addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 0);
+
+    tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, addrlo, addend);
+
+    /* The conditional call must come last, as we're going to return here.  */
+    label_ptr = s->code_ptr;
+    tcg_out_bl_noaddr(s, COND_NE);
+
+    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
+                        s->code_ptr, label_ptr);
+#else /* !CONFIG_SOFTMMU */
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP, guest_base);
+        tcg_out_qemu_st_index(s, COND_AL, opc, datalo,
+                              datahi, addrlo, TCG_REG_TMP);
+    } else {
+        tcg_out_qemu_st_direct(s, opc, datalo, datahi, addrlo);
+    }
+#endif
+}
+
+static tcg_insn_unit *tb_ret_addr;
+
+static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                const TCGArg *args, const int *const_args)
+{
+    TCGArg a0, a1, a2, a3, a4, a5;
+    int c;
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        tcg_out_movi32(s, COND_AL, TCG_REG_R0, args[0]);
+        tcg_out_goto(s, COND_AL, tb_ret_addr);
+        break;
+    case INDEX_op_goto_tb:
+        if (s->tb_jmp_insn_offset) {
+            /* Direct jump method */
+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            tcg_out_b_noaddr(s, COND_AL);
+        } else {
+            /* Indirect jump method */
+            intptr_t ptr = (intptr_t)(s->tb_jmp_target_addr + args[0]);
+            tcg_out_movi32(s, COND_AL, TCG_REG_R0, ptr & ~0xfff);
+            tcg_out_ld32_12(s, COND_AL, TCG_REG_PC, TCG_REG_R0, ptr & 0xfff);
+        }
+        s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+        break;
+    case INDEX_op_br:
+        tcg_out_goto_label(s, COND_AL, arg_label(args[0]));
+        break;
+
+    case INDEX_op_ld8u_i32:
+        tcg_out_ld8u(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld8s_i32:
+        tcg_out_ld8s(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld16u_i32:
+        tcg_out_ld16u(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld16s_i32:
+        tcg_out_ld16s(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld_i32:
+        tcg_out_ld32u(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st8_i32:
+        tcg_out_st8(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st16_i32:
+        tcg_out_st16(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i32:
+        tcg_out_st32(s, COND_AL, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_movcond_i32:
+        /* Constraints mean that v2 is always in the same register as dest,
+         * so we only need to do "if condition passed, move v1 to dest".
+         */
+        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
+                        args[1], args[2], const_args[2]);
+        tcg_out_dat_rIK(s, tcg_cond_to_arm_cond[args[5]], ARITH_MOV,
+                        ARITH_MVN, args[0], 0, args[3], const_args[3]);
+        break;
+    case INDEX_op_add_i32:
+        tcg_out_dat_rIN(s, COND_AL, ARITH_ADD, ARITH_SUB,
+                        args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_sub_i32:
+        if (const_args[1]) {
+            if (const_args[2]) {
+                tcg_out_movi32(s, COND_AL, args[0], args[1] - args[2]);
+            } else {
+                tcg_out_dat_rI(s, COND_AL, ARITH_RSB,
+                               args[0], args[2], args[1], 1);
+            }
+        } else {
+            tcg_out_dat_rIN(s, COND_AL, ARITH_SUB, ARITH_ADD,
+                            args[0], args[1], args[2], const_args[2]);
+        }
+        break;
+    case INDEX_op_and_i32:
+        tcg_out_dat_rIK(s, COND_AL, ARITH_AND, ARITH_BIC,
+                        args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_andc_i32:
+        tcg_out_dat_rIK(s, COND_AL, ARITH_BIC, ARITH_AND,
+                        args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_or_i32:
+        c = ARITH_ORR;
+        goto gen_arith;
+    case INDEX_op_xor_i32:
+        c = ARITH_EOR;
+        /* Fall through.  */
+    gen_arith:
+        tcg_out_dat_rI(s, COND_AL, c, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_add2_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        a3 = args[3], a4 = args[4], a5 = args[5];
+        if (a0 == a3 || (a0 == a5 && !const_args[5])) {
+            a0 = TCG_REG_TMP;
+        }
+        tcg_out_dat_rIN(s, COND_AL, ARITH_ADD | TO_CPSR, ARITH_SUB | TO_CPSR,
+                        a0, a2, a4, const_args[4]);
+        tcg_out_dat_rIK(s, COND_AL, ARITH_ADC, ARITH_SBC,
+                        a1, a3, a5, const_args[5]);
+        tcg_out_mov_reg(s, COND_AL, args[0], a0);
+        break;
+    case INDEX_op_sub2_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        a3 = args[3], a4 = args[4], a5 = args[5];
+        if ((a0 == a3 && !const_args[3]) || (a0 == a5 && !const_args[5])) {
+            a0 = TCG_REG_TMP;
+        }
+        if (const_args[2]) {
+            if (const_args[4]) {
+                tcg_out_movi32(s, COND_AL, a0, a4);
+                a4 = a0;
+            }
+            tcg_out_dat_rI(s, COND_AL, ARITH_RSB | TO_CPSR, a0, a4, a2, 1);
+        } else {
+            tcg_out_dat_rIN(s, COND_AL, ARITH_SUB | TO_CPSR,
+                            ARITH_ADD | TO_CPSR, a0, a2, a4, const_args[4]);
+        }
+        if (const_args[3]) {
+            if (const_args[5]) {
+                tcg_out_movi32(s, COND_AL, a1, a5);
+                a5 = a1;
+            }
+            tcg_out_dat_rI(s, COND_AL, ARITH_RSC, a1, a5, a3, 1);
+        } else {
+            tcg_out_dat_rIK(s, COND_AL, ARITH_SBC, ARITH_ADC,
+                            a1, a3, a5, const_args[5]);
+        }
+        tcg_out_mov_reg(s, COND_AL, args[0], a0);
+        break;
+    case INDEX_op_neg_i32:
+        tcg_out_dat_imm(s, COND_AL, ARITH_RSB, args[0], args[1], 0);
+        break;
+    case INDEX_op_not_i32:
+        tcg_out_dat_reg(s, COND_AL,
+                        ARITH_MVN, args[0], 0, args[1], SHIFT_IMM_LSL(0));
+        break;
+    case INDEX_op_mul_i32:
+        tcg_out_mul32(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_mulu2_i32:
+        tcg_out_umull32(s, COND_AL, args[0], args[1], args[2], args[3]);
+        break;
+    case INDEX_op_muls2_i32:
+        tcg_out_smull32(s, COND_AL, args[0], args[1], args[2], args[3]);
+        break;
+    /* XXX: Perhaps args[2] & 0x1f is wrong */
+    case INDEX_op_shl_i32:
+        c = const_args[2] ?
+                SHIFT_IMM_LSL(args[2] & 0x1f) : SHIFT_REG_LSL(args[2]);
+        goto gen_shift32;
+    case INDEX_op_shr_i32:
+        c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_LSR(args[2] & 0x1f) :
+                SHIFT_IMM_LSL(0) : SHIFT_REG_LSR(args[2]);
+        goto gen_shift32;
+    case INDEX_op_sar_i32:
+        c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_ASR(args[2] & 0x1f) :
+                SHIFT_IMM_LSL(0) : SHIFT_REG_ASR(args[2]);
+        goto gen_shift32;
+    case INDEX_op_rotr_i32:
+        c = const_args[2] ? (args[2] & 0x1f) ? SHIFT_IMM_ROR(args[2] & 0x1f) :
+                SHIFT_IMM_LSL(0) : SHIFT_REG_ROR(args[2]);
+        /* Fall through.  */
+    gen_shift32:
+        tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1], c);
+        break;
+
+    case INDEX_op_rotl_i32:
+        if (const_args[2]) {
+            tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1],
+                            ((0x20 - args[2]) & 0x1f) ?
+                            SHIFT_IMM_ROR((0x20 - args[2]) & 0x1f) :
+                            SHIFT_IMM_LSL(0));
+        } else {
+            tcg_out_dat_imm(s, COND_AL, ARITH_RSB, TCG_REG_TMP, args[2], 0x20);
+            tcg_out_dat_reg(s, COND_AL, ARITH_MOV, args[0], 0, args[1],
+                            SHIFT_REG_ROR(TCG_REG_TMP));
+        }
+        break;
+
+    case INDEX_op_brcond_i32:
+        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
+                       args[0], args[1], const_args[1]);
+        tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[2]],
+                           arg_label(args[3]));
+        break;
+    case INDEX_op_brcond2_i32:
+        /* The resulting conditions are:
+         * TCG_COND_EQ    -->  a0 == a2 && a1 == a3,
+         * TCG_COND_NE    --> (a0 != a2 && a1 == a3) ||  a1 != a3,
+         * TCG_COND_LT(U) --> (a0 <  a2 && a1 == a3) ||  a1 <  a3,
+         * TCG_COND_GE(U) --> (a0 >= a2 && a1 == a3) || (a1 >= a3 && a1 != a3),
+         * TCG_COND_LE(U) --> (a0 <= a2 && a1 == a3) || (a1 <= a3 && a1 != a3),
+         * TCG_COND_GT(U) --> (a0 >  a2 && a1 == a3) ||  a1 >  a3,
+         */
+        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
+                        args[1], args[3], const_args[3]);
+        tcg_out_dat_rIN(s, COND_EQ, ARITH_CMP, ARITH_CMN, 0,
+                        args[0], args[2], const_args[2]);
+        tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[4]],
+                           arg_label(args[5]));
+        break;
+    case INDEX_op_setcond_i32:
+        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
+                        args[1], args[2], const_args[2]);
+        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[3]],
+                        ARITH_MOV, args[0], 0, 1);
+        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[3])],
+                        ARITH_MOV, args[0], 0, 0);
+        break;
+    case INDEX_op_setcond2_i32:
+        /* See brcond2_i32 comment */
+        tcg_out_dat_rIN(s, COND_AL, ARITH_CMP, ARITH_CMN, 0,
+                        args[2], args[4], const_args[4]);
+        tcg_out_dat_rIN(s, COND_EQ, ARITH_CMP, ARITH_CMN, 0,
+                        args[1], args[3], const_args[3]);
+        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[5]],
+                        ARITH_MOV, args[0], 0, 1);
+        tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[5])],
+                        ARITH_MOV, args[0], 0, 0);
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, args, 0);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args, 1);
+        break;
+    case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, args, 0);
+        break;
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args, 1);
+        break;
+
+    case INDEX_op_bswap16_i32:
+        tcg_out_bswap16(s, COND_AL, args[0], args[1]);
+        break;
+    case INDEX_op_bswap32_i32:
+        tcg_out_bswap32(s, COND_AL, args[0], args[1]);
+        break;
+
+    case INDEX_op_ext8s_i32:
+        tcg_out_ext8s(s, COND_AL, args[0], args[1]);
+        break;
+    case INDEX_op_ext16s_i32:
+        tcg_out_ext16s(s, COND_AL, args[0], args[1]);
+        break;
+    case INDEX_op_ext16u_i32:
+        tcg_out_ext16u(s, COND_AL, args[0], args[1]);
+        break;
+
+    case INDEX_op_deposit_i32:
+        tcg_out_deposit(s, COND_AL, args[0], args[2],
+                        args[3], args[4], const_args[2]);
+        break;
+
+    case INDEX_op_div_i32:
+        tcg_out_sdiv(s, COND_AL, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_divu_i32:
+        tcg_out_udiv(s, COND_AL, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+}
+
+static const TCGTargetOpDef arm_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "r", "r" } },
+    { INDEX_op_st16_i32, { "r", "r" } },
+    { INDEX_op_st_i32, { "r", "r" } },
+
+    /* TODO: "r", "r", "ri" */
+    { INDEX_op_add_i32, { "r", "r", "rIN" } },
+    { INDEX_op_sub_i32, { "r", "rI", "rIN" } },
+    { INDEX_op_mul_i32, { "r", "r", "r" } },
+    { INDEX_op_mulu2_i32, { "r", "r", "r", "r" } },
+    { INDEX_op_muls2_i32, { "r", "r", "r", "r" } },
+    { INDEX_op_and_i32, { "r", "r", "rIK" } },
+    { INDEX_op_andc_i32, { "r", "r", "rIK" } },
+    { INDEX_op_or_i32, { "r", "r", "rI" } },
+    { INDEX_op_xor_i32, { "r", "r", "rI" } },
+    { INDEX_op_neg_i32, { "r", "r" } },
+    { INDEX_op_not_i32, { "r", "r" } },
+
+    { INDEX_op_shl_i32, { "r", "r", "ri" } },
+    { INDEX_op_shr_i32, { "r", "r", "ri" } },
+    { INDEX_op_sar_i32, { "r", "r", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
+
+    { INDEX_op_brcond_i32, { "r", "rIN" } },
+    { INDEX_op_setcond_i32, { "r", "r", "rIN" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rIN", "rIK", "0" } },
+
+    { INDEX_op_add2_i32, { "r", "r", "r", "r", "rIN", "rIK" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rI", "rI", "rIN", "rIK" } },
+    { INDEX_op_brcond2_i32, { "r", "r", "rIN", "rIN" } },
+    { INDEX_op_setcond2_i32, { "r", "r", "r", "rIN", "rIN" } },
+
+#if TARGET_LONG_BITS == 32
+    { INDEX_op_qemu_ld_i32, { "r", "l" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "l" } },
+    { INDEX_op_qemu_st_i32, { "s", "s" } },
+    { INDEX_op_qemu_st_i64, { "s", "s", "s" } },
+#else
+    { INDEX_op_qemu_ld_i32, { "r", "l", "l" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "l", "l" } },
+    { INDEX_op_qemu_st_i32, { "s", "s", "s" } },
+    { INDEX_op_qemu_st_i64, { "s", "s", "s", "s" } },
+#endif
+
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
+
+    { INDEX_op_ext8s_i32, { "r", "r" } },
+    { INDEX_op_ext16s_i32, { "r", "r" } },
+    { INDEX_op_ext16u_i32, { "r", "r" } },
+
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+
+    { INDEX_op_div_i32, { "r", "r", "r" } },
+    { INDEX_op_divu_i32, { "r", "r", "r" } },
+
+    { -1 },
+};
+
+static void tcg_target_init(TCGContext *s)
+{
+    /* Only probe for the platform and capabilities if we havn't already
+       determined maximum values at compile time.  */
+#ifndef use_idiv_instructions
+    {
+        unsigned long hwcap = qemu_getauxval(AT_HWCAP);
+        use_idiv_instructions = (hwcap & HWCAP_ARM_IDIVA) != 0;
+    }
+#endif
+    if (__ARM_ARCH < 7) {
+        const char *pl = (const char *)qemu_getauxval(AT_PLATFORM);
+        if (pl != NULL && pl[0] == 'v' && pl[1] >= '4' && pl[1] <= '9') {
+            arm_arch = pl[1] - '0';
+        }
+    }
+
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
+    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
+                     (1 << TCG_REG_R0) |
+                     (1 << TCG_REG_R1) |
+                     (1 << TCG_REG_R2) |
+                     (1 << TCG_REG_R3) |
+                     (1 << TCG_REG_R12) |
+                     (1 << TCG_REG_R14));
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_PC);
+
+    tcg_add_target_add_op_defs(arm_op_defs);
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ld32u(s, COND_AL, arg, arg1, arg2);
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_st32(s, COND_AL, arg, arg1, arg2);
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    return false;
+}
+
+static inline void tcg_out_mov(TCGContext *s, TCGType type,
+                               TCGReg ret, TCGReg arg)
+{
+    tcg_out_dat_reg(s, COND_AL, ARITH_MOV, ret, 0, arg, SHIFT_IMM_LSL(0));
+}
+
+static inline void tcg_out_movi(TCGContext *s, TCGType type,
+                                TCGReg ret, tcg_target_long arg)
+{
+    tcg_out_movi32(s, COND_AL, ret, arg);
+}
+
+/* Compute frame size via macros, to share between tcg_target_qemu_prologue
+   and tcg_register_jit.  */
+
+#define PUSH_SIZE  ((11 - 4 + 1 + 1) * sizeof(tcg_target_long))
+
+#define FRAME_SIZE \
+    ((PUSH_SIZE \
+      + TCG_STATIC_CALL_ARGS_SIZE \
+      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
+      + TCG_TARGET_STACK_ALIGN - 1) \
+     & -TCG_TARGET_STACK_ALIGN)
+
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    int stack_addend;
+
+    /* Calling convention requires us to save r4-r11 and lr.  */
+    /* stmdb sp!, { r4 - r11, lr } */
+    tcg_out32(s, (COND_AL << 28) | 0x092d4ff0);
+
+    /* Reserve callee argument and tcg temp space.  */
+    stack_addend = FRAME_SIZE - PUSH_SIZE;
+
+    tcg_out_dat_rI(s, COND_AL, ARITH_SUB, TCG_REG_CALL_STACK,
+                   TCG_REG_CALL_STACK, stack_addend, 1);
+    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+
+    tcg_out_bx(s, COND_AL, tcg_target_call_iarg_regs[1]);
+    tb_ret_addr = s->code_ptr;
+
+    /* Epilogue.  We branch here via tb_ret_addr.  */
+    tcg_out_dat_rI(s, COND_AL, ARITH_ADD, TCG_REG_CALL_STACK,
+                   TCG_REG_CALL_STACK, stack_addend, 1);
+
+    /* ldmia sp!, { r4 - r11, pc } */
+    tcg_out32(s, (COND_AL << 28) | 0x08bd8ff0);
+}
+
+typedef struct {
+    DebugFrameHeader h;
+    uint8_t fde_def_cfa[4];
+    uint8_t fde_reg_ofs[18];
+} DebugFrame;
+
+#define ELF_HOST_MACHINE EM_ARM
+
+/* We're expecting a 2 byte uleb128 encoded value.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
+
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
+    .h.cie.return_column = 14,
+
+    /* Total FDE size does not include the "len" member.  */
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, 13,                         /* DW_CFA_def_cfa sp, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        /* The following must match the stmdb in the prologue.  */
+        0x8e, 1,                        /* DW_CFA_offset, lr, -4 */
+        0x8b, 2,                        /* DW_CFA_offset, r11, -8 */
+        0x8a, 3,                        /* DW_CFA_offset, r10, -12 */
+        0x89, 4,                        /* DW_CFA_offset, r9, -16 */
+        0x88, 5,                        /* DW_CFA_offset, r8, -20 */
+        0x87, 6,                        /* DW_CFA_offset, r7, -24 */
+        0x86, 7,                        /* DW_CFA_offset, r6, -28 */
+        0x85, 8,                        /* DW_CFA_offset, r5, -32 */
+        0x84, 9,                        /* DW_CFA_offset, r4, -36 */
+    }
+};
+
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
+}
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
deleted file mode 100644
index bde8a62..0000000
--- a/tcg/i386/tcg-target.c
+++ /dev/null
@@ -1,2461 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "tcg-be-ldst.h"
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-#if TCG_TARGET_REG_BITS == 64
-    "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
-    "%r8",  "%r9",  "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
-#else
-    "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
-#endif
-};
-#endif
-
-static const int tcg_target_reg_alloc_order[] = {
-#if TCG_TARGET_REG_BITS == 64
-    TCG_REG_RBP,
-    TCG_REG_RBX,
-    TCG_REG_R12,
-    TCG_REG_R13,
-    TCG_REG_R14,
-    TCG_REG_R15,
-    TCG_REG_R10,
-    TCG_REG_R11,
-    TCG_REG_R9,
-    TCG_REG_R8,
-    TCG_REG_RCX,
-    TCG_REG_RDX,
-    TCG_REG_RSI,
-    TCG_REG_RDI,
-    TCG_REG_RAX,
-#else
-    TCG_REG_EBX,
-    TCG_REG_ESI,
-    TCG_REG_EDI,
-    TCG_REG_EBP,
-    TCG_REG_ECX,
-    TCG_REG_EDX,
-    TCG_REG_EAX,
-#endif
-};
-
-static const int tcg_target_call_iarg_regs[] = {
-#if TCG_TARGET_REG_BITS == 64
-#if defined(_WIN64)
-    TCG_REG_RCX,
-    TCG_REG_RDX,
-#else
-    TCG_REG_RDI,
-    TCG_REG_RSI,
-    TCG_REG_RDX,
-    TCG_REG_RCX,
-#endif
-    TCG_REG_R8,
-    TCG_REG_R9,
-#else
-    /* 32 bit mode uses stack based calling convention (GCC default). */
-#endif
-};
-
-static const int tcg_target_call_oarg_regs[] = {
-    TCG_REG_EAX,
-#if TCG_TARGET_REG_BITS == 32
-    TCG_REG_EDX
-#endif
-};
-
-/* Constants we accept.  */
-#define TCG_CT_CONST_S32 0x100
-#define TCG_CT_CONST_U32 0x200
-#define TCG_CT_CONST_I32 0x400
-
-/* Registers used with L constraint, which are the first argument 
-   registers on x86_64, and two random call clobbered registers on
-   i386. */
-#if TCG_TARGET_REG_BITS == 64
-# define TCG_REG_L0 tcg_target_call_iarg_regs[0]
-# define TCG_REG_L1 tcg_target_call_iarg_regs[1]
-#else
-# define TCG_REG_L0 TCG_REG_EAX
-# define TCG_REG_L1 TCG_REG_EDX
-#endif
-
-/* The host compiler should supply <cpuid.h> to enable runtime features
-   detection, as we're not going to go so far as our own inline assembly.
-   If not available, default values will be assumed.  */
-#if defined(CONFIG_CPUID_H)
-#include <cpuid.h>
-#endif
-
-/* For 32-bit, we are going to attempt to determine at runtime whether cmov
-   is available.  */
-#if TCG_TARGET_REG_BITS == 64
-# define have_cmov 1
-#elif defined(CONFIG_CPUID_H) && defined(bit_CMOV)
-static bool have_cmov;
-#else
-# define have_cmov 0
-#endif
-
-/* If bit_MOVBE is defined in cpuid.h (added in GCC version 4.6), we are
-   going to attempt to determine at runtime whether movbe is available.  */
-#if defined(CONFIG_CPUID_H) && defined(bit_MOVBE)
-static bool have_movbe;
-#else
-# define have_movbe 0
-#endif
-
-/* We need this symbol in tcg-target.h, and we can't properly conditionalize
-   it there.  Therefore we always define the variable.  */
-bool have_bmi1;
-
-#if defined(CONFIG_CPUID_H) && defined(bit_BMI2)
-static bool have_bmi2;
-#else
-# define have_bmi2 0
-#endif
-
-static tcg_insn_unit *tb_ret_addr;
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    value += addend;
-    switch(type) {
-    case R_386_PC32:
-        value -= (uintptr_t)code_ptr;
-        if (value != (int32_t)value) {
-            tcg_abort();
-        }
-        tcg_patch32(code_ptr, value);
-        break;
-    case R_386_PC8:
-        value -= (uintptr_t)code_ptr;
-        if (value != (int8_t)value) {
-            tcg_abort();
-        }
-        tcg_patch8(code_ptr, value);
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch(ct_str[0]) {
-    case 'a':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_EAX);
-        break;
-    case 'b':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_EBX);
-        break;
-    case 'c':
-    case_c:
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_ECX);
-        break;
-    case 'd':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_EDX);
-        break;
-    case 'S':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_ESI);
-        break;
-    case 'D':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_EDI);
-        break;
-    case 'q':
-        ct->ct |= TCG_CT_REG;
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_regset_set32(ct->u.regs, 0, 0xffff);
-        } else {
-            tcg_regset_set32(ct->u.regs, 0, 0xf);
-        }
-        break;
-    case 'Q':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xf);
-        break;
-    case 'r':
-    case_r:
-        ct->ct |= TCG_CT_REG;
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_regset_set32(ct->u.regs, 0, 0xffff);
-        } else {
-            tcg_regset_set32(ct->u.regs, 0, 0xff);
-        }
-        break;
-    case 'C':
-        /* With SHRX et al, we need not use ECX as shift count register.  */
-        if (have_bmi2) {
-            goto case_r;
-        } else {
-            goto case_c;
-        }
-
-        /* qemu_ld/st address constraint */
-    case 'L':
-        ct->ct |= TCG_CT_REG;
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_regset_set32(ct->u.regs, 0, 0xffff);
-        } else {
-            tcg_regset_set32(ct->u.regs, 0, 0xff);
-        }
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L1);
-        break;
-
-    case 'e':
-        ct->ct |= TCG_CT_CONST_S32;
-        break;
-    case 'Z':
-        ct->ct |= TCG_CT_CONST_U32;
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_I32;
-        break;
-
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    }
-    if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
-        return 1;
-    }
-    if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
-        return 1;
-    }
-    if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) {
-        return 1;
-    }
-    return 0;
-}
-
-#if TCG_TARGET_REG_BITS == 64
-# define LOWREGMASK(x)	((x) & 7)
-#else
-# define LOWREGMASK(x)	(x)
-#endif
-
-#define P_EXT		0x100		/* 0x0f opcode prefix */
-#define P_EXT38         0x200           /* 0x0f 0x38 opcode prefix */
-#define P_DATA16        0x400           /* 0x66 opcode prefix */
-#if TCG_TARGET_REG_BITS == 64
-# define P_ADDR32       0x800           /* 0x67 opcode prefix */
-# define P_REXW         0x1000          /* Set REX.W = 1 */
-# define P_REXB_R       0x2000          /* REG field as byte register */
-# define P_REXB_RM      0x4000          /* R/M field as byte register */
-# define P_GS           0x8000          /* gs segment override */
-#else
-# define P_ADDR32	0
-# define P_REXW		0
-# define P_REXB_R	0
-# define P_REXB_RM	0
-# define P_GS           0
-#endif
-#define P_SIMDF3        0x10000         /* 0xf3 opcode prefix */
-#define P_SIMDF2        0x20000         /* 0xf2 opcode prefix */
-
-#define OPC_ARITH_EvIz	(0x81)
-#define OPC_ARITH_EvIb	(0x83)
-#define OPC_ARITH_GvEv	(0x03)		/* ... plus (ARITH_FOO << 3) */
-#define OPC_ANDN        (0xf2 | P_EXT38)
-#define OPC_ADD_GvEv	(OPC_ARITH_GvEv | (ARITH_ADD << 3))
-#define OPC_BSWAP	(0xc8 | P_EXT)
-#define OPC_CALL_Jz	(0xe8)
-#define OPC_CMOVCC      (0x40 | P_EXT)  /* ... plus condition code */
-#define OPC_CMP_GvEv	(OPC_ARITH_GvEv | (ARITH_CMP << 3))
-#define OPC_DEC_r32	(0x48)
-#define OPC_IMUL_GvEv	(0xaf | P_EXT)
-#define OPC_IMUL_GvEvIb	(0x6b)
-#define OPC_IMUL_GvEvIz	(0x69)
-#define OPC_INC_r32	(0x40)
-#define OPC_JCC_long	(0x80 | P_EXT)	/* ... plus condition code */
-#define OPC_JCC_short	(0x70)		/* ... plus condition code */
-#define OPC_JMP_long	(0xe9)
-#define OPC_JMP_short	(0xeb)
-#define OPC_LEA         (0x8d)
-#define OPC_MOVB_EvGv	(0x88)		/* stores, more or less */
-#define OPC_MOVL_EvGv	(0x89)		/* stores, more or less */
-#define OPC_MOVL_GvEv	(0x8b)		/* loads, more or less */
-#define OPC_MOVB_EvIz   (0xc6)
-#define OPC_MOVL_EvIz	(0xc7)
-#define OPC_MOVL_Iv     (0xb8)
-#define OPC_MOVBE_GyMy  (0xf0 | P_EXT38)
-#define OPC_MOVBE_MyGy  (0xf1 | P_EXT38)
-#define OPC_MOVSBL	(0xbe | P_EXT)
-#define OPC_MOVSWL	(0xbf | P_EXT)
-#define OPC_MOVSLQ	(0x63 | P_REXW)
-#define OPC_MOVZBL	(0xb6 | P_EXT)
-#define OPC_MOVZWL	(0xb7 | P_EXT)
-#define OPC_POP_r32	(0x58)
-#define OPC_PUSH_r32	(0x50)
-#define OPC_PUSH_Iv	(0x68)
-#define OPC_PUSH_Ib	(0x6a)
-#define OPC_RET		(0xc3)
-#define OPC_SETCC	(0x90 | P_EXT | P_REXB_RM) /* ... plus cc */
-#define OPC_SHIFT_1	(0xd1)
-#define OPC_SHIFT_Ib	(0xc1)
-#define OPC_SHIFT_cl	(0xd3)
-#define OPC_SARX        (0xf7 | P_EXT38 | P_SIMDF3)
-#define OPC_SHLX        (0xf7 | P_EXT38 | P_DATA16)
-#define OPC_SHRX        (0xf7 | P_EXT38 | P_SIMDF2)
-#define OPC_TESTL	(0x85)
-#define OPC_XCHG_ax_r32	(0x90)
-
-#define OPC_GRP3_Ev	(0xf7)
-#define OPC_GRP5	(0xff)
-
-/* Group 1 opcode extensions for 0x80-0x83.
-   These are also used as modifiers for OPC_ARITH.  */
-#define ARITH_ADD 0
-#define ARITH_OR  1
-#define ARITH_ADC 2
-#define ARITH_SBB 3
-#define ARITH_AND 4
-#define ARITH_SUB 5
-#define ARITH_XOR 6
-#define ARITH_CMP 7
-
-/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3.  */
-#define SHIFT_ROL 0
-#define SHIFT_ROR 1
-#define SHIFT_SHL 4
-#define SHIFT_SHR 5
-#define SHIFT_SAR 7
-
-/* Group 3 opcode extensions for 0xf6, 0xf7.  To be used with OPC_GRP3.  */
-#define EXT3_NOT   2
-#define EXT3_NEG   3
-#define EXT3_MUL   4
-#define EXT3_IMUL  5
-#define EXT3_DIV   6
-#define EXT3_IDIV  7
-
-/* Group 5 opcode extensions for 0xff.  To be used with OPC_GRP5.  */
-#define EXT5_INC_Ev	0
-#define EXT5_DEC_Ev	1
-#define EXT5_CALLN_Ev	2
-#define EXT5_JMPN_Ev	4
-
-/* Condition codes to be added to OPC_JCC_{long,short}.  */
-#define JCC_JMP (-1)
-#define JCC_JO  0x0
-#define JCC_JNO 0x1
-#define JCC_JB  0x2
-#define JCC_JAE 0x3
-#define JCC_JE  0x4
-#define JCC_JNE 0x5
-#define JCC_JBE 0x6
-#define JCC_JA  0x7
-#define JCC_JS  0x8
-#define JCC_JNS 0x9
-#define JCC_JP  0xa
-#define JCC_JNP 0xb
-#define JCC_JL  0xc
-#define JCC_JGE 0xd
-#define JCC_JLE 0xe
-#define JCC_JG  0xf
-
-static const uint8_t tcg_cond_to_jcc[] = {
-    [TCG_COND_EQ] = JCC_JE,
-    [TCG_COND_NE] = JCC_JNE,
-    [TCG_COND_LT] = JCC_JL,
-    [TCG_COND_GE] = JCC_JGE,
-    [TCG_COND_LE] = JCC_JLE,
-    [TCG_COND_GT] = JCC_JG,
-    [TCG_COND_LTU] = JCC_JB,
-    [TCG_COND_GEU] = JCC_JAE,
-    [TCG_COND_LEU] = JCC_JBE,
-    [TCG_COND_GTU] = JCC_JA,
-};
-
-#if TCG_TARGET_REG_BITS == 64
-static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
-{
-    int rex;
-
-    if (opc & P_GS) {
-        tcg_out8(s, 0x65);
-    }
-    if (opc & P_DATA16) {
-        /* We should never be asking for both 16 and 64-bit operation.  */
-        assert((opc & P_REXW) == 0);
-        tcg_out8(s, 0x66);
-    }
-    if (opc & P_ADDR32) {
-        tcg_out8(s, 0x67);
-    }
-
-    rex = 0;
-    rex |= (opc & P_REXW) ? 0x8 : 0x0;  /* REX.W */
-    rex |= (r & 8) >> 1;                /* REX.R */
-    rex |= (x & 8) >> 2;                /* REX.X */
-    rex |= (rm & 8) >> 3;               /* REX.B */
-
-    /* P_REXB_{R,RM} indicates that the given register is the low byte.
-       For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
-       as otherwise the encoding indicates %[abcd]h.  Note that the values
-       that are ORed in merely indicate that the REX byte must be present;
-       those bits get discarded in output.  */
-    rex |= opc & (r >= 4 ? P_REXB_R : 0);
-    rex |= opc & (rm >= 4 ? P_REXB_RM : 0);
-
-    if (rex) {
-        tcg_out8(s, (uint8_t)(rex | 0x40));
-    }
-
-    if (opc & (P_EXT | P_EXT38)) {
-        tcg_out8(s, 0x0f);
-        if (opc & P_EXT38) {
-            tcg_out8(s, 0x38);
-        }
-    }
-
-    tcg_out8(s, opc);
-}
-#else
-static void tcg_out_opc(TCGContext *s, int opc)
-{
-    if (opc & P_DATA16) {
-        tcg_out8(s, 0x66);
-    }
-    if (opc & (P_EXT | P_EXT38)) {
-        tcg_out8(s, 0x0f);
-        if (opc & P_EXT38) {
-            tcg_out8(s, 0x38);
-        }
-    }
-    tcg_out8(s, opc);
-}
-/* Discard the register arguments to tcg_out_opc early, so as not to penalize
-   the 32-bit compilation paths.  This method works with all versions of gcc,
-   whereas relying on optimization may not be able to exclude them.  */
-#define tcg_out_opc(s, opc, r, rm, x)  (tcg_out_opc)(s, opc)
-#endif
-
-static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
-{
-    tcg_out_opc(s, opc, r, rm, 0);
-    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
-}
-
-static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm)
-{
-    int tmp;
-
-    if ((opc & (P_REXW | P_EXT | P_EXT38)) || (rm & 8)) {
-        /* Three byte VEX prefix.  */
-        tcg_out8(s, 0xc4);
-
-        /* VEX.m-mmmm */
-        if (opc & P_EXT38) {
-            tmp = 2;
-        } else if (opc & P_EXT) {
-            tmp = 1;
-        } else {
-            tcg_abort();
-        }
-        tmp |= 0x40;                       /* VEX.X */
-        tmp |= (r & 8 ? 0 : 0x80);         /* VEX.R */
-        tmp |= (rm & 8 ? 0 : 0x20);        /* VEX.B */
-        tcg_out8(s, tmp);
-
-        tmp = (opc & P_REXW ? 0x80 : 0);   /* VEX.W */
-    } else {
-        /* Two byte VEX prefix.  */
-        tcg_out8(s, 0xc5);
-
-        tmp = (r & 8 ? 0 : 0x80);          /* VEX.R */
-    }
-    /* VEX.pp */
-    if (opc & P_DATA16) {
-        tmp |= 1;                          /* 0x66 */
-    } else if (opc & P_SIMDF3) {
-        tmp |= 2;                          /* 0xf3 */
-    } else if (opc & P_SIMDF2) {
-        tmp |= 3;                          /* 0xf2 */
-    }
-    tmp |= (~v & 15) << 3;                 /* VEX.vvvv */
-    tcg_out8(s, tmp);
-    tcg_out8(s, opc);
-    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
-}
-
-/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
-   We handle either RM and INDEX missing with a negative value.  In 64-bit
-   mode for absolute addresses, ~RM is the size of the immediate operand
-   that will follow the instruction.  */
-
-static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
-                                     int index, int shift, intptr_t offset)
-{
-    int mod, len;
-
-    if (index < 0 && rm < 0) {
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* Try for a rip-relative addressing mode.  This has replaced
-               the 32-bit-mode absolute addressing encoding.  */
-            intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm;
-            intptr_t disp = offset - pc;
-            if (disp == (int32_t)disp) {
-                tcg_out_opc(s, opc, r, 0, 0);
-                tcg_out8(s, (LOWREGMASK(r) << 3) | 5);
-                tcg_out32(s, disp);
-                return;
-            }
-
-            /* Try for an absolute address encoding.  This requires the
-               use of the MODRM+SIB encoding and is therefore larger than
-               rip-relative addressing.  */
-            if (offset == (int32_t)offset) {
-                tcg_out_opc(s, opc, r, 0, 0);
-                tcg_out8(s, (LOWREGMASK(r) << 3) | 4);
-                tcg_out8(s, (4 << 3) | 5);
-                tcg_out32(s, offset);
-                return;
-            }
-
-            /* ??? The memory isn't directly addressable.  */
-            tcg_abort();
-        } else {
-            /* Absolute address.  */
-            tcg_out_opc(s, opc, r, 0, 0);
-            tcg_out8(s, (r << 3) | 5);
-            tcg_out32(s, offset);
-            return;
-        }
-    }
-
-    /* Find the length of the immediate addend.  Note that the encoding
-       that would be used for (%ebp) indicates absolute addressing.  */
-    if (rm < 0) {
-        mod = 0, len = 4, rm = 5;
-    } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {
-        mod = 0, len = 0;
-    } else if (offset == (int8_t)offset) {
-        mod = 0x40, len = 1;
-    } else {
-        mod = 0x80, len = 4;
-    }
-
-    /* Use a single byte MODRM format if possible.  Note that the encoding
-       that would be used for %esp is the escape to the two byte form.  */
-    if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {
-        /* Single byte MODRM format.  */
-        tcg_out_opc(s, opc, r, rm, 0);
-        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
-    } else {
-        /* Two byte MODRM+SIB format.  */
-
-        /* Note that the encoding that would place %esp into the index
-           field indicates no index register.  In 64-bit mode, the REX.X
-           bit counts, so %r12 can be used as the index.  */
-        if (index < 0) {
-            index = 4;
-        } else {
-            assert(index != TCG_REG_ESP);
-        }
-
-        tcg_out_opc(s, opc, r, rm, index);
-        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4);
-        tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm));
-    }
-
-    if (len == 1) {
-        tcg_out8(s, offset);
-    } else if (len == 4) {
-        tcg_out32(s, offset);
-    }
-}
-
-/* A simplification of the above with no index or shift.  */
-static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
-                                        int rm, intptr_t offset)
-{
-    tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
-}
-
-/* Generate dest op= src.  Uses the same ARITH_* codes as tgen_arithi.  */
-static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)
-{
-    /* Propagate an opcode prefix, such as P_REXW.  */
-    int ext = subop & ~0x7;
-    subop &= 0x7;
-
-    tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);
-}
-
-static inline void tcg_out_mov(TCGContext *s, TCGType type,
-                               TCGReg ret, TCGReg arg)
-{
-    if (arg != ret) {
-        int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
-        tcg_out_modrm(s, opc, ret, arg);
-    }
-}
-
-static void tcg_out_movi(TCGContext *s, TCGType type,
-                         TCGReg ret, tcg_target_long arg)
-{
-    tcg_target_long diff;
-
-    if (arg == 0) {
-        tgen_arithr(s, ARITH_XOR, ret, ret);
-        return;
-    }
-    if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
-        tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
-        tcg_out32(s, arg);
-        return;
-    }
-    if (arg == (int32_t)arg) {
-        tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
-        tcg_out32(s, arg);
-        return;
-    }
-
-    /* Try a 7 byte pc-relative lea before the 10 byte movq.  */
-    diff = arg - ((uintptr_t)s->code_ptr + 7);
-    if (diff == (int32_t)diff) {
-        tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
-        tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
-        tcg_out32(s, diff);
-        return;
-    }
-
-    tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
-    tcg_out64(s, arg);
-}
-
-static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
-{
-    if (val == (int8_t)val) {
-        tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);
-        tcg_out8(s, val);
-    } else if (val == (int32_t)val) {
-        tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
-        tcg_out32(s, val);
-    } else {
-        tcg_abort();
-    }
-}
-
-static inline void tcg_out_push(TCGContext *s, int reg)
-{
-    tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);
-}
-
-static inline void tcg_out_pop(TCGContext *s, int reg)
-{
-    tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, intptr_t arg2)
-{
-    int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
-    tcg_out_modrm_offset(s, opc, ret, arg1, arg2);
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    int opc = OPC_MOVL_EvGv + (type == TCG_TYPE_I64 ? P_REXW : 0);
-    tcg_out_modrm_offset(s, opc, arg, arg1, arg2);
-}
-
-static inline void tcg_out_sti(TCGContext *s, TCGType type, TCGReg base,
-                               tcg_target_long ofs, tcg_target_long val)
-{
-    int opc = OPC_MOVL_EvIz + (type == TCG_TYPE_I64 ? P_REXW : 0);
-    tcg_out_modrm_offset(s, opc, 0, base, ofs);
-    tcg_out32(s, val);
-}
-
-static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
-{
-    /* Propagate an opcode prefix, such as P_DATA16.  */
-    int ext = subopc & ~0x7;
-    subopc &= 0x7;
-
-    if (count == 1) {
-        tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);
-    } else {
-        tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);
-        tcg_out8(s, count);
-    }
-}
-
-static inline void tcg_out_bswap32(TCGContext *s, int reg)
-{
-    tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);
-}
-
-static inline void tcg_out_rolw_8(TCGContext *s, int reg)
-{
-    tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
-}
-
-static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
-{
-    /* movzbl */
-    assert(src < 4 || TCG_TARGET_REG_BITS == 64);
-    tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
-}
-
-static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
-{
-    /* movsbl */
-    assert(src < 4 || TCG_TARGET_REG_BITS == 64);
-    tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
-}
-
-static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
-{
-    /* movzwl */
-    tcg_out_modrm(s, OPC_MOVZWL, dest, src);
-}
-
-static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
-{
-    /* movsw[lq] */
-    tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
-}
-
-static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
-{
-    /* 32-bit mov zero extends.  */
-    tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
-}
-
-static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
-{
-    tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
-}
-
-static inline void tcg_out_bswap64(TCGContext *s, int reg)
-{
-    tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
-}
-
-static void tgen_arithi(TCGContext *s, int c, int r0,
-                        tcg_target_long val, int cf)
-{
-    int rexw = 0;
-
-    if (TCG_TARGET_REG_BITS == 64) {
-        rexw = c & -8;
-        c &= 7;
-    }
-
-    /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
-       partial flags update stalls on Pentium4 and are not recommended
-       by current Intel optimization manuals.  */
-    if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) {
-        int is_inc = (c == ARITH_ADD) ^ (val < 0);
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* The single-byte increment encodings are re-tasked as the
-               REX prefixes.  Use the MODRM encoding.  */
-            tcg_out_modrm(s, OPC_GRP5 + rexw,
-                          (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);
-        } else {
-            tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0);
-        }
-        return;
-    }
-
-    if (c == ARITH_AND) {
-        if (TCG_TARGET_REG_BITS == 64) {
-            if (val == 0xffffffffu) {
-                tcg_out_ext32u(s, r0, r0);
-                return;
-            }
-            if (val == (uint32_t)val) {
-                /* AND with no high bits set can use a 32-bit operation.  */
-                rexw = 0;
-            }
-        }
-        if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) {
-            tcg_out_ext8u(s, r0, r0);
-            return;
-        }
-        if (val == 0xffffu) {
-            tcg_out_ext16u(s, r0, r0);
-            return;
-        }
-    }
-
-    if (val == (int8_t)val) {
-        tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);
-        tcg_out8(s, val);
-        return;
-    }
-    if (rexw == 0 || val == (int32_t)val) {
-        tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);
-        tcg_out32(s, val);
-        return;
-    }
-
-    tcg_abort();
-}
-
-static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
-{
-    if (val != 0) {
-        tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);
-    }
-}
-
-/* Use SMALL != 0 to force a short forward branch.  */
-static void tcg_out_jxx(TCGContext *s, int opc, int label_index, int small)
-{
-    int32_t val, val1;
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        val = tcg_pcrel_diff(s, l->u.value_ptr);
-        val1 = val - 2;
-        if ((int8_t)val1 == val1) {
-            if (opc == -1) {
-                tcg_out8(s, OPC_JMP_short);
-            } else {
-                tcg_out8(s, OPC_JCC_short + opc);
-            }
-            tcg_out8(s, val1);
-        } else {
-            if (small) {
-                tcg_abort();
-            }
-            if (opc == -1) {
-                tcg_out8(s, OPC_JMP_long);
-                tcg_out32(s, val - 5);
-            } else {
-                tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
-                tcg_out32(s, val - 6);
-            }
-        }
-    } else if (small) {
-        if (opc == -1) {
-            tcg_out8(s, OPC_JMP_short);
-        } else {
-            tcg_out8(s, OPC_JCC_short + opc);
-        }
-        tcg_out_reloc(s, s->code_ptr, R_386_PC8, label_index, -1);
-        s->code_ptr += 1;
-    } else {
-        if (opc == -1) {
-            tcg_out8(s, OPC_JMP_long);
-        } else {
-            tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
-        }
-        tcg_out_reloc(s, s->code_ptr, R_386_PC32, label_index, -4);
-        s->code_ptr += 4;
-    }
-}
-
-static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2,
-                        int const_arg2, int rexw)
-{
-    if (const_arg2) {
-        if (arg2 == 0) {
-            /* test r, r */
-            tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);
-        } else {
-            tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);
-        }
-    } else {
-        tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);
-    }
-}
-
-static void tcg_out_brcond32(TCGContext *s, TCGCond cond,
-                             TCGArg arg1, TCGArg arg2, int const_arg2,
-                             int label_index, int small)
-{
-    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
-    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index, small);
-}
-
-#if TCG_TARGET_REG_BITS == 64
-static void tcg_out_brcond64(TCGContext *s, TCGCond cond,
-                             TCGArg arg1, TCGArg arg2, int const_arg2,
-                             int label_index, int small)
-{
-    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
-    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label_index, small);
-}
-#else
-/* XXX: we implement it at the target level to avoid having to
-   handle cross basic blocks temporaries */
-static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
-                            const int *const_args, int small)
-{
-    int label_next;
-    label_next = gen_new_label();
-    switch(args[4]) {
-    case TCG_COND_EQ:
-        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
-                         label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3],
-                         args[5], small);
-        break;
-    case TCG_COND_NE:
-        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
-                         args[5], small);
-        tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3],
-                         args[5], small);
-        break;
-    case TCG_COND_LT:
-        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_LE:
-        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_GT:
-        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_GE:
-        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_LTU:
-        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_LEU:
-        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_GTU:
-        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    case TCG_COND_GEU:
-        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
-                         args[5], small);
-        tcg_out_jxx(s, JCC_JNE, label_next, 1);
-        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
-                         args[5], small);
-        break;
-    default:
-        tcg_abort();
-    }
-    tcg_out_label(s, label_next, s->code_ptr);
-}
-#endif
-
-static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest,
-                              TCGArg arg1, TCGArg arg2, int const_arg2)
-{
-    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
-    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
-    tcg_out_ext8u(s, dest, dest);
-}
-
-#if TCG_TARGET_REG_BITS == 64
-static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest,
-                              TCGArg arg1, TCGArg arg2, int const_arg2)
-{
-    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
-    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
-    tcg_out_ext8u(s, dest, dest);
-}
-#else
-static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
-                             const int *const_args)
-{
-    TCGArg new_args[6];
-    int label_true, label_over;
-
-    memcpy(new_args, args+1, 5*sizeof(TCGArg));
-
-    if (args[0] == args[1] || args[0] == args[2]
-        || (!const_args[3] && args[0] == args[3])
-        || (!const_args[4] && args[0] == args[4])) {
-        /* When the destination overlaps with one of the argument
-           registers, don't do anything tricky.  */
-        label_true = gen_new_label();
-        label_over = gen_new_label();
-
-        new_args[5] = label_true;
-        tcg_out_brcond2(s, new_args, const_args+1, 1);
-
-        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
-        tcg_out_jxx(s, JCC_JMP, label_over, 1);
-        tcg_out_label(s, label_true, s->code_ptr);
-
-        tcg_out_movi(s, TCG_TYPE_I32, args[0], 1);
-        tcg_out_label(s, label_over, s->code_ptr);
-    } else {
-        /* When the destination does not overlap one of the arguments,
-           clear the destination first, jump if cond false, and emit an
-           increment in the true case.  This results in smaller code.  */
-
-        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
-
-        label_over = gen_new_label();
-        new_args[4] = tcg_invert_cond(new_args[4]);
-        new_args[5] = label_over;
-        tcg_out_brcond2(s, new_args, const_args+1, 1);
-
-        tgen_arithi(s, ARITH_ADD, args[0], 1, 0);
-        tcg_out_label(s, label_over, s->code_ptr);
-    }
-}
-#endif
-
-static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGArg dest,
-                              TCGArg c1, TCGArg c2, int const_c2,
-                              TCGArg v1)
-{
-    tcg_out_cmp(s, c1, c2, const_c2, 0);
-    if (have_cmov) {
-        tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond], dest, v1);
-    } else {
-        int over = gen_new_label();
-        tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
-        tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
-        tcg_out_label(s, over, s->code_ptr);
-    }
-}
-
-#if TCG_TARGET_REG_BITS == 64
-static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGArg dest,
-                              TCGArg c1, TCGArg c2, int const_c2,
-                              TCGArg v1)
-{
-    tcg_out_cmp(s, c1, c2, const_c2, P_REXW);
-    tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | P_REXW, dest, v1);
-}
-#endif
-
-static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest)
-{
-    intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
-
-    if (disp == (int32_t)disp) {
-        tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
-        tcg_out32(s, disp);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R10, (uintptr_t)dest);
-        tcg_out_modrm(s, OPC_GRP5,
-                      call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev, TCG_REG_R10);
-    }
-}
-
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
-{
-    tcg_out_branch(s, 1, dest);
-}
-
-static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest)
-{
-    tcg_out_branch(s, 0, dest);
-}
-
-#if defined(CONFIG_SOFTMMU)
-/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
- *                                     int mmu_idx, uintptr_t ra)
- */
-static void * const qemu_ld_helpers[16] = {
-    [MO_UB]   = helper_ret_ldub_mmu,
-    [MO_LEUW] = helper_le_lduw_mmu,
-    [MO_LEUL] = helper_le_ldul_mmu,
-    [MO_LEQ]  = helper_le_ldq_mmu,
-    [MO_BEUW] = helper_be_lduw_mmu,
-    [MO_BEUL] = helper_be_ldul_mmu,
-    [MO_BEQ]  = helper_be_ldq_mmu,
-};
-
-/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
- *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
- */
-static void * const qemu_st_helpers[16] = {
-    [MO_UB]   = helper_ret_stb_mmu,
-    [MO_LEUW] = helper_le_stw_mmu,
-    [MO_LEUL] = helper_le_stl_mmu,
-    [MO_LEQ]  = helper_le_stq_mmu,
-    [MO_BEUW] = helper_be_stw_mmu,
-    [MO_BEUL] = helper_be_stl_mmu,
-    [MO_BEQ]  = helper_be_stq_mmu,
-};
-
-/* Perform the TLB load and compare.
-
-   Inputs:
-   ADDRLO and ADDRHI contain the low and high part of the address.
-
-   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
-
-   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
-   This should be offsetof addr_read or addr_write.
-
-   Outputs:
-   LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
-   positions of the displacements of forward jumps to the TLB miss case.
-
-   Second argument register is loaded with the low part of the address.
-   In the TLB hit case, it has been adjusted as indicated by the TLB
-   and so is a host address.  In the TLB miss case, it continues to
-   hold a guest address.
-
-   First argument register is clobbered.  */
-
-static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
-                                    int mem_index, TCGMemOp opc,
-                                    tcg_insn_unit **label_ptr, int which)
-{
-    const TCGReg r0 = TCG_REG_L0;
-    const TCGReg r1 = TCG_REG_L1;
-    TCGType ttype = TCG_TYPE_I32;
-    TCGType tlbtype = TCG_TYPE_I32;
-    int trexw = 0, hrexw = 0, tlbrexw = 0;
-    int a_bits = get_alignment_bits(opc);
-    target_ulong tlb_mask;
-
-    if (TCG_TARGET_REG_BITS == 64) {
-        if (TARGET_LONG_BITS == 64) {
-            ttype = TCG_TYPE_I64;
-            trexw = P_REXW;
-        }
-        if (TCG_TYPE_PTR == TCG_TYPE_I64) {
-            hrexw = P_REXW;
-            if (TARGET_PAGE_BITS + CPU_TLB_BITS > 32) {
-                tlbtype = TCG_TYPE_I64;
-                tlbrexw = P_REXW;
-            }
-        }
-    }
-
-    tcg_out_mov(s, tlbtype, r0, addrlo);
-    if (a_bits >= 0) {
-        /* A byte access or an alignment check required */
-        tcg_out_mov(s, ttype, r1, addrlo);
-        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);
-    } else {
-        /* For unaligned access check that we don't cross pages using
-           the page address of the last byte.  */
-        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo,
-                             (1 << (opc & MO_SIZE)) - 1);
-        tlb_mask = TARGET_PAGE_MASK;
-    }
-
-    tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,
-                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-
-    tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0);
-    tgen_arithi(s, ARITH_AND + tlbrexw, r0,
-                (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
-
-    tcg_out_modrm_sib_offset(s, OPC_LEA + hrexw, r0, TCG_AREG0, r0, 0,
-                             offsetof(CPUArchState, tlb_table[mem_index][0])
-                             + which);
-
-    /* cmp 0(r0), r1 */
-    tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, 0);
-
-    /* Prepare for both the fast path add of the tlb addend, and the slow
-       path function argument setup.  There are two cases worth note:
-       For 32-bit guest and x86_64 host, MOVL zero-extends the guest address
-       before the fastpath ADDQ below.  For 64-bit guest and x32 host, MOVQ
-       copies the entire guest address for the slow path, while truncation
-       for the 32-bit host happens with the fastpath ADDL below.  */
-    tcg_out_mov(s, ttype, r1, addrlo);
-
-    /* jne slow_path */
-    tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
-    label_ptr[0] = s->code_ptr;
-    s->code_ptr += 4;
-
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        /* cmp 4(r0), addrhi */
-        tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, 4);
-
-        /* jne slow_path */
-        tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
-        label_ptr[1] = s->code_ptr;
-        s->code_ptr += 4;
-    }
-
-    /* TLB Hit.  */
-
-    /* add addend(r0), r1 */
-    tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0,
-                         offsetof(CPUTLBEntry, addend) - which);
-}
-
-/*
- * Record the context of a call to the out of line helper code for the slow path
- * for a load or store, so that we can later generate the correct helper code
- */
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
-                                TCGReg datalo, TCGReg datahi,
-                                TCGReg addrlo, TCGReg addrhi,
-                                tcg_insn_unit *raddr,
-                                tcg_insn_unit **label_ptr)
-{
-    TCGLabelQemuLdst *label = new_ldst_label(s);
-
-    label->is_ld = is_ld;
-    label->oi = oi;
-    label->datalo_reg = datalo;
-    label->datahi_reg = datahi;
-    label->addrlo_reg = addrlo;
-    label->addrhi_reg = addrhi;
-    label->raddr = raddr;
-    label->label_ptr[0] = label_ptr[0];
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        label->label_ptr[1] = label_ptr[1];
-    }
-}
-
-/*
- * Generate code for the slow path for a load at the end of block
- */
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
-    TCGMemOpIdx oi = l->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGReg data_reg;
-    tcg_insn_unit **label_ptr = &l->label_ptr[0];
-
-    /* resolve label address */
-    tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
-    }
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        int ofs = 0;
-
-        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        if (TARGET_LONG_BITS == 64) {
-            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
-            ofs += 4;
-        }
-
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, oi);
-        ofs += 4;
-
-        tcg_out_sti(s, TCG_TYPE_PTR, TCG_REG_ESP, ofs, (uintptr_t)l->raddr);
-    } else {
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-        /* The second argument is already loaded with addrlo.  */
-        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], oi);
-        tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
-                     (uintptr_t)l->raddr);
-    }
-
-    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-
-    data_reg = l->datalo_reg;
-    switch (opc & MO_SSIZE) {
-    case MO_SB:
-        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-    case MO_SW:
-        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case MO_SL:
-        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
-        break;
-#endif
-    case MO_UB:
-    case MO_UW:
-        /* Note that the helpers have zero-extended to tcg_target_long.  */
-    case MO_UL:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-        break;
-    case MO_Q:
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
-        } else if (data_reg == TCG_REG_EDX) {
-            /* xchg %edx, %eax */
-            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
-            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-
-    /* Jump to the code corresponding to next IR of qemu_st */
-    tcg_out_jmp(s, l->raddr);
-}
-
-/*
- * Generate code for the slow path for a store at the end of block
- */
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
-    TCGMemOpIdx oi = l->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGMemOp s_bits = opc & MO_SIZE;
-    tcg_insn_unit **label_ptr = &l->label_ptr[0];
-    TCGReg retaddr;
-
-    /* resolve label address */
-    tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
-    }
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        int ofs = 0;
-
-        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        if (TARGET_LONG_BITS == 64) {
-            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
-            ofs += 4;
-        }
-
-        tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
-        ofs += 4;
-
-        if (s_bits == MO_64) {
-            tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
-            ofs += 4;
-        }
-
-        tcg_out_sti(s, TCG_TYPE_I32, TCG_REG_ESP, ofs, oi);
-        ofs += 4;
-
-        retaddr = TCG_REG_EAX;
-        tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
-        tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs);
-    } else {
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-        /* The second argument is already loaded with addrlo.  */
-        tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32),
-                    tcg_target_call_iarg_regs[2], l->datalo_reg);
-        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi);
-
-        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
-            retaddr = tcg_target_call_iarg_regs[4];
-            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
-        } else {
-            retaddr = TCG_REG_RAX;
-            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
-            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP,
-                       TCG_TARGET_CALL_STACK_OFFSET);
-        }
-    }
-
-    /* "Tail call" to the helper, with the return address back inline.  */
-    tcg_out_push(s, retaddr);
-    tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-}
-#elif defined(__x86_64__) && defined(__linux__)
-# include <asm/prctl.h>
-# include <sys/prctl.h>
-
-int arch_prctl(int code, unsigned long addr);
-
-static int guest_base_flags;
-static inline void setup_guest_base_seg(void)
-{
-    if (arch_prctl(ARCH_SET_GS, GUEST_BASE) == 0) {
-        guest_base_flags = P_GS;
-    }
-}
-#else
-# define guest_base_flags 0
-static inline void setup_guest_base_seg(void) { }
-#endif /* SOFTMMU */
-
-static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
-                                   TCGReg base, int index, intptr_t ofs,
-                                   int seg, TCGMemOp memop)
-{
-    const TCGMemOp real_bswap = memop & MO_BSWAP;
-    TCGMemOp bswap = real_bswap;
-    int movop = OPC_MOVL_GvEv;
-
-    if (have_movbe && real_bswap) {
-        bswap = 0;
-        movop = OPC_MOVBE_GyMy;
-    }
-
-    switch (memop & MO_SSIZE) {
-    case MO_UB:
-        tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo,
-                                 base, index, 0, ofs);
-        break;
-    case MO_SB:
-        tcg_out_modrm_sib_offset(s, OPC_MOVSBL + P_REXW + seg, datalo,
-                                 base, index, 0, ofs);
-        break;
-    case MO_UW:
-        tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
-                                 base, index, 0, ofs);
-        if (real_bswap) {
-            tcg_out_rolw_8(s, datalo);
-        }
-        break;
-    case MO_SW:
-        if (real_bswap) {
-            if (have_movbe) {
-                tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
-                                         datalo, base, index, 0, ofs);
-            } else {
-                tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
-                                         base, index, 0, ofs);
-                tcg_out_rolw_8(s, datalo);
-            }
-            tcg_out_modrm(s, OPC_MOVSWL + P_REXW, datalo, datalo);
-        } else {
-            tcg_out_modrm_sib_offset(s, OPC_MOVSWL + P_REXW + seg,
-                                     datalo, base, index, 0, ofs);
-        }
-        break;
-    case MO_UL:
-        tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
-        if (bswap) {
-            tcg_out_bswap32(s, datalo);
-        }
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case MO_SL:
-        if (real_bswap) {
-            tcg_out_modrm_sib_offset(s, movop + seg, datalo,
-                                     base, index, 0, ofs);
-            if (bswap) {
-                tcg_out_bswap32(s, datalo);
-            }
-            tcg_out_ext32s(s, datalo, datalo);
-        } else {
-            tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo,
-                                     base, index, 0, ofs);
-        }
-        break;
-#endif
-    case MO_Q:
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
-                                     base, index, 0, ofs);
-            if (bswap) {
-                tcg_out_bswap64(s, datalo);
-            }
-        } else {
-            if (real_bswap) {
-                int t = datalo;
-                datalo = datahi;
-                datahi = t;
-            }
-            if (base != datalo) {
-                tcg_out_modrm_sib_offset(s, movop + seg, datalo,
-                                         base, index, 0, ofs);
-                tcg_out_modrm_sib_offset(s, movop + seg, datahi,
-                                         base, index, 0, ofs + 4);
-            } else {
-                tcg_out_modrm_sib_offset(s, movop + seg, datahi,
-                                         base, index, 0, ofs + 4);
-                tcg_out_modrm_sib_offset(s, movop + seg, datalo,
-                                         base, index, 0, ofs);
-            }
-            if (bswap) {
-                tcg_out_bswap32(s, datalo);
-                tcg_out_bswap32(s, datahi);
-            }
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and
-   EAX. It will be useful once fixed registers globals are less
-   common. */
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
-{
-    TCGReg datalo, datahi, addrlo;
-    TCGReg addrhi __attribute__((unused));
-    TCGMemOpIdx oi;
-    TCGMemOp opc;
-#if defined(CONFIG_SOFTMMU)
-    int mem_index;
-    tcg_insn_unit *label_ptr[2];
-#endif
-
-    datalo = *args++;
-    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
-    addrlo = *args++;
-    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-
-#if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-
-    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
-                     label_ptr, offsetof(CPUTLBEntry, addr_read));
-
-    /* TLB Hit.  */
-    tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc);
-
-    /* Record the current context of a load into ldst label */
-    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
-                        s->code_ptr, label_ptr);
-#else
-    {
-        int32_t offset = GUEST_BASE;
-        TCGReg base = addrlo;
-        int index = -1;
-        int seg = 0;
-
-        /* For a 32-bit guest, the high 32 bits may contain garbage.
-           We can do this with the ADDR32 prefix if we're not using
-           a guest base, or when using segmentation.  Otherwise we
-           need to zero-extend manually.  */
-        if (GUEST_BASE == 0 || guest_base_flags) {
-            seg = guest_base_flags;
-            offset = 0;
-            if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
-                seg |= P_ADDR32;
-            }
-        } else if (TCG_TARGET_REG_BITS == 64) {
-            if (TARGET_LONG_BITS == 32) {
-                tcg_out_ext32u(s, TCG_REG_L0, base);
-                base = TCG_REG_L0;
-            }
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
-                index = TCG_REG_L1;
-                offset = 0;
-            }
-        }
-
-        tcg_out_qemu_ld_direct(s, datalo, datahi,
-                               base, index, offset, seg, opc);
-    }
-#endif
-}
-
-static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
-                                   TCGReg base, intptr_t ofs, int seg,
-                                   TCGMemOp memop)
-{
-    /* ??? Ideally we wouldn't need a scratch register.  For user-only,
-       we could perform the bswap twice to restore the original value
-       instead of moving to the scratch.  But as it is, the L constraint
-       means that TCG_REG_L0 is definitely free here.  */
-    const TCGReg scratch = TCG_REG_L0;
-    const TCGMemOp real_bswap = memop & MO_BSWAP;
-    TCGMemOp bswap = real_bswap;
-    int movop = OPC_MOVL_EvGv;
-
-    if (have_movbe && real_bswap) {
-        bswap = 0;
-        movop = OPC_MOVBE_MyGy;
-    }
-
-    switch (memop & MO_SIZE) {
-    case MO_8:
-        /* In 32-bit mode, 8-bit stores can only happen from [abcd]x.
-           Use the scratch register if necessary.  */
-        if (TCG_TARGET_REG_BITS == 32 && datalo >= 4) {
-            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
-            datalo = scratch;
-        }
-        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
-                             datalo, base, ofs);
-        break;
-    case MO_16:
-        if (bswap) {
-            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
-            tcg_out_rolw_8(s, scratch);
-            datalo = scratch;
-        }
-        tcg_out_modrm_offset(s, movop + P_DATA16 + seg, datalo, base, ofs);
-        break;
-    case MO_32:
-        if (bswap) {
-            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
-            tcg_out_bswap32(s, scratch);
-            datalo = scratch;
-        }
-        tcg_out_modrm_offset(s, movop + seg, datalo, base, ofs);
-        break;
-    case MO_64:
-        if (TCG_TARGET_REG_BITS == 64) {
-            if (bswap) {
-                tcg_out_mov(s, TCG_TYPE_I64, scratch, datalo);
-                tcg_out_bswap64(s, scratch);
-                datalo = scratch;
-            }
-            tcg_out_modrm_offset(s, movop + P_REXW + seg, datalo, base, ofs);
-        } else if (bswap) {
-            tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi);
-            tcg_out_bswap32(s, scratch);
-            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs);
-            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
-            tcg_out_bswap32(s, scratch);
-            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs+4);
-        } else {
-            if (real_bswap) {
-                int t = datalo;
-                datalo = datahi;
-                datahi = t;
-            }
-            tcg_out_modrm_offset(s, movop + seg, datalo, base, ofs);
-            tcg_out_modrm_offset(s, movop + seg, datahi, base, ofs+4);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
-{
-    TCGReg datalo, datahi, addrlo;
-    TCGReg addrhi __attribute__((unused));
-    TCGMemOpIdx oi;
-    TCGMemOp opc;
-#if defined(CONFIG_SOFTMMU)
-    int mem_index;
-    tcg_insn_unit *label_ptr[2];
-#endif
-
-    datalo = *args++;
-    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
-    addrlo = *args++;
-    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-
-#if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-
-    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
-                     label_ptr, offsetof(CPUTLBEntry, addr_write));
-
-    /* TLB Hit.  */
-    tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, 0, 0, opc);
-
-    /* Record the current context of a store into ldst label */
-    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
-                        s->code_ptr, label_ptr);
-#else
-    {
-        int32_t offset = GUEST_BASE;
-        TCGReg base = addrlo;
-        int seg = 0;
-
-        /* See comment in tcg_out_qemu_ld re zero-extension of addrlo.  */
-        if (GUEST_BASE == 0 || guest_base_flags) {
-            seg = guest_base_flags;
-            offset = 0;
-            if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
-                seg |= P_ADDR32;
-            }
-        } else if (TCG_TARGET_REG_BITS == 64) {
-            /* ??? Note that we can't use the same SIB addressing scheme
-               as for loads, since we require L0 free for bswap.  */
-            if (offset != GUEST_BASE) {
-                if (TARGET_LONG_BITS == 32) {
-                    tcg_out_ext32u(s, TCG_REG_L0, base);
-                    base = TCG_REG_L0;
-                }
-                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
-                tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
-                base = TCG_REG_L1;
-                offset = 0;
-            } else if (TARGET_LONG_BITS == 32) {
-                tcg_out_ext32u(s, TCG_REG_L1, base);
-                base = TCG_REG_L1;
-            }
-        }
-
-        tcg_out_qemu_st_direct(s, datalo, datahi, base, offset, seg, opc);
-    }
-#endif
-}
-
-static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                              const TCGArg *args, const int *const_args)
-{
-    int c, vexop, rexw = 0;
-
-#if TCG_TARGET_REG_BITS == 64
-# define OP_32_64(x) \
-        case glue(glue(INDEX_op_, x), _i64): \
-            rexw = P_REXW; /* FALLTHRU */    \
-        case glue(glue(INDEX_op_, x), _i32)
-#else
-# define OP_32_64(x) \
-        case glue(glue(INDEX_op_, x), _i32)
-#endif
-
-    switch(opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, args[0]);
-        tcg_out_jmp(s, tb_ret_addr);
-        break;
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* direct jump method */
-            tcg_out8(s, OPC_JMP_long); /* jmp im */
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            tcg_out32(s, 0);
-        } else {
-            /* indirect jump method */
-            tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
-                                 (intptr_t)(s->tb_next + args[0]));
-        }
-        s->tb_next_offset[args[0]] = tcg_current_code_size(s);
-        break;
-    case INDEX_op_br:
-        tcg_out_jxx(s, JCC_JMP, args[0], 0);
-        break;
-    OP_32_64(ld8u):
-        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
-        tcg_out_modrm_offset(s, OPC_MOVZBL, args[0], args[1], args[2]);
-        break;
-    OP_32_64(ld8s):
-        tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, args[0], args[1], args[2]);
-        break;
-    OP_32_64(ld16u):
-        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
-        tcg_out_modrm_offset(s, OPC_MOVZWL, args[0], args[1], args[2]);
-        break;
-    OP_32_64(ld16s):
-        tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, args[0], args[1], args[2]);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_ld32u_i64:
-#endif
-    case INDEX_op_ld_i32:
-        tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
-        break;
-
-    OP_32_64(st8):
-        if (const_args[0]) {
-            tcg_out_modrm_offset(s, OPC_MOVB_EvIz,
-                                 0, args[1], args[2]);
-            tcg_out8(s, args[0]);
-        } else {
-            tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
-                                 args[0], args[1], args[2]);
-        }
-        break;
-    OP_32_64(st16):
-        if (const_args[0]) {
-            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16,
-                                 0, args[1], args[2]);
-            tcg_out16(s, args[0]);
-        } else {
-            tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
-                                 args[0], args[1], args[2]);
-        }
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_st32_i64:
-#endif
-    case INDEX_op_st_i32:
-        if (const_args[0]) {
-            tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, args[1], args[2]);
-            tcg_out32(s, args[0]);
-        } else {
-            tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
-        }
-        break;
-
-    OP_32_64(add):
-        /* For 3-operand addition, use LEA.  */
-        if (args[0] != args[1]) {
-            TCGArg a0 = args[0], a1 = args[1], a2 = args[2], c3 = 0;
-
-            if (const_args[2]) {
-                c3 = a2, a2 = -1;
-            } else if (a0 == a2) {
-                /* Watch out for dest = src + dest, since we've removed
-                   the matching constraint on the add.  */
-                tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
-                break;
-            }
-
-            tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3);
-            break;
-        }
-        c = ARITH_ADD;
-        goto gen_arith;
-    OP_32_64(sub):
-        c = ARITH_SUB;
-        goto gen_arith;
-    OP_32_64(and):
-        c = ARITH_AND;
-        goto gen_arith;
-    OP_32_64(or):
-        c = ARITH_OR;
-        goto gen_arith;
-    OP_32_64(xor):
-        c = ARITH_XOR;
-        goto gen_arith;
-    gen_arith:
-        if (const_args[2]) {
-            tgen_arithi(s, c + rexw, args[0], args[2], 0);
-        } else {
-            tgen_arithr(s, c + rexw, args[0], args[2]);
-        }
-        break;
-
-    OP_32_64(andc):
-        if (const_args[2]) {
-            tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32,
-                        args[0], args[1]);
-            tgen_arithi(s, ARITH_AND + rexw, args[0], ~args[2], 0);
-        } else {
-            tcg_out_vex_modrm(s, OPC_ANDN + rexw, args[0], args[2], args[1]);
-        }
-        break;
-
-    OP_32_64(mul):
-        if (const_args[2]) {
-            int32_t val;
-            val = args[2];
-            if (val == (int8_t)val) {
-                tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, args[0], args[0]);
-                tcg_out8(s, val);
-            } else {
-                tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, args[0], args[0]);
-                tcg_out32(s, val);
-            }
-        } else {
-            tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, args[0], args[2]);
-        }
-        break;
-
-    OP_32_64(div2):
-        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]);
-        break;
-    OP_32_64(divu2):
-        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]);
-        break;
-
-    OP_32_64(shl):
-        c = SHIFT_SHL;
-        vexop = OPC_SHLX;
-        goto gen_shift_maybe_vex;
-    OP_32_64(shr):
-        c = SHIFT_SHR;
-        vexop = OPC_SHRX;
-        goto gen_shift_maybe_vex;
-    OP_32_64(sar):
-        c = SHIFT_SAR;
-        vexop = OPC_SARX;
-        goto gen_shift_maybe_vex;
-    OP_32_64(rotl):
-        c = SHIFT_ROL;
-        goto gen_shift;
-    OP_32_64(rotr):
-        c = SHIFT_ROR;
-        goto gen_shift;
-    gen_shift_maybe_vex:
-        if (have_bmi2 && !const_args[2]) {
-            tcg_out_vex_modrm(s, vexop + rexw, args[0], args[2], args[1]);
-            break;
-        }
-        /* FALLTHRU */
-    gen_shift:
-        if (const_args[2]) {
-            tcg_out_shifti(s, c + rexw, args[0], args[2]);
-        } else {
-            tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, args[0]);
-        }
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_brcond32(s, args[2], args[0], args[1], const_args[1],
-                         args[3], 0);
-        break;
-    case INDEX_op_setcond_i32:
-        tcg_out_setcond32(s, args[3], args[0], args[1],
-                          args[2], const_args[2]);
-        break;
-    case INDEX_op_movcond_i32:
-        tcg_out_movcond32(s, args[5], args[0], args[1],
-                          args[2], const_args[2], args[3]);
-        break;
-
-    OP_32_64(bswap16):
-        tcg_out_rolw_8(s, args[0]);
-        break;
-    OP_32_64(bswap32):
-        tcg_out_bswap32(s, args[0]);
-        break;
-
-    OP_32_64(neg):
-        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, args[0]);
-        break;
-    OP_32_64(not):
-        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, args[0]);
-        break;
-
-    OP_32_64(ext8s):
-        tcg_out_ext8s(s, args[0], args[1], rexw);
-        break;
-    OP_32_64(ext16s):
-        tcg_out_ext16s(s, args[0], args[1], rexw);
-        break;
-    OP_32_64(ext8u):
-        tcg_out_ext8u(s, args[0], args[1]);
-        break;
-    OP_32_64(ext16u):
-        tcg_out_ext16u(s, args[0], args[1]);
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_qemu_ld(s, args, 0);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, args, 1);
-        break;
-    case INDEX_op_qemu_st_i32:
-        tcg_out_qemu_st(s, args, 0);
-        break;
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args, 1);
-        break;
-
-    OP_32_64(mulu2):
-        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
-        break;
-    OP_32_64(muls2):
-        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]);
-        break;
-    OP_32_64(add2):
-        if (const_args[4]) {
-            tgen_arithi(s, ARITH_ADD + rexw, args[0], args[4], 1);
-        } else {
-            tgen_arithr(s, ARITH_ADD + rexw, args[0], args[4]);
-        }
-        if (const_args[5]) {
-            tgen_arithi(s, ARITH_ADC + rexw, args[1], args[5], 1);
-        } else {
-            tgen_arithr(s, ARITH_ADC + rexw, args[1], args[5]);
-        }
-        break;
-    OP_32_64(sub2):
-        if (const_args[4]) {
-            tgen_arithi(s, ARITH_SUB + rexw, args[0], args[4], 1);
-        } else {
-            tgen_arithr(s, ARITH_SUB + rexw, args[0], args[4]);
-        }
-        if (const_args[5]) {
-            tgen_arithi(s, ARITH_SBB + rexw, args[1], args[5], 1);
-        } else {
-            tgen_arithr(s, ARITH_SBB + rexw, args[1], args[5]);
-        }
-        break;
-
-#if TCG_TARGET_REG_BITS == 32
-    case INDEX_op_brcond2_i32:
-        tcg_out_brcond2(s, args, const_args, 0);
-        break;
-    case INDEX_op_setcond2_i32:
-        tcg_out_setcond2(s, args, const_args);
-        break;
-#else /* TCG_TARGET_REG_BITS == 64 */
-    case INDEX_op_ld32s_i64:
-        tcg_out_modrm_offset(s, OPC_MOVSLQ, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld_i64:
-        tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i64:
-        if (const_args[0]) {
-            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW,
-                                 0, args[1], args[2]);
-            tcg_out32(s, args[0]);
-        } else {
-            tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
-        }
-        break;
-
-    case INDEX_op_brcond_i64:
-        tcg_out_brcond64(s, args[2], args[0], args[1], const_args[1],
-                         args[3], 0);
-        break;
-    case INDEX_op_setcond_i64:
-        tcg_out_setcond64(s, args[3], args[0], args[1],
-                          args[2], const_args[2]);
-        break;
-    case INDEX_op_movcond_i64:
-        tcg_out_movcond64(s, args[5], args[0], args[1],
-                          args[2], const_args[2], args[3]);
-        break;
-
-    case INDEX_op_bswap64_i64:
-        tcg_out_bswap64(s, args[0]);
-        break;
-    case INDEX_op_ext32u_i64:
-        tcg_out_ext32u(s, args[0], args[1]);
-        break;
-    case INDEX_op_ext32s_i64:
-        tcg_out_ext32s(s, args[0], args[1]);
-        break;
-#endif
-
-    OP_32_64(deposit):
-        if (args[3] == 0 && args[4] == 8) {
-            /* load bits 0..7 */
-            tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM,
-                          args[2], args[0]);
-        } else if (args[3] == 8 && args[4] == 8) {
-            /* load bits 8..15 */
-            tcg_out_modrm(s, OPC_MOVB_EvGv, args[2], args[0] + 4);
-        } else if (args[3] == 0 && args[4] == 16) {
-            /* load bits 0..15 */
-            tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, args[2], args[0]);
-        } else {
-            tcg_abort();
-        }
-        break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-
-#undef OP_32_64
-}
-
-static const TCGTargetOpDef x86_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "qi", "r" } },
-    { INDEX_op_st16_i32, { "ri", "r" } },
-    { INDEX_op_st_i32, { "ri", "r" } },
-
-    { INDEX_op_add_i32, { "r", "r", "ri" } },
-    { INDEX_op_sub_i32, { "r", "0", "ri" } },
-    { INDEX_op_mul_i32, { "r", "0", "ri" } },
-    { INDEX_op_div2_i32, { "a", "d", "0", "1", "r" } },
-    { INDEX_op_divu2_i32, { "a", "d", "0", "1", "r" } },
-    { INDEX_op_and_i32, { "r", "0", "ri" } },
-    { INDEX_op_or_i32, { "r", "0", "ri" } },
-    { INDEX_op_xor_i32, { "r", "0", "ri" } },
-    { INDEX_op_andc_i32, { "r", "r", "ri" } },
-
-    { INDEX_op_shl_i32, { "r", "0", "Ci" } },
-    { INDEX_op_shr_i32, { "r", "0", "Ci" } },
-    { INDEX_op_sar_i32, { "r", "0", "Ci" } },
-    { INDEX_op_rotl_i32, { "r", "0", "ci" } },
-    { INDEX_op_rotr_i32, { "r", "0", "ci" } },
-
-    { INDEX_op_brcond_i32, { "r", "ri" } },
-
-    { INDEX_op_bswap16_i32, { "r", "0" } },
-    { INDEX_op_bswap32_i32, { "r", "0" } },
-
-    { INDEX_op_neg_i32, { "r", "0" } },
-
-    { INDEX_op_not_i32, { "r", "0" } },
-
-    { INDEX_op_ext8s_i32, { "r", "q" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-    { INDEX_op_ext8u_i32, { "r", "q" } },
-    { INDEX_op_ext16u_i32, { "r", "r" } },
-
-    { INDEX_op_setcond_i32, { "q", "r", "ri" } },
-
-    { INDEX_op_deposit_i32, { "Q", "0", "Q" } },
-    { INDEX_op_movcond_i32, { "r", "r", "ri", "r", "0" } },
-
-    { INDEX_op_mulu2_i32, { "a", "d", "a", "r" } },
-    { INDEX_op_muls2_i32, { "a", "d", "a", "r" } },
-    { INDEX_op_add2_i32, { "r", "r", "0", "1", "ri", "ri" } },
-    { INDEX_op_sub2_i32, { "r", "r", "0", "1", "ri", "ri" } },
-
-#if TCG_TARGET_REG_BITS == 32
-    { INDEX_op_brcond2_i32, { "r", "r", "ri", "ri" } },
-    { INDEX_op_setcond2_i32, { "r", "r", "r", "ri", "ri" } },
-#else
-    { INDEX_op_ld8u_i64, { "r", "r" } },
-    { INDEX_op_ld8s_i64, { "r", "r" } },
-    { INDEX_op_ld16u_i64, { "r", "r" } },
-    { INDEX_op_ld16s_i64, { "r", "r" } },
-    { INDEX_op_ld32u_i64, { "r", "r" } },
-    { INDEX_op_ld32s_i64, { "r", "r" } },
-    { INDEX_op_ld_i64, { "r", "r" } },
-    { INDEX_op_st8_i64, { "ri", "r" } },
-    { INDEX_op_st16_i64, { "ri", "r" } },
-    { INDEX_op_st32_i64, { "ri", "r" } },
-    { INDEX_op_st_i64, { "re", "r" } },
-
-    { INDEX_op_add_i64, { "r", "r", "re" } },
-    { INDEX_op_mul_i64, { "r", "0", "re" } },
-    { INDEX_op_div2_i64, { "a", "d", "0", "1", "r" } },
-    { INDEX_op_divu2_i64, { "a", "d", "0", "1", "r" } },
-    { INDEX_op_sub_i64, { "r", "0", "re" } },
-    { INDEX_op_and_i64, { "r", "0", "reZ" } },
-    { INDEX_op_or_i64, { "r", "0", "re" } },
-    { INDEX_op_xor_i64, { "r", "0", "re" } },
-    { INDEX_op_andc_i64, { "r", "r", "rI" } },
-
-    { INDEX_op_shl_i64, { "r", "0", "Ci" } },
-    { INDEX_op_shr_i64, { "r", "0", "Ci" } },
-    { INDEX_op_sar_i64, { "r", "0", "Ci" } },
-    { INDEX_op_rotl_i64, { "r", "0", "ci" } },
-    { INDEX_op_rotr_i64, { "r", "0", "ci" } },
-
-    { INDEX_op_brcond_i64, { "r", "re" } },
-    { INDEX_op_setcond_i64, { "r", "r", "re" } },
-
-    { INDEX_op_bswap16_i64, { "r", "0" } },
-    { INDEX_op_bswap32_i64, { "r", "0" } },
-    { INDEX_op_bswap64_i64, { "r", "0" } },
-    { INDEX_op_neg_i64, { "r", "0" } },
-    { INDEX_op_not_i64, { "r", "0" } },
-
-    { INDEX_op_ext8s_i64, { "r", "r" } },
-    { INDEX_op_ext16s_i64, { "r", "r" } },
-    { INDEX_op_ext32s_i64, { "r", "r" } },
-    { INDEX_op_ext8u_i64, { "r", "r" } },
-    { INDEX_op_ext16u_i64, { "r", "r" } },
-    { INDEX_op_ext32u_i64, { "r", "r" } },
-
-    { INDEX_op_deposit_i64, { "Q", "0", "Q" } },
-    { INDEX_op_movcond_i64, { "r", "r", "re", "r", "0" } },
-
-    { INDEX_op_mulu2_i64, { "a", "d", "a", "r" } },
-    { INDEX_op_muls2_i64, { "a", "d", "a", "r" } },
-    { INDEX_op_add2_i64, { "r", "r", "0", "1", "re", "re" } },
-    { INDEX_op_sub2_i64, { "r", "r", "0", "1", "re", "re" } },
-#endif
-
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_qemu_ld_i32, { "r", "L" } },
-    { INDEX_op_qemu_st_i32, { "L", "L" } },
-    { INDEX_op_qemu_ld_i64, { "r", "L" } },
-    { INDEX_op_qemu_st_i64, { "L", "L" } },
-#elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
-    { INDEX_op_qemu_ld_i32, { "r", "L" } },
-    { INDEX_op_qemu_st_i32, { "L", "L" } },
-    { INDEX_op_qemu_ld_i64, { "r", "r", "L" } },
-    { INDEX_op_qemu_st_i64, { "L", "L", "L" } },
-#else
-    { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
-    { INDEX_op_qemu_st_i32, { "L", "L", "L" } },
-    { INDEX_op_qemu_ld_i64, { "r", "r", "L", "L" } },
-    { INDEX_op_qemu_st_i64, { "L", "L", "L", "L" } },
-#endif
-    { -1 },
-};
-
-static int tcg_target_callee_save_regs[] = {
-#if TCG_TARGET_REG_BITS == 64
-    TCG_REG_RBP,
-    TCG_REG_RBX,
-#if defined(_WIN64)
-    TCG_REG_RDI,
-    TCG_REG_RSI,
-#endif
-    TCG_REG_R12,
-    TCG_REG_R13,
-    TCG_REG_R14, /* Currently used for the global env. */
-    TCG_REG_R15,
-#else
-    TCG_REG_EBP, /* Currently used for the global env. */
-    TCG_REG_EBX,
-    TCG_REG_ESI,
-    TCG_REG_EDI,
-#endif
-};
-
-/* Compute frame size via macros, to share between tcg_target_qemu_prologue
-   and tcg_register_jit.  */
-
-#define PUSH_SIZE \
-    ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \
-     * (TCG_TARGET_REG_BITS / 8))
-
-#define FRAME_SIZE \
-    ((PUSH_SIZE \
-      + TCG_STATIC_CALL_ARGS_SIZE \
-      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
-      + TCG_TARGET_STACK_ALIGN - 1) \
-     & ~(TCG_TARGET_STACK_ALIGN - 1))
-
-/* Generate global QEMU prologue and epilogue code */
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int i, stack_addend;
-
-    /* TB prologue */
-
-    /* Reserve some stack space, also for TCG temps.  */
-    stack_addend = FRAME_SIZE - PUSH_SIZE;
-    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-    /* Save all callee saved registers.  */
-    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
-        tcg_out_push(s, tcg_target_callee_save_regs[i]);
-    }
-
-#if TCG_TARGET_REG_BITS == 32
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP,
-               (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4);
-    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
-    /* jmp *tb.  */
-    tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP,
-		         (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4
-			 + stack_addend);
-#else
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
-    /* jmp *tb.  */
-    tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
-#endif
-
-    /* TB epilogue */
-    tb_ret_addr = s->code_ptr;
-
-    tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend);
-
-    for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
-        tcg_out_pop(s, tcg_target_callee_save_regs[i]);
-    }
-    tcg_out_opc(s, OPC_RET, 0, 0, 0);
-
-#if !defined(CONFIG_SOFTMMU)
-    /* Try to set up a segment register to point to GUEST_BASE.  */
-    if (GUEST_BASE) {
-        setup_guest_base_seg();
-    }
-#endif
-}
-
-static void tcg_target_init(TCGContext *s)
-{
-#ifdef CONFIG_CPUID_H
-    unsigned a, b, c, d;
-    int max = __get_cpuid_max(0, 0);
-
-    if (max >= 1) {
-        __cpuid(1, a, b, c, d);
-#ifndef have_cmov
-        /* For 32-bit, 99% certainty that we're running on hardware that
-           supports cmov, but we still need to check.  In case cmov is not
-           available, we'll use a small forward branch.  */
-        have_cmov = (d & bit_CMOV) != 0;
-#endif
-#ifndef have_movbe
-        /* MOVBE is only available on Intel Atom and Haswell CPUs, so we
-           need to probe for it.  */
-        have_movbe = (c & bit_MOVBE) != 0;
-#endif
-    }
-
-    if (max >= 7) {
-        /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs.  */
-        __cpuid_count(7, 0, a, b, c, d);
-#ifdef bit_BMI
-        have_bmi1 = (b & bit_BMI) != 0;
-#endif
-#ifndef have_bmi2
-        have_bmi2 = (b & bit_BMI2) != 0;
-#endif
-    }
-#endif
-
-    if (TCG_TARGET_REG_BITS == 64) {
-        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
-        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffff);
-    } else {
-        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xff);
-    }
-
-    tcg_regset_clear(tcg_target_call_clobber_regs);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX);
-    if (TCG_TARGET_REG_BITS == 64) {
-#if !defined(_WIN64)
-        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI);
-        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI);
-#endif
-        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
-        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
-        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
-        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
-    }
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
-
-    tcg_add_target_add_op_defs(x86_op_defs);
-}
-
-typedef struct {
-    DebugFrameHeader h;
-    uint8_t fde_def_cfa[4];
-    uint8_t fde_reg_ofs[14];
-} DebugFrame;
-
-/* We're expecting a 2 byte uleb128 encoded value.  */
-QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
-
-#if !defined(__ELF__)
-    /* Host machine without ELF. */
-#elif TCG_TARGET_REG_BITS == 64
-#define ELF_HOST_MACHINE EM_X86_64
-static const DebugFrame debug_frame = {
-    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .h.cie.id = -1,
-    .h.cie.version = 1,
-    .h.cie.code_align = 1,
-    .h.cie.data_align = 0x78,             /* sleb128 -8 */
-    .h.cie.return_column = 16,
-
-    /* Total FDE size does not include the "len" member.  */
-    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
-
-    .fde_def_cfa = {
-        12, 7,                          /* DW_CFA_def_cfa %rsp, ... */
-        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
-        (FRAME_SIZE >> 7)
-    },
-    .fde_reg_ofs = {
-        0x90, 1,                        /* DW_CFA_offset, %rip, -8 */
-        /* The following ordering must match tcg_target_callee_save_regs.  */
-        0x86, 2,                        /* DW_CFA_offset, %rbp, -16 */
-        0x83, 3,                        /* DW_CFA_offset, %rbx, -24 */
-        0x8c, 4,                        /* DW_CFA_offset, %r12, -32 */
-        0x8d, 5,                        /* DW_CFA_offset, %r13, -40 */
-        0x8e, 6,                        /* DW_CFA_offset, %r14, -48 */
-        0x8f, 7,                        /* DW_CFA_offset, %r15, -56 */
-    }
-};
-#else
-#define ELF_HOST_MACHINE EM_386
-static const DebugFrame debug_frame = {
-    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .h.cie.id = -1,
-    .h.cie.version = 1,
-    .h.cie.code_align = 1,
-    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
-    .h.cie.return_column = 8,
-
-    /* Total FDE size does not include the "len" member.  */
-    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
-
-    .fde_def_cfa = {
-        12, 4,                          /* DW_CFA_def_cfa %esp, ... */
-        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
-        (FRAME_SIZE >> 7)
-    },
-    .fde_reg_ofs = {
-        0x88, 1,                        /* DW_CFA_offset, %eip, -4 */
-        /* The following ordering must match tcg_target_callee_save_regs.  */
-        0x85, 2,                        /* DW_CFA_offset, %ebp, -8 */
-        0x83, 3,                        /* DW_CFA_offset, %ebx, -12 */
-        0x86, 4,                        /* DW_CFA_offset, %esi, -16 */
-        0x87, 5,                        /* DW_CFA_offset, %edi, -20 */
-    }
-};
-#endif
-
-#if defined(ELF_HOST_MACHINE)
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
-#endif
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index 7a9980e..524cfc6 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -21,10 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_I386 
-#define TCG_TARGET_I386 1
+
+#ifndef I386_TCG_TARGET_H
+#define I386_TCG_TARGET_H
 
 #define TCG_TARGET_INSN_UNIT_SIZE  1
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 31
 
 #ifdef __x86_64__
 # define TCG_TARGET_REG_BITS  64
@@ -101,7 +103,8 @@
 #define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
 #define TCG_TARGET_HAS_ext8s_i64        1
diff --git a/tcg/i386/tcg-target.inc.c b/tcg/i386/tcg-target.inc.c
new file mode 100644
index 0000000..6f8cdca
--- /dev/null
+++ b/tcg/i386/tcg-target.inc.c
@@ -0,0 +1,2496 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg-be-ldst.h"
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+#if TCG_TARGET_REG_BITS == 64
+    "%rax", "%rcx", "%rdx", "%rbx", "%rsp", "%rbp", "%rsi", "%rdi",
+    "%r8",  "%r9",  "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
+#else
+    "%eax", "%ecx", "%edx", "%ebx", "%esp", "%ebp", "%esi", "%edi",
+#endif
+};
+#endif
+
+static const int tcg_target_reg_alloc_order[] = {
+#if TCG_TARGET_REG_BITS == 64
+    TCG_REG_RBP,
+    TCG_REG_RBX,
+    TCG_REG_R12,
+    TCG_REG_R13,
+    TCG_REG_R14,
+    TCG_REG_R15,
+    TCG_REG_R10,
+    TCG_REG_R11,
+    TCG_REG_R9,
+    TCG_REG_R8,
+    TCG_REG_RCX,
+    TCG_REG_RDX,
+    TCG_REG_RSI,
+    TCG_REG_RDI,
+    TCG_REG_RAX,
+#else
+    TCG_REG_EBX,
+    TCG_REG_ESI,
+    TCG_REG_EDI,
+    TCG_REG_EBP,
+    TCG_REG_ECX,
+    TCG_REG_EDX,
+    TCG_REG_EAX,
+#endif
+};
+
+static const int tcg_target_call_iarg_regs[] = {
+#if TCG_TARGET_REG_BITS == 64
+#if defined(_WIN64)
+    TCG_REG_RCX,
+    TCG_REG_RDX,
+#else
+    TCG_REG_RDI,
+    TCG_REG_RSI,
+    TCG_REG_RDX,
+    TCG_REG_RCX,
+#endif
+    TCG_REG_R8,
+    TCG_REG_R9,
+#else
+    /* 32 bit mode uses stack based calling convention (GCC default). */
+#endif
+};
+
+static const int tcg_target_call_oarg_regs[] = {
+    TCG_REG_EAX,
+#if TCG_TARGET_REG_BITS == 32
+    TCG_REG_EDX
+#endif
+};
+
+/* Constants we accept.  */
+#define TCG_CT_CONST_S32 0x100
+#define TCG_CT_CONST_U32 0x200
+#define TCG_CT_CONST_I32 0x400
+
+/* Registers used with L constraint, which are the first argument 
+   registers on x86_64, and two random call clobbered registers on
+   i386. */
+#if TCG_TARGET_REG_BITS == 64
+# define TCG_REG_L0 tcg_target_call_iarg_regs[0]
+# define TCG_REG_L1 tcg_target_call_iarg_regs[1]
+#else
+# define TCG_REG_L0 TCG_REG_EAX
+# define TCG_REG_L1 TCG_REG_EDX
+#endif
+
+/* The host compiler should supply <cpuid.h> to enable runtime features
+   detection, as we're not going to go so far as our own inline assembly.
+   If not available, default values will be assumed.  */
+#if defined(CONFIG_CPUID_H)
+#include <cpuid.h>
+#endif
+
+/* For 32-bit, we are going to attempt to determine at runtime whether cmov
+   is available.  */
+#if TCG_TARGET_REG_BITS == 64
+# define have_cmov 1
+#elif defined(CONFIG_CPUID_H) && defined(bit_CMOV)
+static bool have_cmov;
+#else
+# define have_cmov 0
+#endif
+
+/* If bit_MOVBE is defined in cpuid.h (added in GCC version 4.6), we are
+   going to attempt to determine at runtime whether movbe is available.  */
+#if defined(CONFIG_CPUID_H) && defined(bit_MOVBE)
+static bool have_movbe;
+#else
+# define have_movbe 0
+#endif
+
+/* We need this symbol in tcg-target.h, and we can't properly conditionalize
+   it there.  Therefore we always define the variable.  */
+bool have_bmi1;
+
+#if defined(CONFIG_CPUID_H) && defined(bit_BMI2)
+static bool have_bmi2;
+#else
+# define have_bmi2 0
+#endif
+
+static tcg_insn_unit *tb_ret_addr;
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    value += addend;
+    switch(type) {
+    case R_386_PC32:
+        value -= (uintptr_t)code_ptr;
+        if (value != (int32_t)value) {
+            tcg_abort();
+        }
+        tcg_patch32(code_ptr, value);
+        break;
+    case R_386_PC8:
+        value -= (uintptr_t)code_ptr;
+        if (value != (int8_t)value) {
+            tcg_abort();
+        }
+        tcg_patch8(code_ptr, value);
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch(ct_str[0]) {
+    case 'a':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_EAX);
+        break;
+    case 'b':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_EBX);
+        break;
+    case 'c':
+    case_c:
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_ECX);
+        break;
+    case 'd':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_EDX);
+        break;
+    case 'S':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_ESI);
+        break;
+    case 'D':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_EDI);
+        break;
+    case 'q':
+        ct->ct |= TCG_CT_REG;
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        } else {
+            tcg_regset_set32(ct->u.regs, 0, 0xf);
+        }
+        break;
+    case 'Q':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xf);
+        break;
+    case 'r':
+    case_r:
+        ct->ct |= TCG_CT_REG;
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        } else {
+            tcg_regset_set32(ct->u.regs, 0, 0xff);
+        }
+        break;
+    case 'C':
+        /* With SHRX et al, we need not use ECX as shift count register.  */
+        if (have_bmi2) {
+            goto case_r;
+        } else {
+            goto case_c;
+        }
+
+        /* qemu_ld/st address constraint */
+    case 'L':
+        ct->ct |= TCG_CT_REG;
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        } else {
+            tcg_regset_set32(ct->u.regs, 0, 0xff);
+        }
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L1);
+        break;
+
+    case 'e':
+        ct->ct |= TCG_CT_CONST_S32;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_U32;
+        break;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_I32;
+        break;
+
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
+        return 1;
+    }
+    if ((ct & TCG_CT_CONST_I32) && ~val == (int32_t)~val) {
+        return 1;
+    }
+    return 0;
+}
+
+#if TCG_TARGET_REG_BITS == 64
+# define LOWREGMASK(x)	((x) & 7)
+#else
+# define LOWREGMASK(x)	(x)
+#endif
+
+#define P_EXT		0x100		/* 0x0f opcode prefix */
+#define P_EXT38         0x200           /* 0x0f 0x38 opcode prefix */
+#define P_DATA16        0x400           /* 0x66 opcode prefix */
+#if TCG_TARGET_REG_BITS == 64
+# define P_ADDR32       0x800           /* 0x67 opcode prefix */
+# define P_REXW         0x1000          /* Set REX.W = 1 */
+# define P_REXB_R       0x2000          /* REG field as byte register */
+# define P_REXB_RM      0x4000          /* R/M field as byte register */
+# define P_GS           0x8000          /* gs segment override */
+#else
+# define P_ADDR32	0
+# define P_REXW		0
+# define P_REXB_R	0
+# define P_REXB_RM	0
+# define P_GS           0
+#endif
+#define P_SIMDF3        0x10000         /* 0xf3 opcode prefix */
+#define P_SIMDF2        0x20000         /* 0xf2 opcode prefix */
+
+#define OPC_ARITH_EvIz	(0x81)
+#define OPC_ARITH_EvIb	(0x83)
+#define OPC_ARITH_GvEv	(0x03)		/* ... plus (ARITH_FOO << 3) */
+#define OPC_ANDN        (0xf2 | P_EXT38)
+#define OPC_ADD_GvEv	(OPC_ARITH_GvEv | (ARITH_ADD << 3))
+#define OPC_BSWAP	(0xc8 | P_EXT)
+#define OPC_CALL_Jz	(0xe8)
+#define OPC_CMOVCC      (0x40 | P_EXT)  /* ... plus condition code */
+#define OPC_CMP_GvEv	(OPC_ARITH_GvEv | (ARITH_CMP << 3))
+#define OPC_DEC_r32	(0x48)
+#define OPC_IMUL_GvEv	(0xaf | P_EXT)
+#define OPC_IMUL_GvEvIb	(0x6b)
+#define OPC_IMUL_GvEvIz	(0x69)
+#define OPC_INC_r32	(0x40)
+#define OPC_JCC_long	(0x80 | P_EXT)	/* ... plus condition code */
+#define OPC_JCC_short	(0x70)		/* ... plus condition code */
+#define OPC_JMP_long	(0xe9)
+#define OPC_JMP_short	(0xeb)
+#define OPC_LEA         (0x8d)
+#define OPC_MOVB_EvGv	(0x88)		/* stores, more or less */
+#define OPC_MOVL_EvGv	(0x89)		/* stores, more or less */
+#define OPC_MOVL_GvEv	(0x8b)		/* loads, more or less */
+#define OPC_MOVB_EvIz   (0xc6)
+#define OPC_MOVL_EvIz	(0xc7)
+#define OPC_MOVL_Iv     (0xb8)
+#define OPC_MOVBE_GyMy  (0xf0 | P_EXT38)
+#define OPC_MOVBE_MyGy  (0xf1 | P_EXT38)
+#define OPC_MOVSBL	(0xbe | P_EXT)
+#define OPC_MOVSWL	(0xbf | P_EXT)
+#define OPC_MOVSLQ	(0x63 | P_REXW)
+#define OPC_MOVZBL	(0xb6 | P_EXT)
+#define OPC_MOVZWL	(0xb7 | P_EXT)
+#define OPC_POP_r32	(0x58)
+#define OPC_PUSH_r32	(0x50)
+#define OPC_PUSH_Iv	(0x68)
+#define OPC_PUSH_Ib	(0x6a)
+#define OPC_RET		(0xc3)
+#define OPC_SETCC	(0x90 | P_EXT | P_REXB_RM) /* ... plus cc */
+#define OPC_SHIFT_1	(0xd1)
+#define OPC_SHIFT_Ib	(0xc1)
+#define OPC_SHIFT_cl	(0xd3)
+#define OPC_SARX        (0xf7 | P_EXT38 | P_SIMDF3)
+#define OPC_SHLX        (0xf7 | P_EXT38 | P_DATA16)
+#define OPC_SHRX        (0xf7 | P_EXT38 | P_SIMDF2)
+#define OPC_TESTL	(0x85)
+#define OPC_XCHG_ax_r32	(0x90)
+
+#define OPC_GRP3_Ev	(0xf7)
+#define OPC_GRP5	(0xff)
+
+/* Group 1 opcode extensions for 0x80-0x83.
+   These are also used as modifiers for OPC_ARITH.  */
+#define ARITH_ADD 0
+#define ARITH_OR  1
+#define ARITH_ADC 2
+#define ARITH_SBB 3
+#define ARITH_AND 4
+#define ARITH_SUB 5
+#define ARITH_XOR 6
+#define ARITH_CMP 7
+
+/* Group 2 opcode extensions for 0xc0, 0xc1, 0xd0-0xd3.  */
+#define SHIFT_ROL 0
+#define SHIFT_ROR 1
+#define SHIFT_SHL 4
+#define SHIFT_SHR 5
+#define SHIFT_SAR 7
+
+/* Group 3 opcode extensions for 0xf6, 0xf7.  To be used with OPC_GRP3.  */
+#define EXT3_NOT   2
+#define EXT3_NEG   3
+#define EXT3_MUL   4
+#define EXT3_IMUL  5
+#define EXT3_DIV   6
+#define EXT3_IDIV  7
+
+/* Group 5 opcode extensions for 0xff.  To be used with OPC_GRP5.  */
+#define EXT5_INC_Ev	0
+#define EXT5_DEC_Ev	1
+#define EXT5_CALLN_Ev	2
+#define EXT5_JMPN_Ev	4
+
+/* Condition codes to be added to OPC_JCC_{long,short}.  */
+#define JCC_JMP (-1)
+#define JCC_JO  0x0
+#define JCC_JNO 0x1
+#define JCC_JB  0x2
+#define JCC_JAE 0x3
+#define JCC_JE  0x4
+#define JCC_JNE 0x5
+#define JCC_JBE 0x6
+#define JCC_JA  0x7
+#define JCC_JS  0x8
+#define JCC_JNS 0x9
+#define JCC_JP  0xa
+#define JCC_JNP 0xb
+#define JCC_JL  0xc
+#define JCC_JGE 0xd
+#define JCC_JLE 0xe
+#define JCC_JG  0xf
+
+static const uint8_t tcg_cond_to_jcc[] = {
+    [TCG_COND_EQ] = JCC_JE,
+    [TCG_COND_NE] = JCC_JNE,
+    [TCG_COND_LT] = JCC_JL,
+    [TCG_COND_GE] = JCC_JGE,
+    [TCG_COND_LE] = JCC_JLE,
+    [TCG_COND_GT] = JCC_JG,
+    [TCG_COND_LTU] = JCC_JB,
+    [TCG_COND_GEU] = JCC_JAE,
+    [TCG_COND_LEU] = JCC_JBE,
+    [TCG_COND_GTU] = JCC_JA,
+};
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
+{
+    int rex;
+
+    if (opc & P_GS) {
+        tcg_out8(s, 0x65);
+    }
+    if (opc & P_DATA16) {
+        /* We should never be asking for both 16 and 64-bit operation.  */
+        tcg_debug_assert((opc & P_REXW) == 0);
+        tcg_out8(s, 0x66);
+    }
+    if (opc & P_ADDR32) {
+        tcg_out8(s, 0x67);
+    }
+
+    rex = 0;
+    rex |= (opc & P_REXW) ? 0x8 : 0x0;  /* REX.W */
+    rex |= (r & 8) >> 1;                /* REX.R */
+    rex |= (x & 8) >> 2;                /* REX.X */
+    rex |= (rm & 8) >> 3;               /* REX.B */
+
+    /* P_REXB_{R,RM} indicates that the given register is the low byte.
+       For %[abcd]l we need no REX prefix, but for %{si,di,bp,sp}l we do,
+       as otherwise the encoding indicates %[abcd]h.  Note that the values
+       that are ORed in merely indicate that the REX byte must be present;
+       those bits get discarded in output.  */
+    rex |= opc & (r >= 4 ? P_REXB_R : 0);
+    rex |= opc & (rm >= 4 ? P_REXB_RM : 0);
+
+    if (rex) {
+        tcg_out8(s, (uint8_t)(rex | 0x40));
+    }
+
+    if (opc & (P_EXT | P_EXT38)) {
+        tcg_out8(s, 0x0f);
+        if (opc & P_EXT38) {
+            tcg_out8(s, 0x38);
+        }
+    }
+
+    tcg_out8(s, opc);
+}
+#else
+static void tcg_out_opc(TCGContext *s, int opc)
+{
+    if (opc & P_DATA16) {
+        tcg_out8(s, 0x66);
+    }
+    if (opc & (P_EXT | P_EXT38)) {
+        tcg_out8(s, 0x0f);
+        if (opc & P_EXT38) {
+            tcg_out8(s, 0x38);
+        }
+    }
+    tcg_out8(s, opc);
+}
+/* Discard the register arguments to tcg_out_opc early, so as not to penalize
+   the 32-bit compilation paths.  This method works with all versions of gcc,
+   whereas relying on optimization may not be able to exclude them.  */
+#define tcg_out_opc(s, opc, r, rm, x)  (tcg_out_opc)(s, opc)
+#endif
+
+static void tcg_out_modrm(TCGContext *s, int opc, int r, int rm)
+{
+    tcg_out_opc(s, opc, r, rm, 0);
+    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
+}
+
+static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm)
+{
+    int tmp;
+
+    if ((opc & (P_REXW | P_EXT | P_EXT38)) || (rm & 8)) {
+        /* Three byte VEX prefix.  */
+        tcg_out8(s, 0xc4);
+
+        /* VEX.m-mmmm */
+        if (opc & P_EXT38) {
+            tmp = 2;
+        } else if (opc & P_EXT) {
+            tmp = 1;
+        } else {
+            tcg_abort();
+        }
+        tmp |= 0x40;                       /* VEX.X */
+        tmp |= (r & 8 ? 0 : 0x80);         /* VEX.R */
+        tmp |= (rm & 8 ? 0 : 0x20);        /* VEX.B */
+        tcg_out8(s, tmp);
+
+        tmp = (opc & P_REXW ? 0x80 : 0);   /* VEX.W */
+    } else {
+        /* Two byte VEX prefix.  */
+        tcg_out8(s, 0xc5);
+
+        tmp = (r & 8 ? 0 : 0x80);          /* VEX.R */
+    }
+    /* VEX.pp */
+    if (opc & P_DATA16) {
+        tmp |= 1;                          /* 0x66 */
+    } else if (opc & P_SIMDF3) {
+        tmp |= 2;                          /* 0xf3 */
+    } else if (opc & P_SIMDF2) {
+        tmp |= 3;                          /* 0xf2 */
+    }
+    tmp |= (~v & 15) << 3;                 /* VEX.vvvv */
+    tcg_out8(s, tmp);
+    tcg_out8(s, opc);
+    tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
+}
+
+/* Output an opcode with a full "rm + (index<<shift) + offset" address mode.
+   We handle either RM and INDEX missing with a negative value.  In 64-bit
+   mode for absolute addresses, ~RM is the size of the immediate operand
+   that will follow the instruction.  */
+
+static void tcg_out_modrm_sib_offset(TCGContext *s, int opc, int r, int rm,
+                                     int index, int shift, intptr_t offset)
+{
+    int mod, len;
+
+    if (index < 0 && rm < 0) {
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* Try for a rip-relative addressing mode.  This has replaced
+               the 32-bit-mode absolute addressing encoding.  */
+            intptr_t pc = (intptr_t)s->code_ptr + 5 + ~rm;
+            intptr_t disp = offset - pc;
+            if (disp == (int32_t)disp) {
+                tcg_out_opc(s, opc, r, 0, 0);
+                tcg_out8(s, (LOWREGMASK(r) << 3) | 5);
+                tcg_out32(s, disp);
+                return;
+            }
+
+            /* Try for an absolute address encoding.  This requires the
+               use of the MODRM+SIB encoding and is therefore larger than
+               rip-relative addressing.  */
+            if (offset == (int32_t)offset) {
+                tcg_out_opc(s, opc, r, 0, 0);
+                tcg_out8(s, (LOWREGMASK(r) << 3) | 4);
+                tcg_out8(s, (4 << 3) | 5);
+                tcg_out32(s, offset);
+                return;
+            }
+
+            /* ??? The memory isn't directly addressable.  */
+            tcg_abort();
+        } else {
+            /* Absolute address.  */
+            tcg_out_opc(s, opc, r, 0, 0);
+            tcg_out8(s, (r << 3) | 5);
+            tcg_out32(s, offset);
+            return;
+        }
+    }
+
+    /* Find the length of the immediate addend.  Note that the encoding
+       that would be used for (%ebp) indicates absolute addressing.  */
+    if (rm < 0) {
+        mod = 0, len = 4, rm = 5;
+    } else if (offset == 0 && LOWREGMASK(rm) != TCG_REG_EBP) {
+        mod = 0, len = 0;
+    } else if (offset == (int8_t)offset) {
+        mod = 0x40, len = 1;
+    } else {
+        mod = 0x80, len = 4;
+    }
+
+    /* Use a single byte MODRM format if possible.  Note that the encoding
+       that would be used for %esp is the escape to the two byte form.  */
+    if (index < 0 && LOWREGMASK(rm) != TCG_REG_ESP) {
+        /* Single byte MODRM format.  */
+        tcg_out_opc(s, opc, r, rm, 0);
+        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | LOWREGMASK(rm));
+    } else {
+        /* Two byte MODRM+SIB format.  */
+
+        /* Note that the encoding that would place %esp into the index
+           field indicates no index register.  In 64-bit mode, the REX.X
+           bit counts, so %r12 can be used as the index.  */
+        if (index < 0) {
+            index = 4;
+        } else {
+            tcg_debug_assert(index != TCG_REG_ESP);
+        }
+
+        tcg_out_opc(s, opc, r, rm, index);
+        tcg_out8(s, mod | (LOWREGMASK(r) << 3) | 4);
+        tcg_out8(s, (shift << 6) | (LOWREGMASK(index) << 3) | LOWREGMASK(rm));
+    }
+
+    if (len == 1) {
+        tcg_out8(s, offset);
+    } else if (len == 4) {
+        tcg_out32(s, offset);
+    }
+}
+
+/* A simplification of the above with no index or shift.  */
+static inline void tcg_out_modrm_offset(TCGContext *s, int opc, int r,
+                                        int rm, intptr_t offset)
+{
+    tcg_out_modrm_sib_offset(s, opc, r, rm, -1, 0, offset);
+}
+
+/* Generate dest op= src.  Uses the same ARITH_* codes as tgen_arithi.  */
+static inline void tgen_arithr(TCGContext *s, int subop, int dest, int src)
+{
+    /* Propagate an opcode prefix, such as P_REXW.  */
+    int ext = subop & ~0x7;
+    subop &= 0x7;
+
+    tcg_out_modrm(s, OPC_ARITH_GvEv + (subop << 3) + ext, dest, src);
+}
+
+static inline void tcg_out_mov(TCGContext *s, TCGType type,
+                               TCGReg ret, TCGReg arg)
+{
+    if (arg != ret) {
+        int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
+        tcg_out_modrm(s, opc, ret, arg);
+    }
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type,
+                         TCGReg ret, tcg_target_long arg)
+{
+    tcg_target_long diff;
+
+    if (arg == 0) {
+        tgen_arithr(s, ARITH_XOR, ret, ret);
+        return;
+    }
+    if (arg == (uint32_t)arg || type == TCG_TYPE_I32) {
+        tcg_out_opc(s, OPC_MOVL_Iv + LOWREGMASK(ret), 0, ret, 0);
+        tcg_out32(s, arg);
+        return;
+    }
+    if (arg == (int32_t)arg) {
+        tcg_out_modrm(s, OPC_MOVL_EvIz + P_REXW, 0, ret);
+        tcg_out32(s, arg);
+        return;
+    }
+
+    /* Try a 7 byte pc-relative lea before the 10 byte movq.  */
+    diff = arg - ((uintptr_t)s->code_ptr + 7);
+    if (diff == (int32_t)diff) {
+        tcg_out_opc(s, OPC_LEA | P_REXW, ret, 0, 0);
+        tcg_out8(s, (LOWREGMASK(ret) << 3) | 5);
+        tcg_out32(s, diff);
+        return;
+    }
+
+    tcg_out_opc(s, OPC_MOVL_Iv + P_REXW + LOWREGMASK(ret), 0, ret, 0);
+    tcg_out64(s, arg);
+}
+
+static inline void tcg_out_pushi(TCGContext *s, tcg_target_long val)
+{
+    if (val == (int8_t)val) {
+        tcg_out_opc(s, OPC_PUSH_Ib, 0, 0, 0);
+        tcg_out8(s, val);
+    } else if (val == (int32_t)val) {
+        tcg_out_opc(s, OPC_PUSH_Iv, 0, 0, 0);
+        tcg_out32(s, val);
+    } else {
+        tcg_abort();
+    }
+}
+
+static inline void tcg_out_push(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_PUSH_r32 + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static inline void tcg_out_pop(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_POP_r32 + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
+                              TCGReg arg1, intptr_t arg2)
+{
+    int opc = OPC_MOVL_GvEv + (type == TCG_TYPE_I64 ? P_REXW : 0);
+    tcg_out_modrm_offset(s, opc, ret, arg1, arg2);
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    int opc = OPC_MOVL_EvGv + (type == TCG_TYPE_I64 ? P_REXW : 0);
+    tcg_out_modrm_offset(s, opc, arg, arg1, arg2);
+}
+
+static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                        TCGReg base, intptr_t ofs)
+{
+    int rexw = 0;
+    if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I64) {
+        if (val != (int32_t)val) {
+            return false;
+        }
+        rexw = P_REXW;
+    }
+    tcg_out_modrm_offset(s, OPC_MOVL_EvIz | rexw, 0, base, ofs);
+    tcg_out32(s, val);
+    return true;
+}
+
+static void tcg_out_shifti(TCGContext *s, int subopc, int reg, int count)
+{
+    /* Propagate an opcode prefix, such as P_DATA16.  */
+    int ext = subopc & ~0x7;
+    subopc &= 0x7;
+
+    if (count == 1) {
+        tcg_out_modrm(s, OPC_SHIFT_1 + ext, subopc, reg);
+    } else {
+        tcg_out_modrm(s, OPC_SHIFT_Ib + ext, subopc, reg);
+        tcg_out8(s, count);
+    }
+}
+
+static inline void tcg_out_bswap32(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_BSWAP + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static inline void tcg_out_rolw_8(TCGContext *s, int reg)
+{
+    tcg_out_shifti(s, SHIFT_ROL + P_DATA16, reg, 8);
+}
+
+static inline void tcg_out_ext8u(TCGContext *s, int dest, int src)
+{
+    /* movzbl */
+    tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
+    tcg_out_modrm(s, OPC_MOVZBL + P_REXB_RM, dest, src);
+}
+
+static void tcg_out_ext8s(TCGContext *s, int dest, int src, int rexw)
+{
+    /* movsbl */
+    tcg_debug_assert(src < 4 || TCG_TARGET_REG_BITS == 64);
+    tcg_out_modrm(s, OPC_MOVSBL + P_REXB_RM + rexw, dest, src);
+}
+
+static inline void tcg_out_ext16u(TCGContext *s, int dest, int src)
+{
+    /* movzwl */
+    tcg_out_modrm(s, OPC_MOVZWL, dest, src);
+}
+
+static inline void tcg_out_ext16s(TCGContext *s, int dest, int src, int rexw)
+{
+    /* movsw[lq] */
+    tcg_out_modrm(s, OPC_MOVSWL + rexw, dest, src);
+}
+
+static inline void tcg_out_ext32u(TCGContext *s, int dest, int src)
+{
+    /* 32-bit mov zero extends.  */
+    tcg_out_modrm(s, OPC_MOVL_GvEv, dest, src);
+}
+
+static inline void tcg_out_ext32s(TCGContext *s, int dest, int src)
+{
+    tcg_out_modrm(s, OPC_MOVSLQ, dest, src);
+}
+
+static inline void tcg_out_bswap64(TCGContext *s, int reg)
+{
+    tcg_out_opc(s, OPC_BSWAP + P_REXW + LOWREGMASK(reg), 0, reg, 0);
+}
+
+static void tgen_arithi(TCGContext *s, int c, int r0,
+                        tcg_target_long val, int cf)
+{
+    int rexw = 0;
+
+    if (TCG_TARGET_REG_BITS == 64) {
+        rexw = c & -8;
+        c &= 7;
+    }
+
+    /* ??? While INC is 2 bytes shorter than ADDL $1, they also induce
+       partial flags update stalls on Pentium4 and are not recommended
+       by current Intel optimization manuals.  */
+    if (!cf && (c == ARITH_ADD || c == ARITH_SUB) && (val == 1 || val == -1)) {
+        int is_inc = (c == ARITH_ADD) ^ (val < 0);
+        if (TCG_TARGET_REG_BITS == 64) {
+            /* The single-byte increment encodings are re-tasked as the
+               REX prefixes.  Use the MODRM encoding.  */
+            tcg_out_modrm(s, OPC_GRP5 + rexw,
+                          (is_inc ? EXT5_INC_Ev : EXT5_DEC_Ev), r0);
+        } else {
+            tcg_out8(s, (is_inc ? OPC_INC_r32 : OPC_DEC_r32) + r0);
+        }
+        return;
+    }
+
+    if (c == ARITH_AND) {
+        if (TCG_TARGET_REG_BITS == 64) {
+            if (val == 0xffffffffu) {
+                tcg_out_ext32u(s, r0, r0);
+                return;
+            }
+            if (val == (uint32_t)val) {
+                /* AND with no high bits set can use a 32-bit operation.  */
+                rexw = 0;
+            }
+        }
+        if (val == 0xffu && (r0 < 4 || TCG_TARGET_REG_BITS == 64)) {
+            tcg_out_ext8u(s, r0, r0);
+            return;
+        }
+        if (val == 0xffffu) {
+            tcg_out_ext16u(s, r0, r0);
+            return;
+        }
+    }
+
+    if (val == (int8_t)val) {
+        tcg_out_modrm(s, OPC_ARITH_EvIb + rexw, c, r0);
+        tcg_out8(s, val);
+        return;
+    }
+    if (rexw == 0 || val == (int32_t)val) {
+        tcg_out_modrm(s, OPC_ARITH_EvIz + rexw, c, r0);
+        tcg_out32(s, val);
+        return;
+    }
+
+    tcg_abort();
+}
+
+static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
+{
+    if (val != 0) {
+        tgen_arithi(s, ARITH_ADD + P_REXW, reg, val, 0);
+    }
+}
+
+/* Use SMALL != 0 to force a short forward branch.  */
+static void tcg_out_jxx(TCGContext *s, int opc, TCGLabel *l, int small)
+{
+    int32_t val, val1;
+
+    if (l->has_value) {
+        val = tcg_pcrel_diff(s, l->u.value_ptr);
+        val1 = val - 2;
+        if ((int8_t)val1 == val1) {
+            if (opc == -1) {
+                tcg_out8(s, OPC_JMP_short);
+            } else {
+                tcg_out8(s, OPC_JCC_short + opc);
+            }
+            tcg_out8(s, val1);
+        } else {
+            if (small) {
+                tcg_abort();
+            }
+            if (opc == -1) {
+                tcg_out8(s, OPC_JMP_long);
+                tcg_out32(s, val - 5);
+            } else {
+                tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
+                tcg_out32(s, val - 6);
+            }
+        }
+    } else if (small) {
+        if (opc == -1) {
+            tcg_out8(s, OPC_JMP_short);
+        } else {
+            tcg_out8(s, OPC_JCC_short + opc);
+        }
+        tcg_out_reloc(s, s->code_ptr, R_386_PC8, l, -1);
+        s->code_ptr += 1;
+    } else {
+        if (opc == -1) {
+            tcg_out8(s, OPC_JMP_long);
+        } else {
+            tcg_out_opc(s, OPC_JCC_long + opc, 0, 0, 0);
+        }
+        tcg_out_reloc(s, s->code_ptr, R_386_PC32, l, -4);
+        s->code_ptr += 4;
+    }
+}
+
+static void tcg_out_cmp(TCGContext *s, TCGArg arg1, TCGArg arg2,
+                        int const_arg2, int rexw)
+{
+    if (const_arg2) {
+        if (arg2 == 0) {
+            /* test r, r */
+            tcg_out_modrm(s, OPC_TESTL + rexw, arg1, arg1);
+        } else {
+            tgen_arithi(s, ARITH_CMP + rexw, arg1, arg2, 0);
+        }
+    } else {
+        tgen_arithr(s, ARITH_CMP + rexw, arg1, arg2);
+    }
+}
+
+static void tcg_out_brcond32(TCGContext *s, TCGCond cond,
+                             TCGArg arg1, TCGArg arg2, int const_arg2,
+                             TCGLabel *label, int small)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
+    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small);
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_brcond64(TCGContext *s, TCGCond cond,
+                             TCGArg arg1, TCGArg arg2, int const_arg2,
+                             TCGLabel *label, int small)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
+    tcg_out_jxx(s, tcg_cond_to_jcc[cond], label, small);
+}
+#else
+/* XXX: we implement it at the target level to avoid having to
+   handle cross basic blocks temporaries */
+static void tcg_out_brcond2(TCGContext *s, const TCGArg *args,
+                            const int *const_args, int small)
+{
+    TCGLabel *label_next = gen_new_label();
+    TCGLabel *label_this = arg_label(args[5]);
+
+    switch(args[4]) {
+    case TCG_COND_EQ:
+        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
+                         label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_EQ, args[1], args[3], const_args[3],
+                         label_this, small);
+        break;
+    case TCG_COND_NE:
+        tcg_out_brcond32(s, TCG_COND_NE, args[0], args[2], const_args[2],
+                         label_this, small);
+        tcg_out_brcond32(s, TCG_COND_NE, args[1], args[3], const_args[3],
+                         label_this, small);
+        break;
+    case TCG_COND_LT:
+        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_LE:
+        tcg_out_brcond32(s, TCG_COND_LT, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_GT:
+        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_GE:
+        tcg_out_brcond32(s, TCG_COND_GT, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_LTU:
+        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LTU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_LEU:
+        tcg_out_brcond32(s, TCG_COND_LTU, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_LEU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_GTU:
+        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GTU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    case TCG_COND_GEU:
+        tcg_out_brcond32(s, TCG_COND_GTU, args[1], args[3], const_args[3],
+                         label_this, small);
+        tcg_out_jxx(s, JCC_JNE, label_next, 1);
+        tcg_out_brcond32(s, TCG_COND_GEU, args[0], args[2], const_args[2],
+                         label_this, small);
+        break;
+    default:
+        tcg_abort();
+    }
+    tcg_out_label(s, label_next, s->code_ptr);
+}
+#endif
+
+static void tcg_out_setcond32(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg arg1, TCGArg arg2, int const_arg2)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, 0);
+    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
+    tcg_out_ext8u(s, dest, dest);
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_setcond64(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg arg1, TCGArg arg2, int const_arg2)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2, P_REXW);
+    tcg_out_modrm(s, OPC_SETCC | tcg_cond_to_jcc[cond], 0, dest);
+    tcg_out_ext8u(s, dest, dest);
+}
+#else
+static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
+                             const int *const_args)
+{
+    TCGArg new_args[6];
+    TCGLabel *label_true, *label_over;
+
+    memcpy(new_args, args+1, 5*sizeof(TCGArg));
+
+    if (args[0] == args[1] || args[0] == args[2]
+        || (!const_args[3] && args[0] == args[3])
+        || (!const_args[4] && args[0] == args[4])) {
+        /* When the destination overlaps with one of the argument
+           registers, don't do anything tricky.  */
+        label_true = gen_new_label();
+        label_over = gen_new_label();
+
+        new_args[5] = label_arg(label_true);
+        tcg_out_brcond2(s, new_args, const_args+1, 1);
+
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
+        tcg_out_jxx(s, JCC_JMP, label_over, 1);
+        tcg_out_label(s, label_true, s->code_ptr);
+
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], 1);
+        tcg_out_label(s, label_over, s->code_ptr);
+    } else {
+        /* When the destination does not overlap one of the arguments,
+           clear the destination first, jump if cond false, and emit an
+           increment in the true case.  This results in smaller code.  */
+
+        tcg_out_movi(s, TCG_TYPE_I32, args[0], 0);
+
+        label_over = gen_new_label();
+        new_args[4] = tcg_invert_cond(new_args[4]);
+        new_args[5] = label_arg(label_over);
+        tcg_out_brcond2(s, new_args, const_args+1, 1);
+
+        tgen_arithi(s, ARITH_ADD, args[0], 1, 0);
+        tcg_out_label(s, label_over, s->code_ptr);
+    }
+}
+#endif
+
+static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg c1, TCGArg c2, int const_c2,
+                              TCGArg v1)
+{
+    tcg_out_cmp(s, c1, c2, const_c2, 0);
+    if (have_cmov) {
+        tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond], dest, v1);
+    } else {
+        TCGLabel *over = gen_new_label();
+        tcg_out_jxx(s, tcg_cond_to_jcc[tcg_invert_cond(cond)], over, 1);
+        tcg_out_mov(s, TCG_TYPE_I32, dest, v1);
+        tcg_out_label(s, over, s->code_ptr);
+    }
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg c1, TCGArg c2, int const_c2,
+                              TCGArg v1)
+{
+    tcg_out_cmp(s, c1, c2, const_c2, P_REXW);
+    tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | P_REXW, dest, v1);
+}
+#endif
+
+static void tcg_out_branch(TCGContext *s, int call, tcg_insn_unit *dest)
+{
+    intptr_t disp = tcg_pcrel_diff(s, dest) - 5;
+
+    if (disp == (int32_t)disp) {
+        tcg_out_opc(s, call ? OPC_CALL_Jz : OPC_JMP_long, 0, 0, 0);
+        tcg_out32(s, disp);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R10, (uintptr_t)dest);
+        tcg_out_modrm(s, OPC_GRP5,
+                      call ? EXT5_CALLN_Ev : EXT5_JMPN_Ev, TCG_REG_R10);
+    }
+}
+
+static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+{
+    tcg_out_branch(s, 1, dest);
+}
+
+static void tcg_out_jmp(TCGContext *s, tcg_insn_unit *dest)
+{
+    tcg_out_branch(s, 0, dest);
+}
+
+static void tcg_out_nopn(TCGContext *s, int n)
+{
+    int i;
+    /* Emit 1 or 2 operand size prefixes for the standard one byte nop,
+     * "xchg %eax,%eax", forming "xchg %ax,%ax". All cores accept the
+     * duplicate prefix, and all of the interesting recent cores can
+     * decode and discard the duplicates in a single cycle.
+     */
+    tcg_debug_assert(n >= 1);
+    for (i = 1; i < n; ++i) {
+        tcg_out8(s, 0x66);
+    }
+    tcg_out8(s, 0x90);
+}
+
+#if defined(CONFIG_SOFTMMU)
+/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
+ *                                     int mmu_idx, uintptr_t ra)
+ */
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+};
+
+/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
+ *                                     uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
+};
+
+/* Perform the TLB load and compare.
+
+   Inputs:
+   ADDRLO and ADDRHI contain the low and high part of the address.
+
+   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
+
+   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
+   This should be offsetof addr_read or addr_write.
+
+   Outputs:
+   LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
+   positions of the displacements of forward jumps to the TLB miss case.
+
+   Second argument register is loaded with the low part of the address.
+   In the TLB hit case, it has been adjusted as indicated by the TLB
+   and so is a host address.  In the TLB miss case, it continues to
+   hold a guest address.
+
+   First argument register is clobbered.  */
+
+static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi,
+                                    int mem_index, TCGMemOp opc,
+                                    tcg_insn_unit **label_ptr, int which)
+{
+    const TCGReg r0 = TCG_REG_L0;
+    const TCGReg r1 = TCG_REG_L1;
+    TCGType ttype = TCG_TYPE_I32;
+    TCGType tlbtype = TCG_TYPE_I32;
+    int trexw = 0, hrexw = 0, tlbrexw = 0;
+    int a_bits = get_alignment_bits(opc);
+    target_ulong tlb_mask;
+
+    if (TCG_TARGET_REG_BITS == 64) {
+        if (TARGET_LONG_BITS == 64) {
+            ttype = TCG_TYPE_I64;
+            trexw = P_REXW;
+        }
+        if (TCG_TYPE_PTR == TCG_TYPE_I64) {
+            hrexw = P_REXW;
+            if (TARGET_PAGE_BITS + CPU_TLB_BITS > 32) {
+                tlbtype = TCG_TYPE_I64;
+                tlbrexw = P_REXW;
+            }
+        }
+    }
+
+    tcg_out_mov(s, tlbtype, r0, addrlo);
+    if (a_bits >= 0) {
+        /* A byte access or an alignment check required */
+        tcg_out_mov(s, ttype, r1, addrlo);
+        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);
+    } else {
+        /* For unaligned access check that we don't cross pages using
+           the page address of the last byte.  */
+        tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo,
+                             (1 << (opc & MO_SIZE)) - 1);
+        tlb_mask = TARGET_PAGE_MASK;
+    }
+
+    tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0,
+                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
+
+    tgen_arithi(s, ARITH_AND + trexw, r1, tlb_mask, 0);
+    tgen_arithi(s, ARITH_AND + tlbrexw, r0,
+                (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
+
+    tcg_out_modrm_sib_offset(s, OPC_LEA + hrexw, r0, TCG_AREG0, r0, 0,
+                             offsetof(CPUArchState, tlb_table[mem_index][0])
+                             + which);
+
+    /* cmp 0(r0), r1 */
+    tcg_out_modrm_offset(s, OPC_CMP_GvEv + trexw, r1, r0, 0);
+
+    /* Prepare for both the fast path add of the tlb addend, and the slow
+       path function argument setup.  There are two cases worth note:
+       For 32-bit guest and x86_64 host, MOVL zero-extends the guest address
+       before the fastpath ADDQ below.  For 64-bit guest and x32 host, MOVQ
+       copies the entire guest address for the slow path, while truncation
+       for the 32-bit host happens with the fastpath ADDL below.  */
+    tcg_out_mov(s, ttype, r1, addrlo);
+
+    /* jne slow_path */
+    tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
+    label_ptr[0] = s->code_ptr;
+    s->code_ptr += 4;
+
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        /* cmp 4(r0), addrhi */
+        tcg_out_modrm_offset(s, OPC_CMP_GvEv, addrhi, r0, 4);
+
+        /* jne slow_path */
+        tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
+        label_ptr[1] = s->code_ptr;
+        s->code_ptr += 4;
+    }
+
+    /* TLB Hit.  */
+
+    /* add addend(r0), r1 */
+    tcg_out_modrm_offset(s, OPC_ADD_GvEv + hrexw, r1, r0,
+                         offsetof(CPUTLBEntry, addend) - which);
+}
+
+/*
+ * Record the context of a call to the out of line helper code for the slow path
+ * for a load or store, so that we can later generate the correct helper code
+ */
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
+                                TCGReg datalo, TCGReg datahi,
+                                TCGReg addrlo, TCGReg addrhi,
+                                tcg_insn_unit *raddr,
+                                tcg_insn_unit **label_ptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->oi = oi;
+    label->datalo_reg = datalo;
+    label->datahi_reg = datahi;
+    label->addrlo_reg = addrlo;
+    label->addrhi_reg = addrhi;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr[0];
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        label->label_ptr[1] = label_ptr[1];
+    }
+}
+
+/*
+ * Generate code for the slow path for a load at the end of block
+ */
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOpIdx oi = l->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGReg data_reg;
+    tcg_insn_unit **label_ptr = &l->label_ptr[0];
+
+    /* resolve label address */
+    tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
+    }
+
+    if (TCG_TARGET_REG_BITS == 32) {
+        int ofs = 0;
+
+        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        if (TARGET_LONG_BITS == 64) {
+            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
+            ofs += 4;
+        }
+
+        tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        tcg_out_sti(s, TCG_TYPE_PTR, (uintptr_t)l->raddr, TCG_REG_ESP, ofs);
+    } else {
+        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+        /* The second argument is already loaded with addrlo.  */
+        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], oi);
+        tcg_out_movi(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[3],
+                     (uintptr_t)l->raddr);
+    }
+
+    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+
+    data_reg = l->datalo_reg;
+    switch (opc & MO_SSIZE) {
+    case MO_SB:
+        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+    case MO_SW:
+        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case MO_SL:
+        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
+        break;
+#endif
+    case MO_UB:
+    case MO_UW:
+        /* Note that the helpers have zero-extended to tcg_target_long.  */
+    case MO_UL:
+        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+        break;
+    case MO_Q:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
+        } else if (data_reg == TCG_REG_EDX) {
+            /* xchg %edx, %eax */
+            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
+            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EAX);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_EDX);
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_jmp(s, l->raddr);
+}
+
+/*
+ * Generate code for the slow path for a store at the end of block
+ */
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOpIdx oi = l->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGMemOp s_bits = opc & MO_SIZE;
+    tcg_insn_unit **label_ptr = &l->label_ptr[0];
+    TCGReg retaddr;
+
+    /* resolve label address */
+    tcg_patch32(label_ptr[0], s->code_ptr - label_ptr[0] - 4);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        tcg_patch32(label_ptr[1], s->code_ptr - label_ptr[1] - 4);
+    }
+
+    if (TCG_TARGET_REG_BITS == 32) {
+        int ofs = 0;
+
+        tcg_out_st(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        tcg_out_st(s, TCG_TYPE_I32, l->addrlo_reg, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        if (TARGET_LONG_BITS == 64) {
+            tcg_out_st(s, TCG_TYPE_I32, l->addrhi_reg, TCG_REG_ESP, ofs);
+            ofs += 4;
+        }
+
+        tcg_out_st(s, TCG_TYPE_I32, l->datalo_reg, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        if (s_bits == MO_64) {
+            tcg_out_st(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_ESP, ofs);
+            ofs += 4;
+        }
+
+        tcg_out_sti(s, TCG_TYPE_I32, oi, TCG_REG_ESP, ofs);
+        ofs += 4;
+
+        retaddr = TCG_REG_EAX;
+        tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
+        tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, ofs);
+    } else {
+        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+        /* The second argument is already loaded with addrlo.  */
+        tcg_out_mov(s, (s_bits == MO_64 ? TCG_TYPE_I64 : TCG_TYPE_I32),
+                    tcg_target_call_iarg_regs[2], l->datalo_reg);
+        tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], oi);
+
+        if (ARRAY_SIZE(tcg_target_call_iarg_regs) > 4) {
+            retaddr = tcg_target_call_iarg_regs[4];
+            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
+        } else {
+            retaddr = TCG_REG_RAX;
+            tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
+            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP,
+                       TCG_TARGET_CALL_STACK_OFFSET);
+        }
+    }
+
+    /* "Tail call" to the helper, with the return address back inline.  */
+    tcg_out_push(s, retaddr);
+    tcg_out_jmp(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+}
+#elif defined(__x86_64__) && defined(__linux__)
+# include <asm/prctl.h>
+# include <sys/prctl.h>
+
+int arch_prctl(int code, unsigned long addr);
+
+static int guest_base_flags;
+static inline void setup_guest_base_seg(void)
+{
+    if (arch_prctl(ARCH_SET_GS, guest_base) == 0) {
+        guest_base_flags = P_GS;
+    }
+}
+#else
+# define guest_base_flags 0
+static inline void setup_guest_base_seg(void) { }
+#endif /* SOFTMMU */
+
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, int index, intptr_t ofs,
+                                   int seg, TCGMemOp memop)
+{
+    const TCGMemOp real_bswap = memop & MO_BSWAP;
+    TCGMemOp bswap = real_bswap;
+    int movop = OPC_MOVL_GvEv;
+
+    if (have_movbe && real_bswap) {
+        bswap = 0;
+        movop = OPC_MOVBE_GyMy;
+    }
+
+    switch (memop & MO_SSIZE) {
+    case MO_UB:
+        tcg_out_modrm_sib_offset(s, OPC_MOVZBL + seg, datalo,
+                                 base, index, 0, ofs);
+        break;
+    case MO_SB:
+        tcg_out_modrm_sib_offset(s, OPC_MOVSBL + P_REXW + seg, datalo,
+                                 base, index, 0, ofs);
+        break;
+    case MO_UW:
+        tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
+                                 base, index, 0, ofs);
+        if (real_bswap) {
+            tcg_out_rolw_8(s, datalo);
+        }
+        break;
+    case MO_SW:
+        if (real_bswap) {
+            if (have_movbe) {
+                tcg_out_modrm_sib_offset(s, OPC_MOVBE_GyMy + P_DATA16 + seg,
+                                         datalo, base, index, 0, ofs);
+            } else {
+                tcg_out_modrm_sib_offset(s, OPC_MOVZWL + seg, datalo,
+                                         base, index, 0, ofs);
+                tcg_out_rolw_8(s, datalo);
+            }
+            tcg_out_modrm(s, OPC_MOVSWL + P_REXW, datalo, datalo);
+        } else {
+            tcg_out_modrm_sib_offset(s, OPC_MOVSWL + P_REXW + seg,
+                                     datalo, base, index, 0, ofs);
+        }
+        break;
+    case MO_UL:
+        tcg_out_modrm_sib_offset(s, movop + seg, datalo, base, index, 0, ofs);
+        if (bswap) {
+            tcg_out_bswap32(s, datalo);
+        }
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case MO_SL:
+        if (real_bswap) {
+            tcg_out_modrm_sib_offset(s, movop + seg, datalo,
+                                     base, index, 0, ofs);
+            if (bswap) {
+                tcg_out_bswap32(s, datalo);
+            }
+            tcg_out_ext32s(s, datalo, datalo);
+        } else {
+            tcg_out_modrm_sib_offset(s, OPC_MOVSLQ + seg, datalo,
+                                     base, index, 0, ofs);
+        }
+        break;
+#endif
+    case MO_Q:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_modrm_sib_offset(s, movop + P_REXW + seg, datalo,
+                                     base, index, 0, ofs);
+            if (bswap) {
+                tcg_out_bswap64(s, datalo);
+            }
+        } else {
+            if (real_bswap) {
+                int t = datalo;
+                datalo = datahi;
+                datahi = t;
+            }
+            if (base != datalo) {
+                tcg_out_modrm_sib_offset(s, movop + seg, datalo,
+                                         base, index, 0, ofs);
+                tcg_out_modrm_sib_offset(s, movop + seg, datahi,
+                                         base, index, 0, ofs + 4);
+            } else {
+                tcg_out_modrm_sib_offset(s, movop + seg, datahi,
+                                         base, index, 0, ofs + 4);
+                tcg_out_modrm_sib_offset(s, movop + seg, datalo,
+                                         base, index, 0, ofs);
+            }
+            if (bswap) {
+                tcg_out_bswap32(s, datalo);
+                tcg_out_bswap32(s, datahi);
+            }
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+/* XXX: qemu_ld and qemu_st could be modified to clobber only EDX and
+   EAX. It will be useful once fixed registers globals are less
+   common. */
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64)
+{
+    TCGReg datalo, datahi, addrlo;
+    TCGReg addrhi __attribute__((unused));
+    TCGMemOpIdx oi;
+    TCGMemOp opc;
+#if defined(CONFIG_SOFTMMU)
+    int mem_index;
+    tcg_insn_unit *label_ptr[2];
+#endif
+
+    datalo = *args++;
+    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+
+#if defined(CONFIG_SOFTMMU)
+    mem_index = get_mmuidx(oi);
+
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
+                     label_ptr, offsetof(CPUTLBEntry, addr_read));
+
+    /* TLB Hit.  */
+    tcg_out_qemu_ld_direct(s, datalo, datahi, TCG_REG_L1, -1, 0, 0, opc);
+
+    /* Record the current context of a load into ldst label */
+    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
+                        s->code_ptr, label_ptr);
+#else
+    {
+        int32_t offset = guest_base;
+        TCGReg base = addrlo;
+        int index = -1;
+        int seg = 0;
+
+        /* For a 32-bit guest, the high 32 bits may contain garbage.
+           We can do this with the ADDR32 prefix if we're not using
+           a guest base, or when using segmentation.  Otherwise we
+           need to zero-extend manually.  */
+        if (guest_base == 0 || guest_base_flags) {
+            seg = guest_base_flags;
+            offset = 0;
+            if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
+                seg |= P_ADDR32;
+            }
+        } else if (TCG_TARGET_REG_BITS == 64) {
+            if (TARGET_LONG_BITS == 32) {
+                tcg_out_ext32u(s, TCG_REG_L0, base);
+                base = TCG_REG_L0;
+            }
+            if (offset != guest_base) {
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, guest_base);
+                index = TCG_REG_L1;
+                offset = 0;
+            }
+        }
+
+        tcg_out_qemu_ld_direct(s, datalo, datahi,
+                               base, index, offset, seg, opc);
+    }
+#endif
+}
+
+static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, intptr_t ofs, int seg,
+                                   TCGMemOp memop)
+{
+    /* ??? Ideally we wouldn't need a scratch register.  For user-only,
+       we could perform the bswap twice to restore the original value
+       instead of moving to the scratch.  But as it is, the L constraint
+       means that TCG_REG_L0 is definitely free here.  */
+    const TCGReg scratch = TCG_REG_L0;
+    const TCGMemOp real_bswap = memop & MO_BSWAP;
+    TCGMemOp bswap = real_bswap;
+    int movop = OPC_MOVL_EvGv;
+
+    if (have_movbe && real_bswap) {
+        bswap = 0;
+        movop = OPC_MOVBE_MyGy;
+    }
+
+    switch (memop & MO_SIZE) {
+    case MO_8:
+        /* In 32-bit mode, 8-bit stores can only happen from [abcd]x.
+           Use the scratch register if necessary.  */
+        if (TCG_TARGET_REG_BITS == 32 && datalo >= 4) {
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            datalo = scratch;
+        }
+        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
+                             datalo, base, ofs);
+        break;
+    case MO_16:
+        if (bswap) {
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            tcg_out_rolw_8(s, scratch);
+            datalo = scratch;
+        }
+        tcg_out_modrm_offset(s, movop + P_DATA16 + seg, datalo, base, ofs);
+        break;
+    case MO_32:
+        if (bswap) {
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            tcg_out_bswap32(s, scratch);
+            datalo = scratch;
+        }
+        tcg_out_modrm_offset(s, movop + seg, datalo, base, ofs);
+        break;
+    case MO_64:
+        if (TCG_TARGET_REG_BITS == 64) {
+            if (bswap) {
+                tcg_out_mov(s, TCG_TYPE_I64, scratch, datalo);
+                tcg_out_bswap64(s, scratch);
+                datalo = scratch;
+            }
+            tcg_out_modrm_offset(s, movop + P_REXW + seg, datalo, base, ofs);
+        } else if (bswap) {
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi);
+            tcg_out_bswap32(s, scratch);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs);
+            tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
+            tcg_out_bswap32(s, scratch);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs+4);
+        } else {
+            if (real_bswap) {
+                int t = datalo;
+                datalo = datahi;
+                datahi = t;
+            }
+            tcg_out_modrm_offset(s, movop + seg, datalo, base, ofs);
+            tcg_out_modrm_offset(s, movop + seg, datahi, base, ofs+4);
+        }
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64)
+{
+    TCGReg datalo, datahi, addrlo;
+    TCGReg addrhi __attribute__((unused));
+    TCGMemOpIdx oi;
+    TCGMemOp opc;
+#if defined(CONFIG_SOFTMMU)
+    int mem_index;
+    tcg_insn_unit *label_ptr[2];
+#endif
+
+    datalo = *args++;
+    datahi = (TCG_TARGET_REG_BITS == 32 && is64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TARGET_LONG_BITS > TCG_TARGET_REG_BITS ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+
+#if defined(CONFIG_SOFTMMU)
+    mem_index = get_mmuidx(oi);
+
+    tcg_out_tlb_load(s, addrlo, addrhi, mem_index, opc,
+                     label_ptr, offsetof(CPUTLBEntry, addr_write));
+
+    /* TLB Hit.  */
+    tcg_out_qemu_st_direct(s, datalo, datahi, TCG_REG_L1, 0, 0, opc);
+
+    /* Record the current context of a store into ldst label */
+    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
+                        s->code_ptr, label_ptr);
+#else
+    {
+        int32_t offset = guest_base;
+        TCGReg base = addrlo;
+        int seg = 0;
+
+        /* See comment in tcg_out_qemu_ld re zero-extension of addrlo.  */
+        if (guest_base == 0 || guest_base_flags) {
+            seg = guest_base_flags;
+            offset = 0;
+            if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
+                seg |= P_ADDR32;
+            }
+        } else if (TCG_TARGET_REG_BITS == 64) {
+            /* ??? Note that we can't use the same SIB addressing scheme
+               as for loads, since we require L0 free for bswap.  */
+            if (offset != guest_base) {
+                if (TARGET_LONG_BITS == 32) {
+                    tcg_out_ext32u(s, TCG_REG_L0, base);
+                    base = TCG_REG_L0;
+                }
+                tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, guest_base);
+                tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
+                base = TCG_REG_L1;
+                offset = 0;
+            } else if (TARGET_LONG_BITS == 32) {
+                tcg_out_ext32u(s, TCG_REG_L1, base);
+                base = TCG_REG_L1;
+            }
+        }
+
+        tcg_out_qemu_st_direct(s, datalo, datahi, base, offset, seg, opc);
+    }
+#endif
+}
+
+static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                              const TCGArg *args, const int *const_args)
+{
+    int c, vexop, rexw = 0;
+
+#if TCG_TARGET_REG_BITS == 64
+# define OP_32_64(x) \
+        case glue(glue(INDEX_op_, x), _i64): \
+            rexw = P_REXW; /* FALLTHRU */    \
+        case glue(glue(INDEX_op_, x), _i32)
+#else
+# define OP_32_64(x) \
+        case glue(glue(INDEX_op_, x), _i32)
+#endif
+
+    switch(opc) {
+    case INDEX_op_exit_tb:
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_EAX, args[0]);
+        tcg_out_jmp(s, tb_ret_addr);
+        break;
+    case INDEX_op_goto_tb:
+        if (s->tb_jmp_insn_offset) {
+            /* direct jump method */
+            int gap;
+            /* jump displacement must be aligned for atomic patching;
+             * see if we need to add extra nops before jump
+             */
+            gap = tcg_pcrel_diff(s, QEMU_ALIGN_PTR_UP(s->code_ptr + 1, 4));
+            if (gap != 1) {
+                tcg_out_nopn(s, gap - 1);
+            }
+            tcg_out8(s, OPC_JMP_long); /* jmp im */
+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            tcg_out32(s, 0);
+        } else {
+            /* indirect jump method */
+            tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, -1,
+                                 (intptr_t)(s->tb_jmp_target_addr + args[0]));
+        }
+        s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+        break;
+    case INDEX_op_br:
+        tcg_out_jxx(s, JCC_JMP, arg_label(args[0]), 0);
+        break;
+    OP_32_64(ld8u):
+        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
+        tcg_out_modrm_offset(s, OPC_MOVZBL, args[0], args[1], args[2]);
+        break;
+    OP_32_64(ld8s):
+        tcg_out_modrm_offset(s, OPC_MOVSBL + rexw, args[0], args[1], args[2]);
+        break;
+    OP_32_64(ld16u):
+        /* Note that we can ignore REXW for the zero-extend to 64-bit.  */
+        tcg_out_modrm_offset(s, OPC_MOVZWL, args[0], args[1], args[2]);
+        break;
+    OP_32_64(ld16s):
+        tcg_out_modrm_offset(s, OPC_MOVSWL + rexw, args[0], args[1], args[2]);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_ld32u_i64:
+#endif
+    case INDEX_op_ld_i32:
+        tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        break;
+
+    OP_32_64(st8):
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVB_EvIz,
+                                 0, args[1], args[2]);
+            tcg_out8(s, args[0]);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
+                                 args[0], args[1], args[2]);
+        }
+        break;
+    OP_32_64(st16):
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16,
+                                 0, args[1], args[2]);
+            tcg_out16(s, args[0]);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
+                                 args[0], args[1], args[2]);
+        }
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_st32_i64:
+#endif
+    case INDEX_op_st_i32:
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, args[1], args[2]);
+            tcg_out32(s, args[0]);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        }
+        break;
+
+    OP_32_64(add):
+        /* For 3-operand addition, use LEA.  */
+        if (args[0] != args[1]) {
+            TCGArg a0 = args[0], a1 = args[1], a2 = args[2], c3 = 0;
+
+            if (const_args[2]) {
+                c3 = a2, a2 = -1;
+            } else if (a0 == a2) {
+                /* Watch out for dest = src + dest, since we've removed
+                   the matching constraint on the add.  */
+                tgen_arithr(s, ARITH_ADD + rexw, a0, a1);
+                break;
+            }
+
+            tcg_out_modrm_sib_offset(s, OPC_LEA + rexw, a0, a1, a2, 0, c3);
+            break;
+        }
+        c = ARITH_ADD;
+        goto gen_arith;
+    OP_32_64(sub):
+        c = ARITH_SUB;
+        goto gen_arith;
+    OP_32_64(and):
+        c = ARITH_AND;
+        goto gen_arith;
+    OP_32_64(or):
+        c = ARITH_OR;
+        goto gen_arith;
+    OP_32_64(xor):
+        c = ARITH_XOR;
+        goto gen_arith;
+    gen_arith:
+        if (const_args[2]) {
+            tgen_arithi(s, c + rexw, args[0], args[2], 0);
+        } else {
+            tgen_arithr(s, c + rexw, args[0], args[2]);
+        }
+        break;
+
+    OP_32_64(andc):
+        if (const_args[2]) {
+            tcg_out_mov(s, rexw ? TCG_TYPE_I64 : TCG_TYPE_I32,
+                        args[0], args[1]);
+            tgen_arithi(s, ARITH_AND + rexw, args[0], ~args[2], 0);
+        } else {
+            tcg_out_vex_modrm(s, OPC_ANDN + rexw, args[0], args[2], args[1]);
+        }
+        break;
+
+    OP_32_64(mul):
+        if (const_args[2]) {
+            int32_t val;
+            val = args[2];
+            if (val == (int8_t)val) {
+                tcg_out_modrm(s, OPC_IMUL_GvEvIb + rexw, args[0], args[0]);
+                tcg_out8(s, val);
+            } else {
+                tcg_out_modrm(s, OPC_IMUL_GvEvIz + rexw, args[0], args[0]);
+                tcg_out32(s, val);
+            }
+        } else {
+            tcg_out_modrm(s, OPC_IMUL_GvEv + rexw, args[0], args[2]);
+        }
+        break;
+
+    OP_32_64(div2):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IDIV, args[4]);
+        break;
+    OP_32_64(divu2):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_DIV, args[4]);
+        break;
+
+    OP_32_64(shl):
+        c = SHIFT_SHL;
+        vexop = OPC_SHLX;
+        goto gen_shift_maybe_vex;
+    OP_32_64(shr):
+        c = SHIFT_SHR;
+        vexop = OPC_SHRX;
+        goto gen_shift_maybe_vex;
+    OP_32_64(sar):
+        c = SHIFT_SAR;
+        vexop = OPC_SARX;
+        goto gen_shift_maybe_vex;
+    OP_32_64(rotl):
+        c = SHIFT_ROL;
+        goto gen_shift;
+    OP_32_64(rotr):
+        c = SHIFT_ROR;
+        goto gen_shift;
+    gen_shift_maybe_vex:
+        if (have_bmi2 && !const_args[2]) {
+            tcg_out_vex_modrm(s, vexop + rexw, args[0], args[2], args[1]);
+            break;
+        }
+        /* FALLTHRU */
+    gen_shift:
+        if (const_args[2]) {
+            tcg_out_shifti(s, c + rexw, args[0], args[2]);
+        } else {
+            tcg_out_modrm(s, OPC_SHIFT_cl + rexw, c, args[0]);
+        }
+        break;
+
+    case INDEX_op_brcond_i32:
+        tcg_out_brcond32(s, args[2], args[0], args[1], const_args[1],
+                         arg_label(args[3]), 0);
+        break;
+    case INDEX_op_setcond_i32:
+        tcg_out_setcond32(s, args[3], args[0], args[1],
+                          args[2], const_args[2]);
+        break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond32(s, args[5], args[0], args[1],
+                          args[2], const_args[2], args[3]);
+        break;
+
+    OP_32_64(bswap16):
+        tcg_out_rolw_8(s, args[0]);
+        break;
+    OP_32_64(bswap32):
+        tcg_out_bswap32(s, args[0]);
+        break;
+
+    OP_32_64(neg):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NEG, args[0]);
+        break;
+    OP_32_64(not):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_NOT, args[0]);
+        break;
+
+    OP_32_64(ext8s):
+        tcg_out_ext8s(s, args[0], args[1], rexw);
+        break;
+    OP_32_64(ext16s):
+        tcg_out_ext16s(s, args[0], args[1], rexw);
+        break;
+    OP_32_64(ext8u):
+        tcg_out_ext8u(s, args[0], args[1]);
+        break;
+    OP_32_64(ext16u):
+        tcg_out_ext16u(s, args[0], args[1]);
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, args, 0);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args, 1);
+        break;
+    case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, args, 0);
+        break;
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args, 1);
+        break;
+
+    OP_32_64(mulu2):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_MUL, args[3]);
+        break;
+    OP_32_64(muls2):
+        tcg_out_modrm(s, OPC_GRP3_Ev + rexw, EXT3_IMUL, args[3]);
+        break;
+    OP_32_64(add2):
+        if (const_args[4]) {
+            tgen_arithi(s, ARITH_ADD + rexw, args[0], args[4], 1);
+        } else {
+            tgen_arithr(s, ARITH_ADD + rexw, args[0], args[4]);
+        }
+        if (const_args[5]) {
+            tgen_arithi(s, ARITH_ADC + rexw, args[1], args[5], 1);
+        } else {
+            tgen_arithr(s, ARITH_ADC + rexw, args[1], args[5]);
+        }
+        break;
+    OP_32_64(sub2):
+        if (const_args[4]) {
+            tgen_arithi(s, ARITH_SUB + rexw, args[0], args[4], 1);
+        } else {
+            tgen_arithr(s, ARITH_SUB + rexw, args[0], args[4]);
+        }
+        if (const_args[5]) {
+            tgen_arithi(s, ARITH_SBB + rexw, args[1], args[5], 1);
+        } else {
+            tgen_arithr(s, ARITH_SBB + rexw, args[1], args[5]);
+        }
+        break;
+
+#if TCG_TARGET_REG_BITS == 32
+    case INDEX_op_brcond2_i32:
+        tcg_out_brcond2(s, args, const_args, 0);
+        break;
+    case INDEX_op_setcond2_i32:
+        tcg_out_setcond2(s, args, const_args);
+        break;
+#else /* TCG_TARGET_REG_BITS == 64 */
+    case INDEX_op_ld32s_i64:
+        tcg_out_modrm_offset(s, OPC_MOVSLQ, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i64:
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW,
+                                 0, args[1], args[2]);
+            tcg_out32(s, args[0]);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        }
+        break;
+
+    case INDEX_op_brcond_i64:
+        tcg_out_brcond64(s, args[2], args[0], args[1], const_args[1],
+                         arg_label(args[3]), 0);
+        break;
+    case INDEX_op_setcond_i64:
+        tcg_out_setcond64(s, args[3], args[0], args[1],
+                          args[2], const_args[2]);
+        break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond64(s, args[5], args[0], args[1],
+                          args[2], const_args[2], args[3]);
+        break;
+
+    case INDEX_op_bswap64_i64:
+        tcg_out_bswap64(s, args[0]);
+        break;
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_ext32u_i64:
+        tcg_out_ext32u(s, args[0], args[1]);
+        break;
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_ext32s_i64:
+        tcg_out_ext32s(s, args[0], args[1]);
+        break;
+#endif
+
+    OP_32_64(deposit):
+        if (args[3] == 0 && args[4] == 8) {
+            /* load bits 0..7 */
+            tcg_out_modrm(s, OPC_MOVB_EvGv | P_REXB_R | P_REXB_RM,
+                          args[2], args[0]);
+        } else if (args[3] == 8 && args[4] == 8) {
+            /* load bits 8..15 */
+            tcg_out_modrm(s, OPC_MOVB_EvGv, args[2], args[0] + 4);
+        } else if (args[3] == 0 && args[4] == 16) {
+            /* load bits 0..15 */
+            tcg_out_modrm(s, OPC_MOVL_EvGv | P_DATA16, args[2], args[0]);
+        } else {
+            tcg_abort();
+        }
+        break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+
+#undef OP_32_64
+}
+
+static const TCGTargetOpDef x86_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "qi", "r" } },
+    { INDEX_op_st16_i32, { "ri", "r" } },
+    { INDEX_op_st_i32, { "ri", "r" } },
+
+    { INDEX_op_add_i32, { "r", "r", "ri" } },
+    { INDEX_op_sub_i32, { "r", "0", "ri" } },
+    { INDEX_op_mul_i32, { "r", "0", "ri" } },
+    { INDEX_op_div2_i32, { "a", "d", "0", "1", "r" } },
+    { INDEX_op_divu2_i32, { "a", "d", "0", "1", "r" } },
+    { INDEX_op_and_i32, { "r", "0", "ri" } },
+    { INDEX_op_or_i32, { "r", "0", "ri" } },
+    { INDEX_op_xor_i32, { "r", "0", "ri" } },
+    { INDEX_op_andc_i32, { "r", "r", "ri" } },
+
+    { INDEX_op_shl_i32, { "r", "0", "Ci" } },
+    { INDEX_op_shr_i32, { "r", "0", "Ci" } },
+    { INDEX_op_sar_i32, { "r", "0", "Ci" } },
+    { INDEX_op_rotl_i32, { "r", "0", "ci" } },
+    { INDEX_op_rotr_i32, { "r", "0", "ci" } },
+
+    { INDEX_op_brcond_i32, { "r", "ri" } },
+
+    { INDEX_op_bswap16_i32, { "r", "0" } },
+    { INDEX_op_bswap32_i32, { "r", "0" } },
+
+    { INDEX_op_neg_i32, { "r", "0" } },
+
+    { INDEX_op_not_i32, { "r", "0" } },
+
+    { INDEX_op_ext8s_i32, { "r", "q" } },
+    { INDEX_op_ext16s_i32, { "r", "r" } },
+    { INDEX_op_ext8u_i32, { "r", "q" } },
+    { INDEX_op_ext16u_i32, { "r", "r" } },
+
+    { INDEX_op_setcond_i32, { "q", "r", "ri" } },
+
+    { INDEX_op_deposit_i32, { "Q", "0", "Q" } },
+    { INDEX_op_movcond_i32, { "r", "r", "ri", "r", "0" } },
+
+    { INDEX_op_mulu2_i32, { "a", "d", "a", "r" } },
+    { INDEX_op_muls2_i32, { "a", "d", "a", "r" } },
+    { INDEX_op_add2_i32, { "r", "r", "0", "1", "ri", "ri" } },
+    { INDEX_op_sub2_i32, { "r", "r", "0", "1", "ri", "ri" } },
+
+#if TCG_TARGET_REG_BITS == 32
+    { INDEX_op_brcond2_i32, { "r", "r", "ri", "ri" } },
+    { INDEX_op_setcond2_i32, { "r", "r", "r", "ri", "ri" } },
+#else
+    { INDEX_op_ld8u_i64, { "r", "r" } },
+    { INDEX_op_ld8s_i64, { "r", "r" } },
+    { INDEX_op_ld16u_i64, { "r", "r" } },
+    { INDEX_op_ld16s_i64, { "r", "r" } },
+    { INDEX_op_ld32u_i64, { "r", "r" } },
+    { INDEX_op_ld32s_i64, { "r", "r" } },
+    { INDEX_op_ld_i64, { "r", "r" } },
+    { INDEX_op_st8_i64, { "ri", "r" } },
+    { INDEX_op_st16_i64, { "ri", "r" } },
+    { INDEX_op_st32_i64, { "ri", "r" } },
+    { INDEX_op_st_i64, { "re", "r" } },
+
+    { INDEX_op_add_i64, { "r", "r", "re" } },
+    { INDEX_op_mul_i64, { "r", "0", "re" } },
+    { INDEX_op_div2_i64, { "a", "d", "0", "1", "r" } },
+    { INDEX_op_divu2_i64, { "a", "d", "0", "1", "r" } },
+    { INDEX_op_sub_i64, { "r", "0", "re" } },
+    { INDEX_op_and_i64, { "r", "0", "reZ" } },
+    { INDEX_op_or_i64, { "r", "0", "re" } },
+    { INDEX_op_xor_i64, { "r", "0", "re" } },
+    { INDEX_op_andc_i64, { "r", "r", "rI" } },
+
+    { INDEX_op_shl_i64, { "r", "0", "Ci" } },
+    { INDEX_op_shr_i64, { "r", "0", "Ci" } },
+    { INDEX_op_sar_i64, { "r", "0", "Ci" } },
+    { INDEX_op_rotl_i64, { "r", "0", "ci" } },
+    { INDEX_op_rotr_i64, { "r", "0", "ci" } },
+
+    { INDEX_op_brcond_i64, { "r", "re" } },
+    { INDEX_op_setcond_i64, { "r", "r", "re" } },
+
+    { INDEX_op_bswap16_i64, { "r", "0" } },
+    { INDEX_op_bswap32_i64, { "r", "0" } },
+    { INDEX_op_bswap64_i64, { "r", "0" } },
+    { INDEX_op_neg_i64, { "r", "0" } },
+    { INDEX_op_not_i64, { "r", "0" } },
+
+    { INDEX_op_ext8s_i64, { "r", "r" } },
+    { INDEX_op_ext16s_i64, { "r", "r" } },
+    { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext8u_i64, { "r", "r" } },
+    { INDEX_op_ext16u_i64, { "r", "r" } },
+    { INDEX_op_ext32u_i64, { "r", "r" } },
+
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
+    { INDEX_op_deposit_i64, { "Q", "0", "Q" } },
+    { INDEX_op_movcond_i64, { "r", "r", "re", "r", "0" } },
+
+    { INDEX_op_mulu2_i64, { "a", "d", "a", "r" } },
+    { INDEX_op_muls2_i64, { "a", "d", "a", "r" } },
+    { INDEX_op_add2_i64, { "r", "r", "0", "1", "re", "re" } },
+    { INDEX_op_sub2_i64, { "r", "r", "0", "1", "re", "re" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L" } },
+#elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L", "L" } },
+#else
+    { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r", "L", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L", "L", "L" } },
+#endif
+    { -1 },
+};
+
+static int tcg_target_callee_save_regs[] = {
+#if TCG_TARGET_REG_BITS == 64
+    TCG_REG_RBP,
+    TCG_REG_RBX,
+#if defined(_WIN64)
+    TCG_REG_RDI,
+    TCG_REG_RSI,
+#endif
+    TCG_REG_R12,
+    TCG_REG_R13,
+    TCG_REG_R14, /* Currently used for the global env. */
+    TCG_REG_R15,
+#else
+    TCG_REG_EBP, /* Currently used for the global env. */
+    TCG_REG_EBX,
+    TCG_REG_ESI,
+    TCG_REG_EDI,
+#endif
+};
+
+/* Compute frame size via macros, to share between tcg_target_qemu_prologue
+   and tcg_register_jit.  */
+
+#define PUSH_SIZE \
+    ((1 + ARRAY_SIZE(tcg_target_callee_save_regs)) \
+     * (TCG_TARGET_REG_BITS / 8))
+
+#define FRAME_SIZE \
+    ((PUSH_SIZE \
+      + TCG_STATIC_CALL_ARGS_SIZE \
+      + CPU_TEMP_BUF_NLONGS * sizeof(long) \
+      + TCG_TARGET_STACK_ALIGN - 1) \
+     & ~(TCG_TARGET_STACK_ALIGN - 1))
+
+/* Generate global QEMU prologue and epilogue code */
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    int i, stack_addend;
+
+    /* TB prologue */
+
+    /* Reserve some stack space, also for TCG temps.  */
+    stack_addend = FRAME_SIZE - PUSH_SIZE;
+    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+
+    /* Save all callee saved registers.  */
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
+        tcg_out_push(s, tcg_target_callee_save_regs[i]);
+    }
+
+#if TCG_TARGET_REG_BITS == 32
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP,
+               (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4);
+    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
+    /* jmp *tb.  */
+    tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP,
+		         (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4
+			 + stack_addend);
+#else
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
+    /* jmp *tb.  */
+    tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
+#endif
+
+    /* TB epilogue */
+    tb_ret_addr = s->code_ptr;
+
+    tcg_out_addi(s, TCG_REG_CALL_STACK, stack_addend);
+
+    for (i = ARRAY_SIZE(tcg_target_callee_save_regs) - 1; i >= 0; i--) {
+        tcg_out_pop(s, tcg_target_callee_save_regs[i]);
+    }
+    tcg_out_opc(s, OPC_RET, 0, 0, 0);
+
+#if !defined(CONFIG_SOFTMMU)
+    /* Try to set up a segment register to point to guest_base.  */
+    if (guest_base) {
+        setup_guest_base_seg();
+    }
+#endif
+}
+
+static void tcg_target_init(TCGContext *s)
+{
+#ifdef CONFIG_CPUID_H
+    unsigned a, b, c, d;
+    int max = __get_cpuid_max(0, 0);
+
+    if (max >= 1) {
+        __cpuid(1, a, b, c, d);
+#ifndef have_cmov
+        /* For 32-bit, 99% certainty that we're running on hardware that
+           supports cmov, but we still need to check.  In case cmov is not
+           available, we'll use a small forward branch.  */
+        have_cmov = (d & bit_CMOV) != 0;
+#endif
+#ifndef have_movbe
+        /* MOVBE is only available on Intel Atom and Haswell CPUs, so we
+           need to probe for it.  */
+        have_movbe = (c & bit_MOVBE) != 0;
+#endif
+    }
+
+    if (max >= 7) {
+        /* BMI1 is available on AMD Piledriver and Intel Haswell CPUs.  */
+        __cpuid_count(7, 0, a, b, c, d);
+#ifdef bit_BMI
+        have_bmi1 = (b & bit_BMI) != 0;
+#endif
+#ifndef have_bmi2
+        have_bmi2 = (b & bit_BMI2) != 0;
+#endif
+    }
+#endif
+
+    if (TCG_TARGET_REG_BITS == 64) {
+        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
+        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffff);
+    } else {
+        tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xff);
+    }
+
+    tcg_regset_clear(tcg_target_call_clobber_regs);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EAX);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_EDX);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_ECX);
+    if (TCG_TARGET_REG_BITS == 64) {
+#if !defined(_WIN64)
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RDI);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_RSI);
+#endif
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
+        tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
+    }
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
+
+    tcg_add_target_add_op_defs(x86_op_defs);
+}
+
+typedef struct {
+    DebugFrameHeader h;
+    uint8_t fde_def_cfa[4];
+    uint8_t fde_reg_ofs[14];
+} DebugFrame;
+
+/* We're expecting a 2 byte uleb128 encoded value.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
+
+#if !defined(__ELF__)
+    /* Host machine without ELF. */
+#elif TCG_TARGET_REG_BITS == 64
+#define ELF_HOST_MACHINE EM_X86_64
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x78,             /* sleb128 -8 */
+    .h.cie.return_column = 16,
+
+    /* Total FDE size does not include the "len" member.  */
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, 7,                          /* DW_CFA_def_cfa %rsp, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        0x90, 1,                        /* DW_CFA_offset, %rip, -8 */
+        /* The following ordering must match tcg_target_callee_save_regs.  */
+        0x86, 2,                        /* DW_CFA_offset, %rbp, -16 */
+        0x83, 3,                        /* DW_CFA_offset, %rbx, -24 */
+        0x8c, 4,                        /* DW_CFA_offset, %r12, -32 */
+        0x8d, 5,                        /* DW_CFA_offset, %r13, -40 */
+        0x8e, 6,                        /* DW_CFA_offset, %r14, -48 */
+        0x8f, 7,                        /* DW_CFA_offset, %r15, -56 */
+    }
+};
+#else
+#define ELF_HOST_MACHINE EM_386
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
+    .h.cie.return_column = 8,
+
+    /* Total FDE size does not include the "len" member.  */
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, 4,                          /* DW_CFA_def_cfa %esp, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        0x88, 1,                        /* DW_CFA_offset, %eip, -4 */
+        /* The following ordering must match tcg_target_callee_save_regs.  */
+        0x85, 2,                        /* DW_CFA_offset, %ebp, -8 */
+        0x83, 3,                        /* DW_CFA_offset, %ebx, -12 */
+        0x86, 4,                        /* DW_CFA_offset, %esi, -16 */
+        0x87, 5,                        /* DW_CFA_offset, %edi, -20 */
+    }
+};
+#endif
+
+#if defined(ELF_HOST_MACHINE)
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
+}
+#endif
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
deleted file mode 100644
index 5984406..0000000
--- a/tcg/ia64/tcg-target.c
+++ /dev/null
@@ -1,2450 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2009-2010 Aurelien Jarno <aurelien@aurel32.net>
- * Based on i386/tcg-target.c - Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-/*
- * Register definitions
- */
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
-     "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
-    "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
-    "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
-    "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
-    "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
-    "r48", "r49", "r50", "r51", "r52", "r53", "r54", "r55",
-    "r56", "r57", "r58", "r59", "r60", "r61", "r62", "r63",
-};
-#endif
-
-#ifdef CONFIG_USE_GUEST_BASE
-#define TCG_GUEST_BASE_REG TCG_REG_R55
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
-#endif
-
-/* Branch registers */
-enum {
-    TCG_REG_B0 = 0,
-    TCG_REG_B1,
-    TCG_REG_B2,
-    TCG_REG_B3,
-    TCG_REG_B4,
-    TCG_REG_B5,
-    TCG_REG_B6,
-    TCG_REG_B7,
-};
-
-/* Floating point registers */
-enum {
-    TCG_REG_F0 = 0,
-    TCG_REG_F1,
-    TCG_REG_F2,
-    TCG_REG_F3,
-    TCG_REG_F4,
-    TCG_REG_F5,
-    TCG_REG_F6,
-    TCG_REG_F7,
-    TCG_REG_F8,
-    TCG_REG_F9,
-    TCG_REG_F10,
-    TCG_REG_F11,
-    TCG_REG_F12,
-    TCG_REG_F13,
-    TCG_REG_F14,
-    TCG_REG_F15,
-};
-
-/* Predicate registers */
-enum {
-    TCG_REG_P0 = 0,
-    TCG_REG_P1,
-    TCG_REG_P2,
-    TCG_REG_P3,
-    TCG_REG_P4,
-    TCG_REG_P5,
-    TCG_REG_P6,
-    TCG_REG_P7,
-    TCG_REG_P8,
-    TCG_REG_P9,
-    TCG_REG_P10,
-    TCG_REG_P11,
-    TCG_REG_P12,
-    TCG_REG_P13,
-    TCG_REG_P14,
-    TCG_REG_P15,
-};
-
-/* Application registers */
-enum {
-    TCG_REG_PFS = 64,
-};
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_R35,
-    TCG_REG_R36,
-    TCG_REG_R37,
-    TCG_REG_R38,
-    TCG_REG_R39,
-    TCG_REG_R40,
-    TCG_REG_R41,
-    TCG_REG_R42,
-    TCG_REG_R43,
-    TCG_REG_R44,
-    TCG_REG_R45,
-    TCG_REG_R46,
-    TCG_REG_R47,
-    TCG_REG_R48,
-    TCG_REG_R49,
-    TCG_REG_R50,
-    TCG_REG_R51,
-    TCG_REG_R52,
-    TCG_REG_R53,
-    TCG_REG_R54,
-    TCG_REG_R55,
-    TCG_REG_R14,
-    TCG_REG_R15,
-    TCG_REG_R16,
-    TCG_REG_R17,
-    TCG_REG_R18,
-    TCG_REG_R19,
-    TCG_REG_R20,
-    TCG_REG_R21,
-    TCG_REG_R22,
-    TCG_REG_R23,
-    TCG_REG_R24,
-    TCG_REG_R25,
-    TCG_REG_R26,
-    TCG_REG_R27,
-    TCG_REG_R28,
-    TCG_REG_R29,
-    TCG_REG_R30,
-    TCG_REG_R31,
-    TCG_REG_R56,
-    TCG_REG_R57,
-    TCG_REG_R58,
-    TCG_REG_R59,
-    TCG_REG_R60,
-    TCG_REG_R61,
-    TCG_REG_R62,
-    TCG_REG_R63,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-    TCG_REG_R11
-};
-
-static const int tcg_target_call_iarg_regs[8] = {
-    TCG_REG_R56,
-    TCG_REG_R57,
-    TCG_REG_R58,
-    TCG_REG_R59,
-    TCG_REG_R60,
-    TCG_REG_R61,
-    TCG_REG_R62,
-    TCG_REG_R63,
-};
-
-static const int tcg_target_call_oarg_regs[] = {
-    TCG_REG_R8
-};
-
-/*
- * opcode formation
- */
-
-/* bundle templates: stops (double bar in the IA64 manual) are marked with
-   an uppercase letter. */
-enum {
-    mii = 0x00,
-    miI = 0x01,
-    mIi = 0x02,
-    mII = 0x03,
-    mlx = 0x04,
-    mLX = 0x05,
-    mmi = 0x08,
-    mmI = 0x09,
-    Mmi = 0x0a,
-    MmI = 0x0b,
-    mfi = 0x0c,
-    mfI = 0x0d,
-    mmf = 0x0e,
-    mmF = 0x0f,
-    mib = 0x10,
-    miB = 0x11,
-    mbb = 0x12,
-    mbB = 0x13,
-    bbb = 0x16,
-    bbB = 0x17,
-    mmb = 0x18,
-    mmB = 0x19,
-    mfb = 0x1c,
-    mfB = 0x1d,
-};
-
-enum {
-    OPC_ADD_A1                = 0x10000000000ull,
-    OPC_AND_A1                = 0x10060000000ull,
-    OPC_AND_A3                = 0x10160000000ull,
-    OPC_ANDCM_A1              = 0x10068000000ull,
-    OPC_ANDCM_A3              = 0x10168000000ull,
-    OPC_ADDS_A4               = 0x10800000000ull,
-    OPC_ADDL_A5               = 0x12000000000ull,
-    OPC_ALLOC_M34             = 0x02c00000000ull,
-    OPC_BR_DPTK_FEW_B1        = 0x08400000000ull,
-    OPC_BR_SPTK_MANY_B1       = 0x08000001000ull,
-    OPC_BR_CALL_SPNT_FEW_B3   = 0x0a200000000ull,
-    OPC_BR_SPTK_MANY_B4       = 0x00100001000ull,
-    OPC_BR_CALL_SPTK_MANY_B5  = 0x02100001000ull,
-    OPC_BR_RET_SPTK_MANY_B4   = 0x00108001100ull,
-    OPC_BRL_SPTK_MANY_X3      = 0x18000001000ull,
-    OPC_BRL_CALL_SPNT_MANY_X4 = 0x1a200001000ull,
-    OPC_BRL_CALL_SPTK_MANY_X4 = 0x1a000001000ull,
-    OPC_CMP_LT_A6             = 0x18000000000ull,
-    OPC_CMP_LTU_A6            = 0x1a000000000ull,
-    OPC_CMP_EQ_A6             = 0x1c000000000ull,
-    OPC_CMP4_LT_A6            = 0x18400000000ull,
-    OPC_CMP4_LTU_A6           = 0x1a400000000ull,
-    OPC_CMP4_EQ_A6            = 0x1c400000000ull,
-    OPC_DEP_I14               = 0x0ae00000000ull,
-    OPC_DEP_I15               = 0x08000000000ull,
-    OPC_DEP_Z_I12             = 0x0a600000000ull,
-    OPC_EXTR_I11              = 0x0a400002000ull,
-    OPC_EXTR_U_I11            = 0x0a400000000ull,
-    OPC_FCVT_FX_TRUNC_S1_F10  = 0x004d0000000ull,
-    OPC_FCVT_FXU_TRUNC_S1_F10 = 0x004d8000000ull,
-    OPC_FCVT_XF_F11           = 0x000e0000000ull,
-    OPC_FMA_S1_F1             = 0x10400000000ull,
-    OPC_FNMA_S1_F1            = 0x18400000000ull,
-    OPC_FRCPA_S1_F6           = 0x00600000000ull,
-    OPC_GETF_SIG_M19          = 0x08708000000ull,
-    OPC_LD1_M1                = 0x08000000000ull,
-    OPC_LD1_M3                = 0x0a000000000ull,
-    OPC_LD2_M1                = 0x08040000000ull,
-    OPC_LD2_M3                = 0x0a040000000ull,
-    OPC_LD4_M1                = 0x08080000000ull,
-    OPC_LD4_M3                = 0x0a080000000ull,
-    OPC_LD8_M1                = 0x080c0000000ull,
-    OPC_LD8_M3                = 0x0a0c0000000ull,
-    OPC_MUX1_I3               = 0x0eca0000000ull,
-    OPC_NOP_B9                = 0x04008000000ull,
-    OPC_NOP_F16               = 0x00008000000ull,
-    OPC_NOP_I18               = 0x00008000000ull,
-    OPC_NOP_M48               = 0x00008000000ull,
-    OPC_MOV_I21               = 0x00e00100000ull,
-    OPC_MOV_RET_I21           = 0x00e00500000ull,
-    OPC_MOV_I22               = 0x00188000000ull,
-    OPC_MOV_I_I26             = 0x00150000000ull,
-    OPC_MOVL_X2               = 0x0c000000000ull,
-    OPC_OR_A1                 = 0x10070000000ull,
-    OPC_OR_A3                 = 0x10170000000ull,
-    OPC_SETF_EXP_M18          = 0x0c748000000ull,
-    OPC_SETF_SIG_M18          = 0x0c708000000ull,
-    OPC_SHL_I7                = 0x0f240000000ull,
-    OPC_SHR_I5                = 0x0f220000000ull,
-    OPC_SHR_U_I5              = 0x0f200000000ull,
-    OPC_SHRP_I10              = 0x0ac00000000ull,
-    OPC_SXT1_I29              = 0x000a0000000ull,
-    OPC_SXT2_I29              = 0x000a8000000ull,
-    OPC_SXT4_I29              = 0x000b0000000ull,
-    OPC_ST1_M4                = 0x08c00000000ull,
-    OPC_ST2_M4                = 0x08c40000000ull,
-    OPC_ST4_M4                = 0x08c80000000ull,
-    OPC_ST8_M4                = 0x08cc0000000ull,
-    OPC_SUB_A1                = 0x10028000000ull,
-    OPC_SUB_A3                = 0x10128000000ull,
-    OPC_UNPACK4_L_I2          = 0x0f860000000ull,
-    OPC_XMA_L_F2              = 0x1d000000000ull,
-    OPC_XOR_A1                = 0x10078000000ull,
-    OPC_XOR_A3                = 0x10178000000ull,
-    OPC_ZXT1_I29              = 0x00080000000ull,
-    OPC_ZXT2_I29              = 0x00088000000ull,
-    OPC_ZXT4_I29              = 0x00090000000ull,
-
-    INSN_NOP_M                = OPC_NOP_M48,  /* nop.m 0 */
-    INSN_NOP_I                = OPC_NOP_I18,  /* nop.i 0 */
-};
-
-static inline uint64_t tcg_opc_a1(int qp, uint64_t opc, int r1,
-                                  int r2, int r3)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_a3(int qp, uint64_t opc, int r1,
-                                  uint64_t imm, int r3)
-{
-    return opc
-           | ((imm & 0x80) << 29) /* s */
-           | ((imm & 0x7f) << 13) /* imm7b */
-           | ((r3 & 0x7f) << 20)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_a4(int qp, uint64_t opc, int r1,
-                                  uint64_t imm, int r3)
-{
-    return opc
-           | ((imm & 0x2000) << 23) /* s */
-           | ((imm & 0x1f80) << 20) /* imm6d */
-           | ((imm & 0x007f) << 13) /* imm7b */
-           | ((r3 & 0x7f) << 20)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_a5(int qp, uint64_t opc, int r1,
-                                  uint64_t imm, int r3)
-{
-    return opc
-           | ((imm & 0x200000) << 15) /* s */
-           | ((imm & 0x1f0000) <<  6) /* imm5c */
-           | ((imm & 0x00ff80) << 20) /* imm9d */
-           | ((imm & 0x00007f) << 13) /* imm7b */
-           | ((r3 & 0x03) << 20)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_a6(int qp, uint64_t opc, int p1,
-                                  int p2, int r2, int r3)
-{
-    return opc
-           | ((p2 & 0x3f) << 27)
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((p1 & 0x3f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_b1(int qp, uint64_t opc, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100000) << 16) /* s */
-           | ((imm & 0x0fffff) << 13) /* imm20b */
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_b3(int qp, uint64_t opc, int b1, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100000) << 16) /* s */
-           | ((imm & 0x0fffff) << 13) /* imm20b */
-           | ((b1 & 0x7) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_b4(int qp, uint64_t opc, int b2)
-{
-    return opc
-           | ((b2 & 0x7) << 13)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_b5(int qp, uint64_t opc, int b1, int b2)
-{
-    return opc
-           | ((b2 & 0x7) << 13)
-           | ((b1 & 0x7) << 6)
-           | (qp & 0x3f);
-}
-
-
-static inline uint64_t tcg_opc_b9(int qp, uint64_t opc, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100000) << 16) /* i */
-           | ((imm & 0x0fffff) << 6)  /* imm20a */
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_f1(int qp, uint64_t opc, int f1,
-                                  int f3, int f4, int f2)
-{
-    return opc
-           | ((f4 & 0x7f) << 27)
-           | ((f3 & 0x7f) << 20)
-           | ((f2 & 0x7f) << 13)
-           | ((f1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_f2(int qp, uint64_t opc, int f1,
-                                  int f3, int f4, int f2)
-{
-    return opc
-           | ((f4 & 0x7f) << 27)
-           | ((f3 & 0x7f) << 20)
-           | ((f2 & 0x7f) << 13)
-           | ((f1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_f6(int qp, uint64_t opc, int f1,
-                                  int p2, int f2, int f3)
-{
-    return opc
-           | ((p2 & 0x3f) << 27)
-           | ((f3 & 0x7f) << 20)
-           | ((f2 & 0x7f) << 13)
-           | ((f1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_f10(int qp, uint64_t opc, int f1, int f2)
-{
-    return opc
-           | ((f2 & 0x7f) << 13)
-           | ((f1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_f11(int qp, uint64_t opc, int f1, int f2)
-{
-    return opc
-           | ((f2 & 0x7f) << 13)
-           | ((f1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_f16(int qp, uint64_t opc, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100000) << 16) /* i */
-           | ((imm & 0x0fffff) << 6)  /* imm20a */
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i2(int qp, uint64_t opc, int r1,
-                                  int r2, int r3)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i3(int qp, uint64_t opc, int r1,
-                                  int r2, int mbtype)
-{
-    return opc
-           | ((mbtype & 0x0f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i5(int qp, uint64_t opc, int r1,
-                                  int r3, int r2)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i7(int qp, uint64_t opc, int r1,
-                                  int r2, int r3)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i10(int qp, uint64_t opc, int r1,
-                                   int r2, int r3, uint64_t count)
-{
-    return opc
-           | ((count & 0x3f) << 27)
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i11(int qp, uint64_t opc, int r1,
-                                   int r3, uint64_t pos, uint64_t len)
-{
-    return opc
-           | ((len & 0x3f) << 27)
-           | ((r3 & 0x7f) << 20)
-           | ((pos & 0x3f) << 14)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i12(int qp, uint64_t opc, int r1,
-                                   int r2, uint64_t pos, uint64_t len)
-{
-    return opc
-           | ((len & 0x3f) << 27)
-           | ((pos & 0x3f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i14(int qp, uint64_t opc, int r1, uint64_t imm,
-                                   int r3, uint64_t pos, uint64_t len)
-{
-    return opc
-           | ((imm & 0x01) << 36)
-           | ((len & 0x3f) << 27)
-           | ((r3 & 0x7f) << 20)
-           | ((pos & 0x3f) << 14)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i15(int qp, uint64_t opc, int r1, int r2,
-                                   int r3, uint64_t pos, uint64_t len)
-{
-    return opc
-           | ((pos & 0x3f) << 31)
-           | ((len & 0x0f) << 27)
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i18(int qp, uint64_t opc, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100000) << 16) /* i */
-           | ((imm & 0x0fffff) << 6)  /* imm20a */
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i21(int qp, uint64_t opc, int b1,
-                                   int r2, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x1ff) << 24)
-           | ((r2 & 0x7f) << 13)
-           | ((b1 & 0x7) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i22(int qp, uint64_t opc, int r1, int b2)
-{
-    return opc
-           | ((b2 & 0x7) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i26(int qp, uint64_t opc, int ar3, int r2)
-{
-    return opc
-           | ((ar3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_i29(int qp, uint64_t opc, int r1, int r3)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_l2(uint64_t imm)
-{
-    return (imm & 0x7fffffffffc00000ull) >> 22;
-}
-
-static inline uint64_t tcg_opc_l3(uint64_t imm)
-{
-    return (imm & 0x07fffffffff00000ull) >> 18;
-}
-
-#define tcg_opc_l4  tcg_opc_l3
-
-static inline uint64_t tcg_opc_m1(int qp, uint64_t opc, int r1, int r3)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_m3(int qp, uint64_t opc, int r1,
-                                  int r3, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100) << 28) /* s */
-           | ((imm & 0x080) << 20) /* i */
-           | ((imm & 0x07f) << 13) /* imm7b */
-           | ((r3 & 0x7f) << 20)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_m4(int qp, uint64_t opc, int r2, int r3)
-{
-    return opc
-           | ((r3 & 0x7f) << 20)
-           | ((r2 & 0x7f) << 13)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_m18(int qp, uint64_t opc, int f1, int r2)
-{
-    return opc
-           | ((r2 & 0x7f) << 13)
-           | ((f1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_m19(int qp, uint64_t opc, int r1, int f2)
-{
-    return opc
-           | ((f2 & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_m34(int qp, uint64_t opc, int r1,
-                                   int sof, int sol, int sor)
-{
-    return opc
-           | ((sor & 0x0f) << 27)
-           | ((sol & 0x7f) << 20)
-           | ((sof & 0x7f) << 13)
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_m48(int qp, uint64_t opc, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x100000) << 16) /* i */
-           | ((imm & 0x0fffff) << 6)  /* imm20a */
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_x2(int qp, uint64_t opc,
-                                  int r1, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x8000000000000000ull) >> 27) /* i */
-           |  (imm & 0x0000000000200000ull)        /* ic */
-           | ((imm & 0x00000000001f0000ull) << 6)  /* imm5c */
-           | ((imm & 0x000000000000ff80ull) << 20) /* imm9d */
-           | ((imm & 0x000000000000007full) << 13) /* imm7b */
-           | ((r1 & 0x7f) << 6)
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_x3(int qp, uint64_t opc, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x0800000000000000ull) >> 23) /* i */
-           | ((imm & 0x00000000000fffffull) << 13) /* imm20b */
-           | (qp & 0x3f);
-}
-
-static inline uint64_t tcg_opc_x4(int qp, uint64_t opc, int b1, uint64_t imm)
-{
-    return opc
-           | ((imm & 0x0800000000000000ull) >> 23) /* i */
-           | ((imm & 0x00000000000fffffull) << 13) /* imm20b */
-           | ((b1 & 0x7) << 6)
-           | (qp & 0x3f);
-}
-
-
-/*
- * Relocations - Note that we never encode branches elsewhere than slot 2.
- */
-
-static void reloc_pcrel21b_slot2(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    uint64_t imm = target - pc;
-
-    pc->hi = (pc->hi & 0xf700000fffffffffull)
-             | ((imm & 0x100000) << 39)  /* s */
-             | ((imm & 0x0fffff) << 36); /* imm20b */
-}
-
-static uint64_t get_reloc_pcrel21b_slot2(tcg_insn_unit *pc)
-{
-    int64_t high = pc->hi;
-
-    return ((high >> 39) & 0x100000) + /* s */
-           ((high >> 36) & 0x0fffff);  /* imm20b */
-}
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    assert(addend == 0);
-    assert(type == R_IA64_PCREL21B);
-    reloc_pcrel21b_slot2(code_ptr, (tcg_insn_unit *)value);
-}
-
-/*
- * Constraints
- */
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch(ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set(ct->u.regs, 0xffffffffffffffffull);
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_S22;
-        break;
-    case 'S':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set(ct->u.regs, 0xffffffffffffffffull);
-#if defined(CONFIG_SOFTMMU)
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R56);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R57);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R58);
-#endif
-        break;
-    case 'Z':
-        /* We are cheating a bit here, using the fact that the register
-           r0 is also the register number 0. Hence there is no need
-           to check for const_args in each instruction. */
-        ct->ct |= TCG_CT_CONST_ZERO;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct;
-    ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST)
-        return 1;
-    else if ((ct & TCG_CT_CONST_ZERO) && val == 0)
-        return 1;
-    else if ((ct & TCG_CT_CONST_S22) && val == ((int32_t)val << 10) >> 10)
-        return 1;
-    else
-        return 0;
-}
-
-/*
- * Code generation
- */
-
-static tcg_insn_unit *tb_ret_addr;
-
-static inline void tcg_out_bundle(TCGContext *s, int template,
-                                  uint64_t slot0, uint64_t slot1,
-                                  uint64_t slot2)
-{
-    template &= 0x1f;          /* 5 bits */
-    slot0 &= 0x1ffffffffffull; /* 41 bits */
-    slot1 &= 0x1ffffffffffull; /* 41 bits */
-    slot2 &= 0x1ffffffffffull; /* 41 bits */
-
-    *s->code_ptr++ = (tcg_insn_unit){
-        (slot1 << 46) | (slot0 << 5) | template,
-        (slot2 << 23) | (slot1 >> 18)
-    };
-}
-
-static inline uint64_t tcg_opc_mov_a(int qp, TCGReg dst, TCGReg src)
-{
-    return tcg_opc_a4(qp, OPC_ADDS_A4, dst, 0, src);
-}
-
-static inline void tcg_out_mov(TCGContext *s, TCGType type,
-                               TCGReg ret, TCGReg arg)
-{
-    tcg_out_bundle(s, mmI,
-                   INSN_NOP_M,
-                   INSN_NOP_M,
-                   tcg_opc_mov_a(TCG_REG_P0, ret, arg));
-}
-
-static inline uint64_t tcg_opc_movi_a(int qp, TCGReg dst, int64_t src)
-{
-    assert(src == sextract64(src, 0, 22));
-    return tcg_opc_a5(qp, OPC_ADDL_A5, dst, src, TCG_REG_R0);
-}
-
-static inline void tcg_out_movi(TCGContext *s, TCGType type,
-                                TCGReg reg, tcg_target_long arg)
-{
-    tcg_out_bundle(s, mLX,
-                   INSN_NOP_M,
-                   tcg_opc_l2 (arg),
-                   tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2, reg, arg));
-}
-
-static void tcg_out_br(TCGContext *s, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-    uint64_t imm;
-
-    /* We pay attention here to not modify the branch target by reading
-       the existing value and using it again. This ensure that caches and
-       memory are kept coherent during retranslation. */
-    if (l->has_value) {
-        imm = l->u.value_ptr -  s->code_ptr;
-    } else {
-        imm = get_reloc_pcrel21b_slot2(s->code_ptr);
-        tcg_out_reloc(s, s->code_ptr, R_IA64_PCREL21B, label_index, 0);
-    }
-
-    tcg_out_bundle(s, mmB,
-                   INSN_NOP_M,
-                   INSN_NOP_M,
-                   tcg_opc_b1(TCG_REG_P0, OPC_BR_SPTK_MANY_B1, imm));
-}
-
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *desc)
-{
-    uintptr_t func = desc->lo, gp = desc->hi, disp;
-
-    /* Look through the function descriptor.  */
-    tcg_out_bundle(s, mlx,
-                   INSN_NOP_M,
-                   tcg_opc_l2 (gp),
-                   tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2, TCG_REG_R1, gp));
-    disp = (tcg_insn_unit *)func - s->code_ptr;
-    tcg_out_bundle(s, mLX,
-                   INSN_NOP_M,
-                   tcg_opc_l4 (disp),
-                   tcg_opc_x4 (TCG_REG_P0, OPC_BRL_CALL_SPTK_MANY_X4,
-                               TCG_REG_B0, disp));
-}
-
-static void tcg_out_exit_tb(TCGContext *s, tcg_target_long arg)
-{
-    uint64_t imm, opc1;
-
-    /* At least arg == 0 is a common operation.  */
-    if (arg == sextract64(arg, 0, 22)) {
-        opc1 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R8, arg);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R8, arg);
-        opc1 = INSN_NOP_M;
-    }
-
-    imm = tb_ret_addr - s->code_ptr;
-
-    tcg_out_bundle(s, mLX,
-                   opc1,
-                   tcg_opc_l3 (imm),
-                   tcg_opc_x3 (TCG_REG_P0, OPC_BRL_SPTK_MANY_X3, imm));
-}
-
-static inline void tcg_out_goto_tb(TCGContext *s, TCGArg arg)
-{
-    if (s->tb_jmp_offset) {
-        /* direct jump method */
-        tcg_abort();
-    } else {
-        /* indirect jump method */
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2,
-                     (tcg_target_long)(s->tb_next + arg));
-        tcg_out_bundle(s, MmI,
-                       tcg_opc_m1 (TCG_REG_P0, OPC_LD8_M1,
-                                   TCG_REG_R2, TCG_REG_R2),
-                       INSN_NOP_M,
-                       tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21, TCG_REG_B6,
-                                   TCG_REG_R2, 0));
-        tcg_out_bundle(s, mmB,
-                       INSN_NOP_M,
-                       INSN_NOP_M,
-                       tcg_opc_b4 (TCG_REG_P0, OPC_BR_SPTK_MANY_B4,
-                                   TCG_REG_B6));
-    }
-    s->tb_next_offset[arg] = tcg_current_code_size(s);
-}
-
-static inline void tcg_out_jmp(TCGContext *s, TCGArg addr)
-{
-    tcg_out_bundle(s, mmI,
-                   INSN_NOP_M,
-                   INSN_NOP_M,
-                   tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21, TCG_REG_B6, addr, 0));
-    tcg_out_bundle(s, mmB,
-                   INSN_NOP_M,
-                   INSN_NOP_M,
-                   tcg_opc_b4(TCG_REG_P0, OPC_BR_SPTK_MANY_B4, TCG_REG_B6));
-}
-
-static inline void tcg_out_ld_rel(TCGContext *s, uint64_t opc_m4, TCGArg arg,
-                                  TCGArg arg1, tcg_target_long arg2)
-{
-    if (arg2 == ((int16_t)arg2 >> 2) << 2) {
-        tcg_out_bundle(s, MmI,
-                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4,
-                                  TCG_REG_R2, arg2, arg1),
-                       tcg_opc_m1 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
-                       INSN_NOP_I);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, arg2);
-        tcg_out_bundle(s, MmI,
-                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1,
-                                   TCG_REG_R2, TCG_REG_R2, arg1),
-                       tcg_opc_m1 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
-                       INSN_NOP_I);
-    }
-}
-
-static inline void tcg_out_st_rel(TCGContext *s, uint64_t opc_m4, TCGArg arg,
-                                  TCGArg arg1, tcg_target_long arg2)
-{
-    if (arg2 == ((int16_t)arg2 >> 2) << 2) {
-        tcg_out_bundle(s, MmI,
-                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4,
-                                  TCG_REG_R2, arg2, arg1),
-                       tcg_opc_m4 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
-                       INSN_NOP_I);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, arg2);
-        tcg_out_bundle(s, MmI,
-                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1,
-                                   TCG_REG_R2, TCG_REG_R2, arg1),
-                       tcg_opc_m4 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
-                       INSN_NOP_I);
-    }
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    if (type == TCG_TYPE_I32) {
-        tcg_out_ld_rel(s, OPC_LD4_M1, arg, arg1, arg2);
-    } else {
-        tcg_out_ld_rel(s, OPC_LD8_M1, arg, arg1, arg2);
-    }
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    if (type == TCG_TYPE_I32) {
-        tcg_out_st_rel(s, OPC_ST4_M4, arg, arg1, arg2);
-    } else {
-        tcg_out_st_rel(s, OPC_ST8_M4, arg, arg1, arg2);
-    }
-}
-
-static inline void tcg_out_alu(TCGContext *s, uint64_t opc_a1, uint64_t opc_a3,
-                               TCGReg ret, TCGArg arg1, int const_arg1,
-                               TCGArg arg2, int const_arg2)
-{
-    uint64_t opc1 = 0, opc2 = 0, opc3 = 0;
-
-    if (const_arg2 && arg2 != 0) {
-        opc2 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R3, arg2);
-        arg2 = TCG_REG_R3;
-    }
-    if (const_arg1 && arg1 != 0) {
-        if (opc_a3 && arg1 == (int8_t)arg1) {
-            opc3 = tcg_opc_a3(TCG_REG_P0, opc_a3, ret, arg1, arg2);
-        } else {
-            opc1 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, arg1);
-            arg1 = TCG_REG_R2;
-        }
-    }
-    if (opc3 == 0) {
-        opc3 = tcg_opc_a1(TCG_REG_P0, opc_a1, ret, arg1, arg2);
-    }
-
-    tcg_out_bundle(s, (opc1 || opc2 ? mII : miI),
-                   opc1 ? opc1 : INSN_NOP_M,
-                   opc2 ? opc2 : INSN_NOP_I,
-                   opc3);
-}
-
-static inline void tcg_out_add(TCGContext *s, TCGReg ret, TCGReg arg1,
-                               TCGArg arg2, int const_arg2)
-{
-    if (const_arg2 && arg2 == sextract64(arg2, 0, 14)) {
-        tcg_out_bundle(s, mmI,
-                       INSN_NOP_M,
-                       INSN_NOP_M,
-                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, ret, arg2, arg1));
-    } else {
-        tcg_out_alu(s, OPC_ADD_A1, 0, ret, arg1, 0, arg2, const_arg2);
-    }
-}
-
-static inline void tcg_out_sub(TCGContext *s, TCGReg ret, TCGArg arg1,
-                               int const_arg1, TCGArg arg2, int const_arg2)
-{
-    if (!const_arg1 && const_arg2 && -arg2 == sextract64(-arg2, 0, 14)) {
-        tcg_out_bundle(s, mmI,
-                       INSN_NOP_M,
-                       INSN_NOP_M,
-                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, ret, -arg2, arg1));
-    } else {
-        tcg_out_alu(s, OPC_SUB_A1, OPC_SUB_A3, ret,
-                    arg1, const_arg1, arg2, const_arg2);
-    }
-}
-
-static inline void tcg_out_eqv(TCGContext *s, TCGArg ret,
-                               TCGArg arg1, int const_arg1,
-                               TCGArg arg2, int const_arg2)
-{
-    tcg_out_bundle(s, mII,
-                   INSN_NOP_M,
-                   tcg_opc_a1 (TCG_REG_P0, OPC_XOR_A1, ret, arg1, arg2),
-                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, ret, -1, ret));
-}
-
-static inline void tcg_out_nand(TCGContext *s, TCGArg ret,
-                                TCGArg arg1, int const_arg1,
-                                TCGArg arg2, int const_arg2)
-{
-    tcg_out_bundle(s, mII,
-                   INSN_NOP_M,
-                   tcg_opc_a1 (TCG_REG_P0, OPC_AND_A1, ret, arg1, arg2),
-                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, ret, -1, ret));
-}
-
-static inline void tcg_out_nor(TCGContext *s, TCGArg ret,
-                               TCGArg arg1, int const_arg1,
-                               TCGArg arg2, int const_arg2)
-{
-    tcg_out_bundle(s, mII,
-                   INSN_NOP_M,
-                   tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret, arg1, arg2),
-                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, ret, -1, ret));
-}
-
-static inline void tcg_out_orc(TCGContext *s, TCGArg ret,
-                               TCGArg arg1, int const_arg1,
-                               TCGArg arg2, int const_arg2)
-{
-    tcg_out_bundle(s, mII,
-                   INSN_NOP_M,
-                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, TCG_REG_R2, -1, arg2),
-                   tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret, arg1, TCG_REG_R2));
-}
-
-static inline void tcg_out_mul(TCGContext *s, TCGArg ret,
-                               TCGArg arg1, TCGArg arg2)
-{
-    tcg_out_bundle(s, mmI,
-                   tcg_opc_m18(TCG_REG_P0, OPC_SETF_SIG_M18, TCG_REG_F6, arg1),
-                   tcg_opc_m18(TCG_REG_P0, OPC_SETF_SIG_M18, TCG_REG_F7, arg2),
-                   INSN_NOP_I);
-    tcg_out_bundle(s, mmF,
-                   INSN_NOP_M,
-                   INSN_NOP_M,
-                   tcg_opc_f2 (TCG_REG_P0, OPC_XMA_L_F2, TCG_REG_F6, TCG_REG_F6,
-                               TCG_REG_F7, TCG_REG_F0));
-    tcg_out_bundle(s, miI,
-                   tcg_opc_m19(TCG_REG_P0, OPC_GETF_SIG_M19, ret, TCG_REG_F6),
-                   INSN_NOP_I,
-                   INSN_NOP_I);
-}
-
-static inline void tcg_out_sar_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                   TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_I11,
-                                   ret, arg1, arg2, 31 - arg2));
-    } else {
-        tcg_out_bundle(s, mII,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3,
-                                   TCG_REG_R3, 0x1f, arg2),
-                       tcg_opc_i29(TCG_REG_P0, OPC_SXT4_I29, TCG_REG_R2, arg1),
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_I5, ret,
-                                   TCG_REG_R2, TCG_REG_R3));
-    }
-}
-
-static inline void tcg_out_sar_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                   TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_I11,
-                                   ret, arg1, arg2, 63 - arg2));
-    } else {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_I5, ret, arg1, arg2));
-    }
-}
-
-static inline void tcg_out_shl_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                   TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret,
-                                   arg1, 63 - arg2, 31 - arg2));
-    } else {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R2,
-                                   0x1f, arg2),
-                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, ret,
-                                   arg1, TCG_REG_R2));
-    }
-}
-
-static inline void tcg_out_shl_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                   TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret,
-                                   arg1, 63 - arg2, 63 - arg2));
-    } else {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, ret,
-                                   arg1, arg2));
-    }
-}
-
-static inline void tcg_out_shr_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                   TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
-                                   arg1, arg2, 31 - arg2));
-    } else {
-        tcg_out_bundle(s, mII,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R3,
-                                   0x1f, arg2),
-                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29, TCG_REG_R2, arg1),
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
-                                   TCG_REG_R2, TCG_REG_R3));
-    }
-}
-
-static inline void tcg_out_shr_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                   TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
-                                   arg1, arg2, 63 - arg2));
-    } else {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
-                                   arg1, arg2));
-    }
-}
-
-static inline void tcg_out_rotl_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                    TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
-                                   TCG_REG_R2, arg1, arg1),
-                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
-                                   TCG_REG_R2, 32 - arg2, 31));
-    } else {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
-                                   TCG_REG_R2, arg1, arg1),
-                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R3,
-                                   0x1f, arg2));
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_SUB_A3, TCG_REG_R3,
-                                   0x20, TCG_REG_R3),
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
-                                   TCG_REG_R2, TCG_REG_R3));
-    }
-}
-
-static inline void tcg_out_rotl_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                    TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i10(TCG_REG_P0, OPC_SHRP_I10, ret, arg1,
-                                   arg1, 0x40 - arg2));
-    } else {
-        tcg_out_bundle(s, mII,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_SUB_A3, TCG_REG_R2,
-                                   0x40, arg2),
-                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, TCG_REG_R3,
-                                   arg1, arg2),
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, TCG_REG_R2,
-                                   arg1, TCG_REG_R2));
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret,
-                                   TCG_REG_R2, TCG_REG_R3));
-    }
-}
-
-static inline void tcg_out_rotr_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                    TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
-                                   TCG_REG_R2, arg1, arg1),
-                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
-                                   TCG_REG_R2, arg2, 31));
-    } else {
-        tcg_out_bundle(s, mII,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R3,
-                                   0x1f, arg2),
-                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
-                                   TCG_REG_R2, arg1, arg1),
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
-                                   TCG_REG_R2, TCG_REG_R3));
-    }
-}
-
-static inline void tcg_out_rotr_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
-                                    TCGArg arg2, int const_arg2)
-{
-    if (const_arg2) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_i10(TCG_REG_P0, OPC_SHRP_I10, ret, arg1,
-                                   arg1, arg2));
-    } else {
-        tcg_out_bundle(s, mII,
-                       tcg_opc_a3 (TCG_REG_P0, OPC_SUB_A3, TCG_REG_R2,
-                                   0x40, arg2),
-                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, TCG_REG_R3,
-                                   arg1, arg2),
-                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, TCG_REG_R2,
-                                   arg1, TCG_REG_R2));
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret,
-                                   TCG_REG_R2, TCG_REG_R3));
-    }
-}
-
-static const uint64_t opc_ext_i29[8] = {
-    OPC_ZXT1_I29, OPC_ZXT2_I29, OPC_ZXT4_I29, 0,
-    OPC_SXT1_I29, OPC_SXT2_I29, OPC_SXT4_I29, 0
-};
-
-static inline uint64_t tcg_opc_ext_i(int qp, TCGMemOp opc, TCGReg d, TCGReg s)
-{
-    if ((opc & MO_SIZE) == MO_64) {
-        return tcg_opc_mov_a(qp, d, s);
-    } else {
-        return tcg_opc_i29(qp, opc_ext_i29[opc & MO_SSIZE], d, s);
-    }
-}
-
-static inline void tcg_out_ext(TCGContext *s, uint64_t opc_i29,
-                               TCGArg ret, TCGArg arg)
-{
-    tcg_out_bundle(s, miI,
-                   INSN_NOP_M,
-                   INSN_NOP_I,
-                   tcg_opc_i29(TCG_REG_P0, opc_i29, ret, arg));
-}
-
-static inline uint64_t tcg_opc_bswap64_i(int qp, TCGReg d, TCGReg s)
-{
-    return tcg_opc_i3(qp, OPC_MUX1_I3, d, s, 0xb);
-}
-
-static inline void tcg_out_bswap16(TCGContext *s, TCGArg ret, TCGArg arg)
-{
-    tcg_out_bundle(s, mII,
-                   INSN_NOP_M,
-                   tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret, arg, 15, 15),
-                   tcg_opc_bswap64_i(TCG_REG_P0, ret, ret));
-}
-
-static inline void tcg_out_bswap32(TCGContext *s, TCGArg ret, TCGArg arg)
-{
-    tcg_out_bundle(s, mII,
-                   INSN_NOP_M,
-                   tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret, arg, 31, 31),
-                   tcg_opc_bswap64_i(TCG_REG_P0, ret, ret));
-}
-
-static inline void tcg_out_bswap64(TCGContext *s, TCGArg ret, TCGArg arg)
-{
-    tcg_out_bundle(s, miI,
-                   INSN_NOP_M,
-                   INSN_NOP_I,
-                   tcg_opc_bswap64_i(TCG_REG_P0, ret, arg));
-}
-
-static inline void tcg_out_deposit(TCGContext *s, TCGArg ret, TCGArg a1,
-                                   TCGArg a2, int const_a2, int pos, int len)
-{
-    uint64_t i1 = 0, i2 = 0;
-    int cpos = 63 - pos, lm1 = len - 1;
-
-    if (const_a2) {
-        /* Truncate the value of a constant a2 to the width of the field.  */
-        int mask = (1u << len) - 1;
-        a2 &= mask;
-
-        if (a2 == 0 || a2 == mask) {
-            /* 1-bit signed constant inserted into register.  */
-            i2 = tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, ret, a2, a1, cpos, lm1);
-        } else {
-            /* Otherwise, load any constant into a temporary.  Do this into
-               the first I slot to help out with cross-unit delays.  */
-            i1 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, a2);
-            a2 = TCG_REG_R2;
-        }
-    }
-    if (i2 == 0) {
-        i2 = tcg_opc_i15(TCG_REG_P0, OPC_DEP_I15, ret, a2, a1, cpos, lm1);
-    }
-    tcg_out_bundle(s, (i1 ? mII : miI),
-                   INSN_NOP_M,
-                   i1 ? i1 : INSN_NOP_I,
-                   i2);
-}
-
-static inline uint64_t tcg_opc_cmp_a(int qp, TCGCond cond, TCGArg arg1,
-                                     TCGArg arg2, int cmp4)
-{
-    uint64_t opc_eq_a6, opc_lt_a6, opc_ltu_a6;
-
-    if (cmp4) {
-        opc_eq_a6 = OPC_CMP4_EQ_A6;
-        opc_lt_a6 = OPC_CMP4_LT_A6;
-        opc_ltu_a6 = OPC_CMP4_LTU_A6;
-    } else {
-        opc_eq_a6 = OPC_CMP_EQ_A6;
-        opc_lt_a6 = OPC_CMP_LT_A6;
-        opc_ltu_a6 = OPC_CMP_LTU_A6;
-    }
-
-    switch (cond) {
-    case TCG_COND_EQ:
-        return tcg_opc_a6 (qp, opc_eq_a6,  TCG_REG_P6, TCG_REG_P7, arg1, arg2);
-    case TCG_COND_NE:
-        return tcg_opc_a6 (qp, opc_eq_a6,  TCG_REG_P7, TCG_REG_P6, arg1, arg2);
-    case TCG_COND_LT:
-        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P6, TCG_REG_P7, arg1, arg2);
-    case TCG_COND_LTU:
-        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P6, TCG_REG_P7, arg1, arg2);
-    case TCG_COND_GE:
-        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P7, TCG_REG_P6, arg1, arg2);
-    case TCG_COND_GEU:
-        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P7, TCG_REG_P6, arg1, arg2);
-    case TCG_COND_LE:
-        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P7, TCG_REG_P6, arg2, arg1);
-    case TCG_COND_LEU:
-        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P7, TCG_REG_P6, arg2, arg1);
-    case TCG_COND_GT:
-        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P6, TCG_REG_P7, arg2, arg1);
-    case TCG_COND_GTU:
-        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P6, TCG_REG_P7, arg2, arg1);
-    default:
-        tcg_abort();
-        break;
-    }
-}
-
-static inline void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
-                                  TCGReg arg2, int label_index, int cmp4)
-{
-    TCGLabel *l = &s->labels[label_index];
-    uint64_t imm;
-
-    /* We pay attention here to not modify the branch target by reading
-       the existing value and using it again. This ensure that caches and
-       memory are kept coherent during retranslation. */
-    if (l->has_value) {
-        imm = l->u.value_ptr - s->code_ptr;
-    } else {
-        imm = get_reloc_pcrel21b_slot2(s->code_ptr);
-        tcg_out_reloc(s, s->code_ptr, R_IA64_PCREL21B, label_index, 0);
-    }
-
-    tcg_out_bundle(s, miB,
-                   INSN_NOP_M,
-                   tcg_opc_cmp_a(TCG_REG_P0, cond, arg1, arg2, cmp4),
-                   tcg_opc_b1(TCG_REG_P6, OPC_BR_DPTK_FEW_B1, imm));
-}
-
-static inline void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGArg ret,
-                                   TCGArg arg1, TCGArg arg2, int cmp4)
-{
-    tcg_out_bundle(s, MmI,
-                   tcg_opc_cmp_a(TCG_REG_P0, cond, arg1, arg2, cmp4),
-                   tcg_opc_movi_a(TCG_REG_P6, ret, 1),
-                   tcg_opc_movi_a(TCG_REG_P7, ret, 0));
-}
-
-static inline void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGArg ret,
-                                   TCGArg c1, TCGArg c2,
-                                   TCGArg v1, int const_v1,
-                                   TCGArg v2, int const_v2, int cmp4)
-{
-    uint64_t opc1, opc2;
-
-    if (const_v1) {
-        opc1 = tcg_opc_movi_a(TCG_REG_P6, ret, v1);
-    } else if (ret == v1) {
-        opc1 = INSN_NOP_M;
-    } else {
-        opc1 = tcg_opc_mov_a(TCG_REG_P6, ret, v1);
-    }
-    if (const_v2) {
-        opc2 = tcg_opc_movi_a(TCG_REG_P7, ret, v2);
-    } else if (ret == v2) {
-        opc2 = INSN_NOP_I;
-    } else {
-        opc2 = tcg_opc_mov_a(TCG_REG_P7, ret, v2);
-    }
-
-    tcg_out_bundle(s, MmI,
-                   tcg_opc_cmp_a(TCG_REG_P0, cond, c1, c2, cmp4),
-                   opc1,
-                   opc2);
-}
-
-#if defined(CONFIG_SOFTMMU)
-/* We're expecting to use an signed 22-bit immediate add.  */
-QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
-                  > 0x1fffff)
-
-/* Load and compare a TLB entry, and return the result in (p6, p7).
-   R2 is loaded with the addend TLB entry.
-   R57 is loaded with the address, zero extented on 32-bit targets.
-   R1, R3 are clobbered, leaving R56 free for...
-   BSWAP_1, BSWAP_2 and I-slot insns for swapping data for store.  */
-static inline void tcg_out_qemu_tlb(TCGContext *s, TCGReg addr_reg,
-                                    TCGMemOp s_bits, int off_rw, int off_add,
-                                    uint64_t bswap1, uint64_t bswap2)
-{
-     /*
-        .mii
-        mov	r2 = off_rw
-        extr.u	r3 = addr_reg, ...		# extract tlb page
-        zxt4	r57 = addr_reg                  # or mov for 64-bit guest
-        ;;
-        .mii
-        addl	r2 = r2, areg0
-        shl	r3 = r3, cteb                   # via dep.z
-        dep	r1 = 0, r57, ...                # zero page ofs, keep align
-        ;;
-        .mmi
-        add	r2 = r2, r3
-        ;;
-        ld4	r3 = [r2], off_add-off_rw	# or ld8 for 64-bit guest
-        nop
-        ;;
-        .mmi
-        nop
-        cmp.eq	p6, p7 = r3, r58
-        nop
-        ;;
-      */
-    tcg_out_bundle(s, miI,
-                   tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, off_rw),
-                   tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, TCG_REG_R3,
-                               addr_reg, TARGET_PAGE_BITS, CPU_TLB_BITS - 1),
-                   tcg_opc_ext_i(TCG_REG_P0,
-                                 TARGET_LONG_BITS == 32 ? MO_UL : MO_Q,
-                                 TCG_REG_R57, addr_reg));
-    tcg_out_bundle(s, miI,
-                   tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
-                               TCG_REG_R2, TCG_AREG0),
-                   tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, TCG_REG_R3,
-                               TCG_REG_R3, 63 - CPU_TLB_ENTRY_BITS,
-                               63 - CPU_TLB_ENTRY_BITS),
-                   tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, TCG_REG_R1, 0,
-                               TCG_REG_R57, 63 - s_bits,
-                               TARGET_PAGE_BITS - s_bits - 1));
-    tcg_out_bundle(s, MmI,
-                   tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1,
-                               TCG_REG_R2, TCG_REG_R2, TCG_REG_R3),
-                   tcg_opc_m3 (TCG_REG_P0,
-                               (TARGET_LONG_BITS == 32
-                                ? OPC_LD4_M3 : OPC_LD8_M3), TCG_REG_R3,
-                               TCG_REG_R2, off_add - off_rw),
-                   bswap1);
-    tcg_out_bundle(s, mmI,
-                   tcg_opc_m1 (TCG_REG_P0, OPC_LD8_M1, TCG_REG_R2, TCG_REG_R2),
-                   tcg_opc_a6 (TCG_REG_P0, OPC_CMP_EQ_A6, TCG_REG_P6,
-                               TCG_REG_P7, TCG_REG_R1, TCG_REG_R3),
-                   bswap2);
-}
-
-#define TCG_MAX_QEMU_LDST       640
-
-typedef struct TCGLabelQemuLdst {
-    bool is_ld;
-    TCGMemOp size;
-    tcg_insn_unit *label_ptr;     /* label pointers to be updated */
-} TCGLabelQemuLdst;
-
-typedef struct TCGBackendData {
-    int nb_ldst_labels;
-    TCGLabelQemuLdst ldst_labels[TCG_MAX_QEMU_LDST];
-} TCGBackendData;
-
-static inline void tcg_out_tb_init(TCGContext *s)
-{
-    s->be->nb_ldst_labels = 0;
-}
-
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
-                                tcg_insn_unit *label_ptr)
-{
-    TCGBackendData *be = s->be;
-    TCGLabelQemuLdst *l = &be->ldst_labels[be->nb_ldst_labels++];
-
-    assert(be->nb_ldst_labels <= TCG_MAX_QEMU_LDST);
-    l->is_ld = is_ld;
-    l->size = opc & MO_SIZE;
-    l->label_ptr = label_ptr;
-}
-
-static void tcg_out_tb_finalize(TCGContext *s)
-{
-    static const void * const helpers[8] = {
-        helper_ret_stb_mmu,
-        helper_le_stw_mmu,
-        helper_le_stl_mmu,
-        helper_le_stq_mmu,
-        helper_ret_ldub_mmu,
-        helper_le_lduw_mmu,
-        helper_le_ldul_mmu,
-        helper_le_ldq_mmu,
-    };
-    tcg_insn_unit *thunks[8] = { };
-    TCGBackendData *be = s->be;
-    size_t i, n = be->nb_ldst_labels;
-
-    for (i = 0; i < n; i++) {
-        TCGLabelQemuLdst *l = &be->ldst_labels[i];
-        long x = l->is_ld * 4 + l->size;
-        tcg_insn_unit *dest = thunks[x];
-
-        /* The out-of-line thunks are all the same; load the return address
-           from B0, load the GP, and branch to the code.  Note that we are
-           always post-call, so the register window has rolled, so we're
-           using incomming parameter register numbers, not outgoing.  */
-        if (dest == NULL) {
-            uintptr_t *desc = (uintptr_t *)helpers[x];
-            uintptr_t func = desc[0], gp = desc[1], disp;
-
-            thunks[x] = dest = s->code_ptr;
-
-            tcg_out_bundle(s, mlx,
-                           INSN_NOP_M,
-                           tcg_opc_l2 (gp),
-                           tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2,
-                                       TCG_REG_R1, gp));
-            tcg_out_bundle(s, mii,
-                           INSN_NOP_M,
-                           INSN_NOP_I,
-                           tcg_opc_i22(TCG_REG_P0, OPC_MOV_I22,
-                                       l->is_ld ? TCG_REG_R35 : TCG_REG_R36,
-                                       TCG_REG_B0));
-            disp = (tcg_insn_unit *)func - s->code_ptr;
-            tcg_out_bundle(s, mLX,
-                           INSN_NOP_M,
-                           tcg_opc_l3 (disp),
-                           tcg_opc_x3 (TCG_REG_P0, OPC_BRL_SPTK_MANY_X3, disp));
-        }
-
-        reloc_pcrel21b_slot2(l->label_ptr, dest);
-    }
-}
-
-static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
-{
-    static const uint64_t opc_ld_m1[4] = {
-        OPC_LD1_M1, OPC_LD2_M1, OPC_LD4_M1, OPC_LD8_M1
-    };
-    int addr_reg, data_reg, mem_index;
-    TCGMemOpIdx oi;
-    TCGMemOp opc, s_bits;
-    uint64_t fin1, fin2;
-    tcg_insn_unit *label_ptr;
-
-    data_reg = args[0];
-    addr_reg = args[1];
-    oi = args[2];
-    opc = get_memop(oi);
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
-
-    /* Read the TLB entry */
-    tcg_out_qemu_tlb(s, addr_reg, s_bits,
-                     offsetof(CPUArchState, tlb_table[mem_index][0].addr_read),
-                     offsetof(CPUArchState, tlb_table[mem_index][0].addend),
-                     INSN_NOP_I, INSN_NOP_I);
-
-    /* P6 is the fast path, and P7 the slow path */
-
-    fin2 = 0;
-    if (opc & MO_BSWAP) {
-        fin1 = tcg_opc_bswap64_i(TCG_REG_P0, data_reg, TCG_REG_R8);
-        if (s_bits < MO_64) {
-            int shift = 64 - (8 << s_bits);
-            fin2 = (opc & MO_SIGN ? OPC_EXTR_I11 : OPC_EXTR_U_I11);
-            fin2 = tcg_opc_i11(TCG_REG_P0, fin2,
-                               data_reg, data_reg, shift, 63 - shift);
-        }
-    } else {
-        fin1 = tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, TCG_REG_R8);
-    }
-
-    tcg_out_bundle(s, mmI,
-                   tcg_opc_mov_a(TCG_REG_P7, TCG_REG_R56, TCG_AREG0),
-                   tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R2,
-                               TCG_REG_R2, TCG_REG_R57),
-                   tcg_opc_movi_a(TCG_REG_P7, TCG_REG_R58, oi));
-    label_ptr = s->code_ptr;
-    tcg_out_bundle(s, miB,
-                   tcg_opc_m1 (TCG_REG_P6, opc_ld_m1[s_bits],
-                               TCG_REG_R8, TCG_REG_R2),
-                   INSN_NOP_I,
-                   tcg_opc_b3 (TCG_REG_P7, OPC_BR_CALL_SPNT_FEW_B3, TCG_REG_B0,
-                               get_reloc_pcrel21b_slot2(label_ptr)));
-
-    add_qemu_ldst_label(s, 1, opc, label_ptr);
-
-    /* Note that we always use LE helper functions, so the bswap insns
-       here for the fast path also apply to the slow path.  */
-    tcg_out_bundle(s, (fin2 ? mII : miI),
-                   INSN_NOP_M,
-                   fin1,
-                   fin2 ? fin2 : INSN_NOP_I);
-}
-
-static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
-{
-    static const uint64_t opc_st_m4[4] = {
-        OPC_ST1_M4, OPC_ST2_M4, OPC_ST4_M4, OPC_ST8_M4
-    };
-    TCGReg addr_reg, data_reg;
-    int mem_index;
-    uint64_t pre1, pre2;
-    TCGMemOpIdx oi;
-    TCGMemOp opc, s_bits;
-    tcg_insn_unit *label_ptr;
-
-    data_reg = args[0];
-    addr_reg = args[1];
-    oi = args[2];
-    opc = get_memop(oi);
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
-
-    /* Note that we always use LE helper functions, so the bswap insns
-       that are here for the fast path also apply to the slow path,
-       and move the data into the argument register.  */
-    pre2 = INSN_NOP_I;
-    if (opc & MO_BSWAP) {
-        pre1 = tcg_opc_bswap64_i(TCG_REG_P0, TCG_REG_R58, data_reg);
-        if (s_bits < MO_64) {
-            int shift = 64 - (8 << s_bits);
-            pre2 = tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11,
-                               TCG_REG_R58, TCG_REG_R58, shift, 63 - shift);
-        }
-    } else {
-        /* Just move the data into place for the slow path.  */
-        pre1 = tcg_opc_ext_i(TCG_REG_P0, opc, TCG_REG_R58, data_reg);
-    }
-
-    tcg_out_qemu_tlb(s, addr_reg, s_bits,
-                     offsetof(CPUArchState, tlb_table[mem_index][0].addr_write),
-                     offsetof(CPUArchState, tlb_table[mem_index][0].addend),
-                     pre1, pre2);
-
-    /* P6 is the fast path, and P7 the slow path */
-    tcg_out_bundle(s, mmI,
-                   tcg_opc_mov_a(TCG_REG_P7, TCG_REG_R56, TCG_AREG0),
-                   tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R2,
-                               TCG_REG_R2, TCG_REG_R57),
-                   tcg_opc_movi_a(TCG_REG_P7, TCG_REG_R59, oi));
-    label_ptr = s->code_ptr;
-    tcg_out_bundle(s, miB,
-                   tcg_opc_m4 (TCG_REG_P6, opc_st_m4[s_bits],
-                               TCG_REG_R58, TCG_REG_R2),
-                   INSN_NOP_I,
-                   tcg_opc_b3 (TCG_REG_P7, OPC_BR_CALL_SPNT_FEW_B3, TCG_REG_B0,
-                               get_reloc_pcrel21b_slot2(label_ptr)));
-
-    add_qemu_ldst_label(s, 0, opc, label_ptr);
-}
-
-#else /* !CONFIG_SOFTMMU */
-# include "tcg-be-null.h"
-
-static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
-{
-    static uint64_t const opc_ld_m1[4] = {
-        OPC_LD1_M1, OPC_LD2_M1, OPC_LD4_M1, OPC_LD8_M1
-    };
-    int addr_reg, data_reg;
-    TCGMemOp opc, s_bits, bswap;
-
-    data_reg = args[0];
-    addr_reg = args[1];
-    opc = args[2];
-    s_bits = opc & MO_SIZE;
-    bswap = opc & MO_BSWAP;
-
-#if TARGET_LONG_BITS == 32
-    if (GUEST_BASE != 0) {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
-                                   TCG_REG_R3, addr_reg),
-                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
-                                   TCG_GUEST_BASE_REG, TCG_REG_R3));
-    } else {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
-                                   TCG_REG_R2, addr_reg),
-                       INSN_NOP_I);
-    }
-
-    if (!bswap) {
-        if (!(opc & MO_SIGN)) {
-            tcg_out_bundle(s, miI,
-                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                       data_reg, TCG_REG_R2),
-                           INSN_NOP_I,
-                           INSN_NOP_I);
-        } else {
-            tcg_out_bundle(s, mII,
-                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                       data_reg, TCG_REG_R2),
-                           INSN_NOP_I,
-                           tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, data_reg));
-        }
-    } else if (s_bits == MO_64) {
-            tcg_out_bundle(s, mII,
-                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                       data_reg, TCG_REG_R2),
-                           INSN_NOP_I,
-                           tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
-    } else {
-        if (s_bits == MO_16) {
-            tcg_out_bundle(s, mII,
-                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                       data_reg, TCG_REG_R2),
-                           INSN_NOP_I,
-                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                      data_reg, data_reg, 15, 15));
-        } else {
-            tcg_out_bundle(s, mII,
-                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                       data_reg, TCG_REG_R2),
-                           INSN_NOP_I,
-                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                      data_reg, data_reg, 31, 31));
-        }
-        if (!(opc & MO_SIGN)) {
-            tcg_out_bundle(s, miI,
-                           INSN_NOP_M,
-                           INSN_NOP_I,
-                           tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
-        } else {
-            tcg_out_bundle(s, mII,
-                           INSN_NOP_M,
-                           tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg),
-                           tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, data_reg));
-        }
-    }
-#else
-    if (GUEST_BASE != 0) {
-        tcg_out_bundle(s, MmI,
-                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
-                                   TCG_GUEST_BASE_REG, addr_reg),
-                       tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                   data_reg, TCG_REG_R2),
-                       INSN_NOP_I);
-    } else {
-        tcg_out_bundle(s, mmI,
-                       INSN_NOP_M,
-                       tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
-                                   data_reg, addr_reg),
-                       INSN_NOP_I);
-    }
-
-    if (bswap && s_bits == MO_16) {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                   data_reg, data_reg, 15, 15),
-                       tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
-    } else if (bswap && s_bits == MO_32) {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                   data_reg, data_reg, 31, 31),
-                       tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
-    } else if (bswap && s_bits == MO_64) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
-    }
-    if (opc & MO_SIGN) {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       INSN_NOP_I,
-                       tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, data_reg));
-    }
-#endif
-}
-
-static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
-{
-    static uint64_t const opc_st_m4[4] = {
-        OPC_ST1_M4, OPC_ST2_M4, OPC_ST4_M4, OPC_ST8_M4
-    };
-    int addr_reg, data_reg;
-#if TARGET_LONG_BITS == 64
-    uint64_t add_guest_base;
-#endif
-    TCGMemOp opc, s_bits, bswap;
-
-    data_reg = args[0];
-    addr_reg = args[1];
-    opc = args[2];
-    s_bits = opc & MO_SIZE;
-    bswap = opc & MO_BSWAP;
-
-#if TARGET_LONG_BITS == 32
-    if (GUEST_BASE != 0) {
-        tcg_out_bundle(s, mII,
-                       INSN_NOP_M,
-                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
-                                   TCG_REG_R3, addr_reg),
-                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
-                                   TCG_GUEST_BASE_REG, TCG_REG_R3));
-    } else {
-        tcg_out_bundle(s, miI,
-                       INSN_NOP_M,
-                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
-                                   TCG_REG_R2, addr_reg),
-                       INSN_NOP_I);
-    }
-
-    if (bswap) {
-        if (s_bits == MO_16) {
-            tcg_out_bundle(s, mII,
-                           INSN_NOP_M,
-                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                       TCG_REG_R3, data_reg, 15, 15),
-                           tcg_opc_bswap64_i(TCG_REG_P0,
-                                             TCG_REG_R3, TCG_REG_R3));
-            data_reg = TCG_REG_R3;
-        } else if (s_bits == MO_32) {
-            tcg_out_bundle(s, mII,
-                           INSN_NOP_M,
-                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                       TCG_REG_R3, data_reg, 31, 31),
-                           tcg_opc_bswap64_i(TCG_REG_P0,
-                                             TCG_REG_R3, TCG_REG_R3));
-            data_reg = TCG_REG_R3;
-        } else if (s_bits == MO_64) {
-            tcg_out_bundle(s, miI,
-                           INSN_NOP_M,
-                           INSN_NOP_I,
-                           tcg_opc_bswap64_i(TCG_REG_P0, TCG_REG_R3, data_reg));
-            data_reg = TCG_REG_R3;
-        }
-    }
-    tcg_out_bundle(s, mmI,
-                   tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
-                               data_reg, TCG_REG_R2),
-                   INSN_NOP_M,
-                   INSN_NOP_I);
-#else
-    if (GUEST_BASE != 0) {
-        add_guest_base = tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
-                                     TCG_GUEST_BASE_REG, addr_reg);
-        addr_reg = TCG_REG_R2;
-    } else {
-        add_guest_base = INSN_NOP_M;
-    }
-
-    if (!bswap) {
-        tcg_out_bundle(s, (GUEST_BASE ? MmI : mmI),
-                       add_guest_base,
-                       tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
-                                   data_reg, addr_reg),
-                       INSN_NOP_I);
-    } else {
-        if (s_bits == MO_16) {
-            tcg_out_bundle(s, mII,
-                           add_guest_base,
-                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                       TCG_REG_R3, data_reg, 15, 15),
-                           tcg_opc_bswap64_i(TCG_REG_P0,
-                                             TCG_REG_R3, TCG_REG_R3));
-            data_reg = TCG_REG_R3;
-        } else if (s_bits == MO_32) {
-            tcg_out_bundle(s, mII,
-                           add_guest_base,
-                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
-                                       TCG_REG_R3, data_reg, 31, 31),
-                           tcg_opc_bswap64_i(TCG_REG_P0,
-                                             TCG_REG_R3, TCG_REG_R3));
-            data_reg = TCG_REG_R3;
-        } else if (s_bits == MO_64) {
-            tcg_out_bundle(s, miI,
-                           add_guest_base,
-                           INSN_NOP_I,
-                           tcg_opc_bswap64_i(TCG_REG_P0, TCG_REG_R3, data_reg));
-            data_reg = TCG_REG_R3;
-        }
-        tcg_out_bundle(s, miI,
-                       tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
-                                   data_reg, addr_reg),
-                       INSN_NOP_I,
-                       INSN_NOP_I);
-    }
-#endif
-}
-
-#endif
-
-static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                              const TCGArg *args, const int *const_args)
-{
-    switch(opc) {
-    case INDEX_op_exit_tb:
-        tcg_out_exit_tb(s, args[0]);
-        break;
-    case INDEX_op_br:
-        tcg_out_br(s, args[0]);
-        break;
-    case INDEX_op_goto_tb:
-        tcg_out_goto_tb(s, args[0]);
-        break;
-
-    case INDEX_op_ld8u_i32:
-    case INDEX_op_ld8u_i64:
-        tcg_out_ld_rel(s, OPC_LD1_M1, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld8s_i32:
-    case INDEX_op_ld8s_i64:
-        tcg_out_ld_rel(s, OPC_LD1_M1, args[0], args[1], args[2]);
-        tcg_out_ext(s, OPC_SXT1_I29, args[0], args[0]);
-        break;
-    case INDEX_op_ld16u_i32:
-    case INDEX_op_ld16u_i64:
-        tcg_out_ld_rel(s, OPC_LD2_M1, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld16s_i32:
-    case INDEX_op_ld16s_i64:
-        tcg_out_ld_rel(s, OPC_LD2_M1, args[0], args[1], args[2]);
-        tcg_out_ext(s, OPC_SXT2_I29, args[0], args[0]);
-        break;
-    case INDEX_op_ld_i32:
-    case INDEX_op_ld32u_i64:
-        tcg_out_ld_rel(s, OPC_LD4_M1, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld32s_i64:
-        tcg_out_ld_rel(s, OPC_LD4_M1, args[0], args[1], args[2]);
-        tcg_out_ext(s, OPC_SXT4_I29, args[0], args[0]);
-        break;
-    case INDEX_op_ld_i64:
-        tcg_out_ld_rel(s, OPC_LD8_M1, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st8_i32:
-    case INDEX_op_st8_i64:
-        tcg_out_st_rel(s, OPC_ST1_M4, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st16_i32:
-    case INDEX_op_st16_i64:
-        tcg_out_st_rel(s, OPC_ST2_M4, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i32:
-    case INDEX_op_st32_i64:
-        tcg_out_st_rel(s, OPC_ST4_M4, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i64:
-        tcg_out_st_rel(s, OPC_ST8_M4, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_add_i32:
-    case INDEX_op_add_i64:
-        tcg_out_add(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_sub_i32:
-    case INDEX_op_sub_i64:
-        tcg_out_sub(s, args[0], args[1], const_args[1], args[2], const_args[2]);
-        break;
-
-    case INDEX_op_and_i32:
-    case INDEX_op_and_i64:
-        /* TCG expects arg2 constant; A3 expects arg1 constant.  Swap.  */
-        tcg_out_alu(s, OPC_AND_A1, OPC_AND_A3, args[0],
-                    args[2], const_args[2], args[1], const_args[1]);
-        break;
-    case INDEX_op_andc_i32:
-    case INDEX_op_andc_i64:
-        tcg_out_alu(s, OPC_ANDCM_A1, OPC_ANDCM_A3, args[0],
-                    args[1], const_args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_eqv_i32:
-    case INDEX_op_eqv_i64:
-        tcg_out_eqv(s, args[0], args[1], const_args[1],
-                    args[2], const_args[2]);
-        break;
-    case INDEX_op_nand_i32:
-    case INDEX_op_nand_i64:
-        tcg_out_nand(s, args[0], args[1], const_args[1],
-                     args[2], const_args[2]);
-        break;
-    case INDEX_op_nor_i32:
-    case INDEX_op_nor_i64:
-        tcg_out_nor(s, args[0], args[1], const_args[1],
-                    args[2], const_args[2]);
-        break;
-    case INDEX_op_or_i32:
-    case INDEX_op_or_i64:
-        /* TCG expects arg2 constant; A3 expects arg1 constant.  Swap.  */
-        tcg_out_alu(s, OPC_OR_A1, OPC_OR_A3, args[0],
-                    args[2], const_args[2], args[1], const_args[1]);
-        break;
-    case INDEX_op_orc_i32:
-    case INDEX_op_orc_i64:
-        tcg_out_orc(s, args[0], args[1], const_args[1],
-                    args[2], const_args[2]);
-        break;
-    case INDEX_op_xor_i32:
-    case INDEX_op_xor_i64:
-        /* TCG expects arg2 constant; A3 expects arg1 constant.  Swap.  */
-        tcg_out_alu(s, OPC_XOR_A1, OPC_XOR_A3, args[0],
-                    args[2], const_args[2], args[1], const_args[1]);
-        break;
-
-    case INDEX_op_mul_i32:
-    case INDEX_op_mul_i64:
-        tcg_out_mul(s, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_sar_i32:
-        tcg_out_sar_i32(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_sar_i64:
-        tcg_out_sar_i64(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_shl_i32:
-        tcg_out_shl_i32(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_shl_i64:
-        tcg_out_shl_i64(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_shr_i32:
-        tcg_out_shr_i32(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_shr_i64:
-        tcg_out_shr_i64(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_rotl_i32:
-        tcg_out_rotl_i32(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_rotl_i64:
-        tcg_out_rotl_i64(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_rotr_i32:
-        tcg_out_rotr_i32(s, args[0], args[1], args[2], const_args[2]);
-        break;
-    case INDEX_op_rotr_i64:
-        tcg_out_rotr_i64(s, args[0], args[1], args[2], const_args[2]);
-        break;
-
-    case INDEX_op_ext8s_i32:
-    case INDEX_op_ext8s_i64:
-        tcg_out_ext(s, OPC_SXT1_I29, args[0], args[1]);
-        break;
-    case INDEX_op_ext8u_i32:
-    case INDEX_op_ext8u_i64:
-        tcg_out_ext(s, OPC_ZXT1_I29, args[0], args[1]);
-        break;
-    case INDEX_op_ext16s_i32:
-    case INDEX_op_ext16s_i64:
-        tcg_out_ext(s, OPC_SXT2_I29, args[0], args[1]);
-        break;
-    case INDEX_op_ext16u_i32:
-    case INDEX_op_ext16u_i64:
-        tcg_out_ext(s, OPC_ZXT2_I29, args[0], args[1]);
-        break;
-    case INDEX_op_ext32s_i64:
-        tcg_out_ext(s, OPC_SXT4_I29, args[0], args[1]);
-        break;
-    case INDEX_op_ext32u_i64:
-        tcg_out_ext(s, OPC_ZXT4_I29, args[0], args[1]);
-        break;
-
-    case INDEX_op_bswap16_i32:
-    case INDEX_op_bswap16_i64:
-        tcg_out_bswap16(s, args[0], args[1]);
-        break;
-    case INDEX_op_bswap32_i32:
-    case INDEX_op_bswap32_i64:
-        tcg_out_bswap32(s, args[0], args[1]);
-        break;
-    case INDEX_op_bswap64_i64:
-        tcg_out_bswap64(s, args[0], args[1]);
-        break;
-
-    case INDEX_op_deposit_i32:
-    case INDEX_op_deposit_i64:
-        tcg_out_deposit(s, args[0], args[1], args[2], const_args[2],
-                        args[3], args[4]);
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_brcond(s, args[2], args[0], args[1], args[3], 1);
-        break;
-    case INDEX_op_brcond_i64:
-        tcg_out_brcond(s, args[2], args[0], args[1], args[3], 0);
-        break;
-    case INDEX_op_setcond_i32:
-        tcg_out_setcond(s, args[3], args[0], args[1], args[2], 1);
-        break;
-    case INDEX_op_setcond_i64:
-        tcg_out_setcond(s, args[3], args[0], args[1], args[2], 0);
-        break;
-    case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
-                        args[3], const_args[3], args[4], const_args[4], 1);
-        break;
-    case INDEX_op_movcond_i64:
-        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
-                        args[3], const_args[3], args[4], const_args[4], 0);
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_qemu_ld(s, args);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, args);
-        break;
-    case INDEX_op_qemu_st_i32:
-        tcg_out_qemu_st(s, args);
-        break;
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args);
-        break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef ia64_op_defs[] = {
-    { INDEX_op_br, { } },
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "rZ", "r" } },
-    { INDEX_op_st16_i32, { "rZ", "r" } },
-    { INDEX_op_st_i32, { "rZ", "r" } },
-
-    { INDEX_op_add_i32, { "r", "rZ", "rI" } },
-    { INDEX_op_sub_i32, { "r", "rI", "rI" } },
-
-    { INDEX_op_and_i32, { "r", "rI", "rI" } },
-    { INDEX_op_andc_i32, { "r", "rI", "rI" } },
-    { INDEX_op_eqv_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_nand_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_or_i32, { "r", "rI", "rI" } },
-    { INDEX_op_orc_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_xor_i32, { "r", "rI", "rI" } },
-
-    { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
-
-    { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
-
-    { INDEX_op_ext8s_i32, { "r", "rZ"} },
-    { INDEX_op_ext8u_i32, { "r", "rZ"} },
-    { INDEX_op_ext16s_i32, { "r", "rZ"} },
-    { INDEX_op_ext16u_i32, { "r", "rZ"} },
-
-    { INDEX_op_bswap16_i32, { "r", "rZ" } },
-    { INDEX_op_bswap32_i32, { "r", "rZ" } },
-
-    { INDEX_op_brcond_i32, { "rZ", "rZ" } },
-    { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rI", "rI" } },
-
-    { INDEX_op_ld8u_i64, { "r", "r" } },
-    { INDEX_op_ld8s_i64, { "r", "r" } },
-    { INDEX_op_ld16u_i64, { "r", "r" } },
-    { INDEX_op_ld16s_i64, { "r", "r" } },
-    { INDEX_op_ld32u_i64, { "r", "r" } },
-    { INDEX_op_ld32s_i64, { "r", "r" } },
-    { INDEX_op_ld_i64, { "r", "r" } },
-    { INDEX_op_st8_i64, { "rZ", "r" } },
-    { INDEX_op_st16_i64, { "rZ", "r" } },
-    { INDEX_op_st32_i64, { "rZ", "r" } },
-    { INDEX_op_st_i64, { "rZ", "r" } },
-
-    { INDEX_op_add_i64, { "r", "rZ", "rI" } },
-    { INDEX_op_sub_i64, { "r", "rI", "rI" } },
-
-    { INDEX_op_and_i64, { "r", "rI", "rI" } },
-    { INDEX_op_andc_i64, { "r", "rI", "rI" } },
-    { INDEX_op_eqv_i64, { "r", "rZ", "rZ" } },
-    { INDEX_op_nand_i64, { "r", "rZ", "rZ" } },
-    { INDEX_op_nor_i64, { "r", "rZ", "rZ" } },
-    { INDEX_op_or_i64, { "r", "rI", "rI" } },
-    { INDEX_op_orc_i64, { "r", "rZ", "rZ" } },
-    { INDEX_op_xor_i64, { "r", "rI", "rI" } },
-
-    { INDEX_op_mul_i64, { "r", "rZ", "rZ" } },
-
-    { INDEX_op_sar_i64, { "r", "rZ", "ri" } },
-    { INDEX_op_shl_i64, { "r", "rZ", "ri" } },
-    { INDEX_op_shr_i64, { "r", "rZ", "ri" } },
-    { INDEX_op_rotl_i64, { "r", "rZ", "ri" } },
-    { INDEX_op_rotr_i64, { "r", "rZ", "ri" } },
-
-    { INDEX_op_ext8s_i64, { "r", "rZ"} },
-    { INDEX_op_ext8u_i64, { "r", "rZ"} },
-    { INDEX_op_ext16s_i64, { "r", "rZ"} },
-    { INDEX_op_ext16u_i64, { "r", "rZ"} },
-    { INDEX_op_ext32s_i64, { "r", "rZ"} },
-    { INDEX_op_ext32u_i64, { "r", "rZ"} },
-
-    { INDEX_op_bswap16_i64, { "r", "rZ" } },
-    { INDEX_op_bswap32_i64, { "r", "rZ" } },
-    { INDEX_op_bswap64_i64, { "r", "rZ" } },
-
-    { INDEX_op_brcond_i64, { "rZ", "rZ" } },
-    { INDEX_op_setcond_i64, { "r", "rZ", "rZ" } },
-    { INDEX_op_movcond_i64, { "r", "rZ", "rZ", "rI", "rI" } },
-
-    { INDEX_op_deposit_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_deposit_i64, { "r", "rZ", "ri" } },
-
-    { INDEX_op_qemu_ld_i32, { "r", "r" } },
-    { INDEX_op_qemu_ld_i64, { "r", "r" } },
-    { INDEX_op_qemu_st_i32, { "SZ", "r" } },
-    { INDEX_op_qemu_st_i64, { "SZ", "r" } },
-
-    { -1 },
-};
-
-/* Generate global QEMU prologue and epilogue code */
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int frame_size;
-
-    /* reserve some stack space */
-    frame_size = TCG_STATIC_CALL_ARGS_SIZE +
-                 CPU_TEMP_BUF_NLONGS * sizeof(long);
-    frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
-                 ~(TCG_TARGET_STACK_ALIGN - 1);
-    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-    /* First emit adhoc function descriptor */
-    *s->code_ptr = (tcg_insn_unit){
-        (uint64_t)(s->code_ptr + 1), /* entry point */
-        0                            /* skip gp */
-    };
-    s->code_ptr++;
-
-    /* prologue */
-    tcg_out_bundle(s, miI,
-                   tcg_opc_m34(TCG_REG_P0, OPC_ALLOC_M34,
-                               TCG_REG_R34, 32, 24, 0),
-                   INSN_NOP_I,
-                   tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21,
-                               TCG_REG_B6, TCG_REG_R33, 0));
-
-    /* ??? If GUEST_BASE < 0x200000, we could load the register via
-       an ADDL in the M slot of the next bundle.  */
-    if (GUEST_BASE != 0) {
-        tcg_out_bundle(s, mlx,
-                       INSN_NOP_M,
-                       tcg_opc_l2 (GUEST_BASE),
-                       tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2,
-                                   TCG_GUEST_BASE_REG, GUEST_BASE));
-        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
-    }
-
-    tcg_out_bundle(s, miB,
-                   tcg_opc_a4 (TCG_REG_P0, OPC_ADDS_A4,
-                               TCG_REG_R12, -frame_size, TCG_REG_R12),
-                   tcg_opc_i22(TCG_REG_P0, OPC_MOV_I22,
-                               TCG_REG_R33, TCG_REG_B0),
-                   tcg_opc_b4 (TCG_REG_P0, OPC_BR_SPTK_MANY_B4, TCG_REG_B6));
-
-    /* epilogue */
-    tb_ret_addr = s->code_ptr;
-    tcg_out_bundle(s, miI,
-                   INSN_NOP_M,
-                   tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21,
-                               TCG_REG_B0, TCG_REG_R33, 0),
-                   tcg_opc_a4 (TCG_REG_P0, OPC_ADDS_A4,
-                               TCG_REG_R12, frame_size, TCG_REG_R12));
-    tcg_out_bundle(s, miB,
-                   INSN_NOP_M,
-                   tcg_opc_i26(TCG_REG_P0, OPC_MOV_I_I26,
-                               TCG_REG_PFS, TCG_REG_R34),
-                   tcg_opc_b4 (TCG_REG_P0, OPC_BR_RET_SPTK_MANY_B4,
-                               TCG_REG_B0));
-}
-
-static void tcg_target_init(TCGContext *s)
-{
-    tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I32],
-                   0xffffffffffffffffull);
-    tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I64],
-                   0xffffffffffffffffull);
-
-    tcg_regset_clear(tcg_target_call_clobber_regs);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R15);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R16);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R17);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R18);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R19);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R20);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R21);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R22);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R23);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R24);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R25);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R26);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R27);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R28);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R29);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R30);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R31);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R56);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R57);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R58);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R59);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R60);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R61);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R62);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R63);
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);   /* zero register */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1);   /* global pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2);   /* internal use */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R3);   /* internal use */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R12);  /* stack pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13);  /* thread pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R33);  /* return address */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R34);  /* PFS */
-
-    /* The following 4 are not in use, are call-saved, but *not* saved
-       by the prologue.  Therefore we cannot use them without modifying
-       the prologue.  There doesn't seem to be any good reason to use
-       these as opposed to the windowed registers.  */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R4);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R5);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R6);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R7);
-
-    tcg_add_target_add_op_defs(ia64_op_defs);
-}
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index d675589..6dddb7f 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -22,10 +22,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_IA64 
-#define TCG_TARGET_IA64 1
+
+#ifndef IA64_TCG_TARGET_H
+#define IA64_TCG_TARGET_H
 
 #define TCG_TARGET_INSN_UNIT_SIZE 16
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 21
+
 typedef struct {
     uint64_t lo __attribute__((aligned(16)));
     uint64_t hi;
@@ -158,7 +161,8 @@
 #define TCG_TARGET_HAS_muluh_i64        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 #define TCG_TARGET_HAS_mulsh_i64        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
 #define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
diff --git a/tcg/ia64/tcg-target.inc.c b/tcg/ia64/tcg-target.inc.c
new file mode 100644
index 0000000..c91f392
--- /dev/null
+++ b/tcg/ia64/tcg-target.inc.c
@@ -0,0 +1,2463 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2009-2010 Aurelien Jarno <aurelien@aurel32.net>
+ * Based on i386/tcg-target.c - Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * Register definitions
+ */
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+     "r0",  "r1",  "r2",  "r3",  "r4",  "r5",  "r6",  "r7",
+     "r8",  "r9", "r10", "r11", "r12", "r13", "r14", "r15",
+    "r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
+    "r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
+    "r32", "r33", "r34", "r35", "r36", "r37", "r38", "r39",
+    "r40", "r41", "r42", "r43", "r44", "r45", "r46", "r47",
+    "r48", "r49", "r50", "r51", "r52", "r53", "r54", "r55",
+    "r56", "r57", "r58", "r59", "r60", "r61", "r62", "r63",
+};
+#endif
+
+#ifndef CONFIG_SOFTMMU
+#define TCG_GUEST_BASE_REG TCG_REG_R55
+#endif
+
+/* Branch registers */
+enum {
+    TCG_REG_B0 = 0,
+    TCG_REG_B1,
+    TCG_REG_B2,
+    TCG_REG_B3,
+    TCG_REG_B4,
+    TCG_REG_B5,
+    TCG_REG_B6,
+    TCG_REG_B7,
+};
+
+/* Floating point registers */
+enum {
+    TCG_REG_F0 = 0,
+    TCG_REG_F1,
+    TCG_REG_F2,
+    TCG_REG_F3,
+    TCG_REG_F4,
+    TCG_REG_F5,
+    TCG_REG_F6,
+    TCG_REG_F7,
+    TCG_REG_F8,
+    TCG_REG_F9,
+    TCG_REG_F10,
+    TCG_REG_F11,
+    TCG_REG_F12,
+    TCG_REG_F13,
+    TCG_REG_F14,
+    TCG_REG_F15,
+};
+
+/* Predicate registers */
+enum {
+    TCG_REG_P0 = 0,
+    TCG_REG_P1,
+    TCG_REG_P2,
+    TCG_REG_P3,
+    TCG_REG_P4,
+    TCG_REG_P5,
+    TCG_REG_P6,
+    TCG_REG_P7,
+    TCG_REG_P8,
+    TCG_REG_P9,
+    TCG_REG_P10,
+    TCG_REG_P11,
+    TCG_REG_P12,
+    TCG_REG_P13,
+    TCG_REG_P14,
+    TCG_REG_P15,
+};
+
+/* Application registers */
+enum {
+    TCG_REG_PFS = 64,
+};
+
+static const int tcg_target_reg_alloc_order[] = {
+    TCG_REG_R35,
+    TCG_REG_R36,
+    TCG_REG_R37,
+    TCG_REG_R38,
+    TCG_REG_R39,
+    TCG_REG_R40,
+    TCG_REG_R41,
+    TCG_REG_R42,
+    TCG_REG_R43,
+    TCG_REG_R44,
+    TCG_REG_R45,
+    TCG_REG_R46,
+    TCG_REG_R47,
+    TCG_REG_R48,
+    TCG_REG_R49,
+    TCG_REG_R50,
+    TCG_REG_R51,
+    TCG_REG_R52,
+    TCG_REG_R53,
+    TCG_REG_R54,
+    TCG_REG_R55,
+    TCG_REG_R14,
+    TCG_REG_R15,
+    TCG_REG_R16,
+    TCG_REG_R17,
+    TCG_REG_R18,
+    TCG_REG_R19,
+    TCG_REG_R20,
+    TCG_REG_R21,
+    TCG_REG_R22,
+    TCG_REG_R23,
+    TCG_REG_R24,
+    TCG_REG_R25,
+    TCG_REG_R26,
+    TCG_REG_R27,
+    TCG_REG_R28,
+    TCG_REG_R29,
+    TCG_REG_R30,
+    TCG_REG_R31,
+    TCG_REG_R56,
+    TCG_REG_R57,
+    TCG_REG_R58,
+    TCG_REG_R59,
+    TCG_REG_R60,
+    TCG_REG_R61,
+    TCG_REG_R62,
+    TCG_REG_R63,
+    TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
+    TCG_REG_R11
+};
+
+static const int tcg_target_call_iarg_regs[8] = {
+    TCG_REG_R56,
+    TCG_REG_R57,
+    TCG_REG_R58,
+    TCG_REG_R59,
+    TCG_REG_R60,
+    TCG_REG_R61,
+    TCG_REG_R62,
+    TCG_REG_R63,
+};
+
+static const int tcg_target_call_oarg_regs[] = {
+    TCG_REG_R8
+};
+
+/*
+ * opcode formation
+ */
+
+/* bundle templates: stops (double bar in the IA64 manual) are marked with
+   an uppercase letter. */
+enum {
+    mii = 0x00,
+    miI = 0x01,
+    mIi = 0x02,
+    mII = 0x03,
+    mlx = 0x04,
+    mLX = 0x05,
+    mmi = 0x08,
+    mmI = 0x09,
+    Mmi = 0x0a,
+    MmI = 0x0b,
+    mfi = 0x0c,
+    mfI = 0x0d,
+    mmf = 0x0e,
+    mmF = 0x0f,
+    mib = 0x10,
+    miB = 0x11,
+    mbb = 0x12,
+    mbB = 0x13,
+    bbb = 0x16,
+    bbB = 0x17,
+    mmb = 0x18,
+    mmB = 0x19,
+    mfb = 0x1c,
+    mfB = 0x1d,
+};
+
+enum {
+    OPC_ADD_A1                = 0x10000000000ull,
+    OPC_AND_A1                = 0x10060000000ull,
+    OPC_AND_A3                = 0x10160000000ull,
+    OPC_ANDCM_A1              = 0x10068000000ull,
+    OPC_ANDCM_A3              = 0x10168000000ull,
+    OPC_ADDS_A4               = 0x10800000000ull,
+    OPC_ADDL_A5               = 0x12000000000ull,
+    OPC_ALLOC_M34             = 0x02c00000000ull,
+    OPC_BR_DPTK_FEW_B1        = 0x08400000000ull,
+    OPC_BR_SPTK_MANY_B1       = 0x08000001000ull,
+    OPC_BR_CALL_SPNT_FEW_B3   = 0x0a200000000ull,
+    OPC_BR_SPTK_MANY_B4       = 0x00100001000ull,
+    OPC_BR_CALL_SPTK_MANY_B5  = 0x02100001000ull,
+    OPC_BR_RET_SPTK_MANY_B4   = 0x00108001100ull,
+    OPC_BRL_SPTK_MANY_X3      = 0x18000001000ull,
+    OPC_BRL_CALL_SPNT_MANY_X4 = 0x1a200001000ull,
+    OPC_BRL_CALL_SPTK_MANY_X4 = 0x1a000001000ull,
+    OPC_CMP_LT_A6             = 0x18000000000ull,
+    OPC_CMP_LTU_A6            = 0x1a000000000ull,
+    OPC_CMP_EQ_A6             = 0x1c000000000ull,
+    OPC_CMP4_LT_A6            = 0x18400000000ull,
+    OPC_CMP4_LTU_A6           = 0x1a400000000ull,
+    OPC_CMP4_EQ_A6            = 0x1c400000000ull,
+    OPC_DEP_I14               = 0x0ae00000000ull,
+    OPC_DEP_I15               = 0x08000000000ull,
+    OPC_DEP_Z_I12             = 0x0a600000000ull,
+    OPC_EXTR_I11              = 0x0a400002000ull,
+    OPC_EXTR_U_I11            = 0x0a400000000ull,
+    OPC_FCVT_FX_TRUNC_S1_F10  = 0x004d0000000ull,
+    OPC_FCVT_FXU_TRUNC_S1_F10 = 0x004d8000000ull,
+    OPC_FCVT_XF_F11           = 0x000e0000000ull,
+    OPC_FMA_S1_F1             = 0x10400000000ull,
+    OPC_FNMA_S1_F1            = 0x18400000000ull,
+    OPC_FRCPA_S1_F6           = 0x00600000000ull,
+    OPC_GETF_SIG_M19          = 0x08708000000ull,
+    OPC_LD1_M1                = 0x08000000000ull,
+    OPC_LD1_M3                = 0x0a000000000ull,
+    OPC_LD2_M1                = 0x08040000000ull,
+    OPC_LD2_M3                = 0x0a040000000ull,
+    OPC_LD4_M1                = 0x08080000000ull,
+    OPC_LD4_M3                = 0x0a080000000ull,
+    OPC_LD8_M1                = 0x080c0000000ull,
+    OPC_LD8_M3                = 0x0a0c0000000ull,
+    OPC_MUX1_I3               = 0x0eca0000000ull,
+    OPC_NOP_B9                = 0x04008000000ull,
+    OPC_NOP_F16               = 0x00008000000ull,
+    OPC_NOP_I18               = 0x00008000000ull,
+    OPC_NOP_M48               = 0x00008000000ull,
+    OPC_MOV_I21               = 0x00e00100000ull,
+    OPC_MOV_RET_I21           = 0x00e00500000ull,
+    OPC_MOV_I22               = 0x00188000000ull,
+    OPC_MOV_I_I26             = 0x00150000000ull,
+    OPC_MOVL_X2               = 0x0c000000000ull,
+    OPC_OR_A1                 = 0x10070000000ull,
+    OPC_OR_A3                 = 0x10170000000ull,
+    OPC_SETF_EXP_M18          = 0x0c748000000ull,
+    OPC_SETF_SIG_M18          = 0x0c708000000ull,
+    OPC_SHL_I7                = 0x0f240000000ull,
+    OPC_SHR_I5                = 0x0f220000000ull,
+    OPC_SHR_U_I5              = 0x0f200000000ull,
+    OPC_SHRP_I10              = 0x0ac00000000ull,
+    OPC_SXT1_I29              = 0x000a0000000ull,
+    OPC_SXT2_I29              = 0x000a8000000ull,
+    OPC_SXT4_I29              = 0x000b0000000ull,
+    OPC_ST1_M4                = 0x08c00000000ull,
+    OPC_ST2_M4                = 0x08c40000000ull,
+    OPC_ST4_M4                = 0x08c80000000ull,
+    OPC_ST8_M4                = 0x08cc0000000ull,
+    OPC_SUB_A1                = 0x10028000000ull,
+    OPC_SUB_A3                = 0x10128000000ull,
+    OPC_UNPACK4_L_I2          = 0x0f860000000ull,
+    OPC_XMA_L_F2              = 0x1d000000000ull,
+    OPC_XOR_A1                = 0x10078000000ull,
+    OPC_XOR_A3                = 0x10178000000ull,
+    OPC_ZXT1_I29              = 0x00080000000ull,
+    OPC_ZXT2_I29              = 0x00088000000ull,
+    OPC_ZXT4_I29              = 0x00090000000ull,
+
+    INSN_NOP_M                = OPC_NOP_M48,  /* nop.m 0 */
+    INSN_NOP_I                = OPC_NOP_I18,  /* nop.i 0 */
+};
+
+static inline uint64_t tcg_opc_a1(int qp, uint64_t opc, int r1,
+                                  int r2, int r3)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_a3(int qp, uint64_t opc, int r1,
+                                  uint64_t imm, int r3)
+{
+    return opc
+           | ((imm & 0x80) << 29) /* s */
+           | ((imm & 0x7f) << 13) /* imm7b */
+           | ((r3 & 0x7f) << 20)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_a4(int qp, uint64_t opc, int r1,
+                                  uint64_t imm, int r3)
+{
+    return opc
+           | ((imm & 0x2000) << 23) /* s */
+           | ((imm & 0x1f80) << 20) /* imm6d */
+           | ((imm & 0x007f) << 13) /* imm7b */
+           | ((r3 & 0x7f) << 20)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_a5(int qp, uint64_t opc, int r1,
+                                  uint64_t imm, int r3)
+{
+    return opc
+           | ((imm & 0x200000) << 15) /* s */
+           | ((imm & 0x1f0000) <<  6) /* imm5c */
+           | ((imm & 0x00ff80) << 20) /* imm9d */
+           | ((imm & 0x00007f) << 13) /* imm7b */
+           | ((r3 & 0x03) << 20)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_a6(int qp, uint64_t opc, int p1,
+                                  int p2, int r2, int r3)
+{
+    return opc
+           | ((p2 & 0x3f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((p1 & 0x3f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_b1(int qp, uint64_t opc, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100000) << 16) /* s */
+           | ((imm & 0x0fffff) << 13) /* imm20b */
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_b3(int qp, uint64_t opc, int b1, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100000) << 16) /* s */
+           | ((imm & 0x0fffff) << 13) /* imm20b */
+           | ((b1 & 0x7) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_b4(int qp, uint64_t opc, int b2)
+{
+    return opc
+           | ((b2 & 0x7) << 13)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_b5(int qp, uint64_t opc, int b1, int b2)
+{
+    return opc
+           | ((b2 & 0x7) << 13)
+           | ((b1 & 0x7) << 6)
+           | (qp & 0x3f);
+}
+
+
+static inline uint64_t tcg_opc_b9(int qp, uint64_t opc, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100000) << 16) /* i */
+           | ((imm & 0x0fffff) << 6)  /* imm20a */
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_f1(int qp, uint64_t opc, int f1,
+                                  int f3, int f4, int f2)
+{
+    return opc
+           | ((f4 & 0x7f) << 27)
+           | ((f3 & 0x7f) << 20)
+           | ((f2 & 0x7f) << 13)
+           | ((f1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_f2(int qp, uint64_t opc, int f1,
+                                  int f3, int f4, int f2)
+{
+    return opc
+           | ((f4 & 0x7f) << 27)
+           | ((f3 & 0x7f) << 20)
+           | ((f2 & 0x7f) << 13)
+           | ((f1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_f6(int qp, uint64_t opc, int f1,
+                                  int p2, int f2, int f3)
+{
+    return opc
+           | ((p2 & 0x3f) << 27)
+           | ((f3 & 0x7f) << 20)
+           | ((f2 & 0x7f) << 13)
+           | ((f1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_f10(int qp, uint64_t opc, int f1, int f2)
+{
+    return opc
+           | ((f2 & 0x7f) << 13)
+           | ((f1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_f11(int qp, uint64_t opc, int f1, int f2)
+{
+    return opc
+           | ((f2 & 0x7f) << 13)
+           | ((f1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_f16(int qp, uint64_t opc, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100000) << 16) /* i */
+           | ((imm & 0x0fffff) << 6)  /* imm20a */
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i2(int qp, uint64_t opc, int r1,
+                                  int r2, int r3)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i3(int qp, uint64_t opc, int r1,
+                                  int r2, int mbtype)
+{
+    return opc
+           | ((mbtype & 0x0f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i5(int qp, uint64_t opc, int r1,
+                                  int r3, int r2)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i7(int qp, uint64_t opc, int r1,
+                                  int r2, int r3)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i10(int qp, uint64_t opc, int r1,
+                                   int r2, int r3, uint64_t count)
+{
+    return opc
+           | ((count & 0x3f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i11(int qp, uint64_t opc, int r1,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((len & 0x3f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((pos & 0x3f) << 14)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i12(int qp, uint64_t opc, int r1,
+                                   int r2, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((len & 0x3f) << 27)
+           | ((pos & 0x3f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i14(int qp, uint64_t opc, int r1, uint64_t imm,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((imm & 0x01) << 36)
+           | ((len & 0x3f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((pos & 0x3f) << 14)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i15(int qp, uint64_t opc, int r1, int r2,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((pos & 0x3f) << 31)
+           | ((len & 0x0f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i18(int qp, uint64_t opc, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100000) << 16) /* i */
+           | ((imm & 0x0fffff) << 6)  /* imm20a */
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i21(int qp, uint64_t opc, int b1,
+                                   int r2, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x1ff) << 24)
+           | ((r2 & 0x7f) << 13)
+           | ((b1 & 0x7) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i22(int qp, uint64_t opc, int r1, int b2)
+{
+    return opc
+           | ((b2 & 0x7) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i26(int qp, uint64_t opc, int ar3, int r2)
+{
+    return opc
+           | ((ar3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i29(int qp, uint64_t opc, int r1, int r3)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_l2(uint64_t imm)
+{
+    return (imm & 0x7fffffffffc00000ull) >> 22;
+}
+
+static inline uint64_t tcg_opc_l3(uint64_t imm)
+{
+    return (imm & 0x07fffffffff00000ull) >> 18;
+}
+
+#define tcg_opc_l4  tcg_opc_l3
+
+static inline uint64_t tcg_opc_m1(int qp, uint64_t opc, int r1, int r3)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_m3(int qp, uint64_t opc, int r1,
+                                  int r3, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100) << 28) /* s */
+           | ((imm & 0x080) << 20) /* i */
+           | ((imm & 0x07f) << 13) /* imm7b */
+           | ((r3 & 0x7f) << 20)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_m4(int qp, uint64_t opc, int r2, int r3)
+{
+    return opc
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_m18(int qp, uint64_t opc, int f1, int r2)
+{
+    return opc
+           | ((r2 & 0x7f) << 13)
+           | ((f1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_m19(int qp, uint64_t opc, int r1, int f2)
+{
+    return opc
+           | ((f2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_m34(int qp, uint64_t opc, int r1,
+                                   int sof, int sol, int sor)
+{
+    return opc
+           | ((sor & 0x0f) << 27)
+           | ((sol & 0x7f) << 20)
+           | ((sof & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_m48(int qp, uint64_t opc, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x100000) << 16) /* i */
+           | ((imm & 0x0fffff) << 6)  /* imm20a */
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_x2(int qp, uint64_t opc,
+                                  int r1, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x8000000000000000ull) >> 27) /* i */
+           |  (imm & 0x0000000000200000ull)        /* ic */
+           | ((imm & 0x00000000001f0000ull) << 6)  /* imm5c */
+           | ((imm & 0x000000000000ff80ull) << 20) /* imm9d */
+           | ((imm & 0x000000000000007full) << 13) /* imm7b */
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_x3(int qp, uint64_t opc, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x0800000000000000ull) >> 23) /* i */
+           | ((imm & 0x00000000000fffffull) << 13) /* imm20b */
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_x4(int qp, uint64_t opc, int b1, uint64_t imm)
+{
+    return opc
+           | ((imm & 0x0800000000000000ull) >> 23) /* i */
+           | ((imm & 0x00000000000fffffull) << 13) /* imm20b */
+           | ((b1 & 0x7) << 6)
+           | (qp & 0x3f);
+}
+
+
+/*
+ * Relocations - Note that we never encode branches elsewhere than slot 2.
+ */
+
+static void reloc_pcrel21b_slot2(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    uint64_t imm = target - pc;
+
+    pc->hi = (pc->hi & 0xf700000fffffffffull)
+             | ((imm & 0x100000) << 39)  /* s */
+             | ((imm & 0x0fffff) << 36); /* imm20b */
+}
+
+static uint64_t get_reloc_pcrel21b_slot2(tcg_insn_unit *pc)
+{
+    int64_t high = pc->hi;
+
+    return ((high >> 39) & 0x100000) + /* s */
+           ((high >> 36) & 0x0fffff);  /* imm20b */
+}
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    tcg_debug_assert(addend == 0);
+    tcg_debug_assert(type == R_IA64_PCREL21B);
+    reloc_pcrel21b_slot2(code_ptr, (tcg_insn_unit *)value);
+}
+
+/*
+ * Constraints
+ */
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch(ct_str[0]) {
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set(ct->u.regs, 0xffffffffffffffffull);
+        break;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_S22;
+        break;
+    case 'S':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set(ct->u.regs, 0xffffffffffffffffull);
+#if defined(CONFIG_SOFTMMU)
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R56);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R57);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R58);
+#endif
+        break;
+    case 'Z':
+        /* We are cheating a bit here, using the fact that the register
+           r0 is also the register number 0. Hence there is no need
+           to check for const_args in each instruction. */
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct;
+    ct = arg_ct->ct;
+    if (ct & TCG_CT_CONST)
+        return 1;
+    else if ((ct & TCG_CT_CONST_ZERO) && val == 0)
+        return 1;
+    else if ((ct & TCG_CT_CONST_S22) && val == ((int32_t)val << 10) >> 10)
+        return 1;
+    else
+        return 0;
+}
+
+/*
+ * Code generation
+ */
+
+static tcg_insn_unit *tb_ret_addr;
+
+static inline void tcg_out_bundle(TCGContext *s, int template,
+                                  uint64_t slot0, uint64_t slot1,
+                                  uint64_t slot2)
+{
+    template &= 0x1f;          /* 5 bits */
+    slot0 &= 0x1ffffffffffull; /* 41 bits */
+    slot1 &= 0x1ffffffffffull; /* 41 bits */
+    slot2 &= 0x1ffffffffffull; /* 41 bits */
+
+    *s->code_ptr++ = (tcg_insn_unit){
+        (slot1 << 46) | (slot0 << 5) | template,
+        (slot2 << 23) | (slot1 >> 18)
+    };
+}
+
+static inline uint64_t tcg_opc_mov_a(int qp, TCGReg dst, TCGReg src)
+{
+    return tcg_opc_a4(qp, OPC_ADDS_A4, dst, 0, src);
+}
+
+static inline void tcg_out_mov(TCGContext *s, TCGType type,
+                               TCGReg ret, TCGReg arg)
+{
+    tcg_out_bundle(s, mmI,
+                   INSN_NOP_M,
+                   INSN_NOP_M,
+                   tcg_opc_mov_a(TCG_REG_P0, ret, arg));
+}
+
+static inline uint64_t tcg_opc_movi_a(int qp, TCGReg dst, int64_t src)
+{
+    tcg_debug_assert(src == sextract64(src, 0, 22));
+    return tcg_opc_a5(qp, OPC_ADDL_A5, dst, src, TCG_REG_R0);
+}
+
+static inline void tcg_out_movi(TCGContext *s, TCGType type,
+                                TCGReg reg, tcg_target_long arg)
+{
+    tcg_out_bundle(s, mLX,
+                   INSN_NOP_M,
+                   tcg_opc_l2 (arg),
+                   tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2, reg, arg));
+}
+
+static void tcg_out_br(TCGContext *s, TCGLabel *l)
+{
+    uint64_t imm;
+
+    /* We pay attention here to not modify the branch target by reading
+       the existing value and using it again. This ensure that caches and
+       memory are kept coherent during retranslation. */
+    if (l->has_value) {
+        imm = l->u.value_ptr -  s->code_ptr;
+    } else {
+        imm = get_reloc_pcrel21b_slot2(s->code_ptr);
+        tcg_out_reloc(s, s->code_ptr, R_IA64_PCREL21B, l, 0);
+    }
+
+    tcg_out_bundle(s, mmB,
+                   INSN_NOP_M,
+                   INSN_NOP_M,
+                   tcg_opc_b1(TCG_REG_P0, OPC_BR_SPTK_MANY_B1, imm));
+}
+
+static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *desc)
+{
+    uintptr_t func = desc->lo, gp = desc->hi, disp;
+
+    /* Look through the function descriptor.  */
+    tcg_out_bundle(s, mlx,
+                   INSN_NOP_M,
+                   tcg_opc_l2 (gp),
+                   tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2, TCG_REG_R1, gp));
+    disp = (tcg_insn_unit *)func - s->code_ptr;
+    tcg_out_bundle(s, mLX,
+                   INSN_NOP_M,
+                   tcg_opc_l4 (disp),
+                   tcg_opc_x4 (TCG_REG_P0, OPC_BRL_CALL_SPTK_MANY_X4,
+                               TCG_REG_B0, disp));
+}
+
+static void tcg_out_exit_tb(TCGContext *s, tcg_target_long arg)
+{
+    uint64_t imm, opc1;
+
+    /* At least arg == 0 is a common operation.  */
+    if (arg == sextract64(arg, 0, 22)) {
+        opc1 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R8, arg);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R8, arg);
+        opc1 = INSN_NOP_M;
+    }
+
+    imm = tb_ret_addr - s->code_ptr;
+
+    tcg_out_bundle(s, mLX,
+                   opc1,
+                   tcg_opc_l3 (imm),
+                   tcg_opc_x3 (TCG_REG_P0, OPC_BRL_SPTK_MANY_X3, imm));
+}
+
+static inline void tcg_out_goto_tb(TCGContext *s, TCGArg arg)
+{
+    if (s->tb_jmp_insn_offset) {
+        /* direct jump method */
+        tcg_abort();
+    } else {
+        /* indirect jump method */
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2,
+                     (tcg_target_long)(s->tb_jmp_target_addr + arg));
+        tcg_out_bundle(s, MmI,
+                       tcg_opc_m1 (TCG_REG_P0, OPC_LD8_M1,
+                                   TCG_REG_R2, TCG_REG_R2),
+                       INSN_NOP_M,
+                       tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21, TCG_REG_B6,
+                                   TCG_REG_R2, 0));
+        tcg_out_bundle(s, mmB,
+                       INSN_NOP_M,
+                       INSN_NOP_M,
+                       tcg_opc_b4 (TCG_REG_P0, OPC_BR_SPTK_MANY_B4,
+                                   TCG_REG_B6));
+    }
+    s->tb_jmp_reset_offset[arg] = tcg_current_code_size(s);
+}
+
+static inline void tcg_out_jmp(TCGContext *s, TCGArg addr)
+{
+    tcg_out_bundle(s, mmI,
+                   INSN_NOP_M,
+                   INSN_NOP_M,
+                   tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21, TCG_REG_B6, addr, 0));
+    tcg_out_bundle(s, mmB,
+                   INSN_NOP_M,
+                   INSN_NOP_M,
+                   tcg_opc_b4(TCG_REG_P0, OPC_BR_SPTK_MANY_B4, TCG_REG_B6));
+}
+
+static inline void tcg_out_ld_rel(TCGContext *s, uint64_t opc_m4, TCGArg arg,
+                                  TCGArg arg1, tcg_target_long arg2)
+{
+    if (arg2 == ((int16_t)arg2 >> 2) << 2) {
+        tcg_out_bundle(s, MmI,
+                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4,
+                                  TCG_REG_R2, arg2, arg1),
+                       tcg_opc_m1 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
+                       INSN_NOP_I);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, arg2);
+        tcg_out_bundle(s, MmI,
+                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1,
+                                   TCG_REG_R2, TCG_REG_R2, arg1),
+                       tcg_opc_m1 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
+                       INSN_NOP_I);
+    }
+}
+
+static inline void tcg_out_st_rel(TCGContext *s, uint64_t opc_m4, TCGArg arg,
+                                  TCGArg arg1, tcg_target_long arg2)
+{
+    if (arg2 == ((int16_t)arg2 >> 2) << 2) {
+        tcg_out_bundle(s, MmI,
+                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4,
+                                  TCG_REG_R2, arg2, arg1),
+                       tcg_opc_m4 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
+                       INSN_NOP_I);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, arg2);
+        tcg_out_bundle(s, MmI,
+                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1,
+                                   TCG_REG_R2, TCG_REG_R2, arg1),
+                       tcg_opc_m4 (TCG_REG_P0, opc_m4, arg, TCG_REG_R2),
+                       INSN_NOP_I);
+    }
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    if (type == TCG_TYPE_I32) {
+        tcg_out_ld_rel(s, OPC_LD4_M1, arg, arg1, arg2);
+    } else {
+        tcg_out_ld_rel(s, OPC_LD8_M1, arg, arg1, arg2);
+    }
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    if (type == TCG_TYPE_I32) {
+        tcg_out_st_rel(s, OPC_ST4_M4, arg, arg1, arg2);
+    } else {
+        tcg_out_st_rel(s, OPC_ST8_M4, arg, arg1, arg2);
+    }
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    if (val == 0) {
+        tcg_out_st(s, type, TCG_REG_R0, base, ofs);
+        return true;
+    }
+    return false;
+}
+
+static inline void tcg_out_alu(TCGContext *s, uint64_t opc_a1, uint64_t opc_a3,
+                               TCGReg ret, TCGArg arg1, int const_arg1,
+                               TCGArg arg2, int const_arg2)
+{
+    uint64_t opc1 = 0, opc2 = 0, opc3 = 0;
+
+    if (const_arg2 && arg2 != 0) {
+        opc2 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R3, arg2);
+        arg2 = TCG_REG_R3;
+    }
+    if (const_arg1 && arg1 != 0) {
+        if (opc_a3 && arg1 == (int8_t)arg1) {
+            opc3 = tcg_opc_a3(TCG_REG_P0, opc_a3, ret, arg1, arg2);
+        } else {
+            opc1 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, arg1);
+            arg1 = TCG_REG_R2;
+        }
+    }
+    if (opc3 == 0) {
+        opc3 = tcg_opc_a1(TCG_REG_P0, opc_a1, ret, arg1, arg2);
+    }
+
+    tcg_out_bundle(s, (opc1 || opc2 ? mII : miI),
+                   opc1 ? opc1 : INSN_NOP_M,
+                   opc2 ? opc2 : INSN_NOP_I,
+                   opc3);
+}
+
+static inline void tcg_out_add(TCGContext *s, TCGReg ret, TCGReg arg1,
+                               TCGArg arg2, int const_arg2)
+{
+    if (const_arg2 && arg2 == sextract64(arg2, 0, 14)) {
+        tcg_out_bundle(s, mmI,
+                       INSN_NOP_M,
+                       INSN_NOP_M,
+                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, ret, arg2, arg1));
+    } else {
+        tcg_out_alu(s, OPC_ADD_A1, 0, ret, arg1, 0, arg2, const_arg2);
+    }
+}
+
+static inline void tcg_out_sub(TCGContext *s, TCGReg ret, TCGArg arg1,
+                               int const_arg1, TCGArg arg2, int const_arg2)
+{
+    if (!const_arg1 && const_arg2 && -arg2 == sextract64(-arg2, 0, 14)) {
+        tcg_out_bundle(s, mmI,
+                       INSN_NOP_M,
+                       INSN_NOP_M,
+                       tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, ret, -arg2, arg1));
+    } else {
+        tcg_out_alu(s, OPC_SUB_A1, OPC_SUB_A3, ret,
+                    arg1, const_arg1, arg2, const_arg2);
+    }
+}
+
+static inline void tcg_out_eqv(TCGContext *s, TCGArg ret,
+                               TCGArg arg1, int const_arg1,
+                               TCGArg arg2, int const_arg2)
+{
+    tcg_out_bundle(s, mII,
+                   INSN_NOP_M,
+                   tcg_opc_a1 (TCG_REG_P0, OPC_XOR_A1, ret, arg1, arg2),
+                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, ret, -1, ret));
+}
+
+static inline void tcg_out_nand(TCGContext *s, TCGArg ret,
+                                TCGArg arg1, int const_arg1,
+                                TCGArg arg2, int const_arg2)
+{
+    tcg_out_bundle(s, mII,
+                   INSN_NOP_M,
+                   tcg_opc_a1 (TCG_REG_P0, OPC_AND_A1, ret, arg1, arg2),
+                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, ret, -1, ret));
+}
+
+static inline void tcg_out_nor(TCGContext *s, TCGArg ret,
+                               TCGArg arg1, int const_arg1,
+                               TCGArg arg2, int const_arg2)
+{
+    tcg_out_bundle(s, mII,
+                   INSN_NOP_M,
+                   tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret, arg1, arg2),
+                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, ret, -1, ret));
+}
+
+static inline void tcg_out_orc(TCGContext *s, TCGArg ret,
+                               TCGArg arg1, int const_arg1,
+                               TCGArg arg2, int const_arg2)
+{
+    tcg_out_bundle(s, mII,
+                   INSN_NOP_M,
+                   tcg_opc_a3 (TCG_REG_P0, OPC_ANDCM_A3, TCG_REG_R2, -1, arg2),
+                   tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret, arg1, TCG_REG_R2));
+}
+
+static inline void tcg_out_mul(TCGContext *s, TCGArg ret,
+                               TCGArg arg1, TCGArg arg2)
+{
+    tcg_out_bundle(s, mmI,
+                   tcg_opc_m18(TCG_REG_P0, OPC_SETF_SIG_M18, TCG_REG_F6, arg1),
+                   tcg_opc_m18(TCG_REG_P0, OPC_SETF_SIG_M18, TCG_REG_F7, arg2),
+                   INSN_NOP_I);
+    tcg_out_bundle(s, mmF,
+                   INSN_NOP_M,
+                   INSN_NOP_M,
+                   tcg_opc_f2 (TCG_REG_P0, OPC_XMA_L_F2, TCG_REG_F6, TCG_REG_F6,
+                               TCG_REG_F7, TCG_REG_F0));
+    tcg_out_bundle(s, miI,
+                   tcg_opc_m19(TCG_REG_P0, OPC_GETF_SIG_M19, ret, TCG_REG_F6),
+                   INSN_NOP_I,
+                   INSN_NOP_I);
+}
+
+static inline void tcg_out_sar_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                   TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_I11,
+                                   ret, arg1, arg2, 31 - arg2));
+    } else {
+        tcg_out_bundle(s, mII,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3,
+                                   TCG_REG_R3, 0x1f, arg2),
+                       tcg_opc_i29(TCG_REG_P0, OPC_SXT4_I29, TCG_REG_R2, arg1),
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_I5, ret,
+                                   TCG_REG_R2, TCG_REG_R3));
+    }
+}
+
+static inline void tcg_out_sar_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                   TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_I11,
+                                   ret, arg1, arg2, 63 - arg2));
+    } else {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_I5, ret, arg1, arg2));
+    }
+}
+
+static inline void tcg_out_shl_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                   TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret,
+                                   arg1, 63 - arg2, 31 - arg2));
+    } else {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R2,
+                                   0x1f, arg2),
+                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, ret,
+                                   arg1, TCG_REG_R2));
+    }
+}
+
+static inline void tcg_out_shl_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                   TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret,
+                                   arg1, 63 - arg2, 63 - arg2));
+    } else {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, ret,
+                                   arg1, arg2));
+    }
+}
+
+static inline void tcg_out_shr_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                   TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
+                                   arg1, arg2, 31 - arg2));
+    } else {
+        tcg_out_bundle(s, mII,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R3,
+                                   0x1f, arg2),
+                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29, TCG_REG_R2, arg1),
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
+                                   TCG_REG_R2, TCG_REG_R3));
+    }
+}
+
+static inline void tcg_out_shr_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                   TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
+                                   arg1, arg2, 63 - arg2));
+    } else {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
+                                   arg1, arg2));
+    }
+}
+
+static inline void tcg_out_rotl_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                    TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
+                                   TCG_REG_R2, arg1, arg1),
+                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
+                                   TCG_REG_R2, 32 - arg2, 31));
+    } else {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
+                                   TCG_REG_R2, arg1, arg1),
+                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R3,
+                                   0x1f, arg2));
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_SUB_A3, TCG_REG_R3,
+                                   0x20, TCG_REG_R3),
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
+                                   TCG_REG_R2, TCG_REG_R3));
+    }
+}
+
+static inline void tcg_out_rotl_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                    TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i10(TCG_REG_P0, OPC_SHRP_I10, ret, arg1,
+                                   arg1, 0x40 - arg2));
+    } else {
+        tcg_out_bundle(s, mII,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_SUB_A3, TCG_REG_R2,
+                                   0x40, arg2),
+                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, TCG_REG_R3,
+                                   arg1, arg2),
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, TCG_REG_R2,
+                                   arg1, TCG_REG_R2));
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret,
+                                   TCG_REG_R2, TCG_REG_R3));
+    }
+}
+
+static inline void tcg_out_rotr_i32(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                    TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
+                                   TCG_REG_R2, arg1, arg1),
+                       tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, ret,
+                                   TCG_REG_R2, arg2, 31));
+    } else {
+        tcg_out_bundle(s, mII,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_AND_A3, TCG_REG_R3,
+                                   0x1f, arg2),
+                       tcg_opc_i2 (TCG_REG_P0, OPC_UNPACK4_L_I2,
+                                   TCG_REG_R2, arg1, arg1),
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, ret,
+                                   TCG_REG_R2, TCG_REG_R3));
+    }
+}
+
+static inline void tcg_out_rotr_i64(TCGContext *s, TCGArg ret, TCGArg arg1,
+                                    TCGArg arg2, int const_arg2)
+{
+    if (const_arg2) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_i10(TCG_REG_P0, OPC_SHRP_I10, ret, arg1,
+                                   arg1, arg2));
+    } else {
+        tcg_out_bundle(s, mII,
+                       tcg_opc_a3 (TCG_REG_P0, OPC_SUB_A3, TCG_REG_R2,
+                                   0x40, arg2),
+                       tcg_opc_i5 (TCG_REG_P0, OPC_SHR_U_I5, TCG_REG_R3,
+                                   arg1, arg2),
+                       tcg_opc_i7 (TCG_REG_P0, OPC_SHL_I7, TCG_REG_R2,
+                                   arg1, TCG_REG_R2));
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_a1 (TCG_REG_P0, OPC_OR_A1, ret,
+                                   TCG_REG_R2, TCG_REG_R3));
+    }
+}
+
+static const uint64_t opc_ext_i29[8] = {
+    OPC_ZXT1_I29, OPC_ZXT2_I29, OPC_ZXT4_I29, 0,
+    OPC_SXT1_I29, OPC_SXT2_I29, OPC_SXT4_I29, 0
+};
+
+static inline uint64_t tcg_opc_ext_i(int qp, TCGMemOp opc, TCGReg d, TCGReg s)
+{
+    if ((opc & MO_SIZE) == MO_64) {
+        return tcg_opc_mov_a(qp, d, s);
+    } else {
+        return tcg_opc_i29(qp, opc_ext_i29[opc & MO_SSIZE], d, s);
+    }
+}
+
+static inline void tcg_out_ext(TCGContext *s, uint64_t opc_i29,
+                               TCGArg ret, TCGArg arg)
+{
+    tcg_out_bundle(s, miI,
+                   INSN_NOP_M,
+                   INSN_NOP_I,
+                   tcg_opc_i29(TCG_REG_P0, opc_i29, ret, arg));
+}
+
+static inline uint64_t tcg_opc_bswap64_i(int qp, TCGReg d, TCGReg s)
+{
+    return tcg_opc_i3(qp, OPC_MUX1_I3, d, s, 0xb);
+}
+
+static inline void tcg_out_bswap16(TCGContext *s, TCGArg ret, TCGArg arg)
+{
+    tcg_out_bundle(s, mII,
+                   INSN_NOP_M,
+                   tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret, arg, 15, 15),
+                   tcg_opc_bswap64_i(TCG_REG_P0, ret, ret));
+}
+
+static inline void tcg_out_bswap32(TCGContext *s, TCGArg ret, TCGArg arg)
+{
+    tcg_out_bundle(s, mII,
+                   INSN_NOP_M,
+                   tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, ret, arg, 31, 31),
+                   tcg_opc_bswap64_i(TCG_REG_P0, ret, ret));
+}
+
+static inline void tcg_out_bswap64(TCGContext *s, TCGArg ret, TCGArg arg)
+{
+    tcg_out_bundle(s, miI,
+                   INSN_NOP_M,
+                   INSN_NOP_I,
+                   tcg_opc_bswap64_i(TCG_REG_P0, ret, arg));
+}
+
+static inline void tcg_out_deposit(TCGContext *s, TCGArg ret, TCGArg a1,
+                                   TCGArg a2, int const_a2, int pos, int len)
+{
+    uint64_t i1 = 0, i2 = 0;
+    int cpos = 63 - pos, lm1 = len - 1;
+
+    if (const_a2) {
+        /* Truncate the value of a constant a2 to the width of the field.  */
+        int mask = (1u << len) - 1;
+        a2 &= mask;
+
+        if (a2 == 0 || a2 == mask) {
+            /* 1-bit signed constant inserted into register.  */
+            i2 = tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, ret, a2, a1, cpos, lm1);
+        } else {
+            /* Otherwise, load any constant into a temporary.  Do this into
+               the first I slot to help out with cross-unit delays.  */
+            i1 = tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, a2);
+            a2 = TCG_REG_R2;
+        }
+    }
+    if (i2 == 0) {
+        i2 = tcg_opc_i15(TCG_REG_P0, OPC_DEP_I15, ret, a2, a1, cpos, lm1);
+    }
+    tcg_out_bundle(s, (i1 ? mII : miI),
+                   INSN_NOP_M,
+                   i1 ? i1 : INSN_NOP_I,
+                   i2);
+}
+
+static inline uint64_t tcg_opc_cmp_a(int qp, TCGCond cond, TCGArg arg1,
+                                     TCGArg arg2, int cmp4)
+{
+    uint64_t opc_eq_a6, opc_lt_a6, opc_ltu_a6;
+
+    if (cmp4) {
+        opc_eq_a6 = OPC_CMP4_EQ_A6;
+        opc_lt_a6 = OPC_CMP4_LT_A6;
+        opc_ltu_a6 = OPC_CMP4_LTU_A6;
+    } else {
+        opc_eq_a6 = OPC_CMP_EQ_A6;
+        opc_lt_a6 = OPC_CMP_LT_A6;
+        opc_ltu_a6 = OPC_CMP_LTU_A6;
+    }
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        return tcg_opc_a6 (qp, opc_eq_a6,  TCG_REG_P6, TCG_REG_P7, arg1, arg2);
+    case TCG_COND_NE:
+        return tcg_opc_a6 (qp, opc_eq_a6,  TCG_REG_P7, TCG_REG_P6, arg1, arg2);
+    case TCG_COND_LT:
+        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P6, TCG_REG_P7, arg1, arg2);
+    case TCG_COND_LTU:
+        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P6, TCG_REG_P7, arg1, arg2);
+    case TCG_COND_GE:
+        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P7, TCG_REG_P6, arg1, arg2);
+    case TCG_COND_GEU:
+        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P7, TCG_REG_P6, arg1, arg2);
+    case TCG_COND_LE:
+        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P7, TCG_REG_P6, arg2, arg1);
+    case TCG_COND_LEU:
+        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P7, TCG_REG_P6, arg2, arg1);
+    case TCG_COND_GT:
+        return tcg_opc_a6 (qp, opc_lt_a6,  TCG_REG_P6, TCG_REG_P7, arg2, arg1);
+    case TCG_COND_GTU:
+        return tcg_opc_a6 (qp, opc_ltu_a6, TCG_REG_P6, TCG_REG_P7, arg2, arg1);
+    default:
+        tcg_abort();
+        break;
+    }
+}
+
+static inline void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
+                                  TCGReg arg2, TCGLabel *l, int cmp4)
+{
+    uint64_t imm;
+
+    /* We pay attention here to not modify the branch target by reading
+       the existing value and using it again. This ensure that caches and
+       memory are kept coherent during retranslation. */
+    if (l->has_value) {
+        imm = l->u.value_ptr - s->code_ptr;
+    } else {
+        imm = get_reloc_pcrel21b_slot2(s->code_ptr);
+        tcg_out_reloc(s, s->code_ptr, R_IA64_PCREL21B, l, 0);
+    }
+
+    tcg_out_bundle(s, miB,
+                   INSN_NOP_M,
+                   tcg_opc_cmp_a(TCG_REG_P0, cond, arg1, arg2, cmp4),
+                   tcg_opc_b1(TCG_REG_P6, OPC_BR_DPTK_FEW_B1, imm));
+}
+
+static inline void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGArg ret,
+                                   TCGArg arg1, TCGArg arg2, int cmp4)
+{
+    tcg_out_bundle(s, MmI,
+                   tcg_opc_cmp_a(TCG_REG_P0, cond, arg1, arg2, cmp4),
+                   tcg_opc_movi_a(TCG_REG_P6, ret, 1),
+                   tcg_opc_movi_a(TCG_REG_P7, ret, 0));
+}
+
+static inline void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGArg ret,
+                                   TCGArg c1, TCGArg c2,
+                                   TCGArg v1, int const_v1,
+                                   TCGArg v2, int const_v2, int cmp4)
+{
+    uint64_t opc1, opc2;
+
+    if (const_v1) {
+        opc1 = tcg_opc_movi_a(TCG_REG_P6, ret, v1);
+    } else if (ret == v1) {
+        opc1 = INSN_NOP_M;
+    } else {
+        opc1 = tcg_opc_mov_a(TCG_REG_P6, ret, v1);
+    }
+    if (const_v2) {
+        opc2 = tcg_opc_movi_a(TCG_REG_P7, ret, v2);
+    } else if (ret == v2) {
+        opc2 = INSN_NOP_I;
+    } else {
+        opc2 = tcg_opc_mov_a(TCG_REG_P7, ret, v2);
+    }
+
+    tcg_out_bundle(s, MmI,
+                   tcg_opc_cmp_a(TCG_REG_P0, cond, c1, c2, cmp4),
+                   opc1,
+                   opc2);
+}
+
+#if defined(CONFIG_SOFTMMU)
+/* We're expecting to use an signed 22-bit immediate add.  */
+QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
+                  > 0x1fffff)
+
+/* Load and compare a TLB entry, and return the result in (p6, p7).
+   R2 is loaded with the addend TLB entry.
+   R57 is loaded with the address, zero extented on 32-bit targets.
+   R1, R3 are clobbered, leaving R56 free for...
+   BSWAP_1, BSWAP_2 and I-slot insns for swapping data for store.  */
+static inline void tcg_out_qemu_tlb(TCGContext *s, TCGReg addr_reg,
+                                    TCGMemOp s_bits, int off_rw, int off_add,
+                                    uint64_t bswap1, uint64_t bswap2)
+{
+     /*
+        .mii
+        mov	r2 = off_rw
+        extr.u	r3 = addr_reg, ...		# extract tlb page
+        zxt4	r57 = addr_reg                  # or mov for 64-bit guest
+        ;;
+        .mii
+        addl	r2 = r2, areg0
+        shl	r3 = r3, cteb                   # via dep.z
+        dep	r1 = 0, r57, ...                # zero page ofs, keep align
+        ;;
+        .mmi
+        add	r2 = r2, r3
+        ;;
+        ld4	r3 = [r2], off_add-off_rw	# or ld8 for 64-bit guest
+        nop
+        ;;
+        .mmi
+        nop
+        cmp.eq	p6, p7 = r3, r58
+        nop
+        ;;
+      */
+    tcg_out_bundle(s, miI,
+                   tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, off_rw),
+                   tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, TCG_REG_R3,
+                               addr_reg, TARGET_PAGE_BITS, CPU_TLB_BITS - 1),
+                   tcg_opc_ext_i(TCG_REG_P0,
+                                 TARGET_LONG_BITS == 32 ? MO_UL : MO_Q,
+                                 TCG_REG_R57, addr_reg));
+    tcg_out_bundle(s, miI,
+                   tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
+                               TCG_REG_R2, TCG_AREG0),
+                   tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, TCG_REG_R3,
+                               TCG_REG_R3, 63 - CPU_TLB_ENTRY_BITS,
+                               63 - CPU_TLB_ENTRY_BITS),
+                   tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, TCG_REG_R1, 0,
+                               TCG_REG_R57, 63 - s_bits,
+                               TARGET_PAGE_BITS - s_bits - 1));
+    tcg_out_bundle(s, MmI,
+                   tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1,
+                               TCG_REG_R2, TCG_REG_R2, TCG_REG_R3),
+                   tcg_opc_m3 (TCG_REG_P0,
+                               (TARGET_LONG_BITS == 32
+                                ? OPC_LD4_M3 : OPC_LD8_M3), TCG_REG_R3,
+                               TCG_REG_R2, off_add - off_rw),
+                   bswap1);
+    tcg_out_bundle(s, mmI,
+                   tcg_opc_m1 (TCG_REG_P0, OPC_LD8_M1, TCG_REG_R2, TCG_REG_R2),
+                   tcg_opc_a6 (TCG_REG_P0, OPC_CMP_EQ_A6, TCG_REG_P6,
+                               TCG_REG_P7, TCG_REG_R1, TCG_REG_R3),
+                   bswap2);
+}
+
+typedef struct TCGLabelQemuLdst {
+    bool is_ld;
+    TCGMemOp size;
+    tcg_insn_unit *label_ptr;     /* label pointers to be updated */
+    struct TCGLabelQemuLdst *next;
+} TCGLabelQemuLdst;
+
+typedef struct TCGBackendData {
+    TCGLabelQemuLdst *labels;
+} TCGBackendData;
+
+static inline void tcg_out_tb_init(TCGContext *s)
+{
+    s->be->labels = NULL;
+}
+
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOp opc,
+                                tcg_insn_unit *label_ptr)
+{
+    TCGBackendData *be = s->be;
+    TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l));
+
+    l->is_ld = is_ld;
+    l->size = opc & MO_SIZE;
+    l->label_ptr = label_ptr;
+    l->next = be->labels;
+    be->labels = l;
+}
+
+static bool tcg_out_tb_finalize(TCGContext *s)
+{
+    static const void * const helpers[8] = {
+        helper_ret_stb_mmu,
+        helper_le_stw_mmu,
+        helper_le_stl_mmu,
+        helper_le_stq_mmu,
+        helper_ret_ldub_mmu,
+        helper_le_lduw_mmu,
+        helper_le_ldul_mmu,
+        helper_le_ldq_mmu,
+    };
+    tcg_insn_unit *thunks[8] = { };
+    TCGLabelQemuLdst *l;
+
+    for (l = s->be->labels; l != NULL; l = l->next) {
+        long x = l->is_ld * 4 + l->size;
+        tcg_insn_unit *dest = thunks[x];
+
+        /* The out-of-line thunks are all the same; load the return address
+           from B0, load the GP, and branch to the code.  Note that we are
+           always post-call, so the register window has rolled, so we're
+           using incoming parameter register numbers, not outgoing.  */
+        if (dest == NULL) {
+            uintptr_t *desc = (uintptr_t *)helpers[x];
+            uintptr_t func = desc[0], gp = desc[1], disp;
+
+            thunks[x] = dest = s->code_ptr;
+
+            tcg_out_bundle(s, mlx,
+                           INSN_NOP_M,
+                           tcg_opc_l2 (gp),
+                           tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2,
+                                       TCG_REG_R1, gp));
+            tcg_out_bundle(s, mii,
+                           INSN_NOP_M,
+                           INSN_NOP_I,
+                           tcg_opc_i22(TCG_REG_P0, OPC_MOV_I22,
+                                       l->is_ld ? TCG_REG_R35 : TCG_REG_R36,
+                                       TCG_REG_B0));
+            disp = (tcg_insn_unit *)func - s->code_ptr;
+            tcg_out_bundle(s, mLX,
+                           INSN_NOP_M,
+                           tcg_opc_l3 (disp),
+                           tcg_opc_x3 (TCG_REG_P0, OPC_BRL_SPTK_MANY_X3, disp));
+        }
+
+        reloc_pcrel21b_slot2(l->label_ptr, dest);
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
+{
+    static const uint64_t opc_ld_m1[4] = {
+        OPC_LD1_M1, OPC_LD2_M1, OPC_LD4_M1, OPC_LD8_M1
+    };
+    int addr_reg, data_reg, mem_index;
+    TCGMemOpIdx oi;
+    TCGMemOp opc, s_bits;
+    uint64_t fin1, fin2;
+    tcg_insn_unit *label_ptr;
+
+    data_reg = args[0];
+    addr_reg = args[1];
+    oi = args[2];
+    opc = get_memop(oi);
+    mem_index = get_mmuidx(oi);
+    s_bits = opc & MO_SIZE;
+
+    /* Read the TLB entry */
+    tcg_out_qemu_tlb(s, addr_reg, s_bits,
+                     offsetof(CPUArchState, tlb_table[mem_index][0].addr_read),
+                     offsetof(CPUArchState, tlb_table[mem_index][0].addend),
+                     INSN_NOP_I, INSN_NOP_I);
+
+    /* P6 is the fast path, and P7 the slow path */
+
+    fin2 = 0;
+    if (opc & MO_BSWAP) {
+        fin1 = tcg_opc_bswap64_i(TCG_REG_P0, data_reg, TCG_REG_R8);
+        if (s_bits < MO_64) {
+            int shift = 64 - (8 << s_bits);
+            fin2 = (opc & MO_SIGN ? OPC_EXTR_I11 : OPC_EXTR_U_I11);
+            fin2 = tcg_opc_i11(TCG_REG_P0, fin2,
+                               data_reg, data_reg, shift, 63 - shift);
+        }
+    } else {
+        fin1 = tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, TCG_REG_R8);
+    }
+
+    tcg_out_bundle(s, mmI,
+                   tcg_opc_mov_a(TCG_REG_P7, TCG_REG_R56, TCG_AREG0),
+                   tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R2,
+                               TCG_REG_R2, TCG_REG_R57),
+                   tcg_opc_movi_a(TCG_REG_P7, TCG_REG_R58, oi));
+    label_ptr = s->code_ptr;
+    tcg_out_bundle(s, miB,
+                   tcg_opc_m1 (TCG_REG_P6, opc_ld_m1[s_bits],
+                               TCG_REG_R8, TCG_REG_R2),
+                   INSN_NOP_I,
+                   tcg_opc_b3 (TCG_REG_P7, OPC_BR_CALL_SPNT_FEW_B3, TCG_REG_B0,
+                               get_reloc_pcrel21b_slot2(label_ptr)));
+
+    add_qemu_ldst_label(s, 1, opc, label_ptr);
+
+    /* Note that we always use LE helper functions, so the bswap insns
+       here for the fast path also apply to the slow path.  */
+    tcg_out_bundle(s, (fin2 ? mII : miI),
+                   INSN_NOP_M,
+                   fin1,
+                   fin2 ? fin2 : INSN_NOP_I);
+}
+
+static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
+{
+    static const uint64_t opc_st_m4[4] = {
+        OPC_ST1_M4, OPC_ST2_M4, OPC_ST4_M4, OPC_ST8_M4
+    };
+    TCGReg addr_reg, data_reg;
+    int mem_index;
+    uint64_t pre1, pre2;
+    TCGMemOpIdx oi;
+    TCGMemOp opc, s_bits;
+    tcg_insn_unit *label_ptr;
+
+    data_reg = args[0];
+    addr_reg = args[1];
+    oi = args[2];
+    opc = get_memop(oi);
+    mem_index = get_mmuidx(oi);
+    s_bits = opc & MO_SIZE;
+
+    /* Note that we always use LE helper functions, so the bswap insns
+       that are here for the fast path also apply to the slow path,
+       and move the data into the argument register.  */
+    pre2 = INSN_NOP_I;
+    if (opc & MO_BSWAP) {
+        pre1 = tcg_opc_bswap64_i(TCG_REG_P0, TCG_REG_R58, data_reg);
+        if (s_bits < MO_64) {
+            int shift = 64 - (8 << s_bits);
+            pre2 = tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11,
+                               TCG_REG_R58, TCG_REG_R58, shift, 63 - shift);
+        }
+    } else {
+        /* Just move the data into place for the slow path.  */
+        pre1 = tcg_opc_ext_i(TCG_REG_P0, opc, TCG_REG_R58, data_reg);
+    }
+
+    tcg_out_qemu_tlb(s, addr_reg, s_bits,
+                     offsetof(CPUArchState, tlb_table[mem_index][0].addr_write),
+                     offsetof(CPUArchState, tlb_table[mem_index][0].addend),
+                     pre1, pre2);
+
+    /* P6 is the fast path, and P7 the slow path */
+    tcg_out_bundle(s, mmI,
+                   tcg_opc_mov_a(TCG_REG_P7, TCG_REG_R56, TCG_AREG0),
+                   tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R2,
+                               TCG_REG_R2, TCG_REG_R57),
+                   tcg_opc_movi_a(TCG_REG_P7, TCG_REG_R59, oi));
+    label_ptr = s->code_ptr;
+    tcg_out_bundle(s, miB,
+                   tcg_opc_m4 (TCG_REG_P6, opc_st_m4[s_bits],
+                               TCG_REG_R58, TCG_REG_R2),
+                   INSN_NOP_I,
+                   tcg_opc_b3 (TCG_REG_P7, OPC_BR_CALL_SPNT_FEW_B3, TCG_REG_B0,
+                               get_reloc_pcrel21b_slot2(label_ptr)));
+
+    add_qemu_ldst_label(s, 0, opc, label_ptr);
+}
+
+#else /* !CONFIG_SOFTMMU */
+# include "tcg-be-null.h"
+
+static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args)
+{
+    static uint64_t const opc_ld_m1[4] = {
+        OPC_LD1_M1, OPC_LD2_M1, OPC_LD4_M1, OPC_LD8_M1
+    };
+    int addr_reg, data_reg;
+    TCGMemOp opc, s_bits, bswap;
+
+    data_reg = args[0];
+    addr_reg = args[1];
+    opc = args[2];
+    s_bits = opc & MO_SIZE;
+    bswap = opc & MO_BSWAP;
+
+#if TARGET_LONG_BITS == 32
+    if (guest_base != 0) {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
+                                   TCG_REG_R3, addr_reg),
+                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
+                                   TCG_GUEST_BASE_REG, TCG_REG_R3));
+    } else {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
+                                   TCG_REG_R2, addr_reg),
+                       INSN_NOP_I);
+    }
+
+    if (!bswap) {
+        if (!(opc & MO_SIGN)) {
+            tcg_out_bundle(s, miI,
+                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                       data_reg, TCG_REG_R2),
+                           INSN_NOP_I,
+                           INSN_NOP_I);
+        } else {
+            tcg_out_bundle(s, mII,
+                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                       data_reg, TCG_REG_R2),
+                           INSN_NOP_I,
+                           tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, data_reg));
+        }
+    } else if (s_bits == MO_64) {
+            tcg_out_bundle(s, mII,
+                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                       data_reg, TCG_REG_R2),
+                           INSN_NOP_I,
+                           tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
+    } else {
+        if (s_bits == MO_16) {
+            tcg_out_bundle(s, mII,
+                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                       data_reg, TCG_REG_R2),
+                           INSN_NOP_I,
+                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                      data_reg, data_reg, 15, 15));
+        } else {
+            tcg_out_bundle(s, mII,
+                           tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                       data_reg, TCG_REG_R2),
+                           INSN_NOP_I,
+                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                      data_reg, data_reg, 31, 31));
+        }
+        if (!(opc & MO_SIGN)) {
+            tcg_out_bundle(s, miI,
+                           INSN_NOP_M,
+                           INSN_NOP_I,
+                           tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
+        } else {
+            tcg_out_bundle(s, mII,
+                           INSN_NOP_M,
+                           tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg),
+                           tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, data_reg));
+        }
+    }
+#else
+    if (guest_base != 0) {
+        tcg_out_bundle(s, MmI,
+                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
+                                   TCG_GUEST_BASE_REG, addr_reg),
+                       tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                   data_reg, TCG_REG_R2),
+                       INSN_NOP_I);
+    } else {
+        tcg_out_bundle(s, mmI,
+                       INSN_NOP_M,
+                       tcg_opc_m1 (TCG_REG_P0, opc_ld_m1[s_bits],
+                                   data_reg, addr_reg),
+                       INSN_NOP_I);
+    }
+
+    if (bswap && s_bits == MO_16) {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                   data_reg, data_reg, 15, 15),
+                       tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
+    } else if (bswap && s_bits == MO_32) {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                   data_reg, data_reg, 31, 31),
+                       tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
+    } else if (bswap && s_bits == MO_64) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_bswap64_i(TCG_REG_P0, data_reg, data_reg));
+    }
+    if (opc & MO_SIGN) {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       INSN_NOP_I,
+                       tcg_opc_ext_i(TCG_REG_P0, opc, data_reg, data_reg));
+    }
+#endif
+}
+
+static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args)
+{
+    static uint64_t const opc_st_m4[4] = {
+        OPC_ST1_M4, OPC_ST2_M4, OPC_ST4_M4, OPC_ST8_M4
+    };
+    int addr_reg, data_reg;
+#if TARGET_LONG_BITS == 64
+    uint64_t add_guest_base;
+#endif
+    TCGMemOp opc, s_bits, bswap;
+
+    data_reg = args[0];
+    addr_reg = args[1];
+    opc = args[2];
+    s_bits = opc & MO_SIZE;
+    bswap = opc & MO_BSWAP;
+
+#if TARGET_LONG_BITS == 32
+    if (guest_base != 0) {
+        tcg_out_bundle(s, mII,
+                       INSN_NOP_M,
+                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
+                                   TCG_REG_R3, addr_reg),
+                       tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
+                                   TCG_GUEST_BASE_REG, TCG_REG_R3));
+    } else {
+        tcg_out_bundle(s, miI,
+                       INSN_NOP_M,
+                       tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29,
+                                   TCG_REG_R2, addr_reg),
+                       INSN_NOP_I);
+    }
+
+    if (bswap) {
+        if (s_bits == MO_16) {
+            tcg_out_bundle(s, mII,
+                           INSN_NOP_M,
+                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                       TCG_REG_R3, data_reg, 15, 15),
+                           tcg_opc_bswap64_i(TCG_REG_P0,
+                                             TCG_REG_R3, TCG_REG_R3));
+            data_reg = TCG_REG_R3;
+        } else if (s_bits == MO_32) {
+            tcg_out_bundle(s, mII,
+                           INSN_NOP_M,
+                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                       TCG_REG_R3, data_reg, 31, 31),
+                           tcg_opc_bswap64_i(TCG_REG_P0,
+                                             TCG_REG_R3, TCG_REG_R3));
+            data_reg = TCG_REG_R3;
+        } else if (s_bits == MO_64) {
+            tcg_out_bundle(s, miI,
+                           INSN_NOP_M,
+                           INSN_NOP_I,
+                           tcg_opc_bswap64_i(TCG_REG_P0, TCG_REG_R3, data_reg));
+            data_reg = TCG_REG_R3;
+        }
+    }
+    tcg_out_bundle(s, mmI,
+                   tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
+                               data_reg, TCG_REG_R2),
+                   INSN_NOP_M,
+                   INSN_NOP_I);
+#else
+    if (guest_base != 0) {
+        add_guest_base = tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
+                                     TCG_GUEST_BASE_REG, addr_reg);
+        addr_reg = TCG_REG_R2;
+    } else {
+        add_guest_base = INSN_NOP_M;
+    }
+
+    if (!bswap) {
+        tcg_out_bundle(s, (guest_base ? MmI : mmI),
+                       add_guest_base,
+                       tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
+                                   data_reg, addr_reg),
+                       INSN_NOP_I);
+    } else {
+        if (s_bits == MO_16) {
+            tcg_out_bundle(s, mII,
+                           add_guest_base,
+                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                       TCG_REG_R3, data_reg, 15, 15),
+                           tcg_opc_bswap64_i(TCG_REG_P0,
+                                             TCG_REG_R3, TCG_REG_R3));
+            data_reg = TCG_REG_R3;
+        } else if (s_bits == MO_32) {
+            tcg_out_bundle(s, mII,
+                           add_guest_base,
+                           tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12,
+                                       TCG_REG_R3, data_reg, 31, 31),
+                           tcg_opc_bswap64_i(TCG_REG_P0,
+                                             TCG_REG_R3, TCG_REG_R3));
+            data_reg = TCG_REG_R3;
+        } else if (s_bits == MO_64) {
+            tcg_out_bundle(s, miI,
+                           add_guest_base,
+                           INSN_NOP_I,
+                           tcg_opc_bswap64_i(TCG_REG_P0, TCG_REG_R3, data_reg));
+            data_reg = TCG_REG_R3;
+        }
+        tcg_out_bundle(s, miI,
+                       tcg_opc_m4 (TCG_REG_P0, opc_st_m4[s_bits],
+                                   data_reg, addr_reg),
+                       INSN_NOP_I,
+                       INSN_NOP_I);
+    }
+#endif
+}
+
+#endif
+
+static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                              const TCGArg *args, const int *const_args)
+{
+    switch(opc) {
+    case INDEX_op_exit_tb:
+        tcg_out_exit_tb(s, args[0]);
+        break;
+    case INDEX_op_br:
+        tcg_out_br(s, arg_label(args[0]));
+        break;
+    case INDEX_op_goto_tb:
+        tcg_out_goto_tb(s, args[0]);
+        break;
+
+    case INDEX_op_ld8u_i32:
+    case INDEX_op_ld8u_i64:
+        tcg_out_ld_rel(s, OPC_LD1_M1, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld8s_i32:
+    case INDEX_op_ld8s_i64:
+        tcg_out_ld_rel(s, OPC_LD1_M1, args[0], args[1], args[2]);
+        tcg_out_ext(s, OPC_SXT1_I29, args[0], args[0]);
+        break;
+    case INDEX_op_ld16u_i32:
+    case INDEX_op_ld16u_i64:
+        tcg_out_ld_rel(s, OPC_LD2_M1, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld16s_i32:
+    case INDEX_op_ld16s_i64:
+        tcg_out_ld_rel(s, OPC_LD2_M1, args[0], args[1], args[2]);
+        tcg_out_ext(s, OPC_SXT2_I29, args[0], args[0]);
+        break;
+    case INDEX_op_ld_i32:
+    case INDEX_op_ld32u_i64:
+        tcg_out_ld_rel(s, OPC_LD4_M1, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld32s_i64:
+        tcg_out_ld_rel(s, OPC_LD4_M1, args[0], args[1], args[2]);
+        tcg_out_ext(s, OPC_SXT4_I29, args[0], args[0]);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_ld_rel(s, OPC_LD8_M1, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st8_i32:
+    case INDEX_op_st8_i64:
+        tcg_out_st_rel(s, OPC_ST1_M4, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st16_i32:
+    case INDEX_op_st16_i64:
+        tcg_out_st_rel(s, OPC_ST2_M4, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i32:
+    case INDEX_op_st32_i64:
+        tcg_out_st_rel(s, OPC_ST4_M4, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i64:
+        tcg_out_st_rel(s, OPC_ST8_M4, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_add_i32:
+    case INDEX_op_add_i64:
+        tcg_out_add(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_sub_i32:
+    case INDEX_op_sub_i64:
+        tcg_out_sub(s, args[0], args[1], const_args[1], args[2], const_args[2]);
+        break;
+
+    case INDEX_op_and_i32:
+    case INDEX_op_and_i64:
+        /* TCG expects arg2 constant; A3 expects arg1 constant.  Swap.  */
+        tcg_out_alu(s, OPC_AND_A1, OPC_AND_A3, args[0],
+                    args[2], const_args[2], args[1], const_args[1]);
+        break;
+    case INDEX_op_andc_i32:
+    case INDEX_op_andc_i64:
+        tcg_out_alu(s, OPC_ANDCM_A1, OPC_ANDCM_A3, args[0],
+                    args[1], const_args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_eqv_i32:
+    case INDEX_op_eqv_i64:
+        tcg_out_eqv(s, args[0], args[1], const_args[1],
+                    args[2], const_args[2]);
+        break;
+    case INDEX_op_nand_i32:
+    case INDEX_op_nand_i64:
+        tcg_out_nand(s, args[0], args[1], const_args[1],
+                     args[2], const_args[2]);
+        break;
+    case INDEX_op_nor_i32:
+    case INDEX_op_nor_i64:
+        tcg_out_nor(s, args[0], args[1], const_args[1],
+                    args[2], const_args[2]);
+        break;
+    case INDEX_op_or_i32:
+    case INDEX_op_or_i64:
+        /* TCG expects arg2 constant; A3 expects arg1 constant.  Swap.  */
+        tcg_out_alu(s, OPC_OR_A1, OPC_OR_A3, args[0],
+                    args[2], const_args[2], args[1], const_args[1]);
+        break;
+    case INDEX_op_orc_i32:
+    case INDEX_op_orc_i64:
+        tcg_out_orc(s, args[0], args[1], const_args[1],
+                    args[2], const_args[2]);
+        break;
+    case INDEX_op_xor_i32:
+    case INDEX_op_xor_i64:
+        /* TCG expects arg2 constant; A3 expects arg1 constant.  Swap.  */
+        tcg_out_alu(s, OPC_XOR_A1, OPC_XOR_A3, args[0],
+                    args[2], const_args[2], args[1], const_args[1]);
+        break;
+
+    case INDEX_op_mul_i32:
+    case INDEX_op_mul_i64:
+        tcg_out_mul(s, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_sar_i32:
+        tcg_out_sar_i32(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_sar_i64:
+        tcg_out_sar_i64(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_shl_i32:
+        tcg_out_shl_i32(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_shl_i64:
+        tcg_out_shl_i64(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_shr_i32:
+        tcg_out_shr_i32(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_shr_i64:
+        tcg_out_shr_i64(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_rotl_i32:
+        tcg_out_rotl_i32(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_rotl_i64:
+        tcg_out_rotl_i64(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_rotr_i32:
+        tcg_out_rotr_i32(s, args[0], args[1], args[2], const_args[2]);
+        break;
+    case INDEX_op_rotr_i64:
+        tcg_out_rotr_i64(s, args[0], args[1], args[2], const_args[2]);
+        break;
+
+    case INDEX_op_ext8s_i32:
+    case INDEX_op_ext8s_i64:
+        tcg_out_ext(s, OPC_SXT1_I29, args[0], args[1]);
+        break;
+    case INDEX_op_ext8u_i32:
+    case INDEX_op_ext8u_i64:
+        tcg_out_ext(s, OPC_ZXT1_I29, args[0], args[1]);
+        break;
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
+        tcg_out_ext(s, OPC_SXT2_I29, args[0], args[1]);
+        break;
+    case INDEX_op_ext16u_i32:
+    case INDEX_op_ext16u_i64:
+        tcg_out_ext(s, OPC_ZXT2_I29, args[0], args[1]);
+        break;
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_ext32s_i64:
+        tcg_out_ext(s, OPC_SXT4_I29, args[0], args[1]);
+        break;
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_ext32u_i64:
+        tcg_out_ext(s, OPC_ZXT4_I29, args[0], args[1]);
+        break;
+
+    case INDEX_op_bswap16_i32:
+    case INDEX_op_bswap16_i64:
+        tcg_out_bswap16(s, args[0], args[1]);
+        break;
+    case INDEX_op_bswap32_i32:
+    case INDEX_op_bswap32_i64:
+        tcg_out_bswap32(s, args[0], args[1]);
+        break;
+    case INDEX_op_bswap64_i64:
+        tcg_out_bswap64(s, args[0], args[1]);
+        break;
+
+    case INDEX_op_deposit_i32:
+    case INDEX_op_deposit_i64:
+        tcg_out_deposit(s, args[0], args[1], args[2], const_args[2],
+                        args[3], args[4]);
+        break;
+
+    case INDEX_op_brcond_i32:
+        tcg_out_brcond(s, args[2], args[0], args[1], arg_label(args[3]), 1);
+        break;
+    case INDEX_op_brcond_i64:
+        tcg_out_brcond(s, args[2], args[0], args[1], arg_label(args[3]), 0);
+        break;
+    case INDEX_op_setcond_i32:
+        tcg_out_setcond(s, args[3], args[0], args[1], args[2], 1);
+        break;
+    case INDEX_op_setcond_i64:
+        tcg_out_setcond(s, args[3], args[0], args[1], args[2], 0);
+        break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
+                        args[3], const_args[3], args[4], const_args[4], 1);
+        break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
+                        args[3], const_args[3], args[4], const_args[4], 0);
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, args);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args);
+        break;
+    case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, args);
+        break;
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args);
+        break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+}
+
+static const TCGTargetOpDef ia64_op_defs[] = {
+    { INDEX_op_br, { } },
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "rZ", "r" } },
+    { INDEX_op_st16_i32, { "rZ", "r" } },
+    { INDEX_op_st_i32, { "rZ", "r" } },
+
+    { INDEX_op_add_i32, { "r", "rZ", "rI" } },
+    { INDEX_op_sub_i32, { "r", "rI", "rI" } },
+
+    { INDEX_op_and_i32, { "r", "rI", "rI" } },
+    { INDEX_op_andc_i32, { "r", "rI", "rI" } },
+    { INDEX_op_eqv_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_nand_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_or_i32, { "r", "rI", "rI" } },
+    { INDEX_op_orc_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_xor_i32, { "r", "rI", "rI" } },
+
+    { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
+
+    { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
+
+    { INDEX_op_ext8s_i32, { "r", "rZ"} },
+    { INDEX_op_ext8u_i32, { "r", "rZ"} },
+    { INDEX_op_ext16s_i32, { "r", "rZ"} },
+    { INDEX_op_ext16u_i32, { "r", "rZ"} },
+
+    { INDEX_op_bswap16_i32, { "r", "rZ" } },
+    { INDEX_op_bswap32_i32, { "r", "rZ" } },
+
+    { INDEX_op_brcond_i32, { "rZ", "rZ" } },
+    { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rI", "rI" } },
+
+    { INDEX_op_ld8u_i64, { "r", "r" } },
+    { INDEX_op_ld8s_i64, { "r", "r" } },
+    { INDEX_op_ld16u_i64, { "r", "r" } },
+    { INDEX_op_ld16s_i64, { "r", "r" } },
+    { INDEX_op_ld32u_i64, { "r", "r" } },
+    { INDEX_op_ld32s_i64, { "r", "r" } },
+    { INDEX_op_ld_i64, { "r", "r" } },
+    { INDEX_op_st8_i64, { "rZ", "r" } },
+    { INDEX_op_st16_i64, { "rZ", "r" } },
+    { INDEX_op_st32_i64, { "rZ", "r" } },
+    { INDEX_op_st_i64, { "rZ", "r" } },
+
+    { INDEX_op_add_i64, { "r", "rZ", "rI" } },
+    { INDEX_op_sub_i64, { "r", "rI", "rI" } },
+
+    { INDEX_op_and_i64, { "r", "rI", "rI" } },
+    { INDEX_op_andc_i64, { "r", "rI", "rI" } },
+    { INDEX_op_eqv_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_nand_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_nor_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_or_i64, { "r", "rI", "rI" } },
+    { INDEX_op_orc_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_xor_i64, { "r", "rI", "rI" } },
+
+    { INDEX_op_mul_i64, { "r", "rZ", "rZ" } },
+
+    { INDEX_op_sar_i64, { "r", "rZ", "ri" } },
+    { INDEX_op_shl_i64, { "r", "rZ", "ri" } },
+    { INDEX_op_shr_i64, { "r", "rZ", "ri" } },
+    { INDEX_op_rotl_i64, { "r", "rZ", "ri" } },
+    { INDEX_op_rotr_i64, { "r", "rZ", "ri" } },
+
+    { INDEX_op_ext8s_i64, { "r", "rZ"} },
+    { INDEX_op_ext8u_i64, { "r", "rZ"} },
+    { INDEX_op_ext16s_i64, { "r", "rZ"} },
+    { INDEX_op_ext16u_i64, { "r", "rZ"} },
+    { INDEX_op_ext32s_i64, { "r", "rZ"} },
+    { INDEX_op_ext32u_i64, { "r", "rZ"} },
+    { INDEX_op_ext_i32_i64, { "r", "rZ" } },
+    { INDEX_op_extu_i32_i64, { "r", "rZ" } },
+
+    { INDEX_op_bswap16_i64, { "r", "rZ" } },
+    { INDEX_op_bswap32_i64, { "r", "rZ" } },
+    { INDEX_op_bswap64_i64, { "r", "rZ" } },
+
+    { INDEX_op_brcond_i64, { "rZ", "rZ" } },
+    { INDEX_op_setcond_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_movcond_i64, { "r", "rZ", "rZ", "rI", "rI" } },
+
+    { INDEX_op_deposit_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_deposit_i64, { "r", "rZ", "ri" } },
+
+    { INDEX_op_qemu_ld_i32, { "r", "r" } },
+    { INDEX_op_qemu_ld_i64, { "r", "r" } },
+    { INDEX_op_qemu_st_i32, { "SZ", "r" } },
+    { INDEX_op_qemu_st_i64, { "SZ", "r" } },
+
+    { -1 },
+};
+
+/* Generate global QEMU prologue and epilogue code */
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    int frame_size;
+
+    /* reserve some stack space */
+    frame_size = TCG_STATIC_CALL_ARGS_SIZE +
+                 CPU_TEMP_BUF_NLONGS * sizeof(long);
+    frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
+                 ~(TCG_TARGET_STACK_ALIGN - 1);
+    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+
+    /* First emit adhoc function descriptor */
+    *s->code_ptr = (tcg_insn_unit){
+        (uint64_t)(s->code_ptr + 1), /* entry point */
+        0                            /* skip gp */
+    };
+    s->code_ptr++;
+
+    /* prologue */
+    tcg_out_bundle(s, miI,
+                   tcg_opc_m34(TCG_REG_P0, OPC_ALLOC_M34,
+                               TCG_REG_R34, 32, 24, 0),
+                   INSN_NOP_I,
+                   tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21,
+                               TCG_REG_B6, TCG_REG_R33, 0));
+
+    /* ??? If guest_base < 0x200000, we could load the register via
+       an ADDL in the M slot of the next bundle.  */
+    if (guest_base != 0) {
+        tcg_out_bundle(s, mlx,
+                       INSN_NOP_M,
+                       tcg_opc_l2(guest_base),
+                       tcg_opc_x2 (TCG_REG_P0, OPC_MOVL_X2,
+                                   TCG_GUEST_BASE_REG, guest_base));
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
+    }
+
+    tcg_out_bundle(s, miB,
+                   tcg_opc_a4 (TCG_REG_P0, OPC_ADDS_A4,
+                               TCG_REG_R12, -frame_size, TCG_REG_R12),
+                   tcg_opc_i22(TCG_REG_P0, OPC_MOV_I22,
+                               TCG_REG_R33, TCG_REG_B0),
+                   tcg_opc_b4 (TCG_REG_P0, OPC_BR_SPTK_MANY_B4, TCG_REG_B6));
+
+    /* epilogue */
+    tb_ret_addr = s->code_ptr;
+    tcg_out_bundle(s, miI,
+                   INSN_NOP_M,
+                   tcg_opc_i21(TCG_REG_P0, OPC_MOV_I21,
+                               TCG_REG_B0, TCG_REG_R33, 0),
+                   tcg_opc_a4 (TCG_REG_P0, OPC_ADDS_A4,
+                               TCG_REG_R12, frame_size, TCG_REG_R12));
+    tcg_out_bundle(s, miB,
+                   INSN_NOP_M,
+                   tcg_opc_i26(TCG_REG_P0, OPC_MOV_I_I26,
+                               TCG_REG_PFS, TCG_REG_R34),
+                   tcg_opc_b4 (TCG_REG_P0, OPC_BR_RET_SPTK_MANY_B4,
+                               TCG_REG_B0));
+}
+
+static void tcg_target_init(TCGContext *s)
+{
+    tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I32],
+                   0xffffffffffffffffull);
+    tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I64],
+                   0xffffffffffffffffull);
+
+    tcg_regset_clear(tcg_target_call_clobber_regs);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R15);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R16);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R17);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R18);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R19);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R20);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R21);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R22);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R23);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R24);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R25);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R26);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R27);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R28);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R29);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R30);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R31);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R56);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R57);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R58);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R59);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R60);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R61);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R62);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R63);
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);   /* zero register */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1);   /* global pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2);   /* internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R3);   /* internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R12);  /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13);  /* thread pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R33);  /* return address */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R34);  /* PFS */
+
+    /* The following 4 are not in use, are call-saved, but *not* saved
+       by the prologue.  Therefore we cannot use them without modifying
+       the prologue.  There doesn't seem to be any good reason to use
+       these as opposed to the windowed registers.  */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R4);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R5);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R6);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R7);
+
+    tcg_add_target_add_op_defs(ia64_op_defs);
+}
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
deleted file mode 100644
index 09fefe3..0000000
--- a/tcg/mips/tcg-target.c
+++ /dev/null
@@ -1,1820 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org>
- * Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net>
- * Based on i386/tcg-target.c - Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "tcg-be-ldst.h"
-
-#ifdef HOST_WORDS_BIGENDIAN
-# define MIPS_BE  1
-#else
-# define MIPS_BE  0
-#endif
-
-#define LO_OFF    (MIPS_BE * 4)
-#define HI_OFF    (4 - LO_OFF)
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "zero",
-    "at",
-    "v0",
-    "v1",
-    "a0",
-    "a1",
-    "a2",
-    "a3",
-    "t0",
-    "t1",
-    "t2",
-    "t3",
-    "t4",
-    "t5",
-    "t6",
-    "t7",
-    "s0",
-    "s1",
-    "s2",
-    "s3",
-    "s4",
-    "s5",
-    "s6",
-    "s7",
-    "t8",
-    "t9",
-    "k0",
-    "k1",
-    "gp",
-    "sp",
-    "s8",
-    "ra",
-};
-#endif
-
-#define TCG_TMP0  TCG_REG_AT
-#define TCG_TMP1  TCG_REG_T9
-
-/* check if we really need so many registers :P */
-static const TCGReg tcg_target_reg_alloc_order[] = {
-    /* Call saved registers.  */
-    TCG_REG_S0,
-    TCG_REG_S1,
-    TCG_REG_S2,
-    TCG_REG_S3,
-    TCG_REG_S4,
-    TCG_REG_S5,
-    TCG_REG_S6,
-    TCG_REG_S7,
-    TCG_REG_S8,
-
-    /* Call clobbered registers.  */
-    TCG_REG_T0,
-    TCG_REG_T1,
-    TCG_REG_T2,
-    TCG_REG_T3,
-    TCG_REG_T4,
-    TCG_REG_T5,
-    TCG_REG_T6,
-    TCG_REG_T7,
-    TCG_REG_T8,
-    TCG_REG_T9,
-    TCG_REG_V1,
-    TCG_REG_V0,
-
-    /* Argument registers, opposite order of allocation.  */
-    TCG_REG_A3,
-    TCG_REG_A2,
-    TCG_REG_A1,
-    TCG_REG_A0,
-};
-
-static const TCGReg tcg_target_call_iarg_regs[4] = {
-    TCG_REG_A0,
-    TCG_REG_A1,
-    TCG_REG_A2,
-    TCG_REG_A3
-};
-
-static const TCGReg tcg_target_call_oarg_regs[2] = {
-    TCG_REG_V0,
-    TCG_REG_V1
-};
-
-static tcg_insn_unit *tb_ret_addr;
-
-static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    /* Let the compiler perform the right-shift as part of the arithmetic.  */
-    ptrdiff_t disp = target - (pc + 1);
-    assert(disp == (int16_t)disp);
-    return disp & 0xffff;
-}
-
-static inline void reloc_pc16(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    *pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target));
-}
-
-static inline uint32_t reloc_26_val(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0);
-    return ((uintptr_t)target >> 2) & 0x3ffffff;
-}
-
-static inline void reloc_26(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    *pc = deposit32(*pc, 0, 26, reloc_26_val(pc, target));
-}
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    assert(type == R_MIPS_PC16);
-    assert(addend == 0);
-    reloc_pc16(code_ptr, (tcg_insn_unit *)value);
-}
-
-#define TCG_CT_CONST_ZERO 0x100
-#define TCG_CT_CONST_U16  0x200    /* Unsigned 16-bit: 0 - 0xffff.  */
-#define TCG_CT_CONST_S16  0x400    /* Signed 16-bit: -32768 - 32767 */
-#define TCG_CT_CONST_P2M1 0x800    /* Power of 2 minus 1.  */
-#define TCG_CT_CONST_N16  0x1000   /* "Negatable" 16-bit: -32767 - 32767 */
-
-static inline bool is_p2m1(tcg_target_long val)
-{
-    return val && ((val + 1) & val) == 0;
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch(ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set(ct->u.regs, 0xffffffff);
-        break;
-    case 'L': /* qemu_ld output arg constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set(ct->u.regs, 0xffffffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_V0);
-        break;
-    case 'l': /* qemu_ld input arg constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set(ct->u.regs, 0xffffffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
-#if defined(CONFIG_SOFTMMU)
-        if (TARGET_LONG_BITS == 64) {
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
-        }
-#endif
-        break;
-    case 'S': /* qemu_st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set(ct->u.regs, 0xffffffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
-#if defined(CONFIG_SOFTMMU)
-        if (TARGET_LONG_BITS == 32) {
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A1);
-        } else {
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A3);
-        }
-#endif
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_U16;
-        break;
-    case 'J':
-        ct->ct |= TCG_CT_CONST_S16;
-        break;
-    case 'K':
-        ct->ct |= TCG_CT_CONST_P2M1;
-        break;
-    case 'N':
-        ct->ct |= TCG_CT_CONST_N16;
-        break;
-    case 'Z':
-        /* We are cheating a bit here, using the fact that the register
-           ZERO is also the register number 0. Hence there is no need
-           to check for const_args in each instruction. */
-        ct->ct |= TCG_CT_CONST_ZERO;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct;
-    ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_N16) && val >= -32767 && val <= 32767) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_P2M1)
-               && use_mips32r2_instructions && is_p2m1(val)) {
-        return 1;
-    }
-    return 0;
-}
-
-/* instruction opcodes */
-typedef enum {
-    OPC_J        = 0x02 << 26,
-    OPC_JAL      = 0x03 << 26,
-    OPC_BEQ      = 0x04 << 26,
-    OPC_BNE      = 0x05 << 26,
-    OPC_BLEZ     = 0x06 << 26,
-    OPC_BGTZ     = 0x07 << 26,
-    OPC_ADDIU    = 0x09 << 26,
-    OPC_SLTI     = 0x0A << 26,
-    OPC_SLTIU    = 0x0B << 26,
-    OPC_ANDI     = 0x0C << 26,
-    OPC_ORI      = 0x0D << 26,
-    OPC_XORI     = 0x0E << 26,
-    OPC_LUI      = 0x0F << 26,
-    OPC_LB       = 0x20 << 26,
-    OPC_LH       = 0x21 << 26,
-    OPC_LW       = 0x23 << 26,
-    OPC_LBU      = 0x24 << 26,
-    OPC_LHU      = 0x25 << 26,
-    OPC_LWU      = 0x27 << 26,
-    OPC_SB       = 0x28 << 26,
-    OPC_SH       = 0x29 << 26,
-    OPC_SW       = 0x2B << 26,
-
-    OPC_SPECIAL  = 0x00 << 26,
-    OPC_SLL      = OPC_SPECIAL | 0x00,
-    OPC_SRL      = OPC_SPECIAL | 0x02,
-    OPC_ROTR     = OPC_SPECIAL | (0x01 << 21) | 0x02,
-    OPC_SRA      = OPC_SPECIAL | 0x03,
-    OPC_SLLV     = OPC_SPECIAL | 0x04,
-    OPC_SRLV     = OPC_SPECIAL | 0x06,
-    OPC_ROTRV    = OPC_SPECIAL | (0x01 <<  6) | 0x06,
-    OPC_SRAV     = OPC_SPECIAL | 0x07,
-    OPC_JR       = OPC_SPECIAL | 0x08,
-    OPC_JALR     = OPC_SPECIAL | 0x09,
-    OPC_MOVZ     = OPC_SPECIAL | 0x0A,
-    OPC_MOVN     = OPC_SPECIAL | 0x0B,
-    OPC_MFHI     = OPC_SPECIAL | 0x10,
-    OPC_MFLO     = OPC_SPECIAL | 0x12,
-    OPC_MULT     = OPC_SPECIAL | 0x18,
-    OPC_MULTU    = OPC_SPECIAL | 0x19,
-    OPC_DIV      = OPC_SPECIAL | 0x1A,
-    OPC_DIVU     = OPC_SPECIAL | 0x1B,
-    OPC_ADDU     = OPC_SPECIAL | 0x21,
-    OPC_SUBU     = OPC_SPECIAL | 0x23,
-    OPC_AND      = OPC_SPECIAL | 0x24,
-    OPC_OR       = OPC_SPECIAL | 0x25,
-    OPC_XOR      = OPC_SPECIAL | 0x26,
-    OPC_NOR      = OPC_SPECIAL | 0x27,
-    OPC_SLT      = OPC_SPECIAL | 0x2A,
-    OPC_SLTU     = OPC_SPECIAL | 0x2B,
-
-    OPC_REGIMM   = 0x01 << 26,
-    OPC_BLTZ     = OPC_REGIMM | (0x00 << 16),
-    OPC_BGEZ     = OPC_REGIMM | (0x01 << 16),
-
-    OPC_SPECIAL2 = 0x1c << 26,
-    OPC_MUL      = OPC_SPECIAL2 | 0x002,
-
-    OPC_SPECIAL3 = 0x1f << 26,
-    OPC_EXT      = OPC_SPECIAL3 | 0x000,
-    OPC_INS      = OPC_SPECIAL3 | 0x004,
-    OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
-    OPC_SEB      = OPC_SPECIAL3 | 0x420,
-    OPC_SEH      = OPC_SPECIAL3 | 0x620,
-} MIPSInsn;
-
-/*
- * Type reg
- */
-static inline void tcg_out_opc_reg(TCGContext *s, MIPSInsn opc,
-                                   TCGReg rd, TCGReg rs, TCGReg rt)
-{
-    int32_t inst;
-
-    inst = opc;
-    inst |= (rs & 0x1F) << 21;
-    inst |= (rt & 0x1F) << 16;
-    inst |= (rd & 0x1F) << 11;
-    tcg_out32(s, inst);
-}
-
-/*
- * Type immediate
- */
-static inline void tcg_out_opc_imm(TCGContext *s, MIPSInsn opc,
-                                   TCGReg rt, TCGReg rs, TCGArg imm)
-{
-    int32_t inst;
-
-    inst = opc;
-    inst |= (rs & 0x1F) << 21;
-    inst |= (rt & 0x1F) << 16;
-    inst |= (imm & 0xffff);
-    tcg_out32(s, inst);
-}
-
-/*
- * Type bitfield
- */
-static inline void tcg_out_opc_bf(TCGContext *s, MIPSInsn opc, TCGReg rt,
-                                  TCGReg rs, int msb, int lsb)
-{
-    int32_t inst;
-
-    inst = opc;
-    inst |= (rs & 0x1F) << 21;
-    inst |= (rt & 0x1F) << 16;
-    inst |= (msb & 0x1F) << 11;
-    inst |= (lsb & 0x1F) << 6;
-    tcg_out32(s, inst);
-}
-
-/*
- * Type branch
- */
-static inline void tcg_out_opc_br(TCGContext *s, MIPSInsn opc,
-                                  TCGReg rt, TCGReg rs)
-{
-    /* We pay attention here to not modify the branch target by reading
-       the existing value and using it again. This ensure that caches and
-       memory are kept coherent during retranslation. */
-    uint16_t offset = (uint16_t)*s->code_ptr;
-
-    tcg_out_opc_imm(s, opc, rt, rs, offset);
-}
-
-/*
- * Type sa
- */
-static inline void tcg_out_opc_sa(TCGContext *s, MIPSInsn opc,
-                                  TCGReg rd, TCGReg rt, TCGArg sa)
-{
-    int32_t inst;
-
-    inst = opc;
-    inst |= (rt & 0x1F) << 16;
-    inst |= (rd & 0x1F) << 11;
-    inst |= (sa & 0x1F) <<  6;
-    tcg_out32(s, inst);
-
-}
-
-/*
- * Type jump.
- * Returns true if the branch was in range and the insn was emitted.
- */
-static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target)
-{
-    uintptr_t dest = (uintptr_t)target;
-    uintptr_t from = (uintptr_t)s->code_ptr + 4;
-    int32_t inst;
-
-    /* The pc-region branch happens within the 256MB region of
-       the delay slot (thus the +4).  */
-    if ((from ^ dest) & -(1 << 28)) {
-        return false;
-    }
-    assert((dest & 3) == 0);
-
-    inst = opc;
-    inst |= (dest >> 2) & 0x3ffffff;
-    tcg_out32(s, inst);
-    return true;
-}
-
-static inline void tcg_out_nop(TCGContext *s)
-{
-    tcg_out32(s, 0);
-}
-
-static inline void tcg_out_mov(TCGContext *s, TCGType type,
-                               TCGReg ret, TCGReg arg)
-{
-    /* Simple reg-reg move, optimising out the 'do nothing' case */
-    if (ret != arg) {
-        tcg_out_opc_reg(s, OPC_ADDU, ret, arg, TCG_REG_ZERO);
-    }
-}
-
-static inline void tcg_out_movi(TCGContext *s, TCGType type,
-                                TCGReg reg, tcg_target_long arg)
-{
-    if (arg == (int16_t)arg) {
-        tcg_out_opc_imm(s, OPC_ADDIU, reg, TCG_REG_ZERO, arg);
-    } else if (arg == (uint16_t)arg) {
-        tcg_out_opc_imm(s, OPC_ORI, reg, TCG_REG_ZERO, arg);
-    } else {
-        tcg_out_opc_imm(s, OPC_LUI, reg, TCG_REG_ZERO, arg >> 16);
-        if (arg & 0xffff) {
-            tcg_out_opc_imm(s, OPC_ORI, reg, reg, arg & 0xffff);
-        }
-    }
-}
-
-static inline void tcg_out_bswap16(TCGContext *s, TCGReg ret, TCGReg arg)
-{
-    if (use_mips32r2_instructions) {
-        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
-    } else {
-        /* ret and arg can't be register at */
-        if (ret == TCG_TMP0 || arg == TCG_TMP0) {
-            tcg_abort();
-        }
-
-        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
-        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
-        tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
-    }
-}
-
-static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
-{
-    if (use_mips32r2_instructions) {
-        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
-        tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
-    } else {
-        /* ret and arg can't be register at */
-        if (ret == TCG_TMP0 || arg == TCG_TMP0) {
-            tcg_abort();
-        }
-
-        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
-        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
-        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
-    }
-}
-
-static inline void tcg_out_bswap32(TCGContext *s, TCGReg ret, TCGReg arg)
-{
-    if (use_mips32r2_instructions) {
-        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
-        tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
-    } else {
-        /* ret and arg must be different and can't be register at */
-        if (ret == arg || ret == TCG_TMP0 || arg == TCG_TMP0) {
-            tcg_abort();
-        }
-
-        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
-
-        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 24);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
-
-        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, arg, 0xff00);
-        tcg_out_opc_sa(s, OPC_SLL, TCG_TMP0, TCG_TMP0, 8);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
-
-        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
-        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, TCG_TMP0, 0xff00);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
-    }
-}
-
-static inline void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
-{
-    if (use_mips32r2_instructions) {
-        tcg_out_opc_reg(s, OPC_SEB, ret, 0, arg);
-    } else {
-        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
-        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 24);
-    }
-}
-
-static inline void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
-{
-    if (use_mips32r2_instructions) {
-        tcg_out_opc_reg(s, OPC_SEH, ret, 0, arg);
-    } else {
-        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 16);
-        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
-    }
-}
-
-static void tcg_out_ldst(TCGContext *s, MIPSInsn opc, TCGReg data,
-                         TCGReg addr, intptr_t ofs)
-{
-    int16_t lo = ofs;
-    if (ofs != lo) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - lo);
-        if (addr != TCG_REG_ZERO) {
-            tcg_out_opc_reg(s, OPC_ADDU, TCG_TMP0, TCG_TMP0, addr);
-        }
-        addr = TCG_TMP0;
-    }
-    tcg_out_opc_imm(s, opc, data, addr, lo);
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, OPC_LW, arg, arg1, arg2);
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
-}
-
-static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
-{
-    if (val == (int16_t)val) {
-        tcg_out_opc_imm(s, OPC_ADDIU, reg, reg, val);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, val);
-        tcg_out_opc_reg(s, OPC_ADDU, reg, reg, TCG_TMP0);
-    }
-}
-
-/* Bit 0 set if inversion required; bit 1 set if swapping required.  */
-#define MIPS_CMP_INV  1
-#define MIPS_CMP_SWAP 2
-
-static const uint8_t mips_cmp_map[16] = {
-    [TCG_COND_LT]  = 0,
-    [TCG_COND_LTU] = 0,
-    [TCG_COND_GE]  = MIPS_CMP_INV,
-    [TCG_COND_GEU] = MIPS_CMP_INV,
-    [TCG_COND_LE]  = MIPS_CMP_INV | MIPS_CMP_SWAP,
-    [TCG_COND_LEU] = MIPS_CMP_INV | MIPS_CMP_SWAP,
-    [TCG_COND_GT]  = MIPS_CMP_SWAP,
-    [TCG_COND_GTU] = MIPS_CMP_SWAP,
-};
-
-static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
-                            TCGReg arg1, TCGReg arg2)
-{
-    MIPSInsn s_opc = OPC_SLTU;
-    int cmp_map;
-
-    switch (cond) {
-    case TCG_COND_EQ:
-        if (arg2 != 0) {
-            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
-            arg1 = ret;
-        }
-        tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, 1);
-        break;
-
-    case TCG_COND_NE:
-        if (arg2 != 0) {
-            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
-            arg1 = ret;
-        }
-        tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, arg1);
-        break;
-
-    case TCG_COND_LT:
-    case TCG_COND_GE:
-    case TCG_COND_LE:
-    case TCG_COND_GT:
-        s_opc = OPC_SLT;
-        /* FALLTHRU */
-
-    case TCG_COND_LTU:
-    case TCG_COND_GEU:
-    case TCG_COND_LEU:
-    case TCG_COND_GTU:
-        cmp_map = mips_cmp_map[cond];
-        if (cmp_map & MIPS_CMP_SWAP) {
-            TCGReg t = arg1;
-            arg1 = arg2;
-            arg2 = t;
-        }
-        tcg_out_opc_reg(s, s_opc, ret, arg1, arg2);
-        if (cmp_map & MIPS_CMP_INV) {
-            tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
-        }
-        break;
-
-     default:
-         tcg_abort();
-         break;
-     }
-}
-
-static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
-                           TCGReg arg2, int label_index)
-{
-    static const MIPSInsn b_zero[16] = {
-        [TCG_COND_LT] = OPC_BLTZ,
-        [TCG_COND_GT] = OPC_BGTZ,
-        [TCG_COND_LE] = OPC_BLEZ,
-        [TCG_COND_GE] = OPC_BGEZ,
-    };
-
-    TCGLabel *l;
-    MIPSInsn s_opc = OPC_SLTU;
-    MIPSInsn b_opc;
-    int cmp_map;
-
-    switch (cond) {
-    case TCG_COND_EQ:
-        b_opc = OPC_BEQ;
-        break;
-    case TCG_COND_NE:
-        b_opc = OPC_BNE;
-        break;
-
-    case TCG_COND_LT:
-    case TCG_COND_GT:
-    case TCG_COND_LE:
-    case TCG_COND_GE:
-        if (arg2 == 0) {
-            b_opc = b_zero[cond];
-            arg2 = arg1;
-            arg1 = 0;
-            break;
-        }
-        s_opc = OPC_SLT;
-        /* FALLTHRU */
-
-    case TCG_COND_LTU:
-    case TCG_COND_GTU:
-    case TCG_COND_LEU:
-    case TCG_COND_GEU:
-        cmp_map = mips_cmp_map[cond];
-        if (cmp_map & MIPS_CMP_SWAP) {
-            TCGReg t = arg1;
-            arg1 = arg2;
-            arg2 = t;
-        }
-        tcg_out_opc_reg(s, s_opc, TCG_TMP0, arg1, arg2);
-        b_opc = (cmp_map & MIPS_CMP_INV ? OPC_BEQ : OPC_BNE);
-        arg1 = TCG_TMP0;
-        arg2 = TCG_REG_ZERO;
-        break;
-
-    default:
-        tcg_abort();
-        break;
-    }
-
-    tcg_out_opc_br(s, b_opc, arg1, arg2);
-    l = &s->labels[label_index];
-    if (l->has_value) {
-        reloc_pc16(s->code_ptr - 1, l->u.value_ptr);
-    } else {
-        tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, label_index, 0);
-    }
-    tcg_out_nop(s);
-}
-
-static TCGReg tcg_out_reduce_eq2(TCGContext *s, TCGReg tmp0, TCGReg tmp1,
-                                 TCGReg al, TCGReg ah,
-                                 TCGReg bl, TCGReg bh)
-{
-    /* Merge highpart comparison into AH.  */
-    if (bh != 0) {
-        if (ah != 0) {
-            tcg_out_opc_reg(s, OPC_XOR, tmp0, ah, bh);
-            ah = tmp0;
-        } else {
-            ah = bh;
-        }
-    }
-    /* Merge lowpart comparison into AL.  */
-    if (bl != 0) {
-        if (al != 0) {
-            tcg_out_opc_reg(s, OPC_XOR, tmp1, al, bl);
-            al = tmp1;
-        } else {
-            al = bl;
-        }
-    }
-    /* Merge high and low part comparisons into AL.  */
-    if (ah != 0) {
-        if (al != 0) {
-            tcg_out_opc_reg(s, OPC_OR, tmp0, ah, al);
-            al = tmp0;
-        } else {
-            al = ah;
-        }
-    }
-    return al;
-}
-
-static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
-                             TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh)
-{
-    TCGReg tmp0 = TCG_TMP0;
-    TCGReg tmp1 = ret;
-
-    assert(ret != TCG_TMP0);
-    if (ret == ah || ret == bh) {
-        assert(ret != TCG_TMP1);
-        tmp1 = TCG_TMP1;
-    }
-
-    switch (cond) {
-    case TCG_COND_EQ:
-    case TCG_COND_NE:
-        tmp1 = tcg_out_reduce_eq2(s, tmp0, tmp1, al, ah, bl, bh);
-        tcg_out_setcond(s, cond, ret, tmp1, TCG_REG_ZERO);
-        break;
-
-    default:
-        tcg_out_setcond(s, TCG_COND_EQ, tmp0, ah, bh);
-        tcg_out_setcond(s, tcg_unsigned_cond(cond), tmp1, al, bl);
-        tcg_out_opc_reg(s, OPC_AND, tmp1, tmp1, tmp0);
-        tcg_out_setcond(s, tcg_high_cond(cond), tmp0, ah, bh);
-        tcg_out_opc_reg(s, OPC_OR, ret, tmp1, tmp0);
-        break;
-    }
-}
-
-static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah,
-                            TCGReg bl, TCGReg bh, int label_index)
-{
-    TCGCond b_cond = TCG_COND_NE;
-    TCGReg tmp = TCG_TMP1;
-
-    /* With branches, we emit between 4 and 9 insns with 2 or 3 branches.
-       With setcond, we emit between 3 and 10 insns and only 1 branch,
-       which ought to get better branch prediction.  */
-     switch (cond) {
-     case TCG_COND_EQ:
-     case TCG_COND_NE:
-        b_cond = cond;
-        tmp = tcg_out_reduce_eq2(s, TCG_TMP0, TCG_TMP1, al, ah, bl, bh);
-        break;
-
-    default:
-        /* Minimize code size by preferring a compare not requiring INV.  */
-        if (mips_cmp_map[cond] & MIPS_CMP_INV) {
-            cond = tcg_invert_cond(cond);
-            b_cond = TCG_COND_EQ;
-        }
-        tcg_out_setcond2(s, cond, tmp, al, ah, bl, bh);
-        break;
-    }
-
-    tcg_out_brcond(s, b_cond, tmp, TCG_REG_ZERO, label_index);
-}
-
-static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
-                            TCGReg c1, TCGReg c2, TCGReg v)
-{
-    MIPSInsn m_opc = OPC_MOVN;
-
-    switch (cond) {
-    case TCG_COND_EQ:
-        m_opc = OPC_MOVZ;
-        /* FALLTHRU */
-    case TCG_COND_NE:
-        if (c2 != 0) {
-            tcg_out_opc_reg(s, OPC_XOR, TCG_TMP0, c1, c2);
-            c1 = TCG_TMP0;
-        }
-        break;
-
-    default:
-        /* Minimize code size by preferring a compare not requiring INV.  */
-        if (mips_cmp_map[cond] & MIPS_CMP_INV) {
-            cond = tcg_invert_cond(cond);
-            m_opc = OPC_MOVZ;
-        }
-        tcg_out_setcond(s, cond, TCG_TMP0, c1, c2);
-        c1 = TCG_TMP0;
-        break;
-    }
-
-    tcg_out_opc_reg(s, m_opc, ret, v, c1);
-}
-
-static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
-{
-    /* Note that the ABI requires the called function's address to be
-       loaded into T9, even if a direct branch is in range.  */
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T9, (uintptr_t)arg);
-
-    /* But do try a direct branch, allowing the cpu better insn prefetch.  */
-    if (tail) {
-        if (!tcg_out_opc_jmp(s, OPC_J, arg)) {
-            tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_T9, 0);
-        }
-    } else {
-        if (!tcg_out_opc_jmp(s, OPC_JAL, arg)) {
-            tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, TCG_REG_T9, 0);
-        }
-    }
-}
-
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
-{
-    tcg_out_call_int(s, arg, false);
-    tcg_out_nop(s);
-}
-
-#if defined(CONFIG_SOFTMMU)
-static void * const qemu_ld_helpers[16] = {
-    [MO_UB]   = helper_ret_ldub_mmu,
-    [MO_SB]   = helper_ret_ldsb_mmu,
-    [MO_LEUW] = helper_le_lduw_mmu,
-    [MO_LESW] = helper_le_ldsw_mmu,
-    [MO_LEUL] = helper_le_ldul_mmu,
-    [MO_LEQ]  = helper_le_ldq_mmu,
-    [MO_BEUW] = helper_be_lduw_mmu,
-    [MO_BESW] = helper_be_ldsw_mmu,
-    [MO_BEUL] = helper_be_ldul_mmu,
-    [MO_BEQ]  = helper_be_ldq_mmu,
-};
-
-static void * const qemu_st_helpers[16] = {
-    [MO_UB]   = helper_ret_stb_mmu,
-    [MO_LEUW] = helper_le_stw_mmu,
-    [MO_LEUL] = helper_le_stl_mmu,
-    [MO_LEQ]  = helper_le_stq_mmu,
-    [MO_BEUW] = helper_be_stw_mmu,
-    [MO_BEUL] = helper_be_stl_mmu,
-    [MO_BEQ]  = helper_be_stq_mmu,
-};
-
-/* Helper routines for marshalling helper function arguments into
- * the correct registers and stack.
- * I is where we want to put this argument, and is updated and returned
- * for the next call. ARG is the argument itself.
- *
- * We provide routines for arguments which are: immediate, 32 bit
- * value in register, 16 and 8 bit values in register (which must be zero
- * extended before use) and 64 bit value in a lo:hi register pair.
- */
-
-static int tcg_out_call_iarg_reg(TCGContext *s, int i, TCGReg arg)
-{
-    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
-        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[i], arg);
-    } else {
-        tcg_out_st(s, TCG_TYPE_REG, arg, TCG_REG_SP, 4 * i);
-    }
-    return i + 1;
-}
-
-static int tcg_out_call_iarg_reg8(TCGContext *s, int i, TCGReg arg)
-{
-    TCGReg tmp = TCG_TMP0;
-    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
-        tmp = tcg_target_call_iarg_regs[i];
-    }
-    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xff);
-    return tcg_out_call_iarg_reg(s, i, tmp);
-}
-
-static int tcg_out_call_iarg_reg16(TCGContext *s, int i, TCGReg arg)
-{
-    TCGReg tmp = TCG_TMP0;
-    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
-        tmp = tcg_target_call_iarg_regs[i];
-    }
-    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xffff);
-    return tcg_out_call_iarg_reg(s, i, tmp);
-}
-
-static int tcg_out_call_iarg_imm(TCGContext *s, int i, TCGArg arg)
-{
-    TCGReg tmp = TCG_TMP0;
-    if (arg == 0) {
-        tmp = TCG_REG_ZERO;
-    } else {
-        if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
-            tmp = tcg_target_call_iarg_regs[i];
-        }
-        tcg_out_movi(s, TCG_TYPE_REG, tmp, arg);
-    }
-    return tcg_out_call_iarg_reg(s, i, tmp);
-}
-
-static int tcg_out_call_iarg_reg2(TCGContext *s, int i, TCGReg al, TCGReg ah)
-{
-    i = (i + 1) & ~1;
-    i = tcg_out_call_iarg_reg(s, i, (MIPS_BE ? ah : al));
-    i = tcg_out_call_iarg_reg(s, i, (MIPS_BE ? al : ah));
-    return i;
-}
-
-/* Perform the tlb comparison operation.  The complete host address is
-   placed in BASE.  Clobbers AT, T0, A0.  */
-static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl,
-                             TCGReg addrh, int mem_index, TCGMemOp s_bits,
-                             tcg_insn_unit *label_ptr[2], bool is_load)
-{
-    int cmp_off
-        = (is_load
-           ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
-           : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
-    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
-
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addrl,
-                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0,
-                    (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, TCG_AREG0);
-
-    /* Compensate for very large offsets.  */
-    if (add_off >= 0x8000) {
-        /* Most target env are smaller than 32k; none are larger than 64k.
-           Simplify the logic here merely to offset by 0x7ff0, giving us a
-           range just shy of 64k.  Check this assumption.  */
-        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
-                                   tlb_table[NB_MMU_MODES - 1][1])
-                          > 0x7ff0 + 0x7fff);
-        tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_A0, TCG_REG_A0, 0x7ff0);
-        cmp_off -= 0x7ff0;
-        add_off -= 0x7ff0;
-    }
-
-    /* Load the tlb comparator.  */
-    tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off + LO_OFF);
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_opc_imm(s, OPC_LW, base, TCG_REG_A0, cmp_off + HI_OFF);
-    }
-
-    /* Mask the page bits, keeping the alignment bits to compare against.
-       In between, load the tlb addend for the fast path.  */
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1,
-                 TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
-    tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrl);
-
-    label_ptr[0] = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BNE, TCG_TMP1, TCG_TMP0);
-
-    if (TARGET_LONG_BITS == 64) {
-        /* delay slot */
-        tcg_out_nop(s);
-
-        label_ptr[1] = s->code_ptr;
-        tcg_out_opc_br(s, OPC_BNE, addrh, base);
-    }
-
-    /* delay slot */
-    tcg_out_opc_reg(s, OPC_ADDU, base, TCG_REG_A0, addrl);
-}
-
-static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
-                                TCGReg datalo, TCGReg datahi,
-                                TCGReg addrlo, TCGReg addrhi,
-                                void *raddr, tcg_insn_unit *label_ptr[2])
-{
-    TCGLabelQemuLdst *label = new_ldst_label(s);
-
-    label->is_ld = is_ld;
-    label->oi = oi;
-    label->datalo_reg = datalo;
-    label->datahi_reg = datahi;
-    label->addrlo_reg = addrlo;
-    label->addrhi_reg = addrhi;
-    label->raddr = raddr;
-    label->label_ptr[0] = label_ptr[0];
-    if (TARGET_LONG_BITS == 64) {
-        label->label_ptr[1] = label_ptr[1];
-    }
-}
-
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGReg v0;
-    int i;
-
-    /* resolve label address */
-    reloc_pc16(l->label_ptr[0], s->code_ptr);
-    if (TARGET_LONG_BITS == 64) {
-        reloc_pc16(l->label_ptr[1], s->code_ptr);
-    }
-
-    i = 1;
-    if (TARGET_LONG_BITS == 64) {
-        i = tcg_out_call_iarg_reg2(s, i, l->addrlo_reg, l->addrhi_reg);
-    } else {
-        i = tcg_out_call_iarg_reg(s, i, l->addrlo_reg);
-    }
-    i = tcg_out_call_iarg_imm(s, i, oi);
-    i = tcg_out_call_iarg_imm(s, i, (intptr_t)l->raddr);
-    tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)], false);
-    /* delay slot */
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-
-    v0 = l->datalo_reg;
-    if ((opc & MO_SIZE) == MO_64) {
-        /* We eliminated V0 from the possible output registers, so it
-           cannot be clobbered here.  So we must move V1 first.  */
-        if (MIPS_BE) {
-            tcg_out_mov(s, TCG_TYPE_I32, v0, TCG_REG_V1);
-            v0 = l->datahi_reg;
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_V1);
-        }
-    }
-
-    reloc_pc16(s->code_ptr, l->raddr);
-    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
-    /* delay slot */
-    tcg_out_mov(s, TCG_TYPE_REG, v0, TCG_REG_V0);
-}
-
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
-{
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGMemOp s_bits = opc & MO_SIZE;
-    int i;
-
-    /* resolve label address */
-    reloc_pc16(l->label_ptr[0], s->code_ptr);
-    if (TARGET_LONG_BITS == 64) {
-        reloc_pc16(l->label_ptr[1], s->code_ptr);
-    }
-
-    i = 1;
-    if (TARGET_LONG_BITS == 64) {
-        i = tcg_out_call_iarg_reg2(s, i, l->addrlo_reg, l->addrhi_reg);
-    } else {
-        i = tcg_out_call_iarg_reg(s, i, l->addrlo_reg);
-    }
-    switch (s_bits) {
-    case MO_8:
-        i = tcg_out_call_iarg_reg8(s, i, l->datalo_reg);
-        break;
-    case MO_16:
-        i = tcg_out_call_iarg_reg16(s, i, l->datalo_reg);
-        break;
-    case MO_32:
-        i = tcg_out_call_iarg_reg(s, i, l->datalo_reg);
-        break;
-    case MO_64:
-        i = tcg_out_call_iarg_reg2(s, i, l->datalo_reg, l->datahi_reg);
-        break;
-    default:
-        tcg_abort();
-    }
-    i = tcg_out_call_iarg_imm(s, i, oi);
-
-    /* Tail call to the store helper.  Thus force the return address
-       computation to take place in the return address register.  */
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (intptr_t)l->raddr);
-    i = tcg_out_call_iarg_reg(s, i, TCG_REG_RA);
-    tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)], true);
-    /* delay slot */
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
-}
-#endif
-
-static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
-                                   TCGReg base, TCGMemOp opc)
-{
-    switch (opc) {
-    case MO_UB:
-        tcg_out_opc_imm(s, OPC_LBU, datalo, base, 0);
-        break;
-    case MO_SB:
-        tcg_out_opc_imm(s, OPC_LB, datalo, base, 0);
-        break;
-    case MO_UW | MO_BSWAP:
-        tcg_out_opc_imm(s, OPC_LHU, TCG_TMP1, base, 0);
-        tcg_out_bswap16(s, datalo, TCG_TMP1);
-        break;
-    case MO_UW:
-        tcg_out_opc_imm(s, OPC_LHU, datalo, base, 0);
-        break;
-    case MO_SW | MO_BSWAP:
-        tcg_out_opc_imm(s, OPC_LHU, TCG_TMP1, base, 0);
-        tcg_out_bswap16s(s, datalo, TCG_TMP1);
-        break;
-    case MO_SW:
-        tcg_out_opc_imm(s, OPC_LH, datalo, base, 0);
-        break;
-    case MO_UL | MO_BSWAP:
-        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, 0);
-        tcg_out_bswap32(s, datalo, TCG_TMP1);
-        break;
-    case MO_UL:
-        tcg_out_opc_imm(s, OPC_LW, datalo, base, 0);
-        break;
-    case MO_Q | MO_BSWAP:
-        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, HI_OFF);
-        tcg_out_bswap32(s, datalo, TCG_TMP1);
-        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, LO_OFF);
-        tcg_out_bswap32(s, datahi, TCG_TMP1);
-        break;
-    case MO_Q:
-        tcg_out_opc_imm(s, OPC_LW, datalo, base, LO_OFF);
-        tcg_out_opc_imm(s, OPC_LW, datahi, base, HI_OFF);
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
-{
-    TCGReg addr_regl, addr_regh __attribute__((unused));
-    TCGReg data_regl, data_regh;
-    TCGMemOpIdx oi;
-    TCGMemOp opc;
-#if defined(CONFIG_SOFTMMU)
-    tcg_insn_unit *label_ptr[2];
-    int mem_index;
-    TCGMemOp s_bits;
-#endif
-    /* Note that we've eliminated V0 from the output registers,
-       so we won't overwrite the base register during loading.  */
-    TCGReg base = TCG_REG_V0;
-
-    data_regl = *args++;
-    data_regh = (is_64 ? *args++ : 0);
-    addr_regl = *args++;
-    addr_regh = (TARGET_LONG_BITS == 64 ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-
-#if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & MO_SIZE;
-
-    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 1);
-    tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
-    add_qemu_ldst_label(s, 1, oi, data_regl, data_regh, addr_regl, addr_regh,
-                        s->code_ptr, label_ptr);
-#else
-    if (GUEST_BASE == 0 && data_regl != addr_regl) {
-        base = addr_regl;
-    } else if (GUEST_BASE == (int16_t)GUEST_BASE) {
-        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
-        tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
-    }
-    tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
-#endif
-}
-
-static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
-                                   TCGReg base, TCGMemOp opc)
-{
-    switch (opc) {
-    case MO_8:
-        tcg_out_opc_imm(s, OPC_SB, datalo, base, 0);
-        break;
-
-    case MO_16 | MO_BSWAP:
-        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP1, datalo, 0xffff);
-        tcg_out_bswap16(s, TCG_TMP1, TCG_TMP1);
-        datalo = TCG_TMP1;
-        /* FALLTHRU */
-    case MO_16:
-        tcg_out_opc_imm(s, OPC_SH, datalo, base, 0);
-        break;
-
-    case MO_32 | MO_BSWAP:
-        tcg_out_bswap32(s, TCG_TMP1, datalo);
-        datalo = TCG_TMP1;
-        /* FALLTHRU */
-    case MO_32:
-        tcg_out_opc_imm(s, OPC_SW, datalo, base, 0);
-        break;
-
-    case MO_64 | MO_BSWAP:
-        tcg_out_bswap32(s, TCG_TMP1, datalo);
-        tcg_out_opc_imm(s, OPC_SW, TCG_TMP1, base, HI_OFF);
-        tcg_out_bswap32(s, TCG_TMP1, datahi);
-        tcg_out_opc_imm(s, OPC_SW, TCG_TMP1, base, LO_OFF);
-        break;
-    case MO_64:
-        tcg_out_opc_imm(s, OPC_SW, datalo, base, LO_OFF);
-        tcg_out_opc_imm(s, OPC_SW, datahi, base, HI_OFF);
-        break;
-
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
-                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
-                            bool cbh, bool is_sub)
-{
-    TCGReg th = TCG_TMP1;
-
-    /* If we have a negative constant such that negating it would
-       make the high part zero, we can (usually) eliminate one insn.  */
-    if (cbl && cbh && bh == -1 && bl != 0) {
-        bl = -bl;
-        bh = 0;
-        is_sub = !is_sub;
-    }
-
-    /* By operating on the high part first, we get to use the final
-       carry operation to move back from the temporary.  */
-    if (!cbh) {
-        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
-    } else if (bh != 0 || ah == rl) {
-        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
-    } else {
-        th = ah;
-    }
-
-    /* Note that tcg optimization should eliminate the bl == 0 case.  */
-    if (is_sub) {
-        if (cbl) {
-            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
-            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
-            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
-        }
-        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
-    } else {
-        if (cbl) {
-            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
-            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
-        } else {
-            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
-            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
-        }
-        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
-    }
-}
-
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
-{
-    TCGReg addr_regl, addr_regh __attribute__((unused));
-    TCGReg data_regl, data_regh, base;
-    TCGMemOpIdx oi;
-    TCGMemOp opc;
-#if defined(CONFIG_SOFTMMU)
-    tcg_insn_unit *label_ptr[2];
-    int mem_index;
-    TCGMemOp s_bits;
-#endif
-
-    data_regl = *args++;
-    data_regh = (is_64 ? *args++ : 0);
-    addr_regl = *args++;
-    addr_regh = (TARGET_LONG_BITS == 64 ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-
-#if defined(CONFIG_SOFTMMU)
-    mem_index = get_mmuidx(oi);
-    s_bits = opc & 3;
-
-    /* Note that we eliminated the helper's address argument,
-       so we can reuse that for the base.  */
-    base = (TARGET_LONG_BITS == 32 ? TCG_REG_A1 : TCG_REG_A2);
-    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
-                     s_bits, label_ptr, 0);
-    tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
-    add_qemu_ldst_label(s, 0, oi, data_regl, data_regh, addr_regl, addr_regh,
-                        s->code_ptr, label_ptr);
-#else
-    if (GUEST_BASE == 0) {
-        base = addr_regl;
-    } else {
-        base = TCG_REG_A0;
-        if (GUEST_BASE == (int16_t)GUEST_BASE) {
-            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
-            tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
-        }
-    }
-    tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
-#endif
-}
-
-static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                              const TCGArg *args, const int *const_args)
-{
-    MIPSInsn i1, i2;
-    TCGArg a0, a1, a2;
-    int c2;
-
-    a0 = args[0];
-    a1 = args[1];
-    a2 = args[2];
-    c2 = const_args[2];
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        {
-            TCGReg b0 = TCG_REG_ZERO;
-
-            if (a0 & ~0xffff) {
-                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
-                b0 = TCG_REG_V0;
-            }
-            if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
-                tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0,
-                             (uintptr_t)tb_ret_addr);
-                tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-            }
-            tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
-        }
-        break;
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* direct jump method */
-            s->tb_jmp_offset[a0] = tcg_current_code_size(s);
-            /* Avoid clobbering the address during retranslation.  */
-            tcg_out32(s, OPC_J | (*(uint32_t *)s->code_ptr & 0x3ffffff));
-        } else {
-            /* indirect jump method */
-            tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
-                       (uintptr_t)(s->tb_next + a0));
-            tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
-        }
-        tcg_out_nop(s);
-        s->tb_next_offset[a0] = tcg_current_code_size(s);
-        break;
-    case INDEX_op_br:
-        tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, a0);
-        break;
-
-    case INDEX_op_ld8u_i32:
-        i1 = OPC_LBU;
-        goto do_ldst;
-    case INDEX_op_ld8s_i32:
-        i1 = OPC_LB;
-        goto do_ldst;
-    case INDEX_op_ld16u_i32:
-        i1 = OPC_LHU;
-        goto do_ldst;
-    case INDEX_op_ld16s_i32:
-        i1 = OPC_LH;
-        goto do_ldst;
-    case INDEX_op_ld_i32:
-        i1 = OPC_LW;
-        goto do_ldst;
-    case INDEX_op_st8_i32:
-        i1 = OPC_SB;
-        goto do_ldst;
-    case INDEX_op_st16_i32:
-        i1 = OPC_SH;
-        goto do_ldst;
-    case INDEX_op_st_i32:
-        i1 = OPC_SW;
-    do_ldst:
-        tcg_out_ldst(s, i1, a0, a1, a2);
-        break;
-
-    case INDEX_op_add_i32:
-        i1 = OPC_ADDU, i2 = OPC_ADDIU;
-        goto do_binary;
-    case INDEX_op_or_i32:
-        i1 = OPC_OR, i2 = OPC_ORI;
-        goto do_binary;
-    case INDEX_op_xor_i32:
-        i1 = OPC_XOR, i2 = OPC_XORI;
-    do_binary:
-        if (c2) {
-            tcg_out_opc_imm(s, i2, a0, a1, a2);
-            break;
-        }
-    do_binaryv:
-        tcg_out_opc_reg(s, i1, a0, a1, a2);
-        break;
-
-    case INDEX_op_sub_i32:
-        if (c2) {
-            tcg_out_opc_imm(s, OPC_ADDIU, a0, a1, -a2);
-            break;
-        }
-        i1 = OPC_SUBU;
-        goto do_binary;
-    case INDEX_op_and_i32:
-        if (c2 && a2 != (uint16_t)a2) {
-            int msb = ctz32(~a2) - 1;
-            assert(use_mips32r2_instructions);
-            assert(is_p2m1(a2));
-            tcg_out_opc_bf(s, OPC_EXT, a0, a1, msb, 0);
-            break;
-        }
-        i1 = OPC_AND, i2 = OPC_ANDI;
-        goto do_binary;
-    case INDEX_op_nor_i32:
-        i1 = OPC_NOR;
-        goto do_binaryv;
-
-    case INDEX_op_mul_i32:
-        if (use_mips32_instructions) {
-            tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2);
-            break;
-        }
-        i1 = OPC_MULT, i2 = OPC_MFLO;
-        goto do_hilo1;
-    case INDEX_op_mulsh_i32:
-        i1 = OPC_MULT, i2 = OPC_MFHI;
-        goto do_hilo1;
-    case INDEX_op_muluh_i32:
-        i1 = OPC_MULTU, i2 = OPC_MFHI;
-        goto do_hilo1;
-    case INDEX_op_div_i32:
-        i1 = OPC_DIV, i2 = OPC_MFLO;
-        goto do_hilo1;
-    case INDEX_op_divu_i32:
-        i1 = OPC_DIVU, i2 = OPC_MFLO;
-        goto do_hilo1;
-    case INDEX_op_rem_i32:
-        i1 = OPC_DIV, i2 = OPC_MFHI;
-        goto do_hilo1;
-    case INDEX_op_remu_i32:
-        i1 = OPC_DIVU, i2 = OPC_MFHI;
-    do_hilo1:
-        tcg_out_opc_reg(s, i1, 0, a1, a2);
-        tcg_out_opc_reg(s, i2, a0, 0, 0);
-        break;
-
-    case INDEX_op_muls2_i32:
-        i1 = OPC_MULT;
-        goto do_hilo2;
-    case INDEX_op_mulu2_i32:
-        i1 = OPC_MULTU;
-    do_hilo2:
-        tcg_out_opc_reg(s, i1, 0, a2, args[3]);
-        tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0);
-        tcg_out_opc_reg(s, OPC_MFHI, a1, 0, 0);
-        break;
-
-    case INDEX_op_not_i32:
-        i1 = OPC_NOR;
-        goto do_unary;
-    case INDEX_op_bswap16_i32:
-        i1 = OPC_WSBH;
-        goto do_unary;
-    case INDEX_op_ext8s_i32:
-        i1 = OPC_SEB;
-        goto do_unary;
-    case INDEX_op_ext16s_i32:
-        i1 = OPC_SEH;
-    do_unary:
-        tcg_out_opc_reg(s, i1, a0, TCG_REG_ZERO, a1);
-        break;
-
-    case INDEX_op_sar_i32:
-        i1 = OPC_SRAV, i2 = OPC_SRA;
-        goto do_shift;
-    case INDEX_op_shl_i32:
-        i1 = OPC_SLLV, i2 = OPC_SLL;
-        goto do_shift;
-    case INDEX_op_shr_i32:
-        i1 = OPC_SRLV, i2 = OPC_SRL;
-        goto do_shift;
-    case INDEX_op_rotr_i32:
-        i1 = OPC_ROTRV, i2 = OPC_ROTR;
-    do_shift:
-        if (c2) {
-            tcg_out_opc_sa(s, i2, a0, a1, a2);
-        } else {
-            tcg_out_opc_reg(s, i1, a0, a2, a1);
-        }
-        break;
-    case INDEX_op_rotl_i32:
-        if (c2) {
-            tcg_out_opc_sa(s, OPC_ROTR, a0, a1, 32 - a2);
-        } else {
-            tcg_out_opc_reg(s, OPC_SUBU, TCG_TMP0, TCG_REG_ZERO, a2);
-            tcg_out_opc_reg(s, OPC_ROTRV, a0, TCG_TMP0, a1);
-        }
-        break;
-
-    case INDEX_op_bswap32_i32:
-        tcg_out_opc_reg(s, OPC_WSBH, a0, 0, a1);
-        tcg_out_opc_sa(s, OPC_ROTR, a0, a0, 16);
-        break;
-
-    case INDEX_op_deposit_i32:
-        tcg_out_opc_bf(s, OPC_INS, a0, a2, args[3] + args[4] - 1, args[3]);
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_brcond(s, a2, a0, a1, args[3]);
-        break;
-    case INDEX_op_brcond2_i32:
-        tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], args[5]);
-        break;
-
-    case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, args[5], a0, a1, a2, args[3]);
-        break;
-
-    case INDEX_op_setcond_i32:
-        tcg_out_setcond(s, args[3], a0, a1, a2);
-        break;
-    case INDEX_op_setcond2_i32:
-        tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]);
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_qemu_ld(s, args, false);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, args, true);
-        break;
-    case INDEX_op_qemu_st_i32:
-        tcg_out_qemu_st(s, args, false);
-        break;
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args, true);
-        break;
-
-    case INDEX_op_add2_i32:
-        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5],
-                        const_args[4], const_args[5], false);
-        break;
-    case INDEX_op_sub2_i32:
-        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5],
-                        const_args[4], const_args[5], true);
-        break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef mips_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "rZ", "r" } },
-    { INDEX_op_st16_i32, { "rZ", "r" } },
-    { INDEX_op_st_i32, { "rZ", "r" } },
-
-    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_muls2_i32, { "r", "r", "rZ", "rZ" } },
-    { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
-    { INDEX_op_mulsh_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_muluh_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_divu_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_rem_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_remu_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_sub_i32, { "r", "rZ", "rN" } },
-
-    { INDEX_op_and_i32, { "r", "rZ", "rIK" } },
-    { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_not_i32, { "r", "rZ" } },
-    { INDEX_op_or_i32, { "r", "rZ", "rIZ" } },
-    { INDEX_op_xor_i32, { "r", "rZ", "rIZ" } },
-
-    { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
-    { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
-
-    { INDEX_op_bswap16_i32, { "r", "r" } },
-    { INDEX_op_bswap32_i32, { "r", "r" } },
-
-    { INDEX_op_ext8s_i32, { "r", "rZ" } },
-    { INDEX_op_ext16s_i32, { "r", "rZ" } },
-
-    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
-
-    { INDEX_op_brcond_i32, { "rZ", "rZ" } },
-    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
-    { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
-
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rN", "rN" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rN", "rN" } },
-    { INDEX_op_brcond2_i32, { "rZ", "rZ", "rZ", "rZ" } },
-
-#if TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld_i32, { "L", "lZ" } },
-    { INDEX_op_qemu_st_i32, { "SZ", "SZ" } },
-    { INDEX_op_qemu_ld_i64, { "L", "L", "lZ" } },
-    { INDEX_op_qemu_st_i64, { "SZ", "SZ", "SZ" } },
-#else
-    { INDEX_op_qemu_ld_i32, { "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_st_i32, { "SZ", "SZ", "SZ" } },
-    { INDEX_op_qemu_ld_i64, { "L", "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_st_i64, { "SZ", "SZ", "SZ", "SZ" } },
-#endif
-    { -1 },
-};
-
-static int tcg_target_callee_save_regs[] = {
-    TCG_REG_S0,       /* used for the global env (TCG_AREG0) */
-    TCG_REG_S1,
-    TCG_REG_S2,
-    TCG_REG_S3,
-    TCG_REG_S4,
-    TCG_REG_S5,
-    TCG_REG_S6,
-    TCG_REG_S7,
-    TCG_REG_S8,
-    TCG_REG_RA,       /* should be last for ABI compliance */
-};
-
-/* The Linux kernel doesn't provide any information about the available
-   instruction set. Probe it using a signal handler. */
-
-#include <signal.h>
-
-#ifndef use_movnz_instructions
-bool use_movnz_instructions = false;
-#endif
-
-#ifndef use_mips32_instructions
-bool use_mips32_instructions = false;
-#endif
-
-#ifndef use_mips32r2_instructions
-bool use_mips32r2_instructions = false;
-#endif
-
-static volatile sig_atomic_t got_sigill;
-
-static void sigill_handler(int signo, siginfo_t *si, void *data)
-{
-    /* Skip the faulty instruction */
-    ucontext_t *uc = (ucontext_t *)data;
-    uc->uc_mcontext.pc += 4;
-
-    got_sigill = 1;
-}
-
-static void tcg_target_detect_isa(void)
-{
-    struct sigaction sa_old, sa_new;
-
-    memset(&sa_new, 0, sizeof(sa_new));
-    sa_new.sa_flags = SA_SIGINFO;
-    sa_new.sa_sigaction = sigill_handler;
-    sigaction(SIGILL, &sa_new, &sa_old);
-
-    /* Probe for movn/movz, necessary to implement movcond. */
-#ifndef use_movnz_instructions
-    got_sigill = 0;
-    asm volatile(".set push\n"
-                 ".set mips32\n"
-                 "movn $zero, $zero, $zero\n"
-                 "movz $zero, $zero, $zero\n"
-                 ".set pop\n"
-                 : : : );
-    use_movnz_instructions = !got_sigill;
-#endif
-
-    /* Probe for MIPS32 instructions. As no subsetting is allowed
-       by the specification, it is only necessary to probe for one
-       of the instructions. */
-#ifndef use_mips32_instructions
-    got_sigill = 0;
-    asm volatile(".set push\n"
-                 ".set mips32\n"
-                 "mul $zero, $zero\n"
-                 ".set pop\n"
-                 : : : );
-    use_mips32_instructions = !got_sigill;
-#endif
-
-    /* Probe for MIPS32r2 instructions if MIPS32 instructions are
-       available. As no subsetting is allowed by the specification,
-       it is only necessary to probe for one of the instructions. */
-#ifndef use_mips32r2_instructions
-    if (use_mips32_instructions) {
-        got_sigill = 0;
-        asm volatile(".set push\n"
-                     ".set mips32r2\n"
-                     "seb $zero, $zero\n"
-                     ".set pop\n"
-                     : : : );
-        use_mips32r2_instructions = !got_sigill;
-    }
-#endif
-
-    sigaction(SIGILL, &sa_old, NULL);
-}
-
-/* Generate global QEMU prologue and epilogue code */
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int i, frame_size;
-
-    /* reserve some stack space, also for TCG temps. */
-    frame_size = ARRAY_SIZE(tcg_target_callee_save_regs) * 4
-                 + TCG_STATIC_CALL_ARGS_SIZE
-                 + CPU_TEMP_BUF_NLONGS * sizeof(long);
-    frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
-                 ~(TCG_TARGET_STACK_ALIGN - 1);
-    tcg_set_frame(s, TCG_REG_SP, ARRAY_SIZE(tcg_target_callee_save_regs) * 4
-                  + TCG_STATIC_CALL_ARGS_SIZE,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-    /* TB prologue */
-    tcg_out_addi(s, TCG_REG_SP, -frame_size);
-    for(i = 0 ; i < ARRAY_SIZE(tcg_target_callee_save_regs) ; i++) {
-        tcg_out_st(s, TCG_TYPE_I32, tcg_target_callee_save_regs[i],
-                   TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE + i * 4);
-    }
-
-    /* Call generated code */
-    tcg_out_opc_reg(s, OPC_JR, 0, tcg_target_call_iarg_regs[1], 0);
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-    tb_ret_addr = s->code_ptr;
-
-    /* TB epilogue */
-    for(i = 0 ; i < ARRAY_SIZE(tcg_target_callee_save_regs) ; i++) {
-        tcg_out_ld(s, TCG_TYPE_I32, tcg_target_callee_save_regs[i],
-                   TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE + i * 4);
-    }
-
-    tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_RA, 0);
-    tcg_out_addi(s, TCG_REG_SP, frame_size);
-}
-
-static void tcg_target_init(TCGContext *s)
-{
-    tcg_target_detect_isa();
-    tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I32], 0xffffffff);
-    tcg_regset_set(tcg_target_call_clobber_regs,
-                   (1 << TCG_REG_V0) |
-                   (1 << TCG_REG_V1) |
-                   (1 << TCG_REG_A0) |
-                   (1 << TCG_REG_A1) |
-                   (1 << TCG_REG_A2) |
-                   (1 << TCG_REG_A3) |
-                   (1 << TCG_REG_T0) |
-                   (1 << TCG_REG_T1) |
-                   (1 << TCG_REG_T2) |
-                   (1 << TCG_REG_T3) |
-                   (1 << TCG_REG_T4) |
-                   (1 << TCG_REG_T5) |
-                   (1 << TCG_REG_T6) |
-                   (1 << TCG_REG_T7) |
-                   (1 << TCG_REG_T8) |
-                   (1 << TCG_REG_T9));
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); /* zero register */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_K0);   /* kernel use only */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_K1);   /* kernel use only */
-    tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);     /* internal use */
-    tcg_regset_set_reg(s->reserved_regs, TCG_TMP1);     /* internal use */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);   /* return address */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);   /* stack pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);   /* global pointer */
-
-    tcg_add_target_add_op_defs(mips_op_defs);
-}
-
-void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
-{
-    uint32_t *ptr = (uint32_t *)jmp_addr;
-    *ptr = deposit32(*ptr, 0, 26, addr >> 2);
-    flush_icache_range(jmp_addr, jmp_addr + 4);
-}
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index c88a1c9..3aeac87 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -23,10 +23,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_MIPS 
-#define TCG_TARGET_MIPS 1
+
+#ifndef MIPS_TCG_TARGET_H
+#define MIPS_TCG_TARGET_H
 
 #define TCG_TARGET_INSN_UNIT_SIZE 4
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 16
 #define TCG_TARGET_NB_REGS 32
 
 typedef enum {
@@ -95,6 +97,13 @@
 extern bool use_mips32r2_instructions;
 #endif
 
+/* MIPS32R6 instruction set detection */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 6)
+#define use_mips32r6_instructions  1
+#else
+#define use_mips32r6_instructions  0
+#endif
+
 /* optional instructions */
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_rem_i32          1
@@ -104,8 +113,8 @@
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
-#define TCG_TARGET_HAS_mulu2_i32        1
-#define TCG_TARGET_HAS_muls2_i32        1
+#define TCG_TARGET_HAS_mulu2_i32        (!use_mips32r6_instructions)
+#define TCG_TARGET_HAS_muls2_i32        (!use_mips32r6_instructions)
 #define TCG_TARGET_HAS_muluh_i32        1
 #define TCG_TARGET_HAS_mulsh_i32        1
 
diff --git a/tcg/mips/tcg-target.inc.c b/tcg/mips/tcg-target.inc.c
new file mode 100644
index 0000000..2f9be48
--- /dev/null
+++ b/tcg/mips/tcg-target.inc.c
@@ -0,0 +1,1900 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org>
+ * Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net>
+ * Based on i386/tcg-target.c - Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg-be-ldst.h"
+
+#ifdef HOST_WORDS_BIGENDIAN
+# define MIPS_BE  1
+#else
+# define MIPS_BE  0
+#endif
+
+#define LO_OFF    (MIPS_BE * 4)
+#define HI_OFF    (4 - LO_OFF)
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "zero",
+    "at",
+    "v0",
+    "v1",
+    "a0",
+    "a1",
+    "a2",
+    "a3",
+    "t0",
+    "t1",
+    "t2",
+    "t3",
+    "t4",
+    "t5",
+    "t6",
+    "t7",
+    "s0",
+    "s1",
+    "s2",
+    "s3",
+    "s4",
+    "s5",
+    "s6",
+    "s7",
+    "t8",
+    "t9",
+    "k0",
+    "k1",
+    "gp",
+    "sp",
+    "s8",
+    "ra",
+};
+#endif
+
+#define TCG_TMP0  TCG_REG_AT
+#define TCG_TMP1  TCG_REG_T9
+
+/* check if we really need so many registers :P */
+static const int tcg_target_reg_alloc_order[] = {
+    /* Call saved registers.  */
+    TCG_REG_S0,
+    TCG_REG_S1,
+    TCG_REG_S2,
+    TCG_REG_S3,
+    TCG_REG_S4,
+    TCG_REG_S5,
+    TCG_REG_S6,
+    TCG_REG_S7,
+    TCG_REG_S8,
+
+    /* Call clobbered registers.  */
+    TCG_REG_T0,
+    TCG_REG_T1,
+    TCG_REG_T2,
+    TCG_REG_T3,
+    TCG_REG_T4,
+    TCG_REG_T5,
+    TCG_REG_T6,
+    TCG_REG_T7,
+    TCG_REG_T8,
+    TCG_REG_T9,
+    TCG_REG_V1,
+    TCG_REG_V0,
+
+    /* Argument registers, opposite order of allocation.  */
+    TCG_REG_A3,
+    TCG_REG_A2,
+    TCG_REG_A1,
+    TCG_REG_A0,
+};
+
+static const TCGReg tcg_target_call_iarg_regs[4] = {
+    TCG_REG_A0,
+    TCG_REG_A1,
+    TCG_REG_A2,
+    TCG_REG_A3
+};
+
+static const TCGReg tcg_target_call_oarg_regs[2] = {
+    TCG_REG_V0,
+    TCG_REG_V1
+};
+
+static tcg_insn_unit *tb_ret_addr;
+
+static inline uint32_t reloc_pc16_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    /* Let the compiler perform the right-shift as part of the arithmetic.  */
+    ptrdiff_t disp = target - (pc + 1);
+    tcg_debug_assert(disp == (int16_t)disp);
+    return disp & 0xffff;
+}
+
+static inline void reloc_pc16(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    *pc = deposit32(*pc, 0, 16, reloc_pc16_val(pc, target));
+}
+
+static inline uint32_t reloc_26_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    tcg_debug_assert((((uintptr_t)pc ^ (uintptr_t)target) & 0xf0000000) == 0);
+    return ((uintptr_t)target >> 2) & 0x3ffffff;
+}
+
+static inline void reloc_26(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    *pc = deposit32(*pc, 0, 26, reloc_26_val(pc, target));
+}
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    tcg_debug_assert(type == R_MIPS_PC16);
+    tcg_debug_assert(addend == 0);
+    reloc_pc16(code_ptr, (tcg_insn_unit *)value);
+}
+
+#define TCG_CT_CONST_ZERO 0x100
+#define TCG_CT_CONST_U16  0x200    /* Unsigned 16-bit: 0 - 0xffff.  */
+#define TCG_CT_CONST_S16  0x400    /* Signed 16-bit: -32768 - 32767 */
+#define TCG_CT_CONST_P2M1 0x800    /* Power of 2 minus 1.  */
+#define TCG_CT_CONST_N16  0x1000   /* "Negatable" 16-bit: -32767 - 32767 */
+
+static inline bool is_p2m1(tcg_target_long val)
+{
+    return val && ((val + 1) & val) == 0;
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch(ct_str[0]) {
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set(ct->u.regs, 0xffffffff);
+        break;
+    case 'L': /* qemu_ld output arg constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set(ct->u.regs, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_V0);
+        break;
+    case 'l': /* qemu_ld input arg constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set(ct->u.regs, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
+#if defined(CONFIG_SOFTMMU)
+        if (TARGET_LONG_BITS == 64) {
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
+        }
+#endif
+        break;
+    case 'S': /* qemu_st constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set(ct->u.regs, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
+#if defined(CONFIG_SOFTMMU)
+        if (TARGET_LONG_BITS == 32) {
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A1);
+        } else {
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A3);
+        }
+#endif
+        break;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_U16;
+        break;
+    case 'J':
+        ct->ct |= TCG_CT_CONST_S16;
+        break;
+    case 'K':
+        ct->ct |= TCG_CT_CONST_P2M1;
+        break;
+    case 'N':
+        ct->ct |= TCG_CT_CONST_N16;
+        break;
+    case 'Z':
+        /* We are cheating a bit here, using the fact that the register
+           ZERO is also the register number 0. Hence there is no need
+           to check for const_args in each instruction. */
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct;
+    ct = arg_ct->ct;
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_N16) && val >= -32767 && val <= 32767) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_P2M1)
+               && use_mips32r2_instructions && is_p2m1(val)) {
+        return 1;
+    }
+    return 0;
+}
+
+/* instruction opcodes */
+typedef enum {
+    OPC_J        = 0x02 << 26,
+    OPC_JAL      = 0x03 << 26,
+    OPC_BEQ      = 0x04 << 26,
+    OPC_BNE      = 0x05 << 26,
+    OPC_BLEZ     = 0x06 << 26,
+    OPC_BGTZ     = 0x07 << 26,
+    OPC_ADDIU    = 0x09 << 26,
+    OPC_SLTI     = 0x0A << 26,
+    OPC_SLTIU    = 0x0B << 26,
+    OPC_ANDI     = 0x0C << 26,
+    OPC_ORI      = 0x0D << 26,
+    OPC_XORI     = 0x0E << 26,
+    OPC_LUI      = 0x0F << 26,
+    OPC_LB       = 0x20 << 26,
+    OPC_LH       = 0x21 << 26,
+    OPC_LW       = 0x23 << 26,
+    OPC_LBU      = 0x24 << 26,
+    OPC_LHU      = 0x25 << 26,
+    OPC_LWU      = 0x27 << 26,
+    OPC_SB       = 0x28 << 26,
+    OPC_SH       = 0x29 << 26,
+    OPC_SW       = 0x2B << 26,
+
+    OPC_SPECIAL  = 0x00 << 26,
+    OPC_SLL      = OPC_SPECIAL | 0x00,
+    OPC_SRL      = OPC_SPECIAL | 0x02,
+    OPC_ROTR     = OPC_SPECIAL | (0x01 << 21) | 0x02,
+    OPC_SRA      = OPC_SPECIAL | 0x03,
+    OPC_SLLV     = OPC_SPECIAL | 0x04,
+    OPC_SRLV     = OPC_SPECIAL | 0x06,
+    OPC_ROTRV    = OPC_SPECIAL | (0x01 <<  6) | 0x06,
+    OPC_SRAV     = OPC_SPECIAL | 0x07,
+    OPC_JR_R5    = OPC_SPECIAL | 0x08,
+    OPC_JALR     = OPC_SPECIAL | 0x09,
+    OPC_MOVZ     = OPC_SPECIAL | 0x0A,
+    OPC_MOVN     = OPC_SPECIAL | 0x0B,
+    OPC_MFHI     = OPC_SPECIAL | 0x10,
+    OPC_MFLO     = OPC_SPECIAL | 0x12,
+    OPC_MULT     = OPC_SPECIAL | 0x18,
+    OPC_MUL_R6   = OPC_SPECIAL | (0x02 <<  6) | 0x18,
+    OPC_MUH      = OPC_SPECIAL | (0x03 <<  6) | 0x18,
+    OPC_MULTU    = OPC_SPECIAL | 0x19,
+    OPC_MULU     = OPC_SPECIAL | (0x02 <<  6) | 0x19,
+    OPC_MUHU     = OPC_SPECIAL | (0x03 <<  6) | 0x19,
+    OPC_DIV      = OPC_SPECIAL | 0x1A,
+    OPC_DIV_R6   = OPC_SPECIAL | (0x02 <<  6) | 0x1A,
+    OPC_MOD      = OPC_SPECIAL | (0x03 <<  6) | 0x1A,
+    OPC_DIVU     = OPC_SPECIAL | 0x1B,
+    OPC_DIVU_R6  = OPC_SPECIAL | (0x02 <<  6) | 0x1B,
+    OPC_MODU     = OPC_SPECIAL | (0x03 <<  6) | 0x1B,
+    OPC_ADDU     = OPC_SPECIAL | 0x21,
+    OPC_SUBU     = OPC_SPECIAL | 0x23,
+    OPC_AND      = OPC_SPECIAL | 0x24,
+    OPC_OR       = OPC_SPECIAL | 0x25,
+    OPC_XOR      = OPC_SPECIAL | 0x26,
+    OPC_NOR      = OPC_SPECIAL | 0x27,
+    OPC_SLT      = OPC_SPECIAL | 0x2A,
+    OPC_SLTU     = OPC_SPECIAL | 0x2B,
+    OPC_SELEQZ   = OPC_SPECIAL | 0x35,
+    OPC_SELNEZ   = OPC_SPECIAL | 0x37,
+
+    OPC_REGIMM   = 0x01 << 26,
+    OPC_BLTZ     = OPC_REGIMM | (0x00 << 16),
+    OPC_BGEZ     = OPC_REGIMM | (0x01 << 16),
+
+    OPC_SPECIAL2 = 0x1c << 26,
+    OPC_MUL_R5   = OPC_SPECIAL2 | 0x002,
+
+    OPC_SPECIAL3 = 0x1f << 26,
+    OPC_EXT      = OPC_SPECIAL3 | 0x000,
+    OPC_INS      = OPC_SPECIAL3 | 0x004,
+    OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
+    OPC_SEB      = OPC_SPECIAL3 | 0x420,
+    OPC_SEH      = OPC_SPECIAL3 | 0x620,
+
+    /* MIPS r6 doesn't have JR, JALR should be used instead */
+    OPC_JR       = use_mips32r6_instructions ? OPC_JALR : OPC_JR_R5,
+
+    /*
+     * MIPS r6 replaces MUL with an alternative encoding which is
+     * backwards-compatible at the assembly level.
+     */
+    OPC_MUL      = use_mips32r6_instructions ? OPC_MUL_R6 : OPC_MUL_R5,
+} MIPSInsn;
+
+/*
+ * Type reg
+ */
+static inline void tcg_out_opc_reg(TCGContext *s, MIPSInsn opc,
+                                   TCGReg rd, TCGReg rs, TCGReg rt)
+{
+    int32_t inst;
+
+    inst = opc;
+    inst |= (rs & 0x1F) << 21;
+    inst |= (rt & 0x1F) << 16;
+    inst |= (rd & 0x1F) << 11;
+    tcg_out32(s, inst);
+}
+
+/*
+ * Type immediate
+ */
+static inline void tcg_out_opc_imm(TCGContext *s, MIPSInsn opc,
+                                   TCGReg rt, TCGReg rs, TCGArg imm)
+{
+    int32_t inst;
+
+    inst = opc;
+    inst |= (rs & 0x1F) << 21;
+    inst |= (rt & 0x1F) << 16;
+    inst |= (imm & 0xffff);
+    tcg_out32(s, inst);
+}
+
+/*
+ * Type bitfield
+ */
+static inline void tcg_out_opc_bf(TCGContext *s, MIPSInsn opc, TCGReg rt,
+                                  TCGReg rs, int msb, int lsb)
+{
+    int32_t inst;
+
+    inst = opc;
+    inst |= (rs & 0x1F) << 21;
+    inst |= (rt & 0x1F) << 16;
+    inst |= (msb & 0x1F) << 11;
+    inst |= (lsb & 0x1F) << 6;
+    tcg_out32(s, inst);
+}
+
+/*
+ * Type branch
+ */
+static inline void tcg_out_opc_br(TCGContext *s, MIPSInsn opc,
+                                  TCGReg rt, TCGReg rs)
+{
+    /* We pay attention here to not modify the branch target by reading
+       the existing value and using it again. This ensure that caches and
+       memory are kept coherent during retranslation. */
+    uint16_t offset = (uint16_t)*s->code_ptr;
+
+    tcg_out_opc_imm(s, opc, rt, rs, offset);
+}
+
+/*
+ * Type sa
+ */
+static inline void tcg_out_opc_sa(TCGContext *s, MIPSInsn opc,
+                                  TCGReg rd, TCGReg rt, TCGArg sa)
+{
+    int32_t inst;
+
+    inst = opc;
+    inst |= (rt & 0x1F) << 16;
+    inst |= (rd & 0x1F) << 11;
+    inst |= (sa & 0x1F) <<  6;
+    tcg_out32(s, inst);
+
+}
+
+/*
+ * Type jump.
+ * Returns true if the branch was in range and the insn was emitted.
+ */
+static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target)
+{
+    uintptr_t dest = (uintptr_t)target;
+    uintptr_t from = (uintptr_t)s->code_ptr + 4;
+    int32_t inst;
+
+    /* The pc-region branch happens within the 256MB region of
+       the delay slot (thus the +4).  */
+    if ((from ^ dest) & -(1 << 28)) {
+        return false;
+    }
+    tcg_debug_assert((dest & 3) == 0);
+
+    inst = opc;
+    inst |= (dest >> 2) & 0x3ffffff;
+    tcg_out32(s, inst);
+    return true;
+}
+
+static inline void tcg_out_nop(TCGContext *s)
+{
+    tcg_out32(s, 0);
+}
+
+static inline void tcg_out_mov(TCGContext *s, TCGType type,
+                               TCGReg ret, TCGReg arg)
+{
+    /* Simple reg-reg move, optimising out the 'do nothing' case */
+    if (ret != arg) {
+        tcg_out_opc_reg(s, OPC_ADDU, ret, arg, TCG_REG_ZERO);
+    }
+}
+
+static inline void tcg_out_movi(TCGContext *s, TCGType type,
+                                TCGReg reg, tcg_target_long arg)
+{
+    if (arg == (int16_t)arg) {
+        tcg_out_opc_imm(s, OPC_ADDIU, reg, TCG_REG_ZERO, arg);
+    } else if (arg == (uint16_t)arg) {
+        tcg_out_opc_imm(s, OPC_ORI, reg, TCG_REG_ZERO, arg);
+    } else {
+        tcg_out_opc_imm(s, OPC_LUI, reg, TCG_REG_ZERO, arg >> 16);
+        if (arg & 0xffff) {
+            tcg_out_opc_imm(s, OPC_ORI, reg, reg, arg & 0xffff);
+        }
+    }
+}
+
+static inline void tcg_out_bswap16(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+    } else {
+        /* ret and arg can't be register at */
+        if (ret == TCG_TMP0 || arg == TCG_TMP0) {
+            tcg_abort();
+        }
+
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
+        tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
+    }
+}
+
+static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+        tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
+    } else {
+        /* ret and arg can't be register at */
+        if (ret == TCG_TMP0 || arg == TCG_TMP0) {
+            tcg_abort();
+        }
+
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
+        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
+    }
+}
+
+static inline void tcg_out_bswap32(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+        tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
+    } else {
+        /* ret and arg must be different and can't be register at */
+        if (ret == arg || ret == TCG_TMP0 || arg == TCG_TMP0) {
+            tcg_abort();
+        }
+
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
+
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 24);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
+
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, arg, 0xff00);
+        tcg_out_opc_sa(s, OPC_SLL, TCG_TMP0, TCG_TMP0, 8);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
+
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, TCG_TMP0, 0xff00);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
+    }
+}
+
+static inline void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_SEB, ret, 0, arg);
+    } else {
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
+        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 24);
+    }
+}
+
+static inline void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
+{
+    if (use_mips32r2_instructions) {
+        tcg_out_opc_reg(s, OPC_SEH, ret, 0, arg);
+    } else {
+        tcg_out_opc_sa(s, OPC_SLL, ret, arg, 16);
+        tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
+    }
+}
+
+static void tcg_out_ldst(TCGContext *s, MIPSInsn opc, TCGReg data,
+                         TCGReg addr, intptr_t ofs)
+{
+    int16_t lo = ofs;
+    if (ofs != lo) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - lo);
+        if (addr != TCG_REG_ZERO) {
+            tcg_out_opc_reg(s, OPC_ADDU, TCG_TMP0, TCG_TMP0, addr);
+        }
+        addr = TCG_TMP0;
+    }
+    tcg_out_opc_imm(s, opc, data, addr, lo);
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ldst(s, OPC_LW, arg, arg1, arg2);
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    if (val == 0) {
+        tcg_out_st(s, type, TCG_REG_ZERO, base, ofs);
+        return true;
+    }
+    return false;
+}
+
+static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
+{
+    if (val == (int16_t)val) {
+        tcg_out_opc_imm(s, OPC_ADDIU, reg, reg, val);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, val);
+        tcg_out_opc_reg(s, OPC_ADDU, reg, reg, TCG_TMP0);
+    }
+}
+
+static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
+                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
+                            bool cbh, bool is_sub)
+{
+    TCGReg th = TCG_TMP1;
+
+    /* If we have a negative constant such that negating it would
+       make the high part zero, we can (usually) eliminate one insn.  */
+    if (cbl && cbh && bh == -1 && bl != 0) {
+        bl = -bl;
+        bh = 0;
+        is_sub = !is_sub;
+    }
+
+    /* By operating on the high part first, we get to use the final
+       carry operation to move back from the temporary.  */
+    if (!cbh) {
+        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
+    } else if (bh != 0 || ah == rl) {
+        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
+    } else {
+        th = ah;
+    }
+
+    /* Note that tcg optimization should eliminate the bl == 0 case.  */
+    if (is_sub) {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
+            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
+        }
+        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
+    } else {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
+        } else if (rl == al && rl == bl) {
+            tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, al, 31);
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
+        }
+        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
+    }
+}
+
+/* Bit 0 set if inversion required; bit 1 set if swapping required.  */
+#define MIPS_CMP_INV  1
+#define MIPS_CMP_SWAP 2
+
+static const uint8_t mips_cmp_map[16] = {
+    [TCG_COND_LT]  = 0,
+    [TCG_COND_LTU] = 0,
+    [TCG_COND_GE]  = MIPS_CMP_INV,
+    [TCG_COND_GEU] = MIPS_CMP_INV,
+    [TCG_COND_LE]  = MIPS_CMP_INV | MIPS_CMP_SWAP,
+    [TCG_COND_LEU] = MIPS_CMP_INV | MIPS_CMP_SWAP,
+    [TCG_COND_GT]  = MIPS_CMP_SWAP,
+    [TCG_COND_GTU] = MIPS_CMP_SWAP,
+};
+
+static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGReg arg1, TCGReg arg2)
+{
+    MIPSInsn s_opc = OPC_SLTU;
+    int cmp_map;
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        if (arg2 != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
+            arg1 = ret;
+        }
+        tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, 1);
+        break;
+
+    case TCG_COND_NE:
+        if (arg2 != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
+            arg1 = ret;
+        }
+        tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, arg1);
+        break;
+
+    case TCG_COND_LT:
+    case TCG_COND_GE:
+    case TCG_COND_LE:
+    case TCG_COND_GT:
+        s_opc = OPC_SLT;
+        /* FALLTHRU */
+
+    case TCG_COND_LTU:
+    case TCG_COND_GEU:
+    case TCG_COND_LEU:
+    case TCG_COND_GTU:
+        cmp_map = mips_cmp_map[cond];
+        if (cmp_map & MIPS_CMP_SWAP) {
+            TCGReg t = arg1;
+            arg1 = arg2;
+            arg2 = t;
+        }
+        tcg_out_opc_reg(s, s_opc, ret, arg1, arg2);
+        if (cmp_map & MIPS_CMP_INV) {
+            tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
+        }
+        break;
+
+     default:
+         tcg_abort();
+         break;
+     }
+}
+
+static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
+                           TCGReg arg2, TCGLabel *l)
+{
+    static const MIPSInsn b_zero[16] = {
+        [TCG_COND_LT] = OPC_BLTZ,
+        [TCG_COND_GT] = OPC_BGTZ,
+        [TCG_COND_LE] = OPC_BLEZ,
+        [TCG_COND_GE] = OPC_BGEZ,
+    };
+
+    MIPSInsn s_opc = OPC_SLTU;
+    MIPSInsn b_opc;
+    int cmp_map;
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        b_opc = OPC_BEQ;
+        break;
+    case TCG_COND_NE:
+        b_opc = OPC_BNE;
+        break;
+
+    case TCG_COND_LT:
+    case TCG_COND_GT:
+    case TCG_COND_LE:
+    case TCG_COND_GE:
+        if (arg2 == 0) {
+            b_opc = b_zero[cond];
+            arg2 = arg1;
+            arg1 = 0;
+            break;
+        }
+        s_opc = OPC_SLT;
+        /* FALLTHRU */
+
+    case TCG_COND_LTU:
+    case TCG_COND_GTU:
+    case TCG_COND_LEU:
+    case TCG_COND_GEU:
+        cmp_map = mips_cmp_map[cond];
+        if (cmp_map & MIPS_CMP_SWAP) {
+            TCGReg t = arg1;
+            arg1 = arg2;
+            arg2 = t;
+        }
+        tcg_out_opc_reg(s, s_opc, TCG_TMP0, arg1, arg2);
+        b_opc = (cmp_map & MIPS_CMP_INV ? OPC_BEQ : OPC_BNE);
+        arg1 = TCG_TMP0;
+        arg2 = TCG_REG_ZERO;
+        break;
+
+    default:
+        tcg_abort();
+        break;
+    }
+
+    tcg_out_opc_br(s, b_opc, arg1, arg2);
+    if (l->has_value) {
+        reloc_pc16(s->code_ptr - 1, l->u.value_ptr);
+    } else {
+        tcg_out_reloc(s, s->code_ptr - 1, R_MIPS_PC16, l, 0);
+    }
+    tcg_out_nop(s);
+}
+
+static TCGReg tcg_out_reduce_eq2(TCGContext *s, TCGReg tmp0, TCGReg tmp1,
+                                 TCGReg al, TCGReg ah,
+                                 TCGReg bl, TCGReg bh)
+{
+    /* Merge highpart comparison into AH.  */
+    if (bh != 0) {
+        if (ah != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, tmp0, ah, bh);
+            ah = tmp0;
+        } else {
+            ah = bh;
+        }
+    }
+    /* Merge lowpart comparison into AL.  */
+    if (bl != 0) {
+        if (al != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, tmp1, al, bl);
+            al = tmp1;
+        } else {
+            al = bl;
+        }
+    }
+    /* Merge high and low part comparisons into AL.  */
+    if (ah != 0) {
+        if (al != 0) {
+            tcg_out_opc_reg(s, OPC_OR, tmp0, ah, al);
+            al = tmp0;
+        } else {
+            al = ah;
+        }
+    }
+    return al;
+}
+
+static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
+                             TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh)
+{
+    TCGReg tmp0 = TCG_TMP0;
+    TCGReg tmp1 = ret;
+
+    tcg_debug_assert(ret != TCG_TMP0);
+    if (ret == ah || ret == bh) {
+        tcg_debug_assert(ret != TCG_TMP1);
+        tmp1 = TCG_TMP1;
+    }
+
+    switch (cond) {
+    case TCG_COND_EQ:
+    case TCG_COND_NE:
+        tmp1 = tcg_out_reduce_eq2(s, tmp0, tmp1, al, ah, bl, bh);
+        tcg_out_setcond(s, cond, ret, tmp1, TCG_REG_ZERO);
+        break;
+
+    default:
+        tcg_out_setcond(s, TCG_COND_EQ, tmp0, ah, bh);
+        tcg_out_setcond(s, tcg_unsigned_cond(cond), tmp1, al, bl);
+        tcg_out_opc_reg(s, OPC_AND, tmp1, tmp1, tmp0);
+        tcg_out_setcond(s, tcg_high_cond(cond), tmp0, ah, bh);
+        tcg_out_opc_reg(s, OPC_OR, ret, tmp1, tmp0);
+        break;
+    }
+}
+
+static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah,
+                            TCGReg bl, TCGReg bh, TCGLabel *l)
+{
+    TCGCond b_cond = TCG_COND_NE;
+    TCGReg tmp = TCG_TMP1;
+
+    /* With branches, we emit between 4 and 9 insns with 2 or 3 branches.
+       With setcond, we emit between 3 and 10 insns and only 1 branch,
+       which ought to get better branch prediction.  */
+     switch (cond) {
+     case TCG_COND_EQ:
+     case TCG_COND_NE:
+        b_cond = cond;
+        tmp = tcg_out_reduce_eq2(s, TCG_TMP0, TCG_TMP1, al, ah, bl, bh);
+        break;
+
+    default:
+        /* Minimize code size by preferring a compare not requiring INV.  */
+        if (mips_cmp_map[cond] & MIPS_CMP_INV) {
+            cond = tcg_invert_cond(cond);
+            b_cond = TCG_COND_EQ;
+        }
+        tcg_out_setcond2(s, cond, tmp, al, ah, bl, bh);
+        break;
+    }
+
+    tcg_out_brcond(s, b_cond, tmp, TCG_REG_ZERO, l);
+}
+
+static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGReg c1, TCGReg c2, TCGReg v1, TCGReg v2)
+{
+    bool eqz = false;
+
+    /* If one of the values is zero, put it last to match SEL*Z instructions */
+    if (use_mips32r6_instructions && v1 == 0) {
+        v1 = v2;
+        v2 = 0;
+        cond = tcg_invert_cond(cond);
+    }
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        eqz = true;
+        /* FALLTHRU */
+    case TCG_COND_NE:
+        if (c2 != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, TCG_TMP0, c1, c2);
+            c1 = TCG_TMP0;
+        }
+        break;
+
+    default:
+        /* Minimize code size by preferring a compare not requiring INV.  */
+        if (mips_cmp_map[cond] & MIPS_CMP_INV) {
+            cond = tcg_invert_cond(cond);
+            eqz = true;
+        }
+        tcg_out_setcond(s, cond, TCG_TMP0, c1, c2);
+        c1 = TCG_TMP0;
+        break;
+    }
+
+    if (use_mips32r6_instructions) {
+        MIPSInsn m_opc_t = eqz ? OPC_SELEQZ : OPC_SELNEZ;
+        MIPSInsn m_opc_f = eqz ? OPC_SELNEZ : OPC_SELEQZ;
+
+        if (v2 != 0) {
+            tcg_out_opc_reg(s, m_opc_f, TCG_TMP1, v2, c1);
+        }
+        tcg_out_opc_reg(s, m_opc_t, ret, v1, c1);
+        if (v2 != 0) {
+            tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP1);
+        }
+    } else {
+        MIPSInsn m_opc = eqz ? OPC_MOVZ : OPC_MOVN;
+
+        tcg_out_opc_reg(s, m_opc, ret, v1, c1);
+
+        /* This should be guaranteed via constraints */
+        tcg_debug_assert(v2 == ret);
+    }
+}
+
+static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
+{
+    /* Note that the ABI requires the called function's address to be
+       loaded into T9, even if a direct branch is in range.  */
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T9, (uintptr_t)arg);
+
+    /* But do try a direct branch, allowing the cpu better insn prefetch.  */
+    if (tail) {
+        if (!tcg_out_opc_jmp(s, OPC_J, arg)) {
+            tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_T9, 0);
+        }
+    } else {
+        if (!tcg_out_opc_jmp(s, OPC_JAL, arg)) {
+            tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, TCG_REG_T9, 0);
+        }
+    }
+}
+
+static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+{
+    tcg_out_call_int(s, arg, false);
+    tcg_out_nop(s);
+}
+
+#if defined(CONFIG_SOFTMMU)
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_SB]   = helper_ret_ldsb_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LESW] = helper_le_ldsw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BESW] = helper_be_ldsw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+};
+
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
+};
+
+/* Helper routines for marshalling helper function arguments into
+ * the correct registers and stack.
+ * I is where we want to put this argument, and is updated and returned
+ * for the next call. ARG is the argument itself.
+ *
+ * We provide routines for arguments which are: immediate, 32 bit
+ * value in register, 16 and 8 bit values in register (which must be zero
+ * extended before use) and 64 bit value in a lo:hi register pair.
+ */
+
+static int tcg_out_call_iarg_reg(TCGContext *s, int i, TCGReg arg)
+{
+    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[i], arg);
+    } else {
+        tcg_out_st(s, TCG_TYPE_REG, arg, TCG_REG_SP, 4 * i);
+    }
+    return i + 1;
+}
+
+static int tcg_out_call_iarg_reg8(TCGContext *s, int i, TCGReg arg)
+{
+    TCGReg tmp = TCG_TMP0;
+    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+        tmp = tcg_target_call_iarg_regs[i];
+    }
+    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xff);
+    return tcg_out_call_iarg_reg(s, i, tmp);
+}
+
+static int tcg_out_call_iarg_reg16(TCGContext *s, int i, TCGReg arg)
+{
+    TCGReg tmp = TCG_TMP0;
+    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+        tmp = tcg_target_call_iarg_regs[i];
+    }
+    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xffff);
+    return tcg_out_call_iarg_reg(s, i, tmp);
+}
+
+static int tcg_out_call_iarg_imm(TCGContext *s, int i, TCGArg arg)
+{
+    TCGReg tmp = TCG_TMP0;
+    if (arg == 0) {
+        tmp = TCG_REG_ZERO;
+    } else {
+        if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+            tmp = tcg_target_call_iarg_regs[i];
+        }
+        tcg_out_movi(s, TCG_TYPE_REG, tmp, arg);
+    }
+    return tcg_out_call_iarg_reg(s, i, tmp);
+}
+
+static int tcg_out_call_iarg_reg2(TCGContext *s, int i, TCGReg al, TCGReg ah)
+{
+    i = (i + 1) & ~1;
+    i = tcg_out_call_iarg_reg(s, i, (MIPS_BE ? ah : al));
+    i = tcg_out_call_iarg_reg(s, i, (MIPS_BE ? al : ah));
+    return i;
+}
+
+/* Perform the tlb comparison operation.  The complete host address is
+   placed in BASE.  Clobbers AT, T0, A0.  */
+static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl,
+                             TCGReg addrh, TCGMemOpIdx oi,
+                             tcg_insn_unit *label_ptr[2], bool is_load)
+{
+    TCGMemOp s_bits = get_memop(oi) & MO_SIZE;
+    int mem_index = get_mmuidx(oi);
+    int cmp_off
+        = (is_load
+           ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+           : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
+
+    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addrl,
+                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
+    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0,
+                    (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
+    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, TCG_AREG0);
+
+    /* Compensate for very large offsets.  */
+    if (add_off >= 0x8000) {
+        /* Most target env are smaller than 32k; none are larger than 64k.
+           Simplify the logic here merely to offset by 0x7ff0, giving us a
+           range just shy of 64k.  Check this assumption.  */
+        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
+                                   tlb_table[NB_MMU_MODES - 1][1])
+                          > 0x7ff0 + 0x7fff);
+        tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_A0, TCG_REG_A0, 0x7ff0);
+        cmp_off -= 0x7ff0;
+        add_off -= 0x7ff0;
+    }
+
+    /* Load the (low half) tlb comparator.  */
+    tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0,
+                    cmp_off + (TARGET_LONG_BITS == 64 ? LO_OFF : 0));
+
+    /* Mask the page bits, keeping the alignment bits to compare against.
+       In between on 32-bit targets, load the tlb addend for the fast path.  */
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1,
+                 TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
+    }
+    tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrl);
+
+    label_ptr[0] = s->code_ptr;
+    tcg_out_opc_br(s, OPC_BNE, TCG_TMP1, TCG_TMP0);
+
+    /* Load and test the high half tlb comparator.  */
+    if (TARGET_LONG_BITS == 64) {
+        /* delay slot */
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off + HI_OFF);
+
+        /* Load the tlb addend for the fast path. We can't do it earlier with
+           64-bit targets or we'll clobber a0 before reading the high half tlb
+           comparator.  */
+        tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
+
+        label_ptr[1] = s->code_ptr;
+        tcg_out_opc_br(s, OPC_BNE, addrh, TCG_TMP0);
+    }
+
+    /* delay slot */
+    tcg_out_opc_reg(s, OPC_ADDU, base, TCG_REG_A0, addrl);
+}
+
+static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOpIdx oi,
+                                TCGReg datalo, TCGReg datahi,
+                                TCGReg addrlo, TCGReg addrhi,
+                                void *raddr, tcg_insn_unit *label_ptr[2])
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->oi = oi;
+    label->datalo_reg = datalo;
+    label->datahi_reg = datahi;
+    label->addrlo_reg = addrlo;
+    label->addrhi_reg = addrhi;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr[0];
+    if (TARGET_LONG_BITS == 64) {
+        label->label_ptr[1] = label_ptr[1];
+    }
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOpIdx oi = l->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGReg v0;
+    int i;
+
+    /* resolve label address */
+    reloc_pc16(l->label_ptr[0], s->code_ptr);
+    if (TARGET_LONG_BITS == 64) {
+        reloc_pc16(l->label_ptr[1], s->code_ptr);
+    }
+
+    i = 1;
+    if (TARGET_LONG_BITS == 64) {
+        i = tcg_out_call_iarg_reg2(s, i, l->addrlo_reg, l->addrhi_reg);
+    } else {
+        i = tcg_out_call_iarg_reg(s, i, l->addrlo_reg);
+    }
+    i = tcg_out_call_iarg_imm(s, i, oi);
+    i = tcg_out_call_iarg_imm(s, i, (intptr_t)l->raddr);
+    tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)], false);
+    /* delay slot */
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+
+    v0 = l->datalo_reg;
+    if ((opc & MO_SIZE) == MO_64) {
+        /* We eliminated V0 from the possible output registers, so it
+           cannot be clobbered here.  So we must move V1 first.  */
+        if (MIPS_BE) {
+            tcg_out_mov(s, TCG_TYPE_I32, v0, TCG_REG_V1);
+            v0 = l->datahi_reg;
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_V1);
+        }
+    }
+
+    reloc_pc16(s->code_ptr, l->raddr);
+    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
+    /* delay slot */
+    tcg_out_mov(s, TCG_TYPE_REG, v0, TCG_REG_V0);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOpIdx oi = l->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGMemOp s_bits = opc & MO_SIZE;
+    int i;
+
+    /* resolve label address */
+    reloc_pc16(l->label_ptr[0], s->code_ptr);
+    if (TARGET_LONG_BITS == 64) {
+        reloc_pc16(l->label_ptr[1], s->code_ptr);
+    }
+
+    i = 1;
+    if (TARGET_LONG_BITS == 64) {
+        i = tcg_out_call_iarg_reg2(s, i, l->addrlo_reg, l->addrhi_reg);
+    } else {
+        i = tcg_out_call_iarg_reg(s, i, l->addrlo_reg);
+    }
+    switch (s_bits) {
+    case MO_8:
+        i = tcg_out_call_iarg_reg8(s, i, l->datalo_reg);
+        break;
+    case MO_16:
+        i = tcg_out_call_iarg_reg16(s, i, l->datalo_reg);
+        break;
+    case MO_32:
+        i = tcg_out_call_iarg_reg(s, i, l->datalo_reg);
+        break;
+    case MO_64:
+        i = tcg_out_call_iarg_reg2(s, i, l->datalo_reg, l->datahi_reg);
+        break;
+    default:
+        tcg_abort();
+    }
+    i = tcg_out_call_iarg_imm(s, i, oi);
+
+    /* Tail call to the store helper.  Thus force the return address
+       computation to take place in the return address register.  */
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (intptr_t)l->raddr);
+    i = tcg_out_call_iarg_reg(s, i, TCG_REG_RA);
+    tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)], true);
+    /* delay slot */
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+}
+#endif
+
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, TCGMemOp opc)
+{
+    switch (opc & (MO_SSIZE | MO_BSWAP)) {
+    case MO_UB:
+        tcg_out_opc_imm(s, OPC_LBU, datalo, base, 0);
+        break;
+    case MO_SB:
+        tcg_out_opc_imm(s, OPC_LB, datalo, base, 0);
+        break;
+    case MO_UW | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LHU, TCG_TMP1, base, 0);
+        tcg_out_bswap16(s, datalo, TCG_TMP1);
+        break;
+    case MO_UW:
+        tcg_out_opc_imm(s, OPC_LHU, datalo, base, 0);
+        break;
+    case MO_SW | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LHU, TCG_TMP1, base, 0);
+        tcg_out_bswap16s(s, datalo, TCG_TMP1);
+        break;
+    case MO_SW:
+        tcg_out_opc_imm(s, OPC_LH, datalo, base, 0);
+        break;
+    case MO_UL | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, 0);
+        tcg_out_bswap32(s, datalo, TCG_TMP1);
+        break;
+    case MO_UL:
+        tcg_out_opc_imm(s, OPC_LW, datalo, base, 0);
+        break;
+    case MO_Q | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, HI_OFF);
+        tcg_out_bswap32(s, datalo, TCG_TMP1);
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, LO_OFF);
+        tcg_out_bswap32(s, datahi, TCG_TMP1);
+        break;
+    case MO_Q:
+        tcg_out_opc_imm(s, OPC_LW, datalo, base, LO_OFF);
+        tcg_out_opc_imm(s, OPC_LW, datahi, base, HI_OFF);
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
+{
+    TCGReg addr_regl, addr_regh __attribute__((unused));
+    TCGReg data_regl, data_regh;
+    TCGMemOpIdx oi;
+    TCGMemOp opc;
+#if defined(CONFIG_SOFTMMU)
+    tcg_insn_unit *label_ptr[2];
+#endif
+    /* Note that we've eliminated V0 from the output registers,
+       so we won't overwrite the base register during loading.  */
+    TCGReg base = TCG_REG_V0;
+
+    data_regl = *args++;
+    data_regh = (is_64 ? *args++ : 0);
+    addr_regl = *args++;
+    addr_regh = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+
+#if defined(CONFIG_SOFTMMU)
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 1);
+    tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
+    add_qemu_ldst_label(s, 1, oi, data_regl, data_regh, addr_regl, addr_regh,
+                        s->code_ptr, label_ptr);
+#else
+    if (guest_base == 0 && data_regl != addr_regl) {
+        base = addr_regl;
+    } else if (guest_base == (int16_t)guest_base) {
+        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, guest_base);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base);
+        tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
+    }
+    tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
+#endif
+}
+
+static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, TCGMemOp opc)
+{
+    switch (opc & (MO_SIZE | MO_BSWAP)) {
+    case MO_8:
+        tcg_out_opc_imm(s, OPC_SB, datalo, base, 0);
+        break;
+
+    case MO_16 | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP1, datalo, 0xffff);
+        tcg_out_bswap16(s, TCG_TMP1, TCG_TMP1);
+        datalo = TCG_TMP1;
+        /* FALLTHRU */
+    case MO_16:
+        tcg_out_opc_imm(s, OPC_SH, datalo, base, 0);
+        break;
+
+    case MO_32 | MO_BSWAP:
+        tcg_out_bswap32(s, TCG_TMP1, datalo);
+        datalo = TCG_TMP1;
+        /* FALLTHRU */
+    case MO_32:
+        tcg_out_opc_imm(s, OPC_SW, datalo, base, 0);
+        break;
+
+    case MO_64 | MO_BSWAP:
+        tcg_out_bswap32(s, TCG_TMP1, datalo);
+        tcg_out_opc_imm(s, OPC_SW, TCG_TMP1, base, HI_OFF);
+        tcg_out_bswap32(s, TCG_TMP1, datahi);
+        tcg_out_opc_imm(s, OPC_SW, TCG_TMP1, base, LO_OFF);
+        break;
+    case MO_64:
+        tcg_out_opc_imm(s, OPC_SW, datalo, base, LO_OFF);
+        tcg_out_opc_imm(s, OPC_SW, datahi, base, HI_OFF);
+        break;
+
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
+{
+    TCGReg addr_regl, addr_regh __attribute__((unused));
+    TCGReg data_regl, data_regh, base;
+    TCGMemOpIdx oi;
+    TCGMemOp opc;
+#if defined(CONFIG_SOFTMMU)
+    tcg_insn_unit *label_ptr[2];
+#endif
+
+    data_regl = *args++;
+    data_regh = (is_64 ? *args++ : 0);
+    addr_regl = *args++;
+    addr_regh = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+
+#if defined(CONFIG_SOFTMMU)
+    /* Note that we eliminated the helper's address argument,
+       so we can reuse that for the base.  */
+    base = (TARGET_LONG_BITS == 32 ? TCG_REG_A1 : TCG_REG_A2);
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, oi, label_ptr, 0);
+    tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
+    add_qemu_ldst_label(s, 0, oi, data_regl, data_regh, addr_regl, addr_regh,
+                        s->code_ptr, label_ptr);
+#else
+    if (guest_base == 0) {
+        base = addr_regl;
+    } else {
+        base = TCG_REG_A0;
+        if (guest_base == (int16_t)guest_base) {
+            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, guest_base);
+        } else {
+            tcg_out_movi(s, TCG_TYPE_PTR, base, guest_base);
+            tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
+        }
+    }
+    tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
+#endif
+}
+
+static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                              const TCGArg *args, const int *const_args)
+{
+    MIPSInsn i1, i2;
+    TCGArg a0, a1, a2;
+    int c2;
+
+    a0 = args[0];
+    a1 = args[1];
+    a2 = args[2];
+    c2 = const_args[2];
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        {
+            TCGReg b0 = TCG_REG_ZERO;
+
+            if (a0 & ~0xffff) {
+                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
+                b0 = TCG_REG_V0;
+            }
+            if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
+                tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0,
+                             (uintptr_t)tb_ret_addr);
+                tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
+            }
+            tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
+        }
+        break;
+    case INDEX_op_goto_tb:
+        if (s->tb_jmp_insn_offset) {
+            /* direct jump method */
+            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            /* Avoid clobbering the address during retranslation.  */
+            tcg_out32(s, OPC_J | (*(uint32_t *)s->code_ptr & 0x3ffffff));
+        } else {
+            /* indirect jump method */
+            tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
+                       (uintptr_t)(s->tb_jmp_target_addr + a0));
+            tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
+        }
+        tcg_out_nop(s);
+        s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
+        break;
+    case INDEX_op_br:
+        tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO,
+                       arg_label(a0));
+        break;
+
+    case INDEX_op_ld8u_i32:
+        i1 = OPC_LBU;
+        goto do_ldst;
+    case INDEX_op_ld8s_i32:
+        i1 = OPC_LB;
+        goto do_ldst;
+    case INDEX_op_ld16u_i32:
+        i1 = OPC_LHU;
+        goto do_ldst;
+    case INDEX_op_ld16s_i32:
+        i1 = OPC_LH;
+        goto do_ldst;
+    case INDEX_op_ld_i32:
+        i1 = OPC_LW;
+        goto do_ldst;
+    case INDEX_op_st8_i32:
+        i1 = OPC_SB;
+        goto do_ldst;
+    case INDEX_op_st16_i32:
+        i1 = OPC_SH;
+        goto do_ldst;
+    case INDEX_op_st_i32:
+        i1 = OPC_SW;
+    do_ldst:
+        tcg_out_ldst(s, i1, a0, a1, a2);
+        break;
+
+    case INDEX_op_add_i32:
+        i1 = OPC_ADDU, i2 = OPC_ADDIU;
+        goto do_binary;
+    case INDEX_op_or_i32:
+        i1 = OPC_OR, i2 = OPC_ORI;
+        goto do_binary;
+    case INDEX_op_xor_i32:
+        i1 = OPC_XOR, i2 = OPC_XORI;
+    do_binary:
+        if (c2) {
+            tcg_out_opc_imm(s, i2, a0, a1, a2);
+            break;
+        }
+    do_binaryv:
+        tcg_out_opc_reg(s, i1, a0, a1, a2);
+        break;
+
+    case INDEX_op_sub_i32:
+        if (c2) {
+            tcg_out_opc_imm(s, OPC_ADDIU, a0, a1, -a2);
+            break;
+        }
+        i1 = OPC_SUBU;
+        goto do_binary;
+    case INDEX_op_and_i32:
+        if (c2 && a2 != (uint16_t)a2) {
+            int msb = ctz32(~a2) - 1;
+            tcg_debug_assert(use_mips32r2_instructions);
+            tcg_debug_assert(is_p2m1(a2));
+            tcg_out_opc_bf(s, OPC_EXT, a0, a1, msb, 0);
+            break;
+        }
+        i1 = OPC_AND, i2 = OPC_ANDI;
+        goto do_binary;
+    case INDEX_op_nor_i32:
+        i1 = OPC_NOR;
+        goto do_binaryv;
+
+    case INDEX_op_mul_i32:
+        if (use_mips32_instructions) {
+            tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_MULT, i2 = OPC_MFLO;
+        goto do_hilo1;
+    case INDEX_op_mulsh_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MUH, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_MULT, i2 = OPC_MFHI;
+        goto do_hilo1;
+    case INDEX_op_muluh_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MUHU, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_MULTU, i2 = OPC_MFHI;
+        goto do_hilo1;
+    case INDEX_op_div_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_DIV_R6, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_DIV, i2 = OPC_MFLO;
+        goto do_hilo1;
+    case INDEX_op_divu_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_DIVU_R6, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_DIVU, i2 = OPC_MFLO;
+        goto do_hilo1;
+    case INDEX_op_rem_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MOD, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_DIV, i2 = OPC_MFHI;
+        goto do_hilo1;
+    case INDEX_op_remu_i32:
+        if (use_mips32r6_instructions) {
+            tcg_out_opc_reg(s, OPC_MODU, a0, a1, a2);
+            break;
+        }
+        i1 = OPC_DIVU, i2 = OPC_MFHI;
+    do_hilo1:
+        tcg_out_opc_reg(s, i1, 0, a1, a2);
+        tcg_out_opc_reg(s, i2, a0, 0, 0);
+        break;
+
+    case INDEX_op_muls2_i32:
+        i1 = OPC_MULT;
+        goto do_hilo2;
+    case INDEX_op_mulu2_i32:
+        i1 = OPC_MULTU;
+    do_hilo2:
+        tcg_out_opc_reg(s, i1, 0, a2, args[3]);
+        tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0);
+        tcg_out_opc_reg(s, OPC_MFHI, a1, 0, 0);
+        break;
+
+    case INDEX_op_not_i32:
+        i1 = OPC_NOR;
+        goto do_unary;
+    case INDEX_op_bswap16_i32:
+        i1 = OPC_WSBH;
+        goto do_unary;
+    case INDEX_op_ext8s_i32:
+        i1 = OPC_SEB;
+        goto do_unary;
+    case INDEX_op_ext16s_i32:
+        i1 = OPC_SEH;
+    do_unary:
+        tcg_out_opc_reg(s, i1, a0, TCG_REG_ZERO, a1);
+        break;
+
+    case INDEX_op_sar_i32:
+        i1 = OPC_SRAV, i2 = OPC_SRA;
+        goto do_shift;
+    case INDEX_op_shl_i32:
+        i1 = OPC_SLLV, i2 = OPC_SLL;
+        goto do_shift;
+    case INDEX_op_shr_i32:
+        i1 = OPC_SRLV, i2 = OPC_SRL;
+        goto do_shift;
+    case INDEX_op_rotr_i32:
+        i1 = OPC_ROTRV, i2 = OPC_ROTR;
+    do_shift:
+        if (c2) {
+            tcg_out_opc_sa(s, i2, a0, a1, a2);
+        } else {
+            tcg_out_opc_reg(s, i1, a0, a2, a1);
+        }
+        break;
+    case INDEX_op_rotl_i32:
+        if (c2) {
+            tcg_out_opc_sa(s, OPC_ROTR, a0, a1, 32 - a2);
+        } else {
+            tcg_out_opc_reg(s, OPC_SUBU, TCG_TMP0, TCG_REG_ZERO, a2);
+            tcg_out_opc_reg(s, OPC_ROTRV, a0, TCG_TMP0, a1);
+        }
+        break;
+
+    case INDEX_op_bswap32_i32:
+        tcg_out_opc_reg(s, OPC_WSBH, a0, 0, a1);
+        tcg_out_opc_sa(s, OPC_ROTR, a0, a0, 16);
+        break;
+
+    case INDEX_op_deposit_i32:
+        tcg_out_opc_bf(s, OPC_INS, a0, a2, args[3] + args[4] - 1, args[3]);
+        break;
+
+    case INDEX_op_brcond_i32:
+        tcg_out_brcond(s, a2, a0, a1, arg_label(args[3]));
+        break;
+    case INDEX_op_brcond2_i32:
+        tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], arg_label(args[5]));
+        break;
+
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], a0, a1, a2, args[3], args[4]);
+        break;
+
+    case INDEX_op_setcond_i32:
+        tcg_out_setcond(s, args[3], a0, a1, a2);
+        break;
+    case INDEX_op_setcond2_i32:
+        tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]);
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, args, false);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args, true);
+        break;
+    case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, args, false);
+        break;
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args, true);
+        break;
+
+    case INDEX_op_add2_i32:
+        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5],
+                        const_args[4], const_args[5], false);
+        break;
+    case INDEX_op_sub2_i32:
+        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5],
+                        const_args[4], const_args[5], true);
+        break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+}
+
+static const TCGTargetOpDef mips_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "rZ", "r" } },
+    { INDEX_op_st16_i32, { "rZ", "r" } },
+    { INDEX_op_st_i32, { "rZ", "r" } },
+
+    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
+#if !use_mips32r6_instructions
+    { INDEX_op_muls2_i32, { "r", "r", "rZ", "rZ" } },
+    { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
+#endif
+    { INDEX_op_mulsh_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_muluh_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_divu_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_rem_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_remu_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rN" } },
+
+    { INDEX_op_and_i32, { "r", "rZ", "rIK" } },
+    { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_not_i32, { "r", "rZ" } },
+    { INDEX_op_or_i32, { "r", "rZ", "rIZ" } },
+    { INDEX_op_xor_i32, { "r", "rZ", "rIZ" } },
+
+    { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
+
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
+
+    { INDEX_op_ext8s_i32, { "r", "rZ" } },
+    { INDEX_op_ext16s_i32, { "r", "rZ" } },
+
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+
+    { INDEX_op_brcond_i32, { "rZ", "rZ" } },
+#if use_mips32r6_instructions
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
+#else
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
+#endif
+    { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
+
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rN", "rN" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rN", "rN" } },
+    { INDEX_op_brcond2_i32, { "rZ", "rZ", "rZ", "rZ" } },
+
+#if TARGET_LONG_BITS == 32
+    { INDEX_op_qemu_ld_i32, { "L", "lZ" } },
+    { INDEX_op_qemu_st_i32, { "SZ", "SZ" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "lZ" } },
+    { INDEX_op_qemu_st_i64, { "SZ", "SZ", "SZ" } },
+#else
+    { INDEX_op_qemu_ld_i32, { "L", "lZ", "lZ" } },
+    { INDEX_op_qemu_st_i32, { "SZ", "SZ", "SZ" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "lZ", "lZ" } },
+    { INDEX_op_qemu_st_i64, { "SZ", "SZ", "SZ", "SZ" } },
+#endif
+    { -1 },
+};
+
+static int tcg_target_callee_save_regs[] = {
+    TCG_REG_S0,       /* used for the global env (TCG_AREG0) */
+    TCG_REG_S1,
+    TCG_REG_S2,
+    TCG_REG_S3,
+    TCG_REG_S4,
+    TCG_REG_S5,
+    TCG_REG_S6,
+    TCG_REG_S7,
+    TCG_REG_S8,
+    TCG_REG_RA,       /* should be last for ABI compliance */
+};
+
+/* The Linux kernel doesn't provide any information about the available
+   instruction set. Probe it using a signal handler. */
+
+
+#ifndef use_movnz_instructions
+bool use_movnz_instructions = false;
+#endif
+
+#ifndef use_mips32_instructions
+bool use_mips32_instructions = false;
+#endif
+
+#ifndef use_mips32r2_instructions
+bool use_mips32r2_instructions = false;
+#endif
+
+static volatile sig_atomic_t got_sigill;
+
+static void sigill_handler(int signo, siginfo_t *si, void *data)
+{
+    /* Skip the faulty instruction */
+    ucontext_t *uc = (ucontext_t *)data;
+    uc->uc_mcontext.pc += 4;
+
+    got_sigill = 1;
+}
+
+static void tcg_target_detect_isa(void)
+{
+    struct sigaction sa_old, sa_new;
+
+    memset(&sa_new, 0, sizeof(sa_new));
+    sa_new.sa_flags = SA_SIGINFO;
+    sa_new.sa_sigaction = sigill_handler;
+    sigaction(SIGILL, &sa_new, &sa_old);
+
+    /* Probe for movn/movz, necessary to implement movcond. */
+#ifndef use_movnz_instructions
+    got_sigill = 0;
+    asm volatile(".set push\n"
+                 ".set mips32\n"
+                 "movn $zero, $zero, $zero\n"
+                 "movz $zero, $zero, $zero\n"
+                 ".set pop\n"
+                 : : : );
+    use_movnz_instructions = !got_sigill;
+#endif
+
+    /* Probe for MIPS32 instructions. As no subsetting is allowed
+       by the specification, it is only necessary to probe for one
+       of the instructions. */
+#ifndef use_mips32_instructions
+    got_sigill = 0;
+    asm volatile(".set push\n"
+                 ".set mips32\n"
+                 "mul $zero, $zero\n"
+                 ".set pop\n"
+                 : : : );
+    use_mips32_instructions = !got_sigill;
+#endif
+
+    /* Probe for MIPS32r2 instructions if MIPS32 instructions are
+       available. As no subsetting is allowed by the specification,
+       it is only necessary to probe for one of the instructions. */
+#ifndef use_mips32r2_instructions
+    if (use_mips32_instructions) {
+        got_sigill = 0;
+        asm volatile(".set push\n"
+                     ".set mips32r2\n"
+                     "seb $zero, $zero\n"
+                     ".set pop\n"
+                     : : : );
+        use_mips32r2_instructions = !got_sigill;
+    }
+#endif
+
+    sigaction(SIGILL, &sa_old, NULL);
+}
+
+/* Generate global QEMU prologue and epilogue code */
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    int i, frame_size;
+
+    /* reserve some stack space, also for TCG temps. */
+    frame_size = ARRAY_SIZE(tcg_target_callee_save_regs) * 4
+                 + TCG_STATIC_CALL_ARGS_SIZE
+                 + CPU_TEMP_BUF_NLONGS * sizeof(long);
+    frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
+                 ~(TCG_TARGET_STACK_ALIGN - 1);
+    tcg_set_frame(s, TCG_REG_SP, ARRAY_SIZE(tcg_target_callee_save_regs) * 4
+                  + TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+
+    /* TB prologue */
+    tcg_out_addi(s, TCG_REG_SP, -frame_size);
+    for(i = 0 ; i < ARRAY_SIZE(tcg_target_callee_save_regs) ; i++) {
+        tcg_out_st(s, TCG_TYPE_I32, tcg_target_callee_save_regs[i],
+                   TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE + i * 4);
+    }
+
+    /* Call generated code */
+    tcg_out_opc_reg(s, OPC_JR, 0, tcg_target_call_iarg_regs[1], 0);
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+    tb_ret_addr = s->code_ptr;
+
+    /* TB epilogue */
+    for(i = 0 ; i < ARRAY_SIZE(tcg_target_callee_save_regs) ; i++) {
+        tcg_out_ld(s, TCG_TYPE_I32, tcg_target_callee_save_regs[i],
+                   TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE + i * 4);
+    }
+
+    tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_RA, 0);
+    tcg_out_addi(s, TCG_REG_SP, frame_size);
+}
+
+static void tcg_target_init(TCGContext *s)
+{
+    tcg_target_detect_isa();
+    tcg_regset_set(tcg_target_available_regs[TCG_TYPE_I32], 0xffffffff);
+    tcg_regset_set(tcg_target_call_clobber_regs,
+                   (1 << TCG_REG_V0) |
+                   (1 << TCG_REG_V1) |
+                   (1 << TCG_REG_A0) |
+                   (1 << TCG_REG_A1) |
+                   (1 << TCG_REG_A2) |
+                   (1 << TCG_REG_A3) |
+                   (1 << TCG_REG_T0) |
+                   (1 << TCG_REG_T1) |
+                   (1 << TCG_REG_T2) |
+                   (1 << TCG_REG_T3) |
+                   (1 << TCG_REG_T4) |
+                   (1 << TCG_REG_T5) |
+                   (1 << TCG_REG_T6) |
+                   (1 << TCG_REG_T7) |
+                   (1 << TCG_REG_T8) |
+                   (1 << TCG_REG_T9));
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); /* zero register */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_K0);   /* kernel use only */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_K1);   /* kernel use only */
+    tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);     /* internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_TMP1);     /* internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);   /* return address */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);   /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);   /* global pointer */
+
+    tcg_add_target_add_op_defs(mips_op_defs);
+}
+
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
+{
+    atomic_set((uint32_t *)jmp_addr, deposit32(OPC_J, 0, 26, addr >> 2));
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
diff --git a/tcg/optimize.c b/tcg/optimize.c
index bbc86d0..cffe89b 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -23,26 +23,17 @@
  * THE SOFTWARE.
  */
 
-#include "config.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "exec/cpu-common.h"
 #include "tcg-op.h"
 
 #define CASE_OP_32_64(x)                        \
         glue(glue(case INDEX_op_, x), _i32):    \
         glue(glue(case INDEX_op_, x), _i64)
 
-typedef enum {
-    TCG_TEMP_UNDEF = 0,
-    TCG_TEMP_CONST,
-    TCG_TEMP_COPY,
-} tcg_temp_state;
-
 struct tcg_temp_info {
-    tcg_temp_state state;
+    bool is_const;
     uint16_t prev_copy;
     uint16_t next_copy;
     tcg_target_ulong val;
@@ -50,61 +41,44 @@
 };
 
 static struct tcg_temp_info temps[TCG_MAX_TEMPS];
+static TCGTempSet temps_used;
 
-/* Reset TEMP's state to TCG_TEMP_UNDEF.  If TEMP only had one copy, remove
-   the copy flag from the left temp.  */
-static void reset_temp(TCGArg temp)
+static inline bool temp_is_const(TCGArg arg)
 {
-    if (temps[temp].state == TCG_TEMP_COPY) {
-        if (temps[temp].prev_copy == temps[temp].next_copy) {
-            temps[temps[temp].next_copy].state = TCG_TEMP_UNDEF;
-        } else {
-            temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-            temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-        }
-    }
-    temps[temp].state = TCG_TEMP_UNDEF;
-    temps[temp].mask = -1;
+    return temps[arg].is_const;
 }
 
-static TCGOp *insert_op_before(TCGContext *s, TCGOp *old_op,
-                                TCGOpcode opc, int nargs)
+static inline bool temp_is_copy(TCGArg arg)
 {
-    int oi = s->gen_next_op_idx;
-    int pi = s->gen_next_parm_idx;
-    int prev = old_op->prev;
-    int next = old_op - s->gen_op_buf;
-    TCGOp *new_op;
+    return temps[arg].next_copy != arg;
+}
 
-    tcg_debug_assert(oi < OPC_BUF_SIZE);
-    tcg_debug_assert(pi + nargs <= OPPARAM_BUF_SIZE);
-    s->gen_next_op_idx = oi + 1;
-    s->gen_next_parm_idx = pi + nargs;
-
-    new_op = &s->gen_op_buf[oi];
-    *new_op = (TCGOp){
-        .opc = opc,
-        .args = pi,
-        .prev = prev,
-        .next = next
-    };
-    if (prev >= 0) {
-        s->gen_op_buf[prev].next = oi;
-    } else {
-        s->gen_first_op_idx = oi;
-    }
-    old_op->prev = oi;
-
-    return new_op;
+/* Reset TEMP's state, possibly removing the temp for the list of copies.  */
+static void reset_temp(TCGArg temp)
+{
+    temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
+    temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
+    temps[temp].next_copy = temp;
+    temps[temp].prev_copy = temp;
+    temps[temp].is_const = false;
+    temps[temp].mask = -1;
 }
 
 /* Reset all temporaries, given that there are NB_TEMPS of them.  */
 static void reset_all_temps(int nb_temps)
 {
-    int i;
-    for (i = 0; i < nb_temps; i++) {
-        temps[i].state = TCG_TEMP_UNDEF;
-        temps[i].mask = -1;
+    bitmap_zero(temps_used.l, nb_temps);
+}
+
+/* Initialize and activate a temporary.  */
+static void init_temp_info(TCGArg temp)
+{
+    if (!test_bit(temp, temps_used.l)) {
+        temps[temp].next_copy = temp;
+        temps[temp].prev_copy = temp;
+        temps[temp].is_const = false;
+        temps[temp].mask = -1;
+        set_bit(temp, temps_used.l);
     }
 }
 
@@ -179,8 +153,7 @@
         return true;
     }
 
-    if (temps[arg1].state != TCG_TEMP_COPY
-        || temps[arg2].state != TCG_TEMP_COPY) {
+    if (!temp_is_copy(arg1) || !temp_is_copy(arg2)) {
         return false;
     }
 
@@ -193,10 +166,37 @@
     return false;
 }
 
-static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
-                            TCGOpcode old_op, TCGArg dst, TCGArg src)
+static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg *args,
+                             TCGArg dst, TCGArg val)
 {
-    TCGOpcode new_op = op_to_mov(old_op);
+    TCGOpcode new_op = op_to_movi(op->opc);
+    tcg_target_ulong mask;
+
+    op->opc = new_op;
+
+    reset_temp(dst);
+    temps[dst].is_const = true;
+    temps[dst].val = val;
+    mask = val;
+    if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_movi_i32) {
+        /* High bits of the destination are now garbage.  */
+        mask |= ~0xffffffffull;
+    }
+    temps[dst].mask = mask;
+
+    args[0] = dst;
+    args[1] = val;
+}
+
+static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
+                            TCGArg dst, TCGArg src)
+{
+    if (temps_are_copies(dst, src)) {
+        tcg_op_remove(s, op);
+        return;
+    }
+
+    TCGOpcode new_op = op_to_mov(op->opc);
     tcg_target_ulong mask;
 
     op->opc = new_op;
@@ -209,47 +209,19 @@
     }
     temps[dst].mask = mask;
 
-    assert(temps[src].state != TCG_TEMP_CONST);
-
     if (s->temps[src].type == s->temps[dst].type) {
-        if (temps[src].state != TCG_TEMP_COPY) {
-            temps[src].state = TCG_TEMP_COPY;
-            temps[src].next_copy = src;
-            temps[src].prev_copy = src;
-        }
-        temps[dst].state = TCG_TEMP_COPY;
         temps[dst].next_copy = temps[src].next_copy;
         temps[dst].prev_copy = src;
         temps[temps[dst].next_copy].prev_copy = dst;
         temps[src].next_copy = dst;
+        temps[dst].is_const = temps[src].is_const;
+        temps[dst].val = temps[src].val;
     }
 
     args[0] = dst;
     args[1] = src;
 }
 
-static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg *args,
-                             TCGOpcode old_op, TCGArg dst, TCGArg val)
-{
-    TCGOpcode new_op = op_to_movi(old_op);
-    tcg_target_ulong mask;
-
-    op->opc = new_op;
-
-    reset_temp(dst);
-    temps[dst].state = TCG_TEMP_CONST;
-    temps[dst].val = val;
-    mask = val;
-    if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_movi_i32) {
-        /* High bits of the destination are now garbage.  */
-        mask |= ~0xffffffffull;
-    }
-    temps[dst].mask = mask;
-
-    args[0] = dst;
-    args[1] = val;
-}
-
 static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
 {
     uint64_t l64, h64;
@@ -282,7 +254,6 @@
     case INDEX_op_shr_i32:
         return (uint32_t)x >> (y & 31);
 
-    case INDEX_op_trunc_shr_i32:
     case INDEX_op_shr_i64:
         return (uint64_t)x >> (y & 63);
 
@@ -337,12 +308,18 @@
     CASE_OP_32_64(ext16u):
         return (uint16_t)x;
 
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         return (int32_t)x;
 
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext32u_i64:
         return (uint32_t)x;
 
+    case INDEX_op_extrh_i64_i32:
+        return (uint64_t)x >> 32;
+
     case INDEX_op_muluh_i32:
         return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
     case INDEX_op_mulsh_i32:
@@ -385,7 +362,7 @@
 {
     TCGArg res = do_constant_folding_2(op, x, y);
     if (op_bits(op) == 32) {
-        res &= 0xffffffff;
+        res = (int32_t)res;
     }
     return res;
 }
@@ -471,7 +448,7 @@
 static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
                                        TCGArg y, TCGCond c)
 {
-    if (temps[x].state == TCG_TEMP_CONST && temps[y].state == TCG_TEMP_CONST) {
+    if (temp_is_const(x) && temp_is_const(y)) {
         switch (op_bits(op)) {
         case 32:
             return do_constant_folding_cond_32(temps[x].val, temps[y].val, c);
@@ -482,7 +459,7 @@
         }
     } else if (temps_are_copies(x, y)) {
         return do_constant_folding_cond_eq(c);
-    } else if (temps[y].state == TCG_TEMP_CONST && temps[y].val == 0) {
+    } else if (temp_is_const(y) && temps[y].val == 0) {
         switch (c) {
         case TCG_COND_LTU:
             return 0;
@@ -503,12 +480,10 @@
     TCGArg al = p1[0], ah = p1[1];
     TCGArg bl = p2[0], bh = p2[1];
 
-    if (temps[bl].state == TCG_TEMP_CONST
-        && temps[bh].state == TCG_TEMP_CONST) {
+    if (temp_is_const(bl) && temp_is_const(bh)) {
         uint64_t b = ((uint64_t)temps[bh].val << 32) | (uint32_t)temps[bl].val;
 
-        if (temps[al].state == TCG_TEMP_CONST
-            && temps[ah].state == TCG_TEMP_CONST) {
+        if (temp_is_const(al) && temp_is_const(ah)) {
             uint64_t a;
             a = ((uint64_t)temps[ah].val << 32) | (uint32_t)temps[al].val;
             return do_constant_folding_cond_64(a, b, c);
@@ -534,8 +509,8 @@
 {
     TCGArg a1 = *p1, a2 = *p2;
     int sum = 0;
-    sum += temps[a1].state == TCG_TEMP_CONST;
-    sum -= temps[a2].state == TCG_TEMP_CONST;
+    sum += temp_is_const(a1);
+    sum -= temp_is_const(a2);
 
     /* Prefer the constant in second argument, and then the form
        op a, a, b, which is better handled on non-RISC hosts. */
@@ -550,10 +525,10 @@
 static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
 {
     int sum = 0;
-    sum += temps[p1[0]].state == TCG_TEMP_CONST;
-    sum += temps[p1[1]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[0]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[1]].state == TCG_TEMP_CONST;
+    sum += temp_is_const(p1[0]);
+    sum += temp_is_const(p1[1]);
+    sum -= temp_is_const(p2[0]);
+    sum -= temp_is_const(p2[1]);
     if (sum > 0) {
         TCGArg t;
         t = p1[0], p1[0] = p2[0], p2[0] = t;
@@ -564,7 +539,7 @@
 }
 
 /* Propagate constants and copies, fold constant expressions. */
-static void tcg_constant_folding(TCGContext *s)
+void tcg_optimize(TCGContext *s)
 {
     int oi, oi_next, nb_temps, nb_globals;
 
@@ -577,7 +552,7 @@
     nb_globals = s->nb_globals;
     reset_all_temps(nb_temps);
 
-    for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
+    for (oi = s->gen_op_buf[0].next; oi != 0; oi = oi_next) {
         tcg_target_ulong mask, partmask, affected;
         int nb_oargs, nb_iargs, i;
         TCGArg tmp;
@@ -588,17 +563,29 @@
         const TCGOpDef *def = &tcg_op_defs[opc];
 
         oi_next = op->next;
+
+        /* Count the arguments, and initialize the temps that are
+           going to be used */
         if (opc == INDEX_op_call) {
             nb_oargs = op->callo;
             nb_iargs = op->calli;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                tmp = args[i];
+                if (tmp != TCG_CALL_DUMMY_ARG) {
+                    init_temp_info(tmp);
+                }
+            }
         } else {
             nb_oargs = def->nb_oargs;
             nb_iargs = def->nb_iargs;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                init_temp_info(args[i]);
+            }
         }
 
         /* Do copy propagation */
         for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-            if (temps[args[i]].state == TCG_TEMP_COPY) {
+            if (temp_is_copy(args[i])) {
                 args[i] = find_better_copy(s, args[i]);
             }
         }
@@ -668,9 +655,8 @@
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
-                tcg_opt_gen_movi(s, op, args, opc, args[0], 0);
+            if (temp_is_const(args[1]) && temps[args[1]].val == 0) {
+                tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
             break;
@@ -679,7 +665,7 @@
                 TCGOpcode neg_op;
                 bool have_neg;
 
-                if (temps[args[2]].state == TCG_TEMP_CONST) {
+                if (temp_is_const(args[2])) {
                     /* Proceed with possible constant folding. */
                     break;
                 }
@@ -693,8 +679,7 @@
                 if (!have_neg) {
                     break;
                 }
-                if (temps[args[1]].state == TCG_TEMP_CONST
-                    && temps[args[1]].val == 0) {
+                if (temp_is_const(args[1]) && temps[args[1]].val == 0) {
                     op->opc = neg_op;
                     reset_temp(args[0]);
                     args[1] = args[2];
@@ -704,34 +689,30 @@
             break;
         CASE_OP_32_64(xor):
         CASE_OP_32_64(nand):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == -1) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(nor):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == 0) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(andc):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == -1) {
+            if (!temp_is_const(args[2])
+                && temp_is_const(args[1]) && temps[args[1]].val == -1) {
                 i = 2;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
+            if (!temp_is_const(args[2])
+                && temp_is_const(args[1]) && temps[args[1]].val == 0) {
                 i = 2;
                 goto try_not;
             }
@@ -772,28 +753,21 @@
         CASE_OP_32_64(or):
         CASE_OP_32_64(xor):
         CASE_OP_32_64(andc):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
-                goto do_mov3;
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == 0) {
+                tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+                continue;
             }
             break;
         CASE_OP_32_64(and):
         CASE_OP_32_64(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
-                goto do_mov3;
+            if (!temp_is_const(args[1])
+                && temp_is_const(args[2]) && temps[args[2]].val == -1) {
+                tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+                continue;
             }
             break;
-        do_mov3:
-            if (temps_are_copies(args[0], args[1])) {
-                tcg_op_remove(s, op);
-            } else {
-                tcg_opt_gen_mov(s, op, args, opc, args[0], args[1]);
-            }
-            continue;
         default:
             break;
         }
@@ -827,17 +801,26 @@
 
         CASE_OP_32_64(and):
             mask = temps[args[2]].mask;
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
         and_const:
                 affected = temps[args[1]].mask & ~mask;
             }
             mask = temps[args[1]].mask & mask;
             break;
 
+        case INDEX_op_ext_i32_i64:
+            if ((temps[args[1]].mask & 0x80000000) != 0) {
+                break;
+            }
+        case INDEX_op_extu_i32_i64:
+            /* We do not compute affected as it is a size changing op.  */
+            mask = (uint32_t)temps[args[1]].mask;
+            break;
+
         CASE_OP_32_64(andc):
             /* Known-zeros does not imply known-ones.  Therefore unless
                args[2] is constant, we can't infer anything from it.  */
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 mask = ~temps[args[2]].mask;
                 goto and_const;
             }
@@ -846,37 +829,40 @@
             break;
 
         case INDEX_op_sar_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 31;
                 mask = (int32_t)temps[args[1]].mask >> tmp;
             }
             break;
         case INDEX_op_sar_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 63;
                 mask = (int64_t)temps[args[1]].mask >> tmp;
             }
             break;
 
         case INDEX_op_shr_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 31;
                 mask = (uint32_t)temps[args[1]].mask >> tmp;
             }
             break;
         case INDEX_op_shr_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & 63;
                 mask = (uint64_t)temps[args[1]].mask >> tmp;
             }
             break;
 
-        case INDEX_op_trunc_shr_i32:
-            mask = (uint64_t)temps[args[1]].mask >> args[2];
+        case INDEX_op_extrl_i64_i32:
+            mask = (uint32_t)temps[args[1]].mask;
+            break;
+        case INDEX_op_extrh_i64_i32:
+            mask = (uint64_t)temps[args[1]].mask >> 32;
             break;
 
         CASE_OP_32_64(shl):
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2])) {
                 tmp = temps[args[2]].val & (TCG_TARGET_REG_BITS - 1);
                 mask = temps[args[1]].mask << tmp;
             }
@@ -941,20 +927,13 @@
         }
 
         if (partmask == 0) {
-            assert(nb_oargs == 1);
-            tcg_opt_gen_movi(s, op, args, opc, args[0], 0);
+            tcg_debug_assert(nb_oargs == 1);
+            tcg_opt_gen_movi(s, op, args, args[0], 0);
             continue;
         }
         if (affected == 0) {
-            assert(nb_oargs == 1);
-            if (temps_are_copies(args[0], args[1])) {
-                tcg_op_remove(s, op);
-            } else if (temps[args[1]].state != TCG_TEMP_CONST) {
-                tcg_opt_gen_mov(s, op, args, opc, args[0], args[1]);
-            } else {
-                tcg_opt_gen_movi(s, op, args, opc,
-                                 args[0], temps[args[1]].val);
-            }
+            tcg_debug_assert(nb_oargs == 1);
+            tcg_opt_gen_mov(s, op, args, args[0], args[1]);
             continue;
         }
 
@@ -964,9 +943,8 @@
         CASE_OP_32_64(mul):
         CASE_OP_32_64(muluh):
         CASE_OP_32_64(mulsh):
-            if ((temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0)) {
-                tcg_opt_gen_movi(s, op, args, opc, args[0], 0);
+            if ((temp_is_const(args[2]) && temps[args[2]].val == 0)) {
+                tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
             break;
@@ -979,11 +957,7 @@
         CASE_OP_32_64(or):
         CASE_OP_32_64(and):
             if (temps_are_copies(args[1], args[2])) {
-                if (temps_are_copies(args[0], args[1])) {
-                    tcg_op_remove(s, op);
-                } else {
-                    tcg_opt_gen_mov(s, op, args, opc, args[0], args[1]);
-                }
+                tcg_opt_gen_mov(s, op, args, args[0], args[1]);
                 continue;
             }
             break;
@@ -997,7 +971,7 @@
         CASE_OP_32_64(sub):
         CASE_OP_32_64(xor):
             if (temps_are_copies(args[1], args[2])) {
-                tcg_opt_gen_movi(s, op, args, opc, args[0], 0);
+                tcg_opt_gen_movi(s, op, args, args[0], 0);
                 continue;
             }
             break;
@@ -1010,20 +984,10 @@
            allocator where needed and possible.  Also detect copies. */
         switch (opc) {
         CASE_OP_32_64(mov):
-            if (temps_are_copies(args[0], args[1])) {
-                tcg_op_remove(s, op);
-                break;
-            }
-            if (temps[args[1]].state != TCG_TEMP_CONST) {
-                tcg_opt_gen_mov(s, op, args, opc, args[0], args[1]);
-                break;
-            }
-            /* Source argument is constant.  Rewrite the operation and
-               let movi case handle it. */
-            args[1] = temps[args[1]].val;
-            /* fallthrough */
+            tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+            break;
         CASE_OP_32_64(movi):
-            tcg_opt_gen_movi(s, op, args, opc, args[0], args[1]);
+            tcg_opt_gen_movi(s, op, args, args[0], args[1]);
             break;
 
         CASE_OP_32_64(not):
@@ -1034,17 +998,13 @@
         CASE_OP_32_64(ext16u):
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
+        case INDEX_op_ext_i32_i64:
+        case INDEX_op_extu_i32_i64:
+        case INDEX_op_extrl_i64_i32:
+        case INDEX_op_extrh_i64_i32:
+            if (temp_is_const(args[1])) {
                 tmp = do_constant_folding(opc, temps[args[1]].val, 0);
-                tcg_opt_gen_movi(s, op, args, opc, args[0], tmp);
-                break;
-            }
-            goto do_default;
-
-        case INDEX_op_trunc_shr_i32:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = do_constant_folding(opc, temps[args[1]].val, args[2]);
-                tcg_opt_gen_movi(s, op, args, opc, args[0], tmp);
+                tcg_opt_gen_movi(s, op, args, args[0], tmp);
                 break;
             }
             goto do_default;
@@ -1071,21 +1031,19 @@
         CASE_OP_32_64(divu):
         CASE_OP_32_64(rem):
         CASE_OP_32_64(remu):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[1]) && temp_is_const(args[2])) {
                 tmp = do_constant_folding(opc, temps[args[1]].val,
                                           temps[args[2]].val);
-                tcg_opt_gen_movi(s, op, args, opc, args[0], tmp);
+                tcg_opt_gen_movi(s, op, args, args[0], tmp);
                 break;
             }
             goto do_default;
 
         CASE_OP_32_64(deposit):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[1]) && temp_is_const(args[2])) {
                 tmp = deposit64(temps[args[1]].val, args[3], args[4],
                                 temps[args[2]].val);
-                tcg_opt_gen_movi(s, op, args, opc, args[0], tmp);
+                tcg_opt_gen_movi(s, op, args, args[0], tmp);
                 break;
             }
             goto do_default;
@@ -1093,7 +1051,7 @@
         CASE_OP_32_64(setcond):
             tmp = do_constant_folding_cond(opc, args[1], args[2], args[3]);
             if (tmp != 2) {
-                tcg_opt_gen_movi(s, op, args, opc, args[0], tmp);
+                tcg_opt_gen_movi(s, op, args, args[0], tmp);
                 break;
             }
             goto do_default;
@@ -1115,24 +1073,15 @@
         CASE_OP_32_64(movcond):
             tmp = do_constant_folding_cond(opc, args[1], args[2], args[5]);
             if (tmp != 2) {
-                if (temps_are_copies(args[0], args[4-tmp])) {
-                    tcg_op_remove(s, op);
-                } else if (temps[args[4-tmp]].state == TCG_TEMP_CONST) {
-                    tcg_opt_gen_movi(s, op, args, opc,
-                                     args[0], temps[args[4-tmp]].val);
-                } else {
-                    tcg_opt_gen_mov(s, op, args, opc, args[0], args[4-tmp]);
-                }
+                tcg_opt_gen_mov(s, op, args, args[0], args[4-tmp]);
                 break;
             }
             goto do_default;
 
         case INDEX_op_add2_i32:
         case INDEX_op_sub2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST
-                && temps[args[4]].state == TCG_TEMP_CONST
-                && temps[args[5]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2]) && temp_is_const(args[3])
+                && temp_is_const(args[4]) && temp_is_const(args[5])) {
                 uint32_t al = temps[args[2]].val;
                 uint32_t ah = temps[args[3]].val;
                 uint32_t bl = temps[args[4]].val;
@@ -1140,7 +1089,7 @@
                 uint64_t a = ((uint64_t)ah << 32) | al;
                 uint64_t b = ((uint64_t)bh << 32) | bl;
                 TCGArg rl, rh;
-                TCGOp *op2 = insert_op_before(s, op, INDEX_op_movi_i32, 2);
+                TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_movi_i32, 2);
                 TCGArg *args2 = &s->gen_opparam_buf[op2->args];
 
                 if (opc == INDEX_op_add2_i32) {
@@ -1151,8 +1100,8 @@
 
                 rl = args[0];
                 rh = args[1];
-                tcg_opt_gen_movi(s, op, args, opc, rl, (uint32_t)a);
-                tcg_opt_gen_movi(s, op2, args2, opc, rh, (uint32_t)(a >> 32));
+                tcg_opt_gen_movi(s, op, args, rl, (int32_t)a);
+                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(a >> 32));
 
                 /* We've done all we need to do with the movi.  Skip it.  */
                 oi_next = op2->next;
@@ -1161,19 +1110,18 @@
             goto do_default;
 
         case INDEX_op_mulu2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST) {
+            if (temp_is_const(args[2]) && temp_is_const(args[3])) {
                 uint32_t a = temps[args[2]].val;
                 uint32_t b = temps[args[3]].val;
                 uint64_t r = (uint64_t)a * b;
                 TCGArg rl, rh;
-                TCGOp *op2 = insert_op_before(s, op, INDEX_op_movi_i32, 2);
+                TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_movi_i32, 2);
                 TCGArg *args2 = &s->gen_opparam_buf[op2->args];
 
                 rl = args[0];
                 rh = args[1];
-                tcg_opt_gen_movi(s, op, args, opc, rl, (uint32_t)r);
-                tcg_opt_gen_movi(s, op2, args2, opc, rh, (uint32_t)(r >> 32));
+                tcg_opt_gen_movi(s, op, args, rl, (int32_t)r);
+                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(r >> 32));
 
                 /* We've done all we need to do with the movi.  Skip it.  */
                 oi_next = op2->next;
@@ -1194,10 +1142,8 @@
                     tcg_op_remove(s, op);
                 }
             } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
-                       && temps[args[2]].state == TCG_TEMP_CONST
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[2]].val == 0
-                       && temps[args[3]].val == 0) {
+                       && temp_is_const(args[2]) && temps[args[2]].val == 0
+                       && temp_is_const(args[3]) && temps[args[3]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_brcond_high:
@@ -1257,12 +1203,10 @@
             tmp = do_constant_folding_cond2(&args[1], &args[3], args[5]);
             if (tmp != 2) {
             do_setcond_const:
-                tcg_opt_gen_movi(s, op, args, opc, args[0], tmp);
+                tcg_opt_gen_movi(s, op, args, args[0], tmp);
             } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[4]].state == TCG_TEMP_CONST
-                       && temps[args[3]].val == 0
-                       && temps[args[4]].val == 0) {
+                       && temp_is_const(args[3]) && temps[args[3]].val == 0
+                       && temp_is_const(args[4]) && temps[args[4]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_setcond_high:
@@ -1322,7 +1266,9 @@
             if (!(args[nb_oargs + nb_iargs + 1]
                   & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
                 for (i = 0; i < nb_globals; i++) {
-                    reset_temp(i);
+                    if (test_bit(i, temps_used.l)) {
+                        reset_temp(i);
+                    }
                 }
             }
             goto do_reset_output;
@@ -1351,8 +1297,3 @@
         }
     }
 }
-
-void tcg_optimize(TCGContext *s)
-{
-    tcg_constant_folding(s);
-}
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
deleted file mode 100644
index 9c6fa28..0000000
--- a/tcg/ppc/tcg-target.c
+++ /dev/null
@@ -1,2749 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "tcg-be-ldst.h"
-
-#if defined _CALL_DARWIN || defined __APPLE__
-#define TCG_TARGET_CALL_DARWIN
-#endif
-#ifdef _CALL_SYSV
-# define TCG_TARGET_CALL_ALIGN_ARGS   1
-#endif
-
-/* For some memory operations, we need a scratch that isn't R0.  For the AIX
-   calling convention, we can re-use the TOC register since we'll be reloading
-   it at every call.  Otherwise R12 will do nicely as neither a call-saved
-   register nor a parameter register.  */
-#ifdef _CALL_AIX
-# define TCG_REG_TMP1   TCG_REG_R2
-#else
-# define TCG_REG_TMP1   TCG_REG_R12
-#endif
-
-/* For the 64-bit target, we don't like the 5 insn sequence needed to build
-   full 64-bit addresses.  Better to have a base register to which we can
-   apply a 32-bit displacement.
-
-   There are generally three items of interest:
-   (1) helper functions in the main executable,
-   (2) TranslationBlock data structures,
-   (3) the return address in the epilogue.
-
-   For user-only, we USE_STATIC_CODE_GEN_BUFFER, so the code_gen_buffer
-   will be inside the main executable, and thus near enough to make a
-   pointer to the epilogue be within 2GB of all helper functions.
-
-   For softmmu, we'll let the kernel choose the address of code_gen_buffer,
-   and odds are it'll be somewhere close to the main malloc arena, and so
-   a pointer to the epilogue will be within 2GB of the TranslationBlocks.
-
-   For --enable-pie, everything will be kinda near everything else,
-   somewhere in high memory.
-
-   Thus we choose to keep the return address in a call-saved register.  */
-#define TCG_REG_RA     TCG_REG_R31
-#define USE_REG_RA     (TCG_TARGET_REG_BITS == 64)
-
-/* Shorthand for size of a pointer.  Avoid promotion to unsigned.  */
-#define SZP  ((int)sizeof(void *))
-
-/* Shorthand for size of a register.  */
-#define SZR  (TCG_TARGET_REG_BITS / 8)
-
-#define TCG_CT_CONST_S16  0x100
-#define TCG_CT_CONST_U16  0x200
-#define TCG_CT_CONST_S32  0x400
-#define TCG_CT_CONST_U32  0x800
-#define TCG_CT_CONST_ZERO 0x1000
-#define TCG_CT_CONST_MONE 0x2000
-
-static tcg_insn_unit *tb_ret_addr;
-
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
-#endif
-
-#include "elf.h"
-static bool have_isa_2_06;
-#define HAVE_ISA_2_06  have_isa_2_06
-#define HAVE_ISEL      have_isa_2_06
-
-#ifdef CONFIG_USE_GUEST_BASE
-#define TCG_GUEST_BASE_REG 30
-#else
-#define TCG_GUEST_BASE_REG 0
-#endif
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "r0",
-    "r1",
-    "r2",
-    "r3",
-    "r4",
-    "r5",
-    "r6",
-    "r7",
-    "r8",
-    "r9",
-    "r10",
-    "r11",
-    "r12",
-    "r13",
-    "r14",
-    "r15",
-    "r16",
-    "r17",
-    "r18",
-    "r19",
-    "r20",
-    "r21",
-    "r22",
-    "r23",
-    "r24",
-    "r25",
-    "r26",
-    "r27",
-    "r28",
-    "r29",
-    "r30",
-    "r31"
-};
-#endif
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_R14,  /* call saved registers */
-    TCG_REG_R15,
-    TCG_REG_R16,
-    TCG_REG_R17,
-    TCG_REG_R18,
-    TCG_REG_R19,
-    TCG_REG_R20,
-    TCG_REG_R21,
-    TCG_REG_R22,
-    TCG_REG_R23,
-    TCG_REG_R24,
-    TCG_REG_R25,
-    TCG_REG_R26,
-    TCG_REG_R27,
-    TCG_REG_R28,
-    TCG_REG_R29,
-    TCG_REG_R30,
-    TCG_REG_R31,
-    TCG_REG_R12,  /* call clobbered, non-arguments */
-    TCG_REG_R11,
-    TCG_REG_R2,
-    TCG_REG_R13,
-    TCG_REG_R10,  /* call clobbered, arguments */
-    TCG_REG_R9,
-    TCG_REG_R8,
-    TCG_REG_R7,
-    TCG_REG_R6,
-    TCG_REG_R5,
-    TCG_REG_R4,
-    TCG_REG_R3,
-};
-
-static const int tcg_target_call_iarg_regs[] = {
-    TCG_REG_R3,
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10
-};
-
-static const int tcg_target_call_oarg_regs[] = {
-    TCG_REG_R3,
-    TCG_REG_R4
-};
-
-static const int tcg_target_callee_save_regs[] = {
-#ifdef TCG_TARGET_CALL_DARWIN
-    TCG_REG_R11,
-#endif
-    TCG_REG_R14,
-    TCG_REG_R15,
-    TCG_REG_R16,
-    TCG_REG_R17,
-    TCG_REG_R18,
-    TCG_REG_R19,
-    TCG_REG_R20,
-    TCG_REG_R21,
-    TCG_REG_R22,
-    TCG_REG_R23,
-    TCG_REG_R24,
-    TCG_REG_R25,
-    TCG_REG_R26,
-    TCG_REG_R27, /* currently used for the global env */
-    TCG_REG_R28,
-    TCG_REG_R29,
-    TCG_REG_R30,
-    TCG_REG_R31
-};
-
-static inline bool in_range_b(tcg_target_long target)
-{
-    return target == sextract64(target, 0, 26);
-}
-
-static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
-    assert(in_range_b(disp));
-    return disp & 0x3fffffc;
-}
-
-static void reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    *pc = (*pc & ~0x3fffffc) | reloc_pc24_val(pc, target);
-}
-
-static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
-    assert(disp == (int16_t) disp);
-    return disp & 0xfffc;
-}
-
-static void reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target)
-{
-    *pc = (*pc & ~0xfffc) | reloc_pc14_val(pc, target);
-}
-
-static inline void tcg_out_b_noaddr(TCGContext *s, int insn)
-{
-    unsigned retrans = *s->code_ptr & 0x3fffffc;
-    tcg_out32(s, insn | retrans);
-}
-
-static inline void tcg_out_bc_noaddr(TCGContext *s, int insn)
-{
-    unsigned retrans = *s->code_ptr & 0xfffc;
-    tcg_out32(s, insn | retrans);
-}
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    tcg_insn_unit *target = (tcg_insn_unit *)value;
-
-    assert(addend == 0);
-    switch (type) {
-    case R_PPC_REL14:
-        reloc_pc14(code_ptr, target);
-        break;
-    case R_PPC_REL24:
-        reloc_pc24(code_ptr, target);
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'A': case 'B': case 'C': case 'D':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set_reg(ct->u.regs, 3 + ct_str[0] - 'A');
-        break;
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        break;
-    case 'L':                   /* qemu_ld constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
-#ifdef CONFIG_SOFTMMU
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
-#endif
-        break;
-    case 'S':                   /* qemu_st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
-#ifdef CONFIG_SOFTMMU
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
-#endif
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_S16;
-        break;
-    case 'J':
-        ct->ct |= TCG_CT_CONST_U16;
-        break;
-    case 'M':
-        ct->ct |= TCG_CT_CONST_MONE;
-        break;
-    case 'T':
-        ct->ct |= TCG_CT_CONST_S32;
-        break;
-    case 'U':
-        ct->ct |= TCG_CT_CONST_U32;
-        break;
-    case 'Z':
-        ct->ct |= TCG_CT_CONST_ZERO;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                  const TCGArgConstraint *arg_ct)
-{
-    int ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    }
-
-    /* The only 32-bit constraint we use aside from
-       TCG_CT_CONST is TCG_CT_CONST_S16.  */
-    if (type == TCG_TYPE_I32) {
-        val = (int32_t)val;
-    }
-
-    if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_MONE) && val == -1) {
-        return 1;
-    }
-    return 0;
-}
-
-#define OPCD(opc) ((opc)<<26)
-#define XO19(opc) (OPCD(19)|((opc)<<1))
-#define MD30(opc) (OPCD(30)|((opc)<<2))
-#define MDS30(opc) (OPCD(30)|((opc)<<1))
-#define XO31(opc) (OPCD(31)|((opc)<<1))
-#define XO58(opc) (OPCD(58)|(opc))
-#define XO62(opc) (OPCD(62)|(opc))
-
-#define B      OPCD( 18)
-#define BC     OPCD( 16)
-#define LBZ    OPCD( 34)
-#define LHZ    OPCD( 40)
-#define LHA    OPCD( 42)
-#define LWZ    OPCD( 32)
-#define STB    OPCD( 38)
-#define STH    OPCD( 44)
-#define STW    OPCD( 36)
-
-#define STD    XO62(  0)
-#define STDU   XO62(  1)
-#define STDX   XO31(149)
-
-#define LD     XO58(  0)
-#define LDX    XO31( 21)
-#define LDU    XO58(  1)
-#define LWA    XO58(  2)
-#define LWAX   XO31(341)
-
-#define ADDIC  OPCD( 12)
-#define ADDI   OPCD( 14)
-#define ADDIS  OPCD( 15)
-#define ORI    OPCD( 24)
-#define ORIS   OPCD( 25)
-#define XORI   OPCD( 26)
-#define XORIS  OPCD( 27)
-#define ANDI   OPCD( 28)
-#define ANDIS  OPCD( 29)
-#define MULLI  OPCD(  7)
-#define CMPLI  OPCD( 10)
-#define CMPI   OPCD( 11)
-#define SUBFIC OPCD( 8)
-
-#define LWZU   OPCD( 33)
-#define STWU   OPCD( 37)
-
-#define RLWIMI OPCD( 20)
-#define RLWINM OPCD( 21)
-#define RLWNM  OPCD( 23)
-
-#define RLDICL MD30(  0)
-#define RLDICR MD30(  1)
-#define RLDIMI MD30(  3)
-#define RLDCL  MDS30( 8)
-
-#define BCLR   XO19( 16)
-#define BCCTR  XO19(528)
-#define CRAND  XO19(257)
-#define CRANDC XO19(129)
-#define CRNAND XO19(225)
-#define CROR   XO19(449)
-#define CRNOR  XO19( 33)
-
-#define EXTSB  XO31(954)
-#define EXTSH  XO31(922)
-#define EXTSW  XO31(986)
-#define ADD    XO31(266)
-#define ADDE   XO31(138)
-#define ADDME  XO31(234)
-#define ADDZE  XO31(202)
-#define ADDC   XO31( 10)
-#define AND    XO31( 28)
-#define SUBF   XO31( 40)
-#define SUBFC  XO31(  8)
-#define SUBFE  XO31(136)
-#define SUBFME XO31(232)
-#define SUBFZE XO31(200)
-#define OR     XO31(444)
-#define XOR    XO31(316)
-#define MULLW  XO31(235)
-#define MULHW  XO31( 75)
-#define MULHWU XO31( 11)
-#define DIVW   XO31(491)
-#define DIVWU  XO31(459)
-#define CMP    XO31(  0)
-#define CMPL   XO31( 32)
-#define LHBRX  XO31(790)
-#define LWBRX  XO31(534)
-#define LDBRX  XO31(532)
-#define STHBRX XO31(918)
-#define STWBRX XO31(662)
-#define STDBRX XO31(660)
-#define MFSPR  XO31(339)
-#define MTSPR  XO31(467)
-#define SRAWI  XO31(824)
-#define NEG    XO31(104)
-#define MFCR   XO31( 19)
-#define MFOCRF (MFCR | (1u << 20))
-#define NOR    XO31(124)
-#define CNTLZW XO31( 26)
-#define CNTLZD XO31( 58)
-#define ANDC   XO31( 60)
-#define ORC    XO31(412)
-#define EQV    XO31(284)
-#define NAND   XO31(476)
-#define ISEL   XO31( 15)
-
-#define MULLD  XO31(233)
-#define MULHD  XO31( 73)
-#define MULHDU XO31(  9)
-#define DIVD   XO31(489)
-#define DIVDU  XO31(457)
-
-#define LBZX   XO31( 87)
-#define LHZX   XO31(279)
-#define LHAX   XO31(343)
-#define LWZX   XO31( 23)
-#define STBX   XO31(215)
-#define STHX   XO31(407)
-#define STWX   XO31(151)
-
-#define SPR(a, b) ((((a)<<5)|(b))<<11)
-#define LR     SPR(8, 0)
-#define CTR    SPR(9, 0)
-
-#define SLW    XO31( 24)
-#define SRW    XO31(536)
-#define SRAW   XO31(792)
-
-#define SLD    XO31( 27)
-#define SRD    XO31(539)
-#define SRAD   XO31(794)
-#define SRADI  XO31(413<<1)
-
-#define TW     XO31( 4)
-#define TRAP   (TW | TO(31))
-
-#define NOP    ORI  /* ori 0,0,0 */
-
-#define RT(r) ((r)<<21)
-#define RS(r) ((r)<<21)
-#define RA(r) ((r)<<16)
-#define RB(r) ((r)<<11)
-#define TO(t) ((t)<<21)
-#define SH(s) ((s)<<11)
-#define MB(b) ((b)<<6)
-#define ME(e) ((e)<<1)
-#define BO(o) ((o)<<21)
-#define MB64(b) ((b)<<5)
-#define FXM(b) (1 << (19 - (b)))
-
-#define LK    1
-
-#define TAB(t, a, b) (RT(t) | RA(a) | RB(b))
-#define SAB(s, a, b) (RS(s) | RA(a) | RB(b))
-#define TAI(s, a, i) (RT(s) | RA(a) | ((i) & 0xffff))
-#define SAI(s, a, i) (RS(s) | RA(a) | ((i) & 0xffff))
-
-#define BF(n)    ((n)<<23)
-#define BI(n, c) (((c)+((n)*4))<<16)
-#define BT(n, c) (((c)+((n)*4))<<21)
-#define BA(n, c) (((c)+((n)*4))<<16)
-#define BB(n, c) (((c)+((n)*4))<<11)
-#define BC_(n, c) (((c)+((n)*4))<<6)
-
-#define BO_COND_TRUE  BO(12)
-#define BO_COND_FALSE BO( 4)
-#define BO_ALWAYS     BO(20)
-
-enum {
-    CR_LT,
-    CR_GT,
-    CR_EQ,
-    CR_SO
-};
-
-static const uint32_t tcg_to_bc[] = {
-    [TCG_COND_EQ]  = BC | BI(7, CR_EQ) | BO_COND_TRUE,
-    [TCG_COND_NE]  = BC | BI(7, CR_EQ) | BO_COND_FALSE,
-    [TCG_COND_LT]  = BC | BI(7, CR_LT) | BO_COND_TRUE,
-    [TCG_COND_GE]  = BC | BI(7, CR_LT) | BO_COND_FALSE,
-    [TCG_COND_LE]  = BC | BI(7, CR_GT) | BO_COND_FALSE,
-    [TCG_COND_GT]  = BC | BI(7, CR_GT) | BO_COND_TRUE,
-    [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE,
-    [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE,
-    [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE,
-    [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE,
-};
-
-/* The low bit here is set if the RA and RB fields must be inverted.  */
-static const uint32_t tcg_to_isel[] = {
-    [TCG_COND_EQ]  = ISEL | BC_(7, CR_EQ),
-    [TCG_COND_NE]  = ISEL | BC_(7, CR_EQ) | 1,
-    [TCG_COND_LT]  = ISEL | BC_(7, CR_LT),
-    [TCG_COND_GE]  = ISEL | BC_(7, CR_LT) | 1,
-    [TCG_COND_LE]  = ISEL | BC_(7, CR_GT) | 1,
-    [TCG_COND_GT]  = ISEL | BC_(7, CR_GT),
-    [TCG_COND_LTU] = ISEL | BC_(7, CR_LT),
-    [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1,
-    [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1,
-    [TCG_COND_GTU] = ISEL | BC_(7, CR_GT),
-};
-
-static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
-                             TCGReg base, tcg_target_long offset);
-
-static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
-{
-    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-    if (ret != arg) {
-        tcg_out32(s, OR | SAB(arg, ret, arg));
-    }
-}
-
-static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
-                               int sh, int mb)
-{
-    assert(TCG_TARGET_REG_BITS == 64);
-    sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
-    mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
-    tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
-}
-
-static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
-                               int sh, int mb, int me)
-{
-    tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
-}
-
-static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
-{
-    tcg_out_rld(s, RLDICL, dst, src, 0, 32);
-}
-
-static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
-{
-    tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
-}
-
-static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c)
-{
-    tcg_out_rld(s, RLDICR, dst, src, c, 63 - c);
-}
-
-static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c)
-{
-    tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31);
-}
-
-static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c)
-{
-    tcg_out_rld(s, RLDICL, dst, src, 64 - c, c);
-}
-
-static void tcg_out_movi32(TCGContext *s, TCGReg ret, int32_t arg)
-{
-    if (arg == (int16_t) arg) {
-        tcg_out32(s, ADDI | TAI(ret, 0, arg));
-    } else {
-        tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
-        if (arg & 0xffff) {
-            tcg_out32(s, ORI | SAI(ret, ret, arg));
-        }
-    }
-}
-
-static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
-                         tcg_target_long arg)
-{
-    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-    if (type == TCG_TYPE_I32 || arg == (int32_t)arg) {
-        tcg_out_movi32(s, ret, arg);
-    } else if (arg == (uint32_t)arg && !(arg & 0x8000)) {
-        tcg_out32(s, ADDI | TAI(ret, 0, arg));
-        tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
-    } else {
-        int32_t high;
-
-        if (USE_REG_RA) {
-            intptr_t diff = arg - (intptr_t)tb_ret_addr;
-            if (diff == (int32_t)diff) {
-                tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_RA, diff);
-                return;
-            }
-        }
-
-        high = arg >> 31 >> 1;
-        tcg_out_movi32(s, ret, high);
-        if (high) {
-            tcg_out_shli64(s, ret, ret, 32);
-        }
-        if (arg & 0xffff0000) {
-            tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
-        }
-        if (arg & 0xffff) {
-            tcg_out32(s, ORI | SAI(ret, ret, arg));
-        }
-    }
-}
-
-static bool mask_operand(uint32_t c, int *mb, int *me)
-{
-    uint32_t lsb, test;
-
-    /* Accept a bit pattern like:
-           0....01....1
-           1....10....0
-           0..01..10..0
-       Keep track of the transitions.  */
-    if (c == 0 || c == -1) {
-        return false;
-    }
-    test = c;
-    lsb = test & -test;
-    test += lsb;
-    if (test & (test - 1)) {
-        return false;
-    }
-
-    *me = clz32(lsb);
-    *mb = test ? clz32(test & -test) + 1 : 0;
-    return true;
-}
-
-static bool mask64_operand(uint64_t c, int *mb, int *me)
-{
-    uint64_t lsb;
-
-    if (c == 0) {
-        return false;
-    }
-
-    lsb = c & -c;
-    /* Accept 1..10..0.  */
-    if (c == -lsb) {
-        *mb = 0;
-        *me = clz64(lsb);
-        return true;
-    }
-    /* Accept 0..01..1.  */
-    if (lsb == 1 && (c & (c + 1)) == 0) {
-        *mb = clz64(c + 1) + 1;
-        *me = 63;
-        return true;
-    }
-    return false;
-}
-
-static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
-{
-    int mb, me;
-
-    if ((c & 0xffff) == c) {
-        tcg_out32(s, ANDI | SAI(src, dst, c));
-        return;
-    } else if ((c & 0xffff0000) == c) {
-        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
-        return;
-    } else if (mask_operand(c, &mb, &me)) {
-        tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
-        tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
-    }
-}
-
-static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
-{
-    int mb, me;
-
-    assert(TCG_TARGET_REG_BITS == 64);
-    if ((c & 0xffff) == c) {
-        tcg_out32(s, ANDI | SAI(src, dst, c));
-        return;
-    } else if ((c & 0xffff0000) == c) {
-        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
-        return;
-    } else if (mask64_operand(c, &mb, &me)) {
-        if (mb == 0) {
-            tcg_out_rld(s, RLDICR, dst, src, 0, me);
-        } else {
-            tcg_out_rld(s, RLDICL, dst, src, 0, mb);
-        }
-    } else {
-        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
-        tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
-    }
-}
-
-static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c,
-                           int op_lo, int op_hi)
-{
-    if (c >> 16) {
-        tcg_out32(s, op_hi | SAI(src, dst, c >> 16));
-        src = dst;
-    }
-    if (c & 0xffff) {
-        tcg_out32(s, op_lo | SAI(src, dst, c));
-        src = dst;
-    }
-}
-
-static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
-{
-    tcg_out_zori32(s, dst, src, c, ORI, ORIS);
-}
-
-static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
-{
-    tcg_out_zori32(s, dst, src, c, XORI, XORIS);
-}
-
-static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target)
-{
-    ptrdiff_t disp = tcg_pcrel_diff(s, target);
-    if (in_range_b(disp)) {
-        tcg_out32(s, B | (disp & 0x3fffffc) | mask);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target);
-        tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR);
-        tcg_out32(s, BCCTR | BO_ALWAYS | mask);
-    }
-}
-
-static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
-                             TCGReg base, tcg_target_long offset)
-{
-    tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
-    bool is_store = false;
-    TCGReg rs = TCG_REG_TMP1;
-
-    switch (opi) {
-    case LD: case LWA:
-        align = 3;
-        /* FALLTHRU */
-    default:
-        if (rt != TCG_REG_R0) {
-            rs = rt;
-            break;
-        }
-        break;
-    case STD:
-        align = 3;
-        /* FALLTHRU */
-    case STB: case STH: case STW:
-        is_store = true;
-        break;
-    }
-
-    /* For unaligned, or very large offsets, use the indexed form.  */
-    if (offset & align || offset != (int32_t)offset) {
-        if (rs == base) {
-            rs = TCG_REG_R0;
-        }
-        tcg_debug_assert(!is_store || rs != rt);
-        tcg_out_movi(s, TCG_TYPE_PTR, rs, orig);
-        tcg_out32(s, opx | TAB(rt, base, rs));
-        return;
-    }
-
-    l0 = (int16_t)offset;
-    offset = (offset - l0) >> 16;
-    l1 = (int16_t)offset;
-
-    if (l1 < 0 && orig >= 0) {
-        extra = 0x4000;
-        l1 = (int16_t)(offset - 0x4000);
-    }
-    if (l1) {
-        tcg_out32(s, ADDIS | TAI(rs, base, l1));
-        base = rs;
-    }
-    if (extra) {
-        tcg_out32(s, ADDIS | TAI(rs, base, extra));
-        base = rs;
-    }
-    if (opi != ADDI || base != rt || l0 != 0) {
-        tcg_out32(s, opi | TAI(rt, base, l0));
-    }
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, intptr_t arg2)
-{
-    int opi, opx;
-
-    assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-    if (type == TCG_TYPE_I32) {
-        opi = LWZ, opx = LWZX;
-    } else {
-        opi = LD, opx = LDX;
-    }
-    tcg_out_mem_long(s, opi, opx, ret, arg1, arg2);
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    int opi, opx;
-
-    assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-    if (type == TCG_TYPE_I32) {
-        opi = STW, opx = STWX;
-    } else {
-        opi = STD, opx = STDX;
-    }
-    tcg_out_mem_long(s, opi, opx, arg, arg1, arg2);
-}
-
-static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
-                        int const_arg2, int cr, TCGType type)
-{
-    int imm;
-    uint32_t op;
-
-    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-
-    /* Simplify the comparisons below wrt CMPI.  */
-    if (type == TCG_TYPE_I32) {
-        arg2 = (int32_t)arg2;
-    }
-
-    switch (cond) {
-    case TCG_COND_EQ:
-    case TCG_COND_NE:
-        if (const_arg2) {
-            if ((int16_t) arg2 == arg2) {
-                op = CMPI;
-                imm = 1;
-                break;
-            } else if ((uint16_t) arg2 == arg2) {
-                op = CMPLI;
-                imm = 1;
-                break;
-            }
-        }
-        op = CMPL;
-        imm = 0;
-        break;
-
-    case TCG_COND_LT:
-    case TCG_COND_GE:
-    case TCG_COND_LE:
-    case TCG_COND_GT:
-        if (const_arg2) {
-            if ((int16_t) arg2 == arg2) {
-                op = CMPI;
-                imm = 1;
-                break;
-            }
-        }
-        op = CMP;
-        imm = 0;
-        break;
-
-    case TCG_COND_LTU:
-    case TCG_COND_GEU:
-    case TCG_COND_LEU:
-    case TCG_COND_GTU:
-        if (const_arg2) {
-            if ((uint16_t) arg2 == arg2) {
-                op = CMPLI;
-                imm = 1;
-                break;
-            }
-        }
-        op = CMPL;
-        imm = 0;
-        break;
-
-    default:
-        tcg_abort();
-    }
-    op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
-
-    if (imm) {
-        tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff));
-    } else {
-        if (const_arg2) {
-            tcg_out_movi(s, type, TCG_REG_R0, arg2);
-            arg2 = TCG_REG_R0;
-        }
-        tcg_out32(s, op | RA(arg1) | RB(arg2));
-    }
-}
-
-static void tcg_out_setcond_eq0(TCGContext *s, TCGType type,
-                                TCGReg dst, TCGReg src)
-{
-    if (type == TCG_TYPE_I32) {
-        tcg_out32(s, CNTLZW | RS(src) | RA(dst));
-        tcg_out_shri32(s, dst, dst, 5);
-    } else {
-        tcg_out32(s, CNTLZD | RS(src) | RA(dst));
-        tcg_out_shri64(s, dst, dst, 6);
-    }
-}
-
-static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src)
-{
-    /* X != 0 implies X + -1 generates a carry.  Extra addition
-       trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C.  */
-    if (dst != src) {
-        tcg_out32(s, ADDIC | TAI(dst, src, -1));
-        tcg_out32(s, SUBFE | TAB(dst, dst, src));
-    } else {
-        tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1));
-        tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src));
-    }
-}
-
-static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2,
-                                  bool const_arg2)
-{
-    if (const_arg2) {
-        if ((uint32_t)arg2 == arg2) {
-            tcg_out_xori32(s, TCG_REG_R0, arg1, arg2);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2);
-            tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, TCG_REG_R0));
-        }
-    } else {
-        tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, arg2));
-    }
-    return TCG_REG_R0;
-}
-
-static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
-                            TCGArg arg0, TCGArg arg1, TCGArg arg2,
-                            int const_arg2)
-{
-    int crop, sh;
-
-    assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
-
-    /* Ignore high bits of a potential constant arg2.  */
-    if (type == TCG_TYPE_I32) {
-        arg2 = (uint32_t)arg2;
-    }
-
-    /* Handle common and trivial cases before handling anything else.  */
-    if (arg2 == 0) {
-        switch (cond) {
-        case TCG_COND_EQ:
-            tcg_out_setcond_eq0(s, type, arg0, arg1);
-            return;
-        case TCG_COND_NE:
-            if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
-                tcg_out_ext32u(s, TCG_REG_R0, arg1);
-                arg1 = TCG_REG_R0;
-            }
-            tcg_out_setcond_ne0(s, arg0, arg1);
-            return;
-        case TCG_COND_GE:
-            tcg_out32(s, NOR | SAB(arg1, arg0, arg1));
-            arg1 = arg0;
-            /* FALLTHRU */
-        case TCG_COND_LT:
-            /* Extract the sign bit.  */
-            if (type == TCG_TYPE_I32) {
-                tcg_out_shri32(s, arg0, arg1, 31);
-            } else {
-                tcg_out_shri64(s, arg0, arg1, 63);
-            }
-            return;
-        default:
-            break;
-        }
-    }
-
-    /* If we have ISEL, we can implement everything with 3 or 4 insns.
-       All other cases below are also at least 3 insns, so speed up the
-       code generator by not considering them and always using ISEL.  */
-    if (HAVE_ISEL) {
-        int isel, tab;
-
-        tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
-
-        isel = tcg_to_isel[cond];
-
-        tcg_out_movi(s, type, arg0, 1);
-        if (isel & 1) {
-            /* arg0 = (bc ? 0 : 1) */
-            tab = TAB(arg0, 0, arg0);
-            isel &= ~1;
-        } else {
-            /* arg0 = (bc ? 1 : 0) */
-            tcg_out_movi(s, type, TCG_REG_R0, 0);
-            tab = TAB(arg0, arg0, TCG_REG_R0);
-        }
-        tcg_out32(s, isel | tab);
-        return;
-    }
-
-    switch (cond) {
-    case TCG_COND_EQ:
-        arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
-        tcg_out_setcond_eq0(s, type, arg0, arg1);
-        return;
-
-    case TCG_COND_NE:
-        arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
-        /* Discard the high bits only once, rather than both inputs.  */
-        if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
-            tcg_out_ext32u(s, TCG_REG_R0, arg1);
-            arg1 = TCG_REG_R0;
-        }
-        tcg_out_setcond_ne0(s, arg0, arg1);
-        return;
-
-    case TCG_COND_GT:
-    case TCG_COND_GTU:
-        sh = 30;
-        crop = 0;
-        goto crtest;
-
-    case TCG_COND_LT:
-    case TCG_COND_LTU:
-        sh = 29;
-        crop = 0;
-        goto crtest;
-
-    case TCG_COND_GE:
-    case TCG_COND_GEU:
-        sh = 31;
-        crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT);
-        goto crtest;
-
-    case TCG_COND_LE:
-    case TCG_COND_LEU:
-        sh = 31;
-        crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT);
-    crtest:
-        tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
-        if (crop) {
-            tcg_out32(s, crop);
-        }
-        tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
-        tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31);
-        break;
-
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_bc(TCGContext *s, int bc, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        tcg_out32(s, bc | reloc_pc14_val(s->code_ptr, l->u.value_ptr));
-    } else {
-        tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, label_index, 0);
-        tcg_out_bc_noaddr(s, bc);
-    }
-}
-
-static void tcg_out_brcond(TCGContext *s, TCGCond cond,
-                           TCGArg arg1, TCGArg arg2, int const_arg2,
-                           int label_index, TCGType type)
-{
-    tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
-    tcg_out_bc(s, tcg_to_bc[cond], label_index);
-}
-
-static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
-                            TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1,
-                            TCGArg v2, bool const_c2)
-{
-    /* If for some reason both inputs are zero, don't produce bad code.  */
-    if (v1 == 0 && v2 == 0) {
-        tcg_out_movi(s, type, dest, 0);
-        return;
-    }
-
-    tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type);
-
-    if (HAVE_ISEL) {
-        int isel = tcg_to_isel[cond];
-
-        /* Swap the V operands if the operation indicates inversion.  */
-        if (isel & 1) {
-            int t = v1;
-            v1 = v2;
-            v2 = t;
-            isel &= ~1;
-        }
-        /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand.  */
-        if (v2 == 0) {
-            tcg_out_movi(s, type, TCG_REG_R0, 0);
-        }
-        tcg_out32(s, isel | TAB(dest, v1, v2));
-    } else {
-        if (dest == v2) {
-            cond = tcg_invert_cond(cond);
-            v2 = v1;
-        } else if (dest != v1) {
-            if (v1 == 0) {
-                tcg_out_movi(s, type, dest, 0);
-            } else {
-                tcg_out_mov(s, type, dest, v1);
-            }
-        }
-        /* Branch forward over one insn */
-        tcg_out32(s, tcg_to_bc[cond] | 8);
-        if (v2 == 0) {
-            tcg_out_movi(s, type, dest, 0);
-        } else {
-            tcg_out_mov(s, type, dest, v2);
-        }
-    }
-}
-
-static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
-                         const int *const_args)
-{
-    static const struct { uint8_t bit1, bit2; } bits[] = {
-        [TCG_COND_LT ] = { CR_LT, CR_LT },
-        [TCG_COND_LE ] = { CR_LT, CR_GT },
-        [TCG_COND_GT ] = { CR_GT, CR_GT },
-        [TCG_COND_GE ] = { CR_GT, CR_LT },
-        [TCG_COND_LTU] = { CR_LT, CR_LT },
-        [TCG_COND_LEU] = { CR_LT, CR_GT },
-        [TCG_COND_GTU] = { CR_GT, CR_GT },
-        [TCG_COND_GEU] = { CR_GT, CR_LT },
-    };
-
-    TCGCond cond = args[4], cond2;
-    TCGArg al, ah, bl, bh;
-    int blconst, bhconst;
-    int op, bit1, bit2;
-
-    al = args[0];
-    ah = args[1];
-    bl = args[2];
-    bh = args[3];
-    blconst = const_args[2];
-    bhconst = const_args[3];
-
-    switch (cond) {
-    case TCG_COND_EQ:
-        op = CRAND;
-        goto do_equality;
-    case TCG_COND_NE:
-        op = CRNAND;
-    do_equality:
-        tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32);
-        tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32);
-        tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
-        break;
-
-    case TCG_COND_LT:
-    case TCG_COND_LE:
-    case TCG_COND_GT:
-    case TCG_COND_GE:
-    case TCG_COND_LTU:
-    case TCG_COND_LEU:
-    case TCG_COND_GTU:
-    case TCG_COND_GEU:
-        bit1 = bits[cond].bit1;
-        bit2 = bits[cond].bit2;
-        op = (bit1 != bit2 ? CRANDC : CRAND);
-        cond2 = tcg_unsigned_cond(cond);
-
-        tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32);
-        tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32);
-        tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2));
-        tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ));
-        break;
-
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
-                             const int *const_args)
-{
-    tcg_out_cmp2(s, args + 1, const_args + 1);
-    tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
-    tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31);
-}
-
-static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
-                             const int *const_args)
-{
-    tcg_out_cmp2(s, args, const_args);
-    tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, args[5]);
-}
-
-void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
-{
-    TCGContext s;
-
-    s.code_buf = s.code_ptr = (tcg_insn_unit *)jmp_addr;
-    tcg_out_b(&s, 0, (tcg_insn_unit *)addr);
-    flush_icache_range(jmp_addr, jmp_addr + tcg_current_code_size(&s));
-}
-
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
-{
-#ifdef _CALL_AIX
-    /* Look through the descriptor.  If the branch is in range, and we
-       don't have to spend too much effort on building the toc.  */
-    void *tgt = ((void **)target)[0];
-    uintptr_t toc = ((uintptr_t *)target)[1];
-    intptr_t diff = tcg_pcrel_diff(s, tgt);
-
-    if (in_range_b(diff) && toc == (uint32_t)toc) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc);
-        tcg_out_b(s, LK, tgt);
-    } else {
-        /* Fold the low bits of the constant into the addresses below.  */
-        intptr_t arg = (intptr_t)target;
-        int ofs = (int16_t)arg;
-
-        if (ofs + 8 < 0x8000) {
-            arg -= ofs;
-        } else {
-            ofs = 0;
-        }
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg);
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs);
-        tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR);
-        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP);
-        tcg_out32(s, BCCTR | BO_ALWAYS | LK);
-    }
-#elif defined(_CALL_ELF) && _CALL_ELF == 2
-    intptr_t diff;
-
-    /* In the ELFv2 ABI, we have to set up r12 to contain the destination
-       address, which the callee uses to compute its TOC address.  */
-    /* FIXME: when the branch is in range, we could avoid r12 load if we
-       knew that the destination uses the same TOC, and what its local
-       entry point offset is.  */
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target);
-
-    diff = tcg_pcrel_diff(s, target);
-    if (in_range_b(diff)) {
-        tcg_out_b(s, LK, target);
-    } else {
-        tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR);
-        tcg_out32(s, BCCTR | BO_ALWAYS | LK);
-    }
-#else
-    tcg_out_b(s, LK, target);
-#endif
-}
-
-static const uint32_t qemu_ldx_opc[16] = {
-    [MO_UB] = LBZX,
-    [MO_UW] = LHZX,
-    [MO_UL] = LWZX,
-    [MO_Q]  = LDX,
-    [MO_SW] = LHAX,
-    [MO_SL] = LWAX,
-    [MO_BSWAP | MO_UB] = LBZX,
-    [MO_BSWAP | MO_UW] = LHBRX,
-    [MO_BSWAP | MO_UL] = LWBRX,
-    [MO_BSWAP | MO_Q]  = LDBRX,
-};
-
-static const uint32_t qemu_stx_opc[16] = {
-    [MO_UB] = STBX,
-    [MO_UW] = STHX,
-    [MO_UL] = STWX,
-    [MO_Q]  = STDX,
-    [MO_BSWAP | MO_UB] = STBX,
-    [MO_BSWAP | MO_UW] = STHBRX,
-    [MO_BSWAP | MO_UL] = STWBRX,
-    [MO_BSWAP | MO_Q]  = STDBRX,
-};
-
-static const uint32_t qemu_exts_opc[4] = {
-    EXTSB, EXTSH, EXTSW, 0
-};
-
-#if defined (CONFIG_SOFTMMU)
-/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
- *                                 int mmu_idx, uintptr_t ra)
- */
-static void * const qemu_ld_helpers[16] = {
-    [MO_UB]   = helper_ret_ldub_mmu,
-    [MO_LEUW] = helper_le_lduw_mmu,
-    [MO_LEUL] = helper_le_ldul_mmu,
-    [MO_LEQ]  = helper_le_ldq_mmu,
-    [MO_BEUW] = helper_be_lduw_mmu,
-    [MO_BEUL] = helper_be_ldul_mmu,
-    [MO_BEQ]  = helper_be_ldq_mmu,
-};
-
-/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
- *                                 uintxx_t val, int mmu_idx, uintptr_t ra)
- */
-static void * const qemu_st_helpers[16] = {
-    [MO_UB]   = helper_ret_stb_mmu,
-    [MO_LEUW] = helper_le_stw_mmu,
-    [MO_LEUL] = helper_le_stl_mmu,
-    [MO_LEQ]  = helper_le_stq_mmu,
-    [MO_BEUW] = helper_be_stw_mmu,
-    [MO_BEUL] = helper_be_stl_mmu,
-    [MO_BEQ]  = helper_be_stq_mmu,
-};
-
-/* Perform the TLB load and compare.  Places the result of the comparison
-   in CR7, loads the addend of the TLB into R3, and returns the register
-   containing the guest address (zero-extended into R4).  Clobbers R0 and R2. */
-
-static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc,
-                               TCGReg addrlo, TCGReg addrhi,
-                               int mem_index, bool is_read)
-{
-    int cmp_off
-        = (is_read
-           ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
-           : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
-    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
-    TCGReg base = TCG_AREG0;
-    TCGMemOp s_bits = opc & MO_SIZE;
-    int a_bits = get_alignment_bits(opc);
-
-    /* Extract the page index, shifted into place for tlb index.  */
-    if (TCG_TARGET_REG_BITS == 64) {
-        if (TARGET_LONG_BITS == 32) {
-            /* Zero-extend the address into a place helpful for further use. */
-            tcg_out_ext32u(s, TCG_REG_R4, addrlo);
-            addrlo = TCG_REG_R4;
-        } else {
-            tcg_out_rld(s, RLDICL, TCG_REG_R3, addrlo,
-                        64 - TARGET_PAGE_BITS, 64 - CPU_TLB_BITS);
-        }
-    }
-
-    /* Compensate for very large offsets.  */
-    if (add_off >= 0x8000) {
-        /* Most target env are smaller than 32k; none are larger than 64k.
-           Simplify the logic here merely to offset by 0x7ff0, giving us a
-           range just shy of 64k.  Check this assumption.  */
-        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
-                                   tlb_table[NB_MMU_MODES - 1][1])
-                          > 0x7ff0 + 0x7fff);
-        tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, base, 0x7ff0));
-        base = TCG_REG_TMP1;
-        cmp_off -= 0x7ff0;
-        add_off -= 0x7ff0;
-    }
-
-    /* Extraction and shifting, part 2.  */
-    if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
-        tcg_out_rlw(s, RLWINM, TCG_REG_R3, addrlo,
-                    32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
-                    32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS),
-                    31 - CPU_TLB_ENTRY_BITS);
-    } else {
-        tcg_out_shli64(s, TCG_REG_R3, TCG_REG_R3, CPU_TLB_ENTRY_BITS);
-    }
-
-    tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, base));
-
-    /* Load the tlb comparator.  */
-    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
-        tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R4, TCG_REG_R3, cmp_off);
-        tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP1, TCG_REG_R3, cmp_off + 4);
-    } else {
-        tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP1, TCG_REG_R3, cmp_off);
-    }
-
-    /* Load the TLB addend for use on the fast path.  Do this asap
-       to minimize any load use delay.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, add_off);
-
-    /* Clear the non-page, non-alignment bits from the address */
-    if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
-        /* We don't support unaligned accesses on 32-bits, preserve
-         * the bottom bits and thus trigger a comparison failure on
-         * unaligned accesses
-         */
-        if (a_bits < 0) {
-            a_bits = s_bits;
-        }
-        tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
-                    (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
-    } else if (a_bits) {
-        /* More than byte access, we need to handle alignment */
-        if (a_bits > 0) {
-            /* Alignment required by the front-end, same as 32-bits */
-            tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
-                        64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits);
-            tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
-       } else {
-           /* We support unaligned accesses, we need to make sure we fail
-            * if we cross a page boundary. The trick is to add the
-            * access_size-1 to the address before masking the low bits.
-            * That will make the address overflow to the next page if we
-            * cross a page boundary which will then force a mismatch of
-            * the TLB compare since the next page cannot possibly be in
-            * the same TLB index.
-            */
-            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, (1 << s_bits) - 1));
-            tcg_out_rld(s, RLDICR, TCG_REG_R0, TCG_REG_R0,
-                        0, 63 - TARGET_PAGE_BITS);
-        }
-    } else {
-        /* Byte access, just chop off the bits below the page index */
-        tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo, 0, 63 - TARGET_PAGE_BITS);
-    }
-
-    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
-        tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
-                    0, 7, TCG_TYPE_I32);
-        tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_R4, 0, 6, TCG_TYPE_I32);
-        tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
-    } else {
-        tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
-                    0, 7, TCG_TYPE_TL);
-    }
-
-    return addrlo;
-}
-
-/* Record the context of a call to the out of line helper code for the slow
-   path for a load or store, so that we can later generate the correct
-   helper code.  */
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
-                                TCGReg datalo_reg, TCGReg datahi_reg,
-                                TCGReg addrlo_reg, TCGReg addrhi_reg,
-                                tcg_insn_unit *raddr, tcg_insn_unit *lptr)
-{
-    TCGLabelQemuLdst *label = new_ldst_label(s);
-
-    label->is_ld = is_ld;
-    label->oi = oi;
-    label->datalo_reg = datalo_reg;
-    label->datahi_reg = datahi_reg;
-    label->addrlo_reg = addrlo_reg;
-    label->addrhi_reg = addrhi_reg;
-    label->raddr = raddr;
-    label->label_ptr[0] = lptr;
-}
-
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGReg hi, lo, arg = TCG_REG_R3;
-
-    reloc_pc14(lb->label_ptr[0], s->code_ptr);
-
-    tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
-
-    lo = lb->addrlo_reg;
-    hi = lb->addrhi_reg;
-    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
-        arg |= 1;
-#endif
-        tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
-        tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
-    } else {
-        /* If the address needed to be zero-extended, we'll have already
-           placed it in R4.  The only remaining case is 64-bit guest.  */
-        tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
-    }
-
-    tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
-    tcg_out32(s, MFSPR | RT(arg) | LR);
-
-    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-
-    lo = lb->datalo_reg;
-    hi = lb->datahi_reg;
-    if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
-        tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
-        tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
-    } else if (opc & MO_SIGN) {
-        uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
-        tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
-    } else {
-        tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
-    }
-
-    tcg_out_b(s, 0, lb->raddr);
-}
-
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-    TCGMemOp s_bits = opc & MO_SIZE;
-    TCGReg hi, lo, arg = TCG_REG_R3;
-
-    reloc_pc14(lb->label_ptr[0], s->code_ptr);
-
-    tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
-
-    lo = lb->addrlo_reg;
-    hi = lb->addrhi_reg;
-    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
-        arg |= 1;
-#endif
-        tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
-        tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
-    } else {
-        /* If the address needed to be zero-extended, we'll have already
-           placed it in R4.  The only remaining case is 64-bit guest.  */
-        tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
-    }
-
-    lo = lb->datalo_reg;
-    hi = lb->datahi_reg;
-    if (TCG_TARGET_REG_BITS == 32) {
-        switch (s_bits) {
-        case MO_64:
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
-            arg |= 1;
-#endif
-            tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
-            /* FALLTHRU */
-        case MO_32:
-            tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
-            break;
-        default:
-            tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
-            break;
-        }
-    } else {
-        if (s_bits == MO_64) {
-            tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
-        } else {
-            tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
-        }
-    }
-
-    tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
-    tcg_out32(s, MFSPR | RT(arg) | LR);
-
-    tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-
-    tcg_out_b(s, 0, lb->raddr);
-}
-#endif /* SOFTMMU */
-
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
-{
-    TCGReg datalo, datahi, addrlo, rbase;
-    TCGReg addrhi __attribute__((unused));
-    TCGMemOpIdx oi;
-    TCGMemOp opc, s_bits;
-#ifdef CONFIG_SOFTMMU
-    int mem_index;
-    tcg_insn_unit *label_ptr;
-#endif
-
-    datalo = *args++;
-    datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
-    addrlo = *args++;
-    addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-    s_bits = opc & MO_SIZE;
-
-#ifdef CONFIG_SOFTMMU
-    mem_index = get_mmuidx(oi);
-    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
-
-    /* Load a pointer into the current opcode w/conditional branch-link. */
-    label_ptr = s->code_ptr;
-    tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
-
-    rbase = TCG_REG_R3;
-#else  /* !CONFIG_SOFTMMU */
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
-    if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
-        tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
-        addrlo = TCG_REG_TMP1;
-    }
-#endif
-
-    if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
-        if (opc & MO_BSWAP) {
-            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
-            tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
-            tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0));
-        } else if (rbase != 0) {
-            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
-            tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo));
-            tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0));
-        } else if (addrlo == datahi) {
-            tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
-            tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
-        } else {
-            tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
-            tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
-        }
-    } else {
-        uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)];
-        if (!HAVE_ISA_2_06 && insn == LDBRX) {
-            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
-            tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
-            tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0));
-            tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0);
-        } else if (insn) {
-            tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
-        } else {
-            insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
-            tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
-            insn = qemu_exts_opc[s_bits];
-            tcg_out32(s, insn | RA(datalo) | RS(datalo));
-        }
-    }
-
-#ifdef CONFIG_SOFTMMU
-    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
-                        s->code_ptr, label_ptr);
-#endif
-}
-
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
-{
-    TCGReg datalo, datahi, addrlo, rbase;
-    TCGReg addrhi __attribute__((unused));
-    TCGMemOpIdx oi;
-    TCGMemOp opc, s_bits;
-#ifdef CONFIG_SOFTMMU
-    int mem_index;
-    tcg_insn_unit *label_ptr;
-#endif
-
-    datalo = *args++;
-    datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
-    addrlo = *args++;
-    addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
-    oi = *args++;
-    opc = get_memop(oi);
-    s_bits = opc & MO_SIZE;
-
-#ifdef CONFIG_SOFTMMU
-    mem_index = get_mmuidx(oi);
-    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
-
-    /* Load a pointer into the current opcode w/conditional branch-link. */
-    label_ptr = s->code_ptr;
-    tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
-
-    rbase = TCG_REG_R3;
-#else  /* !CONFIG_SOFTMMU */
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
-    if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
-        tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
-        addrlo = TCG_REG_TMP1;
-    }
-#endif
-
-    if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
-        if (opc & MO_BSWAP) {
-            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
-            tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
-            tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0));
-        } else if (rbase != 0) {
-            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
-            tcg_out32(s, STWX | SAB(datahi, rbase, addrlo));
-            tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0));
-        } else {
-            tcg_out32(s, STW | TAI(datahi, addrlo, 0));
-            tcg_out32(s, STW | TAI(datalo, addrlo, 4));
-        }
-    } else {
-        uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)];
-        if (!HAVE_ISA_2_06 && insn == STDBRX) {
-            tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
-            tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4));
-            tcg_out_shri64(s, TCG_REG_R0, datalo, 32);
-            tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1));
-        } else {
-            tcg_out32(s, insn | SAB(datalo, rbase, addrlo));
-        }
-    }
-
-#ifdef CONFIG_SOFTMMU
-    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
-                        s->code_ptr, label_ptr);
-#endif
-}
-
-/* Parameters for function call generation, used in tcg.c.  */
-#define TCG_TARGET_STACK_ALIGN       16
-#define TCG_TARGET_EXTEND_ARGS       1
-
-#ifdef _CALL_AIX
-# define LINK_AREA_SIZE                (6 * SZR)
-# define LR_OFFSET                     (1 * SZR)
-# define TCG_TARGET_CALL_STACK_OFFSET  (LINK_AREA_SIZE + 8 * SZR)
-#elif defined(TCG_TARGET_CALL_DARWIN)
-# define LINK_AREA_SIZE                (6 * SZR)
-# define LR_OFFSET                     (2 * SZR)
-#elif TCG_TARGET_REG_BITS == 64
-# if defined(_CALL_ELF) && _CALL_ELF == 2
-#  define LINK_AREA_SIZE               (4 * SZR)
-#  define LR_OFFSET                    (1 * SZR)
-# endif
-#else /* TCG_TARGET_REG_BITS == 32 */
-# if defined(_CALL_SYSV)
-#  define LINK_AREA_SIZE               (2 * SZR)
-#  define LR_OFFSET                    (1 * SZR)
-# endif
-#endif
-#ifndef LR_OFFSET
-# error "Unhandled abi"
-#endif
-#ifndef TCG_TARGET_CALL_STACK_OFFSET
-# define TCG_TARGET_CALL_STACK_OFFSET  LINK_AREA_SIZE
-#endif
-
-#define CPU_TEMP_BUF_SIZE  (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
-#define REG_SAVE_SIZE      ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR)
-
-#define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET   \
-                     + TCG_STATIC_CALL_ARGS_SIZE    \
-                     + CPU_TEMP_BUF_SIZE            \
-                     + REG_SAVE_SIZE                \
-                     + TCG_TARGET_STACK_ALIGN - 1)  \
-                    & -TCG_TARGET_STACK_ALIGN)
-
-#define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE)
-
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int i;
-
-#ifdef _CALL_AIX
-    void **desc = (void **)s->code_ptr;
-    desc[0] = desc + 2;                   /* entry point */
-    desc[1] = 0;                          /* environment pointer */
-    s->code_ptr = (void *)(desc + 2);     /* skip over descriptor */
-#endif
-
-    tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
-                  CPU_TEMP_BUF_SIZE);
-
-    /* Prologue */
-    tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR);
-    tcg_out32(s, (SZR == 8 ? STDU : STWU)
-              | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
-
-    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
-        tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
-                   TCG_REG_R1, REG_SAVE_BOT + i * SZR);
-    }
-    tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
-
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
-        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
-    }
-#endif
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-    tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR);
-
-    if (USE_REG_RA) {
-#ifdef _CALL_AIX
-        /* Make the caller load the value as the TOC into R2.  */
-        tb_ret_addr = s->code_ptr + 2;
-        desc[1] = tb_ret_addr;
-        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_RA, TCG_REG_R2);
-        tcg_out32(s, BCCTR | BO_ALWAYS);
-#elif defined(_CALL_ELF) && _CALL_ELF == 2
-        /* Compute from the incoming R12 value.  */
-        tb_ret_addr = s->code_ptr + 2;
-        tcg_out32(s, ADDI | TAI(TCG_REG_RA, TCG_REG_R12,
-                                tcg_ptr_byte_diff(tb_ret_addr, s->code_buf)));
-        tcg_out32(s, BCCTR | BO_ALWAYS);
-#else
-        /* Reserve max 5 insns for the constant load.  */
-        tb_ret_addr = s->code_ptr + 6;
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (intptr_t)tb_ret_addr);
-        tcg_out32(s, BCCTR | BO_ALWAYS);
-        while (s->code_ptr < tb_ret_addr) {
-            tcg_out32(s, NOP);
-        }
-#endif
-    } else {
-        tcg_out32(s, BCCTR | BO_ALWAYS);
-        tb_ret_addr = s->code_ptr;
-    }
-
-    /* Epilogue */
-    assert(tb_ret_addr == s->code_ptr);
-
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
-    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
-        tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
-                   TCG_REG_R1, REG_SAVE_BOT + i * SZR);
-    }
-    tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR);
-    tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
-    tcg_out32(s, BCLR | BO_ALWAYS);
-}
-
-static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
-                       const int *const_args)
-{
-    TCGArg a0, a1, a2;
-    int c;
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        if (USE_REG_RA) {
-            ptrdiff_t disp = tcg_pcrel_diff(s, tb_ret_addr);
-
-            /* If we can use a direct branch, otherwise use the value in RA.
-               Note that the direct branch is always forward.  If it's in
-               range now, it'll still be in range after the movi.  Don't
-               bother about the 20 bytes where the test here fails but it
-               would succeed below.  */
-            if (!in_range_b(disp)) {
-                tcg_out32(s, MTSPR | RS(TCG_REG_RA) | CTR);
-                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
-                tcg_out32(s, BCCTR | BO_ALWAYS);
-                break;
-            }
-        }
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
-        tcg_out_b(s, 0, tb_ret_addr);
-        break;
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* Direct jump method.  */
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            s->code_ptr += 7;
-        } else {
-            /* Indirect jump method.  */
-            tcg_abort();
-        }
-        s->tb_next_offset[args[0]] = tcg_current_code_size(s);
-        break;
-    case INDEX_op_br:
-        {
-            TCGLabel *l = &s->labels[args[0]];
-
-            if (l->has_value) {
-                tcg_out_b(s, 0, l->u.value_ptr);
-            } else {
-                tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, args[0], 0);
-                tcg_out_b_noaddr(s, B);
-            }
-        }
-        break;
-    case INDEX_op_ld8u_i32:
-    case INDEX_op_ld8u_i64:
-        tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld8s_i32:
-    case INDEX_op_ld8s_i64:
-        tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
-        tcg_out32(s, EXTSB | RS(args[0]) | RA(args[0]));
-        break;
-    case INDEX_op_ld16u_i32:
-    case INDEX_op_ld16u_i64:
-        tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld16s_i32:
-    case INDEX_op_ld16s_i64:
-        tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld_i32:
-    case INDEX_op_ld32u_i64:
-        tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld32s_i64:
-        tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_ld_i64:
-        tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st8_i32:
-    case INDEX_op_st8_i64:
-        tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st16_i32:
-    case INDEX_op_st16_i64:
-        tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i32:
-    case INDEX_op_st32_i64:
-        tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i64:
-        tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_add_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-        do_addi_32:
-            tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2);
-        } else {
-            tcg_out32(s, ADD | TAB(a0, a1, a2));
-        }
-        break;
-    case INDEX_op_sub_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[1]) {
-            if (const_args[2]) {
-                tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2);
-            } else {
-                tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
-            }
-        } else if (const_args[2]) {
-            a2 = -a2;
-            goto do_addi_32;
-        } else {
-            tcg_out32(s, SUBF | TAB(a0, a2, a1));
-        }
-        break;
-
-    case INDEX_op_and_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out_andi32(s, a0, a1, a2);
-        } else {
-            tcg_out32(s, AND | SAB(a1, a0, a2));
-        }
-        break;
-    case INDEX_op_and_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out_andi64(s, a0, a1, a2);
-        } else {
-            tcg_out32(s, AND | SAB(a1, a0, a2));
-        }
-        break;
-    case INDEX_op_or_i64:
-    case INDEX_op_or_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out_ori32(s, a0, a1, a2);
-        } else {
-            tcg_out32(s, OR | SAB(a1, a0, a2));
-        }
-        break;
-    case INDEX_op_xor_i64:
-    case INDEX_op_xor_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out_xori32(s, a0, a1, a2);
-        } else {
-            tcg_out32(s, XOR | SAB(a1, a0, a2));
-        }
-        break;
-    case INDEX_op_andc_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out_andi32(s, a0, a1, ~a2);
-        } else {
-            tcg_out32(s, ANDC | SAB(a1, a0, a2));
-        }
-        break;
-    case INDEX_op_andc_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out_andi64(s, a0, a1, ~a2);
-        } else {
-            tcg_out32(s, ANDC | SAB(a1, a0, a2));
-        }
-        break;
-    case INDEX_op_orc_i32:
-        if (const_args[2]) {
-            tcg_out_ori32(s, args[0], args[1], ~args[2]);
-            break;
-        }
-        /* FALLTHRU */
-    case INDEX_op_orc_i64:
-        tcg_out32(s, ORC | SAB(args[1], args[0], args[2]));
-        break;
-    case INDEX_op_eqv_i32:
-        if (const_args[2]) {
-            tcg_out_xori32(s, args[0], args[1], ~args[2]);
-            break;
-        }
-        /* FALLTHRU */
-    case INDEX_op_eqv_i64:
-        tcg_out32(s, EQV | SAB(args[1], args[0], args[2]));
-        break;
-    case INDEX_op_nand_i32:
-    case INDEX_op_nand_i64:
-        tcg_out32(s, NAND | SAB(args[1], args[0], args[2]));
-        break;
-    case INDEX_op_nor_i32:
-    case INDEX_op_nor_i64:
-        tcg_out32(s, NOR | SAB(args[1], args[0], args[2]));
-        break;
-
-    case INDEX_op_mul_i32:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out32(s, MULLI | TAI(a0, a1, a2));
-        } else {
-            tcg_out32(s, MULLW | TAB(a0, a1, a2));
-        }
-        break;
-
-    case INDEX_op_div_i32:
-        tcg_out32(s, DIVW | TAB(args[0], args[1], args[2]));
-        break;
-
-    case INDEX_op_divu_i32:
-        tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
-        break;
-
-    case INDEX_op_shl_i32:
-        if (const_args[2]) {
-            tcg_out_shli32(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out32(s, SLW | SAB(args[1], args[0], args[2]));
-        }
-        break;
-    case INDEX_op_shr_i32:
-        if (const_args[2]) {
-            tcg_out_shri32(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out32(s, SRW | SAB(args[1], args[0], args[2]));
-        }
-        break;
-    case INDEX_op_sar_i32:
-        if (const_args[2]) {
-            tcg_out32(s, SRAWI | RS(args[1]) | RA(args[0]) | SH(args[2]));
-        } else {
-            tcg_out32(s, SRAW | SAB(args[1], args[0], args[2]));
-        }
-        break;
-    case INDEX_op_rotl_i32:
-        if (const_args[2]) {
-            tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31);
-        } else {
-            tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2])
-                         | MB(0) | ME(31));
-        }
-        break;
-    case INDEX_op_rotr_i32:
-        if (const_args[2]) {
-            tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31);
-        } else {
-            tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32));
-            tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0)
-                         | MB(0) | ME(31));
-        }
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
-                       args[3], TCG_TYPE_I32);
-        break;
-    case INDEX_op_brcond_i64:
-        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
-                       args[3], TCG_TYPE_I64);
-        break;
-    case INDEX_op_brcond2_i32:
-        tcg_out_brcond2(s, args, const_args);
-        break;
-
-    case INDEX_op_neg_i32:
-    case INDEX_op_neg_i64:
-        tcg_out32(s, NEG | RT(args[0]) | RA(args[1]));
-        break;
-
-    case INDEX_op_not_i32:
-    case INDEX_op_not_i64:
-        tcg_out32(s, NOR | SAB(args[1], args[0], args[1]));
-        break;
-
-    case INDEX_op_add_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-        do_addi_64:
-            tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
-        } else {
-            tcg_out32(s, ADD | TAB(a0, a1, a2));
-        }
-        break;
-    case INDEX_op_sub_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[1]) {
-            if (const_args[2]) {
-                tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2);
-            } else {
-                tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
-            }
-        } else if (const_args[2]) {
-            a2 = -a2;
-            goto do_addi_64;
-        } else {
-            tcg_out32(s, SUBF | TAB(a0, a2, a1));
-        }
-        break;
-
-    case INDEX_op_shl_i64:
-        if (const_args[2]) {
-            tcg_out_shli64(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out32(s, SLD | SAB(args[1], args[0], args[2]));
-        }
-        break;
-    case INDEX_op_shr_i64:
-        if (const_args[2]) {
-            tcg_out_shri64(s, args[0], args[1], args[2]);
-        } else {
-            tcg_out32(s, SRD | SAB(args[1], args[0], args[2]));
-        }
-        break;
-    case INDEX_op_sar_i64:
-        if (const_args[2]) {
-            int sh = SH(args[2] & 0x1f) | (((args[2] >> 5) & 1) << 1);
-            tcg_out32(s, SRADI | RA(args[0]) | RS(args[1]) | sh);
-        } else {
-            tcg_out32(s, SRAD | SAB(args[1], args[0], args[2]));
-        }
-        break;
-    case INDEX_op_rotl_i64:
-        if (const_args[2]) {
-            tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0);
-        } else {
-            tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0));
-        }
-        break;
-    case INDEX_op_rotr_i64:
-        if (const_args[2]) {
-            tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0);
-        } else {
-            tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64));
-            tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0));
-        }
-        break;
-
-    case INDEX_op_mul_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            tcg_out32(s, MULLI | TAI(a0, a1, a2));
-        } else {
-            tcg_out32(s, MULLD | TAB(a0, a1, a2));
-        }
-        break;
-    case INDEX_op_div_i64:
-        tcg_out32(s, DIVD | TAB(args[0], args[1], args[2]));
-        break;
-    case INDEX_op_divu_i64:
-        tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_qemu_ld(s, args, false);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, args, true);
-        break;
-    case INDEX_op_qemu_st_i32:
-        tcg_out_qemu_st(s, args, false);
-        break;
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args, true);
-        break;
-
-    case INDEX_op_ext8s_i32:
-    case INDEX_op_ext8s_i64:
-        c = EXTSB;
-        goto gen_ext;
-    case INDEX_op_ext16s_i32:
-    case INDEX_op_ext16s_i64:
-        c = EXTSH;
-        goto gen_ext;
-    case INDEX_op_ext32s_i64:
-        c = EXTSW;
-        goto gen_ext;
-    gen_ext:
-        tcg_out32(s, c | RS(args[1]) | RA(args[0]));
-        break;
-
-    case INDEX_op_setcond_i32:
-        tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
-                        const_args[2]);
-        break;
-    case INDEX_op_setcond_i64:
-        tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
-                        const_args[2]);
-        break;
-    case INDEX_op_setcond2_i32:
-        tcg_out_setcond2(s, args, const_args);
-        break;
-
-    case INDEX_op_bswap16_i32:
-    case INDEX_op_bswap16_i64:
-        a0 = args[0], a1 = args[1];
-        /* a1 = abcd */
-        if (a0 != a1) {
-            /* a0 = (a1 r<< 24) & 0xff # 000c */
-            tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
-            /* a0 = (a0 & ~0xff00) | (a1 r<< 8) & 0xff00 # 00dc */
-            tcg_out_rlw(s, RLWIMI, a0, a1, 8, 16, 23);
-        } else {
-            /* r0 = (a1 r<< 8) & 0xff00 # 00d0 */
-            tcg_out_rlw(s, RLWINM, TCG_REG_R0, a1, 8, 16, 23);
-            /* a0 = (a1 r<< 24) & 0xff # 000c */
-            tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
-            /* a0 = a0 | r0 # 00dc */
-            tcg_out32(s, OR | SAB(TCG_REG_R0, a0, a0));
-        }
-        break;
-
-    case INDEX_op_bswap32_i32:
-    case INDEX_op_bswap32_i64:
-        /* Stolen from gcc's builtin_bswap32 */
-        a1 = args[1];
-        a0 = args[0] == a1 ? TCG_REG_R0 : args[0];
-
-        /* a1 = args[1] # abcd */
-        /* a0 = rotate_left (a1, 8) # bcda */
-        tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
-        /* a0 = (a0 & ~0xff000000) | ((a1 r<< 24) & 0xff000000) # dcda */
-        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
-        /* a0 = (a0 & ~0x0000ff00) | ((a1 r<< 24) & 0x0000ff00) # dcba */
-        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
-
-        if (a0 == TCG_REG_R0) {
-            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
-        }
-        break;
-
-    case INDEX_op_bswap64_i64:
-        a0 = args[0], a1 = args[1], a2 = TCG_REG_R0;
-        if (a0 == a1) {
-            a0 = TCG_REG_R0;
-            a2 = a1;
-        }
-
-        /* a1 = # abcd efgh */
-        /* a0 = rl32(a1, 8) # 0000 fghe */
-        tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
-        /* a0 = dep(a0, rl32(a1, 24), 0xff000000) # 0000 hghe */
-        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
-        /* a0 = dep(a0, rl32(a1, 24), 0x0000ff00) # 0000 hgfe */
-        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
-
-        /* a0 = rl64(a0, 32) # hgfe 0000 */
-        /* a2 = rl64(a1, 32) # efgh abcd */
-        tcg_out_rld(s, RLDICL, a0, a0, 32, 0);
-        tcg_out_rld(s, RLDICL, a2, a1, 32, 0);
-
-        /* a0 = dep(a0, rl32(a2, 8), 0xffffffff)  # hgfe bcda */
-        tcg_out_rlw(s, RLWIMI, a0, a2, 8, 0, 31);
-        /* a0 = dep(a0, rl32(a2, 24), 0xff000000) # hgfe dcda */
-        tcg_out_rlw(s, RLWIMI, a0, a2, 24, 0, 7);
-        /* a0 = dep(a0, rl32(a2, 24), 0x0000ff00) # hgfe dcba */
-        tcg_out_rlw(s, RLWIMI, a0, a2, 24, 16, 23);
-
-        if (a0 == 0) {
-            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
-        }
-        break;
-
-    case INDEX_op_deposit_i32:
-        if (const_args[2]) {
-            uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3];
-            tcg_out_andi32(s, args[0], args[0], ~mask);
-        } else {
-            tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3],
-                        32 - args[3] - args[4], 31 - args[3]);
-        }
-        break;
-    case INDEX_op_deposit_i64:
-        if (const_args[2]) {
-            uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3];
-            tcg_out_andi64(s, args[0], args[0], ~mask);
-        } else {
-            tcg_out_rld(s, RLDIMI, args[0], args[2], args[3],
-                        64 - args[3] - args[4]);
-        }
-        break;
-
-    case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2],
-                        args[3], args[4], const_args[2]);
-        break;
-    case INDEX_op_movcond_i64:
-        tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2],
-                        args[3], args[4], const_args[2]);
-        break;
-
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_add2_i64:
-#else
-    case INDEX_op_add2_i32:
-#endif
-        /* Note that the CA bit is defined based on the word size of the
-           environment.  So in 64-bit mode it's always carry-out of bit 63.
-           The fallback code using deposit works just as well for 32-bit.  */
-        a0 = args[0], a1 = args[1];
-        if (a0 == args[3] || (!const_args[5] && a0 == args[5])) {
-            a0 = TCG_REG_R0;
-        }
-        if (const_args[4]) {
-            tcg_out32(s, ADDIC | TAI(a0, args[2], args[4]));
-        } else {
-            tcg_out32(s, ADDC | TAB(a0, args[2], args[4]));
-        }
-        if (const_args[5]) {
-            tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3]));
-        } else {
-            tcg_out32(s, ADDE | TAB(a1, args[3], args[5]));
-        }
-        if (a0 != args[0]) {
-            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
-        }
-        break;
-
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_sub2_i64:
-#else
-    case INDEX_op_sub2_i32:
-#endif
-        a0 = args[0], a1 = args[1];
-        if (a0 == args[5] || (!const_args[3] && a0 == args[3])) {
-            a0 = TCG_REG_R0;
-        }
-        if (const_args[2]) {
-            tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2]));
-        } else {
-            tcg_out32(s, SUBFC | TAB(a0, args[4], args[2]));
-        }
-        if (const_args[3]) {
-            tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5]));
-        } else {
-            tcg_out32(s, SUBFE | TAB(a1, args[5], args[3]));
-        }
-        if (a0 != args[0]) {
-            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
-        }
-        break;
-
-    case INDEX_op_muluh_i32:
-        tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2]));
-        break;
-    case INDEX_op_mulsh_i32:
-        tcg_out32(s, MULHW | TAB(args[0], args[1], args[2]));
-        break;
-    case INDEX_op_muluh_i64:
-        tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2]));
-        break;
-    case INDEX_op_mulsh_i64:
-        tcg_out32(s, MULHD | TAB(args[0], args[1], args[2]));
-        break;
-
-    case INDEX_op_mov_i32:   /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32:  /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef ppc_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-
-    { INDEX_op_st8_i32, { "r", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
-
-    { INDEX_op_add_i32, { "r", "r", "ri" } },
-    { INDEX_op_mul_i32, { "r", "r", "rI" } },
-    { INDEX_op_div_i32, { "r", "r", "r" } },
-    { INDEX_op_divu_i32, { "r", "r", "r" } },
-    { INDEX_op_sub_i32, { "r", "rI", "ri" } },
-    { INDEX_op_and_i32, { "r", "r", "ri" } },
-    { INDEX_op_or_i32, { "r", "r", "ri" } },
-    { INDEX_op_xor_i32, { "r", "r", "ri" } },
-    { INDEX_op_andc_i32, { "r", "r", "ri" } },
-    { INDEX_op_orc_i32, { "r", "r", "ri" } },
-    { INDEX_op_eqv_i32, { "r", "r", "ri" } },
-    { INDEX_op_nand_i32, { "r", "r", "r" } },
-    { INDEX_op_nor_i32, { "r", "r", "r" } },
-
-    { INDEX_op_shl_i32, { "r", "r", "ri" } },
-    { INDEX_op_shr_i32, { "r", "r", "ri" } },
-    { INDEX_op_sar_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
-    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
-
-    { INDEX_op_neg_i32, { "r", "r" } },
-    { INDEX_op_not_i32, { "r", "r" } },
-    { INDEX_op_ext8s_i32, { "r", "r" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-    { INDEX_op_bswap16_i32, { "r", "r" } },
-    { INDEX_op_bswap32_i32, { "r", "r" } },
-
-    { INDEX_op_brcond_i32, { "r", "ri" } },
-    { INDEX_op_setcond_i32, { "r", "r", "ri" } },
-    { INDEX_op_movcond_i32, { "r", "r", "ri", "rZ", "rZ" } },
-
-    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
-
-    { INDEX_op_muluh_i32, { "r", "r", "r" } },
-    { INDEX_op_mulsh_i32, { "r", "r", "r" } },
-
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_ld8u_i64, { "r", "r" } },
-    { INDEX_op_ld8s_i64, { "r", "r" } },
-    { INDEX_op_ld16u_i64, { "r", "r" } },
-    { INDEX_op_ld16s_i64, { "r", "r" } },
-    { INDEX_op_ld32u_i64, { "r", "r" } },
-    { INDEX_op_ld32s_i64, { "r", "r" } },
-    { INDEX_op_ld_i64, { "r", "r" } },
-
-    { INDEX_op_st8_i64, { "r", "r" } },
-    { INDEX_op_st16_i64, { "r", "r" } },
-    { INDEX_op_st32_i64, { "r", "r" } },
-    { INDEX_op_st_i64, { "r", "r" } },
-
-    { INDEX_op_add_i64, { "r", "r", "rT" } },
-    { INDEX_op_sub_i64, { "r", "rI", "rT" } },
-    { INDEX_op_and_i64, { "r", "r", "ri" } },
-    { INDEX_op_or_i64, { "r", "r", "rU" } },
-    { INDEX_op_xor_i64, { "r", "r", "rU" } },
-    { INDEX_op_andc_i64, { "r", "r", "ri" } },
-    { INDEX_op_orc_i64, { "r", "r", "r" } },
-    { INDEX_op_eqv_i64, { "r", "r", "r" } },
-    { INDEX_op_nand_i64, { "r", "r", "r" } },
-    { INDEX_op_nor_i64, { "r", "r", "r" } },
-
-    { INDEX_op_shl_i64, { "r", "r", "ri" } },
-    { INDEX_op_shr_i64, { "r", "r", "ri" } },
-    { INDEX_op_sar_i64, { "r", "r", "ri" } },
-    { INDEX_op_rotl_i64, { "r", "r", "ri" } },
-    { INDEX_op_rotr_i64, { "r", "r", "ri" } },
-
-    { INDEX_op_mul_i64, { "r", "r", "rI" } },
-    { INDEX_op_div_i64, { "r", "r", "r" } },
-    { INDEX_op_divu_i64, { "r", "r", "r" } },
-
-    { INDEX_op_neg_i64, { "r", "r" } },
-    { INDEX_op_not_i64, { "r", "r" } },
-    { INDEX_op_ext8s_i64, { "r", "r" } },
-    { INDEX_op_ext16s_i64, { "r", "r" } },
-    { INDEX_op_ext32s_i64, { "r", "r" } },
-    { INDEX_op_bswap16_i64, { "r", "r" } },
-    { INDEX_op_bswap32_i64, { "r", "r" } },
-    { INDEX_op_bswap64_i64, { "r", "r" } },
-
-    { INDEX_op_brcond_i64, { "r", "ri" } },
-    { INDEX_op_setcond_i64, { "r", "r", "ri" } },
-    { INDEX_op_movcond_i64, { "r", "r", "ri", "rZ", "rZ" } },
-
-    { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
-
-    { INDEX_op_mulsh_i64, { "r", "r", "r" } },
-    { INDEX_op_muluh_i64, { "r", "r", "r" } },
-#endif
-
-#if TCG_TARGET_REG_BITS == 32
-    { INDEX_op_brcond2_i32, { "r", "r", "ri", "ri" } },
-    { INDEX_op_setcond2_i32, { "r", "r", "r", "ri", "ri" } },
-#endif
-
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_add2_i64, { "r", "r", "r", "r", "rI", "rZM" } },
-    { INDEX_op_sub2_i64, { "r", "r", "rI", "rZM", "r", "r" } },
-#else
-    { INDEX_op_add2_i32, { "r", "r", "r", "r", "rI", "rZM" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rI", "rZM", "r", "r" } },
-#endif
-
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_qemu_ld_i32, { "r", "L" } },
-    { INDEX_op_qemu_st_i32, { "S", "S" } },
-    { INDEX_op_qemu_ld_i64, { "r", "L" } },
-    { INDEX_op_qemu_st_i64, { "S", "S" } },
-#elif TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld_i32, { "r", "L" } },
-    { INDEX_op_qemu_st_i32, { "S", "S" } },
-    { INDEX_op_qemu_ld_i64, { "L", "L", "L" } },
-    { INDEX_op_qemu_st_i64, { "S", "S", "S" } },
-#else
-    { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
-    { INDEX_op_qemu_st_i32, { "S", "S", "S" } },
-    { INDEX_op_qemu_ld_i64, { "L", "L", "L", "L" } },
-    { INDEX_op_qemu_st_i64, { "S", "S", "S", "S" } },
-#endif
-
-    { -1 },
-};
-
-static void tcg_target_init(TCGContext *s)
-{
-    unsigned long hwcap = qemu_getauxval(AT_HWCAP);
-    if (hwcap & PPC_FEATURE_ARCH_2_06) {
-        have_isa_2_06 = true;
-    }
-
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
-    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
-                     (1 << TCG_REG_R0) |
-                     (1 << TCG_REG_R2) |
-                     (1 << TCG_REG_R3) |
-                     (1 << TCG_REG_R4) |
-                     (1 << TCG_REG_R5) |
-                     (1 << TCG_REG_R6) |
-                     (1 << TCG_REG_R7) |
-                     (1 << TCG_REG_R8) |
-                     (1 << TCG_REG_R9) |
-                     (1 << TCG_REG_R10) |
-                     (1 << TCG_REG_R11) |
-                     (1 << TCG_REG_R12));
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
-#if defined(_CALL_SYSV)
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */
-#endif
-#if defined(_CALL_SYSV) || TCG_TARGET_REG_BITS == 64
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
-#endif
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); /* mem temp */
-    if (USE_REG_RA) {
-        tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);  /* return addr */
-    }
-
-    tcg_add_target_add_op_defs(ppc_op_defs);
-}
-
-#ifdef __ELF__
-typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
-    uint8_t fde_def_cfa[4];
-    uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
-} DebugFrame;
-
-/* We're expecting a 2 byte uleb128 encoded value.  */
-QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
-
-#if TCG_TARGET_REG_BITS == 64
-# define ELF_HOST_MACHINE EM_PPC64
-#else
-# define ELF_HOST_MACHINE EM_PPC
-#endif
-
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = (-SZR & 0x7f),         /* sleb128 -SZR */
-    .cie.return_column = 65,
-
-    /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
-
-    .fde_def_cfa = {
-        12, TCG_REG_R1,                 /* DW_CFA_def_cfa r1, ... */
-        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
-        (FRAME_SIZE >> 7)
-    },
-    .fde_reg_ofs = {
-        /* DW_CFA_offset_extended_sf, lr, LR_OFFSET */
-        0x11, 65, (LR_OFFSET / -SZR) & 0x7f,
-    }
-};
-
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    uint8_t *p = &debug_frame.fde_reg_ofs[3];
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
-        p[0] = 0x80 + tcg_target_callee_save_regs[i];
-        p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR;
-    }
-
-    debug_frame.fde.func_start = (uintptr_t)buf;
-    debug_frame.fde.func_len = buf_size;
-
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
-#endif /* __ELF__ */
-
-static size_t dcache_bsize = 16;
-static size_t icache_bsize = 16;
-
-void flush_icache_range(uintptr_t start, uintptr_t stop)
-{
-    uintptr_t p, start1, stop1;
-    size_t dsize = dcache_bsize;
-    size_t isize = icache_bsize;
-
-    start1 = start & ~(dsize - 1);
-    stop1 = (stop + dsize - 1) & ~(dsize - 1);
-    for (p = start1; p < stop1; p += dsize) {
-        asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
-    }
-    asm volatile ("sync" : : : "memory");
-
-    start &= start & ~(isize - 1);
-    stop1 = (stop + isize - 1) & ~(isize - 1);
-    for (p = start1; p < stop1; p += isize) {
-        asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
-    }
-    asm volatile ("sync" : : : "memory");
-    asm volatile ("isync" : : : "memory");
-}
-
-#if defined _AIX
-#include <sys/systemcfg.h>
-
-static void __attribute__((constructor)) tcg_cache_init(void)
-{
-    icache_bsize = _system_configuration.icache_line;
-    dcache_bsize = _system_configuration.dcache_line;
-}
-
-#elif defined __linux__
-static void __attribute__((constructor)) tcg_cache_init(void)
-{
-    unsigned long dsize = qemu_getauxval(AT_DCACHEBSIZE);
-    unsigned long isize = qemu_getauxval(AT_ICACHEBSIZE);
-
-    if (dsize == 0 || isize == 0) {
-        if (dsize == 0) {
-            fprintf(stderr, "getauxval AT_DCACHEBSIZE failed\n");
-        }
-        if (isize == 0) {
-            fprintf(stderr, "getauxval AT_ICACHEBSIZE failed\n");
-        }
-        exit(1);
-    }
-    dcache_bsize = dsize;
-    icache_bsize = isize;
-}
-
-#elif defined __APPLE__
-#include <stdio.h>
-#include <sys/types.h>
-#include <sys/sysctl.h>
-
-static void __attribute__((constructor)) tcg_cache_init(void)
-{
-    size_t len;
-    unsigned cacheline;
-    int name[2] = { CTL_HW, HW_CACHELINE };
-
-    len = sizeof(cacheline);
-    if (sysctl(name, 2, &cacheline, &len, NULL, 0)) {
-        perror("sysctl CTL_HW HW_CACHELINE failed");
-        exit(1);
-    }
-    dcache_bsize = cacheline;
-    icache_bsize = cacheline;
-}
-
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/sysctl.h>
-
-static void __attribute__((constructor)) tcg_cache_init(void)
-{
-    size_t len = 4;
-    unsigned cacheline;
-
-    if (sysctlbyname ("machdep.cacheline_size", &cacheline, &len, NULL, 0)) {
-        fprintf(stderr, "sysctlbyname machdep.cacheline_size failed: %s\n",
-                strerror(errno));
-        exit(1);
-    }
-    dcache_bsize = cacheline;
-    icache_bsize = cacheline;
-}
-#endif
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index 32ac442..dd032f2 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -21,8 +21,9 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_PPC64 
-#define TCG_TARGET_PPC64 1
+
+#ifndef PPC_TCG_TARGET_H
+#define PPC_TCG_TARGET_H
 
 #ifdef _ARCH_PPC64
 # define TCG_TARGET_REG_BITS  64
@@ -32,6 +33,7 @@
 
 #define TCG_TARGET_NB_REGS 32
 #define TCG_TARGET_INSN_UNIT_SIZE 4
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 16
 
 typedef enum {
     TCG_REG_R0,  TCG_REG_R1,  TCG_REG_R2,  TCG_REG_R3,
@@ -76,7 +78,8 @@
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_add2_i32         0
 #define TCG_TARGET_HAS_sub2_i32         0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_rot_i64          1
diff --git a/tcg/ppc/tcg-target.inc.c b/tcg/ppc/tcg-target.inc.c
new file mode 100644
index 0000000..eaf1bd9
--- /dev/null
+++ b/tcg/ppc/tcg-target.inc.c
@@ -0,0 +1,2785 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg-be-ldst.h"
+
+#if defined _CALL_DARWIN || defined __APPLE__
+#define TCG_TARGET_CALL_DARWIN
+#endif
+#ifdef _CALL_SYSV
+# define TCG_TARGET_CALL_ALIGN_ARGS   1
+#endif
+
+/* For some memory operations, we need a scratch that isn't R0.  For the AIX
+   calling convention, we can re-use the TOC register since we'll be reloading
+   it at every call.  Otherwise R12 will do nicely as neither a call-saved
+   register nor a parameter register.  */
+#ifdef _CALL_AIX
+# define TCG_REG_TMP1   TCG_REG_R2
+#else
+# define TCG_REG_TMP1   TCG_REG_R12
+#endif
+
+/* For the 64-bit target, we don't like the 5 insn sequence needed to build
+   full 64-bit addresses.  Better to have a base register to which we can
+   apply a 32-bit displacement.
+
+   There are generally three items of interest:
+   (1) helper functions in the main executable,
+   (2) TranslationBlock data structures,
+   (3) the return address in the epilogue.
+
+   For user-only, we USE_STATIC_CODE_GEN_BUFFER, so the code_gen_buffer
+   will be inside the main executable, and thus near enough to make a
+   pointer to the epilogue be within 2GB of all helper functions.
+
+   For softmmu, we'll let the kernel choose the address of code_gen_buffer,
+   and odds are it'll be somewhere close to the main malloc arena, and so
+   a pointer to the epilogue will be within 2GB of the TranslationBlocks.
+
+   For --enable-pie, everything will be kinda near everything else,
+   somewhere in high memory.
+
+   Thus we choose to keep the return address in a call-saved register.  */
+#define TCG_REG_RA     TCG_REG_R31
+#define USE_REG_RA     (TCG_TARGET_REG_BITS == 64)
+
+/* Shorthand for size of a pointer.  Avoid promotion to unsigned.  */
+#define SZP  ((int)sizeof(void *))
+
+/* Shorthand for size of a register.  */
+#define SZR  (TCG_TARGET_REG_BITS / 8)
+
+#define TCG_CT_CONST_S16  0x100
+#define TCG_CT_CONST_U16  0x200
+#define TCG_CT_CONST_S32  0x400
+#define TCG_CT_CONST_U32  0x800
+#define TCG_CT_CONST_ZERO 0x1000
+#define TCG_CT_CONST_MONE 0x2000
+
+static tcg_insn_unit *tb_ret_addr;
+
+#include "elf.h"
+static bool have_isa_2_06;
+#define HAVE_ISA_2_06  have_isa_2_06
+#define HAVE_ISEL      have_isa_2_06
+
+#ifndef CONFIG_SOFTMMU
+#define TCG_GUEST_BASE_REG 30
+#endif
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "r0",
+    "r1",
+    "r2",
+    "r3",
+    "r4",
+    "r5",
+    "r6",
+    "r7",
+    "r8",
+    "r9",
+    "r10",
+    "r11",
+    "r12",
+    "r13",
+    "r14",
+    "r15",
+    "r16",
+    "r17",
+    "r18",
+    "r19",
+    "r20",
+    "r21",
+    "r22",
+    "r23",
+    "r24",
+    "r25",
+    "r26",
+    "r27",
+    "r28",
+    "r29",
+    "r30",
+    "r31"
+};
+#endif
+
+static const int tcg_target_reg_alloc_order[] = {
+    TCG_REG_R14,  /* call saved registers */
+    TCG_REG_R15,
+    TCG_REG_R16,
+    TCG_REG_R17,
+    TCG_REG_R18,
+    TCG_REG_R19,
+    TCG_REG_R20,
+    TCG_REG_R21,
+    TCG_REG_R22,
+    TCG_REG_R23,
+    TCG_REG_R24,
+    TCG_REG_R25,
+    TCG_REG_R26,
+    TCG_REG_R27,
+    TCG_REG_R28,
+    TCG_REG_R29,
+    TCG_REG_R30,
+    TCG_REG_R31,
+    TCG_REG_R12,  /* call clobbered, non-arguments */
+    TCG_REG_R11,
+    TCG_REG_R2,
+    TCG_REG_R13,
+    TCG_REG_R10,  /* call clobbered, arguments */
+    TCG_REG_R9,
+    TCG_REG_R8,
+    TCG_REG_R7,
+    TCG_REG_R6,
+    TCG_REG_R5,
+    TCG_REG_R4,
+    TCG_REG_R3,
+};
+
+static const int tcg_target_call_iarg_regs[] = {
+    TCG_REG_R3,
+    TCG_REG_R4,
+    TCG_REG_R5,
+    TCG_REG_R6,
+    TCG_REG_R7,
+    TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10
+};
+
+static const int tcg_target_call_oarg_regs[] = {
+    TCG_REG_R3,
+    TCG_REG_R4
+};
+
+static const int tcg_target_callee_save_regs[] = {
+#ifdef TCG_TARGET_CALL_DARWIN
+    TCG_REG_R11,
+#endif
+    TCG_REG_R14,
+    TCG_REG_R15,
+    TCG_REG_R16,
+    TCG_REG_R17,
+    TCG_REG_R18,
+    TCG_REG_R19,
+    TCG_REG_R20,
+    TCG_REG_R21,
+    TCG_REG_R22,
+    TCG_REG_R23,
+    TCG_REG_R24,
+    TCG_REG_R25,
+    TCG_REG_R26,
+    TCG_REG_R27, /* currently used for the global env */
+    TCG_REG_R28,
+    TCG_REG_R29,
+    TCG_REG_R30,
+    TCG_REG_R31
+};
+
+static inline bool in_range_b(tcg_target_long target)
+{
+    return target == sextract64(target, 0, 26);
+}
+
+static uint32_t reloc_pc24_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
+    tcg_debug_assert(in_range_b(disp));
+    return disp & 0x3fffffc;
+}
+
+static void reloc_pc24(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    *pc = (*pc & ~0x3fffffc) | reloc_pc24_val(pc, target);
+}
+
+static uint16_t reloc_pc14_val(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
+    tcg_debug_assert(disp == (int16_t) disp);
+    return disp & 0xfffc;
+}
+
+static void reloc_pc14(tcg_insn_unit *pc, tcg_insn_unit *target)
+{
+    *pc = (*pc & ~0xfffc) | reloc_pc14_val(pc, target);
+}
+
+static inline void tcg_out_b_noaddr(TCGContext *s, int insn)
+{
+    unsigned retrans = *s->code_ptr & 0x3fffffc;
+    tcg_out32(s, insn | retrans);
+}
+
+static inline void tcg_out_bc_noaddr(TCGContext *s, int insn)
+{
+    unsigned retrans = *s->code_ptr & 0xfffc;
+    tcg_out32(s, insn | retrans);
+}
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    tcg_insn_unit *target = (tcg_insn_unit *)value;
+
+    tcg_debug_assert(addend == 0);
+    switch (type) {
+    case R_PPC_REL14:
+        reloc_pc14(code_ptr, target);
+        break;
+    case R_PPC_REL24:
+        reloc_pc24(code_ptr, target);
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch (ct_str[0]) {
+    case 'A': case 'B': case 'C': case 'D':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set_reg(ct->u.regs, 3 + ct_str[0] - 'A');
+        break;
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        break;
+    case 'L':                   /* qemu_ld constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
+#ifdef CONFIG_SOFTMMU
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
+#endif
+        break;
+    case 'S':                   /* qemu_st constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
+#ifdef CONFIG_SOFTMMU
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
+#endif
+        break;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_S16;
+        break;
+    case 'J':
+        ct->ct |= TCG_CT_CONST_U16;
+        break;
+    case 'M':
+        ct->ct |= TCG_CT_CONST_MONE;
+        break;
+    case 'T':
+        ct->ct |= TCG_CT_CONST_S32;
+        break;
+    case 'U':
+        ct->ct |= TCG_CT_CONST_U32;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                  const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    }
+
+    /* The only 32-bit constraint we use aside from
+       TCG_CT_CONST is TCG_CT_CONST_S16.  */
+    if (type == TCG_TYPE_I32) {
+        val = (int32_t)val;
+    }
+
+    if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_U32) && val == (uint32_t)val) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_MONE) && val == -1) {
+        return 1;
+    }
+    return 0;
+}
+
+#define OPCD(opc) ((opc)<<26)
+#define XO19(opc) (OPCD(19)|((opc)<<1))
+#define MD30(opc) (OPCD(30)|((opc)<<2))
+#define MDS30(opc) (OPCD(30)|((opc)<<1))
+#define XO31(opc) (OPCD(31)|((opc)<<1))
+#define XO58(opc) (OPCD(58)|(opc))
+#define XO62(opc) (OPCD(62)|(opc))
+
+#define B      OPCD( 18)
+#define BC     OPCD( 16)
+#define LBZ    OPCD( 34)
+#define LHZ    OPCD( 40)
+#define LHA    OPCD( 42)
+#define LWZ    OPCD( 32)
+#define STB    OPCD( 38)
+#define STH    OPCD( 44)
+#define STW    OPCD( 36)
+
+#define STD    XO62(  0)
+#define STDU   XO62(  1)
+#define STDX   XO31(149)
+
+#define LD     XO58(  0)
+#define LDX    XO31( 21)
+#define LDU    XO58(  1)
+#define LWA    XO58(  2)
+#define LWAX   XO31(341)
+
+#define ADDIC  OPCD( 12)
+#define ADDI   OPCD( 14)
+#define ADDIS  OPCD( 15)
+#define ORI    OPCD( 24)
+#define ORIS   OPCD( 25)
+#define XORI   OPCD( 26)
+#define XORIS  OPCD( 27)
+#define ANDI   OPCD( 28)
+#define ANDIS  OPCD( 29)
+#define MULLI  OPCD(  7)
+#define CMPLI  OPCD( 10)
+#define CMPI   OPCD( 11)
+#define SUBFIC OPCD( 8)
+
+#define LWZU   OPCD( 33)
+#define STWU   OPCD( 37)
+
+#define RLWIMI OPCD( 20)
+#define RLWINM OPCD( 21)
+#define RLWNM  OPCD( 23)
+
+#define RLDICL MD30(  0)
+#define RLDICR MD30(  1)
+#define RLDIMI MD30(  3)
+#define RLDCL  MDS30( 8)
+
+#define BCLR   XO19( 16)
+#define BCCTR  XO19(528)
+#define CRAND  XO19(257)
+#define CRANDC XO19(129)
+#define CRNAND XO19(225)
+#define CROR   XO19(449)
+#define CRNOR  XO19( 33)
+
+#define EXTSB  XO31(954)
+#define EXTSH  XO31(922)
+#define EXTSW  XO31(986)
+#define ADD    XO31(266)
+#define ADDE   XO31(138)
+#define ADDME  XO31(234)
+#define ADDZE  XO31(202)
+#define ADDC   XO31( 10)
+#define AND    XO31( 28)
+#define SUBF   XO31( 40)
+#define SUBFC  XO31(  8)
+#define SUBFE  XO31(136)
+#define SUBFME XO31(232)
+#define SUBFZE XO31(200)
+#define OR     XO31(444)
+#define XOR    XO31(316)
+#define MULLW  XO31(235)
+#define MULHW  XO31( 75)
+#define MULHWU XO31( 11)
+#define DIVW   XO31(491)
+#define DIVWU  XO31(459)
+#define CMP    XO31(  0)
+#define CMPL   XO31( 32)
+#define LHBRX  XO31(790)
+#define LWBRX  XO31(534)
+#define LDBRX  XO31(532)
+#define STHBRX XO31(918)
+#define STWBRX XO31(662)
+#define STDBRX XO31(660)
+#define MFSPR  XO31(339)
+#define MTSPR  XO31(467)
+#define SRAWI  XO31(824)
+#define NEG    XO31(104)
+#define MFCR   XO31( 19)
+#define MFOCRF (MFCR | (1u << 20))
+#define NOR    XO31(124)
+#define CNTLZW XO31( 26)
+#define CNTLZD XO31( 58)
+#define ANDC   XO31( 60)
+#define ORC    XO31(412)
+#define EQV    XO31(284)
+#define NAND   XO31(476)
+#define ISEL   XO31( 15)
+
+#define MULLD  XO31(233)
+#define MULHD  XO31( 73)
+#define MULHDU XO31(  9)
+#define DIVD   XO31(489)
+#define DIVDU  XO31(457)
+
+#define LBZX   XO31( 87)
+#define LHZX   XO31(279)
+#define LHAX   XO31(343)
+#define LWZX   XO31( 23)
+#define STBX   XO31(215)
+#define STHX   XO31(407)
+#define STWX   XO31(151)
+
+#define SPR(a, b) ((((a)<<5)|(b))<<11)
+#define LR     SPR(8, 0)
+#define CTR    SPR(9, 0)
+
+#define SLW    XO31( 24)
+#define SRW    XO31(536)
+#define SRAW   XO31(792)
+
+#define SLD    XO31( 27)
+#define SRD    XO31(539)
+#define SRAD   XO31(794)
+#define SRADI  XO31(413<<1)
+
+#define TW     XO31( 4)
+#define TRAP   (TW | TO(31))
+
+#define NOP    ORI  /* ori 0,0,0 */
+
+#define RT(r) ((r)<<21)
+#define RS(r) ((r)<<21)
+#define RA(r) ((r)<<16)
+#define RB(r) ((r)<<11)
+#define TO(t) ((t)<<21)
+#define SH(s) ((s)<<11)
+#define MB(b) ((b)<<6)
+#define ME(e) ((e)<<1)
+#define BO(o) ((o)<<21)
+#define MB64(b) ((b)<<5)
+#define FXM(b) (1 << (19 - (b)))
+
+#define LK    1
+
+#define TAB(t, a, b) (RT(t) | RA(a) | RB(b))
+#define SAB(s, a, b) (RS(s) | RA(a) | RB(b))
+#define TAI(s, a, i) (RT(s) | RA(a) | ((i) & 0xffff))
+#define SAI(s, a, i) (RS(s) | RA(a) | ((i) & 0xffff))
+
+#define BF(n)    ((n)<<23)
+#define BI(n, c) (((c)+((n)*4))<<16)
+#define BT(n, c) (((c)+((n)*4))<<21)
+#define BA(n, c) (((c)+((n)*4))<<16)
+#define BB(n, c) (((c)+((n)*4))<<11)
+#define BC_(n, c) (((c)+((n)*4))<<6)
+
+#define BO_COND_TRUE  BO(12)
+#define BO_COND_FALSE BO( 4)
+#define BO_ALWAYS     BO(20)
+
+enum {
+    CR_LT,
+    CR_GT,
+    CR_EQ,
+    CR_SO
+};
+
+static const uint32_t tcg_to_bc[] = {
+    [TCG_COND_EQ]  = BC | BI(7, CR_EQ) | BO_COND_TRUE,
+    [TCG_COND_NE]  = BC | BI(7, CR_EQ) | BO_COND_FALSE,
+    [TCG_COND_LT]  = BC | BI(7, CR_LT) | BO_COND_TRUE,
+    [TCG_COND_GE]  = BC | BI(7, CR_LT) | BO_COND_FALSE,
+    [TCG_COND_LE]  = BC | BI(7, CR_GT) | BO_COND_FALSE,
+    [TCG_COND_GT]  = BC | BI(7, CR_GT) | BO_COND_TRUE,
+    [TCG_COND_LTU] = BC | BI(7, CR_LT) | BO_COND_TRUE,
+    [TCG_COND_GEU] = BC | BI(7, CR_LT) | BO_COND_FALSE,
+    [TCG_COND_LEU] = BC | BI(7, CR_GT) | BO_COND_FALSE,
+    [TCG_COND_GTU] = BC | BI(7, CR_GT) | BO_COND_TRUE,
+};
+
+/* The low bit here is set if the RA and RB fields must be inverted.  */
+static const uint32_t tcg_to_isel[] = {
+    [TCG_COND_EQ]  = ISEL | BC_(7, CR_EQ),
+    [TCG_COND_NE]  = ISEL | BC_(7, CR_EQ) | 1,
+    [TCG_COND_LT]  = ISEL | BC_(7, CR_LT),
+    [TCG_COND_GE]  = ISEL | BC_(7, CR_LT) | 1,
+    [TCG_COND_LE]  = ISEL | BC_(7, CR_GT) | 1,
+    [TCG_COND_GT]  = ISEL | BC_(7, CR_GT),
+    [TCG_COND_LTU] = ISEL | BC_(7, CR_LT),
+    [TCG_COND_GEU] = ISEL | BC_(7, CR_LT) | 1,
+    [TCG_COND_LEU] = ISEL | BC_(7, CR_GT) | 1,
+    [TCG_COND_GTU] = ISEL | BC_(7, CR_GT),
+};
+
+static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
+                             TCGReg base, tcg_target_long offset);
+
+static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    if (ret != arg) {
+        tcg_out32(s, OR | SAB(arg, ret, arg));
+    }
+}
+
+static inline void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
+                               int sh, int mb)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    sh = SH(sh & 0x1f) | (((sh >> 5) & 1) << 1);
+    mb = MB64((mb >> 5) | ((mb << 1) & 0x3f));
+    tcg_out32(s, op | RA(ra) | RS(rs) | sh | mb);
+}
+
+static inline void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
+                               int sh, int mb, int me)
+{
+    tcg_out32(s, op | RA(ra) | RS(rs) | SH(sh) | MB(mb) | ME(me));
+}
+
+static inline void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
+{
+    tcg_out_rld(s, RLDICL, dst, src, 0, 32);
+}
+
+static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
+{
+    tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
+}
+
+static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c)
+{
+    tcg_out_rld(s, RLDICR, dst, src, c, 63 - c);
+}
+
+static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c)
+{
+    tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31);
+}
+
+static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c)
+{
+    tcg_out_rld(s, RLDICL, dst, src, 64 - c, c);
+}
+
+static void tcg_out_movi32(TCGContext *s, TCGReg ret, int32_t arg)
+{
+    if (arg == (int16_t) arg) {
+        tcg_out32(s, ADDI | TAI(ret, 0, arg));
+    } else {
+        tcg_out32(s, ADDIS | TAI(ret, 0, arg >> 16));
+        if (arg & 0xffff) {
+            tcg_out32(s, ORI | SAI(ret, ret, arg));
+        }
+    }
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
+                         tcg_target_long arg)
+{
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    if (type == TCG_TYPE_I32 || arg == (int32_t)arg) {
+        tcg_out_movi32(s, ret, arg);
+    } else if (arg == (uint32_t)arg && !(arg & 0x8000)) {
+        tcg_out32(s, ADDI | TAI(ret, 0, arg));
+        tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
+    } else {
+        int32_t high;
+
+        if (USE_REG_RA) {
+            intptr_t diff = arg - (intptr_t)tb_ret_addr;
+            if (diff == (int32_t)diff) {
+                tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_RA, diff);
+                return;
+            }
+        }
+
+        high = arg >> 31 >> 1;
+        tcg_out_movi32(s, ret, high);
+        if (high) {
+            tcg_out_shli64(s, ret, ret, 32);
+        }
+        if (arg & 0xffff0000) {
+            tcg_out32(s, ORIS | SAI(ret, ret, arg >> 16));
+        }
+        if (arg & 0xffff) {
+            tcg_out32(s, ORI | SAI(ret, ret, arg));
+        }
+    }
+}
+
+static bool mask_operand(uint32_t c, int *mb, int *me)
+{
+    uint32_t lsb, test;
+
+    /* Accept a bit pattern like:
+           0....01....1
+           1....10....0
+           0..01..10..0
+       Keep track of the transitions.  */
+    if (c == 0 || c == -1) {
+        return false;
+    }
+    test = c;
+    lsb = test & -test;
+    test += lsb;
+    if (test & (test - 1)) {
+        return false;
+    }
+
+    *me = clz32(lsb);
+    *mb = test ? clz32(test & -test) + 1 : 0;
+    return true;
+}
+
+static bool mask64_operand(uint64_t c, int *mb, int *me)
+{
+    uint64_t lsb;
+
+    if (c == 0) {
+        return false;
+    }
+
+    lsb = c & -c;
+    /* Accept 1..10..0.  */
+    if (c == -lsb) {
+        *mb = 0;
+        *me = clz64(lsb);
+        return true;
+    }
+    /* Accept 0..01..1.  */
+    if (lsb == 1 && (c & (c + 1)) == 0) {
+        *mb = clz64(c + 1) + 1;
+        *me = 63;
+        return true;
+    }
+    return false;
+}
+
+static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
+{
+    int mb, me;
+
+    if (mask_operand(c, &mb, &me)) {
+        tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
+    } else if ((c & 0xffff) == c) {
+        tcg_out32(s, ANDI | SAI(src, dst, c));
+        return;
+    } else if ((c & 0xffff0000) == c) {
+        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
+        return;
+    } else {
+        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
+        tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
+    }
+}
+
+static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
+{
+    int mb, me;
+
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
+    if (mask64_operand(c, &mb, &me)) {
+        if (mb == 0) {
+            tcg_out_rld(s, RLDICR, dst, src, 0, me);
+        } else {
+            tcg_out_rld(s, RLDICL, dst, src, 0, mb);
+        }
+    } else if ((c & 0xffff) == c) {
+        tcg_out32(s, ANDI | SAI(src, dst, c));
+        return;
+    } else if ((c & 0xffff0000) == c) {
+        tcg_out32(s, ANDIS | SAI(src, dst, c >> 16));
+        return;
+    } else {
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
+        tcg_out32(s, AND | SAB(src, dst, TCG_REG_R0));
+    }
+}
+
+static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c,
+                           int op_lo, int op_hi)
+{
+    if (c >> 16) {
+        tcg_out32(s, op_hi | SAI(src, dst, c >> 16));
+        src = dst;
+    }
+    if (c & 0xffff) {
+        tcg_out32(s, op_lo | SAI(src, dst, c));
+        src = dst;
+    }
+}
+
+static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
+{
+    tcg_out_zori32(s, dst, src, c, ORI, ORIS);
+}
+
+static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
+{
+    tcg_out_zori32(s, dst, src, c, XORI, XORIS);
+}
+
+static void tcg_out_b(TCGContext *s, int mask, tcg_insn_unit *target)
+{
+    ptrdiff_t disp = tcg_pcrel_diff(s, target);
+    if (in_range_b(disp)) {
+        tcg_out32(s, B | (disp & 0x3fffffc) | mask);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target);
+        tcg_out32(s, MTSPR | RS(TCG_REG_R0) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS | mask);
+    }
+}
+
+static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
+                             TCGReg base, tcg_target_long offset)
+{
+    tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
+    bool is_store = false;
+    TCGReg rs = TCG_REG_TMP1;
+
+    switch (opi) {
+    case LD: case LWA:
+        align = 3;
+        /* FALLTHRU */
+    default:
+        if (rt != TCG_REG_R0) {
+            rs = rt;
+            break;
+        }
+        break;
+    case STD:
+        align = 3;
+        /* FALLTHRU */
+    case STB: case STH: case STW:
+        is_store = true;
+        break;
+    }
+
+    /* For unaligned, or very large offsets, use the indexed form.  */
+    if (offset & align || offset != (int32_t)offset) {
+        if (rs == base) {
+            rs = TCG_REG_R0;
+        }
+        tcg_debug_assert(!is_store || rs != rt);
+        tcg_out_movi(s, TCG_TYPE_PTR, rs, orig);
+        tcg_out32(s, opx | TAB(rt, base, rs));
+        return;
+    }
+
+    l0 = (int16_t)offset;
+    offset = (offset - l0) >> 16;
+    l1 = (int16_t)offset;
+
+    if (l1 < 0 && orig >= 0) {
+        extra = 0x4000;
+        l1 = (int16_t)(offset - 0x4000);
+    }
+    if (l1) {
+        tcg_out32(s, ADDIS | TAI(rs, base, l1));
+        base = rs;
+    }
+    if (extra) {
+        tcg_out32(s, ADDIS | TAI(rs, base, extra));
+        base = rs;
+    }
+    if (opi != ADDI || base != rt || l0 != 0) {
+        tcg_out32(s, opi | TAI(rt, base, l0));
+    }
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
+                              TCGReg arg1, intptr_t arg2)
+{
+    int opi, opx;
+
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    if (type == TCG_TYPE_I32) {
+        opi = LWZ, opx = LWZX;
+    } else {
+        opi = LD, opx = LDX;
+    }
+    tcg_out_mem_long(s, opi, opx, ret, arg1, arg2);
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    int opi, opx;
+
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+    if (type == TCG_TYPE_I32) {
+        opi = STW, opx = STWX;
+    } else {
+        opi = STD, opx = STDX;
+    }
+    tcg_out_mem_long(s, opi, opx, arg, arg1, arg2);
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    return false;
+}
+
+static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
+                        int const_arg2, int cr, TCGType type)
+{
+    int imm;
+    uint32_t op;
+
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+
+    /* Simplify the comparisons below wrt CMPI.  */
+    if (type == TCG_TYPE_I32) {
+        arg2 = (int32_t)arg2;
+    }
+
+    switch (cond) {
+    case TCG_COND_EQ:
+    case TCG_COND_NE:
+        if (const_arg2) {
+            if ((int16_t) arg2 == arg2) {
+                op = CMPI;
+                imm = 1;
+                break;
+            } else if ((uint16_t) arg2 == arg2) {
+                op = CMPLI;
+                imm = 1;
+                break;
+            }
+        }
+        op = CMPL;
+        imm = 0;
+        break;
+
+    case TCG_COND_LT:
+    case TCG_COND_GE:
+    case TCG_COND_LE:
+    case TCG_COND_GT:
+        if (const_arg2) {
+            if ((int16_t) arg2 == arg2) {
+                op = CMPI;
+                imm = 1;
+                break;
+            }
+        }
+        op = CMP;
+        imm = 0;
+        break;
+
+    case TCG_COND_LTU:
+    case TCG_COND_GEU:
+    case TCG_COND_LEU:
+    case TCG_COND_GTU:
+        if (const_arg2) {
+            if ((uint16_t) arg2 == arg2) {
+                op = CMPLI;
+                imm = 1;
+                break;
+            }
+        }
+        op = CMPL;
+        imm = 0;
+        break;
+
+    default:
+        tcg_abort();
+    }
+    op |= BF(cr) | ((type == TCG_TYPE_I64) << 21);
+
+    if (imm) {
+        tcg_out32(s, op | RA(arg1) | (arg2 & 0xffff));
+    } else {
+        if (const_arg2) {
+            tcg_out_movi(s, type, TCG_REG_R0, arg2);
+            arg2 = TCG_REG_R0;
+        }
+        tcg_out32(s, op | RA(arg1) | RB(arg2));
+    }
+}
+
+static void tcg_out_setcond_eq0(TCGContext *s, TCGType type,
+                                TCGReg dst, TCGReg src)
+{
+    if (type == TCG_TYPE_I32) {
+        tcg_out32(s, CNTLZW | RS(src) | RA(dst));
+        tcg_out_shri32(s, dst, dst, 5);
+    } else {
+        tcg_out32(s, CNTLZD | RS(src) | RA(dst));
+        tcg_out_shri64(s, dst, dst, 6);
+    }
+}
+
+static void tcg_out_setcond_ne0(TCGContext *s, TCGReg dst, TCGReg src)
+{
+    /* X != 0 implies X + -1 generates a carry.  Extra addition
+       trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C.  */
+    if (dst != src) {
+        tcg_out32(s, ADDIC | TAI(dst, src, -1));
+        tcg_out32(s, SUBFE | TAB(dst, dst, src));
+    } else {
+        tcg_out32(s, ADDIC | TAI(TCG_REG_R0, src, -1));
+        tcg_out32(s, SUBFE | TAB(dst, TCG_REG_R0, src));
+    }
+}
+
+static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2,
+                                  bool const_arg2)
+{
+    if (const_arg2) {
+        if ((uint32_t)arg2 == arg2) {
+            tcg_out_xori32(s, TCG_REG_R0, arg1, arg2);
+        } else {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2);
+            tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, TCG_REG_R0));
+        }
+    } else {
+        tcg_out32(s, XOR | SAB(arg1, TCG_REG_R0, arg2));
+    }
+    return TCG_REG_R0;
+}
+
+static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
+                            TCGArg arg0, TCGArg arg1, TCGArg arg2,
+                            int const_arg2)
+{
+    int crop, sh;
+
+    tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
+
+    /* Ignore high bits of a potential constant arg2.  */
+    if (type == TCG_TYPE_I32) {
+        arg2 = (uint32_t)arg2;
+    }
+
+    /* Handle common and trivial cases before handling anything else.  */
+    if (arg2 == 0) {
+        switch (cond) {
+        case TCG_COND_EQ:
+            tcg_out_setcond_eq0(s, type, arg0, arg1);
+            return;
+        case TCG_COND_NE:
+            if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
+                tcg_out_ext32u(s, TCG_REG_R0, arg1);
+                arg1 = TCG_REG_R0;
+            }
+            tcg_out_setcond_ne0(s, arg0, arg1);
+            return;
+        case TCG_COND_GE:
+            tcg_out32(s, NOR | SAB(arg1, arg0, arg1));
+            arg1 = arg0;
+            /* FALLTHRU */
+        case TCG_COND_LT:
+            /* Extract the sign bit.  */
+            if (type == TCG_TYPE_I32) {
+                tcg_out_shri32(s, arg0, arg1, 31);
+            } else {
+                tcg_out_shri64(s, arg0, arg1, 63);
+            }
+            return;
+        default:
+            break;
+        }
+    }
+
+    /* If we have ISEL, we can implement everything with 3 or 4 insns.
+       All other cases below are also at least 3 insns, so speed up the
+       code generator by not considering them and always using ISEL.  */
+    if (HAVE_ISEL) {
+        int isel, tab;
+
+        tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
+
+        isel = tcg_to_isel[cond];
+
+        tcg_out_movi(s, type, arg0, 1);
+        if (isel & 1) {
+            /* arg0 = (bc ? 0 : 1) */
+            tab = TAB(arg0, 0, arg0);
+            isel &= ~1;
+        } else {
+            /* arg0 = (bc ? 1 : 0) */
+            tcg_out_movi(s, type, TCG_REG_R0, 0);
+            tab = TAB(arg0, arg0, TCG_REG_R0);
+        }
+        tcg_out32(s, isel | tab);
+        return;
+    }
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
+        tcg_out_setcond_eq0(s, type, arg0, arg1);
+        return;
+
+    case TCG_COND_NE:
+        arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
+        /* Discard the high bits only once, rather than both inputs.  */
+        if (TCG_TARGET_REG_BITS == 64 && type == TCG_TYPE_I32) {
+            tcg_out_ext32u(s, TCG_REG_R0, arg1);
+            arg1 = TCG_REG_R0;
+        }
+        tcg_out_setcond_ne0(s, arg0, arg1);
+        return;
+
+    case TCG_COND_GT:
+    case TCG_COND_GTU:
+        sh = 30;
+        crop = 0;
+        goto crtest;
+
+    case TCG_COND_LT:
+    case TCG_COND_LTU:
+        sh = 29;
+        crop = 0;
+        goto crtest;
+
+    case TCG_COND_GE:
+    case TCG_COND_GEU:
+        sh = 31;
+        crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_LT) | BB(7, CR_LT);
+        goto crtest;
+
+    case TCG_COND_LE:
+    case TCG_COND_LEU:
+        sh = 31;
+        crop = CRNOR | BT(7, CR_EQ) | BA(7, CR_GT) | BB(7, CR_GT);
+    crtest:
+        tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
+        if (crop) {
+            tcg_out32(s, crop);
+        }
+        tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
+        tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31);
+        break;
+
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_bc(TCGContext *s, int bc, TCGLabel *l)
+{
+    if (l->has_value) {
+        tcg_out32(s, bc | reloc_pc14_val(s->code_ptr, l->u.value_ptr));
+    } else {
+        tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0);
+        tcg_out_bc_noaddr(s, bc);
+    }
+}
+
+static void tcg_out_brcond(TCGContext *s, TCGCond cond,
+                           TCGArg arg1, TCGArg arg2, int const_arg2,
+                           TCGLabel *l, TCGType type)
+{
+    tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
+    tcg_out_bc(s, tcg_to_bc[cond], l);
+}
+
+static void tcg_out_movcond(TCGContext *s, TCGType type, TCGCond cond,
+                            TCGArg dest, TCGArg c1, TCGArg c2, TCGArg v1,
+                            TCGArg v2, bool const_c2)
+{
+    /* If for some reason both inputs are zero, don't produce bad code.  */
+    if (v1 == 0 && v2 == 0) {
+        tcg_out_movi(s, type, dest, 0);
+        return;
+    }
+
+    tcg_out_cmp(s, cond, c1, c2, const_c2, 7, type);
+
+    if (HAVE_ISEL) {
+        int isel = tcg_to_isel[cond];
+
+        /* Swap the V operands if the operation indicates inversion.  */
+        if (isel & 1) {
+            int t = v1;
+            v1 = v2;
+            v2 = t;
+            isel &= ~1;
+        }
+        /* V1 == 0 is handled by isel; V2 == 0 must be handled by hand.  */
+        if (v2 == 0) {
+            tcg_out_movi(s, type, TCG_REG_R0, 0);
+        }
+        tcg_out32(s, isel | TAB(dest, v1, v2));
+    } else {
+        if (dest == v2) {
+            cond = tcg_invert_cond(cond);
+            v2 = v1;
+        } else if (dest != v1) {
+            if (v1 == 0) {
+                tcg_out_movi(s, type, dest, 0);
+            } else {
+                tcg_out_mov(s, type, dest, v1);
+            }
+        }
+        /* Branch forward over one insn */
+        tcg_out32(s, tcg_to_bc[cond] | 8);
+        if (v2 == 0) {
+            tcg_out_movi(s, type, dest, 0);
+        } else {
+            tcg_out_mov(s, type, dest, v2);
+        }
+    }
+}
+
+static void tcg_out_cmp2(TCGContext *s, const TCGArg *args,
+                         const int *const_args)
+{
+    static const struct { uint8_t bit1, bit2; } bits[] = {
+        [TCG_COND_LT ] = { CR_LT, CR_LT },
+        [TCG_COND_LE ] = { CR_LT, CR_GT },
+        [TCG_COND_GT ] = { CR_GT, CR_GT },
+        [TCG_COND_GE ] = { CR_GT, CR_LT },
+        [TCG_COND_LTU] = { CR_LT, CR_LT },
+        [TCG_COND_LEU] = { CR_LT, CR_GT },
+        [TCG_COND_GTU] = { CR_GT, CR_GT },
+        [TCG_COND_GEU] = { CR_GT, CR_LT },
+    };
+
+    TCGCond cond = args[4], cond2;
+    TCGArg al, ah, bl, bh;
+    int blconst, bhconst;
+    int op, bit1, bit2;
+
+    al = args[0];
+    ah = args[1];
+    bl = args[2];
+    bh = args[3];
+    blconst = const_args[2];
+    bhconst = const_args[3];
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        op = CRAND;
+        goto do_equality;
+    case TCG_COND_NE:
+        op = CRNAND;
+    do_equality:
+        tcg_out_cmp(s, cond, al, bl, blconst, 6, TCG_TYPE_I32);
+        tcg_out_cmp(s, cond, ah, bh, bhconst, 7, TCG_TYPE_I32);
+        tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
+        break;
+
+    case TCG_COND_LT:
+    case TCG_COND_LE:
+    case TCG_COND_GT:
+    case TCG_COND_GE:
+    case TCG_COND_LTU:
+    case TCG_COND_LEU:
+    case TCG_COND_GTU:
+    case TCG_COND_GEU:
+        bit1 = bits[cond].bit1;
+        bit2 = bits[cond].bit2;
+        op = (bit1 != bit2 ? CRANDC : CRAND);
+        cond2 = tcg_unsigned_cond(cond);
+
+        tcg_out_cmp(s, cond, ah, bh, bhconst, 6, TCG_TYPE_I32);
+        tcg_out_cmp(s, cond2, al, bl, blconst, 7, TCG_TYPE_I32);
+        tcg_out32(s, op | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, bit2));
+        tcg_out32(s, CROR | BT(7, CR_EQ) | BA(6, bit1) | BB(7, CR_EQ));
+        break;
+
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
+                             const int *const_args)
+{
+    tcg_out_cmp2(s, args + 1, const_args + 1);
+    tcg_out32(s, MFOCRF | RT(TCG_REG_R0) | FXM(7));
+    tcg_out_rlw(s, RLWINM, args[0], TCG_REG_R0, 31, 31, 31);
+}
+
+static void tcg_out_brcond2 (TCGContext *s, const TCGArg *args,
+                             const int *const_args)
+{
+    tcg_out_cmp2(s, args, const_args);
+    tcg_out_bc(s, BC | BI(7, CR_EQ) | BO_COND_TRUE, arg_label(args[5]));
+}
+
+#ifdef __powerpc64__
+void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
+{
+    tcg_insn_unit i1, i2;
+    uint64_t pair;
+    intptr_t diff = addr - jmp_addr;
+
+    if (in_range_b(diff)) {
+        i1 = B | (diff & 0x3fffffc);
+        i2 = NOP;
+    } else if (USE_REG_RA) {
+        intptr_t lo, hi;
+        diff = addr - (uintptr_t)tb_ret_addr;
+        lo = (int16_t)diff;
+        hi = (int32_t)(diff - lo);
+        tcg_debug_assert(diff == hi + lo);
+        i1 = ADDIS | TAI(TCG_REG_TMP1, TCG_REG_RA, hi >> 16);
+        i2 = ADDI | TAI(TCG_REG_TMP1, TCG_REG_TMP1, lo);
+    } else {
+        tcg_debug_assert(TCG_TARGET_REG_BITS == 32 || addr == (int32_t)addr);
+        i1 = ADDIS | TAI(TCG_REG_TMP1, 0, addr >> 16);
+        i2 = ORI | SAI(TCG_REG_TMP1, TCG_REG_TMP1, addr);
+    }
+#ifdef HOST_WORDS_BIGENDIAN
+    pair = (uint64_t)i1 << 32 | i2;
+#else
+    pair = (uint64_t)i2 << 32 | i1;
+#endif
+
+    atomic_set((uint64_t *)jmp_addr, pair);
+    flush_icache_range(jmp_addr, jmp_addr + 8);
+}
+#else
+void ppc_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
+{
+    intptr_t diff = addr - jmp_addr;
+    tcg_debug_assert(in_range_b(diff));
+    atomic_set((uint32_t *)jmp_addr, B | (diff & 0x3fffffc));
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
+#endif
+
+static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
+{
+#ifdef _CALL_AIX
+    /* Look through the descriptor.  If the branch is in range, and we
+       don't have to spend too much effort on building the toc.  */
+    void *tgt = ((void **)target)[0];
+    uintptr_t toc = ((uintptr_t *)target)[1];
+    intptr_t diff = tcg_pcrel_diff(s, tgt);
+
+    if (in_range_b(diff) && toc == (uint32_t)toc) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc);
+        tcg_out_b(s, LK, tgt);
+    } else {
+        /* Fold the low bits of the constant into the addresses below.  */
+        intptr_t arg = (intptr_t)target;
+        int ofs = (int16_t)arg;
+
+        if (ofs + 8 < 0x8000) {
+            arg -= ofs;
+        } else {
+            ofs = 0;
+        }
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg);
+        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs);
+        tcg_out32(s, MTSPR | RA(TCG_REG_R0) | CTR);
+        tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP);
+        tcg_out32(s, BCCTR | BO_ALWAYS | LK);
+    }
+#elif defined(_CALL_ELF) && _CALL_ELF == 2
+    intptr_t diff;
+
+    /* In the ELFv2 ABI, we have to set up r12 to contain the destination
+       address, which the callee uses to compute its TOC address.  */
+    /* FIXME: when the branch is in range, we could avoid r12 load if we
+       knew that the destination uses the same TOC, and what its local
+       entry point offset is.  */
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target);
+
+    diff = tcg_pcrel_diff(s, target);
+    if (in_range_b(diff)) {
+        tcg_out_b(s, LK, target);
+    } else {
+        tcg_out32(s, MTSPR | RS(TCG_REG_R12) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS | LK);
+    }
+#else
+    tcg_out_b(s, LK, target);
+#endif
+}
+
+static const uint32_t qemu_ldx_opc[16] = {
+    [MO_UB] = LBZX,
+    [MO_UW] = LHZX,
+    [MO_UL] = LWZX,
+    [MO_Q]  = LDX,
+    [MO_SW] = LHAX,
+    [MO_SL] = LWAX,
+    [MO_BSWAP | MO_UB] = LBZX,
+    [MO_BSWAP | MO_UW] = LHBRX,
+    [MO_BSWAP | MO_UL] = LWBRX,
+    [MO_BSWAP | MO_Q]  = LDBRX,
+};
+
+static const uint32_t qemu_stx_opc[16] = {
+    [MO_UB] = STBX,
+    [MO_UW] = STHX,
+    [MO_UL] = STWX,
+    [MO_Q]  = STDX,
+    [MO_BSWAP | MO_UB] = STBX,
+    [MO_BSWAP | MO_UW] = STHBRX,
+    [MO_BSWAP | MO_UL] = STWBRX,
+    [MO_BSWAP | MO_Q]  = STDBRX,
+};
+
+static const uint32_t qemu_exts_opc[4] = {
+    EXTSB, EXTSH, EXTSW, 0
+};
+
+#if defined (CONFIG_SOFTMMU)
+/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
+ *                                 int mmu_idx, uintptr_t ra)
+ */
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+};
+
+/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
+ *                                 uintxx_t val, int mmu_idx, uintptr_t ra)
+ */
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
+};
+
+/* Perform the TLB load and compare.  Places the result of the comparison
+   in CR7, loads the addend of the TLB into R3, and returns the register
+   containing the guest address (zero-extended into R4).  Clobbers R0 and R2. */
+
+static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc,
+                               TCGReg addrlo, TCGReg addrhi,
+                               int mem_index, bool is_read)
+{
+    int cmp_off
+        = (is_read
+           ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+           : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
+    TCGReg base = TCG_AREG0;
+    TCGMemOp s_bits = opc & MO_SIZE;
+    int a_bits = get_alignment_bits(opc);
+
+    /* Extract the page index, shifted into place for tlb index.  */
+    if (TCG_TARGET_REG_BITS == 64) {
+        if (TARGET_LONG_BITS == 32) {
+            /* Zero-extend the address into a place helpful for further use. */
+            tcg_out_ext32u(s, TCG_REG_R4, addrlo);
+            addrlo = TCG_REG_R4;
+        } else {
+            tcg_out_rld(s, RLDICL, TCG_REG_R3, addrlo,
+                        64 - TARGET_PAGE_BITS, 64 - CPU_TLB_BITS);
+        }
+    }
+
+    /* Compensate for very large offsets.  */
+    if (add_off >= 0x8000) {
+        /* Most target env are smaller than 32k; none are larger than 64k.
+           Simplify the logic here merely to offset by 0x7ff0, giving us a
+           range just shy of 64k.  Check this assumption.  */
+        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
+                                   tlb_table[NB_MMU_MODES - 1][1])
+                          > 0x7ff0 + 0x7fff);
+        tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, base, 0x7ff0));
+        base = TCG_REG_TMP1;
+        cmp_off -= 0x7ff0;
+        add_off -= 0x7ff0;
+    }
+
+    /* Extraction and shifting, part 2.  */
+    if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
+        tcg_out_rlw(s, RLWINM, TCG_REG_R3, addrlo,
+                    32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS),
+                    32 - (CPU_TLB_BITS + CPU_TLB_ENTRY_BITS),
+                    31 - CPU_TLB_ENTRY_BITS);
+    } else {
+        tcg_out_shli64(s, TCG_REG_R3, TCG_REG_R3, CPU_TLB_ENTRY_BITS);
+    }
+
+    tcg_out32(s, ADD | TAB(TCG_REG_R3, TCG_REG_R3, base));
+
+    /* Load the tlb comparator.  */
+    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
+        tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_R4, TCG_REG_R3, cmp_off);
+        tcg_out_ld(s, TCG_TYPE_I32, TCG_REG_TMP1, TCG_REG_R3, cmp_off + 4);
+    } else {
+        tcg_out_ld(s, TCG_TYPE_TL, TCG_REG_TMP1, TCG_REG_R3, cmp_off);
+    }
+
+    /* Load the TLB addend for use on the fast path.  Do this asap
+       to minimize any load use delay.  */
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, add_off);
+
+    /* Clear the non-page, non-alignment bits from the address */
+    if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) {
+        /* We don't support unaligned accesses on 32-bits, preserve
+         * the bottom bits and thus trigger a comparison failure on
+         * unaligned accesses
+         */
+        if (a_bits < 0) {
+            a_bits = s_bits;
+        }
+        tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0,
+                    (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
+    } else if (a_bits) {
+        /* More than byte access, we need to handle alignment */
+        if (a_bits > 0) {
+            /* Alignment required by the front-end, same as 32-bits */
+            tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo,
+                        64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits);
+            tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
+       } else {
+           /* We support unaligned accesses, we need to make sure we fail
+            * if we cross a page boundary. The trick is to add the
+            * access_size-1 to the address before masking the low bits.
+            * That will make the address overflow to the next page if we
+            * cross a page boundary which will then force a mismatch of
+            * the TLB compare since the next page cannot possibly be in
+            * the same TLB index.
+            */
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, (1 << s_bits) - 1));
+            tcg_out_rld(s, RLDICR, TCG_REG_R0, TCG_REG_R0,
+                        0, 63 - TARGET_PAGE_BITS);
+        }
+    } else {
+        /* Byte access, just chop off the bits below the page index */
+        tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo, 0, 63 - TARGET_PAGE_BITS);
+    }
+
+    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
+        tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
+                    0, 7, TCG_TYPE_I32);
+        tcg_out_cmp(s, TCG_COND_EQ, addrhi, TCG_REG_R4, 0, 6, TCG_TYPE_I32);
+        tcg_out32(s, CRAND | BT(7, CR_EQ) | BA(6, CR_EQ) | BB(7, CR_EQ));
+    } else {
+        tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP1,
+                    0, 7, TCG_TYPE_TL);
+    }
+
+    return addrlo;
+}
+
+/* Record the context of a call to the out of line helper code for the slow
+   path for a load or store, so that we can later generate the correct
+   helper code.  */
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
+                                TCGReg datalo_reg, TCGReg datahi_reg,
+                                TCGReg addrlo_reg, TCGReg addrhi_reg,
+                                tcg_insn_unit *raddr, tcg_insn_unit *lptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->oi = oi;
+    label->datalo_reg = datalo_reg;
+    label->datahi_reg = datahi_reg;
+    label->addrlo_reg = addrlo_reg;
+    label->addrhi_reg = addrhi_reg;
+    label->raddr = raddr;
+    label->label_ptr[0] = lptr;
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGReg hi, lo, arg = TCG_REG_R3;
+
+    reloc_pc14(lb->label_ptr[0], s->code_ptr);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
+
+    lo = lb->addrlo_reg;
+    hi = lb->addrhi_reg;
+    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
+#ifdef TCG_TARGET_CALL_ALIGN_ARGS
+        arg |= 1;
+#endif
+        tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
+        tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
+    } else {
+        /* If the address needed to be zero-extended, we'll have already
+           placed it in R4.  The only remaining case is 64-bit guest.  */
+        tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
+    }
+
+    tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
+    tcg_out32(s, MFSPR | RT(arg) | LR);
+
+    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+
+    lo = lb->datalo_reg;
+    hi = lb->datahi_reg;
+    if (TCG_TARGET_REG_BITS == 32 && (opc & MO_SIZE) == MO_64) {
+        tcg_out_mov(s, TCG_TYPE_I32, lo, TCG_REG_R4);
+        tcg_out_mov(s, TCG_TYPE_I32, hi, TCG_REG_R3);
+    } else if (opc & MO_SIGN) {
+        uint32_t insn = qemu_exts_opc[opc & MO_SIZE];
+        tcg_out32(s, insn | RA(lo) | RS(TCG_REG_R3));
+    } else {
+        tcg_out_mov(s, TCG_TYPE_REG, lo, TCG_REG_R3);
+    }
+
+    tcg_out_b(s, 0, lb->raddr);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+    TCGMemOp s_bits = opc & MO_SIZE;
+    TCGReg hi, lo, arg = TCG_REG_R3;
+
+    reloc_pc14(lb->label_ptr[0], s->code_ptr);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, arg++, TCG_AREG0);
+
+    lo = lb->addrlo_reg;
+    hi = lb->addrhi_reg;
+    if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) {
+#ifdef TCG_TARGET_CALL_ALIGN_ARGS
+        arg |= 1;
+#endif
+        tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
+        tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
+    } else {
+        /* If the address needed to be zero-extended, we'll have already
+           placed it in R4.  The only remaining case is 64-bit guest.  */
+        tcg_out_mov(s, TCG_TYPE_TL, arg++, lo);
+    }
+
+    lo = lb->datalo_reg;
+    hi = lb->datahi_reg;
+    if (TCG_TARGET_REG_BITS == 32) {
+        switch (s_bits) {
+        case MO_64:
+#ifdef TCG_TARGET_CALL_ALIGN_ARGS
+            arg |= 1;
+#endif
+            tcg_out_mov(s, TCG_TYPE_I32, arg++, hi);
+            /* FALLTHRU */
+        case MO_32:
+            tcg_out_mov(s, TCG_TYPE_I32, arg++, lo);
+            break;
+        default:
+            tcg_out_rlw(s, RLWINM, arg++, lo, 0, 32 - (8 << s_bits), 31);
+            break;
+        }
+    } else {
+        if (s_bits == MO_64) {
+            tcg_out_mov(s, TCG_TYPE_I64, arg++, lo);
+        } else {
+            tcg_out_rld(s, RLDICL, arg++, lo, 0, 64 - (8 << s_bits));
+        }
+    }
+
+    tcg_out_movi(s, TCG_TYPE_I32, arg++, oi);
+    tcg_out32(s, MFSPR | RT(arg) | LR);
+
+    tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+
+    tcg_out_b(s, 0, lb->raddr);
+}
+#endif /* SOFTMMU */
+
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
+{
+    TCGReg datalo, datahi, addrlo, rbase;
+    TCGReg addrhi __attribute__((unused));
+    TCGMemOpIdx oi;
+    TCGMemOp opc, s_bits;
+#ifdef CONFIG_SOFTMMU
+    int mem_index;
+    tcg_insn_unit *label_ptr;
+#endif
+
+    datalo = *args++;
+    datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+    s_bits = opc & MO_SIZE;
+
+#ifdef CONFIG_SOFTMMU
+    mem_index = get_mmuidx(oi);
+    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, true);
+
+    /* Load a pointer into the current opcode w/conditional branch-link. */
+    label_ptr = s->code_ptr;
+    tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
+
+    rbase = TCG_REG_R3;
+#else  /* !CONFIG_SOFTMMU */
+    rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
+    if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
+        tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
+        addrlo = TCG_REG_TMP1;
+    }
+#endif
+
+    if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
+        if (opc & MO_BSWAP) {
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
+            tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
+            tcg_out32(s, LWBRX | TAB(datahi, rbase, TCG_REG_R0));
+        } else if (rbase != 0) {
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
+            tcg_out32(s, LWZX | TAB(datahi, rbase, addrlo));
+            tcg_out32(s, LWZX | TAB(datalo, rbase, TCG_REG_R0));
+        } else if (addrlo == datahi) {
+            tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
+            tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
+        } else {
+            tcg_out32(s, LWZ | TAI(datahi, addrlo, 0));
+            tcg_out32(s, LWZ | TAI(datalo, addrlo, 4));
+        }
+    } else {
+        uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP | MO_SSIZE)];
+        if (!HAVE_ISA_2_06 && insn == LDBRX) {
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
+            tcg_out32(s, LWBRX | TAB(datalo, rbase, addrlo));
+            tcg_out32(s, LWBRX | TAB(TCG_REG_R0, rbase, TCG_REG_R0));
+            tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0);
+        } else if (insn) {
+            tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
+        } else {
+            insn = qemu_ldx_opc[opc & (MO_SIZE | MO_BSWAP)];
+            tcg_out32(s, insn | TAB(datalo, rbase, addrlo));
+            insn = qemu_exts_opc[s_bits];
+            tcg_out32(s, insn | RA(datalo) | RS(datalo));
+        }
+    }
+
+#ifdef CONFIG_SOFTMMU
+    add_qemu_ldst_label(s, true, oi, datalo, datahi, addrlo, addrhi,
+                        s->code_ptr, label_ptr);
+#endif
+}
+
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
+{
+    TCGReg datalo, datahi, addrlo, rbase;
+    TCGReg addrhi __attribute__((unused));
+    TCGMemOpIdx oi;
+    TCGMemOp opc, s_bits;
+#ifdef CONFIG_SOFTMMU
+    int mem_index;
+    tcg_insn_unit *label_ptr;
+#endif
+
+    datalo = *args++;
+    datahi = (TCG_TARGET_REG_BITS == 32 && is_64 ? *args++ : 0);
+    addrlo = *args++;
+    addrhi = (TCG_TARGET_REG_BITS < TARGET_LONG_BITS ? *args++ : 0);
+    oi = *args++;
+    opc = get_memop(oi);
+    s_bits = opc & MO_SIZE;
+
+#ifdef CONFIG_SOFTMMU
+    mem_index = get_mmuidx(oi);
+    addrlo = tcg_out_tlb_read(s, opc, addrlo, addrhi, mem_index, false);
+
+    /* Load a pointer into the current opcode w/conditional branch-link. */
+    label_ptr = s->code_ptr;
+    tcg_out_bc_noaddr(s, BC | BI(7, CR_EQ) | BO_COND_FALSE | LK);
+
+    rbase = TCG_REG_R3;
+#else  /* !CONFIG_SOFTMMU */
+    rbase = guest_base ? TCG_GUEST_BASE_REG : 0;
+    if (TCG_TARGET_REG_BITS > TARGET_LONG_BITS) {
+        tcg_out_ext32u(s, TCG_REG_TMP1, addrlo);
+        addrlo = TCG_REG_TMP1;
+    }
+#endif
+
+    if (TCG_TARGET_REG_BITS == 32 && s_bits == MO_64) {
+        if (opc & MO_BSWAP) {
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
+            tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
+            tcg_out32(s, STWBRX | SAB(datahi, rbase, TCG_REG_R0));
+        } else if (rbase != 0) {
+            tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, 4));
+            tcg_out32(s, STWX | SAB(datahi, rbase, addrlo));
+            tcg_out32(s, STWX | SAB(datalo, rbase, TCG_REG_R0));
+        } else {
+            tcg_out32(s, STW | TAI(datahi, addrlo, 0));
+            tcg_out32(s, STW | TAI(datalo, addrlo, 4));
+        }
+    } else {
+        uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP | MO_SIZE)];
+        if (!HAVE_ISA_2_06 && insn == STDBRX) {
+            tcg_out32(s, STWBRX | SAB(datalo, rbase, addrlo));
+            tcg_out32(s, ADDI | TAI(TCG_REG_TMP1, addrlo, 4));
+            tcg_out_shri64(s, TCG_REG_R0, datalo, 32);
+            tcg_out32(s, STWBRX | SAB(TCG_REG_R0, rbase, TCG_REG_TMP1));
+        } else {
+            tcg_out32(s, insn | SAB(datalo, rbase, addrlo));
+        }
+    }
+
+#ifdef CONFIG_SOFTMMU
+    add_qemu_ldst_label(s, false, oi, datalo, datahi, addrlo, addrhi,
+                        s->code_ptr, label_ptr);
+#endif
+}
+
+/* Parameters for function call generation, used in tcg.c.  */
+#define TCG_TARGET_STACK_ALIGN       16
+#define TCG_TARGET_EXTEND_ARGS       1
+
+#ifdef _CALL_AIX
+# define LINK_AREA_SIZE                (6 * SZR)
+# define LR_OFFSET                     (1 * SZR)
+# define TCG_TARGET_CALL_STACK_OFFSET  (LINK_AREA_SIZE + 8 * SZR)
+#elif defined(TCG_TARGET_CALL_DARWIN)
+# define LINK_AREA_SIZE                (6 * SZR)
+# define LR_OFFSET                     (2 * SZR)
+#elif TCG_TARGET_REG_BITS == 64
+# if defined(_CALL_ELF) && _CALL_ELF == 2
+#  define LINK_AREA_SIZE               (4 * SZR)
+#  define LR_OFFSET                    (1 * SZR)
+# endif
+#else /* TCG_TARGET_REG_BITS == 32 */
+# if defined(_CALL_SYSV)
+#  define LINK_AREA_SIZE               (2 * SZR)
+#  define LR_OFFSET                    (1 * SZR)
+# endif
+#endif
+#ifndef LR_OFFSET
+# error "Unhandled abi"
+#endif
+#ifndef TCG_TARGET_CALL_STACK_OFFSET
+# define TCG_TARGET_CALL_STACK_OFFSET  LINK_AREA_SIZE
+#endif
+
+#define CPU_TEMP_BUF_SIZE  (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
+#define REG_SAVE_SIZE      ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR)
+
+#define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET   \
+                     + TCG_STATIC_CALL_ARGS_SIZE    \
+                     + CPU_TEMP_BUF_SIZE            \
+                     + REG_SAVE_SIZE                \
+                     + TCG_TARGET_STACK_ALIGN - 1)  \
+                    & -TCG_TARGET_STACK_ALIGN)
+
+#define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE)
+
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    int i;
+
+#ifdef _CALL_AIX
+    void **desc = (void **)s->code_ptr;
+    desc[0] = desc + 2;                   /* entry point */
+    desc[1] = 0;                          /* environment pointer */
+    s->code_ptr = (void *)(desc + 2);     /* skip over descriptor */
+#endif
+
+    tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
+                  CPU_TEMP_BUF_SIZE);
+
+    /* Prologue */
+    tcg_out32(s, MFSPR | RT(TCG_REG_R0) | LR);
+    tcg_out32(s, (SZR == 8 ? STDU : STWU)
+              | SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
+
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
+        tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
+                   TCG_REG_R1, REG_SAVE_BOT + i * SZR);
+    }
+    tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
+
+#ifndef CONFIG_SOFTMMU
+    if (guest_base) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
+    }
+#endif
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+    tcg_out32(s, MTSPR | RS(tcg_target_call_iarg_regs[1]) | CTR);
+
+    if (USE_REG_RA) {
+#ifdef _CALL_AIX
+        /* Make the caller load the value as the TOC into R2.  */
+        tb_ret_addr = s->code_ptr + 2;
+        desc[1] = tb_ret_addr;
+        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_RA, TCG_REG_R2);
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+#elif defined(_CALL_ELF) && _CALL_ELF == 2
+        /* Compute from the incoming R12 value.  */
+        tb_ret_addr = s->code_ptr + 2;
+        tcg_out32(s, ADDI | TAI(TCG_REG_RA, TCG_REG_R12,
+                                tcg_ptr_byte_diff(tb_ret_addr, s->code_buf)));
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+#else
+        /* Reserve max 5 insns for the constant load.  */
+        tb_ret_addr = s->code_ptr + 6;
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (intptr_t)tb_ret_addr);
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+        while (s->code_ptr < tb_ret_addr) {
+            tcg_out32(s, NOP);
+        }
+#endif
+    } else {
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+        tb_ret_addr = s->code_ptr;
+    }
+
+    /* Epilogue */
+    tcg_debug_assert(tb_ret_addr == s->code_ptr);
+
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
+        tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
+                   TCG_REG_R1, REG_SAVE_BOT + i * SZR);
+    }
+    tcg_out32(s, MTSPR | RS(TCG_REG_R0) | LR);
+    tcg_out32(s, ADDI | TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
+    tcg_out32(s, BCLR | BO_ALWAYS);
+}
+
+static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
+                       const int *const_args)
+{
+    TCGArg a0, a1, a2;
+    int c;
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        if (USE_REG_RA) {
+            ptrdiff_t disp = tcg_pcrel_diff(s, tb_ret_addr);
+
+            /* Use a direct branch if we can, otherwise use the value in RA.
+               Note that the direct branch is always backward, thus we need
+               to account for the possibility of 5 insns from the movi.  */
+            if (!in_range_b(disp - 20)) {
+                tcg_out32(s, MTSPR | RS(TCG_REG_RA) | CTR);
+                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
+                tcg_out32(s, BCCTR | BO_ALWAYS);
+                break;
+            }
+        }
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, args[0]);
+        tcg_out_b(s, 0, tb_ret_addr);
+        break;
+    case INDEX_op_goto_tb:
+        tcg_debug_assert(s->tb_jmp_insn_offset);
+        /* Direct jump. */
+#ifdef __powerpc64__
+        /* Ensure the next insns are 8-byte aligned. */
+        if ((uintptr_t)s->code_ptr & 7) {
+            tcg_out32(s, NOP);
+        }
+        s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+        /* To be replaced by either a branch+nop or a load into TMP1.  */
+        s->code_ptr += 2;
+        tcg_out32(s, MTSPR | RS(TCG_REG_TMP1) | CTR);
+        tcg_out32(s, BCCTR | BO_ALWAYS);
+#else
+        /* To be replaced by a branch.  */
+        s->code_ptr++;
+#endif
+        s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+        break;
+    case INDEX_op_br:
+        {
+            TCGLabel *l = arg_label(args[0]);
+
+            if (l->has_value) {
+                tcg_out_b(s, 0, l->u.value_ptr);
+            } else {
+                tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0);
+                tcg_out_b_noaddr(s, B);
+            }
+        }
+        break;
+    case INDEX_op_ld8u_i32:
+    case INDEX_op_ld8u_i64:
+        tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld8s_i32:
+    case INDEX_op_ld8s_i64:
+        tcg_out_mem_long(s, LBZ, LBZX, args[0], args[1], args[2]);
+        tcg_out32(s, EXTSB | RS(args[0]) | RA(args[0]));
+        break;
+    case INDEX_op_ld16u_i32:
+    case INDEX_op_ld16u_i64:
+        tcg_out_mem_long(s, LHZ, LHZX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld16s_i32:
+    case INDEX_op_ld16s_i64:
+        tcg_out_mem_long(s, LHA, LHAX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld_i32:
+    case INDEX_op_ld32u_i64:
+        tcg_out_mem_long(s, LWZ, LWZX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld32s_i64:
+        tcg_out_mem_long(s, LWA, LWAX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_mem_long(s, LD, LDX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st8_i32:
+    case INDEX_op_st8_i64:
+        tcg_out_mem_long(s, STB, STBX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st16_i32:
+    case INDEX_op_st16_i64:
+        tcg_out_mem_long(s, STH, STHX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i32:
+    case INDEX_op_st32_i64:
+        tcg_out_mem_long(s, STW, STWX, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i64:
+        tcg_out_mem_long(s, STD, STDX, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_add_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+        do_addi_32:
+            tcg_out_mem_long(s, ADDI, ADD, a0, a1, (int32_t)a2);
+        } else {
+            tcg_out32(s, ADD | TAB(a0, a1, a2));
+        }
+        break;
+    case INDEX_op_sub_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[1]) {
+            if (const_args[2]) {
+                tcg_out_movi(s, TCG_TYPE_I32, a0, a1 - a2);
+            } else {
+                tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
+            }
+        } else if (const_args[2]) {
+            a2 = -a2;
+            goto do_addi_32;
+        } else {
+            tcg_out32(s, SUBF | TAB(a0, a2, a1));
+        }
+        break;
+
+    case INDEX_op_and_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out_andi32(s, a0, a1, a2);
+        } else {
+            tcg_out32(s, AND | SAB(a1, a0, a2));
+        }
+        break;
+    case INDEX_op_and_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out_andi64(s, a0, a1, a2);
+        } else {
+            tcg_out32(s, AND | SAB(a1, a0, a2));
+        }
+        break;
+    case INDEX_op_or_i64:
+    case INDEX_op_or_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out_ori32(s, a0, a1, a2);
+        } else {
+            tcg_out32(s, OR | SAB(a1, a0, a2));
+        }
+        break;
+    case INDEX_op_xor_i64:
+    case INDEX_op_xor_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out_xori32(s, a0, a1, a2);
+        } else {
+            tcg_out32(s, XOR | SAB(a1, a0, a2));
+        }
+        break;
+    case INDEX_op_andc_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out_andi32(s, a0, a1, ~a2);
+        } else {
+            tcg_out32(s, ANDC | SAB(a1, a0, a2));
+        }
+        break;
+    case INDEX_op_andc_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out_andi64(s, a0, a1, ~a2);
+        } else {
+            tcg_out32(s, ANDC | SAB(a1, a0, a2));
+        }
+        break;
+    case INDEX_op_orc_i32:
+        if (const_args[2]) {
+            tcg_out_ori32(s, args[0], args[1], ~args[2]);
+            break;
+        }
+        /* FALLTHRU */
+    case INDEX_op_orc_i64:
+        tcg_out32(s, ORC | SAB(args[1], args[0], args[2]));
+        break;
+    case INDEX_op_eqv_i32:
+        if (const_args[2]) {
+            tcg_out_xori32(s, args[0], args[1], ~args[2]);
+            break;
+        }
+        /* FALLTHRU */
+    case INDEX_op_eqv_i64:
+        tcg_out32(s, EQV | SAB(args[1], args[0], args[2]));
+        break;
+    case INDEX_op_nand_i32:
+    case INDEX_op_nand_i64:
+        tcg_out32(s, NAND | SAB(args[1], args[0], args[2]));
+        break;
+    case INDEX_op_nor_i32:
+    case INDEX_op_nor_i64:
+        tcg_out32(s, NOR | SAB(args[1], args[0], args[2]));
+        break;
+
+    case INDEX_op_mul_i32:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out32(s, MULLI | TAI(a0, a1, a2));
+        } else {
+            tcg_out32(s, MULLW | TAB(a0, a1, a2));
+        }
+        break;
+
+    case INDEX_op_div_i32:
+        tcg_out32(s, DIVW | TAB(args[0], args[1], args[2]));
+        break;
+
+    case INDEX_op_divu_i32:
+        tcg_out32(s, DIVWU | TAB(args[0], args[1], args[2]));
+        break;
+
+    case INDEX_op_shl_i32:
+        if (const_args[2]) {
+            tcg_out_shli32(s, args[0], args[1], args[2]);
+        } else {
+            tcg_out32(s, SLW | SAB(args[1], args[0], args[2]));
+        }
+        break;
+    case INDEX_op_shr_i32:
+        if (const_args[2]) {
+            tcg_out_shri32(s, args[0], args[1], args[2]);
+        } else {
+            tcg_out32(s, SRW | SAB(args[1], args[0], args[2]));
+        }
+        break;
+    case INDEX_op_sar_i32:
+        if (const_args[2]) {
+            tcg_out32(s, SRAWI | RS(args[1]) | RA(args[0]) | SH(args[2]));
+        } else {
+            tcg_out32(s, SRAW | SAB(args[1], args[0], args[2]));
+        }
+        break;
+    case INDEX_op_rotl_i32:
+        if (const_args[2]) {
+            tcg_out_rlw(s, RLWINM, args[0], args[1], args[2], 0, 31);
+        } else {
+            tcg_out32(s, RLWNM | SAB(args[1], args[0], args[2])
+                         | MB(0) | ME(31));
+        }
+        break;
+    case INDEX_op_rotr_i32:
+        if (const_args[2]) {
+            tcg_out_rlw(s, RLWINM, args[0], args[1], 32 - args[2], 0, 31);
+        } else {
+            tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 32));
+            tcg_out32(s, RLWNM | SAB(args[1], args[0], TCG_REG_R0)
+                         | MB(0) | ME(31));
+        }
+        break;
+
+    case INDEX_op_brcond_i32:
+        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
+                       arg_label(args[3]), TCG_TYPE_I32);
+        break;
+    case INDEX_op_brcond_i64:
+        tcg_out_brcond(s, args[2], args[0], args[1], const_args[1],
+                       arg_label(args[3]), TCG_TYPE_I64);
+        break;
+    case INDEX_op_brcond2_i32:
+        tcg_out_brcond2(s, args, const_args);
+        break;
+
+    case INDEX_op_neg_i32:
+    case INDEX_op_neg_i64:
+        tcg_out32(s, NEG | RT(args[0]) | RA(args[1]));
+        break;
+
+    case INDEX_op_not_i32:
+    case INDEX_op_not_i64:
+        tcg_out32(s, NOR | SAB(args[1], args[0], args[1]));
+        break;
+
+    case INDEX_op_add_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+        do_addi_64:
+            tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
+        } else {
+            tcg_out32(s, ADD | TAB(a0, a1, a2));
+        }
+        break;
+    case INDEX_op_sub_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[1]) {
+            if (const_args[2]) {
+                tcg_out_movi(s, TCG_TYPE_I64, a0, a1 - a2);
+            } else {
+                tcg_out32(s, SUBFIC | TAI(a0, a2, a1));
+            }
+        } else if (const_args[2]) {
+            a2 = -a2;
+            goto do_addi_64;
+        } else {
+            tcg_out32(s, SUBF | TAB(a0, a2, a1));
+        }
+        break;
+
+    case INDEX_op_shl_i64:
+        if (const_args[2]) {
+            tcg_out_shli64(s, args[0], args[1], args[2]);
+        } else {
+            tcg_out32(s, SLD | SAB(args[1], args[0], args[2]));
+        }
+        break;
+    case INDEX_op_shr_i64:
+        if (const_args[2]) {
+            tcg_out_shri64(s, args[0], args[1], args[2]);
+        } else {
+            tcg_out32(s, SRD | SAB(args[1], args[0], args[2]));
+        }
+        break;
+    case INDEX_op_sar_i64:
+        if (const_args[2]) {
+            int sh = SH(args[2] & 0x1f) | (((args[2] >> 5) & 1) << 1);
+            tcg_out32(s, SRADI | RA(args[0]) | RS(args[1]) | sh);
+        } else {
+            tcg_out32(s, SRAD | SAB(args[1], args[0], args[2]));
+        }
+        break;
+    case INDEX_op_rotl_i64:
+        if (const_args[2]) {
+            tcg_out_rld(s, RLDICL, args[0], args[1], args[2], 0);
+        } else {
+            tcg_out32(s, RLDCL | SAB(args[1], args[0], args[2]) | MB64(0));
+        }
+        break;
+    case INDEX_op_rotr_i64:
+        if (const_args[2]) {
+            tcg_out_rld(s, RLDICL, args[0], args[1], 64 - args[2], 0);
+        } else {
+            tcg_out32(s, SUBFIC | TAI(TCG_REG_R0, args[2], 64));
+            tcg_out32(s, RLDCL | SAB(args[1], args[0], TCG_REG_R0) | MB64(0));
+        }
+        break;
+
+    case INDEX_op_mul_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            tcg_out32(s, MULLI | TAI(a0, a1, a2));
+        } else {
+            tcg_out32(s, MULLD | TAB(a0, a1, a2));
+        }
+        break;
+    case INDEX_op_div_i64:
+        tcg_out32(s, DIVD | TAB(args[0], args[1], args[2]));
+        break;
+    case INDEX_op_divu_i64:
+        tcg_out32(s, DIVDU | TAB(args[0], args[1], args[2]));
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, args, false);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args, true);
+        break;
+    case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, args, false);
+        break;
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args, true);
+        break;
+
+    case INDEX_op_ext8s_i32:
+    case INDEX_op_ext8s_i64:
+        c = EXTSB;
+        goto gen_ext;
+    case INDEX_op_ext16s_i32:
+    case INDEX_op_ext16s_i64:
+        c = EXTSH;
+        goto gen_ext;
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_ext32s_i64:
+        c = EXTSW;
+        goto gen_ext;
+    gen_ext:
+        tcg_out32(s, c | RS(args[1]) | RA(args[0]));
+        break;
+    case INDEX_op_extu_i32_i64:
+        tcg_out_ext32u(s, args[0], args[1]);
+        break;
+
+    case INDEX_op_setcond_i32:
+        tcg_out_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1], args[2],
+                        const_args[2]);
+        break;
+    case INDEX_op_setcond_i64:
+        tcg_out_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1], args[2],
+                        const_args[2]);
+        break;
+    case INDEX_op_setcond2_i32:
+        tcg_out_setcond2(s, args, const_args);
+        break;
+
+    case INDEX_op_bswap16_i32:
+    case INDEX_op_bswap16_i64:
+        a0 = args[0], a1 = args[1];
+        /* a1 = abcd */
+        if (a0 != a1) {
+            /* a0 = (a1 r<< 24) & 0xff # 000c */
+            tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
+            /* a0 = (a0 & ~0xff00) | (a1 r<< 8) & 0xff00 # 00dc */
+            tcg_out_rlw(s, RLWIMI, a0, a1, 8, 16, 23);
+        } else {
+            /* r0 = (a1 r<< 8) & 0xff00 # 00d0 */
+            tcg_out_rlw(s, RLWINM, TCG_REG_R0, a1, 8, 16, 23);
+            /* a0 = (a1 r<< 24) & 0xff # 000c */
+            tcg_out_rlw(s, RLWINM, a0, a1, 24, 24, 31);
+            /* a0 = a0 | r0 # 00dc */
+            tcg_out32(s, OR | SAB(TCG_REG_R0, a0, a0));
+        }
+        break;
+
+    case INDEX_op_bswap32_i32:
+    case INDEX_op_bswap32_i64:
+        /* Stolen from gcc's builtin_bswap32 */
+        a1 = args[1];
+        a0 = args[0] == a1 ? TCG_REG_R0 : args[0];
+
+        /* a1 = args[1] # abcd */
+        /* a0 = rotate_left (a1, 8) # bcda */
+        tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
+        /* a0 = (a0 & ~0xff000000) | ((a1 r<< 24) & 0xff000000) # dcda */
+        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
+        /* a0 = (a0 & ~0x0000ff00) | ((a1 r<< 24) & 0x0000ff00) # dcba */
+        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
+
+        if (a0 == TCG_REG_R0) {
+            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
+        }
+        break;
+
+    case INDEX_op_bswap64_i64:
+        a0 = args[0], a1 = args[1], a2 = TCG_REG_R0;
+        if (a0 == a1) {
+            a0 = TCG_REG_R0;
+            a2 = a1;
+        }
+
+        /* a1 = # abcd efgh */
+        /* a0 = rl32(a1, 8) # 0000 fghe */
+        tcg_out_rlw(s, RLWINM, a0, a1, 8, 0, 31);
+        /* a0 = dep(a0, rl32(a1, 24), 0xff000000) # 0000 hghe */
+        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 0, 7);
+        /* a0 = dep(a0, rl32(a1, 24), 0x0000ff00) # 0000 hgfe */
+        tcg_out_rlw(s, RLWIMI, a0, a1, 24, 16, 23);
+
+        /* a0 = rl64(a0, 32) # hgfe 0000 */
+        /* a2 = rl64(a1, 32) # efgh abcd */
+        tcg_out_rld(s, RLDICL, a0, a0, 32, 0);
+        tcg_out_rld(s, RLDICL, a2, a1, 32, 0);
+
+        /* a0 = dep(a0, rl32(a2, 8), 0xffffffff)  # hgfe bcda */
+        tcg_out_rlw(s, RLWIMI, a0, a2, 8, 0, 31);
+        /* a0 = dep(a0, rl32(a2, 24), 0xff000000) # hgfe dcda */
+        tcg_out_rlw(s, RLWIMI, a0, a2, 24, 0, 7);
+        /* a0 = dep(a0, rl32(a2, 24), 0x0000ff00) # hgfe dcba */
+        tcg_out_rlw(s, RLWIMI, a0, a2, 24, 16, 23);
+
+        if (a0 == 0) {
+            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
+        }
+        break;
+
+    case INDEX_op_deposit_i32:
+        if (const_args[2]) {
+            uint32_t mask = ((2u << (args[4] - 1)) - 1) << args[3];
+            tcg_out_andi32(s, args[0], args[0], ~mask);
+        } else {
+            tcg_out_rlw(s, RLWIMI, args[0], args[2], args[3],
+                        32 - args[3] - args[4], 31 - args[3]);
+        }
+        break;
+    case INDEX_op_deposit_i64:
+        if (const_args[2]) {
+            uint64_t mask = ((2ull << (args[4] - 1)) - 1) << args[3];
+            tcg_out_andi64(s, args[0], args[0], ~mask);
+        } else {
+            tcg_out_rld(s, RLDIMI, args[0], args[2], args[3],
+                        64 - args[3] - args[4]);
+        }
+        break;
+
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1], args[2],
+                        args[3], args[4], const_args[2]);
+        break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1], args[2],
+                        args[3], args[4], const_args[2]);
+        break;
+
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_add2_i64:
+#else
+    case INDEX_op_add2_i32:
+#endif
+        /* Note that the CA bit is defined based on the word size of the
+           environment.  So in 64-bit mode it's always carry-out of bit 63.
+           The fallback code using deposit works just as well for 32-bit.  */
+        a0 = args[0], a1 = args[1];
+        if (a0 == args[3] || (!const_args[5] && a0 == args[5])) {
+            a0 = TCG_REG_R0;
+        }
+        if (const_args[4]) {
+            tcg_out32(s, ADDIC | TAI(a0, args[2], args[4]));
+        } else {
+            tcg_out32(s, ADDC | TAB(a0, args[2], args[4]));
+        }
+        if (const_args[5]) {
+            tcg_out32(s, (args[5] ? ADDME : ADDZE) | RT(a1) | RA(args[3]));
+        } else {
+            tcg_out32(s, ADDE | TAB(a1, args[3], args[5]));
+        }
+        if (a0 != args[0]) {
+            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
+        }
+        break;
+
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_sub2_i64:
+#else
+    case INDEX_op_sub2_i32:
+#endif
+        a0 = args[0], a1 = args[1];
+        if (a0 == args[5] || (!const_args[3] && a0 == args[3])) {
+            a0 = TCG_REG_R0;
+        }
+        if (const_args[2]) {
+            tcg_out32(s, SUBFIC | TAI(a0, args[4], args[2]));
+        } else {
+            tcg_out32(s, SUBFC | TAB(a0, args[4], args[2]));
+        }
+        if (const_args[3]) {
+            tcg_out32(s, (args[3] ? SUBFME : SUBFZE) | RT(a1) | RA(args[5]));
+        } else {
+            tcg_out32(s, SUBFE | TAB(a1, args[5], args[3]));
+        }
+        if (a0 != args[0]) {
+            tcg_out_mov(s, TCG_TYPE_REG, args[0], a0);
+        }
+        break;
+
+    case INDEX_op_muluh_i32:
+        tcg_out32(s, MULHWU | TAB(args[0], args[1], args[2]));
+        break;
+    case INDEX_op_mulsh_i32:
+        tcg_out32(s, MULHW | TAB(args[0], args[1], args[2]));
+        break;
+    case INDEX_op_muluh_i64:
+        tcg_out32(s, MULHDU | TAB(args[0], args[1], args[2]));
+        break;
+    case INDEX_op_mulsh_i64:
+        tcg_out32(s, MULHD | TAB(args[0], args[1], args[2]));
+        break;
+
+    case INDEX_op_mov_i32:   /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32:  /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:      /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+}
+
+static const TCGTargetOpDef ppc_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+
+    { INDEX_op_st8_i32, { "r", "r" } },
+    { INDEX_op_st16_i32, { "r", "r" } },
+    { INDEX_op_st_i32, { "r", "r" } },
+
+    { INDEX_op_add_i32, { "r", "r", "ri" } },
+    { INDEX_op_mul_i32, { "r", "r", "rI" } },
+    { INDEX_op_div_i32, { "r", "r", "r" } },
+    { INDEX_op_divu_i32, { "r", "r", "r" } },
+    { INDEX_op_sub_i32, { "r", "rI", "ri" } },
+    { INDEX_op_and_i32, { "r", "r", "ri" } },
+    { INDEX_op_or_i32, { "r", "r", "ri" } },
+    { INDEX_op_xor_i32, { "r", "r", "ri" } },
+    { INDEX_op_andc_i32, { "r", "r", "ri" } },
+    { INDEX_op_orc_i32, { "r", "r", "ri" } },
+    { INDEX_op_eqv_i32, { "r", "r", "ri" } },
+    { INDEX_op_nand_i32, { "r", "r", "r" } },
+    { INDEX_op_nor_i32, { "r", "r", "r" } },
+
+    { INDEX_op_shl_i32, { "r", "r", "ri" } },
+    { INDEX_op_shr_i32, { "r", "r", "ri" } },
+    { INDEX_op_sar_i32, { "r", "r", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "r", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "r", "ri" } },
+
+    { INDEX_op_neg_i32, { "r", "r" } },
+    { INDEX_op_not_i32, { "r", "r" } },
+    { INDEX_op_ext8s_i32, { "r", "r" } },
+    { INDEX_op_ext16s_i32, { "r", "r" } },
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
+
+    { INDEX_op_brcond_i32, { "r", "ri" } },
+    { INDEX_op_setcond_i32, { "r", "r", "ri" } },
+    { INDEX_op_movcond_i32, { "r", "r", "ri", "rZ", "rZ" } },
+
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+
+    { INDEX_op_muluh_i32, { "r", "r", "r" } },
+    { INDEX_op_mulsh_i32, { "r", "r", "r" } },
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_ld8u_i64, { "r", "r" } },
+    { INDEX_op_ld8s_i64, { "r", "r" } },
+    { INDEX_op_ld16u_i64, { "r", "r" } },
+    { INDEX_op_ld16s_i64, { "r", "r" } },
+    { INDEX_op_ld32u_i64, { "r", "r" } },
+    { INDEX_op_ld32s_i64, { "r", "r" } },
+    { INDEX_op_ld_i64, { "r", "r" } },
+
+    { INDEX_op_st8_i64, { "r", "r" } },
+    { INDEX_op_st16_i64, { "r", "r" } },
+    { INDEX_op_st32_i64, { "r", "r" } },
+    { INDEX_op_st_i64, { "r", "r" } },
+
+    { INDEX_op_add_i64, { "r", "r", "rT" } },
+    { INDEX_op_sub_i64, { "r", "rI", "rT" } },
+    { INDEX_op_and_i64, { "r", "r", "ri" } },
+    { INDEX_op_or_i64, { "r", "r", "rU" } },
+    { INDEX_op_xor_i64, { "r", "r", "rU" } },
+    { INDEX_op_andc_i64, { "r", "r", "ri" } },
+    { INDEX_op_orc_i64, { "r", "r", "r" } },
+    { INDEX_op_eqv_i64, { "r", "r", "r" } },
+    { INDEX_op_nand_i64, { "r", "r", "r" } },
+    { INDEX_op_nor_i64, { "r", "r", "r" } },
+
+    { INDEX_op_shl_i64, { "r", "r", "ri" } },
+    { INDEX_op_shr_i64, { "r", "r", "ri" } },
+    { INDEX_op_sar_i64, { "r", "r", "ri" } },
+    { INDEX_op_rotl_i64, { "r", "r", "ri" } },
+    { INDEX_op_rotr_i64, { "r", "r", "ri" } },
+
+    { INDEX_op_mul_i64, { "r", "r", "rI" } },
+    { INDEX_op_div_i64, { "r", "r", "r" } },
+    { INDEX_op_divu_i64, { "r", "r", "r" } },
+
+    { INDEX_op_neg_i64, { "r", "r" } },
+    { INDEX_op_not_i64, { "r", "r" } },
+    { INDEX_op_ext8s_i64, { "r", "r" } },
+    { INDEX_op_ext16s_i64, { "r", "r" } },
+    { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+    { INDEX_op_bswap16_i64, { "r", "r" } },
+    { INDEX_op_bswap32_i64, { "r", "r" } },
+    { INDEX_op_bswap64_i64, { "r", "r" } },
+
+    { INDEX_op_brcond_i64, { "r", "ri" } },
+    { INDEX_op_setcond_i64, { "r", "r", "ri" } },
+    { INDEX_op_movcond_i64, { "r", "r", "ri", "rZ", "rZ" } },
+
+    { INDEX_op_deposit_i64, { "r", "0", "rZ" } },
+
+    { INDEX_op_mulsh_i64, { "r", "r", "r" } },
+    { INDEX_op_muluh_i64, { "r", "r", "r" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 32
+    { INDEX_op_brcond2_i32, { "r", "r", "ri", "ri" } },
+    { INDEX_op_setcond2_i32, { "r", "r", "r", "ri", "ri" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_add2_i64, { "r", "r", "r", "r", "rI", "rZM" } },
+    { INDEX_op_sub2_i64, { "r", "r", "rI", "rZM", "r", "r" } },
+#else
+    { INDEX_op_add2_i32, { "r", "r", "r", "r", "rI", "rZM" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rI", "rZM", "r", "r" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "S", "S" } },
+    { INDEX_op_qemu_ld_i64, { "r", "L" } },
+    { INDEX_op_qemu_st_i64, { "S", "S" } },
+#elif TARGET_LONG_BITS == 32
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "S", "S" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "L" } },
+    { INDEX_op_qemu_st_i64, { "S", "S", "S" } },
+#else
+    { INDEX_op_qemu_ld_i32, { "r", "L", "L" } },
+    { INDEX_op_qemu_st_i32, { "S", "S", "S" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "L", "L" } },
+    { INDEX_op_qemu_st_i64, { "S", "S", "S", "S" } },
+#endif
+
+    { -1 },
+};
+
+static void tcg_target_init(TCGContext *s)
+{
+    unsigned long hwcap = qemu_getauxval(AT_HWCAP);
+    if (hwcap & PPC_FEATURE_ARCH_2_06) {
+        have_isa_2_06 = true;
+    }
+
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
+    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
+                     (1 << TCG_REG_R0) |
+                     (1 << TCG_REG_R2) |
+                     (1 << TCG_REG_R3) |
+                     (1 << TCG_REG_R4) |
+                     (1 << TCG_REG_R5) |
+                     (1 << TCG_REG_R6) |
+                     (1 << TCG_REG_R7) |
+                     (1 << TCG_REG_R8) |
+                     (1 << TCG_REG_R9) |
+                     (1 << TCG_REG_R10) |
+                     (1 << TCG_REG_R11) |
+                     (1 << TCG_REG_R12));
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
+#if defined(_CALL_SYSV)
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */
+#endif
+#if defined(_CALL_SYSV) || TCG_TARGET_REG_BITS == 64
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
+#endif
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1); /* mem temp */
+    if (USE_REG_RA) {
+        tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);  /* return addr */
+    }
+
+    tcg_add_target_add_op_defs(ppc_op_defs);
+}
+
+#ifdef __ELF__
+typedef struct {
+    DebugFrameCIE cie;
+    DebugFrameFDEHeader fde;
+    uint8_t fde_def_cfa[4];
+    uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
+} DebugFrame;
+
+/* We're expecting a 2 byte uleb128 encoded value.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
+
+#if TCG_TARGET_REG_BITS == 64
+# define ELF_HOST_MACHINE EM_PPC64
+#else
+# define ELF_HOST_MACHINE EM_PPC
+#endif
+
+static DebugFrame debug_frame = {
+    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .cie.id = -1,
+    .cie.version = 1,
+    .cie.code_align = 1,
+    .cie.data_align = (-SZR & 0x7f),         /* sleb128 -SZR */
+    .cie.return_column = 65,
+
+    /* Total FDE size does not include the "len" member.  */
+    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, TCG_REG_R1,                 /* DW_CFA_def_cfa r1, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        /* DW_CFA_offset_extended_sf, lr, LR_OFFSET */
+        0x11, 65, (LR_OFFSET / -SZR) & 0x7f,
+    }
+};
+
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    uint8_t *p = &debug_frame.fde_reg_ofs[3];
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
+        p[0] = 0x80 + tcg_target_callee_save_regs[i];
+        p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR;
+    }
+
+    debug_frame.fde.func_start = (uintptr_t)buf;
+    debug_frame.fde.func_len = buf_size;
+
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
+}
+#endif /* __ELF__ */
+
+static size_t dcache_bsize = 16;
+static size_t icache_bsize = 16;
+
+void flush_icache_range(uintptr_t start, uintptr_t stop)
+{
+    uintptr_t p, start1, stop1;
+    size_t dsize = dcache_bsize;
+    size_t isize = icache_bsize;
+
+    start1 = start & ~(dsize - 1);
+    stop1 = (stop + dsize - 1) & ~(dsize - 1);
+    for (p = start1; p < stop1; p += dsize) {
+        asm volatile ("dcbst 0,%0" : : "r"(p) : "memory");
+    }
+    asm volatile ("sync" : : : "memory");
+
+    start &= start & ~(isize - 1);
+    stop1 = (stop + isize - 1) & ~(isize - 1);
+    for (p = start1; p < stop1; p += isize) {
+        asm volatile ("icbi 0,%0" : : "r"(p) : "memory");
+    }
+    asm volatile ("sync" : : : "memory");
+    asm volatile ("isync" : : : "memory");
+}
+
+#if defined _AIX
+#include <sys/systemcfg.h>
+
+static void __attribute__((constructor)) tcg_cache_init(void)
+{
+    icache_bsize = _system_configuration.icache_line;
+    dcache_bsize = _system_configuration.dcache_line;
+}
+
+#elif defined __linux__
+static void __attribute__((constructor)) tcg_cache_init(void)
+{
+    unsigned long dsize = qemu_getauxval(AT_DCACHEBSIZE);
+    unsigned long isize = qemu_getauxval(AT_ICACHEBSIZE);
+
+    if (dsize == 0 || isize == 0) {
+        if (dsize == 0) {
+            fprintf(stderr, "getauxval AT_DCACHEBSIZE failed\n");
+        }
+        if (isize == 0) {
+            fprintf(stderr, "getauxval AT_ICACHEBSIZE failed\n");
+        }
+        exit(1);
+    }
+    dcache_bsize = dsize;
+    icache_bsize = isize;
+}
+
+#elif defined __APPLE__
+#include <sys/sysctl.h>
+
+static void __attribute__((constructor)) tcg_cache_init(void)
+{
+    size_t len;
+    unsigned cacheline;
+    int name[2] = { CTL_HW, HW_CACHELINE };
+
+    len = sizeof(cacheline);
+    if (sysctl(name, 2, &cacheline, &len, NULL, 0)) {
+        perror("sysctl CTL_HW HW_CACHELINE failed");
+        exit(1);
+    }
+    dcache_bsize = cacheline;
+    icache_bsize = cacheline;
+}
+
+#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
+#include <sys/sysctl.h>
+
+static void __attribute__((constructor)) tcg_cache_init(void)
+{
+    size_t len = 4;
+    unsigned cacheline;
+
+    if (sysctlbyname ("machdep.cacheline_size", &cacheline, &len, NULL, 0)) {
+        fprintf(stderr, "sysctlbyname machdep.cacheline_size failed: %s\n",
+                strerror(errno));
+        exit(1);
+    }
+    dcache_bsize = cacheline;
+    icache_bsize = cacheline;
+}
+#endif
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
deleted file mode 100644
index f78a0c1..0000000
--- a/tcg/s390/tcg-target.c
+++ /dev/null
@@ -1,2409 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
- * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
- * Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "tcg-be-ldst.h"
-
-/* We only support generating code for 64-bit mode.  */
-#if TCG_TARGET_REG_BITS != 64
-#error "unsupported code generation mode"
-#endif
-
-#include "elf.h"
-
-/* ??? The translation blocks produced by TCG are generally small enough to
-   be entirely reachable with a 16-bit displacement.  Leaving the option for
-   a 32-bit displacement here Just In Case.  */
-#define USE_LONG_BRANCHES 0
-
-#define TCG_CT_CONST_MULI  0x100
-#define TCG_CT_CONST_ORI   0x200
-#define TCG_CT_CONST_XORI  0x400
-#define TCG_CT_CONST_CMPI  0x800
-#define TCG_CT_CONST_ADLI  0x1000
-
-/* Several places within the instruction set 0 means "no register"
-   rather than TCG_REG_R0.  */
-#define TCG_REG_NONE    0
-
-/* A scratch register that may be be used throughout the backend.  */
-#define TCG_TMP0        TCG_REG_R14
-
-#ifdef CONFIG_USE_GUEST_BASE
-#define TCG_GUEST_BASE_REG TCG_REG_R13
-#else
-#define TCG_GUEST_BASE_REG TCG_REG_R0
-#endif
-
-#ifndef GUEST_BASE
-#define GUEST_BASE 0
-#endif
-
-
-/* All of the following instructions are prefixed with their instruction
-   format, and are defined as 8- or 16-bit quantities, even when the two
-   halves of the 16-bit quantity may appear 32 bits apart in the insn.
-   This makes it easy to copy the values from the tables in Appendix B.  */
-typedef enum S390Opcode {
-    RIL_AFI     = 0xc209,
-    RIL_AGFI    = 0xc208,
-    RIL_ALFI    = 0xc20b,
-    RIL_ALGFI   = 0xc20a,
-    RIL_BRASL   = 0xc005,
-    RIL_BRCL    = 0xc004,
-    RIL_CFI     = 0xc20d,
-    RIL_CGFI    = 0xc20c,
-    RIL_CLFI    = 0xc20f,
-    RIL_CLGFI   = 0xc20e,
-    RIL_IIHF    = 0xc008,
-    RIL_IILF    = 0xc009,
-    RIL_LARL    = 0xc000,
-    RIL_LGFI    = 0xc001,
-    RIL_LGRL    = 0xc408,
-    RIL_LLIHF   = 0xc00e,
-    RIL_LLILF   = 0xc00f,
-    RIL_LRL     = 0xc40d,
-    RIL_MSFI    = 0xc201,
-    RIL_MSGFI   = 0xc200,
-    RIL_NIHF    = 0xc00a,
-    RIL_NILF    = 0xc00b,
-    RIL_OIHF    = 0xc00c,
-    RIL_OILF    = 0xc00d,
-    RIL_SLFI    = 0xc205,
-    RIL_SLGFI   = 0xc204,
-    RIL_XIHF    = 0xc006,
-    RIL_XILF    = 0xc007,
-
-    RI_AGHI     = 0xa70b,
-    RI_AHI      = 0xa70a,
-    RI_BRC      = 0xa704,
-    RI_IIHH     = 0xa500,
-    RI_IIHL     = 0xa501,
-    RI_IILH     = 0xa502,
-    RI_IILL     = 0xa503,
-    RI_LGHI     = 0xa709,
-    RI_LLIHH    = 0xa50c,
-    RI_LLIHL    = 0xa50d,
-    RI_LLILH    = 0xa50e,
-    RI_LLILL    = 0xa50f,
-    RI_MGHI     = 0xa70d,
-    RI_MHI      = 0xa70c,
-    RI_NIHH     = 0xa504,
-    RI_NIHL     = 0xa505,
-    RI_NILH     = 0xa506,
-    RI_NILL     = 0xa507,
-    RI_OIHH     = 0xa508,
-    RI_OIHL     = 0xa509,
-    RI_OILH     = 0xa50a,
-    RI_OILL     = 0xa50b,
-
-    RIE_CGIJ    = 0xec7c,
-    RIE_CGRJ    = 0xec64,
-    RIE_CIJ     = 0xec7e,
-    RIE_CLGRJ   = 0xec65,
-    RIE_CLIJ    = 0xec7f,
-    RIE_CLGIJ   = 0xec7d,
-    RIE_CLRJ    = 0xec77,
-    RIE_CRJ     = 0xec76,
-    RIE_RISBG   = 0xec55,
-
-    RRE_AGR     = 0xb908,
-    RRE_ALGR    = 0xb90a,
-    RRE_ALCR    = 0xb998,
-    RRE_ALCGR   = 0xb988,
-    RRE_CGR     = 0xb920,
-    RRE_CLGR    = 0xb921,
-    RRE_DLGR    = 0xb987,
-    RRE_DLR     = 0xb997,
-    RRE_DSGFR   = 0xb91d,
-    RRE_DSGR    = 0xb90d,
-    RRE_LGBR    = 0xb906,
-    RRE_LCGR    = 0xb903,
-    RRE_LGFR    = 0xb914,
-    RRE_LGHR    = 0xb907,
-    RRE_LGR     = 0xb904,
-    RRE_LLGCR   = 0xb984,
-    RRE_LLGFR   = 0xb916,
-    RRE_LLGHR   = 0xb985,
-    RRE_LRVR    = 0xb91f,
-    RRE_LRVGR   = 0xb90f,
-    RRE_LTGR    = 0xb902,
-    RRE_MLGR    = 0xb986,
-    RRE_MSGR    = 0xb90c,
-    RRE_MSR     = 0xb252,
-    RRE_NGR     = 0xb980,
-    RRE_OGR     = 0xb981,
-    RRE_SGR     = 0xb909,
-    RRE_SLGR    = 0xb90b,
-    RRE_SLBR    = 0xb999,
-    RRE_SLBGR   = 0xb989,
-    RRE_XGR     = 0xb982,
-
-    RRF_LOCR    = 0xb9f2,
-    RRF_LOCGR   = 0xb9e2,
-
-    RR_AR       = 0x1a,
-    RR_ALR      = 0x1e,
-    RR_BASR     = 0x0d,
-    RR_BCR      = 0x07,
-    RR_CLR      = 0x15,
-    RR_CR       = 0x19,
-    RR_DR       = 0x1d,
-    RR_LCR      = 0x13,
-    RR_LR       = 0x18,
-    RR_LTR      = 0x12,
-    RR_NR       = 0x14,
-    RR_OR       = 0x16,
-    RR_SR       = 0x1b,
-    RR_SLR      = 0x1f,
-    RR_XR       = 0x17,
-
-    RSY_RLL     = 0xeb1d,
-    RSY_RLLG    = 0xeb1c,
-    RSY_SLLG    = 0xeb0d,
-    RSY_SRAG    = 0xeb0a,
-    RSY_SRLG    = 0xeb0c,
-
-    RS_SLL      = 0x89,
-    RS_SRA      = 0x8a,
-    RS_SRL      = 0x88,
-
-    RXY_AG      = 0xe308,
-    RXY_AY      = 0xe35a,
-    RXY_CG      = 0xe320,
-    RXY_CY      = 0xe359,
-    RXY_LAY     = 0xe371,
-    RXY_LB      = 0xe376,
-    RXY_LG      = 0xe304,
-    RXY_LGB     = 0xe377,
-    RXY_LGF     = 0xe314,
-    RXY_LGH     = 0xe315,
-    RXY_LHY     = 0xe378,
-    RXY_LLGC    = 0xe390,
-    RXY_LLGF    = 0xe316,
-    RXY_LLGH    = 0xe391,
-    RXY_LMG     = 0xeb04,
-    RXY_LRV     = 0xe31e,
-    RXY_LRVG    = 0xe30f,
-    RXY_LRVH    = 0xe31f,
-    RXY_LY      = 0xe358,
-    RXY_STCY    = 0xe372,
-    RXY_STG     = 0xe324,
-    RXY_STHY    = 0xe370,
-    RXY_STMG    = 0xeb24,
-    RXY_STRV    = 0xe33e,
-    RXY_STRVG   = 0xe32f,
-    RXY_STRVH   = 0xe33f,
-    RXY_STY     = 0xe350,
-
-    RX_A        = 0x5a,
-    RX_C        = 0x59,
-    RX_L        = 0x58,
-    RX_LA       = 0x41,
-    RX_LH       = 0x48,
-    RX_ST       = 0x50,
-    RX_STC      = 0x42,
-    RX_STH      = 0x40,
-} S390Opcode;
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
-    "%r8", "%r9", "%r10" "%r11" "%r12" "%r13" "%r14" "%r15"
-};
-#endif
-
-/* Since R6 is a potential argument register, choose it last of the
-   call-saved registers.  Likewise prefer the call-clobbered registers
-   in reverse order to maximize the chance of avoiding the arguments.  */
-static const int tcg_target_reg_alloc_order[] = {
-    /* Call saved registers.  */
-    TCG_REG_R13,
-    TCG_REG_R12,
-    TCG_REG_R11,
-    TCG_REG_R10,
-    TCG_REG_R9,
-    TCG_REG_R8,
-    TCG_REG_R7,
-    TCG_REG_R6,
-    /* Call clobbered registers.  */
-    TCG_REG_R14,
-    TCG_REG_R0,
-    TCG_REG_R1,
-    /* Argument registers, in reverse order of allocation.  */
-    TCG_REG_R5,
-    TCG_REG_R4,
-    TCG_REG_R3,
-    TCG_REG_R2,
-};
-
-static const int tcg_target_call_iarg_regs[] = {
-    TCG_REG_R2,
-    TCG_REG_R3,
-    TCG_REG_R4,
-    TCG_REG_R5,
-    TCG_REG_R6,
-};
-
-static const int tcg_target_call_oarg_regs[] = {
-    TCG_REG_R2,
-};
-
-#define S390_CC_EQ      8
-#define S390_CC_LT      4
-#define S390_CC_GT      2
-#define S390_CC_OV      1
-#define S390_CC_NE      (S390_CC_LT | S390_CC_GT)
-#define S390_CC_LE      (S390_CC_LT | S390_CC_EQ)
-#define S390_CC_GE      (S390_CC_GT | S390_CC_EQ)
-#define S390_CC_NEVER   0
-#define S390_CC_ALWAYS  15
-
-/* Condition codes that result from a COMPARE and COMPARE LOGICAL.  */
-static const uint8_t tcg_cond_to_s390_cond[] = {
-    [TCG_COND_EQ]  = S390_CC_EQ,
-    [TCG_COND_NE]  = S390_CC_NE,
-    [TCG_COND_LT]  = S390_CC_LT,
-    [TCG_COND_LE]  = S390_CC_LE,
-    [TCG_COND_GT]  = S390_CC_GT,
-    [TCG_COND_GE]  = S390_CC_GE,
-    [TCG_COND_LTU] = S390_CC_LT,
-    [TCG_COND_LEU] = S390_CC_LE,
-    [TCG_COND_GTU] = S390_CC_GT,
-    [TCG_COND_GEU] = S390_CC_GE,
-};
-
-/* Condition codes that result from a LOAD AND TEST.  Here, we have no
-   unsigned instruction variation, however since the test is vs zero we
-   can re-map the outcomes appropriately.  */
-static const uint8_t tcg_cond_to_ltr_cond[] = {
-    [TCG_COND_EQ]  = S390_CC_EQ,
-    [TCG_COND_NE]  = S390_CC_NE,
-    [TCG_COND_LT]  = S390_CC_LT,
-    [TCG_COND_LE]  = S390_CC_LE,
-    [TCG_COND_GT]  = S390_CC_GT,
-    [TCG_COND_GE]  = S390_CC_GE,
-    [TCG_COND_LTU] = S390_CC_NEVER,
-    [TCG_COND_LEU] = S390_CC_EQ,
-    [TCG_COND_GTU] = S390_CC_NE,
-    [TCG_COND_GEU] = S390_CC_ALWAYS,
-};
-
-#ifdef CONFIG_SOFTMMU
-static void * const qemu_ld_helpers[16] = {
-    [MO_UB]   = helper_ret_ldub_mmu,
-    [MO_SB]   = helper_ret_ldsb_mmu,
-    [MO_LEUW] = helper_le_lduw_mmu,
-    [MO_LESW] = helper_le_ldsw_mmu,
-    [MO_LEUL] = helper_le_ldul_mmu,
-    [MO_LESL] = helper_le_ldsl_mmu,
-    [MO_LEQ]  = helper_le_ldq_mmu,
-    [MO_BEUW] = helper_be_lduw_mmu,
-    [MO_BESW] = helper_be_ldsw_mmu,
-    [MO_BEUL] = helper_be_ldul_mmu,
-    [MO_BESL] = helper_be_ldsl_mmu,
-    [MO_BEQ]  = helper_be_ldq_mmu,
-};
-
-static void * const qemu_st_helpers[16] = {
-    [MO_UB]   = helper_ret_stb_mmu,
-    [MO_LEUW] = helper_le_stw_mmu,
-    [MO_LEUL] = helper_le_stl_mmu,
-    [MO_LEQ]  = helper_le_stq_mmu,
-    [MO_BEUW] = helper_be_stw_mmu,
-    [MO_BEUL] = helper_be_stl_mmu,
-    [MO_BEQ]  = helper_be_stq_mmu,
-};
-#endif
-
-static tcg_insn_unit *tb_ret_addr;
-
-/* A list of relevant facilities used by this translator.  Some of these
-   are required for proper operation, and these are checked at startup.  */
-
-#define FACILITY_ZARCH_ACTIVE	(1ULL << (63 - 2))
-#define FACILITY_LONG_DISP	(1ULL << (63 - 18))
-#define FACILITY_EXT_IMM	(1ULL << (63 - 21))
-#define FACILITY_GEN_INST_EXT	(1ULL << (63 - 34))
-#define FACILITY_LOAD_ON_COND   (1ULL << (63 - 45))
-
-static uint64_t facilities;
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    intptr_t pcrel2 = (tcg_insn_unit *)value - (code_ptr - 1);
-    assert(addend == -2);
-
-    switch (type) {
-    case R_390_PC16DBL:
-        assert(pcrel2 == (int16_t)pcrel2);
-        tcg_patch16(code_ptr, pcrel2);
-        break;
-    case R_390_PC32DBL:
-        assert(pcrel2 == (int32_t)pcrel2);
-        tcg_patch32(code_ptr, pcrel2);
-        break;
-    default:
-        tcg_abort();
-        break;
-    }
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str = *pct_str;
-
-    switch (ct_str[0]) {
-    case 'r':                  /* all registers */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffff);
-        break;
-    case 'R':                  /* not R0 */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffff);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
-        break;
-    case 'L':                  /* qemu_ld/st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffff);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R2);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R3);
-        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
-        break;
-    case 'a':                  /* force R2 for division */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_clear(ct->u.regs);
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_R2);
-        break;
-    case 'b':                  /* force R3 for division */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_clear(ct->u.regs);
-        tcg_regset_set_reg(ct->u.regs, TCG_REG_R3);
-        break;
-    case 'A':
-        ct->ct |= TCG_CT_CONST_ADLI;
-        break;
-    case 'K':
-        ct->ct |= TCG_CT_CONST_MULI;
-        break;
-    case 'O':
-        ct->ct |= TCG_CT_CONST_ORI;
-        break;
-    case 'X':
-        ct->ct |= TCG_CT_CONST_XORI;
-        break;
-    case 'C':
-        ct->ct |= TCG_CT_CONST_CMPI;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-
-    return 0;
-}
-
-/* Immediates to be used with logical OR.  This is an optimization only,
-   since a full 64-bit immediate OR can always be performed with 4 sequential
-   OI[LH][LH] instructions.  What we're looking for is immediates that we
-   can load efficiently, and the immediate load plus the reg-reg OR is
-   smaller than the sequential OI's.  */
-
-static int tcg_match_ori(TCGType type, tcg_target_long val)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        if (type == TCG_TYPE_I32) {
-            /* All 32-bit ORs can be performed with 1 48-bit insn.  */
-            return 1;
-        }
-    }
-
-    /* Look for negative values.  These are best to load with LGHI.  */
-    if (val < 0) {
-        if (val == (int16_t)val) {
-            return 0;
-        }
-        if (facilities & FACILITY_EXT_IMM) {
-            if (val == (int32_t)val) {
-                return 0;
-            }
-        }
-    }
-
-    return 1;
-}
-
-/* Immediates to be used with logical XOR.  This is almost, but not quite,
-   only an optimization.  XOR with immediate is only supported with the
-   extended-immediate facility.  That said, there are a few patterns for
-   which it is better to load the value into a register first.  */
-
-static int tcg_match_xori(TCGType type, tcg_target_long val)
-{
-    if ((facilities & FACILITY_EXT_IMM) == 0) {
-        return 0;
-    }
-
-    if (type == TCG_TYPE_I32) {
-        /* All 32-bit XORs can be performed with 1 48-bit insn.  */
-        return 1;
-    }
-
-    /* Look for negative values.  These are best to load with LGHI.  */
-    if (val < 0 && val == (int32_t)val) {
-        return 0;
-    }
-
-    return 1;
-}
-
-/* Imediates to be used with comparisons.  */
-
-static int tcg_match_cmpi(TCGType type, tcg_target_long val)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        /* The COMPARE IMMEDIATE instruction is available.  */
-        if (type == TCG_TYPE_I32) {
-            /* We have a 32-bit immediate and can compare against anything.  */
-            return 1;
-        } else {
-            /* ??? We have no insight here into whether the comparison is
-               signed or unsigned.  The COMPARE IMMEDIATE insn uses a 32-bit
-               signed immediate, and the COMPARE LOGICAL IMMEDIATE insn uses
-               a 32-bit unsigned immediate.  If we were to use the (semi)
-               obvious "val == (int32_t)val" we would be enabling unsigned
-               comparisons vs very large numbers.  The only solution is to
-               take the intersection of the ranges.  */
-            /* ??? Another possible solution is to simply lie and allow all
-               constants here and force the out-of-range values into a temp
-               register in tgen_cmp when we have knowledge of the actual
-               comparison code in use.  */
-            return val >= 0 && val <= 0x7fffffff;
-        }
-    } else {
-        /* Only the LOAD AND TEST instruction is available.  */
-        return val == 0;
-    }
-}
-
-/* Immediates to be used with add2/sub2.  */
-
-static int tcg_match_add2i(TCGType type, tcg_target_long val)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        if (type == TCG_TYPE_I32) {
-            return 1;
-        } else if (val >= -0xffffffffll && val <= 0xffffffffll) {
-            return 1;
-        }
-    }
-    return 0;
-}
-
-/* Test if a constant matches the constraint. */
-static int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                  const TCGArgConstraint *arg_ct)
-{
-    int ct = arg_ct->ct;
-
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    }
-
-    if (type == TCG_TYPE_I32) {
-        val = (int32_t)val;
-    }
-
-    /* The following are mutually exclusive.  */
-    if (ct & TCG_CT_CONST_MULI) {
-        /* Immediates that may be used with multiply.  If we have the
-           general-instruction-extensions, then we have MULTIPLY SINGLE
-           IMMEDIATE with a signed 32-bit, otherwise we have only
-           MULTIPLY HALFWORD IMMEDIATE, with a signed 16-bit.  */
-        if (facilities & FACILITY_GEN_INST_EXT) {
-            return val == (int32_t)val;
-        } else {
-            return val == (int16_t)val;
-        }
-    } else if (ct & TCG_CT_CONST_ADLI) {
-        return tcg_match_add2i(type, val);
-    } else if (ct & TCG_CT_CONST_ORI) {
-        return tcg_match_ori(type, val);
-    } else if (ct & TCG_CT_CONST_XORI) {
-        return tcg_match_xori(type, val);
-    } else if (ct & TCG_CT_CONST_CMPI) {
-        return tcg_match_cmpi(type, val);
-    }
-
-    return 0;
-}
-
-/* Emit instructions according to the given instruction format.  */
-
-static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
-{
-    tcg_out16(s, (op << 8) | (r1 << 4) | r2);
-}
-
-static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
-                             TCGReg r1, TCGReg r2)
-{
-    tcg_out32(s, (op << 16) | (r1 << 4) | r2);
-}
-
-static void tcg_out_insn_RRF(TCGContext *s, S390Opcode op,
-                             TCGReg r1, TCGReg r2, int m3)
-{
-    tcg_out32(s, (op << 16) | (m3 << 12) | (r1 << 4) | r2);
-}
-
-static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
-{
-    tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff));
-}
-
-static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
-{
-    tcg_out16(s, op | (r1 << 4));
-    tcg_out32(s, i2);
-}
-
-static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
-                            TCGReg b2, TCGReg r3, int disp)
-{
-    tcg_out32(s, (op << 24) | (r1 << 20) | (r3 << 16) | (b2 << 12)
-              | (disp & 0xfff));
-}
-
-static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
-                             TCGReg b2, TCGReg r3, int disp)
-{
-    tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3);
-    tcg_out32(s, (op & 0xff) | (b2 << 28)
-              | ((disp & 0xfff) << 16) | ((disp & 0xff000) >> 4));
-}
-
-#define tcg_out_insn_RX   tcg_out_insn_RS
-#define tcg_out_insn_RXY  tcg_out_insn_RSY
-
-/* Emit an opcode with "type-checking" of the format.  */
-#define tcg_out_insn(S, FMT, OP, ...) \
-    glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
-
-
-/* emit 64-bit shifts */
-static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
-                         TCGReg src, TCGReg sh_reg, int sh_imm)
-{
-    tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
-}
-
-/* emit 32-bit shifts */
-static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
-                         TCGReg sh_reg, int sh_imm)
-{
-    tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
-}
-
-static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
-{
-    if (src != dst) {
-        if (type == TCG_TYPE_I32) {
-            tcg_out_insn(s, RR, LR, dst, src);
-        } else {
-            tcg_out_insn(s, RRE, LGR, dst, src);
-        }
-    }
-}
-
-/* load a register with an immediate value */
-static void tcg_out_movi(TCGContext *s, TCGType type,
-                         TCGReg ret, tcg_target_long sval)
-{
-    static const S390Opcode lli_insns[4] = {
-        RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
-    };
-
-    tcg_target_ulong uval = sval;
-    int i;
-
-    if (type == TCG_TYPE_I32) {
-        uval = (uint32_t)sval;
-        sval = (int32_t)sval;
-    }
-
-    /* Try all 32-bit insns that can load it in one go.  */
-    if (sval >= -0x8000 && sval < 0x8000) {
-        tcg_out_insn(s, RI, LGHI, ret, sval);
-        return;
-    }
-
-    for (i = 0; i < 4; i++) {
-        tcg_target_long mask = 0xffffull << i*16;
-        if ((uval & mask) == uval) {
-            tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i*16);
-            return;
-        }
-    }
-
-    /* Try all 48-bit insns that can load it in one go.  */
-    if (facilities & FACILITY_EXT_IMM) {
-        if (sval == (int32_t)sval) {
-            tcg_out_insn(s, RIL, LGFI, ret, sval);
-            return;
-        }
-        if (uval <= 0xffffffff) {
-            tcg_out_insn(s, RIL, LLILF, ret, uval);
-            return;
-        }
-        if ((uval & 0xffffffff) == 0) {
-            tcg_out_insn(s, RIL, LLIHF, ret, uval >> 31 >> 1);
-            return;
-        }
-    }
-
-    /* Try for PC-relative address load.  */
-    if ((sval & 1) == 0) {
-        ptrdiff_t off = tcg_pcrel_diff(s, (void *)sval) >> 1;
-        if (off == (int32_t)off) {
-            tcg_out_insn(s, RIL, LARL, ret, off);
-            return;
-        }
-    }
-
-    /* If extended immediates are not present, then we may have to issue
-       several instructions to load the low 32 bits.  */
-    if (!(facilities & FACILITY_EXT_IMM)) {
-        /* A 32-bit unsigned value can be loaded in 2 insns.  And given
-           that the lli_insns loop above did not succeed, we know that
-           both insns are required.  */
-        if (uval <= 0xffffffff) {
-            tcg_out_insn(s, RI, LLILL, ret, uval);
-            tcg_out_insn(s, RI, IILH, ret, uval >> 16);
-            return;
-        }
-
-        /* If all high bits are set, the value can be loaded in 2 or 3 insns.
-           We first want to make sure that all the high bits get set.  With
-           luck the low 16-bits can be considered negative to perform that for
-           free, otherwise we load an explicit -1.  */
-        if (sval >> 31 >> 1 == -1) {
-            if (uval & 0x8000) {
-                tcg_out_insn(s, RI, LGHI, ret, uval);
-            } else {
-                tcg_out_insn(s, RI, LGHI, ret, -1);
-                tcg_out_insn(s, RI, IILL, ret, uval);
-            }
-            tcg_out_insn(s, RI, IILH, ret, uval >> 16);
-            return;
-        }
-    }
-
-    /* If we get here, both the high and low parts have non-zero bits.  */
-
-    /* Recurse to load the lower 32-bits.  */
-    tcg_out_movi(s, TCG_TYPE_I64, ret, uval & 0xffffffff);
-
-    /* Insert data into the high 32-bits.  */
-    uval = uval >> 31 >> 1;
-    if (facilities & FACILITY_EXT_IMM) {
-        if (uval < 0x10000) {
-            tcg_out_insn(s, RI, IIHL, ret, uval);
-        } else if ((uval & 0xffff) == 0) {
-            tcg_out_insn(s, RI, IIHH, ret, uval >> 16);
-        } else {
-            tcg_out_insn(s, RIL, IIHF, ret, uval);
-        }
-    } else {
-        if (uval & 0xffff) {
-            tcg_out_insn(s, RI, IIHL, ret, uval);
-        }
-        if (uval & 0xffff0000) {
-            tcg_out_insn(s, RI, IIHH, ret, uval >> 16);
-        }
-    }
-}
-
-
-/* Emit a load/store type instruction.  Inputs are:
-   DATA:     The register to be loaded or stored.
-   BASE+OFS: The effective address.
-   OPC_RX:   If the operation has an RX format opcode (e.g. STC), otherwise 0.
-   OPC_RXY:  The RXY format opcode for the operation (e.g. STCY).  */
-
-static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
-                        TCGReg data, TCGReg base, TCGReg index,
-                        tcg_target_long ofs)
-{
-    if (ofs < -0x80000 || ofs >= 0x80000) {
-        /* Combine the low 20 bits of the offset with the actual load insn;
-           the high 44 bits must come from an immediate load.  */
-        tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
-        ofs = low;
-
-        /* If we were already given an index register, add it in.  */
-        if (index != TCG_REG_NONE) {
-            tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
-        }
-        index = TCG_TMP0;
-    }
-
-    if (opc_rx && ofs >= 0 && ofs < 0x1000) {
-        tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
-    } else {
-        tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
-    }
-}
-
-
-/* load data without address translation or endianness conversion */
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
-                              TCGReg base, intptr_t ofs)
-{
-    if (type == TCG_TYPE_I32) {
-        tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
-    } else {
-        tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
-    }
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
-                              TCGReg base, intptr_t ofs)
-{
-    if (type == TCG_TYPE_I32) {
-        tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
-    } else {
-        tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
-    }
-}
-
-/* load data from an absolute host address */
-static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs)
-{
-    intptr_t addr = (intptr_t)abs;
-
-    if ((facilities & FACILITY_GEN_INST_EXT) && !(addr & 1)) {
-        ptrdiff_t disp = tcg_pcrel_diff(s, abs) >> 1;
-        if (disp == (int32_t)disp) {
-            if (type == TCG_TYPE_I32) {
-                tcg_out_insn(s, RIL, LRL, dest, disp);
-            } else {
-                tcg_out_insn(s, RIL, LGRL, dest, disp);
-            }
-            return;
-        }
-    }
-
-    tcg_out_movi(s, TCG_TYPE_PTR, dest, addr & ~0xffff);
-    tcg_out_ld(s, type, dest, dest, addr & 0xffff);
-}
-
-static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
-                                 int msb, int lsb, int ofs, int z)
-{
-    /* Format RIE-f */
-    tcg_out16(s, (RIE_RISBG & 0xff00) | (dest << 4) | src);
-    tcg_out16(s, (msb << 8) | (z << 7) | lsb);
-    tcg_out16(s, (ofs << 8) | (RIE_RISBG & 0xff));
-}
-
-static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        tcg_out_insn(s, RRE, LGBR, dest, src);
-        return;
-    }
-
-    if (type == TCG_TYPE_I32) {
-        if (dest == src) {
-            tcg_out_sh32(s, RS_SLL, dest, TCG_REG_NONE, 24);
-        } else {
-            tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 24);
-        }
-        tcg_out_sh32(s, RS_SRA, dest, TCG_REG_NONE, 24);
-    } else {
-        tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 56);
-        tcg_out_sh64(s, RSY_SRAG, dest, dest, TCG_REG_NONE, 56);
-    }
-}
-
-static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        tcg_out_insn(s, RRE, LLGCR, dest, src);
-        return;
-    }
-
-    if (dest == src) {
-        tcg_out_movi(s, type, TCG_TMP0, 0xff);
-        src = TCG_TMP0;
-    } else {
-        tcg_out_movi(s, type, dest, 0xff);
-    }
-    if (type == TCG_TYPE_I32) {
-        tcg_out_insn(s, RR, NR, dest, src);
-    } else {
-        tcg_out_insn(s, RRE, NGR, dest, src);
-    }
-}
-
-static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        tcg_out_insn(s, RRE, LGHR, dest, src);
-        return;
-    }
-
-    if (type == TCG_TYPE_I32) {
-        if (dest == src) {
-            tcg_out_sh32(s, RS_SLL, dest, TCG_REG_NONE, 16);
-        } else {
-            tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 16);
-        }
-        tcg_out_sh32(s, RS_SRA, dest, TCG_REG_NONE, 16);
-    } else {
-        tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 48);
-        tcg_out_sh64(s, RSY_SRAG, dest, dest, TCG_REG_NONE, 48);
-    }
-}
-
-static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
-{
-    if (facilities & FACILITY_EXT_IMM) {
-        tcg_out_insn(s, RRE, LLGHR, dest, src);
-        return;
-    }
-
-    if (dest == src) {
-        tcg_out_movi(s, type, TCG_TMP0, 0xffff);
-        src = TCG_TMP0;
-    } else {
-        tcg_out_movi(s, type, dest, 0xffff);
-    }
-    if (type == TCG_TYPE_I32) {
-        tcg_out_insn(s, RR, NR, dest, src);
-    } else {
-        tcg_out_insn(s, RRE, NGR, dest, src);
-    }
-}
-
-static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
-{
-    tcg_out_insn(s, RRE, LGFR, dest, src);
-}
-
-static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
-{
-    tcg_out_insn(s, RRE, LLGFR, dest, src);
-}
-
-/* Accept bit patterns like these:
-    0....01....1
-    1....10....0
-    1..10..01..1
-    0..01..10..0
-   Copied from gcc sources.  */
-static inline bool risbg_mask(uint64_t c)
-{
-    uint64_t lsb;
-    /* We don't change the number of transitions by inverting,
-       so make sure we start with the LSB zero.  */
-    if (c & 1) {
-        c = ~c;
-    }
-    /* Reject all zeros or all ones.  */
-    if (c == 0) {
-        return false;
-    }
-    /* Find the first transition.  */
-    lsb = c & -c;
-    /* Invert to look for a second transition.  */
-    c = ~c;
-    /* Erase the first transition.  */
-    c &= -lsb;
-    /* Find the second transition, if any.  */
-    lsb = c & -c;
-    /* Match if all the bits are 1's, or if c is zero.  */
-    return c == -lsb;
-}
-
-static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
-{
-    int msb, lsb;
-    if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
-        /* Achieve wraparound by swapping msb and lsb.  */
-        msb = 64 - ctz64(~val);
-        lsb = clz64(~val) - 1;
-    } else {
-        msb = clz64(val);
-        lsb = 63 - ctz64(val);
-    }
-    tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
-}
-
-static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
-{
-    static const S390Opcode ni_insns[4] = {
-        RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
-    };
-    static const S390Opcode nif_insns[2] = {
-        RIL_NILF, RIL_NIHF
-    };
-    uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
-    int i;
-
-    /* Look for the zero-extensions.  */
-    if ((val & valid) == 0xffffffff) {
-        tgen_ext32u(s, dest, dest);
-        return;
-    }
-    if (facilities & FACILITY_EXT_IMM) {
-        if ((val & valid) == 0xff) {
-            tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
-            return;
-        }
-        if ((val & valid) == 0xffff) {
-            tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
-            return;
-        }
-    }
-
-    /* Try all 32-bit insns that can perform it in one go.  */
-    for (i = 0; i < 4; i++) {
-        tcg_target_ulong mask = ~(0xffffull << i*16);
-        if (((val | ~valid) & mask) == mask) {
-            tcg_out_insn_RI(s, ni_insns[i], dest, val >> i*16);
-            return;
-        }
-    }
-
-    /* Try all 48-bit insns that can perform it in one go.  */
-    if (facilities & FACILITY_EXT_IMM) {
-        for (i = 0; i < 2; i++) {
-            tcg_target_ulong mask = ~(0xffffffffull << i*32);
-            if (((val | ~valid) & mask) == mask) {
-                tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i*32);
-                return;
-            }
-        }
-    }
-    if ((facilities & FACILITY_GEN_INST_EXT) && risbg_mask(val)) {
-        tgen_andi_risbg(s, dest, dest, val);
-        return;
-    }
-
-    /* Fall back to loading the constant.  */
-    tcg_out_movi(s, type, TCG_TMP0, val);
-    if (type == TCG_TYPE_I32) {
-        tcg_out_insn(s, RR, NR, dest, TCG_TMP0);
-    } else {
-        tcg_out_insn(s, RRE, NGR, dest, TCG_TMP0);
-    }
-}
-
-static void tgen64_ori(TCGContext *s, TCGReg dest, tcg_target_ulong val)
-{
-    static const S390Opcode oi_insns[4] = {
-        RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
-    };
-    static const S390Opcode nif_insns[2] = {
-        RIL_OILF, RIL_OIHF
-    };
-
-    int i;
-
-    /* Look for no-op.  */
-    if (val == 0) {
-        return;
-    }
-
-    if (facilities & FACILITY_EXT_IMM) {
-        /* Try all 32-bit insns that can perform it in one go.  */
-        for (i = 0; i < 4; i++) {
-            tcg_target_ulong mask = (0xffffull << i*16);
-            if ((val & mask) != 0 && (val & ~mask) == 0) {
-                tcg_out_insn_RI(s, oi_insns[i], dest, val >> i*16);
-                return;
-            }
-        }
-
-        /* Try all 48-bit insns that can perform it in one go.  */
-        for (i = 0; i < 2; i++) {
-            tcg_target_ulong mask = (0xffffffffull << i*32);
-            if ((val & mask) != 0 && (val & ~mask) == 0) {
-                tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i*32);
-                return;
-            }
-        }
-
-        /* Perform the OR via sequential modifications to the high and
-           low parts.  Do this via recursion to handle 16-bit vs 32-bit
-           masks in each half.  */
-        tgen64_ori(s, dest, val & 0x00000000ffffffffull);
-        tgen64_ori(s, dest, val & 0xffffffff00000000ull);
-    } else {
-        /* With no extended-immediate facility, we don't need to be so
-           clever.  Just iterate over the insns and mask in the constant.  */
-        for (i = 0; i < 4; i++) {
-            tcg_target_ulong mask = (0xffffull << i*16);
-            if ((val & mask) != 0) {
-                tcg_out_insn_RI(s, oi_insns[i], dest, val >> i*16);
-            }
-        }
-    }
-}
-
-static void tgen64_xori(TCGContext *s, TCGReg dest, tcg_target_ulong val)
-{
-    /* Perform the xor by parts.  */
-    if (val & 0xffffffff) {
-        tcg_out_insn(s, RIL, XILF, dest, val);
-    }
-    if (val > 0xffffffff) {
-        tcg_out_insn(s, RIL, XIHF, dest, val >> 31 >> 1);
-    }
-}
-
-static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
-                    TCGArg c2, int c2const)
-{
-    bool is_unsigned = is_unsigned_cond(c);
-    if (c2const) {
-        if (c2 == 0) {
-            if (type == TCG_TYPE_I32) {
-                tcg_out_insn(s, RR, LTR, r1, r1);
-            } else {
-                tcg_out_insn(s, RRE, LTGR, r1, r1);
-            }
-            return tcg_cond_to_ltr_cond[c];
-        } else {
-            if (is_unsigned) {
-                if (type == TCG_TYPE_I32) {
-                    tcg_out_insn(s, RIL, CLFI, r1, c2);
-                } else {
-                    tcg_out_insn(s, RIL, CLGFI, r1, c2);
-                }
-            } else {
-                if (type == TCG_TYPE_I32) {
-                    tcg_out_insn(s, RIL, CFI, r1, c2);
-                } else {
-                    tcg_out_insn(s, RIL, CGFI, r1, c2);
-                }
-            }
-        }
-    } else {
-        if (is_unsigned) {
-            if (type == TCG_TYPE_I32) {
-                tcg_out_insn(s, RR, CLR, r1, c2);
-            } else {
-                tcg_out_insn(s, RRE, CLGR, r1, c2);
-            }
-        } else {
-            if (type == TCG_TYPE_I32) {
-                tcg_out_insn(s, RR, CR, r1, c2);
-            } else {
-                tcg_out_insn(s, RRE, CGR, r1, c2);
-            }
-        }
-    }
-    return tcg_cond_to_s390_cond[c];
-}
-
-static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
-                         TCGReg dest, TCGReg c1, TCGArg c2, int c2const)
-{
-    int cc;
-
-    switch (cond) {
-    case TCG_COND_GTU:
-    case TCG_COND_GT:
-    do_greater:
-        /* The result of a compare has CC=2 for GT and CC=3 unused.
-           ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit.  */
-        tgen_cmp(s, type, cond, c1, c2, c2const);
-        tcg_out_movi(s, type, dest, 0);
-        tcg_out_insn(s, RRE, ALCGR, dest, dest);
-        return;
-
-    case TCG_COND_GEU:
-    do_geu:
-        /* We need "real" carry semantics, so use SUBTRACT LOGICAL
-           instead of COMPARE LOGICAL.  This needs an extra move.  */
-        tcg_out_mov(s, type, TCG_TMP0, c1);
-        if (c2const) {
-            tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
-            if (type == TCG_TYPE_I32) {
-                tcg_out_insn(s, RIL, SLFI, TCG_TMP0, c2);
-            } else {
-                tcg_out_insn(s, RIL, SLGFI, TCG_TMP0, c2);
-            }
-        } else {
-            if (type == TCG_TYPE_I32) {
-                tcg_out_insn(s, RR, SLR, TCG_TMP0, c2);
-            } else {
-                tcg_out_insn(s, RRE, SLGR, TCG_TMP0, c2);
-            }
-            tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
-        }
-        tcg_out_insn(s, RRE, ALCGR, dest, dest);
-        return;
-
-    case TCG_COND_LEU:
-    case TCG_COND_LTU:
-    case TCG_COND_LT:
-        /* Swap operands so that we can use GEU/GTU/GT.  */
-        if (c2const) {
-            tcg_out_movi(s, type, TCG_TMP0, c2);
-            c2 = c1;
-            c2const = 0;
-            c1 = TCG_TMP0;
-        } else {
-            TCGReg t = c1;
-            c1 = c2;
-            c2 = t;
-        }
-        if (cond == TCG_COND_LEU) {
-            goto do_geu;
-        }
-        cond = tcg_swap_cond(cond);
-        goto do_greater;
-
-    case TCG_COND_NE:
-        /* X != 0 is X > 0.  */
-        if (c2const && c2 == 0) {
-            cond = TCG_COND_GTU;
-            goto do_greater;
-        }
-        break;
-
-    case TCG_COND_EQ:
-        /* X == 0 is X <= 0 is 0 >= X.  */
-        if (c2const && c2 == 0) {
-            tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 0);
-            c2 = c1;
-            c2const = 0;
-            c1 = TCG_TMP0;
-            goto do_geu;
-        }
-        break;
-
-    default:
-        break;
-    }
-
-    cc = tgen_cmp(s, type, cond, c1, c2, c2const);
-    if (facilities & FACILITY_LOAD_ON_COND) {
-        /* Emit: d = 0, t = 1, d = (cc ? t : d).  */
-        tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
-        tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 1);
-        tcg_out_insn(s, RRF, LOCGR, dest, TCG_TMP0, cc);
-    } else {
-        /* Emit: d = 1; if (cc) goto over; d = 0; over:  */
-        tcg_out_movi(s, type, dest, 1);
-        tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
-        tcg_out_movi(s, type, dest, 0);
-    }
-}
-
-static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest,
-                         TCGReg c1, TCGArg c2, int c2const, TCGReg r3)
-{
-    int cc;
-    if (facilities & FACILITY_LOAD_ON_COND) {
-        cc = tgen_cmp(s, type, c, c1, c2, c2const);
-        tcg_out_insn(s, RRF, LOCGR, dest, r3, cc);
-    } else {
-        c = tcg_invert_cond(c);
-        cc = tgen_cmp(s, type, c, c1, c2, c2const);
-
-        /* Emit: if (cc) goto over; dest = r3; over:  */
-        tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
-        tcg_out_insn(s, RRE, LGR, dest, r3);
-    }
-}
-
-bool tcg_target_deposit_valid(int ofs, int len)
-{
-    return (facilities & FACILITY_GEN_INST_EXT) != 0;
-}
-
-static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src,
-                         int ofs, int len)
-{
-    int lsb = (63 - ofs);
-    int msb = lsb - (len - 1);
-    tcg_out_risbg(s, dest, src, msb, lsb, ofs, 0);
-}
-
-static void tgen_gotoi(TCGContext *s, int cc, tcg_insn_unit *dest)
-{
-    ptrdiff_t off = dest - s->code_ptr;
-    if (off == (int16_t)off) {
-        tcg_out_insn(s, RI, BRC, cc, off);
-    } else if (off == (int32_t)off) {
-        tcg_out_insn(s, RIL, BRCL, cc, off);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
-        tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
-    }
-}
-
-static void tgen_branch(TCGContext *s, int cc, int labelno)
-{
-    TCGLabel* l = &s->labels[labelno];
-    if (l->has_value) {
-        tgen_gotoi(s, cc, l->u.value_ptr);
-    } else if (USE_LONG_BRANCHES) {
-        tcg_out16(s, RIL_BRCL | (cc << 4));
-        tcg_out_reloc(s, s->code_ptr, R_390_PC32DBL, labelno, -2);
-        s->code_ptr += 2;
-    } else {
-        tcg_out16(s, RI_BRC | (cc << 4));
-        tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, labelno, -2);
-        s->code_ptr += 1;
-    }
-}
-
-static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
-                                TCGReg r1, TCGReg r2, int labelno)
-{
-    TCGLabel* l = &s->labels[labelno];
-    intptr_t off;
-
-    if (l->has_value) {
-        off = l->u.value_ptr - s->code_ptr;
-    } else {
-        /* We need to keep the offset unchanged for retranslation.  */
-        off = s->code_ptr[1];
-        tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, labelno, -2);
-    }
-
-    tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
-    tcg_out16(s, off);
-    tcg_out16(s, cc << 12 | (opc & 0xff));
-}
-
-static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
-                                    TCGReg r1, int i2, int labelno)
-{
-    TCGLabel* l = &s->labels[labelno];
-    tcg_target_long off;
-
-    if (l->has_value) {
-        off = l->u.value_ptr - s->code_ptr;
-    } else {
-        /* We need to keep the offset unchanged for retranslation.  */
-        off = s->code_ptr[1];
-        tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, labelno, -2);
-    }
-
-    tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
-    tcg_out16(s, off);
-    tcg_out16(s, (i2 << 8) | (opc & 0xff));
-}
-
-static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
-                        TCGReg r1, TCGArg c2, int c2const, int labelno)
-{
-    int cc;
-
-    if (facilities & FACILITY_GEN_INST_EXT) {
-        bool is_unsigned = is_unsigned_cond(c);
-        bool in_range;
-        S390Opcode opc;
-
-        cc = tcg_cond_to_s390_cond[c];
-
-        if (!c2const) {
-            opc = (type == TCG_TYPE_I32
-                   ? (is_unsigned ? RIE_CLRJ : RIE_CRJ)
-                   : (is_unsigned ? RIE_CLGRJ : RIE_CGRJ));
-            tgen_compare_branch(s, opc, cc, r1, c2, labelno);
-            return;
-        }
-
-        /* COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
-           If the immediate we've been given does not fit that range, we'll
-           fall back to separate compare and branch instructions using the
-           larger comparison range afforded by COMPARE IMMEDIATE.  */
-        if (type == TCG_TYPE_I32) {
-            if (is_unsigned) {
-                opc = RIE_CLIJ;
-                in_range = (uint32_t)c2 == (uint8_t)c2;
-            } else {
-                opc = RIE_CIJ;
-                in_range = (int32_t)c2 == (int8_t)c2;
-            }
-        } else {
-            if (is_unsigned) {
-                opc = RIE_CLGIJ;
-                in_range = (uint64_t)c2 == (uint8_t)c2;
-            } else {
-                opc = RIE_CGIJ;
-                in_range = (int64_t)c2 == (int8_t)c2;
-            }
-        }
-        if (in_range) {
-            tgen_compare_imm_branch(s, opc, cc, r1, c2, labelno);
-            return;
-        }
-    }
-
-    cc = tgen_cmp(s, type, c, r1, c2, c2const);
-    tgen_branch(s, cc, labelno);
-}
-
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
-{
-    ptrdiff_t off = dest - s->code_ptr;
-    if (off == (int32_t)off) {
-        tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
-        tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
-    }
-}
-
-static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp opc, TCGReg data,
-                                   TCGReg base, TCGReg index, int disp)
-{
-    switch (opc) {
-    case MO_UB:
-        tcg_out_insn(s, RXY, LLGC, data, base, index, disp);
-        break;
-    case MO_SB:
-        tcg_out_insn(s, RXY, LGB, data, base, index, disp);
-        break;
-
-    case MO_UW | MO_BSWAP:
-        /* swapped unsigned halfword load with upper bits zeroed */
-        tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
-        tgen_ext16u(s, TCG_TYPE_I64, data, data);
-        break;
-    case MO_UW:
-        tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
-        break;
-
-    case MO_SW | MO_BSWAP:
-        /* swapped sign-extended halfword load */
-        tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
-        tgen_ext16s(s, TCG_TYPE_I64, data, data);
-        break;
-    case MO_SW:
-        tcg_out_insn(s, RXY, LGH, data, base, index, disp);
-        break;
-
-    case MO_UL | MO_BSWAP:
-        /* swapped unsigned int load with upper bits zeroed */
-        tcg_out_insn(s, RXY, LRV, data, base, index, disp);
-        tgen_ext32u(s, data, data);
-        break;
-    case MO_UL:
-        tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
-        break;
-
-    case MO_SL | MO_BSWAP:
-        /* swapped sign-extended int load */
-        tcg_out_insn(s, RXY, LRV, data, base, index, disp);
-        tgen_ext32s(s, data, data);
-        break;
-    case MO_SL:
-        tcg_out_insn(s, RXY, LGF, data, base, index, disp);
-        break;
-
-    case MO_Q | MO_BSWAP:
-        tcg_out_insn(s, RXY, LRVG, data, base, index, disp);
-        break;
-    case MO_Q:
-        tcg_out_insn(s, RXY, LG, data, base, index, disp);
-        break;
-
-    default:
-        tcg_abort();
-    }
-}
-
-static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp opc, TCGReg data,
-                                   TCGReg base, TCGReg index, int disp)
-{
-    switch (opc) {
-    case MO_UB:
-        if (disp >= 0 && disp < 0x1000) {
-            tcg_out_insn(s, RX, STC, data, base, index, disp);
-        } else {
-            tcg_out_insn(s, RXY, STCY, data, base, index, disp);
-        }
-        break;
-
-    case MO_UW | MO_BSWAP:
-        tcg_out_insn(s, RXY, STRVH, data, base, index, disp);
-        break;
-    case MO_UW:
-        if (disp >= 0 && disp < 0x1000) {
-            tcg_out_insn(s, RX, STH, data, base, index, disp);
-        } else {
-            tcg_out_insn(s, RXY, STHY, data, base, index, disp);
-        }
-        break;
-
-    case MO_UL | MO_BSWAP:
-        tcg_out_insn(s, RXY, STRV, data, base, index, disp);
-        break;
-    case MO_UL:
-        if (disp >= 0 && disp < 0x1000) {
-            tcg_out_insn(s, RX, ST, data, base, index, disp);
-        } else {
-            tcg_out_insn(s, RXY, STY, data, base, index, disp);
-        }
-        break;
-
-    case MO_Q | MO_BSWAP:
-        tcg_out_insn(s, RXY, STRVG, data, base, index, disp);
-        break;
-    case MO_Q:
-        tcg_out_insn(s, RXY, STG, data, base, index, disp);
-        break;
-
-    default:
-        tcg_abort();
-    }
-}
-
-#if defined(CONFIG_SOFTMMU)
-/* We're expecting to use a 20-bit signed offset on the tlb memory ops.
-   Using the offset of the second entry in the last tlb table ensures
-   that we can index all of the elements of the first entry.  */
-QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
-                  > 0x7ffff);
-
-/* Load and compare a TLB entry, leaving the flags set.  Loads the TLB
-   addend into R2.  Returns a register with the santitized guest address.  */
-static TCGReg tcg_out_tlb_read(TCGContext* s, TCGReg addr_reg, TCGMemOp opc,
-                               int mem_index, bool is_ld)
-{
-    int a_bits = get_alignment_bits(opc);
-    int ofs, a_off;
-    uint64_t tlb_mask;
-
-    /* For aligned accesses, we check the first byte and include the alignment
-       bits within the address.  For unaligned access, we check that we don't
-       cross pages using the address of the last byte of the access.  */
-    if (a_bits >= 0) {
-        /* A byte access or an alignment check required */
-        a_off = 0;
-        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);
-    } else {
-        a_off = (1 << (opc & MO_SIZE)) - 1;
-        tlb_mask = TARGET_PAGE_MASK;
-    }
-
-    if (facilities & FACILITY_GEN_INST_EXT) {
-        tcg_out_risbg(s, TCG_REG_R2, addr_reg,
-                      64 - CPU_TLB_BITS - CPU_TLB_ENTRY_BITS,
-                      63 - CPU_TLB_ENTRY_BITS,
-                      64 + CPU_TLB_ENTRY_BITS - TARGET_PAGE_BITS, 1);
-        if (a_off) {
-            tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
-            tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
-        } else {
-            tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
-        }
-    } else {
-        tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
-                     TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-        tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
-        tgen_andi(s, TCG_TYPE_I64, TCG_REG_R2,
-                  (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-        tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
-    }
-
-    if (is_ld) {
-        ofs = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
-    } else {
-        ofs = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
-    }
-    if (TARGET_LONG_BITS == 32) {
-        tcg_out_mem(s, RX_C, RXY_CY, TCG_REG_R3, TCG_REG_R2, TCG_AREG0, ofs);
-    } else {
-        tcg_out_mem(s, 0, RXY_CG, TCG_REG_R3, TCG_REG_R2, TCG_AREG0, ofs);
-    }
-
-    ofs = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
-    tcg_out_mem(s, 0, RXY_LG, TCG_REG_R2, TCG_REG_R2, TCG_AREG0, ofs);
-
-    if (TARGET_LONG_BITS == 32) {
-        tgen_ext32u(s, TCG_REG_R3, addr_reg);
-        return TCG_REG_R3;
-    }
-    return addr_reg;
-}
-
-static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
-                                TCGReg data, TCGReg addr,
-                                tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
-{
-    TCGLabelQemuLdst *label = new_ldst_label(s);
-
-    label->is_ld = is_ld;
-    label->oi = oi;
-    label->datalo_reg = data;
-    label->addrlo_reg = addr;
-    label->raddr = raddr;
-    label->label_ptr[0] = label_ptr;
-}
-
-static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGReg addr_reg = lb->addrlo_reg;
-    TCGReg data_reg = lb->datalo_reg;
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-
-    patch_reloc(lb->label_ptr[0], R_390_PC16DBL, (intptr_t)s->code_ptr, -2);
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
-    }
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi);
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr);
-    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]);
-    tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2);
-
-    tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
-}
-
-static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
-{
-    TCGReg addr_reg = lb->addrlo_reg;
-    TCGReg data_reg = lb->datalo_reg;
-    TCGMemOpIdx oi = lb->oi;
-    TCGMemOp opc = get_memop(oi);
-
-    patch_reloc(lb->label_ptr[0], R_390_PC16DBL, (intptr_t)s->code_ptr, -2);
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
-    }
-    switch (opc & MO_SIZE) {
-    case MO_UB:
-        tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
-        break;
-    case MO_UW:
-        tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
-        break;
-    case MO_UL:
-        tgen_ext32u(s, TCG_REG_R4, data_reg);
-        break;
-    case MO_Q:
-        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
-        break;
-    default:
-        tcg_abort();
-    }
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
-    tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
-
-    tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
-}
-#else
-static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
-                                  TCGReg *index_reg, tcg_target_long *disp)
-{
-    if (TARGET_LONG_BITS == 32) {
-        tgen_ext32u(s, TCG_TMP0, *addr_reg);
-        *addr_reg = TCG_TMP0;
-    }
-    if (GUEST_BASE < 0x80000) {
-        *index_reg = TCG_REG_NONE;
-        *disp = GUEST_BASE;
-    } else {
-        *index_reg = TCG_GUEST_BASE_REG;
-        *disp = 0;
-    }
-}
-#endif /* CONFIG_SOFTMMU */
-
-static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
-                            TCGMemOpIdx oi)
-{
-    TCGMemOp opc = get_memop(oi);
-#ifdef CONFIG_SOFTMMU
-    unsigned mem_index = get_mmuidx(oi);
-    tcg_insn_unit *label_ptr;
-    TCGReg base_reg;
-
-    base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1);
-
-    label_ptr = s->code_ptr + 1;
-    tcg_out_insn(s, RI, BRC, S390_CC_NE, 0);
-
-    tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
-
-    add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
-#else
-    TCGReg index_reg;
-    tcg_target_long disp;
-
-    tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
-    tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp);
-#endif
-}
-
-static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
-                            TCGMemOpIdx oi)
-{
-    TCGMemOp opc = get_memop(oi);
-#ifdef CONFIG_SOFTMMU
-    unsigned mem_index = get_mmuidx(oi);
-    tcg_insn_unit *label_ptr;
-    TCGReg base_reg;
-
-    base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0);
-
-    label_ptr = s->code_ptr + 1;
-    tcg_out_insn(s, RI, BRC, S390_CC_NE, 0);
-
-    tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
-
-    add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
-#else
-    TCGReg index_reg;
-    tcg_target_long disp;
-
-    tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
-    tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp);
-#endif
-}
-
-# define OP_32_64(x) \
-        case glue(glue(INDEX_op_,x),_i32): \
-        case glue(glue(INDEX_op_,x),_i64)
-
-static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                const TCGArg *args, const int *const_args)
-{
-    S390Opcode op;
-    TCGArg a0, a1, a2;
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        /* return value */
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, args[0]);
-        tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
-        break;
-
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            s->code_ptr += 2;
-        } else {
-            /* load address stored at s->tb_next + args[0] */
-            tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_TMP0, s->tb_next + args[0]);
-            /* and go there */
-            tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_TMP0);
-        }
-        s->tb_next_offset[args[0]] = tcg_current_code_size(s);
-        break;
-
-    OP_32_64(ld8u):
-        /* ??? LLC (RXY format) is only present with the extended-immediate
-           facility, whereas LLGC is always present.  */
-        tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-
-    OP_32_64(ld8s):
-        /* ??? LB is no smaller than LGB, so no point to using it.  */
-        tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-
-    OP_32_64(ld16u):
-        /* ??? LLH (RXY format) is only present with the extended-immediate
-           facility, whereas LLGH is always present.  */
-        tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-
-    case INDEX_op_ld16s_i32:
-        tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-
-    case INDEX_op_ld_i32:
-        tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
-        break;
-
-    OP_32_64(st8):
-        tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1],
-                    TCG_REG_NONE, args[2]);
-        break;
-
-    OP_32_64(st16):
-        tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1],
-                    TCG_REG_NONE, args[2]);
-        break;
-
-    case INDEX_op_st_i32:
-        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_add_i32:
-        a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
-        if (const_args[2]) {
-        do_addi_32:
-            if (a0 == a1) {
-                if (a2 == (int16_t)a2) {
-                    tcg_out_insn(s, RI, AHI, a0, a2);
-                    break;
-                }
-                if (facilities & FACILITY_EXT_IMM) {
-                    tcg_out_insn(s, RIL, AFI, a0, a2);
-                    break;
-                }
-            }
-            tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
-        } else if (a0 == a1) {
-            tcg_out_insn(s, RR, AR, a0, a2);
-        } else {
-            tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
-        }
-        break;
-    case INDEX_op_sub_i32:
-        a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
-        if (const_args[2]) {
-            a2 = -a2;
-            goto do_addi_32;
-        }
-        tcg_out_insn(s, RR, SR, args[0], args[2]);
-        break;
-
-    case INDEX_op_and_i32:
-        if (const_args[2]) {
-            tgen_andi(s, TCG_TYPE_I32, args[0], args[2]);
-        } else {
-            tcg_out_insn(s, RR, NR, args[0], args[2]);
-        }
-        break;
-    case INDEX_op_or_i32:
-        if (const_args[2]) {
-            tgen64_ori(s, args[0], args[2] & 0xffffffff);
-        } else {
-            tcg_out_insn(s, RR, OR, args[0], args[2]);
-        }
-        break;
-    case INDEX_op_xor_i32:
-        if (const_args[2]) {
-            tgen64_xori(s, args[0], args[2] & 0xffffffff);
-        } else {
-            tcg_out_insn(s, RR, XR, args[0], args[2]);
-        }
-        break;
-
-    case INDEX_op_neg_i32:
-        tcg_out_insn(s, RR, LCR, args[0], args[1]);
-        break;
-
-    case INDEX_op_mul_i32:
-        if (const_args[2]) {
-            if ((int32_t)args[2] == (int16_t)args[2]) {
-                tcg_out_insn(s, RI, MHI, args[0], args[2]);
-            } else {
-                tcg_out_insn(s, RIL, MSFI, args[0], args[2]);
-            }
-        } else {
-            tcg_out_insn(s, RRE, MSR, args[0], args[2]);
-        }
-        break;
-
-    case INDEX_op_div2_i32:
-        tcg_out_insn(s, RR, DR, TCG_REG_R2, args[4]);
-        break;
-    case INDEX_op_divu2_i32:
-        tcg_out_insn(s, RRE, DLR, TCG_REG_R2, args[4]);
-        break;
-
-    case INDEX_op_shl_i32:
-        op = RS_SLL;
-    do_shift32:
-        if (const_args[2]) {
-            tcg_out_sh32(s, op, args[0], TCG_REG_NONE, args[2]);
-        } else {
-            tcg_out_sh32(s, op, args[0], args[2], 0);
-        }
-        break;
-    case INDEX_op_shr_i32:
-        op = RS_SRL;
-        goto do_shift32;
-    case INDEX_op_sar_i32:
-        op = RS_SRA;
-        goto do_shift32;
-
-    case INDEX_op_rotl_i32:
-        /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol.  */
-        if (const_args[2]) {
-            tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]);
-        } else {
-            tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0);
-        }
-        break;
-    case INDEX_op_rotr_i32:
-        if (const_args[2]) {
-            tcg_out_sh64(s, RSY_RLL, args[0], args[1],
-                         TCG_REG_NONE, (32 - args[2]) & 31);
-        } else {
-            tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
-            tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0);
-        }
-        break;
-
-    case INDEX_op_ext8s_i32:
-        tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
-    case INDEX_op_ext16s_i32:
-        tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
-    case INDEX_op_ext8u_i32:
-        tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
-    case INDEX_op_ext16u_i32:
-        tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
-        break;
-
-    OP_32_64(bswap16):
-        /* The TCG bswap definition requires bits 0-47 already be zero.
-           Thus we don't need the G-type insns to implement bswap16_i64.  */
-        tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
-        tcg_out_sh32(s, RS_SRL, args[0], TCG_REG_NONE, 16);
-        break;
-    OP_32_64(bswap32):
-        tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
-        break;
-
-    case INDEX_op_add2_i32:
-        if (const_args[4]) {
-            tcg_out_insn(s, RIL, ALFI, args[0], args[4]);
-        } else {
-            tcg_out_insn(s, RR, ALR, args[0], args[4]);
-        }
-        tcg_out_insn(s, RRE, ALCR, args[1], args[5]);
-        break;
-    case INDEX_op_sub2_i32:
-        if (const_args[4]) {
-            tcg_out_insn(s, RIL, SLFI, args[0], args[4]);
-        } else {
-            tcg_out_insn(s, RR, SLR, args[0], args[4]);
-        }
-        tcg_out_insn(s, RRE, SLBR, args[1], args[5]);
-        break;
-
-    case INDEX_op_br:
-        tgen_branch(s, S390_CC_ALWAYS, args[0]);
-        break;
-
-    case INDEX_op_brcond_i32:
-        tgen_brcond(s, TCG_TYPE_I32, args[2], args[0],
-                    args[1], const_args[1], args[3]);
-        break;
-    case INDEX_op_setcond_i32:
-        tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1],
-                     args[2], const_args[2]);
-        break;
-    case INDEX_op_movcond_i32:
-        tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1],
-                     args[2], const_args[2], args[3]);
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        /* ??? Technically we can use a non-extending instruction.  */
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_qemu_st_i32:
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_ld16s_i64:
-        tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-    case INDEX_op_ld32u_i64:
-        tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-    case INDEX_op_ld32s_i64:
-        tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]);
-        break;
-    case INDEX_op_ld_i64:
-        tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_st32_i64:
-        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_st_i64:
-        tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
-        break;
-
-    case INDEX_op_add_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-        do_addi_64:
-            if (a0 == a1) {
-                if (a2 == (int16_t)a2) {
-                    tcg_out_insn(s, RI, AGHI, a0, a2);
-                    break;
-                }
-                if (facilities & FACILITY_EXT_IMM) {
-                    if (a2 == (int32_t)a2) {
-                        tcg_out_insn(s, RIL, AGFI, a0, a2);
-                        break;
-                    } else if (a2 == (uint32_t)a2) {
-                        tcg_out_insn(s, RIL, ALGFI, a0, a2);
-                        break;
-                    } else if (-a2 == (uint32_t)-a2) {
-                        tcg_out_insn(s, RIL, SLGFI, a0, -a2);
-                        break;
-                    }
-                }
-            }
-            tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
-        } else if (a0 == a1) {
-            tcg_out_insn(s, RRE, AGR, a0, a2);
-        } else {
-            tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
-        }
-        break;
-    case INDEX_op_sub_i64:
-        a0 = args[0], a1 = args[1], a2 = args[2];
-        if (const_args[2]) {
-            a2 = -a2;
-            goto do_addi_64;
-        } else {
-            tcg_out_insn(s, RRE, SGR, args[0], args[2]);
-        }
-        break;
-
-    case INDEX_op_and_i64:
-        if (const_args[2]) {
-            tgen_andi(s, TCG_TYPE_I64, args[0], args[2]);
-        } else {
-            tcg_out_insn(s, RRE, NGR, args[0], args[2]);
-        }
-        break;
-    case INDEX_op_or_i64:
-        if (const_args[2]) {
-            tgen64_ori(s, args[0], args[2]);
-        } else {
-            tcg_out_insn(s, RRE, OGR, args[0], args[2]);
-        }
-        break;
-    case INDEX_op_xor_i64:
-        if (const_args[2]) {
-            tgen64_xori(s, args[0], args[2]);
-        } else {
-            tcg_out_insn(s, RRE, XGR, args[0], args[2]);
-        }
-        break;
-
-    case INDEX_op_neg_i64:
-        tcg_out_insn(s, RRE, LCGR, args[0], args[1]);
-        break;
-    case INDEX_op_bswap64_i64:
-        tcg_out_insn(s, RRE, LRVGR, args[0], args[1]);
-        break;
-
-    case INDEX_op_mul_i64:
-        if (const_args[2]) {
-            if (args[2] == (int16_t)args[2]) {
-                tcg_out_insn(s, RI, MGHI, args[0], args[2]);
-            } else {
-                tcg_out_insn(s, RIL, MSGFI, args[0], args[2]);
-            }
-        } else {
-            tcg_out_insn(s, RRE, MSGR, args[0], args[2]);
-        }
-        break;
-
-    case INDEX_op_div2_i64:
-        /* ??? We get an unnecessary sign-extension of the dividend
-           into R3 with this definition, but as we do in fact always
-           produce both quotient and remainder using INDEX_op_div_i64
-           instead requires jumping through even more hoops.  */
-        tcg_out_insn(s, RRE, DSGR, TCG_REG_R2, args[4]);
-        break;
-    case INDEX_op_divu2_i64:
-        tcg_out_insn(s, RRE, DLGR, TCG_REG_R2, args[4]);
-        break;
-    case INDEX_op_mulu2_i64:
-        tcg_out_insn(s, RRE, MLGR, TCG_REG_R2, args[3]);
-        break;
-
-    case INDEX_op_shl_i64:
-        op = RSY_SLLG;
-    do_shift64:
-        if (const_args[2]) {
-            tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]);
-        } else {
-            tcg_out_sh64(s, op, args[0], args[1], args[2], 0);
-        }
-        break;
-    case INDEX_op_shr_i64:
-        op = RSY_SRLG;
-        goto do_shift64;
-    case INDEX_op_sar_i64:
-        op = RSY_SRAG;
-        goto do_shift64;
-
-    case INDEX_op_rotl_i64:
-        if (const_args[2]) {
-            tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
-                         TCG_REG_NONE, args[2]);
-        } else {
-            tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0);
-        }
-        break;
-    case INDEX_op_rotr_i64:
-        if (const_args[2]) {
-            tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
-                         TCG_REG_NONE, (64 - args[2]) & 63);
-        } else {
-            /* We can use the smaller 32-bit negate because only the
-               low 6 bits are examined for the rotate.  */
-            tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
-            tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0);
-        }
-        break;
-
-    case INDEX_op_ext8s_i64:
-        tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
-    case INDEX_op_ext16s_i64:
-        tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
-    case INDEX_op_ext32s_i64:
-        tgen_ext32s(s, args[0], args[1]);
-        break;
-    case INDEX_op_ext8u_i64:
-        tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
-    case INDEX_op_ext16u_i64:
-        tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
-        break;
-    case INDEX_op_ext32u_i64:
-        tgen_ext32u(s, args[0], args[1]);
-        break;
-
-    case INDEX_op_add2_i64:
-        if (const_args[4]) {
-            if ((int64_t)args[4] >= 0) {
-                tcg_out_insn(s, RIL, ALGFI, args[0], args[4]);
-            } else {
-                tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]);
-            }
-        } else {
-            tcg_out_insn(s, RRE, ALGR, args[0], args[4]);
-        }
-        tcg_out_insn(s, RRE, ALCGR, args[1], args[5]);
-        break;
-    case INDEX_op_sub2_i64:
-        if (const_args[4]) {
-            if ((int64_t)args[4] >= 0) {
-                tcg_out_insn(s, RIL, SLGFI, args[0], args[4]);
-            } else {
-                tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]);
-            }
-        } else {
-            tcg_out_insn(s, RRE, SLGR, args[0], args[4]);
-        }
-        tcg_out_insn(s, RRE, SLBGR, args[1], args[5]);
-        break;
-
-    case INDEX_op_brcond_i64:
-        tgen_brcond(s, TCG_TYPE_I64, args[2], args[0],
-                    args[1], const_args[1], args[3]);
-        break;
-    case INDEX_op_setcond_i64:
-        tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1],
-                     args[2], const_args[2]);
-        break;
-    case INDEX_op_movcond_i64:
-        tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1],
-                     args[2], const_args[2], args[3]);
-        break;
-
-    OP_32_64(deposit):
-        tgen_deposit(s, args[0], args[2], args[3], args[4]);
-        break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef s390_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "r", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
-
-    { INDEX_op_add_i32, { "r", "r", "ri" } },
-    { INDEX_op_sub_i32, { "r", "0", "ri" } },
-    { INDEX_op_mul_i32, { "r", "0", "rK" } },
-
-    { INDEX_op_div2_i32, { "b", "a", "0", "1", "r" } },
-    { INDEX_op_divu2_i32, { "b", "a", "0", "1", "r" } },
-
-    { INDEX_op_and_i32, { "r", "0", "ri" } },
-    { INDEX_op_or_i32, { "r", "0", "rO" } },
-    { INDEX_op_xor_i32, { "r", "0", "rX" } },
-
-    { INDEX_op_neg_i32, { "r", "r" } },
-
-    { INDEX_op_shl_i32, { "r", "0", "Ri" } },
-    { INDEX_op_shr_i32, { "r", "0", "Ri" } },
-    { INDEX_op_sar_i32, { "r", "0", "Ri" } },
-
-    { INDEX_op_rotl_i32, { "r", "r", "Ri" } },
-    { INDEX_op_rotr_i32, { "r", "r", "Ri" } },
-
-    { INDEX_op_ext8s_i32, { "r", "r" } },
-    { INDEX_op_ext8u_i32, { "r", "r" } },
-    { INDEX_op_ext16s_i32, { "r", "r" } },
-    { INDEX_op_ext16u_i32, { "r", "r" } },
-
-    { INDEX_op_bswap16_i32, { "r", "r" } },
-    { INDEX_op_bswap32_i32, { "r", "r" } },
-
-    { INDEX_op_add2_i32, { "r", "r", "0", "1", "rA", "r" } },
-    { INDEX_op_sub2_i32, { "r", "r", "0", "1", "rA", "r" } },
-
-    { INDEX_op_brcond_i32, { "r", "rC" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rC" } },
-    { INDEX_op_movcond_i32, { "r", "r", "rC", "r", "0" } },
-    { INDEX_op_deposit_i32, { "r", "0", "r" } },
-
-    { INDEX_op_qemu_ld_i32, { "r", "L" } },
-    { INDEX_op_qemu_ld_i64, { "r", "L" } },
-    { INDEX_op_qemu_st_i32, { "L", "L" } },
-    { INDEX_op_qemu_st_i64, { "L", "L" } },
-
-    { INDEX_op_ld8u_i64, { "r", "r" } },
-    { INDEX_op_ld8s_i64, { "r", "r" } },
-    { INDEX_op_ld16u_i64, { "r", "r" } },
-    { INDEX_op_ld16s_i64, { "r", "r" } },
-    { INDEX_op_ld32u_i64, { "r", "r" } },
-    { INDEX_op_ld32s_i64, { "r", "r" } },
-    { INDEX_op_ld_i64, { "r", "r" } },
-
-    { INDEX_op_st8_i64, { "r", "r" } },
-    { INDEX_op_st16_i64, { "r", "r" } },
-    { INDEX_op_st32_i64, { "r", "r" } },
-    { INDEX_op_st_i64, { "r", "r" } },
-
-    { INDEX_op_add_i64, { "r", "r", "ri" } },
-    { INDEX_op_sub_i64, { "r", "0", "ri" } },
-    { INDEX_op_mul_i64, { "r", "0", "rK" } },
-
-    { INDEX_op_div2_i64, { "b", "a", "0", "1", "r" } },
-    { INDEX_op_divu2_i64, { "b", "a", "0", "1", "r" } },
-    { INDEX_op_mulu2_i64, { "b", "a", "0", "r" } },
-
-    { INDEX_op_and_i64, { "r", "0", "ri" } },
-    { INDEX_op_or_i64, { "r", "0", "rO" } },
-    { INDEX_op_xor_i64, { "r", "0", "rX" } },
-
-    { INDEX_op_neg_i64, { "r", "r" } },
-
-    { INDEX_op_shl_i64, { "r", "r", "Ri" } },
-    { INDEX_op_shr_i64, { "r", "r", "Ri" } },
-    { INDEX_op_sar_i64, { "r", "r", "Ri" } },
-
-    { INDEX_op_rotl_i64, { "r", "r", "Ri" } },
-    { INDEX_op_rotr_i64, { "r", "r", "Ri" } },
-
-    { INDEX_op_ext8s_i64, { "r", "r" } },
-    { INDEX_op_ext8u_i64, { "r", "r" } },
-    { INDEX_op_ext16s_i64, { "r", "r" } },
-    { INDEX_op_ext16u_i64, { "r", "r" } },
-    { INDEX_op_ext32s_i64, { "r", "r" } },
-    { INDEX_op_ext32u_i64, { "r", "r" } },
-
-    { INDEX_op_bswap16_i64, { "r", "r" } },
-    { INDEX_op_bswap32_i64, { "r", "r" } },
-    { INDEX_op_bswap64_i64, { "r", "r" } },
-
-    { INDEX_op_add2_i64, { "r", "r", "0", "1", "rA", "r" } },
-    { INDEX_op_sub2_i64, { "r", "r", "0", "1", "rA", "r" } },
-
-    { INDEX_op_brcond_i64, { "r", "rC" } },
-    { INDEX_op_setcond_i64, { "r", "r", "rC" } },
-    { INDEX_op_movcond_i64, { "r", "r", "rC", "r", "0" } },
-    { INDEX_op_deposit_i64, { "r", "0", "r" } },
-
-    { -1 },
-};
-
-static void query_facilities(void)
-{
-    unsigned long hwcap = qemu_getauxval(AT_HWCAP);
-
-    /* Is STORE FACILITY LIST EXTENDED available?  Honestly, I believe this
-       is present on all 64-bit systems, but let's check for it anyway.  */
-    if (hwcap & HWCAP_S390_STFLE) {
-        register int r0 __asm__("0");
-        register void *r1 __asm__("1");
-
-        /* stfle 0(%r1) */
-        r1 = &facilities;
-        asm volatile(".word 0xb2b0,0x1000"
-                     : "=r"(r0) : "0"(0), "r"(r1) : "memory", "cc");
-    }
-}
-
-static void tcg_target_init(TCGContext *s)
-{
-    query_facilities();
-
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffff);
-
-    tcg_regset_clear(tcg_target_call_clobber_regs);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
-    /* The r6 register is technically call-saved, but it's also a parameter
-       register, so it can get killed by setup for the qemu_st helper.  */
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
-    /* The return register can be considered call-clobbered.  */
-    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
-    /* XXX many insns can't be used with R0, so we better avoid it for now */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
-
-    tcg_add_target_add_op_defs(s390_op_defs);
-}
-
-#define FRAME_SIZE  ((int)(TCG_TARGET_CALL_STACK_OFFSET          \
-                           + TCG_STATIC_CALL_ARGS_SIZE           \
-                           + CPU_TEMP_BUF_NLONGS * sizeof(long)))
-
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    /* stmg %r6,%r15,48(%r15) (save registers) */
-    tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
-
-    /* aghi %r15,-frame_size */
-    tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
-
-    tcg_set_frame(s, TCG_REG_CALL_STACK,
-                  TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-
-    if (GUEST_BASE >= 0x80000) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
-        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
-    }
-
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-    /* br %r3 (go to TB) */
-    tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
-
-    tb_ret_addr = s->code_ptr;
-
-    /* lmg %r6,%r15,fs+48(%r15) (restore registers) */
-    tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
-                 FRAME_SIZE + 48);
-
-    /* br %r14 (return) */
-    tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
-}
-
-typedef struct {
-    DebugFrameHeader h;
-    uint8_t fde_def_cfa[4];
-    uint8_t fde_reg_ofs[18];
-} DebugFrame;
-
-/* We're expecting a 2 byte uleb128 encoded value.  */
-QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
-
-#define ELF_HOST_MACHINE  EM_S390
-
-static const DebugFrame debug_frame = {
-    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .h.cie.id = -1,
-    .h.cie.version = 1,
-    .h.cie.code_align = 1,
-    .h.cie.data_align = 8,                /* sleb128 8 */
-    .h.cie.return_column = TCG_REG_R14,
-
-    /* Total FDE size does not include the "len" member.  */
-    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
-
-    .fde_def_cfa = {
-        12, TCG_REG_CALL_STACK,         /* DW_CFA_def_cfa %r15, ... */
-        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
-        (FRAME_SIZE >> 7)
-    },
-    .fde_reg_ofs = {
-        0x86, 6,                        /* DW_CFA_offset, %r6, 48 */
-        0x87, 7,                        /* DW_CFA_offset, %r7, 56 */
-        0x88, 8,                        /* DW_CFA_offset, %r8, 64 */
-        0x89, 9,                        /* DW_CFA_offset, %r92, 72 */
-        0x8a, 10,                       /* DW_CFA_offset, %r10, 80 */
-        0x8b, 11,                       /* DW_CFA_offset, %r11, 88 */
-        0x8c, 12,                       /* DW_CFA_offset, %r12, 96 */
-        0x8d, 13,                       /* DW_CFA_offset, %r13, 104 */
-        0x8e, 14,                       /* DW_CFA_offset, %r14, 112 */
-    }
-};
-
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index 5acc28c..0c1af24 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -21,10 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_S390 
-#define TCG_TARGET_S390 1
+
+#ifndef S390_TCG_TARGET_H
+#define S390_TCG_TARGET_H
 
 #define TCG_TARGET_INSN_UNIT_SIZE 2
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 19
 
 typedef enum TCGReg {
     TCG_REG_R0 = 0,
@@ -71,7 +73,8 @@
 #define TCG_TARGET_HAS_muls2_i32        0
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 
 #define TCG_TARGET_HAS_div2_i64         1
 #define TCG_TARGET_HAS_rot_i64          1
diff --git a/tcg/s390/tcg-target.inc.c b/tcg/s390/tcg-target.inc.c
new file mode 100644
index 0000000..5a7495b
--- /dev/null
+++ b/tcg/s390/tcg-target.inc.c
@@ -0,0 +1,2425 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
+ * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
+ * Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg-be-ldst.h"
+
+/* We only support generating code for 64-bit mode.  */
+#if TCG_TARGET_REG_BITS != 64
+#error "unsupported code generation mode"
+#endif
+
+#include "elf.h"
+
+/* ??? The translation blocks produced by TCG are generally small enough to
+   be entirely reachable with a 16-bit displacement.  Leaving the option for
+   a 32-bit displacement here Just In Case.  */
+#define USE_LONG_BRANCHES 0
+
+#define TCG_CT_CONST_MULI  0x100
+#define TCG_CT_CONST_ORI   0x200
+#define TCG_CT_CONST_XORI  0x400
+#define TCG_CT_CONST_CMPI  0x800
+#define TCG_CT_CONST_ADLI  0x1000
+
+/* Several places within the instruction set 0 means "no register"
+   rather than TCG_REG_R0.  */
+#define TCG_REG_NONE    0
+
+/* A scratch register that may be be used throughout the backend.  */
+#define TCG_TMP0        TCG_REG_R14
+
+#ifndef CONFIG_SOFTMMU
+#define TCG_GUEST_BASE_REG TCG_REG_R13
+#endif
+
+/* All of the following instructions are prefixed with their instruction
+   format, and are defined as 8- or 16-bit quantities, even when the two
+   halves of the 16-bit quantity may appear 32 bits apart in the insn.
+   This makes it easy to copy the values from the tables in Appendix B.  */
+typedef enum S390Opcode {
+    RIL_AFI     = 0xc209,
+    RIL_AGFI    = 0xc208,
+    RIL_ALFI    = 0xc20b,
+    RIL_ALGFI   = 0xc20a,
+    RIL_BRASL   = 0xc005,
+    RIL_BRCL    = 0xc004,
+    RIL_CFI     = 0xc20d,
+    RIL_CGFI    = 0xc20c,
+    RIL_CLFI    = 0xc20f,
+    RIL_CLGFI   = 0xc20e,
+    RIL_IIHF    = 0xc008,
+    RIL_IILF    = 0xc009,
+    RIL_LARL    = 0xc000,
+    RIL_LGFI    = 0xc001,
+    RIL_LGRL    = 0xc408,
+    RIL_LLIHF   = 0xc00e,
+    RIL_LLILF   = 0xc00f,
+    RIL_LRL     = 0xc40d,
+    RIL_MSFI    = 0xc201,
+    RIL_MSGFI   = 0xc200,
+    RIL_NIHF    = 0xc00a,
+    RIL_NILF    = 0xc00b,
+    RIL_OIHF    = 0xc00c,
+    RIL_OILF    = 0xc00d,
+    RIL_SLFI    = 0xc205,
+    RIL_SLGFI   = 0xc204,
+    RIL_XIHF    = 0xc006,
+    RIL_XILF    = 0xc007,
+
+    RI_AGHI     = 0xa70b,
+    RI_AHI      = 0xa70a,
+    RI_BRC      = 0xa704,
+    RI_IIHH     = 0xa500,
+    RI_IIHL     = 0xa501,
+    RI_IILH     = 0xa502,
+    RI_IILL     = 0xa503,
+    RI_LGHI     = 0xa709,
+    RI_LLIHH    = 0xa50c,
+    RI_LLIHL    = 0xa50d,
+    RI_LLILH    = 0xa50e,
+    RI_LLILL    = 0xa50f,
+    RI_MGHI     = 0xa70d,
+    RI_MHI      = 0xa70c,
+    RI_NIHH     = 0xa504,
+    RI_NIHL     = 0xa505,
+    RI_NILH     = 0xa506,
+    RI_NILL     = 0xa507,
+    RI_OIHH     = 0xa508,
+    RI_OIHL     = 0xa509,
+    RI_OILH     = 0xa50a,
+    RI_OILL     = 0xa50b,
+
+    RIE_CGIJ    = 0xec7c,
+    RIE_CGRJ    = 0xec64,
+    RIE_CIJ     = 0xec7e,
+    RIE_CLGRJ   = 0xec65,
+    RIE_CLIJ    = 0xec7f,
+    RIE_CLGIJ   = 0xec7d,
+    RIE_CLRJ    = 0xec77,
+    RIE_CRJ     = 0xec76,
+    RIE_RISBG   = 0xec55,
+
+    RRE_AGR     = 0xb908,
+    RRE_ALGR    = 0xb90a,
+    RRE_ALCR    = 0xb998,
+    RRE_ALCGR   = 0xb988,
+    RRE_CGR     = 0xb920,
+    RRE_CLGR    = 0xb921,
+    RRE_DLGR    = 0xb987,
+    RRE_DLR     = 0xb997,
+    RRE_DSGFR   = 0xb91d,
+    RRE_DSGR    = 0xb90d,
+    RRE_LGBR    = 0xb906,
+    RRE_LCGR    = 0xb903,
+    RRE_LGFR    = 0xb914,
+    RRE_LGHR    = 0xb907,
+    RRE_LGR     = 0xb904,
+    RRE_LLGCR   = 0xb984,
+    RRE_LLGFR   = 0xb916,
+    RRE_LLGHR   = 0xb985,
+    RRE_LRVR    = 0xb91f,
+    RRE_LRVGR   = 0xb90f,
+    RRE_LTGR    = 0xb902,
+    RRE_MLGR    = 0xb986,
+    RRE_MSGR    = 0xb90c,
+    RRE_MSR     = 0xb252,
+    RRE_NGR     = 0xb980,
+    RRE_OGR     = 0xb981,
+    RRE_SGR     = 0xb909,
+    RRE_SLGR    = 0xb90b,
+    RRE_SLBR    = 0xb999,
+    RRE_SLBGR   = 0xb989,
+    RRE_XGR     = 0xb982,
+
+    RRF_LOCR    = 0xb9f2,
+    RRF_LOCGR   = 0xb9e2,
+
+    RR_AR       = 0x1a,
+    RR_ALR      = 0x1e,
+    RR_BASR     = 0x0d,
+    RR_BCR      = 0x07,
+    RR_CLR      = 0x15,
+    RR_CR       = 0x19,
+    RR_DR       = 0x1d,
+    RR_LCR      = 0x13,
+    RR_LR       = 0x18,
+    RR_LTR      = 0x12,
+    RR_NR       = 0x14,
+    RR_OR       = 0x16,
+    RR_SR       = 0x1b,
+    RR_SLR      = 0x1f,
+    RR_XR       = 0x17,
+
+    RSY_RLL     = 0xeb1d,
+    RSY_RLLG    = 0xeb1c,
+    RSY_SLLG    = 0xeb0d,
+    RSY_SRAG    = 0xeb0a,
+    RSY_SRLG    = 0xeb0c,
+
+    RS_SLL      = 0x89,
+    RS_SRA      = 0x8a,
+    RS_SRL      = 0x88,
+
+    RXY_AG      = 0xe308,
+    RXY_AY      = 0xe35a,
+    RXY_CG      = 0xe320,
+    RXY_CY      = 0xe359,
+    RXY_LAY     = 0xe371,
+    RXY_LB      = 0xe376,
+    RXY_LG      = 0xe304,
+    RXY_LGB     = 0xe377,
+    RXY_LGF     = 0xe314,
+    RXY_LGH     = 0xe315,
+    RXY_LHY     = 0xe378,
+    RXY_LLGC    = 0xe390,
+    RXY_LLGF    = 0xe316,
+    RXY_LLGH    = 0xe391,
+    RXY_LMG     = 0xeb04,
+    RXY_LRV     = 0xe31e,
+    RXY_LRVG    = 0xe30f,
+    RXY_LRVH    = 0xe31f,
+    RXY_LY      = 0xe358,
+    RXY_STCY    = 0xe372,
+    RXY_STG     = 0xe324,
+    RXY_STHY    = 0xe370,
+    RXY_STMG    = 0xeb24,
+    RXY_STRV    = 0xe33e,
+    RXY_STRVG   = 0xe32f,
+    RXY_STRVH   = 0xe33f,
+    RXY_STY     = 0xe350,
+
+    RX_A        = 0x5a,
+    RX_C        = 0x59,
+    RX_L        = 0x58,
+    RX_LA       = 0x41,
+    RX_LH       = 0x48,
+    RX_ST       = 0x50,
+    RX_STC      = 0x42,
+    RX_STH      = 0x40,
+
+    NOP         = 0x0707,
+} S390Opcode;
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
+    "%r8", "%r9", "%r10" "%r11" "%r12" "%r13" "%r14" "%r15"
+};
+#endif
+
+/* Since R6 is a potential argument register, choose it last of the
+   call-saved registers.  Likewise prefer the call-clobbered registers
+   in reverse order to maximize the chance of avoiding the arguments.  */
+static const int tcg_target_reg_alloc_order[] = {
+    /* Call saved registers.  */
+    TCG_REG_R13,
+    TCG_REG_R12,
+    TCG_REG_R11,
+    TCG_REG_R10,
+    TCG_REG_R9,
+    TCG_REG_R8,
+    TCG_REG_R7,
+    TCG_REG_R6,
+    /* Call clobbered registers.  */
+    TCG_REG_R14,
+    TCG_REG_R0,
+    TCG_REG_R1,
+    /* Argument registers, in reverse order of allocation.  */
+    TCG_REG_R5,
+    TCG_REG_R4,
+    TCG_REG_R3,
+    TCG_REG_R2,
+};
+
+static const int tcg_target_call_iarg_regs[] = {
+    TCG_REG_R2,
+    TCG_REG_R3,
+    TCG_REG_R4,
+    TCG_REG_R5,
+    TCG_REG_R6,
+};
+
+static const int tcg_target_call_oarg_regs[] = {
+    TCG_REG_R2,
+};
+
+#define S390_CC_EQ      8
+#define S390_CC_LT      4
+#define S390_CC_GT      2
+#define S390_CC_OV      1
+#define S390_CC_NE      (S390_CC_LT | S390_CC_GT)
+#define S390_CC_LE      (S390_CC_LT | S390_CC_EQ)
+#define S390_CC_GE      (S390_CC_GT | S390_CC_EQ)
+#define S390_CC_NEVER   0
+#define S390_CC_ALWAYS  15
+
+/* Condition codes that result from a COMPARE and COMPARE LOGICAL.  */
+static const uint8_t tcg_cond_to_s390_cond[] = {
+    [TCG_COND_EQ]  = S390_CC_EQ,
+    [TCG_COND_NE]  = S390_CC_NE,
+    [TCG_COND_LT]  = S390_CC_LT,
+    [TCG_COND_LE]  = S390_CC_LE,
+    [TCG_COND_GT]  = S390_CC_GT,
+    [TCG_COND_GE]  = S390_CC_GE,
+    [TCG_COND_LTU] = S390_CC_LT,
+    [TCG_COND_LEU] = S390_CC_LE,
+    [TCG_COND_GTU] = S390_CC_GT,
+    [TCG_COND_GEU] = S390_CC_GE,
+};
+
+/* Condition codes that result from a LOAD AND TEST.  Here, we have no
+   unsigned instruction variation, however since the test is vs zero we
+   can re-map the outcomes appropriately.  */
+static const uint8_t tcg_cond_to_ltr_cond[] = {
+    [TCG_COND_EQ]  = S390_CC_EQ,
+    [TCG_COND_NE]  = S390_CC_NE,
+    [TCG_COND_LT]  = S390_CC_LT,
+    [TCG_COND_LE]  = S390_CC_LE,
+    [TCG_COND_GT]  = S390_CC_GT,
+    [TCG_COND_GE]  = S390_CC_GE,
+    [TCG_COND_LTU] = S390_CC_NEVER,
+    [TCG_COND_LEU] = S390_CC_EQ,
+    [TCG_COND_GTU] = S390_CC_NE,
+    [TCG_COND_GEU] = S390_CC_ALWAYS,
+};
+
+#ifdef CONFIG_SOFTMMU
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_SB]   = helper_ret_ldsb_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LESW] = helper_le_ldsw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LESL] = helper_le_ldsl_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BESW] = helper_be_ldsw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BESL] = helper_be_ldsl_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
+};
+
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
+};
+#endif
+
+static tcg_insn_unit *tb_ret_addr;
+
+/* A list of relevant facilities used by this translator.  Some of these
+   are required for proper operation, and these are checked at startup.  */
+
+#define FACILITY_ZARCH_ACTIVE	(1ULL << (63 - 2))
+#define FACILITY_LONG_DISP	(1ULL << (63 - 18))
+#define FACILITY_EXT_IMM	(1ULL << (63 - 21))
+#define FACILITY_GEN_INST_EXT	(1ULL << (63 - 34))
+#define FACILITY_LOAD_ON_COND   (1ULL << (63 - 45))
+
+static uint64_t facilities;
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    intptr_t pcrel2 = (tcg_insn_unit *)value - (code_ptr - 1);
+    tcg_debug_assert(addend == -2);
+
+    switch (type) {
+    case R_390_PC16DBL:
+        tcg_debug_assert(pcrel2 == (int16_t)pcrel2);
+        tcg_patch16(code_ptr, pcrel2);
+        break;
+    case R_390_PC32DBL:
+        tcg_debug_assert(pcrel2 == (int32_t)pcrel2);
+        tcg_patch32(code_ptr, pcrel2);
+        break;
+    default:
+        tcg_abort();
+        break;
+    }
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str = *pct_str;
+
+    switch (ct_str[0]) {
+    case 'r':                  /* all registers */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        break;
+    case 'R':                  /* not R0 */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
+        break;
+    case 'L':                  /* qemu_ld/st constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffff);
+        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R2);
+        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R3);
+        tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
+        break;
+    case 'a':                  /* force R2 for division */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_clear(ct->u.regs);
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_R2);
+        break;
+    case 'b':                  /* force R3 for division */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_clear(ct->u.regs);
+        tcg_regset_set_reg(ct->u.regs, TCG_REG_R3);
+        break;
+    case 'A':
+        ct->ct |= TCG_CT_CONST_ADLI;
+        break;
+    case 'K':
+        ct->ct |= TCG_CT_CONST_MULI;
+        break;
+    case 'O':
+        ct->ct |= TCG_CT_CONST_ORI;
+        break;
+    case 'X':
+        ct->ct |= TCG_CT_CONST_XORI;
+        break;
+    case 'C':
+        ct->ct |= TCG_CT_CONST_CMPI;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+
+    return 0;
+}
+
+/* Immediates to be used with logical OR.  This is an optimization only,
+   since a full 64-bit immediate OR can always be performed with 4 sequential
+   OI[LH][LH] instructions.  What we're looking for is immediates that we
+   can load efficiently, and the immediate load plus the reg-reg OR is
+   smaller than the sequential OI's.  */
+
+static int tcg_match_ori(TCGType type, tcg_target_long val)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        if (type == TCG_TYPE_I32) {
+            /* All 32-bit ORs can be performed with 1 48-bit insn.  */
+            return 1;
+        }
+    }
+
+    /* Look for negative values.  These are best to load with LGHI.  */
+    if (val < 0) {
+        if (val == (int16_t)val) {
+            return 0;
+        }
+        if (facilities & FACILITY_EXT_IMM) {
+            if (val == (int32_t)val) {
+                return 0;
+            }
+        }
+    }
+
+    return 1;
+}
+
+/* Immediates to be used with logical XOR.  This is almost, but not quite,
+   only an optimization.  XOR with immediate is only supported with the
+   extended-immediate facility.  That said, there are a few patterns for
+   which it is better to load the value into a register first.  */
+
+static int tcg_match_xori(TCGType type, tcg_target_long val)
+{
+    if ((facilities & FACILITY_EXT_IMM) == 0) {
+        return 0;
+    }
+
+    if (type == TCG_TYPE_I32) {
+        /* All 32-bit XORs can be performed with 1 48-bit insn.  */
+        return 1;
+    }
+
+    /* Look for negative values.  These are best to load with LGHI.  */
+    if (val < 0 && val == (int32_t)val) {
+        return 0;
+    }
+
+    return 1;
+}
+
+/* Imediates to be used with comparisons.  */
+
+static int tcg_match_cmpi(TCGType type, tcg_target_long val)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        /* The COMPARE IMMEDIATE instruction is available.  */
+        if (type == TCG_TYPE_I32) {
+            /* We have a 32-bit immediate and can compare against anything.  */
+            return 1;
+        } else {
+            /* ??? We have no insight here into whether the comparison is
+               signed or unsigned.  The COMPARE IMMEDIATE insn uses a 32-bit
+               signed immediate, and the COMPARE LOGICAL IMMEDIATE insn uses
+               a 32-bit unsigned immediate.  If we were to use the (semi)
+               obvious "val == (int32_t)val" we would be enabling unsigned
+               comparisons vs very large numbers.  The only solution is to
+               take the intersection of the ranges.  */
+            /* ??? Another possible solution is to simply lie and allow all
+               constants here and force the out-of-range values into a temp
+               register in tgen_cmp when we have knowledge of the actual
+               comparison code in use.  */
+            return val >= 0 && val <= 0x7fffffff;
+        }
+    } else {
+        /* Only the LOAD AND TEST instruction is available.  */
+        return val == 0;
+    }
+}
+
+/* Immediates to be used with add2/sub2.  */
+
+static int tcg_match_add2i(TCGType type, tcg_target_long val)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        if (type == TCG_TYPE_I32) {
+            return 1;
+        } else if (val >= -0xffffffffll && val <= 0xffffffffll) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+/* Test if a constant matches the constraint. */
+static int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                  const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    }
+
+    if (type == TCG_TYPE_I32) {
+        val = (int32_t)val;
+    }
+
+    /* The following are mutually exclusive.  */
+    if (ct & TCG_CT_CONST_MULI) {
+        /* Immediates that may be used with multiply.  If we have the
+           general-instruction-extensions, then we have MULTIPLY SINGLE
+           IMMEDIATE with a signed 32-bit, otherwise we have only
+           MULTIPLY HALFWORD IMMEDIATE, with a signed 16-bit.  */
+        if (facilities & FACILITY_GEN_INST_EXT) {
+            return val == (int32_t)val;
+        } else {
+            return val == (int16_t)val;
+        }
+    } else if (ct & TCG_CT_CONST_ADLI) {
+        return tcg_match_add2i(type, val);
+    } else if (ct & TCG_CT_CONST_ORI) {
+        return tcg_match_ori(type, val);
+    } else if (ct & TCG_CT_CONST_XORI) {
+        return tcg_match_xori(type, val);
+    } else if (ct & TCG_CT_CONST_CMPI) {
+        return tcg_match_cmpi(type, val);
+    }
+
+    return 0;
+}
+
+/* Emit instructions according to the given instruction format.  */
+
+static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
+{
+    tcg_out16(s, (op << 8) | (r1 << 4) | r2);
+}
+
+static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
+                             TCGReg r1, TCGReg r2)
+{
+    tcg_out32(s, (op << 16) | (r1 << 4) | r2);
+}
+
+static void tcg_out_insn_RRF(TCGContext *s, S390Opcode op,
+                             TCGReg r1, TCGReg r2, int m3)
+{
+    tcg_out32(s, (op << 16) | (m3 << 12) | (r1 << 4) | r2);
+}
+
+static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
+{
+    tcg_out32(s, (op << 16) | (r1 << 20) | (i2 & 0xffff));
+}
+
+static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
+{
+    tcg_out16(s, op | (r1 << 4));
+    tcg_out32(s, i2);
+}
+
+static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
+                            TCGReg b2, TCGReg r3, int disp)
+{
+    tcg_out32(s, (op << 24) | (r1 << 20) | (r3 << 16) | (b2 << 12)
+              | (disp & 0xfff));
+}
+
+static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
+                             TCGReg b2, TCGReg r3, int disp)
+{
+    tcg_out16(s, (op & 0xff00) | (r1 << 4) | r3);
+    tcg_out32(s, (op & 0xff) | (b2 << 28)
+              | ((disp & 0xfff) << 16) | ((disp & 0xff000) >> 4));
+}
+
+#define tcg_out_insn_RX   tcg_out_insn_RS
+#define tcg_out_insn_RXY  tcg_out_insn_RSY
+
+/* Emit an opcode with "type-checking" of the format.  */
+#define tcg_out_insn(S, FMT, OP, ...) \
+    glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
+
+
+/* emit 64-bit shifts */
+static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
+                         TCGReg src, TCGReg sh_reg, int sh_imm)
+{
+    tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
+}
+
+/* emit 32-bit shifts */
+static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
+                         TCGReg sh_reg, int sh_imm)
+{
+    tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
+}
+
+static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
+{
+    if (src != dst) {
+        if (type == TCG_TYPE_I32) {
+            tcg_out_insn(s, RR, LR, dst, src);
+        } else {
+            tcg_out_insn(s, RRE, LGR, dst, src);
+        }
+    }
+}
+
+/* load a register with an immediate value */
+static void tcg_out_movi(TCGContext *s, TCGType type,
+                         TCGReg ret, tcg_target_long sval)
+{
+    static const S390Opcode lli_insns[4] = {
+        RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
+    };
+
+    tcg_target_ulong uval = sval;
+    int i;
+
+    if (type == TCG_TYPE_I32) {
+        uval = (uint32_t)sval;
+        sval = (int32_t)sval;
+    }
+
+    /* Try all 32-bit insns that can load it in one go.  */
+    if (sval >= -0x8000 && sval < 0x8000) {
+        tcg_out_insn(s, RI, LGHI, ret, sval);
+        return;
+    }
+
+    for (i = 0; i < 4; i++) {
+        tcg_target_long mask = 0xffffull << i*16;
+        if ((uval & mask) == uval) {
+            tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i*16);
+            return;
+        }
+    }
+
+    /* Try all 48-bit insns that can load it in one go.  */
+    if (facilities & FACILITY_EXT_IMM) {
+        if (sval == (int32_t)sval) {
+            tcg_out_insn(s, RIL, LGFI, ret, sval);
+            return;
+        }
+        if (uval <= 0xffffffff) {
+            tcg_out_insn(s, RIL, LLILF, ret, uval);
+            return;
+        }
+        if ((uval & 0xffffffff) == 0) {
+            tcg_out_insn(s, RIL, LLIHF, ret, uval >> 31 >> 1);
+            return;
+        }
+    }
+
+    /* Try for PC-relative address load.  */
+    if ((sval & 1) == 0) {
+        ptrdiff_t off = tcg_pcrel_diff(s, (void *)sval) >> 1;
+        if (off == (int32_t)off) {
+            tcg_out_insn(s, RIL, LARL, ret, off);
+            return;
+        }
+    }
+
+    /* If extended immediates are not present, then we may have to issue
+       several instructions to load the low 32 bits.  */
+    if (!(facilities & FACILITY_EXT_IMM)) {
+        /* A 32-bit unsigned value can be loaded in 2 insns.  And given
+           that the lli_insns loop above did not succeed, we know that
+           both insns are required.  */
+        if (uval <= 0xffffffff) {
+            tcg_out_insn(s, RI, LLILL, ret, uval);
+            tcg_out_insn(s, RI, IILH, ret, uval >> 16);
+            return;
+        }
+
+        /* If all high bits are set, the value can be loaded in 2 or 3 insns.
+           We first want to make sure that all the high bits get set.  With
+           luck the low 16-bits can be considered negative to perform that for
+           free, otherwise we load an explicit -1.  */
+        if (sval >> 31 >> 1 == -1) {
+            if (uval & 0x8000) {
+                tcg_out_insn(s, RI, LGHI, ret, uval);
+            } else {
+                tcg_out_insn(s, RI, LGHI, ret, -1);
+                tcg_out_insn(s, RI, IILL, ret, uval);
+            }
+            tcg_out_insn(s, RI, IILH, ret, uval >> 16);
+            return;
+        }
+    }
+
+    /* If we get here, both the high and low parts have non-zero bits.  */
+
+    /* Recurse to load the lower 32-bits.  */
+    tcg_out_movi(s, TCG_TYPE_I64, ret, uval & 0xffffffff);
+
+    /* Insert data into the high 32-bits.  */
+    uval = uval >> 31 >> 1;
+    if (facilities & FACILITY_EXT_IMM) {
+        if (uval < 0x10000) {
+            tcg_out_insn(s, RI, IIHL, ret, uval);
+        } else if ((uval & 0xffff) == 0) {
+            tcg_out_insn(s, RI, IIHH, ret, uval >> 16);
+        } else {
+            tcg_out_insn(s, RIL, IIHF, ret, uval);
+        }
+    } else {
+        if (uval & 0xffff) {
+            tcg_out_insn(s, RI, IIHL, ret, uval);
+        }
+        if (uval & 0xffff0000) {
+            tcg_out_insn(s, RI, IIHH, ret, uval >> 16);
+        }
+    }
+}
+
+
+/* Emit a load/store type instruction.  Inputs are:
+   DATA:     The register to be loaded or stored.
+   BASE+OFS: The effective address.
+   OPC_RX:   If the operation has an RX format opcode (e.g. STC), otherwise 0.
+   OPC_RXY:  The RXY format opcode for the operation (e.g. STCY).  */
+
+static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
+                        TCGReg data, TCGReg base, TCGReg index,
+                        tcg_target_long ofs)
+{
+    if (ofs < -0x80000 || ofs >= 0x80000) {
+        /* Combine the low 20 bits of the offset with the actual load insn;
+           the high 44 bits must come from an immediate load.  */
+        tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
+        ofs = low;
+
+        /* If we were already given an index register, add it in.  */
+        if (index != TCG_REG_NONE) {
+            tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
+        }
+        index = TCG_TMP0;
+    }
+
+    if (opc_rx && ofs >= 0 && ofs < 0x1000) {
+        tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
+    } else {
+        tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
+    }
+}
+
+
+/* load data without address translation or endianness conversion */
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
+                              TCGReg base, intptr_t ofs)
+{
+    if (type == TCG_TYPE_I32) {
+        tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
+    } else {
+        tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
+    }
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
+                              TCGReg base, intptr_t ofs)
+{
+    if (type == TCG_TYPE_I32) {
+        tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
+    } else {
+        tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
+    }
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    return false;
+}
+
+/* load data from an absolute host address */
+static void tcg_out_ld_abs(TCGContext *s, TCGType type, TCGReg dest, void *abs)
+{
+    intptr_t addr = (intptr_t)abs;
+
+    if ((facilities & FACILITY_GEN_INST_EXT) && !(addr & 1)) {
+        ptrdiff_t disp = tcg_pcrel_diff(s, abs) >> 1;
+        if (disp == (int32_t)disp) {
+            if (type == TCG_TYPE_I32) {
+                tcg_out_insn(s, RIL, LRL, dest, disp);
+            } else {
+                tcg_out_insn(s, RIL, LGRL, dest, disp);
+            }
+            return;
+        }
+    }
+
+    tcg_out_movi(s, TCG_TYPE_PTR, dest, addr & ~0xffff);
+    tcg_out_ld(s, type, dest, dest, addr & 0xffff);
+}
+
+static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
+                                 int msb, int lsb, int ofs, int z)
+{
+    /* Format RIE-f */
+    tcg_out16(s, (RIE_RISBG & 0xff00) | (dest << 4) | src);
+    tcg_out16(s, (msb << 8) | (z << 7) | lsb);
+    tcg_out16(s, (ofs << 8) | (RIE_RISBG & 0xff));
+}
+
+static void tgen_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        tcg_out_insn(s, RRE, LGBR, dest, src);
+        return;
+    }
+
+    if (type == TCG_TYPE_I32) {
+        if (dest == src) {
+            tcg_out_sh32(s, RS_SLL, dest, TCG_REG_NONE, 24);
+        } else {
+            tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 24);
+        }
+        tcg_out_sh32(s, RS_SRA, dest, TCG_REG_NONE, 24);
+    } else {
+        tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 56);
+        tcg_out_sh64(s, RSY_SRAG, dest, dest, TCG_REG_NONE, 56);
+    }
+}
+
+static void tgen_ext8u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        tcg_out_insn(s, RRE, LLGCR, dest, src);
+        return;
+    }
+
+    if (dest == src) {
+        tcg_out_movi(s, type, TCG_TMP0, 0xff);
+        src = TCG_TMP0;
+    } else {
+        tcg_out_movi(s, type, dest, 0xff);
+    }
+    if (type == TCG_TYPE_I32) {
+        tcg_out_insn(s, RR, NR, dest, src);
+    } else {
+        tcg_out_insn(s, RRE, NGR, dest, src);
+    }
+}
+
+static void tgen_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        tcg_out_insn(s, RRE, LGHR, dest, src);
+        return;
+    }
+
+    if (type == TCG_TYPE_I32) {
+        if (dest == src) {
+            tcg_out_sh32(s, RS_SLL, dest, TCG_REG_NONE, 16);
+        } else {
+            tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 16);
+        }
+        tcg_out_sh32(s, RS_SRA, dest, TCG_REG_NONE, 16);
+    } else {
+        tcg_out_sh64(s, RSY_SLLG, dest, src, TCG_REG_NONE, 48);
+        tcg_out_sh64(s, RSY_SRAG, dest, dest, TCG_REG_NONE, 48);
+    }
+}
+
+static void tgen_ext16u(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
+{
+    if (facilities & FACILITY_EXT_IMM) {
+        tcg_out_insn(s, RRE, LLGHR, dest, src);
+        return;
+    }
+
+    if (dest == src) {
+        tcg_out_movi(s, type, TCG_TMP0, 0xffff);
+        src = TCG_TMP0;
+    } else {
+        tcg_out_movi(s, type, dest, 0xffff);
+    }
+    if (type == TCG_TYPE_I32) {
+        tcg_out_insn(s, RR, NR, dest, src);
+    } else {
+        tcg_out_insn(s, RRE, NGR, dest, src);
+    }
+}
+
+static inline void tgen_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_insn(s, RRE, LGFR, dest, src);
+}
+
+static inline void tgen_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
+{
+    tcg_out_insn(s, RRE, LLGFR, dest, src);
+}
+
+/* Accept bit patterns like these:
+    0....01....1
+    1....10....0
+    1..10..01..1
+    0..01..10..0
+   Copied from gcc sources.  */
+static inline bool risbg_mask(uint64_t c)
+{
+    uint64_t lsb;
+    /* We don't change the number of transitions by inverting,
+       so make sure we start with the LSB zero.  */
+    if (c & 1) {
+        c = ~c;
+    }
+    /* Reject all zeros or all ones.  */
+    if (c == 0) {
+        return false;
+    }
+    /* Find the first transition.  */
+    lsb = c & -c;
+    /* Invert to look for a second transition.  */
+    c = ~c;
+    /* Erase the first transition.  */
+    c &= -lsb;
+    /* Find the second transition, if any.  */
+    lsb = c & -c;
+    /* Match if all the bits are 1's, or if c is zero.  */
+    return c == -lsb;
+}
+
+static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
+{
+    int msb, lsb;
+    if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
+        /* Achieve wraparound by swapping msb and lsb.  */
+        msb = 64 - ctz64(~val);
+        lsb = clz64(~val) - 1;
+    } else {
+        msb = clz64(val);
+        lsb = 63 - ctz64(val);
+    }
+    tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
+}
+
+static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
+{
+    static const S390Opcode ni_insns[4] = {
+        RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
+    };
+    static const S390Opcode nif_insns[2] = {
+        RIL_NILF, RIL_NIHF
+    };
+    uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
+    int i;
+
+    /* Look for the zero-extensions.  */
+    if ((val & valid) == 0xffffffff) {
+        tgen_ext32u(s, dest, dest);
+        return;
+    }
+    if (facilities & FACILITY_EXT_IMM) {
+        if ((val & valid) == 0xff) {
+            tgen_ext8u(s, TCG_TYPE_I64, dest, dest);
+            return;
+        }
+        if ((val & valid) == 0xffff) {
+            tgen_ext16u(s, TCG_TYPE_I64, dest, dest);
+            return;
+        }
+    }
+
+    /* Try all 32-bit insns that can perform it in one go.  */
+    for (i = 0; i < 4; i++) {
+        tcg_target_ulong mask = ~(0xffffull << i*16);
+        if (((val | ~valid) & mask) == mask) {
+            tcg_out_insn_RI(s, ni_insns[i], dest, val >> i*16);
+            return;
+        }
+    }
+
+    /* Try all 48-bit insns that can perform it in one go.  */
+    if (facilities & FACILITY_EXT_IMM) {
+        for (i = 0; i < 2; i++) {
+            tcg_target_ulong mask = ~(0xffffffffull << i*32);
+            if (((val | ~valid) & mask) == mask) {
+                tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i*32);
+                return;
+            }
+        }
+    }
+    if ((facilities & FACILITY_GEN_INST_EXT) && risbg_mask(val)) {
+        tgen_andi_risbg(s, dest, dest, val);
+        return;
+    }
+
+    /* Fall back to loading the constant.  */
+    tcg_out_movi(s, type, TCG_TMP0, val);
+    if (type == TCG_TYPE_I32) {
+        tcg_out_insn(s, RR, NR, dest, TCG_TMP0);
+    } else {
+        tcg_out_insn(s, RRE, NGR, dest, TCG_TMP0);
+    }
+}
+
+static void tgen64_ori(TCGContext *s, TCGReg dest, tcg_target_ulong val)
+{
+    static const S390Opcode oi_insns[4] = {
+        RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
+    };
+    static const S390Opcode nif_insns[2] = {
+        RIL_OILF, RIL_OIHF
+    };
+
+    int i;
+
+    /* Look for no-op.  */
+    if (val == 0) {
+        return;
+    }
+
+    if (facilities & FACILITY_EXT_IMM) {
+        /* Try all 32-bit insns that can perform it in one go.  */
+        for (i = 0; i < 4; i++) {
+            tcg_target_ulong mask = (0xffffull << i*16);
+            if ((val & mask) != 0 && (val & ~mask) == 0) {
+                tcg_out_insn_RI(s, oi_insns[i], dest, val >> i*16);
+                return;
+            }
+        }
+
+        /* Try all 48-bit insns that can perform it in one go.  */
+        for (i = 0; i < 2; i++) {
+            tcg_target_ulong mask = (0xffffffffull << i*32);
+            if ((val & mask) != 0 && (val & ~mask) == 0) {
+                tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i*32);
+                return;
+            }
+        }
+
+        /* Perform the OR via sequential modifications to the high and
+           low parts.  Do this via recursion to handle 16-bit vs 32-bit
+           masks in each half.  */
+        tgen64_ori(s, dest, val & 0x00000000ffffffffull);
+        tgen64_ori(s, dest, val & 0xffffffff00000000ull);
+    } else {
+        /* With no extended-immediate facility, we don't need to be so
+           clever.  Just iterate over the insns and mask in the constant.  */
+        for (i = 0; i < 4; i++) {
+            tcg_target_ulong mask = (0xffffull << i*16);
+            if ((val & mask) != 0) {
+                tcg_out_insn_RI(s, oi_insns[i], dest, val >> i*16);
+            }
+        }
+    }
+}
+
+static void tgen64_xori(TCGContext *s, TCGReg dest, tcg_target_ulong val)
+{
+    /* Perform the xor by parts.  */
+    if (val & 0xffffffff) {
+        tcg_out_insn(s, RIL, XILF, dest, val);
+    }
+    if (val > 0xffffffff) {
+        tcg_out_insn(s, RIL, XIHF, dest, val >> 31 >> 1);
+    }
+}
+
+static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
+                    TCGArg c2, int c2const)
+{
+    bool is_unsigned = is_unsigned_cond(c);
+    if (c2const) {
+        if (c2 == 0) {
+            if (type == TCG_TYPE_I32) {
+                tcg_out_insn(s, RR, LTR, r1, r1);
+            } else {
+                tcg_out_insn(s, RRE, LTGR, r1, r1);
+            }
+            return tcg_cond_to_ltr_cond[c];
+        } else {
+            if (is_unsigned) {
+                if (type == TCG_TYPE_I32) {
+                    tcg_out_insn(s, RIL, CLFI, r1, c2);
+                } else {
+                    tcg_out_insn(s, RIL, CLGFI, r1, c2);
+                }
+            } else {
+                if (type == TCG_TYPE_I32) {
+                    tcg_out_insn(s, RIL, CFI, r1, c2);
+                } else {
+                    tcg_out_insn(s, RIL, CGFI, r1, c2);
+                }
+            }
+        }
+    } else {
+        if (is_unsigned) {
+            if (type == TCG_TYPE_I32) {
+                tcg_out_insn(s, RR, CLR, r1, c2);
+            } else {
+                tcg_out_insn(s, RRE, CLGR, r1, c2);
+            }
+        } else {
+            if (type == TCG_TYPE_I32) {
+                tcg_out_insn(s, RR, CR, r1, c2);
+            } else {
+                tcg_out_insn(s, RRE, CGR, r1, c2);
+            }
+        }
+    }
+    return tcg_cond_to_s390_cond[c];
+}
+
+static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
+                         TCGReg dest, TCGReg c1, TCGArg c2, int c2const)
+{
+    int cc;
+
+    switch (cond) {
+    case TCG_COND_GTU:
+    case TCG_COND_GT:
+    do_greater:
+        /* The result of a compare has CC=2 for GT and CC=3 unused.
+           ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit.  */
+        tgen_cmp(s, type, cond, c1, c2, c2const);
+        tcg_out_movi(s, type, dest, 0);
+        tcg_out_insn(s, RRE, ALCGR, dest, dest);
+        return;
+
+    case TCG_COND_GEU:
+    do_geu:
+        /* We need "real" carry semantics, so use SUBTRACT LOGICAL
+           instead of COMPARE LOGICAL.  This needs an extra move.  */
+        tcg_out_mov(s, type, TCG_TMP0, c1);
+        if (c2const) {
+            tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
+            if (type == TCG_TYPE_I32) {
+                tcg_out_insn(s, RIL, SLFI, TCG_TMP0, c2);
+            } else {
+                tcg_out_insn(s, RIL, SLGFI, TCG_TMP0, c2);
+            }
+        } else {
+            if (type == TCG_TYPE_I32) {
+                tcg_out_insn(s, RR, SLR, TCG_TMP0, c2);
+            } else {
+                tcg_out_insn(s, RRE, SLGR, TCG_TMP0, c2);
+            }
+            tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
+        }
+        tcg_out_insn(s, RRE, ALCGR, dest, dest);
+        return;
+
+    case TCG_COND_LEU:
+    case TCG_COND_LTU:
+    case TCG_COND_LT:
+        /* Swap operands so that we can use GEU/GTU/GT.  */
+        if (c2const) {
+            tcg_out_movi(s, type, TCG_TMP0, c2);
+            c2 = c1;
+            c2const = 0;
+            c1 = TCG_TMP0;
+        } else {
+            TCGReg t = c1;
+            c1 = c2;
+            c2 = t;
+        }
+        if (cond == TCG_COND_LEU) {
+            goto do_geu;
+        }
+        cond = tcg_swap_cond(cond);
+        goto do_greater;
+
+    case TCG_COND_NE:
+        /* X != 0 is X > 0.  */
+        if (c2const && c2 == 0) {
+            cond = TCG_COND_GTU;
+            goto do_greater;
+        }
+        break;
+
+    case TCG_COND_EQ:
+        /* X == 0 is X <= 0 is 0 >= X.  */
+        if (c2const && c2 == 0) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 0);
+            c2 = c1;
+            c2const = 0;
+            c1 = TCG_TMP0;
+            goto do_geu;
+        }
+        break;
+
+    default:
+        break;
+    }
+
+    cc = tgen_cmp(s, type, cond, c1, c2, c2const);
+    if (facilities & FACILITY_LOAD_ON_COND) {
+        /* Emit: d = 0, t = 1, d = (cc ? t : d).  */
+        tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
+        tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 1);
+        tcg_out_insn(s, RRF, LOCGR, dest, TCG_TMP0, cc);
+    } else {
+        /* Emit: d = 1; if (cc) goto over; d = 0; over:  */
+        tcg_out_movi(s, type, dest, 1);
+        tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
+        tcg_out_movi(s, type, dest, 0);
+    }
+}
+
+static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c, TCGReg dest,
+                         TCGReg c1, TCGArg c2, int c2const, TCGReg r3)
+{
+    int cc;
+    if (facilities & FACILITY_LOAD_ON_COND) {
+        cc = tgen_cmp(s, type, c, c1, c2, c2const);
+        tcg_out_insn(s, RRF, LOCGR, dest, r3, cc);
+    } else {
+        c = tcg_invert_cond(c);
+        cc = tgen_cmp(s, type, c, c1, c2, c2const);
+
+        /* Emit: if (cc) goto over; dest = r3; over:  */
+        tcg_out_insn(s, RI, BRC, cc, (4 + 4) >> 1);
+        tcg_out_insn(s, RRE, LGR, dest, r3);
+    }
+}
+
+bool tcg_target_deposit_valid(int ofs, int len)
+{
+    return (facilities & FACILITY_GEN_INST_EXT) != 0;
+}
+
+static void tgen_deposit(TCGContext *s, TCGReg dest, TCGReg src,
+                         int ofs, int len)
+{
+    int lsb = (63 - ofs);
+    int msb = lsb - (len - 1);
+    tcg_out_risbg(s, dest, src, msb, lsb, ofs, 0);
+}
+
+static void tgen_gotoi(TCGContext *s, int cc, tcg_insn_unit *dest)
+{
+    ptrdiff_t off = dest - s->code_ptr;
+    if (off == (int16_t)off) {
+        tcg_out_insn(s, RI, BRC, cc, off);
+    } else if (off == (int32_t)off) {
+        tcg_out_insn(s, RIL, BRCL, cc, off);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
+        tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
+    }
+}
+
+static void tgen_branch(TCGContext *s, int cc, TCGLabel *l)
+{
+    if (l->has_value) {
+        tgen_gotoi(s, cc, l->u.value_ptr);
+    } else if (USE_LONG_BRANCHES) {
+        tcg_out16(s, RIL_BRCL | (cc << 4));
+        tcg_out_reloc(s, s->code_ptr, R_390_PC32DBL, l, -2);
+        s->code_ptr += 2;
+    } else {
+        tcg_out16(s, RI_BRC | (cc << 4));
+        tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, l, -2);
+        s->code_ptr += 1;
+    }
+}
+
+static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
+                                TCGReg r1, TCGReg r2, TCGLabel *l)
+{
+    intptr_t off;
+
+    if (l->has_value) {
+        off = l->u.value_ptr - s->code_ptr;
+    } else {
+        /* We need to keep the offset unchanged for retranslation.  */
+        off = s->code_ptr[1];
+        tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, -2);
+    }
+
+    tcg_out16(s, (opc & 0xff00) | (r1 << 4) | r2);
+    tcg_out16(s, off);
+    tcg_out16(s, cc << 12 | (opc & 0xff));
+}
+
+static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
+                                    TCGReg r1, int i2, TCGLabel *l)
+{
+    tcg_target_long off;
+
+    if (l->has_value) {
+        off = l->u.value_ptr - s->code_ptr;
+    } else {
+        /* We need to keep the offset unchanged for retranslation.  */
+        off = s->code_ptr[1];
+        tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, -2);
+    }
+
+    tcg_out16(s, (opc & 0xff00) | (r1 << 4) | cc);
+    tcg_out16(s, off);
+    tcg_out16(s, (i2 << 8) | (opc & 0xff));
+}
+
+static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
+                        TCGReg r1, TCGArg c2, int c2const, TCGLabel *l)
+{
+    int cc;
+
+    if (facilities & FACILITY_GEN_INST_EXT) {
+        bool is_unsigned = is_unsigned_cond(c);
+        bool in_range;
+        S390Opcode opc;
+
+        cc = tcg_cond_to_s390_cond[c];
+
+        if (!c2const) {
+            opc = (type == TCG_TYPE_I32
+                   ? (is_unsigned ? RIE_CLRJ : RIE_CRJ)
+                   : (is_unsigned ? RIE_CLGRJ : RIE_CGRJ));
+            tgen_compare_branch(s, opc, cc, r1, c2, l);
+            return;
+        }
+
+        /* COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
+           If the immediate we've been given does not fit that range, we'll
+           fall back to separate compare and branch instructions using the
+           larger comparison range afforded by COMPARE IMMEDIATE.  */
+        if (type == TCG_TYPE_I32) {
+            if (is_unsigned) {
+                opc = RIE_CLIJ;
+                in_range = (uint32_t)c2 == (uint8_t)c2;
+            } else {
+                opc = RIE_CIJ;
+                in_range = (int32_t)c2 == (int8_t)c2;
+            }
+        } else {
+            if (is_unsigned) {
+                opc = RIE_CLGIJ;
+                in_range = (uint64_t)c2 == (uint8_t)c2;
+            } else {
+                opc = RIE_CGIJ;
+                in_range = (int64_t)c2 == (int8_t)c2;
+            }
+        }
+        if (in_range) {
+            tgen_compare_imm_branch(s, opc, cc, r1, c2, l);
+            return;
+        }
+    }
+
+    cc = tgen_cmp(s, type, c, r1, c2, c2const);
+    tgen_branch(s, cc, l);
+}
+
+static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+{
+    ptrdiff_t off = dest - s->code_ptr;
+    if (off == (int32_t)off) {
+        tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
+        tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
+    }
+}
+
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGMemOp opc, TCGReg data,
+                                   TCGReg base, TCGReg index, int disp)
+{
+    switch (opc & (MO_SSIZE | MO_BSWAP)) {
+    case MO_UB:
+        tcg_out_insn(s, RXY, LLGC, data, base, index, disp);
+        break;
+    case MO_SB:
+        tcg_out_insn(s, RXY, LGB, data, base, index, disp);
+        break;
+
+    case MO_UW | MO_BSWAP:
+        /* swapped unsigned halfword load with upper bits zeroed */
+        tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
+        tgen_ext16u(s, TCG_TYPE_I64, data, data);
+        break;
+    case MO_UW:
+        tcg_out_insn(s, RXY, LLGH, data, base, index, disp);
+        break;
+
+    case MO_SW | MO_BSWAP:
+        /* swapped sign-extended halfword load */
+        tcg_out_insn(s, RXY, LRVH, data, base, index, disp);
+        tgen_ext16s(s, TCG_TYPE_I64, data, data);
+        break;
+    case MO_SW:
+        tcg_out_insn(s, RXY, LGH, data, base, index, disp);
+        break;
+
+    case MO_UL | MO_BSWAP:
+        /* swapped unsigned int load with upper bits zeroed */
+        tcg_out_insn(s, RXY, LRV, data, base, index, disp);
+        tgen_ext32u(s, data, data);
+        break;
+    case MO_UL:
+        tcg_out_insn(s, RXY, LLGF, data, base, index, disp);
+        break;
+
+    case MO_SL | MO_BSWAP:
+        /* swapped sign-extended int load */
+        tcg_out_insn(s, RXY, LRV, data, base, index, disp);
+        tgen_ext32s(s, data, data);
+        break;
+    case MO_SL:
+        tcg_out_insn(s, RXY, LGF, data, base, index, disp);
+        break;
+
+    case MO_Q | MO_BSWAP:
+        tcg_out_insn(s, RXY, LRVG, data, base, index, disp);
+        break;
+    case MO_Q:
+        tcg_out_insn(s, RXY, LG, data, base, index, disp);
+        break;
+
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_st_direct(TCGContext *s, TCGMemOp opc, TCGReg data,
+                                   TCGReg base, TCGReg index, int disp)
+{
+    switch (opc & (MO_SIZE | MO_BSWAP)) {
+    case MO_UB:
+        if (disp >= 0 && disp < 0x1000) {
+            tcg_out_insn(s, RX, STC, data, base, index, disp);
+        } else {
+            tcg_out_insn(s, RXY, STCY, data, base, index, disp);
+        }
+        break;
+
+    case MO_UW | MO_BSWAP:
+        tcg_out_insn(s, RXY, STRVH, data, base, index, disp);
+        break;
+    case MO_UW:
+        if (disp >= 0 && disp < 0x1000) {
+            tcg_out_insn(s, RX, STH, data, base, index, disp);
+        } else {
+            tcg_out_insn(s, RXY, STHY, data, base, index, disp);
+        }
+        break;
+
+    case MO_UL | MO_BSWAP:
+        tcg_out_insn(s, RXY, STRV, data, base, index, disp);
+        break;
+    case MO_UL:
+        if (disp >= 0 && disp < 0x1000) {
+            tcg_out_insn(s, RX, ST, data, base, index, disp);
+        } else {
+            tcg_out_insn(s, RXY, STY, data, base, index, disp);
+        }
+        break;
+
+    case MO_Q | MO_BSWAP:
+        tcg_out_insn(s, RXY, STRVG, data, base, index, disp);
+        break;
+    case MO_Q:
+        tcg_out_insn(s, RXY, STG, data, base, index, disp);
+        break;
+
+    default:
+        tcg_abort();
+    }
+}
+
+#if defined(CONFIG_SOFTMMU)
+/* We're expecting to use a 20-bit signed offset on the tlb memory ops.
+   Using the offset of the second entry in the last tlb table ensures
+   that we can index all of the elements of the first entry.  */
+QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1])
+                  > 0x7ffff);
+
+/* Load and compare a TLB entry, leaving the flags set.  Loads the TLB
+   addend into R2.  Returns a register with the santitized guest address.  */
+static TCGReg tcg_out_tlb_read(TCGContext* s, TCGReg addr_reg, TCGMemOp opc,
+                               int mem_index, bool is_ld)
+{
+    int a_bits = get_alignment_bits(opc);
+    int ofs, a_off;
+    uint64_t tlb_mask;
+
+    /* For aligned accesses, we check the first byte and include the alignment
+       bits within the address.  For unaligned access, we check that we don't
+       cross pages using the address of the last byte of the access.  */
+    if (a_bits >= 0) {
+        /* A byte access or an alignment check required */
+        a_off = 0;
+        tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1);
+    } else {
+        a_off = (1 << (opc & MO_SIZE)) - 1;
+        tlb_mask = TARGET_PAGE_MASK;
+    }
+
+    if (facilities & FACILITY_GEN_INST_EXT) {
+        tcg_out_risbg(s, TCG_REG_R2, addr_reg,
+                      64 - CPU_TLB_BITS - CPU_TLB_ENTRY_BITS,
+                      63 - CPU_TLB_ENTRY_BITS,
+                      64 + CPU_TLB_ENTRY_BITS - TARGET_PAGE_BITS, 1);
+        if (a_off) {
+            tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
+            tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
+        } else {
+            tgen_andi_risbg(s, TCG_REG_R3, addr_reg, tlb_mask);
+        }
+    } else {
+        tcg_out_sh64(s, RSY_SRLG, TCG_REG_R2, addr_reg, TCG_REG_NONE,
+                     TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
+        tcg_out_insn(s, RX, LA, TCG_REG_R3, addr_reg, TCG_REG_NONE, a_off);
+        tgen_andi(s, TCG_TYPE_I64, TCG_REG_R2,
+                  (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
+        tgen_andi(s, TCG_TYPE_TL, TCG_REG_R3, tlb_mask);
+    }
+
+    if (is_ld) {
+        ofs = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
+    } else {
+        ofs = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+    }
+    if (TARGET_LONG_BITS == 32) {
+        tcg_out_mem(s, RX_C, RXY_CY, TCG_REG_R3, TCG_REG_R2, TCG_AREG0, ofs);
+    } else {
+        tcg_out_mem(s, 0, RXY_CG, TCG_REG_R3, TCG_REG_R2, TCG_AREG0, ofs);
+    }
+
+    ofs = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
+    tcg_out_mem(s, 0, RXY_LG, TCG_REG_R2, TCG_REG_R2, TCG_AREG0, ofs);
+
+    if (TARGET_LONG_BITS == 32) {
+        tgen_ext32u(s, TCG_REG_R3, addr_reg);
+        return TCG_REG_R3;
+    }
+    return addr_reg;
+}
+
+static void add_qemu_ldst_label(TCGContext *s, bool is_ld, TCGMemOpIdx oi,
+                                TCGReg data, TCGReg addr,
+                                tcg_insn_unit *raddr, tcg_insn_unit *label_ptr)
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->oi = oi;
+    label->datalo_reg = data;
+    label->addrlo_reg = addr;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr;
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGReg addr_reg = lb->addrlo_reg;
+    TCGReg data_reg = lb->datalo_reg;
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+
+    patch_reloc(lb->label_ptr[0], R_390_PC16DBL, (intptr_t)s->code_ptr, -2);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
+    }
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi);
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr);
+    tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]);
+    tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2);
+
+    tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
+{
+    TCGReg addr_reg = lb->addrlo_reg;
+    TCGReg data_reg = lb->datalo_reg;
+    TCGMemOpIdx oi = lb->oi;
+    TCGMemOp opc = get_memop(oi);
+
+    patch_reloc(lb->label_ptr[0], R_390_PC16DBL, (intptr_t)s->code_ptr, -2);
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_AREG0);
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R3, addr_reg);
+    }
+    switch (opc & MO_SIZE) {
+    case MO_UB:
+        tgen_ext8u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
+        break;
+    case MO_UW:
+        tgen_ext16u(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
+        break;
+    case MO_UL:
+        tgen_ext32u(s, TCG_REG_R4, data_reg);
+        break;
+    case MO_Q:
+        tcg_out_mov(s, TCG_TYPE_I64, TCG_REG_R4, data_reg);
+        break;
+    default:
+        tcg_abort();
+    }
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi);
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr);
+    tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]);
+
+    tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
+}
+#else
+static void tcg_prepare_user_ldst(TCGContext *s, TCGReg *addr_reg,
+                                  TCGReg *index_reg, tcg_target_long *disp)
+{
+    if (TARGET_LONG_BITS == 32) {
+        tgen_ext32u(s, TCG_TMP0, *addr_reg);
+        *addr_reg = TCG_TMP0;
+    }
+    if (guest_base < 0x80000) {
+        *index_reg = TCG_REG_NONE;
+        *disp = guest_base;
+    } else {
+        *index_reg = TCG_GUEST_BASE_REG;
+        *disp = 0;
+    }
+}
+#endif /* CONFIG_SOFTMMU */
+
+static void tcg_out_qemu_ld(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
+                            TCGMemOpIdx oi)
+{
+    TCGMemOp opc = get_memop(oi);
+#ifdef CONFIG_SOFTMMU
+    unsigned mem_index = get_mmuidx(oi);
+    tcg_insn_unit *label_ptr;
+    TCGReg base_reg;
+
+    base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 1);
+
+    /* We need to keep the offset unchanged for retranslation.  */
+    tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
+    label_ptr = s->code_ptr;
+    s->code_ptr += 1;
+
+    tcg_out_qemu_ld_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
+
+    add_qemu_ldst_label(s, 1, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
+#else
+    TCGReg index_reg;
+    tcg_target_long disp;
+
+    tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
+    tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, index_reg, disp);
+#endif
+}
+
+static void tcg_out_qemu_st(TCGContext* s, TCGReg data_reg, TCGReg addr_reg,
+                            TCGMemOpIdx oi)
+{
+    TCGMemOp opc = get_memop(oi);
+#ifdef CONFIG_SOFTMMU
+    unsigned mem_index = get_mmuidx(oi);
+    tcg_insn_unit *label_ptr;
+    TCGReg base_reg;
+
+    base_reg = tcg_out_tlb_read(s, addr_reg, opc, mem_index, 0);
+
+    /* We need to keep the offset unchanged for retranslation.  */
+    tcg_out16(s, RI_BRC | (S390_CC_NE << 4));
+    label_ptr = s->code_ptr;
+    s->code_ptr += 1;
+
+    tcg_out_qemu_st_direct(s, opc, data_reg, base_reg, TCG_REG_R2, 0);
+
+    add_qemu_ldst_label(s, 0, oi, data_reg, addr_reg, s->code_ptr, label_ptr);
+#else
+    TCGReg index_reg;
+    tcg_target_long disp;
+
+    tcg_prepare_user_ldst(s, &addr_reg, &index_reg, &disp);
+    tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, index_reg, disp);
+#endif
+}
+
+# define OP_32_64(x) \
+        case glue(glue(INDEX_op_,x),_i32): \
+        case glue(glue(INDEX_op_,x),_i64)
+
+static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                const TCGArg *args, const int *const_args)
+{
+    S390Opcode op;
+    TCGArg a0, a1, a2;
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        /* return value */
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, args[0]);
+        tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
+        break;
+
+    case INDEX_op_goto_tb:
+        if (s->tb_jmp_insn_offset) {
+            /* branch displacement must be aligned for atomic patching;
+             * see if we need to add extra nop before branch
+             */
+            if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
+                tcg_out16(s, NOP);
+            }
+            tcg_out16(s, RIL_BRCL | (S390_CC_ALWAYS << 4));
+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            s->code_ptr += 2;
+        } else {
+            /* load address stored at s->tb_jmp_target_addr + args[0] */
+            tcg_out_ld_abs(s, TCG_TYPE_PTR, TCG_TMP0,
+                           s->tb_jmp_target_addr + args[0]);
+            /* and go there */
+            tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_TMP0);
+        }
+        s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+        break;
+
+    OP_32_64(ld8u):
+        /* ??? LLC (RXY format) is only present with the extended-immediate
+           facility, whereas LLGC is always present.  */
+        tcg_out_mem(s, 0, RXY_LLGC, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+
+    OP_32_64(ld8s):
+        /* ??? LB is no smaller than LGB, so no point to using it.  */
+        tcg_out_mem(s, 0, RXY_LGB, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+
+    OP_32_64(ld16u):
+        /* ??? LLH (RXY format) is only present with the extended-immediate
+           facility, whereas LLGH is always present.  */
+        tcg_out_mem(s, 0, RXY_LLGH, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+
+    case INDEX_op_ld16s_i32:
+        tcg_out_mem(s, RX_LH, RXY_LHY, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+
+    case INDEX_op_ld_i32:
+        tcg_out_ld(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        break;
+
+    OP_32_64(st8):
+        tcg_out_mem(s, RX_STC, RXY_STCY, args[0], args[1],
+                    TCG_REG_NONE, args[2]);
+        break;
+
+    OP_32_64(st16):
+        tcg_out_mem(s, RX_STH, RXY_STHY, args[0], args[1],
+                    TCG_REG_NONE, args[2]);
+        break;
+
+    case INDEX_op_st_i32:
+        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_add_i32:
+        a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
+        if (const_args[2]) {
+        do_addi_32:
+            if (a0 == a1) {
+                if (a2 == (int16_t)a2) {
+                    tcg_out_insn(s, RI, AHI, a0, a2);
+                    break;
+                }
+                if (facilities & FACILITY_EXT_IMM) {
+                    tcg_out_insn(s, RIL, AFI, a0, a2);
+                    break;
+                }
+            }
+            tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
+        } else if (a0 == a1) {
+            tcg_out_insn(s, RR, AR, a0, a2);
+        } else {
+            tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
+        }
+        break;
+    case INDEX_op_sub_i32:
+        a0 = args[0], a1 = args[1], a2 = (int32_t)args[2];
+        if (const_args[2]) {
+            a2 = -a2;
+            goto do_addi_32;
+        }
+        tcg_out_insn(s, RR, SR, args[0], args[2]);
+        break;
+
+    case INDEX_op_and_i32:
+        if (const_args[2]) {
+            tgen_andi(s, TCG_TYPE_I32, args[0], args[2]);
+        } else {
+            tcg_out_insn(s, RR, NR, args[0], args[2]);
+        }
+        break;
+    case INDEX_op_or_i32:
+        if (const_args[2]) {
+            tgen64_ori(s, args[0], args[2] & 0xffffffff);
+        } else {
+            tcg_out_insn(s, RR, OR, args[0], args[2]);
+        }
+        break;
+    case INDEX_op_xor_i32:
+        if (const_args[2]) {
+            tgen64_xori(s, args[0], args[2] & 0xffffffff);
+        } else {
+            tcg_out_insn(s, RR, XR, args[0], args[2]);
+        }
+        break;
+
+    case INDEX_op_neg_i32:
+        tcg_out_insn(s, RR, LCR, args[0], args[1]);
+        break;
+
+    case INDEX_op_mul_i32:
+        if (const_args[2]) {
+            if ((int32_t)args[2] == (int16_t)args[2]) {
+                tcg_out_insn(s, RI, MHI, args[0], args[2]);
+            } else {
+                tcg_out_insn(s, RIL, MSFI, args[0], args[2]);
+            }
+        } else {
+            tcg_out_insn(s, RRE, MSR, args[0], args[2]);
+        }
+        break;
+
+    case INDEX_op_div2_i32:
+        tcg_out_insn(s, RR, DR, TCG_REG_R2, args[4]);
+        break;
+    case INDEX_op_divu2_i32:
+        tcg_out_insn(s, RRE, DLR, TCG_REG_R2, args[4]);
+        break;
+
+    case INDEX_op_shl_i32:
+        op = RS_SLL;
+    do_shift32:
+        if (const_args[2]) {
+            tcg_out_sh32(s, op, args[0], TCG_REG_NONE, args[2]);
+        } else {
+            tcg_out_sh32(s, op, args[0], args[2], 0);
+        }
+        break;
+    case INDEX_op_shr_i32:
+        op = RS_SRL;
+        goto do_shift32;
+    case INDEX_op_sar_i32:
+        op = RS_SRA;
+        goto do_shift32;
+
+    case INDEX_op_rotl_i32:
+        /* ??? Using tcg_out_sh64 here for the format; it is a 32-bit rol.  */
+        if (const_args[2]) {
+            tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_REG_NONE, args[2]);
+        } else {
+            tcg_out_sh64(s, RSY_RLL, args[0], args[1], args[2], 0);
+        }
+        break;
+    case INDEX_op_rotr_i32:
+        if (const_args[2]) {
+            tcg_out_sh64(s, RSY_RLL, args[0], args[1],
+                         TCG_REG_NONE, (32 - args[2]) & 31);
+        } else {
+            tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
+            tcg_out_sh64(s, RSY_RLL, args[0], args[1], TCG_TMP0, 0);
+        }
+        break;
+
+    case INDEX_op_ext8s_i32:
+        tgen_ext8s(s, TCG_TYPE_I32, args[0], args[1]);
+        break;
+    case INDEX_op_ext16s_i32:
+        tgen_ext16s(s, TCG_TYPE_I32, args[0], args[1]);
+        break;
+    case INDEX_op_ext8u_i32:
+        tgen_ext8u(s, TCG_TYPE_I32, args[0], args[1]);
+        break;
+    case INDEX_op_ext16u_i32:
+        tgen_ext16u(s, TCG_TYPE_I32, args[0], args[1]);
+        break;
+
+    OP_32_64(bswap16):
+        /* The TCG bswap definition requires bits 0-47 already be zero.
+           Thus we don't need the G-type insns to implement bswap16_i64.  */
+        tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
+        tcg_out_sh32(s, RS_SRL, args[0], TCG_REG_NONE, 16);
+        break;
+    OP_32_64(bswap32):
+        tcg_out_insn(s, RRE, LRVR, args[0], args[1]);
+        break;
+
+    case INDEX_op_add2_i32:
+        if (const_args[4]) {
+            tcg_out_insn(s, RIL, ALFI, args[0], args[4]);
+        } else {
+            tcg_out_insn(s, RR, ALR, args[0], args[4]);
+        }
+        tcg_out_insn(s, RRE, ALCR, args[1], args[5]);
+        break;
+    case INDEX_op_sub2_i32:
+        if (const_args[4]) {
+            tcg_out_insn(s, RIL, SLFI, args[0], args[4]);
+        } else {
+            tcg_out_insn(s, RR, SLR, args[0], args[4]);
+        }
+        tcg_out_insn(s, RRE, SLBR, args[1], args[5]);
+        break;
+
+    case INDEX_op_br:
+        tgen_branch(s, S390_CC_ALWAYS, arg_label(args[0]));
+        break;
+
+    case INDEX_op_brcond_i32:
+        tgen_brcond(s, TCG_TYPE_I32, args[2], args[0],
+                    args[1], const_args[1], arg_label(args[3]));
+        break;
+    case INDEX_op_setcond_i32:
+        tgen_setcond(s, TCG_TYPE_I32, args[3], args[0], args[1],
+                     args[2], const_args[2]);
+        break;
+    case INDEX_op_movcond_i32:
+        tgen_movcond(s, TCG_TYPE_I32, args[5], args[0], args[1],
+                     args[2], const_args[2], args[3]);
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        /* ??? Technically we can use a non-extending instruction.  */
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_qemu_st_i32:
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_ld16s_i64:
+        tcg_out_mem(s, 0, RXY_LGH, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+    case INDEX_op_ld32u_i64:
+        tcg_out_mem(s, 0, RXY_LLGF, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+    case INDEX_op_ld32s_i64:
+        tcg_out_mem(s, 0, RXY_LGF, args[0], args[1], TCG_REG_NONE, args[2]);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_st32_i64:
+        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        break;
+    case INDEX_op_st_i64:
+        tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        break;
+
+    case INDEX_op_add_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+        do_addi_64:
+            if (a0 == a1) {
+                if (a2 == (int16_t)a2) {
+                    tcg_out_insn(s, RI, AGHI, a0, a2);
+                    break;
+                }
+                if (facilities & FACILITY_EXT_IMM) {
+                    if (a2 == (int32_t)a2) {
+                        tcg_out_insn(s, RIL, AGFI, a0, a2);
+                        break;
+                    } else if (a2 == (uint32_t)a2) {
+                        tcg_out_insn(s, RIL, ALGFI, a0, a2);
+                        break;
+                    } else if (-a2 == (uint32_t)-a2) {
+                        tcg_out_insn(s, RIL, SLGFI, a0, -a2);
+                        break;
+                    }
+                }
+            }
+            tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
+        } else if (a0 == a1) {
+            tcg_out_insn(s, RRE, AGR, a0, a2);
+        } else {
+            tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
+        }
+        break;
+    case INDEX_op_sub_i64:
+        a0 = args[0], a1 = args[1], a2 = args[2];
+        if (const_args[2]) {
+            a2 = -a2;
+            goto do_addi_64;
+        } else {
+            tcg_out_insn(s, RRE, SGR, args[0], args[2]);
+        }
+        break;
+
+    case INDEX_op_and_i64:
+        if (const_args[2]) {
+            tgen_andi(s, TCG_TYPE_I64, args[0], args[2]);
+        } else {
+            tcg_out_insn(s, RRE, NGR, args[0], args[2]);
+        }
+        break;
+    case INDEX_op_or_i64:
+        if (const_args[2]) {
+            tgen64_ori(s, args[0], args[2]);
+        } else {
+            tcg_out_insn(s, RRE, OGR, args[0], args[2]);
+        }
+        break;
+    case INDEX_op_xor_i64:
+        if (const_args[2]) {
+            tgen64_xori(s, args[0], args[2]);
+        } else {
+            tcg_out_insn(s, RRE, XGR, args[0], args[2]);
+        }
+        break;
+
+    case INDEX_op_neg_i64:
+        tcg_out_insn(s, RRE, LCGR, args[0], args[1]);
+        break;
+    case INDEX_op_bswap64_i64:
+        tcg_out_insn(s, RRE, LRVGR, args[0], args[1]);
+        break;
+
+    case INDEX_op_mul_i64:
+        if (const_args[2]) {
+            if (args[2] == (int16_t)args[2]) {
+                tcg_out_insn(s, RI, MGHI, args[0], args[2]);
+            } else {
+                tcg_out_insn(s, RIL, MSGFI, args[0], args[2]);
+            }
+        } else {
+            tcg_out_insn(s, RRE, MSGR, args[0], args[2]);
+        }
+        break;
+
+    case INDEX_op_div2_i64:
+        /* ??? We get an unnecessary sign-extension of the dividend
+           into R3 with this definition, but as we do in fact always
+           produce both quotient and remainder using INDEX_op_div_i64
+           instead requires jumping through even more hoops.  */
+        tcg_out_insn(s, RRE, DSGR, TCG_REG_R2, args[4]);
+        break;
+    case INDEX_op_divu2_i64:
+        tcg_out_insn(s, RRE, DLGR, TCG_REG_R2, args[4]);
+        break;
+    case INDEX_op_mulu2_i64:
+        tcg_out_insn(s, RRE, MLGR, TCG_REG_R2, args[3]);
+        break;
+
+    case INDEX_op_shl_i64:
+        op = RSY_SLLG;
+    do_shift64:
+        if (const_args[2]) {
+            tcg_out_sh64(s, op, args[0], args[1], TCG_REG_NONE, args[2]);
+        } else {
+            tcg_out_sh64(s, op, args[0], args[1], args[2], 0);
+        }
+        break;
+    case INDEX_op_shr_i64:
+        op = RSY_SRLG;
+        goto do_shift64;
+    case INDEX_op_sar_i64:
+        op = RSY_SRAG;
+        goto do_shift64;
+
+    case INDEX_op_rotl_i64:
+        if (const_args[2]) {
+            tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
+                         TCG_REG_NONE, args[2]);
+        } else {
+            tcg_out_sh64(s, RSY_RLLG, args[0], args[1], args[2], 0);
+        }
+        break;
+    case INDEX_op_rotr_i64:
+        if (const_args[2]) {
+            tcg_out_sh64(s, RSY_RLLG, args[0], args[1],
+                         TCG_REG_NONE, (64 - args[2]) & 63);
+        } else {
+            /* We can use the smaller 32-bit negate because only the
+               low 6 bits are examined for the rotate.  */
+            tcg_out_insn(s, RR, LCR, TCG_TMP0, args[2]);
+            tcg_out_sh64(s, RSY_RLLG, args[0], args[1], TCG_TMP0, 0);
+        }
+        break;
+
+    case INDEX_op_ext8s_i64:
+        tgen_ext8s(s, TCG_TYPE_I64, args[0], args[1]);
+        break;
+    case INDEX_op_ext16s_i64:
+        tgen_ext16s(s, TCG_TYPE_I64, args[0], args[1]);
+        break;
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_ext32s_i64:
+        tgen_ext32s(s, args[0], args[1]);
+        break;
+    case INDEX_op_ext8u_i64:
+        tgen_ext8u(s, TCG_TYPE_I64, args[0], args[1]);
+        break;
+    case INDEX_op_ext16u_i64:
+        tgen_ext16u(s, TCG_TYPE_I64, args[0], args[1]);
+        break;
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_ext32u_i64:
+        tgen_ext32u(s, args[0], args[1]);
+        break;
+
+    case INDEX_op_add2_i64:
+        if (const_args[4]) {
+            if ((int64_t)args[4] >= 0) {
+                tcg_out_insn(s, RIL, ALGFI, args[0], args[4]);
+            } else {
+                tcg_out_insn(s, RIL, SLGFI, args[0], -args[4]);
+            }
+        } else {
+            tcg_out_insn(s, RRE, ALGR, args[0], args[4]);
+        }
+        tcg_out_insn(s, RRE, ALCGR, args[1], args[5]);
+        break;
+    case INDEX_op_sub2_i64:
+        if (const_args[4]) {
+            if ((int64_t)args[4] >= 0) {
+                tcg_out_insn(s, RIL, SLGFI, args[0], args[4]);
+            } else {
+                tcg_out_insn(s, RIL, ALGFI, args[0], -args[4]);
+            }
+        } else {
+            tcg_out_insn(s, RRE, SLGR, args[0], args[4]);
+        }
+        tcg_out_insn(s, RRE, SLBGR, args[1], args[5]);
+        break;
+
+    case INDEX_op_brcond_i64:
+        tgen_brcond(s, TCG_TYPE_I64, args[2], args[0],
+                    args[1], const_args[1], arg_label(args[3]));
+        break;
+    case INDEX_op_setcond_i64:
+        tgen_setcond(s, TCG_TYPE_I64, args[3], args[0], args[1],
+                     args[2], const_args[2]);
+        break;
+    case INDEX_op_movcond_i64:
+        tgen_movcond(s, TCG_TYPE_I64, args[5], args[0], args[1],
+                     args[2], const_args[2], args[3]);
+        break;
+
+    OP_32_64(deposit):
+        tgen_deposit(s, args[0], args[2], args[3], args[4]);
+        break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+}
+
+static const TCGTargetOpDef s390_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "r", "r" } },
+    { INDEX_op_st16_i32, { "r", "r" } },
+    { INDEX_op_st_i32, { "r", "r" } },
+
+    { INDEX_op_add_i32, { "r", "r", "ri" } },
+    { INDEX_op_sub_i32, { "r", "0", "ri" } },
+    { INDEX_op_mul_i32, { "r", "0", "rK" } },
+
+    { INDEX_op_div2_i32, { "b", "a", "0", "1", "r" } },
+    { INDEX_op_divu2_i32, { "b", "a", "0", "1", "r" } },
+
+    { INDEX_op_and_i32, { "r", "0", "ri" } },
+    { INDEX_op_or_i32, { "r", "0", "rO" } },
+    { INDEX_op_xor_i32, { "r", "0", "rX" } },
+
+    { INDEX_op_neg_i32, { "r", "r" } },
+
+    { INDEX_op_shl_i32, { "r", "0", "Ri" } },
+    { INDEX_op_shr_i32, { "r", "0", "Ri" } },
+    { INDEX_op_sar_i32, { "r", "0", "Ri" } },
+
+    { INDEX_op_rotl_i32, { "r", "r", "Ri" } },
+    { INDEX_op_rotr_i32, { "r", "r", "Ri" } },
+
+    { INDEX_op_ext8s_i32, { "r", "r" } },
+    { INDEX_op_ext8u_i32, { "r", "r" } },
+    { INDEX_op_ext16s_i32, { "r", "r" } },
+    { INDEX_op_ext16u_i32, { "r", "r" } },
+
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
+
+    { INDEX_op_add2_i32, { "r", "r", "0", "1", "rA", "r" } },
+    { INDEX_op_sub2_i32, { "r", "r", "0", "1", "rA", "r" } },
+
+    { INDEX_op_brcond_i32, { "r", "rC" } },
+    { INDEX_op_setcond_i32, { "r", "r", "rC" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rC", "r", "0" } },
+    { INDEX_op_deposit_i32, { "r", "0", "r" } },
+
+    { INDEX_op_qemu_ld_i32, { "r", "L" } },
+    { INDEX_op_qemu_ld_i64, { "r", "L" } },
+    { INDEX_op_qemu_st_i32, { "L", "L" } },
+    { INDEX_op_qemu_st_i64, { "L", "L" } },
+
+    { INDEX_op_ld8u_i64, { "r", "r" } },
+    { INDEX_op_ld8s_i64, { "r", "r" } },
+    { INDEX_op_ld16u_i64, { "r", "r" } },
+    { INDEX_op_ld16s_i64, { "r", "r" } },
+    { INDEX_op_ld32u_i64, { "r", "r" } },
+    { INDEX_op_ld32s_i64, { "r", "r" } },
+    { INDEX_op_ld_i64, { "r", "r" } },
+
+    { INDEX_op_st8_i64, { "r", "r" } },
+    { INDEX_op_st16_i64, { "r", "r" } },
+    { INDEX_op_st32_i64, { "r", "r" } },
+    { INDEX_op_st_i64, { "r", "r" } },
+
+    { INDEX_op_add_i64, { "r", "r", "ri" } },
+    { INDEX_op_sub_i64, { "r", "0", "ri" } },
+    { INDEX_op_mul_i64, { "r", "0", "rK" } },
+
+    { INDEX_op_div2_i64, { "b", "a", "0", "1", "r" } },
+    { INDEX_op_divu2_i64, { "b", "a", "0", "1", "r" } },
+    { INDEX_op_mulu2_i64, { "b", "a", "0", "r" } },
+
+    { INDEX_op_and_i64, { "r", "0", "ri" } },
+    { INDEX_op_or_i64, { "r", "0", "rO" } },
+    { INDEX_op_xor_i64, { "r", "0", "rX" } },
+
+    { INDEX_op_neg_i64, { "r", "r" } },
+
+    { INDEX_op_shl_i64, { "r", "r", "Ri" } },
+    { INDEX_op_shr_i64, { "r", "r", "Ri" } },
+    { INDEX_op_sar_i64, { "r", "r", "Ri" } },
+
+    { INDEX_op_rotl_i64, { "r", "r", "Ri" } },
+    { INDEX_op_rotr_i64, { "r", "r", "Ri" } },
+
+    { INDEX_op_ext8s_i64, { "r", "r" } },
+    { INDEX_op_ext8u_i64, { "r", "r" } },
+    { INDEX_op_ext16s_i64, { "r", "r" } },
+    { INDEX_op_ext16u_i64, { "r", "r" } },
+    { INDEX_op_ext32s_i64, { "r", "r" } },
+    { INDEX_op_ext32u_i64, { "r", "r" } },
+
+    { INDEX_op_ext_i32_i64, { "r", "r" } },
+    { INDEX_op_extu_i32_i64, { "r", "r" } },
+
+    { INDEX_op_bswap16_i64, { "r", "r" } },
+    { INDEX_op_bswap32_i64, { "r", "r" } },
+    { INDEX_op_bswap64_i64, { "r", "r" } },
+
+    { INDEX_op_add2_i64, { "r", "r", "0", "1", "rA", "r" } },
+    { INDEX_op_sub2_i64, { "r", "r", "0", "1", "rA", "r" } },
+
+    { INDEX_op_brcond_i64, { "r", "rC" } },
+    { INDEX_op_setcond_i64, { "r", "r", "rC" } },
+    { INDEX_op_movcond_i64, { "r", "r", "rC", "r", "0" } },
+    { INDEX_op_deposit_i64, { "r", "0", "r" } },
+
+    { -1 },
+};
+
+static void query_facilities(void)
+{
+    unsigned long hwcap = qemu_getauxval(AT_HWCAP);
+
+    /* Is STORE FACILITY LIST EXTENDED available?  Honestly, I believe this
+       is present on all 64-bit systems, but let's check for it anyway.  */
+    if (hwcap & HWCAP_S390_STFLE) {
+        register int r0 __asm__("0");
+        register void *r1 __asm__("1");
+
+        /* stfle 0(%r1) */
+        r1 = &facilities;
+        asm volatile(".word 0xb2b0,0x1000"
+                     : "=r"(r0) : "0"(0), "r"(r1) : "memory", "cc");
+    }
+}
+
+static void tcg_target_init(TCGContext *s)
+{
+    query_facilities();
+
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffff);
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffff);
+
+    tcg_regset_clear(tcg_target_call_clobber_regs);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
+    /* The r6 register is technically call-saved, but it's also a parameter
+       register, so it can get killed by setup for the qemu_st helper.  */
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
+    /* The return register can be considered call-clobbered.  */
+    tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
+    /* XXX many insns can't be used with R0, so we better avoid it for now */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
+
+    tcg_add_target_add_op_defs(s390_op_defs);
+}
+
+#define FRAME_SIZE  ((int)(TCG_TARGET_CALL_STACK_OFFSET          \
+                           + TCG_STATIC_CALL_ARGS_SIZE           \
+                           + CPU_TEMP_BUF_NLONGS * sizeof(long)))
+
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    /* stmg %r6,%r15,48(%r15) (save registers) */
+    tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
+
+    /* aghi %r15,-frame_size */
+    tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
+
+    tcg_set_frame(s, TCG_REG_CALL_STACK,
+                  TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+
+#ifndef CONFIG_SOFTMMU
+    if (guest_base >= 0x80000) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
+    }
+#endif
+
+    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
+    /* br %r3 (go to TB) */
+    tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
+
+    tb_ret_addr = s->code_ptr;
+
+    /* lmg %r6,%r15,fs+48(%r15) (restore registers) */
+    tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
+                 FRAME_SIZE + 48);
+
+    /* br %r14 (return) */
+    tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
+}
+
+typedef struct {
+    DebugFrameHeader h;
+    uint8_t fde_def_cfa[4];
+    uint8_t fde_reg_ofs[18];
+} DebugFrame;
+
+/* We're expecting a 2 byte uleb128 encoded value.  */
+QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
+
+#define ELF_HOST_MACHINE  EM_S390
+
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 8,                /* sleb128 8 */
+    .h.cie.return_column = TCG_REG_R14,
+
+    /* Total FDE size does not include the "len" member.  */
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
+
+    .fde_def_cfa = {
+        12, TCG_REG_CALL_STACK,         /* DW_CFA_def_cfa %r15, ... */
+        (FRAME_SIZE & 0x7f) | 0x80,     /* ... uleb128 FRAME_SIZE */
+        (FRAME_SIZE >> 7)
+    },
+    .fde_reg_ofs = {
+        0x86, 6,                        /* DW_CFA_offset, %r6, 48 */
+        0x87, 7,                        /* DW_CFA_offset, %r7, 56 */
+        0x88, 8,                        /* DW_CFA_offset, %r8, 64 */
+        0x89, 9,                        /* DW_CFA_offset, %r92, 72 */
+        0x8a, 10,                       /* DW_CFA_offset, %r10, 80 */
+        0x8b, 11,                       /* DW_CFA_offset, %r11, 88 */
+        0x8c, 12,                       /* DW_CFA_offset, %r12, 96 */
+        0x8d, 13,                       /* DW_CFA_offset, %r13, 104 */
+        0x8e, 14,                       /* DW_CFA_offset, %r14, 112 */
+    }
+};
+
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
+}
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
deleted file mode 100644
index e081828..0000000
--- a/tcg/sparc/tcg-target.c
+++ /dev/null
@@ -1,1652 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "tcg-be-null.h"
-
-#ifndef NDEBUG
-static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "%g0",
-    "%g1",
-    "%g2",
-    "%g3",
-    "%g4",
-    "%g5",
-    "%g6",
-    "%g7",
-    "%o0",
-    "%o1",
-    "%o2",
-    "%o3",
-    "%o4",
-    "%o5",
-    "%o6",
-    "%o7",
-    "%l0",
-    "%l1",
-    "%l2",
-    "%l3",
-    "%l4",
-    "%l5",
-    "%l6",
-    "%l7",
-    "%i0",
-    "%i1",
-    "%i2",
-    "%i3",
-    "%i4",
-    "%i5",
-    "%i6",
-    "%i7",
-};
-#endif
-
-#ifdef __arch64__
-# define SPARC64 1
-#else
-# define SPARC64 0
-#endif
-
-/* Note that sparcv8plus can only hold 64 bit quantities in %g and %o
-   registers.  These are saved manually by the kernel in full 64-bit
-   slots.  The %i and %l registers are saved by the register window
-   mechanism, which only allocates space for 32 bits.  Given that this
-   window spill/fill can happen on any signal, we must consider the
-   high bits of the %i and %l registers garbage at all times.  */
-#if SPARC64
-# define ALL_64  0xffffffffu
-#else
-# define ALL_64  0xffffu
-#endif
-
-/* Define some temporary registers.  T2 is used for constant generation.  */
-#define TCG_REG_T1  TCG_REG_G1
-#define TCG_REG_T2  TCG_REG_O7
-
-#ifdef CONFIG_USE_GUEST_BASE
-# define TCG_GUEST_BASE_REG TCG_REG_I5
-#else
-# define TCG_GUEST_BASE_REG TCG_REG_G0
-#endif
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_L0,
-    TCG_REG_L1,
-    TCG_REG_L2,
-    TCG_REG_L3,
-    TCG_REG_L4,
-    TCG_REG_L5,
-    TCG_REG_L6,
-    TCG_REG_L7,
-
-    TCG_REG_I0,
-    TCG_REG_I1,
-    TCG_REG_I2,
-    TCG_REG_I3,
-    TCG_REG_I4,
-    TCG_REG_I5,
-
-    TCG_REG_G2,
-    TCG_REG_G3,
-    TCG_REG_G4,
-    TCG_REG_G5,
-
-    TCG_REG_O0,
-    TCG_REG_O1,
-    TCG_REG_O2,
-    TCG_REG_O3,
-    TCG_REG_O4,
-    TCG_REG_O5,
-};
-
-static const int tcg_target_call_iarg_regs[6] = {
-    TCG_REG_O0,
-    TCG_REG_O1,
-    TCG_REG_O2,
-    TCG_REG_O3,
-    TCG_REG_O4,
-    TCG_REG_O5,
-};
-
-static const int tcg_target_call_oarg_regs[] = {
-    TCG_REG_O0,
-    TCG_REG_O1,
-    TCG_REG_O2,
-    TCG_REG_O3,
-};
-
-#define INSN_OP(x)  ((x) << 30)
-#define INSN_OP2(x) ((x) << 22)
-#define INSN_OP3(x) ((x) << 19)
-#define INSN_OPF(x) ((x) << 5)
-#define INSN_RD(x)  ((x) << 25)
-#define INSN_RS1(x) ((x) << 14)
-#define INSN_RS2(x) (x)
-#define INSN_ASI(x) ((x) << 5)
-
-#define INSN_IMM10(x) ((1 << 13) | ((x) & 0x3ff))
-#define INSN_IMM11(x) ((1 << 13) | ((x) & 0x7ff))
-#define INSN_IMM13(x) ((1 << 13) | ((x) & 0x1fff))
-#define INSN_OFF16(x) ((((x) >> 2) & 0x3fff) | ((((x) >> 16) & 3) << 20))
-#define INSN_OFF19(x) (((x) >> 2) & 0x07ffff)
-#define INSN_COND(x) ((x) << 25)
-
-#define COND_N     0x0
-#define COND_E     0x1
-#define COND_LE    0x2
-#define COND_L     0x3
-#define COND_LEU   0x4
-#define COND_CS    0x5
-#define COND_NEG   0x6
-#define COND_VS    0x7
-#define COND_A     0x8
-#define COND_NE    0x9
-#define COND_G     0xa
-#define COND_GE    0xb
-#define COND_GU    0xc
-#define COND_CC    0xd
-#define COND_POS   0xe
-#define COND_VC    0xf
-#define BA         (INSN_OP(0) | INSN_COND(COND_A) | INSN_OP2(0x2))
-
-#define RCOND_Z    1
-#define RCOND_LEZ  2
-#define RCOND_LZ   3
-#define RCOND_NZ   5
-#define RCOND_GZ   6
-#define RCOND_GEZ  7
-
-#define MOVCC_ICC  (1 << 18)
-#define MOVCC_XCC  (1 << 18 | 1 << 12)
-
-#define BPCC_ICC   0
-#define BPCC_XCC   (2 << 20)
-#define BPCC_PT    (1 << 19)
-#define BPCC_PN    0
-#define BPCC_A     (1 << 29)
-
-#define BPR_PT     BPCC_PT
-
-#define ARITH_ADD  (INSN_OP(2) | INSN_OP3(0x00))
-#define ARITH_ADDCC (INSN_OP(2) | INSN_OP3(0x10))
-#define ARITH_AND  (INSN_OP(2) | INSN_OP3(0x01))
-#define ARITH_ANDN (INSN_OP(2) | INSN_OP3(0x05))
-#define ARITH_OR   (INSN_OP(2) | INSN_OP3(0x02))
-#define ARITH_ORCC (INSN_OP(2) | INSN_OP3(0x12))
-#define ARITH_ORN  (INSN_OP(2) | INSN_OP3(0x06))
-#define ARITH_XOR  (INSN_OP(2) | INSN_OP3(0x03))
-#define ARITH_SUB  (INSN_OP(2) | INSN_OP3(0x04))
-#define ARITH_SUBCC (INSN_OP(2) | INSN_OP3(0x14))
-#define ARITH_ADDC (INSN_OP(2) | INSN_OP3(0x08))
-#define ARITH_SUBC (INSN_OP(2) | INSN_OP3(0x0c))
-#define ARITH_UMUL (INSN_OP(2) | INSN_OP3(0x0a))
-#define ARITH_SMUL (INSN_OP(2) | INSN_OP3(0x0b))
-#define ARITH_UDIV (INSN_OP(2) | INSN_OP3(0x0e))
-#define ARITH_SDIV (INSN_OP(2) | INSN_OP3(0x0f))
-#define ARITH_MULX (INSN_OP(2) | INSN_OP3(0x09))
-#define ARITH_UDIVX (INSN_OP(2) | INSN_OP3(0x0d))
-#define ARITH_SDIVX (INSN_OP(2) | INSN_OP3(0x2d))
-#define ARITH_MOVCC (INSN_OP(2) | INSN_OP3(0x2c))
-#define ARITH_MOVR (INSN_OP(2) | INSN_OP3(0x2f))
-
-#define ARITH_ADDXC (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x11))
-#define ARITH_UMULXHI (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x16))
-
-#define SHIFT_SLL  (INSN_OP(2) | INSN_OP3(0x25))
-#define SHIFT_SRL  (INSN_OP(2) | INSN_OP3(0x26))
-#define SHIFT_SRA  (INSN_OP(2) | INSN_OP3(0x27))
-
-#define SHIFT_SLLX (INSN_OP(2) | INSN_OP3(0x25) | (1 << 12))
-#define SHIFT_SRLX (INSN_OP(2) | INSN_OP3(0x26) | (1 << 12))
-#define SHIFT_SRAX (INSN_OP(2) | INSN_OP3(0x27) | (1 << 12))
-
-#define RDY        (INSN_OP(2) | INSN_OP3(0x28) | INSN_RS1(0))
-#define WRY        (INSN_OP(2) | INSN_OP3(0x30) | INSN_RD(0))
-#define JMPL       (INSN_OP(2) | INSN_OP3(0x38))
-#define RETURN     (INSN_OP(2) | INSN_OP3(0x39))
-#define SAVE       (INSN_OP(2) | INSN_OP3(0x3c))
-#define RESTORE    (INSN_OP(2) | INSN_OP3(0x3d))
-#define SETHI      (INSN_OP(0) | INSN_OP2(0x4))
-#define CALL       INSN_OP(1)
-#define LDUB       (INSN_OP(3) | INSN_OP3(0x01))
-#define LDSB       (INSN_OP(3) | INSN_OP3(0x09))
-#define LDUH       (INSN_OP(3) | INSN_OP3(0x02))
-#define LDSH       (INSN_OP(3) | INSN_OP3(0x0a))
-#define LDUW       (INSN_OP(3) | INSN_OP3(0x00))
-#define LDSW       (INSN_OP(3) | INSN_OP3(0x08))
-#define LDX        (INSN_OP(3) | INSN_OP3(0x0b))
-#define STB        (INSN_OP(3) | INSN_OP3(0x05))
-#define STH        (INSN_OP(3) | INSN_OP3(0x06))
-#define STW        (INSN_OP(3) | INSN_OP3(0x04))
-#define STX        (INSN_OP(3) | INSN_OP3(0x0e))
-#define LDUBA      (INSN_OP(3) | INSN_OP3(0x11))
-#define LDSBA      (INSN_OP(3) | INSN_OP3(0x19))
-#define LDUHA      (INSN_OP(3) | INSN_OP3(0x12))
-#define LDSHA      (INSN_OP(3) | INSN_OP3(0x1a))
-#define LDUWA      (INSN_OP(3) | INSN_OP3(0x10))
-#define LDSWA      (INSN_OP(3) | INSN_OP3(0x18))
-#define LDXA       (INSN_OP(3) | INSN_OP3(0x1b))
-#define STBA       (INSN_OP(3) | INSN_OP3(0x15))
-#define STHA       (INSN_OP(3) | INSN_OP3(0x16))
-#define STWA       (INSN_OP(3) | INSN_OP3(0x14))
-#define STXA       (INSN_OP(3) | INSN_OP3(0x1e))
-
-#ifndef ASI_PRIMARY_LITTLE
-#define ASI_PRIMARY_LITTLE 0x88
-#endif
-
-#define LDUH_LE    (LDUHA | INSN_ASI(ASI_PRIMARY_LITTLE))
-#define LDSH_LE    (LDSHA | INSN_ASI(ASI_PRIMARY_LITTLE))
-#define LDUW_LE    (LDUWA | INSN_ASI(ASI_PRIMARY_LITTLE))
-#define LDSW_LE    (LDSWA | INSN_ASI(ASI_PRIMARY_LITTLE))
-#define LDX_LE     (LDXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
-
-#define STH_LE     (STHA  | INSN_ASI(ASI_PRIMARY_LITTLE))
-#define STW_LE     (STWA  | INSN_ASI(ASI_PRIMARY_LITTLE))
-#define STX_LE     (STXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
-
-#ifndef use_vis3_instructions
-bool use_vis3_instructions;
-#endif
-
-static inline int check_fit_i64(int64_t val, unsigned int bits)
-{
-    return val == sextract64(val, 0, bits);
-}
-
-static inline int check_fit_i32(int32_t val, unsigned int bits)
-{
-    return val == sextract32(val, 0, bits);
-}
-
-#define check_fit_tl    check_fit_i64
-#if SPARC64
-# define check_fit_ptr  check_fit_i64
-#else
-# define check_fit_ptr  check_fit_i32
-#endif
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    uint32_t insn;
-
-    assert(addend == 0);
-    value = tcg_ptr_byte_diff((tcg_insn_unit *)value, code_ptr);
-
-    switch (type) {
-    case R_SPARC_WDISP16:
-        if (!check_fit_ptr(value >> 2, 16)) {
-            tcg_abort();
-        }
-        insn = *code_ptr;
-        insn &= ~INSN_OFF16(-1);
-        insn |= INSN_OFF16(value);
-        *code_ptr = insn;
-        break;
-    case R_SPARC_WDISP19:
-        if (!check_fit_ptr(value >> 2, 19)) {
-            tcg_abort();
-        }
-        insn = *code_ptr;
-        insn &= ~INSN_OFF19(-1);
-        insn |= INSN_OFF19(value);
-        *code_ptr = insn;
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        break;
-    case 'R':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, ALL_64);
-        break;
-    case 'A': /* qemu_ld/st address constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0,
-                         TARGET_LONG_BITS == 64 ? ALL_64 : 0xffffffff);
-    reserve_helpers:
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O1);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O2);
-        break;
-    case 's': /* qemu_st data 32-bit constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        goto reserve_helpers;
-    case 'S': /* qemu_st data 64-bit constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, ALL_64);
-        goto reserve_helpers;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_S11;
-        break;
-    case 'J':
-        ct->ct |= TCG_CT_CONST_S13;
-        break;
-    case 'Z':
-        ct->ct |= TCG_CT_CONST_ZERO;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct = arg_ct->ct;
-
-    if (ct & TCG_CT_CONST) {
-        return 1;
-    }
-
-    if (type == TCG_TYPE_I32) {
-        val = (int32_t)val;
-    }
-
-    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11)) {
-        return 1;
-    } else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13)) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-static inline void tcg_out_arith(TCGContext *s, TCGReg rd, TCGReg rs1,
-                                 TCGReg rs2, int op)
-{
-    tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1) | INSN_RS2(rs2));
-}
-
-static inline void tcg_out_arithi(TCGContext *s, TCGReg rd, TCGReg rs1,
-                                  int32_t offset, int op)
-{
-    tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1) | INSN_IMM13(offset));
-}
-
-static void tcg_out_arithc(TCGContext *s, TCGReg rd, TCGReg rs1,
-			   int32_t val2, int val2const, int op)
-{
-    tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1)
-              | (val2const ? INSN_IMM13(val2) : INSN_RS2(val2)));
-}
-
-static inline void tcg_out_mov(TCGContext *s, TCGType type,
-                               TCGReg ret, TCGReg arg)
-{
-    if (ret != arg) {
-        tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
-    }
-}
-
-static inline void tcg_out_sethi(TCGContext *s, TCGReg ret, uint32_t arg)
-{
-    tcg_out32(s, SETHI | INSN_RD(ret) | ((arg & 0xfffffc00) >> 10));
-}
-
-static inline void tcg_out_movi_imm13(TCGContext *s, TCGReg ret, int32_t arg)
-{
-    tcg_out_arithi(s, ret, TCG_REG_G0, arg, ARITH_OR);
-}
-
-static void tcg_out_movi(TCGContext *s, TCGType type,
-                         TCGReg ret, tcg_target_long arg)
-{
-    tcg_target_long hi, lo = (int32_t)arg;
-
-    /* Make sure we test 32-bit constants for imm13 properly.  */
-    if (type == TCG_TYPE_I32) {
-        arg = lo;
-    }
-
-    /* A 13-bit constant sign-extended to 64-bits.  */
-    if (check_fit_tl(arg, 13)) {
-        tcg_out_movi_imm13(s, ret, arg);
-        return;
-    }
-
-    /* A 32-bit constant, or 32-bit zero-extended to 64-bits.  */
-    if (type == TCG_TYPE_I32 || arg == (uint32_t)arg) {
-        tcg_out_sethi(s, ret, arg);
-        if (arg & 0x3ff) {
-            tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR);
-        }
-        return;
-    }
-
-    /* A 32-bit constant sign-extended to 64-bits.  */
-    if (arg == lo) {
-        tcg_out_sethi(s, ret, ~arg);
-        tcg_out_arithi(s, ret, ret, (arg & 0x3ff) | -0x400, ARITH_XOR);
-        return;
-    }
-
-    /* A 64-bit constant decomposed into 2 32-bit pieces.  */
-    if (check_fit_i32(lo, 13)) {
-        hi = (arg - lo) >> 32;
-        tcg_out_movi(s, TCG_TYPE_I32, ret, hi);
-        tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
-        tcg_out_arithi(s, ret, ret, lo, ARITH_ADD);
-    } else {
-        hi = arg >> 32;
-        tcg_out_movi(s, TCG_TYPE_I32, ret, hi);
-        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T2, lo);
-        tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
-        tcg_out_arith(s, ret, ret, TCG_REG_T2, ARITH_OR);
-    }
-}
-
-static inline void tcg_out_ldst_rr(TCGContext *s, TCGReg data, TCGReg a1,
-                                   TCGReg a2, int op)
-{
-    tcg_out32(s, op | INSN_RD(data) | INSN_RS1(a1) | INSN_RS2(a2));
-}
-
-static void tcg_out_ldst(TCGContext *s, TCGReg ret, TCGReg addr,
-                         intptr_t offset, int op)
-{
-    if (check_fit_ptr(offset, 13)) {
-        tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(addr) |
-                  INSN_IMM13(offset));
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, offset);
-        tcg_out_ldst_rr(s, ret, addr, TCG_REG_T1, op);
-    }
-}
-
-static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, ret, arg1, arg2, (type == TCG_TYPE_I32 ? LDUW : LDX));
-}
-
-static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
-                              TCGReg arg1, intptr_t arg2)
-{
-    tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
-}
-
-static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, uintptr_t arg)
-{
-    tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff);
-    tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, arg & 0x3ff);
-}
-
-static inline void tcg_out_sety(TCGContext *s, TCGReg rs)
-{
-    tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs));
-}
-
-static inline void tcg_out_rdy(TCGContext *s, TCGReg rd)
-{
-    tcg_out32(s, RDY | INSN_RD(rd));
-}
-
-static void tcg_out_div32(TCGContext *s, TCGReg rd, TCGReg rs1,
-                          int32_t val2, int val2const, int uns)
-{
-    /* Load Y with the sign/zero extension of RS1 to 64-bits.  */
-    if (uns) {
-        tcg_out_sety(s, TCG_REG_G0);
-    } else {
-        tcg_out_arithi(s, TCG_REG_T1, rs1, 31, SHIFT_SRA);
-        tcg_out_sety(s, TCG_REG_T1);
-    }
-
-    tcg_out_arithc(s, rd, rs1, val2, val2const,
-                   uns ? ARITH_UDIV : ARITH_SDIV);
-}
-
-static inline void tcg_out_nop(TCGContext *s)
-{
-    tcg_out_sethi(s, TCG_REG_G0, 0);
-}
-
-static const uint8_t tcg_cond_to_bcond[] = {
-    [TCG_COND_EQ] = COND_E,
-    [TCG_COND_NE] = COND_NE,
-    [TCG_COND_LT] = COND_L,
-    [TCG_COND_GE] = COND_GE,
-    [TCG_COND_LE] = COND_LE,
-    [TCG_COND_GT] = COND_G,
-    [TCG_COND_LTU] = COND_CS,
-    [TCG_COND_GEU] = COND_CC,
-    [TCG_COND_LEU] = COND_LEU,
-    [TCG_COND_GTU] = COND_GU,
-};
-
-static const uint8_t tcg_cond_to_rcond[] = {
-    [TCG_COND_EQ] = RCOND_Z,
-    [TCG_COND_NE] = RCOND_NZ,
-    [TCG_COND_LT] = RCOND_LZ,
-    [TCG_COND_GT] = RCOND_GZ,
-    [TCG_COND_LE] = RCOND_LEZ,
-    [TCG_COND_GE] = RCOND_GEZ
-};
-
-static void tcg_out_bpcc0(TCGContext *s, int scond, int flags, int off19)
-{
-    tcg_out32(s, INSN_OP(0) | INSN_OP2(1) | INSN_COND(scond) | flags | off19);
-}
-
-static void tcg_out_bpcc(TCGContext *s, int scond, int flags, int label)
-{
-    TCGLabel *l = &s->labels[label];
-    int off19;
-
-    if (l->has_value) {
-        off19 = INSN_OFF19(tcg_pcrel_diff(s, l->u.value_ptr));
-    } else {
-        /* Make sure to preserve destinations during retranslation.  */
-        off19 = *s->code_ptr & INSN_OFF19(-1);
-        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, label, 0);
-    }
-    tcg_out_bpcc0(s, scond, flags, off19);
-}
-
-static void tcg_out_cmp(TCGContext *s, TCGReg c1, int32_t c2, int c2const)
-{
-    tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const, ARITH_SUBCC);
-}
-
-static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGReg arg1,
-                               int32_t arg2, int const_arg2, int label)
-{
-    tcg_out_cmp(s, arg1, arg2, const_arg2);
-    tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_ICC | BPCC_PT, label);
-    tcg_out_nop(s);
-}
-
-static void tcg_out_movcc(TCGContext *s, TCGCond cond, int cc, TCGReg ret,
-                          int32_t v1, int v1const)
-{
-    tcg_out32(s, ARITH_MOVCC | cc | INSN_RD(ret)
-              | INSN_RS1(tcg_cond_to_bcond[cond])
-              | (v1const ? INSN_IMM11(v1) : INSN_RS2(v1)));
-}
-
-static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
-                                TCGReg c1, int32_t c2, int c2const,
-                                int32_t v1, int v1const)
-{
-    tcg_out_cmp(s, c1, c2, c2const);
-    tcg_out_movcc(s, cond, MOVCC_ICC, ret, v1, v1const);
-}
-
-static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1,
-                               int32_t arg2, int const_arg2, int label)
-{
-    /* For 64-bit signed comparisons vs zero, we can avoid the compare.  */
-    if (arg2 == 0 && !is_unsigned_cond(cond)) {
-        TCGLabel *l = &s->labels[label];
-        int off16;
-
-        if (l->has_value) {
-            off16 = INSN_OFF16(tcg_pcrel_diff(s, l->u.value_ptr));
-        } else {
-            /* Make sure to preserve destinations during retranslation.  */
-            off16 = *s->code_ptr & INSN_OFF16(-1);
-            tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP16, label, 0);
-        }
-        tcg_out32(s, INSN_OP(0) | INSN_OP2(3) | BPR_PT | INSN_RS1(arg1)
-                  | INSN_COND(tcg_cond_to_rcond[cond]) | off16);
-    } else {
-        tcg_out_cmp(s, arg1, arg2, const_arg2);
-        tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_XCC | BPCC_PT, label);
-    }
-    tcg_out_nop(s);
-}
-
-static void tcg_out_movr(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg c1,
-                         int32_t v1, int v1const)
-{
-    tcg_out32(s, ARITH_MOVR | INSN_RD(ret) | INSN_RS1(c1)
-              | (tcg_cond_to_rcond[cond] << 10)
-              | (v1const ? INSN_IMM10(v1) : INSN_RS2(v1)));
-}
-
-static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
-                                TCGReg c1, int32_t c2, int c2const,
-                                int32_t v1, int v1const)
-{
-    /* For 64-bit signed comparisons vs zero, we can avoid the compare.
-       Note that the immediate range is one bit smaller, so we must check
-       for that as well.  */
-    if (c2 == 0 && !is_unsigned_cond(cond)
-        && (!v1const || check_fit_i32(v1, 10))) {
-        tcg_out_movr(s, cond, ret, c1, v1, v1const);
-    } else {
-        tcg_out_cmp(s, c1, c2, c2const);
-        tcg_out_movcc(s, cond, MOVCC_XCC, ret, v1, v1const);
-    }
-}
-
-static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
-                                TCGReg c1, int32_t c2, int c2const)
-{
-    /* For 32-bit comparisons, we can play games with ADDC/SUBC.  */
-    switch (cond) {
-    case TCG_COND_LTU:
-    case TCG_COND_GEU:
-        /* The result of the comparison is in the carry bit.  */
-        break;
-
-    case TCG_COND_EQ:
-    case TCG_COND_NE:
-        /* For equality, we can transform to inequality vs zero.  */
-        if (c2 != 0) {
-            tcg_out_arithc(s, TCG_REG_T1, c1, c2, c2const, ARITH_XOR);
-            c2 = TCG_REG_T1;
-        } else {
-            c2 = c1;
-        }
-        c1 = TCG_REG_G0, c2const = 0;
-        cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU);
-	break;
-
-    case TCG_COND_GTU:
-    case TCG_COND_LEU:
-        /* If we don't need to load a constant into a register, we can
-           swap the operands on GTU/LEU.  There's no benefit to loading
-           the constant into a temporary register.  */
-        if (!c2const || c2 == 0) {
-            TCGReg t = c1;
-            c1 = c2;
-            c2 = t;
-            c2const = 0;
-            cond = tcg_swap_cond(cond);
-            break;
-        }
-        /* FALLTHRU */
-
-    default:
-        tcg_out_cmp(s, c1, c2, c2const);
-        tcg_out_movi_imm13(s, ret, 0);
-        tcg_out_movcc(s, cond, MOVCC_ICC, ret, 1, 1);
-        return;
-    }
-
-    tcg_out_cmp(s, c1, c2, c2const);
-    if (cond == TCG_COND_LTU) {
-        tcg_out_arithi(s, ret, TCG_REG_G0, 0, ARITH_ADDC);
-    } else {
-        tcg_out_arithi(s, ret, TCG_REG_G0, -1, ARITH_SUBC);
-    }
-}
-
-static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
-                                TCGReg c1, int32_t c2, int c2const)
-{
-    if (use_vis3_instructions) {
-        switch (cond) {
-        case TCG_COND_NE:
-            if (c2 != 0) {
-                break;
-            }
-            c2 = c1, c2const = 0, c1 = TCG_REG_G0;
-            /* FALLTHRU */
-        case TCG_COND_LTU:
-            tcg_out_cmp(s, c1, c2, c2const);
-            tcg_out_arith(s, ret, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC);
-            return;
-        default:
-            break;
-        }
-    }
-
-    /* For 64-bit signed comparisons vs zero, we can avoid the compare
-       if the input does not overlap the output.  */
-    if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) {
-        tcg_out_movi_imm13(s, ret, 0);
-        tcg_out_movr(s, cond, ret, c1, 1, 1);
-    } else {
-        tcg_out_cmp(s, c1, c2, c2const);
-        tcg_out_movi_imm13(s, ret, 0);
-        tcg_out_movcc(s, cond, MOVCC_XCC, ret, 1, 1);
-    }
-}
-
-static void tcg_out_addsub2_i32(TCGContext *s, TCGReg rl, TCGReg rh,
-                                TCGReg al, TCGReg ah, int32_t bl, int blconst,
-                                int32_t bh, int bhconst, int opl, int oph)
-{
-    TCGReg tmp = TCG_REG_T1;
-
-    /* Note that the low parts are fully consumed before tmp is set.  */
-    if (rl != ah && (bhconst || rl != bh)) {
-        tmp = rl;
-    }
-
-    tcg_out_arithc(s, tmp, al, bl, blconst, opl);
-    tcg_out_arithc(s, rh, ah, bh, bhconst, oph);
-    tcg_out_mov(s, TCG_TYPE_I32, rl, tmp);
-}
-
-static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh,
-                                TCGReg al, TCGReg ah, int32_t bl, int blconst,
-                                int32_t bh, int bhconst, bool is_sub)
-{
-    TCGReg tmp = TCG_REG_T1;
-
-    /* Note that the low parts are fully consumed before tmp is set.  */
-    if (rl != ah && (bhconst || rl != bh)) {
-        tmp = rl;
-    }
-
-    tcg_out_arithc(s, tmp, al, bl, blconst, is_sub ? ARITH_SUBCC : ARITH_ADDCC);
-
-    if (use_vis3_instructions && !is_sub) {
-        /* Note that ADDXC doesn't accept immediates.  */
-        if (bhconst && bh != 0) {
-           tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh);
-           bh = TCG_REG_T2;
-        }
-        tcg_out_arith(s, rh, ah, bh, ARITH_ADDXC);
-    } else if (bh == TCG_REG_G0) {
-	/* If we have a zero, we can perform the operation in two insns,
-           with the arithmetic first, and a conditional move into place.  */
-	if (rh == ah) {
-            tcg_out_arithi(s, TCG_REG_T2, ah, 1,
-			   is_sub ? ARITH_SUB : ARITH_ADD);
-            tcg_out_movcc(s, TCG_COND_LTU, MOVCC_XCC, rh, TCG_REG_T2, 0);
-	} else {
-            tcg_out_arithi(s, rh, ah, 1, is_sub ? ARITH_SUB : ARITH_ADD);
-	    tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, rh, ah, 0);
-	}
-    } else {
-        /* Otherwise adjust BH as if there is carry into T2 ... */
-        if (bhconst) {
-            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh + (is_sub ? -1 : 1));
-        } else {
-            tcg_out_arithi(s, TCG_REG_T2, bh, 1,
-                           is_sub ? ARITH_SUB : ARITH_ADD);
-        }
-        /* ... smoosh T2 back to original BH if carry is clear ... */
-        tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, TCG_REG_T2, bh, bhconst);
-	/* ... and finally perform the arithmetic with the new operand.  */
-        tcg_out_arith(s, rh, ah, TCG_REG_T2, is_sub ? ARITH_SUB : ARITH_ADD);
-    }
-
-    tcg_out_mov(s, TCG_TYPE_I64, rl, tmp);
-}
-
-static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest)
-{
-    ptrdiff_t disp = tcg_pcrel_diff(s, dest);
-
-    if (disp == (int32_t)disp) {
-        tcg_out32(s, CALL | (uint32_t)disp >> 2);
-    } else {
-        uintptr_t desti = (uintptr_t)dest;
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, desti & ~0xfff);
-        tcg_out_arithi(s, TCG_REG_O7, TCG_REG_T1, desti & 0xfff, JMPL);
-    }
-}
-
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
-{
-    tcg_out_call_nodelay(s, dest);
-    tcg_out_nop(s);
-}
-
-#ifdef CONFIG_SOFTMMU
-static tcg_insn_unit *qemu_ld_trampoline[16];
-static tcg_insn_unit *qemu_st_trampoline[16];
-
-static void build_trampolines(TCGContext *s)
-{
-    static void * const qemu_ld_helpers[16] = {
-        [MO_UB]   = helper_ret_ldub_mmu,
-        [MO_SB]   = helper_ret_ldsb_mmu,
-        [MO_LEUW] = helper_le_lduw_mmu,
-        [MO_LESW] = helper_le_ldsw_mmu,
-        [MO_LEUL] = helper_le_ldul_mmu,
-        [MO_LEQ]  = helper_le_ldq_mmu,
-        [MO_BEUW] = helper_be_lduw_mmu,
-        [MO_BESW] = helper_be_ldsw_mmu,
-        [MO_BEUL] = helper_be_ldul_mmu,
-        [MO_BEQ]  = helper_be_ldq_mmu,
-    };
-    static void * const qemu_st_helpers[16] = {
-        [MO_UB]   = helper_ret_stb_mmu,
-        [MO_LEUW] = helper_le_stw_mmu,
-        [MO_LEUL] = helper_le_stl_mmu,
-        [MO_LEQ]  = helper_le_stq_mmu,
-        [MO_BEUW] = helper_be_stw_mmu,
-        [MO_BEUL] = helper_be_stl_mmu,
-        [MO_BEQ]  = helper_be_stq_mmu,
-    };
-
-    int i;
-    TCGReg ra;
-
-    for (i = 0; i < 16; ++i) {
-        if (qemu_ld_helpers[i] == NULL) {
-            continue;
-        }
-
-        /* May as well align the trampoline.  */
-        while ((uintptr_t)s->code_ptr & 15) {
-            tcg_out_nop(s);
-        }
-        qemu_ld_trampoline[i] = s->code_ptr;
-
-        if (SPARC64 || TARGET_LONG_BITS == 32) {
-            ra = TCG_REG_O3;
-        } else {
-            /* Install the high part of the address.  */
-            tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O2, 32, SHIFT_SRLX);
-            ra = TCG_REG_O4;
-        }
-
-        /* Set the retaddr operand.  */
-        tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7);
-        /* Set the env operand.  */
-        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O0, TCG_AREG0);
-        /* Tail call.  */
-        tcg_out_call_nodelay(s, qemu_ld_helpers[i]);
-        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra);
-    }
-
-    for (i = 0; i < 16; ++i) {
-        if (qemu_st_helpers[i] == NULL) {
-            continue;
-        }
-
-        /* May as well align the trampoline.  */
-        while ((uintptr_t)s->code_ptr & 15) {
-            tcg_out_nop(s);
-        }
-        qemu_st_trampoline[i] = s->code_ptr;
-
-        if (SPARC64) {
-            ra = TCG_REG_O4;
-        } else {
-            ra = TCG_REG_O1;
-            if (TARGET_LONG_BITS == 64) {
-                /* Install the high part of the address.  */
-                tcg_out_arithi(s, ra, ra + 1, 32, SHIFT_SRLX);
-                ra += 2;
-            } else {
-                ra += 1;
-            }
-            if ((i & MO_SIZE) == MO_64) {
-                /* Install the high part of the data.  */
-                tcg_out_arithi(s, ra, ra + 1, 32, SHIFT_SRLX);
-                ra += 2;
-            } else {
-                ra += 1;
-            }
-            /* Skip the oi argument.  */
-            ra += 1;
-        }
-                
-        /* Set the retaddr operand.  */
-        if (ra >= TCG_REG_O6) {
-            tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_O7, TCG_REG_CALL_STACK,
-                       TCG_TARGET_CALL_STACK_OFFSET);
-            ra = TCG_REG_G1;
-        }
-        tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7);
-        /* Set the env operand.  */
-        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O0, TCG_AREG0);
-        /* Tail call.  */
-        tcg_out_call_nodelay(s, qemu_st_helpers[i]);
-        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra);
-    }
-}
-#endif
-
-/* Generate global QEMU prologue and epilogue code */
-static void tcg_target_qemu_prologue(TCGContext *s)
-{
-    int tmp_buf_size, frame_size;
-
-    /* The TCG temp buffer is at the top of the frame, immediately
-       below the frame pointer.  */
-    tmp_buf_size = CPU_TEMP_BUF_NLONGS * (int)sizeof(long);
-    tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_STACK_BIAS - tmp_buf_size,
-                  tmp_buf_size);
-
-    /* TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is
-       otherwise the minimal frame usable by callees.  */
-    frame_size = TCG_TARGET_CALL_STACK_OFFSET - TCG_TARGET_STACK_BIAS;
-    frame_size += TCG_STATIC_CALL_ARGS_SIZE + tmp_buf_size;
-    frame_size += TCG_TARGET_STACK_ALIGN - 1;
-    frame_size &= -TCG_TARGET_STACK_ALIGN;
-    tcg_out32(s, SAVE | INSN_RD(TCG_REG_O6) | INSN_RS1(TCG_REG_O6) |
-              INSN_IMM13(-frame_size));
-
-#ifdef CONFIG_USE_GUEST_BASE
-    if (GUEST_BASE != 0) {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
-        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
-    }
-#endif
-
-    tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL);
-    /* delay slot */
-    tcg_out_nop(s);
-
-    /* No epilogue required.  We issue ret + restore directly in the TB.  */
-
-#ifdef CONFIG_SOFTMMU
-    build_trampolines(s);
-#endif
-}
-
-#if defined(CONFIG_SOFTMMU)
-/* Perform the TLB load and compare.
-
-   Inputs:
-   ADDRLO and ADDRHI contain the possible two parts of the address.
-
-   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
-
-   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
-   This should be offsetof addr_read or addr_write.
-
-   The result of the TLB comparison is in %[ix]cc.  The sanitized address
-   is in the returned register, maybe %o0.  The TLB addend is in %o1.  */
-
-static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index,
-                               TCGMemOp s_bits, int which)
-{
-    const TCGReg r0 = TCG_REG_O0;
-    const TCGReg r1 = TCG_REG_O1;
-    const TCGReg r2 = TCG_REG_O2;
-    int tlb_ofs;
-
-    /* Shift the page number down.  */
-    tcg_out_arithi(s, r1, addr, TARGET_PAGE_BITS, SHIFT_SRL);
-
-    /* Mask out the page offset, except for the required alignment.  */
-    tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_T1,
-                 TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-
-    /* Mask the tlb index.  */
-    tcg_out_arithi(s, r1, r1, CPU_TLB_SIZE - 1, ARITH_AND);
-    
-    /* Mask page, part 2.  */
-    tcg_out_arith(s, r0, addr, TCG_REG_T1, ARITH_AND);
-
-    /* Shift the tlb index into place.  */
-    tcg_out_arithi(s, r1, r1, CPU_TLB_ENTRY_BITS, SHIFT_SLL);
-
-    /* Relative to the current ENV.  */
-    tcg_out_arith(s, r1, TCG_AREG0, r1, ARITH_ADD);
-
-    /* Find a base address that can load both tlb comparator and addend.  */
-    tlb_ofs = offsetof(CPUArchState, tlb_table[mem_index][0]);
-    if (!check_fit_ptr(tlb_ofs + sizeof(CPUTLBEntry), 13)) {
-        if (tlb_ofs & ~0x3ff) {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, tlb_ofs & ~0x3ff);
-            tcg_out_arith(s, r1, r1, TCG_REG_T1, ARITH_ADD);
-        }
-        tlb_ofs &= 0x3ff;
-    }
-
-    /* Load the tlb comparator and the addend.  */
-    tcg_out_ld(s, TCG_TYPE_TL, r2, r1, tlb_ofs + which);
-    tcg_out_ld(s, TCG_TYPE_PTR, r1, r1, tlb_ofs+offsetof(CPUTLBEntry, addend));
-
-    /* subcc arg0, arg2, %g0 */
-    tcg_out_cmp(s, r0, r2, 0);
-
-    /* If the guest address must be zero-extended, do so now.  */
-    if (SPARC64 && TARGET_LONG_BITS == 32) {
-        tcg_out_arithi(s, r0, addr, 0, SHIFT_SRL);
-        return r0;
-    }
-    return addr;
-}
-#endif /* CONFIG_SOFTMMU */
-
-static const int qemu_ld_opc[16] = {
-    [MO_UB]   = LDUB,
-    [MO_SB]   = LDSB,
-
-    [MO_BEUW] = LDUH,
-    [MO_BESW] = LDSH,
-    [MO_BEUL] = LDUW,
-    [MO_BESL] = LDSW,
-    [MO_BEQ]  = LDX,
-
-    [MO_LEUW] = LDUH_LE,
-    [MO_LESW] = LDSH_LE,
-    [MO_LEUL] = LDUW_LE,
-    [MO_LESL] = LDSW_LE,
-    [MO_LEQ]  = LDX_LE,
-};
-
-static const int qemu_st_opc[16] = {
-    [MO_UB]   = STB,
-
-    [MO_BEUW] = STH,
-    [MO_BEUL] = STW,
-    [MO_BEQ]  = STX,
-
-    [MO_LEUW] = STH_LE,
-    [MO_LEUL] = STW_LE,
-    [MO_LEQ]  = STX_LE,
-};
-
-static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
-                            TCGMemOpIdx oi, bool is_64)
-{
-    TCGMemOp memop = get_memop(oi);
-#ifdef CONFIG_SOFTMMU
-    unsigned memi = get_mmuidx(oi);
-    TCGReg addrz, param;
-    tcg_insn_unit *func;
-    tcg_insn_unit *label_ptr;
-
-    addrz = tcg_out_tlb_load(s, addr, memi, memop & MO_SIZE,
-                             offsetof(CPUTLBEntry, addr_read));
-
-    /* The fast path is exactly one insn.  Thus we can perform the
-       entire TLB Hit in the (annulled) delay slot of the branch
-       over the TLB Miss case.  */
-
-    /* beq,a,pt %[xi]cc, label0 */
-    label_ptr = s->code_ptr;
-    tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
-                  | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
-    /* delay slot */
-    tcg_out_ldst_rr(s, data, addrz, TCG_REG_O1,
-                    qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]);
-
-    /* TLB Miss.  */
-
-    param = TCG_REG_O1;
-    if (!SPARC64 && TARGET_LONG_BITS == 64) {
-        /* Skip the high-part; we'll perform the extract in the trampoline.  */
-        param++;
-    }
-    tcg_out_mov(s, TCG_TYPE_REG, param++, addr);
-
-    /* We use the helpers to extend SB and SW data, leaving the case
-       of SL needing explicit extending below.  */
-    if ((memop & MO_SSIZE) == MO_SL) {
-        func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SIZE)];
-    } else {
-        func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SSIZE)];
-    }
-    assert(func != NULL);
-    tcg_out_call_nodelay(s, func);
-    /* delay slot */
-    tcg_out_movi(s, TCG_TYPE_I32, param, oi);
-
-    /* Recall that all of the helpers return 64-bit results.
-       Which complicates things for sparcv8plus.  */
-    if (SPARC64) {
-        /* We let the helper sign-extend SB and SW, but leave SL for here.  */
-        if (is_64 && (memop & MO_SSIZE) == MO_SL) {
-            tcg_out_arithi(s, data, TCG_REG_O0, 0, SHIFT_SRA);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0);
-        }
-    } else {
-        if ((memop & MO_SIZE) == MO_64) {
-            tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, 32, SHIFT_SLLX);
-            tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O1, 0, SHIFT_SRL);
-            tcg_out_arith(s, data, TCG_REG_O0, TCG_REG_O1, ARITH_OR);
-        } else if (is_64) {
-            /* Re-extend from 32-bit rather than reassembling when we
-               know the high register must be an extension.  */
-            tcg_out_arithi(s, data, TCG_REG_O1, 0,
-                           memop & MO_SIGN ? SHIFT_SRA : SHIFT_SRL);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, data, TCG_REG_O1);
-        }
-    }
-
-    *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr));
-#else
-    if (SPARC64 && TARGET_LONG_BITS == 32) {
-        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
-        addr = TCG_REG_T1;
-    }
-    tcg_out_ldst_rr(s, data, addr,
-                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
-                    qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]);
-#endif /* CONFIG_SOFTMMU */
-}
-
-static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
-                            TCGMemOpIdx oi)
-{
-    TCGMemOp memop = get_memop(oi);
-#ifdef CONFIG_SOFTMMU
-    unsigned memi = get_mmuidx(oi);
-    TCGReg addrz, param;
-    tcg_insn_unit *func;
-    tcg_insn_unit *label_ptr;
-
-    addrz = tcg_out_tlb_load(s, addr, memi, memop & MO_SIZE,
-                             offsetof(CPUTLBEntry, addr_write));
-
-    /* The fast path is exactly one insn.  Thus we can perform the entire
-       TLB Hit in the (annulled) delay slot of the branch over TLB Miss.  */
-    /* beq,a,pt %[xi]cc, label0 */
-    label_ptr = s->code_ptr;
-    tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
-                  | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
-    /* delay slot */
-    tcg_out_ldst_rr(s, data, addrz, TCG_REG_O1,
-                    qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]);
-
-    /* TLB Miss.  */
-
-    param = TCG_REG_O1;
-    if (!SPARC64 && TARGET_LONG_BITS == 64) {
-        /* Skip the high-part; we'll perform the extract in the trampoline.  */
-        param++;
-    }
-    tcg_out_mov(s, TCG_TYPE_REG, param++, addr);
-    if (!SPARC64 && (memop & MO_SIZE) == MO_64) {
-        /* Skip the high-part; we'll perform the extract in the trampoline.  */
-        param++;
-    }
-    tcg_out_mov(s, TCG_TYPE_REG, param++, data);
-
-    func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)];
-    assert(func != NULL);
-    tcg_out_call_nodelay(s, func);
-    /* delay slot */
-    tcg_out_movi(s, TCG_TYPE_I32, param, oi);
-
-    *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr));
-#else
-    if (SPARC64 && TARGET_LONG_BITS == 32) {
-        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
-        addr = TCG_REG_T1;
-    }
-    tcg_out_ldst_rr(s, data, addr,
-                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
-                    qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]);
-#endif /* CONFIG_SOFTMMU */
-}
-
-static void tcg_out_op(TCGContext *s, TCGOpcode opc,
-                       const TCGArg args[TCG_MAX_OP_ARGS],
-                       const int const_args[TCG_MAX_OP_ARGS])
-{
-    TCGArg a0, a1, a2;
-    int c, c2;
-
-    /* Hoist the loads of the most common arguments.  */
-    a0 = args[0];
-    a1 = args[1];
-    a2 = args[2];
-    c2 = const_args[2];
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        if (check_fit_ptr(a0, 13)) {
-            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
-            tcg_out_movi_imm13(s, TCG_REG_O0, a0);
-        } else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
-            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
-            tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
-        }
-        break;
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* direct jump method */
-            s->tb_jmp_offset[a0] = tcg_current_code_size(s);
-            /* Make sure to preserve links during retranslation.  */
-            tcg_out32(s, CALL | (*s->code_ptr & ~INSN_OP(-1)));
-        } else {
-            /* indirect jump method */
-            tcg_out_ld_ptr(s, TCG_REG_T1, (uintptr_t)(s->tb_next + a0));
-            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_T1, 0, JMPL);
-        }
-        tcg_out_nop(s);
-        s->tb_next_offset[a0] = tcg_current_code_size(s);
-        break;
-    case INDEX_op_br:
-        tcg_out_bpcc(s, COND_A, BPCC_PT, a0);
-        tcg_out_nop(s);
-        break;
-
-#define OP_32_64(x)                             \
-        glue(glue(case INDEX_op_, x), _i32):    \
-        glue(glue(case INDEX_op_, x), _i64)
-
-    OP_32_64(ld8u):
-        tcg_out_ldst(s, a0, a1, a2, LDUB);
-        break;
-    OP_32_64(ld8s):
-        tcg_out_ldst(s, a0, a1, a2, LDSB);
-        break;
-    OP_32_64(ld16u):
-        tcg_out_ldst(s, a0, a1, a2, LDUH);
-        break;
-    OP_32_64(ld16s):
-        tcg_out_ldst(s, a0, a1, a2, LDSH);
-        break;
-    case INDEX_op_ld_i32:
-    case INDEX_op_ld32u_i64:
-        tcg_out_ldst(s, a0, a1, a2, LDUW);
-        break;
-    OP_32_64(st8):
-        tcg_out_ldst(s, a0, a1, a2, STB);
-        break;
-    OP_32_64(st16):
-        tcg_out_ldst(s, a0, a1, a2, STH);
-        break;
-    case INDEX_op_st_i32:
-    case INDEX_op_st32_i64:
-        tcg_out_ldst(s, a0, a1, a2, STW);
-        break;
-    OP_32_64(add):
-        c = ARITH_ADD;
-        goto gen_arith;
-    OP_32_64(sub):
-        c = ARITH_SUB;
-        goto gen_arith;
-    OP_32_64(and):
-        c = ARITH_AND;
-        goto gen_arith;
-    OP_32_64(andc):
-        c = ARITH_ANDN;
-        goto gen_arith;
-    OP_32_64(or):
-        c = ARITH_OR;
-        goto gen_arith;
-    OP_32_64(orc):
-        c = ARITH_ORN;
-        goto gen_arith;
-    OP_32_64(xor):
-        c = ARITH_XOR;
-        goto gen_arith;
-    case INDEX_op_shl_i32:
-        c = SHIFT_SLL;
-    do_shift32:
-        /* Limit immediate shift count lest we create an illegal insn.  */
-        tcg_out_arithc(s, a0, a1, a2 & 31, c2, c);
-        break;
-    case INDEX_op_shr_i32:
-        c = SHIFT_SRL;
-        goto do_shift32;
-    case INDEX_op_sar_i32:
-        c = SHIFT_SRA;
-        goto do_shift32;
-    case INDEX_op_mul_i32:
-        c = ARITH_UMUL;
-        goto gen_arith;
-
-    OP_32_64(neg):
-	c = ARITH_SUB;
-	goto gen_arith1;
-    OP_32_64(not):
-	c = ARITH_ORN;
-	goto gen_arith1;
-
-    case INDEX_op_div_i32:
-        tcg_out_div32(s, a0, a1, a2, c2, 0);
-        break;
-    case INDEX_op_divu_i32:
-        tcg_out_div32(s, a0, a1, a2, c2, 1);
-        break;
-
-    case INDEX_op_brcond_i32:
-        tcg_out_brcond_i32(s, a2, a0, a1, const_args[1], args[3]);
-        break;
-    case INDEX_op_setcond_i32:
-        tcg_out_setcond_i32(s, args[3], a0, a1, a2, c2);
-        break;
-    case INDEX_op_movcond_i32:
-        tcg_out_movcond_i32(s, args[5], a0, a1, a2, c2, args[3], const_args[3]);
-        break;
-
-    case INDEX_op_add2_i32:
-        tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3],
-                            args[4], const_args[4], args[5], const_args[5],
-                            ARITH_ADDCC, ARITH_ADDC);
-        break;
-    case INDEX_op_sub2_i32:
-        tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3],
-                            args[4], const_args[4], args[5], const_args[5],
-                            ARITH_SUBCC, ARITH_SUBC);
-        break;
-    case INDEX_op_mulu2_i32:
-        c = ARITH_UMUL;
-        goto do_mul2;
-    case INDEX_op_muls2_i32:
-        c = ARITH_SMUL;
-    do_mul2:
-        /* The 32-bit multiply insns produce a full 64-bit result.  If the
-           destination register can hold it, we can avoid the slower RDY.  */
-        tcg_out_arithc(s, a0, a2, args[3], const_args[3], c);
-        if (SPARC64 || a0 <= TCG_REG_O7) {
-            tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX);
-        } else {
-            tcg_out_rdy(s, a1);
-        }
-        break;
-
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_qemu_ld(s, a0, a1, a2, false);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_qemu_ld(s, a0, a1, a2, true);
-        break;
-    case INDEX_op_qemu_st_i32:
-    case INDEX_op_qemu_st_i64:
-        tcg_out_qemu_st(s, a0, a1, a2);
-        break;
-
-    case INDEX_op_ld32s_i64:
-        tcg_out_ldst(s, a0, a1, a2, LDSW);
-        break;
-    case INDEX_op_ld_i64:
-        tcg_out_ldst(s, a0, a1, a2, LDX);
-        break;
-    case INDEX_op_st_i64:
-        tcg_out_ldst(s, a0, a1, a2, STX);
-        break;
-    case INDEX_op_shl_i64:
-        c = SHIFT_SLLX;
-    do_shift64:
-        /* Limit immediate shift count lest we create an illegal insn.  */
-        tcg_out_arithc(s, a0, a1, a2 & 63, c2, c);
-        break;
-    case INDEX_op_shr_i64:
-        c = SHIFT_SRLX;
-        goto do_shift64;
-    case INDEX_op_sar_i64:
-        c = SHIFT_SRAX;
-        goto do_shift64;
-    case INDEX_op_mul_i64:
-        c = ARITH_MULX;
-        goto gen_arith;
-    case INDEX_op_div_i64:
-        c = ARITH_SDIVX;
-        goto gen_arith;
-    case INDEX_op_divu_i64:
-        c = ARITH_UDIVX;
-        goto gen_arith;
-    case INDEX_op_ext32s_i64:
-        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRA);
-        break;
-    case INDEX_op_ext32u_i64:
-        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRL);
-        break;
-    case INDEX_op_trunc_shr_i32:
-        if (a2 == 0) {
-            tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
-        } else {
-            tcg_out_arithi(s, a0, a1, a2, SHIFT_SRLX);
-        }
-        break;
-
-    case INDEX_op_brcond_i64:
-        tcg_out_brcond_i64(s, a2, a0, a1, const_args[1], args[3]);
-        break;
-    case INDEX_op_setcond_i64:
-        tcg_out_setcond_i64(s, args[3], a0, a1, a2, c2);
-        break;
-    case INDEX_op_movcond_i64:
-        tcg_out_movcond_i64(s, args[5], a0, a1, a2, c2, args[3], const_args[3]);
-        break;
-    case INDEX_op_add2_i64:
-        tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4],
-                            const_args[4], args[5], const_args[5], false);
-        break;
-    case INDEX_op_sub2_i64:
-        tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4],
-                            const_args[4], args[5], const_args[5], true);
-        break;
-    case INDEX_op_muluh_i64:
-        tcg_out_arith(s, args[0], args[1], args[2], ARITH_UMULXHI);
-        break;
-
-    gen_arith:
-        tcg_out_arithc(s, a0, a1, a2, c2, c);
-        break;
-
-    gen_arith1:
-	tcg_out_arithc(s, a0, TCG_REG_G0, a1, const_args[1], c);
-	break;
-
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-}
-
-static const TCGTargetOpDef sparc_op_defs[] = {
-    { INDEX_op_exit_tb, { } },
-    { INDEX_op_goto_tb, { } },
-    { INDEX_op_br, { } },
-
-    { INDEX_op_ld8u_i32, { "r", "r" } },
-    { INDEX_op_ld8s_i32, { "r", "r" } },
-    { INDEX_op_ld16u_i32, { "r", "r" } },
-    { INDEX_op_ld16s_i32, { "r", "r" } },
-    { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "rZ", "r" } },
-    { INDEX_op_st16_i32, { "rZ", "r" } },
-    { INDEX_op_st_i32, { "rZ", "r" } },
-
-    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_mul_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_div_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_divu_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_and_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_andc_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_or_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_orc_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_xor_i32, { "r", "rZ", "rJ" } },
-
-    { INDEX_op_shl_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_shr_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_sar_i32, { "r", "rZ", "rJ" } },
-
-    { INDEX_op_neg_i32, { "r", "rJ" } },
-    { INDEX_op_not_i32, { "r", "rJ" } },
-
-    { INDEX_op_brcond_i32, { "rZ", "rJ" } },
-    { INDEX_op_setcond_i32, { "r", "rZ", "rJ" } },
-    { INDEX_op_movcond_i32, { "r", "rZ", "rJ", "rI", "0" } },
-
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
-    { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rJ" } },
-    { INDEX_op_muls2_i32, { "r", "r", "rZ", "rJ" } },
-
-    { INDEX_op_ld8u_i64, { "R", "r" } },
-    { INDEX_op_ld8s_i64, { "R", "r" } },
-    { INDEX_op_ld16u_i64, { "R", "r" } },
-    { INDEX_op_ld16s_i64, { "R", "r" } },
-    { INDEX_op_ld32u_i64, { "R", "r" } },
-    { INDEX_op_ld32s_i64, { "R", "r" } },
-    { INDEX_op_ld_i64, { "R", "r" } },
-    { INDEX_op_st8_i64, { "RZ", "r" } },
-    { INDEX_op_st16_i64, { "RZ", "r" } },
-    { INDEX_op_st32_i64, { "RZ", "r" } },
-    { INDEX_op_st_i64, { "RZ", "r" } },
-
-    { INDEX_op_add_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_mul_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_div_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_divu_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_sub_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_and_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_andc_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_or_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_orc_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_xor_i64, { "R", "RZ", "RJ" } },
-
-    { INDEX_op_shl_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_shr_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_sar_i64, { "R", "RZ", "RJ" } },
-
-    { INDEX_op_neg_i64, { "R", "RJ" } },
-    { INDEX_op_not_i64, { "R", "RJ" } },
-
-    { INDEX_op_ext32s_i64, { "R", "r" } },
-    { INDEX_op_ext32u_i64, { "R", "r" } },
-    { INDEX_op_trunc_shr_i32,  { "r", "R" } },
-
-    { INDEX_op_brcond_i64, { "RZ", "RJ" } },
-    { INDEX_op_setcond_i64, { "R", "RZ", "RJ" } },
-    { INDEX_op_movcond_i64, { "R", "RZ", "RJ", "RI", "0" } },
-
-    { INDEX_op_add2_i64, { "R", "R", "RZ", "RZ", "RJ", "RI" } },
-    { INDEX_op_sub2_i64, { "R", "R", "RZ", "RZ", "RJ", "RI" } },
-    { INDEX_op_muluh_i64, { "R", "RZ", "RZ" } },
-
-    { INDEX_op_qemu_ld_i32, { "r", "A" } },
-    { INDEX_op_qemu_ld_i64, { "R", "A" } },
-    { INDEX_op_qemu_st_i32, { "sZ", "A" } },
-    { INDEX_op_qemu_st_i64, { "SZ", "A" } },
-
-    { -1 },
-};
-
-static void tcg_target_init(TCGContext *s)
-{
-    /* Only probe for the platform and capabilities if we havn't already
-       determined maximum values at compile time.  */
-#ifndef use_vis3_instructions
-    {
-        unsigned long hwcap = qemu_getauxval(AT_HWCAP);
-        use_vis3_instructions = (hwcap & HWCAP_SPARC_VIS3) != 0;
-    }
-#endif
-
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, ALL_64);
-
-    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
-                     (1 << TCG_REG_G1) |
-                     (1 << TCG_REG_G2) |
-                     (1 << TCG_REG_G3) |
-                     (1 << TCG_REG_G4) |
-                     (1 << TCG_REG_G5) |
-                     (1 << TCG_REG_G6) |
-                     (1 << TCG_REG_G7) |
-                     (1 << TCG_REG_O0) |
-                     (1 << TCG_REG_O1) |
-                     (1 << TCG_REG_O2) |
-                     (1 << TCG_REG_O3) |
-                     (1 << TCG_REG_O4) |
-                     (1 << TCG_REG_O5) |
-                     (1 << TCG_REG_O7));
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0); /* zero */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G6); /* reserved for os */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G7); /* thread pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6); /* frame pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7); /* return address */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6); /* stack pointer */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T1); /* for internal use */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */
-
-    tcg_add_target_add_op_defs(sparc_op_defs);
-}
-
-#if SPARC64
-# define ELF_HOST_MACHINE  EM_SPARCV9
-#else
-# define ELF_HOST_MACHINE  EM_SPARC32PLUS
-# define ELF_HOST_FLAGS    EF_SPARC_32PLUS
-#endif
-
-typedef struct {
-    DebugFrameHeader h;
-    uint8_t fde_def_cfa[SPARC64 ? 4 : 2];
-    uint8_t fde_win_save;
-    uint8_t fde_ret_save[3];
-} DebugFrame;
-
-static const DebugFrame debug_frame = {
-    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .h.cie.id = -1,
-    .h.cie.version = 1,
-    .h.cie.code_align = 1,
-    .h.cie.data_align = -sizeof(void *) & 0x7f,
-    .h.cie.return_column = 15,            /* o7 */
-
-    /* Total FDE size does not include the "len" member.  */
-    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
-
-    .fde_def_cfa = {
-#if SPARC64
-        12, 30,                         /* DW_CFA_def_cfa i6, 2047 */
-        (2047 & 0x7f) | 0x80, (2047 >> 7)
-#else
-        13, 30                          /* DW_CFA_def_cfa_register i6 */
-#endif
-    },
-    .fde_win_save = 0x2d,               /* DW_CFA_GNU_window_save */
-    .fde_ret_save = { 9, 15, 31 },      /* DW_CFA_register o7, i7 */
-};
-
-void tcg_register_jit(void *buf, size_t buf_size)
-{
-    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
-}
-
-void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
-{
-    uint32_t *ptr = (uint32_t *)jmp_addr;
-    uintptr_t disp = addr - jmp_addr;
-
-    /* We can reach the entire address space for 32-bit.  For 64-bit
-       the code_gen_buffer can't be larger than 2GB.  */
-    assert(disp == (int32_t)disp);
-
-    *ptr = CALL | (uint32_t)disp >> 2;
-    flush_icache_range(jmp_addr, jmp_addr + 4);
-}
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index 0c4c8af..88f9c90 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -21,12 +21,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef TCG_TARGET_SPARC 
-#define TCG_TARGET_SPARC 1
+
+#ifndef SPARC_TCG_TARGET_H
+#define SPARC_TCG_TARGET_H
 
 #define TCG_TARGET_REG_BITS 64
 
 #define TCG_TARGET_INSN_UNIT_SIZE 4
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 32
 #define TCG_TARGET_NB_REGS 32
 
 typedef enum {
@@ -117,7 +119,8 @@
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 
-#define TCG_TARGET_HAS_trunc_shr_i32    1
+#define TCG_TARGET_HAS_extrl_i64_i32    1
+#define TCG_TARGET_HAS_extrh_i64_i32    1
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_rot_i64          0
diff --git a/tcg/sparc/tcg-target.inc.c b/tcg/sparc/tcg-target.inc.c
new file mode 100644
index 0000000..8e98172
--- /dev/null
+++ b/tcg/sparc/tcg-target.inc.c
@@ -0,0 +1,1663 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg-be-null.h"
+
+#ifdef CONFIG_DEBUG_TCG
+static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "%g0",
+    "%g1",
+    "%g2",
+    "%g3",
+    "%g4",
+    "%g5",
+    "%g6",
+    "%g7",
+    "%o0",
+    "%o1",
+    "%o2",
+    "%o3",
+    "%o4",
+    "%o5",
+    "%o6",
+    "%o7",
+    "%l0",
+    "%l1",
+    "%l2",
+    "%l3",
+    "%l4",
+    "%l5",
+    "%l6",
+    "%l7",
+    "%i0",
+    "%i1",
+    "%i2",
+    "%i3",
+    "%i4",
+    "%i5",
+    "%i6",
+    "%i7",
+};
+#endif
+
+#ifdef __arch64__
+# define SPARC64 1
+#else
+# define SPARC64 0
+#endif
+
+/* Note that sparcv8plus can only hold 64 bit quantities in %g and %o
+   registers.  These are saved manually by the kernel in full 64-bit
+   slots.  The %i and %l registers are saved by the register window
+   mechanism, which only allocates space for 32 bits.  Given that this
+   window spill/fill can happen on any signal, we must consider the
+   high bits of the %i and %l registers garbage at all times.  */
+#if SPARC64
+# define ALL_64  0xffffffffu
+#else
+# define ALL_64  0xffffu
+#endif
+
+/* Define some temporary registers.  T2 is used for constant generation.  */
+#define TCG_REG_T1  TCG_REG_G1
+#define TCG_REG_T2  TCG_REG_O7
+
+#ifndef CONFIG_SOFTMMU
+# define TCG_GUEST_BASE_REG TCG_REG_I5
+#endif
+
+static const int tcg_target_reg_alloc_order[] = {
+    TCG_REG_L0,
+    TCG_REG_L1,
+    TCG_REG_L2,
+    TCG_REG_L3,
+    TCG_REG_L4,
+    TCG_REG_L5,
+    TCG_REG_L6,
+    TCG_REG_L7,
+
+    TCG_REG_I0,
+    TCG_REG_I1,
+    TCG_REG_I2,
+    TCG_REG_I3,
+    TCG_REG_I4,
+    TCG_REG_I5,
+
+    TCG_REG_G2,
+    TCG_REG_G3,
+    TCG_REG_G4,
+    TCG_REG_G5,
+
+    TCG_REG_O0,
+    TCG_REG_O1,
+    TCG_REG_O2,
+    TCG_REG_O3,
+    TCG_REG_O4,
+    TCG_REG_O5,
+};
+
+static const int tcg_target_call_iarg_regs[6] = {
+    TCG_REG_O0,
+    TCG_REG_O1,
+    TCG_REG_O2,
+    TCG_REG_O3,
+    TCG_REG_O4,
+    TCG_REG_O5,
+};
+
+static const int tcg_target_call_oarg_regs[] = {
+    TCG_REG_O0,
+    TCG_REG_O1,
+    TCG_REG_O2,
+    TCG_REG_O3,
+};
+
+#define INSN_OP(x)  ((x) << 30)
+#define INSN_OP2(x) ((x) << 22)
+#define INSN_OP3(x) ((x) << 19)
+#define INSN_OPF(x) ((x) << 5)
+#define INSN_RD(x)  ((x) << 25)
+#define INSN_RS1(x) ((x) << 14)
+#define INSN_RS2(x) (x)
+#define INSN_ASI(x) ((x) << 5)
+
+#define INSN_IMM10(x) ((1 << 13) | ((x) & 0x3ff))
+#define INSN_IMM11(x) ((1 << 13) | ((x) & 0x7ff))
+#define INSN_IMM13(x) ((1 << 13) | ((x) & 0x1fff))
+#define INSN_OFF16(x) ((((x) >> 2) & 0x3fff) | ((((x) >> 16) & 3) << 20))
+#define INSN_OFF19(x) (((x) >> 2) & 0x07ffff)
+#define INSN_COND(x) ((x) << 25)
+
+#define COND_N     0x0
+#define COND_E     0x1
+#define COND_LE    0x2
+#define COND_L     0x3
+#define COND_LEU   0x4
+#define COND_CS    0x5
+#define COND_NEG   0x6
+#define COND_VS    0x7
+#define COND_A     0x8
+#define COND_NE    0x9
+#define COND_G     0xa
+#define COND_GE    0xb
+#define COND_GU    0xc
+#define COND_CC    0xd
+#define COND_POS   0xe
+#define COND_VC    0xf
+#define BA         (INSN_OP(0) | INSN_COND(COND_A) | INSN_OP2(0x2))
+
+#define RCOND_Z    1
+#define RCOND_LEZ  2
+#define RCOND_LZ   3
+#define RCOND_NZ   5
+#define RCOND_GZ   6
+#define RCOND_GEZ  7
+
+#define MOVCC_ICC  (1 << 18)
+#define MOVCC_XCC  (1 << 18 | 1 << 12)
+
+#define BPCC_ICC   0
+#define BPCC_XCC   (2 << 20)
+#define BPCC_PT    (1 << 19)
+#define BPCC_PN    0
+#define BPCC_A     (1 << 29)
+
+#define BPR_PT     BPCC_PT
+
+#define ARITH_ADD  (INSN_OP(2) | INSN_OP3(0x00))
+#define ARITH_ADDCC (INSN_OP(2) | INSN_OP3(0x10))
+#define ARITH_AND  (INSN_OP(2) | INSN_OP3(0x01))
+#define ARITH_ANDN (INSN_OP(2) | INSN_OP3(0x05))
+#define ARITH_OR   (INSN_OP(2) | INSN_OP3(0x02))
+#define ARITH_ORCC (INSN_OP(2) | INSN_OP3(0x12))
+#define ARITH_ORN  (INSN_OP(2) | INSN_OP3(0x06))
+#define ARITH_XOR  (INSN_OP(2) | INSN_OP3(0x03))
+#define ARITH_SUB  (INSN_OP(2) | INSN_OP3(0x04))
+#define ARITH_SUBCC (INSN_OP(2) | INSN_OP3(0x14))
+#define ARITH_ADDC (INSN_OP(2) | INSN_OP3(0x08))
+#define ARITH_SUBC (INSN_OP(2) | INSN_OP3(0x0c))
+#define ARITH_UMUL (INSN_OP(2) | INSN_OP3(0x0a))
+#define ARITH_SMUL (INSN_OP(2) | INSN_OP3(0x0b))
+#define ARITH_UDIV (INSN_OP(2) | INSN_OP3(0x0e))
+#define ARITH_SDIV (INSN_OP(2) | INSN_OP3(0x0f))
+#define ARITH_MULX (INSN_OP(2) | INSN_OP3(0x09))
+#define ARITH_UDIVX (INSN_OP(2) | INSN_OP3(0x0d))
+#define ARITH_SDIVX (INSN_OP(2) | INSN_OP3(0x2d))
+#define ARITH_MOVCC (INSN_OP(2) | INSN_OP3(0x2c))
+#define ARITH_MOVR (INSN_OP(2) | INSN_OP3(0x2f))
+
+#define ARITH_ADDXC (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x11))
+#define ARITH_UMULXHI (INSN_OP(2) | INSN_OP3(0x36) | INSN_OPF(0x16))
+
+#define SHIFT_SLL  (INSN_OP(2) | INSN_OP3(0x25))
+#define SHIFT_SRL  (INSN_OP(2) | INSN_OP3(0x26))
+#define SHIFT_SRA  (INSN_OP(2) | INSN_OP3(0x27))
+
+#define SHIFT_SLLX (INSN_OP(2) | INSN_OP3(0x25) | (1 << 12))
+#define SHIFT_SRLX (INSN_OP(2) | INSN_OP3(0x26) | (1 << 12))
+#define SHIFT_SRAX (INSN_OP(2) | INSN_OP3(0x27) | (1 << 12))
+
+#define RDY        (INSN_OP(2) | INSN_OP3(0x28) | INSN_RS1(0))
+#define WRY        (INSN_OP(2) | INSN_OP3(0x30) | INSN_RD(0))
+#define JMPL       (INSN_OP(2) | INSN_OP3(0x38))
+#define RETURN     (INSN_OP(2) | INSN_OP3(0x39))
+#define SAVE       (INSN_OP(2) | INSN_OP3(0x3c))
+#define RESTORE    (INSN_OP(2) | INSN_OP3(0x3d))
+#define SETHI      (INSN_OP(0) | INSN_OP2(0x4))
+#define CALL       INSN_OP(1)
+#define LDUB       (INSN_OP(3) | INSN_OP3(0x01))
+#define LDSB       (INSN_OP(3) | INSN_OP3(0x09))
+#define LDUH       (INSN_OP(3) | INSN_OP3(0x02))
+#define LDSH       (INSN_OP(3) | INSN_OP3(0x0a))
+#define LDUW       (INSN_OP(3) | INSN_OP3(0x00))
+#define LDSW       (INSN_OP(3) | INSN_OP3(0x08))
+#define LDX        (INSN_OP(3) | INSN_OP3(0x0b))
+#define STB        (INSN_OP(3) | INSN_OP3(0x05))
+#define STH        (INSN_OP(3) | INSN_OP3(0x06))
+#define STW        (INSN_OP(3) | INSN_OP3(0x04))
+#define STX        (INSN_OP(3) | INSN_OP3(0x0e))
+#define LDUBA      (INSN_OP(3) | INSN_OP3(0x11))
+#define LDSBA      (INSN_OP(3) | INSN_OP3(0x19))
+#define LDUHA      (INSN_OP(3) | INSN_OP3(0x12))
+#define LDSHA      (INSN_OP(3) | INSN_OP3(0x1a))
+#define LDUWA      (INSN_OP(3) | INSN_OP3(0x10))
+#define LDSWA      (INSN_OP(3) | INSN_OP3(0x18))
+#define LDXA       (INSN_OP(3) | INSN_OP3(0x1b))
+#define STBA       (INSN_OP(3) | INSN_OP3(0x15))
+#define STHA       (INSN_OP(3) | INSN_OP3(0x16))
+#define STWA       (INSN_OP(3) | INSN_OP3(0x14))
+#define STXA       (INSN_OP(3) | INSN_OP3(0x1e))
+
+#ifndef ASI_PRIMARY_LITTLE
+#define ASI_PRIMARY_LITTLE 0x88
+#endif
+
+#define LDUH_LE    (LDUHA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDSH_LE    (LDSHA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDUW_LE    (LDUWA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDSW_LE    (LDSWA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDX_LE     (LDXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+
+#define STH_LE     (STHA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define STW_LE     (STWA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define STX_LE     (STXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+
+#ifndef use_vis3_instructions
+bool use_vis3_instructions;
+#endif
+
+static inline int check_fit_i64(int64_t val, unsigned int bits)
+{
+    return val == sextract64(val, 0, bits);
+}
+
+static inline int check_fit_i32(int32_t val, unsigned int bits)
+{
+    return val == sextract32(val, 0, bits);
+}
+
+#define check_fit_tl    check_fit_i64
+#if SPARC64
+# define check_fit_ptr  check_fit_i64
+#else
+# define check_fit_ptr  check_fit_i32
+#endif
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    uint32_t insn;
+
+    tcg_debug_assert(addend == 0);
+    value = tcg_ptr_byte_diff((tcg_insn_unit *)value, code_ptr);
+
+    switch (type) {
+    case R_SPARC_WDISP16:
+        if (!check_fit_ptr(value >> 2, 16)) {
+            tcg_abort();
+        }
+        insn = *code_ptr;
+        insn &= ~INSN_OFF16(-1);
+        insn |= INSN_OFF16(value);
+        *code_ptr = insn;
+        break;
+    case R_SPARC_WDISP19:
+        if (!check_fit_ptr(value >> 2, 19)) {
+            tcg_abort();
+        }
+        insn = *code_ptr;
+        insn &= ~INSN_OFF19(-1);
+        insn |= INSN_OFF19(value);
+        *code_ptr = insn;
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch (ct_str[0]) {
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        break;
+    case 'R':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, ALL_64);
+        break;
+    case 'A': /* qemu_ld/st address constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0,
+                         TARGET_LONG_BITS == 64 ? ALL_64 : 0xffffffff);
+    reserve_helpers:
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O1);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O2);
+        break;
+    case 's': /* qemu_st data 32-bit constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        goto reserve_helpers;
+    case 'S': /* qemu_st data 64-bit constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, ALL_64);
+        goto reserve_helpers;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_S11;
+        break;
+    case 'J':
+        ct->ct |= TCG_CT_CONST_S13;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static inline int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    }
+
+    if (type == TCG_TYPE_I32) {
+        val = (int32_t)val;
+    }
+
+    if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11)) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+static inline void tcg_out_arith(TCGContext *s, TCGReg rd, TCGReg rs1,
+                                 TCGReg rs2, int op)
+{
+    tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1) | INSN_RS2(rs2));
+}
+
+static inline void tcg_out_arithi(TCGContext *s, TCGReg rd, TCGReg rs1,
+                                  int32_t offset, int op)
+{
+    tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1) | INSN_IMM13(offset));
+}
+
+static void tcg_out_arithc(TCGContext *s, TCGReg rd, TCGReg rs1,
+			   int32_t val2, int val2const, int op)
+{
+    tcg_out32(s, op | INSN_RD(rd) | INSN_RS1(rs1)
+              | (val2const ? INSN_IMM13(val2) : INSN_RS2(val2)));
+}
+
+static inline void tcg_out_mov(TCGContext *s, TCGType type,
+                               TCGReg ret, TCGReg arg)
+{
+    if (ret != arg) {
+        tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
+    }
+}
+
+static inline void tcg_out_sethi(TCGContext *s, TCGReg ret, uint32_t arg)
+{
+    tcg_out32(s, SETHI | INSN_RD(ret) | ((arg & 0xfffffc00) >> 10));
+}
+
+static inline void tcg_out_movi_imm13(TCGContext *s, TCGReg ret, int32_t arg)
+{
+    tcg_out_arithi(s, ret, TCG_REG_G0, arg, ARITH_OR);
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type,
+                         TCGReg ret, tcg_target_long arg)
+{
+    tcg_target_long hi, lo = (int32_t)arg;
+
+    /* Make sure we test 32-bit constants for imm13 properly.  */
+    if (type == TCG_TYPE_I32) {
+        arg = lo;
+    }
+
+    /* A 13-bit constant sign-extended to 64-bits.  */
+    if (check_fit_tl(arg, 13)) {
+        tcg_out_movi_imm13(s, ret, arg);
+        return;
+    }
+
+    /* A 32-bit constant, or 32-bit zero-extended to 64-bits.  */
+    if (type == TCG_TYPE_I32 || arg == (uint32_t)arg) {
+        tcg_out_sethi(s, ret, arg);
+        if (arg & 0x3ff) {
+            tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR);
+        }
+        return;
+    }
+
+    /* A 32-bit constant sign-extended to 64-bits.  */
+    if (arg == lo) {
+        tcg_out_sethi(s, ret, ~arg);
+        tcg_out_arithi(s, ret, ret, (arg & 0x3ff) | -0x400, ARITH_XOR);
+        return;
+    }
+
+    /* A 64-bit constant decomposed into 2 32-bit pieces.  */
+    if (check_fit_i32(lo, 13)) {
+        hi = (arg - lo) >> 32;
+        tcg_out_movi(s, TCG_TYPE_I32, ret, hi);
+        tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
+        tcg_out_arithi(s, ret, ret, lo, ARITH_ADD);
+    } else {
+        hi = arg >> 32;
+        tcg_out_movi(s, TCG_TYPE_I32, ret, hi);
+        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T2, lo);
+        tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
+        tcg_out_arith(s, ret, ret, TCG_REG_T2, ARITH_OR);
+    }
+}
+
+static inline void tcg_out_ldst_rr(TCGContext *s, TCGReg data, TCGReg a1,
+                                   TCGReg a2, int op)
+{
+    tcg_out32(s, op | INSN_RD(data) | INSN_RS1(a1) | INSN_RS2(a2));
+}
+
+static void tcg_out_ldst(TCGContext *s, TCGReg ret, TCGReg addr,
+                         intptr_t offset, int op)
+{
+    if (check_fit_ptr(offset, 13)) {
+        tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(addr) |
+                  INSN_IMM13(offset));
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, offset);
+        tcg_out_ldst_rr(s, ret, addr, TCG_REG_T1, op);
+    }
+}
+
+static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ldst(s, ret, arg1, arg2, (type == TCG_TYPE_I32 ? LDUW : LDX));
+}
+
+static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
+                              TCGReg arg1, intptr_t arg2)
+{
+    tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    if (val == 0) {
+        tcg_out_st(s, type, TCG_REG_G0, base, ofs);
+        return true;
+    }
+    return false;
+}
+
+static void tcg_out_ld_ptr(TCGContext *s, TCGReg ret, uintptr_t arg)
+{
+    tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff);
+    tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, arg & 0x3ff);
+}
+
+static inline void tcg_out_sety(TCGContext *s, TCGReg rs)
+{
+    tcg_out32(s, WRY | INSN_RS1(TCG_REG_G0) | INSN_RS2(rs));
+}
+
+static inline void tcg_out_rdy(TCGContext *s, TCGReg rd)
+{
+    tcg_out32(s, RDY | INSN_RD(rd));
+}
+
+static void tcg_out_div32(TCGContext *s, TCGReg rd, TCGReg rs1,
+                          int32_t val2, int val2const, int uns)
+{
+    /* Load Y with the sign/zero extension of RS1 to 64-bits.  */
+    if (uns) {
+        tcg_out_sety(s, TCG_REG_G0);
+    } else {
+        tcg_out_arithi(s, TCG_REG_T1, rs1, 31, SHIFT_SRA);
+        tcg_out_sety(s, TCG_REG_T1);
+    }
+
+    tcg_out_arithc(s, rd, rs1, val2, val2const,
+                   uns ? ARITH_UDIV : ARITH_SDIV);
+}
+
+static inline void tcg_out_nop(TCGContext *s)
+{
+    tcg_out_sethi(s, TCG_REG_G0, 0);
+}
+
+static const uint8_t tcg_cond_to_bcond[] = {
+    [TCG_COND_EQ] = COND_E,
+    [TCG_COND_NE] = COND_NE,
+    [TCG_COND_LT] = COND_L,
+    [TCG_COND_GE] = COND_GE,
+    [TCG_COND_LE] = COND_LE,
+    [TCG_COND_GT] = COND_G,
+    [TCG_COND_LTU] = COND_CS,
+    [TCG_COND_GEU] = COND_CC,
+    [TCG_COND_LEU] = COND_LEU,
+    [TCG_COND_GTU] = COND_GU,
+};
+
+static const uint8_t tcg_cond_to_rcond[] = {
+    [TCG_COND_EQ] = RCOND_Z,
+    [TCG_COND_NE] = RCOND_NZ,
+    [TCG_COND_LT] = RCOND_LZ,
+    [TCG_COND_GT] = RCOND_GZ,
+    [TCG_COND_LE] = RCOND_LEZ,
+    [TCG_COND_GE] = RCOND_GEZ
+};
+
+static void tcg_out_bpcc0(TCGContext *s, int scond, int flags, int off19)
+{
+    tcg_out32(s, INSN_OP(0) | INSN_OP2(1) | INSN_COND(scond) | flags | off19);
+}
+
+static void tcg_out_bpcc(TCGContext *s, int scond, int flags, TCGLabel *l)
+{
+    int off19;
+
+    if (l->has_value) {
+        off19 = INSN_OFF19(tcg_pcrel_diff(s, l->u.value_ptr));
+    } else {
+        /* Make sure to preserve destinations during retranslation.  */
+        off19 = *s->code_ptr & INSN_OFF19(-1);
+        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, l, 0);
+    }
+    tcg_out_bpcc0(s, scond, flags, off19);
+}
+
+static void tcg_out_cmp(TCGContext *s, TCGReg c1, int32_t c2, int c2const)
+{
+    tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const, ARITH_SUBCC);
+}
+
+static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGReg arg1,
+                               int32_t arg2, int const_arg2, TCGLabel *l)
+{
+    tcg_out_cmp(s, arg1, arg2, const_arg2);
+    tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_ICC | BPCC_PT, l);
+    tcg_out_nop(s);
+}
+
+static void tcg_out_movcc(TCGContext *s, TCGCond cond, int cc, TCGReg ret,
+                          int32_t v1, int v1const)
+{
+    tcg_out32(s, ARITH_MOVCC | cc | INSN_RD(ret)
+              | INSN_RS1(tcg_cond_to_bcond[cond])
+              | (v1const ? INSN_IMM11(v1) : INSN_RS2(v1)));
+}
+
+static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
+                                TCGReg c1, int32_t c2, int c2const,
+                                int32_t v1, int v1const)
+{
+    tcg_out_cmp(s, c1, c2, c2const);
+    tcg_out_movcc(s, cond, MOVCC_ICC, ret, v1, v1const);
+}
+
+static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1,
+                               int32_t arg2, int const_arg2, TCGLabel *l)
+{
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare.  */
+    if (arg2 == 0 && !is_unsigned_cond(cond)) {
+        int off16;
+
+        if (l->has_value) {
+            off16 = INSN_OFF16(tcg_pcrel_diff(s, l->u.value_ptr));
+        } else {
+            /* Make sure to preserve destinations during retranslation.  */
+            off16 = *s->code_ptr & INSN_OFF16(-1);
+            tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP16, l, 0);
+        }
+        tcg_out32(s, INSN_OP(0) | INSN_OP2(3) | BPR_PT | INSN_RS1(arg1)
+                  | INSN_COND(tcg_cond_to_rcond[cond]) | off16);
+    } else {
+        tcg_out_cmp(s, arg1, arg2, const_arg2);
+        tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_XCC | BPCC_PT, l);
+    }
+    tcg_out_nop(s);
+}
+
+static void tcg_out_movr(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg c1,
+                         int32_t v1, int v1const)
+{
+    tcg_out32(s, ARITH_MOVR | INSN_RD(ret) | INSN_RS1(c1)
+              | (tcg_cond_to_rcond[cond] << 10)
+              | (v1const ? INSN_IMM10(v1) : INSN_RS2(v1)));
+}
+
+static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
+                                TCGReg c1, int32_t c2, int c2const,
+                                int32_t v1, int v1const)
+{
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare.
+       Note that the immediate range is one bit smaller, so we must check
+       for that as well.  */
+    if (c2 == 0 && !is_unsigned_cond(cond)
+        && (!v1const || check_fit_i32(v1, 10))) {
+        tcg_out_movr(s, cond, ret, c1, v1, v1const);
+    } else {
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movcc(s, cond, MOVCC_XCC, ret, v1, v1const);
+    }
+}
+
+static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
+                                TCGReg c1, int32_t c2, int c2const)
+{
+    /* For 32-bit comparisons, we can play games with ADDC/SUBC.  */
+    switch (cond) {
+    case TCG_COND_LTU:
+    case TCG_COND_GEU:
+        /* The result of the comparison is in the carry bit.  */
+        break;
+
+    case TCG_COND_EQ:
+    case TCG_COND_NE:
+        /* For equality, we can transform to inequality vs zero.  */
+        if (c2 != 0) {
+            tcg_out_arithc(s, TCG_REG_T1, c1, c2, c2const, ARITH_XOR);
+            c2 = TCG_REG_T1;
+        } else {
+            c2 = c1;
+        }
+        c1 = TCG_REG_G0, c2const = 0;
+        cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU);
+	break;
+
+    case TCG_COND_GTU:
+    case TCG_COND_LEU:
+        /* If we don't need to load a constant into a register, we can
+           swap the operands on GTU/LEU.  There's no benefit to loading
+           the constant into a temporary register.  */
+        if (!c2const || c2 == 0) {
+            TCGReg t = c1;
+            c1 = c2;
+            c2 = t;
+            c2const = 0;
+            cond = tcg_swap_cond(cond);
+            break;
+        }
+        /* FALLTHRU */
+
+    default:
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movcc(s, cond, MOVCC_ICC, ret, 1, 1);
+        return;
+    }
+
+    tcg_out_cmp(s, c1, c2, c2const);
+    if (cond == TCG_COND_LTU) {
+        tcg_out_arithi(s, ret, TCG_REG_G0, 0, ARITH_ADDC);
+    } else {
+        tcg_out_arithi(s, ret, TCG_REG_G0, -1, ARITH_SUBC);
+    }
+}
+
+static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
+                                TCGReg c1, int32_t c2, int c2const)
+{
+    if (use_vis3_instructions) {
+        switch (cond) {
+        case TCG_COND_NE:
+            if (c2 != 0) {
+                break;
+            }
+            c2 = c1, c2const = 0, c1 = TCG_REG_G0;
+            /* FALLTHRU */
+        case TCG_COND_LTU:
+            tcg_out_cmp(s, c1, c2, c2const);
+            tcg_out_arith(s, ret, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC);
+            return;
+        default:
+            break;
+        }
+    }
+
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare
+       if the input does not overlap the output.  */
+    if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) {
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movr(s, cond, ret, c1, 1, 1);
+    } else {
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movcc(s, cond, MOVCC_XCC, ret, 1, 1);
+    }
+}
+
+static void tcg_out_addsub2_i32(TCGContext *s, TCGReg rl, TCGReg rh,
+                                TCGReg al, TCGReg ah, int32_t bl, int blconst,
+                                int32_t bh, int bhconst, int opl, int oph)
+{
+    TCGReg tmp = TCG_REG_T1;
+
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (rl != ah && (bhconst || rl != bh)) {
+        tmp = rl;
+    }
+
+    tcg_out_arithc(s, tmp, al, bl, blconst, opl);
+    tcg_out_arithc(s, rh, ah, bh, bhconst, oph);
+    tcg_out_mov(s, TCG_TYPE_I32, rl, tmp);
+}
+
+static void tcg_out_addsub2_i64(TCGContext *s, TCGReg rl, TCGReg rh,
+                                TCGReg al, TCGReg ah, int32_t bl, int blconst,
+                                int32_t bh, int bhconst, bool is_sub)
+{
+    TCGReg tmp = TCG_REG_T1;
+
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (rl != ah && (bhconst || rl != bh)) {
+        tmp = rl;
+    }
+
+    tcg_out_arithc(s, tmp, al, bl, blconst, is_sub ? ARITH_SUBCC : ARITH_ADDCC);
+
+    if (use_vis3_instructions && !is_sub) {
+        /* Note that ADDXC doesn't accept immediates.  */
+        if (bhconst && bh != 0) {
+           tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh);
+           bh = TCG_REG_T2;
+        }
+        tcg_out_arith(s, rh, ah, bh, ARITH_ADDXC);
+    } else if (bh == TCG_REG_G0) {
+	/* If we have a zero, we can perform the operation in two insns,
+           with the arithmetic first, and a conditional move into place.  */
+	if (rh == ah) {
+            tcg_out_arithi(s, TCG_REG_T2, ah, 1,
+			   is_sub ? ARITH_SUB : ARITH_ADD);
+            tcg_out_movcc(s, TCG_COND_LTU, MOVCC_XCC, rh, TCG_REG_T2, 0);
+	} else {
+            tcg_out_arithi(s, rh, ah, 1, is_sub ? ARITH_SUB : ARITH_ADD);
+	    tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, rh, ah, 0);
+	}
+    } else {
+        /* Otherwise adjust BH as if there is carry into T2 ... */
+        if (bhconst) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_T2, bh + (is_sub ? -1 : 1));
+        } else {
+            tcg_out_arithi(s, TCG_REG_T2, bh, 1,
+                           is_sub ? ARITH_SUB : ARITH_ADD);
+        }
+        /* ... smoosh T2 back to original BH if carry is clear ... */
+        tcg_out_movcc(s, TCG_COND_GEU, MOVCC_XCC, TCG_REG_T2, bh, bhconst);
+	/* ... and finally perform the arithmetic with the new operand.  */
+        tcg_out_arith(s, rh, ah, TCG_REG_T2, is_sub ? ARITH_SUB : ARITH_ADD);
+    }
+
+    tcg_out_mov(s, TCG_TYPE_I64, rl, tmp);
+}
+
+static void tcg_out_call_nodelay(TCGContext *s, tcg_insn_unit *dest)
+{
+    ptrdiff_t disp = tcg_pcrel_diff(s, dest);
+
+    if (disp == (int32_t)disp) {
+        tcg_out32(s, CALL | (uint32_t)disp >> 2);
+    } else {
+        uintptr_t desti = (uintptr_t)dest;
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, desti & ~0xfff);
+        tcg_out_arithi(s, TCG_REG_O7, TCG_REG_T1, desti & 0xfff, JMPL);
+    }
+}
+
+static void tcg_out_call(TCGContext *s, tcg_insn_unit *dest)
+{
+    tcg_out_call_nodelay(s, dest);
+    tcg_out_nop(s);
+}
+
+#ifdef CONFIG_SOFTMMU
+static tcg_insn_unit *qemu_ld_trampoline[16];
+static tcg_insn_unit *qemu_st_trampoline[16];
+
+static void build_trampolines(TCGContext *s)
+{
+    static void * const qemu_ld_helpers[16] = {
+        [MO_UB]   = helper_ret_ldub_mmu,
+        [MO_SB]   = helper_ret_ldsb_mmu,
+        [MO_LEUW] = helper_le_lduw_mmu,
+        [MO_LESW] = helper_le_ldsw_mmu,
+        [MO_LEUL] = helper_le_ldul_mmu,
+        [MO_LEQ]  = helper_le_ldq_mmu,
+        [MO_BEUW] = helper_be_lduw_mmu,
+        [MO_BESW] = helper_be_ldsw_mmu,
+        [MO_BEUL] = helper_be_ldul_mmu,
+        [MO_BEQ]  = helper_be_ldq_mmu,
+    };
+    static void * const qemu_st_helpers[16] = {
+        [MO_UB]   = helper_ret_stb_mmu,
+        [MO_LEUW] = helper_le_stw_mmu,
+        [MO_LEUL] = helper_le_stl_mmu,
+        [MO_LEQ]  = helper_le_stq_mmu,
+        [MO_BEUW] = helper_be_stw_mmu,
+        [MO_BEUL] = helper_be_stl_mmu,
+        [MO_BEQ]  = helper_be_stq_mmu,
+    };
+
+    int i;
+    TCGReg ra;
+
+    for (i = 0; i < 16; ++i) {
+        if (qemu_ld_helpers[i] == NULL) {
+            continue;
+        }
+
+        /* May as well align the trampoline.  */
+        while ((uintptr_t)s->code_ptr & 15) {
+            tcg_out_nop(s);
+        }
+        qemu_ld_trampoline[i] = s->code_ptr;
+
+        if (SPARC64 || TARGET_LONG_BITS == 32) {
+            ra = TCG_REG_O3;
+        } else {
+            /* Install the high part of the address.  */
+            tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O2, 32, SHIFT_SRLX);
+            ra = TCG_REG_O4;
+        }
+
+        /* Set the retaddr operand.  */
+        tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7);
+        /* Set the env operand.  */
+        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O0, TCG_AREG0);
+        /* Tail call.  */
+        tcg_out_call_nodelay(s, qemu_ld_helpers[i]);
+        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra);
+    }
+
+    for (i = 0; i < 16; ++i) {
+        if (qemu_st_helpers[i] == NULL) {
+            continue;
+        }
+
+        /* May as well align the trampoline.  */
+        while ((uintptr_t)s->code_ptr & 15) {
+            tcg_out_nop(s);
+        }
+        qemu_st_trampoline[i] = s->code_ptr;
+
+        if (SPARC64) {
+            ra = TCG_REG_O4;
+        } else {
+            ra = TCG_REG_O1;
+            if (TARGET_LONG_BITS == 64) {
+                /* Install the high part of the address.  */
+                tcg_out_arithi(s, ra, ra + 1, 32, SHIFT_SRLX);
+                ra += 2;
+            } else {
+                ra += 1;
+            }
+            if ((i & MO_SIZE) == MO_64) {
+                /* Install the high part of the data.  */
+                tcg_out_arithi(s, ra, ra + 1, 32, SHIFT_SRLX);
+                ra += 2;
+            } else {
+                ra += 1;
+            }
+            /* Skip the oi argument.  */
+            ra += 1;
+        }
+                
+        /* Set the retaddr operand.  */
+        if (ra >= TCG_REG_O6) {
+            tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_O7, TCG_REG_CALL_STACK,
+                       TCG_TARGET_CALL_STACK_OFFSET);
+            ra = TCG_REG_G1;
+        }
+        tcg_out_mov(s, TCG_TYPE_PTR, ra, TCG_REG_O7);
+        /* Set the env operand.  */
+        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O0, TCG_AREG0);
+        /* Tail call.  */
+        tcg_out_call_nodelay(s, qemu_st_helpers[i]);
+        tcg_out_mov(s, TCG_TYPE_PTR, TCG_REG_O7, ra);
+    }
+}
+#endif
+
+/* Generate global QEMU prologue and epilogue code */
+static void tcg_target_qemu_prologue(TCGContext *s)
+{
+    int tmp_buf_size, frame_size;
+
+    /* The TCG temp buffer is at the top of the frame, immediately
+       below the frame pointer.  */
+    tmp_buf_size = CPU_TEMP_BUF_NLONGS * (int)sizeof(long);
+    tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_STACK_BIAS - tmp_buf_size,
+                  tmp_buf_size);
+
+    /* TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is
+       otherwise the minimal frame usable by callees.  */
+    frame_size = TCG_TARGET_CALL_STACK_OFFSET - TCG_TARGET_STACK_BIAS;
+    frame_size += TCG_STATIC_CALL_ARGS_SIZE + tmp_buf_size;
+    frame_size += TCG_TARGET_STACK_ALIGN - 1;
+    frame_size &= -TCG_TARGET_STACK_ALIGN;
+    tcg_out32(s, SAVE | INSN_RD(TCG_REG_O6) | INSN_RS1(TCG_REG_O6) |
+              INSN_IMM13(-frame_size));
+
+#ifndef CONFIG_SOFTMMU
+    if (guest_base != 0) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
+    }
+#endif
+
+    tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL);
+    /* delay slot */
+    tcg_out_nop(s);
+
+    /* No epilogue required.  We issue ret + restore directly in the TB.  */
+
+#ifdef CONFIG_SOFTMMU
+    build_trampolines(s);
+#endif
+}
+
+#if defined(CONFIG_SOFTMMU)
+/* Perform the TLB load and compare.
+
+   Inputs:
+   ADDRLO and ADDRHI contain the possible two parts of the address.
+
+   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
+
+   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
+   This should be offsetof addr_read or addr_write.
+
+   The result of the TLB comparison is in %[ix]cc.  The sanitized address
+   is in the returned register, maybe %o0.  The TLB addend is in %o1.  */
+
+static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index,
+                               TCGMemOp s_bits, int which)
+{
+    const TCGReg r0 = TCG_REG_O0;
+    const TCGReg r1 = TCG_REG_O1;
+    const TCGReg r2 = TCG_REG_O2;
+    int tlb_ofs;
+
+    /* Shift the page number down.  */
+    tcg_out_arithi(s, r1, addr, TARGET_PAGE_BITS, SHIFT_SRL);
+
+    /* Mask out the page offset, except for the required alignment.  */
+    tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_T1,
+                 TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+
+    /* Mask the tlb index.  */
+    tcg_out_arithi(s, r1, r1, CPU_TLB_SIZE - 1, ARITH_AND);
+    
+    /* Mask page, part 2.  */
+    tcg_out_arith(s, r0, addr, TCG_REG_T1, ARITH_AND);
+
+    /* Shift the tlb index into place.  */
+    tcg_out_arithi(s, r1, r1, CPU_TLB_ENTRY_BITS, SHIFT_SLL);
+
+    /* Relative to the current ENV.  */
+    tcg_out_arith(s, r1, TCG_AREG0, r1, ARITH_ADD);
+
+    /* Find a base address that can load both tlb comparator and addend.  */
+    tlb_ofs = offsetof(CPUArchState, tlb_table[mem_index][0]);
+    if (!check_fit_ptr(tlb_ofs + sizeof(CPUTLBEntry), 13)) {
+        if (tlb_ofs & ~0x3ff) {
+            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, tlb_ofs & ~0x3ff);
+            tcg_out_arith(s, r1, r1, TCG_REG_T1, ARITH_ADD);
+        }
+        tlb_ofs &= 0x3ff;
+    }
+
+    /* Load the tlb comparator and the addend.  */
+    tcg_out_ld(s, TCG_TYPE_TL, r2, r1, tlb_ofs + which);
+    tcg_out_ld(s, TCG_TYPE_PTR, r1, r1, tlb_ofs+offsetof(CPUTLBEntry, addend));
+
+    /* subcc arg0, arg2, %g0 */
+    tcg_out_cmp(s, r0, r2, 0);
+
+    /* If the guest address must be zero-extended, do so now.  */
+    if (SPARC64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, r0, addr, 0, SHIFT_SRL);
+        return r0;
+    }
+    return addr;
+}
+#endif /* CONFIG_SOFTMMU */
+
+static const int qemu_ld_opc[16] = {
+    [MO_UB]   = LDUB,
+    [MO_SB]   = LDSB,
+
+    [MO_BEUW] = LDUH,
+    [MO_BESW] = LDSH,
+    [MO_BEUL] = LDUW,
+    [MO_BESL] = LDSW,
+    [MO_BEQ]  = LDX,
+
+    [MO_LEUW] = LDUH_LE,
+    [MO_LESW] = LDSH_LE,
+    [MO_LEUL] = LDUW_LE,
+    [MO_LESL] = LDSW_LE,
+    [MO_LEQ]  = LDX_LE,
+};
+
+static const int qemu_st_opc[16] = {
+    [MO_UB]   = STB,
+
+    [MO_BEUW] = STH,
+    [MO_BEUL] = STW,
+    [MO_BEQ]  = STX,
+
+    [MO_LEUW] = STH_LE,
+    [MO_LEUL] = STW_LE,
+    [MO_LEQ]  = STX_LE,
+};
+
+static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr,
+                            TCGMemOpIdx oi, bool is_64)
+{
+    TCGMemOp memop = get_memop(oi);
+#ifdef CONFIG_SOFTMMU
+    unsigned memi = get_mmuidx(oi);
+    TCGReg addrz, param;
+    tcg_insn_unit *func;
+    tcg_insn_unit *label_ptr;
+
+    addrz = tcg_out_tlb_load(s, addr, memi, memop & MO_SIZE,
+                             offsetof(CPUTLBEntry, addr_read));
+
+    /* The fast path is exactly one insn.  Thus we can perform the
+       entire TLB Hit in the (annulled) delay slot of the branch
+       over the TLB Miss case.  */
+
+    /* beq,a,pt %[xi]cc, label0 */
+    label_ptr = s->code_ptr;
+    tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
+                  | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+    /* delay slot */
+    tcg_out_ldst_rr(s, data, addrz, TCG_REG_O1,
+                    qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]);
+
+    /* TLB Miss.  */
+
+    param = TCG_REG_O1;
+    if (!SPARC64 && TARGET_LONG_BITS == 64) {
+        /* Skip the high-part; we'll perform the extract in the trampoline.  */
+        param++;
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, param++, addr);
+
+    /* We use the helpers to extend SB and SW data, leaving the case
+       of SL needing explicit extending below.  */
+    if ((memop & MO_SSIZE) == MO_SL) {
+        func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SIZE)];
+    } else {
+        func = qemu_ld_trampoline[memop & (MO_BSWAP | MO_SSIZE)];
+    }
+    tcg_debug_assert(func != NULL);
+    tcg_out_call_nodelay(s, func);
+    /* delay slot */
+    tcg_out_movi(s, TCG_TYPE_I32, param, oi);
+
+    /* Recall that all of the helpers return 64-bit results.
+       Which complicates things for sparcv8plus.  */
+    if (SPARC64) {
+        /* We let the helper sign-extend SB and SW, but leave SL for here.  */
+        if (is_64 && (memop & MO_SSIZE) == MO_SL) {
+            tcg_out_arithi(s, data, TCG_REG_O0, 0, SHIFT_SRA);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_REG, data, TCG_REG_O0);
+        }
+    } else {
+        if ((memop & MO_SIZE) == MO_64) {
+            tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, 32, SHIFT_SLLX);
+            tcg_out_arithi(s, TCG_REG_O1, TCG_REG_O1, 0, SHIFT_SRL);
+            tcg_out_arith(s, data, TCG_REG_O0, TCG_REG_O1, ARITH_OR);
+        } else if (is_64) {
+            /* Re-extend from 32-bit rather than reassembling when we
+               know the high register must be an extension.  */
+            tcg_out_arithi(s, data, TCG_REG_O1, 0,
+                           memop & MO_SIGN ? SHIFT_SRA : SHIFT_SRL);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, data, TCG_REG_O1);
+        }
+    }
+
+    *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr));
+#else
+    if (SPARC64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
+        addr = TCG_REG_T1;
+    }
+    tcg_out_ldst_rr(s, data, addr,
+                    (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    qemu_ld_opc[memop & (MO_BSWAP | MO_SSIZE)]);
+#endif /* CONFIG_SOFTMMU */
+}
+
+static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr,
+                            TCGMemOpIdx oi)
+{
+    TCGMemOp memop = get_memop(oi);
+#ifdef CONFIG_SOFTMMU
+    unsigned memi = get_mmuidx(oi);
+    TCGReg addrz, param;
+    tcg_insn_unit *func;
+    tcg_insn_unit *label_ptr;
+
+    addrz = tcg_out_tlb_load(s, addr, memi, memop & MO_SIZE,
+                             offsetof(CPUTLBEntry, addr_write));
+
+    /* The fast path is exactly one insn.  Thus we can perform the entire
+       TLB Hit in the (annulled) delay slot of the branch over TLB Miss.  */
+    /* beq,a,pt %[xi]cc, label0 */
+    label_ptr = s->code_ptr;
+    tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
+                  | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+    /* delay slot */
+    tcg_out_ldst_rr(s, data, addrz, TCG_REG_O1,
+                    qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]);
+
+    /* TLB Miss.  */
+
+    param = TCG_REG_O1;
+    if (!SPARC64 && TARGET_LONG_BITS == 64) {
+        /* Skip the high-part; we'll perform the extract in the trampoline.  */
+        param++;
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, param++, addr);
+    if (!SPARC64 && (memop & MO_SIZE) == MO_64) {
+        /* Skip the high-part; we'll perform the extract in the trampoline.  */
+        param++;
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, param++, data);
+
+    func = qemu_st_trampoline[memop & (MO_BSWAP | MO_SIZE)];
+    tcg_debug_assert(func != NULL);
+    tcg_out_call_nodelay(s, func);
+    /* delay slot */
+    tcg_out_movi(s, TCG_TYPE_I32, param, oi);
+
+    *label_ptr |= INSN_OFF19(tcg_ptr_byte_diff(s->code_ptr, label_ptr));
+#else
+    if (SPARC64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, TCG_REG_T1, addr, 0, SHIFT_SRL);
+        addr = TCG_REG_T1;
+    }
+    tcg_out_ldst_rr(s, data, addr,
+                    (guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    qemu_st_opc[memop & (MO_BSWAP | MO_SIZE)]);
+#endif /* CONFIG_SOFTMMU */
+}
+
+static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+                       const TCGArg args[TCG_MAX_OP_ARGS],
+                       const int const_args[TCG_MAX_OP_ARGS])
+{
+    TCGArg a0, a1, a2;
+    int c, c2;
+
+    /* Hoist the loads of the most common arguments.  */
+    a0 = args[0];
+    a1 = args[1];
+    a2 = args[2];
+    c2 = const_args[2];
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        if (check_fit_ptr(a0, 13)) {
+            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
+            tcg_out_movi_imm13(s, TCG_REG_O0, a0);
+        } else {
+            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
+            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
+            tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
+        }
+        break;
+    case INDEX_op_goto_tb:
+        if (s->tb_jmp_insn_offset) {
+            /* direct jump method */
+            s->tb_jmp_insn_offset[a0] = tcg_current_code_size(s);
+            /* Make sure to preserve links during retranslation.  */
+            tcg_out32(s, CALL | (*s->code_ptr & ~INSN_OP(-1)));
+        } else {
+            /* indirect jump method */
+            tcg_out_ld_ptr(s, TCG_REG_T1,
+                           (uintptr_t)(s->tb_jmp_target_addr + a0));
+            tcg_out_arithi(s, TCG_REG_G0, TCG_REG_T1, 0, JMPL);
+        }
+        tcg_out_nop(s);
+        s->tb_jmp_reset_offset[a0] = tcg_current_code_size(s);
+        break;
+    case INDEX_op_br:
+        tcg_out_bpcc(s, COND_A, BPCC_PT, arg_label(a0));
+        tcg_out_nop(s);
+        break;
+
+#define OP_32_64(x)                             \
+        glue(glue(case INDEX_op_, x), _i32):    \
+        glue(glue(case INDEX_op_, x), _i64)
+
+    OP_32_64(ld8u):
+        tcg_out_ldst(s, a0, a1, a2, LDUB);
+        break;
+    OP_32_64(ld8s):
+        tcg_out_ldst(s, a0, a1, a2, LDSB);
+        break;
+    OP_32_64(ld16u):
+        tcg_out_ldst(s, a0, a1, a2, LDUH);
+        break;
+    OP_32_64(ld16s):
+        tcg_out_ldst(s, a0, a1, a2, LDSH);
+        break;
+    case INDEX_op_ld_i32:
+    case INDEX_op_ld32u_i64:
+        tcg_out_ldst(s, a0, a1, a2, LDUW);
+        break;
+    OP_32_64(st8):
+        tcg_out_ldst(s, a0, a1, a2, STB);
+        break;
+    OP_32_64(st16):
+        tcg_out_ldst(s, a0, a1, a2, STH);
+        break;
+    case INDEX_op_st_i32:
+    case INDEX_op_st32_i64:
+        tcg_out_ldst(s, a0, a1, a2, STW);
+        break;
+    OP_32_64(add):
+        c = ARITH_ADD;
+        goto gen_arith;
+    OP_32_64(sub):
+        c = ARITH_SUB;
+        goto gen_arith;
+    OP_32_64(and):
+        c = ARITH_AND;
+        goto gen_arith;
+    OP_32_64(andc):
+        c = ARITH_ANDN;
+        goto gen_arith;
+    OP_32_64(or):
+        c = ARITH_OR;
+        goto gen_arith;
+    OP_32_64(orc):
+        c = ARITH_ORN;
+        goto gen_arith;
+    OP_32_64(xor):
+        c = ARITH_XOR;
+        goto gen_arith;
+    case INDEX_op_shl_i32:
+        c = SHIFT_SLL;
+    do_shift32:
+        /* Limit immediate shift count lest we create an illegal insn.  */
+        tcg_out_arithc(s, a0, a1, a2 & 31, c2, c);
+        break;
+    case INDEX_op_shr_i32:
+        c = SHIFT_SRL;
+        goto do_shift32;
+    case INDEX_op_sar_i32:
+        c = SHIFT_SRA;
+        goto do_shift32;
+    case INDEX_op_mul_i32:
+        c = ARITH_UMUL;
+        goto gen_arith;
+
+    OP_32_64(neg):
+	c = ARITH_SUB;
+	goto gen_arith1;
+    OP_32_64(not):
+	c = ARITH_ORN;
+	goto gen_arith1;
+
+    case INDEX_op_div_i32:
+        tcg_out_div32(s, a0, a1, a2, c2, 0);
+        break;
+    case INDEX_op_divu_i32:
+        tcg_out_div32(s, a0, a1, a2, c2, 1);
+        break;
+
+    case INDEX_op_brcond_i32:
+        tcg_out_brcond_i32(s, a2, a0, a1, const_args[1], arg_label(args[3]));
+        break;
+    case INDEX_op_setcond_i32:
+        tcg_out_setcond_i32(s, args[3], a0, a1, a2, c2);
+        break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond_i32(s, args[5], a0, a1, a2, c2, args[3], const_args[3]);
+        break;
+
+    case INDEX_op_add2_i32:
+        tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3],
+                            args[4], const_args[4], args[5], const_args[5],
+                            ARITH_ADDCC, ARITH_ADDC);
+        break;
+    case INDEX_op_sub2_i32:
+        tcg_out_addsub2_i32(s, args[0], args[1], args[2], args[3],
+                            args[4], const_args[4], args[5], const_args[5],
+                            ARITH_SUBCC, ARITH_SUBC);
+        break;
+    case INDEX_op_mulu2_i32:
+        c = ARITH_UMUL;
+        goto do_mul2;
+    case INDEX_op_muls2_i32:
+        c = ARITH_SMUL;
+    do_mul2:
+        /* The 32-bit multiply insns produce a full 64-bit result.  If the
+           destination register can hold it, we can avoid the slower RDY.  */
+        tcg_out_arithc(s, a0, a2, args[3], const_args[3], c);
+        if (SPARC64 || a0 <= TCG_REG_O7) {
+            tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX);
+        } else {
+            tcg_out_rdy(s, a1);
+        }
+        break;
+
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, a0, a1, a2, false);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, a0, a1, a2, true);
+        break;
+    case INDEX_op_qemu_st_i32:
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, a0, a1, a2);
+        break;
+
+    case INDEX_op_ld32s_i64:
+        tcg_out_ldst(s, a0, a1, a2, LDSW);
+        break;
+    case INDEX_op_ld_i64:
+        tcg_out_ldst(s, a0, a1, a2, LDX);
+        break;
+    case INDEX_op_st_i64:
+        tcg_out_ldst(s, a0, a1, a2, STX);
+        break;
+    case INDEX_op_shl_i64:
+        c = SHIFT_SLLX;
+    do_shift64:
+        /* Limit immediate shift count lest we create an illegal insn.  */
+        tcg_out_arithc(s, a0, a1, a2 & 63, c2, c);
+        break;
+    case INDEX_op_shr_i64:
+        c = SHIFT_SRLX;
+        goto do_shift64;
+    case INDEX_op_sar_i64:
+        c = SHIFT_SRAX;
+        goto do_shift64;
+    case INDEX_op_mul_i64:
+        c = ARITH_MULX;
+        goto gen_arith;
+    case INDEX_op_div_i64:
+        c = ARITH_SDIVX;
+        goto gen_arith;
+    case INDEX_op_divu_i64:
+        c = ARITH_UDIVX;
+        goto gen_arith;
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_ext32s_i64:
+        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRA);
+        break;
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_ext32u_i64:
+        tcg_out_arithi(s, a0, a1, 0, SHIFT_SRL);
+        break;
+    case INDEX_op_extrl_i64_i32:
+        tcg_out_mov(s, TCG_TYPE_I32, a0, a1);
+        break;
+    case INDEX_op_extrh_i64_i32:
+        tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX);
+        break;
+
+    case INDEX_op_brcond_i64:
+        tcg_out_brcond_i64(s, a2, a0, a1, const_args[1], arg_label(args[3]));
+        break;
+    case INDEX_op_setcond_i64:
+        tcg_out_setcond_i64(s, args[3], a0, a1, a2, c2);
+        break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond_i64(s, args[5], a0, a1, a2, c2, args[3], const_args[3]);
+        break;
+    case INDEX_op_add2_i64:
+        tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4],
+                            const_args[4], args[5], const_args[5], false);
+        break;
+    case INDEX_op_sub2_i64:
+        tcg_out_addsub2_i64(s, args[0], args[1], args[2], args[3], args[4],
+                            const_args[4], args[5], const_args[5], true);
+        break;
+    case INDEX_op_muluh_i64:
+        tcg_out_arith(s, args[0], args[1], args[2], ARITH_UMULXHI);
+        break;
+
+    gen_arith:
+        tcg_out_arithc(s, a0, a1, a2, c2, c);
+        break;
+
+    gen_arith1:
+	tcg_out_arithc(s, a0, TCG_REG_G0, a1, const_args[1], c);
+	break;
+
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+}
+
+static const TCGTargetOpDef sparc_op_defs[] = {
+    { INDEX_op_exit_tb, { } },
+    { INDEX_op_goto_tb, { } },
+    { INDEX_op_br, { } },
+
+    { INDEX_op_ld8u_i32, { "r", "r" } },
+    { INDEX_op_ld8s_i32, { "r", "r" } },
+    { INDEX_op_ld16u_i32, { "r", "r" } },
+    { INDEX_op_ld16s_i32, { "r", "r" } },
+    { INDEX_op_ld_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "rZ", "r" } },
+    { INDEX_op_st16_i32, { "rZ", "r" } },
+    { INDEX_op_st_i32, { "rZ", "r" } },
+
+    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_mul_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_div_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_divu_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_and_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_andc_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_or_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_orc_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_xor_i32, { "r", "rZ", "rJ" } },
+
+    { INDEX_op_shl_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sar_i32, { "r", "rZ", "rJ" } },
+
+    { INDEX_op_neg_i32, { "r", "rJ" } },
+    { INDEX_op_not_i32, { "r", "rJ" } },
+
+    { INDEX_op_brcond_i32, { "rZ", "rJ" } },
+    { INDEX_op_setcond_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rJ", "rI", "0" } },
+
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rJ" } },
+    { INDEX_op_muls2_i32, { "r", "r", "rZ", "rJ" } },
+
+    { INDEX_op_ld8u_i64, { "R", "r" } },
+    { INDEX_op_ld8s_i64, { "R", "r" } },
+    { INDEX_op_ld16u_i64, { "R", "r" } },
+    { INDEX_op_ld16s_i64, { "R", "r" } },
+    { INDEX_op_ld32u_i64, { "R", "r" } },
+    { INDEX_op_ld32s_i64, { "R", "r" } },
+    { INDEX_op_ld_i64, { "R", "r" } },
+    { INDEX_op_st8_i64, { "RZ", "r" } },
+    { INDEX_op_st16_i64, { "RZ", "r" } },
+    { INDEX_op_st32_i64, { "RZ", "r" } },
+    { INDEX_op_st_i64, { "RZ", "r" } },
+
+    { INDEX_op_add_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_mul_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_div_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_divu_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_sub_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_and_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_andc_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_or_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_orc_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_xor_i64, { "R", "RZ", "RJ" } },
+
+    { INDEX_op_shl_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_shr_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_sar_i64, { "R", "RZ", "RJ" } },
+
+    { INDEX_op_neg_i64, { "R", "RJ" } },
+    { INDEX_op_not_i64, { "R", "RJ" } },
+
+    { INDEX_op_ext32s_i64, { "R", "R" } },
+    { INDEX_op_ext32u_i64, { "R", "R" } },
+    { INDEX_op_ext_i32_i64, { "R", "r" } },
+    { INDEX_op_extu_i32_i64, { "R", "r" } },
+    { INDEX_op_extrl_i64_i32,  { "r", "R" } },
+    { INDEX_op_extrh_i64_i32,  { "r", "R" } },
+
+    { INDEX_op_brcond_i64, { "RZ", "RJ" } },
+    { INDEX_op_setcond_i64, { "R", "RZ", "RJ" } },
+    { INDEX_op_movcond_i64, { "R", "RZ", "RJ", "RI", "0" } },
+
+    { INDEX_op_add2_i64, { "R", "R", "RZ", "RZ", "RJ", "RI" } },
+    { INDEX_op_sub2_i64, { "R", "R", "RZ", "RZ", "RJ", "RI" } },
+    { INDEX_op_muluh_i64, { "R", "RZ", "RZ" } },
+
+    { INDEX_op_qemu_ld_i32, { "r", "A" } },
+    { INDEX_op_qemu_ld_i64, { "R", "A" } },
+    { INDEX_op_qemu_st_i32, { "sZ", "A" } },
+    { INDEX_op_qemu_st_i64, { "SZ", "A" } },
+
+    { -1 },
+};
+
+static void tcg_target_init(TCGContext *s)
+{
+    /* Only probe for the platform and capabilities if we havn't already
+       determined maximum values at compile time.  */
+#ifndef use_vis3_instructions
+    {
+        unsigned long hwcap = qemu_getauxval(AT_HWCAP);
+        use_vis3_instructions = (hwcap & HWCAP_SPARC_VIS3) != 0;
+    }
+#endif
+
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, ALL_64);
+
+    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
+                     (1 << TCG_REG_G1) |
+                     (1 << TCG_REG_G2) |
+                     (1 << TCG_REG_G3) |
+                     (1 << TCG_REG_G4) |
+                     (1 << TCG_REG_G5) |
+                     (1 << TCG_REG_G6) |
+                     (1 << TCG_REG_G7) |
+                     (1 << TCG_REG_O0) |
+                     (1 << TCG_REG_O1) |
+                     (1 << TCG_REG_O2) |
+                     (1 << TCG_REG_O3) |
+                     (1 << TCG_REG_O4) |
+                     (1 << TCG_REG_O5) |
+                     (1 << TCG_REG_O7));
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0); /* zero */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G6); /* reserved for os */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G7); /* thread pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6); /* frame pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7); /* return address */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6); /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T1); /* for internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */
+
+    tcg_add_target_add_op_defs(sparc_op_defs);
+}
+
+#if SPARC64
+# define ELF_HOST_MACHINE  EM_SPARCV9
+#else
+# define ELF_HOST_MACHINE  EM_SPARC32PLUS
+# define ELF_HOST_FLAGS    EF_SPARC_32PLUS
+#endif
+
+typedef struct {
+    DebugFrameHeader h;
+    uint8_t fde_def_cfa[SPARC64 ? 4 : 2];
+    uint8_t fde_win_save;
+    uint8_t fde_ret_save[3];
+} DebugFrame;
+
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = -sizeof(void *) & 0x7f,
+    .h.cie.return_column = 15,            /* o7 */
+
+    /* Total FDE size does not include the "len" member.  */
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
+
+    .fde_def_cfa = {
+#if SPARC64
+        12, 30,                         /* DW_CFA_def_cfa i6, 2047 */
+        (2047 & 0x7f) | 0x80, (2047 >> 7)
+#else
+        13, 30                          /* DW_CFA_def_cfa_register i6 */
+#endif
+    },
+    .fde_win_save = 0x2d,               /* DW_CFA_GNU_window_save */
+    .fde_ret_save = { 9, 15, 31 },      /* DW_CFA_register o7, i7 */
+};
+
+void tcg_register_jit(void *buf, size_t buf_size)
+{
+    tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
+}
+
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
+{
+    uint32_t *ptr = (uint32_t *)jmp_addr;
+    uintptr_t disp = addr - jmp_addr;
+
+    /* We can reach the entire address space for 32-bit.  For 64-bit
+       the code_gen_buffer can't be larger than 2GB.  */
+    tcg_debug_assert(disp == (int32_t)disp);
+
+    atomic_set(ptr, deposit32(CALL, 0, 30, disp >> 2));
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
diff --git a/tcg/tcg-be-ldst.h b/tcg/tcg-be-ldst.h
index cafe8ed..17777ae 100644
--- a/tcg/tcg-be-ldst.h
+++ b/tcg/tcg-be-ldst.h
@@ -21,7 +21,6 @@
  */
 
 #ifdef CONFIG_SOFTMMU
-#define TCG_MAX_QEMU_LDST       640
 
 typedef struct TCGLabelQemuLdst {
     bool is_ld;             /* qemu_ld: true, qemu_st: false */
@@ -33,11 +32,11 @@
     TCGReg datahi_reg;      /* reg index for high word to be loaded or stored */
     tcg_insn_unit *raddr;   /* gen code addr of the next IR of qemu_ld/st IR */
     tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
+    struct TCGLabelQemuLdst *next;
 } TCGLabelQemuLdst;
 
 typedef struct TCGBackendData {
-    int nb_ldst_labels;
-    TCGLabelQemuLdst ldst_labels[TCG_MAX_QEMU_LDST];
+    TCGLabelQemuLdst *labels;
 } TCGBackendData;
 
 
@@ -47,7 +46,7 @@
 
 static inline void tcg_out_tb_init(TCGContext *s)
 {
-    s->be->nb_ldst_labels = 0;
+    s->be->labels = NULL;
 }
 
 /*
@@ -57,19 +56,27 @@
 static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
 static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
 
-static void tcg_out_tb_finalize(TCGContext *s)
+static bool tcg_out_tb_finalize(TCGContext *s)
 {
-    TCGLabelQemuLdst *lb = s->be->ldst_labels;
-    int i, n = s->be->nb_ldst_labels;
+    TCGLabelQemuLdst *lb;
 
     /* qemu_ld/st slow paths */
-    for (i = 0; i < n; i++) {
-        if (lb[i].is_ld) {
-            tcg_out_qemu_ld_slow_path(s, lb + i);
+    for (lb = s->be->labels; lb != NULL; lb = lb->next) {
+        if (lb->is_ld) {
+            tcg_out_qemu_ld_slow_path(s, lb);
         } else {
-            tcg_out_qemu_st_slow_path(s, lb + i);
+            tcg_out_qemu_st_slow_path(s, lb);
+        }
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return false;
         }
     }
+    return true;
 }
 
 /*
@@ -79,11 +86,11 @@
 static inline TCGLabelQemuLdst *new_ldst_label(TCGContext *s)
 {
     TCGBackendData *be = s->be;
-    int n = be->nb_ldst_labels;
+    TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l));
 
-    assert(n < TCG_MAX_QEMU_LDST);
-    be->nb_ldst_labels = n + 1;
-    return &be->ldst_labels[n];
+    l->next = be->labels;
+    be->labels = l;
+    return l;
 }
 #else
 #include "tcg-be-null.h"
diff --git a/tcg/tcg-be-null.h b/tcg/tcg-be-null.h
index 74c57d5..5222fe2 100644
--- a/tcg/tcg-be-null.h
+++ b/tcg/tcg-be-null.h
@@ -38,6 +38,7 @@
  * Generate TB finalization at the end of block
  */
 
-static inline void tcg_out_tb_finalize(TCGContext *s)
+static inline bool tcg_out_tb_finalize(TCGContext *s)
 {
+    return true;
 }
diff --git a/tcg/tcg-common.c b/tcg/tcg-common.c
new file mode 100644
index 0000000..2f139de
--- /dev/null
+++ b/tcg/tcg-common.c
@@ -0,0 +1,40 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2008 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "exec/cpu-common.h"
+#include "tcg/tcg.h"
+
+#if defined(CONFIG_TCG_INTERPRETER)
+uintptr_t tci_tb_ptr;
+#endif
+
+TCGOpDef tcg_op_defs[] = {
+#define DEF(s, oargs, iargs, cargs, flags) \
+         { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
+#include "tcg-opc.h"
+#undef DEF
+};
+const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c
index 7d3de33..0243c99 100644
--- a/tcg/tcg-op.c
+++ b/tcg/tcg-op.c
@@ -22,8 +22,14 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "cpu.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
 #include "tcg-op.h"
+#include "trace-tcg.h"
+#include "trace/mem.h"
 
 /* Reduce the number of ifdefs below.  This assumes that all uses of
    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
@@ -46,7 +52,7 @@
     int pi = oi - 1;
 
     tcg_debug_assert(oi < OPC_BUF_SIZE);
-    ctx->gen_last_op_idx = oi;
+    ctx->gen_op_buf[0].prev = oi;
     ctx->gen_next_op_idx = ni;
 
     ctx->gen_op_buf[oi] = (TCGOp){
@@ -275,20 +281,24 @@
     }
 }
 
-void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, int label)
+void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
 {
     if (cond == TCG_COND_ALWAYS) {
-        tcg_gen_br(label);
+        tcg_gen_br(l);
     } else if (cond != TCG_COND_NEVER) {
-        tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label);
+        tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
     }
 }
 
-void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, int label)
+void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
 {
-    TCGv_i32 t0 = tcg_const_i32(arg2);
-    tcg_gen_brcond_i32(cond, arg1, t0, label);
-    tcg_temp_free_i32(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(l);
+    } else if (cond != TCG_COND_NEVER) {
+        TCGv_i32 t0 = tcg_const_i32(arg2);
+        tcg_gen_brcond_i32(cond, arg1, t0, l);
+        tcg_temp_free_i32(t0);
+    }
 }
 
 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
@@ -546,7 +556,11 @@
 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
 {
-    if (TCG_TARGET_HAS_movcond_i32) {
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_mov_i32(ret, v1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_mov_i32(ret, v2);
+    } else if (TCG_TARGET_HAS_movcond_i32) {
         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
     } else {
         TCGv_i32 t0 = tcg_temp_new_i32();
@@ -800,7 +814,7 @@
 {
     /* Since arg2 and ret have different types,
        they cannot be the same temporary */
-#ifdef TCG_TARGET_WORDS_BIGENDIAN
+#ifdef HOST_WORDS_BIGENDIAN
     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
 #else
@@ -811,7 +825,7 @@
 
 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
 {
-#ifdef TCG_TARGET_WORDS_BIGENDIAN
+#ifdef HOST_WORDS_BIGENDIAN
     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
 #else
@@ -1084,28 +1098,29 @@
     }
 }
 
-void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, int label)
+void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
 {
     if (cond == TCG_COND_ALWAYS) {
-        tcg_gen_br(label);
+        tcg_gen_br(l);
     } else if (cond != TCG_COND_NEVER) {
         if (TCG_TARGET_REG_BITS == 32) {
             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
-                              TCGV_HIGH(arg2), cond, label);
+                              TCGV_HIGH(arg2), cond, label_arg(l));
         } else {
-            tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label);
+            tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
+                              label_arg(l));
         }
     }
 }
 
-void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, int label)
+void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
 {
     if (cond == TCG_COND_ALWAYS) {
-        tcg_gen_br(label);
+        tcg_gen_br(l);
     } else if (cond != TCG_COND_NEVER) {
         TCGv_i64 t0 = tcg_const_i64(arg2);
-        tcg_gen_brcond_i64(cond, arg1, t0, label);
+        tcg_gen_brcond_i64(cond, arg1, t0, l);
         tcg_temp_free_i64(t0);
     }
 }
@@ -1589,7 +1604,11 @@
 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
 {
-    if (TCG_TARGET_REG_BITS == 32) {
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_mov_i64(ret, v1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_mov_i64(ret, v2);
+    } else if (TCG_TARGET_REG_BITS == 32) {
         TCGv_i32 t0 = tcg_temp_new_i32();
         TCGv_i32 t1 = tcg_temp_new_i32();
         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
@@ -1724,28 +1743,28 @@
 
 /* Size changing operations.  */
 
-void tcg_gen_trunc_shr_i64_i32(TCGv_i32 ret, TCGv_i64 arg, unsigned count)
+void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
 {
-    tcg_debug_assert(count < 64);
     if (TCG_TARGET_REG_BITS == 32) {
-        if (count >= 32) {
-            tcg_gen_shri_i32(ret, TCGV_HIGH(arg), count - 32);
-        } else if (count == 0) {
-            tcg_gen_mov_i32(ret, TCGV_LOW(arg));
-        } else {
-            TCGv_i64 t = tcg_temp_new_i64();
-            tcg_gen_shri_i64(t, arg, count);
-            tcg_gen_mov_i32(ret, TCGV_LOW(t));
-            tcg_temp_free_i64(t);
-        }
-    } else if (TCG_TARGET_HAS_trunc_shr_i32) {
-        tcg_gen_op3i_i32(INDEX_op_trunc_shr_i32, ret,
-                         MAKE_TCGV_I32(GET_TCGV_I64(arg)), count);
-    } else if (count == 0) {
+        tcg_gen_mov_i32(ret, TCGV_LOW(arg));
+    } else if (TCG_TARGET_HAS_extrl_i64_i32) {
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32,
+                    GET_TCGV_I32(ret), GET_TCGV_I64(arg));
+    } else {
         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
+    }
+}
+
+void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
+{
+    if (TCG_TARGET_REG_BITS == 32) {
+        tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
+    } else if (TCG_TARGET_HAS_extrh_i64_i32) {
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32,
+                    GET_TCGV_I32(ret), GET_TCGV_I64(arg));
     } else {
         TCGv_i64 t = tcg_temp_new_i64();
-        tcg_gen_shri_i64(t, arg, count);
+        tcg_gen_shri_i64(t, arg, 32);
         tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t)));
         tcg_temp_free_i64(t);
     }
@@ -1757,9 +1776,8 @@
         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
     } else {
-        /* Note: we assume the target supports move between
-           32 and 64 bit registers.  */
-        tcg_gen_ext32u_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+        tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64,
+                    GET_TCGV_I64(ret), GET_TCGV_I32(arg));
     }
 }
 
@@ -1769,9 +1787,8 @@
         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
     } else {
-        /* Note: we assume the target supports move between
-           32 and 64 bit registers.  */
-        tcg_gen_ext32s_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+        tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64,
+                    GET_TCGV_I64(ret), GET_TCGV_I32(arg));
     }
 }
 
@@ -1807,8 +1824,8 @@
         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
     } else {
-        tcg_gen_trunc_shr_i64_i32(lo, arg, 0);
-        tcg_gen_trunc_shr_i64_i32(hi, arg, 32);
+        tcg_gen_extrl_i64_i32(lo, arg);
+        tcg_gen_extrh_i64_i32(hi, arg);
     }
 }
 
@@ -1898,12 +1915,16 @@
 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
 {
     memop = tcg_canonicalize_memop(memop, 0, 0);
+    trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
+                               addr, trace_mem_get_info(memop, 0));
     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
 }
 
 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
 {
     memop = tcg_canonicalize_memop(memop, 0, 1);
+    trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
+                               addr, trace_mem_get_info(memop, 1));
     gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
 }
 
@@ -1920,6 +1941,8 @@
     }
 
     memop = tcg_canonicalize_memop(memop, 1, 0);
+    trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
+                               addr, trace_mem_get_info(memop, 0));
     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
 }
 
@@ -1931,5 +1954,7 @@
     }
 
     memop = tcg_canonicalize_memop(memop, 1, 1);
+    trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
+                               addr, trace_mem_get_info(memop, 1));
     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
 }
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index 96adf9a..f217e80 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -251,19 +251,16 @@
 
 /* Generic ops.  */
 
-int gen_new_label(void);
-
-static inline void gen_set_label(int n)
+static inline void gen_set_label(TCGLabel *l)
 {
-    tcg_gen_op1(&tcg_ctx, INDEX_op_set_label, n);
+    tcg_gen_op1(&tcg_ctx, INDEX_op_set_label, label_arg(l));
 }
 
-static inline void tcg_gen_br(int label)
+static inline void tcg_gen_br(TCGLabel *l)
 {
-    tcg_gen_op1(&tcg_ctx, INDEX_op_br, label);
+    tcg_gen_op1(&tcg_ctx, INDEX_op_br, label_arg(l));
 }
 
-
 /* Helper calls. */
 
 /* 32 bit ops */
@@ -293,8 +290,8 @@
 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2);
 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
                          unsigned int ofs, unsigned int len);
-void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, int label);
-void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, int label);
+void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *);
+void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *);
 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
                          TCGv_i32 arg1, TCGv_i32 arg2);
 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
@@ -469,8 +466,8 @@
 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2);
 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
                          unsigned int ofs, unsigned int len);
-void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, int label);
-void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, int label);
+void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *);
+void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *);
 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
                          TCGv_i64 arg1, TCGv_i64 arg2);
 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
@@ -687,7 +684,8 @@
 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg);
 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg);
 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high);
-void tcg_gen_trunc_shr_i64_i32(TCGv_i32 ret, TCGv_i64 arg, unsigned int c);
+void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg);
+void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg);
 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg);
 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg);
 
@@ -696,38 +694,81 @@
     tcg_gen_deposit_i64(ret, lo, hi, 32, 32);
 }
 
-static inline void tcg_gen_trunc_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
-{
-    tcg_gen_trunc_shr_i64_i32(ret, arg, 0);
-}
-
 /* QEMU specific operations.  */
 
 #ifndef TARGET_LONG_BITS
 #error must include QEMU headers
 #endif
 
-/* debug info: write the PC of the corresponding QEMU CPU instruction */
-static inline void tcg_gen_debug_insn_start(uint64_t pc)
+#if TARGET_INSN_START_WORDS == 1
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc)
 {
-    /* XXX: must really use a 32 bit size for TCGArg in all cases */
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-    tcg_gen_op2ii(INDEX_op_debug_insn_start,
-                  (uint32_t)(pc), (uint32_t)(pc >> 32));
-#else
-    tcg_gen_op1i(INDEX_op_debug_insn_start, pc);
-#endif
+    tcg_gen_op1(&tcg_ctx, INDEX_op_insn_start, pc);
 }
+# else
+static inline void tcg_gen_insn_start(target_ulong pc)
+{
+    tcg_gen_op2(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32));
+}
+# endif
+#elif TARGET_INSN_START_WORDS == 2
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1)
+{
+    tcg_gen_op2(&tcg_ctx, INDEX_op_insn_start, pc, a1);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1)
+{
+    tcg_gen_op4(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32),
+                (uint32_t)a1, (uint32_t)(a1 >> 32));
+}
+# endif
+#elif TARGET_INSN_START_WORDS == 3
+# if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1,
+                                      target_ulong a2)
+{
+    tcg_gen_op3(&tcg_ctx, INDEX_op_insn_start, pc, a1, a2);
+}
+# else
+static inline void tcg_gen_insn_start(target_ulong pc, target_ulong a1,
+                                      target_ulong a2)
+{
+    tcg_gen_op6(&tcg_ctx, INDEX_op_insn_start,
+                (uint32_t)pc, (uint32_t)(pc >> 32),
+                (uint32_t)a1, (uint32_t)(a1 >> 32),
+                (uint32_t)a2, (uint32_t)(a2 >> 32));
+}
+# endif
+#else
+# error "Unhandled number of operands to insn_start"
+#endif
 
 static inline void tcg_gen_exit_tb(uintptr_t val)
 {
     tcg_gen_op1i(INDEX_op_exit_tb, val);
 }
 
+/**
+ * tcg_gen_goto_tb() - output goto_tb TCG operation
+ * @idx: Direct jump slot index (0 or 1)
+ *
+ * See tcg/README for more info about this TCG operation.
+ *
+ * NOTE: In softmmu emulation, direct jumps with goto_tb are only safe within
+ * the pages this TB resides in because we don't take care of direct jumps when
+ * address mapping changes, e.g. in tlb_flush(). In user mode, there's only a
+ * static address translation, so the destination address is always valid, TBs
+ * are always invalidated properly, and direct jumps are reset when mapping
+ * changes.
+ */
 void tcg_gen_goto_tb(unsigned idx);
 
 #if TARGET_LONG_BITS == 32
-#define TCGv TCGv_i32
 #define tcg_temp_new() tcg_temp_new_i32()
 #define tcg_global_reg_new tcg_global_reg_new_i32
 #define tcg_global_mem_new tcg_global_mem_new_i32
@@ -739,7 +780,6 @@
 #define tcg_gen_qemu_ld_tl tcg_gen_qemu_ld_i32
 #define tcg_gen_qemu_st_tl tcg_gen_qemu_st_i32
 #else
-#define TCGv TCGv_i64
 #define tcg_temp_new() tcg_temp_new_i64()
 #define tcg_global_reg_new tcg_global_reg_new_i64
 #define tcg_global_mem_new tcg_global_mem_new_i64
@@ -856,7 +896,7 @@
 #define tcg_gen_divu_tl tcg_gen_divu_i64
 #define tcg_gen_remu_tl tcg_gen_remu_i64
 #define tcg_gen_discard_tl tcg_gen_discard_i64
-#define tcg_gen_trunc_tl_i32 tcg_gen_trunc_i64_i32
+#define tcg_gen_trunc_tl_i32 tcg_gen_extrl_i64_i32
 #define tcg_gen_trunc_i64_tl tcg_gen_mov_i64
 #define tcg_gen_extu_i32_tl tcg_gen_extu_i32_i64
 #define tcg_gen_ext_i32_tl tcg_gen_ext_i32_i64
@@ -935,7 +975,7 @@
 #define tcg_gen_remu_tl tcg_gen_remu_i32
 #define tcg_gen_discard_tl tcg_gen_discard_i32
 #define tcg_gen_trunc_tl_i32 tcg_gen_mov_i32
-#define tcg_gen_trunc_i64_tl tcg_gen_trunc_i64_i32
+#define tcg_gen_trunc_i64_tl tcg_gen_extrl_i64_i32
 #define tcg_gen_extu_i32_tl tcg_gen_mov_i32
 #define tcg_gen_ext_i32_tl tcg_gen_mov_i32
 #define tcg_gen_extu_tl_i64 tcg_gen_extu_i32_i64
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index 13ccb60..6d0410c 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -138,8 +138,14 @@
 DEF(rotr_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(deposit_i64, 1, 2, 2, IMPL64 | IMPL(TCG_TARGET_HAS_deposit_i64))
 
-DEF(trunc_shr_i32, 1, 1, 1,
-    IMPL(TCG_TARGET_HAS_trunc_shr_i32)
+/* size changing ops */
+DEF(ext_i32_i64, 1, 1, 0, IMPL64)
+DEF(extu_i32_i64, 1, 1, 0, IMPL64)
+DEF(extrl_i64_i32, 1, 1, 0,
+    IMPL(TCG_TARGET_HAS_extrl_i64_i32)
+    | (TCG_TARGET_REG_BITS == 32 ? TCG_OPF_NOT_PRESENT : 0))
+DEF(extrh_i64_i32, 1, 1, 0,
+    IMPL(TCG_TARGET_HAS_extrh_i64_i32)
     | (TCG_TARGET_REG_BITS == 32 ? TCG_OPF_NOT_PRESENT : 0))
 
 DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | IMPL64)
@@ -167,18 +173,15 @@
 DEF(muluh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_muluh_i64))
 DEF(mulsh_i64, 1, 2, 0, IMPL(TCG_TARGET_HAS_mulsh_i64))
 
+#define TLADDR_ARGS  (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
+#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
+
 /* QEMU specific */
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-DEF(debug_insn_start, 0, 0, 2, TCG_OPF_NOT_PRESENT)
-#else
-DEF(debug_insn_start, 0, 0, 1, TCG_OPF_NOT_PRESENT)
-#endif
+DEF(insn_start, 0, 0, TLADDR_ARGS * TARGET_INSN_START_WORDS,
+    TCG_OPF_NOT_PRESENT)
 DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
 DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
 
-#define TLADDR_ARGS    (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
-#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
-
 DEF(qemu_ld_i32, 1, TLADDR_ARGS, 1,
     TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
 DEF(qemu_st_i32, 0, TLADDR_ARGS + 1, 1,
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 585ca6c..42417bd 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -23,20 +23,14 @@
  */
 
 /* define it to use liveness analysis (better code) */
-#define USE_LIVENESS_ANALYSIS
 #define USE_TCG_OPTIMIZATIONS
 
-#include "config.h"
+#include "qemu/osdep.h"
 
 /* Define to jump the ELF file used to communicate with GDB.  */
 #undef DEBUG_JIT
 
-#if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
-/* define it to suppress various consistency checks (faster) */
-#define NDEBUG
-#endif
-
-#include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/host-utils.h"
 #include "qemu/timer.h"
 
@@ -46,6 +40,11 @@
 #define NO_CPU_IO_DEFS
 #include "cpu.h"
 
+#include "qemu/host-utils.h"
+#include "qemu/timer.h"
+#include "exec/cpu-common.h"
+#include "exec/exec-all.h"
+
 #include "tcg-op.h"
 
 #if UINTPTR_MAX == UINT32_MAX
@@ -60,8 +59,10 @@
 #endif
 
 #include "elf.h"
+#include "exec/log.h"
 
-/* Forward declarations for functions declared in tcg-target.c and used here. */
+/* Forward declarations for functions declared in tcg-target.inc.c and
+   used here. */
 static void tcg_target_init(TCGContext *s);
 static void tcg_target_qemu_prologue(TCGContext *s);
 static void patch_reloc(tcg_insn_unit *code_ptr, int type,
@@ -95,7 +96,7 @@
                                  size_t debug_frame_size)
     __attribute__((unused));
 
-/* Forward declarations for functions declared and used in tcg-target.c. */
+/* Forward declarations for functions declared and used in tcg-target.inc.c. */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
                        intptr_t arg2);
@@ -106,19 +107,15 @@
                        const int *const_args);
 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
                        intptr_t arg2);
+static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                        TCGReg base, intptr_t ofs);
 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
 static int tcg_target_const_match(tcg_target_long val, TCGType type,
                                   const TCGArgConstraint *arg_ct);
 static void tcg_out_tb_init(TCGContext *s);
-static void tcg_out_tb_finalize(TCGContext *s);
+static bool tcg_out_tb_finalize(TCGContext *s);
 
 
-TCGOpDef tcg_op_defs[] = {
-#define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
-#include "tcg-opc.h"
-#undef DEF
-};
-const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
 
 static TCGRegSet tcg_target_available_regs[2];
 static TCGRegSet tcg_target_call_clobber_regs;
@@ -208,12 +205,10 @@
 /* label relocation processing */
 
 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
-                          int label_index, intptr_t addend)
+                          TCGLabel *l, intptr_t addend)
 {
-    TCGLabel *l;
     TCGRelocation *r;
 
-    l = &s->labels[label_index];
     if (l->has_value) {
         /* FIXME: This may break relocations on RISC targets that
            modify instruction fields in place.  The caller may not have 
@@ -230,13 +225,12 @@
     }
 }
 
-static void tcg_out_label(TCGContext *s, int label_index, tcg_insn_unit *ptr)
+static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
 {
-    TCGLabel *l = &s->labels[label_index];
     intptr_t value = (intptr_t)ptr;
     TCGRelocation *r;
 
-    assert(!l->has_value);
+    tcg_debug_assert(!l->has_value);
 
     for (r = l->u.first_reloc; r != NULL; r = r->next) {
         patch_reloc(r->ptr, r->type, value, r->addend);
@@ -246,22 +240,19 @@
     l->u.value_ptr = ptr;
 }
 
-int gen_new_label(void)
+TCGLabel *gen_new_label(void)
 {
     TCGContext *s = &tcg_ctx;
-    int idx;
-    TCGLabel *l;
+    TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
 
-    if (s->nb_labels >= TCG_MAX_LABELS)
-        tcg_abort();
-    idx = s->nb_labels++;
-    l = &s->labels[idx];
-    l->has_value = 0;
-    l->u.first_reloc = NULL;
-    return idx;
+    *l = (TCGLabel){
+        .id = s->nb_labels++
+    };
+
+    return l;
 }
 
-#include "tcg-target.c"
+#include "tcg-target.inc.c"
 
 /* pool based memory allocation */
 void *tcg_malloc_internal(TCGContext *s, int size)
@@ -329,6 +320,8 @@
 #include "exec/helper-tcg.h"
 };
 
+static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
+
 void tcg_context_init(TCGContext *s)
 {
     int op, total_args, n, i;
@@ -339,7 +332,7 @@
 
     memset(s, 0, sizeof(*s));
     s->nb_globals = 0;
-    
+
     /* Count total number of arguments and allocate the corresponding
        space */
     total_args = 0;
@@ -371,34 +364,64 @@
     }
 
     tcg_target_init(s);
+
+    /* Reverse the order of the saved registers, assuming they're all at
+       the start of tcg_target_reg_alloc_order.  */
+    for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
+        int r = tcg_target_reg_alloc_order[n];
+        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
+            break;
+        }
+    }
+    for (i = 0; i < n; ++i) {
+        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
+    }
+    for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
+        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
+    }
 }
 
 void tcg_prologue_init(TCGContext *s)
 {
-    /* init global prologue and epilogue */
-    s->code_buf = s->code_gen_prologue;
-    s->code_ptr = s->code_buf;
+    size_t prologue_size, total_size;
+    void *buf0, *buf1;
+
+    /* Put the prologue at the beginning of code_gen_buffer.  */
+    buf0 = s->code_gen_buffer;
+    s->code_ptr = buf0;
+    s->code_buf = buf0;
+    s->code_gen_prologue = buf0;
+
+    /* Generate the prologue.  */
     tcg_target_qemu_prologue(s);
-    flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
+    buf1 = s->code_ptr;
+    flush_icache_range((uintptr_t)buf0, (uintptr_t)buf1);
+
+    /* Deduct the prologue from the buffer.  */
+    prologue_size = tcg_current_code_size(s);
+    s->code_gen_ptr = buf1;
+    s->code_gen_buffer = buf1;
+    s->code_buf = buf1;
+    total_size = s->code_gen_buffer_size - prologue_size;
+    s->code_gen_buffer_size = total_size;
+
+    /* Compute a high-water mark, at which we voluntarily flush the buffer
+       and start over.  The size here is arbitrary, significantly larger
+       than we expect the code generation for any one opcode to require.  */
+    s->code_gen_highwater = s->code_gen_buffer + (total_size - 1024);
+
+    tcg_register_jit(s->code_gen_buffer, total_size);
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
-        size_t size = tcg_current_code_size(s);
-        qemu_log("PROLOGUE: [size=%zu]\n", size);
-        log_disas(s->code_buf, size);
+        qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
+        log_disas(buf0, prologue_size);
         qemu_log("\n");
         qemu_log_flush();
     }
 #endif
 }
 
-void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size)
-{
-    s->frame_start = start;
-    s->frame_end = start + size;
-    s->frame_reg = reg;
-}
-
 void tcg_func_start(TCGContext *s)
 {
     tcg_pool_reset(s);
@@ -414,136 +437,145 @@
     s->goto_tb_issue_mask = 0;
 #endif
 
-    s->gen_first_op_idx = 0;
-    s->gen_last_op_idx = -1;
-    s->gen_next_op_idx = 0;
+    s->gen_op_buf[0].next = 1;
+    s->gen_op_buf[0].prev = 0;
+    s->gen_next_op_idx = 1;
     s->gen_next_parm_idx = 0;
 
     s->be = tcg_malloc(sizeof(TCGBackendData));
 }
 
-static inline void tcg_temp_alloc(TCGContext *s, int n)
+static inline int temp_idx(TCGContext *s, TCGTemp *ts)
 {
-    if (n > TCG_MAX_TEMPS)
-        tcg_abort();
+    ptrdiff_t n = ts - s->temps;
+    tcg_debug_assert(n >= 0 && n < s->nb_temps);
+    return n;
 }
 
-static inline int tcg_global_reg_new_internal(TCGType type, int reg,
-                                              const char *name)
+static inline TCGTemp *tcg_temp_alloc(TCGContext *s)
 {
-    TCGContext *s = &tcg_ctx;
-    TCGTemp *ts;
-    int idx;
+    int n = s->nb_temps++;
+    tcg_debug_assert(n < TCG_MAX_TEMPS);
+    return memset(&s->temps[n], 0, sizeof(TCGTemp));
+}
 
-#if TCG_TARGET_REG_BITS == 32
-    if (type != TCG_TYPE_I32)
+static inline TCGTemp *tcg_global_alloc(TCGContext *s)
+{
+    tcg_debug_assert(s->nb_globals == s->nb_temps);
+    s->nb_globals++;
+    return tcg_temp_alloc(s);
+}
+
+static int tcg_global_reg_new_internal(TCGContext *s, TCGType type,
+                                       TCGReg reg, const char *name)
+{
+    TCGTemp *ts;
+
+    if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
         tcg_abort();
-#endif
-    if (tcg_regset_test_reg(s->reserved_regs, reg))
-        tcg_abort();
-    idx = s->nb_globals;
-    tcg_temp_alloc(s, s->nb_globals + 1);
-    ts = &s->temps[s->nb_globals];
+    }
+
+    ts = tcg_global_alloc(s);
     ts->base_type = type;
     ts->type = type;
     ts->fixed_reg = 1;
     ts->reg = reg;
     ts->name = name;
-    s->nb_globals++;
     tcg_regset_set_reg(s->reserved_regs, reg);
-    return idx;
+
+    return temp_idx(s, ts);
 }
 
-TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name)
+void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
 {
     int idx;
+    s->frame_start = start;
+    s->frame_end = start + size;
+    idx = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
+    s->frame_temp = &s->temps[idx];
+}
 
-    idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name);
+TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name)
+{
+    TCGContext *s = &tcg_ctx;
+    int idx;
+
+    if (tcg_regset_test_reg(s->reserved_regs, reg)) {
+        tcg_abort();
+    }
+    idx = tcg_global_reg_new_internal(s, TCG_TYPE_I32, reg, name);
     return MAKE_TCGV_I32(idx);
 }
 
-TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name)
+TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name)
 {
+    TCGContext *s = &tcg_ctx;
     int idx;
 
-    idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name);
+    if (tcg_regset_test_reg(s->reserved_regs, reg)) {
+        tcg_abort();
+    }
+    idx = tcg_global_reg_new_internal(s, TCG_TYPE_I64, reg, name);
     return MAKE_TCGV_I64(idx);
 }
 
-static inline int tcg_global_mem_new_internal(TCGType type, int reg,
-                                              intptr_t offset,
-                                              const char *name)
+int tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
+                                intptr_t offset, const char *name)
 {
     TCGContext *s = &tcg_ctx;
-    TCGTemp *ts;
-    int idx;
-
-    idx = s->nb_globals;
-#if TCG_TARGET_REG_BITS == 32
-    if (type == TCG_TYPE_I64) {
-        char buf[64];
-        tcg_temp_alloc(s, s->nb_globals + 2);
-        ts = &s->temps[s->nb_globals];
-        ts->base_type = type;
-        ts->type = TCG_TYPE_I32;
-        ts->fixed_reg = 0;
-        ts->mem_allocated = 1;
-        ts->mem_reg = reg;
+    TCGTemp *base_ts = &s->temps[GET_TCGV_PTR(base)];
+    TCGTemp *ts = tcg_global_alloc(s);
+    int indirect_reg = 0, bigendian = 0;
 #ifdef HOST_WORDS_BIGENDIAN
-        ts->mem_offset = offset + 4;
-#else
-        ts->mem_offset = offset;
+    bigendian = 1;
 #endif
+
+    if (!base_ts->fixed_reg) {
+        /* We do not support double-indirect registers.  */
+        tcg_debug_assert(!base_ts->indirect_reg);
+        base_ts->indirect_base = 1;
+        s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
+                            ? 2 : 1);
+        indirect_reg = 1;
+    }
+
+    if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
+        TCGTemp *ts2 = tcg_global_alloc(s);
+        char buf[64];
+
+        ts->base_type = TCG_TYPE_I64;
+        ts->type = TCG_TYPE_I32;
+        ts->indirect_reg = indirect_reg;
+        ts->mem_allocated = 1;
+        ts->mem_base = base_ts;
+        ts->mem_offset = offset + bigendian * 4;
         pstrcpy(buf, sizeof(buf), name);
         pstrcat(buf, sizeof(buf), "_0");
         ts->name = strdup(buf);
-        ts++;
 
-        ts->base_type = type;
-        ts->type = TCG_TYPE_I32;
-        ts->fixed_reg = 0;
-        ts->mem_allocated = 1;
-        ts->mem_reg = reg;
-#ifdef HOST_WORDS_BIGENDIAN
-        ts->mem_offset = offset;
-#else
-        ts->mem_offset = offset + 4;
-#endif
+        tcg_debug_assert(ts2 == ts + 1);
+        ts2->base_type = TCG_TYPE_I64;
+        ts2->type = TCG_TYPE_I32;
+        ts2->indirect_reg = indirect_reg;
+        ts2->mem_allocated = 1;
+        ts2->mem_base = base_ts;
+        ts2->mem_offset = offset + (1 - bigendian) * 4;
         pstrcpy(buf, sizeof(buf), name);
         pstrcat(buf, sizeof(buf), "_1");
-        ts->name = strdup(buf);
-
-        s->nb_globals += 2;
-    } else
-#endif
-    {
-        tcg_temp_alloc(s, s->nb_globals + 1);
-        ts = &s->temps[s->nb_globals];
+        ts2->name = strdup(buf);
+    } else {
         ts->base_type = type;
         ts->type = type;
-        ts->fixed_reg = 0;
+        ts->indirect_reg = indirect_reg;
         ts->mem_allocated = 1;
-        ts->mem_reg = reg;
+        ts->mem_base = base_ts;
         ts->mem_offset = offset;
         ts->name = name;
-        s->nb_globals++;
     }
-    return idx;
+    return temp_idx(s, ts);
 }
 
-TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name)
-{
-    int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
-    return MAKE_TCGV_I32(idx);
-}
-
-TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name)
-{
-    int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
-    return MAKE_TCGV_I64(idx);
-}
-
-static inline int tcg_temp_new_internal(TCGType type, int temp_local)
+static int tcg_temp_new_internal(TCGType type, int temp_local)
 {
     TCGContext *s = &tcg_ctx;
     TCGTemp *ts;
@@ -557,38 +589,30 @@
 
         ts = &s->temps[idx];
         ts->temp_allocated = 1;
-        assert(ts->base_type == type);
-        assert(ts->temp_local == temp_local);
+        tcg_debug_assert(ts->base_type == type);
+        tcg_debug_assert(ts->temp_local == temp_local);
     } else {
-        idx = s->nb_temps;
-#if TCG_TARGET_REG_BITS == 32
-        if (type == TCG_TYPE_I64) {
-            tcg_temp_alloc(s, s->nb_temps + 2);
-            ts = &s->temps[s->nb_temps];
+        ts = tcg_temp_alloc(s);
+        if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
+            TCGTemp *ts2 = tcg_temp_alloc(s);
+
             ts->base_type = type;
             ts->type = TCG_TYPE_I32;
             ts->temp_allocated = 1;
             ts->temp_local = temp_local;
-            ts->name = NULL;
-            ts++;
-            ts->base_type = type;
-            ts->type = TCG_TYPE_I32;
-            ts->temp_allocated = 1;
-            ts->temp_local = temp_local;
-            ts->name = NULL;
-            s->nb_temps += 2;
-        } else
-#endif
-        {
-            tcg_temp_alloc(s, s->nb_temps + 1);
-            ts = &s->temps[s->nb_temps];
+
+            tcg_debug_assert(ts2 == ts + 1);
+            ts2->base_type = TCG_TYPE_I64;
+            ts2->type = TCG_TYPE_I32;
+            ts2->temp_allocated = 1;
+            ts2->temp_local = temp_local;
+        } else {
             ts->base_type = type;
             ts->type = type;
             ts->temp_allocated = 1;
             ts->temp_local = temp_local;
-            ts->name = NULL;
-            s->nb_temps++;
         }
+        idx = temp_idx(s, ts);
     }
 
 #if defined(CONFIG_DEBUG_TCG)
@@ -626,9 +650,9 @@
     }
 #endif
 
-    assert(idx >= s->nb_globals && idx < s->nb_temps);
+    tcg_debug_assert(idx >= s->nb_globals && idx < s->nb_temps);
     ts = &s->temps[idx];
-    assert(ts->temp_allocated != 0);
+    tcg_debug_assert(ts->temp_allocated != 0);
     ts->temp_allocated = 0;
 
     k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
@@ -804,16 +828,16 @@
                 real_args++;
             }
 #endif
-	    /* If stack grows up, then we will be placing successive
-	       arguments at lower addresses, which means we need to
-	       reverse the order compared to how we would normally
-	       treat either big or little-endian.  For those arguments
-	       that will wind up in registers, this still works for
-	       HPPA (the only current STACK_GROWSUP target) since the
-	       argument registers are *also* allocated in decreasing
-	       order.  If another such target is added, this logic may
-	       have to get more complicated to differentiate between
-	       stack arguments and register arguments.  */
+           /* If stack grows up, then we will be placing successive
+              arguments at lower addresses, which means we need to
+              reverse the order compared to how we would normally
+              treat either big or little-endian.  For those arguments
+              that will wind up in registers, this still works for
+              HPPA (the only current STACK_GROWSUP target) since the
+              argument registers are *also* allocated in decreasing
+              order.  If another such target is added, this logic may
+              have to get more complicated to differentiate between
+              stack arguments and register arguments.  */
 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
             s->gen_opparam_buf[pi++] = args[i] + 1;
             s->gen_opparam_buf[pi++] = args[i];
@@ -848,7 +872,7 @@
     /* Make sure the calli field didn't overflow.  */
     tcg_debug_assert(s->gen_op_buf[i].calli == real_args);
 
-    s->gen_last_op_idx = i;
+    s->gen_op_buf[0].prev = i;
     s->gen_next_op_idx = i + 1;
     s->gen_next_parm_idx = pi;
 
@@ -907,37 +931,30 @@
         ts->mem_allocated = 0;
         ts->fixed_reg = 0;
     }
-    for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
-        s->reg_to_temp[i] = -1;
-    }
+
+    memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
 }
 
-static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size,
-                                 int idx)
+static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
+                                 TCGTemp *ts)
 {
-    TCGTemp *ts;
+    int idx = temp_idx(s, ts);
 
-    assert(idx >= 0 && idx < s->nb_temps);
-    ts = &s->temps[idx];
     if (idx < s->nb_globals) {
         pstrcpy(buf, buf_size, ts->name);
+    } else if (ts->temp_local) {
+        snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
     } else {
-        if (ts->temp_local) 
-            snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
-        else
-            snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
+        snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
     }
     return buf;
 }
 
-char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg)
+static char *tcg_get_arg_str_idx(TCGContext *s, char *buf,
+                                 int buf_size, int idx)
 {
-    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg));
-}
-
-char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg)
-{
-    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg));
+    tcg_debug_assert(idx >= 0 && idx < s->nb_temps);
+    return tcg_get_arg_str_ptr(s, buf, buf_size, &s->temps[idx]);
 }
 
 /* Find helper name.  */
@@ -1007,28 +1024,30 @@
     TCGOp *op;
     int oi;
 
-    for (oi = s->gen_first_op_idx; oi >= 0; oi = op->next) {
+    for (oi = s->gen_op_buf[0].next; oi != 0; oi = op->next) {
         int i, k, nb_oargs, nb_iargs, nb_cargs;
         const TCGOpDef *def;
         const TCGArg *args;
         TCGOpcode c;
+        int col = 0;
 
         op = &s->gen_op_buf[oi];
         c = op->opc;
         def = &tcg_op_defs[c];
         args = &s->gen_opparam_buf[op->args];
 
-        if (c == INDEX_op_debug_insn_start) {
-            uint64_t pc;
+        if (c == INDEX_op_insn_start) {
+            col += qemu_log("%s ----", oi != s->gen_op_buf[0].next ? "\n" : "");
+
+            for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+                target_ulong a;
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-            pc = ((uint64_t)args[1] << 32) | args[0];
+                a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
 #else
-            pc = args[0];
+                a = args[i];
 #endif
-            if (oi != s->gen_first_op_idx) {
-                qemu_log("\n");
+                col += qemu_log(" " TARGET_FMT_lx, a);
             }
-            qemu_log(" ---- 0x%" PRIx64, pc);
         } else if (c == INDEX_op_call) {
             /* variable number of arguments */
             nb_oargs = op->callo;
@@ -1036,12 +1055,12 @@
             nb_cargs = def->nb_cargs;
 
             /* function name, flags, out args */
-            qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
-                     tcg_find_helper(s, args[nb_oargs + nb_iargs]),
-                     args[nb_oargs + nb_iargs + 1], nb_oargs);
+            col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
+                            tcg_find_helper(s, args[nb_oargs + nb_iargs]),
+                            args[nb_oargs + nb_iargs + 1], nb_oargs);
             for (i = 0; i < nb_oargs; i++) {
-                qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
-                                                   args[i]));
+                col += qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
+                                                           args[i]));
             }
             for (i = 0; i < nb_iargs; i++) {
                 TCGArg arg = args[nb_oargs + i];
@@ -1049,10 +1068,10 @@
                 if (arg != TCG_CALL_DUMMY_ARG) {
                     t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg);
                 }
-                qemu_log(",%s", t);
+                col += qemu_log(",%s", t);
             }
         } else {
-            qemu_log(" %s ", def->name);
+            col += qemu_log(" %s ", def->name);
 
             nb_oargs = def->nb_oargs;
             nb_iargs = def->nb_iargs;
@@ -1061,17 +1080,17 @@
             k = 0;
             for (i = 0; i < nb_oargs; i++) {
                 if (k != 0) {
-                    qemu_log(",");
+                    col += qemu_log(",");
                 }
-                qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
-                                                   args[k++]));
+                col += qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
+                                                          args[k++]));
             }
             for (i = 0; i < nb_iargs; i++) {
                 if (k != 0) {
-                    qemu_log(",");
+                    col += qemu_log(",");
                 }
-                qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
-                                                   args[k++]));
+                col += qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
+                                                          args[k++]));
             }
             switch (c) {
             case INDEX_op_brcond_i32:
@@ -1083,9 +1102,9 @@
             case INDEX_op_setcond_i64:
             case INDEX_op_movcond_i64:
                 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {
-                    qemu_log(",%s", cond_name[args[k++]]);
+                    col += qemu_log(",%s", cond_name[args[k++]]);
                 } else {
-                    qemu_log(",$0x%" TCG_PRIlx, args[k++]);
+                    col += qemu_log(",$0x%" TCG_PRIlx, args[k++]);
                 }
                 i = 1;
                 break;
@@ -1099,12 +1118,12 @@
                     unsigned ix = get_mmuidx(oi);
 
                     if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
-                        qemu_log(",$0x%x,%u", op, ix);
+                        col += qemu_log(",$0x%x,%u", op, ix);
                     } else {
                         const char *s_al, *s_op;
                         s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
                         s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
-                        qemu_log(",%s%s,%u", s_al, s_op, ix);
+                        col += qemu_log(",%s%s,%u", s_al, s_op, ix);
                     }
                     i = 1;
                 }
@@ -1113,11 +1132,45 @@
                 i = 0;
                 break;
             }
-            for (; i < nb_cargs; i++) {
-                if (k != 0) {
-                    qemu_log(",");
+            switch (c) {
+            case INDEX_op_set_label:
+            case INDEX_op_br:
+            case INDEX_op_brcond_i32:
+            case INDEX_op_brcond_i64:
+            case INDEX_op_brcond2_i32:
+                col += qemu_log("%s$L%d", k ? "," : "", arg_label(args[k])->id);
+                i++, k++;
+                break;
+            default:
+                break;
+            }
+            for (; i < nb_cargs; i++, k++) {
+                col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", args[k]);
+            }
+        }
+        if (op->life) {
+            unsigned life = op->life;
+
+            for (; col < 48; ++col) {
+                putc(' ', qemu_logfile);
+            }
+
+            if (life & (SYNC_ARG * 3)) {
+                qemu_log("  sync:");
+                for (i = 0; i < 2; ++i) {
+                    if (life & (SYNC_ARG << i)) {
+                        qemu_log(" %d", i);
+                    }
                 }
-                qemu_log("$0x%" TCG_PRIlx, args[k++]);
+            }
+            life /= DEAD_ARG;
+            if (life) {
+                qemu_log("  dead:");
+                for (i = 0; life; ++i, life >>= 1) {
+                    if (life & 1) {
+                        qemu_log(" %d", i);
+                    }
+                }
             }
         }
         qemu_log("\n");
@@ -1179,25 +1232,25 @@
         if (tdefs->op == (TCGOpcode)-1)
             break;
         op = tdefs->op;
-        assert((unsigned)op < NB_OPS);
+        tcg_debug_assert((unsigned)op < NB_OPS);
         def = &tcg_op_defs[op];
 #if defined(CONFIG_DEBUG_TCG)
         /* Duplicate entry in op definitions? */
-        assert(!def->used);
+        tcg_debug_assert(!def->used);
         def->used = 1;
 #endif
         nb_args = def->nb_iargs + def->nb_oargs;
         for(i = 0; i < nb_args; i++) {
             ct_str = tdefs->args_ct_str[i];
             /* Incomplete TCGTargetOpDef entry? */
-            assert(ct_str != NULL);
+            tcg_debug_assert(ct_str != NULL);
             tcg_regset_clear(def->args_ct[i].u.regs);
             def->args_ct[i].ct = 0;
             if (ct_str[0] >= '0' && ct_str[0] <= '9') {
                 int oarg;
                 oarg = ct_str[0] - '0';
-                assert(oarg < def->nb_oargs);
-                assert(def->args_ct[oarg].ct & TCG_CT_REG);
+                tcg_debug_assert(oarg < def->nb_oargs);
+                tcg_debug_assert(def->args_ct[oarg].ct & TCG_CT_REG);
                 /* TCG_CT_ALIAS is for the output arguments. The input
                    argument is tagged with TCG_CT_IALIAS. */
                 def->args_ct[i] = def->args_ct[oarg];
@@ -1226,7 +1279,7 @@
         }
 
         /* TCGTargetOpDef entry with too much information? */
-        assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
+        tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
 
         /* sort the constraints (XXX: this is just an heuristic) */
         sort_constraints(def, 0, def->nb_oargs);
@@ -1247,7 +1300,7 @@
 
 #if defined(CONFIG_DEBUG_TCG)
     i = 0;
-    for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) {
+    for (op = 0; op < tcg_op_defs_max; op++) {
         const TCGOpDef *def = &tcg_op_defs[op];
         if (def->flags & TCG_OPF_NOT_PRESENT) {
             /* Wrong entry in op definitions? */
@@ -1274,71 +1327,116 @@
     int next = op->next;
     int prev = op->prev;
 
-    if (next >= 0) {
-        s->gen_op_buf[next].prev = prev;
-    } else {
-        s->gen_last_op_idx = prev;
-    }
-    if (prev >= 0) {
-        s->gen_op_buf[prev].next = next;
-    } else {
-        s->gen_first_op_idx = next;
-    }
+    /* We should never attempt to remove the list terminator.  */
+    tcg_debug_assert(op != &s->gen_op_buf[0]);
 
-    memset(op, -1, sizeof(*op));
+    s->gen_op_buf[next].prev = prev;
+    s->gen_op_buf[prev].next = next;
+
+    memset(op, 0, sizeof(*op));
 
 #ifdef CONFIG_PROFILER
     s->del_op_count++;
 #endif
 }
 
-#ifdef USE_LIVENESS_ANALYSIS
+TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
+                            TCGOpcode opc, int nargs)
+{
+    int oi = s->gen_next_op_idx;
+    int pi = s->gen_next_parm_idx;
+    int prev = old_op->prev;
+    int next = old_op - s->gen_op_buf;
+    TCGOp *new_op;
+
+    tcg_debug_assert(oi < OPC_BUF_SIZE);
+    tcg_debug_assert(pi + nargs <= OPPARAM_BUF_SIZE);
+    s->gen_next_op_idx = oi + 1;
+    s->gen_next_parm_idx = pi + nargs;
+
+    new_op = &s->gen_op_buf[oi];
+    *new_op = (TCGOp){
+        .opc = opc,
+        .args = pi,
+        .prev = prev,
+        .next = next
+    };
+    s->gen_op_buf[prev].next = oi;
+    old_op->prev = oi;
+
+    return new_op;
+}
+
+TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
+                           TCGOpcode opc, int nargs)
+{
+    int oi = s->gen_next_op_idx;
+    int pi = s->gen_next_parm_idx;
+    int prev = old_op - s->gen_op_buf;
+    int next = old_op->next;
+    TCGOp *new_op;
+
+    tcg_debug_assert(oi < OPC_BUF_SIZE);
+    tcg_debug_assert(pi + nargs <= OPPARAM_BUF_SIZE);
+    s->gen_next_op_idx = oi + 1;
+    s->gen_next_parm_idx = pi + nargs;
+
+    new_op = &s->gen_op_buf[oi];
+    *new_op = (TCGOp){
+        .opc = opc,
+        .args = pi,
+        .prev = prev,
+        .next = next
+    };
+    s->gen_op_buf[next].prev = oi;
+    old_op->next = oi;
+
+    return new_op;
+}
+
+#define TS_DEAD  1
+#define TS_MEM   2
+
+#define IS_DEAD_ARG(n)   (arg_life & (DEAD_ARG << (n)))
+#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
+
 /* liveness analysis: end of function: all temps are dead, and globals
    should be in memory. */
-static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps,
-                                   uint8_t *mem_temps)
+static inline void tcg_la_func_end(TCGContext *s, uint8_t *temp_state)
 {
-    memset(dead_temps, 1, s->nb_temps);
-    memset(mem_temps, 1, s->nb_globals);
-    memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals);
+    memset(temp_state, TS_DEAD | TS_MEM, s->nb_globals);
+    memset(temp_state + s->nb_globals, TS_DEAD, s->nb_temps - s->nb_globals);
 }
 
 /* liveness analysis: end of basic block: all temps are dead, globals
    and local temps should be in memory. */
-static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
-                                 uint8_t *mem_temps)
+static inline void tcg_la_bb_end(TCGContext *s, uint8_t *temp_state)
 {
-    int i;
+    int i, n;
 
-    memset(dead_temps, 1, s->nb_temps);
-    memset(mem_temps, 1, s->nb_globals);
-    for(i = s->nb_globals; i < s->nb_temps; i++) {
-        mem_temps[i] = s->temps[i].temp_local;
+    tcg_la_func_end(s, temp_state);
+    for (i = s->nb_globals, n = s->nb_temps; i < n; i++) {
+        if (s->temps[i].temp_local) {
+            temp_state[i] |= TS_MEM;
+        }
     }
 }
 
-/* Liveness analysis : update the opc_dead_args array to tell if a
+/* Liveness analysis : update the opc_arg_life array to tell if a
    given input arguments is dead. Instructions updating dead
    temporaries are removed. */
-static void tcg_liveness_analysis(TCGContext *s)
+static void liveness_pass_1(TCGContext *s, uint8_t *temp_state)
 {
-    uint8_t *dead_temps, *mem_temps;
-    int oi, oi_prev, nb_ops;
+    int nb_globals = s->nb_globals;
+    int oi, oi_prev;
 
-    nb_ops = s->gen_next_op_idx;
-    s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
-    s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
-    
-    dead_temps = tcg_malloc(s->nb_temps);
-    mem_temps = tcg_malloc(s->nb_temps);
-    tcg_la_func_end(s, dead_temps, mem_temps);
+    tcg_la_func_end(s, temp_state);
 
-    for (oi = s->gen_last_op_idx; oi >= 0; oi = oi_prev) {
+    for (oi = s->gen_op_buf[0].prev; oi != 0; oi = oi_prev) {
         int i, nb_iargs, nb_oargs;
         TCGOpcode opc_new, opc_new2;
         bool have_opc_new2;
-        uint16_t dead_args;
-        uint8_t sync_args;
+        TCGLifeData arg_life = 0;
         TCGArg arg;
 
         TCGOp * const op = &s->gen_op_buf[oi];
@@ -1361,7 +1459,7 @@
                 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
                     for (i = 0; i < nb_oargs; i++) {
                         arg = args[i];
-                        if (!dead_temps[arg] || mem_temps[arg]) {
+                        if (temp_state[arg] != TS_DEAD) {
                             goto do_not_remove_call;
                         }
                     }
@@ -1370,55 +1468,52 @@
                 do_not_remove_call:
 
                     /* output args are dead */
-                    dead_args = 0;
-                    sync_args = 0;
                     for (i = 0; i < nb_oargs; i++) {
                         arg = args[i];
-                        if (dead_temps[arg]) {
-                            dead_args |= (1 << i);
+                        if (temp_state[arg] & TS_DEAD) {
+                            arg_life |= DEAD_ARG << i;
                         }
-                        if (mem_temps[arg]) {
-                            sync_args |= (1 << i);
+                        if (temp_state[arg] & TS_MEM) {
+                            arg_life |= SYNC_ARG << i;
                         }
-                        dead_temps[arg] = 1;
-                        mem_temps[arg] = 0;
+                        temp_state[arg] = TS_DEAD;
                     }
 
-                    if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
-                        /* globals should be synced to memory */
-                        memset(mem_temps, 1, s->nb_globals);
-                    }
                     if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
                                         TCG_CALL_NO_READ_GLOBALS))) {
                         /* globals should go back to memory */
-                        memset(dead_temps, 1, s->nb_globals);
+                        memset(temp_state, TS_DEAD | TS_MEM, nb_globals);
+                    } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
+                        /* globals should be synced to memory */
+                        for (i = 0; i < nb_globals; i++) {
+                            temp_state[i] |= TS_MEM;
+                        }
                     }
 
                     /* record arguments that die in this helper */
                     for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
                         arg = args[i];
                         if (arg != TCG_CALL_DUMMY_ARG) {
-                            if (dead_temps[arg]) {
-                                dead_args |= (1 << i);
+                            if (temp_state[arg] & TS_DEAD) {
+                                arg_life |= DEAD_ARG << i;
                             }
                         }
                     }
-                    /* input arguments are live for preceeding opcodes */
-                    for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
+                    /* input arguments are live for preceding opcodes */
+                    for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
                         arg = args[i];
-                        dead_temps[arg] = 0;
+                        if (arg != TCG_CALL_DUMMY_ARG) {
+                            temp_state[arg] &= ~TS_DEAD;
+                        }
                     }
-                    s->op_dead_args[oi] = dead_args;
-                    s->op_sync_args[oi] = sync_args;
                 }
             }
             break;
-        case INDEX_op_debug_insn_start:
+        case INDEX_op_insn_start:
             break;
         case INDEX_op_discard:
             /* mark the temporary as dead */
-            dead_temps[args[0]] = 1;
-            mem_temps[args[0]] = 0;
+            temp_state[args[0]] = TS_DEAD;
             break;
 
         case INDEX_op_add2_i32:
@@ -1439,8 +1534,8 @@
                the low part.  The result can be optimized to a simple
                add or sub.  This happens often for x86_64 guest when the
                cpu mode is set to 32 bit.  */
-            if (dead_temps[args[1]] && !mem_temps[args[1]]) {
-                if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+            if (temp_state[args[1]] == TS_DEAD) {
+                if (temp_state[args[0]] == TS_DEAD) {
                     goto do_remove;
                 }
                 /* Replace the opcode and adjust the args in place,
@@ -1477,8 +1572,8 @@
         do_mul2:
             nb_iargs = 2;
             nb_oargs = 2;
-            if (dead_temps[args[1]] && !mem_temps[args[1]]) {
-                if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+            if (temp_state[args[1]] == TS_DEAD) {
+                if (temp_state[args[0]] == TS_DEAD) {
                     /* Both parts of the operation are dead.  */
                     goto do_remove;
                 }
@@ -1486,8 +1581,7 @@
                 op->opc = opc = opc_new;
                 args[1] = args[2];
                 args[2] = args[3];
-            } else if (have_opc_new2 && dead_temps[args[0]]
-                       && !mem_temps[args[0]]) {
+            } else if (temp_state[args[0]] == TS_DEAD && have_opc_new2) {
                 /* The low part of the operation is dead; generate the high. */
                 op->opc = opc = opc_new2;
                 args[0] = args[1];
@@ -1510,8 +1604,7 @@
                implies side effects */
             if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
                 for (i = 0; i < nb_oargs; i++) {
-                    arg = args[i];
-                    if (!dead_temps[arg] || mem_temps[arg]) {
+                    if (temp_state[args[i]] != TS_DEAD) {
                         goto do_not_remove;
                     }
                 }
@@ -1520,62 +1613,205 @@
             } else {
             do_not_remove:
                 /* output args are dead */
-                dead_args = 0;
-                sync_args = 0;
                 for (i = 0; i < nb_oargs; i++) {
                     arg = args[i];
-                    if (dead_temps[arg]) {
-                        dead_args |= (1 << i);
+                    if (temp_state[arg] & TS_DEAD) {
+                        arg_life |= DEAD_ARG << i;
                     }
-                    if (mem_temps[arg]) {
-                        sync_args |= (1 << i);
+                    if (temp_state[arg] & TS_MEM) {
+                        arg_life |= SYNC_ARG << i;
                     }
-                    dead_temps[arg] = 1;
-                    mem_temps[arg] = 0;
+                    temp_state[arg] = TS_DEAD;
                 }
 
                 /* if end of basic block, update */
                 if (def->flags & TCG_OPF_BB_END) {
-                    tcg_la_bb_end(s, dead_temps, mem_temps);
+                    tcg_la_bb_end(s, temp_state);
                 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
                     /* globals should be synced to memory */
-                    memset(mem_temps, 1, s->nb_globals);
+                    for (i = 0; i < nb_globals; i++) {
+                        temp_state[i] |= TS_MEM;
+                    }
                 }
 
                 /* record arguments that die in this opcode */
                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
                     arg = args[i];
-                    if (dead_temps[arg]) {
-                        dead_args |= (1 << i);
+                    if (temp_state[arg] & TS_DEAD) {
+                        arg_life |= DEAD_ARG << i;
                     }
                 }
-                /* input arguments are live for preceeding opcodes */
+                /* input arguments are live for preceding opcodes */
                 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-                    arg = args[i];
-                    dead_temps[arg] = 0;
+                    temp_state[args[i]] &= ~TS_DEAD;
                 }
-                s->op_dead_args[oi] = dead_args;
-                s->op_sync_args[oi] = sync_args;
             }
             break;
         }
+        op->life = arg_life;
     }
 }
-#else
-/* dummy liveness analysis */
-static void tcg_liveness_analysis(TCGContext *s)
+
+/* Liveness analysis: Convert indirect regs to direct temporaries.  */
+static bool liveness_pass_2(TCGContext *s, uint8_t *temp_state)
 {
-    int nb_ops;
-    nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
+    int nb_globals = s->nb_globals;
+    int16_t *dir_temps;
+    int i, oi, oi_next;
+    bool changes = false;
 
-    s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
-    memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
-    s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
-    memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t));
+    dir_temps = tcg_malloc(nb_globals * sizeof(int16_t));
+    memset(dir_temps, 0, nb_globals * sizeof(int16_t));
+
+    /* Create a temporary for each indirect global.  */
+    for (i = 0; i < nb_globals; ++i) {
+        TCGTemp *its = &s->temps[i];
+        if (its->indirect_reg) {
+            TCGTemp *dts = tcg_temp_alloc(s);
+            dts->type = its->type;
+            dts->base_type = its->base_type;
+            dir_temps[i] = temp_idx(s, dts);
+        }
+    }
+
+    memset(temp_state, TS_DEAD, nb_globals);
+
+    for (oi = s->gen_op_buf[0].next; oi != 0; oi = oi_next) {
+        TCGOp *op = &s->gen_op_buf[oi];
+        TCGArg *args = &s->gen_opparam_buf[op->args];
+        TCGOpcode opc = op->opc;
+        const TCGOpDef *def = &tcg_op_defs[opc];
+        TCGLifeData arg_life = op->life;
+        int nb_iargs, nb_oargs, call_flags;
+        TCGArg arg, dir;
+
+        oi_next = op->next;
+
+        if (opc == INDEX_op_call) {
+            nb_oargs = op->callo;
+            nb_iargs = op->calli;
+            call_flags = args[nb_oargs + nb_iargs + 1];
+        } else {
+            nb_iargs = def->nb_iargs;
+            nb_oargs = def->nb_oargs;
+
+            /* Set flags similar to how calls require.  */
+            if (def->flags & TCG_OPF_BB_END) {
+                /* Like writing globals: save_globals */
+                call_flags = 0;
+            } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
+                /* Like reading globals: sync_globals */
+                call_flags = TCG_CALL_NO_WRITE_GLOBALS;
+            } else {
+                /* No effect on globals.  */
+                call_flags = (TCG_CALL_NO_READ_GLOBALS |
+                              TCG_CALL_NO_WRITE_GLOBALS);
+            }
+        }
+
+        /* Make sure that input arguments are available.  */
+        for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
+            arg = args[i];
+            /* Note this unsigned test catches TCG_CALL_ARG_DUMMY too.  */
+            if (arg < nb_globals) {
+                dir = dir_temps[arg];
+                if (dir != 0 && temp_state[arg] == TS_DEAD) {
+                    TCGTemp *its = &s->temps[arg];
+                    TCGOpcode lopc = (its->type == TCG_TYPE_I32
+                                      ? INDEX_op_ld_i32
+                                      : INDEX_op_ld_i64);
+                    TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3);
+                    TCGArg *largs = &s->gen_opparam_buf[lop->args];
+
+                    largs[0] = dir;
+                    largs[1] = temp_idx(s, its->mem_base);
+                    largs[2] = its->mem_offset;
+
+                    /* Loaded, but synced with memory.  */
+                    temp_state[arg] = TS_MEM;
+                }
+            }
+        }
+
+        /* Perform input replacement, and mark inputs that became dead.
+           No action is required except keeping temp_state up to date
+           so that we reload when needed.  */
+        for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
+            arg = args[i];
+            if (arg < nb_globals) {
+                dir = dir_temps[arg];
+                if (dir != 0) {
+                    args[i] = dir;
+                    changes = true;
+                    if (IS_DEAD_ARG(i)) {
+                        temp_state[arg] = TS_DEAD;
+                    }
+                }
+            }
+        }
+
+        /* Liveness analysis should ensure that the following are
+           all correct, for call sites and basic block end points.  */
+        if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
+            /* Nothing to do */
+        } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
+            for (i = 0; i < nb_globals; ++i) {
+                /* Liveness should see that globals are synced back,
+                   that is, either TS_DEAD or TS_MEM.  */
+                tcg_debug_assert(dir_temps[i] == 0
+                                 || temp_state[i] != 0);
+            }
+        } else {
+            for (i = 0; i < nb_globals; ++i) {
+                /* Liveness should see that globals are saved back,
+                   that is, TS_DEAD, waiting to be reloaded.  */
+                tcg_debug_assert(dir_temps[i] == 0
+                                 || temp_state[i] == TS_DEAD);
+            }
+        }
+
+        /* Outputs become available.  */
+        for (i = 0; i < nb_oargs; i++) {
+            arg = args[i];
+            if (arg >= nb_globals) {
+                continue;
+            }
+            dir = dir_temps[arg];
+            if (dir == 0) {
+                continue;
+            }
+            args[i] = dir;
+            changes = true;
+
+            /* The output is now live and modified.  */
+            temp_state[arg] = 0;
+
+            /* Sync outputs upon their last write.  */
+            if (NEED_SYNC_ARG(i)) {
+                TCGTemp *its = &s->temps[arg];
+                TCGOpcode sopc = (its->type == TCG_TYPE_I32
+                                  ? INDEX_op_st_i32
+                                  : INDEX_op_st_i64);
+                TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3);
+                TCGArg *sargs = &s->gen_opparam_buf[sop->args];
+
+                sargs[0] = dir;
+                sargs[1] = temp_idx(s, its->mem_base);
+                sargs[2] = its->mem_offset;
+
+                temp_state[arg] = TS_MEM;
+            }
+            /* Drop outputs that are dead.  */
+            if (IS_DEAD_ARG(i)) {
+                temp_state[arg] = TS_DEAD;
+            }
+        }
+    }
+
+    return changes;
 }
-#endif
 
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
 static void dump_regs(TCGContext *s)
 {
     TCGTemp *ts;
@@ -1590,7 +1826,8 @@
             printf("%s", tcg_target_reg_names[ts->reg]);
             break;
         case TEMP_VAL_MEM:
-            printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]);
+            printf("%d(%s)", (int)ts->mem_offset,
+                   tcg_target_reg_names[ts->mem_base->reg]);
             break;
         case TEMP_VAL_CONST:
             printf("$0x%" TCG_PRIlx, ts->val);
@@ -1606,43 +1843,41 @@
     }
 
     for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
-        if (s->reg_to_temp[i] >= 0) {
+        if (s->reg_to_temp[i] != NULL) {
             printf("%s: %s\n", 
                    tcg_target_reg_names[i], 
-                   tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i]));
+                   tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
         }
     }
 }
 
 static void check_regs(TCGContext *s)
 {
-    int reg, k;
+    int reg;
+    int k;
     TCGTemp *ts;
     char buf[64];
 
-    for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
-        k = s->reg_to_temp[reg];
-        if (k >= 0) {
-            ts = &s->temps[k];
-            if (ts->val_type != TEMP_VAL_REG ||
-                ts->reg != reg) {
+    for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
+        ts = s->reg_to_temp[reg];
+        if (ts != NULL) {
+            if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
                 printf("Inconsistency for register %s:\n", 
                        tcg_target_reg_names[reg]);
                 goto fail;
             }
         }
     }
-    for(k = 0; k < s->nb_temps; k++) {
+    for (k = 0; k < s->nb_temps; k++) {
         ts = &s->temps[k];
-        if (ts->val_type == TEMP_VAL_REG &&
-            !ts->fixed_reg &&
-            s->reg_to_temp[ts->reg] != k) {
-                printf("Inconsistency for temp %s:\n", 
-                       tcg_get_arg_str_idx(s, buf, sizeof(buf), k));
+        if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg
+            && s->reg_to_temp[ts->reg] != ts) {
+            printf("Inconsistency for temp %s:\n",
+                   tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
         fail:
-                printf("reg state:\n");
-                dump_regs(s);
-                tcg_abort();
+            printf("reg state:\n");
+            dump_regs(s);
+            tcg_abort();
         }
     }
 }
@@ -1663,62 +1898,115 @@
         tcg_abort();
     }
     ts->mem_offset = s->current_frame_offset;
-    ts->mem_reg = s->frame_reg;
+    ts->mem_base = s->frame_temp;
     ts->mem_allocated = 1;
     s->current_frame_offset += sizeof(tcg_target_long);
 }
 
-/* sync register 'reg' by saving it to the corresponding temporary */
-static inline void tcg_reg_sync(TCGContext *s, int reg)
-{
-    TCGTemp *ts;
-    int temp;
+static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet);
 
-    temp = s->reg_to_temp[reg];
-    ts = &s->temps[temp];
-    assert(ts->val_type == TEMP_VAL_REG);
-    if (!ts->mem_coherent && !ts->fixed_reg) {
-        if (!ts->mem_allocated) {
-            temp_allocate_frame(s, temp);
-        }
-        tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
+/* Mark a temporary as free or dead.  If 'free_or_dead' is negative,
+   mark it free; otherwise mark it dead.  */
+static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
+{
+    if (ts->fixed_reg) {
+        return;
     }
-    ts->mem_coherent = 1;
+    if (ts->val_type == TEMP_VAL_REG) {
+        s->reg_to_temp[ts->reg] = NULL;
+    }
+    ts->val_type = (free_or_dead < 0
+                    || ts->temp_local
+                    || temp_idx(s, ts) < s->nb_globals
+                    ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
+}
+
+/* Mark a temporary as dead.  */
+static inline void temp_dead(TCGContext *s, TCGTemp *ts)
+{
+    temp_free_or_dead(s, ts, 1);
+}
+
+/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
+   registers needs to be allocated to store a constant.  If 'free_or_dead'
+   is non-zero, subsequently release the temporary; if it is positive, the
+   temp is dead; if it is negative, the temp is free.  */
+static void temp_sync(TCGContext *s, TCGTemp *ts,
+                      TCGRegSet allocated_regs, int free_or_dead)
+{
+    if (ts->fixed_reg) {
+        return;
+    }
+    if (!ts->mem_coherent) {
+        if (!ts->mem_allocated) {
+            temp_allocate_frame(s, temp_idx(s, ts));
+        }
+        switch (ts->val_type) {
+        case TEMP_VAL_CONST:
+            /* If we're going to free the temp immediately, then we won't
+               require it later in a register, so attempt to store the
+               constant to memory directly.  */
+            if (free_or_dead
+                && tcg_out_sti(s, ts->type, ts->val,
+                               ts->mem_base->reg, ts->mem_offset)) {
+                break;
+            }
+            temp_load(s, ts, tcg_target_available_regs[ts->type],
+                      allocated_regs);
+            /* fallthrough */
+
+        case TEMP_VAL_REG:
+            tcg_out_st(s, ts->type, ts->reg,
+                       ts->mem_base->reg, ts->mem_offset);
+            break;
+
+        case TEMP_VAL_MEM:
+            break;
+
+        case TEMP_VAL_DEAD:
+        default:
+            tcg_abort();
+        }
+        ts->mem_coherent = 1;
+    }
+    if (free_or_dead) {
+        temp_free_or_dead(s, ts, free_or_dead);
+    }
 }
 
 /* free register 'reg' by spilling the corresponding temporary if necessary */
-static void tcg_reg_free(TCGContext *s, int reg)
+static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
 {
-    int temp;
-
-    temp = s->reg_to_temp[reg];
-    if (temp != -1) {
-        tcg_reg_sync(s, reg);
-        s->temps[temp].val_type = TEMP_VAL_MEM;
-        s->reg_to_temp[reg] = -1;
+    TCGTemp *ts = s->reg_to_temp[reg];
+    if (ts != NULL) {
+        temp_sync(s, ts, allocated_regs, -1);
     }
 }
 
 /* Allocate a register belonging to reg1 & ~reg2 */
-static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2)
+static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet desired_regs,
+                            TCGRegSet allocated_regs, bool rev)
 {
-    int i, reg;
+    int i, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
+    const int *order;
+    TCGReg reg;
     TCGRegSet reg_ct;
 
-    tcg_regset_andnot(reg_ct, reg1, reg2);
+    tcg_regset_andnot(reg_ct, desired_regs, allocated_regs);
+    order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
 
     /* first try free registers */
-    for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) {
-        reg = tcg_target_reg_alloc_order[i];
-        if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1)
+    for(i = 0; i < n; i++) {
+        reg = order[i];
+        if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == NULL)
             return reg;
     }
 
     /* XXX: do better spill choice */
-    for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) {
-        reg = tcg_target_reg_alloc_order[i];
+    for(i = 0; i < n; i++) {
+        reg = order[i];
         if (tcg_regset_test_reg(reg_ct, reg)) {
-            tcg_reg_free(s, reg);
+            tcg_reg_free(s, reg, allocated_regs);
             return reg;
         }
     }
@@ -1726,65 +2014,42 @@
     tcg_abort();
 }
 
-/* mark a temporary as dead. */
-static inline void temp_dead(TCGContext *s, int temp)
+/* Make sure the temporary is in a register.  If needed, allocate the register
+   from DESIRED while avoiding ALLOCATED.  */
+static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
+                      TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
+    TCGReg reg;
 
-    ts = &s->temps[temp];
-    if (!ts->fixed_reg) {
-        if (ts->val_type == TEMP_VAL_REG) {
-            s->reg_to_temp[ts->reg] = -1;
-        }
-        if (temp < s->nb_globals || ts->temp_local) {
-            ts->val_type = TEMP_VAL_MEM;
-        } else {
-            ts->val_type = TEMP_VAL_DEAD;
-        }
+    switch (ts->val_type) {
+    case TEMP_VAL_REG:
+        return;
+    case TEMP_VAL_CONST:
+        reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
+        tcg_out_movi(s, ts->type, reg, ts->val);
+        ts->mem_coherent = 0;
+        break;
+    case TEMP_VAL_MEM:
+        reg = tcg_reg_alloc(s, desired_regs, allocated_regs, ts->indirect_base);
+        tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
+        ts->mem_coherent = 1;
+        break;
+    case TEMP_VAL_DEAD:
+    default:
+        tcg_abort();
     }
+    ts->reg = reg;
+    ts->val_type = TEMP_VAL_REG;
+    s->reg_to_temp[reg] = ts;
 }
 
-/* sync a temporary to memory. 'allocated_regs' is used in case a
-   temporary registers needs to be allocated to store a constant. */
-static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs)
+/* Save a temporary to memory. 'allocated_regs' is used in case a
+   temporary registers needs to be allocated to store a constant.  */
+static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
-
-    ts = &s->temps[temp];
-    if (!ts->fixed_reg) {
-        switch(ts->val_type) {
-        case TEMP_VAL_CONST:
-            ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
-                                    allocated_regs);
-            ts->val_type = TEMP_VAL_REG;
-            s->reg_to_temp[ts->reg] = temp;
-            ts->mem_coherent = 0;
-            tcg_out_movi(s, ts->type, ts->reg, ts->val);
-            /* fallthrough*/
-        case TEMP_VAL_REG:
-            tcg_reg_sync(s, ts->reg);
-            break;
-        case TEMP_VAL_DEAD:
-        case TEMP_VAL_MEM:
-            break;
-        default:
-            tcg_abort();
-        }
-    }
-}
-
-/* save a temporary to memory. 'allocated_regs' is used in case a
-   temporary registers needs to be allocated to store a constant. */
-static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
-{
-#ifdef USE_LIVENESS_ANALYSIS
     /* The liveness analysis already ensures that globals are back
-       in memory. Keep an assert for safety. */
-    assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg);
-#else
-    temp_sync(s, temp, allocated_regs);
-    temp_dead(s, temp);
-#endif
+       in memory. Keep an tcg_debug_assert for safety. */
+    tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg);
 }
 
 /* save globals to their canonical location and assume they can be
@@ -1794,8 +2059,8 @@
 {
     int i;
 
-    for(i = 0; i < s->nb_globals; i++) {
-        temp_save(s, i, allocated_regs);
+    for (i = 0; i < s->nb_globals; i++) {
+        temp_save(s, &s->temps[i], allocated_regs);
     }
 }
 
@@ -1807,12 +2072,10 @@
     int i;
 
     for (i = 0; i < s->nb_globals; i++) {
-#ifdef USE_LIVENESS_ANALYSIS
-        assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg ||
-               s->temps[i].mem_coherent);
-#else
-        temp_sync(s, i, allocated_regs);
-#endif
+        TCGTemp *ts = &s->temps[i];
+        tcg_debug_assert(ts->val_type != TEMP_VAL_REG
+                         || ts->fixed_reg
+                         || ts->mem_coherent);
     }
 }
 
@@ -1820,32 +2083,24 @@
    all globals are stored at their canonical location. */
 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
 {
-    TCGTemp *ts;
     int i;
 
-    for(i = s->nb_globals; i < s->nb_temps; i++) {
-        ts = &s->temps[i];
+    for (i = s->nb_globals; i < s->nb_temps; i++) {
+        TCGTemp *ts = &s->temps[i];
         if (ts->temp_local) {
-            temp_save(s, i, allocated_regs);
+            temp_save(s, ts, allocated_regs);
         } else {
-#ifdef USE_LIVENESS_ANALYSIS
             /* The liveness analysis already ensures that temps are dead.
-               Keep an assert for safety. */
-            assert(ts->val_type == TEMP_VAL_DEAD);
-#else
-            temp_dead(s, i);
-#endif
+               Keep an tcg_debug_assert for safety. */
+            tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
         }
     }
 
     save_globals(s, allocated_regs);
 }
 
-#define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
-#define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
-
 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
-                               uint16_t dead_args, uint8_t sync_args)
+                               TCGLifeData arg_life)
 {
     TCGTemp *ots;
     tcg_target_ulong val;
@@ -1854,27 +2109,27 @@
     val = args[1];
 
     if (ots->fixed_reg) {
-        /* for fixed registers, we do not do any constant
-           propagation */
+        /* For fixed registers, we do not do any constant propagation.  */
         tcg_out_movi(s, ots->type, ots->reg, val);
-    } else {
-        /* The movi is not explicitly generated here */
-        if (ots->val_type == TEMP_VAL_REG)
-            s->reg_to_temp[ots->reg] = -1;
-        ots->val_type = TEMP_VAL_CONST;
-        ots->val = val;
+        return;
     }
+
+    /* The movi is not explicitly generated here.  */
+    if (ots->val_type == TEMP_VAL_REG) {
+        s->reg_to_temp[ots->reg] = NULL;
+    }
+    ots->val_type = TEMP_VAL_CONST;
+    ots->val = val;
+    ots->mem_coherent = 0;
     if (NEED_SYNC_ARG(0)) {
-        temp_sync(s, args[0], s->reserved_regs);
-    }
-    if (IS_DEAD_ARG(0)) {
-        temp_dead(s, args[0]);
+        temp_sync(s, ots, s->reserved_regs, IS_DEAD_ARG(0));
+    } else if (IS_DEAD_ARG(0)) {
+        temp_dead(s, ots);
     }
 }
 
 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
-                              const TCGArg *args, uint16_t dead_args,
-                              uint8_t sync_args)
+                              const TCGArg *args, TCGLifeData arg_life)
 {
     TCGRegSet allocated_regs;
     TCGTemp *ts, *ots;
@@ -1894,76 +2149,70 @@
        we don't have to reload SOURCE the next time it is used. */
     if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
         || ts->val_type == TEMP_VAL_MEM) {
-        ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[itype],
-                                allocated_regs);
-        if (ts->val_type == TEMP_VAL_MEM) {
-            tcg_out_ld(s, itype, ts->reg, ts->mem_reg, ts->mem_offset);
-            ts->mem_coherent = 1;
-        } else if (ts->val_type == TEMP_VAL_CONST) {
-            tcg_out_movi(s, itype, ts->reg, ts->val);
-        }
-        s->reg_to_temp[ts->reg] = args[1];
-        ts->val_type = TEMP_VAL_REG;
+        temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs);
     }
 
     if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
         /* mov to a non-saved dead register makes no sense (even with
            liveness analysis disabled). */
-        assert(NEED_SYNC_ARG(0));
+        tcg_debug_assert(NEED_SYNC_ARG(0));
         /* The code above should have moved the temp to a register. */
-        assert(ts->val_type == TEMP_VAL_REG);
+        tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
         if (!ots->mem_allocated) {
             temp_allocate_frame(s, args[0]);
         }
-        tcg_out_st(s, otype, ts->reg, ots->mem_reg, ots->mem_offset);
+        tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
         if (IS_DEAD_ARG(1)) {
-            temp_dead(s, args[1]);
+            temp_dead(s, ts);
         }
-        temp_dead(s, args[0]);
+        temp_dead(s, ots);
     } else if (ts->val_type == TEMP_VAL_CONST) {
         /* propagate constant */
         if (ots->val_type == TEMP_VAL_REG) {
-            s->reg_to_temp[ots->reg] = -1;
+            s->reg_to_temp[ots->reg] = NULL;
         }
         ots->val_type = TEMP_VAL_CONST;
         ots->val = ts->val;
+        if (IS_DEAD_ARG(1)) {
+            temp_dead(s, ts);
+        }
     } else {
         /* The code in the first if block should have moved the
            temp to a register. */
-        assert(ts->val_type == TEMP_VAL_REG);
+        tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
         if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
             /* the mov can be suppressed */
             if (ots->val_type == TEMP_VAL_REG) {
-                s->reg_to_temp[ots->reg] = -1;
+                s->reg_to_temp[ots->reg] = NULL;
             }
             ots->reg = ts->reg;
-            temp_dead(s, args[1]);
+            temp_dead(s, ts);
         } else {
             if (ots->val_type != TEMP_VAL_REG) {
                 /* When allocating a new register, make sure to not spill the
                    input one. */
                 tcg_regset_set_reg(allocated_regs, ts->reg);
                 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
-                                         allocated_regs);
+                                         allocated_regs, ots->indirect_base);
             }
             tcg_out_mov(s, otype, ots->reg, ts->reg);
         }
         ots->val_type = TEMP_VAL_REG;
         ots->mem_coherent = 0;
-        s->reg_to_temp[ots->reg] = args[0];
+        s->reg_to_temp[ots->reg] = ots;
         if (NEED_SYNC_ARG(0)) {
-            tcg_reg_sync(s, ots->reg);
+            temp_sync(s, ots, allocated_regs, 0);
         }
     }
 }
 
 static void tcg_reg_alloc_op(TCGContext *s, 
                              const TCGOpDef *def, TCGOpcode opc,
-                             const TCGArg *args, uint16_t dead_args,
-                             uint8_t sync_args)
+                             const TCGArg *args, TCGLifeData arg_life)
 {
     TCGRegSet allocated_regs;
-    int i, k, nb_iargs, nb_oargs, reg;
+    int i, k, nb_iargs, nb_oargs;
+    TCGReg reg;
     TCGArg arg;
     const TCGArgConstraint *arg_ct;
     TCGTemp *ts;
@@ -1985,30 +2234,17 @@
         arg = args[i];
         arg_ct = &def->args_ct[i];
         ts = &s->temps[arg];
-        if (ts->val_type == TEMP_VAL_MEM) {
-            reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
-            tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-            ts->val_type = TEMP_VAL_REG;
-            ts->reg = reg;
-            ts->mem_coherent = 1;
-            s->reg_to_temp[reg] = arg;
-        } else if (ts->val_type == TEMP_VAL_CONST) {
-            if (tcg_target_const_match(ts->val, ts->type, arg_ct)) {
-                /* constant is OK for instruction */
-                const_args[i] = 1;
-                new_args[i] = ts->val;
-                goto iarg_end;
-            } else {
-                /* need to move to a register */
-                reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
-                tcg_out_movi(s, ts->type, reg, ts->val);
-                ts->val_type = TEMP_VAL_REG;
-                ts->reg = reg;
-                ts->mem_coherent = 0;
-                s->reg_to_temp[reg] = arg;
-            }
+
+        if (ts->val_type == TEMP_VAL_CONST
+            && tcg_target_const_match(ts->val, ts->type, arg_ct)) {
+            /* constant is OK for instruction */
+            const_args[i] = 1;
+            new_args[i] = ts->val;
+            goto iarg_end;
         }
-        assert(ts->val_type == TEMP_VAL_REG);
+
+        temp_load(s, ts, arg_ct->u.regs, allocated_regs);
+
         if (arg_ct->ct & TCG_CT_IALIAS) {
             if (ts->fixed_reg) {
                 /* if fixed register, we must allocate a new register
@@ -2041,7 +2277,8 @@
         allocate_in_reg:
             /* allocate a new register matching the constraint 
                and move the temporary register into it */
-            reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
+            reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
+                                ts->indirect_base);
             tcg_out_mov(s, ts->type, reg, ts->reg);
         }
         new_args[i] = reg;
@@ -2053,7 +2290,7 @@
     /* mark dead temporaries and free the associated registers */
     for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
         if (IS_DEAD_ARG(i)) {
-            temp_dead(s, args[i]);
+            temp_dead(s, &s->temps[args[i]]);
         }
     }
 
@@ -2062,9 +2299,9 @@
     } else {
         if (def->flags & TCG_OPF_CALL_CLOBBER) {
             /* XXX: permit generic clobber register list ? */ 
-            for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
-                if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) {
-                    tcg_reg_free(s, reg);
+            for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
+                if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
+                    tcg_reg_free(s, i, allocated_regs);
                 }
             }
         }
@@ -2090,20 +2327,21 @@
                     tcg_regset_test_reg(arg_ct->u.regs, reg)) {
                     goto oarg_end;
                 }
-                reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
+                reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs,
+                                    ts->indirect_base);
             }
             tcg_regset_set_reg(allocated_regs, reg);
             /* if a fixed register is used, then a move will be done afterwards */
             if (!ts->fixed_reg) {
                 if (ts->val_type == TEMP_VAL_REG) {
-                    s->reg_to_temp[ts->reg] = -1;
+                    s->reg_to_temp[ts->reg] = NULL;
                 }
                 ts->val_type = TEMP_VAL_REG;
                 ts->reg = reg;
                 /* temp value is modified, so the value kept in memory is
                    potentially not the same */
                 ts->mem_coherent = 0;
-                s->reg_to_temp[reg] = arg;
+                s->reg_to_temp[reg] = ts;
             }
         oarg_end:
             new_args[i] = reg;
@@ -2121,10 +2359,9 @@
             tcg_out_mov(s, ts->type, ts->reg, reg);
         }
         if (NEED_SYNC_ARG(i)) {
-            tcg_reg_sync(s, reg);
-        }
-        if (IS_DEAD_ARG(i)) {
-            temp_dead(s, args[i]);
+            temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i));
+        } else if (IS_DEAD_ARG(i)) {
+            temp_dead(s, ts);
         }
     }
 }
@@ -2136,10 +2373,10 @@
 #endif
 
 static void tcg_reg_alloc_call(TCGContext *s, int nb_oargs, int nb_iargs,
-                               const TCGArg * const args, uint16_t dead_args,
-                               uint8_t sync_args)
+                               const TCGArg * const args, TCGLifeData arg_life)
 {
-    int flags, nb_regs, i, reg;
+    int flags, nb_regs, i;
+    TCGReg reg;
     TCGArg arg;
     TCGTemp *ts;
     intptr_t stack_offset;
@@ -2175,23 +2412,9 @@
 #endif
         if (arg != TCG_CALL_DUMMY_ARG) {
             ts = &s->temps[arg];
-            if (ts->val_type == TEMP_VAL_REG) {
-                tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
-            } else if (ts->val_type == TEMP_VAL_MEM) {
-                reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 
-                                    s->reserved_regs);
-                /* XXX: not correct if reading values from the stack */
-                tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-                tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset);
-            } else if (ts->val_type == TEMP_VAL_CONST) {
-                reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 
-                                    s->reserved_regs);
-                /* XXX: sign extend may be needed on some targets */
-                tcg_out_movi(s, ts->type, reg, ts->val);
-                tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset);
-            } else {
-                tcg_abort();
-            }
+            temp_load(s, ts, tcg_target_available_regs[ts->type],
+                      s->reserved_regs);
+            tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
         }
 #ifndef TCG_TARGET_STACK_GROWSUP
         stack_offset += sizeof(tcg_target_long);
@@ -2205,19 +2428,20 @@
         if (arg != TCG_CALL_DUMMY_ARG) {
             ts = &s->temps[arg];
             reg = tcg_target_call_iarg_regs[i];
-            tcg_reg_free(s, reg);
+            tcg_reg_free(s, reg, allocated_regs);
+
             if (ts->val_type == TEMP_VAL_REG) {
                 if (ts->reg != reg) {
                     tcg_out_mov(s, ts->type, reg, ts->reg);
                 }
-            } else if (ts->val_type == TEMP_VAL_MEM) {
-                tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-            } else if (ts->val_type == TEMP_VAL_CONST) {
-                /* XXX: sign extend ? */
-                tcg_out_movi(s, ts->type, reg, ts->val);
             } else {
-                tcg_abort();
+                TCGRegSet arg_set;
+
+                tcg_regset_clear(arg_set);
+                tcg_regset_set_reg(arg_set, reg);
+                temp_load(s, ts, arg_set, allocated_regs);
             }
+
             tcg_regset_set_reg(allocated_regs, reg);
         }
     }
@@ -2225,14 +2449,14 @@
     /* mark dead temporaries and free the associated registers */
     for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
         if (IS_DEAD_ARG(i)) {
-            temp_dead(s, args[i]);
+            temp_dead(s, &s->temps[args[i]]);
         }
     }
     
     /* clobber call registers */
-    for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
-        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) {
-            tcg_reg_free(s, reg);
+    for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
+        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
+            tcg_reg_free(s, i, allocated_regs);
         }
     }
 
@@ -2253,7 +2477,7 @@
         arg = args[i];
         ts = &s->temps[arg];
         reg = tcg_target_call_oarg_regs[i];
-        assert(s->reg_to_temp[reg] == -1);
+        tcg_debug_assert(s->reg_to_temp[reg] == NULL);
 
         if (ts->fixed_reg) {
             if (ts->reg != reg) {
@@ -2261,17 +2485,16 @@
             }
         } else {
             if (ts->val_type == TEMP_VAL_REG) {
-                s->reg_to_temp[ts->reg] = -1;
+                s->reg_to_temp[ts->reg] = NULL;
             }
             ts->val_type = TEMP_VAL_REG;
             ts->reg = reg;
             ts->mem_coherent = 0;
-            s->reg_to_temp[reg] = arg;
+            s->reg_to_temp[reg] = ts;
             if (NEED_SYNC_ARG(i)) {
-                tcg_reg_sync(s, reg);
-            }
-            if (IS_DEAD_ARG(i)) {
-                temp_dead(s, args[i]);
+                temp_sync(s, ts, allocated_regs, IS_DEAD_ARG(i));
+            } else if (IS_DEAD_ARG(i)) {
+                temp_dead(s, ts);
             }
         }
     }
@@ -2281,25 +2504,48 @@
 
 static int64_t tcg_table_op_count[NB_OPS];
 
-static void dump_op_count(void)
+void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
 {
     int i;
 
     for (i = 0; i < NB_OPS; i++) {
-        qemu_log("%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]);
+        cpu_fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name,
+                    tcg_table_op_count[i]);
     }
 }
+#else
+void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf)
+{
+    cpu_fprintf(f, "[TCG profiler not compiled]\n");
+}
 #endif
 
 
-static inline int tcg_gen_code_common(TCGContext *s,
-                                      tcg_insn_unit *gen_code_buf,
-                                      long search_pc)
+int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
 {
-    int oi, oi_next;
+    int i, oi, oi_next, num_insns;
+
+#ifdef CONFIG_PROFILER
+    {
+        int n;
+
+        n = s->gen_op_buf[0].prev + 1;
+        s->op_count += n;
+        if (n > s->op_count_max) {
+            s->op_count_max = n;
+        }
+
+        n = s->nb_temps;
+        s->temp_count += n;
+        if (n > s->temp_count_max) {
+            s->temp_count_max = n;
+        }
+    }
+#endif
 
 #ifdef DEBUG_DISAS
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
+                 && qemu_log_in_addr_range(tb->pc))) {
         qemu_log("OP:\n");
         tcg_dump_ops(s);
         qemu_log("\n");
@@ -2319,14 +2565,35 @@
     s->la_time -= profile_getclock();
 #endif
 
-    tcg_liveness_analysis(s);
+    {
+        uint8_t *temp_state = tcg_malloc(s->nb_temps + s->nb_indirects);
+
+        liveness_pass_1(s, temp_state);
+
+        if (s->nb_indirects > 0) {
+#ifdef DEBUG_DISAS
+            if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
+                         && qemu_log_in_addr_range(tb->pc))) {
+                qemu_log("OP before indirect lowering:\n");
+                tcg_dump_ops(s);
+                qemu_log("\n");
+            }
+#endif
+            /* Replace indirect temps with direct temps.  */
+            if (liveness_pass_2(s, temp_state)) {
+                /* If changes were made, re-run liveness.  */
+                liveness_pass_1(s, temp_state);
+            }
+        }
+    }
 
 #ifdef CONFIG_PROFILER
     s->la_time += profile_getclock();
 #endif
 
 #ifdef DEBUG_DISAS
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
+                 && qemu_log_in_addr_range(tb->pc))) {
         qemu_log("OP after optimization and liveness analysis:\n");
         tcg_dump_ops(s);
         qemu_log("\n");
@@ -2335,18 +2602,18 @@
 
     tcg_reg_alloc_start(s);
 
-    s->code_buf = gen_code_buf;
-    s->code_ptr = gen_code_buf;
+    s->code_buf = tb->tc_ptr;
+    s->code_ptr = tb->tc_ptr;
 
     tcg_out_tb_init(s);
 
-    for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
+    num_insns = -1;
+    for (oi = s->gen_op_buf[0].next; oi != 0; oi = oi_next) {
         TCGOp * const op = &s->gen_op_buf[oi];
         TCGArg * const args = &s->gen_opparam_buf[op->args];
         TCGOpcode opc = op->opc;
         const TCGOpDef *def = &tcg_op_defs[opc];
-        uint16_t dead_args = s->op_dead_args[oi];
-        uint8_t sync_args = s->op_sync_args[oi];
+        TCGLifeData arg_life = op->life;
 
         oi_next = op->next;
 #ifdef CONFIG_PROFILER
@@ -2356,24 +2623,36 @@
         switch (opc) {
         case INDEX_op_mov_i32:
         case INDEX_op_mov_i64:
-            tcg_reg_alloc_mov(s, def, args, dead_args, sync_args);
+            tcg_reg_alloc_mov(s, def, args, arg_life);
             break;
         case INDEX_op_movi_i32:
         case INDEX_op_movi_i64:
-            tcg_reg_alloc_movi(s, args, dead_args, sync_args);
+            tcg_reg_alloc_movi(s, args, arg_life);
             break;
-        case INDEX_op_debug_insn_start:
+        case INDEX_op_insn_start:
+            if (num_insns >= 0) {
+                s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
+            }
+            num_insns++;
+            for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
+                target_ulong a;
+#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
+                a = ((target_ulong)args[i * 2 + 1] << 32) | args[i * 2];
+#else
+                a = args[i];
+#endif
+                s->gen_insn_data[num_insns][i] = a;
+            }
             break;
         case INDEX_op_discard:
-            temp_dead(s, args[0]);
+            temp_dead(s, &s->temps[args[0]]);
             break;
         case INDEX_op_set_label:
             tcg_reg_alloc_bb_end(s, s->reserved_regs);
-            tcg_out_label(s, args[0], s->code_ptr);
+            tcg_out_label(s, arg_label(args[0]), s->code_ptr);
             break;
         case INDEX_op_call:
-            tcg_reg_alloc_call(s, op->callo, op->calli, args,
-                               dead_args, sync_args);
+            tcg_reg_alloc_call(s, op->callo, op->calli, args, arg_life);
             break;
         default:
             /* Sanity check that we've not introduced any unhandled opcodes. */
@@ -2383,43 +2662,27 @@
             /* Note: in order to speed up the code, it would be much
                faster to have specialized register allocator functions for
                some common argument patterns */
-            tcg_reg_alloc_op(s, def, opc, args, dead_args, sync_args);
+            tcg_reg_alloc_op(s, def, opc, args, arg_life);
             break;
         }
-        if (search_pc >= 0 && search_pc < tcg_current_code_size(s)) {
-            return oi;
-        }
-#ifndef NDEBUG
+#ifdef CONFIG_DEBUG_TCG
         check_regs(s);
 #endif
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one operation beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           generating code without having to check during generation.  */
+        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
+            return -1;
+        }
     }
+    tcg_debug_assert(num_insns >= 0);
+    s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
 
     /* Generate TB finalization at the end of block */
-    tcg_out_tb_finalize(s);
-    return -1;
-}
-
-int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf)
-{
-#ifdef CONFIG_PROFILER
-    {
-        int n;
-
-        n = s->gen_last_op_idx + 1;
-        s->op_count += n;
-        if (n > s->op_count_max) {
-            s->op_count_max = n;
-        }
-
-        n = s->nb_temps;
-        s->temp_count += n;
-        if (n > s->temp_count_max) {
-            s->temp_count_max = n;
-        }
+    if (!tcg_out_tb_finalize(s)) {
+        return -1;
     }
-#endif
-
-    tcg_gen_code_common(s, gen_code_buf, -1);
 
     /* flush instruction cache */
     flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
@@ -2427,38 +2690,30 @@
     return tcg_current_code_size(s);
 }
 
-/* Return the index of the micro operation such as the pc after is <
-   offset bytes from the start of the TB.  The contents of gen_code_buf must
-   not be changed, though writing the same values is ok.
-   Return -1 if not found. */
-int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf,
-                           long offset)
-{
-    return tcg_gen_code_common(s, gen_code_buf, offset);
-}
-
 #ifdef CONFIG_PROFILER
 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
 {
     TCGContext *s = &tcg_ctx;
-    int64_t tot;
+    int64_t tb_count = s->tb_count;
+    int64_t tb_div_count = tb_count ? tb_count : 1;
+    int64_t tot = s->interm_time + s->code_time;
 
-    tot = s->interm_time + s->code_time;
     cpu_fprintf(f, "JIT cycles          %" PRId64 " (%0.3f s at 2.4 GHz)\n",
                 tot, tot / 2.4e9);
     cpu_fprintf(f, "translated TBs      %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 
-                s->tb_count, 
-                s->tb_count1 - s->tb_count,
-                s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0);
+                tb_count, s->tb_count1 - tb_count,
+                (double)(s->tb_count1 - s->tb_count)
+                / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
     cpu_fprintf(f, "avg ops/TB          %0.1f max=%d\n", 
-                s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max);
+                (double)s->op_count / tb_div_count, s->op_count_max);
     cpu_fprintf(f, "deleted ops/TB      %0.2f\n",
-                s->tb_count ? 
-                (double)s->del_op_count / s->tb_count : 0);
+                (double)s->del_op_count / tb_div_count);
     cpu_fprintf(f, "avg temps/TB        %0.2f max=%d\n",
-                s->tb_count ? 
-                (double)s->temp_count / s->tb_count : 0,
-                s->temp_count_max);
+                (double)s->temp_count / tb_div_count, s->temp_count_max);
+    cpu_fprintf(f, "avg host code/TB    %0.1f\n",
+                (double)s->code_out_len / tb_div_count);
+    cpu_fprintf(f, "avg search data/TB  %0.1f\n",
+                (double)s->search_out_len / tb_div_count);
     
     cpu_fprintf(f, "cycles/op           %0.1f\n", 
                 s->op_count ? (double)tot / s->op_count : 0);
@@ -2466,8 +2721,11 @@
                 s->code_in_len ? (double)tot / s->code_in_len : 0);
     cpu_fprintf(f, "cycles/out byte     %0.1f\n", 
                 s->code_out_len ? (double)tot / s->code_out_len : 0);
-    if (tot == 0)
+    cpu_fprintf(f, "cycles/search byte     %0.1f\n",
+                s->search_out_len ? (double)tot / s->search_out_len : 0);
+    if (tot == 0) {
         tot = 1;
+    }
     cpu_fprintf(f, "  gen_interm time   %0.1f%%\n", 
                 (double)s->interm_time / tot * 100.0);
     cpu_fprintf(f, "  gen_code time     %0.1f%%\n", 
@@ -2481,8 +2739,6 @@
                 s->restore_count);
     cpu_fprintf(f, "  avg cycles        %0.1f\n",
                 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
-
-    dump_op_count();
 }
 #else
 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
diff --git a/tcg/tcg.h b/tcg/tcg.h
index b27ee14..1bcabca 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -26,9 +26,34 @@
 #define TCG_H
 
 #include "qemu-common.h"
+#include "cpu.h"
+#include "exec/tb-context.h"
 #include "qemu/bitops.h"
 #include "tcg-target.h"
 
+/* XXX: make safe guess about sizes */
+#define MAX_OP_PER_INSTR 266
+
+#if HOST_LONG_BITS == 32
+#define MAX_OPC_PARAM_PER_ARG 2
+#else
+#define MAX_OPC_PARAM_PER_ARG 1
+#endif
+#define MAX_OPC_PARAM_IARGS 5
+#define MAX_OPC_PARAM_OARGS 1
+#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
+
+/* A Call op needs up to 4 + 2N parameters on 32-bit archs,
+ * and up to 4 + N parameters on 64-bit archs
+ * (N = number of input arguments + output arguments).  */
+#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS))
+#define OPC_BUF_SIZE 640
+#define OPC_MAX_SIZE (OPC_BUF_SIZE - MAX_OP_PER_INSTR)
+
+#define OPPARAM_BUF_SIZE (OPC_BUF_SIZE * MAX_OPC_PARAM)
+
+#define CPU_TEMP_BUF_NLONGS 128
+
 /* Default target word size to pointer size.  */
 #ifndef TCG_TARGET_REG_BITS
 # if UINTPTR_MAX == UINT32_MAX
@@ -64,7 +89,8 @@
 
 #if TCG_TARGET_REG_BITS == 32
 /* Turn some undef macros into false macros.  */
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_div_i64          0
 #define TCG_TARGET_HAS_rem_i64          0
 #define TCG_TARGET_HAS_div2_i64         0
@@ -126,6 +152,12 @@
 # error "Missing unsigned widening multiply"
 #endif
 
+#ifndef TARGET_INSN_START_EXTRA_WORDS
+# define TARGET_INSN_START_WORDS 1
+#else
+# define TARGET_INSN_START_WORDS (1 + TARGET_INSN_START_EXTRA_WORDS)
+#endif
+
 typedef enum TCGOpcode {
 #define DEF(name, oargs, iargs, cargs, flags) INDEX_op_ ## name,
 #include "tcg-opc.h"
@@ -159,7 +191,7 @@
 #endif
 
 
-#ifdef CONFIG_DEBUG_TCG
+#if defined CONFIG_DEBUG_TCG || defined QEMU_STATIC_ANALYSIS
 # define tcg_debug_assert(X) do { assert(X); } while (0)
 #elif QEMU_GNUC_PREREQ(4, 5)
 # define tcg_debug_assert(X) \
@@ -176,7 +208,8 @@
 } TCGRelocation; 
 
 typedef struct TCGLabel {
-    int has_value;
+    unsigned has_value : 1;
+    unsigned id : 31;
     union {
         uintptr_t value;
         tcg_insn_unit *value_ptr;
@@ -192,9 +225,8 @@
 
 #define TCG_POOL_CHUNK_SIZE 32768
 
-#define TCG_MAX_LABELS 512
-
 #define TCG_MAX_TEMPS 512
+#define TCG_MAX_INSNS 512
 
 /* when the size of the arguments of a called function is smaller than
    this value, they are statically allocated in the TB stack frame */
@@ -369,6 +401,14 @@
 typedef struct TCGv_i32_d *TCGv_i32;
 typedef struct TCGv_i64_d *TCGv_i64;
 typedef struct TCGv_ptr_d *TCGv_ptr;
+typedef TCGv_ptr TCGv_env;
+#if TARGET_LONG_BITS == 32
+#define TCGv TCGv_i32
+#elif TARGET_LONG_BITS == 64
+#define TCGv TCGv_i64
+#else
+#error Unhandled TARGET_LONG_BITS value
+#endif
 
 static inline TCGv_i32 QEMU_ARTIFICIAL MAKE_TCGV_I32(intptr_t i)
 {
@@ -501,27 +541,31 @@
     }
 }
 
-#define TEMP_VAL_DEAD  0
-#define TEMP_VAL_REG   1
-#define TEMP_VAL_MEM   2
-#define TEMP_VAL_CONST 3
+typedef enum TCGTempVal {
+    TEMP_VAL_DEAD,
+    TEMP_VAL_REG,
+    TEMP_VAL_MEM,
+    TEMP_VAL_CONST,
+} TCGTempVal;
 
-/* XXX: optimize memory layout */
 typedef struct TCGTemp {
-    TCGType base_type;
-    TCGType type;
-    int val_type;
-    int reg;
-    tcg_target_long val;
-    int mem_reg;
-    intptr_t mem_offset;
+    TCGReg reg:8;
+    TCGTempVal val_type:8;
+    TCGType base_type:8;
+    TCGType type:8;
     unsigned int fixed_reg:1;
+    unsigned int indirect_reg:1;
+    unsigned int indirect_base:1;
     unsigned int mem_coherent:1;
     unsigned int mem_allocated:1;
     unsigned int temp_local:1; /* If true, the temp is saved across
                                   basic blocks. Otherwise, it is not
                                   preserved across basic blocks. */
     unsigned int temp_allocated:1; /* never used for code gen */
+
+    tcg_target_long val;
+    struct TCGTemp *mem_base;
+    intptr_t mem_offset;
     const char *name;
 } TCGTemp;
 
@@ -531,24 +575,41 @@
     unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)];
 } TCGTempSet;
 
+/* While we limit helpers to 6 arguments, for 32-bit hosts, with padding,
+   this imples a max of 6*2 (64-bit in) + 2 (64-bit out) = 14 operands.
+   There are never more than 2 outputs, which means that we can store all
+   dead + sync data within 16 bits.  */
+#define DEAD_ARG  4
+#define SYNC_ARG  1
+typedef uint16_t TCGLifeData;
+
+/* The layout here is designed to avoid crossing of a 32-bit boundary.
+   If we do so, gcc adds padding, expanding the size to 12.  */
 typedef struct TCGOp {
-    TCGOpcode opc   : 8;
+    TCGOpcode opc   : 8;        /*  8 */
+
+    /* Index of the prev/next op, or 0 for the end of the list.  */
+    unsigned prev   : 10;       /* 18 */
+    unsigned next   : 10;       /* 28 */
 
     /* The number of out and in parameter for a call.  */
-    unsigned callo  : 2;
-    unsigned calli  : 6;
+    unsigned calli  : 4;        /* 32 */
+    unsigned callo  : 2;        /* 34 */
 
-    /* Index of the arguments for this op, or -1 for zero-operand ops.  */
-    signed args     : 16;
+    /* Index of the arguments for this op, or 0 for zero-operand ops.  */
+    unsigned args   : 14;       /* 48 */
 
-    /* Index of the prex/next op, or -1 for the end of the list.  */
-    signed prev     : 16;
-    signed next     : 16;
+    /* Lifetime data of the operands.  */
+    unsigned life   : 16;       /* 64 */
 } TCGOp;
 
-QEMU_BUILD_BUG_ON(NB_OPS > 0xff);
-QEMU_BUILD_BUG_ON(OPC_BUF_SIZE >= 0x7fff);
-QEMU_BUILD_BUG_ON(OPPARAM_BUF_SIZE >= 0x7fff);
+/* Make sure operands fit in the bitfields above.  */
+QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8));
+QEMU_BUILD_BUG_ON(OPC_BUF_SIZE > (1 << 10));
+QEMU_BUILD_BUG_ON(OPPARAM_BUF_SIZE > (1 << 14));
+
+/* Make sure that we don't overflow 64 bits without noticing.  */
+QEMU_BUILD_BUG_ON(sizeof(TCGOp) > 8);
 
 struct TCGContext {
     uint8_t *pool_cur, *pool_end;
@@ -556,25 +617,19 @@
     int nb_labels;
     int nb_globals;
     int nb_temps;
+    int nb_indirects;
 
     /* goto_tb support */
     tcg_insn_unit *code_buf;
-    uintptr_t *tb_next;
-    uint16_t *tb_next_offset;
-    uint16_t *tb_jmp_offset; /* != NULL if USE_DIRECT_JUMP */
+    uint16_t *tb_jmp_reset_offset; /* tb->jmp_reset_offset */
+    uint16_t *tb_jmp_insn_offset; /* tb->jmp_insn_offset if USE_DIRECT_JUMP */
+    uintptr_t *tb_jmp_target_addr; /* tb->jmp_target_addr if !USE_DIRECT_JUMP */
 
-    /* liveness analysis */
-    uint16_t *op_dead_args; /* for each operation, each bit tells if the
-                               corresponding argument is dead */
-    uint8_t *op_sync_args;  /* for each operation, each bit tells if the
-                               corresponding output argument needs to be
-                               sync to memory. */
-    
     TCGRegSet reserved_regs;
     intptr_t current_frame_offset;
     intptr_t frame_start;
     intptr_t frame_end;
-    int frame_reg;
+    TCGTemp *frame_temp;
 
     tcg_insn_unit *code_ptr;
 
@@ -591,6 +646,7 @@
     int64_t del_op_count;
     int64_t code_in_len;
     int64_t code_out_len;
+    int64_t search_out_len;
     int64_t interm_time;
     int64_t code_time;
     int64_t la_time;
@@ -604,8 +660,6 @@
     int goto_tb_issue_mask;
 #endif
 
-    int gen_first_op_idx;
-    int gen_last_op_idx;
     int gen_next_op_idx;
     int gen_next_parm_idx;
 
@@ -617,34 +671,42 @@
     void *code_gen_prologue;
     void *code_gen_buffer;
     size_t code_gen_buffer_size;
-    /* threshold to flush the translated code buffer */
-    size_t code_gen_buffer_max_size;
     void *code_gen_ptr;
 
+    /* Threshold to flush the translated code buffer.  */
+    void *code_gen_highwater;
+
     TBContext tb_ctx;
 
-    /* The TCGBackendData structure is private to tcg-target.c.  */
+    /* Track which vCPU triggers events */
+    CPUState *cpu;                      /* *_trans */
+    TCGv_env tcg_env;                   /* *_exec  */
+
+    /* The TCGBackendData structure is private to tcg-target.inc.c.  */
     struct TCGBackendData *be;
 
     TCGTempSet free_temps[TCG_TYPE_COUNT * 2];
     TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
 
-    /* tells in which temporary a given register is. It does not take
-       into account fixed registers */
-    int reg_to_temp[TCG_TARGET_NB_REGS];
+    /* Tells which temporary holds a given register.
+       It does not take into account fixed registers */
+    TCGTemp *reg_to_temp[TCG_TARGET_NB_REGS];
 
     TCGOp gen_op_buf[OPC_BUF_SIZE];
     TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
 
-    target_ulong gen_opc_pc[OPC_BUF_SIZE];
-    uint16_t gen_opc_icount[OPC_BUF_SIZE];
-    uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
-
-    TCGLabel labels[TCG_MAX_LABELS];
+    uint16_t gen_insn_end_off[TCG_MAX_INSNS];
+    target_ulong gen_insn_data[TCG_MAX_INSNS][TARGET_INSN_START_WORDS];
 };
 
 extern TCGContext tcg_ctx;
 
+static inline void tcg_set_insn_param(int op_idx, int arg, TCGArg v)
+{
+    int op_argi = tcg_ctx.gen_op_buf[op_idx].args;
+    tcg_ctx.gen_opparam_buf[op_argi + arg] = v;
+}
+
 /* The number of opcodes emitted so far.  */
 static inline int tcg_op_buf_count(void)
 {
@@ -663,6 +725,10 @@
 void tcg_pool_reset(TCGContext *s);
 void tcg_pool_delete(TCGContext *s);
 
+void tb_lock(void);
+void tb_unlock(void);
+void tb_lock_reset(void);
+
 static inline void *tcg_malloc(int size)
 {
     TCGContext *s = &tcg_ctx;
@@ -682,39 +748,54 @@
 void tcg_prologue_init(TCGContext *s);
 void tcg_func_start(TCGContext *s);
 
-int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf);
-int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf,
-                           long offset);
+int tcg_gen_code(TCGContext *s, TranslationBlock *tb);
 
-void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size);
+void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size);
 
-TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name);
-TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name);
+int tcg_global_mem_new_internal(TCGType, TCGv_ptr, intptr_t, const char *);
+
+TCGv_i32 tcg_global_reg_new_i32(TCGReg reg, const char *name);
+TCGv_i64 tcg_global_reg_new_i64(TCGReg reg, const char *name);
+
 TCGv_i32 tcg_temp_new_internal_i32(int temp_local);
+TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
+
+void tcg_temp_free_i32(TCGv_i32 arg);
+void tcg_temp_free_i64(TCGv_i64 arg);
+
+static inline TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t offset,
+                                              const char *name)
+{
+    int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
+    return MAKE_TCGV_I32(idx);
+}
+
 static inline TCGv_i32 tcg_temp_new_i32(void)
 {
     return tcg_temp_new_internal_i32(0);
 }
+
 static inline TCGv_i32 tcg_temp_local_new_i32(void)
 {
     return tcg_temp_new_internal_i32(1);
 }
-void tcg_temp_free_i32(TCGv_i32 arg);
-char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg);
 
-TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name);
-TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name);
-TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
+static inline TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t offset,
+                                              const char *name)
+{
+    int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
+    return MAKE_TCGV_I64(idx);
+}
+
 static inline TCGv_i64 tcg_temp_new_i64(void)
 {
     return tcg_temp_new_internal_i64(0);
 }
+
 static inline TCGv_i64 tcg_temp_local_new_i64(void)
 {
     return tcg_temp_new_internal_i64(1);
 }
-void tcg_temp_free_i64(TCGv_i64 arg);
-char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg);
 
 #if defined(CONFIG_DEBUG_TCG)
 /* If you call tcg_clear_temp_count() at the start of a section of
@@ -730,6 +811,7 @@
 #endif
 
 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf);
+void tcg_dump_op_count(FILE *f, fprintf_function cpu_fprintf);
 
 #define TCG_CT_ALIAS  0x80
 #define TCG_CT_IALIAS 0x40
@@ -817,6 +899,9 @@
                    TCGArg ret, int nargs, TCGArg *args);
 
 void tcg_op_remove(TCGContext *s, TCGOp *op);
+TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc, int narg);
+TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc, int narg);
+
 void tcg_optimize(TCGContext *s);
 
 /* only used for debugging purposes */
@@ -828,6 +913,33 @@
 TCGv_i32 tcg_const_local_i32(int32_t val);
 TCGv_i64 tcg_const_local_i64(int64_t val);
 
+TCGLabel *gen_new_label(void);
+
+/**
+ * label_arg
+ * @l: label
+ *
+ * Encode a label for storage in the TCG opcode stream.
+ */
+
+static inline TCGArg label_arg(TCGLabel *l)
+{
+    return (uintptr_t)l;
+}
+
+/**
+ * arg_label
+ * @i: value
+ *
+ * The opposite of label_arg.  Retrieve a label from the
+ * encoding of the TCG opcode stream.
+ */
+
+static inline TCGLabel *arg_label(TCGArg i)
+{
+    return (TCGLabel *)(uintptr_t)i;
+}
+
 /**
  * tcg_ptr_byte_diff
  * @a, @b: addresses to be differenced
@@ -913,7 +1025,7 @@
 
 /**
  * tcg_qemu_tb_exec:
- * @env: CPUArchState * for the CPU
+ * @env: pointer to CPUArchState for the CPU
  * @tb_ptr: address of generated code for the TB to execute
  *
  * Start executing code from a given translation block.
@@ -924,30 +1036,33 @@
  * which has not yet been directly linked, or an asynchronous
  * event such as an interrupt needs handling.
  *
- * The return value is a pointer to the next TB to execute
- * (if known; otherwise zero). This pointer is assumed to be
- * 4-aligned, and the bottom two bits are used to return further
- * information:
+ * Return: The return value is the value passed to the corresponding
+ * tcg_gen_exit_tb() at translation time of the last TB attempted to execute.
+ * The value is either zero or a 4-byte aligned pointer to that TB combined
+ * with additional information in its two least significant bits. The
+ * additional information is encoded as follows:
  *  0, 1: the link between this TB and the next is via the specified
  *        TB index (0 or 1). That is, we left the TB via (the equivalent
  *        of) "goto_tb <index>". The main loop uses this to determine
  *        how to link the TB just executed to the next.
  *  2:    we are using instruction counting code generation, and we
  *        did not start executing this TB because the instruction counter
- *        would hit zero midway through it. In this case the next-TB pointer
+ *        would hit zero midway through it. In this case the pointer
  *        returned is the TB we were about to execute, and the caller must
  *        arrange to execute the remaining count of instructions.
  *  3:    we stopped because the CPU's exit_request flag was set
  *        (usually meaning that there is an interrupt that needs to be
- *        handled). The next-TB pointer returned is the TB we were
- *        about to execute when we noticed the pending exit request.
+ *        handled). The pointer returned is the TB we were about to execute
+ *        when we noticed the pending exit request.
  *
  * If the bottom two bits indicate an exit-via-index then the CPU
  * state is correctly synchronised and ready for execution of the next
  * TB (and in particular the guest PC is the address to execute next).
  * Otherwise, we gave up on execution of this TB before it started, and
- * the caller must fix up the CPU state by calling cpu_pc_from_tb()
- * with the next-TB pointer we return.
+ * the caller must fix up the CPU state by calling the CPU's
+ * synchronize_from_tb() method with the TB pointer we return (falling
+ * back to calling the CPU's set_pc method with tb->pb if no
+ * synchronize_from_tb() method exists).
  *
  * Note that TCG targets may use a different definition of tcg_qemu_tb_exec
  * to this default (which just calls the prologue.code emitted by
@@ -959,7 +1074,9 @@
 #define TB_EXIT_ICOUNT_EXPIRED 2
 #define TB_EXIT_REQUESTED 3
 
-#if !defined(tcg_qemu_tb_exec)
+#ifdef HAVE_TCG_QEMU_TB_EXEC
+uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
+#else
 # define tcg_qemu_tb_exec(env, tb_ptr) \
     ((uintptr_t (*)(void *, void *))tcg_ctx.code_gen_prologue)(env, tb_ptr)
 #endif
@@ -1013,25 +1130,48 @@
 void helper_be_stq_mmu(CPUArchState *env, target_ulong addr, uint64_t val,
                        TCGMemOpIdx oi, uintptr_t retaddr);
 
+uint8_t helper_ret_ldb_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint16_t helper_le_ldw_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint32_t helper_le_ldl_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_le_ldq_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint16_t helper_be_ldw_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint32_t helper_be_ldl_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+uint64_t helper_be_ldq_cmmu(CPUArchState *env, target_ulong addr,
+                            TCGMemOpIdx oi, uintptr_t retaddr);
+
 /* Temporary aliases until backends are converted.  */
 #ifdef TARGET_WORDS_BIGENDIAN
 # define helper_ret_ldsw_mmu  helper_be_ldsw_mmu
 # define helper_ret_lduw_mmu  helper_be_lduw_mmu
 # define helper_ret_ldsl_mmu  helper_be_ldsl_mmu
 # define helper_ret_ldul_mmu  helper_be_ldul_mmu
+# define helper_ret_ldl_mmu   helper_be_ldul_mmu
 # define helper_ret_ldq_mmu   helper_be_ldq_mmu
 # define helper_ret_stw_mmu   helper_be_stw_mmu
 # define helper_ret_stl_mmu   helper_be_stl_mmu
 # define helper_ret_stq_mmu   helper_be_stq_mmu
+# define helper_ret_ldw_cmmu  helper_be_ldw_cmmu
+# define helper_ret_ldl_cmmu  helper_be_ldl_cmmu
+# define helper_ret_ldq_cmmu  helper_be_ldq_cmmu
 #else
 # define helper_ret_ldsw_mmu  helper_le_ldsw_mmu
 # define helper_ret_lduw_mmu  helper_le_lduw_mmu
 # define helper_ret_ldsl_mmu  helper_le_ldsl_mmu
 # define helper_ret_ldul_mmu  helper_le_ldul_mmu
+# define helper_ret_ldl_mmu   helper_le_ldul_mmu
 # define helper_ret_ldq_mmu   helper_le_ldq_mmu
 # define helper_ret_stw_mmu   helper_le_stw_mmu
 # define helper_ret_stl_mmu   helper_le_stl_mmu
 # define helper_ret_stq_mmu   helper_le_stq_mmu
+# define helper_ret_ldw_cmmu  helper_le_ldw_cmmu
+# define helper_ret_ldl_cmmu  helper_le_ldl_cmmu
+# define helper_ret_ldq_cmmu  helper_le_ldq_cmmu
 #endif
 
 #endif /* CONFIG_SOFTMMU */
diff --git a/tcg/tci/README b/tcg/tci/README
index dc57f07..3786b09 100644
--- a/tcg/tci/README
+++ b/tcg/tci/README
@@ -21,7 +21,7 @@
 2) Implementation
 
 Like each TCG host frontend, TCI implements the code generator in
-tcg-target.c, tcg-target.h. Both files are in directory tcg/tci.
+tcg-target.inc.c, tcg-target.h. Both files are in directory tcg/tci.
 
 The additional file tcg/tci.c adds the interpreter.
 
@@ -123,7 +123,7 @@
   would also improve speed for hosts which support byte alignment).
 
 * A better disassembler for the pseudo code would be nice (a very primitive
-  disassembler is included in tcg-target.c).
+  disassembler is included in tcg-target.inc.c).
 
 * It might be useful to have a runtime option which selects the native TCG
   or TCI, so QEMU would have to include two TCGs. Today, selecting TCI
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
deleted file mode 100644
index def4311..0000000
--- a/tcg/tci/tcg-target.c
+++ /dev/null
@@ -1,882 +0,0 @@
-/*
- * Tiny Code Generator for QEMU
- *
- * Copyright (c) 2009, 2011 Stefan Weil
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "tcg-be-null.h"
-
-/* TODO list:
- * - See TODO comments in code.
- */
-
-/* Marker for missing code. */
-#define TODO() \
-    do { \
-        fprintf(stderr, "TODO %s:%u: %s()\n", \
-                __FILE__, __LINE__, __func__); \
-        tcg_abort(); \
-    } while (0)
-
-/* Bitfield n...m (in 32 bit value). */
-#define BITS(n, m) (((0xffffffffU << (31 - n)) >> (31 - n + m)) << m)
-
-/* Macros used in tcg_target_op_defs. */
-#define R       "r"
-#define RI      "ri"
-#if TCG_TARGET_REG_BITS == 32
-# define R64    "r", "r"
-#else
-# define R64    "r"
-#endif
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-# define L      "L", "L"
-# define S      "S", "S"
-#else
-# define L      "L"
-# define S      "S"
-#endif
-
-/* TODO: documentation. */
-static const TCGTargetOpDef tcg_target_op_defs[] = {
-    { INDEX_op_exit_tb, { NULL } },
-    { INDEX_op_goto_tb, { NULL } },
-    { INDEX_op_br, { NULL } },
-
-    { INDEX_op_ld8u_i32, { R, R } },
-    { INDEX_op_ld8s_i32, { R, R } },
-    { INDEX_op_ld16u_i32, { R, R } },
-    { INDEX_op_ld16s_i32, { R, R } },
-    { INDEX_op_ld_i32, { R, R } },
-    { INDEX_op_st8_i32, { R, R } },
-    { INDEX_op_st16_i32, { R, R } },
-    { INDEX_op_st_i32, { R, R } },
-
-    { INDEX_op_add_i32, { R, RI, RI } },
-    { INDEX_op_sub_i32, { R, RI, RI } },
-    { INDEX_op_mul_i32, { R, RI, RI } },
-#if TCG_TARGET_HAS_div_i32
-    { INDEX_op_div_i32, { R, R, R } },
-    { INDEX_op_divu_i32, { R, R, R } },
-    { INDEX_op_rem_i32, { R, R, R } },
-    { INDEX_op_remu_i32, { R, R, R } },
-#elif TCG_TARGET_HAS_div2_i32
-    { INDEX_op_div2_i32, { R, R, "0", "1", R } },
-    { INDEX_op_divu2_i32, { R, R, "0", "1", R } },
-#endif
-    /* TODO: Does R, RI, RI result in faster code than R, R, RI?
-       If both operands are constants, we can optimize. */
-    { INDEX_op_and_i32, { R, RI, RI } },
-#if TCG_TARGET_HAS_andc_i32
-    { INDEX_op_andc_i32, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_eqv_i32
-    { INDEX_op_eqv_i32, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_nand_i32
-    { INDEX_op_nand_i32, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_nor_i32
-    { INDEX_op_nor_i32, { R, RI, RI } },
-#endif
-    { INDEX_op_or_i32, { R, RI, RI } },
-#if TCG_TARGET_HAS_orc_i32
-    { INDEX_op_orc_i32, { R, RI, RI } },
-#endif
-    { INDEX_op_xor_i32, { R, RI, RI } },
-    { INDEX_op_shl_i32, { R, RI, RI } },
-    { INDEX_op_shr_i32, { R, RI, RI } },
-    { INDEX_op_sar_i32, { R, RI, RI } },
-#if TCG_TARGET_HAS_rot_i32
-    { INDEX_op_rotl_i32, { R, RI, RI } },
-    { INDEX_op_rotr_i32, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_deposit_i32
-    { INDEX_op_deposit_i32, { R, "0", R } },
-#endif
-
-    { INDEX_op_brcond_i32, { R, RI } },
-
-    { INDEX_op_setcond_i32, { R, R, RI } },
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_setcond_i64, { R, R, RI } },
-#endif /* TCG_TARGET_REG_BITS == 64 */
-
-#if TCG_TARGET_REG_BITS == 32
-    /* TODO: Support R, R, R, R, RI, RI? Will it be faster? */
-    { INDEX_op_add2_i32, { R, R, R, R, R, R } },
-    { INDEX_op_sub2_i32, { R, R, R, R, R, R } },
-    { INDEX_op_brcond2_i32, { R, R, RI, RI } },
-    { INDEX_op_mulu2_i32, { R, R, R, R } },
-    { INDEX_op_setcond2_i32, { R, R, R, RI, RI } },
-#endif
-
-#if TCG_TARGET_HAS_not_i32
-    { INDEX_op_not_i32, { R, R } },
-#endif
-#if TCG_TARGET_HAS_neg_i32
-    { INDEX_op_neg_i32, { R, R } },
-#endif
-
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_ld8u_i64, { R, R } },
-    { INDEX_op_ld8s_i64, { R, R } },
-    { INDEX_op_ld16u_i64, { R, R } },
-    { INDEX_op_ld16s_i64, { R, R } },
-    { INDEX_op_ld32u_i64, { R, R } },
-    { INDEX_op_ld32s_i64, { R, R } },
-    { INDEX_op_ld_i64, { R, R } },
-
-    { INDEX_op_st8_i64, { R, R } },
-    { INDEX_op_st16_i64, { R, R } },
-    { INDEX_op_st32_i64, { R, R } },
-    { INDEX_op_st_i64, { R, R } },
-
-    { INDEX_op_add_i64, { R, RI, RI } },
-    { INDEX_op_sub_i64, { R, RI, RI } },
-    { INDEX_op_mul_i64, { R, RI, RI } },
-#if TCG_TARGET_HAS_div_i64
-    { INDEX_op_div_i64, { R, R, R } },
-    { INDEX_op_divu_i64, { R, R, R } },
-    { INDEX_op_rem_i64, { R, R, R } },
-    { INDEX_op_remu_i64, { R, R, R } },
-#elif TCG_TARGET_HAS_div2_i64
-    { INDEX_op_div2_i64, { R, R, "0", "1", R } },
-    { INDEX_op_divu2_i64, { R, R, "0", "1", R } },
-#endif
-    { INDEX_op_and_i64, { R, RI, RI } },
-#if TCG_TARGET_HAS_andc_i64
-    { INDEX_op_andc_i64, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_eqv_i64
-    { INDEX_op_eqv_i64, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_nand_i64
-    { INDEX_op_nand_i64, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_nor_i64
-    { INDEX_op_nor_i64, { R, RI, RI } },
-#endif
-    { INDEX_op_or_i64, { R, RI, RI } },
-#if TCG_TARGET_HAS_orc_i64
-    { INDEX_op_orc_i64, { R, RI, RI } },
-#endif
-    { INDEX_op_xor_i64, { R, RI, RI } },
-    { INDEX_op_shl_i64, { R, RI, RI } },
-    { INDEX_op_shr_i64, { R, RI, RI } },
-    { INDEX_op_sar_i64, { R, RI, RI } },
-#if TCG_TARGET_HAS_rot_i64
-    { INDEX_op_rotl_i64, { R, RI, RI } },
-    { INDEX_op_rotr_i64, { R, RI, RI } },
-#endif
-#if TCG_TARGET_HAS_deposit_i64
-    { INDEX_op_deposit_i64, { R, "0", R } },
-#endif
-    { INDEX_op_brcond_i64, { R, RI } },
-
-#if TCG_TARGET_HAS_ext8s_i64
-    { INDEX_op_ext8s_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext16s_i64
-    { INDEX_op_ext16s_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext32s_i64
-    { INDEX_op_ext32s_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext8u_i64
-    { INDEX_op_ext8u_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext16u_i64
-    { INDEX_op_ext16u_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext32u_i64
-    { INDEX_op_ext32u_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_bswap16_i64
-    { INDEX_op_bswap16_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_bswap32_i64
-    { INDEX_op_bswap32_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_bswap64_i64
-    { INDEX_op_bswap64_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_not_i64
-    { INDEX_op_not_i64, { R, R } },
-#endif
-#if TCG_TARGET_HAS_neg_i64
-    { INDEX_op_neg_i64, { R, R } },
-#endif
-#endif /* TCG_TARGET_REG_BITS == 64 */
-
-    { INDEX_op_qemu_ld_i32, { R, L } },
-    { INDEX_op_qemu_ld_i64, { R64, L } },
-
-    { INDEX_op_qemu_st_i32, { R, S } },
-    { INDEX_op_qemu_st_i64, { R64, S } },
-
-#if TCG_TARGET_HAS_ext8s_i32
-    { INDEX_op_ext8s_i32, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext16s_i32
-    { INDEX_op_ext16s_i32, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext8u_i32
-    { INDEX_op_ext8u_i32, { R, R } },
-#endif
-#if TCG_TARGET_HAS_ext16u_i32
-    { INDEX_op_ext16u_i32, { R, R } },
-#endif
-
-#if TCG_TARGET_HAS_bswap16_i32
-    { INDEX_op_bswap16_i32, { R, R } },
-#endif
-#if TCG_TARGET_HAS_bswap32_i32
-    { INDEX_op_bswap32_i32, { R, R } },
-#endif
-
-    { -1 },
-};
-
-static const int tcg_target_reg_alloc_order[] = {
-    TCG_REG_R0,
-    TCG_REG_R1,
-    TCG_REG_R2,
-    TCG_REG_R3,
-#if 0 /* used for TCG_REG_CALL_STACK */
-    TCG_REG_R4,
-#endif
-    TCG_REG_R5,
-    TCG_REG_R6,
-    TCG_REG_R7,
-#if TCG_TARGET_NB_REGS >= 16
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-    TCG_REG_R11,
-    TCG_REG_R12,
-    TCG_REG_R13,
-    TCG_REG_R14,
-    TCG_REG_R15,
-#endif
-};
-
-#if MAX_OPC_PARAM_IARGS != 5
-# error Fix needed, number of supported input arguments changed!
-#endif
-
-static const int tcg_target_call_iarg_regs[] = {
-    TCG_REG_R0,
-    TCG_REG_R1,
-    TCG_REG_R2,
-    TCG_REG_R3,
-#if 0 /* used for TCG_REG_CALL_STACK */
-    TCG_REG_R4,
-#endif
-    TCG_REG_R5,
-#if TCG_TARGET_REG_BITS == 32
-    /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */
-    TCG_REG_R6,
-    TCG_REG_R7,
-#if TCG_TARGET_NB_REGS >= 16
-    TCG_REG_R8,
-    TCG_REG_R9,
-    TCG_REG_R10,
-#else
-# error Too few input registers available
-#endif
-#endif
-};
-
-static const int tcg_target_call_oarg_regs[] = {
-    TCG_REG_R0,
-#if TCG_TARGET_REG_BITS == 32
-    TCG_REG_R1
-#endif
-};
-
-#ifndef NDEBUG
-static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
-    "r00",
-    "r01",
-    "r02",
-    "r03",
-    "r04",
-    "r05",
-    "r06",
-    "r07",
-#if TCG_TARGET_NB_REGS >= 16
-    "r08",
-    "r09",
-    "r10",
-    "r11",
-    "r12",
-    "r13",
-    "r14",
-    "r15",
-#if TCG_TARGET_NB_REGS >= 32
-    "r16",
-    "r17",
-    "r18",
-    "r19",
-    "r20",
-    "r21",
-    "r22",
-    "r23",
-    "r24",
-    "r25",
-    "r26",
-    "r27",
-    "r28",
-    "r29",
-    "r30",
-    "r31"
-#endif
-#endif
-};
-#endif
-
-static void patch_reloc(tcg_insn_unit *code_ptr, int type,
-                        intptr_t value, intptr_t addend)
-{
-    /* tcg_out_reloc always uses the same type, addend. */
-    assert(type == sizeof(tcg_target_long));
-    assert(addend == 0);
-    assert(value != 0);
-    if (TCG_TARGET_REG_BITS == 32) {
-        tcg_patch32(code_ptr, value);
-    } else {
-        tcg_patch64(code_ptr, value);
-    }
-}
-
-/* Parse target specific constraints. */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'r':
-    case 'L':                   /* qemu_ld constraint */
-    case 'S':                   /* qemu_st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, BIT(TCG_TARGET_NB_REGS) - 1);
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-#if defined(CONFIG_DEBUG_TCG_INTERPRETER)
-/* Show current bytecode. Used by tcg interpreter. */
-void tci_disas(uint8_t opc)
-{
-    const TCGOpDef *def = &tcg_op_defs[opc];
-    fprintf(stderr, "TCG %s %u, %u, %u\n",
-            def->name, def->nb_oargs, def->nb_iargs, def->nb_cargs);
-}
-#endif
-
-/* Write value (native size). */
-static void tcg_out_i(TCGContext *s, tcg_target_ulong v)
-{
-    if (TCG_TARGET_REG_BITS == 32) {
-        tcg_out32(s, v);
-    } else {
-        tcg_out64(s, v);
-    }
-}
-
-/* Write opcode. */
-static void tcg_out_op_t(TCGContext *s, TCGOpcode op)
-{
-    tcg_out8(s, op);
-    tcg_out8(s, 0);
-}
-
-/* Write register. */
-static void tcg_out_r(TCGContext *s, TCGArg t0)
-{
-    assert(t0 < TCG_TARGET_NB_REGS);
-    tcg_out8(s, t0);
-}
-
-/* Write register or constant (native size). */
-static void tcg_out_ri(TCGContext *s, int const_arg, TCGArg arg)
-{
-    if (const_arg) {
-        assert(const_arg == 1);
-        tcg_out8(s, TCG_CONST);
-        tcg_out_i(s, arg);
-    } else {
-        tcg_out_r(s, arg);
-    }
-}
-
-/* Write register or constant (32 bit). */
-static void tcg_out_ri32(TCGContext *s, int const_arg, TCGArg arg)
-{
-    if (const_arg) {
-        assert(const_arg == 1);
-        tcg_out8(s, TCG_CONST);
-        tcg_out32(s, arg);
-    } else {
-        tcg_out_r(s, arg);
-    }
-}
-
-#if TCG_TARGET_REG_BITS == 64
-/* Write register or constant (64 bit). */
-static void tcg_out_ri64(TCGContext *s, int const_arg, TCGArg arg)
-{
-    if (const_arg) {
-        assert(const_arg == 1);
-        tcg_out8(s, TCG_CONST);
-        tcg_out64(s, arg);
-    } else {
-        tcg_out_r(s, arg);
-    }
-}
-#endif
-
-/* Write label. */
-static void tci_out_label(TCGContext *s, TCGArg arg)
-{
-    TCGLabel *label = &s->labels[arg];
-    if (label->has_value) {
-        tcg_out_i(s, label->u.value);
-        assert(label->u.value);
-    } else {
-        tcg_out_reloc(s, s->code_ptr, sizeof(tcg_target_ulong), arg, 0);
-        s->code_ptr += sizeof(tcg_target_ulong);
-    }
-}
-
-static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
-                       intptr_t arg2)
-{
-    uint8_t *old_code_ptr = s->code_ptr;
-    if (type == TCG_TYPE_I32) {
-        tcg_out_op_t(s, INDEX_op_ld_i32);
-        tcg_out_r(s, ret);
-        tcg_out_r(s, arg1);
-        tcg_out32(s, arg2);
-    } else {
-        assert(type == TCG_TYPE_I64);
-#if TCG_TARGET_REG_BITS == 64
-        tcg_out_op_t(s, INDEX_op_ld_i64);
-        tcg_out_r(s, ret);
-        tcg_out_r(s, arg1);
-        assert(arg2 == (int32_t)arg2);
-        tcg_out32(s, arg2);
-#else
-        TODO();
-#endif
-    }
-    old_code_ptr[1] = s->code_ptr - old_code_ptr;
-}
-
-static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
-{
-    uint8_t *old_code_ptr = s->code_ptr;
-    assert(ret != arg);
-#if TCG_TARGET_REG_BITS == 32
-    tcg_out_op_t(s, INDEX_op_mov_i32);
-#else
-    tcg_out_op_t(s, INDEX_op_mov_i64);
-#endif
-    tcg_out_r(s, ret);
-    tcg_out_r(s, arg);
-    old_code_ptr[1] = s->code_ptr - old_code_ptr;
-}
-
-static void tcg_out_movi(TCGContext *s, TCGType type,
-                         TCGReg t0, tcg_target_long arg)
-{
-    uint8_t *old_code_ptr = s->code_ptr;
-    uint32_t arg32 = arg;
-    if (type == TCG_TYPE_I32 || arg == arg32) {
-        tcg_out_op_t(s, INDEX_op_movi_i32);
-        tcg_out_r(s, t0);
-        tcg_out32(s, arg32);
-    } else {
-        assert(type == TCG_TYPE_I64);
-#if TCG_TARGET_REG_BITS == 64
-        tcg_out_op_t(s, INDEX_op_movi_i64);
-        tcg_out_r(s, t0);
-        tcg_out64(s, arg);
-#else
-        TODO();
-#endif
-    }
-    old_code_ptr[1] = s->code_ptr - old_code_ptr;
-}
-
-static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
-{
-    uint8_t *old_code_ptr = s->code_ptr;
-    tcg_out_op_t(s, INDEX_op_call);
-    tcg_out_ri(s, 1, (uintptr_t)arg);
-    old_code_ptr[1] = s->code_ptr - old_code_ptr;
-}
-
-static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
-                       const int *const_args)
-{
-    uint8_t *old_code_ptr = s->code_ptr;
-
-    tcg_out_op_t(s, opc);
-
-    switch (opc) {
-    case INDEX_op_exit_tb:
-        tcg_out64(s, args[0]);
-        break;
-    case INDEX_op_goto_tb:
-        if (s->tb_jmp_offset) {
-            /* Direct jump method. */
-            assert(args[0] < ARRAY_SIZE(s->tb_jmp_offset));
-            s->tb_jmp_offset[args[0]] = tcg_current_code_size(s);
-            tcg_out32(s, 0);
-        } else {
-            /* Indirect jump method. */
-            TODO();
-        }
-        assert(args[0] < ARRAY_SIZE(s->tb_next_offset));
-        s->tb_next_offset[args[0]] = tcg_current_code_size(s);
-        break;
-    case INDEX_op_br:
-        tci_out_label(s, args[0]);
-        break;
-    case INDEX_op_setcond_i32:
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_ri32(s, const_args[2], args[2]);
-        tcg_out8(s, args[3]);   /* condition */
-        break;
-#if TCG_TARGET_REG_BITS == 32
-    case INDEX_op_setcond2_i32:
-        /* setcond2_i32 cond, t0, t1_low, t1_high, t2_low, t2_high */
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_r(s, args[2]);
-        tcg_out_ri32(s, const_args[3], args[3]);
-        tcg_out_ri32(s, const_args[4], args[4]);
-        tcg_out8(s, args[5]);   /* condition */
-        break;
-#elif TCG_TARGET_REG_BITS == 64
-    case INDEX_op_setcond_i64:
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_ri64(s, const_args[2], args[2]);
-        tcg_out8(s, args[3]);   /* condition */
-        break;
-#endif
-    case INDEX_op_ld8u_i32:
-    case INDEX_op_ld8s_i32:
-    case INDEX_op_ld16u_i32:
-    case INDEX_op_ld16s_i32:
-    case INDEX_op_ld_i32:
-    case INDEX_op_st8_i32:
-    case INDEX_op_st16_i32:
-    case INDEX_op_st_i32:
-    case INDEX_op_ld8u_i64:
-    case INDEX_op_ld8s_i64:
-    case INDEX_op_ld16u_i64:
-    case INDEX_op_ld16s_i64:
-    case INDEX_op_ld32u_i64:
-    case INDEX_op_ld32s_i64:
-    case INDEX_op_ld_i64:
-    case INDEX_op_st8_i64:
-    case INDEX_op_st16_i64:
-    case INDEX_op_st32_i64:
-    case INDEX_op_st_i64:
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        assert(args[2] == (int32_t)args[2]);
-        tcg_out32(s, args[2]);
-        break;
-    case INDEX_op_add_i32:
-    case INDEX_op_sub_i32:
-    case INDEX_op_mul_i32:
-    case INDEX_op_and_i32:
-    case INDEX_op_andc_i32:     /* Optional (TCG_TARGET_HAS_andc_i32). */
-    case INDEX_op_eqv_i32:      /* Optional (TCG_TARGET_HAS_eqv_i32). */
-    case INDEX_op_nand_i32:     /* Optional (TCG_TARGET_HAS_nand_i32). */
-    case INDEX_op_nor_i32:      /* Optional (TCG_TARGET_HAS_nor_i32). */
-    case INDEX_op_or_i32:
-    case INDEX_op_orc_i32:      /* Optional (TCG_TARGET_HAS_orc_i32). */
-    case INDEX_op_xor_i32:
-    case INDEX_op_shl_i32:
-    case INDEX_op_shr_i32:
-    case INDEX_op_sar_i32:
-    case INDEX_op_rotl_i32:     /* Optional (TCG_TARGET_HAS_rot_i32). */
-    case INDEX_op_rotr_i32:     /* Optional (TCG_TARGET_HAS_rot_i32). */
-        tcg_out_r(s, args[0]);
-        tcg_out_ri32(s, const_args[1], args[1]);
-        tcg_out_ri32(s, const_args[2], args[2]);
-        break;
-    case INDEX_op_deposit_i32:  /* Optional (TCG_TARGET_HAS_deposit_i32). */
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_r(s, args[2]);
-        assert(args[3] <= UINT8_MAX);
-        tcg_out8(s, args[3]);
-        assert(args[4] <= UINT8_MAX);
-        tcg_out8(s, args[4]);
-        break;
-
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_add_i64:
-    case INDEX_op_sub_i64:
-    case INDEX_op_mul_i64:
-    case INDEX_op_and_i64:
-    case INDEX_op_andc_i64:     /* Optional (TCG_TARGET_HAS_andc_i64). */
-    case INDEX_op_eqv_i64:      /* Optional (TCG_TARGET_HAS_eqv_i64). */
-    case INDEX_op_nand_i64:     /* Optional (TCG_TARGET_HAS_nand_i64). */
-    case INDEX_op_nor_i64:      /* Optional (TCG_TARGET_HAS_nor_i64). */
-    case INDEX_op_or_i64:
-    case INDEX_op_orc_i64:      /* Optional (TCG_TARGET_HAS_orc_i64). */
-    case INDEX_op_xor_i64:
-    case INDEX_op_shl_i64:
-    case INDEX_op_shr_i64:
-    case INDEX_op_sar_i64:
-    case INDEX_op_rotl_i64:     /* Optional (TCG_TARGET_HAS_rot_i64). */
-    case INDEX_op_rotr_i64:     /* Optional (TCG_TARGET_HAS_rot_i64). */
-        tcg_out_r(s, args[0]);
-        tcg_out_ri64(s, const_args[1], args[1]);
-        tcg_out_ri64(s, const_args[2], args[2]);
-        break;
-    case INDEX_op_deposit_i64:  /* Optional (TCG_TARGET_HAS_deposit_i64). */
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_r(s, args[2]);
-        assert(args[3] <= UINT8_MAX);
-        tcg_out8(s, args[3]);
-        assert(args[4] <= UINT8_MAX);
-        tcg_out8(s, args[4]);
-        break;
-    case INDEX_op_div_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
-    case INDEX_op_divu_i64:     /* Optional (TCG_TARGET_HAS_div_i64). */
-    case INDEX_op_rem_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
-    case INDEX_op_remu_i64:     /* Optional (TCG_TARGET_HAS_div_i64). */
-        TODO();
-        break;
-    case INDEX_op_div2_i64:     /* Optional (TCG_TARGET_HAS_div2_i64). */
-    case INDEX_op_divu2_i64:    /* Optional (TCG_TARGET_HAS_div2_i64). */
-        TODO();
-        break;
-    case INDEX_op_brcond_i64:
-        tcg_out_r(s, args[0]);
-        tcg_out_ri64(s, const_args[1], args[1]);
-        tcg_out8(s, args[2]);           /* condition */
-        tci_out_label(s, args[3]);
-        break;
-    case INDEX_op_bswap16_i64:  /* Optional (TCG_TARGET_HAS_bswap16_i64). */
-    case INDEX_op_bswap32_i64:  /* Optional (TCG_TARGET_HAS_bswap32_i64). */
-    case INDEX_op_bswap64_i64:  /* Optional (TCG_TARGET_HAS_bswap64_i64). */
-    case INDEX_op_not_i64:      /* Optional (TCG_TARGET_HAS_not_i64). */
-    case INDEX_op_neg_i64:      /* Optional (TCG_TARGET_HAS_neg_i64). */
-    case INDEX_op_ext8s_i64:    /* Optional (TCG_TARGET_HAS_ext8s_i64). */
-    case INDEX_op_ext8u_i64:    /* Optional (TCG_TARGET_HAS_ext8u_i64). */
-    case INDEX_op_ext16s_i64:   /* Optional (TCG_TARGET_HAS_ext16s_i64). */
-    case INDEX_op_ext16u_i64:   /* Optional (TCG_TARGET_HAS_ext16u_i64). */
-    case INDEX_op_ext32s_i64:   /* Optional (TCG_TARGET_HAS_ext32s_i64). */
-    case INDEX_op_ext32u_i64:   /* Optional (TCG_TARGET_HAS_ext32u_i64). */
-#endif /* TCG_TARGET_REG_BITS == 64 */
-    case INDEX_op_neg_i32:      /* Optional (TCG_TARGET_HAS_neg_i32). */
-    case INDEX_op_not_i32:      /* Optional (TCG_TARGET_HAS_not_i32). */
-    case INDEX_op_ext8s_i32:    /* Optional (TCG_TARGET_HAS_ext8s_i32). */
-    case INDEX_op_ext16s_i32:   /* Optional (TCG_TARGET_HAS_ext16s_i32). */
-    case INDEX_op_ext8u_i32:    /* Optional (TCG_TARGET_HAS_ext8u_i32). */
-    case INDEX_op_ext16u_i32:   /* Optional (TCG_TARGET_HAS_ext16u_i32). */
-    case INDEX_op_bswap16_i32:  /* Optional (TCG_TARGET_HAS_bswap16_i32). */
-    case INDEX_op_bswap32_i32:  /* Optional (TCG_TARGET_HAS_bswap32_i32). */
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        break;
-    case INDEX_op_div_i32:      /* Optional (TCG_TARGET_HAS_div_i32). */
-    case INDEX_op_divu_i32:     /* Optional (TCG_TARGET_HAS_div_i32). */
-    case INDEX_op_rem_i32:      /* Optional (TCG_TARGET_HAS_div_i32). */
-    case INDEX_op_remu_i32:     /* Optional (TCG_TARGET_HAS_div_i32). */
-        tcg_out_r(s, args[0]);
-        tcg_out_ri32(s, const_args[1], args[1]);
-        tcg_out_ri32(s, const_args[2], args[2]);
-        break;
-    case INDEX_op_div2_i32:     /* Optional (TCG_TARGET_HAS_div2_i32). */
-    case INDEX_op_divu2_i32:    /* Optional (TCG_TARGET_HAS_div2_i32). */
-        TODO();
-        break;
-#if TCG_TARGET_REG_BITS == 32
-    case INDEX_op_add2_i32:
-    case INDEX_op_sub2_i32:
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_r(s, args[2]);
-        tcg_out_r(s, args[3]);
-        tcg_out_r(s, args[4]);
-        tcg_out_r(s, args[5]);
-        break;
-    case INDEX_op_brcond2_i32:
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_ri32(s, const_args[2], args[2]);
-        tcg_out_ri32(s, const_args[3], args[3]);
-        tcg_out8(s, args[4]);           /* condition */
-        tci_out_label(s, args[5]);
-        break;
-    case INDEX_op_mulu2_i32:
-        tcg_out_r(s, args[0]);
-        tcg_out_r(s, args[1]);
-        tcg_out_r(s, args[2]);
-        tcg_out_r(s, args[3]);
-        break;
-#endif
-    case INDEX_op_brcond_i32:
-        tcg_out_r(s, args[0]);
-        tcg_out_ri32(s, const_args[1], args[1]);
-        tcg_out8(s, args[2]);           /* condition */
-        tci_out_label(s, args[3]);
-        break;
-    case INDEX_op_qemu_ld_i32:
-        tcg_out_r(s, *args++);
-        tcg_out_r(s, *args++);
-        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-            tcg_out_r(s, *args++);
-        }
-        tcg_out_i(s, *args++);
-        break;
-    case INDEX_op_qemu_ld_i64:
-        tcg_out_r(s, *args++);
-        if (TCG_TARGET_REG_BITS == 32) {
-            tcg_out_r(s, *args++);
-        }
-        tcg_out_r(s, *args++);
-        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-            tcg_out_r(s, *args++);
-        }
-        tcg_out_i(s, *args++);
-        break;
-    case INDEX_op_qemu_st_i32:
-        tcg_out_r(s, *args++);
-        tcg_out_r(s, *args++);
-        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-            tcg_out_r(s, *args++);
-        }
-        tcg_out_i(s, *args++);
-#ifdef CONFIG_SOFTMMU
-        tcg_out_i(s, *args);
-#endif
-        break;
-    case INDEX_op_qemu_st_i64:
-        tcg_out_r(s, *args++);
-        if (TCG_TARGET_REG_BITS == 32) {
-            tcg_out_r(s, *args++);
-        }
-        tcg_out_r(s, *args++);
-        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-            tcg_out_r(s, *args++);
-        }
-        tcg_out_i(s, *args++);
-#ifdef CONFIG_SOFTMMU
-        tcg_out_i(s, *args);
-#endif
-        break;
-    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
-    case INDEX_op_mov_i64:
-    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
-    case INDEX_op_movi_i64:
-    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
-    default:
-        tcg_abort();
-    }
-    old_code_ptr[1] = s->code_ptr - old_code_ptr;
-}
-
-static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
-                       intptr_t arg2)
-{
-    uint8_t *old_code_ptr = s->code_ptr;
-    if (type == TCG_TYPE_I32) {
-        tcg_out_op_t(s, INDEX_op_st_i32);
-        tcg_out_r(s, arg);
-        tcg_out_r(s, arg1);
-        tcg_out32(s, arg2);
-    } else {
-        assert(type == TCG_TYPE_I64);
-#if TCG_TARGET_REG_BITS == 64
-        tcg_out_op_t(s, INDEX_op_st_i64);
-        tcg_out_r(s, arg);
-        tcg_out_r(s, arg1);
-        tcg_out32(s, arg2);
-#else
-        TODO();
-#endif
-    }
-    old_code_ptr[1] = s->code_ptr - old_code_ptr;
-}
-
-/* Test if a constant matches the constraint. */
-static int tcg_target_const_match(tcg_target_long val, TCGType type,
-                                  const TCGArgConstraint *arg_ct)
-{
-    /* No need to return 0 or 1, 0 or != 0 is good enough. */
-    return arg_ct->ct & TCG_CT_CONST;
-}
-
-static void tcg_target_init(TCGContext *s)
-{
-#if defined(CONFIG_DEBUG_TCG_INTERPRETER)
-    const char *envval = getenv("DEBUG_TCG");
-    if (envval) {
-        qemu_set_log(strtol(envval, NULL, 0));
-    }
-#endif
-
-    /* The current code uses uint8_t for tcg operations. */
-    assert(ARRAY_SIZE(tcg_op_defs) <= UINT8_MAX);
-
-    /* Registers available for 32 bit operations. */
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0,
-                     BIT(TCG_TARGET_NB_REGS) - 1);
-    /* Registers available for 64 bit operations. */
-    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0,
-                     BIT(TCG_TARGET_NB_REGS) - 1);
-    /* TODO: Which registers should be set here? */
-    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
-                     BIT(TCG_TARGET_NB_REGS) - 1);
-
-    tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
-    tcg_add_target_add_op_defs(tcg_target_op_defs);
-
-    /* We use negative offsets from "sp" so that we can distinguish
-       stores that might pretend to be call arguments.  */
-    tcg_set_frame(s, TCG_REG_CALL_STACK,
-                  -CPU_TEMP_BUF_NLONGS * sizeof(long),
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-}
-
-/* Generate global QEMU prologue and epilogue code. */
-static inline void tcg_target_qemu_prologue(TCGContext *s)
-{
-}
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index bd1e974..868228b 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -37,13 +37,12 @@
  * Therefore, we need both 32 and 64 bit virtual machines (interpreter).
  */
 
-#if !defined(TCG_TARGET_H)
+#ifndef TCG_TARGET_H
 #define TCG_TARGET_H
 
-#include "config-host.h"
-
 #define TCG_TARGET_INTERPRETER 1
 #define TCG_TARGET_INSN_UNIT_SIZE 1
+#define TCG_TARGET_TLB_DISPLACEMENT_BITS 32
 
 #if UINTPTR_MAX == UINT32_MAX
 # define TCG_TARGET_REG_BITS 32
@@ -83,7 +82,8 @@
 #define TCG_TARGET_HAS_mulsh_i32        0
 
 #if TCG_TARGET_REG_BITS == 64
-#define TCG_TARGET_HAS_trunc_shr_i32    0
+#define TCG_TARGET_HAS_extrl_i64_i32    0
+#define TCG_TARGET_HAS_extrh_i64_i32    0
 #define TCG_TARGET_HAS_bswap16_i64      1
 #define TCG_TARGET_HAS_bswap32_i64      1
 #define TCG_TARGET_HAS_bswap64_i64      1
@@ -175,8 +175,7 @@
 
 void tci_disas(uint8_t opc);
 
-uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr);
-#define tcg_qemu_tb_exec tcg_qemu_tb_exec
+#define HAVE_TCG_QEMU_TB_EXEC
 
 static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
diff --git a/tcg/tci/tcg-target.inc.c b/tcg/tci/tcg-target.inc.c
new file mode 100644
index 0000000..3c47ea7
--- /dev/null
+++ b/tcg/tci/tcg-target.inc.c
@@ -0,0 +1,887 @@
+/*
+ * Tiny Code Generator for QEMU
+ *
+ * Copyright (c) 2009, 2011 Stefan Weil
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "tcg-be-null.h"
+
+/* TODO list:
+ * - See TODO comments in code.
+ */
+
+/* Marker for missing code. */
+#define TODO() \
+    do { \
+        fprintf(stderr, "TODO %s:%u: %s()\n", \
+                __FILE__, __LINE__, __func__); \
+        tcg_abort(); \
+    } while (0)
+
+/* Bitfield n...m (in 32 bit value). */
+#define BITS(n, m) (((0xffffffffU << (31 - n)) >> (31 - n + m)) << m)
+
+/* Macros used in tcg_target_op_defs. */
+#define R       "r"
+#define RI      "ri"
+#if TCG_TARGET_REG_BITS == 32
+# define R64    "r", "r"
+#else
+# define R64    "r"
+#endif
+#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
+# define L      "L", "L"
+# define S      "S", "S"
+#else
+# define L      "L"
+# define S      "S"
+#endif
+
+/* TODO: documentation. */
+static const TCGTargetOpDef tcg_target_op_defs[] = {
+    { INDEX_op_exit_tb, { NULL } },
+    { INDEX_op_goto_tb, { NULL } },
+    { INDEX_op_br, { NULL } },
+
+    { INDEX_op_ld8u_i32, { R, R } },
+    { INDEX_op_ld8s_i32, { R, R } },
+    { INDEX_op_ld16u_i32, { R, R } },
+    { INDEX_op_ld16s_i32, { R, R } },
+    { INDEX_op_ld_i32, { R, R } },
+    { INDEX_op_st8_i32, { R, R } },
+    { INDEX_op_st16_i32, { R, R } },
+    { INDEX_op_st_i32, { R, R } },
+
+    { INDEX_op_add_i32, { R, RI, RI } },
+    { INDEX_op_sub_i32, { R, RI, RI } },
+    { INDEX_op_mul_i32, { R, RI, RI } },
+#if TCG_TARGET_HAS_div_i32
+    { INDEX_op_div_i32, { R, R, R } },
+    { INDEX_op_divu_i32, { R, R, R } },
+    { INDEX_op_rem_i32, { R, R, R } },
+    { INDEX_op_remu_i32, { R, R, R } },
+#elif TCG_TARGET_HAS_div2_i32
+    { INDEX_op_div2_i32, { R, R, "0", "1", R } },
+    { INDEX_op_divu2_i32, { R, R, "0", "1", R } },
+#endif
+    /* TODO: Does R, RI, RI result in faster code than R, R, RI?
+       If both operands are constants, we can optimize. */
+    { INDEX_op_and_i32, { R, RI, RI } },
+#if TCG_TARGET_HAS_andc_i32
+    { INDEX_op_andc_i32, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_eqv_i32
+    { INDEX_op_eqv_i32, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_nand_i32
+    { INDEX_op_nand_i32, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_nor_i32
+    { INDEX_op_nor_i32, { R, RI, RI } },
+#endif
+    { INDEX_op_or_i32, { R, RI, RI } },
+#if TCG_TARGET_HAS_orc_i32
+    { INDEX_op_orc_i32, { R, RI, RI } },
+#endif
+    { INDEX_op_xor_i32, { R, RI, RI } },
+    { INDEX_op_shl_i32, { R, RI, RI } },
+    { INDEX_op_shr_i32, { R, RI, RI } },
+    { INDEX_op_sar_i32, { R, RI, RI } },
+#if TCG_TARGET_HAS_rot_i32
+    { INDEX_op_rotl_i32, { R, RI, RI } },
+    { INDEX_op_rotr_i32, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_deposit_i32
+    { INDEX_op_deposit_i32, { R, "0", R } },
+#endif
+
+    { INDEX_op_brcond_i32, { R, RI } },
+
+    { INDEX_op_setcond_i32, { R, R, RI } },
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_setcond_i64, { R, R, RI } },
+#endif /* TCG_TARGET_REG_BITS == 64 */
+
+#if TCG_TARGET_REG_BITS == 32
+    /* TODO: Support R, R, R, R, RI, RI? Will it be faster? */
+    { INDEX_op_add2_i32, { R, R, R, R, R, R } },
+    { INDEX_op_sub2_i32, { R, R, R, R, R, R } },
+    { INDEX_op_brcond2_i32, { R, R, RI, RI } },
+    { INDEX_op_mulu2_i32, { R, R, R, R } },
+    { INDEX_op_setcond2_i32, { R, R, R, RI, RI } },
+#endif
+
+#if TCG_TARGET_HAS_not_i32
+    { INDEX_op_not_i32, { R, R } },
+#endif
+#if TCG_TARGET_HAS_neg_i32
+    { INDEX_op_neg_i32, { R, R } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_ld8u_i64, { R, R } },
+    { INDEX_op_ld8s_i64, { R, R } },
+    { INDEX_op_ld16u_i64, { R, R } },
+    { INDEX_op_ld16s_i64, { R, R } },
+    { INDEX_op_ld32u_i64, { R, R } },
+    { INDEX_op_ld32s_i64, { R, R } },
+    { INDEX_op_ld_i64, { R, R } },
+
+    { INDEX_op_st8_i64, { R, R } },
+    { INDEX_op_st16_i64, { R, R } },
+    { INDEX_op_st32_i64, { R, R } },
+    { INDEX_op_st_i64, { R, R } },
+
+    { INDEX_op_add_i64, { R, RI, RI } },
+    { INDEX_op_sub_i64, { R, RI, RI } },
+    { INDEX_op_mul_i64, { R, RI, RI } },
+#if TCG_TARGET_HAS_div_i64
+    { INDEX_op_div_i64, { R, R, R } },
+    { INDEX_op_divu_i64, { R, R, R } },
+    { INDEX_op_rem_i64, { R, R, R } },
+    { INDEX_op_remu_i64, { R, R, R } },
+#elif TCG_TARGET_HAS_div2_i64
+    { INDEX_op_div2_i64, { R, R, "0", "1", R } },
+    { INDEX_op_divu2_i64, { R, R, "0", "1", R } },
+#endif
+    { INDEX_op_and_i64, { R, RI, RI } },
+#if TCG_TARGET_HAS_andc_i64
+    { INDEX_op_andc_i64, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_eqv_i64
+    { INDEX_op_eqv_i64, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_nand_i64
+    { INDEX_op_nand_i64, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_nor_i64
+    { INDEX_op_nor_i64, { R, RI, RI } },
+#endif
+    { INDEX_op_or_i64, { R, RI, RI } },
+#if TCG_TARGET_HAS_orc_i64
+    { INDEX_op_orc_i64, { R, RI, RI } },
+#endif
+    { INDEX_op_xor_i64, { R, RI, RI } },
+    { INDEX_op_shl_i64, { R, RI, RI } },
+    { INDEX_op_shr_i64, { R, RI, RI } },
+    { INDEX_op_sar_i64, { R, RI, RI } },
+#if TCG_TARGET_HAS_rot_i64
+    { INDEX_op_rotl_i64, { R, RI, RI } },
+    { INDEX_op_rotr_i64, { R, RI, RI } },
+#endif
+#if TCG_TARGET_HAS_deposit_i64
+    { INDEX_op_deposit_i64, { R, "0", R } },
+#endif
+    { INDEX_op_brcond_i64, { R, RI } },
+
+#if TCG_TARGET_HAS_ext8s_i64
+    { INDEX_op_ext8s_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext16s_i64
+    { INDEX_op_ext16s_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext32s_i64
+    { INDEX_op_ext32s_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext8u_i64
+    { INDEX_op_ext8u_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext16u_i64
+    { INDEX_op_ext16u_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext32u_i64
+    { INDEX_op_ext32u_i64, { R, R } },
+#endif
+    { INDEX_op_ext_i32_i64, { R, R } },
+    { INDEX_op_extu_i32_i64, { R, R } },
+#if TCG_TARGET_HAS_bswap16_i64
+    { INDEX_op_bswap16_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_bswap32_i64
+    { INDEX_op_bswap32_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_bswap64_i64
+    { INDEX_op_bswap64_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_not_i64
+    { INDEX_op_not_i64, { R, R } },
+#endif
+#if TCG_TARGET_HAS_neg_i64
+    { INDEX_op_neg_i64, { R, R } },
+#endif
+#endif /* TCG_TARGET_REG_BITS == 64 */
+
+    { INDEX_op_qemu_ld_i32, { R, L } },
+    { INDEX_op_qemu_ld_i64, { R64, L } },
+
+    { INDEX_op_qemu_st_i32, { R, S } },
+    { INDEX_op_qemu_st_i64, { R64, S } },
+
+#if TCG_TARGET_HAS_ext8s_i32
+    { INDEX_op_ext8s_i32, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext16s_i32
+    { INDEX_op_ext16s_i32, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext8u_i32
+    { INDEX_op_ext8u_i32, { R, R } },
+#endif
+#if TCG_TARGET_HAS_ext16u_i32
+    { INDEX_op_ext16u_i32, { R, R } },
+#endif
+
+#if TCG_TARGET_HAS_bswap16_i32
+    { INDEX_op_bswap16_i32, { R, R } },
+#endif
+#if TCG_TARGET_HAS_bswap32_i32
+    { INDEX_op_bswap32_i32, { R, R } },
+#endif
+
+    { -1 },
+};
+
+static const int tcg_target_reg_alloc_order[] = {
+    TCG_REG_R0,
+    TCG_REG_R1,
+    TCG_REG_R2,
+    TCG_REG_R3,
+#if 0 /* used for TCG_REG_CALL_STACK */
+    TCG_REG_R4,
+#endif
+    TCG_REG_R5,
+    TCG_REG_R6,
+    TCG_REG_R7,
+#if TCG_TARGET_NB_REGS >= 16
+    TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
+    TCG_REG_R11,
+    TCG_REG_R12,
+    TCG_REG_R13,
+    TCG_REG_R14,
+    TCG_REG_R15,
+#endif
+};
+
+#if MAX_OPC_PARAM_IARGS != 5
+# error Fix needed, number of supported input arguments changed!
+#endif
+
+static const int tcg_target_call_iarg_regs[] = {
+    TCG_REG_R0,
+    TCG_REG_R1,
+    TCG_REG_R2,
+    TCG_REG_R3,
+#if 0 /* used for TCG_REG_CALL_STACK */
+    TCG_REG_R4,
+#endif
+    TCG_REG_R5,
+#if TCG_TARGET_REG_BITS == 32
+    /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */
+    TCG_REG_R6,
+    TCG_REG_R7,
+#if TCG_TARGET_NB_REGS >= 16
+    TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
+#else
+# error Too few input registers available
+#endif
+#endif
+};
+
+static const int tcg_target_call_oarg_regs[] = {
+    TCG_REG_R0,
+#if TCG_TARGET_REG_BITS == 32
+    TCG_REG_R1
+#endif
+};
+
+#ifdef CONFIG_DEBUG_TCG
+static const char *const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
+    "r00",
+    "r01",
+    "r02",
+    "r03",
+    "r04",
+    "r05",
+    "r06",
+    "r07",
+#if TCG_TARGET_NB_REGS >= 16
+    "r08",
+    "r09",
+    "r10",
+    "r11",
+    "r12",
+    "r13",
+    "r14",
+    "r15",
+#if TCG_TARGET_NB_REGS >= 32
+    "r16",
+    "r17",
+    "r18",
+    "r19",
+    "r20",
+    "r21",
+    "r22",
+    "r23",
+    "r24",
+    "r25",
+    "r26",
+    "r27",
+    "r28",
+    "r29",
+    "r30",
+    "r31"
+#endif
+#endif
+};
+#endif
+
+static void patch_reloc(tcg_insn_unit *code_ptr, int type,
+                        intptr_t value, intptr_t addend)
+{
+    /* tcg_out_reloc always uses the same type, addend. */
+    tcg_debug_assert(type == sizeof(tcg_target_long));
+    tcg_debug_assert(addend == 0);
+    tcg_debug_assert(value != 0);
+    if (TCG_TARGET_REG_BITS == 32) {
+        tcg_patch32(code_ptr, value);
+    } else {
+        tcg_patch64(code_ptr, value);
+    }
+}
+
+/* Parse target specific constraints. */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str = *pct_str;
+    switch (ct_str[0]) {
+    case 'r':
+    case 'L':                   /* qemu_ld constraint */
+    case 'S':                   /* qemu_st constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, BIT(TCG_TARGET_NB_REGS) - 1);
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+#if defined(CONFIG_DEBUG_TCG_INTERPRETER)
+/* Show current bytecode. Used by tcg interpreter. */
+void tci_disas(uint8_t opc)
+{
+    const TCGOpDef *def = &tcg_op_defs[opc];
+    fprintf(stderr, "TCG %s %u, %u, %u\n",
+            def->name, def->nb_oargs, def->nb_iargs, def->nb_cargs);
+}
+#endif
+
+/* Write value (native size). */
+static void tcg_out_i(TCGContext *s, tcg_target_ulong v)
+{
+    if (TCG_TARGET_REG_BITS == 32) {
+        tcg_out32(s, v);
+    } else {
+        tcg_out64(s, v);
+    }
+}
+
+/* Write opcode. */
+static void tcg_out_op_t(TCGContext *s, TCGOpcode op)
+{
+    tcg_out8(s, op);
+    tcg_out8(s, 0);
+}
+
+/* Write register. */
+static void tcg_out_r(TCGContext *s, TCGArg t0)
+{
+    tcg_debug_assert(t0 < TCG_TARGET_NB_REGS);
+    tcg_out8(s, t0);
+}
+
+/* Write register or constant (native size). */
+static void tcg_out_ri(TCGContext *s, int const_arg, TCGArg arg)
+{
+    if (const_arg) {
+        tcg_debug_assert(const_arg == 1);
+        tcg_out8(s, TCG_CONST);
+        tcg_out_i(s, arg);
+    } else {
+        tcg_out_r(s, arg);
+    }
+}
+
+/* Write register or constant (32 bit). */
+static void tcg_out_ri32(TCGContext *s, int const_arg, TCGArg arg)
+{
+    if (const_arg) {
+        tcg_debug_assert(const_arg == 1);
+        tcg_out8(s, TCG_CONST);
+        tcg_out32(s, arg);
+    } else {
+        tcg_out_r(s, arg);
+    }
+}
+
+#if TCG_TARGET_REG_BITS == 64
+/* Write register or constant (64 bit). */
+static void tcg_out_ri64(TCGContext *s, int const_arg, TCGArg arg)
+{
+    if (const_arg) {
+        tcg_debug_assert(const_arg == 1);
+        tcg_out8(s, TCG_CONST);
+        tcg_out64(s, arg);
+    } else {
+        tcg_out_r(s, arg);
+    }
+}
+#endif
+
+/* Write label. */
+static void tci_out_label(TCGContext *s, TCGLabel *label)
+{
+    if (label->has_value) {
+        tcg_out_i(s, label->u.value);
+        tcg_debug_assert(label->u.value);
+    } else {
+        tcg_out_reloc(s, s->code_ptr, sizeof(tcg_target_ulong), label, 0);
+        s->code_ptr += sizeof(tcg_target_ulong);
+    }
+}
+
+static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
+                       intptr_t arg2)
+{
+    uint8_t *old_code_ptr = s->code_ptr;
+    if (type == TCG_TYPE_I32) {
+        tcg_out_op_t(s, INDEX_op_ld_i32);
+        tcg_out_r(s, ret);
+        tcg_out_r(s, arg1);
+        tcg_out32(s, arg2);
+    } else {
+        tcg_debug_assert(type == TCG_TYPE_I64);
+#if TCG_TARGET_REG_BITS == 64
+        tcg_out_op_t(s, INDEX_op_ld_i64);
+        tcg_out_r(s, ret);
+        tcg_out_r(s, arg1);
+        tcg_debug_assert(arg2 == (int32_t)arg2);
+        tcg_out32(s, arg2);
+#else
+        TODO();
+#endif
+    }
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
+}
+
+static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
+{
+    uint8_t *old_code_ptr = s->code_ptr;
+    tcg_debug_assert(ret != arg);
+#if TCG_TARGET_REG_BITS == 32
+    tcg_out_op_t(s, INDEX_op_mov_i32);
+#else
+    tcg_out_op_t(s, INDEX_op_mov_i64);
+#endif
+    tcg_out_r(s, ret);
+    tcg_out_r(s, arg);
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
+}
+
+static void tcg_out_movi(TCGContext *s, TCGType type,
+                         TCGReg t0, tcg_target_long arg)
+{
+    uint8_t *old_code_ptr = s->code_ptr;
+    uint32_t arg32 = arg;
+    if (type == TCG_TYPE_I32 || arg == arg32) {
+        tcg_out_op_t(s, INDEX_op_movi_i32);
+        tcg_out_r(s, t0);
+        tcg_out32(s, arg32);
+    } else {
+        tcg_debug_assert(type == TCG_TYPE_I64);
+#if TCG_TARGET_REG_BITS == 64
+        tcg_out_op_t(s, INDEX_op_movi_i64);
+        tcg_out_r(s, t0);
+        tcg_out64(s, arg);
+#else
+        TODO();
+#endif
+    }
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
+}
+
+static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
+{
+    uint8_t *old_code_ptr = s->code_ptr;
+    tcg_out_op_t(s, INDEX_op_call);
+    tcg_out_ri(s, 1, (uintptr_t)arg);
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
+}
+
+static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
+                       const int *const_args)
+{
+    uint8_t *old_code_ptr = s->code_ptr;
+
+    tcg_out_op_t(s, opc);
+
+    switch (opc) {
+    case INDEX_op_exit_tb:
+        tcg_out64(s, args[0]);
+        break;
+    case INDEX_op_goto_tb:
+        if (s->tb_jmp_insn_offset) {
+            /* Direct jump method. */
+            tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_insn_offset));
+            /* Align for atomic patching and thread safety */
+            s->code_ptr = QEMU_ALIGN_PTR_UP(s->code_ptr, 4);
+            s->tb_jmp_insn_offset[args[0]] = tcg_current_code_size(s);
+            tcg_out32(s, 0);
+        } else {
+            /* Indirect jump method. */
+            TODO();
+        }
+        tcg_debug_assert(args[0] < ARRAY_SIZE(s->tb_jmp_reset_offset));
+        s->tb_jmp_reset_offset[args[0]] = tcg_current_code_size(s);
+        break;
+    case INDEX_op_br:
+        tci_out_label(s, arg_label(args[0]));
+        break;
+    case INDEX_op_setcond_i32:
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_ri32(s, const_args[2], args[2]);
+        tcg_out8(s, args[3]);   /* condition */
+        break;
+#if TCG_TARGET_REG_BITS == 32
+    case INDEX_op_setcond2_i32:
+        /* setcond2_i32 cond, t0, t1_low, t1_high, t2_low, t2_high */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        tcg_out_ri32(s, const_args[3], args[3]);
+        tcg_out_ri32(s, const_args[4], args[4]);
+        tcg_out8(s, args[5]);   /* condition */
+        break;
+#elif TCG_TARGET_REG_BITS == 64
+    case INDEX_op_setcond_i64:
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_ri64(s, const_args[2], args[2]);
+        tcg_out8(s, args[3]);   /* condition */
+        break;
+#endif
+    case INDEX_op_ld8u_i32:
+    case INDEX_op_ld8s_i32:
+    case INDEX_op_ld16u_i32:
+    case INDEX_op_ld16s_i32:
+    case INDEX_op_ld_i32:
+    case INDEX_op_st8_i32:
+    case INDEX_op_st16_i32:
+    case INDEX_op_st_i32:
+    case INDEX_op_ld8u_i64:
+    case INDEX_op_ld8s_i64:
+    case INDEX_op_ld16u_i64:
+    case INDEX_op_ld16s_i64:
+    case INDEX_op_ld32u_i64:
+    case INDEX_op_ld32s_i64:
+    case INDEX_op_ld_i64:
+    case INDEX_op_st8_i64:
+    case INDEX_op_st16_i64:
+    case INDEX_op_st32_i64:
+    case INDEX_op_st_i64:
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_debug_assert(args[2] == (int32_t)args[2]);
+        tcg_out32(s, args[2]);
+        break;
+    case INDEX_op_add_i32:
+    case INDEX_op_sub_i32:
+    case INDEX_op_mul_i32:
+    case INDEX_op_and_i32:
+    case INDEX_op_andc_i32:     /* Optional (TCG_TARGET_HAS_andc_i32). */
+    case INDEX_op_eqv_i32:      /* Optional (TCG_TARGET_HAS_eqv_i32). */
+    case INDEX_op_nand_i32:     /* Optional (TCG_TARGET_HAS_nand_i32). */
+    case INDEX_op_nor_i32:      /* Optional (TCG_TARGET_HAS_nor_i32). */
+    case INDEX_op_or_i32:
+    case INDEX_op_orc_i32:      /* Optional (TCG_TARGET_HAS_orc_i32). */
+    case INDEX_op_xor_i32:
+    case INDEX_op_shl_i32:
+    case INDEX_op_shr_i32:
+    case INDEX_op_sar_i32:
+    case INDEX_op_rotl_i32:     /* Optional (TCG_TARGET_HAS_rot_i32). */
+    case INDEX_op_rotr_i32:     /* Optional (TCG_TARGET_HAS_rot_i32). */
+        tcg_out_r(s, args[0]);
+        tcg_out_ri32(s, const_args[1], args[1]);
+        tcg_out_ri32(s, const_args[2], args[2]);
+        break;
+    case INDEX_op_deposit_i32:  /* Optional (TCG_TARGET_HAS_deposit_i32). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        tcg_debug_assert(args[3] <= UINT8_MAX);
+        tcg_out8(s, args[3]);
+        tcg_debug_assert(args[4] <= UINT8_MAX);
+        tcg_out8(s, args[4]);
+        break;
+
+#if TCG_TARGET_REG_BITS == 64
+    case INDEX_op_add_i64:
+    case INDEX_op_sub_i64:
+    case INDEX_op_mul_i64:
+    case INDEX_op_and_i64:
+    case INDEX_op_andc_i64:     /* Optional (TCG_TARGET_HAS_andc_i64). */
+    case INDEX_op_eqv_i64:      /* Optional (TCG_TARGET_HAS_eqv_i64). */
+    case INDEX_op_nand_i64:     /* Optional (TCG_TARGET_HAS_nand_i64). */
+    case INDEX_op_nor_i64:      /* Optional (TCG_TARGET_HAS_nor_i64). */
+    case INDEX_op_or_i64:
+    case INDEX_op_orc_i64:      /* Optional (TCG_TARGET_HAS_orc_i64). */
+    case INDEX_op_xor_i64:
+    case INDEX_op_shl_i64:
+    case INDEX_op_shr_i64:
+    case INDEX_op_sar_i64:
+    case INDEX_op_rotl_i64:     /* Optional (TCG_TARGET_HAS_rot_i64). */
+    case INDEX_op_rotr_i64:     /* Optional (TCG_TARGET_HAS_rot_i64). */
+        tcg_out_r(s, args[0]);
+        tcg_out_ri64(s, const_args[1], args[1]);
+        tcg_out_ri64(s, const_args[2], args[2]);
+        break;
+    case INDEX_op_deposit_i64:  /* Optional (TCG_TARGET_HAS_deposit_i64). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        tcg_debug_assert(args[3] <= UINT8_MAX);
+        tcg_out8(s, args[3]);
+        tcg_debug_assert(args[4] <= UINT8_MAX);
+        tcg_out8(s, args[4]);
+        break;
+    case INDEX_op_div_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
+    case INDEX_op_divu_i64:     /* Optional (TCG_TARGET_HAS_div_i64). */
+    case INDEX_op_rem_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
+    case INDEX_op_remu_i64:     /* Optional (TCG_TARGET_HAS_div_i64). */
+        TODO();
+        break;
+    case INDEX_op_div2_i64:     /* Optional (TCG_TARGET_HAS_div2_i64). */
+    case INDEX_op_divu2_i64:    /* Optional (TCG_TARGET_HAS_div2_i64). */
+        TODO();
+        break;
+    case INDEX_op_brcond_i64:
+        tcg_out_r(s, args[0]);
+        tcg_out_ri64(s, const_args[1], args[1]);
+        tcg_out8(s, args[2]);           /* condition */
+        tci_out_label(s, arg_label(args[3]));
+        break;
+    case INDEX_op_bswap16_i64:  /* Optional (TCG_TARGET_HAS_bswap16_i64). */
+    case INDEX_op_bswap32_i64:  /* Optional (TCG_TARGET_HAS_bswap32_i64). */
+    case INDEX_op_bswap64_i64:  /* Optional (TCG_TARGET_HAS_bswap64_i64). */
+    case INDEX_op_not_i64:      /* Optional (TCG_TARGET_HAS_not_i64). */
+    case INDEX_op_neg_i64:      /* Optional (TCG_TARGET_HAS_neg_i64). */
+    case INDEX_op_ext8s_i64:    /* Optional (TCG_TARGET_HAS_ext8s_i64). */
+    case INDEX_op_ext8u_i64:    /* Optional (TCG_TARGET_HAS_ext8u_i64). */
+    case INDEX_op_ext16s_i64:   /* Optional (TCG_TARGET_HAS_ext16s_i64). */
+    case INDEX_op_ext16u_i64:   /* Optional (TCG_TARGET_HAS_ext16u_i64). */
+    case INDEX_op_ext32s_i64:   /* Optional (TCG_TARGET_HAS_ext32s_i64). */
+    case INDEX_op_ext32u_i64:   /* Optional (TCG_TARGET_HAS_ext32u_i64). */
+    case INDEX_op_ext_i32_i64:
+    case INDEX_op_extu_i32_i64:
+#endif /* TCG_TARGET_REG_BITS == 64 */
+    case INDEX_op_neg_i32:      /* Optional (TCG_TARGET_HAS_neg_i32). */
+    case INDEX_op_not_i32:      /* Optional (TCG_TARGET_HAS_not_i32). */
+    case INDEX_op_ext8s_i32:    /* Optional (TCG_TARGET_HAS_ext8s_i32). */
+    case INDEX_op_ext16s_i32:   /* Optional (TCG_TARGET_HAS_ext16s_i32). */
+    case INDEX_op_ext8u_i32:    /* Optional (TCG_TARGET_HAS_ext8u_i32). */
+    case INDEX_op_ext16u_i32:   /* Optional (TCG_TARGET_HAS_ext16u_i32). */
+    case INDEX_op_bswap16_i32:  /* Optional (TCG_TARGET_HAS_bswap16_i32). */
+    case INDEX_op_bswap32_i32:  /* Optional (TCG_TARGET_HAS_bswap32_i32). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        break;
+    case INDEX_op_div_i32:      /* Optional (TCG_TARGET_HAS_div_i32). */
+    case INDEX_op_divu_i32:     /* Optional (TCG_TARGET_HAS_div_i32). */
+    case INDEX_op_rem_i32:      /* Optional (TCG_TARGET_HAS_div_i32). */
+    case INDEX_op_remu_i32:     /* Optional (TCG_TARGET_HAS_div_i32). */
+        tcg_out_r(s, args[0]);
+        tcg_out_ri32(s, const_args[1], args[1]);
+        tcg_out_ri32(s, const_args[2], args[2]);
+        break;
+    case INDEX_op_div2_i32:     /* Optional (TCG_TARGET_HAS_div2_i32). */
+    case INDEX_op_divu2_i32:    /* Optional (TCG_TARGET_HAS_div2_i32). */
+        TODO();
+        break;
+#if TCG_TARGET_REG_BITS == 32
+    case INDEX_op_add2_i32:
+    case INDEX_op_sub2_i32:
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        tcg_out_r(s, args[3]);
+        tcg_out_r(s, args[4]);
+        tcg_out_r(s, args[5]);
+        break;
+    case INDEX_op_brcond2_i32:
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_ri32(s, const_args[2], args[2]);
+        tcg_out_ri32(s, const_args[3], args[3]);
+        tcg_out8(s, args[4]);           /* condition */
+        tci_out_label(s, arg_label(args[5]));
+        break;
+    case INDEX_op_mulu2_i32:
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        tcg_out_r(s, args[3]);
+        break;
+#endif
+    case INDEX_op_brcond_i32:
+        tcg_out_r(s, args[0]);
+        tcg_out_ri32(s, const_args[1], args[1]);
+        tcg_out8(s, args[2]);           /* condition */
+        tci_out_label(s, arg_label(args[3]));
+        break;
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_r(s, *args++);
+        tcg_out_r(s, *args++);
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
+        break;
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_r(s, *args++);
+        if (TCG_TARGET_REG_BITS == 32) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_r(s, *args++);
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
+        break;
+    case INDEX_op_qemu_st_i32:
+        tcg_out_r(s, *args++);
+        tcg_out_r(s, *args++);
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
+        break;
+    case INDEX_op_qemu_st_i64:
+        tcg_out_r(s, *args++);
+        if (TCG_TARGET_REG_BITS == 32) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_r(s, *args++);
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
+        break;
+    case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
+    case INDEX_op_mov_i64:
+    case INDEX_op_movi_i32: /* Always emitted via tcg_out_movi.  */
+    case INDEX_op_movi_i64:
+    case INDEX_op_call:     /* Always emitted via tcg_out_call.  */
+    default:
+        tcg_abort();
+    }
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
+}
+
+static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
+                       intptr_t arg2)
+{
+    uint8_t *old_code_ptr = s->code_ptr;
+    if (type == TCG_TYPE_I32) {
+        tcg_out_op_t(s, INDEX_op_st_i32);
+        tcg_out_r(s, arg);
+        tcg_out_r(s, arg1);
+        tcg_out32(s, arg2);
+    } else {
+        tcg_debug_assert(type == TCG_TYPE_I64);
+#if TCG_TARGET_REG_BITS == 64
+        tcg_out_op_t(s, INDEX_op_st_i64);
+        tcg_out_r(s, arg);
+        tcg_out_r(s, arg1);
+        tcg_out32(s, arg2);
+#else
+        TODO();
+#endif
+    }
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
+}
+
+static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
+                               TCGReg base, intptr_t ofs)
+{
+    return false;
+}
+
+/* Test if a constant matches the constraint. */
+static int tcg_target_const_match(tcg_target_long val, TCGType type,
+                                  const TCGArgConstraint *arg_ct)
+{
+    /* No need to return 0 or 1, 0 or != 0 is good enough. */
+    return arg_ct->ct & TCG_CT_CONST;
+}
+
+static void tcg_target_init(TCGContext *s)
+{
+#if defined(CONFIG_DEBUG_TCG_INTERPRETER)
+    const char *envval = getenv("DEBUG_TCG");
+    if (envval) {
+        qemu_set_log(strtol(envval, NULL, 0));
+    }
+#endif
+
+    /* The current code uses uint8_t for tcg operations. */
+    tcg_debug_assert(tcg_op_defs_max <= UINT8_MAX);
+
+    /* Registers available for 32 bit operations. */
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0,
+                     BIT(TCG_TARGET_NB_REGS) - 1);
+    /* Registers available for 64 bit operations. */
+    tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0,
+                     BIT(TCG_TARGET_NB_REGS) - 1);
+    /* TODO: Which registers should be set here? */
+    tcg_regset_set32(tcg_target_call_clobber_regs, 0,
+                     BIT(TCG_TARGET_NB_REGS) - 1);
+
+    tcg_regset_clear(s->reserved_regs);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
+    tcg_add_target_add_op_defs(tcg_target_op_defs);
+
+    /* We use negative offsets from "sp" so that we can distinguish
+       stores that might pretend to be call arguments.  */
+    tcg_set_frame(s, TCG_REG_CALL_STACK,
+                  -CPU_TEMP_BUF_NLONGS * sizeof(long),
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
+}
+
+/* Generate global QEMU prologue and epilogue code. */
+static inline void tcg_target_qemu_prologue(TCGContext *s)
+{
+}
diff --git a/tci.c b/tci.c
index 8444948..b488c0d 100644
--- a/tci.c
+++ b/tci.c
@@ -1,7 +1,7 @@
 /*
  * Tiny Code Interpreter for QEMU
  *
- * Copyright (c) 2009, 2011 Stefan Weil
+ * Copyright (c) 2009, 2011, 2016 Stefan Weil
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -17,15 +17,18 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 
-/* Defining NDEBUG disables assertions (which makes the code faster). */
-#if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
-# define NDEBUG
+/* Enable TCI assertions only when debugging TCG (and without NDEBUG defined).
+ * Without assertions, the interpreter runs much faster. */
+#if defined(CONFIG_DEBUG_TCG)
+# define tci_assert(cond) assert(cond)
+#else
+# define tci_assert(cond) ((void)0)
 #endif
 
 #include "qemu-common.h"
-#include "exec/exec-all.h"           /* MAX_OPC_PARAM_IARGS */
+#include "tcg/tcg.h"           /* MAX_OPC_PARAM_IARGS */
 #include "exec/cpu_ldst.h"
 #include "tcg-op.h"
 
@@ -52,17 +55,11 @@
                                     tcg_target_ulong);
 #endif
 
-/* Targets which don't use GETPC also don't need tci_tb_ptr
-   which makes them a little faster. */
-#if defined(GETPC)
-uintptr_t tci_tb_ptr;
-#endif
-
 static tcg_target_ulong tci_reg[TCG_TARGET_NB_REGS];
 
 static tcg_target_ulong tci_read_reg(TCGReg index)
 {
-    assert(index < ARRAY_SIZE(tci_reg));
+    tci_assert(index < ARRAY_SIZE(tci_reg));
     return tci_reg[index];
 }
 
@@ -111,9 +108,9 @@
 
 static void tci_write_reg(TCGReg index, tcg_target_ulong value)
 {
-    assert(index < ARRAY_SIZE(tci_reg));
-    assert(index != TCG_AREG0);
-    assert(index != TCG_REG_CALL_STACK);
+    tci_assert(index < ARRAY_SIZE(tci_reg));
+    tci_assert(index != TCG_AREG0);
+    tci_assert(index != TCG_REG_CALL_STACK);
     tci_reg[index] = value;
 }
 
@@ -331,7 +328,7 @@
 static tcg_target_ulong tci_read_label(uint8_t **tb_ptr)
 {
     tcg_target_ulong label = tci_read_i(tb_ptr);
-    assert(label != 0);
+    tci_assert(label != 0);
     return label;
 }
 
@@ -470,15 +467,15 @@
 {
     long tcg_temps[CPU_TEMP_BUF_NLONGS];
     uintptr_t sp_value = (uintptr_t)(tcg_temps + CPU_TEMP_BUF_NLONGS);
-    uintptr_t next_tb = 0;
+    uintptr_t ret = 0;
 
     tci_reg[TCG_AREG0] = (tcg_target_ulong)env;
     tci_reg[TCG_REG_CALL_STACK] = sp_value;
-    assert(tb_ptr);
+    tci_assert(tb_ptr);
 
     for (;;) {
         TCGOpcode opc = tb_ptr[0];
-#if !defined(NDEBUG)
+#if defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
         uint8_t op_size = tb_ptr[1];
         uint8_t *old_code_ptr = tb_ptr;
 #endif
@@ -531,7 +528,7 @@
             break;
         case INDEX_op_br:
             label = tci_read_label(&tb_ptr);
-            assert(tb_ptr == old_code_ptr + op_size);
+            tci_assert(tb_ptr == old_code_ptr + op_size);
             tb_ptr = (uint8_t *)label;
             continue;
         case INDEX_op_setcond_i32:
@@ -606,7 +603,7 @@
             t0 = tci_read_r32(&tb_ptr);
             t1 = tci_read_r(&tb_ptr);
             t2 = tci_read_s32(&tb_ptr);
-            assert(t1 != sp_value || (int32_t)t2 < 0);
+            tci_assert(t1 != sp_value || (int32_t)t2 < 0);
             *(uint32_t *)(t1 + t2) = t0;
             break;
 
@@ -731,7 +728,7 @@
             condition = *tb_ptr++;
             label = tci_read_label(&tb_ptr);
             if (tci_compare32(t0, t1, condition)) {
-                assert(tb_ptr == old_code_ptr + op_size);
+                tci_assert(tb_ptr == old_code_ptr + op_size);
                 tb_ptr = (uint8_t *)label;
                 continue;
             }
@@ -757,7 +754,7 @@
             condition = *tb_ptr++;
             label = tci_read_label(&tb_ptr);
             if (tci_compare64(tmp64, v64, condition)) {
-                assert(tb_ptr == old_code_ptr + op_size);
+                tci_assert(tb_ptr == old_code_ptr + op_size);
                 tb_ptr = (uint8_t *)label;
                 continue;
             }
@@ -891,7 +888,7 @@
             t0 = tci_read_r64(&tb_ptr);
             t1 = tci_read_r(&tb_ptr);
             t2 = tci_read_s32(&tb_ptr);
-            assert(t1 != sp_value || (int32_t)t2 < 0);
+            tci_assert(t1 != sp_value || (int32_t)t2 < 0);
             *(uint64_t *)(t1 + t2) = t0;
             break;
 
@@ -998,7 +995,7 @@
             condition = *tb_ptr++;
             label = tci_read_label(&tb_ptr);
             if (tci_compare64(t0, t1, condition)) {
-                assert(tb_ptr == old_code_ptr + op_size);
+                tci_assert(tb_ptr == old_code_ptr + op_size);
                 tb_ptr = (uint8_t *)label;
                 continue;
             }
@@ -1033,18 +1030,20 @@
 #endif
 #if TCG_TARGET_HAS_ext32s_i64
         case INDEX_op_ext32s_i64:
+#endif
+        case INDEX_op_ext_i32_i64:
             t0 = *tb_ptr++;
             t1 = tci_read_r32s(&tb_ptr);
             tci_write_reg64(t0, t1);
             break;
-#endif
 #if TCG_TARGET_HAS_ext32u_i64
         case INDEX_op_ext32u_i64:
+#endif
+        case INDEX_op_extu_i32_i64:
             t0 = *tb_ptr++;
             t1 = tci_read_r32(&tb_ptr);
             tci_write_reg64(t0, t1);
             break;
-#endif
 #if TCG_TARGET_HAS_bswap16_i64
         case INDEX_op_bswap16_i64:
             TODO();
@@ -1085,22 +1084,16 @@
 
             /* QEMU specific operations. */
 
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        case INDEX_op_debug_insn_start:
-            TODO();
-            break;
-#else
-        case INDEX_op_debug_insn_start:
-            TODO();
-            break;
-#endif
         case INDEX_op_exit_tb:
-            next_tb = *(uint64_t *)tb_ptr;
+            ret = *(uint64_t *)tb_ptr;
             goto exit;
             break;
         case INDEX_op_goto_tb:
-            t0 = tci_read_i32(&tb_ptr);
-            assert(tb_ptr == old_code_ptr + op_size);
+            /* Jump address is aligned */
+            tb_ptr = QEMU_ALIGN_PTR_UP(tb_ptr, 4);
+            t0 = atomic_read((int32_t *)tb_ptr);
+            tb_ptr += sizeof(int32_t);
+            tci_assert(tb_ptr == old_code_ptr + op_size);
             tb_ptr += (int32_t)t0;
             continue;
         case INDEX_op_qemu_ld_i32:
@@ -1247,8 +1240,8 @@
             TODO();
             break;
         }
-        assert(tb_ptr == old_code_ptr + op_size);
+        tci_assert(tb_ptr == old_code_ptr + op_size);
     }
 exit:
-    return next_tb;
+    return ret;
 }
diff --git a/tests/.gitignore b/tests/.gitignore
index e2e4957..dbb5263 100644
--- a/tests/.gitignore
+++ b/tests/.gitignore
@@ -3,15 +3,46 @@
 check-qint
 check-qjson
 check-qlist
+check-qnull
 check-qstring
 check-qom-interface
+check-qom-proplist
+qht-bench
+rcutorture
 test-aio
+test-base64
 test-bitops
+test-blockjob-txn
+test-clone-visitor
 test-coroutine
+test-crypto-afsplit
+test-crypto-block
+test-crypto-cipher
+test-crypto-hash
+test-crypto-ivgen
+test-crypto-pbkdf
+test-crypto-secret
+test-crypto-tlscredsx509
+test-crypto-tlscredsx509-work/
+test-crypto-tlscredsx509-certs/
+test-crypto-tlssession
+test-crypto-tlssession-work/
+test-crypto-tlssession-client/
+test-crypto-tlssession-server/
+test-crypto-xts
 test-cutils
 test-hbitmap
 test-int128
 test-iov
+test-io-channel-buffer
+test-io-channel-command
+test-io-channel-command.fifo
+test-io-channel-file
+test-io-channel-file.txt
+test-io-channel-socket
+test-io-channel-tls
+test-io-task
+test-logging
 test-mul64
 test-opts-visitor
 test-qapi-event.[ch]
@@ -19,21 +50,32 @@
 test-qapi-visit.[ch]
 test-qdev-global-props
 test-qemu-opts
+test-qdist
+test-qga
+test-qht
+test-qht-par
 test-qmp-commands
 test-qmp-commands.h
 test-qmp-event
 test-qmp-input-strict
 test-qmp-input-visitor
+test-qmp-introspect.[ch]
 test-qmp-marshal.c
 test-qmp-output-visitor
+test-rcu-list
 test-rfifolock
 test-string-input-visitor
 test-string-output-visitor
 test-thread-pool
 test-throttle
+test-timed-average
 test-visitor-serialization
 test-vmstate
+test-write-threshold
 test-x86-cpuid
 test-xbzrle
+test-netfilter
+test-filter-mirror
+test-filter-redirector
 *-test
 qapi-schema/*.test.*
diff --git a/tests/Makefile b/tests/Makefile
deleted file mode 100644
index 16f0e4c..0000000
--- a/tests/Makefile
+++ /dev/null
@@ -1,479 +0,0 @@
-export SRC_PATH
-
-# Get the list of all supported sysemu targets
-SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \
-   $(wildcard $(SRC_PATH)/default-configs/*-softmmu.mak)))
-
-check-unit-y = tests/check-qdict$(EXESUF)
-gcov-files-check-qdict-y = qobject/qdict.c
-check-unit-y += tests/check-qfloat$(EXESUF)
-gcov-files-check-qfloat-y = qobject/qfloat.c
-check-unit-y += tests/check-qint$(EXESUF)
-gcov-files-check-qint-y = qobject/qint.c
-check-unit-y += tests/check-qstring$(EXESUF)
-gcov-files-check-qstring-y = qobject/qstring.c
-check-unit-y += tests/check-qlist$(EXESUF)
-gcov-files-check-qlist-y = qobject/qlist.c
-check-unit-y += tests/check-qjson$(EXESUF)
-gcov-files-check-qjson-y = qobject/qjson.c
-check-unit-y += tests/test-qmp-output-visitor$(EXESUF)
-gcov-files-test-qmp-output-visitor-y = qapi/qmp-output-visitor.c
-check-unit-y += tests/test-qmp-input-visitor$(EXESUF)
-gcov-files-test-qmp-input-visitor-y = qapi/qmp-input-visitor.c
-check-unit-y += tests/test-qmp-input-strict$(EXESUF)
-check-unit-y += tests/test-qmp-commands$(EXESUF)
-gcov-files-test-qmp-commands-y = qapi/qmp-dispatch.c
-check-unit-y += tests/test-string-input-visitor$(EXESUF)
-gcov-files-test-string-input-visitor-y = qapi/string-input-visitor.c
-check-unit-y += tests/test-string-output-visitor$(EXESUF)
-gcov-files-test-string-output-visitor-y = qapi/string-output-visitor.c
-check-unit-y += tests/test-qmp-event$(EXESUF)
-gcov-files-test-qmp-event-y += qapi/qmp-event.c
-check-unit-y += tests/test-opts-visitor$(EXESUF)
-gcov-files-test-opts-visitor-y = qapi/opts-visitor.c
-check-unit-y += tests/test-coroutine$(EXESUF)
-gcov-files-test-coroutine-y = coroutine-$(CONFIG_COROUTINE_BACKEND).c
-check-unit-y += tests/test-visitor-serialization$(EXESUF)
-check-unit-y += tests/test-iov$(EXESUF)
-gcov-files-test-iov-y = util/iov.c
-check-unit-y += tests/test-aio$(EXESUF)
-check-unit-$(CONFIG_POSIX) += tests/test-rfifolock$(EXESUF)
-check-unit-y += tests/test-throttle$(EXESUF)
-gcov-files-test-aio-$(CONFIG_WIN32) = aio-win32.c
-gcov-files-test-aio-$(CONFIG_POSIX) = aio-posix.c
-check-unit-y += tests/test-thread-pool$(EXESUF)
-gcov-files-test-thread-pool-y = thread-pool.c
-gcov-files-test-hbitmap-y = util/hbitmap.c
-check-unit-y += tests/test-hbitmap$(EXESUF)
-check-unit-y += tests/test-x86-cpuid$(EXESUF)
-# all code tested by test-x86-cpuid is inside topology.h
-gcov-files-test-x86-cpuid-y =
-check-unit-y += tests/test-xbzrle$(EXESUF)
-gcov-files-test-xbzrle-y = xbzrle.c
-check-unit-y += tests/test-cutils$(EXESUF)
-gcov-files-test-cutils-y += util/cutils.c
-check-unit-y += tests/test-mul64$(EXESUF)
-gcov-files-test-mul64-y = util/host-utils.c
-check-unit-y += tests/test-int128$(EXESUF)
-# all code tested by test-int128 is inside int128.h
-gcov-files-test-int128-y =
-check-unit-y += tests/test-bitops$(EXESUF)
-check-unit-$(CONFIG_HAS_GLIB_SUBPROCESS_TESTS) += tests/test-qdev-global-props$(EXESUF)
-check-unit-y += tests/check-qom-interface$(EXESUF)
-gcov-files-check-qom-interface-y = qom/object.c
-check-unit-$(CONFIG_POSIX) += tests/test-vmstate$(EXESUF)
-check-unit-y += tests/test-qemu-opts$(EXESUF)
-gcov-files-test-qemu-opts-y = qom/test-qemu-opts.c
-
-check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
-
-# All QTests for now are POSIX-only, but the dependencies are
-# really in libqtest, not in the testcases themselves.
-
-gcov-files-ipack-y += hw/ipack/ipack.c
-check-qtest-ipack-y += tests/ipoctal232-test$(EXESUF)
-gcov-files-ipack-y += hw/char/ipoctal232.c
-
-check-qtest-virtioserial-y += tests/virtio-console-test$(EXESUF)
-gcov-files-virtioserial-y += hw/char/virtio-console.c
-
-gcov-files-virtio-y += i386-softmmu/hw/virtio/virtio.c
-check-qtest-virtio-y += tests/virtio-net-test$(EXESUF)
-gcov-files-virtio-y += i386-softmmu/hw/net/virtio-net.c
-check-qtest-virtio-y += tests/virtio-balloon-test$(EXESUF)
-gcov-files-virtio-y += i386-softmmu/hw/virtio/virtio-balloon.c
-check-qtest-virtio-y += tests/virtio-blk-test$(EXESUF)
-gcov-files-virtio-y += i386-softmmu/hw/block/virtio-blk.c
-check-qtest-virtio-y += tests/virtio-rng-test$(EXESUF)
-gcov-files-virtio-y += hw/virtio/virtio-rng.c
-check-qtest-virtio-y += tests/virtio-scsi-test$(EXESUF)
-gcov-files-virtio-y += i386-softmmu/hw/scsi/virtio-scsi.c
-ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
-check-qtest-virtio-y += tests/virtio-9p-test$(EXESUF)
-gcov-files-virtio-y += hw/9pfs/virtio-9p.c
-gcov-files-virtio-y += i386-softmmu/hw/9pfs/virtio-9p-device.c
-endif
-check-qtest-virtio-y += tests/virtio-serial-test$(EXESUF)
-gcov-files-virtio-y += i386-softmmu/hw/char/virtio-serial-bus.c
-check-qtest-virtio-y += $(check-qtest-virtioserial-y)
-gcov-files-virtio-y += $(gcov-files-virtioserial-y)
-
-check-qtest-pci-y += tests/e1000-test$(EXESUF)
-gcov-files-pci-y += hw/net/e1000.c
-check-qtest-pci-y += tests/rtl8139-test$(EXESUF)
-gcov-files-pci-y += hw/net/rtl8139.c
-check-qtest-pci-y += tests/pcnet-test$(EXESUF)
-gcov-files-pci-y += hw/net/pcnet.c
-gcov-files-pci-y += hw/net/pcnet-pci.c
-check-qtest-pci-y += tests/eepro100-test$(EXESUF)
-gcov-files-pci-y += hw/net/eepro100.c
-check-qtest-pci-y += tests/ne2000-test$(EXESUF)
-gcov-files-pci-y += hw/net/ne2000.c
-check-qtest-pci-y += tests/nvme-test$(EXESUF)
-gcov-files-pci-y += hw/block/nvme.c
-check-qtest-pci-y += tests/ac97-test$(EXESUF)
-gcov-files-pci-y += hw/audio/ac97.c
-check-qtest-pci-y += tests/es1370-test$(EXESUF)
-gcov-files-pci-y += hw/audio/es1370.c
-check-qtest-pci-y += $(check-qtest-virtio-y)
-gcov-files-pci-y += $(gcov-files-virtio-y) hw/virtio/virtio-pci.c
-check-qtest-pci-y += tests/tpci200-test$(EXESUF)
-gcov-files-pci-y += hw/ipack/tpci200.c
-check-qtest-pci-y += $(check-qtest-ipack-y)
-gcov-files-pci-y += $(gcov-files-ipack-y)
-check-qtest-pci-y += tests/display-vga-test$(EXESUF)
-gcov-files-pci-y += hw/display/vga.c
-gcov-files-pci-y += hw/display/cirrus_vga.c
-gcov-files-pci-y += hw/display/vga-pci.c
-check-qtest-pci-y += tests/intel-hda-test$(EXESUF)
-gcov-files-pci-y += hw/audio/intel-hda.c hw/audio/hda-codec.c
-
-check-qtest-i386-y = tests/endianness-test$(EXESUF)
-check-qtest-i386-y += tests/fdc-test$(EXESUF)
-gcov-files-i386-y = hw/block/fdc.c
-check-qtest-i386-y += tests/ide-test$(EXESUF)
-check-qtest-i386-y += tests/ahci-test$(EXESUF)
-check-qtest-i386-y += tests/hd-geo-test$(EXESUF)
-gcov-files-i386-y += hw/block/hd-geometry.c
-check-qtest-i386-y += tests/boot-order-test$(EXESUF)
-check-qtest-i386-y += tests/bios-tables-test$(EXESUF)
-check-qtest-i386-y += tests/rtc-test$(EXESUF)
-check-qtest-i386-y += tests/i440fx-test$(EXESUF)
-check-qtest-i386-y += tests/fw_cfg-test$(EXESUF)
-check-qtest-i386-y += tests/drive_del-test$(EXESUF)
-check-qtest-i386-y += tests/wdt_ib700-test$(EXESUF)
-gcov-files-i386-y += hw/watchdog/watchdog.c hw/watchdog/wdt_ib700.c
-check-qtest-i386-y += $(check-qtest-pci-y)
-gcov-files-i386-y += $(gcov-files-pci-y)
-check-qtest-i386-y += tests/vmxnet3-test$(EXESUF)
-gcov-files-i386-y += hw/net/vmxnet3.c
-gcov-files-i386-y += hw/net/vmxnet_rx_pkt.c
-gcov-files-i386-y += hw/net/vmxnet_tx_pkt.c
-check-qtest-i386-y += tests/pvpanic-test$(EXESUF)
-gcov-files-i386-y += i386-softmmu/hw/misc/pvpanic.c
-check-qtest-i386-y += tests/i82801b11-test$(EXESUF)
-gcov-files-i386-y += hw/pci-bridge/i82801b11.c
-check-qtest-i386-y += tests/ioh3420-test$(EXESUF)
-gcov-files-i386-y += hw/pci-bridge/ioh3420.c
-check-qtest-i386-y += tests/usb-hcd-ohci-test$(EXESUF)
-gcov-files-i386-y += hw/usb/hcd-ohci.c
-check-qtest-i386-y += tests/usb-hcd-uhci-test$(EXESUF)
-gcov-files-i386-y += hw/usb/hcd-uhci.c
-check-qtest-i386-y += tests/usb-hcd-ehci-test$(EXESUF)
-gcov-files-i386-y += hw/usb/hcd-ehci.c
-gcov-files-i386-y += hw/usb/dev-hid.c
-gcov-files-i386-y += hw/usb/dev-storage.c
-check-qtest-i386-y += tests/usb-hcd-xhci-test$(EXESUF)
-gcov-files-i386-y += hw/usb/hcd-xhci.c
-check-qtest-i386-$(CONFIG_LINUX) += tests/vhost-user-test$(EXESUF)
-check-qtest-x86_64-y = $(check-qtest-i386-y)
-gcov-files-i386-y += i386-softmmu/hw/timer/mc146818rtc.c
-gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
-check-qtest-mips-y = tests/endianness-test$(EXESUF)
-check-qtest-mips64-y = tests/endianness-test$(EXESUF)
-check-qtest-mips64el-y = tests/endianness-test$(EXESUF)
-check-qtest-ppc-y = tests/endianness-test$(EXESUF)
-check-qtest-ppc64-y = tests/endianness-test$(EXESUF)
-check-qtest-sh4-y = tests/endianness-test$(EXESUF)
-check-qtest-sh4eb-y = tests/endianness-test$(EXESUF)
-check-qtest-sparc64-y = tests/endianness-test$(EXESUF)
-#check-qtest-sparc-y = tests/m48t59-test$(EXESUF)
-#check-qtest-sparc64-y += tests/m48t59-test$(EXESUF)
-gcov-files-sparc-y += hw/timer/m48t59.c
-gcov-files-sparc64-y += hw/timer/m48t59.c
-check-qtest-arm-y = tests/tmp105-test$(EXESUF)
-gcov-files-arm-y += hw/misc/tmp105.c
-check-qtest-ppc-y += tests/boot-order-test$(EXESUF)
-check-qtest-ppc64-y += tests/boot-order-test$(EXESUF)
-check-qtest-ppc64-y += tests/spapr-phb-test$(EXESUF)
-gcov-files-ppc64-y += ppc64-softmmu/hw/ppc/spapr_pci.c
-check-qtest-microblazeel-y = $(check-qtest-microblaze-y)
-check-qtest-xtensaeb-y = $(check-qtest-xtensa-y)
-
-# qom-test works for all sysemu architectures:
-$(foreach target,$(SYSEMU_TARGET_LIST), \
-	$(if $(findstring tests/qom-test$(EXESUF), $(check-qtest-$(target)-y)),, \
-		$(eval check-qtest-$(target)-y += tests/qom-test$(EXESUF))))
-
-check-qapi-schema-y := $(addprefix tests/qapi-schema/, \
-	comments.json empty.json funny-char.json indented-expr.json \
-	missing-colon.json missing-comma-list.json \
-	missing-comma-object.json non-objects.json \
-	qapi-schema-test.json quoted-structural-chars.json \
-	trailing-comma-list.json trailing-comma-object.json \
-	unclosed-list.json unclosed-object.json unclosed-string.json \
-	duplicate-key.json union-invalid-base.json flat-union-no-base.json \
-	flat-union-invalid-discriminator.json \
-	flat-union-invalid-branch-key.json flat-union-reverse-define.json \
-	flat-union-string-discriminator.json \
-	include-simple.json include-relpath.json include-format-err.json \
-	include-non-file.json include-no-file.json include-before-err.json \
-	include-nested-err.json include-self-cycle.json include-cycle.json \
-	include-repetition.json event-nest-struct.json)
-
-GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h \
-		     tests/test-qmp-commands.h tests/test-qapi-event.h
-
-test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
-	tests/check-qlist.o tests/check-qfloat.o tests/check-qjson.o \
-	tests/test-coroutine.o tests/test-string-output-visitor.o \
-	tests/test-string-input-visitor.o tests/test-qmp-output-visitor.o \
-	tests/test-qmp-input-visitor.o tests/test-qmp-input-strict.o \
-	tests/test-qmp-commands.o tests/test-visitor-serialization.o \
-	tests/test-x86-cpuid.o tests/test-mul64.o tests/test-int128.o \
-	tests/test-opts-visitor.o tests/test-qmp-event.o
-
-test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
-		  tests/test-qapi-event.o
-
-$(test-obj-y): QEMU_INCLUDES += -Itests
-QEMU_CFLAGS += -I$(SRC_PATH)/tests
-qom-core-obj = qom/object.o qom/qom-qobject.o qom/container.o
-
-tests/test-x86-cpuid.o: QEMU_INCLUDES += -I$(SRC_PATH)/target-i386
-
-tests/check-qint$(EXESUF): tests/check-qint.o libqemuutil.a
-tests/check-qstring$(EXESUF): tests/check-qstring.o libqemuutil.a
-tests/check-qdict$(EXESUF): tests/check-qdict.o libqemuutil.a
-tests/check-qlist$(EXESUF): tests/check-qlist.o libqemuutil.a
-tests/check-qfloat$(EXESUF): tests/check-qfloat.o libqemuutil.a
-tests/check-qjson$(EXESUF): tests/check-qjson.o libqemuutil.a libqemustub.a
-tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(qom-core-obj) libqemuutil.a libqemustub.a
-tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-aio$(EXESUF): tests/test-aio.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-rfifolock$(EXESUF): tests/test-rfifolock.o libqemuutil.a libqemustub.a
-tests/test-throttle$(EXESUF): tests/test-throttle.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(block-obj-y) libqemuutil.a libqemustub.a
-tests/test-iov$(EXESUF): tests/test-iov.o libqemuutil.a
-tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o libqemuutil.a libqemustub.a
-tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
-tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o xbzrle.o page_cache.o libqemuutil.a
-tests/test-cutils$(EXESUF): tests/test-cutils.o util/cutils.o
-tests/test-int128$(EXESUF): tests/test-int128.o
-tests/test-qdev-global-props$(EXESUF): tests/test-qdev-global-props.o \
-	hw/core/qdev.o hw/core/qdev-properties.o hw/core/hotplug.o\
-	hw/core/irq.o \
-	hw/core/fw-path-provider.o \
-	$(qom-core-obj) \
-	$(test-qapi-obj-y) \
-	libqemuutil.a libqemustub.a
-tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
-	vmstate.o qemu-file.o qemu-file-unix.o \
-	libqemuutil.a libqemustub.a
-
-tests/test-qapi-types.c tests/test-qapi-types.h :\
-$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
-		$(gen-out-type) -o tests -p "test-" -i $<, \
-		"  GEN   $@")
-tests/test-qapi-visit.c tests/test-qapi-visit.h :\
-$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-visit.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
-		$(gen-out-type) -o tests -p "test-" -i $<, \
-		"  GEN   $@")
-tests/test-qmp-commands.h tests/test-qmp-marshal.c :\
-$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-commands.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
-		$(gen-out-type) -o tests -p "test-" -i $<, \
-		"  GEN   $@")
-tests/test-qapi-event.c tests/test-qapi-event.h :\
-$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-event.py
-	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-event.py \
-		$(gen-out-type) -o tests -p "test-" -i $<, \
-		"  GEN   $@")
-
-tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y) libqemuutil.a libqemustub.a
-
-tests/test-mul64$(EXESUF): tests/test-mul64.o libqemuutil.a
-tests/test-bitops$(EXESUF): tests/test-bitops.o libqemuutil.a
-
-libqos-obj-y = tests/libqos/pci.o tests/libqos/fw_cfg.o
-libqos-obj-y += tests/libqos/i2c.o
-libqos-pc-obj-y = $(libqos-obj-y) tests/libqos/pci-pc.o
-libqos-pc-obj-y += tests/libqos/malloc-pc.o
-libqos-omap-obj-y = $(libqos-obj-y) tests/libqos/i2c-omap.o
-libqos-virtio-obj-y = $(libqos-obj-y) $(libqos-pc-obj-y) tests/libqos/virtio.o tests/libqos/virtio-pci.o
-libqos-usb-obj-y = $(libqos-pc-obj-y) tests/libqos/usb.o
-
-tests/rtc-test$(EXESUF): tests/rtc-test.o
-tests/m48t59-test$(EXESUF): tests/m48t59-test.o
-tests/endianness-test$(EXESUF): tests/endianness-test.o
-tests/spapr-phb-test$(EXESUF): tests/spapr-phb-test.o $(libqos-obj-y)
-tests/fdc-test$(EXESUF): tests/fdc-test.o
-tests/ide-test$(EXESUF): tests/ide-test.o $(libqos-pc-obj-y)
-tests/ahci-test$(EXESUF): tests/ahci-test.o $(libqos-pc-obj-y)
-tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
-tests/boot-order-test$(EXESUF): tests/boot-order-test.o $(libqos-obj-y)
-tests/bios-tables-test$(EXESUF): tests/bios-tables-test.o $(libqos-obj-y)
-tests/tmp105-test$(EXESUF): tests/tmp105-test.o $(libqos-omap-obj-y)
-tests/i440fx-test$(EXESUF): tests/i440fx-test.o $(libqos-pc-obj-y)
-tests/fw_cfg-test$(EXESUF): tests/fw_cfg-test.o $(libqos-pc-obj-y)
-tests/e1000-test$(EXESUF): tests/e1000-test.o
-tests/rtl8139-test$(EXESUF): tests/rtl8139-test.o
-tests/pcnet-test$(EXESUF): tests/pcnet-test.o
-tests/eepro100-test$(EXESUF): tests/eepro100-test.o
-tests/vmxnet3-test$(EXESUF): tests/vmxnet3-test.o
-tests/ne2000-test$(EXESUF): tests/ne2000-test.o
-tests/wdt_ib700-test$(EXESUF): tests/wdt_ib700-test.o
-tests/virtio-balloon-test$(EXESUF): tests/virtio-balloon-test.o
-tests/virtio-blk-test$(EXESUF): tests/virtio-blk-test.o $(libqos-virtio-obj-y)
-tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y)
-tests/virtio-rng-test$(EXESUF): tests/virtio-rng-test.o $(libqos-pc-obj-y)
-tests/virtio-scsi-test$(EXESUF): tests/virtio-scsi-test.o
-tests/virtio-9p-test$(EXESUF): tests/virtio-9p-test.o
-tests/virtio-serial-test$(EXESUF): tests/virtio-serial-test.o
-tests/virtio-console-test$(EXESUF): tests/virtio-console-test.o
-tests/tpci200-test$(EXESUF): tests/tpci200-test.o
-tests/display-vga-test$(EXESUF): tests/display-vga-test.o
-tests/ipoctal232-test$(EXESUF): tests/ipoctal232-test.o
-tests/qom-test$(EXESUF): tests/qom-test.o
-tests/drive_del-test$(EXESUF): tests/drive_del-test.o $(libqos-pc-obj-y)
-tests/qdev-monitor-test$(EXESUF): tests/qdev-monitor-test.o $(libqos-pc-obj-y)
-tests/nvme-test$(EXESUF): tests/nvme-test.o
-tests/pvpanic-test$(EXESUF): tests/pvpanic-test.o
-tests/i82801b11-test$(EXESUF): tests/i82801b11-test.o
-tests/ac97-test$(EXESUF): tests/ac97-test.o
-tests/es1370-test$(EXESUF): tests/es1370-test.o
-tests/intel-hda-test$(EXESUF): tests/intel-hda-test.o
-tests/ioh3420-test$(EXESUF): tests/ioh3420-test.o
-tests/usb-hcd-ohci-test$(EXESUF): tests/usb-hcd-ohci-test.o $(libqos-usb-obj-y)
-tests/usb-hcd-uhci-test$(EXESUF): tests/usb-hcd-uhci-test.o $(libqos-usb-obj-y)
-tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o $(libqos-usb-obj-y)
-tests/usb-hcd-xhci-test$(EXESUF): tests/usb-hcd-xhci-test.o $(libqos-usb-obj-y)
-tests/vhost-user-test$(EXESUF): tests/vhost-user-test.o qemu-char.o qemu-timer.o $(qtest-obj-y)
-tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
-tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o libqemuutil.a libqemustub.a
-
-ifeq ($(CONFIG_POSIX),y)
-LIBS += -lutil
-endif
-
-# QTest rules
-
-TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS)))
-ifeq ($(CONFIG_POSIX),y)
-QTEST_TARGETS=$(foreach TARGET,$(TARGETS), $(if $(check-qtest-$(TARGET)-y), $(TARGET),))
-check-qtest-y=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y))
-endif
-
-qtest-obj-y = tests/libqtest.o libqemuutil.a libqemustub.a
-$(check-qtest-y): $(qtest-obj-y)
-
-.PHONY: check-help
-check-help:
-	@echo "Regression testing targets:"
-	@echo
-	@echo " make check                Run all tests"
-	@echo " make check-qtest-TARGET   Run qtest tests for given target"
-	@echo " make check-qtest          Run qtest tests"
-	@echo " make check-unit           Run qobject tests"
-	@echo " make check-qapi-schema    Run QAPI schema tests"
-	@echo " make check-block          Run block tests"
-	@echo " make check-report.html    Generates an HTML test report"
-	@echo " make check-clean          Clean the tests"
-	@echo
-	@echo "Please note that HTML reports do not regenerate if the unit tests"
-	@echo "has not changed."
-	@echo
-	@echo "The variable SPEED can be set to control the gtester speed setting."
-	@echo "Default options are -k and (for make V=1) --verbose; they can be"
-	@echo "changed with variable GTESTER_OPTIONS."
-
-SPEED = quick
-GTESTER_OPTIONS = -k $(if $(V),--verbose,-q)
-GCOV_OPTIONS = -n $(if $(V),-f,)
-
-# gtester tests, possibly with verbose output
-
-.PHONY: $(patsubst %, check-qtest-%, $(QTEST_TARGETS))
-$(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: $(check-qtest-y)
-	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
-	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
-		MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$((RANDOM % 255 + 1))} \
-		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
-	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$*-y); do \
-	  echo Gcov report for $$f:;\
-	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
-	done,)
-
-.PHONY: $(patsubst %, check-%, $(check-unit-y))
-$(patsubst %, check-%, $(check-unit-y)): check-%: %
-	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
-	$(call quiet-command, \
-		MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$((RANDOM % 255 + 1))} \
-		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $*,"GTESTER $*")
-	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$(subst tests/,,$*)-y); do \
-	  echo Gcov report for $$f:;\
-	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
-	done,)
-
-# gtester tests with XML output
-
-$(patsubst %, check-report-qtest-%.xml, $(QTEST_TARGETS)): check-report-qtest-%.xml: $(check-qtest-y)
-	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
-	  gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $(check-qtest-$*-y),"GTESTER $@")
-
-check-report-unit.xml: $(check-unit-y)
-	$(call quiet-command,gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $^, "GTESTER $@")
-
-# Reports and overall runs
-
-check-report.xml: $(patsubst %,check-report-qtest-%.xml, $(QTEST_TARGETS)) check-report-unit.xml
-	$(call quiet-command,$(SRC_PATH)/scripts/gtester-cat $^ > $@, "  GEN    $@")
-
-check-report.html: check-report.xml
-	$(call quiet-command,gtester-report $< > $@, "  GEN    $@")
-
-
-# Other tests
-
-QEMU_IOTESTS_HELPERS-$(CONFIG_LINUX) = tests/qemu-iotests/socket_scm_helper$(EXESUF)
-
-.PHONY: check-tests/qemu-iotests-quick.sh
-check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF) $(QEMU_IOTESTS_HELPERS-y)
-	$<
-
-.PHONY: check-tests/test-qapi.py
-check-tests/test-qapi.py: tests/test-qapi.py
-
-.PHONY: $(patsubst %, check-%, $(check-qapi-schema-y))
-$(patsubst %, check-%, $(check-qapi-schema-y)): check-%.json: $(SRC_PATH)/%.json
-	$(call quiet-command, PYTHONPATH=$(SRC_PATH)/scripts \
-		$(PYTHON) $(SRC_PATH)/tests/qapi-schema/test-qapi.py \
-		$^ >$*.test.out 2>$*.test.err; \
-		echo $$? >$*.test.exit, \
-		"  TEST  $*.out")
-	@diff -q $(SRC_PATH)/$*.out $*.test.out
-	@# Sanitize error messages (make them independent of build directory)
-	@perl -p -e 's|\Q$(SRC_PATH)\E/||g' $*.test.err | diff -q $(SRC_PATH)/$*.err -
-	@diff -q $(SRC_PATH)/$*.exit $*.test.exit
-
-# Consolidated targets
-
-.PHONY: check-qapi-schema check-qtest check-unit check check-clean
-check-qapi-schema: $(patsubst %,check-%, $(check-qapi-schema-y))
-check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS))
-check-unit: $(patsubst %,check-%, $(check-unit-y))
-check-block: $(patsubst %,check-%, $(check-block-y))
-check: check-qapi-schema check-unit check-qtest
-check-clean:
-	$(MAKE) -C tests/tcg clean
-	rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y)
-	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)))
-
-clean: check-clean
-
-# Build the help program automatically
-
-all: $(QEMU_IOTESTS_HELPERS-y)
-
--include $(wildcard tests/*.d)
--include $(wildcard tests/libqos/*.d)
diff --git a/tests/Makefile.include b/tests/Makefile.include
new file mode 100644
index 0000000..14be491
--- /dev/null
+++ b/tests/Makefile.include
@@ -0,0 +1,778 @@
+export SRC_PATH
+
+qapi-py = $(SRC_PATH)/scripts/qapi.py $(SRC_PATH)/scripts/ordereddict.py
+
+# Get the list of all supported sysemu targets
+SYSEMU_TARGET_LIST := $(subst -softmmu.mak,,$(notdir \
+   $(wildcard $(SRC_PATH)/default-configs/*-softmmu.mak)))
+
+check-unit-y = tests/check-qdict$(EXESUF)
+gcov-files-check-qdict-y = qobject/qdict.c
+check-unit-y += tests/check-qfloat$(EXESUF)
+gcov-files-check-qfloat-y = qobject/qfloat.c
+check-unit-y += tests/check-qint$(EXESUF)
+gcov-files-check-qint-y = qobject/qint.c
+check-unit-y += tests/check-qstring$(EXESUF)
+gcov-files-check-qstring-y = qobject/qstring.c
+check-unit-y += tests/check-qlist$(EXESUF)
+gcov-files-check-qlist-y = qobject/qlist.c
+check-unit-y += tests/check-qnull$(EXESUF)
+gcov-files-check-qnull-y = qobject/qnull.c
+check-unit-y += tests/check-qjson$(EXESUF)
+gcov-files-check-qjson-y = qobject/qjson.c
+check-unit-y += tests/test-qmp-output-visitor$(EXESUF)
+gcov-files-test-qmp-output-visitor-y = qapi/qmp-output-visitor.c
+check-unit-y += tests/test-clone-visitor$(EXESUF)
+gcov-files-test-clone-visitor-y = qapi/qapi-clone-visitor.c
+check-unit-y += tests/test-qmp-input-visitor$(EXESUF)
+gcov-files-test-qmp-input-visitor-y = qapi/qmp-input-visitor.c
+check-unit-y += tests/test-qmp-input-strict$(EXESUF)
+check-unit-y += tests/test-qmp-commands$(EXESUF)
+gcov-files-test-qmp-commands-y = qapi/qmp-dispatch.c
+check-unit-y += tests/test-string-input-visitor$(EXESUF)
+gcov-files-test-string-input-visitor-y = qapi/string-input-visitor.c
+check-unit-y += tests/test-string-output-visitor$(EXESUF)
+gcov-files-test-string-output-visitor-y = qapi/string-output-visitor.c
+check-unit-y += tests/test-qmp-event$(EXESUF)
+gcov-files-test-qmp-event-y += qapi/qmp-event.c
+check-unit-y += tests/test-opts-visitor$(EXESUF)
+gcov-files-test-opts-visitor-y = qapi/opts-visitor.c
+check-unit-y += tests/test-coroutine$(EXESUF)
+gcov-files-test-coroutine-y = coroutine-$(CONFIG_COROUTINE_BACKEND).c
+check-unit-y += tests/test-visitor-serialization$(EXESUF)
+check-unit-y += tests/test-iov$(EXESUF)
+gcov-files-test-iov-y = util/iov.c
+check-unit-y += tests/test-aio$(EXESUF)
+check-unit-$(CONFIG_POSIX) += tests/test-rfifolock$(EXESUF)
+check-unit-y += tests/test-throttle$(EXESUF)
+gcov-files-test-aio-$(CONFIG_WIN32) = aio-win32.c
+gcov-files-test-aio-$(CONFIG_POSIX) = aio-posix.c
+check-unit-y += tests/test-thread-pool$(EXESUF)
+gcov-files-test-thread-pool-y = thread-pool.c
+gcov-files-test-hbitmap-y = util/hbitmap.c
+check-unit-y += tests/test-hbitmap$(EXESUF)
+gcov-files-test-hbitmap-y = blockjob.c
+check-unit-y += tests/test-blockjob$(EXESUF)
+check-unit-y += tests/test-blockjob-txn$(EXESUF)
+check-unit-y += tests/test-x86-cpuid$(EXESUF)
+# all code tested by test-x86-cpuid is inside topology.h
+gcov-files-test-x86-cpuid-y =
+ifeq ($(CONFIG_SOFTMMU),y)
+check-unit-y += tests/test-xbzrle$(EXESUF)
+gcov-files-test-xbzrle-y = migration/xbzrle.c
+check-unit-$(CONFIG_POSIX) += tests/test-vmstate$(EXESUF)
+endif
+check-unit-y += tests/test-cutils$(EXESUF)
+gcov-files-test-cutils-y += util/cutils.c
+check-unit-y += tests/test-mul64$(EXESUF)
+gcov-files-test-mul64-y = util/host-utils.c
+check-unit-y += tests/test-int128$(EXESUF)
+# all code tested by test-int128 is inside int128.h
+gcov-files-test-int128-y =
+check-unit-y += tests/rcutorture$(EXESUF)
+gcov-files-rcutorture-y = util/rcu.c
+check-unit-y += tests/test-rcu-list$(EXESUF)
+gcov-files-test-rcu-list-y = util/rcu.c
+check-unit-y += tests/test-qdist$(EXESUF)
+gcov-files-test-qdist-y = util/qdist.c
+check-unit-y += tests/test-qht$(EXESUF)
+gcov-files-test-qht-y = util/qht.c
+check-unit-y += tests/test-qht-par$(EXESUF)
+gcov-files-test-qht-par-y = util/qht.c
+check-unit-y += tests/test-bitops$(EXESUF)
+check-unit-$(CONFIG_HAS_GLIB_SUBPROCESS_TESTS) += tests/test-qdev-global-props$(EXESUF)
+check-unit-y += tests/check-qom-interface$(EXESUF)
+gcov-files-check-qom-interface-y = qom/object.c
+check-unit-y += tests/check-qom-proplist$(EXESUF)
+gcov-files-check-qom-proplist-y = qom/object.c
+check-unit-y += tests/test-qemu-opts$(EXESUF)
+gcov-files-test-qemu-opts-y = qom/test-qemu-opts.c
+check-unit-y += tests/test-write-threshold$(EXESUF)
+gcov-files-test-write-threshold-y = block/write-threshold.c
+check-unit-y += tests/test-crypto-hash$(EXESUF)
+check-unit-y += tests/test-crypto-cipher$(EXESUF)
+check-unit-y += tests/test-crypto-secret$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-crypto-tlscredsx509$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-crypto-tlssession$(EXESUF)
+ifneq (,$(findstring qemu-ga,$(TOOLS)))
+check-unit-$(CONFIG_LINUX) += tests/test-qga$(EXESUF)
+endif
+check-unit-y += tests/test-timed-average$(EXESUF)
+check-unit-y += tests/test-io-task$(EXESUF)
+check-unit-y += tests/test-io-channel-socket$(EXESUF)
+check-unit-y += tests/test-io-channel-file$(EXESUF)
+check-unit-$(CONFIG_GNUTLS) += tests/test-io-channel-tls$(EXESUF)
+check-unit-y += tests/test-io-channel-command$(EXESUF)
+check-unit-y += tests/test-io-channel-buffer$(EXESUF)
+check-unit-y += tests/test-base64$(EXESUF)
+check-unit-$(if $(CONFIG_NETTLE_KDF),y,$(CONFIG_GCRYPT_KDF)) += tests/test-crypto-pbkdf$(EXESUF)
+check-unit-y += tests/test-crypto-ivgen$(EXESUF)
+check-unit-y += tests/test-crypto-afsplit$(EXESUF)
+check-unit-y += tests/test-crypto-xts$(EXESUF)
+check-unit-y += tests/test-crypto-block$(EXESUF)
+gcov-files-test-logging-y = tests/test-logging.c
+check-unit-y += tests/test-logging$(EXESUF)
+
+check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
+
+# All QTests for now are POSIX-only, but the dependencies are
+# really in libqtest, not in the testcases themselves.
+
+check-qtest-generic-y = tests/device-introspect-test$(EXESUF)
+gcov-files-generic-y = qdev-monitor.c qmp.c
+
+gcov-files-ipack-y += hw/ipack/ipack.c
+check-qtest-ipack-y += tests/ipoctal232-test$(EXESUF)
+gcov-files-ipack-y += hw/char/ipoctal232.c
+
+check-qtest-virtioserial-y += tests/virtio-console-test$(EXESUF)
+gcov-files-virtioserial-y += hw/char/virtio-console.c
+
+gcov-files-virtio-y += i386-softmmu/hw/virtio/virtio.c
+check-qtest-virtio-y += tests/virtio-net-test$(EXESUF)
+gcov-files-virtio-y += i386-softmmu/hw/net/virtio-net.c
+check-qtest-virtio-y += tests/virtio-balloon-test$(EXESUF)
+gcov-files-virtio-y += i386-softmmu/hw/virtio/virtio-balloon.c
+check-qtest-virtio-y += tests/virtio-blk-test$(EXESUF)
+gcov-files-virtio-y += i386-softmmu/hw/block/virtio-blk.c
+check-qtest-virtio-y += tests/virtio-rng-test$(EXESUF)
+gcov-files-virtio-y += hw/virtio/virtio-rng.c
+check-qtest-virtio-y += tests/virtio-scsi-test$(EXESUF)
+gcov-files-virtio-y += i386-softmmu/hw/scsi/virtio-scsi.c
+ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
+check-qtest-virtio-y += tests/virtio-9p-test$(EXESUF)
+gcov-files-virtio-y += hw/9pfs/virtio-9p.c
+gcov-files-virtio-y += i386-softmmu/hw/9pfs/virtio-9p-device.c
+endif
+check-qtest-virtio-y += tests/virtio-serial-test$(EXESUF)
+gcov-files-virtio-y += i386-softmmu/hw/char/virtio-serial-bus.c
+check-qtest-virtio-y += $(check-qtest-virtioserial-y)
+gcov-files-virtio-y += $(gcov-files-virtioserial-y)
+
+check-qtest-pci-y += tests/e1000-test$(EXESUF)
+gcov-files-pci-y += hw/net/e1000.c
+check-qtest-pci-y += tests/e1000e-test$(EXESUF)
+gcov-files-pci-y += hw/net/e1000e.c hw/net/e1000e_core.c
+check-qtest-pci-y += tests/rtl8139-test$(EXESUF)
+gcov-files-pci-y += hw/net/rtl8139.c
+check-qtest-pci-y += tests/pcnet-test$(EXESUF)
+gcov-files-pci-y += hw/net/pcnet.c
+gcov-files-pci-y += hw/net/pcnet-pci.c
+check-qtest-pci-y += tests/eepro100-test$(EXESUF)
+gcov-files-pci-y += hw/net/eepro100.c
+check-qtest-pci-y += tests/ne2000-test$(EXESUF)
+gcov-files-pci-y += hw/net/ne2000.c
+check-qtest-pci-y += tests/nvme-test$(EXESUF)
+gcov-files-pci-y += hw/block/nvme.c
+check-qtest-pci-y += tests/ac97-test$(EXESUF)
+gcov-files-pci-y += hw/audio/ac97.c
+check-qtest-pci-y += tests/es1370-test$(EXESUF)
+gcov-files-pci-y += hw/audio/es1370.c
+check-qtest-pci-y += $(check-qtest-virtio-y)
+gcov-files-pci-y += $(gcov-files-virtio-y) hw/virtio/virtio-pci.c
+check-qtest-pci-y += tests/tpci200-test$(EXESUF)
+gcov-files-pci-y += hw/ipack/tpci200.c
+check-qtest-pci-y += $(check-qtest-ipack-y)
+gcov-files-pci-y += $(gcov-files-ipack-y)
+check-qtest-pci-y += tests/display-vga-test$(EXESUF)
+gcov-files-pci-y += hw/display/vga.c
+gcov-files-pci-y += hw/display/cirrus_vga.c
+gcov-files-pci-y += hw/display/vga-pci.c
+gcov-files-pci-y += hw/display/virtio-gpu.c
+gcov-files-pci-y += hw/display/virtio-gpu-pci.c
+gcov-files-pci-$(CONFIG_VIRTIO_VGA) += hw/display/virtio-vga.c
+check-qtest-pci-y += tests/intel-hda-test$(EXESUF)
+gcov-files-pci-y += hw/audio/intel-hda.c hw/audio/hda-codec.c
+check-qtest-pci-$(CONFIG_EVENTFD) += tests/ivshmem-test$(EXESUF)
+gcov-files-pci-y += hw/misc/ivshmem.c
+
+check-qtest-i386-y = tests/endianness-test$(EXESUF)
+check-qtest-i386-y += tests/fdc-test$(EXESUF)
+gcov-files-i386-y = hw/block/fdc.c
+check-qtest-i386-y += tests/ide-test$(EXESUF)
+check-qtest-i386-y += tests/ahci-test$(EXESUF)
+check-qtest-i386-y += tests/hd-geo-test$(EXESUF)
+gcov-files-i386-y += hw/block/hd-geometry.c
+check-qtest-i386-y += tests/boot-order-test$(EXESUF)
+check-qtest-i386-y += tests/bios-tables-test$(EXESUF)
+check-qtest-i386-y += tests/pxe-test$(EXESUF)
+check-qtest-i386-y += tests/rtc-test$(EXESUF)
+check-qtest-i386-y += tests/ipmi-kcs-test$(EXESUF)
+check-qtest-i386-y += tests/ipmi-bt-test$(EXESUF)
+check-qtest-i386-y += tests/i440fx-test$(EXESUF)
+check-qtest-i386-y += tests/fw_cfg-test$(EXESUF)
+check-qtest-i386-y += tests/drive_del-test$(EXESUF)
+check-qtest-i386-y += tests/wdt_ib700-test$(EXESUF)
+check-qtest-i386-y += tests/tco-test$(EXESUF)
+gcov-files-i386-y += hw/watchdog/watchdog.c hw/watchdog/wdt_ib700.c
+check-qtest-i386-y += $(check-qtest-pci-y)
+gcov-files-i386-y += $(gcov-files-pci-y)
+check-qtest-i386-y += tests/vmxnet3-test$(EXESUF)
+gcov-files-i386-y += hw/net/vmxnet3.c
+gcov-files-i386-y += hw/net/net_rx_pkt.c
+gcov-files-i386-y += hw/net/net_tx_pkt.c
+check-qtest-i386-y += tests/pvpanic-test$(EXESUF)
+gcov-files-i386-y += i386-softmmu/hw/misc/pvpanic.c
+check-qtest-i386-y += tests/i82801b11-test$(EXESUF)
+gcov-files-i386-y += hw/pci-bridge/i82801b11.c
+check-qtest-i386-y += tests/ioh3420-test$(EXESUF)
+gcov-files-i386-y += hw/pci-bridge/ioh3420.c
+check-qtest-i386-y += tests/usb-hcd-ohci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-ohci.c
+check-qtest-i386-y += tests/usb-hcd-uhci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-uhci.c
+check-qtest-i386-y += tests/usb-hcd-ehci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-ehci.c
+gcov-files-i386-y += hw/usb/dev-hid.c
+gcov-files-i386-y += hw/usb/dev-storage.c
+check-qtest-i386-y += tests/usb-hcd-xhci-test$(EXESUF)
+gcov-files-i386-y += hw/usb/hcd-xhci.c
+check-qtest-i386-y += tests/pc-cpu-test$(EXESUF)
+check-qtest-i386-y += tests/q35-test$(EXESUF)
+gcov-files-i386-y += hw/pci-host/q35.c
+check-qtest-i386-$(CONFIG_VHOST_NET_TEST_i386) += tests/vhost-user-test$(EXESUF)
+ifeq ($(CONFIG_VHOST_NET_TEST_i386),)
+check-qtest-x86_64-$(CONFIG_VHOST_NET_TEST_x86_64) += tests/vhost-user-test$(EXESUF)
+endif
+check-qtest-i386-y += tests/test-netfilter$(EXESUF)
+check-qtest-i386-y += tests/test-filter-mirror$(EXESUF)
+check-qtest-i386-y += tests/test-filter-redirector$(EXESUF)
+check-qtest-i386-y += tests/postcopy-test$(EXESUF)
+check-qtest-x86_64-y += $(check-qtest-i386-y)
+gcov-files-i386-y += i386-softmmu/hw/timer/mc146818rtc.c
+gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
+check-qtest-mips-y = tests/endianness-test$(EXESUF)
+check-qtest-mips64-y = tests/endianness-test$(EXESUF)
+check-qtest-mips64el-y = tests/endianness-test$(EXESUF)
+check-qtest-ppc-y = tests/endianness-test$(EXESUF)
+check-qtest-ppc64-y = tests/endianness-test$(EXESUF)
+check-qtest-sh4-y = tests/endianness-test$(EXESUF)
+check-qtest-sh4eb-y = tests/endianness-test$(EXESUF)
+check-qtest-sparc64-y = tests/endianness-test$(EXESUF)
+#check-qtest-sparc-y = tests/m48t59-test$(EXESUF)
+#check-qtest-sparc64-y += tests/m48t59-test$(EXESUF)
+gcov-files-sparc-y += hw/timer/m48t59.c
+gcov-files-sparc64-y += hw/timer/m48t59.c
+check-qtest-arm-y = tests/tmp105-test$(EXESUF)
+check-qtest-arm-y += tests/ds1338-test$(EXESUF)
+gcov-files-arm-y += hw/misc/tmp105.c
+check-qtest-arm-y += tests/virtio-blk-test$(EXESUF)
+gcov-files-arm-y += arm-softmmu/hw/block/virtio-blk.c
+check-qtest-ppc-y += tests/boot-order-test$(EXESUF)
+check-qtest-ppc64-y += tests/boot-order-test$(EXESUF)
+check-qtest-ppc-y += tests/drive_del-test$(EXESUF)
+check-qtest-ppc64-y += tests/drive_del-test$(EXESUF)
+check-qtest-ppc64-y += tests/spapr-phb-test$(EXESUF)
+gcov-files-ppc64-y += ppc64-softmmu/hw/ppc/spapr_pci.c
+check-qtest-ppc-y += tests/prom-env-test$(EXESUF)
+check-qtest-ppc64-y += tests/prom-env-test$(EXESUF)
+check-qtest-sparc-y += tests/prom-env-test$(EXESUF)
+#Disabled for now, triggers a TCG bug on 32-bit hosts
+#check-qtest-sparc64-y += tests/prom-env-test$(EXESUF)
+check-qtest-microblazeel-y = $(check-qtest-microblaze-y)
+check-qtest-xtensaeb-y = $(check-qtest-xtensa-y)
+check-qtest-ppc64-y += tests/postcopy-test$(EXESUF)
+
+check-qtest-generic-y += tests/qom-test$(EXESUF)
+
+qapi-schema += alternate-any.json
+qapi-schema += alternate-array.json
+qapi-schema += alternate-base.json
+qapi-schema += alternate-clash.json
+qapi-schema += alternate-conflict-dict.json
+qapi-schema += alternate-conflict-string.json
+qapi-schema += alternate-empty.json
+qapi-schema += alternate-nested.json
+qapi-schema += alternate-unknown.json
+qapi-schema += args-alternate.json
+qapi-schema += args-any.json
+qapi-schema += args-array-empty.json
+qapi-schema += args-array-unknown.json
+qapi-schema += args-bad-boxed.json
+qapi-schema += args-boxed-anon.json
+qapi-schema += args-boxed-empty.json
+qapi-schema += args-boxed-string.json
+qapi-schema += args-int.json
+qapi-schema += args-invalid.json
+qapi-schema += args-member-array-bad.json
+qapi-schema += args-member-case.json
+qapi-schema += args-member-unknown.json
+qapi-schema += args-name-clash.json
+qapi-schema += args-union.json
+qapi-schema += args-unknown.json
+qapi-schema += bad-base.json
+qapi-schema += bad-data.json
+qapi-schema += bad-ident.json
+qapi-schema += bad-type-bool.json
+qapi-schema += bad-type-dict.json
+qapi-schema += bad-type-int.json
+qapi-schema += base-cycle-direct.json
+qapi-schema += base-cycle-indirect.json
+qapi-schema += command-int.json
+qapi-schema += comments.json
+qapi-schema += double-data.json
+qapi-schema += double-type.json
+qapi-schema += duplicate-key.json
+qapi-schema += empty.json
+qapi-schema += enum-bad-name.json
+qapi-schema += enum-bad-prefix.json
+qapi-schema += enum-clash-member.json
+qapi-schema += enum-dict-member.json
+qapi-schema += enum-int-member.json
+qapi-schema += enum-member-case.json
+qapi-schema += enum-missing-data.json
+qapi-schema += enum-wrong-data.json
+qapi-schema += escape-outside-string.json
+qapi-schema += escape-too-big.json
+qapi-schema += escape-too-short.json
+qapi-schema += event-boxed-empty.json
+qapi-schema += event-case.json
+qapi-schema += event-nest-struct.json
+qapi-schema += flat-union-array-branch.json
+qapi-schema += flat-union-bad-base.json
+qapi-schema += flat-union-bad-discriminator.json
+qapi-schema += flat-union-base-any.json
+qapi-schema += flat-union-base-union.json
+qapi-schema += flat-union-clash-member.json
+qapi-schema += flat-union-empty.json
+qapi-schema += flat-union-incomplete-branch.json
+qapi-schema += flat-union-inline.json
+qapi-schema += flat-union-int-branch.json
+qapi-schema += flat-union-invalid-branch-key.json
+qapi-schema += flat-union-invalid-discriminator.json
+qapi-schema += flat-union-no-base.json
+qapi-schema += flat-union-optional-discriminator.json
+qapi-schema += flat-union-string-discriminator.json
+qapi-schema += funny-char.json
+qapi-schema += ident-with-escape.json
+qapi-schema += include-before-err.json
+qapi-schema += include-cycle.json
+qapi-schema += include-format-err.json
+qapi-schema += include-nested-err.json
+qapi-schema += include-no-file.json
+qapi-schema += include-non-file.json
+qapi-schema += include-relpath.json
+qapi-schema += include-repetition.json
+qapi-schema += include-self-cycle.json
+qapi-schema += include-simple.json
+qapi-schema += indented-expr.json
+qapi-schema += leading-comma-list.json
+qapi-schema += leading-comma-object.json
+qapi-schema += missing-colon.json
+qapi-schema += missing-comma-list.json
+qapi-schema += missing-comma-object.json
+qapi-schema += missing-type.json
+qapi-schema += nested-struct-data.json
+qapi-schema += non-objects.json
+qapi-schema += qapi-schema-test.json
+qapi-schema += quoted-structural-chars.json
+qapi-schema += redefined-builtin.json
+qapi-schema += redefined-command.json
+qapi-schema += redefined-event.json
+qapi-schema += redefined-type.json
+qapi-schema += reserved-command-q.json
+qapi-schema += reserved-enum-q.json
+qapi-schema += reserved-member-has.json
+qapi-schema += reserved-member-q.json
+qapi-schema += reserved-member-u.json
+qapi-schema += reserved-member-underscore.json
+qapi-schema += reserved-type-kind.json
+qapi-schema += reserved-type-list.json
+qapi-schema += returns-alternate.json
+qapi-schema += returns-array-bad.json
+qapi-schema += returns-dict.json
+qapi-schema += returns-unknown.json
+qapi-schema += returns-whitelist.json
+qapi-schema += struct-base-clash-deep.json
+qapi-schema += struct-base-clash.json
+qapi-schema += struct-data-invalid.json
+qapi-schema += struct-member-invalid.json
+qapi-schema += trailing-comma-list.json
+qapi-schema += trailing-comma-object.json
+qapi-schema += type-bypass-bad-gen.json
+qapi-schema += unclosed-list.json
+qapi-schema += unclosed-object.json
+qapi-schema += unclosed-string.json
+qapi-schema += unicode-str.json
+qapi-schema += union-base-no-discriminator.json
+qapi-schema += union-branch-case.json
+qapi-schema += union-clash-branches.json
+qapi-schema += union-empty.json
+qapi-schema += union-invalid-base.json
+qapi-schema += union-optional-branch.json
+qapi-schema += union-unknown.json
+qapi-schema += unknown-escape.json
+qapi-schema += unknown-expr-key.json
+check-qapi-schema-y := $(addprefix tests/qapi-schema/, $(qapi-schema))
+
+GENERATED_HEADERS += tests/test-qapi-types.h tests/test-qapi-visit.h \
+	tests/test-qmp-commands.h tests/test-qapi-event.h \
+	tests/test-qmp-introspect.h
+
+test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
+	tests/check-qlist.o tests/check-qfloat.o tests/check-qnull.o \
+	tests/check-qjson.o \
+	tests/test-coroutine.o tests/test-string-output-visitor.o \
+	tests/test-string-input-visitor.o tests/test-qmp-output-visitor.o \
+	tests/test-clone-visitor.o \
+	tests/test-qmp-input-visitor.o tests/test-qmp-input-strict.o \
+	tests/test-qmp-commands.o tests/test-visitor-serialization.o \
+	tests/test-x86-cpuid.o tests/test-mul64.o tests/test-int128.o \
+	tests/test-opts-visitor.o tests/test-qmp-event.o \
+	tests/rcutorture.o tests/test-rcu-list.o \
+	tests/test-qdist.o \
+	tests/test-qht.o tests/qht-bench.o tests/test-qht-par.o
+
+$(test-obj-y): QEMU_INCLUDES += -Itests
+QEMU_CFLAGS += -I$(SRC_PATH)/tests
+
+
+# Deps that are common to various different sets of tests below
+test-util-obj-y = libqemuutil.a libqemustub.a
+test-qom-obj-y = $(qom-obj-y) $(test-util-obj-y)
+test-qapi-obj-y = tests/test-qapi-visit.o tests/test-qapi-types.o \
+	tests/test-qapi-event.o tests/test-qmp-introspect.o \
+	$(test-qom-obj-y)
+test-crypto-obj-y = $(crypto-obj-y) $(test-qom-obj-y)
+test-io-obj-y = $(io-obj-y) $(test-crypto-obj-y)
+test-block-obj-y = $(block-obj-y) $(test-io-obj-y)
+
+tests/check-qint$(EXESUF): tests/check-qint.o $(test-util-obj-y)
+tests/check-qstring$(EXESUF): tests/check-qstring.o $(test-util-obj-y)
+tests/check-qdict$(EXESUF): tests/check-qdict.o $(test-util-obj-y)
+tests/check-qlist$(EXESUF): tests/check-qlist.o $(test-util-obj-y)
+tests/check-qfloat$(EXESUF): tests/check-qfloat.o $(test-util-obj-y)
+tests/check-qnull$(EXESUF): tests/check-qnull.o $(test-util-obj-y)
+tests/check-qjson$(EXESUF): tests/check-qjson.o $(test-util-obj-y)
+tests/check-qom-interface$(EXESUF): tests/check-qom-interface.o $(test-qom-obj-y)
+tests/check-qom-proplist$(EXESUF): tests/check-qom-proplist.o $(test-qom-obj-y)
+tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(test-block-obj-y)
+tests/test-aio$(EXESUF): tests/test-aio.o $(test-block-obj-y)
+tests/test-rfifolock$(EXESUF): tests/test-rfifolock.o $(test-util-obj-y)
+tests/test-throttle$(EXESUF): tests/test-throttle.o $(test-block-obj-y)
+tests/test-blockjob$(EXESUF): tests/test-blockjob.o $(test-block-obj-y) $(test-util-obj-y)
+tests/test-blockjob-txn$(EXESUF): tests/test-blockjob-txn.o $(test-block-obj-y) $(test-util-obj-y)
+tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(test-block-obj-y)
+tests/test-iov$(EXESUF): tests/test-iov.o $(test-util-obj-y)
+tests/test-hbitmap$(EXESUF): tests/test-hbitmap.o $(test-util-obj-y)
+tests/test-x86-cpuid$(EXESUF): tests/test-x86-cpuid.o
+tests/test-xbzrle$(EXESUF): tests/test-xbzrle.o migration/xbzrle.o page_cache.o $(test-util-obj-y)
+tests/test-cutils$(EXESUF): tests/test-cutils.o util/cutils.o
+tests/test-int128$(EXESUF): tests/test-int128.o
+tests/rcutorture$(EXESUF): tests/rcutorture.o $(test-util-obj-y)
+tests/test-rcu-list$(EXESUF): tests/test-rcu-list.o $(test-util-obj-y)
+tests/test-qdist$(EXESUF): tests/test-qdist.o $(test-util-obj-y)
+tests/test-qht$(EXESUF): tests/test-qht.o $(test-util-obj-y)
+tests/test-qht-par$(EXESUF): tests/test-qht-par.o tests/qht-bench$(EXESUF) $(test-util-obj-y)
+tests/qht-bench$(EXESUF): tests/qht-bench.o $(test-util-obj-y)
+
+tests/test-qdev-global-props$(EXESUF): tests/test-qdev-global-props.o \
+	hw/core/qdev.o hw/core/qdev-properties.o hw/core/hotplug.o\
+	hw/core/bus.o \
+	hw/core/irq.o \
+	hw/core/fw-path-provider.o \
+	$(test-qapi-obj-y)
+tests/test-vmstate$(EXESUF): tests/test-vmstate.o \
+	migration/vmstate.o migration/qemu-file.o \
+        migration/qemu-file-channel.o migration/qjson.o \
+	$(test-io-obj-y)
+tests/test-timed-average$(EXESUF): tests/test-timed-average.o qemu-timer.o \
+	$(test-util-obj-y)
+tests/test-base64$(EXESUF): tests/test-base64.o \
+	libqemuutil.a libqemustub.a
+
+tests/test-logging$(EXESUF): tests/test-logging.o $(test-util-obj-y)
+
+tests/test-qapi-types.c tests/test-qapi-types.h :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-types.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-types.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
+tests/test-qapi-visit.c tests/test-qapi-visit.h :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-visit.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-visit.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
+tests/test-qmp-commands.h tests/test-qmp-marshal.c :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-commands.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
+tests/test-qapi-event.c tests/test-qapi-event.h :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-event.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-event.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
+tests/test-qmp-introspect.c tests/test-qmp-introspect.h :\
+$(SRC_PATH)/tests/qapi-schema/qapi-schema-test.json $(SRC_PATH)/scripts/qapi-introspect.py $(qapi-py)
+	$(call quiet-command,$(PYTHON) $(SRC_PATH)/scripts/qapi-introspect.py \
+		$(gen-out-type) -o tests -p "test-" $<, \
+		"  GEN   $@")
+
+tests/test-string-output-visitor$(EXESUF): tests/test-string-output-visitor.o $(test-qapi-obj-y)
+tests/test-string-input-visitor$(EXESUF): tests/test-string-input-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-event$(EXESUF): tests/test-qmp-event.o $(test-qapi-obj-y)
+tests/test-qmp-output-visitor$(EXESUF): tests/test-qmp-output-visitor.o $(test-qapi-obj-y)
+tests/test-clone-visitor$(EXESUF): tests/test-clone-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-input-visitor$(EXESUF): tests/test-qmp-input-visitor.o $(test-qapi-obj-y)
+tests/test-qmp-input-strict$(EXESUF): tests/test-qmp-input-strict.o $(test-qapi-obj-y)
+tests/test-qmp-commands$(EXESUF): tests/test-qmp-commands.o tests/test-qmp-marshal.o $(test-qapi-obj-y)
+tests/test-visitor-serialization$(EXESUF): tests/test-visitor-serialization.o $(test-qapi-obj-y)
+tests/test-opts-visitor$(EXESUF): tests/test-opts-visitor.o $(test-qapi-obj-y)
+
+tests/test-mul64$(EXESUF): tests/test-mul64.o $(test-util-obj-y)
+tests/test-bitops$(EXESUF): tests/test-bitops.o $(test-util-obj-y)
+tests/test-crypto-hash$(EXESUF): tests/test-crypto-hash.o $(test-crypto-obj-y)
+tests/test-crypto-cipher$(EXESUF): tests/test-crypto-cipher.o $(test-crypto-obj-y)
+tests/test-crypto-secret$(EXESUF): tests/test-crypto-secret.o $(test-crypto-obj-y)
+tests/test-crypto-xts$(EXESUF): tests/test-crypto-xts.o $(test-crypto-obj-y)
+
+tests/crypto-tls-x509-helpers.o-cflags := $(TASN1_CFLAGS)
+tests/crypto-tls-x509-helpers.o-libs := $(TASN1_LIBS)
+tests/pkix_asn1_tab.o-cflags := $(TASN1_CFLAGS)
+
+tests/test-crypto-tlscredsx509.o-cflags := $(TASN1_CFLAGS)
+tests/test-crypto-tlscredsx509$(EXESUF): tests/test-crypto-tlscredsx509.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o $(test-crypto-obj-y)
+
+tests/test-crypto-tlssession.o-cflags := $(TASN1_CFLAGS)
+tests/test-crypto-tlssession$(EXESUF): tests/test-crypto-tlssession.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o $(test-crypto-obj-y)
+tests/test-io-task$(EXESUF): tests/test-io-task.o $(test-io-obj-y)
+tests/test-io-channel-socket$(EXESUF): tests/test-io-channel-socket.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-file$(EXESUF): tests/test-io-channel-file.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-tls$(EXESUF): tests/test-io-channel-tls.o \
+	tests/crypto-tls-x509-helpers.o tests/pkix_asn1_tab.o \
+	tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-command$(EXESUF): tests/test-io-channel-command.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-io-channel-buffer$(EXESUF): tests/test-io-channel-buffer.o \
+        tests/io-channel-helpers.o $(test-io-obj-y)
+tests/test-crypto-pbkdf$(EXESUF): tests/test-crypto-pbkdf.o $(test-crypto-obj-y)
+tests/test-crypto-ivgen$(EXESUF): tests/test-crypto-ivgen.o $(test-crypto-obj-y)
+tests/test-crypto-afsplit$(EXESUF): tests/test-crypto-afsplit.o $(test-crypto-obj-y)
+tests/test-crypto-block$(EXESUF): tests/test-crypto-block.o $(test-crypto-obj-y)
+
+libqos-obj-y = tests/libqos/pci.o tests/libqos/fw_cfg.o tests/libqos/malloc.o
+libqos-obj-y += tests/libqos/i2c.o tests/libqos/libqos.o
+libqos-pc-obj-y = $(libqos-obj-y) tests/libqos/pci-pc.o
+libqos-pc-obj-y += tests/libqos/malloc-pc.o tests/libqos/libqos-pc.o
+libqos-pc-obj-y += tests/libqos/ahci.o
+libqos-omap-obj-y = $(libqos-obj-y) tests/libqos/i2c-omap.o
+libqos-imx-obj-y = $(libqos-obj-y) tests/libqos/i2c-imx.o
+libqos-usb-obj-y = $(libqos-pc-obj-y) tests/libqos/usb.o
+libqos-virtio-obj-y = $(libqos-pc-obj-y) tests/libqos/virtio.o tests/libqos/virtio-pci.o tests/libqos/virtio-mmio.o tests/libqos/malloc-generic.o
+
+tests/device-introspect-test$(EXESUF): tests/device-introspect-test.o
+tests/rtc-test$(EXESUF): tests/rtc-test.o
+tests/m48t59-test$(EXESUF): tests/m48t59-test.o
+tests/endianness-test$(EXESUF): tests/endianness-test.o
+tests/spapr-phb-test$(EXESUF): tests/spapr-phb-test.o $(libqos-obj-y)
+tests/prom-env-test$(EXESUF): tests/prom-env-test.o $(libqos-obj-y)
+tests/fdc-test$(EXESUF): tests/fdc-test.o
+tests/ide-test$(EXESUF): tests/ide-test.o $(libqos-pc-obj-y)
+tests/ahci-test$(EXESUF): tests/ahci-test.o $(libqos-pc-obj-y)
+tests/ipmi-kcs-test$(EXESUF): tests/ipmi-kcs-test.o
+tests/ipmi-bt-test$(EXESUF): tests/ipmi-bt-test.o
+tests/hd-geo-test$(EXESUF): tests/hd-geo-test.o
+tests/boot-order-test$(EXESUF): tests/boot-order-test.o $(libqos-obj-y)
+tests/bios-tables-test$(EXESUF): tests/bios-tables-test.o \
+	tests/boot-sector.o $(libqos-obj-y)
+tests/pxe-test$(EXESUF): tests/pxe-test.o tests/boot-sector.o $(libqos-obj-y)
+tests/tmp105-test$(EXESUF): tests/tmp105-test.o $(libqos-omap-obj-y)
+tests/ds1338-test$(EXESUF): tests/ds1338-test.o $(libqos-imx-obj-y)
+tests/i440fx-test$(EXESUF): tests/i440fx-test.o $(libqos-pc-obj-y)
+tests/q35-test$(EXESUF): tests/q35-test.o $(libqos-pc-obj-y)
+tests/fw_cfg-test$(EXESUF): tests/fw_cfg-test.o $(libqos-pc-obj-y)
+tests/e1000-test$(EXESUF): tests/e1000-test.o
+tests/e1000e-test$(EXESUF): tests/e1000e-test.o $(libqos-pc-obj-y)
+tests/rtl8139-test$(EXESUF): tests/rtl8139-test.o $(libqos-pc-obj-y)
+tests/pcnet-test$(EXESUF): tests/pcnet-test.o
+tests/eepro100-test$(EXESUF): tests/eepro100-test.o
+tests/vmxnet3-test$(EXESUF): tests/vmxnet3-test.o
+tests/ne2000-test$(EXESUF): tests/ne2000-test.o
+tests/wdt_ib700-test$(EXESUF): tests/wdt_ib700-test.o
+tests/tco-test$(EXESUF): tests/tco-test.o $(libqos-pc-obj-y)
+tests/virtio-balloon-test$(EXESUF): tests/virtio-balloon-test.o
+tests/virtio-blk-test$(EXESUF): tests/virtio-blk-test.o $(libqos-virtio-obj-y)
+tests/virtio-net-test$(EXESUF): tests/virtio-net-test.o $(libqos-pc-obj-y) $(libqos-virtio-obj-y)
+tests/virtio-rng-test$(EXESUF): tests/virtio-rng-test.o $(libqos-pc-obj-y)
+tests/virtio-scsi-test$(EXESUF): tests/virtio-scsi-test.o $(libqos-virtio-obj-y)
+tests/virtio-9p-test$(EXESUF): tests/virtio-9p-test.o
+tests/virtio-serial-test$(EXESUF): tests/virtio-serial-test.o
+tests/virtio-console-test$(EXESUF): tests/virtio-console-test.o
+tests/tpci200-test$(EXESUF): tests/tpci200-test.o
+tests/display-vga-test$(EXESUF): tests/display-vga-test.o
+tests/ipoctal232-test$(EXESUF): tests/ipoctal232-test.o
+tests/qom-test$(EXESUF): tests/qom-test.o
+tests/drive_del-test$(EXESUF): tests/drive_del-test.o $(libqos-pc-obj-y)
+tests/qdev-monitor-test$(EXESUF): tests/qdev-monitor-test.o $(libqos-pc-obj-y)
+tests/nvme-test$(EXESUF): tests/nvme-test.o
+tests/pvpanic-test$(EXESUF): tests/pvpanic-test.o
+tests/i82801b11-test$(EXESUF): tests/i82801b11-test.o
+tests/ac97-test$(EXESUF): tests/ac97-test.o
+tests/es1370-test$(EXESUF): tests/es1370-test.o
+tests/intel-hda-test$(EXESUF): tests/intel-hda-test.o
+tests/ioh3420-test$(EXESUF): tests/ioh3420-test.o
+tests/usb-hcd-ohci-test$(EXESUF): tests/usb-hcd-ohci-test.o $(libqos-usb-obj-y)
+tests/usb-hcd-uhci-test$(EXESUF): tests/usb-hcd-uhci-test.o $(libqos-usb-obj-y)
+tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o $(libqos-usb-obj-y)
+tests/usb-hcd-xhci-test$(EXESUF): tests/usb-hcd-xhci-test.o $(libqos-usb-obj-y)
+tests/pc-cpu-test$(EXESUF): tests/pc-cpu-test.o
+tests/postcopy-test$(EXESUF): tests/postcopy-test.o
+tests/vhost-user-test$(EXESUF): tests/vhost-user-test.o qemu-char.o qemu-timer.o $(qtest-obj-y) $(test-io-obj-y)
+tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
+tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o $(test-util-obj-y)
+tests/test-write-threshold$(EXESUF): tests/test-write-threshold.o $(test-block-obj-y)
+tests/test-netfilter$(EXESUF): tests/test-netfilter.o $(qtest-obj-y)
+tests/test-filter-mirror$(EXESUF): tests/test-filter-mirror.o $(qtest-obj-y)
+tests/test-filter-redirector$(EXESUF): tests/test-filter-redirector.o $(qtest-obj-y)
+tests/ivshmem-test$(EXESUF): tests/ivshmem-test.o contrib/ivshmem-server/ivshmem-server.o $(libqos-pc-obj-y)
+tests/vhost-user-bridge$(EXESUF): tests/vhost-user-bridge.o
+
+tests/migration/stress$(EXESUF): tests/migration/stress.o
+	$(call quiet-command, $(LINKPROG) -static -O3 $(PTHREAD_LIB) -o $@ $< ,"  LINK  $(TARGET_DIR)$@")
+
+INITRD_WORK_DIR=tests/migration/initrd
+
+tests/migration/initrd-stress.img: tests/migration/stress$(EXESUF)
+	mkdir -p $(INITRD_WORK_DIR)
+	cp $< $(INITRD_WORK_DIR)/init
+	(cd $(INITRD_WORK_DIR) && (find | cpio --quiet -o -H newc | gzip -9)) > $@
+	rm $(INITRD_WORK_DIR)/init
+	rmdir $(INITRD_WORK_DIR)
+
+ifeq ($(CONFIG_POSIX),y)
+LIBS += -lutil
+endif
+
+# QTest rules
+
+TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS)))
+ifeq ($(CONFIG_POSIX),y)
+QTEST_TARGETS = $(TARGETS)
+check-qtest-y=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y))
+check-qtest-y += $(check-qtest-generic-y)
+else
+QTEST_TARGETS =
+endif
+
+qtest-obj-y = tests/libqtest.o $(test-util-obj-y)
+$(check-qtest-y): $(qtest-obj-y)
+
+tests/test-qga: tests/test-qga.o $(qtest-obj-y)
+
+.PHONY: check-help
+check-help:
+	@echo "Regression testing targets:"
+	@echo
+	@echo " make check                Run all tests"
+	@echo " make check-qtest-TARGET   Run qtest tests for given target"
+	@echo " make check-qtest          Run qtest tests"
+	@echo " make check-unit           Run qobject tests"
+	@echo " make check-qapi-schema    Run QAPI schema tests"
+	@echo " make check-block          Run block tests"
+	@echo " make check-report.html    Generates an HTML test report"
+	@echo " make check-clean          Clean the tests"
+	@echo
+	@echo "Please note that HTML reports do not regenerate if the unit tests"
+	@echo "has not changed."
+	@echo
+	@echo "The variable SPEED can be set to control the gtester speed setting."
+	@echo "Default options are -k and (for make V=1) --verbose; they can be"
+	@echo "changed with variable GTESTER_OPTIONS."
+
+SPEED = quick
+GTESTER_OPTIONS = -k $(if $(V),--verbose,-q)
+GCOV_OPTIONS = -n $(if $(V),-f,)
+
+# gtester tests, possibly with verbose output
+
+.PHONY: $(patsubst %, check-qtest-%, $(QTEST_TARGETS))
+$(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: $(check-qtest-y)
+	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
+	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
+		QTEST_QEMU_IMG=qemu-img$(EXESUF) \
+		MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$((RANDOM % 255 + 1))} \
+		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $(check-qtest-$*-y) $(check-qtest-generic-y),"GTESTER $@")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$*-y) $(gcov-files-generic-y); do \
+	  echo Gcov report for $$f:;\
+	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
+	done,)
+
+.PHONY: $(patsubst %, check-%, $(check-unit-y))
+$(patsubst %, check-%, $(check-unit-y)): check-%: %
+	$(if $(CONFIG_GCOV),@rm -f *.gcda */*.gcda */*/*.gcda */*/*/*.gcda,)
+	$(call quiet-command, \
+		MALLOC_PERTURB_=$${MALLOC_PERTURB_:-$$((RANDOM % 255 + 1))} \
+		gtester $(GTESTER_OPTIONS) -m=$(SPEED) $*,"GTESTER $*")
+	$(if $(CONFIG_GCOV),@for f in $(gcov-files-$(subst tests/,,$*)-y) $(gcov-files-generic-y); do \
+	  echo Gcov report for $$f:;\
+	  $(GCOV) $(GCOV_OPTIONS) $$f -o `dirname $$f`; \
+	done,)
+
+# gtester tests with XML output
+
+$(patsubst %, check-report-qtest-%.xml, $(QTEST_TARGETS)): check-report-qtest-%.xml: $(check-qtest-y)
+	$(call quiet-command,QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \
+		QTEST_QEMU_IMG=qemu-img$(EXESUF) \
+	  gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $(check-qtest-$*-y) $(check-qtest-generic-y),"GTESTER $@")
+
+check-report-unit.xml: $(check-unit-y)
+	$(call quiet-command,gtester -q $(GTESTER_OPTIONS) -o $@ -m=$(SPEED) $^, "GTESTER $@")
+
+# Reports and overall runs
+
+check-report.xml: $(patsubst %,check-report-qtest-%.xml, $(QTEST_TARGETS)) check-report-unit.xml
+	$(call quiet-command,$(SRC_PATH)/scripts/gtester-cat $^ > $@, "  GEN   $@")
+
+check-report.html: check-report.xml
+	$(call quiet-command,gtester-report $< > $@, "  GEN   $@")
+
+
+# Other tests
+
+QEMU_IOTESTS_HELPERS-$(CONFIG_LINUX) = tests/qemu-iotests/socket_scm_helper$(EXESUF)
+
+.PHONY: check-tests/qemu-iotests-quick.sh
+check-tests/qemu-iotests-quick.sh: tests/qemu-iotests-quick.sh qemu-img$(EXESUF) qemu-io$(EXESUF) $(QEMU_IOTESTS_HELPERS-y)
+	$<
+
+.PHONY: check-tests/test-qapi.py
+check-tests/test-qapi.py: tests/test-qapi.py
+
+.PHONY: $(patsubst %, check-%, $(check-qapi-schema-y))
+$(patsubst %, check-%, $(check-qapi-schema-y)): check-%.json: $(SRC_PATH)/%.json
+	$(call quiet-command, PYTHONPATH=$(SRC_PATH)/scripts \
+		$(PYTHON) $(SRC_PATH)/tests/qapi-schema/test-qapi.py \
+		$^ >$*.test.out 2>$*.test.err; \
+		echo $$? >$*.test.exit, \
+		"  TEST  $*.out")
+	@diff -q $(SRC_PATH)/$*.out $*.test.out
+	@# Sanitize error messages (make them independent of build directory)
+	@perl -p -e 's|\Q$(SRC_PATH)\E/||g' $*.test.err | diff -q $(SRC_PATH)/$*.err -
+	@diff -q $(SRC_PATH)/$*.exit $*.test.exit
+
+# Consolidated targets
+
+.PHONY: check-qapi-schema check-qtest check-unit check check-clean
+check-qapi-schema: $(patsubst %,check-%, $(check-qapi-schema-y))
+check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS))
+check-unit: $(patsubst %,check-%, $(check-unit-y))
+check-block: $(patsubst %,check-%, $(check-block-y))
+check: check-qapi-schema check-unit check-qtest
+check-clean:
+	$(MAKE) -C tests/tcg clean
+	rm -rf $(check-unit-y) tests/*.o $(QEMU_IOTESTS_HELPERS-y)
+	rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y)) $(check-qtest-generic-y))
+
+clean: check-clean
+
+# Build the help program automatically
+
+all: $(QEMU_IOTESTS_HELPERS-y)
+
+-include $(wildcard tests/*.d)
+-include $(wildcard tests/libqos/*.d)
diff --git a/tests/ac97-test.c b/tests/ac97-test.c
index af30ea1..e0d177b 100644
--- a/tests/ac97-test.c
+++ b/tests/ac97-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/acpi-test-data/pc/APIC.cphp b/tests/acpi-test-data/pc/APIC.cphp
new file mode 100644
index 0000000..1bf8a0a
--- /dev/null
+++ b/tests/acpi-test-data/pc/APIC.cphp
Binary files differ
diff --git a/tests/acpi-test-data/pc/DSDT b/tests/acpi-test-data/pc/DSDT
index ee9cc67..8053d71 100644
--- a/tests/acpi-test-data/pc/DSDT
+++ b/tests/acpi-test-data/pc/DSDT
Binary files differ
diff --git a/tests/acpi-test-data/pc/DSDT.bridge b/tests/acpi-test-data/pc/DSDT.bridge
new file mode 100644
index 0000000..850e71a
--- /dev/null
+++ b/tests/acpi-test-data/pc/DSDT.bridge
Binary files differ
diff --git a/tests/acpi-test-data/pc/DSDT.cphp b/tests/acpi-test-data/pc/DSDT.cphp
new file mode 100644
index 0000000..e8b1462
--- /dev/null
+++ b/tests/acpi-test-data/pc/DSDT.cphp
Binary files differ
diff --git a/tests/acpi-test-data/pc/DSDT.ipmikcs b/tests/acpi-test-data/pc/DSDT.ipmikcs
new file mode 100644
index 0000000..8ac48af
--- /dev/null
+++ b/tests/acpi-test-data/pc/DSDT.ipmikcs
Binary files differ
diff --git a/tests/acpi-test-data/pc/SSDT b/tests/acpi-test-data/pc/SSDT
deleted file mode 100644
index 558e4c8..0000000
--- a/tests/acpi-test-data/pc/SSDT
+++ /dev/null
Binary files differ
diff --git a/tests/acpi-test-data/q35/APIC.cphp b/tests/acpi-test-data/q35/APIC.cphp
new file mode 100644
index 0000000..1bf8a0a
--- /dev/null
+++ b/tests/acpi-test-data/q35/APIC.cphp
Binary files differ
diff --git a/tests/acpi-test-data/q35/DSDT b/tests/acpi-test-data/q35/DSDT
index ef0c75f..58fbb3d 100644
--- a/tests/acpi-test-data/q35/DSDT
+++ b/tests/acpi-test-data/q35/DSDT
Binary files differ
diff --git a/tests/acpi-test-data/q35/DSDT.bridge b/tests/acpi-test-data/q35/DSDT.bridge
new file mode 100644
index 0000000..c392802
--- /dev/null
+++ b/tests/acpi-test-data/q35/DSDT.bridge
Binary files differ
diff --git a/tests/acpi-test-data/q35/DSDT.cphp b/tests/acpi-test-data/q35/DSDT.cphp
new file mode 100644
index 0000000..6cc28c6
--- /dev/null
+++ b/tests/acpi-test-data/q35/DSDT.cphp
Binary files differ
diff --git a/tests/acpi-test-data/q35/DSDT.ipmibt b/tests/acpi-test-data/q35/DSDT.ipmibt
new file mode 100644
index 0000000..0ea38e1
--- /dev/null
+++ b/tests/acpi-test-data/q35/DSDT.ipmibt
Binary files differ
diff --git a/tests/acpi-test-data/q35/SSDT b/tests/acpi-test-data/q35/SSDT
deleted file mode 100644
index 4e45510..0000000
--- a/tests/acpi-test-data/q35/SSDT
+++ /dev/null
Binary files differ
diff --git a/tests/ahci-test.c b/tests/ahci-test.c
index 4c77ebe..9c0adce 100644
--- a/tests/ahci-test.c
+++ b/tests/ahci-test.c
@@ -22,15 +22,13 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include <getopt.h>
-#include <glib.h>
 
 #include "libqtest.h"
+#include "libqos/libqos-pc.h"
+#include "libqos/ahci.h"
 #include "libqos/pci-pc.h"
-#include "libqos/malloc-pc.h"
 
 #include "qemu-common.h"
 #include "qemu/host-utils.h"
@@ -38,367 +36,34 @@
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/pci_regs.h"
 
-/* Test-specific defines. */
-#define TEST_IMAGE_SIZE    (64 * 1024 * 1024)
-
-/*** Supplementary PCI Config Space IDs & Masks ***/
-#define PCI_DEVICE_ID_INTEL_Q35_AHCI   (0x2922)
-#define PCI_MSI_FLAGS_RESERVED         (0xFF00)
-#define PCI_PM_CTRL_RESERVED             (0xFC)
-#define PCI_BCC(REG32)          ((REG32) >> 24)
-#define PCI_PI(REG32)   (((REG32) >> 8) & 0xFF)
-#define PCI_SCC(REG32) (((REG32) >> 16) & 0xFF)
-
-/*** Recognized AHCI Device Types ***/
-#define AHCI_INTEL_ICH9 (PCI_DEVICE_ID_INTEL_Q35_AHCI << 16 | \
-                         PCI_VENDOR_ID_INTEL)
-
-/*** AHCI/HBA Register Offsets and Bitmasks ***/
-#define AHCI_CAP                          (0)
-#define AHCI_CAP_NP                    (0x1F)
-#define AHCI_CAP_SXS                   (0x20)
-#define AHCI_CAP_EMS                   (0x40)
-#define AHCI_CAP_CCCS                  (0x80)
-#define AHCI_CAP_NCS                 (0x1F00)
-#define AHCI_CAP_PSC                 (0x2000)
-#define AHCI_CAP_SSC                 (0x4000)
-#define AHCI_CAP_PMD                 (0x8000)
-#define AHCI_CAP_FBSS               (0x10000)
-#define AHCI_CAP_SPM                (0x20000)
-#define AHCI_CAP_SAM                (0x40000)
-#define AHCI_CAP_RESERVED           (0x80000)
-#define AHCI_CAP_ISS               (0xF00000)
-#define AHCI_CAP_SCLO             (0x1000000)
-#define AHCI_CAP_SAL              (0x2000000)
-#define AHCI_CAP_SALP             (0x4000000)
-#define AHCI_CAP_SSS              (0x8000000)
-#define AHCI_CAP_SMPS            (0x10000000)
-#define AHCI_CAP_SSNTF           (0x20000000)
-#define AHCI_CAP_SNCQ            (0x40000000)
-#define AHCI_CAP_S64A            (0x80000000)
-
-#define AHCI_GHC                          (1)
-#define AHCI_GHC_HR                    (0x01)
-#define AHCI_GHC_IE                    (0x02)
-#define AHCI_GHC_MRSM                  (0x04)
-#define AHCI_GHC_RESERVED        (0x7FFFFFF8)
-#define AHCI_GHC_AE              (0x80000000)
-
-#define AHCI_IS                           (2)
-#define AHCI_PI                           (3)
-#define AHCI_VS                           (4)
-
-#define AHCI_CCCCTL                       (5)
-#define AHCI_CCCCTL_EN                 (0x01)
-#define AHCI_CCCCTL_RESERVED           (0x06)
-#define AHCI_CCCCTL_CC               (0xFF00)
-#define AHCI_CCCCTL_TV           (0xFFFF0000)
-
-#define AHCI_CCCPORTS                     (6)
-#define AHCI_EMLOC                        (7)
-
-#define AHCI_EMCTL                        (8)
-#define AHCI_EMCTL_STSMR               (0x01)
-#define AHCI_EMCTL_CTLTM              (0x100)
-#define AHCI_EMCTL_CTLRST             (0x200)
-#define AHCI_EMCTL_RESERVED      (0xF0F0FCFE)
-
-#define AHCI_CAP2                         (9)
-#define AHCI_CAP2_BOH                  (0x01)
-#define AHCI_CAP2_NVMP                 (0x02)
-#define AHCI_CAP2_APST                 (0x04)
-#define AHCI_CAP2_RESERVED       (0xFFFFFFF8)
-
-#define AHCI_BOHC                        (10)
-#define AHCI_RESERVED                    (11)
-#define AHCI_NVMHCI                      (24)
-#define AHCI_VENDOR                      (40)
-#define AHCI_PORTS                       (64)
-
-/*** Port Memory Offsets & Bitmasks ***/
-#define AHCI_PX_CLB                       (0)
-#define AHCI_PX_CLB_RESERVED          (0x1FF)
-
-#define AHCI_PX_CLBU                      (1)
-
-#define AHCI_PX_FB                        (2)
-#define AHCI_PX_FB_RESERVED            (0xFF)
-
-#define AHCI_PX_FBU                       (3)
-
-#define AHCI_PX_IS                        (4)
-#define AHCI_PX_IS_DHRS                 (0x1)
-#define AHCI_PX_IS_PSS                  (0x2)
-#define AHCI_PX_IS_DSS                  (0x4)
-#define AHCI_PX_IS_SDBS                 (0x8)
-#define AHCI_PX_IS_UFS                 (0x10)
-#define AHCI_PX_IS_DPS                 (0x20)
-#define AHCI_PX_IS_PCS                 (0x40)
-#define AHCI_PX_IS_DMPS                (0x80)
-#define AHCI_PX_IS_RESERVED       (0x23FFF00)
-#define AHCI_PX_IS_PRCS            (0x400000)
-#define AHCI_PX_IS_IPMS            (0x800000)
-#define AHCI_PX_IS_OFS            (0x1000000)
-#define AHCI_PX_IS_INFS           (0x4000000)
-#define AHCI_PX_IS_IFS            (0x8000000)
-#define AHCI_PX_IS_HBDS          (0x10000000)
-#define AHCI_PX_IS_HBFS          (0x20000000)
-#define AHCI_PX_IS_TFES          (0x40000000)
-#define AHCI_PX_IS_CPDS          (0x80000000)
-
-#define AHCI_PX_IE                        (5)
-#define AHCI_PX_IE_DHRE                 (0x1)
-#define AHCI_PX_IE_PSE                  (0x2)
-#define AHCI_PX_IE_DSE                  (0x4)
-#define AHCI_PX_IE_SDBE                 (0x8)
-#define AHCI_PX_IE_UFE                 (0x10)
-#define AHCI_PX_IE_DPE                 (0x20)
-#define AHCI_PX_IE_PCE                 (0x40)
-#define AHCI_PX_IE_DMPE                (0x80)
-#define AHCI_PX_IE_RESERVED       (0x23FFF00)
-#define AHCI_PX_IE_PRCE            (0x400000)
-#define AHCI_PX_IE_IPME            (0x800000)
-#define AHCI_PX_IE_OFE            (0x1000000)
-#define AHCI_PX_IE_INFE           (0x4000000)
-#define AHCI_PX_IE_IFE            (0x8000000)
-#define AHCI_PX_IE_HBDE          (0x10000000)
-#define AHCI_PX_IE_HBFE          (0x20000000)
-#define AHCI_PX_IE_TFEE          (0x40000000)
-#define AHCI_PX_IE_CPDE          (0x80000000)
-
-#define AHCI_PX_CMD                       (6)
-#define AHCI_PX_CMD_ST                  (0x1)
-#define AHCI_PX_CMD_SUD                 (0x2)
-#define AHCI_PX_CMD_POD                 (0x4)
-#define AHCI_PX_CMD_CLO                 (0x8)
-#define AHCI_PX_CMD_FRE                (0x10)
-#define AHCI_PX_CMD_RESERVED           (0xE0)
-#define AHCI_PX_CMD_CCS              (0x1F00)
-#define AHCI_PX_CMD_MPSS             (0x2000)
-#define AHCI_PX_CMD_FR               (0x4000)
-#define AHCI_PX_CMD_CR               (0x8000)
-#define AHCI_PX_CMD_CPS             (0x10000)
-#define AHCI_PX_CMD_PMA             (0x20000)
-#define AHCI_PX_CMD_HPCP            (0x40000)
-#define AHCI_PX_CMD_MPSP            (0x80000)
-#define AHCI_PX_CMD_CPD            (0x100000)
-#define AHCI_PX_CMD_ESP            (0x200000)
-#define AHCI_PX_CMD_FBSCP          (0x400000)
-#define AHCI_PX_CMD_APSTE          (0x800000)
-#define AHCI_PX_CMD_ATAPI         (0x1000000)
-#define AHCI_PX_CMD_DLAE          (0x2000000)
-#define AHCI_PX_CMD_ALPE          (0x4000000)
-#define AHCI_PX_CMD_ASP           (0x8000000)
-#define AHCI_PX_CMD_ICC          (0xF0000000)
-
-#define AHCI_PX_RES1                      (7)
-
-#define AHCI_PX_TFD                       (8)
-#define AHCI_PX_TFD_STS                (0xFF)
-#define AHCI_PX_TFD_STS_ERR            (0x01)
-#define AHCI_PX_TFD_STS_CS1            (0x06)
-#define AHCI_PX_TFD_STS_DRQ            (0x08)
-#define AHCI_PX_TFD_STS_CS2            (0x70)
-#define AHCI_PX_TFD_STS_BSY            (0x80)
-#define AHCI_PX_TFD_ERR              (0xFF00)
-#define AHCI_PX_TFD_RESERVED     (0xFFFF0000)
-
-#define AHCI_PX_SIG                       (9)
-#define AHCI_PX_SIG_SECTOR_COUNT       (0xFF)
-#define AHCI_PX_SIG_LBA_LOW          (0xFF00)
-#define AHCI_PX_SIG_LBA_MID        (0xFF0000)
-#define AHCI_PX_SIG_LBA_HIGH     (0xFF000000)
-
-#define AHCI_PX_SSTS                     (10)
-#define AHCI_PX_SSTS_DET               (0x0F)
-#define AHCI_PX_SSTS_SPD               (0xF0)
-#define AHCI_PX_SSTS_IPM              (0xF00)
-#define AHCI_PX_SSTS_RESERVED    (0xFFFFF000)
-#define SSTS_DET_NO_DEVICE             (0x00)
-#define SSTS_DET_PRESENT               (0x01)
-#define SSTS_DET_ESTABLISHED           (0x03)
-#define SSTS_DET_OFFLINE               (0x04)
-
-#define AHCI_PX_SCTL                     (11)
-
-#define AHCI_PX_SERR                     (12)
-#define AHCI_PX_SERR_ERR             (0xFFFF)
-#define AHCI_PX_SERR_DIAG        (0xFFFF0000)
-#define AHCI_PX_SERR_DIAG_X      (0x04000000)
-
-#define AHCI_PX_SACT                     (13)
-#define AHCI_PX_CI                       (14)
-#define AHCI_PX_SNTF                     (15)
-
-#define AHCI_PX_FBS                      (16)
-#define AHCI_PX_FBS_EN                  (0x1)
-#define AHCI_PX_FBS_DEC                 (0x2)
-#define AHCI_PX_FBS_SDE                 (0x4)
-#define AHCI_PX_FBS_DEV               (0xF00)
-#define AHCI_PX_FBS_ADO              (0xF000)
-#define AHCI_PX_FBS_DWE             (0xF0000)
-#define AHCI_PX_FBS_RESERVED     (0xFFF000F8)
-
-#define AHCI_PX_RES2                     (17)
-#define AHCI_PX_VS                       (28)
-
-#define HBA_DATA_REGION_SIZE            (256)
-#define HBA_PORT_DATA_SIZE              (128)
-#define HBA_PORT_NUM_REG (HBA_PORT_DATA_SIZE/4)
-
-#define AHCI_VERSION_0_95        (0x00000905)
-#define AHCI_VERSION_1_0         (0x00010000)
-#define AHCI_VERSION_1_1         (0x00010100)
-#define AHCI_VERSION_1_2         (0x00010200)
-#define AHCI_VERSION_1_3         (0x00010300)
-
-/*** Structures ***/
-
-/**
- * Generic FIS structure.
- */
-typedef struct FIS {
-    uint8_t fis_type;
-    uint8_t flags;
-    char data[0];
-} __attribute__((__packed__)) FIS;
-
-/**
- * Register device-to-host FIS structure.
- */
-typedef struct RegD2HFIS {
-    /* DW0 */
-    uint8_t fis_type;
-    uint8_t flags;
-    uint8_t status;
-    uint8_t error;
-    /* DW1 */
-    uint8_t lba_low;
-    uint8_t lba_mid;
-    uint8_t lba_high;
-    uint8_t device;
-    /* DW2 */
-    uint8_t lba3;
-    uint8_t lba4;
-    uint8_t lba5;
-    uint8_t res1;
-    /* DW3 */
-    uint16_t count;
-    uint8_t res2;
-    uint8_t res3;
-    /* DW4 */
-    uint16_t res4;
-    uint16_t res5;
-} __attribute__((__packed__)) RegD2HFIS;
-
-/**
- * Register host-to-device FIS structure.
- */
-typedef struct RegH2DFIS {
-    /* DW0 */
-    uint8_t fis_type;
-    uint8_t flags;
-    uint8_t command;
-    uint8_t feature_low;
-    /* DW1 */
-    uint8_t lba_low;
-    uint8_t lba_mid;
-    uint8_t lba_high;
-    uint8_t device;
-    /* DW2 */
-    uint8_t lba3;
-    uint8_t lba4;
-    uint8_t lba5;
-    uint8_t feature_high;
-    /* DW3 */
-    uint16_t count;
-    uint8_t icc;
-    uint8_t control;
-    /* DW4 */
-    uint32_t aux;
-} __attribute__((__packed__)) RegH2DFIS;
-
-/**
- * Command List entry structure.
- * The command list contains between 1-32 of these structures.
- */
-typedef struct AHCICommand {
-    uint8_t b1;
-    uint8_t b2;
-    uint16_t prdtl; /* Phys Region Desc. Table Length */
-    uint32_t prdbc; /* Phys Region Desc. Byte Count */
-    uint32_t ctba;  /* Command Table Descriptor Base Address */
-    uint32_t ctbau; /*                                    '' Upper */
-    uint32_t res[4];
-} __attribute__((__packed__)) AHCICommand;
-
-/**
- * Physical Region Descriptor; pointed to by the Command List Header,
- * struct ahci_command.
- */
-typedef struct PRD {
-    uint32_t dba;  /* Data Base Address */
-    uint32_t dbau; /* Data Base Address Upper */
-    uint32_t res;  /* Reserved */
-    uint32_t dbc;  /* Data Byte Count (0-indexed) & Interrupt Flag (bit 2^31) */
-} PRD;
-
-typedef struct HBACap {
-    uint32_t cap;
-    uint32_t cap2;
-} HBACap;
+/* Test images sizes in MB */
+#define TEST_IMAGE_SIZE_MB_LARGE (200 * 1024)
+#define TEST_IMAGE_SIZE_MB_SMALL 64
 
 /*** Globals ***/
-static QGuestAllocator *guest_malloc;
-static QPCIBus *pcibus;
-static uint64_t barsize;
 static char tmp_path[] = "/tmp/qtest.XXXXXX";
+static char debug_path[] = "/tmp/qtest-blkdebug.XXXXXX";
+static char mig_socket[] = "/tmp/qtest-migration.XXXXXX";
 static bool ahci_pedantic;
-static uint32_t ahci_fingerprint;
-
-/*** Macro Utilities ***/
-#define BITANY(data, mask) (((data) & (mask)) != 0)
-#define BITSET(data, mask) (((data) & (mask)) == (mask))
-#define BITCLR(data, mask) (((data) & (mask)) == 0)
-#define ASSERT_BIT_SET(data, mask) g_assert_cmphex((data) & (mask), ==, (mask))
-#define ASSERT_BIT_CLEAR(data, mask) g_assert_cmphex((data) & (mask), ==, 0)
-
-/*** IO macros for the AHCI memory registers. ***/
-#define AHCI_READ(OFST) qpci_io_readl(ahci, hba_base + (OFST))
-#define AHCI_WRITE(OFST, VAL) qpci_io_writel(ahci, hba_base + (OFST), (VAL))
-#define AHCI_RREG(regno)      AHCI_READ(4 * (regno))
-#define AHCI_WREG(regno, val) AHCI_WRITE(4 * (regno), (val))
-#define AHCI_SET(regno, mask) AHCI_WREG((regno), AHCI_RREG(regno) | (mask))
-#define AHCI_CLR(regno, mask) AHCI_WREG((regno), AHCI_RREG(regno) & ~(mask))
-
-/*** IO macros for port-specific offsets inside of AHCI memory. ***/
-#define PX_OFST(port, regno) (HBA_PORT_NUM_REG * (port) + AHCI_PORTS + (regno))
-#define PX_RREG(port, regno)      AHCI_RREG(PX_OFST((port), (regno)))
-#define PX_WREG(port, regno, val) AHCI_WREG(PX_OFST((port), (regno)), (val))
-#define PX_SET(port, reg, mask)   PX_WREG((port), (reg),                \
-                                          PX_RREG((port), (reg)) | (mask));
-#define PX_CLR(port, reg, mask)   PX_WREG((port), (reg),                \
-                                          PX_RREG((port), (reg)) & ~(mask));
-
-/* For calculating how big the PRD table needs to be: */
-#define CMD_TBL_SIZ(n) ((0x80 + ((n) * sizeof(PRD)) + 0x7F) & ~0x7F)
-
+static const char *imgfmt;
+static unsigned test_image_size_mb;
 
 /*** Function Declarations ***/
-static QPCIDevice *get_ahci_device(void);
-static QPCIDevice *start_ahci_device(QPCIDevice *dev, void **hba_base);
-static void free_ahci_device(QPCIDevice *dev);
-static void ahci_test_port_spec(QPCIDevice *ahci, void *hba_base,
-                                HBACap *hcap, uint8_t port);
-static void ahci_test_pci_spec(QPCIDevice *ahci);
-static void ahci_test_pci_caps(QPCIDevice *ahci, uint16_t header,
+static void ahci_test_port_spec(AHCIQState *ahci, uint8_t port);
+static void ahci_test_pci_spec(AHCIQState *ahci);
+static void ahci_test_pci_caps(AHCIQState *ahci, uint16_t header,
                                uint8_t offset);
-static void ahci_test_satacap(QPCIDevice *ahci, uint8_t offset);
-static void ahci_test_msicap(QPCIDevice *ahci, uint8_t offset);
-static void ahci_test_pmcap(QPCIDevice *ahci, uint8_t offset);
+static void ahci_test_satacap(AHCIQState *ahci, uint8_t offset);
+static void ahci_test_msicap(AHCIQState *ahci, uint8_t offset);
+static void ahci_test_pmcap(AHCIQState *ahci, uint8_t offset);
 
 /*** Utilities ***/
 
+static uint64_t mb_to_sectors(uint64_t image_size_mb)
+{
+    return (image_size_mb * 1024 * 1024) / AHCI_SECTOR_SIZE;
+}
+
 static void string_bswap16(uint16_t *s, size_t bytes)
 {
     g_assert_cmphex((bytes & 1), ==, 0);
@@ -411,287 +76,180 @@
 }
 
 /**
- * Locate, verify, and return a handle to the AHCI device.
+ * Verify that the transfer did not corrupt our state at all.
  */
-static QPCIDevice *get_ahci_device(void)
+static void verify_state(AHCIQState *ahci)
 {
-    QPCIDevice *ahci;
+    int i, j;
+    uint32_t ahci_fingerprint;
+    uint64_t hba_base;
+    uint64_t hba_stored;
+    AHCICommandHeader cmd;
 
-    pcibus = qpci_init_pc();
+    ahci_fingerprint = qpci_config_readl(ahci->dev, PCI_VENDOR_ID);
+    g_assert_cmphex(ahci_fingerprint, ==, ahci->fingerprint);
 
-    /* Find the AHCI PCI device and verify it's the right one. */
-    ahci = qpci_device_find(pcibus, QPCI_DEVFN(0x1F, 0x02));
-    g_assert(ahci != NULL);
-
-    ahci_fingerprint = qpci_config_readl(ahci, PCI_VENDOR_ID);
-
-    switch (ahci_fingerprint) {
-    case AHCI_INTEL_ICH9:
-        break;
-    default:
-        /* Unknown device. */
-        g_assert_not_reached();
+    /* If we haven't initialized, this is as much as can be validated. */
+    if (!ahci->hba_base) {
+        return;
     }
 
-    return ahci;
+    hba_base = (uint64_t)qpci_config_readl(ahci->dev, PCI_BASE_ADDRESS_5);
+    hba_stored = (uint64_t)(uintptr_t)ahci->hba_base;
+    g_assert_cmphex(hba_base, ==, hba_stored);
+
+    g_assert_cmphex(ahci_rreg(ahci, AHCI_CAP), ==, ahci->cap);
+    g_assert_cmphex(ahci_rreg(ahci, AHCI_CAP2), ==, ahci->cap2);
+
+    for (i = 0; i < 32; i++) {
+        g_assert_cmphex(ahci_px_rreg(ahci, i, AHCI_PX_FB), ==,
+                        ahci->port[i].fb);
+        g_assert_cmphex(ahci_px_rreg(ahci, i, AHCI_PX_CLB), ==,
+                        ahci->port[i].clb);
+        for (j = 0; j < 32; j++) {
+            ahci_get_command_header(ahci, i, j, &cmd);
+            g_assert_cmphex(cmd.prdtl, ==, ahci->port[i].prdtl[j]);
+            g_assert_cmphex(cmd.ctba, ==, ahci->port[i].ctba[j]);
+        }
+    }
 }
 
-static void free_ahci_device(QPCIDevice *ahci)
+static void ahci_migrate(AHCIQState *from, AHCIQState *to, const char *uri)
 {
-    /* libqos doesn't have a function for this, so free it manually */
-    g_free(ahci);
+    QOSState *tmp = to->parent;
+    QPCIDevice *dev = to->dev;
+    char *uri_local = NULL;
 
-    if (pcibus) {
-        qpci_free_pc(pcibus);
-        pcibus = NULL;
+    if (uri == NULL) {
+        uri_local = g_strdup_printf("%s%s", "unix:", mig_socket);
+        uri = uri_local;
     }
 
-    /* Clear our cached barsize information. */
-    barsize = 0;
+    /* context will be 'to' after completion. */
+    migrate(from->parent, to->parent, uri);
+
+    /* We'd like for the AHCIState objects to still point
+     * to information specific to its specific parent
+     * instance, but otherwise just inherit the new data. */
+    memcpy(to, from, sizeof(AHCIQState));
+    to->parent = tmp;
+    to->dev = dev;
+
+    tmp = from->parent;
+    dev = from->dev;
+    memset(from, 0x00, sizeof(AHCIQState));
+    from->parent = tmp;
+    from->dev = dev;
+
+    verify_state(to);
+    g_free(uri_local);
 }
 
 /*** Test Setup & Teardown ***/
 
 /**
- * Launch QEMU with the given command line,
- * and then set up interrupts and our guest malloc interface.
+ * Start a Q35 machine and bookmark a handle to the AHCI device.
  */
-static void qtest_boot(const char *cmdline_fmt, ...)
+static AHCIQState *ahci_vboot(const char *cli, va_list ap)
 {
-    va_list ap;
-    char *cmdline;
+    AHCIQState *s;
 
-    va_start(ap, cmdline_fmt);
-    cmdline = g_strdup_vprintf(cmdline_fmt, ap);
-    va_end(ap);
+    s = g_malloc0(sizeof(AHCIQState));
+    s->parent = qtest_pc_vboot(cli, ap);
+    alloc_set_flags(s->parent->alloc, ALLOC_LEAK_ASSERT);
 
-    qtest_start(cmdline);
-    qtest_irq_intercept_in(global_qtest, "ioapic");
-    guest_malloc = pc_alloc_init();
+    /* Verify that we have an AHCI device present. */
+    s->dev = get_ahci_device(&s->fingerprint);
 
-    g_free(cmdline);
-}
-
-/**
- * Tear down the QEMU instance.
- */
-static void qtest_shutdown(void)
-{
-    g_free(guest_malloc);
-    guest_malloc = NULL;
-    qtest_end();
+    return s;
 }
 
 /**
  * Start a Q35 machine and bookmark a handle to the AHCI device.
  */
-static QPCIDevice *ahci_boot(void)
+static AHCIQState *ahci_boot(const char *cli, ...)
 {
-    qtest_boot("-drive if=none,id=drive0,file=%s,cache=writeback,serial=%s"
-               " -M q35 "
-               "-device ide-hd,drive=drive0 "
-               "-global ide-hd.ver=%s",
-               tmp_path, "testdisk", "version");
+    AHCIQState *s;
+    va_list ap;
 
-    /* Verify that we have an AHCI device present. */
-    return get_ahci_device();
+    if (cli) {
+        va_start(ap, cli);
+        s = ahci_vboot(cli, ap);
+        va_end(ap);
+    } else {
+        cli = "-drive if=none,id=drive0,file=%s,cache=writeback,serial=%s"
+            ",format=%s"
+            " -M q35 "
+            "-device ide-hd,drive=drive0 "
+            "-global ide-hd.ver=%s";
+        s = ahci_boot(cli, tmp_path, "testdisk", imgfmt, "version");
+    }
+
+    return s;
 }
 
 /**
  * Clean up the PCI device, then terminate the QEMU instance.
  */
-static void ahci_shutdown(QPCIDevice *ahci)
+static void ahci_shutdown(AHCIQState *ahci)
 {
-    free_ahci_device(ahci);
-    qtest_shutdown();
+    QOSState *qs = ahci->parent;
+
+    set_context(qs);
+    ahci_clean_mem(ahci);
+    free_ahci_device(ahci->dev);
+    g_free(ahci);
+    qtest_shutdown(qs);
 }
 
-/*** Logical Device Initialization ***/
-
 /**
- * Start the PCI device and sanity-check default operation.
+ * Boot and fully enable the HBA device.
+ * @see ahci_boot, ahci_pci_enable and ahci_hba_enable.
  */
-static void ahci_pci_enable(QPCIDevice *ahci, void **hba_base)
+static AHCIQState *ahci_boot_and_enable(const char *cli, ...)
 {
-    uint8_t reg;
+    AHCIQState *ahci;
+    va_list ap;
+    uint16_t buff[256];
+    uint8_t port;
+    uint8_t hello;
 
-    start_ahci_device(ahci, hba_base);
-
-    switch (ahci_fingerprint) {
-    case AHCI_INTEL_ICH9:
-        /* ICH9 has a register at PCI 0x92 that
-         * acts as a master port enabler mask. */
-        reg = qpci_config_readb(ahci, 0x92);
-        reg |= 0x3F;
-        qpci_config_writeb(ahci, 0x92, reg);
-        /* 0...0111111b -- bit significant, ports 0-5 enabled. */
-        ASSERT_BIT_SET(qpci_config_readb(ahci, 0x92), 0x3F);
-        break;
+    if (cli) {
+        va_start(ap, cli);
+        ahci = ahci_vboot(cli, ap);
+        va_end(ap);
+    } else {
+        ahci = ahci_boot(NULL);
     }
 
-}
-
-/**
- * Map BAR5/ABAR, and engage the PCI device.
- */
-static QPCIDevice *start_ahci_device(QPCIDevice *ahci, void **hba_base)
-{
-    /* Map AHCI's ABAR (BAR5) */
-    *hba_base = qpci_iomap(ahci, 5, &barsize);
-
-    /* turns on pci.cmd.iose, pci.cmd.mse and pci.cmd.bme */
-    qpci_device_enable(ahci);
+    ahci_pci_enable(ahci);
+    ahci_hba_enable(ahci);
+    /* Initialize test device */
+    port = ahci_port_select(ahci);
+    ahci_port_clear(ahci, port);
+    if (is_atapi(ahci, port)) {
+        hello = CMD_PACKET_ID;
+    } else {
+        hello = CMD_IDENTIFY;
+    }
+    ahci_io(ahci, port, hello, &buff, sizeof(buff), 0);
 
     return ahci;
 }
 
-/**
- * Test and initialize the AHCI's HBA memory areas.
- * Initialize and start any ports with devices attached.
- * Bring the HBA into the idle state.
- */
-static void ahci_hba_enable(QPCIDevice *ahci, void *hba_base)
-{
-    /* Bits of interest in this section:
-     * GHC.AE     Global Host Control / AHCI Enable
-     * PxCMD.ST   Port Command: Start
-     * PxCMD.SUD  "Spin Up Device"
-     * PxCMD.POD  "Power On Device"
-     * PxCMD.FRE  "FIS Receive Enable"
-     * PxCMD.FR   "FIS Receive Running"
-     * PxCMD.CR   "Command List Running"
-     */
-
-    g_assert(ahci != NULL);
-    g_assert(hba_base != NULL);
-
-    uint32_t reg, ports_impl, clb, fb;
-    uint16_t i;
-    uint8_t num_cmd_slots;
-
-    g_assert(hba_base != 0);
-
-    /* Set GHC.AE to 1 */
-    AHCI_SET(AHCI_GHC, AHCI_GHC_AE);
-    reg = AHCI_RREG(AHCI_GHC);
-    ASSERT_BIT_SET(reg, AHCI_GHC_AE);
-
-    /* Read CAP.NCS, how many command slots do we have? */
-    reg = AHCI_RREG(AHCI_CAP);
-    num_cmd_slots = ((reg & AHCI_CAP_NCS) >> ctzl(AHCI_CAP_NCS)) + 1;
-    g_test_message("Number of Command Slots: %u", num_cmd_slots);
-
-    /* Determine which ports are implemented. */
-    ports_impl = AHCI_RREG(AHCI_PI);
-
-    for (i = 0; ports_impl; ports_impl >>= 1, ++i) {
-        if (!(ports_impl & 0x01)) {
-            continue;
-        }
-
-        g_test_message("Initializing port %u", i);
-
-        reg = PX_RREG(i, AHCI_PX_CMD);
-        if (BITCLR(reg, AHCI_PX_CMD_ST | AHCI_PX_CMD_CR |
-                   AHCI_PX_CMD_FRE | AHCI_PX_CMD_FR)) {
-            g_test_message("port is idle");
-        } else {
-            g_test_message("port needs to be idled");
-            PX_CLR(i, AHCI_PX_CMD, (AHCI_PX_CMD_ST | AHCI_PX_CMD_FRE));
-            /* The port has 500ms to disengage. */
-            usleep(500000);
-            reg = PX_RREG(i, AHCI_PX_CMD);
-            ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CR);
-            ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FR);
-            g_test_message("port is now idle");
-            /* The spec does allow for possibly needing a PORT RESET
-             * or HBA reset if we fail to idle the port. */
-        }
-
-        /* Allocate Memory for the Command List Buffer & FIS Buffer */
-        /* PxCLB space ... 0x20 per command, as in 4.2.2 p 36 */
-        clb = guest_alloc(guest_malloc, num_cmd_slots * 0x20);
-        g_test_message("CLB: 0x%08x", clb);
-        PX_WREG(i, AHCI_PX_CLB, clb);
-        g_assert_cmphex(clb, ==, PX_RREG(i, AHCI_PX_CLB));
-
-        /* PxFB space ... 0x100, as in 4.2.1 p 35 */
-        fb = guest_alloc(guest_malloc, 0x100);
-        g_test_message("FB: 0x%08x", fb);
-        PX_WREG(i, AHCI_PX_FB, fb);
-        g_assert_cmphex(fb, ==, PX_RREG(i, AHCI_PX_FB));
-
-        /* Clear PxSERR, PxIS, then IS.IPS[x] by writing '1's. */
-        PX_WREG(i, AHCI_PX_SERR, 0xFFFFFFFF);
-        PX_WREG(i, AHCI_PX_IS, 0xFFFFFFFF);
-        AHCI_WREG(AHCI_IS, (1 << i));
-
-        /* Verify Interrupts Cleared */
-        reg = PX_RREG(i, AHCI_PX_SERR);
-        g_assert_cmphex(reg, ==, 0);
-
-        reg = PX_RREG(i, AHCI_PX_IS);
-        g_assert_cmphex(reg, ==, 0);
-
-        reg = AHCI_RREG(AHCI_IS);
-        ASSERT_BIT_CLEAR(reg, (1 << i));
-
-        /* Enable All Interrupts: */
-        PX_WREG(i, AHCI_PX_IE, 0xFFFFFFFF);
-        reg = PX_RREG(i, AHCI_PX_IE);
-        g_assert_cmphex(reg, ==, ~((uint32_t)AHCI_PX_IE_RESERVED));
-
-        /* Enable the FIS Receive Engine. */
-        PX_SET(i, AHCI_PX_CMD, AHCI_PX_CMD_FRE);
-        reg = PX_RREG(i, AHCI_PX_CMD);
-        ASSERT_BIT_SET(reg, AHCI_PX_CMD_FR);
-
-        /* AHCI 1.3 spec: if !STS.BSY, !STS.DRQ and PxSSTS.DET indicates
-         * physical presence, a device is present and may be started. However,
-         * PxSERR.DIAG.X /may/ need to be cleared a priori. */
-        reg = PX_RREG(i, AHCI_PX_SERR);
-        if (BITSET(reg, AHCI_PX_SERR_DIAG_X)) {
-            PX_SET(i, AHCI_PX_SERR, AHCI_PX_SERR_DIAG_X);
-        }
-
-        reg = PX_RREG(i, AHCI_PX_TFD);
-        if (BITCLR(reg, AHCI_PX_TFD_STS_BSY | AHCI_PX_TFD_STS_DRQ)) {
-            reg = PX_RREG(i, AHCI_PX_SSTS);
-            if ((reg & AHCI_PX_SSTS_DET) == SSTS_DET_ESTABLISHED) {
-                /* Device Found: set PxCMD.ST := 1 */
-                PX_SET(i, AHCI_PX_CMD, AHCI_PX_CMD_ST);
-                ASSERT_BIT_SET(PX_RREG(i, AHCI_PX_CMD), AHCI_PX_CMD_CR);
-                g_test_message("Started Device %u", i);
-            } else if ((reg & AHCI_PX_SSTS_DET)) {
-                /* Device present, but in some unknown state. */
-                g_assert_not_reached();
-            }
-        }
-    }
-
-    /* Enable GHC.IE */
-    AHCI_SET(AHCI_GHC, AHCI_GHC_IE);
-    reg = AHCI_RREG(AHCI_GHC);
-    ASSERT_BIT_SET(reg, AHCI_GHC_IE);
-
-    /* TODO: The device should now be idling and waiting for commands.
-     * In the future, a small test-case to inspect the Register D2H FIS
-     * and clear the initial interrupts might be good. */
-}
-
 /*** Specification Adherence Tests ***/
 
 /**
  * Implementation for test_pci_spec. Ensures PCI configuration space is sane.
  */
-static void ahci_test_pci_spec(QPCIDevice *ahci)
+static void ahci_test_pci_spec(AHCIQState *ahci)
 {
     uint8_t datab;
     uint16_t data;
     uint32_t datal;
 
     /* Most of these bits should start cleared until we turn them on. */
-    data = qpci_config_readw(ahci, PCI_COMMAND);
+    data = qpci_config_readw(ahci->dev, PCI_COMMAND);
     ASSERT_BIT_CLEAR(data, PCI_COMMAND_MEMORY);
     ASSERT_BIT_CLEAR(data, PCI_COMMAND_MASTER);
     ASSERT_BIT_CLEAR(data, PCI_COMMAND_SPECIAL);     /* Reserved */
@@ -703,7 +261,7 @@
     ASSERT_BIT_CLEAR(data, PCI_COMMAND_INTX_DISABLE);
     ASSERT_BIT_CLEAR(data, 0xF800);                  /* Reserved */
 
-    data = qpci_config_readw(ahci, PCI_STATUS);
+    data = qpci_config_readw(ahci->dev, PCI_STATUS);
     ASSERT_BIT_CLEAR(data, 0x01 | 0x02 | 0x04);     /* Reserved */
     ASSERT_BIT_CLEAR(data, PCI_STATUS_INTERRUPT);
     ASSERT_BIT_SET(data, PCI_STATUS_CAP_LIST);      /* must be set */
@@ -716,7 +274,7 @@
     ASSERT_BIT_CLEAR(data, PCI_STATUS_DETECTED_PARITY);
 
     /* RID occupies the low byte, CCs occupy the high three. */
-    datal = qpci_config_readl(ahci, PCI_CLASS_REVISION);
+    datal = qpci_config_readl(ahci->dev, PCI_CLASS_REVISION);
     if (ahci_pedantic) {
         /* AHCI 1.3 specifies that at-boot, the RID should reset to 0x00,
          * Though in practice this is likely seldom true. */
@@ -739,40 +297,40 @@
         g_assert_not_reached();
     }
 
-    datab = qpci_config_readb(ahci, PCI_CACHE_LINE_SIZE);
+    datab = qpci_config_readb(ahci->dev, PCI_CACHE_LINE_SIZE);
     g_assert_cmphex(datab, ==, 0);
 
-    datab = qpci_config_readb(ahci, PCI_LATENCY_TIMER);
+    datab = qpci_config_readb(ahci->dev, PCI_LATENCY_TIMER);
     g_assert_cmphex(datab, ==, 0);
 
     /* Only the bottom 7 bits must be off. */
-    datab = qpci_config_readb(ahci, PCI_HEADER_TYPE);
+    datab = qpci_config_readb(ahci->dev, PCI_HEADER_TYPE);
     ASSERT_BIT_CLEAR(datab, 0x7F);
 
     /* BIST is optional, but the low 7 bits must always start off regardless. */
-    datab = qpci_config_readb(ahci, PCI_BIST);
+    datab = qpci_config_readb(ahci->dev, PCI_BIST);
     ASSERT_BIT_CLEAR(datab, 0x7F);
 
     /* BARS 0-4 do not have a boot spec, but ABAR/BAR5 must be clean. */
-    datal = qpci_config_readl(ahci, PCI_BASE_ADDRESS_5);
+    datal = qpci_config_readl(ahci->dev, PCI_BASE_ADDRESS_5);
     g_assert_cmphex(datal, ==, 0);
 
-    qpci_config_writel(ahci, PCI_BASE_ADDRESS_5, 0xFFFFFFFF);
-    datal = qpci_config_readl(ahci, PCI_BASE_ADDRESS_5);
+    qpci_config_writel(ahci->dev, PCI_BASE_ADDRESS_5, 0xFFFFFFFF);
+    datal = qpci_config_readl(ahci->dev, PCI_BASE_ADDRESS_5);
     /* ABAR must be 32-bit, memory mapped, non-prefetchable and
      * must be >= 512 bytes. To that end, bits 0-8 must be off. */
     ASSERT_BIT_CLEAR(datal, 0xFF);
 
     /* Capability list MUST be present, */
-    datal = qpci_config_readl(ahci, PCI_CAPABILITY_LIST);
+    datal = qpci_config_readl(ahci->dev, PCI_CAPABILITY_LIST);
     /* But these bits are reserved. */
     ASSERT_BIT_CLEAR(datal, ~0xFF);
     g_assert_cmphex(datal, !=, 0);
 
     /* Check specification adherence for capability extenstions. */
-    data = qpci_config_readw(ahci, datal);
+    data = qpci_config_readw(ahci->dev, datal);
 
-    switch (ahci_fingerprint) {
+    switch (ahci->fingerprint) {
     case AHCI_INTEL_ICH9:
         /* Intel ICH9 Family Datasheet 14.1.19 p.550 */
         g_assert_cmphex((data & 0xFF), ==, PCI_CAP_ID_MSI);
@@ -785,18 +343,18 @@
     ahci_test_pci_caps(ahci, data, (uint8_t)datal);
 
     /* Reserved. */
-    datal = qpci_config_readl(ahci, PCI_CAPABILITY_LIST + 4);
+    datal = qpci_config_readl(ahci->dev, PCI_CAPABILITY_LIST + 4);
     g_assert_cmphex(datal, ==, 0);
 
     /* IPIN might vary, but ILINE must be off. */
-    datab = qpci_config_readb(ahci, PCI_INTERRUPT_LINE);
+    datab = qpci_config_readb(ahci->dev, PCI_INTERRUPT_LINE);
     g_assert_cmphex(datab, ==, 0);
 }
 
 /**
  * Test PCI capabilities for AHCI specification adherence.
  */
-static void ahci_test_pci_caps(QPCIDevice *ahci, uint16_t header,
+static void ahci_test_pci_caps(AHCIQState *ahci, uint16_t header,
                                uint8_t offset)
 {
     uint8_t cid = header & 0xFF;
@@ -820,14 +378,14 @@
     }
 
     if (next) {
-        ahci_test_pci_caps(ahci, qpci_config_readw(ahci, next), next);
+        ahci_test_pci_caps(ahci, qpci_config_readw(ahci->dev, next), next);
     }
 }
 
 /**
  * Test SATA PCI capabilitity for AHCI specification adherence.
  */
-static void ahci_test_satacap(QPCIDevice *ahci, uint8_t offset)
+static void ahci_test_satacap(AHCIQState *ahci, uint8_t offset)
 {
     uint16_t dataw;
     uint32_t datal;
@@ -835,11 +393,11 @@
     g_test_message("Verifying SATACAP");
 
     /* Assert that the SATACAP version is 1.0, And reserved bits are empty. */
-    dataw = qpci_config_readw(ahci, offset + 2);
+    dataw = qpci_config_readw(ahci->dev, offset + 2);
     g_assert_cmphex(dataw, ==, 0x10);
 
     /* Grab the SATACR1 register. */
-    datal = qpci_config_readw(ahci, offset + 4);
+    datal = qpci_config_readw(ahci->dev, offset + 4);
 
     switch (datal & 0x0F) {
     case 0x04: /* BAR0 */
@@ -862,30 +420,30 @@
 /**
  * Test MSI PCI capability for AHCI specification adherence.
  */
-static void ahci_test_msicap(QPCIDevice *ahci, uint8_t offset)
+static void ahci_test_msicap(AHCIQState *ahci, uint8_t offset)
 {
     uint16_t dataw;
     uint32_t datal;
 
     g_test_message("Verifying MSICAP");
 
-    dataw = qpci_config_readw(ahci, offset + PCI_MSI_FLAGS);
+    dataw = qpci_config_readw(ahci->dev, offset + PCI_MSI_FLAGS);
     ASSERT_BIT_CLEAR(dataw, PCI_MSI_FLAGS_ENABLE);
     ASSERT_BIT_CLEAR(dataw, PCI_MSI_FLAGS_QSIZE);
     ASSERT_BIT_CLEAR(dataw, PCI_MSI_FLAGS_RESERVED);
 
-    datal = qpci_config_readl(ahci, offset + PCI_MSI_ADDRESS_LO);
+    datal = qpci_config_readl(ahci->dev, offset + PCI_MSI_ADDRESS_LO);
     g_assert_cmphex(datal, ==, 0);
 
     if (dataw & PCI_MSI_FLAGS_64BIT) {
         g_test_message("MSICAP is 64bit");
-        datal = qpci_config_readl(ahci, offset + PCI_MSI_ADDRESS_HI);
+        datal = qpci_config_readl(ahci->dev, offset + PCI_MSI_ADDRESS_HI);
         g_assert_cmphex(datal, ==, 0);
-        dataw = qpci_config_readw(ahci, offset + PCI_MSI_DATA_64);
+        dataw = qpci_config_readw(ahci->dev, offset + PCI_MSI_DATA_64);
         g_assert_cmphex(dataw, ==, 0);
     } else {
         g_test_message("MSICAP is 32bit");
-        dataw = qpci_config_readw(ahci, offset + PCI_MSI_DATA_32);
+        dataw = qpci_config_readw(ahci->dev, offset + PCI_MSI_DATA_32);
         g_assert_cmphex(dataw, ==, 0);
     }
 }
@@ -893,36 +451,34 @@
 /**
  * Test Power Management PCI capability for AHCI specification adherence.
  */
-static void ahci_test_pmcap(QPCIDevice *ahci, uint8_t offset)
+static void ahci_test_pmcap(AHCIQState *ahci, uint8_t offset)
 {
     uint16_t dataw;
 
     g_test_message("Verifying PMCAP");
 
-    dataw = qpci_config_readw(ahci, offset + PCI_PM_PMC);
+    dataw = qpci_config_readw(ahci->dev, offset + PCI_PM_PMC);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_PME_CLOCK);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_RESERVED);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_D1);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CAP_D2);
 
-    dataw = qpci_config_readw(ahci, offset + PCI_PM_CTRL);
+    dataw = qpci_config_readw(ahci->dev, offset + PCI_PM_CTRL);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_STATE_MASK);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_RESERVED);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_DATA_SEL_MASK);
     ASSERT_BIT_CLEAR(dataw, PCI_PM_CTRL_DATA_SCALE_MASK);
 }
 
-static void ahci_test_hba_spec(QPCIDevice *ahci, void *hba_base)
+static void ahci_test_hba_spec(AHCIQState *ahci)
 {
-    HBACap hcap;
     unsigned i;
-    uint32_t cap, cap2, reg;
+    uint32_t reg;
     uint32_t ports;
     uint8_t nports_impl;
     uint8_t maxports;
 
-    g_assert(ahci != 0);
-    g_assert(hba_base != 0);
+    g_assert(ahci != NULL);
 
     /*
      * Note that the AHCI spec does expect the BIOS to set up a few things:
@@ -945,15 +501,15 @@
      */
 
     /* 1 CAP - Capabilities Register */
-    cap = AHCI_RREG(AHCI_CAP);
-    ASSERT_BIT_CLEAR(cap, AHCI_CAP_RESERVED);
+    ahci->cap = ahci_rreg(ahci, AHCI_CAP);
+    ASSERT_BIT_CLEAR(ahci->cap, AHCI_CAP_RESERVED);
 
     /* 2 GHC - Global Host Control */
-    reg = AHCI_RREG(AHCI_GHC);
+    reg = ahci_rreg(ahci, AHCI_GHC);
     ASSERT_BIT_CLEAR(reg, AHCI_GHC_HR);
     ASSERT_BIT_CLEAR(reg, AHCI_GHC_IE);
     ASSERT_BIT_CLEAR(reg, AHCI_GHC_MRSM);
-    if (BITSET(cap, AHCI_CAP_SAM)) {
+    if (BITSET(ahci->cap, AHCI_CAP_SAM)) {
         g_test_message("Supports AHCI-Only Mode: GHC_AE is Read-Only.");
         ASSERT_BIT_SET(reg, AHCI_GHC_AE);
     } else {
@@ -962,27 +518,27 @@
     }
 
     /* 3 IS - Interrupt Status */
-    reg = AHCI_RREG(AHCI_IS);
+    reg = ahci_rreg(ahci, AHCI_IS);
     g_assert_cmphex(reg, ==, 0);
 
     /* 4 PI - Ports Implemented */
-    ports = AHCI_RREG(AHCI_PI);
+    ports = ahci_rreg(ahci, AHCI_PI);
     /* Ports Implemented must be non-zero. */
     g_assert_cmphex(ports, !=, 0);
     /* Ports Implemented must be <= Number of Ports. */
     nports_impl = ctpopl(ports);
-    g_assert_cmpuint(((AHCI_CAP_NP & cap) + 1), >=, nports_impl);
+    g_assert_cmpuint(((AHCI_CAP_NP & ahci->cap) + 1), >=, nports_impl);
 
-    g_assert_cmphex(barsize, >, 0);
     /* Ports must be within the proper range. Given a mapping of SIZE,
      * 256 bytes are used for global HBA control, and the rest is used
      * for ports data, at 0x80 bytes each. */
-    maxports = (barsize - HBA_DATA_REGION_SIZE) / HBA_PORT_DATA_SIZE;
+    g_assert_cmphex(ahci->barsize, >, 0);
+    maxports = (ahci->barsize - HBA_DATA_REGION_SIZE) / HBA_PORT_DATA_SIZE;
     /* e.g, 30 ports for 4K of memory. (4096 - 256) / 128 = 30 */
     g_assert_cmphex((reg >> maxports), ==, 0);
 
     /* 5 AHCI Version */
-    reg = AHCI_RREG(AHCI_VS);
+    reg = ahci_rreg(ahci, AHCI_VS);
     switch (reg) {
     case AHCI_VERSION_0_95:
     case AHCI_VERSION_1_0:
@@ -995,8 +551,8 @@
     }
 
     /* 6 Command Completion Coalescing Control: depends on CAP.CCCS. */
-    reg = AHCI_RREG(AHCI_CCCCTL);
-    if (BITSET(cap, AHCI_CAP_CCCS)) {
+    reg = ahci_rreg(ahci, AHCI_CCCCTL);
+    if (BITSET(ahci->cap, AHCI_CAP_CCCS)) {
         ASSERT_BIT_CLEAR(reg, AHCI_CCCCTL_EN);
         ASSERT_BIT_CLEAR(reg, AHCI_CCCCTL_RESERVED);
         ASSERT_BIT_SET(reg, AHCI_CCCCTL_CC);
@@ -1006,19 +562,19 @@
     }
 
     /* 7 CCC_PORTS */
-    reg = AHCI_RREG(AHCI_CCCPORTS);
+    reg = ahci_rreg(ahci, AHCI_CCCPORTS);
     /* Must be zeroes initially regardless of CAP.CCCS */
     g_assert_cmphex(reg, ==, 0);
 
     /* 8 EM_LOC */
-    reg = AHCI_RREG(AHCI_EMLOC);
-    if (BITCLR(cap, AHCI_CAP_EMS)) {
+    reg = ahci_rreg(ahci, AHCI_EMLOC);
+    if (BITCLR(ahci->cap, AHCI_CAP_EMS)) {
         g_assert_cmphex(reg, ==, 0);
     }
 
     /* 9 EM_CTL */
-    reg = AHCI_RREG(AHCI_EMCTL);
-    if (BITSET(cap, AHCI_CAP_EMS)) {
+    reg = ahci_rreg(ahci, AHCI_EMCTL);
+    if (BITSET(ahci->cap, AHCI_CAP_EMS)) {
         ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_STSMR);
         ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_CTLTM);
         ASSERT_BIT_CLEAR(reg, AHCI_EMCTL_CTLRST);
@@ -1028,25 +584,25 @@
     }
 
     /* 10 CAP2 -- Capabilities Extended */
-    cap2 = AHCI_RREG(AHCI_CAP2);
-    ASSERT_BIT_CLEAR(cap2, AHCI_CAP2_RESERVED);
+    ahci->cap2 = ahci_rreg(ahci, AHCI_CAP2);
+    ASSERT_BIT_CLEAR(ahci->cap2, AHCI_CAP2_RESERVED);
 
     /* 11 BOHC -- Bios/OS Handoff Control */
-    reg = AHCI_RREG(AHCI_BOHC);
+    reg = ahci_rreg(ahci, AHCI_BOHC);
     g_assert_cmphex(reg, ==, 0);
 
     /* 12 -- 23: Reserved */
     g_test_message("Verifying HBA reserved area is empty.");
     for (i = AHCI_RESERVED; i < AHCI_NVMHCI; ++i) {
-        reg = AHCI_RREG(i);
+        reg = ahci_rreg(ahci, i);
         g_assert_cmphex(reg, ==, 0);
     }
 
     /* 24 -- 39: NVMHCI */
-    if (BITCLR(cap2, AHCI_CAP2_NVMP)) {
+    if (BITCLR(ahci->cap2, AHCI_CAP2_NVMP)) {
         g_test_message("Verifying HBA/NVMHCI area is empty.");
         for (i = AHCI_NVMHCI; i < AHCI_VENDOR; ++i) {
-            reg = AHCI_RREG(i);
+            reg = ahci_rreg(ahci, i);
             g_assert_cmphex(reg, ==, 0);
         }
     }
@@ -1054,17 +610,15 @@
     /* 40 -- 63: Vendor */
     g_test_message("Verifying HBA/Vendor area is empty.");
     for (i = AHCI_VENDOR; i < AHCI_PORTS; ++i) {
-        reg = AHCI_RREG(i);
+        reg = ahci_rreg(ahci, i);
         g_assert_cmphex(reg, ==, 0);
     }
 
     /* 64 -- XX: Port Space */
-    hcap.cap = cap;
-    hcap.cap2 = cap2;
     for (i = 0; ports || (i < maxports); ports >>= 1, ++i) {
         if (BITSET(ports, 0x1)) {
             g_test_message("Testing port %u for spec", i);
-            ahci_test_port_spec(ahci, hba_base, &hcap, i);
+            ahci_test_port_spec(ahci, i);
         } else {
             uint16_t j;
             uint16_t low = AHCI_PORTS + (32 * i);
@@ -1073,7 +627,7 @@
                            "(reg [%u-%u]) is empty.",
                            i, low, high - 1);
             for (j = low; j < high; ++j) {
-                reg = AHCI_RREG(j);
+                reg = ahci_rreg(ahci, j);
                 g_assert_cmphex(reg, ==, 0);
             }
         }
@@ -1083,42 +637,41 @@
 /**
  * Test the memory space for one port for specification adherence.
  */
-static void ahci_test_port_spec(QPCIDevice *ahci, void *hba_base,
-                                HBACap *hcap, uint8_t port)
+static void ahci_test_port_spec(AHCIQState *ahci, uint8_t port)
 {
     uint32_t reg;
     unsigned i;
 
     /* (0) CLB */
-    reg = PX_RREG(port, AHCI_PX_CLB);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_CLB);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_CLB_RESERVED);
 
     /* (1) CLBU */
-    if (BITCLR(hcap->cap, AHCI_CAP_S64A)) {
-        reg = PX_RREG(port, AHCI_PX_CLBU);
+    if (BITCLR(ahci->cap, AHCI_CAP_S64A)) {
+        reg = ahci_px_rreg(ahci, port, AHCI_PX_CLBU);
         g_assert_cmphex(reg, ==, 0);
     }
 
     /* (2) FB */
-    reg = PX_RREG(port, AHCI_PX_FB);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_FB);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FB_RESERVED);
 
     /* (3) FBU */
-    if (BITCLR(hcap->cap, AHCI_CAP_S64A)) {
-        reg = PX_RREG(port, AHCI_PX_FBU);
+    if (BITCLR(ahci->cap, AHCI_CAP_S64A)) {
+        reg = ahci_px_rreg(ahci, port, AHCI_PX_FBU);
         g_assert_cmphex(reg, ==, 0);
     }
 
     /* (4) IS */
-    reg = PX_RREG(port, AHCI_PX_IS);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_IS);
     g_assert_cmphex(reg, ==, 0);
 
     /* (5) IE */
-    reg = PX_RREG(port, AHCI_PX_IE);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_IE);
     g_assert_cmphex(reg, ==, 0);
 
     /* (6) CMD */
-    reg = PX_RREG(port, AHCI_PX_CMD);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_CMD);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FRE);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_RESERVED);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CCS);
@@ -1140,27 +693,27 @@
         ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_MPSS);
     }
     /* If we do not support MPS, MPSS and MPSP must be off. */
-    if (BITCLR(hcap->cap, AHCI_CAP_SMPS)) {
+    if (BITCLR(ahci->cap, AHCI_CAP_SMPS)) {
         ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_MPSS);
         ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_MPSP);
     }
     /* If, via CPD or MPSP we detect a drive, HPCP must be on. */
-    if (BITANY(reg, AHCI_PX_CMD_CPD || AHCI_PX_CMD_MPSP)) {
+    if (BITANY(reg, AHCI_PX_CMD_CPD | AHCI_PX_CMD_MPSP)) {
         ASSERT_BIT_SET(reg, AHCI_PX_CMD_HPCP);
     }
     /* HPCP and ESP cannot both be active. */
     g_assert(!BITSET(reg, AHCI_PX_CMD_HPCP | AHCI_PX_CMD_ESP));
     /* If CAP.FBSS is not set, FBSCP must not be set. */
-    if (BITCLR(hcap->cap, AHCI_CAP_FBSS)) {
+    if (BITCLR(ahci->cap, AHCI_CAP_FBSS)) {
         ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FBSCP);
     }
 
     /* (7) RESERVED */
-    reg = PX_RREG(port, AHCI_PX_RES1);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_RES1);
     g_assert_cmphex(reg, ==, 0);
 
     /* (8) TFD */
-    reg = PX_RREG(port, AHCI_PX_TFD);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_TFD);
     /* At boot, prior to an FIS being received, the TFD register should be 0x7F,
      * which breaks down as follows, as seen in AHCI 1.3 sec 3.3.8, p. 27. */
     ASSERT_BIT_SET(reg, AHCI_PX_TFD_STS_ERR);
@@ -1178,53 +731,53 @@
      * so we cannot expect a value here. AHCI 1.3, sec 3.3.9, pp 27-28 */
 
     /* (10) SSTS / SCR0: SStatus */
-    reg = PX_RREG(port, AHCI_PX_SSTS);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SSTS);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_SSTS_RESERVED);
     /* Even though the register should be 0 at boot, it is asynchronous and
      * prone to change, so we cannot test any well known value. */
 
     /* (11) SCTL / SCR2: SControl */
-    reg = PX_RREG(port, AHCI_PX_SCTL);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SCTL);
     g_assert_cmphex(reg, ==, 0);
 
     /* (12) SERR / SCR1: SError */
-    reg = PX_RREG(port, AHCI_PX_SERR);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SERR);
     g_assert_cmphex(reg, ==, 0);
 
     /* (13) SACT / SCR3: SActive */
-    reg = PX_RREG(port, AHCI_PX_SACT);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SACT);
     g_assert_cmphex(reg, ==, 0);
 
     /* (14) CI */
-    reg = PX_RREG(port, AHCI_PX_CI);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_CI);
     g_assert_cmphex(reg, ==, 0);
 
     /* (15) SNTF */
-    reg = PX_RREG(port, AHCI_PX_SNTF);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SNTF);
     g_assert_cmphex(reg, ==, 0);
 
     /* (16) FBS */
-    reg = PX_RREG(port, AHCI_PX_FBS);
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_FBS);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_EN);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_DEC);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_SDE);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_DEV);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_DWE);
     ASSERT_BIT_CLEAR(reg, AHCI_PX_FBS_RESERVED);
-    if (BITSET(hcap->cap, AHCI_CAP_FBSS)) {
+    if (BITSET(ahci->cap, AHCI_CAP_FBSS)) {
         /* if Port-Multiplier FIS-based switching avail, ADO must >= 2 */
         g_assert((reg & AHCI_PX_FBS_ADO) >> ctzl(AHCI_PX_FBS_ADO) >= 2);
     }
 
     /* [17 -- 27] RESERVED */
     for (i = AHCI_PX_RES2; i < AHCI_PX_VS; ++i) {
-        reg = PX_RREG(port, i);
+        reg = ahci_px_rreg(ahci, port, i);
         g_assert_cmphex(reg, ==, 0);
     }
 
     /* [28 -- 31] Vendor-Specific */
     for (i = AHCI_PX_VS; i < 32; ++i) {
-        reg = PX_RREG(port, i);
+        reg = ahci_px_rreg(ahci, port, i);
         if (reg) {
             g_test_message("INFO: Vendor register %u non-empty", i);
         }
@@ -1235,164 +788,46 @@
  * Utilizing an initialized AHCI HBA, issue an IDENTIFY command to the first
  * device we see, then read and check the response.
  */
-static void ahci_test_identify(QPCIDevice *ahci, void *hba_base)
+static void ahci_test_identify(AHCIQState *ahci)
 {
-    RegD2HFIS *d2h = g_malloc0(0x20);
-    RegD2HFIS *pio = g_malloc0(0x20);
-    RegH2DFIS fis;
-    AHCICommand cmd;
-    PRD prd;
-    uint32_t ports, reg, clb, table, fb, data_ptr;
     uint16_t buff[256];
-    unsigned i;
+    unsigned px;
     int rc;
+    uint16_t sect_size;
+    const size_t buffsize = 512;
 
     g_assert(ahci != NULL);
-    g_assert(hba_base != NULL);
 
-    /* We need to:
-     * (1) Create a Command Table Buffer and update the Command List Slot #0
-     *     to point to this buffer.
-     * (2) Construct an FIS host-to-device command structure, and write it to
-     *     the top of the command table buffer.
-     * (3) Create a data buffer for the IDENTIFY response to be sent to
-     * (4) Create a Physical Region Descriptor that points to the data buffer,
-     *     and write it to the bottom (offset 0x80) of the command table.
-     * (5) Now, PxCLB points to the command list, command 0 points to
-     *     our table, and our table contains an FIS instruction and a
-     *     PRD that points to our rx buffer.
-     * (6) We inform the HBA via PxCI that there is a command ready in slot #0.
+    /**
+     * This serves as a bit of a tutorial on AHCI device programming:
+     *
+     * (1) Create a data buffer for the IDENTIFY response to be sent to
+     * (2) Create a Command Table buffer, where we will store the
+     *     command and PRDT (Physical Region Descriptor Table)
+     * (3) Construct an FIS host-to-device command structure, and write it to
+     *     the top of the Command Table buffer.
+     * (4) Create one or more Physical Region Descriptors (PRDs) that describe
+     *     a location in memory where data may be stored/retrieved.
+     * (5) Write these PRDTs to the bottom (offset 0x80) of the Command Table.
+     * (6) Each AHCI port has up to 32 command slots. Each slot contains a
+     *     header that points to a Command Table buffer. Pick an unused slot
+     *     and update it to point to the Command Table we have built.
+     * (7) Now: Command #n points to our Command Table, and our Command Table
+     *     contains the FIS (that describes our command) and the PRDTL, which
+     *     describes our buffer.
+     * (8) We inform the HBA via PxCI (Command Issue) that the command in slot
+     *     #n is ready for processing.
      */
 
     /* Pick the first implemented and running port */
-    ports = AHCI_RREG(AHCI_PI);
-    for (i = 0; i < 32; ports >>= 1, ++i) {
-        if (ports == 0) {
-            i = 32;
-        }
+    px = ahci_port_select(ahci);
+    g_test_message("Selected port %u for test", px);
 
-        if (!(ports & 0x01)) {
-            continue;
-        }
+    /* Clear out the FIS Receive area and any pending interrupts. */
+    ahci_port_clear(ahci, px);
 
-        reg = PX_RREG(i, AHCI_PX_CMD);
-        if (BITSET(reg, AHCI_PX_CMD_ST)) {
-            break;
-        }
-    }
-    g_assert_cmphex(i, <, 32);
-    g_test_message("Selected port %u for test", i);
-
-    /* Clear out this port's interrupts (ignore the init register d2h fis) */
-    reg = PX_RREG(i, AHCI_PX_IS);
-    PX_WREG(i, AHCI_PX_IS, reg);
-    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
-
-    /* Wipe the FIS-Recieve Buffer */
-    fb = PX_RREG(i, AHCI_PX_FB);
-    g_assert_cmphex(fb, !=, 0);
-    qmemset(fb, 0x00, 0x100);
-
-    /* Create a Command Table buffer. 0x80 is the smallest with a PRDTL of 0. */
-    /* We need at least one PRD, so round up to the nearest 0x80 multiple.    */
-    table = guest_alloc(guest_malloc, CMD_TBL_SIZ(1));
-    g_assert(table);
-    ASSERT_BIT_CLEAR(table, 0x7F);
-
-    /* Create a data buffer ... where we will dump the IDENTIFY data to. */
-    data_ptr = guest_alloc(guest_malloc, 512);
-    g_assert(data_ptr);
-
-    /* Grab the Command List Buffer pointer */
-    clb = PX_RREG(i, AHCI_PX_CLB);
-    g_assert(clb);
-
-    /* Copy the existing Command #0 structure from the CLB into local memory,
-     * and build a new command #0. */
-    memread(clb, &cmd, sizeof(cmd));
-    cmd.b1 = 5;    /* reg_h2d_fis is 5 double-words long */
-    cmd.b2 = 0x04; /* clear PxTFD.STS.BSY when done */
-    cmd.prdtl = cpu_to_le16(1); /* One PRD table entry. */
-    cmd.prdbc = 0;
-    cmd.ctba = cpu_to_le32(table);
-    cmd.ctbau = 0;
-
-    /* Construct our PRD, noting that DBC is 0-indexed. */
-    prd.dba = cpu_to_le32(data_ptr);
-    prd.dbau = 0;
-    prd.res = 0;
-    /* 511+1 bytes, request DPS interrupt */
-    prd.dbc = cpu_to_le32(511 | 0x80000000);
-
-    /* Construct our Command FIS, Based on http://wiki.osdev.org/AHCI */
-    memset(&fis, 0x00, sizeof(fis));
-    fis.fis_type = 0x27; /* Register Host-to-Device FIS */
-    fis.command = 0xEC;  /* IDENTIFY */
-    fis.device = 0;
-    fis.flags = 0x80;    /* Indicate this is a command FIS */
-
-    /* We've committed nothing yet, no interrupts should be posted yet. */
-    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
-
-    /* Commit the Command FIS to the Command Table */
-    memwrite(table, &fis, sizeof(fis));
-
-    /* Commit the PRD entry to the Command Table */
-    memwrite(table + 0x80, &prd, sizeof(prd));
-
-    /* Commit Command #0, pointing to the Table, to the Command List Buffer. */
-    memwrite(clb, &cmd, sizeof(cmd));
-
-    /* Everything is in place, but we haven't given the go-ahead yet. */
-    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
-
-    /* Issue Command #0 via PxCI */
-    PX_WREG(i, AHCI_PX_CI, (1 << 0));
-    while (BITSET(PX_RREG(i, AHCI_PX_TFD), AHCI_PX_TFD_STS_BSY)) {
-        usleep(50);
-    }
-
-    /* Check for expected interrupts */
-    reg = PX_RREG(i, AHCI_PX_IS);
-    ASSERT_BIT_SET(reg, AHCI_PX_IS_DHRS);
-    ASSERT_BIT_SET(reg, AHCI_PX_IS_PSS);
-    /* BUG: we expect AHCI_PX_IS_DPS to be set. */
-    ASSERT_BIT_CLEAR(reg, AHCI_PX_IS_DPS);
-
-    /* Clear expected interrupts and assert all interrupts now cleared. */
-    PX_WREG(i, AHCI_PX_IS, AHCI_PX_IS_DHRS | AHCI_PX_IS_PSS | AHCI_PX_IS_DPS);
-    g_assert_cmphex(PX_RREG(i, AHCI_PX_IS), ==, 0);
-
-    /* Check for errors. */
-    reg = PX_RREG(i, AHCI_PX_SERR);
-    g_assert_cmphex(reg, ==, 0);
-    reg = PX_RREG(i, AHCI_PX_TFD);
-    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_STS_ERR);
-    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_ERR);
-
-    /* Investigate CMD #0, assert that we read 512 bytes */
-    memread(clb, &cmd, sizeof(cmd));
-    g_assert_cmphex(512, ==, le32_to_cpu(cmd.prdbc));
-
-    /* Investigate FIS responses */
-    memread(fb + 0x20, pio, 0x20);
-    memread(fb + 0x40, d2h, 0x20);
-    g_assert_cmphex(pio->fis_type, ==, 0x5f);
-    g_assert_cmphex(d2h->fis_type, ==, 0x34);
-    g_assert_cmphex(pio->flags, ==, d2h->flags);
-    g_assert_cmphex(pio->status, ==, d2h->status);
-    g_assert_cmphex(pio->error, ==, d2h->error);
-
-    reg = PX_RREG(i, AHCI_PX_TFD);
-    g_assert_cmphex((reg & AHCI_PX_TFD_ERR), ==, pio->error);
-    g_assert_cmphex((reg & AHCI_PX_TFD_STS), ==, pio->status);
-    /* The PIO Setup FIS contains a "bytes read" field, which is a
-     * 16-bit value. The Physical Region Descriptor Byte Count is
-     * 32-bit, but for small transfers using one PRD, it should match. */
-    g_assert_cmphex(le16_to_cpu(pio->res4), ==, le32_to_cpu(cmd.prdbc));
-
-    /* Last, but not least: Investigate the IDENTIFY response data. */
-    memread(data_ptr, &buff, 512);
+    /* "Read" 512 bytes using CMD_IDENTIFY into the host buffer. */
+    ahci_io(ahci, px, CMD_IDENTIFY, &buff, buffsize, 0);
 
     /* Check serial number/version in the buffer */
     /* NB: IDENTIFY strings are packed in 16bit little endian chunks.
@@ -1407,10 +842,94 @@
     rc = memcmp(&buff[23], "version ", 8);
     g_assert_cmphex(rc, ==, 0);
 
-    g_free(d2h);
-    g_free(pio);
+    sect_size = le16_to_cpu(*((uint16_t *)(&buff[5])));
+    g_assert_cmphex(sect_size, ==, AHCI_SECTOR_SIZE);
 }
 
+static void ahci_test_io_rw_simple(AHCIQState *ahci, unsigned bufsize,
+                                   uint64_t sector, uint8_t read_cmd,
+                                   uint8_t write_cmd)
+{
+    uint64_t ptr;
+    uint8_t port;
+    unsigned char *tx = g_malloc(bufsize);
+    unsigned char *rx = g_malloc0(bufsize);
+
+    g_assert(ahci != NULL);
+
+    /* Pick the first running port and clear it. */
+    port = ahci_port_select(ahci);
+    ahci_port_clear(ahci, port);
+
+    /*** Create pattern and transfer to guest ***/
+    /* Data buffer in the guest */
+    ptr = ahci_alloc(ahci, bufsize);
+    g_assert(ptr);
+
+    /* Write some indicative pattern to our buffer. */
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
+    bufwrite(ptr, tx, bufsize);
+
+    /* Write this buffer to disk, then read it back to the DMA buffer. */
+    ahci_guest_io(ahci, port, write_cmd, ptr, bufsize, sector);
+    qmemset(ptr, 0x00, bufsize);
+    ahci_guest_io(ahci, port, read_cmd, ptr, bufsize, sector);
+
+    /*** Read back the Data ***/
+    bufread(ptr, rx, bufsize);
+    g_assert_cmphex(memcmp(tx, rx, bufsize), ==, 0);
+
+    ahci_free(ahci, ptr);
+    g_free(tx);
+    g_free(rx);
+}
+
+static uint8_t ahci_test_nondata(AHCIQState *ahci, uint8_t ide_cmd)
+{
+    uint8_t port;
+
+    /* Sanitize */
+    port = ahci_port_select(ahci);
+    ahci_port_clear(ahci, port);
+
+    ahci_io(ahci, port, ide_cmd, NULL, 0, 0);
+
+    return port;
+}
+
+static void ahci_test_flush(AHCIQState *ahci)
+{
+    ahci_test_nondata(ahci, CMD_FLUSH_CACHE);
+}
+
+static void ahci_test_max(AHCIQState *ahci)
+{
+    RegD2HFIS *d2h = g_malloc0(0x20);
+    uint64_t nsect;
+    uint8_t port;
+    uint8_t cmd;
+    uint64_t config_sect = mb_to_sectors(test_image_size_mb) - 1;
+
+    if (config_sect > 0xFFFFFF) {
+        cmd = CMD_READ_MAX_EXT;
+    } else {
+        cmd = CMD_READ_MAX;
+    }
+
+    port = ahci_test_nondata(ahci, cmd);
+    memread(ahci->port[port].fb + 0x40, d2h, 0x20);
+    nsect = (uint64_t)d2h->lba_hi[2] << 40 |
+        (uint64_t)d2h->lba_hi[1] << 32 |
+        (uint64_t)d2h->lba_hi[0] << 24 |
+        (uint64_t)d2h->lba_lo[2] << 16 |
+        (uint64_t)d2h->lba_lo[1] << 8 |
+        (uint64_t)d2h->lba_lo[0];
+
+    g_assert_cmphex(nsect, ==, config_sect);
+    g_free(d2h);
+}
+
+
 /******************************************************************************/
 /* Test Interfaces                                                            */
 /******************************************************************************/
@@ -1420,8 +939,8 @@
  */
 static void test_sanity(void)
 {
-    QPCIDevice *ahci;
-    ahci = ahci_boot();
+    AHCIQState *ahci;
+    ahci = ahci_boot(NULL);
     ahci_shutdown(ahci);
 }
 
@@ -1431,8 +950,8 @@
  */
 static void test_pci_spec(void)
 {
-    QPCIDevice *ahci;
-    ahci = ahci_boot();
+    AHCIQState *ahci;
+    ahci = ahci_boot(NULL);
     ahci_test_pci_spec(ahci);
     ahci_shutdown(ahci);
 }
@@ -1443,10 +962,9 @@
  */
 static void test_pci_enable(void)
 {
-    QPCIDevice *ahci;
-    void *hba_base;
-    ahci = ahci_boot();
-    ahci_pci_enable(ahci, &hba_base);
+    AHCIQState *ahci;
+    ahci = ahci_boot(NULL);
+    ahci_pci_enable(ahci);
     ahci_shutdown(ahci);
 }
 
@@ -1456,12 +974,11 @@
  */
 static void test_hba_spec(void)
 {
-    QPCIDevice *ahci;
-    void *hba_base;
+    AHCIQState *ahci;
 
-    ahci = ahci_boot();
-    ahci_pci_enable(ahci, &hba_base);
-    ahci_test_hba_spec(ahci, hba_base);
+    ahci = ahci_boot(NULL);
+    ahci_pci_enable(ahci);
+    ahci_test_hba_spec(ahci);
     ahci_shutdown(ahci);
 }
 
@@ -1471,12 +988,11 @@
  */
 static void test_hba_enable(void)
 {
-    QPCIDevice *ahci;
-    void *hba_base;
+    AHCIQState *ahci;
 
-    ahci = ahci_boot();
-    ahci_pci_enable(ahci, &hba_base);
-    ahci_hba_enable(ahci, hba_base);
+    ahci = ahci_boot(NULL);
+    ahci_pci_enable(ahci);
+    ahci_hba_enable(ahci);
     ahci_shutdown(ahci);
 }
 
@@ -1486,24 +1002,723 @@
  */
 static void test_identify(void)
 {
-    QPCIDevice *ahci;
-    void *hba_base;
+    AHCIQState *ahci;
 
-    ahci = ahci_boot();
-    ahci_pci_enable(ahci, &hba_base);
-    ahci_hba_enable(ahci, hba_base);
-    ahci_test_identify(ahci, hba_base);
+    ahci = ahci_boot_and_enable(NULL);
+    ahci_test_identify(ahci);
     ahci_shutdown(ahci);
 }
 
+/**
+ * Fragmented DMA test: Perform a standard 4K DMA read/write
+ * test, but make sure the physical regions are fragmented to
+ * be very small, each just 32 bytes, to see how AHCI performs
+ * with chunks defined to be much less than a sector.
+ */
+static void test_dma_fragmented(void)
+{
+    AHCIQState *ahci;
+    AHCICommand *cmd;
+    uint8_t px;
+    size_t bufsize = 4096;
+    unsigned char *tx = g_malloc(bufsize);
+    unsigned char *rx = g_malloc0(bufsize);
+    uint64_t ptr;
+
+    ahci = ahci_boot_and_enable(NULL);
+    px = ahci_port_select(ahci);
+    ahci_port_clear(ahci, px);
+
+    /* create pattern */
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
+
+    /* Create a DMA buffer in guest memory, and write our pattern to it. */
+    ptr = guest_alloc(ahci->parent->alloc, bufsize);
+    g_assert(ptr);
+    bufwrite(ptr, tx, bufsize);
+
+    cmd = ahci_command_create(CMD_WRITE_DMA);
+    ahci_command_adjust(cmd, 0, ptr, bufsize, 32);
+    ahci_command_commit(ahci, cmd, px);
+    ahci_command_issue(ahci, cmd);
+    ahci_command_verify(ahci, cmd);
+    ahci_command_free(cmd);
+
+    cmd = ahci_command_create(CMD_READ_DMA);
+    ahci_command_adjust(cmd, 0, ptr, bufsize, 32);
+    ahci_command_commit(ahci, cmd, px);
+    ahci_command_issue(ahci, cmd);
+    ahci_command_verify(ahci, cmd);
+    ahci_command_free(cmd);
+
+    /* Read back the guest's receive buffer into local memory */
+    bufread(ptr, rx, bufsize);
+    guest_free(ahci->parent->alloc, ptr);
+
+    g_assert_cmphex(memcmp(tx, rx, bufsize), ==, 0);
+
+    ahci_shutdown(ahci);
+
+    g_free(rx);
+    g_free(tx);
+}
+
+/*
+ * Write sector 1 with random data to make AHCI storage dirty
+ * Needed for flush tests so that flushes actually go though the block layer
+ */
+static void make_dirty(AHCIQState* ahci, uint8_t port)
+{
+    uint64_t ptr;
+    unsigned bufsize = 512;
+
+    ptr = ahci_alloc(ahci, bufsize);
+    g_assert(ptr);
+
+    ahci_guest_io(ahci, port, CMD_WRITE_DMA, ptr, bufsize, 1);
+    ahci_free(ahci, ptr);
+}
+
+static void test_flush(void)
+{
+    AHCIQState *ahci;
+    uint8_t port;
+
+    ahci = ahci_boot_and_enable(NULL);
+
+    port = ahci_port_select(ahci);
+    ahci_port_clear(ahci, port);
+
+    make_dirty(ahci, port);
+
+    ahci_test_flush(ahci);
+    ahci_shutdown(ahci);
+}
+
+static void test_flush_retry(void)
+{
+    AHCIQState *ahci;
+    AHCICommand *cmd;
+    uint8_t port;
+
+    prepare_blkdebug_script(debug_path, "flush_to_disk");
+    ahci = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
+                                "format=%s,cache=writeback,"
+                                "rerror=stop,werror=stop "
+                                "-M q35 "
+                                "-device ide-hd,drive=drive0 ",
+                                debug_path,
+                                tmp_path, imgfmt);
+
+    port = ahci_port_select(ahci);
+    ahci_port_clear(ahci, port);
+
+    /* Issue write so that flush actually goes to disk */
+    make_dirty(ahci, port);
+
+    /* Issue Flush Command and wait for error */
+    cmd = ahci_guest_io_halt(ahci, port, CMD_FLUSH_CACHE, 0, 0, 0);
+    ahci_guest_io_resume(ahci, cmd);
+
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Basic sanity test to boot a machine, find an AHCI device, and shutdown.
+ */
+static void test_migrate_sanity(void)
+{
+    AHCIQState *src, *dst;
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
+
+    src = ahci_boot("-m 1024 -M q35 "
+                    "-drive if=ide,file=%s,format=%s ", tmp_path, imgfmt);
+    dst = ahci_boot("-m 1024 -M q35 "
+                    "-drive if=ide,file=%s,format=%s "
+                    "-incoming %s", tmp_path, imgfmt, uri);
+
+    ahci_migrate(src, dst, uri);
+
+    ahci_shutdown(src);
+    ahci_shutdown(dst);
+    g_free(uri);
+}
+
+/**
+ * Simple migration test: Write a pattern, migrate, then read.
+ */
+static void ahci_migrate_simple(uint8_t cmd_read, uint8_t cmd_write)
+{
+    AHCIQState *src, *dst;
+    uint8_t px;
+    size_t bufsize = 4096;
+    unsigned char *tx = g_malloc(bufsize);
+    unsigned char *rx = g_malloc0(bufsize);
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
+
+    src = ahci_boot_and_enable("-m 1024 -M q35 "
+                               "-drive if=ide,format=%s,file=%s ",
+                               imgfmt, tmp_path);
+    dst = ahci_boot("-m 1024 -M q35 "
+                    "-drive if=ide,format=%s,file=%s "
+                    "-incoming %s", imgfmt, tmp_path, uri);
+
+    set_context(src->parent);
+
+    /* initialize */
+    px = ahci_port_select(src);
+    ahci_port_clear(src, px);
+
+    /* create pattern */
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
+
+    /* Write, migrate, then read. */
+    ahci_io(src, px, cmd_write, tx, bufsize, 0);
+    ahci_migrate(src, dst, uri);
+    ahci_io(dst, px, cmd_read, rx, bufsize, 0);
+
+    /* Verify pattern */
+    g_assert_cmphex(memcmp(tx, rx, bufsize), ==, 0);
+
+    ahci_shutdown(src);
+    ahci_shutdown(dst);
+    g_free(rx);
+    g_free(tx);
+    g_free(uri);
+}
+
+static void test_migrate_dma(void)
+{
+    ahci_migrate_simple(CMD_READ_DMA, CMD_WRITE_DMA);
+}
+
+static void test_migrate_ncq(void)
+{
+    ahci_migrate_simple(READ_FPDMA_QUEUED, WRITE_FPDMA_QUEUED);
+}
+
+/**
+ * Halted IO Error Test
+ *
+ * Simulate an error on first write, Try to write a pattern,
+ * Confirm the VM has stopped, resume the VM, verify command
+ * has completed, then read back the data and verify.
+ */
+static void ahci_halted_io_test(uint8_t cmd_read, uint8_t cmd_write)
+{
+    AHCIQState *ahci;
+    uint8_t port;
+    size_t bufsize = 4096;
+    unsigned char *tx = g_malloc(bufsize);
+    unsigned char *rx = g_malloc0(bufsize);
+    uint64_t ptr;
+    AHCICommand *cmd;
+
+    prepare_blkdebug_script(debug_path, "write_aio");
+
+    ahci = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
+                                "format=%s,cache=writeback,"
+                                "rerror=stop,werror=stop "
+                                "-M q35 "
+                                "-device ide-hd,drive=drive0 ",
+                                debug_path,
+                                tmp_path, imgfmt);
+
+    /* Initialize and prepare */
+    port = ahci_port_select(ahci);
+    ahci_port_clear(ahci, port);
+
+    /* create DMA source buffer and write pattern */
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
+    ptr = ahci_alloc(ahci, bufsize);
+    g_assert(ptr);
+    memwrite(ptr, tx, bufsize);
+
+    /* Attempt to write (and fail) */
+    cmd = ahci_guest_io_halt(ahci, port, cmd_write,
+                             ptr, bufsize, 0);
+
+    /* Attempt to resume the command */
+    ahci_guest_io_resume(ahci, cmd);
+    ahci_free(ahci, ptr);
+
+    /* Read back and verify */
+    ahci_io(ahci, port, cmd_read, rx, bufsize, 0);
+    g_assert_cmphex(memcmp(tx, rx, bufsize), ==, 0);
+
+    /* Cleanup and go home */
+    ahci_shutdown(ahci);
+    g_free(rx);
+    g_free(tx);
+}
+
+static void test_halted_dma(void)
+{
+    ahci_halted_io_test(CMD_READ_DMA, CMD_WRITE_DMA);
+}
+
+static void test_halted_ncq(void)
+{
+    ahci_halted_io_test(READ_FPDMA_QUEUED, WRITE_FPDMA_QUEUED);
+}
+
+/**
+ * IO Error Migration Test
+ *
+ * Simulate an error on first write, Try to write a pattern,
+ * Confirm the VM has stopped, migrate, resume the VM,
+ * verify command has completed, then read back the data and verify.
+ */
+static void ahci_migrate_halted_io(uint8_t cmd_read, uint8_t cmd_write)
+{
+    AHCIQState *src, *dst;
+    uint8_t port;
+    size_t bufsize = 4096;
+    unsigned char *tx = g_malloc(bufsize);
+    unsigned char *rx = g_malloc0(bufsize);
+    uint64_t ptr;
+    AHCICommand *cmd;
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
+
+    prepare_blkdebug_script(debug_path, "write_aio");
+
+    src = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
+                               "format=%s,cache=writeback,"
+                               "rerror=stop,werror=stop "
+                               "-M q35 "
+                               "-device ide-hd,drive=drive0 ",
+                               debug_path,
+                               tmp_path, imgfmt);
+
+    dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
+                    "format=%s,cache=writeback,"
+                    "rerror=stop,werror=stop "
+                    "-M q35 "
+                    "-device ide-hd,drive=drive0 "
+                    "-incoming %s",
+                    tmp_path, imgfmt, uri);
+
+    set_context(src->parent);
+
+    /* Initialize and prepare */
+    port = ahci_port_select(src);
+    ahci_port_clear(src, port);
+    generate_pattern(tx, bufsize, AHCI_SECTOR_SIZE);
+
+    /* create DMA source buffer and write pattern */
+    ptr = ahci_alloc(src, bufsize);
+    g_assert(ptr);
+    memwrite(ptr, tx, bufsize);
+
+    /* Write, trigger the VM to stop, migrate, then resume. */
+    cmd = ahci_guest_io_halt(src, port, cmd_write,
+                             ptr, bufsize, 0);
+    ahci_migrate(src, dst, uri);
+    ahci_guest_io_resume(dst, cmd);
+    ahci_free(dst, ptr);
+
+    /* Read back */
+    ahci_io(dst, port, cmd_read, rx, bufsize, 0);
+
+    /* Verify TX and RX are identical */
+    g_assert_cmphex(memcmp(tx, rx, bufsize), ==, 0);
+
+    /* Cleanup and go home. */
+    ahci_shutdown(src);
+    ahci_shutdown(dst);
+    g_free(rx);
+    g_free(tx);
+    g_free(uri);
+}
+
+static void test_migrate_halted_dma(void)
+{
+    ahci_migrate_halted_io(CMD_READ_DMA, CMD_WRITE_DMA);
+}
+
+static void test_migrate_halted_ncq(void)
+{
+    ahci_migrate_halted_io(READ_FPDMA_QUEUED, WRITE_FPDMA_QUEUED);
+}
+
+/**
+ * Migration test: Try to flush, migrate, then resume.
+ */
+static void test_flush_migrate(void)
+{
+    AHCIQState *src, *dst;
+    AHCICommand *cmd;
+    uint8_t px;
+    const char *s;
+    char *uri = g_strdup_printf("unix:%s", mig_socket);
+
+    prepare_blkdebug_script(debug_path, "flush_to_disk");
+
+    src = ahci_boot_and_enable("-drive file=blkdebug:%s:%s,if=none,id=drive0,"
+                               "cache=writeback,rerror=stop,werror=stop,"
+                               "format=%s "
+                               "-M q35 "
+                               "-device ide-hd,drive=drive0 ",
+                               debug_path, tmp_path, imgfmt);
+    dst = ahci_boot("-drive file=%s,if=none,id=drive0,"
+                    "cache=writeback,rerror=stop,werror=stop,"
+                    "format=%s "
+                    "-M q35 "
+                    "-device ide-hd,drive=drive0 "
+                    "-incoming %s", tmp_path, imgfmt, uri);
+
+    set_context(src->parent);
+
+    px = ahci_port_select(src);
+    ahci_port_clear(src, px);
+
+    /* Dirty device so that flush reaches disk */
+    make_dirty(src, px);
+
+    /* Issue Flush Command */
+    cmd = ahci_command_create(CMD_FLUSH_CACHE);
+    ahci_command_commit(src, cmd, px);
+    ahci_command_issue_async(src, cmd);
+    qmp_eventwait("STOP");
+
+    /* Migrate over */
+    ahci_migrate(src, dst, uri);
+
+    /* Complete the command */
+    s = "{'execute':'cont' }";
+    qmp_async(s);
+    qmp_eventwait("RESUME");
+    ahci_command_wait(dst, cmd);
+    ahci_command_verify(dst, cmd);
+
+    ahci_command_free(cmd);
+    ahci_shutdown(src);
+    ahci_shutdown(dst);
+    g_free(uri);
+}
+
+static void test_max(void)
+{
+    AHCIQState *ahci;
+
+    ahci = ahci_boot_and_enable(NULL);
+    ahci_test_max(ahci);
+    ahci_shutdown(ahci);
+}
+
+static void test_reset(void)
+{
+    AHCIQState *ahci;
+    int i;
+
+    ahci = ahci_boot(NULL);
+    ahci_test_pci_spec(ahci);
+    ahci_pci_enable(ahci);
+
+    for (i = 0; i < 2; i++) {
+        ahci_test_hba_spec(ahci);
+        ahci_hba_enable(ahci);
+        ahci_test_identify(ahci);
+        ahci_test_io_rw_simple(ahci, 4096, 0,
+                               CMD_READ_DMA_EXT,
+                               CMD_WRITE_DMA_EXT);
+        ahci_set(ahci, AHCI_GHC, AHCI_GHC_HR);
+        ahci_clean_mem(ahci);
+    }
+
+    ahci_shutdown(ahci);
+}
+
+static void test_ncq_simple(void)
+{
+    AHCIQState *ahci;
+
+    ahci = ahci_boot_and_enable(NULL);
+    ahci_test_io_rw_simple(ahci, 4096, 0,
+                           READ_FPDMA_QUEUED,
+                           WRITE_FPDMA_QUEUED);
+    ahci_shutdown(ahci);
+}
+
+static int prepare_iso(size_t size, unsigned char **buf, char **name)
+{
+    char cdrom_path[] = "/tmp/qtest.iso.XXXXXX";
+    unsigned char *patt;
+    ssize_t ret;
+    int fd = mkstemp(cdrom_path);
+
+    g_assert(buf);
+    g_assert(name);
+    patt = g_malloc(size);
+
+    /* Generate a pattern and build a CDROM image to read from */
+    generate_pattern(patt, size, ATAPI_SECTOR_SIZE);
+    ret = write(fd, patt, size);
+    g_assert(ret == size);
+
+    *name = g_strdup(cdrom_path);
+    *buf = patt;
+    return fd;
+}
+
+static void remove_iso(int fd, char *name)
+{
+    unlink(name);
+    g_free(name);
+    close(fd);
+}
+
+static int ahci_cb_cmp_buff(AHCIQState *ahci, AHCICommand *cmd,
+                            const AHCIOpts *opts)
+{
+    unsigned char *tx = opts->opaque;
+    unsigned char *rx = g_malloc0(opts->size);
+
+    bufread(opts->buffer, rx, opts->size);
+    g_assert_cmphex(memcmp(tx, rx, opts->size), ==, 0);
+    g_free(rx);
+
+    return 0;
+}
+
+static void ahci_test_cdrom(int nsectors, bool dma)
+{
+    AHCIQState *ahci;
+    unsigned char *tx;
+    char *iso;
+    int fd;
+    AHCIOpts opts = {
+        .size = (ATAPI_SECTOR_SIZE * nsectors),
+        .atapi = true,
+        .atapi_dma = dma,
+        .post_cb = ahci_cb_cmp_buff,
+    };
+
+    /* Prepare ISO and fill 'tx' buffer */
+    fd = prepare_iso(1024 * 1024, &tx, &iso);
+    opts.opaque = tx;
+
+    /* Standard startup wonkery, but use ide-cd and our special iso file */
+    ahci = ahci_boot_and_enable("-drive if=none,id=drive0,file=%s,format=raw "
+                                "-M q35 "
+                                "-device ide-cd,drive=drive0 ", iso);
+
+    /* Build & Send AHCI command */
+    ahci_exec(ahci, ahci_port_select(ahci), CMD_ATAPI_READ_10, &opts);
+
+    /* Cleanup */
+    g_free(tx);
+    ahci_shutdown(ahci);
+    remove_iso(fd, iso);
+}
+
+static void test_cdrom_dma(void)
+{
+    ahci_test_cdrom(1, true);
+}
+
+static void test_cdrom_dma_multi(void)
+{
+    ahci_test_cdrom(3, true);
+}
+
+static void test_cdrom_pio(void)
+{
+    ahci_test_cdrom(1, false);
+}
+
+static void test_cdrom_pio_multi(void)
+{
+    ahci_test_cdrom(3, false);
+}
+
+/******************************************************************************/
+/* AHCI I/O Test Matrix Definitions                                           */
+
+enum BuffLen {
+    LEN_BEGIN = 0,
+    LEN_SIMPLE = LEN_BEGIN,
+    LEN_DOUBLE,
+    LEN_LONG,
+    LEN_SHORT,
+    NUM_LENGTHS
+};
+
+static const char *buff_len_str[NUM_LENGTHS] = { "simple", "double",
+                                                 "long", "short" };
+
+enum AddrMode {
+    ADDR_MODE_BEGIN = 0,
+    ADDR_MODE_LBA28 = ADDR_MODE_BEGIN,
+    ADDR_MODE_LBA48,
+    NUM_ADDR_MODES
+};
+
+static const char *addr_mode_str[NUM_ADDR_MODES] = { "lba28", "lba48" };
+
+enum IOMode {
+    MODE_BEGIN = 0,
+    MODE_PIO = MODE_BEGIN,
+    MODE_DMA,
+    NUM_MODES
+};
+
+static const char *io_mode_str[NUM_MODES] = { "pio", "dma" };
+
+enum IOOps {
+    IO_BEGIN = 0,
+    IO_READ = IO_BEGIN,
+    IO_WRITE,
+    NUM_IO_OPS
+};
+
+enum OffsetType {
+    OFFSET_BEGIN = 0,
+    OFFSET_ZERO = OFFSET_BEGIN,
+    OFFSET_LOW,
+    OFFSET_HIGH,
+    NUM_OFFSETS
+};
+
+static const char *offset_str[NUM_OFFSETS] = { "zero", "low", "high" };
+
+typedef struct AHCIIOTestOptions {
+    enum BuffLen length;
+    enum AddrMode address_type;
+    enum IOMode io_type;
+    enum OffsetType offset;
+} AHCIIOTestOptions;
+
+static uint64_t offset_sector(enum OffsetType ofst,
+                              enum AddrMode addr_type,
+                              uint64_t buffsize)
+{
+    uint64_t ceil;
+    uint64_t nsectors;
+
+    switch (ofst) {
+    case OFFSET_ZERO:
+        return 0;
+    case OFFSET_LOW:
+        return 1;
+    case OFFSET_HIGH:
+        ceil = (addr_type == ADDR_MODE_LBA28) ? 0xfffffff : 0xffffffffffff;
+        ceil = MIN(ceil, mb_to_sectors(test_image_size_mb) - 1);
+        nsectors = buffsize / AHCI_SECTOR_SIZE;
+        return ceil - nsectors + 1;
+    default:
+        g_assert_not_reached();
+    }
+}
+
+/**
+ * Table of possible I/O ATA commands given a set of enumerations.
+ */
+static const uint8_t io_cmds[NUM_MODES][NUM_ADDR_MODES][NUM_IO_OPS] = {
+    [MODE_PIO] = {
+        [ADDR_MODE_LBA28] = {
+            [IO_READ] = CMD_READ_PIO,
+            [IO_WRITE] = CMD_WRITE_PIO },
+        [ADDR_MODE_LBA48] = {
+            [IO_READ] = CMD_READ_PIO_EXT,
+            [IO_WRITE] = CMD_WRITE_PIO_EXT }
+    },
+    [MODE_DMA] = {
+        [ADDR_MODE_LBA28] = {
+            [IO_READ] = CMD_READ_DMA,
+            [IO_WRITE] = CMD_WRITE_DMA },
+        [ADDR_MODE_LBA48] = {
+            [IO_READ] = CMD_READ_DMA_EXT,
+            [IO_WRITE] = CMD_WRITE_DMA_EXT }
+    }
+};
+
+/**
+ * Test a Read/Write pattern using various commands, addressing modes,
+ * transfer modes, and buffer sizes.
+ */
+static void test_io_rw_interface(enum AddrMode lba48, enum IOMode dma,
+                                 unsigned bufsize, uint64_t sector)
+{
+    AHCIQState *ahci;
+
+    ahci = ahci_boot_and_enable(NULL);
+    ahci_test_io_rw_simple(ahci, bufsize, sector,
+                           io_cmds[dma][lba48][IO_READ],
+                           io_cmds[dma][lba48][IO_WRITE]);
+    ahci_shutdown(ahci);
+}
+
+/**
+ * Demultiplex the test data and invoke the actual test routine.
+ */
+static void test_io_interface(gconstpointer opaque)
+{
+    AHCIIOTestOptions *opts = (AHCIIOTestOptions *)opaque;
+    unsigned bufsize;
+    uint64_t sector;
+
+    switch (opts->length) {
+    case LEN_SIMPLE:
+        bufsize = 4096;
+        break;
+    case LEN_DOUBLE:
+        bufsize = 8192;
+        break;
+    case LEN_LONG:
+        bufsize = 4096 * 64;
+        break;
+    case LEN_SHORT:
+        bufsize = 512;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    sector = offset_sector(opts->offset, opts->address_type, bufsize);
+    test_io_rw_interface(opts->address_type, opts->io_type, bufsize, sector);
+    g_free(opts);
+    return;
+}
+
+static void create_ahci_io_test(enum IOMode type, enum AddrMode addr,
+                                enum BuffLen len, enum OffsetType offset)
+{
+    char *name;
+    AHCIIOTestOptions *opts;
+
+    opts = g_malloc(sizeof(AHCIIOTestOptions));
+    opts->length = len;
+    opts->address_type = addr;
+    opts->io_type = type;
+    opts->offset = offset;
+
+    name = g_strdup_printf("ahci/io/%s/%s/%s/%s",
+                           io_mode_str[type],
+                           addr_mode_str[addr],
+                           buff_len_str[len],
+                           offset_str[offset]);
+
+    if ((addr == ADDR_MODE_LBA48) && (offset == OFFSET_HIGH) &&
+        (mb_to_sectors(test_image_size_mb) <= 0xFFFFFFF)) {
+        g_test_message("%s: skipped; test image too small", name);
+        g_free(name);
+        return;
+    }
+
+    qtest_add_data_func(name, opts, test_io_interface);
+    g_free(name);
+}
+
 /******************************************************************************/
 
 int main(int argc, char **argv)
 {
     const char *arch;
-    int fd;
     int ret;
+    int fd;
     int c;
+    int i, j, k, m;
 
     static struct option long_options[] = {
         {"pedantic", no_argument, 0, 'p' },
@@ -1537,11 +1752,31 @@
         return 0;
     }
 
-    /* Create a temporary raw image */
+    /* Create a temporary image */
     fd = mkstemp(tmp_path);
     g_assert(fd >= 0);
-    ret = ftruncate(fd, TEST_IMAGE_SIZE);
-    g_assert(ret == 0);
+    if (have_qemu_img()) {
+        imgfmt = "qcow2";
+        test_image_size_mb = TEST_IMAGE_SIZE_MB_LARGE;
+        mkqcow2(tmp_path, TEST_IMAGE_SIZE_MB_LARGE);
+    } else {
+        g_test_message("QTEST_QEMU_IMG not set or qemu-img missing; "
+                       "skipping LBA48 high-sector tests");
+        imgfmt = "raw";
+        test_image_size_mb = TEST_IMAGE_SIZE_MB_SMALL;
+        ret = ftruncate(fd, test_image_size_mb * 1024 * 1024);
+        g_assert(ret == 0);
+    }
+    close(fd);
+
+    /* Create temporary blkdebug instructions */
+    fd = mkstemp(debug_path);
+    g_assert(fd >= 0);
+    close(fd);
+
+    /* Reserve a hollow file to use as a socket for migration tests */
+    fd = mkstemp(mig_socket);
+    g_assert(fd >= 0);
     close(fd);
 
     /* Run the tests */
@@ -1552,10 +1787,46 @@
     qtest_add_func("/ahci/hba_enable", test_hba_enable);
     qtest_add_func("/ahci/identify",   test_identify);
 
+    for (i = MODE_BEGIN; i < NUM_MODES; i++) {
+        for (j = ADDR_MODE_BEGIN; j < NUM_ADDR_MODES; j++) {
+            for (k = LEN_BEGIN; k < NUM_LENGTHS; k++) {
+                for (m = OFFSET_BEGIN; m < NUM_OFFSETS; m++) {
+                    create_ahci_io_test(i, j, k, m);
+                }
+            }
+        }
+    }
+
+    qtest_add_func("/ahci/io/dma/lba28/fragmented", test_dma_fragmented);
+
+    qtest_add_func("/ahci/flush/simple", test_flush);
+    qtest_add_func("/ahci/flush/retry", test_flush_retry);
+    qtest_add_func("/ahci/flush/migrate", test_flush_migrate);
+
+    qtest_add_func("/ahci/migrate/sanity", test_migrate_sanity);
+    qtest_add_func("/ahci/migrate/dma/simple", test_migrate_dma);
+    qtest_add_func("/ahci/io/dma/lba28/retry", test_halted_dma);
+    qtest_add_func("/ahci/migrate/dma/halted", test_migrate_halted_dma);
+
+    qtest_add_func("/ahci/max", test_max);
+    qtest_add_func("/ahci/reset", test_reset);
+
+    qtest_add_func("/ahci/io/ncq/simple", test_ncq_simple);
+    qtest_add_func("/ahci/migrate/ncq/simple", test_migrate_ncq);
+    qtest_add_func("/ahci/io/ncq/retry", test_halted_ncq);
+    qtest_add_func("/ahci/migrate/ncq/halted", test_migrate_halted_ncq);
+
+    qtest_add_func("/ahci/cdrom/dma/single", test_cdrom_dma);
+    qtest_add_func("/ahci/cdrom/dma/multi", test_cdrom_dma_multi);
+    qtest_add_func("/ahci/cdrom/pio/single", test_cdrom_pio);
+    qtest_add_func("/ahci/cdrom/pio/multi", test_cdrom_pio_multi);
+
     ret = g_test_run();
 
     /* Cleanup */
     unlink(tmp_path);
+    unlink(debug_path);
+    unlink(mig_socket);
 
     return ret;
 }
diff --git a/tests/bios-tables-test.c b/tests/bios-tables-test.c
index 9e4d205..de4019e 100644
--- a/tests/bios-tables-test.c
+++ b/tests/bios-tables-test.c
@@ -10,16 +10,14 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
-#include <stdio.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include <glib/gstdio.h>
 #include "qemu-common.h"
 #include "libqtest.h"
-#include "qemu/compiler.h"
-#include "hw/i386/acpi-defs.h"
-#include "hw/i386/smbios.h"
+#include "hw/acpi/acpi-defs.h"
+#include "hw/smbios/smbios.h"
 #include "qemu/bitmap.h"
+#include "boot-sector.h"
 
 #define MACHINE_PC "pc"
 #define MACHINE_Q35 "q35"
@@ -40,6 +38,7 @@
 
 typedef struct {
     const char *machine;
+    const char *variant;
     uint32_t rsdp_addr;
     AcpiRsdpDescriptor rsdp_table;
     AcpiRsdtDescriptorRev1 rsdt_table;
@@ -49,16 +48,11 @@
     int rsdt_tables_nr;
     GArray *tables;
     uint32_t smbios_ep_addr;
-    struct smbios_entry_point smbios_ep_table;
+    struct smbios_21_entry_point smbios_ep_table;
+    uint8_t *required_struct_types;
+    int required_struct_types_len;
 } test_data;
 
-#define LOW(x) ((x) & 0xff)
-#define HIGH(x) ((x) >> 8)
-
-#define SIGNATURE 0xdead
-#define SIGNATURE_OFFSET 0x10
-#define BOOT_SECTOR_ADDRESS 0x7c00
-
 #define ACPI_READ_FIELD(field, addr)           \
     do {                                       \
         switch (sizeof(field)) {               \
@@ -118,35 +112,6 @@
     g_assert_cmpstr(ACPI_ASSERT_CMP_str, ==, expected); \
 } while (0)
 
-/* Boot sector code: write SIGNATURE into memory,
- * then halt.
- * Q35 machine requires a minimum 0x7e000 bytes disk.
- * (bug or feature?)
- */
-static uint8_t boot_sector[0x7e000] = {
-    /* 7c00: mov $0xdead,%ax */
-    [0x00] = 0xb8,
-    [0x01] = LOW(SIGNATURE),
-    [0x02] = HIGH(SIGNATURE),
-    /* 7c03:  mov %ax,0x7c10 */
-    [0x03] = 0xa3,
-    [0x04] = LOW(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
-    [0x05] = HIGH(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
-    /* 7c06: cli */
-    [0x06] = 0xfa,
-    /* 7c07: hlt */
-    [0x07] = 0xf4,
-    /* 7c08: jmp 0x7c07=0x7c0a-3 */
-    [0x08] = 0xeb,
-    [0x09] = LOW(-3),
-    /* We mov 0xdead here: set value to make debugging easier */
-    [SIGNATURE_OFFSET] = LOW(0xface),
-    [SIGNATURE_OFFSET + 1] = HIGH(0xface),
-    /* End of boot sector marker */
-    [0x1FE] = 0x55,
-    [0x1FF] = 0xAA,
-};
-
 static const char *disk = "tests/acpi-test-disk.raw";
 static const char *data_dir = "tests/acpi-test-data";
 #ifdef CONFIG_IASL
@@ -160,31 +125,23 @@
     AcpiSdtTable *temp;
     int i;
 
-    if (data->rsdt_tables_addr) {
-        g_free(data->rsdt_tables_addr);
-    }
+    g_free(data->rsdt_tables_addr);
 
     for (i = 0; i < data->tables->len; ++i) {
         temp = &g_array_index(data->tables, AcpiSdtTable, i);
-        if (temp->aml) {
-            g_free(temp->aml);
+        g_free(temp->aml);
+        if (temp->aml_file &&
+            !temp->tmp_files_retain &&
+            g_strstr_len(temp->aml_file, -1, "aml-")) {
+            unlink(temp->aml_file);
         }
-        if (temp->aml_file) {
-            if (!temp->tmp_files_retain &&
-                g_strstr_len(temp->aml_file, -1, "aml-")) {
-                unlink(temp->aml_file);
-            }
-            g_free(temp->aml_file);
+        g_free(temp->aml_file);
+        g_free(temp->asl);
+        if (temp->asl_file &&
+            !temp->tmp_files_retain) {
+            unlink(temp->asl_file);
         }
-        if (temp->asl) {
-            g_free(temp->asl);
-        }
-        if (temp->asl_file) {
-            if (!temp->tmp_files_retain) {
-                unlink(temp->asl_file);
-            }
-            g_free(temp->asl_file);
-        }
+        g_free(temp->asl_file);
     }
 
     g_array_free(data->tables, false);
@@ -379,7 +336,7 @@
     for (i = 0; i < tables_nr; i++) {
         AcpiSdtTable ssdt_table;
 
-        memset(&ssdt_table, 0 , sizeof(ssdt_table));
+        memset(&ssdt_table, 0, sizeof(ssdt_table));
         uint32_t addr = data->rsdt_tables_addr[i + 1]; /* fadt is first */
         test_dst_table(&ssdt_table, addr);
         g_array_append_val(data->tables, ssdt_table);
@@ -396,13 +353,14 @@
     int i;
 
     for (i = 0; i < data->tables->len; ++i) {
+        const char *ext = data->variant ? data->variant : "";
         sdt = &g_array_index(data->tables, AcpiSdtTable, i);
         g_assert(sdt->aml);
 
         if (rebuild) {
             uint32_t signature = cpu_to_le32(sdt->header.signature);
-            aml_file = g_strdup_printf("%s/%s/%.4s", data_dir, data->machine,
-                                       (gchar *)&signature);
+            aml_file = g_strdup_printf("%s/%s/%.4s%s", data_dir, data->machine,
+                                       (gchar *)&signature, ext);
             fd = g_open(aml_file, O_WRONLY|O_TRUNC|O_CREAT,
                         S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH);
         } else {
@@ -418,9 +376,7 @@
 
         close(fd);
 
-        if (aml_file) {
-            g_free(aml_file);
-        }
+        g_free(aml_file);
     }
 }
 
@@ -477,7 +433,7 @@
 
 #define COMMENT_END "*/"
 #define DEF_BLOCK "DefinitionBlock ("
-#define BLOCK_NAME_END ".aml"
+#define BLOCK_NAME_END ","
 
 static GString *normalize_asl(gchar *asl_code)
 {
@@ -509,7 +465,6 @@
 {
     int i;
     AcpiSdtTable *sdt;
-    gchar *aml_file;
     GError *error = NULL;
     gboolean ret;
 
@@ -517,6 +472,8 @@
     for (i = 0; i < data->tables->len; ++i) {
         AcpiSdtTable exp_sdt;
         uint32_t signature;
+        gchar *aml_file = NULL;
+        const char *ext = data->variant ? data->variant : "";
 
         sdt = &g_array_index(data->tables, AcpiSdtTable, i);
 
@@ -524,10 +481,25 @@
         exp_sdt.header.signature = sdt->header.signature;
 
         signature = cpu_to_le32(sdt->header.signature);
-        aml_file = g_strdup_printf("%s/%s/%.4s", data_dir, data->machine,
-                                   (gchar *)&signature);
-        exp_sdt.aml_file = aml_file;
-        g_assert(g_file_test(aml_file, G_FILE_TEST_EXISTS));
+
+try_again:
+        aml_file = g_strdup_printf("%s/%s/%.4s%s", data_dir, data->machine,
+                                   (gchar *)&signature, ext);
+        if (getenv("V")) {
+            fprintf(stderr, "\nLooking for expected file '%s'\n", aml_file);
+        }
+        if (g_file_test(aml_file, G_FILE_TEST_EXISTS)) {
+            exp_sdt.aml_file = aml_file;
+        } else if (*ext != '\0') {
+            /* try fallback to generic (extention less) expected file */
+            ext = "";
+            g_free(aml_file);
+            goto try_again;
+        }
+        g_assert(exp_sdt.aml_file);
+        if (getenv("V")) {
+            fprintf(stderr, "\nUsing expected file '%s'\n", aml_file);
+        }
         ret = g_file_get_contents(aml_file, &exp_sdt.aml,
                                   &exp_sdt.aml_len, &error);
         g_assert(ret);
@@ -580,6 +552,22 @@
                         (gchar *)&signature,
                         sdt->asl_file, sdt->aml_file,
                         exp_sdt->asl_file, exp_sdt->aml_file);
+                if (getenv("V")) {
+                    const char *diff_cmd = getenv("DIFF");
+                    if (diff_cmd) {
+                        int ret G_GNUC_UNUSED;
+                        char *diff = g_strdup_printf("%s %s %s", diff_cmd,
+                            exp_sdt->asl_file, sdt->asl_file);
+                        ret = system(diff) ;
+                        g_free(diff);
+                    } else {
+                        fprintf(stderr, "acpi-test: Warning. not showing "
+                            "difference since no diff utility is specified. "
+                            "Set 'DIFF' environment variable to a preferred "
+                            "diff utility and run 'make V=1 check' again to "
+                            "see ASL difference.");
+                    }
+                }
           }
         }
         g_string_free(asl, true);
@@ -589,7 +577,45 @@
     free_test_data(&exp_data);
 }
 
-static void test_smbios_ep_address(test_data *data)
+static bool smbios_ep_table_ok(test_data *data)
+{
+    struct smbios_21_entry_point *ep_table = &data->smbios_ep_table;
+    uint32_t addr = data->smbios_ep_addr;
+
+    ACPI_READ_ARRAY(ep_table->anchor_string, addr);
+    if (memcmp(ep_table->anchor_string, "_SM_", 4)) {
+        return false;
+    }
+    ACPI_READ_FIELD(ep_table->checksum, addr);
+    ACPI_READ_FIELD(ep_table->length, addr);
+    ACPI_READ_FIELD(ep_table->smbios_major_version, addr);
+    ACPI_READ_FIELD(ep_table->smbios_minor_version, addr);
+    ACPI_READ_FIELD(ep_table->max_structure_size, addr);
+    ACPI_READ_FIELD(ep_table->entry_point_revision, addr);
+    ACPI_READ_ARRAY(ep_table->formatted_area, addr);
+    ACPI_READ_ARRAY(ep_table->intermediate_anchor_string, addr);
+    if (memcmp(ep_table->intermediate_anchor_string, "_DMI_", 5)) {
+        return false;
+    }
+    ACPI_READ_FIELD(ep_table->intermediate_checksum, addr);
+    ACPI_READ_FIELD(ep_table->structure_table_length, addr);
+    if (ep_table->structure_table_length == 0) {
+        return false;
+    }
+    ACPI_READ_FIELD(ep_table->structure_table_address, addr);
+    ACPI_READ_FIELD(ep_table->number_of_structures, addr);
+    if (ep_table->number_of_structures == 0) {
+        return false;
+    }
+    ACPI_READ_FIELD(ep_table->smbios_bcd_revision, addr);
+    if (acpi_checksum((uint8_t *)ep_table, sizeof *ep_table) ||
+        acpi_checksum((uint8_t *)ep_table + 0x10, sizeof *ep_table - 0x10)) {
+        return false;
+    }
+    return true;
+}
+
+static void test_smbios_entry_point(test_data *data)
 {
     uint32_t off;
 
@@ -603,40 +629,15 @@
         }
 
         if (!memcmp(sig, "_SM_", sizeof sig)) {
-            break;
+            /* signature match, but is this a valid entry point? */
+            data->smbios_ep_addr = off;
+            if (smbios_ep_table_ok(data)) {
+                break;
+            }
         }
     }
 
     g_assert_cmphex(off, <, 0x100000);
-    data->smbios_ep_addr = off;
-}
-
-static void test_smbios_ep_table(test_data *data)
-{
-    struct smbios_entry_point *ep_table = &data->smbios_ep_table;
-    uint32_t addr = data->smbios_ep_addr;
-
-    ACPI_READ_ARRAY(ep_table->anchor_string, addr);
-    g_assert(!memcmp(ep_table->anchor_string, "_SM_", 4));
-    ACPI_READ_FIELD(ep_table->checksum, addr);
-    ACPI_READ_FIELD(ep_table->length, addr);
-    ACPI_READ_FIELD(ep_table->smbios_major_version, addr);
-    ACPI_READ_FIELD(ep_table->smbios_minor_version, addr);
-    ACPI_READ_FIELD(ep_table->max_structure_size, addr);
-    ACPI_READ_FIELD(ep_table->entry_point_revision, addr);
-    ACPI_READ_ARRAY(ep_table->formatted_area, addr);
-    ACPI_READ_ARRAY(ep_table->intermediate_anchor_string, addr);
-    g_assert(!memcmp(ep_table->intermediate_anchor_string, "_DMI_", 5));
-    ACPI_READ_FIELD(ep_table->intermediate_checksum, addr);
-    ACPI_READ_FIELD(ep_table->structure_table_length, addr);
-    g_assert_cmpuint(ep_table->structure_table_length, >, 0);
-    ACPI_READ_FIELD(ep_table->structure_table_address, addr);
-    ACPI_READ_FIELD(ep_table->number_of_structures, addr);
-    g_assert_cmpuint(ep_table->number_of_structures, >, 0);
-    ACPI_READ_FIELD(ep_table->smbios_bcd_revision, addr);
-    g_assert(!acpi_checksum((uint8_t *)ep_table, sizeof *ep_table));
-    g_assert(!acpi_checksum((uint8_t *)ep_table + 0x10,
-                            sizeof *ep_table - 0x10));
 }
 
 static inline bool smbios_single_instance(uint8_t type)
@@ -658,11 +659,10 @@
 static void test_smbios_structs(test_data *data)
 {
     DECLARE_BITMAP(struct_bitmap, SMBIOS_MAX_TYPE+1) = { 0 };
-    struct smbios_entry_point *ep_table = &data->smbios_ep_table;
+    struct smbios_21_entry_point *ep_table = &data->smbios_ep_table;
     uint32_t addr = ep_table->structure_table_address;
     int i, len, max_len = 0;
     uint8_t type, prv, crt;
-    uint8_t required_struct_types[] = {0, 1, 3, 4, 16, 17, 19, 32, 127};
 
     /* walk the smbios tables */
     for (i = 0; i < ep_table->number_of_structures; i++) {
@@ -702,44 +702,23 @@
     g_assert_cmpuint(ep_table->max_structure_size, ==, max_len);
 
     /* required struct types must all be present */
-    for (i = 0; i < ARRAY_SIZE(required_struct_types); i++) {
-        g_assert(test_bit(required_struct_types[i], struct_bitmap));
+    for (i = 0; i < data->required_struct_types_len; i++) {
+        g_assert(test_bit(data->required_struct_types[i], struct_bitmap));
     }
 }
 
 static void test_acpi_one(const char *params, test_data *data)
 {
     char *args;
-    uint8_t signature_low;
-    uint8_t signature_high;
-    uint16_t signature;
-    int i;
 
     args = g_strdup_printf("-net none -display none %s "
-                           "-drive id=hd0,if=none,file=%s "
+                           "-drive id=hd0,if=none,file=%s,format=raw "
                            "-device ide-hd,drive=hd0 ",
                            params ? params : "", disk);
 
     qtest_start(args);
 
-   /* Wait at most 1 minute */
-#define TEST_DELAY (1 * G_USEC_PER_SEC / 10)
-#define TEST_CYCLES MAX((60 * G_USEC_PER_SEC / TEST_DELAY), 1)
-
-    /* Poll until code has run and modified memory.  Once it has we know BIOS
-     * initialization is done.  TODO: check that IP reached the halt
-     * instruction.
-     */
-    for (i = 0; i < TEST_CYCLES; ++i) {
-        signature_low = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET);
-        signature_high = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET + 1);
-        signature = (signature_high << 8) | signature_low;
-        if (signature == SIGNATURE) {
-            break;
-        }
-        g_usleep(TEST_DELAY);
-    }
-    g_assert_cmphex(signature, ==, SIGNATURE);
+    boot_sector_test();
 
     test_acpi_rsdp_address(data);
     test_acpi_rsdp_table(data);
@@ -757,15 +736,18 @@
         }
     }
 
-    test_smbios_ep_address(data);
-    test_smbios_ep_table(data);
+    test_smbios_entry_point(data);
     test_smbios_structs(data);
 
     qtest_quit(global_qtest);
     g_free(args);
 }
 
-static void test_acpi_tcg(void)
+static uint8_t base_required_struct_types[] = {
+    0, 1, 3, 4, 16, 17, 19, 32, 127
+};
+
+static void test_acpi_piix4_tcg(void)
 {
     test_data data;
 
@@ -774,34 +756,136 @@
      */
     memset(&data, 0, sizeof(data));
     data.machine = MACHINE_PC;
+    data.required_struct_types = base_required_struct_types;
+    data.required_struct_types_len = ARRAY_SIZE(base_required_struct_types);
     test_acpi_one("-machine accel=tcg", &data);
     free_test_data(&data);
+}
+
+static void test_acpi_piix4_tcg_bridge(void)
+{
+    test_data data;
+
+    memset(&data, 0, sizeof(data));
+    data.machine = MACHINE_PC;
+    data.variant = ".bridge";
+    data.required_struct_types = base_required_struct_types;
+    data.required_struct_types_len = ARRAY_SIZE(base_required_struct_types);
+    test_acpi_one("-machine accel=tcg -device pci-bridge,chassis_nr=1", &data);
+    free_test_data(&data);
+}
+
+static void test_acpi_q35_tcg(void)
+{
+    test_data data;
 
     memset(&data, 0, sizeof(data));
     data.machine = MACHINE_Q35;
+    data.required_struct_types = base_required_struct_types;
+    data.required_struct_types_len = ARRAY_SIZE(base_required_struct_types);
     test_acpi_one("-machine q35,accel=tcg", &data);
     free_test_data(&data);
 }
 
+static void test_acpi_q35_tcg_bridge(void)
+{
+    test_data data;
+
+    memset(&data, 0, sizeof(data));
+    data.machine = MACHINE_Q35;
+    data.variant = ".bridge";
+    data.required_struct_types = base_required_struct_types;
+    data.required_struct_types_len = ARRAY_SIZE(base_required_struct_types);
+    test_acpi_one("-machine q35,accel=tcg -device pci-bridge,chassis_nr=1",
+                  &data);
+    free_test_data(&data);
+}
+
+static void test_acpi_piix4_tcg_cphp(void)
+{
+    test_data data;
+
+    memset(&data, 0, sizeof(data));
+    data.machine = MACHINE_PC;
+    data.variant = ".cphp";
+    test_acpi_one("-machine accel=tcg"
+                  " -smp 2,cores=3,sockets=2,maxcpus=6",
+                  &data);
+    free_test_data(&data);
+}
+
+static void test_acpi_q35_tcg_cphp(void)
+{
+    test_data data;
+
+    memset(&data, 0, sizeof(data));
+    data.machine = MACHINE_Q35;
+    data.variant = ".cphp";
+    test_acpi_one("-machine q35,accel=tcg"
+                  " -smp 2,cores=3,sockets=2,maxcpus=6",
+                  &data);
+    free_test_data(&data);
+}
+
+static uint8_t ipmi_required_struct_types[] = {
+    0, 1, 3, 4, 16, 17, 19, 32, 38, 127
+};
+
+static void test_acpi_q35_tcg_ipmi(void)
+{
+    test_data data;
+
+    memset(&data, 0, sizeof(data));
+    data.machine = MACHINE_Q35;
+    data.variant = ".ipmibt";
+    data.required_struct_types = ipmi_required_struct_types;
+    data.required_struct_types_len = ARRAY_SIZE(ipmi_required_struct_types);
+    test_acpi_one("-machine q35,accel=tcg -device ipmi-bmc-sim,id=bmc0"
+                  " -device isa-ipmi-bt,bmc=bmc0",
+                  &data);
+    free_test_data(&data);
+}
+
+static void test_acpi_piix4_tcg_ipmi(void)
+{
+    test_data data;
+
+    /* Supplying -machine accel argument overrides the default (qtest).
+     * This is to make guest actually run.
+     */
+    memset(&data, 0, sizeof(data));
+    data.machine = MACHINE_PC;
+    data.variant = ".ipmikcs";
+    data.required_struct_types = ipmi_required_struct_types;
+    data.required_struct_types_len = ARRAY_SIZE(ipmi_required_struct_types);
+    test_acpi_one("-machine accel=tcg -device ipmi-bmc-sim,id=bmc0"
+                  " -device isa-ipmi-kcs,irq=0,bmc=bmc0",
+                  &data);
+    free_test_data(&data);
+}
+
 int main(int argc, char *argv[])
 {
     const char *arch = qtest_get_arch();
-    FILE *f = fopen(disk, "w");
     int ret;
 
-    if (!f) {
-        fprintf(stderr, "Couldn't open \"%s\": %s", disk, strerror(errno));
-        return 1;
-    }
-    fwrite(boot_sector, 1, sizeof boot_sector, f);
-    fclose(f);
+    ret = boot_sector_init(disk);
+    if(ret)
+        return ret;
 
     g_test_init(&argc, &argv, NULL);
 
     if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
-        qtest_add_func("acpi/tcg", test_acpi_tcg);
+        qtest_add_func("acpi/piix4/tcg", test_acpi_piix4_tcg);
+        qtest_add_func("acpi/piix4/tcg/bridge", test_acpi_piix4_tcg_bridge);
+        qtest_add_func("acpi/q35/tcg", test_acpi_q35_tcg);
+        qtest_add_func("acpi/q35/tcg/bridge", test_acpi_q35_tcg_bridge);
+        qtest_add_func("acpi/piix4/tcg/ipmi", test_acpi_piix4_tcg_ipmi);
+        qtest_add_func("acpi/q35/tcg/ipmi", test_acpi_q35_tcg_ipmi);
+        qtest_add_func("acpi/piix4/tcg/cpuhp", test_acpi_piix4_tcg_cphp);
+        qtest_add_func("acpi/q35/tcg/cpuhp", test_acpi_q35_tcg_cphp);
     }
     ret = g_test_run();
-    unlink(disk);
+    boot_sector_cleanup(disk);
     return ret;
 }
diff --git a/tests/boot-order-test.c b/tests/boot-order-test.c
index 360a691..fc1e794 100644
--- a/tests/boot-order-test.c
+++ b/tests/boot-order-test.c
@@ -10,14 +10,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "libqos/fw_cfg.h"
 #include "libqtest.h"
 
-#define NO_QEMU_PROTOS
-#include "hw/nvram/fw_cfg.h"
-#undef NO_QEMU_PROTOS
+#include "hw/nvram/fw_cfg_keys.h"
 
 typedef struct {
     const char *args;
diff --git a/tests/boot-sector.c b/tests/boot-sector.c
new file mode 100644
index 0000000..3ffe298
--- /dev/null
+++ b/tests/boot-sector.c
@@ -0,0 +1,118 @@
+/*
+ * QEMU boot sector testing helpers.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *  Victor Kaplansky <victork@redhat.com>    
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "boot-sector.h"
+#include "qemu-common.h"
+#include "libqtest.h"
+
+#define LOW(x) ((x) & 0xff)
+#define HIGH(x) ((x) >> 8)
+
+#define SIGNATURE 0xdead
+#define SIGNATURE_OFFSET 0x10
+#define BOOT_SECTOR_ADDRESS 0x7c00
+
+/* Boot sector code: write SIGNATURE into memory,
+ * then halt.
+ * Q35 machine requires a minimum 0x7e000 bytes disk.
+ * (bug or feature?)
+ */
+static uint8_t boot_sector[0x7e000] = {
+    /* The first sector will be placed at RAM address 00007C00, and
+     * the BIOS transfers control to 00007C00
+     */
+
+    /* Data Segment register should be initialized, since pxe
+     * boot loader can leave it dirty.
+     */
+
+    /* 7c00: move $0000,%ax */
+    [0x00] = 0xb8,
+    [0x01] = 0x00,
+    [0x02] = 0x00,
+    /* 7c03: move %ax,%ds */
+    [0x03] = 0x8e,
+    [0x04] = 0xd8,
+
+    /* 7c05: mov $0xdead,%ax */
+    [0x05] = 0xb8,
+    [0x06] = LOW(SIGNATURE),
+    [0x07] = HIGH(SIGNATURE),
+    /* 7c08:  mov %ax,0x7c10 */
+    [0x08] = 0xa3,
+    [0x09] = LOW(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
+    [0x0a] = HIGH(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET),
+
+    /* 7c0b cli */
+    [0x0b] = 0xfa,
+    /* 7c0c: hlt */
+    [0x0c] = 0xf4,
+    /* 7c0e: jmp 0x7c07=0x7c0f-3 */
+    [0x0d] = 0xeb,
+    [0x0e] = LOW(-3),
+    /* We mov 0xdead here: set value to make debugging easier */
+    [SIGNATURE_OFFSET] = LOW(0xface),
+    [SIGNATURE_OFFSET + 1] = HIGH(0xface),
+    /* End of boot sector marker */
+    [0x1FE] = 0x55,
+    [0x1FF] = 0xAA,
+};
+
+/* Create boot disk file.  */
+int boot_sector_init(const char *fname)
+{
+    FILE *f = fopen(fname, "w");
+
+    if (!f) {
+        fprintf(stderr, "Couldn't open \"%s\": %s", fname, strerror(errno));
+        return 1;
+    }
+    fwrite(boot_sector, 1, sizeof boot_sector, f);
+    fclose(f);
+    return 0;
+}
+
+/* Loop until signature in memory is OK.  */
+void boot_sector_test(void)
+{
+    uint8_t signature_low;
+    uint8_t signature_high;
+    uint16_t signature;
+    int i;
+
+   /* Wait at most 1 minute */
+#define TEST_DELAY (1 * G_USEC_PER_SEC / 10)
+#define TEST_CYCLES MAX((60 * G_USEC_PER_SEC / TEST_DELAY), 1)
+
+    /* Poll until code has run and modified memory.  Once it has we know BIOS
+     * initialization is done.  TODO: check that IP reached the halt
+     * instruction.
+     */
+    for (i = 0; i < TEST_CYCLES; ++i) {
+        signature_low = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET);
+        signature_high = readb(BOOT_SECTOR_ADDRESS + SIGNATURE_OFFSET + 1);
+        signature = (signature_high << 8) | signature_low;
+        if (signature == SIGNATURE) {
+            break;
+        }
+        g_usleep(TEST_DELAY);
+    }
+
+    g_assert_cmphex(signature, ==, SIGNATURE);
+}
+
+/* unlink boot disk file.  */
+void boot_sector_cleanup(const char *fname)
+{
+    unlink(fname);
+}
diff --git a/tests/boot-sector.h b/tests/boot-sector.h
new file mode 100644
index 0000000..f64b477
--- /dev/null
+++ b/tests/boot-sector.h
@@ -0,0 +1,26 @@
+/*
+ * QEMU boot sector testing helpers.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *  Victor Kaplansky <victork@redhat.com>    
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef TEST_BOOT_SECTOR_H
+#define TEST_BOOT_SECTOR_H
+
+/* Create boot disk file.  */
+int boot_sector_init(const char *fname);
+
+/* Loop until signature in memory is OK.  */
+void boot_sector_test(void);
+
+/* unlink boot disk file.  */
+void boot_sector_cleanup(const char *fname);
+
+#endif /* TEST_BOOT_SECTOR_H */
diff --git a/tests/check-block.sh b/tests/check-block.sh
index b9d9c6a..a37797a 100755
--- a/tests/check-block.sh
+++ b/tests/check-block.sh
@@ -9,7 +9,7 @@
     exit 1
 fi
 
-cd $SRC_PATH/tests/qemu-iotests
+cd tests/qemu-iotests
 
 ret=0
 ./check -T -nocache -raw || ret=1
diff --git a/tests/check-qdict.c b/tests/check-qdict.c
index a9296f0..42da1e6 100644
--- a/tests/check-qdict.c
+++ b/tests/check-qdict.c
@@ -9,7 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qdict.h"
@@ -152,6 +152,28 @@
     QDECREF(tests_dict);
 }
 
+static void qdict_defaults_test(void)
+{
+    QDict *dict, *copy;
+
+    dict = qdict_new();
+    copy = qdict_new();
+
+    qdict_set_default_str(dict, "foo", "abc");
+    qdict_set_default_str(dict, "foo", "def");
+    g_assert_cmpstr(qdict_get_str(dict, "foo"), ==, "abc");
+    qdict_set_default_str(dict, "bar", "ghi");
+
+    qdict_copy_default(copy, dict, "foo");
+    g_assert_cmpstr(qdict_get_str(copy, "foo"), ==, "abc");
+    qdict_set_default_str(copy, "bar", "xyz");
+    qdict_copy_default(copy, dict, "bar");
+    g_assert_cmpstr(qdict_get_str(copy, "bar"), ==, "xyz");
+
+    QDECREF(copy);
+    QDECREF(dict);
+}
+
 static void qdict_haskey_not_test(void)
 {
     QDict *tests_dict = qdict_new();
@@ -444,6 +466,49 @@
     QDECREF(test_dict);
 }
 
+static void qdict_array_entries_test(void)
+{
+    QDict *dict = qdict_new();
+
+    g_assert_cmpint(qdict_array_entries(dict, "foo."), ==, 0);
+
+    qdict_put(dict, "bar", qint_from_int(0));
+    qdict_put(dict, "baz.0", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, "foo."), ==, 0);
+
+    qdict_put(dict, "foo.1", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, "foo."), ==, -EINVAL);
+    qdict_put(dict, "foo.0", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, "foo."), ==, 2);
+    qdict_put(dict, "foo.bar", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, "foo."), ==, -EINVAL);
+    qdict_del(dict, "foo.bar");
+
+    qdict_put(dict, "foo.2.a", qint_from_int(0));
+    qdict_put(dict, "foo.2.b", qint_from_int(0));
+    qdict_put(dict, "foo.2.c", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, "foo."), ==, 3);
+    g_assert_cmpint(qdict_array_entries(dict, ""), ==, -EINVAL);
+
+    QDECREF(dict);
+
+    dict = qdict_new();
+    qdict_put(dict, "1", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, ""), ==, -EINVAL);
+    qdict_put(dict, "0", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, ""), ==, 2);
+    qdict_put(dict, "bar", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, ""), ==, -EINVAL);
+    qdict_del(dict, "bar");
+
+    qdict_put(dict, "2.a", qint_from_int(0));
+    qdict_put(dict, "2.b", qint_from_int(0));
+    qdict_put(dict, "2.c", qint_from_int(0));
+    g_assert_cmpint(qdict_array_entries(dict, ""), ==, 3);
+
+    QDECREF(dict);
+}
+
 static void qdict_join_test(void)
 {
     QDict *dict1, *dict2;
@@ -663,6 +728,7 @@
     g_test_add_func("/public/get_try_int", qdict_get_try_int_test);
     g_test_add_func("/public/get_str", qdict_get_str_test);
     g_test_add_func("/public/get_try_str", qdict_get_try_str_test);
+    g_test_add_func("/public/defaults", qdict_defaults_test);
     g_test_add_func("/public/haskey_not", qdict_haskey_not_test);
     g_test_add_func("/public/haskey", qdict_haskey_test);
     g_test_add_func("/public/del", qdict_del_test);
@@ -670,6 +736,7 @@
     g_test_add_func("/public/iterapi", qdict_iterapi_test);
     g_test_add_func("/public/flatten", qdict_flatten_test);
     g_test_add_func("/public/array_split", qdict_array_split_test);
+    g_test_add_func("/public/array_entries", qdict_array_entries_test);
     g_test_add_func("/public/join", qdict_join_test);
 
     g_test_add_func("/errors/put_exists", qdict_put_exists_test);
diff --git a/tests/check-qfloat.c b/tests/check-qfloat.c
index 6404ac8..1da2cda 100644
--- a/tests/check-qfloat.c
+++ b/tests/check-qfloat.c
@@ -10,7 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qapi/qmp/qfloat.h"
 #include "qemu-common.h"
diff --git a/tests/check-qint.c b/tests/check-qint.c
index 8686884..b6e4555 100644
--- a/tests/check-qint.c
+++ b/tests/check-qint.c
@@ -9,7 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qapi/qmp/qint.h"
 #include "qemu-common.h"
diff --git a/tests/check-qjson.c b/tests/check-qjson.c
index 95497a0..8595574 100644
--- a/tests/check-qjson.c
+++ b/tests/check-qjson.c
@@ -1,6 +1,6 @@
 /*
  * Copyright IBM, Corp. 2009
- * Copyright (c) 2013 Red Hat Inc.
+ * Copyright (c) 2013, 2015 Red Hat Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
@@ -10,16 +10,10 @@
  * See the COPYING.LIB file in the top-level directory.
  *
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
-#include "qapi/qmp/qstring.h"
-#include "qapi/qmp/qint.h"
-#include "qapi/qmp/qdict.h"
-#include "qapi/qmp/qlist.h"
-#include "qapi/qmp/qfloat.h"
-#include "qapi/qmp/qbool.h"
+#include "qapi/qmp/types.h"
 #include "qapi/qmp/qjson.h"
-
 #include "qemu-common.h"
 
 static void escaped_string(void)
@@ -1005,6 +999,7 @@
 {
     QObject *obj;
     QBool *qbool;
+    QObject *null;
     QString *str;
 
     obj = qobject_from_json("true");
@@ -1012,7 +1007,7 @@
     g_assert(qobject_type(obj) == QTYPE_QBOOL);
 
     qbool = qobject_to_qbool(obj);
-    g_assert(qbool_get_int(qbool) != 0);
+    g_assert(qbool_get_bool(qbool) == true);
 
     str = qobject_to_json(obj);
     g_assert(strcmp(qstring_get_str(str), "true") == 0);
@@ -1025,7 +1020,7 @@
     g_assert(qobject_type(obj) == QTYPE_QBOOL);
 
     qbool = qobject_to_qbool(obj);
-    g_assert(qbool_get_int(qbool) == 0);
+    g_assert(qbool_get_bool(qbool) == false);
 
     str = qobject_to_json(obj);
     g_assert(strcmp(qstring_get_str(str), "false") == 0);
@@ -1038,18 +1033,29 @@
     g_assert(qobject_type(obj) == QTYPE_QBOOL);
 
     qbool = qobject_to_qbool(obj);
-    g_assert(qbool_get_int(qbool) == 0);
+    g_assert(qbool_get_bool(qbool) == false);
 
     QDECREF(qbool);
-    
-    obj = qobject_from_jsonf("%i", true);
+
+    /* Test that non-zero values other than 1 get collapsed to true */
+    obj = qobject_from_jsonf("%i", 2);
     g_assert(obj != NULL);
     g_assert(qobject_type(obj) == QTYPE_QBOOL);
 
     qbool = qobject_to_qbool(obj);
-    g_assert(qbool_get_int(qbool) != 0);
+    g_assert(qbool_get_bool(qbool) == true);
 
     QDECREF(qbool);
+
+    obj = qobject_from_json("null");
+    g_assert(obj != NULL);
+    g_assert(qobject_type(obj) == QTYPE_QNULL);
+
+    null = qnull();
+    g_assert(null == obj);
+
+    qobject_decref(obj);
+    qobject_decref(null);
 }
 
 typedef struct LiteralQDictEntry LiteralQDictEntry;
@@ -1472,6 +1478,30 @@
     g_assert(obj == NULL);
 }
 
+static char *make_nest(char *buf, size_t cnt)
+{
+    memset(buf, '[', cnt - 1);
+    buf[cnt - 1] = '{';
+    buf[cnt] = '}';
+    memset(buf + cnt + 1, ']', cnt - 1);
+    buf[2 * cnt] = 0;
+    return buf;
+}
+
+static void limits_nesting(void)
+{
+    enum { max_nesting = 1024 }; /* see qobject/json-streamer.c */
+    char buf[2 * (max_nesting + 1) + 1];
+    QObject *obj;
+
+    obj = qobject_from_json(make_nest(buf, max_nesting));
+    g_assert(obj != NULL);
+    qobject_decref(obj);
+
+    obj = qobject_from_json(make_nest(buf, max_nesting + 1));
+    g_assert(obj == NULL);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -1507,6 +1537,7 @@
     g_test_add_func("/errors/invalid_array_comma", invalid_array_comma);
     g_test_add_func("/errors/invalid_dict_comma", invalid_dict_comma);
     g_test_add_func("/errors/unterminated/literal", unterminated_literal);
+    g_test_add_func("/errors/limits/nesting", limits_nesting);
 
     return g_test_run();
 }
diff --git a/tests/check-qlist.c b/tests/check-qlist.c
index b9c05d4..e16da5e 100644
--- a/tests/check-qlist.c
+++ b/tests/check-qlist.c
@@ -9,7 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qapi/qmp/qint.h"
 #include "qapi/qmp/qlist.h"
diff --git a/tests/check-qnull.c b/tests/check-qnull.c
new file mode 100644
index 0000000..dc906b1
--- /dev/null
+++ b/tests/check-qnull.c
@@ -0,0 +1,73 @@
+/*
+ * QNull unit-tests.
+ *
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+
+#include "qapi/qmp/qobject.h"
+#include "qemu-common.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi/error.h"
+
+/*
+ * Public Interface test-cases
+ *
+ * (with some violations to access 'private' data)
+ */
+
+static void qnull_ref_test(void)
+{
+    QObject *obj;
+
+    g_assert(qnull_.refcnt == 1);
+    obj = qnull();
+    g_assert(obj);
+    g_assert(obj == &qnull_);
+    g_assert(qnull_.refcnt == 2);
+    g_assert(qobject_type(obj) == QTYPE_QNULL);
+    qobject_decref(obj);
+    g_assert(qnull_.refcnt == 1);
+}
+
+static void qnull_visit_test(void)
+{
+    QObject *obj;
+    Visitor *v;
+
+    /*
+     * Most tests of interactions between QObject and visitors are in
+     * test-qmp-*-visitor; but these tests live here because they
+     * depend on layering violations to check qnull_ refcnt.
+     */
+
+    g_assert(qnull_.refcnt == 1);
+    obj = qnull();
+    v = qmp_input_visitor_new(obj, true);
+    qobject_decref(obj);
+    visit_type_null(v, NULL, &error_abort);
+    visit_free(v);
+
+    v = qmp_output_visitor_new(&obj);
+    visit_type_null(v, NULL, &error_abort);
+    visit_complete(v, &obj);
+    g_assert(obj == &qnull_);
+    qobject_decref(obj);
+    visit_free(v);
+
+    g_assert(qnull_.refcnt == 1);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/public/qnull_ref", qnull_ref_test);
+    g_test_add_func("/public/qnull_visit", qnull_visit_test);
+
+    return g_test_run();
+}
diff --git a/tests/check-qom-interface.c b/tests/check-qom-interface.c
index f06380e..719ddcf 100644
--- a/tests/check-qom-interface.c
+++ b/tests/check-qom-interface.c
@@ -9,7 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qom/object.h"
 #include "qemu/module.h"
diff --git a/tests/check-qom-proplist.c b/tests/check-qom-proplist.c
new file mode 100644
index 0000000..42defe7
--- /dev/null
+++ b/tests/check-qom-proplist.c
@@ -0,0 +1,518 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "qapi/error.h"
+#include "qom/object.h"
+#include "qemu/module.h"
+
+
+#define TYPE_DUMMY "qemu-dummy"
+
+typedef struct DummyObject DummyObject;
+typedef struct DummyObjectClass DummyObjectClass;
+
+#define DUMMY_OBJECT(obj)                               \
+    OBJECT_CHECK(DummyObject, (obj), TYPE_DUMMY)
+
+typedef enum DummyAnimal DummyAnimal;
+
+enum DummyAnimal {
+    DUMMY_FROG,
+    DUMMY_ALLIGATOR,
+    DUMMY_PLATYPUS,
+
+    DUMMY_LAST,
+};
+
+static const char *const dummy_animal_map[DUMMY_LAST + 1] = {
+    [DUMMY_FROG] = "frog",
+    [DUMMY_ALLIGATOR] = "alligator",
+    [DUMMY_PLATYPUS] = "platypus",
+    [DUMMY_LAST] = NULL,
+};
+
+struct DummyObject {
+    Object parent_obj;
+
+    bool bv;
+    DummyAnimal av;
+    char *sv;
+};
+
+struct DummyObjectClass {
+    ObjectClass parent_class;
+};
+
+
+static void dummy_set_bv(Object *obj,
+                         bool value,
+                         Error **errp)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    dobj->bv = value;
+}
+
+static bool dummy_get_bv(Object *obj,
+                         Error **errp)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    return dobj->bv;
+}
+
+
+static void dummy_set_av(Object *obj,
+                         int value,
+                         Error **errp)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    dobj->av = value;
+}
+
+static int dummy_get_av(Object *obj,
+                        Error **errp)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    return dobj->av;
+}
+
+
+static void dummy_set_sv(Object *obj,
+                         const char *value,
+                         Error **errp)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    g_free(dobj->sv);
+    dobj->sv = g_strdup(value);
+}
+
+static char *dummy_get_sv(Object *obj,
+                          Error **errp)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    return g_strdup(dobj->sv);
+}
+
+
+static void dummy_init(Object *obj)
+{
+    object_property_add_bool(obj, "bv",
+                             dummy_get_bv,
+                             dummy_set_bv,
+                             NULL);
+}
+
+
+static void dummy_class_init(ObjectClass *cls, void *data)
+{
+    object_class_property_add_bool(cls, "bv",
+                                   dummy_get_bv,
+                                   dummy_set_bv,
+                                   NULL);
+    object_class_property_add_str(cls, "sv",
+                                  dummy_get_sv,
+                                  dummy_set_sv,
+                                  NULL);
+    object_class_property_add_enum(cls, "av",
+                                   "DummyAnimal",
+                                   dummy_animal_map,
+                                   dummy_get_av,
+                                   dummy_set_av,
+                                   NULL);
+}
+
+
+static void dummy_finalize(Object *obj)
+{
+    DummyObject *dobj = DUMMY_OBJECT(obj);
+
+    g_free(dobj->sv);
+}
+
+
+static const TypeInfo dummy_info = {
+    .name          = TYPE_DUMMY,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyObject),
+    .instance_init = dummy_init,
+    .instance_finalize = dummy_finalize,
+    .class_size = sizeof(DummyObjectClass),
+    .class_init = dummy_class_init,
+};
+
+
+/*
+ * The following 3 object classes are used to
+ * simulate the kind of relationships seen in
+ * qdev, which result in complex object
+ * property destruction ordering.
+ *
+ * DummyDev has a 'bus' child to a DummyBus
+ * DummyBus has a 'backend' child to a DummyBackend
+ * DummyDev has a 'backend' link to DummyBackend
+ *
+ * When DummyDev is finalized, it unparents the
+ * DummyBackend, which unparents the DummyDev
+ * which deletes the 'backend' link from DummyDev
+ * to DummyBackend. This illustrates that the
+ * object_property_del_all() method needs to
+ * cope with the list of properties being changed
+ * while it iterates over them.
+ */
+typedef struct DummyDev DummyDev;
+typedef struct DummyDevClass DummyDevClass;
+typedef struct DummyBus DummyBus;
+typedef struct DummyBusClass DummyBusClass;
+typedef struct DummyBackend DummyBackend;
+typedef struct DummyBackendClass DummyBackendClass;
+
+#define TYPE_DUMMY_DEV "qemu-dummy-dev"
+#define TYPE_DUMMY_BUS "qemu-dummy-bus"
+#define TYPE_DUMMY_BACKEND "qemu-dummy-backend"
+
+#define DUMMY_DEV(obj)                               \
+    OBJECT_CHECK(DummyDev, (obj), TYPE_DUMMY_DEV)
+#define DUMMY_BUS(obj)                               \
+    OBJECT_CHECK(DummyBus, (obj), TYPE_DUMMY_BUS)
+#define DUMMY_BACKEND(obj)                               \
+    OBJECT_CHECK(DummyBackend, (obj), TYPE_DUMMY_BACKEND)
+
+struct DummyDev {
+    Object parent_obj;
+
+    DummyBus *bus;
+};
+
+struct DummyDevClass {
+    ObjectClass parent_class;
+};
+
+struct DummyBus {
+    Object parent_obj;
+
+    DummyBackend *backend;
+};
+
+struct DummyBusClass {
+    ObjectClass parent_class;
+};
+
+struct DummyBackend {
+    Object parent_obj;
+};
+
+struct DummyBackendClass {
+    ObjectClass parent_class;
+};
+
+
+static void dummy_dev_init(Object *obj)
+{
+    DummyDev *dev = DUMMY_DEV(obj);
+    DummyBus *bus = DUMMY_BUS(object_new(TYPE_DUMMY_BUS));
+    DummyBackend *backend = DUMMY_BACKEND(object_new(TYPE_DUMMY_BACKEND));
+
+    object_property_add_child(obj, "bus", OBJECT(bus), NULL);
+    dev->bus = bus;
+    object_property_add_child(OBJECT(bus), "backend", OBJECT(backend), NULL);
+    bus->backend = backend;
+
+    object_property_add_link(obj, "backend", TYPE_DUMMY_BACKEND,
+                             (Object **)&bus->backend, NULL, 0, NULL);
+}
+
+static void dummy_dev_unparent(Object *obj)
+{
+    DummyDev *dev = DUMMY_DEV(obj);
+    object_unparent(OBJECT(dev->bus));
+}
+
+static void dummy_dev_class_init(ObjectClass *klass, void *opaque)
+{
+    klass->unparent = dummy_dev_unparent;
+}
+
+
+static void dummy_bus_init(Object *obj)
+{
+}
+
+static void dummy_bus_unparent(Object *obj)
+{
+    DummyBus *bus = DUMMY_BUS(obj);
+    object_property_del(obj->parent, "backend", NULL);
+    object_unparent(OBJECT(bus->backend));
+}
+
+static void dummy_bus_class_init(ObjectClass *klass, void *opaque)
+{
+    klass->unparent = dummy_bus_unparent;
+}
+
+static void dummy_backend_init(Object *obj)
+{
+}
+
+
+static const TypeInfo dummy_dev_info = {
+    .name          = TYPE_DUMMY_DEV,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyDev),
+    .instance_init = dummy_dev_init,
+    .class_size = sizeof(DummyDevClass),
+    .class_init = dummy_dev_class_init,
+};
+
+static const TypeInfo dummy_bus_info = {
+    .name          = TYPE_DUMMY_BUS,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyBus),
+    .instance_init = dummy_bus_init,
+    .class_size = sizeof(DummyBusClass),
+    .class_init = dummy_bus_class_init,
+};
+
+static const TypeInfo dummy_backend_info = {
+    .name          = TYPE_DUMMY_BACKEND,
+    .parent        = TYPE_OBJECT,
+    .instance_size = sizeof(DummyBackend),
+    .instance_init = dummy_backend_init,
+    .class_size = sizeof(DummyBackendClass),
+};
+
+
+
+static void test_dummy_createv(void)
+{
+    Error *err = NULL;
+    Object *parent = object_get_objects_root();
+    DummyObject *dobj = DUMMY_OBJECT(
+        object_new_with_props(TYPE_DUMMY,
+                              parent,
+                              "dummy0",
+                              &err,
+                              "bv", "yes",
+                              "sv", "Hiss hiss hiss",
+                              "av", "platypus",
+                              NULL));
+
+    g_assert(err == NULL);
+    g_assert_cmpstr(dobj->sv, ==, "Hiss hiss hiss");
+    g_assert(dobj->bv == true);
+    g_assert(dobj->av == DUMMY_PLATYPUS);
+
+    g_assert(object_resolve_path_component(parent, "dummy0")
+             == OBJECT(dobj));
+
+    object_unparent(OBJECT(dobj));
+}
+
+
+static Object *new_helper(Error **errp,
+                          Object *parent,
+                          ...)
+{
+    va_list vargs;
+    Object *obj;
+
+    va_start(vargs, parent);
+    obj = object_new_with_propv(TYPE_DUMMY,
+                                parent,
+                                "dummy0",
+                                errp,
+                                vargs);
+    va_end(vargs);
+    return obj;
+}
+
+static void test_dummy_createlist(void)
+{
+    Error *err = NULL;
+    Object *parent = object_get_objects_root();
+    DummyObject *dobj = DUMMY_OBJECT(
+        new_helper(&err,
+                   parent,
+                   "bv", "yes",
+                   "sv", "Hiss hiss hiss",
+                   "av", "platypus",
+                   NULL));
+
+    g_assert(err == NULL);
+    g_assert_cmpstr(dobj->sv, ==, "Hiss hiss hiss");
+    g_assert(dobj->bv == true);
+    g_assert(dobj->av == DUMMY_PLATYPUS);
+
+    g_assert(object_resolve_path_component(parent, "dummy0")
+             == OBJECT(dobj));
+
+    object_unparent(OBJECT(dobj));
+}
+
+static void test_dummy_badenum(void)
+{
+    Error *err = NULL;
+    Object *parent = object_get_objects_root();
+    Object *dobj =
+        object_new_with_props(TYPE_DUMMY,
+                              parent,
+                              "dummy0",
+                              &err,
+                              "bv", "yes",
+                              "sv", "Hiss hiss hiss",
+                              "av", "yeti",
+                              NULL);
+
+    g_assert(dobj == NULL);
+    g_assert(err != NULL);
+    g_assert_cmpstr(error_get_pretty(err), ==,
+                    "Invalid parameter 'yeti'");
+
+    g_assert(object_resolve_path_component(parent, "dummy0")
+             == NULL);
+
+    error_free(err);
+}
+
+
+static void test_dummy_getenum(void)
+{
+    Error *err = NULL;
+    int val;
+    Object *parent = object_get_objects_root();
+    DummyObject *dobj = DUMMY_OBJECT(
+        object_new_with_props(TYPE_DUMMY,
+                         parent,
+                         "dummy0",
+                         &err,
+                         "av", "platypus",
+                         NULL));
+
+    g_assert(err == NULL);
+    g_assert(dobj->av == DUMMY_PLATYPUS);
+
+    val = object_property_get_enum(OBJECT(dobj),
+                                   "av",
+                                   "DummyAnimal",
+                                   &err);
+    g_assert(err == NULL);
+    g_assert(val == DUMMY_PLATYPUS);
+
+    /* A bad enum type name */
+    val = object_property_get_enum(OBJECT(dobj),
+                                   "av",
+                                   "BadAnimal",
+                                   &err);
+    g_assert(err != NULL);
+    error_free(err);
+    err = NULL;
+
+    /* A non-enum property name */
+    val = object_property_get_enum(OBJECT(dobj),
+                                   "iv",
+                                   "DummyAnimal",
+                                   &err);
+    g_assert(err != NULL);
+    error_free(err);
+
+    object_unparent(OBJECT(dobj));
+}
+
+
+static void test_dummy_iterator(void)
+{
+    Object *parent = object_get_objects_root();
+    DummyObject *dobj = DUMMY_OBJECT(
+        object_new_with_props(TYPE_DUMMY,
+                              parent,
+                              "dummy0",
+                              &error_abort,
+                              "bv", "yes",
+                              "sv", "Hiss hiss hiss",
+                              "av", "platypus",
+                              NULL));
+
+    ObjectProperty *prop;
+    ObjectPropertyIterator iter;
+    bool seenbv = false, seensv = false, seenav = false, seentype;
+
+    object_property_iter_init(&iter, OBJECT(dobj));
+    while ((prop = object_property_iter_next(&iter))) {
+        if (g_str_equal(prop->name, "bv")) {
+            seenbv = true;
+        } else if (g_str_equal(prop->name, "sv")) {
+            seensv = true;
+        } else if (g_str_equal(prop->name, "av")) {
+            seenav = true;
+        } else if (g_str_equal(prop->name, "type")) {
+            /* This prop comes from the base Object class */
+            seentype = true;
+        } else {
+            g_printerr("Found prop '%s'\n", prop->name);
+            g_assert_not_reached();
+        }
+    }
+    g_assert(seenbv);
+    g_assert(seenav);
+    g_assert(seensv);
+    g_assert(seentype);
+
+    object_unparent(OBJECT(dobj));
+}
+
+
+static void test_dummy_delchild(void)
+{
+    Object *parent = object_get_objects_root();
+    DummyDev *dev = DUMMY_DEV(
+        object_new_with_props(TYPE_DUMMY_DEV,
+                              parent,
+                              "dev0",
+                              &error_abort,
+                              NULL));
+
+    object_unparent(OBJECT(dev));
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    module_call_init(MODULE_INIT_QOM);
+    type_register_static(&dummy_info);
+    type_register_static(&dummy_dev_info);
+    type_register_static(&dummy_bus_info);
+    type_register_static(&dummy_backend_info);
+
+    g_test_add_func("/qom/proplist/createlist", test_dummy_createlist);
+    g_test_add_func("/qom/proplist/createv", test_dummy_createv);
+    g_test_add_func("/qom/proplist/badenum", test_dummy_badenum);
+    g_test_add_func("/qom/proplist/getenum", test_dummy_getenum);
+    g_test_add_func("/qom/proplist/iterator", test_dummy_iterator);
+    g_test_add_func("/qom/proplist/delchild", test_dummy_delchild);
+
+    return g_test_run();
+}
diff --git a/tests/check-qstring.c b/tests/check-qstring.c
index 95dc9e3..239e9d9 100644
--- a/tests/check-qstring.c
+++ b/tests/check-qstring.c
@@ -9,7 +9,7 @@
  * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qapi/qmp/qstring.h"
 #include "qemu-common.h"
diff --git a/tests/crypto-tls-x509-helpers.c b/tests/crypto-tls-x509-helpers.c
new file mode 100644
index 0000000..64073d3
--- /dev/null
+++ b/tests/crypto-tls-x509-helpers.c
@@ -0,0 +1,485 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "qemu/sockets.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+/*
+ * This stores some static data that is needed when
+ * encoding extensions in the x509 certs
+ */
+ASN1_TYPE pkix_asn1;
+
+/*
+ * To avoid consuming random entropy to generate keys,
+ * here's one we prepared earlier :-)
+ */
+gnutls_x509_privkey_t privkey;
+# define PRIVATE_KEY                                              \
+    "-----BEGIN PRIVATE KEY-----\n"                               \
+    "MIICdQIBADANBgkqhkiG9w0BAQEFAASCAl8wggJbAgEAAoGBALVcr\n"     \
+    "BL40Tm6yq88FBhJNw1aaoCjmtg0l4dWQZ/e9Fimx4ARxFpT+ji4FE\n"     \
+    "Cgl9s/SGqC+1nvlkm9ViSo0j7MKDbnDB+VRHDvMAzQhA2X7e8M0n9\n"     \
+    "rPolUY2lIVC83q0BBaOBkCj2RSmT2xTEbbC2xLukSrg2WP/ihVOxc\n"     \
+    "kXRuyFtzAgMBAAECgYB7slBexDwXrtItAMIH6m/U+LUpNe0Xx48OL\n"     \
+    "IOn4a4whNgO/o84uIwygUK27ZGFZT0kAGAk8CdF9hA6ArcbQ62s1H\n"     \
+    "myxrUbF9/mrLsQw1NEqpuUk9Ay2Tx5U/wPx35S3W/X2AvR/ZpTnCn\n"     \
+    "2q/7ym9fyiSoj86drD7BTvmKXlOnOwQJBAPOFMp4mMa9NGpGuEssO\n"     \
+    "m3Uwbp6lhcP0cA9MK+iOmeANpoKWfBdk5O34VbmeXnGYWEkrnX+9J\n"     \
+    "bM4wVhnnBWtgBMCQQC+qAEmvwcfhauERKYznMVUVksyeuhxhCe7EK\n"     \
+    "mPh+U2+g0WwdKvGDgO0PPt1gq0ILEjspMDeMHVdTwkaVBo/uMhAkA\n"     \
+    "Z5SsZyCP2aTOPFDypXRdI4eqRcjaEPOUBq27r3uYb/jeboVb2weLa\n"     \
+    "L1MmVuHiIHoa5clswPdWVI2y0em2IGoDAkBPSp/v9VKJEZabk9Frd\n"     \
+    "a+7u4fanrM9QrEjY3KhduslSilXZZSxrWjjAJPyPiqFb3M8XXA26W\n"     \
+    "nz1KYGnqYKhLcBAkB7dt57n9xfrhDpuyVEv+Uv1D3VVAhZlsaZ5Pp\n"     \
+    "dcrhrkJn2sa/+O8OKvdrPSeeu/N5WwYhJf61+CPoenMp7IFci\n"         \
+    "-----END PRIVATE KEY-----\n"
+
+/*
+ * This loads the private key we defined earlier
+ */
+static gnutls_x509_privkey_t test_tls_load_key(void)
+{
+    gnutls_x509_privkey_t key;
+    const gnutls_datum_t data = { (unsigned char *)PRIVATE_KEY,
+                                  strlen(PRIVATE_KEY) };
+    int err;
+
+    err = gnutls_x509_privkey_init(&key);
+    if (err < 0) {
+        g_critical("Failed to init key %s", gnutls_strerror(err));
+        abort();
+    }
+
+    err = gnutls_x509_privkey_import(key, &data,
+                                     GNUTLS_X509_FMT_PEM);
+    if (err < 0) {
+        if (err != GNUTLS_E_BASE64_UNEXPECTED_HEADER_ERROR &&
+            err != GNUTLS_E_REQUESTED_DATA_NOT_AVAILABLE) {
+            g_critical("Failed to import key %s", gnutls_strerror(err));
+            abort();
+        }
+
+        err = gnutls_x509_privkey_import_pkcs8(
+            key, &data, GNUTLS_X509_FMT_PEM, NULL, 0);
+        if (err < 0) {
+            g_critical("Failed to import PKCS8 key %s", gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    return key;
+}
+
+
+void test_tls_init(const char *keyfile)
+{
+    gnutls_global_init();
+
+    if (asn1_array2tree(pkix_asn1_tab, &pkix_asn1, NULL) != ASN1_SUCCESS) {
+        abort();
+    }
+
+    privkey = test_tls_load_key();
+    if (!g_file_set_contents(keyfile, PRIVATE_KEY, -1, NULL)) {
+        abort();
+    }
+}
+
+
+void test_tls_cleanup(const char *keyfile)
+{
+    asn1_delete_structure(&pkix_asn1);
+    unlink(keyfile);
+}
+
+/*
+ * Turns an ASN1 object into a DER encoded byte array
+ */
+static void test_tls_der_encode(ASN1_TYPE src,
+                                const char *src_name,
+                                gnutls_datum_t *res)
+{
+  int size;
+  char *data = NULL;
+
+  size = 0;
+  asn1_der_coding(src, src_name, NULL, &size, NULL);
+
+  data = g_new0(char, size);
+
+  asn1_der_coding(src, src_name, data, &size, NULL);
+
+  res->data = (unsigned char *)data;
+  res->size = size;
+}
+
+
+static void
+test_tls_get_ipaddr(const char *addrstr,
+                    char **data,
+                    int *datalen)
+{
+    struct addrinfo *res;
+    struct addrinfo hints;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_flags = AI_NUMERICHOST;
+    g_assert(getaddrinfo(addrstr, NULL, &hints, &res) == 0);
+
+    *datalen = res->ai_addrlen;
+    *data = g_new(char, *datalen);
+    memcpy(*data, res->ai_addr, *datalen);
+    freeaddrinfo(res);
+}
+
+/*
+ * This is a fairly lame x509 certificate generator.
+ *
+ * Do not copy/use this code for generating real certificates
+ * since it leaves out many things that you would want in
+ * certificates for real world usage.
+ *
+ * This is good enough only for doing tests of the QEMU
+ * TLS certificate code
+ */
+void
+test_tls_generate_cert(QCryptoTLSTestCertReq *req,
+                       gnutls_x509_crt_t ca)
+{
+    gnutls_x509_crt_t crt;
+    int err;
+    static char buffer[1024 * 1024];
+    size_t size = sizeof(buffer);
+    char serial[5] = { 1, 2, 3, 4, 0 };
+    gnutls_datum_t der;
+    time_t start = time(NULL) + (60 * 60 * req->start_offset);
+    time_t expire = time(NULL) + (60 * 60 * (req->expire_offset
+                                             ? req->expire_offset : 24));
+
+    /*
+     * Prepare our new certificate object
+     */
+    err = gnutls_x509_crt_init(&crt);
+    if (err < 0) {
+        g_critical("Failed to initialize certificate %s", gnutls_strerror(err));
+        abort();
+    }
+    err = gnutls_x509_crt_set_key(crt, privkey);
+    if (err < 0) {
+        g_critical("Failed to set certificate key %s", gnutls_strerror(err));
+        abort();
+    }
+
+    /*
+     * A v3 certificate is required in order to be able
+     * set any of the basic constraints, key purpose and
+     * key usage data
+     */
+    gnutls_x509_crt_set_version(crt, 3);
+
+    if (req->country) {
+        err = gnutls_x509_crt_set_dn_by_oid(
+            crt, GNUTLS_OID_X520_COUNTRY_NAME, 0,
+            req->country, strlen(req->country));
+        if (err < 0) {
+            g_critical("Failed to set certificate country name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+    if (req->cn) {
+        err = gnutls_x509_crt_set_dn_by_oid(
+            crt, GNUTLS_OID_X520_COMMON_NAME, 0,
+            req->cn, strlen(req->cn));
+        if (err < 0) {
+            g_critical("Failed to set certificate common name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    /*
+     * Setup the subject altnames, which are used
+     * for hostname checks in live sessions
+     */
+    if (req->altname1) {
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_DNSNAME,
+            req->altname1,
+            strlen(req->altname1),
+            GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+    if (req->altname2) {
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_DNSNAME,
+            req->altname2,
+            strlen(req->altname2),
+            GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate %s alt name",
+                       gnutls_strerror(err));
+            abort();
+        }
+    }
+
+    /*
+     * IP address need to be put into the cert in their
+     * raw byte form, not strings, hence this is a little
+     * more complicated
+     */
+    if (req->ipaddr1) {
+        char *data;
+        int len;
+
+        test_tls_get_ipaddr(req->ipaddr1, &data, &len);
+
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_IPADDRESS,
+            data, len, GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+        g_free(data);
+    }
+    if (req->ipaddr2) {
+        char *data;
+        int len;
+
+        test_tls_get_ipaddr(req->ipaddr2, &data, &len);
+
+        err = gnutls_x509_crt_set_subject_alt_name(
+            crt, GNUTLS_SAN_IPADDRESS,
+            data, len, GNUTLS_FSAN_APPEND);
+        if (err < 0) {
+            g_critical("Failed to set certificate alt name %s",
+                       gnutls_strerror(err));
+            abort();
+        }
+        g_free(data);
+    }
+
+
+    /*
+     * Basic constraints are used to decide if the cert
+     * is for a CA or not. We can't use the convenient
+     * gnutls API for setting this, since it hardcodes
+     * the 'critical' field which we want control over
+     */
+    if (req->basicConstraintsEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+
+        asn1_create_element(pkix_asn1, "PKIX1.BasicConstraints", &ext);
+        asn1_write_value(ext, "cA",
+                         req->basicConstraintsIsCA ? "TRUE" : "FALSE", 1);
+        asn1_write_value(ext, "pathLenConstraint", NULL, 0);
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.19",
+            der.data, der.size,
+            req->basicConstraintsCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate basic constraints %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Next up the key usage extension. Again we can't
+     * use the gnutls API since it hardcodes the extension
+     * to be 'critical'
+     */
+    if (req->keyUsageEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+        char str[2];
+
+        str[0] = req->keyUsageValue & 0xff;
+        str[1] = (req->keyUsageValue >> 8) & 0xff;
+
+        asn1_create_element(pkix_asn1, "PKIX1.KeyUsage", &ext);
+        asn1_write_value(ext, "", str, 9);
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.15",
+            der.data, der.size,
+            req->keyUsageCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate key usage %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Finally the key purpose extension. This time
+     * gnutls has the opposite problem, always hardcoding
+     * it to be non-critical. So once again we have to
+     * set this the hard way building up ASN1 data ourselves
+     */
+    if (req->keyPurposeEnable) {
+        ASN1_TYPE ext = ASN1_TYPE_EMPTY;
+
+        asn1_create_element(pkix_asn1, "PKIX1.ExtKeyUsageSyntax", &ext);
+        if (req->keyPurposeOID1) {
+            asn1_write_value(ext, "", "NEW", 1);
+            asn1_write_value(ext, "?LAST", req->keyPurposeOID1, 1);
+        }
+        if (req->keyPurposeOID2) {
+            asn1_write_value(ext, "", "NEW", 1);
+            asn1_write_value(ext, "?LAST", req->keyPurposeOID2, 1);
+        }
+        test_tls_der_encode(ext, "", &der);
+        err = gnutls_x509_crt_set_extension_by_oid(
+            crt, "2.5.29.37",
+            der.data, der.size,
+            req->keyPurposeCritical);
+        if (err < 0) {
+            g_critical("Failed to set certificate key purpose %s",
+                       gnutls_strerror(err));
+            g_free(der.data);
+            abort();
+        }
+        asn1_delete_structure(&ext);
+        g_free(der.data);
+    }
+
+    /*
+     * Any old serial number will do, so lets pick 5
+     */
+    err = gnutls_x509_crt_set_serial(crt, serial, 5);
+    if (err < 0) {
+        g_critical("Failed to set certificate serial %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+    err = gnutls_x509_crt_set_activation_time(crt, start);
+    if (err < 0) {
+        g_critical("Failed to set certificate activation %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+    err = gnutls_x509_crt_set_expiration_time(crt, expire);
+    if (err < 0) {
+        g_critical("Failed to set certificate expiration %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+
+    /*
+     * If no 'ca' is set then we are self signing
+     * the cert. This is done for the root CA certs
+     */
+    err = gnutls_x509_crt_sign(crt, ca ? ca : crt, privkey);
+    if (err < 0) {
+        g_critical("Failed to sign certificate %s",
+                   gnutls_strerror(err));
+        abort();
+    }
+
+    /*
+     * Finally write the new cert out to disk
+     */
+    err = gnutls_x509_crt_export(
+        crt, GNUTLS_X509_FMT_PEM, buffer, &size);
+    if (err < 0) {
+        g_critical("Failed to export certificate %s: %d",
+                   gnutls_strerror(err), err);
+        abort();
+    }
+
+    if (!g_file_set_contents(req->filename, buffer, -1, NULL)) {
+        g_critical("Failed to write certificate %s",
+                   req->filename);
+        abort();
+    }
+
+    req->crt = crt;
+}
+
+
+void test_tls_write_cert_chain(const char *filename,
+                               gnutls_x509_crt_t *certs,
+                               size_t ncerts)
+{
+    size_t i;
+    size_t capacity = 1024, offset = 0;
+    char *buffer = g_new0(char, capacity);
+    int err;
+
+    for (i = 0; i < ncerts; i++) {
+        size_t len = capacity - offset;
+    retry:
+        err = gnutls_x509_crt_export(certs[i], GNUTLS_X509_FMT_PEM,
+                                     buffer + offset, &len);
+        if (err < 0) {
+            if (err == GNUTLS_E_SHORT_MEMORY_BUFFER) {
+                buffer = g_renew(char, buffer, offset + len);
+                capacity = offset + len;
+                goto retry;
+            }
+            g_critical("Failed to export certificate chain %s: %d",
+                       gnutls_strerror(err), err);
+            abort();
+        }
+        offset += len;
+    }
+
+    if (!g_file_set_contents(filename, buffer, offset, NULL)) {
+        abort();
+    }
+    g_free(buffer);
+}
+
+
+void test_tls_discard_cert(QCryptoTLSTestCertReq *req)
+{
+    if (!req->crt) {
+        return;
+    }
+
+    gnutls_x509_crt_deinit(req->crt);
+    req->crt = NULL;
+
+    if (getenv("QEMU_TEST_DEBUG_CERTS") == NULL) {
+        unlink(req->filename);
+    }
+}
+
+#endif /* QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/crypto-tls-x509-helpers.h b/tests/crypto-tls-x509-helpers.h
new file mode 100644
index 0000000..356b49c
--- /dev/null
+++ b/tests/crypto-tls-x509-helpers.h
@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#include <gnutls/gnutls.h>
+#include <gnutls/x509.h>
+
+#if !(defined WIN32) && \
+    defined(CONFIG_TASN1) && \
+    defined(LIBGNUTLS_VERSION_NUMBER) && \
+    (LIBGNUTLS_VERSION_NUMBER >= 0x020600)
+# define QCRYPTO_HAVE_TLS_TEST_SUPPORT
+#endif
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+# include <libtasn1.h>
+
+# include "qemu-common.h"
+
+/*
+ * This contains parameter about how to generate
+ * certificates.
+ */
+typedef struct QCryptoTLSTestCertReq QCryptoTLSTestCertReq;
+struct QCryptoTLSTestCertReq {
+    gnutls_x509_crt_t crt;
+
+    const char *filename;
+
+    /* Identifying information */
+    const char *country;
+    const char *cn;
+    const char *altname1;
+    const char *altname2;
+    const char *ipaddr1;
+    const char *ipaddr2;
+
+    /* Basic constraints */
+    bool basicConstraintsEnable;
+    bool basicConstraintsCritical;
+    bool basicConstraintsIsCA;
+
+    /* Key usage */
+    bool keyUsageEnable;
+    bool keyUsageCritical;
+    int keyUsageValue;
+
+    /* Key purpose (aka Extended key usage) */
+    bool keyPurposeEnable;
+    bool keyPurposeCritical;
+    const char *keyPurposeOID1;
+    const char *keyPurposeOID2;
+
+    /* zero for current time, or non-zero for hours from now */
+    int start_offset;
+    /* zero for 24 hours from now, or non-zero for hours from now */
+    int expire_offset;
+};
+
+void test_tls_generate_cert(QCryptoTLSTestCertReq *req,
+                            gnutls_x509_crt_t ca);
+void test_tls_write_cert_chain(const char *filename,
+                               gnutls_x509_crt_t *certs,
+                               size_t ncerts);
+void test_tls_discard_cert(QCryptoTLSTestCertReq *req);
+
+void test_tls_init(const char *keyfile);
+void test_tls_cleanup(const char *keyfile);
+
+# define TLS_CERT_REQ(varname, cavarname,                               \
+                      country, commonname,                              \
+                      altname1, altname2,                               \
+                      ipaddr1, ipaddr2,                                 \
+                      basicconsenable, basicconscritical, basicconsca,  \
+                      keyusageenable, keyusagecritical, keyusagevalue,  \
+                      keypurposeenable, keypurposecritical,             \
+                      keypurposeoid1, keypurposeoid2,                   \
+                      startoffset, endoffset)                           \
+    static QCryptoTLSTestCertReq varname = {                            \
+        NULL, WORKDIR #varname "-ctx.pem",                              \
+        country, commonname, altname1, altname2,                        \
+        ipaddr1, ipaddr2,                                               \
+        basicconsenable, basicconscritical, basicconsca,                \
+        keyusageenable, keyusagecritical, keyusagevalue,                \
+        keypurposeenable, keypurposecritical,                           \
+        keypurposeoid1, keypurposeoid2,                                 \
+        startoffset, endoffset                                          \
+    };                                                                  \
+    test_tls_generate_cert(&varname, cavarname.crt)
+
+# define TLS_ROOT_REQ(varname,                                          \
+                      country, commonname,                              \
+                      altname1, altname2,                               \
+                      ipaddr1, ipaddr2,                                 \
+                      basicconsenable, basicconscritical, basicconsca,  \
+                      keyusageenable, keyusagecritical, keyusagevalue,  \
+                      keypurposeenable, keypurposecritical,             \
+                      keypurposeoid1, keypurposeoid2,                   \
+                      startoffset, endoffset)                           \
+    static QCryptoTLSTestCertReq varname = {                            \
+        NULL, WORKDIR #varname "-ctx.pem",                              \
+        country, commonname, altname1, altname2,                        \
+        ipaddr1, ipaddr2,                                               \
+        basicconsenable, basicconscritical, basicconsca,                \
+        keyusageenable, keyusagecritical, keyusagevalue,                \
+        keypurposeenable, keypurposecritical,                           \
+        keypurposeoid1, keypurposeoid2,                                 \
+        startoffset, endoffset                                          \
+    };                                                                  \
+    test_tls_generate_cert(&varname, NULL)
+
+extern const ASN1_ARRAY_TYPE pkix_asn1_tab[];
+
+#endif /* QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/data/test-qga-config b/tests/data/test-qga-config
new file mode 100644
index 0000000..4bb721a
--- /dev/null
+++ b/tests/data/test-qga-config
@@ -0,0 +1,8 @@
+[general]
+daemon=false
+method=virtio-serial
+path=/path/to/org.qemu.guest_agent.0
+pidfile=/var/foo/qemu-ga.pid
+statedir=/var/state
+verbose=true
+blacklist=guest-ping;guest-get-time
diff --git a/tests/device-introspect-test.c b/tests/device-introspect-test.c
new file mode 100644
index 0000000..37debc1
--- /dev/null
+++ b/tests/device-introspect-test.c
@@ -0,0 +1,123 @@
+/*
+ * Device introspection test cases
+ *
+ * Copyright (c) 2015 Red Hat Inc.
+ *
+ * Authors:
+ *  Markus Armbruster <armbru@redhat.com>,
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+/*
+ * Covers QMP device-list-properties and HMP device_add help.  We
+ * currently don't check that their output makes sense, only that QEMU
+ * survives.  Useful since we've had an astounding number of crash
+ * bugs around here.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qapi/qmp/qstring.h"
+#include "libqtest.h"
+
+const char common_args[] = "-nodefaults -machine none";
+
+static QList *device_type_list(bool abstract)
+{
+    QDict *resp;
+    QList *ret;
+
+    resp = qmp("{'execute': 'qom-list-types',"
+               " 'arguments': {'implements': 'device', 'abstract': %i}}",
+               abstract);
+    g_assert(qdict_haskey(resp, "return"));
+    ret = qdict_get_qlist(resp, "return");
+    QINCREF(ret);
+    QDECREF(resp);
+    return ret;
+}
+
+static void test_one_device(const char *type)
+{
+    QDict *resp;
+    char *help, *qom_tree;
+
+    resp = qmp("{'execute': 'device-list-properties',"
+               " 'arguments': {'typename': %s}}",
+               type);
+    QDECREF(resp);
+
+    help = hmp("device_add \"%s,help\"", type);
+    g_free(help);
+
+    /*
+     * Some devices leave dangling pointers in QOM behind.
+     * "info qom-tree" has a good chance at crashing then
+     */
+    qom_tree = hmp("info qom-tree");
+    g_free(qom_tree);
+}
+
+static void test_device_intro_list(void)
+{
+    QList *types;
+    char *help;
+
+    qtest_start(common_args);
+
+    types = device_type_list(true);
+    QDECREF(types);
+
+    help = hmp("device_add help");
+    g_free(help);
+
+    qtest_end();
+}
+
+static void test_device_intro_none(void)
+{
+    qtest_start(common_args);
+    test_one_device("nonexistent");
+    qtest_end();
+}
+
+static void test_device_intro_abstract(void)
+{
+    qtest_start(common_args);
+    test_one_device("device");
+    qtest_end();
+}
+
+static void test_device_intro_concrete(void)
+{
+    QList *types;
+    QListEntry *entry;
+    const char *type;
+
+    qtest_start(common_args);
+    types = device_type_list(false);
+
+    QLIST_FOREACH_ENTRY(types, entry) {
+        type = qdict_get_try_str(qobject_to_qdict(qlist_entry_obj(entry)),
+                                "name");
+        g_assert(type);
+        test_one_device(type);
+    }
+
+    QDECREF(types);
+    qtest_end();
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("device/introspect/list", test_device_intro_list);
+    qtest_add_func("device/introspect/none", test_device_intro_none);
+    qtest_add_func("device/introspect/abstract", test_device_intro_abstract);
+    qtest_add_func("device/introspect/concrete", test_device_intro_concrete);
+
+    return g_test_run();
+}
diff --git a/tests/display-vga-test.c b/tests/display-vga-test.c
index 17f5910..9146021 100644
--- a/tests/display-vga-test.c
+++ b/tests/display-vga-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 static void pci_cirrus(void)
 {
@@ -36,17 +34,31 @@
     qtest_end();
 }
 
+static void pci_virtio_gpu(void)
+{
+    qtest_start("-vga none -device virtio-gpu-pci");
+    qtest_end();
+}
+
+#ifdef CONFIG_VIRTIO_VGA
+static void pci_virtio_vga(void)
+{
+    qtest_start("-vga none -device virtio-vga");
+    qtest_end();
+}
+#endif
+
 int main(int argc, char **argv)
 {
-    int ret;
-
     g_test_init(&argc, &argv, NULL);
 
     qtest_add_func("/display/pci/cirrus", pci_cirrus);
     qtest_add_func("/display/pci/stdvga", pci_stdvga);
     qtest_add_func("/display/pci/secondary", pci_secondary);
     qtest_add_func("/display/pci/multihead", pci_multihead);
-    ret = g_test_run();
-
-    return ret;
+    qtest_add_func("/display/pci/virtio-gpu", pci_virtio_gpu);
+#ifdef CONFIG_VIRTIO_VGA
+    qtest_add_func("/display/pci/virtio-vga", pci_virtio_vga);
+#endif
+    return g_test_run();
 }
diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include
new file mode 100644
index 0000000..4f4707d
--- /dev/null
+++ b/tests/docker/Makefile.include
@@ -0,0 +1,128 @@
+# Makefile for Docker tests
+
+.PHONY: docker docker-test docker-clean docker-image docker-qemu-src
+
+DOCKER_SUFFIX := .docker
+DOCKER_FILES_DIR := $(SRC_PATH)/tests/docker/dockerfiles
+DOCKER_IMAGES := $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.docker)))
+DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES))
+# Use a global constant ccache directory to speed up repetitive builds
+DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache
+
+DOCKER_TESTS := $(notdir $(shell \
+	find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f))
+
+DOCKER_TOOLS := travis
+
+TESTS ?= %
+IMAGES ?= %
+
+# Make archive from git repo $1 to tar.gz $2
+make-archive-maybe = $(if $(wildcard $1/*), \
+	$(call quiet-command, \
+		(cd $1; if git diff-index --quiet HEAD -- &>/dev/null; then \
+			git archive -1 HEAD --format=tar.gz; \
+		else \
+			git archive -1 $$(git stash create) --format=tar.gz; \
+		fi) > $2, \
+		"  ARCHIVE $(notdir $2)"))
+
+CUR_TIME := $(shell date +%Y-%m-%d-%H.%M.%S.$$$$)
+DOCKER_SRC_COPY := docker-src.$(CUR_TIME)
+
+$(DOCKER_SRC_COPY):
+	@mkdir $@
+	$(call make-archive-maybe, $(SRC_PATH), $@/qemu.tgz)
+	$(call make-archive-maybe, $(SRC_PATH)/dtc, $@/dtc.tgz)
+	$(call make-archive-maybe, $(SRC_PATH)/pixman, $@/pixman.tgz)
+	$(call quiet-command, cp $(SRC_PATH)/tests/docker/run $@/run, \
+		"  COPY RUNNER")
+
+docker-qemu-src: $(DOCKER_SRC_COPY)
+
+docker-image: ${DOCKER_TARGETS}
+
+# General rule for building docker images
+docker-image-%: $(DOCKER_FILES_DIR)/%.docker
+	$(call quiet-command,\
+		$(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \
+		$(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \
+		$(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\
+		"  BUILD $*")
+
+# Expand all the pre-requistes for each docker image and test combination
+$(foreach i,$(DOCKER_IMAGES), \
+	$(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \
+		$(eval .PHONY: docker-$t@$i) \
+		$(eval docker-$t@$i: docker-image-$i docker-run-$t@$i) \
+	) \
+	$(foreach t,$(DOCKER_TESTS), \
+		$(eval docker-test: docker-$t@$i) \
+	) \
+)
+
+docker:
+	@echo 'Build QEMU and run tests inside Docker containers'
+	@echo
+	@echo 'Available targets:'
+	@echo
+	@echo '    docker:              Print this help.'
+	@echo '    docker-test:         Run all image/test combinations.'
+	@echo '    docker-clean:        Kill and remove residual docker testing containers.'
+	@echo '    docker-TEST@IMAGE:   Run "TEST" in container "IMAGE".'
+	@echo '                         Note: "TEST" is one of the listed test name,'
+	@echo '                         or a script name under $$QEMU_SRC/tests/docker/;'
+	@echo '                         "IMAGE" is one of the listed container name."'
+	@echo '    docker-image:        Build all images.'
+	@echo '    docker-image-IMAGE:  Build image "IMAGE".'
+	@echo
+	@echo 'Available container images:'
+	@echo '    $(DOCKER_IMAGES)'
+	@echo
+	@echo 'Available tests:'
+	@echo '    $(DOCKER_TESTS)'
+	@echo
+	@echo 'Available tools:'
+	@echo '    $(DOCKER_TOOLS)'
+	@echo
+	@echo 'Special variables:'
+	@echo '    TARGET_LIST=a,b,c    Override target list in builds.'
+	@echo '    EXTRA_CONFIGURE_OPTS="..."'
+	@echo '                         Extra configure options.'
+	@echo '    IMAGES="a b c ..":   Filters which images to build or run.'
+	@echo '    TESTS="x y z .."     Filters which tests to run (for docker-test).'
+	@echo '    J=[0..9]*            Overrides the -jN parameter for make commands'
+	@echo '                         (default is 1)'
+	@echo '    DEBUG=1              Stop and drop to shell in the created container'
+	@echo '                         before running the command.'
+	@echo '    NOCACHE=1            Ignore cache when build images.'
+	@echo '    EXECUTABLE=<path>    Include executable in image.'
+
+docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\1/')
+docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-\([^@]*\)@\(.*\)/\2/')
+docker-run-%: docker-qemu-src
+	@mkdir -p "$(DOCKER_CCACHE_DIR)"
+	@if test -z "$(IMAGE)" || test -z "$(CMD)"; \
+		then echo "Invalid target"; exit 1; \
+	fi
+	$(if $(filter $(TESTS),$(CMD)),$(if $(filter $(IMAGES),$(IMAGE)), \
+		$(call quiet-command,\
+			if $(SRC_PATH)/tests/docker/docker.py images | \
+				awk '$$1=="qemu" && $$2=="$(IMAGE)"{found=1} END{exit(!found)}'; then \
+				$(SRC_PATH)/tests/docker/docker.py run $(if $V,,--rm) \
+				-t \
+				$(if $(DEBUG),-i,--net=none) \
+				-e TARGET_LIST=$(TARGET_LIST) \
+				-e EXTRA_CONFIGURE_OPTS=$(EXTRA_CONFIGURE_OPTS) \
+				-e V=$V -e J=$J -e DEBUG=$(DEBUG)\
+				-e CCACHE_DIR=/var/tmp/ccache \
+				-v $$(realpath $(DOCKER_SRC_COPY)):/var/tmp/qemu:z$(COMMA)ro \
+				-v $(DOCKER_CCACHE_DIR):/var/tmp/ccache:z \
+				qemu:$(IMAGE) \
+				/var/tmp/qemu/run \
+				$(CMD); \
+			fi \
+			, "  RUN $(CMD) in $(IMAGE)")))
+
+docker-clean:
+	$(call quiet-command, $(SRC_PATH)/tests/docker/docker.py clean)
diff --git a/tests/docker/common.rc b/tests/docker/common.rc
new file mode 100755
index 0000000..0c6d8d5
--- /dev/null
+++ b/tests/docker/common.rc
@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Common routines for docker test scripts.
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+requires()
+{
+    for c in $@; do
+        if ! echo "$FEATURES" | grep -wq -e "$c"; then
+            echo "Prerequisite '$c' not present, skip"
+            exit 0
+        fi
+    done
+}
+
+build_qemu()
+{
+    $QEMU_SRC/configure \
+        --enable-werror \
+        ${TARGET_LIST:+"--target-list=${TARGET_LIST}"} \
+        --prefix="$PWD/install" \
+        $EXTRA_CONFIGURE_OPTS \
+        "$@"
+    make $MAKEFLAGS
+}
diff --git a/tests/docker/docker.py b/tests/docker/docker.py
new file mode 100755
index 0000000..222a105
--- /dev/null
+++ b/tests/docker/docker.py
@@ -0,0 +1,335 @@
+#!/usr/bin/env python2
+#
+# Docker controlling module
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+import os
+import sys
+import subprocess
+import json
+import hashlib
+import atexit
+import uuid
+import argparse
+import tempfile
+import re
+from tarfile import TarFile, TarInfo
+from StringIO import StringIO
+from shutil import copy, rmtree
+
+def _text_checksum(text):
+    """Calculate a digest string unique to the text content"""
+    return hashlib.sha1(text).hexdigest()
+
+def _guess_docker_command():
+    """ Guess a working docker command or raise exception if not found"""
+    commands = [["docker"], ["sudo", "-n", "docker"]]
+    for cmd in commands:
+        if subprocess.call(cmd + ["images"],
+                           stdout=subprocess.PIPE,
+                           stderr=subprocess.PIPE) == 0:
+            return cmd
+    commands_txt = "\n".join(["  " + " ".join(x) for x in commands])
+    raise Exception("Cannot find working docker command. Tried:\n%s" % \
+                    commands_txt)
+
+def _copy_with_mkdir(src, root_dir, sub_path):
+    """Copy src into root_dir, creating sub_path as needed."""
+    dest_dir = os.path.normpath("%s/%s" % (root_dir, sub_path))
+    try:
+        os.makedirs(dest_dir)
+    except OSError:
+        # we can safely ignore already created directories
+        pass
+
+    dest_file = "%s/%s" % (dest_dir, os.path.basename(src))
+    copy(src, dest_file)
+
+
+def _get_so_libs(executable):
+    """Return a list of libraries associated with an executable.
+
+    The paths may be symbolic links which would need to be resolved to
+    ensure theright data is copied."""
+
+    libs = []
+    ldd_re = re.compile(r"(/.*/)(\S*)")
+    try:
+        ldd_output = subprocess.check_output(["ldd", executable])
+        for line in ldd_output.split("\n"):
+            search = ldd_re.search(line)
+            if search and len(search.groups()) == 2:
+                so_path = search.groups()[0]
+                so_lib = search.groups()[1]
+                libs.append("%s/%s" % (so_path, so_lib))
+    except subprocess.CalledProcessError:
+        print "%s had no associated libraries (static build?)" % (executable)
+
+    return libs
+
+def _copy_binary_with_libs(src, dest_dir):
+    """Copy a binary executable and all its dependant libraries.
+
+    This does rely on the host file-system being fairly multi-arch
+    aware so the file don't clash with the guests layout."""
+
+    _copy_with_mkdir(src, dest_dir, "/usr/bin")
+
+    libs = _get_so_libs(src)
+    if libs:
+        for l in libs:
+            so_path = os.path.dirname(l)
+            _copy_with_mkdir(l , dest_dir, so_path)
+
+class Docker(object):
+    """ Running Docker commands """
+    def __init__(self):
+        self._command = _guess_docker_command()
+        self._instances = []
+        atexit.register(self._kill_instances)
+
+    def _do(self, cmd, quiet=True, infile=None, **kwargs):
+        if quiet:
+            kwargs["stdout"] = subprocess.PIPE
+        if infile:
+            kwargs["stdin"] = infile
+        return subprocess.call(self._command + cmd, **kwargs)
+
+    def _do_kill_instances(self, only_known, only_active=True):
+        cmd = ["ps", "-q"]
+        if not only_active:
+            cmd.append("-a")
+        for i in self._output(cmd).split():
+            resp = self._output(["inspect", i])
+            labels = json.loads(resp)[0]["Config"]["Labels"]
+            active = json.loads(resp)[0]["State"]["Running"]
+            if not labels:
+                continue
+            instance_uuid = labels.get("com.qemu.instance.uuid", None)
+            if not instance_uuid:
+                continue
+            if only_known and instance_uuid not in self._instances:
+                continue
+            print "Terminating", i
+            if active:
+                self._do(["kill", i])
+            self._do(["rm", i])
+
+    def clean(self):
+        self._do_kill_instances(False, False)
+        return 0
+
+    def _kill_instances(self):
+        return self._do_kill_instances(True)
+
+    def _output(self, cmd, **kwargs):
+        return subprocess.check_output(self._command + cmd,
+                                       stderr=subprocess.STDOUT,
+                                       **kwargs)
+
+    def get_image_dockerfile_checksum(self, tag):
+        resp = self._output(["inspect", tag])
+        labels = json.loads(resp)[0]["Config"].get("Labels", {})
+        return labels.get("com.qemu.dockerfile-checksum", "")
+
+    def build_image(self, tag, docker_dir, dockerfile, quiet=True, argv=None):
+        if argv == None:
+            argv = []
+
+        tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker")
+        tmp_df.write(dockerfile)
+
+        tmp_df.write("\n")
+        tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" %
+                     _text_checksum(dockerfile))
+        tmp_df.flush()
+
+        self._do(["build", "-t", tag, "-f", tmp_df.name] + argv + \
+                 [docker_dir],
+                 quiet=quiet)
+
+    def update_image(self, tag, tarball, quiet=True):
+        "Update a tagged image using "
+
+        self._do(["build", "-t", tag, "-"], quiet=quiet, infile=tarball)
+
+    def image_matches_dockerfile(self, tag, dockerfile):
+        try:
+            checksum = self.get_image_dockerfile_checksum(tag)
+        except Exception:
+            return False
+        return checksum == _text_checksum(dockerfile)
+
+    def run(self, cmd, keep, quiet):
+        label = uuid.uuid1().hex
+        if not keep:
+            self._instances.append(label)
+        ret = self._do(["run", "--label",
+                        "com.qemu.instance.uuid=" + label] + cmd,
+                       quiet=quiet)
+        if not keep:
+            self._instances.remove(label)
+        return ret
+
+    def command(self, cmd, argv, quiet):
+        return self._do([cmd] + argv, quiet=quiet)
+
+class SubCommand(object):
+    """A SubCommand template base class"""
+    name = None # Subcommand name
+    def shared_args(self, parser):
+        parser.add_argument("--quiet", action="store_true",
+                            help="Run quietly unless an error occured")
+
+    def args(self, parser):
+        """Setup argument parser"""
+        pass
+    def run(self, args, argv):
+        """Run command.
+        args: parsed argument by argument parser.
+        argv: remaining arguments from sys.argv.
+        """
+        pass
+
+class RunCommand(SubCommand):
+    """Invoke docker run and take care of cleaning up"""
+    name = "run"
+    def args(self, parser):
+        parser.add_argument("--keep", action="store_true",
+                            help="Don't remove image when command completes")
+    def run(self, args, argv):
+        return Docker().run(argv, args.keep, quiet=args.quiet)
+
+class BuildCommand(SubCommand):
+    """ Build docker image out of a dockerfile. Arguments: <tag> <dockerfile>"""
+    name = "build"
+    def args(self, parser):
+        parser.add_argument("--include-executable", "-e",
+                            help="""Specify a binary that will be copied to the
+                            container together with all its dependent
+                            libraries""")
+        parser.add_argument("tag",
+                            help="Image Tag")
+        parser.add_argument("dockerfile",
+                            help="Dockerfile name")
+
+    def run(self, args, argv):
+        dockerfile = open(args.dockerfile, "rb").read()
+        tag = args.tag
+
+        dkr = Docker()
+        if dkr.image_matches_dockerfile(tag, dockerfile):
+            if not args.quiet:
+                print "Image is up to date."
+        else:
+            # Create a docker context directory for the build
+            docker_dir = tempfile.mkdtemp(prefix="docker_build")
+
+            # Is there a .pre file to run in the build context?
+            docker_pre = os.path.splitext(args.dockerfile)[0]+".pre"
+            if os.path.exists(docker_pre):
+                rc = subprocess.call(os.path.realpath(docker_pre),
+                                     cwd=docker_dir)
+                if rc == 3:
+                    print "Skip"
+                    return 0
+                elif rc != 0:
+                    print "%s exited with code %d" % (docker_pre, rc)
+                    return 1
+
+            # Do we include a extra binary?
+            if args.include_executable:
+                _copy_binary_with_libs(args.include_executable,
+                                       docker_dir)
+
+            dkr.build_image(tag, docker_dir, dockerfile,
+                            quiet=args.quiet, argv=argv)
+
+            rmtree(docker_dir)
+
+        return 0
+
+class UpdateCommand(SubCommand):
+    """ Update a docker image with new executables. Arguments: <tag> <executable>"""
+    name = "update"
+    def args(self, parser):
+        parser.add_argument("tag",
+                            help="Image Tag")
+        parser.add_argument("executable",
+                            help="Executable to copy")
+
+    def run(self, args, argv):
+        # Create a temporary tarball with our whole build context and
+        # dockerfile for the update
+        tmp = tempfile.NamedTemporaryFile(suffix="dckr.tar.gz")
+        tmp_tar = TarFile(fileobj=tmp, mode='w')
+
+        # Add the executable to the tarball
+        bn = os.path.basename(args.executable)
+        ff = "/usr/bin/%s" % bn
+        tmp_tar.add(args.executable, arcname=ff)
+
+        # Add any associated libraries
+        libs = _get_so_libs(args.executable)
+        if libs:
+            for l in libs:
+                tmp_tar.add(os.path.realpath(l), arcname=l)
+
+        # Create a Docker buildfile
+        df = StringIO()
+        df.write("FROM %s\n" % args.tag)
+        df.write("ADD . /\n")
+        df.seek(0)
+
+        df_tar = TarInfo(name="Dockerfile")
+        df_tar.size = len(df.buf)
+        tmp_tar.addfile(df_tar, fileobj=df)
+
+        tmp_tar.close()
+
+        # reset the file pointers
+        tmp.flush()
+        tmp.seek(0)
+
+        # Run the build with our tarball context
+        dkr = Docker()
+        dkr.update_image(args.tag, tmp, quiet=args.quiet)
+
+        return 0
+
+class CleanCommand(SubCommand):
+    """Clean up docker instances"""
+    name = "clean"
+    def run(self, args, argv):
+        Docker().clean()
+        return 0
+
+class ImagesCommand(SubCommand):
+    """Run "docker images" command"""
+    name = "images"
+    def run(self, args, argv):
+        return Docker().command("images", argv, args.quiet)
+
+def main():
+    parser = argparse.ArgumentParser(description="A Docker helper",
+            usage="%s <subcommand> ..." % os.path.basename(sys.argv[0]))
+    subparsers = parser.add_subparsers(title="subcommands", help=None)
+    for cls in SubCommand.__subclasses__():
+        cmd = cls()
+        subp = subparsers.add_parser(cmd.name, help=cmd.__doc__)
+        cmd.shared_args(subp)
+        cmd.args(subp)
+        subp.set_defaults(cmdobj=cmd)
+    args, argv = parser.parse_known_args()
+    return args.cmdobj.run(args, argv)
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tests/docker/dockerfiles/centos6.docker b/tests/docker/dockerfiles/centos6.docker
new file mode 100644
index 0000000..8f4fe46
--- /dev/null
+++ b/tests/docker/dockerfiles/centos6.docker
@@ -0,0 +1,6 @@
+FROM centos:6
+RUN yum install -y \
+    tar git make gcc g++ \
+    zlib-devel glib2-devel SDL-devel pixman-devel \
+    epel-release
+RUN yum install -y libfdt-devel ccache
diff --git a/tests/docker/dockerfiles/debian-bootstrap.docker b/tests/docker/dockerfiles/debian-bootstrap.docker
new file mode 100644
index 0000000..3a9125e
--- /dev/null
+++ b/tests/docker/dockerfiles/debian-bootstrap.docker
@@ -0,0 +1,21 @@
+# Create Debian Bootstrap Image
+#
+# This is intended to be pre-poluated by:
+#  - a first stage debootstrap (see debian-bootstrap.pre)
+#  - a native qemu-$arch that binfmt_misc will run
+FROM scratch
+
+# Add everything from the context into the container
+ADD . /
+
+# Patch all mounts as docker already has stuff set up
+RUN sed -i 's/in_target mount/echo not for docker in_target mount/g' /debootstrap/functions
+
+# Run stage 2
+RUN /debootstrap/debootstrap --second-stage
+
+# At this point we can install additional packages if we want
+# Duplicate deb line as deb-src
+RUN cat /etc/apt/sources.list | sed "s/deb/deb-src/" >> /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get -y build-dep qemu
diff --git a/tests/docker/dockerfiles/debian-bootstrap.pre b/tests/docker/dockerfiles/debian-bootstrap.pre
new file mode 100755
index 0000000..5d9c8d5
--- /dev/null
+++ b/tests/docker/dockerfiles/debian-bootstrap.pre
@@ -0,0 +1,87 @@
+#!/bin/sh
+#
+# Simple wrapper for debootstrap, run in the docker build context
+#
+FAKEROOT=`which fakeroot 2> /dev/null`
+
+exit_and_skip()
+{
+    exit 3
+}
+
+#
+# fakeroot is needed to run the bootstrap stage
+#
+if [ -z $FAKEROOT ]; then
+    echo "Please install fakeroot to enable bootstraping"
+    exit_and_skip
+fi
+
+# We check in order for
+#
+#  - DEBOOTSTRAP_DIR pointing at a development checkout
+#  - PATH for the debootstrap script (installed)
+#
+# If neither option works then we checkout debootstrap from its
+# upstream SCM and run it from there.
+#
+
+if [ -z $DEBOOTSTRAP_DIR ]; then
+    DEBOOTSTRAP=`which debootstrap 2> /dev/null`
+    if [ -z $DEBOOTSTRAP ]; then
+        echo "No debootstrap installed, attempting to install from SCM"
+        DEBOOTSTRAP_SOURCE=https://anonscm.debian.org/git/d-i/debootstrap.git
+        git clone ${DEBOOTSTRAP_SOURCE} ./debootstrap.git
+        export DEBOOTSTRAP_DIR=./debootstrap.git
+        DEBOOTSTRAP=./debootstrap.git/debootstrap
+    fi
+else
+    DEBOOTSTRAP=${DEBOOTSTRAP_DIR}/debootstrap
+    if [ ! -f $DEBOOTSTRAP ]; then
+        echo "Couldn't find script at ${DEBOOTSTRAP}"
+        exit_and_skip
+    fi
+fi
+
+#
+# Finally check to see if any qemu's are installed
+#
+BINFMT_DIR=/proc/sys/fs/binfmt_misc
+if [ ! -e $BINFMT_DIR ]; then
+   echo "binfmt_misc needs enabling for a QEMU bootstrap to work"
+   exit_and_skip
+else
+    # DEB_ARCH and QEMU arch names are not totally aligned
+    case "${DEB_ARCH}" in
+        amd64)
+            QEMU=qemu-i386
+            ;;
+        armel|armhf)
+            QEMU=qemu-arm
+            ;;
+        arm64)
+            QEMU=qemu-aarch64
+            ;;
+        powerpc)
+            QEMU=qemu-ppc
+            ;;
+        ppc64el)
+            QEMU=qemu-ppc64le
+            ;;
+        s390)
+            QEMU=qemu-s390x
+            ;;
+        *)
+            QEMU=qemu-${DEB_ARCH}
+        ;;
+    esac
+    if [ ! -e "${BINFMT_DIR}/$QEMU" ]; then
+        echo "No binfmt_misc rule to run $QEMU, can't bootstrap"
+        exit_and_skip
+    fi
+fi
+
+echo "Building a rootfs using ${FAKEROOT} and ${DEBOOTSTRAP} ${DEB_ARCH}/${DEB_TYPE}"
+
+${FAKEROOT} ${DEBOOTSTRAP} --variant=buildd --foreign --arch=$DEB_ARCH $DEB_TYPE . http://httpredir.debian.org/debian || exit 1
+exit 0
diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker
new file mode 100644
index 0000000..1d26a8e
--- /dev/null
+++ b/tests/docker/dockerfiles/fedora.docker
@@ -0,0 +1,7 @@
+FROM fedora:23
+RUN dnf install -y \
+    ccache git tar PyYAML sparse flex bison \
+    glib2-devel pixman-devel zlib-devel SDL-devel libfdt-devel \
+    gcc gcc-c++ clang make perl which bc findutils \
+    mingw{32,64}-{pixman,glib2,gmp,SDL,pkg-config,gtk2,gtk3,gnutls,nettle,libtasn1,libjpeg-turbo,libpng,curl,libssh2,bzip2}
+ENV FEATURES mingw clang pyyaml
diff --git a/tests/docker/dockerfiles/ubuntu.docker b/tests/docker/dockerfiles/ubuntu.docker
new file mode 100644
index 0000000..a8b88c3
--- /dev/null
+++ b/tests/docker/dockerfiles/ubuntu.docker
@@ -0,0 +1,11 @@
+FROM ubuntu:14.04
+RUN echo "deb http://archive.ubuntu.com/ubuntu/ trusty universe multiverse" >> \
+    /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get -y install flex bison \
+    libusb-1.0-0-dev libiscsi-dev librados-dev libncurses5-dev \
+    libseccomp-dev libgnutls-dev libssh2-1-dev  libspice-server-dev \
+    libspice-protocol-dev libnss3-dev libfdt-dev \
+    libgtk-3-dev libvte-2.90-dev libsdl1.2-dev libpng12-dev libpixman-1-dev \
+    git make ccache python-yaml gcc clang sparse
+ENV FEATURES clang pyyaml
diff --git a/tests/docker/run b/tests/docker/run
new file mode 100755
index 0000000..d85d49a
--- /dev/null
+++ b/tests/docker/run
@@ -0,0 +1,68 @@
+#!/bin/bash -e
+#
+# Docker test runner
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+set -e
+
+if test -n "$V"; then
+    set -x
+fi
+
+BASE="$(dirname $(readlink -e $0))"
+
+# Prepare the environment
+. /etc/profile || true
+export PATH=/usr/lib/ccache:$PATH
+
+if test -n "$J"; then
+    export MAKEFLAGS="$MAKEFLAGS -j$J"
+fi
+
+# We are in the container so the whole file system belong to us
+export TEST_DIR=/tmp/qemu-test
+mkdir -p $TEST_DIR/{src,build,install}
+
+# Extract the source tarballs
+tar -C $TEST_DIR/src -xzf $BASE/qemu.tgz
+for p in dtc pixman; do
+    if test -f $BASE/$p.tgz; then
+        tar -C $TEST_DIR/src/$p -xzf $BASE/$p.tgz
+        export FEATURES="$FEATURES $p"
+    fi
+done
+
+export QEMU_SRC="$TEST_DIR/src"
+
+cd "$QEMU_SRC/tests/docker"
+
+CMD="$QEMU_SRC/tests/docker/$@"
+
+if test -n "$DEBUG"; then
+    echo "* Prepared to run command:"
+    echo "  $CMD"
+    echo "* Hit Ctrl-D to continue, or type 'exit 1' to abort"
+    echo
+    $SHELL
+fi
+
+if "$CMD"; then
+    exit 0
+elif test -n "$DEBUG"; then
+    echo "* Command failed:"
+    echo "  $CMD"
+    echo "* Hit Ctrl-D to exit"
+    echo
+    # Force error after shell exits
+    $SHELL && exit 1
+else
+    exit 1
+fi
diff --git a/tests/docker/test-clang b/tests/docker/test-clang
new file mode 100755
index 0000000..60e4e97
--- /dev/null
+++ b/tests/docker/test-clang
@@ -0,0 +1,24 @@
+#!/bin/bash -e
+#
+# Compile and check with clang.
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+. common.rc
+
+requires clang
+
+OPTS="--enable-debug --cxx=clang++ --cc=clang --host-cc=clang"
+# -fsanitize=undefined is broken on Fedora 23, skip it for now
+# See also: https://bugzilla.redhat.com/show_bug.cgi?id=1263834
+#OPTS="$OPTS --extra-cflags=-fsanitize=undefined \
+    #--extra-cflags=-fno-sanitize=float-divide-by-zero"
+build_qemu $OPTS
+make $MAKEFLAGS check
diff --git a/tests/docker/test-full b/tests/docker/test-full
new file mode 100755
index 0000000..fd9b798
--- /dev/null
+++ b/tests/docker/test-full
@@ -0,0 +1,17 @@
+#!/bin/bash -e
+#
+# Compile all the targets.
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+. common.rc
+
+build_qemu
+make check $MAKEFLAGS
diff --git a/tests/docker/test-mingw b/tests/docker/test-mingw
new file mode 100755
index 0000000..c03757a
--- /dev/null
+++ b/tests/docker/test-mingw
@@ -0,0 +1,34 @@
+#!/bin/bash -e
+#
+# Cross compile QEMU with mingw toolchain on Linux.
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+. common.rc
+
+requires mingw dtc
+
+for prefix in x86_64-w64-mingw32- i686-w64-mingw32-; do
+    TARGET_LIST=x86_64-softmmu,aarch64-softmmu \
+        build_qemu --cross-prefix=$prefix \
+        --enable-trace-backends=simple \
+        --enable-debug \
+        --enable-gnutls \
+        --enable-nettle \
+        --enable-curl \
+        --enable-vnc \
+        --enable-bzip2 \
+        --enable-guest-agent \
+        --with-sdlabi=1.2 \
+        --with-gtkabi=2.0
+    make clean
+
+done
+
diff --git a/tests/docker/test-quick b/tests/docker/test-quick
new file mode 100755
index 0000000..07cdc59
--- /dev/null
+++ b/tests/docker/test-quick
@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+# Quick compiling test that everyone already does. But why not automate it?
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+. common.rc
+
+DEF_TARGET_LIST="$(echo {x86_64,aarch64}-softmmu)"
+TARGET_LIST=${TARGET_LIST:-$DEF_TARGET_LIST} \
+build_qemu
+make check $MAKEFLAGS
diff --git a/tests/docker/travis b/tests/docker/travis
new file mode 100755
index 0000000..d345393
--- /dev/null
+++ b/tests/docker/travis
@@ -0,0 +1,21 @@
+#!/bin/bash -e
+#
+# Mimic a travis testing matrix
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+. common.rc
+
+requires pyyaml
+cmdfile=/tmp/travis_cmd_list.sh
+$QEMU_SRC/tests/docker/travis.py $QEMU_SRC/.travis.yml > $cmdfile
+chmod +x $cmdfile
+cd "$QEMU_SRC"
+$cmdfile
diff --git a/tests/docker/travis.py b/tests/docker/travis.py
new file mode 100755
index 0000000..8dcc964
--- /dev/null
+++ b/tests/docker/travis.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# Travis YAML config parser
+#
+# Copyright (c) 2016 Red Hat Inc.
+#
+# Authors:
+#  Fam Zheng <famz@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2
+# or (at your option) any later version. See the COPYING file in
+# the top-level directory.
+
+import sys
+import yaml
+import itertools
+
+def load_yaml(fname):
+    return yaml.load(open(fname, "r").read())
+
+def conf_iter(conf):
+    def env_to_list(env):
+        return env if isinstance(env, list) else [env]
+    global_env = conf["env"]["global"]
+    for entry in conf["matrix"]["include"]:
+        yield {"env": global_env + env_to_list(entry["env"]),
+               "compiler": entry["compiler"]}
+    for entry in itertools.product(conf["compiler"],
+                                   conf["env"]["matrix"]):
+        yield {"env": global_env + env_to_list(entry[1]),
+               "compiler": entry[0]}
+
+def main():
+    if len(sys.argv) < 2:
+        sys.stderr.write("Usage: %s <travis-file>\n" % sys.argv[0])
+        return 1
+    conf = load_yaml(sys.argv[1])
+    for config in conf_iter(conf):
+        print "("
+        print "\n".join(config["env"])
+        print "alias cc=" + config["compiler"]
+        print "\n".join(conf["before_script"])
+        print "\n".join(conf["script"])
+        print ")"
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tests/drive_del-test.c b/tests/drive_del-test.c
index 53fa969..121b9c9 100644
--- a/tests/drive_del-test.c
+++ b/tests/drive_del-test.c
@@ -10,34 +10,23 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
+#include "qemu/osdep.h"
 #include "libqtest.h"
 
 static void drive_add(void)
 {
-    QDict *response;
+    char *resp = hmp("drive_add 0 if=none,id=drive0");
 
-    response = qmp("{'execute': 'human-monitor-command',"
-                   " 'arguments': {"
-                   "   'command-line': 'drive_add 0 if=none,id=drive0'"
-                   "}}");
-    g_assert(response);
-    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "OK\r\n");
-    QDECREF(response);
+    g_assert_cmpstr(resp, ==, "OK\r\n");
+    g_free(resp);
 }
 
 static void drive_del(void)
 {
-    QDict *response;
+    char *resp = hmp("drive_del drive0");
 
-    response = qmp("{'execute': 'human-monitor-command',"
-                   " 'arguments': {"
-                   "   'command-line': 'drive_del drive0'"
-                   "}}");
-    g_assert(response);
-    g_assert_cmpstr(qdict_get_try_str(response, "return"), ==, "");
-    QDECREF(response);
+    g_assert_cmpstr(resp, ==, "");
+    g_free(resp);
 }
 
 static void device_del(void)
@@ -103,7 +92,7 @@
 static void test_drive_del_device_del(void)
 {
     /* Start with a drive used by a device that unplugs instantaneously */
-    qtest_start("-drive if=none,id=drive0,file=/dev/null"
+    qtest_start("-drive if=none,id=drive0,file=/dev/null,format=raw"
                 " -device virtio-scsi-pci"
                 " -device scsi-hd,drive=drive0,id=dev0");
 
@@ -126,7 +115,8 @@
     qtest_add_func("/drive_del/without-dev", test_drive_without_dev);
 
     /* TODO I guess any arch with PCI would do */
-    if (!strcmp(arch, "i386") || !strcmp(arch, "x86_64")) {
+    if (!strcmp(arch, "i386") || !strcmp(arch, "x86_64") ||
+        !strcmp(arch, "ppc") || !strcmp(arch, "ppc64")) {
         qtest_add_func("/drive_del/after_failed_device_add",
                        test_after_failed_device_add);
         qtest_add_func("/blockdev/drive_del_device_del",
diff --git a/tests/ds1338-test.c b/tests/ds1338-test.c
new file mode 100644
index 0000000..26968bc
--- /dev/null
+++ b/tests/ds1338-test.c
@@ -0,0 +1,77 @@
+/*
+ * QTest testcase for the DS1338 RTC
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "libqos/i2c.h"
+
+#define IMX25_I2C_0_BASE 0x43F80000
+
+#define DS1338_ADDR 0x68
+
+static I2CAdapter *i2c;
+static uint8_t addr;
+
+static inline uint8_t bcd2bin(uint8_t x)
+{
+    return ((x) & 0x0f) + ((x) >> 4) * 10;
+}
+
+static void send_and_receive(void)
+{
+    uint8_t cmd[1];
+    uint8_t resp[7];
+    time_t now = time(NULL);
+    struct tm *tm_ptr = gmtime(&now);
+
+    /* reset the index in the RTC memory */
+    cmd[0] = 0;
+    i2c_send(i2c, addr, cmd, 1);
+
+    /* retrieve the date */
+    i2c_recv(i2c, addr, resp, 7);
+
+    /* check retrieved time againt local time */
+    g_assert_cmpuint(bcd2bin(resp[4]), == , tm_ptr->tm_mday);
+    g_assert_cmpuint(bcd2bin(resp[5]), == , 1 + tm_ptr->tm_mon);
+    g_assert_cmpuint(2000 + bcd2bin(resp[6]), == , 1900 + tm_ptr->tm_year);
+}
+
+int main(int argc, char **argv)
+{
+    QTestState *s = NULL;
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    s = qtest_start("-display none -machine imx25-pdk");
+    i2c = imx_i2c_create(IMX25_I2C_0_BASE);
+    addr = DS1338_ADDR;
+
+    qtest_add_func("/ds1338/tx-rx", send_and_receive);
+
+    ret = g_test_run();
+
+    if (s) {
+        qtest_quit(s);
+    }
+    g_free(i2c);
+
+    return ret;
+}
diff --git a/tests/e1000-test.c b/tests/e1000-test.c
index 81f164d..59cab68 100644
--- a/tests/e1000-test.c
+++ b/tests/e1000-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void test_device(gconstpointer data)
@@ -44,8 +42,8 @@
     for (i = 0; i < ARRAY_SIZE(models); i++) {
         char *path;
 
-        path = g_strdup_printf("/%s/e1000/%s", qtest_get_arch(), models[i]);
-        g_test_add_data_func(path, models[i], test_device);
+        path = g_strdup_printf("e1000/%s", models[i]);
+        qtest_add_data_func(path, models[i], test_device);
     }
 
     return g_test_run();
diff --git a/tests/e1000e-test.c b/tests/e1000e-test.c
new file mode 100644
index 0000000..d497b08
--- /dev/null
+++ b/tests/e1000e-test.c
@@ -0,0 +1,478 @@
+ /*
+ * QTest testcase for e1000e NIC
+ *
+ * Copyright (c) 2015 Ravello Systems LTD (http://ravellosystems.com)
+ * Developed by Daynix Computing LTD (http://www.daynix.com)
+ *
+ * Authors:
+ * Dmitry Fleytman <dmitry@daynix.com>
+ * Leonid Bloch <leonid@daynix.com>
+ * Yan Vugenfirer <yan@daynix.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "qemu-common.h"
+#include "libqos/pci-pc.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
+#include "qemu/bitops.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+#include "libqos/malloc-generic.h"
+
+#define E1000E_IMS      (0x00d0)
+
+#define E1000E_STATUS   (0x0008)
+#define E1000E_STATUS_LU BIT(1)
+#define E1000E_STATUS_ASDV1000 BIT(9)
+
+#define E1000E_CTRL     (0x0000)
+#define E1000E_CTRL_RESET BIT(26)
+
+#define E1000E_RCTL     (0x0100)
+#define E1000E_RCTL_EN  BIT(1)
+#define E1000E_RCTL_UPE BIT(3)
+#define E1000E_RCTL_MPE BIT(4)
+
+#define E1000E_RFCTL     (0x5008)
+#define E1000E_RFCTL_EXTEN  BIT(15)
+
+#define E1000E_TCTL     (0x0400)
+#define E1000E_TCTL_EN  BIT(1)
+
+#define E1000E_CTRL_EXT             (0x0018)
+#define E1000E_CTRL_EXT_DRV_LOAD    BIT(28)
+#define E1000E_CTRL_EXT_TXLSFLOW    BIT(22)
+
+#define E1000E_RX0_MSG_ID           (0)
+#define E1000E_TX0_MSG_ID           (1)
+#define E1000E_OTHER_MSG_ID         (2)
+
+#define E1000E_IVAR                 (0x00E4)
+#define E1000E_IVAR_TEST_CFG        ((E1000E_RX0_MSG_ID << 0)    | BIT(3)  | \
+                                     (E1000E_TX0_MSG_ID << 8)    | BIT(11) | \
+                                     (E1000E_OTHER_MSG_ID << 16) | BIT(19) | \
+                                     BIT(31))
+
+#define E1000E_RING_LEN             (0x1000)
+#define E1000E_TXD_LEN              (16)
+#define E1000E_RXD_LEN              (16)
+
+#define E1000E_TDBAL    (0x3800)
+#define E1000E_TDBAH    (0x3804)
+#define E1000E_TDLEN    (0x3808)
+#define E1000E_TDH      (0x3810)
+#define E1000E_TDT      (0x3818)
+
+#define E1000E_RDBAL    (0x2800)
+#define E1000E_RDBAH    (0x2804)
+#define E1000E_RDLEN    (0x2808)
+#define E1000E_RDH      (0x2810)
+#define E1000E_RDT      (0x2818)
+
+typedef struct e1000e_device {
+    QPCIDevice *pci_dev;
+    void *mac_regs;
+
+    uint64_t tx_ring;
+    uint64_t rx_ring;
+} e1000e_device;
+
+static int test_sockets[2];
+static QGuestAllocator *test_alloc;
+static QPCIBus *test_bus;
+
+static void e1000e_pci_foreach_callback(QPCIDevice *dev, int devfn, void *data)
+{
+    *(QPCIDevice **) data = dev;
+}
+
+static QPCIDevice *e1000e_device_find(QPCIBus *bus)
+{
+    static const int e1000e_vendor_id = 0x8086;
+    static const int e1000e_dev_id = 0x10D3;
+
+    QPCIDevice *e1000e_dev = NULL;
+
+    qpci_device_foreach(bus, e1000e_vendor_id, e1000e_dev_id,
+        e1000e_pci_foreach_callback, &e1000e_dev);
+
+    g_assert_nonnull(e1000e_dev);
+
+    return e1000e_dev;
+}
+
+static void e1000e_macreg_write(e1000e_device *d, uint32_t reg, uint32_t val)
+{
+    qpci_io_writel(d->pci_dev, d->mac_regs + reg, val);
+}
+
+static uint32_t e1000e_macreg_read(e1000e_device *d, uint32_t reg)
+{
+    return qpci_io_readl(d->pci_dev, d->mac_regs + reg);
+}
+
+static void e1000e_device_init(QPCIBus *bus, e1000e_device *d)
+{
+    uint32_t val;
+
+    d->pci_dev = e1000e_device_find(bus);
+
+    /* Enable the device */
+    qpci_device_enable(d->pci_dev);
+
+    /* Map BAR0 (mac registers) */
+    d->mac_regs = qpci_iomap(d->pci_dev, 0, NULL);
+    g_assert_nonnull(d->mac_regs);
+
+    /* Reset the device */
+    val = e1000e_macreg_read(d, E1000E_CTRL);
+    e1000e_macreg_write(d, E1000E_CTRL, val | E1000E_CTRL_RESET);
+
+    /* Enable and configure MSI-X */
+    qpci_msix_enable(d->pci_dev);
+    e1000e_macreg_write(d, E1000E_IVAR, E1000E_IVAR_TEST_CFG);
+
+    /* Check the device status - link and speed */
+    val = e1000e_macreg_read(d, E1000E_STATUS);
+    g_assert_cmphex(val & (E1000E_STATUS_LU | E1000E_STATUS_ASDV1000),
+        ==, E1000E_STATUS_LU | E1000E_STATUS_ASDV1000);
+
+    /* Initialize TX/RX logic */
+    e1000e_macreg_write(d, E1000E_RCTL, 0);
+    e1000e_macreg_write(d, E1000E_TCTL, 0);
+
+    /* Notify the device that the driver is ready */
+    val = e1000e_macreg_read(d, E1000E_CTRL_EXT);
+    e1000e_macreg_write(d, E1000E_CTRL_EXT,
+        val | E1000E_CTRL_EXT_DRV_LOAD | E1000E_CTRL_EXT_TXLSFLOW);
+
+    /* Allocate and setup TX ring */
+    d->tx_ring = guest_alloc(test_alloc, E1000E_RING_LEN);
+    g_assert(d->tx_ring != 0);
+
+    e1000e_macreg_write(d, E1000E_TDBAL, (uint32_t) d->tx_ring);
+    e1000e_macreg_write(d, E1000E_TDBAH, (uint32_t) (d->tx_ring >> 32));
+    e1000e_macreg_write(d, E1000E_TDLEN, E1000E_RING_LEN);
+    e1000e_macreg_write(d, E1000E_TDT, 0);
+    e1000e_macreg_write(d, E1000E_TDH, 0);
+
+    /* Enable transmit */
+    e1000e_macreg_write(d, E1000E_TCTL, E1000E_TCTL_EN);
+
+    /* Allocate and setup RX ring */
+    d->rx_ring = guest_alloc(test_alloc, E1000E_RING_LEN);
+    g_assert(d->rx_ring != 0);
+
+    e1000e_macreg_write(d, E1000E_RDBAL, (uint32_t)d->rx_ring);
+    e1000e_macreg_write(d, E1000E_RDBAH, (uint32_t)(d->rx_ring >> 32));
+    e1000e_macreg_write(d, E1000E_RDLEN, E1000E_RING_LEN);
+    e1000e_macreg_write(d, E1000E_RDT, 0);
+    e1000e_macreg_write(d, E1000E_RDH, 0);
+
+    /* Enable receive */
+    e1000e_macreg_write(d, E1000E_RFCTL, E1000E_RFCTL_EXTEN);
+    e1000e_macreg_write(d, E1000E_RCTL, E1000E_RCTL_EN  |
+                                        E1000E_RCTL_UPE |
+                                        E1000E_RCTL_MPE);
+
+    /* Enable all interrupts */
+    e1000e_macreg_write(d, E1000E_IMS, 0xFFFFFFFF);
+}
+
+static void e1000e_tx_ring_push(e1000e_device *d, void *descr)
+{
+    uint32_t tail = e1000e_macreg_read(d, E1000E_TDT);
+    uint32_t len = e1000e_macreg_read(d, E1000E_TDLEN) / E1000E_TXD_LEN;
+
+    memwrite(d->tx_ring + tail * E1000E_TXD_LEN, descr, E1000E_TXD_LEN);
+    e1000e_macreg_write(d, E1000E_TDT, (tail + 1) % len);
+
+    /* Read WB data for the packet transmitted */
+    memread(d->tx_ring + tail * E1000E_TXD_LEN, descr, E1000E_TXD_LEN);
+}
+
+static void e1000e_rx_ring_push(e1000e_device *d, void *descr)
+{
+    uint32_t tail = e1000e_macreg_read(d, E1000E_RDT);
+    uint32_t len = e1000e_macreg_read(d, E1000E_RDLEN) / E1000E_RXD_LEN;
+
+    memwrite(d->rx_ring + tail * E1000E_RXD_LEN, descr, E1000E_RXD_LEN);
+    e1000e_macreg_write(d, E1000E_RDT, (tail + 1) % len);
+
+    /* Read WB data for the packet received */
+    memread(d->rx_ring + tail * E1000E_RXD_LEN, descr, E1000E_RXD_LEN);
+}
+
+static void e1000e_wait_isr(e1000e_device *d, uint16_t msg_id)
+{
+    guint64 end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+
+    do {
+        if (qpci_msix_pending(d->pci_dev, msg_id)) {
+            return;
+        }
+        clock_step(10000);
+    } while (g_get_monotonic_time() < end_time);
+
+    g_error("Timeout expired");
+}
+
+static void e1000e_send_verify(e1000e_device *d)
+{
+    struct {
+        uint64_t buffer_addr;
+        union {
+            uint32_t data;
+            struct {
+                uint16_t length;
+                uint8_t cso;
+                uint8_t cmd;
+            } flags;
+        } lower;
+        union {
+            uint32_t data;
+            struct {
+                uint8_t status;
+                uint8_t css;
+                uint16_t special;
+            } fields;
+        } upper;
+    } descr;
+
+    static const uint32_t dtyp_data = BIT(20);
+    static const uint32_t dtyp_ext  = BIT(29);
+    static const uint32_t dcmd_rs   = BIT(27);
+    static const uint32_t dcmd_eop  = BIT(24);
+    static const uint32_t dsta_dd   = BIT(0);
+    static const int data_len = 64;
+    char buffer[64];
+    int ret;
+    uint32_t recv_len;
+
+    /* Prepare test data buffer */
+    uint64_t data = guest_alloc(test_alloc, data_len);
+    memwrite(data, "TEST", 5);
+
+    /* Prepare TX descriptor */
+    memset(&descr, 0, sizeof(descr));
+    descr.buffer_addr = cpu_to_le64(data);
+    descr.lower.data = cpu_to_le32(dcmd_rs   |
+                                   dcmd_eop  |
+                                   dtyp_ext  |
+                                   dtyp_data |
+                                   data_len);
+
+    /* Put descriptor to the ring */
+    e1000e_tx_ring_push(d, &descr);
+
+    /* Wait for TX WB interrupt */
+    e1000e_wait_isr(d, E1000E_TX0_MSG_ID);
+
+    /* Check DD bit */
+    g_assert_cmphex(le32_to_cpu(descr.upper.data) & dsta_dd, ==, dsta_dd);
+
+    /* Check data sent to the backend */
+    ret = qemu_recv(test_sockets[0], &recv_len, sizeof(recv_len), 0);
+    g_assert_cmpint(ret, == , sizeof(recv_len));
+    qemu_recv(test_sockets[0], buffer, 64, 0);
+    g_assert_cmpstr(buffer, == , "TEST");
+
+    /* Free test data buffer */
+    guest_free(test_alloc, data);
+}
+
+static void e1000e_receive_verify(e1000e_device *d)
+{
+    union {
+        struct {
+            uint64_t buffer_addr;
+            uint64_t reserved;
+        } read;
+        struct {
+            struct {
+                uint32_t mrq;
+                union {
+                    uint32_t rss;
+                    struct {
+                        uint16_t ip_id;
+                        uint16_t csum;
+                    } csum_ip;
+                } hi_dword;
+            } lower;
+            struct {
+                uint32_t status_error;
+                uint16_t length;
+                uint16_t vlan;
+            } upper;
+        } wb;
+    } descr;
+
+    static const uint32_t esta_dd = BIT(0);
+
+    char test[] = "TEST";
+    int len = htonl(sizeof(test));
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        },{
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+
+    static const int data_len = 64;
+    char buffer[64];
+    int ret;
+
+    /* Send a dummy packet to device's socket*/
+    ret = iov_send(test_sockets[0], iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, == , sizeof(test) + sizeof(len));
+
+    /* Prepare test data buffer */
+    uint64_t data = guest_alloc(test_alloc, data_len);
+
+    /* Prepare RX descriptor */
+    memset(&descr, 0, sizeof(descr));
+    descr.read.buffer_addr = cpu_to_le64(data);
+
+    /* Put descriptor to the ring */
+    e1000e_rx_ring_push(d, &descr);
+
+    /* Wait for TX WB interrupt */
+    e1000e_wait_isr(d, E1000E_RX0_MSG_ID);
+
+    /* Check DD bit */
+    g_assert_cmphex(le32_to_cpu(descr.wb.upper.status_error) &
+        esta_dd, ==, esta_dd);
+
+    /* Check data sent to the backend */
+    memread(data, buffer, sizeof(buffer));
+    g_assert_cmpstr(buffer, == , "TEST");
+
+    /* Free test data buffer */
+    guest_free(test_alloc, data);
+}
+
+static void e1000e_device_clear(QPCIBus *bus, e1000e_device *d)
+{
+    qpci_iounmap(d->pci_dev, d->mac_regs);
+    qpci_msix_disable(d->pci_dev);
+}
+
+static void data_test_init(e1000e_device *d)
+{
+    char *cmdline;
+
+    int ret = socketpair(PF_UNIX, SOCK_STREAM, 0, test_sockets);
+    g_assert_cmpint(ret, != , -1);
+
+    cmdline = g_strdup_printf("-netdev socket,fd=%d,id=hs0 "
+                              "-device e1000e,netdev=hs0", test_sockets[1]);
+    g_assert_nonnull(cmdline);
+
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    test_bus = qpci_init_pc();
+    g_assert_nonnull(test_bus);
+
+    test_alloc = pc_alloc_init();
+    g_assert_nonnull(test_alloc);
+
+    e1000e_device_init(test_bus, d);
+}
+
+static void data_test_clear(e1000e_device *d)
+{
+    e1000e_device_clear(test_bus, d);
+    close(test_sockets[0]);
+    pc_alloc_uninit(test_alloc);
+    qpci_free_pc(test_bus);
+    qtest_end();
+}
+
+static void test_e1000e_init(gconstpointer data)
+{
+    e1000e_device d;
+
+    data_test_init(&d);
+    data_test_clear(&d);
+}
+
+static void test_e1000e_tx(gconstpointer data)
+{
+    e1000e_device d;
+
+    data_test_init(&d);
+    e1000e_send_verify(&d);
+    data_test_clear(&d);
+}
+
+static void test_e1000e_rx(gconstpointer data)
+{
+    e1000e_device d;
+
+    data_test_init(&d);
+    e1000e_receive_verify(&d);
+    data_test_clear(&d);
+}
+
+static void test_e1000e_multiple_transfers(gconstpointer data)
+{
+    static const long iterations = 4 * 1024;
+    long i;
+
+    e1000e_device d;
+
+    data_test_init(&d);
+
+    for (i = 0; i < iterations; i++) {
+        e1000e_send_verify(&d);
+        e1000e_receive_verify(&d);
+    }
+
+    data_test_clear(&d);
+}
+
+static void test_e1000e_hotplug(gconstpointer data)
+{
+    static const uint8_t slot = 0x06;
+
+    qtest_start("-device e1000e");
+
+    qpci_plug_device_test("e1000e", "e1000e_net", slot, NULL);
+    qpci_unplug_acpi_device_test("e1000e_net", slot);
+
+    qtest_end();
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_data_func("e1000e/init", NULL, test_e1000e_init);
+    qtest_add_data_func("e1000e/tx", NULL, test_e1000e_tx);
+    qtest_add_data_func("e1000e/rx", NULL, test_e1000e_rx);
+    qtest_add_data_func("e1000e/multiple_transfers", NULL,
+        test_e1000e_multiple_transfers);
+    qtest_add_data_func("e1000e/hotplug", NULL, test_e1000e_hotplug);
+
+    return g_test_run();
+}
diff --git a/tests/eepro100-test.c b/tests/eepro100-test.c
index bf82526..ed23258 100644
--- a/tests/eepro100-test.c
+++ b/tests/eepro100-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 static void test_device(gconstpointer data)
 {
@@ -54,9 +52,8 @@
     for (i = 0; i < ARRAY_SIZE(models); i++) {
         char *path;
 
-        path = g_strdup_printf("/%s/eepro100/%s",
-                               qtest_get_arch(), models[i]);
-        g_test_add_data_func(path, models[i], test_device);
+        path = g_strdup_printf("eepro100/%s", models[i]);
+        qtest_add_data_func(path, models[i], test_device);
     }
 
     return g_test_run();
diff --git a/tests/endianness-test.c b/tests/endianness-test.c
index 92e17d2..b7a120e 100644
--- a/tests/endianness-test.c
+++ b/tests/endianness-test.c
@@ -11,11 +11,7 @@
  *
  */
 
-#include <glib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 #include "qemu/bswap.h"
@@ -31,8 +27,6 @@
 
 static const TestCase test_cases[] = {
     { "i386", "pc", -1 },
-    { "mips", "magnum", 0x90000000, .bswap = true },
-    { "mips", "pica61", 0x90000000, .bswap = true },
     { "mips", "mips", 0x14000000, .bswap = true },
     { "mips", "malta", 0x10000000, .bswap = true },
     { "mips64", "magnum", 0x90000000, .bswap = true },
@@ -288,7 +282,6 @@
 int main(int argc, char **argv)
 {
     const char *arch = qtest_get_arch();
-    int ret;
     int i;
 
     g_test_init(&argc, &argv, NULL);
@@ -298,20 +291,18 @@
         if (strcmp(test_cases[i].arch, arch) != 0) {
             continue;
         }
-        path = g_strdup_printf("/%s/endianness/%s",
-                               arch, test_cases[i].machine);
-        g_test_add_data_func(path, &test_cases[i], test_endianness);
+        path = g_strdup_printf("endianness/%s",
+                               test_cases[i].machine);
+        qtest_add_data_func(path, &test_cases[i], test_endianness);
 
-        path = g_strdup_printf("/%s/endianness/split/%s",
-                               arch, test_cases[i].machine);
-        g_test_add_data_func(path, &test_cases[i], test_endianness_split);
+        path = g_strdup_printf("endianness/split/%s",
+                               test_cases[i].machine);
+        qtest_add_data_func(path, &test_cases[i], test_endianness_split);
 
-        path = g_strdup_printf("/%s/endianness/combine/%s",
-                               arch, test_cases[i].machine);
-        g_test_add_data_func(path, &test_cases[i], test_endianness_combine);
+        path = g_strdup_printf("endianness/combine/%s",
+                               test_cases[i].machine);
+        qtest_add_data_func(path, &test_cases[i], test_endianness_combine);
     }
 
-    ret = g_test_run();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/es1370-test.c b/tests/es1370-test.c
index cc23fb5..199fe19 100644
--- a/tests/es1370-test.c
+++ b/tests/es1370-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/fdc-test.c b/tests/fdc-test.c
index 203074c..738c6b4 100644
--- a/tests/fdc-test.c
+++ b/tests/fdc-test.c
@@ -22,11 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
-#include <glib.h>
 
 #include "libqtest.h"
 #include "qemu-common.h"
@@ -218,6 +215,10 @@
         inb(FLOPPY_BASE + reg_fifo);
     }
 
+    msr = inb(FLOPPY_BASE + reg_msr);
+    assert_bit_set(msr, BUSY | RQM | DIO);
+    g_assert(get_irq(FLOPPY_IRQ));
+
     st0 = floppy_recv();
     if (st0 != expected_st0) {
         ret = 1;
@@ -228,8 +229,15 @@
     floppy_recv();
     floppy_recv();
     floppy_recv();
+    g_assert(get_irq(FLOPPY_IRQ));
     floppy_recv();
 
+    /* Check that we're back in command phase */
+    msr = inb(FLOPPY_BASE + reg_msr);
+    assert_bit_clear(msr, BUSY | DIO);
+    assert_bit_set(msr, RQM);
+    g_assert(!get_irq(FLOPPY_IRQ));
+
     return ret;
 }
 
@@ -256,7 +264,7 @@
     uint8_t cmos;
 
     cmos = cmos_read(CMOS_FLOPPY);
-    g_assert(cmos == 0x40);
+    g_assert(cmos == 0x40 || cmos == 0x50);
 }
 
 static void test_no_media_on_start(void)
@@ -291,11 +299,8 @@
     /* Insert media in drive. DSKCHK should not be reset until a step pulse
      * is sent. */
     qmp_discard_response("{'execute':'change', 'arguments':{"
-                         " 'device':'floppy0', 'target': %s }}",
+                         " 'device':'floppy0', 'target': %s, 'arg': 'raw' }}",
                          test_image);
-    qmp_discard_response(""); /* ignore event
-                                 (FIXME open -> open transition?!) */
-    qmp_discard_response(""); /* ignore event */
 
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_set(dir, DSKCHG);
@@ -326,7 +331,6 @@
      * reset the bit. */
     qmp_discard_response("{'execute':'eject', 'arguments':{"
                          " 'device':'floppy0' }}");
-    qmp_discard_response(""); /* ignore event */
 
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_set(dir, DSKCHG);
@@ -403,6 +407,7 @@
     uint8_t head = 0;
     uint8_t cyl;
     uint8_t st0;
+    uint8_t msr;
 
     /* Seek to track 0 and check with READ ID */
     send_seek(0);
@@ -411,18 +416,29 @@
     g_assert(!get_irq(FLOPPY_IRQ));
     floppy_send(head << 2 | drive);
 
+    msr = inb(FLOPPY_BASE + reg_msr);
+    if (!get_irq(FLOPPY_IRQ)) {
+        assert_bit_set(msr, BUSY);
+        assert_bit_clear(msr, RQM);
+    }
+
     while (!get_irq(FLOPPY_IRQ)) {
         /* qemu involves a timer with READ ID... */
         clock_step(1000000000LL / 50);
     }
 
+    msr = inb(FLOPPY_BASE + reg_msr);
+    assert_bit_set(msr, BUSY | RQM | DIO);
+
     st0 = floppy_recv();
     floppy_recv();
     floppy_recv();
     cyl = floppy_recv();
     head = floppy_recv();
     floppy_recv();
+    g_assert(get_irq(FLOPPY_IRQ));
     floppy_recv();
+    g_assert(!get_irq(FLOPPY_IRQ));
 
     g_assert_cmpint(cyl, ==, 0);
     g_assert_cmpint(head, ==, 0);
@@ -443,18 +459,29 @@
     g_assert(!get_irq(FLOPPY_IRQ));
     floppy_send(head << 2 | drive);
 
+    msr = inb(FLOPPY_BASE + reg_msr);
+    if (!get_irq(FLOPPY_IRQ)) {
+        assert_bit_set(msr, BUSY);
+        assert_bit_clear(msr, RQM);
+    }
+
     while (!get_irq(FLOPPY_IRQ)) {
         /* qemu involves a timer with READ ID... */
         clock_step(1000000000LL / 50);
     }
 
+    msr = inb(FLOPPY_BASE + reg_msr);
+    assert_bit_set(msr, BUSY | RQM | DIO);
+
     st0 = floppy_recv();
     floppy_recv();
     floppy_recv();
     cyl = floppy_recv();
     head = floppy_recv();
     floppy_recv();
+    g_assert(get_irq(FLOPPY_IRQ));
     floppy_recv();
+    g_assert(!get_irq(FLOPPY_IRQ));
 
     g_assert_cmpint(cyl, ==, 8);
     g_assert_cmpint(head, ==, 1);
diff --git a/tests/fw_cfg-test.c b/tests/fw_cfg-test.c
index 5c8f8d6..688342b 100644
--- a/tests/fw_cfg-test.c
+++ b/tests/fw_cfg-test.c
@@ -10,12 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
-#define NO_QEMU_PROTOS
-#include "hw/nvram/fw_cfg.h"
+#include "hw/nvram/fw_cfg_keys.h"
 #include "libqos/fw_cfg.h"
 
 static uint64_t ram_size = 128 << 20;
@@ -37,7 +35,9 @@
 
 static void test_fw_cfg_id(void)
 {
-    g_assert_cmpint(qfw_cfg_get_u32(fw_cfg, FW_CFG_ID), ==, 1);
+    uint32_t id = qfw_cfg_get_u32(fw_cfg, FW_CFG_ID);
+    g_assert((id == 1) ||
+             (id == 3));
 }
 
 static void test_fw_cfg_uuid(void)
@@ -109,21 +109,21 @@
 
     fw_cfg = pc_fw_cfg_init();
 
-    g_test_add_func("/fw_cfg/signature", test_fw_cfg_signature);
-    g_test_add_func("/fw_cfg/id", test_fw_cfg_id);
-    g_test_add_func("/fw_cfg/uuid", test_fw_cfg_uuid);
-    g_test_add_func("/fw_cfg/ram_size", test_fw_cfg_ram_size);
-    g_test_add_func("/fw_cfg/nographic", test_fw_cfg_nographic);
-    g_test_add_func("/fw_cfg/nb_cpus", test_fw_cfg_nb_cpus);
+    qtest_add_func("fw_cfg/signature", test_fw_cfg_signature);
+    qtest_add_func("fw_cfg/id", test_fw_cfg_id);
+    qtest_add_func("fw_cfg/uuid", test_fw_cfg_uuid);
+    qtest_add_func("fw_cfg/ram_size", test_fw_cfg_ram_size);
+    qtest_add_func("fw_cfg/nographic", test_fw_cfg_nographic);
+    qtest_add_func("fw_cfg/nb_cpus", test_fw_cfg_nb_cpus);
 #if 0
-    g_test_add_func("/fw_cfg/machine_id", test_fw_cfg_machine_id);
-    g_test_add_func("/fw_cfg/kernel", test_fw_cfg_kernel);
-    g_test_add_func("/fw_cfg/initrd", test_fw_cfg_initrd);
-    g_test_add_func("/fw_cfg/boot_device", test_fw_cfg_boot_device);
+    qtest_add_func("fw_cfg/machine_id", test_fw_cfg_machine_id);
+    qtest_add_func("fw_cfg/kernel", test_fw_cfg_kernel);
+    qtest_add_func("fw_cfg/initrd", test_fw_cfg_initrd);
+    qtest_add_func("fw_cfg/boot_device", test_fw_cfg_boot_device);
 #endif
-    g_test_add_func("/fw_cfg/max_cpus", test_fw_cfg_max_cpus);
-    g_test_add_func("/fw_cfg/numa", test_fw_cfg_numa);
-    g_test_add_func("/fw_cfg/boot_menu", test_fw_cfg_boot_menu);
+    qtest_add_func("fw_cfg/max_cpus", test_fw_cfg_max_cpus);
+    qtest_add_func("fw_cfg/numa", test_fw_cfg_numa);
+    qtest_add_func("fw_cfg/boot_menu", test_fw_cfg_boot_menu);
 
     cmdline = g_strdup_printf("-uuid 4600cb32-38ec-4b2f-8acb-81c6ea54f2d8 ");
     s = qtest_start(cmdline);
diff --git a/tests/guest-debug/test-gdbstub.py b/tests/guest-debug/test-gdbstub.py
new file mode 100644
index 0000000..31ba6c9
--- /dev/null
+++ b/tests/guest-debug/test-gdbstub.py
@@ -0,0 +1,176 @@
+#
+# This script needs to be run on startup
+# qemu -kernel ${KERNEL} -s -S
+# and then:
+# gdb ${KERNEL}.vmlinux -x ${QEMU_SRC}/tests/guest-debug/test-gdbstub.py
+
+import gdb
+
+failcount = 0
+
+
+def report(cond, msg):
+    "Report success/fail of test"
+    if cond:
+        print ("PASS: %s" % (msg))
+    else:
+        print ("FAIL: %s" % (msg))
+        failcount += 1
+
+
+def check_step():
+    "Step an instruction, check it moved."
+    start_pc = gdb.parse_and_eval('$pc')
+    gdb.execute("si")
+    end_pc = gdb.parse_and_eval('$pc')
+
+    return not (start_pc == end_pc)
+
+
+def check_break(sym_name):
+    "Setup breakpoint, continue and check we stopped."
+    sym, ok = gdb.lookup_symbol(sym_name)
+    bp = gdb.Breakpoint(sym_name)
+
+    gdb.execute("c")
+
+    # hopefully we came back
+    end_pc = gdb.parse_and_eval('$pc')
+    print ("%s == %s %d" % (end_pc, sym.value(), bp.hit_count))
+    bp.delete()
+
+    # can we test we hit bp?
+    return end_pc == sym.value()
+
+
+# We need to do hbreak manually as the python interface doesn't export it
+def check_hbreak(sym_name):
+    "Setup hardware breakpoint, continue and check we stopped."
+    sym, ok = gdb.lookup_symbol(sym_name)
+    gdb.execute("hbreak %s" % (sym_name))
+    gdb.execute("c")
+
+    # hopefully we came back
+    end_pc = gdb.parse_and_eval('$pc')
+    print ("%s == %s" % (end_pc, sym.value()))
+
+    if end_pc == sym.value():
+        gdb.execute("d 1")
+        return True
+    else:
+        return False
+
+
+class WatchPoint(gdb.Breakpoint):
+
+    def get_wpstr(self, sym_name):
+        "Setup sym and wp_str for given symbol."
+        self.sym, ok = gdb.lookup_symbol(sym_name)
+        wp_addr = gdb.parse_and_eval(sym_name).address
+        self.wp_str = '*(%(type)s)(&%(address)s)' % dict(
+            type = wp_addr.type, address = sym_name)
+
+        return(self.wp_str)
+
+    def __init__(self, sym_name, type):
+        wp_str = self.get_wpstr(sym_name)
+        super(WatchPoint, self).__init__(wp_str, gdb.BP_WATCHPOINT, type)
+
+    def stop(self):
+        end_pc = gdb.parse_and_eval('$pc')
+        print ("HIT WP @ %s" % (end_pc))
+        return True
+
+
+def do_one_watch(sym, wtype, text):
+
+    wp = WatchPoint(sym, wtype)
+    gdb.execute("c")
+    report_str = "%s for %s (%s)" % (text, sym, wp.sym.value())
+
+    if wp.hit_count > 0:
+        report(True, report_str)
+        wp.delete()
+    else:
+        report(False, report_str)
+
+
+def check_watches(sym_name):
+    "Watch a symbol for any access."
+
+    # Should hit for any read
+    do_one_watch(sym_name, gdb.WP_ACCESS, "awatch")
+
+    # Again should hit for reads
+    do_one_watch(sym_name, gdb.WP_READ, "rwatch")
+
+    # Finally when it is written
+    do_one_watch(sym_name, gdb.WP_WRITE, "watch")
+
+
+class CatchBreakpoint(gdb.Breakpoint):
+    def __init__(self, sym_name):
+        super(CatchBreakpoint, self).__init__(sym_name)
+        self.sym, ok = gdb.lookup_symbol(sym_name)
+
+    def stop(self):
+        end_pc = gdb.parse_and_eval('$pc')
+        print ("CB: %s == %s" % (end_pc, self.sym.value()))
+        if end_pc == self.sym.value():
+            report(False, "Hit final catchpoint")
+
+
+def run_test():
+    "Run throught the tests one by one"
+
+    print ("Checking we can step the first few instructions")
+    step_ok = 0
+    for i in range(3):
+        if check_step():
+            step_ok += 1
+
+    report(step_ok == 3, "single step in boot code")
+
+    print ("Checking HW breakpoint works")
+    break_ok = check_hbreak("kernel_init")
+    report(break_ok, "hbreak @ kernel_init")
+
+    # Can't set this up until we are in the kernel proper
+    # if we make it to run_init_process we've over-run and
+    # one of the tests failed
+    print ("Setup catch-all for run_init_process")
+    cbp = CatchBreakpoint("run_init_process")
+    cpb2 = CatchBreakpoint("try_to_run_init_process")
+
+    print ("Checking Normal breakpoint works")
+    break_ok = check_break("wait_for_completion")
+    report(break_ok, "break @ wait_for_completion")
+
+    print ("Checking watchpoint works")
+    check_watches("system_state")
+
+#
+# This runs as the script it sourced (via -x)
+#
+
+try:
+    print ("Connecting to remote")
+    gdb.execute("target remote localhost:1234")
+
+    # These are not very useful in scripts
+    gdb.execute("set pagination off")
+    gdb.execute("set confirm off")
+
+    # Run the actual tests
+    run_test()
+
+except:
+    print ("GDB Exception: %s" % (sys.exc_info()[0]))
+    failcount += 1
+    import code
+    code.InteractiveConsole(locals=globals()).interact()
+    raise
+
+# Finally kill the inferior and exit gdb with a count of failures
+gdb.execute("kill")
+exit(failcount)
diff --git a/tests/hd-geo-test.c b/tests/hd-geo-test.c
index c84d1e7..12ee392 100644
--- a/tests/hd-geo-test.c
+++ b/tests/hd-geo-test.c
@@ -15,15 +15,10 @@
  * Improvements welcome.
  */
 
-#include <glib.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "libqtest.h"
 
-static const char test_image[] = "/tmp/qtest.XXXXXX";
-
 static char *create_test_img(int secs)
 {
     char *template = strdup("/tmp/qtest.XXXXXX");
@@ -214,7 +209,7 @@
 
     if (img_secs[img_idx] >= 0) {
         setup_mbr(img_idx, mbr);
-        s3 = g_strdup_printf(",file=%s", img_file_name[img_idx]);
+        s3 = g_strdup_printf(",format=raw,file=%s", img_file_name[img_idx]);
     } else {
         s3 = g_strdup(",media=cdrom");
     }
diff --git a/tests/i440fx-test.c b/tests/i440fx-test.c
index ad232b5..3542ad1 100644
--- a/tests/i440fx-test.c
+++ b/tests/i440fx-test.c
@@ -12,13 +12,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <errno.h>
-#include <sys/mman.h>
-#include <stdlib.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 #include "libqos/pci.h"
@@ -27,8 +21,6 @@
 
 #define BROKEN 1
 
-#define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
-
 typedef struct TestData
 {
     int num_cpus;
@@ -191,7 +183,7 @@
     uint32_t size = end - start + 1;
     uint8_t *data;
 
-    data = g_malloc0(size);
+    data = g_malloc(size);
     memset(data, value, size);
     memwrite(start, data, size);
 
@@ -342,8 +334,9 @@
     g_assert(fw_pathname != NULL);
 
     /* Better hope the user didn't put metacharacters in TMPDIR and co. */
-    cmdline = g_strdup_printf("-S %s %s",
-                              fixture->is_bios ? "-bios" : "-pflash",
+    cmdline = g_strdup_printf("-S %s%s", fixture->is_bios
+                                         ? "-bios "
+                                         : "-drive if=pflash,format=raw,file=",
                               fw_pathname);
     g_test_message("qemu cmdline: %s", cmdline);
     qtest_start(cmdline);
@@ -382,8 +375,8 @@
                               void (*setup_fixture)(FirmwareTestFixture *f,
                                                     gconstpointer test_data))
 {
-    g_test_add(testpath, FirmwareTestFixture, NULL, setup_fixture,
-               test_i440fx_firmware, NULL);
+    qtest_add(testpath, FirmwareTestFixture, NULL, setup_fixture,
+              test_i440fx_firmware, NULL);
 }
 
 static void request_bios(FirmwareTestFixture *fixture,
@@ -401,17 +394,15 @@
 int main(int argc, char **argv)
 {
     TestData data;
-    int ret;
 
     g_test_init(&argc, &argv, NULL);
 
     data.num_cpus = 1;
 
-    g_test_add_data_func("/i440fx/defaults", &data, test_i440fx_defaults);
-    g_test_add_data_func("/i440fx/pam", &data, test_i440fx_pam);
-    add_firmware_test("/i440fx/firmware/bios", request_bios);
-    add_firmware_test("/i440fx/firmware/pflash", request_pflash);
+    qtest_add_data_func("i440fx/defaults", &data, test_i440fx_defaults);
+    qtest_add_data_func("i440fx/pam", &data, test_i440fx_pam);
+    add_firmware_test("i440fx/firmware/bios", request_bios);
+    add_firmware_test("i440fx/firmware/pflash", request_pflash);
 
-    ret = g_test_run();
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/i82801b11-test.c b/tests/i82801b11-test.c
index 78d9ce0..a6e3159 100644
--- a/tests/i82801b11-test.c
+++ b/tests/i82801b11-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/ide-test.c b/tests/ide-test.c
index b7a97e9..1e51af2 100644
--- a/tests/ide-test.c
+++ b/tests/ide-test.c
@@ -22,17 +22,16 @@
  * THE SOFTWARE.
  */
 
-#include <stdint.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
-#include <glib.h>
 
 #include "libqtest.h"
+#include "libqos/libqos.h"
 #include "libqos/pci-pc.h"
 #include "libqos/malloc-pc.h"
 
 #include "qemu-common.h"
+#include "qemu/bswap.h"
 #include "hw/pci/pci_ids.h"
 #include "hw/pci/pci_regs.h"
 
@@ -44,8 +43,15 @@
 #define IDE_BASE 0x1f0
 #define IDE_PRIMARY_IRQ 14
 
+#define ATAPI_BLOCK_SIZE 2048
+
+/* How many bytes to receive via ATAPI PIO at one time.
+ * Must be less than 0xFFFF. */
+#define BYTE_COUNT_LIMIT 5120
+
 enum {
     reg_data        = 0x0,
+    reg_feature     = 0x1,
     reg_nsectors    = 0x2,
     reg_lba_low     = 0x3,
     reg_lba_middle  = 0x4,
@@ -79,6 +85,7 @@
     CMD_WRITE_DMA   = 0xca,
     CMD_FLUSH_CACHE = 0xe7,
     CMD_IDENTIFY    = 0xec,
+    CMD_PACKET      = 0xa0,
 
     CMDF_ABORT      = 0x100,
     CMDF_NO_BM      = 0x200,
@@ -118,7 +125,6 @@
     va_end(ap);
 
     qtest_start(cmdline);
-    qtest_irq_intercept_in(global_qtest, "ioapic");
     guest_malloc = pc_alloc_init();
 
     g_free(cmdline);
@@ -172,7 +178,8 @@
 #define assert_bit_clear(data, mask) g_assert_cmphex((data) & (mask), ==, 0)
 
 static int send_dma_request(int cmd, uint64_t sector, int nb_sectors,
-                            PrdtEntry *prdt, int prdt_entries)
+                            PrdtEntry *prdt, int prdt_entries,
+                            void(*post_exec)(uint64_t sector, int nb_sectors))
 {
     QPCIDevice *dev;
     uint16_t bmdma_base;
@@ -189,6 +196,9 @@
 
     switch (cmd) {
     case CMD_READ_DMA:
+    case CMD_PACKET:
+        /* Assuming we only test data reads w/ ATAPI, otherwise we need to know
+         * the SCSI command being sent in the packet, too. */
         from_dev = true;
         break;
     case CMD_WRITE_DMA:
@@ -217,14 +227,22 @@
     outl(bmdma_base + bmreg_prdt, guest_prdt);
 
     /* ATA DMA command */
-    outb(IDE_BASE + reg_nsectors, nb_sectors);
-
-    outb(IDE_BASE + reg_lba_low,    sector & 0xff);
-    outb(IDE_BASE + reg_lba_middle, (sector >> 8) & 0xff);
-    outb(IDE_BASE + reg_lba_high,   (sector >> 16) & 0xff);
+    if (cmd == CMD_PACKET) {
+        /* Enables ATAPI DMA; otherwise PIO is attempted */
+        outb(IDE_BASE + reg_feature, 0x01);
+    } else {
+        outb(IDE_BASE + reg_nsectors, nb_sectors);
+        outb(IDE_BASE + reg_lba_low,    sector & 0xff);
+        outb(IDE_BASE + reg_lba_middle, (sector >> 8) & 0xff);
+        outb(IDE_BASE + reg_lba_high,   (sector >> 16) & 0xff);
+    }
 
     outb(IDE_BASE + reg_command, cmd);
 
+    if (post_exec) {
+        post_exec(sector, nb_sectors);
+    }
+
     /* Start DMA transfer */
     outb(bmdma_base + bmreg_cmd, BM_CMD_START | (from_dev ? BM_CMD_WRITE : 0));
 
@@ -278,7 +296,8 @@
     memset(buf, 0x55, len);
     memwrite(guest_buf, buf, len);
 
-    status = send_dma_request(CMD_WRITE_DMA, 0, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_WRITE_DMA, 0, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -286,14 +305,15 @@
     memset(buf, 0xaa, len);
     memwrite(guest_buf, buf, len);
 
-    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Read and verify 0x55 pattern in sector 0 */
     memset(cmpbuf, 0x55, len);
 
-    status = send_dma_request(CMD_READ_DMA, 0, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_READ_DMA, 0, 1, prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -303,7 +323,7 @@
     /* Read and verify 0xaa pattern in sector 1 */
     memset(cmpbuf, 0xaa, len);
 
-    status = send_dma_request(CMD_READ_DMA, 1, 1, prdt, ARRAY_SIZE(prdt));
+    status = send_dma_request(CMD_READ_DMA, 1, 1, prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
@@ -328,13 +348,38 @@
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
+    g_assert_cmphex(status, ==, 0);
+    assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
+}
+
+static void test_bmdma_one_sector_short_prdt(void)
+{
+    uint8_t status;
+
+    /* Read 2 sectors but only give 1 sector in PRDT */
+    PrdtEntry prdt[] = {
+        {
+            .addr = 0,
+            .size = cpu_to_le32(0x200 | PRDT_EOT),
+        },
+    };
+
+    /* Normal request */
+    status = send_dma_request(CMD_READ_DMA, 0, 2,
+                              prdt, ARRAY_SIZE(prdt), NULL);
+    g_assert_cmphex(status, ==, 0);
+    assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
+
+    /* Abort the request before it completes */
+    status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 2,
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, 0);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -352,13 +397,13 @@
 
     /* Normal request */
     status = send_dma_request(CMD_READ_DMA, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_ACTIVE | BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 
     /* Abort the request before it completes */
     status = send_dma_request(CMD_READ_DMA | CMDF_ABORT, 0, 1,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
     g_assert_cmphex(status, ==, BM_STS_INTR);
     assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
 }
@@ -374,7 +419,7 @@
     PrdtEntry prdt[4096] = { };
 
     status = send_dma_request(CMD_READ_DMA | CMDF_NO_BM, 0, 512,
-                              prdt, ARRAY_SIZE(prdt));
+                              prdt, ARRAY_SIZE(prdt), NULL);
 
     /* Not entirely clear what the expected result is, but this is what we get
      * in practice. At least we want to be aware of any changes. */
@@ -385,9 +430,10 @@
 static void test_bmdma_setup(void)
 {
     ide_test_start(
-        "-drive file=%s,if=ide,serial=%s,cache=writeback "
+        "-drive file=%s,if=ide,serial=%s,cache=writeback,format=raw "
         "-global ide-hd.ver=%s",
         tmp_path, "testdisk", "version");
+    qtest_irq_intercept_in(global_qtest, "ioapic");
 }
 
 static void test_bmdma_teardown(void)
@@ -414,7 +460,7 @@
     int ret;
 
     ide_test_start(
-        "-drive file=%s,if=ide,serial=%s,cache=writeback "
+        "-drive file=%s,if=ide,serial=%s,cache=writeback,format=raw "
         "-global ide-hd.ver=%s",
         tmp_path, "testdisk", "version");
 
@@ -453,18 +499,54 @@
     ide_test_quit();
 }
 
+/*
+ * Write sector 1 with random data to make IDE storage dirty
+ * Needed for flush tests so that flushes actually go though the block layer
+ */
+static void make_dirty(uint8_t device)
+{
+    uint8_t status;
+    size_t len = 512;
+    uintptr_t guest_buf;
+    void* buf;
+
+    guest_buf = guest_alloc(guest_malloc, len);
+    buf = g_malloc(len);
+    g_assert(guest_buf);
+    g_assert(buf);
+
+    memwrite(guest_buf, buf, len);
+
+    PrdtEntry prdt[] = {
+        {
+            .addr = cpu_to_le32(guest_buf),
+            .size = cpu_to_le32(len | PRDT_EOT),
+        },
+    };
+
+    status = send_dma_request(CMD_WRITE_DMA, 1, 1, prdt,
+                              ARRAY_SIZE(prdt), NULL);
+    g_assert_cmphex(status, ==, BM_STS_INTR);
+    assert_bit_clear(inb(IDE_BASE + reg_status), DF | ERR);
+
+    g_free(buf);
+}
+
 static void test_flush(void)
 {
     uint8_t data;
 
     ide_test_start(
-        "-drive file=blkdebug::%s,if=ide,cache=writeback",
+        "-drive file=blkdebug::%s,if=ide,cache=writeback,format=raw",
         tmp_path);
 
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    /* Dirty media so that CMD_FLUSH_CACHE will actually go to disk */
+    make_dirty(0);
+
     /* Delay the completion of the flush request until we explicitly do it */
-    qmp_discard_response("{'execute':'human-monitor-command', 'arguments': {"
-                         " 'command-line':"
-                         " 'qemu-io ide0-hd0 \"break flush_to_os A\"'} }");
+    g_free(hmp("qemu-io ide0-hd0 \"break flush_to_os A\""));
 
     /* FLUSH CACHE command on device 0*/
     outb(IDE_BASE + reg_device, 0);
@@ -476,9 +558,7 @@
     assert_bit_clear(data, DF | ERR | DRQ);
 
     /* Complete the command */
-    qmp_discard_response("{'execute':'human-monitor-command', 'arguments': {"
-                         " 'command-line':"
-                         " 'qemu-io ide0-hd0 \"resume A\"'} }");
+    g_free(hmp("qemu-io ide0-hd0 \"resume A\""));
 
     /* Check registers */
     data = inb(IDE_BASE + reg_device);
@@ -494,41 +574,24 @@
     ide_test_quit();
 }
 
-static void prepare_blkdebug_script(const char *debug_fn, const char *event)
-{
-    FILE *debug_file = fopen(debug_fn, "w");
-    int ret;
-
-    fprintf(debug_file, "[inject-error]\n");
-    fprintf(debug_file, "event = \"%s\"\n", event);
-    fprintf(debug_file, "errno = \"5\"\n");
-    fprintf(debug_file, "state = \"1\"\n");
-    fprintf(debug_file, "immediately = \"off\"\n");
-    fprintf(debug_file, "once = \"on\"\n");
-
-    fprintf(debug_file, "[set-state]\n");
-    fprintf(debug_file, "event = \"%s\"\n", event);
-    fprintf(debug_file, "new_state = \"2\"\n");
-    fflush(debug_file);
-    g_assert(!ferror(debug_file));
-
-    ret = fclose(debug_file);
-    g_assert(ret == 0);
-}
-
-static void test_retry_flush(void)
+static void test_retry_flush(const char *machine)
 {
     uint8_t data;
     const char *s;
-    QDict *response;
 
     prepare_blkdebug_script(debug_path, "flush_to_disk");
 
     ide_test_start(
         "-vnc none "
-        "-drive file=blkdebug:%s:%s,if=ide,cache=writeback,rerror=stop,werror=stop",
+        "-drive file=blkdebug:%s:%s,if=ide,cache=writeback,format=raw,"
+        "rerror=stop,werror=stop",
         debug_path, tmp_path);
 
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    /* Dirty media so that CMD_FLUSH_CACHE will actually go to disk */
+    make_dirty(0);
+
     /* FLUSH CACHE command on device 0*/
     outb(IDE_BASE + reg_device, 0);
     outb(IDE_BASE + reg_command, CMD_FLUSH_CACHE);
@@ -538,15 +601,7 @@
     assert_bit_set(data, BSY | DRDY);
     assert_bit_clear(data, DF | ERR | DRQ);
 
-    for (;; response = NULL) {
-        response = qmp_receive();
-        if ((qdict_haskey(response, "event")) &&
-            (strcmp(qdict_get_str(response, "event"), "STOP") == 0)) {
-            QDECREF(response);
-            break;
-        }
-        QDECREF(response);
-    }
+    qmp_eventwait("STOP");
 
     /* Complete the command */
     s = "{'execute':'cont' }";
@@ -579,6 +634,215 @@
     ide_test_quit();
 }
 
+static void test_pci_retry_flush(void)
+{
+    test_retry_flush("pc");
+}
+
+static void test_isa_retry_flush(void)
+{
+    test_retry_flush("isapc");
+}
+
+typedef struct Read10CDB {
+    uint8_t opcode;
+    uint8_t flags;
+    uint32_t lba;
+    uint8_t reserved;
+    uint16_t nblocks;
+    uint8_t control;
+    uint16_t padding;
+} __attribute__((__packed__)) Read10CDB;
+
+static void send_scsi_cdb_read10(uint64_t lba, int nblocks)
+{
+    Read10CDB pkt = { .padding = 0 };
+    int i;
+
+    g_assert_cmpint(lba, <=, UINT32_MAX);
+    g_assert_cmpint(nblocks, <=, UINT16_MAX);
+    g_assert_cmpint(nblocks, >=, 0);
+
+    /* Construct SCSI CDB packet */
+    pkt.opcode = 0x28;
+    pkt.lba = cpu_to_be32(lba);
+    pkt.nblocks = cpu_to_be16(nblocks);
+
+    /* Send Packet */
+    for (i = 0; i < sizeof(Read10CDB)/2; i++) {
+        outw(IDE_BASE + reg_data, cpu_to_le16(((uint16_t *)&pkt)[i]));
+    }
+}
+
+static void nsleep(int64_t nsecs)
+{
+    const struct timespec val = { .tv_nsec = nsecs };
+    nanosleep(&val, NULL);
+    clock_set(nsecs);
+}
+
+static uint8_t ide_wait_clear(uint8_t flag)
+{
+    uint8_t data;
+    time_t st;
+
+    /* Wait with a 5 second timeout */
+    time(&st);
+    while (true) {
+        data = inb(IDE_BASE + reg_status);
+        if (!(data & flag)) {
+            return data;
+        }
+        if (difftime(time(NULL), st) > 5.0) {
+            break;
+        }
+        nsleep(400);
+    }
+    g_assert_not_reached();
+}
+
+static void ide_wait_intr(int irq)
+{
+    time_t st;
+    bool intr;
+
+    time(&st);
+    while (true) {
+        intr = get_irq(irq);
+        if (intr) {
+            return;
+        }
+        if (difftime(time(NULL), st) > 5.0) {
+            break;
+        }
+        nsleep(400);
+    }
+
+    g_assert_not_reached();
+}
+
+static void cdrom_pio_impl(int nblocks)
+{
+    FILE *fh;
+    int patt_blocks = MAX(16, nblocks);
+    size_t patt_len = ATAPI_BLOCK_SIZE * patt_blocks;
+    char *pattern = g_malloc(patt_len);
+    size_t rxsize = ATAPI_BLOCK_SIZE * nblocks;
+    uint16_t *rx = g_malloc0(rxsize);
+    int i, j;
+    uint8_t data;
+    uint16_t limit;
+
+    /* Prepopulate the CDROM with an interesting pattern */
+    generate_pattern(pattern, patt_len, ATAPI_BLOCK_SIZE);
+    fh = fopen(tmp_path, "w+");
+    fwrite(pattern, ATAPI_BLOCK_SIZE, patt_blocks, fh);
+    fclose(fh);
+
+    ide_test_start("-drive if=none,file=%s,media=cdrom,format=raw,id=sr0,index=0 "
+                   "-device ide-cd,drive=sr0,bus=ide.0", tmp_path);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    /* PACKET command on device 0 */
+    outb(IDE_BASE + reg_device, 0);
+    outb(IDE_BASE + reg_lba_middle, BYTE_COUNT_LIMIT & 0xFF);
+    outb(IDE_BASE + reg_lba_high, (BYTE_COUNT_LIMIT >> 8 & 0xFF));
+    outb(IDE_BASE + reg_command, CMD_PACKET);
+    /* HP0: Check_Status_A State */
+    nsleep(400);
+    data = ide_wait_clear(BSY);
+    /* HP1: Send_Packet State */
+    assert_bit_set(data, DRQ | DRDY);
+    assert_bit_clear(data, ERR | DF | BSY);
+
+    /* SCSI CDB (READ10) -- read n*2048 bytes from block 0 */
+    send_scsi_cdb_read10(0, nblocks);
+
+    /* Read data back: occurs in bursts of 'BYTE_COUNT_LIMIT' bytes.
+     * If BYTE_COUNT_LIMIT is odd, we transfer BYTE_COUNT_LIMIT - 1 bytes.
+     * We allow an odd limit only when the remaining transfer size is
+     * less than BYTE_COUNT_LIMIT. However, SCSI's read10 command can only
+     * request n blocks, so our request size is always even.
+     * For this reason, we assume there is never a hanging byte to fetch. */
+    g_assert(!(rxsize & 1));
+    limit = BYTE_COUNT_LIMIT & ~1;
+    for (i = 0; i < DIV_ROUND_UP(rxsize, limit); i++) {
+        size_t offset = i * (limit / 2);
+        size_t rem = (rxsize / 2) - offset;
+
+        /* HP3: INTRQ_Wait */
+        ide_wait_intr(IDE_PRIMARY_IRQ);
+
+        /* HP2: Check_Status_B (and clear IRQ) */
+        data = ide_wait_clear(BSY);
+        assert_bit_set(data, DRQ | DRDY);
+        assert_bit_clear(data, ERR | DF | BSY);
+
+        /* HP4: Transfer_Data */
+        for (j = 0; j < MIN((limit / 2), rem); j++) {
+            rx[offset + j] = le16_to_cpu(inw(IDE_BASE + reg_data));
+        }
+    }
+
+    /* Check for final completion IRQ */
+    ide_wait_intr(IDE_PRIMARY_IRQ);
+
+    /* Sanity check final state */
+    data = ide_wait_clear(DRQ);
+    assert_bit_set(data, DRDY);
+    assert_bit_clear(data, DRQ | ERR | DF | BSY);
+
+    g_assert_cmpint(memcmp(pattern, rx, rxsize), ==, 0);
+    g_free(pattern);
+    g_free(rx);
+    test_bmdma_teardown();
+}
+
+static void test_cdrom_pio(void)
+{
+    cdrom_pio_impl(1);
+}
+
+static void test_cdrom_pio_large(void)
+{
+    /* Test a few loops of the PIO DRQ mechanism. */
+    cdrom_pio_impl(BYTE_COUNT_LIMIT * 4 / ATAPI_BLOCK_SIZE);
+}
+
+
+static void test_cdrom_dma(void)
+{
+    static const size_t len = ATAPI_BLOCK_SIZE;
+    char *pattern = g_malloc(ATAPI_BLOCK_SIZE * 16);
+    char *rx = g_malloc0(len);
+    uintptr_t guest_buf;
+    PrdtEntry prdt[1];
+    FILE *fh;
+
+    ide_test_start("-drive if=none,file=%s,media=cdrom,format=raw,id=sr0,index=0 "
+                   "-device ide-cd,drive=sr0,bus=ide.0", tmp_path);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+
+    guest_buf = guest_alloc(guest_malloc, len);
+    prdt[0].addr = cpu_to_le32(guest_buf);
+    prdt[0].size = cpu_to_le32(len | PRDT_EOT);
+
+    generate_pattern(pattern, ATAPI_BLOCK_SIZE * 16, ATAPI_BLOCK_SIZE);
+    fh = fopen(tmp_path, "w+");
+    fwrite(pattern, ATAPI_BLOCK_SIZE, 16, fh);
+    fclose(fh);
+
+    send_dma_request(CMD_PACKET, 0, 1, prdt, 1, send_scsi_cdb_read10);
+
+    /* Read back data from guest memory into local qtest memory */
+    memread(guest_buf, rx, len);
+    g_assert_cmpint(memcmp(pattern, rx, len), ==, 0);
+
+    g_free(pattern);
+    g_free(rx);
+    test_bmdma_teardown();
+}
+
 int main(int argc, char **argv)
 {
     const char *arch = qtest_get_arch();
@@ -611,14 +875,20 @@
     qtest_add_func("/ide/bmdma/setup", test_bmdma_setup);
     qtest_add_func("/ide/bmdma/simple_rw", test_bmdma_simple_rw);
     qtest_add_func("/ide/bmdma/short_prdt", test_bmdma_short_prdt);
+    qtest_add_func("/ide/bmdma/one_sector_short_prdt",
+                   test_bmdma_one_sector_short_prdt);
     qtest_add_func("/ide/bmdma/long_prdt", test_bmdma_long_prdt);
     qtest_add_func("/ide/bmdma/no_busmaster", test_bmdma_no_busmaster);
     qtest_add_func("/ide/bmdma/teardown", test_bmdma_teardown);
 
     qtest_add_func("/ide/flush", test_flush);
-    qtest_add_func("/ide/flush_nodev", test_flush_nodev);
+    qtest_add_func("/ide/flush/nodev", test_flush_nodev);
+    qtest_add_func("/ide/flush/retry_pci", test_pci_retry_flush);
+    qtest_add_func("/ide/flush/retry_isa", test_isa_retry_flush);
 
-    qtest_add_func("/ide/retry/flush", test_retry_flush);
+    qtest_add_func("/ide/cdrom/pio", test_cdrom_pio);
+    qtest_add_func("/ide/cdrom/pio_large", test_cdrom_pio_large);
+    qtest_add_func("/ide/cdrom/dma", test_cdrom_dma);
 
     ret = g_test_run();
 
diff --git a/tests/image-fuzzer/runner.py b/tests/image-fuzzer/runner.py
index 0a8743e..96a1c11 100755
--- a/tests/image-fuzzer/runner.py
+++ b/tests/image-fuzzer/runner.py
@@ -157,7 +157,7 @@
 
         try:
             os.makedirs(self.current_dir)
-        except OSError, e:
+        except OSError as e:
             print >>sys.stderr, \
                 "Error: The working directory '%s' cannot be used. Reason: %s"\
                 % (self.work_dir, e[1])
@@ -244,7 +244,7 @@
             temp_log = StringIO.StringIO()
             try:
                 retcode = run_app(temp_log, current_cmd)
-            except OSError, e:
+            except OSError as e:
                 multilog("%sError: Start of '%s' failed. Reason: %s\n\n"
                          % (test_summary, os.path.basename(current_cmd[0]),
                             e[1]),
@@ -301,7 +301,7 @@
         JSON:
 
         '--command' accepts a JSON array of commands. Each command presents
-        an application under test with all its paramaters as a list of strings,
+        an application under test with all its parameters as a list of strings,
         e.g. ["qemu-io", "$test_img", "-c", "write $off $len"].
 
         Supported application aliases: 'qemu-img' and 'qemu-io'.
@@ -356,7 +356,7 @@
         opts, args = getopt.gnu_getopt(sys.argv[1:], 'c:hs:kvd:',
                                        ['command=', 'help', 'seed=', 'config=',
                                         'keep_passed', 'verbose', 'duration='])
-    except getopt.error, e:
+    except getopt.error as e:
         print >>sys.stderr, \
             "Error: %s\n\nTry 'runner.py --help' for more information" % e
         sys.exit(1)
@@ -374,7 +374,7 @@
         elif opt in ('-c', '--command'):
             try:
                 command = json.loads(arg)
-            except (TypeError, ValueError, NameError), e:
+            except (TypeError, ValueError, NameError) as e:
                 print >>sys.stderr, \
                     "Error: JSON array of test commands cannot be loaded.\n" \
                     "Reason: %s" % e
@@ -390,7 +390,7 @@
         elif opt == '--config':
             try:
                 config = json.loads(arg)
-            except (TypeError, ValueError, NameError), e:
+            except (TypeError, ValueError, NameError) as e:
                 print >>sys.stderr, \
                     "Error: JSON array with the fuzzer configuration cannot" \
                     " be loaded\nReason: %s" % e
@@ -414,7 +414,7 @@
 
     try:
         image_generator = __import__(generator_name)
-    except ImportError, e:
+    except ImportError as e:
         print >>sys.stderr, \
             "Error: The image generator '%s' cannot be imported.\n" \
             "Reason: %s" % (generator_name, e)
diff --git a/tests/intel-hda-test.c b/tests/intel-hda-test.c
index d89b407..b782b2e 100644
--- a/tests/intel-hda-test.c
+++ b/tests/intel-hda-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 #define HDA_ID "hda0"
 #define CODEC_DEVICES " -device hda-output,bus=" HDA_ID ".0" \
@@ -33,13 +31,9 @@
 
 int main(int argc, char **argv)
 {
-    int ret;
-
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/intel-hda/ich6", ich6_test);
     qtest_add_func("/intel-hda/ich9", ich9_test);
 
-    ret = g_test_run();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/io-channel-helpers.c b/tests/io-channel-helpers.c
new file mode 100644
index 0000000..05e5579
--- /dev/null
+++ b/tests/io-channel-helpers.c
@@ -0,0 +1,248 @@
+/*
+ * QEMU I/O channel test helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+struct QIOChannelTest {
+    QIOChannel *src;
+    QIOChannel *dst;
+    bool blocking;
+    size_t len;
+    size_t niov;
+    char *input;
+    struct iovec *inputv;
+    char *output;
+    struct iovec *outputv;
+    Error *writeerr;
+    Error *readerr;
+};
+
+
+static void test_skip_iovec(struct iovec **iov,
+                            size_t *niov,
+                            size_t skip,
+                            struct iovec *old)
+{
+    size_t offset = 0;
+    size_t i;
+
+    for (i = 0; i < *niov; i++) {
+        if (skip < (*iov)[i].iov_len) {
+            old->iov_len = (*iov)[i].iov_len;
+            old->iov_base = (*iov)[i].iov_base;
+
+            (*iov)[i].iov_len -= skip;
+            (*iov)[i].iov_base += skip;
+            break;
+        } else {
+            skip -= (*iov)[i].iov_len;
+
+            if (i == 0 && old->iov_base) {
+                (*iov)[i].iov_len = old->iov_len;
+                (*iov)[i].iov_base = old->iov_base;
+                old->iov_len = 0;
+                old->iov_base = NULL;
+            }
+
+            offset++;
+        }
+    }
+
+    *iov = *iov + offset;
+    *niov -= offset;
+}
+
+
+/* This thread sends all data using iovecs */
+static gpointer test_io_thread_writer(gpointer opaque)
+{
+    QIOChannelTest *data = opaque;
+    struct iovec *iov = data->inputv;
+    size_t niov = data->niov;
+    struct iovec old = { 0 };
+
+    qio_channel_set_blocking(data->src, data->blocking, NULL);
+
+    while (niov) {
+        ssize_t ret;
+        ret = qio_channel_writev(data->src,
+                                 iov,
+                                 niov,
+                                 &data->writeerr);
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            if (data->blocking) {
+                error_setg(&data->writeerr,
+                           "Unexpected I/O blocking");
+                break;
+            } else {
+                qio_channel_wait(data->src,
+                                 G_IO_OUT);
+                continue;
+            }
+        } else if (ret < 0) {
+            break;
+        } else if (ret == 0) {
+            error_setg(&data->writeerr,
+                       "Unexpected zero length write");
+            break;
+        }
+
+        test_skip_iovec(&iov, &niov, ret, &old);
+    }
+
+    return NULL;
+}
+
+
+/* This thread receives all data using iovecs */
+static gpointer test_io_thread_reader(gpointer opaque)
+{
+    QIOChannelTest *data = opaque;
+    struct iovec *iov = data->outputv;
+    size_t niov = data->niov;
+    struct iovec old = { 0 };
+
+    qio_channel_set_blocking(data->dst, data->blocking, NULL);
+
+    while (niov) {
+        ssize_t ret;
+
+        ret = qio_channel_readv(data->dst,
+                                iov,
+                                niov,
+                                &data->readerr);
+
+        if (ret == QIO_CHANNEL_ERR_BLOCK) {
+            if (data->blocking) {
+                error_setg(&data->readerr,
+                           "Unexpected I/O blocking");
+                break;
+            } else {
+                qio_channel_wait(data->dst,
+                                 G_IO_IN);
+                continue;
+            }
+        } else if (ret < 0) {
+            break;
+        } else if (ret == 0) {
+            break;
+        }
+
+        test_skip_iovec(&iov, &niov, ret, &old);
+    }
+
+    return NULL;
+}
+
+
+QIOChannelTest *qio_channel_test_new(void)
+{
+    QIOChannelTest *data = g_new0(QIOChannelTest, 1);
+    size_t i;
+    size_t offset;
+
+
+    /* We'll send 1 MB of data */
+#define CHUNK_COUNT 250
+#define CHUNK_LEN 4194
+
+    data->len = CHUNK_COUNT * CHUNK_LEN;
+    data->input = g_new0(char, data->len);
+    data->output = g_new0(gchar, data->len);
+
+    /* Fill input with a pattern */
+    for (i = 0; i < data->len; i += CHUNK_LEN) {
+        memset(data->input + i, (i / CHUNK_LEN), CHUNK_LEN);
+    }
+
+    /* We'll split the data across a bunch of IO vecs */
+    data->niov = CHUNK_COUNT;
+    data->inputv = g_new0(struct iovec, data->niov);
+    data->outputv = g_new0(struct iovec, data->niov);
+
+    for (i = 0, offset = 0; i < data->niov; i++, offset += CHUNK_LEN) {
+        data->inputv[i].iov_base = data->input + offset;
+        data->outputv[i].iov_base = data->output + offset;
+        data->inputv[i].iov_len = CHUNK_LEN;
+        data->outputv[i].iov_len = CHUNK_LEN;
+    }
+
+    return data;
+}
+
+void qio_channel_test_run_threads(QIOChannelTest *test,
+                                  bool blocking,
+                                  QIOChannel *src,
+                                  QIOChannel *dst)
+{
+    GThread *reader, *writer;
+
+    test->src = src;
+    test->dst = dst;
+    test->blocking = blocking;
+
+    reader = g_thread_new("reader",
+                          test_io_thread_reader,
+                          test);
+    writer = g_thread_new("writer",
+                          test_io_thread_writer,
+                          test);
+
+    g_thread_join(reader);
+    g_thread_join(writer);
+
+    test->dst = test->src = NULL;
+}
+
+
+void qio_channel_test_run_writer(QIOChannelTest *test,
+                                 QIOChannel *src)
+{
+    test->src = src;
+    test_io_thread_writer(test);
+    test->src = NULL;
+}
+
+
+void qio_channel_test_run_reader(QIOChannelTest *test,
+                                 QIOChannel *dst)
+{
+    test->dst = dst;
+    test_io_thread_reader(test);
+    test->dst = NULL;
+}
+
+
+void qio_channel_test_validate(QIOChannelTest *test)
+{
+    g_assert(test->readerr == NULL);
+    g_assert(test->writeerr == NULL);
+    g_assert_cmpint(memcmp(test->input,
+                           test->output,
+                           test->len), ==, 0);
+
+    g_free(test->inputv);
+    g_free(test->outputv);
+    g_free(test->input);
+    g_free(test->output);
+    g_free(test);
+}
diff --git a/tests/io-channel-helpers.h b/tests/io-channel-helpers.h
new file mode 100644
index 0000000..fedc64f
--- /dev/null
+++ b/tests/io-channel-helpers.h
@@ -0,0 +1,42 @@
+/*
+ * QEMU I/O channel test helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "io/channel.h"
+
+#ifndef TEST_IO_CHANNEL_HELPERS
+#define TEST_IO_CHANNEL_HELPERS
+
+typedef struct QIOChannelTest QIOChannelTest;
+
+QIOChannelTest *qio_channel_test_new(void);
+
+void qio_channel_test_run_threads(QIOChannelTest *test,
+                                  bool blocking,
+                                  QIOChannel *src,
+                                  QIOChannel *dst);
+
+void qio_channel_test_run_writer(QIOChannelTest *test,
+                                 QIOChannel *src);
+void qio_channel_test_run_reader(QIOChannelTest *test,
+                                 QIOChannel *dst);
+
+void qio_channel_test_validate(QIOChannelTest *test);
+
+#endif /* TEST_IO_CHANNEL_HELPERS */
diff --git a/tests/ioh3420-test.c b/tests/ioh3420-test.c
index c991a5f..b54c4b9 100644
--- a/tests/ioh3420-test.c
+++ b/tests/ioh3420-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/ipmi-bt-test.c b/tests/ipmi-bt-test.c
new file mode 100644
index 0000000..be9005e
--- /dev/null
+++ b/tests/ipmi-bt-test.c
@@ -0,0 +1,432 @@
+/*
+ * IPMI BT test cases, using the external interface for checking
+ *
+ * Copyright (c) 2012 Corey Minyard <cminyard@mvista.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+
+
+#include "libqtest.h"
+#include "qemu-common.h"
+
+#define IPMI_IRQ        5
+
+#define IPMI_BT_BASE    0xe4
+
+#define IPMI_BT_CTLREG_CLR_WR_PTR  0
+#define IPMI_BT_CTLREG_CLR_RD_PTR  1
+#define IPMI_BT_CTLREG_H2B_ATN     2
+#define IPMI_BT_CTLREG_B2H_ATN     3
+#define IPMI_BT_CTLREG_SMS_ATN     4
+#define IPMI_BT_CTLREG_H_BUSY      6
+#define IPMI_BT_CTLREG_B_BUSY      7
+
+#define IPMI_BT_CTLREG_GET(b) ((bt_get_ctrlreg() >> (b)) & 1)
+#define IPMI_BT_CTLREG_GET_H2B_ATN() IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_H2B_ATN)
+#define IPMI_BT_CTLREG_GET_B2H_ATN() IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_B2H_ATN)
+#define IPMI_BT_CTLREG_GET_SMS_ATN() IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_SMS_ATN)
+#define IPMI_BT_CTLREG_GET_H_BUSY()  IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_H_BUSY)
+#define IPMI_BT_CTLREG_GET_B_BUSY()  IPMI_BT_CTLREG_GET(IPMI_BT_CTLREG_B_BUSY)
+
+#define IPMI_BT_CTLREG_SET(b) bt_write_ctrlreg(1 << (b))
+#define IPMI_BT_CTLREG_SET_CLR_WR_PTR() IPMI_BT_CTLREG_SET( \
+                                                IPMI_BT_CTLREG_CLR_WR_PTR)
+#define IPMI_BT_CTLREG_SET_CLR_RD_PTR() IPMI_BT_CTLREG_SET( \
+                                                IPMI_BT_CTLREG_CLR_RD_PTR)
+#define IPMI_BT_CTLREG_SET_H2B_ATN()  IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_H2B_ATN)
+#define IPMI_BT_CTLREG_SET_B2H_ATN()  IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_B2H_ATN)
+#define IPMI_BT_CTLREG_SET_SMS_ATN()  IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_SMS_ATN)
+#define IPMI_BT_CTLREG_SET_H_BUSY()   IPMI_BT_CTLREG_SET(IPMI_BT_CTLREG_H_BUSY)
+
+static int bt_ints_enabled;
+
+static uint8_t bt_get_ctrlreg(void)
+{
+    return inb(IPMI_BT_BASE);
+}
+
+static void bt_write_ctrlreg(uint8_t val)
+{
+    outb(IPMI_BT_BASE, val);
+}
+
+static uint8_t bt_get_buf(void)
+{
+    return inb(IPMI_BT_BASE + 1);
+}
+
+static void bt_write_buf(uint8_t val)
+{
+    outb(IPMI_BT_BASE + 1, val);
+}
+
+static uint8_t bt_get_irqreg(void)
+{
+    return inb(IPMI_BT_BASE + 2);
+}
+
+static void bt_write_irqreg(uint8_t val)
+{
+    outb(IPMI_BT_BASE + 2, val);
+}
+
+static void bt_wait_b_busy(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_BT_CTLREG_GET_B_BUSY() != 0) {
+        g_assert(--count != 0);
+    }
+}
+
+static void bt_wait_b2h_atn(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_BT_CTLREG_GET_B2H_ATN() == 0) {
+        g_assert(--count != 0);
+    }
+}
+
+
+static int emu_lfd;
+static int emu_fd;
+static in_port_t emu_port;
+static uint8_t inbuf[100];
+static unsigned int inbuf_len;
+static unsigned int inbuf_pos;
+static int last_was_aa;
+
+static void read_emu_data(void)
+{
+    fd_set readfds;
+    int rv;
+    struct timeval tv;
+
+    FD_ZERO(&readfds);
+    FD_SET(emu_fd, &readfds);
+    tv.tv_sec = 10;
+    tv.tv_usec = 0;
+    rv = select(emu_fd + 1, &readfds, NULL, NULL, &tv);
+    if (rv == -1) {
+        perror("select");
+    }
+    g_assert(rv == 1);
+    rv = read(emu_fd, inbuf, sizeof(inbuf));
+    if (rv == -1) {
+        perror("read");
+    }
+    g_assert(rv > 0);
+    inbuf_len = rv;
+    inbuf_pos = 0;
+}
+
+static void write_emu_msg(uint8_t *msg, unsigned int len)
+{
+    int rv;
+
+#ifdef DEBUG_TEST
+    {
+        unsigned int i;
+        printf("sending:");
+        for (i = 0; i < len; i++) {
+            printf(" %2.2x", msg[i]);
+        }
+        printf("\n");
+    }
+#endif
+    rv = write(emu_fd, msg, len);
+    g_assert(rv == len);
+}
+
+static void get_emu_msg(uint8_t *msg, unsigned int *len)
+{
+    unsigned int outpos = 0;
+
+    for (;;) {
+        while (inbuf_pos < inbuf_len) {
+            uint8_t ch = inbuf[inbuf_pos++];
+
+            g_assert(outpos < *len);
+            if (last_was_aa) {
+                assert(ch & 0x10);
+                msg[outpos++] = ch & ~0x10;
+                last_was_aa = 0;
+            } else if (ch == 0xaa) {
+                last_was_aa = 1;
+            } else {
+                msg[outpos++] = ch;
+                if ((ch == 0xa0) || (ch == 0xa1)) {
+                    /* Message complete */
+                    *len = outpos;
+                    goto done;
+                }
+            }
+        }
+        read_emu_data();
+    }
+ done:
+#ifdef DEBUG_TEST
+    {
+        unsigned int i;
+        printf("Msg:");
+        for (i = 0; i < outpos; i++) {
+            printf(" %2.2x", msg[i]);
+        }
+        printf("\n");
+    }
+#endif
+    return;
+}
+
+static uint8_t
+ipmb_checksum(const unsigned char *data, int size, unsigned char start)
+{
+        unsigned char csum = start;
+
+        for (; size > 0; size--, data++) {
+                csum += *data;
+        }
+        return csum;
+}
+
+static uint8_t get_dev_id_cmd[] = { 0x18, 0x01 };
+static uint8_t get_dev_id_rsp[] = { 0x1c, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,
+                                    0x02, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+static uint8_t set_bmc_globals_cmd[] = { 0x18, 0x2e, 0x0f };
+static uint8_t set_bmc_globals_rsp[] = { 0x1c, 0x2e, 0x00 };
+static uint8_t enable_irq_cmd[] = { 0x05, 0xa1 };
+
+static void emu_msg_handler(void)
+{
+    uint8_t msg[100];
+    unsigned int msg_len = sizeof(msg);
+
+    get_emu_msg(msg, &msg_len);
+    g_assert(msg_len >= 5);
+    g_assert(msg[msg_len - 1] == 0xa0);
+    msg_len--;
+    g_assert(ipmb_checksum(msg, msg_len, 0) == 0);
+    msg_len--;
+    if ((msg[1] == get_dev_id_cmd[0]) && (msg[2] == get_dev_id_cmd[1])) {
+        memcpy(msg + 1, get_dev_id_rsp, sizeof(get_dev_id_rsp));
+        msg_len = sizeof(get_dev_id_rsp) + 1;
+        msg[msg_len] = -ipmb_checksum(msg, msg_len, 0);
+        msg_len++;
+        msg[msg_len++] = 0xa0;
+        write_emu_msg(msg, msg_len);
+    } else if ((msg[1] == set_bmc_globals_cmd[0]) &&
+               (msg[2] == set_bmc_globals_cmd[1])) {
+        memcpy(msg + 1, set_bmc_globals_rsp, sizeof(set_bmc_globals_rsp));
+        msg_len = sizeof(set_bmc_globals_rsp) + 1;
+        msg[msg_len] = -ipmb_checksum(msg, msg_len, 0);
+        msg_len++;
+        msg[msg_len++] = 0xa0;
+        write_emu_msg(msg, msg_len);
+        write_emu_msg(enable_irq_cmd, sizeof(enable_irq_cmd));
+    } else {
+        g_assert(0);
+    }
+}
+
+static void bt_cmd(uint8_t *cmd, unsigned int cmd_len,
+                    uint8_t *rsp, unsigned int *rsp_len)
+{
+    unsigned int i, len, j = 0;
+    uint8_t seq = 5;
+
+    /* Should be idle */
+    g_assert(bt_get_ctrlreg() == 0);
+
+    bt_wait_b_busy();
+    IPMI_BT_CTLREG_SET_CLR_WR_PTR();
+    bt_write_buf(cmd_len + 1);
+    bt_write_buf(cmd[0]);
+    bt_write_buf(seq);
+    for (i = 1; i < cmd_len; i++) {
+        bt_write_buf(cmd[i]);
+    }
+    IPMI_BT_CTLREG_SET_H2B_ATN();
+
+    emu_msg_handler(); /* We should get a message on the socket here. */
+
+    bt_wait_b2h_atn();
+    if (bt_ints_enabled) {
+        g_assert((bt_get_irqreg() & 0x02) == 0x02);
+        g_assert(get_irq(IPMI_IRQ));
+        bt_write_irqreg(0x03);
+    } else {
+        g_assert(!get_irq(IPMI_IRQ));
+    }
+    IPMI_BT_CTLREG_SET_H_BUSY();
+    IPMI_BT_CTLREG_SET_B2H_ATN();
+    IPMI_BT_CTLREG_SET_CLR_RD_PTR();
+    len = bt_get_buf();
+    g_assert(len >= 4);
+    rsp[0] = bt_get_buf();
+    assert(bt_get_buf() == seq);
+    len--;
+    for (j = 1; j < len; j++) {
+        rsp[j] = bt_get_buf();
+    }
+    IPMI_BT_CTLREG_SET_H_BUSY();
+    *rsp_len = j;
+}
+
+
+/*
+ * We should get a connect request and a short message with capabilities.
+ */
+static void test_connect(void)
+{
+    fd_set readfds;
+    int rv;
+    int val;
+    struct timeval tv;
+    uint8_t msg[100];
+    unsigned int msglen;
+    static uint8_t exp1[] = { 0xff, 0x01, 0xa1 }; /* A protocol version */
+    static uint8_t exp2[] = { 0x08, 0x1f, 0xa1 }; /* A capabilities cmd */
+
+    FD_ZERO(&readfds);
+    FD_SET(emu_lfd, &readfds);
+    tv.tv_sec = 10;
+    tv.tv_usec = 0;
+    rv = select(emu_lfd + 1, &readfds, NULL, NULL, &tv);
+    g_assert(rv == 1);
+    emu_fd = accept(emu_lfd, NULL, 0);
+    if (emu_fd < 0) {
+        perror("accept");
+    }
+    g_assert(emu_fd >= 0);
+
+    val = 1;
+    rv = setsockopt(emu_fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val));
+    g_assert(rv != -1);
+
+    /* Report our version */
+    write_emu_msg(exp1, sizeof(exp1));
+
+    /* Validate that we get the info we expect. */
+    msglen = sizeof(msg);
+    get_emu_msg(msg, &msglen);
+    g_assert(msglen == sizeof(exp1));
+    g_assert(memcmp(msg, exp1, msglen) == 0);
+    msglen = sizeof(msg);
+    get_emu_msg(msg, &msglen);
+    g_assert(msglen == sizeof(exp2));
+    g_assert(memcmp(msg, exp2, msglen) == 0);
+}
+
+/*
+ * Send a get_device_id to do a basic test.
+ */
+static void test_bt_base(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    bt_cmd(get_dev_id_cmd, sizeof(get_dev_id_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(get_dev_id_rsp));
+    g_assert(memcmp(get_dev_id_rsp, rsp, rsplen) == 0);
+}
+
+/*
+ * Enable IRQs for the interface.
+ */
+static void test_enable_irq(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    bt_cmd(set_bmc_globals_cmd, sizeof(set_bmc_globals_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(set_bmc_globals_rsp));
+    g_assert(memcmp(set_bmc_globals_rsp, rsp, rsplen) == 0);
+    bt_write_irqreg(0x01);
+    bt_ints_enabled = 1;
+}
+
+/*
+ * Create a local TCP socket with any port, then save off the port we got.
+ */
+static void open_socket(void)
+{
+    struct sockaddr_in myaddr;
+    socklen_t addrlen;
+
+    myaddr.sin_family = AF_INET;
+    myaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+    myaddr.sin_port = 0;
+    emu_lfd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+    if (emu_lfd == -1) {
+        perror("socket");
+        exit(1);
+    }
+    if (bind(emu_lfd, (struct sockaddr *) &myaddr, sizeof(myaddr)) == -1) {
+        perror("bind");
+        exit(1);
+    }
+    addrlen = sizeof(myaddr);
+    if (getsockname(emu_lfd, (struct sockaddr *) &myaddr , &addrlen) == -1) {
+        perror("getsockname");
+        exit(1);
+    }
+    emu_port = ntohs(myaddr.sin_port);
+    assert(listen(emu_lfd, 1) != -1);
+}
+
+int main(int argc, char **argv)
+{
+    const char *arch = qtest_get_arch();
+    char *cmdline;
+    int ret;
+
+    /* Check architecture */
+    if (strcmp(arch, "i386") && strcmp(arch, "x86_64")) {
+        g_test_message("Skipping test for non-x86\n");
+        return 0;
+    }
+
+    open_socket();
+
+    /* Run the tests */
+    g_test_init(&argc, &argv, NULL);
+
+    cmdline = g_strdup_printf("-vnc none"
+          " -chardev socket,id=ipmi0,host=localhost,port=%d,reconnect=10"
+          " -device ipmi-bmc-extern,chardev=ipmi0,id=bmc0"
+          " -device isa-ipmi-bt,bmc=bmc0", emu_port);
+    qtest_start(cmdline);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+    qtest_add_func("/ipmi/extern/connect", test_connect);
+    qtest_add_func("/ipmi/extern/bt_base", test_bt_base);
+    qtest_add_func("/ipmi/extern/bt_enable_irq", test_enable_irq);
+    qtest_add_func("/ipmi/extern/bt_base_irq", test_bt_base);
+    ret = g_test_run();
+    qtest_quit(global_qtest);
+
+    return ret;
+}
diff --git a/tests/ipmi-kcs-test.c b/tests/ipmi-kcs-test.c
new file mode 100644
index 0000000..3750389
--- /dev/null
+++ b/tests/ipmi-kcs-test.c
@@ -0,0 +1,292 @@
+/*
+ * IPMI KCS test cases, using the local interface.
+ *
+ * Copyright (c) 2012 Corey Minyard <cminyard@mvista.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+
+#include "libqtest.h"
+
+#define IPMI_IRQ        5
+
+#define IPMI_KCS_BASE   0xca2
+
+#define IPMI_KCS_STATUS_ABORT           0x60
+#define IPMI_KCS_CMD_WRITE_START        0x61
+#define IPMI_KCS_CMD_WRITE_END          0x62
+#define IPMI_KCS_CMD_READ               0x68
+
+#define IPMI_KCS_ABORTED_BY_CMD         0x01
+
+#define IPMI_KCS_CMDREG_GET_STATE() ((kcs_get_cmdreg() >> 6) & 3)
+#define IPMI_KCS_STATE_IDLE     0
+#define IPMI_KCS_STATE_READ     1
+#define IPMI_KCS_STATE_WRITE    2
+#define IPMI_KCS_STATE_ERROR    3
+#define IPMI_KCS_CMDREG_GET_CD()    ((kcs_get_cmdreg() >> 3) & 1)
+#define IPMI_KCS_CMDREG_GET_ATN()   ((kcs_get_cmdreg() >> 2) & 1)
+#define IPMI_KCS_CMDREG_GET_IBF()   ((kcs_get_cmdreg() >> 1) & 1)
+#define IPMI_KCS_CMDREG_GET_OBF()   ((kcs_get_cmdreg() >> 0) & 1)
+
+static int kcs_ints_enabled;
+
+static uint8_t kcs_get_cmdreg(void)
+{
+    return inb(IPMI_KCS_BASE + 1);
+}
+
+static void kcs_write_cmdreg(uint8_t val)
+{
+    outb(IPMI_KCS_BASE + 1, val);
+}
+
+static uint8_t kcs_get_datareg(void)
+{
+    return inb(IPMI_KCS_BASE);
+}
+
+static void kcs_write_datareg(uint8_t val)
+{
+    outb(IPMI_KCS_BASE, val);
+}
+
+static void kcs_wait_ibf(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_KCS_CMDREG_GET_IBF() != 0) {
+        g_assert(--count != 0);
+    }
+}
+
+static void kcs_wait_obf(void)
+{
+    unsigned int count = 1000;
+    while (IPMI_KCS_CMDREG_GET_OBF() == 0) {
+        g_assert(--count != 0);
+    }
+}
+
+static void kcs_clear_obf(void)
+{
+    if (kcs_ints_enabled) {
+        g_assert(get_irq(IPMI_IRQ));
+    } else {
+        g_assert(!get_irq(IPMI_IRQ));
+    }
+    g_assert(IPMI_KCS_CMDREG_GET_OBF() == 1);
+    kcs_get_datareg();
+    g_assert(IPMI_KCS_CMDREG_GET_OBF() == 0);
+    g_assert(!get_irq(IPMI_IRQ));
+}
+
+static void kcs_check_state(uint8_t state)
+{
+    g_assert(IPMI_KCS_CMDREG_GET_STATE() == state);
+}
+
+static void kcs_cmd(uint8_t *cmd, unsigned int cmd_len,
+                    uint8_t *rsp, unsigned int *rsp_len)
+{
+    unsigned int i, j = 0;
+
+    /* Should be idle */
+    g_assert(kcs_get_cmdreg() == 0);
+
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_START);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    for (i = 0; i < cmd_len; i++) {
+        kcs_write_datareg(cmd[i]);
+        kcs_wait_ibf();
+        kcs_check_state(IPMI_KCS_STATE_WRITE);
+        kcs_clear_obf();
+    }
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_END);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    kcs_write_datareg(0);
+ next_read_byte:
+    kcs_wait_ibf();
+    switch (IPMI_KCS_CMDREG_GET_STATE()) {
+    case IPMI_KCS_STATE_READ:
+        kcs_wait_obf();
+        g_assert(j < *rsp_len);
+        rsp[j++] = kcs_get_datareg();
+        kcs_write_datareg(IPMI_KCS_CMD_READ);
+        goto next_read_byte;
+        break;
+
+    case IPMI_KCS_STATE_IDLE:
+        kcs_wait_obf();
+        kcs_get_datareg();
+        break;
+
+    default:
+        g_assert(0);
+    }
+    *rsp_len = j;
+}
+
+static void kcs_abort(uint8_t *cmd, unsigned int cmd_len,
+                      uint8_t *rsp, unsigned int *rsp_len)
+{
+    unsigned int i, j = 0;
+    unsigned int retries = 4;
+
+    /* Should be idle */
+    g_assert(kcs_get_cmdreg() == 0);
+
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_START);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    for (i = 0; i < cmd_len; i++) {
+        kcs_write_datareg(cmd[i]);
+        kcs_wait_ibf();
+        kcs_check_state(IPMI_KCS_STATE_WRITE);
+        kcs_clear_obf();
+    }
+    kcs_write_cmdreg(IPMI_KCS_CMD_WRITE_END);
+    kcs_wait_ibf();
+    kcs_check_state(IPMI_KCS_STATE_WRITE);
+    kcs_clear_obf();
+    kcs_write_datareg(0);
+    kcs_wait_ibf();
+    switch (IPMI_KCS_CMDREG_GET_STATE()) {
+    case IPMI_KCS_STATE_READ:
+        kcs_wait_obf();
+        g_assert(j < *rsp_len);
+        rsp[j++] = kcs_get_datareg();
+        kcs_write_datareg(IPMI_KCS_CMD_READ);
+        break;
+
+    default:
+        g_assert(0);
+    }
+
+    /* Start the abort here */
+ retry_abort:
+    g_assert(retries > 0);
+
+    kcs_wait_ibf();
+    kcs_write_cmdreg(IPMI_KCS_STATUS_ABORT);
+    kcs_wait_ibf();
+    kcs_clear_obf();
+    kcs_write_datareg(0);
+    kcs_wait_ibf();
+    if (IPMI_KCS_CMDREG_GET_STATE() != IPMI_KCS_STATE_READ) {
+        retries--;
+        goto retry_abort;
+    }
+    kcs_wait_obf();
+    rsp[0] = kcs_get_datareg();
+    kcs_write_datareg(IPMI_KCS_CMD_READ);
+    kcs_wait_ibf();
+    if (IPMI_KCS_CMDREG_GET_STATE() != IPMI_KCS_STATE_IDLE) {
+        retries--;
+        goto retry_abort;
+    }
+    kcs_wait_obf();
+    kcs_clear_obf();
+
+    *rsp_len = j;
+}
+
+
+static uint8_t get_dev_id_cmd[] = { 0x18, 0x01 };
+static uint8_t get_dev_id_rsp[] = { 0x1c, 0x01, 0x00, 0x20, 0x00, 0x00, 0x00,
+                                    0x02, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+/*
+ * Send a get_device_id to do a basic test.
+ */
+static void test_kcs_base(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    kcs_cmd(get_dev_id_cmd, sizeof(get_dev_id_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(get_dev_id_rsp));
+    g_assert(memcmp(get_dev_id_rsp, rsp, rsplen) == 0);
+}
+
+/*
+ * Abort a kcs operation while reading
+ */
+static void test_kcs_abort(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    kcs_abort(get_dev_id_cmd, sizeof(get_dev_id_cmd), rsp, &rsplen);
+    g_assert(rsp[0] == IPMI_KCS_ABORTED_BY_CMD);
+}
+
+static uint8_t set_bmc_globals_cmd[] = { 0x18, 0x2e, 0x0f };
+static uint8_t set_bmc_globals_rsp[] = { 0x1c, 0x2e, 0x00 };
+
+/*
+ * Enable interrupts
+ */
+static void test_enable_irq(void)
+{
+    uint8_t rsp[20];
+    unsigned int rsplen = sizeof(rsp);
+
+    kcs_cmd(set_bmc_globals_cmd, sizeof(set_bmc_globals_cmd), rsp, &rsplen);
+    g_assert(rsplen == sizeof(set_bmc_globals_rsp));
+    g_assert(memcmp(set_bmc_globals_rsp, rsp, rsplen) == 0);
+    kcs_ints_enabled = 1;
+}
+
+int main(int argc, char **argv)
+{
+    const char *arch = qtest_get_arch();
+    char *cmdline;
+    int ret;
+
+    /* Check architecture */
+    if (strcmp(arch, "i386") && strcmp(arch, "x86_64")) {
+        g_test_message("Skipping test for non-x86\n");
+        return 0;
+    }
+
+    /* Run the tests */
+    g_test_init(&argc, &argv, NULL);
+
+    cmdline = g_strdup_printf("-vnc none -device ipmi-bmc-sim,id=bmc0"
+                              " -device isa-ipmi-kcs,bmc=bmc0");
+    qtest_start(cmdline);
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+    qtest_add_func("/ipmi/local/kcs_base", test_kcs_base);
+    qtest_add_func("/ipmi/local/kcs_abort", test_kcs_abort);
+    qtest_add_func("/ipmi/local/kcs_enable_irq", test_enable_irq);
+    qtest_add_func("/ipmi/local/kcs_base_irq", test_kcs_base);
+    qtest_add_func("/ipmi/local/kcs_abort_irq", test_kcs_abort);
+    ret = g_test_run();
+    qtest_quit(global_qtest);
+
+    return ret;
+}
diff --git a/tests/ipoctal232-test.c b/tests/ipoctal232-test.c
index 3ac1714..6849141 100644
--- a/tests/ipoctal232-test.c
+++ b/tests/ipoctal232-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/ivshmem-test.c b/tests/ivshmem-test.c
new file mode 100644
index 0000000..0957ee7
--- /dev/null
+++ b/tests/ivshmem-test.c
@@ -0,0 +1,516 @@
+/*
+ * QTest testcase for ivshmem
+ *
+ * Copyright (c) 2014 SUSE LINUX Products GmbH
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib/gstdio.h>
+#include "contrib/ivshmem-server/ivshmem-server.h"
+#include "libqos/pci-pc.h"
+#include "libqtest.h"
+#include "qemu-common.h"
+
+#define TMPSHMSIZE (1 << 20)
+static char *tmpshm;
+static void *tmpshmem;
+static char *tmpdir;
+static char *tmpserver;
+
+static void save_fn(QPCIDevice *dev, int devfn, void *data)
+{
+    QPCIDevice **pdev = (QPCIDevice **) data;
+
+    *pdev = dev;
+}
+
+static QPCIDevice *get_device(QPCIBus *pcibus)
+{
+    QPCIDevice *dev;
+
+    dev = NULL;
+    qpci_device_foreach(pcibus, 0x1af4, 0x1110, save_fn, &dev);
+    g_assert(dev != NULL);
+
+    return dev;
+}
+
+typedef struct _IVState {
+    QTestState *qtest;
+    void *reg_base, *mem_base;
+    QPCIBus *pcibus;
+    QPCIDevice *dev;
+} IVState;
+
+enum Reg {
+    INTRMASK = 0,
+    INTRSTATUS = 4,
+    IVPOSITION = 8,
+    DOORBELL = 12,
+};
+
+static const char* reg2str(enum Reg reg) {
+    switch (reg) {
+    case INTRMASK:
+        return "IntrMask";
+    case INTRSTATUS:
+        return "IntrStatus";
+    case IVPOSITION:
+        return "IVPosition";
+    case DOORBELL:
+        return "DoorBell";
+    default:
+        return NULL;
+    }
+}
+
+static inline unsigned in_reg(IVState *s, enum Reg reg)
+{
+    const char *name = reg2str(reg);
+    QTestState *qtest = global_qtest;
+    unsigned res;
+
+    global_qtest = s->qtest;
+    res = qpci_io_readl(s->dev, s->reg_base + reg);
+    g_test_message("*%s -> %x\n", name, res);
+    global_qtest = qtest;
+
+    return res;
+}
+
+static inline void out_reg(IVState *s, enum Reg reg, unsigned v)
+{
+    const char *name = reg2str(reg);
+    QTestState *qtest = global_qtest;
+
+    global_qtest = s->qtest;
+    g_test_message("%x -> *%s\n", v, name);
+    qpci_io_writel(s->dev, s->reg_base + reg, v);
+    global_qtest = qtest;
+}
+
+static void cleanup_vm(IVState *s)
+{
+    g_free(s->dev);
+    qpci_free_pc(s->pcibus);
+    qtest_quit(s->qtest);
+}
+
+static void setup_vm_cmd(IVState *s, const char *cmd, bool msix)
+{
+    uint64_t barsize;
+
+    s->qtest = qtest_start(cmd);
+    s->pcibus = qpci_init_pc();
+    s->dev = get_device(s->pcibus);
+
+    s->reg_base = qpci_iomap(s->dev, 0, &barsize);
+    g_assert_nonnull(s->reg_base);
+    g_assert_cmpuint(barsize, ==, 256);
+
+    if (msix) {
+        qpci_msix_enable(s->dev);
+    }
+
+    s->mem_base = qpci_iomap(s->dev, 2, &barsize);
+    g_assert_nonnull(s->mem_base);
+    g_assert_cmpuint(barsize, ==, TMPSHMSIZE);
+
+    qpci_device_enable(s->dev);
+}
+
+static void setup_vm(IVState *s)
+{
+    char *cmd = g_strdup_printf("-object memory-backend-file"
+                                ",id=mb1,size=1M,share,mem-path=/dev/shm%s"
+                                " -device ivshmem-plain,memdev=mb1", tmpshm);
+
+    setup_vm_cmd(s, cmd, false);
+
+    g_free(cmd);
+}
+
+static void test_ivshmem_single(void)
+{
+    IVState state, *s;
+    uint32_t data[1024];
+    int i;
+
+    setup_vm(&state);
+    s = &state;
+
+    /* initial state of readable registers */
+    g_assert_cmpuint(in_reg(s, INTRMASK), ==, 0);
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 0);
+    g_assert_cmpuint(in_reg(s, IVPOSITION), ==, 0);
+
+    /* trigger interrupt via registers */
+    out_reg(s, INTRMASK, 0xffffffff);
+    g_assert_cmpuint(in_reg(s, INTRMASK), ==, 0xffffffff);
+    out_reg(s, INTRSTATUS, 1);
+    /* check interrupt status */
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 1);
+    /* reading clears */
+    g_assert_cmpuint(in_reg(s, INTRSTATUS), ==, 0);
+    /* TODO intercept actual interrupt (needs qtest work) */
+
+    /* invalid register access */
+    out_reg(s, IVPOSITION, 1);
+    in_reg(s, DOORBELL);
+
+    /* ring the (non-functional) doorbell */
+    out_reg(s, DOORBELL, 8 << 16);
+
+    /* write shared memory */
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        data[i] = i;
+    }
+    qtest_memwrite(s->qtest, (uintptr_t)s->mem_base, data, sizeof(data));
+
+    /* verify write */
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        g_assert_cmpuint(((uint32_t *)tmpshmem)[i], ==, i);
+    }
+
+    /* read it back and verify read */
+    memset(data, 0, sizeof(data));
+    qtest_memread(s->qtest, (uintptr_t)s->mem_base, data, sizeof(data));
+    for (i = 0; i < G_N_ELEMENTS(data); i++) {
+        g_assert_cmpuint(data[i], ==, i);
+    }
+
+    cleanup_vm(s);
+}
+
+static void test_ivshmem_pair(void)
+{
+    IVState state1, state2, *s1, *s2;
+    char *data;
+    int i;
+
+    setup_vm(&state1);
+    s1 = &state1;
+    setup_vm(&state2);
+    s2 = &state2;
+
+    data = g_malloc0(TMPSHMSIZE);
+
+    /* host write, guest 1 & 2 read */
+    memset(tmpshmem, 0x42, TMPSHMSIZE);
+    qtest_memread(s1->qtest, (uintptr_t)s1->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x42);
+    }
+    qtest_memread(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x42);
+    }
+
+    /* guest 1 write, guest 2 read */
+    memset(data, 0x43, TMPSHMSIZE);
+    qtest_memwrite(s1->qtest, (uintptr_t)s1->mem_base, data, TMPSHMSIZE);
+    memset(data, 0, TMPSHMSIZE);
+    qtest_memread(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x43);
+    }
+
+    /* guest 2 write, guest 1 read */
+    memset(data, 0x44, TMPSHMSIZE);
+    qtest_memwrite(s2->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    memset(data, 0, TMPSHMSIZE);
+    qtest_memread(s1->qtest, (uintptr_t)s2->mem_base, data, TMPSHMSIZE);
+    for (i = 0; i < TMPSHMSIZE; i++) {
+        g_assert_cmpuint(data[i], ==, 0x44);
+    }
+
+    cleanup_vm(s1);
+    cleanup_vm(s2);
+    g_free(data);
+}
+
+typedef struct ServerThread {
+    GThread *thread;
+    IvshmemServer *server;
+    int pipe[2]; /* to handle quit */
+} ServerThread;
+
+static void *server_thread(void *data)
+{
+    ServerThread *t = data;
+    IvshmemServer *server = t->server;
+
+    while (true) {
+        fd_set fds;
+        int maxfd, ret;
+
+        FD_ZERO(&fds);
+        FD_SET(t->pipe[0], &fds);
+        maxfd = t->pipe[0] + 1;
+
+        ivshmem_server_get_fds(server, &fds, &maxfd);
+
+        ret = select(maxfd, &fds, NULL, NULL, NULL);
+
+        if (ret < 0) {
+            if (errno == EINTR) {
+                continue;
+            }
+
+            g_critical("select error: %s\n", strerror(errno));
+            break;
+        }
+        if (ret == 0) {
+            continue;
+        }
+
+        if (FD_ISSET(t->pipe[0], &fds)) {
+            break;
+        }
+
+        if (ivshmem_server_handle_fds(server, &fds, maxfd) < 0) {
+            g_critical("ivshmem_server_handle_fds() failed\n");
+            break;
+        }
+    }
+
+    return NULL;
+}
+
+static void setup_vm_with_server(IVState *s, int nvectors, bool msi)
+{
+    char *cmd = g_strdup_printf("-chardev socket,id=chr0,path=%s,nowait "
+                                "-device ivshmem%s,chardev=chr0,vectors=%d",
+                                tmpserver,
+                                msi ? "-doorbell" : ",size=1M,msi=off",
+                                nvectors);
+
+    setup_vm_cmd(s, cmd, msi);
+
+    g_free(cmd);
+}
+
+static void test_ivshmem_server(bool msi)
+{
+    IVState state1, state2, *s1, *s2;
+    ServerThread thread;
+    IvshmemServer server;
+    int ret, vm1, vm2;
+    int nvectors = 2;
+    guint64 end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+
+    ret = ivshmem_server_init(&server, tmpserver, tmpshm, true,
+                              TMPSHMSIZE, nvectors,
+                              g_test_verbose());
+    g_assert_cmpint(ret, ==, 0);
+
+    ret = ivshmem_server_start(&server);
+    g_assert_cmpint(ret, ==, 0);
+
+    thread.server = &server;
+    ret = pipe(thread.pipe);
+    g_assert_cmpint(ret, ==, 0);
+    thread.thread = g_thread_new("ivshmem-server", server_thread, &thread);
+    g_assert(thread.thread != NULL);
+
+    setup_vm_with_server(&state1, nvectors, msi);
+    s1 = &state1;
+    setup_vm_with_server(&state2, nvectors, msi);
+    s2 = &state2;
+
+    /* check got different VM ids */
+    vm1 = in_reg(s1, IVPOSITION);
+    vm2 = in_reg(s2, IVPOSITION);
+    g_assert_cmpint(vm1, >=, 0);
+    g_assert_cmpint(vm2, >=, 0);
+    g_assert_cmpint(vm1, !=, vm2);
+
+    /* check number of MSI-X vectors */
+    global_qtest = s1->qtest;
+    if (msi) {
+        ret = qpci_msix_table_size(s1->dev);
+        g_assert_cmpuint(ret, ==, nvectors);
+    }
+
+    /* TODO test behavior before MSI-X is enabled */
+
+    /* ping vm2 -> vm1 on vector 0 */
+    if (msi) {
+        ret = qpci_msix_pending(s1->dev, 0);
+        g_assert_cmpuint(ret, ==, 0);
+    } else {
+        g_assert_cmpuint(in_reg(s1, INTRSTATUS), ==, 0);
+    }
+    out_reg(s2, DOORBELL, vm1 << 16);
+    do {
+        g_usleep(10000);
+        ret = msi ? qpci_msix_pending(s1->dev, 0) : in_reg(s1, INTRSTATUS);
+    } while (ret == 0 && g_get_monotonic_time() < end_time);
+    g_assert_cmpuint(ret, !=, 0);
+
+    /* ping vm1 -> vm2 on vector 1 */
+    global_qtest = s2->qtest;
+    if (msi) {
+        ret = qpci_msix_pending(s2->dev, 1);
+        g_assert_cmpuint(ret, ==, 0);
+    } else {
+        g_assert_cmpuint(in_reg(s2, INTRSTATUS), ==, 0);
+    }
+    out_reg(s1, DOORBELL, vm2 << 16 | 1);
+    do {
+        g_usleep(10000);
+        ret = msi ? qpci_msix_pending(s2->dev, 1) : in_reg(s2, INTRSTATUS);
+    } while (ret == 0 && g_get_monotonic_time() < end_time);
+    g_assert_cmpuint(ret, !=, 0);
+
+    cleanup_vm(s2);
+    cleanup_vm(s1);
+
+    if (qemu_write_full(thread.pipe[1], "q", 1) != 1) {
+        g_error("qemu_write_full: %s", g_strerror(errno));
+    }
+
+    g_thread_join(thread.thread);
+
+    ivshmem_server_close(&server);
+    close(thread.pipe[1]);
+    close(thread.pipe[0]);
+}
+
+static void test_ivshmem_server_msi(void)
+{
+    test_ivshmem_server(true);
+}
+
+static void test_ivshmem_server_irq(void)
+{
+    test_ivshmem_server(false);
+}
+
+#define PCI_SLOT_HP             0x06
+
+static void test_ivshmem_hotplug(void)
+{
+    gchar *opts;
+
+    qtest_start("");
+
+    opts = g_strdup_printf("'shm': '%s', 'size': '1M'", tmpshm);
+
+    qpci_plug_device_test("ivshmem", "iv1", PCI_SLOT_HP, opts);
+    qpci_unplug_acpi_device_test("iv1", PCI_SLOT_HP);
+
+    qtest_end();
+    g_free(opts);
+}
+
+static void test_ivshmem_memdev(void)
+{
+    IVState state;
+
+    /* just for the sake of checking memory-backend property */
+    setup_vm_cmd(&state, "-object memory-backend-ram,size=1M,id=mb1"
+                 " -device ivshmem-plain,memdev=mb1", false);
+
+    cleanup_vm(&state);
+}
+
+static void cleanup(void)
+{
+    if (tmpshmem) {
+        munmap(tmpshmem, TMPSHMSIZE);
+        tmpshmem = NULL;
+    }
+
+    if (tmpshm) {
+        shm_unlink(tmpshm);
+        g_free(tmpshm);
+        tmpshm = NULL;
+    }
+
+    if (tmpserver) {
+        g_unlink(tmpserver);
+        g_free(tmpserver);
+        tmpserver = NULL;
+    }
+
+    if (tmpdir) {
+        g_rmdir(tmpdir);
+        tmpdir = NULL;
+    }
+}
+
+static void abrt_handler(void *data)
+{
+    cleanup();
+}
+
+static gchar *mktempshm(int size, int *fd)
+{
+    while (true) {
+        gchar *name;
+
+        name = g_strdup_printf("/qtest-%u-%u", getpid(), g_random_int());
+        *fd = shm_open(name, O_CREAT|O_RDWR|O_EXCL,
+                       S_IRWXU|S_IRWXG|S_IRWXO);
+        if (*fd > 0) {
+            g_assert(ftruncate(*fd, size) == 0);
+            return name;
+        }
+
+        g_free(name);
+
+        if (errno != EEXIST) {
+            perror("shm_open");
+            return NULL;
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int ret, fd;
+    gchar dir[] = "/tmp/ivshmem-test.XXXXXX";
+
+#if !GLIB_CHECK_VERSION(2, 31, 0)
+    if (!g_thread_supported()) {
+        g_thread_init(NULL);
+    }
+#endif
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_abrt_handler(abrt_handler, NULL);
+    /* shm */
+    tmpshm = mktempshm(TMPSHMSIZE, &fd);
+    if (!tmpshm) {
+        return 0;
+    }
+    tmpshmem = mmap(0, TMPSHMSIZE, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
+    g_assert(tmpshmem != MAP_FAILED);
+    /* server */
+    if (mkdtemp(dir) == NULL) {
+        g_error("mkdtemp: %s", g_strerror(errno));
+    }
+    tmpdir = dir;
+    tmpserver = g_strconcat(tmpdir, "/server", NULL);
+
+    qtest_add_func("/ivshmem/single", test_ivshmem_single);
+    qtest_add_func("/ivshmem/hotplug", test_ivshmem_hotplug);
+    qtest_add_func("/ivshmem/memdev", test_ivshmem_memdev);
+    if (g_test_slow()) {
+        qtest_add_func("/ivshmem/pair", test_ivshmem_pair);
+        qtest_add_func("/ivshmem/server-msi", test_ivshmem_server_msi);
+        qtest_add_func("/ivshmem/server-irq", test_ivshmem_server_irq);
+    }
+
+    ret = g_test_run();
+
+    cleanup();
+
+    return ret;
+}
diff --git a/tests/libqos/ahci.c b/tests/libqos/ahci.c
new file mode 100644
index 0000000..f3be550
--- /dev/null
+++ b/tests/libqos/ahci.c
@@ -0,0 +1,1123 @@
+/*
+ * libqos AHCI functions
+ *
+ * Copyright (c) 2014 John Snow <jsnow@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include "libqtest.h"
+#include "libqos/ahci.h"
+#include "libqos/pci-pc.h"
+
+#include "qemu-common.h"
+#include "qemu/host-utils.h"
+
+#include "hw/pci/pci_ids.h"
+#include "hw/pci/pci_regs.h"
+
+typedef struct AHCICommandProp {
+    uint8_t  cmd;        /* Command Code */
+    bool     data;       /* Data transfer command? */
+    bool     pio;
+    bool     dma;
+    bool     lba28;
+    bool     lba48;
+    bool     read;
+    bool     write;
+    bool     atapi;
+    bool     ncq;
+    uint64_t size;       /* Static transfer size, for commands like IDENTIFY. */
+    uint32_t interrupts; /* Expected interrupts for this command. */
+} AHCICommandProp;
+
+AHCICommandProp ahci_command_properties[] = {
+    { .cmd = CMD_READ_PIO,       .data = true,  .pio = true,
+                                 .lba28 = true, .read = true },
+    { .cmd = CMD_WRITE_PIO,      .data = true,  .pio = true,
+                                 .lba28 = true, .write = true },
+    { .cmd = CMD_READ_PIO_EXT,   .data = true,  .pio = true,
+                                 .lba48 = true, .read = true },
+    { .cmd = CMD_WRITE_PIO_EXT,  .data = true,  .pio = true,
+                                 .lba48 = true, .write = true },
+    { .cmd = CMD_READ_DMA,       .data = true,  .dma = true,
+                                 .lba28 = true, .read = true },
+    { .cmd = CMD_WRITE_DMA,      .data = true,  .dma = true,
+                                 .lba28 = true, .write = true },
+    { .cmd = CMD_READ_DMA_EXT,   .data = true,  .dma = true,
+                                 .lba48 = true, .read = true },
+    { .cmd = CMD_WRITE_DMA_EXT,  .data = true,  .dma = true,
+                                 .lba48 = true, .write = true },
+    { .cmd = CMD_IDENTIFY,       .data = true,  .pio = true,
+                                 .size = 512,   .read = true },
+    { .cmd = READ_FPDMA_QUEUED,  .data = true,  .dma = true,
+                                 .lba48 = true, .read = true, .ncq = true },
+    { .cmd = WRITE_FPDMA_QUEUED, .data = true,  .dma = true,
+                                 .lba48 = true, .write = true, .ncq = true },
+    { .cmd = CMD_READ_MAX,       .lba28 = true },
+    { .cmd = CMD_READ_MAX_EXT,   .lba48 = true },
+    { .cmd = CMD_FLUSH_CACHE,    .data = false },
+    { .cmd = CMD_PACKET,         .data = true,  .size = 16,
+                                 .atapi = true, .pio = true },
+    { .cmd = CMD_PACKET_ID,      .data = true,  .pio = true,
+                                 .size = 512,   .read = true }
+};
+
+struct AHCICommand {
+    /* Test Management Data */
+    uint8_t name;
+    uint8_t port;
+    uint8_t slot;
+    uint32_t interrupts;
+    uint64_t xbytes;
+    uint32_t prd_size;
+    uint64_t buffer;
+    AHCICommandProp *props;
+    /* Data to be transferred to the guest */
+    AHCICommandHeader header;
+    RegH2DFIS fis;
+    unsigned char *atapi_cmd;
+};
+
+/**
+ * Allocate space in the guest using information in the AHCIQState object.
+ */
+uint64_t ahci_alloc(AHCIQState *ahci, size_t bytes)
+{
+    g_assert(ahci);
+    g_assert(ahci->parent);
+    return qmalloc(ahci->parent, bytes);
+}
+
+void ahci_free(AHCIQState *ahci, uint64_t addr)
+{
+    g_assert(ahci);
+    g_assert(ahci->parent);
+    qfree(ahci->parent, addr);
+}
+
+bool is_atapi(AHCIQState *ahci, uint8_t port)
+{
+    return ahci_px_rreg(ahci, port, AHCI_PX_SIG) == AHCI_SIGNATURE_CDROM;
+}
+
+/**
+ * Locate, verify, and return a handle to the AHCI device.
+ */
+QPCIDevice *get_ahci_device(uint32_t *fingerprint)
+{
+    QPCIDevice *ahci;
+    uint32_t ahci_fingerprint;
+    QPCIBus *pcibus;
+
+    pcibus = qpci_init_pc();
+
+    /* Find the AHCI PCI device and verify it's the right one. */
+    ahci = qpci_device_find(pcibus, QPCI_DEVFN(0x1F, 0x02));
+    g_assert(ahci != NULL);
+
+    ahci_fingerprint = qpci_config_readl(ahci, PCI_VENDOR_ID);
+
+    switch (ahci_fingerprint) {
+    case AHCI_INTEL_ICH9:
+        break;
+    default:
+        /* Unknown device. */
+        g_assert_not_reached();
+    }
+
+    if (fingerprint) {
+        *fingerprint = ahci_fingerprint;
+    }
+    return ahci;
+}
+
+void free_ahci_device(QPCIDevice *dev)
+{
+    QPCIBus *pcibus = dev ? dev->bus : NULL;
+
+    /* libqos doesn't have a function for this, so free it manually */
+    g_free(dev);
+    qpci_free_pc(pcibus);
+}
+
+/* Free all memory in-use by the AHCI device. */
+void ahci_clean_mem(AHCIQState *ahci)
+{
+    uint8_t port, slot;
+
+    for (port = 0; port < 32; ++port) {
+        if (ahci->port[port].fb) {
+            ahci_free(ahci, ahci->port[port].fb);
+            ahci->port[port].fb = 0;
+        }
+        if (ahci->port[port].clb) {
+            for (slot = 0; slot < 32; slot++) {
+                ahci_destroy_command(ahci, port, slot);
+            }
+            ahci_free(ahci, ahci->port[port].clb);
+            ahci->port[port].clb = 0;
+        }
+    }
+}
+
+/*** Logical Device Initialization ***/
+
+/**
+ * Start the PCI device and sanity-check default operation.
+ */
+void ahci_pci_enable(AHCIQState *ahci)
+{
+    uint8_t reg;
+
+    start_ahci_device(ahci);
+
+    switch (ahci->fingerprint) {
+    case AHCI_INTEL_ICH9:
+        /* ICH9 has a register at PCI 0x92 that
+         * acts as a master port enabler mask. */
+        reg = qpci_config_readb(ahci->dev, 0x92);
+        reg |= 0x3F;
+        qpci_config_writeb(ahci->dev, 0x92, reg);
+        /* 0...0111111b -- bit significant, ports 0-5 enabled. */
+        ASSERT_BIT_SET(qpci_config_readb(ahci->dev, 0x92), 0x3F);
+        break;
+    }
+
+}
+
+/**
+ * Map BAR5/ABAR, and engage the PCI device.
+ */
+void start_ahci_device(AHCIQState *ahci)
+{
+    /* Map AHCI's ABAR (BAR5) */
+    ahci->hba_base = qpci_iomap(ahci->dev, 5, &ahci->barsize);
+    g_assert(ahci->hba_base);
+
+    /* turns on pci.cmd.iose, pci.cmd.mse and pci.cmd.bme */
+    qpci_device_enable(ahci->dev);
+}
+
+/**
+ * Test and initialize the AHCI's HBA memory areas.
+ * Initialize and start any ports with devices attached.
+ * Bring the HBA into the idle state.
+ */
+void ahci_hba_enable(AHCIQState *ahci)
+{
+    /* Bits of interest in this section:
+     * GHC.AE     Global Host Control / AHCI Enable
+     * PxCMD.ST   Port Command: Start
+     * PxCMD.SUD  "Spin Up Device"
+     * PxCMD.POD  "Power On Device"
+     * PxCMD.FRE  "FIS Receive Enable"
+     * PxCMD.FR   "FIS Receive Running"
+     * PxCMD.CR   "Command List Running"
+     */
+    uint32_t reg, ports_impl;
+    uint16_t i;
+    uint8_t num_cmd_slots;
+
+    g_assert(ahci != NULL);
+
+    /* Set GHC.AE to 1 */
+    ahci_set(ahci, AHCI_GHC, AHCI_GHC_AE);
+    reg = ahci_rreg(ahci, AHCI_GHC);
+    ASSERT_BIT_SET(reg, AHCI_GHC_AE);
+
+    /* Cache CAP and CAP2. */
+    ahci->cap = ahci_rreg(ahci, AHCI_CAP);
+    ahci->cap2 = ahci_rreg(ahci, AHCI_CAP2);
+
+    /* Read CAP.NCS, how many command slots do we have? */
+    num_cmd_slots = ((ahci->cap & AHCI_CAP_NCS) >> ctzl(AHCI_CAP_NCS)) + 1;
+    g_test_message("Number of Command Slots: %u", num_cmd_slots);
+
+    /* Determine which ports are implemented. */
+    ports_impl = ahci_rreg(ahci, AHCI_PI);
+
+    for (i = 0; ports_impl; ports_impl >>= 1, ++i) {
+        if (!(ports_impl & 0x01)) {
+            continue;
+        }
+
+        g_test_message("Initializing port %u", i);
+
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_CMD);
+        if (BITCLR(reg, AHCI_PX_CMD_ST | AHCI_PX_CMD_CR |
+                   AHCI_PX_CMD_FRE | AHCI_PX_CMD_FR)) {
+            g_test_message("port is idle");
+        } else {
+            g_test_message("port needs to be idled");
+            ahci_px_clr(ahci, i, AHCI_PX_CMD,
+                        (AHCI_PX_CMD_ST | AHCI_PX_CMD_FRE));
+            /* The port has 500ms to disengage. */
+            usleep(500000);
+            reg = ahci_px_rreg(ahci, i, AHCI_PX_CMD);
+            ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_CR);
+            ASSERT_BIT_CLEAR(reg, AHCI_PX_CMD_FR);
+            g_test_message("port is now idle");
+            /* The spec does allow for possibly needing a PORT RESET
+             * or HBA reset if we fail to idle the port. */
+        }
+
+        /* Allocate Memory for the Command List Buffer & FIS Buffer */
+        /* PxCLB space ... 0x20 per command, as in 4.2.2 p 36 */
+        ahci->port[i].clb = ahci_alloc(ahci, num_cmd_slots * 0x20);
+        qmemset(ahci->port[i].clb, 0x00, num_cmd_slots * 0x20);
+        g_test_message("CLB: 0x%08" PRIx64, ahci->port[i].clb);
+        ahci_px_wreg(ahci, i, AHCI_PX_CLB, ahci->port[i].clb);
+        g_assert_cmphex(ahci->port[i].clb, ==,
+                        ahci_px_rreg(ahci, i, AHCI_PX_CLB));
+
+        /* PxFB space ... 0x100, as in 4.2.1 p 35 */
+        ahci->port[i].fb = ahci_alloc(ahci, 0x100);
+        qmemset(ahci->port[i].fb, 0x00, 0x100);
+        g_test_message("FB: 0x%08" PRIx64, ahci->port[i].fb);
+        ahci_px_wreg(ahci, i, AHCI_PX_FB, ahci->port[i].fb);
+        g_assert_cmphex(ahci->port[i].fb, ==,
+                        ahci_px_rreg(ahci, i, AHCI_PX_FB));
+
+        /* Clear PxSERR, PxIS, then IS.IPS[x] by writing '1's. */
+        ahci_px_wreg(ahci, i, AHCI_PX_SERR, 0xFFFFFFFF);
+        ahci_px_wreg(ahci, i, AHCI_PX_IS, 0xFFFFFFFF);
+        ahci_wreg(ahci, AHCI_IS, (1 << i));
+
+        /* Verify Interrupts Cleared */
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_SERR);
+        g_assert_cmphex(reg, ==, 0);
+
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_IS);
+        g_assert_cmphex(reg, ==, 0);
+
+        reg = ahci_rreg(ahci, AHCI_IS);
+        ASSERT_BIT_CLEAR(reg, (1 << i));
+
+        /* Enable All Interrupts: */
+        ahci_px_wreg(ahci, i, AHCI_PX_IE, 0xFFFFFFFF);
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_IE);
+        g_assert_cmphex(reg, ==, ~((uint32_t)AHCI_PX_IE_RESERVED));
+
+        /* Enable the FIS Receive Engine. */
+        ahci_px_set(ahci, i, AHCI_PX_CMD, AHCI_PX_CMD_FRE);
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_CMD);
+        ASSERT_BIT_SET(reg, AHCI_PX_CMD_FR);
+
+        /* AHCI 1.3 spec: if !STS.BSY, !STS.DRQ and PxSSTS.DET indicates
+         * physical presence, a device is present and may be started. However,
+         * PxSERR.DIAG.X /may/ need to be cleared a priori. */
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_SERR);
+        if (BITSET(reg, AHCI_PX_SERR_DIAG_X)) {
+            ahci_px_set(ahci, i, AHCI_PX_SERR, AHCI_PX_SERR_DIAG_X);
+        }
+
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_TFD);
+        if (BITCLR(reg, AHCI_PX_TFD_STS_BSY | AHCI_PX_TFD_STS_DRQ)) {
+            reg = ahci_px_rreg(ahci, i, AHCI_PX_SSTS);
+            if ((reg & AHCI_PX_SSTS_DET) == SSTS_DET_ESTABLISHED) {
+                /* Device Found: set PxCMD.ST := 1 */
+                ahci_px_set(ahci, i, AHCI_PX_CMD, AHCI_PX_CMD_ST);
+                ASSERT_BIT_SET(ahci_px_rreg(ahci, i, AHCI_PX_CMD),
+                               AHCI_PX_CMD_CR);
+                g_test_message("Started Device %u", i);
+            } else if ((reg & AHCI_PX_SSTS_DET)) {
+                /* Device present, but in some unknown state. */
+                g_assert_not_reached();
+            }
+        }
+    }
+
+    /* Enable GHC.IE */
+    ahci_set(ahci, AHCI_GHC, AHCI_GHC_IE);
+    reg = ahci_rreg(ahci, AHCI_GHC);
+    ASSERT_BIT_SET(reg, AHCI_GHC_IE);
+
+    /* TODO: The device should now be idling and waiting for commands.
+     * In the future, a small test-case to inspect the Register D2H FIS
+     * and clear the initial interrupts might be good. */
+}
+
+/**
+ * Pick the first implemented and running port
+ */
+unsigned ahci_port_select(AHCIQState *ahci)
+{
+    uint32_t ports, reg;
+    unsigned i;
+
+    ports = ahci_rreg(ahci, AHCI_PI);
+    for (i = 0; i < 32; ports >>= 1, ++i) {
+        if (ports == 0) {
+            i = 32;
+        }
+
+        if (!(ports & 0x01)) {
+            continue;
+        }
+
+        reg = ahci_px_rreg(ahci, i, AHCI_PX_CMD);
+        if (BITSET(reg, AHCI_PX_CMD_ST)) {
+            break;
+        }
+    }
+    g_assert(i < 32);
+    return i;
+}
+
+/**
+ * Clear a port's interrupts and status information prior to a test.
+ */
+void ahci_port_clear(AHCIQState *ahci, uint8_t port)
+{
+    uint32_t reg;
+
+    /* Clear out this port's interrupts (ignore the init register d2h fis) */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_IS);
+    ahci_px_wreg(ahci, port, AHCI_PX_IS, reg);
+    g_assert_cmphex(ahci_px_rreg(ahci, port, AHCI_PX_IS), ==, 0);
+
+    /* Wipe the FIS-Receive Buffer */
+    qmemset(ahci->port[port].fb, 0x00, 0x100);
+}
+
+/**
+ * Check a port for errors.
+ */
+void ahci_port_check_error(AHCIQState *ahci, uint8_t port)
+{
+    uint32_t reg;
+
+    /* The upper 9 bits of the IS register all indicate errors. */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_IS);
+    reg >>= 23;
+    g_assert_cmphex(reg, ==, 0);
+
+    /* The Sata Error Register should be empty. */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SERR);
+    g_assert_cmphex(reg, ==, 0);
+
+    /* The TFD also has two error sections. */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_TFD);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_STS_ERR);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_ERR);
+}
+
+void ahci_port_check_interrupts(AHCIQState *ahci, uint8_t port,
+                                uint32_t intr_mask)
+{
+    uint32_t reg;
+
+    /* Check for expected interrupts */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_IS);
+    ASSERT_BIT_SET(reg, intr_mask);
+
+    /* Clear expected interrupts and assert all interrupts now cleared. */
+    ahci_px_wreg(ahci, port, AHCI_PX_IS, intr_mask);
+    g_assert_cmphex(ahci_px_rreg(ahci, port, AHCI_PX_IS), ==, 0);
+}
+
+void ahci_port_check_nonbusy(AHCIQState *ahci, uint8_t port, uint8_t slot)
+{
+    uint32_t reg;
+
+    /* Assert that the command slot is no longer busy (NCQ) */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_SACT);
+    ASSERT_BIT_CLEAR(reg, (1 << slot));
+
+    /* Non-NCQ */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_CI);
+    ASSERT_BIT_CLEAR(reg, (1 << slot));
+
+    /* And assert that we are generally not busy. */
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_TFD);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_STS_BSY);
+    ASSERT_BIT_CLEAR(reg, AHCI_PX_TFD_STS_DRQ);
+}
+
+void ahci_port_check_d2h_sanity(AHCIQState *ahci, uint8_t port, uint8_t slot)
+{
+    RegD2HFIS *d2h = g_malloc0(0x20);
+    uint32_t reg;
+
+    memread(ahci->port[port].fb + 0x40, d2h, 0x20);
+    g_assert_cmphex(d2h->fis_type, ==, 0x34);
+
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_TFD);
+    g_assert_cmphex((reg & AHCI_PX_TFD_ERR) >> 8, ==, d2h->error);
+    g_assert_cmphex((reg & AHCI_PX_TFD_STS), ==, d2h->status);
+
+    g_free(d2h);
+}
+
+void ahci_port_check_pio_sanity(AHCIQState *ahci, uint8_t port,
+                                uint8_t slot, size_t buffsize)
+{
+    PIOSetupFIS *pio = g_malloc0(0x20);
+
+    /* We cannot check the Status or E_Status registers, because
+     * the status may have again changed between the PIO Setup FIS
+     * and the conclusion of the command with the D2H Register FIS. */
+    memread(ahci->port[port].fb + 0x20, pio, 0x20);
+    g_assert_cmphex(pio->fis_type, ==, 0x5f);
+
+    /* BUG: PIO Setup FIS as utilized by QEMU tries to fit the entire
+     * transfer size in a uint16_t field. The maximum transfer size can
+     * eclipse this; the field is meant to convey the size of data per
+     * each Data FIS, not the entire operation as a whole. For now,
+     * we will sanity check the broken case where applicable. */
+    if (buffsize <= UINT16_MAX) {
+        g_assert_cmphex(le16_to_cpu(pio->tx_count), ==, buffsize);
+    }
+
+    g_free(pio);
+}
+
+void ahci_port_check_cmd_sanity(AHCIQState *ahci, AHCICommand *cmd)
+{
+    AHCICommandHeader cmdh;
+
+    ahci_get_command_header(ahci, cmd->port, cmd->slot, &cmdh);
+    /* Physical Region Descriptor Byte Count is not required to work for NCQ. */
+    if (!cmd->props->ncq) {
+        g_assert_cmphex(cmd->xbytes, ==, cmdh.prdbc);
+    }
+}
+
+/* Get the command in #slot of port #port. */
+void ahci_get_command_header(AHCIQState *ahci, uint8_t port,
+                             uint8_t slot, AHCICommandHeader *cmd)
+{
+    uint64_t ba = ahci->port[port].clb;
+    ba += slot * sizeof(AHCICommandHeader);
+    memread(ba, cmd, sizeof(AHCICommandHeader));
+
+    cmd->flags = le16_to_cpu(cmd->flags);
+    cmd->prdtl = le16_to_cpu(cmd->prdtl);
+    cmd->prdbc = le32_to_cpu(cmd->prdbc);
+    cmd->ctba = le64_to_cpu(cmd->ctba);
+}
+
+/* Set the command in #slot of port #port. */
+void ahci_set_command_header(AHCIQState *ahci, uint8_t port,
+                             uint8_t slot, AHCICommandHeader *cmd)
+{
+    AHCICommandHeader tmp = { .flags = 0 };
+    uint64_t ba = ahci->port[port].clb;
+    ba += slot * sizeof(AHCICommandHeader);
+
+    tmp.flags = cpu_to_le16(cmd->flags);
+    tmp.prdtl = cpu_to_le16(cmd->prdtl);
+    tmp.prdbc = cpu_to_le32(cmd->prdbc);
+    tmp.ctba = cpu_to_le64(cmd->ctba);
+
+    memwrite(ba, &tmp, sizeof(AHCICommandHeader));
+}
+
+void ahci_destroy_command(AHCIQState *ahci, uint8_t port, uint8_t slot)
+{
+    AHCICommandHeader cmd;
+
+    /* Obtain the Nth Command Header */
+    ahci_get_command_header(ahci, port, slot, &cmd);
+    if (cmd.ctba == 0) {
+        /* No address in it, so just return -- it's empty. */
+        goto tidy;
+    }
+
+    /* Free the Table */
+    ahci_free(ahci, cmd.ctba);
+
+ tidy:
+    /* NULL the header. */
+    memset(&cmd, 0x00, sizeof(cmd));
+    ahci_set_command_header(ahci, port, slot, &cmd);
+    ahci->port[port].ctba[slot] = 0;
+    ahci->port[port].prdtl[slot] = 0;
+}
+
+void ahci_write_fis(AHCIQState *ahci, AHCICommand *cmd)
+{
+    RegH2DFIS tmp = cmd->fis;
+    uint64_t addr = cmd->header.ctba;
+
+    /* NCQ commands use exclusively 8 bit fields and needs no adjustment.
+     * Only the count field needs to be adjusted for non-NCQ commands.
+     * The auxiliary FIS fields are defined per-command and are not currently
+     * implemented in libqos/ahci.o, but may or may not need to be flipped. */
+    if (!cmd->props->ncq) {
+        tmp.count = cpu_to_le16(tmp.count);
+    }
+
+    memwrite(addr, &tmp, sizeof(tmp));
+}
+
+unsigned ahci_pick_cmd(AHCIQState *ahci, uint8_t port)
+{
+    unsigned i;
+    unsigned j;
+    uint32_t reg;
+
+    reg = ahci_px_rreg(ahci, port, AHCI_PX_CI);
+
+    /* Pick the least recently used command slot that's available */
+    for (i = 0; i < 32; ++i) {
+        j = ((ahci->port[port].next + i) % 32);
+        if (reg & (1 << j)) {
+            continue;
+        }
+        ahci_destroy_command(ahci, port, j);
+        ahci->port[port].next = (j + 1) % 32;
+        return j;
+    }
+
+    g_test_message("All command slots were busy.");
+    g_assert_not_reached();
+}
+
+inline unsigned size_to_prdtl(unsigned bytes, unsigned bytes_per_prd)
+{
+    /* Each PRD can describe up to 4MiB */
+    g_assert_cmphex(bytes_per_prd, <=, 4096 * 1024);
+    g_assert_cmphex(bytes_per_prd & 0x01, ==, 0x00);
+    return (bytes + bytes_per_prd - 1) / bytes_per_prd;
+}
+
+const AHCIOpts default_opts = { .size = 0 };
+
+/**
+ * ahci_exec: execute a given command on a specific
+ * AHCI port.
+ *
+ * @ahci: The device to send the command to
+ * @port: The port number of the SATA device we wish
+ *        to have execute this command
+ * @op:   The S/ATA command to execute, or if opts.atapi
+ *        is true, the SCSI command code.
+ * @opts: Optional arguments to modify execution behavior.
+ */
+void ahci_exec(AHCIQState *ahci, uint8_t port,
+               uint8_t op, const AHCIOpts *opts_in)
+{
+    AHCICommand *cmd;
+    int rc;
+    AHCIOpts *opts;
+
+    opts = g_memdup((opts_in == NULL ? &default_opts : opts_in),
+                    sizeof(AHCIOpts));
+
+    /* No guest buffer provided, create one. */
+    if (opts->size && !opts->buffer) {
+        opts->buffer = ahci_alloc(ahci, opts->size);
+        g_assert(opts->buffer);
+        qmemset(opts->buffer, 0x00, opts->size);
+    }
+
+    /* Command creation */
+    if (opts->atapi) {
+        cmd = ahci_atapi_command_create(op);
+        if (opts->atapi_dma) {
+            ahci_command_enable_atapi_dma(cmd);
+        }
+    } else {
+        cmd = ahci_command_create(op);
+    }
+    ahci_command_adjust(cmd, opts->lba, opts->buffer,
+                        opts->size, opts->prd_size);
+
+    if (opts->pre_cb) {
+        rc = opts->pre_cb(ahci, cmd, opts);
+        g_assert_cmpint(rc, ==, 0);
+    }
+
+    /* Write command to memory and issue it */
+    ahci_command_commit(ahci, cmd, port);
+    ahci_command_issue_async(ahci, cmd);
+    if (opts->error) {
+        qmp_eventwait("STOP");
+    }
+    if (opts->mid_cb) {
+        rc = opts->mid_cb(ahci, cmd, opts);
+        g_assert_cmpint(rc, ==, 0);
+    }
+    if (opts->error) {
+        qmp_async("{'execute':'cont' }");
+        qmp_eventwait("RESUME");
+    }
+
+    /* Wait for command to complete and verify sanity */
+    ahci_command_wait(ahci, cmd);
+    ahci_command_verify(ahci, cmd);
+    if (opts->post_cb) {
+        rc = opts->post_cb(ahci, cmd, opts);
+        g_assert_cmpint(rc, ==, 0);
+    }
+    ahci_command_free(cmd);
+    if (opts->buffer != opts_in->buffer) {
+        ahci_free(ahci, opts->buffer);
+    }
+    g_free(opts);
+}
+
+/* Issue a command, expecting it to fail and STOP the VM */
+AHCICommand *ahci_guest_io_halt(AHCIQState *ahci, uint8_t port,
+                                uint8_t ide_cmd, uint64_t buffer,
+                                size_t bufsize, uint64_t sector)
+{
+    AHCICommand *cmd;
+
+    cmd = ahci_command_create(ide_cmd);
+    ahci_command_adjust(cmd, sector, buffer, bufsize, 0);
+    ahci_command_commit(ahci, cmd, port);
+    ahci_command_issue_async(ahci, cmd);
+    qmp_eventwait("STOP");
+
+    return cmd;
+}
+
+/* Resume a previously failed command and verify/finalize */
+void ahci_guest_io_resume(AHCIQState *ahci, AHCICommand *cmd)
+{
+    /* Complete the command */
+    qmp_async("{'execute':'cont' }");
+    qmp_eventwait("RESUME");
+    ahci_command_wait(ahci, cmd);
+    ahci_command_verify(ahci, cmd);
+    ahci_command_free(cmd);
+}
+
+/* Given a guest buffer address, perform an IO operation */
+void ahci_guest_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
+                   uint64_t buffer, size_t bufsize, uint64_t sector)
+{
+    AHCICommand *cmd;
+    cmd = ahci_command_create(ide_cmd);
+    ahci_command_set_buffer(cmd, buffer);
+    ahci_command_set_size(cmd, bufsize);
+    if (sector) {
+        ahci_command_set_offset(cmd, sector);
+    }
+    ahci_command_commit(ahci, cmd, port);
+    ahci_command_issue(ahci, cmd);
+    ahci_command_verify(ahci, cmd);
+    ahci_command_free(cmd);
+}
+
+static AHCICommandProp *ahci_command_find(uint8_t command_name)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(ahci_command_properties); i++) {
+        if (ahci_command_properties[i].cmd == command_name) {
+            return &ahci_command_properties[i];
+        }
+    }
+
+    return NULL;
+}
+
+/* Given a HOST buffer, create a buffer address and perform an IO operation. */
+void ahci_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
+             void *buffer, size_t bufsize, uint64_t sector)
+{
+    uint64_t ptr;
+    AHCICommandProp *props;
+
+    props = ahci_command_find(ide_cmd);
+    g_assert(props);
+    ptr = ahci_alloc(ahci, bufsize);
+    g_assert(!bufsize || ptr);
+    qmemset(ptr, 0x00, bufsize);
+
+    if (bufsize && props->write) {
+        bufwrite(ptr, buffer, bufsize);
+    }
+
+    ahci_guest_io(ahci, port, ide_cmd, ptr, bufsize, sector);
+
+    if (bufsize && props->read) {
+        bufread(ptr, buffer, bufsize);
+    }
+
+    ahci_free(ahci, ptr);
+}
+
+/**
+ * Initializes a basic command header in memory.
+ * We assume that this is for an ATA command using RegH2DFIS.
+ */
+static void command_header_init(AHCICommand *cmd)
+{
+    AHCICommandHeader *hdr = &cmd->header;
+    AHCICommandProp *props = cmd->props;
+
+    hdr->flags = 5;             /* RegH2DFIS is 5 DW long. Must be < 32 */
+    hdr->flags |= CMDH_CLR_BSY; /* Clear the BSY bit when done */
+    if (props->write) {
+        hdr->flags |= CMDH_WRITE;
+    }
+    if (props->atapi) {
+        hdr->flags |= CMDH_ATAPI;
+    }
+    /* Other flags: PREFETCH, RESET, and BIST */
+    hdr->prdtl = size_to_prdtl(cmd->xbytes, cmd->prd_size);
+    hdr->prdbc = 0;
+    hdr->ctba = 0;
+}
+
+static void command_table_init(AHCICommand *cmd)
+{
+    RegH2DFIS *fis = &(cmd->fis);
+    uint16_t sect_count = (cmd->xbytes / AHCI_SECTOR_SIZE);
+
+    fis->fis_type = REG_H2D_FIS;
+    fis->flags = REG_H2D_FIS_CMD; /* "Command" bit */
+    fis->command = cmd->name;
+
+    if (cmd->props->ncq) {
+        NCQFIS *ncqfis = (NCQFIS *)fis;
+        /* NCQ is weird and re-uses FIS frames for unrelated data.
+         * See SATA 3.2, 13.6.4.1 READ FPDMA QUEUED for an example. */
+        ncqfis->sector_low = sect_count & 0xFF;
+        ncqfis->sector_hi = (sect_count >> 8) & 0xFF;
+        ncqfis->device = NCQ_DEVICE_MAGIC;
+        /* Force Unit Access is bit 7 in the device register */
+        ncqfis->tag = 0;  /* bits 3-7 are the NCQ tag */
+        ncqfis->prio = 0; /* bits 6,7 are a prio tag */
+        /* RARC bit is bit 0 of TAG field */
+    } else {
+        fis->feature_low = 0x00;
+        fis->feature_high = 0x00;
+        if (cmd->props->lba28 || cmd->props->lba48) {
+            fis->device = ATA_DEVICE_LBA;
+        }
+        fis->count = (cmd->xbytes / AHCI_SECTOR_SIZE);
+    }
+    fis->icc = 0x00;
+    fis->control = 0x00;
+    memset(fis->aux, 0x00, ARRAY_SIZE(fis->aux));
+}
+
+void ahci_command_enable_atapi_dma(AHCICommand *cmd)
+{
+    RegH2DFIS *fis = &(cmd->fis);
+    g_assert(cmd->props->atapi);
+    fis->feature_low |= 0x01;
+    cmd->interrupts &= ~AHCI_PX_IS_PSS;
+    cmd->props->dma = true;
+    cmd->props->pio = false;
+    /* BUG: We expect the DMA Setup interrupt for DMA commands */
+    /* cmd->interrupts |= AHCI_PX_IS_DSS; */
+}
+
+AHCICommand *ahci_command_create(uint8_t command_name)
+{
+    AHCICommandProp *props = ahci_command_find(command_name);
+    AHCICommand *cmd;
+
+    g_assert(props);
+    cmd = g_malloc0(sizeof(AHCICommand));
+    g_assert(!(props->dma && props->pio));
+    g_assert(!(props->lba28 && props->lba48));
+    g_assert(!(props->read && props->write));
+    g_assert(!props->size || props->data);
+    g_assert(!props->ncq || props->lba48);
+
+    /* Defaults and book-keeping */
+    cmd->props = g_memdup(props, sizeof(AHCICommandProp));
+    cmd->name = command_name;
+    cmd->xbytes = props->size;
+    cmd->prd_size = 4096;
+    cmd->buffer = 0xabad1dea;
+
+    if (!cmd->props->ncq) {
+        cmd->interrupts = AHCI_PX_IS_DHRS;
+    }
+    /* BUG: We expect the DPS interrupt for data commands */
+    /* cmd->interrupts |= props->data ? AHCI_PX_IS_DPS : 0; */
+    /* BUG: We expect the DMA Setup interrupt for DMA commands */
+    /* cmd->interrupts |= props->dma ? AHCI_PX_IS_DSS : 0; */
+    cmd->interrupts |= props->pio ? AHCI_PX_IS_PSS : 0;
+    cmd->interrupts |= props->ncq ? AHCI_PX_IS_SDBS : 0;
+
+    command_header_init(cmd);
+    command_table_init(cmd);
+
+    return cmd;
+}
+
+AHCICommand *ahci_atapi_command_create(uint8_t scsi_cmd)
+{
+    AHCICommand *cmd = ahci_command_create(CMD_PACKET);
+    cmd->atapi_cmd = g_malloc0(16);
+    cmd->atapi_cmd[0] = scsi_cmd;
+    /* ATAPI needs a PIO transfer chunk size set inside of the LBA registers.
+     * The block/sector size is a natural default. */
+    cmd->fis.lba_lo[1] = ATAPI_SECTOR_SIZE >> 8 & 0xFF;
+    cmd->fis.lba_lo[2] = ATAPI_SECTOR_SIZE & 0xFF;
+
+    return cmd;
+}
+
+void ahci_command_free(AHCICommand *cmd)
+{
+    g_free(cmd->atapi_cmd);
+    g_free(cmd->props);
+    g_free(cmd);
+}
+
+void ahci_command_set_flags(AHCICommand *cmd, uint16_t cmdh_flags)
+{
+    cmd->header.flags |= cmdh_flags;
+}
+
+void ahci_command_clr_flags(AHCICommand *cmd, uint16_t cmdh_flags)
+{
+    cmd->header.flags &= ~cmdh_flags;
+}
+
+static void ahci_atapi_command_set_offset(AHCICommand *cmd, uint64_t lba)
+{
+    unsigned char *cbd = cmd->atapi_cmd;
+    g_assert(cbd);
+
+    switch (cbd[0]) {
+    case CMD_ATAPI_READ_10:
+        g_assert_cmpuint(lba, <=, UINT32_MAX);
+        stl_be_p(&cbd[2], lba);
+        break;
+    default:
+        /* SCSI doesn't have uniform packet formats,
+         * so you have to add support for it manually. Sorry! */
+        g_assert_not_reached();
+    }
+}
+
+void ahci_command_set_offset(AHCICommand *cmd, uint64_t lba_sect)
+{
+    RegH2DFIS *fis = &(cmd->fis);
+
+    if (cmd->props->atapi) {
+        ahci_atapi_command_set_offset(cmd, lba_sect);
+        return;
+    } else if (!cmd->props->data && !lba_sect) {
+        /* Not meaningful, ignore. */
+        return;
+    } else if (cmd->props->lba28) {
+        g_assert_cmphex(lba_sect, <=, 0xFFFFFFF);
+    } else if (cmd->props->lba48 || cmd->props->ncq) {
+        g_assert_cmphex(lba_sect, <=, 0xFFFFFFFFFFFF);
+    } else {
+        /* Can't set offset if we don't know the format. */
+        g_assert_not_reached();
+    }
+
+    /* LBA28 uses the low nibble of the device/control register for LBA24:27 */
+    fis->lba_lo[0] = (lba_sect & 0xFF);
+    fis->lba_lo[1] = (lba_sect >> 8) & 0xFF;
+    fis->lba_lo[2] = (lba_sect >> 16) & 0xFF;
+    if (cmd->props->lba28) {
+        fis->device = (fis->device & 0xF0) | ((lba_sect >> 24) & 0x0F);
+    }
+    fis->lba_hi[0] = (lba_sect >> 24) & 0xFF;
+    fis->lba_hi[1] = (lba_sect >> 32) & 0xFF;
+    fis->lba_hi[2] = (lba_sect >> 40) & 0xFF;
+}
+
+void ahci_command_set_buffer(AHCICommand *cmd, uint64_t buffer)
+{
+    cmd->buffer = buffer;
+}
+
+static void ahci_atapi_set_size(AHCICommand *cmd, uint64_t xbytes)
+{
+    unsigned char *cbd = cmd->atapi_cmd;
+    uint64_t nsectors = xbytes / 2048;
+    g_assert(cbd);
+
+    switch (cbd[0]) {
+    case CMD_ATAPI_READ_10:
+        g_assert_cmpuint(nsectors, <=, UINT16_MAX);
+        stw_be_p(&cbd[7], nsectors);
+        break;
+    default:
+        /* SCSI doesn't have uniform packet formats,
+         * so you have to add support for it manually. Sorry! */
+        g_assert_not_reached();
+    }
+}
+
+void ahci_command_set_sizes(AHCICommand *cmd, uint64_t xbytes,
+                            unsigned prd_size)
+{
+    uint16_t sect_count;
+
+    /* Each PRD can describe up to 4MiB, and must not be odd. */
+    g_assert_cmphex(prd_size, <=, 4096 * 1024);
+    g_assert_cmphex(prd_size & 0x01, ==, 0x00);
+    if (prd_size) {
+        cmd->prd_size = prd_size;
+    }
+    cmd->xbytes = xbytes;
+    sect_count = (cmd->xbytes / AHCI_SECTOR_SIZE);
+
+    if (cmd->props->ncq) {
+        NCQFIS *nfis = (NCQFIS *)&(cmd->fis);
+        nfis->sector_low = sect_count & 0xFF;
+        nfis->sector_hi = (sect_count >> 8) & 0xFF;
+    } else if (cmd->props->atapi) {
+        ahci_atapi_set_size(cmd, xbytes);
+    } else {
+        cmd->fis.count = sect_count;
+    }
+    cmd->header.prdtl = size_to_prdtl(cmd->xbytes, cmd->prd_size);
+}
+
+void ahci_command_set_size(AHCICommand *cmd, uint64_t xbytes)
+{
+    ahci_command_set_sizes(cmd, xbytes, cmd->prd_size);
+}
+
+void ahci_command_set_prd_size(AHCICommand *cmd, unsigned prd_size)
+{
+    ahci_command_set_sizes(cmd, cmd->xbytes, prd_size);
+}
+
+void ahci_command_adjust(AHCICommand *cmd, uint64_t offset, uint64_t buffer,
+                         uint64_t xbytes, unsigned prd_size)
+{
+    ahci_command_set_sizes(cmd, xbytes, prd_size);
+    ahci_command_set_buffer(cmd, buffer);
+    ahci_command_set_offset(cmd, offset);
+}
+
+void ahci_command_commit(AHCIQState *ahci, AHCICommand *cmd, uint8_t port)
+{
+    uint16_t i, prdtl;
+    uint64_t table_size, table_ptr, remaining;
+    PRD prd;
+
+    /* This command is now tied to this port/command slot */
+    cmd->port = port;
+    cmd->slot = ahci_pick_cmd(ahci, port);
+
+    if (cmd->props->ncq) {
+        NCQFIS *nfis = (NCQFIS *)&cmd->fis;
+        nfis->tag = (cmd->slot << 3) & 0xFC;
+    }
+
+    /* Create a buffer for the command table */
+    prdtl = size_to_prdtl(cmd->xbytes, cmd->prd_size);
+    table_size = CMD_TBL_SIZ(prdtl);
+    table_ptr = ahci_alloc(ahci, table_size);
+    g_assert(table_ptr);
+    /* AHCI 1.3: Must be aligned to 0x80 */
+    g_assert((table_ptr & 0x7F) == 0x00);
+    cmd->header.ctba = table_ptr;
+
+    /* Commit the command header (part of the Command List Buffer) */
+    ahci_set_command_header(ahci, port, cmd->slot, &(cmd->header));
+    /* Now, write the command table (FIS, ACMD, and PRDT) -- FIS first, */
+    ahci_write_fis(ahci, cmd);
+    /* Then ATAPI CMD, if needed */
+    if (cmd->props->atapi) {
+        memwrite(table_ptr + 0x40, cmd->atapi_cmd, 16);
+    }
+
+    /* Construct and write the PRDs to the command table */
+    g_assert_cmphex(prdtl, ==, cmd->header.prdtl);
+    remaining = cmd->xbytes;
+    for (i = 0; i < prdtl; ++i) {
+        prd.dba = cpu_to_le64(cmd->buffer + (cmd->prd_size * i));
+        prd.res = 0;
+        if (remaining > cmd->prd_size) {
+            /* Note that byte count is 0-based. */
+            prd.dbc = cpu_to_le32(cmd->prd_size - 1);
+            remaining -= cmd->prd_size;
+        } else {
+            /* Again, dbc is 0-based. */
+            prd.dbc = cpu_to_le32(remaining - 1);
+            remaining = 0;
+        }
+        prd.dbc |= cpu_to_le32(0x80000000); /* Request DPS Interrupt */
+
+        /* Commit the PRD entry to the Command Table */
+        memwrite(table_ptr + 0x80 + (i * sizeof(PRD)),
+                 &prd, sizeof(PRD));
+    }
+
+    /* Bookmark the PRDTL and CTBA values */
+    ahci->port[port].ctba[cmd->slot] = table_ptr;
+    ahci->port[port].prdtl[cmd->slot] = prdtl;
+}
+
+void ahci_command_issue_async(AHCIQState *ahci, AHCICommand *cmd)
+{
+    if (cmd->props->ncq) {
+        ahci_px_wreg(ahci, cmd->port, AHCI_PX_SACT, (1 << cmd->slot));
+    }
+
+    ahci_px_wreg(ahci, cmd->port, AHCI_PX_CI, (1 << cmd->slot));
+}
+
+void ahci_command_wait(AHCIQState *ahci, AHCICommand *cmd)
+{
+    /* We can't rely on STS_BSY until the command has started processing.
+     * Therefore, we also use the Command Issue bit as indication of
+     * a command in-flight. */
+
+#define RSET(REG, MASK) (BITSET(ahci_px_rreg(ahci, cmd->port, (REG)), (MASK)))
+
+    while (RSET(AHCI_PX_TFD, AHCI_PX_TFD_STS_BSY) ||
+           RSET(AHCI_PX_CI, 1 << cmd->slot) ||
+           (cmd->props->ncq && RSET(AHCI_PX_SACT, 1 << cmd->slot))) {
+        usleep(50);
+    }
+
+}
+
+void ahci_command_issue(AHCIQState *ahci, AHCICommand *cmd)
+{
+    ahci_command_issue_async(ahci, cmd);
+    ahci_command_wait(ahci, cmd);
+}
+
+void ahci_command_verify(AHCIQState *ahci, AHCICommand *cmd)
+{
+    uint8_t slot = cmd->slot;
+    uint8_t port = cmd->port;
+
+    ahci_port_check_error(ahci, port);
+    ahci_port_check_interrupts(ahci, port, cmd->interrupts);
+    ahci_port_check_nonbusy(ahci, port, slot);
+    ahci_port_check_cmd_sanity(ahci, cmd);
+    if (cmd->interrupts & AHCI_PX_IS_DHRS) {
+        ahci_port_check_d2h_sanity(ahci, port, slot);
+    }
+    if (cmd->props->pio) {
+        ahci_port_check_pio_sanity(ahci, port, slot, cmd->xbytes);
+    }
+}
+
+uint8_t ahci_command_slot(AHCICommand *cmd)
+{
+    return cmd->slot;
+}
diff --git a/tests/libqos/ahci.h b/tests/libqos/ahci.h
new file mode 100644
index 0000000..c69fb5a
--- /dev/null
+++ b/tests/libqos/ahci.h
@@ -0,0 +1,627 @@
+#ifndef LIBQOS_AHCI_H
+#define LIBQOS_AHCI_H
+
+/*
+ * AHCI qtest library functions and definitions
+ *
+ * Copyright (c) 2014 John Snow <jsnow@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "libqos/libqos.h"
+#include "libqos/pci.h"
+#include "libqos/malloc-pc.h"
+
+/*** Supplementary PCI Config Space IDs & Masks ***/
+#define PCI_DEVICE_ID_INTEL_Q35_AHCI   (0x2922)
+#define PCI_MSI_FLAGS_RESERVED         (0xFF00)
+#define PCI_PM_CTRL_RESERVED             (0xFC)
+#define PCI_BCC(REG32)          ((REG32) >> 24)
+#define PCI_PI(REG32)   (((REG32) >> 8) & 0xFF)
+#define PCI_SCC(REG32) (((REG32) >> 16) & 0xFF)
+
+/*** Recognized AHCI Device Types ***/
+#define AHCI_INTEL_ICH9 (PCI_DEVICE_ID_INTEL_Q35_AHCI << 16 | \
+                         PCI_VENDOR_ID_INTEL)
+
+/*** AHCI/HBA Register Offsets and Bitmasks ***/
+#define AHCI_CAP                          (0)
+#define AHCI_CAP_NP                    (0x1F)
+#define AHCI_CAP_SXS                   (0x20)
+#define AHCI_CAP_EMS                   (0x40)
+#define AHCI_CAP_CCCS                  (0x80)
+#define AHCI_CAP_NCS                 (0x1F00)
+#define AHCI_CAP_PSC                 (0x2000)
+#define AHCI_CAP_SSC                 (0x4000)
+#define AHCI_CAP_PMD                 (0x8000)
+#define AHCI_CAP_FBSS               (0x10000)
+#define AHCI_CAP_SPM                (0x20000)
+#define AHCI_CAP_SAM                (0x40000)
+#define AHCI_CAP_RESERVED           (0x80000)
+#define AHCI_CAP_ISS               (0xF00000)
+#define AHCI_CAP_SCLO             (0x1000000)
+#define AHCI_CAP_SAL              (0x2000000)
+#define AHCI_CAP_SALP             (0x4000000)
+#define AHCI_CAP_SSS              (0x8000000)
+#define AHCI_CAP_SMPS            (0x10000000)
+#define AHCI_CAP_SSNTF           (0x20000000)
+#define AHCI_CAP_SNCQ            (0x40000000)
+#define AHCI_CAP_S64A            (0x80000000)
+
+#define AHCI_GHC                          (1)
+#define AHCI_GHC_HR                    (0x01)
+#define AHCI_GHC_IE                    (0x02)
+#define AHCI_GHC_MRSM                  (0x04)
+#define AHCI_GHC_RESERVED        (0x7FFFFFF8)
+#define AHCI_GHC_AE              (0x80000000)
+
+#define AHCI_IS                           (2)
+#define AHCI_PI                           (3)
+#define AHCI_VS                           (4)
+
+#define AHCI_CCCCTL                       (5)
+#define AHCI_CCCCTL_EN                 (0x01)
+#define AHCI_CCCCTL_RESERVED           (0x06)
+#define AHCI_CCCCTL_CC               (0xFF00)
+#define AHCI_CCCCTL_TV           (0xFFFF0000)
+
+#define AHCI_CCCPORTS                     (6)
+#define AHCI_EMLOC                        (7)
+
+#define AHCI_EMCTL                        (8)
+#define AHCI_EMCTL_STSMR               (0x01)
+#define AHCI_EMCTL_CTLTM              (0x100)
+#define AHCI_EMCTL_CTLRST             (0x200)
+#define AHCI_EMCTL_RESERVED      (0xF0F0FCFE)
+
+#define AHCI_CAP2                         (9)
+#define AHCI_CAP2_BOH                  (0x01)
+#define AHCI_CAP2_NVMP                 (0x02)
+#define AHCI_CAP2_APST                 (0x04)
+#define AHCI_CAP2_RESERVED       (0xFFFFFFF8)
+
+#define AHCI_BOHC                        (10)
+#define AHCI_RESERVED                    (11)
+#define AHCI_NVMHCI                      (24)
+#define AHCI_VENDOR                      (40)
+#define AHCI_PORTS                       (64)
+
+/*** Port Memory Offsets & Bitmasks ***/
+#define AHCI_PX_CLB                       (0)
+#define AHCI_PX_CLB_RESERVED          (0x1FF)
+
+#define AHCI_PX_CLBU                      (1)
+
+#define AHCI_PX_FB                        (2)
+#define AHCI_PX_FB_RESERVED            (0xFF)
+
+#define AHCI_PX_FBU                       (3)
+
+#define AHCI_PX_IS                        (4)
+#define AHCI_PX_IS_DHRS                 (0x1)
+#define AHCI_PX_IS_PSS                  (0x2)
+#define AHCI_PX_IS_DSS                  (0x4)
+#define AHCI_PX_IS_SDBS                 (0x8)
+#define AHCI_PX_IS_UFS                 (0x10)
+#define AHCI_PX_IS_DPS                 (0x20)
+#define AHCI_PX_IS_PCS                 (0x40)
+#define AHCI_PX_IS_DMPS                (0x80)
+#define AHCI_PX_IS_RESERVED       (0x23FFF00)
+#define AHCI_PX_IS_PRCS            (0x400000)
+#define AHCI_PX_IS_IPMS            (0x800000)
+#define AHCI_PX_IS_OFS            (0x1000000)
+#define AHCI_PX_IS_INFS           (0x4000000)
+#define AHCI_PX_IS_IFS            (0x8000000)
+#define AHCI_PX_IS_HBDS          (0x10000000)
+#define AHCI_PX_IS_HBFS          (0x20000000)
+#define AHCI_PX_IS_TFES          (0x40000000)
+#define AHCI_PX_IS_CPDS          (0x80000000)
+
+#define AHCI_PX_IE                        (5)
+#define AHCI_PX_IE_DHRE                 (0x1)
+#define AHCI_PX_IE_PSE                  (0x2)
+#define AHCI_PX_IE_DSE                  (0x4)
+#define AHCI_PX_IE_SDBE                 (0x8)
+#define AHCI_PX_IE_UFE                 (0x10)
+#define AHCI_PX_IE_DPE                 (0x20)
+#define AHCI_PX_IE_PCE                 (0x40)
+#define AHCI_PX_IE_DMPE                (0x80)
+#define AHCI_PX_IE_RESERVED       (0x23FFF00)
+#define AHCI_PX_IE_PRCE            (0x400000)
+#define AHCI_PX_IE_IPME            (0x800000)
+#define AHCI_PX_IE_OFE            (0x1000000)
+#define AHCI_PX_IE_INFE           (0x4000000)
+#define AHCI_PX_IE_IFE            (0x8000000)
+#define AHCI_PX_IE_HBDE          (0x10000000)
+#define AHCI_PX_IE_HBFE          (0x20000000)
+#define AHCI_PX_IE_TFEE          (0x40000000)
+#define AHCI_PX_IE_CPDE          (0x80000000)
+
+#define AHCI_PX_CMD                       (6)
+#define AHCI_PX_CMD_ST                  (0x1)
+#define AHCI_PX_CMD_SUD                 (0x2)
+#define AHCI_PX_CMD_POD                 (0x4)
+#define AHCI_PX_CMD_CLO                 (0x8)
+#define AHCI_PX_CMD_FRE                (0x10)
+#define AHCI_PX_CMD_RESERVED           (0xE0)
+#define AHCI_PX_CMD_CCS              (0x1F00)
+#define AHCI_PX_CMD_MPSS             (0x2000)
+#define AHCI_PX_CMD_FR               (0x4000)
+#define AHCI_PX_CMD_CR               (0x8000)
+#define AHCI_PX_CMD_CPS             (0x10000)
+#define AHCI_PX_CMD_PMA             (0x20000)
+#define AHCI_PX_CMD_HPCP            (0x40000)
+#define AHCI_PX_CMD_MPSP            (0x80000)
+#define AHCI_PX_CMD_CPD            (0x100000)
+#define AHCI_PX_CMD_ESP            (0x200000)
+#define AHCI_PX_CMD_FBSCP          (0x400000)
+#define AHCI_PX_CMD_APSTE          (0x800000)
+#define AHCI_PX_CMD_ATAPI         (0x1000000)
+#define AHCI_PX_CMD_DLAE          (0x2000000)
+#define AHCI_PX_CMD_ALPE          (0x4000000)
+#define AHCI_PX_CMD_ASP           (0x8000000)
+#define AHCI_PX_CMD_ICC          (0xF0000000)
+
+#define AHCI_PX_RES1                      (7)
+
+#define AHCI_PX_TFD                       (8)
+#define AHCI_PX_TFD_STS                (0xFF)
+#define AHCI_PX_TFD_STS_ERR            (0x01)
+#define AHCI_PX_TFD_STS_CS1            (0x06)
+#define AHCI_PX_TFD_STS_DRQ            (0x08)
+#define AHCI_PX_TFD_STS_CS2            (0x70)
+#define AHCI_PX_TFD_STS_BSY            (0x80)
+#define AHCI_PX_TFD_ERR              (0xFF00)
+#define AHCI_PX_TFD_RESERVED     (0xFFFF0000)
+
+#define AHCI_PX_SIG                       (9)
+#define AHCI_PX_SIG_SECTOR_COUNT       (0xFF)
+#define AHCI_PX_SIG_LBA_LOW          (0xFF00)
+#define AHCI_PX_SIG_LBA_MID        (0xFF0000)
+#define AHCI_PX_SIG_LBA_HIGH     (0xFF000000)
+
+#define AHCI_PX_SSTS                     (10)
+#define AHCI_PX_SSTS_DET               (0x0F)
+#define AHCI_PX_SSTS_SPD               (0xF0)
+#define AHCI_PX_SSTS_IPM              (0xF00)
+#define AHCI_PX_SSTS_RESERVED    (0xFFFFF000)
+#define SSTS_DET_NO_DEVICE             (0x00)
+#define SSTS_DET_PRESENT               (0x01)
+#define SSTS_DET_ESTABLISHED           (0x03)
+#define SSTS_DET_OFFLINE               (0x04)
+
+#define AHCI_PX_SCTL                     (11)
+
+#define AHCI_PX_SERR                     (12)
+#define AHCI_PX_SERR_ERR             (0xFFFF)
+#define AHCI_PX_SERR_DIAG        (0xFFFF0000)
+#define AHCI_PX_SERR_DIAG_X      (0x04000000)
+
+#define AHCI_PX_SACT                     (13)
+#define AHCI_PX_CI                       (14)
+#define AHCI_PX_SNTF                     (15)
+
+#define AHCI_PX_FBS                      (16)
+#define AHCI_PX_FBS_EN                  (0x1)
+#define AHCI_PX_FBS_DEC                 (0x2)
+#define AHCI_PX_FBS_SDE                 (0x4)
+#define AHCI_PX_FBS_DEV               (0xF00)
+#define AHCI_PX_FBS_ADO              (0xF000)
+#define AHCI_PX_FBS_DWE             (0xF0000)
+#define AHCI_PX_FBS_RESERVED     (0xFFF000F8)
+
+#define AHCI_PX_RES2                     (17)
+#define AHCI_PX_VS                       (28)
+
+#define HBA_DATA_REGION_SIZE            (256)
+#define HBA_PORT_DATA_SIZE              (128)
+#define HBA_PORT_NUM_REG (HBA_PORT_DATA_SIZE/4)
+
+#define AHCI_VERSION_0_95        (0x00000905)
+#define AHCI_VERSION_1_0         (0x00010000)
+#define AHCI_VERSION_1_1         (0x00010100)
+#define AHCI_VERSION_1_2         (0x00010200)
+#define AHCI_VERSION_1_3         (0x00010300)
+
+#define AHCI_SECTOR_SIZE                (512)
+#define ATAPI_SECTOR_SIZE              (2048)
+
+#define AHCI_SIGNATURE_CDROM     (0xeb140101)
+#define AHCI_SIGNATURE_DISK      (0x00000101)
+
+/* FIS types */
+enum {
+    REG_H2D_FIS = 0x27,
+    REG_D2H_FIS = 0x34,
+    DMA_ACTIVATE_FIS = 0x39,
+    DMA_SETUP_FIS = 0x41,
+    DATA_FIS = 0x46,
+    BIST_ACTIVATE_FIS = 0x58,
+    PIO_SETUP_FIS = 0x5F,
+    SDB_FIS = 0xA1
+};
+
+/* FIS flags */
+#define REG_H2D_FIS_CMD  0x80
+
+/* ATA Commands */
+enum {
+    /* DMA */
+    CMD_READ_DMA       = 0xC8,
+    CMD_READ_DMA_EXT   = 0x25,
+    CMD_WRITE_DMA      = 0xCA,
+    CMD_WRITE_DMA_EXT  = 0x35,
+    /* PIO */
+    CMD_READ_PIO       = 0x20,
+    CMD_READ_PIO_EXT   = 0x24,
+    CMD_WRITE_PIO      = 0x30,
+    CMD_WRITE_PIO_EXT  = 0x34,
+    /* Misc */
+    CMD_READ_MAX       = 0xF8,
+    CMD_READ_MAX_EXT   = 0x27,
+    CMD_FLUSH_CACHE    = 0xE7,
+    CMD_IDENTIFY       = 0xEC,
+    CMD_PACKET         = 0xA0,
+    CMD_PACKET_ID      = 0xA1,
+    /* NCQ */
+    READ_FPDMA_QUEUED  = 0x60,
+    WRITE_FPDMA_QUEUED = 0x61,
+};
+
+/* ATAPI Commands */
+enum {
+    CMD_ATAPI_READ_10 = 0x28,
+};
+
+/* AHCI Command Header Flags & Masks*/
+#define CMDH_CFL        (0x1F)
+#define CMDH_ATAPI      (0x20)
+#define CMDH_WRITE      (0x40)
+#define CMDH_PREFETCH   (0x80)
+#define CMDH_RESET     (0x100)
+#define CMDH_BIST      (0x200)
+#define CMDH_CLR_BSY   (0x400)
+#define CMDH_RES       (0x800)
+#define CMDH_PMP      (0xF000)
+
+/* ATA device register masks */
+#define ATA_DEVICE_MAGIC 0xA0 /* used in ata1-3 */
+#define ATA_DEVICE_LBA   0x40
+#define NCQ_DEVICE_MAGIC 0x40 /* for ncq device registers */
+#define ATA_DEVICE_DRIVE 0x10
+#define ATA_DEVICE_HEAD  0x0F
+
+/*** Structures ***/
+
+typedef struct AHCIPortQState {
+    uint64_t fb;
+    uint64_t clb;
+    uint64_t ctba[32];
+    uint16_t prdtl[32];
+    uint8_t next; /** Next Command Slot to Use **/
+} AHCIPortQState;
+
+typedef struct AHCIQState {
+    QOSState *parent;
+    QPCIDevice *dev;
+    void *hba_base;
+    uint64_t barsize;
+    uint32_t fingerprint;
+    uint32_t cap;
+    uint32_t cap2;
+    AHCIPortQState port[32];
+} AHCIQState;
+
+/**
+ * Generic FIS structure.
+ */
+typedef struct FIS {
+    uint8_t fis_type;
+    uint8_t flags;
+    char data[0];
+} __attribute__((__packed__)) FIS;
+
+/**
+ * Register device-to-host FIS structure.
+ */
+typedef struct RegD2HFIS {
+    /* DW0 */
+    uint8_t fis_type;
+    uint8_t flags;
+    uint8_t status;
+    uint8_t error;
+    /* DW1 */
+    uint8_t lba_lo[3];
+    uint8_t device;
+    /* DW2 */
+    uint8_t lba_hi[3];
+    uint8_t res0;
+    /* DW3 */
+    uint16_t count;
+    uint16_t res1;
+    /* DW4 */
+    uint32_t res2;
+} __attribute__((__packed__)) RegD2HFIS;
+
+/**
+ * Register device-to-host FIS structure;
+ * PIO Setup variety.
+ */
+typedef struct PIOSetupFIS {
+    /* DW0 */
+    uint8_t fis_type;
+    uint8_t flags;
+    uint8_t status;
+    uint8_t error;
+    /* DW1 */
+    uint8_t lba_lo[3];
+    uint8_t device;
+    /* DW2 */
+    uint8_t lba_hi[3];
+    uint8_t res0;
+    /* DW3 */
+    uint16_t count;
+    uint8_t res1;
+    uint8_t e_status;
+    /* DW4 */
+    uint16_t tx_count;
+    uint16_t res2;
+} __attribute__((__packed__)) PIOSetupFIS;
+
+/**
+ * Register host-to-device FIS structure.
+ */
+typedef struct RegH2DFIS {
+    /* DW0 */
+    uint8_t fis_type;
+    uint8_t flags;
+    uint8_t command;
+    uint8_t feature_low;
+    /* DW1 */
+    uint8_t lba_lo[3];
+    uint8_t device;
+    /* DW2 */
+    uint8_t lba_hi[3];
+    uint8_t feature_high;
+    /* DW3 */
+    uint16_t count;
+    uint8_t icc;
+    uint8_t control;
+    /* DW4 */
+    uint8_t aux[4];
+} __attribute__((__packed__)) RegH2DFIS;
+
+/**
+ * Register host-to-device FIS structure, for NCQ commands.
+ * Actually just a RegH2DFIS, but with fields repurposed.
+ * Repurposed fields are annotated below.
+ */
+typedef struct NCQFIS {
+    /* DW0 */
+    uint8_t fis_type;
+    uint8_t flags;
+    uint8_t command;
+    uint8_t sector_low; /* H2D: Feature 7:0 */
+    /* DW1 */
+    uint8_t lba_lo[3];
+    uint8_t device;
+    /* DW2 */
+    uint8_t lba_hi[3];
+    uint8_t sector_hi; /* H2D: Feature 15:8 */
+    /* DW3 */
+    uint8_t tag;       /* H2D: Count 0:7 */
+    uint8_t prio;      /* H2D: Count 15:8 */
+    uint8_t icc;
+    uint8_t control;
+    /* DW4 */
+    uint8_t aux[4];
+} __attribute__((__packed__)) NCQFIS;
+
+/**
+ * Command List entry structure.
+ * The command list contains between 1-32 of these structures.
+ */
+typedef struct AHCICommandHeader {
+    uint16_t flags; /* Cmd-Fis-Len, PMP#, and flags. */
+    uint16_t prdtl; /* Phys Region Desc. Table Length */
+    uint32_t prdbc; /* Phys Region Desc. Byte Count */
+    uint64_t ctba;  /* Command Table Descriptor Base Address */
+    uint32_t res[4];
+} __attribute__((__packed__)) AHCICommandHeader;
+
+/**
+ * Physical Region Descriptor; pointed to by the Command List Header,
+ * struct ahci_command.
+ */
+typedef struct PRD {
+    uint64_t dba;  /* Data Base Address */
+    uint32_t res;  /* Reserved */
+    uint32_t dbc;  /* Data Byte Count (0-indexed) & Interrupt Flag (bit 2^31) */
+} __attribute__((__packed__)) PRD;
+
+/* Opaque, defined within ahci.c */
+typedef struct AHCICommand AHCICommand;
+
+/* Options to ahci_exec */
+typedef struct AHCIOpts {
+    size_t size;
+    unsigned prd_size;
+    uint64_t lba;
+    uint64_t buffer;
+    bool atapi;
+    bool atapi_dma;
+    bool error;
+    int (*pre_cb)(AHCIQState*, AHCICommand*, const struct AHCIOpts *);
+    int (*mid_cb)(AHCIQState*, AHCICommand*, const struct AHCIOpts *);
+    int (*post_cb)(AHCIQState*, AHCICommand*, const struct AHCIOpts *);
+    void *opaque;
+} AHCIOpts;
+
+/*** Macro Utilities ***/
+#define BITANY(data, mask) (((data) & (mask)) != 0)
+#define BITSET(data, mask) (((data) & (mask)) == (mask))
+#define BITCLR(data, mask) (((data) & (mask)) == 0)
+#define ASSERT_BIT_SET(data, mask) g_assert_cmphex((data) & (mask), ==, (mask))
+#define ASSERT_BIT_CLEAR(data, mask) g_assert_cmphex((data) & (mask), ==, 0)
+
+/* For calculating how big the PRD table needs to be: */
+#define CMD_TBL_SIZ(n) ((0x80 + ((n) * sizeof(PRD)) + 0x7F) & ~0x7F)
+
+/* Helpers for reading/writing AHCI HBA register values */
+
+static inline uint32_t ahci_mread(AHCIQState *ahci, size_t offset)
+{
+    return qpci_io_readl(ahci->dev, ahci->hba_base + offset);
+}
+
+static inline void ahci_mwrite(AHCIQState *ahci, size_t offset, uint32_t value)
+{
+    qpci_io_writel(ahci->dev, ahci->hba_base + offset, value);
+}
+
+static inline uint32_t ahci_rreg(AHCIQState *ahci, uint32_t reg_num)
+{
+    return ahci_mread(ahci, 4 * reg_num);
+}
+
+static inline void ahci_wreg(AHCIQState *ahci, uint32_t reg_num, uint32_t value)
+{
+    ahci_mwrite(ahci, 4 * reg_num, value);
+}
+
+static inline void ahci_set(AHCIQState *ahci, uint32_t reg_num, uint32_t mask)
+{
+    ahci_wreg(ahci, reg_num, ahci_rreg(ahci, reg_num) | mask);
+}
+
+static inline void ahci_clr(AHCIQState *ahci, uint32_t reg_num, uint32_t mask)
+{
+    ahci_wreg(ahci, reg_num, ahci_rreg(ahci, reg_num) & ~mask);
+}
+
+static inline size_t ahci_px_offset(uint8_t port, uint32_t reg_num)
+{
+    return AHCI_PORTS + (HBA_PORT_NUM_REG * port) + reg_num;
+}
+
+static inline uint32_t ahci_px_rreg(AHCIQState *ahci, uint8_t port,
+                                    uint32_t reg_num)
+{
+    return ahci_rreg(ahci, ahci_px_offset(port, reg_num));
+}
+
+static inline void ahci_px_wreg(AHCIQState *ahci, uint8_t port,
+                                uint32_t reg_num, uint32_t value)
+{
+    ahci_wreg(ahci, ahci_px_offset(port, reg_num), value);
+}
+
+static inline void ahci_px_set(AHCIQState *ahci, uint8_t port,
+                               uint32_t reg_num, uint32_t mask)
+{
+    ahci_px_wreg(ahci, port, reg_num,
+                 ahci_px_rreg(ahci, port, reg_num) | mask);
+}
+
+static inline void ahci_px_clr(AHCIQState *ahci, uint8_t port,
+                               uint32_t reg_num, uint32_t mask)
+{
+    ahci_px_wreg(ahci, port, reg_num,
+                 ahci_px_rreg(ahci, port, reg_num) & ~mask);
+}
+
+/*** Prototypes ***/
+uint64_t ahci_alloc(AHCIQState *ahci, size_t bytes);
+void ahci_free(AHCIQState *ahci, uint64_t addr);
+void ahci_clean_mem(AHCIQState *ahci);
+
+/* Device management */
+QPCIDevice *get_ahci_device(uint32_t *fingerprint);
+void free_ahci_device(QPCIDevice *dev);
+void ahci_pci_enable(AHCIQState *ahci);
+void start_ahci_device(AHCIQState *ahci);
+void ahci_hba_enable(AHCIQState *ahci);
+
+/* Port Management */
+unsigned ahci_port_select(AHCIQState *ahci);
+void ahci_port_clear(AHCIQState *ahci, uint8_t port);
+
+/* Command header / table management */
+unsigned ahci_pick_cmd(AHCIQState *ahci, uint8_t port);
+void ahci_get_command_header(AHCIQState *ahci, uint8_t port,
+                             uint8_t slot, AHCICommandHeader *cmd);
+void ahci_set_command_header(AHCIQState *ahci, uint8_t port,
+                             uint8_t slot, AHCICommandHeader *cmd);
+void ahci_destroy_command(AHCIQState *ahci, uint8_t port, uint8_t slot);
+
+/* AHCI sanity check routines */
+void ahci_port_check_error(AHCIQState *ahci, uint8_t port);
+void ahci_port_check_interrupts(AHCIQState *ahci, uint8_t port,
+                                uint32_t intr_mask);
+void ahci_port_check_nonbusy(AHCIQState *ahci, uint8_t port, uint8_t slot);
+void ahci_port_check_d2h_sanity(AHCIQState *ahci, uint8_t port, uint8_t slot);
+void ahci_port_check_pio_sanity(AHCIQState *ahci, uint8_t port,
+                                uint8_t slot, size_t buffsize);
+void ahci_port_check_cmd_sanity(AHCIQState *ahci, AHCICommand *cmd);
+
+/* Misc */
+bool is_atapi(AHCIQState *ahci, uint8_t port);
+unsigned size_to_prdtl(unsigned bytes, unsigned bytes_per_prd);
+
+/* Command: Macro level execution */
+void ahci_guest_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
+                   uint64_t gbuffer, size_t size, uint64_t sector);
+AHCICommand *ahci_guest_io_halt(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
+                                uint64_t gbuffer, size_t size, uint64_t sector);
+void ahci_guest_io_resume(AHCIQState *ahci, AHCICommand *cmd);
+void ahci_io(AHCIQState *ahci, uint8_t port, uint8_t ide_cmd,
+             void *buffer, size_t bufsize, uint64_t sector);
+void ahci_exec(AHCIQState *ahci, uint8_t port,
+               uint8_t op, const AHCIOpts *opts);
+
+/* Command: Fine-grained lifecycle */
+AHCICommand *ahci_command_create(uint8_t command_name);
+AHCICommand *ahci_atapi_command_create(uint8_t scsi_cmd);
+void ahci_command_commit(AHCIQState *ahci, AHCICommand *cmd, uint8_t port);
+void ahci_command_issue(AHCIQState *ahci, AHCICommand *cmd);
+void ahci_command_issue_async(AHCIQState *ahci, AHCICommand *cmd);
+void ahci_command_wait(AHCIQState *ahci, AHCICommand *cmd);
+void ahci_command_verify(AHCIQState *ahci, AHCICommand *cmd);
+void ahci_command_free(AHCICommand *cmd);
+
+/* Command: adjustments */
+void ahci_command_set_flags(AHCICommand *cmd, uint16_t cmdh_flags);
+void ahci_command_clr_flags(AHCICommand *cmd, uint16_t cmdh_flags);
+void ahci_command_set_offset(AHCICommand *cmd, uint64_t lba_sect);
+void ahci_command_set_buffer(AHCICommand *cmd, uint64_t buffer);
+void ahci_command_set_size(AHCICommand *cmd, uint64_t xbytes);
+void ahci_command_set_prd_size(AHCICommand *cmd, unsigned prd_size);
+void ahci_command_set_sizes(AHCICommand *cmd, uint64_t xbytes,
+                            unsigned prd_size);
+void ahci_command_set_acmd(AHCICommand *cmd, void *acmd);
+void ahci_command_enable_atapi_dma(AHCICommand *cmd);
+void ahci_command_adjust(AHCICommand *cmd, uint64_t lba_sect, uint64_t gbuffer,
+                         uint64_t xbytes, unsigned prd_size);
+
+/* Command: Misc */
+uint8_t ahci_command_slot(AHCICommand *cmd);
+void ahci_write_fis(AHCIQState *ahci, AHCICommand *cmd);
+
+#endif
diff --git a/tests/libqos/fw_cfg.c b/tests/libqos/fw_cfg.c
index ef00fed..4d9dc3f 100644
--- a/tests/libqos/fw_cfg.c
+++ b/tests/libqos/fw_cfg.c
@@ -12,7 +12,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "libqos/fw_cfg.h"
 #include "libqtest.h"
 #include "qemu/bswap.h"
diff --git a/tests/libqos/fw_cfg.h b/tests/libqos/fw_cfg.h
index 61b1548..e8371b2 100644
--- a/tests/libqos/fw_cfg.h
+++ b/tests/libqos/fw_cfg.h
@@ -13,8 +13,6 @@
 #ifndef LIBQOS_FW_CFG_H
 #define LIBQOS_FW_CFG_H
 
-#include <stdint.h>
-#include <sys/types.h>
 
 typedef struct QFWCFG QFWCFG;
 
diff --git a/tests/libqos/i2c-imx.c b/tests/libqos/i2c-imx.c
new file mode 100644
index 0000000..1c4b431
--- /dev/null
+++ b/tests/libqos/i2c-imx.c
@@ -0,0 +1,207 @@
+/*
+ * QTest i.MX I2C driver
+ *
+ * Copyright (c) 2013 Jean-Christophe Dubois
+ *
+ *  This program is free software; you can redistribute it and/or modify it
+ *  under the terms of the GNU General Public License as published by the
+ *  Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ *  for more details.
+ *
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "libqos/i2c.h"
+
+
+#include "libqtest.h"
+
+#include "hw/i2c/imx_i2c.h"
+
+enum IMXI2CDirection {
+    IMX_I2C_READ,
+    IMX_I2C_WRITE,
+};
+
+typedef struct IMXI2C {
+    I2CAdapter parent;
+
+    uint64_t addr;
+} IMXI2C;
+
+
+static void imx_i2c_set_slave_addr(IMXI2C *s, uint8_t addr,
+                                   enum IMXI2CDirection direction)
+{
+    writeb(s->addr + I2DR_ADDR, (addr << 1) |
+           (direction == IMX_I2C_READ ? 1 : 0));
+}
+
+static void imx_i2c_send(I2CAdapter *i2c, uint8_t addr,
+                         const uint8_t *buf, uint16_t len)
+{
+    IMXI2C *s = (IMXI2C *)i2c;
+    uint8_t data;
+    uint8_t status;
+    uint16_t size = 0;
+
+    if (!len) {
+        return;
+    }
+
+    /* set the bus for write */
+    data = I2CR_IEN |
+           I2CR_IIEN |
+           I2CR_MSTA |
+           I2CR_MTX |
+           I2CR_TXAK;
+
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* set the slave address */
+    imx_i2c_set_slave_addr(s, addr, IMX_I2C_WRITE);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+    g_assert((status & I2SR_RXAK) == 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    while (size < len) {
+        /* check we are still busy */
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IBB) != 0);
+
+        /* write the data */
+        writeb(s->addr + I2DR_ADDR, buf[size]);
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) != 0);
+        g_assert((status & I2SR_RXAK) == 0);
+
+        /* ack the interrupt */
+        writeb(s->addr + I2SR_ADDR, 0);
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) == 0);
+
+        size++;
+    }
+
+    /* release the bus */
+    data &= ~(I2CR_MSTA | I2CR_MTX);
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) == 0);
+}
+
+static void imx_i2c_recv(I2CAdapter *i2c, uint8_t addr,
+                         uint8_t *buf, uint16_t len)
+{
+    IMXI2C *s = (IMXI2C *)i2c;
+    uint8_t data;
+    uint8_t status;
+    uint16_t size = 0;
+
+    if (!len) {
+        return;
+    }
+
+    /* set the bus for write */
+    data = I2CR_IEN |
+           I2CR_IIEN |
+           I2CR_MSTA |
+           I2CR_MTX |
+           I2CR_TXAK;
+
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* set the slave address */
+    imx_i2c_set_slave_addr(s, addr, IMX_I2C_READ);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+    g_assert((status & I2SR_RXAK) == 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    /* set the bus for read */
+    data &= ~I2CR_MTX;
+    /* if only one byte don't ack */
+    if (len != 1) {
+        data &= ~I2CR_TXAK;
+    }
+    writeb(s->addr + I2CR_ADDR, data);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) != 0);
+
+    /* dummy read */
+    readb(s->addr + I2DR_ADDR);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) != 0);
+
+    /* ack the interrupt */
+    writeb(s->addr + I2SR_ADDR, 0);
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IIF) == 0);
+
+    while (size < len) {
+        /* check we are still busy */
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IBB) != 0);
+
+        if (size == (len - 1)) {
+            /* stop the read transaction */
+            data &= ~(I2CR_MSTA | I2CR_MTX);
+        } else {
+            /* ack the data read */
+            data |= I2CR_TXAK;
+        }
+        writeb(s->addr + I2CR_ADDR, data);
+
+        /* read the data */
+        buf[size] = readb(s->addr + I2DR_ADDR);
+
+        if (size != (len - 1)) {
+            status = readb(s->addr + I2SR_ADDR);
+            g_assert((status & I2SR_IIF) != 0);
+
+            /* ack the interrupt */
+            writeb(s->addr + I2SR_ADDR, 0);
+        }
+
+        status = readb(s->addr + I2SR_ADDR);
+        g_assert((status & I2SR_IIF) == 0);
+
+        size++;
+    }
+
+    status = readb(s->addr + I2SR_ADDR);
+    g_assert((status & I2SR_IBB) == 0);
+}
+
+I2CAdapter *imx_i2c_create(uint64_t addr)
+{
+    IMXI2C *s = g_malloc0(sizeof(*s));
+    I2CAdapter *i2c = (I2CAdapter *)s;
+
+    s->addr = addr;
+
+    i2c->send = imx_i2c_send;
+    i2c->recv = imx_i2c_recv;
+
+    return i2c;
+}
diff --git a/tests/libqos/i2c-omap.c b/tests/libqos/i2c-omap.c
index 3d4d45d..f603fdf 100644
--- a/tests/libqos/i2c-omap.c
+++ b/tests/libqos/i2c-omap.c
@@ -6,12 +6,10 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "libqos/i2c.h"
 
-#include <glib.h>
-#include <string.h>
 
-#include "qemu/osdep.h"
 #include "qemu/bswap.h"
 #include "libqtest.h"
 
diff --git a/tests/libqos/i2c.c b/tests/libqos/i2c.c
index da7592f..23bc2a3 100644
--- a/tests/libqos/i2c.c
+++ b/tests/libqos/i2c.c
@@ -6,6 +6,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
+#include "qemu/osdep.h"
 #include "libqos/i2c.h"
 #include "libqtest.h"
 
diff --git a/tests/libqos/i2c.h b/tests/libqos/i2c.h
index 1ce9af4..6e648f9 100644
--- a/tests/libqos/i2c.h
+++ b/tests/libqos/i2c.h
@@ -9,7 +9,6 @@
 #ifndef LIBQOS_I2C_H
 #define LIBQOS_I2C_H
 
-#include <stdint.h>
 
 typedef struct I2CAdapter I2CAdapter;
 struct I2CAdapter {
@@ -27,4 +26,7 @@
 /* libi2c-omap.c */
 I2CAdapter *omap_i2c_create(uint64_t addr);
 
+/* libi2c-imx.c */
+I2CAdapter *imx_i2c_create(uint64_t addr);
+
 #endif
diff --git a/tests/libqos/libqos-pc.c b/tests/libqos/libqos-pc.c
new file mode 100644
index 0000000..72b5e3b
--- /dev/null
+++ b/tests/libqos/libqos-pc.c
@@ -0,0 +1,30 @@
+#include "qemu/osdep.h"
+#include "libqos/libqos-pc.h"
+#include "libqos/malloc-pc.h"
+
+static QOSOps qos_ops = {
+    .init_allocator = pc_alloc_init_flags,
+    .uninit_allocator = pc_alloc_uninit
+};
+
+QOSState *qtest_pc_vboot(const char *cmdline_fmt, va_list ap)
+{
+    return qtest_vboot(&qos_ops, cmdline_fmt, ap);
+}
+
+QOSState *qtest_pc_boot(const char *cmdline_fmt, ...)
+{
+    QOSState *qs;
+    va_list ap;
+
+    va_start(ap, cmdline_fmt);
+    qs = qtest_vboot(&qos_ops, cmdline_fmt, ap);
+    va_end(ap);
+
+    return qs;
+}
+
+void qtest_pc_shutdown(QOSState *qs)
+{
+    return qtest_shutdown(qs);
+}
diff --git a/tests/libqos/libqos-pc.h b/tests/libqos/libqos-pc.h
new file mode 100644
index 0000000..a0e4c45
--- /dev/null
+++ b/tests/libqos/libqos-pc.h
@@ -0,0 +1,10 @@
+#ifndef LIBQOS_PC_H
+#define LIBQOS_PC_H
+
+#include "libqos/libqos.h"
+
+QOSState *qtest_pc_vboot(const char *cmdline_fmt, va_list ap);
+QOSState *qtest_pc_boot(const char *cmdline_fmt, ...);
+void qtest_pc_shutdown(QOSState *qs);
+
+#endif
diff --git a/tests/libqos/libqos.c b/tests/libqos/libqos.c
new file mode 100644
index 0000000..c7ba441
--- /dev/null
+++ b/tests/libqos/libqos.c
@@ -0,0 +1,253 @@
+#include "qemu/osdep.h"
+#include <sys/wait.h>
+
+#include "libqtest.h"
+#include "libqos/libqos.h"
+#include "libqos/pci.h"
+
+/*** Test Setup & Teardown ***/
+
+/**
+ * Launch QEMU with the given command line,
+ * and then set up interrupts and our guest malloc interface.
+ */
+QOSState *qtest_vboot(QOSOps *ops, const char *cmdline_fmt, va_list ap)
+{
+    char *cmdline;
+
+    struct QOSState *qs = g_malloc(sizeof(QOSState));
+
+    cmdline = g_strdup_vprintf(cmdline_fmt, ap);
+    qs->qts = qtest_start(cmdline);
+    qs->ops = ops;
+    qtest_irq_intercept_in(global_qtest, "ioapic");
+    if (ops && ops->init_allocator) {
+        qs->alloc = ops->init_allocator(ALLOC_NO_FLAGS);
+    }
+
+    g_free(cmdline);
+    return qs;
+}
+
+/**
+ * Launch QEMU with the given command line,
+ * and then set up interrupts and our guest malloc interface.
+ */
+QOSState *qtest_boot(QOSOps *ops, const char *cmdline_fmt, ...)
+{
+    QOSState *qs;
+    va_list ap;
+
+    va_start(ap, cmdline_fmt);
+    qs = qtest_vboot(ops, cmdline_fmt, ap);
+    va_end(ap);
+
+    return qs;
+}
+
+/**
+ * Tear down the QEMU instance.
+ */
+void qtest_shutdown(QOSState *qs)
+{
+    if (qs->alloc && qs->ops && qs->ops->uninit_allocator) {
+        qs->ops->uninit_allocator(qs->alloc);
+        qs->alloc = NULL;
+    }
+    qtest_quit(qs->qts);
+    g_free(qs);
+}
+
+void set_context(QOSState *s)
+{
+    global_qtest = s->qts;
+}
+
+static QDict *qmp_execute(const char *command)
+{
+    char *fmt;
+    QDict *rsp;
+
+    fmt = g_strdup_printf("{ 'execute': '%s' }", command);
+    rsp = qmp(fmt);
+    g_free(fmt);
+
+    return rsp;
+}
+
+void migrate(QOSState *from, QOSState *to, const char *uri)
+{
+    const char *st;
+    char *s;
+    QDict *rsp, *sub;
+    bool running;
+
+    set_context(from);
+
+    /* Is the machine currently running? */
+    rsp = qmp_execute("query-status");
+    g_assert(qdict_haskey(rsp, "return"));
+    sub = qdict_get_qdict(rsp, "return");
+    g_assert(qdict_haskey(sub, "running"));
+    running = qdict_get_bool(sub, "running");
+    QDECREF(rsp);
+
+    /* Issue the migrate command. */
+    s = g_strdup_printf("{ 'execute': 'migrate',"
+                        "'arguments': { 'uri': '%s' } }",
+                        uri);
+    rsp = qmp(s);
+    g_free(s);
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    /* Wait for STOP event, but only if we were running: */
+    if (running) {
+        qmp_eventwait("STOP");
+    }
+
+    /* If we were running, we can wait for an event. */
+    if (running) {
+        migrate_allocator(from->alloc, to->alloc);
+        set_context(to);
+        qmp_eventwait("RESUME");
+        return;
+    }
+
+    /* Otherwise, we need to wait: poll until migration is completed. */
+    while (1) {
+        rsp = qmp_execute("query-migrate");
+        g_assert(qdict_haskey(rsp, "return"));
+        sub = qdict_get_qdict(rsp, "return");
+        g_assert(qdict_haskey(sub, "status"));
+        st = qdict_get_str(sub, "status");
+
+        /* "setup", "active", "completed", "failed", "cancelled" */
+        if (strcmp(st, "completed") == 0) {
+            QDECREF(rsp);
+            break;
+        }
+
+        if ((strcmp(st, "setup") == 0) || (strcmp(st, "active") == 0)) {
+            QDECREF(rsp);
+            g_usleep(5000);
+            continue;
+        }
+
+        fprintf(stderr, "Migration did not complete, status: %s\n", st);
+        g_assert_not_reached();
+    }
+
+    migrate_allocator(from->alloc, to->alloc);
+    set_context(to);
+}
+
+bool have_qemu_img(void)
+{
+    char *rpath;
+    const char *path = getenv("QTEST_QEMU_IMG");
+    if (!path) {
+        return false;
+    }
+
+    rpath = realpath(path, NULL);
+    if (!rpath) {
+        return false;
+    } else {
+        free(rpath);
+        return true;
+    }
+}
+
+void mkimg(const char *file, const char *fmt, unsigned size_mb)
+{
+    gchar *cli;
+    bool ret;
+    int rc;
+    GError *err = NULL;
+    char *qemu_img_path;
+    gchar *out, *out2;
+    char *qemu_img_abs_path;
+
+    qemu_img_path = getenv("QTEST_QEMU_IMG");
+    g_assert(qemu_img_path);
+    qemu_img_abs_path = realpath(qemu_img_path, NULL);
+    g_assert(qemu_img_abs_path);
+
+    cli = g_strdup_printf("%s create -f %s %s %uM", qemu_img_abs_path,
+                          fmt, file, size_mb);
+    ret = g_spawn_command_line_sync(cli, &out, &out2, &rc, &err);
+    if (err) {
+        fprintf(stderr, "%s\n", err->message);
+        g_error_free(err);
+    }
+    g_assert(ret && !err);
+
+    /* In glib 2.34, we have g_spawn_check_exit_status. in 2.12, we don't.
+     * glib 2.43.91 implementation assumes that any non-zero is an error for
+     * windows, but uses extra precautions for Linux. However,
+     * 0 is only possible if the program exited normally, so that should be
+     * sufficient for our purposes on all platforms, here. */
+    if (rc) {
+        fprintf(stderr, "qemu-img returned status code %d\n", rc);
+    }
+    g_assert(!rc);
+
+    g_free(out);
+    g_free(out2);
+    g_free(cli);
+    free(qemu_img_abs_path);
+}
+
+void mkqcow2(const char *file, unsigned size_mb)
+{
+    return mkimg(file, "qcow2", size_mb);
+}
+
+void prepare_blkdebug_script(const char *debug_fn, const char *event)
+{
+    FILE *debug_file = fopen(debug_fn, "w");
+    int ret;
+
+    fprintf(debug_file, "[inject-error]\n");
+    fprintf(debug_file, "event = \"%s\"\n", event);
+    fprintf(debug_file, "errno = \"5\"\n");
+    fprintf(debug_file, "state = \"1\"\n");
+    fprintf(debug_file, "immediately = \"off\"\n");
+    fprintf(debug_file, "once = \"on\"\n");
+
+    fprintf(debug_file, "[set-state]\n");
+    fprintf(debug_file, "event = \"%s\"\n", event);
+    fprintf(debug_file, "new_state = \"2\"\n");
+    fflush(debug_file);
+    g_assert(!ferror(debug_file));
+
+    ret = fclose(debug_file);
+    g_assert(ret == 0);
+}
+
+void generate_pattern(void *buffer, size_t len, size_t cycle_len)
+{
+    int i, j;
+    unsigned char *tx = (unsigned char *)buffer;
+    unsigned char p;
+    size_t *sx;
+
+    /* Write an indicative pattern that varies and is unique per-cycle */
+    p = rand() % 256;
+    for (i = 0; i < len; i++) {
+        tx[i] = p++ % 256;
+        if (i % cycle_len == 0) {
+            p = rand() % 256;
+        }
+    }
+
+    /* force uniqueness by writing an id per-cycle */
+    for (i = 0; i < len / cycle_len; i++) {
+        j = i * cycle_len;
+        if (j + sizeof(*sx) <= len) {
+            sx = (size_t *)&tx[j];
+            *sx = i;
+        }
+    }
+}
diff --git a/tests/libqos/libqos.h b/tests/libqos/libqos.h
new file mode 100644
index 0000000..604980d
--- /dev/null
+++ b/tests/libqos/libqos.h
@@ -0,0 +1,40 @@
+#ifndef LIBQOS_H
+#define LIBQOS_H
+
+#include "libqtest.h"
+#include "libqos/pci.h"
+#include "libqos/malloc-pc.h"
+
+typedef struct QOSOps {
+    QGuestAllocator *(*init_allocator)(QAllocOpts);
+    void (*uninit_allocator)(QGuestAllocator *);
+} QOSOps;
+
+typedef struct QOSState {
+    QTestState *qts;
+    QGuestAllocator *alloc;
+    QOSOps *ops;
+} QOSState;
+
+QOSState *qtest_vboot(QOSOps *ops, const char *cmdline_fmt, va_list ap);
+QOSState *qtest_boot(QOSOps *ops, const char *cmdline_fmt, ...);
+void qtest_shutdown(QOSState *qs);
+bool have_qemu_img(void);
+void mkimg(const char *file, const char *fmt, unsigned size_mb);
+void mkqcow2(const char *file, unsigned size_mb);
+void set_context(QOSState *s);
+void migrate(QOSState *from, QOSState *to, const char *uri);
+void prepare_blkdebug_script(const char *debug_fn, const char *event);
+void generate_pattern(void *buffer, size_t len, size_t cycle_len);
+
+static inline uint64_t qmalloc(QOSState *q, size_t bytes)
+{
+    return guest_alloc(q->alloc, bytes);
+}
+
+static inline void qfree(QOSState *q, uint64_t addr)
+{
+    guest_free(q->alloc, addr);
+}
+
+#endif
diff --git a/tests/libqos/malloc-generic.c b/tests/libqos/malloc-generic.c
new file mode 100644
index 0000000..33ce90b
--- /dev/null
+++ b/tests/libqos/malloc-generic.c
@@ -0,0 +1,39 @@
+/*
+ * Basic libqos generic malloc support
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqos/malloc-generic.h"
+#include "libqos/malloc.h"
+
+/*
+ * Mostly for valgrind happiness, but it does offer
+ * a chokepoint for debugging guest memory leaks, too.
+ */
+void generic_alloc_uninit(QGuestAllocator *allocator)
+{
+    alloc_uninit(allocator);
+}
+
+QGuestAllocator *generic_alloc_init_flags(uint64_t base_addr, uint64_t size,
+                                        uint32_t page_size, QAllocOpts flags)
+{
+    QGuestAllocator *s;
+    uint64_t start = base_addr + (1 << 20); /* Start at 1MB */
+
+    s = alloc_init_flags(flags, start, start + size);
+    alloc_set_page_size(s, page_size);
+
+    return s;
+}
+
+inline QGuestAllocator *generic_alloc_init(uint64_t base_addr, uint64_t size,
+                                                            uint32_t page_size)
+{
+    return generic_alloc_init_flags(base_addr, size, page_size, ALLOC_NO_FLAGS);
+}
diff --git a/tests/libqos/malloc-generic.h b/tests/libqos/malloc-generic.h
new file mode 100644
index 0000000..90104ec
--- /dev/null
+++ b/tests/libqos/malloc-generic.h
@@ -0,0 +1,21 @@
+/*
+ * Basic libqos generic malloc support
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef LIBQOS_MALLOC_GENERIC_H
+#define LIBQOS_MALLOC_GENERIC_H
+
+#include "libqos/malloc.h"
+
+QGuestAllocator *generic_alloc_init(uint64_t base_addr, uint64_t size,
+                                                            uint32_t page_size);
+QGuestAllocator *generic_alloc_init_flags(uint64_t base_addr, uint64_t size,
+                                        uint32_t page_size, QAllocOpts flags);
+void generic_alloc_uninit(QGuestAllocator *allocator);
+
+#endif
diff --git a/tests/libqos/malloc-pc.c b/tests/libqos/malloc-pc.c
index f4218c6..dd2b900 100644
--- a/tests/libqos/malloc-pc.c
+++ b/tests/libqos/malloc-pc.c
@@ -10,325 +10,42 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqos/malloc-pc.h"
 #include "libqos/fw_cfg.h"
 
-#define NO_QEMU_PROTOS
-#include "hw/nvram/fw_cfg.h"
+#include "hw/nvram/fw_cfg_keys.h"
 
 #include "qemu-common.h"
-#include "qemu/queue.h"
-#include <glib.h>
 
 #define PAGE_SIZE (4096)
 
-#define MLIST_ENTNAME entries
-typedef QTAILQ_HEAD(MemList, MemBlock) MemList;
-typedef struct MemBlock {
-    QTAILQ_ENTRY(MemBlock) MLIST_ENTNAME;
-    uint64_t size;
-    uint64_t addr;
-} MemBlock;
-
-typedef struct PCAlloc
-{
-    QGuestAllocator alloc;
-    PCAllocOpts opts;
-    uint64_t start;
-    uint64_t end;
-
-    MemList used;
-    MemList free;
-} PCAlloc;
-
-static MemBlock *mlist_new(uint64_t addr, uint64_t size)
-{
-    MemBlock *block;
-
-    if (!size) {
-        return NULL;
-    }
-    block = g_malloc0(sizeof(MemBlock));
-
-    block->addr = addr;
-    block->size = size;
-
-    return block;
-}
-
-static void mlist_delete(MemList *list, MemBlock *node)
-{
-    g_assert(list && node);
-    QTAILQ_REMOVE(list, node, MLIST_ENTNAME);
-    g_free(node);
-}
-
-static MemBlock *mlist_find_key(MemList *head, uint64_t addr)
-{
-    MemBlock *node;
-    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
-        if (node->addr == addr) {
-            return node;
-        }
-    }
-    return NULL;
-}
-
-static MemBlock *mlist_find_space(MemList *head, uint64_t size)
-{
-    MemBlock *node;
-
-    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
-        if (node->size >= size) {
-            return node;
-        }
-    }
-    return NULL;
-}
-
-static MemBlock *mlist_sort_insert(MemList *head, MemBlock *insr)
-{
-    MemBlock *node;
-    g_assert(head && insr);
-
-    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
-        if (insr->addr < node->addr) {
-            QTAILQ_INSERT_BEFORE(node, insr, MLIST_ENTNAME);
-            return insr;
-        }
-    }
-
-    QTAILQ_INSERT_TAIL(head, insr, MLIST_ENTNAME);
-    return insr;
-}
-
-static inline uint64_t mlist_boundary(MemBlock *node)
-{
-    return node->size + node->addr;
-}
-
-static MemBlock *mlist_join(MemList *head, MemBlock *left, MemBlock *right)
-{
-    g_assert(head && left && right);
-
-    left->size += right->size;
-    mlist_delete(head, right);
-    return left;
-}
-
-static void mlist_coalesce(MemList *head, MemBlock *node)
-{
-    g_assert(node);
-    MemBlock *left;
-    MemBlock *right;
-    char merge;
-
-    do {
-        merge = 0;
-        left = QTAILQ_PREV(node, MemList, MLIST_ENTNAME);
-        right = QTAILQ_NEXT(node, MLIST_ENTNAME);
-
-        /* clowns to the left of me */
-        if (left && mlist_boundary(left) == node->addr) {
-            node = mlist_join(head, left, node);
-            merge = 1;
-        }
-
-        /* jokers to the right */
-        if (right && mlist_boundary(node) == right->addr) {
-            node = mlist_join(head, node, right);
-            merge = 1;
-        }
-
-    } while (merge);
-}
-
-static uint64_t pc_mlist_fulfill(PCAlloc *s, MemBlock *freenode, uint64_t size)
-{
-    uint64_t addr;
-    MemBlock *usednode;
-
-    g_assert(freenode);
-    g_assert_cmpint(freenode->size, >=, size);
-
-    addr = freenode->addr;
-    if (freenode->size == size) {
-        /* re-use this freenode as our used node */
-        QTAILQ_REMOVE(&s->free, freenode, MLIST_ENTNAME);
-        usednode = freenode;
-    } else {
-        /* adjust the free node and create a new used node */
-        freenode->addr += size;
-        freenode->size -= size;
-        usednode = mlist_new(addr, size);
-    }
-
-    mlist_sort_insert(&s->used, usednode);
-    return addr;
-}
-
-/* To assert the correctness of the list.
- * Used only if PC_ALLOC_PARANOID is set. */
-static void pc_mlist_check(PCAlloc *s)
-{
-    MemBlock *node;
-    uint64_t addr = s->start > 0 ? s->start - 1 : 0;
-    uint64_t next = s->start;
-
-    QTAILQ_FOREACH(node, &s->free, MLIST_ENTNAME) {
-        g_assert_cmpint(node->addr, >, addr);
-        g_assert_cmpint(node->addr, >=, next);
-        addr = node->addr;
-        next = node->addr + node->size;
-    }
-
-    addr = s->start > 0 ? s->start - 1 : 0;
-    next = s->start;
-    QTAILQ_FOREACH(node, &s->used, MLIST_ENTNAME) {
-        g_assert_cmpint(node->addr, >, addr);
-        g_assert_cmpint(node->addr, >=, next);
-        addr = node->addr;
-        next = node->addr + node->size;
-    }
-}
-
-static uint64_t pc_mlist_alloc(PCAlloc *s, uint64_t size)
-{
-    MemBlock *node;
-
-    node = mlist_find_space(&s->free, size);
-    if (!node) {
-        fprintf(stderr, "Out of guest memory.\n");
-        g_assert_not_reached();
-    }
-    return pc_mlist_fulfill(s, node, size);
-}
-
-static void pc_mlist_free(PCAlloc *s, uint64_t addr)
-{
-    MemBlock *node;
-
-    if (addr == 0) {
-        return;
-    }
-
-    node = mlist_find_key(&s->used, addr);
-    if (!node) {
-        fprintf(stderr, "Error: no record found for an allocation at "
-                "0x%016" PRIx64 ".\n",
-                addr);
-        g_assert_not_reached();
-    }
-
-    /* Rip it out of the used list and re-insert back into the free list. */
-    QTAILQ_REMOVE(&s->used, node, MLIST_ENTNAME);
-    mlist_sort_insert(&s->free, node);
-    mlist_coalesce(&s->free, node);
-}
-
-static uint64_t pc_alloc(QGuestAllocator *allocator, size_t size)
-{
-    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
-    uint64_t rsize = size;
-    uint64_t naddr;
-
-    rsize += (PAGE_SIZE - 1);
-    rsize &= -PAGE_SIZE;
-    g_assert_cmpint((s->start + rsize), <=, s->end);
-    g_assert_cmpint(rsize, >=, size);
-
-    naddr = pc_mlist_alloc(s, rsize);
-    if (s->opts & PC_ALLOC_PARANOID) {
-        pc_mlist_check(s);
-    }
-
-    return naddr;
-}
-
-static void pc_free(QGuestAllocator *allocator, uint64_t addr)
-{
-    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
-
-    pc_mlist_free(s, addr);
-    if (s->opts & PC_ALLOC_PARANOID) {
-        pc_mlist_check(s);
-    }
-}
-
 /*
  * Mostly for valgrind happiness, but it does offer
  * a chokepoint for debugging guest memory leaks, too.
  */
 void pc_alloc_uninit(QGuestAllocator *allocator)
 {
-    PCAlloc *s = container_of(allocator, PCAlloc, alloc);
-    MemBlock *node;
-    MemBlock *tmp;
-    PCAllocOpts mask;
-
-    /* Check for guest leaks, and destroy the list. */
-    QTAILQ_FOREACH_SAFE(node, &s->used, MLIST_ENTNAME, tmp) {
-        if (s->opts & (PC_ALLOC_LEAK_WARN | PC_ALLOC_LEAK_ASSERT)) {
-            fprintf(stderr, "guest malloc leak @ 0x%016" PRIx64 "; "
-                    "size 0x%016" PRIx64 ".\n",
-                    node->addr, node->size);
-        }
-        if (s->opts & (PC_ALLOC_LEAK_ASSERT)) {
-            g_assert_not_reached();
-        }
-        g_free(node);
-    }
-
-    /* If we have previously asserted that there are no leaks, then there
-     * should be only one node here with a specific address and size. */
-    mask = PC_ALLOC_LEAK_ASSERT | PC_ALLOC_PARANOID;
-    QTAILQ_FOREACH_SAFE(node, &s->free, MLIST_ENTNAME, tmp) {
-        if ((s->opts & mask) == mask) {
-            if ((node->addr != s->start) ||
-                (node->size != s->end - s->start)) {
-                fprintf(stderr, "Free list is corrupted.\n");
-                g_assert_not_reached();
-            }
-        }
-
-        g_free(node);
-    }
-
-    g_free(s);
+    alloc_uninit(allocator);
 }
 
-QGuestAllocator *pc_alloc_init_flags(PCAllocOpts flags)
+QGuestAllocator *pc_alloc_init_flags(QAllocOpts flags)
 {
-    PCAlloc *s = g_malloc0(sizeof(*s));
+    QGuestAllocator *s;
     uint64_t ram_size;
     QFWCFG *fw_cfg = pc_fw_cfg_init();
-    MemBlock *node;
-
-    s->opts = flags;
-    s->alloc.alloc = pc_alloc;
-    s->alloc.free = pc_free;
 
     ram_size = qfw_cfg_get_u64(fw_cfg, FW_CFG_RAM_SIZE);
-
-    /* Start at 1MB */
-    s->start = 1 << 20;
-
-    /* Respect PCI hole */
-    s->end = MIN(ram_size, 0xE0000000);
+    s = alloc_init_flags(flags, 1 << 20, MIN(ram_size, 0xE0000000));
+    alloc_set_page_size(s, PAGE_SIZE);
 
     /* clean-up */
     g_free(fw_cfg);
 
-    QTAILQ_INIT(&s->used);
-    QTAILQ_INIT(&s->free);
-
-    node = mlist_new(s->start, s->end - s->start);
-    QTAILQ_INSERT_HEAD(&s->free, node, MLIST_ENTNAME);
-
-    return &s->alloc;
+    return s;
 }
 
 inline QGuestAllocator *pc_alloc_init(void)
 {
-    return pc_alloc_init_flags(PC_ALLOC_NO_FLAGS);
+    return pc_alloc_init_flags(ALLOC_NO_FLAGS);
 }
diff --git a/tests/libqos/malloc-pc.h b/tests/libqos/malloc-pc.h
index 9f525e3..86ab9f0 100644
--- a/tests/libqos/malloc-pc.h
+++ b/tests/libqos/malloc-pc.h
@@ -15,15 +15,8 @@
 
 #include "libqos/malloc.h"
 
-typedef enum {
-    PC_ALLOC_NO_FLAGS    = 0x00,
-    PC_ALLOC_LEAK_WARN   = 0x01,
-    PC_ALLOC_LEAK_ASSERT = 0x02,
-    PC_ALLOC_PARANOID    = 0x04
-} PCAllocOpts;
-
 QGuestAllocator *pc_alloc_init(void);
-QGuestAllocator *pc_alloc_init_flags(PCAllocOpts flags);
-void             pc_alloc_uninit(QGuestAllocator *allocator);
+QGuestAllocator *pc_alloc_init_flags(QAllocOpts flags);
+void pc_alloc_uninit(QGuestAllocator *allocator);
 
 #endif
diff --git a/tests/libqos/malloc.c b/tests/libqos/malloc.c
new file mode 100644
index 0000000..b8eff5f
--- /dev/null
+++ b/tests/libqos/malloc.c
@@ -0,0 +1,377 @@
+/*
+ * libqos malloc support
+ *
+ * Copyright (c) 2014
+ *
+ * Author:
+ *  John Snow <jsnow@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqos/malloc.h"
+#include "qemu-common.h"
+#include "qemu/host-utils.h"
+
+typedef QTAILQ_HEAD(MemList, MemBlock) MemList;
+
+typedef struct MemBlock {
+    QTAILQ_ENTRY(MemBlock) MLIST_ENTNAME;
+    uint64_t size;
+    uint64_t addr;
+} MemBlock;
+
+struct QGuestAllocator {
+    QAllocOpts opts;
+    uint64_t start;
+    uint64_t end;
+    uint32_t page_size;
+
+    MemList *used;
+    MemList *free;
+};
+
+#define DEFAULT_PAGE_SIZE 4096
+
+static void mlist_delete(MemList *list, MemBlock *node)
+{
+    g_assert(list && node);
+    QTAILQ_REMOVE(list, node, MLIST_ENTNAME);
+    g_free(node);
+}
+
+static MemBlock *mlist_find_key(MemList *head, uint64_t addr)
+{
+    MemBlock *node;
+    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
+        if (node->addr == addr) {
+            return node;
+        }
+    }
+    return NULL;
+}
+
+static MemBlock *mlist_find_space(MemList *head, uint64_t size)
+{
+    MemBlock *node;
+
+    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
+        if (node->size >= size) {
+            return node;
+        }
+    }
+    return NULL;
+}
+
+static MemBlock *mlist_sort_insert(MemList *head, MemBlock *insr)
+{
+    MemBlock *node;
+    g_assert(head && insr);
+
+    QTAILQ_FOREACH(node, head, MLIST_ENTNAME) {
+        if (insr->addr < node->addr) {
+            QTAILQ_INSERT_BEFORE(node, insr, MLIST_ENTNAME);
+            return insr;
+        }
+    }
+
+    QTAILQ_INSERT_TAIL(head, insr, MLIST_ENTNAME);
+    return insr;
+}
+
+static inline uint64_t mlist_boundary(MemBlock *node)
+{
+    return node->size + node->addr;
+}
+
+static MemBlock *mlist_join(MemList *head, MemBlock *left, MemBlock *right)
+{
+    g_assert(head && left && right);
+
+    left->size += right->size;
+    mlist_delete(head, right);
+    return left;
+}
+
+static void mlist_coalesce(MemList *head, MemBlock *node)
+{
+    g_assert(node);
+    MemBlock *left;
+    MemBlock *right;
+    char merge;
+
+    do {
+        merge = 0;
+        left = QTAILQ_PREV(node, MemList, MLIST_ENTNAME);
+        right = QTAILQ_NEXT(node, MLIST_ENTNAME);
+
+        /* clowns to the left of me */
+        if (left && mlist_boundary(left) == node->addr) {
+            node = mlist_join(head, left, node);
+            merge = 1;
+        }
+
+        /* jokers to the right */
+        if (right && mlist_boundary(node) == right->addr) {
+            node = mlist_join(head, node, right);
+            merge = 1;
+        }
+
+    } while (merge);
+}
+
+static MemBlock *mlist_new(uint64_t addr, uint64_t size)
+{
+    MemBlock *block;
+
+    if (!size) {
+        return NULL;
+    }
+    block = g_malloc0(sizeof(MemBlock));
+
+    block->addr = addr;
+    block->size = size;
+
+    return block;
+}
+
+static uint64_t mlist_fulfill(QGuestAllocator *s, MemBlock *freenode,
+                                                                uint64_t size)
+{
+    uint64_t addr;
+    MemBlock *usednode;
+
+    g_assert(freenode);
+    g_assert_cmpint(freenode->size, >=, size);
+
+    addr = freenode->addr;
+    if (freenode->size == size) {
+        /* re-use this freenode as our used node */
+        QTAILQ_REMOVE(s->free, freenode, MLIST_ENTNAME);
+        usednode = freenode;
+    } else {
+        /* adjust the free node and create a new used node */
+        freenode->addr += size;
+        freenode->size -= size;
+        usednode = mlist_new(addr, size);
+    }
+
+    mlist_sort_insert(s->used, usednode);
+    return addr;
+}
+
+/* To assert the correctness of the list.
+ * Used only if ALLOC_PARANOID is set. */
+static void mlist_check(QGuestAllocator *s)
+{
+    MemBlock *node;
+    uint64_t addr = s->start > 0 ? s->start - 1 : 0;
+    uint64_t next = s->start;
+
+    QTAILQ_FOREACH(node, s->free, MLIST_ENTNAME) {
+        g_assert_cmpint(node->addr, >, addr);
+        g_assert_cmpint(node->addr, >=, next);
+        addr = node->addr;
+        next = node->addr + node->size;
+    }
+
+    addr = s->start > 0 ? s->start - 1 : 0;
+    next = s->start;
+    QTAILQ_FOREACH(node, s->used, MLIST_ENTNAME) {
+        g_assert_cmpint(node->addr, >, addr);
+        g_assert_cmpint(node->addr, >=, next);
+        addr = node->addr;
+        next = node->addr + node->size;
+    }
+}
+
+static uint64_t mlist_alloc(QGuestAllocator *s, uint64_t size)
+{
+    MemBlock *node;
+
+    node = mlist_find_space(s->free, size);
+    if (!node) {
+        fprintf(stderr, "Out of guest memory.\n");
+        g_assert_not_reached();
+    }
+    return mlist_fulfill(s, node, size);
+}
+
+static void mlist_free(QGuestAllocator *s, uint64_t addr)
+{
+    MemBlock *node;
+
+    if (addr == 0) {
+        return;
+    }
+
+    node = mlist_find_key(s->used, addr);
+    if (!node) {
+        fprintf(stderr, "Error: no record found for an allocation at "
+                "0x%016" PRIx64 ".\n",
+                addr);
+        g_assert_not_reached();
+    }
+
+    /* Rip it out of the used list and re-insert back into the free list. */
+    QTAILQ_REMOVE(s->used, node, MLIST_ENTNAME);
+    mlist_sort_insert(s->free, node);
+    mlist_coalesce(s->free, node);
+}
+
+/*
+ * Mostly for valgrind happiness, but it does offer
+ * a chokepoint for debugging guest memory leaks, too.
+ */
+void alloc_uninit(QGuestAllocator *allocator)
+{
+    MemBlock *node;
+    MemBlock *tmp;
+    QAllocOpts mask;
+
+    /* Check for guest leaks, and destroy the list. */
+    QTAILQ_FOREACH_SAFE(node, allocator->used, MLIST_ENTNAME, tmp) {
+        if (allocator->opts & (ALLOC_LEAK_WARN | ALLOC_LEAK_ASSERT)) {
+            fprintf(stderr, "guest malloc leak @ 0x%016" PRIx64 "; "
+                    "size 0x%016" PRIx64 ".\n",
+                    node->addr, node->size);
+        }
+        if (allocator->opts & (ALLOC_LEAK_ASSERT)) {
+            g_assert_not_reached();
+        }
+        g_free(node);
+    }
+
+    /* If we have previously asserted that there are no leaks, then there
+     * should be only one node here with a specific address and size. */
+    mask = ALLOC_LEAK_ASSERT | ALLOC_PARANOID;
+    QTAILQ_FOREACH_SAFE(node, allocator->free, MLIST_ENTNAME, tmp) {
+        if ((allocator->opts & mask) == mask) {
+            if ((node->addr != allocator->start) ||
+                (node->size != allocator->end - allocator->start)) {
+                fprintf(stderr, "Free list is corrupted.\n");
+                g_assert_not_reached();
+            }
+        }
+
+        g_free(node);
+    }
+
+    g_free(allocator->used);
+    g_free(allocator->free);
+    g_free(allocator);
+}
+
+uint64_t guest_alloc(QGuestAllocator *allocator, size_t size)
+{
+    uint64_t rsize = size;
+    uint64_t naddr;
+
+    if (!size) {
+        return 0;
+    }
+
+    rsize += (allocator->page_size - 1);
+    rsize &= -allocator->page_size;
+    g_assert_cmpint((allocator->start + rsize), <=, allocator->end);
+    g_assert_cmpint(rsize, >=, size);
+
+    naddr = mlist_alloc(allocator, rsize);
+    if (allocator->opts & ALLOC_PARANOID) {
+        mlist_check(allocator);
+    }
+
+    return naddr;
+}
+
+void guest_free(QGuestAllocator *allocator, uint64_t addr)
+{
+    if (!addr) {
+        return;
+    }
+    mlist_free(allocator, addr);
+    if (allocator->opts & ALLOC_PARANOID) {
+        mlist_check(allocator);
+    }
+}
+
+QGuestAllocator *alloc_init(uint64_t start, uint64_t end)
+{
+    QGuestAllocator *s = g_malloc0(sizeof(*s));
+    MemBlock *node;
+
+    s->start = start;
+    s->end = end;
+
+    s->used = g_malloc(sizeof(MemList));
+    s->free = g_malloc(sizeof(MemList));
+    QTAILQ_INIT(s->used);
+    QTAILQ_INIT(s->free);
+
+    node = mlist_new(s->start, s->end - s->start);
+    QTAILQ_INSERT_HEAD(s->free, node, MLIST_ENTNAME);
+
+    s->page_size = DEFAULT_PAGE_SIZE;
+
+    return s;
+}
+
+QGuestAllocator *alloc_init_flags(QAllocOpts opts,
+                                  uint64_t start, uint64_t end)
+{
+    QGuestAllocator *s = alloc_init(start, end);
+    s->opts = opts;
+    return s;
+}
+
+void alloc_set_page_size(QGuestAllocator *allocator, size_t page_size)
+{
+    /* Can't alter the page_size for an allocator in-use */
+    g_assert(QTAILQ_EMPTY(allocator->used));
+
+    g_assert(is_power_of_2(page_size));
+    allocator->page_size = page_size;
+}
+
+void alloc_set_flags(QGuestAllocator *allocator, QAllocOpts opts)
+{
+    allocator->opts |= opts;
+}
+
+void migrate_allocator(QGuestAllocator *src,
+                       QGuestAllocator *dst)
+{
+    MemBlock *node, *tmp;
+    MemList *tmpused, *tmpfree;
+
+    /* The general memory layout should be equivalent,
+     * though opts can differ. */
+    g_assert_cmphex(src->start, ==, dst->start);
+    g_assert_cmphex(src->end, ==, dst->end);
+
+    /* Destroy (silently, regardless of options) the dest-list: */
+    QTAILQ_FOREACH_SAFE(node, dst->used, MLIST_ENTNAME, tmp) {
+        g_free(node);
+    }
+    QTAILQ_FOREACH_SAFE(node, dst->free, MLIST_ENTNAME, tmp) {
+        g_free(node);
+    }
+
+    tmpused = dst->used;
+    tmpfree = dst->free;
+
+    /* Inherit the lists of the source allocator: */
+    dst->used = src->used;
+    dst->free = src->free;
+
+    /* Source is now re-initialized, the source memory is 'invalid' now: */
+    src->used = tmpused;
+    src->free = tmpfree;
+    QTAILQ_INIT(src->used);
+    QTAILQ_INIT(src->free);
+    node = mlist_new(src->start, src->end - src->start);
+    QTAILQ_INSERT_HEAD(src->free, node, MLIST_ENTNAME);
+    return;
+}
diff --git a/tests/libqos/malloc.h b/tests/libqos/malloc.h
index 5565381..ae9dac8 100644
--- a/tests/libqos/malloc.h
+++ b/tests/libqos/malloc.h
@@ -13,26 +13,28 @@
 #ifndef LIBQOS_MALLOC_H
 #define LIBQOS_MALLOC_H
 
-#include <stdint.h>
-#include <sys/types.h>
+#include "qemu/queue.h"
+
+typedef enum {
+    ALLOC_NO_FLAGS    = 0x00,
+    ALLOC_LEAK_WARN   = 0x01,
+    ALLOC_LEAK_ASSERT = 0x02,
+    ALLOC_PARANOID    = 0x04
+} QAllocOpts;
 
 typedef struct QGuestAllocator QGuestAllocator;
 
-struct QGuestAllocator
-{
-    uint64_t (*alloc)(QGuestAllocator *allocator, size_t size);
-    void (*free)(QGuestAllocator *allocator, uint64_t addr);
-};
+void alloc_uninit(QGuestAllocator *allocator);
 
 /* Always returns page aligned values */
-static inline uint64_t guest_alloc(QGuestAllocator *allocator, size_t size)
-{
-    return allocator->alloc(allocator, size);
-}
+uint64_t guest_alloc(QGuestAllocator *allocator, size_t size);
+void guest_free(QGuestAllocator *allocator, uint64_t addr);
+void migrate_allocator(QGuestAllocator *src, QGuestAllocator *dst);
 
-static inline void guest_free(QGuestAllocator *allocator, uint64_t addr)
-{
-    allocator->free(allocator, addr);
-}
+QGuestAllocator *alloc_init(uint64_t start, uint64_t end);
+QGuestAllocator *alloc_init_flags(QAllocOpts flags,
+                                  uint64_t start, uint64_t end);
+void alloc_set_page_size(QGuestAllocator *allocator, size_t page_size);
+void alloc_set_flags(QGuestAllocator *allocator, QAllocOpts opts);
 
 #endif
diff --git a/tests/libqos/pci-pc.c b/tests/libqos/pci-pc.c
index 6dba0db..1ae2d37 100644
--- a/tests/libqos/pci-pc.c
+++ b/tests/libqos/pci-pc.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "libqos/pci-pc.h"
 
@@ -18,7 +19,6 @@
 #include "qemu-common.h"
 #include "qemu/host-utils.h"
 
-#include <glib.h>
 
 #define ACPI_PCIHP_ADDR         0xae00
 #define PCI_EJ_BASE             0x0008
@@ -183,7 +183,9 @@
     if (io_type == PCI_BASE_ADDRESS_SPACE_IO) {
         uint16_t loc;
 
-        g_assert((s->pci_iohole_alloc + size) <= s->pci_iohole_size);
+        g_assert(QEMU_ALIGN_UP(s->pci_iohole_alloc, size) + size
+                 <= s->pci_iohole_size);
+        s->pci_iohole_alloc = QEMU_ALIGN_UP(s->pci_iohole_alloc, size);
         loc = s->pci_iohole_start + s->pci_iohole_alloc;
         s->pci_iohole_alloc += size;
 
@@ -193,7 +195,9 @@
     } else {
         uint64_t loc;
 
-        g_assert((s->pci_hole_alloc + size) <= s->pci_hole_size);
+        g_assert(QEMU_ALIGN_UP(s->pci_hole_alloc, size) + size
+                 <= s->pci_hole_size);
+        s->pci_hole_alloc = QEMU_ALIGN_UP(s->pci_hole_alloc, size);
         loc = s->pci_hole_start + s->pci_hole_alloc;
         s->pci_hole_alloc += size;
 
diff --git a/tests/libqos/pci.c b/tests/libqos/pci.c
index 4e630c2..ed78d91 100644
--- a/tests/libqos/pci.c
+++ b/tests/libqos/pci.c
@@ -10,10 +10,10 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "libqos/pci.h"
 
 #include "hw/pci/pci_regs.h"
-#include <glib.h>
 
 void qpci_device_foreach(QPCIBus *bus, int vendor_id, int device_id,
                          void (*func)(QPCIDevice *dev, int devfn, void *data),
@@ -34,11 +34,13 @@
 
             if (vendor_id != -1 &&
                 qpci_config_readw(dev, PCI_VENDOR_ID) != vendor_id) {
+                g_free(dev);
                 continue;
             }
 
             if (device_id != -1 &&
                 qpci_config_readw(dev, PCI_DEVICE_ID) != device_id) {
+                g_free(dev);
                 continue;
             }
 
diff --git a/tests/libqos/pci.h b/tests/libqos/pci.h
index dfaee9e..c06add8 100644
--- a/tests/libqos/pci.h
+++ b/tests/libqos/pci.h
@@ -13,7 +13,6 @@
 #ifndef LIBQOS_PCI_H
 #define LIBQOS_PCI_H
 
-#include <stdint.h>
 #include "libqtest.h"
 
 #define QPCI_DEVFN(dev, fn) (((dev) << 3) | (fn))
diff --git a/tests/libqos/usb.c b/tests/libqos/usb.c
index 41d89b8..f794d92 100644
--- a/tests/libqos/usb.c
+++ b/tests/libqos/usb.c
@@ -11,10 +11,8 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 #include "hw/usb/uhci-regs.h"
 #include "libqos/usb.h"
 
diff --git a/tests/libqos/virtio-mmio.c b/tests/libqos/virtio-mmio.c
new file mode 100644
index 0000000..0cab38f
--- /dev/null
+++ b/tests/libqos/virtio-mmio.c
@@ -0,0 +1,206 @@
+/*
+ * libqos virtio MMIO driver
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-mmio.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-generic.h"
+#include "standard-headers/linux/virtio_ring.h"
+
+static uint8_t qvirtio_mmio_config_readb(QVirtioDevice *d, uint64_t addr)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return readb(dev->addr + addr);
+}
+
+static uint16_t qvirtio_mmio_config_readw(QVirtioDevice *d, uint64_t addr)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return readw(dev->addr + addr);
+}
+
+static uint32_t qvirtio_mmio_config_readl(QVirtioDevice *d, uint64_t addr)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return readl(dev->addr + addr);
+}
+
+static uint64_t qvirtio_mmio_config_readq(QVirtioDevice *d, uint64_t addr)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return readq(dev->addr + addr);
+}
+
+static uint32_t qvirtio_mmio_get_features(QVirtioDevice *d)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    writel(dev->addr + QVIRTIO_MMIO_HOST_FEATURES_SEL, 0);
+    return readl(dev->addr + QVIRTIO_MMIO_HOST_FEATURES);
+}
+
+static void qvirtio_mmio_set_features(QVirtioDevice *d, uint32_t features)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    dev->features = features;
+    writel(dev->addr + QVIRTIO_MMIO_GUEST_FEATURES_SEL, 0);
+    writel(dev->addr + QVIRTIO_MMIO_GUEST_FEATURES, features);
+}
+
+static uint32_t qvirtio_mmio_get_guest_features(QVirtioDevice *d)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return dev->features;
+}
+
+static uint8_t qvirtio_mmio_get_status(QVirtioDevice *d)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return (uint8_t)readl(dev->addr + QVIRTIO_MMIO_DEVICE_STATUS);
+}
+
+static void qvirtio_mmio_set_status(QVirtioDevice *d, uint8_t status)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    writel(dev->addr + QVIRTIO_MMIO_DEVICE_STATUS, (uint32_t)status);
+}
+
+static bool qvirtio_mmio_get_queue_isr_status(QVirtioDevice *d, QVirtQueue *vq)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    uint32_t isr;
+
+    isr = readl(dev->addr + QVIRTIO_MMIO_INTERRUPT_STATUS) & 1;
+    if (isr != 0) {
+        writel(dev->addr + QVIRTIO_MMIO_INTERRUPT_ACK, 1);
+        return true;
+    }
+
+    return false;
+}
+
+static bool qvirtio_mmio_get_config_isr_status(QVirtioDevice *d)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    uint32_t isr;
+
+    isr = readl(dev->addr + QVIRTIO_MMIO_INTERRUPT_STATUS) & 2;
+    if (isr != 0) {
+        writel(dev->addr + QVIRTIO_MMIO_INTERRUPT_ACK, 2);
+        return true;
+    }
+
+    return false;
+}
+
+static void qvirtio_mmio_queue_select(QVirtioDevice *d, uint16_t index)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    writel(dev->addr + QVIRTIO_MMIO_QUEUE_SEL, (uint32_t)index);
+
+    g_assert_cmphex(readl(dev->addr + QVIRTIO_MMIO_QUEUE_PFN), ==, 0);
+}
+
+static uint16_t qvirtio_mmio_get_queue_size(QVirtioDevice *d)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    return (uint16_t)readl(dev->addr + QVIRTIO_MMIO_QUEUE_NUM_MAX);
+}
+
+static void qvirtio_mmio_set_queue_address(QVirtioDevice *d, uint32_t pfn)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    writel(dev->addr + QVIRTIO_MMIO_QUEUE_PFN, pfn);
+}
+
+static QVirtQueue *qvirtio_mmio_virtqueue_setup(QVirtioDevice *d,
+                                        QGuestAllocator *alloc, uint16_t index)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    QVirtQueue *vq;
+    uint64_t addr;
+
+    vq = g_malloc0(sizeof(*vq));
+    qvirtio_mmio_queue_select(d, index);
+    writel(dev->addr + QVIRTIO_MMIO_QUEUE_ALIGN, dev->page_size);
+
+    vq->index = index;
+    vq->size = qvirtio_mmio_get_queue_size(d);
+    vq->free_head = 0;
+    vq->num_free = vq->size;
+    vq->align = dev->page_size;
+    vq->indirect = (dev->features & (1u << VIRTIO_RING_F_INDIRECT_DESC)) != 0;
+    vq->event = (dev->features & (1u << VIRTIO_RING_F_EVENT_IDX)) != 0;
+
+    writel(dev->addr + QVIRTIO_MMIO_QUEUE_NUM, vq->size);
+
+    /* Check different than 0 */
+    g_assert_cmpint(vq->size, !=, 0);
+
+    /* Check power of 2 */
+    g_assert_cmpint(vq->size & (vq->size - 1), ==, 0);
+
+    addr = guest_alloc(alloc, qvring_size(vq->size, dev->page_size));
+    qvring_init(alloc, vq, addr);
+    qvirtio_mmio_set_queue_address(d, vq->desc / dev->page_size);
+
+    return vq;
+}
+
+static void qvirtio_mmio_virtqueue_cleanup(QVirtQueue *vq,
+                                           QGuestAllocator *alloc)
+{
+    guest_free(alloc, vq->desc);
+    g_free(vq);
+}
+
+static void qvirtio_mmio_virtqueue_kick(QVirtioDevice *d, QVirtQueue *vq)
+{
+    QVirtioMMIODevice *dev = (QVirtioMMIODevice *)d;
+    writel(dev->addr + QVIRTIO_MMIO_QUEUE_NOTIFY, vq->index);
+}
+
+const QVirtioBus qvirtio_mmio = {
+    .config_readb = qvirtio_mmio_config_readb,
+    .config_readw = qvirtio_mmio_config_readw,
+    .config_readl = qvirtio_mmio_config_readl,
+    .config_readq = qvirtio_mmio_config_readq,
+    .get_features = qvirtio_mmio_get_features,
+    .set_features = qvirtio_mmio_set_features,
+    .get_guest_features = qvirtio_mmio_get_guest_features,
+    .get_status = qvirtio_mmio_get_status,
+    .set_status = qvirtio_mmio_set_status,
+    .get_queue_isr_status = qvirtio_mmio_get_queue_isr_status,
+    .get_config_isr_status = qvirtio_mmio_get_config_isr_status,
+    .queue_select = qvirtio_mmio_queue_select,
+    .get_queue_size = qvirtio_mmio_get_queue_size,
+    .set_queue_address = qvirtio_mmio_set_queue_address,
+    .virtqueue_setup = qvirtio_mmio_virtqueue_setup,
+    .virtqueue_cleanup = qvirtio_mmio_virtqueue_cleanup,
+    .virtqueue_kick = qvirtio_mmio_virtqueue_kick,
+};
+
+QVirtioMMIODevice *qvirtio_mmio_init_device(uint64_t addr, uint32_t page_size)
+{
+    QVirtioMMIODevice *dev;
+    uint32_t magic;
+    dev = g_malloc0(sizeof(*dev));
+
+    magic = readl(addr + QVIRTIO_MMIO_MAGIC_VALUE);
+    g_assert(magic == ('v' | 'i' << 8 | 'r' << 16 | 't' << 24));
+
+    dev->addr = addr;
+    dev->page_size = page_size;
+    dev->vdev.device_type = readl(addr + QVIRTIO_MMIO_DEVICE_ID);
+
+    writel(addr + QVIRTIO_MMIO_GUEST_PAGE_SIZE, page_size);
+
+    return dev;
+}
diff --git a/tests/libqos/virtio-mmio.h b/tests/libqos/virtio-mmio.h
new file mode 100644
index 0000000..e3e52b9
--- /dev/null
+++ b/tests/libqos/virtio-mmio.h
@@ -0,0 +1,46 @@
+/*
+ * libqos virtio MMIO definitions
+ *
+ * Copyright (c) 2014 Marc Marí
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef LIBQOS_VIRTIO_MMIO_H
+#define LIBQOS_VIRTIO_MMIO_H
+
+#include "libqos/virtio.h"
+
+#define QVIRTIO_MMIO_MAGIC_VALUE        0x000
+#define QVIRTIO_MMIO_VERSION            0x004
+#define QVIRTIO_MMIO_DEVICE_ID          0x008
+#define QVIRTIO_MMIO_VENDOR_ID          0x00C
+#define QVIRTIO_MMIO_HOST_FEATURES      0x010
+#define QVIRTIO_MMIO_HOST_FEATURES_SEL  0x014
+#define QVIRTIO_MMIO_GUEST_FEATURES     0x020
+#define QVIRTIO_MMIO_GUEST_FEATURES_SEL 0x024
+#define QVIRTIO_MMIO_GUEST_PAGE_SIZE    0x028
+#define QVIRTIO_MMIO_QUEUE_SEL          0x030
+#define QVIRTIO_MMIO_QUEUE_NUM_MAX      0x034
+#define QVIRTIO_MMIO_QUEUE_NUM          0x038
+#define QVIRTIO_MMIO_QUEUE_ALIGN        0x03C
+#define QVIRTIO_MMIO_QUEUE_PFN          0x040
+#define QVIRTIO_MMIO_QUEUE_NOTIFY       0x050
+#define QVIRTIO_MMIO_INTERRUPT_STATUS   0x060
+#define QVIRTIO_MMIO_INTERRUPT_ACK      0x064
+#define QVIRTIO_MMIO_DEVICE_STATUS      0x070
+#define QVIRTIO_MMIO_DEVICE_SPECIFIC    0x100
+
+typedef struct QVirtioMMIODevice {
+    QVirtioDevice vdev;
+    uint64_t addr;
+    uint32_t page_size;
+    uint32_t features; /* As it cannot be read later, save it */
+} QVirtioMMIODevice;
+
+extern const QVirtioBus qvirtio_mmio;
+
+QVirtioMMIODevice *qvirtio_mmio_init_device(uint64_t addr, uint32_t page_size);
+
+#endif
diff --git a/tests/libqos/virtio-pci.c b/tests/libqos/virtio-pci.c
index 788ebaf..18b92b9 100644
--- a/tests/libqos/virtio-pci.c
+++ b/tests/libqos/virtio-pci.c
@@ -7,8 +7,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-pci.h"
@@ -16,7 +15,10 @@
 #include "libqos/pci-pc.h"
 #include "libqos/malloc.h"
 #include "libqos/malloc-pc.h"
+#include "standard-headers/linux/virtio_ring.h"
+#include "standard-headers/linux/virtio_pci.h"
 
+#include "hw/pci/pci.h"
 #include "hw/pci/pci_regs.h"
 
 typedef struct QVirtioPCIForeachData {
@@ -60,25 +62,25 @@
     *vpcidev = (QVirtioPCIDevice *)d;
 }
 
-static uint8_t qvirtio_pci_config_readb(QVirtioDevice *d, void *addr)
+static uint8_t qvirtio_pci_config_readb(QVirtioDevice *d, uint64_t addr)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readb(dev->pdev, addr);
+    return qpci_io_readb(dev->pdev, (void *)(uintptr_t)addr);
 }
 
-static uint16_t qvirtio_pci_config_readw(QVirtioDevice *d, void *addr)
+static uint16_t qvirtio_pci_config_readw(QVirtioDevice *d, uint64_t addr)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readw(dev->pdev, addr);
+    return qpci_io_readw(dev->pdev, (void *)(uintptr_t)addr);
 }
 
-static uint32_t qvirtio_pci_config_readl(QVirtioDevice *d, void *addr)
+static uint32_t qvirtio_pci_config_readl(QVirtioDevice *d, uint64_t addr)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readl(dev->pdev, addr);
+    return qpci_io_readl(dev->pdev, (void *)(uintptr_t)addr);
 }
 
-static uint64_t qvirtio_pci_config_readq(QVirtioDevice *d, void *addr)
+static uint64_t qvirtio_pci_config_readq(QVirtioDevice *d, uint64_t addr)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
     int i;
@@ -86,11 +88,13 @@
 
     if (qtest_big_endian()) {
         for (i = 0; i < 8; ++i) {
-            u64 |= (uint64_t)qpci_io_readb(dev->pdev, addr + i) << (7 - i) * 8;
+            u64 |= (uint64_t)qpci_io_readb(dev->pdev,
+                                (void *)(uintptr_t)addr + i) << (7 - i) * 8;
         }
     } else {
         for (i = 0; i < 8; ++i) {
-            u64 |= (uint64_t)qpci_io_readb(dev->pdev, addr + i) << i * 8;
+            u64 |= (uint64_t)qpci_io_readb(dev->pdev,
+                                (void *)(uintptr_t)addr + i) << i * 8;
         }
     }
 
@@ -100,31 +104,31 @@
 static uint32_t qvirtio_pci_get_features(QVirtioDevice *d)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readl(dev->pdev, dev->addr + QVIRTIO_DEVICE_FEATURES);
+    return qpci_io_readl(dev->pdev, dev->addr + VIRTIO_PCI_HOST_FEATURES);
 }
 
 static void qvirtio_pci_set_features(QVirtioDevice *d, uint32_t features)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    qpci_io_writel(dev->pdev, dev->addr + QVIRTIO_GUEST_FEATURES, features);
+    qpci_io_writel(dev->pdev, dev->addr + VIRTIO_PCI_GUEST_FEATURES, features);
 }
 
 static uint32_t qvirtio_pci_get_guest_features(QVirtioDevice *d)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readl(dev->pdev, dev->addr + QVIRTIO_GUEST_FEATURES);
+    return qpci_io_readl(dev->pdev, dev->addr + VIRTIO_PCI_GUEST_FEATURES);
 }
 
 static uint8_t qvirtio_pci_get_status(QVirtioDevice *d)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readb(dev->pdev, dev->addr + QVIRTIO_DEVICE_STATUS);
+    return qpci_io_readb(dev->pdev, dev->addr + VIRTIO_PCI_STATUS);
 }
 
 static void qvirtio_pci_set_status(QVirtioDevice *d, uint8_t status)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    qpci_io_writeb(dev->pdev, dev->addr + QVIRTIO_DEVICE_STATUS, status);
+    qpci_io_writeb(dev->pdev, dev->addr + VIRTIO_PCI_STATUS, status);
 }
 
 static bool qvirtio_pci_get_queue_isr_status(QVirtioDevice *d, QVirtQueue *vq)
@@ -140,11 +144,15 @@
             return qpci_msix_pending(dev->pdev, vqpci->msix_entry);
         } else {
             data = readl(vqpci->msix_addr);
-            writel(vqpci->msix_addr, 0);
-            return data == vqpci->msix_data;
+            if (data == vqpci->msix_data) {
+                writel(vqpci->msix_addr, 0);
+                return true;
+            } else {
+                return false;
+            }
         }
     } else {
-        return qpci_io_readb(dev->pdev, dev->addr + QVIRTIO_ISR_STATUS) & 1;
+        return qpci_io_readb(dev->pdev, dev->addr + VIRTIO_PCI_ISR) & 1;
     }
 }
 
@@ -160,30 +168,34 @@
             return qpci_msix_pending(dev->pdev, dev->config_msix_entry);
         } else {
             data = readl(dev->config_msix_addr);
-            writel(dev->config_msix_addr, 0);
-            return data == dev->config_msix_data;
+            if (data == dev->config_msix_data) {
+                writel(dev->config_msix_addr, 0);
+                return true;
+            } else {
+                return false;
+            }
         }
     } else {
-        return qpci_io_readb(dev->pdev, dev->addr + QVIRTIO_ISR_STATUS) & 2;
+        return qpci_io_readb(dev->pdev, dev->addr + VIRTIO_PCI_ISR) & 2;
     }
 }
 
 static void qvirtio_pci_queue_select(QVirtioDevice *d, uint16_t index)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    qpci_io_writeb(dev->pdev, dev->addr + QVIRTIO_QUEUE_SELECT, index);
+    qpci_io_writeb(dev->pdev, dev->addr + VIRTIO_PCI_QUEUE_SEL, index);
 }
 
 static uint16_t qvirtio_pci_get_queue_size(QVirtioDevice *d)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    return qpci_io_readw(dev->pdev, dev->addr + QVIRTIO_QUEUE_SIZE);
+    return qpci_io_readw(dev->pdev, dev->addr + VIRTIO_PCI_QUEUE_NUM);
 }
 
 static void qvirtio_pci_set_queue_address(QVirtioDevice *d, uint32_t pfn)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    qpci_io_writel(dev->pdev, dev->addr + QVIRTIO_QUEUE_ADDRESS, pfn);
+    qpci_io_writel(dev->pdev, dev->addr + VIRTIO_PCI_QUEUE_PFN, pfn);
 }
 
 static QVirtQueue *qvirtio_pci_virtqueue_setup(QVirtioDevice *d,
@@ -201,9 +213,9 @@
     vqpci->vq.size = qvirtio_pci_get_queue_size(d);
     vqpci->vq.free_head = 0;
     vqpci->vq.num_free = vqpci->vq.size;
-    vqpci->vq.align = QVIRTIO_PCI_ALIGN;
-    vqpci->vq.indirect = (feat & QVIRTIO_F_RING_INDIRECT_DESC) != 0;
-    vqpci->vq.event = (feat & QVIRTIO_F_RING_EVENT_IDX) != 0;
+    vqpci->vq.align = VIRTIO_PCI_VRING_ALIGN;
+    vqpci->vq.indirect = (feat & (1u << VIRTIO_RING_F_INDIRECT_DESC)) != 0;
+    vqpci->vq.event = (feat & (1u << VIRTIO_RING_F_EVENT_IDX)) != 0;
 
     vqpci->msix_entry = -1;
     vqpci->msix_addr = 0;
@@ -215,17 +227,27 @@
     /* Check power of 2 */
     g_assert_cmpint(vqpci->vq.size & (vqpci->vq.size - 1), ==, 0);
 
-    addr = guest_alloc(alloc, qvring_size(vqpci->vq.size, QVIRTIO_PCI_ALIGN));
+    addr = guest_alloc(alloc, qvring_size(vqpci->vq.size,
+                                          VIRTIO_PCI_VRING_ALIGN));
     qvring_init(alloc, &vqpci->vq, addr);
-    qvirtio_pci_set_queue_address(d, vqpci->vq.desc / QVIRTIO_PCI_ALIGN);
+    qvirtio_pci_set_queue_address(d, vqpci->vq.desc / VIRTIO_PCI_VRING_ALIGN);
 
     return &vqpci->vq;
 }
 
+static void qvirtio_pci_virtqueue_cleanup(QVirtQueue *vq,
+                                          QGuestAllocator *alloc)
+{
+    QVirtQueuePCI *vqpci = container_of(vq, QVirtQueuePCI, vq);
+
+    guest_free(alloc, vq->desc);
+    g_free(vqpci);
+}
+
 static void qvirtio_pci_virtqueue_kick(QVirtioDevice *d, QVirtQueue *vq)
 {
     QVirtioPCIDevice *dev = (QVirtioPCIDevice *)d;
-    qpci_io_writew(dev->pdev, dev->addr + QVIRTIO_QUEUE_NOTIFY, vq->index);
+    qpci_io_writew(dev->pdev, dev->addr + VIRTIO_PCI_QUEUE_NOTIFY, vq->index);
 }
 
 const QVirtioBus qvirtio_pci = {
@@ -244,6 +266,7 @@
     .get_queue_size = qvirtio_pci_get_queue_size,
     .set_queue_address = qvirtio_pci_set_queue_address,
     .virtqueue_setup = qvirtio_pci_virtqueue_setup,
+    .virtqueue_cleanup = qvirtio_pci_virtqueue_cleanup,
     .virtqueue_kick = qvirtio_pci_virtqueue_kick,
 };
 
@@ -254,7 +277,7 @@
                                 .device_type = device_type,
                                 .user_data = data };
 
-    qpci_device_foreach(bus, QVIRTIO_VENDOR_ID, -1,
+    qpci_device_foreach(bus, PCI_VENDOR_ID_REDHAT_QUMRANET, -1,
                                 qvirtio_pci_foreach_callback, &d);
 }
 
@@ -305,9 +328,9 @@
                                         control & ~PCI_MSIX_ENTRY_CTRL_MASKBIT);
 
     qvirtio_pci_queue_select(&d->vdev, vqpci->vq.index);
-    qpci_io_writew(d->pdev, d->addr + QVIRTIO_MSIX_QUEUE_VECTOR, entry);
-    vector = qpci_io_readw(d->pdev, d->addr + QVIRTIO_MSIX_QUEUE_VECTOR);
-    g_assert_cmphex(vector, !=, QVIRTIO_MSI_NO_VECTOR);
+    qpci_io_writew(d->pdev, d->addr + VIRTIO_MSI_QUEUE_VECTOR, entry);
+    vector = qpci_io_readw(d->pdev, d->addr + VIRTIO_MSI_QUEUE_VECTOR);
+    g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR);
 }
 
 void qvirtio_pci_set_msix_configuration_vector(QVirtioPCIDevice *d,
@@ -337,7 +360,7 @@
     qpci_io_writel(d->pdev, addr + PCI_MSIX_ENTRY_VECTOR_CTRL,
                                         control & ~PCI_MSIX_ENTRY_CTRL_MASKBIT);
 
-    qpci_io_writew(d->pdev, d->addr + QVIRTIO_MSIX_CONF_VECTOR, entry);
-    vector = qpci_io_readw(d->pdev, d->addr + QVIRTIO_MSIX_CONF_VECTOR);
-    g_assert_cmphex(vector, !=, QVIRTIO_MSI_NO_VECTOR);
+    qpci_io_writew(d->pdev, d->addr + VIRTIO_MSI_CONFIG_VECTOR, entry);
+    vector = qpci_io_readw(d->pdev, d->addr + VIRTIO_MSI_CONFIG_VECTOR);
+    g_assert_cmphex(vector, !=, VIRTIO_MSI_NO_VECTOR);
 }
diff --git a/tests/libqos/virtio-pci.h b/tests/libqos/virtio-pci.h
index 883f7ff..efcac2d 100644
--- a/tests/libqos/virtio-pci.h
+++ b/tests/libqos/virtio-pci.h
@@ -13,23 +13,6 @@
 #include "libqos/virtio.h"
 #include "libqos/pci.h"
 
-#define QVIRTIO_DEVICE_FEATURES         0x00
-#define QVIRTIO_GUEST_FEATURES          0x04
-#define QVIRTIO_QUEUE_ADDRESS           0x08
-#define QVIRTIO_QUEUE_SIZE              0x0C
-#define QVIRTIO_QUEUE_SELECT            0x0E
-#define QVIRTIO_QUEUE_NOTIFY            0x10
-#define QVIRTIO_DEVICE_STATUS           0x12
-#define QVIRTIO_ISR_STATUS              0x13
-#define QVIRTIO_MSIX_CONF_VECTOR        0x14
-#define QVIRTIO_MSIX_QUEUE_VECTOR       0x16
-#define QVIRTIO_DEVICE_SPECIFIC_MSIX    0x18
-#define QVIRTIO_DEVICE_SPECIFIC_NO_MSIX 0x14
-
-#define QVIRTIO_PCI_ALIGN   4096
-
-#define QVIRTIO_MSI_NO_VECTOR   0xFFFF
-
 typedef struct QVirtioPCIDevice {
     QVirtioDevice vdev;
     QPCIDevice *pdev;
diff --git a/tests/libqos/virtio.c b/tests/libqos/virtio.c
index a061289..d8c2970 100644
--- a/tests/libqos/virtio.c
+++ b/tests/libqos/virtio.c
@@ -7,30 +7,32 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "libqos/virtio.h"
+#include "standard-headers/linux/virtio_config.h"
+#include "standard-headers/linux/virtio_ring.h"
 
 uint8_t qvirtio_config_readb(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr)
+                                                                uint64_t addr)
 {
     return bus->config_readb(d, addr);
 }
 
 uint16_t qvirtio_config_readw(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr)
+                                                                uint64_t addr)
 {
     return bus->config_readw(d, addr);
 }
 
 uint32_t qvirtio_config_readl(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr)
+                                                                uint64_t addr)
 {
     return bus->config_readl(d, addr);
 }
 
 uint64_t qvirtio_config_readq(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr)
+                                                                uint64_t addr)
 {
     return bus->config_readq(d, addr);
 }
@@ -52,30 +54,36 @@
     return bus->virtqueue_setup(d, alloc, index);
 }
 
+void qvirtqueue_cleanup(const QVirtioBus *bus, QVirtQueue *vq,
+                        QGuestAllocator *alloc)
+{
+    return bus->virtqueue_cleanup(vq, alloc);
+}
+
 void qvirtio_reset(const QVirtioBus *bus, QVirtioDevice *d)
 {
-    bus->set_status(d, QVIRTIO_RESET);
-    g_assert_cmphex(bus->get_status(d), ==, QVIRTIO_RESET);
+    bus->set_status(d, 0);
+    g_assert_cmphex(bus->get_status(d), ==, 0);
 }
 
 void qvirtio_set_acknowledge(const QVirtioBus *bus, QVirtioDevice *d)
 {
-    bus->set_status(d, bus->get_status(d) | QVIRTIO_ACKNOWLEDGE);
-    g_assert_cmphex(bus->get_status(d), ==, QVIRTIO_ACKNOWLEDGE);
+    bus->set_status(d, bus->get_status(d) | VIRTIO_CONFIG_S_ACKNOWLEDGE);
+    g_assert_cmphex(bus->get_status(d), ==, VIRTIO_CONFIG_S_ACKNOWLEDGE);
 }
 
 void qvirtio_set_driver(const QVirtioBus *bus, QVirtioDevice *d)
 {
-    bus->set_status(d, bus->get_status(d) | QVIRTIO_DRIVER);
+    bus->set_status(d, bus->get_status(d) | VIRTIO_CONFIG_S_DRIVER);
     g_assert_cmphex(bus->get_status(d), ==,
-                                    QVIRTIO_DRIVER | QVIRTIO_ACKNOWLEDGE);
+                    VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_ACKNOWLEDGE);
 }
 
 void qvirtio_set_driver_ok(const QVirtioBus *bus, QVirtioDevice *d)
 {
-    bus->set_status(d, bus->get_status(d) | QVIRTIO_DRIVER_OK);
-    g_assert_cmphex(bus->get_status(d), ==,
-                QVIRTIO_DRIVER_OK | QVIRTIO_DRIVER | QVIRTIO_ACKNOWLEDGE);
+    bus->set_status(d, bus->get_status(d) | VIRTIO_CONFIG_S_DRIVER_OK);
+    g_assert_cmphex(bus->get_status(d), ==, VIRTIO_CONFIG_S_DRIVER_OK |
+                    VIRTIO_CONFIG_S_DRIVER | VIRTIO_CONFIG_S_ACKNOWLEDGE);
 }
 
 void qvirtio_wait_queue_isr(const QVirtioBus *bus, QVirtioDevice *d,
@@ -133,7 +141,7 @@
     int i;
 
     vq->desc = addr;
-    vq->avail = vq->desc + vq->size*sizeof(QVRingDesc);
+    vq->avail = vq->desc + vq->size * sizeof(struct vring_desc);
     vq->used = (uint64_t)((vq->avail + sizeof(uint16_t) * (3 + vq->size)
         + vq->align - 1) & ~(vq->align - 1));
 
@@ -154,7 +162,7 @@
     /* vq->used->flags */
     writew(vq->used, 0);
     /* vq->used->avail_event */
-    writew(vq->used+2+(sizeof(struct QVRingUsedElem)*vq->size), 0);
+    writew(vq->used + 2 + sizeof(struct vring_used_elem) * vq->size, 0);
 }
 
 QVRingIndirectDesc *qvring_indirect_desc_setup(QVirtioDevice *d,
@@ -165,13 +173,13 @@
 
     indirect->index = 0;
     indirect->elem = elem;
-    indirect->desc = guest_alloc(alloc, sizeof(QVRingDesc)*elem);
+    indirect->desc = guest_alloc(alloc, sizeof(struct vring_desc) * elem);
 
     for (i = 0; i < elem - 1; ++i) {
         /* indirect->desc[i].addr */
         writeq(indirect->desc + (16 * i), 0);
         /* indirect->desc[i].flags */
-        writew(indirect->desc + (16 * i) + 12, QVRING_DESC_F_NEXT);
+        writew(indirect->desc + (16 * i) + 12, VRING_DESC_F_NEXT);
         /* indirect->desc[i].next */
         writew(indirect->desc + (16 * i) + 14, i + 1);
     }
@@ -189,7 +197,7 @@
     flags = readw(indirect->desc + (16 * indirect->index) + 12);
 
     if (write) {
-        flags |= QVRING_DESC_F_WRITE;
+        flags |= VRING_DESC_F_WRITE;
     }
 
     /* indirect->desc[indirect->index].addr */
@@ -209,11 +217,11 @@
     vq->num_free--;
 
     if (write) {
-        flags |= QVRING_DESC_F_WRITE;
+        flags |= VRING_DESC_F_WRITE;
     }
 
     if (next) {
-        flags |= QVRING_DESC_F_NEXT;
+        flags |= VRING_DESC_F_NEXT;
     }
 
     /* vq->desc[vq->free_head].addr */
@@ -238,9 +246,9 @@
     writeq(vq->desc + (16 * vq->free_head), indirect->desc);
     /* vq->desc[vq->free_head].len */
     writel(vq->desc + (16 * vq->free_head) + 8,
-                                        sizeof(QVRingDesc) * indirect->elem);
+           sizeof(struct vring_desc) * indirect->elem);
     /* vq->desc[vq->free_head].flags */
-    writew(vq->desc + (16 * vq->free_head) + 12, QVRING_DESC_F_INDIRECT);
+    writew(vq->desc + (16 * vq->free_head) + 12, VRING_DESC_F_INDIRECT);
 
     return vq->free_head++; /* Return and increase, in this order */
 }
@@ -263,10 +271,10 @@
     /* Must read after idx is updated */
     flags = readw(vq->avail);
     avail_event = readw(vq->used + 4 +
-                                (sizeof(struct QVRingUsedElem) * vq->size));
+                                sizeof(struct vring_used_elem) * vq->size);
 
     /* < 1 because we add elements to avail queue one by one */
-    if ((flags & QVRING_USED_F_NO_NOTIFY) == 0 &&
+    if ((flags & VRING_USED_F_NO_NOTIFY) == 0 &&
                             (!vq->event || (uint16_t)(idx-avail_event) < 1)) {
         bus->virtqueue_kick(d, vq);
     }
diff --git a/tests/libqos/virtio.h b/tests/libqos/virtio.h
index 29fbacb..0250842 100644
--- a/tests/libqos/virtio.h
+++ b/tests/libqos/virtio.h
@@ -11,72 +11,19 @@
 #define LIBQOS_VIRTIO_H
 
 #include "libqos/malloc.h"
+#include "standard-headers/linux/virtio_ring.h"
 
-#define QVIRTIO_VENDOR_ID       0x1AF4
-
-#define QVIRTIO_RESET           0x0
-#define QVIRTIO_ACKNOWLEDGE     0x1
-#define QVIRTIO_DRIVER          0x2
-#define QVIRTIO_DRIVER_OK       0x4
-
-#define QVIRTIO_NET_DEVICE_ID   0x1
-#define QVIRTIO_BLK_DEVICE_ID   0x2
-
-#define QVIRTIO_F_NOTIFY_ON_EMPTY       0x01000000
-#define QVIRTIO_F_ANY_LAYOUT            0x08000000
-#define QVIRTIO_F_RING_INDIRECT_DESC    0x10000000
-#define QVIRTIO_F_RING_EVENT_IDX        0x20000000
 #define QVIRTIO_F_BAD_FEATURE           0x40000000
 
-#define QVRING_DESC_F_NEXT      0x1
-#define QVRING_DESC_F_WRITE     0x2
-#define QVRING_DESC_F_INDIRECT  0x4
-
-#define QVIRTIO_F_NOTIFY_ON_EMPTY       0x01000000
-#define QVIRTIO_F_ANY_LAYOUT            0x08000000
-#define QVIRTIO_F_RING_INDIRECT_DESC    0x10000000
-#define QVIRTIO_F_RING_EVENT_IDX        0x20000000
-#define QVIRTIO_F_BAD_FEATURE           0x40000000
-
-#define QVRING_AVAIL_F_NO_INTERRUPT     1
-
-#define QVRING_USED_F_NO_NOTIFY     1
-
 typedef struct QVirtioDevice {
     /* Device type */
     uint16_t device_type;
 } QVirtioDevice;
 
-typedef struct QVRingDesc {
-    uint64_t addr;
-    uint32_t len;
-    uint16_t flags;
-    uint16_t next;
-} QVRingDesc;
-
-typedef struct QVRingAvail {
-    uint16_t flags;
-    uint16_t idx;
-    uint16_t ring[0]; /* This is an array of uint16_t */
-    uint16_t used_event;
-} QVRingAvail;
-
-typedef struct QVRingUsedElem {
-    uint32_t id;
-    uint32_t len;
-} QVRingUsedElem;
-
-typedef struct QVRingUsed {
-    uint16_t flags;
-    uint16_t idx;
-    QVRingUsedElem ring[0]; /* This is an array of QVRingUsedElem structs */
-    uint16_t avail_event;
-} QVRingUsed;
-
 typedef struct QVirtQueue {
-    uint64_t desc; /* This points to an array of QVRingDesc */
-    uint64_t avail; /* This points to a QVRingAvail */
-    uint64_t used; /* This points to a QVRingDesc */
+    uint64_t desc; /* This points to an array of struct vring_desc */
+    uint64_t avail; /* This points to a struct vring_avail */
+    uint64_t used; /* This points to a struct vring_desc */
     uint16_t index;
     uint32_t size;
     uint32_t free_head;
@@ -87,16 +34,16 @@
 } QVirtQueue;
 
 typedef struct QVRingIndirectDesc {
-    uint64_t desc; /* This points to an array fo QVRingDesc */
+    uint64_t desc; /* This points to an array fo struct vring_desc */
     uint16_t index;
     uint16_t elem;
 } QVRingIndirectDesc;
 
 typedef struct QVirtioBus {
-    uint8_t (*config_readb)(QVirtioDevice *d, void *addr);
-    uint16_t (*config_readw)(QVirtioDevice *d, void *addr);
-    uint32_t (*config_readl)(QVirtioDevice *d, void *addr);
-    uint64_t (*config_readq)(QVirtioDevice *d, void *addr);
+    uint8_t (*config_readb)(QVirtioDevice *d, uint64_t addr);
+    uint16_t (*config_readw)(QVirtioDevice *d, uint64_t addr);
+    uint32_t (*config_readl)(QVirtioDevice *d, uint64_t addr);
+    uint64_t (*config_readq)(QVirtioDevice *d, uint64_t addr);
 
     /* Get features of the device */
     uint32_t (*get_features)(QVirtioDevice *d);
@@ -132,25 +79,28 @@
     QVirtQueue *(*virtqueue_setup)(QVirtioDevice *d, QGuestAllocator *alloc,
                                                                 uint16_t index);
 
+    /* Free virtqueue resources */
+    void (*virtqueue_cleanup)(QVirtQueue *vq, QGuestAllocator *alloc);
+
     /* Notify changes in virtqueue */
     void (*virtqueue_kick)(QVirtioDevice *d, QVirtQueue *vq);
 } QVirtioBus;
 
 static inline uint32_t qvring_size(uint32_t num, uint32_t align)
 {
-    return ((sizeof(struct QVRingDesc) * num + sizeof(uint16_t) * (3 + num)
+    return ((sizeof(struct vring_desc) * num + sizeof(uint16_t) * (3 + num)
         + align - 1) & ~(align - 1))
-        + sizeof(uint16_t) * 3 + sizeof(struct QVRingUsedElem) * num;
+        + sizeof(uint16_t) * 3 + sizeof(struct vring_used_elem) * num;
 }
 
 uint8_t qvirtio_config_readb(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr);
+                                                                uint64_t addr);
 uint16_t qvirtio_config_readw(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr);
+                                                                uint64_t addr);
 uint32_t qvirtio_config_readl(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr);
+                                                                uint64_t addr);
 uint64_t qvirtio_config_readq(const QVirtioBus *bus, QVirtioDevice *d,
-                                                                void *addr);
+                                                                uint64_t addr);
 uint32_t qvirtio_get_features(const QVirtioBus *bus, QVirtioDevice *d);
 void qvirtio_set_features(const QVirtioBus *bus, QVirtioDevice *d,
                                                             uint32_t features);
@@ -171,6 +121,8 @@
                              gint64 timeout_us);
 QVirtQueue *qvirtqueue_setup(const QVirtioBus *bus, QVirtioDevice *d,
                                         QGuestAllocator *alloc, uint16_t index);
+void qvirtqueue_cleanup(const QVirtioBus *bus, QVirtQueue *vq,
+                        QGuestAllocator *alloc);
 
 void qvring_init(const QGuestAllocator *alloc, QVirtQueue *vq, uint64_t addr);
 QVRingIndirectDesc *qvring_indirect_desc_setup(QVirtioDevice *d,
diff --git a/tests/libqtest.c b/tests/libqtest.c
index 9a92aa7..eb00f13 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -14,28 +14,19 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "libqtest.h"
 
-#include <glib.h>
-#include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/wait.h>
 #include <sys/un.h>
-#include <inttypes.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
 
-#include "qemu/compiler.h"
-#include "qemu/osdep.h"
 #include "qapi/qmp/json-parser.h"
 #include "qapi/qmp/json-streamer.h"
 #include "qapi/qmp/qjson.h"
 
 #define MAX_IRQ 256
-#define SOCKET_TIMEOUT 5
+#define SOCKET_TIMEOUT 50
 
 QTestState *global_qtest;
 
@@ -46,9 +37,9 @@
     bool irq_level[MAX_IRQ];
     GString *rx;
     pid_t qemu_pid;  /* our child QEMU process */
-    struct sigaction sigact_old; /* restored on exit */
 };
 
+static GHookList abrt_hooks;
 static GList *qtest_instances;
 static struct sigaction sigact_old;
 
@@ -110,12 +101,14 @@
     }
 }
 
+static void kill_qemu_hook_func(void *s)
+{
+    kill_qemu(s);
+}
+
 static void sigabrt_handler(int signo)
 {
-    GList *elem;
-    for (elem = qtest_instances; elem; elem = elem->next) {
-        kill_qemu(elem->data);
-    }
+    g_hook_list_invoke(&abrt_hooks, FALSE);
 }
 
 static void setup_sigabrt_handler(void)
@@ -136,6 +129,23 @@
     sigaction(SIGABRT, &sigact_old, NULL);
 }
 
+void qtest_add_abrt_handler(GHookFunc fn, const void *data)
+{
+    GHook *hook;
+
+    /* Only install SIGABRT handler once */
+    if (!abrt_hooks.is_setup) {
+        g_hook_list_init(&abrt_hooks, sizeof(GHook));
+        setup_sigabrt_handler();
+    }
+
+    hook = g_hook_alloc(&abrt_hooks);
+    hook->func = fn;
+    hook->data = (void *)data;
+
+    g_hook_prepend(&abrt_hooks, hook);
+}
+
 QTestState *qtest_init(const char *extra_args)
 {
     QTestState *s;
@@ -156,12 +166,7 @@
     sock = init_socket(socket_path);
     qmpsock = init_socket(qmp_socket_path);
 
-    /* Only install SIGABRT handler once */
-    if (!qtest_instances) {
-        setup_sigabrt_handler();
-    }
-
-    qtest_instances = g_list_prepend(qtest_instances, s);
+    qtest_add_abrt_handler(kill_qemu_hook_func, s);
 
     s->qemu_pid = fork();
     if (s->qemu_pid == 0) {
@@ -209,13 +214,14 @@
 
 void qtest_quit(QTestState *s)
 {
+    qtest_instances = g_list_remove(qtest_instances, s);
+    g_hook_destroy_link(&abrt_hooks, g_hook_find_data(&abrt_hooks, TRUE, s));
+
     /* Uninstall SIGABRT handler on last instance */
-    if (qtest_instances && !qtest_instances->next) {
+    if (!qtest_instances) {
         cleanup_sigabrt_handler();
     }
 
-    qtest_instances = g_list_remove(qtest_instances, s);
-
     kill_qemu(s);
     close(s->fd);
     close(s->qmp_fd);
@@ -341,7 +347,7 @@
     QDict *response;
 } QMPResponseParser;
 
-static void qmp_response(JSONMessageParser *parser, QList *tokens)
+static void qmp_response(JSONMessageParser *parser, GQueue *tokens)
 {
     QMPResponseParser *qmp = container_of(parser, QMPResponseParser, parser);
     QObject *obj;
@@ -357,7 +363,7 @@
     qmp->response = (QDict *)obj;
 }
 
-QDict *qtest_qmp_receive(QTestState *s)
+QDict *qmp_fd_receive(int fd)
 {
     QMPResponseParser qmp;
     bool log = getenv("QTEST_LOG") != NULL;
@@ -368,7 +374,7 @@
         ssize_t len;
         char c;
 
-        len = read(s->qmp_fd, &c, 1);
+        len = read(fd, &c, 1);
         if (len == -1 && errno == EINTR) {
             continue;
         }
@@ -388,7 +394,17 @@
     return qmp.response;
 }
 
-QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
+QDict *qtest_qmp_receive(QTestState *s)
+{
+    return qmp_fd_receive(s->qmp_fd);
+}
+
+/**
+ * Allow users to send a message without waiting for the reply,
+ * in the case that they choose to discard all replies up until
+ * a particular EVENT is received.
+ */
+void qmp_fd_sendv(int fd, const char *fmt, va_list ap)
 {
     va_list ap_copy;
     QObject *qobj;
@@ -412,16 +428,53 @@
             fprintf(stderr, "%s", str);
         }
         /* Send QMP request */
-        socket_send(s->qmp_fd, str, size);
+        socket_send(fd, str, size);
 
         QDECREF(qstr);
         qobject_decref(qobj);
     }
+}
+
+void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap)
+{
+    qmp_fd_sendv(s->qmp_fd, fmt, ap);
+}
+
+QDict *qmp_fdv(int fd, const char *fmt, va_list ap)
+{
+    qmp_fd_sendv(fd, fmt, ap);
+
+    return qmp_fd_receive(fd);
+}
+
+QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap)
+{
+    qtest_async_qmpv(s, fmt, ap);
 
     /* Receive reply */
     return qtest_qmp_receive(s);
 }
 
+QDict *qmp_fd(int fd, const char *fmt, ...)
+{
+    va_list ap;
+    QDict *response;
+
+    va_start(ap, fmt);
+    response = qmp_fdv(fd, fmt, ap);
+    va_end(ap);
+    return response;
+}
+
+void qmp_fd_send(int fd, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    qmp_fd_sendv(fd, fmt, ap);
+    va_end(ap);
+}
+
 QDict *qtest_qmp(QTestState *s, const char *fmt, ...)
 {
     va_list ap;
@@ -433,6 +486,15 @@
     return response;
 }
 
+void qtest_async_qmp(QTestState *s, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    qtest_async_qmpv(s, fmt, ap);
+    va_end(ap);
+}
+
 void qtest_qmpv_discard_response(QTestState *s, const char *fmt, va_list ap)
 {
     QDict *response = qtest_qmpv(s, fmt, ap);
@@ -450,9 +512,53 @@
     QDECREF(response);
 }
 
+void qtest_qmp_eventwait(QTestState *s, const char *event)
+{
+    QDict *response;
+
+    for (;;) {
+        response = qtest_qmp_receive(s);
+        if ((qdict_haskey(response, "event")) &&
+            (strcmp(qdict_get_str(response, "event"), event) == 0)) {
+            QDECREF(response);
+            break;
+        }
+        QDECREF(response);
+    }
+}
+
+char *qtest_hmpv(QTestState *s, const char *fmt, va_list ap)
+{
+    char *cmd;
+    QDict *resp;
+    char *ret;
+
+    cmd = g_strdup_vprintf(fmt, ap);
+    resp = qtest_qmp(s, "{'execute': 'human-monitor-command',"
+                     " 'arguments': {'command-line': %s}}",
+                     cmd);
+    ret = g_strdup(qdict_get_try_str(resp, "return"));
+    g_assert(ret);
+    QDECREF(resp);
+    g_free(cmd);
+    return ret;
+}
+
+char *qtest_hmp(QTestState *s, const char *fmt, ...)
+{
+    va_list ap;
+    char *ret;
+
+    va_start(ap, fmt);
+    ret = qtest_hmpv(s, fmt, ap);
+    va_end(ap);
+    return ret;
+}
+
 const char *qtest_get_arch(void)
 {
     const char *qemu = getenv("QTEST_QEMU_BINARY");
+    g_assert(qemu != NULL);
     const char *end = strrchr(qemu, '/');
 
     return end + strlen("/qemu-system-");
@@ -645,35 +751,70 @@
     g_strfreev(args);
 }
 
-void qtest_add_func(const char *str, void (*fn))
+void qtest_add_func(const char *str, void (*fn)(void))
 {
     gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
     g_test_add_func(path, fn);
     g_free(path);
 }
 
+void qtest_add_data_func(const char *str, const void *data,
+                         void (*fn)(const void *))
+{
+    gchar *path = g_strdup_printf("/%s/%s", qtest_get_arch(), str);
+    g_test_add_data_func(path, data, fn);
+    g_free(path);
+}
+
+void qtest_bufwrite(QTestState *s, uint64_t addr, const void *data, size_t size)
+{
+    gchar *bdata;
+
+    bdata = g_base64_encode(data, size);
+    qtest_sendf(s, "b64write 0x%" PRIx64 " 0x%zx ", addr, size);
+    socket_send(s->fd, bdata, strlen(bdata));
+    socket_send(s->fd, "\n", 1);
+    qtest_rsp(s, 0);
+    g_free(bdata);
+}
+
+void qtest_bufread(QTestState *s, uint64_t addr, void *data, size_t size)
+{
+    gchar **args;
+    size_t len;
+
+    qtest_sendf(s, "b64read 0x%" PRIx64 " 0x%zx\n", addr, size);
+    args = qtest_rsp(s, 2);
+
+    g_base64_decode_inplace(args[1], &len);
+    if (size != len) {
+        fprintf(stderr, "bufread: asked for %zu bytes but decoded %zu\n",
+                size, len);
+        len = MIN(len, size);
+    }
+
+    memcpy(data, args[1], len);
+    g_strfreev(args);
+}
+
 void qtest_memwrite(QTestState *s, uint64_t addr, const void *data, size_t size)
 {
     const uint8_t *ptr = data;
     size_t i;
+    char *enc = g_malloc(2 * size + 1);
 
-    qtest_sendf(s, "write 0x%" PRIx64 " 0x%zx 0x", addr, size);
     for (i = 0; i < size; i++) {
-        qtest_sendf(s, "%02x", ptr[i]);
+        sprintf(&enc[i * 2], "%02x", ptr[i]);
     }
-    qtest_sendf(s, "\n");
+
+    qtest_sendf(s, "write 0x%" PRIx64 " 0x%zx 0x%s\n", addr, size, enc);
     qtest_rsp(s, 0);
+    g_free(enc);
 }
 
 void qtest_memset(QTestState *s, uint64_t addr, uint8_t pattern, size_t size)
 {
-    size_t i;
-
-    qtest_sendf(s, "write 0x%" PRIx64 " 0x%zx 0x", addr, size);
-    for (i = 0; i < size; i++) {
-        qtest_sendf(s, "%02x", pattern);
-    }
-    qtest_sendf(s, "\n");
+    qtest_sendf(s, "memset 0x%" PRIx64 " 0x%zx 0x%02x\n", addr, size, pattern);
     qtest_rsp(s, 0);
 }
 
@@ -688,6 +829,15 @@
     return response;
 }
 
+void qmp_async(const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    qtest_async_qmpv(global_qtest, fmt, ap);
+    va_end(ap);
+}
+
 void qmp_discard_response(const char *fmt, ...)
 {
     va_list ap;
@@ -696,6 +846,16 @@
     qtest_qmpv_discard_response(global_qtest, fmt, ap);
     va_end(ap);
 }
+char *hmp(const char *fmt, ...)
+{
+    va_list ap;
+    char *ret;
+
+    va_start(ap, fmt);
+    ret = qtest_hmpv(global_qtest, fmt, ap);
+    va_end(ap);
+    return ret;
+}
 
 bool qtest_big_endian(void)
 {
diff --git a/tests/libqtest.h b/tests/libqtest.h
index e7413d5..37f37ad 100644
--- a/tests/libqtest.h
+++ b/tests/libqtest.h
@@ -17,13 +17,7 @@
 #ifndef LIBQTEST_H
 #define LIBQTEST_H
 
-#include <stddef.h>
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdarg.h>
-#include <sys/types.h>
 #include "qapi/qmp/qdict.h"
-#include "glib-compat.h"
 
 typedef struct QTestState QTestState;
 
@@ -64,6 +58,15 @@
 QDict *qtest_qmp(QTestState *s, const char *fmt, ...);
 
 /**
+ * qtest_async_qmp:
+ * @s: #QTestState instance to operate on.
+ * @fmt...: QMP message to send to qemu
+ *
+ * Sends a QMP message to QEMU and leaves the response in the stream.
+ */
+void qtest_async_qmp(QTestState *s, const char *fmt, ...);
+
+/**
  * qtest_qmpv_discard_response:
  * @s: #QTestState instance to operate on.
  * @fmt: QMP message to send to QEMU
@@ -84,6 +87,16 @@
 QDict *qtest_qmpv(QTestState *s, const char *fmt, va_list ap);
 
 /**
+ * qtest_async_qmpv:
+ * @s: #QTestState instance to operate on.
+ * @fmt: QMP message to send to QEMU
+ * @ap: QMP message arguments
+ *
+ * Sends a QMP message to QEMU and leaves the response in the stream.
+ */
+void qtest_async_qmpv(QTestState *s, const char *fmt, va_list ap);
+
+/**
  * qtest_receive:
  * @s: #QTestState instance to operate on.
  *
@@ -92,6 +105,38 @@
 QDict *qtest_qmp_receive(QTestState *s);
 
 /**
+ * qtest_qmp_eventwait:
+ * @s: #QTestState instance to operate on.
+ * @s: #event event to wait for.
+ *
+ * Continuosly polls for QMP responses until it receives the desired event.
+ */
+void qtest_qmp_eventwait(QTestState *s, const char *event);
+
+/**
+ * qtest_hmpv:
+ * @s: #QTestState instance to operate on.
+ * @fmt...: HMP command to send to QEMU
+ *
+ * Send HMP command to QEMU via QMP's human-monitor-command.
+ *
+ * Returns: the command's output.  The caller should g_free() it.
+ */
+char *qtest_hmp(QTestState *s, const char *fmt, ...);
+
+/**
+ * qtest_hmpv:
+ * @s: #QTestState instance to operate on.
+ * @fmt: HMP command to send to QEMU
+ * @ap: HMP command arguments
+ *
+ * Send HMP command to QEMU via QMP's human-monitor-command.
+ *
+ * Returns: the command's output.  The caller should g_free() it.
+ */
+char *qtest_hmpv(QTestState *s, const char *fmt, va_list ap);
+
+/**
  * qtest_get_irq:
  * @s: #QTestState instance to operate on.
  * @num: Interrupt to observe.
@@ -273,6 +318,17 @@
 void qtest_memread(QTestState *s, uint64_t addr, void *data, size_t size);
 
 /**
+ * qtest_bufread:
+ * @s: #QTestState instance to operate on.
+ * @addr: Guest address to read from.
+ * @data: Pointer to where memory contents will be stored.
+ * @size: Number of bytes to read.
+ *
+ * Read guest memory into a buffer and receive using a base64 encoding.
+ */
+void qtest_bufread(QTestState *s, uint64_t addr, void *data, size_t size);
+
+/**
  * qtest_memwrite:
  * @s: #QTestState instance to operate on.
  * @addr: Guest address to write to.
@@ -284,6 +340,18 @@
 void qtest_memwrite(QTestState *s, uint64_t addr, const void *data, size_t size);
 
 /**
+ * qtest_bufwrite:
+ * @s: #QTestState instance to operate on.
+ * @addr: Guest address to write to.
+ * @data: Pointer to the bytes that will be written to guest memory.
+ * @size: Number of bytes to write.
+ *
+ * Write a buffer to guest memory and transmit using a base64 encoding.
+ */
+void qtest_bufwrite(QTestState *s, uint64_t addr,
+                    const void *data, size_t size);
+
+/**
  * qtest_memset:
  * @s: #QTestState instance to operate on.
  * @addr: Guest address to write to.
@@ -342,7 +410,42 @@
  * The path is prefixed with the architecture under test, as
  * returned by qtest_get_arch().
  */
-void qtest_add_func(const char *str, void (*fn));
+void qtest_add_func(const char *str, void (*fn)(void));
+
+/**
+ * qtest_add_data_func:
+ * @str: Test case path.
+ * @data: Test case data
+ * @fn: Test case function
+ *
+ * Add a GTester testcase with the given name, data and function.
+ * The path is prefixed with the architecture under test, as
+ * returned by qtest_get_arch().
+ */
+void qtest_add_data_func(const char *str, const void *data,
+                         void (*fn)(const void *));
+
+/**
+ * qtest_add:
+ * @testpath: Test case path
+ * @Fixture: Fixture type
+ * @tdata: Test case data
+ * @fsetup: Test case setup function
+ * @ftest: Test case function
+ * @fteardown: Test case teardown function
+ *
+ * Add a GTester testcase with the given name, data and functions.
+ * The path is prefixed with the architecture under test, as
+ * returned by qtest_get_arch().
+ */
+#define qtest_add(testpath, Fixture, tdata, fsetup, ftest, fteardown) \
+    do { \
+        char *path = g_strdup_printf("/%s/%s", qtest_get_arch(), testpath); \
+        g_test_add(path, Fixture, tdata, fsetup, ftest, fteardown); \
+        g_free(path); \
+    } while (0)
+
+void qtest_add_abrt_handler(GHookFunc fn, const void *data);
 
 /**
  * qtest_start:
@@ -379,6 +482,14 @@
 QDict *qmp(const char *fmt, ...);
 
 /**
+ * qmp_async:
+ * @fmt...: QMP message to send to qemu
+ *
+ * Sends a QMP message to QEMU and leaves the response in the stream.
+ */
+void qmp_async(const char *fmt, ...);
+
+/**
  * qmp_discard_response:
  * @fmt...: QMP message to send to qemu
  *
@@ -397,6 +508,27 @@
 }
 
 /**
+ * qmp_eventwait:
+ * @s: #event event to wait for.
+ *
+ * Continuosly polls for QMP responses until it receives the desired event.
+ */
+static inline void qmp_eventwait(const char *event)
+{
+    return qtest_qmp_eventwait(global_qtest, event);
+}
+
+/**
+ * hmp:
+ * @fmt...: HMP command to send to QEMU
+ *
+ * Send HMP command to QEMU via QMP's human-monitor-command.
+ *
+ * Returns: the command's output.  The caller should g_free() it.
+ */
+char *hmp(const char *fmt, ...);
+
+/**
  * get_irq:
  * @num: Interrupt to observe.
  *
@@ -620,6 +752,19 @@
 }
 
 /**
+ * bufread:
+ * @addr: Guest address to read from.
+ * @data: Pointer to where memory contents will be stored.
+ * @size: Number of bytes to read.
+ *
+ * Read guest memory into a buffer, receive using a base64 encoding.
+ */
+static inline void bufread(uint64_t addr, void *data, size_t size)
+{
+    qtest_bufread(global_qtest, addr, data, size);
+}
+
+/**
  * memwrite:
  * @addr: Guest address to write to.
  * @data: Pointer to the bytes that will be written to guest memory.
@@ -633,6 +778,19 @@
 }
 
 /**
+ * bufwrite:
+ * @addr: Guest address to write to.
+ * @data: Pointer to the bytes that will be written to guest memory.
+ * @size: Number of bytes to write.
+ *
+ * Write a buffer to guest memory, transmit using a base64 encoding.
+ */
+static inline void bufwrite(uint64_t addr, const void *data, size_t size)
+{
+    qtest_bufwrite(global_qtest, addr, data, size);
+}
+
+/**
  * qmemset:
  * @addr: Guest address to write to.
  * @patt: Byte pattern to fill the guest memory region with.
@@ -690,4 +848,11 @@
  */
 bool qtest_big_endian(void);
 
+
+QDict *qmp_fd_receive(int fd);
+void qmp_fd_sendv(int fd, const char *fmt, va_list ap);
+void qmp_fd_send(int fd, const char *fmt, ...);
+QDict *qmp_fdv(int fd, const char *fmt, va_list ap);
+QDict *qmp_fd(int fd, const char *fmt, ...);
+
 #endif
diff --git a/tests/m48t59-test.c b/tests/m48t59-test.c
index 71b4f28..0f921ef 100644
--- a/tests/m48t59-test.c
+++ b/tests/m48t59-test.c
@@ -12,11 +12,7 @@
  *
  */
 
-#include <glib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 
diff --git a/tests/migration/.gitignore b/tests/migration/.gitignore
new file mode 100644
index 0000000..84f3755
--- /dev/null
+++ b/tests/migration/.gitignore
@@ -0,0 +1,2 @@
+initrd-stress.img
+stress
diff --git a/tests/migration/guestperf-batch.py b/tests/migration/guestperf-batch.py
new file mode 100755
index 0000000..cb150ce
--- /dev/null
+++ b/tests/migration/guestperf-batch.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+#
+# Migration test batch comparison invokation
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+
+from guestperf.shell import BatchShell
+
+shell = BatchShell()
+sys.exit(shell.run(sys.argv[1:]))
diff --git a/tests/migration/guestperf-plot.py b/tests/migration/guestperf-plot.py
new file mode 100755
index 0000000..d70bb7a
--- /dev/null
+++ b/tests/migration/guestperf-plot.py
@@ -0,0 +1,26 @@
+#!/usr/bin/python
+#
+# Migration test graph plotting command
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+
+from guestperf.shell import PlotShell
+
+shell = PlotShell()
+sys.exit(shell.run(sys.argv[1:]))
diff --git a/tests/migration/guestperf.py b/tests/migration/guestperf.py
new file mode 100755
index 0000000..99b027e
--- /dev/null
+++ b/tests/migration/guestperf.py
@@ -0,0 +1,27 @@
+#!/usr/bin/python
+#
+# Migration test direct invokation command
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import sys
+
+from guestperf.shell import Shell
+
+shell = Shell()
+sys.exit(shell.run(sys.argv[1:]))
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/migration/guestperf/__init__.py
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/migration/guestperf/__init__.py
diff --git a/tests/migration/guestperf/comparison.py b/tests/migration/guestperf/comparison.py
new file mode 100644
index 0000000..d0b7df9
--- /dev/null
+++ b/tests/migration/guestperf/comparison.py
@@ -0,0 +1,124 @@
+#
+# Migration test scenario comparison mapping
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+from guestperf.scenario import Scenario
+
+class Comparison(object):
+    def __init__(self, name, scenarios):
+        self._name = name
+        self._scenarios = scenarios
+
+COMPARISONS = [
+    # Looking at effect of pausing guest during migration
+    # at various stages of iteration over RAM
+    Comparison("pause-iters", scenarios = [
+        Scenario("pause-iters-0",
+                 pause=True, pause_iters=0),
+        Scenario("pause-iters-1",
+                 pause=True, pause_iters=1),
+        Scenario("pause-iters-5",
+                 pause=True, pause_iters=5),
+        Scenario("pause-iters-20",
+                 pause=True, pause_iters=20),
+    ]),
+
+
+    # Looking at use of post-copy in relation to bandwidth
+    # available for migration
+    Comparison("post-copy-bandwidth", scenarios = [
+        Scenario("post-copy-bw-100mbs",
+                 post_copy=True, bandwidth=12),
+        Scenario("post-copy-bw-300mbs",
+                 post_copy=True, bandwidth=37),
+        Scenario("post-copy-bw-1gbs",
+                 post_copy=True, bandwidth=125),
+        Scenario("post-copy-bw-10gbs",
+                 post_copy=True, bandwidth=1250),
+        Scenario("post-copy-bw-100gbs",
+                 post_copy=True, bandwidth=12500),
+    ]),
+
+
+    # Looking at effect of starting post-copy at different
+    # stages of the migration
+    Comparison("post-copy-iters", scenarios = [
+        Scenario("post-copy-iters-0",
+                 post_copy=True, post_copy_iters=0),
+        Scenario("post-copy-iters-1",
+                 post_copy=True, post_copy_iters=1),
+        Scenario("post-copy-iters-5",
+                 post_copy=True, post_copy_iters=5),
+        Scenario("post-copy-iters-20",
+                 post_copy=True, post_copy_iters=20),
+    ]),
+
+
+    # Looking at effect of auto-converge with different
+    # throttling percentage step rates
+    Comparison("auto-converge-iters", scenarios = [
+        Scenario("auto-converge-step-5",
+                 auto_converge=True, auto_converge_step=5),
+        Scenario("auto-converge-step-10",
+                 auto_converge=True, auto_converge_step=10),
+        Scenario("auto-converge-step-20",
+                 auto_converge=True, auto_converge_step=20),
+    ]),
+
+
+    # Looking at use of auto-converge in relation to bandwidth
+    # available for migration
+    Comparison("auto-converge-bandwidth", scenarios = [
+        Scenario("auto-converge-bw-100mbs",
+                 auto_converge=True, bandwidth=12),
+        Scenario("auto-converge-bw-300mbs",
+                 auto_converge=True, bandwidth=37),
+        Scenario("auto-converge-bw-1gbs",
+                 auto_converge=True, bandwidth=125),
+        Scenario("auto-converge-bw-10gbs",
+                 auto_converge=True, bandwidth=1250),
+        Scenario("auto-converge-bw-100gbs",
+                 auto_converge=True, bandwidth=12500),
+    ]),
+
+
+    # Looking at effect of multi-thread compression with
+    # varying numbers of threads
+    Comparison("compr-mt", scenarios = [
+        Scenario("compr-mt-threads-1",
+                 compression_mt=True, compression_mt_threads=1),
+        Scenario("compr-mt-threads-2",
+                 compression_mt=True, compression_mt_threads=2),
+        Scenario("compr-mt-threads-4",
+                 compression_mt=True, compression_mt_threads=4),
+    ]),
+
+
+    # Looking at effect of xbzrle compression with varying
+    # cache sizes
+    Comparison("compr-xbzrle", scenarios = [
+        Scenario("compr-xbzrle-cache-5",
+                 compression_xbzrle=True, compression_xbzrle_cache=5),
+        Scenario("compr-xbzrle-cache-10",
+                 compression_xbzrle=True, compression_xbzrle_cache=10),
+        Scenario("compr-xbzrle-cache-20",
+                 compression_xbzrle=True, compression_xbzrle_cache=10),
+        Scenario("compr-xbzrle-cache-50",
+                 compression_xbzrle=True, compression_xbzrle_cache=50),
+    ]),
+]
diff --git a/tests/migration/guestperf/engine.py b/tests/migration/guestperf/engine.py
new file mode 100644
index 0000000..0a13050
--- /dev/null
+++ b/tests/migration/guestperf/engine.py
@@ -0,0 +1,439 @@
+#
+# Migration test main engine
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import os
+import re
+import sys
+import time
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', '..', 'scripts'))
+import qemu
+import qmp.qmp
+from guestperf.progress import Progress, ProgressStats
+from guestperf.report import Report
+from guestperf.timings import TimingRecord, Timings
+
+
+class Engine(object):
+
+    def __init__(self, binary, dst_host, kernel, initrd, transport="tcp",
+                 sleep=15, verbose=False, debug=False):
+
+        self._binary = binary # Path to QEMU binary
+        self._dst_host = dst_host # Hostname of target host
+        self._kernel = kernel # Path to kernel image
+        self._initrd = initrd # Path to stress initrd
+        self._transport = transport # 'unix' or 'tcp' or 'rdma'
+        self._sleep = sleep
+        self._verbose = verbose
+        self._debug = debug
+
+        if debug:
+            self._verbose = debug
+
+    def _vcpu_timing(self, pid, tid_list):
+        records = []
+        now = time.time()
+
+        jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
+        for tid in tid_list:
+            statfile = "/proc/%d/task/%d/stat" % (pid, tid)
+            with open(statfile, "r") as fh:
+                stat = fh.readline()
+                fields = stat.split(" ")
+                stime = int(fields[13])
+                utime = int(fields[14])
+                records.append(TimingRecord(tid, now, 1000 * (stime + utime) / jiffies_per_sec))
+        return records
+
+    def _cpu_timing(self, pid):
+        records = []
+        now = time.time()
+
+        jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK'])
+        statfile = "/proc/%d/stat" % pid
+        with open(statfile, "r") as fh:
+            stat = fh.readline()
+            fields = stat.split(" ")
+            stime = int(fields[13])
+            utime = int(fields[14])
+            return TimingRecord(pid, now, 1000 * (stime + utime) / jiffies_per_sec)
+
+    def _migrate_progress(self, vm):
+        info = vm.command("query-migrate")
+
+        if "ram" not in info:
+            info["ram"] = {}
+
+        return Progress(
+            info.get("status", "active"),
+            ProgressStats(
+                info["ram"].get("transferred", 0),
+                info["ram"].get("remaining", 0),
+                info["ram"].get("total", 0),
+                info["ram"].get("duplicate", 0),
+                info["ram"].get("skipped", 0),
+                info["ram"].get("normal", 0),
+                info["ram"].get("normal-bytes", 0),
+                info["ram"].get("dirty-pages-rate", 0),
+                info["ram"].get("mbps", 0),
+                info["ram"].get("dirty-sync-count", 0)
+            ),
+            time.time(),
+            info.get("total-time", 0),
+            info.get("downtime", 0),
+            info.get("expected-downtime", 0),
+            info.get("setup-time", 0),
+            info.get("x-cpu-throttle-percentage", 0),
+        )
+
+    def _migrate(self, hardware, scenario, src, dst, connect_uri):
+        src_qemu_time = []
+        src_vcpu_time = []
+        src_pid = src.get_pid()
+
+        vcpus = src.command("query-cpus")
+        src_threads = []
+        for vcpu in vcpus:
+            src_threads.append(vcpu["thread_id"])
+
+        # XXX how to get dst timings on remote host ?
+
+        if self._verbose:
+            print "Sleeping %d seconds for initial guest workload run" % self._sleep
+        sleep_secs = self._sleep
+        while sleep_secs > 1:
+            src_qemu_time.append(self._cpu_timing(src_pid))
+            src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads))
+            time.sleep(1)
+            sleep_secs -= 1
+
+        if self._verbose:
+            print "Starting migration"
+        if scenario._auto_converge:
+            resp = src.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "auto-converge",
+                                     "state": True }
+                               ])
+            resp = src.command("migrate-set-parameters",
+                               x_cpu_throttle_increment=scenario._auto_converge_step)
+
+        if scenario._post_copy:
+            resp = src.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "postcopy-ram",
+                                     "state": True }
+                               ])
+            resp = dst.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "postcopy-ram",
+                                     "state": True }
+                               ])
+
+        resp = src.command("migrate_set_speed",
+                           value=scenario._bandwidth * 1024 * 1024)
+
+        resp = src.command("migrate_set_downtime",
+                           value=scenario._downtime / 1024.0)
+
+        if scenario._compression_mt:
+            resp = src.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "compress",
+                                     "state": True }
+                               ])
+            resp = src.command("migrate-set-parameters",
+                               compress_threads=scenario._compression_mt_threads)
+            resp = dst.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "compress",
+                                     "state": True }
+                               ])
+            resp = dst.command("migrate-set-parameters",
+                               decompress_threads=scenario._compression_mt_threads)
+
+        if scenario._compression_xbzrle:
+            resp = src.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "xbzrle",
+                                     "state": True }
+                               ])
+            resp = dst.command("migrate-set-capabilities",
+                               capabilities = [
+                                   { "capability": "xbzrle",
+                                     "state": True }
+                               ])
+            resp = src.command("migrate-set-cache-size",
+                               value=(hardware._mem * 1024 * 1024 * 1024 / 100 *
+                                      scenario._compression_xbzrle_cache))
+
+        resp = src.command("migrate", uri=connect_uri)
+
+        post_copy = False
+        paused = False
+
+        progress_history = []
+
+        start = time.time()
+        loop = 0
+        while True:
+            loop = loop + 1
+            time.sleep(0.05)
+
+            progress = self._migrate_progress(src)
+            if (loop % 20) == 0:
+                src_qemu_time.append(self._cpu_timing(src_pid))
+                src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads))
+
+            if (len(progress_history) == 0 or
+                (progress_history[-1]._ram._iterations <
+                 progress._ram._iterations)):
+                progress_history.append(progress)
+
+            if progress._status in ("completed", "failed", "cancelled"):
+                if progress._status == "completed" and paused:
+                    dst.command("cont")
+                if progress_history[-1] != progress:
+                    progress_history.append(progress)
+
+                if progress._status == "completed":
+                    if self._verbose:
+                        print "Sleeping %d seconds for final guest workload run" % self._sleep
+                    sleep_secs = self._sleep
+                    while sleep_secs > 1:
+                        time.sleep(1)
+                        src_qemu_time.append(self._cpu_timing(src_pid))
+                        src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads))
+                        sleep_secs -= 1
+
+                return [progress_history, src_qemu_time, src_vcpu_time]
+
+            if self._verbose and (loop % 20) == 0:
+                print "Iter %d: remain %5dMB of %5dMB (total %5dMB @ %5dMb/sec)" % (
+                    progress._ram._iterations,
+                    progress._ram._remaining_bytes / (1024 * 1024),
+                    progress._ram._total_bytes / (1024 * 1024),
+                    progress._ram._transferred_bytes / (1024 * 1024),
+                    progress._ram._transfer_rate_mbs,
+                )
+
+            if progress._ram._iterations > scenario._max_iters:
+                if self._verbose:
+                    print "No completion after %d iterations over RAM" % scenario._max_iters
+                src.command("migrate_cancel")
+                continue
+
+            if time.time() > (start + scenario._max_time):
+                if self._verbose:
+                    print "No completion after %d seconds" % scenario._max_time
+                src.command("migrate_cancel")
+                continue
+
+            if (scenario._post_copy and
+                progress._ram._iterations >= scenario._post_copy_iters and
+                not post_copy):
+                if self._verbose:
+                    print "Switching to post-copy after %d iterations" % scenario._post_copy_iters
+                resp = src.command("migrate-start-postcopy")
+                post_copy = True
+
+            if (scenario._pause and
+                progress._ram._iterations >= scenario._pause_iters and
+                not paused):
+                if self._verbose:
+                    print "Pausing VM after %d iterations" % scenario._pause_iters
+                resp = src.command("stop")
+                paused = True
+
+    def _get_common_args(self, hardware, tunnelled=False):
+        args = [
+            "noapic",
+            "edd=off",
+            "printk.time=1",
+            "noreplace-smp",
+            "cgroup_disable=memory",
+            "pci=noearly",
+            "console=ttyS0",
+        ]
+        if self._debug:
+            args.append("debug")
+        else:
+            args.append("quiet")
+
+        args.append("ramsize=%s" % hardware._mem)
+
+        cmdline = " ".join(args)
+        if tunnelled:
+            cmdline = "'" + cmdline + "'"
+
+        argv = [
+            "-machine", "accel=kvm",
+            "-cpu", "host",
+            "-kernel", self._kernel,
+            "-initrd", self._initrd,
+            "-append", cmdline,
+            "-chardev", "stdio,id=cdev0",
+            "-device", "isa-serial,chardev=cdev0",
+            "-m", str((hardware._mem * 1024) + 512),
+            "-smp", str(hardware._cpus),
+        ]
+
+        if self._debug:
+            argv.extend(["-device", "sga"])
+
+        if hardware._prealloc_pages:
+            argv_source += ["-mem-path", "/dev/shm",
+                            "-mem-prealloc"]
+        if hardware._locked_pages:
+            argv_source += ["-realtime", "mlock=on"]
+        if hardware._huge_pages:
+            pass
+
+        return argv
+
+    def _get_src_args(self, hardware):
+        return self._get_common_args(hardware)
+
+    def _get_dst_args(self, hardware, uri):
+        tunnelled = False
+        if self._dst_host != "localhost":
+            tunnelled = True
+        argv = self._get_common_args(hardware, tunnelled)
+        return argv + ["-incoming", uri]
+
+    @staticmethod
+    def _get_common_wrapper(cpu_bind, mem_bind):
+        wrapper = []
+        if len(cpu_bind) > 0 or len(mem_bind) > 0:
+            wrapper.append("numactl")
+            if cpu_bind:
+                wrapper.append("--physcpubind=%s" % ",".join(cpu_bind))
+            if mem_bind:
+                wrapper.append("--membind=%s" % ",".join(mem_bind))
+
+        return wrapper
+
+    def _get_src_wrapper(self, hardware):
+        return self._get_common_wrapper(hardware._src_cpu_bind, hardware._src_mem_bind)
+
+    def _get_dst_wrapper(self, hardware):
+        wrapper = self._get_common_wrapper(hardware._dst_cpu_bind, hardware._dst_mem_bind)
+        if self._dst_host != "localhost":
+            return ["ssh",
+                    "-R", "9001:localhost:9001",
+                    self._dst_host] + wrapper
+        else:
+            return wrapper
+
+    def _get_timings(self, vm):
+        log = vm.get_log()
+        if not log:
+            return []
+        if self._debug:
+            print log
+
+        regex = r"[^\s]+\s\((\d+)\):\sINFO:\s(\d+)ms\scopied\s\d+\sGB\sin\s(\d+)ms"
+        matcher = re.compile(regex)
+        records = []
+        for line in log.split("\n"):
+            match = matcher.match(line)
+            if match:
+                records.append(TimingRecord(int(match.group(1)),
+                                            int(match.group(2)) / 1000.0,
+                                            int(match.group(3))))
+        return records
+
+    def run(self, hardware, scenario, result_dir=os.getcwd()):
+        abs_result_dir = os.path.join(result_dir, scenario._name)
+
+        if self._transport == "tcp":
+            uri = "tcp:%s:9000" % self._dst_host
+        elif self._transport == "rdma":
+            uri = "rdma:%s:9000" % self._dst_host
+        elif self._transport == "unix":
+            if self._dst_host != "localhost":
+                raise Exception("Running use unix migration transport for non-local host")
+            uri = "unix:/var/tmp/qemu-migrate-%d.migrate" % os.getpid()
+            try:
+                os.remove(uri[5:])
+                os.remove(monaddr)
+            except:
+                pass
+
+        if self._dst_host != "localhost":
+            dstmonaddr = ("localhost", 9001)
+        else:
+            dstmonaddr = "/var/tmp/qemu-dst-%d-monitor.sock" % os.getpid()
+        srcmonaddr = "/var/tmp/qemu-src-%d-monitor.sock" % os.getpid()
+
+        src = qemu.QEMUMachine(self._binary,
+                               args=self._get_src_args(hardware),
+                               wrapper=self._get_src_wrapper(hardware),
+                               name="qemu-src-%d" % os.getpid(),
+                               monitor_address=srcmonaddr,
+                               debug=self._debug)
+
+        dst = qemu.QEMUMachine(self._binary,
+                               args=self._get_dst_args(hardware, uri),
+                               wrapper=self._get_dst_wrapper(hardware),
+                               name="qemu-dst-%d" % os.getpid(),
+                               monitor_address=dstmonaddr,
+                               debug=self._debug)
+
+        try:
+            src.launch()
+            dst.launch()
+
+            ret = self._migrate(hardware, scenario, src, dst, uri)
+            progress_history = ret[0]
+            qemu_timings = ret[1]
+            vcpu_timings = ret[2]
+            if uri[0:5] == "unix:":
+                os.remove(uri[5:])
+            if self._verbose:
+                print "Finished migration"
+
+            src.shutdown()
+            dst.shutdown()
+
+            return Report(hardware, scenario, progress_history,
+                          Timings(self._get_timings(src) + self._get_timings(dst)),
+                          Timings(qemu_timings),
+                          Timings(vcpu_timings),
+                          self._binary, self._dst_host, self._kernel,
+                          self._initrd, self._transport, self._sleep)
+        except Exception as e:
+            if self._debug:
+                print "Failed: %s" % str(e)
+            try:
+                src.shutdown()
+            except:
+                pass
+            try:
+                dst.shutdown()
+            except:
+                pass
+
+            if self._debug:
+                print src.get_log()
+                print dst.get_log()
+            raise
+
diff --git a/tests/migration/guestperf/hardware.py b/tests/migration/guestperf/hardware.py
new file mode 100644
index 0000000..a66c9dd
--- /dev/null
+++ b/tests/migration/guestperf/hardware.py
@@ -0,0 +1,62 @@
+#
+# Migration test hardware configuration description
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class Hardware(object):
+    def __init__(self, cpus=1, mem=1,
+                 src_cpu_bind=None, src_mem_bind=None,
+                 dst_cpu_bind=None, dst_mem_bind=None,
+                 prealloc_pages = False,
+                 huge_pages=False, locked_pages=False):
+        self._cpus = cpus
+        self._mem = mem # GiB
+        self._src_mem_bind = src_mem_bind # List of NUMA nodes
+        self._src_cpu_bind = src_cpu_bind # List of pCPUs
+        self._dst_mem_bind = dst_mem_bind # List of NUMA nodes
+        self._dst_cpu_bind = dst_cpu_bind # List of pCPUs
+        self._prealloc_pages = prealloc_pages
+        self._huge_pages = huge_pages
+        self._locked_pages = locked_pages
+
+
+    def serialize(self):
+        return {
+            "cpus": self._cpus,
+            "mem": self._mem,
+            "src_mem_bind": self._src_mem_bind,
+            "dst_mem_bind": self._dst_mem_bind,
+            "src_cpu_bind": self._src_cpu_bind,
+            "dst_cpu_bind": self._dst_cpu_bind,
+            "prealloc_pages": self._prealloc_pages,
+            "huge_pages": self._huge_pages,
+            "locked_pages": self._locked_pages,
+        }
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(
+            data["cpus"],
+            data["mem"],
+            data["src_cpu_bind"],
+            data["src_mem_bind"],
+            data["dst_cpu_bind"],
+            data["dst_mem_bind"],
+            data["prealloc_pages"],
+            data["huge_pages"],
+            data["locked_pages"])
diff --git a/tests/migration/guestperf/plot.py b/tests/migration/guestperf/plot.py
new file mode 100644
index 0000000..bc42249
--- /dev/null
+++ b/tests/migration/guestperf/plot.py
@@ -0,0 +1,623 @@
+#
+# Migration test graph plotting
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+import sys
+
+
+class Plot(object):
+
+    # Generated using
+    # http://tools.medialab.sciences-po.fr/iwanthue/
+    COLORS = ["#CD54D0",
+              "#79D94C",
+              "#7470CD",
+              "#D2D251",
+              "#863D79",
+              "#76DDA6",
+              "#D4467B",
+              "#61923D",
+              "#CB9CCA",
+              "#D98F36",
+              "#8CC8DA",
+              "#CE4831",
+              "#5E7693",
+              "#9B803F",
+              "#412F4C",
+              "#CECBA6",
+              "#6D3229",
+              "#598B73",
+              "#C8827C",
+              "#394427"]
+
+    def __init__(self,
+                 reports,
+                 migration_iters,
+                 total_guest_cpu,
+                 split_guest_cpu,
+                 qemu_cpu,
+                 vcpu_cpu):
+
+        self._reports = reports
+        self._migration_iters = migration_iters
+        self._total_guest_cpu = total_guest_cpu
+        self._split_guest_cpu = split_guest_cpu
+        self._qemu_cpu = qemu_cpu
+        self._vcpu_cpu = vcpu_cpu
+        self._color_idx = 0
+
+    def _next_color(self):
+        color = self.COLORS[self._color_idx]
+        self._color_idx += 1
+        if self._color_idx >= len(self.COLORS):
+            self._color_idx = 0
+        return color
+
+    def _get_progress_label(self, progress):
+        if progress:
+            return "\n\n" + "\n".join(
+                ["Status: %s" % progress._status,
+                 "Iteration: %d" % progress._ram._iterations,
+                 "Throttle: %02d%%" % progress._throttle_pcent,
+                 "Dirty rate: %dMB/s" % (progress._ram._dirty_rate_pps * 4 / 1024.0)])
+        else:
+            return "\n\n" + "\n".join(
+                ["Status: %s" % "none",
+                 "Iteration: %d" % 0])
+
+    def _find_start_time(self, report):
+        startqemu = report._qemu_timings._records[0]._timestamp
+        startguest = report._guest_timings._records[0]._timestamp
+        if startqemu < startguest:
+            return startqemu
+        else:
+            return stasrtguest
+
+    def _get_guest_max_value(self, report):
+        maxvalue = 0
+        for record in report._guest_timings._records:
+            if record._value > maxvalue:
+                maxvalue = record._value
+        return maxvalue
+
+    def _get_qemu_max_value(self, report):
+        maxvalue = 0
+        oldvalue = None
+        oldtime = None
+        for record in report._qemu_timings._records:
+            if oldvalue is not None:
+                cpudelta = (record._value - oldvalue) / 1000.0
+                timedelta = record._timestamp - oldtime
+                if timedelta == 0:
+                    continue
+                util = cpudelta / timedelta * 100.0
+            else:
+                util = 0
+            oldvalue = record._value
+            oldtime = record._timestamp
+
+            if util > maxvalue:
+                maxvalue = util
+        return maxvalue
+
+    def _get_total_guest_cpu_graph(self, report, starttime):
+        xaxis = []
+        yaxis = []
+        labels = []
+        progress_idx = -1
+        for record in report._guest_timings._records:
+            while ((progress_idx + 1) < len(report._progress_history) and
+                   report._progress_history[progress_idx + 1]._now < record._timestamp):
+                progress_idx = progress_idx + 1
+
+            if progress_idx >= 0:
+                progress = report._progress_history[progress_idx]
+            else:
+                progress = None
+
+            xaxis.append(record._timestamp - starttime)
+            yaxis.append(record._value)
+            labels.append(self._get_progress_label(progress))
+
+        from plotly import graph_objs as go
+        return go.Scatter(x=xaxis,
+                          y=yaxis,
+                          name="Guest PIDs: %s" % report._scenario._name,
+                          mode='lines',
+                          line={
+                              "dash": "solid",
+                              "color": self._next_color(),
+                              "shape": "linear",
+                              "width": 1
+                          },
+                          text=labels)
+
+    def _get_split_guest_cpu_graphs(self, report, starttime):
+        threads = {}
+        for record in report._guest_timings._records:
+            if record._tid in threads:
+                continue
+            threads[record._tid] = {
+                "xaxis": [],
+                "yaxis": [],
+                "labels": [],
+            }
+
+        progress_idx = -1
+        for record in report._guest_timings._records:
+            while ((progress_idx + 1) < len(report._progress_history) and
+                   report._progress_history[progress_idx + 1]._now < record._timestamp):
+                progress_idx = progress_idx + 1
+
+            if progress_idx >= 0:
+                progress = report._progress_history[progress_idx]
+            else:
+                progress = None
+
+            threads[record._tid]["xaxis"].append(record._timestamp - starttime)
+            threads[record._tid]["yaxis"].append(record._value)
+            threads[record._tid]["labels"].append(self._get_progress_label(progress))
+
+
+        graphs = []
+        from plotly import graph_objs as go
+        for tid in threads.keys():
+            graphs.append(
+                go.Scatter(x=threads[tid]["xaxis"],
+                           y=threads[tid]["yaxis"],
+                           name="PID %s: %s" % (tid, report._scenario._name),
+                           mode="lines",
+                           line={
+                               "dash": "solid",
+                               "color": self._next_color(),
+                               "shape": "linear",
+                               "width": 1
+                           },
+                           text=threads[tid]["labels"]))
+        return graphs
+
+    def _get_migration_iters_graph(self, report, starttime):
+        xaxis = []
+        yaxis = []
+        labels = []
+        for progress in report._progress_history:
+            xaxis.append(progress._now - starttime)
+            yaxis.append(0)
+            labels.append(self._get_progress_label(progress))
+
+        from plotly import graph_objs as go
+        return go.Scatter(x=xaxis,
+                          y=yaxis,
+                          text=labels,
+                          name="Migration iterations",
+                          mode="markers",
+                          marker={
+                              "color": self._next_color(),
+                              "symbol": "star",
+                              "size": 5
+                          })
+
+    def _get_qemu_cpu_graph(self, report, starttime):
+        xaxis = []
+        yaxis = []
+        labels = []
+        progress_idx = -1
+
+        first = report._qemu_timings._records[0]
+        abstimestamps = [first._timestamp]
+        absvalues = [first._value]
+
+        for record in report._qemu_timings._records[1:]:
+            while ((progress_idx + 1) < len(report._progress_history) and
+                   report._progress_history[progress_idx + 1]._now < record._timestamp):
+                progress_idx = progress_idx + 1
+
+            if progress_idx >= 0:
+                progress = report._progress_history[progress_idx]
+            else:
+                progress = None
+
+            oldvalue = absvalues[-1]
+            oldtime = abstimestamps[-1]
+
+            cpudelta = (record._value - oldvalue) / 1000.0
+            timedelta = record._timestamp - oldtime
+            if timedelta == 0:
+                continue
+            util = cpudelta / timedelta * 100.0
+
+            abstimestamps.append(record._timestamp)
+            absvalues.append(record._value)
+
+            xaxis.append(record._timestamp - starttime)
+            yaxis.append(util)
+            labels.append(self._get_progress_label(progress))
+
+        from plotly import graph_objs as go
+        return go.Scatter(x=xaxis,
+                          y=yaxis,
+                          yaxis="y2",
+                          name="QEMU: %s" % report._scenario._name,
+                          mode='lines',
+                          line={
+                              "dash": "solid",
+                              "color": self._next_color(),
+                              "shape": "linear",
+                              "width": 1
+                          },
+                          text=labels)
+
+    def _get_vcpu_cpu_graphs(self, report, starttime):
+        threads = {}
+        for record in report._vcpu_timings._records:
+            if record._tid in threads:
+                continue
+            threads[record._tid] = {
+                "xaxis": [],
+                "yaxis": [],
+                "labels": [],
+                "absvalue": [record._value],
+                "abstime": [record._timestamp],
+            }
+
+        progress_idx = -1
+        for record in report._vcpu_timings._records:
+            while ((progress_idx + 1) < len(report._progress_history) and
+                   report._progress_history[progress_idx + 1]._now < record._timestamp):
+                progress_idx = progress_idx + 1
+
+            if progress_idx >= 0:
+                progress = report._progress_history[progress_idx]
+            else:
+                progress = None
+
+            oldvalue = threads[record._tid]["absvalue"][-1]
+            oldtime = threads[record._tid]["abstime"][-1]
+
+            cpudelta = (record._value - oldvalue) / 1000.0
+            timedelta = record._timestamp - oldtime
+            if timedelta == 0:
+                continue
+            util = cpudelta / timedelta * 100.0
+            if util > 100:
+                util = 100
+
+            threads[record._tid]["absvalue"].append(record._value)
+            threads[record._tid]["abstime"].append(record._timestamp)
+
+            threads[record._tid]["xaxis"].append(record._timestamp - starttime)
+            threads[record._tid]["yaxis"].append(util)
+            threads[record._tid]["labels"].append(self._get_progress_label(progress))
+
+
+        graphs = []
+        from plotly import graph_objs as go
+        for tid in threads.keys():
+            graphs.append(
+                go.Scatter(x=threads[tid]["xaxis"],
+                           y=threads[tid]["yaxis"],
+                           yaxis="y2",
+                           name="VCPU %s: %s" % (tid, report._scenario._name),
+                           mode="lines",
+                           line={
+                               "dash": "solid",
+                               "color": self._next_color(),
+                               "shape": "linear",
+                               "width": 1
+                           },
+                           text=threads[tid]["labels"]))
+        return graphs
+
+    def _generate_chart_report(self, report):
+        graphs = []
+        starttime = self._find_start_time(report)
+        if self._total_guest_cpu:
+            graphs.append(self._get_total_guest_cpu_graph(report, starttime))
+        if self._split_guest_cpu:
+            graphs.extend(self._get_split_guest_cpu_graphs(report, starttime))
+        if self._qemu_cpu:
+            graphs.append(self._get_qemu_cpu_graph(report, starttime))
+        if self._vcpu_cpu:
+            graphs.extend(self._get_vcpu_cpu_graphs(report, starttime))
+        if self._migration_iters:
+            graphs.append(self._get_migration_iters_graph(report, starttime))
+        return graphs
+
+    def _generate_annotation(self, starttime, progress):
+        return {
+            "text": progress._status,
+            "x": progress._now - starttime,
+            "y": 10,
+        }
+
+    def _generate_annotations(self, report):
+        starttime = self._find_start_time(report)
+        annotations = {}
+        started = False
+        for progress in report._progress_history:
+            if progress._status == "setup":
+                continue
+            if progress._status not in annotations:
+                annotations[progress._status] = self._generate_annotation(starttime, progress)
+
+        return annotations.values()
+
+    def _generate_chart(self):
+        from plotly.offline import plot
+        from plotly import graph_objs as go
+
+        graphs = []
+        yaxismax = 0
+        yaxismax2 = 0
+        for report in self._reports:
+            graphs.extend(self._generate_chart_report(report))
+
+            maxvalue = self._get_guest_max_value(report)
+            if maxvalue > yaxismax:
+                yaxismax = maxvalue
+
+            maxvalue = self._get_qemu_max_value(report)
+            if maxvalue > yaxismax2:
+                yaxismax2 = maxvalue
+
+        yaxismax += 100
+        if not self._qemu_cpu:
+            yaxismax2 = 110
+        yaxismax2 += 10
+
+        annotations = []
+        if self._migration_iters:
+            for report in self._reports:
+                annotations.extend(self._generate_annotations(report))
+
+        layout = go.Layout(title="Migration comparison",
+                           xaxis={
+                               "title": "Wallclock time (secs)",
+                               "showgrid": False,
+                           },
+                           yaxis={
+                               "title": "Memory update speed (ms/GB)",
+                               "showgrid": False,
+                               "range": [0, yaxismax],
+                           },
+                           yaxis2={
+                               "title": "Hostutilization (%)",
+                               "overlaying": "y",
+                               "side": "right",
+                               "range": [0, yaxismax2],
+                               "showgrid": False,
+                           },
+                           annotations=annotations)
+
+        figure = go.Figure(data=graphs, layout=layout)
+
+        return plot(figure,
+                    show_link=False,
+                    include_plotlyjs=False,
+                    output_type="div")
+
+
+    def _generate_report(self):
+        pieces = []
+        for report in self._reports:
+            pieces.append("""
+<h3>Report %s</h3>
+<table>
+""" % report._scenario._name)
+
+            pieces.append("""
+  <tr class="subhead">
+    <th colspan="2">Test config</th>
+  </tr>
+  <tr>
+    <th>Emulator:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Kernel:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Ramdisk:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Transport:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Host:</th>
+    <td>%s</td>
+  </tr>
+""" % (report._binary, report._kernel,
+       report._initrd, report._transport, report._dst_host))
+
+            hardware = report._hardware
+            pieces.append("""
+  <tr class="subhead">
+    <th colspan="2">Hardware config</th>
+  </tr>
+  <tr>
+    <th>CPUs:</th>
+    <td>%d</td>
+  </tr>
+  <tr>
+    <th>RAM:</th>
+    <td>%d GB</td>
+  </tr>
+  <tr>
+    <th>Source CPU bind:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Source RAM bind:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Dest CPU bind:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Dest RAM bind:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Preallocate RAM:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Locked RAM:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Huge pages:</th>
+    <td>%s</td>
+  </tr>
+""" % (hardware._cpus, hardware._mem,
+       ",".join(hardware._src_cpu_bind),
+       ",".join(hardware._src_mem_bind),
+       ",".join(hardware._dst_cpu_bind),
+       ",".join(hardware._dst_mem_bind),
+       "yes" if hardware._prealloc_pages else "no",
+       "yes" if hardware._locked_pages else "no",
+       "yes" if hardware._huge_pages else "no"))
+
+            scenario = report._scenario
+            pieces.append("""
+  <tr class="subhead">
+    <th colspan="2">Scenario config</th>
+  </tr>
+  <tr>
+    <th>Max downtime:</th>
+    <td>%d milli-sec</td>
+  </tr>
+  <tr>
+    <th>Max bandwidth:</th>
+    <td>%d MB/sec</td>
+  </tr>
+  <tr>
+    <th>Max iters:</th>
+    <td>%d</td>
+  </tr>
+  <tr>
+    <th>Max time:</th>
+    <td>%d secs</td>
+  </tr>
+  <tr>
+    <th>Pause:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Pause iters:</th>
+    <td>%d</td>
+  </tr>
+  <tr>
+    <th>Post-copy:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Post-copy iters:</th>
+    <td>%d</td>
+  </tr>
+  <tr>
+    <th>Auto-converge:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>Auto-converge iters:</th>
+    <td>%d</td>
+  </tr>
+  <tr>
+    <th>MT compression:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>MT compression threads:</th>
+    <td>%d</td>
+  </tr>
+  <tr>
+    <th>XBZRLE compression:</th>
+    <td>%s</td>
+  </tr>
+  <tr>
+    <th>XBZRLE compression cache:</th>
+    <td>%d%% of RAM</td>
+  </tr>
+""" % (scenario._downtime, scenario._bandwidth,
+       scenario._max_iters, scenario._max_time,
+       "yes" if scenario._pause else "no", scenario._pause_iters,
+       "yes" if scenario._post_copy else "no", scenario._post_copy_iters,
+       "yes" if scenario._auto_converge else "no", scenario._auto_converge_step,
+       "yes" if scenario._compression_mt else "no", scenario._compression_mt_threads,
+       "yes" if scenario._compression_xbzrle else "no", scenario._compression_xbzrle_cache))
+
+            pieces.append("""
+</table>
+""")
+
+        return "\n".join(pieces)
+
+    def _generate_style(self):
+        return """
+#report table tr th {
+    text-align: right;
+}
+#report table tr td {
+    text-align: left;
+}
+#report table tr.subhead th {
+    background: rgb(192, 192, 192);
+    text-align: center;
+}
+
+"""
+
+    def generate_html(self, fh):
+        print >>fh, """<html>
+  <head>
+    <script type="text/javascript" src="plotly.min.js">
+    </script>
+    <style type="text/css">
+%s
+    </style>
+    <title>Migration report</title>
+  </head>
+  <body>
+    <h1>Migration report</h1>
+    <h2>Chart summary</h2>
+    <div id="chart">
+""" % self._generate_style()
+        print >>fh, self._generate_chart()
+        print >>fh, """
+    </div>
+    <h2>Report details</h2>
+    <div id="report">
+"""
+        print >>fh, self._generate_report()
+        print >>fh, """
+    </div>
+  </body>
+</html>
+"""
+
+    def generate(self, filename):
+        if filename is None:
+            self.generate_html(sys.stdout)
+        else:
+            with open(filename, "w") as fh:
+                self.generate_html(fh)
diff --git a/tests/migration/guestperf/progress.py b/tests/migration/guestperf/progress.py
new file mode 100644
index 0000000..46d2157
--- /dev/null
+++ b/tests/migration/guestperf/progress.py
@@ -0,0 +1,117 @@
+#
+# Migration test migration operation progress
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class ProgressStats(object):
+
+    def __init__(self,
+                 transferred_bytes,
+                 remaining_bytes,
+                 total_bytes,
+                 duplicate_pages,
+                 skipped_pages,
+                 normal_pages,
+                 normal_bytes,
+                 dirty_rate_pps,
+                 transfer_rate_mbs,
+                 iterations):
+        self._transferred_bytes = transferred_bytes
+        self._remaining_bytes = remaining_bytes
+        self._total_bytes = total_bytes
+        self._duplicate_pages = duplicate_pages
+        self._skipped_pages = skipped_pages
+        self._normal_pages = normal_pages
+        self._normal_bytes = normal_bytes
+        self._dirty_rate_pps = dirty_rate_pps
+        self._transfer_rate_mbs = transfer_rate_mbs
+        self._iterations = iterations
+
+    def serialize(self):
+        return {
+            "transferred_bytes": self._transferred_bytes,
+            "remaining_bytes": self._remaining_bytes,
+            "total_bytes": self._total_bytes,
+            "duplicate_pages": self._duplicate_pages,
+            "skipped_pages": self._skipped_pages,
+            "normal_pages": self._normal_pages,
+            "normal_bytes": self._normal_bytes,
+            "dirty_rate_pps": self._dirty_rate_pps,
+            "transfer_rate_mbs": self._transfer_rate_mbs,
+            "iterations": self._iterations,
+        }
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(
+            data["transferred_bytes"],
+            data["remaining_bytes"],
+            data["total_bytes"],
+            data["duplicate_pages"],
+            data["skipped_pages"],
+            data["normal_pages"],
+            data["normal_bytes"],
+            data["dirty_rate_pps"],
+            data["transfer_rate_mbs"],
+            data["iterations"])
+
+
+class Progress(object):
+
+    def __init__(self,
+                 status,
+                 ram,
+                 now,
+                 duration,
+                 downtime,
+                 downtime_expected,
+                 setup_time,
+                 throttle_pcent):
+
+        self._status = status
+        self._ram = ram
+        self._now = now
+        self._duration = duration
+        self._downtime = downtime
+        self._downtime_expected = downtime_expected
+        self._setup_time = setup_time
+        self._throttle_pcent = throttle_pcent
+
+    def serialize(self):
+        return {
+            "status": self._status,
+            "ram": self._ram.serialize(),
+            "now": self._now,
+            "duration": self._duration,
+            "downtime": self._downtime,
+            "downtime_expected": self._downtime_expected,
+            "setup_time": self._setup_time,
+            "throttle_pcent": self._throttle_pcent,
+        }
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(
+            data["status"],
+            ProgressStats.deserialize(data["ram"]),
+            data["now"],
+            data["duration"],
+            data["downtime"],
+            data["downtime_expected"],
+            data["setup_time"],
+            data["throttle_pcent"])
diff --git a/tests/migration/guestperf/report.py b/tests/migration/guestperf/report.py
new file mode 100644
index 0000000..6a1f971
--- /dev/null
+++ b/tests/migration/guestperf/report.py
@@ -0,0 +1,98 @@
+#
+# Migration test output result reporting
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+import json
+
+from guestperf.hardware import Hardware
+from guestperf.scenario import Scenario
+from guestperf.progress import Progress
+from guestperf.timings import Timings
+
+class Report(object):
+
+    def __init__(self,
+                 hardware,
+                 scenario,
+                 progress_history,
+                 guest_timings,
+                 qemu_timings,
+                 vcpu_timings,
+                 binary,
+                 dst_host,
+                 kernel,
+                 initrd,
+                 transport,
+                 sleep):
+
+        self._hardware = hardware
+        self._scenario = scenario
+        self._progress_history = progress_history
+        self._guest_timings = guest_timings
+        self._qemu_timings = qemu_timings
+        self._vcpu_timings = vcpu_timings
+        self._binary = binary
+        self._dst_host = dst_host
+        self._kernel = kernel
+        self._initrd = initrd
+        self._transport = transport
+        self._sleep = sleep
+
+    def serialize(self):
+        return {
+            "hardware": self._hardware.serialize(),
+            "scenario": self._scenario.serialize(),
+            "progress_history": [progress.serialize() for progress in self._progress_history],
+            "guest_timings": self._guest_timings.serialize(),
+            "qemu_timings": self._qemu_timings.serialize(),
+            "vcpu_timings": self._vcpu_timings.serialize(),
+            "binary": self._binary,
+            "dst_host": self._dst_host,
+            "kernel": self._kernel,
+            "initrd": self._initrd,
+            "transport": self._transport,
+            "sleep": self._sleep,
+        }
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(
+            Hardware.deserialize(data["hardware"]),
+            Scenario.deserialize(data["scenario"]),
+            [Progress.deserialize(record) for record in data["progress_history"]],
+            Timings.deserialize(data["guest_timings"]),
+            Timings.deserialize(data["qemu_timings"]),
+            Timings.deserialize(data["vcpu_timings"]),
+            data["binary"],
+            data["dst_host"],
+            data["kernel"],
+            data["initrd"],
+            data["transport"],
+            data["sleep"])
+
+    def to_json(self):
+        return json.dumps(self.serialize(), indent=4)
+
+    @classmethod
+    def from_json(cls, data):
+        return cls.deserialize(json.loads(data))
+
+    @classmethod
+    def from_json_file(cls, filename):
+        with open(filename, "r") as fh:
+            return cls.deserialize(json.load(fh))
diff --git a/tests/migration/guestperf/scenario.py b/tests/migration/guestperf/scenario.py
new file mode 100644
index 0000000..705c2e8
--- /dev/null
+++ b/tests/migration/guestperf/scenario.py
@@ -0,0 +1,95 @@
+#
+# Migration test scenario parameter description
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class Scenario(object):
+
+    def __init__(self, name,
+                 downtime=500,
+                 bandwidth=125000, # 1000 gig-e, effectively unlimited
+                 max_iters=30,
+                 max_time=300,
+                 pause=False, pause_iters=5,
+                 post_copy=False, post_copy_iters=5,
+                 auto_converge=False, auto_converge_step=10,
+                 compression_mt=False, compression_mt_threads=1,
+                 compression_xbzrle=False, compression_xbzrle_cache=10):
+
+        self._name = name
+
+        # General migration tunables
+        self._downtime = downtime  # milliseconds
+        self._bandwidth = bandwidth # MiB per second
+        self._max_iters = max_iters
+        self._max_time = max_time # seconds
+
+
+        # Strategies for ensuring completion
+        self._pause = pause
+        self._pause_iters = pause_iters
+
+        self._post_copy = post_copy
+        self._post_copy_iters = post_copy_iters
+
+        self._auto_converge = auto_converge
+        self._auto_converge_step = auto_converge_step # percentage CPU time
+
+        self._compression_mt = compression_mt
+        self._compression_mt_threads = compression_mt_threads
+
+        self._compression_xbzrle = compression_xbzrle
+        self._compression_xbzrle_cache = compression_xbzrle_cache # percentage of guest RAM
+
+    def serialize(self):
+        return {
+            "name": self._name,
+            "downtime": self._downtime,
+            "bandwidth": self._bandwidth,
+            "max_iters": self._max_iters,
+            "max_time": self._max_time,
+            "pause": self._pause,
+            "pause_iters": self._pause_iters,
+            "post_copy": self._post_copy,
+            "post_copy_iters": self._post_copy_iters,
+            "auto_converge": self._auto_converge,
+            "auto_converge_step": self._auto_converge_step,
+            "compression_mt": self._compression_mt,
+            "compression_mt_threads": self._compression_mt_threads,
+            "compression_xbzrle": self._compression_xbzrle,
+            "compression_xbzrle_cache": self._compression_xbzrle_cache,
+        }
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(
+            data["name"],
+            data["downtime"],
+            data["bandwidth"],
+            data["max_iters"],
+            data["max_time"],
+            data["pause"],
+            data["pause_iters"],
+            data["post_copy"],
+            data["post_copy_iters"],
+            data["auto_converge"],
+            data["auto_converge_step"],
+            data["compression_mt"],
+            data["compression_mt_threads"],
+            data["compression_xbzrle"],
+            data["compression_xbzrle_cache"])
diff --git a/tests/migration/guestperf/shell.py b/tests/migration/guestperf/shell.py
new file mode 100644
index 0000000..185c569
--- /dev/null
+++ b/tests/migration/guestperf/shell.py
@@ -0,0 +1,255 @@
+#
+# Migration test command line shell integration
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+import argparse
+import fnmatch
+import os
+import os.path
+import platform
+import sys
+
+from guestperf.hardware import Hardware
+from guestperf.engine import Engine
+from guestperf.scenario import Scenario
+from guestperf.comparison import COMPARISONS
+from guestperf.plot import Plot
+from guestperf.report import Report
+
+
+class BaseShell(object):
+
+    def __init__(self):
+        parser = argparse.ArgumentParser(description="Migration Test Tool")
+
+        # Test args
+        parser.add_argument("--debug", dest="debug", default=False, action="store_true")
+        parser.add_argument("--verbose", dest="verbose", default=False, action="store_true")
+        parser.add_argument("--sleep", dest="sleep", default=15, type=int)
+        parser.add_argument("--binary", dest="binary", default="/usr/bin/qemu-system-x86_64")
+        parser.add_argument("--dst-host", dest="dst_host", default="localhost")
+        parser.add_argument("--kernel", dest="kernel", default="/boot/vmlinuz-%s" % platform.release())
+        parser.add_argument("--initrd", dest="initrd", default="tests/migration/initrd-stress.img")
+        parser.add_argument("--transport", dest="transport", default="unix")
+
+
+        # Hardware args
+        parser.add_argument("--cpus", dest="cpus", default=1, type=int)
+        parser.add_argument("--mem", dest="mem", default=1, type=int)
+        parser.add_argument("--src-cpu-bind", dest="src_cpu_bind", default="")
+        parser.add_argument("--src-mem-bind", dest="src_mem_bind", default="")
+        parser.add_argument("--dst-cpu-bind", dest="dst_cpu_bind", default="")
+        parser.add_argument("--dst-mem-bind", dest="dst_mem_bind", default="")
+        parser.add_argument("--prealloc-pages", dest="prealloc_pages", default=False)
+        parser.add_argument("--huge-pages", dest="huge_pages", default=False)
+        parser.add_argument("--locked-pages", dest="locked_pages", default=False)
+
+        self._parser = parser
+
+    def get_engine(self, args):
+        return Engine(binary=args.binary,
+                      dst_host=args.dst_host,
+                      kernel=args.kernel,
+                      initrd=args.initrd,
+                      transport=args.transport,
+                      sleep=args.sleep,
+                      debug=args.debug,
+                      verbose=args.verbose)
+
+    def get_hardware(self, args):
+        def split_map(value):
+            if value == "":
+                return []
+            return value.split(",")
+
+        return Hardware(cpus=args.cpus,
+                        mem=args.mem,
+
+                        src_cpu_bind=split_map(args.src_cpu_bind),
+                        src_mem_bind=split_map(args.src_mem_bind),
+                        dst_cpu_bind=split_map(args.dst_cpu_bind),
+                        dst_mem_bind=split_map(args.dst_mem_bind),
+
+                        locked_pages=args.locked_pages,
+                        huge_pages=args.huge_pages,
+                        prealloc_pages=args.prealloc_pages)
+
+
+class Shell(BaseShell):
+
+    def __init__(self):
+        super(Shell, self).__init__()
+
+        parser = self._parser
+
+        parser.add_argument("--output", dest="output", default=None)
+
+        # Scenario args
+        parser.add_argument("--max-iters", dest="max_iters", default=30, type=int)
+        parser.add_argument("--max-time", dest="max_time", default=300, type=int)
+        parser.add_argument("--bandwidth", dest="bandwidth", default=125000, type=int)
+        parser.add_argument("--downtime", dest="downtime", default=500, type=int)
+
+        parser.add_argument("--pause", dest="pause", default=False, action="store_true")
+        parser.add_argument("--pause-iters", dest="pause_iters", default=5, type=int)
+
+        parser.add_argument("--post-copy", dest="post_copy", default=False, action="store_true")
+        parser.add_argument("--post-copy-iters", dest="post_copy_iters", default=5, type=int)
+
+        parser.add_argument("--auto-converge", dest="auto_converge", default=False, action="store_true")
+        parser.add_argument("--auto-converge-step", dest="auto_converge_step", default=10, type=int)
+
+        parser.add_argument("--compression-mt", dest="compression_mt", default=False, action="store_true")
+        parser.add_argument("--compression-mt-threads", dest="compression_mt_threads", default=1, type=int)
+
+        parser.add_argument("--compression-xbzrle", dest="compression_xbzrle", default=False, action="store_true")
+        parser.add_argument("--compression-xbzrle-cache", dest="compression_xbzrle_cache", default=10, type=int)
+
+    def get_scenario(self, args):
+        return Scenario(name="perfreport",
+                        downtime=args.downtime,
+                        bandwidth=args.bandwidth,
+                        max_iters=args.max_iters,
+                        max_time=args.max_time,
+
+                        pause=args.pause,
+                        pause_iters=args.pause_iters,
+
+                        post_copy=args.post_copy,
+                        post_copy_iters=args.post_copy_iters,
+
+                        auto_converge=args.auto_converge,
+                        auto_converge_step=args.auto_converge_step,
+
+                        compression_mt=args.compression_mt,
+                        compression_mt_threads=args.compression_mt_threads,
+
+                        compression_xbzrle=args.compression_xbzrle,
+                        compression_xbzrle_cache=args.compression_xbzrle_cache)
+
+    def run(self, argv):
+        args = self._parser.parse_args(argv)
+
+        engine = self.get_engine(args)
+        hardware = self.get_hardware(args)
+        scenario = self.get_scenario(args)
+
+        try:
+            report = engine.run(hardware, scenario)
+            if args.output is None:
+                print report.to_json()
+            else:
+                with open(args.output, "w") as fh:
+                    print >>fh, report.to_json()
+            return 0
+        except Exception as e:
+            print >>sys.stderr, "Error: %s" % str(e)
+            if args.debug:
+                raise
+            return 1
+
+
+class BatchShell(BaseShell):
+
+    def __init__(self):
+        super(BatchShell, self).__init__()
+
+        parser = self._parser
+
+        parser.add_argument("--filter", dest="filter", default="*")
+        parser.add_argument("--output", dest="output", default=os.getcwd())
+
+    def run(self, argv):
+        args = self._parser.parse_args(argv)
+
+        engine = self.get_engine(args)
+        hardware = self.get_hardware(args)
+
+        try:
+            for comparison in COMPARISONS:
+                compdir = os.path.join(args.output, comparison._name)
+                for scenario in comparison._scenarios:
+                    name = os.path.join(comparison._name, scenario._name)
+                    if not fnmatch.fnmatch(name, args.filter):
+                        if args.verbose:
+                            print "Skipping %s" % name
+                        continue
+
+                    if args.verbose:
+                        print "Running %s" % name
+
+                    dirname = os.path.join(args.output, comparison._name)
+                    filename = os.path.join(dirname, scenario._name + ".json")
+                    if not os.path.exists(dirname):
+                        os.makedirs(dirname)
+                    report = engine.run(hardware, scenario)
+                    with open(filename, "w") as fh:
+                        print >>fh, report.to_json()
+        except Exception as e:
+            print >>sys.stderr, "Error: %s" % str(e)
+            if args.debug:
+                raise
+
+
+class PlotShell(object):
+
+    def __init__(self):
+        super(PlotShell, self).__init__()
+
+        self._parser = argparse.ArgumentParser(description="Migration Test Tool")
+
+        self._parser.add_argument("--output", dest="output", default=None)
+
+        self._parser.add_argument("--debug", dest="debug", default=False, action="store_true")
+        self._parser.add_argument("--verbose", dest="verbose", default=False, action="store_true")
+
+        self._parser.add_argument("--migration-iters", dest="migration_iters", default=False, action="store_true")
+        self._parser.add_argument("--total-guest-cpu", dest="total_guest_cpu", default=False, action="store_true")
+        self._parser.add_argument("--split-guest-cpu", dest="split_guest_cpu", default=False, action="store_true")
+        self._parser.add_argument("--qemu-cpu", dest="qemu_cpu", default=False, action="store_true")
+        self._parser.add_argument("--vcpu-cpu", dest="vcpu_cpu", default=False, action="store_true")
+
+        self._parser.add_argument("reports", nargs='*')
+
+    def run(self, argv):
+        args = self._parser.parse_args(argv)
+
+        if len(args.reports) == 0:
+            print >>sys.stderr, "At least one report required"
+            return 1
+
+        if not (args.qemu_cpu or
+                args.vcpu_cpu or
+                args.total_guest_cpu or
+                args.split_guest_cpu):
+            print >>sys.stderr, "At least one chart type is required"
+            return 1
+
+        reports = []
+        for report in args.reports:
+            reports.append(Report.from_json_file(report))
+
+        plot = Plot(reports,
+                    args.migration_iters,
+                    args.total_guest_cpu,
+                    args.split_guest_cpu,
+                    args.qemu_cpu,
+                    args.vcpu_cpu)
+
+        plot.generate(args.output)
diff --git a/tests/migration/guestperf/timings.py b/tests/migration/guestperf/timings.py
new file mode 100644
index 0000000..f94d809
--- /dev/null
+++ b/tests/migration/guestperf/timings.py
@@ -0,0 +1,55 @@
+#
+# Migration test timing records
+#
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, see <http://www.gnu.org/licenses/>.
+#
+
+
+class TimingRecord(object):
+
+    def __init__(self, tid, timestamp, value):
+
+        self._tid = tid
+        self._timestamp = timestamp
+        self._value = value
+
+    def serialize(self):
+        return {
+            "tid": self._tid,
+            "timestamp": self._timestamp,
+            "value": self._value
+        }
+
+    @classmethod
+    def deserialize(cls, data):
+        return cls(
+            data["tid"],
+            data["timestamp"],
+            data["value"])
+
+
+class Timings(object):
+
+    def __init__(self, records):
+
+        self._records = records
+
+    def serialize(self):
+        return [record.serialize() for record in self._records]
+
+    @classmethod
+    def deserialize(cls, data):
+        return Timings([TimingRecord.deserialize(record) for record in data])
diff --git a/tests/migration/stress.c b/tests/migration/stress.c
new file mode 100644
index 0000000..cf8ce8b
--- /dev/null
+++ b/tests/migration/stress.c
@@ -0,0 +1,367 @@
+/*
+ * Migration stress workload
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <getopt.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <unistd.h>
+#include <sys/reboot.h>
+#include <sys/syscall.h>
+#include <linux/random.h>
+#include <sys/time.h>
+#include <pthread.h>
+#include <fcntl.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+const char *argv0;
+
+#define PAGE_SIZE 4096
+
+static int gettid(void)
+{
+    return syscall(SYS_gettid);
+}
+
+static __attribute__((noreturn)) void exit_failure(void)
+{
+    if (getpid() == 1) {
+        sync();
+        reboot(RB_POWER_OFF);
+        fprintf(stderr, "%s (%05d): ERROR: cannot reboot: %s\n",
+                argv0, gettid(), strerror(errno));
+        abort();
+    } else {
+        exit(1);
+    }
+}
+
+static __attribute__((noreturn)) void exit_success(void)
+{
+    if (getpid() == 1) {
+        sync();
+        reboot(RB_POWER_OFF);
+        fprintf(stderr, "%s (%05d): ERROR: cannot reboot: %s\n",
+                argv0, gettid(), strerror(errno));
+        abort();
+    } else {
+        exit(0);
+    }
+}
+
+static int get_command_arg_str(const char *name,
+                               char **val)
+{
+    static char line[1024];
+    FILE *fp = fopen("/proc/cmdline", "r");
+    char *start, *end;
+
+    if (fp == NULL) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot open /proc/cmdline: %s\n",
+                argv0, gettid(), strerror(errno));
+        return -1;
+    }
+
+    if (!fgets(line, sizeof line, fp)) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot read /proc/cmdline: %s\n",
+                argv0, gettid(), strerror(errno));
+        fclose(fp);
+        return -1;
+    }
+    fclose(fp);
+
+    start = strstr(line, name);
+    if (!start)
+        return 0;
+
+    start += strlen(name);
+
+    if (*start != '=') {
+        fprintf(stderr, "%s (%05d): ERROR: no value provided for '%s' in /proc/cmdline\n",
+                argv0, gettid(), name);
+    }
+    start++;
+
+    end = strstr(start, " ");
+    if (!end)
+        end = strstr(start, "\n");
+
+    if (end == start) {
+        fprintf(stderr, "%s (%05d): ERROR: no value provided for '%s' in /proc/cmdline\n",
+                argv0, gettid(), name);
+        return -1;
+    }
+
+    if (end)
+        *val = strndup(start, end - start);
+    else
+        *val = strdup(start);
+    return 1;
+}
+
+
+static int get_command_arg_ull(const char *name,
+                               unsigned long long *val)
+{
+    char *valstr;
+    char *end;
+
+    int ret = get_command_arg_str(name, &valstr);
+    if (ret <= 0)
+        return ret;
+
+    errno = 0;
+    *val = strtoll(valstr, &end, 10);
+    if (errno || *end) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot parse %s value %s\n",
+                argv0, gettid(), name, valstr);
+        free(valstr);
+        return -1;
+    }
+    free(valstr);
+    return 0;
+}
+
+
+static int random_bytes(char *buf, size_t len)
+{
+    int fd;
+
+    fd = open("/dev/urandom", O_RDONLY);
+    if (fd < 0) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot open /dev/urandom: %s\n",
+                argv0, gettid(), strerror(errno));
+        return -1;
+    }
+
+    if (read(fd, buf, len) != len) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot read /dev/urandom: %s\n",
+                argv0, gettid(), strerror(errno));
+        close(fd);
+        return -1;
+    }
+
+    close(fd);
+
+    return 0;
+}
+
+
+static unsigned long long now(void)
+{
+    struct timeval tv;
+
+    gettimeofday(&tv, NULL);
+
+    return (tv.tv_sec * 1000ull) + (tv.tv_usec / 1000ull);
+}
+
+static int stressone(unsigned long long ramsizeMB)
+{
+    size_t pagesPerMB = 1024 * 1024 / PAGE_SIZE;
+    char *ram = malloc(ramsizeMB * 1024 * 1024);
+    char *ramptr;
+    size_t i, j, k;
+    char *data = malloc(PAGE_SIZE);
+    char *dataptr;
+    size_t nMB = 0;
+    unsigned long long before, after;
+
+    if (!ram) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot allocate %llu MB of RAM: %s\n",
+                argv0, gettid(), ramsizeMB, strerror(errno));
+        return -1;
+    }
+    if (!data) {
+        fprintf(stderr, "%s (%d): ERROR: cannot allocate %d bytes of RAM: %s\n",
+                argv0, gettid(), PAGE_SIZE, strerror(errno));
+        free(ram);
+        return -1;
+    }
+
+    /* We don't care about initial state, but we do want
+     * to fault it all into RAM, otherwise the first iter
+     * of the loop below will be quite slow. We cna't use
+     * 0x0 as the byte as gcc optimizes that away into a
+     * calloc instead :-) */
+    memset(ram, 0xfe, ramsizeMB * 1024 * 1024);
+
+    if (random_bytes(data, PAGE_SIZE) < 0) {
+        free(ram);
+        free(data);
+        return -1;
+    }
+
+    before = now();
+
+    while (1) {
+
+        ramptr = ram;
+        for (i = 0; i < ramsizeMB; i++, nMB++) {
+            for (j = 0; j < pagesPerMB; j++) {
+                dataptr = data;
+                for (k = 0; k < PAGE_SIZE; k += sizeof(long long)) {
+                    ramptr += sizeof(long long);
+                    dataptr += sizeof(long long);
+                    *(unsigned long long *)ramptr ^= *(unsigned long long *)dataptr;
+                }
+            }
+
+            if (nMB == 1024) {
+                after = now();
+                fprintf(stderr, "%s (%05d): INFO: %06llums copied 1 GB in %05llums\n",
+                        argv0, gettid(), after, after - before);
+                before = now();
+                nMB = 0;
+            }
+        }
+    }
+
+    free(data);
+    free(ram);
+}
+
+
+static void *stressthread(void *arg)
+{
+    unsigned long long ramsizeMB = *(unsigned long long *)arg;
+
+    stressone(ramsizeMB);
+
+    return NULL;
+}
+
+static int stress(unsigned long long ramsizeGB, int ncpus)
+{
+    size_t i;
+    unsigned long long ramsizeMB = ramsizeGB * 1024 / ncpus;
+    ncpus--;
+
+    for (i = 0; i < ncpus; i++) {
+        pthread_t thr;
+        pthread_create(&thr, NULL,
+                       stressthread,   &ramsizeMB);
+    }
+
+    stressone(ramsizeMB);
+
+    return 0;
+}
+
+
+static int mount_misc(const char *fstype, const char *dir)
+{
+    if (mkdir(dir, 0755) < 0 && errno != EEXIST) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot create %s: %s\n",
+                argv0, gettid(), dir, strerror(errno));
+        return -1;
+    }
+
+    if (mount("none", dir, fstype, 0, NULL) < 0) {
+        fprintf(stderr, "%s (%05d): ERROR: cannot mount %s: %s\n",
+                argv0, gettid(), dir, strerror(errno));
+        return -1;
+    }
+
+    return 0;
+}
+
+static int mount_all(void)
+{
+    if (mount_misc("proc", "/proc") < 0 ||
+        mount_misc("sysfs", "/sys") < 0 ||
+        mount_misc("tmpfs", "/dev") < 0)
+        return -1;
+
+    mknod("/dev/urandom", 0777 | S_IFCHR, makedev(1, 9));
+    mknod("/dev/random", 0777 | S_IFCHR, makedev(1, 8));
+
+    return 0;
+}
+
+int main(int argc, char **argv)
+{
+    unsigned long long ramsizeGB = 1;
+    char *end;
+    int ch;
+    int opt_ind = 0;
+    const char *sopt = "hr:c:";
+    struct option lopt[] = {
+        { "help", no_argument, NULL, 'h' },
+        { "ramsize", required_argument, NULL, 'r' },
+        { "cpus", required_argument, NULL, 'c' },
+        { NULL, 0, NULL, 0 }
+    };
+    int ret;
+    int ncpus = 0;
+
+    argv0 = argv[0];
+
+    while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) {
+        switch (ch) {
+        case 'r':
+            errno = 0;
+            ramsizeGB = strtoll(optarg, &end, 10);
+            if (errno != 0 || *end) {
+                fprintf(stderr, "%s (%05d): ERROR: Cannot parse RAM size %s\n",
+                        argv0, gettid(), optarg);
+                exit_failure();
+            }
+            break;
+
+        case 'c':
+            errno = 0;
+            ncpus = strtoll(optarg, &end, 10);
+            if (errno != 0 || *end) {
+                fprintf(stderr, "%s (%05d): ERROR: Cannot parse CPU count %s\n",
+                        argv0, gettid(), optarg);
+                exit_failure();
+            }
+            break;
+
+        case '?':
+        case 'h':
+            fprintf(stderr, "%s: [--help][--ramsize GB][--cpus N]\n", argv0);
+            exit_failure();
+        }
+    }
+
+    if (getpid() == 1) {
+        if (mount_all() < 0)
+            exit_failure();
+
+        ret = get_command_arg_ull("ramsize", &ramsizeGB);
+        if (ret < 0)
+            exit_failure();
+    }
+
+    if (ncpus == 0)
+        ncpus = sysconf(_SC_NPROCESSORS_ONLN);
+
+    fprintf(stdout, "%s (%05d): INFO: RAM %llu GiB across %d CPUs\n",
+            argv0, gettid(), ramsizeGB, ncpus);
+
+    if (stress(ramsizeGB, ncpus) < 0)
+        exit_failure();
+
+    exit_success();
+}
diff --git a/tests/multiboot/Makefile b/tests/multiboot/Makefile
index 34cdd81..36f01dc 100644
--- a/tests/multiboot/Makefile
+++ b/tests/multiboot/Makefile
@@ -6,11 +6,14 @@
 LDFLAGS=-melf_i386 -T link.ld
 LIBS=$(shell $(CC) $(CCFLAGS) -print-libgcc-file-name)
 
-all: mmap.elf
+all: mmap.elf modules.elf
 
 mmap.elf: start.o mmap.o libc.o
 	$(LD) $(LDFLAGS) -o $@ $^ $(LIBS)
 
+modules.elf: start.o modules.o libc.o
+	$(LD) $(LDFLAGS) -o $@ $^ $(LIBS)
+
 %.o: %.c
 	$(CC) $(CCFLAGS) -c -o $@ $^
 
diff --git a/tests/multiboot/libc.c b/tests/multiboot/libc.c
index 05abbd9..6df9bda 100644
--- a/tests/multiboot/libc.c
+++ b/tests/multiboot/libc.c
@@ -22,6 +22,18 @@
 
 #include "libc.h"
 
+void* memcpy(void *dest, const void *src, int n)
+{
+    char *d = dest;
+    const char *s = src;
+
+    while (n--) {
+        *d++ = *s++;
+    }
+
+    return dest;
+}
+
 static void print_char(char c)
 {
     outb(0xe9, c);
diff --git a/tests/multiboot/libc.h b/tests/multiboot/libc.h
index 80eec5b..04c9922 100644
--- a/tests/multiboot/libc.h
+++ b/tests/multiboot/libc.h
@@ -57,5 +57,6 @@
 /* Misc functions */
 
 void printf(const char *fmt, ...);
+void* memcpy(void *dest, const void *src, int n);
 
 #endif
diff --git a/tests/multiboot/mmap.out b/tests/multiboot/mmap.out
index e70b6eb..003e109 100644
--- a/tests/multiboot/mmap.out
+++ b/tests/multiboot/mmap.out
@@ -4,14 +4,14 @@
 === Running test case: mmap.elf  ===
 
 Lower memory: 639k
-Upper memory: 130040k
+Upper memory: 129920k
 
 e820 memory map:
 0x0 - 0x9fc00: type 1 [entry size: 20]
 0x9fc00 - 0xa0000: type 2 [entry size: 20]
 0xf0000 - 0x100000: type 2 [entry size: 20]
-0x100000 - 0x7ffe000: type 1 [entry size: 20]
-0x7ffe000 - 0x8000000: type 2 [entry size: 20]
+0x100000 - 0x7fe0000: type 1 [entry size: 20]
+0x7fe0000 - 0x8000000: type 2 [entry size: 20]
 0xfffc0000 - 0x100000000: type 2 [entry size: 20]
 
 mmap start:       0x9000
@@ -22,32 +22,31 @@
 === Running test case: mmap.elf -m 1.1M ===
 
 Lower memory: 639k
-Upper memory: 96k
+Upper memory: 104k
 
 e820 memory map:
 0x0 - 0x9fc00: type 1 [entry size: 20]
 0x9fc00 - 0xa0000: type 2 [entry size: 20]
 0xf0000 - 0x100000: type 2 [entry size: 20]
-0x100000 - 0x118000: type 1 [entry size: 20]
-0x118000 - 0x11a000: type 2 [entry size: 20]
+0x100000 - 0x11a000: type 1 [entry size: 20]
 0xfffc0000 - 0x100000000: type 2 [entry size: 20]
 
 mmap start:       0x9000
-mmap end:         0x9090
-real mmap end:    0x9090
+mmap end:         0x9078
+real mmap end:    0x9078
 
 
 === Running test case: mmap.elf -m 2G ===
 
 Lower memory: 639k
-Upper memory: 2096120k
+Upper memory: 2096000k
 
 e820 memory map:
 0x0 - 0x9fc00: type 1 [entry size: 20]
 0x9fc00 - 0xa0000: type 2 [entry size: 20]
 0xf0000 - 0x100000: type 2 [entry size: 20]
-0x100000 - 0x7fffe000: type 1 [entry size: 20]
-0x7fffe000 - 0x80000000: type 2 [entry size: 20]
+0x100000 - 0x7ffe0000: type 1 [entry size: 20]
+0x7ffe0000 - 0x80000000: type 2 [entry size: 20]
 0xfffc0000 - 0x100000000: type 2 [entry size: 20]
 
 mmap start:       0x9000
@@ -58,16 +57,16 @@
 === Running test case: mmap.elf -m 4G ===
 
 Lower memory: 639k
-Upper memory: 3668984k
+Upper memory: 3144576k
 
 e820 memory map:
 0x0 - 0x9fc00: type 1 [entry size: 20]
 0x9fc00 - 0xa0000: type 2 [entry size: 20]
 0xf0000 - 0x100000: type 2 [entry size: 20]
-0x100000 - 0xdfffe000: type 1 [entry size: 20]
-0xdfffe000 - 0xe0000000: type 2 [entry size: 20]
+0x100000 - 0xbffe0000: type 1 [entry size: 20]
+0xbffe0000 - 0xc0000000: type 2 [entry size: 20]
 0xfffc0000 - 0x100000000: type 2 [entry size: 20]
-0x100000000 - 0x120000000: type 1 [entry size: 20]
+0x100000000 - 0x140000000: type 1 [entry size: 20]
 
 mmap start:       0x9000
 mmap end:         0x90a8
@@ -77,16 +76,16 @@
 === Running test case: mmap.elf -m 8G ===
 
 Lower memory: 639k
-Upper memory: 3668984k
+Upper memory: 3144576k
 
 e820 memory map:
 0x0 - 0x9fc00: type 1 [entry size: 20]
 0x9fc00 - 0xa0000: type 2 [entry size: 20]
 0xf0000 - 0x100000: type 2 [entry size: 20]
-0x100000 - 0xdfffe000: type 1 [entry size: 20]
-0xdfffe000 - 0xe0000000: type 2 [entry size: 20]
+0x100000 - 0xbffe0000: type 1 [entry size: 20]
+0xbffe0000 - 0xc0000000: type 2 [entry size: 20]
 0xfffc0000 - 0x100000000: type 2 [entry size: 20]
-0x100000000 - 0x220000000: type 1 [entry size: 20]
+0x100000000 - 0x240000000: type 1 [entry size: 20]
 
 mmap start:       0x9000
 mmap end:         0x90a8
diff --git a/tests/multiboot/module.txt b/tests/multiboot/module.txt
new file mode 100644
index 0000000..54c1d27
--- /dev/null
+++ b/tests/multiboot/module.txt
@@ -0,0 +1 @@
+This is a test file that is used as a multiboot module.
diff --git a/tests/multiboot/modules.c b/tests/multiboot/modules.c
new file mode 100644
index 0000000..531601f
--- /dev/null
+++ b/tests/multiboot/modules.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2015 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "libc.h"
+#include "multiboot.h"
+
+int test_main(uint32_t magic, struct mb_info *mbi)
+{
+    struct mb_module *mod;
+    unsigned int i;
+
+    (void) magic;
+
+    printf("Module list with %d entries at %x\n",
+           mbi->mods_count, mbi->mods_addr);
+
+    for (i = 0, mod = (struct mb_module*) mbi->mods_addr;
+         i < mbi->mods_count;
+         i++, mod++)
+    {
+        char buf[1024];
+        unsigned int size = mod->mod_end - mod->mod_start;
+
+        printf("[%p] Module: %x - %x (%d bytes) '%s'\n",
+               mod, mod->mod_start, mod->mod_end, size, mod->string);
+
+        /* Print test file, but remove the newline at the end */
+        if (size < sizeof(buf)) {
+            memcpy(buf, (void*) mod->mod_start, size);
+            buf[size - 1] = '\0';
+            printf("         Content: '%s'\n", buf);
+        }
+    }
+
+    return 0;
+}
diff --git a/tests/multiboot/modules.out b/tests/multiboot/modules.out
new file mode 100644
index 0000000..1636708
--- /dev/null
+++ b/tests/multiboot/modules.out
@@ -0,0 +1,38 @@
+
+
+
+=== Running test case: modules.elf  ===
+
+Module list with 0 entries at 102000
+
+
+=== Running test case: modules.elf -initrd module.txt ===
+
+Module list with 1 entries at 102000
+[102000] Module: 103000 - 103038 (56 bytes) 'module.txt'
+         Content: 'This is a test file that is used as a multiboot module.'
+
+
+=== Running test case: modules.elf -initrd module.txt argument ===
+
+Module list with 1 entries at 102000
+[102000] Module: 103000 - 103038 (56 bytes) 'module.txt argument'
+         Content: 'This is a test file that is used as a multiboot module.'
+
+
+=== Running test case: modules.elf -initrd module.txt argument,,with,,commas ===
+
+Module list with 1 entries at 102000
+[102000] Module: 103000 - 103038 (56 bytes) 'module.txt argument,with,commas'
+         Content: 'This is a test file that is used as a multiboot module.'
+
+
+=== Running test case: modules.elf -initrd module.txt,module.txt argument,module.txt ===
+
+Module list with 3 entries at 102000
+[102000] Module: 103000 - 103038 (56 bytes) 'module.txt'
+         Content: 'This is a test file that is used as a multiboot module.'
+[102010] Module: 104000 - 104038 (56 bytes) 'module.txt argument'
+         Content: 'This is a test file that is used as a multiboot module.'
+[102020] Module: 105000 - 105038 (56 bytes) 'module.txt'
+         Content: 'This is a test file that is used as a multiboot module.'
diff --git a/tests/multiboot/run_test.sh b/tests/multiboot/run_test.sh
index 97a9a49..78d7edf 100755
--- a/tests/multiboot/run_test.sh
+++ b/tests/multiboot/run_test.sh
@@ -48,10 +48,17 @@
     run_qemu mmap.elf -m 8G
 }
 
+modules() {
+    run_qemu modules.elf
+    run_qemu modules.elf -initrd module.txt
+    run_qemu modules.elf -initrd "module.txt argument"
+    run_qemu modules.elf -initrd "module.txt argument,,with,,commas"
+    run_qemu modules.elf -initrd "module.txt,module.txt argument,module.txt"
+}
 
 make all
 
-for t in mmap; do
+for t in mmap modules; do
 
     echo > test.log
     $t
diff --git a/tests/ne2000-test.c b/tests/ne2000-test.c
index 61a678a..b7cf3dd 100644
--- a/tests/ne2000-test.c
+++ b/tests/ne2000-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/tests/nvme-test.c b/tests/nvme-test.c
index 85768e8..c8bece4 100644
--- a/tests/nvme-test.c
+++ b/tests/nvme-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
@@ -24,7 +22,7 @@
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/nvme/nop", nop);
 
-    qtest_start("-drive id=drv0,if=none,file=/dev/null "
+    qtest_start("-drive id=drv0,if=none,file=/dev/null,format=raw "
                 "-device nvme,drive=drv0,serial=foo");
     ret = g_test_run();
 
diff --git a/tests/pc-cpu-test.c b/tests/pc-cpu-test.c
new file mode 100644
index 0000000..4428cea
--- /dev/null
+++ b/tests/pc-cpu-test.c
@@ -0,0 +1,144 @@
+/*
+ * QTest testcase for PC CPUs
+ *
+ * Copyright (c) 2015 SUSE Linux GmbH
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+
+#include "qemu-common.h"
+#include "libqtest.h"
+#include "qapi/qmp/types.h"
+
+struct PCTestData {
+    const char *machine;
+    const char *cpu_model;
+    unsigned sockets;
+    unsigned cores;
+    unsigned threads;
+    unsigned maxcpus;
+};
+typedef struct PCTestData PCTestData;
+
+static void test_pc_with_cpu_add(gconstpointer data)
+{
+    const PCTestData *s = data;
+    char *args;
+    QDict *response;
+    unsigned int i;
+
+    args = g_strdup_printf("-machine %s -cpu %s "
+                           "-smp sockets=%u,cores=%u,threads=%u,maxcpus=%u",
+                           s->machine, s->cpu_model,
+                           s->sockets, s->cores, s->threads, s->maxcpus);
+    qtest_start(args);
+
+    for (i = s->sockets * s->cores * s->threads; i < s->maxcpus; i++) {
+        response = qmp("{ 'execute': 'cpu-add',"
+                       "  'arguments': { 'id': %d } }", i);
+        g_assert(response);
+        g_assert(!qdict_haskey(response, "error"));
+        QDECREF(response);
+    }
+
+    qtest_end();
+    g_free(args);
+}
+
+static void test_pc_without_cpu_add(gconstpointer data)
+{
+    const PCTestData *s = data;
+    char *args;
+    QDict *response;
+
+    args = g_strdup_printf("-machine %s -cpu %s "
+                           "-smp sockets=%u,cores=%u,threads=%u,maxcpus=%u",
+                           s->machine, s->cpu_model,
+                           s->sockets, s->cores, s->threads, s->maxcpus);
+    qtest_start(args);
+
+    response = qmp("{ 'execute': 'cpu-add',"
+                   "  'arguments': { 'id': %d } }",
+                   s->sockets * s->cores * s->threads);
+    g_assert(response);
+    g_assert(qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    qtest_end();
+    g_free(args);
+}
+
+static void add_pc_test_cases(void)
+{
+    QDict *response, *minfo;
+    QList *list;
+    const QListEntry *p;
+    QObject *qobj;
+    QString *qstr;
+    const char *mname, *path;
+    PCTestData *data;
+
+    qtest_start("-machine none");
+    response = qmp("{ 'execute': 'query-machines' }");
+    g_assert(response);
+    list = qdict_get_qlist(response, "return");
+    g_assert(list);
+
+    for (p = qlist_first(list); p; p = qlist_next(p)) {
+        minfo = qobject_to_qdict(qlist_entry_obj(p));
+        g_assert(minfo);
+        qobj = qdict_get(minfo, "name");
+        g_assert(qobj);
+        qstr = qobject_to_qstring(qobj);
+        g_assert(qstr);
+        mname = qstring_get_str(qstr);
+        if (!g_str_has_prefix(mname, "pc-")) {
+            continue;
+        }
+        data = g_malloc(sizeof(PCTestData));
+        data->machine = mname;
+        data->cpu_model = "Haswell"; /* 1.3+ theoretically */
+        data->sockets = 1;
+        data->cores = 3;
+        data->threads = 2;
+        data->maxcpus = data->sockets * data->cores * data->threads * 2;
+        if (g_str_has_suffix(mname, "-1.4") ||
+            (strcmp(mname, "pc-1.3") == 0) ||
+            (strcmp(mname, "pc-1.2") == 0) ||
+            (strcmp(mname, "pc-1.1") == 0) ||
+            (strcmp(mname, "pc-1.0") == 0) ||
+            (strcmp(mname, "pc-0.15") == 0) ||
+            (strcmp(mname, "pc-0.14") == 0) ||
+            (strcmp(mname, "pc-0.13") == 0) ||
+            (strcmp(mname, "pc-0.12") == 0) ||
+            (strcmp(mname, "pc-0.11") == 0) ||
+            (strcmp(mname, "pc-0.10") == 0)) {
+            path = g_strdup_printf("cpu/%s/init/%ux%ux%u&maxcpus=%u",
+                                   mname, data->sockets, data->cores,
+                                   data->threads, data->maxcpus);
+            qtest_add_data_func(path, data, test_pc_without_cpu_add);
+        } else {
+            path = g_strdup_printf("cpu/%s/add/%ux%ux%u&maxcpus=%u",
+                                   mname, data->sockets, data->cores,
+                                   data->threads, data->maxcpus);
+            qtest_add_data_func(path, data, test_pc_with_cpu_add);
+        }
+    }
+    qtest_end();
+}
+
+int main(int argc, char **argv)
+{
+    const char *arch = qtest_get_arch();
+
+    g_test_init(&argc, &argv, NULL);
+
+    if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
+        add_pc_test_cases();
+    }
+
+    return g_test_run();
+}
diff --git a/tests/pcnet-test.c b/tests/pcnet-test.c
index 84af4f3..efb1ef4 100644
--- a/tests/pcnet-test.c
+++ b/tests/pcnet-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/tests/pkix_asn1_tab.c b/tests/pkix_asn1_tab.c
new file mode 100644
index 0000000..903bc02
--- /dev/null
+++ b/tests/pkix_asn1_tab.c
@@ -0,0 +1,1105 @@
+/*
+ * This file is taken from gnutls 1.6.3 under the GPLv2+
+ * and is under copyright of various GNUTLS contributors.
+ */
+
+#include "qemu/osdep.h"
+#include <libtasn1.h>
+
+const ASN1_ARRAY_TYPE pkix_asn1_tab[] = {
+  {"PKIX1", 536875024, 0},
+  {0, 1073741836, 0},
+  {"id-ce", 1879048204, 0},
+  {"joint-iso-ccitt", 1073741825, "2"},
+  {"ds", 1073741825, "5"},
+  {0, 1, "29"},
+  {"id-ce-authorityKeyIdentifier", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "35"},
+  {"AuthorityKeyIdentifier", 1610612741, 0},
+  {"keyIdentifier", 1610637314, "KeyIdentifier"},
+  {0, 4104, "0"},
+  {"authorityCertIssuer", 1610637314, "GeneralNames"},
+  {0, 4104, "1"},
+  {"authorityCertSerialNumber", 536895490, "CertificateSerialNumber"},
+  {0, 4104, "2"},
+  {"KeyIdentifier", 1073741831, 0},
+  {"id-ce-subjectKeyIdentifier", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "14"},
+  {"SubjectKeyIdentifier", 1073741826, "KeyIdentifier"},
+  {"id-ce-keyUsage", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "15"},
+  {"KeyUsage", 1610874886, 0},
+  {"digitalSignature", 1073741825, "0"},
+  {"nonRepudiation", 1073741825, "1"},
+  {"keyEncipherment", 1073741825, "2"},
+  {"dataEncipherment", 1073741825, "3"},
+  {"keyAgreement", 1073741825, "4"},
+  {"keyCertSign", 1073741825, "5"},
+  {"cRLSign", 1073741825, "6"},
+  {"encipherOnly", 1073741825, "7"},
+  {"decipherOnly", 1, "8"},
+  {"id-ce-privateKeyUsagePeriod", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "16"},
+  {"PrivateKeyUsagePeriod", 1610612741, 0},
+  {"notBefore", 1619025937, 0},
+  {0, 4104, "0"},
+  {"notAfter", 545284113, 0},
+  {0, 4104, "1"},
+  {"id-ce-certificatePolicies", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "32"},
+  {"CertificatePolicies", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "PolicyInformation"},
+  {"PolicyInformation", 1610612741, 0},
+  {"policyIdentifier", 1073741826, "CertPolicyId"},
+  {"policyQualifiers", 538984459, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "PolicyQualifierInfo"},
+  {"CertPolicyId", 1073741836, 0},
+  {"PolicyQualifierInfo", 1610612741, 0},
+  {"policyQualifierId", 1073741826, "PolicyQualifierId"},
+  {"qualifier", 541065229, 0},
+  {"policyQualifierId", 1, 0},
+  {"PolicyQualifierId", 1073741836, 0},
+  {"CPSuri", 1073741826, "IA5String"},
+  {"UserNotice", 1610612741, 0},
+  {"noticeRef", 1073758210, "NoticeReference"},
+  {"explicitText", 16386, "DisplayText"},
+  {"NoticeReference", 1610612741, 0},
+  {"organization", 1073741826, "DisplayText"},
+  {"noticeNumbers", 536870923, 0},
+  {0, 3, 0},
+  {"DisplayText", 1610612754, 0},
+  {"visibleString", 1612709890, "VisibleString"},
+  {"200", 524298, "1"},
+  {"bmpString", 1612709890, "BMPString"},
+  {"200", 524298, "1"},
+  {"utf8String", 538968066, "UTF8String"},
+  {"200", 524298, "1"},
+  {"id-ce-policyMappings", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "33"},
+  {"PolicyMappings", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 536870917, 0},
+  {"issuerDomainPolicy", 1073741826, "CertPolicyId"},
+  {"subjectDomainPolicy", 2, "CertPolicyId"},
+  {"DirectoryString", 1610612754, 0},
+  {"teletexString", 1612709890, "TeletexString"},
+  {"MAX", 524298, "1"},
+  {"printableString", 1612709890, "PrintableString"},
+  {"MAX", 524298, "1"},
+  {"universalString", 1612709890, "UniversalString"},
+  {"MAX", 524298, "1"},
+  {"utf8String", 1612709890, "UTF8String"},
+  {"MAX", 524298, "1"},
+  {"bmpString", 1612709890, "BMPString"},
+  {"MAX", 524298, "1"},
+  {"ia5String", 538968066, "IA5String"},
+  {"MAX", 524298, "1"},
+  {"id-ce-subjectAltName", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "17"},
+  {"SubjectAltName", 1073741826, "GeneralNames"},
+  {"GeneralNames", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "GeneralName"},
+  {"GeneralName", 1610612754, 0},
+  {"otherName", 1610620930, "AnotherName"},
+  {0, 4104, "0"},
+  {"rfc822Name", 1610620930, "IA5String"},
+  {0, 4104, "1"},
+  {"dNSName", 1610620930, "IA5String"},
+  {0, 4104, "2"},
+  {"x400Address", 1610620930, "ORAddress"},
+  {0, 4104, "3"},
+  {"directoryName", 1610620930, "RDNSequence"},
+  {0, 2056, "4"},
+  {"ediPartyName", 1610620930, "EDIPartyName"},
+  {0, 4104, "5"},
+  {"uniformResourceIdentifier", 1610620930, "IA5String"},
+  {0, 4104, "6"},
+  {"iPAddress", 1610620935, 0},
+  {0, 4104, "7"},
+  {"registeredID", 536879116, 0},
+  {0, 4104, "8"},
+  {"AnotherName", 1610612741, 0},
+  {"type-id", 1073741836, 0},
+  {"value", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"type-id", 1, 0},
+  {"EDIPartyName", 1610612741, 0},
+  {"nameAssigner", 1610637314, "DirectoryString"},
+  {0, 4104, "0"},
+  {"partyName", 536879106, "DirectoryString"},
+  {0, 4104, "1"},
+  {"id-ce-issuerAltName", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "18"},
+  {"IssuerAltName", 1073741826, "GeneralNames"},
+  {"id-ce-subjectDirectoryAttributes", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "9"},
+  {"SubjectDirectoryAttributes", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Attribute"},
+  {"id-ce-basicConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "19"},
+  {"BasicConstraints", 1610612741, 0},
+  {"cA", 1610645508, 0},
+  {0, 131081, 0},
+  {"pathLenConstraint", 537411587, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-nameConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "30"},
+  {"NameConstraints", 1610612741, 0},
+  {"permittedSubtrees", 1610637314, "GeneralSubtrees"},
+  {0, 4104, "0"},
+  {"excludedSubtrees", 536895490, "GeneralSubtrees"},
+  {0, 4104, "1"},
+  {"GeneralSubtrees", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "GeneralSubtree"},
+  {"GeneralSubtree", 1610612741, 0},
+  {"base", 1073741826, "GeneralName"},
+  {"minimum", 1610653698, "BaseDistance"},
+  {0, 1073741833, "0"},
+  {0, 4104, "0"},
+  {"maximum", 536895490, "BaseDistance"},
+  {0, 4104, "1"},
+  {"BaseDistance", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-policyConstraints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "36"},
+  {"PolicyConstraints", 1610612741, 0},
+  {"requireExplicitPolicy", 1610637314, "SkipCerts"},
+  {0, 4104, "0"},
+  {"inhibitPolicyMapping", 536895490, "SkipCerts"},
+  {0, 4104, "1"},
+  {"SkipCerts", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-cRLDistributionPoints", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "31"},
+  {"CRLDistributionPoints", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "DistributionPoint"},
+  {"DistributionPoint", 1610612741, 0},
+  {"distributionPoint", 1610637314, "DistributionPointName"},
+  {0, 2056, "0"},
+  {"reasons", 1610637314, "ReasonFlags"},
+  {0, 4104, "1"},
+  {"cRLIssuer", 536895490, "GeneralNames"},
+  {0, 4104, "2"},
+  {"DistributionPointName", 1610612754, 0},
+  {"fullName", 1610620930, "GeneralNames"},
+  {0, 4104, "0"},
+  {"nameRelativeToCRLIssuer", 536879106, "RelativeDistinguishedName"},
+  {0, 4104, "1"},
+  {"ReasonFlags", 1610874886, 0},
+  {"unused", 1073741825, "0"},
+  {"keyCompromise", 1073741825, "1"},
+  {"cACompromise", 1073741825, "2"},
+  {"affiliationChanged", 1073741825, "3"},
+  {"superseded", 1073741825, "4"},
+  {"cessationOfOperation", 1073741825, "5"},
+  {"certificateHold", 1073741825, "6"},
+  {"privilegeWithdrawn", 1073741825, "7"},
+  {"aACompromise", 1, "8"},
+  {"id-ce-extKeyUsage", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "37"},
+  {"ExtKeyUsageSyntax", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "KeyPurposeId"},
+  {"KeyPurposeId", 1073741836, 0},
+  {"id-kp-serverAuth", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "1"},
+  {"id-kp-clientAuth", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "2"},
+  {"id-kp-codeSigning", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "3"},
+  {"id-kp-emailProtection", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "4"},
+  {"id-kp-ipsecEndSystem", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "5"},
+  {"id-kp-ipsecTunnel", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "6"},
+  {"id-kp-ipsecUser", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "7"},
+  {"id-kp-timeStamping", 1879048204, 0},
+  {0, 1073741825, "id-kp"},
+  {0, 1, "8"},
+  {"id-pe-authorityInfoAccess", 1879048204, 0},
+  {0, 1073741825, "id-pe"},
+  {0, 1, "1"},
+  {"AuthorityInfoAccessSyntax", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "AccessDescription"},
+  {"AccessDescription", 1610612741, 0},
+  {"accessMethod", 1073741836, 0},
+  {"accessLocation", 2, "GeneralName"},
+  {"id-ce-cRLNumber", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "20"},
+  {"CRLNumber", 1611137027, 0},
+  {"0", 10, "MAX"},
+  {"id-ce-issuingDistributionPoint", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "28"},
+  {"IssuingDistributionPoint", 1610612741, 0},
+  {"distributionPoint", 1610637314, "DistributionPointName"},
+  {0, 4104, "0"},
+  {"onlyContainsUserCerts", 1610653700, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "1"},
+  {"onlyContainsCACerts", 1610653700, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "2"},
+  {"onlySomeReasons", 1610637314, "ReasonFlags"},
+  {0, 4104, "3"},
+  {"indirectCRL", 536911876, 0},
+  {0, 1073872905, 0},
+  {0, 4104, "4"},
+  {"id-ce-deltaCRLIndicator", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "27"},
+  {"BaseCRLNumber", 1073741826, "CRLNumber"},
+  {"id-ce-cRLReasons", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "21"},
+  {"CRLReason", 1610874901, 0},
+  {"unspecified", 1073741825, "0"},
+  {"keyCompromise", 1073741825, "1"},
+  {"cACompromise", 1073741825, "2"},
+  {"affiliationChanged", 1073741825, "3"},
+  {"superseded", 1073741825, "4"},
+  {"cessationOfOperation", 1073741825, "5"},
+  {"certificateHold", 1073741825, "6"},
+  {"removeFromCRL", 1, "8"},
+  {"id-ce-certificateIssuer", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "29"},
+  {"CertificateIssuer", 1073741826, "GeneralNames"},
+  {"id-ce-holdInstructionCode", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "23"},
+  {"HoldInstructionCode", 1073741836, 0},
+  {"holdInstruction", 1879048204, 0},
+  {"joint-iso-itu-t", 1073741825, "2"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9cm", 1073741825, "10040"},
+  {0, 1, "2"},
+  {"id-holdinstruction-none", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "1"},
+  {"id-holdinstruction-callissuer", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "2"},
+  {"id-holdinstruction-reject", 1879048204, 0},
+  {0, 1073741825, "holdInstruction"},
+  {0, 1, "3"},
+  {"id-ce-invalidityDate", 1879048204, 0},
+  {0, 1073741825, "id-ce"},
+  {0, 1, "24"},
+  {"InvalidityDate", 1082130449, 0},
+  {"VisibleString", 1610620935, 0},
+  {0, 4360, "26"},
+  {"NumericString", 1610620935, 0},
+  {0, 4360, "18"},
+  {"IA5String", 1610620935, 0},
+  {0, 4360, "22"},
+  {"TeletexString", 1610620935, 0},
+  {0, 4360, "20"},
+  {"PrintableString", 1610620935, 0},
+  {0, 4360, "19"},
+  {"UniversalString", 1610620935, 0},
+  {0, 4360, "28"},
+  {"BMPString", 1610620935, 0},
+  {0, 4360, "30"},
+  {"UTF8String", 1610620935, 0},
+  {0, 4360, "12"},
+  {"id-pkix", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"identified-organization", 1073741825, "3"},
+  {"dod", 1073741825, "6"},
+  {"internet", 1073741825, "1"},
+  {"security", 1073741825, "5"},
+  {"mechanisms", 1073741825, "5"},
+  {"pkix", 1, "7"},
+  {"id-pe", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "1"},
+  {"id-qt", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "2"},
+  {"id-kp", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "3"},
+  {"id-ad", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "48"},
+  {"id-qt-cps", 1879048204, 0},
+  {0, 1073741825, "id-qt"},
+  {0, 1, "1"},
+  {"id-qt-unotice", 1879048204, 0},
+  {0, 1073741825, "id-qt"},
+  {0, 1, "2"},
+  {"id-ad-ocsp", 1879048204, 0},
+  {0, 1073741825, "id-ad"},
+  {0, 1, "1"},
+  {"id-ad-caIssuers", 1879048204, 0},
+  {0, 1073741825, "id-ad"},
+  {0, 1, "2"},
+  {"Attribute", 1610612741, 0},
+  {"type", 1073741826, "AttributeType"},
+  {"values", 536870927, 0},
+  {0, 2, "AttributeValue"},
+  {"AttributeType", 1073741836, 0},
+  {"AttributeValue", 1614807053, 0},
+  {"type", 1, 0},
+  {"AttributeTypeAndValue", 1610612741, 0},
+  {"type", 1073741826, "AttributeType"},
+  {"value", 2, "AttributeValue"},
+  {"id-at", 1879048204, 0},
+  {"joint-iso-ccitt", 1073741825, "2"},
+  {"ds", 1073741825, "5"},
+  {0, 1, "4"},
+  {"id-at-initials", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "43"},
+  {"X520initials", 1073741826, "DirectoryString"},
+  {"id-at-generationQualifier", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "44"},
+  {"X520generationQualifier", 1073741826, "DirectoryString"},
+  {"id-at-surname", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "4"},
+  {"X520surName", 1073741826, "DirectoryString"},
+  {"id-at-givenName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "42"},
+  {"X520givenName", 1073741826, "DirectoryString"},
+  {"id-at-name", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "41"},
+  {"X520name", 1073741826, "DirectoryString"},
+  {"id-at-commonName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "3"},
+  {"X520CommonName", 1073741826, "DirectoryString"},
+  {"id-at-localityName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "7"},
+  {"X520LocalityName", 1073741826, "DirectoryString"},
+  {"id-at-stateOrProvinceName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "8"},
+  {"X520StateOrProvinceName", 1073741826, "DirectoryString"},
+  {"id-at-organizationName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "10"},
+  {"X520OrganizationName", 1073741826, "DirectoryString"},
+  {"id-at-organizationalUnitName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "11"},
+  {"X520OrganizationalUnitName", 1073741826, "DirectoryString"},
+  {"id-at-title", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "12"},
+  {"X520Title", 1073741826, "DirectoryString"},
+  {"id-at-description", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "13"},
+  {"X520Description", 1073741826, "DirectoryString"},
+  {"id-at-dnQualifier", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "46"},
+  {"X520dnQualifier", 1073741826, "PrintableString"},
+  {"id-at-countryName", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "6"},
+  {"X520countryName", 1612709890, "PrintableString"},
+  {0, 1048586, "2"},
+  {"id-at-serialNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "5"},
+  {"X520serialNumber", 1073741826, "PrintableString"},
+  {"id-at-telephoneNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "20"},
+  {"X520telephoneNumber", 1073741826, "PrintableString"},
+  {"id-at-facsimileTelephoneNumber", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "23"},
+  {"X520facsimileTelephoneNumber", 1073741826, "PrintableString"},
+  {"id-at-pseudonym", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "65"},
+  {"X520pseudonym", 1073741826, "DirectoryString"},
+  {"id-at-name", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "41"},
+  {"X520name", 1073741826, "DirectoryString"},
+  {"id-at-streetAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "9"},
+  {"X520streetAddress", 1073741826, "DirectoryString"},
+  {"id-at-postalAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-at"},
+  {0, 1, "16"},
+  {"X520postalAddress", 1073741826, "PostalAddress"},
+  {"PostalAddress", 1610612747, 0},
+  {0, 2, "DirectoryString"},
+  {"pkcs", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1, "1"},
+  {"pkcs-9", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "9"},
+  {"emailAddress", 1880096780, "AttributeType"},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "1"},
+  {"Pkcs9email", 1612709890, "IA5String"},
+  {"ub-emailaddress-length", 524298, "1"},
+  {"Name", 1610612754, 0},
+  {"rdnSequence", 2, "RDNSequence"},
+  {"RDNSequence", 1610612747, 0},
+  {0, 2, "RelativeDistinguishedName"},
+  {"DistinguishedName", 1073741826, "RDNSequence"},
+  {"RelativeDistinguishedName", 1612709903, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "AttributeTypeAndValue"},
+  {"Certificate", 1610612741, 0},
+  {"tbsCertificate", 1073741826, "TBSCertificate"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"TBSCertificate", 1610612741, 0},
+  {"version", 1610653698, "Version"},
+  {0, 1073741833, "v1"},
+  {0, 2056, "0"},
+  {"serialNumber", 1073741826, "CertificateSerialNumber"},
+  {"signature", 1073741826, "AlgorithmIdentifier"},
+  {"issuer", 1073741826, "Name"},
+  {"validity", 1073741826, "Validity"},
+  {"subject", 1073741826, "Name"},
+  {"subjectPublicKeyInfo", 1073741826, "SubjectPublicKeyInfo"},
+  {"issuerUniqueID", 1610637314, "UniqueIdentifier"},
+  {0, 4104, "1"},
+  {"subjectUniqueID", 1610637314, "UniqueIdentifier"},
+  {0, 4104, "2"},
+  {"extensions", 536895490, "Extensions"},
+  {0, 2056, "3"},
+  {"Version", 1610874883, 0},
+  {"v1", 1073741825, "0"},
+  {"v2", 1073741825, "1"},
+  {"v3", 1, "2"},
+  {"CertificateSerialNumber", 1073741827, 0},
+  {"Validity", 1610612741, 0},
+  {"notBefore", 1073741826, "Time"},
+  {"notAfter", 2, "Time"},
+  {"Time", 1610612754, 0},
+  {"utcTime", 1090519057, 0},
+  {"generalTime", 8388625, 0},
+  {"UniqueIdentifier", 1073741830, 0},
+  {"SubjectPublicKeyInfo", 1610612741, 0},
+  {"algorithm", 1073741826, "AlgorithmIdentifier"},
+  {"subjectPublicKey", 6, 0},
+  {"Extensions", 1612709899, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Extension"},
+  {"Extension", 1610612741, 0},
+  {"extnID", 1073741836, 0},
+  {"critical", 1610645508, 0},
+  {0, 131081, 0},
+  {"extnValue", 7, 0},
+  {"CertificateList", 1610612741, 0},
+  {"tbsCertList", 1073741826, "TBSCertList"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"TBSCertList", 1610612741, 0},
+  {"version", 1073758210, "Version"},
+  {"signature", 1073741826, "AlgorithmIdentifier"},
+  {"issuer", 1073741826, "Name"},
+  {"thisUpdate", 1073741826, "Time"},
+  {"nextUpdate", 1073758210, "Time"},
+  {"revokedCertificates", 1610629131, 0},
+  {0, 536870917, 0},
+  {"userCertificate", 1073741826, "CertificateSerialNumber"},
+  {"revocationDate", 1073741826, "Time"},
+  {"crlEntryExtensions", 16386, "Extensions"},
+  {"crlExtensions", 536895490, "Extensions"},
+  {0, 2056, "0"},
+  {"AlgorithmIdentifier", 1610612741, 0},
+  {"algorithm", 1073741836, 0},
+  {"parameters", 541081613, 0},
+  {"algorithm", 1, 0},
+  {"pkcs-1", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "1"},
+  {"rsaEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "1"},
+  {"md2WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "2"},
+  {"md5WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "4"},
+  {"sha1WithRSAEncryption", 1879048204, 0},
+  {0, 1073741825, "pkcs-1"},
+  {0, 1, "5"},
+  {"id-dsa-with-sha1", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9-57", 1073741825, "10040"},
+  {"x9algorithm", 1073741825, "4"},
+  {0, 1, "3"},
+  {"Dss-Sig-Value", 1610612741, 0},
+  {"r", 1073741827, 0},
+  {"s", 3, 0},
+  {"dhpublicnumber", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"ansi-x942", 1073741825, "10046"},
+  {"number-type", 1073741825, "2"},
+  {0, 1, "1"},
+  {"DomainParameters", 1610612741, 0},
+  {"p", 1073741827, 0},
+  {"g", 1073741827, 0},
+  {"q", 1073741827, 0},
+  {"j", 1073758211, 0},
+  {"validationParms", 16386, "ValidationParms"},
+  {"ValidationParms", 1610612741, 0},
+  {"seed", 1073741830, 0},
+  {"pgenCounter", 3, 0},
+  {"id-dsa", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"x9-57", 1073741825, "10040"},
+  {"x9algorithm", 1073741825, "4"},
+  {0, 1, "1"},
+  {"Dss-Parms", 1610612741, 0},
+  {"p", 1073741827, 0},
+  {"q", 1073741827, 0},
+  {"g", 3, 0},
+  {"ORAddress", 1610612741, 0},
+  {"built-in-standard-attributes", 1073741826, "BuiltInStandardAttributes"},
+  {"built-in-domain-defined-attributes", 1073758210,
+   "BuiltInDomainDefinedAttributes"},
+  {"extension-attributes", 16386, "ExtensionAttributes"},
+  {"BuiltInStandardAttributes", 1610612741, 0},
+  {"country-name", 1073758210, "CountryName"},
+  {"administration-domain-name", 1073758210, "AdministrationDomainName"},
+  {"network-address", 1610637314, "NetworkAddress"},
+  {0, 2056, "0"},
+  {"terminal-identifier", 1610637314, "TerminalIdentifier"},
+  {0, 2056, "1"},
+  {"private-domain-name", 1610637314, "PrivateDomainName"},
+  {0, 2056, "2"},
+  {"organization-name", 1610637314, "OrganizationName"},
+  {0, 2056, "3"},
+  {"numeric-user-identifier", 1610637314, "NumericUserIdentifier"},
+  {0, 2056, "4"},
+  {"personal-name", 1610637314, "PersonalName"},
+  {0, 2056, "5"},
+  {"organizational-unit-names", 536895490, "OrganizationalUnitNames"},
+  {0, 2056, "6"},
+  {"CountryName", 1610620946, 0},
+  {0, 1073746952, "1"},
+  {"x121-dcc-code", 1612709890, "NumericString"},
+  {0, 1048586, "ub-country-name-numeric-length"},
+  {"iso-3166-alpha2-code", 538968066, "PrintableString"},
+  {0, 1048586, "ub-country-name-alpha-length"},
+  {"AdministrationDomainName", 1610620946, 0},
+  {0, 1073744904, "2"},
+  {"numeric", 1612709890, "NumericString"},
+  {"ub-domain-name-length", 524298, "0"},
+  {"printable", 538968066, "PrintableString"},
+  {"ub-domain-name-length", 524298, "0"},
+  {"NetworkAddress", 1073741826, "X121Address"},
+  {"X121Address", 1612709890, "NumericString"},
+  {"ub-x121-address-length", 524298, "1"},
+  {"TerminalIdentifier", 1612709890, "PrintableString"},
+  {"ub-terminal-id-length", 524298, "1"},
+  {"PrivateDomainName", 1610612754, 0},
+  {"numeric", 1612709890, "NumericString"},
+  {"ub-domain-name-length", 524298, "1"},
+  {"printable", 538968066, "PrintableString"},
+  {"ub-domain-name-length", 524298, "1"},
+  {"OrganizationName", 1612709890, "PrintableString"},
+  {"ub-organization-name-length", 524298, "1"},
+  {"NumericUserIdentifier", 1612709890, "NumericString"},
+  {"ub-numeric-user-id-length", 524298, "1"},
+  {"PersonalName", 1610612750, 0},
+  {"surname", 1814044674, "PrintableString"},
+  {0, 1073745928, "0"},
+  {"ub-surname-length", 524298, "1"},
+  {"given-name", 1814061058, "PrintableString"},
+  {0, 1073745928, "1"},
+  {"ub-given-name-length", 524298, "1"},
+  {"initials", 1814061058, "PrintableString"},
+  {0, 1073745928, "2"},
+  {"ub-initials-length", 524298, "1"},
+  {"generation-qualifier", 740319234, "PrintableString"},
+  {0, 1073745928, "3"},
+  {"ub-generation-qualifier-length", 524298, "1"},
+  {"OrganizationalUnitNames", 1612709899, 0},
+  {"ub-organizational-units", 1074266122, "1"},
+  {0, 2, "OrganizationalUnitName"},
+  {"OrganizationalUnitName", 1612709890, "PrintableString"},
+  {"ub-organizational-unit-name-length", 524298, "1"},
+  {"BuiltInDomainDefinedAttributes", 1612709899, 0},
+  {"ub-domain-defined-attributes", 1074266122, "1"},
+  {0, 2, "BuiltInDomainDefinedAttribute"},
+  {"BuiltInDomainDefinedAttribute", 1610612741, 0},
+  {"type", 1612709890, "PrintableString"},
+  {"ub-domain-defined-attribute-type-length", 524298, "1"},
+  {"value", 538968066, "PrintableString"},
+  {"ub-domain-defined-attribute-value-length", 524298, "1"},
+  {"ExtensionAttributes", 1612709903, 0},
+  {"ub-extension-attributes", 1074266122, "1"},
+  {0, 2, "ExtensionAttribute"},
+  {"ExtensionAttribute", 1610612741, 0},
+  {"extension-attribute-type", 1611145219, 0},
+  {0, 1073743880, "0"},
+  {"0", 10, "ub-extension-attributes"},
+  {"extension-attribute-value", 541073421, 0},
+  {0, 1073743880, "1"},
+  {"extension-attribute-type", 1, 0},
+  {"common-name", 1342177283, "1"},
+  {"CommonName", 1612709890, "PrintableString"},
+  {"ub-common-name-length", 524298, "1"},
+  {"teletex-common-name", 1342177283, "2"},
+  {"TeletexCommonName", 1612709890, "TeletexString"},
+  {"ub-common-name-length", 524298, "1"},
+  {"teletex-organization-name", 1342177283, "3"},
+  {"TeletexOrganizationName", 1612709890, "TeletexString"},
+  {"ub-organization-name-length", 524298, "1"},
+  {"teletex-personal-name", 1342177283, "4"},
+  {"TeletexPersonalName", 1610612750, 0},
+  {"surname", 1814044674, "TeletexString"},
+  {0, 1073743880, "0"},
+  {"ub-surname-length", 524298, "1"},
+  {"given-name", 1814061058, "TeletexString"},
+  {0, 1073743880, "1"},
+  {"ub-given-name-length", 524298, "1"},
+  {"initials", 1814061058, "TeletexString"},
+  {0, 1073743880, "2"},
+  {"ub-initials-length", 524298, "1"},
+  {"generation-qualifier", 740319234, "TeletexString"},
+  {0, 1073743880, "3"},
+  {"ub-generation-qualifier-length", 524298, "1"},
+  {"teletex-organizational-unit-names", 1342177283, "5"},
+  {"TeletexOrganizationalUnitNames", 1612709899, 0},
+  {"ub-organizational-units", 1074266122, "1"},
+  {0, 2, "TeletexOrganizationalUnitName"},
+  {"TeletexOrganizationalUnitName", 1612709890, "TeletexString"},
+  {"ub-organizational-unit-name-length", 524298, "1"},
+  {"pds-name", 1342177283, "7"},
+  {"PDSName", 1612709890, "PrintableString"},
+  {"ub-pds-name-length", 524298, "1"},
+  {"physical-delivery-country-name", 1342177283, "8"},
+  {"PhysicalDeliveryCountryName", 1610612754, 0},
+  {"x121-dcc-code", 1612709890, "NumericString"},
+  {0, 1048586, "ub-country-name-numeric-length"},
+  {"iso-3166-alpha2-code", 538968066, "PrintableString"},
+  {0, 1048586, "ub-country-name-alpha-length"},
+  {"postal-code", 1342177283, "9"},
+  {"PostalCode", 1610612754, 0},
+  {"numeric-code", 1612709890, "NumericString"},
+  {"ub-postal-code-length", 524298, "1"},
+  {"printable-code", 538968066, "PrintableString"},
+  {"ub-postal-code-length", 524298, "1"},
+  {"physical-delivery-office-name", 1342177283, "10"},
+  {"PhysicalDeliveryOfficeName", 1073741826, "PDSParameter"},
+  {"physical-delivery-office-number", 1342177283, "11"},
+  {"PhysicalDeliveryOfficeNumber", 1073741826, "PDSParameter"},
+  {"extension-OR-address-components", 1342177283, "12"},
+  {"ExtensionORAddressComponents", 1073741826, "PDSParameter"},
+  {"physical-delivery-personal-name", 1342177283, "13"},
+  {"PhysicalDeliveryPersonalName", 1073741826, "PDSParameter"},
+  {"physical-delivery-organization-name", 1342177283, "14"},
+  {"PhysicalDeliveryOrganizationName", 1073741826, "PDSParameter"},
+  {"extension-physical-delivery-address-components", 1342177283, "15"},
+  {"ExtensionPhysicalDeliveryAddressComponents", 1073741826, "PDSParameter"},
+  {"unformatted-postal-address", 1342177283, "16"},
+  {"UnformattedPostalAddress", 1610612750, 0},
+  {"printable-address", 1814052875, 0},
+  {"ub-pds-physical-address-lines", 1074266122, "1"},
+  {0, 538968066, "PrintableString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"teletex-string", 740311042, "TeletexString"},
+  {"ub-unformatted-address-length", 524298, "1"},
+  {"street-address", 1342177283, "17"},
+  {"StreetAddress", 1073741826, "PDSParameter"},
+  {"post-office-box-address", 1342177283, "18"},
+  {"PostOfficeBoxAddress", 1073741826, "PDSParameter"},
+  {"poste-restante-address", 1342177283, "19"},
+  {"PosteRestanteAddress", 1073741826, "PDSParameter"},
+  {"unique-postal-name", 1342177283, "20"},
+  {"UniquePostalName", 1073741826, "PDSParameter"},
+  {"local-postal-attributes", 1342177283, "21"},
+  {"LocalPostalAttributes", 1073741826, "PDSParameter"},
+  {"PDSParameter", 1610612750, 0},
+  {"printable-string", 1814052866, "PrintableString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"teletex-string", 740311042, "TeletexString"},
+  {"ub-pds-parameter-length", 524298, "1"},
+  {"extended-network-address", 1342177283, "22"},
+  {"ExtendedNetworkAddress", 1610612754, 0},
+  {"e163-4-address", 1610612741, 0},
+  {"number", 1612718082, "NumericString"},
+  {0, 1073743880, "0"},
+  {"ub-e163-4-number-length", 524298, "1"},
+  {"sub-address", 538992642, "NumericString"},
+  {0, 1073743880, "1"},
+  {"ub-e163-4-sub-address-length", 524298, "1"},
+  {"psap-address", 536879106, "PresentationAddress"},
+  {0, 2056, "0"},
+  {"PresentationAddress", 1610612741, 0},
+  {"pSelector", 1610637319, 0},
+  {0, 2056, "0"},
+  {"sSelector", 1610637319, 0},
+  {0, 2056, "1"},
+  {"tSelector", 1610637319, 0},
+  {0, 2056, "2"},
+  {"nAddresses", 538976271, 0},
+  {0, 1073743880, "3"},
+  {"MAX", 1074266122, "1"},
+  {0, 7, 0},
+  {"terminal-type", 1342177283, "23"},
+  {"TerminalType", 1610874883, 0},
+  {"telex", 1073741825, "3"},
+  {"teletex", 1073741825, "4"},
+  {"g3-facsimile", 1073741825, "5"},
+  {"g4-facsimile", 1073741825, "6"},
+  {"ia5-terminal", 1073741825, "7"},
+  {"videotex", 1, "8"},
+  {"teletex-domain-defined-attributes", 1342177283, "6"},
+  {"TeletexDomainDefinedAttributes", 1612709899, 0},
+  {"ub-domain-defined-attributes", 1074266122, "1"},
+  {0, 2, "TeletexDomainDefinedAttribute"},
+  {"TeletexDomainDefinedAttribute", 1610612741, 0},
+  {"type", 1612709890, "TeletexString"},
+  {"ub-domain-defined-attribute-type-length", 524298, "1"},
+  {"value", 538968066, "TeletexString"},
+  {"ub-domain-defined-attribute-value-length", 524298, "1"},
+  {"ub-name", 1342177283, "32768"},
+  {"ub-common-name", 1342177283, "64"},
+  {"ub-locality-name", 1342177283, "128"},
+  {"ub-state-name", 1342177283, "128"},
+  {"ub-organization-name", 1342177283, "64"},
+  {"ub-organizational-unit-name", 1342177283, "64"},
+  {"ub-title", 1342177283, "64"},
+  {"ub-match", 1342177283, "128"},
+  {"ub-emailaddress-length", 1342177283, "128"},
+  {"ub-common-name-length", 1342177283, "64"},
+  {"ub-country-name-alpha-length", 1342177283, "2"},
+  {"ub-country-name-numeric-length", 1342177283, "3"},
+  {"ub-domain-defined-attributes", 1342177283, "4"},
+  {"ub-domain-defined-attribute-type-length", 1342177283, "8"},
+  {"ub-domain-defined-attribute-value-length", 1342177283, "128"},
+  {"ub-domain-name-length", 1342177283, "16"},
+  {"ub-extension-attributes", 1342177283, "256"},
+  {"ub-e163-4-number-length", 1342177283, "15"},
+  {"ub-e163-4-sub-address-length", 1342177283, "40"},
+  {"ub-generation-qualifier-length", 1342177283, "3"},
+  {"ub-given-name-length", 1342177283, "16"},
+  {"ub-initials-length", 1342177283, "5"},
+  {"ub-integer-options", 1342177283, "256"},
+  {"ub-numeric-user-id-length", 1342177283, "32"},
+  {"ub-organization-name-length", 1342177283, "64"},
+  {"ub-organizational-unit-name-length", 1342177283, "32"},
+  {"ub-organizational-units", 1342177283, "4"},
+  {"ub-pds-name-length", 1342177283, "16"},
+  {"ub-pds-parameter-length", 1342177283, "30"},
+  {"ub-pds-physical-address-lines", 1342177283, "6"},
+  {"ub-postal-code-length", 1342177283, "16"},
+  {"ub-surname-length", 1342177283, "40"},
+  {"ub-terminal-id-length", 1342177283, "24"},
+  {"ub-unformatted-address-length", 1342177283, "180"},
+  {"ub-x121-address-length", 1342177283, "16"},
+  {"pkcs-7-ContentInfo", 1610612741, 0},
+  {"contentType", 1073741826, "pkcs-7-ContentType"},
+  {"content", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"contentType", 1, 0},
+  {"pkcs-7-DigestInfo", 1610612741, 0},
+  {"digestAlgorithm", 1073741826, "pkcs-7-DigestAlgorithmIdentifier"},
+  {"digest", 2, "pkcs-7-Digest"},
+  {"pkcs-7-Digest", 1073741831, 0},
+  {"pkcs-7-ContentType", 1073741836, 0},
+  {"pkcs-7-SignedData", 1610612741, 0},
+  {"version", 1073741826, "pkcs-7-CMSVersion"},
+  {"digestAlgorithms", 1073741826, "pkcs-7-DigestAlgorithmIdentifiers"},
+  {"encapContentInfo", 1073741826, "pkcs-7-EncapsulatedContentInfo"},
+  {"certificates", 1610637314, "pkcs-7-CertificateSet"},
+  {0, 4104, "0"},
+  {"crls", 1610637314, "pkcs-7-CertificateRevocationLists"},
+  {0, 4104, "1"},
+  {"signerInfos", 2, "pkcs-7-SignerInfos"},
+  {"pkcs-7-CMSVersion", 1610874883, 0},
+  {"v0", 1073741825, "0"},
+  {"v1", 1073741825, "1"},
+  {"v2", 1073741825, "2"},
+  {"v3", 1073741825, "3"},
+  {"v4", 1, "4"},
+  {"pkcs-7-DigestAlgorithmIdentifiers", 1610612751, 0},
+  {0, 2, "pkcs-7-DigestAlgorithmIdentifier"},
+  {"pkcs-7-DigestAlgorithmIdentifier", 1073741826, "AlgorithmIdentifier"},
+  {"pkcs-7-EncapsulatedContentInfo", 1610612741, 0},
+  {"eContentType", 1073741826, "pkcs-7-ContentType"},
+  {"eContent", 536895495, 0},
+  {0, 2056, "0"},
+  {"pkcs-7-CertificateRevocationLists", 1610612751, 0},
+  {0, 13, 0},
+  {"pkcs-7-CertificateChoices", 1610612754, 0},
+  {"certificate", 13, 0},
+  {"pkcs-7-CertificateSet", 1610612751, 0},
+  {0, 2, "pkcs-7-CertificateChoices"},
+  {"pkcs-7-SignerInfos", 1610612751, 0},
+  {0, 13, 0},
+  {"pkcs-10-CertificationRequestInfo", 1610612741, 0},
+  {"version", 1610874883, 0},
+  {"v1", 1, "0"},
+  {"subject", 1073741826, "Name"},
+  {"subjectPKInfo", 1073741826, "SubjectPublicKeyInfo"},
+  {"attributes", 536879106, "Attributes"},
+  {0, 4104, "0"},
+  {"Attributes", 1610612751, 0},
+  {0, 2, "Attribute"},
+  {"pkcs-10-CertificationRequest", 1610612741, 0},
+  {"certificationRequestInfo", 1073741826, "pkcs-10-CertificationRequestInfo"},
+  {"signatureAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"signature", 6, 0},
+  {"pkcs-9-ub-challengePassword", 1342177283, "255"},
+  {"pkcs-9-certTypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "22"},
+  {"pkcs-9-crlTypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "23"},
+  {"pkcs-9-at-challengePassword", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "7"},
+  {"pkcs-9-challengePassword", 1610612754, 0},
+  {"printableString", 1612709890, "PrintableString"},
+  {"pkcs-9-ub-challengePassword", 524298, "1"},
+  {"utf8String", 538968066, "UTF8String"},
+  {"pkcs-9-ub-challengePassword", 524298, "1"},
+  {"pkcs-9-at-localKeyId", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "21"},
+  {"pkcs-9-localKeyId", 1073741831, 0},
+  {"pkcs-9-at-friendlyName", 1879048204, 0},
+  {0, 1073741825, "pkcs-9"},
+  {0, 1, "20"},
+  {"pkcs-9-friendlyName", 1612709890, "BMPString"},
+  {"255", 524298, "1"},
+  {"pkcs-8-PrivateKeyInfo", 1610612741, 0},
+  {"version", 1073741826, "pkcs-8-Version"},
+  {"privateKeyAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"privateKey", 1073741826, "pkcs-8-PrivateKey"},
+  {"attributes", 536895490, "Attributes"},
+  {0, 4104, "0"},
+  {"pkcs-8-Version", 1610874883, 0},
+  {"v1", 1, "0"},
+  {"pkcs-8-PrivateKey", 1073741831, 0},
+  {"pkcs-8-Attributes", 1610612751, 0},
+  {0, 2, "Attribute"},
+  {"pkcs-8-EncryptedPrivateKeyInfo", 1610612741, 0},
+  {"encryptionAlgorithm", 1073741826, "AlgorithmIdentifier"},
+  {"encryptedData", 2, "pkcs-8-EncryptedData"},
+  {"pkcs-8-EncryptedData", 1073741831, 0},
+  {"pkcs-5", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "5"},
+  {"pkcs-5-encryptionAlgorithm", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {0, 1, "3"},
+  {"pkcs-5-des-EDE3-CBC", 1879048204, 0},
+  {0, 1073741825, "pkcs-5-encryptionAlgorithm"},
+  {0, 1, "7"},
+  {"pkcs-5-des-EDE3-CBC-params", 1612709895, 0},
+  {0, 1048586, "8"},
+  {"pkcs-5-id-PBES2", 1879048204, 0},
+  {0, 1073741825, "pkcs-5"},
+  {0, 1, "13"},
+  {"pkcs-5-PBES2-params", 1610612741, 0},
+  {"keyDerivationFunc", 1073741826, "AlgorithmIdentifier"},
+  {"encryptionScheme", 2, "AlgorithmIdentifier"},
+  {"pkcs-5-id-PBKDF2", 1879048204, 0},
+  {0, 1073741825, "pkcs-5"},
+  {0, 1, "12"},
+  {"pkcs-5-PBKDF2-params", 1610612741, 0},
+  {"salt", 1610612754, 0},
+  {"specified", 1073741831, 0},
+  {"otherSource", 2, "AlgorithmIdentifier"},
+  {"iterationCount", 1611137027, 0},
+  {"1", 10, "MAX"},
+  {"keyLength", 1611153411, 0},
+  {"1", 10, "MAX"},
+  {"prf", 16386, "AlgorithmIdentifier"},
+  {"pkcs-12", 1879048204, 0},
+  {0, 1073741825, "pkcs"},
+  {0, 1, "12"},
+  {"pkcs-12-PFX", 1610612741, 0},
+  {"version", 1610874883, 0},
+  {"v3", 1, "3"},
+  {"authSafe", 1073741826, "pkcs-7-ContentInfo"},
+  {"macData", 16386, "pkcs-12-MacData"},
+  {"pkcs-12-PbeParams", 1610612741, 0},
+  {"salt", 1073741831, 0},
+  {"iterations", 3, 0},
+  {"pkcs-12-MacData", 1610612741, 0},
+  {"mac", 1073741826, "pkcs-7-DigestInfo"},
+  {"macSalt", 1073741831, 0},
+  {"iterations", 536903683, 0},
+  {0, 9, "1"},
+  {"pkcs-12-AuthenticatedSafe", 1610612747, 0},
+  {0, 2, "pkcs-7-ContentInfo"},
+  {"pkcs-12-SafeContents", 1610612747, 0},
+  {0, 2, "pkcs-12-SafeBag"},
+  {"pkcs-12-SafeBag", 1610612741, 0},
+  {"bagId", 1073741836, 0},
+  {"bagValue", 1614815245, 0},
+  {0, 1073743880, "0"},
+  {"badId", 1, 0},
+  {"bagAttributes", 536887311, 0},
+  {0, 2, "pkcs-12-PKCS12Attribute"},
+  {"pkcs-12-bagtypes", 1879048204, 0},
+  {0, 1073741825, "pkcs-12"},
+  {0, 1073741825, "10"},
+  {0, 1, "1"},
+  {"pkcs-12-keyBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "1"},
+  {"pkcs-12-pkcs8ShroudedKeyBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "2"},
+  {"pkcs-12-certBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "3"},
+  {"pkcs-12-crlBag", 1879048204, 0},
+  {0, 1073741825, "pkcs-12-bagtypes"},
+  {0, 1, "4"},
+  {"pkcs-12-KeyBag", 1073741826, "pkcs-8-PrivateKeyInfo"},
+  {"pkcs-12-PKCS8ShroudedKeyBag", 1073741826, "pkcs-8-EncryptedPrivateKeyInfo"},
+  {"pkcs-12-CertBag", 1610612741, 0},
+  {"certId", 1073741836, 0},
+  {"certValue", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"certId", 1, 0},
+  {"pkcs-12-CRLBag", 1610612741, 0},
+  {"crlId", 1073741836, 0},
+  {"crlValue", 541073421, 0},
+  {0, 1073743880, "0"},
+  {"crlId", 1, 0},
+  {"pkcs-12-PKCS12Attribute", 1073741826, "Attribute"},
+  {"pkcs-7-data", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1073741825, "1"},
+  {"pkcs7", 1073741825, "7"},
+  {0, 1, "1"},
+  {"pkcs-7-encryptedData", 1879048204, 0},
+  {"iso", 1073741825, "1"},
+  {"member-body", 1073741825, "2"},
+  {"us", 1073741825, "840"},
+  {"rsadsi", 1073741825, "113549"},
+  {"pkcs", 1073741825, "1"},
+  {"pkcs7", 1073741825, "7"},
+  {0, 1, "6"},
+  {"pkcs-7-Data", 1073741831, 0},
+  {"pkcs-7-EncryptedData", 1610612741, 0},
+  {"version", 1073741826, "pkcs-7-CMSVersion"},
+  {"encryptedContentInfo", 1073741826, "pkcs-7-EncryptedContentInfo"},
+  {"unprotectedAttrs", 536895490, "pkcs-7-UnprotectedAttributes"},
+  {0, 4104, "1"},
+  {"pkcs-7-EncryptedContentInfo", 1610612741, 0},
+  {"contentType", 1073741826, "pkcs-7-ContentType"},
+  {"contentEncryptionAlgorithm", 1073741826,
+   "pkcs-7-ContentEncryptionAlgorithmIdentifier"},
+  {"encryptedContent", 536895490, "pkcs-7-EncryptedContent"},
+  {0, 4104, "0"},
+  {"pkcs-7-ContentEncryptionAlgorithmIdentifier", 1073741826,
+   "AlgorithmIdentifier"},
+  {"pkcs-7-EncryptedContent", 1073741831, 0},
+  {"pkcs-7-UnprotectedAttributes", 1612709903, 0},
+  {"MAX", 1074266122, "1"},
+  {0, 2, "Attribute"},
+  {"id-at-ldap-DC", 1880096780, "AttributeType"},
+  {0, 1073741825, "0"},
+  {0, 1073741825, "9"},
+  {0, 1073741825, "2342"},
+  {0, 1073741825, "19200300"},
+  {0, 1073741825, "100"},
+  {0, 1073741825, "1"},
+  {0, 1, "25"},
+  {"ldap-DC", 1073741826, "IA5String"},
+  {"id-at-ldap-UID", 1880096780, "AttributeType"},
+  {0, 1073741825, "0"},
+  {0, 1073741825, "9"},
+  {0, 1073741825, "2342"},
+  {0, 1073741825, "19200300"},
+  {0, 1073741825, "100"},
+  {0, 1073741825, "1"},
+  {0, 1, "1"},
+  {"ldap-UID", 1073741826, "DirectoryString"},
+  {"id-pda", 1879048204, 0},
+  {0, 1073741825, "id-pkix"},
+  {0, 1, "9"},
+  {"id-pda-dateOfBirth", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "1"},
+  {"DateOfBirth", 1082130449, 0},
+  {"id-pda-placeOfBirth", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "2"},
+  {"PlaceOfBirth", 1073741826, "DirectoryString"},
+  {"id-pda-gender", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "3"},
+  {"Gender", 1612709890, "PrintableString"},
+  {0, 1048586, "1"},
+  {"id-pda-countryOfCitizenship", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "4"},
+  {"CountryOfCitizenship", 1612709890, "PrintableString"},
+  {0, 1048586, "2"},
+  {"id-pda-countryOfResidence", 1880096780, "AttributeType"},
+  {0, 1073741825, "id-pda"},
+  {0, 1, "5"},
+  {"CountryOfResidence", 538968066, "PrintableString"},
+  {0, 1048586, "2"},
+  {0, 0, 0}
+};
diff --git a/tests/postcopy-test.c b/tests/postcopy-test.c
new file mode 100644
index 0000000..229e9e9
--- /dev/null
+++ b/tests/postcopy-test.c
@@ -0,0 +1,530 @@
+/*
+ * QTest testcase for postcopy
+ *
+ * Copyright (c) 2016 Red Hat, Inc. and/or its affiliates
+ *   based on the vhost-user-test.c that is:
+ *      Copyright (c) 2014 Virtual Open Systems Sarl.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "libqtest.h"
+#include "qemu/option.h"
+#include "qemu/range.h"
+#include "qemu/sockets.h"
+#include "sysemu/char.h"
+#include "sysemu/sysemu.h"
+#include "hw/nvram/openbios_firmware_abi.h"
+
+#define MIN_NVRAM_SIZE 8192 /* from spapr_nvram.c */
+
+const unsigned start_address = 1024 * 1024;
+const unsigned end_address = 100 * 1024 * 1024;
+bool got_stop;
+
+#if defined(__linux__)
+#include <sys/syscall.h>
+#include <sys/vfs.h>
+#endif
+
+#if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD)
+#include <sys/eventfd.h>
+#include <sys/ioctl.h>
+#include <linux/userfaultfd.h>
+
+static bool ufd_version_check(void)
+{
+    struct uffdio_api api_struct;
+    uint64_t ioctl_mask;
+
+    int ufd = ufd = syscall(__NR_userfaultfd, O_CLOEXEC);
+
+    if (ufd == -1) {
+        g_test_message("Skipping test: userfaultfd not available");
+        return false;
+    }
+
+    api_struct.api = UFFD_API;
+    api_struct.features = 0;
+    if (ioctl(ufd, UFFDIO_API, &api_struct)) {
+        g_test_message("Skipping test: UFFDIO_API failed");
+        return false;
+    }
+
+    ioctl_mask = (__u64)1 << _UFFDIO_REGISTER |
+                 (__u64)1 << _UFFDIO_UNREGISTER;
+    if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) {
+        g_test_message("Skipping test: Missing userfault feature");
+        return false;
+    }
+
+    return true;
+}
+
+#else
+static bool ufd_version_check(void)
+{
+    g_test_message("Skipping test: Userfault not available (builtdtime)");
+    return false;
+}
+
+#endif
+
+static const char *tmpfs;
+
+/* A simple PC boot sector that modifies memory (1-100MB) quickly
+ * outputing a 'B' every so often if it's still running.
+ */
+unsigned char bootsect[] = {
+  0xfa, 0x0f, 0x01, 0x16, 0x74, 0x7c, 0x66, 0xb8, 0x01, 0x00, 0x00, 0x00,
+  0x0f, 0x22, 0xc0, 0x66, 0xea, 0x20, 0x7c, 0x00, 0x00, 0x08, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x92, 0x0c, 0x02,
+  0xe6, 0x92, 0xb8, 0x10, 0x00, 0x00, 0x00, 0x8e, 0xd8, 0x66, 0xb8, 0x41,
+  0x00, 0x66, 0xba, 0xf8, 0x03, 0xee, 0xb3, 0x00, 0xb8, 0x00, 0x00, 0x10,
+  0x00, 0xfe, 0x00, 0x05, 0x00, 0x10, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x40,
+  0x06, 0x7c, 0xf2, 0xfe, 0xc3, 0x75, 0xe9, 0x66, 0xb8, 0x42, 0x00, 0x66,
+  0xba, 0xf8, 0x03, 0xee, 0xeb, 0xde, 0x66, 0x90, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x9a, 0xcf, 0x00,
+  0xff, 0xff, 0x00, 0x00, 0x00, 0x92, 0xcf, 0x00, 0x27, 0x00, 0x5c, 0x7c,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0xaa
+};
+
+static void init_bootfile_x86(const char *bootpath)
+{
+    FILE *bootfile = fopen(bootpath, "wb");
+
+    g_assert_cmpint(fwrite(bootsect, 512, 1, bootfile), ==, 1);
+    fclose(bootfile);
+}
+
+static void init_bootfile_ppc(const char *bootpath)
+{
+    FILE *bootfile;
+    char buf[MIN_NVRAM_SIZE];
+    struct OpenBIOS_nvpart_v1 *header = (struct OpenBIOS_nvpart_v1 *)buf;
+
+    memset(buf, 0, MIN_NVRAM_SIZE);
+
+    /* Create a "common" partition in nvram to store boot-command property */
+
+    header->signature = OPENBIOS_PART_SYSTEM;
+    memcpy(header->name, "common", 6);
+    OpenBIOS_finish_partition(header, MIN_NVRAM_SIZE);
+
+    /* FW_MAX_SIZE is 4MB, but slof.bin is only 900KB,
+     * so let's modify memory between 1MB and 100MB
+     * to do like PC bootsector
+     */
+
+    sprintf(buf + 16,
+            "boot-command=hex .\" _\" begin %x %x do i c@ 1 + i c! 1000 +loop "
+            ".\" B\" 0 until", end_address, start_address);
+
+    /* Write partition to the NVRAM file */
+
+    bootfile = fopen(bootpath, "wb");
+    g_assert_cmpint(fwrite(buf, MIN_NVRAM_SIZE, 1, bootfile), ==, 1);
+    fclose(bootfile);
+}
+
+/*
+ * Wait for some output in the serial output file,
+ * we get an 'A' followed by an endless string of 'B's
+ * but on the destination we won't have the A.
+ */
+static void wait_for_serial(const char *side)
+{
+    char *serialpath = g_strdup_printf("%s/%s", tmpfs, side);
+    FILE *serialfile = fopen(serialpath, "r");
+    const char *arch = qtest_get_arch();
+    int started = (strcmp(side, "src_serial") == 0 &&
+                   strcmp(arch, "ppc64") == 0) ? 0 : 1;
+
+    do {
+        int readvalue = fgetc(serialfile);
+
+        if (!started) {
+            /* SLOF prints its banner before starting test,
+             * to ignore it, mark the start of the test with '_',
+             * ignore all characters until this marker
+             */
+            switch (readvalue) {
+            case '_':
+                started = 1;
+                break;
+            case EOF:
+                fseek(serialfile, 0, SEEK_SET);
+                usleep(1000);
+                break;
+            }
+            continue;
+        }
+        switch (readvalue) {
+        case 'A':
+            /* Fine */
+            break;
+
+        case 'B':
+            /* It's alive! */
+            fclose(serialfile);
+            g_free(serialpath);
+            return;
+
+        case EOF:
+            started = (strcmp(side, "src_serial") == 0 &&
+                       strcmp(arch, "ppc64") == 0) ? 0 : 1;
+            fseek(serialfile, 0, SEEK_SET);
+            usleep(1000);
+            break;
+
+        default:
+            fprintf(stderr, "Unexpected %d on %s serial\n", readvalue, side);
+            g_assert_not_reached();
+        }
+    } while (true);
+}
+
+/*
+ * Events can get in the way of responses we are actually waiting for.
+ */
+static QDict *return_or_event(QDict *response)
+{
+    const char *event_string;
+    if (!qdict_haskey(response, "event")) {
+        return response;
+    }
+
+    /* OK, it was an event */
+    event_string = qdict_get_str(response, "event");
+    if (!strcmp(event_string, "STOP")) {
+        got_stop = true;
+    }
+    QDECREF(response);
+    return return_or_event(qtest_qmp_receive(global_qtest));
+}
+
+
+/*
+ * It's tricky to use qemu's migration event capability with qtest,
+ * events suddenly appearing confuse the qmp()/hmp() responses.
+ * so wait for a couple of passes to have happened before
+ * going postcopy.
+ */
+
+static uint64_t get_migration_pass(void)
+{
+    QDict *rsp, *rsp_return, *rsp_ram;
+    uint64_t result;
+
+    rsp = return_or_event(qmp("{ 'execute': 'query-migrate' }"));
+    rsp_return = qdict_get_qdict(rsp, "return");
+    if (!qdict_haskey(rsp_return, "ram")) {
+        /* Still in setup */
+        result = 0;
+    } else {
+        rsp_ram = qdict_get_qdict(rsp_return, "ram");
+        result = qdict_get_try_int(rsp_ram, "dirty-sync-count", 0);
+        QDECREF(rsp);
+    }
+    return result;
+}
+
+static void wait_for_migration_complete(void)
+{
+    QDict *rsp, *rsp_return;
+    bool completed;
+
+    do {
+        const char *status;
+
+        rsp = return_or_event(qmp("{ 'execute': 'query-migrate' }"));
+        rsp_return = qdict_get_qdict(rsp, "return");
+        status = qdict_get_str(rsp_return, "status");
+        completed = strcmp(status, "completed") == 0;
+        g_assert_cmpstr(status, !=,  "failed");
+        QDECREF(rsp);
+        usleep(1000 * 100);
+    } while (!completed);
+}
+
+static void wait_for_migration_pass(void)
+{
+    uint64_t initial_pass = get_migration_pass();
+    uint64_t pass;
+
+    /* Wait for the 1st sync */
+    do {
+        initial_pass = get_migration_pass();
+        if (got_stop || initial_pass) {
+            break;
+        }
+        usleep(1000 * 100);
+    } while (true);
+
+    do {
+        usleep(1000 * 100);
+        pass = get_migration_pass();
+    } while (pass == initial_pass && !got_stop);
+}
+
+static void check_guests_ram(void)
+{
+    /* Our ASM test will have been incrementing one byte from each page from
+     * 1MB to <100MB in order.
+     * This gives us a constraint that any page's byte should be equal or less
+     * than the previous pages byte (mod 256); and they should all be equal
+     * except for one transition at the point where we meet the incrementer.
+     * (We're running this with the guest stopped).
+     */
+    unsigned address;
+    uint8_t first_byte;
+    uint8_t last_byte;
+    bool hit_edge = false;
+    bool bad = false;
+
+    qtest_memread(global_qtest, start_address, &first_byte, 1);
+    last_byte = first_byte;
+
+    for (address = start_address + 4096; address < end_address; address += 4096)
+    {
+        uint8_t b;
+        qtest_memread(global_qtest, address, &b, 1);
+        if (b != last_byte) {
+            if (((b + 1) % 256) == last_byte && !hit_edge) {
+                /* This is OK, the guest stopped at the point of
+                 * incrementing the previous page but didn't get
+                 * to us yet.
+                 */
+                hit_edge = true;
+            } else {
+                fprintf(stderr, "Memory content inconsistency at %x"
+                                " first_byte = %x last_byte = %x current = %x"
+                                " hit_edge = %x\n",
+                                address, first_byte, last_byte, b, hit_edge);
+                bad = true;
+            }
+        }
+        last_byte = b;
+    }
+    g_assert_false(bad);
+}
+
+static void cleanup(const char *filename)
+{
+    char *path = g_strdup_printf("%s/%s", tmpfs, filename);
+
+    unlink(path);
+}
+
+static void test_migrate(void)
+{
+    char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
+    QTestState *global = global_qtest, *from, *to;
+    unsigned char dest_byte_a, dest_byte_b, dest_byte_c, dest_byte_d;
+    gchar *cmd, *cmd_src, *cmd_dst;
+    QDict *rsp;
+
+    char *bootpath = g_strdup_printf("%s/bootsect", tmpfs);
+    const char *arch = qtest_get_arch();
+
+    got_stop = false;
+
+    if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
+        init_bootfile_x86(bootpath);
+        cmd_src = g_strdup_printf("-machine accel=kvm:tcg -m 150M"
+                                  " -name pcsource,debug-threads=on"
+                                  " -serial file:%s/src_serial"
+                                  " -drive file=%s,format=raw",
+                                  tmpfs, bootpath);
+        cmd_dst = g_strdup_printf("-machine accel=kvm:tcg -m 150M"
+                                  " -name pcdest,debug-threads=on"
+                                  " -serial file:%s/dest_serial"
+                                  " -drive file=%s,format=raw"
+                                  " -incoming %s",
+                                  tmpfs, bootpath, uri);
+    } else if (strcmp(arch, "ppc64") == 0) {
+        init_bootfile_ppc(bootpath);
+        cmd_src = g_strdup_printf("-machine accel=kvm:tcg -m 256M"
+                                  " -name pcsource,debug-threads=on"
+                                  " -serial file:%s/src_serial"
+                                  " -drive file=%s,if=pflash,format=raw",
+                                  tmpfs, bootpath);
+        cmd_dst = g_strdup_printf("-machine accel=kvm:tcg -m 256M"
+                                  " -name pcdest,debug-threads=on"
+                                  " -serial file:%s/dest_serial"
+                                  " -incoming %s",
+                                  tmpfs, uri);
+    } else {
+        g_assert_not_reached();
+    }
+
+    from = qtest_start(cmd_src);
+    g_free(cmd_src);
+
+    to = qtest_init(cmd_dst);
+    g_free(cmd_dst);
+
+    global_qtest = from;
+    rsp = qmp("{ 'execute': 'migrate-set-capabilities',"
+                  "'arguments': { "
+                      "'capabilities': [ {"
+                          "'capability': 'postcopy-ram',"
+                          "'state': true } ] } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    global_qtest = to;
+    rsp = qmp("{ 'execute': 'migrate-set-capabilities',"
+                  "'arguments': { "
+                      "'capabilities': [ {"
+                          "'capability': 'postcopy-ram',"
+                          "'state': true } ] } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    /* We want to pick a speed slow enough that the test completes
+     * quickly, but that it doesn't complete precopy even on a slow
+     * machine, so also set the downtime.
+     */
+    global_qtest = from;
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 100000000 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    /* 1ms downtime - it should never finish precopy */
+    rsp = qmp("{ 'execute': 'migrate_set_downtime',"
+              "'arguments': { 'value': 0.001 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+
+    /* Wait for the first serial output from the source */
+    wait_for_serial("src_serial");
+
+    cmd = g_strdup_printf("{ 'execute': 'migrate',"
+                          "'arguments': { 'uri': '%s' } }",
+                          uri);
+    rsp = qmp(cmd);
+    g_free(cmd);
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    wait_for_migration_pass();
+
+    rsp = return_or_event(qmp("{ 'execute': 'migrate-start-postcopy' }"));
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    if (!got_stop) {
+        qmp_eventwait("STOP");
+    }
+
+    global_qtest = to;
+    qmp_eventwait("RESUME");
+
+    wait_for_serial("dest_serial");
+    global_qtest = from;
+    wait_for_migration_complete();
+
+    qtest_quit(from);
+
+    global_qtest = to;
+
+    qtest_memread(to, start_address, &dest_byte_a, 1);
+
+    /* Destination still running, wait for a byte to change */
+    do {
+        qtest_memread(to, start_address, &dest_byte_b, 1);
+        usleep(10 * 1000);
+    } while (dest_byte_a == dest_byte_b);
+
+    qmp("{ 'execute' : 'stop'}");
+    /* With it stopped, check nothing changes */
+    qtest_memread(to, start_address, &dest_byte_c, 1);
+    sleep(1);
+    qtest_memread(to, start_address, &dest_byte_d, 1);
+    g_assert_cmpint(dest_byte_c, ==, dest_byte_d);
+
+    check_guests_ram();
+
+    qtest_quit(to);
+    g_free(uri);
+
+    global_qtest = global;
+
+    cleanup("bootsect");
+    cleanup("migsocket");
+    cleanup("src_serial");
+    cleanup("dest_serial");
+}
+
+int main(int argc, char **argv)
+{
+    char template[] = "/tmp/postcopy-test-XXXXXX";
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    if (!ufd_version_check()) {
+        return 0;
+    }
+
+    tmpfs = mkdtemp(template);
+    if (!tmpfs) {
+        g_test_message("mkdtemp on path (%s): %s\n", template, strerror(errno));
+    }
+    g_assert(tmpfs);
+
+    module_call_init(MODULE_INIT_QOM);
+
+    qtest_add_func("/postcopy", test_migrate);
+
+    ret = g_test_run();
+
+    g_assert_cmpint(ret, ==, 0);
+
+    ret = rmdir(tmpfs);
+    if (ret != 0) {
+        g_test_message("unable to rmdir: path (%s): %s\n",
+                       tmpfs, strerror(errno));
+    }
+
+    return ret;
+}
diff --git a/tests/prom-env-test.c b/tests/prom-env-test.c
new file mode 100644
index 0000000..7a62857
--- /dev/null
+++ b/tests/prom-env-test.c
@@ -0,0 +1,90 @@
+/*
+ * Test OpenBIOS-based machines.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Author:
+ *    Thomas Huth <thuth@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2
+ * or later. See the COPYING file in the top-level directory.
+ *
+ * This test is used to check that some OpenBIOS machines can be started
+ * successfully in TCG mode. To do this, we first put some Forth code into
+ * the "boot-command" Open Firmware environment variable. This Forth code
+ * writes a well-known magic value to a known location in memory. Then we
+ * start the guest so that OpenBIOS can boot and finally run the Forth code.
+ * The testing code here then can finally check whether the value has been
+ * successfully written into the guest memory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+
+#define MAGIC   0xcafec0de
+#define ADDRESS 0x4000
+
+static void check_guest_memory(void)
+{
+    uint32_t signature;
+    int i;
+
+    /* Poll until code has run and modified memory. Wait at most 30 seconds */
+    for (i = 0; i < 10000; ++i) {
+        signature = readl(ADDRESS);
+        if (signature == MAGIC) {
+            break;
+        }
+        g_usleep(10000);
+    }
+
+    g_assert_cmphex(signature, ==, MAGIC);
+}
+
+static void test_machine(const void *machine)
+{
+    char *args;
+
+    args = g_strdup_printf("-M %s,accel=tcg -prom-env 'boot-command=%x %x l!'",
+                           (const char *)machine, MAGIC, ADDRESS);
+
+    qtest_start(args);
+    check_guest_memory();
+    qtest_quit(global_qtest);
+
+    g_free(args);
+}
+
+static void add_tests(const char *machines[])
+{
+    int i;
+    char *name;
+
+    for (i = 0; machines[i] != NULL; i++) {
+        name = g_strdup_printf("prom-env/%s", machines[i]);
+        qtest_add_data_func(name, machines[i], test_machine);
+        g_free(name);
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    const char *sparc_machines[] = { "SPARCbook", "Voyager", "SS-20", NULL };
+    const char *sparc64_machines[] = { "sun4u", "sun4v", NULL };
+    const char *mac_machines[] = { "mac99", "g3beige", NULL };
+    const char *arch = qtest_get_arch();
+
+    g_test_init(&argc, &argv, NULL);
+
+    if (!strcmp(arch, "ppc") || !strcmp(arch, "ppc64")) {
+        add_tests(mac_machines);
+    } else if (!strcmp(arch, "sparc")) {
+        add_tests(sparc_machines);
+    } else if (!strcmp(arch, "sparc64")) {
+        add_tests(sparc64_machines);
+    } else {
+        g_assert_not_reached();
+    }
+
+    return g_test_run();
+}
diff --git a/tests/pvpanic-test.c b/tests/pvpanic-test.c
index a7ad6b3..3bfa678 100644
--- a/tests/pvpanic-test.c
+++ b/tests/pvpanic-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 static void test_panic(void)
 {
diff --git a/tests/pxe-test.c b/tests/pxe-test.c
new file mode 100644
index 0000000..b2cc355
--- /dev/null
+++ b/tests/pxe-test.c
@@ -0,0 +1,67 @@
+/*
+ * PXE test cases.
+ *
+ * Copyright (c) 2016 Red Hat Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>,
+ *  Victor Kaplansky <victork@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib/gstdio.h>
+#include "qemu-common.h"
+#include "libqtest.h"
+#include "boot-sector.h"
+
+#define NETNAME "net0"
+
+static const char *disk = "tests/pxe-test-disk.raw";
+
+static void test_pxe_one(const char *params)
+{
+    char *args;
+
+    args = g_strdup_printf("-machine accel=tcg "
+                           "-netdev user,id=" NETNAME ",tftp=./,bootfile=%s "
+                           "%s ",
+                           disk, params);
+
+    qtest_start(args);
+    boot_sector_test();
+    qtest_quit(global_qtest);
+    g_free(args);
+}
+
+static void test_pxe_e1000(void)
+{
+    test_pxe_one("-device e1000,netdev=" NETNAME);
+}
+
+static void test_pxe_virtio_pci(void)
+{
+    test_pxe_one("-device virtio-net-pci,netdev=" NETNAME);
+}
+
+int main(int argc, char *argv[])
+{
+    int ret;
+    const char *arch = qtest_get_arch();
+
+    ret = boot_sector_init(disk);
+    if(ret)
+        return ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
+        qtest_add_func("pxe/e1000", test_pxe_e1000);
+        qtest_add_func("pxe/virtio", test_pxe_virtio_pci);
+    }
+    ret = g_test_run();
+    boot_sector_cleanup(disk);
+    return ret;
+}
diff --git a/tests/q35-test.c b/tests/q35-test.c
new file mode 100644
index 0000000..71538fc
--- /dev/null
+++ b/tests/q35-test.c
@@ -0,0 +1,89 @@
+/*
+ * QTest testcase for Q35 northbridge
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Author: Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "libqos/pci.h"
+#include "libqos/pci-pc.h"
+#include "hw/pci-host/q35.h"
+
+static void smram_set_bit(QPCIDevice *pcidev, uint8_t mask, bool enabled)
+{
+    uint8_t smram;
+
+    smram = qpci_config_readb(pcidev, MCH_HOST_BRIDGE_SMRAM);
+    if (enabled) {
+        smram |= mask;
+    } else {
+        smram &= ~mask;
+    }
+    qpci_config_writeb(pcidev, MCH_HOST_BRIDGE_SMRAM, smram);
+}
+
+static bool smram_test_bit(QPCIDevice *pcidev, uint8_t mask)
+{
+    uint8_t smram;
+
+    smram = qpci_config_readb(pcidev, MCH_HOST_BRIDGE_SMRAM);
+    return smram & mask;
+}
+
+static void test_smram_lock(void)
+{
+    QPCIBus *pcibus;
+    QPCIDevice *pcidev;
+    QDict *response;
+
+    pcibus = qpci_init_pc();
+    g_assert(pcibus != NULL);
+
+    pcidev = qpci_device_find(pcibus, 0);
+    g_assert(pcidev != NULL);
+
+    /* check open is settable */
+    smram_set_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN, false);
+    g_assert(smram_test_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN) == false);
+    smram_set_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN, true);
+    g_assert(smram_test_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN) == true);
+
+    /* lock, check open is cleared & not settable */
+    smram_set_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_LCK, true);
+    g_assert(smram_test_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN) == false);
+    smram_set_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN, true);
+    g_assert(smram_test_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN) == false);
+
+    /* reset */
+    response = qmp("{'execute': 'system_reset', 'arguments': {} }");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* check open is settable again */
+    smram_set_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN, false);
+    g_assert(smram_test_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN) == false);
+    smram_set_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN, true);
+    g_assert(smram_test_bit(pcidev, MCH_HOST_BRIDGE_SMRAM_D_OPEN) == true);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/q35/smram/lock", test_smram_lock);
+
+    qtest_start("-M q35");
+    ret = g_test_run();
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/qapi-schema/alternate-any.err b/tests/qapi-schema/alternate-any.err
new file mode 100644
index 0000000..aaa0154
--- /dev/null
+++ b/tests/qapi-schema/alternate-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-any.json:2: Alternate 'Alt' member 'one' cannot use type 'any'
diff --git a/tests/qapi-schema/alternate-any.exit b/tests/qapi-schema/alternate-any.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-any.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-any.json b/tests/qapi-schema/alternate-any.json
new file mode 100644
index 0000000..e47a73a
--- /dev/null
+++ b/tests/qapi-schema/alternate-any.json
@@ -0,0 +1,4 @@
+# we do not allow the 'any' type as an alternate branch
+{ 'alternate': 'Alt',
+  'data': { 'one': 'any',
+            'two': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-any.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-any.out
diff --git a/tests/qapi-schema/alternate-array.err b/tests/qapi-schema/alternate-array.err
new file mode 100644
index 0000000..7b930c6
--- /dev/null
+++ b/tests/qapi-schema/alternate-array.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-array.json:5: Member 'two' of alternate 'Alt' cannot be an array
diff --git a/tests/qapi-schema/alternate-array.exit b/tests/qapi-schema/alternate-array.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-array.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-array.json b/tests/qapi-schema/alternate-array.json
new file mode 100644
index 0000000..f241aac
--- /dev/null
+++ b/tests/qapi-schema/alternate-array.json
@@ -0,0 +1,7 @@
+# we do not allow array branches in alternates
+# TODO: should we support this?
+{ 'struct': 'One',
+  'data': { 'name': 'str' } }
+{ 'alternate': 'Alt',
+  'data': { 'one': 'One',
+            'two': [ 'int' ] } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-array.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-array.out
diff --git a/tests/qapi-schema/alternate-base.err b/tests/qapi-schema/alternate-base.err
new file mode 100644
index 0000000..30d8a34
--- /dev/null
+++ b/tests/qapi-schema/alternate-base.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-base.json:4: Unknown key 'base' in alternate 'Alt'
diff --git a/tests/qapi-schema/alternate-base.exit b/tests/qapi-schema/alternate-base.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-base.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-base.json b/tests/qapi-schema/alternate-base.json
new file mode 100644
index 0000000..529430ec
--- /dev/null
+++ b/tests/qapi-schema/alternate-base.json
@@ -0,0 +1,6 @@
+# we reject alternate with base type
+{ 'struct': 'Base',
+  'data': { 'string': 'str' } }
+{ 'alternate': 'Alt',
+  'base': 'Base',
+  'data': { 'number': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-base.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-base.out
diff --git a/tests/qapi-schema/alternate-clash.err b/tests/qapi-schema/alternate-clash.err
new file mode 100644
index 0000000..604d849
--- /dev/null
+++ b/tests/qapi-schema/alternate-clash.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-clash.json:7: 'a_b' (branch of Alt1) collides with 'a-b' (branch of Alt1)
diff --git a/tests/qapi-schema/alternate-clash.exit b/tests/qapi-schema/alternate-clash.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-clash.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-clash.json b/tests/qapi-schema/alternate-clash.json
new file mode 100644
index 0000000..6d73bc5
--- /dev/null
+++ b/tests/qapi-schema/alternate-clash.json
@@ -0,0 +1,8 @@
+# Alternate branch name collision
+# Reject an alternate that would result in a collision in generated C
+# names (this would try to generate two enum values 'ALT1_KIND_A_B').
+# TODO: In the future, if alternates are simplified to not generate
+# the implicit Alt1Kind enum, we would still have a collision with the
+# resulting C union trying to have two members named 'a_b'.
+{ 'alternate': 'Alt1',
+  'data': { 'a-b': 'str', 'a_b': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-clash.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-clash.out
diff --git a/tests/qapi-schema/alternate-conflict-dict.err b/tests/qapi-schema/alternate-conflict-dict.err
new file mode 100644
index 0000000..0f411f4
--- /dev/null
+++ b/tests/qapi-schema/alternate-conflict-dict.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-conflict-dict.json:6: Alternate 'Alt' member 'two' can't be distinguished from member 'one'
diff --git a/tests/qapi-schema/alternate-conflict-dict.exit b/tests/qapi-schema/alternate-conflict-dict.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-conflict-dict.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-conflict-dict.json b/tests/qapi-schema/alternate-conflict-dict.json
new file mode 100644
index 0000000..d566cca
--- /dev/null
+++ b/tests/qapi-schema/alternate-conflict-dict.json
@@ -0,0 +1,8 @@
+# we reject alternates with multiple object branches
+{ 'struct': 'One',
+  'data': { 'name': 'str' } }
+{ 'struct': 'Two',
+  'data': { 'value': 'int' } }
+{ 'alternate': 'Alt',
+  'data': { 'one': 'One',
+            'two': 'Two' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-conflict-dict.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-conflict-dict.out
diff --git a/tests/qapi-schema/alternate-conflict-string.err b/tests/qapi-schema/alternate-conflict-string.err
new file mode 100644
index 0000000..fc523b0
--- /dev/null
+++ b/tests/qapi-schema/alternate-conflict-string.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-conflict-string.json:4: Alternate 'Alt' member 'two' can't be distinguished from member 'one'
diff --git a/tests/qapi-schema/alternate-conflict-string.exit b/tests/qapi-schema/alternate-conflict-string.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-conflict-string.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-conflict-string.json b/tests/qapi-schema/alternate-conflict-string.json
new file mode 100644
index 0000000..72f04a8
--- /dev/null
+++ b/tests/qapi-schema/alternate-conflict-string.json
@@ -0,0 +1,6 @@
+# we reject alternates with multiple string-like branches
+{ 'enum': 'Enum',
+  'data': [ 'hello', 'world' ] }
+{ 'alternate': 'Alt',
+  'data': { 'one': 'str',
+            'two': 'Enum' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-conflict-string.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-conflict-string.out
diff --git a/tests/qapi-schema/alternate-empty.err b/tests/qapi-schema/alternate-empty.err
new file mode 100644
index 0000000..bb06c5b
--- /dev/null
+++ b/tests/qapi-schema/alternate-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-empty.json:2: Alternate 'Alt' should have at least two branches in 'data'
diff --git a/tests/qapi-schema/alternate-empty.exit b/tests/qapi-schema/alternate-empty.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-empty.json b/tests/qapi-schema/alternate-empty.json
new file mode 100644
index 0000000..fff15ba
--- /dev/null
+++ b/tests/qapi-schema/alternate-empty.json
@@ -0,0 +1,2 @@
+# alternates must list at least two types to be useful
+{ 'alternate': 'Alt', 'data': { 'i': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-empty.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-empty.out
diff --git a/tests/qapi-schema/alternate-nested.err b/tests/qapi-schema/alternate-nested.err
new file mode 100644
index 0000000..4d1187e
--- /dev/null
+++ b/tests/qapi-schema/alternate-nested.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-nested.json:4: Member 'nested' of alternate 'Alt2' cannot use alternate type 'Alt1'
diff --git a/tests/qapi-schema/alternate-nested.exit b/tests/qapi-schema/alternate-nested.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-nested.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-nested.json b/tests/qapi-schema/alternate-nested.json
new file mode 100644
index 0000000..8e22186
--- /dev/null
+++ b/tests/qapi-schema/alternate-nested.json
@@ -0,0 +1,5 @@
+# we reject a nested alternate branch
+{ 'alternate': 'Alt1',
+  'data': { 'name': 'str', 'value': 'int' } }
+{ 'alternate': 'Alt2',
+  'data': { 'nested': 'Alt1', 'b': 'bool' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-nested.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-nested.out
diff --git a/tests/qapi-schema/alternate-unknown.err b/tests/qapi-schema/alternate-unknown.err
new file mode 100644
index 0000000..dea45dc
--- /dev/null
+++ b/tests/qapi-schema/alternate-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/alternate-unknown.json:2: Member 'unknown' of alternate 'Alt' uses unknown type 'MissingType'
diff --git a/tests/qapi-schema/alternate-unknown.exit b/tests/qapi-schema/alternate-unknown.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/alternate-unknown.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/alternate-unknown.json b/tests/qapi-schema/alternate-unknown.json
new file mode 100644
index 0000000..08c80dc
--- /dev/null
+++ b/tests/qapi-schema/alternate-unknown.json
@@ -0,0 +1,3 @@
+# we reject an alternate with unknown type in branch
+{ 'alternate': 'Alt',
+  'data': { 'unknown': 'MissingType', 'i': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/alternate-unknown.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/alternate-unknown.out
diff --git a/tests/qapi-schema/args-alternate.err b/tests/qapi-schema/args-alternate.err
new file mode 100644
index 0000000..3086eae
--- /dev/null
+++ b/tests/qapi-schema/args-alternate.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-alternate.json:3: 'data' for command 'oops' cannot use alternate type 'Alt'
diff --git a/tests/qapi-schema/args-alternate.exit b/tests/qapi-schema/args-alternate.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-alternate.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-alternate.json b/tests/qapi-schema/args-alternate.json
new file mode 100644
index 0000000..69e94d4
--- /dev/null
+++ b/tests/qapi-schema/args-alternate.json
@@ -0,0 +1,3 @@
+# we do not allow alternate arguments
+{ 'alternate': 'Alt', 'data': { 'case1': 'int', 'case2': 'str' } }
+{ 'command': 'oops', 'data': 'Alt' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-alternate.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-alternate.out
diff --git a/tests/qapi-schema/args-any.err b/tests/qapi-schema/args-any.err
new file mode 100644
index 0000000..bf9b5e0
--- /dev/null
+++ b/tests/qapi-schema/args-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-any.json:2: 'data' for command 'oops' cannot use built-in type 'any'
diff --git a/tests/qapi-schema/args-any.exit b/tests/qapi-schema/args-any.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-any.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-any.json b/tests/qapi-schema/args-any.json
new file mode 100644
index 0000000..58fe5e4
--- /dev/null
+++ b/tests/qapi-schema/args-any.json
@@ -0,0 +1,2 @@
+# we do not allow an 'any' argument
+{ 'command': 'oops', 'data': 'any' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-any.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-any.out
diff --git a/tests/qapi-schema/args-array-empty.err b/tests/qapi-schema/args-array-empty.err
new file mode 100644
index 0000000..cb7ed33
--- /dev/null
+++ b/tests/qapi-schema/args-array-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-array-empty.json:2: Member 'empty' of 'data' for command 'oops': array type must contain single type name
diff --git a/tests/qapi-schema/args-array-empty.exit b/tests/qapi-schema/args-array-empty.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-array-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-array-empty.json b/tests/qapi-schema/args-array-empty.json
new file mode 100644
index 0000000..652dcfb
--- /dev/null
+++ b/tests/qapi-schema/args-array-empty.json
@@ -0,0 +1,2 @@
+# we reject an array for data if it does not contain a known type
+{ 'command': 'oops', 'data': { 'empty': [ ] } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-array-empty.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-array-empty.out
diff --git a/tests/qapi-schema/args-array-unknown.err b/tests/qapi-schema/args-array-unknown.err
new file mode 100644
index 0000000..cd7a0f9
--- /dev/null
+++ b/tests/qapi-schema/args-array-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-array-unknown.json:2: Member 'array' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/args-array-unknown.exit b/tests/qapi-schema/args-array-unknown.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-array-unknown.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-array-unknown.json b/tests/qapi-schema/args-array-unknown.json
new file mode 100644
index 0000000..6f3e883
--- /dev/null
+++ b/tests/qapi-schema/args-array-unknown.json
@@ -0,0 +1,2 @@
+# we reject an array for data if it does not contain a known type
+{ 'command': 'oops', 'data': { 'array': [ 'NoSuchType' ] } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-array-unknown.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-array-unknown.out
diff --git a/tests/qapi-schema/args-bad-boxed.err b/tests/qapi-schema/args-bad-boxed.err
new file mode 100644
index 0000000..ad0d417
--- /dev/null
+++ b/tests/qapi-schema/args-bad-boxed.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-bad-boxed.json:2: 'boxed' of command 'foo' should only use true value
diff --git a/tests/qapi-schema/args-bad-boxed.exit b/tests/qapi-schema/args-bad-boxed.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-bad-boxed.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-bad-boxed.json b/tests/qapi-schema/args-bad-boxed.json
new file mode 100644
index 0000000..dea0cd0
--- /dev/null
+++ b/tests/qapi-schema/args-bad-boxed.json
@@ -0,0 +1,2 @@
+# 'boxed' should only appear with value true
+{ 'command': 'foo', 'boxed': false }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-bad-boxed.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-bad-boxed.out
diff --git a/tests/qapi-schema/args-boxed-anon.err b/tests/qapi-schema/args-boxed-anon.err
new file mode 100644
index 0000000..f24f345
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-anon.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-boxed-anon.json:2: 'data' for command 'foo' should be a type name
diff --git a/tests/qapi-schema/args-boxed-anon.exit b/tests/qapi-schema/args-boxed-anon.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-anon.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-boxed-anon.json b/tests/qapi-schema/args-boxed-anon.json
new file mode 100644
index 0000000..95f60da
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-anon.json
@@ -0,0 +1,2 @@
+# 'boxed' can only be used with named types
+{ 'command': 'foo', 'boxed': true, 'data': { 'string': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-boxed-anon.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-boxed-anon.out
diff --git a/tests/qapi-schema/args-boxed-empty.err b/tests/qapi-schema/args-boxed-empty.err
new file mode 100644
index 0000000..039603e
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-boxed-empty.json:3: Cannot use 'boxed' with empty type
diff --git a/tests/qapi-schema/args-boxed-empty.exit b/tests/qapi-schema/args-boxed-empty.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-boxed-empty.json b/tests/qapi-schema/args-boxed-empty.json
new file mode 100644
index 0000000..52717e0
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-empty.json
@@ -0,0 +1,3 @@
+# 'boxed' requires a non-empty type
+{ 'struct': 'Empty', 'data': {} }
+{ 'command': 'foo', 'boxed': true, 'data': 'Empty' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-boxed-empty.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-boxed-empty.out
diff --git a/tests/qapi-schema/args-boxed-string.err b/tests/qapi-schema/args-boxed-string.err
new file mode 100644
index 0000000..d326b48
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-string.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-boxed-string.json:2: 'data' for command 'foo' cannot use built-in type 'str'
diff --git a/tests/qapi-schema/args-boxed-string.exit b/tests/qapi-schema/args-boxed-string.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-string.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-boxed-string.json b/tests/qapi-schema/args-boxed-string.json
new file mode 100644
index 0000000..f91a150
--- /dev/null
+++ b/tests/qapi-schema/args-boxed-string.json
@@ -0,0 +1,2 @@
+# 'boxed' requires a complex (not built-in) type
+{ 'command': 'foo', 'boxed': true, 'data': 'str' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-boxed-string.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-boxed-string.out
diff --git a/tests/qapi-schema/args-int.err b/tests/qapi-schema/args-int.err
new file mode 100644
index 0000000..dc1d250
--- /dev/null
+++ b/tests/qapi-schema/args-int.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-int.json:2: 'data' for command 'oops' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/args-int.exit b/tests/qapi-schema/args-int.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-int.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-int.json b/tests/qapi-schema/args-int.json
new file mode 100644
index 0000000..a334d92
--- /dev/null
+++ b/tests/qapi-schema/args-int.json
@@ -0,0 +1,2 @@
+# we reject commands where data is not an array or complex type
+{ 'command': 'oops', 'data': 'int' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-int.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-int.out
diff --git a/tests/qapi-schema/args-invalid.err b/tests/qapi-schema/args-invalid.err
new file mode 100644
index 0000000..fe1e949
--- /dev/null
+++ b/tests/qapi-schema/args-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-invalid.json:1: 'data' for command 'foo' should be a dictionary or type name
diff --git a/tests/qapi-schema/args-invalid.exit b/tests/qapi-schema/args-invalid.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-invalid.json b/tests/qapi-schema/args-invalid.json
new file mode 100644
index 0000000..db09813
--- /dev/null
+++ b/tests/qapi-schema/args-invalid.json
@@ -0,0 +1,2 @@
+{ 'command': 'foo',
+  'data': false }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-invalid.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-invalid.out
diff --git a/tests/qapi-schema/args-member-array-bad.err b/tests/qapi-schema/args-member-array-bad.err
new file mode 100644
index 0000000..881b4d9
--- /dev/null
+++ b/tests/qapi-schema/args-member-array-bad.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-member-array-bad.json:2: Member 'member' of 'data' for command 'oops': array type must contain single type name
diff --git a/tests/qapi-schema/args-member-array-bad.exit b/tests/qapi-schema/args-member-array-bad.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-member-array-bad.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-member-array-bad.json b/tests/qapi-schema/args-member-array-bad.json
new file mode 100644
index 0000000..b2ff144
--- /dev/null
+++ b/tests/qapi-schema/args-member-array-bad.json
@@ -0,0 +1,2 @@
+# we reject data if it does not contain a valid array type
+{ 'command': 'oops', 'data': { 'member': [ { 'nested': 'str' } ] } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-member-array-bad.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-member-array-bad.out
diff --git a/tests/qapi-schema/args-member-case.err b/tests/qapi-schema/args-member-case.err
new file mode 100644
index 0000000..19c4426
--- /dev/null
+++ b/tests/qapi-schema/args-member-case.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-member-case.json:2: 'Arg' (parameter of no-way-this-will-get-whitelisted) should not use uppercase
diff --git a/tests/qapi-schema/args-member-case.exit b/tests/qapi-schema/args-member-case.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-member-case.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-member-case.json b/tests/qapi-schema/args-member-case.json
new file mode 100644
index 0000000..93439be
--- /dev/null
+++ b/tests/qapi-schema/args-member-case.json
@@ -0,0 +1,2 @@
+# Member names should be 'lower-case' unless the struct/command is whitelisted
+{ 'command': 'no-way-this-will-get-whitelisted', 'data': { 'Arg': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-member-case.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-member-case.out
diff --git a/tests/qapi-schema/args-member-unknown.err b/tests/qapi-schema/args-member-unknown.err
new file mode 100644
index 0000000..f6f8282
--- /dev/null
+++ b/tests/qapi-schema/args-member-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-member-unknown.json:2: Member 'member' of 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/args-member-unknown.exit b/tests/qapi-schema/args-member-unknown.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-member-unknown.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-member-unknown.json b/tests/qapi-schema/args-member-unknown.json
new file mode 100644
index 0000000..342a41e
--- /dev/null
+++ b/tests/qapi-schema/args-member-unknown.json
@@ -0,0 +1,2 @@
+# we reject data if it does not contain a known type
+{ 'command': 'oops', 'data': { 'member': 'NoSuchType' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-member-unknown.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-member-unknown.out
diff --git a/tests/qapi-schema/args-name-clash.err b/tests/qapi-schema/args-name-clash.err
new file mode 100644
index 0000000..d953e8d
--- /dev/null
+++ b/tests/qapi-schema/args-name-clash.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-name-clash.json:4: 'a_b' (parameter of oops) collides with 'a-b' (parameter of oops)
diff --git a/tests/qapi-schema/args-name-clash.exit b/tests/qapi-schema/args-name-clash.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-name-clash.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-name-clash.json b/tests/qapi-schema/args-name-clash.json
new file mode 100644
index 0000000..61423cb
--- /dev/null
+++ b/tests/qapi-schema/args-name-clash.json
@@ -0,0 +1,4 @@
+# C member name collision
+# Reject members that clash when mapped to C names (we would have two 'a_b'
+# members).
+{ 'command': 'oops', 'data': { 'a-b': 'str', 'a_b': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-name-clash.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-name-clash.out
diff --git a/tests/qapi-schema/args-union.err b/tests/qapi-schema/args-union.err
new file mode 100644
index 0000000..f8ad223
--- /dev/null
+++ b/tests/qapi-schema/args-union.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-union.json:3: 'data' for command 'oops' cannot use union type 'Uni'
diff --git a/tests/qapi-schema/args-union.exit b/tests/qapi-schema/args-union.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-union.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-union.json b/tests/qapi-schema/args-union.json
new file mode 100644
index 0000000..2fcaeaa
--- /dev/null
+++ b/tests/qapi-schema/args-union.json
@@ -0,0 +1,3 @@
+# use of union arguments requires 'boxed':true
+{ 'union': 'Uni', 'data': { 'case1': 'int', 'case2': 'str' } }
+{ 'command': 'oops', 'data': 'Uni' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-union.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-union.out
diff --git a/tests/qapi-schema/args-unknown.err b/tests/qapi-schema/args-unknown.err
new file mode 100644
index 0000000..4d91ec8
--- /dev/null
+++ b/tests/qapi-schema/args-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/args-unknown.json:2: 'data' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/args-unknown.exit b/tests/qapi-schema/args-unknown.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/args-unknown.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/args-unknown.json b/tests/qapi-schema/args-unknown.json
new file mode 100644
index 0000000..32aba43
--- /dev/null
+++ b/tests/qapi-schema/args-unknown.json
@@ -0,0 +1,2 @@
+# we reject data if it does not contain a known type
+{ 'command': 'oops', 'data': 'NoSuchType' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/args-unknown.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/args-unknown.out
diff --git a/tests/qapi-schema/bad-base.err b/tests/qapi-schema/bad-base.err
new file mode 100644
index 0000000..154274b
--- /dev/null
+++ b/tests/qapi-schema/bad-base.err
@@ -0,0 +1 @@
+tests/qapi-schema/bad-base.json:3: 'base' for struct 'MyType' cannot use union type 'Union'
diff --git a/tests/qapi-schema/bad-base.exit b/tests/qapi-schema/bad-base.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/bad-base.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/bad-base.json b/tests/qapi-schema/bad-base.json
new file mode 100644
index 0000000..a634331
--- /dev/null
+++ b/tests/qapi-schema/bad-base.json
@@ -0,0 +1,3 @@
+# we reject a base that is not a struct
+{ 'union': 'Union', 'data': { 'a': 'int', 'b': 'str' } }
+{ 'struct': 'MyType', 'base': 'Union', 'data': { 'c': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/bad-base.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/bad-base.out
diff --git a/tests/qapi-schema/bad-data.err b/tests/qapi-schema/bad-data.err
new file mode 100644
index 0000000..8523ac4
--- /dev/null
+++ b/tests/qapi-schema/bad-data.err
@@ -0,0 +1 @@
+tests/qapi-schema/bad-data.json:2: 'data' for command 'oops' cannot be an array
diff --git a/tests/qapi-schema/bad-data.exit b/tests/qapi-schema/bad-data.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/bad-data.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/bad-data.json b/tests/qapi-schema/bad-data.json
new file mode 100644
index 0000000..832eeb7
--- /dev/null
+++ b/tests/qapi-schema/bad-data.json
@@ -0,0 +1,2 @@
+# we ensure 'data' is a dictionary for all but enums
+{ 'command': 'oops', 'data': [ ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/bad-data.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/bad-data.out
diff --git a/tests/qapi-schema/bad-ident.err b/tests/qapi-schema/bad-ident.err
new file mode 100644
index 0000000..c419060
--- /dev/null
+++ b/tests/qapi-schema/bad-ident.err
@@ -0,0 +1 @@
+tests/qapi-schema/bad-ident.json:2: 'struct' does not allow optional name '*oops'
diff --git a/tests/qapi-schema/bad-ident.exit b/tests/qapi-schema/bad-ident.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/bad-ident.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/bad-ident.json b/tests/qapi-schema/bad-ident.json
new file mode 100644
index 0000000..763627a
--- /dev/null
+++ b/tests/qapi-schema/bad-ident.json
@@ -0,0 +1,2 @@
+# we reject creating a type name with bad name
+{ 'struct': '*oops', 'data': { 'i': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/bad-ident.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/bad-ident.out
diff --git a/tests/qapi-schema/bad-type-bool.err b/tests/qapi-schema/bad-type-bool.err
new file mode 100644
index 0000000..62fd70b
--- /dev/null
+++ b/tests/qapi-schema/bad-type-bool.err
@@ -0,0 +1 @@
+tests/qapi-schema/bad-type-bool.json:2: 'struct' key must have a string value
diff --git a/tests/qapi-schema/bad-type-bool.exit b/tests/qapi-schema/bad-type-bool.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/bad-type-bool.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/bad-type-bool.json b/tests/qapi-schema/bad-type-bool.json
new file mode 100644
index 0000000..bde17b5
--- /dev/null
+++ b/tests/qapi-schema/bad-type-bool.json
@@ -0,0 +1,2 @@
+# we reject an expression with a metatype that is not a string
+{ 'struct': true, 'data': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/bad-type-bool.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/bad-type-bool.out
diff --git a/tests/qapi-schema/bad-type-dict.err b/tests/qapi-schema/bad-type-dict.err
new file mode 100644
index 0000000..0b2a2ae
--- /dev/null
+++ b/tests/qapi-schema/bad-type-dict.err
@@ -0,0 +1 @@
+tests/qapi-schema/bad-type-dict.json:2: 'command' key must have a string value
diff --git a/tests/qapi-schema/bad-type-dict.exit b/tests/qapi-schema/bad-type-dict.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/bad-type-dict.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/bad-type-dict.json b/tests/qapi-schema/bad-type-dict.json
new file mode 100644
index 0000000..2a91b24
--- /dev/null
+++ b/tests/qapi-schema/bad-type-dict.json
@@ -0,0 +1,2 @@
+# we reject an expression with a metatype that is not a string
+{ 'command': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/bad-type-dict.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/bad-type-dict.out
diff --git a/tests/qapi-schema/bad-type-int.err b/tests/qapi-schema/bad-type-int.err
new file mode 100644
index 0000000..da89895
--- /dev/null
+++ b/tests/qapi-schema/bad-type-int.err
@@ -0,0 +1 @@
+tests/qapi-schema/bad-type-int.json:3:13: Stray "1"
diff --git a/tests/qapi-schema/bad-type-int.exit b/tests/qapi-schema/bad-type-int.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/bad-type-int.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/bad-type-int.json b/tests/qapi-schema/bad-type-int.json
new file mode 100644
index 0000000..56fc6f8
--- /dev/null
+++ b/tests/qapi-schema/bad-type-int.json
@@ -0,0 +1,3 @@
+# we reject an expression with a metatype that is not a string
+# FIXME: once the parser understands integer inputs, improve the error message
+{ 'struct': 1, 'data': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/bad-type-int.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/bad-type-int.out
diff --git a/tests/qapi-schema/base-cycle-direct.err b/tests/qapi-schema/base-cycle-direct.err
new file mode 100644
index 0000000..9c68f65
--- /dev/null
+++ b/tests/qapi-schema/base-cycle-direct.err
@@ -0,0 +1 @@
+tests/qapi-schema/base-cycle-direct.json:2: Object Loopy contains itself
diff --git a/tests/qapi-schema/base-cycle-direct.exit b/tests/qapi-schema/base-cycle-direct.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/base-cycle-direct.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/base-cycle-direct.json b/tests/qapi-schema/base-cycle-direct.json
new file mode 100644
index 0000000..4fc66d0
--- /dev/null
+++ b/tests/qapi-schema/base-cycle-direct.json
@@ -0,0 +1,2 @@
+# we reject a loop in base classes
+{ 'struct': 'Loopy', 'base': 'Loopy', 'data': {} }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/base-cycle-direct.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/base-cycle-direct.out
diff --git a/tests/qapi-schema/base-cycle-indirect.err b/tests/qapi-schema/base-cycle-indirect.err
new file mode 100644
index 0000000..fc92fe4
--- /dev/null
+++ b/tests/qapi-schema/base-cycle-indirect.err
@@ -0,0 +1 @@
+tests/qapi-schema/base-cycle-indirect.json:2: Object Base1 contains itself
diff --git a/tests/qapi-schema/base-cycle-indirect.exit b/tests/qapi-schema/base-cycle-indirect.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/base-cycle-indirect.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/base-cycle-indirect.json b/tests/qapi-schema/base-cycle-indirect.json
new file mode 100644
index 0000000..2866772
--- /dev/null
+++ b/tests/qapi-schema/base-cycle-indirect.json
@@ -0,0 +1,3 @@
+# we reject a loop in base classes
+{ 'struct': 'Base1', 'base': 'Base2', 'data': {} }
+{ 'struct': 'Base2', 'base': 'Base1', 'data': {} }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/base-cycle-indirect.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/base-cycle-indirect.out
diff --git a/tests/qapi-schema/command-int.err b/tests/qapi-schema/command-int.err
new file mode 100644
index 0000000..0f93006
--- /dev/null
+++ b/tests/qapi-schema/command-int.err
@@ -0,0 +1 @@
+tests/qapi-schema/command-int.json:2: built-in 'int' is already defined
diff --git a/tests/qapi-schema/command-int.exit b/tests/qapi-schema/command-int.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/command-int.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/command-int.json b/tests/qapi-schema/command-int.json
new file mode 100644
index 0000000..9a62554
--- /dev/null
+++ b/tests/qapi-schema/command-int.json
@@ -0,0 +1,2 @@
+# we reject collisions between commands and types
+{ 'command': 'int', 'data': { 'character': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/command-int.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/command-int.out
diff --git a/tests/qapi-schema/comments.out b/tests/qapi-schema/comments.out
index 4ce3dcf..5d7c13c 100644
--- a/tests/qapi-schema/comments.out
+++ b/tests/qapi-schema/comments.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/tests/qapi-schema/double-data.err b/tests/qapi-schema/double-data.err
new file mode 100644
index 0000000..cc765c4
--- /dev/null
+++ b/tests/qapi-schema/double-data.err
@@ -0,0 +1 @@
+tests/qapi-schema/double-data.json:2:41: Duplicate key "data"
diff --git a/tests/qapi-schema/double-data.exit b/tests/qapi-schema/double-data.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/double-data.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/double-data.json b/tests/qapi-schema/double-data.json
new file mode 100644
index 0000000..e76b519
--- /dev/null
+++ b/tests/qapi-schema/double-data.json
@@ -0,0 +1,2 @@
+# we reject an expression with duplicate top-level keys
+{ 'struct': 'bar', 'data': { }, 'data': { 'string': 'str'} }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/double-data.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/double-data.out
diff --git a/tests/qapi-schema/double-type.err b/tests/qapi-schema/double-type.err
new file mode 100644
index 0000000..f9613c6
--- /dev/null
+++ b/tests/qapi-schema/double-type.err
@@ -0,0 +1 @@
+tests/qapi-schema/double-type.json:2: Unknown key 'command' in struct 'bar'
diff --git a/tests/qapi-schema/double-type.exit b/tests/qapi-schema/double-type.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/double-type.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/double-type.json b/tests/qapi-schema/double-type.json
new file mode 100644
index 0000000..911fa7a
--- /dev/null
+++ b/tests/qapi-schema/double-type.json
@@ -0,0 +1,2 @@
+# we reject an expression with ambiguous metatype
+{ 'command': 'foo', 'struct': 'bar', 'data': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/double-type.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/double-type.out
diff --git a/tests/qapi-schema/duplicate-key.err b/tests/qapi-schema/duplicate-key.err
index 768b276..6d02f83 100644
--- a/tests/qapi-schema/duplicate-key.err
+++ b/tests/qapi-schema/duplicate-key.err
@@ -1 +1 @@
-tests/qapi-schema/duplicate-key.json:2:10: Duplicate key "key"
+tests/qapi-schema/duplicate-key.json:3:10: Duplicate key "key"
diff --git a/tests/qapi-schema/duplicate-key.json b/tests/qapi-schema/duplicate-key.json
index 1b55d88..14ac0e8 100644
--- a/tests/qapi-schema/duplicate-key.json
+++ b/tests/qapi-schema/duplicate-key.json
@@ -1,2 +1,3 @@
+# QAPI cannot include the same key more than once in any {}
 { 'key': 'value',
   'key': 'value' }
diff --git a/tests/qapi-schema/empty.out b/tests/qapi-schema/empty.out
index b7f89a4..8a5b034 100644
--- a/tests/qapi-schema/empty.out
+++ b/tests/qapi-schema/empty.out
@@ -1,3 +1,3 @@
-[]
-[]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+object q_empty
diff --git a/tests/qapi-schema/enum-bad-name.err b/tests/qapi-schema/enum-bad-name.err
new file mode 100644
index 0000000..9c3c100
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-name.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-bad-name.json:2: Member of enum 'MyEnum' uses invalid name 'not^possible'
diff --git a/tests/qapi-schema/enum-bad-name.exit b/tests/qapi-schema/enum-bad-name.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-name.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-bad-name.json b/tests/qapi-schema/enum-bad-name.json
new file mode 100644
index 0000000..8506562
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-name.json
@@ -0,0 +1,2 @@
+# we ensure all enum names can map to C
+{ 'enum': 'MyEnum', 'data': [ 'not^possible' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-bad-name.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-bad-name.out
diff --git a/tests/qapi-schema/enum-bad-prefix.err b/tests/qapi-schema/enum-bad-prefix.err
new file mode 100644
index 0000000..399f5f7
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-prefix.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-bad-prefix.json:2: Enum 'MyEnum' requires a string for 'prefix'
diff --git a/tests/qapi-schema/enum-bad-prefix.exit b/tests/qapi-schema/enum-bad-prefix.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-prefix.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-bad-prefix.json b/tests/qapi-schema/enum-bad-prefix.json
new file mode 100644
index 0000000..996f628
--- /dev/null
+++ b/tests/qapi-schema/enum-bad-prefix.json
@@ -0,0 +1,2 @@
+# The prefix must be a string type
+{ 'enum': 'MyEnum', 'data': [ 'one' ], 'prefix': [ 'fish' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-bad-prefix.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-bad-prefix.out
diff --git a/tests/qapi-schema/enum-clash-member.err b/tests/qapi-schema/enum-clash-member.err
new file mode 100644
index 0000000..5403c78
--- /dev/null
+++ b/tests/qapi-schema/enum-clash-member.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-clash-member.json:2: 'one_two' (member of MyEnum) collides with 'one-two' (member of MyEnum)
diff --git a/tests/qapi-schema/enum-clash-member.exit b/tests/qapi-schema/enum-clash-member.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-clash-member.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-clash-member.json b/tests/qapi-schema/enum-clash-member.json
new file mode 100644
index 0000000..b6928b8
--- /dev/null
+++ b/tests/qapi-schema/enum-clash-member.json
@@ -0,0 +1,2 @@
+# we reject enums where members will clash when mapped to C enum
+{ 'enum': 'MyEnum', 'data': [ 'one-two', 'one_two' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-clash-member.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-clash-member.out
diff --git a/tests/qapi-schema/enum-dict-member.err b/tests/qapi-schema/enum-dict-member.err
new file mode 100644
index 0000000..8ca146e
--- /dev/null
+++ b/tests/qapi-schema/enum-dict-member.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-dict-member.json:2: Member of enum 'MyEnum' requires a string name
diff --git a/tests/qapi-schema/enum-dict-member.exit b/tests/qapi-schema/enum-dict-member.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-dict-member.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-dict-member.json b/tests/qapi-schema/enum-dict-member.json
new file mode 100644
index 0000000..79672e0
--- /dev/null
+++ b/tests/qapi-schema/enum-dict-member.json
@@ -0,0 +1,2 @@
+# we reject any enum member that is not a string
+{ 'enum': 'MyEnum', 'data': [ { 'value': 'str' } ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-dict-member.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-dict-member.out
diff --git a/tests/qapi-schema/enum-int-member.err b/tests/qapi-schema/enum-int-member.err
new file mode 100644
index 0000000..071c521
--- /dev/null
+++ b/tests/qapi-schema/enum-int-member.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-int-member.json:3:31: Stray "1"
diff --git a/tests/qapi-schema/enum-int-member.exit b/tests/qapi-schema/enum-int-member.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-int-member.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-int-member.json b/tests/qapi-schema/enum-int-member.json
new file mode 100644
index 0000000..6c9c32e
--- /dev/null
+++ b/tests/qapi-schema/enum-int-member.json
@@ -0,0 +1,3 @@
+# we reject any enum member that is not a string
+# FIXME: once the parser understands integer inputs, improve the error message
+{ 'enum': 'MyEnum', 'data': [ 1 ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-int-member.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-int-member.out
diff --git a/tests/qapi-schema/enum-member-case.err b/tests/qapi-schema/enum-member-case.err
new file mode 100644
index 0000000..b652e9a
--- /dev/null
+++ b/tests/qapi-schema/enum-member-case.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-member-case.json:3: 'Value' (member of NoWayThisWillGetWhitelisted) should not use uppercase
diff --git a/tests/qapi-schema/enum-member-case.exit b/tests/qapi-schema/enum-member-case.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-member-case.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-member-case.json b/tests/qapi-schema/enum-member-case.json
new file mode 100644
index 0000000..2096b35
--- /dev/null
+++ b/tests/qapi-schema/enum-member-case.json
@@ -0,0 +1,3 @@
+# Member names should be 'lower-case' unless the enum is whitelisted
+{ 'enum': 'UuidInfo', 'data': [ 'Value' ] } # UuidInfo is whitelisted
+{ 'enum': 'NoWayThisWillGetWhitelisted', 'data': [ 'Value' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-member-case.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-member-case.out
diff --git a/tests/qapi-schema/enum-missing-data.err b/tests/qapi-schema/enum-missing-data.err
new file mode 100644
index 0000000..ba4873a
--- /dev/null
+++ b/tests/qapi-schema/enum-missing-data.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-missing-data.json:2: Key 'data' is missing from enum 'MyEnum'
diff --git a/tests/qapi-schema/enum-missing-data.exit b/tests/qapi-schema/enum-missing-data.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-missing-data.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-missing-data.json b/tests/qapi-schema/enum-missing-data.json
new file mode 100644
index 0000000..558fd35
--- /dev/null
+++ b/tests/qapi-schema/enum-missing-data.json
@@ -0,0 +1,2 @@
+# we require that all QAPI enums have a data array
+{ 'enum': 'MyEnum' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-missing-data.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-missing-data.out
diff --git a/tests/qapi-schema/enum-wrong-data.err b/tests/qapi-schema/enum-wrong-data.err
new file mode 100644
index 0000000..11b4347
--- /dev/null
+++ b/tests/qapi-schema/enum-wrong-data.err
@@ -0,0 +1 @@
+tests/qapi-schema/enum-wrong-data.json:2: Enum 'MyEnum' requires an array for 'data'
diff --git a/tests/qapi-schema/enum-wrong-data.exit b/tests/qapi-schema/enum-wrong-data.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/enum-wrong-data.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/enum-wrong-data.json b/tests/qapi-schema/enum-wrong-data.json
new file mode 100644
index 0000000..7b3e255
--- /dev/null
+++ b/tests/qapi-schema/enum-wrong-data.json
@@ -0,0 +1,2 @@
+# we require that all qapi enums have an array for data
+{ 'enum': 'MyEnum', 'data': { 'value': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/enum-wrong-data.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/enum-wrong-data.out
diff --git a/tests/qapi-schema/escape-outside-string.err b/tests/qapi-schema/escape-outside-string.err
new file mode 100644
index 0000000..b9b8837
--- /dev/null
+++ b/tests/qapi-schema/escape-outside-string.err
@@ -0,0 +1 @@
+tests/qapi-schema/escape-outside-string.json:3:27: Stray "\"
diff --git a/tests/qapi-schema/escape-outside-string.exit b/tests/qapi-schema/escape-outside-string.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/escape-outside-string.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/escape-outside-string.json b/tests/qapi-schema/escape-outside-string.json
new file mode 100644
index 0000000..482f795
--- /dev/null
+++ b/tests/qapi-schema/escape-outside-string.json
@@ -0,0 +1,3 @@
+# escape sequences are permitted only inside strings
+# { 'command': 'foo', 'data': {} }
+{ 'command': 'foo', 'data'\u003a{} }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/escape-outside-string.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/escape-outside-string.out
diff --git a/tests/qapi-schema/escape-too-big.err b/tests/qapi-schema/escape-too-big.err
new file mode 100644
index 0000000..d9aeb5d
--- /dev/null
+++ b/tests/qapi-schema/escape-too-big.err
@@ -0,0 +1 @@
+tests/qapi-schema/escape-too-big.json:3:14: For now, \u escape only supports non-zero values up to \u007f
diff --git a/tests/qapi-schema/escape-too-big.exit b/tests/qapi-schema/escape-too-big.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/escape-too-big.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/escape-too-big.json b/tests/qapi-schema/escape-too-big.json
new file mode 100644
index 0000000..62bcecd
--- /dev/null
+++ b/tests/qapi-schema/escape-too-big.json
@@ -0,0 +1,3 @@
+# we don't support full Unicode strings, yet
+# { 'command': 'é' }
+{ 'command': '\u00e9' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/escape-too-big.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/escape-too-big.out
diff --git a/tests/qapi-schema/escape-too-short.err b/tests/qapi-schema/escape-too-short.err
new file mode 100644
index 0000000..934de59
--- /dev/null
+++ b/tests/qapi-schema/escape-too-short.err
@@ -0,0 +1 @@
+tests/qapi-schema/escape-too-short.json:3:14: \u escape needs 4 hex digits
diff --git a/tests/qapi-schema/escape-too-short.exit b/tests/qapi-schema/escape-too-short.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/escape-too-short.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/escape-too-short.json b/tests/qapi-schema/escape-too-short.json
new file mode 100644
index 0000000..6cb1dec
--- /dev/null
+++ b/tests/qapi-schema/escape-too-short.json
@@ -0,0 +1,3 @@
+# the \u escape requires 4 hex digits
+# { 'command': 'a' }
+{ 'command': '\u61' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/escape-too-short.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/escape-too-short.out
diff --git a/tests/qapi-schema/event-boxed-empty.err b/tests/qapi-schema/event-boxed-empty.err
new file mode 100644
index 0000000..68ec6f2
--- /dev/null
+++ b/tests/qapi-schema/event-boxed-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/event-boxed-empty.json:2: Use of 'boxed' requires 'data'
diff --git a/tests/qapi-schema/event-boxed-empty.exit b/tests/qapi-schema/event-boxed-empty.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/event-boxed-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/event-boxed-empty.json b/tests/qapi-schema/event-boxed-empty.json
new file mode 100644
index 0000000..cb145f1
--- /dev/null
+++ b/tests/qapi-schema/event-boxed-empty.json
@@ -0,0 +1,2 @@
+# 'boxed' requires a non-empty type
+{ 'event': 'FOO', 'boxed': true }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/event-boxed-empty.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/event-boxed-empty.out
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/event-case.err
similarity index 100%
rename from tests/qapi-schema/flat-union-reverse-define.err
rename to tests/qapi-schema/event-case.err
diff --git a/tests/qapi-schema/flat-union-reverse-define.exit b/tests/qapi-schema/event-case.exit
similarity index 100%
rename from tests/qapi-schema/flat-union-reverse-define.exit
rename to tests/qapi-schema/event-case.exit
diff --git a/tests/qapi-schema/event-case.json b/tests/qapi-schema/event-case.json
new file mode 100644
index 0000000..3a92d8b
--- /dev/null
+++ b/tests/qapi-schema/event-case.json
@@ -0,0 +1,3 @@
+# TODO: might be nice to enforce naming conventions; but until then this works
+# even though events should usually be ALL_CAPS
+{ 'event': 'oops' }
diff --git a/tests/qapi-schema/event-case.out b/tests/qapi-schema/event-case.out
new file mode 100644
index 0000000..5a0f2bf
--- /dev/null
+++ b/tests/qapi-schema/event-case.out
@@ -0,0 +1,5 @@
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+event oops None
+   boxed=False
+object q_empty
diff --git a/tests/qapi-schema/event-nest-struct.err b/tests/qapi-schema/event-nest-struct.err
index 91bde1c..5a42701 100644
--- a/tests/qapi-schema/event-nest-struct.err
+++ b/tests/qapi-schema/event-nest-struct.err
@@ -1 +1 @@
-tests/qapi-schema/event-nest-struct.json:1: Nested structure define in event is not supported, event 'EVENT_A', argname 'a'
+tests/qapi-schema/event-nest-struct.json:1: Member 'a' of 'data' for event 'EVENT_A' should be a type name
diff --git a/tests/qapi-schema/flat-union-array-branch.err b/tests/qapi-schema/flat-union-array-branch.err
new file mode 100644
index 0000000..8ea91ea
--- /dev/null
+++ b/tests/qapi-schema/flat-union-array-branch.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-array-branch.json:8: Member 'value1' of union 'TestUnion' cannot be an array
diff --git a/tests/qapi-schema/flat-union-array-branch.exit b/tests/qapi-schema/flat-union-array-branch.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-array-branch.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-array-branch.json b/tests/qapi-schema/flat-union-array-branch.json
new file mode 100644
index 0000000..0b98820
--- /dev/null
+++ b/tests/qapi-schema/flat-union-array-branch.json
@@ -0,0 +1,12 @@
+# we require flat union branches to be a struct
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'Base',
+  'data': { 'enum1': 'TestEnum' } }
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'discriminator': 'enum1',
+  'data': { 'value1': ['TestTypeB'],
+            'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-array-branch.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-array-branch.out
diff --git a/tests/qapi-schema/flat-union-bad-base.err b/tests/qapi-schema/flat-union-bad-base.err
new file mode 100644
index 0000000..bee24a2
--- /dev/null
+++ b/tests/qapi-schema/flat-union-bad-base.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-bad-base.json:8: 'string' (member of TestTypeA) collides with 'string' (base of TestUnion)
diff --git a/tests/qapi-schema/flat-union-bad-base.exit b/tests/qapi-schema/flat-union-bad-base.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-bad-base.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-bad-base.json b/tests/qapi-schema/flat-union-bad-base.json
new file mode 100644
index 0000000..74dd421
--- /dev/null
+++ b/tests/qapi-schema/flat-union-bad-base.json
@@ -0,0 +1,12 @@
+# we allow anonymous base, but enforce no duplicate keys
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'TestTypeA',
+  'data': { 'string': 'str' } }
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+{ 'union': 'TestUnion',
+  'base': { 'enum1': 'TestEnum', 'string': 'str' },
+  'discriminator': 'enum1',
+  'data': { 'value1': 'TestTypeA',
+            'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-bad-base.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-bad-base.out
diff --git a/tests/qapi-schema/flat-union-bad-discriminator.err b/tests/qapi-schema/flat-union-bad-discriminator.err
new file mode 100644
index 0000000..c38cc8e
--- /dev/null
+++ b/tests/qapi-schema/flat-union-bad-discriminator.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-bad-discriminator.json:11: Discriminator of flat union 'TestUnion' requires a string name
diff --git a/tests/qapi-schema/flat-union-bad-discriminator.exit b/tests/qapi-schema/flat-union-bad-discriminator.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-bad-discriminator.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-bad-discriminator.json b/tests/qapi-schema/flat-union-bad-discriminator.json
new file mode 100644
index 0000000..cd10b9d
--- /dev/null
+++ b/tests/qapi-schema/flat-union-bad-discriminator.json
@@ -0,0 +1,15 @@
+# we require the discriminator to be a string naming a base-type member
+# this tests the old syntax for anonymous unions before we added alternates
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'TestBase',
+  'data': { 'enum1': 'TestEnum', 'kind': 'str' } }
+{ 'struct': 'TestTypeA',
+  'data': { 'string': 'str' } }
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+{ 'union': 'TestUnion',
+  'base': 'TestBase',
+  'discriminator': {},
+  'data': { 'kind1': 'TestTypeA',
+            'kind2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-bad-discriminator.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-bad-discriminator.out
diff --git a/tests/qapi-schema/flat-union-base-any.err b/tests/qapi-schema/flat-union-base-any.err
new file mode 100644
index 0000000..646f1c9
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-any.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-base-any.json:8: 'base' for union 'TestUnion' cannot use built-in type 'any'
diff --git a/tests/qapi-schema/flat-union-base-any.exit b/tests/qapi-schema/flat-union-base-any.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-any.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-base-any.json b/tests/qapi-schema/flat-union-base-any.json
new file mode 100644
index 0000000..fe66b71
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-any.json
@@ -0,0 +1,12 @@
+# we require the base to be an existing struct
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'TestTypeA',
+  'data': { 'string': 'str' } }
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+{ 'union': 'TestUnion',
+  'base': 'any',
+  'discriminator': 'enum1',
+  'data': { 'value1': 'TestTypeA',
+            'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-base-any.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-base-any.out
diff --git a/tests/qapi-schema/flat-union-base-union.err b/tests/qapi-schema/flat-union-base-union.err
new file mode 100644
index 0000000..f138395
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-union.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-base-union.json:14: 'base' for union 'TestUnion' cannot use union type 'UnionBase'
diff --git a/tests/qapi-schema/flat-union-base-union.exit b/tests/qapi-schema/flat-union-base-union.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-union.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-base-union.json b/tests/qapi-schema/flat-union-base-union.json
new file mode 100644
index 0000000..98b4eba
--- /dev/null
+++ b/tests/qapi-schema/flat-union-base-union.json
@@ -0,0 +1,18 @@
+# For now, we require the base to be a struct without variants
+# TODO: It would be possible to allow a union as a base, as long as all
+# permutations of QMP names exposed by base do not clash with any QMP
+# member names added by local variants.
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'TestTypeA',
+  'data': { 'string': 'str' } }
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+{ 'union': 'UnionBase',
+  'data': { 'kind1': 'TestTypeA',
+            'kind2': 'TestTypeB' } }
+{ 'union': 'TestUnion',
+  'base': 'UnionBase',
+  'discriminator': 'type',
+  'data': { 'kind1': 'TestTypeA',
+            'kind2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-base-union.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-base-union.out
diff --git a/tests/qapi-schema/flat-union-clash-member.err b/tests/qapi-schema/flat-union-clash-member.err
new file mode 100644
index 0000000..2adf697
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-member.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-clash-member.json:11: 'name' (member of Branch1) collides with 'name' (member of Base)
diff --git a/tests/qapi-schema/flat-union-clash-member.exit b/tests/qapi-schema/flat-union-clash-member.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-member.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-clash-member.json b/tests/qapi-schema/flat-union-clash-member.json
new file mode 100644
index 0000000..9efc771
--- /dev/null
+++ b/tests/qapi-schema/flat-union-clash-member.json
@@ -0,0 +1,15 @@
+# We check for no duplicate keys between branch members and base
+# base's member 'name' clashes with Branch1's
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'Base',
+  'data': { 'enum1': 'TestEnum', '*name': 'str' } }
+{ 'struct': 'Branch1',
+  'data': { 'name': 'str' } }
+{ 'struct': 'Branch2',
+  'data': { 'value': 'int' } }
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'discriminator': 'enum1',
+  'data': { 'value1': 'Branch1',
+            'value2': 'Branch2' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-clash-member.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-clash-member.out
diff --git a/tests/qapi-schema/flat-union-empty.err b/tests/qapi-schema/flat-union-empty.err
new file mode 100644
index 0000000..15754f5
--- /dev/null
+++ b/tests/qapi-schema/flat-union-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-empty.json:4: Union 'Union' cannot have empty 'data'
diff --git a/tests/qapi-schema/flat-union-empty.exit b/tests/qapi-schema/flat-union-empty.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-empty.json b/tests/qapi-schema/flat-union-empty.json
new file mode 100644
index 0000000..77f1d9a
--- /dev/null
+++ b/tests/qapi-schema/flat-union-empty.json
@@ -0,0 +1,4 @@
+# flat unions cannot be empty
+{ 'enum': 'Empty', 'data': [ ] }
+{ 'struct': 'Base', 'data': { 'type': 'Empty' } }
+{ 'union': 'Union', 'base': 'Base', 'discriminator': 'type', 'data': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-empty.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-empty.out
diff --git a/tests/qapi-schema/flat-union-incomplete-branch.err b/tests/qapi-schema/flat-union-incomplete-branch.err
new file mode 100644
index 0000000..e826bf0
--- /dev/null
+++ b/tests/qapi-schema/flat-union-incomplete-branch.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-incomplete-branch.json:6: Union 'TestUnion' data missing 'value2' branch
diff --git a/tests/qapi-schema/flat-union-incomplete-branch.exit b/tests/qapi-schema/flat-union-incomplete-branch.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-incomplete-branch.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-incomplete-branch.json b/tests/qapi-schema/flat-union-incomplete-branch.json
new file mode 100644
index 0000000..25a411b
--- /dev/null
+++ b/tests/qapi-schema/flat-union-incomplete-branch.json
@@ -0,0 +1,9 @@
+# we require all branches of the union to be covered
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'TestTypeA',
+  'data': { 'string': 'str' } }
+{ 'union': 'TestUnion',
+  'base': { 'type': 'TestEnum' },
+  'discriminator': 'type',
+  'data': { 'value1': 'TestTypeA' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-incomplete-branch.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-incomplete-branch.out
diff --git a/tests/qapi-schema/flat-union-inline.err b/tests/qapi-schema/flat-union-inline.err
new file mode 100644
index 0000000..2333358
--- /dev/null
+++ b/tests/qapi-schema/flat-union-inline.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-inline.json:7: Member 'value1' of union 'TestUnion' should be a type name
diff --git a/tests/qapi-schema/flat-union-inline.exit b/tests/qapi-schema/flat-union-inline.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-inline.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-inline.json b/tests/qapi-schema/flat-union-inline.json
new file mode 100644
index 0000000..62c7cda
--- /dev/null
+++ b/tests/qapi-schema/flat-union-inline.json
@@ -0,0 +1,11 @@
+# we require branches to be a struct name
+# TODO: should we allow anonymous inline branch types?
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'Base',
+  'data': { 'enum1': 'TestEnum', 'kind': 'str' } }
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'discriminator': 'enum1',
+  'data': { 'value1': { 'string': 'str' },
+            'value2': { 'integer': 'int' } } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-inline.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-inline.out
diff --git a/tests/qapi-schema/flat-union-int-branch.err b/tests/qapi-schema/flat-union-int-branch.err
new file mode 100644
index 0000000..faf0157
--- /dev/null
+++ b/tests/qapi-schema/flat-union-int-branch.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-int-branch.json:8: Member 'value1' of union 'TestUnion' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/flat-union-int-branch.exit b/tests/qapi-schema/flat-union-int-branch.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-int-branch.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-int-branch.json b/tests/qapi-schema/flat-union-int-branch.json
new file mode 100644
index 0000000..9370c34
--- /dev/null
+++ b/tests/qapi-schema/flat-union-int-branch.json
@@ -0,0 +1,12 @@
+# we require flat union branches to be a struct
+{ 'enum': 'TestEnum',
+  'data': [ 'value1', 'value2' ] }
+{ 'struct': 'Base',
+  'data': { 'enum1': 'TestEnum' } }
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'discriminator': 'enum1',
+  'data': { 'value1': 'int',
+            'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-int-branch.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-int-branch.out
diff --git a/tests/qapi-schema/flat-union-invalid-branch-key.json b/tests/qapi-schema/flat-union-invalid-branch-key.json
index a624282..95ff774 100644
--- a/tests/qapi-schema/flat-union-invalid-branch-key.json
+++ b/tests/qapi-schema/flat-union-invalid-branch-key.json
@@ -1,13 +1,13 @@
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 
-{ 'type': 'TestBase',
+{ 'struct': 'TestBase',
   'data': { 'enum1': 'TestEnum' } }
 
-{ 'type': 'TestTypeA',
+{ 'struct': 'TestTypeA',
   'data': { 'string': 'str' } }
 
-{ 'type': 'TestTypeB',
+{ 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 
 { 'union': 'TestUnion',
diff --git a/tests/qapi-schema/flat-union-invalid-discriminator.err b/tests/qapi-schema/flat-union-invalid-discriminator.err
index 790b675..5f40556 100644
--- a/tests/qapi-schema/flat-union-invalid-discriminator.err
+++ b/tests/qapi-schema/flat-union-invalid-discriminator.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-invalid-discriminator.json:13: Discriminator 'enum_wrong' is not a member of base type 'TestBase'
+tests/qapi-schema/flat-union-invalid-discriminator.json:13: Discriminator 'enum_wrong' is not a member of base struct 'TestBase'
diff --git a/tests/qapi-schema/flat-union-invalid-discriminator.json b/tests/qapi-schema/flat-union-invalid-discriminator.json
index 887157e..48b94c3 100644
--- a/tests/qapi-schema/flat-union-invalid-discriminator.json
+++ b/tests/qapi-schema/flat-union-invalid-discriminator.json
@@ -1,13 +1,13 @@
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 
-{ 'type': 'TestBase',
+{ 'struct': 'TestBase',
   'data': { 'enum1': 'TestEnum' } }
 
-{ 'type': 'TestTypeA',
+{ 'struct': 'TestTypeA',
   'data': { 'string': 'str' } }
 
-{ 'type': 'TestTypeB',
+{ 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 
 { 'union': 'TestUnion',
diff --git a/tests/qapi-schema/flat-union-no-base.err b/tests/qapi-schema/flat-union-no-base.err
index a59749e..841c93b 100644
--- a/tests/qapi-schema/flat-union-no-base.err
+++ b/tests/qapi-schema/flat-union-no-base.err
@@ -1 +1 @@
-tests/qapi-schema/flat-union-no-base.json:7: Flat union 'TestUnion' must have a base field
+tests/qapi-schema/flat-union-no-base.json:9: Flat union 'TestUnion' must have a base
diff --git a/tests/qapi-schema/flat-union-no-base.json b/tests/qapi-schema/flat-union-no-base.json
index 50f2673..ffc4c6f 100644
--- a/tests/qapi-schema/flat-union-no-base.json
+++ b/tests/qapi-schema/flat-union-no-base.json
@@ -1,10 +1,12 @@
-{ 'type': 'TestTypeA',
+# flat unions require a base
+# TODO: simple unions should be able to use an enum discriminator
+{ 'struct': 'TestTypeA',
   'data': { 'string': 'str' } }
-
-{ 'type': 'TestTypeB',
+{ 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
-
+{ 'enum': 'Enum',
+  'data': [ 'value1', 'value2' ] }
 { 'union': 'TestUnion',
-  'discriminator': 'enum1',
+  'discriminator': 'Enum',
   'data': { 'value1': 'TestTypeA',
             'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-optional-discriminator.err b/tests/qapi-schema/flat-union-optional-discriminator.err
new file mode 100644
index 0000000..aaabedb
--- /dev/null
+++ b/tests/qapi-schema/flat-union-optional-discriminator.err
@@ -0,0 +1 @@
+tests/qapi-schema/flat-union-optional-discriminator.json:6: Discriminator of flat union 'MyUnion' does not allow optional name '*switch'
diff --git a/tests/qapi-schema/flat-union-optional-discriminator.exit b/tests/qapi-schema/flat-union-optional-discriminator.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/flat-union-optional-discriminator.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/flat-union-optional-discriminator.json b/tests/qapi-schema/flat-union-optional-discriminator.json
new file mode 100644
index 0000000..08a8f7e
--- /dev/null
+++ b/tests/qapi-schema/flat-union-optional-discriminator.json
@@ -0,0 +1,10 @@
+# we require the discriminator to be non-optional
+{ 'enum': 'Enum', 'data': [ 'one', 'two' ] }
+{ 'struct': 'Base',
+  'data': { '*switch': 'Enum' } }
+{ 'struct': 'Branch', 'data': { 'name': 'str' } }
+{ 'union': 'MyUnion',
+  'base': 'Base',
+  'discriminator': '*switch',
+  'data': { 'one': 'Branch',
+            'two': 'Branch' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/flat-union-optional-discriminator.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/flat-union-optional-discriminator.out
diff --git a/tests/qapi-schema/flat-union-reverse-define.json b/tests/qapi-schema/flat-union-reverse-define.json
deleted file mode 100644
index 9ea7e72..0000000
--- a/tests/qapi-schema/flat-union-reverse-define.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{ 'union': 'TestUnion',
-  'base': 'TestBase',
-  'discriminator': 'enum1',
-  'data': { 'value1': 'TestTypeA',
-            'value2': 'TestTypeB' } }
-
-{ 'type': 'TestBase',
-  'data': { 'enum1': 'TestEnum' } }
-
-{ 'enum': 'TestEnum',
-  'data': [ 'value1', 'value2' ] }
-
-{ 'type': 'TestTypeA',
-  'data': { 'string': 'str' } }
-
-{ 'type': 'TestTypeB',
-  'data': { 'integer': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.out b/tests/qapi-schema/flat-union-reverse-define.out
deleted file mode 100644
index 03c952e..0000000
--- a/tests/qapi-schema/flat-union-reverse-define.out
+++ /dev/null
@@ -1,9 +0,0 @@
-[OrderedDict([('union', 'TestUnion'), ('base', 'TestBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'TestTypeA'), ('value2', 'TestTypeB')]))]),
- OrderedDict([('type', 'TestBase'), ('data', OrderedDict([('enum1', 'TestEnum')]))]),
- OrderedDict([('enum', 'TestEnum'), ('data', ['value1', 'value2'])]),
- OrderedDict([('type', 'TestTypeA'), ('data', OrderedDict([('string', 'str')]))]),
- OrderedDict([('type', 'TestTypeB'), ('data', OrderedDict([('integer', 'int')]))])]
-[{'enum_name': 'TestEnum', 'enum_values': ['value1', 'value2']}]
-[OrderedDict([('type', 'TestBase'), ('data', OrderedDict([('enum1', 'TestEnum')]))]),
- OrderedDict([('type', 'TestTypeA'), ('data', OrderedDict([('string', 'str')]))]),
- OrderedDict([('type', 'TestTypeB'), ('data', OrderedDict([('integer', 'int')]))])]
diff --git a/tests/qapi-schema/flat-union-string-discriminator.json b/tests/qapi-schema/flat-union-string-discriminator.json
index e966aeb..8af6033 100644
--- a/tests/qapi-schema/flat-union-string-discriminator.json
+++ b/tests/qapi-schema/flat-union-string-discriminator.json
@@ -1,13 +1,13 @@
 { 'enum': 'TestEnum',
   'data': [ 'value1', 'value2' ] }
 
-{ 'type': 'TestBase',
+{ 'struct': 'TestBase',
   'data': { 'enum1': 'TestEnum', 'kind': 'str' } }
 
-{ 'type': 'TestTypeA',
+{ 'struct': 'TestTypeA',
   'data': { 'string': 'str' } }
 
-{ 'type': 'TestTypeB',
+{ 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 
 { 'union': 'TestUnion',
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/ident-with-escape.err
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/ident-with-escape.err
diff --git a/tests/qapi-schema/flat-union-reverse-define.exit b/tests/qapi-schema/ident-with-escape.exit
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.exit
copy to tests/qapi-schema/ident-with-escape.exit
diff --git a/tests/qapi-schema/ident-with-escape.json b/tests/qapi-schema/ident-with-escape.json
new file mode 100644
index 0000000..5661750
--- /dev/null
+++ b/tests/qapi-schema/ident-with-escape.json
@@ -0,0 +1,4 @@
+# we allow escape sequences in strings, if they map back to ASCII
+# { 'command': 'fooA', 'data': { 'bar1': 'str' } }
+{ 'c\u006fmmand': '\u0066\u006f\u006FA',
+  'd\u0061ta': { '\u0062\u0061\u00721': '\u0073\u0074\u0072' } }
diff --git a/tests/qapi-schema/ident-with-escape.out b/tests/qapi-schema/ident-with-escape.out
new file mode 100644
index 0000000..1d2722c
--- /dev/null
+++ b/tests/qapi-schema/ident-with-escape.out
@@ -0,0 +1,7 @@
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+command fooA q_obj_fooA-arg -> None
+   gen=True success_response=True boxed=False
+object q_empty
+object q_obj_fooA-arg
+    member bar1: str optional=False
diff --git a/tests/qapi-schema/include-cycle.err b/tests/qapi-schema/include-cycle.err
index 602cf62..bdcd07d 100644
--- a/tests/qapi-schema/include-cycle.err
+++ b/tests/qapi-schema/include-cycle.err
@@ -1,3 +1,3 @@
 In file included from tests/qapi-schema/include-cycle.json:1:
-In file included from include-cycle-b.json:1:
-include-cycle-c.json:1: Inclusion loop for include-cycle.json
+In file included from tests/qapi-schema/include-cycle-b.json:1:
+tests/qapi-schema/include-cycle-c.json:1: Inclusion loop for include-cycle.json
diff --git a/tests/qapi-schema/include-nested-err.err b/tests/qapi-schema/include-nested-err.err
index 1dacbda..1b7b227 100644
--- a/tests/qapi-schema/include-nested-err.err
+++ b/tests/qapi-schema/include-nested-err.err
@@ -1,2 +1,2 @@
 In file included from tests/qapi-schema/include-nested-err.json:1:
-missing-colon.json:1:10: Expected ":"
+tests/qapi-schema/missing-colon.json:1:10: Expected ":"
diff --git a/tests/qapi-schema/include-non-file.err b/tests/qapi-schema/include-non-file.err
index 9658c78..faae1ea 100644
--- a/tests/qapi-schema/include-non-file.err
+++ b/tests/qapi-schema/include-non-file.err
@@ -1 +1 @@
-tests/qapi-schema/include-non-file.json:1: Expected a file name (string), got: ['foo', 'bar']
+tests/qapi-schema/include-non-file.json:1: Value of 'include' must be a string
diff --git a/tests/qapi-schema/include-non-file.json b/tests/qapi-schema/include-non-file.json
index cd43c3f..4711aa4 100644
--- a/tests/qapi-schema/include-non-file.json
+++ b/tests/qapi-schema/include-non-file.json
@@ -1 +1 @@
-{ 'include': [ 'foo', 'bar' ] }
+{ 'include': {} }
diff --git a/tests/qapi-schema/include-relpath.out b/tests/qapi-schema/include-relpath.out
index 4ce3dcf..5d7c13c 100644
--- a/tests/qapi-schema/include-relpath.out
+++ b/tests/qapi-schema/include-relpath.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/tests/qapi-schema/include-repetition.out b/tests/qapi-schema/include-repetition.out
index 4ce3dcf..5d7c13c 100644
--- a/tests/qapi-schema/include-repetition.out
+++ b/tests/qapi-schema/include-repetition.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/tests/qapi-schema/include-simple.out b/tests/qapi-schema/include-simple.out
index 4ce3dcf..5d7c13c 100644
--- a/tests/qapi-schema/include-simple.out
+++ b/tests/qapi-schema/include-simple.out
@@ -1,3 +1,4 @@
-[OrderedDict([('enum', 'Status'), ('data', ['good', 'bad', 'ugly'])])]
-[{'enum_name': 'Status', 'enum_values': ['good', 'bad', 'ugly']}]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+enum Status ['good', 'bad', 'ugly']
+object q_empty
diff --git a/tests/qapi-schema/indented-expr.json b/tests/qapi-schema/indented-expr.json
index d80af60..7115d31 100644
--- a/tests/qapi-schema/indented-expr.json
+++ b/tests/qapi-schema/indented-expr.json
@@ -1,2 +1,2 @@
-{ 'id' : 'eins' }
- { 'id' : 'zwei' }
+{ 'command' : 'eins' }
+ { 'command' : 'zwei' }
diff --git a/tests/qapi-schema/indented-expr.out b/tests/qapi-schema/indented-expr.out
index 98af89a..e8171c9 100644
--- a/tests/qapi-schema/indented-expr.out
+++ b/tests/qapi-schema/indented-expr.out
@@ -1,3 +1,7 @@
-[OrderedDict([('id', 'eins')]), OrderedDict([('id', 'zwei')])]
-[]
-[]
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+command eins None -> None
+   gen=True success_response=True boxed=False
+object q_empty
+command zwei None -> None
+   gen=True success_response=True boxed=False
diff --git a/tests/qapi-schema/leading-comma-list.err b/tests/qapi-schema/leading-comma-list.err
new file mode 100644
index 0000000..f5c870b
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.err
@@ -0,0 +1 @@
+tests/qapi-schema/leading-comma-list.json:2:13: Expected "{", "[", "]", string, boolean or "null"
diff --git a/tests/qapi-schema/leading-comma-list.exit b/tests/qapi-schema/leading-comma-list.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/leading-comma-list.json b/tests/qapi-schema/leading-comma-list.json
new file mode 100644
index 0000000..c5ba501
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-list.json
@@ -0,0 +1,2 @@
+{ 'enum': 'Status',
+  'data': [ , 'good', 'bad', 'ugly' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/leading-comma-list.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/leading-comma-list.out
diff --git a/tests/qapi-schema/leading-comma-object.err b/tests/qapi-schema/leading-comma-object.err
new file mode 100644
index 0000000..f767b95
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.err
@@ -0,0 +1 @@
+tests/qapi-schema/leading-comma-object.json:1:3: Expected string or "}"
diff --git a/tests/qapi-schema/leading-comma-object.exit b/tests/qapi-schema/leading-comma-object.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/leading-comma-object.json b/tests/qapi-schema/leading-comma-object.json
new file mode 100644
index 0000000..c89023f
--- /dev/null
+++ b/tests/qapi-schema/leading-comma-object.json
@@ -0,0 +1,2 @@
+{ , 'enum': 'Status',
+  'data': [ 'good', 'bad', 'ugly' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/leading-comma-object.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/leading-comma-object.out
diff --git a/tests/qapi-schema/missing-type.err b/tests/qapi-schema/missing-type.err
new file mode 100644
index 0000000..b3e7b14
--- /dev/null
+++ b/tests/qapi-schema/missing-type.err
@@ -0,0 +1 @@
+tests/qapi-schema/missing-type.json:2: Expression is missing metatype
diff --git a/tests/qapi-schema/missing-type.exit b/tests/qapi-schema/missing-type.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/missing-type.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/missing-type.json b/tests/qapi-schema/missing-type.json
new file mode 100644
index 0000000..ff5349d
--- /dev/null
+++ b/tests/qapi-schema/missing-type.json
@@ -0,0 +1,2 @@
+# we reject an expression with missing metatype
+{ 'data': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/missing-type.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/missing-type.out
diff --git a/tests/qapi-schema/nested-struct-data.err b/tests/qapi-schema/nested-struct-data.err
new file mode 100644
index 0000000..da767ba
--- /dev/null
+++ b/tests/qapi-schema/nested-struct-data.err
@@ -0,0 +1 @@
+tests/qapi-schema/nested-struct-data.json:2: Member 'a' of 'data' for command 'foo' should be a type name
diff --git a/tests/qapi-schema/nested-struct-data.exit b/tests/qapi-schema/nested-struct-data.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/nested-struct-data.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/nested-struct-data.json b/tests/qapi-schema/nested-struct-data.json
new file mode 100644
index 0000000..efbe773
--- /dev/null
+++ b/tests/qapi-schema/nested-struct-data.json
@@ -0,0 +1,3 @@
+# inline subtypes collide with our desired future use of defaults
+{ 'command': 'foo',
+  'data': { 'a' : { 'string' : 'str', 'integer': 'int' }, 'b' : 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/nested-struct-data.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/nested-struct-data.out
diff --git a/tests/qapi-schema/qapi-schema-test.json b/tests/qapi-schema/qapi-schema-test.json
index d43b5fd..1719463 100644
--- a/tests/qapi-schema/qapi-schema-test.json
+++ b/tests/qapi-schema/qapi-schema-test.json
@@ -1,65 +1,103 @@
 # *-*- Mode: Python -*-*
 
+# This file is a stress test of supported qapi constructs that must
+# parse and compile correctly.
+
+{ 'struct': 'TestStruct',
+  'data': { 'integer': 'int', 'boolean': 'bool', 'string': 'str' } }
+
 # for testing enums
-{ 'enum': 'EnumOne',
-  'data': [ 'value1', 'value2', 'value3' ] }
-{ 'type': 'NestedEnumsOne',
-  'data': { 'enum1': 'EnumOne', '*enum2': 'EnumOne', 'enum3': 'EnumOne', '*enum4': 'EnumOne' } }
+{ 'struct': 'NestedEnumsOne',
+  'data': { 'enum1': 'EnumOne',   # Intentional forward reference
+            '*enum2': 'EnumOne', 'enum3': 'EnumOne', '*enum4': 'EnumOne' } }
+
+# An empty enum, although unusual, is currently acceptable
+{ 'enum': 'MyEnum', 'data': [ ] }
+
+# Likewise for an empty struct, including an empty base
+{ 'struct': 'Empty1', 'data': { } }
+{ 'struct': 'Empty2', 'base': 'Empty1', 'data': { } }
+
+{ 'command': 'user_def_cmd0', 'data': 'Empty2', 'returns': 'Empty2' }
+
+# for testing override of default naming heuristic
+{ 'enum': 'QEnumTwo',
+  'prefix': 'QENUM_TWO',
+  'data': [ 'value1', 'value2' ] }
 
 # for testing nested structs
-{ 'type': 'UserDefZero',
+{ 'struct': 'UserDefOne',
+  'base': 'UserDefZero',        # intentional forward reference
+  'data': { 'string': 'str',
+            '*enum1': 'EnumOne' } }   # intentional forward reference
+
+{ 'enum': 'EnumOne',
+  'data': [ 'value1', 'value2', 'value3' ] }
+
+{ 'struct': 'UserDefZero',
   'data': { 'integer': 'int' } }
 
-{ 'type': 'UserDefOne',
-  'base': 'UserDefZero',
-  'data': { 'string': 'str', '*enum1': 'EnumOne' } }
+{ 'struct': 'UserDefTwoDictDict',
+  'data': { 'userdef': 'UserDefOne', 'string': 'str' } }
 
-{ 'type': 'UserDefTwo',
-  'data': { 'string': 'str',
-            'dict': { 'string': 'str',
-                      'dict': { 'userdef': 'UserDefOne', 'string': 'str' },
-                      '*dict2': { 'userdef': 'UserDefOne', 'string': 'str' } } } }
+{ 'struct': 'UserDefTwoDict',
+  'data': { 'string1': 'str',
+            'dict2': 'UserDefTwoDictDict',
+            '*dict3': 'UserDefTwoDictDict' } }
 
-{ 'type': 'UserDefNested',
+{ 'struct': 'UserDefTwo',
   'data': { 'string0': 'str',
-            'dict1': { 'string1': 'str',
-                       'dict2': { 'userdef1': 'UserDefOne', 'string2': 'str' },
-                       '*dict3': { 'userdef2': 'UserDefOne', 'string3': 'str' } } } }
+            'dict1': 'UserDefTwoDict' } }
+
+# dummy struct to force generation of array types not otherwise mentioned
+{ 'struct': 'ForceArrays',
+  'data': { 'unused1':['UserDefOne'], 'unused2':['UserDefTwo'],
+            'unused3':['TestStruct'] } }
 
 # for testing unions
-{ 'type': 'UserDefA',
-  'data': { 'boolean': 'bool' } }
+# Among other things, test that a name collision between branches does
+# not cause any problems (since only one branch can be in use at a time),
+# by intentionally using two branches that both have a C member 'a_b'
+{ 'struct': 'UserDefA',
+  'data': { 'boolean': 'bool', '*a_b': 'int' } }
 
-{ 'type': 'UserDefB',
-  'data': { 'integer': 'int' } }
-
-{ 'type': 'UserDefC',
-  'data': { 'string1': 'str', 'string2': 'str' } }
-
-{ 'union': 'UserDefUnion',
-  'base': 'UserDefZero',
-  'data': { 'a' : 'UserDefA', 'b' : 'UserDefB' } }
-
-{ 'type': 'UserDefUnionBase',
-  'data': { 'string': 'str', 'enum1': 'EnumOne' } }
+{ 'struct': 'UserDefB',
+  'data': { 'intb': 'int', '*a-b': 'bool' } }
 
 { 'union': 'UserDefFlatUnion',
-  'base': 'UserDefUnionBase',
+  'base': 'UserDefUnionBase',   # intentional forward reference
   'discriminator': 'enum1',
-  'data': { 'value1' : 'UserDefA', 'value2' : 'UserDefB', 'value3' : 'UserDefB' } }
-# FIXME generated struct UserDefFlatUnion has members for direct base
-# UserDefOne, but lacks members for indirect base UserDefZero
+  'data': { 'value1' : 'UserDefA',
+            'value2' : 'UserDefB',
+            'value3' : 'UserDefB' } }
 
-# this variant of UserDefFlatUnion defaults to a union that uses fields with
+{ 'struct': 'UserDefUnionBase',
+  'base': 'UserDefZero',
+  'data': { 'string': 'str', 'enum1': 'EnumOne' } }
+
+# this variant of UserDefFlatUnion defaults to a union that uses members with
 # allocated types to test corner cases in the cleanup/dealloc visitor
 { 'union': 'UserDefFlatUnion2',
-  'base': 'UserDefUnionBase',
+  'base': { '*integer': 'int', 'string': 'str', 'enum1': 'QEnumTwo' },
   'discriminator': 'enum1',
-  'data': { 'value1' : 'UserDefC', 'value2' : 'UserDefB', 'value3' : 'UserDefA' } }
+  'data': { 'value1' : 'UserDefC', # intentional forward reference
+            'value2' : 'UserDefB' } }
 
-{ 'union': 'UserDefAnonUnion',
-  'discriminator': {},
-  'data': { 'uda': 'UserDefA', 's': 'str', 'i': 'int' } }
+{ 'struct': 'WrapAlternate',
+  'data': { 'alt': 'UserDefAlternate' } }
+{ 'alternate': 'UserDefAlternate',
+  'data': { 'udfu': 'UserDefFlatUnion', 's': 'str', 'i': 'int' } }
+
+{ 'struct': 'UserDefC',
+  'data': { 'string1': 'str', 'string2': 'str' } }
+
+# for testing use of 'number' within alternates
+{ 'alternate': 'AltStrBool', 'data': { 's': 'str', 'b': 'bool' } }
+{ 'alternate': 'AltStrNum', 'data': { 's': 'str', 'n': 'number' } }
+{ 'alternate': 'AltNumStr', 'data': { 'n': 'number', 's': 'str' } }
+{ 'alternate': 'AltStrInt', 'data': { 's': 'str', 'i': 'int' } }
+{ 'alternate': 'AltIntNum', 'data': { 'i': 'int', 'n': 'number' } }
+{ 'alternate': 'AltNumInt', 'data': { 'n': 'number', 'i': 'int' } }
 
 # for testing native lists
 { 'union': 'UserDefNativeListUnion',
@@ -74,7 +112,9 @@
             'u64': ['uint64'],
             'number': ['number'],
             'boolean': ['bool'],
-            'string': ['str'] } }
+            'string': ['str'],
+            'sizes': ['size'],
+            'any': ['any'] } }
 
 # testing commands
 { 'command': 'user_def_cmd', 'data': {} }
@@ -82,8 +122,13 @@
 { 'command': 'user_def_cmd2',
   'data': {'ud1a': 'UserDefOne', '*ud1b': 'UserDefOne'},
   'returns': 'UserDefTwo' }
-{ 'command': 'user_def_cmd3', 'data': {'a': 'int', '*b': 'int' },
+
+# Returning a non-dictionary requires a name from the whitelist
+{ 'command': 'guest-get-time', 'data': {'a': 'int', '*b': 'int' },
   'returns': 'int' }
+{ 'command': 'guest-sync', 'data': { 'arg': 'any' }, 'returns': 'any' }
+{ 'command': 'boxed-struct', 'boxed': true, 'data': 'UserDefZero' }
+{ 'command': 'boxed-union', 'data': 'UserDefNativeListUnion', 'boxed': true }
 
 # For testing integer range flattening in opts-visitor. The following schema
 # corresponds to the option format:
@@ -92,7 +137,7 @@
 #
 # For simplicity, this example doesn't use [type=]discriminator nor optargs
 # specific to discriminator values.
-{ 'type': 'UserDefOptions',
+{ 'struct': 'UserDefOptions',
   'data': {
     '*i64' : [ 'int'    ],
     '*u64' : [ 'uint64' ],
@@ -101,7 +146,7 @@
     '*u64x':   'uint64'  } }
 
 # testing event
-{ 'type': 'EventStructOne',
+{ 'struct': 'EventStructOne',
   'data': { 'struct1': 'UserDefOne', 'string': 'str', '*enum2': 'EnumOne' } }
 
 { 'event': 'EVENT_A' }
@@ -111,3 +156,26 @@
   'data': { '*a': 'int', '*b': 'UserDefOne', 'c': 'str' } }
 { 'event': 'EVENT_D',
   'data': { 'a' : 'EventStructOne', 'b' : 'str', '*c': 'str', '*enum3': 'EnumOne' } }
+{ 'event': 'EVENT_E', 'boxed': true, 'data': 'UserDefZero' }
+{ 'event': 'EVENT_F', 'boxed': true, 'data': 'UserDefAlternate' }
+
+# test that we correctly compile downstream extensions, as well as munge
+# ticklish names
+{ 'enum': '__org.qemu_x-Enum', 'data': [ '__org.qemu_x-value' ] }
+{ 'struct': '__org.qemu_x-Base',
+  'data': { '__org.qemu_x-member1': '__org.qemu_x-Enum' } }
+{ 'struct': '__org.qemu_x-Struct', 'base': '__org.qemu_x-Base',
+  'data': { '__org.qemu_x-member2': 'str', '*wchar-t': 'int' } }
+{ 'union': '__org.qemu_x-Union1', 'data': { '__org.qemu_x-branch': 'str' } }
+{ 'struct': '__org.qemu_x-Struct2',
+  'data': { 'array': ['__org.qemu_x-Union1'] } }
+{ 'union': '__org.qemu_x-Union2', 'base': '__org.qemu_x-Base',
+  'discriminator': '__org.qemu_x-member1',
+  'data': { '__org.qemu_x-value': '__org.qemu_x-Struct2' } }
+{ 'alternate': '__org.qemu_x-Alt',
+  'data': { '__org.qemu_x-branch': 'str', 'b': '__org.qemu_x-Base' } }
+{ 'event': '__ORG.QEMU_X-EVENT', 'data': '__org.qemu_x-Struct' }
+{ 'command': '__org.qemu_x-command',
+  'data': { 'a': ['__org.qemu_x-Enum'], 'b': ['__org.qemu_x-Struct'],
+            'c': '__org.qemu_x-Union2', 'd': '__org.qemu_x-Alt' },
+  'returns': '__org.qemu_x-Union1' }
diff --git a/tests/qapi-schema/qapi-schema-test.out b/tests/qapi-schema/qapi-schema-test.out
index 08d7304..9d99c4e 100644
--- a/tests/qapi-schema/qapi-schema-test.out
+++ b/tests/qapi-schema/qapi-schema-test.out
@@ -1,40 +1,234 @@
-[OrderedDict([('enum', 'EnumOne'), ('data', ['value1', 'value2', 'value3'])]),
- OrderedDict([('type', 'NestedEnumsOne'), ('data', OrderedDict([('enum1', 'EnumOne'), ('*enum2', 'EnumOne'), ('enum3', 'EnumOne'), ('*enum4', 'EnumOne')]))]),
- OrderedDict([('type', 'UserDefZero'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('type', 'UserDefOne'), ('base', 'UserDefZero'), ('data', OrderedDict([('string', 'str'), ('*enum1', 'EnumOne')]))]),
- OrderedDict([('type', 'UserDefTwo'), ('data', OrderedDict([('string', 'str'), ('dict', OrderedDict([('string', 'str'), ('dict', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')])), ('*dict2', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')]))]))]))]),
- OrderedDict([('type', 'UserDefNested'), ('data', OrderedDict([('string0', 'str'), ('dict1', OrderedDict([('string1', 'str'), ('dict2', OrderedDict([('userdef1', 'UserDefOne'), ('string2', 'str')])), ('*dict3', OrderedDict([('userdef2', 'UserDefOne'), ('string3', 'str')]))]))]))]),
- OrderedDict([('type', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
- OrderedDict([('type', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('type', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
- OrderedDict([('union', 'UserDefUnion'), ('base', 'UserDefZero'), ('data', OrderedDict([('a', 'UserDefA'), ('b', 'UserDefB')]))]),
- OrderedDict([('type', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
- OrderedDict([('union', 'UserDefFlatUnion'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefA'), ('value2', 'UserDefB'), ('value3', 'UserDefB')]))]),
- OrderedDict([('union', 'UserDefFlatUnion2'), ('base', 'UserDefUnionBase'), ('discriminator', 'enum1'), ('data', OrderedDict([('value1', 'UserDefC'), ('value2', 'UserDefB'), ('value3', 'UserDefA')]))]),
- OrderedDict([('union', 'UserDefAnonUnion'), ('discriminator', OrderedDict()), ('data', OrderedDict([('uda', 'UserDefA'), ('s', 'str'), ('i', 'int')]))]),
- OrderedDict([('union', 'UserDefNativeListUnion'), ('data', OrderedDict([('integer', ['int']), ('s8', ['int8']), ('s16', ['int16']), ('s32', ['int32']), ('s64', ['int64']), ('u8', ['uint8']), ('u16', ['uint16']), ('u32', ['uint32']), ('u64', ['uint64']), ('number', ['number']), ('boolean', ['bool']), ('string', ['str'])]))]),
- OrderedDict([('command', 'user_def_cmd'), ('data', OrderedDict())]),
- OrderedDict([('command', 'user_def_cmd1'), ('data', OrderedDict([('ud1a', 'UserDefOne')]))]),
- OrderedDict([('command', 'user_def_cmd2'), ('data', OrderedDict([('ud1a', 'UserDefOne'), ('*ud1b', 'UserDefOne')])), ('returns', 'UserDefTwo')]),
- OrderedDict([('command', 'user_def_cmd3'), ('data', OrderedDict([('a', 'int'), ('*b', 'int')])), ('returns', 'int')]),
- OrderedDict([('type', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
- OrderedDict([('type', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))]),
- OrderedDict([('event', 'EVENT_A')]),
- OrderedDict([('event', 'EVENT_B'), ('data', OrderedDict())]),
- OrderedDict([('event', 'EVENT_C'), ('data', OrderedDict([('*a', 'int'), ('*b', 'UserDefOne'), ('c', 'str')]))]),
- OrderedDict([('event', 'EVENT_D'), ('data', OrderedDict([('a', 'EventStructOne'), ('b', 'str'), ('*c', 'str'), ('*enum3', 'EnumOne')]))])]
-[{'enum_name': 'EnumOne', 'enum_values': ['value1', 'value2', 'value3']},
- {'enum_name': 'UserDefUnionKind', 'enum_values': None},
- {'enum_name': 'UserDefAnonUnionKind', 'enum_values': None},
- {'enum_name': 'UserDefNativeListUnionKind', 'enum_values': None}]
-[OrderedDict([('type', 'NestedEnumsOne'), ('data', OrderedDict([('enum1', 'EnumOne'), ('*enum2', 'EnumOne'), ('enum3', 'EnumOne'), ('*enum4', 'EnumOne')]))]),
- OrderedDict([('type', 'UserDefZero'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('type', 'UserDefOne'), ('base', 'UserDefZero'), ('data', OrderedDict([('string', 'str'), ('*enum1', 'EnumOne')]))]),
- OrderedDict([('type', 'UserDefTwo'), ('data', OrderedDict([('string', 'str'), ('dict', OrderedDict([('string', 'str'), ('dict', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')])), ('*dict2', OrderedDict([('userdef', 'UserDefOne'), ('string', 'str')]))]))]))]),
- OrderedDict([('type', 'UserDefNested'), ('data', OrderedDict([('string0', 'str'), ('dict1', OrderedDict([('string1', 'str'), ('dict2', OrderedDict([('userdef1', 'UserDefOne'), ('string2', 'str')])), ('*dict3', OrderedDict([('userdef2', 'UserDefOne'), ('string3', 'str')]))]))]))]),
- OrderedDict([('type', 'UserDefA'), ('data', OrderedDict([('boolean', 'bool')]))]),
- OrderedDict([('type', 'UserDefB'), ('data', OrderedDict([('integer', 'int')]))]),
- OrderedDict([('type', 'UserDefC'), ('data', OrderedDict([('string1', 'str'), ('string2', 'str')]))]),
- OrderedDict([('type', 'UserDefUnionBase'), ('data', OrderedDict([('string', 'str'), ('enum1', 'EnumOne')]))]),
- OrderedDict([('type', 'UserDefOptions'), ('data', OrderedDict([('*i64', ['int']), ('*u64', ['uint64']), ('*u16', ['uint16']), ('*i64x', 'int'), ('*u64x', 'uint64')]))]),
- OrderedDict([('type', 'EventStructOne'), ('data', OrderedDict([('struct1', 'UserDefOne'), ('string', 'str'), ('*enum2', 'EnumOne')]))])]
+alternate AltIntNum
+    tag type
+    case i: int
+    case n: number
+alternate AltNumInt
+    tag type
+    case n: number
+    case i: int
+alternate AltNumStr
+    tag type
+    case n: number
+    case s: str
+alternate AltStrBool
+    tag type
+    case s: str
+    case b: bool
+alternate AltStrInt
+    tag type
+    case s: str
+    case i: int
+alternate AltStrNum
+    tag type
+    case s: str
+    case n: number
+event EVENT_A None
+   boxed=False
+event EVENT_B None
+   boxed=False
+event EVENT_C q_obj_EVENT_C-arg
+   boxed=False
+event EVENT_D q_obj_EVENT_D-arg
+   boxed=False
+event EVENT_E UserDefZero
+   boxed=True
+event EVENT_F UserDefAlternate
+   boxed=True
+object Empty1
+object Empty2
+    base Empty1
+enum EnumOne ['value1', 'value2', 'value3']
+object EventStructOne
+    member struct1: UserDefOne optional=False
+    member string: str optional=False
+    member enum2: EnumOne optional=True
+object ForceArrays
+    member unused1: UserDefOneList optional=False
+    member unused2: UserDefTwoList optional=False
+    member unused3: TestStructList optional=False
+enum MyEnum []
+object NestedEnumsOne
+    member enum1: EnumOne optional=False
+    member enum2: EnumOne optional=True
+    member enum3: EnumOne optional=False
+    member enum4: EnumOne optional=True
+enum QEnumTwo ['value1', 'value2']
+    prefix QENUM_TWO
+enum QType ['none', 'qnull', 'qint', 'qstring', 'qdict', 'qlist', 'qfloat', 'qbool']
+    prefix QTYPE
+object TestStruct
+    member integer: int optional=False
+    member boolean: bool optional=False
+    member string: str optional=False
+object UserDefA
+    member boolean: bool optional=False
+    member a_b: int optional=True
+alternate UserDefAlternate
+    tag type
+    case udfu: UserDefFlatUnion
+    case s: str
+    case i: int
+object UserDefB
+    member intb: int optional=False
+    member a-b: bool optional=True
+object UserDefC
+    member string1: str optional=False
+    member string2: str optional=False
+object UserDefFlatUnion
+    base UserDefUnionBase
+    tag enum1
+    case value1: UserDefA
+    case value2: UserDefB
+    case value3: UserDefB
+object UserDefFlatUnion2
+    base q_obj_UserDefFlatUnion2-base
+    tag enum1
+    case value1: UserDefC
+    case value2: UserDefB
+object UserDefNativeListUnion
+    member type: UserDefNativeListUnionKind optional=False
+    tag type
+    case integer: q_obj_intList-wrapper
+    case s8: q_obj_int8List-wrapper
+    case s16: q_obj_int16List-wrapper
+    case s32: q_obj_int32List-wrapper
+    case s64: q_obj_int64List-wrapper
+    case u8: q_obj_uint8List-wrapper
+    case u16: q_obj_uint16List-wrapper
+    case u32: q_obj_uint32List-wrapper
+    case u64: q_obj_uint64List-wrapper
+    case number: q_obj_numberList-wrapper
+    case boolean: q_obj_boolList-wrapper
+    case string: q_obj_strList-wrapper
+    case sizes: q_obj_sizeList-wrapper
+    case any: q_obj_anyList-wrapper
+enum UserDefNativeListUnionKind ['integer', 's8', 's16', 's32', 's64', 'u8', 'u16', 'u32', 'u64', 'number', 'boolean', 'string', 'sizes', 'any']
+object UserDefOne
+    base UserDefZero
+    member string: str optional=False
+    member enum1: EnumOne optional=True
+object UserDefOptions
+    member i64: intList optional=True
+    member u64: uint64List optional=True
+    member u16: uint16List optional=True
+    member i64x: int optional=True
+    member u64x: uint64 optional=True
+object UserDefTwo
+    member string0: str optional=False
+    member dict1: UserDefTwoDict optional=False
+object UserDefTwoDict
+    member string1: str optional=False
+    member dict2: UserDefTwoDictDict optional=False
+    member dict3: UserDefTwoDictDict optional=True
+object UserDefTwoDictDict
+    member userdef: UserDefOne optional=False
+    member string: str optional=False
+object UserDefUnionBase
+    base UserDefZero
+    member string: str optional=False
+    member enum1: EnumOne optional=False
+object UserDefZero
+    member integer: int optional=False
+object WrapAlternate
+    member alt: UserDefAlternate optional=False
+event __ORG.QEMU_X-EVENT __org.qemu_x-Struct
+   boxed=False
+alternate __org.qemu_x-Alt
+    tag type
+    case __org.qemu_x-branch: str
+    case b: __org.qemu_x-Base
+object __org.qemu_x-Base
+    member __org.qemu_x-member1: __org.qemu_x-Enum optional=False
+enum __org.qemu_x-Enum ['__org.qemu_x-value']
+object __org.qemu_x-Struct
+    base __org.qemu_x-Base
+    member __org.qemu_x-member2: str optional=False
+    member wchar-t: int optional=True
+object __org.qemu_x-Struct2
+    member array: __org.qemu_x-Union1List optional=False
+object __org.qemu_x-Union1
+    member type: __org.qemu_x-Union1Kind optional=False
+    tag type
+    case __org.qemu_x-branch: q_obj_str-wrapper
+enum __org.qemu_x-Union1Kind ['__org.qemu_x-branch']
+object __org.qemu_x-Union2
+    base __org.qemu_x-Base
+    tag __org.qemu_x-member1
+    case __org.qemu_x-value: __org.qemu_x-Struct2
+command __org.qemu_x-command q_obj___org.qemu_x-command-arg -> __org.qemu_x-Union1
+   gen=True success_response=True boxed=False
+command boxed-struct UserDefZero -> None
+   gen=True success_response=True boxed=True
+command boxed-union UserDefNativeListUnion -> None
+   gen=True success_response=True boxed=True
+command guest-get-time q_obj_guest-get-time-arg -> int
+   gen=True success_response=True boxed=False
+command guest-sync q_obj_guest-sync-arg -> any
+   gen=True success_response=True boxed=False
+object q_empty
+object q_obj_EVENT_C-arg
+    member a: int optional=True
+    member b: UserDefOne optional=True
+    member c: str optional=False
+object q_obj_EVENT_D-arg
+    member a: EventStructOne optional=False
+    member b: str optional=False
+    member c: str optional=True
+    member enum3: EnumOne optional=True
+object q_obj_UserDefFlatUnion2-base
+    member integer: int optional=True
+    member string: str optional=False
+    member enum1: QEnumTwo optional=False
+object q_obj___org.qemu_x-command-arg
+    member a: __org.qemu_x-EnumList optional=False
+    member b: __org.qemu_x-StructList optional=False
+    member c: __org.qemu_x-Union2 optional=False
+    member d: __org.qemu_x-Alt optional=False
+object q_obj_anyList-wrapper
+    member data: anyList optional=False
+object q_obj_boolList-wrapper
+    member data: boolList optional=False
+object q_obj_guest-get-time-arg
+    member a: int optional=False
+    member b: int optional=True
+object q_obj_guest-sync-arg
+    member arg: any optional=False
+object q_obj_int16List-wrapper
+    member data: int16List optional=False
+object q_obj_int32List-wrapper
+    member data: int32List optional=False
+object q_obj_int64List-wrapper
+    member data: int64List optional=False
+object q_obj_int8List-wrapper
+    member data: int8List optional=False
+object q_obj_intList-wrapper
+    member data: intList optional=False
+object q_obj_numberList-wrapper
+    member data: numberList optional=False
+object q_obj_sizeList-wrapper
+    member data: sizeList optional=False
+object q_obj_str-wrapper
+    member data: str optional=False
+object q_obj_strList-wrapper
+    member data: strList optional=False
+object q_obj_uint16List-wrapper
+    member data: uint16List optional=False
+object q_obj_uint32List-wrapper
+    member data: uint32List optional=False
+object q_obj_uint64List-wrapper
+    member data: uint64List optional=False
+object q_obj_uint8List-wrapper
+    member data: uint8List optional=False
+object q_obj_user_def_cmd1-arg
+    member ud1a: UserDefOne optional=False
+object q_obj_user_def_cmd2-arg
+    member ud1a: UserDefOne optional=False
+    member ud1b: UserDefOne optional=True
+command user_def_cmd None -> None
+   gen=True success_response=True boxed=False
+command user_def_cmd0 Empty2 -> Empty2
+   gen=True success_response=True boxed=False
+command user_def_cmd1 q_obj_user_def_cmd1-arg -> None
+   gen=True success_response=True boxed=False
+command user_def_cmd2 q_obj_user_def_cmd2-arg -> UserDefTwo
+   gen=True success_response=True boxed=False
diff --git a/tests/qapi-schema/redefined-builtin.err b/tests/qapi-schema/redefined-builtin.err
new file mode 100644
index 0000000..b275722
--- /dev/null
+++ b/tests/qapi-schema/redefined-builtin.err
@@ -0,0 +1 @@
+tests/qapi-schema/redefined-builtin.json:2: built-in 'size' is already defined
diff --git a/tests/qapi-schema/redefined-builtin.exit b/tests/qapi-schema/redefined-builtin.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/redefined-builtin.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/redefined-builtin.json b/tests/qapi-schema/redefined-builtin.json
new file mode 100644
index 0000000..45b8a55
--- /dev/null
+++ b/tests/qapi-schema/redefined-builtin.json
@@ -0,0 +1,2 @@
+# we reject types that duplicate builtin names
+{ 'struct': 'size', 'data': { 'myint': 'size' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/redefined-builtin.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/redefined-builtin.out
diff --git a/tests/qapi-schema/redefined-command.err b/tests/qapi-schema/redefined-command.err
new file mode 100644
index 0000000..82ae256
--- /dev/null
+++ b/tests/qapi-schema/redefined-command.err
@@ -0,0 +1 @@
+tests/qapi-schema/redefined-command.json:3: command 'foo' is already defined
diff --git a/tests/qapi-schema/redefined-command.exit b/tests/qapi-schema/redefined-command.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/redefined-command.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/redefined-command.json b/tests/qapi-schema/redefined-command.json
new file mode 100644
index 0000000..247e401
--- /dev/null
+++ b/tests/qapi-schema/redefined-command.json
@@ -0,0 +1,3 @@
+# we reject commands defined more than once
+{ 'command': 'foo', 'data': { 'one': 'str' } }
+{ 'command': 'foo', 'data': { '*two': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/redefined-command.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/redefined-command.out
diff --git a/tests/qapi-schema/redefined-event.err b/tests/qapi-schema/redefined-event.err
new file mode 100644
index 0000000..35429cb
--- /dev/null
+++ b/tests/qapi-schema/redefined-event.err
@@ -0,0 +1 @@
+tests/qapi-schema/redefined-event.json:3: event 'EVENT_A' is already defined
diff --git a/tests/qapi-schema/redefined-event.exit b/tests/qapi-schema/redefined-event.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/redefined-event.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/redefined-event.json b/tests/qapi-schema/redefined-event.json
new file mode 100644
index 0000000..7717e91
--- /dev/null
+++ b/tests/qapi-schema/redefined-event.json
@@ -0,0 +1,3 @@
+# we reject duplicate events
+{ 'event': 'EVENT_A', 'data': { 'myint': 'int' } }
+{ 'event': 'EVENT_A', 'data': { 'myint': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/redefined-event.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/redefined-event.out
diff --git a/tests/qapi-schema/redefined-type.err b/tests/qapi-schema/redefined-type.err
new file mode 100644
index 0000000..06ea78c
--- /dev/null
+++ b/tests/qapi-schema/redefined-type.err
@@ -0,0 +1 @@
+tests/qapi-schema/redefined-type.json:3: struct 'foo' is already defined
diff --git a/tests/qapi-schema/redefined-type.exit b/tests/qapi-schema/redefined-type.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/redefined-type.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/redefined-type.json b/tests/qapi-schema/redefined-type.json
new file mode 100644
index 0000000..a09e768
--- /dev/null
+++ b/tests/qapi-schema/redefined-type.json
@@ -0,0 +1,3 @@
+# we reject types defined more than once
+{ 'struct': 'foo', 'data': { 'one': 'str' } }
+{ 'enum': 'foo', 'data': [ 'two' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/redefined-type.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/redefined-type.out
diff --git a/tests/qapi-schema/reserved-command-q.err b/tests/qapi-schema/reserved-command-q.err
new file mode 100644
index 0000000..f939e04
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-command-q.json:5: 'command' uses invalid name 'q-unix'
diff --git a/tests/qapi-schema/reserved-command-q.exit b/tests/qapi-schema/reserved-command-q.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-command-q.json b/tests/qapi-schema/reserved-command-q.json
new file mode 100644
index 0000000..99f8aae
--- /dev/null
+++ b/tests/qapi-schema/reserved-command-q.json
@@ -0,0 +1,5 @@
+# C entity name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'command': 'unix' }
+{ 'command': 'q-unix' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-command-q.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-command-q.out
diff --git a/tests/qapi-schema/reserved-enum-q.err b/tests/qapi-schema/reserved-enum-q.err
new file mode 100644
index 0000000..e1c3480
--- /dev/null
+++ b/tests/qapi-schema/reserved-enum-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-enum-q.json:4: Member of enum 'Foo' uses invalid name 'q-Unix'
diff --git a/tests/qapi-schema/reserved-enum-q.exit b/tests/qapi-schema/reserved-enum-q.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-enum-q.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-enum-q.json b/tests/qapi-schema/reserved-enum-q.json
new file mode 100644
index 0000000..3593a76
--- /dev/null
+++ b/tests/qapi-schema/reserved-enum-q.json
@@ -0,0 +1,4 @@
+# C entity name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'enum': 'Foo', 'data': [ 'unix', 'q-Unix' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-enum-q.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-enum-q.out
diff --git a/tests/qapi-schema/reserved-member-has.err b/tests/qapi-schema/reserved-member-has.err
new file mode 100644
index 0000000..e755771
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-has.json:5: Member of 'data' for command 'oops' uses reserved name 'has-a'
diff --git a/tests/qapi-schema/reserved-member-has.exit b/tests/qapi-schema/reserved-member-has.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-has.json b/tests/qapi-schema/reserved-member-has.json
new file mode 100644
index 0000000..45b9109
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-has.json
@@ -0,0 +1,5 @@
+# C member name collision
+# We reject names like 'has-a', because they can collide with the flag
+# for an optional 'a' in generated C.
+# TODO we could munge the optional flag name to avoid the collision.
+{ 'command': 'oops', 'data': { '*a': 'str', 'has-a': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-member-has.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-member-has.out
diff --git a/tests/qapi-schema/reserved-member-q.err b/tests/qapi-schema/reserved-member-q.err
new file mode 100644
index 0000000..f3d5dd7
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-q.json:4: Member of 'data' for struct 'Foo' uses invalid name 'q-unix'
diff --git a/tests/qapi-schema/reserved-member-q.exit b/tests/qapi-schema/reserved-member-q.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-q.json b/tests/qapi-schema/reserved-member-q.json
new file mode 100644
index 0000000..62fed8f
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-q.json
@@ -0,0 +1,4 @@
+# C member name collision
+# We reject names like 'q-unix', because they can collide with the mangled
+# name for 'unix' in generated C.
+{ 'struct': 'Foo', 'data': { 'unix':'int', 'q-unix':'bool' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-member-q.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-member-q.out
diff --git a/tests/qapi-schema/reserved-member-u.err b/tests/qapi-schema/reserved-member-u.err
new file mode 100644
index 0000000..87d4229
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-u.json:7: Member of 'data' for struct 'Oops' uses reserved name 'u'
diff --git a/tests/qapi-schema/reserved-member-u.exit b/tests/qapi-schema/reserved-member-u.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-u.json b/tests/qapi-schema/reserved-member-u.json
new file mode 100644
index 0000000..1eaf0f3
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-u.json
@@ -0,0 +1,7 @@
+# Potential C member name collision
+# We reject use of 'u' as a member name, to allow it for internal use in
+# putting union branch members in a separate namespace from QMP members.
+# This is true even for non-unions, because it is possible to convert a
+# struct to flat union while remaining backwards compatible in QMP.
+# TODO - we could munge the member name to 'q_u' to avoid the collision
+{ 'struct': 'Oops', 'data': { 'u': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-member-u.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-member-u.out
diff --git a/tests/qapi-schema/reserved-member-underscore.err b/tests/qapi-schema/reserved-member-underscore.err
new file mode 100644
index 0000000..65ff0da
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-underscore.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-member-underscore.json:4: Member of 'data' for struct 'Oops' uses invalid name '_oops'
diff --git a/tests/qapi-schema/reserved-member-underscore.exit b/tests/qapi-schema/reserved-member-underscore.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-underscore.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-member-underscore.json b/tests/qapi-schema/reserved-member-underscore.json
new file mode 100644
index 0000000..4a3a017
--- /dev/null
+++ b/tests/qapi-schema/reserved-member-underscore.json
@@ -0,0 +1,4 @@
+# C member name collision
+# We reject use of a single leading underscore in all names (names must
+# begin with a letter or a downstream extension double-underscore prefix).
+{ 'struct': 'Oops', 'data': { '_oops': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-member-underscore.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-member-underscore.out
diff --git a/tests/qapi-schema/reserved-type-kind.err b/tests/qapi-schema/reserved-type-kind.err
new file mode 100644
index 0000000..0a38efa
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-kind.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-type-kind.json:2: enum 'UnionKind' should not end in 'Kind'
diff --git a/tests/qapi-schema/reserved-type-kind.exit b/tests/qapi-schema/reserved-type-kind.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-kind.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-type-kind.json b/tests/qapi-schema/reserved-type-kind.json
new file mode 100644
index 0000000..9ecaba1
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-kind.json
@@ -0,0 +1,2 @@
+# we reject types that would conflict with implicit union enum
+{ 'enum': 'UnionKind', 'data': [ 'oops' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-type-kind.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-type-kind.out
diff --git a/tests/qapi-schema/reserved-type-list.err b/tests/qapi-schema/reserved-type-list.err
new file mode 100644
index 0000000..4510fa6
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.err
@@ -0,0 +1 @@
+tests/qapi-schema/reserved-type-list.json:5: struct 'FooList' should not end in 'List'
diff --git a/tests/qapi-schema/reserved-type-list.exit b/tests/qapi-schema/reserved-type-list.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/reserved-type-list.json b/tests/qapi-schema/reserved-type-list.json
new file mode 100644
index 0000000..98d53bf
--- /dev/null
+++ b/tests/qapi-schema/reserved-type-list.json
@@ -0,0 +1,5 @@
+# Potential C name collision
+# We reserve names ending in 'List' for use by array types.
+# TODO - we could choose array names to avoid collision with user types,
+# in order to let this compile
+{ 'struct': 'FooList', 'data': { 's': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/reserved-type-list.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/reserved-type-list.out
diff --git a/tests/qapi-schema/returns-alternate.err b/tests/qapi-schema/returns-alternate.err
new file mode 100644
index 0000000..dfbb419
--- /dev/null
+++ b/tests/qapi-schema/returns-alternate.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-alternate.json:3: 'returns' for command 'oops' cannot use alternate type 'Alt'
diff --git a/tests/qapi-schema/returns-alternate.exit b/tests/qapi-schema/returns-alternate.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/returns-alternate.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/returns-alternate.json b/tests/qapi-schema/returns-alternate.json
new file mode 100644
index 0000000..972390c
--- /dev/null
+++ b/tests/qapi-schema/returns-alternate.json
@@ -0,0 +1,3 @@
+# we reject returns if it is an alternate type
+{ 'alternate': 'Alt', 'data': { 'a': 'int', 'b': 'str' } }
+{ 'command': 'oops', 'returns': 'Alt' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/returns-alternate.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/returns-alternate.out
diff --git a/tests/qapi-schema/returns-array-bad.err b/tests/qapi-schema/returns-array-bad.err
new file mode 100644
index 0000000..138095c
--- /dev/null
+++ b/tests/qapi-schema/returns-array-bad.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-array-bad.json:2: 'returns' for command 'oops': array type must contain single type name
diff --git a/tests/qapi-schema/returns-array-bad.exit b/tests/qapi-schema/returns-array-bad.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/returns-array-bad.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/returns-array-bad.json b/tests/qapi-schema/returns-array-bad.json
new file mode 100644
index 0000000..09b0b1f
--- /dev/null
+++ b/tests/qapi-schema/returns-array-bad.json
@@ -0,0 +1,2 @@
+# we reject an array return that is not a single type
+{ 'command': 'oops', 'returns': [ 'str', 'str' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/returns-array-bad.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/returns-array-bad.out
diff --git a/tests/qapi-schema/returns-dict.err b/tests/qapi-schema/returns-dict.err
new file mode 100644
index 0000000..eb2d0c4
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-dict.json:2: 'returns' for command 'oops' should be a type name
diff --git a/tests/qapi-schema/returns-dict.exit b/tests/qapi-schema/returns-dict.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/returns-dict.json b/tests/qapi-schema/returns-dict.json
new file mode 100644
index 0000000..1cfef3e
--- /dev/null
+++ b/tests/qapi-schema/returns-dict.json
@@ -0,0 +1,2 @@
+# we reject inline struct return type
+{ 'command': 'oops', 'returns': { 'a': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/returns-dict.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/returns-dict.out
diff --git a/tests/qapi-schema/returns-unknown.err b/tests/qapi-schema/returns-unknown.err
new file mode 100644
index 0000000..1f43e3a
--- /dev/null
+++ b/tests/qapi-schema/returns-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-unknown.json:2: 'returns' for command 'oops' uses unknown type 'NoSuchType'
diff --git a/tests/qapi-schema/returns-unknown.exit b/tests/qapi-schema/returns-unknown.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/returns-unknown.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/returns-unknown.json b/tests/qapi-schema/returns-unknown.json
new file mode 100644
index 0000000..25bd498
--- /dev/null
+++ b/tests/qapi-schema/returns-unknown.json
@@ -0,0 +1,2 @@
+# we reject returns if it does not contain a known type
+{ 'command': 'oops', 'returns': 'NoSuchType' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/returns-unknown.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/returns-unknown.out
diff --git a/tests/qapi-schema/returns-whitelist.err b/tests/qapi-schema/returns-whitelist.err
new file mode 100644
index 0000000..f47c1ee
--- /dev/null
+++ b/tests/qapi-schema/returns-whitelist.err
@@ -0,0 +1 @@
+tests/qapi-schema/returns-whitelist.json:10: 'returns' for command 'no-way-this-will-get-whitelisted' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/returns-whitelist.exit b/tests/qapi-schema/returns-whitelist.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/returns-whitelist.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/returns-whitelist.json b/tests/qapi-schema/returns-whitelist.json
new file mode 100644
index 0000000..e8b3cea
--- /dev/null
+++ b/tests/qapi-schema/returns-whitelist.json
@@ -0,0 +1,11 @@
+# we enforce that 'returns' be a dict or array of dict unless whitelisted
+{ 'command': 'human-monitor-command',
+  'data': {'command-line': 'str', '*cpu-index': 'int'},
+  'returns': 'str' }
+{ 'enum': 'TpmModel', 'data': [ 'tpm-tis' ] }
+{ 'command': 'query-tpm-models', 'returns': ['TpmModel'] }
+{ 'command': 'guest-get-time',
+  'returns': 'int' }
+
+{ 'command': 'no-way-this-will-get-whitelisted',
+  'returns': [ 'int' ] }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/returns-whitelist.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/returns-whitelist.out
diff --git a/tests/qapi-schema/struct-base-clash-deep.err b/tests/qapi-schema/struct-base-clash-deep.err
new file mode 100644
index 0000000..e2d7943
--- /dev/null
+++ b/tests/qapi-schema/struct-base-clash-deep.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-base-clash-deep.json:10: 'name' (member of Sub) collides with 'name' (member of Base)
diff --git a/tests/qapi-schema/struct-base-clash-deep.exit b/tests/qapi-schema/struct-base-clash-deep.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/struct-base-clash-deep.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/struct-base-clash-deep.json b/tests/qapi-schema/struct-base-clash-deep.json
new file mode 100644
index 0000000..fa873ab
--- /dev/null
+++ b/tests/qapi-schema/struct-base-clash-deep.json
@@ -0,0 +1,12 @@
+# Reject attempts to duplicate QMP members
+# Here, 'name' would have to appear twice on the wire, locally and
+# indirectly for the grandparent base; the collision doesn't care that
+# one instance is optional.
+{ 'struct': 'Base',
+  'data': { 'name': 'str' } }
+{ 'struct': 'Mid',
+  'base': 'Base',
+  'data': { 'value': 'int' } }
+{ 'struct': 'Sub',
+  'base': 'Mid',
+  'data': { '*name': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/struct-base-clash-deep.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/struct-base-clash-deep.out
diff --git a/tests/qapi-schema/struct-base-clash.err b/tests/qapi-schema/struct-base-clash.err
new file mode 100644
index 0000000..c52f33d
--- /dev/null
+++ b/tests/qapi-schema/struct-base-clash.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-base-clash.json:5: 'name' (member of Sub) collides with 'name' (member of Base)
diff --git a/tests/qapi-schema/struct-base-clash.exit b/tests/qapi-schema/struct-base-clash.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/struct-base-clash.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/struct-base-clash.json b/tests/qapi-schema/struct-base-clash.json
new file mode 100644
index 0000000..11aec80
--- /dev/null
+++ b/tests/qapi-schema/struct-base-clash.json
@@ -0,0 +1,7 @@
+# Reject attempts to duplicate QMP members
+# Here, 'name' would have to appear twice on the wire, locally and for base.
+{ 'struct': 'Base',
+  'data': { 'name': 'str' } }
+{ 'struct': 'Sub',
+  'base': 'Base',
+  'data': { 'name': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/struct-base-clash.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/struct-base-clash.out
diff --git a/tests/qapi-schema/struct-data-invalid.err b/tests/qapi-schema/struct-data-invalid.err
new file mode 100644
index 0000000..6644f4c
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-data-invalid.json:1: 'data' for struct 'foo' should be a dictionary or type name
diff --git a/tests/qapi-schema/struct-data-invalid.exit b/tests/qapi-schema/struct-data-invalid.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/struct-data-invalid.json b/tests/qapi-schema/struct-data-invalid.json
new file mode 100644
index 0000000..9adbc3b
--- /dev/null
+++ b/tests/qapi-schema/struct-data-invalid.json
@@ -0,0 +1,2 @@
+{ 'struct': 'foo',
+  'data': false }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/struct-data-invalid.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/struct-data-invalid.out
diff --git a/tests/qapi-schema/struct-member-invalid.err b/tests/qapi-schema/struct-member-invalid.err
new file mode 100644
index 0000000..69a326d
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.err
@@ -0,0 +1 @@
+tests/qapi-schema/struct-member-invalid.json:1: Member 'a' of 'data' for struct 'foo' should be a type name
diff --git a/tests/qapi-schema/struct-member-invalid.exit b/tests/qapi-schema/struct-member-invalid.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/struct-member-invalid.json b/tests/qapi-schema/struct-member-invalid.json
new file mode 100644
index 0000000..8f172f7
--- /dev/null
+++ b/tests/qapi-schema/struct-member-invalid.json
@@ -0,0 +1,2 @@
+{ 'struct': 'foo',
+  'data': { 'a': false } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/struct-member-invalid.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/struct-member-invalid.out
diff --git a/tests/qapi-schema/test-qapi.py b/tests/qapi-schema/test-qapi.py
index 634ef2d..ef74e2c 100644
--- a/tests/qapi-schema/test-qapi.py
+++ b/tests/qapi-schema/test-qapi.py
@@ -15,11 +15,43 @@
 import os
 import sys
 
-try:
-    exprs = parse_schema(sys.argv[1])
-except SystemExit:
-    raise
 
-pprint(exprs)
-pprint(enum_types)
-pprint(struct_types)
+class QAPISchemaTestVisitor(QAPISchemaVisitor):
+    def visit_enum_type(self, name, info, values, prefix):
+        print 'enum %s %s' % (name, values)
+        if prefix:
+            print '    prefix %s' % prefix
+
+    def visit_object_type(self, name, info, base, members, variants):
+        print 'object %s' % name
+        if base:
+            print '    base %s' % base.name
+        for m in members:
+            print '    member %s: %s optional=%s' % \
+                (m.name, m.type.name, m.optional)
+        self._print_variants(variants)
+
+    def visit_alternate_type(self, name, info, variants):
+        print 'alternate %s' % name
+        self._print_variants(variants)
+
+    def visit_command(self, name, info, arg_type, ret_type,
+                      gen, success_response, boxed):
+        print 'command %s %s -> %s' % \
+            (name, arg_type and arg_type.name, ret_type and ret_type.name)
+        print '   gen=%s success_response=%s boxed=%s' % \
+            (gen, success_response, boxed)
+
+    def visit_event(self, name, info, arg_type, boxed):
+        print 'event %s %s' % (name, arg_type and arg_type.name)
+        print '   boxed=%s' % boxed
+
+    @staticmethod
+    def _print_variants(variants):
+        if variants:
+            print '    tag %s' % variants.tag_member.name
+            for v in variants.variants:
+                print '    case %s: %s' % (v.name, v.type.name)
+
+schema = QAPISchema(sys.argv[1])
+schema.visit(QAPISchemaTestVisitor())
diff --git a/tests/qapi-schema/type-bypass-bad-gen.err b/tests/qapi-schema/type-bypass-bad-gen.err
new file mode 100644
index 0000000..a83c3c6
--- /dev/null
+++ b/tests/qapi-schema/type-bypass-bad-gen.err
@@ -0,0 +1 @@
+tests/qapi-schema/type-bypass-bad-gen.json:2: 'gen' of command 'foo' should only use false value
diff --git a/tests/qapi-schema/type-bypass-bad-gen.exit b/tests/qapi-schema/type-bypass-bad-gen.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/type-bypass-bad-gen.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/type-bypass-bad-gen.json b/tests/qapi-schema/type-bypass-bad-gen.json
new file mode 100644
index 0000000..e8dec34
--- /dev/null
+++ b/tests/qapi-schema/type-bypass-bad-gen.json
@@ -0,0 +1,2 @@
+# 'gen' should only appear with value false
+{ 'command': 'foo', 'gen': 'whatever' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/type-bypass-bad-gen.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/type-bypass-bad-gen.out
diff --git a/tests/qapi-schema/unicode-str.err b/tests/qapi-schema/unicode-str.err
new file mode 100644
index 0000000..f621cd6
--- /dev/null
+++ b/tests/qapi-schema/unicode-str.err
@@ -0,0 +1 @@
+tests/qapi-schema/unicode-str.json:2: 'command' uses invalid name 'é'
diff --git a/tests/qapi-schema/unicode-str.exit b/tests/qapi-schema/unicode-str.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/unicode-str.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/unicode-str.json b/tests/qapi-schema/unicode-str.json
new file mode 100644
index 0000000..5253a1b
--- /dev/null
+++ b/tests/qapi-schema/unicode-str.json
@@ -0,0 +1,2 @@
+# we don't support full Unicode strings, yet
+{ 'command': 'é' }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/unicode-str.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/unicode-str.out
diff --git a/tests/qapi-schema/union-base-no-discriminator.err b/tests/qapi-schema/union-base-no-discriminator.err
new file mode 100644
index 0000000..8b7a242
--- /dev/null
+++ b/tests/qapi-schema/union-base-no-discriminator.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-base-no-discriminator.json:11: Simple union 'TestUnion' must not have a base
diff --git a/tests/qapi-schema/union-base-no-discriminator.exit b/tests/qapi-schema/union-base-no-discriminator.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/union-base-no-discriminator.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-base-no-discriminator.json b/tests/qapi-schema/union-base-no-discriminator.json
new file mode 100644
index 0000000..1409cf5
--- /dev/null
+++ b/tests/qapi-schema/union-base-no-discriminator.json
@@ -0,0 +1,14 @@
+# we reject simple unions with a base (or flat unions without discriminator)
+{ 'struct': 'TestTypeA',
+  'data': { 'string': 'str' } }
+
+{ 'struct': 'TestTypeB',
+  'data': { 'integer': 'int' } }
+
+{ 'struct': 'Base',
+  'data': { 'string': 'str' } }
+
+{ 'union': 'TestUnion',
+  'base': 'Base',
+  'data': { 'value1': 'TestTypeA',
+            'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/union-base-no-discriminator.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/union-base-no-discriminator.out
diff --git a/tests/qapi-schema/union-branch-case.err b/tests/qapi-schema/union-branch-case.err
new file mode 100644
index 0000000..1152190
--- /dev/null
+++ b/tests/qapi-schema/union-branch-case.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-branch-case.json:2: 'Branch' (branch of NoWayThisWillGetWhitelisted) should not use uppercase
diff --git a/tests/qapi-schema/union-branch-case.exit b/tests/qapi-schema/union-branch-case.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/union-branch-case.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-branch-case.json b/tests/qapi-schema/union-branch-case.json
new file mode 100644
index 0000000..e6565dc
--- /dev/null
+++ b/tests/qapi-schema/union-branch-case.json
@@ -0,0 +1,2 @@
+# Branch names should be 'lower-case' unless the union is whitelisted
+{ 'union': 'NoWayThisWillGetWhitelisted', 'data': { 'Branch': 'int' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/union-branch-case.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/union-branch-case.out
diff --git a/tests/qapi-schema/union-clash-branches.err b/tests/qapi-schema/union-clash-branches.err
new file mode 100644
index 0000000..e5b2113
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-clash-branches.json:4: 'a_b' (branch of TestUnion) collides with 'a-b' (branch of TestUnion)
diff --git a/tests/qapi-schema/union-clash-branches.exit b/tests/qapi-schema/union-clash-branches.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-clash-branches.json b/tests/qapi-schema/union-clash-branches.json
new file mode 100644
index 0000000..3bece8c
--- /dev/null
+++ b/tests/qapi-schema/union-clash-branches.json
@@ -0,0 +1,5 @@
+# Union branch name collision
+# Reject a union that would result in a collision in generated C names (this
+# would try to generate two members 'a_b').
+{ 'union': 'TestUnion',
+  'data': { 'a-b': 'int', 'a_b': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/union-clash-branches.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/union-clash-branches.out
diff --git a/tests/qapi-schema/union-empty.err b/tests/qapi-schema/union-empty.err
new file mode 100644
index 0000000..12c2022
--- /dev/null
+++ b/tests/qapi-schema/union-empty.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-empty.json:2: Union 'Union' cannot have empty 'data'
diff --git a/tests/qapi-schema/union-empty.exit b/tests/qapi-schema/union-empty.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/union-empty.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-empty.json b/tests/qapi-schema/union-empty.json
new file mode 100644
index 0000000..1f0b13c
--- /dev/null
+++ b/tests/qapi-schema/union-empty.json
@@ -0,0 +1,2 @@
+# unions cannot be empty
+{ 'union': 'Union', 'data': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/union-empty.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/union-empty.out
diff --git a/tests/qapi-schema/union-invalid-base.err b/tests/qapi-schema/union-invalid-base.err
index 938f969..03d7b97 100644
--- a/tests/qapi-schema/union-invalid-base.err
+++ b/tests/qapi-schema/union-invalid-base.err
@@ -1 +1 @@
-tests/qapi-schema/union-invalid-base.json:7: Base 'TestBaseWrong' is not a valid type
+tests/qapi-schema/union-invalid-base.json:8: 'base' for union 'TestUnion' cannot use built-in type 'int'
diff --git a/tests/qapi-schema/union-invalid-base.json b/tests/qapi-schema/union-invalid-base.json
index 1fa4930..92be39d 100644
--- a/tests/qapi-schema/union-invalid-base.json
+++ b/tests/qapi-schema/union-invalid-base.json
@@ -1,10 +1,12 @@
-{ 'type': 'TestTypeA',
+# a union base type must be a struct
+{ 'struct': 'TestTypeA',
   'data': { 'string': 'str' } }
 
-{ 'type': 'TestTypeB',
+{ 'struct': 'TestTypeB',
   'data': { 'integer': 'int' } }
 
 { 'union': 'TestUnion',
-  'base': 'TestBaseWrong',
+  'base': 'int',
+  'discriminator': 'int',
   'data': { 'value1': 'TestTypeA',
             'value2': 'TestTypeB' } }
diff --git a/tests/qapi-schema/union-optional-branch.err b/tests/qapi-schema/union-optional-branch.err
new file mode 100644
index 0000000..3ada133
--- /dev/null
+++ b/tests/qapi-schema/union-optional-branch.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-optional-branch.json:2: Member of union 'Union' does not allow optional name '*a'
diff --git a/tests/qapi-schema/union-optional-branch.exit b/tests/qapi-schema/union-optional-branch.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/union-optional-branch.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-optional-branch.json b/tests/qapi-schema/union-optional-branch.json
new file mode 100644
index 0000000..591615f
--- /dev/null
+++ b/tests/qapi-schema/union-optional-branch.json
@@ -0,0 +1,2 @@
+# union branches cannot be optional
+{ 'union': 'Union', 'data': { '*a': 'int', 'b': 'str' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/union-optional-branch.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/union-optional-branch.out
diff --git a/tests/qapi-schema/union-unknown.err b/tests/qapi-schema/union-unknown.err
new file mode 100644
index 0000000..54fe456
--- /dev/null
+++ b/tests/qapi-schema/union-unknown.err
@@ -0,0 +1 @@
+tests/qapi-schema/union-unknown.json:2: Member 'unknown' of union 'Union' uses unknown type 'MissingType'
diff --git a/tests/qapi-schema/union-unknown.exit b/tests/qapi-schema/union-unknown.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/union-unknown.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/union-unknown.json b/tests/qapi-schema/union-unknown.json
new file mode 100644
index 0000000..aa7e814
--- /dev/null
+++ b/tests/qapi-schema/union-unknown.json
@@ -0,0 +1,3 @@
+# we reject a union with unknown type in branch
+{ 'union': 'Union',
+  'data': { 'unknown': 'MissingType' } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/union-unknown.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/union-unknown.out
diff --git a/tests/qapi-schema/unknown-escape.err b/tests/qapi-schema/unknown-escape.err
new file mode 100644
index 0000000..000e30d
--- /dev/null
+++ b/tests/qapi-schema/unknown-escape.err
@@ -0,0 +1 @@
+tests/qapi-schema/unknown-escape.json:3:21: Unknown escape \x
diff --git a/tests/qapi-schema/unknown-escape.exit b/tests/qapi-schema/unknown-escape.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/unknown-escape.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/unknown-escape.json b/tests/qapi-schema/unknown-escape.json
new file mode 100644
index 0000000..8e6891e
--- /dev/null
+++ b/tests/qapi-schema/unknown-escape.json
@@ -0,0 +1,3 @@
+# we only recognize JSON escape sequences, plus our \' extension (no \x)
+# { 'command': 'foo', 'data': {} }
+{ 'command': 'foo', 'dat\x61':{} }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/unknown-escape.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/unknown-escape.out
diff --git a/tests/qapi-schema/unknown-expr-key.err b/tests/qapi-schema/unknown-expr-key.err
new file mode 100644
index 0000000..12f5ed5
--- /dev/null
+++ b/tests/qapi-schema/unknown-expr-key.err
@@ -0,0 +1 @@
+tests/qapi-schema/unknown-expr-key.json:2: Unknown key 'bogus' in struct 'bar'
diff --git a/tests/qapi-schema/unknown-expr-key.exit b/tests/qapi-schema/unknown-expr-key.exit
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/qapi-schema/unknown-expr-key.exit
@@ -0,0 +1 @@
+1
diff --git a/tests/qapi-schema/unknown-expr-key.json b/tests/qapi-schema/unknown-expr-key.json
new file mode 100644
index 0000000..3b2be00
--- /dev/null
+++ b/tests/qapi-schema/unknown-expr-key.json
@@ -0,0 +1,2 @@
+# we reject an expression with unknown top-level keys
+{ 'struct': 'bar', 'data': { 'string': 'str'}, 'bogus': { } }
diff --git a/tests/qapi-schema/flat-union-reverse-define.err b/tests/qapi-schema/unknown-expr-key.out
similarity index 100%
copy from tests/qapi-schema/flat-union-reverse-define.err
copy to tests/qapi-schema/unknown-expr-key.out
diff --git a/tests/qemu-iotests-quick.sh b/tests/qemu-iotests-quick.sh
index 12af731..0e554bb 100755
--- a/tests/qemu-iotests-quick.sh
+++ b/tests/qemu-iotests-quick.sh
@@ -3,6 +3,6 @@
 cd tests/qemu-iotests
 
 ret=0
-./check -T -qcow2 -g quick || ret=1
+TEST_DIR=${TEST_DIR:-/tmp/qemu-iotests-quick-$$} ./check -T -qcow2 -g quick || ret=1
 
 exit $ret
diff --git a/tests/qemu-iotests/.gitignore b/tests/qemu-iotests/.gitignore
index 0541f80..0711cbd 100644
--- a/tests/qemu-iotests/.gitignore
+++ b/tests/qemu-iotests/.gitignore
@@ -1,5 +1,6 @@
 check.log
 check.time
+common.env
 *.out.bad
 *.notrun
 socket_scm_helper
diff --git a/tests/qemu-iotests/001 b/tests/qemu-iotests/001
index 4e16469..ffd14e2 100755
--- a/tests/qemu-iotests/001
+++ b/tests/qemu-iotests/001
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/001.out b/tests/qemu-iotests/001.out
index 330f5c9..b80e098 100644
--- a/tests/qemu-iotests/001.out
+++ b/tests/qemu-iotests/001.out
@@ -1,5 +1,5 @@
 QA output created by 001
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0
diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002
index 6a865aa..d4f8e91 100755
--- a/tests/qemu-iotests/002
+++ b/tests/qemu-iotests/002
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/002.out b/tests/qemu-iotests/002.out
index cd6aa0f..1b46b34 100644
--- a/tests/qemu-iotests/002.out
+++ b/tests/qemu-iotests/002.out
@@ -1,5 +1,5 @@
 QA output created by 002
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0
diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003
index 98638d4..19889b9 100755
--- a/tests/qemu-iotests/003
+++ b/tests/qemu-iotests/003
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/003.out b/tests/qemu-iotests/003.out
index 4942482..e764123 100644
--- a/tests/qemu-iotests/003.out
+++ b/tests/qemu-iotests/003.out
@@ -1,5 +1,5 @@
 QA output created by 003
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0
diff --git a/tests/qemu-iotests/004 b/tests/qemu-iotests/004
index 651072e..6f2aa3d 100755
--- a/tests/qemu-iotests/004
+++ b/tests/qemu-iotests/004
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -38,7 +37,7 @@
 . ./common.rc
 . ./common.filter
 
-_supported_fmt generic
+_supported_fmt raw qcow qcow2 qed vdi vmdk vhdx luks
 _supported_proto generic
 _supported_os Linux
 
diff --git a/tests/qemu-iotests/004.out b/tests/qemu-iotests/004.out
index f1ed4d8..fb85932 100644
--- a/tests/qemu-iotests/004.out
+++ b/tests/qemu-iotests/004.out
@@ -1,5 +1,5 @@
 QA output created by 004
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 write before image boundary
 wrote 1048576/1048576 bytes at offset 133169152
diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005
index ba1236d..4447377 100755
--- a/tests/qemu-iotests/005
+++ b/tests/qemu-iotests/005
@@ -28,7 +28,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/005.out b/tests/qemu-iotests/005.out
index 2d3e7df..10044c3 100644
--- a/tests/qemu-iotests/005.out
+++ b/tests/qemu-iotests/005.out
@@ -1,7 +1,7 @@
 QA output created by 005
 
 creating large image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5368709120000 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5368709120000
 
 small read
 read 4096/4096 bytes at offset 1024
diff --git a/tests/qemu-iotests/006 b/tests/qemu-iotests/006
deleted file mode 100755
index 0c0cf5d..0000000
--- a/tests/qemu-iotests/006
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-#
-# Make sure qemu-img rejects > 127GB images for the vpc format as the format
-# doesn't support this.
-#
-# Copyright (C) 2009 Red Hat, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-# creator
-owner=hch@lst.de
-
-seq=`basename $0`
-echo "QA output created by $seq"
-
-here=`pwd`
-tmp=/tmp/$$
-status=1	# failure is the default!
-
-_cleanup()
-{
-	_cleanup_test_img
-}
-trap "_cleanup; exit \$status" 0 1 2 3 15
-
-# get standard environment, filters and checks
-. ./common.rc
-. ./common.filter
-
-_supported_fmt vpc
-_supported_proto generic
-_supported_os Linux
-
-
-echo
-echo "creating 128GB image"
-_make_test_img 128G
-
-# success, all done
-echo "*** done"
-rm -f $seq.full
-status=0
diff --git a/tests/qemu-iotests/006.out b/tests/qemu-iotests/006.out
deleted file mode 100644
index f82fbfb..0000000
--- a/tests/qemu-iotests/006.out
+++ /dev/null
@@ -1,6 +0,0 @@
-QA output created by 006
-
-creating 128GB image
-qemu-img: The image size is too large for file format 'vpc'
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=137438953472 
-*** done
diff --git a/tests/qemu-iotests/007 b/tests/qemu-iotests/007
index fe1a743..fa543ee 100755
--- a/tests/qemu-iotests/007
+++ b/tests/qemu-iotests/007
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -43,6 +42,9 @@
 _supported_fmt qcow2
 _supported_proto generic
 _supported_os Linux
+# refcount_bits must be at least 4 so we can create ten internal snapshots
+# (1 bit supports none, 2 bits support two, 4 bits support 14)
+_unsupported_imgopts 'refcount_bits=\(1\|2\)[^0-9]'
 
 echo
 echo "creating image"
diff --git a/tests/qemu-iotests/007.out b/tests/qemu-iotests/007.out
index cf3d17d..5715053 100644
--- a/tests/qemu-iotests/007.out
+++ b/tests/qemu-iotests/007.out
@@ -1,7 +1,7 @@
 QA output created by 007
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
 savevm 1
 savevm 2
 savevm 3
diff --git a/tests/qemu-iotests/008 b/tests/qemu-iotests/008
index 2d28efd..8e89d74 100755
--- a/tests/qemu-iotests/008
+++ b/tests/qemu-iotests/008
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/008.out b/tests/qemu-iotests/008.out
index 0acbb86..08475c8 100644
--- a/tests/qemu-iotests/008.out
+++ b/tests/qemu-iotests/008.out
@@ -1,5 +1,5 @@
 QA output created by 008
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0
diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009
index 57a43f5..16e4475 100755
--- a/tests/qemu-iotests/009
+++ b/tests/qemu-iotests/009
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/009.out b/tests/qemu-iotests/009.out
index 180e451..19e15d0 100644
--- a/tests/qemu-iotests/009.out
+++ b/tests/qemu-iotests/009.out
@@ -1,7 +1,7 @@
 QA output created by 009
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 
 creating pattern
 wrote 4096/4096 bytes at offset 2097152
diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010
index 896a005..151dac2 100755
--- a/tests/qemu-iotests/010
+++ b/tests/qemu-iotests/010
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/010.out b/tests/qemu-iotests/010.out
index cc5c941..1bee037 100644
--- a/tests/qemu-iotests/010.out
+++ b/tests/qemu-iotests/010.out
@@ -1,7 +1,7 @@
 QA output created by 010
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 
 creating pattern
 wrote 4096/4096 bytes at offset 2097152
diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011
index 1c5158a..f8d044e 100755
--- a/tests/qemu-iotests/011
+++ b/tests/qemu-iotests/011
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/011.out b/tests/qemu-iotests/011.out
index dfec17d..65c998e 100644
--- a/tests/qemu-iotests/011.out
+++ b/tests/qemu-iotests/011.out
@@ -1,7 +1,7 @@
 QA output created by 011
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 
 overlapping I/O
 wrote 1048576/1048576 bytes at offset XXX
diff --git a/tests/qemu-iotests/012 b/tests/qemu-iotests/012
index 7c5b689..01a770d 100755
--- a/tests/qemu-iotests/012
+++ b/tests/qemu-iotests/012
@@ -27,7 +27,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -44,13 +43,16 @@
 _supported_proto file
 _supported_os Linux
 
+# Remove once all tests are fixed to use TEST_IMG_FILE
+# correctly and common.rc sets it unconditionally
+test -z "$TEST_IMG_FILE" && TEST_IMG_FILE=$TEST_IMG
 
 size=128M
 _make_test_img $size
 
 echo
 echo "== mark image read-only"
-chmod a-w "$TEST_IMG"
+chmod a-w "$TEST_IMG_FILE"
 
 echo
 echo "== read from read-only image"
diff --git a/tests/qemu-iotests/012.out b/tests/qemu-iotests/012.out
index 5f9b990..c4ffa8f 100644
--- a/tests/qemu-iotests/012.out
+++ b/tests/qemu-iotests/012.out
@@ -1,5 +1,5 @@
 QA output created by 012
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == mark image read-only
 
diff --git a/tests/qemu-iotests/013 b/tests/qemu-iotests/013
index ea3cab9..d013f87 100755
--- a/tests/qemu-iotests/013
+++ b/tests/qemu-iotests/013
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/013.out b/tests/qemu-iotests/013.out
index 43a414c..317cdf4 100644
--- a/tests/qemu-iotests/013.out
+++ b/tests/qemu-iotests/013.out
@@ -1,5 +1,5 @@
 QA output created by 013
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 Testing empty image
 
 At offset 0:
diff --git a/tests/qemu-iotests/014 b/tests/qemu-iotests/014
index b23c2db..2ea79e8 100755
--- a/tests/qemu-iotests/014
+++ b/tests/qemu-iotests/014
@@ -27,7 +27,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/014.out b/tests/qemu-iotests/014.out
index 4744b4b..f19341c 100644
--- a/tests/qemu-iotests/014.out
+++ b/tests/qemu-iotests/014.out
@@ -1,5 +1,5 @@
 QA output created by 014
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 Testing empty image:
 test2: With offset 0
 === Clusters to be compressed [1]
diff --git a/tests/qemu-iotests/015 b/tests/qemu-iotests/015
index 099d757..aaf9c3f 100755
--- a/tests/qemu-iotests/015
+++ b/tests/qemu-iotests/015
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -43,6 +42,8 @@
 _supported_fmt qcow2
 _supported_proto generic
 _supported_os Linux
+# Internal snapshots are (currently) impossible with refcount_bits=1
+_unsupported_imgopts 'refcount_bits=1[^0-9]'
 
 echo
 echo "creating image"
diff --git a/tests/qemu-iotests/015.out b/tests/qemu-iotests/015.out
index d4b961c..3414afd 100644
--- a/tests/qemu-iotests/015.out
+++ b/tests/qemu-iotests/015.out
@@ -1,7 +1,7 @@
 QA output created by 015
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=37748736 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=37748736
 creating first snapshot
 wrote 37748736/37748736 bytes at offset 0
 36 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/qemu-iotests/016 b/tests/qemu-iotests/016
deleted file mode 100755
index 7ea9e94..0000000
--- a/tests/qemu-iotests/016
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/bin/bash
-#
-# Test I/O after EOF for growable images.
-#
-# Copyright (C) 2009 Red Hat, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-# creator
-owner=hch@lst.de
-
-seq=`basename $0`
-echo "QA output created by $seq"
-
-here=`pwd`
-tmp=/tmp/$$
-status=1	# failure is the default!
-
-_cleanup()
-{
-	_cleanup_test_img
-}
-trap "_cleanup; exit \$status" 0 1 2 3 15
-
-# get standard environment, filters and checks
-. ./common.rc
-. ./common.filter
-
-_supported_fmt raw
-_supported_proto file sheepdog nfs
-_supported_os Linux
-
-
-size=128M
-_make_test_img $size
-
-echo
-echo "== reading at EOF =="
-$QEMU_IO -g -c "read -P 0 $size 512" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== reading far past EOF =="
-$QEMU_IO -g -c "read -P 0 256M 512" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== writing at EOF =="
-$QEMU_IO -g -c "write -P 66 $size 512" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 66 $size 512" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== writing far past EOF =="
-$QEMU_IO -g -c "write -P 66 256M 512" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 66 256M 512" "$TEST_IMG" | _filter_qemu_io
-
-# success, all done
-echo "*** done"
-rm -f $seq.full
-status=0
diff --git a/tests/qemu-iotests/016.out b/tests/qemu-iotests/016.out
deleted file mode 100644
index 1867978..0000000
--- a/tests/qemu-iotests/016.out
+++ /dev/null
@@ -1,23 +0,0 @@
-QA output created by 016
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-
-== reading at EOF ==
-read 512/512 bytes at offset 134217728
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== reading far past EOF ==
-read 512/512 bytes at offset 268435456
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== writing at EOF ==
-wrote 512/512 bytes at offset 134217728
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 134217728
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== writing far past EOF ==
-wrote 512/512 bytes at offset 268435456
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 268435456
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-*** done
diff --git a/tests/qemu-iotests/017 b/tests/qemu-iotests/017
index 3af3cdf..e3f9e75 100755
--- a/tests/qemu-iotests/017
+++ b/tests/qemu-iotests/017
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/017.out b/tests/qemu-iotests/017.out
index 75ea614..8fc9241 100644
--- a/tests/qemu-iotests/017.out
+++ b/tests/qemu-iotests/017.out
@@ -269,7 +269,7 @@
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018
index d8a7d43..1d39d35 100755
--- a/tests/qemu-iotests/018
+++ b/tests/qemu-iotests/018
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -66,8 +65,8 @@
 echo "Creating test image with backing file"
 echo
 
-TEST_IMG=$TEST_IMG_SAVE
-_make_test_img -b "$TEST_IMG.base" 6G
+TEST_IMG="$TEST_IMG_SAVE.orig"
+_make_test_img -b "$TEST_IMG_SAVE.base" 6G
 
 echo "Filling test image"
 echo
@@ -81,8 +80,8 @@
 done
 _check_test_img
 
-mv "$TEST_IMG" "$TEST_IMG.orig"
-$QEMU_IMG convert -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
+TEST_IMG="$TEST_IMG_SAVE"
+$QEMU_IMG convert -f $IMGFMT -O $IMGFMT "$TEST_IMG.orig" "$TEST_IMG"
 
 echo "Reading"
 echo
diff --git a/tests/qemu-iotests/018.out b/tests/qemu-iotests/018.out
index 25e7b95..5df9667 100644
--- a/tests/qemu-iotests/018.out
+++ b/tests/qemu-iotests/018.out
@@ -269,7 +269,7 @@
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019
index f5ecbf5..24a789a 100755
--- a/tests/qemu-iotests/019
+++ b/tests/qemu-iotests/019
@@ -27,7 +27,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -54,6 +53,9 @@
 TEST_OFFSETS="0 4294967296"
 CLUSTER_SIZE=65536
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img 6G
 
 echo "Filling base image"
@@ -71,8 +73,8 @@
 echo "Creating test image with backing file"
 echo
 
-mv "$TEST_IMG" "$TEST_IMG.base"
-_make_test_img -b "$TEST_IMG.base" 6G
+TEST_IMG="$TEST_IMG_SAVE.orig"
+_make_test_img -b "$TEST_IMG_SAVE.base" 6G
 
 echo "Filling test image"
 echo
@@ -86,9 +88,7 @@
 done
 _check_test_img
 
-mv "$TEST_IMG" "$TEST_IMG.orig"
-
-
+TEST_IMG="$TEST_IMG_SAVE"
 
 # Test the conversion twice: One test with the old-style -B option and another
 # one with -o backing_file
@@ -98,7 +98,7 @@
     echo
     echo Testing conversion with $backing_option"$TEST_IMG.base" | _filter_testdir | _filter_imgfmt
     echo
-    $QEMU_IMG convert -O $IMGFMT $backing_option"$TEST_IMG.base" "$TEST_IMG.orig" "$TEST_IMG"
+    $QEMU_IMG convert -f $IMGFMT -O $IMGFMT $backing_option"$TEST_IMG.base" "$TEST_IMG.orig" "$TEST_IMG"
 
     echo "Checking if backing clusters are allocated when they shouldn't"
     echo
diff --git a/tests/qemu-iotests/019.out b/tests/qemu-iotests/019.out
index f0c6e63..0124264 100644
--- a/tests/qemu-iotests/019.out
+++ b/tests/qemu-iotests/019.out
@@ -1,5 +1,5 @@
 QA output created by 019
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=6442450944
 Filling base image
 
 === IO: pattern 42
@@ -269,7 +269,7 @@
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 43
diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020
index 2f258dc..9c4a68c 100755
--- a/tests/qemu-iotests/020
+++ b/tests/qemu-iotests/020
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -56,6 +55,9 @@
 
 TEST_OFFSETS="0 4294967296"
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img 6G
 
 echo "Filling base image"
@@ -73,7 +75,7 @@
 echo "Creating test image with backing file"
 echo
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 _make_test_img -b "$TEST_IMG.base" 6G
 
 echo "Filling test image"
@@ -89,7 +91,7 @@
 _check_test_img
 
 $QEMU_IMG commit "$TEST_IMG"
-mv "$TEST_IMG.base" "$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
 
 echo "Reading from the backing file"
 echo
diff --git a/tests/qemu-iotests/020.out b/tests/qemu-iotests/020.out
index fc9a63c..42f6c1b 100644
--- a/tests/qemu-iotests/020.out
+++ b/tests/qemu-iotests/020.out
@@ -1,5 +1,5 @@
 QA output created by 020
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=6442450944
 Filling base image
 
 === IO: pattern 0
@@ -269,7 +269,7 @@
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 1
diff --git a/tests/qemu-iotests/021 b/tests/qemu-iotests/021
index 1c69024..11e8ed7 100755
--- a/tests/qemu-iotests/021
+++ b/tests/qemu-iotests/021
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/021.out b/tests/qemu-iotests/021.out
index ed4448c..8533f8e 100644
--- a/tests/qemu-iotests/021.out
+++ b/tests/qemu-iotests/021.out
@@ -1,5 +1,5 @@
 QA output created by 021
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == testing writev -P -1 ==
 -1 is not a valid pattern byte
diff --git a/tests/qemu-iotests/022 b/tests/qemu-iotests/022
index d35b816..2452a9f 100755
--- a/tests/qemu-iotests/022
+++ b/tests/qemu-iotests/022
@@ -27,7 +27,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/022.out b/tests/qemu-iotests/022.out
index 5a729e0..dbe9cdb 100644
--- a/tests/qemu-iotests/022.out
+++ b/tests/qemu-iotests/022.out
@@ -1,5 +1,5 @@
 QA output created by 022
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 Testing empty image
 
 At offset 10485760:
diff --git a/tests/qemu-iotests/023 b/tests/qemu-iotests/023
index 9ad06b9..497ae1e 100755
--- a/tests/qemu-iotests/023
+++ b/tests/qemu-iotests/023
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/023.out b/tests/qemu-iotests/023.out
index ec32341..664871b 100644
--- a/tests/qemu-iotests/023.out
+++ b/tests/qemu-iotests/023.out
@@ -1,7 +1,7 @@
 QA output created by 023
 Creating new image; cluster size: 1024
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 Testing empty image
 
 At offset 0:
@@ -225,42 +225,78 @@
 wrote 512/512 bytes at offset 109568
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 110848 is not sector aligned
-offset 111872 is not sector aligned
-offset 112896 is not sector aligned
-offset 113920 is not sector aligned
-offset 114944 is not sector aligned
-offset 115968 is not sector aligned
-offset 116992 is not sector aligned
-offset 118016 is not sector aligned
-offset 119040 is not sector aligned
-offset 120064 is not sector aligned
-offset 121088 is not sector aligned
-offset 122112 is not sector aligned
-offset 123136 is not sector aligned
-offset 124160 is not sector aligned
-offset 125184 is not sector aligned
-offset 126208 is not sector aligned
-offset 127232 is not sector aligned
-offset 128256 is not sector aligned
-offset 129280 is not sector aligned
-offset 130304 is not sector aligned
-offset 131328 is not sector aligned
-offset 132352 is not sector aligned
-offset 133376 is not sector aligned
-offset 134400 is not sector aligned
-offset 135424 is not sector aligned
-offset 136448 is not sector aligned
-offset 137472 is not sector aligned
-offset 138496 is not sector aligned
-offset 139520 is not sector aligned
-offset 140544 is not sector aligned
-offset 141568 is not sector aligned
-offset 142592 is not sector aligned
-offset 143616 is not sector aligned
-offset 144640 is not sector aligned
-offset 145664 is not sector aligned
-offset 146688 is not sector aligned
+wrote 512/512 bytes at offset 110848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 111872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 112896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 113920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 114944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 115968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 116992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 118016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 119040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 120064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 121088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 122112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 123136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 124160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 125184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 126208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 127232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 128256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 129280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 130304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 131328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 132352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 133376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 134400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 135424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 136448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 137472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 138496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 139520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 140544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 141568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 142592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 143616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 144640
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 145664
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 146688
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 wrote 2048/2048 bytes at offset 147968
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -507,42 +543,78 @@
 read 512/512 bytes at offset 109568
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 110848 is not sector aligned
-offset 111872 is not sector aligned
-offset 112896 is not sector aligned
-offset 113920 is not sector aligned
-offset 114944 is not sector aligned
-offset 115968 is not sector aligned
-offset 116992 is not sector aligned
-offset 118016 is not sector aligned
-offset 119040 is not sector aligned
-offset 120064 is not sector aligned
-offset 121088 is not sector aligned
-offset 122112 is not sector aligned
-offset 123136 is not sector aligned
-offset 124160 is not sector aligned
-offset 125184 is not sector aligned
-offset 126208 is not sector aligned
-offset 127232 is not sector aligned
-offset 128256 is not sector aligned
-offset 129280 is not sector aligned
-offset 130304 is not sector aligned
-offset 131328 is not sector aligned
-offset 132352 is not sector aligned
-offset 133376 is not sector aligned
-offset 134400 is not sector aligned
-offset 135424 is not sector aligned
-offset 136448 is not sector aligned
-offset 137472 is not sector aligned
-offset 138496 is not sector aligned
-offset 139520 is not sector aligned
-offset 140544 is not sector aligned
-offset 141568 is not sector aligned
-offset 142592 is not sector aligned
-offset 143616 is not sector aligned
-offset 144640 is not sector aligned
-offset 145664 is not sector aligned
-offset 146688 is not sector aligned
+read 512/512 bytes at offset 110848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 111872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 112896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 113920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 114944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 115968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 116992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 118016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 119040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 120064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 121088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 122112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 123136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 124160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 125184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 126208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 127232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 128256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 129280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 130304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 131328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 133376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 134400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 135424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 136448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 137472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 138496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 139520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 140544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 141568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 142592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 143616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 144640
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 145664
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 146688
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 147968
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -789,42 +861,78 @@
 wrote 512/512 bytes at offset 109568
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 110848 is not sector aligned
-offset 111872 is not sector aligned
-offset 112896 is not sector aligned
-offset 113920 is not sector aligned
-offset 114944 is not sector aligned
-offset 115968 is not sector aligned
-offset 116992 is not sector aligned
-offset 118016 is not sector aligned
-offset 119040 is not sector aligned
-offset 120064 is not sector aligned
-offset 121088 is not sector aligned
-offset 122112 is not sector aligned
-offset 123136 is not sector aligned
-offset 124160 is not sector aligned
-offset 125184 is not sector aligned
-offset 126208 is not sector aligned
-offset 127232 is not sector aligned
-offset 128256 is not sector aligned
-offset 129280 is not sector aligned
-offset 130304 is not sector aligned
-offset 131328 is not sector aligned
-offset 132352 is not sector aligned
-offset 133376 is not sector aligned
-offset 134400 is not sector aligned
-offset 135424 is not sector aligned
-offset 136448 is not sector aligned
-offset 137472 is not sector aligned
-offset 138496 is not sector aligned
-offset 139520 is not sector aligned
-offset 140544 is not sector aligned
-offset 141568 is not sector aligned
-offset 142592 is not sector aligned
-offset 143616 is not sector aligned
-offset 144640 is not sector aligned
-offset 145664 is not sector aligned
-offset 146688 is not sector aligned
+wrote 512/512 bytes at offset 110848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 111872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 112896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 113920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 114944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 115968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 116992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 118016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 119040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 120064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 121088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 122112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 123136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 124160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 125184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 126208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 127232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 128256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 129280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 130304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 131328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 132352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 133376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 134400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 135424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 136448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 137472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 138496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 139520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 140544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 141568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 142592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 143616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 144640
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 145664
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 146688
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 wrote 2048/2048 bytes at offset 147968
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -1071,42 +1179,78 @@
 read 512/512 bytes at offset 109568
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 110848 is not sector aligned
-offset 111872 is not sector aligned
-offset 112896 is not sector aligned
-offset 113920 is not sector aligned
-offset 114944 is not sector aligned
-offset 115968 is not sector aligned
-offset 116992 is not sector aligned
-offset 118016 is not sector aligned
-offset 119040 is not sector aligned
-offset 120064 is not sector aligned
-offset 121088 is not sector aligned
-offset 122112 is not sector aligned
-offset 123136 is not sector aligned
-offset 124160 is not sector aligned
-offset 125184 is not sector aligned
-offset 126208 is not sector aligned
-offset 127232 is not sector aligned
-offset 128256 is not sector aligned
-offset 129280 is not sector aligned
-offset 130304 is not sector aligned
-offset 131328 is not sector aligned
-offset 132352 is not sector aligned
-offset 133376 is not sector aligned
-offset 134400 is not sector aligned
-offset 135424 is not sector aligned
-offset 136448 is not sector aligned
-offset 137472 is not sector aligned
-offset 138496 is not sector aligned
-offset 139520 is not sector aligned
-offset 140544 is not sector aligned
-offset 141568 is not sector aligned
-offset 142592 is not sector aligned
-offset 143616 is not sector aligned
-offset 144640 is not sector aligned
-offset 145664 is not sector aligned
-offset 146688 is not sector aligned
+read 512/512 bytes at offset 110848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 111872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 112896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 113920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 114944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 115968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 116992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 118016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 119040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 120064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 121088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 122112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 123136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 124160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 125184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 126208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 127232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 128256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 129280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 130304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 131328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 133376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 134400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 135424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 136448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 137472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 138496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 139520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 140544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 141568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 142592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 143616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 144640
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 145664
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 146688
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 147968
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -1355,42 +1499,78 @@
 wrote 512/512 bytes at offset 4295076864
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 4295078144 is not sector aligned
-offset 4295079168 is not sector aligned
-offset 4295080192 is not sector aligned
-offset 4295081216 is not sector aligned
-offset 4295082240 is not sector aligned
-offset 4295083264 is not sector aligned
-offset 4295084288 is not sector aligned
-offset 4295085312 is not sector aligned
-offset 4295086336 is not sector aligned
-offset 4295087360 is not sector aligned
-offset 4295088384 is not sector aligned
-offset 4295089408 is not sector aligned
-offset 4295090432 is not sector aligned
-offset 4295091456 is not sector aligned
-offset 4295092480 is not sector aligned
-offset 4295093504 is not sector aligned
-offset 4295094528 is not sector aligned
-offset 4295095552 is not sector aligned
-offset 4295096576 is not sector aligned
-offset 4295097600 is not sector aligned
-offset 4295098624 is not sector aligned
-offset 4295099648 is not sector aligned
-offset 4295100672 is not sector aligned
-offset 4295101696 is not sector aligned
-offset 4295102720 is not sector aligned
-offset 4295103744 is not sector aligned
-offset 4295104768 is not sector aligned
-offset 4295105792 is not sector aligned
-offset 4295106816 is not sector aligned
-offset 4295107840 is not sector aligned
-offset 4295108864 is not sector aligned
-offset 4295109888 is not sector aligned
-offset 4295110912 is not sector aligned
-offset 4295111936 is not sector aligned
-offset 4295112960 is not sector aligned
-offset 4295113984 is not sector aligned
+wrote 512/512 bytes at offset 4295078144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295079168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295080192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295081216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295082240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295083264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295084288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295085312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295086336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295087360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295088384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295089408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295090432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295091456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295092480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295093504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295094528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295095552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295096576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295097600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295098624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295099648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295100672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295101696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295102720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295103744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295104768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295105792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295106816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295107840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295108864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295109888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295110912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295111936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295112960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295113984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 wrote 2048/2048 bytes at offset 4295115264
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -1637,42 +1817,78 @@
 read 512/512 bytes at offset 4295076864
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 4295078144 is not sector aligned
-offset 4295079168 is not sector aligned
-offset 4295080192 is not sector aligned
-offset 4295081216 is not sector aligned
-offset 4295082240 is not sector aligned
-offset 4295083264 is not sector aligned
-offset 4295084288 is not sector aligned
-offset 4295085312 is not sector aligned
-offset 4295086336 is not sector aligned
-offset 4295087360 is not sector aligned
-offset 4295088384 is not sector aligned
-offset 4295089408 is not sector aligned
-offset 4295090432 is not sector aligned
-offset 4295091456 is not sector aligned
-offset 4295092480 is not sector aligned
-offset 4295093504 is not sector aligned
-offset 4295094528 is not sector aligned
-offset 4295095552 is not sector aligned
-offset 4295096576 is not sector aligned
-offset 4295097600 is not sector aligned
-offset 4295098624 is not sector aligned
-offset 4295099648 is not sector aligned
-offset 4295100672 is not sector aligned
-offset 4295101696 is not sector aligned
-offset 4295102720 is not sector aligned
-offset 4295103744 is not sector aligned
-offset 4295104768 is not sector aligned
-offset 4295105792 is not sector aligned
-offset 4295106816 is not sector aligned
-offset 4295107840 is not sector aligned
-offset 4295108864 is not sector aligned
-offset 4295109888 is not sector aligned
-offset 4295110912 is not sector aligned
-offset 4295111936 is not sector aligned
-offset 4295112960 is not sector aligned
-offset 4295113984 is not sector aligned
+read 512/512 bytes at offset 4295078144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295079168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295080192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295081216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295082240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295083264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295084288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295085312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295086336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295087360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295088384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295089408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295090432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295091456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295092480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295093504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295094528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295095552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295096576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295097600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295098624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295099648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295100672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295101696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295102720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295103744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295104768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295105792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295106816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295107840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295108864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295109888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295110912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295111936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295112960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295113984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 4295115264
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -1919,42 +2135,78 @@
 wrote 512/512 bytes at offset 4295076864
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 4295078144 is not sector aligned
-offset 4295079168 is not sector aligned
-offset 4295080192 is not sector aligned
-offset 4295081216 is not sector aligned
-offset 4295082240 is not sector aligned
-offset 4295083264 is not sector aligned
-offset 4295084288 is not sector aligned
-offset 4295085312 is not sector aligned
-offset 4295086336 is not sector aligned
-offset 4295087360 is not sector aligned
-offset 4295088384 is not sector aligned
-offset 4295089408 is not sector aligned
-offset 4295090432 is not sector aligned
-offset 4295091456 is not sector aligned
-offset 4295092480 is not sector aligned
-offset 4295093504 is not sector aligned
-offset 4295094528 is not sector aligned
-offset 4295095552 is not sector aligned
-offset 4295096576 is not sector aligned
-offset 4295097600 is not sector aligned
-offset 4295098624 is not sector aligned
-offset 4295099648 is not sector aligned
-offset 4295100672 is not sector aligned
-offset 4295101696 is not sector aligned
-offset 4295102720 is not sector aligned
-offset 4295103744 is not sector aligned
-offset 4295104768 is not sector aligned
-offset 4295105792 is not sector aligned
-offset 4295106816 is not sector aligned
-offset 4295107840 is not sector aligned
-offset 4295108864 is not sector aligned
-offset 4295109888 is not sector aligned
-offset 4295110912 is not sector aligned
-offset 4295111936 is not sector aligned
-offset 4295112960 is not sector aligned
-offset 4295113984 is not sector aligned
+wrote 512/512 bytes at offset 4295078144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295079168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295080192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295081216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295082240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295083264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295084288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295085312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295086336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295087360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295088384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295089408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295090432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295091456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295092480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295093504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295094528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295095552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295096576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295097600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295098624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295099648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295100672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295101696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295102720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295103744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295104768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295105792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295106816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295107840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295108864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295109888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295110912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295111936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295112960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295113984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 wrote 2048/2048 bytes at offset 4295115264
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -2201,42 +2453,78 @@
 read 512/512 bytes at offset 4295076864
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 4295078144 is not sector aligned
-offset 4295079168 is not sector aligned
-offset 4295080192 is not sector aligned
-offset 4295081216 is not sector aligned
-offset 4295082240 is not sector aligned
-offset 4295083264 is not sector aligned
-offset 4295084288 is not sector aligned
-offset 4295085312 is not sector aligned
-offset 4295086336 is not sector aligned
-offset 4295087360 is not sector aligned
-offset 4295088384 is not sector aligned
-offset 4295089408 is not sector aligned
-offset 4295090432 is not sector aligned
-offset 4295091456 is not sector aligned
-offset 4295092480 is not sector aligned
-offset 4295093504 is not sector aligned
-offset 4295094528 is not sector aligned
-offset 4295095552 is not sector aligned
-offset 4295096576 is not sector aligned
-offset 4295097600 is not sector aligned
-offset 4295098624 is not sector aligned
-offset 4295099648 is not sector aligned
-offset 4295100672 is not sector aligned
-offset 4295101696 is not sector aligned
-offset 4295102720 is not sector aligned
-offset 4295103744 is not sector aligned
-offset 4295104768 is not sector aligned
-offset 4295105792 is not sector aligned
-offset 4295106816 is not sector aligned
-offset 4295107840 is not sector aligned
-offset 4295108864 is not sector aligned
-offset 4295109888 is not sector aligned
-offset 4295110912 is not sector aligned
-offset 4295111936 is not sector aligned
-offset 4295112960 is not sector aligned
-offset 4295113984 is not sector aligned
+read 512/512 bytes at offset 4295078144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295079168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295080192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295081216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295082240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295083264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295084288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295085312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295086336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295087360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295088384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295089408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295090432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295091456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295092480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295093504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295094528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295095552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295096576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295097600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295098624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295099648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295100672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295101696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295102720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295103744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295104768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295105792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295106816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295107840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295108864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295109888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295110912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295111936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295112960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295113984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 4295115264
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -2489,42 +2777,78 @@
 read 512/512 bytes at offset 109568
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 110848 is not sector aligned
-offset 111872 is not sector aligned
-offset 112896 is not sector aligned
-offset 113920 is not sector aligned
-offset 114944 is not sector aligned
-offset 115968 is not sector aligned
-offset 116992 is not sector aligned
-offset 118016 is not sector aligned
-offset 119040 is not sector aligned
-offset 120064 is not sector aligned
-offset 121088 is not sector aligned
-offset 122112 is not sector aligned
-offset 123136 is not sector aligned
-offset 124160 is not sector aligned
-offset 125184 is not sector aligned
-offset 126208 is not sector aligned
-offset 127232 is not sector aligned
-offset 128256 is not sector aligned
-offset 129280 is not sector aligned
-offset 130304 is not sector aligned
-offset 131328 is not sector aligned
-offset 132352 is not sector aligned
-offset 133376 is not sector aligned
-offset 134400 is not sector aligned
-offset 135424 is not sector aligned
-offset 136448 is not sector aligned
-offset 137472 is not sector aligned
-offset 138496 is not sector aligned
-offset 139520 is not sector aligned
-offset 140544 is not sector aligned
-offset 141568 is not sector aligned
-offset 142592 is not sector aligned
-offset 143616 is not sector aligned
-offset 144640 is not sector aligned
-offset 145664 is not sector aligned
-offset 146688 is not sector aligned
+read 512/512 bytes at offset 110848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 111872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 112896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 113920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 114944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 115968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 116992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 118016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 119040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 120064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 121088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 122112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 123136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 124160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 125184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 126208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 127232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 128256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 129280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 130304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 131328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 133376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 134400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 135424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 136448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 137472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 138496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 139520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 140544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 141568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 142592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 143616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 144640
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 145664
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 146688
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 147968
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -2771,42 +3095,78 @@
 read 512/512 bytes at offset 109568
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 110848 is not sector aligned
-offset 111872 is not sector aligned
-offset 112896 is not sector aligned
-offset 113920 is not sector aligned
-offset 114944 is not sector aligned
-offset 115968 is not sector aligned
-offset 116992 is not sector aligned
-offset 118016 is not sector aligned
-offset 119040 is not sector aligned
-offset 120064 is not sector aligned
-offset 121088 is not sector aligned
-offset 122112 is not sector aligned
-offset 123136 is not sector aligned
-offset 124160 is not sector aligned
-offset 125184 is not sector aligned
-offset 126208 is not sector aligned
-offset 127232 is not sector aligned
-offset 128256 is not sector aligned
-offset 129280 is not sector aligned
-offset 130304 is not sector aligned
-offset 131328 is not sector aligned
-offset 132352 is not sector aligned
-offset 133376 is not sector aligned
-offset 134400 is not sector aligned
-offset 135424 is not sector aligned
-offset 136448 is not sector aligned
-offset 137472 is not sector aligned
-offset 138496 is not sector aligned
-offset 139520 is not sector aligned
-offset 140544 is not sector aligned
-offset 141568 is not sector aligned
-offset 142592 is not sector aligned
-offset 143616 is not sector aligned
-offset 144640 is not sector aligned
-offset 145664 is not sector aligned
-offset 146688 is not sector aligned
+read 512/512 bytes at offset 110848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 111872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 112896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 113920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 114944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 115968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 116992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 118016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 119040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 120064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 121088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 122112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 123136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 124160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 125184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 126208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 127232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 128256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 129280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 130304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 131328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 133376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 134400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 135424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 136448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 137472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 138496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 139520
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 140544
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 141568
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 142592
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 143616
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 144640
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 145664
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 146688
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 147968
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -3055,42 +3415,78 @@
 read 512/512 bytes at offset 4295076864
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 4295078144 is not sector aligned
-offset 4295079168 is not sector aligned
-offset 4295080192 is not sector aligned
-offset 4295081216 is not sector aligned
-offset 4295082240 is not sector aligned
-offset 4295083264 is not sector aligned
-offset 4295084288 is not sector aligned
-offset 4295085312 is not sector aligned
-offset 4295086336 is not sector aligned
-offset 4295087360 is not sector aligned
-offset 4295088384 is not sector aligned
-offset 4295089408 is not sector aligned
-offset 4295090432 is not sector aligned
-offset 4295091456 is not sector aligned
-offset 4295092480 is not sector aligned
-offset 4295093504 is not sector aligned
-offset 4295094528 is not sector aligned
-offset 4295095552 is not sector aligned
-offset 4295096576 is not sector aligned
-offset 4295097600 is not sector aligned
-offset 4295098624 is not sector aligned
-offset 4295099648 is not sector aligned
-offset 4295100672 is not sector aligned
-offset 4295101696 is not sector aligned
-offset 4295102720 is not sector aligned
-offset 4295103744 is not sector aligned
-offset 4295104768 is not sector aligned
-offset 4295105792 is not sector aligned
-offset 4295106816 is not sector aligned
-offset 4295107840 is not sector aligned
-offset 4295108864 is not sector aligned
-offset 4295109888 is not sector aligned
-offset 4295110912 is not sector aligned
-offset 4295111936 is not sector aligned
-offset 4295112960 is not sector aligned
-offset 4295113984 is not sector aligned
+read 512/512 bytes at offset 4295078144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295079168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295080192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295081216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295082240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295083264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295084288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295085312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295086336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295087360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295088384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295089408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295090432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295091456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295092480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295093504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295094528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295095552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295096576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295097600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295098624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295099648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295100672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295101696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295102720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295103744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295104768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295105792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295106816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295107840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295108864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295109888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295110912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295111936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295112960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295113984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 4295115264
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -3337,42 +3733,78 @@
 read 512/512 bytes at offset 4295076864
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 216
-offset 4295078144 is not sector aligned
-offset 4295079168 is not sector aligned
-offset 4295080192 is not sector aligned
-offset 4295081216 is not sector aligned
-offset 4295082240 is not sector aligned
-offset 4295083264 is not sector aligned
-offset 4295084288 is not sector aligned
-offset 4295085312 is not sector aligned
-offset 4295086336 is not sector aligned
-offset 4295087360 is not sector aligned
-offset 4295088384 is not sector aligned
-offset 4295089408 is not sector aligned
-offset 4295090432 is not sector aligned
-offset 4295091456 is not sector aligned
-offset 4295092480 is not sector aligned
-offset 4295093504 is not sector aligned
-offset 4295094528 is not sector aligned
-offset 4295095552 is not sector aligned
-offset 4295096576 is not sector aligned
-offset 4295097600 is not sector aligned
-offset 4295098624 is not sector aligned
-offset 4295099648 is not sector aligned
-offset 4295100672 is not sector aligned
-offset 4295101696 is not sector aligned
-offset 4295102720 is not sector aligned
-offset 4295103744 is not sector aligned
-offset 4295104768 is not sector aligned
-offset 4295105792 is not sector aligned
-offset 4295106816 is not sector aligned
-offset 4295107840 is not sector aligned
-offset 4295108864 is not sector aligned
-offset 4295109888 is not sector aligned
-offset 4295110912 is not sector aligned
-offset 4295111936 is not sector aligned
-offset 4295112960 is not sector aligned
-offset 4295113984 is not sector aligned
+read 512/512 bytes at offset 4295078144
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295079168
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295080192
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295081216
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295082240
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295083264
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295084288
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295085312
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295086336
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295087360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295088384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295089408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295090432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295091456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295092480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295093504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295094528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295095552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295096576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295097600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295098624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295099648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295100672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295101696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295102720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295103744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295104768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295105792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295106816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295107840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295108864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295109888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295110912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295111936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295112960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295113984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 33
 read 2048/2048 bytes at offset 4295115264
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -3623,42 +4055,78 @@
 wrote 512/512 bytes at offset 110080
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 111360 is not sector aligned
-offset 112384 is not sector aligned
-offset 113408 is not sector aligned
-offset 114432 is not sector aligned
-offset 115456 is not sector aligned
-offset 116480 is not sector aligned
-offset 117504 is not sector aligned
-offset 118528 is not sector aligned
-offset 119552 is not sector aligned
-offset 120576 is not sector aligned
-offset 121600 is not sector aligned
-offset 122624 is not sector aligned
-offset 123648 is not sector aligned
-offset 124672 is not sector aligned
-offset 125696 is not sector aligned
-offset 126720 is not sector aligned
-offset 127744 is not sector aligned
-offset 128768 is not sector aligned
-offset 129792 is not sector aligned
-offset 130816 is not sector aligned
-offset 131840 is not sector aligned
-offset 132864 is not sector aligned
-offset 133888 is not sector aligned
-offset 134912 is not sector aligned
-offset 135936 is not sector aligned
-offset 136960 is not sector aligned
-offset 137984 is not sector aligned
-offset 139008 is not sector aligned
-offset 140032 is not sector aligned
-offset 141056 is not sector aligned
-offset 142080 is not sector aligned
-offset 143104 is not sector aligned
-offset 144128 is not sector aligned
-offset 145152 is not sector aligned
-offset 146176 is not sector aligned
-offset 147200 is not sector aligned
+wrote 512/512 bytes at offset 111360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 112384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 113408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 114432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 115456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 116480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 117504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 118528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 119552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 120576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 121600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 122624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 123648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 124672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 125696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 126720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 127744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 128768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 129792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 130816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 131840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 132864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 133888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 134912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 135936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 136960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 137984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 139008
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 140032
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 141056
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 142080
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 143104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 144128
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 145152
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 146176
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 147200
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 wrote 2048/2048 bytes at offset 148480
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -3905,42 +4373,78 @@
 read 512/512 bytes at offset 110080
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 111360 is not sector aligned
-offset 112384 is not sector aligned
-offset 113408 is not sector aligned
-offset 114432 is not sector aligned
-offset 115456 is not sector aligned
-offset 116480 is not sector aligned
-offset 117504 is not sector aligned
-offset 118528 is not sector aligned
-offset 119552 is not sector aligned
-offset 120576 is not sector aligned
-offset 121600 is not sector aligned
-offset 122624 is not sector aligned
-offset 123648 is not sector aligned
-offset 124672 is not sector aligned
-offset 125696 is not sector aligned
-offset 126720 is not sector aligned
-offset 127744 is not sector aligned
-offset 128768 is not sector aligned
-offset 129792 is not sector aligned
-offset 130816 is not sector aligned
-offset 131840 is not sector aligned
-offset 132864 is not sector aligned
-offset 133888 is not sector aligned
-offset 134912 is not sector aligned
-offset 135936 is not sector aligned
-offset 136960 is not sector aligned
-offset 137984 is not sector aligned
-offset 139008 is not sector aligned
-offset 140032 is not sector aligned
-offset 141056 is not sector aligned
-offset 142080 is not sector aligned
-offset 143104 is not sector aligned
-offset 144128 is not sector aligned
-offset 145152 is not sector aligned
-offset 146176 is not sector aligned
-offset 147200 is not sector aligned
+read 512/512 bytes at offset 111360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 112384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 113408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 114432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 115456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 116480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 117504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 118528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 119552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 120576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 121600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 122624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 123648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 124672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 125696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 126720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 127744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 128768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 129792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 130816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 131840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 133888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 134912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 135936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 136960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 137984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 139008
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 140032
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 141056
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 142080
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 143104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 144128
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 145152
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 146176
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 147200
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 read 2048/2048 bytes at offset 148480
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -4187,42 +4691,78 @@
 wrote 512/512 bytes at offset 110080
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 111360 is not sector aligned
-offset 112384 is not sector aligned
-offset 113408 is not sector aligned
-offset 114432 is not sector aligned
-offset 115456 is not sector aligned
-offset 116480 is not sector aligned
-offset 117504 is not sector aligned
-offset 118528 is not sector aligned
-offset 119552 is not sector aligned
-offset 120576 is not sector aligned
-offset 121600 is not sector aligned
-offset 122624 is not sector aligned
-offset 123648 is not sector aligned
-offset 124672 is not sector aligned
-offset 125696 is not sector aligned
-offset 126720 is not sector aligned
-offset 127744 is not sector aligned
-offset 128768 is not sector aligned
-offset 129792 is not sector aligned
-offset 130816 is not sector aligned
-offset 131840 is not sector aligned
-offset 132864 is not sector aligned
-offset 133888 is not sector aligned
-offset 134912 is not sector aligned
-offset 135936 is not sector aligned
-offset 136960 is not sector aligned
-offset 137984 is not sector aligned
-offset 139008 is not sector aligned
-offset 140032 is not sector aligned
-offset 141056 is not sector aligned
-offset 142080 is not sector aligned
-offset 143104 is not sector aligned
-offset 144128 is not sector aligned
-offset 145152 is not sector aligned
-offset 146176 is not sector aligned
-offset 147200 is not sector aligned
+wrote 512/512 bytes at offset 111360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 112384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 113408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 114432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 115456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 116480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 117504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 118528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 119552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 120576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 121600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 122624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 123648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 124672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 125696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 126720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 127744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 128768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 129792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 130816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 131840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 132864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 133888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 134912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 135936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 136960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 137984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 139008
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 140032
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 141056
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 142080
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 143104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 144128
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 145152
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 146176
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 147200
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 wrote 2048/2048 bytes at offset 148480
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -4469,42 +5009,78 @@
 read 512/512 bytes at offset 110080
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 111360 is not sector aligned
-offset 112384 is not sector aligned
-offset 113408 is not sector aligned
-offset 114432 is not sector aligned
-offset 115456 is not sector aligned
-offset 116480 is not sector aligned
-offset 117504 is not sector aligned
-offset 118528 is not sector aligned
-offset 119552 is not sector aligned
-offset 120576 is not sector aligned
-offset 121600 is not sector aligned
-offset 122624 is not sector aligned
-offset 123648 is not sector aligned
-offset 124672 is not sector aligned
-offset 125696 is not sector aligned
-offset 126720 is not sector aligned
-offset 127744 is not sector aligned
-offset 128768 is not sector aligned
-offset 129792 is not sector aligned
-offset 130816 is not sector aligned
-offset 131840 is not sector aligned
-offset 132864 is not sector aligned
-offset 133888 is not sector aligned
-offset 134912 is not sector aligned
-offset 135936 is not sector aligned
-offset 136960 is not sector aligned
-offset 137984 is not sector aligned
-offset 139008 is not sector aligned
-offset 140032 is not sector aligned
-offset 141056 is not sector aligned
-offset 142080 is not sector aligned
-offset 143104 is not sector aligned
-offset 144128 is not sector aligned
-offset 145152 is not sector aligned
-offset 146176 is not sector aligned
-offset 147200 is not sector aligned
+read 512/512 bytes at offset 111360
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 112384
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 113408
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 114432
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 115456
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 116480
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 117504
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 118528
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 119552
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 120576
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 121600
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 122624
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 123648
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 124672
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 125696
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 126720
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 127744
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 128768
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 129792
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 130816
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 131840
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132864
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 133888
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 134912
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 135936
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 136960
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 137984
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 139008
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 140032
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 141056
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 142080
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 143104
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 144128
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 145152
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 146176
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 147200
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 read 2048/2048 bytes at offset 148480
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -4753,42 +5329,78 @@
 wrote 512/512 bytes at offset 4295077376
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 4295078656 is not sector aligned
-offset 4295079680 is not sector aligned
-offset 4295080704 is not sector aligned
-offset 4295081728 is not sector aligned
-offset 4295082752 is not sector aligned
-offset 4295083776 is not sector aligned
-offset 4295084800 is not sector aligned
-offset 4295085824 is not sector aligned
-offset 4295086848 is not sector aligned
-offset 4295087872 is not sector aligned
-offset 4295088896 is not sector aligned
-offset 4295089920 is not sector aligned
-offset 4295090944 is not sector aligned
-offset 4295091968 is not sector aligned
-offset 4295092992 is not sector aligned
-offset 4295094016 is not sector aligned
-offset 4295095040 is not sector aligned
-offset 4295096064 is not sector aligned
-offset 4295097088 is not sector aligned
-offset 4295098112 is not sector aligned
-offset 4295099136 is not sector aligned
-offset 4295100160 is not sector aligned
-offset 4295101184 is not sector aligned
-offset 4295102208 is not sector aligned
-offset 4295103232 is not sector aligned
-offset 4295104256 is not sector aligned
-offset 4295105280 is not sector aligned
-offset 4295106304 is not sector aligned
-offset 4295107328 is not sector aligned
-offset 4295108352 is not sector aligned
-offset 4295109376 is not sector aligned
-offset 4295110400 is not sector aligned
-offset 4295111424 is not sector aligned
-offset 4295112448 is not sector aligned
-offset 4295113472 is not sector aligned
-offset 4295114496 is not sector aligned
+wrote 512/512 bytes at offset 4295078656
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295079680
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295080704
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295081728
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295082752
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295083776
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295084800
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295085824
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295086848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295087872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295088896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295089920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295090944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295091968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295092992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295094016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295095040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295096064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295097088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295098112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295099136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295100160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295101184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295102208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295103232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295104256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295105280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295106304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295107328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295108352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295109376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295110400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295111424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295112448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295113472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295114496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 wrote 2048/2048 bytes at offset 4295115776
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -5035,42 +5647,78 @@
 read 512/512 bytes at offset 4295077376
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 4295078656 is not sector aligned
-offset 4295079680 is not sector aligned
-offset 4295080704 is not sector aligned
-offset 4295081728 is not sector aligned
-offset 4295082752 is not sector aligned
-offset 4295083776 is not sector aligned
-offset 4295084800 is not sector aligned
-offset 4295085824 is not sector aligned
-offset 4295086848 is not sector aligned
-offset 4295087872 is not sector aligned
-offset 4295088896 is not sector aligned
-offset 4295089920 is not sector aligned
-offset 4295090944 is not sector aligned
-offset 4295091968 is not sector aligned
-offset 4295092992 is not sector aligned
-offset 4295094016 is not sector aligned
-offset 4295095040 is not sector aligned
-offset 4295096064 is not sector aligned
-offset 4295097088 is not sector aligned
-offset 4295098112 is not sector aligned
-offset 4295099136 is not sector aligned
-offset 4295100160 is not sector aligned
-offset 4295101184 is not sector aligned
-offset 4295102208 is not sector aligned
-offset 4295103232 is not sector aligned
-offset 4295104256 is not sector aligned
-offset 4295105280 is not sector aligned
-offset 4295106304 is not sector aligned
-offset 4295107328 is not sector aligned
-offset 4295108352 is not sector aligned
-offset 4295109376 is not sector aligned
-offset 4295110400 is not sector aligned
-offset 4295111424 is not sector aligned
-offset 4295112448 is not sector aligned
-offset 4295113472 is not sector aligned
-offset 4295114496 is not sector aligned
+read 512/512 bytes at offset 4295078656
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295079680
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295080704
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295081728
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295082752
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295083776
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295084800
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295085824
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295086848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295087872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295088896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295089920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295090944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295091968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295092992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295094016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295095040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295096064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295097088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295098112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295099136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295100160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295101184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295102208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295103232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295104256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295105280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295106304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295107328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295108352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295109376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295110400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295111424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295112448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295113472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295114496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 read 2048/2048 bytes at offset 4295115776
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -5317,42 +5965,78 @@
 wrote 512/512 bytes at offset 4295077376
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 4295078656 is not sector aligned
-offset 4295079680 is not sector aligned
-offset 4295080704 is not sector aligned
-offset 4295081728 is not sector aligned
-offset 4295082752 is not sector aligned
-offset 4295083776 is not sector aligned
-offset 4295084800 is not sector aligned
-offset 4295085824 is not sector aligned
-offset 4295086848 is not sector aligned
-offset 4295087872 is not sector aligned
-offset 4295088896 is not sector aligned
-offset 4295089920 is not sector aligned
-offset 4295090944 is not sector aligned
-offset 4295091968 is not sector aligned
-offset 4295092992 is not sector aligned
-offset 4295094016 is not sector aligned
-offset 4295095040 is not sector aligned
-offset 4295096064 is not sector aligned
-offset 4295097088 is not sector aligned
-offset 4295098112 is not sector aligned
-offset 4295099136 is not sector aligned
-offset 4295100160 is not sector aligned
-offset 4295101184 is not sector aligned
-offset 4295102208 is not sector aligned
-offset 4295103232 is not sector aligned
-offset 4295104256 is not sector aligned
-offset 4295105280 is not sector aligned
-offset 4295106304 is not sector aligned
-offset 4295107328 is not sector aligned
-offset 4295108352 is not sector aligned
-offset 4295109376 is not sector aligned
-offset 4295110400 is not sector aligned
-offset 4295111424 is not sector aligned
-offset 4295112448 is not sector aligned
-offset 4295113472 is not sector aligned
-offset 4295114496 is not sector aligned
+wrote 512/512 bytes at offset 4295078656
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295079680
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295080704
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295081728
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295082752
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295083776
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295084800
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295085824
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295086848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295087872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295088896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295089920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295090944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295091968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295092992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295094016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295095040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295096064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295097088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295098112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295099136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295100160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295101184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295102208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295103232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295104256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295105280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295106304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295107328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295108352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295109376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295110400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295111424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295112448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295113472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 4295114496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 wrote 2048/2048 bytes at offset 4295115776
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -5599,42 +6283,78 @@
 read 512/512 bytes at offset 4295077376
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 217
-offset 4295078656 is not sector aligned
-offset 4295079680 is not sector aligned
-offset 4295080704 is not sector aligned
-offset 4295081728 is not sector aligned
-offset 4295082752 is not sector aligned
-offset 4295083776 is not sector aligned
-offset 4295084800 is not sector aligned
-offset 4295085824 is not sector aligned
-offset 4295086848 is not sector aligned
-offset 4295087872 is not sector aligned
-offset 4295088896 is not sector aligned
-offset 4295089920 is not sector aligned
-offset 4295090944 is not sector aligned
-offset 4295091968 is not sector aligned
-offset 4295092992 is not sector aligned
-offset 4295094016 is not sector aligned
-offset 4295095040 is not sector aligned
-offset 4295096064 is not sector aligned
-offset 4295097088 is not sector aligned
-offset 4295098112 is not sector aligned
-offset 4295099136 is not sector aligned
-offset 4295100160 is not sector aligned
-offset 4295101184 is not sector aligned
-offset 4295102208 is not sector aligned
-offset 4295103232 is not sector aligned
-offset 4295104256 is not sector aligned
-offset 4295105280 is not sector aligned
-offset 4295106304 is not sector aligned
-offset 4295107328 is not sector aligned
-offset 4295108352 is not sector aligned
-offset 4295109376 is not sector aligned
-offset 4295110400 is not sector aligned
-offset 4295111424 is not sector aligned
-offset 4295112448 is not sector aligned
-offset 4295113472 is not sector aligned
-offset 4295114496 is not sector aligned
+read 512/512 bytes at offset 4295078656
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295079680
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295080704
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295081728
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295082752
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295083776
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295084800
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295085824
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295086848
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295087872
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295088896
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295089920
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295090944
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295091968
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295092992
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295094016
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295095040
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295096064
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295097088
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295098112
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295099136
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295100160
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295101184
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295102208
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295103232
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295104256
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295105280
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295106304
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295107328
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295108352
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295109376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295110400
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295111424
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295112448
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295113472
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 4295114496
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 === IO: pattern 34
 read 2048/2048 bytes at offset 4295115776
 2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -5664,7 +6384,7 @@
 No errors were found on the image.
 Creating another new image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 More complex patterns
 
 test2: With offset 0
@@ -5887,7 +6607,7 @@
 No errors were found on the image.
 Creating new image; cluster size: 4096
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 Testing empty image
 
 At offset 0:
@@ -12270,7 +12990,7 @@
 No errors were found on the image.
 Creating another new image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 More complex patterns
 
 test2: With offset 0
@@ -12493,7 +13213,7 @@
 No errors were found on the image.
 Creating new image; cluster size: 16384
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 Testing empty image
 
 At offset 0:
@@ -18876,7 +19596,7 @@
 No errors were found on the image.
 Creating another new image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 More complex patterns
 
 test2: With offset 0
@@ -19099,7 +19819,7 @@
 No errors were found on the image.
 Creating new image; cluster size: 65536
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 Testing empty image
 
 At offset 0:
@@ -25482,7 +26202,7 @@
 No errors were found on the image.
 Creating another new image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=8589934592
 More complex patterns
 
 test2: With offset 0
diff --git a/tests/qemu-iotests/024 b/tests/qemu-iotests/024
index 9bf99e1..e0d77ce 100755
--- a/tests/qemu-iotests/024
+++ b/tests/qemu-iotests/024
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -60,16 +59,22 @@
 echo "Creating backing file"
 echo
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base_old"
+
 _make_test_img 1G
 io_pattern writev 0 $CLUSTER_SIZE $((2 * CLUSTER_SIZE)) 8 0x11
-mv "$TEST_IMG" "$TEST_IMG.base_old"
+
+TEST_IMG="$TEST_IMG_SAVE.base_new"
 
 echo "Creating new backing file"
 echo
 
 _make_test_img 1G
 io_pattern writev 0 $((2 * CLUSTER_SIZE)) $((4 * CLUSTER_SIZE)) 4 0x22
-mv "$TEST_IMG" "$TEST_IMG.base_new"
+
+
+TEST_IMG="$TEST_IMG_SAVE"
 
 echo "Creating COW image"
 echo
diff --git a/tests/qemu-iotests/024.out b/tests/qemu-iotests/024.out
index e84b973..33cfaf5 100644
--- a/tests/qemu-iotests/024.out
+++ b/tests/qemu-iotests/024.out
@@ -1,7 +1,7 @@
 QA output created by 024
 Creating backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT.base_old', fmt=IMGFMT size=1073741824
 === IO: pattern 0x11
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -21,7 +21,7 @@
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Creating new backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT.base_new', fmt=IMGFMT size=1073741824
 === IO: pattern 0x22
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -33,7 +33,7 @@
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Creating COW image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 backing_file='TEST_DIR/t.IMGFMT.base_old' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 backing_file=TEST_DIR/t.IMGFMT.base_old
 === IO: pattern 0x33
 wrote 262144/262144 bytes at offset 0
 256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/qemu-iotests/025 b/tests/qemu-iotests/025
index 467a4b7..c41370f 100755
--- a/tests/qemu-iotests/025
+++ b/tests/qemu-iotests/025
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/025.out b/tests/qemu-iotests/025.out
index 8c695e6..f13fc28 100644
--- a/tests/qemu-iotests/025.out
+++ b/tests/qemu-iotests/025.out
@@ -1,7 +1,7 @@
 QA output created by 025
 === Creating image
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 === Writing whole image
 === IO: pattern 0xc5
diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026
index df2884b..f5a7f02 100755
--- a/tests/qemu-iotests/026
+++ b/tests/qemu-iotests/026
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -46,6 +45,13 @@
 _supported_os Linux
 _default_cache_mode "writethrough"
 _supported_cache_modes "writethrough" "none"
+# The refcount table tests expect a certain minimum width for refcount entries
+# (so that the refcount table actually needs to grow); that minimum is 16 bits,
+# being the default refcount entry width.
+# 32 and 64 bits do not work either, however, due to different leaked cluster
+# count on error.
+# Thus, the only remaining option is refcount_bits=16.
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 echo "Errors while writing 128 kB"
 echo
@@ -59,7 +65,7 @@
     \
     l2_load \
     l2_update \
-    l2_alloc.write \
+    l2_alloc_write \
     \
     write_aio \
     \
@@ -119,11 +125,11 @@
 
 
 for event in \
-    refblock_alloc.hookup \
-    refblock_alloc.write \
-    refblock_alloc.write_blocks \
-    refblock_alloc.write_table \
-    refblock_alloc.switch_table \
+    refblock_alloc_hookup \
+    refblock_alloc_write \
+    refblock_alloc_write_blocks \
+    refblock_alloc_write_table \
+    refblock_alloc_switch_table \
 
 do
 
@@ -163,9 +169,9 @@
 
 
 for event in \
-    l1_grow.alloc_table \
-    l1_grow.write_table \
-    l1_grow.activate_table \
+    l1_grow_alloc_table \
+    l1_grow_write_table \
+    l1_grow_activate_table \
 
 do
 
diff --git a/tests/qemu-iotests/026.out b/tests/qemu-iotests/026.out
index f7c78e7..8531735 100644
--- a/tests/qemu-iotests/026.out
+++ b/tests/qemu-iotests/026.out
@@ -1,63 +1,67 @@
 QA output created by 026
 Errors while writing 128 kB
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_update; errno: 5; imm: off; once: on; write 
+Event: l1_update; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_update; errno: 5; imm: off; once: off; write 
+Event: l1_update; errno: 5; imm: off; once: off; write
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_update; errno: 28; imm: off; once: on; write 
+Event: l1_update; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_update; errno: 28; imm: off; once: off; write 
+Event: l1_update; errno: 28; imm: off; once: off; write
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l1_update; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_load; errno: 5; imm: off; once: on; write 
+Event: l2_load; errno: 5; imm: off; once: on; write
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 write failed: Input/output error
 read failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_load; errno: 5; imm: off; once: on; write -b
 wrote 131072/131072 bytes at offset 0
@@ -65,15 +69,15 @@
 write failed: Input/output error
 read failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_load; errno: 5; imm: off; once: off; write 
+Event: l2_load; errno: 5; imm: off; once: off; write
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 write failed: Input/output error
 read failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_load; errno: 5; imm: off; once: off; write -b
 wrote 131072/131072 bytes at offset 0
@@ -81,15 +85,15 @@
 write failed: Input/output error
 read failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_load; errno: 28; imm: off; once: on; write 
+Event: l2_load; errno: 28; imm: off; once: on; write
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 write failed: No space left on device
 read failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_load; errno: 28; imm: off; once: on; write -b
 wrote 131072/131072 bytes at offset 0
@@ -97,15 +101,15 @@
 write failed: No space left on device
 read failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_load; errno: 28; imm: off; once: off; write 
+Event: l2_load; errno: 28; imm: off; once: off; write
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 write failed: No space left on device
 read failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_load; errno: 28; imm: off; once: off; write -b
 wrote 131072/131072 bytes at offset 0
@@ -113,302 +117,330 @@
 write failed: No space left on device
 read failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_update; errno: 5; imm: off; once: on; write 
+Event: l2_update; errno: 5; imm: off; once: on; write
 write failed: Input/output error
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
-
-Event: l2_update; errno: 5; imm: off; once: off; write 
+Event: l2_update; errno: 5; imm: off; once: off; write
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 127 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 127 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_update; errno: 28; imm: off; once: on; write 
+Event: l2_update; errno: 28; imm: off; once: on; write
 write failed: No space left on device
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
-
-Event: l2_update; errno: 28; imm: off; once: off; write 
+Event: l2_update; errno: 28; imm: off; once: off; write
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 127 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 127 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write 
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write 
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write 
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write 
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: write_aio; errno: 5; imm: off; once: on; write 
+Event: write_aio; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: write_aio; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: write_aio; errno: 5; imm: off; once: off; write 
+Event: write_aio; errno: 5; imm: off; once: off; write
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: write_aio; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: write_aio; errno: 28; imm: off; once: on; write 
+Event: write_aio; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: write_aio; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: write_aio; errno: 28; imm: off; once: off; write 
+Event: write_aio; errno: 28; imm: off; once: off; write
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: write_aio; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_load; errno: 5; imm: off; once: on; write 
+Event: refblock_load; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_load; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_load; errno: 5; imm: off; once: off; write 
+Event: refblock_load; errno: 5; imm: off; once: off; write
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_load; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_load; errno: 28; imm: off; once: on; write 
+Event: refblock_load; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_load; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_load; errno: 28; imm: off; once: off; write 
+Event: refblock_load; errno: 28; imm: off; once: off; write
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_load; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_update_part; errno: 5; imm: off; once: on; write 
+Event: refblock_update_part; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_update_part; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_update_part; errno: 5; imm: off; once: off; write 
+Event: refblock_update_part; errno: 5; imm: off; once: off; write
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_update_part; errno: 28; imm: off; once: on; write 
+Event: refblock_update_part; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_update_part; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_update_part; errno: 28; imm: off; once: off; write 
+Event: refblock_update_part; errno: 28; imm: off; once: off; write
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc; errno: 5; imm: off; once: on; write 
+Event: refblock_alloc; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_alloc; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc; errno: 5; imm: off; once: off; write 
+Event: refblock_alloc; errno: 5; imm: off; once: off; write
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_alloc; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc; errno: 28; imm: off; once: off; write
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: cluster_alloc; errno: 5; imm: off; once: on; write 
+Event: cluster_alloc; errno: 5; imm: off; once: on; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: cluster_alloc; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: cluster_alloc; errno: 5; imm: off; once: off; write 
+Event: cluster_alloc; errno: 5; imm: off; once: off; write
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: cluster_alloc; errno: 5; imm: off; once: off; write -b
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: cluster_alloc; errno: 28; imm: off; once: on; write 
+Event: cluster_alloc; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: cluster_alloc; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: cluster_alloc; errno: 28; imm: off; once: off; write 
+Event: cluster_alloc; errno: 28; imm: off; once: off; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: cluster_alloc; errno: 28; imm: off; once: off; write -b
 write failed: No space left on device
@@ -416,118 +448,134 @@
 
 === Refcout table growth tests ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 55 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 251 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
@@ -535,66 +583,72 @@
 
 === L1 growth tests ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: on
+Event: l1_grow_write_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: off
+Event: l1_grow_write_table; errno: 5; imm: off; once: off
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: on
+Event: l1_grow_write_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: off
+Event: l1_grow_write_table; errno: 28; imm: off; once: off
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: on
+Event: l1_grow_activate_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: off
+Event: l1_grow_activate_table; errno: 5; imm: off; once: off
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 96 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: on
+Event: l1_grow_activate_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: off
+Event: l1_grow_activate_table; errno: 28; imm: off; once: off
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 96 leaked clusters were found on the image.
diff --git a/tests/qemu-iotests/026.out.nocache b/tests/qemu-iotests/026.out.nocache
index c9d242e..672d77c 100644
--- a/tests/qemu-iotests/026.out.nocache
+++ b/tests/qemu-iotests/026.out.nocache
@@ -14,6 +14,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write 
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
@@ -21,6 +22,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
@@ -38,6 +40,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write 
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
@@ -45,6 +48,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l1_update; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
@@ -118,20 +122,18 @@
 Event: l2_update; errno: 5; imm: off; once: on; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 5; imm: off; once: on; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -140,6 +142,8 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_update; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -150,20 +154,18 @@
 Event: l2_update; errno: 28; imm: off; once: on; write 
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 28; imm: off; once: on; write -b
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-127 leaked clusters were found on the image.
-This means waste of disk space, but no harm to data.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+No errors were found on the image.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 Event: l2_update; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -172,6 +174,8 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: l2_update; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
@@ -179,44 +183,52 @@
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write 
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write 
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: on; write -b
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write 
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 5; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 1 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write 
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write 
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l2_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: l2_alloc_write; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 1 leaked clusters were found on the image.
@@ -234,11 +246,15 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 5; imm: off; once: off; write 
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 5; imm: off; once: off; write -b
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -254,11 +270,15 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: write_aio; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -274,11 +294,13 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 5; imm: off; once: off; write 
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -294,11 +316,13 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 28; imm: off; once: off; write 
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_load; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -314,11 +338,13 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write 
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -334,11 +360,13 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write 
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_update_part; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -354,11 +382,13 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write 
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 5; imm: off; once: off; write -b
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -374,11 +404,13 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write 
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
 Event: refblock_alloc; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
@@ -426,116 +458,132 @@
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 55 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.hookup; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_hookup; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 251 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write 
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
-10 leaked clusters were found on the image.
+11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_blocks; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_blocks; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write 
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
-10 leaked clusters were found on the image.
+11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.write_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_write_table; errno: 28; imm: off; once: off; write -b
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write 
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write 
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: on; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: on; write -b
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write 
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write 
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
-10 leaked clusters were found on the image.
+11 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: refblock_alloc.switch_table; errno: 28; imm: off; once: off; write -b
+Event: refblock_alloc_switch_table; errno: 28; imm: off; once: off; write -b
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 23 leaked clusters were found on the image.
@@ -545,64 +593,70 @@
 
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 5; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 5; imm: off; once: off
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: on
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.alloc_table; errno: 28; imm: off; once: off
+Event: l1_grow_alloc_table; errno: 28; imm: off; once: off
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: on
+Event: l1_grow_write_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 5; imm: off; once: off
+Event: l1_grow_write_table; errno: 5; imm: off; once: off
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: on
+Event: l1_grow_write_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.write_table; errno: 28; imm: off; once: off
+Event: l1_grow_write_table; errno: 28; imm: off; once: off
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: on
+Event: l1_grow_activate_table; errno: 5; imm: off; once: on
 write failed: Input/output error
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 5; imm: off; once: off
+Event: l1_grow_activate_table; errno: 5; imm: off; once: off
+Failed to flush the L2 table cache: Input/output error
+Failed to flush the refcount block cache: Input/output error
 write failed: Input/output error
 
 96 leaked clusters were found on the image.
 This means waste of disk space, but no harm to data.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: on
+Event: l1_grow_activate_table; errno: 28; imm: off; once: on
 write failed: No space left on device
 No errors were found on the image.
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
 
-Event: l1_grow.activate_table; errno: 28; imm: off; once: off
+Event: l1_grow_activate_table; errno: 28; imm: off; once: off
+Failed to flush the L2 table cache: No space left on device
+Failed to flush the refcount block cache: No space left on device
 write failed: No space left on device
 
 96 leaked clusters were found on the image.
diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027
index 3fa81b8..08593da 100755
--- a/tests/qemu-iotests/027
+++ b/tests/qemu-iotests/027
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/027.out b/tests/qemu-iotests/027.out
index 4fcb416..c7bca71 100644
--- a/tests/qemu-iotests/027.out
+++ b/tests/qemu-iotests/027.out
@@ -1,5 +1,5 @@
 QA output created by 027
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == writing first cluster to populate metadata ==
 wrote 65536/65536 bytes at offset 65536
diff --git a/tests/qemu-iotests/028 b/tests/qemu-iotests/028
index a1f4423..7783e57 100755
--- a/tests/qemu-iotests/028
+++ b/tests/qemu-iotests/028
@@ -28,7 +28,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -60,6 +59,9 @@
 
 offset=$(( base_size - 32 * 1024 ))
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $base_size
 
 echo "Filling base image"
@@ -73,7 +75,7 @@
 echo "Creating test image with backing file"
 echo
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 _make_test_img -b "$TEST_IMG.base" $image_size
 
 echo "Filling test image"
@@ -111,10 +113,12 @@
 QEMU_COMM_TIMEOUT=1
 
 # Silence output since it contains the disk image path and QEMU's readline
-# character echoing makes it very hard to filter the output
+# character echoing makes it very hard to filter the output. Plus, there
+# is no telling how many times the command will repeat before succeeding.
 _send_qemu_cmd $h "drive_backup disk ${TEST_IMG}.copy" "(qemu)" >/dev/null
 _send_qemu_cmd $h "" "Formatting" | _filter_img_create
-qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs"
+qemu_cmd_repeat=20 _send_qemu_cmd $h "info block-jobs" "No active jobs" >/dev/null
+_send_qemu_cmd $h "info block-jobs" "No active jobs"
 _send_qemu_cmd $h 'quit' ""
 
 # Base image sectors
diff --git a/tests/qemu-iotests/028.out b/tests/qemu-iotests/028.out
index e8d0245..acd2870 100644
--- a/tests/qemu-iotests/028.out
+++ b/tests/qemu-iotests/028.out
@@ -1,5 +1,5 @@
 QA output created by 028
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=3221227008 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=3221227008
 Filling base image
 
 === IO: pattern 195
@@ -70,7 +70,7 @@
 No errors were found on the image.
 Creating test image with backing file
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base
 Filling test image
 
 === IO: pattern 196
@@ -468,11 +468,8 @@
 
 block-backup
 
-Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file='TEST_DIR/t.IMGFMT.base' backing_fmt='IMGFMT'
-(qemu) 
+Formatting 'TEST_DIR/t.IMGFMT.copy', fmt=IMGFMT size=4294968832 backing_file=TEST_DIR/t.IMGFMT.base backing_fmt=IMGFMT
 (qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block-[K[D[D[D[D[D[D[D[D[D[D[Dinfo block-j[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block-jo[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block-job[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block-jobs[K
-Type backup, device disk: Completed 0 of 4294968832 bytes, speed limit 0 bytes/s
-i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block-[K[D[D[D[D[D[D[D[D[D[D[Dinfo block-j[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block-jo[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block-job[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block-jobs[K
 No active jobs
 === IO: pattern 195
 read 512/512 bytes at offset 3221194240
diff --git a/tests/qemu-iotests/029 b/tests/qemu-iotests/029
index fa46ace..e639ac0 100755
--- a/tests/qemu-iotests/029
+++ b/tests/qemu-iotests/029
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -44,6 +43,8 @@
 _supported_fmt qcow2
 _supported_proto generic
 _supported_os Linux
+# Internal snapshots are (currently) impossible with refcount_bits=1
+_unsupported_imgopts 'refcount_bits=1[^0-9]'
 
 offset_size=24
 offset_l1_size=36
diff --git a/tests/qemu-iotests/029.out b/tests/qemu-iotests/029.out
index ce0e64d..5bc93e0 100644
--- a/tests/qemu-iotests/029.out
+++ b/tests/qemu-iotests/029.out
@@ -3,18 +3,18 @@
 Test loading internal snapshots where the L1 table of the snapshot
 is smaller than the current L1 table.
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=16777216 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=16777216
 wrote 4194304/4194304 bytes at offset 0
 4 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
 
 Try using a huge VM state
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 4096/4096 bytes at offset 1099511627776
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 4096/4096 bytes at offset 1099511627776
@@ -23,5 +23,5 @@
 
 qcow2_snapshot_load_tmp() should take the L1 size from the snapshot
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 *** done
diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 8ce2373..3ac2443 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -34,7 +34,8 @@
         iotests.create_image(backing_img, TestSingleDrive.image_len)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
-        qemu_io('-c', 'write -P 0x1 0 512', backing_img)
+        qemu_io('-f', 'raw', '-c', 'write -P 0x1 0 512', backing_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0x1 524288 512', mid_img)
         self.vm = iotests.VM().add_drive("blkdebug::" + test_img)
         self.vm.launch()
 
@@ -55,8 +56,8 @@
         self.assert_no_active_block_jobs()
         self.vm.shutdown()
 
-        self.assertEqual(qemu_io('-c', 'map', backing_img),
-                         qemu_io('-c', 'map', test_img),
+        self.assertEqual(qemu_io('-f', 'raw', '-c', 'map', backing_img),
+                         qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
 
     def test_stream_pause(self):
@@ -86,13 +87,17 @@
         self.assert_no_active_block_jobs()
         self.vm.shutdown()
 
-        self.assertEqual(qemu_io('-c', 'map', backing_img),
-                         qemu_io('-c', 'map', test_img),
+        self.assertEqual(qemu_io('-f', 'raw', '-c', 'map', backing_img),
+                         qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
 
-    def test_stream_partial(self):
+    def test_stream_no_op(self):
         self.assert_no_active_block_jobs()
 
+        # The image map is empty before the operation
+        empty_map = qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img)
+
+        # This is a no-op: no data should ever be copied from the base image
         result = self.vm.qmp('block-stream', device='drive0', base=mid_img)
         self.assert_qmp(result, 'return', {})
 
@@ -101,8 +106,22 @@
         self.assert_no_active_block_jobs()
         self.vm.shutdown()
 
-        self.assertEqual(qemu_io('-c', 'map', mid_img),
-                         qemu_io('-c', 'map', test_img),
+        self.assertEqual(qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
+                         empty_map, 'image file map changed after a no-op')
+
+    def test_stream_partial(self):
+        self.assert_no_active_block_jobs()
+
+        result = self.vm.qmp('block-stream', device='drive0', base=backing_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_until_completed()
+
+        self.assert_no_active_block_jobs()
+        self.vm.shutdown()
+
+        self.assertEqual(qemu_io('-f', iotests.imgfmt, '-c', 'map', mid_img),
+                         qemu_io('-f', iotests.imgfmt, '-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
 
     def test_device_not_found(self):
@@ -245,6 +264,7 @@
         while not completed:
             for event in self.vm.get_qmp_events(wait=True):
                 if event['event'] == 'BLOCK_JOB_ERROR':
+                    error = True
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/operation', 'read')
 
@@ -257,9 +277,11 @@
                     self.assert_qmp(result, 'return', {})
 
                     result = self.vm.qmp('query-block-jobs')
+                    if result == {'return': []}:
+                        # Race; likely already finished. Check.
+                        continue
                     self.assert_qmp(result, 'return[0]/paused', False)
                     self.assert_qmp(result, 'return[0]/io-status', 'ok')
-                    error = True
                 elif event['event'] == 'BLOCK_JOB_COMPLETED':
                     self.assertTrue(error, 'job completed unexpectedly')
                     self.assert_qmp(event, 'data/type', 'stream')
@@ -359,9 +381,9 @@
 
     def setUp(self):
         qemu_img('create', backing_img, str(TestStreamStop.image_len))
-        qemu_io('-c', 'write -P 0x1 0 32M', backing_img)
+        qemu_io('-f', 'raw', '-c', 'write -P 0x1 0 32M', backing_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
-        qemu_io('-c', 'write -P 0x1 32M 32M', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0x1 32M 32M', test_img)
         self.vm = iotests.VM().add_drive("blkdebug::" + test_img)
         self.vm.launch()
 
@@ -388,9 +410,9 @@
 
     def setUp(self):
         qemu_img('create', backing_img, str(TestSetSpeed.image_len))
-        qemu_io('-c', 'write -P 0x1 0 32M', backing_img)
+        qemu_io('-f', 'raw', '-c', 'write -P 0x1 0 32M', backing_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
-        qemu_io('-c', 'write -P 0x1 32M 32M', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0x1 32M 32M', test_img)
         self.vm = iotests.VM().add_drive('blkdebug::' + test_img)
         self.vm.launch()
 
diff --git a/tests/qemu-iotests/030.out b/tests/qemu-iotests/030.out
index fa16b5c..6323079 100644
--- a/tests/qemu-iotests/030.out
+++ b/tests/qemu-iotests/030.out
@@ -1,5 +1,5 @@
-.............
+..............
 ----------------------------------------------------------------------
-Ran 13 tests
+Ran 14 tests
 
 OK
diff --git a/tests/qemu-iotests/031 b/tests/qemu-iotests/031
index 2a77ba8..1e08abc 100755
--- a/tests/qemu-iotests/031
+++ b/tests/qemu-iotests/031
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out
index a943344..7f5050b 100644
--- a/tests/qemu-iotests/031.out
+++ b/tests/qemu-iotests/031.out
@@ -4,7 +4,7 @@
 
 === Create image with unknown header extension ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 magic                     0x514649fb
 version                   2
 backing_file_offset       0x0
@@ -53,11 +53,6 @@
 header_length             72
 
 Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
-Header extension:
 magic                     0x12345678
 length                    31
 data                      'This is a test header extension'
@@ -68,7 +63,7 @@
 
 magic                     0x514649fb
 version                   2
-backing_file_offset       0x128
+backing_file_offset       0x90
 backing_file_size         0x17
 cluster_bits              16
 size                      67108864
@@ -91,11 +86,6 @@
 data                      'host_device'
 
 Header extension:
-magic                     0x6803f857
-length                    144
-data                      <binary>
-
-Header extension:
 magic                     0x12345678
 length                    31
 data                      'This is a test header extension'
@@ -105,7 +95,7 @@
 
 === Create image with unknown header extension ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -126,6 +116,11 @@
 header_length             104
 
 Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
+Header extension:
 magic                     0x12345678
 length                    31
 data                      'This is a test header extension'
diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032
index b1ba5c3..24bcb52 100755
--- a/tests/qemu-iotests/032
+++ b/tests/qemu-iotests/032
@@ -27,7 +27,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/032.out b/tests/qemu-iotests/032.out
index ca20de6..ce27d5d 100644
--- a/tests/qemu-iotests/032.out
+++ b/tests/qemu-iotests/032.out
@@ -2,7 +2,7 @@
 
 === Prepare image ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 131072
diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033
index ea3351c..16edcf2 100755
--- a/tests/qemu-iotests/033
+++ b/tests/qemu-iotests/033
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -46,26 +45,54 @@
 size=128M
 _make_test_img $size
 
-echo
-echo "== preparing image =="
-$QEMU_IO -c "write -P 0xa 0x200 0x400" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "write -P 0xa 0x20000 0x600" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "write -z 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
+do_test()
+{
+	local align=$1
+	local iocmd=$2
+	local img=$3
+	{
+		echo "open -o driver=$IMGFMT,file.align=$align blkdebug::$img"
+		echo $iocmd
+	} | $QEMU_IO
+}
 
-echo
-echo "== verifying patterns (1) =="
-$QEMU_IO -c "read -P 0xa 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0x0 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0xa 0x20400 0x200" "$TEST_IMG" | _filter_qemu_io
+for write_zero_cmd in "write -z" "aio_write -z"; do
+for align in 512 4k; do
+	echo
+	echo "== preparing image =="
+	do_test $align "write -P 0xa 0x200 0x400" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "write -P 0xa 0x20000 0x600" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "$write_zero_cmd 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
 
-echo
-echo "== rewriting zeroes =="
-$QEMU_IO -c "write -P 0xb 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "write -z 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
+	echo
+	echo "== verifying patterns (1) =="
+	do_test $align "read -P 0xa 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "read -P 0x0 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "read -P 0xa 0x20400 0x200" "$TEST_IMG" | _filter_qemu_io
 
-echo
-echo "== verifying patterns (2) =="
-$QEMU_IO -c "read -P 0x0 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
+	echo
+	echo "== rewriting zeroes =="
+	do_test $align "write -P 0xb 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "$write_zero_cmd 0x10000 0x10000" "$TEST_IMG" | _filter_qemu_io
+
+	echo
+	echo "== verifying patterns (2) =="
+	do_test $align "read -P 0x0 0x400 0x20000" "$TEST_IMG" | _filter_qemu_io
+
+	echo
+	echo "== rewriting unaligned zeroes =="
+	do_test $align "write -P 0xb 0x0 0x1000" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "$write_zero_cmd 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
+
+	echo
+	echo "== verifying patterns (3) =="
+	do_test $align "read -P 0xb 0x0 0x200" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "read -P 0x0 0x200 0x200" "$TEST_IMG" | _filter_qemu_io
+	do_test $align "read -P 0xb 0x400 0xc00" "$TEST_IMG" | _filter_qemu_io
+
+	echo
+done
+done
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/033.out b/tests/qemu-iotests/033.out
index 2fe74df..95929ef 100644
--- a/tests/qemu-iotests/033.out
+++ b/tests/qemu-iotests/033.out
@@ -1,5 +1,5 @@
 QA output created by 033
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == preparing image ==
 wrote 1024/1024 bytes at offset 512
@@ -26,4 +26,142 @@
 == verifying patterns (2) ==
 read 131072/131072 bytes at offset 1024
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting unaligned zeroes ==
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (3) ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 1024
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+
+== preparing image ==
+wrote 1024/1024 bytes at offset 512
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1536/1536 bytes at offset 131072
+1.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (1) ==
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132096
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting zeroes ==
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (2) ==
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting unaligned zeroes ==
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (3) ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 1024
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+
+== preparing image ==
+wrote 1024/1024 bytes at offset 512
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1536/1536 bytes at offset 131072
+1.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (1) ==
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132096
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting zeroes ==
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (2) ==
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting unaligned zeroes ==
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (3) ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 1024
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+
+== preparing image ==
+wrote 1024/1024 bytes at offset 512
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1536/1536 bytes at offset 131072
+1.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (1) ==
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 132096
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting zeroes ==
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (2) ==
+read 131072/131072 bytes at offset 1024
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting unaligned zeroes ==
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verifying patterns (3) ==
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 1024
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
 *** done
diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034
index 69c7858..1b28bda 100755
--- a/tests/qemu-iotests/034
+++ b/tests/qemu-iotests/034
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Test bdrv_write_zeroes with backing files
+# Test bdrv_pwrite_zeroes with backing files (see also 154)
 #
 # Copyright (C) 2012 Red Hat, Inc.
 #
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,9 +50,13 @@
 echo
 echo "== creating backing file for COW tests =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $size
 $QEMU_IO -c "write -P 0x55 0 1M" "$TEST_IMG" | _filter_qemu_io
-mv "$TEST_IMG" "$TEST_IMG.base"
+
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.base" 6G
 
diff --git a/tests/qemu-iotests/034.out b/tests/qemu-iotests/034.out
index e82dae5..0764ead 100644
--- a/tests/qemu-iotests/034.out
+++ b/tests/qemu-iotests/034.out
@@ -1,10 +1,10 @@
 QA output created by 034
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == zero write with backing file ==
 wrote 196608/196608 bytes at offset 65536
diff --git a/tests/qemu-iotests/035 b/tests/qemu-iotests/035
index ebe9b8c..efc38e4 100755
--- a/tests/qemu-iotests/035
+++ b/tests/qemu-iotests/035
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/035.out b/tests/qemu-iotests/035.out
index cde21d8..dd3bd20 100644
--- a/tests/qemu-iotests/035.out
+++ b/tests/qemu-iotests/035.out
@@ -1,7 +1,7 @@
 QA output created by 035
 
 creating image
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944
 wrote 512/512 bytes at offset XXX
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset XXX
diff --git a/tests/qemu-iotests/036 b/tests/qemu-iotests/036
index 392f1ef..ce638d6 100755
--- a/tests/qemu-iotests/036
+++ b/tests/qemu-iotests/036
@@ -28,7 +28,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -57,6 +56,7 @@
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 63
 
 # Without feature table
+$PYTHON qcow2.py "$TEST_IMG" del-header-ext 0x6803f857
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 _img_info
 
@@ -73,6 +73,7 @@
 $PYTHON qcow2.py "$TEST_IMG" set-feature-bit incompatible 63
 
 # Without feature table
+$PYTHON qcow2.py "$TEST_IMG" del-header-ext 0x6803f857
 _img_info
 
 # With feature table containing bit 63
diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out
index 720bd89..9b009b8 100644
--- a/tests/qemu-iotests/036.out
+++ b/tests/qemu-iotests/036.out
@@ -2,7 +2,7 @@
 
 === Image with unknown incompatible feature bit ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -22,21 +22,21 @@
 refcount_order            4
 header_length             104
 
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Unknown incompatible feature: 8000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Test feature
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Unknown incompatible feature: 8000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature
 
 === Image with multiple incompatible feature bits ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Unknown incompatible feature: e000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Test feature, Unknown incompatible feature: 6000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: Test feature, Unknown incompatible feature: c000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: test1, test2, Unknown incompatible feature: 8000000000000000
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: test1, test2, test3
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': 'image' uses a IMGFMT feature which is not supported by this qemu version: test2, Unknown incompatible feature: a000000000000000
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Unknown incompatible feature: e000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature, Unknown incompatible feature: 6000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): Test feature, Unknown incompatible feature: c000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): test1, test2, Unknown incompatible feature: 8000000000000000
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): test1, test2, test3
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Unsupported IMGFMT feature(s): test2, Unknown incompatible feature: a000000000000000
 === Create image with unknown autoclear feature bit ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 magic                     0x514649fb
 version                   3
 backing_file_offset       0x0
@@ -56,6 +56,11 @@
 refcount_order            4
 header_length             104
 
+Header extension:
+magic                     0x6803f857
+length                    144
+data                      <binary>
+
 
 === Repair image ===
 
diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037
index 9171d8c..c476b82 100755
--- a/tests/qemu-iotests/037
+++ b/tests/qemu-iotests/037
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,6 +50,9 @@
 echo
 echo "== creating backing file for COW tests =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $size
 
 function backing_io()
@@ -71,7 +73,7 @@
 
 backing_io 0 256 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.base" 6G
 
diff --git a/tests/qemu-iotests/037.out b/tests/qemu-iotests/037.out
index 4eb84ed..cd6710c 100644
--- a/tests/qemu-iotests/037.out
+++ b/tests/qemu-iotests/037.out
@@ -1,7 +1,7 @@
 QA output created by 037
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 512
@@ -514,7 +514,7 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 130560
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == COW in a single cluster ==
 wrote 2048/2048 bytes at offset 0
diff --git a/tests/qemu-iotests/038 b/tests/qemu-iotests/038
index cfaf00a..d99a150 100755
--- a/tests/qemu-iotests/038
+++ b/tests/qemu-iotests/038
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -48,6 +47,9 @@
 echo
 echo "== creating backing file for COW tests =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.base"
+
 _make_test_img $size
 
 function backing_io()
@@ -68,7 +70,7 @@
 
 backing_io 0 256 write | $QEMU_IO "$TEST_IMG" | _filter_qemu_io
 
-mv "$TEST_IMG" "$TEST_IMG.base"
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.base" 6G
 
diff --git a/tests/qemu-iotests/038.out b/tests/qemu-iotests/038.out
index a71c3fa..0bdfb19 100644
--- a/tests/qemu-iotests/038.out
+++ b/tests/qemu-iotests/038.out
@@ -1,7 +1,7 @@
 QA output created by 038
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 65536
@@ -514,7 +514,7 @@
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 16711680
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == Some concurrent requests touching the same cluster ==
 wrote 65536/65536 bytes at offset XXX
diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index 84c9167..1f48339 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -28,7 +28,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -47,11 +46,6 @@
 _default_cache_mode "writethrough"
 _supported_cache_modes "writethrough"
 
-_no_dump_exec()
-{
-    (ulimit -c 0; exec "$@")
-}
-
 size=128M
 
 echo
@@ -72,7 +66,9 @@
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-_no_dump_exec $QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" 2>&1 | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
 
 # The dirty bit must be set
 $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
@@ -105,7 +101,9 @@
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-_no_dump_exec $QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" 2>&1 | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
 
 # The dirty bit must be set
 $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
@@ -121,7 +119,9 @@
 IMGOPTS="compat=1.1,lazy_refcounts=off"
 _make_test_img $size
 
-_no_dump_exec $QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" 2>&1 | _filter_qemu_io
+$QEMU_IO -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
 
 # The dirty bit must not be set since lazy_refcounts=off
 $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
@@ -146,6 +146,33 @@
 _check_test_img
 TEST_IMG="$TEST_IMG".base _check_test_img
 
+echo
+echo "== Changing lazy_refcounts setting at runtime =="
+
+IMGOPTS="compat=1.1,lazy_refcounts=off"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=on" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
+IMGOPTS="compat=1.1,lazy_refcounts=on"
+_make_test_img $size
+
+$QEMU_IO -c "reopen -o lazy-refcounts=off" \
+         -c "write -P 0x5a 0 512" \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \
+    | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+_check_test_img
+
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out
index 0adf153..c6e0ac2 100644
--- a/tests/qemu-iotests/039.out
+++ b/tests/qemu-iotests/039.out
@@ -1,17 +1,21 @@
 QA output created by 039
 
 == Checking that image is clean on shutdown ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 incompatible_features     0x0
 No errors were found on the image.
 
 == Creating a dirty image file ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Aborted                 ( ulimit -c 0; exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+fi )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
@@ -43,10 +47,14 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Opening a dirty image read/write should repair it ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Aborted                 ( ulimit -c 0; exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+fi )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 Rebuilding refcount structure
@@ -57,16 +65,20 @@
 incompatible_features     0x0
 
 == Creating an image file with lazy_refcounts=off ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-./039: Aborted                 ( ulimit -c 0; exec "$@" )
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+fi )
 incompatible_features     0x0
 No errors were found on the image.
 
 == Committing to a backing file with lazy_refcounts=on ==
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Image committed.
@@ -74,4 +86,30 @@
 incompatible_features     0x0
 No errors were found on the image.
 No errors were found on the image.
+
+== Changing lazy_refcounts setting at runtime ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+fi )
+incompatible_features     0x1
+ERROR cluster 5 refcount=0 reference=1
+ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
+
+2 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+fi )
+incompatible_features     0x0
+No errors were found on the image.
 *** done
diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
index 2b432ad..5bdaf3d 100755
--- a/tests/qemu-iotests/040
+++ b/tests/qemu-iotests/040
@@ -41,6 +41,7 @@
         while not completed:
             for event in self.vm.get_qmp_events(wait=True):
                 if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp_absent(event, 'data/error')
                     self.assert_qmp(event, 'data/type', 'commit')
                     self.assert_qmp(event, 'data/device', 'drive0')
                     self.assert_qmp(event, 'data/offset', event['data']['len'])
@@ -76,8 +77,8 @@
         iotests.create_image(backing_img, self.image_len)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
-        qemu_io('-c', 'write -P 0xab 0 524288', backing_img)
-        qemu_io('-c', 'write -P 0xef 524288 524288', mid_img)
+        qemu_io('-f', 'raw', '-c', 'write -P 0xab 0 524288', backing_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xef 524288 524288', mid_img)
         self.vm = iotests.VM().add_drive(test_img)
         self.vm.launch()
 
@@ -89,8 +90,8 @@
 
     def test_commit(self):
         self.run_commit_test(mid_img, backing_img)
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
 
     def test_device_not_found(self):
         result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % mid_img)
@@ -116,13 +117,13 @@
 
     def test_top_is_active(self):
         self.run_commit_test(test_img, backing_img, need_ready=True)
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
 
     def test_top_is_default_active(self):
         self.run_default_commit_test()
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
 
     def test_top_and_base_reversed(self):
         self.assert_no_active_block_jobs()
@@ -159,8 +160,8 @@
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.mid_img_abs, self.test_img)
         qemu_img('rebase', '-u', '-b', self.backing_img, self.mid_img_abs)
         qemu_img('rebase', '-u', '-b', self.mid_img, self.test_img)
-        qemu_io('-c', 'write -P 0xab 0 524288', self.backing_img_abs)
-        qemu_io('-c', 'write -P 0xef 524288 524288', self.mid_img_abs)
+        qemu_io('-f', 'raw', '-c', 'write -P 0xab 0 524288', self.backing_img_abs)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xef 524288 524288', self.mid_img_abs)
         self.vm = iotests.VM().add_drive(self.test_img)
         self.vm.launch()
 
@@ -179,8 +180,8 @@
 
     def test_commit(self):
         self.run_commit_test(self.mid_img, self.backing_img)
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', self.backing_img_abs).find("verification failed"))
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', self.backing_img_abs).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', self.backing_img_abs).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', self.backing_img_abs).find("verification failed"))
 
     def test_device_not_found(self):
         result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % self.mid_img)
@@ -206,8 +207,8 @@
 
     def test_top_is_active(self):
         self.run_commit_test(self.test_img, self.backing_img)
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', self.backing_img_abs).find("verification failed"))
-        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', self.backing_img_abs).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xab 0 524288', self.backing_img_abs).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-f', 'raw', '-c', 'read -P 0xef 524288 524288', self.backing_img_abs).find("verification failed"))
 
     def test_top_and_base_reversed(self):
         self.assert_no_active_block_jobs()
@@ -223,8 +224,8 @@
         qemu_img('create', backing_img, str(TestSetSpeed.image_len))
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
-        qemu_io('-c', 'write -P 0x1 0 512', test_img)
-        qemu_io('-c', 'write -P 0xef 524288 524288', mid_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0x1 0 512', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xef 524288 524288', mid_img)
         self.vm = iotests.VM().add_drive(test_img)
         self.vm.launch()
 
@@ -251,5 +252,34 @@
 class TestActiveZeroLengthImage(TestSingleDrive):
     image_len = 0
 
+class TestReopenOverlay(ImageCommitTestCase):
+    image_len = 1024 * 1024
+    img0 = os.path.join(iotests.test_dir, '0.img')
+    img1 = os.path.join(iotests.test_dir, '1.img')
+    img2 = os.path.join(iotests.test_dir, '2.img')
+    img3 = os.path.join(iotests.test_dir, '3.img')
+
+    def setUp(self):
+        iotests.create_image(self.img0, self.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img0, self.img1)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img1, self.img2)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.img2, self.img3)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P 0xab 0 128K', self.img1)
+        self.vm = iotests.VM().add_drive(self.img3)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.img0)
+        os.remove(self.img1)
+        os.remove(self.img2)
+        os.remove(self.img3)
+
+    # This tests what happens when the overlay image of the 'top' node
+    # needs to be reopened in read-write mode in order to update the
+    # backing image string.
+    def test_reopen_overlay(self):
+        self.run_commit_test(self.img1, self.img0)
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out
index 42314e9..4fd1c2d 100644
--- a/tests/qemu-iotests/040.out
+++ b/tests/qemu-iotests/040.out
@@ -1,5 +1,5 @@
-........................
+.........................
 ----------------------------------------------------------------------
-Ran 24 tests
+Ran 25 tests
 
 OK
diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041
index 59a8f73..cbf5e0b 100755
--- a/tests/qemu-iotests/041
+++ b/tests/qemu-iotests/041
@@ -34,44 +34,18 @@
 quorum_repair_img = os.path.join(iotests.test_dir, 'quorum_repair.img')
 quorum_snapshot_file = os.path.join(iotests.test_dir, 'quorum_snapshot.img')
 
-class ImageMirroringTestCase(iotests.QMPTestCase):
-    '''Abstract base class for image mirroring test cases'''
-
-    def wait_ready(self, drive='drive0'):
-        '''Wait until a block job BLOCK_JOB_READY event'''
-        ready = False
-        while not ready:
-            for event in self.vm.get_qmp_events(wait=True):
-                if event['event'] == 'BLOCK_JOB_READY':
-                    self.assert_qmp(event, 'data/type', 'mirror')
-                    self.assert_qmp(event, 'data/device', drive)
-                    ready = True
-
-    def wait_ready_and_cancel(self, drive='drive0'):
-        self.wait_ready(drive=drive)
-        event = self.cancel_and_wait(drive=drive)
-        self.assertEquals(event['event'], 'BLOCK_JOB_COMPLETED')
-        self.assert_qmp(event, 'data/type', 'mirror')
-        self.assert_qmp(event, 'data/offset', event['data']['len'])
-
-    def complete_and_wait(self, drive='drive0', wait_ready=True):
-        '''Complete a block job and wait for it to finish'''
-        if wait_ready:
-            self.wait_ready(drive=drive)
-
-        result = self.vm.qmp('block-job-complete', device=drive)
-        self.assert_qmp(result, 'return', {})
-
-        event = self.wait_until_completed(drive=drive)
-        self.assert_qmp(event, 'data/type', 'mirror')
-
-class TestSingleDrive(ImageMirroringTestCase):
+class TestSingleDrive(iotests.QMPTestCase):
     image_len = 1 * 1024 * 1024 # MB
+    qmp_cmd = 'drive-mirror'
+    qmp_target = target_img
+    not_found_error = 'DeviceNotFound'
 
     def setUp(self):
         iotests.create_image(backing_img, self.image_len)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
         self.vm = iotests.VM().add_drive(test_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
@@ -86,8 +60,8 @@
     def test_complete(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -100,8 +74,8 @@
     def test_cancel(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.cancel_and_wait(force=True)
@@ -112,8 +86,8 @@
     def test_cancel_after_ready(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.wait_ready_and_cancel()
@@ -126,8 +100,8 @@
     def test_pause(self):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         result = self.vm.qmp('block-job-pause', device='drive0')
@@ -153,8 +127,8 @@
         self.assert_no_active_block_jobs()
 
         # A small buffer is rounded up automatically
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             buf_size=4096, target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             buf_size=4096, target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -169,8 +143,8 @@
 
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,size=%d'
                         % (self.image_len, self.image_len), target_img)
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             buf_size=65536, mode='existing', target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             buf_size=65536, mode='existing', target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -185,8 +159,8 @@
 
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
                         % (self.image_len, backing_img), target_img)
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             mode='existing', target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             mode='existing', target=self.qmp_target)
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait()
@@ -197,41 +171,61 @@
                         'target image does not match source after mirroring')
 
     def test_medium_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
-                             target=target_img)
-        self.assert_qmp(result, 'error/class', 'GenericError')
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp(self.qmp_cmd, device='ide1-cd0', sync='full',
+                             target=self.qmp_target)
+        self.assert_qmp(result, 'error/class', self.not_found_error)
 
     def test_image_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
-                             mode='existing', target=target_img)
+        result = self.vm.qmp(self.qmp_cmd, device='drive0', sync='full',
+                             mode='existing', target=self.qmp_target)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
     def test_device_not_found(self):
-        result = self.vm.qmp('drive-mirror', device='nonexistent', sync='full',
-                             target=target_img)
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        result = self.vm.qmp(self.qmp_cmd, device='nonexistent', sync='full',
+                             target=self.qmp_target)
+        self.assert_qmp(result, 'error/class', self.not_found_error)
+
+class TestSingleBlockdev(TestSingleDrive):
+    qmp_cmd = 'blockdev-mirror'
+    qmp_target = 'node1'
+    not_found_error = 'GenericError'
+
+    def setUp(self):
+        TestSingleDrive.setUp(self)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        args = {'options':
+                    {'driver': iotests.imgfmt,
+                     'node-name': self.qmp_target,
+                     'file': { 'filename': target_img, 'driver': 'file' } } }
+        result = self.vm.qmp("blockdev-add", **args)
+        self.assert_qmp(result, 'return', {})
+
+    test_large_cluster = None
+    test_image_not_found = None
+    test_small_buffer2 = None
 
 class TestSingleDriveZeroLength(TestSingleDrive):
     image_len = 0
     test_small_buffer2 = None
     test_large_cluster = None
 
+class TestSingleBlockdevZeroLength(TestSingleBlockdev):
+    image_len = 0
+
 class TestSingleDriveUnalignedLength(TestSingleDrive):
     image_len = 1025 * 1024
     test_small_buffer2 = None
     test_large_cluster = None
 
-class TestMirrorNoBacking(ImageMirroringTestCase):
+class TestSingleBlockdevUnalignedLength(TestSingleBlockdev):
+    image_len = 1025 * 1024
+
+class TestMirrorNoBacking(iotests.QMPTestCase):
     image_len = 2 * 1024 * 1024 # MB
 
-    def complete_and_wait(self, drive='drive0', wait_ready=True):
-        iotests.create_image(target_backing_img, TestMirrorNoBacking.image_len)
-        return ImageMirroringTestCase.complete_and_wait(self, drive, wait_ready)
-
-    def compare_images(self, img1, img2):
-        iotests.create_image(target_backing_img, TestMirrorNoBacking.image_len)
-        return iotests.compare_images(img1, img2)
-
     def setUp(self):
         iotests.create_image(backing_img, TestMirrorNoBacking.image_len)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
@@ -242,7 +236,10 @@
         self.vm.shutdown()
         os.remove(test_img)
         os.remove(backing_img)
-        os.remove(target_backing_img)
+        try:
+            os.remove(target_backing_img)
+        except:
+            pass
         os.remove(target_img)
 
     def test_complete(self):
@@ -257,7 +254,7 @@
         result = self.vm.qmp('query-block')
         self.assert_qmp(result, 'return[0]/inserted/file', target_img)
         self.vm.shutdown()
-        self.assertTrue(self.compare_images(test_img, target_img),
+        self.assertTrue(iotests.compare_images(test_img, target_img),
                         'target image does not match source after mirroring')
 
     def test_cancel(self):
@@ -272,7 +269,7 @@
         result = self.vm.qmp('query-block')
         self.assert_qmp(result, 'return[0]/inserted/file', test_img)
         self.vm.shutdown()
-        self.assertTrue(self.compare_images(test_img, target_img),
+        self.assertTrue(iotests.compare_images(test_img, target_img),
                         'target image does not match source after mirroring')
 
     def test_large_cluster(self):
@@ -283,7 +280,6 @@
                         %(TestMirrorNoBacking.image_len), target_backing_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
                         % (TestMirrorNoBacking.image_len, target_backing_img), target_img)
-        os.remove(target_backing_img)
 
         result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
                              mode='existing', target=target_img)
@@ -293,10 +289,10 @@
         result = self.vm.qmp('query-block')
         self.assert_qmp(result, 'return[0]/inserted/file', target_img)
         self.vm.shutdown()
-        self.assertTrue(self.compare_images(test_img, target_img),
+        self.assertTrue(iotests.compare_images(test_img, target_img),
                         'target image does not match source after mirroring')
 
-class TestMirrorResized(ImageMirroringTestCase):
+class TestMirrorResized(iotests.QMPTestCase):
     backing_len = 1 * 1024 * 1024 # MB
     image_len = 2 * 1024 * 1024 # MB
 
@@ -344,7 +340,7 @@
         self.assertTrue(iotests.compare_images(test_img, target_img),
                         'target image does not match source after mirroring')
 
-class TestReadErrors(ImageMirroringTestCase):
+class TestReadErrors(iotests.QMPTestCase):
     image_len = 2 * 1024 * 1024 # MB
 
     # this should be a multiple of twice the default granularity
@@ -498,7 +494,7 @@
         self.assert_no_active_block_jobs()
         self.vm.shutdown()
 
-class TestWriteErrors(ImageMirroringTestCase):
+class TestWriteErrors(iotests.QMPTestCase):
     image_len = 2 * 1024 * 1024 # MB
 
     # this should be a multiple of twice the default granularity
@@ -624,7 +620,7 @@
         self.assert_no_active_block_jobs()
         self.vm.shutdown()
 
-class TestSetSpeed(ImageMirroringTestCase):
+class TestSetSpeed(iotests.QMPTestCase):
     image_len = 80 * 1024 * 1024 # MB
 
     def setUp(self):
@@ -690,7 +686,7 @@
 
         self.wait_ready_and_cancel()
 
-class TestUnbackedSource(ImageMirroringTestCase):
+class TestUnbackedSource(iotests.QMPTestCase):
     image_len = 2 * 1024 * 1024 # MB
 
     def setUp(self):
@@ -731,7 +727,37 @@
         self.complete_and_wait()
         self.assert_no_active_block_jobs()
 
-class TestRepairQuorum(ImageMirroringTestCase):
+class TestGranularity(iotests.QMPTestCase):
+    image_len = 10 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, test_img,
+                 str(TestGranularity.image_len))
+        qemu_io('-c', 'write 0 %d' % (self.image_len),
+                test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        self.assertTrue(iotests.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+        os.remove(test_img)
+        os.remove(target_img)
+
+    def test_granularity(self):
+        self.assert_no_active_block_jobs()
+        result = self.vm.qmp('drive-mirror', device='drive0',
+                             sync='full', target=target_img,
+                             mode='absolute-paths', granularity=8192)
+        self.assert_qmp(result, 'return', {})
+        event = self.vm.get_qmp_event(wait=60.0)
+        # Failures will manifest as COMPLETED/ERROR.
+        self.assert_qmp(event, 'event', 'BLOCK_JOB_READY')
+        self.complete_and_wait(drive='drive0', wait_ready=False)
+        self.assert_no_active_block_jobs()
+
+class TestRepairQuorum(iotests.QMPTestCase):
     """ This class test quorum file repair using drive-mirror.
         It's mostly a fork of TestSingleDrive """
     image_len = 1 * 1024 * 1024 # MB
@@ -743,6 +769,9 @@
     def setUp(self):
         self.vm = iotests.VM()
 
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
+
         # Add each individual quorum images
         for i in self.IMAGES:
             qemu_img('create', '-f', iotests.imgfmt, i,
@@ -784,8 +813,7 @@
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait(drive="quorum0")
-        result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_repair_img)
+        self.assert_has_block_node("repair0", quorum_repair_img)
         # TODO: a better test requiring some QEMU infrastructure will be added
         #       to check that this file is really driven by quorum
         self.vm.shutdown()
@@ -807,8 +835,7 @@
         self.cancel_and_wait(drive="quorum0", force=True)
         # here we check that the last registered quorum file has not been
         # swapped out and unref
-        result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_img3)
+        self.assert_has_block_node(None, quorum_img3)
         self.vm.shutdown()
 
     def test_cancel_after_ready(self):
@@ -824,10 +851,9 @@
         self.assert_qmp(result, 'return', {})
 
         self.wait_ready_and_cancel(drive="quorum0")
-        result = self.vm.qmp('query-named-block-nodes')
         # here we check that the last registered quorum file has not been
         # swapped out and unref
-        self.assert_qmp(result, 'return[0]/file', quorum_img3)
+        self.assert_has_block_node(None, quorum_img3)
         self.vm.shutdown()
         self.assertTrue(iotests.compare_images(quorum_img2, quorum_repair_img),
                         'target image does not match source after mirroring')
@@ -867,7 +893,11 @@
         if not self.has_quorum():
             return
 
-        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-mirror', device='drive0', # CD-ROM
+                             sync='full',
                              node_name='repair0',
                              replaces='img1',
                              target=quorum_repair_img, format=iotests.imgfmt)
@@ -913,7 +943,7 @@
                              target=quorum_repair_img, format=iotests.imgfmt)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
-    def test_unexistant_replaces(self):
+    def test_nonexistent_replaces(self):
         if not self.has_quorum():
             return
 
@@ -944,8 +974,7 @@
         self.assert_qmp(result, 'return', {})
 
         self.complete_and_wait(drive="quorum0")
-        result = self.vm.qmp('query-named-block-nodes')
-        self.assert_qmp(result, 'return[0]/file', quorum_repair_img)
+        self.assert_has_block_node("repair0", quorum_repair_img)
         # TODO: a better test requiring some QEMU infrastructure will be added
         #       to check that this file is really driven by quorum
         self.vm.shutdown()
diff --git a/tests/qemu-iotests/041.out b/tests/qemu-iotests/041.out
index 24093bc..b67d050 100644
--- a/tests/qemu-iotests/041.out
+++ b/tests/qemu-iotests/041.out
@@ -1,5 +1,5 @@
-......................................................
+............................................................................
 ----------------------------------------------------------------------
-Ran 54 tests
+Ran 76 tests
 
 OK
diff --git a/tests/qemu-iotests/042 b/tests/qemu-iotests/042
index 94ce3a9..351b283 100755
--- a/tests/qemu-iotests/042
+++ b/tests/qemu-iotests/042
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/042.out b/tests/qemu-iotests/042.out
index dc80f4b..8ab1520 100644
--- a/tests/qemu-iotests/042.out
+++ b/tests/qemu-iotests/042.out
@@ -1,7 +1,7 @@
 QA output created by 042
 
 == Creating zero size image ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
 No errors were found on the image.
 
 == Converting the image ==
diff --git a/tests/qemu-iotests/043 b/tests/qemu-iotests/043
index b316b97..1c6c22d 100755
--- a/tests/qemu-iotests/043
+++ b/tests/qemu-iotests/043
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/043.out b/tests/qemu-iotests/043.out
index ad23337..b37d2a3 100644
--- a/tests/qemu-iotests/043.out
+++ b/tests/qemu-iotests/043.out
@@ -1,23 +1,23 @@
 QA output created by 043
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == backing file references self ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 
 == parent references self ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.3.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.1.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.2.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.3.base
 
 == ancestor references another ancestor ==
 qemu-img: Backing file 'TEST_DIR/t.IMGFMT.2.base' creates an infinite loop.
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.1.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.2.base
 
 == finite chain of length 3 (human) ==
 image: TEST_DIR/t.IMGFMT
@@ -40,26 +40,28 @@
 == finite chain of length 3 (json) ==
 [
     {
-        "virtual-size": 134217728, 
-        "filename": "TEST_DIR/t.IMGFMT", 
-        "cluster-size": 65536, 
-        "format": "IMGFMT", 
-        "backing-filename": "TEST_DIR/t.IMGFMT.2.base", 
+        "virtual-size": 134217728,
+        "filename": "TEST_DIR/t.IMGFMT",
+        "cluster-size": 65536,
+        "format": "IMGFMT",
+        "full-backing-filename": "TEST_DIR/t.IMGFMT.2.base",
+        "backing-filename": "TEST_DIR/t.IMGFMT.2.base",
         "dirty-flag": false
-    }, 
+    },
     {
-        "virtual-size": 134217728, 
-        "filename": "TEST_DIR/t.IMGFMT.2.base", 
-        "cluster-size": 65536, 
-        "format": "IMGFMT", 
-        "backing-filename": "TEST_DIR/t.IMGFMT.1.base", 
+        "virtual-size": 134217728,
+        "filename": "TEST_DIR/t.IMGFMT.2.base",
+        "cluster-size": 65536,
+        "format": "IMGFMT",
+        "full-backing-filename": "TEST_DIR/t.IMGFMT.1.base",
+        "backing-filename": "TEST_DIR/t.IMGFMT.1.base",
         "dirty-flag": false
-    }, 
+    },
     {
-        "virtual-size": 134217728, 
-        "filename": "TEST_DIR/t.IMGFMT.1.base", 
-        "cluster-size": 65536, 
-        "format": "IMGFMT", 
+        "virtual-size": 134217728,
+        "filename": "TEST_DIR/t.IMGFMT.1.base",
+        "cluster-size": 65536,
+        "format": "IMGFMT",
         "dirty-flag": false
     }
 ]
diff --git a/tests/qemu-iotests/046 b/tests/qemu-iotests/046
index e0be46c..e528b67 100755
--- a/tests/qemu-iotests/046
+++ b/tests/qemu-iotests/046
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/046.out b/tests/qemu-iotests/046.out
index 65d584b..ca2c740 100644
--- a/tests/qemu-iotests/046.out
+++ b/tests/qemu-iotests/046.out
@@ -1,7 +1,7 @@
 QA output created by 046
 
 == creating backing file for COW tests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 65536
@@ -66,7 +66,7 @@
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 2031616
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=6442450944 backing_file=TEST_DIR/t.IMGFMT.base
 
 == Some concurrent requests touching the same cluster ==
 blkdebug: Suspended request 'A'
diff --git a/tests/qemu-iotests/047 b/tests/qemu-iotests/047
index c35cd09..1b8f3d4 100755
--- a/tests/qemu-iotests/047
+++ b/tests/qemu-iotests/047
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/047.out b/tests/qemu-iotests/047.out
index 959f2af..856a2bc 100644
--- a/tests/qemu-iotests/047.out
+++ b/tests/qemu-iotests/047.out
@@ -1,5 +1,5 @@
 QA output created by 047
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 327680/327680 bytes at offset 0
 320 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 327680
diff --git a/tests/qemu-iotests/048 b/tests/qemu-iotests/048
index 65da46d..203c04f 100755
--- a/tests/qemu-iotests/048
+++ b/tests/qemu-iotests/048
@@ -31,13 +31,13 @@
 {
     echo "Cleanup"
     _cleanup_test_img
-    rm "${TEST_IMG2}"
+    rm "${TEST_IMG_FILE2}"
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
 
 _compare()
 {
-    $QEMU_IMG compare "$@" "$TEST_IMG" "${TEST_IMG2}"
+    $QEMU_IMG compare $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" "${TEST_IMG2}"
     echo $?
 }
 
@@ -46,25 +46,37 @@
 . ./common.filter
 . ./common.pattern
 
-_supported_fmt raw qcow qcow2 qed
+_supported_fmt raw qcow qcow2 qed luks
 _supported_proto file
 _supported_os Linux
 
+# Remove once all tests are fixed to use TEST_IMG_FILE
+# correctly and common.rc sets it unconditionally
+test -z "$TEST_IMG_FILE" && TEST_IMG_FILE=$TEST_IMG
+
 # Setup test basic parameters
 TEST_IMG2=$TEST_IMG.2
+TEST_IMG_FILE2=$TEST_IMG_FILE.2
 CLUSTER_SIZE=4096
-size=1024M
+size=128M
 
 _make_test_img $size
 io_pattern write 524288 $CLUSTER_SIZE $CLUSTER_SIZE 4 45
 
 # Compare identical images
-cp "$TEST_IMG" "${TEST_IMG2}"
+cp "$TEST_IMG_FILE" "${TEST_IMG_FILE2}"
 _compare
 _compare -q
 
 # Compare images with different size
-$QEMU_IMG resize "$TEST_IMG" +512M
+if [ "$IMGOPTSSYNTAX" = "true" ]; then
+    $QEMU_IMG resize $QEMU_IMG_EXTRA_ARGS "$TEST_IMG" +32M
+else
+    $QEMU_IMG resize -f $IMGFMT "$TEST_IMG" +32M
+fi
+# Ensure extended space is zero-initialized
+$QEMU_IO "$TEST_IMG" -c "write -z $size 32M" | _filter_qemu_io
+
 _compare
 _compare -s
 
@@ -77,7 +89,7 @@
 # Test unaligned case of mismatch offsets in allocated clusters
 _make_test_img $size
 io_pattern write 0 512 0 1 100
-cp "$TEST_IMG" "$TEST_IMG2"
+cp "$TEST_IMG_FILE" "$TEST_IMG_FILE2"
 io_pattern write 512 512 0 1 101
 _compare
 
diff --git a/tests/qemu-iotests/048.out b/tests/qemu-iotests/048.out
index c0f380d..0bcf663 100644
--- a/tests/qemu-iotests/048.out
+++ b/tests/qemu-iotests/048.out
@@ -1,5 +1,5 @@
 QA output created by 048
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 === IO: pattern 45
 wrote 4096/4096 bytes at offset 524288
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -13,6 +13,8 @@
 0
 0
 Image resized.
+wrote 33554432/33554432 bytes at offset 134217728
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Warning: Image size mismatch!
 Images are identical.
 0
@@ -28,7 +30,7 @@
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 Content mismatch at offset 0!
 1
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 === IO: pattern 100
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
diff --git a/tests/qemu-iotests/049 b/tests/qemu-iotests/049
index 93aa0ea..fff0760 100755
--- a/tests/qemu-iotests/049
+++ b/tests/qemu-iotests/049
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/049.out b/tests/qemu-iotests/049.out
index 09ca0ae..4673b67 100644
--- a/tests/qemu-iotests/049.out
+++ b/tests/qemu-iotests/049.out
@@ -4,90 +4,90 @@
 == 1. Traditional size parameter ==
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1024
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1024b
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1k
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1K
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1048576 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1048576 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1T
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1099511627776 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1099511627776 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1024.0
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1024.0b
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1.5k
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1.5K
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1.5M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1572864 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1572864 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1.5G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1610612736 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1610612736 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 1.5T
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1649267441664 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1649267441664 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == 2. Specifying size via -o ==
 
 qemu-img create -f qcow2 -o size=1024 TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1024b TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1k TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1K TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1M TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1048576 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1048576 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1G TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1073741824 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1T TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1099511627776 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1099511627776 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1024.0 TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1024.0b TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1.5k TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1.5K TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1536 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1.5M TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1572864 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1572864 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1.5G TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1610612736 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1610612736 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o size=1.5T TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1649267441664 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1649267441664 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == 3. Invalid sizes ==
 
@@ -95,114 +95,113 @@
 qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1024 TEST_DIR/t.qcow2
-qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+qemu-img: Parameter 'size' expects a non-negative number below 2^64
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1k
 qemu-img: Image size must be less than 8 EiB!
 
 qemu-img create -f qcow2 -o size=-1k TEST_DIR/t.qcow2
-qemu-img: qcow2 doesn't support shrinking images yet
-qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+qemu-img: Parameter 'size' expects a non-negative number below 2^64
+qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- 1kilobyte
-qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for 
+qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for
 qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes.
 
 qemu-img create -f qcow2 -o size=1kilobyte TEST_DIR/t.qcow2
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- foobar
-qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for 
+qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for
 qemu-img: kilobytes, megabytes, gigabytes, terabytes, petabytes and exabytes.
 
 qemu-img create -f qcow2 -o size=foobar TEST_DIR/t.qcow2
 qemu-img: Parameter 'size' expects a size
+You may use k, M, G or T suffixes for kilobytes, megabytes, gigabytes and terabytes.
 qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2'
 
 == Check correct interpretation of suffixes for cluster size ==
 
 qemu-img create -f qcow2 -o cluster_size=1024 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=1024b TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=1k TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=1K TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=1M TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1048576 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1048576 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=1024.0 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=1024.0b TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=1024 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=0.5k TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=512 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=512 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=0.5K TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=512 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=512 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o cluster_size=0.5M TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=524288 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=524288 lazy_refcounts=off refcount_bits=16
 
 == Check compat level option ==
 
 qemu-img create -f qcow2 -o compat=0.10 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=1.1 TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.42 TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: '0.42'
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.42' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.42 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=foobar TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Invalid compatibility level: 'foobar'
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='foobar' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=foobar encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == Check preallocation option ==
 
 qemu-img create -f qcow2 -o preallocation=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='off' lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=off lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o preallocation=metadata TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=metadata lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o preallocation=1234 TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: invalid parameter value: 1234
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation='1234' lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 preallocation=1234 lazy_refcounts=off refcount_bits=16
 
 == Check encryption option ==
 
 qemu-img create -f qcow2 -o encryption=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o encryption=on TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=on cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 encryption=on cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 == Check lazy_refcounts option (only with v3) ==
 
 qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=1.1,lazy_refcounts=on TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='1.1' encryption=off cluster_size=65536 lazy_refcounts=on 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=1.1 encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=off TEST_DIR/t.qcow2 64M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 qemu-img create -f qcow2 -o compat=0.10,lazy_refcounts=on TEST_DIR/t.qcow2 64M
 qemu-img: TEST_DIR/t.qcow2: Lazy refcounts only supported with compatibility level 1.1 and above (use compat=1.1 or greater)
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat='0.10' encryption=off cluster_size=65536 lazy_refcounts=on 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=67108864 compat=0.10 encryption=off cluster_size=65536 lazy_refcounts=on refcount_bits=16
 
 *** done
diff --git a/tests/qemu-iotests/050 b/tests/qemu-iotests/050
index 07802bc..03b4a5d 100755
--- a/tests/qemu-iotests/050
+++ b/tests/qemu-iotests/050
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,14 +50,19 @@
 echo
 echo "== Creating images =="
 
+TEST_IMG_SAVE="$TEST_IMG"
+TEST_IMG="$TEST_IMG.old"
+
 size=10M
 _make_test_img $size
 $QEMU_IO -c "write -P 0x40 0 1048576" "$TEST_IMG" | _filter_qemu_io
-mv "$TEST_IMG" "$TEST_IMG.old"
+
+TEST_IMG="$TEST_IMG_SAVE.new"
 
 _make_test_img $size
 $QEMU_IO -c "write -P 0x5a 0 1048576" "$TEST_IMG" | _filter_qemu_io
-mv "$TEST_IMG" "$TEST_IMG.new"
+
+TEST_IMG="$TEST_IMG_SAVE"
 
 _make_test_img -b "$TEST_IMG.old" $size
 $QEMU_IO -c "write -z 0 1048576" "$TEST_IMG" | _filter_qemu_io
diff --git a/tests/qemu-iotests/050.out b/tests/qemu-iotests/050.out
index 3f5f7e1..3602d58 100644
--- a/tests/qemu-iotests/050.out
+++ b/tests/qemu-iotests/050.out
@@ -1,13 +1,13 @@
 QA output created by 050
 
 == Creating images ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 
+Formatting 'TEST_DIR/t.IMGFMT.old', fmt=IMGFMT size=10485760
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 
+Formatting 'TEST_DIR/t.IMGFMT.new', fmt=IMGFMT size=10485760
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 backing_file='TEST_DIR/t.IMGFMT.old' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=10485760 backing_file=TEST_DIR/t.IMGFMT.old
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/tests/qemu-iotests/051 b/tests/qemu-iotests/051
index 11c858f..630cb7a 100755
--- a/tests/qemu-iotests/051
+++ b/tests/qemu-iotests/051
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -41,6 +40,9 @@
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# A compat=0.10 image is created in this test which does not support anything
+# other than refcount_bits=16
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 function do_run_qemu()
 {
@@ -58,10 +60,11 @@
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_generated_node_ids
 }
 
 size=128M
+device_id="drive0"
 
 _make_test_img $size
 cp "$TEST_IMG" "$TEST_IMG.orig"
@@ -72,10 +75,10 @@
 echo === Unknown option ===
 echo
 
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=on
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=1234
-run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=foo
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=,if=none,id=$device_id
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=on,if=none,id=$device_id
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=1234,if=none,id=$device_id
+run_qemu -drive file="$TEST_IMG",format=qcow2,unknown_opt=foo,if=none,id=$device_id
 
 echo
 echo === Unknown protocol option ===
@@ -93,12 +96,20 @@
 run_qemu -drive file="$TEST_IMG",format=foo
 run_qemu -drive file="$TEST_IMG",driver=foo
 run_qemu -drive file="$TEST_IMG",driver=raw,format=qcow2
+run_qemu -drive file="$TEST_IMG",driver=qcow2,format=qcow2
+
+echo
+echo === Device without drive ===
+echo
+
+run_qemu -device virtio-scsi-pci -device scsi-hd
 
 echo
 echo === Overriding backing file ===
 echo
 
-echo "info block" | run_qemu -drive file="$TEST_IMG",driver=qcow2,backing.file.filename="$TEST_IMG.orig" -nodefaults
+echo "info block" | run_qemu -drive file="$TEST_IMG",driver=qcow2,backing.file.filename="$TEST_IMG.orig",if=none,id=$device_id -nodefaults\
+                  | _filter_generated_node_ids
 
 # Drivers that don't support backing files
 run_qemu -drive file="$TEST_IMG",driver=raw,backing.file.filename="$TEST_IMG.orig"
@@ -129,41 +140,63 @@
 echo === No medium ===
 echo
 
-run_qemu -drive if=floppy
-run_qemu -drive if=ide,media=cdrom
-run_qemu -drive if=scsi,media=cdrom
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive if=floppy
+        run_qemu -drive if=ide,media=cdrom
+        run_qemu -drive if=scsi,media=cdrom
+        run_qemu -drive if=ide
+        run_qemu -drive if=scsi
+        ;;
+     *)
+        ;;
+esac
 
-run_qemu -drive if=ide
 run_qemu -drive if=virtio
-run_qemu -drive if=scsi
 
-run_qemu -drive if=none,id=disk -device ide-cd,drive=disk
-run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
-
-run_qemu -drive if=none,id=disk -device ide-drive,drive=disk
-run_qemu -drive if=none,id=disk -device ide-hd,drive=disk
-run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
-run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive if=none,id=disk -device ide-cd,drive=disk
+        run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
+        run_qemu -drive if=none,id=disk -device ide-drive,drive=disk
+        run_qemu -drive if=none,id=disk -device ide-hd,drive=disk
+        run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
+        run_qemu -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
+        ;;
+     *)
+        ;;
+esac
 
 echo
 echo === Read-only ===
 echo
 
-run_qemu -drive file="$TEST_IMG",if=floppy,readonly=on
-run_qemu -drive file="$TEST_IMG",if=ide,media=cdrom,readonly=on
-run_qemu -drive file="$TEST_IMG",if=scsi,media=cdrom,readonly=on
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive file="$TEST_IMG",if=floppy,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=ide,media=cdrom,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=scsi,media=cdrom,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=ide,readonly=on
+        run_qemu -drive file="$TEST_IMG",if=scsi,readonly=on
+        ;;
+     *)
+        ;;
+esac
 
-run_qemu -drive file="$TEST_IMG",if=ide,readonly=on
 run_qemu -drive file="$TEST_IMG",if=virtio,readonly=on
-run_qemu -drive file="$TEST_IMG",if=scsi,readonly=on
 
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-cd,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
-
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-drive,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-hd,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
-run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+case "$QEMU_DEFAULT_MACHINE" in
+    pc)
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-cd,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-drive,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device ide-hd,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
+        run_qemu -drive file="$TEST_IMG",if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+        ;;
+     *)
+        ;;
+esac
 
 echo
 echo === Cache modes ===
@@ -172,19 +205,26 @@
 # Cannot use the test image because cache=none might not work on the host FS
 # Use cdrom so that we won't get errors about missing media
 
-run_qemu -drive media=cdrom,cache=none
-run_qemu -drive media=cdrom,cache=directsync
-run_qemu -drive media=cdrom,cache=writeback
-run_qemu -drive media=cdrom,cache=writethrough
-run_qemu -drive media=cdrom,cache=unsafe
-run_qemu -drive media=cdrom,cache=invalid_value
+run_qemu -drive driver=null-co,cache=none
+run_qemu -drive driver=null-co,cache=directsync
+run_qemu -drive driver=null-co,cache=writeback
+run_qemu -drive driver=null-co,cache=writethrough
+run_qemu -drive driver=null-co,cache=unsafe
+run_qemu -drive driver=null-co,cache=invalid_value
+
+# Can't test direct=on here because O_DIRECT might not be supported on this FS
+# Test 142 checks the direct=on cases
+
+for cache in writeback writethrough unsafe invalid_value; do
+    echo -e "info block\ninfo block file\ninfo block backing\ninfo block backing-file" | \
+    run_qemu -drive file="$TEST_IMG",cache=$cache,backing.file.filename="$TEST_IMG.base",backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=$device_id -nodefaults
+done
 
 echo
 echo === Specifying the protocol layer ===
 echo
 
 run_qemu -drive file="$TEST_IMG",file.driver=file
-run_qemu -drive file="$TEST_IMG",file.driver=qcow2
 
 echo
 echo === Leaving out required options ===
@@ -222,6 +262,24 @@
 run_qemu -drive file="$TEST_IMG",readonly=on,read-only=off
 
 echo
+echo === Catching negative/large throttling values ===
+echo
+
+run_qemu -drive file="$TEST_IMG",iops=-1
+run_qemu -drive file="$TEST_IMG",bps=-2
+run_qemu -drive file="$TEST_IMG",bps_rd=-3
+run_qemu -drive file="$TEST_IMG",bps_rd_max=-3
+run_qemu -drive file="$TEST_IMG",throttling.iops-total=-4
+run_qemu -drive file="$TEST_IMG",throttling.bps-total=-5
+# These are accepted
+run_qemu -drive file="$TEST_IMG",bps=0
+run_qemu -drive file="$TEST_IMG",bps=1
+run_qemu -drive file="$TEST_IMG",bps=1000000000000000
+# While these are not
+run_qemu -drive file="$TEST_IMG",bps=1000000000000001
+run_qemu -drive file="$TEST_IMG",bps=9999999999999999
+
+echo
 echo === Parsing protocol from file name ===
 echo
 
@@ -243,26 +301,30 @@
 
 $QEMU_IO -c "write -P 0x11 0 4k" "$TEST_IMG" | _filter_qemu_io
 
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG" -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG",snapshot=on | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2,snapshot=on | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2 -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="file:$TEST_IMG" -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="file:$TEST_IMG",snapshot=on | _filter_qemu_io
+
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",if=none,id=$device_id -snapshot | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",snapshot=on,if=none,id=$device_id | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2,snapshot=on,if=none,id=$device_id\
+                                                 | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file.filename="$TEST_IMG",driver=qcow2,if=none,id=$device_id -snapshot\
+                                                 | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="file:$TEST_IMG",if=none,id=$device_id -snapshot | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="file:$TEST_IMG",snapshot=on,if=none,id=$device_id | _filter_qemu_io
 
 # Opening a read-only file r/w with snapshot=on
 chmod u-w "$TEST_IMG"
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG" -snapshot | _filter_qemu_io
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG",snapshot=on | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",if=none,id=$device_id -snapshot | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",snapshot=on,if=none,id=$device_id | _filter_qemu_io
 chmod u+w "$TEST_IMG"
 
 $QEMU_IO -c "read -P 0x11 0 4k" "$TEST_IMG" | _filter_qemu_io
 
-echo 'qemu-io ide0-hd0 "write -P 0x22 0 4k"' | run_qemu -drive file="$TEST_IMG",snapshot=off | _filter_qemu_io
+echo "qemu-io $device_id \"write -P 0x22 0 4k\"" | run_qemu -drive file="$TEST_IMG",snapshot=off,if=none,id=$device_id | _filter_qemu_io
 
 $QEMU_IO -c "read -P 0x22 0 4k" "$TEST_IMG" | _filter_qemu_io
 
-echo -e 'qemu-io ide0-hd0 "write -P 0x33 0 4k"\ncommit ide0-hd0' | run_qemu -drive file="$TEST_IMG",snapshot=on | _filter_qemu_io
+echo -e "qemu-io $device_id \"write -P 0x33 0 4k\"\ncommit $device_id" | run_qemu -drive file="$TEST_IMG",snapshot=on,if=none,id=$device_id\
+                                                                       | _filter_qemu_io
 
 $QEMU_IO -c "read -P 0x33 0 4k" "$TEST_IMG" | _filter_qemu_io
 
diff --git a/tests/qemu-iotests/051.out b/tests/qemu-iotests/051.out
index 2c7e808..408d613 100644
--- a/tests/qemu-iotests/051.out
+++ b/tests/qemu-iotests/051.out
@@ -1,66 +1,78 @@
 QA output created by 051
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 
 === Unknown option ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device 'ide0-hd0' doesn't support the option 'unknown_opt'
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
 
 
 === Unknown protocol option ===
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=: could not open disk image TEST_DIR/t.qcow2: Block protocol 'file' doesn't support the option 'unknown_opt'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=: Block protocol 'file' doesn't support the option 'unknown_opt'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on: could not open disk image TEST_DIR/t.qcow2: Block protocol 'file' doesn't support the option 'unknown_opt'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on: Block protocol 'file' doesn't support the option 'unknown_opt'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234: could not open disk image TEST_DIR/t.qcow2: Block protocol 'file' doesn't support the option 'unknown_opt'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234: Block protocol 'file' doesn't support the option 'unknown_opt'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo: could not open disk image TEST_DIR/t.qcow2: Block protocol 'file' doesn't support the option 'unknown_opt'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo: Block protocol 'file' doesn't support the option 'unknown_opt'
 
 
 === Invalid format ===
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=foo: 'foo' invalid format
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=foo: Unknown driver 'foo'
 
 Testing: -drive file=TEST_DIR/t.qcow2,driver=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=foo: could not open disk image TEST_DIR/t.qcow2: Unknown driver 'foo'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=foo: Unknown driver 'foo'
 
 Testing: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2: could not open disk image TEST_DIR/t.qcow2: Driver specified twice
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2: Cannot specify both 'driver' and 'format'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,format=qcow2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=qcow2,format=qcow2: Cannot specify both 'driver' and 'format'
+
+
+=== Device without drive ===
+
+Testing: -device virtio-scsi-pci -device scsi-hd
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-hd: drive property not set
 
 
 === Overriding backing file ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig -nodefaults
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig,if=none,id=drive0 -nodefaults
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
-ide0-hd0: TEST_DIR/t.qcow2 (qcow2)
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
     Backing file:     TEST_DIR/t.qcow2.orig (chain depth: 1)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
 Testing: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig: could not open disk image TEST_DIR/t.qcow2: Driver doesn't support backing files
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
 
 Testing: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig: could not open disk image TEST_DIR/t.qcow2: Driver doesn't support backing files
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
 
 Testing: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig: could not open disk image TEST_DIR/t.qcow2: Driver doesn't support backing files
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
 
 
 === Enable and disable lazy refcounting on the command line, plus some invalid values ===
@@ -74,20 +86,20 @@
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: could not open disk image TEST_DIR/t.qcow2: Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: Parameter 'lazy-refcounts' expects 'on' or 'off'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: could not open disk image TEST_DIR/t.qcow2: Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: Parameter 'lazy-refcounts' expects 'on' or 'off'
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: could not open disk image TEST_DIR/t.qcow2: Parameter 'lazy-refcounts' expects 'on' or 'off'
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: Parameter 'lazy-refcounts' expects 'on' or 'off'
 
 
 === With version 2 images enabling lazy refcounts must fail ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: could not open disk image TEST_DIR/t.qcow2: Lazy refcounts require a qcow2 image with at least qemu 1.1 compatibility level
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: Lazy refcounts require a qcow2 image with at least qemu 1.1 compatibility level
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=off
 QEMU X.Y.Z monitor - type 'help' for more information
@@ -96,148 +108,105 @@
 
 === No medium ===
 
-Testing: -drive if=floppy
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive if=ide,media=cdrom
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive if=scsi,media=cdrom
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive if=ide
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: Device needs media, but drive is empty
-QEMU_PROG: Device initialization failed.
-QEMU_PROG: Initialization of device ide-hd failed
-
 Testing: -drive if=virtio
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) QEMU_PROG: -drive if=virtio: Device needs media, but drive is empty
-QEMU_PROG: -drive if=virtio: Device initialization failed.
-QEMU_PROG: -drive if=virtio: Device 'virtio-blk-pci' could not be initialized
-
-Testing: -drive if=scsi
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -drive if=scsi: Device needs media, but drive is empty
-QEMU_PROG: Device initialization failed.
-QEMU_PROG: Initialization of device lsi53c895a failed
-
-Testing: -drive if=none,id=disk -device ide-cd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive if=none,id=disk -device ide-drive,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
-QEMU_PROG: -device ide-drive,drive=disk: Device 'ide-drive' could not be initialized
-
-Testing: -drive if=none,id=disk -device ide-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-hd,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
-QEMU_PROG: -device ide-hd,drive=disk: Device 'ide-hd' could not be initialized
-
-Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device scsi-disk,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device scsi-disk,drive=disk: Device 'scsi-disk' could not be initialized
-
-Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device scsi-hd,drive=disk: Device needs media, but drive is empty
-QEMU_PROG: -device scsi-hd,drive=disk: Device 'scsi-hd' could not be initialized
 
 
 === Read-only ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=floppy,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=ide,media=cdrom,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,media=cdrom,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=ide,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: Can't use a read-only drive
-QEMU_PROG: Device initialization failed.
-QEMU_PROG: Initialization of device ide-hd failed
-
 Testing: -drive file=TEST_DIR/t.qcow2,if=virtio,readonly=on
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,readonly=on
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-cd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-drive,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-drive,drive=disk: Can't use a read-only drive
-QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
-QEMU_PROG: -device ide-drive,drive=disk: Device 'ide-drive' could not be initialized
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) QEMU_PROG: -device ide-hd,drive=disk: Can't use a read-only drive
-QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
-QEMU_PROG: -device ide-hd,drive=disk: Device 'ide-hd' could not be initialized
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
-Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
-QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
-
 
 === Cache modes ===
 
-Testing: -drive media=cdrom,cache=none
+Testing: -drive driver=null-co,cache=none
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive media=cdrom,cache=directsync
+Testing: -drive driver=null-co,cache=directsync
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive media=cdrom,cache=writeback
+Testing: -drive driver=null-co,cache=writeback
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive media=cdrom,cache=writethrough
+Testing: -drive driver=null-co,cache=writethrough
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive media=cdrom,cache=unsafe
+Testing: -drive driver=null-co,cache=unsafe
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive media=cdrom,cache=invalid_value
-QEMU_PROG: -drive media=cdrom,cache=invalid_value: invalid cache option
+Testing: -drive driver=null-co,cache=invalid_value
+QEMU_PROG: -drive driver=null-co,cache=invalid_value: invalid cache option
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writeback,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block f[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block file[K
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-f[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-file[K
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writethrough,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writethrough
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block f[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block file[K
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-f[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-file[K
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=unsafe,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback, ignore flushes
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block f[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block file[K
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-f[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-file[K
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0: invalid cache option
 
 
 === Specifying the protocol layer ===
@@ -246,32 +215,29 @@
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=TEST_DIR/t.qcow2,file.driver=qcow2
-QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.driver=qcow2: could not open disk image TEST_DIR/t.qcow2: Block format 'qcow2' used by device '' doesn't support the option 'filename'
-
 
 === Leaving out required options ===
 
 Testing: -drive driver=file
-QEMU_PROG: -drive driver=file: could not open disk image ide0-hd0: The 'file' block driver requires a file name
+QEMU_PROG: -drive driver=file: The 'file' block driver requires a file name
 
 Testing: -drive driver=nbd
-QEMU_PROG: -drive driver=nbd: could not open disk image ide0-hd0: one of path and host must be specified.
+QEMU_PROG: -drive driver=nbd: one of path and host must be specified.
 
 Testing: -drive driver=raw
-QEMU_PROG: -drive driver=raw: could not open disk image ide0-hd0: Can't use 'raw' as a block driver for the protocol level
+QEMU_PROG: -drive driver=raw: Can't use 'raw' as a block driver for the protocol level
 
 Testing: -drive file.driver=file
-QEMU_PROG: -drive file.driver=file: could not open disk image ide0-hd0: The 'file' block driver requires a file name
+QEMU_PROG: -drive file.driver=file: The 'file' block driver requires a file name
 
 Testing: -drive file.driver=nbd
-QEMU_PROG: -drive file.driver=nbd: could not open disk image ide0-hd0: one of path and host must be specified.
+QEMU_PROG: -drive file.driver=nbd: one of path and host must be specified.
 
 Testing: -drive file.driver=raw
-QEMU_PROG: -drive file.driver=raw: could not open disk image ide0-hd0: Can't use 'raw' as a block driver for the protocol level
+QEMU_PROG: -drive file.driver=raw: Can't use 'raw' as a block driver for the protocol level
 
 Testing: -drive foo=bar
-QEMU_PROG: -drive foo=bar: could not open disk image ide0-hd0: Must specify either driver or file
+QEMU_PROG: -drive foo=bar: Must specify either driver or file
 
 
 === Specifying both an option and its legacy alias ===
@@ -319,16 +285,55 @@
 QEMU_PROG: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off: 'read-only' and its alias 'readonly' can't be used at the same time
 
 
+=== Catching negative/large throttling values ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=-1
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=-1: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=-2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=-2: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000000
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999: bps/iops/max values must be within [0, 1000000000000000]
+
+
 === Parsing protocol from file name ===
 
 Testing: -hda foo:bar
-QEMU_PROG: -hda foo:bar: could not open disk image foo:bar: Unknown protocol
+QEMU_PROG: -hda foo:bar: Unknown protocol 'foo'
 
 Testing: -drive file=foo:bar
-QEMU_PROG: -drive file=foo:bar: could not open disk image foo:bar: Unknown protocol
+QEMU_PROG: -drive file=foo:bar: Unknown protocol 'foo'
 
 Testing: -drive file.filename=foo:bar
-QEMU_PROG: -drive file.filename=foo:bar: could not open disk image ide0-hd0: Could not open 'foo:bar': No such file or directory
+QEMU_PROG: -drive file.filename=foo:bar: Could not open 'foo:bar': No such file or directory
 
 Testing: -hda file:TEST_DIR/t.qcow2
 QEMU X.Y.Z monitor - type 'help' for more information
@@ -339,86 +344,86 @@
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
 Testing: -drive file.filename=file:TEST_DIR/t.qcow2
-QEMU_PROG: -drive file.filename=file:TEST_DIR/t.qcow2: could not open disk image ide0-hd0: Could not open 'file:TEST_DIR/t.qcow2': No such file or directory
+QEMU_PROG: -drive file.filename=file:TEST_DIR/t.qcow2: Could not open 'file:TEST_DIR/t.qcow2': No such file or directory
 
 
 === Snapshot mode ===
 
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Testing: -drive file=TEST_DIR/t.qcow2 -snapshot
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,snapshot=on
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2 -snapshot
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=file:TEST_DIR/t.qcow2 -snapshot
+Testing: -drive file=file:TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=file:TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=file:TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=TEST_DIR/t.qcow2 -snapshot
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
 read 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=off
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=off,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x22 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
 read 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
 QEMU X.Y.Z monitor - type 'help' for more information
-(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io i[K[D[D[D[D[D[D[D[D[Dqemu-io id[K[D[D[D[D[D[D[D[D[D[Dqemu-io ide[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x3[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io ide0-hd0 "write -P 0x33 0 4k"[K
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x3[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 4k"[K
 wrote 4096/4096 bytes at offset 0
 4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-(qemu) c[K[Dco[K[D[Dcom[K[D[D[Dcomm[K[D[D[D[Dcommi[K[D[D[D[D[Dcommit[K[D[D[D[D[D[Dcommit [K[D[D[D[D[D[D[Dcommit i[K[D[D[D[D[D[D[D[Dcommit id[K[D[D[D[D[D[D[D[D[Dcommit ide[K[D[D[D[D[D[D[D[D[D[Dcommit ide0[K[D[D[D[D[D[D[D[D[D[D[Dcommit ide0-[K[D[D[D[D[D[D[D[D[D[D[D[Dcommit ide0-h[K[D[D[D[D[D[D[D[D[D[D[D[D[Dcommit ide0-hd[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dcommit ide0-hd0[K
+(qemu) c[K[Dco[K[D[Dcom[K[D[D[Dcomm[K[D[D[D[Dcommi[K[D[D[D[D[Dcommit[K[D[D[D[D[D[Dcommit [K[D[D[D[D[D[D[Dcommit d[K[D[D[D[D[D[D[D[Dcommit dr[K[D[D[D[D[D[D[D[D[Dcommit dri[K[D[D[D[D[D[D[D[D[D[Dcommit driv[K[D[D[D[D[D[D[D[D[D[D[Dcommit drive[K[D[D[D[D[D[D[D[D[D[D[D[Dcommit drive0[K
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
 
 read 4096/4096 bytes at offset 0
diff --git a/tests/qemu-iotests/051.pc.out b/tests/qemu-iotests/051.pc.out
new file mode 100644
index 0000000..ec6d222
--- /dev/null
+++ b/tests/qemu-iotests/051.pc.out
@@ -0,0 +1,525 @@
+QA output created by 051
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+
+=== Unknown option ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=on,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=1234,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,unknown_opt=foo,if=none,id=drive0: Block format 'qcow2' does not support the option 'unknown_opt'
+
+
+=== Unknown protocol option ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=on: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=1234: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,file.unknown_opt=foo: Block protocol 'file' doesn't support the option 'unknown_opt'
+
+
+=== Invalid format ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=foo: Unknown driver 'foo'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=foo: Unknown driver 'foo'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,format=qcow2: Cannot specify both 'driver' and 'format'
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,format=qcow2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=qcow2,format=qcow2: Cannot specify both 'driver' and 'format'
+
+
+=== Device without drive ===
+
+Testing: -device virtio-scsi-pci -device scsi-hd
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-hd: drive property not set
+
+
+=== Overriding backing file ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=qcow2,backing.file.filename=TEST_DIR/t.qcow2.orig,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
+    Backing file:     TEST_DIR/t.qcow2.orig (chain depth: 1)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,driver=raw,backing.file.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
+
+Testing: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=file,file.backing.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
+
+Testing: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,file.backing.driver=qcow2,file.backing.file.filename=TEST_DIR/t.qcow2.orig: Driver doesn't support backing files
+
+
+=== Enable and disable lazy refcounting on the command line, plus some invalid values ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=off
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=: Parameter 'lazy-refcounts' expects 'on' or 'off'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=42: Parameter 'lazy-refcounts' expects 'on' or 'off'
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=foo: Parameter 'lazy-refcounts' expects 'on' or 'off'
+
+
+=== With version 2 images enabling lazy refcounts must fail ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=on: Lazy refcounts require a qcow2 image with at least qemu 1.1 compatibility level
+
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,lazy-refcounts=off
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+
+=== No medium ===
+
+Testing: -drive if=floppy
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive if=ide,media=cdrom
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive if=scsi,media=cdrom
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive if=ide
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: Device needs media, but drive is empty
+QEMU_PROG: Initialization of device ide-hd failed: Device initialization failed.
+
+Testing: -drive if=scsi
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: Initialization of device lsi53c895a failed: Device needs media, but drive is empty
+
+Testing: -drive if=virtio
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -drive if=virtio: Device needs media, but drive is empty
+
+Testing: -drive if=none,id=disk -device ide-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive if=none,id=disk -device ide-drive,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: Device needs media, but drive is empty
+QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
+
+Testing: -drive if=none,id=disk -device ide-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-hd,drive=disk: Device needs media, but drive is empty
+QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
+
+Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-disk,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-disk,drive=disk: Device needs media, but drive is empty
+
+Testing: -drive if=none,id=disk -device lsi53c895a -device scsi-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device scsi-hd,drive=disk: Device needs media, but drive is empty
+
+
+=== Read-only ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=floppy,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=ide,media=cdrom,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,media=cdrom,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=ide,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: Can't use a read-only drive
+QEMU_PROG: Initialization of device ide-hd failed: Device initialization failed.
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=scsi,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=virtio,readonly=on
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-cd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-drive,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-drive,drive=disk: Can't use a read-only drive
+QEMU_PROG: -device ide-drive,drive=disk: Device initialization failed.
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device ide-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) QEMU_PROG: -device ide-hd,drive=disk: Can't use a read-only drive
+QEMU_PROG: -device ide-hd,drive=disk: Device initialization failed.
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-disk,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=disk,readonly=on -device lsi53c895a -device scsi-hd,drive=disk
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+
+=== Cache modes ===
+
+Testing: -drive driver=null-co,cache=none
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive driver=null-co,cache=directsync
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive driver=null-co,cache=writeback
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive driver=null-co,cache=writethrough
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive driver=null-co,cache=unsafe
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive driver=null-co,cache=invalid_value
+QEMU_PROG: -drive driver=null-co,cache=invalid_value: invalid cache option
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writeback,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block f[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block file[K
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-f[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-file[K
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writethrough,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writethrough
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block f[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block file[K
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-f[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-file[K
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=unsafe,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K
+drive0 (NODE_NAME): TEST_DIR/t.qcow2 (qcow2)
+    Removable device: not locked, tray closed
+    Cache mode:       writeback, ignore flushes
+    Backing file:     TEST_DIR/t.qcow2.base (chain depth: 1)
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block f[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block file[K
+
+file: TEST_DIR/t.qcow2 (file)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K
+backing: TEST_DIR/t.qcow2.base (qcow2, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) i[K[Din[K[D[Dinf[K[D[D[Dinfo[K[D[D[D[Dinfo [K[D[D[D[D[Dinfo b[K[D[D[D[D[D[Dinfo bl[K[D[D[D[D[D[D[Dinfo blo[K[D[D[D[D[D[D[D[Dinfo bloc[K[D[D[D[D[D[D[D[D[Dinfo block[K[D[D[D[D[D[D[D[D[D[Dinfo block [K[D[D[D[D[D[D[D[D[D[D[Dinfo block b[K[D[D[D[D[D[D[D[D[D[D[D[Dinfo block ba[K[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block bac[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block back[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backin[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-f[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fi[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-fil[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dinfo block backing-file[K
+
+backing-file: TEST_DIR/t.qcow2.base (file, read-only)
+    Cache mode:       writeback, ignore flushes
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0 -nodefaults
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,cache=invalid_value,backing.file.filename=TEST_DIR/t.qcow2.base,backing.cache.no-flush=on,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,if=none,id=drive0: invalid cache option
+
+
+=== Specifying the protocol layer ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,file.driver=file
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+
+=== Leaving out required options ===
+
+Testing: -drive driver=file
+QEMU_PROG: -drive driver=file: The 'file' block driver requires a file name
+
+Testing: -drive driver=nbd
+QEMU_PROG: -drive driver=nbd: one of path and host must be specified.
+
+Testing: -drive driver=raw
+QEMU_PROG: -drive driver=raw: Can't use 'raw' as a block driver for the protocol level
+
+Testing: -drive file.driver=file
+QEMU_PROG: -drive file.driver=file: The 'file' block driver requires a file name
+
+Testing: -drive file.driver=nbd
+QEMU_PROG: -drive file.driver=nbd: one of path and host must be specified.
+
+Testing: -drive file.driver=raw
+QEMU_PROG: -drive file.driver=raw: Can't use 'raw' as a block driver for the protocol level
+
+Testing: -drive foo=bar
+QEMU_PROG: -drive foo=bar: Must specify either driver or file
+
+
+=== Specifying both an option and its legacy alias ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=1234,throttling.iops-total=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=1234,throttling.iops-total=5678: 'throttling.iops-total' and its alias 'iops' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_rd=1234,throttling.iops-read=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_rd=1234,throttling.iops-read=5678: 'throttling.iops-read' and its alias 'iops_rd' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_wr=1234,throttling.iops-write=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_wr=1234,throttling.iops-write=5678: 'throttling.iops-write' and its alias 'iops_wr' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1234,throttling.bps-total=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1234,throttling.bps-total=5678: 'throttling.bps-total' and its alias 'bps' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=1234,throttling.bps-read=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=1234,throttling.bps-read=5678: 'throttling.bps-read' and its alias 'bps_rd' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_wr=1234,throttling.bps-write=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_wr=1234,throttling.bps-write=5678: 'throttling.bps-write' and its alias 'bps_wr' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_max=1234,throttling.iops-total-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_max=1234,throttling.iops-total-max=5678: 'throttling.iops-total-max' and its alias 'iops_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_rd_max=1234,throttling.iops-read-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_rd_max=1234,throttling.iops-read-max=5678: 'throttling.iops-read-max' and its alias 'iops_rd_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_wr_max=1234,throttling.iops-write-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_wr_max=1234,throttling.iops-write-max=5678: 'throttling.iops-write-max' and its alias 'iops_wr_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_max=1234,throttling.bps-total-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_max=1234,throttling.bps-total-max=5678: 'throttling.bps-total-max' and its alias 'bps_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=1234,throttling.bps-read-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=1234,throttling.bps-read-max=5678: 'throttling.bps-read-max' and its alias 'bps_rd_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_wr_max=1234,throttling.bps-write-max=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_wr_max=1234,throttling.bps-write-max=5678: 'throttling.bps-write-max' and its alias 'bps_wr_max' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops_size=1234,throttling.iops-size=5678
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops_size=1234,throttling.iops-size=5678: 'throttling.iops-size' and its alias 'iops_size' can't be used at the same time
+
+Testing: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,readonly=on,read-only=off: 'read-only' and its alias 'readonly' can't be used at the same time
+
+
+=== Catching negative/large throttling values ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,iops=-1
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,iops=-1: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=-2
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=-2: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps_rd_max=-3: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.iops-total=-4: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,throttling.bps-total=-5: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000000
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=1000000000000001: bps/iops/max values must be within [0, 1000000000000000]
+
+Testing: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,bps=9999999999999999: bps/iops/max values must be within [0, 1000000000000000]
+
+
+=== Parsing protocol from file name ===
+
+Testing: -hda foo:bar
+QEMU_PROG: -hda foo:bar: Unknown protocol 'foo'
+
+Testing: -drive file=foo:bar
+QEMU_PROG: -drive file=foo:bar: Unknown protocol 'foo'
+
+Testing: -drive file.filename=foo:bar
+QEMU_PROG: -drive file.filename=foo:bar: Could not open 'foo:bar': No such file or directory
+
+Testing: -hda file:TEST_DIR/t.qcow2
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=file:TEST_DIR/t.qcow2
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file.filename=file:TEST_DIR/t.qcow2
+QEMU_PROG: -drive file.filename=file:TEST_DIR/t.qcow2: Could not open 'file:TEST_DIR/t.qcow2': No such file or directory
+
+
+=== Snapshot mode ===
+
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file.filename=TEST_DIR/t.qcow2,driver=qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=file:TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=file:TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,if=none,id=drive0 -snapshot
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=off,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x2[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x22 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Testing: -drive file=TEST_DIR/t.qcow2,snapshot=on,if=none,id=drive0
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqe[K[D[Dqem[K[D[D[Dqemu[K[D[D[D[Dqemu-[K[D[D[D[D[Dqemu-i[K[D[D[D[D[D[Dqemu-io[K[D[D[D[D[D[D[Dqemu-io [K[D[D[D[D[D[D[D[Dqemu-io d[K[D[D[D[D[D[D[D[D[Dqemu-io dr[K[D[D[D[D[D[D[D[D[D[Dqemu-io dri[K[D[D[D[D[D[D[D[D[D[D[Dqemu-io driv[K[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive[K[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "w[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wr[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "wri[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "writ[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x3[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 [K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 4[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 4k[K[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[D[Dqemu-io drive0 "write -P 0x33 0 4k"[K
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+(qemu) c[K[Dco[K[D[Dcom[K[D[D[Dcomm[K[D[D[D[Dcommi[K[D[D[D[D[Dcommit[K[D[D[D[D[D[Dcommit [K[D[D[D[D[D[D[Dcommit d[K[D[D[D[D[D[D[D[Dcommit dr[K[D[D[D[D[D[D[D[D[Dcommit dri[K[D[D[D[D[D[D[D[D[D[Dcommit driv[K[D[D[D[D[D[D[D[D[D[D[Dcommit drive[K[D[D[D[D[D[D[D[D[D[D[D[Dcommit drive0[K
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
index 61959e2..842eace 100755
--- a/tests/qemu-iotests/052
+++ b/tests/qemu-iotests/052
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -49,6 +48,10 @@
 _make_test_img $size
 
 echo
+echo "== initializing whole image =="
+$QEMU_IO -c "write -z 0 $size" "$TEST_IMG" | _filter_qemu_io
+
+echo
 echo "== reading whole image =="
 $QEMU_IO -s -c "read 0 $size" "$TEST_IMG" | _filter_qemu_io
 
diff --git a/tests/qemu-iotests/052.out b/tests/qemu-iotests/052.out
index 8617aa2..a377d30 100644
--- a/tests/qemu-iotests/052.out
+++ b/tests/qemu-iotests/052.out
@@ -1,5 +1,9 @@
 QA output created by 052
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+
+== initializing whole image ==
+wrote 134217728/134217728 bytes at offset 0
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == reading whole image ==
 read 134217728/134217728 bytes at offset 0
diff --git a/tests/qemu-iotests/053 b/tests/qemu-iotests/053
index e589e5f..2a04f5f 100755
--- a/tests/qemu-iotests/053
+++ b/tests/qemu-iotests/053
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/053.out b/tests/qemu-iotests/053.out
index 16464e6..8e793b6 100644
--- a/tests/qemu-iotests/053.out
+++ b/tests/qemu-iotests/053.out
@@ -1,7 +1,7 @@
 QA output created by 053
 
 == Creating single sector image ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=512 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=512
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/tests/qemu-iotests/054 b/tests/qemu-iotests/054
index bd94153..bf47ef9 100755
--- a/tests/qemu-iotests/054
+++ b/tests/qemu-iotests/054
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/054.out b/tests/qemu-iotests/054.out
index 7161d6e..e6ec430 100644
--- a/tests/qemu-iotests/054.out
+++ b/tests/qemu-iotests/054.out
@@ -2,9 +2,9 @@
 
 creating too large image (1 EB)
 qemu-img: TEST_DIR/t.IMGFMT: The image size is too large for file format 'IMGFMT' (try using a larger cluster size)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1152921504606846976 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1152921504606846976
 
 creating too large image (1 EB) using qcow2.py
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296
 qemu-img: Could not open 'TEST_DIR/t.qcow2': Image is too big
 *** done
diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055
index 451b67d..c8e3578 100755
--- a/tests/qemu-iotests/055
+++ b/tests/qemu-iotests/055
@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 #
-# Tests for drive-backup
+# Tests for drive-backup and blockdev-backup
 #
-# Copyright (C) 2013 Red Hat, Inc.
+# Copyright (C) 2013, 2014 Red Hat, Inc.
 #
 # Based on 041.
 #
@@ -27,6 +27,7 @@
 
 test_img = os.path.join(iotests.test_dir, 'test.img')
 target_img = os.path.join(iotests.test_dir, 'target.img')
+blockdev_target_img = os.path.join(iotests.test_dir, 'blockdev-target.img')
 
 class TestSingleDrive(iotests.QMPTestCase):
     image_len = 64 * 1024 * 1024 # MB
@@ -34,38 +35,47 @@
     def setUp(self):
         # Write data to the image so we can compare later
         qemu_img('create', '-f', iotests.imgfmt, test_img, str(TestSingleDrive.image_len))
-        qemu_io('-c', 'write -P0x5d 0 64k', test_img)
-        qemu_io('-c', 'write -P0xd5 1M 32k', test_img)
-        qemu_io('-c', 'write -P0xdc 32M 124k', test_img)
-        qemu_io('-c', 'write -P0xdc 67043328 64k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 0 64k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0xd5 1M 32k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0xdc 32M 124k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0xdc 67043328 64k', test_img)
+        qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
 
-        self.vm = iotests.VM().add_drive(test_img)
+        self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
         self.vm.shutdown()
         os.remove(test_img)
+        os.remove(blockdev_target_img)
         try:
             os.remove(target_img)
         except OSError:
             pass
 
-    def test_cancel(self):
+    def do_test_cancel(self, cmd, target):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-backup', device='drive0',
-                             target=target_img, sync='full')
+        result = self.vm.qmp(cmd, device='drive0', target=target, sync='full')
         self.assert_qmp(result, 'return', {})
 
         event = self.cancel_and_wait()
         self.assert_qmp(event, 'data/type', 'backup')
 
-    def test_pause(self):
+    def test_cancel_drive_backup(self):
+        self.do_test_cancel('drive-backup', target_img)
+
+    def test_cancel_blockdev_backup(self):
+        self.do_test_cancel('blockdev-backup', 'drive1')
+
+    def do_test_pause(self, cmd, target, image):
         self.assert_no_active_block_jobs()
 
         self.vm.pause_drive('drive0')
-        result = self.vm.qmp('drive-backup', device='drive0',
-                             target=target_img, sync='full')
+        result = self.vm.qmp(cmd, device='drive0',
+                             target=target, sync='full')
         self.assert_qmp(result, 'return', {})
 
         result = self.vm.qmp('block-job-pause', device='drive0')
@@ -86,14 +96,31 @@
         self.wait_until_completed()
 
         self.vm.shutdown()
-        self.assertTrue(iotests.compare_images(test_img, target_img),
+        self.assertTrue(iotests.compare_images(test_img, image),
                         'target image does not match source after backup')
 
+    def test_pause_drive_backup(self):
+        self.do_test_pause('drive-backup', target_img, target_img)
+
+    def test_pause_blockdev_backup(self):
+        self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
+
     def test_medium_not_found(self):
-        result = self.vm.qmp('drive-backup', device='ide1-cd0',
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('drive-backup', device='drive2', # CD-ROM
                              target=target_img, sync='full')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
+    def test_medium_not_found_blockdev_backup(self):
+        if iotests.qemu_default_machine != 'pc':
+            return
+
+        result = self.vm.qmp('blockdev-backup', device='drive2', # CD-ROM
+                             target='drive1', sync='full')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
     def test_image_not_found(self):
         result = self.vm.qmp('drive-backup', device='drive0',
                              target=target_img, sync='full', mode='existing')
@@ -105,31 +132,56 @@
                              format='spaghetti-noodles')
         self.assert_qmp(result, 'error/class', 'GenericError')
 
+    def do_test_device_not_found(self, cmd, **args):
+        result = self.vm.qmp(cmd, **args)
+        if cmd == 'drive-backup':
+            self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        else:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+
     def test_device_not_found(self):
-        result = self.vm.qmp('drive-backup', device='nonexistent',
-                             target=target_img, sync='full')
-        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+        self.do_test_device_not_found('drive-backup', device='nonexistent',
+                                      target=target_img, sync='full')
+
+        self.do_test_device_not_found('blockdev-backup', device='nonexistent',
+                                      target='drive0', sync='full')
+
+        self.do_test_device_not_found('blockdev-backup', device='drive0',
+                                      target='nonexistent', sync='full')
+
+        self.do_test_device_not_found('blockdev-backup', device='nonexistent',
+                                      target='nonexistent', sync='full')
+
+    def test_target_is_source(self):
+        result = self.vm.qmp('blockdev-backup', device='drive0',
+                             target='drive0', sync='full')
+        self.assert_qmp(result, 'error/class', 'GenericError')
 
 class TestSetSpeed(iotests.QMPTestCase):
     image_len = 80 * 1024 * 1024 # MB
 
     def setUp(self):
         qemu_img('create', '-f', iotests.imgfmt, test_img, str(TestSetSpeed.image_len))
-        qemu_io('-c', 'write -P1 0 512', test_img)
-        self.vm = iotests.VM().add_drive(test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P1 0 512', test_img)
+        qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
+
+        self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
         self.vm.launch()
 
     def tearDown(self):
         self.vm.shutdown()
         os.remove(test_img)
-        os.remove(target_img)
+        os.remove(blockdev_target_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
 
-    def test_set_speed(self):
+    def do_test_set_speed(self, cmd, target):
         self.assert_no_active_block_jobs()
 
         self.vm.pause_drive('drive0')
-        result = self.vm.qmp('drive-backup', device='drive0',
-                             target=target_img, sync='full')
+        result = self.vm.qmp(cmd, device='drive0', target=target, sync='full')
         self.assert_qmp(result, 'return', {})
 
         # Default speed is 0
@@ -148,10 +200,10 @@
         event = self.cancel_and_wait(resume=True)
         self.assert_qmp(event, 'data/type', 'backup')
 
-        # Check setting speed in drive-backup works
+        # Check setting speed option works
         self.vm.pause_drive('drive0')
-        result = self.vm.qmp('drive-backup', device='drive0',
-                             target=target_img, sync='full', speed=4*1024*1024)
+        result = self.vm.qmp(cmd, device='drive0',
+                             target=target, sync='full', speed=4*1024*1024)
         self.assert_qmp(result, 'return', {})
 
         result = self.vm.qmp('query-block-jobs')
@@ -161,18 +213,24 @@
         event = self.cancel_and_wait(resume=True)
         self.assert_qmp(event, 'data/type', 'backup')
 
-    def test_set_speed_invalid(self):
+    def test_set_speed_drive_backup(self):
+        self.do_test_set_speed('drive-backup', target_img)
+
+    def test_set_speed_blockdev_backup(self):
+        self.do_test_set_speed('blockdev-backup', 'drive1')
+
+    def do_test_set_speed_invalid(self, cmd, target):
         self.assert_no_active_block_jobs()
 
-        result = self.vm.qmp('drive-backup', device='drive0',
-                             target=target_img, sync='full', speed=-1)
+        result = self.vm.qmp(cmd, device='drive0',
+                             target=target, sync='full', speed=-1)
         self.assert_qmp(result, 'error/class', 'GenericError')
 
         self.assert_no_active_block_jobs()
 
         self.vm.pause_drive('drive0')
-        result = self.vm.qmp('drive-backup', device='drive0',
-                             target=target_img, sync='full')
+        result = self.vm.qmp(cmd, device='drive0',
+                             target=target, sync='full')
         self.assert_qmp(result, 'return', {})
 
         result = self.vm.qmp('block-job-set-speed', device='drive0', speed=-1)
@@ -181,50 +239,67 @@
         event = self.cancel_and_wait(resume=True)
         self.assert_qmp(event, 'data/type', 'backup')
 
+    def test_set_speed_invalid_drive_backup(self):
+        self.do_test_set_speed_invalid('drive-backup', target_img)
+
+    def test_set_speed_invalid_blockdev_backup(self):
+        self.do_test_set_speed_invalid('blockdev-backup',  'drive1')
+
 class TestSingleTransaction(iotests.QMPTestCase):
     image_len = 64 * 1024 * 1024 # MB
 
     def setUp(self):
         qemu_img('create', '-f', iotests.imgfmt, test_img, str(TestSingleTransaction.image_len))
-        qemu_io('-c', 'write -P0x5d 0 64k', test_img)
-        qemu_io('-c', 'write -P0xd5 1M 32k', test_img)
-        qemu_io('-c', 'write -P0xdc 32M 124k', test_img)
-        qemu_io('-c', 'write -P0xdc 67043328 64k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 0 64k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0xd5 1M 32k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0xdc 32M 124k', test_img)
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0xdc 67043328 64k', test_img)
+        qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len))
 
-        self.vm = iotests.VM().add_drive(test_img)
+        self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img)
+        if iotests.qemu_default_machine == 'pc':
+            self.vm.add_drive(None, 'media=cdrom', 'ide')
         self.vm.launch()
 
     def tearDown(self):
         self.vm.shutdown()
         os.remove(test_img)
+        os.remove(blockdev_target_img)
         try:
             os.remove(target_img)
         except OSError:
             pass
 
-    def test_cancel(self):
+    def do_test_cancel(self, cmd, target):
         self.assert_no_active_block_jobs()
 
         result = self.vm.qmp('transaction', actions=[{
-                'type': 'drive-backup',
+                'type': cmd,
                 'data': { 'device': 'drive0',
-                          'target': target_img,
+                          'target': target,
                           'sync': 'full' },
             }
         ])
+
         self.assert_qmp(result, 'return', {})
 
         event = self.cancel_and_wait()
         self.assert_qmp(event, 'data/type', 'backup')
 
-    def test_pause(self):
+    def test_cancel_drive_backup(self):
+        self.do_test_cancel('drive-backup', target_img)
+
+    def test_cancel_blockdev_backup(self):
+        self.do_test_cancel('blockdev-backup', 'drive1')
+
+    def do_test_pause(self, cmd, target, image):
         self.assert_no_active_block_jobs()
 
         self.vm.pause_drive('drive0')
         result = self.vm.qmp('transaction', actions=[{
-                'type': 'drive-backup',
+                'type': cmd,
                 'data': { 'device': 'drive0',
-                          'target': target_img,
+                          'target': target,
                           'sync': 'full' },
             }
         ])
@@ -248,19 +323,34 @@
         self.wait_until_completed()
 
         self.vm.shutdown()
-        self.assertTrue(iotests.compare_images(test_img, target_img),
+        self.assertTrue(iotests.compare_images(test_img, image),
                         'target image does not match source after backup')
 
-    def test_medium_not_found(self):
+    def test_pause_drive_backup(self):
+        self.do_test_pause('drive-backup', target_img, target_img)
+
+    def test_pause_blockdev_backup(self):
+        self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img)
+
+    def do_test_medium_not_found(self, cmd, target):
+        if iotests.qemu_default_machine != 'pc':
+            return
+
         result = self.vm.qmp('transaction', actions=[{
-                'type': 'drive-backup',
-                'data': { 'device': 'ide1-cd0',
-                          'target': target_img,
+                'type': cmd,
+                'data': { 'device': 'drive2', # CD-ROM
+                          'target': target,
                           'sync': 'full' },
             }
         ])
         self.assert_qmp(result, 'error/class', 'GenericError')
 
+    def test_medium_not_found_drive_backup(self):
+        self.do_test_medium_not_found('drive-backup', target_img)
+
+    def test_medium_not_found_blockdev_backup(self):
+        self.do_test_medium_not_found('blockdev-backup', 'drive1')
+
     def test_image_not_found(self):
         result = self.vm.qmp('transaction', actions=[{
                 'type': 'drive-backup',
@@ -283,6 +373,43 @@
         ])
         self.assert_qmp(result, 'error/class', 'DeviceNotFound')
 
+        result = self.vm.qmp('transaction', actions=[{
+                'type': 'blockdev-backup',
+                'data': { 'device': 'nonexistent',
+                          'target': 'drive1',
+                          'sync': 'full' },
+            }
+        ])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        result = self.vm.qmp('transaction', actions=[{
+                'type': 'blockdev-backup',
+                'data': { 'device': 'drive0',
+                          'target': 'nonexistent',
+                          'sync': 'full' },
+            }
+        ])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        result = self.vm.qmp('transaction', actions=[{
+                'type': 'blockdev-backup',
+                'data': { 'device': 'nonexistent',
+                          'target': 'nonexistent',
+                          'sync': 'full' },
+            }
+        ])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_target_is_source(self):
+        result = self.vm.qmp('transaction', actions=[{
+                'type': 'blockdev-backup',
+                'data': { 'device': 'drive0',
+                          'target': 'drive0',
+                          'sync': 'full' },
+            }
+        ])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
     def test_abort(self):
         result = self.vm.qmp('transaction', actions=[{
                 'type': 'drive-backup',
@@ -298,5 +425,31 @@
         self.assert_qmp(result, 'error/class', 'GenericError')
         self.assert_no_active_block_jobs()
 
+        result = self.vm.qmp('transaction', actions=[{
+                'type': 'blockdev-backup',
+                'data': { 'device': 'nonexistent',
+                          'target': 'drive1',
+                          'sync': 'full' },
+            }, {
+                'type': 'Abort',
+                'data': {},
+            }
+        ])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_no_active_block_jobs()
+
+        result = self.vm.qmp('transaction', actions=[{
+                'type': 'blockdev-backup',
+                'data': { 'device': 'drive0',
+                          'target': 'nonexistent',
+                          'sync': 'full' },
+            }, {
+                'type': 'Abort',
+                'data': {},
+            }
+        ])
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_no_active_block_jobs()
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['raw', 'qcow2'])
diff --git a/tests/qemu-iotests/055.out b/tests/qemu-iotests/055.out
index 6323079..42314e9 100644
--- a/tests/qemu-iotests/055.out
+++ b/tests/qemu-iotests/055.out
@@ -1,5 +1,5 @@
-..............
+........................
 ----------------------------------------------------------------------
-Ran 14 tests
+Ran 24 tests
 
 OK
diff --git a/tests/qemu-iotests/056 b/tests/qemu-iotests/056
index 54e4bd0..04f2c3c 100755
--- a/tests/qemu-iotests/056
+++ b/tests/qemu-iotests/056
@@ -82,6 +82,31 @@
         time.sleep(1)
         self.assertEqual(-1, qemu_io('-c', 'read -P0x41 0 512', target_img).find("verification failed"))
 
+class TestBeforeWriteNotifier(iotests.QMPTestCase):
+    def setUp(self):
+        self.vm = iotests.VM().add_drive_raw("file=blkdebug::null-co://,id=drive0,align=65536,driver=blkdebug")
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(target_img)
+
+    def test_before_write_notifier(self):
+        self.vm.pause_drive("drive0")
+        result = self.vm.qmp('drive-backup', device='drive0',
+                             sync='full', target=target_img,
+                             format="file", speed=1)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp('block-job-pause', device="drive0")
+        self.assert_qmp(result, 'return', {})
+        # Speed is low enough that this must be an uncopied range, which will
+        # trigger the before write notifier
+        self.vm.hmp_qemu_io('drive0', 'aio_write -P 1 512512 512')
+        self.vm.resume_drive("drive0")
+        result = self.vm.qmp('block-job-resume', device="drive0")
+        self.assert_qmp(result, 'return', {})
+        event = self.cancel_and_wait()
+        self.assert_qmp(event, 'data/type', 'backup')
 
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/056.out b/tests/qemu-iotests/056.out
index fbc63e6..8d7e996 100644
--- a/tests/qemu-iotests/056.out
+++ b/tests/qemu-iotests/056.out
@@ -1,5 +1,5 @@
-..
+...
 ----------------------------------------------------------------------
-Ran 2 tests
+Ran 3 tests
 
 OK
diff --git a/tests/qemu-iotests/058 b/tests/qemu-iotests/058
index 14584cd..2253c6a 100755
--- a/tests/qemu-iotests/058
+++ b/tests/qemu-iotests/058
@@ -27,16 +27,21 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 nbd_unix_socket=$TEST_DIR/test_qemu_nbd_socket
 nbd_snapshot_img="nbd:unix:$nbd_unix_socket"
+rm -f "${TEST_DIR}/qemu-nbd.pid"
 
 _cleanup_nbd()
 {
-    if [ -n "$NBD_SNAPSHOT_PID" ]; then
-        kill "$NBD_SNAPSHOT_PID"
+    local NBD_SNAPSHOT_PID
+    if [ -f "${TEST_DIR}/qemu-nbd.pid" ]; then
+        read NBD_SNAPSHOT_PID < "${TEST_DIR}/qemu-nbd.pid"
+        rm -f "${TEST_DIR}/qemu-nbd.pid"
+        if [ -n "$NBD_SNAPSHOT_PID" ]; then
+            kill "$NBD_SNAPSHOT_PID"
+        fi
     fi
     rm -f "$nbd_unix_socket"
 }
@@ -60,7 +65,6 @@
 {
     _cleanup_nbd
     $QEMU_NBD -v -t -k "$nbd_unix_socket" "$TEST_IMG" -l $1 &
-    NBD_SNAPSHOT_PID=$!
     _wait_for_nbd
 }
 
@@ -68,7 +72,6 @@
 {
     _cleanup_nbd
     $QEMU_NBD -v -t -k "$nbd_unix_socket" "$TEST_IMG" -l snapshot.name=$1 &
-    NBD_SNAPSHOT_PID=$!
     _wait_for_nbd
 }
 
@@ -87,7 +90,13 @@
 
 _supported_fmt qcow2
 _supported_proto file
+_supported_os Linux
 _require_command QEMU_NBD
+# Internal snapshots are (currently) impossible with refcount_bits=1
+_unsupported_imgopts 'refcount_bits=1[^0-9]'
+
+# Use -f raw instead of -f $IMGFMT for the NBD connection
+QEMU_IO_NBD="$QEMU_IO -f raw --cache=$CACHEMODE"
 
 echo
 echo "== preparing image =="
@@ -108,15 +117,15 @@
 
 echo
 echo "== verifying the exported snapshot with patterns, method 1 =="
-$QEMU_IO -c 'read -P 0xa 0x1000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
-$QEMU_IO -c 'read -P 0xb 0x2000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
+$QEMU_IO_NBD -c 'read -P 0xa 0x1000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
+$QEMU_IO_NBD -c 'read -P 0xb 0x2000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
 
 _export_nbd_snapshot1 sn1
 
 echo
 echo "== verifying the exported snapshot with patterns, method 2 =="
-$QEMU_IO -c 'read -P 0xa 0x1000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
-$QEMU_IO -c 'read -P 0xb 0x2000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
+$QEMU_IO_NBD -c 'read -P 0xa 0x1000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
+$QEMU_IO_NBD -c 'read -P 0xb 0x2000 0x1000' "$nbd_snapshot_img" | _filter_qemu_io
 
 $QEMU_IMG convert "$TEST_IMG" -l sn1 -O qcow2 "$converted_image"
 
diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index 3c053c2..6655aaf 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -106,11 +105,17 @@
 echo
 echo "=== Converting to streamOptimized from image with small cluster size==="
 TEST_IMG="$TEST_IMG.qcow2" IMGFMT=qcow2 IMGOPTS="cluster_size=4096" _make_test_img 1G
-$QEMU_IO -c "write -P 0xa 0 512" "$TEST_IMG.qcow2" | _filter_qemu_io
-$QEMU_IO -c "write -P 0xb 10240 512" "$TEST_IMG.qcow2" | _filter_qemu_io
+$QEMU_IO -f qcow2 -c "write -P 0xa 0 512" "$TEST_IMG.qcow2" | _filter_qemu_io
+$QEMU_IO -f qcow2 -c "write -P 0xb 10240 512" "$TEST_IMG.qcow2" | _filter_qemu_io
 $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2" "$TEST_IMG" 2>&1
 
 echo
+echo "=== Testing monolithicFlat with internally generated JSON file name ==="
+IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
+$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \
+    | _filter_testdir | _filter_imgfmt
+
+echo
 echo "=== Testing version 3 ==="
 _use_sample_img iotest-version3.vmdk.bz2
 _img_info
@@ -126,6 +131,21 @@
 $QEMU_IO -c "write -P 0xa 900G 512" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "read -v 900G 1024" "$TEST_IMG" | _filter_qemu_io
 
+echo
+echo "=== Testing qemu-img map on extents ==="
+for fmt in monolithicSparse twoGbMaxExtentSparse; do
+    IMGOPTS="subformat=$fmt" _make_test_img 31G
+    $QEMU_IO -c "write 65024 1k" "$TEST_IMG" | _filter_qemu_io
+    $QEMU_IO -c "write 2147483136 1k" "$TEST_IMG" | _filter_qemu_io
+    $QEMU_IO -c "write 5G 1k" "$TEST_IMG" | _filter_qemu_io
+    $QEMU_IMG map "$TEST_IMG" | _filter_testdir
+done
+
+echo
+echo "=== Testing afl image with a very large capacity ==="
+_use_sample_img afl9.vmdk.bz2
+_img_info
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index 0dadba6..678adb4 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -2,31 +2,31 @@
 
 === Testing invalid granularity ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vmdk: Invalid granularity, image may be corrupt
+can't open device TEST_DIR/t.vmdk: Invalid granularity, image may be corrupt
 no file open, try 'help open'
 
 === Testing too big L2 table size ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vmdk: L2 table size too big
+can't open device TEST_DIR/t.vmdk: L2 table size too big
 no file open, try 'help open'
 
 === Testing too big L1 table size ===
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
+can't open device TEST_DIR/t.vmdk: L1 size too big
 no file open, try 'help open'
 
 === Testing monolithicFlat creation and opening ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 2.0G (2147483648 bytes)
 
 === Testing monolithicFlat with zeroed_grain ===
 qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
 
 === Testing big twoGbMaxExtentFlat ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat
 image: TEST_DIR/t.vmdk
 file format: vmdk
 virtual size: 1.0T (1073741824000 bytes)
@@ -2038,12 +2038,10 @@
             format: FLAT
 
 === Testing malformed VMFS extent description line ===
-qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent lines:
-RW 12582912 VMFS "dummy.IMGFMT" 1
-
+qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1
 
 === Testing truncated sparse ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes
 
 === Converting to streamOptimized from image with small cluster size===
@@ -2053,6 +2051,10 @@
 wrote 512/512 bytes at offset 10240
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
+=== Testing monolithicFlat with internally generated JSON file name ===
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat
+can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
+
 === Testing version 3 ===
 image: TEST_DIR/iotest-version3.IMGFMT
 file format: IMGFMT
@@ -2260,7 +2262,7 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing 4TB monolithicFlat creation and IO ===
-Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
 image: TEST_DIR/iotest-version3.IMGFMT
 file format: IMGFMT
 virtual size: 4.0T (4398046511104 bytes)
@@ -2332,4 +2334,32 @@
 e1000003f0:  00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00  ................
 read 1024/1024 bytes at offset 966367641600
 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing qemu-img map on extents ===
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
+wrote 1024/1024 bytes at offset 65024
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 2147483136
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 5368709120
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          Mapped to       File
+0               0x20000         0x3f0000        TEST_DIR/iotest-version3.vmdk
+0x7fff0000      0x20000         0x410000        TEST_DIR/iotest-version3.vmdk
+0x140000000     0x10000         0x430000        TEST_DIR/iotest-version3.vmdk
+Formatting 'TEST_DIR/iotest-version3.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
+wrote 1024/1024 bytes at offset 65024
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 2147483136
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 5368709120
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          Mapped to       File
+0               0x20000         0x50000         TEST_DIR/iotest-version3-s001.vmdk
+0x7fff0000      0x10000         0x70000         TEST_DIR/iotest-version3-s001.vmdk
+0x80000000      0x10000         0x50000         TEST_DIR/iotest-version3-s002.vmdk
+0x140000000     0x10000         0x50000         TEST_DIR/iotest-version3-s003.vmdk
+
+=== Testing afl image with a very large capacity ===
+qemu-img: Can't get size of device 'image': File too large
 *** done
diff --git a/tests/qemu-iotests/060 b/tests/qemu-iotests/060
index 9772d36..8e95c45 100755
--- a/tests/qemu-iotests/060
+++ b/tests/qemu-iotests/060
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -77,7 +76,7 @@
 $PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
 
 # This information should be available through qemu-img info
-$QEMU_IMG info "$TEST_IMG" | _filter_testdir
+_img_info --format-specific
 
 # Try to open the image R/W (which should fail)
 $QEMU_IO -c "$OPEN_RW" -c "read 0 512" 2>&1 | _filter_qemu_io \
@@ -186,6 +185,12 @@
 poke_file "$TEST_IMG" "$l1_offset" "\x80\x00\x00\x00\x00\x04\x2a\x00"
 $QEMU_IO -c "read 0 64k" "$TEST_IMG" | _filter_qemu_io
 
+# Test how well zero cluster expansion can cope with this
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$l1_offset" "\x80\x00\x00\x00\x00\x04\x2a\x00"
+$QEMU_IMG amend -o compat=0.10 "$TEST_IMG"
+
 echo
 echo "=== Testing unaligned L2 entry ==="
 echo
@@ -195,6 +200,15 @@
 $QEMU_IO -c "read 0 64k" "$TEST_IMG" | _filter_qemu_io
 
 echo
+echo "=== Testing unaligned pre-allocated zero cluster ==="
+echo
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$l2_offset" "\x80\x00\x00\x00\x00\x05\x2a\x01"
+# zero cluster expansion
+$QEMU_IMG amend -o compat=0.10 "$TEST_IMG"
+
+echo
 echo "=== Testing unaligned reftable entry ==="
 echo
 _make_test_img 64M
diff --git a/tests/qemu-iotests/060.out b/tests/qemu-iotests/060.out
index 9419da1..5d40206 100644
--- a/tests/qemu-iotests/060.out
+++ b/tests/qemu-iotests/060.out
@@ -2,7 +2,7 @@
 
 === Testing L2 reference into L1 ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 ERROR cluster 3 refcount=1 reference=3
 
 1 errors were found on the image.
@@ -11,22 +11,22 @@
 qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with active L1 table); further corruption events will be suppressed
 write failed: Input/output error
 incompatible_features     0x2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 64M (67108864 bytes)
-disk size: 196K
 cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    refcount bits: 16
     corrupt: true
-qemu-io: can't open device TEST_DIR/t.IMGFMT: IMGFMT: Image is corrupt; cannot be opened read/write
+can't open device TEST_DIR/t.IMGFMT: IMGFMT: Image is corrupt; cannot be opened read/write
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing cluster data reference into refcount block ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 ERROR refcount block 0 refcount=2
 ERROR cluster 2 refcount=1 reference=2
 
@@ -55,7 +55,7 @@
 
 === Testing cluster data reference into inactive L2 table ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 0
@@ -96,7 +96,7 @@
 
 === Testing overlap while COW is in flight ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 536870912
@@ -111,7 +111,7 @@
 
 === Testing unallocated image header ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
@@ -119,29 +119,42 @@
 
 === Testing unaligned L1 entry ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qcow2: Marking image as corrupt: L2 table offset 0x42a00 unaligned (L1 index: 0); further corruption events will be suppressed
 read failed: Input/output error
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Marking image as corrupt: L2 table offset 0x42a00 unaligned (L1 index: 0); further corruption events will be suppressed
+qemu-img: Error while amending options: Input/output error
 
 === Testing unaligned L2 entry ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qcow2: Marking image as corrupt: Data cluster offset 0x52a00 unaligned (L2 offset: 0x40000, L2 index: 0); further corruption events will be suppressed
 read failed: Input/output error
 
+=== Testing unaligned pre-allocated zero cluster ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qcow2: Marking image as corrupt: Data cluster offset 0x52a00 unaligned (L2 offset: 0x40000, L2 index: 0); further corruption events will be suppressed
+qemu-img: Error while amending options: Input/output error
+
 === Testing unaligned reftable entry ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 qcow2: Marking image as corrupt: Refblock offset 0x22a00 unaligned (reftable index: 0); further corruption events will be suppressed
 write failed: Input/output error
 
 === Testing non-fatal corruption on freeing ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qcow2: Image is corrupt: Cannot free unaligned cluster 0x52a00; further non-fatal corruption events will be suppressed
@@ -150,7 +163,7 @@
 
 === Testing read-only corruption report ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qcow2: Image is corrupt: Data cluster offset 0x52a00 unaligned (L2 offset: 0x40000, L2 index: 0); further non-fatal corruption events will be suppressed
@@ -159,7 +172,7 @@
 
 === Testing non-fatal and then fatal corruption report ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 131072/131072 bytes at offset 0
 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qcow2: Image is corrupt: Cannot free unaligned cluster 0x52a00; further non-fatal corruption events will be suppressed
diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061
index 8d37f8a..f5678b1 100755
--- a/tests/qemu-iotests/061
+++ b/tests/qemu-iotests/061
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -58,7 +57,8 @@
 echo "=== Testing dirty version downgrade ==="
 echo
 IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
-$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG"
 $PYTHON qcow2.py "$TEST_IMG" dump-header
@@ -91,7 +91,8 @@
 echo "=== Testing dirty lazy_refcounts=off ==="
 echo
 IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M
-$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -P 0x2a 0 128k" -c flush \
+         -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 | _filter_qemu_io
 $PYTHON qcow2.py "$TEST_IMG" dump-header
 $QEMU_IMG amend -o "lazy_refcounts=off" "$TEST_IMG"
 $PYTHON qcow2.py "$TEST_IMG" dump-header
diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
index 9045544..a431b7f 100644
--- a/tests/qemu-iotests/061.out
+++ b/tests/qemu-iotests/061.out
Binary files differ
diff --git a/tests/qemu-iotests/062 b/tests/qemu-iotests/062
index 0511246..051fb9f 100755
--- a/tests/qemu-iotests/062
+++ b/tests/qemu-iotests/062
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/062.out b/tests/qemu-iotests/062.out
index 442d761..9ddf22b 100644
--- a/tests/qemu-iotests/062.out
+++ b/tests/qemu-iotests/062.out
@@ -2,7 +2,7 @@
 
 === Testing snapshotting an image with zero clusters ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 262144/262144 bytes at offset 0
 256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 No errors were found on the image.
diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063
index a47493a..352e78c 100755
--- a/tests/qemu-iotests/063
+++ b/tests/qemu-iotests/063
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/064 b/tests/qemu-iotests/064
index 1c74c31..5792fbb 100755
--- a/tests/qemu-iotests/064
+++ b/tests/qemu-iotests/064
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -54,7 +53,15 @@
 
 echo
 echo "=== Verify pattern 0x00, 66M - 1024M ==="
-$QEMU_IO -r -c "read -pP 0x00 66M 958M" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -r -c "read -pP 0x00 66M 62M" \
+            -c "read -pP 0x00 128M 128M" \
+            -c "read -pP 0x00 256M 128M" \
+            -c "read -pP 0x00 384M 128M" \
+            -c "read -pP 0x00 512M 128M" \
+            -c "read -pP 0x00 640M 128M" \
+            -c "read -pP 0x00 768M 128M" \
+            -c "read -pP 0x00 896M 128M" \
+            "$TEST_IMG" | _filter_qemu_io
 
 echo
 echo "=== Verify pattern write, 0xc3 99M-157M ==="
@@ -63,7 +70,14 @@
 $QEMU_IO -c "read -pP 0xa5 0 33M" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "read -pP 0x96 33M 33M" "$TEST_IMG" | _filter_qemu_io
 $QEMU_IO -c "read -pP 0x00 66M 33M" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -pP 0x00 157MM 867MM" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -pP 0x00 157M 99M" \
+         -c "read -pP 0x00 256M 128M" \
+         -c "read -pP 0x00 384M 128M" \
+         -c "read -pP 0x00 512M 128M" \
+         -c "read -pP 0x00 640M 128M" \
+         -c "read -pP 0x00 768M 128M" \
+         -c "read -pP 0x00 896M 128M" \
+         "$TEST_IMG" | _filter_qemu_io
 # now verify what we should have actually written
 $QEMU_IO -c "read -pP 0xc3 99M 58M" "$TEST_IMG" | _filter_qemu_io
 
diff --git a/tests/qemu-iotests/064.out b/tests/qemu-iotests/064.out
index 5346a4e..1a5b9e2 100644
--- a/tests/qemu-iotests/064.out
+++ b/tests/qemu-iotests/064.out
@@ -9,8 +9,22 @@
 33 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Verify pattern 0x00, 66M - 1024M ===
-read 1004535808/1004535808 bytes at offset 69206016
-958 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65011712/65011712 bytes at offset 69206016
+62 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 134217728
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 268435456
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 402653184
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 536870912
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 671088640
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 805306368
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 939524096
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Verify pattern write, 0xc3 99M-157M ===
 wrote 60817408/60817408 bytes at offset 103809024
@@ -21,8 +35,20 @@
 33 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 34603008/34603008 bytes at offset 69206016
 33 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 909115392/909115392 bytes at offset 164626432
-867 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 103809024/103809024 bytes at offset 164626432
+99 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 268435456
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 402653184
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 536870912
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 671088640
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 805306368
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 134217728/134217728 bytes at offset 939524096
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 60817408/60817408 bytes at offset 103809024
 58 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 *** done
diff --git a/tests/qemu-iotests/065 b/tests/qemu-iotests/065
index 8d3a9c9..72aa970 100755
--- a/tests/qemu-iotests/065
+++ b/tests/qemu-iotests/065
@@ -88,34 +88,41 @@
 class TestQCow2(TestQemuImgInfo):
     '''Testing a qcow2 version 2 image'''
     img_options = 'compat=0.10'
-    json_compare = { 'compat': '0.10' }
-    human_compare = [ 'compat: 0.10' ]
+    json_compare = { 'compat': '0.10', 'refcount-bits': 16 }
+    human_compare = [ 'compat: 0.10', 'refcount bits: 16' ]
 
 class TestQCow3NotLazy(TestQemuImgInfo):
     '''Testing a qcow2 version 3 image with lazy refcounts disabled'''
     img_options = 'compat=1.1,lazy_refcounts=off'
-    json_compare = { 'compat': '1.1', 'lazy-refcounts': False, 'corrupt': False }
-    human_compare = [ 'compat: 1.1', 'lazy refcounts: false', 'corrupt: false' ]
+    json_compare = { 'compat': '1.1', 'lazy-refcounts': False,
+                     'refcount-bits': 16, 'corrupt': False }
+    human_compare = [ 'compat: 1.1', 'lazy refcounts: false',
+                      'refcount bits: 16', 'corrupt: false' ]
 
 class TestQCow3Lazy(TestQemuImgInfo):
     '''Testing a qcow2 version 3 image with lazy refcounts enabled'''
     img_options = 'compat=1.1,lazy_refcounts=on'
-    json_compare = { 'compat': '1.1', 'lazy-refcounts': True, 'corrupt': False }
-    human_compare = [ 'compat: 1.1', 'lazy refcounts: true', 'corrupt: false' ]
+    json_compare = { 'compat': '1.1', 'lazy-refcounts': True,
+                     'refcount-bits': 16, 'corrupt': False }
+    human_compare = [ 'compat: 1.1', 'lazy refcounts: true',
+                      'refcount bits: 16', 'corrupt: false' ]
 
 class TestQCow3NotLazyQMP(TestQMP):
     '''Testing a qcow2 version 3 image with lazy refcounts disabled, opening
        with lazy refcounts enabled'''
     img_options = 'compat=1.1,lazy_refcounts=off'
     qemu_options = 'lazy-refcounts=on'
-    compare = { 'compat': '1.1', 'lazy-refcounts': False, 'corrupt': False }
+    compare = { 'compat': '1.1', 'lazy-refcounts': False,
+                'refcount-bits': 16, 'corrupt': False }
+
 
 class TestQCow3LazyQMP(TestQMP):
     '''Testing a qcow2 version 3 image with lazy refcounts enabled, opening
        with lazy refcounts disabled'''
     img_options = 'compat=1.1,lazy_refcounts=on'
     qemu_options = 'lazy-refcounts=off'
-    compare = { 'compat': '1.1', 'lazy-refcounts': True, 'corrupt': False }
+    compare = { 'compat': '1.1', 'lazy-refcounts': True,
+                'refcount-bits': 16, 'corrupt': False }
 
 TestImageInfoSpecific = None
 TestQemuImgInfo = None
diff --git a/tests/qemu-iotests/066 b/tests/qemu-iotests/066
index 1c2452b..364166d 100755
--- a/tests/qemu-iotests/066
+++ b/tests/qemu-iotests/066
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/066.out b/tests/qemu-iotests/066.out
index 9139780..7bc9a10 100644
--- a/tests/qemu-iotests/066.out
+++ b/tests/qemu-iotests/066.out
@@ -2,7 +2,7 @@
 
 === Testing snapshotting an image with zero clusters ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 262144/262144 bytes at offset 0
 256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 262144/262144 bytes at offset 0
diff --git a/tests/qemu-iotests/067 b/tests/qemu-iotests/067
index d025192..c1df48e 100755
--- a/tests/qemu-iotests/067
+++ b/tests/qemu-iotests/067
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 # get standard environment, filters and checks
@@ -35,17 +34,30 @@
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# Because anything other than 16 would change the output of query-block
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 function do_run_qemu()
 {
     echo Testing: "$@"
-    $QEMU -nographic -qmp stdio -serial none "$@"
+    $QEMU -nographic -qmp-pretty stdio -serial none "$@"
     echo
 }
 
+# Remove QMP events from (pretty-printed) output. Doesn't handle
+# nested dicts correctly, but we don't get any of those in this test.
+_filter_qmp_events()
+{
+    tr '\n' '\t' | sed -e \
+	's/{\s*"timestamp":\s*{[^}]*},\s*"event":[^,}]*\(,\s*"data":\s*{[^}]*}\)\?\s*}\s*//g' \
+	| tr '\t' '\n'
+}
+
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu \
+                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g' \
+                          | _filter_generated_node_ids | _filter_qmp_events
 }
 
 size=128M
@@ -56,7 +68,7 @@
 echo === -drive/-device and device_del ===
 echo
 
-run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0 <<EOF
+run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk,drive=disk,id=virtio0 <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "query-block" }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
@@ -73,7 +85,7 @@
 { "execute": "qmp_capabilities" }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
@@ -91,7 +103,7 @@
   "arguments": { "command-line": "drive_add 0 file=$TEST_IMG,format=$IMGFMT,if=none,id=disk" } }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
@@ -119,7 +131,7 @@
   }
 { "execute": "query-block" }
 { "execute": "device_add",
-   "arguments": { "driver": "virtio-blk-pci", "drive": "disk",
+   "arguments": { "driver": "virtio-blk", "drive": "disk",
                   "id": "virtio0" } }
 { "execute": "device_del", "arguments": { "id": "virtio0" } }
 { "execute": "system_reset" }
diff --git a/tests/qemu-iotests/067.out b/tests/qemu-iotests/067.out
index 0f72dcf..7e25a49 100644
--- a/tests/qemu-iotests/067.out
+++ b/tests/qemu-iotests/067.out
@@ -1,80 +1,373 @@
 QA output created by 067
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 === -drive/-device and device_del ===
 
-Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0
-QMP_VERSION
-{"return": {}}
-{"return": [{"io-status": "ok", "device": "disk", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
-
+Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk,drive=disk,id=virtio0
+{
+    QMP_VERSION
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+        {
+            "io-status": "ok",
+            "device": "disk",
+            "locked": false,
+            "removable": false,
+            "inserted": {
+                "iops_rd": 0,
+                "detect_zeroes": "off",
+                "image": {
+                    "virtual-size": 134217728,
+                    "filename": "TEST_DIR/t.qcow2",
+                    "cluster-size": 65536,
+                    "format": "qcow2",
+                    "actual-size": SIZE,
+                    "format-specific": {
+                        "type": "qcow2",
+                        "data": {
+                            "compat": "1.1",
+                            "lazy-refcounts": false,
+                            "refcount-bits": 16,
+                            "corrupt": false
+                        }
+                    },
+                    "dirty-flag": false
+                },
+                "iops_wr": 0,
+                "ro": false,
+                "node-name": "NODE_NAME",
+                "backing_file_depth": 0,
+                "drv": "qcow2",
+                "iops": 0,
+                "bps_wr": 0,
+                "write_threshold": 0,
+                "encrypted": false,
+                "bps": 0,
+                "bps_rd": 0,
+                "cache": {
+                    "no-flush": false,
+                    "direct": false,
+                    "writeback": true
+                },
+                "file": "TEST_DIR/t.qcow2",
+                "encryption_key_missing": false
+            },
+            "type": "unknown"
+        }
+    ]
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+    ]
+}
+{
+    "return": {
+    }
+}
 
 === -drive/device_add and device_del ===
 
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
-QMP_VERSION
-{"return": {}}
-{"return": [{"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"return": {}}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
-
+{
+    QMP_VERSION
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+        {
+            "device": "disk",
+            "locked": false,
+            "removable": true,
+            "inserted": {
+                "iops_rd": 0,
+                "detect_zeroes": "off",
+                "image": {
+                    "virtual-size": 134217728,
+                    "filename": "TEST_DIR/t.qcow2",
+                    "cluster-size": 65536,
+                    "format": "qcow2",
+                    "actual-size": SIZE,
+                    "format-specific": {
+                        "type": "qcow2",
+                        "data": {
+                            "compat": "1.1",
+                            "lazy-refcounts": false,
+                            "refcount-bits": 16,
+                            "corrupt": false
+                        }
+                    },
+                    "dirty-flag": false
+                },
+                "iops_wr": 0,
+                "ro": false,
+                "node-name": "NODE_NAME",
+                "backing_file_depth": 0,
+                "drv": "qcow2",
+                "iops": 0,
+                "bps_wr": 0,
+                "write_threshold": 0,
+                "encrypted": false,
+                "bps": 0,
+                "bps_rd": 0,
+                "cache": {
+                    "no-flush": false,
+                    "direct": false,
+                    "writeback": true
+                },
+                "file": "TEST_DIR/t.qcow2",
+                "encryption_key_missing": false
+            },
+            "type": "unknown"
+        }
+    ]
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+    ]
+}
+{
+    "return": {
+    }
+}
 
 === drive_add/device_add and device_del ===
 
 Testing:
-QMP_VERSION
-{"return": {}}
-{"return": "OK\r\n"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"return": {}}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
-
+{
+    QMP_VERSION
+}
+{
+    "return": {
+    }
+}
+{
+    "return": "OK\r\n"
+}
+{
+    "return": [
+        {
+            "device": "disk",
+            "locked": false,
+            "removable": true,
+            "inserted": {
+                "iops_rd": 0,
+                "detect_zeroes": "off",
+                "image": {
+                    "virtual-size": 134217728,
+                    "filename": "TEST_DIR/t.qcow2",
+                    "cluster-size": 65536,
+                    "format": "qcow2",
+                    "actual-size": SIZE,
+                    "format-specific": {
+                        "type": "qcow2",
+                        "data": {
+                            "compat": "1.1",
+                            "lazy-refcounts": false,
+                            "refcount-bits": 16,
+                            "corrupt": false
+                        }
+                    },
+                    "dirty-flag": false
+                },
+                "iops_wr": 0,
+                "ro": false,
+                "node-name": "NODE_NAME",
+                "backing_file_depth": 0,
+                "drv": "qcow2",
+                "iops": 0,
+                "bps_wr": 0,
+                "write_threshold": 0,
+                "encrypted": false,
+                "bps": 0,
+                "bps_rd": 0,
+                "cache": {
+                    "no-flush": false,
+                    "direct": false,
+                    "writeback": true
+                },
+                "file": "TEST_DIR/t.qcow2",
+                "encryption_key_missing": false
+            },
+            "type": "unknown"
+        }
+    ]
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+    ]
+}
+{
+    "return": {
+    }
+}
 
 === blockdev_add/device_add and device_del ===
 
 Testing:
-QMP_VERSION
-{"return": {}}
-{"return": {}}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"return": {}}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false, "corrupt": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
-{"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
-
+{
+    QMP_VERSION
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+        {
+            "device": "disk",
+            "locked": false,
+            "removable": true,
+            "inserted": {
+                "iops_rd": 0,
+                "detect_zeroes": "off",
+                "image": {
+                    "virtual-size": 134217728,
+                    "filename": "TEST_DIR/t.qcow2",
+                    "cluster-size": 65536,
+                    "format": "qcow2",
+                    "actual-size": SIZE,
+                    "format-specific": {
+                        "type": "qcow2",
+                        "data": {
+                            "compat": "1.1",
+                            "lazy-refcounts": false,
+                            "refcount-bits": 16,
+                            "corrupt": false
+                        }
+                    },
+                    "dirty-flag": false
+                },
+                "iops_wr": 0,
+                "ro": false,
+                "node-name": "NODE_NAME",
+                "backing_file_depth": 0,
+                "drv": "qcow2",
+                "iops": 0,
+                "bps_wr": 0,
+                "write_threshold": 0,
+                "encrypted": false,
+                "bps": 0,
+                "bps_rd": 0,
+                "cache": {
+                    "no-flush": false,
+                    "direct": false,
+                    "writeback": true
+                },
+                "file": "TEST_DIR/t.qcow2",
+                "encryption_key_missing": false
+            },
+            "type": "unknown"
+        }
+    ]
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": {
+    }
+}
+{
+    "return": [
+        {
+            "io-status": "ok",
+            "device": "disk",
+            "locked": false,
+            "removable": true,
+            "inserted": {
+                "iops_rd": 0,
+                "detect_zeroes": "off",
+                "image": {
+                    "virtual-size": 134217728,
+                    "filename": "TEST_DIR/t.qcow2",
+                    "cluster-size": 65536,
+                    "format": "qcow2",
+                    "actual-size": SIZE,
+                    "format-specific": {
+                        "type": "qcow2",
+                        "data": {
+                            "compat": "1.1",
+                            "lazy-refcounts": false,
+                            "refcount-bits": 16,
+                            "corrupt": false
+                        }
+                    },
+                    "dirty-flag": false
+                },
+                "iops_wr": 0,
+                "ro": false,
+                "node-name": "NODE_NAME",
+                "backing_file_depth": 0,
+                "drv": "qcow2",
+                "iops": 0,
+                "bps_wr": 0,
+                "write_threshold": 0,
+                "encrypted": false,
+                "bps": 0,
+                "bps_rd": 0,
+                "cache": {
+                    "no-flush": false,
+                    "direct": false,
+                    "writeback": true
+                },
+                "file": "TEST_DIR/t.qcow2",
+                "encryption_key_missing": false
+            },
+            "type": "unknown"
+        }
+    ]
+}
+{
+    "return": {
+    }
+}
 *** done
diff --git a/tests/qemu-iotests/068 b/tests/qemu-iotests/068
index b72e555..68f6e82 100755
--- a/tests/qemu-iotests/068
+++ b/tests/qemu-iotests/068
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -50,13 +49,23 @@
 echo "=== Saving and reloading a VM state to/from a qcow2 image ==="
 echo
 _make_test_img $IMG_SIZE
+
+case "$QEMU_DEFAULT_MACHINE" in
+  s390-ccw-virtio)
+      platform_parm="-no-shutdown"
+      ;;
+  *)
+      platform_parm=""
+      ;;
+esac
+
 # Give qemu some time to boot before saving the VM state
 bash -c 'sleep 1; echo -e "savevm 0\nquit"' |\
-    $QEMU -nographic -monitor stdio -serial none -hda "$TEST_IMG" |\
+    $QEMU $platform_parm -nographic -monitor stdio -serial none -hda "$TEST_IMG" |\
     _filter_qemu
 # Now try to continue from that VM state (this should just work)
 echo quit |\
-    $QEMU -nographic -monitor stdio -serial none -hda "$TEST_IMG" -loadvm 0 |\
+    $QEMU $platform_parm -nographic -monitor stdio -serial none -hda "$TEST_IMG" -loadvm 0 |\
     _filter_qemu
 
 # success, all done
diff --git a/tests/qemu-iotests/068.out b/tests/qemu-iotests/068.out
index abe35a9..84f19b4 100644
--- a/tests/qemu-iotests/068.out
+++ b/tests/qemu-iotests/068.out
@@ -2,7 +2,7 @@
 
 === Saving and reloading a VM state to/from a qcow2 image ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072
 QEMU X.Y.Z monitor - type 'help' for more information
 (qemu) s[K[Dsa[K[D[Dsav[K[D[D[Dsave[K[D[D[D[Dsavev[K[D[D[D[D[Dsavevm[K[D[D[D[D[D[Dsavevm [K[D[D[D[D[D[D[Dsavevm 0[K
 (qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
diff --git a/tests/qemu-iotests/069 b/tests/qemu-iotests/069
index ce9e054..96e55ef 100755
--- a/tests/qemu-iotests/069
+++ b/tests/qemu-iotests/069
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/069.out b/tests/qemu-iotests/069.out
index b48306d..f975856 100644
--- a/tests/qemu-iotests/069.out
+++ b/tests/qemu-iotests/069.out
@@ -2,7 +2,7 @@
 
 === Creating an image with a backing file and deleting that file ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=131072 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 backing_file='TEST_DIR/t.IMGFMT.base' 
-qemu-io: can't open device TEST_DIR/t.IMGFMT: Could not open backing file: Could not open 'TEST_DIR/t.IMGFMT.base': No such file or directory
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=131072
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 backing_file=TEST_DIR/t.IMGFMT.base
+can't open device TEST_DIR/t.IMGFMT: Could not open backing file: Could not open 'TEST_DIR/t.IMGFMT.base': No such file or directory
 *** done
diff --git a/tests/qemu-iotests/070 b/tests/qemu-iotests/070
index d649ddf..8d08d74 100755
--- a/tests/qemu-iotests/070
+++ b/tests/qemu-iotests/070
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/070.out b/tests/qemu-iotests/070.out
index ca74383..131a5b1 100644
--- a/tests/qemu-iotests/070.out
+++ b/tests/qemu-iotests/070.out
@@ -1,8 +1,9 @@
 QA output created by 070
 
 === Verify open image read-only fails, due to dirty log ===
-qemu-io: can't open device TEST_DIR/iotest-dirtylog-10G-4M.vhdx: VHDX image file 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx' opened read-only, but contains a log that needs to be replayed.  To replay the log, execute:
- qemu-img check -r all 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx': Operation not permitted
+can't open device TEST_DIR/iotest-dirtylog-10G-4M.vhdx: VHDX image file 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx' opened read-only, but contains a log that needs to be replayed
+To replay the log, run:
+qemu-img check -r all 'TEST_DIR/iotest-dirtylog-10G-4M.vhdx'
  no file open, try 'help open'
 === Verify open image replays log  ===
 read 18874368/18874368 bytes at offset 0
diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071
index 3924e51..bdfd91f 100755
--- a/tests/qemu-iotests/071
+++ b/tests/qemu-iotests/071
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -51,7 +50,7 @@
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu_io
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp | _filter_qemu_io
 }
 
 IMG_SIZE=64M
@@ -63,12 +62,12 @@
 TEST_IMG="$TEST_IMG.base" IMGOPTS="" IMGFMT="raw" _make_test_img $IMG_SIZE |\
     _filter_imgfmt
 _make_test_img $IMG_SIZE
-$QEMU_IO -c "open -o file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
+$QEMU_IO -c "open -o driver=raw,file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
          -c 'read 0 512' -c 'write -P 42 0x38000 512' -c 'read -P 42 0x38000 512' | _filter_qemu_io
 
 $QEMU_IO -c 'write -P 42 0 512' "$TEST_IMG" | _filter_qemu_io
 
-$QEMU_IO -c "open -o file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
+$QEMU_IO -c "open -o driver=raw,file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
          -c 'read -P 42 0 512' | _filter_qemu_io
 
 echo
@@ -78,12 +77,12 @@
 TEST_IMG="$TEST_IMG.base" IMGOPTS="" IMGFMT="raw" _make_test_img $IMG_SIZE |\
     _filter_imgfmt
 _make_test_img $IMG_SIZE
-$QEMU_IO -c "open -o file.driver=blkverify,file.raw.filename=$TEST_IMG.base,file.test.driver=$IMGFMT,file.test.file.filename=$TEST_IMG" \
+$QEMU_IO -c "open -o driver=raw,file.driver=blkverify,file.raw.filename=$TEST_IMG.base,file.test.driver=$IMGFMT,file.test.file.filename=$TEST_IMG" \
          -c 'read 0 512' -c 'write -P 42 0x38000 512' -c 'read -P 42 0x38000 512' | _filter_qemu_io
 
 $QEMU_IO -c 'write -P 42 0 512' "$TEST_IMG" | _filter_qemu_io
 
-$QEMU_IO -c "open -o file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
+$QEMU_IO -c "open -o driver=raw,file.driver=blkverify,file.raw.filename=$TEST_IMG.base $TEST_IMG" \
          -c 'read -P 42 0 512' | _filter_qemu_io
 
 echo
@@ -104,11 +103,20 @@
 echo "=== Testing blkdebug on existing block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "$IMGFMT",
             "id": "drive0-debug",
             "file": {
@@ -133,20 +141,29 @@
 echo "=== Testing blkverify on existing block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=$IMGFMT,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "$IMGFMT",
+            "file": {
+                "driver": "file",
+                "filename": "$TEST_IMG"
+            }
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "blkverify",
             "id": "drive0-verify",
             "test": "drive0",
             "raw": {
-                "driver": "raw",
-                "file": {
-                    "driver": "file",
-                    "filename": "$TEST_IMG.base"
-                }
+                "driver": "file",
+                "filename": "$TEST_IMG.base"
             }
         }
     }
@@ -163,11 +180,20 @@
 echo "=== Testing blkverify on existing raw block device ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG.base,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG.base"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "blkverify",
             "id": "drive0-verify",
             "test": {
@@ -193,11 +219,20 @@
 echo "=== Testing blkdebug's set-state through QMP ==="
 echo
 
-run_qemu -drive "file=$TEST_IMG,format=raw,if=none,id=drive0" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive0",
+            "driver": "file",
+            "filename": "$TEST_IMG"
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "$IMGFMT",
             "id": "drive0-debug",
             "file": {
diff --git a/tests/qemu-iotests/071.out b/tests/qemu-iotests/071.out
index 5f840a9..8c6851e 100644
--- a/tests/qemu-iotests/071.out
+++ b/tests/qemu-iotests/071.out
@@ -2,8 +2,8 @@
 
 === Testing blkverify through filename ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 229376
@@ -12,12 +12,12 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-blkverify: read sector_num=0 nb_sectors=4 contents mismatch in sector 0
+blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 === Testing blkverify through file blockref ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 229376
@@ -26,7 +26,7 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-blkverify: read sector_num=0 nb_sectors=4 contents mismatch in sector 0
+blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 === Testing blkdebug through filename ===
 
@@ -38,42 +38,44 @@
 
 === Testing blkdebug on existing block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 read failed: Input/output error
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Testing blkverify on existing block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=IMGFMT,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 
 === Testing blkverify on existing raw block device ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT.base,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 blkverify: read sector_num=0 nb_sectors=1 contents mismatch in sector 0
 
 
 === Testing blkdebug's set-state through QMP ===
 
-Testing: -drive file=TEST_DIR/t.IMGFMT,format=raw,if=none,id=drive0
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
+{"return": {}}
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 {"return": ""}
@@ -84,7 +86,5 @@
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 *** done
diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072
index e4a723d..aa027c7 100755
--- a/tests/qemu-iotests/072
+++ b/tests/qemu-iotests/072
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/072.out b/tests/qemu-iotests/072.out
index efe577c..fe949d4 100644
--- a/tests/qemu-iotests/072.out
+++ b/tests/qemu-iotests/072.out
@@ -2,7 +2,7 @@
 
 === Testing nested image formats ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 512
diff --git a/tests/qemu-iotests/073 b/tests/qemu-iotests/073
index 392db54..ad37a61 100755
--- a/tests/qemu-iotests/073
+++ b/tests/qemu-iotests/073
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/073.out b/tests/qemu-iotests/073.out
index c9b0076..de54524 100644
--- a/tests/qemu-iotests/073.out
+++ b/tests/qemu-iotests/073.out
@@ -1,8 +1,8 @@
 QA output created by 073
 
 == creating backing file ==
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 134217728/134217728 bytes at offset 0
 128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
diff --git a/tests/qemu-iotests/075 b/tests/qemu-iotests/075
index 6117660..770d51c 100755
--- a/tests/qemu-iotests/075
+++ b/tests/qemu-iotests/075
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/075.out b/tests/qemu-iotests/075.out
index 5f1d6c1..87beae4 100644
--- a/tests/qemu-iotests/075.out
+++ b/tests/qemu-iotests/075.out
@@ -9,30 +9,30 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == block_size must be a multiple of 512 ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: block_size 513 must be a multiple of 512
+can't open device TEST_DIR/simple-pattern.cloop: block_size 513 must be a multiple of 512
 no file open, try 'help open'
 
 == block_size cannot be zero ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: block_size cannot be zero
+can't open device TEST_DIR/simple-pattern.cloop: block_size cannot be zero
 no file open, try 'help open'
 
 == huge block_size ===
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: block_size 4294966784 must be 64 MB or less
+can't open device TEST_DIR/simple-pattern.cloop: block_size 4294966784 must be 64 MB or less
 no file open, try 'help open'
 
 == offsets_size overflow ===
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: n_blocks 4294967295 must be 536870911 or less
+can't open device TEST_DIR/simple-pattern.cloop: n_blocks 4294967295 must be 536870911 or less
 no file open, try 'help open'
 
 == refuse images that require too many offsets ===
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: image requires too many offsets, try increasing block size
+can't open device TEST_DIR/simple-pattern.cloop: image requires too many offsets, try increasing block size
 no file open, try 'help open'
 
 == refuse images with non-monotonically increasing offsets ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: offsets not monotonically increasing at index 1, image file is corrupt
+can't open device TEST_DIR/simple-pattern.cloop: offsets not monotonically increasing at index 1, image file is corrupt
 no file open, try 'help open'
 
 == refuse images with invalid compressed block size ==
-qemu-io: can't open device TEST_DIR/simple-pattern.cloop: invalid compressed block size at index 1, image file is corrupt
+can't open device TEST_DIR/simple-pattern.cloop: invalid compressed block size at index 1, image file is corrupt
 no file open, try 'help open'
 *** done
diff --git a/tests/qemu-iotests/076 b/tests/qemu-iotests/076
index ed2be35..ef9e6a4 100755
--- a/tests/qemu-iotests/076
+++ b/tests/qemu-iotests/076
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -49,31 +48,36 @@
 echo
 echo "== Read from a valid v1 image =="
 _use_sample_img parallels-v1.bz2
-{ $QEMU_IO -c "read -P 0x11 0 64k" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read -P 0x11 0 64k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Negative catalog size =="
 _use_sample_img parallels-v1.bz2
 poke_file "$TEST_IMG" "$catalog_entries_offset" "\xff\xff\xff\xff"
-{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read 0 512" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Overflow in catalog allocation =="
 _use_sample_img parallels-v1.bz2
 poke_file "$TEST_IMG" "$nb_sectors_offset" "\xff\xff\xff\xff"
 poke_file "$TEST_IMG" "$catalog_entries_offset" "\x01\x00\x00\x40"
-{ $QEMU_IO -c "read 64M 64M" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read 64M 64M" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Zero sectors per track =="
 _use_sample_img parallels-v1.bz2
 poke_file "$TEST_IMG" "$tracks_offset" "\x00\x00\x00\x00"
-{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read 0 512" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 echo
 echo "== Read from a valid v2 image =="
 _use_sample_img parallels-v2.bz2
-{ $QEMU_IO -c "read -P 0x11 0 64k" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read -P 0x11 0 64k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "write -P 0x21 1024k 1k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "write -P 0x22 1025k 1k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read -P 0x21 1024k 1k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read -P 0x22 1025k 1k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+{ $QEMU_IO -c "read -P 0 1026k 62k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/076.out b/tests/qemu-iotests/076.out
index 32ade08..72645b2 100644
--- a/tests/qemu-iotests/076.out
+++ b/tests/qemu-iotests/076.out
@@ -5,18 +5,28 @@
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Negative catalog size ==
-qemu-io: can't open device TEST_DIR/parallels-v1: Catalog too large
+can't open device TEST_DIR/parallels-v1: Catalog too large
 no file open, try 'help open'
 
 == Overflow in catalog allocation ==
-qemu-io: can't open device TEST_DIR/parallels-v1: Catalog too large
+can't open device TEST_DIR/parallels-v1: Catalog too large
 no file open, try 'help open'
 
 == Zero sectors per track ==
-qemu-io: can't open device TEST_DIR/parallels-v1: Invalid image: Zero sectors per track
+can't open device TEST_DIR/parallels-v1: Invalid image: Zero sectors per track
 no file open, try 'help open'
 
 == Read from a valid v2 image ==
 read 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 1048576
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 1049600
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 1048576
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 1049600
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 63488/63488 bytes at offset 1050624
+62 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 *** done
diff --git a/tests/qemu-iotests/077 b/tests/qemu-iotests/077
index 4dd1bdd..d2d2a2d 100755
--- a/tests/qemu-iotests/077
+++ b/tests/qemu-iotests/077
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -52,7 +51,7 @@
 
 function test_io()
 {
-echo "open -o file.align=4k blkdebug::$TEST_IMG"
+echo "open -o driver=$IMGFMT,file.align=4k blkdebug::$TEST_IMG"
 # A simple RMW request
 cat  <<EOF
 aio_write -P 10 0x200 0x200
@@ -61,9 +60,9 @@
 
 # Sequential RMW requests on the same physical sector
 off=0x1000
-for ev in "head" "after_head" "tail" "after_tail"; do
+for ev in "head" "after_head"; do
 cat  <<EOF
-break pwritev_rmw.$ev A
+break pwritev_rmw_$ev A
 aio_write -P 10 $((off + 0x200)) 0x200
 wait_break A
 aio_write -P 11 $((off + 0x400)) 0x200
@@ -76,7 +75,7 @@
 
 # Chained dependencies
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0x5000 0x200
 wait_break A
 aio_write -P 11 0x5200 0x200
@@ -93,10 +92,10 @@
 
 # Overlapping multiple requests
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0x6000 0x200
 wait_break A
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0x7e00 0x200
 wait_break B
 aio_write -P 11 0x6800 0x1000
@@ -107,10 +106,10 @@
 EOF
 
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0x8000 0x200
 wait_break A
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0x9e00 0x200
 wait_break B
 aio_write -P 11 0x8800 0x1000
@@ -121,11 +120,11 @@
 EOF
 
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0xa000 0x200
 wait_break A
 aio_write -P 11 0xa800 0x1000
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0xbe00 0x200
 wait_break B
 resume A
@@ -135,11 +134,11 @@
 EOF
 
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0xc000 0x200
 wait_break A
 aio_write -P 11 0xc800 0x1000
-break pwritev_rmw.after_head B
+break pwritev_rmw_after_head B
 aio_write -P 10 0xde00 0x200
 wait_break B
 resume B
@@ -150,7 +149,7 @@
 
 # Only RMW for the tail part
 cat  <<EOF
-break pwritev_rmw.after_tail A
+break pwritev_rmw_after_tail A
 aio_write -P 10 0xe000 0x1800
 wait_break A
 aio_write -P 11 0xf000 0xc00
@@ -163,7 +162,7 @@
 break pwritev A
 aio_write -P 10 0x10000 0x800
 wait_break A
-break pwritev_rmw.after_tail B
+break pwritev_rmw_after_tail B
 aio_write -P 11 0x10000 0x400
 break pwritev_done C
 resume A
@@ -212,16 +211,6 @@
     echo read -P 11 0x2400 0x200
     echo read -P 0  0x2600 0xa00
 
-    echo read -P 0  0x3000 0x200
-    echo read -P 10 0x3200 0x200
-    echo read -P 11 0x3400 0x200
-    echo read -P 0  0x3600 0xa00
-
-    echo read -P 0  0x4000 0x200
-    echo read -P 10 0x4200 0x200
-    echo read -P 11 0x4400 0x200
-    echo read -P 0  0x4600 0xa00
-
     # Chained dependencies
     echo read -P 10 0x5000 0x200
     echo read -P 11 0x5200 0x200
diff --git a/tests/qemu-iotests/077.out b/tests/qemu-iotests/077.out
index ab61234..16f951f 100644
--- a/tests/qemu-iotests/077.out
+++ b/tests/qemu-iotests/077.out
@@ -1,5 +1,5 @@
 QA output created by 077
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
 == Some concurrent requests involving RMW ==
 wrote XXX/XXX bytes at offset XXX
@@ -19,16 +19,6 @@
 XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote XXX/XXX bytes at offset XXX
 XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-blkdebug: Resuming request 'A'
-wrote XXX/XXX bytes at offset XXX
-XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-wrote XXX/XXX bytes at offset XXX
-XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-blkdebug: Resuming request 'A'
-wrote XXX/XXX bytes at offset XXX
-XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-wrote XXX/XXX bytes at offset XXX
-XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote XXX/XXX bytes at offset XXX
 XXX bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote XXX/XXX bytes at offset XXX
@@ -114,22 +104,6 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 2560/2560 bytes at offset 9728
 2.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 12288
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 12800
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 13312
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 2560/2560 bytes at offset 13824
-2.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 16384
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 16896
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 512/512 bytes at offset 17408
-512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 2560/2560 bytes at offset 17920
-2.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 512/512 bytes at offset 20480
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 512/512 bytes at offset 20992
diff --git a/tests/qemu-iotests/078 b/tests/qemu-iotests/078
index 7be2c3f..f333e9a 100755
--- a/tests/qemu-iotests/078
+++ b/tests/qemu-iotests/078
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/078.out b/tests/qemu-iotests/078.out
index ca18d2e..42b8a83 100644
--- a/tests/qemu-iotests/078.out
+++ b/tests/qemu-iotests/078.out
@@ -5,24 +5,24 @@
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Negative catalog size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too large
+can't open device TEST_DIR/empty.bochs: Catalog size is too large
 no file open, try 'help open'
 
 == Overflow for catalog size * sizeof(uint32_t) ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too large
+can't open device TEST_DIR/empty.bochs: Catalog size is too large
 no file open, try 'help open'
 
 == Too small catalog bitmap for image size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
+can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
+can't open device TEST_DIR/empty.bochs: Catalog size is too small for this disk size
 no file open, try 'help open'
 
 == Negative extent size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Extent size 2147483648 is too large
+can't open device TEST_DIR/empty.bochs: Extent size 2147483648 is too large
 no file open, try 'help open'
 
 == Zero extent size ==
-qemu-io: can't open device TEST_DIR/empty.bochs: Extent size must be at least 512
+can't open device TEST_DIR/empty.bochs: Extent size must be at least 512
 no file open, try 'help open'
 *** done
diff --git a/tests/qemu-iotests/079 b/tests/qemu-iotests/079
index 6613cfb..b2e3f74 100755
--- a/tests/qemu-iotests/079
+++ b/tests/qemu-iotests/079
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -42,19 +41,13 @@
 _supported_proto file nfs
 _supported_os Linux
 
-function test_qemu_img()
-{
-    echo qemu-img "$@" | _filter_testdir
-    $QEMU_IMG "$@" 2>&1 | _filter_testdir
-    echo
-}
-
 echo "=== Check option preallocation and cluster_size ==="
 echo
 cluster_sizes="16384 32768 65536 131072 262144 524288 1048576 2097152 4194304"
 
 for s in $cluster_sizes; do
-    test_qemu_img create -f $IMGFMT -o preallocation=metadata,cluster_size=$s "$TEST_IMG" 4G
+    IMGOPTS=$(_optstr_add "$IMGOPTS" "preallocation=metadata,cluster_size=$s") \
+        _make_test_img 4G
 done
 
 # success, all done
diff --git a/tests/qemu-iotests/079.out b/tests/qemu-iotests/079.out
index ef4b8c9..aab922f 100644
--- a/tests/qemu-iotests/079.out
+++ b/tests/qemu-iotests/079.out
@@ -1,32 +1,14 @@
 QA output created by 079
 === Check option preallocation and cluster_size ===
 
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=16384 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=16384 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=32768 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=32768 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=65536 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=65536 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=131072 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=131072 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=262144 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=262144 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=524288 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=524288 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=1048576 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=1048576 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=2097152 TEST_DIR/t.qcow2 4G
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=2097152 preallocation='metadata' lazy_refcounts=off
-
-qemu-img create -f qcow2 -o preallocation=metadata,cluster_size=4194304 TEST_DIR/t.qcow2 4G
-qemu-img: TEST_DIR/t.qcow2: Cluster size must be a power of two between 512 and 2048k
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=4294967296 encryption=off cluster_size=4194304 preallocation='metadata' lazy_refcounts=off
-
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
+qemu-img: TEST_DIR/t.IMGFMT: Cluster size must be a power of two between 512 and 2048k
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4294967296 preallocation=metadata
 *** done
diff --git a/tests/qemu-iotests/080 b/tests/qemu-iotests/080
index 9de337c..55044c7 100755
--- a/tests/qemu-iotests/080
+++ b/tests/qemu-iotests/080
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -42,6 +41,8 @@
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# Internal snapshots are (currently) impossible with refcount_bits=1
+_unsupported_imgopts 'refcount_bits=1[^0-9]'
 
 header_size=104
 
@@ -78,6 +79,8 @@
 poke_file "$TEST_IMG" "$offset_ext_magic" "\x12\x34\x56\x78"
 poke_file "$TEST_IMG" "$offset_ext_size" "\x7f\xff\xff\xff"
 { $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_backing_file_offset" "\x00\x00\x00\x00\x00\x00\x00\x$(printf %x $offset_ext_size)"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
 poke_file "$TEST_IMG" "$offset_backing_file_offset" "\x00\x00\x00\x00\x00\x00\x00\x00"
 { $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
 
diff --git a/tests/qemu-iotests/080.out b/tests/qemu-iotests/080.out
index f7a943c..0daac48 100644
--- a/tests/qemu-iotests/080.out
+++ b/tests/qemu-iotests/080.out
@@ -1,82 +1,84 @@
 QA output created by 080
 
 == Huge header size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
+can't open device TEST_DIR/t.qcow2: qcow2 header exceeds cluster size
 no file open, try 'help open'
 
 == Huge unknown header extension ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid backing file offset
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Invalid backing file offset
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Header extension too large
+can't open device TEST_DIR/t.qcow2: Header extension too large
+no file open, try 'help open'
+can't open device TEST_DIR/t.qcow2: Header extension too large
 no file open, try 'help open'
 
 == Huge refcount table size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Reference count table too large
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Reference count table too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Reference count table too large
+can't open device TEST_DIR/t.qcow2: Reference count table too large
 no file open, try 'help open'
 
 == Misaligned refcount table ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
 no file open, try 'help open'
 
 == Huge refcount offset ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Invalid reference count table offset
 no file open, try 'help open'
 
 == Invalid snapshot table ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Too many snapshots
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Too many snapshots
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Too many snapshots
+can't open device TEST_DIR/t.qcow2: Too many snapshots
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
+can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
+can't open device TEST_DIR/t.qcow2: Invalid snapshot table offset
 no file open, try 'help open'
 
 == Hitting snapshot table size limit ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 qemu-img: Could not create snapshot 'test': -27 (File too large)
 read 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == Invalid L1 table ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Active L1 table too large
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Active L1 table too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Active L1 table too large
+can't open device TEST_DIR/t.qcow2: Active L1 table too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
+can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
+can't open device TEST_DIR/t.qcow2: Invalid L1 table offset
 no file open, try 'help open'
 
 == Invalid L1 table (with internal snapshot in the image) ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 qemu-img: Could not open 'TEST_DIR/t.IMGFMT': L1 table is too small
 
 == Invalid backing file size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow2: Backing file name too long
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow2: Backing file name too long
 no file open, try 'help open'
 
 == Invalid L2 entry (huge physical offset) ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not create snapshot 'test': -27 (File too large)
 qemu-img: Could not create snapshot 'test': -11 (Resource temporarily unavailable)
 
 == Invalid snapshot L1 table ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Failed to load snapshot: Snapshot L1 table too large
diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081
index ed3c29e..d89fabc 100755
--- a/tests/qemu-iotests/081
+++ b/tests/qemu-iotests/081
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -53,15 +52,20 @@
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu_io
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp\
+                          | _filter_qemu_io | _filter_generated_node_ids
 }
 
 test_quorum=$($QEMU_IMG --help|grep quorum)
 [ "$test_quorum" = "" ] && _supported_fmt quorum
 
-quorum="file.driver=quorum,file.children.0.file.filename=$TEST_DIR/1.raw"
+quorum="driver=raw,file.driver=quorum,file.vote-threshold=2"
+quorum="$quorum,file.children.0.file.filename=$TEST_DIR/1.raw"
 quorum="$quorum,file.children.1.file.filename=$TEST_DIR/2.raw"
-quorum="$quorum,file.children.2.file.filename=$TEST_DIR/3.raw,file.vote-threshold=2"
+quorum="$quorum,file.children.2.file.filename=$TEST_DIR/3.raw"
+quorum="$quorum,file.children.0.driver=raw"
+quorum="$quorum,file.children.1.driver=raw"
+quorum="$quorum,file.children.2.driver=raw"
 
 echo
 echo "== creating quorum files =="
@@ -97,17 +101,29 @@
 echo
 echo "== checking mixed reference/option specification =="
 
-run_qemu -drive "file=$TEST_DIR/2.raw,format=$IMGFMT,if=none,id=drive2" <<EOF
+run_qemu <<EOF
 { "execute": "qmp_capabilities" }
 { "execute": "blockdev-add",
     "arguments": {
         "options": {
+            "node-name": "drive2",
+            "driver": "$IMGFMT",
+            "file": {
+                "driver": "file",
+                "filename": "$TEST_DIR/2.raw"
+            }
+        }
+    }
+}
+{ "execute": "blockdev-add",
+    "arguments": {
+        "options": {
             "driver": "quorum",
             "id": "drive0-quorum",
             "vote-threshold": 2,
             "children": [
                 {
-                    "driver": "raw",
+                    "driver": "$IMGFMT",
                     "file": {
                         "driver": "file",
                         "filename": "$TEST_DIR/1.raw"
@@ -115,7 +131,7 @@
                 },
                 "drive2",
                 {
-                    "driver": "raw",
+                    "driver": "$IMGFMT",
                     "file": {
                         "driver": "file",
                         "filename": "$TEST_DIR/3.raw"
diff --git a/tests/qemu-iotests/081.out b/tests/qemu-iotests/081.out
index 073515e..97df69d 100644
--- a/tests/qemu-iotests/081.out
+++ b/tests/qemu-iotests/081.out
@@ -1,9 +1,9 @@
 QA output created by 081
 
 == creating quorum files ==
-Formatting 'TEST_DIR/1.IMGFMT', fmt=IMGFMT size=10485760 
-Formatting 'TEST_DIR/2.IMGFMT', fmt=IMGFMT size=10485760 
-Formatting 'TEST_DIR/3.IMGFMT', fmt=IMGFMT size=10485760 
+Formatting 'TEST_DIR/1.IMGFMT', fmt=IMGFMT size=10485760
+Formatting 'TEST_DIR/2.IMGFMT', fmt=IMGFMT size=10485760
+Formatting 'TEST_DIR/3.IMGFMT', fmt=IMGFMT size=10485760
 
 == writing images ==
 wrote 10485760/10485760 bytes at offset 0
@@ -26,18 +26,17 @@
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == checking mixed reference/option specification ==
-Testing: -drive file=TEST_DIR/2.IMGFMT,format=IMGFMT,if=none,id=drive2
+Testing:
 QMP_VERSION
 {"return": {}}
 {"return": {}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "QUORUM_REPORT_BAD", "data": {"node-name": "", "sectors-count": 20480, "sector-num": 0}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "QUORUM_REPORT_BAD", "data": {"node-name": "drive2", "sectors-count": 20480, "sector-num": 0, "type": "read"}}
 read 10485760/10485760 bytes at offset 0
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 {"return": ""}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 == using quorum rewrite corrupted mode ==
@@ -55,5 +54,5 @@
 10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 == checking that quorum is broken ==
-qemu-io: can't open: Could not read image for determining its format: Input/output error
+read failed: Input/output error
 *** done
diff --git a/tests/qemu-iotests/082 b/tests/qemu-iotests/082
index e64de27..ad1d9fa 100755
--- a/tests/qemu-iotests/082
+++ b/tests/qemu-iotests/082
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -60,11 +59,11 @@
 
 # Multiple -o should be merged
 run_qemu_img create -f $IMGFMT -o cluster_size=4k -o lazy_refcounts=on "$TEST_IMG" $size
-run_qemu_img info "$TEST_IMG"
+_img_info --format-specific
 
 # If the same -o key is specified more than once, the last one wins
 run_qemu_img create -f $IMGFMT -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k "$TEST_IMG" $size
-run_qemu_img info "$TEST_IMG"
+_img_info --format-specific
 run_qemu_img create -f $IMGFMT -o cluster_size=4k,cluster_size=8k "$TEST_IMG" $size
 _img_info
 
@@ -114,11 +113,11 @@
 
 # Multiple -o should be merged
 run_qemu_img convert -O $IMGFMT -o cluster_size=4k -o lazy_refcounts=on "$TEST_IMG" "$TEST_IMG".base
-run_qemu_img info "$TEST_IMG".base
+TEST_IMG="${TEST_IMG}.base" _img_info --format-specific
 
 # If the same -o key is specified more than once, the last one wins
 run_qemu_img convert -O $IMGFMT -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k "$TEST_IMG" "$TEST_IMG".base
-run_qemu_img info "$TEST_IMG".base
+TEST_IMG="${TEST_IMG}.base" _img_info --format-specific
 run_qemu_img convert -O $IMGFMT -o cluster_size=4k,cluster_size=8k "$TEST_IMG" "$TEST_IMG".base
 TEST_IMG="${TEST_IMG}.base" _img_info
 
@@ -157,15 +156,15 @@
 
 # Last -f should win
 run_qemu_img amend -f foo -f $IMGFMT -o lazy_refcounts=on "$TEST_IMG"
-run_qemu_img info "$TEST_IMG"
+_img_info --format-specific
 
 # Multiple -o should be merged
 run_qemu_img amend -f $IMGFMT -o size=130M -o lazy_refcounts=off "$TEST_IMG"
-run_qemu_img info "$TEST_IMG"
+_img_info --format-specific
 
 # If the same -o key is specified more than once, the last one wins
 run_qemu_img amend -f $IMGFMT -o size=8M -o lazy_refcounts=on -o size=132M "$TEST_IMG"
-run_qemu_img info "$TEST_IMG"
+_img_info --format-specific
 run_qemu_img amend -f $IMGFMT -o size=4M,size=148M "$TEST_IMG"
 _img_info
 
diff --git a/tests/qemu-iotests/082.out b/tests/qemu-iotests/082.out
index 0a3ab5a..a952330 100644
--- a/tests/qemu-iotests/082.out
+++ b/tests/qemu-iotests/082.out
@@ -3,42 +3,38 @@
 === create: Options specified more than once ===
 
 Testing: create -f foo -f qcow2 TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 128M (134217728 bytes)
 cluster_size: 65536
 
 Testing: create -f qcow2 -o cluster_size=4k -o lazy_refcounts=on TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=4096 lazy_refcounts=on 
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=4096 lazy_refcounts=on refcount_bits=16
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 16K
 cluster_size: 4096
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    refcount bits: 16
     corrupt: false
 
 Testing: create -f qcow2 -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=8192 lazy_refcounts=on 
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=8192 lazy_refcounts=on refcount_bits=16
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 28K
 cluster_size: 8192
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    refcount bits: 16
     corrupt: false
 
 Testing: create -f qcow2 -o cluster_size=4k,cluster_size=8k TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=8192 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=8192 lazy_refcounts=off refcount_bits=16
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 128M (134217728 bytes)
@@ -56,6 +52,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o ? TEST_DIR/t.qcow2 128M
@@ -68,6 +65,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o cluster_size=4k,help TEST_DIR/t.qcow2 128M
@@ -80,6 +78,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o cluster_size=4k,? TEST_DIR/t.qcow2 128M
@@ -92,6 +91,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o help,cluster_size=4k TEST_DIR/t.qcow2 128M
@@ -104,6 +104,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o ?,cluster_size=4k TEST_DIR/t.qcow2 128M
@@ -116,6 +117,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o cluster_size=4k -o help TEST_DIR/t.qcow2 128M
@@ -128,6 +130,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o cluster_size=4k -o ? TEST_DIR/t.qcow2 128M
@@ -140,13 +143,14 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,help TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2,help' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2,,help encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,? TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2,?' encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2,,? encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: create -f qcow2 -o backing_file=TEST_DIR/t.qcow2, -o help TEST_DIR/t.qcow2 128M
 qemu-img: Invalid option list: backing_file=TEST_DIR/t.qcow2,
@@ -167,6 +171,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 
 Testing: create -o help
 Supported options:
@@ -175,7 +180,7 @@
 === convert: Options specified more than once ===
 
 Testing: create -f qcow2 TEST_DIR/t.qcow2 128M
-Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=65536 lazy_refcounts=off 
+Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=134217728 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 
 Testing: convert -f foo -f qcow2 TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
 image: TEST_DIR/t.IMGFMT.base
@@ -189,29 +194,25 @@
 cluster_size: 65536
 
 Testing: convert -O qcow2 -o cluster_size=4k -o lazy_refcounts=on TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
-
-Testing: info TEST_DIR/t.qcow2.base
-image: TEST_DIR/t.qcow2.base
-file format: qcow2
+image: TEST_DIR/t.IMGFMT.base
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 16K
 cluster_size: 4096
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    refcount bits: 16
     corrupt: false
 
 Testing: convert -O qcow2 -o cluster_size=4k -o lazy_refcounts=on -o cluster_size=8k TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
-
-Testing: info TEST_DIR/t.qcow2.base
-image: TEST_DIR/t.qcow2.base
-file format: qcow2
+image: TEST_DIR/t.IMGFMT.base
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 28K
 cluster_size: 8192
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    refcount bits: 16
     corrupt: false
 
 Testing: convert -O qcow2 -o cluster_size=4k,cluster_size=8k TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -232,6 +233,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o ? TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -244,6 +246,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o cluster_size=4k,help TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -256,6 +259,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o cluster_size=4k,? TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -268,6 +272,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o help,cluster_size=4k TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -280,6 +285,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o ?,cluster_size=4k TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -292,6 +298,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o cluster_size=4k -o help TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -304,6 +311,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o cluster_size=4k -o ? TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -316,6 +324,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: convert -O qcow2 -o backing_file=TEST_DIR/t.qcow2,,help TEST_DIR/t.qcow2 TEST_DIR/t.qcow2.base
@@ -343,6 +352,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 
 Testing: convert -o help
 Supported options:
@@ -351,42 +361,36 @@
 === amend: Options specified more than once ===
 
 Testing: amend -f foo -f qcow2 -o lazy_refcounts=on TEST_DIR/t.qcow2
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 128M (134217728 bytes)
-disk size: 196K
 cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    refcount bits: 16
     corrupt: false
 
 Testing: amend -f qcow2 -o size=130M -o lazy_refcounts=off TEST_DIR/t.qcow2
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 130M (136314880 bytes)
-disk size: 196K
 cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    refcount bits: 16
     corrupt: false
 
 Testing: amend -f qcow2 -o size=8M -o lazy_refcounts=on -o size=132M TEST_DIR/t.qcow2
-
-Testing: info TEST_DIR/t.qcow2
-image: TEST_DIR/t.qcow2
-file format: qcow2
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
 virtual size: 132M (138412032 bytes)
-disk size: 196K
 cluster_size: 65536
 Format specific information:
     compat: 1.1
     lazy refcounts: true
+    refcount bits: 16
     corrupt: false
 
 Testing: amend -f qcow2 -o size=4M,size=148M TEST_DIR/t.qcow2
@@ -407,6 +411,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o ? TEST_DIR/t.qcow2
@@ -419,6 +424,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o cluster_size=4k,help TEST_DIR/t.qcow2
@@ -431,6 +437,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o cluster_size=4k,? TEST_DIR/t.qcow2
@@ -443,6 +450,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o help,cluster_size=4k TEST_DIR/t.qcow2
@@ -455,6 +463,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o ?,cluster_size=4k TEST_DIR/t.qcow2
@@ -467,6 +476,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o cluster_size=4k -o help TEST_DIR/t.qcow2
@@ -479,6 +489,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o cluster_size=4k -o ? TEST_DIR/t.qcow2
@@ -491,6 +502,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 nocow            Turn off copy-on-write (valid only on btrfs)
 
 Testing: amend -f qcow2 -o backing_file=TEST_DIR/t.qcow2,,help TEST_DIR/t.qcow2
@@ -520,6 +532,7 @@
 cluster_size     qcow2 cluster size
 preallocation    Preallocation mode (allowed values: off, metadata, falloc, full)
 lazy_refcounts   Postpone refcount updates
+refcount_bits    Width of a reference count entry in bits
 
 Testing: convert -o help
 Supported options:
diff --git a/tests/qemu-iotests/083 b/tests/qemu-iotests/083
index 991a9d9..bff9360 100755
--- a/tests/qemu-iotests/083
+++ b/tests/qemu-iotests/083
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 # get standard environment, filters and checks
@@ -44,21 +43,11 @@
 
 wait_for_tcp_port() {
 	while ! (netstat --tcp --listening --numeric | \
-		 grep "$1.*0\\.0\\.0\\.0:\\*.*LISTEN") 2>&1 >/dev/null; do
+		 grep "$1.*0\\.0\\.0\\.0:\\*.*LISTEN") >/dev/null 2>&1; do
 		sleep 0.1
 	done
 }
 
-filter_nbd() {
-	# nbd.c error messages contain function names and line numbers that are prone
-	# to change.  Message ordering depends on timing between send and receive
-	# callbacks sometimes, making them unreliable.
-	#
-	# Filter out the TCP port number since this changes between runs.
-	sed -e 's#^.*nbd\.c:.*##g' \
-	    -e 's#nbd:127\.0\.0\.1:[^:]*:#nbd:127\.0\.0\.1:PORT:#g'
-}
-
 check_disconnect() {
 	event=$1
 	when=$2
@@ -81,9 +70,9 @@
 		nbd_url="nbd:127.0.0.1:$port:exportname=foo"
 	fi
 
-	$PYTHON nbd-fault-injector.py $extra_args "127.0.0.1:$port" "$TEST_DIR/nbd-fault-injector.conf" 2>&1 >/dev/null &
+	$PYTHON nbd-fault-injector.py $extra_args "127.0.0.1:$port" "$TEST_DIR/nbd-fault-injector.conf" >/dev/null 2>&1 &
 	wait_for_tcp_port "127\\.0\\.0\\.1:$port"
-	$QEMU_IO -c "read 0 512" "$nbd_url" 2>&1 | _filter_qemu_io | filter_nbd
+	$QEMU_IO -c "read 0 512" "$nbd_url" 2>&1 | _filter_qemu_io | _filter_nbd
 
 	echo
 }
diff --git a/tests/qemu-iotests/083.out b/tests/qemu-iotests/083.out
index 85ee8d6..ef3d1e3 100644
--- a/tests/qemu-iotests/083.out
+++ b/tests/qemu-iotests/083.out
@@ -1,163 +1,128 @@
 QA output created by 083
 === Check disconnect before neg1 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect after neg1 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 8 neg1 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 16 neg1 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect before export ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect after export ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 4 export ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 12 export ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 16 export ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect before neg2 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect after neg2 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect 8 neg2 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect 10 neg2 ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT:exportname=foo
 no file open, try 'help open'
 
 === Check disconnect before request ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect after request ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect before reply ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect after reply ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect 4 reply ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect 8 reply ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect before data ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT:exportname=foo: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 === Check disconnect after data ===
 
-
-read failed: Input/output error
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Check disconnect before neg-classic ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 8 neg-classic ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 16 neg-classic ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 24 neg-classic ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect 28 neg-classic ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT: Could not open image: Invalid argument
+can't open device nbd:127.0.0.1:PORT
 no file open, try 'help open'
 
 === Check disconnect after neg-classic ===
 
-
-qemu-io: can't open device nbd:127.0.0.1:PORT: Could not read image for determining its format: Input/output error
-no file open, try 'help open'
+read failed: Input/output error
 
 *** done
diff --git a/tests/qemu-iotests/084 b/tests/qemu-iotests/084
index 733018d..04f2aa9 100755
--- a/tests/qemu-iotests/084
+++ b/tests/qemu-iotests/084
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/084.out b/tests/qemu-iotests/084.out
index 5ece829..5c5ab92 100644
--- a/tests/qemu-iotests/084.out
+++ b/tests/qemu-iotests/084.out
@@ -2,7 +2,7 @@
 
 === Statically allocated image creation ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
@@ -11,7 +11,7 @@
 
 === Testing image size bounds ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 64M (67108864 bytes)
diff --git a/tests/qemu-iotests/085 b/tests/qemu-iotests/085
index 56cd6f8..aa77eca 100755
--- a/tests/qemu-iotests/085
+++ b/tests/qemu-iotests/085
@@ -7,6 +7,7 @@
 # snapshots are performed.
 #
 # Copyright (C) 2014 Red Hat, Inc.
+# Copyright (C) 2015 Igalia, S.L.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -34,17 +35,17 @@
 snapshot_virt0="snapshot-v0.qcow2"
 snapshot_virt1="snapshot-v1.qcow2"
 
-MAX_SNAPSHOTS=10
+SNAPSHOTS=10
 
 _cleanup()
 {
     _cleanup_qemu
-    for i in $(seq 1 ${MAX_SNAPSHOTS})
+    for i in $(seq 1 ${SNAPSHOTS})
     do
         rm -f "${TEST_DIR}/${i}-${snapshot_virt0}"
         rm -f "${TEST_DIR}/${i}-${snapshot_virt1}"
     done
-	_cleanup_test_img
+    rm -f "${TEST_IMG}.1" "${TEST_IMG}.2"
 
 }
 trap "_cleanup; exit \$status" 0 1 2 3 15
@@ -64,7 +65,7 @@
 {
     cmd="{ 'execute': 'blockdev-snapshot-sync',
                       'arguments': { 'device': 'virtio0',
-                                     'snapshot-file':'"${TEST_DIR}/${1}-${snapshot_virt0}"',
+                                     'snapshot-file':'${TEST_DIR}/${1}-${snapshot_virt0}',
                                      'format': 'qcow2' } }"
     _send_qemu_cmd $h "${cmd}" "return"
 }
@@ -76,27 +77,60 @@
            {'actions': [
                { 'type': 'blockdev-snapshot-sync', 'data' :
                    { 'device': 'virtio0',
-                      'snapshot-file': '"${TEST_DIR}/${1}-${snapshot_virt0}"' } },
+                      'snapshot-file': '${TEST_DIR}/${1}-${snapshot_virt0}' } },
                { 'type': 'blockdev-snapshot-sync', 'data' :
                    { 'device': 'virtio1',
-                       'snapshot-file': '"${TEST_DIR}/${1}-${snapshot_virt1}"' } } ]
+                       'snapshot-file': '${TEST_DIR}/${1}-${snapshot_virt1}' } } ]
              } }"
 
     _send_qemu_cmd $h "${cmd}" "return"
 }
 
+# ${1}: unique identifier for the snapshot filename
+# ${2}: true: open backing images; false: don't open them (default)
+function add_snapshot_image()
+{
+    if [ "${2}" = "true" ]; then
+        extra_params=""
+    else
+        extra_params="'backing': '', "
+    fi
+    base_image="${TEST_DIR}/$((${1}-1))-${snapshot_virt0}"
+    snapshot_file="${TEST_DIR}/${1}-${snapshot_virt0}"
+    _make_test_img -b "${base_image}" "$size"
+    mv "${TEST_IMG}" "${snapshot_file}"
+    cmd="{ 'execute': 'blockdev-add', 'arguments':
+           { 'options':
+             { 'driver': 'qcow2', 'node-name': 'snap_${1}', ${extra_params}
+               'file':
+               { 'driver': 'file', 'filename': '${snapshot_file}',
+                 'node-name': 'file_${1}' } } } }"
+    _send_qemu_cmd $h "${cmd}" "return"
+}
+
+# ${1}: unique identifier for the snapshot filename
+# ${2}: expected response, defaults to 'return'
+function blockdev_snapshot()
+{
+    cmd="{ 'execute': 'blockdev-snapshot',
+                      'arguments': { 'node': 'virtio0',
+                                     'overlay':'snap_${1}' } }"
+    _send_qemu_cmd $h "${cmd}" "${2:-return}"
+}
+
 size=128M
 
 _make_test_img $size
-mv "${TEST_IMG}" "${TEST_IMG}.orig"
+mv "${TEST_IMG}" "${TEST_IMG}.1"
 _make_test_img $size
+mv "${TEST_IMG}" "${TEST_IMG}.2"
 
 echo
 echo === Running QEMU ===
 echo
 
 qemu_comm_method="qmp"
-_launch_qemu -drive file="${TEST_IMG}.orig",if=virtio -drive file="${TEST_IMG}",if=virtio
+_launch_qemu -drive file="${TEST_IMG}.1",if=virtio -drive file="${TEST_IMG}.2",if=virtio
 h=$QEMU_HANDLE
 
 echo
@@ -105,6 +139,8 @@
 
 _send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" "return"
 
+# Tests for the blockdev-snapshot-sync command
+
 echo
 echo === Create a single snapshot on virtio0 ===
 echo
@@ -117,7 +153,7 @@
 echo
 
 _send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
-                         'arguments': { 'snapshot-file':'"${TEST_DIR}/1-${snapshot_virt0}"',
+                         'arguments': { 'snapshot-file':'${TEST_DIR}/1-${snapshot_virt0}',
                                      'format': 'qcow2' } }" "error"
 
 echo
@@ -132,11 +168,75 @@
 echo === Create several transactional group snapshots ===
 echo
 
-for i in $(seq 2 ${MAX_SNAPSHOTS})
+for i in $(seq 2 ${SNAPSHOTS})
 do
     create_group_snapshot ${i}
 done
 
+# Tests for the blockdev-snapshot command
+
+echo
+echo === Create a couple of snapshots using blockdev-snapshot ===
+echo
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS}
+blockdev_snapshot ${SNAPSHOTS}
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS}
+blockdev_snapshot ${SNAPSHOTS}
+
+echo
+echo === Invalid command - cannot create a snapshot using a file BDS ===
+echo
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'file_${SNAPSHOTS}' }
+                   }" "error"
+
+echo
+echo === Invalid command - snapshot node used as active layer ===
+echo
+
+blockdev_snapshot ${SNAPSHOTS} error
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'virtio0' }
+                   }" "error"
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'virtio0',
+                                    'overlay':'virtio1' }
+                   }" "error"
+
+echo
+echo === Invalid command - snapshot node used as backing hd ===
+echo
+
+blockdev_snapshot $((${SNAPSHOTS}-1)) error
+
+echo
+echo === Invalid command - snapshot node has a backing image ===
+echo
+
+SNAPSHOTS=$((${SNAPSHOTS}+1))
+add_snapshot_image ${SNAPSHOTS} true
+blockdev_snapshot ${SNAPSHOTS} error
+
+echo
+echo === Invalid command - The node does not exist ===
+echo
+
+blockdev_snapshot $((${SNAPSHOTS}+1)) error
+
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot',
+                     'arguments': { 'node':'nodevice',
+                                    'overlay':'snap_${SNAPSHOTS}' }
+                   }" "error"
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/085.out b/tests/qemu-iotests/085.out
index 0f2b17f..01c78d6 100644
--- a/tests/qemu-iotests/085.out
+++ b/tests/qemu-iotests/085.out
@@ -11,7 +11,7 @@
 
 === Create a single snapshot on virtio0 ===
 
-Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2.orig' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/1-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2.1 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
 
 === Invalid command - missing device and nodename ===
@@ -25,31 +25,65 @@
 
 === Create several transactional group snapshots ===
 
-Formatting 'TEST_DIR/2-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/1-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/2-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/t.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/2-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/1-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/2-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/t.qcow2.2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/3-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/2-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/3-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/2-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/3-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/2-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/3-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/2-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/4-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/3-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/4-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/3-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/4-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/3-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/4-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/3-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/5-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/4-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/5-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/4-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/5-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/4-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/5-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/4-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/6-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/5-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/6-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/5-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/6-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/5-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/6-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/5-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/7-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/6-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/7-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/6-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/7-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/6-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/7-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/6-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/8-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/7-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/8-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/7-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/8-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/7-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/8-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/7-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/9-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/8-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/9-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/8-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/9-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/8-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/9-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/8-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
-Formatting 'TEST_DIR/10-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/9-snapshot-v0.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
-Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file='TEST_DIR/9-snapshot-v1.qcow2' backing_fmt='qcow2' encryption=off cluster_size=65536 lazy_refcounts=off
+Formatting 'TEST_DIR/10-snapshot-v0.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/9-snapshot-v0.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+Formatting 'TEST_DIR/10-snapshot-v1.qcow2', fmt=qcow2 size=134217728 backing_file=TEST_DIR/9-snapshot-v1.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
 {"return": {}}
+
+=== Create a couple of snapshots using blockdev-snapshot ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/10-snapshot-v0.IMGFMT
+{"return": {}}
+{"return": {}}
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/11-snapshot-v0.IMGFMT
+{"return": {}}
+{"return": {}}
+
+=== Invalid command - cannot create a snapshot using a file BDS ===
+
+{"error": {"class": "GenericError", "desc": "The snapshot does not support backing images"}}
+
+=== Invalid command - snapshot node used as active layer ===
+
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio0"}}
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio0"}}
+{"error": {"class": "GenericError", "desc": "The snapshot is already in use by virtio1"}}
+
+=== Invalid command - snapshot node used as backing hd ===
+
+{"error": {"class": "GenericError", "desc": "Node 'snap_11' is busy: node is used as backing hd of 'virtio0'"}}
+
+=== Invalid command - snapshot node has a backing image ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/12-snapshot-v0.IMGFMT
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "The snapshot already has a backing image"}}
+
+=== Invalid command - The node does not exist ===
+
+{"error": {"class": "GenericError", "desc": "Cannot find device=snap_14 nor node_name=snap_14"}}
+{"error": {"class": "GenericError", "desc": "Cannot find device=nodevice nor node_name=nodevice"}}
 *** done
diff --git a/tests/qemu-iotests/086 b/tests/qemu-iotests/086
index 234eb9a..cd4494a 100755
--- a/tests/qemu-iotests/086
+++ b/tests/qemu-iotests/086
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -38,7 +37,7 @@
 . ./common.rc
 . ./common.filter
 
-_supported_fmt qcow2
+_supported_fmt qcow2 raw
 _supported_proto file nfs
 _supported_os Linux
 
diff --git a/tests/qemu-iotests/086.out b/tests/qemu-iotests/086.out
index 9c0bf23..5ff9961 100644
--- a/tests/qemu-iotests/086.out
+++ b/tests/qemu-iotests/086.out
@@ -1,5 +1,5 @@
 QA output created by 086
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 wrote 1048576/1048576 bytes at offset 0
 1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 1048576/1048576 bytes at offset 2097152
diff --git a/tests/qemu-iotests/087 b/tests/qemu-iotests/087
index d7454d1..e7bca37 100755
--- a/tests/qemu-iotests/087
+++ b/tests/qemu-iotests/087
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 # get standard environment, filters and checks
@@ -45,7 +44,9 @@
 
 function run_qemu()
 {
-    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
+                          | _filter_qemu | _filter_imgfmt \
+                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
 }
 
 size=128M
@@ -53,7 +54,7 @@
 _make_test_img $size
 
 echo
-echo === Missing ID ===
+echo === Missing ID and node-name ===
 echo
 
 run_qemu <<EOF
diff --git a/tests/qemu-iotests/087.out b/tests/qemu-iotests/087.out
index e8795b3..a95c4b0 100644
--- a/tests/qemu-iotests/087.out
+++ b/tests/qemu-iotests/087.out
@@ -1,16 +1,14 @@
 QA output created by 087
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
 
-=== Missing ID ===
+=== Missing ID and node-name ===
 
 Testing:
 QMP_VERSION
 {"return": {}}
-{"error": {"class": "GenericError", "desc": "Block device needs an ID"}}
+{"error": {"class": "GenericError", "desc": "'id' and/or 'node-name' need to be specified for the root node"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Duplicate ID ===
@@ -21,14 +19,11 @@
 {"return": {}}
 {"error": {"class": "GenericError", "desc": "Device with id 'disk' already exists"}}
 {"error": {"class": "GenericError", "desc": "Device name 'test-node' conflicts with an existing node name"}}
-main-loop: WARNING: I/O thread spun for 1000 iterations
-{"error": {"class": "GenericError", "desc": "could not open disk image disk2: node-name=disk is conflicting with a device id"}}
-{"error": {"class": "GenericError", "desc": "could not open disk image disk2: Duplicate node name"}}
-{"error": {"class": "GenericError", "desc": "could not open disk image disk3: node-name=disk3 is conflicting with a device id"}}
+{"error": {"class": "GenericError", "desc": "node-name=disk is conflicting with a device id"}}
+{"error": {"class": "GenericError", "desc": "Duplicate node name"}}
+{"error": {"class": "GenericError", "desc": "Device name 'disk3' conflicts with an existing node name"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === aio=native without O_DIRECT ===
@@ -39,42 +34,34 @@
 {"error": {"class": "GenericError", "desc": "aio=native requires cache.direct=true"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Encrypted image ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on
 Testing: -S
 QMP_VERSION
 {"return": {}}
-{"error": {"class": "GenericError", "desc": "blockdev-add doesn't support encrypted devices"}}
+{"error": {"class": "GenericError", "desc": "Use of AES-CBC encrypted IMGFMT images is no longer supported in system emulators"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 Testing:
 QMP_VERSION
 {"return": {}}
-{"error": {"class": "GenericError", "desc": "could not open disk image disk: Guest must be stopped for opening of encrypted image"}}
+{"error": {"class": "GenericError", "desc": "Use of AES-CBC encrypted IMGFMT images is no longer supported in system emulators"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 
 === Missing driver ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on
 Testing: -S
 QMP_VERSION
 {"return": {}}
 {"error": {"class": "GenericError", "desc": "Invalid parameter type for 'driver', expected: string"}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
-{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "floppy0", "tray-open": true}}
 
 *** done
diff --git a/tests/qemu-iotests/088 b/tests/qemu-iotests/088
index f9c3129..b8076f2 100755
--- a/tests/qemu-iotests/088
+++ b/tests/qemu-iotests/088
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/088.out b/tests/qemu-iotests/088.out
index d961609..a2a83b8 100644
--- a/tests/qemu-iotests/088.out
+++ b/tests/qemu-iotests/088.out
@@ -1,17 +1,17 @@
 QA output created by 088
 
 == Invalid block size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 0
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.vpc: Invalid block size 0
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 0
+can't open device TEST_DIR/t.vpc: Invalid block size 0
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 128
+can't open device TEST_DIR/t.vpc: Invalid block size 128
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 128
+can't open device TEST_DIR/t.vpc: Invalid block size 128
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 305419896
+can't open device TEST_DIR/t.vpc: Invalid block size 305419896
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.vpc: Invalid block size 305419896
+can't open device TEST_DIR/t.vpc: Invalid block size 305419896
 no file open, try 'help open'
 *** done
diff --git a/tests/qemu-iotests/089 b/tests/qemu-iotests/089
index dffc977..9bfe230 100755
--- a/tests/qemu-iotests/089
+++ b/tests/qemu-iotests/089
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -41,6 +40,8 @@
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# Because anything other than 16 would change the output of qemu_io -c info
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 # Using an image filename containing quotation marks will render the JSON data
 # below invalid. In that case, we have little choice but simply not to run this
@@ -65,7 +66,8 @@
 
 $QEMU_IMG convert -f raw -O $IMGFMT "$TEST_IMG.base" "$TEST_IMG"
 
-$QEMU_IO -c 'read -P 42 0 512' -c 'read -P 23 512 512' \
+$QEMU_IO_PROG --cache $CACHEMODE \
+         -c 'read -P 42 0 512' -c 'read -P 23 512 512' \
          -c 'read -P 66 1024 512' "json:{
     \"driver\": \"$IMGFMT\",
     \"file\": {
@@ -91,7 +93,8 @@
 
 # The "image.filename" part tests whether "a": { "b": "c" } and "a.b": "c" do
 # the same (which they should).
-$QEMU_IO -c 'read -P 42 0x38000 512' "json:{
+$QEMU_IO_PROG --cache $CACHEMODE \
+     -c 'read -P 42 0x38000 512' "json:{
     \"driver\": \"$IMGFMT\",
     \"file\": {
         \"driver\": \"blkdebug\",
diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
index b2b0390..18f5fdd 100644
--- a/tests/qemu-iotests/089.out
+++ b/tests/qemu-iotests/089.out
@@ -2,7 +2,7 @@
 
 === Testing nested image formats ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 512/512 bytes at offset 512
@@ -21,7 +21,7 @@
 
 === Testing blkdebug ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 512/512 bytes at offset 229376
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read failed: Input/output error
@@ -41,6 +41,7 @@
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    refcount bits: 16
     corrupt: false
 format name: IMGFMT
 cluster size: 64 KiB
@@ -48,5 +49,6 @@
 Format specific information:
     compat: 1.1
     lazy refcounts: false
+    refcount bits: 16
     corrupt: false
 *** done
diff --git a/tests/qemu-iotests/090 b/tests/qemu-iotests/090
index 70b5a6f..7380503 100755
--- a/tests/qemu-iotests/090
+++ b/tests/qemu-iotests/090
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/090.out b/tests/qemu-iotests/090.out
index 2df93e0..66ca5d5 100644
--- a/tests/qemu-iotests/090.out
+++ b/tests/qemu-iotests/090.out
@@ -1,5 +1,5 @@
 QA output created by 090
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 65536/65536 bytes at offset 65536
diff --git a/tests/qemu-iotests/091.out b/tests/qemu-iotests/091.out
index a2e0122..5017f8c 100644
--- a/tests/qemu-iotests/091.out
+++ b/tests/qemu-iotests/091.out
@@ -1,5 +1,5 @@
 QA output created by 091
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824
 
 === Starting QEMU VM1 ===
 
diff --git a/tests/qemu-iotests/092 b/tests/qemu-iotests/092
index 52c529b..5bbdd07 100755
--- a/tests/qemu-iotests/092
+++ b/tests/qemu-iotests/092
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/092.out b/tests/qemu-iotests/092.out
index 496d8f0..e18f54c 100644
--- a/tests/qemu-iotests/092.out
+++ b/tests/qemu-iotests/092.out
@@ -1,38 +1,38 @@
 QA output created by 092
 
 == Invalid cluster size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
 no file open, try 'help open'
 
 == Invalid L2 table size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
 no file open, try 'help open'
 
 == Invalid size ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow: Image too large
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow: Image too large
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Image too large
+can't open device TEST_DIR/t.qcow: Image too large
 no file open, try 'help open'
 
 == Invalid backing file length ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
-qemu-io: can't open device TEST_DIR/t.qcow: Backing file name too long
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+can't open device TEST_DIR/t.qcow: Backing file name too long
 no file open, try 'help open'
-qemu-io: can't open device TEST_DIR/t.qcow: Backing file name too long
+can't open device TEST_DIR/t.qcow: Backing file name too long
 no file open, try 'help open'
 *** done
diff --git a/tests/qemu-iotests/093 b/tests/qemu-iotests/093
new file mode 100755
index 0000000..ffcb271
--- /dev/null
+++ b/tests/qemu-iotests/093
@@ -0,0 +1,286 @@
+#!/usr/bin/env python
+#
+# Tests for IO throttling
+#
+# Copyright (C) 2015 Red Hat, Inc.
+# Copyright (C) 2015-2016 Igalia, S.L.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+
+nsec_per_sec = 1000000000
+
+class ThrottleTestCase(iotests.QMPTestCase):
+    test_img = "null-aio://"
+    max_drives = 3
+
+    def blockstats(self, device):
+        result = self.vm.qmp("query-blockstats")
+        for r in result['return']:
+            if r['device'] == device:
+                stat = r['stats']
+                return stat['rd_bytes'], stat['rd_operations'], stat['wr_bytes'], stat['wr_operations']
+        raise Exception("Device not found for blockstats: %s" % device)
+
+    def setUp(self):
+        self.vm = iotests.VM()
+        for i in range(0, self.max_drives):
+            self.vm.add_drive(self.test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+
+    def configure_throttle(self, ndrives, params):
+        params['group'] = 'test'
+
+        # Set the I/O throttling parameters to all drives
+        for i in range(0, ndrives):
+            params['device'] = 'drive%d' % i
+            result = self.vm.qmp("block_set_io_throttle", conv_keys=False, **params)
+            self.assert_qmp(result, 'return', {})
+
+    def do_test_throttle(self, ndrives, seconds, params):
+        def check_limit(limit, num):
+            # IO throttling algorithm is discrete, allow 10% error so the test
+            # is more robust
+            return limit == 0 or \
+                   (num < seconds * limit * 1.1 / ndrives
+                   and num > seconds * limit * 0.9 / ndrives)
+
+        # Set vm clock to a known value
+        ns = seconds * nsec_per_sec
+        self.vm.qtest("clock_step %d" % ns)
+
+        # Submit enough requests so the throttling mechanism kicks
+        # in. The throttled requests won't be executed until we
+        # advance the virtual clock.
+        rq_size = 512
+        rd_nr = max(params['bps'] / rq_size / 2,
+                    params['bps_rd'] / rq_size,
+                    params['iops'] / 2,
+                    params['iops_rd'])
+        rd_nr *= seconds * 2
+        rd_nr /= ndrives
+        wr_nr = max(params['bps'] / rq_size / 2,
+                    params['bps_wr'] / rq_size,
+                    params['iops'] / 2,
+                    params['iops_wr'])
+        wr_nr *= seconds * 2
+        wr_nr /= ndrives
+
+        # Send I/O requests to all drives
+        for i in range(rd_nr):
+            for drive in range(0, ndrives):
+                self.vm.hmp_qemu_io("drive%d" % drive, "aio_read %d %d" %
+                                    (i * rq_size, rq_size))
+
+        for i in range(wr_nr):
+            for drive in range(0, ndrives):
+                self.vm.hmp_qemu_io("drive%d" % drive, "aio_write %d %d" %
+                                    (i * rq_size, rq_size))
+
+        # We'll store the I/O stats for each drive in these arrays
+        start_rd_bytes = [0] * ndrives
+        start_rd_iops  = [0] * ndrives
+        start_wr_bytes = [0] * ndrives
+        start_wr_iops  = [0] * ndrives
+        end_rd_bytes   = [0] * ndrives
+        end_rd_iops    = [0] * ndrives
+        end_wr_bytes   = [0] * ndrives
+        end_wr_iops    = [0] * ndrives
+
+        # Read the stats before advancing the clock
+        for i in range(0, ndrives):
+            start_rd_bytes[i], start_rd_iops[i], start_wr_bytes[i], \
+                start_wr_iops[i] = self.blockstats('drive%d' % i)
+
+        self.vm.qtest("clock_step %d" % ns)
+
+        # Read the stats after advancing the clock
+        for i in range(0, ndrives):
+            end_rd_bytes[i], end_rd_iops[i], end_wr_bytes[i], \
+                end_wr_iops[i] = self.blockstats('drive%d' % i)
+
+        # Check that the I/O is within the limits and evenly distributed
+        for i in range(0, ndrives):
+            rd_bytes = end_rd_bytes[i] - start_rd_bytes[i]
+            rd_iops = end_rd_iops[i] - start_rd_iops[i]
+            wr_bytes = end_wr_bytes[i] - start_wr_bytes[i]
+            wr_iops = end_wr_iops[i] - start_wr_iops[i]
+
+            self.assertTrue(check_limit(params['bps'], rd_bytes + wr_bytes))
+            self.assertTrue(check_limit(params['bps_rd'], rd_bytes))
+            self.assertTrue(check_limit(params['bps_wr'], wr_bytes))
+            self.assertTrue(check_limit(params['iops'], rd_iops + wr_iops))
+            self.assertTrue(check_limit(params['iops_rd'], rd_iops))
+            self.assertTrue(check_limit(params['iops_wr'], wr_iops))
+
+    def test_all(self):
+        params = {"bps": 4096,
+                  "bps_rd": 4096,
+                  "bps_wr": 4096,
+                  "iops": 10,
+                  "iops_rd": 10,
+                  "iops_wr": 10,
+                 }
+        # Repeat the test with different numbers of drives
+        for ndrives in range(1, self.max_drives + 1):
+            # Pick each out of all possible params and test
+            for tk in params:
+                limits = dict([(k, 0) for k in params])
+                limits[tk] = params[tk] * ndrives
+                self.configure_throttle(ndrives, limits)
+                self.do_test_throttle(ndrives, 5, limits)
+
+    def test_burst(self):
+        params = {"bps": 4096,
+                  "bps_rd": 4096,
+                  "bps_wr": 4096,
+                  "iops": 10,
+                  "iops_rd": 10,
+                  "iops_wr": 10,
+                 }
+        ndrives = 1
+        # Pick each out of all possible params and test
+        for tk in params:
+            rate = params[tk] * ndrives
+            burst_rate = rate * 7
+            burst_length = 4
+
+            # Configure the throttling settings
+            settings = dict([(k, 0) for k in params])
+            settings[tk] = rate
+            settings['%s_max' % tk] = burst_rate
+            settings['%s_max_length' % tk] = burst_length
+            self.configure_throttle(ndrives, settings)
+
+            # Wait for the bucket to empty so we can do bursts
+            wait_ns = nsec_per_sec * burst_length * burst_rate / rate
+            self.vm.qtest("clock_step %d" % wait_ns)
+
+            # Test I/O at the max burst rate
+            limits = dict([(k, 0) for k in params])
+            limits[tk] = burst_rate
+            self.do_test_throttle(ndrives, burst_length, limits)
+
+            # Now test I/O at the normal rate
+            limits[tk] = rate
+            self.do_test_throttle(ndrives, 5, limits)
+
+class ThrottleTestCoroutine(ThrottleTestCase):
+    test_img = "null-co://"
+
+class ThrottleTestGroupNames(iotests.QMPTestCase):
+    test_img = "null-aio://"
+    max_drives = 3
+
+    def setUp(self):
+        self.vm = iotests.VM()
+        for i in range(0, self.max_drives):
+            self.vm.add_drive(self.test_img, "throttling.iops-total=100")
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+
+    def set_io_throttle(self, device, params):
+        params["device"] = device
+        result = self.vm.qmp("block_set_io_throttle", conv_keys=False, **params)
+        self.assert_qmp(result, 'return', {})
+
+    def verify_name(self, device, name):
+        result = self.vm.qmp("query-block")
+        for r in result["return"]:
+            if r["device"] == device:
+                info = r["inserted"]
+                if name:
+                    self.assertEqual(info["group"], name)
+                else:
+                    self.assertFalse(info.has_key('group'))
+                return
+
+        raise Exception("No group information found for '%s'" % device)
+
+    def test_group_naming(self):
+        params = {"bps": 0,
+                  "bps_rd": 0,
+                  "bps_wr": 0,
+                  "iops": 0,
+                  "iops_rd": 0,
+                  "iops_wr": 0}
+
+        # Check the drives added using the command line.
+        # The default throttling group name is the device name.
+        for i in range(self.max_drives):
+            devname = "drive%d" % i
+            self.verify_name(devname, devname)
+
+        # Clear throttling settings => the group name is gone.
+        for i in range(self.max_drives):
+            devname = "drive%d" % i
+            self.set_io_throttle(devname, params)
+            self.verify_name(devname, None)
+
+        # Set throttling settings using block_set_io_throttle and
+        # check the default group names.
+        params["iops"] = 10
+        for i in range(self.max_drives):
+            devname = "drive%d" % i
+            self.set_io_throttle(devname, params)
+            self.verify_name(devname, devname)
+
+        # Set a custom group name for each device
+        for i in range(3):
+            devname = "drive%d" % i
+            groupname = "group%d" % i
+            params['group'] = groupname
+            self.set_io_throttle(devname, params)
+            self.verify_name(devname, groupname)
+
+        # Put drive0 in group1 and check that all other devices remain
+        # unchanged
+        params['group'] = 'group1'
+        self.set_io_throttle('drive0', params)
+        self.verify_name('drive0', 'group1')
+        for i in range(1, self.max_drives):
+            devname = "drive%d" % i
+            groupname = "group%d" % i
+            self.verify_name(devname, groupname)
+
+        # Put drive0 in group2 and check that all other devices remain
+        # unchanged
+        params['group'] = 'group2'
+        self.set_io_throttle('drive0', params)
+        self.verify_name('drive0', 'group2')
+        for i in range(1, self.max_drives):
+            devname = "drive%d" % i
+            groupname = "group%d" % i
+            self.verify_name(devname, groupname)
+
+        # Clear throttling settings from drive0 check that all other
+        # devices remain unchanged
+        params["iops"] = 0
+        self.set_io_throttle('drive0', params)
+        self.verify_name('drive0', None)
+        for i in range(1, self.max_drives):
+            devname = "drive%d" % i
+            groupname = "group%d" % i
+            self.verify_name(devname, groupname)
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["raw"])
diff --git a/tests/qemu-iotests/093.out b/tests/qemu-iotests/093.out
new file mode 100644
index 0000000..914e373
--- /dev/null
+++ b/tests/qemu-iotests/093.out
@@ -0,0 +1,5 @@
+.....
+----------------------------------------------------------------------
+Ran 5 tests
+
+OK
diff --git a/tests/qemu-iotests/094 b/tests/qemu-iotests/094
new file mode 100755
index 0000000..0ba0b0c
--- /dev/null
+++ b/tests/qemu-iotests/094
@@ -0,0 +1,82 @@
+#!/bin/bash
+#
+# Test case for drive-mirror to NBD
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+trap "exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt generic
+_supported_proto nbd
+_supported_os Linux
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+
+_make_test_img 64M
+$QEMU_IMG create -f $IMGFMT "$TEST_DIR/source.$IMGFMT" 64M | _filter_img_create
+
+_launch_qemu -drive if=none,id=src,file="$TEST_DIR/source.$IMGFMT",format=raw \
+             -nodefaults
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{'execute': 'qmp_capabilities'}" \
+    'return'
+
+# 'format': 'nbd' is not actually "correct", but this was the only way to
+# test the bug fixed in commit f53a829.  Though the bug's related code
+# bdrv_swap() was replaced later, let's make sure we don't fall in the same
+# pit again.
+_send_qemu_cmd $QEMU_HANDLE  \
+    "{'execute': 'drive-mirror',
+      'arguments': {'device': 'src',
+                    'target': '$TEST_IMG',
+                    'format': 'nbd',
+                    'sync':'full',
+                    'mode':'existing'}}" \
+    'BLOCK_JOB_READY'
+
+_send_qemu_cmd $QEMU_HANDLE  \
+    "{'execute': 'block-job-complete',
+      'arguments': {'device': 'src'}}" \
+    'BLOCK_JOB_COMPLETE'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{'execute': 'quit'}" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+_cleanup_test_img
+rm -f "$TEST_DIR/source.$IMGFMT"
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/094.out b/tests/qemu-iotests/094.out
new file mode 100644
index 0000000..b66dc07
--- /dev/null
+++ b/tests/qemu-iotests/094.out
@@ -0,0 +1,11 @@
+QA output created by 094
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/source.IMGFMT', fmt=IMGFMT size=67108864
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 67108864, "offset": 67108864, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 67108864, "offset": 67108864, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+*** done
diff --git a/tests/qemu-iotests/095 b/tests/qemu-iotests/095
index 6630181..030adb2 100755
--- a/tests/qemu-iotests/095
+++ b/tests/qemu-iotests/095
@@ -50,17 +50,15 @@
 size_smaller=5M
 size_larger=100M
 
-_make_test_img $size_smaller
-mv "${TEST_IMG}" "${TEST_IMG}.base"
+TEST_IMG="$TEST_IMG.base" _make_test_img $size_smaller
 
-_make_test_img -b "${TEST_IMG}.base" $size_larger
-mv "${TEST_IMG}" "${TEST_IMG}.snp1"
+TEST_IMG="$TEST_IMG.snp1" _make_test_img -b "$TEST_IMG.base" $size_larger
 
 _make_test_img -b "${TEST_IMG}.snp1" $size_larger
 
 echo
 echo "=== Base image info before commit and resize ==="
-TEST_IMG="${TEST_IMG}.base" _img_info
+TEST_IMG="${TEST_IMG}.base" _img_info | _filter_img_info
 
 echo
 echo === Running QEMU Live Commit Test ===
@@ -76,9 +74,11 @@
                                  'arguments': { 'device': 'test',
                                  'top': '"${TEST_IMG}.snp1"' } }" "BLOCK_JOB_COMPLETED"
 
+_cleanup_qemu
+
 echo
 echo "=== Base image info after commit and resize ==="
-TEST_IMG="${TEST_IMG}.base" _img_info
+TEST_IMG="${TEST_IMG}.base" _img_info | _filter_img_info
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/095.out b/tests/qemu-iotests/095.out
index cc86efa..73875ca 100644
--- a/tests/qemu-iotests/095.out
+++ b/tests/qemu-iotests/095.out
@@ -1,13 +1,12 @@
 QA output created by 095
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=5242880 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file='TEST_DIR/t.IMGFMT.snp1' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=5242880
+Formatting 'TEST_DIR/t.IMGFMT.snp1', fmt=IMGFMT size=104857600 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=104857600 backing_file=TEST_DIR/t.IMGFMT.snp1
 
 === Base image info before commit and resize ===
 image: TEST_DIR/t.IMGFMT.base
 file format: IMGFMT
 virtual size: 5.0M (5242880 bytes)
-cluster_size: 65536
 
 === Running QEMU Live Commit Test ===
 
@@ -19,5 +18,4 @@
 image: TEST_DIR/t.IMGFMT.base
 file format: IMGFMT
 virtual size: 100M (104857600 bytes)
-cluster_size: 65536
 *** done
diff --git a/tests/qemu-iotests/096 b/tests/qemu-iotests/096
new file mode 100644
index 0000000..aeeb375
--- /dev/null
+++ b/tests/qemu-iotests/096
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# Test that snapshots move the throttling configuration to the active
+# layer
+#
+# Copyright (C) 2015 Igalia, S.L.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+import os
+
+class TestLiveSnapshot(iotests.QMPTestCase):
+    base_img = os.path.join(iotests.test_dir, 'base.img')
+    target_img = os.path.join(iotests.test_dir, 'target.img')
+    group = 'mygroup'
+    iops = 6000
+    iops_size = 1024
+
+    def setUp(self):
+        opts = []
+        opts.append('node-name=base')
+        opts.append('throttling.group=%s' % self.group)
+        opts.append('throttling.iops-total=%d' % self.iops)
+        opts.append('throttling.iops-size=%d' % self.iops_size)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, self.base_img, '100M')
+        self.vm = iotests.VM().add_drive(self.base_img, ','.join(opts))
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.base_img)
+        os.remove(self.target_img)
+
+    def checkConfig(self, active_layer):
+        result = self.vm.qmp('query-block')
+        for r in result['return']:
+            r = r['inserted']
+            if r['node-name'] == active_layer:
+                self.assertEqual(r['group'], self.group)
+                self.assertEqual(r['iops'], self.iops)
+                self.assertEqual(r['iops_size'], self.iops_size)
+            else:
+                self.assertFalse(r.has_key('group'))
+                self.assertEqual(r['iops'], 0)
+                self.assertFalse(r.has_key('iops_size'))
+
+    def testSnapshot(self):
+        self.checkConfig('base')
+        self.vm.qmp('blockdev-snapshot-sync',
+                    node_name = 'base',
+                    snapshot_node_name = 'target',
+                    snapshot_file = self.target_img,
+                    format = iotests.imgfmt)
+        self.checkConfig('target')
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/096.out b/tests/qemu-iotests/096.out
new file mode 100644
index 0000000..ae1213e
--- /dev/null
+++ b/tests/qemu-iotests/096.out
@@ -0,0 +1,5 @@
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK
diff --git a/tests/qemu-iotests/097 b/tests/qemu-iotests/097
index c7a613b..01d8dd0 100755
--- a/tests/qemu-iotests/097
+++ b/tests/qemu-iotests/097
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/097.out b/tests/qemu-iotests/097.out
index 1cb7764..48abd2e 100644
--- a/tests/qemu-iotests/097.out
+++ b/tests/qemu-iotests/097.out
@@ -2,9 +2,9 @@
 
 === Test pass 0 ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -31,9 +31,9 @@
 
 === Test pass 1 ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -61,9 +61,9 @@
 
 === Test pass 2 ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
@@ -91,9 +91,9 @@
 
 === Test pass 3 ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.itmd' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT.itmd', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.itmd
 wrote 196608/196608 bytes at offset 0
 192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 wrote 131072/131072 bytes at offset 65536
diff --git a/tests/qemu-iotests/098 b/tests/qemu-iotests/098
index e2230ad..b002e96 100755
--- a/tests/qemu-iotests/098
+++ b/tests/qemu-iotests/098
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/098.out b/tests/qemu-iotests/098.out
index 586dfc8..7634d0e 100644
--- a/tests/qemu-iotests/098.out
+++ b/tests/qemu-iotests/098.out
@@ -2,8 +2,8 @@
 
 === l1_update ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -11,8 +11,8 @@
 
 === empty_image_prepare ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -24,8 +24,8 @@
 
 === reftable_update ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
@@ -38,8 +38,8 @@
 
 === refblock_alloc ===
 
-Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file='TEST_DIR/t.IMGFMT.base' 
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 qemu-img: Could not empty blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT: Input/output error
diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099
index ffc7ea7..caaf58e 100755
--- a/tests/qemu-iotests/099
+++ b/tests/qemu-iotests/099
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -44,7 +43,8 @@
 _supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc
 _supported_proto file
 _supported_os Linux
-
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+    "subformat=twoGbMaxExtentSparse"
 
 function do_run_qemu()
 {
@@ -56,7 +56,7 @@
     # Get the "file": "foo" entry ($foo may only contain escaped double quotes,
     # which is how we can extract it)
     do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_imgfmt | _filter_qmp \
-        | grep "drv0" \
+        | _filter_qemu | grep "drv0" \
         | sed -e 's/^.*"file": "\(\(\\"\|[^"]\)*\)".*$/\1/' -e 's/\\"/"/g'
 }
 
@@ -107,8 +107,21 @@
 # generate a JSON object here as well
 test_qemu "file.driver=blkverify,file.raw.filename=$TEST_IMG.compare,file.test.file.driver=blkdebug,file.test.file.image.filename=$TEST_IMG,file.test.file.inject-error.0.event=l1_update"
 
+echo
+echo '=== Testing plain filename for blkdebug ==='
+echo
 
-rm -f "$TEST_IMG.compare"
+touch "$TEST_DIR/blkdebug.conf"
+test_qemu "file.driver=blkdebug,file.config=$TEST_DIR/blkdebug.conf,file.image.filename=$TEST_IMG"
+
+echo
+echo '=== Testing plain filename for blkdebug without configuration file ==='
+echo
+
+test_qemu "file.driver=blkdebug,file.image.filename=$TEST_IMG"
+
+
+rm -f "$TEST_IMG.compare" "$TEST_DIR/blkdebug.conf"
 
 # success, all done
 echo "*** done"
diff --git a/tests/qemu-iotests/099.out b/tests/qemu-iotests/099.out
index 55be4d4..8cce627 100644
--- a/tests/qemu-iotests/099.out
+++ b/tests/qemu-iotests/099.out
@@ -1,6 +1,6 @@
 QA output created by 099
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072 
-Formatting 'TEST_DIR/t.IMGFMT.compare', fmt=raw size=131072 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=131072
+Formatting 'TEST_DIR/t.IMGFMT.compare', fmt=raw size=131072
 
 === Testing simple filename for blkverify ===
 
@@ -12,9 +12,17 @@
 
 === Testing JSON filename for blkdebug ===
 
-json:{"driver": "IMGFMT", "file": {"inject-error": [{"immediately": false, "once": false, "state": 0, "sector": -1, "event": "l1_update", "errno": 5}], "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug"}}
+json:{"driver": "IMGFMT", "file": {"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "l1_update"}}
 
 === Testing indirectly enforced JSON filename ===
 
-json:{"driver": "raw", "file": {"test": {"driver": "IMGFMT", "file": {"inject-error": [{"immediately": false, "once": false, "state": 0, "sector": -1, "event": "l1_update", "errno": 5}], "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug"}}, "driver": "blkverify", "raw": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT.compare"}}}
+json:{"driver": "raw", "file": {"test": {"driver": "IMGFMT", "file": {"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "l1_update"}}, "driver": "blkverify", "raw": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT.compare"}}}
+
+=== Testing plain filename for blkdebug ===
+
+blkdebug:TEST_DIR/blkdebug.conf:TEST_DIR/t.IMGFMT
+
+=== Testing plain filename for blkdebug without configuration file ===
+
+blkdebug::TEST_DIR/t.IMGFMT
 *** done
diff --git a/tests/qemu-iotests/100 b/tests/qemu-iotests/100
deleted file mode 100755
index 9124aba..0000000
--- a/tests/qemu-iotests/100
+++ /dev/null
@@ -1,134 +0,0 @@
-#!/bin/bash
-#
-# Test simple read/write using plain bdrv_read/bdrv_write
-#
-# Copyright (C) 2014 Red Hat, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-# creator
-owner=stefanha@redhat.com
-
-seq=`basename $0`
-echo "QA output created by $seq"
-
-here=`pwd`
-tmp=/tmp/$$
-status=1	# failure is the default!
-
-_cleanup()
-{
-	_cleanup_test_img
-}
-trap "_cleanup; exit \$status" 0 1 2 3 15
-
-# get standard environment, filters and checks
-. ./common.rc
-. ./common.filter
-
-_supported_fmt generic
-_supported_proto generic
-_supported_os Linux
-
-
-size=128M
-
-echo
-echo "== Single request =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 0 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-$QEMU_IO -c "read -P 0xcd 0 4k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== Sequential requests =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 0 4k ; 4k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-$QEMU_IO -c "read -P 0xcd 0 4k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0xce 4k 4k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 8k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== Superset overlapping requests =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 0 4k ; 1k 2k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-# Order of overlapping in-flight requests is not guaranteed so we cannot verify
-# [1k, 3k) since it could have either pattern 0xcd or 0xce.
-$QEMU_IO -c "read -P 0xcd 0 1k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0xcd 3k 1k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== Subset overlapping requests =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 1k 2k ; 0k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-# Order of overlapping in-flight requests is not guaranteed so we cannot verify
-# [1k, 3k) since it could have either pattern 0xcd or 0xce.
-$QEMU_IO -c "read -P 0xce 0 1k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0xce 3k 1k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== Head overlapping requests =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 0k 2k ; 0k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-# Order of overlapping in-flight requests is not guaranteed so we cannot verify
-# [0k, 2k) since it could have either pattern 0xcd or 0xce.
-$QEMU_IO -c "read -P 0xce 2k 2k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== Tail overlapping requests =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 2k 2k ; 0k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-# Order of overlapping in-flight requests is not guaranteed so we cannot verify
-# [2k, 4k) since it could have either pattern 0xcd or 0xce.
-$QEMU_IO -c "read -P 0xce 0k 2k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== Disjoint requests =="
-_make_test_img $size
-$QEMU_IO -c "multiwrite 0 4k ; 64k 4k" "$TEST_IMG" | _filter_qemu_io
-
-echo
-echo "== verify pattern =="
-$QEMU_IO -c "read -P 0xcd 0 4k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 4k 60k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0xce 64k 4k" "$TEST_IMG" | _filter_qemu_io
-$QEMU_IO -c "read -P 0 68k 4k" "$TEST_IMG" | _filter_qemu_io
-
-# success, all done
-echo "*** done"
-rm -f $seq.full
-status=0
diff --git a/tests/qemu-iotests/100.out b/tests/qemu-iotests/100.out
deleted file mode 100644
index 2d6e9f0..0000000
--- a/tests/qemu-iotests/100.out
+++ /dev/null
@@ -1,89 +0,0 @@
-QA output created by 100
-
-== Single request ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 4096/4096 bytes at offset 0
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 4096/4096 bytes at offset 0
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 4096
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== Sequential requests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 8192/8192 bytes at offset 0
-8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 4096/4096 bytes at offset 0
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 4096
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 8192
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== Superset overlapping requests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 6144/6144 bytes at offset 0
-6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 1024/1024 bytes at offset 0
-1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 1024/1024 bytes at offset 3072
-1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 4096
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== Subset overlapping requests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 6144/6144 bytes at offset 1024
-6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 1024/1024 bytes at offset 0
-1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 1024/1024 bytes at offset 3072
-1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 4096
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== Head overlapping requests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 6144/6144 bytes at offset 0
-6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 2048/2048 bytes at offset 2048
-2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 4096
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== Tail overlapping requests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 6144/6144 bytes at offset 2048
-6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 2048/2048 bytes at offset 0
-2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 4096
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== Disjoint requests ==
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
-wrote 8192/8192 bytes at offset 0
-8 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-
-== verify pattern ==
-read 4096/4096 bytes at offset 0
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 61440/61440 bytes at offset 4096
-60 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 65536
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-read 4096/4096 bytes at offset 69632
-4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-*** done
diff --git a/tests/qemu-iotests/101 b/tests/qemu-iotests/101
index 70fbf25..ea53f8b 100755
--- a/tests/qemu-iotests/101
+++ b/tests/qemu-iotests/101
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/102 b/tests/qemu-iotests/102
index 161b197..64b4af9 100755
--- a/tests/qemu-iotests/102
+++ b/tests/qemu-iotests/102
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=$PWD
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/103 b/tests/qemu-iotests/103
index ccab551..ecbd8eb 100755
--- a/tests/qemu-iotests/103
+++ b/tests/qemu-iotests/103
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=$PWD
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -93,6 +92,16 @@
          -c 'read -P 42 0 64k' \
     | _filter_qemu_io
 
+echo
+echo '=== Testing minimal L2 cache and COW ==='
+echo
+
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+# This requires a COW operation, which accesses two L2 tables simultaneously
+# (COW source and destination), so there must be enough space in the cache to
+# place both tables there (and qemu should not crash)
+$QEMU_IO -c "open -o cache-size=0 $TEST_IMG" -c 'write 0 64k' | _filter_qemu_io
+
 # success, all done
 echo '*** done'
 rm -f $seq.full
diff --git a/tests/qemu-iotests/103.out b/tests/qemu-iotests/103.out
index ddf6b5a..b7aaadf 100644
--- a/tests/qemu-iotests/103.out
+++ b/tests/qemu-iotests/103.out
@@ -1,14 +1,14 @@
 QA output created by 103
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 
 === Testing invalid option combinations ===
 
-qemu-io: can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
-qemu-io: can't open device TEST_DIR/t.IMGFMT: l2-cache-size may not exceed cache-size
-qemu-io: can't open device TEST_DIR/t.IMGFMT: refcount-cache-size may not exceed cache-size
-qemu-io: can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+can't open device TEST_DIR/t.IMGFMT: l2-cache-size may not exceed cache-size
+can't open device TEST_DIR/t.IMGFMT: refcount-cache-size may not exceed cache-size
+can't open device TEST_DIR/t.IMGFMT: cache-size, l2-cache-size and refcount-cache-size may not be set the same time
 
 === Testing valid option combinations ===
 
@@ -26,4 +26,9 @@
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing minimal L2 cache and COW ===
+
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 *** done
diff --git a/tests/qemu-iotests/104 b/tests/qemu-iotests/104
index b471aa5..726d467 100755
--- a/tests/qemu-iotests/104
+++ b/tests/qemu-iotests/104
@@ -25,20 +25,15 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
-_cleanup()
-{
-	_cleanup_test_img
-}
-trap "_cleanup; exit \$status" 0 1 2 3 15
+trap "exit \$status" 0 1 2 3 15
 
 # get standard environment, filters and checks
 . ./common.rc
 . ./common.filter
 
-_supported_fmt generic
+_supported_fmt raw qcow qcow2 qed vdi vmdk vhdx
 _supported_proto generic
 _supported_os Linux
 
@@ -47,8 +42,9 @@
 image_sizes="1024 1234"
 
 for s in $image_sizes; do
-    _make_test_img $s | _filter_img_create
+    _make_test_img $s
     _img_info | _filter_img_info
+    _cleanup_test_img
 done
 
 # success, all done
diff --git a/tests/qemu-iotests/104.out b/tests/qemu-iotests/104.out
index de27852..ab8d892 100644
--- a/tests/qemu-iotests/104.out
+++ b/tests/qemu-iotests/104.out
@@ -1,11 +1,11 @@
 QA output created by 104
 === Check qemu-img info output ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1024 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1024
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 1.0K (1024 bytes)
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1234 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1234
 image: TEST_DIR/t.IMGFMT
 file format: IMGFMT
 virtual size: 1.5K (1536 bytes)
diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105
index 9bae49e..3db4ce3 100755
--- a/tests/qemu-iotests/105
+++ b/tests/qemu-iotests/105
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here=`pwd`
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/107 b/tests/qemu-iotests/107
index 9862030..d7222dc 100755
--- a/tests/qemu-iotests/107
+++ b/tests/qemu-iotests/107
@@ -25,7 +25,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/107.out b/tests/qemu-iotests/107.out
index 93445b7..24f85d8 100644
--- a/tests/qemu-iotests/107.out
+++ b/tests/qemu-iotests/107.out
@@ -2,7 +2,7 @@
 
 === Updates should not write random data ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 read 65528/65528 bytes at offset 196616
diff --git a/tests/qemu-iotests/108 b/tests/qemu-iotests/108
index 12fc92a..2355d98 100755
--- a/tests/qemu-iotests/108
+++ b/tests/qemu-iotests/108
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
@@ -43,6 +42,8 @@
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+# This test directly modifies a refblock so it relies on refcount_bits being 16
+_unsupported_imgopts 'refcount_bits=\([^1]\|.\([^6]\|$\)\)'
 
 echo
 echo '=== Repairing an image without any refcount table ==='
diff --git a/tests/qemu-iotests/108.out b/tests/qemu-iotests/108.out
index 824d5cf..75bab8d 100644
--- a/tests/qemu-iotests/108.out
+++ b/tests/qemu-iotests/108.out
@@ -2,7 +2,7 @@
 
 === Repairing an image without any refcount table ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 65536/65536 bytes at offset 0
 64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 ERROR cluster 0 refcount=0 reference=1
@@ -22,7 +22,7 @@
 
 === Repairing unreferenced data cluster in new refblock area ===
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 wrote 111104/111104 bytes at offset 0
 108.500 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
 131072
@@ -49,7 +49,7 @@
 
 --- Otherwise clean ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 Repairing refcount block 1 is outside image
 The following inconsistencies were found and repaired:
 
@@ -61,7 +61,7 @@
 
 --- Refblock is unallocated ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 Repairing refcount block 1 is outside image
 ERROR cluster 16 refcount=0 reference=1
 Rebuilding refcount structure
@@ -78,7 +78,7 @@
 
 --- Signed overflow after the refblock ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 Repairing refcount block 1 is outside image
 ERROR could not resize image: Invalid argument
 Rebuilding refcount structure
@@ -94,7 +94,7 @@
 
 --- Unsigned overflow after the refblock ---
 
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
 Repairing refcount block 1 is outside image
 ERROR could not resize image: Invalid argument
 Rebuilding refcount structure
diff --git a/tests/qemu-iotests/109 b/tests/qemu-iotests/109
new file mode 100755
index 0000000..280ed27
--- /dev/null
+++ b/tests/qemu-iotests/109
@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# Test writing image headers of other formats into raw images
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    rm -f $TEST_IMG.src
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt raw
+_supported_proto file
+_supported_os Linux
+
+qemu_comm_method=qmp
+
+function run_qemu()
+{
+    local raw_img="$1"
+    local source_img="$2"
+    local qmp_format="$3"
+    local qmp_event="$4"
+
+    _launch_qemu -drive file="${source_img}",format=raw,cache=${CACHEMODE},id=src
+    _send_qemu_cmd $QEMU_HANDLE "{ 'execute': 'qmp_capabilities' }" "return"
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute':'drive-mirror', 'arguments':{
+            'device': 'src', 'target': '$raw_img', $qmp_format
+            'mode': 'existing', 'sync': 'full'}}" \
+        "return"
+
+    _send_qemu_cmd $QEMU_HANDLE '' "$qmp_event"
+    _send_qemu_cmd $QEMU_HANDLE '{"execute":"query-block-jobs"}' "return"
+    _cleanup_qemu
+}
+
+for fmt in qcow qcow2 qed vdi vmdk vpc; do
+
+    echo
+    echo "=== Writing a $fmt header into raw ==="
+    echo
+
+    _make_test_img 64M
+    TEST_IMG="$TEST_IMG.src" IMGFMT=$fmt _make_test_img 64M
+
+    # This first test should fail: The image format was probed, we may not
+    # write an image header at the start of the image
+    run_qemu "$TEST_IMG" "$TEST_IMG.src" "" "BLOCK_JOB_ERROR"
+    $QEMU_IO -c 'read -P 0 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+
+    # When raw was explicitly specified, the same must succeed
+    run_qemu "$TEST_IMG" "$TEST_IMG.src" "'format': 'raw'," "BLOCK_JOB_READY"
+    $QEMU_IMG compare -f raw -F raw "$TEST_IMG" "$TEST_IMG.src"
+
+done
+
+
+for sample_img in empty.bochs iotest-dirtylog-10G-4M.vhdx parallels-v1 \
+                  simple-pattern.cloop; do
+
+    echo
+    echo "=== Copying sample image $sample_img into raw ==="
+    echo
+
+    # Can't use _use_sample_img because that isn't designed to be used multiple
+    # times and it overwrites $TEST_IMG (both breaks cleanup)
+    _make_test_img 64M
+    bzcat "$SAMPLE_IMG_DIR/$sample_img.bz2" > "$TEST_IMG.src"
+
+    run_qemu "$TEST_IMG" "$TEST_IMG.src" "" "BLOCK_JOB_ERROR" | _filter_block_job_offset
+    $QEMU_IO -c 'read -P 0 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+    run_qemu "$TEST_IMG" "$TEST_IMG.src" "'format': 'raw'," "BLOCK_JOB_READY"
+    $QEMU_IMG compare -f raw -F raw "$TEST_IMG" "$TEST_IMG.src"
+done
+
+echo
+echo "=== Write legitimate MBR into raw ==="
+echo
+
+for sample_img in grub_mbr.raw; do
+    _make_test_img 64M
+    bzcat "$SAMPLE_IMG_DIR/$sample_img.bz2" > "$TEST_IMG.src"
+
+    run_qemu "$TEST_IMG" "$TEST_IMG.src" "" "BLOCK_JOB_READY"
+    $QEMU_IMG compare -f raw -F raw "$TEST_IMG" "$TEST_IMG.src"
+
+    run_qemu "$TEST_IMG" "$TEST_IMG.src" "'format': 'raw'," "BLOCK_JOB_READY"
+    $QEMU_IMG compare -f raw -F raw "$TEST_IMG" "$TEST_IMG.src"
+done
+
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/109.out b/tests/qemu-iotests/109.out
new file mode 100644
index 0000000..e5d70d7
--- /dev/null
+++ b/tests/qemu-iotests/109.out
@@ -0,0 +1,227 @@
+QA output created by 109
+
+=== Writing a qcow header into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 1024, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 1024, "offset": 1024, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 1024, "offset": 1024, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Writing a qcow2 header into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 197120, "offset": 512, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 197120, "offset": 197120, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 197120, "offset": 197120, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Writing a qed header into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 327680, "offset": 262144, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 327680, "offset": 327680, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 327680, "offset": 327680, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Writing a vdi header into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 1024, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 1024, "offset": 1024, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 1024, "offset": 1024, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Writing a vmdk header into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 65536, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 65536, "offset": 65536, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Writing a vpc header into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.raw.src', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2560, "offset": 0, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 2560, "offset": 2560, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 2560, "offset": 2560, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Copying sample image empty.bochs into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2560, "offset": OFFSET, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 2560, "offset": 2560, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 2560, "offset": 2560, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Copying sample image iotest-dirtylog-10G-4M.vhdx into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 31457280, "offset": OFFSET, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 31457280, "offset": 31457280, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 31457280, "offset": 31457280, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Copying sample image parallels-v1 into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 327680, "offset": OFFSET, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 327680, "offset": 327680, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 327680, "offset": 327680, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Copying sample image simple-pattern.cloop into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_ERROR", "data": {"device": "src", "operation": "write", "action": "report"}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "src", "len": 2048, "offset": OFFSET, "speed": 0, "type": "mirror", "error": "Operation not permitted"}}
+{"return": []}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 2048, "offset": 2048, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 2048, "offset": 2048, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+
+=== Write legitimate MBR into raw ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+{"return": {}}
+WARNING: Image format was not specified for 'TEST_DIR/t.raw' and probing guessed raw.
+Automatically detecting the format is dangerous for raw images, write operations on block 0 will be restricted.
+Specify the 'raw' format explicitly to remove the restrictions.
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 512, "offset": 512, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 512, "offset": 512, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "src", "len": 512, "offset": 512, "speed": 0, "type": "mirror"}}
+{"return": [{"io-status": "ok", "device": "src", "busy": false, "len": 512, "offset": 512, "paused": false, "speed": 0, "ready": true, "type": "mirror"}]}
+Warning: Image size mismatch!
+Images are identical.
+*** done
diff --git a/tests/qemu-iotests/110 b/tests/qemu-iotests/110
new file mode 100755
index 0000000..9de7369
--- /dev/null
+++ b/tests/qemu-iotests/110
@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# Test case for relative backing file names in complex BDS trees
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# Any format supporting backing files
+_supported_fmt qed qcow qcow2 vmdk
+_supported_proto file
+_supported_os Linux
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+
+TEST_IMG_REL=$(basename "$TEST_IMG")
+
+echo
+echo '=== Reconstructable filename ==='
+echo
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+_make_test_img -b "$TEST_IMG_REL.base" 64M
+# qemu should be able to reconstruct the filename, so relative backing names
+# should work
+TEST_IMG="json:{'driver':'$IMGFMT','file':{'driver':'file','filename':'$TEST_IMG'}}" \
+    _img_info | _filter_img_info
+
+echo
+echo '=== Non-reconstructable filename ==='
+echo
+
+# Across blkdebug without a config file, you cannot reconstruct filenames, so
+# qemu is incapable of knowing the directory of the top image
+TEST_IMG="json:{
+    'driver': '$IMGFMT',
+    'file': {
+        'driver': 'blkdebug',
+        'image': {
+            'driver': 'file',
+            'filename': '$TEST_IMG'
+        },
+        'set-state': [
+            {
+                'event': 'read_aio',
+                'new_state': 42
+            }
+        ]
+    }
+}" _img_info | _filter_img_info
+
+echo
+echo '=== Backing name is always relative to the backed image ==='
+echo
+
+# omit the image size; it should work anyway
+_make_test_img -b "$TEST_IMG_REL.base"
+
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/110.out b/tests/qemu-iotests/110.out
new file mode 100644
index 0000000..b3584ff
--- /dev/null
+++ b/tests/qemu-iotests/110.out
@@ -0,0 +1,22 @@
+QA output created by 110
+
+=== Reconstructable filename ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.base
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+backing file: t.IMGFMT.base (actual path: TEST_DIR/t.IMGFMT.base)
+
+=== Non-reconstructable filename ===
+
+image: json:{"driver": "IMGFMT", "file": {"set-state.0.event": "read_aio", "image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "set-state.0.new_state": 42}}
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+backing file: t.IMGFMT.base (cannot determine actual path)
+
+=== Backing name is always relative to the backed image ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=t.IMGFMT.base
+*** done
diff --git a/tests/qemu-iotests/111 b/tests/qemu-iotests/111
index 6011c94..a1c152d 100755
--- a/tests/qemu-iotests/111
+++ b/tests/qemu-iotests/111
@@ -26,7 +26,6 @@
 echo "QA output created by $seq"
 
 here="$PWD"
-tmp=/tmp/$$
 status=1	# failure is the default!
 
 _cleanup()
diff --git a/tests/qemu-iotests/112 b/tests/qemu-iotests/112
new file mode 100755
index 0000000..28eb9aa
--- /dev/null
+++ b/tests/qemu-iotests/112
@@ -0,0 +1,295 @@
+#!/bin/bash
+#
+# Test cases for different refcount_bits values
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qcow2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+# This test will set refcount_bits on its own which would conflict with the
+# manual setting; compat will be overridden as well
+_unsupported_imgopts refcount_bits 'compat=0.10'
+
+function print_refcount_bits()
+{
+    $QEMU_IMG info "$TEST_IMG" | sed -n '/refcount bits:/ s/^ *//p'
+}
+
+echo
+echo '=== refcount_bits limits ==='
+echo
+
+# Must be positive (non-zero)
+IMGOPTS="$IMGOPTS,refcount_bits=0" _make_test_img 64M
+# Must be positive (non-negative)
+IMGOPTS="$IMGOPTS,refcount_bits=-1" _make_test_img 64M
+# May not exceed 64
+IMGOPTS="$IMGOPTS,refcount_bits=128" _make_test_img 64M
+# Must be a power of two
+IMGOPTS="$IMGOPTS,refcount_bits=42" _make_test_img 64M
+
+# 1 is the minimum
+IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+print_refcount_bits
+
+# 64 is the maximum
+IMGOPTS="$IMGOPTS,refcount_bits=64" _make_test_img 64M
+print_refcount_bits
+
+# 16 is the default
+_make_test_img 64M
+print_refcount_bits
+
+echo
+echo '=== refcount_bits and compat=0.10 ==='
+echo
+
+# Should work
+IMGOPTS="$IMGOPTS,compat=0.10,refcount_bits=16" _make_test_img 64M
+print_refcount_bits
+
+# Should not work
+IMGOPTS="$IMGOPTS,compat=0.10,refcount_bits=1" _make_test_img 64M
+IMGOPTS="$IMGOPTS,compat=0.10,refcount_bits=64" _make_test_img 64M
+
+
+echo
+echo '=== Snapshot limit on refcount_bits=1 ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+print_refcount_bits
+
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Should fail for now; in the future, this might be supported by automatically
+# copying all clusters with overflowing refcount
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+
+# The new L1 table could/should be leaked
+_check_test_img
+
+echo
+echo '=== Snapshot limit on refcount_bits=2 ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=2" _make_test_img 64M
+print_refcount_bits
+
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Should succeed
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+$QEMU_IMG snapshot -c bar "$TEST_IMG"
+# Should fail (4th reference)
+$QEMU_IMG snapshot -c baz "$TEST_IMG"
+
+# The new L1 table could/should be leaked
+_check_test_img
+
+echo
+echo '=== Compressed clusters with refcount_bits=1 ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+print_refcount_bits
+
+# Both should fit into a single host cluster; instead of failing to increase the
+# refcount of that cluster, qemu should just allocate a new cluster and make
+# this operation succeed
+$QEMU_IO -c 'write -P 0 -c  0  64k' \
+         -c 'write -P 1 -c 64k 64k' \
+         "$TEST_IMG" | _filter_qemu_io
+
+_check_test_img
+
+echo
+echo '=== MSb set in 64 bit refcount ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=64" _make_test_img 64M
+print_refcount_bits
+
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Set the MSb in the refblock entry of the data cluster
+poke_file "$TEST_IMG" $((0x20028)) "\x80\x00\x00\x00\x00\x00\x00\x00"
+
+# Clear OFLAG_COPIED in the L2 entry of the data cluster
+poke_file "$TEST_IMG" $((0x40000)) "\x00\x00\x00\x00\x00\x05\x00\x00"
+
+# Try to write to that cluster (should work, even though the MSb is set)
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo '=== Snapshot on maximum 64 bit refcount value ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=64" _make_test_img 64M
+print_refcount_bits
+
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Set the refblock entry to the maximum value possible
+poke_file "$TEST_IMG" $((0x20028)) "\xff\xff\xff\xff\xff\xff\xff\xff"
+
+# Clear OFLAG_COPIED in the L2 entry of the data cluster
+poke_file "$TEST_IMG" $((0x40000)) "\x00\x00\x00\x00\x00\x05\x00\x00"
+
+# Try a snapshot (should correctly identify the overflow; may work in the future
+# by falling back to COW)
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+
+# The new L1 table could/should be leaked; and obviously the data cluster is
+# leaked (refcount=UINT64_MAX reference=1)
+_check_test_img
+
+echo
+echo '=== Amend from refcount_bits=16 to refcount_bits=1 ==='
+echo
+
+_make_test_img 64M
+print_refcount_bits
+
+$QEMU_IO -c 'write 16M 32M' "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG amend -o refcount_bits=1 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+echo
+echo '=== Amend from refcount_bits=1 to refcount_bits=64 ==='
+echo
+
+$QEMU_IMG amend -o refcount_bits=64 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+echo
+echo '=== Amend to compat=0.10 ==='
+echo
+
+# Should not work because refcount_bits needs to be 16 for compat=0.10
+$QEMU_IMG amend -o compat=0.10 "$TEST_IMG"
+print_refcount_bits
+# Should work
+$QEMU_IMG amend -o compat=0.10,refcount_bits=16 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+# Get back to compat=1.1 and refcount_bits=16
+$QEMU_IMG amend -o compat=1.1 "$TEST_IMG"
+print_refcount_bits
+# Should not work
+$QEMU_IMG amend -o refcount_bits=32,compat=0.10 "$TEST_IMG"
+print_refcount_bits
+
+echo
+echo '=== Amend with snapshot ==='
+echo
+
+$QEMU_IMG snapshot -c foo "$TEST_IMG"
+# Just to have different refcounts across the image
+$QEMU_IO -c 'write 0 16M' "$TEST_IMG" | _filter_qemu_io
+
+# Should not work (may work in the future by first decreasing all refcounts so
+# they fit into the target range by copying them)
+$QEMU_IMG amend -o refcount_bits=1 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+# Should work
+$QEMU_IMG amend -o refcount_bits=2 "$TEST_IMG"
+_check_test_img
+print_refcount_bits
+
+echo
+echo '=== Testing too many references for check ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=1" _make_test_img 64M
+print_refcount_bits
+
+# This cluster should be created at 0x50000
+$QEMU_IO -c 'write 0 64k' "$TEST_IMG" | _filter_qemu_io
+# Now make the second L2 entry (the L2 table should be at 0x40000) point to that
+# cluster, so we have two references
+poke_file "$TEST_IMG" $((0x40008)) "\x80\x00\x00\x00\x00\x05\x00\x00"
+
+# This should say "please use amend"
+_check_test_img -r all
+
+# So we do that
+$QEMU_IMG amend -o refcount_bits=2 "$TEST_IMG"
+print_refcount_bits
+
+# And try again
+_check_test_img -r all
+
+echo
+echo '=== Multiple walks necessary during amend ==='
+echo
+
+IMGOPTS="$IMGOPTS,refcount_bits=1,cluster_size=512" _make_test_img 64k
+
+# Cluster 0 is the image header, clusters 1 to 4 are used by the L1 table, a
+# single L2 table, the reftable and a single refblock. This creates 58 data
+# clusters (actually, the L2 table is created here, too), so in total there are
+# then 63 used clusters in the image. With a refcount width of 64, one refblock
+# describes 64 clusters (512 bytes / 64 bits/entry = 64 entries), so this will
+# make the first refblock in the amended image have exactly one free entry.
+$QEMU_IO -c "write 0 $((58 * 512))" "$TEST_IMG" | _filter_qemu_io
+
+# Now change the refcount width; since the first new refblock will have exactly
+# one free entry, that entry will be used to store its own reference. No other
+# refblocks are needed, so then the new reftable will be allocated; since the
+# first new refblock is completely filled up, this will require a new refblock
+# which is why the refcount width changing function will need to run through
+# everything one more time until the allocations are stable.
+# Having more walks than usual should be visible as regressing progress (from
+# 66.67 % (2/3 walks) to 50.00 % (2/4 walks)).
+$QEMU_IMG amend -o refcount_bits=64 -p "$TEST_IMG" | tr '\r' '\n' \
+                                                   | grep -A 1 '66.67'
+print_refcount_bits
+
+_check_test_img
+
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/112.out b/tests/qemu-iotests/112.out
new file mode 100644
index 0000000..81b04d1
--- /dev/null
+++ b/tests/qemu-iotests/112.out
@@ -0,0 +1,155 @@
+QA output created by 112
+
+=== refcount_bits limits ===
+
+qemu-img: TEST_DIR/t.IMGFMT: Refcount width must be a power of two and may not exceed 64 bits
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+qemu-img: TEST_DIR/t.IMGFMT: Refcount width must be a power of two and may not exceed 64 bits
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 refcount_bits=-1
+qemu-img: TEST_DIR/t.IMGFMT: Refcount width must be a power of two and may not exceed 64 bits
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+qemu-img: TEST_DIR/t.IMGFMT: Refcount width must be a power of two and may not exceed 64 bits
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 1
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 64
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 16
+
+=== refcount_bits and compat=0.10 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 16
+qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use or greater)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+qemu-img: TEST_DIR/t.IMGFMT: Different refcount widths than 16 bits require compatibility level 1.1 or above (use or greater)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+
+=== Snapshot limit on refcount_bits=1 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 1
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not create snapshot 'foo': -22 (Invalid argument)
+Leaked cluster 6 refcount=1 reference=0
+
+1 leaked clusters were found on the image.
+This means waste of disk space, but no harm to data.
+
+=== Snapshot limit on refcount_bits=2 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 2
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not create snapshot 'baz': -22 (Invalid argument)
+Leaked cluster 7 refcount=1 reference=0
+
+1 leaked clusters were found on the image.
+This means waste of disk space, but no harm to data.
+
+=== Compressed clusters with refcount_bits=1 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 1
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+=== MSb set in 64 bit refcount ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 64
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Snapshot on maximum 64 bit refcount value ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 64
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Could not create snapshot 'foo': -22 (Invalid argument)
+Leaked cluster 5 refcount=18446744073709551615 reference=1
+Leaked cluster 6 refcount=1 reference=0
+
+2 leaked clusters were found on the image.
+This means waste of disk space, but no harm to data.
+
+=== Amend from refcount_bits=16 to refcount_bits=1 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 16
+wrote 33554432/33554432 bytes at offset 16777216
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+refcount bits: 1
+
+=== Amend from refcount_bits=1 to refcount_bits=64 ===
+
+No errors were found on the image.
+refcount bits: 64
+
+=== Amend to compat=0.10 ===
+
+qemu-img: compat=0.10 requires refcount_bits=16
+qemu-img: Error while amending options: Operation not supported
+refcount bits: 64
+No errors were found on the image.
+refcount bits: 16
+refcount bits: 16
+qemu-img: Different refcount widths than 16 bits require compatibility level 1.1 or above (use compat=1.1 or greater)
+qemu-img: Error while amending options: Invalid argument
+refcount bits: 16
+
+=== Amend with snapshot ===
+
+wrote 16777216/16777216 bytes at offset 0
+16 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Cannot decrease refcount entry width to 1 bits: Cluster at offset 0x50000 has a refcount of 2
+qemu-img: Error while amending options: Invalid argument
+No errors were found on the image.
+refcount bits: 16
+No errors were found on the image.
+refcount bits: 2
+
+=== Testing too many references for check ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+refcount bits: 1
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+ERROR: overflow cluster offset=0x50000
+Use qemu-img amend to increase the refcount entry width or qemu-img convert to create a clean copy if the image cannot be opened for writing
+
+1 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+refcount bits: 2
+ERROR cluster 5 refcount=1 reference=2
+Repairing cluster 5 refcount=1 reference=2
+Repairing OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=2
+Repairing OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=2
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    3 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+
+=== Multiple walks necessary during amend ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+wrote 29696/29696 bytes at offset 0
+29 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+    (66.67/100%)
+    (50.00/100%)
+refcount bits: 64
+No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/113 b/tests/qemu-iotests/113
new file mode 100755
index 0000000..19b68b2
--- /dev/null
+++ b/tests/qemu-iotests/113
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Test case for accessing creation options on image formats and
+# protocols not supporting image creation
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# We can only test one format here because we need its sample file
+_supported_fmt bochs
+_supported_proto nbd
+_supported_os Linux
+
+echo
+echo '=== Unsupported image creation in qemu-img create ==='
+echo
+
+$QEMU_IMG create -f $IMGFMT nbd://example.com 2>&1 64M | _filter_imgfmt
+
+echo
+echo '=== Unsupported image creation in qemu-img convert ==='
+echo
+
+# We could use any input image format here, but this is a bochs test, so just
+# use the bochs image
+_use_sample_img empty.bochs.bz2
+$QEMU_IMG convert -f $IMGFMT -O $IMGFMT "$TEST_IMG" nbd://example.com 2>&1 \
+    | _filter_imgfmt
+
+echo
+echo '=== Unsupported format in qemu-img amend ==='
+echo
+
+# The protocol does not matter here
+_use_sample_img empty.bochs.bz2
+$QEMU_IMG amend -f $IMGFMT -o foo=bar "$TEST_IMG" 2>&1 | _filter_imgfmt
+
+
+# success, all done
+echo
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/113.out b/tests/qemu-iotests/113.out
new file mode 100644
index 0000000..00bdfd6
--- /dev/null
+++ b/tests/qemu-iotests/113.out
@@ -0,0 +1,15 @@
+QA output created by 113
+
+=== Unsupported image creation in qemu-img create ===
+
+qemu-img: nbd://example.com: Format driver 'IMGFMT' does not support image creation
+
+=== Unsupported image creation in qemu-img convert ===
+
+qemu-img: Format driver 'IMGFMT' does not support image creation
+
+=== Unsupported format in qemu-img amend ===
+
+qemu-img: Format driver 'IMGFMT' does not support any options to amend
+
+*** done
diff --git a/tests/qemu-iotests/114 b/tests/qemu-iotests/114
new file mode 100755
index 0000000..f110d4f
--- /dev/null
+++ b/tests/qemu-iotests/114
@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Test invalid backing file format in qcow2 images
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+_make_test_img -b "$TEST_IMG.base" 64M
+
+# Set an invalid backing file format
+$PYTHON qcow2.py "$TEST_IMG" add-header-ext 0xE2792ACA "foo"
+_img_info
+
+# Try opening the image. Should fail (and not probe) in the first case, but
+# overriding the backing file format should be possible.
+$QEMU_IO -c "open $TEST_IMG" -c "read 0 4k" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "open -o backing.driver=$IMGFMT $TEST_IMG" -c "read 0 4k" | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/114.out b/tests/qemu-iotests/114.out
new file mode 100644
index 0000000..b6d10e4
--- /dev/null
+++ b/tests/qemu-iotests/114.out
@@ -0,0 +1,13 @@
+QA output created by 114
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base 
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+cluster_size: 65536
+backing file: TEST_DIR/t.IMGFMT.base
+backing file format: foo
+can't open device TEST_DIR/t.qcow2: Could not open backing file: Unknown driver 'foo'
+read 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/115 b/tests/qemu-iotests/115
new file mode 100755
index 0000000..665c2ea
--- /dev/null
+++ b/tests/qemu-iotests/115
@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# Test case for non-self-referential qcow2 refcount blocks
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+# This test relies on refcounts being 64 bits wide (which does not work with
+# compat=0.10)
+_unsupported_imgopts 'refcount_bits=\([^6]\|.\([^4]\|$\)\)' 'compat=0.10'
+
+echo
+echo '=== Testing large refcount and L1 table ==='
+echo
+
+# Create an image with an L1 table and a refcount table that each span twice the
+# number of clusters which can be described by a single refblock; therefore, at
+# least two refblocks cannot count their own refcounts because all the clusters
+# they describe are part of the L1 table or refcount table.
+
+# One refblock can describe (with cluster_size=512 and refcount_bits=64)
+# 512/8 = 64 clusters, therefore the L1 table should cover 128 clusters, which
+# equals 128 * (512/8) = 8192 entries (actually, 8192 - 512/8 = 8129 would
+# suffice, but it does not really matter). 8192 L2 tables can in turn describe
+# 8192 * 512/8 = 524,288 clusters which cover a space of 256 MB.
+
+# Since with refcount_bits=64 every refcount block entry is 64 bits wide (just
+# like the L2 table entries), the same calculation applies to the refcount table
+# as well; the difference is that while for the L1 table the guest disk size is
+# concerned, for the refcount table it is the image length that has to be at
+# least 256 MB. We can achieve that by using preallocation=metadata for an image
+# which has a guest disk size of 256 MB.
+
+IMGOPTS="$IMGOPTS,refcount_bits=64,cluster_size=512,preallocation=metadata" \
+    _make_test_img 256M
+
+# We know for sure that the L1 and refcount tables do not overlap with any other
+# structure because the metadata overlap checks would have caught that case.
+
+# Because qemu refuses to open qcow2 files whose L1 table does not cover the
+# whole guest disk size, it is definitely large enough. On the other hand, to
+# test whether the refcount table is large enough, we simply have to verify that
+# indeed all the clusters are allocated, which is done by qemu-img check.
+
+# The final thing we need to test is whether the tables are actually covered by
+# refcount blocks; since all clusters of the tables are referenced, we can use
+# qemu-img check for that purpose, too.
+
+$QEMU_IMG check "$TEST_IMG" | \
+    sed -e 's/^.* = \([0-9]\+\.[0-9]\+% allocated\).*\(clusters\)$/\1 \2/' \
+        -e '/^Image end offset/d'
+
+# (Note that we cannot use _check_test_img because that function filters out the
+# allocation status)
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/115.out b/tests/qemu-iotests/115.out
new file mode 100644
index 0000000..dbdad17
--- /dev/null
+++ b/tests/qemu-iotests/115.out
@@ -0,0 +1,8 @@
+QA output created by 115
+
+=== Testing large refcount and L1 table ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=268435456 preallocation=metadata
+No errors were found on the image.
+100.00% allocated clusters
+*** done
diff --git a/tests/qemu-iotests/116 b/tests/qemu-iotests/116
new file mode 100755
index 0000000..df0172f
--- /dev/null
+++ b/tests/qemu-iotests/116
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Test error code paths for invalid QED images
+#
+# The aim of this test is to exercise the error paths in qed_open() to ensure
+# there are no crashes with invalid input files.
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qed
+_supported_proto generic
+_supported_os Linux
+
+
+size=128M
+
+echo
+echo "== truncated header cluster =="
+_make_test_img $size
+truncate -s 512 "$TEST_IMG"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== invalid header magic =="
+_make_test_img $size
+poke_file "$TEST_IMG" "0" "QEDX"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== invalid cluster size =="
+_make_test_img $size
+poke_file "$TEST_IMG" "4" "\xff\xff\xff\xff"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== invalid table size =="
+_make_test_img $size
+poke_file "$TEST_IMG" "8" "\xff\xff\xff\xff"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== invalid header size =="
+_make_test_img $size
+poke_file "$TEST_IMG" "12" "\xff\xff\xff\xff"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== invalid L1 table offset =="
+_make_test_img $size
+poke_file "$TEST_IMG" "40" "\xff\xff\xff\xff\xff\xff\xff\xff"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== invalid image size =="
+_make_test_img $size
+poke_file "$TEST_IMG" "48" "\xff\xff\xff\xff\xff\xff\xff\xff"
+$QEMU_IO -f "$IMGFMT" -c "read 0 $size" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/116.out b/tests/qemu-iotests/116.out
new file mode 100644
index 0000000..1f11d44
--- /dev/null
+++ b/tests/qemu-iotests/116.out
@@ -0,0 +1,37 @@
+QA output created by 116
+
+== truncated header cluster ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+no file open, try 'help open'
+
+== invalid header magic ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Image not in QED format
+no file open, try 'help open'
+
+== invalid cluster size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+no file open, try 'help open'
+
+== invalid table size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+no file open, try 'help open'
+
+== invalid header size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+no file open, try 'help open'
+
+== invalid L1 table offset ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+no file open, try 'help open'
+
+== invalid image size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+can't open device TEST_DIR/t.qed: Could not open 'TEST_DIR/t.qed': Invalid argument
+no file open, try 'help open'
+*** done
diff --git a/tests/qemu-iotests/117 b/tests/qemu-iotests/117
new file mode 100755
index 0000000..9385b3f
--- /dev/null
+++ b/tests/qemu-iotests/117
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Test case for shared BDS between backend trees
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+_make_test_img 64k
+
+_launch_qemu
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'blockdev-add',
+       'arguments': { 'options': { 'id': 'protocol',
+                                   'driver': 'file',
+                                   'filename': '$TEST_IMG' } } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'blockdev-add',
+       'arguments': { 'options': { 'id': 'format',
+                                   'driver': '$IMGFMT',
+                                   'file': 'protocol' } } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line': 'qemu-io format \"write -P 42 0 64k\"' } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+_check_test_img
+
+$QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/117.out b/tests/qemu-iotests/117.out
new file mode 100644
index 0000000..f52dc1a
--- /dev/null
+++ b/tests/qemu-iotests/117.out
@@ -0,0 +1,14 @@
+QA output created by 117
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+{"return": {}}
+{"return": {}}
+{"return": {}}
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+No errors were found on the image.
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/118 b/tests/qemu-iotests/118
new file mode 100755
index 0000000..9e5951f
--- /dev/null
+++ b/tests/qemu-iotests/118
@@ -0,0 +1,677 @@
+#!/usr/bin/env python
+#
+# Test case for the QMP 'change' command and all other associated
+# commands
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import stat
+import time
+import iotests
+from iotests import qemu_img
+
+old_img = os.path.join(iotests.test_dir, 'test0.img')
+new_img = os.path.join(iotests.test_dir, 'test1.img')
+
+class ChangeBaseClass(iotests.QMPTestCase):
+    has_opened = False
+    has_closed = False
+
+    def process_events(self):
+        for event in self.vm.get_qmp_events(wait=False):
+            if (event['event'] == 'DEVICE_TRAY_MOVED' and
+                event['data']['device'] == 'drive0'):
+                if event['data']['tray-open'] == False:
+                    self.has_closed = True
+                else:
+                    self.has_opened = True
+
+    def wait_for_open(self):
+        if not self.has_real_tray:
+            return
+
+        timeout = time.clock() + 3
+        while not self.has_opened and time.clock() < timeout:
+            self.process_events()
+        if not self.has_opened:
+            self.fail('Timeout while waiting for the tray to open')
+
+    def wait_for_close(self):
+        if not self.has_real_tray:
+            return
+
+        timeout = time.clock() + 3
+        while not self.has_closed and time.clock() < timeout:
+            self.process_events()
+        if not self.has_opened:
+            self.fail('Timeout while waiting for the tray to close')
+
+class GeneralChangeTestsBaseClass(ChangeBaseClass):
+    def test_change(self):
+        result = self.vm.qmp('change', device='drive0', target=new_img,
+                                       arg=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_blockdev_change_medium(self):
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_eject(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+    def test_tray_eject_change(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_tray_open_close(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        if self.has_real_tray or not self.was_empty:
+            self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_tray_eject_close(self):
+        result = self.vm.qmp('eject', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+    def test_tray_open_change(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_cycle(self):
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0', force=True)
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        if self.was_empty == True:
+            self.assert_qmp_absent(result, 'return[0]/inserted')
+        else:
+            self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_close()
+
+        result = self.vm.qmp('query-block')
+        if self.has_real_tray:
+            self.assert_qmp(result, 'return[0]/tray_open', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_close_on_closed(self):
+        result = self.vm.qmp('blockdev-close-tray', device='drive0')
+        # Should be a no-op
+        self.assert_qmp(result, 'return', {})
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+    def test_remove_on_closed(self):
+        if not self.has_real_tray:
+            return
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_insert_on_closed(self):
+        if not self.has_real_tray:
+            return
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestInitiallyFilled(GeneralChangeTestsBaseClass):
+    was_empty = False
+
+    def setUp(self, media, interface):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1440k')
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM().add_drive(old_img, 'media=%s' % media, interface)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(old_img)
+        os.remove(new_img)
+
+    def test_insert_on_filled(self):
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('blockdev-open-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+class TestInitiallyEmpty(GeneralChangeTestsBaseClass):
+    was_empty = True
+
+    def setUp(self, media, interface):
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM().add_drive(None, 'media=%s' % media, interface)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(new_img)
+
+    def test_remove_on_empty(self):
+        result = self.vm.qmp('blockdev-open-tray', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.wait_for_open()
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        # Should be a no-op
+        self.assert_qmp(result, 'return', {})
+
+class TestCDInitiallyFilled(TestInitiallyFilled):
+    TestInitiallyFilled = TestInitiallyFilled
+    has_real_tray = True
+
+    def setUp(self):
+        self.TestInitiallyFilled.setUp(self, 'cdrom', 'ide')
+
+class TestCDInitiallyEmpty(TestInitiallyEmpty):
+    TestInitiallyEmpty = TestInitiallyEmpty
+    has_real_tray = True
+
+    def setUp(self):
+        self.TestInitiallyEmpty.setUp(self, 'cdrom', 'ide')
+
+class TestFloppyInitiallyFilled(TestInitiallyFilled):
+    TestInitiallyFilled = TestInitiallyFilled
+    has_real_tray = False
+
+    def setUp(self):
+        self.TestInitiallyFilled.setUp(self, 'disk', 'floppy')
+
+class TestFloppyInitiallyEmpty(TestInitiallyEmpty):
+    TestInitiallyEmpty = TestInitiallyEmpty
+    has_real_tray = False
+
+    def setUp(self):
+        self.TestInitiallyEmpty.setUp(self, 'disk', 'floppy')
+        # FDDs not having a real tray and there not being a medium inside the
+        # tray at startup means the tray will be considered open
+        self.has_opened = True
+
+class TestChangeReadOnly(ChangeBaseClass):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1440k')
+        qemu_img('create', '-f', iotests.imgfmt, new_img, '1440k')
+        self.vm = iotests.VM()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.chmod(old_img, 0666)
+        os.chmod(new_img, 0666)
+        os.remove(old_img)
+        os.remove(new_img)
+
+    def test_ro_ro_retain(self):
+        os.chmod(old_img, 0444)
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_ro_rw_retain(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_rw_ro_retain(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assertEquals(self.vm.get_qmp_events(wait=False), [])
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_ro_rw(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-write')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_rw_ro(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-only')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_rw_ro(self):
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-only')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_ro_rw(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium',
+                             device='drive0',
+                             filename=new_img,
+                             format=iotests.imgfmt,
+                             read_only_mode='read-write')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_make_rw_ro_by_retain(self):
+        os.chmod(old_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk,read-only=on', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+    def test_make_ro_rw_by_retain(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-change-medium', device='drive0',
+                                                       filename=new_img,
+                                                       format=iotests.imgfmt,
+                                                       read_only_mode='retain')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def test_rw_ro_cycle(self):
+        os.chmod(new_img, 0444)
+        self.vm.add_drive(old_img, 'media=disk', 'floppy')
+        self.vm.launch()
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'new',
+                                      'driver': iotests.imgfmt,
+                                      'read-only': True,
+                                      'file': {'filename': new_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', False)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='new')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/ro', True)
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+
+GeneralChangeTestsBaseClass = None
+TestInitiallyFilled = None
+TestInitiallyEmpty = None
+
+
+class TestBlockJobsAfterCycle(ChangeBaseClass):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, old_img, '1M')
+
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'id': 'drive0',
+                                      'driver': 'null-co'})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/format', 'null-co')
+
+        # For device-less BBs, calling blockdev-open-tray or blockdev-close-tray
+        # is not necessary
+        result = self.vm.qmp('x-blockdev-remove-medium', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp_absent(result, 'return[0]/inserted')
+
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'node0',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'filename': old_img,
+                                               'driver': 'file'}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium', device='drive0',
+                                                       node_name='node0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', old_img)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(old_img)
+        try:
+            os.remove(new_img)
+        except OSError:
+            pass
+
+    def test_snapshot_and_commit(self):
+        # We need backing file support
+        if iotests.imgfmt != 'qcow2' and iotests.imgfmt != 'qed':
+            return
+
+        result = self.vm.qmp('blockdev-snapshot-sync', device='drive0',
+                                                       snapshot_file=new_img,
+                                                       format=iotests.imgfmt)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/image/filename', new_img)
+        self.assert_qmp(result,
+                        'return[0]/inserted/image/backing-image/filename',
+                        old_img)
+
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait(name='BLOCK_JOB_READY')
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+
+        result = self.vm.qmp('block-job-complete', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait(name='BLOCK_JOB_COMPLETED')
+
+
+if __name__ == '__main__':
+    if iotests.qemu_default_machine != 'pc':
+        # We need floppy and IDE CD-ROM
+        iotests.notrun('not suitable for this machine type: %s' %
+                       iotests.qemu_default_machine)
+    # Need to support image creation
+    iotests.main(supported_fmts=['vpc', 'parallels', 'qcow', 'vdi', 'qcow2',
+                                 'vmdk', 'raw', 'vhdx', 'qed'])
diff --git a/tests/qemu-iotests/118.out b/tests/qemu-iotests/118.out
new file mode 100644
index 0000000..6a91713
--- /dev/null
+++ b/tests/qemu-iotests/118.out
@@ -0,0 +1,5 @@
+...........................................................
+----------------------------------------------------------------------
+Ran 59 tests
+
+OK
diff --git a/tests/qemu-iotests/119 b/tests/qemu-iotests/119
new file mode 100755
index 0000000..4f34fb4
--- /dev/null
+++ b/tests/qemu-iotests/119
@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# NBD test case for overriding BDRV_O_PROTOCOL by explicitly specifying
+# a driver
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw
+_supported_proto nbd
+_supported_os Linux
+
+_make_test_img 64M
+# This should not crash
+echo "{'execute': 'qmp_capabilities'}
+      {'execute': 'human-monitor-command',
+       'arguments': {'command-line': 'qemu-io drv \"read -P 0 0 64k\"'}}
+      {'execute': 'quit'}" \
+    | $QEMU -nographic -drive id=drv,if=none,file="$TEST_IMG",driver=nbd \
+            -qmp stdio -nodefaults \
+    | _filter_qmp | _filter_qemu_io
+
+# success, all done
+echo
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/119.out b/tests/qemu-iotests/119.out
new file mode 100644
index 0000000..58e7114
--- /dev/null
+++ b/tests/qemu-iotests/119.out
@@ -0,0 +1,11 @@
+QA output created by 119
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+QMP_VERSION
+{"return": {}}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+
+*** done
diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120
new file mode 100755
index 0000000..4f88a67
--- /dev/null
+++ b/tests/qemu-iotests/120
@@ -0,0 +1,64 @@
+#!/bin/bash
+#
+# Non-NBD test cases for overriding BDRV_O_PROTOCOL by explicitly
+# specifying a driver
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_proto file
+_supported_os Linux
+
+_make_test_img 64M
+
+echo "{'execute': 'qmp_capabilities'}
+      {'execute': 'human-monitor-command',
+       'arguments': {'command-line': 'qemu-io drv \"write -P 42 0 64k\"'}}
+      {'execute': 'quit'}" \
+    | $QEMU -qmp stdio -nographic -nodefaults \
+            -drive id=drv,if=none,file="$TEST_IMG",driver=raw,file.driver=$IMGFMT \
+    | _filter_qmp | _filter_qemu_io
+$QEMU_IO -c 'read -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IO_PROG -c 'read -P 42 0 64k' \
+    "json:{'driver': 'raw', 'file': {'driver': '$IMGFMT', 'file': {'filename': '$TEST_IMG'}}}" \
+    | _filter_qemu_io
+
+# success, all done
+echo
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/120.out b/tests/qemu-iotests/120.out
new file mode 100644
index 0000000..9131b1b
--- /dev/null
+++ b/tests/qemu-iotests/120.out
@@ -0,0 +1,15 @@
+QA output created by 120
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+QMP_VERSION
+{"return": {}}
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+*** done
diff --git a/tests/qemu-iotests/121 b/tests/qemu-iotests/121
new file mode 100755
index 0000000..1307b4e
--- /dev/null
+++ b/tests/qemu-iotests/121
@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Test cases for qcow2 refcount table growth
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo
+echo '=== New refcount structures may not conflict with existing structures ==='
+
+echo
+echo '--- Test 1 ---'
+echo
+
+# Preallocation speeds up the write operation, but preallocating everything will
+# destroy the purpose of the write; so preallocate one KB less than what would
+# cause a reftable growth...
+IMGOPTS='preallocation=metadata,cluster_size=1k' _make_test_img 64512K
+# ...and make the image the desired size afterwards.
+$QEMU_IMG resize "$TEST_IMG" 65M
+
+# The first write results in a growth of the refcount table during an allocation
+# which has precisely the required size so that the new refcount block allocated
+# in alloc_refcount_block() is right after cluster_index; this did lead to a
+# different refcount block being written to disk (a zeroed cluster) than what is
+# cached (a refblock with one entry having a refcount of 1), and the second
+# write would then result in that cached cluster being marked dirty and then
+# in it being written to disk.
+# This should not happen, the new refcount structures may not conflict with
+# new_block.
+# (Note that for some reason, 'write 63M 1K' does not trigger the problem)
+$QEMU_IO -c 'write 62M 1025K' -c 'write 64M 1M' "$TEST_IMG" | _filter_qemu_io
+
+_check_test_img
+
+
+echo
+echo '--- Test 2 ---'
+echo
+
+IMGOPTS='preallocation=metadata,cluster_size=1k' _make_test_img 64513K
+# This results in an L1 table growth which in turn results in some clusters at
+# the start of the image becoming free
+$QEMU_IMG resize "$TEST_IMG" 65M
+
+# This write results in a refcount table growth; but the refblock allocated
+# immediately before that (new_block) takes cluster index 4 (which is now free)
+# and is thus not self-describing (in contrast to test 1, where new_block was
+# self-describing). The refcount table growth algorithm then used to place the
+# new refcount structures at cluster index 65536 (which is the same as the
+# cluster_index parameter in this case), allocating a new refcount block for
+# that cluster while new_block already existed, leaking new_block.
+# Therefore, the new refcount structures may not be put at cluster_index
+# (because new_block already describes that cluster, and the new structures try
+# to be self-describing).
+$QEMU_IO -c 'write 63M 130K' "$TEST_IMG" | _filter_qemu_io
+
+_check_test_img
+
+
+# success, all done
+echo
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/121.out b/tests/qemu-iotests/121.out
new file mode 100644
index 0000000..5961a44
--- /dev/null
+++ b/tests/qemu-iotests/121.out
@@ -0,0 +1,23 @@
+QA output created by 121
+
+=== New refcount structures may not conflict with existing structures ===
+
+--- Test 1 ---
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66060288 preallocation=metadata
+Image resized.
+wrote 1049600/1049600 bytes at offset 65011712
+1.001 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1048576/1048576 bytes at offset 67108864
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+--- Test 2 ---
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=66061312 preallocation=metadata
+Image resized.
+wrote 133120/133120 bytes at offset 66060288
+130 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+No errors were found on the image.
+
+*** done
diff --git a/tests/qemu-iotests/122 b/tests/qemu-iotests/122
new file mode 100755
index 0000000..45b359c
--- /dev/null
+++ b/tests/qemu-iotests/122
@@ -0,0 +1,222 @@
+#!/bin/bash
+#
+# Test some qemu-img convert cases
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    rm -f "$TEST_IMG".[123]
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+
+TEST_IMG="$TEST_IMG".base _make_test_img 64M
+$QEMU_IO -c "write -P 0x11 0 64M" "$TEST_IMG".base 2>&1 | _filter_qemu_io | _filter_testdir
+
+
+echo
+echo "=== Check allocation status regression with -B ==="
+echo
+
+_make_test_img -b "$TEST_IMG".base
+$QEMU_IO -c "write -P 0x22 0 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG convert -O $IMGFMT -B "$TEST_IMG".base "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IMG map "$TEST_IMG".orig | _filter_qemu_img_map
+
+
+echo
+echo "=== Check that zero clusters are kept in overlay ==="
+echo
+
+_make_test_img -b "$TEST_IMG".base
+
+$QEMU_IO -c "write -P 0 0 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG convert -O $IMGFMT -B "$TEST_IMG".base "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG convert -O $IMGFMT -c -B "$TEST_IMG".base "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+
+$QEMU_IO -c "write -z 0 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG convert -O $IMGFMT -B "$TEST_IMG".base "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG convert -O $IMGFMT -c -B "$TEST_IMG".base "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+
+
+echo
+echo "=== Concatenate multiple source images ==="
+echo
+
+TEST_IMG="$TEST_IMG".1 _make_test_img 4M
+TEST_IMG="$TEST_IMG".2 _make_test_img 4M
+TEST_IMG="$TEST_IMG".3 _make_test_img 4M
+
+$QEMU_IO -c "write -P 0x11 0 64k" "$TEST_IMG".1 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write -P 0x22 0 64k" "$TEST_IMG".2 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write -P 0x33 0 64k" "$TEST_IMG".3 2>&1 | _filter_qemu_io | _filter_testdir
+
+$QEMU_IMG convert -O $IMGFMT "$TEST_IMG".[123] "$TEST_IMG"
+$QEMU_IMG map "$TEST_IMG" | _filter_qemu_img_map
+$QEMU_IO -c "read -P 0x11 0 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x22 4M 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x33 8M 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+$QEMU_IMG convert -c -O $IMGFMT "$TEST_IMG".[123] "$TEST_IMG"
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+$QEMU_IO -c "read -P 0x11 0 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x22 4M 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x33 8M 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+# -B can't be combined with concatenation
+$QEMU_IMG convert -O $IMGFMT -B "$TEST_IMG".base "$TEST_IMG".[123] "$TEST_IMG"
+$QEMU_IMG convert -O $IMGFMT -c -B "$TEST_IMG".base "$TEST_IMG".[123] "$TEST_IMG"
+
+
+echo
+echo "=== Compression with misaligned allocations and image sizes ==="
+echo
+
+TEST_IMG="$TEST_IMG".1 _make_test_img 1023k -o cluster_size=1024
+TEST_IMG="$TEST_IMG".2 _make_test_img 1023k -o cluster_size=1024
+
+$QEMU_IO -c "write -P 0x11   16k  16k" "$TEST_IMG".1 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write -P 0x22  130k 130k" "$TEST_IMG".1 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write -P 0x33 1022k   1k" "$TEST_IMG".1 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write -P 0x44    0k   1k" "$TEST_IMG".2 2>&1 | _filter_qemu_io | _filter_testdir
+
+$QEMU_IMG convert -c -O $IMGFMT "$TEST_IMG".[12] "$TEST_IMG"
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+$QEMU_IO -c "read -P 0       0k   16k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x11   16k   16k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0      32k   98k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x22  130k  130k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0     260k  762k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x33 1022k    1k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x44 1023k    1k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0    1024k 1022k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+
+echo
+echo "=== Full allocation with -S 0 ==="
+echo
+
+# Standalone image
+_make_test_img 64M
+$QEMU_IO -c "write -P 0x22 0 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write -P 0 3M 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo convert -S 0:
+$QEMU_IMG convert -O $IMGFMT -S 0 "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0x22 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0 3M 61M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+echo
+echo convert -c -S 0:
+$QEMU_IMG convert -O $IMGFMT -c -S 0 "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0x22 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0 3M 61M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+# With backing file
+TEST_IMG="$TEST_IMG".base _make_test_img 64M
+$QEMU_IO -c "write -P 0x11 0 32M" "$TEST_IMG".base 2>&1 | _filter_qemu_io | _filter_testdir
+
+_make_test_img -b "$TEST_IMG".base 64M
+$QEMU_IO -c "write -P 0x22 0 3M" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo convert -S 0 with source backing file:
+$QEMU_IMG convert -O $IMGFMT -S 0 "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0x22 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x11 3M 29M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0 32M 32M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+echo
+echo convert -c -S 0 with source backing file:
+$QEMU_IMG convert -O $IMGFMT -c -S 0 "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0x22 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x11 3M 29M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0 32M 32M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+# With keeping the backing file
+echo
+echo convert -S 0 -B ...
+$QEMU_IMG convert -O $IMGFMT -S 0 "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0x22 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x11 3M 29M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0 32M 32M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+echo
+echo convert -c -S 0 -B ...
+$QEMU_IMG convert -O $IMGFMT -c -S 0 "$TEST_IMG" "$TEST_IMG".orig
+$QEMU_IO -c "read -P 0x22 0 3M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0x11 3M 29M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "read -P 0 32M 32M" "$TEST_IMG".orig 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+
+echo
+echo "=== Non-zero -S ==="
+echo
+
+_make_test_img 64M -o cluster_size=1k
+$QEMU_IO -c "write -P 0 0 64k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write 0 1k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write 8k 1k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+$QEMU_IO -c "write 17k 1k" "$TEST_IMG" 2>&1 | _filter_qemu_io | _filter_testdir
+
+for min_sparse in 4k 8k; do
+    echo
+    echo convert -S $min_sparse
+    $QEMU_IMG convert -O $IMGFMT -o cluster_size=1k -S $min_sparse "$TEST_IMG" "$TEST_IMG".orig
+    $QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+
+    echo
+    echo convert -c -S $min_sparse
+    # For compressed images, -S values other than 0 are ignored
+    $QEMU_IMG convert -O $IMGFMT -o cluster_size=1k -c -S $min_sparse "$TEST_IMG" "$TEST_IMG".orig
+    $QEMU_IMG map --output=json "$TEST_IMG".orig | _filter_qemu_img_map
+done
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/122.out b/tests/qemu-iotests/122.out
new file mode 100644
index 0000000..98814de
--- /dev/null
+++ b/tests/qemu-iotests/122.out
@@ -0,0 +1,207 @@
+QA output created by 122
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+wrote 67108864/67108864 bytes at offset 0
+64 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Check allocation status regression with -B ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          File
+0               0x300000        TEST_DIR/t.IMGFMT.orig
+0x300000        0x3d00000       TEST_DIR/t.IMGFMT.base
+
+=== Check that zero clusters are kept in overlay ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Concatenate multiple source images ===
+
+Formatting 'TEST_DIR/t.IMGFMT.1', fmt=IMGFMT size=4194304
+Formatting 'TEST_DIR/t.IMGFMT.2', fmt=IMGFMT size=4194304
+Formatting 'TEST_DIR/t.IMGFMT.3', fmt=IMGFMT size=4194304
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Offset          Length          File
+0               0x10000         TEST_DIR/t.IMGFMT
+0x400000        0x10000         TEST_DIR/t.IMGFMT
+0x800000        0x10000         TEST_DIR/t.IMGFMT
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 4194304
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 8388608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 65536, "depth": 0, "zero": false, "data": true},
+{ "start": 65536, "length": 4128768, "depth": 0, "zero": true, "data": false},
+{ "start": 4194304, "length": 65536, "depth": 0, "zero": false, "data": true},
+{ "start": 4259840, "length": 4128768, "depth": 0, "zero": true, "data": false},
+{ "start": 8388608, "length": 65536, "depth": 0, "zero": false, "data": true},
+{ "start": 8454144, "length": 4128768, "depth": 0, "zero": true, "data": false}]
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 4194304
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 8388608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: -B makes no sense when concatenating multiple input images
+qemu-img: -B makes no sense when concatenating multiple input images
+
+=== Compression with misaligned allocations and image sizes ===
+
+Formatting 'TEST_DIR/t.IMGFMT.1', fmt=IMGFMT size=1047552
+Formatting 'TEST_DIR/t.IMGFMT.2', fmt=IMGFMT size=1047552
+wrote 16384/16384 bytes at offset 16384
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 133120/133120 bytes at offset 133120
+130 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 1046528
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 0
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 65536, "depth": 0, "zero": false, "data": true},
+{ "start": 65536, "length": 65536, "depth": 0, "zero": true, "data": false},
+{ "start": 131072, "length": 196608, "depth": 0, "zero": false, "data": true},
+{ "start": 327680, "length": 655360, "depth": 0, "zero": true, "data": false},
+{ "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true},
+{ "start": 1048576, "length": 1046528, "depth": 0, "zero": true, "data": false}]
+read 16384/16384 bytes at offset 0
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 16384/16384 bytes at offset 16384
+16 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 100352/100352 bytes at offset 32768
+98 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 133120/133120 bytes at offset 133120
+130 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 780288/780288 bytes at offset 266240
+762 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 1046528
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 1047552
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1046528/1046528 bytes at offset 1048576
+1022 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Full allocation with -S 0 ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 3145728/3145728 bytes at offset 3145728
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+convert -S 0:
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 63963136/63963136 bytes at offset 3145728
+61 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true, "offset": 327680}]
+
+convert -c -S 0:
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 63963136/63963136 bytes at offset 3145728
+61 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true}]
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+convert -S 0 with source backing file:
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 30408704/30408704 bytes at offset 3145728
+29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 33554432
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true, "offset": 327680}]
+
+convert -c -S 0 with source backing file:
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 30408704/30408704 bytes at offset 3145728
+29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 33554432
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true}]
+
+convert -S 0 -B ...
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 30408704/30408704 bytes at offset 3145728
+29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 33554432
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true, "offset": 327680}]
+
+convert -c -S 0 -B ...
+read 3145728/3145728 bytes at offset 0
+3 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 30408704/30408704 bytes at offset 3145728
+29 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 33554432
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 67108864, "depth": 0, "zero": false, "data": true}]
+
+=== Non-zero -S ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 0
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 8192
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 17408
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+convert -S 4k
+[{ "start": 0, "length": 1024, "depth": 0, "zero": false, "data": true, "offset": 8192},
+{ "start": 1024, "length": 7168, "depth": 0, "zero": true, "data": false},
+{ "start": 8192, "length": 1024, "depth": 0, "zero": false, "data": true, "offset": 9216},
+{ "start": 9216, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 17408, "length": 1024, "depth": 0, "zero": false, "data": true, "offset": 10240},
+{ "start": 18432, "length": 67090432, "depth": 0, "zero": true, "data": false}]
+
+convert -c -S 4k
+[{ "start": 0, "length": 1024, "depth": 0, "zero": false, "data": true},
+{ "start": 1024, "length": 7168, "depth": 0, "zero": true, "data": false},
+{ "start": 8192, "length": 1024, "depth": 0, "zero": false, "data": true},
+{ "start": 9216, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 17408, "length": 1024, "depth": 0, "zero": false, "data": true},
+{ "start": 18432, "length": 67090432, "depth": 0, "zero": true, "data": false}]
+
+convert -S 8k
+[{ "start": 0, "length": 9216, "depth": 0, "zero": false, "data": true, "offset": 8192},
+{ "start": 9216, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 17408, "length": 1024, "depth": 0, "zero": false, "data": true, "offset": 17408},
+{ "start": 18432, "length": 67090432, "depth": 0, "zero": true, "data": false}]
+
+convert -c -S 8k
+[{ "start": 0, "length": 1024, "depth": 0, "zero": false, "data": true},
+{ "start": 1024, "length": 7168, "depth": 0, "zero": true, "data": false},
+{ "start": 8192, "length": 1024, "depth": 0, "zero": false, "data": true},
+{ "start": 9216, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 17408, "length": 1024, "depth": 0, "zero": false, "data": true},
+{ "start": 18432, "length": 67090432, "depth": 0, "zero": true, "data": false}]
+*** done
diff --git a/tests/qemu-iotests/123 b/tests/qemu-iotests/123
new file mode 100755
index 0000000..b18e3fc
--- /dev/null
+++ b/tests/qemu-iotests/123
@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Test case for qemu-img convert to NBD
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$SRC_IMG"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw
+_supported_proto nbd
+_supported_os Linux
+
+SRC_IMG="$TEST_DIR/source.$IMGFMT"
+
+_make_test_img 1M
+$QEMU_IMG create -f $IMGFMT "$SRC_IMG" 1M | _filter_img_create
+
+$QEMU_IO -c 'write -P 42 0 1M' "$SRC_IMG" | _filter_qemu_io
+
+$QEMU_IMG convert -n -f $IMGFMT -O raw "$SRC_IMG" "$TEST_IMG"
+
+$QEMU_IO -c 'read -P 42 0 1M' "$TEST_IMG" | _filter_qemu_io
+
+
+# success, all done
+echo
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/123.out b/tests/qemu-iotests/123.out
new file mode 100644
index 0000000..0b818d3
--- /dev/null
+++ b/tests/qemu-iotests/123.out
@@ -0,0 +1,9 @@
+QA output created by 123
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
+Formatting 'TEST_DIR/source.IMGFMT', fmt=IMGFMT size=1048576
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+*** done
diff --git a/tests/qemu-iotests/124 b/tests/qemu-iotests/124
new file mode 100644
index 0000000..de7cdbe
--- /dev/null
+++ b/tests/qemu-iotests/124
@@ -0,0 +1,605 @@
+#!/usr/bin/env python
+#
+# Tests for incremental drive-backup
+#
+# Copyright (C) 2015 John Snow for Red Hat, Inc.
+#
+# Based on 056.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+
+
+def io_write_patterns(img, patterns):
+    for pattern in patterns:
+        iotests.qemu_io('-c', 'write -P%s %s %s' % pattern, img)
+
+
+def try_remove(img):
+    try:
+        os.remove(img)
+    except OSError:
+        pass
+
+
+def transaction_action(action, **kwargs):
+    return {
+        'type': action,
+        'data': dict((k.replace('_', '-'), v) for k, v in kwargs.iteritems())
+    }
+
+
+def transaction_bitmap_clear(node, name, **kwargs):
+    return transaction_action('block-dirty-bitmap-clear',
+                              node=node, name=name, **kwargs)
+
+
+def transaction_drive_backup(device, target, **kwargs):
+    return transaction_action('drive-backup', device=device, target=target,
+                              **kwargs)
+
+
+class Bitmap:
+    def __init__(self, name, drive):
+        self.name = name
+        self.drive = drive
+        self.num = 0
+        self.backups = list()
+
+    def base_target(self):
+        return (self.drive['backup'], None)
+
+    def new_target(self, num=None):
+        if num is None:
+            num = self.num
+        self.num = num + 1
+        base = os.path.join(iotests.test_dir,
+                            "%s.%s." % (self.drive['id'], self.name))
+        suff = "%i.%s" % (num, self.drive['fmt'])
+        target = base + "inc" + suff
+        reference = base + "ref" + suff
+        self.backups.append((target, reference))
+        return (target, reference)
+
+    def last_target(self):
+        if self.backups:
+            return self.backups[-1]
+        return self.base_target()
+
+    def del_target(self):
+        for image in self.backups.pop():
+            try_remove(image)
+        self.num -= 1
+
+    def cleanup(self):
+        for backup in self.backups:
+            for image in backup:
+                try_remove(image)
+
+
+class TestIncrementalBackupBase(iotests.QMPTestCase):
+    def __init__(self, *args):
+        super(TestIncrementalBackupBase, self).__init__(*args)
+        self.bitmaps = list()
+        self.files = list()
+        self.drives = list()
+        self.vm = iotests.VM()
+        self.err_img = os.path.join(iotests.test_dir, 'err.%s' % iotests.imgfmt)
+
+
+    def setUp(self):
+        # Create a base image with a distinctive patterning
+        drive0 = self.add_node('drive0')
+        self.img_create(drive0['file'], drive0['fmt'])
+        self.vm.add_drive(drive0['file'])
+        self.write_default_pattern(drive0['file'])
+        self.vm.launch()
+
+
+    def write_default_pattern(self, target):
+        io_write_patterns(target, (('0x41', 0, 512),
+                                   ('0xd5', '1M', '32k'),
+                                   ('0xdc', '32M', '124k')))
+
+
+    def add_node(self, node_id, fmt=iotests.imgfmt, path=None, backup=None):
+        if path is None:
+            path = os.path.join(iotests.test_dir, '%s.%s' % (node_id, fmt))
+        if backup is None:
+            backup = os.path.join(iotests.test_dir,
+                                  '%s.full.backup.%s' % (node_id, fmt))
+
+        self.drives.append({
+            'id': node_id,
+            'file': path,
+            'backup': backup,
+            'fmt': fmt })
+        return self.drives[-1]
+
+
+    def img_create(self, img, fmt=iotests.imgfmt, size='64M',
+                   parent=None, parentFormat=None, **kwargs):
+        optargs = []
+        for k,v in kwargs.iteritems():
+            optargs = optargs + ['-o', '%s=%s' % (k,v)]
+        args = ['create', '-f', fmt] + optargs + [img, size]
+        if parent:
+            if parentFormat is None:
+                parentFormat = fmt
+            args = args + ['-b', parent, '-F', parentFormat]
+        iotests.qemu_img(*args)
+        self.files.append(img)
+
+
+    def do_qmp_backup(self, error='Input/output error', **kwargs):
+        res = self.vm.qmp('drive-backup', **kwargs)
+        self.assert_qmp(res, 'return', {})
+        return self.wait_qmp_backup(kwargs['device'], error)
+
+
+    def wait_qmp_backup(self, device, error='Input/output error'):
+        event = self.vm.event_wait(name="BLOCK_JOB_COMPLETED",
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+        try:
+            failure = self.dictpath(event, 'data/error')
+        except AssertionError:
+            # Backup succeeded.
+            self.assert_qmp(event, 'data/offset', event['data']['len'])
+            return True
+        else:
+            # Backup failed.
+            self.assert_qmp(event, 'data/error', error)
+            return False
+
+
+    def wait_qmp_backup_cancelled(self, device):
+        event = self.vm.event_wait(name='BLOCK_JOB_CANCELLED',
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+
+    def create_anchor_backup(self, drive=None):
+        if drive is None:
+            drive = self.drives[-1]
+        res = self.do_qmp_backup(device=drive['id'], sync='full',
+                                 format=drive['fmt'], target=drive['backup'])
+        self.assertTrue(res)
+        self.files.append(drive['backup'])
+        return drive['backup']
+
+
+    def make_reference_backup(self, bitmap=None):
+        if bitmap is None:
+            bitmap = self.bitmaps[-1]
+        _, reference = bitmap.last_target()
+        res = self.do_qmp_backup(device=bitmap.drive['id'], sync='full',
+                                 format=bitmap.drive['fmt'], target=reference)
+        self.assertTrue(res)
+
+
+    def add_bitmap(self, name, drive, **kwargs):
+        bitmap = Bitmap(name, drive)
+        self.bitmaps.append(bitmap)
+        result = self.vm.qmp('block-dirty-bitmap-add', node=drive['id'],
+                             name=bitmap.name, **kwargs)
+        self.assert_qmp(result, 'return', {})
+        return bitmap
+
+
+    def prepare_backup(self, bitmap=None, parent=None):
+        if bitmap is None:
+            bitmap = self.bitmaps[-1]
+        if parent is None:
+            parent, _ = bitmap.last_target()
+
+        target, _ = bitmap.new_target()
+        self.img_create(target, bitmap.drive['fmt'], parent=parent)
+        return target
+
+
+    def create_incremental(self, bitmap=None, parent=None,
+                           parentFormat=None, validate=True):
+        if bitmap is None:
+            bitmap = self.bitmaps[-1]
+        if parent is None:
+            parent, _ = bitmap.last_target()
+
+        target = self.prepare_backup(bitmap, parent)
+        res = self.do_qmp_backup(device=bitmap.drive['id'],
+                                 sync='incremental', bitmap=bitmap.name,
+                                 format=bitmap.drive['fmt'], target=target,
+                                 mode='existing')
+        if not res:
+            bitmap.del_target();
+            self.assertFalse(validate)
+        else:
+            self.make_reference_backup(bitmap)
+        return res
+
+
+    def check_backups(self):
+        for bitmap in self.bitmaps:
+            for incremental, reference in bitmap.backups:
+                self.assertTrue(iotests.compare_images(incremental, reference))
+            last = bitmap.last_target()[0]
+            self.assertTrue(iotests.compare_images(last, bitmap.drive['file']))
+
+
+    def hmp_io_writes(self, drive, patterns):
+        for pattern in patterns:
+            self.vm.hmp_qemu_io(drive, 'write -P%s %s %s' % pattern)
+        self.vm.hmp_qemu_io(drive, 'flush')
+
+
+    def do_incremental_simple(self, **kwargs):
+        self.create_anchor_backup()
+        self.add_bitmap('bitmap0', self.drives[0], **kwargs)
+
+        # Sanity: Create a "hollow" incremental backup
+        self.create_incremental()
+        # Three writes: One complete overwrite, one new segment,
+        # and one partial overlap.
+        self.hmp_io_writes(self.drives[0]['id'], (('0xab', 0, 512),
+                                                  ('0xfe', '16M', '256k'),
+                                                  ('0x64', '32736k', '64k')))
+        self.create_incremental()
+        # Three more writes, one of each kind, like above
+        self.hmp_io_writes(self.drives[0]['id'], (('0x9a', 0, 512),
+                                                  ('0x55', '8M', '352k'),
+                                                  ('0x78', '15872k', '1M')))
+        self.create_incremental()
+        self.vm.shutdown()
+        self.check_backups()
+
+
+    def tearDown(self):
+        self.vm.shutdown()
+        for bitmap in self.bitmaps:
+            bitmap.cleanup()
+        for filename in self.files:
+            try_remove(filename)
+
+
+
+class TestIncrementalBackup(TestIncrementalBackupBase):
+    def test_incremental_simple(self):
+        '''
+        Test: Create and verify three incremental backups.
+
+        Create a bitmap and a full backup before VM execution begins,
+        then create a series of three incremental backups "during execution,"
+        i.e.; after IO requests begin modifying the drive.
+        '''
+        return self.do_incremental_simple()
+
+
+    def test_small_granularity(self):
+        '''
+        Test: Create and verify backups made with a small granularity bitmap.
+
+        Perform the same test as test_incremental_simple, but with a granularity
+        of only 32KiB instead of the present default of 64KiB.
+        '''
+        return self.do_incremental_simple(granularity=32768)
+
+
+    def test_large_granularity(self):
+        '''
+        Test: Create and verify backups made with a large granularity bitmap.
+
+        Perform the same test as test_incremental_simple, but with a granularity
+        of 128KiB instead of the present default of 64KiB.
+        '''
+        return self.do_incremental_simple(granularity=131072)
+
+
+    def test_larger_cluster_target(self):
+        '''
+        Test: Create and verify backups made to a larger cluster size target.
+
+        With a default granularity of 64KiB, verify that backups made to a
+        larger cluster size target of 128KiB without a backing file works.
+        '''
+        drive0 = self.drives[0]
+
+        # Create a cluster_size=128k full backup / "anchor" backup
+        self.img_create(drive0['backup'], cluster_size='128k')
+        self.assertTrue(self.do_qmp_backup(device=drive0['id'], sync='full',
+                                           format=drive0['fmt'],
+                                           target=drive0['backup'],
+                                           mode='existing'))
+
+        # Create bitmap and dirty it with some new writes.
+        # overwrite [32736, 32799] which will dirty bitmap clusters at
+        # 32M-64K and 32M. 32M+64K will be left undirtied.
+        bitmap0 = self.add_bitmap('bitmap0', drive0)
+        self.hmp_io_writes(drive0['id'],
+                           (('0xab', 0, 512),
+                            ('0xfe', '16M', '256k'),
+                            ('0x64', '32736k', '64k')))
+
+
+        # Prepare a cluster_size=128k backup target without a backing file.
+        (target, _) = bitmap0.new_target()
+        self.img_create(target, bitmap0.drive['fmt'], cluster_size='128k')
+
+        # Perform Incremental Backup
+        self.assertTrue(self.do_qmp_backup(device=bitmap0.drive['id'],
+                                           sync='incremental',
+                                           bitmap=bitmap0.name,
+                                           format=bitmap0.drive['fmt'],
+                                           target=target,
+                                           mode='existing'))
+        self.make_reference_backup(bitmap0)
+
+        # Add the backing file, then compare and exit.
+        iotests.qemu_img('rebase', '-f', drive0['fmt'], '-u', '-b',
+                         drive0['backup'], '-F', drive0['fmt'], target)
+        self.vm.shutdown()
+        self.check_backups()
+
+
+    def test_incremental_transaction(self):
+        '''Test: Verify backups made from transactionally created bitmaps.
+
+        Create a bitmap "before" VM execution begins, then create a second
+        bitmap AFTER writes have already occurred. Use transactions to create
+        a full backup and synchronize both bitmaps to this backup.
+        Create an incremental backup through both bitmaps and verify that
+        both backups match the current drive0 image.
+        '''
+
+        drive0 = self.drives[0]
+        bitmap0 = self.add_bitmap('bitmap0', drive0)
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+        bitmap1 = self.add_bitmap('bitmap1', drive0)
+
+        result = self.vm.qmp('transaction', actions=[
+            transaction_bitmap_clear(bitmap0.drive['id'], bitmap0.name),
+            transaction_bitmap_clear(bitmap1.drive['id'], bitmap1.name),
+            transaction_drive_backup(drive0['id'], drive0['backup'],
+                                     sync='full', format=drive0['fmt'])
+        ])
+        self.assert_qmp(result, 'return', {})
+        self.wait_until_completed(drive0['id'])
+        self.files.append(drive0['backup'])
+
+        self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
+                                          ('0x55', '8M', '352k'),
+                                          ('0x78', '15872k', '1M')))
+        # Both bitmaps should be correctly in sync.
+        self.create_incremental(bitmap0)
+        self.create_incremental(bitmap1)
+        self.vm.shutdown()
+        self.check_backups()
+
+
+    def test_transaction_failure(self):
+        '''Test: Verify backups made from a transaction that partially fails.
+
+        Add a second drive with its own unique pattern, and add a bitmap to each
+        drive. Use blkdebug to interfere with the backup on just one drive and
+        attempt to create a coherent incremental backup across both drives.
+
+        verify a failure in one but not both, then delete the failed stubs and
+        re-run the same transaction.
+
+        verify that both incrementals are created successfully.
+        '''
+
+        # Create a second drive, with pattern:
+        drive1 = self.add_node('drive1')
+        self.img_create(drive1['file'], drive1['fmt'])
+        io_write_patterns(drive1['file'], (('0x14', 0, 512),
+                                           ('0x5d', '1M', '32k'),
+                                           ('0xcd', '32M', '124k')))
+
+        # Create a blkdebug interface to this img as 'drive1'
+        result = self.vm.qmp('blockdev-add', options={
+            'id': drive1['id'],
+            'driver': drive1['fmt'],
+            'file': {
+                'driver': 'blkdebug',
+                'image': {
+                    'driver': 'file',
+                    'filename': drive1['file']
+                },
+                'set-state': [{
+                    'event': 'flush_to_disk',
+                    'state': 1,
+                    'new_state': 2
+                }],
+                'inject-error': [{
+                    'event': 'read_aio',
+                    'errno': 5,
+                    'state': 2,
+                    'immediately': False,
+                    'once': True
+                }],
+            }
+        })
+        self.assert_qmp(result, 'return', {})
+
+        # Create bitmaps and full backups for both drives
+        drive0 = self.drives[0]
+        dr0bm0 = self.add_bitmap('bitmap0', drive0)
+        dr1bm0 = self.add_bitmap('bitmap0', drive1)
+        self.create_anchor_backup(drive0)
+        self.create_anchor_backup(drive1)
+        self.assert_no_active_block_jobs()
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+
+        # Emulate some writes
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+        self.hmp_io_writes(drive1['id'], (('0xba', 0, 512),
+                                          ('0xef', '16M', '256k'),
+                                          ('0x46', '32736k', '64k')))
+
+        # Create incremental backup targets
+        target0 = self.prepare_backup(dr0bm0)
+        target1 = self.prepare_backup(dr1bm0)
+
+        # Ask for a new incremental backup per-each drive,
+        # expecting drive1's backup to fail:
+        transaction = [
+            transaction_drive_backup(drive0['id'], target0, sync='incremental',
+                                     format=drive0['fmt'], mode='existing',
+                                     bitmap=dr0bm0.name),
+            transaction_drive_backup(drive1['id'], target1, sync='incremental',
+                                     format=drive1['fmt'], mode='existing',
+                                     bitmap=dr1bm0.name)
+        ]
+        result = self.vm.qmp('transaction', actions=transaction,
+                             properties={'completion-mode': 'grouped'} )
+        self.assert_qmp(result, 'return', {})
+
+        # Observe that drive0's backup is cancelled and drive1 completes with
+        # an error.
+        self.wait_qmp_backup_cancelled(drive0['id'])
+        self.assertFalse(self.wait_qmp_backup(drive1['id']))
+        error = self.vm.event_wait('BLOCK_JOB_ERROR')
+        self.assert_qmp(error, 'data', {'device': drive1['id'],
+                                        'action': 'report',
+                                        'operation': 'read'})
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+        self.assert_no_active_block_jobs()
+
+        # Delete drive0's successful target and eliminate our record of the
+        # unsuccessful drive1 target. Then re-run the same transaction.
+        dr0bm0.del_target()
+        dr1bm0.del_target()
+        target0 = self.prepare_backup(dr0bm0)
+        target1 = self.prepare_backup(dr1bm0)
+
+        # Re-run the exact same transaction.
+        result = self.vm.qmp('transaction', actions=transaction,
+                             properties={'completion-mode':'grouped'})
+        self.assert_qmp(result, 'return', {})
+
+        # Both should complete successfully this time.
+        self.assertTrue(self.wait_qmp_backup(drive0['id']))
+        self.assertTrue(self.wait_qmp_backup(drive1['id']))
+        self.make_reference_backup(dr0bm0)
+        self.make_reference_backup(dr1bm0)
+        self.assertFalse(self.vm.get_qmp_events(wait=False))
+        self.assert_no_active_block_jobs()
+
+        # And the images should of course validate.
+        self.vm.shutdown()
+        self.check_backups()
+
+
+    def test_sync_dirty_bitmap_missing(self):
+        self.assert_no_active_block_jobs()
+        self.files.append(self.err_img)
+        result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
+                             sync='incremental', format=self.drives[0]['fmt'],
+                             target=self.err_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+
+    def test_sync_dirty_bitmap_not_found(self):
+        self.assert_no_active_block_jobs()
+        self.files.append(self.err_img)
+        result = self.vm.qmp('drive-backup', device=self.drives[0]['id'],
+                             sync='incremental', bitmap='unknown',
+                             format=self.drives[0]['fmt'], target=self.err_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+
+    def test_sync_dirty_bitmap_bad_granularity(self):
+        '''
+        Test: Test what happens if we provide an improper granularity.
+
+        The granularity must always be a power of 2.
+        '''
+        self.assert_no_active_block_jobs()
+        self.assertRaises(AssertionError, self.add_bitmap,
+                          'bitmap0', self.drives[0],
+                          granularity=64000)
+
+
+class TestIncrementalBackupBlkdebug(TestIncrementalBackupBase):
+    '''Incremental backup tests that utilize a BlkDebug filter on drive0.'''
+
+    def setUp(self):
+        drive0 = self.add_node('drive0')
+        self.img_create(drive0['file'], drive0['fmt'])
+        self.write_default_pattern(drive0['file'])
+        self.vm.launch()
+
+    def test_incremental_failure(self):
+        '''Test: Verify backups made after a failure are correct.
+
+        Simulate a failure during an incremental backup block job,
+        emulate additional writes, then create another incremental backup
+        afterwards and verify that the backup created is correct.
+        '''
+
+        drive0 = self.drives[0]
+        result = self.vm.qmp('blockdev-add', options={
+            'id': drive0['id'],
+            'driver': drive0['fmt'],
+            'file': {
+                'driver': 'blkdebug',
+                'image': {
+                    'driver': 'file',
+                    'filename': drive0['file']
+                },
+                'set-state': [{
+                    'event': 'flush_to_disk',
+                    'state': 1,
+                    'new_state': 2
+                }],
+                'inject-error': [{
+                    'event': 'read_aio',
+                    'errno': 5,
+                    'state': 2,
+                    'immediately': False,
+                    'once': True
+                }],
+            }
+        })
+        self.assert_qmp(result, 'return', {})
+
+        self.create_anchor_backup(drive0)
+        self.add_bitmap('bitmap0', drive0)
+        # Note: at this point, during a normal execution,
+        # Assume that the VM resumes and begins issuing IO requests here.
+
+        self.hmp_io_writes(drive0['id'], (('0xab', 0, 512),
+                                          ('0xfe', '16M', '256k'),
+                                          ('0x64', '32736k', '64k')))
+
+        result = self.create_incremental(validate=False)
+        self.assertFalse(result)
+        self.hmp_io_writes(drive0['id'], (('0x9a', 0, 512),
+                                          ('0x55', '8M', '352k'),
+                                          ('0x78', '15872k', '1M')))
+        self.create_incremental()
+        self.vm.shutdown()
+        self.check_backups()
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/124.out b/tests/qemu-iotests/124.out
new file mode 100644
index 0000000..36376be
--- /dev/null
+++ b/tests/qemu-iotests/124.out
@@ -0,0 +1,5 @@
+..........
+----------------------------------------------------------------------
+Ran 10 tests
+
+OK
diff --git a/tests/qemu-iotests/128 b/tests/qemu-iotests/128
new file mode 100755
index 0000000..0976a18
--- /dev/null
+++ b/tests/qemu-iotests/128
@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Test that opening O_DIRECT succeeds when image file I/O produces EIO
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+devname="eiodev$$"
+sudo=""
+
+_sudo_qemu_io_wrapper()
+{
+    (exec $sudo "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@")
+}
+
+_setup_eiodev()
+{
+	# This test should either be run as root or with passwordless sudo
+	for cmd in "" "sudo -n"; do
+		echo "0 $((1024 * 1024 * 1024 / 512)) error" | \
+			$cmd dmsetup create "$devname" 2>/dev/null
+		if [ "$?" -eq 0 ]; then
+			sudo="$cmd"
+			return
+		fi
+	done
+	_notrun "root privileges required to run dmsetup"
+}
+
+_cleanup_eiodev()
+{
+	for cmd in "" "sudo -n"; do
+		$cmd dmsetup remove "$devname" 2>/dev/null
+		if [ "$?" -eq 0 ]; then
+			return
+		fi
+	done
+}
+
+_cleanup()
+{
+	_cleanup_eiodev
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw
+_supported_proto file
+_supported_os Linux
+
+_setup_eiodev
+
+TEST_IMG="/dev/mapper/$devname"
+
+echo
+echo "== reading from error device =="
+# Opening image should succeed but the read operation should fail
+_sudo_qemu_io_wrapper --format "$IMGFMT" --nocache \
+                      -c "read 0 65536" "$TEST_IMG" \
+    | _filter_qemu_io
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/128.out b/tests/qemu-iotests/128.out
new file mode 100644
index 0000000..4e43f5f
--- /dev/null
+++ b/tests/qemu-iotests/128.out
@@ -0,0 +1,5 @@
+QA output created by 128
+
+== reading from error device ==
+read failed: Input/output error
+*** done
diff --git a/tests/qemu-iotests/129 b/tests/qemu-iotests/129
new file mode 100644
index 0000000..9e87e1c
--- /dev/null
+++ b/tests/qemu-iotests/129
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+#
+# Tests that "bdrv_drain_all" doesn't drain block jobs
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+import time
+
+class TestStopWithBlockJob(iotests.QMPTestCase):
+    test_img = os.path.join(iotests.test_dir, 'test.img')
+    target_img = os.path.join(iotests.test_dir, 'target.img')
+    base_img = os.path.join(iotests.test_dir, 'base.img')
+
+    def setUp(self):
+        iotests.qemu_img('create', '-f', iotests.imgfmt, self.base_img, "1G")
+        iotests.qemu_img('create', '-f', iotests.imgfmt, self.test_img, "-b", self.base_img)
+        iotests.qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 1M 128M', self.test_img)
+        self.vm = iotests.VM().add_drive(self.test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        params = {"device": "drive0",
+                  "bps": 0,
+                  "bps_rd": 0,
+                  "bps_wr": 0,
+                  "iops": 0,
+                  "iops_rd": 0,
+                  "iops_wr": 0,
+                 }
+        result = self.vm.qmp("block_set_io_throttle", conv_keys=False,
+                             **params)
+        self.vm.shutdown()
+
+    def do_test_stop(self, cmd, **args):
+        """Test 'stop' while block job is running on a throttled drive.
+        The 'stop' command shouldn't drain the job"""
+        params = {"device": "drive0",
+                  "bps": 1024,
+                  "bps_rd": 0,
+                  "bps_wr": 0,
+                  "iops": 0,
+                  "iops_rd": 0,
+                  "iops_wr": 0,
+                 }
+        result = self.vm.qmp("block_set_io_throttle", conv_keys=False,
+                             **params)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp(cmd, **args)
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp("stop")
+        self.assert_qmp(result, 'return', {})
+        result = self.vm.qmp("query-block-jobs")
+        self.assert_qmp(result, 'return[0]/busy', True)
+        self.assert_qmp(result, 'return[0]/ready', False)
+
+    def test_drive_mirror(self):
+        self.do_test_stop("drive-mirror", device="drive0",
+                          target=self.target_img,
+                          sync="full")
+
+    def test_drive_backup(self):
+        self.do_test_stop("drive-backup", device="drive0",
+                          target=self.target_img,
+                          sync="full")
+
+    def test_block_commit(self):
+        self.do_test_stop("block-commit", device="drive0")
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["qcow2"])
diff --git a/tests/qemu-iotests/129.out b/tests/qemu-iotests/129.out
new file mode 100644
index 0000000..8d7e996
--- /dev/null
+++ b/tests/qemu-iotests/129.out
@@ -0,0 +1,5 @@
+...
+----------------------------------------------------------------------
+Ran 3 tests
+
+OK
diff --git a/tests/qemu-iotests/130 b/tests/qemu-iotests/130
new file mode 100755
index 0000000..ecc8a5b
--- /dev/null
+++ b/tests/qemu-iotests/130
@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# Test that temporary backing file overrides (on the command line or in
+# blockdev-add) don't replace the original path stored in the image during
+# header updates.
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+qemu_comm_method="monitor"
+
+
+TEST_IMG="$TEST_IMG.orig" _make_test_img 64M
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+_make_test_img 64M
+_img_info | _filter_img_info
+
+echo
+echo "=== HMP commit ==="
+echo
+# bdrv_make_empty() involves a header update for qcow2
+
+# Test that a backing file isn't written
+_launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base"
+_send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
+_send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
+_cleanup_qemu
+_img_info | _filter_img_info
+
+# Make sure that if there was a backing file that was just overridden on the
+# command line, that backing file is retained, with the right format
+_make_test_img -F raw -b "$TEST_IMG.orig" 64M
+_launch_qemu -drive id=testdisk,file="$TEST_IMG",backing.file.filename="$TEST_IMG.base",backing.driver=$IMGFMT
+_send_qemu_cmd $QEMU_HANDLE "commit testdisk" "(qemu)"
+_send_qemu_cmd $QEMU_HANDLE '' '(qemu)'
+_cleanup_qemu
+_img_info | _filter_img_info
+
+echo
+echo "=== Marking image dirty (lazy refcounts) ==="
+echo
+
+# Test that a backing file isn't written
+_make_test_img 64M
+$QEMU_IO -c "open -o backing.file.filename=$TEST_IMG.base,lazy-refcounts=on $TEST_IMG" -c "write 0 4k" | _filter_qemu_io
+_img_info | _filter_img_info
+
+# Make sure that if there was a backing file that was just overridden on the
+# command line, that backing file is retained, with the right format
+_make_test_img -F raw -b "$TEST_IMG.orig" 64M
+$QEMU_IO -c "open -o backing.file.filename=$TEST_IMG.base,backing.driver=$IMGFMT,lazy-refcounts=on $TEST_IMG" -c "write 0 4k" | _filter_qemu_io
+_img_info | _filter_img_info
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/130.out b/tests/qemu-iotests/130.out
new file mode 100644
index 0000000..ae95b50
--- /dev/null
+++ b/tests/qemu-iotests/130.out
@@ -0,0 +1,43 @@
+QA output created by 130
+Formatting 'TEST_DIR/t.IMGFMT.orig', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+
+=== HMP commit ===
+
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) c[K[Dco[K[D[Dcom[K[D[D[Dcomm[K[D[D[D[Dcommi[K[D[D[D[D[Dcommit[K[D[D[D[D[D[Dcommit [K[D[D[D[D[D[D[Dcommit t[K[D[D[D[D[D[D[D[Dcommit te[K[D[D[D[D[D[D[D[D[Dcommit tes[K[D[D[D[D[D[D[D[D[D[Dcommit test[K[D[D[D[D[D[D[D[D[D[D[Dcommit testd[K[D[D[D[D[D[D[D[D[D[D[D[Dcommit testdi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dcommit testdis[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dcommit testdisk[K
+(qemu) 
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) c[K[Dco[K[D[Dcom[K[D[D[Dcomm[K[D[D[D[Dcommi[K[D[D[D[D[Dcommit[K[D[D[D[D[D[Dcommit [K[D[D[D[D[D[D[Dcommit t[K[D[D[D[D[D[D[D[Dcommit te[K[D[D[D[D[D[D[D[D[Dcommit tes[K[D[D[D[D[D[D[D[D[D[Dcommit test[K[D[D[D[D[D[D[D[D[D[D[Dcommit testd[K[D[D[D[D[D[D[D[D[D[D[D[Dcommit testdi[K[D[D[D[D[D[D[D[D[D[D[D[D[Dcommit testdis[K[D[D[D[D[D[D[D[D[D[D[D[D[D[Dcommit testdisk[K
+(qemu) 
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+backing file: TEST_DIR/t.IMGFMT.orig
+backing file format: raw
+
+=== Marking image dirty (lazy refcounts) ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.orig backing_fmt=raw
+wrote 4096/4096 bytes at offset 0
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+backing file: TEST_DIR/t.IMGFMT.orig
+backing file format: raw
+*** done
diff --git a/tests/qemu-iotests/131 b/tests/qemu-iotests/131
new file mode 100755
index 0000000..94a9ae7
--- /dev/null
+++ b/tests/qemu-iotests/131
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# parallels format validation tests (created by QEMU)
+#
+# Copyright (C) 2014 Denis V. Lunev <den@openvz.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=den@openvz.org
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt parallels
+_supported_proto file
+_supported_os Linux
+
+inuse_offset=$((0x2c))
+
+size=64M
+CLUSTER_SIZE=64k
+IMGFMT=parallels
+_make_test_img $size
+
+echo == read empty image ==
+{ $QEMU_IO -c "read -P 0 32k 64k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+echo == write more than 1 block in a row ==
+{ $QEMU_IO -c "write -P 0x11 32k 128k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+echo == read less than block ==
+{ $QEMU_IO -c "read -P 0x11 32k 32k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+echo == read exactly 1 block ==
+{ $QEMU_IO -c "read -P 0x11 64k 64k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+echo == read more than 1 block ==
+{ $QEMU_IO -c "read -P 0x11 32k 128k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+echo == check that there is no trash after written ==
+{ $QEMU_IO -c "read -P 0 160k 32k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+echo == check that there is no trash before written ==
+{ $QEMU_IO -c "read -P 0 0 32k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo "== Corrupt image =="
+poke_file "$TEST_IMG" "$inuse_offset" "\x59\x6e\x6f\x74"
+{ $QEMU_IO -c "read -P 0x11 64k 64k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+_check_test_img
+_check_test_img -r all
+{ $QEMU_IO -c "read -P 0x11 64k 64k" "$TEST_IMG"; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/131.out b/tests/qemu-iotests/131.out
new file mode 100644
index 0000000..ae2412e
--- /dev/null
+++ b/tests/qemu-iotests/131.out
@@ -0,0 +1,41 @@
+QA output created by 131
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+== read empty image ==
+read 65536/65536 bytes at offset 32768
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== write more than 1 block in a row ==
+wrote 131072/131072 bytes at offset 32768
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== read less than block ==
+read 32768/32768 bytes at offset 32768
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== read exactly 1 block ==
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== read more than 1 block ==
+read 131072/131072 bytes at offset 32768
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== check that there is no trash after written ==
+read 32768/32768 bytes at offset 163840
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== check that there is no trash before written ==
+read 32768/32768 bytes at offset 0
+32 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+== Corrupt image ==
+can't open device TEST_DIR/t.parallels: parallels: Image was not closed correctly; cannot be opened read/write
+no file open, try 'help open'
+ERROR image was not closed correctly
+
+1 errors were found on the image.
+Data may be corrupted, or further writes to the image may corrupt it.
+Repairing image was not closed correctly
+The following inconsistencies were found and repaired:
+
+    0 leaked clusters
+    1 corruptions
+
+Double checking the fixed image now...
+No errors were found on the image.
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/132 b/tests/qemu-iotests/132
new file mode 100644
index 0000000..f53ef6e
--- /dev/null
+++ b/tests/qemu-iotests/132
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+#
+# Test mirror with unmap
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+import os
+import iotests
+from iotests import qemu_img, qemu_io
+
+test_img = os.path.join(iotests.test_dir, 'test.img')
+target_img = os.path.join(iotests.test_dir, 'target.img')
+
+class TestSingleDrive(iotests.QMPTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    def setUp(self):
+        # Write data to the image so we can compare later
+        qemu_img('create', '-f', iotests.imgfmt, test_img, str(TestSingleDrive.image_len))
+        qemu_io('-f', iotests.imgfmt, '-c', 'write -P0x5d 0 2M', test_img)
+
+        self.vm = iotests.VM().add_drive(test_img, 'discard=unmap')
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def test_mirror_discard(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+        self.vm.hmp_qemu_io('drive0', 'discard 0 64k')
+        self.complete_and_wait('drive0')
+        self.vm.shutdown()
+        self.assertTrue(iotests.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['raw', 'qcow2'])
diff --git a/tests/qemu-iotests/132.out b/tests/qemu-iotests/132.out
new file mode 100644
index 0000000..ae1213e
--- /dev/null
+++ b/tests/qemu-iotests/132.out
@@ -0,0 +1,5 @@
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK
diff --git a/tests/qemu-iotests/133 b/tests/qemu-iotests/133
new file mode 100755
index 0000000..9d35a6a
--- /dev/null
+++ b/tests/qemu-iotests/133
@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Test for reopen
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+TEST_IMG="$TEST_IMG.base" _make_test_img 64M
+_make_test_img -b "$TEST_IMG.base"
+
+echo
+echo "=== Check that node-name can't be changed ==="
+echo
+
+$QEMU_IO -c 'reopen -o node-name=foo' $TEST_IMG
+$QEMU_IO -c 'reopen -o file.node-name=foo' $TEST_IMG
+$QEMU_IO -c 'reopen -o backing.node-name=foo' $TEST_IMG
+
+echo
+echo "=== Check that unchanged node-name is okay ==="
+echo
+
+# Explicitly repeated
+$QEMU_IO -c "open -o node-name=foo $TEST_IMG" -c 'reopen -o node-name=foo'
+$QEMU_IO -c "open -o file.node-name=foo $TEST_IMG" -c 'reopen -o file.node-name=foo'
+$QEMU_IO -c "open -o backing.node-name=foo $TEST_IMG" -c 'reopen -o backing.node-name=foo'
+
+# Implicitly retained
+$QEMU_IO -c "open -o node-name=foo $TEST_IMG" -c 'reopen'
+$QEMU_IO -c "open -o file.node-name=foo $TEST_IMG" -c 'reopen'
+$QEMU_IO -c "open -o backing.node-name=foo $TEST_IMG" -c 'reopen'
+
+echo
+echo "=== Check that driver can't be changed ==="
+echo
+
+$QEMU_IO -c 'reopen -o driver=raw' $TEST_IMG
+$QEMU_IO -c 'reopen -o file.driver=qcow2' $TEST_IMG
+$QEMU_IO -c 'reopen -o backing.driver=file' $TEST_IMG
+
+echo
+echo "=== Check that unchanged driver is okay ==="
+echo
+
+# Explicitly repeated (implicit case is covered in node-name test)
+$QEMU_IO -c 'reopen -o driver=qcow2' $TEST_IMG
+$QEMU_IO -c 'reopen -o file.driver=file' $TEST_IMG
+$QEMU_IO -c 'reopen -o backing.driver=qcow2' $TEST_IMG
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/133.out b/tests/qemu-iotests/133.out
new file mode 100644
index 0000000..cc86b94
--- /dev/null
+++ b/tests/qemu-iotests/133.out
@@ -0,0 +1,22 @@
+QA output created by 133
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 backing_file=TEST_DIR/t.IMGFMT.base
+
+=== Check that node-name can't be changed ===
+
+Cannot change the option 'node-name'
+Cannot change the option 'node-name'
+Cannot change the option 'node-name'
+
+=== Check that unchanged node-name is okay ===
+
+
+=== Check that driver can't be changed ===
+
+Cannot change the option 'driver'
+Cannot change the option 'driver'
+Cannot change the option 'driver'
+
+=== Check that unchanged driver is okay ===
+
+*** done
diff --git a/tests/qemu-iotests/134 b/tests/qemu-iotests/134
new file mode 100755
index 0000000..af618b8
--- /dev/null
+++ b/tests/qemu-iotests/134
@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# Test encrypted read/write using plain bdrv_read/bdrv_write
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=berrange@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+
+size=128M
+IMGOPTS="encryption=on" _make_test_img $size
+
+echo
+echo "== reading whole image =="
+echo "astrochicken" | $QEMU_IO -c "read 0 $size" "$TEST_IMG" | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== rewriting whole image =="
+echo "astrochicken" | $QEMU_IO -c "write -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== verify pattern =="
+echo "astrochicken" | $QEMU_IO -c "read -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== verify pattern failure with wrong password =="
+echo "platypus" | $QEMU_IO -c "read -P 0xa 0 $size" "$TEST_IMG" | _filter_qemu_io | _filter_testdir
+
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/134.out b/tests/qemu-iotests/134.out
new file mode 100644
index 0000000..6493704
--- /dev/null
+++ b/tests/qemu-iotests/134.out
@@ -0,0 +1,28 @@
+QA output created by 134
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 encryption=on
+
+== reading whole image ==
+Disk image 'TEST_DIR/t.qcow2' is encrypted.
+password:
+read 134217728/134217728 bytes at offset 0
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== rewriting whole image ==
+Disk image 'TEST_DIR/t.qcow2' is encrypted.
+password:
+wrote 134217728/134217728 bytes at offset 0
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern ==
+Disk image 'TEST_DIR/t.qcow2' is encrypted.
+password:
+read 134217728/134217728 bytes at offset 0
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+== verify pattern failure with wrong password ==
+Disk image 'TEST_DIR/t.qcow2' is encrypted.
+password:
+Pattern verification failed at offset 0, 134217728 bytes
+read 134217728/134217728 bytes at offset 0
+128 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+*** done
diff --git a/tests/qemu-iotests/135 b/tests/qemu-iotests/135
new file mode 100755
index 0000000..ce60831
--- /dev/null
+++ b/tests/qemu-iotests/135
@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+# Test VPC open of image with large Max Table Entries value.
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt vpc
+_supported_proto generic
+_supported_os Linux
+
+_use_sample_img afl5.img.bz2
+
+echo
+echo "=== Verify image open and failure ===="
+$QEMU_IMG info "$TEST_IMG" 2>&1| _filter_testdir
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/135.out b/tests/qemu-iotests/135.out
new file mode 100644
index 0000000..793898b
--- /dev/null
+++ b/tests/qemu-iotests/135.out
@@ -0,0 +1,5 @@
+QA output created by 135
+
+=== Verify image open and failure ====
+qemu-img: Could not open 'TEST_DIR/afl5.img': Max Table Entries too large (1073741825)
+*** done
diff --git a/tests/qemu-iotests/136 b/tests/qemu-iotests/136
new file mode 100644
index 0000000..635b977
--- /dev/null
+++ b/tests/qemu-iotests/136
@@ -0,0 +1,330 @@
+#!/usr/bin/env python
+#
+# Tests for block device statistics
+#
+# Copyright (C) 2015 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import iotests
+import os
+
+interval_length = 10
+nsec_per_sec = 1000000000
+op_latency = nsec_per_sec / 1000 # See qtest_latency_ns in accounting.c
+bad_sector = 8192
+bad_offset = bad_sector * 512
+blkdebug_file = os.path.join(iotests.test_dir, 'blkdebug.conf')
+
+class BlockDeviceStatsTestCase(iotests.QMPTestCase):
+    test_img = "null-aio://"
+    total_rd_bytes = 0
+    total_rd_ops = 0
+    total_wr_bytes = 0
+    total_wr_ops = 0
+    total_wr_merged = 0
+    total_flush_ops = 0
+    failed_rd_ops = 0
+    failed_wr_ops = 0
+    invalid_rd_ops = 0
+    invalid_wr_ops = 0
+    wr_highest_offset = 0
+    account_invalid = False
+    account_failed = False
+
+    def blockstats(self, device):
+        result = self.vm.qmp("query-blockstats")
+        for r in result['return']:
+            if r['device'] == device:
+                return r['stats']
+        raise Exception("Device not found for blockstats: %s" % device)
+
+    def create_blkdebug_file(self):
+        file = open(blkdebug_file, 'w')
+        file.write('''
+[inject-error]
+event = "read_aio"
+errno = "5"
+sector = "%d"
+
+[inject-error]
+event = "write_aio"
+errno = "5"
+sector = "%d"
+''' % (bad_sector, bad_sector))
+        file.close()
+
+    def setUp(self):
+        drive_args = []
+        drive_args.append("stats-intervals.0=%d" % interval_length)
+        drive_args.append("stats-account-invalid=%s" %
+                          (self.account_invalid and "on" or "off"))
+        drive_args.append("stats-account-failed=%s" %
+                          (self.account_failed and "on" or "off"))
+        self.create_blkdebug_file()
+        self.vm = iotests.VM().add_drive('blkdebug:%s:%s ' %
+                                         (blkdebug_file, self.test_img),
+                                         ','.join(drive_args))
+        self.vm.launch()
+        # Set an initial value for the clock
+        self.vm.qtest("clock_step %d" % nsec_per_sec)
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(blkdebug_file)
+
+    def accounted_ops(self, read = False, write = False, flush = False):
+        ops = 0
+        if write:
+            ops += self.total_wr_ops
+            if self.account_failed:
+                ops += self.failed_wr_ops
+            if self.account_invalid:
+                ops += self.invalid_wr_ops
+        if read:
+            ops += self.total_rd_ops
+            if self.account_failed:
+                ops += self.failed_rd_ops
+            if self.account_invalid:
+                ops += self.invalid_rd_ops
+        if flush:
+            ops += self.total_flush_ops
+        return ops
+
+    def accounted_latency(self, read = False, write = False, flush = False):
+        latency = 0
+        if write:
+            latency += self.total_wr_ops * op_latency
+            if self.account_failed:
+                latency += self.failed_wr_ops * op_latency
+        if read:
+            latency += self.total_rd_ops * op_latency
+            if self.account_failed:
+                latency += self.failed_rd_ops * op_latency
+        if flush:
+            latency += self.total_flush_ops * op_latency
+        return latency
+
+    def check_values(self):
+        stats = self.blockstats('drive0')
+
+        # Check that the totals match with what we have calculated
+        self.assertEqual(self.total_rd_bytes, stats['rd_bytes'])
+        self.assertEqual(self.total_wr_bytes, stats['wr_bytes'])
+        self.assertEqual(self.total_rd_ops, stats['rd_operations'])
+        self.assertEqual(self.total_wr_ops, stats['wr_operations'])
+        self.assertEqual(self.total_flush_ops, stats['flush_operations'])
+        self.assertEqual(self.wr_highest_offset, stats['wr_highest_offset'])
+        self.assertEqual(self.failed_rd_ops, stats['failed_rd_operations'])
+        self.assertEqual(self.failed_wr_ops, stats['failed_wr_operations'])
+        self.assertEqual(self.invalid_rd_ops, stats['invalid_rd_operations'])
+        self.assertEqual(self.invalid_wr_ops, stats['invalid_wr_operations'])
+        self.assertEqual(self.account_invalid, stats['account_invalid'])
+        self.assertEqual(self.account_failed, stats['account_failed'])
+        self.assertEqual(self.total_wr_merged, stats['wr_merged'])
+
+        # Check that there's exactly one interval with the length we defined
+        self.assertEqual(1, len(stats['timed_stats']))
+        timed_stats = stats['timed_stats'][0]
+        self.assertEqual(interval_length, timed_stats['interval_length'])
+
+        total_rd_latency = self.accounted_latency(read = True)
+        if (total_rd_latency != 0):
+            self.assertEqual(total_rd_latency, stats['rd_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_rd_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_rd_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_rd_latency_ns'])
+            self.assertLess(0, timed_stats['avg_rd_queue_depth'])
+        else:
+            self.assertEqual(0, stats['rd_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['max_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_rd_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_rd_queue_depth'])
+
+        # min read latency <= avg read latency <= max read latency
+        self.assertLessEqual(timed_stats['min_rd_latency_ns'],
+                             timed_stats['avg_rd_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_rd_latency_ns'],
+                             timed_stats['max_rd_latency_ns'])
+
+        total_wr_latency = self.accounted_latency(write = True)
+        if (total_wr_latency != 0):
+            self.assertEqual(total_wr_latency, stats['wr_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_wr_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_wr_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_wr_latency_ns'])
+            self.assertLess(0, timed_stats['avg_wr_queue_depth'])
+        else:
+            self.assertEqual(0, stats['wr_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['max_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_wr_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_wr_queue_depth'])
+
+        # min write latency <= avg write latency <= max write latency
+        self.assertLessEqual(timed_stats['min_wr_latency_ns'],
+                             timed_stats['avg_wr_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_wr_latency_ns'],
+                             timed_stats['max_wr_latency_ns'])
+
+        total_flush_latency = self.accounted_latency(flush = True)
+        if (total_flush_latency != 0):
+            self.assertEqual(total_flush_latency, stats['flush_total_time_ns'])
+            self.assertEqual(op_latency, timed_stats['min_flush_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['max_flush_latency_ns'])
+            self.assertEqual(op_latency, timed_stats['avg_flush_latency_ns'])
+        else:
+            self.assertEqual(0, stats['flush_total_time_ns'])
+            self.assertEqual(0, timed_stats['min_flush_latency_ns'])
+            self.assertEqual(0, timed_stats['max_flush_latency_ns'])
+            self.assertEqual(0, timed_stats['avg_flush_latency_ns'])
+
+        # min flush latency <= avg flush latency <= max flush latency
+        self.assertLessEqual(timed_stats['min_flush_latency_ns'],
+                             timed_stats['avg_flush_latency_ns'])
+        self.assertLessEqual(timed_stats['avg_flush_latency_ns'],
+                             timed_stats['max_flush_latency_ns'])
+
+        # idle_time_ns must be > 0 if we have performed any operation
+        if (self.accounted_ops(read = True, write = True, flush = True) != 0):
+            self.assertLess(0, stats['idle_time_ns'])
+        else:
+            self.assertFalse(stats.has_key('idle_time_ns'))
+
+        # This test does not alter these, so they must be all 0
+        self.assertEqual(0, stats['rd_merged'])
+        self.assertEqual(0, stats['failed_flush_operations'])
+        self.assertEqual(0, stats['invalid_flush_operations'])
+
+    def do_test_stats(self, rd_size = 0, rd_ops = 0, wr_size = 0, wr_ops = 0,
+                      flush_ops = 0, invalid_rd_ops = 0, invalid_wr_ops = 0,
+                      failed_rd_ops = 0, failed_wr_ops = 0, wr_merged = 0):
+        # The 'ops' list will contain all the requested I/O operations
+        ops = []
+        for i in range(rd_ops):
+            ops.append("aio_read %d %d" % (i * rd_size, rd_size))
+
+        for i in range(wr_ops):
+            ops.append("aio_write %d %d" % (i * wr_size, wr_size))
+
+        for i in range(flush_ops):
+            ops.append("aio_flush")
+
+        highest_offset = wr_ops * wr_size
+
+        for i in range(invalid_rd_ops):
+            ops.append("aio_read -i 0 512")
+
+        for i in range(invalid_wr_ops):
+            ops.append("aio_write -i 0 512")
+
+        for i in range(failed_rd_ops):
+            ops.append("aio_read %d 512" % bad_offset)
+
+        for i in range(failed_wr_ops):
+            ops.append("aio_write %d 512" % bad_offset)
+
+        if failed_wr_ops > 0:
+            highest_offset = max(highest_offset, bad_offset + 512)
+
+        # Now perform all operations
+        for op in ops:
+            self.vm.hmp_qemu_io("drive0", op)
+
+        # Update the expected totals
+        self.total_rd_bytes += rd_ops * rd_size
+        self.total_rd_ops += rd_ops
+        self.total_wr_bytes += wr_ops * wr_size
+        self.total_wr_ops += wr_ops
+        self.total_wr_merged += wr_merged
+        self.total_flush_ops += flush_ops
+        self.invalid_rd_ops += invalid_rd_ops
+        self.invalid_wr_ops += invalid_wr_ops
+        self.failed_rd_ops += failed_rd_ops
+        self.failed_wr_ops += failed_wr_ops
+
+        self.wr_highest_offset = max(self.wr_highest_offset, highest_offset)
+
+        # Advance the clock so idle_time_ns has a meaningful value
+        self.vm.qtest("clock_step %d" % nsec_per_sec)
+
+        # And check that the actual statistics match the expected ones
+        self.check_values()
+
+    def test_read_only(self):
+        test_values = [[512,    1],
+                       [65536,  1],
+                       [512,   12],
+                       [65536, 12]]
+        for i in test_values:
+            self.do_test_stats(rd_size = i[0], rd_ops = i[1])
+
+    def test_write_only(self):
+        test_values = [[512,    1],
+                       [65536,  1],
+                       [512,   12],
+                       [65536, 12]]
+        for i in test_values:
+            self.do_test_stats(wr_size = i[0], wr_ops = i[1])
+
+    def test_invalid(self):
+        self.do_test_stats(invalid_rd_ops = 7)
+        self.do_test_stats(invalid_wr_ops = 3)
+        self.do_test_stats(invalid_rd_ops = 4, invalid_wr_ops = 5)
+
+    def test_failed(self):
+        self.do_test_stats(failed_rd_ops = 8)
+        self.do_test_stats(failed_wr_ops = 6)
+        self.do_test_stats(failed_rd_ops = 5, failed_wr_ops = 12)
+
+    def test_flush(self):
+        self.do_test_stats(flush_ops = 8)
+
+    def test_all(self):
+        # rd_size, rd_ops, wr_size, wr_ops, flush_ops
+        # invalid_rd_ops,  invalid_wr_ops,
+        # failed_rd_ops,   failed_wr_ops
+        # wr_merged
+        test_values = [[512,    1, 512,   1, 1, 4, 7, 5, 2, 0],
+                       [65536,  1, 2048, 12, 7, 7, 5, 2, 5, 0],
+                       [32768,  9, 8192,  1, 4, 3, 2, 4, 6, 0],
+                       [16384, 11, 3584, 16, 9, 8, 6, 7, 3, 0]]
+        for i in test_values:
+            self.do_test_stats(*i)
+
+    def test_no_op(self):
+        # All values must be sane before doing any I/O
+        self.check_values()
+
+
+class BlockDeviceStatsTestAccountInvalid(BlockDeviceStatsTestCase):
+    account_invalid = True
+    account_failed = False
+
+class BlockDeviceStatsTestAccountFailed(BlockDeviceStatsTestCase):
+    account_invalid = False
+    account_failed = True
+
+class BlockDeviceStatsTestAccountBoth(BlockDeviceStatsTestCase):
+    account_invalid = True
+    account_failed = True
+
+class BlockDeviceStatsTestCoroutine(BlockDeviceStatsTestCase):
+    test_img = "null-co://"
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["raw"])
diff --git a/tests/qemu-iotests/136.out b/tests/qemu-iotests/136.out
new file mode 100644
index 0000000..cfa5c0d
--- /dev/null
+++ b/tests/qemu-iotests/136.out
@@ -0,0 +1,5 @@
+...................................
+----------------------------------------------------------------------
+Ran 35 tests
+
+OK
diff --git a/tests/qemu-iotests/137 b/tests/qemu-iotests/137
new file mode 100755
index 0000000..e5e30de
--- /dev/null
+++ b/tests/qemu-iotests/137
@@ -0,0 +1,144 @@
+#!/bin/bash
+#
+# Test qcow2 reopen
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto generic
+_supported_os Linux
+
+
+_make_test_img 64M
+
+echo === Try setting valid values for all options ===
+echo
+
+# Try all options and then check that all of the basic I/O operations still
+# work on this image.
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,pass-discard-request=on" \
+    -c "reopen -o lazy-refcounts=off,pass-discard-request=off" \
+    -c "reopen -o pass-discard-snapshot=on,pass-discard-other=on" \
+    -c "reopen -o pass-discard-snapshot=off,pass-discard-other=off" \
+    -c "reopen -o overlap-check=all" \
+    -c "reopen -o overlap-check=none" \
+    -c "reopen -o overlap-check=cached" \
+    -c "reopen -o overlap-check=constant" \
+    -c "reopen -o overlap-check.template=all" \
+    -c "reopen -o overlap-check.template=none" \
+    -c "reopen -o overlap-check.template=cached" \
+    -c "reopen -o overlap-check.template=constant" \
+    -c "reopen -o overlap-check.main-header=on" \
+    -c "reopen -o overlap-check.main-header=off" \
+    -c "reopen -o overlap-check.active-l1=on" \
+    -c "reopen -o overlap-check.active-l1=off" \
+    -c "reopen -o overlap-check.active-l2=on" \
+    -c "reopen -o overlap-check.active-l2=off" \
+    -c "reopen -o overlap-check.refcount-table=on" \
+    -c "reopen -o overlap-check.refcount-table=off" \
+    -c "reopen -o overlap-check.refcount-block=on" \
+    -c "reopen -o overlap-check.refcount-block=off" \
+    -c "reopen -o overlap-check.snapshot-table=on" \
+    -c "reopen -o overlap-check.snapshot-table=off" \
+    -c "reopen -o overlap-check.inactive-l1=on" \
+    -c "reopen -o overlap-check.inactive-l1=off" \
+    -c "reopen -o overlap-check.inactive-l2=on" \
+    -c "reopen -o overlap-check.inactive-l2=off" \
+    -c "reopen -o cache-size=1M" \
+    -c "reopen -o l2-cache-size=512k" \
+    -c "reopen -o refcount-cache-size=128k" \
+    -c "reopen -o cache-clean-interval=5" \
+    -c "reopen -o cache-clean-interval=0" \
+    -c "reopen -o cache-clean-interval=10" \
+    \
+    -c "write -P 55 0 32M" \
+    -c "read -P 55 0 32M" \
+    -c "discard 0 32M" \
+    -c "write -z 0 32M" \
+    -c "read -P 0 0 32M" \
+    \
+    "$TEST_IMG" | _filter_qemu_io
+
+
+echo
+echo === Try setting some invalid values ===
+echo
+
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=42" \
+    -c "reopen -o cache-size=1M,l2-cache-size=64k,refcount-cache-size=64k" \
+    -c "reopen -o cache-size=1M,l2-cache-size=2M" \
+    -c "reopen -o cache-size=1M,refcount-cache-size=2M" \
+    -c "reopen -o l2-cache-size=256T" \
+    -c "reopen -o refcount-cache-size=256T" \
+    -c "reopen -o overlap-check=constant,overlap-check.template=all" \
+    -c "reopen -o overlap-check=blubb" \
+    -c "reopen -o overlap-check.template=blubb" \
+    -c "reopen -o cache-clean-interval=-1" \
+    "$TEST_IMG" | _filter_qemu_io
+
+echo
+echo === Test transaction semantics ===
+echo
+
+# Whether lazy-refcounts was actually enabled can easily be tested: Check if
+# the dirty bit is set after a crash
+$QEMU_IO \
+    -c "reopen -o lazy-refcounts=on,overlap-check=blubb" \
+    -c "write -P 0x5a 0 512" \
+    -c "sigraise $(kill -l KILL)" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# The dirty bit must not be set
+$PYTHON qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
+
+# Similarly we can test whether corruption detection has been enabled:
+# Create L1/L2, overwrite first entry in refcount block, allocate something.
+# Disabling the checks should fail, so the corruption must be detected.
+_make_test_img 64M
+$QEMU_IO -c "write 0 64k" "$TEST_IMG" | _filter_qemu_io
+poke_file "$TEST_IMG" "$((0x20000))" "\x00\x00"
+$QEMU_IO \
+    -c "reopen -o overlap-check=none,lazy-refcounts=42" \
+    -c "write 64k 64k" \
+    "$TEST_IMG" 2>&1 | _filter_qemu_io
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/137.out b/tests/qemu-iotests/137.out
new file mode 100644
index 0000000..c0e7534
--- /dev/null
+++ b/tests/qemu-iotests/137.out
@@ -0,0 +1,46 @@
+QA output created by 137
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+=== Try setting valid values for all options ===
+
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+discard 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 33554432/33554432 bytes at offset 0
+32 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Try setting some invalid values ===
+
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+cache-size, l2-cache-size and refcount-cache-size may not be set the same time
+l2-cache-size may not exceed cache-size
+refcount-cache-size may not exceed cache-size
+L2 cache size too big
+L2 cache size too big
+Conflicting values for qcow2 options 'overlap-check' ('constant') and 'overlap-check.template' ('all')
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+Cache clean interval too big
+
+=== Test transaction semantics ===
+
+Unsupported value 'blubb' for qcow2 option 'overlap-check'. Allowed are any of the following: none, constant, cached, all
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./common.config: Killed                  ( if [ "${VALGRIND_QEMU}" == "y" ]; then
+    exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+else
+    exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@";
+fi )
+incompatible_features     0x0
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Parameter 'lazy-refcounts' expects 'on' or 'off'
+qcow2: Marking image as corrupt: Preventing invalid write on metadata (overlaps with qcow2_header); further corruption events will be suppressed
+write failed: Input/output error
+*** done
diff --git a/tests/qemu-iotests/138 b/tests/qemu-iotests/138
new file mode 100755
index 0000000..21650d8
--- /dev/null
+++ b/tests/qemu-iotests/138
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# General test case for qcow2's image check
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# This tests qocw2-specific low-level functionality
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+echo
+echo '=== Check on an image with a multiple of 2^32 clusters ==='
+echo
+
+IMGOPTS=$(_optstr_add "$IMGOPTS" "cluster_size=512") \
+    _make_test_img 512
+
+# Allocate L2 table
+$QEMU_IO -c 'write 0 512' "$TEST_IMG" | _filter_qemu_io
+
+# Put the data cluster at a multiple of 2 TB, resulting in the image apparently
+# having a multiple of 2^32 clusters
+# (To be more specific: It is at 32 PB)
+poke_file "$TEST_IMG" 2048 "\x80\x80\x00\x00\x00\x00\x00\x00"
+
+# An offset of 32 PB results in qemu-img check having to allocate an in-memory
+# refcount table of 128 TB (16 bit refcounts, 512 byte clusters).
+# This should be generally too much for any system and thus fail.
+# What this test is checking is that the qcow2 driver actually tries to allocate
+# such a large amount of memory (and is consequently aborting) instead of having
+# truncated the cluster count somewhere (which would result in much less memory
+# being allocated and then a segfault occurring).
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/138.out b/tests/qemu-iotests/138.out
new file mode 100644
index 0000000..3fe911f
--- /dev/null
+++ b/tests/qemu-iotests/138.out
@@ -0,0 +1,9 @@
+QA output created by 138
+
+=== Check on an image with a multiple of 2^32 clusters ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=512
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+qemu-img: Check failed: Cannot allocate memory
+*** done
diff --git a/tests/qemu-iotests/139 b/tests/qemu-iotests/139
new file mode 100644
index 0000000..a4b9694
--- /dev/null
+++ b/tests/qemu-iotests/139
@@ -0,0 +1,416 @@
+#!/usr/bin/env python
+#
+# Test cases for the QMP 'x-blockdev-del' command
+#
+# Copyright (C) 2015 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+import time
+
+base_img = os.path.join(iotests.test_dir, 'base.img')
+new_img = os.path.join(iotests.test_dir, 'new.img')
+
+class TestBlockdevDel(iotests.QMPTestCase):
+
+    def setUp(self):
+        iotests.qemu_img('create', '-f', iotests.imgfmt, base_img, '1M')
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(base_img)
+        if os.path.isfile(new_img):
+            os.remove(new_img)
+
+    # Check whether a BlockBackend exists
+    def checkBlockBackend(self, backend, node, must_exist = True):
+        result = self.vm.qmp('query-block')
+        backends = filter(lambda x: x['device'] == backend, result['return'])
+        self.assertLessEqual(len(backends), 1)
+        self.assertEqual(must_exist, len(backends) == 1)
+        if must_exist:
+            if node:
+                self.assertEqual(backends[0]['inserted']['node-name'], node)
+            else:
+                self.assertFalse(backends[0].has_key('inserted'))
+
+    # Check whether a BlockDriverState exists
+    def checkBlockDriverState(self, node, must_exist = True):
+        result = self.vm.qmp('query-named-block-nodes')
+        nodes = filter(lambda x: x['node-name'] == node, result['return'])
+        self.assertLessEqual(len(nodes), 1)
+        self.assertEqual(must_exist, len(nodes) == 1)
+
+    # Add a new BlockBackend (with its attached BlockDriverState)
+    def addBlockBackend(self, backend, node):
+        file_node = '%s_file' % node
+        self.checkBlockBackend(backend, node, False)
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(file_node, False)
+        opts = {'driver': iotests.imgfmt,
+                'id': backend,
+                'node-name': node,
+                'file': {'driver': 'file',
+                         'node-name': file_node,
+                         'filename': base_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(file_node)
+
+    # Add a BlockDriverState without a BlockBackend
+    def addBlockDriverState(self, node):
+        file_node = '%s_file' % node
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(file_node, False)
+        opts = {'driver': iotests.imgfmt,
+                'node-name': node,
+                'file': {'driver': 'file',
+                         'node-name': file_node,
+                         'filename': base_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(file_node)
+
+    # Add a BlockDriverState that will be used as overlay for the base_img BDS
+    def addBlockDriverStateOverlay(self, node):
+        self.checkBlockDriverState(node, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt,
+                         '-b', base_img, new_img, '1M')
+        opts = {'driver': iotests.imgfmt,
+                'node-name': node,
+                'backing': '',
+                'file': {'driver': 'file',
+                         'filename': new_img}}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+
+    # Delete a BlockBackend
+    def delBlockBackend(self, backend, node, expect_error = False,
+                        destroys_media = True):
+        self.checkBlockBackend(backend, node)
+        if node:
+            self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-del', id = backend)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+            if node:
+                self.checkBlockDriverState(node)
+        else:
+            self.assert_qmp(result, 'return', {})
+            if node:
+                self.checkBlockDriverState(node, not destroys_media)
+        self.checkBlockBackend(backend, node, must_exist = expect_error)
+
+    # Delete a BlockDriverState
+    def delBlockDriverState(self, node, expect_error = False):
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-del', node_name = node)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+        else:
+            self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node, expect_error)
+
+    # Add a device model
+    def addDeviceModel(self, device, backend):
+        result = self.vm.qmp('device_add', id = device,
+                             driver = 'virtio-blk-pci', drive = backend)
+        self.assert_qmp(result, 'return', {})
+
+    # Delete a device model
+    def delDeviceModel(self, device):
+        result = self.vm.qmp('device_del', id = device)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('system_reset')
+        self.assert_qmp(result, 'return', {})
+
+        device_path = '/machine/peripheral/%s/virtio-backend' % device
+        event = self.vm.event_wait(name="DEVICE_DELETED",
+                                   match={'data': {'path': device_path}})
+        self.assertNotEqual(event, None)
+
+        event = self.vm.event_wait(name="DEVICE_DELETED",
+                                   match={'data': {'device': device}})
+        self.assertNotEqual(event, None)
+
+    # Remove a BlockDriverState
+    def ejectDrive(self, backend, node, expect_error = False,
+                   destroys_media = True):
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('eject', device = backend)
+        if expect_error:
+            self.assert_qmp(result, 'error/class', 'GenericError')
+            self.checkBlockDriverState(node)
+            self.checkBlockBackend(backend, node)
+        else:
+            self.assert_qmp(result, 'return', {})
+            self.checkBlockDriverState(node, not destroys_media)
+            self.checkBlockBackend(backend, None)
+
+    # Insert a BlockDriverState
+    def insertDrive(self, backend, node):
+        self.checkBlockBackend(backend, None)
+        self.checkBlockDriverState(node)
+        result = self.vm.qmp('x-blockdev-insert-medium',
+                             device = backend, node_name = node)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(node)
+
+    # Create a snapshot using 'blockdev-snapshot-sync'
+    def createSnapshotSync(self, node, overlay):
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay, False)
+        opts = {'node-name': node,
+                'snapshot-file': new_img,
+                'snapshot-node-name': overlay,
+                'format': iotests.imgfmt}
+        result = self.vm.qmp('blockdev-snapshot-sync', conv_keys=False, **opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+
+    # Create a snapshot using 'blockdev-snapshot'
+    def createSnapshot(self, node, overlay):
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+        result = self.vm.qmp('blockdev-snapshot',
+                             node = node, overlay = overlay)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(overlay)
+
+    # Create a mirror
+    def createMirror(self, backend, node, new_node):
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(new_node, False)
+        opts = {'device': backend,
+                'target': new_img,
+                'node-name': new_node,
+                'sync': 'top',
+                'format': iotests.imgfmt}
+        result = self.vm.qmp('drive-mirror', conv_keys=False, **opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockBackend(backend, node)
+        self.checkBlockDriverState(new_node)
+
+    # Complete an existing block job
+    def completeBlockJob(self, backend, node_before, node_after):
+        self.checkBlockBackend(backend, node_before)
+        result = self.vm.qmp('block-job-complete', device=backend)
+        self.assert_qmp(result, 'return', {})
+        self.wait_until_completed(backend)
+        self.checkBlockBackend(backend, node_after)
+
+    # Add a BlkDebug node
+    # Note that the purpose of this is to test the x-blockdev-del
+    # sanity checks, not to create a usable blkdebug drive
+    def addBlkDebug(self, debug, node):
+        self.checkBlockDriverState(node, False)
+        self.checkBlockDriverState(debug, False)
+        image = {'driver': iotests.imgfmt,
+                 'node-name': node,
+                 'file': {'driver': 'file',
+                          'filename': base_img}}
+        opts = {'driver': 'blkdebug',
+                'node-name': debug,
+                'image': image}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(node)
+        self.checkBlockDriverState(debug)
+
+    # Add a BlkVerify node
+    # Note that the purpose of this is to test the x-blockdev-del
+    # sanity checks, not to create a usable blkverify drive
+    def addBlkVerify(self, blkverify, test, raw):
+        self.checkBlockDriverState(test, False)
+        self.checkBlockDriverState(raw, False)
+        self.checkBlockDriverState(blkverify, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, new_img, '1M')
+        node_0 = {'driver': iotests.imgfmt,
+                  'node-name': test,
+                  'file': {'driver': 'file',
+                           'filename': base_img}}
+        node_1 = {'driver': iotests.imgfmt,
+                  'node-name': raw,
+                  'file': {'driver': 'file',
+                           'filename': new_img}}
+        opts = {'driver': 'blkverify',
+                'node-name': blkverify,
+                'test': node_0,
+                'raw': node_1}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(test)
+        self.checkBlockDriverState(raw)
+        self.checkBlockDriverState(blkverify)
+
+    # Add a Quorum node
+    def addQuorum(self, quorum, child0, child1):
+        self.checkBlockDriverState(child0, False)
+        self.checkBlockDriverState(child1, False)
+        self.checkBlockDriverState(quorum, False)
+        iotests.qemu_img('create', '-f', iotests.imgfmt, new_img, '1M')
+        child_0 = {'driver': iotests.imgfmt,
+                   'node-name': child0,
+                   'file': {'driver': 'file',
+                            'filename': base_img}}
+        child_1 = {'driver': iotests.imgfmt,
+                   'node-name': child1,
+                   'file': {'driver': 'file',
+                            'filename': new_img}}
+        opts = {'driver': 'quorum',
+                'node-name': quorum,
+                'vote-threshold': 1,
+                'children': [ child_0, child_1 ]}
+        result = self.vm.qmp('blockdev-add', conv_keys = False, options = opts)
+        self.assert_qmp(result, 'return', {})
+        self.checkBlockDriverState(child0)
+        self.checkBlockDriverState(child1)
+        self.checkBlockDriverState(quorum)
+
+    ########################
+    # The tests start here #
+    ########################
+
+    def testWrongParameters(self):
+        self.addBlockBackend('drive0', 'node0')
+        result = self.vm.qmp('x-blockdev-del')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        result = self.vm.qmp('x-blockdev-del', id='drive0', node_name='node0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.delBlockBackend('drive0', 'node0')
+
+    def testBlockBackend(self):
+        self.addBlockBackend('drive0', 'node0')
+        # You cannot delete a BDS that is attached to a backend
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockBackend('drive0', 'node0')
+
+    def testBlockDriverState(self):
+        self.addBlockDriverState('node0')
+        # You cannot delete a file BDS directly
+        self.delBlockDriverState('node0_file', expect_error = True)
+        self.delBlockDriverState('node0')
+
+    def testEject(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.ejectDrive('drive0', 'node0')
+        self.delBlockBackend('drive0', None)
+
+    def testDeviceModel(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.addDeviceModel('device0', 'drive0')
+        self.ejectDrive('drive0', 'node0', expect_error = True)
+        self.delBlockBackend('drive0', 'node0', expect_error = True)
+        self.delDeviceModel('device0')
+        self.delBlockBackend('drive0', 'node0')
+
+    def testAttachMedia(self):
+        # This creates a BlockBackend and removes its media
+        self.addBlockBackend('drive0', 'node0')
+        self.ejectDrive('drive0', 'node0')
+        # This creates a new BlockDriverState and inserts it into the backend
+        self.addBlockDriverState('node1')
+        self.insertDrive('drive0', 'node1')
+        # The backend can't be removed: the new BDS has an extra reference
+        self.delBlockBackend('drive0', 'node1', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # The BDS still exists after being ejected, but now it can be removed
+        self.ejectDrive('drive0', 'node1', destroys_media = False)
+        self.delBlockDriverState('node1')
+        self.delBlockBackend('drive0', None)
+
+    def testSnapshotSync(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.createSnapshotSync('node0', 'overlay0')
+        # This fails because node0 is now being used as a backing image
+        self.delBlockDriverState('node0', expect_error = True)
+        # This succeeds because overlay0 only has the backend reference
+        self.delBlockBackend('drive0', 'overlay0')
+        self.checkBlockDriverState('node0', False)
+
+    def testSnapshot(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.addBlockDriverStateOverlay('overlay0')
+        self.createSnapshot('node0', 'overlay0')
+        self.delBlockBackend('drive0', 'overlay0', expect_error = True)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('overlay0', expect_error = True)
+        self.ejectDrive('drive0', 'overlay0', destroys_media = False)
+        self.delBlockBackend('drive0', None)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('overlay0')
+        self.checkBlockDriverState('node0', False)
+
+    def testMirror(self):
+        self.addBlockBackend('drive0', 'node0')
+        self.createMirror('drive0', 'node0', 'mirror0')
+        # The block job prevents removing the device
+        self.delBlockBackend('drive0', 'node0', expect_error = True)
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('mirror0', expect_error = True)
+        self.wait_ready('drive0')
+        self.completeBlockJob('drive0', 'node0', 'mirror0')
+        self.assert_no_active_block_jobs()
+        self.checkBlockDriverState('node0', False)
+        # This succeeds because the backend now points to mirror0
+        self.delBlockBackend('drive0', 'mirror0')
+
+    def testBlkDebug(self):
+        self.addBlkDebug('debug0', 'node0')
+        # 'node0' is used by the blkdebug node
+        self.delBlockDriverState('node0', expect_error = True)
+        # But we can remove the blkdebug node directly
+        self.delBlockDriverState('debug0')
+        self.checkBlockDriverState('node0', False)
+
+    def testBlkVerify(self):
+        self.addBlkVerify('verify0', 'node0', 'node1')
+        # We cannot remove the children of a blkverify device
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # But we can remove the blkverify node directly
+        self.delBlockDriverState('verify0')
+        self.checkBlockDriverState('node0', False)
+        self.checkBlockDriverState('node1', False)
+
+    def testQuorum(self):
+        if not 'quorum' in iotests.qemu_img_pipe('--help'):
+            return
+        self.addQuorum('quorum0', 'node0', 'node1')
+        # We cannot remove the children of a Quorum device
+        self.delBlockDriverState('node0', expect_error = True)
+        self.delBlockDriverState('node1', expect_error = True)
+        # But we can remove the Quorum node directly
+        self.delBlockDriverState('quorum0')
+        self.checkBlockDriverState('node0', False)
+        self.checkBlockDriverState('node1', False)
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=["qcow2"])
diff --git a/tests/qemu-iotests/139.out b/tests/qemu-iotests/139.out
new file mode 100644
index 0000000..281b69e
--- /dev/null
+++ b/tests/qemu-iotests/139.out
@@ -0,0 +1,5 @@
+............
+----------------------------------------------------------------------
+Ran 12 tests
+
+OK
diff --git a/tests/qemu-iotests/140 b/tests/qemu-iotests/140
new file mode 100755
index 0000000..49f9df4
--- /dev/null
+++ b/tests/qemu-iotests/140
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Test case for ejecting a BlockBackend with an NBD server attached to it
+#
+# Verify that the NBD server stops offering the drive when ejecting a
+# BlockDriverState tree from a BlockBackend (that is, a medium from a
+# drive) exposed via an NBD server.
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_DIR/nbd"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt generic
+_supported_proto file
+_supported_os Linux
+
+_make_test_img 64k
+
+$QEMU_IO -c 'write -P 42 0 64k' "$TEST_IMG" | _filter_qemu_io
+
+keep_stderr=y \
+_launch_qemu -drive if=none,media=cdrom,id=drv,file="$TEST_IMG",format=$IMGFMT \
+    2> >(_filter_nbd)
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'nbd-server-start',
+       'arguments': { 'addr': { 'type': 'unix',
+                                'data': { 'path': '$TEST_DIR/nbd' }}}}" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'nbd-server-add',
+       'arguments': { 'device': 'drv' }}" \
+    'return'
+
+$QEMU_IO_PROG -f raw -c 'read -P 42 0 64k' \
+    "nbd+unix:///drv?socket=$TEST_DIR/nbd" 2>&1 \
+    | _filter_qemu_io | _filter_nbd
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'eject',
+       'arguments': { 'device': 'drv' }}" \
+    'return'
+
+$QEMU_IO_PROG -f raw -c close \
+    "nbd+unix:///drv?socket=$TEST_DIR/nbd" 2>&1 \
+    | _filter_qemu_io | _filter_nbd
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/140.out b/tests/qemu-iotests/140.out
new file mode 100644
index 0000000..0409cd0
--- /dev/null
+++ b/tests/qemu-iotests/140.out
@@ -0,0 +1,15 @@
+QA output created by 140
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+wrote 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"return": {}}
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+can't open device nbd+unix:///drv?socket=TEST_DIR/nbd: No export with name 'drv' available
+no file open, try 'help open'
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+*** done
diff --git a/tests/qemu-iotests/141 b/tests/qemu-iotests/141
new file mode 100755
index 0000000..b2617e5
--- /dev/null
+++ b/tests/qemu-iotests/141
@@ -0,0 +1,185 @@
+#!/bin/bash
+#
+# Test case for ejecting BDSs with block jobs still running on them
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f "$TEST_DIR/{b,m,o}.$IMGFMT"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+# Needs backing file and backing format support
+_supported_fmt qcow2 qed
+_supported_proto file
+_supported_os Linux
+
+
+test_blockjob()
+{
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'blockdev-add',
+          'arguments': {
+              'options': {
+                  'id': 'drv0',
+                  'driver': '$IMGFMT',
+                  'file': {
+                      'driver': 'file',
+                      'filename': '$TEST_IMG'
+                  }}}}" \
+        'return'
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "$1" \
+        "$2" \
+        | _filter_img_create
+
+    # We want this to return an error because the block job is still running
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'x-blockdev-remove-medium',
+          'arguments': {'device': 'drv0'}}" \
+        'error'
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'block-job-cancel',
+          'arguments': {'device': 'drv0'}}" \
+        "$3"
+
+    _send_qemu_cmd $QEMU_HANDLE \
+        "{'execute': 'x-blockdev-del',
+          'arguments': {'id': 'drv0'}}" \
+        'return'
+}
+
+
+TEST_IMG="$TEST_DIR/b.$IMGFMT" _make_test_img 1M
+TEST_IMG="$TEST_DIR/m.$IMGFMT" _make_test_img -b "$TEST_DIR/b.$IMGFMT" 1M
+_make_test_img -b "$TEST_DIR/m.$IMGFMT" 1M
+
+_launch_qemu -nodefaults
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{'execute': 'qmp_capabilities'}" \
+    'return'
+
+echo
+echo '=== Testing drive-backup ==='
+echo
+
+# drive-backup will not send BLOCK_JOB_READY by itself, and cancelling the job
+# will consequently result in BLOCK_JOB_CANCELLED being emitted.
+
+test_blockjob \
+    "{'execute': 'drive-backup',
+      'arguments': {'device': 'drv0',
+                    'target': '$TEST_DIR/o.$IMGFMT',
+                    'format': '$IMGFMT',
+                    'sync': 'none'}}" \
+    'return' \
+    'BLOCK_JOB_CANCELLED'
+
+echo
+echo '=== Testing drive-mirror ==='
+echo
+
+# drive-mirror will send BLOCK_JOB_READY basically immediately, and cancelling
+# the job will consequently result in BLOCK_JOB_COMPLETED being emitted.
+
+test_blockjob \
+    "{'execute': 'drive-mirror',
+      'arguments': {'device': 'drv0',
+                    'target': '$TEST_DIR/o.$IMGFMT',
+                    'format': '$IMGFMT',
+                    'sync': 'none'}}" \
+    'BLOCK_JOB_READY' \
+    'BLOCK_JOB_COMPLETED'
+
+echo
+echo '=== Testing active block-commit ==='
+echo
+
+# An active block-commit will send BLOCK_JOB_READY basically immediately, and
+# cancelling the job will consequently result in BLOCK_JOB_COMPLETED being
+# emitted.
+
+test_blockjob \
+    "{'execute': 'block-commit',
+      'arguments': {'device': 'drv0'}}" \
+    'BLOCK_JOB_READY' \
+    'BLOCK_JOB_COMPLETED'
+
+echo
+echo '=== Testing non-active block-commit ==='
+echo
+
+# Give block-commit something to work on, otherwise it would be done
+# immediately, send a BLOCK_JOB_COMPLETED and ejecting the BDS would work just
+# fine without the block job still running.
+
+$QEMU_IO -c 'write 0 1M' "$TEST_DIR/m.$IMGFMT" | _filter_qemu_io
+
+test_blockjob \
+    "{'execute': 'block-commit',
+      'arguments': {'device': 'drv0',
+                    'top':    '$TEST_DIR/m.$IMGFMT',
+                    'speed':  1}}" \
+    'return' \
+    'BLOCK_JOB_CANCELLED'
+
+echo
+echo '=== Testing block-stream ==='
+echo
+
+# Give block-stream something to work on, otherwise it would be done
+# immediately, send a BLOCK_JOB_COMPLETED and ejecting the BDS would work just
+# fine without the block job still running.
+
+$QEMU_IO -c 'write 0 1M' "$TEST_DIR/b.$IMGFMT" | _filter_qemu_io
+
+# With some data to stream (and @speed set to 1), block-stream will not complete
+# until we send the block-job-cancel command. Therefore, no event other than
+# BLOCK_JOB_CANCELLED will be emitted.
+
+test_blockjob \
+    "{'execute': 'block-stream',
+      'arguments': {'device': 'drv0',
+                    'speed': 1}}" \
+    'return' \
+    'BLOCK_JOB_CANCELLED'
+
+_cleanup_qemu
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/141.out b/tests/qemu-iotests/141.out
new file mode 100644
index 0000000..eaf1e60
--- /dev/null
+++ b/tests/qemu-iotests/141.out
@@ -0,0 +1,59 @@
+QA output created by 141
+Formatting 'TEST_DIR/b.IMGFMT', fmt=IMGFMT size=1048576
+Formatting 'TEST_DIR/m.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/b.IMGFMT
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/m.IMGFMT
+{"return": {}}
+
+=== Testing drive-backup ===
+
+{"return": {}}
+Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: backup"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "drv0", "len": 1048576, "offset": 0, "speed": 0, "type": "backup"}}
+{"return": {}}
+
+=== Testing drive-mirror ===
+
+{"return": {}}
+Formatting 'TEST_DIR/o.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT backing_fmt=IMGFMT
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: mirror"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "mirror"}}
+{"return": {}}
+
+=== Testing active block-commit ===
+
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "drv0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
+
+=== Testing non-active block-commit ===
+
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: commit"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "drv0", "len": 1048576, "offset": 524288, "speed": 1, "type": "commit"}}
+{"return": {}}
+
+=== Testing block-stream ===
+
+wrote 1048576/1048576 bytes at offset 0
+1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"error": {"class": "GenericError", "desc": "Node 'drv0' is busy: block device is in use by block job: stream"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_CANCELLED", "data": {"device": "drv0", "len": 1048576, "offset": 524288, "speed": 1, "type": "stream"}}
+{"return": {}}
+*** done
diff --git a/tests/qemu-iotests/142 b/tests/qemu-iotests/142
new file mode 100755
index 0000000..29c0606
--- /dev/null
+++ b/tests/qemu-iotests/142
@@ -0,0 +1,358 @@
+#!/bin/bash
+#
+# Test for configuring cache modes of arbitrary nodes (requires O_DIRECT)
+#
+# Copyright (C) 2015 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f $TEST_IMG.snap
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+# We test all cache modes anyway, but O_DIRECT needs to be supported
+_default_cache_mode none
+_supported_cache_modes none directsync
+
+function do_run_qemu()
+{
+    echo Testing: "$@"
+    (
+        if ! test -t 0; then
+            while read cmd; do
+                echo $cmd
+            done
+        fi
+        echo quit
+    ) | $QEMU -nographic -monitor stdio -nodefaults "$@"
+    echo
+}
+
+function run_qemu()
+{
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu
+}
+
+size=128M
+
+TEST_IMG="$TEST_IMG.base" _make_test_img $size
+TEST_IMG="$TEST_IMG.snap" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base" $size
+
+echo
+echo === Simple test for all cache modes ===
+echo
+
+run_qemu -drive file="$TEST_IMG",cache=none
+run_qemu -drive file="$TEST_IMG",cache=directsync
+run_qemu -drive file="$TEST_IMG",cache=writeback
+run_qemu -drive file="$TEST_IMG",cache=writethrough
+run_qemu -drive file="$TEST_IMG",cache=unsafe
+run_qemu -drive file="$TEST_IMG",cache=invalid_value
+
+echo
+echo === Check inheritance of cache modes ===
+echo
+
+files="if=none,file=$TEST_IMG,backing.file.filename=$TEST_IMG.base"
+ids="node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file"
+
+function check_cache_all()
+{
+    # cache.direct is supposed to be inherited by both bs->file and
+    # bs->backing
+
+    echo -e "cache.direct=on on none0"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+    echo -e "\ncache.direct=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",file.cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+    echo -e "\ncache.direct=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+    echo -e "\ncache.direct=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.file.cache.direct=on | grep -e "Cache" -e "[Cc]annot|[Cc]ould not|[Cc]an't"
+
+    # cache.writeback is supposed to be inherited by bs->backing; bs->file
+    # always gets cache.writeback=on
+
+    echo -e "\n\ncache.writeback=off on none0"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",cache.writeback=off | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",file.cache.writeback=off | grep -e "doesn't" -e "does not"
+    echo -e "\ncache.writeback=off on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.cache.writeback=off | grep -e "doesn't" -e "does not"
+    echo -e "\ncache.writeback=off on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.file.cache.writeback=off | grep -e "doesn't" -e "does not"
+
+    # cache.no-flush is supposed to be inherited by both bs->file and bs->backing
+
+    echo -e "\n\ncache.no-flush=on on none0"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",file.cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$files","$ids",backing.file.cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+}
+
+echo
+echo "--- Configure cache modes on the command line ---"
+echo
+
+# First check the inherited cache mode after opening the image.
+
+hmp_cmds="info block none0
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all
+
+echo
+echo "--- Cache modes after reopen (live snapshot) ---"
+echo
+
+# Then trigger a reopen and check that the cache modes are still the same.
+
+hmp_cmds="snapshot_blkdev -n none0 $TEST_IMG.snap $IMGFMT
+info block
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all
+
+echo
+echo "--- Change cache modes with reopen (qemu-io command, flags) ---"
+echo
+
+# This one actually changes the cache mode with the reopen. For this test, the
+# new cache mode is specified in the flags, not as an option.
+
+hmp_cmds='qemu-io none0 "reopen -c none"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all
+
+echo
+echo "--- Change cache modes with reopen (qemu-io command, options) ---"
+echo
+
+# This one actually changes the cache mode with the reopen. For this test, the
+# new cache mode is specified as an option, not in the flags.
+
+hmp_cmds='qemu-io none0 "reopen -o cache.direct=on"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all
+
+echo
+echo "--- Change cache modes after snapshot ---"
+echo
+
+# This checks that the original image doesn't inherit from the snapshot
+
+hmp_cmds="snapshot_blkdev -n none0 $TEST_IMG.snap $IMGFMT
+qemu-io none0 \"reopen -c none\"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all
+
+echo
+echo "--- Change cache mode in parent, child has explicit option in JSON ---"
+echo
+
+# This checks that children with options explicitly set by the json:
+# pseudo-protocol don't inherit these options from their parents.
+#
+# Yes, blkdebug::json:... is criminal, but I can't see another way to have a
+# BDS initialised with the json: pseudo-protocol, but still have it inherit
+# options from its parent node.
+
+hmp_cmds="qemu-io none0 \"reopen -o cache.direct=on,cache.no-flush=on\"
+info block none0
+info block image
+info block blkdebug
+info block file"
+
+echo "$hmp_cmds" | run_qemu -drive if=none,file="blkdebug::json:{\"filename\":\"$TEST_IMG\",,\"cache\":{\"direct\":false}}",node-name=image,file.node-name=blkdebug,file.image.node-name=file | grep "Cache"
+
+echo
+echo "=== Check that referenced BDSes don't inherit ==="
+echo
+
+drv_bkfile="if=none,driver=file,filename=$TEST_IMG.base,node-name=backing-file"
+drv_bk="if=none,file=json:{'driver':'$IMGFMT',,'file':'backing-file',,'node-name':'backing'}"
+drv_file="if=none,driver=file,filename=$TEST_IMG,node-name=file"
+drv_img="if=none,id=blk,file=json:{'driver':'$IMGFMT',,'file':'file',,'backing':'backing',,'node-name':'image'}"
+
+function check_cache_all_separate()
+{
+    # Check cache.direct
+
+    echo -e "cache.direct=on on blk"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img",cache.direct=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.direct=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file",cache.direct=on -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.direct=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk",cache.direct=on -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.direct=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile",cache.direct=on -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+
+    # Check cache.writeback
+
+    echo -e "\n\ncache.writeback=off on blk"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img",cache.writeback=off | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file",cache.writeback=off -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk",cache.writeback=off -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.writeback=off on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile",cache.writeback=off -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+
+    # Check cache.no-flush
+
+    echo -e "\n\ncache.no-flush=on on blk"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img",cache.no-flush=on | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk" -drive "$drv_file",cache.no-flush=on -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile" -drive "$drv_bk",cache.no-flush=on -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+    echo -e "\ncache.no-flush=on on backing-file"
+    echo "$hmp_cmds" | run_qemu -drive "$drv_bkfile",cache.no-flush=on -drive "$drv_bk" -drive "$drv_file" -drive "$drv_img" | grep -e "Cache" -e "[Cc]annot\|[Cc]ould not\|[Cc]an't"
+}
+
+echo
+echo "--- Configure cache modes on the command line ---"
+echo
+
+# First check the inherited cache mode after opening the image.
+
+hmp_cmds="info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all_separate
+
+echo
+echo "--- Cache modes after reopen (live snapshot) ---"
+echo
+
+# Then trigger a reopen and check that the cache modes are still the same.
+
+hmp_cmds="snapshot_blkdev -n blk $TEST_IMG.snap $IMGFMT
+info block blk
+info block image
+info block file
+info block backing
+info block backing-file"
+
+check_cache_all_separate
+
+echo
+echo "--- Change cache modes with reopen (qemu-io command, flags) ---"
+echo
+
+# This one actually changes the cache mode with the reopen. For this test, the
+# new cache mode is specified as flags, not as option.
+
+hmp_cmds='qemu-io blk "reopen -c none"
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all_separate
+
+
+echo
+echo "=== Reopening children instead of the root ==="
+echo
+
+files="if=none,file=$TEST_IMG,backing.file.filename=$TEST_IMG.base"
+ids="node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file"
+
+echo
+echo "--- Basic reopen ---"
+echo
+
+hmp_cmds='qemu-io none0 "reopen -o backing.cache.direct=on"
+info block none0
+info block image
+info block file
+info block backing
+info block backing-file'
+
+check_cache_all
+
+echo
+echo "--- Change cache mode after reopening child ---"
+echo
+
+# This checks that children with options explicitly set with reopen don't
+# inherit these options from their parents any more
+
+# TODO Implement node-name support for 'qemu-io' HMP command for -c
+# Can use only -o to access child node options for now
+
+hmp_cmds="qemu-io none0 \"reopen -o file.cache.direct=off,file.cache.no-flush=off\"
+qemu-io none0 \"reopen -o backing.file.cache.direct=off,backing.file.cache.no-flush=on\"
+qemu-io none0 \"reopen -c none\"
+info block image
+info block file
+info block backing
+info block backing-file"
+
+echo "$hmp_cmds" | run_qemu -drive "$files","$ids" | grep "Cache"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/142.out b/tests/qemu-iotests/142.out
new file mode 100644
index 0000000..600beca
--- /dev/null
+++ b/tests/qemu-iotests/142.out
@@ -0,0 +1,750 @@
+QA output created by 142
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT.snap', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+
+=== Simple test for all cache modes ===
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=none
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=directsync
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writeback
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=writethrough
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=unsafe
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+
+Testing: -drive file=TEST_DIR/t.qcow2,cache=invalid_value
+QEMU_PROG: -drive file=TEST_DIR/t.qcow2,cache=invalid_value: invalid cache option
+
+
+=== Check inheritance of cache modes ===
+
+
+--- Configure cache modes on the command line ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Cache modes after reopen (live snapshot) ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Change cache modes with reopen (qemu-io command, flags) ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+
+--- Change cache modes with reopen (qemu-io command, options) ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+
+--- Change cache modes after snapshot ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Change cache mode in parent, child has explicit option in JSON ---
+
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, ignore flushes
+
+=== Check that referenced BDSes don't inherit ===
+
+
+--- Configure cache modes on the command line ---
+
+cache.direct=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on blk
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+
+cache.no-flush=on on blk
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Cache modes after reopen (live snapshot) ---
+
+cache.direct=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on blk
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+
+cache.no-flush=on on blk
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+--- Change cache modes with reopen (qemu-io command, flags) ---
+
+cache.direct=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.direct=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+
+cache.direct=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.writeback=off on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+
+cache.no-flush=on on blk
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+    Cache mode:       writeback
+
+cache.no-flush=on on backing
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+
+=== Reopening children instead of the root ===
+
+
+--- Basic reopen ---
+
+cache.direct=on on none0
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.direct=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+
+cache.writeback=off on none0
+    Cache mode:       writethrough
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.writeback=off on file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,file.cache.writeback=off: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+cache.writeback=off on backing
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.cache.writeback=off: Could not open backing file: Block format 'qcow2' does not support the option 'cache.writeback'
+
+cache.writeback=off on backing-file
+QEMU_PROG: -drive if=none,file=TEST_DIR/t.qcow2,backing.file.filename=TEST_DIR/t.qcow2.base,node-name=image,backing.node-name=backing,backing.file.node-name=backing-file,file.node-name=file,backing.file.cache.writeback=off: Could not open backing file: Block protocol 'file' doesn't support the option 'cache.writeback'
+
+
+cache.no-flush=on on none0
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, ignore flushes
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct
+
+cache.no-flush=on on backing
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct, ignore flushes
+    Cache mode:       writeback, direct, ignore flushes
+
+cache.no-flush=on on backing-file
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, direct, ignore flushes
+
+--- Change cache mode after reopening child ---
+
+    Cache mode:       writeback, direct
+    Cache mode:       writeback
+    Cache mode:       writeback, direct
+    Cache mode:       writeback, ignore flushes
+*** done
diff --git a/tests/qemu-iotests/143 b/tests/qemu-iotests/143
new file mode 100755
index 0000000..ec4ef22
--- /dev/null
+++ b/tests/qemu-iotests/143
@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# Test case for connecting to a non-existing NBD export name
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    rm -f "$TEST_DIR/nbd"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt generic
+_supported_proto generic
+_supported_os Linux
+
+keep_stderr=y \
+_launch_qemu 2> >(_filter_nbd)
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'nbd-server-start',
+       'arguments': { 'addr': { 'type': 'unix',
+                                'data': { 'path': '$TEST_DIR/nbd' }}}}" \
+    'return'
+
+# This should just result in a client error, not in the server crashing
+$QEMU_IO_PROG -f raw -c quit \
+    "nbd+unix:///no_such_export?socket=$TEST_DIR/nbd" 2>&1 \
+    | _filter_qemu_io | _filter_nbd
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/143.out b/tests/qemu-iotests/143.out
new file mode 100644
index 0000000..d24ad20
--- /dev/null
+++ b/tests/qemu-iotests/143.out
@@ -0,0 +1,7 @@
+QA output created by 143
+{"return": {}}
+{"return": {}}
+can't open device nbd+unix:///no_such_export?socket=TEST_DIR/nbd: No export with name 'no_such_export' available
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+*** done
diff --git a/tests/qemu-iotests/144 b/tests/qemu-iotests/144
new file mode 100755
index 0000000..00de3c3
--- /dev/null
+++ b/tests/qemu-iotests/144
@@ -0,0 +1,114 @@
+#!/bin/bash
+# Check live snapshot, followed by active commit, and another snapshot.
+#
+# This test is to catch the error case of BZ #1300209:
+# https://bugzilla.redhat.com/show_bug.cgi?id=1300209
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+TMP_SNAP1=${TEST_DIR}/tmp.qcow2
+TMP_SNAP2=${TEST_DIR}/tmp2.qcow2
+
+_cleanup()
+{
+    _cleanup_qemu
+    rm -f "${TEST_IMG}" "${TMP_SNAP1}" "${TMP_SNAP2}"
+}
+
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+size=512M
+
+_make_test_img $size
+
+echo
+echo === Launching QEMU ===
+echo
+
+qemu_comm_method="qmp"
+_launch_qemu -drive file="${TEST_IMG}",if=virtio
+h=$QEMU_HANDLE
+
+
+echo
+echo === Performing Live Snapshot 1 ===
+echo
+
+_send_qemu_cmd $h "{ 'execute': 'qmp_capabilities' }" "return"
+
+
+# First live snapshot, new overlay as active layer
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
+                                'arguments': {
+                                             'device': 'virtio0',
+                                             'snapshot-file':'${TMP_SNAP1}',
+                                             'format': 'qcow2'
+                                             }
+                    }" "return"
+
+echo
+echo === Performing block-commit on active layer ===
+echo
+
+# Block commit on active layer, push the new overlay into base
+_send_qemu_cmd $h "{ 'execute': 'block-commit',
+                                'arguments': {
+                                                 'device': 'virtio0'
+                                              }
+                    }" "READY"
+
+_send_qemu_cmd $h "{ 'execute': 'block-job-complete',
+                                'arguments': {
+                                                'device': 'virtio0'
+                                              }
+                   }" "COMPLETED"
+
+echo
+echo === Performing Live Snapshot 2 ===
+echo
+
+# New live snapshot, new overlays as active layer
+_send_qemu_cmd $h "{ 'execute': 'blockdev-snapshot-sync',
+                                'arguments': {
+                                                'device': 'virtio0',
+                                                'snapshot-file':'${TMP_SNAP2}',
+                                                'format': 'qcow2'
+                                              }
+                   }" "return"
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/144.out b/tests/qemu-iotests/144.out
new file mode 100644
index 0000000..387855c
--- /dev/null
+++ b/tests/qemu-iotests/144.out
@@ -0,0 +1,24 @@
+QA output created by 144
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=536870912
+
+=== Launching QEMU ===
+
+
+=== Performing Live Snapshot 1 ===
+
+{"return": {}}
+Formatting 'TEST_DIR/tmp.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+{"return": {}}
+
+=== Performing block-commit on active layer ===
+
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "virtio0", "len": 0, "offset": 0, "speed": 0, "type": "commit"}}
+
+=== Performing Live Snapshot 2 ===
+
+Formatting 'TEST_DIR/tmp2.qcow2', fmt=qcow2 size=536870912 backing_file=TEST_DIR/t.qcow2 backing_fmt=qcow2 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16
+{"return": {}}
+*** done
diff --git a/tests/qemu-iotests/145 b/tests/qemu-iotests/145
new file mode 100755
index 0000000..1eca0e8
--- /dev/null
+++ b/tests/qemu-iotests/145
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Test the combination of -incoming and snapshot=on
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+	true
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_proto generic
+_supported_os Linux
+
+_make_test_img 1M
+echo quit | $QEMU -nographic -hda "$TEST_IMG" -incoming 'exec:true' -snapshot -serial none -monitor stdio | _filter_qemu
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/145.out b/tests/qemu-iotests/145.out
new file mode 100644
index 0000000..75b5c8a
--- /dev/null
+++ b/tests/qemu-iotests/145.out
@@ -0,0 +1,5 @@
+QA output created by 145
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
+QEMU X.Y.Z monitor - type 'help' for more information
+(qemu) q[K[Dqu[K[D[Dqui[K[D[D[Dquit[K
+*** done
diff --git a/tests/qemu-iotests/146 b/tests/qemu-iotests/146
new file mode 100755
index 0000000..043711b
--- /dev/null
+++ b/tests/qemu-iotests/146
@@ -0,0 +1,165 @@
+#!/bin/bash
+#
+# Test VHD image format creator detection and override
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=jcody@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1    # failure is the default!
+
+_cleanup()
+{
+    _cleanup_qemu
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt vpc
+_supported_proto file
+_supported_os Linux
+
+
+qemu_comm_method="monitor"
+silent=
+
+echo
+echo === Testing VPC Autodetect ===
+echo
+_use_sample_img virtualpc-dynamic.vhd.bz2
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing VPC with current_size force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing VPC with chs force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+_cleanup_test_img
+
+echo
+echo === Testing Hyper-V Autodetect ===
+echo
+_use_sample_img hyperv2012r2-dynamic.vhd.bz2
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing Hyper-V with current_size force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing Hyper-V with chs force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+_cleanup_test_img
+
+echo
+echo === Testing d2v Autodetect ===
+echo
+_use_sample_img d2v-zerofilled.vhd.bz2
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing d2v with current_size force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing d2v with chs force ===
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+_cleanup_test_img
+
+echo
+echo === Testing Image create, default ===
+echo
+
+TEST_IMG="${TEST_DIR}/vpc-create-test.vpc"
+
+_make_test_img 4G
+
+echo
+echo === Read created image, default opts ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=chs ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=current_size ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo
+echo === Testing Image create, force_size ===
+echo
+
+_make_test_img -o force_size 4G
+
+echo
+echo === Read created image, default opts ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=chs ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=chs ${TEST_IMG}" -c 'map'
+
+echo
+echo === Read created image, force_size_calc=current_size ====
+echo
+
+${QEMU_IO} -c "open -o driver=vpc,force_size_calc=current_size ${TEST_IMG}" -c 'map'
+
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/146.out b/tests/qemu-iotests/146.out
new file mode 100644
index 0000000..4f334d8
--- /dev/null
+++ b/tests/qemu-iotests/146.out
@@ -0,0 +1,70 @@
+QA output created by 146
+
+=== Testing VPC Autodetect ===
+
+[                       0]  266334240/ 266334240 sectors not allocated at offset 0 bytes (0)
+
+=== Testing VPC with current_size force ===
+
+[                       0]  266338304/ 266338304 sectors not allocated at offset 0 bytes (0)
+
+=== Testing VPC with chs force ===
+
+[                       0]  266334240/ 266334240 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Hyper-V Autodetect ===
+
+[                       0]  266338304/ 266338304 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Hyper-V with current_size force ===
+
+[                       0]  266338304/ 266338304 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Hyper-V with chs force ===
+
+[                       0]  266334240/ 266334240 sectors not allocated at offset 0 bytes (0)
+
+=== Testing d2v Autodetect ===
+
+[                       0]   514560/  514560 sectors     allocated at offset 0 bytes (1)
+
+=== Testing d2v with current_size force ===
+
+[                       0]   514560/  514560 sectors     allocated at offset 0 bytes (1)
+
+=== Testing d2v with chs force ===
+
+[                       0]   514560/  514560 sectors     allocated at offset 0 bytes (1)
+
+=== Testing Image create, default ===
+
+Formatting 'TEST_DIR/IMGFMT-create-test.IMGFMT', fmt=IMGFMT size=4294967296
+
+=== Read created image, default opts ====
+
+[                       0]  8389584/ 8389584 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=chs ====
+
+[                       0]  8389584/ 8389584 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=current_size ====
+
+[                       0]  8389584/ 8389584 sectors not allocated at offset 0 bytes (0)
+
+=== Testing Image create, force_size ===
+
+Formatting 'TEST_DIR/IMGFMT-create-test.IMGFMT', fmt=IMGFMT size=4294967296 force_size=on
+
+=== Read created image, default opts ====
+
+[                       0]  8388608/ 8388608 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=chs ====
+
+[                       0]  8388608/ 8388608 sectors not allocated at offset 0 bytes (0)
+
+=== Read created image, force_size_calc=current_size ====
+
+[                       0]  8388608/ 8388608 sectors not allocated at offset 0 bytes (0)
+*** done
diff --git a/tests/qemu-iotests/148 b/tests/qemu-iotests/148
new file mode 100644
index 0000000..e01b061
--- /dev/null
+++ b/tests/qemu-iotests/148
@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+#
+# Test the rate limit of QMP events
+#
+# Copyright (C) 2016 Igalia, S.L.
+# Author: Alberto Garcia <berto@igalia.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+
+imgs = (os.path.join(iotests.test_dir, 'quorum0.img'),
+        os.path.join(iotests.test_dir, 'quorum1.img'),
+        os.path.join(iotests.test_dir, 'quorum2.img'))
+
+img_conf = (os.path.join(iotests.test_dir, 'quorum0.conf'),
+            os.path.join(iotests.test_dir, 'quorum1.conf'),
+            os.path.join(iotests.test_dir, 'quorum2.conf'))
+
+event_rate = 1000000000
+sector_size = 512
+offset = 10
+
+class TestQuorumEvents(iotests.QMPTestCase):
+    read_pattern = 'quorum'
+
+    def create_blkdebug_file(self, blkdebug_file, bad_sector):
+        file = open(blkdebug_file, 'w')
+        file.write('''
+[inject-error]
+event = "read_aio"
+errno = "5"
+sector = "%d"
+''' % bad_sector)
+        file.close()
+
+    def setUp(self):
+        driveopts = ['driver=quorum', 'vote-threshold=2']
+        driveopts.append('read-pattern=%s' % self.read_pattern)
+        for i in range(len(imgs)):
+            iotests.qemu_img('create', '-f', iotests.imgfmt, imgs[i], '1M')
+            self.create_blkdebug_file(img_conf[i], i + offset)
+            driveopts.append('children.%d.driver=%s' % (i, iotests.imgfmt))
+            driveopts.append('children.%d.file.driver=blkdebug' % i)
+            driveopts.append('children.%d.file.config=%s' % (i, img_conf[i]))
+            driveopts.append('children.%d.file.image.filename=%s' % (i, imgs[i]))
+            driveopts.append('children.%d.node-name=img%d' % (i, i))
+        self.vm = iotests.VM()
+        self.vm.add_drive(None, opts = ','.join(driveopts))
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        for i in range(len(imgs)):
+            os.remove(imgs[i])
+            os.remove(img_conf[i])
+
+    def do_check_event(self, node, sector = 0):
+        if node == None:
+            self.assertEqual(self.vm.get_qmp_event(), None)
+            return
+
+        for event in self.vm.get_qmp_events(wait=True):
+            if event['event'] == 'QUORUM_REPORT_BAD':
+                self.assert_qmp(event, 'data/node-name', node)
+                self.assert_qmp(event, 'data/sector-num', sector)
+
+    def testQuorum(self):
+        # Generate an error and get an event
+        self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                            (offset * sector_size, sector_size))
+        self.vm.qtest("clock_step 10")
+        self.do_check_event('img0', offset)
+
+        # I/O errors in the same child: only one event is emitted
+        delay = 10
+        for i in range(3):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                (offset * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            self.do_check_event(None)
+
+        # Wait enough so the event is finally emitted
+        self.vm.qtest("clock_step %d" % (2 * event_rate))
+        self.do_check_event('img0', offset)
+
+        # I/O errors in the same child: all events are emitted
+        delay = 2 * event_rate
+        for i in range(3):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                (offset * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            self.do_check_event('img0', offset)
+
+        # I/O errors in different children: all events are emitted
+        delay = 10
+        for i in range(len(imgs)):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                ((offset + i) * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            # In fifo mode only errors in the first child are detected
+            if i > 0 and self.read_pattern == 'fifo':
+                self.do_check_event(None)
+            else:
+                self.do_check_event('img%d' % i, offset + i)
+
+        # I/O errors in different children: all events are emitted
+        delay = 2 * event_rate
+        for i in range(len(imgs)):
+            self.vm.hmp_qemu_io("drive0", "aio_read %d %d" %
+                                ((offset + i) * sector_size, sector_size))
+            self.vm.qtest("clock_step %d" % delay)
+            # In fifo mode only errors in the first child are detected
+            if i > 0 and self.read_pattern == 'fifo':
+                self.do_check_event(None)
+            else:
+                self.do_check_event('img%d' % i, offset + i)
+
+        # No more pending events
+        self.do_check_event(None)
+
+class TestFifoQuorumEvents(TestQuorumEvents):
+    read_pattern = 'fifo'
+
+if __name__ == '__main__':
+    iotests.verify_quorum()
+    iotests.main(supported_fmts=["raw"])
diff --git a/tests/qemu-iotests/148.out b/tests/qemu-iotests/148.out
new file mode 100644
index 0000000..fbc63e6
--- /dev/null
+++ b/tests/qemu-iotests/148.out
@@ -0,0 +1,5 @@
+..
+----------------------------------------------------------------------
+Ran 2 tests
+
+OK
diff --git a/tests/qemu-iotests/149 b/tests/qemu-iotests/149
new file mode 100755
index 0000000..8407251
--- /dev/null
+++ b/tests/qemu-iotests/149
@@ -0,0 +1,531 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Creator/Owner: Daniel P. Berrange <berrange@redhat.com>
+#
+# Exercise the QEMU 'luks' block driver to validate interoperability
+# with the Linux dm-crypt + cryptsetup implementation
+
+import subprocess
+import os
+import os.path
+
+import base64
+
+import iotests
+
+
+class LUKSConfig(object):
+    """Represent configuration parameters for a single LUKS
+       setup to be tested"""
+
+    def __init__(self, name, cipher, keylen, mode, ivgen,
+                 ivgen_hash, hash, password=None, passwords=None):
+
+        self.name = name
+        self.cipher = cipher
+        self.keylen = keylen
+        self.mode = mode
+        self.ivgen = ivgen
+        self.ivgen_hash = ivgen_hash
+        self.hash = hash
+
+        if passwords is not None:
+            self.passwords = passwords
+        else:
+            self.passwords = {}
+
+            if password is None:
+                self.passwords["0"] = "123456"
+            else:
+                self.passwords["0"] = password
+
+    def __repr__(self):
+        return self.name
+
+    def image_name(self):
+        return "luks-%s.img" % self.name
+
+    def image_path(self):
+        return os.path.join(iotests.test_dir, self.image_name())
+
+    def device_name(self):
+        return "qiotest-145-%s" % self.name
+
+    def device_path(self):
+        return "/dev/mapper/" + self.device_name()
+
+    def first_password(self):
+        for i in range(8):
+            slot = str(i)
+            if slot in self.passwords:
+                return (self.passwords[slot], slot)
+        raise Exception("No password found")
+
+    def first_password_base64(self):
+        (pw, slot) = self.first_password()
+        return base64.b64encode(pw)
+
+    def active_slots(self):
+        slots = []
+        for i in range(8):
+            slot = str(i)
+            if slot in self.passwords:
+                slots.append(slot)
+        return slots
+
+def verify_passwordless_sudo():
+    """Check whether sudo is configured to allow
+       password-less access to commands"""
+
+    args = ["sudo", "-n", "/bin/true"]
+
+    proc = subprocess.Popen(args,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    msg = proc.communicate()[0]
+
+    if proc.returncode != 0:
+        iotests.notrun('requires password-less sudo access: %s' % msg)
+
+
+def cryptsetup(args, password=None):
+    """Run the cryptsetup command in batch mode"""
+
+    fullargs = ["sudo", "cryptsetup", "-q", "-v"]
+    fullargs.extend(args)
+
+    iotests.log(" ".join(fullargs), filters=[iotests.filter_test_dir])
+    proc = subprocess.Popen(fullargs,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    msg = proc.communicate(password)[0]
+
+    if proc.returncode != 0:
+        raise Exception(msg)
+
+
+def cryptsetup_add_password(config, slot):
+    """Add another password to a LUKS key slot"""
+
+    (password, mainslot) = config.first_password()
+
+    pwfile = os.path.join(iotests.test_dir, "passwd.txt")
+    with open(pwfile, "w") as fh:
+        fh.write(config.passwords[slot])
+
+    try:
+        args = ["luksAddKey", config.image_path(),
+                "--key-slot", slot,
+                "--key-file", "-",
+                pwfile]
+
+        cryptsetup(args, password)
+    finally:
+        os.unlink(pwfile)
+
+
+def cryptsetup_format(config):
+    """Format a new LUKS volume with cryptsetup, adding the
+    first key slot only"""
+
+    (password, slot) = config.first_password()
+
+    args = ["luksFormat"]
+    cipher = config.cipher + "-" + config.mode + "-" + config.ivgen
+    if config.ivgen_hash is not None:
+        cipher = cipher + ":" + config.ivgen_hash
+    elif config.ivgen == "essiv":
+        cipher = cipher + ":" + "sha256"
+    args.extend(["--cipher", cipher])
+    if config.mode == "xts":
+        args.extend(["--key-size", str(config.keylen * 2)])
+    else:
+        args.extend(["--key-size", str(config.keylen)])
+    if config.hash is not None:
+        args.extend(["--hash", config.hash])
+    args.extend(["--key-slot", slot])
+    args.extend(["--key-file", "-"])
+    args.append(config.image_path())
+
+    cryptsetup(args, password)
+
+
+def chown(config):
+    """Set the ownership of a open LUKS device to this user"""
+
+    path = config.device_path()
+
+    args = ["sudo", "chown", "%d:%d" % (os.getuid(), os.getgid()), path]
+    iotests.log(" ".join(args), filters=[iotests.filter_chown])
+    proc = subprocess.Popen(args,
+                            stdin=subprocess.PIPE,
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    msg = proc.communicate()[0]
+
+    if proc.returncode != 0:
+        raise Exception("Cannot change owner on %s" % path)
+
+
+def cryptsetup_open(config):
+    """Open an image as a LUKS device"""
+
+    (password, slot) = config.first_password()
+
+    args = ["luksOpen", config.image_path(), config.device_name()]
+
+    cryptsetup(args, password)
+
+
+def cryptsetup_close(config):
+    """Close an active LUKS device """
+
+    args = ["luksClose", config.device_name()]
+    cryptsetup(args)
+
+
+def delete_image(config):
+    """Delete a disk image"""
+
+    try:
+        os.unlink(config.image_path())
+        iotests.log("unlink %s" % config.image_path(),
+                    filters=[iotests.filter_test_dir])
+    except Exception as e:
+        pass
+
+
+def create_image(config, size_mb):
+    """Create a bare disk image with requested size"""
+
+    delete_image(config)
+    iotests.log("truncate %s --size %dMB" % (config.image_path(), size_mb),
+                filters=[iotests.filter_test_dir])
+    with open(config.image_path(), "w") as fn:
+        fn.truncate(size_mb * 1024 * 1024)
+
+
+def qemu_img_create(config, size_mb):
+    """Create and format a disk image with LUKS using qemu-img"""
+
+    opts = [
+        "key-secret=sec0",
+        "cipher-alg=%s-%d" % (config.cipher, config.keylen),
+        "cipher-mode=%s" % config.mode,
+        "ivgen-alg=%s" % config.ivgen,
+        "hash-alg=%s" % config.hash,
+    ]
+    if config.ivgen_hash is not None:
+        opts.append("ivgen-hash-alg=%s" % config.ivgen_hash)
+
+    args = ["create", "-f", "luks",
+            "--object",
+            ("secret,id=sec0,data=%s,format=base64" %
+             config.first_password_base64()),
+            "-o", ",".join(opts),
+            config.image_path(),
+            "%dM" % size_mb]
+
+    iotests.log("qemu-img " + " ".join(args), filters=[iotests.filter_test_dir])
+    iotests.log(iotests.qemu_img_pipe(*args), filters=[iotests.filter_test_dir])
+
+def qemu_io_image_args(config, dev=False):
+    """Get the args for access an image or device with qemu-io"""
+
+    if dev:
+        return [
+            "--image-opts",
+            "driver=file,filename=%s" % config.device_path()]
+    else:
+        return [
+            "--object",
+            ("secret,id=sec0,data=%s,format=base64" %
+             config.first_password_base64()),
+            "--image-opts",
+            ("driver=luks,key-secret=sec0,file.filename=%s" %
+             config.image_path())]
+
+def qemu_io_write_pattern(config, pattern, offset_mb, size_mb, dev=False):
+    """Write a pattern of data to a LUKS image or device"""
+
+    args = ["-c", "write -P 0x%x %dM %dM" % (pattern, offset_mb, size_mb)]
+    args.extend(qemu_io_image_args(config, dev))
+    iotests.log("qemu-io " + " ".join(args), filters=[iotests.filter_test_dir])
+    iotests.log(iotests.qemu_io(*args), filters=[iotests.filter_test_dir,
+                                                 iotests.filter_qemu_io])
+
+
+def qemu_io_read_pattern(config, pattern, offset_mb, size_mb, dev=False):
+    """Read a pattern of data to a LUKS image or device"""
+
+    args = ["-c", "read -P 0x%x %dM %dM" % (pattern, offset_mb, size_mb)]
+    args.extend(qemu_io_image_args(config, dev))
+    iotests.log("qemu-io " + " ".join(args), filters=[iotests.filter_test_dir])
+    iotests.log(iotests.qemu_io(*args), filters=[iotests.filter_test_dir,
+                                                 iotests.filter_qemu_io])
+
+
+def test_once(config, qemu_img=False):
+    """Run the test with a desired LUKS configuration. Can either
+       use qemu-img for creating the initial volume, or cryptsetup,
+       in order to test interoperability in both directions"""
+
+    iotests.log("# ================= %s %s =================" % (
+        "qemu-img" if qemu_img else "dm-crypt", config))
+
+    oneKB = 1024
+    oneMB = oneKB * 1024
+    oneGB = oneMB * 1024
+    oneTB = oneGB * 1024
+
+    # 4 TB, so that we pass the 32-bit sector number boundary.
+    # Important for testing correctness of some IV generators
+    # The files are sparse, so not actually using this much space
+    image_size = 4 * oneTB
+    if qemu_img:
+        iotests.log("# Create image")
+        qemu_img_create(config, image_size / oneMB)
+    else:
+        iotests.log("# Create image")
+        create_image(config, image_size / oneMB)
+
+    lowOffsetMB = 100
+    highOffsetMB = 3 * oneTB / oneMB
+
+    try:
+        if not qemu_img:
+            iotests.log("# Format image")
+            cryptsetup_format(config)
+
+            for slot in config.active_slots()[1:]:
+                iotests.log("# Add password slot %s" % slot)
+                cryptsetup_add_password(config, slot)
+
+        # First we'll open the image using cryptsetup and write a
+        # known pattern of data that we'll then verify with QEMU
+
+        iotests.log("# Open dev")
+        cryptsetup_open(config)
+
+        try:
+            iotests.log("# Set dev owner")
+            chown(config)
+
+            iotests.log("# Write test pattern 0xa7")
+            qemu_io_write_pattern(config, 0xa7, lowOffsetMB, 10, dev=True)
+            iotests.log("# Write test pattern 0x13")
+            qemu_io_write_pattern(config, 0x13, highOffsetMB, 10, dev=True)
+        finally:
+            iotests.log("# Close dev")
+            cryptsetup_close(config)
+
+        # Ok, now we're using QEMU to verify the pattern just
+        # written via dm-crypt
+
+        iotests.log("# Read test pattern 0xa7")
+        qemu_io_read_pattern(config, 0xa7, lowOffsetMB, 10, dev=False)
+        iotests.log("# Read test pattern 0x13")
+        qemu_io_read_pattern(config, 0x13, highOffsetMB, 10, dev=False)
+
+
+        # Write a new pattern to the image, which we'll later
+        # verify with dm-crypt
+        iotests.log("# Write test pattern 0x91")
+        qemu_io_write_pattern(config, 0x91, lowOffsetMB, 10, dev=False)
+        iotests.log("# Write test pattern 0x5e")
+        qemu_io_write_pattern(config, 0x5e, highOffsetMB, 10, dev=False)
+
+
+        # Now we're opening the image with dm-crypt once more
+        # and verifying what QEMU wrote, completing the circle
+        iotests.log("# Open dev")
+        cryptsetup_open(config)
+
+        try:
+            iotests.log("# Set dev owner")
+            chown(config)
+
+            iotests.log("# Read test pattern 0x91")
+            qemu_io_read_pattern(config, 0x91, lowOffsetMB, 10, dev=True)
+            iotests.log("# Read test pattern 0x5e")
+            qemu_io_read_pattern(config, 0x5e, highOffsetMB, 10, dev=True)
+        finally:
+            iotests.log("# Close dev")
+            cryptsetup_close(config)
+    finally:
+        iotests.log("# Delete image")
+        delete_image(config)
+        print
+
+
+# Obviously we only work with the luks image format
+iotests.verify_image_format(supported_fmts=['luks'])
+iotests.verify_platform()
+
+# We need sudo in order to run cryptsetup to create
+# dm-crypt devices. This is safe to use on any
+# machine, since all dm-crypt devices are backed
+# by newly created plain files, and have a dm-crypt
+# name prefix of 'qiotest' to avoid clashing with
+# user LUKS volumes
+verify_passwordless_sudo()
+
+
+# If we look at all permutations of cipher, key size,
+# mode, ivgen, hash, there are ~1000 possible configs.
+#
+# We certainly don't want/need to test every permutation
+# to get good validation of interoperability between QEMU
+# and dm-crypt/cryptsetup.
+#
+# The configs below are a representative set that aim to
+# exercise each axis of configurability.
+#
+configs = [
+    # A common LUKS default
+    LUKSConfig("aes-256-xts-plain64-sha1",
+               "aes", 256, "xts", "plain64", None, "sha1"),
+
+
+    # LUKS default but diff ciphers
+    LUKSConfig("twofish-256-xts-plain64-sha1",
+               "twofish", 256, "xts", "plain64", None, "sha1"),
+    LUKSConfig("serpent-256-xts-plain64-sha1",
+               "serpent", 256, "xts", "plain64", None, "sha1"),
+    # Should really be xts, but kernel doesn't support xts+cast5
+    # nor does it do essiv+cast5
+    LUKSConfig("cast5-128-cbc-plain64-sha1",
+               "cast5", 128, "cbc", "plain64", None, "sha1"),
+    LUKSConfig("cast6-256-xts-plain64-sha1",
+               "cast6", 256, "xts", "plain64", None, "sha1"),
+
+
+    # LUKS default but diff modes / ivgens
+    LUKSConfig("aes-256-cbc-plain-sha1",
+               "aes", 256, "cbc", "plain", None, "sha1"),
+    LUKSConfig("aes-256-cbc-plain64-sha1",
+               "aes", 256, "cbc", "plain64", None, "sha1"),
+    LUKSConfig("aes-256-cbc-essiv-sha256-sha1",
+               "aes", 256, "cbc", "essiv", "sha256", "sha1"),
+    LUKSConfig("aes-256-xts-essiv-sha256-sha1",
+               "aes", 256, "xts", "essiv", "sha256", "sha1"),
+
+
+    # LUKS default but smaller key sizes
+    LUKSConfig("aes-128-xts-plain64-sha256-sha1",
+               "aes", 128, "xts", "plain64", None, "sha1"),
+    LUKSConfig("aes-192-xts-plain64-sha256-sha1",
+               "aes", 192, "xts", "plain64", None, "sha1"),
+
+    LUKSConfig("twofish-128-xts-plain64-sha1",
+               "twofish", 128, "xts", "plain64", None, "sha1"),
+    LUKSConfig("twofish-192-xts-plain64-sha1",
+               "twofish", 192, "xts", "plain64", None, "sha1"),
+
+    LUKSConfig("serpent-128-xts-plain64-sha1",
+               "serpent", 128, "xts", "plain64", None, "sha1"),
+    LUKSConfig("serpent-192-xts-plain64-sha1",
+               "serpent", 192, "xts", "plain64", None, "sha1"),
+
+    LUKSConfig("cast6-128-xts-plain64-sha1",
+               "cast6", 128, "xts", "plain", None, "sha1"),
+    LUKSConfig("cast6-192-xts-plain64-sha1",
+               "cast6", 192, "xts", "plain64", None, "sha1"),
+
+
+    # LUKS default but diff hash
+    LUKSConfig("aes-256-xts-plain64-sha256",
+               "aes", 256, "xts", "plain64", None, "sha256"),
+    LUKSConfig("aes-256-xts-plain64-sha512",
+               "aes", 256, "xts", "plain64", None, "sha512"),
+    LUKSConfig("aes-256-xts-plain64-ripemd160",
+               "aes", 256, "xts", "plain64", None, "ripemd160"),
+
+    # Password in slot 3
+    LUKSConfig("aes-256-xts-plain-sha1-pwslot3",
+               "aes", 256, "xts", "plain", None, "sha1",
+               passwords={
+                   "3": "slot3",
+               }),
+
+    # Passwords in every slot
+    LUKSConfig("aes-256-xts-plain-sha1-pwallslots",
+               "aes", 256, "xts", "plain", None, "sha1",
+               passwords={
+                   "0": "slot1",
+                   "1": "slot1",
+                   "2": "slot2",
+                   "3": "slot3",
+                   "4": "slot4",
+                   "5": "slot5",
+                   "6": "slot6",
+                   "7": "slot7",
+               }),
+
+    # Check handling of default hash alg (sha256) with essiv
+    LUKSConfig("aes-256-cbc-essiv-auto-sha1",
+               "aes", 256, "cbc", "essiv", None, "sha1"),
+
+    # Check that a useless hash provided for 'plain64' iv gen
+    # is ignored and no error raised
+    LUKSConfig("aes-256-cbc-plain64-sha256-sha1",
+               "aes", 256, "cbc", "plain64", "sha256", "sha1"),
+
+]
+
+blacklist = [
+    # We don't have a cast-6 cipher impl for QEMU yet
+    "cast6-256-xts-plain64-sha1",
+    "cast6-128-xts-plain64-sha1",
+    "cast6-192-xts-plain64-sha1",
+
+    # GCrypt doesn't support Twofish with 192 bit key
+    "twofish-192-xts-plain64-sha1",
+
+    # We don't have sha512 hash wired up yet
+    "aes-256-xts-plain64-sha512",
+
+    # We don't have ripemd160 hash wired up yet
+    "aes-256-xts-plain64-ripemd160",
+]
+
+whitelist = []
+if "LUKS_CONFIG" in os.environ:
+    whitelist = os.environ["LUKS_CONFIG"].split(",")
+
+for config in configs:
+    if config.name in blacklist:
+        iotests.log("Skipping %s in blacklist" % config.name)
+        continue
+
+    if len(whitelist) > 0 and config.name not in whitelist:
+        iotests.log("Skipping %s not in whitelist" % config.name)
+        continue
+
+    test_once(config, qemu_img=False)
+
+    # XXX we should support setting passwords in a non-0
+    # key slot with 'qemu-img create' in future
+    (pw, slot) = config.first_password()
+    if slot == "0":
+        test_once(config, qemu_img=True)
diff --git a/tests/qemu-iotests/149.out b/tests/qemu-iotests/149.out
new file mode 100644
index 0000000..90b5b55
--- /dev/null
+++ b/tests/qemu-iotests/149.out
@@ -0,0 +1,2120 @@
+# ================= dm-crypt aes-256-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+
+# ================= qemu-img aes-256-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-256-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha1.img qiotest-145-aes-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha1.img
+
+# ================= dm-crypt twofish-256-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-twofish-256-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher twofish-xts-plain64 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+
+# ================= qemu-img twofish-256-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=twofish-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-twofish-256-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-twofish-256-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=twofish-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-256-xts-plain64-sha1.img qiotest-145-twofish-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-256-xts-plain64-sha1.img
+
+# ================= dm-crypt serpent-256-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-serpent-256-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher serpent-xts-plain64 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+
+# ================= qemu-img serpent-256-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=serpent-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-serpent-256-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-serpent-256-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=serpent-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-256-xts-plain64-sha1.img qiotest-145-serpent-256-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-256-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-256-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-256-xts-plain64-sha1.img
+
+# ================= dm-crypt cast5-128-cbc-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher cast5-cbc-plain64 --key-size 128 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+
+# ================= qemu-img cast5-128-cbc-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=cast5-128,cipher-mode=cbc,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=cast5-128 cipher-mode=cbc ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img qiotest-145-cast5-128-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-cast5-128-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-cast5-128-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-cast5-128-cbc-plain64-sha1.img
+
+Skipping cast6-256-xts-plain64-sha1 in blacklist
+# ================= dm-crypt aes-256-cbc-plain-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-plain-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-plain --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+
+# ================= qemu-img aes-256-cbc-plain-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=plain,hash-alg=sha1 TEST_DIR/luks-aes-256-cbc-plain-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-plain-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=plain hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain-sha1.img qiotest-145-aes-256-cbc-plain-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain-sha1.img
+
+# ================= dm-crypt aes-256-cbc-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+
+# ================= qemu-img aes-256-cbc-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-256-cbc-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha1.img qiotest-145-aes-256-cbc-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain64-sha1.img
+
+# ================= dm-crypt aes-256-cbc-essiv-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-essiv:sha256 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+
+# ================= qemu-img aes-256-cbc-essiv-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=essiv,hash-alg=sha1,ivgen-hash-alg=sha256 TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=essiv ivgen-hash-alg=sha256 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-essiv-sha256-sha1.img
+
+# ================= dm-crypt aes-256-xts-essiv-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-essiv:sha256 --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+
+# ================= qemu-img aes-256-xts-essiv-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=essiv,hash-alg=sha1,ivgen-hash-alg=sha256 TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=essiv ivgen-hash-alg=sha256 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-essiv-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-essiv-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-essiv-sha256-sha1.img
+
+# ================= dm-crypt aes-128-xts-plain64-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+
+# ================= qemu-img aes-128-xts-plain64-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-128,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-128 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-128-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-128-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-128-xts-plain64-sha256-sha1.img
+
+# ================= dm-crypt aes-192-xts-plain64-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 384 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+
+# ================= qemu-img aes-192-xts-plain64-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-192,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-192 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-192-xts-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-192-xts-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-192-xts-plain64-sha256-sha1.img
+
+# ================= dm-crypt twofish-128-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-twofish-128-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher twofish-xts-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+
+# ================= qemu-img twofish-128-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=twofish-128,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-twofish-128-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-twofish-128-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=twofish-128 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-twofish-128-xts-plain64-sha1.img qiotest-145-twofish-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-twofish-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-twofish-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-twofish-128-xts-plain64-sha1.img
+
+Skipping twofish-192-xts-plain64-sha1 in blacklist
+# ================= dm-crypt serpent-128-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-serpent-128-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher serpent-xts-plain64 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+
+# ================= qemu-img serpent-128-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=serpent-128,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-serpent-128-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-serpent-128-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=serpent-128 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-128-xts-plain64-sha1.img qiotest-145-serpent-128-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-128-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-128-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-128-xts-plain64-sha1.img
+
+# ================= dm-crypt serpent-192-xts-plain64-sha1 =================
+# Create image
+truncate TEST_DIR/luks-serpent-192-xts-plain64-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher serpent-xts-plain64 --key-size 384 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+
+# ================= qemu-img serpent-192-xts-plain64-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=serpent-192,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha1 TEST_DIR/luks-serpent-192-xts-plain64-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-serpent-192-xts-plain64-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=serpent-192 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-serpent-192-xts-plain64-sha1.img qiotest-145-serpent-192-xts-plain64-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-serpent-192-xts-plain64-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-serpent-192-xts-plain64-sha1
+# Delete image
+unlink TEST_DIR/luks-serpent-192-xts-plain64-sha1.img
+
+Skipping cast6-128-xts-plain64-sha1 in blacklist
+Skipping cast6-192-xts-plain64-sha1 in blacklist
+# ================= dm-crypt aes-256-xts-plain64-sha256 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain64-sha256.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain64 --key-size 512 --hash sha256 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+
+# ================= qemu-img aes-256-xts-plain64-sha256 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=plain64,hash-alg=sha256 TEST_DIR/luks-aes-256-xts-plain64-sha256.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-plain64-sha256.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=plain64 hash-alg=sha256
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain64-sha256.img qiotest-145-aes-256-xts-plain64-sha256
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain64-sha256
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain64-sha256
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain64-sha256.img
+
+Skipping aes-256-xts-plain64-sha512 in blacklist
+Skipping aes-256-xts-plain64-ripemd160 in blacklist
+# ================= dm-crypt aes-256-xts-plain-sha1-pwslot3 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain --key-size 512 --hash sha1 --key-slot 3 --key-file - TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=c2xvdDM=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwslot3
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwslot3
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain-sha1-pwslot3.img
+
+# ================= dm-crypt aes-256-xts-plain-sha1-pwallslots =================
+# Create image
+truncate TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-xts-plain --key-size 512 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+# Add password slot 1
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 1 --key-file - TEST_DIR/passwd.txt
+# Add password slot 2
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 2 --key-file - TEST_DIR/passwd.txt
+# Add password slot 3
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 3 --key-file - TEST_DIR/passwd.txt
+# Add password slot 4
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 4 --key-file - TEST_DIR/passwd.txt
+# Add password slot 5
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 5 --key-file - TEST_DIR/passwd.txt
+# Add password slot 6
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 6 --key-file - TEST_DIR/passwd.txt
+# Add password slot 7
+sudo cryptsetup -q -v luksAddKey TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img --key-slot 7 --key-file - TEST_DIR/passwd.txt
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+
+# ================= qemu-img aes-256-xts-plain-sha1-pwallslots =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=c2xvdDE=,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=xts,ivgen-alg=plain,hash-alg=sha1 TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=xts ivgen-alg=plain hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=c2xvdDE=,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-xts-plain-sha1-pwallslots
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-xts-plain-sha1-pwallslots
+# Delete image
+unlink TEST_DIR/luks-aes-256-xts-plain-sha1-pwallslots.img
+
+# ================= dm-crypt aes-256-cbc-essiv-auto-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-essiv:sha256 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+
+# ================= qemu-img aes-256-cbc-essiv-auto-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=essiv,hash-alg=sha1 TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=essiv hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-essiv-auto-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-essiv-auto-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-essiv-auto-sha1.img
+
+# ================= dm-crypt aes-256-cbc-plain64-sha256-sha1 =================
+# Create image
+truncate TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img --size 4194304MB
+# Format image
+sudo cryptsetup -q -v luksFormat --cipher aes-cbc-plain64:sha256 --key-size 256 --hash sha1 --key-slot 0 --key-file - TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+
+# ================= qemu-img aes-256-cbc-plain64-sha256-sha1 =================
+# Create image
+qemu-img create -f luks --object secret,id=sec0,data=MTIzNDU2,format=base64 -o key-secret=sec0,cipher-alg=aes-256,cipher-mode=cbc,ivgen-alg=plain64,hash-alg=sha1,ivgen-hash-alg=sha256 TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img 4194304M
+Formatting 'TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img', fmt=luks size=4398046511104 key-secret=sec0 cipher-alg=aes-256 cipher-mode=cbc ivgen-alg=plain64 ivgen-hash-alg=sha256 hash-alg=sha1
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Write test pattern 0xa7
+qemu-io -c write -P 0xa7 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x13
+qemu-io -c write -P 0x13 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Read test pattern 0xa7
+qemu-io -c read -P 0xa7 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x13
+qemu-io -c read -P 0x13 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x91
+qemu-io -c write -P 0x91 100M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Write test pattern 0x5e
+qemu-io -c write -P 0x5e 3145728M 10M --object secret,id=sec0,data=MTIzNDU2,format=base64 --image-opts driver=luks,key-secret=sec0,file.filename=TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+wrote 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Open dev
+sudo cryptsetup -q -v luksOpen TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Set dev owner
+sudo chown UID:GID /dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Read test pattern 0x91
+qemu-io -c read -P 0x91 100M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 104857600
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Read test pattern 0x5e
+qemu-io -c read -P 0x5e 3145728M 10M --image-opts driver=file,filename=/dev/mapper/qiotest-145-aes-256-cbc-plain64-sha256-sha1
+read 10485760/10485760 bytes at offset 3298534883328
+10 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+# Close dev
+sudo cryptsetup -q -v luksClose qiotest-145-aes-256-cbc-plain64-sha256-sha1
+# Delete image
+unlink TEST_DIR/luks-aes-256-cbc-plain64-sha256-sha1.img
+
diff --git a/tests/qemu-iotests/150 b/tests/qemu-iotests/150
new file mode 100755
index 0000000..ee8f637
--- /dev/null
+++ b/tests/qemu-iotests/150
@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# Test that qemu-img convert -S 0 fully allocates the target image
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt raw qcow2
+_supported_proto file
+_supported_os Linux
+
+
+img_size=1048576
+
+
+echo
+echo '=== Mapping sparse conversion ==='
+echo
+
+$QEMU_IMG_PROG convert -O "$IMGFMT" -S 512 \
+    "json:{ 'driver': 'null-co', 'size': $img_size, 'read-zeroes': true }" \
+    "$TEST_IMG"
+
+$QEMU_IMG map "$TEST_IMG" | _filter_qemu_img_map
+
+
+echo
+echo '=== Mapping non-sparse conversion ==='
+echo
+
+$QEMU_IMG convert -O "$IMGFMT" -S 0 \
+    "json:{ 'driver': 'null-co', 'size': $img_size, 'read-zeroes': true }" \
+    "$TEST_IMG"
+
+$QEMU_IMG map "$TEST_IMG" | _filter_qemu_img_map
+
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/150.out b/tests/qemu-iotests/150.out
new file mode 100644
index 0000000..2a54e8d
--- /dev/null
+++ b/tests/qemu-iotests/150.out
@@ -0,0 +1,11 @@
+QA output created by 150
+
+=== Mapping sparse conversion ===
+
+Offset          Length          File
+
+=== Mapping non-sparse conversion ===
+
+Offset          Length          File
+0               0x100000        TEST_DIR/t.IMGFMT
+*** done
diff --git a/tests/qemu-iotests/152 b/tests/qemu-iotests/152
new file mode 100644
index 0000000..fec546d
--- /dev/null
+++ b/tests/qemu-iotests/152
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#
+# Tests for drive-mirror with source size unaligned to granularity
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+from iotests import qemu_img
+
+test_img = os.path.join(iotests.test_dir, 'test.img')
+target_img = os.path.join(iotests.test_dir, 'target.img')
+
+class TestUnaligned(iotests.QMPTestCase):
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, test_img, '512')
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def test_unaligned(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             granularity=65536, target=target_img)
+        self.complete_and_wait()
+        self.vm.shutdown()
+        self.assertEqual(iotests.image_size(test_img), iotests.image_size(target_img),
+                         "Target size doesn't match source when granularity when unaligend")
+
+    def test_unaligned_with_update(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             granularity=65536, target=target_img)
+        self.wait_ready()
+        self.vm.hmp_qemu_io('drive0', 'write 0 512')
+        self.complete_and_wait(wait_ready=False)
+        self.vm.shutdown()
+        self.assertEqual(iotests.image_size(test_img), iotests.image_size(target_img),
+                         "Target size doesn't match source when granularity when unaligend")
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['raw', 'qcow2'])
diff --git a/tests/qemu-iotests/152.out b/tests/qemu-iotests/152.out
new file mode 100644
index 0000000..fbc63e6
--- /dev/null
+++ b/tests/qemu-iotests/152.out
@@ -0,0 +1,5 @@
+..
+----------------------------------------------------------------------
+Ran 2 tests
+
+OK
diff --git a/tests/qemu-iotests/154 b/tests/qemu-iotests/154
new file mode 100755
index 0000000..7ca7219
--- /dev/null
+++ b/tests/qemu-iotests/154
@@ -0,0 +1,305 @@
+#!/bin/bash
+#
+# qcow2 specific bdrv_pwrite_zeroes tests with backing files (complements 034)
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+CLUSTER_SIZE=4k
+size=128M
+
+echo
+echo == backing file contains zeros ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Make sure that the whole cluster is allocated even for partial write_zeroes
+# when the backing file contains zeros
+
+# X = non-zero data sector in backing file
+# - = sector unallocated in whole backing chain
+# 0 = sector touched by write_zeroes request
+
+# 1. Tail unaligned:    00 00 -- --
+# 2. Head unaligned:    -- -- 00 00
+# 3. Both unaligned:    -- 00 00 --
+# 4. Both, 2 clusters:  -- -- -- 00 | 00 -- -- --
+
+$QEMU_IO -c "write -z 0 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -z 10k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -z 17k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "write -z 27k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == backing file contains non-zero data before write_zeroes ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Single cluster; non-zero data at the cluster start
+# ... | XX -- 00 -- | ...
+$QEMU_IO -c "write -P 0x11 32k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 34k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 32k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 33k 3k" "$TEST_IMG" | _filter_qemu_io
+
+# Single cluster; non-zero data exists, but not at the cluster start
+# ... | -- XX 00 -- | ...
+$QEMU_IO -c "write -P 0x11 65k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 66k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 65k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 64k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 66k 2k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == backing file contains non-zero data after write_zeroes ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Single cluster; non-zero data directly after request
+# ... | -- 00 XX -- | ...
+$QEMU_IO -c "write -P 0x11 34k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 33k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 32k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 34k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 35k 1k" "$TEST_IMG" | _filter_qemu_io
+
+# Single cluster; non-zero data exists, but not directly after request
+# ... | -- 00 -- XX | ...
+$QEMU_IO -c "write -P 0x11 43k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 41k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 43k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 40k 3k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == write_zeroes covers non-zero data ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# non-zero data at front of request
+# Backing file: -- XX -- --
+# Active layer: -- 00 00 --
+
+$QEMU_IO -c "write -P 0x11 5k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 5k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 4k 4k" "$TEST_IMG" | _filter_qemu_io
+
+# non-zero data at end of request
+# Backing file: -- -- XX --
+# Active layer: -- 00 00 --
+
+$QEMU_IO -c "write -P 0x11 14k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 13k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 12k 4k" "$TEST_IMG" | _filter_qemu_io
+
+# non-zero data matches size of request
+# Backing file: -- XX XX --
+# Active layer: -- 00 00 --
+
+$QEMU_IO -c "write -P 0x11 21k 2k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 21k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 20k 4k" "$TEST_IMG" | _filter_qemu_io
+
+# non-zero data smaller than request
+# Backing file: -- -X X- --
+# Active layer: -- 00 00 --
+
+$QEMU_IO -c "write -P 0x11 30208 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 29k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 28k 4k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning two clusters, non-zero before request ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Two clusters; non-zero data before request:
+# 1. At cluster start:          32k: XX -- -- 00 | 00 -- -- --
+# 2. Between unallocated space: 48k: -- XX -- 00 | 00 -- -- --
+# 3. Directly before request:   64k: -- -- XX 00 | 00 -- -- --
+
+$QEMU_IO -c "write -P 0x11 32k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 35k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 32k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 33k 7k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IO -c "write -P 0x11 49k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 51k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 48k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 49k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 50k 6k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IO -c "write -P 0x11 66k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 67k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 64k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 66k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 67k 5k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning two clusters, non-zero after request ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Two clusters; non-zero data after request:
+# 1. Directly after request:    32k: -- -- -- 00 | 00 XX -- --
+# 2. Between unallocated space: 48k: -- -- -- 00 | 00 -- XX --
+# 3. At cluster end:            64k: -- -- -- 00 | 00 -- -- XX
+
+$QEMU_IO -c "write -P 0x11 37k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 35k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 32k 5k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 37k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 38k 2k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IO -c "write -P 0x11 54k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 51k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 48k 6k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 54k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 55k 1k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IO -c "write -P 0x11 71k 1k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 67k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 64k 7k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 71k 1k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning two clusters, partially overwriting backing file ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Backing file: -- -- XX XX | XX XX -- --
+# Active layer: -- -- XX 00 | 00 XX -- --
+
+$QEMU_IO -c "write -P 0x11 2k 4k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 3k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 0k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 2k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 3k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 5k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 6k 2k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning multiple clusters, non-zero in first cluster ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Backing file: 64k: XX XX -- -- | -- -- -- -- | -- -- -- --
+# Active layer: 64k: XX XX 00 00 | 00 00 00 00 | 00 -- -- --
+
+$QEMU_IO -c "write -P 0x11 64k 2k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 66k 7k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 64k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 66k 10k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning multiple clusters, non-zero in intermediate cluster ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Backing file: 64k: -- -- -- -- | -- XX XX -- | -- -- -- --
+# Active layer: 64k: -- -- 00 00 | 00 00 00 00 | 00 -- -- --
+
+$QEMU_IO -c "write -P 0x11 69k 2k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 66k 7k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 64k 12k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning multiple clusters, non-zero in final cluster ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Backing file: 64k: -- -- -- -- | -- -- -- -- | -- -- XX XX
+# Active layer: 64k: -- -- 00 00 | 00 00 00 00 | 00 -- XX XX
+
+$QEMU_IO -c "write -P 0x11 74k 2k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 66k 7k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 64k 10k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 74k 2k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+echo
+echo == spanning multiple clusters, partially overwriting backing file ==
+
+CLUSTER_SIZE=512 TEST_IMG="$TEST_IMG.base" _make_test_img $size
+_make_test_img -b "$TEST_IMG.base"
+
+# Backing file: 64k: -- XX XX XX | XX XX XX XX | XX XX XX --
+# Active layer: 64k: -- XX 00 00 | 00 00 00 00 | 00 XX XX --
+
+$QEMU_IO -c "write -P 0x11 65k 10k" "$TEST_IMG.base" | _filter_qemu_io
+$QEMU_IO -c "write -z 66k 7k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 64k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 65k 1k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 66k 7k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0x11 73k 2k" "$TEST_IMG" | _filter_qemu_io
+$QEMU_IO -c "read -P 0 75k 1k" "$TEST_IMG" | _filter_qemu_io
+
+$QEMU_IMG map --output=json "$TEST_IMG" | _filter_qemu_img_map
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/154.out b/tests/qemu-iotests/154.out
new file mode 100644
index 0000000..da9eabd
--- /dev/null
+++ b/tests/qemu-iotests/154.out
@@ -0,0 +1,285 @@
+QA output created by 154
+
+== backing file contains zeros ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 2048/2048 bytes at offset 0
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 10240
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 17408
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 27648
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 4096, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 8192, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 12288, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 16384, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 20480, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 24576, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 32768, "length": 134184960, "depth": 1, "zero": true, "data": false}]
+
+== backing file contains non-zero data before write_zeroes ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 1024/1024 bytes at offset 32768
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 34816
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 32768
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 33792
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 66560
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 67584
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 66560
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 65536
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 67584
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 32768, "depth": 1, "zero": true, "data": false},
+{ "start": 32768, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 36864, "length": 28672, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 24576},
+{ "start": 69632, "length": 134148096, "depth": 1, "zero": true, "data": false}]
+
+== backing file contains non-zero data after write_zeroes ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 1024/1024 bytes at offset 34816
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 33792
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 32768
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 34816
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 35840
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 44032
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 41984
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 44032
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 3072/3072 bytes at offset 40960
+3 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 32768, "depth": 1, "zero": true, "data": false},
+{ "start": 32768, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 36864, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 40960, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 24576},
+{ "start": 45056, "length": 134172672, "depth": 1, "zero": true, "data": false}]
+
+== write_zeroes covers non-zero data ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 1024/1024 bytes at offset 5120
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 5120
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 4096
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 14336
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 13312
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 12288
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 21504
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 21504
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 20480
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 30208
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 29696
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 4096/4096 bytes at offset 28672
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 4096, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 8192, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 12288, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 16384, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 20480, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 24576, "length": 4096, "depth": 1, "zero": true, "data": false},
+{ "start": 28672, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 32768, "length": 134184960, "depth": 1, "zero": true, "data": false}]
+
+== spanning two clusters, non-zero before request ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 1024/1024 bytes at offset 32768
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 35840
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 32768
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 7168/7168 bytes at offset 33792
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 50176
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 52224
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 49152
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 50176
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 6144/6144 bytes at offset 51200
+6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 67584
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 68608
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 65536
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 67584
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 5120/5120 bytes at offset 68608
+5 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 32768, "depth": 1, "zero": true, "data": false},
+{ "start": 32768, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 36864, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 40960, "length": 8192, "depth": 1, "zero": true, "data": false},
+{ "start": 49152, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 24576},
+{ "start": 53248, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 57344, "length": 8192, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 28672},
+{ "start": 69632, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 73728, "length": 134144000, "depth": 1, "zero": true, "data": false}]
+
+== spanning two clusters, non-zero after request ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 1024/1024 bytes at offset 37888
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 35840
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 5120/5120 bytes at offset 32768
+5 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 37888
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 38912
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 55296
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 52224
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 6144/6144 bytes at offset 49152
+6 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 55296
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 56320
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 1024/1024 bytes at offset 72704
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 68608
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 7168/7168 bytes at offset 65536
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 72704
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 32768, "depth": 1, "zero": true, "data": false},
+{ "start": 32768, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 36864, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 40960, "length": 8192, "depth": 1, "zero": true, "data": false},
+{ "start": 49152, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 53248, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 24576},
+{ "start": 57344, "length": 8192, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 69632, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 28672},
+{ "start": 73728, "length": 134144000, "depth": 1, "zero": true, "data": false}]
+
+== spanning two clusters, partially overwriting backing file ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 4096/4096 bytes at offset 2048
+4 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 2048/2048 bytes at offset 3072
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 0
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 2048
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 3072
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 5120
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 6144
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 8192, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 8192, "length": 134209536, "depth": 1, "zero": true, "data": false}]
+
+== spanning multiple clusters, non-zero in first cluster ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 2048/2048 bytes at offset 65536
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 7168/7168 bytes at offset 67584
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 65536
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 10240/10240 bytes at offset 67584
+10 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 65536, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 69632, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 77824, "length": 134139904, "depth": 1, "zero": true, "data": false}]
+
+== spanning multiple clusters, non-zero in intermediate cluster ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 2048/2048 bytes at offset 70656
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 7168/7168 bytes at offset 67584
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 12288/12288 bytes at offset 65536
+12 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 65536, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 12288, "depth": 0, "zero": true, "data": false},
+{ "start": 77824, "length": 134139904, "depth": 1, "zero": true, "data": false}]
+
+== spanning multiple clusters, non-zero in final cluster ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 2048/2048 bytes at offset 75776
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 7168/7168 bytes at offset 67584
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 10240/10240 bytes at offset 65536
+10 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 75776
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 65536, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 8192, "depth": 0, "zero": true, "data": false},
+{ "start": 73728, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 77824, "length": 134139904, "depth": 1, "zero": true, "data": false}]
+
+== spanning multiple clusters, partially overwriting backing file ==
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=134217728
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file=TEST_DIR/t.IMGFMT.base
+wrote 10240/10240 bytes at offset 66560
+10 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 7168/7168 bytes at offset 67584
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 65536
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 66560
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 7168/7168 bytes at offset 67584
+7 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 2048/2048 bytes at offset 74752
+2 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 1024/1024 bytes at offset 76800
+1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+[{ "start": 0, "length": 65536, "depth": 1, "zero": true, "data": false},
+{ "start": 65536, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 20480},
+{ "start": 69632, "length": 4096, "depth": 0, "zero": true, "data": false},
+{ "start": 73728, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": 24576},
+{ "start": 77824, "length": 134139904, "depth": 1, "zero": true, "data": false}]
+*** done
diff --git a/tests/qemu-iotests/155 b/tests/qemu-iotests/155
new file mode 100755
index 0000000..4057b5e
--- /dev/null
+++ b/tests/qemu-iotests/155
@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+#
+# Test whether the backing BDSs are correct after completion of a
+# mirror block job; in "existing" modes (drive-mirror with
+# mode=existing and blockdev-mirror) the backing chain should not be
+# overridden.
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import iotests
+from iotests import qemu_img
+
+back0_img = os.path.join(iotests.test_dir, 'back0.' + iotests.imgfmt)
+back1_img = os.path.join(iotests.test_dir, 'back1.' + iotests.imgfmt)
+back2_img = os.path.join(iotests.test_dir, 'back2.' + iotests.imgfmt)
+source_img = os.path.join(iotests.test_dir, 'source.' + iotests.imgfmt)
+target_img = os.path.join(iotests.test_dir, 'target.' + iotests.imgfmt)
+
+
+# Class variables for controlling its behavior:
+#
+# existing: If True, explicitly create the target image and blockdev-add it
+# target_backing: If existing is True: Use this filename as the backing file
+#                 of the target image
+#                 (None: no backing file)
+# target_blockdev_backing: If existing is True: Pass this dict as "backing"
+#                          for the blockdev-add command
+#                          (None: do not pass "backing")
+# target_real_backing: If existing is True: The real filename of the backing
+#                      image during runtime, only makes sense if
+#                      target_blockdev_backing is not None
+#                      (None: same as target_backing)
+
+class BaseClass(iotests.QMPTestCase):
+    target_blockdev_backing = None
+    target_real_backing = None
+
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, back0_img, '1M')
+        qemu_img('create', '-f', iotests.imgfmt, '-b', back0_img, back1_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-b', back1_img, back2_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-b', back2_img, source_img)
+
+        self.vm = iotests.VM()
+        self.vm.add_drive(None, '', 'none')
+        self.vm.launch()
+
+        # Add the BDS via blockdev-add so it stays around after the mirror block
+        # job has been completed
+        result = self.vm.qmp('blockdev-add',
+                             options={'node-name': 'source',
+                                      'driver': iotests.imgfmt,
+                                      'file': {'driver': 'file',
+                                               'filename': source_img}})
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('x-blockdev-insert-medium',
+                             device='drive0', node_name='source')
+        self.assert_qmp(result, 'return', {})
+
+        self.assertIntactSourceBackingChain()
+
+        if self.existing:
+            if self.target_backing:
+                qemu_img('create', '-f', iotests.imgfmt,
+                         '-b', self.target_backing, target_img, '1M')
+            else:
+                qemu_img('create', '-f', iotests.imgfmt, target_img, '1M')
+
+            if self.cmd == 'blockdev-mirror':
+                options = { 'node-name': 'target',
+                            'driver': iotests.imgfmt,
+                            'file': { 'driver': 'file',
+                                      'filename': target_img } }
+                if self.target_blockdev_backing:
+                    options['backing'] = self.target_blockdev_backing
+
+                result = self.vm.qmp('blockdev-add', options=options)
+                self.assert_qmp(result, 'return', {})
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(source_img)
+        os.remove(back2_img)
+        os.remove(back1_img)
+        os.remove(back0_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def findBlockNode(self, node_name, id=None):
+        if id:
+            result = self.vm.qmp('query-block')
+            for device in result['return']:
+                if device['device'] == id:
+                    if node_name:
+                        self.assert_qmp(device, 'inserted/node-name', node_name)
+                    return device['inserted']
+        else:
+            result = self.vm.qmp('query-named-block-nodes')
+            for node in result['return']:
+                if node['node-name'] == node_name:
+                    return node
+
+        self.fail('Cannot find node %s/%s' % (id, node_name))
+
+    def assertIntactSourceBackingChain(self):
+        node = self.findBlockNode('source')
+
+        self.assert_qmp(node, 'image' + '/backing-image' * 0 + '/filename',
+                        source_img)
+        self.assert_qmp(node, 'image' + '/backing-image' * 1 + '/filename',
+                        back2_img)
+        self.assert_qmp(node, 'image' + '/backing-image' * 2 + '/filename',
+                        back1_img)
+        self.assert_qmp(node, 'image' + '/backing-image' * 3 + '/filename',
+                        back0_img)
+        self.assert_qmp_absent(node, 'image' + '/backing-image' * 4)
+
+    def assertCorrectBackingImage(self, node, default_image):
+        if self.existing:
+            if self.target_real_backing:
+                image = self.target_real_backing
+            else:
+                image = self.target_backing
+        else:
+            image = default_image
+
+        if image:
+            self.assert_qmp(node, 'image/backing-image/filename', image)
+        else:
+            self.assert_qmp_absent(node, 'image/backing-image')
+
+
+# Class variables for controlling its behavior:
+#
+# cmd: Mirroring command to execute, either drive-mirror or blockdev-mirror
+
+class MirrorBaseClass(BaseClass):
+    def runMirror(self, sync):
+        if self.cmd == 'blockdev-mirror':
+            result = self.vm.qmp(self.cmd, device='drive0', sync=sync,
+                                 target='target')
+        else:
+            if self.existing:
+                mode = 'existing'
+            else:
+                mode = 'absolute-paths'
+            result = self.vm.qmp(self.cmd, device='drive0', sync=sync,
+                                 target=target_img, format=iotests.imgfmt,
+                                 mode=mode, node_name='target')
+
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait('BLOCK_JOB_READY')
+
+        result = self.vm.qmp('block-job-complete', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait('BLOCK_JOB_COMPLETED')
+
+    def testFull(self):
+        self.runMirror('full')
+
+        node = self.findBlockNode('target', 'drive0')
+        self.assertCorrectBackingImage(node, None)
+        self.assertIntactSourceBackingChain()
+
+    def testTop(self):
+        self.runMirror('top')
+
+        node = self.findBlockNode('target', 'drive0')
+        self.assertCorrectBackingImage(node, back2_img)
+        self.assertIntactSourceBackingChain()
+
+    def testNone(self):
+        self.runMirror('none')
+
+        node = self.findBlockNode('target', 'drive0')
+        self.assertCorrectBackingImage(node, source_img)
+        self.assertIntactSourceBackingChain()
+
+
+class TestDriveMirrorAbsolutePaths(MirrorBaseClass):
+    cmd = 'drive-mirror'
+    existing = False
+
+class TestDriveMirrorExistingNoBacking(MirrorBaseClass):
+    cmd = 'drive-mirror'
+    existing = True
+    target_backing = None
+
+class TestDriveMirrorExistingBacking(MirrorBaseClass):
+    cmd = 'drive-mirror'
+    existing = True
+    target_backing = 'null-co://'
+
+class TestBlockdevMirrorNoBacking(MirrorBaseClass):
+    cmd = 'blockdev-mirror'
+    existing = True
+    target_backing = None
+
+class TestBlockdevMirrorBacking(MirrorBaseClass):
+    cmd = 'blockdev-mirror'
+    existing = True
+    target_backing = 'null-co://'
+
+class TestBlockdevMirrorForcedBacking(MirrorBaseClass):
+    cmd = 'blockdev-mirror'
+    existing = True
+    target_backing = None
+    target_blockdev_backing = { 'driver': 'null-co' }
+    target_real_backing = 'null-co://'
+
+
+class TestCommit(BaseClass):
+    existing = False
+
+    def testCommit(self):
+        result = self.vm.qmp('block-commit', device='drive0', base=back1_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait('BLOCK_JOB_READY')
+
+        result = self.vm.qmp('block-job-complete', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.vm.event_wait('BLOCK_JOB_COMPLETED')
+
+        node = self.findBlockNode(None, 'drive0')
+        self.assert_qmp(node, 'image' + '/backing-image' * 0 + '/filename',
+                        back1_img)
+        self.assert_qmp(node, 'image' + '/backing-image' * 1 + '/filename',
+                        back0_img)
+        self.assert_qmp_absent(node, 'image' + '/backing-image' * 2 +
+                               '/filename')
+
+        self.assertIntactSourceBackingChain()
+
+
+BaseClass = None
+MirrorBaseClass = None
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/155.out b/tests/qemu-iotests/155.out
new file mode 100644
index 0000000..4176bb9
--- /dev/null
+++ b/tests/qemu-iotests/155.out
@@ -0,0 +1,5 @@
+...................
+----------------------------------------------------------------------
+Ran 19 tests
+
+OK
diff --git a/tests/qemu-iotests/156 b/tests/qemu-iotests/156
new file mode 100755
index 0000000..cc95ff1
--- /dev/null
+++ b/tests/qemu-iotests/156
@@ -0,0 +1,174 @@
+#!/bin/bash
+#
+# Tests oVirt-like storage migration:
+#  - Create snapshot
+#  - Create target image with (not yet existing) target backing chain
+#    (i.e. just write the name of a soon-to-be-copied-over backing file into it)
+#  - drive-mirror the snapshot to the target with mode=existing and sync=top
+#  - In the meantime, copy the original source files to the destination via
+#    conventional means (i.e. outside of qemu)
+#  - Complete the drive-mirror job
+#  - Delete all source images
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+    rm -f "$TEST_IMG{,.target}{,.backing,.overlay}"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2 qed
+_supported_proto generic
+_supported_os Linux
+
+# Create source disk
+TEST_IMG="$TEST_IMG.backing" _make_test_img 1M
+_make_test_img -b "$TEST_IMG.backing" 1M
+
+$QEMU_IO -c 'write -P 1 0 256k' "$TEST_IMG.backing" | _filter_qemu_io
+$QEMU_IO -c 'write -P 2 64k 192k' "$TEST_IMG" | _filter_qemu_io
+
+_launch_qemu -drive if=none,id=source,file="$TEST_IMG"
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+# Create snapshot
+TEST_IMG="$TEST_IMG.overlay" _make_test_img -b "$TEST_IMG" 1M
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'blockdev-snapshot-sync',
+       'arguments': { 'device': 'source',
+                      'snapshot-file': '$TEST_IMG.overlay',
+                      'format': '$IMGFMT',
+                      'mode': 'existing' } }" \
+    'return'
+
+# Write something to the snapshot
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line':
+                      'qemu-io source \"write -P 3 128k 128k\"' } }" \
+    'return'
+
+# Create target image
+TEST_IMG="$TEST_IMG.target.overlay" _make_test_img -b "$TEST_IMG.target" 1M
+
+# Mirror snapshot
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'drive-mirror',
+       'arguments': { 'device': 'source',
+                      'target': '$TEST_IMG.target.overlay',
+                      'mode': 'existing',
+                      'sync': 'top' } }" \
+    'return'
+
+# Wait for convergence
+_send_qemu_cmd $QEMU_HANDLE \
+    '' \
+    'BLOCK_JOB_READY'
+
+# Write some more
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line':
+                      'qemu-io source \"write -P 4 192k 64k\"' } }" \
+    'return'
+
+# Copy source backing chain to the target before completing the job
+cp "$TEST_IMG.backing" "$TEST_IMG.target.backing"
+cp "$TEST_IMG" "$TEST_IMG.target"
+$QEMU_IMG rebase -u -b "$TEST_IMG.target.backing" "$TEST_IMG.target"
+
+# Complete block job
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'block-job-complete',
+       'arguments': { 'device': 'source' } }" \
+    ''
+
+_send_qemu_cmd $QEMU_HANDLE \
+    '' \
+    'BLOCK_JOB_COMPLETED'
+
+# Remove the source images
+rm -f "$TEST_IMG{,.backing,.overlay}"
+
+echo
+
+# Check online disk contents
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line':
+                      'qemu-io source \"read -P 1 0k 64k\"' } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line':
+                      'qemu-io source \"read -P 2 64k 64k\"' } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line':
+                      'qemu-io source \"read -P 3 128k 64k\"' } }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'human-monitor-command',
+       'arguments': { 'command-line':
+                      'qemu-io source \"read -P 4 192k 64k\"' } }" \
+    'return'
+
+echo
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=1 _cleanup_qemu
+
+echo
+
+# Check offline disk contents
+$QEMU_IO -c 'read -P 1 0k 64k' \
+         -c 'read -P 2 64k 64k' \
+         -c 'read -P 3 128k 64k' \
+         -c 'read -P 4 192k 64k' \
+         "$TEST_IMG.target.overlay" | _filter_qemu_io
+
+echo
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/156.out b/tests/qemu-iotests/156.out
new file mode 100644
index 0000000..3af82ae
--- /dev/null
+++ b/tests/qemu-iotests/156.out
@@ -0,0 +1,48 @@
+QA output created by 156
+Formatting 'TEST_DIR/t.IMGFMT.backing', fmt=IMGFMT size=1048576
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT.backing
+wrote 262144/262144 bytes at offset 0
+256 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 196608/196608 bytes at offset 65536
+192 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+Formatting 'TEST_DIR/t.IMGFMT.overlay', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT
+{"return": {}}
+wrote 131072/131072 bytes at offset 131072
+128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+Formatting 'TEST_DIR/t.IMGFMT.target.overlay', fmt=IMGFMT size=1048576 backing_file=TEST_DIR/t.IMGFMT.target
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "source", "len": 131072, "offset": 131072, "speed": 0, "type": "mirror"}}
+wrote 65536/65536 bytes at offset 196608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "source", "len": 196608, "offset": 196608, "speed": 0, "type": "mirror"}}
+
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+read 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+read 65536/65536 bytes at offset 196608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": ""}
+
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
+
+read 65536/65536 bytes at offset 0
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 65536
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 131072
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 65536/65536 bytes at offset 196608
+64 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+*** done
diff --git a/tests/qemu-iotests/157 b/tests/qemu-iotests/157
new file mode 100755
index 0000000..8d939cb
--- /dev/null
+++ b/tests/qemu-iotests/157
@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# Test command line configuration of block devices with qdev
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_proto file
+_supported_os Linux
+
+function do_run_qemu()
+{
+    echo Testing: "$@"
+    (
+        if ! test -t 0; then
+            while read cmd; do
+                echo $cmd
+            done
+        fi
+        echo quit
+    ) | $QEMU -nodefaults -nographic -monitor stdio -serial none "$@"
+    echo
+}
+
+function run_qemu()
+{
+    do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_imgfmt \
+                          | _filter_qemu | _filter_generated_node_ids
+}
+
+
+size=128M
+drive="if=none,file=$TEST_IMG,driver=$IMGFMT"
+
+_make_test_img $size
+
+echo
+echo "=== Setting WCE with qdev and with manually created BB ==="
+echo
+
+# The qdev option takes precedence, but if it isn't given or 'auto', the BB
+# option is used instead.
+
+for cache in "writeback" "writethrough"; do
+    for wce in "" ",write-cache=auto" ",write-cache=on" ",write-cache=off"; do
+        echo "info block" \
+            | run_qemu -drive "$drive,cache=$cache" \
+                       -device "virtio-blk,drive=none0$wce" \
+            | grep -e "Testing" -e "Cache mode"
+    done
+done
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/157.out b/tests/qemu-iotests/157.out
new file mode 100644
index 0000000..77a9c03
--- /dev/null
+++ b/tests/qemu-iotests/157.out
@@ -0,0 +1,22 @@
+QA output created by 157
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728
+
+=== Setting WCE with qdev and with manually created BB ===
+
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writeback -device virtio-blk,drive=none0
+    Cache mode:       writeback
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writeback -device virtio-blk,drive=none0,write-cache=auto
+    Cache mode:       writeback
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writeback -device virtio-blk,drive=none0,write-cache=on
+    Cache mode:       writeback
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writeback -device virtio-blk,drive=none0,write-cache=off
+    Cache mode:       writethrough
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writethrough -device virtio-blk,drive=none0
+    Cache mode:       writethrough
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writethrough -device virtio-blk,drive=none0,write-cache=auto
+    Cache mode:       writethrough
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writethrough -device virtio-blk,drive=none0,write-cache=on
+    Cache mode:       writeback
+Testing: -drive if=none,file=TEST_DIR/t.IMGFMT,driver=IMGFMT,cache=writethrough -device virtio-blk,drive=none0,write-cache=off
+    Cache mode:       writethrough
+*** done
diff --git a/tests/qemu-iotests/162 b/tests/qemu-iotests/162
new file mode 100755
index 0000000..0b43ea3
--- /dev/null
+++ b/tests/qemu-iotests/162
@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# Test case for specifying runtime options of the wrong type to some
+# block drivers
+#
+# Copyright (C) 2016 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+status=1	# failure is the default!
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt generic
+_supported_os Linux
+
+echo
+echo '=== NBD ==='
+# NBD expects all of its arguments to be strings
+
+# So this should not crash
+$QEMU_IMG info 'json:{"driver": "nbd", "host": 42}'
+
+# And this should not treat @port as if it had not been specified
+# (We cannot use localhost with an invalid port here, but we need to use a
+#  non-existing domain, because otherwise the error message will not contain
+#  the port)
+$QEMU_IMG info 'json:{"driver": "nbd", "host": "does.not.exist.example.com", "port": 42}'
+
+# This is a test for NBD's bdrv_refresh_filename() implementation: It expects
+# either host or path to be set, but it must not assume that they are set to
+# strings in the options QDict
+$QEMU_NBD -k "$PWD/42" -f raw null-co:// &
+sleep 0.5
+$QEMU_IMG info 'json:{"driver": "nbd", "path": 42}' | grep '^image'
+rm -f 42
+
+
+echo
+echo '=== SSH ==='
+# SSH expects all of its arguments to be strings, except for @port, which is
+# expected to be an integer
+
+# So "0" should be converted to an integer here (instead of crashing)
+$QEMU_IMG info 'json:{"driver": "ssh", "host": "localhost", "port": "0", "path": "/foo"}'
+# The same, basically (all values for --image-opts are seen as strings in qemu)
+$QEMU_IMG info --image-opts \
+    driver=ssh,host=localhost,port=0,path=/foo
+
+# This, however, should fail because of the wrong type
+$QEMU_IMG info 'json:{"driver": "ssh", "host": "localhost", "port": 0.42, "path": "/foo"}'
+# Not really the same: Here, "0.42" will be passed instead of 0.42, but still,
+# qemu should not try to convert "0.42" to an integer
+$QEMU_IMG info --image-opts \
+    driver=ssh,host=localhost,port=0.42,path=/foo
+
+
+echo
+echo '=== blkdebug ==='
+# blkdebug expects all of its arguments to be strings, but its
+# bdrv_refresh_filename() implementation should not assume that they have been
+# passed as strings in the original options QDict.
+# So this should emit blkdebug:42:null-co:// as the filename:
+touch 42
+$QEMU_IMG info 'json:{"driver": "blkdebug", "config": 42,
+                      "image.driver": "null-co"}' \
+    | grep '^image'
+rm -f 42
+
+
+# success, all done
+echo
+echo '*** done'
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/162.out b/tests/qemu-iotests/162.out
new file mode 100644
index 0000000..9bba723
--- /dev/null
+++ b/tests/qemu-iotests/162.out
@@ -0,0 +1,17 @@
+QA output created by 162
+
+=== NBD ===
+qemu-img: Could not open 'json:{"driver": "nbd", "host": 42}': Failed to connect socket: Invalid argument
+qemu-img: Could not open 'json:{"driver": "nbd", "host": "does.not.exist.example.com", "port": 42}': address resolution failed for does.not.exist.example.com:42: Name or service not known
+image: nbd+unix://?socket=42
+
+=== SSH ===
+qemu-img: Could not open 'json:{"driver": "ssh", "host": "localhost", "port": "0", "path": "/foo"}': Failed to connect socket: Connection refused
+qemu-img: Could not open 'driver=ssh,host=localhost,port=0,path=/foo': Failed to connect socket: Connection refused
+qemu-img: Could not open 'json:{"driver": "ssh", "host": "localhost", "port": 0.42, "path": "/foo"}': Parameter 'port' expects a number
+qemu-img: Could not open 'driver=ssh,host=localhost,port=0.42,path=/foo': Parameter 'port' expects a number
+
+=== blkdebug ===
+image: blkdebug:42:null-co://
+
+*** done
diff --git a/tests/qemu-iotests/README b/tests/qemu-iotests/README
index 4ccfdd1..6079b40 100644
--- a/tests/qemu-iotests/README
+++ b/tests/qemu-iotests/README
@@ -17,4 +17,5 @@
 * Feedback and patches
 
 Please send improvements to the test suite, general feedback or just
-reports of failing tests cases to qemu-devel@savannah.nongnu.org.
+reports of failing tests cases to qemu-devel@nongnu.org with a CC:
+to qemu-block@nongnu.org.
diff --git a/tests/qemu-iotests/check b/tests/qemu-iotests/check
index 8ca4011..4cba215 100755
--- a/tests/qemu-iotests/check
+++ b/tests/qemu-iotests/check
@@ -19,7 +19,6 @@
 # Control script for QA
 #
 
-tmp=/tmp/$$
 status=0
 needwrap=true
 try=0
@@ -130,6 +129,8 @@
 #    exit 1
 #fi
 
+tmp="${TEST_DIR}"/$$
+
 _wallclock()
 {
     date "+%H %M %S" | $AWK_PROG '{ print $1*3600 + $2*60 + $3 }'
@@ -146,8 +147,8 @@
     # for hangcheck ...
     # remove files that were used by hangcheck
     #
-    [ -f /tmp/check.pid ] && rm -rf /tmp/check.pid
-    [ -f /tmp/check.sts ] && rm -rf /tmp/check.sts
+    [ -f "${TEST_DIR}"/check.pid ] && rm -rf "${TEST_DIR}"/check.pid
+    [ -f "${TEST_DIR}"/check.sts ] && rm -rf "${TEST_DIR}"/check.sts
 
     if $showme
     then
@@ -197,8 +198,8 @@
         needwrap=false
     fi
 
-    rm -f /tmp/*.out /tmp/*.err /tmp/*.time
-    rm -f /tmp/check.pid /tmp/check.sts
+    rm -f "${TEST_DIR}"/*.out "${TEST_DIR}"/*.err "${TEST_DIR}"/*.time
+    rm -f "${TEST_DIR}"/check.pid "${TEST_DIR}"/check.sts
     rm -f $tmp.*
 }
 
@@ -208,16 +209,16 @@
 # Save pid of check in a well known place, so that hangcheck can be sure it
 # has the right pid (getting the pid from ps output is not reliable enough).
 #
-rm -rf /tmp/check.pid
-echo $$ >/tmp/check.pid
+rm -rf "${TEST_DIR}"/check.pid
+echo $$ > "${TEST_DIR}"/check.pid
 
 # for hangcheck ...
 # Save the status of check in a well known place, so that hangcheck can be
 # sure to know where check is up to (getting test number from ps output is
 # not reliable enough since the trace stuff has been introduced).
 #
-rm -rf /tmp/check.sts
-echo "preamble" >/tmp/check.sts
+rm -rf "${TEST_DIR}"/check.sts
+echo "preamble" > "${TEST_DIR}"/check.sts
 
 # don't leave old full output behind on a clean run
 rm -f check.full
@@ -231,13 +232,14 @@
 #FULL_MOUNT_OPTIONS=`_scratch_mount_options`
 
 cat <<EOF
-QEMU          -- $QEMU
-QEMU_IMG      -- $QEMU_IMG
-QEMU_IO       -- $QEMU_IO
-QEMU_NBD      -- $QEMU_NBD
+QEMU          -- "$QEMU_PROG" $QEMU_OPTIONS
+QEMU_IMG      -- "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS
+QEMU_IO       -- "$QEMU_IO_PROG" $QEMU_IO_OPTIONS
+QEMU_NBD      -- "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS
 IMGFMT        -- $FULL_IMGFMT_DETAILS
 IMGPROTO      -- $FULL_IMGPROTO_DETAILS
 PLATFORM      -- $FULL_HOST_DETAILS
+TEST_DIR      -- $TEST_DIR
 SOCKET_SCM_HELPER -- $SOCKET_SCM_HELPER
 
 EOF
@@ -284,7 +286,7 @@
         rm -f core $seq.notrun
 
         # for hangcheck ...
-        echo "$seq" >/tmp/check.sts
+        echo "$seq" > "${TEST_DIR}"/check.sts
 
         start=`_wallclock`
         $timestamp && echo -n "        ["`date "+%T"`"]"
@@ -295,9 +297,15 @@
             run_command="./$seq"
         fi
         export OUTPUT_DIR=$PWD
-        (cd "$source_iotests";
-        MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
-                $run_command >$tmp.out 2>&1)
+        if $debug; then
+            (cd "$source_iotests";
+            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
+                    $run_command -d 2>&1 | tee $tmp.out)
+        else
+            (cd "$source_iotests";
+            MALLOC_PERTURB_=${MALLOC_PERTURB_:-$(($RANDOM % 255 + 1))} \
+                    $run_command >$tmp.out 2>&1)
+        fi
         sts=$?
         $timestamp && _timestamp
         stop=`_wallclock`
@@ -323,6 +331,11 @@
             fi
 
             reference="$source_iotests/$seq.out"
+            reference_machine="$source_iotests/$seq.$QEMU_DEFAULT_MACHINE.out"
+            if [ -f "$reference_machine" ]; then
+                reference="$reference_machine"
+            fi
+
             if [ "$CACHEMODE" = "none" ]; then
                 [ -f "$source_iotests/$seq.out.nocache" ] && reference="$source_iotests/$seq.out.nocache"
             fi
diff --git a/tests/qemu-iotests/common b/tests/qemu-iotests/common
index 9e12bec..d60ea2c 100644
--- a/tests/qemu-iotests/common
+++ b/tests/qemu-iotests/common
@@ -32,6 +32,7 @@
 
 diff="diff -u"
 verbose=false
+debug=false
 group=false
 xgroup=false
 imgopts=false
@@ -40,7 +41,6 @@
 expunge=true
 have_test_arg=false
 randomize=false
-valgrind=false
 cachemode=false
 rm -f $tmp.list $tmp.tmp $tmp.sed
 
@@ -51,6 +51,10 @@
 export CACHEMODE="writeback"
 export QEMU_IO_OPTIONS=""
 export CACHEMODE_IS_DEFAULT=true
+export QEMU_OPTIONS="-nodefaults"
+export VALGRIND_QEMU=
+export IMGKEYSECRET=
+export IMGOPTSSYNTAX=false
 
 for r
 do
@@ -132,6 +136,7 @@
 
 common options
     -v                  verbose
+    -d                  debug
 
 check options
     -raw                test raw (default)
@@ -152,6 +157,7 @@
     -ssh                test ssh
     -nfs                test nfs
     -archipelago        test archipelago
+    -luks               test luks
     -xdiff              graphical mode diff
     -nocache            use O_DIRECT on backing file
     -misalign           misalign memory allocations
@@ -203,6 +209,13 @@
             xpand=false
             ;;
 
+        -luks)
+            IMGOPTSSYNTAX=true
+            IMGFMT=luks
+            IMGKEYSECRET=123456
+            xpand=false
+            ;;
+
         -qed)
             IMGFMT=qed
             xpand=false
@@ -275,7 +288,7 @@
             ;;
 
         -valgrind)
-            valgrind=true
+            VALGRIND_QEMU='y'
             xpand=false
             ;;
 
@@ -289,10 +302,10 @@
 
             if [ ! -z "$DISPLAY" ]
             then
-                which xdiff >/dev/null 2>&1 && diff=xdiff
-                which gdiff >/dev/null 2>&1 && diff=gdiff
-                which tkdiff >/dev/null 2>&1 && diff=tkdiff
-                which xxdiff >/dev/null 2>&1 && diff=xxdiff
+                command -v xdiff >/dev/null 2>&1 && diff=xdiff
+                command -v gdiff >/dev/null 2>&1 && diff=gdiff
+                command -v tkdiff >/dev/null 2>&1 && diff=tkdiff
+                command -v xxdiff >/dev/null 2>&1 && diff=xxdiff
             fi
             ;;
 
@@ -322,6 +335,10 @@
             verbose=true
             xpand=false
             ;;
+        -d)
+            debug=true
+            xpand=false
+            ;;
         -x)        # -x group ... exclude from group file
             xgroup=true
             xpand=false
@@ -391,7 +408,11 @@
 done
 
 # Set qemu-io cache mode with $CACHEMODE we have
-QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"
+if [ "$IMGOPTSSYNTAX" = "true" ]; then
+    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --cache $CACHEMODE"
+else
+    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS -f $IMGFMT --cache $CACHEMODE"
+fi
 
 # Set default options for qemu-img create -o if they were not specified
 _set_default_imgopts
@@ -429,8 +450,3 @@
 if [ "$IMGPROTO" = "nbd" ] ; then
     [ "$QEMU_NBD" = "" ] && _fatal "qemu-nbd not found"
 fi
-
-if $valgrind; then
-    export REAL_QEMU_IO="$QEMU_IO_PROG"
-    export QEMU_IO_PROG=valgrind_qemu_io
-fi
diff --git a/tests/qemu-iotests/common.config b/tests/qemu-iotests/common.config
index bd6790b..f6384fb 100644
--- a/tests/qemu-iotests/common.config
+++ b/tests/qemu-iotests/common.config
@@ -44,10 +44,12 @@
 export CHECK_OPTIONS=${CHECK_OPTIONS:="-g auto"}
 export PWD=`pwd`
 
+export _QEMU_HANDLE=0
+
 # $1 = prog to look for, $2* = default pathnames if not found in $PATH
 set_prog_path()
 {
-    p=`which $1 2> /dev/null`
+    p=`command -v $1 2> /dev/null`
     if [ -n "$p" -a -x "$p" ]; then
         echo $p
         return 0
@@ -103,10 +105,80 @@
     export QEMU_NBD_PROG="`set_prog_path qemu-nbd`"
 fi
 
-export QEMU=$QEMU_PROG
-export QEMU_IMG=$QEMU_IMG_PROG
-export QEMU_IO="$QEMU_IO_PROG $QEMU_IO_OPTIONS"
-export QEMU_NBD=$QEMU_NBD_PROG
+_qemu_wrapper()
+{
+    (
+        if [ -n "${QEMU_NEED_PID}" ]; then
+            echo $BASHPID > "${TEST_DIR}/qemu-${_QEMU_HANDLE}.pid"
+        fi
+        exec "$QEMU_PROG" $QEMU_OPTIONS "$@"
+    )
+}
+
+_qemu_img_wrapper()
+{
+    (exec "$QEMU_IMG_PROG" $QEMU_IMG_OPTIONS "$@")
+}
+
+_qemu_io_wrapper()
+{
+    local VALGRIND_LOGFILE="${TEST_DIR}"/$$.valgrind
+    local QEMU_IO_ARGS="$QEMU_IO_OPTIONS"
+    if [ "$IMGOPTSSYNTAX" = "true" ]; then
+        QEMU_IO_ARGS="--image-opts $QEMU_IO_ARGS"
+        if [ -n "$IMGKEYSECRET" ]; then
+            QEMU_IO_ARGS="--object secret,id=keysec0,data=$IMGKEYSECRET $QEMU_IO_ARGS"
+        fi
+    fi
+    local RETVAL
+    (
+        if [ "${VALGRIND_QEMU}" == "y" ]; then
+            exec valgrind --log-file="${VALGRIND_LOGFILE}" --error-exitcode=99 "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@"
+        else
+            exec "$QEMU_IO_PROG" $QEMU_IO_ARGS "$@"
+        fi
+    )
+    RETVAL=$?
+    if [ "${VALGRIND_QEMU}" == "y" ]; then
+        if [ $RETVAL == 99 ]; then
+            cat "${VALGRIND_LOGFILE}"
+        fi
+        rm -f "${VALGRIND_LOGFILE}"
+    fi
+    (exit $RETVAL)
+}
+
+_qemu_nbd_wrapper()
+{
+    (
+        echo $BASHPID > "${TEST_DIR}/qemu-nbd.pid"
+        exec "$QEMU_NBD_PROG" $QEMU_NBD_OPTIONS "$@"
+    )
+}
+
+export QEMU=_qemu_wrapper
+export QEMU_IMG=_qemu_img_wrapper
+export QEMU_IO=_qemu_io_wrapper
+export QEMU_NBD=_qemu_nbd_wrapper
+
+QEMU_IMG_EXTRA_ARGS=
+if [ "$IMGOPTSSYNTAX" = "true" ]; then
+    QEMU_IMG_EXTRA_ARGS="--image-opts $QEMU_IMG_EXTRA_ARGS"
+    if [ -n "$IMGKEYSECRET" ]; then
+        QEMU_IMG_EXTRA_ARGS="--object secret,id=keysec0,data=$IMGKEYSECRET $QEMU_IMG_EXTRA_ARGS"
+    fi
+fi
+export QEMU_IMG_EXTRA_ARGS
+
+
+default_machine=$($QEMU -machine help | sed -n '/(default)/ s/ .*//p')
+default_alias_machine=$($QEMU -machine help | \
+   sed -n "/(alias of $default_machine)/ { s/ .*//p; q; }")
+if [[ "$default_alias_machine" ]]; then
+    default_machine="$default_alias_machine"
+fi
+
+export QEMU_DEFAULT_MACHINE="$default_machine"
 
 [ -f /etc/qemu-iotest.config ]       && . /etc/qemu-iotest.config
 
@@ -155,4 +227,4 @@
 }
 
 # make sure this script returns success
-/bin/true
+true
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index 3acdb30..3ab6e4d 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -19,107 +19,6 @@
 # standard filters
 #
 
-# Checks that given_value is in range of correct_value +/- tolerance.
-# Tolerance can be an absolute value or a percentage of the correct value
-# (see examples with tolerances below).
-# Outputs suitable message to stdout if it's not in range.
-#
-# A verbose option, -v, may be used as the LAST argument
-#
-# e.g.
-# foo: 0.0298 = 0.03 +/- 5%
-# _within_tolerance "foo" 0.0298 0.03 5%
-#
-# foo: 0.0298 = 0.03 +/- 0.01
-# _within_tolerance "foo" 0.0298 0.03 0.01
-#
-# foo: 0.0298 = 0.03 -0.01 +0.002
-# _within_tolerance "foo" 0.0298 0.03 0.01 0.002
-#
-# foo: verbose output of 0.0298 = 0.03 +/- 5%
-# _within_tolerance "foo" 0.0298 0.03 5% -v
-_within_tolerance()
-{
-  _name=$1
-  _given_val=$2
-  _correct_val=$3
-  _mintol=$4
-  _maxtol=$_mintol
-  _verbose=0
-  _debug=false
-
-  # maxtol arg is optional
-  # verbose arg is optional
-  if [ $# -ge 5 ]
-  then
-     if [ "$5" = "-v" ]
-     then
-        _verbose=1
-     else
-        _maxtol=$5
-     fi
-  fi
-  if [ $# -ge 6 ]
-  then
-     [ "$6" = "-v" ] && _verbose=1
-  fi
-
-  # find min with or without %
-  _mintolerance=`echo $_mintol | sed -e 's/%//'`
-  if [ $_mintol = $_mintolerance ]
-  then
-      _min=`echo "scale=5; $_correct_val-$_mintolerance" | bc`
-  else
-      _min=`echo "scale=5; $_correct_val-$_mintolerance*0.01*$_correct_val" | bc`
-  fi
-
-  # find max with or without %
-  _maxtolerance=`echo $_maxtol | sed -e 's/%//'`
-  if [ $_maxtol = $_maxtolerance ]
-  then
-      _max=`echo "scale=5; $_correct_val+$_maxtolerance" | bc`
-  else
-      _max=`echo "scale=5; $_correct_val+$_maxtolerance*0.01*$_correct_val" | bc`
-  fi
-
-  $_debug && echo "min = $_min"
-  $_debug && echo "max = $_max"
-
-  cat <<EOF >$tmp.bc.1
-scale=5;
-if ($_min <= $_given_val) 1;
-if ($_min > $_given_val) 0;
-EOF
-
-  cat <<EOF >$tmp.bc.2
-scale=5;
-if ($_given_val <= $_max) 1;
-if ($_given_val > $_max) 0;
-EOF
-
-  _above_min=`bc <$tmp.bc.1`
-  _below_max=`bc <$tmp.bc.2`
-
-  rm -f $tmp.bc.[12]
-
-  _in_range=`expr $_above_min \& $_below_max`
-
-  # fix up min, max precision for output
-  # can vary for 5.3, 6.2
-  _min=`echo $_min | sed -e 's/0*$//'` # get rid of trailling zeroes
-  _max=`echo $_max | sed -e 's/0*$//'` # get rid of trailling zeroes
-
-  if [ $_in_range -eq 1 ]
-  then
-        [ $_verbose -eq 1 ] && echo $_name is in range
-        return 0
-  else
-        [ $_verbose -eq 1 ] && echo $_name has value of $_given_val
-        [ $_verbose -eq 1 ] && echo $_name is NOT in range $_min .. $_max
-        return 1
-  fi
-}
-
 # ctime(3) dates
 #
 _filter_date()
@@ -128,6 +27,11 @@
         -e 's/[A-Z][a-z][a-z] [A-z][a-z][a-z]  *[0-9][0-9]* [0-9][0-9]:[0-9][0-9]:[0-9][0-9] [0-9][0-9][0-9][0-9]$/DATE/'
 }
 
+_filter_generated_node_ids()
+{
+     sed -re 's/\#block[0-9]{3,}/NODE_NAME/'
+}
+
 # replace occurrences of the actual TEST_DIR value with TEST_DIR
 _filter_testdir()
 {
@@ -150,7 +54,7 @@
 _filter_qemu_io()
 {
     _filter_win32 | sed -e "s/[0-9]* ops\; [0-9/:. sec]* ([0-9/.inf]* [EPTGMKiBbytes]*\/sec and [0-9/.inf]* ops\/sec)/X ops\; XX:XX:XX.X (XXX YYY\/sec and XXX ops\/sec)/" \
-        -e "s/: line [0-9][0-9]*:  *[0-9][0-9]*\( Aborted\)/:\1/" \
+        -e "s/: line [0-9][0-9]*:  *[0-9][0-9]*\( Aborted\| Killed\)/:\1/" \
         -e "s/qemu-io> //g"
 }
 
@@ -159,6 +63,7 @@
 {
     sed -e "s#\\(^\\|(qemu) \\)$(basename $QEMU_PROG):#\1QEMU_PROG:#" \
         -e 's#^QEMU [0-9]\+\.[0-9]\+\.[0-9]\+ monitor#QEMU X.Y.Z monitor#' \
+        -e '/main-loop: WARNING: I\/O thread spun for [0-9]\+ iterations/d' \
         -e $'s#\r##' # QEMU monitor uses \r\n line endings
 }
 
@@ -167,7 +72,15 @@
 {
     _filter_win32 | \
     sed -e 's#\("\(micro\)\?seconds": \)[0-9]\+#\1 TIMESTAMP#g' \
-        -e 's#^{"QMP":.*}$#QMP_VERSION#'
+        -e 's#^{"QMP":.*}$#QMP_VERSION#' \
+        -e '/^    "QMP": {\s*$/, /^    }\s*$/ c\' \
+        -e '    QMP_VERSION'
+}
+
+# replace block job offset
+_filter_block_job_offset()
+{
+    sed -e 's/, "offset": [0-9]\+,/, "offset": OFFSET,/'
 }
 
 # replace driver-specific options in the "Formatting..." line
@@ -179,17 +92,20 @@
         -e "s# encryption=off##g" \
         -e "s# cluster_size=[0-9]\\+##g" \
         -e "s# table_size=[0-9]\\+##g" \
-        -e "s# compat='[^']*'##g" \
+        -e "s# compat=[^ ]*##g" \
         -e "s# compat6=\\(on\\|off\\)##g" \
         -e "s# static=\\(on\\|off\\)##g" \
         -e "s# zeroed_grain=\\(on\\|off\\)##g" \
         -e "s# subformat='[^']*'##g" \
         -e "s# adapter_type='[^']*'##g" \
+        -e "s# hwversion=[^ ]*##g" \
         -e "s# lazy_refcounts=\\(on\\|off\\)##g" \
         -e "s# block_size=[0-9]\\+##g" \
         -e "s# block_state_zero=\\(on\\|off\\)##g" \
         -e "s# log_size=[0-9]\\+##g" \
-        -e "s/archipelago:a/TEST_DIR\//g"
+        -e "s/archipelago:a/TEST_DIR\//g" \
+        -e "s# refcount_bits=[0-9]\\+##g" \
+        -e "s# key-secret=[a-zA-Z0-9]\\+##g"
 }
 
 _filter_img_info()
@@ -197,6 +113,7 @@
     sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
         -e "s#$TEST_DIR#TEST_DIR#g" \
         -e "s#$IMGFMT#IMGFMT#g" \
+        -e 's#nbd://127.0.0.1:10810$#TEST_DIR/t.IMGFMT#g' \
         -e "/encrypted: yes/d" \
         -e "/cluster_size: [0-9]\\+/d" \
         -e "/table_size: [0-9]\\+/d" \
@@ -206,6 +123,7 @@
         -e "/zeroed_grain: \\(on\\|off\\)/d" \
         -e "/subformat: '[^']*'/d" \
         -e "/adapter_type: '[^']*'/d" \
+        -e "/hwversion: '[^']*'/d" \
         -e "/lazy_refcounts: \\(on\\|off\\)/d" \
         -e "/block_size: [0-9]\\+/d" \
         -e "/block_state_zero: \\(on\\|off\\)/d" \
@@ -220,5 +138,18 @@
         -e 's/Mapped to *//' | _filter_testdir | _filter_imgfmt
 }
 
+_filter_nbd()
+{
+    # nbd.c error messages contain function names and line numbers that are
+    # prone to change.  Message ordering depends on timing between send and
+    # receive callbacks sometimes, making them unreliable.
+    #
+    # Filter out the TCP port number since this changes between runs.
+    sed -e '/nbd\/.*\.c:/d' \
+        -e 's#nbd:\(//\)\?127\.0\.0\.1:[0-9]*#nbd:\1127.0.0.1:PORT#g' \
+        -e "s#?socket=$TEST_DIR#?socket=TEST_DIR#g" \
+        -e 's#\(exportname=foo\|PORT\): Failed to .*$#\1#'
+}
+
 # make sure this script returns success
-/bin/true
+true
diff --git a/tests/qemu-iotests/common.qemu b/tests/qemu-iotests/common.qemu
index ee7ebb4..2548a87 100644
--- a/tests/qemu-iotests/common.qemu
+++ b/tests/qemu-iotests/common.qemu
@@ -30,8 +30,6 @@
 QEMU_FIFO_IN="${TEST_DIR}/qmp-in-$$"
 QEMU_FIFO_OUT="${TEST_DIR}/qmp-out-$$"
 
-QEMU_PID=
-_QEMU_HANDLE=0
 QEMU_HANDLE=0
 
 # If bash version is >= 4.1, these will be overwritten and dynamic
@@ -131,6 +129,8 @@
 # $qemu_comm_method: set this variable to 'monitor' (case insensitive)
 #                    to use the QEMU HMP monitor for communication.
 #                    Otherwise, the default of QMP is used.
+# $keep_stderr: Set this variable to 'y' to keep QEMU's stderr output on stderr.
+#               If this variable is empty, stderr will be redirected to stdout.
 # Returns:
 # $QEMU_HANDLE: set to a handle value to communicate with this QEMU instance.
 #
@@ -153,10 +153,20 @@
     mkfifo "${fifo_out}"
     mkfifo "${fifo_in}"
 
-    "${QEMU}" -nographic -serial none ${comm} -machine accel=qtest "${@}" 2>&1 \
+    if [ -z "$keep_stderr" ]; then
+        QEMU_NEED_PID='y'\
+        ${QEMU} -nographic -serial none ${comm} -machine accel=qtest "${@}" \
+                                                                >"${fifo_out}" \
+                                                                2>&1 \
+                                                                <"${fifo_in}" &
+    elif [ "$keep_stderr" = "y" ]; then
+        QEMU_NEED_PID='y'\
+        ${QEMU} -nographic -serial none ${comm} -machine accel=qtest "${@}" \
                                                                 >"${fifo_out}" \
                                                                 <"${fifo_in}" &
-    QEMU_PID[${_QEMU_HANDLE}]=$!
+    else
+        exit 1
+    fi
 
     if [[ "${BASH_VERSINFO[0]}" -ge "5" ||
         ("${BASH_VERSINFO[0]}" -ge "4"  &&  "${BASH_VERSINFO[1]}" -ge "1") ]]
@@ -186,13 +196,31 @@
 
 
 # Silenty kills the QEMU process
+#
+# If $wait is set to anything other than the empty string, the process will not
+# be killed but only waited for, and any output will be forwarded to stdout. If
+# $wait is empty, the process will be killed and all output will be suppressed.
 function _cleanup_qemu()
 {
     # QEMU_PID[], QEMU_IN[], QEMU_OUT[] all use same indices
     for i in "${!QEMU_OUT[@]}"
     do
-        kill -KILL ${QEMU_PID[$i]} 2>/dev/null
-        wait ${QEMU_PID[$i]} 2>/dev/null # silent kill
+        local QEMU_PID
+        if [ -f "${TEST_DIR}/qemu-${i}.pid" ]; then
+            read QEMU_PID < "${TEST_DIR}/qemu-${i}.pid"
+            rm -f "${TEST_DIR}/qemu-${i}.pid"
+            if [ -z "${wait}" ] && [ -n "${QEMU_PID}" ]; then
+                kill -KILL ${QEMU_PID} 2>/dev/null
+            fi
+            if [ -n "${QEMU_PID}" ]; then
+                wait ${QEMU_PID} 2>/dev/null # silent kill
+            fi
+        fi
+
+        if [ -n "${wait}" ]; then
+            cat <&${QEMU_OUT[$i]} | _filter_testdir | _filter_qemu \
+                                  | _filter_qemu_io | _filter_qmp
+        fi
         rm -f "${QEMU_FIFO_IN}_${i}" "${QEMU_FIFO_OUT}_${i}"
         eval "exec ${QEMU_IN[$i]}<&-"   # close file descriptors
         eval "exec ${QEMU_OUT[$i]}<&-"
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index 9c49deb..306b00c 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -53,32 +53,46 @@
 # make sure we have a standard umask
 umask 022
 
-if [ "$IMGPROTO" = "file" ]; then
-    TEST_IMG=$TEST_DIR/t.$IMGFMT
-elif [ "$IMGPROTO" = "nbd" ]; then
-    TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
-    TEST_IMG="nbd:127.0.0.1:10810"
-elif [ "$IMGPROTO" = "ssh" ]; then
-    TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
-    TEST_IMG="ssh://127.0.0.1$TEST_IMG_FILE"
-elif [ "$IMGPROTO" = "nfs" ]; then
-    TEST_DIR="nfs://127.0.0.1/$TEST_DIR"
-    TEST_IMG=$TEST_DIR/t.$IMGFMT
-elif [ "$IMGPROTO" = "archipelago" ]; then
-    TEST_IMG="archipelago:at.$IMGFMT"
-else
-    TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
-fi
-
-function valgrind_qemu_io()
-{
-    valgrind --log-file=/tmp/$$.valgrind --error-exitcode=99 $REAL_QEMU_IO "$@"
-    if [ $? != 0 ]; then
-        cat /tmp/$$.valgrind
+if [ "$IMGOPTSSYNTAX" = "true" ]; then
+    DRIVER="driver=$IMGFMT"
+    if [ "$IMGFMT" = "luks" ]; then
+        DRIVER="$DRIVER,key-secret=keysec0"
     fi
-    rm -f /tmp/$$.valgrind
-}
-
+    if [ "$IMGPROTO" = "file" ]; then
+        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+        TEST_IMG="$DRIVER,file.filename=$TEST_DIR/t.$IMGFMT"
+    elif [ "$IMGPROTO" = "nbd" ]; then
+        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+        TEST_IMG="$DRIVER,file.driver=nbd,file.host=127.0.0.1,file.port=10810"
+    elif [ "$IMGPROTO" = "ssh" ]; then
+        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+        TEST_IMG="$DRIVER,file.driver=ssh,file.host=127.0.0.1,file.path=$TEST_IMG_FILE"
+    elif [ "$IMGPROTO" = "nfs" ]; then
+        TEST_DIR="$DRIVER,file.driver=nfs,file.filename=nfs://127.0.0.1/$TEST_DIR"
+        TEST_IMG=$TEST_DIR_OPTS/t.$IMGFMT
+    elif [ "$IMGPROTO" = "archipelago" ]; then
+        TEST_IMG="$DRIVER,file.driver=archipelago,file.volume=:at.$IMGFMT"
+    else
+        TEST_IMG="$DRIVER,file.driver=$IMGPROTO,file.filename=$TEST_DIR/t.$IMGFMT"
+    fi
+else
+    if [ "$IMGPROTO" = "file" ]; then
+        TEST_IMG=$TEST_DIR/t.$IMGFMT
+    elif [ "$IMGPROTO" = "nbd" ]; then
+        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+        TEST_IMG="nbd:127.0.0.1:10810"
+    elif [ "$IMGPROTO" = "ssh" ]; then
+        TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+        TEST_IMG="ssh://127.0.0.1$TEST_IMG_FILE"
+    elif [ "$IMGPROTO" = "nfs" ]; then
+        TEST_DIR="nfs://127.0.0.1/$TEST_DIR"
+        TEST_IMG=$TEST_DIR/t.$IMGFMT
+    elif [ "$IMGPROTO" = "archipelago" ]; then
+        TEST_IMG="archipelago:at.$IMGFMT"
+    else
+        TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
+    fi
+fi
 
 _optstr_add()
 {
@@ -118,6 +132,7 @@
     local img_name=""
     local use_backing=0
     local backing_file=""
+    local object_options=""
 
     if [ -n "$TEST_IMG_FILE" ]; then
         img_name=$TEST_IMG_FILE
@@ -128,6 +143,10 @@
     if [ -n "$IMGOPTS" ]; then
         optstr=$(_optstr_add "$optstr" "$IMGOPTS")
     fi
+    if [ -n "$IMGKEYSECRET" ]; then
+        object_options="--object secret,id=keysec0,data=$IMGKEYSECRET"
+        optstr=$(_optstr_add "$optstr" "key-secret=keysec0")
+    fi
 
     if [ "$1" = "-b" ]; then
         use_backing=1
@@ -145,16 +164,15 @@
     # XXX(hch): have global image options?
     (
      if [ $use_backing = 1 ]; then
-        $QEMU_IMG create -f $IMGFMT $extra_img_options -b "$backing_file" "$img_name" $image_size 2>&1
+        $QEMU_IMG create $object_options -f $IMGFMT $extra_img_options -b "$backing_file" "$img_name" $image_size 2>&1
      else
-        $QEMU_IMG create -f $IMGFMT $extra_img_options "$img_name" $image_size 2>&1
+        $QEMU_IMG create $object_options -f $IMGFMT $extra_img_options "$img_name" $image_size 2>&1
      fi
     ) | _filter_img_create
 
     # Start an NBD server on the image file, which is what we'll be talking to
     if [ $IMGPROTO = "nbd" ]; then
-        eval "$QEMU_NBD -v -t -b 127.0.0.1 -p 10810  $TEST_IMG_FILE &"
-        QEMU_NBD_PID=$!
+        eval "$QEMU_NBD -v -t -b 127.0.0.1 -p 10810 -f $IMGFMT  $TEST_IMG_FILE &"
         sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
     fi
 }
@@ -175,7 +193,12 @@
     case "$IMGPROTO" in
 
         nbd)
-            kill $QEMU_NBD_PID
+            if [ -f "${TEST_DIR}/qemu-nbd.pid" ]; then
+                local QEMU_NBD_PID
+                read QEMU_NBD_PID < "${TEST_DIR}/qemu-nbd.pid"
+                kill ${QEMU_NBD_PID}
+                rm -f "${TEST_DIR}/qemu-nbd.pid"
+            fi
             rm -f "$TEST_IMG_FILE"
             ;;
         file)
@@ -205,7 +228,13 @@
 
 _check_test_img()
 {
-    $QEMU_IMG check "$@" -f $IMGFMT "$TEST_IMG" 2>&1 | _filter_testdir | \
+    (
+        if [ "$IMGOPTSSYNTAX" = "true" ]; then
+            $QEMU_IMG check $QEMU_IMG_EXTRA_ARGS "$@" "$TEST_IMG" 2>&1
+        else
+            $QEMU_IMG check "$@" -f $IMGFMT "$TEST_IMG" 2>&1
+        fi
+    ) | _filter_testdir | \
         sed -e '/allocated.*fragmented.*compressed clusters/d' \
             -e 's/qemu-img: This image format does not support checks/No errors were found on the image./' \
             -e '/Image end offset: [0-9]\+/d'
@@ -213,6 +242,13 @@
 
 _img_info()
 {
+    if [[ "$1" == "--format-specific" ]]; then
+        local format_specific=1
+        shift
+    else
+        local format_specific=0
+    fi
+
     discard=0
     regex_json_spec_start='^ *"format-specific": \{'
     $QEMU_IMG info "$@" "$TEST_IMG" 2>&1 | \
@@ -222,7 +258,9 @@
             -e "/^disk size:/ D" \
             -e "/actual-size/ D" | \
         while IFS='' read line; do
-            if [[ $line == "Format specific information:" ]]; then
+            if [[ $format_specific == 1 ]]; then
+                discard=0
+            elif [[ $line == "Format specific information:" ]]; then
                 discard=1
             elif [[ $line =~ $regex_json_spec_start ]]; then
                 discard=2
@@ -284,52 +322,6 @@
     fi
 }
 
-
-# Do a command, log it to $seq.full, optionally test return status
-# and die if command fails. If called with one argument _do executes the
-# command, logs it, and returns its exit status. With two arguments _do
-# first prints the message passed in the first argument, and then "done"
-# or "fail" depending on the return status of the command passed in the
-# second argument. If the command fails and the variable _do_die_on_error
-# is set to "always" or the two argument form is used and _do_die_on_error
-# is set to "message_only" _do will print an error message to
-# $seq.out and exit.
-
-_do()
-{
-    if [ $# -eq 1 ]; then
-        _cmd=$1
-    elif [ $# -eq 2 ]; then
-        _note=$1
-        _cmd=$2
-        echo -n "$_note... "
-    else
-        echo "Usage: _do [note] cmd" 1>&2
-        status=1; exit
-    fi
-
-    (eval "echo '---' \"$_cmd\"") >>"$OUTPUT_DIR/$seq.full"
-    (eval "$_cmd") >$tmp._out 2>&1; ret=$?
-    cat $tmp._out >>"$OUTPUT_DIR/$seq.full"
-    if [ $# -eq 2 ]; then
-        if [ $ret -eq 0 ]; then
-            echo "done"
-        else
-            echo "fail"
-        fi
-    fi
-    if [ $ret -ne 0  ] \
-        && [ "$_do_die_on_error" = "always" \
-            -o \( $# -eq 2 -a "$_do_die_on_error" = "message_only" \) ]
-    then
-        [ $# -ne 2 ] && echo
-        eval "echo \"$_cmd\" failed \(returned $ret\): see $seq.full"
-        status=1; exit
-    fi
-
-    return $ret
-}
-
 # bail out, setting up .notrun file
 #
 _notrun()
@@ -428,7 +420,17 @@
 #
 _require_command()
 {
-    eval c=\$$1
+    if [ "$1" = "QEMU" ]; then
+        c=$QEMU_PROG
+    elif [ "$1" = "QEMU_IMG" ]; then
+        c=$QEMU_IMG_PROG
+    elif [ "$1" = "QEMU_IO" ]; then
+        c=$QEMU_IO_PROG
+    elif [ "$1" = "QEMU_NBD" ]; then
+        c=$QEMU_NBD_PROG
+    else
+        eval c=\$$1
+    fi
     [ -x "$c" ] || _notrun "$1 utility required, skipped this test"
 }
 
@@ -479,4 +481,4 @@
 }
 
 # make sure this script returns success
-/bin/true
+true
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index 7dfe469..50ddeed 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -12,7 +12,7 @@
 003 rw auto
 004 rw auto quick
 005 img auto quick
-006 img auto
+# 006 was removed, do not reuse
 007 snapshot auto
 008 rw auto quick
 009 rw auto quick
@@ -22,7 +22,7 @@
 013 rw auto
 014 rw auto
 015 rw snapshot auto
-016 rw auto quick
+# 016 was removed, do not reuse
 017 rw backing auto quick
 018 rw backing auto quick
 019 rw backing auto quick
@@ -97,13 +97,16 @@
 088 rw auto quick
 089 rw auto quick
 090 rw auto quick
-091 rw auto quick
+091 rw auto
 092 rw auto quick
+093 auto
+094 rw auto quick
 095 rw auto quick
+096 rw auto quick
 097 rw auto backing
 098 rw auto backing quick
 099 rw auto quick
-100 rw auto quick
+# 100 was removed, do not reuse
 101 rw auto quick
 102 rw auto quick
 103 rw auto quick
@@ -111,4 +114,47 @@
 105 rw auto quick
 107 rw auto quick
 108 rw auto quick
+109 rw auto
+110 rw auto backing quick
 111 rw auto quick
+112 rw auto
+113 rw auto quick
+114 rw auto quick
+115 rw auto
+116 rw auto quick
+117 rw auto
+118 rw auto
+119 rw auto quick
+120 rw auto quick
+121 rw auto
+122 rw auto
+123 rw auto quick
+124 rw auto backing
+128 rw auto quick
+129 rw auto quick
+130 rw auto quick
+131 rw auto quick
+132 rw auto quick
+133 auto quick
+134 rw auto quick
+135 rw auto
+136 rw auto
+137 rw auto
+138 rw auto quick
+139 rw auto quick
+140 rw auto quick
+141 rw auto quick
+142 auto
+143 auto quick
+144 rw auto quick
+145 auto quick
+146 auto quick
+148 rw auto quick
+149 rw auto sudo
+150 rw auto quick
+152 rw auto quick
+154 rw auto backing quick
+155 rw auto
+156 rw auto quick
+157 auto
+162 auto quick
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index f57f154..dbe0ee5 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -16,49 +16,75 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import errno
 import os
 import re
 import subprocess
 import string
 import unittest
-import sys; sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts', 'qmp'))
-import qmp
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'scripts'))
+import qtest
 import struct
+import json
 
-__all__ = ['imgfmt', 'imgproto', 'test_dir' 'qemu_img', 'qemu_io',
-           'VM', 'QMPTestCase', 'notrun', 'main']
 
-# This will not work if arguments or path contain spaces but is necessary if we
+# This will not work if arguments contain spaces but is necessary if we
 # want to support the override options that ./check supports.
-qemu_img_args = os.environ.get('QEMU_IMG', 'qemu-img').strip().split(' ')
-qemu_io_args = os.environ.get('QEMU_IO', 'qemu-io').strip().split(' ')
-qemu_args = os.environ.get('QEMU', 'qemu').strip().split(' ')
+qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')]
+if os.environ.get('QEMU_IMG_OPTIONS'):
+    qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ')
+
+qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')]
+if os.environ.get('QEMU_IO_OPTIONS'):
+    qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ')
+
+qemu_prog = os.environ.get('QEMU_PROG', 'qemu')
+qemu_opts = os.environ.get('QEMU_OPTIONS', '').strip().split(' ')
 
 imgfmt = os.environ.get('IMGFMT', 'raw')
 imgproto = os.environ.get('IMGPROTO', 'file')
-test_dir = os.environ.get('TEST_DIR', '/var/tmp')
+test_dir = os.environ.get('TEST_DIR')
 output_dir = os.environ.get('OUTPUT_DIR', '.')
 cachemode = os.environ.get('CACHEMODE')
+qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE')
 
 socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
 
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
-    return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return exitcode
 
 def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
-    return subprocess.call(qemu_img_args + list(args))
+    exitcode = subprocess.call(qemu_img_args + list(args))
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return exitcode
 
 def qemu_img_pipe(*args):
     '''Run qemu-img and return its output'''
-    return subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE).communicate()[0]
+    subp = subprocess.Popen(qemu_img_args + list(args),
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
+    return subp.communicate()[0]
 
 def qemu_io(*args):
     '''Run qemu-io and return the stdout data'''
     args = qemu_io_args + list(args)
-    return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0]
+    subp = subprocess.Popen(args, stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+    exitcode = subp.wait()
+    if exitcode < 0:
+        sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args)))
+    return subp.communicate()[0]
 
 def compare_images(img1, img2):
     '''Return True if two image files are identical'''
@@ -75,32 +101,56 @@
         i = i + 512
     file.close()
 
-class VM(object):
+def image_size(img):
+    '''Return image's virtual size'''
+    r = qemu_img_pipe('info', '--output=json', '-f', imgfmt, img)
+    return json.loads(r)['virtual-size']
+
+test_dir_re = re.compile(r"%s" % test_dir)
+def filter_test_dir(msg):
+    return test_dir_re.sub("TEST_DIR", msg)
+
+win32_re = re.compile(r"\r")
+def filter_win32(msg):
+    return win32_re.sub("", msg)
+
+qemu_io_re = re.compile(r"[0-9]* ops; [0-9\/:. sec]* \([0-9\/.inf]* [EPTGMKiBbytes]*\/sec and [0-9\/.inf]* ops\/sec\)")
+def filter_qemu_io(msg):
+    msg = filter_win32(msg)
+    return qemu_io_re.sub("X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)", msg)
+
+chown_re = re.compile(r"chown [0-9]+:[0-9]+")
+def filter_chown(msg):
+    return chown_re.sub("chown UID:GID", msg)
+
+def log(msg, filters=[]):
+    for flt in filters:
+        msg = flt(msg)
+    print msg
+
+class VM(qtest.QEMUQtestMachine):
     '''A QEMU VM'''
 
     def __init__(self):
-        self._monitor_path = os.path.join(test_dir, 'qemu-mon.%d' % os.getpid())
-        self._qemu_log_path = os.path.join(test_dir, 'qemu-log.%d' % os.getpid())
-        self._args = qemu_args + ['-chardev',
-                     'socket,id=mon,path=' + self._monitor_path,
-                     '-mon', 'chardev=mon,mode=control',
-                     '-qtest', 'stdio', '-machine', 'accel=qtest',
-                     '-display', 'none', '-vga', 'none']
+        super(VM, self).__init__(qemu_prog, qemu_opts, test_dir=test_dir,
+                                 socket_scm_helper=socket_scm_helper)
         self._num_drives = 0
 
-    # This can be used to add an unused monitor instance.
-    def add_monitor_telnet(self, ip, port):
-        args = 'tcp:%s:%d,server,nowait,telnet' % (ip, port)
-        self._args.append('-monitor')
-        self._args.append(args)
+    def add_drive_raw(self, opts):
+        self._args.append('-drive')
+        self._args.append(opts)
+        return self
 
-    def add_drive(self, path, opts=''):
+    def add_drive(self, path, opts='', interface='virtio'):
         '''Add a virtio-blk drive to the VM'''
-        options = ['if=virtio',
-                   'format=%s' % imgfmt,
-                   'cache=%s' % cachemode,
-                   'file=%s' % path,
+        options = ['if=%s' % interface,
                    'id=drive%d' % self._num_drives]
+
+        if path is not None:
+            options.append('file=%s' % path)
+            options.append('format=%s' % imgfmt)
+            options.append('cache=%s' % cachemode)
+
         if opts:
             options.append(opts)
 
@@ -127,73 +177,6 @@
         return self.qmp('human-monitor-command',
                         command_line='qemu-io %s "%s"' % (drive, cmd))
 
-    def add_fd(self, fd, fdset, opaque, opts=''):
-        '''Pass a file descriptor to the VM'''
-        options = ['fd=%d' % fd,
-                   'set=%d' % fdset,
-                   'opaque=%s' % opaque]
-        if opts:
-            options.append(opts)
-
-        self._args.append('-add-fd')
-        self._args.append(','.join(options))
-        return self
-
-    def send_fd_scm(self, fd_file_path):
-        # In iotest.py, the qmp should always use unix socket.
-        assert self._qmp.is_scm_available()
-        bin = socket_scm_helper
-        if os.path.exists(bin) == False:
-            print "Scm help program does not present, path '%s'." % bin
-            return -1
-        fd_param = ["%s" % bin,
-                    "%d" % self._qmp.get_sock_fd(),
-                    "%s" % fd_file_path]
-        devnull = open('/dev/null', 'rb')
-        p = subprocess.Popen(fd_param, stdin=devnull, stdout=sys.stdout,
-                             stderr=sys.stderr)
-        return p.wait()
-
-    def launch(self):
-        '''Launch the VM and establish a QMP connection'''
-        devnull = open('/dev/null', 'rb')
-        qemulog = open(self._qemu_log_path, 'wb')
-        try:
-            self._qmp = qmp.QEMUMonitorProtocol(self._monitor_path, server=True)
-            self._popen = subprocess.Popen(self._args, stdin=devnull, stdout=qemulog,
-                                           stderr=subprocess.STDOUT)
-            self._qmp.accept()
-        except:
-            os.remove(self._monitor_path)
-            raise
-
-    def shutdown(self):
-        '''Terminate the VM and clean up'''
-        if not self._popen is None:
-            self._qmp.cmd('quit')
-            self._popen.wait()
-            os.remove(self._monitor_path)
-            os.remove(self._qemu_log_path)
-            self._popen = None
-
-    underscore_to_dash = string.maketrans('_', '-')
-    def qmp(self, cmd, **args):
-        '''Invoke a QMP command and return the result dict'''
-        qmp_args = dict()
-        for k in args.keys():
-            qmp_args[k.translate(self.underscore_to_dash)] = args[k]
-
-        return self._qmp.cmd(cmd, args=qmp_args)
-
-    def get_qmp_event(self, wait=False):
-        '''Poll for one queued QMP events and return it'''
-        return self._qmp.pull_event(wait=wait)
-
-    def get_qmp_events(self, wait=False):
-        '''Poll for queued QMP events and return a list of dicts'''
-        events = self._qmp.get_events(wait=wait)
-        self._qmp.clear_events()
-        return events
 
 index_re = re.compile(r'([^\[]+)\[([^\]]+)\]')
 
@@ -237,6 +220,20 @@
         result = self.vm.qmp('query-block-jobs')
         self.assert_qmp(result, 'return', [])
 
+    def assert_has_block_node(self, node_name=None, file_name=None):
+        """Issue a query-named-block-nodes and assert node_name and/or
+        file_name is present in the result"""
+        def check_equal_or_none(a, b):
+            return a == None or b == None or a == b
+        assert node_name or file_name
+        result = self.vm.qmp('query-named-block-nodes')
+        for x in result["return"]:
+            if check_equal_or_none(x.get("node-name"), node_name) and \
+                    check_equal_or_none(x.get("file"), file_name):
+                return
+        self.assertTrue(False, "Cannot find %s %s in result:\n%s" % \
+                (node_name, file_name, result))
+
     def cancel_and_wait(self, drive='drive0', force=False, resume=False):
         '''Cancel a block job and wait for it to finish, returning the event'''
         result = self.vm.qmp('block-job-cancel', device=drive, force=force)
@@ -273,6 +270,29 @@
         self.assert_no_active_block_jobs()
         return event
 
+    def wait_ready(self, drive='drive0'):
+        '''Wait until a block job BLOCK_JOB_READY event'''
+        f = {'data': {'type': 'mirror', 'device': drive } }
+        event = self.vm.event_wait(name='BLOCK_JOB_READY', match=f)
+
+    def wait_ready_and_cancel(self, drive='drive0'):
+        self.wait_ready(drive=drive)
+        event = self.cancel_and_wait(drive=drive)
+        self.assertEquals(event['event'], 'BLOCK_JOB_COMPLETED')
+        self.assert_qmp(event, 'data/type', 'mirror')
+        self.assert_qmp(event, 'data/offset', event['data']['len'])
+
+    def complete_and_wait(self, drive='drive0', wait_ready=True):
+        '''Complete a block job and wait for it to finish'''
+        if wait_ready:
+            self.wait_ready(drive=drive)
+
+        result = self.vm.qmp('block-job-complete', device=drive)
+        self.assert_qmp(result, 'return', {})
+
+        event = self.wait_until_completed(drive=drive)
+        self.assert_qmp(event, 'data/type', 'mirror')
+
 def notrun(reason):
     '''Skip this test suite'''
     # Each test in qemu-iotests has a number ("seq")
@@ -282,23 +302,52 @@
     print '%s not run: %s' % (seq, reason)
     sys.exit(0)
 
-def main(supported_fmts=[]):
-    '''Run tests'''
-
+def verify_image_format(supported_fmts=[]):
     if supported_fmts and (imgfmt not in supported_fmts):
         notrun('not suitable for this image format: %s' % imgfmt)
 
+def verify_platform(supported_oses=['linux']):
+    if True not in [sys.platform.startswith(x) for x in supported_oses]:
+        notrun('not suitable for this OS: %s' % sys.platform)
+
+def verify_quorum():
+    '''Skip test suite if quorum support is not available'''
+    if 'quorum' not in qemu_img_pipe('--help'):
+        notrun('quorum support missing')
+
+def main(supported_fmts=[], supported_oses=['linux']):
+    '''Run tests'''
+
+    # We are using TEST_DIR and QEMU_DEFAULT_MACHINE as proxies to
+    # indicate that we're not being run via "check". There may be
+    # other things set up by "check" that individual test cases rely
+    # on.
+    if test_dir is None or qemu_default_machine is None:
+        sys.stderr.write('Please run this test via the "check" script\n')
+        sys.exit(os.EX_USAGE)
+
+    debug = '-d' in sys.argv
+    verbosity = 1
+    verify_image_format(supported_fmts)
+    verify_platform(supported_oses)
+
     # We need to filter out the time taken from the output so that qemu-iotest
     # can reliably diff the results against master output.
     import StringIO
-    output = StringIO.StringIO()
+    if debug:
+        output = sys.stdout
+        verbosity = 2
+        sys.argv.remove('-d')
+    else:
+        output = StringIO.StringIO()
 
     class MyTestRunner(unittest.TextTestRunner):
-        def __init__(self, stream=output, descriptions=True, verbosity=1):
+        def __init__(self, stream=output, descriptions=True, verbosity=verbosity):
             unittest.TextTestRunner.__init__(self, stream, descriptions, verbosity)
 
     # unittest.main() will use sys.exit() so expect a SystemExit exception
     try:
         unittest.main(testRunner=MyTestRunner)
     finally:
-        sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', output.getvalue()))
+        if not debug:
+            sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', output.getvalue()))
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 2058596..9cc4cf7 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -7,6 +7,10 @@
 class QcowHeaderExtension:
 
     def __init__(self, magic, length, data):
+        if length % 8 != 0:
+            padding = 8 - (length % 8)
+            data += "\0" * padding
+
         self.magic  = magic
         self.length = length
         self.data   = data
diff --git a/tests/qemu-iotests/qed.py b/tests/qemu-iotests/qed.py
index 52ff845..748068d 100755
--- a/tests/qemu-iotests/qed.py
+++ b/tests/qemu-iotests/qed.py
@@ -227,7 +227,7 @@
     qed = QED(open(filename, 'r+b'))
     try:
         globals()[cmd](qed, *sys.argv[3:])
-    except TypeError, e:
+    except TypeError as e:
         sys.stderr.write(globals()[cmd].__doc__ + '\n')
         sys.exit(1)
 
diff --git a/tests/qemu-iotests/sample_images/afl5.img.bz2 b/tests/qemu-iotests/sample_images/afl5.img.bz2
new file mode 100644
index 0000000..1614348
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/afl5.img.bz2
Binary files differ
diff --git a/tests/qemu-iotests/sample_images/afl9.vmdk.bz2 b/tests/qemu-iotests/sample_images/afl9.vmdk.bz2
new file mode 100644
index 0000000..03615d3
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/afl9.vmdk.bz2
Binary files differ
diff --git a/tests/qemu-iotests/sample_images/d2v-zerofilled.vhd.bz2 b/tests/qemu-iotests/sample_images/d2v-zerofilled.vhd.bz2
new file mode 100644
index 0000000..f12cb92
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/d2v-zerofilled.vhd.bz2
Binary files differ
diff --git a/tests/qemu-iotests/sample_images/grub_mbr.raw.bz2 b/tests/qemu-iotests/sample_images/grub_mbr.raw.bz2
new file mode 100644
index 0000000..8585878
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/grub_mbr.raw.bz2
Binary files differ
diff --git a/tests/qemu-iotests/sample_images/hyperv2012r2-dynamic.vhd.bz2 b/tests/qemu-iotests/sample_images/hyperv2012r2-dynamic.vhd.bz2
new file mode 100644
index 0000000..bfeccf7
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/hyperv2012r2-dynamic.vhd.bz2
Binary files differ
diff --git a/tests/qemu-iotests/sample_images/virtualpc-dynamic.vhd.bz2 b/tests/qemu-iotests/sample_images/virtualpc-dynamic.vhd.bz2
new file mode 100644
index 0000000..783be3c
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/virtualpc-dynamic.vhd.bz2
Binary files differ
diff --git a/tests/qemu-iotests/socket_scm_helper.c b/tests/qemu-iotests/socket_scm_helper.c
index 8195983..80cadf4 100644
--- a/tests/qemu-iotests/socket_scm_helper.c
+++ b/tests/qemu-iotests/socket_scm_helper.c
@@ -10,15 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include <stdio.h>
-#include <errno.h>
+#include "qemu/osdep.h"
 #include <sys/socket.h>
 #include <sys/un.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
 
 /* #define SOCKET_SCM_DEBUG */
 
diff --git a/tests/qht-bench.c b/tests/qht-bench.c
new file mode 100644
index 0000000..76360a0
--- /dev/null
+++ b/tests/qht-bench.c
@@ -0,0 +1,487 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/processor.h"
+#include "qemu/atomic.h"
+#include "qemu/qht.h"
+#include "qemu/rcu.h"
+#include "exec/tb-hash-xx.h"
+
+struct thread_stats {
+    size_t rd;
+    size_t not_rd;
+    size_t in;
+    size_t not_in;
+    size_t rm;
+    size_t not_rm;
+    size_t rz;
+    size_t not_rz;
+};
+
+struct thread_info {
+    void (*func)(struct thread_info *);
+    struct thread_stats stats;
+    uint64_t r;
+    bool write_op; /* writes alternate between insertions and removals */
+    bool resize_down;
+} QEMU_ALIGNED(64); /* avoid false sharing among threads */
+
+static struct qht ht;
+static QemuThread *rw_threads;
+
+#define DEFAULT_RANGE (4096)
+#define DEFAULT_QHT_N_ELEMS DEFAULT_RANGE
+
+static unsigned int duration = 1;
+static unsigned int n_rw_threads = 1;
+static unsigned long lookup_range = DEFAULT_RANGE;
+static unsigned long update_range = DEFAULT_RANGE;
+static size_t init_range = DEFAULT_RANGE;
+static size_t init_size = DEFAULT_RANGE;
+static size_t n_ready_threads;
+static long populate_offset;
+static long *keys;
+
+static size_t resize_min;
+static size_t resize_max;
+static struct thread_info *rz_info;
+static unsigned long resize_delay = 1000;
+static double resize_rate; /* 0.0 to 1.0 */
+static unsigned int n_rz_threads = 1;
+static QemuThread *rz_threads;
+
+static double update_rate; /* 0.0 to 1.0 */
+static uint64_t update_threshold;
+static uint64_t resize_threshold;
+
+static size_t qht_n_elems = DEFAULT_QHT_N_ELEMS;
+static int qht_mode;
+
+static bool test_start;
+static bool test_stop;
+
+static struct thread_info *rw_info;
+
+static const char commands_string[] =
+    " -d = duration, in seconds\n"
+    " -n = number of threads\n"
+    "\n"
+    " -o = offset at which keys start\n"
+    "\n"
+    " -g = set -s,-k,-K,-l,-r to the same value\n"
+    " -s = initial size hint\n"
+    " -k = initial number of keys\n"
+    " -K = initial range of keys (will be rounded up to pow2)\n"
+    " -l = lookup range of keys (will be rounded up to pow2)\n"
+    " -r = update range of keys (will be rounded up to pow2)\n"
+    "\n"
+    " -u = update rate (0.0 to 100.0), 50/50 split of insertions/removals\n"
+    "\n"
+    " -R = enable auto-resize\n"
+    " -S = resize rate (0.0 to 100.0)\n"
+    " -D = delay (in us) between potential resizes\n"
+    " -N = number of resize threads";
+
+static void usage_complete(int argc, char *argv[])
+{
+    fprintf(stderr, "Usage: %s [options]\n", argv[0]);
+    fprintf(stderr, "options:\n%s\n", commands_string);
+    exit(-1);
+}
+
+static bool is_equal(const void *obj, const void *userp)
+{
+    const long *a = obj;
+    const long *b = userp;
+
+    return *a == *b;
+}
+
+static inline uint32_t h(unsigned long v)
+{
+    return tb_hash_func5(v, 0, 0);
+}
+
+/*
+ * From: https://en.wikipedia.org/wiki/Xorshift
+ * This is faster than rand_r(), and gives us a wider range (RAND_MAX is only
+ * guaranteed to be >= INT_MAX).
+ */
+static uint64_t xorshift64star(uint64_t x)
+{
+    x ^= x >> 12; /* a */
+    x ^= x << 25; /* b */
+    x ^= x >> 27; /* c */
+    return x * UINT64_C(2685821657736338717);
+}
+
+static void do_rz(struct thread_info *info)
+{
+    struct thread_stats *stats = &info->stats;
+
+    if (info->r < resize_threshold) {
+        size_t size = info->resize_down ? resize_min : resize_max;
+        bool resized;
+
+        resized = qht_resize(&ht, size);
+        info->resize_down = !info->resize_down;
+
+        if (resized) {
+            stats->rz++;
+        } else {
+            stats->not_rz++;
+        }
+    }
+    g_usleep(resize_delay);
+}
+
+static void do_rw(struct thread_info *info)
+{
+    struct thread_stats *stats = &info->stats;
+    uint32_t hash;
+    long *p;
+
+    if (info->r >= update_threshold) {
+        bool read;
+
+        p = &keys[info->r & (lookup_range - 1)];
+        hash = h(*p);
+        read = qht_lookup(&ht, is_equal, p, hash);
+        if (read) {
+            stats->rd++;
+        } else {
+            stats->not_rd++;
+        }
+    } else {
+        p = &keys[info->r & (update_range - 1)];
+        hash = h(*p);
+        if (info->write_op) {
+            bool written = false;
+
+            if (qht_lookup(&ht, is_equal, p, hash) == NULL) {
+                written = qht_insert(&ht, p, hash);
+            }
+            if (written) {
+                stats->in++;
+            } else {
+                stats->not_in++;
+            }
+        } else {
+            bool removed = false;
+
+            if (qht_lookup(&ht, is_equal, p, hash)) {
+                removed = qht_remove(&ht, p, hash);
+            }
+            if (removed) {
+                stats->rm++;
+            } else {
+                stats->not_rm++;
+            }
+        }
+        info->write_op = !info->write_op;
+    }
+}
+
+static void *thread_func(void *p)
+{
+    struct thread_info *info = p;
+
+    rcu_register_thread();
+
+    atomic_inc(&n_ready_threads);
+    while (!atomic_mb_read(&test_start)) {
+        cpu_relax();
+    }
+
+    rcu_read_lock();
+    while (!atomic_read(&test_stop)) {
+        info->r = xorshift64star(info->r);
+        info->func(info);
+    }
+    rcu_read_unlock();
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+/* sets everything except info->func */
+static void prepare_thread_info(struct thread_info *info, int i)
+{
+    /* seed for the RNG; each thread should have a different one */
+    info->r = (i + 1) ^ time(NULL);
+    /* the first update will be a write */
+    info->write_op = true;
+    /* the first resize will be down */
+    info->resize_down = true;
+
+    memset(&info->stats, 0, sizeof(info->stats));
+}
+
+static void
+th_create_n(QemuThread **threads, struct thread_info **infos, const char *name,
+            void (*func)(struct thread_info *), int offset, int n)
+{
+    struct thread_info *info;
+    QemuThread *th;
+    int i;
+
+    th = g_malloc(sizeof(*th) * n);
+    *threads = th;
+
+    info = qemu_memalign(64, sizeof(*info) * n);
+    *infos = info;
+
+    for (i = 0; i < n; i++) {
+        prepare_thread_info(&info[i], offset + i);
+        info[i].func = func;
+        qemu_thread_create(&th[i], name, thread_func, &info[i],
+                           QEMU_THREAD_JOINABLE);
+    }
+}
+
+static void create_threads(void)
+{
+    th_create_n(&rw_threads, &rw_info, "rw", do_rw, 0, n_rw_threads);
+    th_create_n(&rz_threads, &rz_info, "rz", do_rz, n_rw_threads, n_rz_threads);
+}
+
+static void pr_params(void)
+{
+    printf("Parameters:\n");
+    printf(" duration:          %d s\n", duration);
+    printf(" # of threads:      %u\n", n_rw_threads);
+    printf(" initial # of keys: %zu\n", init_size);
+    printf(" initial size hint: %zu\n", qht_n_elems);
+    printf(" auto-resize:       %s\n",
+           qht_mode & QHT_MODE_AUTO_RESIZE ? "on" : "off");
+    if (resize_rate) {
+        printf(" resize_rate:       %f%%\n", resize_rate * 100.0);
+        printf(" resize range:      %zu-%zu\n", resize_min, resize_max);
+        printf(" # resize threads   %u\n", n_rz_threads);
+    }
+    printf(" update rate:       %f%%\n", update_rate * 100.0);
+    printf(" offset:            %ld\n", populate_offset);
+    printf(" initial key range: %zu\n", init_range);
+    printf(" lookup range:      %lu\n", lookup_range);
+    printf(" update range:      %lu\n", update_range);
+}
+
+static void do_threshold(double rate, uint64_t *threshold)
+{
+    if (rate == 1.0) {
+        *threshold = UINT64_MAX;
+    } else {
+        *threshold = rate * UINT64_MAX;
+    }
+}
+
+static void htable_init(void)
+{
+    unsigned long n = MAX(init_range, update_range);
+    uint64_t r = time(NULL);
+    size_t retries = 0;
+    size_t i;
+
+    /* avoid allocating memory later by allocating all the keys now */
+    keys = g_malloc(sizeof(*keys) * n);
+    for (i = 0; i < n; i++) {
+        keys[i] = populate_offset + i;
+    }
+
+    /* some sanity checks */
+    g_assert_cmpuint(lookup_range, <=, n);
+
+    /* compute thresholds */
+    do_threshold(update_rate, &update_threshold);
+    do_threshold(resize_rate, &resize_threshold);
+
+    if (resize_rate) {
+        resize_min = n / 2;
+        resize_max = n;
+        assert(resize_min < resize_max);
+    } else {
+        n_rz_threads = 0;
+    }
+
+    /* initialize the hash table */
+    qht_init(&ht, qht_n_elems, qht_mode);
+    assert(init_size <= init_range);
+
+    pr_params();
+
+    fprintf(stderr, "Initialization: populating %zu items...", init_size);
+    for (i = 0; i < init_size; i++) {
+        for (;;) {
+            uint32_t hash;
+            long *p;
+
+            r = xorshift64star(r);
+            p = &keys[r & (init_range - 1)];
+            hash = h(*p);
+            if (qht_insert(&ht, p, hash)) {
+                break;
+            }
+            retries++;
+        }
+    }
+    fprintf(stderr, " populated after %zu retries\n", retries);
+}
+
+static void add_stats(struct thread_stats *s, struct thread_info *info, int n)
+{
+    int i;
+
+    for (i = 0; i < n; i++) {
+        struct thread_stats *stats = &info[i].stats;
+
+        s->rd += stats->rd;
+        s->not_rd += stats->not_rd;
+
+        s->in += stats->in;
+        s->not_in += stats->not_in;
+
+        s->rm += stats->rm;
+        s->not_rm += stats->not_rm;
+
+        s->rz += stats->rz;
+        s->not_rz += stats->not_rz;
+    }
+}
+
+static void pr_stats(void)
+{
+    struct thread_stats s = {};
+    double tx;
+
+    add_stats(&s, rw_info, n_rw_threads);
+    add_stats(&s, rz_info, n_rz_threads);
+
+    printf("Results:\n");
+
+    if (resize_rate) {
+        printf(" Resizes:           %zu (%.2f%% of %zu)\n",
+               s.rz, (double)s.rz / (s.rz + s.not_rz) * 100, s.rz + s.not_rz);
+    }
+
+    printf(" Read:              %.2f M (%.2f%% of %.2fM)\n",
+           (double)s.rd / 1e6,
+           (double)s.rd / (s.rd + s.not_rd) * 100,
+           (double)(s.rd + s.not_rd) / 1e6);
+    printf(" Inserted:          %.2f M (%.2f%% of %.2fM)\n",
+           (double)s.in / 1e6,
+           (double)s.in / (s.in + s.not_in) * 100,
+           (double)(s.in + s.not_in) / 1e6);
+    printf(" Removed:           %.2f M (%.2f%% of %.2fM)\n",
+           (double)s.rm / 1e6,
+           (double)s.rm / (s.rm + s.not_rm) * 100,
+           (double)(s.rm + s.not_rm) / 1e6);
+
+    tx = (s.rd + s.not_rd + s.in + s.not_in + s.rm + s.not_rm) / 1e6 / duration;
+    printf(" Throughput:        %.2f MT/s\n", tx);
+    printf(" Throughput/thread: %.2f MT/s/thread\n", tx / n_rw_threads);
+}
+
+static void run_test(void)
+{
+    unsigned int remaining;
+    int i;
+
+    while (atomic_read(&n_ready_threads) != n_rw_threads + n_rz_threads) {
+        cpu_relax();
+    }
+    atomic_mb_set(&test_start, true);
+    do {
+        remaining = sleep(duration);
+    } while (remaining);
+    atomic_mb_set(&test_stop, true);
+
+    for (i = 0; i < n_rw_threads; i++) {
+        qemu_thread_join(&rw_threads[i]);
+    }
+    for (i = 0; i < n_rz_threads; i++) {
+        qemu_thread_join(&rz_threads[i]);
+    }
+}
+
+static void parse_args(int argc, char *argv[])
+{
+    int c;
+
+    for (;;) {
+        c = getopt(argc, argv, "d:D:g:k:K:l:hn:N:o:r:Rs:S:u:");
+        if (c < 0) {
+            break;
+        }
+        switch (c) {
+        case 'd':
+            duration = atoi(optarg);
+            break;
+        case 'D':
+            resize_delay = atol(optarg);
+            break;
+        case 'g':
+            init_range = pow2ceil(atol(optarg));
+            lookup_range = pow2ceil(atol(optarg));
+            update_range = pow2ceil(atol(optarg));
+            qht_n_elems = atol(optarg);
+            init_size = atol(optarg);
+            break;
+        case 'h':
+            usage_complete(argc, argv);
+            exit(0);
+        case 'k':
+            init_size = atol(optarg);
+            break;
+        case 'K':
+            init_range = pow2ceil(atol(optarg));
+            break;
+        case 'l':
+            lookup_range = pow2ceil(atol(optarg));
+            break;
+        case 'n':
+            n_rw_threads = atoi(optarg);
+            break;
+        case 'N':
+            n_rz_threads = atoi(optarg);
+            break;
+        case 'o':
+            populate_offset = atol(optarg);
+            break;
+        case 'r':
+            update_range = pow2ceil(atol(optarg));
+            break;
+        case 'R':
+            qht_mode |= QHT_MODE_AUTO_RESIZE;
+            break;
+        case 's':
+            qht_n_elems = atol(optarg);
+            break;
+        case 'S':
+            resize_rate = atof(optarg) / 100.0;
+            if (resize_rate > 1.0) {
+                resize_rate = 1.0;
+            }
+            break;
+        case 'u':
+            update_rate = atof(optarg) / 100.0;
+            if (update_rate > 1.0) {
+                update_rate = 1.0;
+            }
+            break;
+        }
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    parse_args(argc, argv);
+    htable_init();
+    create_threads();
+    run_test();
+    pr_stats();
+    return 0;
+}
diff --git a/tests/qom-test.c b/tests/qom-test.c
index 4246382..23493a2 100644
--- a/tests/qom-test.c
+++ b/tests/qom-test.c
@@ -7,12 +7,11 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "libqtest.h"
-#include "qemu/osdep.h"
 #include "qapi/qmp/types.h"
 
 static const char *blacklist_x86[] = {
@@ -47,7 +46,7 @@
 static void test_properties(const char *path, bool recurse)
 {
     char *child_path;
-    QDict *response, *tuple;
+    QDict *response, *tuple, *tmp;
     QList *list;
     QListEntry *entry;
 
@@ -57,6 +56,7 @@
     g_assert(response);
 
     if (!recurse) {
+        QDECREF(response);
         return;
     }
 
@@ -75,14 +75,16 @@
         } else {
             const char *prop = qdict_get_str(tuple, "name");
             g_test_message("Testing property %s.%s", path, prop);
-            response = qmp("{ 'execute': 'qom-get',"
-                           "  'arguments': { 'path': %s,"
-                           "                 'property': %s } }",
-                           path, prop);
+            tmp = qmp("{ 'execute': 'qom-get',"
+                      "  'arguments': { 'path': %s,"
+                      "                 'property': %s } }",
+                      path, prop);
             /* qom-get may fail but should not, e.g., segfault. */
-            g_assert(response);
+            g_assert(tmp);
+            QDECREF(tmp);
         }
     }
+    QDECREF(response);
 }
 
 static void test_machine(gconstpointer data)
@@ -98,9 +100,11 @@
 
     response = qmp("{ 'execute': 'quit' }");
     g_assert(qdict_haskey(response, "return"));
+    QDECREF(response);
 
     qtest_end();
     g_free(args);
+    g_free((void *)machine);
 }
 
 static void add_machine_test_cases(void)
@@ -128,11 +132,13 @@
         g_assert(qstr);
         mname = qstring_get_str(qstr);
         if (!is_blacklisted(arch, mname)) {
-            path = g_strdup_printf("/%s/qom/%s", arch, mname);
-            g_test_add_data_func(path, mname, test_machine);
+            path = g_strdup_printf("qom/%s", mname);
+            qtest_add_data_func(path, g_strdup(mname), test_machine);
         }
     }
+
     qtest_end();
+    QDECREF(response);
 }
 
 int main(int argc, char **argv)
diff --git a/tests/rcutorture.c b/tests/rcutorture.c
new file mode 100644
index 0000000..4002ecf
--- /dev/null
+++ b/tests/rcutorture.c
@@ -0,0 +1,459 @@
+/*
+ * rcutorture.c: simple user-level performance/stress test of RCU.
+ *
+ * Usage:
+ *     ./rcu <nreaders> rperf [ <seconds> ]
+ *         Run a read-side performance test with the specified
+ *         number of readers for <seconds> seconds.
+ *     ./rcu <nupdaters> uperf [ <seconds> ]
+ *         Run an update-side performance test with the specified
+ *         number of updaters and specified duration.
+ *     ./rcu <nreaders> perf [ <seconds> ]
+ *         Run a combined read/update performance test with the specified
+ *         number of readers and one updater and specified duration.
+ *
+ * The above tests produce output as follows:
+ *
+ * n_reads: 46008000  n_updates: 146026  nreaders: 2  nupdaters: 1 duration: 1
+ * ns/read: 43.4707  ns/update: 6848.1
+ *
+ * The first line lists the total number of RCU reads and updates executed
+ * during the test, the number of reader threads, the number of updater
+ * threads, and the duration of the test in seconds.  The second line
+ * lists the average duration of each type of operation in nanoseconds,
+ * or "nan" if the corresponding type of operation was not performed.
+ *
+ *     ./rcu <nreaders> stress [ <seconds> ]
+ *         Run a stress test with the specified number of readers and
+ *         one updater.
+ *
+ * This test produces output as follows:
+ *
+ * n_reads: 114633217  n_updates: 3903415  n_mberror: 0
+ * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
+ *
+ * The first line lists the number of RCU read and update operations
+ * executed, followed by the number of memory-ordering violations
+ * (which will be zero in a correct RCU implementation).  The second
+ * line lists the number of readers observing progressively more stale
+ * data.  A correct RCU implementation will have all but the first two
+ * numbers non-zero.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
+ */
+
+/*
+ * Test variables.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/atomic.h"
+#include "qemu/rcu.h"
+#include "qemu/thread.h"
+
+long long n_reads = 0LL;
+long n_updates = 0L;
+int nthreadsrunning;
+
+#define GOFLAG_INIT 0
+#define GOFLAG_RUN  1
+#define GOFLAG_STOP 2
+
+static volatile int goflag = GOFLAG_INIT;
+
+#define RCU_READ_RUN 1000
+
+#define NR_THREADS 100
+static QemuMutex counts_mutex;
+static QemuThread threads[NR_THREADS];
+static struct rcu_reader_data *data[NR_THREADS];
+static int n_threads;
+
+static void create_thread(void *(*func)(void *))
+{
+    if (n_threads >= NR_THREADS) {
+        fprintf(stderr, "Thread limit of %d exceeded!\n", NR_THREADS);
+        exit(-1);
+    }
+    qemu_thread_create(&threads[n_threads], "test", func, &data[n_threads],
+                       QEMU_THREAD_JOINABLE);
+    n_threads++;
+}
+
+static void wait_all_threads(void)
+{
+    int i;
+
+    for (i = 0; i < n_threads; i++) {
+        qemu_thread_join(&threads[i]);
+    }
+    n_threads = 0;
+}
+
+/*
+ * Performance test.
+ */
+
+static void *rcu_read_perf_test(void *arg)
+{
+    int i;
+    long long n_reads_local = 0;
+
+    rcu_register_thread();
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    atomic_inc(&nthreadsrunning);
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+    while (goflag == GOFLAG_RUN) {
+        for (i = 0; i < RCU_READ_RUN; i++) {
+            rcu_read_lock();
+            rcu_read_unlock();
+        }
+        n_reads_local += RCU_READ_RUN;
+    }
+    qemu_mutex_lock(&counts_mutex);
+    n_reads += n_reads_local;
+    qemu_mutex_unlock(&counts_mutex);
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+static void *rcu_update_perf_test(void *arg)
+{
+    long long n_updates_local = 0;
+
+    rcu_register_thread();
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    atomic_inc(&nthreadsrunning);
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+    while (goflag == GOFLAG_RUN) {
+        synchronize_rcu();
+        n_updates_local++;
+    }
+    qemu_mutex_lock(&counts_mutex);
+    n_updates += n_updates_local;
+    qemu_mutex_unlock(&counts_mutex);
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+static void perftestinit(void)
+{
+    nthreadsrunning = 0;
+}
+
+static void perftestrun(int nthreads, int duration, int nreaders, int nupdaters)
+{
+    while (atomic_read(&nthreadsrunning) < nthreads) {
+        g_usleep(1000);
+    }
+    goflag = GOFLAG_RUN;
+    g_usleep(duration * G_USEC_PER_SEC);
+    goflag = GOFLAG_STOP;
+    wait_all_threads();
+    printf("n_reads: %lld  n_updates: %ld  nreaders: %d  nupdaters: %d duration: %d\n",
+           n_reads, n_updates, nreaders, nupdaters, duration);
+    printf("ns/read: %g  ns/update: %g\n",
+           ((duration * 1000*1000*1000.*(double)nreaders) /
+        (double)n_reads),
+           ((duration * 1000*1000*1000.*(double)nupdaters) /
+        (double)n_updates));
+    exit(0);
+}
+
+static void perftest(int nreaders, int duration)
+{
+    int i;
+
+    perftestinit();
+    for (i = 0; i < nreaders; i++) {
+        create_thread(rcu_read_perf_test);
+    }
+    create_thread(rcu_update_perf_test);
+    perftestrun(i + 1, duration, nreaders, 1);
+}
+
+static void rperftest(int nreaders, int duration)
+{
+    int i;
+
+    perftestinit();
+    for (i = 0; i < nreaders; i++) {
+        create_thread(rcu_read_perf_test);
+    }
+    perftestrun(i, duration, nreaders, 0);
+}
+
+static void uperftest(int nupdaters, int duration)
+{
+    int i;
+
+    perftestinit();
+    for (i = 0; i < nupdaters; i++) {
+        create_thread(rcu_update_perf_test);
+    }
+    perftestrun(i, duration, 0, nupdaters);
+}
+
+/*
+ * Stress test.
+ */
+
+#define RCU_STRESS_PIPE_LEN 10
+
+struct rcu_stress {
+    int pipe_count;
+    int mbtest;
+};
+
+struct rcu_stress rcu_stress_array[RCU_STRESS_PIPE_LEN] = { { 0 } };
+struct rcu_stress *rcu_stress_current;
+int rcu_stress_idx;
+
+int n_mberror;
+long long rcu_stress_count[RCU_STRESS_PIPE_LEN + 1];
+
+
+static void *rcu_read_stress_test(void *arg)
+{
+    int i;
+    int itercnt = 0;
+    struct rcu_stress *p;
+    int pc;
+    long long n_reads_local = 0;
+    long long rcu_stress_local[RCU_STRESS_PIPE_LEN + 1] = { 0 };
+    volatile int garbage = 0;
+
+    rcu_register_thread();
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+    while (goflag == GOFLAG_RUN) {
+        rcu_read_lock();
+        p = atomic_rcu_read(&rcu_stress_current);
+        if (p->mbtest == 0) {
+            n_mberror++;
+        }
+        rcu_read_lock();
+        for (i = 0; i < 100; i++) {
+            garbage++;
+        }
+        rcu_read_unlock();
+        pc = p->pipe_count;
+        rcu_read_unlock();
+        if ((pc > RCU_STRESS_PIPE_LEN) || (pc < 0)) {
+            pc = RCU_STRESS_PIPE_LEN;
+        }
+        rcu_stress_local[pc]++;
+        n_reads_local++;
+        if ((++itercnt % 0x1000) == 0) {
+            synchronize_rcu();
+        }
+    }
+    qemu_mutex_lock(&counts_mutex);
+    n_reads += n_reads_local;
+    for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
+        rcu_stress_count[i] += rcu_stress_local[i];
+    }
+    qemu_mutex_unlock(&counts_mutex);
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+static void *rcu_update_stress_test(void *arg)
+{
+    int i;
+    struct rcu_stress *p;
+
+    rcu_register_thread();
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+    while (goflag == GOFLAG_RUN) {
+        i = rcu_stress_idx + 1;
+        if (i >= RCU_STRESS_PIPE_LEN) {
+            i = 0;
+        }
+        p = &rcu_stress_array[i];
+        p->mbtest = 0;
+        smp_mb();
+        p->pipe_count = 0;
+        p->mbtest = 1;
+        atomic_rcu_set(&rcu_stress_current, p);
+        rcu_stress_idx = i;
+        for (i = 0; i < RCU_STRESS_PIPE_LEN; i++) {
+            if (i != rcu_stress_idx) {
+                rcu_stress_array[i].pipe_count++;
+            }
+        }
+        synchronize_rcu();
+        n_updates++;
+    }
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+static void *rcu_fake_update_stress_test(void *arg)
+{
+    rcu_register_thread();
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+    while (goflag == GOFLAG_RUN) {
+        synchronize_rcu();
+        g_usleep(1000);
+    }
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+static void stresstest(int nreaders, int duration)
+{
+    int i;
+
+    rcu_stress_current = &rcu_stress_array[0];
+    rcu_stress_current->pipe_count = 0;
+    rcu_stress_current->mbtest = 1;
+    for (i = 0; i < nreaders; i++) {
+        create_thread(rcu_read_stress_test);
+    }
+    create_thread(rcu_update_stress_test);
+    for (i = 0; i < 5; i++) {
+        create_thread(rcu_fake_update_stress_test);
+    }
+    goflag = GOFLAG_RUN;
+    g_usleep(duration * G_USEC_PER_SEC);
+    goflag = GOFLAG_STOP;
+    wait_all_threads();
+    printf("n_reads: %lld  n_updates: %ld  n_mberror: %d\n",
+           n_reads, n_updates, n_mberror);
+    printf("rcu_stress_count:");
+    for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) {
+        printf(" %lld", rcu_stress_count[i]);
+    }
+    printf("\n");
+    exit(0);
+}
+
+/* GTest interface */
+
+static void gtest_stress(int nreaders, int duration)
+{
+    int i;
+
+    rcu_stress_current = &rcu_stress_array[0];
+    rcu_stress_current->pipe_count = 0;
+    rcu_stress_current->mbtest = 1;
+    for (i = 0; i < nreaders; i++) {
+        create_thread(rcu_read_stress_test);
+    }
+    create_thread(rcu_update_stress_test);
+    for (i = 0; i < 5; i++) {
+        create_thread(rcu_fake_update_stress_test);
+    }
+    goflag = GOFLAG_RUN;
+    g_usleep(duration * G_USEC_PER_SEC);
+    goflag = GOFLAG_STOP;
+    wait_all_threads();
+    g_assert_cmpint(n_mberror, ==, 0);
+    for (i = 2; i <= RCU_STRESS_PIPE_LEN; i++) {
+        g_assert_cmpint(rcu_stress_count[i], ==, 0);
+    }
+}
+
+static void gtest_stress_1_1(void)
+{
+    gtest_stress(1, 1);
+}
+
+static void gtest_stress_10_1(void)
+{
+    gtest_stress(10, 1);
+}
+
+static void gtest_stress_1_5(void)
+{
+    gtest_stress(1, 5);
+}
+
+static void gtest_stress_10_5(void)
+{
+    gtest_stress(10, 5);
+}
+
+/*
+ * Mainprogram.
+ */
+
+static void usage(int argc, char *argv[])
+{
+    fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]);
+    exit(-1);
+}
+
+int main(int argc, char *argv[])
+{
+    int nreaders = 1;
+    int duration = 1;
+
+    qemu_mutex_init(&counts_mutex);
+    if (argc >= 2 && argv[1][0] == '-') {
+        g_test_init(&argc, &argv, NULL);
+        if (g_test_quick()) {
+            g_test_add_func("/rcu/torture/1reader", gtest_stress_1_1);
+            g_test_add_func("/rcu/torture/10readers", gtest_stress_10_1);
+        } else {
+            g_test_add_func("/rcu/torture/1reader", gtest_stress_1_5);
+            g_test_add_func("/rcu/torture/10readers", gtest_stress_10_5);
+        }
+        return g_test_run();
+    }
+
+    if (argc >= 2) {
+        nreaders = strtoul(argv[1], NULL, 0);
+    }
+    if (argc > 3) {
+        duration = strtoul(argv[3], NULL, 0);
+    }
+    if (argc < 3 || strcmp(argv[2], "stress") == 0) {
+        stresstest(nreaders, duration);
+    } else if (strcmp(argv[2], "rperf") == 0) {
+        rperftest(nreaders, duration);
+    } else if (strcmp(argv[2], "uperf") == 0) {
+        uperftest(nreaders, duration);
+    } else if (strcmp(argv[2], "perf") == 0) {
+        perftest(nreaders, duration);
+    }
+    usage(argc, argv);
+    return 0;
+}
diff --git a/tests/rocker/README b/tests/rocker/README
new file mode 100644
index 0000000..531e673
--- /dev/null
+++ b/tests/rocker/README
@@ -0,0 +1,5 @@
+Tests require simp (simple network simulator) found here:
+
+https://github.com/scottfeldman/simp
+
+Run 'all' to run all tests.
diff --git a/tests/rocker/all b/tests/rocker/all
new file mode 100755
index 0000000..d5ae963
--- /dev/null
+++ b/tests/rocker/all
@@ -0,0 +1,19 @@
+echo -n "Running port test...              "
+./port
+if [ $? -eq 0 ]; then echo "pass"; else echo "FAILED"; exit 1; fi
+
+echo -n "Running bridge test...            "
+./bridge
+if [ $? -eq 0 ]; then echo "pass"; else echo "FAILED"; exit 1; fi
+
+echo -n "Running bridge STP test...        "
+./bridge-stp
+if [ $? -eq 0 ]; then echo "pass"; else echo "FAILED"; exit 1; fi
+
+echo -n "Running bridge VLAN test...       "
+./bridge-vlan
+if [ $? -eq 0 ]; then echo "pass"; else echo "FAILED"; exit 1; fi
+
+echo -n "Running bridge VLAN STP test...   "
+./bridge-vlan-stp
+if [ $? -eq 0 ]; then echo "pass"; else echo "FAILED"; exit 1; fi
diff --git a/tests/rocker/bridge b/tests/rocker/bridge
new file mode 100755
index 0000000..46abc6f
--- /dev/null
+++ b/tests/rocker/bridge
@@ -0,0 +1,43 @@
+simp destroy ".*"
+simp create -o sw1:rocker:sw1 tut tut.dot
+simp start tut
+sleep 10
+while ! simp ssh tut sw1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h2 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+
+# configure a 2-port bridge
+
+simp ssh tut sw1 --cmd "sudo /sbin/ip link add name br0 type bridge"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p1 master br0"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p2 master br0"
+
+# turn off vlan default_pvid on br0
+
+simp ssh tut sw1 --cmd "echo 0 | sudo dd of=/sys/class/net/br0/bridge/default_pvid 2> /dev/null"
+
+# turn off learning and flooding in SW
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 learning off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 learning off"
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 flood off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 flood off"
+
+# bring up bridge and ports
+
+simp ssh tut sw1 --cmd "sudo ifconfig br0 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p2 up"
+simp ssh tut sw1 --cmd "sudo ifconfig br0 11.0.0.3/24"
+
+# config IP on hosts
+
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1 11.0.0.1/24"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1 11.0.0.2/24"
+
+# test...
+
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+if [ $? -ne 0 ]; then exit 1; fi
+simp ssh tut h1 --cmd "ping -c10 11.0.0.3 >/dev/null"
diff --git a/tests/rocker/bridge-stp b/tests/rocker/bridge-stp
new file mode 100755
index 0000000..008568a
--- /dev/null
+++ b/tests/rocker/bridge-stp
@@ -0,0 +1,52 @@
+simp destroy ".*"
+simp create -o sw1:rocker:sw1 tut tut.dot
+simp start tut
+sleep 10
+while ! simp ssh tut sw1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h2 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+
+# configure a 2-port bridge
+
+simp ssh tut sw1 --cmd "sudo /sbin/ip link add name br0 type bridge"
+simp ssh tut sw1 --cmd "sudo brctl stp br0 on"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p1 master br0"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p2 master br0"
+
+# turn off vlan default_pvid on br0
+
+simp ssh tut sw1 --cmd "echo 0 | sudo dd of=/sys/class/net/br0/bridge/default_pvid 2> /dev/null"
+
+# turn off learning and flooding in SW
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 learning off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 learning off"
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 flood off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 flood off"
+
+# config IP on hosts
+
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1 11.0.0.1/24"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1 11.0.0.2/24"
+
+# bring up bridge and ports
+
+simp ssh tut sw1 --cmd "sudo ifconfig br0 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p2 up"
+
+# test...
+
+simp ssh tut h1 --cmd "ping -w 1 -c1 11.0.0.2 >/dev/null"
+if [ $? -eq 0 ]; then exit 1; fi
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
diff --git a/tests/rocker/bridge-vlan b/tests/rocker/bridge-vlan
new file mode 100755
index 0000000..897d82c
--- /dev/null
+++ b/tests/rocker/bridge-vlan
@@ -0,0 +1,52 @@
+simp destroy ".*"
+simp create -o sw1:rocker:sw1 tut tut.dot
+simp start tut
+sleep 10
+while ! simp ssh tut sw1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h2 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+
+# configure a 2-port bridge
+
+simp ssh tut sw1 --cmd "sudo /sbin/ip link add name br0 type bridge"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p1 master br0"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p2 master br0"
+
+# turn off vlan default_pvid on br0
+# turn on vlan filtering on br0
+
+simp ssh tut sw1 --cmd "echo 0 | sudo dd of=/sys/class/net/br0/bridge/default_pvid 2> /dev/null"
+simp ssh tut sw1 --cmd "echo 1 | sudo dd of=/sys/class/net/br0/bridge/vlan_filtering 2> /dev/null"
+
+# add both ports to VLAN 57
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge vlan add vid 57 dev sw1p1"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge vlan add vid 57 dev sw1p2"
+
+# turn off learning and flooding in SW
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 learning off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 learning off"
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 flood off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 flood off"
+
+# bring up bridge and ports
+
+simp ssh tut sw1 --cmd "sudo ifconfig br0 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p2 up"
+
+# config IP on host VLANs
+
+simp ssh tut h1 --cmd "sudo vconfig add sw1p1 57 >/dev/null 2>&1"
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1.57 11.0.0.1/24"
+
+simp ssh tut h2 --cmd "sudo vconfig add sw1p1 57 >/dev/null 2>&1"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1.57 11.0.0.2/24"
+
+# test...
+
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
diff --git a/tests/rocker/bridge-vlan-stp b/tests/rocker/bridge-vlan-stp
new file mode 100755
index 0000000..85d2646
--- /dev/null
+++ b/tests/rocker/bridge-vlan-stp
@@ -0,0 +1,64 @@
+simp destroy ".*"
+simp create -o sw1:rocker:sw1 tut tut.dot
+simp start tut
+sleep 10
+while ! simp ssh tut sw1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h2 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+
+# configure a 2-port bridge
+
+simp ssh tut sw1 --cmd "sudo /sbin/ip link add name br0 type bridge"
+simp ssh tut sw1 --cmd "sudo brctl stp br0 on"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p1 master br0"
+simp ssh tut sw1 --cmd "sudo /sbin/ip link set dev sw1p2 master br0"
+
+# turn off vlan default_pvid on br0
+# turn on vlan filtering on br0
+
+simp ssh tut sw1 --cmd "echo 0 | sudo dd of=/sys/class/net/br0/bridge/default_pvid 2> /dev/null"
+simp ssh tut sw1 --cmd "echo 1 | sudo dd of=/sys/class/net/br0/bridge/vlan_filtering 2> /dev/null"
+
+# add both ports to VLAN 57
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge vlan add vid 57 dev sw1p1"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge vlan add vid 57 dev sw1p2"
+
+# turn off learning and flooding in SW
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 learning off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 learning off"
+
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p1 flood off"
+simp ssh tut sw1 --cmd "sudo /sbin/bridge link set dev sw1p2 flood off"
+
+# config IP on host VLANs
+
+simp ssh tut h1 --cmd "sudo vconfig add sw1p1 57 >/dev/null 2>&1"
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1.57 11.0.0.1/24"
+
+simp ssh tut h2 --cmd "sudo vconfig add sw1p1 57 >/dev/null 2>&1"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1.57 11.0.0.2/24"
+
+# bring up bridge and ports
+
+simp ssh tut sw1 --cmd "sudo ifconfig br0 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p1 up"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p2 up"
+
+# test...
+
+simp ssh tut h1 --cmd "ping -w 1 -c1 11.0.0.2 >/dev/null"
+if [ $? -eq 0 ]; then exit 1; fi
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
+sleep 10
+simp ssh tut h1 --cmd "ping -c10 11.0.0.2 >/dev/null"
diff --git a/tests/rocker/port b/tests/rocker/port
new file mode 100755
index 0000000..5f2c248
--- /dev/null
+++ b/tests/rocker/port
@@ -0,0 +1,22 @@
+simp destroy ".*"
+simp create -o sw1:rocker:sw1 tut tut.dot
+simp start tut
+while ! simp ssh tut sw1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h1 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+while ! simp ssh tut h2 --cmd "ping -c 1 localhost >/dev/null"; do sleep 1; done
+
+# bring up DUT ports
+
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p1 11.0.0.1/24"
+simp ssh tut sw1 --cmd "sudo ifconfig sw1p2 12.0.0.1/24"
+
+# config IP on hosts
+
+simp ssh tut h1 --cmd "sudo ifconfig sw1p1 11.0.0.2/24"
+simp ssh tut h2 --cmd "sudo ifconfig sw1p1 12.0.0.2/24"
+
+# test...
+
+simp ssh tut h1 --cmd "ping -c10 11.0.0.1 >/dev/null"
+if [ $? -eq 1 ]; then exit 1; fi
+simp ssh tut h2 --cmd "ping -c10 12.0.0.1 >/dev/null"
diff --git a/tests/rocker/tut.dot b/tests/rocker/tut.dot
new file mode 100644
index 0000000..87f7266
--- /dev/null
+++ b/tests/rocker/tut.dot
@@ -0,0 +1,8 @@
+graph G {
+	graph [hostidtype="hostname", version="1:0", date="04/12/2013"];
+	edge [dir=none, notify="log"];
+	sw1:swp1 -- h1:swp1;
+	sw1:swp2 -- h2:swp1;
+	sw1:swp3 -- h3:swp1;
+	sw1:swp4 -- h4:swp1;
+}
diff --git a/tests/rtc-test.c b/tests/rtc-test.c
index 4243624..a086efd 100644
--- a/tests/rtc-test.c
+++ b/tests/rtc-test.c
@@ -11,11 +11,7 @@
  *
  */
 
-#include <glib.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 #include "hw/timer/mc146818rtc_regs.h"
diff --git a/tests/rtl8139-test.c b/tests/rtl8139-test.c
index f6a1be3..13de7ee 100644
--- a/tests/rtl8139-test.c
+++ b/tests/rtl8139-test.c
@@ -7,22 +7,201 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
+#include "libqos/pci-pc.h"
+#include "qemu/timer.h"
+#include "qemu-common.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
 {
 }
 
+#define CLK 33333333
+
+static QPCIBus *pcibus;
+static QPCIDevice *dev;
+static void *dev_base;
+
+static void save_fn(QPCIDevice *dev, int devfn, void *data)
+{
+    QPCIDevice **pdev = (QPCIDevice **) data;
+
+    *pdev = dev;
+}
+
+static QPCIDevice *get_device(void)
+{
+    QPCIDevice *dev;
+
+    pcibus = qpci_init_pc();
+    qpci_device_foreach(pcibus, 0x10ec, 0x8139, save_fn, &dev);
+    g_assert(dev != NULL);
+
+    return dev;
+}
+
+#define PORT(name, len, val) \
+static unsigned __attribute__((unused)) in_##name(void) \
+{ \
+    unsigned res = qpci_io_read##len(dev, dev_base+(val)); \
+    g_test_message("*%s -> %x\n", #name, res); \
+    return res; \
+} \
+static void out_##name(unsigned v) \
+{ \
+    g_test_message("%x -> *%s\n", v, #name); \
+    qpci_io_write##len(dev, dev_base+(val), v); \
+}
+
+PORT(Timer, l, 0x48)
+PORT(IntrMask, w, 0x3c)
+PORT(IntrStatus, w, 0x3E)
+PORT(TimerInt, l, 0x54)
+
+#define fatal(...) do { g_test_message(__VA_ARGS__); g_assert(0); } while (0)
+
+static void test_timer(void)
+{
+    const unsigned from = 0.95 * CLK;
+    const unsigned to = 1.6 * CLK;
+    unsigned prev, curr, next;
+    unsigned cnt, diff;
+
+    out_IntrMask(0);
+
+    in_IntrStatus();
+    in_Timer();
+    in_Timer();
+
+    /* Test 1. test counter continue and continue */
+    out_TimerInt(0); /* disable timer */
+    out_IntrStatus(0x4000);
+    out_Timer(12345); /* reset timer to 0 */
+    curr = in_Timer();
+    if (curr > 0.1 * CLK) {
+        fatal("time too big %u\n", curr);
+    }
+    for (cnt = 0; ; ) {
+        clock_step(1 * NANOSECONDS_PER_SECOND);
+        prev = curr;
+        curr = in_Timer();
+
+        /* test skip is in a specific range */
+        diff = (curr-prev) & 0xffffffffu;
+        if (diff < from || diff > to) {
+            fatal("Invalid diff %u (%u-%u)\n", diff, from, to);
+        }
+        if (curr < prev && ++cnt == 3) {
+            break;
+        }
+    }
+
+    /* Test 2. Check we didn't get an interrupt with TimerInt == 0 */
+    if (in_IntrStatus() & 0x4000) {
+        fatal("got an interrupt\n");
+    }
+
+    /* Test 3. Setting TimerInt to 1 and Timer to 0 get interrupt */
+    out_TimerInt(1);
+    out_Timer(0);
+    clock_step(40);
+    if ((in_IntrStatus() & 0x4000) == 0) {
+        fatal("we should have an interrupt here!\n");
+    }
+
+    /* Test 3. Check acknowledge */
+    out_IntrStatus(0x4000);
+    if (in_IntrStatus() & 0x4000) {
+        fatal("got an interrupt\n");
+    }
+
+    /* Test. Status set after Timer reset */
+    out_Timer(0);
+    out_TimerInt(0);
+    out_IntrStatus(0x4000);
+    curr = in_Timer();
+    out_TimerInt(curr + 0.5 * CLK);
+    clock_step(1 * NANOSECONDS_PER_SECOND);
+    out_Timer(0);
+    if ((in_IntrStatus() & 0x4000) == 0) {
+        fatal("we should have an interrupt here!\n");
+    }
+
+    /* Test. Status set after TimerInt reset */
+    out_Timer(0);
+    out_TimerInt(0);
+    out_IntrStatus(0x4000);
+    curr = in_Timer();
+    out_TimerInt(curr + 0.5 * CLK);
+    clock_step(1 * NANOSECONDS_PER_SECOND);
+    out_TimerInt(0);
+    if ((in_IntrStatus() & 0x4000) == 0) {
+        fatal("we should have an interrupt here!\n");
+    }
+
+    /* Test 4. Increment TimerInt we should see an interrupt */
+    curr = in_Timer();
+    next = curr + 5.0 * CLK;
+    out_TimerInt(next);
+    for (cnt = 0; ; ) {
+        clock_step(1 * NANOSECONDS_PER_SECOND);
+        prev = curr;
+        curr = in_Timer();
+        diff = (curr-prev) & 0xffffffffu;
+        if (diff < from || diff > to) {
+            fatal("Invalid diff %u (%u-%u)\n", diff, from, to);
+        }
+        if (cnt < 3 && curr > next) {
+            if ((in_IntrStatus() & 0x4000) == 0) {
+                fatal("we should have an interrupt here!\n");
+            }
+            out_IntrStatus(0x4000);
+            next = curr + 5.0 * CLK;
+            out_TimerInt(next);
+            if (++cnt == 3) {
+                out_TimerInt(1);
+            }
+        /* Test 5. Second time we pass from 0 should see an interrupt */
+        } else if (cnt >= 3 && curr < prev) {
+            /* here we should have an interrupt */
+            if ((in_IntrStatus() & 0x4000) == 0) {
+                fatal("we should have an interrupt here!\n");
+            }
+            out_IntrStatus(0x4000);
+            if (++cnt == 5) {
+                break;
+            }
+        }
+    }
+
+    g_test_message("Everythink is ok!\n");
+}
+
+
+static void test_init(void)
+{
+    uint64_t barsize;
+
+    dev = get_device();
+
+    dev_base = qpci_iomap(dev, 0, &barsize);
+
+    g_assert(dev_base != NULL);
+
+    qpci_device_enable(dev);
+
+    test_timer();
+}
+
 int main(int argc, char **argv)
 {
     int ret;
 
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/rtl8139/nop", nop);
+    qtest_add_func("/rtl8139/timer", test_init);
 
     qtest_start("-device rtl8139");
     ret = g_test_run();
diff --git a/tests/spapr-phb-test.c b/tests/spapr-phb-test.c
index b629de4..21004a7 100644
--- a/tests/spapr-phb-test.c
+++ b/tests/spapr-phb-test.c
@@ -7,7 +7,7 @@
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 
diff --git a/tests/tcg/cris/check_addo.c b/tests/tcg/cris/check_addo.c
index 3d8e789..4235e5f 100644
--- a/tests/tcg/cris/check_addo.c
+++ b/tests/tcg/cris/check_addo.c
@@ -51,7 +51,7 @@
 	t = (unsigned char *)x;
 	t -= 32768;
 	p = (unsigned char *) &y.v1;
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_pi_d(p, t);
@@ -62,7 +62,7 @@
 
 
 	t += 32770;
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_pi_w(p, t);
@@ -71,7 +71,7 @@
 	if (*r != 0x4455aa77)
 		err();
 
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_d(p, r);
@@ -81,7 +81,7 @@
 	if (*r != 0xee19ccff)
 		err();
 
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_pi_b(p, t);
@@ -90,7 +90,7 @@
 	if (*(uint16_t*)r != 0xff22)
 		err();
 
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_b(p, r);
@@ -100,7 +100,7 @@
 	if (*r != 0x4455aa77)
 		err();
 
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_w(p, r);
@@ -110,7 +110,7 @@
 	if (*r != 0xff224455)
 		err();
 
-	mb(); /* dont reorder anything beyond here.  */
+	mb(); /* don't reorder anything beyond here.  */
 	cris_tst_cc_init();
 	asm volatile ("setf\tzvnc\n");
 	cris_addo_pi_d(p, t);
diff --git a/tests/tcg/linux-test.c b/tests/tcg/linux-test.c
index 1c6c013..6081c61 100644
--- a/tests/tcg/linux-test.c
+++ b/tests/tcg/linux-test.c
@@ -39,6 +39,7 @@
 #include <dirent.h>
 #include <setjmp.h>
 #include <sys/shm.h>
+#include "qemu/cutils.h"
 
 #define TESTPATH "/tmp/linux-test.tmp"
 #define TESTPORT 7654
@@ -151,7 +152,7 @@
     tbuf.modtime = 1000;
     chk_error(truncate("file2", 100));
     chk_error(utime("file2", &tbuf));
-    chk_error(stat("file2", &st));
+    chk_error(qemu_stat("file2", &st));
     if (st.st_size != 100)
         error("stat size");
     if (!S_ISREG(st.st_mode))
@@ -162,7 +163,7 @@
         st.st_mtime != 1000)
         error("stat time");
 
-    chk_error(stat(TESTPATH, &st));
+    chk_error(qemu_stat(TESTPATH, &st));
     if (!S_ISDIR(st.st_mode))
         error("stat mode");
 
@@ -179,7 +180,7 @@
 
     /* symlink/lstat */
     chk_error(symlink("file2", "file3"));
-    chk_error(lstat("file3", &st));
+    chk_error(qemu_lstat("file3", &st));
     if (!S_ISLNK(st.st_mode))
         error("stat mode");
 
diff --git a/tests/tcg/test-i386-fprem.c b/tests/tcg/test-i386-fprem.c
index e91fb1a..1a71623 100644
--- a/tests/tcg/test-i386-fprem.c
+++ b/tests/tcg/test-i386-fprem.c
@@ -22,10 +22,8 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "qemu/compiler.h"
+
 #include "qemu/osdep.h"
-#include <stdio.h>
-#include <inttypes.h>
 
 /*
  * Inspired by <ieee754.h>'s union ieee854_long_double, but with single
diff --git a/tests/tcg/testthread.c b/tests/tcg/testthread.c
index 2679af1..810ba5d 100644
--- a/tests/tcg/testthread.c
+++ b/tests/tcg/testthread.c
@@ -2,7 +2,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
-#include <signal.h>
 #include <unistd.h>
 #include <inttypes.h>
 #include <pthread.h>
diff --git a/tests/tcg/xtensa/linker.ld.S b/tests/tcg/xtensa/linker.ld.S
index f1e7fa9..5902302 100644
--- a/tests/tcg/xtensa/linker.ld.S
+++ b/tests/tcg/xtensa/linker.ld.S
@@ -1,4 +1,4 @@
-#include <core-isa.h>
+#include "core-isa.h"
 
 #if XTENSA_HAVE_BE
 OUTPUT_FORMAT("elf32-xtensa-be")
diff --git a/tests/tcg/xtensa/test_mmu.S b/tests/tcg/xtensa/test_mmu.S
index 58c5bca..a15316f 100644
--- a/tests/tcg/xtensa/test_mmu.S
+++ b/tests/tcg/xtensa/test_mmu.S
@@ -641,7 +641,7 @@
     witlb   a2, a3
     wdtlb   a2, a3
 
-    movi    a2, 0x00007ffd
+    movi    a2, 0x00007ffc
     movi    a3, 20f
     movi    a4, 21f
     sub     a4, a4, a3
@@ -651,7 +651,7 @@
     addi    a2, a2, 1
     addi    a3, a3, 1
 1:
-    movi    a2, 0x00007ffd
+    movi    a2, 0x00007ffc
     movi    a3, 0x00008000
     /* DTLB: OK, ITLB: OK */
     jx      a2
@@ -668,10 +668,10 @@
     movi    a3, 1
     assert  eq, a2, a3
     rsr     a2, epc1
-    movi    a3, 0x8000
+    movi    a3, 0x7fff
     assert  eq, a2, a3
     rsr     a2, excsave1
-    movi    a3, 0x00007ffd
+    movi    a3, 0x00007ffc
     assert  ne, a2, a3
 
     reset_ps
@@ -680,7 +680,7 @@
     movi    a2, 0x0400000c /* PPN */
     movi    a3, 0x00008000 /* VPN */
     wdtlb   a2, a3
-    movi    a2, 0x00007ffd
+    movi    a2, 0x00007ffc
     movi    a3, 0x00008000
     /* DTLB: FAIL, ITLB: OK */
     jx      a2
@@ -689,10 +689,10 @@
     movi    a3, 28
     assert  eq, a2, a3
     rsr     a2, epc1
-    movi    a3, 0x7ffd
+    movi    a3, 0x7ffc
     assert  eq, a2, a3
     rsr     a2, excsave1
-    movi    a3, 0x00007ffd
+    movi    a3, 0x00007ffc
     assert  eq, a2, a3
 
     reset_ps
@@ -703,7 +703,7 @@
     witlb   a2, a3
     movi    a2, 0x04000003 /* PPN */
     wdtlb   a2, a3
-    movi    a2, 0x00007ffd
+    movi    a2, 0x00007ffc
     movi    a3, 0x00008000
     /* DTLB: OK, ITLB: FAIL */
     jx      a2
@@ -712,10 +712,10 @@
     movi    a3, 20
     assert  eq, a2, a3
     rsr     a2, epc1
-    movi    a3, 0x8000
+    movi    a3, 0x7fff
     assert  eq, a2, a3
     rsr     a2, excsave1
-    movi    a3, 0x00007ffd
+    movi    a3, 0x00007ffc
     assert  ne, a2, a3
 
     reset_ps
@@ -724,7 +724,7 @@
     movi    a2, 0x0400000c /* PPN */
     movi    a3, 0x00008000 /* VPN */
     wdtlb   a2, a3
-    movi    a2, 0x00007ffd
+    movi    a2, 0x00007ffc
     movi    a3, 0x00008000
     /* DTLB: FAIL, ITLB: FAIL */
     jx      a2
@@ -733,10 +733,10 @@
     movi    a3, 28
     assert  eq, a2, a3
     rsr     a2, epc1
-    movi    a3, 0x7ffd
+    movi    a3, 0x7ffc
     assert  eq, a2, a3
     rsr     a2, excsave1
-    movi    a3, 0x00007ffd
+    movi    a3, 0x00007ffc
     assert  eq, a2, a3
 test_end
 
diff --git a/tests/tco-test.c b/tests/tco-test.c
new file mode 100644
index 0000000..0d13aa8
--- /dev/null
+++ b/tests/tco-test.c
@@ -0,0 +1,462 @@
+/*
+ * QEMU ICH9 TCO emulation tests
+ *
+ * Copyright (c) 2015 Paulo Alcantara <pcacjr@zytor.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+
+#include "libqtest.h"
+#include "libqos/pci.h"
+#include "libqos/pci-pc.h"
+#include "hw/pci/pci_regs.h"
+#include "hw/i386/ich9.h"
+#include "hw/acpi/ich9.h"
+#include "hw/acpi/tco.h"
+
+#define RCBA_BASE_ADDR    0xfed1c000
+#define PM_IO_BASE_ADDR   0xb000
+
+enum {
+    TCO_RLD_DEFAULT         = 0x0000,
+    TCO_DAT_IN_DEFAULT      = 0x00,
+    TCO_DAT_OUT_DEFAULT     = 0x00,
+    TCO1_STS_DEFAULT        = 0x0000,
+    TCO2_STS_DEFAULT        = 0x0000,
+    TCO1_CNT_DEFAULT        = 0x0000,
+    TCO2_CNT_DEFAULT        = 0x0008,
+    TCO_MESSAGE1_DEFAULT    = 0x00,
+    TCO_MESSAGE2_DEFAULT    = 0x00,
+    TCO_WDCNT_DEFAULT       = 0x00,
+    TCO_TMR_DEFAULT         = 0x0004,
+    SW_IRQ_GEN_DEFAULT      = 0x03,
+};
+
+#define TCO_SECS_TO_TICKS(secs)     (((secs) * 10) / 6)
+#define TCO_TICKS_TO_SECS(ticks)    (((ticks) * 6) / 10)
+
+typedef struct {
+    const char *args;
+    bool noreboot;
+    QPCIDevice *dev;
+    void *tco_io_base;
+} TestData;
+
+static void test_init(TestData *d)
+{
+    QPCIBus *bus;
+    QTestState *qs;
+    char *s;
+
+    s = g_strdup_printf("-machine q35 %s %s",
+                        d->noreboot ? "" : "-global ICH9-LPC.noreboot=false",
+                        !d->args ? "" : d->args);
+    qs = qtest_start(s);
+    qtest_irq_intercept_in(qs, "ioapic");
+    g_free(s);
+
+    bus = qpci_init_pc();
+    d->dev = qpci_device_find(bus, QPCI_DEVFN(0x1f, 0x00));
+    g_assert(d->dev != NULL);
+
+    qpci_device_enable(d->dev);
+
+    /* set ACPI PM I/O space base address */
+    qpci_config_writel(d->dev, ICH9_LPC_PMBASE, PM_IO_BASE_ADDR | 0x1);
+    /* enable ACPI I/O */
+    qpci_config_writeb(d->dev, ICH9_LPC_ACPI_CTRL, 0x80);
+    /* set Root Complex BAR */
+    qpci_config_writel(d->dev, ICH9_LPC_RCBA, RCBA_BASE_ADDR | 0x1);
+
+    d->tco_io_base = (void *)((uintptr_t)PM_IO_BASE_ADDR + 0x60);
+}
+
+static void stop_tco(const TestData *d)
+{
+    uint32_t val;
+
+    val = qpci_io_readw(d->dev, d->tco_io_base + TCO1_CNT);
+    val |= TCO_TMR_HLT;
+    qpci_io_writew(d->dev, d->tco_io_base + TCO1_CNT, val);
+}
+
+static void start_tco(const TestData *d)
+{
+    uint32_t val;
+
+    val = qpci_io_readw(d->dev, d->tco_io_base + TCO1_CNT);
+    val &= ~TCO_TMR_HLT;
+    qpci_io_writew(d->dev, d->tco_io_base + TCO1_CNT, val);
+}
+
+static void load_tco(const TestData *d)
+{
+    qpci_io_writew(d->dev, d->tco_io_base + TCO_RLD, 4);
+}
+
+static void set_tco_timeout(const TestData *d, uint16_t ticks)
+{
+    qpci_io_writew(d->dev, d->tco_io_base + TCO_TMR, ticks);
+}
+
+static void clear_tco_status(const TestData *d)
+{
+    qpci_io_writew(d->dev, d->tco_io_base + TCO1_STS, 0x0008);
+    qpci_io_writew(d->dev, d->tco_io_base + TCO2_STS, 0x0002);
+    qpci_io_writew(d->dev, d->tco_io_base + TCO2_STS, 0x0004);
+}
+
+static void reset_on_second_timeout(bool enable)
+{
+    uint32_t val;
+
+    val = readl(RCBA_BASE_ADDR + ICH9_CC_GCS);
+    if (enable) {
+        val &= ~ICH9_CC_GCS_NO_REBOOT;
+    } else {
+        val |= ICH9_CC_GCS_NO_REBOOT;
+    }
+    writel(RCBA_BASE_ADDR + ICH9_CC_GCS, val);
+}
+
+static void test_tco_defaults(void)
+{
+    TestData d;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO_RLD), ==,
+                    TCO_RLD_DEFAULT);
+    /* TCO_DAT_IN & TCO_DAT_OUT */
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO_DAT_IN), ==,
+                    (TCO_DAT_OUT_DEFAULT << 8) | TCO_DAT_IN_DEFAULT);
+    /* TCO1_STS & TCO2_STS */
+    g_assert_cmpint(qpci_io_readl(d.dev, d.tco_io_base + TCO1_STS), ==,
+                    (TCO2_STS_DEFAULT << 16) | TCO1_STS_DEFAULT);
+    /* TCO1_CNT & TCO2_CNT */
+    g_assert_cmpint(qpci_io_readl(d.dev, d.tco_io_base + TCO1_CNT), ==,
+                    (TCO2_CNT_DEFAULT << 16) | TCO1_CNT_DEFAULT);
+    /* TCO_MESSAGE1 & TCO_MESSAGE2 */
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO_MESSAGE1), ==,
+                    (TCO_MESSAGE2_DEFAULT << 8) | TCO_MESSAGE1_DEFAULT);
+    g_assert_cmpint(qpci_io_readb(d.dev, d.tco_io_base + TCO_WDCNT), ==,
+                    TCO_WDCNT_DEFAULT);
+    g_assert_cmpint(qpci_io_readb(d.dev, d.tco_io_base + SW_IRQ_GEN), ==,
+                    SW_IRQ_GEN_DEFAULT);
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO_TMR), ==,
+                    TCO_TMR_DEFAULT);
+    qtest_end();
+}
+
+static void test_tco_timeout(void)
+{
+    TestData d;
+    const uint16_t ticks = TCO_SECS_TO_TICKS(4);
+    uint32_t val;
+    int ret;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+
+    stop_tco(&d);
+    clear_tco_status(&d);
+    reset_on_second_timeout(false);
+    set_tco_timeout(&d, ticks);
+    load_tco(&d);
+    start_tco(&d);
+    clock_step(ticks * TCO_TICK_NSEC);
+
+    /* test first timeout */
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS);
+    ret = val & TCO_TIMEOUT ? 1 : 0;
+    g_assert(ret == 1);
+
+    /* test clearing timeout bit */
+    val |= TCO_TIMEOUT;
+    qpci_io_writew(d.dev, d.tco_io_base + TCO1_STS, val);
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS);
+    ret = val & TCO_TIMEOUT ? 1 : 0;
+    g_assert(ret == 0);
+
+    /* test second timeout */
+    clock_step(ticks * TCO_TICK_NSEC);
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS);
+    ret = val & TCO_TIMEOUT ? 1 : 0;
+    g_assert(ret == 1);
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO2_STS);
+    ret = val & TCO_SECOND_TO_STS ? 1 : 0;
+    g_assert(ret == 1);
+
+    stop_tco(&d);
+    qtest_end();
+}
+
+static void test_tco_max_timeout(void)
+{
+    TestData d;
+    const uint16_t ticks = 0xffff;
+    uint32_t val;
+    int ret;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+
+    stop_tco(&d);
+    clear_tco_status(&d);
+    reset_on_second_timeout(false);
+    set_tco_timeout(&d, ticks);
+    load_tco(&d);
+    start_tco(&d);
+    clock_step(((ticks & TCO_TMR_MASK) - 1) * TCO_TICK_NSEC);
+
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO_RLD);
+    g_assert_cmpint(val & TCO_RLD_MASK, ==, 1);
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS);
+    ret = val & TCO_TIMEOUT ? 1 : 0;
+    g_assert(ret == 0);
+    clock_step(TCO_TICK_NSEC);
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS);
+    ret = val & TCO_TIMEOUT ? 1 : 0;
+    g_assert(ret == 1);
+
+    stop_tco(&d);
+    qtest_end();
+}
+
+static QDict *get_watchdog_action(void)
+{
+    QDict *ev = qmp("");
+    QDict *data;
+    g_assert(!strcmp(qdict_get_str(ev, "event"), "WATCHDOG"));
+
+    data = qdict_get_qdict(ev, "data");
+    QINCREF(data);
+    QDECREF(ev);
+    return data;
+}
+
+static void test_tco_second_timeout_pause(void)
+{
+    TestData td;
+    const uint16_t ticks = TCO_SECS_TO_TICKS(32);
+    QDict *ad;
+
+    td.args = "-watchdog-action pause";
+    td.noreboot = false;
+    test_init(&td);
+
+    stop_tco(&td);
+    clear_tco_status(&td);
+    reset_on_second_timeout(true);
+    set_tco_timeout(&td, TCO_SECS_TO_TICKS(16));
+    load_tco(&td);
+    start_tco(&td);
+    clock_step(ticks * TCO_TICK_NSEC * 2);
+    ad = get_watchdog_action();
+    g_assert(!strcmp(qdict_get_str(ad, "action"), "pause"));
+    QDECREF(ad);
+
+    stop_tco(&td);
+    qtest_end();
+}
+
+static void test_tco_second_timeout_reset(void)
+{
+    TestData td;
+    const uint16_t ticks = TCO_SECS_TO_TICKS(16);
+    QDict *ad;
+
+    td.args = "-watchdog-action reset";
+    td.noreboot = false;
+    test_init(&td);
+
+    stop_tco(&td);
+    clear_tco_status(&td);
+    reset_on_second_timeout(true);
+    set_tco_timeout(&td, TCO_SECS_TO_TICKS(16));
+    load_tco(&td);
+    start_tco(&td);
+    clock_step(ticks * TCO_TICK_NSEC * 2);
+    ad = get_watchdog_action();
+    g_assert(!strcmp(qdict_get_str(ad, "action"), "reset"));
+    QDECREF(ad);
+
+    stop_tco(&td);
+    qtest_end();
+}
+
+static void test_tco_second_timeout_shutdown(void)
+{
+    TestData td;
+    const uint16_t ticks = TCO_SECS_TO_TICKS(128);
+    QDict *ad;
+
+    td.args = "-watchdog-action shutdown";
+    td.noreboot = false;
+    test_init(&td);
+
+    stop_tco(&td);
+    clear_tco_status(&td);
+    reset_on_second_timeout(true);
+    set_tco_timeout(&td, ticks);
+    load_tco(&td);
+    start_tco(&td);
+    clock_step(ticks * TCO_TICK_NSEC * 2);
+    ad = get_watchdog_action();
+    g_assert(!strcmp(qdict_get_str(ad, "action"), "shutdown"));
+    QDECREF(ad);
+
+    stop_tco(&td);
+    qtest_end();
+}
+
+static void test_tco_second_timeout_none(void)
+{
+    TestData td;
+    const uint16_t ticks = TCO_SECS_TO_TICKS(256);
+    QDict *ad;
+
+    td.args = "-watchdog-action none";
+    td.noreboot = false;
+    test_init(&td);
+
+    stop_tco(&td);
+    clear_tco_status(&td);
+    reset_on_second_timeout(true);
+    set_tco_timeout(&td, ticks);
+    load_tco(&td);
+    start_tco(&td);
+    clock_step(ticks * TCO_TICK_NSEC * 2);
+    ad = get_watchdog_action();
+    g_assert(!strcmp(qdict_get_str(ad, "action"), "none"));
+    QDECREF(ad);
+
+    stop_tco(&td);
+    qtest_end();
+}
+
+static void test_tco_ticks_counter(void)
+{
+    TestData d;
+    uint16_t ticks = TCO_SECS_TO_TICKS(8);
+    uint16_t rld;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+
+    stop_tco(&d);
+    clear_tco_status(&d);
+    reset_on_second_timeout(false);
+    set_tco_timeout(&d, ticks);
+    load_tco(&d);
+    start_tco(&d);
+
+    do {
+        rld = qpci_io_readw(d.dev, d.tco_io_base + TCO_RLD) & TCO_RLD_MASK;
+        g_assert_cmpint(rld, ==, ticks);
+        clock_step(TCO_TICK_NSEC);
+        ticks--;
+    } while (!(qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS) & TCO_TIMEOUT));
+
+    stop_tco(&d);
+    qtest_end();
+}
+
+static void test_tco1_control_bits(void)
+{
+    TestData d;
+    uint16_t val;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+
+    val = TCO_LOCK;
+    qpci_io_writew(d.dev, d.tco_io_base + TCO1_CNT, val);
+    val &= ~TCO_LOCK;
+    qpci_io_writew(d.dev, d.tco_io_base + TCO1_CNT, val);
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO1_CNT), ==,
+                    TCO_LOCK);
+    qtest_end();
+}
+
+static void test_tco1_status_bits(void)
+{
+    TestData d;
+    uint16_t ticks = 8;
+    uint16_t val;
+    int ret;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+
+    stop_tco(&d);
+    clear_tco_status(&d);
+    reset_on_second_timeout(false);
+    set_tco_timeout(&d, ticks);
+    load_tco(&d);
+    start_tco(&d);
+    clock_step(ticks * TCO_TICK_NSEC);
+
+    qpci_io_writeb(d.dev, d.tco_io_base + TCO_DAT_IN, 0);
+    qpci_io_writeb(d.dev, d.tco_io_base + TCO_DAT_OUT, 0);
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS);
+    ret = val & (TCO_TIMEOUT | SW_TCO_SMI | TCO_INT_STS) ? 1 : 0;
+    g_assert(ret == 1);
+    qpci_io_writew(d.dev, d.tco_io_base + TCO1_STS, val);
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO1_STS), ==, 0);
+    qtest_end();
+}
+
+static void test_tco2_status_bits(void)
+{
+    TestData d;
+    uint16_t ticks = 8;
+    uint16_t val;
+    int ret;
+
+    d.args = NULL;
+    d.noreboot = true;
+    test_init(&d);
+
+    stop_tco(&d);
+    clear_tco_status(&d);
+    reset_on_second_timeout(true);
+    set_tco_timeout(&d, ticks);
+    load_tco(&d);
+    start_tco(&d);
+    clock_step(ticks * TCO_TICK_NSEC * 2);
+
+    val = qpci_io_readw(d.dev, d.tco_io_base + TCO2_STS);
+    ret = val & (TCO_SECOND_TO_STS | TCO_BOOT_STS) ? 1 : 0;
+    g_assert(ret == 1);
+    qpci_io_writew(d.dev, d.tco_io_base + TCO2_STS, val);
+    g_assert_cmpint(qpci_io_readw(d.dev, d.tco_io_base + TCO2_STS), ==, 0);
+    qtest_end();
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("tco/defaults", test_tco_defaults);
+    qtest_add_func("tco/timeout/no_action", test_tco_timeout);
+    qtest_add_func("tco/timeout/no_action/max", test_tco_max_timeout);
+    qtest_add_func("tco/second_timeout/pause", test_tco_second_timeout_pause);
+    qtest_add_func("tco/second_timeout/reset", test_tco_second_timeout_reset);
+    qtest_add_func("tco/second_timeout/shutdown",
+                   test_tco_second_timeout_shutdown);
+    qtest_add_func("tco/second_timeout/none", test_tco_second_timeout_none);
+    qtest_add_func("tco/counter", test_tco_ticks_counter);
+    qtest_add_func("tco/tco1_control/bits", test_tco1_control_bits);
+    qtest_add_func("tco/tco1_status/bits", test_tco1_status_bits);
+    qtest_add_func("tco/tco2_status/bits", test_tco2_status_bits);
+    return g_test_run();
+}
diff --git a/tests/test-aio.c b/tests/test-aio.c
index a7cb5c9..03aa846 100644
--- a/tests/test-aio.c
+++ b/tests/test-aio.c
@@ -10,8 +10,9 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "block/aio.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/sockets.h"
 #include "qemu/error-report.h"
@@ -97,14 +98,6 @@
 
 /* Tests using aio_*.  */
 
-static void test_notify(void)
-{
-    g_assert(!aio_poll(ctx, false));
-    aio_notify(ctx);
-    g_assert(!aio_poll(ctx, true));
-    g_assert(!aio_poll(ctx, false));
-}
-
 typedef struct {
     QemuMutex start_lock;
     bool thread_acquired;
@@ -126,6 +119,12 @@
     return NULL;
 }
 
+static void set_event_notifier(AioContext *ctx, EventNotifier *notifier,
+                               EventNotifierHandler *handler)
+{
+    aio_set_event_notifier(ctx, notifier, false, handler);
+}
+
 static void dummy_notifier_read(EventNotifier *unused)
 {
     g_assert(false); /* should never be invoked */
@@ -139,7 +138,7 @@
 
     /* Dummy event notifier ensures aio_poll() will block */
     event_notifier_init(&notifier, false);
-    aio_set_event_notifier(ctx, &notifier, dummy_notifier_read);
+    set_event_notifier(ctx, &notifier, dummy_notifier_read);
     g_assert(!aio_poll(ctx, false)); /* consume aio_notify() */
 
     qemu_mutex_init(&data.start_lock);
@@ -157,7 +156,7 @@
     aio_context_release(ctx);
 
     qemu_thread_join(&thread);
-    aio_set_event_notifier(ctx, &notifier, NULL);
+    set_event_notifier(ctx, &notifier, NULL);
     event_notifier_cleanup(&notifier);
 
     g_assert(data.thread_acquired);
@@ -316,11 +315,11 @@
 {
     EventNotifierTestData data = { .n = 0, .active = 0 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     event_notifier_cleanup(&data.e);
@@ -330,8 +329,8 @@
 {
     EventNotifierTestData data = { .n = 0, .active = 1 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
-    g_assert(!aio_poll(ctx, false));
+    set_event_notifier(ctx, &data.e, event_ready_cb);
+    while (aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 1);
 
@@ -344,7 +343,7 @@
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 1);
 
@@ -355,8 +354,8 @@
 {
     EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
-    g_assert(!aio_poll(ctx, false));
+    set_event_notifier(ctx, &data.e, event_ready_cb);
+    while (aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 10);
 
@@ -371,18 +370,42 @@
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!aio_poll(ctx, false));
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     event_notifier_cleanup(&data.e);
 }
 
+static void test_aio_external_client(void)
+{
+    int i, j;
+
+    for (i = 1; i < 3; i++) {
+        EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+        event_notifier_init(&data.e, false);
+        aio_set_event_notifier(ctx, &data.e, true, event_ready_cb);
+        event_notifier_set(&data.e);
+        for (j = 0; j < i; j++) {
+            aio_disable_external(ctx);
+        }
+        for (j = 0; j < i; j++) {
+            assert(!aio_poll(ctx, false));
+            assert(event_notifier_test_and_clear(&data.e));
+            event_notifier_set(&data.e);
+            aio_enable_external(ctx);
+        }
+        assert(aio_poll(ctx, false));
+        set_event_notifier(ctx, &data.e, NULL);
+        event_notifier_cleanup(&data.e);
+    }
+}
+
 static void test_wait_event_notifier_noflush(void)
 {
     EventNotifierTestData data = { .n = 0 };
     EventNotifierTestData dummy = { .n = 0, .active = 1 };
 
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
 
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 0);
@@ -395,7 +418,7 @@
 
     /* An active event notifier forces aio_poll to look at EventNotifiers.  */
     event_notifier_init(&dummy.e, false);
-    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb);
+    set_event_notifier(ctx, &dummy.e, event_ready_cb);
 
     event_notifier_set(&data.e);
     g_assert(aio_poll(ctx, false));
@@ -415,10 +438,10 @@
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &dummy.e, NULL);
+    set_event_notifier(ctx, &dummy.e, NULL);
     event_notifier_cleanup(&dummy.e);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
@@ -429,14 +452,14 @@
 {
     TimerTestData data = { .n = 0, .ctx = ctx, .ns = SCALE_MS * 750LL,
                            .max = 2,
-                           .clock_type = QEMU_CLOCK_VIRTUAL };
+                           .clock_type = QEMU_CLOCK_REALTIME };
     EventNotifier e;
 
     /* aio_poll will not block to wait for timers to complete unless it has
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
     event_notifier_init(&e, false);
-    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
+    set_event_notifier(ctx, &e, dummy_io_handler_read);
     aio_poll(ctx, false);
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -475,7 +498,7 @@
     g_assert(!aio_poll(ctx, false));
     g_assert_cmpint(data.n, ==, 2);
 
-    aio_set_event_notifier(ctx, &e, NULL);
+    set_event_notifier(ctx, &e, NULL);
     event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
@@ -494,14 +517,6 @@
  *   works well, and that's what I am using.
  */
 
-static void test_source_notify(void)
-{
-    while (g_main_context_iteration(NULL, false));
-    aio_notify(ctx);
-    g_assert(g_main_context_iteration(NULL, true));
-    g_assert(!g_main_context_iteration(NULL, false));
-}
-
 static void test_source_flush(void)
 {
     g_assert(!g_main_context_iteration(NULL, false));
@@ -654,11 +669,11 @@
 {
     EventNotifierTestData data = { .n = 0, .active = 0 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     event_notifier_cleanup(&data.e);
@@ -668,8 +683,8 @@
 {
     EventNotifierTestData data = { .n = 0, .active = 1 };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
-    g_assert(g_main_context_iteration(NULL, false));
+    set_event_notifier(ctx, &data.e, event_ready_cb);
+    while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 1);
 
@@ -682,7 +697,7 @@
     g_assert_cmpint(data.n, ==, 1);
     g_assert_cmpint(data.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 1);
 
@@ -693,8 +708,8 @@
 {
     EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
-    g_assert(g_main_context_iteration(NULL, false));
+    set_event_notifier(ctx, &data.e, event_ready_cb);
+    while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
     g_assert_cmpint(data.active, ==, 10);
 
@@ -709,7 +724,7 @@
     g_assert_cmpint(data.active, ==, 0);
     g_assert(!g_main_context_iteration(NULL, false));
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     event_notifier_cleanup(&data.e);
 }
@@ -720,7 +735,7 @@
     EventNotifierTestData dummy = { .n = 0, .active = 1 };
 
     event_notifier_init(&data.e, false);
-    aio_set_event_notifier(ctx, &data.e, event_ready_cb);
+    set_event_notifier(ctx, &data.e, event_ready_cb);
 
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 0);
@@ -733,7 +748,7 @@
 
     /* An active event notifier forces aio_poll to look at EventNotifiers.  */
     event_notifier_init(&dummy.e, false);
-    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb);
+    set_event_notifier(ctx, &dummy.e, event_ready_cb);
 
     event_notifier_set(&data.e);
     g_assert(g_main_context_iteration(NULL, false));
@@ -753,10 +768,10 @@
     g_assert_cmpint(dummy.n, ==, 1);
     g_assert_cmpint(dummy.active, ==, 0);
 
-    aio_set_event_notifier(ctx, &dummy.e, NULL);
+    set_event_notifier(ctx, &dummy.e, NULL);
     event_notifier_cleanup(&dummy.e);
 
-    aio_set_event_notifier(ctx, &data.e, NULL);
+    set_event_notifier(ctx, &data.e, NULL);
     while (g_main_context_iteration(NULL, false));
     g_assert_cmpint(data.n, ==, 2);
 
@@ -767,7 +782,7 @@
 {
     TimerTestData data = { .n = 0, .ctx = ctx, .ns = SCALE_MS * 750LL,
                            .max = 2,
-                           .clock_type = QEMU_CLOCK_VIRTUAL };
+                           .clock_type = QEMU_CLOCK_REALTIME };
     EventNotifier e;
     int64_t expiry;
 
@@ -775,7 +790,7 @@
      * an fd to wait on. Fixing this breaks other tests. So create a dummy one.
      */
     event_notifier_init(&e, false);
-    aio_set_event_notifier(ctx, &e, dummy_io_handler_read);
+    set_event_notifier(ctx, &e, dummy_io_handler_read);
     do {} while (g_main_context_iteration(NULL, false));
 
     aio_timer_init(ctx, &data.timer, data.clock_type,
@@ -800,7 +815,7 @@
     g_assert_cmpint(data.n, ==, 2);
     g_assert(qemu_clock_get_ns(data.clock_type) > expiry);
 
-    aio_set_event_notifier(ctx, &e, NULL);
+    set_event_notifier(ctx, &e, NULL);
     event_notifier_cleanup(&e);
 
     timer_del(&data.timer);
@@ -818,9 +833,7 @@
 
     ctx = aio_context_new(&local_error);
     if (!ctx) {
-        error_report("Failed to create AIO Context: '%s'",
-                     error_get_pretty(local_error));
-        error_free(local_error);
+        error_reportf_err(local_error, "Failed to create AIO Context: ");
         exit(1);
     }
     src = aio_get_g_source(ctx);
@@ -830,7 +843,6 @@
     while (g_main_context_iteration(NULL, false));
 
     g_test_init(&argc, &argv, NULL);
-    g_test_add_func("/aio/notify",                  test_notify);
     g_test_add_func("/aio/acquire",                 test_acquire);
     g_test_add_func("/aio/bh/schedule",             test_bh_schedule);
     g_test_add_func("/aio/bh/schedule10",           test_bh_schedule10);
@@ -843,9 +855,9 @@
     g_test_add_func("/aio/event/wait",              test_wait_event_notifier);
     g_test_add_func("/aio/event/wait/no-flush-cb",  test_wait_event_notifier_noflush);
     g_test_add_func("/aio/event/flush",             test_flush_event_notifier);
+    g_test_add_func("/aio/external-client",         test_aio_external_client);
     g_test_add_func("/aio/timer/schedule",          test_timer_schedule);
 
-    g_test_add_func("/aio-gsource/notify",                  test_source_notify);
     g_test_add_func("/aio-gsource/flush",                   test_source_flush);
     g_test_add_func("/aio-gsource/bh/schedule",             test_source_bh_schedule);
     g_test_add_func("/aio-gsource/bh/schedule10",           test_source_bh_schedule10);
diff --git a/tests/test-base64.c b/tests/test-base64.c
new file mode 100644
index 0000000..ec122ce
--- /dev/null
+++ b/tests/test-base64.c
@@ -0,0 +1,109 @@
+/*
+ * QEMU base64 helper test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "qapi/error.h"
+#include "qemu/base64.h"
+
+static void test_base64_good(void)
+{
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW\n"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+    const char expect[] = "Because we focused on the snake, "
+        "we missed the scorpion.";
+
+    size_t len;
+    uint8_t *actual = qbase64_decode(input,
+                                     -1,
+                                     &len,
+                                     &error_abort);
+
+    g_assert(actual != NULL);
+    g_assert_cmpint(len, ==, strlen(expect));
+    g_assert_cmpstr((char *)actual, ==, expect);
+    g_free(actual);
+}
+
+
+static void test_base64_bad(const char *input,
+                            size_t input_len)
+{
+    size_t len;
+    Error *err = NULL;
+    uint8_t *actual = qbase64_decode(input,
+                                     input_len,
+                                     &len,
+                                     &err);
+
+    g_assert(err != NULL);
+    g_assert(actual == NULL);
+    g_assert_cmpint(len, ==, 0);
+    error_free(err);
+}
+
+
+static void test_base64_embedded_nul(void)
+{
+    /* We put a NUL character in the middle of the base64
+     * text which is invalid data, given the expected length */
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW\0"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+
+    test_base64_bad(input, G_N_ELEMENTS(input) - 1);
+}
+
+
+static void test_base64_not_nul_terminated(void)
+{
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW\n"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+
+    /* Using '-2' to make us drop the trailing NUL, thus
+     * creating an invalid base64 sequence for decoding */
+    test_base64_bad(input, G_N_ELEMENTS(input) - 2);
+}
+
+
+static void test_base64_invalid_chars(void)
+{
+    /* We put a single quote character in the middle
+     * of the base64 text which is invalid data */
+    const char input[] =
+        "QmVjYXVzZSB3ZSBmb2N1c2VkIG9uIHRoZSBzbmFrZSwgd2UgbW'"
+        "lzc2VkIHRoZSBzY29ycGlvbi4=";
+
+    test_base64_bad(input, strlen(input));
+}
+
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/util/base64/good", test_base64_good);
+    g_test_add_func("/util/base64/embedded-nul", test_base64_embedded_nul);
+    g_test_add_func("/util/base64/not-nul-terminated",
+                    test_base64_not_nul_terminated);
+    g_test_add_func("/util/base64/invalid-chars", test_base64_invalid_chars);
+    return g_test_run();
+}
diff --git a/tests/test-bitops.c b/tests/test-bitops.c
index 47b5d3e..5a791d2 100644
--- a/tests/test-bitops.c
+++ b/tests/test-bitops.c
@@ -6,8 +6,6 @@
  *
  */
 
-#include <glib.h>
-#include <stdint.h>
 #include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
@@ -67,10 +65,82 @@
     }
 }
 
+typedef struct {
+    uint32_t unshuffled;
+    uint32_t shuffled;
+} Shuffle32Test;
+
+typedef struct {
+    uint64_t unshuffled;
+    uint64_t shuffled;
+} Shuffle64Test;
+
+static const Shuffle32Test test_shuffle32_data[] = {
+    { 0x0000FFFF, 0x55555555 },
+    { 0x000081C5, 0x40015011 },
+};
+
+static const Shuffle64Test test_shuffle64_data[] = {
+    { 0x00000000FFFFFFFFULL, 0x5555555555555555ULL },
+    { 0x00000000493AB02CULL, 0x1041054445000450ULL },
+};
+
+static void test_half_shuffle32(void)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(test_shuffle32_data); i++) {
+        const Shuffle32Test *test = &test_shuffle32_data[i];
+        uint32_t r = half_shuffle32(test->unshuffled);
+
+        g_assert_cmpint(r, ==, test->shuffled);
+    }
+}
+
+static void test_half_shuffle64(void)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(test_shuffle64_data); i++) {
+        const Shuffle64Test *test = &test_shuffle64_data[i];
+        uint64_t r = half_shuffle64(test->unshuffled);
+
+        g_assert_cmpint(r, ==, test->shuffled);
+    }
+}
+
+static void test_half_unshuffle32(void)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(test_shuffle32_data); i++) {
+        const Shuffle32Test *test = &test_shuffle32_data[i];
+        uint32_t r = half_unshuffle32(test->shuffled);
+
+        g_assert_cmpint(r, ==, test->unshuffled);
+    }
+}
+
+static void test_half_unshuffle64(void)
+{
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(test_shuffle64_data); i++) {
+        const Shuffle64Test *test = &test_shuffle64_data[i];
+        uint64_t r = half_unshuffle64(test->shuffled);
+
+        g_assert_cmpint(r, ==, test->unshuffled);
+    }
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
     g_test_add_func("/bitops/sextract32", test_sextract32);
     g_test_add_func("/bitops/sextract64", test_sextract64);
+    g_test_add_func("/bitops/half_shuffle32", test_half_shuffle32);
+    g_test_add_func("/bitops/half_shuffle64", test_half_shuffle64);
+    g_test_add_func("/bitops/half_unshuffle32", test_half_unshuffle32);
+    g_test_add_func("/bitops/half_unshuffle64", test_half_unshuffle64);
     return g_test_run();
 }
diff --git a/tests/test-blockjob-txn.c b/tests/test-blockjob-txn.c
new file mode 100644
index 0000000..d049cba
--- /dev/null
+++ b/tests/test-blockjob-txn.c
@@ -0,0 +1,254 @@
+/*
+ * Blockjob transactions tests
+ *
+ * Copyright Red Hat, Inc. 2015
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/main-loop.h"
+#include "block/blockjob.h"
+#include "sysemu/block-backend.h"
+
+typedef struct {
+    BlockJob common;
+    unsigned int iterations;
+    bool use_timer;
+    int rc;
+    int *result;
+} TestBlockJob;
+
+static const BlockJobDriver test_block_job_driver = {
+    .instance_size = sizeof(TestBlockJob),
+};
+
+static void test_block_job_complete(BlockJob *job, void *opaque)
+{
+    BlockDriverState *bs = blk_bs(job->blk);
+    int rc = (intptr_t)opaque;
+
+    if (block_job_is_cancelled(job)) {
+        rc = -ECANCELED;
+    }
+
+    block_job_completed(job, rc);
+    bdrv_unref(bs);
+}
+
+static void coroutine_fn test_block_job_run(void *opaque)
+{
+    TestBlockJob *s = opaque;
+    BlockJob *job = &s->common;
+
+    while (s->iterations--) {
+        if (s->use_timer) {
+            block_job_sleep_ns(job, QEMU_CLOCK_REALTIME, 0);
+        } else {
+            block_job_yield(job);
+        }
+
+        if (block_job_is_cancelled(job)) {
+            break;
+        }
+    }
+
+    block_job_defer_to_main_loop(job, test_block_job_complete,
+                                 (void *)(intptr_t)s->rc);
+}
+
+typedef struct {
+    TestBlockJob *job;
+    int *result;
+} TestBlockJobCBData;
+
+static void test_block_job_cb(void *opaque, int ret)
+{
+    TestBlockJobCBData *data = opaque;
+    if (!ret && block_job_is_cancelled(&data->job->common)) {
+        ret = -ECANCELED;
+    }
+    *data->result = ret;
+    g_free(data);
+}
+
+/* Create a block job that completes with a given return code after a given
+ * number of event loop iterations.  The return code is stored in the given
+ * result pointer.
+ *
+ * The event loop iterations can either be handled automatically with a 0 delay
+ * timer, or they can be stepped manually by entering the coroutine.
+ */
+static BlockJob *test_block_job_start(unsigned int iterations,
+                                      bool use_timer,
+                                      int rc, int *result)
+{
+    BlockDriverState *bs;
+    TestBlockJob *s;
+    TestBlockJobCBData *data;
+    static unsigned counter;
+    char job_id[24];
+
+    data = g_new0(TestBlockJobCBData, 1);
+    bs = bdrv_new();
+    snprintf(job_id, sizeof(job_id), "job%u", counter++);
+    s = block_job_create(job_id, &test_block_job_driver, bs, 0,
+                         test_block_job_cb, data, &error_abort);
+    s->iterations = iterations;
+    s->use_timer = use_timer;
+    s->rc = rc;
+    s->result = result;
+    s->common.co = qemu_coroutine_create(test_block_job_run, s);
+    data->job = s;
+    data->result = result;
+    qemu_coroutine_enter(s->common.co);
+    return &s->common;
+}
+
+static void test_single_job(int expected)
+{
+    BlockJob *job;
+    BlockJobTxn *txn;
+    int result = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job = test_block_job_start(1, true, expected, &result);
+    block_job_txn_add_job(txn, job);
+
+    if (expected == -ECANCELED) {
+        block_job_cancel(job);
+    }
+
+    while (result == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+    g_assert_cmpint(result, ==, expected);
+
+    block_job_txn_unref(txn);
+}
+
+static void test_single_job_success(void)
+{
+    test_single_job(0);
+}
+
+static void test_single_job_failure(void)
+{
+    test_single_job(-EIO);
+}
+
+static void test_single_job_cancel(void)
+{
+    test_single_job(-ECANCELED);
+}
+
+static void test_pair_jobs(int expected1, int expected2)
+{
+    BlockJob *job1;
+    BlockJob *job2;
+    BlockJobTxn *txn;
+    int result1 = -EINPROGRESS;
+    int result2 = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job1 = test_block_job_start(1, true, expected1, &result1);
+    block_job_txn_add_job(txn, job1);
+    job2 = test_block_job_start(2, true, expected2, &result2);
+    block_job_txn_add_job(txn, job2);
+
+    if (expected1 == -ECANCELED) {
+        block_job_cancel(job1);
+    }
+    if (expected2 == -ECANCELED) {
+        block_job_cancel(job2);
+    }
+
+    while (result1 == -EINPROGRESS || result2 == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    /* Failure or cancellation of one job cancels the other job */
+    if (expected1 != 0) {
+        expected2 = -ECANCELED;
+    } else if (expected2 != 0) {
+        expected1 = -ECANCELED;
+    }
+
+    g_assert_cmpint(result1, ==, expected1);
+    g_assert_cmpint(result2, ==, expected2);
+
+    block_job_txn_unref(txn);
+}
+
+static void test_pair_jobs_success(void)
+{
+    test_pair_jobs(0, 0);
+}
+
+static void test_pair_jobs_failure(void)
+{
+    /* Test both orderings.  The two jobs run for a different number of
+     * iterations so the code path is different depending on which job fails
+     * first.
+     */
+    test_pair_jobs(-EIO, 0);
+    test_pair_jobs(0, -EIO);
+}
+
+static void test_pair_jobs_cancel(void)
+{
+    test_pair_jobs(-ECANCELED, 0);
+    test_pair_jobs(0, -ECANCELED);
+}
+
+static void test_pair_jobs_fail_cancel_race(void)
+{
+    BlockJob *job1;
+    BlockJob *job2;
+    BlockJobTxn *txn;
+    int result1 = -EINPROGRESS;
+    int result2 = -EINPROGRESS;
+
+    txn = block_job_txn_new();
+    job1 = test_block_job_start(1, true, -ECANCELED, &result1);
+    block_job_txn_add_job(txn, job1);
+    job2 = test_block_job_start(2, false, 0, &result2);
+    block_job_txn_add_job(txn, job2);
+
+    block_job_cancel(job1);
+
+    /* Now make job2 finish before the main loop kicks jobs.  This simulates
+     * the race between a pending kick and another job completing.
+     */
+    block_job_enter(job2);
+    block_job_enter(job2);
+
+    while (result1 == -EINPROGRESS || result2 == -EINPROGRESS) {
+        aio_poll(qemu_get_aio_context(), true);
+    }
+
+    g_assert_cmpint(result1, ==, -ECANCELED);
+    g_assert_cmpint(result2, ==, -ECANCELED);
+
+    block_job_txn_unref(txn);
+}
+
+int main(int argc, char **argv)
+{
+    qemu_init_main_loop(&error_abort);
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/single/success", test_single_job_success);
+    g_test_add_func("/single/failure", test_single_job_failure);
+    g_test_add_func("/single/cancel", test_single_job_cancel);
+    g_test_add_func("/pair/success", test_pair_jobs_success);
+    g_test_add_func("/pair/failure", test_pair_jobs_failure);
+    g_test_add_func("/pair/cancel", test_pair_jobs_cancel);
+    g_test_add_func("/pair/fail-cancel-race", test_pair_jobs_fail_cancel_race);
+    return g_test_run();
+}
diff --git a/tests/test-blockjob.c b/tests/test-blockjob.c
new file mode 100644
index 0000000..5b0e934
--- /dev/null
+++ b/tests/test-blockjob.c
@@ -0,0 +1,147 @@
+/*
+ * Blockjob tests
+ *
+ * Copyright Igalia, S.L. 2016
+ *
+ * Authors:
+ *  Alberto Garcia   <berto@igalia.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/main-loop.h"
+#include "block/blockjob.h"
+#include "sysemu/block-backend.h"
+
+static const BlockJobDriver test_block_job_driver = {
+    .instance_size = sizeof(BlockJob),
+};
+
+static void block_job_cb(void *opaque, int ret)
+{
+}
+
+static BlockJob *do_test_id(BlockBackend *blk, const char *id,
+                            bool should_succeed)
+{
+    BlockJob *job;
+    Error *errp = NULL;
+
+    job = block_job_create(id, &test_block_job_driver, blk_bs(blk), 0,
+                           block_job_cb, NULL, &errp);
+    if (should_succeed) {
+        g_assert_null(errp);
+        g_assert_nonnull(job);
+        if (id) {
+            g_assert_cmpstr(job->id, ==, id);
+        } else {
+            g_assert_cmpstr(job->id, ==, blk_name(blk));
+        }
+    } else {
+        g_assert_nonnull(errp);
+        g_assert_null(job);
+        error_free(errp);
+    }
+
+    return job;
+}
+
+/* This creates a BlockBackend (optionally with a name) with a
+ * BlockDriverState inserted. */
+static BlockBackend *create_blk(const char *name)
+{
+    BlockBackend *blk = blk_new();
+    BlockDriverState *bs = bdrv_new();
+
+    blk_insert_bs(blk, bs);
+    bdrv_unref(bs);
+
+    if (name) {
+        Error *errp = NULL;
+        monitor_add_blk(blk, name, &errp);
+        g_assert_null(errp);
+    }
+
+    return blk;
+}
+
+/* This destroys the backend */
+static void destroy_blk(BlockBackend *blk)
+{
+    if (blk_name(blk)[0] != '\0') {
+        monitor_remove_blk(blk);
+    }
+
+    blk_remove_bs(blk);
+    blk_unref(blk);
+}
+
+static void test_job_ids(void)
+{
+    BlockBackend *blk[3];
+    BlockJob *job[3];
+
+    blk[0] = create_blk(NULL);
+    blk[1] = create_blk("drive1");
+    blk[2] = create_blk("drive2");
+
+    /* No job ID provided and the block backend has no name */
+    job[0] = do_test_id(blk[0], NULL, false);
+
+    /* These are all invalid job IDs */
+    job[0] = do_test_id(blk[0], "0id", false);
+    job[0] = do_test_id(blk[0], "",    false);
+    job[0] = do_test_id(blk[0], "   ", false);
+    job[0] = do_test_id(blk[0], "123", false);
+    job[0] = do_test_id(blk[0], "_id", false);
+    job[0] = do_test_id(blk[0], "-id", false);
+    job[0] = do_test_id(blk[0], ".id", false);
+    job[0] = do_test_id(blk[0], "#id", false);
+
+    /* This one is valid */
+    job[0] = do_test_id(blk[0], "id0", true);
+
+    /* We cannot have two jobs in the same BDS */
+    do_test_id(blk[0], "id1", false);
+
+    /* Duplicate job IDs are not allowed */
+    job[1] = do_test_id(blk[1], "id0", false);
+
+    /* But once job[0] finishes we can reuse its ID */
+    block_job_unref(job[0]);
+    job[1] = do_test_id(blk[1], "id0", true);
+
+    /* No job ID specified, defaults to the backend name ('drive1') */
+    block_job_unref(job[1]);
+    job[1] = do_test_id(blk[1], NULL, true);
+
+    /* Duplicate job ID */
+    job[2] = do_test_id(blk[2], "drive1", false);
+
+    /* The ID of job[2] would default to 'drive2' but it is already in use */
+    job[0] = do_test_id(blk[0], "drive2", true);
+    job[2] = do_test_id(blk[2], NULL, false);
+
+    /* This one is valid */
+    job[2] = do_test_id(blk[2], "id_2", true);
+
+    block_job_unref(job[0]);
+    block_job_unref(job[1]);
+    block_job_unref(job[2]);
+
+    destroy_blk(blk[0]);
+    destroy_blk(blk[1]);
+    destroy_blk(blk[2]);
+}
+
+int main(int argc, char **argv)
+{
+    qemu_init_main_loop(&error_abort);
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/blockjob/ids", test_job_ids);
+    return g_test_run();
+}
diff --git a/tests/test-clone-visitor.c b/tests/test-clone-visitor.c
new file mode 100644
index 0000000..df0c045
--- /dev/null
+++ b/tests/test-clone-visitor.c
@@ -0,0 +1,206 @@
+/*
+ * QAPI Clone Visitor unit-tests.
+ *
+ * Copyright (C) 2016 Red Hat Inc.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include <glib.h>
+
+#include "qemu-common.h"
+#include "qapi/clone-visitor.h"
+#include "test-qapi-types.h"
+#include "test-qapi-visit.h"
+#include "qapi/qmp/types.h"
+
+static void test_clone_struct(void)
+{
+    UserDefOne *src, *dst;
+
+    src = g_new0(UserDefOne, 1);
+    src->integer = 42;
+    src->string = g_strdup("Hello");
+    src->has_enum1 = false;
+    src->enum1 = ENUM_ONE_VALUE2;
+
+    dst = QAPI_CLONE(UserDefOne, src);
+    g_assert(dst);
+    g_assert_cmpint(dst->integer, ==, 42);
+    g_assert(dst->string != src->string);
+    g_assert_cmpstr(dst->string, ==, "Hello");
+    g_assert_cmpint(dst->has_enum1, ==, false);
+    /* Our implementation does this, but it is not required:
+    g_assert_cmpint(dst->enum1, ==, ENUM_ONE_VALUE2);
+    */
+
+    qapi_free_UserDefOne(src);
+    qapi_free_UserDefOne(dst);
+}
+
+static void test_clone_alternate(void)
+{
+    AltStrBool *b_src, *s_src, *b_dst, *s_dst;
+
+    b_src = g_new0(AltStrBool, 1);
+    b_src->type = QTYPE_QBOOL;
+    b_src->u.b = true;
+    s_src = g_new0(AltStrBool, 1);
+    s_src->type = QTYPE_QSTRING;
+    s_src->u.s = g_strdup("World");
+
+    b_dst = QAPI_CLONE(AltStrBool, b_src);
+    g_assert(b_dst);
+    g_assert_cmpint(b_dst->type, ==, b_src->type);
+    g_assert_cmpint(b_dst->u.b, ==, b_src->u.b);
+    s_dst = QAPI_CLONE(AltStrBool, s_src);
+    g_assert(s_dst);
+    g_assert_cmpint(s_dst->type, ==, s_src->type);
+    g_assert_cmpstr(s_dst->u.s, ==, s_src->u.s);
+    g_assert(s_dst->u.s != s_src->u.s);
+
+    qapi_free_AltStrBool(b_src);
+    qapi_free_AltStrBool(s_src);
+    qapi_free_AltStrBool(b_dst);
+    qapi_free_AltStrBool(s_dst);
+}
+
+static void test_clone_native_list(void)
+{
+    uint8List *src, *dst;
+    uint8List *tmp = NULL;
+    int i;
+
+    /* Build list in reverse */
+    for (i = 10; i; i--) {
+        src = g_new0(uint8List, 1);
+        src->next = tmp;
+        src->value = i;
+        tmp = src;
+    }
+
+    dst = QAPI_CLONE(uint8List, src);
+    for (tmp = dst, i = 1; i <= 10; i++) {
+        g_assert(tmp);
+        g_assert_cmpint(tmp->value, ==, i);
+        tmp = tmp->next;
+    }
+    g_assert(!tmp);
+
+    qapi_free_uint8List(src);
+    qapi_free_uint8List(dst);
+}
+
+static void test_clone_empty(void)
+{
+    Empty2 *src, *dst;
+
+    src = g_new0(Empty2, 1);
+    dst = QAPI_CLONE(Empty2, src);
+    g_assert(dst);
+    qapi_free_Empty2(src);
+    qapi_free_Empty2(dst);
+}
+
+static void test_clone_complex1(void)
+{
+    UserDefNativeListUnion *src, *dst;
+
+    src = g_new0(UserDefNativeListUnion, 1);
+    src->type = USER_DEF_NATIVE_LIST_UNION_KIND_STRING;
+
+    dst = QAPI_CLONE(UserDefNativeListUnion, src);
+    g_assert(dst);
+    g_assert_cmpint(dst->type, ==, src->type);
+    g_assert(!dst->u.string.data);
+
+    qapi_free_UserDefNativeListUnion(src);
+    qapi_free_UserDefNativeListUnion(dst);
+}
+
+static void test_clone_complex2(void)
+{
+    WrapAlternate *src, *dst;
+
+    src = g_new0(WrapAlternate, 1);
+    src->alt = g_new(UserDefAlternate, 1);
+    src->alt->type = QTYPE_QDICT;
+    src->alt->u.udfu.integer = 42;
+    /* Clone intentionally converts NULL into "" for strings */
+    src->alt->u.udfu.string = NULL;
+    src->alt->u.udfu.enum1 = ENUM_ONE_VALUE3;
+    src->alt->u.udfu.u.value3.intb = 99;
+    src->alt->u.udfu.u.value3.has_a_b = true;
+    src->alt->u.udfu.u.value3.a_b = true;
+
+    dst = QAPI_CLONE(WrapAlternate, src);
+    g_assert(dst);
+    g_assert(dst->alt);
+    g_assert_cmpint(dst->alt->type, ==, QTYPE_QDICT);
+    g_assert_cmpint(dst->alt->u.udfu.integer, ==, 42);
+    g_assert_cmpstr(dst->alt->u.udfu.string, ==, "");
+    g_assert_cmpint(dst->alt->u.udfu.enum1, ==, ENUM_ONE_VALUE3);
+    g_assert_cmpint(dst->alt->u.udfu.u.value3.intb, ==, 99);
+    g_assert_cmpint(dst->alt->u.udfu.u.value3.has_a_b, ==, true);
+    g_assert_cmpint(dst->alt->u.udfu.u.value3.a_b, ==, true);
+
+    qapi_free_WrapAlternate(src);
+    qapi_free_WrapAlternate(dst);
+}
+
+static void test_clone_complex3(void)
+{
+    __org_qemu_x_Struct2 *src, *dst;
+    __org_qemu_x_Union1List *tmp;
+
+    src = g_new0(__org_qemu_x_Struct2, 1);
+    tmp = src->array = g_new0(__org_qemu_x_Union1List, 1);
+    tmp->value = g_new0(__org_qemu_x_Union1, 1);
+    tmp->value->type = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
+    tmp->value->u.__org_qemu_x_branch.data = g_strdup("one");
+    tmp = tmp->next = g_new0(__org_qemu_x_Union1List, 1);
+    tmp->value = g_new0(__org_qemu_x_Union1, 1);
+    tmp->value->type = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
+    tmp->value->u.__org_qemu_x_branch.data = g_strdup("two");
+    tmp = tmp->next = g_new0(__org_qemu_x_Union1List, 1);
+    tmp->value = g_new0(__org_qemu_x_Union1, 1);
+    tmp->value->type = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
+    tmp->value->u.__org_qemu_x_branch.data = g_strdup("three");
+
+    dst = QAPI_CLONE(__org_qemu_x_Struct2, src);
+    g_assert(dst);
+    tmp = dst->array;
+    g_assert(tmp);
+    g_assert(tmp->value);
+    g_assert_cmpstr(tmp->value->u.__org_qemu_x_branch.data, ==, "one");
+    tmp = tmp->next;
+    g_assert(tmp);
+    g_assert(tmp->value);
+    g_assert_cmpstr(tmp->value->u.__org_qemu_x_branch.data, ==, "two");
+    tmp = tmp->next;
+    g_assert(tmp);
+    g_assert(tmp->value);
+    g_assert_cmpstr(tmp->value->u.__org_qemu_x_branch.data, ==, "three");
+    tmp = tmp->next;
+    g_assert(!tmp);
+
+    qapi_free___org_qemu_x_Struct2(src);
+    qapi_free___org_qemu_x_Struct2(dst);
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/visitor/clone/struct", test_clone_struct);
+    g_test_add_func("/visitor/clone/alternate", test_clone_alternate);
+    g_test_add_func("/visitor/clone/native_list", test_clone_native_list);
+    g_test_add_func("/visitor/clone/empty", test_clone_empty);
+    g_test_add_func("/visitor/clone/complex1", test_clone_complex1);
+    g_test_add_func("/visitor/clone/complex2", test_clone_complex2);
+    g_test_add_func("/visitor/clone/complex3", test_clone_complex3);
+
+    return g_test_run();
+}
diff --git a/tests/test-coroutine.c b/tests/test-coroutine.c
index e22fae1..ee5e06d 100644
--- a/tests/test-coroutine.c
+++ b/tests/test-coroutine.c
@@ -11,8 +11,9 @@
  *
  */
 
-#include <glib.h>
-#include "block/coroutine.h"
+#include "qemu/osdep.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
 
 /*
  * Check that qemu_in_coroutine() works
@@ -29,8 +30,8 @@
 
     g_assert(!qemu_in_coroutine());
 
-    coroutine = qemu_coroutine_create(verify_in_coroutine);
-    qemu_coroutine_enter(coroutine, NULL);
+    coroutine = qemu_coroutine_create(verify_in_coroutine, NULL);
+    qemu_coroutine_enter(coroutine);
 }
 
 /*
@@ -39,15 +40,16 @@
 
 static void coroutine_fn verify_self(void *opaque)
 {
-    g_assert(qemu_coroutine_self() == opaque);
+    Coroutine **p_co = opaque;
+    g_assert(qemu_coroutine_self() == *p_co);
 }
 
 static void test_self(void)
 {
     Coroutine *coroutine;
 
-    coroutine = qemu_coroutine_create(verify_self);
-    qemu_coroutine_enter(coroutine, coroutine);
+    coroutine = qemu_coroutine_create(verify_self, &coroutine);
+    qemu_coroutine_enter(coroutine);
 }
 
 /*
@@ -69,8 +71,8 @@
     if (nd->n_enter < nd->max) {
         Coroutine *child;
 
-        child = qemu_coroutine_create(nest);
-        qemu_coroutine_enter(child, nd);
+        child = qemu_coroutine_create(nest, nd);
+        qemu_coroutine_enter(child);
     }
 
     nd->n_return++;
@@ -85,8 +87,8 @@
         .max      = 128,
     };
 
-    root = qemu_coroutine_create(nest);
-    qemu_coroutine_enter(root, &nd);
+    root = qemu_coroutine_create(nest, &nd);
+    qemu_coroutine_enter(root);
 
     /* Must enter and return from max nesting level */
     g_assert_cmpint(nd.n_enter, ==, nd.max);
@@ -114,14 +116,38 @@
     bool done = false;
     int i = -1; /* one extra time to return from coroutine */
 
-    coroutine = qemu_coroutine_create(yield_5_times);
+    coroutine = qemu_coroutine_create(yield_5_times, &done);
     while (!done) {
-        qemu_coroutine_enter(coroutine, &done);
+        qemu_coroutine_enter(coroutine);
         i++;
     }
     g_assert_cmpint(i, ==, 5); /* coroutine must yield 5 times */
 }
 
+static void coroutine_fn c2_fn(void *opaque)
+{
+    qemu_coroutine_yield();
+}
+
+static void coroutine_fn c1_fn(void *opaque)
+{
+    Coroutine *c2 = opaque;
+    qemu_coroutine_enter(c2);
+}
+
+static void test_co_queue(void)
+{
+    Coroutine *c1;
+    Coroutine *c2;
+
+    c2 = qemu_coroutine_create(c2_fn, NULL);
+    c1 = qemu_coroutine_create(c1_fn, c2);
+
+    qemu_coroutine_enter(c1);
+    memset(c1, 0xff, sizeof(Coroutine));
+    qemu_coroutine_enter(c2);
+}
+
 /*
  * Check that creation, enter, and return work
  */
@@ -139,14 +165,14 @@
     bool done = false;
 
     /* Create, enter, and return from coroutine */
-    coroutine = qemu_coroutine_create(set_and_exit);
-    qemu_coroutine_enter(coroutine, &done);
+    coroutine = qemu_coroutine_create(set_and_exit, &done);
+    qemu_coroutine_enter(coroutine);
     g_assert(done); /* expect done to be true (first time) */
 
     /* Repeat to check that no state affects this test */
     done = false;
-    coroutine = qemu_coroutine_create(set_and_exit);
-    qemu_coroutine_enter(coroutine, &done);
+    coroutine = qemu_coroutine_create(set_and_exit, &done);
+    qemu_coroutine_enter(coroutine);
     g_assert(done); /* expect done to be true (second time) */
 }
 
@@ -180,12 +206,12 @@
 {
     Coroutine *co;
 
-    co = qemu_coroutine_create(co_order_test);
+    co = qemu_coroutine_create(co_order_test, NULL);
     record_push(1, 1);
-    qemu_coroutine_enter(co, NULL);
+    qemu_coroutine_enter(co);
     record_push(1, 2);
     g_assert(!qemu_in_coroutine());
-    qemu_coroutine_enter(co, NULL);
+    qemu_coroutine_enter(co);
     record_push(1, 3);
     g_assert(!qemu_in_coroutine());
 }
@@ -222,8 +248,8 @@
 
     g_test_timer_start();
     for (i = 0; i < max; i++) {
-        coroutine = qemu_coroutine_create(empty_coroutine);
-        qemu_coroutine_enter(coroutine, NULL);
+        coroutine = qemu_coroutine_create(empty_coroutine, NULL);
+        qemu_coroutine_enter(coroutine);
     }
     duration = g_test_timer_elapsed();
 
@@ -246,8 +272,8 @@
             .n_return = 0,
             .max      = maxnesting,
         };
-        root = qemu_coroutine_create(nest);
-        qemu_coroutine_enter(root, &nd);
+        root = qemu_coroutine_create(nest, &nd);
+        qemu_coroutine_enter(root);
     }
     duration = g_test_timer_elapsed();
 
@@ -276,11 +302,11 @@
 
     maxcycles = 100000000;
     i = maxcycles;
-    Coroutine *coroutine = qemu_coroutine_create(yield_loop);
+    Coroutine *coroutine = qemu_coroutine_create(yield_loop, &i);
 
     g_test_timer_start();
     while (i > 0) {
-        qemu_coroutine_enter(coroutine, &i);
+        qemu_coroutine_enter(coroutine);
     }
     duration = g_test_timer_elapsed();
 
@@ -326,9 +352,9 @@
 
     g_test_timer_start();
     while (i++ < maxcycles) {
-        co = qemu_coroutine_create(perf_cost_func);
-        qemu_coroutine_enter(co, &i);
-        qemu_coroutine_enter(co, NULL);
+        co = qemu_coroutine_create(perf_cost_func, &i);
+        qemu_coroutine_enter(co);
+        qemu_coroutine_enter(co);
     }
     duration = g_test_timer_elapsed();
     ops = (long)(maxcycles / (duration * 1000));
@@ -337,12 +363,21 @@
                    "%luns per coroutine",
                    maxcycles,
                    duration, ops,
-                   (unsigned long)(1000000000 * duration) / maxcycles);
+                   (unsigned long)(1000000000.0 * duration / maxcycles));
 }
 
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
+
+    /* This test assumes there is a freelist and marks freed coroutine memory
+     * with a sentinel value.  If there is no freelist this would legitimately
+     * crash, so skip it.
+     */
+    if (CONFIG_COROUTINE_POOL) {
+        g_test_add_func("/basic/co_queue", test_co_queue);
+    }
+
     g_test_add_func("/basic/lifecycle", test_lifecycle);
     g_test_add_func("/basic/yield", test_yield);
     g_test_add_func("/basic/nesting", test_nesting);
diff --git a/tests/test-crypto-afsplit.c b/tests/test-crypto-afsplit.c
new file mode 100644
index 0000000..f9f2fcd
--- /dev/null
+++ b/tests/test-crypto-afsplit.c
@@ -0,0 +1,194 @@
+/*
+ * QEMU Crypto anti-forensic splitter
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/init.h"
+#include "crypto/afsplit.h"
+
+typedef struct QCryptoAFSplitTestData QCryptoAFSplitTestData;
+struct QCryptoAFSplitTestData {
+    const char *path;
+    QCryptoHashAlgorithm hash;
+    uint32_t stripes;
+    size_t blocklen;
+    const uint8_t *key;
+    const uint8_t *splitkey;
+};
+
+static QCryptoAFSplitTestData test_data[] = {
+    {
+        .path = "/crypto/afsplit/sha256/5",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .stripes = 5,
+        .blocklen = 32,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+            "\xa8\xa9\xaa\xab\xac\xad\xae\xaf",
+        .splitkey = (const uint8_t *)
+            "\xfd\xd2\x73\xb1\x7d\x99\x93\x34"
+            "\x70\xde\xfa\x07\xc5\xac\x58\xd2"
+            "\x30\x67\x2f\x1a\x35\x43\x60\x7d"
+            "\x77\x02\xdb\x62\x3c\xcb\x2c\x33"
+            "\x48\x08\xb6\xf1\x7c\xa3\x20\xa0"
+            "\xad\x2d\x4c\xf3\xcd\x18\x6f\x53"
+            "\xf9\xe8\xe7\x59\x27\x3c\xa9\x54"
+            "\x61\x87\xb3\xaf\xf6\xf7\x7e\x64"
+            "\x86\xaa\x89\x7f\x1f\x9f\xdb\x86"
+            "\xf4\xa2\x16\xff\xa3\x4f\x8c\xa1"
+            "\x59\xc4\x23\x34\x28\xc4\x77\x71"
+            "\x83\xd4\xcd\x8e\x89\x1b\xc7\xc5"
+            "\xae\x4d\xa9\xcd\xc9\x72\x85\x70"
+            "\x13\x68\x52\x83\xfc\xb8\x11\x72"
+            "\xba\x3d\xc6\x4a\x28\xfa\xe2\x86"
+            "\x7b\x27\xab\x58\xe1\xa4\xca\xf6"
+            "\x9e\xbc\xfe\x0c\x92\x79\xb3\xec"
+            "\x1c\x5f\x79\x3b\x0d\x1e\xaa\x1a"
+            "\x77\x0f\x70\x19\x4b\xc8\x80\xee"
+            "\x27\x7c\x6e\x4a\x91\x96\x5c\xf4"
+    },
+    {
+        .path = "/crypto/afsplit/sha256/5000",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .stripes = 5000,
+        .blocklen = 16,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+    },
+    {
+        .path = "/crypto/afsplit/sha1/1000",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .stripes = 1000,
+        .blocklen = 32,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7"
+            "\xa8\xa9\xaa\xab\xac\xad\xae\xaf",
+    },
+    {
+        .path = "/crypto/afsplit/sha256/big",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .stripes = 1000,
+        .blocklen = 64,
+        .key = (const uint8_t *)
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+            "\x00\x01\x02\x03\x04\x05\x06\x07"
+            "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+    },
+};
+
+
+static inline char hex(int i)
+{
+    if (i < 10) {
+        return '0' + i;
+    }
+    return 'a' + (i - 10);
+}
+
+static char *hex_string(const uint8_t *bytes,
+                        size_t len)
+{
+    char *hexstr = g_new0(char, len * 2 + 1);
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        hexstr[i * 2] = hex((bytes[i] >> 4) & 0xf);
+        hexstr[i * 2 + 1] = hex(bytes[i] & 0xf);
+    }
+    hexstr[len * 2] = '\0';
+
+    return hexstr;
+}
+
+static void test_afsplit(const void *opaque)
+{
+    const QCryptoAFSplitTestData *data = opaque;
+    size_t splitlen = data->blocklen * data->stripes;
+    uint8_t *splitkey = g_new0(uint8_t, splitlen);
+    uint8_t *key = g_new0(uint8_t, data->blocklen);
+    gchar *expect, *actual;
+
+    /* First time we round-trip the key */
+    qcrypto_afsplit_encode(data->hash,
+                           data->blocklen, data->stripes,
+                           data->key, splitkey,
+                           &error_abort);
+
+    qcrypto_afsplit_decode(data->hash,
+                           data->blocklen, data->stripes,
+                           splitkey, key,
+                           &error_abort);
+
+    expect = hex_string(data->key, data->blocklen);
+    actual = hex_string(key, data->blocklen);
+
+    g_assert_cmpstr(actual, ==, expect);
+
+    g_free(actual);
+    g_free(expect);
+
+    /* Second time we merely try decoding a previous split */
+    if (data->splitkey) {
+        memset(key, 0, data->blocklen);
+
+        qcrypto_afsplit_decode(data->hash,
+                               data->blocklen, data->stripes,
+                               data->splitkey, key,
+                               &error_abort);
+
+        expect = hex_string(data->key, data->blocklen);
+        actual = hex_string(key, data->blocklen);
+
+        g_assert_cmpstr(actual, ==, expect);
+
+        g_free(actual);
+        g_free(expect);
+    }
+
+    g_free(key);
+    g_free(splitkey);
+}
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (!qcrypto_hash_supports(test_data[i].hash)) {
+            continue;
+        }
+        g_test_add_data_func(test_data[i].path, &test_data[i], test_afsplit);
+    }
+    return g_test_run();
+}
diff --git a/tests/test-crypto-block.c b/tests/test-crypto-block.c
new file mode 100644
index 0000000..a38110d
--- /dev/null
+++ b/tests/test-crypto-block.c
@@ -0,0 +1,363 @@
+/*
+ * QEMU Crypto block encryption
+ *
+ * Copyright (c) 2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/init.h"
+#include "crypto/block.h"
+#include "qemu/buffer.h"
+#include "crypto/secret.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#if defined(CONFIG_UUID) && (defined(_WIN32) || defined RUSAGE_THREAD)
+#define TEST_LUKS
+#else
+#undef TEST_LUKS
+#endif
+
+static QCryptoBlockCreateOptions qcow_create_opts = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_QCOW,
+    .u.qcow = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+static QCryptoBlockOpenOptions qcow_open_opts = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_QCOW,
+    .u.qcow = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+
+#ifdef TEST_LUKS
+static QCryptoBlockOpenOptions luks_open_opts = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+
+/* Creation with all default values */
+static QCryptoBlockCreateOptions luks_create_opts_default = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+    },
+};
+
+
+/* ...and with explicit values */
+static QCryptoBlockCreateOptions luks_create_opts_aes256_cbc_plain64 = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+        .has_cipher_alg = true,
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .has_cipher_mode = true,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .has_ivgen_alg = true,
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+    },
+};
+
+
+static QCryptoBlockCreateOptions luks_create_opts_aes256_cbc_essiv = {
+    .format = Q_CRYPTO_BLOCK_FORMAT_LUKS,
+    .u.luks = {
+        .has_key_secret = true,
+        .key_secret = (char *)"sec0",
+        .has_cipher_alg = true,
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .has_cipher_mode = true,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .has_ivgen_alg = true,
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .has_ivgen_hash_alg = true,
+        .ivgen_hash_alg = QCRYPTO_HASH_ALG_SHA256,
+        .has_hash_alg = true,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA1,
+    },
+};
+#endif /* TEST_LUKS */
+
+
+static struct QCryptoBlockTestData {
+    const char *path;
+    QCryptoBlockCreateOptions *create_opts;
+    QCryptoBlockOpenOptions *open_opts;
+
+    bool expect_header;
+
+    QCryptoCipherAlgorithm cipher_alg;
+    QCryptoCipherMode cipher_mode;
+    QCryptoHashAlgorithm hash_alg;
+
+    QCryptoIVGenAlgorithm ivgen_alg;
+    QCryptoHashAlgorithm ivgen_hash;
+
+    bool slow;
+} test_data[] = {
+    {
+        .path = "/crypto/block/qcow",
+        .create_opts = &qcow_create_opts,
+        .open_opts = &qcow_open_opts,
+
+        .expect_header = false,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+    },
+#ifdef TEST_LUKS
+    {
+        .path = "/crypto/block/luks/default",
+        .create_opts = &luks_create_opts_default,
+        .open_opts = &luks_open_opts,
+
+        .expect_header = true,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_XTS,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA256,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+
+        .slow = true,
+    },
+    {
+        .path = "/crypto/block/luks/aes-256-cbc-plain64",
+        .create_opts = &luks_create_opts_aes256_cbc_plain64,
+        .open_opts = &luks_open_opts,
+
+        .expect_header = true,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA256,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_PLAIN64,
+
+        .slow = true,
+    },
+    {
+        .path = "/crypto/block/luks/aes-256-cbc-essiv",
+        .create_opts = &luks_create_opts_aes256_cbc_essiv,
+        .open_opts = &luks_open_opts,
+
+        .expect_header = true,
+
+        .cipher_alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .cipher_mode = QCRYPTO_CIPHER_MODE_CBC,
+        .hash_alg = QCRYPTO_HASH_ALG_SHA1,
+
+        .ivgen_alg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .ivgen_hash = QCRYPTO_HASH_ALG_SHA256,
+
+        .slow = true,
+    },
+#endif
+};
+
+
+static ssize_t test_block_read_func(QCryptoBlock *block,
+                                    size_t offset,
+                                    uint8_t *buf,
+                                    size_t buflen,
+                                    Error **errp,
+                                    void *opaque)
+{
+    Buffer *header = opaque;
+
+    g_assert_cmpint(offset + buflen, <=, header->capacity);
+
+    memcpy(buf, header->buffer + offset, buflen);
+
+    return buflen;
+}
+
+
+static ssize_t test_block_init_func(QCryptoBlock *block,
+                                    size_t headerlen,
+                                    Error **errp,
+                                    void *opaque)
+{
+    Buffer *header = opaque;
+
+    g_assert_cmpint(header->capacity, ==, 0);
+
+    buffer_reserve(header, headerlen);
+
+    return headerlen;
+}
+
+
+static ssize_t test_block_write_func(QCryptoBlock *block,
+                                     size_t offset,
+                                     const uint8_t *buf,
+                                     size_t buflen,
+                                     Error **errp,
+                                     void *opaque)
+{
+    Buffer *header = opaque;
+
+    g_assert_cmpint(buflen + offset, <=, header->capacity);
+
+    memcpy(header->buffer + offset, buf, buflen);
+    header->offset = offset + buflen;
+
+    return buflen;
+}
+
+
+static Object *test_block_secret(void)
+{
+    return object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        NULL);
+}
+
+static void test_block_assert_setup(const struct QCryptoBlockTestData *data,
+                                    QCryptoBlock *blk)
+{
+    QCryptoIVGen *ivgen;
+    QCryptoCipher *cipher;
+
+    ivgen = qcrypto_block_get_ivgen(blk);
+    cipher = qcrypto_block_get_cipher(blk);
+
+    g_assert(ivgen);
+    g_assert(cipher);
+
+    g_assert_cmpint(data->cipher_alg, ==, cipher->alg);
+    g_assert_cmpint(data->cipher_mode, ==, cipher->mode);
+    g_assert_cmpint(data->hash_alg, ==,
+                    qcrypto_block_get_kdf_hash(blk));
+
+    g_assert_cmpint(data->ivgen_alg, ==,
+                    qcrypto_ivgen_get_algorithm(ivgen));
+    g_assert_cmpint(data->ivgen_hash, ==,
+                    qcrypto_ivgen_get_hash(ivgen));
+}
+
+
+static void test_block(gconstpointer opaque)
+{
+    const struct QCryptoBlockTestData *data = opaque;
+    QCryptoBlock *blk;
+    Buffer header;
+    Object *sec = test_block_secret();
+
+    memset(&header, 0, sizeof(header));
+    buffer_init(&header, "header");
+
+    blk = qcrypto_block_create(data->create_opts,
+                               test_block_init_func,
+                               test_block_write_func,
+                               &header,
+                               &error_abort);
+    g_assert(blk);
+
+    if (data->expect_header) {
+        g_assert_cmpint(header.capacity, >, 0);
+    } else {
+        g_assert_cmpint(header.capacity, ==, 0);
+    }
+
+    test_block_assert_setup(data, blk);
+
+    qcrypto_block_free(blk);
+    object_unparent(sec);
+
+    /* Ensure we can't open without the secret */
+    blk = qcrypto_block_open(data->open_opts,
+                             test_block_read_func,
+                             &header,
+                             0,
+                             NULL);
+    g_assert(blk == NULL);
+
+    /* Ensure we can't open without the secret, unless NO_IO */
+    blk = qcrypto_block_open(data->open_opts,
+                             test_block_read_func,
+                             &header,
+                             QCRYPTO_BLOCK_OPEN_NO_IO,
+                             &error_abort);
+
+    g_assert(qcrypto_block_get_cipher(blk) == NULL);
+    g_assert(qcrypto_block_get_ivgen(blk) == NULL);
+
+    qcrypto_block_free(blk);
+
+
+    /* Now open for real with secret */
+    sec = test_block_secret();
+    blk = qcrypto_block_open(data->open_opts,
+                             test_block_read_func,
+                             &header,
+                             0,
+                             &error_abort);
+    g_assert(blk);
+
+    test_block_assert_setup(data, blk);
+
+    qcrypto_block_free(blk);
+
+    object_unparent(sec);
+
+    buffer_free(&header);
+}
+
+
+int main(int argc, char **argv)
+{
+    gsize i;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (test_data[i].open_opts->format == Q_CRYPTO_BLOCK_FORMAT_LUKS &&
+            !qcrypto_hash_supports(test_data[i].hash_alg)) {
+            continue;
+        }
+        if (!test_data[i].slow ||
+            g_test_slow()) {
+            g_test_add_data_func(test_data[i].path, &test_data[i], test_block);
+        }
+    }
+
+    return g_test_run();
+}
diff --git a/tests/test-crypto-cipher.c b/tests/test-crypto-cipher.c
new file mode 100644
index 0000000..1b5130d
--- /dev/null
+++ b/tests/test-crypto-cipher.c
@@ -0,0 +1,604 @@
+/*
+ * QEMU Crypto cipher algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/init.h"
+#include "crypto/cipher.h"
+#include "qapi/error.h"
+
+typedef struct QCryptoCipherTestData QCryptoCipherTestData;
+struct QCryptoCipherTestData {
+    const char *path;
+    QCryptoCipherAlgorithm alg;
+    QCryptoCipherMode mode;
+    const char *key;
+    const char *plaintext;
+    const char *ciphertext;
+    const char *iv;
+};
+
+/* AES test data comes from appendix F of:
+ *
+ * http://csrc.nist.gov/publications/nistpubs/800-38a/sp800-38a.pdf
+ */
+static QCryptoCipherTestData test_data[] = {
+    {
+        /* NIST F.1.1 ECB-AES128.Encrypt */
+        .path = "/crypto/cipher/aes-ecb-128",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "2b7e151628aed2a6abf7158809cf4f3c",
+        .plaintext =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "3ad77bb40d7a3660a89ecaf32466ef97"
+            "f5d3d58503b9699de785895a96fdbaaf"
+            "43b1cd7f598ece23881b00e3ed030688"
+            "7b0c785e27e8ad3f8223207104725dd4"
+    },
+    {
+        /* NIST F.1.3 ECB-AES192.Encrypt */
+        .path = "/crypto/cipher/aes-ecb-192",
+        .alg = QCRYPTO_CIPHER_ALG_AES_192,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b",
+        .plaintext  =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "bd334f1d6e45f25ff712a214571fa5cc"
+            "974104846d0ad3ad7734ecb3ecee4eef"
+            "ef7afd2270e2e60adce0ba2face6444e"
+            "9a4b41ba738d6c72fb16691603c18e0e"
+    },
+    {
+        /* NIST F.1.5 ECB-AES256.Encrypt */
+        .path = "/crypto/cipher/aes-ecb-256",
+        .alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key =
+            "603deb1015ca71be2b73aef0857d7781"
+            "1f352c073b6108d72d9810a30914dff4",
+        .plaintext  =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "f3eed1bdb5d2a03c064b5a7e3db181f8"
+            "591ccb10d410ed26dc5ba74a31362870"
+            "b6ed21b99ca6f4f9f153e7b1beafed1d"
+            "23304b7a39f9f3ff067d8d8f9e24ecc7",
+    },
+    {
+        /* NIST F.2.1 CBC-AES128.Encrypt */
+        .path = "/crypto/cipher/aes-cbc-128",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_CBC,
+        .key = "2b7e151628aed2a6abf7158809cf4f3c",
+        .iv = "000102030405060708090a0b0c0d0e0f",
+        .plaintext  =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "7649abac8119b246cee98e9b12e9197d"
+            "5086cb9b507219ee95db113a917678b2"
+            "73bed6b8e3c1743b7116e69e22229516"
+            "3ff1caa1681fac09120eca307586e1a7",
+    },
+    {
+        /* NIST F.2.3 CBC-AES128.Encrypt */
+        .path = "/crypto/cipher/aes-cbc-192",
+        .alg = QCRYPTO_CIPHER_ALG_AES_192,
+        .mode = QCRYPTO_CIPHER_MODE_CBC,
+        .key = "8e73b0f7da0e6452c810f32b809079e562f8ead2522c6b7b",
+        .iv = "000102030405060708090a0b0c0d0e0f",
+        .plaintext  =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "4f021db243bc633d7178183a9fa071e8"
+            "b4d9ada9ad7dedf4e5e738763f69145a"
+            "571b242012fb7ae07fa9baac3df102e0"
+            "08b0e27988598881d920a9e64f5615cd",
+    },
+    {
+        /* NIST F.2.5 CBC-AES128.Encrypt */
+        .path = "/crypto/cipher/aes-cbc-256",
+        .alg = QCRYPTO_CIPHER_ALG_AES_256,
+        .mode = QCRYPTO_CIPHER_MODE_CBC,
+        .key =
+            "603deb1015ca71be2b73aef0857d7781"
+            "1f352c073b6108d72d9810a30914dff4",
+        .iv = "000102030405060708090a0b0c0d0e0f",
+        .plaintext  =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "f58c4c04d6e5f1ba779eabfb5f7bfbd6"
+            "9cfc4e967edb808d679f777bc6702c7d"
+            "39f23369a9d9bacfa530e26304231461"
+            "b2eb05e2c39be9fcda6c19078c6a9d1b",
+    },
+    {
+        .path = "/crypto/cipher/des-rfb-ecb-56",
+        .alg = QCRYPTO_CIPHER_ALG_DES_RFB,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "0123456789abcdef",
+        .plaintext =
+            "6bc1bee22e409f96e93d7e117393172a"
+            "ae2d8a571e03ac9c9eb76fac45af8e51"
+            "30c81c46a35ce411e5fbc1191a0a52ef"
+            "f69f2445df4f9b17ad2b417be66c3710",
+        .ciphertext =
+            "8f346aaf64eaf24040720d80648c52e7"
+            "aefc616be53ab1a3d301e69d91e01838"
+            "ffd29f1bb5596ad94ea2d8e6196b7f09"
+            "30d8ed0bf2773af36dd82a6280c20926",
+    },
+    {
+        /* RFC 2144, Appendix B.1 */
+        .path = "/crypto/cipher/cast5-128",
+        .alg = QCRYPTO_CIPHER_ALG_CAST5_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "0123456712345678234567893456789A",
+        .plaintext = "0123456789abcdef",
+        .ciphertext = "238b4fe5847e44b2",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-128",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000",
+        .plaintext = "d29d576fcea3a3a7ed9099f29273d78e",
+        .ciphertext = "b2288b968ae8b08648d1ce9606fd992d",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-192",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_192,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000"
+               "0000000000000000",
+        .plaintext = "d29d576fceaba3a7ed9899f2927bd78e",
+        .ciphertext = "130e353e1037c22405e8faefb2c3c3e9",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-256a",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000"
+               "00000000000000000000000000000000",
+        .plaintext = "d095576fcea3e3a7ed98d9f29073d78e",
+        .ciphertext = "b90ee5862de69168f2bdd5125b45472b",
+    },
+    {
+        /* libgcrypt serpent.c */
+        .path = "/crypto/cipher/serpent-256b",
+        .alg = QCRYPTO_CIPHER_ALG_SERPENT_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "00000000000000000000000000000000"
+               "00000000000000000000000000000000",
+        .plaintext = "00000000010000000200000003000000",
+        .ciphertext = "2061a42782bd52ec691ec383b03ba77c",
+    },
+    {
+        /* Twofish paper "Known Answer Test" */
+        .path = "/crypto/cipher/twofish-128",
+        .alg = QCRYPTO_CIPHER_ALG_TWOFISH_128,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "d491db16e7b1c39e86cb086b789f5419",
+        .plaintext = "019f9809de1711858faac3a3ba20fbc3",
+        .ciphertext = "6363977de839486297e661c6c9d668eb",
+    },
+    {
+        /* Twofish paper "Known Answer Test", I=3 */
+        .path = "/crypto/cipher/twofish-192",
+        .alg = QCRYPTO_CIPHER_ALG_TWOFISH_192,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "88b2b2706b105e36b446bb6d731a1e88"
+               "efa71f788965bd44",
+        .plaintext = "39da69d6ba4997d585b6dc073ca341b2",
+        .ciphertext = "182b02d81497ea45f9daacdc29193a65",
+    },
+    {
+        /* Twofish paper "Known Answer Test", I=4 */
+        .path = "/crypto/cipher/twofish-256",
+        .alg = QCRYPTO_CIPHER_ALG_TWOFISH_256,
+        .mode = QCRYPTO_CIPHER_MODE_ECB,
+        .key = "d43bb7556ea32e46f2a282b7d45b4e0d"
+               "57ff739d4dc92c1bd7fc01700cc8216f",
+        .plaintext = "90afe91bb288544f2c32dc239b2635e6",
+        .ciphertext = "6cb4561c40bf0a9705931cb6d408e7fa",
+    },
+    {
+        /* #1 32 byte key, 32 byte PTX */
+        .path = "/crypto/cipher/aes-xts-128-1",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "00000000000000000000000000000000"
+            "00000000000000000000000000000000",
+        .iv =
+            "00000000000000000000000000000000",
+        .plaintext =
+            "00000000000000000000000000000000"
+            "00000000000000000000000000000000",
+        .ciphertext =
+            "917cf69ebd68b2ec9b9fe9a3eadda692"
+            "cd43d2f59598ed858c02c2652fbf922e",
+    },
+    {
+        /* #2, 32 byte key, 32 byte PTX */
+        .path = "/crypto/cipher/aes-xts-128-2",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "11111111111111111111111111111111"
+            "22222222222222222222222222222222",
+        .iv =
+            "33333333330000000000000000000000",
+        .plaintext =
+            "44444444444444444444444444444444"
+            "44444444444444444444444444444444",
+        .ciphertext =
+            "c454185e6a16936e39334038acef838b"
+            "fb186fff7480adc4289382ecd6d394f0",
+    },
+    {
+        /* #5 from xts.7, 32 byte key, 32 byte PTX */
+        .path = "/crypto/cipher/aes-xts-128-3",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "fffefdfcfbfaf9f8f7f6f5f4f3f2f1f0"
+            "bfbebdbcbbbab9b8b7b6b5b4b3b2b1b0",
+        .iv =
+            "9a785634120000000000000000000000",
+        .plaintext =
+            "44444444444444444444444444444444"
+            "44444444444444444444444444444444",
+        .ciphertext =
+            "b01f86f8edc1863706fa8a4253e34f28"
+            "af319de38334870f4dd1f94cbe9832f1",
+    },
+    {
+        /* #4, 32 byte key, 512 byte PTX  */
+        .path = "/crypto/cipher/aes-xts-128-4",
+        .alg = QCRYPTO_CIPHER_ALG_AES_128,
+        .mode = QCRYPTO_CIPHER_MODE_XTS,
+        .key =
+            "27182818284590452353602874713526"
+            "31415926535897932384626433832795",
+        .iv =
+            "00000000000000000000000000000000",
+        .plaintext =
+            "000102030405060708090a0b0c0d0e0f"
+            "101112131415161718191a1b1c1d1e1f"
+            "202122232425262728292a2b2c2d2e2f"
+            "303132333435363738393a3b3c3d3e3f"
+            "404142434445464748494a4b4c4d4e4f"
+            "505152535455565758595a5b5c5d5e5f"
+            "606162636465666768696a6b6c6d6e6f"
+            "707172737475767778797a7b7c7d7e7f"
+            "808182838485868788898a8b8c8d8e8f"
+            "909192939495969798999a9b9c9d9e9f"
+            "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+            "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+            "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+            "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+            "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+            "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"
+            "000102030405060708090a0b0c0d0e0f"
+            "101112131415161718191a1b1c1d1e1f"
+            "202122232425262728292a2b2c2d2e2f"
+            "303132333435363738393a3b3c3d3e3f"
+            "404142434445464748494a4b4c4d4e4f"
+            "505152535455565758595a5b5c5d5e5f"
+            "606162636465666768696a6b6c6d6e6f"
+            "707172737475767778797a7b7c7d7e7f"
+            "808182838485868788898a8b8c8d8e8f"
+            "909192939495969798999a9b9c9d9e9f"
+            "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf"
+            "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf"
+            "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf"
+            "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf"
+            "e0e1e2e3e4e5e6e7e8e9eaebecedeeef"
+            "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff",
+        .ciphertext =
+            "27a7479befa1d476489f308cd4cfa6e2"
+            "a96e4bbe3208ff25287dd3819616e89c"
+            "c78cf7f5e543445f8333d8fa7f560000"
+            "05279fa5d8b5e4ad40e736ddb4d35412"
+            "328063fd2aab53e5ea1e0a9f332500a5"
+            "df9487d07a5c92cc512c8866c7e860ce"
+            "93fdf166a24912b422976146ae20ce84"
+            "6bb7dc9ba94a767aaef20c0d61ad0265"
+            "5ea92dc4c4e41a8952c651d33174be51"
+            "a10c421110e6d81588ede82103a252d8"
+            "a750e8768defffed9122810aaeb99f91"
+            "72af82b604dc4b8e51bcb08235a6f434"
+            "1332e4ca60482a4ba1a03b3e65008fc5"
+            "da76b70bf1690db4eae29c5f1badd03c"
+            "5ccf2a55d705ddcd86d449511ceb7ec3"
+            "0bf12b1fa35b913f9f747a8afd1b130e"
+            "94bff94effd01a91735ca1726acd0b19"
+            "7c4e5b03393697e126826fb6bbde8ecc"
+            "1e08298516e2c9ed03ff3c1b7860f6de"
+            "76d4cecd94c8119855ef5297ca67e9f3"
+            "e7ff72b1e99785ca0a7e7720c5b36dc6"
+            "d72cac9574c8cbbc2f801e23e56fd344"
+            "b07f22154beba0f08ce8891e643ed995"
+            "c94d9a69c9f1b5f499027a78572aeebd"
+            "74d20cc39881c213ee770b1010e4bea7"
+            "18846977ae119f7a023ab58cca0ad752"
+            "afe656bb3c17256a9f6e9bf19fdd5a38"
+            "fc82bbe872c5539edb609ef4f79c203e"
+            "bb140f2e583cb2ad15b4aa5b655016a8"
+            "449277dbd477ef2c8d6c017db738b18d"
+            "eb4a427d1923ce3ff262735779a418f2"
+            "0a282df920147beabe421ee5319d0568",
+    },
+};
+
+
+static inline int unhex(char c)
+{
+    if (c >= 'a' && c <= 'f') {
+        return 10 + (c - 'a');
+    }
+    if (c >= 'A' && c <= 'F') {
+        return 10 + (c - 'A');
+    }
+    return c - '0';
+}
+
+static inline char hex(int i)
+{
+    if (i < 10) {
+        return '0' + i;
+    }
+    return 'a' + (i - 10);
+}
+
+static size_t unhex_string(const char *hexstr,
+                           uint8_t **data)
+{
+    size_t len;
+    size_t i;
+
+    if (!hexstr) {
+        *data = NULL;
+        return 0;
+    }
+
+    len = strlen(hexstr);
+    *data = g_new0(uint8_t, len / 2);
+
+    for (i = 0; i < len; i += 2) {
+        (*data)[i/2] = (unhex(hexstr[i]) << 4) | unhex(hexstr[i+1]);
+    }
+    return len / 2;
+}
+
+static char *hex_string(const uint8_t *bytes,
+                        size_t len)
+{
+    char *hexstr = g_new0(char, len * 2 + 1);
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        hexstr[i*2] = hex((bytes[i] >> 4) & 0xf);
+        hexstr[i*2+1] = hex(bytes[i] & 0xf);
+    }
+    hexstr[len*2] = '\0';
+
+    return hexstr;
+}
+
+static void test_cipher(const void *opaque)
+{
+    const QCryptoCipherTestData *data = opaque;
+
+    QCryptoCipher *cipher;
+    uint8_t *key, *iv, *ciphertext, *plaintext, *outtext;
+    size_t nkey, niv, nciphertext, nplaintext;
+    char *outtexthex;
+    size_t ivsize, keysize, blocksize;
+
+    nkey = unhex_string(data->key, &key);
+    niv = unhex_string(data->iv, &iv);
+    nciphertext = unhex_string(data->ciphertext, &ciphertext);
+    nplaintext = unhex_string(data->plaintext, &plaintext);
+
+    g_assert(nciphertext == nplaintext);
+
+    outtext = g_new0(uint8_t, nciphertext);
+
+    cipher = qcrypto_cipher_new(
+        data->alg, data->mode,
+        key, nkey,
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    keysize = qcrypto_cipher_get_key_len(data->alg);
+    blocksize = qcrypto_cipher_get_block_len(data->alg);
+    ivsize = qcrypto_cipher_get_iv_len(data->alg, data->mode);
+
+    if (data->mode == QCRYPTO_CIPHER_MODE_XTS) {
+        g_assert_cmpint(keysize * 2, ==, nkey);
+    } else {
+        g_assert_cmpint(keysize, ==, nkey);
+    }
+    g_assert_cmpint(ivsize, ==, niv);
+    if (niv) {
+        g_assert_cmpint(blocksize, ==, niv);
+    }
+
+    if (iv) {
+        g_assert(qcrypto_cipher_setiv(cipher,
+                                      iv, niv,
+                                      &error_abort) == 0);
+    }
+    g_assert(qcrypto_cipher_encrypt(cipher,
+                                    plaintext,
+                                    outtext,
+                                    nplaintext,
+                                    &error_abort) == 0);
+
+    outtexthex = hex_string(outtext, nciphertext);
+
+    g_assert_cmpstr(outtexthex, ==, data->ciphertext);
+
+    g_free(outtexthex);
+
+    if (iv) {
+        g_assert(qcrypto_cipher_setiv(cipher,
+                                      iv, niv,
+                                      &error_abort) == 0);
+    }
+    g_assert(qcrypto_cipher_decrypt(cipher,
+                                    ciphertext,
+                                    outtext,
+                                    nplaintext,
+                                    &error_abort) == 0);
+
+    outtexthex = hex_string(outtext, nplaintext);
+
+    g_assert_cmpstr(outtexthex, ==, data->plaintext);
+
+    g_free(outtext);
+    g_free(outtexthex);
+    g_free(key);
+    g_free(iv);
+    g_free(ciphertext);
+    g_free(plaintext);
+    qcrypto_cipher_free(cipher);
+}
+
+
+static void test_cipher_null_iv(void)
+{
+    QCryptoCipher *cipher;
+    uint8_t key[32] = { 0 };
+    uint8_t plaintext[32] = { 0 };
+    uint8_t ciphertext[32] = { 0 };
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_256,
+        QCRYPTO_CIPHER_MODE_CBC,
+        key, sizeof(key),
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    /* Don't call qcrypto_cipher_setiv */
+
+    qcrypto_cipher_encrypt(cipher,
+                           plaintext,
+                           ciphertext,
+                           sizeof(plaintext),
+                           &error_abort);
+
+    qcrypto_cipher_free(cipher);
+}
+
+static void test_cipher_short_plaintext(void)
+{
+    Error *err = NULL;
+    QCryptoCipher *cipher;
+    uint8_t key[32] = { 0 };
+    uint8_t plaintext1[20] = { 0 };
+    uint8_t ciphertext1[20] = { 0 };
+    uint8_t plaintext2[40] = { 0 };
+    uint8_t ciphertext2[40] = { 0 };
+    int ret;
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_AES_256,
+        QCRYPTO_CIPHER_MODE_CBC,
+        key, sizeof(key),
+        &error_abort);
+    g_assert(cipher != NULL);
+
+    /* Should report an error as plaintext is shorter
+     * than block size
+     */
+    ret = qcrypto_cipher_encrypt(cipher,
+                                 plaintext1,
+                                 ciphertext1,
+                                 sizeof(plaintext1),
+                                 &err);
+    g_assert(ret == -1);
+    g_assert(err != NULL);
+
+    error_free(err);
+    err = NULL;
+
+    /* Should report an error as plaintext is larger than
+     * block size, but not a multiple of block size
+     */
+    ret = qcrypto_cipher_encrypt(cipher,
+                                 plaintext2,
+                                 ciphertext2,
+                                 sizeof(plaintext2),
+                                 &err);
+    g_assert(ret == -1);
+    g_assert(err != NULL);
+
+    error_free(err);
+    qcrypto_cipher_free(cipher);
+}
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (qcrypto_cipher_supports(test_data[i].alg)) {
+            g_test_add_data_func(test_data[i].path, &test_data[i], test_cipher);
+        }
+    }
+
+    g_test_add_func("/crypto/cipher/null-iv",
+                    test_cipher_null_iv);
+
+    g_test_add_func("/crypto/cipher/short-plaintext",
+                    test_cipher_short_plaintext);
+
+    return g_test_run();
+}
diff --git a/tests/test-crypto-hash.c b/tests/test-crypto-hash.c
new file mode 100644
index 0000000..42fc77a
--- /dev/null
+++ b/tests/test-crypto-hash.c
@@ -0,0 +1,263 @@
+/*
+ * QEMU Crypto hash algorithms
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/init.h"
+#include "crypto/hash.h"
+
+#define INPUT_TEXT "Hiss hisss Hissss hiss Hiss hisss Hiss hiss"
+#define INPUT_TEXT1 "Hiss hisss "
+#define INPUT_TEXT2 "Hissss hiss "
+#define INPUT_TEXT3 "Hiss hisss Hiss hiss"
+
+#define OUTPUT_MD5 "628d206371563035ab8ef62f492bdec9"
+#define OUTPUT_SHA1 "b2e74f26758a3a421e509cee045244b78753cc02"
+#define OUTPUT_SHA224 "e2f7415aad33ef79f6516b0986d7175f" \
+                      "9ca3389a85bf6cfed078737b"
+#define OUTPUT_SHA256 "bc757abb0436586f392b437e5dd24096" \
+                      "f7f224de6b74d4d86e2abc6121b160d0"
+#define OUTPUT_SHA384 "887ce52efb4f46700376356583b7e279" \
+                      "4f612bd024e4495087ddb946c448c69d" \
+                      "56dbf7152a94a5e63a80f3ba9f0eed78"
+#define OUTPUT_SHA512 "3a90d79638235ec6c4c11bebd84d83c0" \
+                      "549bc1e84edc4b6ec7086487641256cb" \
+                      "63b54e4cb2d2032b393994aa263c0dbb" \
+                      "e00a9f2fe9ef6037352232a1eec55ee7"
+#define OUTPUT_RIPEMD160 "f3d658fad3fdfb2b52c9369cf0d441249ddfa8a0"
+
+#define OUTPUT_MD5_B64 "Yo0gY3FWMDWrjvYvSSveyQ=="
+#define OUTPUT_SHA1_B64 "sudPJnWKOkIeUJzuBFJEt4dTzAI="
+#define OUTPUT_SHA224_B64 "4vdBWq0z73n2UWsJhtcXX5yjOJqFv2z+0Hhzew=="
+#define OUTPUT_SHA256_B64 "vHV6uwQ2WG85K0N+XdJAlvfyJN5rdNTYbiq8YSGxYNA="
+#define OUTPUT_SHA384_B64 "iHzlLvtPRnADdjVlg7fieU9hK9Ak5ElQh925RsRI" \
+                          "xp1W2/cVKpSl5jqA87qfDu14"
+#define OUTPUT_SHA512_B64 "OpDXljgjXsbEwRvr2E2DwFSbwehO3Etuxwhkh2QS" \
+                          "VstjtU5MstIDKzk5lKomPA274AqfL+nvYDc1IjKh" \
+                          "7sVe5w=="
+#define OUTPUT_RIPEMD160_B64 "89ZY+tP9+ytSyTac8NRBJJ3fqKA="
+
+static const char *expected_outputs[] = {
+    [QCRYPTO_HASH_ALG_MD5] = OUTPUT_MD5,
+    [QCRYPTO_HASH_ALG_SHA1] = OUTPUT_SHA1,
+    [QCRYPTO_HASH_ALG_SHA224] = OUTPUT_SHA224,
+    [QCRYPTO_HASH_ALG_SHA256] = OUTPUT_SHA256,
+    [QCRYPTO_HASH_ALG_SHA384] = OUTPUT_SHA384,
+    [QCRYPTO_HASH_ALG_SHA512] = OUTPUT_SHA512,
+    [QCRYPTO_HASH_ALG_RIPEMD160] = OUTPUT_RIPEMD160,
+};
+static const char *expected_outputs_b64[] = {
+    [QCRYPTO_HASH_ALG_MD5] = OUTPUT_MD5_B64,
+    [QCRYPTO_HASH_ALG_SHA1] = OUTPUT_SHA1_B64,
+    [QCRYPTO_HASH_ALG_SHA224] = OUTPUT_SHA224_B64,
+    [QCRYPTO_HASH_ALG_SHA256] = OUTPUT_SHA256_B64,
+    [QCRYPTO_HASH_ALG_SHA384] = OUTPUT_SHA384_B64,
+    [QCRYPTO_HASH_ALG_SHA512] = OUTPUT_SHA512_B64,
+    [QCRYPTO_HASH_ALG_RIPEMD160] = OUTPUT_RIPEMD160_B64,
+};
+static const int expected_lens[] = {
+    [QCRYPTO_HASH_ALG_MD5] = 16,
+    [QCRYPTO_HASH_ALG_SHA1] = 20,
+    [QCRYPTO_HASH_ALG_SHA224] = 28,
+    [QCRYPTO_HASH_ALG_SHA256] = 32,
+    [QCRYPTO_HASH_ALG_SHA384] = 48,
+    [QCRYPTO_HASH_ALG_SHA512] = 64,
+    [QCRYPTO_HASH_ALG_RIPEMD160] = 20,
+};
+
+static const char hex[] = "0123456789abcdef";
+
+/* Test with dynamic allocation */
+static void test_hash_alloc(void)
+{
+    size_t i;
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(expected_outputs) ; i++) {
+        uint8_t *result = NULL;
+        size_t resultlen = 0;
+        int ret;
+        size_t j;
+
+        if (!qcrypto_hash_supports(i)) {
+            continue;
+        }
+
+        ret = qcrypto_hash_bytes(i,
+                                 INPUT_TEXT,
+                                 strlen(INPUT_TEXT),
+                                 &result,
+                                 &resultlen,
+                                 NULL);
+        g_assert(ret == 0);
+        g_assert(resultlen == expected_lens[i]);
+
+        for (j = 0; j < resultlen; j++) {
+            g_assert(expected_outputs[i][j * 2] == hex[(result[j] >> 4) & 0xf]);
+            g_assert(expected_outputs[i][j * 2 + 1] == hex[result[j] & 0xf]);
+        }
+        g_free(result);
+    }
+}
+
+/* Test with caller preallocating */
+static void test_hash_prealloc(void)
+{
+    size_t i;
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(expected_outputs) ; i++) {
+        uint8_t *result;
+        size_t resultlen;
+        int ret;
+        size_t j;
+
+        if (!qcrypto_hash_supports(i)) {
+            continue;
+        }
+
+        resultlen = expected_lens[i];
+        result = g_new0(uint8_t, resultlen);
+
+        ret = qcrypto_hash_bytes(i,
+                                 INPUT_TEXT,
+                                 strlen(INPUT_TEXT),
+                                 &result,
+                                 &resultlen,
+                                 NULL);
+        g_assert(ret == 0);
+
+        g_assert(resultlen == expected_lens[i]);
+        for (j = 0; j < resultlen; j++) {
+            g_assert(expected_outputs[i][j * 2] == hex[(result[j] >> 4) & 0xf]);
+            g_assert(expected_outputs[i][j * 2 + 1] == hex[result[j] & 0xf]);
+        }
+        g_free(result);
+    }
+}
+
+
+/* Test with dynamic allocation */
+static void test_hash_iov(void)
+{
+    size_t i;
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(expected_outputs) ; i++) {
+        struct iovec iov[3] = {
+            { .iov_base = (char *)INPUT_TEXT1, .iov_len = strlen(INPUT_TEXT1) },
+            { .iov_base = (char *)INPUT_TEXT2, .iov_len = strlen(INPUT_TEXT2) },
+            { .iov_base = (char *)INPUT_TEXT3, .iov_len = strlen(INPUT_TEXT3) },
+        };
+        uint8_t *result = NULL;
+        size_t resultlen = 0;
+        int ret;
+        size_t j;
+
+        if (!qcrypto_hash_supports(i)) {
+            continue;
+        }
+
+        ret = qcrypto_hash_bytesv(i,
+                                  iov, 3,
+                                  &result,
+                                  &resultlen,
+                                  NULL);
+        g_assert(ret == 0);
+        g_assert(resultlen == expected_lens[i]);
+        for (j = 0; j < resultlen; j++) {
+            g_assert(expected_outputs[i][j * 2] == hex[(result[j] >> 4) & 0xf]);
+            g_assert(expected_outputs[i][j * 2 + 1] == hex[result[j] & 0xf]);
+        }
+        g_free(result);
+    }
+}
+
+
+/* Test with printable hashing */
+static void test_hash_digest(void)
+{
+    size_t i;
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(expected_outputs) ; i++) {
+        int ret;
+        char *digest;
+        size_t digestsize;
+
+        if (!qcrypto_hash_supports(i)) {
+            continue;
+        }
+
+        digestsize = qcrypto_hash_digest_len(i);
+
+        g_assert_cmpint(digestsize * 2, ==, strlen(expected_outputs[i]));
+
+        ret = qcrypto_hash_digest(i,
+                                  INPUT_TEXT,
+                                  strlen(INPUT_TEXT),
+                                  &digest,
+                                  NULL);
+        g_assert(ret == 0);
+        g_assert_cmpstr(digest, ==, expected_outputs[i]);
+        g_free(digest);
+    }
+}
+
+/* Test with base64 encoding */
+static void test_hash_base64(void)
+{
+    size_t i;
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(expected_outputs) ; i++) {
+        int ret;
+        char *digest;
+
+        if (!qcrypto_hash_supports(i)) {
+            continue;
+        }
+
+        ret = qcrypto_hash_base64(i,
+                                  INPUT_TEXT,
+                                  strlen(INPUT_TEXT),
+                                  &digest,
+                                  NULL);
+        g_assert(ret == 0);
+        g_assert_cmpstr(digest, ==, expected_outputs_b64[i]);
+        g_free(digest);
+    }
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/crypto/hash/iov", test_hash_iov);
+    g_test_add_func("/crypto/hash/alloc", test_hash_alloc);
+    g_test_add_func("/crypto/hash/prealloc", test_hash_prealloc);
+    g_test_add_func("/crypto/hash/digest", test_hash_digest);
+    g_test_add_func("/crypto/hash/base64", test_hash_base64);
+    return g_test_run();
+}
diff --git a/tests/test-crypto-ivgen.c b/tests/test-crypto-ivgen.c
new file mode 100644
index 0000000..a5ff5d3
--- /dev/null
+++ b/tests/test-crypto-ivgen.c
@@ -0,0 +1,174 @@
+/*
+ * QEMU Crypto IV generator algorithms
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/ivgen.h"
+
+
+struct QCryptoIVGenTestData {
+    const char *path;
+    uint64_t sector;
+    QCryptoIVGenAlgorithm ivalg;
+    QCryptoHashAlgorithm hashalg;
+    QCryptoCipherAlgorithm cipheralg;
+    const uint8_t *key;
+    size_t nkey;
+    const uint8_t *iv;
+    size_t niv;
+} test_data[] = {
+    /* Small */
+    {
+        "/crypto/ivgen/plain/1",
+        .sector = 0x1,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN,
+        .iv = (const uint8_t *)"\x01\x00\x00\x00\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Big ! */
+    {
+        "/crypto/ivgen/plain/1f2e3d4c",
+        .sector = 0x1f2e3d4cULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN,
+        .iv = (const uint8_t *)"\x4c\x3d\x2e\x1f\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Truncation */
+    {
+        "/crypto/ivgen/plain/1f2e3d4c5b6a7988",
+        .sector = 0x1f2e3d4c5b6a7988ULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN,
+        .iv = (const uint8_t *)"\x88\x79\x6a\x5b\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Small */
+    {
+        "/crypto/ivgen/plain64/1",
+        .sector = 0x1,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN64,
+        .iv = (const uint8_t *)"\x01\x00\x00\x00\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Big ! */
+    {
+        "/crypto/ivgen/plain64/1f2e3d4c",
+        .sector = 0x1f2e3d4cULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN64,
+        .iv = (const uint8_t *)"\x4c\x3d\x2e\x1f\x00\x00\x00\x00"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* No Truncation */
+    {
+        "/crypto/ivgen/plain64/1f2e3d4c5b6a7988",
+        .sector = 0x1f2e3d4c5b6a7988ULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_PLAIN64,
+        .iv = (const uint8_t *)"\x88\x79\x6a\x5b\x4c\x3d\x2e\x1f"
+                               "\x00\x00\x00\x00\x00\x00\x00\x00",
+        .niv = 16,
+    },
+    /* Small */
+    {
+        "/crypto/ivgen/essiv/1",
+        .sector = 0x1,
+        .ivalg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .cipheralg = QCRYPTO_CIPHER_ALG_AES_128,
+        .hashalg = QCRYPTO_HASH_ALG_SHA256,
+        .key = (const uint8_t *)"\x00\x01\x02\x03\x04\x05\x06\x07"
+                                "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        .nkey = 16,
+        .iv = (const uint8_t *)"\xd4\x83\x71\xb2\xa1\x94\x53\x88"
+                               "\x1c\x7a\x2d\06\x2d\x0b\x65\x46",
+        .niv = 16,
+    },
+    /* Big ! */
+    {
+        "/crypto/ivgen/essiv/1f2e3d4c",
+        .sector = 0x1f2e3d4cULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .cipheralg = QCRYPTO_CIPHER_ALG_AES_128,
+        .hashalg = QCRYPTO_HASH_ALG_SHA256,
+        .key = (const uint8_t *)"\x00\x01\x02\x03\x04\x05\x06\x07"
+                                "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        .nkey = 16,
+        .iv = (const uint8_t *)"\x5d\x36\x09\x5d\xc6\x9e\x5e\xe9"
+                               "\xe3\x02\x8d\xd8\x7a\x3d\xe7\x8f",
+        .niv = 16,
+    },
+    /* No Truncation */
+    {
+        "/crypto/ivgen/essiv/1f2e3d4c5b6a7988",
+        .sector = 0x1f2e3d4c5b6a7988ULL,
+        .ivalg = QCRYPTO_IVGEN_ALG_ESSIV,
+        .cipheralg = QCRYPTO_CIPHER_ALG_AES_128,
+        .hashalg = QCRYPTO_HASH_ALG_SHA256,
+        .key = (const uint8_t *)"\x00\x01\x02\x03\x04\x05\x06\x07"
+                                "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f",
+        .nkey = 16,
+        .iv = (const uint8_t *)"\x58\xbb\x81\x94\x51\x83\x23\x23"
+                               "\x7a\x08\x93\xa9\xdc\xd2\xd9\xab",
+        .niv = 16,
+    },
+};
+
+
+static void test_ivgen(const void *opaque)
+{
+    const struct QCryptoIVGenTestData *data = opaque;
+    uint8_t *iv = g_new0(uint8_t, data->niv);
+    QCryptoIVGen *ivgen = qcrypto_ivgen_new(
+        data->ivalg,
+        data->cipheralg,
+        data->hashalg,
+        data->key,
+        data->nkey,
+        &error_abort);
+
+    qcrypto_ivgen_calculate(ivgen,
+                            data->sector,
+                            iv,
+                            data->niv,
+                            &error_abort);
+
+    g_assert(memcmp(iv, data->iv, data->niv) == 0);
+
+    qcrypto_ivgen_free(ivgen);
+    g_free(iv);
+}
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    g_test_init(&argc, &argv, NULL);
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (test_data[i].ivalg == QCRYPTO_IVGEN_ALG_ESSIV &&
+            !qcrypto_hash_supports(test_data[i].hashalg)) {
+            continue;
+        }
+        g_test_add_data_func(test_data[i].path,
+                             &(test_data[i]),
+                             test_ivgen);
+    }
+    return g_test_run();
+}
diff --git a/tests/test-crypto-pbkdf.c b/tests/test-crypto-pbkdf.c
new file mode 100644
index 0000000..8ceceb1
--- /dev/null
+++ b/tests/test-crypto-pbkdf.c
@@ -0,0 +1,393 @@
+/*
+ * QEMU Crypto cipher algorithms
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "crypto/init.h"
+#ifndef _WIN32
+#include <sys/resource.h>
+#endif
+
+#if ((defined(CONFIG_NETTLE) || defined(CONFIG_GCRYPT)) && \
+     (defined(_WIN32) || defined(RUSAGE_THREAD)))
+#include "crypto/pbkdf.h"
+
+typedef struct QCryptoPbkdfTestData QCryptoPbkdfTestData;
+struct QCryptoPbkdfTestData {
+    const char *path;
+    QCryptoHashAlgorithm hash;
+    unsigned int iterations;
+    const char *key;
+    size_t nkey;
+    const char *salt;
+    size_t nsalt;
+    const char *out;
+    size_t nout;
+    bool slow;
+};
+
+/* This test data comes from cryptsetup package
+ *
+ *  $SRC/lib/crypto_backend/pbkdf2_generic.c
+ *
+ * under LGPLv2.1+ license
+ */
+static QCryptoPbkdfTestData test_data[] = {
+    /* RFC 3962 test data */
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1,
+        .key = "password",
+        .nkey = 8,
+        .salt = "ATHENA.MIT.EDUraeburn",
+        .nsalt = 21,
+        .out = "\xcd\xed\xb5\x28\x1b\xb2\xf8\x01"
+               "\x56\x5a\x11\x22\xb2\x56\x35\x15"
+               "\x0a\xd1\xf7\xa0\x4b\xb9\xf3\xa3"
+               "\x33\xec\xc0\xe2\xe1\xf7\x08\x37",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter2",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 2,
+        .key = "password",
+        .nkey = 8,
+        .salt = "ATHENA.MIT.EDUraeburn",
+        .nsalt = 21,
+        .out = "\x01\xdb\xee\x7f\x4a\x9e\x24\x3e"
+               "\x98\x8b\x62\xc7\x3c\xda\x93\x5d"
+               "\xa0\x53\x78\xb9\x32\x44\xec\x8f"
+               "\x48\xa9\x9e\x61\xad\x79\x9d\x86",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1200a",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1200,
+        .key = "password",
+        .nkey = 8,
+        .salt = "ATHENA.MIT.EDUraeburn",
+        .nsalt = 21,
+        .out = "\x5c\x08\xeb\x61\xfd\xf7\x1e\x4e"
+               "\x4e\xc3\xcf\x6b\xa1\xf5\x51\x2b"
+               "\xa7\xe5\x2d\xdb\xc5\xe5\x14\x2f"
+               "\x70\x8a\x31\xe2\xe6\x2b\x1e\x13",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter5",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 5,
+        .key = "password",
+        .nkey = 8,
+        .salt = "\0224VxxV4\022", /* "\x1234567878563412 */
+        .nsalt = 8,
+        .out = "\xd1\xda\xa7\x86\x15\xf2\x87\xe6"
+               "\xa1\xc8\xb1\x20\xd7\x06\x2a\x49"
+               "\x3f\x98\xd2\x03\xe6\xbe\x49\xa6"
+               "\xad\xf4\xfa\x57\x4b\x6e\x64\xee",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1200b",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 64,
+        .salt = "pass phrase equals block size",
+        .nsalt = 29,
+        .out = "\x13\x9c\x30\xc0\x96\x6b\xc3\x2b"
+               "\xa5\x5f\xdb\xf2\x12\x53\x0a\xc9"
+               "\xc5\xec\x59\xf1\xa4\x52\xf5\xcc"
+               "\x9a\xd9\x40\xfe\xa0\x59\x8e\xd1",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter1200c",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 65,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x9c\xca\xd6\xd4\x68\x77\x0c\xd5"
+               "\x1b\x10\xe6\xa6\x87\x21\xbe\x61"
+               "\x1a\x8b\x4d\x28\x26\x01\xdb\x3b"
+               "\x36\xbe\x92\x46\x91\x5e\xc8\x2a",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/rfc3962/sha1/iter50",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 50,
+        .key = "\360\235\204\236", /* g-clef ("\xf09d849e) */
+        .nkey = 4,
+        .salt = "EXAMPLE.COMpianist",
+        .nsalt = 18,
+        .out = "\x6b\x9c\xf2\x6d\x45\x45\x5a\x43"
+               "\xa5\xb8\xbb\x27\x6a\x40\x3b\x39"
+               "\xe7\xfe\x37\xa0\xc4\x1e\x02\xc2"
+               "\x81\xff\x30\x69\xe1\xe9\x4f\x52",
+        .nout = 32
+    },
+
+    /* RFC-6070 test data */
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter1",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 1,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\x0c\x60\xc8\x0f\x96\x1f\x0e\x71\xf3\xa9"
+               "\xb5\x24\xaf\x60\x12\x06\x2f\xe0\x37\xa6",
+        .nout = 20
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter2",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 2,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\xea\x6c\x01\x4d\xc7\x2d\x6f\x8c\xcd\x1e"
+               "\xd9\x2a\xce\x1d\x41\xf0\xd8\xde\x89\x57",
+        .nout = 20
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter4096",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 4096,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\x4b\x00\x79\x01\xb7\x65\x48\x9a\xbe\xad"
+               "\x49\xd9\x26\xf7\x21\xd0\x65\xa4\x29\xc1",
+        .nout = 20
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter16777216",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 16777216,
+        .key = "password",
+        .nkey = 8,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\xee\xfe\x3d\x61\xcd\x4d\xa4\xe4\xe9\x94"
+               "\x5b\x3d\x6b\xa2\x15\x8c\x26\x34\xe9\x84",
+        .nout = 20,
+        .slow = true,
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter4096a",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 4096,
+        .key = "passwordPASSWORDpassword",
+        .nkey = 24,
+        .salt = "saltSALTsaltSALTsaltSALTsaltSALTsalt",
+        .nsalt = 36,
+        .out = "\x3d\x2e\xec\x4f\xe4\x1c\x84\x9b\x80\xc8"
+               "\xd8\x36\x62\xc0\xe4\x4a\x8b\x29\x1a\x96"
+               "\x4c\xf2\xf0\x70\x38",
+        .nout = 25
+    },
+    {
+        .path = "/crypto/pbkdf/rfc6070/sha1/iter4096b",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 4096,
+        .key = "pass\0word",
+        .nkey = 9,
+        .salt = "sa\0lt",
+        .nsalt = 5,
+        .out = "\x56\xfa\x6a\xa7\x55\x48\x09\x9d\xcc\x37"
+               "\xd7\xf0\x34\x25\xe0\xc3",
+        .nout = 16
+    },
+
+    /* non-RFC misc test data */
+#ifdef CONFIG_NETTLE
+    {
+        /* empty password test.
+         * Broken with libgcrypt <= 1.5.0, hence CONFIG_NETTLE */
+        .path = "/crypto/pbkdf/nonrfc/sha1/iter2",
+        .hash = QCRYPTO_HASH_ALG_SHA1,
+        .iterations = 2,
+        .key = "",
+        .nkey = 0,
+        .salt = "salt",
+        .nsalt = 4,
+        .out = "\x13\x3a\x4c\xe8\x37\xb4\xd2\x52\x1e\xe2"
+               "\xbf\x03\xe1\x1c\x71\xca\x79\x4e\x07\x97",
+        .nout = 20
+    },
+#endif
+    {
+        /* Password exceeds block size test */
+        .path = "/crypto/pbkdf/nonrfc/sha256/iter1200",
+        .hash = QCRYPTO_HASH_ALG_SHA256,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 65,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x22\x34\x4b\xc4\xb6\xe3\x26\x75"
+               "\xa8\x09\x0f\x3e\xa8\x0b\xe0\x1d"
+               "\x5f\x95\x12\x6a\x2c\xdd\xc3\xfa"
+               "\xcc\x4a\x5e\x6d\xca\x04\xec\x58",
+        .nout = 32
+    },
+#if 0
+    {
+        .path = "/crypto/pbkdf/nonrfc/sha512/iter1200",
+        .hash = QCRYPTO_HASH_ALG_SHA512,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 129,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x0f\xb2\xed\x2c\x0e\x6e\xfb\x7d"
+               "\x7d\x8e\xdd\x58\x01\xb4\x59\x72"
+               "\x99\x92\x16\x30\x5e\xa4\x36\x8d"
+               "\x76\x14\x80\xf3\xe3\x7a\x22\xb9",
+        .nout = 32
+    },
+    {
+        .path = "/crypto/pbkdf/nonrfc/whirlpool/iter1200",
+        .hash = QCRYPTO_HASH_ALG_WHIRLPOOL,
+        .iterations = 1200,
+        .key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
+               "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
+        .nkey = 65,
+        .salt = "pass phrase exceeds block size",
+        .nsalt = 30,
+        .out = "\x9c\x1c\x74\xf5\x88\x26\xe7\x6a"
+               "\x53\x58\xf4\x0c\x39\xe7\x80\x89"
+               "\x07\xc0\x31\x19\x9a\x50\xa2\x48"
+               "\xf1\xd9\xfe\x78\x64\xe5\x84\x50",
+        .nout = 32
+    }
+#endif
+};
+
+
+static inline char hex(int i)
+{
+    if (i < 10) {
+        return '0' + i;
+    }
+    return 'a' + (i - 10);
+}
+
+static char *hex_string(const uint8_t *bytes,
+                        size_t len)
+{
+    char *hexstr = g_new0(char, len * 2 + 1);
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        hexstr[i * 2] = hex((bytes[i] >> 4) & 0xf);
+        hexstr[i * 2 + 1] = hex(bytes[i] & 0xf);
+    }
+    hexstr[len * 2] = '\0';
+
+    return hexstr;
+}
+
+static void test_pbkdf(const void *opaque)
+{
+    const QCryptoPbkdfTestData *data = opaque;
+    size_t nout = data->nout;
+    uint8_t *out = g_new0(uint8_t, nout);
+    gchar *expect, *actual;
+
+    qcrypto_pbkdf2(data->hash,
+                   (uint8_t *)data->key, data->nkey,
+                   (uint8_t *)data->salt, data->nsalt,
+                   data->iterations,
+                   (uint8_t *)out, nout,
+                   &error_abort);
+
+    expect = hex_string((const uint8_t *)data->out, data->nout);
+    actual = hex_string(out, nout);
+
+    g_assert_cmpstr(actual, ==, expect);
+
+    g_free(actual);
+    g_free(expect);
+    g_free(out);
+}
+
+
+static void test_pbkdf_timing(void)
+{
+    uint8_t key[32];
+    uint8_t salt[32];
+    int iters;
+
+    memset(key, 0x5d, sizeof(key));
+    memset(salt, 0x7c, sizeof(salt));
+
+    iters = qcrypto_pbkdf2_count_iters(QCRYPTO_HASH_ALG_SHA256,
+                                       key, sizeof(key),
+                                       salt, sizeof(salt),
+                                       &error_abort);
+
+    g_assert(iters >= (1 << 15));
+}
+
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        if (!test_data[i].slow ||
+            g_test_slow()) {
+            g_test_add_data_func(test_data[i].path, &test_data[i], test_pbkdf);
+        }
+    }
+
+    if (g_test_slow()) {
+        g_test_add_func("/crypt0/pbkdf/timing", test_pbkdf_timing);
+    }
+
+    return g_test_run();
+}
+#else
+int main(int argc, char **argv)
+{
+    return 0;
+}
+#endif
diff --git a/tests/test-crypto-secret.c b/tests/test-crypto-secret.c
new file mode 100644
index 0000000..13fc6c4
--- /dev/null
+++ b/tests/test-crypto-secret.c
@@ -0,0 +1,456 @@
+/*
+ * QEMU Crypto secret handling
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto/init.h"
+#include "crypto/secret.h"
+#include "qapi/error.h"
+#include "qemu/module.h"
+
+static void test_secret_direct(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_indirect_good(void)
+{
+    Object *sec;
+    char *fname = NULL;
+    int fd = g_file_open_tmp("qemu-test-crypto-secret-XXXXXX",
+                             &fname,
+                             NULL);
+
+    g_assert(fd >= 0);
+    g_assert_nonnull(fname);
+
+    g_assert(write(fd, "123456", 6) == 6);
+
+    sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "file", fname,
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+    close(fd);
+    unlink(fname);
+    g_free(fname);
+}
+
+
+static void test_secret_indirect_badfile(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "file", "does-not-exist",
+        NULL);
+
+    g_assert(sec == NULL);
+}
+
+
+static void test_secret_indirect_emptyfile(void)
+{
+    Object *sec;
+    char *fname = NULL;
+    int fd = g_file_open_tmp("qemu-test-crypto-secretXXXXXX",
+                             &fname,
+                             NULL);
+
+    g_assert(fd >= 0);
+    g_assert_nonnull(fname);
+
+    sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "file", fname,
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "");
+
+    object_unparent(sec);
+    g_free(pw);
+    close(fd);
+    unlink(fname);
+    g_free(fname);
+}
+
+
+static void test_secret_noconv_base64_good(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "MTIzNDU2",
+        "format", "base64",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_base64("sec0",
+                                               &error_abort);
+
+    g_assert_cmpstr(pw, ==, "MTIzNDU2");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_noconv_base64_bad(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "MTI$NDU2",
+        "format", "base64",
+        NULL);
+
+    g_assert(sec == NULL);
+}
+
+
+static void test_secret_noconv_utf8(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        "format", "raw",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_conv_base64_utf8valid(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "MTIzNDU2",
+        "format", "base64",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_conv_base64_utf8invalid(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "f0VMRgIBAQAAAA==",
+        "format", "base64",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             NULL);
+    g_assert(pw == NULL);
+
+    object_unparent(sec);
+}
+
+
+static void test_secret_conv_utf8_base64(void)
+{
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "123456",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_base64("sec0",
+                                               &error_abort);
+
+    g_assert_cmpstr(pw, ==, "MTIzNDU2");
+
+    object_unparent(sec);
+    g_free(pw);
+}
+
+
+static void test_secret_crypt_raw(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data",
+        "\xCC\xBF\xF7\x09\x46\x19\x0B\x52\x2A\x3A\xB4\x6B\xCD\x7A\xB0\xB0",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2apQ3g==",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    object_unparent(master);
+    g_free(pw);
+}
+
+
+static void test_secret_crypt_base64(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        &error_abort,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "base64",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2apQ3g==",
+        NULL);
+
+    char *pw = qcrypto_secret_lookup_as_utf8("sec0",
+                                             &error_abort);
+
+    g_assert_cmpstr(pw, ==, "123456");
+
+    object_unparent(sec);
+    object_unparent(master);
+    g_free(pw);
+}
+
+
+static void test_secret_crypt_short_key(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVc",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2apQ3g==",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+static void test_secret_crypt_short_iv(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TKuA+Old2W2a",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+static void test_secret_crypt_missing_iv(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+static void test_secret_crypt_bad_iv(void)
+{
+    Object *master = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "master",
+        &error_abort,
+        "data", "9miloPQCzGy+TL6aonfzVcptibCmCIhKzrnlfwiWivk=",
+        "format", "base64",
+        NULL);
+    Object *sec = object_new_with_props(
+        TYPE_QCRYPTO_SECRET,
+        object_get_objects_root(),
+        "sec0",
+        NULL,
+        "data", "zL/3CUYZC1IqOrRrzXqwsA==",
+        "format", "raw",
+        "keyid", "master",
+        "iv", "0I7Gw/TK$$uA+Old2W2a",
+        NULL);
+
+    g_assert(sec == NULL);
+    object_unparent(master);
+}
+
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    g_test_add_func("/crypto/secret/direct",
+                    test_secret_direct);
+    g_test_add_func("/crypto/secret/indirect/good",
+                    test_secret_indirect_good);
+    g_test_add_func("/crypto/secret/indirect/badfile",
+                    test_secret_indirect_badfile);
+    g_test_add_func("/crypto/secret/indirect/emptyfile",
+                    test_secret_indirect_emptyfile);
+
+    g_test_add_func("/crypto/secret/noconv/base64/good",
+                    test_secret_noconv_base64_good);
+    g_test_add_func("/crypto/secret/noconv/base64/bad",
+                    test_secret_noconv_base64_bad);
+    g_test_add_func("/crypto/secret/noconv/utf8",
+                    test_secret_noconv_utf8);
+    g_test_add_func("/crypto/secret/conv/base64/utf8valid",
+                    test_secret_conv_base64_utf8valid);
+    g_test_add_func("/crypto/secret/conv/base64/utf8invalid",
+                    test_secret_conv_base64_utf8invalid);
+    g_test_add_func("/crypto/secret/conv/utf8/base64",
+                    test_secret_conv_utf8_base64);
+
+    g_test_add_func("/crypto/secret/crypt/raw",
+                    test_secret_crypt_raw);
+    g_test_add_func("/crypto/secret/crypt/base64",
+                    test_secret_crypt_base64);
+    g_test_add_func("/crypto/secret/crypt/shortkey",
+                    test_secret_crypt_short_key);
+    g_test_add_func("/crypto/secret/crypt/shortiv",
+                    test_secret_crypt_short_iv);
+    g_test_add_func("/crypto/secret/crypt/missingiv",
+                    test_secret_crypt_missing_iv);
+    g_test_add_func("/crypto/secret/crypt/badiv",
+                    test_secret_crypt_bad_iv);
+
+    return g_test_run();
+}
diff --git a/tests/test-crypto-tlscredsx509.c b/tests/test-crypto-tlscredsx509.c
new file mode 100644
index 0000000..af2f80e
--- /dev/null
+++ b/tests/test-crypto-tlscredsx509.c
@@ -0,0 +1,730 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "qapi/error.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-crypto-tlscredsx509-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QCryptoTLSCredsTestData {
+    bool isServer;
+    const char *cacrt;
+    const char *crt;
+    bool expectFail;
+};
+
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        "testtlscreds",
+        errp,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        "sanity-check", "yes",
+        NULL);
+
+    if (*errp) {
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+/*
+ * This tests sanity checking of our own certificates
+ *
+ * The code being tested is used when TLS creds are created,
+ * and aim to ensure QMEU has been configured with sane
+ * certificates. This allows us to give much much much
+ * clearer error messages to the admin when they misconfigure
+ * things.
+ */
+static void test_tls_creds(const void *opaque)
+{
+    struct QCryptoTLSCredsTestData *data =
+        (struct QCryptoTLSCredsTestData *)opaque;
+    QCryptoTLSCreds *creds;
+    Error *err = NULL;
+
+#define CERT_DIR "tests/test-crypto-tlscredsx509-certs/"
+    mkdir(CERT_DIR, 0700);
+
+    unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    if (data->isServer) {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+    } else {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+    }
+
+    if (access(data->cacrt, R_OK) == 0) {
+        g_assert(link(data->cacrt,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    }
+    if (data->isServer) {
+        if (access(data->crt, R_OK) == 0) {
+            g_assert(link(data->crt,
+                          CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+        }
+        g_assert(link(KEYFILE,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+    } else {
+        if (access(data->crt, R_OK) == 0) {
+            g_assert(link(data->crt,
+                          CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+        }
+        g_assert(link(KEYFILE,
+                      CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+    }
+
+    creds = test_tls_creds_create(
+        (data->isServer ?
+         QCRYPTO_TLS_CREDS_ENDPOINT_SERVER :
+         QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT),
+        CERT_DIR,
+        &err);
+
+    if (data->expectFail) {
+        error_free(err);
+        g_assert(creds == NULL);
+    } else {
+        if (err) {
+            g_printerr("Failed to generate creds: %s\n",
+                       error_get_pretty(err));
+            error_free(err);
+        }
+        g_assert(creds != NULL);
+    }
+
+    unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    if (data->isServer) {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+    } else {
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+        unlink(CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+    }
+    rmdir(CERT_DIR);
+    if (creds) {
+        object_unparent(OBJECT(creds));
+    }
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TLS_TEST_REG(name, isServer, caCrt, crt, expectFail)           \
+    struct QCryptoTLSCredsTestData name = {                             \
+        isServer, caCrt, crt, expectFail                                \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlscredsx509/" # name,               \
+                         &name, test_tls_creds);                        \
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(perfectserver, true,
+                 cacertreq.filename, servercertreq.filename, false);
+    TLS_TEST_REG(perfectclient, false,
+                 cacertreq.filename, clientcertreq.filename, false);
+
+
+    /* Some other CAs which are good */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacert1req,
+                 "UK", "qemu CA 1", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert1req, cacert1req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Basic:CA:not-critical */
+    TLS_ROOT_REQ(cacert2req,
+                 "UK", "qemu CA 2", NULL, NULL, NULL, NULL,
+                 true, false, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert2req, cacert2req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Key usage:cert-sign:critical */
+    TLS_ROOT_REQ(cacert3req,
+                 "UK", "qemu CA 3", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert3req, cacert3req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(goodca1, true,
+                 cacert1req.filename, servercert1req.filename, false);
+    TLS_TEST_REG(goodca2, true,
+                 cacert2req.filename, servercert2req.filename, false);
+    TLS_TEST_REG(goodca3, true,
+                 cacert3req.filename, servercert3req.filename, false);
+
+    /* Now some bad certs */
+
+    /* Key usage:dig-sig:not-critical */
+    TLS_ROOT_REQ(cacert4req,
+                 "UK", "qemu CA 4", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, false, GNUTLS_KEY_DIGITAL_SIGNATURE,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert4req, cacert4req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* no-basic */
+    TLS_ROOT_REQ(cacert5req,
+                 "UK", "qemu CA 5", NULL, NULL, NULL, NULL,
+                 false, false, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert5req, cacert5req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* Key usage:dig-sig:critical */
+    TLS_ROOT_REQ(cacert6req,
+                 "UK", "qemu CA 6", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_DIGITAL_SIGNATURE,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercert6req, cacert6req,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    /* Technically a CA cert with basic constraints
+     * key purpose == key signing + non-critical should
+     * be rejected. GNUTLS < 3.1 does not reject it and
+     * we don't anticipate them changing this behaviour
+     */
+    TLS_TEST_REG(badca1, true, cacert4req.filename, servercert4req.filename,
+                (GNUTLS_VERSION_MAJOR == 3 && GNUTLS_VERSION_MINOR >= 1) ||
+                GNUTLS_VERSION_MAJOR > 3);
+    TLS_TEST_REG(badca2, true,
+                 cacert5req.filename, servercert5req.filename, true);
+    TLS_TEST_REG(badca3, true,
+                 cacert6req.filename, servercert6req.filename, true);
+
+
+    /* Various good servers */
+    /* no usage or purpose */
+    TLS_CERT_REQ(servercert7req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign+dig-sig+encipher:critical */
+    TLS_CERT_REQ(servercert8req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT |
+                 GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign:not-critical */
+    TLS_CERT_REQ(servercert9req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, false, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:server:critical */
+    TLS_CERT_REQ(servercert10req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* purpose:server:not-critical */
+    TLS_CERT_REQ(servercert11req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* purpose:client+server:critical */
+    TLS_CERT_REQ(servercert12req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+    /* purpose:client+server:not-critical */
+    TLS_CERT_REQ(servercert13req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+
+    TLS_TEST_REG(goodserver1, true,
+                 cacertreq.filename, servercert7req.filename, false);
+    TLS_TEST_REG(goodserver2, true,
+                 cacertreq.filename, servercert8req.filename, false);
+    TLS_TEST_REG(goodserver3, true,
+                 cacertreq.filename, servercert9req.filename, false);
+    TLS_TEST_REG(goodserver4, true,
+                 cacertreq.filename, servercert10req.filename, false);
+    TLS_TEST_REG(goodserver5, true,
+                 cacertreq.filename, servercert11req.filename, false);
+    TLS_TEST_REG(goodserver6, true,
+                 cacertreq.filename, servercert12req.filename, false);
+    TLS_TEST_REG(goodserver7, true,
+                 cacertreq.filename, servercert13req.filename, false);
+
+    /* Bad servers */
+
+    /* usage:cert-sign:critical */
+    TLS_CERT_REQ(servercert14req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(servercert15req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* usage: none:critical */
+    TLS_CERT_REQ(servercert16req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(badserver1, true,
+                 cacertreq.filename, servercert14req.filename, true);
+    TLS_TEST_REG(badserver2, true,
+                 cacertreq.filename, servercert15req.filename, true);
+    TLS_TEST_REG(badserver3, true,
+                 cacertreq.filename, servercert16req.filename, true);
+
+
+
+    /* Various good clients */
+    /* no usage or purpose */
+    TLS_CERT_REQ(clientcert1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign+dig-sig+encipher:critical */
+    TLS_CERT_REQ(clientcert2req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT |
+                 GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* usage:cert-sign:not-critical */
+    TLS_CERT_REQ(clientcert3req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, false, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(clientcert4req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* purpose:client:not-critical */
+    TLS_CERT_REQ(clientcert5req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+    /* purpose:client+client:critical */
+    TLS_CERT_REQ(clientcert6req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+    /* purpose:client+client:not-critical */
+    TLS_CERT_REQ(clientcert7req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, false,
+                 GNUTLS_KP_TLS_WWW_CLIENT, GNUTLS_KP_TLS_WWW_SERVER,
+                 0, 0);
+
+    TLS_TEST_REG(goodclient1, false,
+                 cacertreq.filename, clientcert1req.filename, false);
+    TLS_TEST_REG(goodclient2, false,
+                 cacertreq.filename, clientcert2req.filename, false);
+    TLS_TEST_REG(goodclient3, false,
+                 cacertreq.filename, clientcert3req.filename, false);
+    TLS_TEST_REG(goodclient4, false,
+                 cacertreq.filename, clientcert4req.filename, false);
+    TLS_TEST_REG(goodclient5, false,
+                 cacertreq.filename, clientcert5req.filename, false);
+    TLS_TEST_REG(goodclient6, false,
+                 cacertreq.filename, clientcert6req.filename, false);
+    TLS_TEST_REG(goodclient7, false,
+                 cacertreq.filename, clientcert7req.filename, false);
+
+    /* Bad clients */
+
+    /* usage:cert-sign:critical */
+    TLS_CERT_REQ(clientcert8req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    /* purpose:client:critical */
+    TLS_CERT_REQ(clientcert9req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 false, false, 0,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* usage: none:critical */
+    TLS_CERT_REQ(clientcert10req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_TEST_REG(badclient1, false,
+                 cacertreq.filename, clientcert8req.filename, true);
+    TLS_TEST_REG(badclient2, false,
+                 cacertreq.filename, clientcert9req.filename, true);
+    TLS_TEST_REG(badclient3, false,
+                 cacertreq.filename, clientcert10req.filename, true);
+
+
+
+    /* Expired stuff */
+
+    TLS_ROOT_REQ(cacertexpreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, -1);
+    TLS_CERT_REQ(servercertexpreq, cacertexpreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertexp1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, -1);
+    TLS_CERT_REQ(clientcertexp1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, -1);
+
+    TLS_TEST_REG(expired1, true,
+                 cacertexpreq.filename, servercertexpreq.filename, true);
+    TLS_TEST_REG(expired2, true,
+                 cacertreq.filename, servercertexp1req.filename, true);
+    TLS_TEST_REG(expired3, false,
+                 cacertreq.filename, clientcertexp1req.filename, true);
+
+
+    /* Not activated stuff */
+
+    TLS_ROOT_REQ(cacertnewreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 1, 2);
+    TLS_CERT_REQ(servercertnewreq, cacertnewreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertnew1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 1, 2);
+    TLS_CERT_REQ(clientcertnew1req, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 1, 2);
+
+    TLS_TEST_REG(inactive1, true,
+                 cacertnewreq.filename, servercertnewreq.filename, true);
+    TLS_TEST_REG(inactive2, true,
+                 cacertreq.filename, servercertnew1req.filename, true);
+    TLS_TEST_REG(inactive3, false,
+                 cacertreq.filename, clientcertnew1req.filename, true);
+
+    TLS_ROOT_REQ(cacertrootreq,
+                 "UK", "qemu root", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1areq, cacertrootreq,
+                 "UK", "qemu level 1a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1breq, cacertrootreq,
+                 "UK", "qemu level 1b", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel2areq, cacertlevel1areq,
+                 "UK", "qemu level 2a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertlevel3areq, cacertlevel2areq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertlevel2breq, cacertlevel1breq,
+                 "UK", "qemu client level 2b", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    gnutls_x509_crt_t certchain[] = {
+        cacertrootreq.crt,
+        cacertlevel1areq.crt,
+        cacertlevel1breq.crt,
+        cacertlevel2areq.crt,
+    };
+
+    test_tls_write_cert_chain(WORKDIR "cacertchain-ctx.pem",
+                              certchain,
+                              G_N_ELEMENTS(certchain));
+
+    TLS_TEST_REG(chain1, true,
+                 WORKDIR "cacertchain-ctx.pem",
+                 servercertlevel3areq.filename, false);
+    TLS_TEST_REG(chain2, false,
+                 WORKDIR "cacertchain-ctx.pem",
+                 clientcertlevel2breq.filename, false);
+
+    /* Some missing certs - first two are fatal, the last
+     * is ok
+     */
+    TLS_TEST_REG(missingca, true,
+                 "cacertdoesnotexist.pem",
+                 servercert1req.filename, true);
+    TLS_TEST_REG(missingserver, true,
+                 cacert1req.filename,
+                 "servercertdoesnotexist.pem", true);
+    TLS_TEST_REG(missingclient, false,
+                 cacert1req.filename,
+                 "clientcertdoesnotexist.pem", false);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&cacertreq);
+    test_tls_discard_cert(&cacert1req);
+    test_tls_discard_cert(&cacert2req);
+    test_tls_discard_cert(&cacert3req);
+    test_tls_discard_cert(&cacert4req);
+    test_tls_discard_cert(&cacert5req);
+    test_tls_discard_cert(&cacert6req);
+
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&servercert1req);
+    test_tls_discard_cert(&servercert2req);
+    test_tls_discard_cert(&servercert3req);
+    test_tls_discard_cert(&servercert4req);
+    test_tls_discard_cert(&servercert5req);
+    test_tls_discard_cert(&servercert6req);
+    test_tls_discard_cert(&servercert7req);
+    test_tls_discard_cert(&servercert8req);
+    test_tls_discard_cert(&servercert9req);
+    test_tls_discard_cert(&servercert10req);
+    test_tls_discard_cert(&servercert11req);
+    test_tls_discard_cert(&servercert12req);
+    test_tls_discard_cert(&servercert13req);
+    test_tls_discard_cert(&servercert14req);
+    test_tls_discard_cert(&servercert15req);
+    test_tls_discard_cert(&servercert16req);
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&clientcert1req);
+    test_tls_discard_cert(&clientcert2req);
+    test_tls_discard_cert(&clientcert3req);
+    test_tls_discard_cert(&clientcert4req);
+    test_tls_discard_cert(&clientcert5req);
+    test_tls_discard_cert(&clientcert6req);
+    test_tls_discard_cert(&clientcert7req);
+    test_tls_discard_cert(&clientcert8req);
+    test_tls_discard_cert(&clientcert9req);
+    test_tls_discard_cert(&clientcert10req);
+
+    test_tls_discard_cert(&cacertexpreq);
+    test_tls_discard_cert(&servercertexpreq);
+    test_tls_discard_cert(&servercertexp1req);
+    test_tls_discard_cert(&clientcertexp1req);
+
+    test_tls_discard_cert(&cacertnewreq);
+    test_tls_discard_cert(&servercertnewreq);
+    test_tls_discard_cert(&servercertnew1req);
+    test_tls_discard_cert(&clientcertnew1req);
+
+    test_tls_discard_cert(&cacertrootreq);
+    test_tls_discard_cert(&cacertlevel1areq);
+    test_tls_discard_cert(&cacertlevel1breq);
+    test_tls_discard_cert(&cacertlevel2areq);
+    test_tls_discard_cert(&servercertlevel3areq);
+    test_tls_discard_cert(&clientcertlevel2breq);
+    unlink(WORKDIR "cacertchain-ctx.pem");
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/test-crypto-tlssession.c b/tests/test-crypto-tlssession.c
new file mode 100644
index 0000000..1a4a066
--- /dev/null
+++ b/tests/test-crypto-tlssession.c
@@ -0,0 +1,534 @@
+/*
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "crypto/tlssession.h"
+#include "qom/object_interfaces.h"
+#include "qapi/error.h"
+#include "qemu/sockets.h"
+#include "qemu/acl.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-crypto-tlssession-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QCryptoTLSSessionTestData {
+    const char *servercacrt;
+    const char *clientcacrt;
+    const char *servercrt;
+    const char *clientcrt;
+    bool expectServerFail;
+    bool expectClientFail;
+    const char *hostname;
+    const char *const *wildcards;
+};
+
+
+static ssize_t testWrite(const char *buf, size_t len, void *opaque)
+{
+    int *fd = opaque;
+
+    return write(*fd, buf, len);
+}
+
+static ssize_t testRead(char *buf, size_t len, void *opaque)
+{
+    int *fd = opaque;
+
+    return read(*fd, buf, len);
+}
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Error *err = NULL;
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+         "testtlscredsserver" : "testtlscredsclient"),
+        &err,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        /* We skip initial sanity checks here because we
+         * want to make sure that problems are being
+         * detected at the TLS session validation stage,
+         * and the test-crypto-tlscreds test already
+         * validate the sanity check code.
+         */
+        "sanity-check", "no",
+        NULL
+        );
+
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
+/*
+ * This tests validation checking of peer certificates
+ *
+ * This is replicating the checks that are done for an
+ * active TLS session after handshake completes. To
+ * simulate that we create our TLS contexts, skipping
+ * sanity checks. We then get a socketpair, and
+ * initiate a TLS session across them. Finally do
+ * do actual cert validation tests
+ */
+static void test_crypto_tls_session(const void *opaque)
+{
+    struct QCryptoTLSSessionTestData *data =
+        (struct QCryptoTLSSessionTestData *)opaque;
+    QCryptoTLSCreds *clientCreds;
+    QCryptoTLSCreds *serverCreds;
+    QCryptoTLSSession *clientSess = NULL;
+    QCryptoTLSSession *serverSess = NULL;
+    qemu_acl *acl;
+    const char * const *wildcards;
+    int channel[2];
+    bool clientShake = false;
+    bool serverShake = false;
+    Error *err = NULL;
+    int ret;
+
+    /* We'll use this for our fake client-server connection */
+    ret = socketpair(AF_UNIX, SOCK_STREAM, 0, channel);
+    g_assert(ret == 0);
+
+    /*
+     * We have an evil loop to do the handshake in a single
+     * thread, so we need these non-blocking to avoid deadlock
+     * of ourselves
+     */
+    qemu_set_nonblock(channel[0]);
+    qemu_set_nonblock(channel[1]);
+
+#define CLIENT_CERT_DIR "tests/test-crypto-tlssession-client/"
+#define SERVER_CERT_DIR "tests/test-crypto-tlssession-server/"
+    mkdir(CLIENT_CERT_DIR, 0700);
+    mkdir(SERVER_CERT_DIR, 0700);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    g_assert(link(data->servercacrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->servercrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+
+    g_assert(link(data->clientcacrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->clientcrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+
+    clientCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        CLIENT_CERT_DIR,
+        &err);
+    g_assert(clientCreds != NULL);
+
+    serverCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        SERVER_CERT_DIR,
+        &err);
+    g_assert(serverCreds != NULL);
+
+    acl = qemu_acl_init("tlssessionacl");
+    qemu_acl_reset(acl);
+    wildcards = data->wildcards;
+    while (wildcards && *wildcards) {
+        qemu_acl_append(acl, 0, *wildcards);
+        wildcards++;
+    }
+
+    /* Now the real part of the test, setup the sessions */
+    clientSess = qcrypto_tls_session_new(
+        clientCreds, data->hostname, NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT, &err);
+    serverSess = qcrypto_tls_session_new(
+        serverCreds, NULL,
+        data->wildcards ? "tlssessionacl" : NULL,
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER, &err);
+
+    g_assert(clientSess != NULL);
+    g_assert(serverSess != NULL);
+
+    /* For handshake to work, we need to set the I/O callbacks
+     * to read/write over the socketpair
+     */
+    qcrypto_tls_session_set_callbacks(serverSess,
+                                      testWrite, testRead,
+                                      &channel[0]);
+    qcrypto_tls_session_set_callbacks(clientSess,
+                                      testWrite, testRead,
+                                      &channel[1]);
+
+    /*
+     * Finally we loop around & around doing handshake on each
+     * session until we get an error, or the handshake completes.
+     * This relies on the socketpair being nonblocking to avoid
+     * deadlocking ourselves upon handshake
+     */
+    do {
+        int rv;
+        if (!serverShake) {
+            rv = qcrypto_tls_session_handshake(serverSess,
+                                               &err);
+            g_assert(rv >= 0);
+            if (qcrypto_tls_session_get_handshake_status(serverSess) ==
+                QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+                serverShake = true;
+            }
+        }
+        if (!clientShake) {
+            rv = qcrypto_tls_session_handshake(clientSess,
+                                               &err);
+            g_assert(rv >= 0);
+            if (qcrypto_tls_session_get_handshake_status(clientSess) ==
+                QCRYPTO_TLS_HANDSHAKE_COMPLETE) {
+                clientShake = true;
+            }
+        }
+    } while (!clientShake && !serverShake);
+
+
+    /* Finally make sure the server validation does what
+     * we were expecting
+     */
+    if (qcrypto_tls_session_check_credentials(serverSess, &err) < 0) {
+        g_assert(data->expectServerFail);
+        error_free(err);
+        err = NULL;
+    } else {
+        g_assert(!data->expectServerFail);
+    }
+
+    /*
+     * And the same for the client validation check
+     */
+    if (qcrypto_tls_session_check_credentials(clientSess, &err) < 0) {
+        g_assert(data->expectClientFail);
+        error_free(err);
+        err = NULL;
+    } else {
+        g_assert(!data->expectClientFail);
+    }
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    rmdir(CLIENT_CERT_DIR);
+    rmdir(SERVER_CERT_DIR);
+
+    object_unparent(OBJECT(serverCreds));
+    object_unparent(OBJECT(clientCreds));
+
+    qcrypto_tls_session_free(serverSess);
+    qcrypto_tls_session_free(clientSess);
+
+    close(channel[0]);
+    close(channel[1]);
+}
+
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TEST_SESS_REG(name, caCrt,                                     \
+                       serverCrt, clientCrt,                            \
+                       expectServerFail, expectClientFail,              \
+                       hostname, wildcards)                             \
+    struct QCryptoTLSSessionTestData name = {                           \
+        caCrt, caCrt, serverCrt, clientCrt,                             \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlssession/" # name,                 \
+                         &name, test_crypto_tls_session);               \
+
+
+# define TEST_SESS_REG_EXT(name, serverCaCrt, clientCaCrt,              \
+                           serverCrt, clientCrt,                        \
+                           expectServerFail, expectClientFail,          \
+                           hostname, wildcards)                         \
+    struct QCryptoTLSSessionTestData name = {                           \
+        serverCaCrt, clientCaCrt, serverCrt, clientCrt,                 \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qcrypto/tlssession/" # name,                 \
+                         &name, test_crypto_tls_session);               \
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_ROOT_REQ(altcacertreq,
+                 "UK", "qemu CA 1", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 false, false, 0,
+                 false, false, NULL, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TLS_CERT_REQ(clientcertaltreq, altcacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    TEST_SESS_REG(basicca, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", NULL);
+    TEST_SESS_REG_EXT(differentca, cacertreq.filename,
+                      altcacertreq.filename, servercertreq.filename,
+                      clientcertaltreq.filename, true, true, "qemu.org", NULL);
+
+
+    /* When an altname is set, the CN is ignored, so it must be duplicated
+     * as an altname for it to match */
+    TLS_CERT_REQ(servercertalt1req, cacertreq,
+                 "UK", "qemu.org", "www.qemu.org", "qemu.org",
+                 "192.168.122.1", "fec0::dead:beaf",
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    /* This intentionally doesn't replicate */
+    TLS_CERT_REQ(servercertalt2req, cacertreq,
+                 "UK", "qemu.org", "www.qemu.org", "wiki.qemu.org",
+                 "192.168.122.1", "fec0::dead:beaf",
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+
+    TEST_SESS_REG(altname1, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, false, "qemu.org", NULL);
+    TEST_SESS_REG(altname2, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, false, "www.qemu.org", NULL);
+    TEST_SESS_REG(altname3, cacertreq.filename,
+                  servercertalt1req.filename, clientcertreq.filename,
+                  false, true, "wiki.qemu.org", NULL);
+
+    TEST_SESS_REG(altname4, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, true, "qemu.org", NULL);
+    TEST_SESS_REG(altname5, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, false, "www.qemu.org", NULL);
+    TEST_SESS_REG(altname6, cacertreq.filename,
+                  servercertalt2req.filename, clientcertreq.filename,
+                  false, false, "wiki.qemu.org", NULL);
+
+    const char *const wildcards1[] = {
+        "C=UK,CN=dogfood",
+        NULL,
+    };
+    const char *const wildcards2[] = {
+        "C=UK,CN=qemu",
+        NULL,
+    };
+    const char *const wildcards3[] = {
+        "C=UK,CN=dogfood",
+        "C=UK,CN=qemu",
+        NULL,
+    };
+    const char *const wildcards4[] = {
+        "C=UK,CN=qemustuff",
+        NULL,
+    };
+    const char *const wildcards5[] = {
+        "C=UK,CN=qemu*",
+        NULL,
+    };
+    const char *const wildcards6[] = {
+        "C=UK,CN=*emu*",
+        NULL,
+    };
+
+    TEST_SESS_REG(wildcard1, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  true, false, "qemu.org", wildcards1);
+    TEST_SESS_REG(wildcard2, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards2);
+    TEST_SESS_REG(wildcard3, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards3);
+    TEST_SESS_REG(wildcard4, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  true, false, "qemu.org", wildcards4);
+    TEST_SESS_REG(wildcard5, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards5);
+    TEST_SESS_REG(wildcard6, cacertreq.filename,
+                  servercertreq.filename, clientcertreq.filename,
+                  false, false, "qemu.org", wildcards6);
+
+    TLS_ROOT_REQ(cacertrootreq,
+                 "UK", "qemu root", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1areq, cacertrootreq,
+                 "UK", "qemu level 1a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel1breq, cacertrootreq,
+                 "UK", "qemu level 1b", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(cacertlevel2areq, cacertlevel1areq,
+                 "UK", "qemu level 2a", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertlevel3areq, cacertlevel2areq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertlevel2breq, cacertlevel1breq,
+                 "UK", "qemu client level 2b", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    gnutls_x509_crt_t certchain[] = {
+        cacertrootreq.crt,
+        cacertlevel1areq.crt,
+        cacertlevel1breq.crt,
+        cacertlevel2areq.crt,
+    };
+
+    test_tls_write_cert_chain(WORKDIR "cacertchain-sess.pem",
+                              certchain,
+                              G_N_ELEMENTS(certchain));
+
+    TEST_SESS_REG(cachain, WORKDIR "cacertchain-sess.pem",
+                  servercertlevel3areq.filename, clientcertlevel2breq.filename,
+                  false, false, "qemu.org", NULL);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&clientcertaltreq);
+
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&servercertalt1req);
+    test_tls_discard_cert(&servercertalt2req);
+
+    test_tls_discard_cert(&cacertreq);
+    test_tls_discard_cert(&altcacertreq);
+
+    test_tls_discard_cert(&cacertrootreq);
+    test_tls_discard_cert(&cacertlevel1areq);
+    test_tls_discard_cert(&cacertlevel1breq);
+    test_tls_discard_cert(&cacertlevel2areq);
+    test_tls_discard_cert(&servercertlevel3areq);
+    test_tls_discard_cert(&clientcertlevel2breq);
+    unlink(WORKDIR "cacertchain-sess.pem");
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/test-crypto-xts.c b/tests/test-crypto-xts.c
new file mode 100644
index 0000000..1f1412c
--- /dev/null
+++ b/tests/test-crypto-xts.c
@@ -0,0 +1,423 @@
+/*
+ * QEMU Crypto XTS cipher mode
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ * This code is originally derived from public domain / WTFPL code in
+ * LibTomCrypt crytographic library http://libtom.org. The XTS code
+ * was donated by Elliptic Semiconductor Inc (www.ellipticsemi.com)
+ * to the LibTom Projects
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "crypto/init.h"
+#include "crypto/xts.h"
+#include "crypto/aes.h"
+
+typedef struct {
+    const char *path;
+    int keylen;
+    unsigned char key1[32];
+    unsigned char key2[32];
+    uint64_t seqnum;
+    unsigned long PTLEN;
+    unsigned char PTX[512], CTX[512];
+} QCryptoXTSTestData;
+
+static const QCryptoXTSTestData test_data[] = {
+    /* #1 32 byte key, 32 byte PTX */
+    {
+        "/crypto/xts/t-1-key-32-ptx-32",
+        32,
+        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+        0,
+        32,
+        { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+          0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
+        { 0x91, 0x7c, 0xf6, 0x9e, 0xbd, 0x68, 0xb2, 0xec,
+          0x9b, 0x9f, 0xe9, 0xa3, 0xea, 0xdd, 0xa6, 0x92,
+          0xcd, 0x43, 0xd2, 0xf5, 0x95, 0x98, 0xed, 0x85,
+          0x8c, 0x02, 0xc2, 0x65, 0x2f, 0xbf, 0x92, 0x2e },
+    },
+
+    /* #2, 32 byte key, 32 byte PTX */
+    {
+        "/crypto/xts/t-2-key-32-ptx-32",
+        32,
+        { 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11,
+          0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
+        { 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22,
+          0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22 },
+        0x3333333333LL,
+        32,
+        { 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44 },
+        { 0xc4, 0x54, 0x18, 0x5e, 0x6a, 0x16, 0x93, 0x6e,
+          0x39, 0x33, 0x40, 0x38, 0xac, 0xef, 0x83, 0x8b,
+          0xfb, 0x18, 0x6f, 0xff, 0x74, 0x80, 0xad, 0xc4,
+          0x28, 0x93, 0x82, 0xec, 0xd6, 0xd3, 0x94, 0xf0 },
+    },
+
+    /* #5 from xts.7, 32 byte key, 32 byte PTX */
+    {
+        "/crypto/xts/t-5-key-32-ptx-32",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        32,
+        { 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44,
+          0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44 },
+        { 0xb0, 0x1f, 0x86, 0xf8, 0xed, 0xc1, 0x86, 0x37,
+          0x06, 0xfa, 0x8a, 0x42, 0x53, 0xe3, 0x4f, 0x28,
+          0xaf, 0x31, 0x9d, 0xe3, 0x83, 0x34, 0x87, 0x0f,
+          0x4d, 0xd1, 0xf9, 0x4c, 0xbe, 0x98, 0x32, 0xf1 },
+    },
+
+    /* #4, 32 byte key, 512 byte PTX  */
+    {
+        "/crypto/xts/t-4-key-32-ptx-512",
+        32,
+        { 0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45,
+          0x23, 0x53, 0x60, 0x28, 0x74, 0x71, 0x35, 0x26 },
+        { 0x31, 0x41, 0x59, 0x26, 0x53, 0x58, 0x97, 0x93,
+          0x23, 0x84, 0x62, 0x64, 0x33, 0x83, 0x27, 0x95 },
+        0,
+        512,
+        {
+            0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+            0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+            0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+            0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+            0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+            0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+            0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+            0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+            0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+            0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+            0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+            0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+            0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+            0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+            0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+            0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
+            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+            0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+            0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+            0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+            0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+            0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+            0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+            0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+            0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+            0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+            0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+            0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+            0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+            0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+            0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+            0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
+            0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+            0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+            0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+            0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
+            0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,
+            0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,
+            0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
+            0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,
+            0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
+            0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f,
+            0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57,
+            0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,
+            0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,
+            0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,
+            0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,
+            0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,
+            0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+            0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,
+            0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+            0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,
+            0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,
+            0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,
+            0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,
+            0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,
+            0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,
+            0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,
+            0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
+            0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,
+            0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,
+            0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
+            0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,
+            0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,
+        },
+        {
+            0x27, 0xa7, 0x47, 0x9b, 0xef, 0xa1, 0xd4, 0x76,
+            0x48, 0x9f, 0x30, 0x8c, 0xd4, 0xcf, 0xa6, 0xe2,
+            0xa9, 0x6e, 0x4b, 0xbe, 0x32, 0x08, 0xff, 0x25,
+            0x28, 0x7d, 0xd3, 0x81, 0x96, 0x16, 0xe8, 0x9c,
+            0xc7, 0x8c, 0xf7, 0xf5, 0xe5, 0x43, 0x44, 0x5f,
+            0x83, 0x33, 0xd8, 0xfa, 0x7f, 0x56, 0x00, 0x00,
+            0x05, 0x27, 0x9f, 0xa5, 0xd8, 0xb5, 0xe4, 0xad,
+            0x40, 0xe7, 0x36, 0xdd, 0xb4, 0xd3, 0x54, 0x12,
+            0x32, 0x80, 0x63, 0xfd, 0x2a, 0xab, 0x53, 0xe5,
+            0xea, 0x1e, 0x0a, 0x9f, 0x33, 0x25, 0x00, 0xa5,
+            0xdf, 0x94, 0x87, 0xd0, 0x7a, 0x5c, 0x92, 0xcc,
+            0x51, 0x2c, 0x88, 0x66, 0xc7, 0xe8, 0x60, 0xce,
+            0x93, 0xfd, 0xf1, 0x66, 0xa2, 0x49, 0x12, 0xb4,
+            0x22, 0x97, 0x61, 0x46, 0xae, 0x20, 0xce, 0x84,
+            0x6b, 0xb7, 0xdc, 0x9b, 0xa9, 0x4a, 0x76, 0x7a,
+            0xae, 0xf2, 0x0c, 0x0d, 0x61, 0xad, 0x02, 0x65,
+            0x5e, 0xa9, 0x2d, 0xc4, 0xc4, 0xe4, 0x1a, 0x89,
+            0x52, 0xc6, 0x51, 0xd3, 0x31, 0x74, 0xbe, 0x51,
+            0xa1, 0x0c, 0x42, 0x11, 0x10, 0xe6, 0xd8, 0x15,
+            0x88, 0xed, 0xe8, 0x21, 0x03, 0xa2, 0x52, 0xd8,
+            0xa7, 0x50, 0xe8, 0x76, 0x8d, 0xef, 0xff, 0xed,
+            0x91, 0x22, 0x81, 0x0a, 0xae, 0xb9, 0x9f, 0x91,
+            0x72, 0xaf, 0x82, 0xb6, 0x04, 0xdc, 0x4b, 0x8e,
+            0x51, 0xbc, 0xb0, 0x82, 0x35, 0xa6, 0xf4, 0x34,
+            0x13, 0x32, 0xe4, 0xca, 0x60, 0x48, 0x2a, 0x4b,
+            0xa1, 0xa0, 0x3b, 0x3e, 0x65, 0x00, 0x8f, 0xc5,
+            0xda, 0x76, 0xb7, 0x0b, 0xf1, 0x69, 0x0d, 0xb4,
+            0xea, 0xe2, 0x9c, 0x5f, 0x1b, 0xad, 0xd0, 0x3c,
+            0x5c, 0xcf, 0x2a, 0x55, 0xd7, 0x05, 0xdd, 0xcd,
+            0x86, 0xd4, 0x49, 0x51, 0x1c, 0xeb, 0x7e, 0xc3,
+            0x0b, 0xf1, 0x2b, 0x1f, 0xa3, 0x5b, 0x91, 0x3f,
+            0x9f, 0x74, 0x7a, 0x8a, 0xfd, 0x1b, 0x13, 0x0e,
+            0x94, 0xbf, 0xf9, 0x4e, 0xff, 0xd0, 0x1a, 0x91,
+            0x73, 0x5c, 0xa1, 0x72, 0x6a, 0xcd, 0x0b, 0x19,
+            0x7c, 0x4e, 0x5b, 0x03, 0x39, 0x36, 0x97, 0xe1,
+            0x26, 0x82, 0x6f, 0xb6, 0xbb, 0xde, 0x8e, 0xcc,
+            0x1e, 0x08, 0x29, 0x85, 0x16, 0xe2, 0xc9, 0xed,
+            0x03, 0xff, 0x3c, 0x1b, 0x78, 0x60, 0xf6, 0xde,
+            0x76, 0xd4, 0xce, 0xcd, 0x94, 0xc8, 0x11, 0x98,
+            0x55, 0xef, 0x52, 0x97, 0xca, 0x67, 0xe9, 0xf3,
+            0xe7, 0xff, 0x72, 0xb1, 0xe9, 0x97, 0x85, 0xca,
+            0x0a, 0x7e, 0x77, 0x20, 0xc5, 0xb3, 0x6d, 0xc6,
+            0xd7, 0x2c, 0xac, 0x95, 0x74, 0xc8, 0xcb, 0xbc,
+            0x2f, 0x80, 0x1e, 0x23, 0xe5, 0x6f, 0xd3, 0x44,
+            0xb0, 0x7f, 0x22, 0x15, 0x4b, 0xeb, 0xa0, 0xf0,
+            0x8c, 0xe8, 0x89, 0x1e, 0x64, 0x3e, 0xd9, 0x95,
+            0xc9, 0x4d, 0x9a, 0x69, 0xc9, 0xf1, 0xb5, 0xf4,
+            0x99, 0x02, 0x7a, 0x78, 0x57, 0x2a, 0xee, 0xbd,
+            0x74, 0xd2, 0x0c, 0xc3, 0x98, 0x81, 0xc2, 0x13,
+            0xee, 0x77, 0x0b, 0x10, 0x10, 0xe4, 0xbe, 0xa7,
+            0x18, 0x84, 0x69, 0x77, 0xae, 0x11, 0x9f, 0x7a,
+            0x02, 0x3a, 0xb5, 0x8c, 0xca, 0x0a, 0xd7, 0x52,
+            0xaf, 0xe6, 0x56, 0xbb, 0x3c, 0x17, 0x25, 0x6a,
+            0x9f, 0x6e, 0x9b, 0xf1, 0x9f, 0xdd, 0x5a, 0x38,
+            0xfc, 0x82, 0xbb, 0xe8, 0x72, 0xc5, 0x53, 0x9e,
+            0xdb, 0x60, 0x9e, 0xf4, 0xf7, 0x9c, 0x20, 0x3e,
+            0xbb, 0x14, 0x0f, 0x2e, 0x58, 0x3c, 0xb2, 0xad,
+            0x15, 0xb4, 0xaa, 0x5b, 0x65, 0x50, 0x16, 0xa8,
+            0x44, 0x92, 0x77, 0xdb, 0xd4, 0x77, 0xef, 0x2c,
+            0x8d, 0x6c, 0x01, 0x7d, 0xb7, 0x38, 0xb1, 0x8d,
+            0xeb, 0x4a, 0x42, 0x7d, 0x19, 0x23, 0xce, 0x3f,
+            0xf2, 0x62, 0x73, 0x57, 0x79, 0xa4, 0x18, 0xf2,
+            0x0a, 0x28, 0x2d, 0xf9, 0x20, 0x14, 0x7b, 0xea,
+            0xbe, 0x42, 0x1e, 0xe5, 0x31, 0x9d, 0x05, 0x68,
+        }
+    },
+
+    /* #7, 32 byte key, 17 byte PTX */
+    {
+        "/crypto/xts/t-7-key-32-ptx-17",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        17,
+        { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+          0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10 },
+        { 0x6c, 0x16, 0x25, 0xdb, 0x46, 0x71, 0x52, 0x2d,
+          0x3d, 0x75, 0x99, 0x60, 0x1d, 0xe7, 0xca, 0x09, 0xed },
+    },
+
+    /* #15, 32 byte key, 25 byte PTX */
+    {
+        "/crypto/xts/t-15-key-32-ptx-25",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        25,
+        { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+          0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+          0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18 },
+        { 0x8f, 0x4d, 0xcb, 0xad, 0x55, 0x55, 0x8d, 0x7b,
+          0x4e, 0x01, 0xd9, 0x37, 0x9c, 0xd4, 0xea, 0x22,
+          0xed, 0xbf, 0x9d, 0xac, 0xe4, 0x5d, 0x6f, 0x6a, 0x73 },
+    },
+
+    /* #21, 32 byte key, 31 byte PTX */
+    {
+        "/crypto/xts/t-21-key-32-ptx-31",
+        32,
+        { 0xff, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8,
+          0xf7, 0xf6, 0xf5, 0xf4, 0xf3, 0xf2, 0xf1, 0xf0 },
+        { 0xbf, 0xbe, 0xbd, 0xbc, 0xbb, 0xba, 0xb9, 0xb8,
+          0xb7, 0xb6, 0xb5, 0xb4, 0xb3, 0xb2, 0xb1, 0xb0 },
+        0x123456789aLL,
+        31,
+        { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+          0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
+          0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,
+          0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e },
+        { 0xd0, 0x5b, 0xc0, 0x90, 0xa8, 0xe0, 0x4f, 0x1b,
+          0x3d, 0x3e, 0xcd, 0xd5, 0xba, 0xec, 0x0f, 0xd4,
+          0xed, 0xbf, 0x9d, 0xac, 0xe4, 0x5d, 0x6f, 0x6a,
+          0x73, 0x06, 0xe6, 0x4b, 0xe5, 0xdd, 0x82 },
+    },
+};
+
+#define STORE64L(x, y)                                                  \
+    do {                                                                \
+        (y)[7] = (unsigned char)(((x) >> 56) & 255);                    \
+        (y)[6] = (unsigned char)(((x) >> 48) & 255);                    \
+        (y)[5] = (unsigned char)(((x) >> 40) & 255);                    \
+        (y)[4] = (unsigned char)(((x) >> 32) & 255);                    \
+        (y)[3] = (unsigned char)(((x) >> 24) & 255);                    \
+        (y)[2] = (unsigned char)(((x) >> 16) & 255);                    \
+        (y)[1] = (unsigned char)(((x) >> 8) & 255);                     \
+        (y)[0] = (unsigned char)((x) & 255);                            \
+    } while (0)
+
+struct TestAES {
+    AES_KEY enc;
+    AES_KEY dec;
+};
+
+static void test_xts_aes_encrypt(const void *ctx,
+                                 size_t length,
+                                 uint8_t *dst,
+                                 const uint8_t *src)
+{
+    const struct TestAES *aesctx = ctx;
+
+    AES_encrypt(src, dst, &aesctx->enc);
+}
+
+
+static void test_xts_aes_decrypt(const void *ctx,
+                                 size_t length,
+                                 uint8_t *dst,
+                                 const uint8_t *src)
+{
+    const struct TestAES *aesctx = ctx;
+
+    AES_decrypt(src, dst, &aesctx->dec);
+}
+
+
+static void test_xts(const void *opaque)
+{
+    const QCryptoXTSTestData *data = opaque;
+    unsigned char out[512], Torg[16], T[16];
+    uint64_t seq;
+    int j;
+    unsigned long len;
+    struct TestAES aesdata;
+    struct TestAES aestweak;
+
+    for (j = 0; j < 2; j++) {
+        /* skip the cases where
+         * the length is smaller than 2*blocklen
+         * or the length is not a multiple of 32
+         */
+        if ((j == 1) && ((data->PTLEN < 32) || (data->PTLEN % 32))) {
+            continue;
+        }
+        len = data->PTLEN / 2;
+
+        AES_set_encrypt_key(data->key1, data->keylen / 2 * 8, &aesdata.enc);
+        AES_set_decrypt_key(data->key1, data->keylen / 2 * 8, &aesdata.dec);
+        AES_set_encrypt_key(data->key2, data->keylen / 2 * 8, &aestweak.enc);
+        AES_set_decrypt_key(data->key2, data->keylen / 2 * 8, &aestweak.dec);
+
+        seq = data->seqnum;
+        STORE64L(seq, Torg);
+        memset(Torg + 8, 0, 8);
+
+        memcpy(T, Torg, sizeof(T));
+        if (j == 0) {
+            xts_encrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, data->PTLEN, out, data->PTX);
+        } else {
+            xts_encrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, out, data->PTX);
+            xts_encrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, &out[len], &data->PTX[len]);
+        }
+
+        g_assert(memcmp(out, data->CTX, data->PTLEN) == 0);
+
+        memcpy(T, Torg, sizeof(T));
+        if (j == 0) {
+            xts_decrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, data->PTLEN, out, data->CTX);
+        } else {
+            xts_decrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, out, data->CTX);
+            xts_decrypt(&aesdata, &aestweak,
+                        test_xts_aes_encrypt,
+                        test_xts_aes_decrypt,
+                        T, len, &out[len], &data->CTX[len]);
+        }
+
+        g_assert(memcmp(out, data->PTX, data->PTLEN) == 0);
+    }
+}
+
+
+int main(int argc, char **argv)
+{
+    size_t i;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_assert(qcrypto_init(NULL) == 0);
+
+    for (i = 0; i < G_N_ELEMENTS(test_data); i++) {
+        g_test_add_data_func(test_data[i].path, &test_data[i], test_xts);
+    }
+
+    return g_test_run();
+}
diff --git a/tests/test-cutils.c b/tests/test-cutils.c
index 2a4556d..64e3e95 100644
--- a/tests/test-cutils.c
+++ b/tests/test-cutils.c
@@ -25,12 +25,9 @@
  * THE SOFTWARE.
  */
 
-#include <glib.h>
-#include <errno.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
-#include "qemu-common.h"
-
+#include "qemu/cutils.h"
 
 static void test_parse_uint_null(void)
 {
@@ -226,6 +223,1212 @@
     g_assert_cmpint(i, ==, 123);
 }
 
+static void test_qemu_strtol_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtol_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtol_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtol_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_max(void)
+{
+    const char *str = g_strdup_printf("%ld", LONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err  = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LONG_MIN);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtol_full_correct(void)
+{
+    const char *str = "123";
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtol_full_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtol_full_empty(void)
+{
+    const char *str = "";
+    long res = 999L;
+    int err;
+
+    err =  qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_full_negative(void)
+{
+    const char *str = " \t -321";
+    long res = 999;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+}
+
+static void test_qemu_strtol_full_trailing(void)
+{
+    const char *str = "123xxx";
+    long res;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtol_full_max(void)
+{
+    const char *str = g_strdup_printf("%ld", LONG_MAX);
+    long res;
+    int err;
+
+    err = qemu_strtol(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LONG_MAX);
+}
+
+static void test_qemu_strtoul_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoul_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoul_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoul_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    res = 999;
+    endptr = &f;
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_max(void)
+{
+    const char *str = g_strdup_printf("%lu", ULONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err  = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, -1ul);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321ul);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoul_full_correct(void)
+{
+    const char *str = "123";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtoul_full_null(void)
+{
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_full_empty(void)
+{
+    const char *str = "";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+static void test_qemu_strtoul_full_negative(void)
+{
+    const char *str = " \t -321";
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321ul);
+}
+
+static void test_qemu_strtoul_full_trailing(void)
+{
+    const char *str = "123xxx";
+    unsigned long res;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoul_full_max(void)
+{
+    const char *str = g_strdup_printf("%lu", ULONG_MAX);
+    unsigned long res = 999;
+    int err;
+
+    err = qemu_strtoul(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULONG_MAX);
+}
+
+static void test_qemu_strtoll_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoll_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoll_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoll_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_max(void)
+{
+    const char *str = g_strdup_printf("%lld", LLONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err  = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, LLONG_MIN);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoll_full_correct(void)
+{
+    const char *str = "123";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+}
+
+static void test_qemu_strtoll_full_null(void)
+{
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_empty(void)
+{
+    const char *str = "";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_negative(void)
+{
+    const char *str = " \t -321";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+}
+
+static void test_qemu_strtoll_full_trailing(void)
+{
+    const char *str = "123xxx";
+    int64_t res = 999;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoll_full_max(void)
+{
+
+    const char *str = g_strdup_printf("%lld", LLONG_MAX);
+    int64_t res;
+    int err;
+
+    err = qemu_strtoll(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, LLONG_MAX);
+}
+
+static void test_qemu_strtoull_correct(void)
+{
+    const char *str = "12345 foo";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 12345);
+    g_assert(endptr == str + 5);
+}
+
+static void test_qemu_strtoull_null(void)
+{
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(NULL, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+    g_assert(endptr == NULL);
+}
+
+static void test_qemu_strtoull_empty(void)
+{
+    const char *str = "";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_whitespace(void)
+{
+    const char *str = "  \t  ";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_invalid(void)
+{
+    const char *str = "   xxxx  \t abc";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_trailing(void)
+{
+    const char *str = "123xxx";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + 3);
+}
+
+static void test_qemu_strtoull_octal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 8, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_decimal(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 10, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_hex(void)
+{
+    const char *str = "0123";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 16, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+
+    str = "0x123";
+    endptr = &f;
+    res = 999;
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 0x123);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_max(void)
+{
+    const char *str = g_strdup_printf("%llu", ULLONG_MAX);
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_overflow(void)
+{
+    const char *str = "99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_underflow(void)
+{
+    const char *str = "-99999999999999999999999999999999999999999999";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err  = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, -ERANGE);
+    g_assert_cmpint(res, ==, -1);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_negative(void)
+{
+    const char *str = "  \t -321";
+    char f = 'X';
+    const char *endptr = &f;
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, &endptr, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, -321);
+    g_assert(endptr == str + strlen(str));
+}
+
+static void test_qemu_strtoull_full_correct(void)
+{
+    const char *str = "18446744073709551614";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 18446744073709551614LLU);
+}
+
+static void test_qemu_strtoull_full_null(void)
+{
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(NULL, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_empty(void)
+{
+    const char *str = "";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_negative(void)
+{
+    const char *str = " \t -321";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, 18446744073709551295LLU);
+}
+
+static void test_qemu_strtoull_full_trailing(void)
+{
+    const char *str = "18446744073709551614xxxxxx";
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, -EINVAL);
+}
+
+static void test_qemu_strtoull_full_max(void)
+{
+    const char *str = g_strdup_printf("%lld", ULLONG_MAX);
+    uint64_t res = 999;
+    int err;
+
+    err = qemu_strtoull(str, NULL, 0, &res);
+
+    g_assert_cmpint(err, ==, 0);
+    g_assert_cmpint(res, ==, ULLONG_MAX);
+}
+
+static void test_qemu_strtosz_simple(void)
+{
+    const char *str = "12345M";
+    char *endptr = NULL;
+    int64_t res;
+
+    res = qemu_strtosz(str, &endptr);
+    g_assert_cmpint(res, ==, 12345 * M_BYTE);
+    g_assert(endptr == str + 6);
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, 12345 * M_BYTE);
+}
+
+static void test_qemu_strtosz_units(void)
+{
+    const char *none = "1";
+    const char *b = "1B";
+    const char *k = "1K";
+    const char *m = "1M";
+    const char *g = "1G";
+    const char *t = "1T";
+    const char *p = "1P";
+    const char *e = "1E";
+    int64_t res;
+
+    /* default is M */
+    res = qemu_strtosz(none, NULL);
+    g_assert_cmpint(res, ==, M_BYTE);
+
+    res = qemu_strtosz(b, NULL);
+    g_assert_cmpint(res, ==, 1);
+
+    res = qemu_strtosz(k, NULL);
+    g_assert_cmpint(res, ==, K_BYTE);
+
+    res = qemu_strtosz(m, NULL);
+    g_assert_cmpint(res, ==, M_BYTE);
+
+    res = qemu_strtosz(g, NULL);
+    g_assert_cmpint(res, ==, G_BYTE);
+
+    res = qemu_strtosz(t, NULL);
+    g_assert_cmpint(res, ==, T_BYTE);
+
+    res = qemu_strtosz(p, NULL);
+    g_assert_cmpint(res, ==, P_BYTE);
+
+    res = qemu_strtosz(e, NULL);
+    g_assert_cmpint(res, ==, E_BYTE);
+}
+
+static void test_qemu_strtosz_float(void)
+{
+    const char *str = "12.345M";
+    int64_t res;
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, 12.345 * M_BYTE);
+}
+
+static void test_qemu_strtosz_erange(void)
+{
+    const char *str = "10E";
+    int64_t res;
+
+    res = qemu_strtosz(str, NULL);
+    g_assert_cmpint(res, ==, -ERANGE);
+}
+
+static void test_qemu_strtosz_suffix_unit(void)
+{
+    const char *str = "12345";
+    int64_t res;
+
+    res = qemu_strtosz_suffix_unit(str, NULL,
+                                   QEMU_STRTOSZ_DEFSUFFIX_KB, 1000);
+    g_assert_cmpint(res, ==, 12345000);
+}
+
 int main(int argc, char **argv)
 {
     g_test_init(&argc, &argv, NULL);
@@ -247,5 +1450,145 @@
     g_test_add_func("/cutils/parse_uint_full/correct",
                     test_parse_uint_full_correct);
 
+    /* qemu_strtol() tests */
+    g_test_add_func("/cutils/qemu_strtol/correct", test_qemu_strtol_correct);
+    g_test_add_func("/cutils/qemu_strtol/null", test_qemu_strtol_null);
+    g_test_add_func("/cutils/qemu_strtol/empty", test_qemu_strtol_empty);
+    g_test_add_func("/cutils/qemu_strtol/whitespace",
+                    test_qemu_strtol_whitespace);
+    g_test_add_func("/cutils/qemu_strtol/invalid", test_qemu_strtol_invalid);
+    g_test_add_func("/cutils/qemu_strtol/trailing", test_qemu_strtol_trailing);
+    g_test_add_func("/cutils/qemu_strtol/octal", test_qemu_strtol_octal);
+    g_test_add_func("/cutils/qemu_strtol/decimal", test_qemu_strtol_decimal);
+    g_test_add_func("/cutils/qemu_strtol/hex", test_qemu_strtol_hex);
+    g_test_add_func("/cutils/qemu_strtol/max", test_qemu_strtol_max);
+    g_test_add_func("/cutils/qemu_strtol/overflow", test_qemu_strtol_overflow);
+    g_test_add_func("/cutils/qemu_strtol/underflow",
+                    test_qemu_strtol_underflow);
+    g_test_add_func("/cutils/qemu_strtol/negative", test_qemu_strtol_negative);
+    g_test_add_func("/cutils/qemu_strtol_full/correct",
+                    test_qemu_strtol_full_correct);
+    g_test_add_func("/cutils/qemu_strtol_full/null",
+                    test_qemu_strtol_full_null);
+    g_test_add_func("/cutils/qemu_strtol_full/empty",
+                    test_qemu_strtol_full_empty);
+    g_test_add_func("/cutils/qemu_strtol_full/negative",
+                    test_qemu_strtol_full_negative);
+    g_test_add_func("/cutils/qemu_strtol_full/trailing",
+                    test_qemu_strtol_full_trailing);
+    g_test_add_func("/cutils/qemu_strtol_full/max",
+                    test_qemu_strtol_full_max);
+
+    /* qemu_strtoul() tests */
+    g_test_add_func("/cutils/qemu_strtoul/correct", test_qemu_strtoul_correct);
+    g_test_add_func("/cutils/qemu_strtoul/null", test_qemu_strtoul_null);
+    g_test_add_func("/cutils/qemu_strtoul/empty", test_qemu_strtoul_empty);
+    g_test_add_func("/cutils/qemu_strtoul/whitespace",
+                    test_qemu_strtoul_whitespace);
+    g_test_add_func("/cutils/qemu_strtoul/invalid", test_qemu_strtoul_invalid);
+    g_test_add_func("/cutils/qemu_strtoul/trailing",
+                    test_qemu_strtoul_trailing);
+    g_test_add_func("/cutils/qemu_strtoul/octal", test_qemu_strtoul_octal);
+    g_test_add_func("/cutils/qemu_strtoul/decimal", test_qemu_strtoul_decimal);
+    g_test_add_func("/cutils/qemu_strtoul/hex", test_qemu_strtoul_hex);
+    g_test_add_func("/cutils/qemu_strtoul/max", test_qemu_strtoul_max);
+    g_test_add_func("/cutils/qemu_strtoul/overflow",
+                    test_qemu_strtoul_overflow);
+    g_test_add_func("/cutils/qemu_strtoul/underflow",
+                    test_qemu_strtoul_underflow);
+    g_test_add_func("/cutils/qemu_strtoul/negative",
+                    test_qemu_strtoul_negative);
+    g_test_add_func("/cutils/qemu_strtoul_full/correct",
+                    test_qemu_strtoul_full_correct);
+    g_test_add_func("/cutils/qemu_strtoul_full/null",
+                    test_qemu_strtoul_full_null);
+    g_test_add_func("/cutils/qemu_strtoul_full/empty",
+                    test_qemu_strtoul_full_empty);
+    g_test_add_func("/cutils/qemu_strtoul_full/negative",
+                    test_qemu_strtoul_full_negative);
+    g_test_add_func("/cutils/qemu_strtoul_full/trailing",
+                    test_qemu_strtoul_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoul_full/max",
+                    test_qemu_strtoul_full_max);
+
+    /* qemu_strtoll() tests */
+    g_test_add_func("/cutils/qemu_strtoll/correct", test_qemu_strtoll_correct);
+    g_test_add_func("/cutils/qemu_strtoll/null", test_qemu_strtoll_null);
+    g_test_add_func("/cutils/qemu_strtoll/empty", test_qemu_strtoll_empty);
+    g_test_add_func("/cutils/qemu_strtoll/whitespace",
+                    test_qemu_strtoll_whitespace);
+    g_test_add_func("/cutils/qemu_strtoll/invalid", test_qemu_strtoll_invalid);
+    g_test_add_func("/cutils/qemu_strtoll/trailing",
+                    test_qemu_strtoll_trailing);
+    g_test_add_func("/cutils/qemu_strtoll/octal", test_qemu_strtoll_octal);
+    g_test_add_func("/cutils/qemu_strtoll/decimal", test_qemu_strtoll_decimal);
+    g_test_add_func("/cutils/qemu_strtoll/hex", test_qemu_strtoll_hex);
+    g_test_add_func("/cutils/qemu_strtoll/max", test_qemu_strtoll_max);
+    g_test_add_func("/cutils/qemu_strtoll/overflow",
+                    test_qemu_strtoll_overflow);
+    g_test_add_func("/cutils/qemu_strtoll/underflow",
+                    test_qemu_strtoll_underflow);
+    g_test_add_func("/cutils/qemu_strtoll/negative",
+                    test_qemu_strtoll_negative);
+    g_test_add_func("/cutils/qemu_strtoll_full/correct",
+                    test_qemu_strtoll_full_correct);
+    g_test_add_func("/cutils/qemu_strtoll_full/null",
+                    test_qemu_strtoll_full_null);
+    g_test_add_func("/cutils/qemu_strtoll_full/empty",
+                    test_qemu_strtoll_full_empty);
+    g_test_add_func("/cutils/qemu_strtoll_full/negative",
+                    test_qemu_strtoll_full_negative);
+    g_test_add_func("/cutils/qemu_strtoll_full/trailing",
+                    test_qemu_strtoll_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoll_full/max",
+                    test_qemu_strtoll_full_max);
+
+    /* qemu_strtoull() tests */
+    g_test_add_func("/cutils/qemu_strtoull/correct",
+                    test_qemu_strtoull_correct);
+    g_test_add_func("/cutils/qemu_strtoull/null",
+                    test_qemu_strtoull_null);
+    g_test_add_func("/cutils/qemu_strtoull/empty", test_qemu_strtoull_empty);
+    g_test_add_func("/cutils/qemu_strtoull/whitespace",
+                    test_qemu_strtoull_whitespace);
+    g_test_add_func("/cutils/qemu_strtoull/invalid",
+                    test_qemu_strtoull_invalid);
+    g_test_add_func("/cutils/qemu_strtoull/trailing",
+                    test_qemu_strtoull_trailing);
+    g_test_add_func("/cutils/qemu_strtoull/octal", test_qemu_strtoull_octal);
+    g_test_add_func("/cutils/qemu_strtoull/decimal",
+                    test_qemu_strtoull_decimal);
+    g_test_add_func("/cutils/qemu_strtoull/hex", test_qemu_strtoull_hex);
+    g_test_add_func("/cutils/qemu_strtoull/max", test_qemu_strtoull_max);
+    g_test_add_func("/cutils/qemu_strtoull/overflow",
+                    test_qemu_strtoull_overflow);
+    g_test_add_func("/cutils/qemu_strtoull/underflow",
+                    test_qemu_strtoull_underflow);
+    g_test_add_func("/cutils/qemu_strtoull/negative",
+                    test_qemu_strtoull_negative);
+    g_test_add_func("/cutils/qemu_strtoull_full/correct",
+                    test_qemu_strtoull_full_correct);
+    g_test_add_func("/cutils/qemu_strtoull_full/null",
+                    test_qemu_strtoull_full_null);
+    g_test_add_func("/cutils/qemu_strtoull_full/empty",
+                    test_qemu_strtoull_full_empty);
+    g_test_add_func("/cutils/qemu_strtoull_full/negative",
+                    test_qemu_strtoull_full_negative);
+    g_test_add_func("/cutils/qemu_strtoull_full/trailing",
+                    test_qemu_strtoull_full_trailing);
+    g_test_add_func("/cutils/qemu_strtoull_full/max",
+                    test_qemu_strtoull_full_max);
+
+    g_test_add_func("/cutils/strtosz/simple",
+                    test_qemu_strtosz_simple);
+    g_test_add_func("/cutils/strtosz/units",
+                    test_qemu_strtosz_units);
+    g_test_add_func("/cutils/strtosz/float",
+                    test_qemu_strtosz_float);
+    g_test_add_func("/cutils/strtosz/erange",
+                    test_qemu_strtosz_erange);
+    g_test_add_func("/cutils/strtosz/suffix-unit",
+                    test_qemu_strtosz_suffix_unit);
+
     return g_test_run();
 }
diff --git a/tests/test-filter-mirror.c b/tests/test-filter-mirror.c
new file mode 100644
index 0000000..ffaaffa
--- /dev/null
+++ b/tests/test-filter-mirror.c
@@ -0,0 +1,92 @@
+/*
+ * QTest testcase for filter-mirror
+ *
+ * Copyright (c) 2016 FUJITSU LIMITED
+ * Author: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+
+static void test_mirror(void)
+{
+#ifndef _WIN32
+/* socketpair(PF_UNIX) which does not exist on windows */
+
+    int send_sock[2], recv_sock;
+    char *cmdline;
+    uint32_t ret = 0, len = 0;
+    char send_buf[] = "Hello! filter-mirror~";
+    char sock_path[] = "filter-mirror.XXXXXX";
+    char *recv_buf;
+    uint32_t size = sizeof(send_buf);
+    size = htonl(size);
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, send_sock);
+    g_assert_cmpint(ret, !=, -1);
+
+    ret = mkstemp(sock_path);
+    g_assert_cmpint(ret, !=, -1);
+
+    cmdline = g_strdup_printf("-netdev socket,id=qtest-bn0,fd=%d "
+                 "-device e1000,netdev=qtest-bn0,id=qtest-e0 "
+                 "-chardev socket,id=mirror0,path=%s,server,nowait "
+                 "-object filter-mirror,id=qtest-f0,netdev=qtest-bn0,queue=tx,outdev=mirror0 "
+                 , send_sock[1], sock_path);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    recv_sock = unix_connect(sock_path, NULL);
+    g_assert_cmpint(recv_sock, !=, -1);
+
+    struct iovec iov[] = {
+        {
+            .iov_base = &size,
+            .iov_len = sizeof(size),
+        }, {
+            .iov_base = send_buf,
+            .iov_len = sizeof(send_buf),
+        },
+    };
+
+    /* send a qmp command to guarantee that 'connected' is setting to true. */
+    qmp("{ 'execute' : 'query-status'}");
+    ret = iov_send(send_sock[0], iov, 2, 0, sizeof(size) + sizeof(send_buf));
+    g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size));
+    close(send_sock[0]);
+
+    ret = qemu_recv(recv_sock, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    g_assert_cmpint(len, ==, sizeof(send_buf));
+    recv_buf = g_malloc(len);
+    ret = qemu_recv(recv_sock, recv_buf, len, 0);
+    g_assert_cmpstr(recv_buf, ==, send_buf);
+
+    g_free(recv_buf);
+    close(recv_sock);
+    unlink(sock_path);
+
+#endif
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+
+    qtest_add_func("/netfilter/mirror", test_mirror);
+    ret = g_test_run();
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/test-filter-redirector.c b/tests/test-filter-redirector.c
new file mode 100644
index 0000000..c63b68f
--- /dev/null
+++ b/tests/test-filter-redirector.c
@@ -0,0 +1,216 @@
+/*
+ * QTest testcase for filter-redirector
+ *
+ * Copyright (c) 2016 FUJITSU LIMITED
+ * Author: Zhang Chen <zhangchen.fnst@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ *
+ * Case 1, tx traffic flow:
+ *
+ * qemu side              | test side
+ *                        |
+ * +---------+            |  +-------+
+ * | backend <---------------+ sock0 |
+ * +----+----+            |  +-------+
+ *      |                 |
+ * +----v----+  +-------+ |
+ * |  rd0    +->+chardev| |
+ * +---------+  +---+---+ |
+ *                  |     |
+ * +---------+      |     |
+ * |  rd1    <------+     |
+ * +----+----+            |
+ *      |                 |
+ * +----v----+            |  +-------+
+ * |  rd2    +--------------->sock1  |
+ * +---------+            |  +-------+
+ *                        +
+ *
+ * --------------------------------------
+ * Case 2, rx traffic flow
+ * qemu side              | test side
+ *                        |
+ * +---------+            |  +-------+
+ * | backend +---------------> sock1 |
+ * +----^----+            |  +-------+
+ *      |                 |
+ * +----+----+  +-------+ |
+ * |  rd0    +<-+chardev| |
+ * +---------+  +---+---+ |
+ *                  ^     |
+ * +---------+      |     |
+ * |  rd1    +------+     |
+ * +----^----+            |
+ *      |                 |
+ * +----+----+            |  +-------+
+ * |  rd2    <---------------+sock0  |
+ * +---------+            |  +-------+
+ *                        +
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+#include "qemu/iov.h"
+#include "qemu/sockets.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+
+static void test_redirector_tx(void)
+{
+#ifndef _WIN32
+/* socketpair(PF_UNIX) which does not exist on windows */
+
+    int backend_sock[2], recv_sock;
+    char *cmdline;
+    uint32_t ret = 0, len = 0;
+    char send_buf[] = "Hello!!";
+    char sock_path0[] = "filter-redirector0.XXXXXX";
+    char sock_path1[] = "filter-redirector1.XXXXXX";
+    char *recv_buf;
+    uint32_t size = sizeof(send_buf);
+    size = htonl(size);
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, backend_sock);
+    g_assert_cmpint(ret, !=, -1);
+
+    ret = mkstemp(sock_path0);
+    g_assert_cmpint(ret, !=, -1);
+    ret = mkstemp(sock_path1);
+    g_assert_cmpint(ret, !=, -1);
+
+    cmdline = g_strdup_printf("-netdev socket,id=qtest-bn0,fd=%d "
+                "-device rtl8139,netdev=qtest-bn0,id=qtest-e0 "
+                "-chardev socket,id=redirector0,path=%s,server,nowait "
+                "-chardev socket,id=redirector1,path=%s,server,nowait "
+                "-chardev socket,id=redirector2,path=%s,nowait "
+                "-object filter-redirector,id=qtest-f0,netdev=qtest-bn0,"
+                "queue=tx,outdev=redirector0 "
+                "-object filter-redirector,id=qtest-f1,netdev=qtest-bn0,"
+                "queue=tx,indev=redirector2 "
+                "-object filter-redirector,id=qtest-f2,netdev=qtest-bn0,"
+                "queue=tx,outdev=redirector1 "
+                , backend_sock[1], sock_path0, sock_path1, sock_path0);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    recv_sock = unix_connect(sock_path1, NULL);
+    g_assert_cmpint(recv_sock, !=, -1);
+
+    /* send a qmp command to guarantee that 'connected' is setting to true. */
+    qmp("{ 'execute' : 'query-status'}");
+
+    struct iovec iov[] = {
+        {
+            .iov_base = &size,
+            .iov_len = sizeof(size),
+        }, {
+            .iov_base = send_buf,
+            .iov_len = sizeof(send_buf),
+        },
+    };
+
+    ret = iov_send(backend_sock[0], iov, 2, 0, sizeof(size) + sizeof(send_buf));
+    g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size));
+    close(backend_sock[0]);
+
+    ret = qemu_recv(recv_sock, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    g_assert_cmpint(len, ==, sizeof(send_buf));
+    recv_buf = g_malloc(len);
+    ret = qemu_recv(recv_sock, recv_buf, len, 0);
+    g_assert_cmpstr(recv_buf, ==, send_buf);
+
+    g_free(recv_buf);
+    close(recv_sock);
+    unlink(sock_path0);
+    unlink(sock_path1);
+    qtest_end();
+
+#endif
+}
+
+static void test_redirector_rx(void)
+{
+#ifndef _WIN32
+/* socketpair(PF_UNIX) which does not exist on windows */
+
+    int backend_sock[2], send_sock;
+    char *cmdline;
+    uint32_t ret = 0, len = 0;
+    char send_buf[] = "Hello!!";
+    char sock_path0[] = "filter-redirector0.XXXXXX";
+    char sock_path1[] = "filter-redirector1.XXXXXX";
+    char *recv_buf;
+    uint32_t size = sizeof(send_buf);
+    size = htonl(size);
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, backend_sock);
+    g_assert_cmpint(ret, !=, -1);
+
+    ret = mkstemp(sock_path0);
+    g_assert_cmpint(ret, !=, -1);
+    ret = mkstemp(sock_path1);
+    g_assert_cmpint(ret, !=, -1);
+
+    cmdline = g_strdup_printf("-netdev socket,id=qtest-bn0,fd=%d "
+                "-device rtl8139,netdev=qtest-bn0,id=qtest-e0 "
+                "-chardev socket,id=redirector0,path=%s,server,nowait "
+                "-chardev socket,id=redirector1,path=%s,server,nowait "
+                "-chardev socket,id=redirector2,path=%s,nowait "
+                "-object filter-redirector,id=qtest-f0,netdev=qtest-bn0,"
+                "queue=rx,indev=redirector0 "
+                "-object filter-redirector,id=qtest-f1,netdev=qtest-bn0,"
+                "queue=rx,outdev=redirector2 "
+                "-object filter-redirector,id=qtest-f2,netdev=qtest-bn0,"
+                "queue=rx,indev=redirector1 "
+                , backend_sock[1], sock_path0, sock_path1, sock_path0);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    struct iovec iov[] = {
+        {
+            .iov_base = &size,
+            .iov_len = sizeof(size),
+        }, {
+            .iov_base = send_buf,
+            .iov_len = sizeof(send_buf),
+        },
+    };
+
+    send_sock = unix_connect(sock_path1, NULL);
+    g_assert_cmpint(send_sock, !=, -1);
+    /* send a qmp command to guarantee that 'connected' is setting to true. */
+    qmp("{ 'execute' : 'query-status'}");
+
+    ret = iov_send(send_sock, iov, 2, 0, sizeof(size) + sizeof(send_buf));
+    g_assert_cmpint(ret, ==, sizeof(send_buf) + sizeof(size));
+    close(send_sock);
+
+    ret = qemu_recv(backend_sock[0], &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    g_assert_cmpint(len, ==, sizeof(send_buf));
+    recv_buf = g_malloc(len);
+    ret = qemu_recv(backend_sock[0], recv_buf, len, 0);
+    g_assert_cmpstr(recv_buf, ==, send_buf);
+
+    g_free(recv_buf);
+    unlink(sock_path0);
+    unlink(sock_path1);
+    qtest_end();
+
+#endif
+}
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    qtest_add_func("/netfilter/redirector_tx", test_redirector_tx);
+    qtest_add_func("/netfilter/redirector_rx", test_redirector_rx);
+    return g_test_run();
+}
diff --git a/tests/test-hbitmap.c b/tests/test-hbitmap.c
index 8c902f2..c0e9895 100644
--- a/tests/test-hbitmap.c
+++ b/tests/test-hbitmap.c
@@ -9,8 +9,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 #include "qemu/hbitmap.h"
 
 #define LOG_BITS_PER_LONG          (BITS_PER_LONG == 32 ? 5 : 6)
@@ -23,6 +22,7 @@
     HBitmap       *hb;
     unsigned long *bits;
     size_t         size;
+    size_t         old_size;
     int            granularity;
 } TestHBitmapData;
 
@@ -79,7 +79,7 @@
     size_t n;
     data->hb = hbitmap_alloc(size, granularity);
 
-    n = (size + BITS_PER_LONG - 1) / BITS_PER_LONG;
+    n = DIV_ROUND_UP(size, BITS_PER_LONG);
     if (n == 0) {
         n = 1;
     }
@@ -91,6 +91,44 @@
     }
 }
 
+static inline size_t hbitmap_test_array_size(size_t bits)
+{
+    size_t n = DIV_ROUND_UP(bits, BITS_PER_LONG);
+    return n ? n : 1;
+}
+
+static void hbitmap_test_truncate_impl(TestHBitmapData *data,
+                                       size_t size)
+{
+    size_t n;
+    size_t m;
+    data->old_size = data->size;
+    data->size = size;
+
+    if (data->size == data->old_size) {
+        return;
+    }
+
+    n = hbitmap_test_array_size(size);
+    m = hbitmap_test_array_size(data->old_size);
+    data->bits = g_realloc(data->bits, sizeof(unsigned long) * n);
+    if (n > m) {
+        memset(&data->bits[m], 0x00, sizeof(unsigned long) * (n - m));
+    }
+
+    /* If we shrink to an uneven multiple of sizeof(unsigned long),
+     * scrub the leftover memory. */
+    if (data->size < data->old_size) {
+        m = size % (sizeof(unsigned long) * 8);
+        if (m) {
+            unsigned long mask = (1ULL << m) - 1;
+            data->bits[n-1] &= mask;
+        }
+    }
+
+    hbitmap_truncate(data->hb, size);
+}
+
 static void hbitmap_test_teardown(TestHBitmapData *data,
                                   const void *unused)
 {
@@ -98,10 +136,8 @@
         hbitmap_free(data->hb);
         data->hb = NULL;
     }
-    if (data->bits) {
-        g_free(data->bits);
-        data->bits = NULL;
-    }
+    g_free(data->bits);
+    data->bits = NULL;
 }
 
 /* Set a range in the HBitmap and in the shadow "simple" bitmap.
@@ -143,6 +179,23 @@
     }
 }
 
+static void hbitmap_test_reset_all(TestHBitmapData *data)
+{
+    size_t n;
+
+    hbitmap_reset_all(data->hb);
+
+    n = DIV_ROUND_UP(data->size, BITS_PER_LONG);
+    if (n == 0) {
+        n = 1;
+    }
+    memset(data->bits, 0, n * sizeof(unsigned long));
+
+    if (data->granularity == 0) {
+        hbitmap_test_check(data, 0);
+    }
+}
+
 static void hbitmap_test_check_get(TestHBitmapData *data)
 {
     uint64_t count = 0;
@@ -323,6 +376,26 @@
     hbitmap_test_set(data, L3 / 2, L3);
 }
 
+static void test_hbitmap_reset_all(TestHBitmapData *data,
+                                   const void *unused)
+{
+    hbitmap_test_init(data, L3 * 2, 0);
+    hbitmap_test_set(data, L1 - 1, L1 + 2);
+    hbitmap_test_reset_all(data);
+    hbitmap_test_set(data, 0, L1 * 3);
+    hbitmap_test_reset_all(data);
+    hbitmap_test_set(data, L2, L1);
+    hbitmap_test_reset_all(data);
+    hbitmap_test_set(data, L2, L3 - L2 + 1);
+    hbitmap_test_reset_all(data);
+    hbitmap_test_set(data, L3 - 1, 3);
+    hbitmap_test_reset_all(data);
+    hbitmap_test_set(data, 0, L3 * 2);
+    hbitmap_test_reset_all(data);
+    hbitmap_test_set(data, L3 / 2, L3);
+    hbitmap_test_reset_all(data);
+}
+
 static void test_hbitmap_granularity(TestHBitmapData *data,
                                      const void *unused)
 {
@@ -369,6 +442,198 @@
     g_assert_cmpint(hbitmap_iter_next(&hbi), <, 0);
 }
 
+static void hbitmap_test_set_boundary_bits(TestHBitmapData *data, ssize_t diff)
+{
+    size_t size = data->size;
+
+    /* First bit */
+    hbitmap_test_set(data, 0, 1);
+    if (diff < 0) {
+        /* Last bit in new, shortened map */
+        hbitmap_test_set(data, size + diff - 1, 1);
+
+        /* First bit to be truncated away */
+        hbitmap_test_set(data, size + diff, 1);
+    }
+    /* Last bit */
+    hbitmap_test_set(data, size - 1, 1);
+    if (data->granularity == 0) {
+        hbitmap_test_check_get(data);
+    }
+}
+
+static void hbitmap_test_check_boundary_bits(TestHBitmapData *data)
+{
+    size_t size = MIN(data->size, data->old_size);
+
+    if (data->granularity == 0) {
+        hbitmap_test_check_get(data);
+        hbitmap_test_check(data, 0);
+    } else {
+        /* If a granularity was set, note that every distinct
+         * (bit >> granularity) value that was set will increase
+         * the bit pop count by 2^granularity, not just 1.
+         *
+         * The hbitmap_test_check facility does not currently tolerate
+         * non-zero granularities, so test the boundaries and the population
+         * count manually.
+         */
+        g_assert(hbitmap_get(data->hb, 0));
+        g_assert(hbitmap_get(data->hb, size - 1));
+        g_assert_cmpint(2 << data->granularity, ==, hbitmap_count(data->hb));
+    }
+}
+
+/* Generic truncate test. */
+static void hbitmap_test_truncate(TestHBitmapData *data,
+                                  size_t size,
+                                  ssize_t diff,
+                                  int granularity)
+{
+    hbitmap_test_init(data, size, granularity);
+    hbitmap_test_set_boundary_bits(data, diff);
+    hbitmap_test_truncate_impl(data, size + diff);
+    hbitmap_test_check_boundary_bits(data);
+}
+
+static void test_hbitmap_truncate_nop(TestHBitmapData *data,
+                                      const void *unused)
+{
+    hbitmap_test_truncate(data, L2, 0, 0);
+}
+
+/**
+ * Grow by an amount smaller than the granularity, without crossing
+ * a granularity alignment boundary. Effectively a NOP.
+ */
+static void test_hbitmap_truncate_grow_negligible(TestHBitmapData *data,
+                                                  const void *unused)
+{
+    size_t size = L2 - 1;
+    size_t diff = 1;
+    int granularity = 1;
+
+    hbitmap_test_truncate(data, size, diff, granularity);
+}
+
+/**
+ * Shrink by an amount smaller than the granularity, without crossing
+ * a granularity alignment boundary. Effectively a NOP.
+ */
+static void test_hbitmap_truncate_shrink_negligible(TestHBitmapData *data,
+                                                    const void *unused)
+{
+    size_t size = L2;
+    ssize_t diff = -1;
+    int granularity = 1;
+
+    hbitmap_test_truncate(data, size, diff, granularity);
+}
+
+/**
+ * Grow by an amount smaller than the granularity, but crossing over
+ * a granularity alignment boundary.
+ */
+static void test_hbitmap_truncate_grow_tiny(TestHBitmapData *data,
+                                            const void *unused)
+{
+    size_t size = L2 - 2;
+    ssize_t diff = 1;
+    int granularity = 1;
+
+    hbitmap_test_truncate(data, size, diff, granularity);
+}
+
+/**
+ * Shrink by an amount smaller than the granularity, but crossing over
+ * a granularity alignment boundary.
+ */
+static void test_hbitmap_truncate_shrink_tiny(TestHBitmapData *data,
+                                              const void *unused)
+{
+    size_t size = L2 - 1;
+    ssize_t diff = -1;
+    int granularity = 1;
+
+    hbitmap_test_truncate(data, size, diff, granularity);
+}
+
+/**
+ * Grow by an amount smaller than sizeof(long), and not crossing over
+ * a sizeof(long) alignment boundary.
+ */
+static void test_hbitmap_truncate_grow_small(TestHBitmapData *data,
+                                             const void *unused)
+{
+    size_t size = L2 + 1;
+    size_t diff = sizeof(long) / 2;
+
+    hbitmap_test_truncate(data, size, diff, 0);
+}
+
+/**
+ * Shrink by an amount smaller than sizeof(long), and not crossing over
+ * a sizeof(long) alignment boundary.
+ */
+static void test_hbitmap_truncate_shrink_small(TestHBitmapData *data,
+                                               const void *unused)
+{
+    size_t size = L2;
+    size_t diff = sizeof(long) / 2;
+
+    hbitmap_test_truncate(data, size, -diff, 0);
+}
+
+/**
+ * Grow by an amount smaller than sizeof(long), while crossing over
+ * a sizeof(long) alignment boundary.
+ */
+static void test_hbitmap_truncate_grow_medium(TestHBitmapData *data,
+                                              const void *unused)
+{
+    size_t size = L2 - 1;
+    size_t diff = sizeof(long) / 2;
+
+    hbitmap_test_truncate(data, size, diff, 0);
+}
+
+/**
+ * Shrink by an amount smaller than sizeof(long), while crossing over
+ * a sizeof(long) alignment boundary.
+ */
+static void test_hbitmap_truncate_shrink_medium(TestHBitmapData *data,
+                                                const void *unused)
+{
+    size_t size = L2 + 1;
+    size_t diff = sizeof(long) / 2;
+
+    hbitmap_test_truncate(data, size, -diff, 0);
+}
+
+/**
+ * Grow by an amount larger than sizeof(long).
+ */
+static void test_hbitmap_truncate_grow_large(TestHBitmapData *data,
+                                             const void *unused)
+{
+    size_t size = L2;
+    size_t diff = 8 * sizeof(long);
+
+    hbitmap_test_truncate(data, size, diff, 0);
+}
+
+/**
+ * Shrink by an amount larger than sizeof(long).
+ */
+static void test_hbitmap_truncate_shrink_large(TestHBitmapData *data,
+                                               const void *unused)
+{
+    size_t size = L2;
+    size_t diff = 8 * sizeof(long);
+
+    hbitmap_test_truncate(data, size, -diff, 0);
+}
+
 static void hbitmap_test_add(const char *testpath,
                                    void (*test_func)(TestHBitmapData *data, const void *user_data))
 {
@@ -394,7 +659,30 @@
     hbitmap_test_add("/hbitmap/set/overlap", test_hbitmap_set_overlap);
     hbitmap_test_add("/hbitmap/reset/empty", test_hbitmap_reset_empty);
     hbitmap_test_add("/hbitmap/reset/general", test_hbitmap_reset);
+    hbitmap_test_add("/hbitmap/reset/all", test_hbitmap_reset_all);
     hbitmap_test_add("/hbitmap/granularity", test_hbitmap_granularity);
+
+    hbitmap_test_add("/hbitmap/truncate/nop", test_hbitmap_truncate_nop);
+    hbitmap_test_add("/hbitmap/truncate/grow/negligible",
+                     test_hbitmap_truncate_grow_negligible);
+    hbitmap_test_add("/hbitmap/truncate/shrink/negligible",
+                     test_hbitmap_truncate_shrink_negligible);
+    hbitmap_test_add("/hbitmap/truncate/grow/tiny",
+                     test_hbitmap_truncate_grow_tiny);
+    hbitmap_test_add("/hbitmap/truncate/shrink/tiny",
+                     test_hbitmap_truncate_shrink_tiny);
+    hbitmap_test_add("/hbitmap/truncate/grow/small",
+                     test_hbitmap_truncate_grow_small);
+    hbitmap_test_add("/hbitmap/truncate/shrink/small",
+                     test_hbitmap_truncate_shrink_small);
+    hbitmap_test_add("/hbitmap/truncate/grow/medium",
+                     test_hbitmap_truncate_grow_medium);
+    hbitmap_test_add("/hbitmap/truncate/shrink/medium",
+                     test_hbitmap_truncate_shrink_medium);
+    hbitmap_test_add("/hbitmap/truncate/grow/large",
+                     test_hbitmap_truncate_grow_large);
+    hbitmap_test_add("/hbitmap/truncate/shrink/large",
+                     test_hbitmap_truncate_shrink_large);
     g_test_run();
 
     return 0;
diff --git a/tests/test-int128.c b/tests/test-int128.c
index 0772ef7..4390123 100644
--- a/tests/test-int128.c
+++ b/tests/test-int128.c
@@ -6,10 +6,8 @@
  *
  */
 
-#include <glib.h>
-#include <stdio.h>
-#include "qemu/int128.h"
 #include "qemu/osdep.h"
+#include "qemu/int128.h"
 
 /* clang doesn't support __noclone__ but it does have a mechanism for
  * telling us this. We assume that if we don't have __has_attribute()
diff --git a/tests/test-io-channel-buffer.c b/tests/test-io-channel-buffer.c
new file mode 100644
index 0000000..64722a2
--- /dev/null
+++ b/tests/test-io-channel-buffer.c
@@ -0,0 +1,51 @@
+/*
+ * QEMU I/O channel buffer test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-buffer.h"
+#include "io-channel-helpers.h"
+
+
+static void test_io_channel_buf(void)
+{
+    QIOChannelBuffer *buf;
+    QIOChannelTest *test;
+
+    buf = qio_channel_buffer_new(0);
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_writer(test, QIO_CHANNEL(buf));
+    buf->offset = 0;
+    qio_channel_test_run_reader(test, QIO_CHANNEL(buf));
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(buf));
+}
+
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/io/channel/buf", test_io_channel_buf);
+    return g_test_run();
+}
diff --git a/tests/test-io-channel-command.c b/tests/test-io-channel-command.c
new file mode 100644
index 0000000..1d1f461
--- /dev/null
+++ b/tests/test-io-channel-command.c
@@ -0,0 +1,131 @@
+/*
+ * QEMU I/O channel command test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-command.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+#ifndef WIN32
+static void test_io_channel_command_fifo(bool async)
+{
+#define TEST_FIFO "tests/test-io-channel-command.fifo"
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+    char *srcfifo = g_strdup_printf("PIPE:%s,wronly", TEST_FIFO);
+    char *dstfifo = g_strdup_printf("PIPE:%s,rdonly", TEST_FIFO);
+    const char *srcargv[] = {
+        "/bin/socat", "-", srcfifo, NULL,
+    };
+    const char *dstargv[] = {
+        "/bin/socat", dstfifo, "-", NULL,
+    };
+
+    unlink(TEST_FIFO);
+    if (access("/bin/socat", X_OK) < 0) {
+        return; /* Pretend success if socat is not present */
+    }
+    if (mkfifo(TEST_FIFO, 0600) < 0) {
+        abort();
+    }
+    src = QIO_CHANNEL(qio_channel_command_new_spawn(srcargv,
+                                                    O_WRONLY,
+                                                    &error_abort));
+    dst = QIO_CHANNEL(qio_channel_command_new_spawn(dstargv,
+                                                    O_RDONLY,
+                                                    &error_abort));
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, async, src, dst);
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+
+    g_free(srcfifo);
+    g_free(dstfifo);
+    unlink(TEST_FIFO);
+}
+
+
+static void test_io_channel_command_fifo_async(void)
+{
+    test_io_channel_command_fifo(true);
+}
+
+static void test_io_channel_command_fifo_sync(void)
+{
+    test_io_channel_command_fifo(false);
+}
+
+
+static void test_io_channel_command_echo(bool async)
+{
+    QIOChannel *ioc;
+    QIOChannelTest *test;
+    const char *socatargv[] = {
+        "/bin/socat", "-", "-", NULL,
+    };
+
+    if (access("/bin/socat", X_OK) < 0) {
+        return; /* Pretend success if socat is not present */
+    }
+
+    ioc = QIO_CHANNEL(qio_channel_command_new_spawn(socatargv,
+                                                    O_RDWR,
+                                                    &error_abort));
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, async, ioc, ioc);
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(ioc));
+}
+
+
+static void test_io_channel_command_echo_async(void)
+{
+    test_io_channel_command_echo(true);
+}
+
+static void test_io_channel_command_echo_sync(void)
+{
+    test_io_channel_command_echo(false);
+}
+#endif
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+
+    g_test_init(&argc, &argv, NULL);
+
+#ifndef WIN32
+    g_test_add_func("/io/channel/command/fifo/sync",
+                    test_io_channel_command_fifo_sync);
+    g_test_add_func("/io/channel/command/fifo/async",
+                    test_io_channel_command_fifo_async);
+    g_test_add_func("/io/channel/command/echo/sync",
+                    test_io_channel_command_echo_sync);
+    g_test_add_func("/io/channel/command/echo/async",
+                    test_io_channel_command_echo_async);
+#endif
+
+    return g_test_run();
+}
diff --git a/tests/test-io-channel-file.c b/tests/test-io-channel-file.c
new file mode 100644
index 0000000..6bfede6
--- /dev/null
+++ b/tests/test-io-channel-file.c
@@ -0,0 +1,123 @@
+/*
+ * QEMU I/O channel file test
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-file.h"
+#include "io/channel-util.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+static void test_io_channel_file(void)
+{
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+
+#define TEST_FILE "tests/test-io-channel-file.txt"
+    unlink(TEST_FILE);
+    src = QIO_CHANNEL(qio_channel_file_new_path(
+                          TEST_FILE,
+                          O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0600,
+                          &error_abort));
+    dst = QIO_CHANNEL(qio_channel_file_new_path(
+                          TEST_FILE,
+                          O_RDONLY | O_BINARY, 0,
+                          &error_abort));
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_writer(test, src);
+    qio_channel_test_run_reader(test, dst);
+    qio_channel_test_validate(test);
+
+    unlink(TEST_FILE);
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+}
+
+
+static void test_io_channel_fd(void)
+{
+    QIOChannel *ioc;
+    int fd = -1;
+
+#define TEST_FILE "tests/test-io-channel-file.txt"
+    fd = open(TEST_FILE, O_CREAT | O_TRUNC | O_WRONLY, 0600);
+    g_assert_cmpint(fd, >, -1);
+
+    ioc = qio_channel_new_fd(fd, &error_abort);
+
+    g_assert_cmpstr(object_get_typename(OBJECT(ioc)),
+                    ==,
+                    TYPE_QIO_CHANNEL_FILE);
+
+    unlink(TEST_FILE);
+    object_unref(OBJECT(ioc));
+}
+
+
+#ifndef _WIN32
+static void test_io_channel_pipe(bool async)
+{
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+    int fd[2];
+
+    if (pipe(fd) < 0) {
+        perror("pipe");
+        abort();
+    }
+
+    src = QIO_CHANNEL(qio_channel_file_new_fd(fd[1]));
+    dst = QIO_CHANNEL(qio_channel_file_new_fd(fd[0]));
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, async, src, dst);
+    qio_channel_test_validate(test);
+
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+}
+
+
+static void test_io_channel_pipe_async(void)
+{
+    test_io_channel_pipe(true);
+}
+
+static void test_io_channel_pipe_sync(void)
+{
+    test_io_channel_pipe(false);
+}
+#endif /* ! _WIN32 */
+
+
+int main(int argc, char **argv)
+{
+    module_call_init(MODULE_INIT_QOM);
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/io/channel/file", test_io_channel_file);
+    g_test_add_func("/io/channel/file/fd", test_io_channel_fd);
+#ifndef _WIN32
+    g_test_add_func("/io/channel/pipe/sync", test_io_channel_pipe_sync);
+    g_test_add_func("/io/channel/pipe/async", test_io_channel_pipe_async);
+#endif
+    return g_test_run();
+}
diff --git a/tests/test-io-channel-socket.c b/tests/test-io-channel-socket.c
new file mode 100644
index 0000000..f73e063
--- /dev/null
+++ b/tests/test-io-channel-socket.c
@@ -0,0 +1,568 @@
+/*
+ * QEMU I/O channel sockets test
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "io/channel-socket.h"
+#include "io/channel-util.h"
+#include "io-channel-helpers.h"
+#include "qapi/error.h"
+
+#ifndef AI_ADDRCONFIG
+# define AI_ADDRCONFIG 0
+#endif
+#ifndef EAI_ADDRFAMILY
+# define EAI_ADDRFAMILY 0
+#endif
+
+static int check_bind(const char *hostname, bool *has_proto)
+{
+    int fd = -1;
+    struct addrinfo ai, *res = NULL;
+    int rc;
+    int ret = -1;
+
+    memset(&ai, 0, sizeof(ai));
+    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
+    ai.ai_family = AF_UNSPEC;
+    ai.ai_socktype = SOCK_STREAM;
+
+    /* lookup */
+    rc = getaddrinfo(hostname, NULL, &ai, &res);
+    if (rc != 0) {
+        if (rc == EAI_ADDRFAMILY ||
+            rc == EAI_FAMILY) {
+            *has_proto = false;
+            goto done;
+        }
+        goto cleanup;
+    }
+
+    fd = qemu_socket(res->ai_family, res->ai_socktype, res->ai_protocol);
+    if (fd < 0) {
+        goto cleanup;
+    }
+
+    if (bind(fd, res->ai_addr, res->ai_addrlen) < 0) {
+        if (errno == EADDRNOTAVAIL) {
+            *has_proto = false;
+            goto done;
+        }
+        goto cleanup;
+    }
+
+    *has_proto = true;
+ done:
+    ret = 0;
+
+ cleanup:
+    if (fd != -1) {
+        close(fd);
+    }
+    if (res) {
+        freeaddrinfo(res);
+    }
+    return ret;
+}
+
+static int check_protocol_support(bool *has_ipv4, bool *has_ipv6)
+{
+    if (check_bind("127.0.0.1", has_ipv4) < 0) {
+        return -1;
+    }
+    if (check_bind("::1", has_ipv6) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static void test_io_channel_set_socket_bufs(QIOChannel *src,
+                                            QIOChannel *dst)
+{
+    int buflen = 64 * 1024;
+
+    /*
+     * Make the socket buffers small so that we see
+     * the effects of partial reads/writes
+     */
+    setsockopt(((QIOChannelSocket *)src)->fd,
+               SOL_SOCKET, SO_SNDBUF,
+               (char *)&buflen,
+               sizeof(buflen));
+
+    setsockopt(((QIOChannelSocket *)dst)->fd,
+               SOL_SOCKET, SO_SNDBUF,
+               (char *)&buflen,
+               sizeof(buflen));
+}
+
+
+static void test_io_channel_setup_sync(SocketAddress *listen_addr,
+                                       SocketAddress *connect_addr,
+                                       QIOChannel **src,
+                                       QIOChannel **dst)
+{
+    QIOChannelSocket *lioc;
+
+    lioc = qio_channel_socket_new();
+    qio_channel_socket_listen_sync(lioc, listen_addr, &error_abort);
+
+    if (listen_addr->type == SOCKET_ADDRESS_KIND_INET) {
+        SocketAddress *laddr = qio_channel_socket_get_local_address(
+            lioc, &error_abort);
+
+        g_free(connect_addr->u.inet.data->port);
+        connect_addr->u.inet.data->port = g_strdup(laddr->u.inet.data->port);
+
+        qapi_free_SocketAddress(laddr);
+    }
+
+    *src = QIO_CHANNEL(qio_channel_socket_new());
+    qio_channel_socket_connect_sync(
+        QIO_CHANNEL_SOCKET(*src), connect_addr, &error_abort);
+    qio_channel_set_delay(*src, false);
+
+    qio_channel_wait(QIO_CHANNEL(lioc), G_IO_IN);
+    *dst = QIO_CHANNEL(qio_channel_socket_accept(lioc, &error_abort));
+    g_assert(*dst);
+
+    test_io_channel_set_socket_bufs(*src, *dst);
+
+    object_unref(OBJECT(lioc));
+}
+
+
+struct TestIOChannelData {
+    bool err;
+    GMainLoop *loop;
+};
+
+
+static void test_io_channel_complete(Object *src,
+                                     Error *err,
+                                     gpointer opaque)
+{
+    struct TestIOChannelData *data = opaque;
+    data->err = err != NULL;
+    g_main_loop_quit(data->loop);
+}
+
+
+static void test_io_channel_setup_async(SocketAddress *listen_addr,
+                                        SocketAddress *connect_addr,
+                                        QIOChannel **src,
+                                        QIOChannel **dst)
+{
+    QIOChannelSocket *lioc;
+    struct TestIOChannelData data;
+
+    data.loop = g_main_loop_new(g_main_context_default(),
+                                TRUE);
+
+    lioc = qio_channel_socket_new();
+    qio_channel_socket_listen_async(
+        lioc, listen_addr,
+        test_io_channel_complete, &data, NULL);
+
+    g_main_loop_run(data.loop);
+    g_main_context_iteration(g_main_context_default(), FALSE);
+
+    g_assert(!data.err);
+
+    if (listen_addr->type == SOCKET_ADDRESS_KIND_INET) {
+        SocketAddress *laddr = qio_channel_socket_get_local_address(
+            lioc, &error_abort);
+
+        g_free(connect_addr->u.inet.data->port);
+        connect_addr->u.inet.data->port = g_strdup(laddr->u.inet.data->port);
+
+        qapi_free_SocketAddress(laddr);
+    }
+
+    *src = QIO_CHANNEL(qio_channel_socket_new());
+
+    qio_channel_socket_connect_async(
+        QIO_CHANNEL_SOCKET(*src), connect_addr,
+        test_io_channel_complete, &data, NULL);
+
+    g_main_loop_run(data.loop);
+    g_main_context_iteration(g_main_context_default(), FALSE);
+
+    g_assert(!data.err);
+
+    qio_channel_wait(QIO_CHANNEL(lioc), G_IO_IN);
+    *dst = QIO_CHANNEL(qio_channel_socket_accept(lioc, &error_abort));
+    g_assert(*dst);
+
+    qio_channel_set_delay(*src, false);
+    test_io_channel_set_socket_bufs(*src, *dst);
+
+    object_unref(OBJECT(lioc));
+
+    g_main_loop_unref(data.loop);
+}
+
+
+static void test_io_channel(bool async,
+                            SocketAddress *listen_addr,
+                            SocketAddress *connect_addr,
+                            bool passFD)
+{
+    QIOChannel *src, *dst;
+    QIOChannelTest *test;
+    if (async) {
+        test_io_channel_setup_async(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, true, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+
+        test_io_channel_setup_async(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, false, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+    } else {
+        test_io_channel_setup_sync(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, true, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+
+        test_io_channel_setup_sync(listen_addr, connect_addr, &src, &dst);
+
+        g_assert(!passFD ||
+                 qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+        g_assert(!passFD ||
+                 qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+        test = qio_channel_test_new();
+        qio_channel_test_run_threads(test, false, src, dst);
+        qio_channel_test_validate(test);
+
+        object_unref(OBJECT(src));
+        object_unref(OBJECT(dst));
+    }
+}
+
+
+static void test_io_channel_ipv4(bool async)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+
+    listen_addr->type = SOCKET_ADDRESS_KIND_INET;
+    listen_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *listen_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("127.0.0.1"),
+        .port = NULL, /* Auto-select */
+    };
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_INET;
+    connect_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *connect_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("127.0.0.1"),
+        .port = NULL, /* Filled in later */
+    };
+
+    test_io_channel(async, listen_addr, connect_addr, false);
+
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+}
+
+
+static void test_io_channel_ipv4_sync(void)
+{
+    return test_io_channel_ipv4(false);
+}
+
+
+static void test_io_channel_ipv4_async(void)
+{
+    return test_io_channel_ipv4(true);
+}
+
+
+static void test_io_channel_ipv6(bool async)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+
+    listen_addr->type = SOCKET_ADDRESS_KIND_INET;
+    listen_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *listen_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("::1"),
+        .port = NULL, /* Auto-select */
+    };
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_INET;
+    connect_addr->u.inet.data = g_new(InetSocketAddress, 1);
+    *connect_addr->u.inet.data = (InetSocketAddress) {
+        .host = g_strdup("::1"),
+        .port = NULL, /* Filled in later */
+    };
+
+    test_io_channel(async, listen_addr, connect_addr, false);
+
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+}
+
+
+static void test_io_channel_ipv6_sync(void)
+{
+    return test_io_channel_ipv6(false);
+}
+
+
+static void test_io_channel_ipv6_async(void)
+{
+    return test_io_channel_ipv6(true);
+}
+
+
+#ifndef _WIN32
+static void test_io_channel_unix(bool async)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+
+#define TEST_SOCKET "test-io-channel-socket.sock"
+    listen_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    listen_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    listen_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    connect_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    connect_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    test_io_channel(async, listen_addr, connect_addr, true);
+
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+    g_assert(g_file_test(TEST_SOCKET, G_FILE_TEST_EXISTS) == FALSE);
+}
+
+
+static void test_io_channel_unix_sync(void)
+{
+    return test_io_channel_unix(false);
+}
+
+
+static void test_io_channel_unix_async(void)
+{
+    return test_io_channel_unix(true);
+}
+
+static void test_io_channel_unix_fd_pass(void)
+{
+    SocketAddress *listen_addr = g_new0(SocketAddress, 1);
+    SocketAddress *connect_addr = g_new0(SocketAddress, 1);
+    QIOChannel *src, *dst;
+    int testfd;
+    int fdsend[3];
+    int *fdrecv = NULL;
+    size_t nfdrecv = 0;
+    size_t i;
+    char bufsend[12], bufrecv[12];
+    struct iovec iosend[1], iorecv[1];
+
+#define TEST_SOCKET "test-io-channel-socket.sock"
+#define TEST_FILE "test-io-channel-socket.txt"
+
+    testfd = open(TEST_FILE, O_RDWR|O_TRUNC|O_CREAT, 0700);
+    g_assert(testfd != -1);
+    fdsend[0] = testfd;
+    fdsend[1] = testfd;
+    fdsend[2] = testfd;
+
+    listen_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    listen_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    listen_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    connect_addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    connect_addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    connect_addr->u.q_unix.data->path = g_strdup(TEST_SOCKET);
+
+    test_io_channel_setup_sync(listen_addr, connect_addr, &src, &dst);
+
+    memcpy(bufsend, "Hello World", G_N_ELEMENTS(bufsend));
+
+    iosend[0].iov_base = bufsend;
+    iosend[0].iov_len = G_N_ELEMENTS(bufsend);
+
+    iorecv[0].iov_base = bufrecv;
+    iorecv[0].iov_len = G_N_ELEMENTS(bufrecv);
+
+    g_assert(qio_channel_has_feature(src, QIO_CHANNEL_FEATURE_FD_PASS));
+    g_assert(qio_channel_has_feature(dst, QIO_CHANNEL_FEATURE_FD_PASS));
+
+    qio_channel_writev_full(src,
+                            iosend,
+                            G_N_ELEMENTS(iosend),
+                            fdsend,
+                            G_N_ELEMENTS(fdsend),
+                            &error_abort);
+
+    qio_channel_readv_full(dst,
+                           iorecv,
+                           G_N_ELEMENTS(iorecv),
+                           &fdrecv,
+                           &nfdrecv,
+                           &error_abort);
+
+    g_assert(nfdrecv == G_N_ELEMENTS(fdsend));
+    /* Each recvd FD should be different from sent FD */
+    for (i = 0; i < nfdrecv; i++) {
+        g_assert_cmpint(fdrecv[i], !=, testfd);
+    }
+    /* Each recvd FD should be different from each other */
+    g_assert_cmpint(fdrecv[0], !=, fdrecv[1]);
+    g_assert_cmpint(fdrecv[0], !=, fdrecv[2]);
+    g_assert_cmpint(fdrecv[1], !=, fdrecv[2]);
+
+    /* Check the I/O buf we sent at the same time matches */
+    g_assert(memcmp(bufsend, bufrecv, G_N_ELEMENTS(bufsend)) == 0);
+
+    /* Write some data into the FD we received */
+    g_assert(write(fdrecv[0], bufsend, G_N_ELEMENTS(bufsend)) ==
+             G_N_ELEMENTS(bufsend));
+
+    /* Read data from the original FD and make sure it matches */
+    memset(bufrecv, 0, G_N_ELEMENTS(bufrecv));
+    g_assert(lseek(testfd, 0, SEEK_SET) == 0);
+    g_assert(read(testfd, bufrecv, G_N_ELEMENTS(bufrecv)) ==
+             G_N_ELEMENTS(bufrecv));
+    g_assert(memcmp(bufsend, bufrecv, G_N_ELEMENTS(bufsend)) == 0);
+
+    object_unref(OBJECT(src));
+    object_unref(OBJECT(dst));
+    qapi_free_SocketAddress(listen_addr);
+    qapi_free_SocketAddress(connect_addr);
+    unlink(TEST_SOCKET);
+    unlink(TEST_FILE);
+    close(testfd);
+    for (i = 0; i < nfdrecv; i++) {
+        close(fdrecv[i]);
+    }
+    g_free(fdrecv);
+}
+#endif /* _WIN32 */
+
+
+static void test_io_channel_ipv4_fd(void)
+{
+    QIOChannel *ioc;
+    int fd = -1;
+    struct sockaddr_in sa = {
+        .sin_family = AF_INET,
+        .sin_addr = {
+            .s_addr =  htonl(INADDR_LOOPBACK),
+        }
+        /* Leave port unset for auto-assign */
+    };
+    socklen_t salen = sizeof(sa);
+
+    fd = socket(AF_INET, SOCK_STREAM, 0);
+    g_assert_cmpint(fd, >, -1);
+
+    g_assert_cmpint(bind(fd, (struct sockaddr *)&sa, salen), ==, 0);
+
+    ioc = qio_channel_new_fd(fd, &error_abort);
+
+    g_assert_cmpstr(object_get_typename(OBJECT(ioc)),
+                    ==,
+                    TYPE_QIO_CHANNEL_SOCKET);
+
+    object_unref(OBJECT(ioc));
+}
+
+
+int main(int argc, char **argv)
+{
+    bool has_ipv4, has_ipv6;
+
+    module_call_init(MODULE_INIT_QOM);
+    socket_init();
+
+    g_test_init(&argc, &argv, NULL);
+
+    /* We're creating actual IPv4/6 sockets, so we should
+     * check if the host running tests actually supports
+     * each protocol to avoid breaking tests on machines
+     * with either IPv4 or IPv6 disabled.
+     */
+    if (check_protocol_support(&has_ipv4, &has_ipv6) < 0) {
+        return 1;
+    }
+
+    if (has_ipv4) {
+        g_test_add_func("/io/channel/socket/ipv4-sync",
+                        test_io_channel_ipv4_sync);
+        g_test_add_func("/io/channel/socket/ipv4-async",
+                        test_io_channel_ipv4_async);
+        g_test_add_func("/io/channel/socket/ipv4-fd",
+                        test_io_channel_ipv4_fd);
+    }
+    if (has_ipv6) {
+        g_test_add_func("/io/channel/socket/ipv6-sync",
+                        test_io_channel_ipv6_sync);
+        g_test_add_func("/io/channel/socket/ipv6-async",
+                        test_io_channel_ipv6_async);
+    }
+
+#ifndef _WIN32
+    g_test_add_func("/io/channel/socket/unix-sync",
+                    test_io_channel_unix_sync);
+    g_test_add_func("/io/channel/socket/unix-async",
+                    test_io_channel_unix_async);
+    g_test_add_func("/io/channel/socket/unix-fd-pass",
+                    test_io_channel_unix_fd_pass);
+#endif /* _WIN32 */
+
+    return g_test_run();
+}
diff --git a/tests/test-io-channel-tls.c b/tests/test-io-channel-tls.c
new file mode 100644
index 0000000..3c361a7
--- /dev/null
+++ b/tests/test-io-channel-tls.c
@@ -0,0 +1,340 @@
+/*
+ * QEMU I/O channel TLS test
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Author: Daniel P. Berrange <berrange@redhat.com>
+ */
+
+
+#include "qemu/osdep.h"
+
+#include "crypto-tls-x509-helpers.h"
+#include "io/channel-tls.h"
+#include "io/channel-socket.h"
+#include "io-channel-helpers.h"
+#include "crypto/tlscredsx509.h"
+#include "qemu/acl.h"
+#include "qom/object_interfaces.h"
+
+#ifdef QCRYPTO_HAVE_TLS_TEST_SUPPORT
+
+#define WORKDIR "tests/test-io-channel-tls-work/"
+#define KEYFILE WORKDIR "key-ctx.pem"
+
+struct QIOChannelTLSTestData {
+    const char *servercacrt;
+    const char *clientcacrt;
+    const char *servercrt;
+    const char *clientcrt;
+    bool expectServerFail;
+    bool expectClientFail;
+    const char *hostname;
+    const char *const *wildcards;
+};
+
+struct QIOChannelTLSHandshakeData {
+    bool finished;
+    bool failed;
+};
+
+static void test_tls_handshake_done(Object *source,
+                                    Error *err,
+                                    gpointer opaque)
+{
+    struct QIOChannelTLSHandshakeData *data = opaque;
+
+    data->finished = true;
+    data->failed = err != NULL;
+}
+
+
+static QCryptoTLSCreds *test_tls_creds_create(QCryptoTLSCredsEndpoint endpoint,
+                                              const char *certdir,
+                                              Error **errp)
+{
+    Object *parent = object_get_objects_root();
+    Object *creds = object_new_with_props(
+        TYPE_QCRYPTO_TLS_CREDS_X509,
+        parent,
+        (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+         "testtlscredsserver" : "testtlscredsclient"),
+        errp,
+        "endpoint", (endpoint == QCRYPTO_TLS_CREDS_ENDPOINT_SERVER ?
+                     "server" : "client"),
+        "dir", certdir,
+        "verify-peer", "yes",
+        /* We skip initial sanity checks here because we
+         * want to make sure that problems are being
+         * detected at the TLS session validation stage,
+         * and the test-crypto-tlscreds test already
+         * validate the sanity check code.
+         */
+        "sanity-check", "no",
+        NULL
+        );
+
+    if (*errp) {
+        return NULL;
+    }
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
+/*
+ * This tests validation checking of peer certificates
+ *
+ * This is replicating the checks that are done for an
+ * active TLS session after handshake completes. To
+ * simulate that we create our TLS contexts, skipping
+ * sanity checks. When then get a socketpair, and
+ * initiate a TLS session across them. Finally do
+ * do actual cert validation tests
+ */
+static void test_io_channel_tls(const void *opaque)
+{
+    struct QIOChannelTLSTestData *data =
+        (struct QIOChannelTLSTestData *)opaque;
+    QCryptoTLSCreds *clientCreds;
+    QCryptoTLSCreds *serverCreds;
+    QIOChannelTLS *clientChanTLS;
+    QIOChannelTLS *serverChanTLS;
+    QIOChannelSocket *clientChanSock;
+    QIOChannelSocket *serverChanSock;
+    qemu_acl *acl;
+    const char * const *wildcards;
+    int channel[2];
+    struct QIOChannelTLSHandshakeData clientHandshake = { false, false };
+    struct QIOChannelTLSHandshakeData serverHandshake = { false, false };
+    Error *err = NULL;
+    QIOChannelTest *test;
+    GMainContext *mainloop;
+
+    /* We'll use this for our fake client-server connection */
+    g_assert(socketpair(AF_UNIX, SOCK_STREAM, 0, channel) == 0);
+
+#define CLIENT_CERT_DIR "tests/test-crypto-tlssession-client/"
+#define SERVER_CERT_DIR "tests/test-crypto-tlssession-server/"
+    mkdir(CLIENT_CERT_DIR, 0700);
+    mkdir(SERVER_CERT_DIR, 0700);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    g_assert(link(data->servercacrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->servercrt,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY) == 0);
+
+    g_assert(link(data->clientcacrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT) == 0);
+    g_assert(link(data->clientcrt,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT) == 0);
+    g_assert(link(KEYFILE,
+                  CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY) == 0);
+
+    clientCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_CLIENT,
+        CLIENT_CERT_DIR,
+        &err);
+    g_assert(clientCreds != NULL);
+
+    serverCreds = test_tls_creds_create(
+        QCRYPTO_TLS_CREDS_ENDPOINT_SERVER,
+        SERVER_CERT_DIR,
+        &err);
+    g_assert(serverCreds != NULL);
+
+    acl = qemu_acl_init("channeltlsacl");
+    qemu_acl_reset(acl);
+    wildcards = data->wildcards;
+    while (wildcards && *wildcards) {
+        qemu_acl_append(acl, 0, *wildcards);
+        wildcards++;
+    }
+
+    clientChanSock = qio_channel_socket_new_fd(
+        channel[0], &err);
+    g_assert(clientChanSock != NULL);
+    serverChanSock = qio_channel_socket_new_fd(
+        channel[1], &err);
+    g_assert(serverChanSock != NULL);
+
+    /*
+     * We have an evil loop to do the handshake in a single
+     * thread, so we need these non-blocking to avoid deadlock
+     * of ourselves
+     */
+    qio_channel_set_blocking(QIO_CHANNEL(clientChanSock), false, NULL);
+    qio_channel_set_blocking(QIO_CHANNEL(serverChanSock), false, NULL);
+
+    /* Now the real part of the test, setup the sessions */
+    clientChanTLS = qio_channel_tls_new_client(
+        QIO_CHANNEL(clientChanSock), clientCreds,
+        data->hostname, &err);
+    g_assert(clientChanTLS != NULL);
+
+    serverChanTLS = qio_channel_tls_new_server(
+        QIO_CHANNEL(serverChanSock), serverCreds,
+        "channeltlsacl", &err);
+    g_assert(serverChanTLS != NULL);
+
+    qio_channel_tls_handshake(clientChanTLS,
+                              test_tls_handshake_done,
+                              &clientHandshake,
+                              NULL);
+    qio_channel_tls_handshake(serverChanTLS,
+                              test_tls_handshake_done,
+                              &serverHandshake,
+                              NULL);
+
+    /*
+     * Finally we loop around & around doing handshake on each
+     * session until we get an error, or the handshake completes.
+     * This relies on the socketpair being nonblocking to avoid
+     * deadlocking ourselves upon handshake
+     */
+    mainloop = g_main_context_default();
+    do {
+        g_main_context_iteration(mainloop, TRUE);
+    } while (!clientHandshake.finished &&
+             !serverHandshake.finished);
+
+    g_assert(clientHandshake.failed == data->expectClientFail);
+    g_assert(serverHandshake.failed == data->expectServerFail);
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, false,
+                                 QIO_CHANNEL(clientChanTLS),
+                                 QIO_CHANNEL(serverChanTLS));
+    qio_channel_test_validate(test);
+
+    test = qio_channel_test_new();
+    qio_channel_test_run_threads(test, true,
+                                 QIO_CHANNEL(clientChanTLS),
+                                 QIO_CHANNEL(serverChanTLS));
+    qio_channel_test_validate(test);
+
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_CERT);
+    unlink(SERVER_CERT_DIR QCRYPTO_TLS_CREDS_X509_SERVER_KEY);
+
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CA_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_CERT);
+    unlink(CLIENT_CERT_DIR QCRYPTO_TLS_CREDS_X509_CLIENT_KEY);
+
+    rmdir(CLIENT_CERT_DIR);
+    rmdir(SERVER_CERT_DIR);
+
+    object_unparent(OBJECT(serverCreds));
+    object_unparent(OBJECT(clientCreds));
+
+    object_unref(OBJECT(serverChanTLS));
+    object_unref(OBJECT(clientChanTLS));
+
+    object_unref(OBJECT(serverChanSock));
+    object_unref(OBJECT(clientChanSock));
+
+    close(channel[0]);
+    close(channel[1]);
+}
+
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    module_call_init(MODULE_INIT_QOM);
+    g_test_init(&argc, &argv, NULL);
+    setenv("GNUTLS_FORCE_FIPS_MODE", "2", 1);
+
+    mkdir(WORKDIR, 0700);
+
+    test_tls_init(KEYFILE);
+
+# define TEST_CHANNEL(name, caCrt,                                      \
+                      serverCrt, clientCrt,                             \
+                      expectServerFail, expectClientFail,               \
+                      hostname, wildcards)                              \
+    struct QIOChannelTLSTestData name = {                               \
+        caCrt, caCrt, serverCrt, clientCrt,                             \
+        expectServerFail, expectClientFail,                             \
+        hostname, wildcards                                             \
+    };                                                                  \
+    g_test_add_data_func("/qio/channel/tls/" # name,                    \
+                         &name, test_io_channel_tls);
+
+    /* A perfect CA, perfect client & perfect server */
+
+    /* Basic:CA:critical */
+    TLS_ROOT_REQ(cacertreq,
+                 "UK", "qemu CA", NULL, NULL, NULL, NULL,
+                 true, true, true,
+                 true, true, GNUTLS_KEY_KEY_CERT_SIGN,
+                 false, false, NULL, NULL,
+                 0, 0);
+    TLS_CERT_REQ(servercertreq, cacertreq,
+                 "UK", "qemu.org", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_SERVER, NULL,
+                 0, 0);
+    TLS_CERT_REQ(clientcertreq, cacertreq,
+                 "UK", "qemu", NULL, NULL, NULL, NULL,
+                 true, true, false,
+                 true, true,
+                 GNUTLS_KEY_DIGITAL_SIGNATURE | GNUTLS_KEY_KEY_ENCIPHERMENT,
+                 true, true, GNUTLS_KP_TLS_WWW_CLIENT, NULL,
+                 0, 0);
+
+    const char *const wildcards[] = {
+        "C=UK,CN=qemu*",
+        NULL,
+    };
+    TEST_CHANNEL(basic, cacertreq.filename, servercertreq.filename,
+                 clientcertreq.filename, false, false,
+                 "qemu.org", wildcards);
+
+    ret = g_test_run();
+
+    test_tls_discard_cert(&clientcertreq);
+    test_tls_discard_cert(&servercertreq);
+    test_tls_discard_cert(&cacertreq);
+
+    test_tls_cleanup(KEYFILE);
+    rmdir(WORKDIR);
+
+    return ret == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
+}
+
+#else /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
+
+int
+main(void)
+{
+    return EXIT_SUCCESS;
+}
+
+#endif /* ! QCRYPTO_HAVE_TLS_TEST_SUPPORT */
diff --git a/tests/test-io-task.c b/tests/test-io-task.c
new file mode 100644
index 0000000..e091c12
--- /dev/null
+++ b/tests/test-io-task.c
@@ -0,0 +1,269 @@
+/*
+ * QEMU I/O task tests
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+
+#include "io/task.h"
+#include "qapi/error.h"
+
+#define TYPE_DUMMY "qemu:dummy"
+
+typedef struct DummyObject DummyObject;
+typedef struct DummyObjectClass DummyObjectClass;
+
+struct DummyObject {
+    Object parent;
+};
+
+struct DummyObjectClass {
+    ObjectClass parent;
+};
+
+static const TypeInfo dummy_info = {
+    .parent = TYPE_OBJECT,
+    .name = TYPE_DUMMY,
+    .instance_size = sizeof(DummyObject),
+    .class_size = sizeof(DummyObjectClass),
+};
+
+struct TestTaskData {
+    Object *source;
+    Error *err;
+    bool freed;
+};
+
+
+static void task_callback(Object *source,
+                          Error *err,
+                          gpointer opaque)
+{
+    struct TestTaskData *data = opaque;
+
+    data->source = source;
+    data->err = err;
+}
+
+
+static void test_task_complete(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    Object *src;
+    struct TestTaskData data = { NULL, NULL, false };
+
+    task = qio_task_new(obj, task_callback, &data, NULL);
+    src = qio_task_get_source(task);
+
+    qio_task_complete(task);
+
+    g_assert(obj == src);
+
+    object_unref(obj);
+    object_unref(src);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == NULL);
+    g_assert(data.freed == false);
+}
+
+
+static void task_data_free(gpointer opaque)
+{
+    struct TestTaskData *data = opaque;
+
+    data->freed = true;
+}
+
+
+static void test_task_data_free(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestTaskData data = { NULL, NULL, false };
+
+    task = qio_task_new(obj, task_callback, &data, task_data_free);
+
+    qio_task_complete(task);
+
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == NULL);
+    g_assert(data.freed == true);
+}
+
+
+static void test_task_failure(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestTaskData data = { NULL, NULL, false };
+    Error *err = NULL;
+
+    task = qio_task_new(obj, task_callback, &data, NULL);
+
+    error_setg(&err, "Some error");
+
+    qio_task_abort(task, err);
+
+    error_free(err);
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == err);
+    g_assert(data.freed == false);
+
+}
+
+
+struct TestThreadWorkerData {
+    Object *source;
+    Error *err;
+    bool fail;
+    GThread *worker;
+    GThread *complete;
+    GMainLoop *loop;
+};
+
+static int test_task_thread_worker(QIOTask *task,
+                                   Error **errp,
+                                   gpointer opaque)
+{
+    struct TestThreadWorkerData *data = opaque;
+
+    data->worker = g_thread_self();
+
+    if (data->fail) {
+        error_setg(errp, "Testing fail");
+        return -1;
+    }
+
+    return 0;
+}
+
+
+static void test_task_thread_callback(Object *source,
+                                      Error *err,
+                                      gpointer opaque)
+{
+    struct TestThreadWorkerData *data = opaque;
+
+    data->source = source;
+    data->err = err;
+
+    data->complete = g_thread_self();
+
+    g_main_loop_quit(data->loop);
+}
+
+
+static void test_task_thread_complete(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestThreadWorkerData data = { 0 };
+    GThread *self;
+
+    data.loop = g_main_loop_new(g_main_context_default(),
+                                TRUE);
+
+    task = qio_task_new(obj,
+                        test_task_thread_callback,
+                        &data,
+                        NULL);
+
+    qio_task_run_in_thread(task,
+                           test_task_thread_worker,
+                           &data,
+                           NULL);
+
+    g_main_loop_run(data.loop);
+
+    g_main_loop_unref(data.loop);
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err == NULL);
+
+    self = g_thread_self();
+
+    /* Make sure the test_task_thread_worker actually got
+     * run in a different thread */
+    g_assert(data.worker != self);
+
+    /* And that the test_task_thread_callback got rnu in
+     * the main loop thread (ie this one) */
+    g_assert(data.complete == self);
+}
+
+
+static void test_task_thread_failure(void)
+{
+    QIOTask *task;
+    Object *obj = object_new(TYPE_DUMMY);
+    struct TestThreadWorkerData data = { 0 };
+    GThread *self;
+
+    data.loop = g_main_loop_new(g_main_context_default(),
+                                TRUE);
+    data.fail = true;
+
+    task = qio_task_new(obj,
+                        test_task_thread_callback,
+                        &data,
+                        NULL);
+
+    qio_task_run_in_thread(task,
+                           test_task_thread_worker,
+                           &data,
+                           NULL);
+
+    g_main_loop_run(data.loop);
+
+    g_main_loop_unref(data.loop);
+    object_unref(obj);
+
+    g_assert(data.source == obj);
+    g_assert(data.err != NULL);
+
+    self = g_thread_self();
+
+    /* Make sure the test_task_thread_worker actually got
+     * run in a different thread */
+    g_assert(data.worker != self);
+
+    /* And that the test_task_thread_callback got rnu in
+     * the main loop thread (ie this one) */
+    g_assert(data.complete == self);
+}
+
+
+int main(int argc, char **argv)
+{
+    g_test_init(&argc, &argv, NULL);
+    module_call_init(MODULE_INIT_QOM);
+    type_register_static(&dummy_info);
+    g_test_add_func("/crypto/task/complete", test_task_complete);
+    g_test_add_func("/crypto/task/datafree", test_task_data_free);
+    g_test_add_func("/crypto/task/failure", test_task_failure);
+    g_test_add_func("/crypto/task/thread_complete", test_task_thread_complete);
+    g_test_add_func("/crypto/task/thread_failure", test_task_thread_failure);
+    return g_test_run();
+}
diff --git a/tests/test-iov.c b/tests/test-iov.c
index 46e4ddd..46ae25e 100644
--- a/tests/test-iov.c
+++ b/tests/test-iov.c
@@ -1,4 +1,4 @@
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
diff --git a/tests/test-logging.c b/tests/test-logging.c
new file mode 100644
index 0000000..a12585f
--- /dev/null
+++ b/tests/test-logging.c
@@ -0,0 +1,143 @@
+/*
+ * logging unit-tests
+ *
+ * Copyright (C) 2016 Linaro Ltd.
+ *
+ *  Author: Alex Bennée <alex.bennee@linaro.org>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include <glib/gstdio.h>
+
+#include "qemu-common.h"
+#include "qapi/error.h"
+#include "qemu/log.h"
+
+static void test_parse_range(void)
+{
+    Error *err = NULL;
+
+    qemu_set_dfilter_ranges("0x1000+0x100", &error_abort);
+
+    g_assert_false(qemu_log_in_addr_range(0xfff));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert(qemu_log_in_addr_range(0x1001));
+    g_assert(qemu_log_in_addr_range(0x10ff));
+    g_assert_false(qemu_log_in_addr_range(0x1100));
+
+    qemu_set_dfilter_ranges("0x1000-0x100", &error_abort);
+
+    g_assert_false(qemu_log_in_addr_range(0x1001));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert(qemu_log_in_addr_range(0x0f01));
+    g_assert_false(qemu_log_in_addr_range(0x0f00));
+
+    qemu_set_dfilter_ranges("0x1000..0x1100", &error_abort);
+
+    g_assert_false(qemu_log_in_addr_range(0xfff));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert(qemu_log_in_addr_range(0x1100));
+    g_assert_false(qemu_log_in_addr_range(0x1101));
+
+    qemu_set_dfilter_ranges("0x1000..0x1000", &error_abort);
+
+    g_assert_false(qemu_log_in_addr_range(0xfff));
+    g_assert(qemu_log_in_addr_range(0x1000));
+    g_assert_false(qemu_log_in_addr_range(0x1001));
+
+    qemu_set_dfilter_ranges("0x1000+0x100,0x2100-0x100,0x3000..0x3100",
+                            &error_abort);
+    g_assert(qemu_log_in_addr_range(0x1050));
+    g_assert(qemu_log_in_addr_range(0x2050));
+    g_assert(qemu_log_in_addr_range(0x3050));
+
+    qemu_set_dfilter_ranges("0xffffffffffffffff-1", &error_abort);
+    g_assert(qemu_log_in_addr_range(UINT64_MAX));
+    g_assert_false(qemu_log_in_addr_range(UINT64_MAX - 1));
+
+    qemu_set_dfilter_ranges("0..0xffffffffffffffff", &err);
+    g_assert(qemu_log_in_addr_range(0));
+    g_assert(qemu_log_in_addr_range(UINT64_MAX));
+ 
+    qemu_set_dfilter_ranges("2..1", &err);
+    error_free_or_abort(&err);
+
+    qemu_set_dfilter_ranges("0x1000+onehundred", &err);
+    error_free_or_abort(&err);
+
+    qemu_set_dfilter_ranges("0x1000+0", &err);
+    error_free_or_abort(&err);
+}
+
+static void set_log_path_tmp(char const *dir, char const *tpl, Error **errp)
+{
+    gchar *file_path = g_build_filename(dir, tpl, NULL);
+
+    qemu_set_log_filename(file_path, errp);
+    g_free(file_path);
+}
+
+static void test_parse_path(gconstpointer data)
+{
+    gchar const *tmp_path = data;
+    Error *err = NULL;
+
+    set_log_path_tmp(tmp_path, "qemu.log", &error_abort);
+    set_log_path_tmp(tmp_path, "qemu-%d.log", &error_abort);
+    set_log_path_tmp(tmp_path, "qemu.log.%d", &error_abort);
+
+    set_log_path_tmp(tmp_path, "qemu-%d%d.log", &err);
+    error_free_or_abort(&err);
+}
+
+/* Remove a directory and all its entries (non-recursive). */
+static void rmdir_full(gchar const *root)
+{
+    GDir *root_gdir = g_dir_open(root, 0, NULL);
+    gchar const *entry_name;
+
+    g_assert_nonnull(root_gdir);
+    while ((entry_name = g_dir_read_name(root_gdir)) != NULL) {
+        gchar *entry_path = g_build_filename(root, entry_name, NULL);
+        g_assert(g_remove(entry_path) == 0);
+        g_free(entry_path);
+    }
+    g_dir_close(root_gdir);
+    g_assert(g_rmdir(root) == 0);
+}
+
+int main(int argc, char **argv)
+{
+    gchar *tmp_path = g_dir_make_tmp("qemu-test-logging.XXXXXX", NULL);
+    int rc;
+
+    g_test_init(&argc, &argv, NULL);
+    g_assert_nonnull(tmp_path);
+
+    g_test_add_func("/logging/parse_range", test_parse_range);
+    g_test_add_data_func("/logging/parse_path", tmp_path, test_parse_path);
+
+    rc = g_test_run();
+
+    rmdir_full(tmp_path);
+    g_free(tmp_path);
+    return rc;
+}
diff --git a/tests/test-mul64.c b/tests/test-mul64.c
index a0a17f7..9be775d 100644
--- a/tests/test-mul64.c
+++ b/tests/test-mul64.c
@@ -6,10 +6,8 @@
  *
  */
 
-#include <glib.h>
-#include <stdint.h>
-#include "qemu/host-utils.h"
 #include "qemu/osdep.h"
+#include "qemu/host-utils.h"
 
 
 typedef struct {
diff --git a/tests/test-netfilter.c b/tests/test-netfilter.c
new file mode 100644
index 0000000..8b5a9b2
--- /dev/null
+++ b/tests/test-netfilter.c
@@ -0,0 +1,200 @@
+/*
+ * QTest testcase for netfilter
+ *
+ * Copyright (c) 2015 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "libqtest.h"
+
+/* add a netfilter to a netdev and then remove it */
+static void add_one_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add a netfilter to a netdev and then remove the netdev */
+static void remove_netdev_with_one_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'netdev_del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* add back the netdev */
+    response = qmp("{'execute': 'netdev_add',"
+                   " 'arguments': {"
+                   "   'type': 'user',"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add multi(2) netfilters to a netdev and then remove them */
+static void add_multi_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f1',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-f1'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+/* add multi(2) netfilters to a netdev and then remove the netdev */
+static void remove_netdev_with_multi_netfilter(void)
+{
+    QDict *response;
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f0',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'object-add',"
+                   " 'arguments': {"
+                   "   'qom-type': 'filter-buffer',"
+                   "   'id': 'qtest-f1',"
+                   "   'props': {"
+                   "     'netdev': 'qtest-bn0',"
+                   "     'queue': 'rx',"
+                   "     'interval': 1000"
+                   "}}}");
+
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    response = qmp("{'execute': 'netdev_del',"
+                   " 'arguments': {"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+
+    /* add back the netdev */
+    response = qmp("{'execute': 'netdev_add',"
+                   " 'arguments': {"
+                   "   'type': 'user',"
+                   "   'id': 'qtest-bn0'"
+                   "}}");
+    g_assert(response);
+    g_assert(!qdict_haskey(response, "error"));
+    QDECREF(response);
+}
+
+int main(int argc, char **argv)
+{
+    int ret;
+
+    g_test_init(&argc, &argv, NULL);
+    qtest_add_func("/netfilter/addremove_one", add_one_netfilter);
+    qtest_add_func("/netfilter/remove_netdev_one",
+                   remove_netdev_with_one_netfilter);
+    qtest_add_func("/netfilter/addremove_multi", add_multi_netfilter);
+    qtest_add_func("/netfilter/remove_netdev_multi",
+                   remove_netdev_with_multi_netfilter);
+
+    qtest_start("-netdev user,id=qtest-bn0 -device e1000,netdev=qtest-bn0");
+    ret = g_test_run();
+
+    qtest_end();
+
+    return ret;
+}
diff --git a/tests/test-opts-visitor.c b/tests/test-opts-visitor.c
index ebeee5d..0a9e75f 100644
--- a/tests/test-opts-visitor.c
+++ b/tests/test-opts-visitor.c
@@ -10,13 +10,13 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qemu/config-file.h"     /* qemu_add_opts() */
 #include "qemu/option.h"          /* qemu_opts_parse() */
+#include "qapi/error.h"
 #include "qapi/opts-visitor.h"    /* opts_visitor_new() */
 #include "test-qapi-visit.h"      /* visit_type_UserDefOptions() */
-#include "qapi/dealloc-visitor.h" /* qapi_dealloc_visitor_new() */
 
 static QemuOptsList userdef_opts = {
     .name = "userdef",
@@ -37,15 +37,15 @@
 {
     const char *opts_string = test_data;
     QemuOpts *opts;
-    OptsVisitor *ov;
+    Visitor *v;
 
-    opts = qemu_opts_parse(qemu_find_opts("userdef"), opts_string, 0);
+    opts = qemu_opts_parse(qemu_find_opts("userdef"), opts_string, false,
+                           NULL);
     g_assert(opts != NULL);
 
-    ov = opts_visitor_new(opts);
-    visit_type_UserDefOptions(opts_get_visitor(ov), &f->userdef, NULL,
-                              &f->err);
-    opts_visitor_cleanup(ov);
+    v = opts_visitor_new(opts);
+    visit_type_UserDefOptions(v, NULL, &f->userdef, &f->err);
+    visit_free(v);
     qemu_opts_del(opts);
 }
 
@@ -53,14 +53,7 @@
 static void
 teardown_fixture(OptsVisitorFixture *f, gconstpointer test_data)
 {
-    if (f->userdef != NULL) {
-        QapiDeallocVisitor *dv;
-
-        dv = qapi_dealloc_visitor_new();
-        visit_type_UserDefOptions(qapi_dealloc_get_visitor(dv), &f->userdef,
-                                  NULL, NULL);
-        qapi_dealloc_visitor_cleanup(dv);
-    }
+    qapi_free_UserDefOptions(f->userdef);
     error_free(f->err);
 }
 
diff --git a/tests/test-qdev-global-props.c b/tests/test-qdev-global-props.c
index 0be9835..48e5b73 100644
--- a/tests/test-qdev-global-props.c
+++ b/tests/test-qdev-global-props.c
@@ -22,8 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include <glib.h>
-#include <stdint.h>
+#include "qemu/osdep.h"
 
 #include "hw/qdev.h"
 #include "qom/object.h"
@@ -116,26 +115,20 @@
 #define TYPE_UNUSED_HOTPLUG   "hotplug-type"
 #define TYPE_UNUSED_NOHOTPLUG "nohotplug-type"
 
-static void prop1_accessor(Object *obj,
-                           Visitor *v,
-                           void *opaque,
-                           const char *name,
-                           Error **errp)
+static void prop1_accessor(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     MyType *mt = DYNAMIC_TYPE(obj);
 
-    visit_type_uint32(v, &mt->prop1, name, errp);
+    visit_type_uint32(v, name, &mt->prop1, errp);
 }
 
-static void prop2_accessor(Object *obj,
-                           Visitor *v,
-                           void *opaque,
-                           const char *name,
-                           Error **errp)
+static void prop2_accessor(Object *obj, Visitor *v, const char *name,
+                           void *opaque, Error **errp)
 {
     MyType *mt = DYNAMIC_TYPE(obj);
 
-    visit_type_uint32(v, &mt->prop2, name, errp);
+    visit_type_uint32(v, name, &mt->prop2, errp);
 }
 
 static void dynamic_instance_init(Object *obj)
diff --git a/tests/test-qdist.c b/tests/test-qdist.c
new file mode 100644
index 0000000..9541ce3
--- /dev/null
+++ b/tests/test-qdist.c
@@ -0,0 +1,389 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/qdist.h"
+
+#include <math.h>
+
+struct entry_desc {
+    double x;
+    unsigned long count;
+
+    /* 0 prints a space, 1-8 prints from qdist_blocks[] */
+    int fill_code;
+};
+
+/* See: https://en.wikipedia.org/wiki/Block_Elements */
+static const gunichar qdist_blocks[] = {
+    0x2581,
+    0x2582,
+    0x2583,
+    0x2584,
+    0x2585,
+    0x2586,
+    0x2587,
+    0x2588
+};
+
+#define QDIST_NR_BLOCK_CODES ARRAY_SIZE(qdist_blocks)
+
+static char *pr_hist(const struct entry_desc *darr, size_t n)
+{
+    GString *s = g_string_new("");
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        int fill = darr[i].fill_code;
+
+        if (fill) {
+            assert(fill <= QDIST_NR_BLOCK_CODES);
+            g_string_append_unichar(s, qdist_blocks[fill - 1]);
+        } else {
+            g_string_append_c(s, ' ');
+        }
+    }
+    return g_string_free(s, FALSE);
+}
+
+static void
+histogram_check(const struct qdist *dist, const struct entry_desc *darr,
+                size_t n, size_t n_bins)
+{
+    char *pr = qdist_pr_plain(dist, n_bins);
+    char *str = pr_hist(darr, n);
+
+    g_assert_cmpstr(pr, ==, str);
+    g_free(pr);
+    g_free(str);
+}
+
+static void histogram_check_single_full(const struct qdist *dist, size_t n_bins)
+{
+    struct entry_desc desc = { .fill_code = 8 };
+
+    histogram_check(dist, &desc, 1, n_bins);
+}
+
+static void
+entries_check(const struct qdist *dist, const struct entry_desc *darr, size_t n)
+{
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        struct qdist_entry *e = &dist->entries[i];
+
+        g_assert_cmpuint(e->count, ==, darr[i].count);
+    }
+}
+
+static void
+entries_insert(struct qdist *dist, const struct entry_desc *darr, size_t n)
+{
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        qdist_add(dist, darr[i].x, darr[i].count);
+    }
+}
+
+static void do_test_bin(const struct entry_desc *a, size_t n_a,
+                        const struct entry_desc *b, size_t n_b)
+{
+    struct qdist qda;
+    struct qdist qdb;
+
+    qdist_init(&qda);
+
+    entries_insert(&qda, a, n_a);
+    qdist_inc(&qda, a[0].x);
+    qdist_add(&qda, a[0].x, -1);
+
+    g_assert_cmpuint(qdist_unique_entries(&qda), ==, n_a);
+    g_assert_cmpfloat(qdist_xmin(&qda), ==, a[0].x);
+    g_assert_cmpfloat(qdist_xmax(&qda), ==, a[n_a - 1].x);
+    histogram_check(&qda, a, n_a, 0);
+    histogram_check(&qda, a, n_a, n_a);
+
+    qdist_bin__internal(&qdb, &qda, n_b);
+    g_assert_cmpuint(qdb.n, ==, n_b);
+    entries_check(&qdb, b, n_b);
+    g_assert_cmpuint(qdist_sample_count(&qda), ==, qdist_sample_count(&qdb));
+    /*
+     * No histogram_check() for $qdb, since we'd rebin it and that is a bug.
+     * Instead, regenerate it from $qda.
+     */
+    histogram_check(&qda, b, n_b, n_b);
+
+    qdist_destroy(&qdb);
+    qdist_destroy(&qda);
+}
+
+static void do_test_pr(uint32_t opt)
+{
+    static const struct entry_desc desc[] = {
+        [0] = { 1, 900, 8 },
+        [1] = { 2, 1, 1 },
+        [2] = { 3, 2, 1 }
+    };
+    static const char border[] = "|";
+    const char *llabel = NULL;
+    const char *rlabel = NULL;
+    struct qdist dist;
+    GString *s;
+    char *str;
+    char *pr;
+    size_t n;
+
+    n = ARRAY_SIZE(desc);
+    qdist_init(&dist);
+
+    entries_insert(&dist, desc, n);
+    histogram_check(&dist, desc, n, 0);
+
+    s = g_string_new("");
+
+    if (opt & QDIST_PR_LABELS) {
+        unsigned int lopts = opt & (QDIST_PR_NODECIMAL |
+                                    QDIST_PR_PERCENT |
+                                    QDIST_PR_100X |
+                                    QDIST_PR_NOBINRANGE);
+
+        if (lopts == 0) {
+            llabel = "[1.0,1.7)";
+            rlabel = "[2.3,3.0]";
+        } else if (lopts == QDIST_PR_NODECIMAL) {
+            llabel = "[1,2)";
+            rlabel = "[2,3]";
+        } else if (lopts == (QDIST_PR_PERCENT | QDIST_PR_NODECIMAL)) {
+            llabel = "[1,2)%";
+            rlabel = "[2,3]%";
+        } else if (lopts == QDIST_PR_100X) {
+            llabel = "[100.0,166.7)";
+            rlabel = "[233.3,300.0]";
+        } else if (lopts == (QDIST_PR_NOBINRANGE | QDIST_PR_NODECIMAL)) {
+            llabel = "1";
+            rlabel = "3";
+        } else {
+            g_assert_cmpstr("BUG", ==, "This is not meant to be exhaustive");
+        }
+    }
+
+    if (llabel) {
+        g_string_append(s, llabel);
+    }
+    if (opt & QDIST_PR_BORDER) {
+        g_string_append(s, border);
+    }
+
+    str = pr_hist(desc, n);
+    g_string_append(s, str);
+    g_free(str);
+
+    if (opt & QDIST_PR_BORDER) {
+        g_string_append(s, border);
+    }
+    if (rlabel) {
+        g_string_append(s, rlabel);
+    }
+
+    str = g_string_free(s, FALSE);
+    pr = qdist_pr(&dist, n, opt);
+    g_assert_cmpstr(pr, ==, str);
+    g_free(pr);
+    g_free(str);
+
+    qdist_destroy(&dist);
+}
+
+static inline void do_test_pr_label(uint32_t opt)
+{
+    opt |= QDIST_PR_LABELS;
+    do_test_pr(opt);
+}
+
+static void test_pr(void)
+{
+    do_test_pr(0);
+
+    do_test_pr(QDIST_PR_BORDER);
+
+    /* 100X should be ignored because we're not setting LABELS */
+    do_test_pr(QDIST_PR_100X);
+
+    do_test_pr_label(0);
+    do_test_pr_label(QDIST_PR_NODECIMAL);
+    do_test_pr_label(QDIST_PR_PERCENT | QDIST_PR_NODECIMAL);
+    do_test_pr_label(QDIST_PR_100X);
+    do_test_pr_label(QDIST_PR_NOBINRANGE | QDIST_PR_NODECIMAL);
+}
+
+static void test_bin_shrink(void)
+{
+    static const struct entry_desc a[] = {
+        [0] = { 0.0,   42922, 7 },
+        [1] = { 0.25,  47834, 8 },
+        [2] = { 0.50,  26628, 0 },
+        [3] = { 0.625, 597,   4 },
+        [4] = { 0.75,  10298, 1 },
+        [5] = { 0.875, 22,    2 },
+        [6] = { 1.0,   2771,  1 }
+    };
+    static const struct entry_desc b[] = {
+        [0] = { 0.0, 42922, 7 },
+        [1] = { 0.25, 47834, 8 },
+        [2] = { 0.50, 27225, 3 },
+        [3] = { 0.75, 13091, 1 }
+    };
+
+    return do_test_bin(a, ARRAY_SIZE(a), b, ARRAY_SIZE(b));
+}
+
+static void test_bin_expand(void)
+{
+    static const struct entry_desc a[] = {
+        [0] = { 0.0,   11713, 5 },
+        [1] = { 0.25,  20294, 0 },
+        [2] = { 0.50,  17266, 8 },
+        [3] = { 0.625, 1506,  0 },
+        [4] = { 0.75,  10355, 6 },
+        [5] = { 0.833, 2,     1 },
+        [6] = { 0.875, 99,    4 },
+        [7] = { 1.0,   4301,  2 }
+    };
+    static const struct entry_desc b[] = {
+        [0] = { 0.0, 11713, 5 },
+        [1] = { 0.0, 0,     0 },
+        [2] = { 0.0, 20294, 8 },
+        [3] = { 0.0, 0,     0 },
+        [4] = { 0.0, 0,     0 },
+        [5] = { 0.0, 17266, 6 },
+        [6] = { 0.0, 1506,  1 },
+        [7] = { 0.0, 10355, 4 },
+        [8] = { 0.0, 101,   1 },
+        [9] = { 0.0, 4301,  2 }
+    };
+
+    return do_test_bin(a, ARRAY_SIZE(a), b, ARRAY_SIZE(b));
+}
+
+static void test_bin_precision(void)
+{
+    static const struct entry_desc a[] = {
+        [0] = { 0, 213549, 8 },
+        [1] = { 1, 70, 1 },
+    };
+    static const struct entry_desc b[] = {
+        [0] = { 0, 213549, 8 },
+        [1] = { 0, 70, 1 },
+    };
+
+    return do_test_bin(a, ARRAY_SIZE(a), b, ARRAY_SIZE(b));
+}
+
+static void test_bin_simple(void)
+{
+    static const struct entry_desc a[] = {
+        [0] = { 10, 101, 8 },
+        [1] = { 11, 0, 0 },
+        [2] = { 12, 2, 1 }
+    };
+    static const struct entry_desc b[] = {
+        [0] = { 0, 101, 8 },
+        [1] = { 0, 0, 0 },
+        [2] = { 0, 0, 0 },
+        [3] = { 0, 0, 0 },
+        [4] = { 0, 2, 1 }
+    };
+
+    return do_test_bin(a, ARRAY_SIZE(a), b, ARRAY_SIZE(b));
+}
+
+static void test_single_full(void)
+{
+    struct qdist dist;
+
+    qdist_init(&dist);
+
+    qdist_add(&dist, 3, 102);
+    g_assert_cmpfloat(qdist_avg(&dist), ==, 3);
+    g_assert_cmpfloat(qdist_xmin(&dist), ==, 3);
+    g_assert_cmpfloat(qdist_xmax(&dist), ==, 3);
+
+    histogram_check_single_full(&dist, 0);
+    histogram_check_single_full(&dist, 1);
+    histogram_check_single_full(&dist, 10);
+
+    qdist_destroy(&dist);
+}
+
+static void test_single_empty(void)
+{
+    struct qdist dist;
+    char *pr;
+
+    qdist_init(&dist);
+
+    qdist_add(&dist, 3, 0);
+    g_assert_cmpuint(qdist_sample_count(&dist), ==, 0);
+    g_assert(isnan(qdist_avg(&dist)));
+    g_assert_cmpfloat(qdist_xmin(&dist), ==, 3);
+    g_assert_cmpfloat(qdist_xmax(&dist), ==, 3);
+
+    pr = qdist_pr_plain(&dist, 0);
+    g_assert_cmpstr(pr, ==, " ");
+    g_free(pr);
+
+    pr = qdist_pr_plain(&dist, 1);
+    g_assert_cmpstr(pr, ==, " ");
+    g_free(pr);
+
+    pr = qdist_pr_plain(&dist, 2);
+    g_assert_cmpstr(pr, ==, " ");
+    g_free(pr);
+
+    qdist_destroy(&dist);
+}
+
+static void test_none(void)
+{
+    struct qdist dist;
+    char *pr;
+
+    qdist_init(&dist);
+
+    g_assert(isnan(qdist_avg(&dist)));
+    g_assert(isnan(qdist_xmin(&dist)));
+    g_assert(isnan(qdist_xmax(&dist)));
+
+    pr = qdist_pr_plain(&dist, 0);
+    g_assert_cmpstr(pr, ==, "(empty)");
+    g_free(pr);
+
+    pr = qdist_pr_plain(&dist, 2);
+    g_assert_cmpstr(pr, ==, "(empty)");
+    g_free(pr);
+
+    pr = qdist_pr(&dist, 0, QDIST_PR_BORDER);
+    g_assert_cmpstr(pr, ==, "(empty)");
+    g_free(pr);
+
+    qdist_destroy(&dist);
+}
+
+int main(int argc, char *argv[])
+{
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/qdist/none", test_none);
+    g_test_add_func("/qdist/single/empty", test_single_empty);
+    g_test_add_func("/qdist/single/full", test_single_full);
+    g_test_add_func("/qdist/binning/simple", test_bin_simple);
+    g_test_add_func("/qdist/binning/precision", test_bin_precision);
+    g_test_add_func("/qdist/binning/expand", test_bin_expand);
+    g_test_add_func("/qdist/binning/shrink", test_bin_shrink);
+    g_test_add_func("/qdist/pr", test_pr);
+    return g_test_run();
+}
diff --git a/tests/test-qemu-opts.c b/tests/test-qemu-opts.c
index ca08ac5..a505a3e 100644
--- a/tests/test-qemu-opts.c
+++ b/tests/test-qemu-opts.c
@@ -7,12 +7,11 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qapi/error.h"
 #include "qapi/qmp/qstring.h"
 #include "qemu/config-file.h"
 
-#include <glib.h>
-#include <string.h>
 
 static QemuOptsList opts_list_01 = {
     .name = "opts_list_01",
@@ -148,13 +147,13 @@
     opt = qemu_opt_get(opts, "str2");
     g_assert(opt == NULL);
 
-    qemu_opt_set(opts, "str2", "value");
+    qemu_opt_set(opts, "str2", "value", &error_abort);
 
     /* now we have set str2, should know about it */
     opt = qemu_opt_get(opts, "str2");
     g_assert_cmpstr(opt, ==, "value");
 
-    qemu_opt_set(opts, "str2", "value2");
+    qemu_opt_set(opts, "str2", "value2", &error_abort);
 
     /* having reset the value, the returned should be the reset one */
     opt = qemu_opt_get(opts, "str2");
@@ -169,10 +168,10 @@
 
 static void test_qemu_opt_get_bool(void)
 {
+    Error *err = NULL;
     QemuOptsList *list;
     QemuOpts *opts;
     bool opt;
-    int ret;
 
     list = qemu_find_opts("opts_list_02");
     g_assert(list != NULL);
@@ -192,16 +191,16 @@
     opt = qemu_opt_get_bool(opts, "bool1", false);
     g_assert(opt == false);
 
-    ret = qemu_opt_set_bool(opts, "bool1", true);
-    g_assert(ret == 0);
+    qemu_opt_set_bool(opts, "bool1", true, &err);
+    g_assert(!err);
 
     /* now we have set bool1, should know about it */
     opt = qemu_opt_get_bool(opts, "bool1", false);
     g_assert(opt == true);
 
     /* having reset the value, opt should be the reset one not defval */
-    ret = qemu_opt_set_bool(opts, "bool1", false);
-    g_assert(ret == 0);
+    qemu_opt_set_bool(opts, "bool1", false, &err);
+    g_assert(!err);
 
     opt = qemu_opt_get_bool(opts, "bool1", true);
     g_assert(opt == false);
@@ -215,10 +214,10 @@
 
 static void test_qemu_opt_get_number(void)
 {
+    Error *err = NULL;
     QemuOptsList *list;
     QemuOpts *opts;
     uint64_t opt;
-    int ret;
 
     list = qemu_find_opts("opts_list_01");
     g_assert(list != NULL);
@@ -238,16 +237,16 @@
     opt = qemu_opt_get_number(opts, "number1", 5);
     g_assert(opt == 5);
 
-    ret = qemu_opt_set_number(opts, "number1", 10);
-    g_assert(ret == 0);
+    qemu_opt_set_number(opts, "number1", 10, &err);
+    g_assert(!err);
 
     /* now we have set number1, should know about it */
     opt = qemu_opt_get_number(opts, "number1", 5);
     g_assert(opt == 10);
 
     /* having reset it, the returned should be the reset one not defval */
-    ret = qemu_opt_set_number(opts, "number1", 15);
-    g_assert(ret == 0);
+    qemu_opt_set_number(opts, "number1", 15, &err);
+    g_assert(!err);
 
     opt = qemu_opt_get_number(opts, "number1", 5);
     g_assert(opt == 15);
@@ -323,7 +322,7 @@
     int ret;
 
     /* dynamically initialized (parsed) opts */
-    opts = qemu_opts_parse(&opts_list_03, "key=value", 0);
+    opts = qemu_opts_parse(&opts_list_03, "key=value", false, NULL);
     g_assert(opts != NULL);
 
     /* check default/parsed value */
@@ -331,7 +330,7 @@
     g_assert_cmpstr(value, ==, "value");
 
     /* reset it to value2 */
-    qemu_opt_set(opts, "key", "value2");
+    qemu_opt_set(opts, "key", "value2", &error_abort);
 
     value = qemu_opt_get(opts, "key");
     g_assert_cmpstr(value, ==, "value2");
@@ -349,10 +348,10 @@
 
 static void test_qemu_opts_reset(void)
 {
+    Error *err = NULL;
     QemuOptsList *list;
     QemuOpts *opts;
     uint64_t opt;
-    int ret;
 
     list = qemu_find_opts("opts_list_01");
     g_assert(list != NULL);
@@ -372,8 +371,8 @@
     opt = qemu_opt_get_number(opts, "number1", 5);
     g_assert(opt == 5);
 
-    ret = qemu_opt_set_number(opts, "number1", 10);
-    g_assert(ret == 0);
+    qemu_opt_set_number(opts, "number1", 10, &err);
+    g_assert(!err);
 
     /* now we have set number1, should know about it */
     opt = qemu_opt_get_number(opts, "number1", 5);
@@ -388,9 +387,9 @@
 
 static void test_qemu_opts_set(void)
 {
+    Error *err = NULL;
     QemuOptsList *list;
     QemuOpts *opts;
-    int ret;
     const char *opt;
 
     list = qemu_find_opts("opts_list_01");
@@ -403,8 +402,8 @@
     g_assert(opts == NULL);
 
     /* implicitly create opts and set str3 value */
-    ret = qemu_opts_set(list, NULL, "str3", "value");
-    g_assert(ret == 0);
+    qemu_opts_set(list, NULL, "str3", "value", &err);
+    g_assert(!err);
     g_assert(!QTAILQ_EMPTY(&list->head));
 
     /* get the just created opts */
diff --git a/tests/test-qga.c b/tests/test-qga.c
new file mode 100644
index 0000000..dac8fb8
--- /dev/null
+++ b/tests/test-qga.c
@@ -0,0 +1,930 @@
+#include "qemu/osdep.h"
+#include <locale.h>
+#include <glib/gstdio.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include "libqtest.h"
+
+typedef struct {
+    char *test_dir;
+    GMainLoop *loop;
+    int fd;
+    GPid pid;
+} TestFixture;
+
+static int connect_qga(char *path)
+{
+    int s, ret, len, i = 0;
+    struct sockaddr_un remote;
+
+    s = socket(AF_UNIX, SOCK_STREAM, 0);
+    g_assert(s != -1);
+
+    remote.sun_family = AF_UNIX;
+    do {
+        strcpy(remote.sun_path, path);
+        len = strlen(remote.sun_path) + sizeof(remote.sun_family);
+        ret = connect(s, (struct sockaddr *)&remote, len);
+        if (ret == -1) {
+            g_usleep(G_USEC_PER_SEC);
+        }
+        if (i++ == 10) {
+            return -1;
+        }
+    } while (ret == -1);
+
+    return s;
+}
+
+static void qga_watch(GPid pid, gint status, gpointer user_data)
+{
+    TestFixture *fixture = user_data;
+
+    g_assert_cmpint(status, ==, 0);
+    g_main_loop_quit(fixture->loop);
+}
+
+static void
+fixture_setup(TestFixture *fixture, gconstpointer data)
+{
+    const gchar *extra_arg = data;
+    GError *error = NULL;
+    gchar *cwd, *path, *cmd, **argv = NULL;
+
+    fixture->loop = g_main_loop_new(NULL, FALSE);
+
+    fixture->test_dir = g_strdup("/tmp/qgatest.XXXXXX");
+    g_assert_nonnull(mkdtemp(fixture->test_dir));
+
+    path = g_build_filename(fixture->test_dir, "sock", NULL);
+    cwd = g_get_current_dir();
+    cmd = g_strdup_printf("%s%cqemu-ga -m unix-listen -t %s -p %s %s %s",
+                          cwd, G_DIR_SEPARATOR,
+                          fixture->test_dir, path,
+                          getenv("QTEST_LOG") ? "-v" : "",
+                          extra_arg ?: "");
+    g_shell_parse_argv(cmd, NULL, &argv, &error);
+    g_assert_no_error(error);
+
+    g_spawn_async(fixture->test_dir, argv, NULL,
+                  G_SPAWN_SEARCH_PATH|G_SPAWN_DO_NOT_REAP_CHILD,
+                  NULL, NULL, &fixture->pid, &error);
+    g_assert_no_error(error);
+
+    g_child_watch_add(fixture->pid, qga_watch, fixture);
+
+    fixture->fd = connect_qga(path);
+    g_assert_cmpint(fixture->fd, !=, -1);
+
+    g_strfreev(argv);
+    g_free(cmd);
+    g_free(cwd);
+    g_free(path);
+}
+
+static void
+fixture_tear_down(TestFixture *fixture, gconstpointer data)
+{
+    gchar *tmp;
+
+    kill(fixture->pid, SIGTERM);
+
+    g_main_loop_run(fixture->loop);
+    g_main_loop_unref(fixture->loop);
+
+    g_spawn_close_pid(fixture->pid);
+
+    tmp = g_build_filename(fixture->test_dir, "foo", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    tmp = g_build_filename(fixture->test_dir, "qga.state", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    tmp = g_build_filename(fixture->test_dir, "sock", NULL);
+    g_unlink(tmp);
+    g_free(tmp);
+
+    g_rmdir(fixture->test_dir);
+    g_free(fixture->test_dir);
+}
+
+static void qmp_assertion_message_error(const char     *domain,
+                                        const char     *file,
+                                        int             line,
+                                        const char     *func,
+                                        const char     *expr,
+                                        QDict          *dict)
+{
+    const char *class, *desc;
+    char *s;
+    QDict *error;
+
+    error = qdict_get_qdict(dict, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+
+    s = g_strdup_printf("assertion failed %s: %s %s", expr, class, desc);
+    g_assertion_message(domain, file, line, func, s);
+    g_free(s);
+}
+
+#define qmp_assert_no_error(err) do {                                   \
+    if (qdict_haskey(err, "error")) {                                   \
+        qmp_assertion_message_error(G_LOG_DOMAIN, __FILE__, __LINE__,   \
+                                    G_STRFUNC, #err, err);              \
+    }                                                                   \
+} while (0)
+
+static void test_qga_sync_delimited(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    guint32 v, r = g_random_int();
+    unsigned char c;
+    QDict *ret;
+    gchar *cmd;
+
+    cmd = g_strdup_printf("%c{'execute': 'guest-sync-delimited',"
+                          " 'arguments': {'id': %u } }", 0xff, r);
+    qmp_fd_send(fixture->fd, cmd);
+    g_free(cmd);
+
+    v = read(fixture->fd, &c, 1);
+    g_assert_cmpint(v, ==, 1);
+    g_assert_cmpint(c, ==, 0xff);
+
+    ret = qmp_fd_receive(fixture->fd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    v = qdict_get_int(ret, "return");
+    g_assert_cmpint(r, ==, v);
+
+    QDECREF(ret);
+}
+
+static void test_qga_sync(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    guint32 v, r = g_random_int();
+    QDict *ret;
+    gchar *cmd;
+
+    cmd = g_strdup_printf("%c{'execute': 'guest-sync',"
+                          " 'arguments': {'id': %u } }", 0xff, r);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_free(cmd);
+
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    v = qdict_get_int(ret, "return");
+    g_assert_cmpint(r, ==, v);
+
+    QDECREF(ret);
+}
+
+static void test_qga_ping(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-ping'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    QDECREF(ret);
+}
+
+static void test_qga_invalid_cmd(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-invalid-cmd'}");
+    g_assert_nonnull(ret);
+
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+
+    g_assert_cmpstr(class, ==, "CommandNotFound");
+    g_assert_cmpint(strlen(desc), >, 0);
+
+    QDECREF(ret);
+}
+
+static void test_qga_info(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    const gchar *version;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-info'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    version = qdict_get_try_str(val, "version");
+    g_assert_cmpstr(version, ==, QEMU_VERSION);
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_vcpus(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-vcpus'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* check there is at least a cpu */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "online"));
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "logical-id"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_fsinfo(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-fsinfo'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* sanity-check the response if there are any filesystems */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    if (entry) {
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "name"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "mountpoint"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "type"));
+        g_assert(qdict_haskey(qobject_to_qdict(entry->value), "disk"));
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_memory_block_info(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    int64_t size;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-memory-block-info'}");
+    g_assert_nonnull(ret);
+
+    /* some systems might not expose memory block info in sysfs */
+    if (!qdict_haskey(ret, "error")) {
+        /* check there is at least some memory */
+        val = qdict_get_qdict(ret, "return");
+        size = qdict_get_int(val, "size");
+        g_assert_cmpint(size, >, 0);
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_get_memory_blocks(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-memory-blocks'}");
+    g_assert_nonnull(ret);
+
+    /* some systems might not expose memory block info in sysfs */
+    if (!qdict_haskey(ret, "error")) {
+        list = qdict_get_qlist(ret, "return");
+        entry = qlist_first(list);
+        /* newer versions of qga may return empty list without error */
+        if (entry) {
+            g_assert(qdict_haskey(qobject_to_qdict(entry->value), "phys-index"));
+            g_assert(qdict_haskey(qobject_to_qdict(entry->value), "online"));
+        }
+    }
+
+    QDECREF(ret);
+}
+
+static void test_qga_network_get_interfaces(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-network-get-interfaces'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    /* check there is at least an interface */
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "name"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_file_ops(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    const unsigned char helloworld[] = "Hello World!\n";
+    const char *b64;
+    gchar *cmd, *path, *enc;
+    unsigned char *dec;
+    QDict *ret, *val;
+    int64_t id, eof;
+    gsize count;
+    FILE *f;
+    char tmp[100];
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'w+' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    enc = g_base64_encode(helloworld, sizeof(helloworld));
+    /* write */
+    cmd = g_strdup_printf("{'execute': 'guest-file-write',"
+                          " 'arguments': { 'handle': %" PRId64 ","
+                          " 'buf-b64': '%s' } }", id, enc);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert_cmpint(eof, ==, 0);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* flush */
+    cmd = g_strdup_printf("{'execute': 'guest-file-flush',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* check content */
+    path = g_build_filename(fixture->test_dir, "foo", NULL);
+    f = fopen(path, "r");
+    g_assert_nonnull(f);
+    count = fread(tmp, 1, sizeof(tmp), f);
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    tmp[count] = 0;
+    g_assert_cmpstr(tmp, ==, (char *)helloworld);
+    fclose(f);
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'r' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    /* read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, enc);
+
+    QDECREF(ret);
+    g_free(cmd);
+    g_free(enc);
+
+    /* read eof */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, "");
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* seek */
+    cmd = g_strdup_printf("{'execute': 'guest-file-seek',"
+                          " 'arguments': { 'handle': %" PRId64 ", "
+                          " 'offset': %d, 'whence': '%s' } }",
+                          id, 6, "set");
+    ret = qmp_fd(fixture->fd, cmd);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "position");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, 6);
+    g_assert(!eof);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* partial read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld) - 6);
+    g_assert(eof);
+    dec = g_base64_decode(b64, &count);
+    g_assert_cmpint(count, ==, sizeof(helloworld) - 6);
+    g_assert_cmpmem(dec, count, helloworld + 6, sizeof(helloworld) - 6);
+    g_free(dec);
+
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+}
+
+static void test_qga_file_write_read(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    const unsigned char helloworld[] = "Hello World!\n";
+    const char *b64;
+    gchar *cmd, *enc;
+    QDict *ret, *val;
+    int64_t id, eof;
+    gsize count;
+
+    /* open */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-file-open',"
+                 " 'arguments': { 'path': 'foo', 'mode': 'w+' } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    id = qdict_get_int(ret, "return");
+    QDECREF(ret);
+
+    enc = g_base64_encode(helloworld, sizeof(helloworld));
+    /* write */
+    cmd = g_strdup_printf("{'execute': 'guest-file-write',"
+                          " 'arguments': { 'handle': %" PRId64 ","
+                          " 'buf-b64': '%s' } }", id, enc);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert_cmpint(eof, ==, 0);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* read (check implicit flush) */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, "");
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* seek to 0 */
+    cmd = g_strdup_printf("{'execute': 'guest-file-seek',"
+                          " 'arguments': { 'handle': %" PRId64 ", "
+                          " 'offset': %d, 'whence': '%s' } }",
+                          id, 0, "set");
+    ret = qmp_fd(fixture->fd, cmd);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "position");
+    eof = qdict_get_bool(val, "eof");
+    g_assert_cmpint(count, ==, 0);
+    g_assert(!eof);
+    QDECREF(ret);
+    g_free(cmd);
+
+    /* read */
+    cmd = g_strdup_printf("{'execute': 'guest-file-read',"
+                          " 'arguments': { 'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    val = qdict_get_qdict(ret, "return");
+    count = qdict_get_int(val, "count");
+    eof = qdict_get_bool(val, "eof");
+    b64 = qdict_get_str(val, "buf-b64");
+    g_assert_cmpint(count, ==, sizeof(helloworld));
+    g_assert(eof);
+    g_assert_cmpstr(b64, ==, enc);
+    QDECREF(ret);
+    g_free(cmd);
+    g_free(enc);
+
+    /* close */
+    cmd = g_strdup_printf("{'execute': 'guest-file-close',"
+                          " 'arguments': {'handle': %" PRId64 "} }",
+                          id);
+    ret = qmp_fd(fixture->fd, cmd);
+    QDECREF(ret);
+    g_free(cmd);
+}
+
+static void test_qga_get_time(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    int64_t time;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    time = qdict_get_int(ret, "return");
+    g_assert_cmpint(time, >, 0);
+
+    QDECREF(ret);
+}
+
+static void test_qga_set_time(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    int64_t current, time;
+    gchar *cmd;
+
+    /* get current time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    current = qdict_get_int(ret, "return");
+    g_assert_cmpint(current, >, 0);
+    QDECREF(ret);
+
+    /* set some old time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-set-time',"
+                 " 'arguments': { 'time': 1000 } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    /* check old time */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    time = qdict_get_int(ret, "return");
+    g_assert_cmpint(time / 1000, <, G_USEC_PER_SEC * 10);
+    QDECREF(ret);
+
+    /* set back current time */
+    cmd = g_strdup_printf("{'execute': 'guest-set-time',"
+                          " 'arguments': { 'time': %" PRId64 " } }",
+                          current + time * 1000);
+    ret = qmp_fd(fixture->fd, cmd);
+    g_free(cmd);
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+}
+
+static void test_qga_fstrim(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    QList *list;
+    const QListEntry *entry;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fstrim',"
+                 " arguments: { minimum: 4194304 } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    list = qdict_get_qlist(ret, "return");
+    entry = qlist_first(list);
+    g_assert(qdict_haskey(qobject_to_qdict(entry->value), "paths"));
+
+    QDECREF(ret);
+}
+
+static void test_qga_blacklist(gconstpointer data)
+{
+    TestFixture fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    fixture_setup(&fix, "-b guest-ping,guest-get-time");
+
+    /* check blacklist */
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-ping'}");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_nonnull(g_strstr_len(desc, -1, "has been disabled"));
+    QDECREF(ret);
+
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-get-time'}");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    class = qdict_get_try_str(error, "class");
+    desc = qdict_get_try_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_nonnull(g_strstr_len(desc, -1, "has been disabled"));
+    QDECREF(ret);
+
+    /* check something work */
+    ret = qmp_fd(fix.fd, "{'execute': 'guest-get-fsinfo'}");
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    fixture_tear_down(&fix, NULL);
+}
+
+static void test_qga_config(gconstpointer data)
+{
+    GError *error = NULL;
+    char *cwd, *cmd, *out, *err, *str, **strv, **argv = NULL;
+    char *env[2];
+    int status;
+    gsize n;
+    GKeyFile *kf;
+
+    cwd = g_get_current_dir();
+    cmd = g_strdup_printf("%s%cqemu-ga -D",
+                          cwd, G_DIR_SEPARATOR);
+    g_shell_parse_argv(cmd, NULL, &argv, &error);
+    g_assert_no_error(error);
+
+    env[0] = g_strdup_printf("QGA_CONF=tests%cdata%ctest-qga-config",
+                             G_DIR_SEPARATOR, G_DIR_SEPARATOR);
+    env[1] = NULL;
+    g_spawn_sync(NULL, argv, env, 0,
+                 NULL, NULL, &out, &err, &status, &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(err, ==, "");
+    g_assert_cmpint(status, ==, 0);
+
+    kf = g_key_file_new();
+    g_key_file_load_from_data(kf, out, -1, G_KEY_FILE_NONE, &error);
+    g_assert_no_error(error);
+
+    str = g_key_file_get_start_group(kf);
+    g_assert_cmpstr(str, ==, "general");
+    g_free(str);
+
+    g_assert_false(g_key_file_get_boolean(kf, "general", "daemon", &error));
+    g_assert_no_error(error);
+
+    str = g_key_file_get_string(kf, "general", "method", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "virtio-serial");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "path", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/path/to/org.qemu.guest_agent.0");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "pidfile", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/var/foo/qemu-ga.pid");
+    g_free(str);
+
+    str = g_key_file_get_string(kf, "general", "statedir", &error);
+    g_assert_no_error(error);
+    g_assert_cmpstr(str, ==, "/var/state");
+    g_free(str);
+
+    g_assert_true(g_key_file_get_boolean(kf, "general", "verbose", &error));
+    g_assert_no_error(error);
+
+    strv = g_key_file_get_string_list(kf, "general", "blacklist", &n, &error);
+    g_assert_cmpint(n, ==, 2);
+#if GLIB_CHECK_VERSION(2, 44, 0)
+    g_assert_true(g_strv_contains((const char * const *)strv,
+                                  "guest-ping"));
+    g_assert_true(g_strv_contains((const char * const *)strv,
+                                  "guest-get-time"));
+#endif
+    g_assert_no_error(error);
+    g_strfreev(strv);
+
+    g_free(out);
+    g_free(err);
+    g_free(env[0]);
+    g_key_file_free(kf);
+}
+
+static void test_qga_fsfreeze_status(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    const gchar *status;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-status'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+
+    status = qdict_get_try_str(ret, "return");
+    g_assert_cmpstr(status, ==, "thawed");
+
+    QDECREF(ret);
+}
+
+static void test_qga_fsfreeze_and_thaw(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret;
+    const gchar *status;
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-freeze'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-status'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    status = qdict_get_try_str(ret, "return");
+    g_assert_cmpstr(status, ==, "frozen");
+    QDECREF(ret);
+
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-fsfreeze-thaw'}");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    QDECREF(ret);
+}
+
+static void test_qga_guest_exec(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *val;
+    const gchar *out;
+    guchar *decoded;
+    int64_t pid, now, exitcode;
+    gsize len;
+    bool exited;
+
+    /* exec 'echo foo bar' */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-exec', 'arguments': {"
+                 " 'path': '/bin/echo', 'arg': [ '-n', '\" test_str \"' ],"
+                 " 'capture-output': true } }");
+    g_assert_nonnull(ret);
+    qmp_assert_no_error(ret);
+    val = qdict_get_qdict(ret, "return");
+    pid = qdict_get_int(val, "pid");
+    g_assert_cmpint(pid, >, 0);
+    QDECREF(ret);
+
+    /* wait for completion */
+    now = g_get_monotonic_time();
+    do {
+        ret = qmp_fd(fixture->fd, "{'execute': 'guest-exec-status',"
+                     " 'arguments': { 'pid': %" PRId64 "  } }", pid);
+        g_assert_nonnull(ret);
+        val = qdict_get_qdict(ret, "return");
+        exited = qdict_get_bool(val, "exited");
+        if (!exited) {
+            QDECREF(ret);
+        }
+    } while (!exited &&
+             g_get_monotonic_time() < now + 5 * G_TIME_SPAN_SECOND);
+    g_assert(exited);
+
+    /* check stdout */
+    exitcode = qdict_get_int(val, "exitcode");
+    g_assert_cmpint(exitcode, ==, 0);
+    out = qdict_get_str(val, "out-data");
+    decoded = g_base64_decode(out, &len);
+    g_assert_cmpint(len, ==, 12);
+    g_assert_cmpstr((char *)decoded, ==, "\" test_str \"");
+    g_free(decoded);
+    QDECREF(ret);
+}
+
+static void test_qga_guest_exec_invalid(gconstpointer fix)
+{
+    const TestFixture *fixture = fix;
+    QDict *ret, *error;
+    const gchar *class, *desc;
+
+    /* invalid command */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-exec', 'arguments': {"
+                 " 'path': '/bin/invalid-cmd42' } }");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    g_assert_nonnull(error);
+    class = qdict_get_str(error, "class");
+    desc = qdict_get_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_cmpint(strlen(desc), >, 0);
+    QDECREF(ret);
+
+    /* invalid pid */
+    ret = qmp_fd(fixture->fd, "{'execute': 'guest-exec-status',"
+                 " 'arguments': { 'pid': 0 } }");
+    g_assert_nonnull(ret);
+    error = qdict_get_qdict(ret, "error");
+    g_assert_nonnull(error);
+    class = qdict_get_str(error, "class");
+    desc = qdict_get_str(error, "desc");
+    g_assert_cmpstr(class, ==, "GenericError");
+    g_assert_cmpint(strlen(desc), >, 0);
+    QDECREF(ret);
+}
+
+int main(int argc, char **argv)
+{
+    TestFixture fix;
+    int ret;
+
+    setlocale (LC_ALL, "");
+    g_test_init(&argc, &argv, NULL);
+    fixture_setup(&fix, NULL);
+
+    g_test_add_data_func("/qga/sync-delimited", &fix, test_qga_sync_delimited);
+    g_test_add_data_func("/qga/sync", &fix, test_qga_sync);
+    g_test_add_data_func("/qga/ping", &fix, test_qga_ping);
+    g_test_add_data_func("/qga/info", &fix, test_qga_info);
+    g_test_add_data_func("/qga/network-get-interfaces", &fix,
+                         test_qga_network_get_interfaces);
+    g_test_add_data_func("/qga/get-vcpus", &fix, test_qga_get_vcpus);
+    g_test_add_data_func("/qga/get-fsinfo", &fix, test_qga_get_fsinfo);
+    g_test_add_data_func("/qga/get-memory-block-info", &fix,
+                         test_qga_get_memory_block_info);
+    g_test_add_data_func("/qga/get-memory-blocks", &fix,
+                         test_qga_get_memory_blocks);
+    g_test_add_data_func("/qga/file-ops", &fix, test_qga_file_ops);
+    g_test_add_data_func("/qga/file-write-read", &fix, test_qga_file_write_read);
+    g_test_add_data_func("/qga/get-time", &fix, test_qga_get_time);
+    g_test_add_data_func("/qga/invalid-cmd", &fix, test_qga_invalid_cmd);
+    g_test_add_data_func("/qga/fsfreeze-status", &fix,
+                         test_qga_fsfreeze_status);
+
+    g_test_add_data_func("/qga/blacklist", NULL, test_qga_blacklist);
+    g_test_add_data_func("/qga/config", NULL, test_qga_config);
+    g_test_add_data_func("/qga/guest-exec", &fix, test_qga_guest_exec);
+    g_test_add_data_func("/qga/guest-exec-invalid", &fix,
+                         test_qga_guest_exec_invalid);
+
+    if (g_getenv("QGA_TEST_SIDE_EFFECTING")) {
+        g_test_add_data_func("/qga/fsfreeze-and-thaw", &fix,
+                             test_qga_fsfreeze_and_thaw);
+        g_test_add_data_func("/qga/set-time", &fix, test_qga_set_time);
+        g_test_add_data_func("/qga/fstrim", &fix, test_qga_fstrim);
+    }
+
+    ret = g_test_run();
+
+    fixture_tear_down(&fix, NULL);
+
+    return ret;
+}
diff --git a/tests/test-qht-par.c b/tests/test-qht-par.c
new file mode 100644
index 0000000..d8a83ca
--- /dev/null
+++ b/tests/test-qht-par.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+
+#define TEST_QHT_STRING "tests/qht-bench 1>/dev/null 2>&1 -R -S0.1 -D10000 -N1 "
+
+static void test_qht(int n_threads, int update_rate, int duration)
+{
+    char *str;
+    int rc;
+
+    str = g_strdup_printf(TEST_QHT_STRING "-n %d -u %d -d %d",
+                          n_threads, update_rate, duration);
+    rc = system(str);
+    g_free(str);
+    g_assert_cmpint(rc, ==, 0);
+}
+
+static void test_2th0u1s(void)
+{
+    test_qht(2, 0, 1);
+}
+
+static void test_2th20u1s(void)
+{
+    test_qht(2, 20, 1);
+}
+
+static void test_2th0u5s(void)
+{
+    test_qht(2, 0, 5);
+}
+
+static void test_2th20u5s(void)
+{
+    test_qht(2, 20, 5);
+}
+
+int main(int argc, char *argv[])
+{
+    g_test_init(&argc, &argv, NULL);
+
+    if (g_test_quick()) {
+        g_test_add_func("/qht/parallel/2threads-0%updates-1s", test_2th0u1s);
+        g_test_add_func("/qht/parallel/2threads-20%updates-1s", test_2th20u1s);
+    } else {
+        g_test_add_func("/qht/parallel/2threads-0%updates-5s", test_2th0u5s);
+        g_test_add_func("/qht/parallel/2threads-20%updates-5s", test_2th20u5s);
+    }
+    return g_test_run();
+}
diff --git a/tests/test-qht.c b/tests/test-qht.c
new file mode 100644
index 0000000..46a64b6
--- /dev/null
+++ b/tests/test-qht.c
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/qht.h"
+
+#define N 5000
+
+static struct qht ht;
+static int32_t arr[N * 2];
+
+static bool is_equal(const void *obj, const void *userp)
+{
+    const int32_t *a = obj;
+    const int32_t *b = userp;
+
+    return *a == *b;
+}
+
+static void insert(int a, int b)
+{
+    int i;
+
+    for (i = a; i < b; i++) {
+        uint32_t hash;
+
+        arr[i] = i;
+        hash = i;
+
+        qht_insert(&ht, &arr[i], hash);
+    }
+}
+
+static void rm(int init, int end)
+{
+    int i;
+
+    for (i = init; i < end; i++) {
+        uint32_t hash;
+
+        hash = arr[i];
+        g_assert_true(qht_remove(&ht, &arr[i], hash));
+    }
+}
+
+static void check(int a, int b, bool expected)
+{
+    struct qht_stats stats;
+    int i;
+
+    for (i = a; i < b; i++) {
+        void *p;
+        uint32_t hash;
+        int32_t val;
+
+        val = i;
+        hash = i;
+        p = qht_lookup(&ht, is_equal, &val, hash);
+        g_assert_true(!!p == expected);
+    }
+    qht_statistics_init(&ht, &stats);
+    if (stats.used_head_buckets) {
+        g_assert_cmpfloat(qdist_avg(&stats.chain), >=, 1.0);
+    }
+    g_assert_cmpuint(stats.head_buckets, >, 0);
+    qht_statistics_destroy(&stats);
+}
+
+static void count_func(struct qht *ht, void *p, uint32_t hash, void *userp)
+{
+    unsigned int *curr = userp;
+
+    (*curr)++;
+}
+
+static void check_n(size_t expected)
+{
+    struct qht_stats stats;
+
+    qht_statistics_init(&ht, &stats);
+    g_assert_cmpuint(stats.entries, ==, expected);
+    qht_statistics_destroy(&stats);
+}
+
+static void iter_check(unsigned int count)
+{
+    unsigned int curr = 0;
+
+    qht_iter(&ht, count_func, &curr);
+    g_assert_cmpuint(curr, ==, count);
+}
+
+static void qht_do_test(unsigned int mode, size_t init_entries)
+{
+    /* under KVM we might fetch stats from an uninitialized qht */
+    check_n(0);
+
+    qht_init(&ht, 0, mode);
+
+    check_n(0);
+    insert(0, N);
+    check(0, N, true);
+    check_n(N);
+    check(-N, -1, false);
+    iter_check(N);
+
+    rm(101, 102);
+    check_n(N - 1);
+    insert(N, N * 2);
+    check_n(N + N - 1);
+    rm(N, N * 2);
+    check_n(N - 1);
+    insert(101, 102);
+    check_n(N);
+
+    rm(10, 200);
+    check_n(N - 190);
+    insert(150, 200);
+    check_n(N - 190 + 50);
+    insert(10, 150);
+    check_n(N);
+
+    rm(1, 2);
+    check_n(N - 1);
+    qht_reset_size(&ht, 0);
+    check_n(0);
+    check(0, N, false);
+
+    qht_destroy(&ht);
+}
+
+static void qht_test(unsigned int mode)
+{
+    qht_do_test(mode, 0);
+    qht_do_test(mode, 1);
+    qht_do_test(mode, 2);
+    qht_do_test(mode, 8);
+    qht_do_test(mode, 16);
+    qht_do_test(mode, 8192);
+    qht_do_test(mode, 16384);
+}
+
+static void test_default(void)
+{
+    qht_test(0);
+}
+
+static void test_resize(void)
+{
+    qht_test(QHT_MODE_AUTO_RESIZE);
+}
+
+int main(int argc, char *argv[])
+{
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/qht/mode/default", test_default);
+    g_test_add_func("/qht/mode/resize", test_resize);
+    return g_test_run();
+}
diff --git a/tests/test-qmp-commands.c b/tests/test-qmp-commands.c
index 554e222..261fd9e 100644
--- a/tests/test-qmp-commands.c
+++ b/tests/test-qmp-commands.c
@@ -1,4 +1,4 @@
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qapi/qmp/types.h"
 #include "test-qmp-commands.h"
@@ -12,6 +12,11 @@
 {
 }
 
+Empty2 *qmp_user_def_cmd0(Error **errp)
+{
+    return g_new0(Empty2, 1);
+}
+
 void qmp_user_def_cmd1(UserDefOne * ud1, Error **errp)
 {
 }
@@ -25,29 +30,62 @@
     UserDefOne *ud1d = g_malloc0(sizeof(UserDefOne));
 
     ud1c->string = strdup(ud1a->string);
-    ud1c->base = g_new0(UserDefZero, 1);
-    ud1c->base->integer = ud1a->base->integer;
+    ud1c->integer = ud1a->integer;
     ud1d->string = strdup(has_udb1 ? ud1b->string : "blah0");
-    ud1d->base = g_new0(UserDefZero, 1);
-    ud1d->base->integer = has_udb1 ? ud1b->base->integer : 0;
+    ud1d->integer = has_udb1 ? ud1b->integer : 0;
 
-    ret = g_malloc0(sizeof(UserDefTwo));
-    ret->string = strdup("blah1");
-    ret->dict.string = strdup("blah2");
-    ret->dict.dict.userdef = ud1c;
-    ret->dict.dict.string = strdup("blah3");
-    ret->dict.has_dict2 = true;
-    ret->dict.dict2.userdef = ud1d;
-    ret->dict.dict2.string = strdup("blah4");
+    ret = g_new0(UserDefTwo, 1);
+    ret->string0 = strdup("blah1");
+    ret->dict1 = g_new0(UserDefTwoDict, 1);
+    ret->dict1->string1 = strdup("blah2");
+    ret->dict1->dict2 = g_new0(UserDefTwoDictDict, 1);
+    ret->dict1->dict2->userdef = ud1c;
+    ret->dict1->dict2->string = strdup("blah3");
+    ret->dict1->dict3 = g_new0(UserDefTwoDictDict, 1);
+    ret->dict1->has_dict3 = true;
+    ret->dict1->dict3->userdef = ud1d;
+    ret->dict1->dict3->string = strdup("blah4");
 
     return ret;
 }
 
-int64_t qmp_user_def_cmd3(int64_t a, bool has_b, int64_t b, Error **errp)
+int64_t qmp_guest_get_time(int64_t a, bool has_b, int64_t b, Error **errp)
 {
     return a + (has_b ? b : 0);
 }
 
+QObject *qmp_guest_sync(QObject *arg, Error **errp)
+{
+    return arg;
+}
+
+void qmp_boxed_struct(UserDefZero *arg, Error **errp)
+{
+}
+
+void qmp_boxed_union(UserDefNativeListUnion *arg, Error **errp)
+{
+}
+
+__org_qemu_x_Union1 *qmp___org_qemu_x_command(__org_qemu_x_EnumList *a,
+                                              __org_qemu_x_StructList *b,
+                                              __org_qemu_x_Union2 *c,
+                                              __org_qemu_x_Alt *d,
+                                              Error **errp)
+{
+    __org_qemu_x_Union1 *ret = g_new0(__org_qemu_x_Union1, 1);
+
+    ret->type = ORG_QEMU_X_UNION1_KIND___ORG_QEMU_X_BRANCH;
+    ret->u.__org_qemu_x_branch.data = strdup("blah1");
+
+    /* Also test that 'wchar-t' was munged to 'q_wchar_t' */
+    if (b && b->value && !b->value->has_q_wchar_t) {
+        b->value->q_wchar_t = 1;
+    }
+    return ret;
+}
+
+
 /* test commands with no input and no return value */
 static void test_dispatch_cmd(void)
 {
@@ -65,7 +103,7 @@
 }
 
 /* test commands that return an error due to invalid parameters */
-static void test_dispatch_cmd_error(void)
+static void test_dispatch_cmd_failure(void)
 {
     QDict *req = qdict_new();
     QObject *resp;
@@ -120,15 +158,15 @@
 
     ret = qobject_to_qdict(test_qmp_dispatch(req));
 
-    assert(!strcmp(qdict_get_str(ret, "string"), "blah1"));
-    ret_dict = qdict_get_qdict(ret, "dict");
-    assert(!strcmp(qdict_get_str(ret_dict, "string"), "blah2"));
-    ret_dict_dict = qdict_get_qdict(ret_dict, "dict");
+    assert(!strcmp(qdict_get_str(ret, "string0"), "blah1"));
+    ret_dict = qdict_get_qdict(ret, "dict1");
+    assert(!strcmp(qdict_get_str(ret_dict, "string1"), "blah2"));
+    ret_dict_dict = qdict_get_qdict(ret_dict, "dict2");
     ret_dict_dict_userdef = qdict_get_qdict(ret_dict_dict, "userdef");
     assert(qdict_get_int(ret_dict_dict_userdef, "integer") == 42);
     assert(!strcmp(qdict_get_str(ret_dict_dict_userdef, "string"), "hello"));
     assert(!strcmp(qdict_get_str(ret_dict_dict, "string"), "blah3"));
-    ret_dict_dict2 = qdict_get_qdict(ret_dict, "dict2");
+    ret_dict_dict2 = qdict_get_qdict(ret_dict, "dict3");
     ret_dict_dict2_userdef = qdict_get_qdict(ret_dict_dict2, "userdef");
     assert(qdict_get_int(ret_dict_dict2_userdef, "integer") == 422);
     assert(!strcmp(qdict_get_str(ret_dict_dict2_userdef, "string"), "hello2"));
@@ -137,7 +175,7 @@
 
     qdict_put(args3, "a", qint_from_int(66));
     qdict_put(req, "arguments", args3);
-    qdict_put(req, "execute", qstring_from_str("user_def_cmd3"));
+    qdict_put(req, "execute", qstring_from_str("guest-get-time"));
 
     ret3 = qobject_to_qint(test_qmp_dispatch(req));
     assert(qint_get_int(ret3) == 66);
@@ -153,20 +191,17 @@
     UserDefOneList *ud1list;
 
     ud1test = g_malloc0(sizeof(UserDefOne));
-    ud1test->base = g_new0(UserDefZero, 1);
-    ud1test->base->integer = 42;
+    ud1test->integer = 42;
     ud1test->string = g_strdup("hi there 42");
 
     qapi_free_UserDefOne(ud1test);
 
     ud1a = g_malloc0(sizeof(UserDefOne));
-    ud1a->base = g_new0(UserDefZero, 1);
-    ud1a->base->integer = 43;
+    ud1a->integer = 43;
     ud1a->string = g_strdup("hi there 43");
 
     ud1b = g_malloc0(sizeof(UserDefOne));
-    ud1b->base = g_new0(UserDefZero, 1);
-    ud1b->base->integer = 44;
+    ud1b->integer = 44;
     ud1b->string = g_strdup("hi there 44");
 
     ud1list = g_malloc0(sizeof(UserDefOneList));
@@ -189,26 +224,24 @@
     /* create partial object */
     {
         QDict *ud2_dict;
-        QmpInputVisitor *qiv;
+        Visitor *v;
 
         ud2_dict = qdict_new();
-        qdict_put_obj(ud2_dict, "string", QOBJECT(qstring_from_str(text)));
+        qdict_put_obj(ud2_dict, "string0", QOBJECT(qstring_from_str(text)));
 
-        qiv = qmp_input_visitor_new(QOBJECT(ud2_dict));
-        visit_type_UserDefTwo(qmp_input_get_visitor(qiv), &ud2, NULL, &err);
-        qmp_input_visitor_cleanup(qiv);
+        v = qmp_input_visitor_new(QOBJECT(ud2_dict), true);
+        visit_type_UserDefTwo(v, NULL, &ud2, &err);
+        visit_free(v);
         QDECREF(ud2_dict);
     }
 
-    /* verify partial success */
-    assert(ud2 != NULL);
-    assert(ud2->string != NULL);
-    assert(strcmp(ud2->string, text) == 0);
-    assert(ud2->dict.dict.userdef == NULL);
+    /* verify that visit_type_XXX() cleans up properly on error */
+    error_free_or_abort(&err);
+    assert(!ud2);
 
-    /* confirm & release construction error */
-    assert(err != NULL);
-    error_free(err);
+    /* Manually create a partial object, leaving ud2->dict1 at NULL */
+    ud2 = g_new0(UserDefTwo, 1);
+    ud2->string0 = g_strdup(text);
 
     /* tear down partial object */
     qapi_free_UserDefTwo(ud2);
@@ -220,7 +253,7 @@
     g_test_init(&argc, &argv, NULL);
 
     g_test_add_func("/0.15/dispatch_cmd", test_dispatch_cmd);
-    g_test_add_func("/0.15/dispatch_cmd_error", test_dispatch_cmd_error);
+    g_test_add_func("/0.15/dispatch_cmd_failure", test_dispatch_cmd_failure);
     g_test_add_func("/0.15/dispatch_cmd_io", test_dispatch_cmd_io);
     g_test_add_func("/0.15/dealloc_types", test_dealloc_types);
     g_test_add_func("/0.15/dealloc_partial", test_dealloc_partial);
diff --git a/tests/test-qmp-event.c b/tests/test-qmp-event.c
index cb354e6..633dc87 100644
--- a/tests/test-qmp-event.c
+++ b/tests/test-qmp-event.c
@@ -11,8 +11,7 @@
  *
  */
 
-#include <glib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "test-qapi-types.h"
@@ -60,8 +59,8 @@
 
     switch (qobject_type(obj1)) {
     case QTYPE_QBOOL:
-        d->result = (qbool_get_int(qobject_to_qbool(obj1)) ==
-                     qbool_get_int(qobject_to_qbool(obj2)));
+        d->result = (qbool_get_bool(qobject_to_qbool(obj1)) ==
+                     qbool_get_bool(qobject_to_qbool(obj2)));
         return;
     case QTYPE_QINT:
         d->result = (qint_get_int(qobject_to_qint(obj1)) ==
@@ -94,7 +93,7 @@
 
 /* This function is hooked as final emit function, which can verify the
    correctness. */
-static void event_test_emit(TEST_QAPIEvent event, QDict *d, Error **errp)
+static void event_test_emit(test_QAPIEvent event, QDict *d, Error **errp)
 {
     QObject *obj;
     QDict *t;
@@ -179,9 +178,7 @@
     QDict *d, *d_data, *d_b;
 
     UserDefOne b;
-    UserDefZero z;
-    z.integer = 2;
-    b.base = &z;
+    b.integer = 2;
     b.string = g_strdup("test1");
     b.has_enum1 = false;
 
@@ -209,11 +206,9 @@
 {
     UserDefOne struct1;
     EventStructOne a;
-    UserDefZero z;
     QDict *d, *d_data, *d_a, *d_struct1;
 
-    z.integer = 2;
-    struct1.base = &z;
+    struct1.integer = 2;
     struct1.string = g_strdup("test1");
     struct1.has_enum1 = true;
     struct1.enum1 = ENUM_ONE_VALUE1;
diff --git a/tests/test-qmp-input-strict.c b/tests/test-qmp-input-strict.c
index d5360c6..814550a 100644
--- a/tests/test-qmp-input-strict.c
+++ b/tests/test-qmp-input-strict.c
@@ -1,7 +1,7 @@
 /*
  * QMP Input Visitor unit-tests (strict mode).
  *
- * Copyright (C) 2011-2012 Red Hat Inc.
+ * Copyright (C) 2011-2012, 2015 Red Hat Inc.
  *
  * Authors:
  *  Luiz Capitulino <lcapitulino@redhat.com>
@@ -11,18 +11,22 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
+#include "qapi/qmp/qjson.h"
+#include "test-qmp-introspect.h"
+#include "qmp-introspect.h"
+#include "qapi-visit.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
-    QmpInputVisitor *qiv;
+    Visitor *qiv;
 } TestInputVisitorData;
 
 static void validate_teardown(TestInputVisitorData *data,
@@ -32,14 +36,28 @@
     data->obj = NULL;
 
     if (data->qiv) {
-        qmp_input_visitor_cleanup(data->qiv);
+        visit_free(data->qiv);
         data->qiv = NULL;
     }
 }
 
-/* This is provided instead of a test setup function so that the JSON
-   string used by the tests are kept in the test functions (and not
-   int main()) */
+/* The various test_init functions are provided instead of a test setup
+   function so that the JSON string used by the tests are kept in the test
+   functions (and not in main()). */
+static Visitor *validate_test_init_internal(TestInputVisitorData *data,
+                                            const char *json_string,
+                                            va_list *ap)
+{
+    validate_teardown(data, NULL);
+
+    data->obj = qobject_from_jsonv(json_string, ap);
+    g_assert(data->obj);
+
+    data->qiv = qmp_input_visitor_new(data->obj, true);
+    g_assert(data->qiv);
+    return data->qiv;
+}
+
 static GCC_FMT_ATTR(2, 3)
 Visitor *validate_test_init(TestInputVisitorData *data,
                              const char *json_string, ...)
@@ -48,67 +66,34 @@
     va_list ap;
 
     va_start(ap, json_string);
-    data->obj = qobject_from_jsonv(json_string, &ap);
+    v = validate_test_init_internal(data, json_string, &ap);
     va_end(ap);
-
-    g_assert(data->obj != NULL);
-
-    data->qiv = qmp_input_visitor_new_strict(data->obj);
-    g_assert(data->qiv != NULL);
-
-    v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
-
     return v;
 }
 
-typedef struct TestStruct
+/* similar to validate_test_init(), but does not expect a string
+ * literal/format json_string argument and so can be used for
+ * programatically generated strings (and we can't pass in programatically
+ * generated strings via %s format parameters since qobject_from_jsonv()
+ * will wrap those in double-quotes and treat the entire object as a
+ * string)
+ */
+static Visitor *validate_test_init_raw(TestInputVisitorData *data,
+                                       const char *json_string)
 {
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
+    return validate_test_init_internal(data, json_string, NULL);
 }
 
+
 static void test_validate_struct(TestInputVisitorData *data,
                                   const void *unused)
 {
     TestStruct *p = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(v, NULL, &p, &error_abort);
     g_free(p->string);
     g_free(p);
 }
@@ -116,43 +101,40 @@
 static void test_validate_struct_nested(TestInputVisitorData *data,
                                          const void *unused)
 {
-    UserDefNested *udp = NULL;
-    Error *err = NULL;
+    UserDefTwo *udp = NULL;
     Visitor *v;
 
-    v = validate_test_init(data, "{ 'string0': 'string0', 'dict1': { 'string1': 'string1', 'dict2': { 'userdef1': { 'integer': 42, 'string': 'string' }, 'string2': 'string2'}}}");
+    v = validate_test_init(data, "{ 'string0': 'string0', "
+                           "'dict1': { 'string1': 'string1', "
+                           "'dict2': { 'userdef': { 'integer': 42, "
+                           "'string': 'string' }, 'string': 'string2'}}}");
 
-    visit_type_UserDefNested(v, &udp, NULL, &err);
-    g_assert(!err);
-    qapi_free_UserDefNested(udp);
+    visit_type_UserDefTwo(v, NULL, &udp, &error_abort);
+    qapi_free_UserDefTwo(udp);
 }
 
 static void test_validate_list(TestInputVisitorData *data,
                                 const void *unused)
 {
     UserDefOneList *head = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44 } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefOneList(v, NULL, &head, &error_abort);
     qapi_free_UserDefOneList(head);
 }
 
-static void test_validate_union(TestInputVisitorData *data,
-                                 const void *unused)
+static void test_validate_union_native_list(TestInputVisitorData *data,
+                                            const void *unused)
 {
-    UserDefUnion *tmp = NULL;
+    UserDefNativeListUnion *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
-    v = validate_test_init(data, "{ 'type': 'b', 'integer': 41, 'data' : { 'integer': 42 } }");
+    v = validate_test_init(data, "{ 'type': 'integer', 'data' : [ 1, 2 ] }");
 
-    visit_type_UserDefUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
-    qapi_free_UserDefUnion(tmp);
+    visit_type_UserDefNativeListUnion(v, NULL, &tmp, &error_abort);
+    qapi_free_UserDefNativeListUnion(tmp);
 }
 
 static void test_validate_union_flat(TestInputVisitorData *data,
@@ -160,31 +142,27 @@
 {
     UserDefFlatUnion *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data,
                            "{ 'enum1': 'value1', "
+                           "'integer': 41, "
                            "'string': 'str', "
                            "'boolean': true }");
-    /* TODO when generator bug is fixed, add 'integer': 41 */
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefFlatUnion(v, NULL, &tmp, &error_abort);
     qapi_free_UserDefFlatUnion(tmp);
 }
 
-static void test_validate_union_anon(TestInputVisitorData *data,
-                                     const void *unused)
+static void test_validate_alternate(TestInputVisitorData *data,
+                                    const void *unused)
 {
-    UserDefAnonUnion *tmp = NULL;
+    UserDefAlternate *tmp = NULL;
     Visitor *v;
-    Error *err = NULL;
 
     v = validate_test_init(data, "42");
 
-    visit_type_UserDefAnonUnion(v, &tmp, NULL, &err);
-    g_assert(!err);
-    qapi_free_UserDefAnonUnion(tmp);
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    qapi_free_UserDefAlternate(tmp);
 }
 
 static void test_validate_fail_struct(TestInputVisitorData *data,
@@ -196,26 +174,23 @@
 
     v = validate_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo', 'extra': 42 }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(err);
-    if (p) {
-        g_free(p->string);
-    }
-    g_free(p);
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
 }
 
 static void test_validate_fail_struct_nested(TestInputVisitorData *data,
                                               const void *unused)
 {
-    UserDefNested *udp = NULL;
+    UserDefTwo *udp = NULL;
     Error *err = NULL;
     Visitor *v;
 
     v = validate_test_init(data, "{ 'string0': 'string0', 'dict1': { 'string1': 'string1', 'dict2': { 'userdef1': { 'integer': 42, 'string': 'string', 'extra': [42, 23, {'foo':'bar'}] }, 'string2': 'string2'}}}");
 
-    visit_type_UserDefNested(v, &udp, NULL, &err);
-    g_assert(err);
-    qapi_free_UserDefNested(udp);
+    visit_type_UserDefTwo(v, NULL, &udp, &err);
+    error_free_or_abort(&err);
+    g_assert(!udp);
 }
 
 static void test_validate_fail_list(TestInputVisitorData *data,
@@ -227,23 +202,24 @@
 
     v = validate_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44, 'extra': 'ggg' } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(err);
-    qapi_free_UserDefOneList(head);
+    visit_type_UserDefOneList(v, NULL, &head, &err);
+    error_free_or_abort(&err);
+    g_assert(!head);
 }
 
-static void test_validate_fail_union(TestInputVisitorData *data,
-                                      const void *unused)
+static void test_validate_fail_union_native_list(TestInputVisitorData *data,
+                                                 const void *unused)
 {
-    UserDefUnion *tmp = NULL;
+    UserDefNativeListUnion *tmp = NULL;
     Error *err = NULL;
     Visitor *v;
 
-    v = validate_test_init(data, "{ 'type': 'b', 'data' : { 'integer': 42 } }");
+    v = validate_test_init(data,
+                           "{ 'type': 'integer', 'data' : [ 'string' ] }");
 
-    visit_type_UserDefUnion(v, &tmp, NULL, &err);
-    g_assert(err);
-    qapi_free_UserDefUnion(tmp);
+    visit_type_UserDefNativeListUnion(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
+    g_assert(!tmp);
 }
 
 static void test_validate_fail_union_flat(TestInputVisitorData *data,
@@ -255,9 +231,9 @@
 
     v = validate_test_init(data, "{ 'string': 'c', 'integer': 41, 'boolean': true }");
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(err);
-    qapi_free_UserDefFlatUnion(tmp);
+    visit_type_UserDefFlatUnion(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
+    g_assert(!tmp);
 }
 
 static void test_validate_fail_union_flat_no_discrim(TestInputVisitorData *data,
@@ -268,25 +244,46 @@
     Visitor *v;
 
     /* test situation where discriminator field ('enum1' here) is missing */
-    v = validate_test_init(data, "{ 'string': 'c', 'string1': 'd', 'string2': 'e' }");
+    v = validate_test_init(data, "{ 'integer': 42, 'string': 'c', 'string1': 'd', 'string2': 'e' }");
 
-    visit_type_UserDefFlatUnion2(v, &tmp, NULL, &err);
-    g_assert(err);
-    qapi_free_UserDefFlatUnion2(tmp);
+    visit_type_UserDefFlatUnion2(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
+    g_assert(!tmp);
 }
 
-static void test_validate_fail_union_anon(TestInputVisitorData *data,
-                                          const void *unused)
+static void test_validate_fail_alternate(TestInputVisitorData *data,
+                                         const void *unused)
 {
-    UserDefAnonUnion *tmp = NULL;
+    UserDefAlternate *tmp;
     Visitor *v;
     Error *err = NULL;
 
     v = validate_test_init(data, "3.14");
 
-    visit_type_UserDefAnonUnion(v, &tmp, NULL, &err);
-    g_assert(err);
-    qapi_free_UserDefAnonUnion(tmp);
+    visit_type_UserDefAlternate(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
+    g_assert(!tmp);
+}
+
+static void do_test_validate_qmp_introspect(TestInputVisitorData *data,
+                                            const char *schema_json)
+{
+    SchemaInfoList *schema = NULL;
+    Visitor *v;
+
+    v = validate_test_init_raw(data, schema_json);
+
+    visit_type_SchemaInfoList(v, NULL, &schema, &error_abort);
+    g_assert(schema);
+
+    qapi_free_SchemaInfoList(schema);
+}
+
+static void test_validate_qmp_introspect(TestInputVisitorData *data,
+                                           const void *unused)
+{
+    do_test_validate_qmp_introspect(data, test_qmp_schema_json);
+    do_test_validate_qmp_introspect(data, qmp_schema_json);
 }
 
 static void validate_test_add(const char *testpath,
@@ -304,31 +301,33 @@
     g_test_init(&argc, &argv, NULL);
 
     validate_test_add("/visitor/input-strict/pass/struct",
-                       &testdata, test_validate_struct);
+                      &testdata, test_validate_struct);
     validate_test_add("/visitor/input-strict/pass/struct-nested",
-                       &testdata, test_validate_struct_nested);
+                      &testdata, test_validate_struct_nested);
     validate_test_add("/visitor/input-strict/pass/list",
-                       &testdata, test_validate_list);
-    validate_test_add("/visitor/input-strict/pass/union",
-                       &testdata, test_validate_union);
+                      &testdata, test_validate_list);
     validate_test_add("/visitor/input-strict/pass/union-flat",
-                       &testdata, test_validate_union_flat);
-    validate_test_add("/visitor/input-strict/pass/union-anon",
-                       &testdata, test_validate_union_anon);
+                      &testdata, test_validate_union_flat);
+    validate_test_add("/visitor/input-strict/pass/alternate",
+                      &testdata, test_validate_alternate);
+    validate_test_add("/visitor/input-strict/pass/union-native-list",
+                      &testdata, test_validate_union_native_list);
     validate_test_add("/visitor/input-strict/fail/struct",
-                       &testdata, test_validate_fail_struct);
+                      &testdata, test_validate_fail_struct);
     validate_test_add("/visitor/input-strict/fail/struct-nested",
-                       &testdata, test_validate_fail_struct_nested);
+                      &testdata, test_validate_fail_struct_nested);
     validate_test_add("/visitor/input-strict/fail/list",
-                       &testdata, test_validate_fail_list);
-    validate_test_add("/visitor/input-strict/fail/union",
-                       &testdata, test_validate_fail_union);
+                      &testdata, test_validate_fail_list);
     validate_test_add("/visitor/input-strict/fail/union-flat",
-                       &testdata, test_validate_fail_union_flat);
+                      &testdata, test_validate_fail_union_flat);
     validate_test_add("/visitor/input-strict/fail/union-flat-no-discriminator",
-                       &testdata, test_validate_fail_union_flat_no_discrim);
-    validate_test_add("/visitor/input-strict/fail/union-anon",
-                       &testdata, test_validate_fail_union_anon);
+                      &testdata, test_validate_fail_union_flat_no_discrim);
+    validate_test_add("/visitor/input-strict/fail/alternate",
+                      &testdata, test_validate_fail_alternate);
+    validate_test_add("/visitor/input-strict/fail/union-native-list",
+                      &testdata, test_validate_fail_union_native_list);
+    validate_test_add("/visitor/input-strict/pass/qmp-introspect",
+                      &testdata, test_validate_qmp_introspect);
 
     g_test_run();
 
diff --git a/tests/test-qmp-input-visitor.c b/tests/test-qmp-input-visitor.c
index 1c8e872..f583dce 100644
--- a/tests/test-qmp-input-visitor.c
+++ b/tests/test-qmp-input-visitor.c
@@ -1,7 +1,7 @@
 /*
  * QMP Input Visitor unit-tests.
  *
- * Copyright (C) 2011 Red Hat Inc.
+ * Copyright (C) 2011-2016 Red Hat Inc.
  *
  * Authors:
  *  Luiz Capitulino <lcapitulino@redhat.com>
@@ -10,18 +10,19 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/qmp-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
+#include "qapi/qmp/qjson.h"
 
 typedef struct TestInputVisitorData {
     QObject *obj;
-    QmpInputVisitor *qiv;
+    Visitor *qiv;
 } TestInputVisitorData;
 
 static void visitor_input_teardown(TestInputVisitorData *data,
@@ -31,14 +32,28 @@
     data->obj = NULL;
 
     if (data->qiv) {
-        qmp_input_visitor_cleanup(data->qiv);
+        visit_free(data->qiv);
         data->qiv = NULL;
     }
 }
 
-/* This is provided instead of a test setup function so that the JSON
-   string used by the tests are kept in the test functions (and not
-   int main()) */
+/* The various test_init functions are provided instead of a test setup
+   function so that the JSON string used by the tests are kept in the test
+   functions (and not in main()). */
+static Visitor *visitor_input_test_init_internal(TestInputVisitorData *data,
+                                                 const char *json_string,
+                                                 va_list *ap)
+{
+    visitor_input_teardown(data, NULL);
+
+    data->obj = qobject_from_jsonv(json_string, ap);
+    g_assert(data->obj);
+
+    data->qiv = qmp_input_visitor_new(data->obj, false);
+    g_assert(data->qiv);
+    return data->qiv;
+}
+
 static GCC_FMT_ATTR(2, 3)
 Visitor *visitor_input_test_init(TestInputVisitorData *data,
                                  const char *json_string, ...)
@@ -47,17 +62,8 @@
     va_list ap;
 
     va_start(ap, json_string);
-    data->obj = qobject_from_jsonv(json_string, &ap);
+    v = visitor_input_test_init_internal(data, json_string, &ap);
     va_end(ap);
-
-    g_assert(data->obj != NULL);
-
-    data->qiv = qmp_input_visitor_new(data->obj);
-    g_assert(data->qiv != NULL);
-
-    v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
-
     return v;
 }
 
@@ -71,32 +77,18 @@
 static Visitor *visitor_input_test_init_raw(TestInputVisitorData *data,
                                             const char *json_string)
 {
-    Visitor *v;
-
-    data->obj = qobject_from_json(json_string);
-
-    g_assert(data->obj != NULL);
-
-    data->qiv = qmp_input_visitor_new(data->obj);
-    g_assert(data->qiv != NULL);
-
-    v = qmp_input_get_visitor(data->qiv);
-    g_assert(v != NULL);
-
-    return v;
+    return visitor_input_test_init_internal(data, json_string, NULL);
 }
 
 static void test_visitor_in_int(TestInputVisitorData *data,
                                 const void *unused)
 {
     int64_t res = 0, value = -42;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%" PRId64, value);
 
-    visit_type_int(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_int(v, NULL, &res, &error_abort);
     g_assert_cmpint(res, ==, value);
 }
 
@@ -113,22 +105,19 @@
      */
     v = visitor_input_test_init(data, "%f", DBL_MAX);
 
-    visit_type_int(v, &res, NULL, &err);
-    g_assert(err);
-    error_free(err);
+    visit_type_int(v, NULL, &res, &err);
+    error_free_or_abort(&err);
 }
 
 static void test_visitor_in_bool(TestInputVisitorData *data,
                                  const void *unused)
 {
-    Error *err = NULL;
     bool res = false;
     Visitor *v;
 
     v = visitor_input_test_init(data, "true");
 
-    visit_type_bool(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_bool(v, NULL, &res, &error_abort);
     g_assert_cmpint(res, ==, true);
 }
 
@@ -136,13 +125,11 @@
                                    const void *unused)
 {
     double res = 0, value = 3.14;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%f", value);
 
-    visit_type_number(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_number(v, NULL, &res, &error_abort);
     g_assert_cmpfloat(res, ==, value);
 }
 
@@ -150,13 +137,11 @@
                                    const void *unused)
 {
     char *res = NULL, *value = (char *) "Q E M U";
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "%s", value);
 
-    visit_type_str(v, &res, NULL, &err);
-    g_assert(!err);
+    visit_type_str(v, NULL, &res, &error_abort);
     g_assert_cmpstr(res, ==, value);
 
     g_free(res);
@@ -165,7 +150,6 @@
 static void test_visitor_in_enum(TestInputVisitorData *data,
                                  const void *unused)
 {
-    Error *err = NULL;
     Visitor *v;
     EnumOne i;
 
@@ -174,63 +158,21 @@
 
         v = visitor_input_test_init(data, "%s", EnumOne_lookup[i]);
 
-        visit_type_EnumOne(v, &res, NULL, &err);
-        g_assert(!err);
+        visit_type_EnumOne(v, NULL, &res, &error_abort);
         g_assert_cmpint(i, ==, res);
-
-        visitor_input_teardown(data, NULL);
     }
-
-    data->obj = NULL;
-    data->qiv = NULL;
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static void test_visitor_in_struct(TestInputVisitorData *data,
                                    const void *unused)
 {
     TestStruct *p = NULL;
-    Error *err = NULL;
     Visitor *v;
 
     v = visitor_input_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(v, NULL, &p, &error_abort);
     g_assert_cmpint(p->integer, ==, -42);
     g_assert(p->boolean == true);
     g_assert_cmpstr(p->string, ==, "foo");
@@ -239,47 +181,39 @@
     g_free(p);
 }
 
-static void check_and_free_str(char *str, const char *cmp)
-{
-    g_assert_cmpstr(str, ==, cmp);
-    g_free(str);
-}
-
 static void test_visitor_in_struct_nested(TestInputVisitorData *data,
                                           const void *unused)
 {
-    UserDefNested *udp = NULL;
-    Error *err = NULL;
+    UserDefTwo *udp = NULL;
     Visitor *v;
 
-    v = visitor_input_test_init(data, "{ 'string0': 'string0', 'dict1': { 'string1': 'string1', 'dict2': { 'userdef1': { 'integer': 42, 'string': 'string' }, 'string2': 'string2'}}}");
+    v = visitor_input_test_init(data, "{ 'string0': 'string0', "
+                                "'dict1': { 'string1': 'string1', "
+                                "'dict2': { 'userdef': { 'integer': 42, "
+                                "'string': 'string' }, 'string': 'string2'}}}");
 
-    visit_type_UserDefNested(v, &udp, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefTwo(v, NULL, &udp, &error_abort);
 
-    check_and_free_str(udp->string0, "string0");
-    check_and_free_str(udp->dict1.string1, "string1");
-    g_assert_cmpint(udp->dict1.dict2.userdef1->base->integer, ==, 42);
-    check_and_free_str(udp->dict1.dict2.userdef1->string, "string");
-    check_and_free_str(udp->dict1.dict2.string2, "string2");
-    g_assert(udp->dict1.has_dict3 == false);
+    g_assert_cmpstr(udp->string0, ==, "string0");
+    g_assert_cmpstr(udp->dict1->string1, ==, "string1");
+    g_assert_cmpint(udp->dict1->dict2->userdef->integer, ==, 42);
+    g_assert_cmpstr(udp->dict1->dict2->userdef->string, ==, "string");
+    g_assert_cmpstr(udp->dict1->dict2->string, ==, "string2");
+    g_assert(udp->dict1->has_dict3 == false);
 
-    g_free(udp->dict1.dict2.userdef1);
-    g_free(udp);
+    qapi_free_UserDefTwo(udp);
 }
 
 static void test_visitor_in_list(TestInputVisitorData *data,
                                  const void *unused)
 {
     UserDefOneList *item, *head = NULL;
-    Error *err = NULL;
     Visitor *v;
     int i;
 
     v = visitor_input_test_init(data, "[ { 'string': 'string0', 'integer': 42 }, { 'string': 'string1', 'integer': 43 }, { 'string': 'string2', 'integer': 44 } ]");
 
-    visit_type_UserDefOneList(v, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_UserDefOneList(v, NULL, &head, &error_abort);
     g_assert(head != NULL);
 
     for (i = 0, item = head; item; item = item->next, i++) {
@@ -287,65 +221,255 @@
 
         snprintf(string, sizeof(string), "string%d", i);
         g_assert_cmpstr(item->value->string, ==, string);
-        g_assert_cmpint(item->value->base->integer, ==, 42 + i);
+        g_assert_cmpint(item->value->integer, ==, 42 + i);
     }
 
     qapi_free_UserDefOneList(head);
+    head = NULL;
+
+    /* An empty list is valid */
+    v = visitor_input_test_init(data, "[]");
+    visit_type_UserDefOneList(v, NULL, &head, &error_abort);
+    g_assert(!head);
 }
 
-static void test_visitor_in_union(TestInputVisitorData *data,
-                                  const void *unused)
+static void test_visitor_in_any(TestInputVisitorData *data,
+                                const void *unused)
+{
+    QObject *res = NULL;
+    Visitor *v;
+    QInt *qint;
+    QBool *qbool;
+    QString *qstring;
+    QDict *qdict;
+    QObject *qobj;
+
+    v = visitor_input_test_init(data, "-42");
+    visit_type_any(v, NULL, &res, &error_abort);
+    qint = qobject_to_qint(res);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobject_decref(res);
+
+    v = visitor_input_test_init(data, "{ 'integer': -42, 'boolean': true, 'string': 'foo' }");
+    visit_type_any(v, NULL, &res, &error_abort);
+    qdict = qobject_to_qdict(res);
+    g_assert(qdict && qdict_size(qdict) == 3);
+    qobj = qdict_get(qdict, "integer");
+    g_assert(qobj);
+    qint = qobject_to_qint(qobj);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobj = qdict_get(qdict, "boolean");
+    g_assert(qobj);
+    qbool = qobject_to_qbool(qobj);
+    g_assert(qbool);
+    g_assert(qbool_get_bool(qbool) == true);
+    qobj = qdict_get(qdict, "string");
+    g_assert(qobj);
+    qstring = qobject_to_qstring(qobj);
+    g_assert(qstring);
+    g_assert_cmpstr(qstring_get_str(qstring), ==, "foo");
+    qobject_decref(res);
+}
+
+static void test_visitor_in_null(TestInputVisitorData *data,
+                                 const void *unused)
 {
     Visitor *v;
     Error *err = NULL;
-    UserDefUnion *tmp;
+    char *tmp;
 
-    v = visitor_input_test_init(data, "{ 'type': 'b', 'integer': 41, 'data' : { 'integer': 42 } }");
+    /*
+     * FIXME: Since QAPI doesn't know the 'null' type yet, we can't
+     * test visit_type_null() by reading into a QAPI struct then
+     * checking that it was populated correctly.  The best we can do
+     * for now is ensure that we consumed null from the input, proven
+     * by the fact that we can't re-read the key; and that we detect
+     * when input is not null.
+     */
 
-    visit_type_UserDefUnion(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, USER_DEF_UNION_KIND_B);
-    g_assert_cmpint(tmp->integer, ==, 41);
-    g_assert_cmpint(tmp->b->integer, ==, 42);
-    qapi_free_UserDefUnion(tmp);
+    v = visitor_input_test_init(data, "{ 'a': null, 'b': '' }");
+    visit_start_struct(v, NULL, NULL, 0, &error_abort);
+    visit_type_null(v, "a", &error_abort);
+    visit_type_str(v, "a", &tmp, &err);
+    g_assert(!tmp);
+    error_free_or_abort(&err);
+    visit_type_null(v, "b", &err);
+    error_free_or_abort(&err);
+    visit_check_struct(v, &error_abort);
+    visit_end_struct(v, NULL);
 }
 
 static void test_visitor_in_union_flat(TestInputVisitorData *data,
                                        const void *unused)
 {
     Visitor *v;
-    Error *err = NULL;
     UserDefFlatUnion *tmp;
+    UserDefUnionBase *base;
 
     v = visitor_input_test_init(data,
                                 "{ 'enum1': 'value1', "
+                                "'integer': 41, "
                                 "'string': 'str', "
                                 "'boolean': true }");
-    /* TODO when generator bug is fixed, add 'integer': 41 */
 
-    visit_type_UserDefFlatUnion(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, ENUM_ONE_VALUE1);
+    visit_type_UserDefFlatUnion(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->enum1, ==, ENUM_ONE_VALUE1);
     g_assert_cmpstr(tmp->string, ==, "str");
-    /* TODO g_assert_cmpint(tmp->integer, ==, 41); */
-    g_assert_cmpint(tmp->value1->boolean, ==, true);
+    g_assert_cmpint(tmp->integer, ==, 41);
+    g_assert_cmpint(tmp->u.value1.boolean, ==, true);
+
+    base = qapi_UserDefFlatUnion_base(tmp);
+    g_assert(&base->enum1 == &tmp->enum1);
+
     qapi_free_UserDefFlatUnion(tmp);
 }
 
-static void test_visitor_in_union_anon(TestInputVisitorData *data,
-                                       const void *unused)
+static void test_visitor_in_alternate(TestInputVisitorData *data,
+                                      const void *unused)
 {
     Visitor *v;
     Error *err = NULL;
-    UserDefAnonUnion *tmp;
+    UserDefAlternate *tmp;
+    WrapAlternate *wrap;
 
     v = visitor_input_test_init(data, "42");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->type, ==, QTYPE_QINT);
+    g_assert_cmpint(tmp->u.i, ==, 42);
+    qapi_free_UserDefAlternate(tmp);
 
-    visit_type_UserDefAnonUnion(v, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    g_assert_cmpint(tmp->kind, ==, USER_DEF_ANON_UNION_KIND_I);
-    g_assert_cmpint(tmp->i, ==, 42);
-    qapi_free_UserDefAnonUnion(tmp);
+    v = visitor_input_test_init(data, "'string'");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->type, ==, QTYPE_QSTRING);
+    g_assert_cmpstr(tmp->u.s, ==, "string");
+    qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "{'integer':1, 'string':'str', "
+                                "'enum1':'value1', 'boolean':true}");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &error_abort);
+    g_assert_cmpint(tmp->type, ==, QTYPE_QDICT);
+    g_assert_cmpint(tmp->u.udfu.integer, ==, 1);
+    g_assert_cmpstr(tmp->u.udfu.string, ==, "str");
+    g_assert_cmpint(tmp->u.udfu.enum1, ==, ENUM_ONE_VALUE1);
+    g_assert_cmpint(tmp->u.udfu.u.value1.boolean, ==, true);
+    g_assert_cmpint(tmp->u.udfu.u.value1.has_a_b, ==, false);
+    qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "false");
+    visit_type_UserDefAlternate(v, NULL, &tmp, &err);
+    error_free_or_abort(&err);
+    qapi_free_UserDefAlternate(tmp);
+
+    v = visitor_input_test_init(data, "{ 'alt': 42 }");
+    visit_type_WrapAlternate(v, NULL, &wrap, &error_abort);
+    g_assert_cmpint(wrap->alt->type, ==, QTYPE_QINT);
+    g_assert_cmpint(wrap->alt->u.i, ==, 42);
+    qapi_free_WrapAlternate(wrap);
+
+    v = visitor_input_test_init(data, "{ 'alt': 'string' }");
+    visit_type_WrapAlternate(v, NULL, &wrap, &error_abort);
+    g_assert_cmpint(wrap->alt->type, ==, QTYPE_QSTRING);
+    g_assert_cmpstr(wrap->alt->u.s, ==, "string");
+    qapi_free_WrapAlternate(wrap);
+
+    v = visitor_input_test_init(data, "{ 'alt': {'integer':1, 'string':'str', "
+                                "'enum1':'value1', 'boolean':true} }");
+    visit_type_WrapAlternate(v, NULL, &wrap, &error_abort);
+    g_assert_cmpint(wrap->alt->type, ==, QTYPE_QDICT);
+    g_assert_cmpint(wrap->alt->u.udfu.integer, ==, 1);
+    g_assert_cmpstr(wrap->alt->u.udfu.string, ==, "str");
+    g_assert_cmpint(wrap->alt->u.udfu.enum1, ==, ENUM_ONE_VALUE1);
+    g_assert_cmpint(wrap->alt->u.udfu.u.value1.boolean, ==, true);
+    g_assert_cmpint(wrap->alt->u.udfu.u.value1.has_a_b, ==, false);
+    qapi_free_WrapAlternate(wrap);
+}
+
+static void test_visitor_in_alternate_number(TestInputVisitorData *data,
+                                             const void *unused)
+{
+    Visitor *v;
+    Error *err = NULL;
+    AltStrBool *asb;
+    AltStrNum *asn;
+    AltNumStr *ans;
+    AltStrInt *asi;
+    AltIntNum *ain;
+    AltNumInt *ani;
+
+    /* Parsing an int */
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrBool(v, NULL, &asb, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrBool(asb);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrNum(v, NULL, &asn, &error_abort);
+    g_assert_cmpint(asn->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(asn->u.n, ==, 42);
+    qapi_free_AltStrNum(asn);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltNumStr(v, NULL, &ans, &error_abort);
+    g_assert_cmpint(ans->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ans->u.n, ==, 42);
+    qapi_free_AltNumStr(ans);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltStrInt(v, NULL, &asi, &error_abort);
+    g_assert_cmpint(asi->type, ==, QTYPE_QINT);
+    g_assert_cmpint(asi->u.i, ==, 42);
+    qapi_free_AltStrInt(asi);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltIntNum(v, NULL, &ain, &error_abort);
+    g_assert_cmpint(ain->type, ==, QTYPE_QINT);
+    g_assert_cmpint(ain->u.i, ==, 42);
+    qapi_free_AltIntNum(ain);
+
+    v = visitor_input_test_init(data, "42");
+    visit_type_AltNumInt(v, NULL, &ani, &error_abort);
+    g_assert_cmpint(ani->type, ==, QTYPE_QINT);
+    g_assert_cmpint(ani->u.i, ==, 42);
+    qapi_free_AltNumInt(ani);
+
+    /* Parsing a double */
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrBool(v, NULL, &asb, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrBool(asb);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrNum(v, NULL, &asn, &error_abort);
+    g_assert_cmpint(asn->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(asn->u.n, ==, 42.5);
+    qapi_free_AltStrNum(asn);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltNumStr(v, NULL, &ans, &error_abort);
+    g_assert_cmpint(ans->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ans->u.n, ==, 42.5);
+    qapi_free_AltNumStr(ans);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltStrInt(v, NULL, &asi, &err);
+    error_free_or_abort(&err);
+    qapi_free_AltStrInt(asi);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltIntNum(v, NULL, &ain, &error_abort);
+    g_assert_cmpint(ain->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ain->u.n, ==, 42.5);
+    qapi_free_AltIntNum(ain);
+
+    v = visitor_input_test_init(data, "42.5");
+    visit_type_AltNumInt(v, NULL, &ani, &error_abort);
+    g_assert_cmpint(ani->type, ==, QTYPE_QFLOAT);
+    g_assert_cmpfloat(ani->u.n, ==, 42.5);
+    qapi_free_AltNumInt(ani);
 }
 
 static void test_native_list_integer_helper(TestInputVisitorData *data,
@@ -353,7 +477,6 @@
                                             UserDefNativeListUnionKind kind)
 {
     UserDefNativeListUnion *cvalue = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -370,71 +493,71 @@
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, kind);
+    g_assert_cmpint(cvalue->type, ==, kind);
 
     switch (kind) {
     case USER_DEF_NATIVE_LIST_UNION_KIND_INTEGER: {
         intList *elem = NULL;
-        for (i = 0, elem = cvalue->integer; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.integer.data;
+             elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S8: {
         int8List *elem = NULL;
-        for (i = 0, elem = cvalue->s8; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s8.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S16: {
         int16List *elem = NULL;
-        for (i = 0, elem = cvalue->s16; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s16.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S32: {
         int32List *elem = NULL;
-        for (i = 0, elem = cvalue->s32; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s32.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S64: {
         int64List *elem = NULL;
-        for (i = 0, elem = cvalue->s64; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.s64.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U8: {
         uint8List *elem = NULL;
-        for (i = 0, elem = cvalue->u8; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u8.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U16: {
         uint16List *elem = NULL;
-        for (i = 0, elem = cvalue->u16; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u16.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U32: {
         uint32List *elem = NULL;
-        for (i = 0, elem = cvalue->u32; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u32.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U64: {
         uint64List *elem = NULL;
-        for (i = 0, elem = cvalue->u64; elem; elem = elem->next, i++) {
+        for (i = 0, elem = cvalue->u.u64.data; elem; elem = elem->next, i++) {
             g_assert_cmpint(elem->value, ==, i);
         }
         break;
@@ -516,7 +639,6 @@
 {
     UserDefNativeListUnion *cvalue = NULL;
     boolList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -533,12 +655,11 @@
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN);
 
-    for (i = 0, elem = cvalue->boolean; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.boolean.data; elem; elem = elem->next, i++) {
         g_assert_cmpint(elem->value, ==, (i % 3 == 0) ? 1 : 0);
     }
 
@@ -552,7 +673,6 @@
 {
     UserDefNativeListUnion *cvalue = NULL;
     strList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -568,12 +688,11 @@
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_STRING);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_STRING);
 
-    for (i = 0, elem = cvalue->string; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.string.data; elem; elem = elem->next, i++) {
         gchar str[8];
         sprintf(str, "%d", i);
         g_assert_cmpstr(elem->value, ==, str);
@@ -591,7 +710,6 @@
 {
     UserDefNativeListUnion *cvalue = NULL;
     numberList *elem = NULL;
-    Error *err = NULL;
     Visitor *v;
     GString *gstr_list = g_string_new("");
     GString *gstr_union = g_string_new("");
@@ -607,12 +725,11 @@
                            gstr_list->str);
     v = visitor_input_test_init_raw(data,  gstr_union->str);
 
-    visit_type_UserDefNativeListUnion(v, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(v, NULL, &cvalue, &error_abort);
     g_assert(cvalue != NULL);
-    g_assert_cmpint(cvalue->kind, ==, USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER);
+    g_assert_cmpint(cvalue->type, ==, USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER);
 
-    for (i = 0, elem = cvalue->number; elem; elem = elem->next, i++) {
+    for (i = 0, elem = cvalue->u.number.data; elem; elem = elem->next, i++) {
         GString *double_expected = g_string_new("");
         GString *double_actual = g_string_new("");
 
@@ -643,16 +760,75 @@
     TestStruct *p = NULL;
     Error *err = NULL;
     Visitor *v;
+    strList *q = NULL;
+    UserDefTwo *r = NULL;
+    WrapAlternate *s = NULL;
 
-    v = visitor_input_test_init(data, "{ 'integer': false, 'boolean': 'foo', 'string': -42 }");
+    v = visitor_input_test_init(data, "{ 'integer': false, 'boolean': 'foo', "
+                                "'string': -42 }");
 
-    visit_type_TestStruct(v, &p, NULL, &err);
-    g_assert(err);
-    g_assert(p->string == NULL);
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
 
-    error_free(err);
-    g_free(p->string);
-    g_free(p);
+    v = visitor_input_test_init(data, "[ '1', '2', false, '3' ]");
+    visit_type_strList(v, NULL, &q, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    v = visitor_input_test_init(data, "{ 'str':'hi' }");
+    visit_type_UserDefTwo(v, NULL, &r, &err);
+    error_free_or_abort(&err);
+    assert(!r);
+
+    v = visitor_input_test_init(data, "{ }");
+    visit_type_WrapAlternate(v, NULL, &s, &err);
+    error_free_or_abort(&err);
+    assert(!s);
+}
+
+static void test_visitor_in_wrong_type(TestInputVisitorData *data,
+                                       const void *unused)
+{
+    TestStruct *p = NULL;
+    Visitor *v;
+    strList *q = NULL;
+    int64_t i;
+    Error *err = NULL;
+
+    /* Make sure arrays and structs cannot be confused */
+
+    v = visitor_input_test_init(data, "[]");
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
+
+    v = visitor_input_test_init(data, "{}");
+    visit_type_strList(v, NULL, &q, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    /* Make sure primitives and struct cannot be confused */
+
+    v = visitor_input_test_init(data, "1");
+    visit_type_TestStruct(v, NULL, &p, &err);
+    error_free_or_abort(&err);
+    g_assert(!p);
+
+    v = visitor_input_test_init(data, "{}");
+    visit_type_int(v, NULL, &i, &err);
+    error_free_or_abort(&err);
+
+    /* Make sure primitives and arrays cannot be confused */
+
+    v = visitor_input_test_init(data, "1");
+    visit_type_strList(v, NULL, &q, &err);
+    error_free_or_abort(&err);
+    assert(!q);
+
+    v = visitor_input_test_init(data, "[]");
+    visit_type_int(v, NULL, &i, &err);
+    error_free_or_abort(&err);
 }
 
 int main(int argc, char **argv)
@@ -670,55 +846,64 @@
     input_visitor_test_add("/visitor/input/number",
                            &in_visitor_data, test_visitor_in_number);
     input_visitor_test_add("/visitor/input/string",
-                            &in_visitor_data, test_visitor_in_string);
+                           &in_visitor_data, test_visitor_in_string);
     input_visitor_test_add("/visitor/input/enum",
-                            &in_visitor_data, test_visitor_in_enum);
+                           &in_visitor_data, test_visitor_in_enum);
     input_visitor_test_add("/visitor/input/struct",
-                            &in_visitor_data, test_visitor_in_struct);
+                           &in_visitor_data, test_visitor_in_struct);
     input_visitor_test_add("/visitor/input/struct-nested",
-                            &in_visitor_data, test_visitor_in_struct_nested);
+                           &in_visitor_data, test_visitor_in_struct_nested);
     input_visitor_test_add("/visitor/input/list",
-                            &in_visitor_data, test_visitor_in_list);
-    input_visitor_test_add("/visitor/input/union",
-                            &in_visitor_data, test_visitor_in_union);
+                           &in_visitor_data, test_visitor_in_list);
+    input_visitor_test_add("/visitor/input/any",
+                           &in_visitor_data, test_visitor_in_any);
+    input_visitor_test_add("/visitor/input/null",
+                           &in_visitor_data, test_visitor_in_null);
     input_visitor_test_add("/visitor/input/union-flat",
-                            &in_visitor_data, test_visitor_in_union_flat);
-    input_visitor_test_add("/visitor/input/union-anon",
-                            &in_visitor_data, test_visitor_in_union_anon);
+                           &in_visitor_data, test_visitor_in_union_flat);
+    input_visitor_test_add("/visitor/input/alternate",
+                           &in_visitor_data, test_visitor_in_alternate);
     input_visitor_test_add("/visitor/input/errors",
-                            &in_visitor_data, test_visitor_in_errors);
+                           &in_visitor_data, test_visitor_in_errors);
+    input_visitor_test_add("/visitor/input/wrong-type",
+                           &in_visitor_data, test_visitor_in_wrong_type);
+    input_visitor_test_add("/visitor/input/alternate-number",
+                           &in_visitor_data, test_visitor_in_alternate_number);
     input_visitor_test_add("/visitor/input/native_list/int",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_int);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_int);
     input_visitor_test_add("/visitor/input/native_list/int8",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_int8);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_int8);
     input_visitor_test_add("/visitor/input/native_list/int16",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_int16);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_int16);
     input_visitor_test_add("/visitor/input/native_list/int32",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_int32);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_int32);
     input_visitor_test_add("/visitor/input/native_list/int64",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_int64);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_int64);
     input_visitor_test_add("/visitor/input/native_list/uint8",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_uint8);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_uint8);
     input_visitor_test_add("/visitor/input/native_list/uint16",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_uint16);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_uint16);
     input_visitor_test_add("/visitor/input/native_list/uint32",
-                            &in_visitor_data,
-                            test_visitor_in_native_list_uint32);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_uint32);
     input_visitor_test_add("/visitor/input/native_list/uint64",
-                            &in_visitor_data, test_visitor_in_native_list_uint64);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_uint64);
     input_visitor_test_add("/visitor/input/native_list/bool",
-                            &in_visitor_data, test_visitor_in_native_list_bool);
+                           &in_visitor_data, test_visitor_in_native_list_bool);
     input_visitor_test_add("/visitor/input/native_list/str",
-                            &in_visitor_data, test_visitor_in_native_list_string);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_string);
     input_visitor_test_add("/visitor/input/native_list/number",
-                            &in_visitor_data, test_visitor_in_native_list_number);
+                           &in_visitor_data,
+                           test_visitor_in_native_list_number);
 
     g_test_run();
 
diff --git a/tests/test-qmp-output-visitor.c b/tests/test-qmp-output-visitor.c
index 74020de..513d71f 100644
--- a/tests/test-qmp-output-visitor.c
+++ b/tests/test-qmp-output-visitor.c
@@ -1,7 +1,7 @@
 /*
  * QMP Output Visitor unit-tests.
  *
- * Copyright (C) 2011 Red Hat Inc.
+ * Copyright (C) 2011-2016 Red Hat Inc.
  *
  * Authors:
  *  Luiz Capitulino <lcapitulino@redhat.com>
@@ -10,197 +10,148 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/qmp-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
+#include "qapi/qmp/qjson.h"
 
 typedef struct TestOutputVisitorData {
-    QmpOutputVisitor *qov;
     Visitor *ov;
+    QObject *obj;
 } TestOutputVisitorData;
 
 static void visitor_output_setup(TestOutputVisitorData *data,
                                  const void *unused)
 {
-    data->qov = qmp_output_visitor_new();
-    g_assert(data->qov != NULL);
-
-    data->ov = qmp_output_get_visitor(data->qov);
-    g_assert(data->ov != NULL);
+    data->ov = qmp_output_visitor_new(&data->obj);
+    g_assert(data->ov);
 }
 
 static void visitor_output_teardown(TestOutputVisitorData *data,
                                     const void *unused)
 {
-    qmp_output_visitor_cleanup(data->qov);
-    data->qov = NULL;
+    visit_free(data->ov);
     data->ov = NULL;
+    qobject_decref(data->obj);
+    data->obj = NULL;
+}
+
+static QObject *visitor_get(TestOutputVisitorData *data)
+{
+    visit_complete(data->ov, &data->obj);
+    g_assert(data->obj);
+    return data->obj;
+}
+
+static void visitor_reset(TestOutputVisitorData *data)
+{
+    visitor_output_teardown(data, NULL);
+    visitor_output_setup(data, NULL);
 }
 
 static void test_visitor_out_int(TestOutputVisitorData *data,
                                  const void *unused)
 {
     int64_t value = -42;
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_int(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_int(data->ov, NULL, &value, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QINT);
     g_assert_cmpint(qint_get_int(qobject_to_qint(obj)), ==, value);
-
-    qobject_decref(obj);
 }
 
 static void test_visitor_out_bool(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     bool value = true;
     QObject *obj;
 
-    visit_type_bool(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_bool(data->ov, NULL, &value, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QBOOL);
-    g_assert(qbool_get_int(qobject_to_qbool(obj)) == value);
-
-    qobject_decref(obj);
+    g_assert(qbool_get_bool(qobject_to_qbool(obj)) == value);
 }
 
 static void test_visitor_out_number(TestOutputVisitorData *data,
                                     const void *unused)
 {
     double value = 3.14;
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_number(data->ov, &value, NULL, &err);
-    g_assert(!err);
+    visit_type_number(data->ov, NULL, &value, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QFLOAT);
     g_assert(qfloat_get_double(qobject_to_qfloat(obj)) == value);
-
-    qobject_decref(obj);
 }
 
 static void test_visitor_out_string(TestOutputVisitorData *data,
                                     const void *unused)
 {
     char *string = (char *) "Q E M U";
-    Error *err = NULL;
     QObject *obj;
 
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, NULL, &string, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QSTRING);
     g_assert_cmpstr(qstring_get_str(qobject_to_qstring(obj)), ==, string);
-
-    qobject_decref(obj);
 }
 
 static void test_visitor_out_no_string(TestOutputVisitorData *data,
                                        const void *unused)
 {
     char *string = NULL;
-    Error *err = NULL;
     QObject *obj;
 
     /* A null string should return "" */
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, NULL, &string, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QSTRING);
     g_assert_cmpstr(qstring_get_str(qobject_to_qstring(obj)), ==, "");
-
-    qobject_decref(obj);
 }
 
 static void test_visitor_out_enum(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     QObject *obj;
     EnumOne i;
 
-    for (i = 0; i < ENUM_ONE_MAX; i++) {
-        visit_type_EnumOne(data->ov, &i, "unused", &err);
-        g_assert(!err);
+    for (i = 0; i < ENUM_ONE__MAX; i++) {
+        visit_type_EnumOne(data->ov, "unused", &i, &error_abort);
 
-        obj = qmp_output_get_qobject(data->qov);
-        g_assert(obj != NULL);
+        obj = visitor_get(data);
         g_assert(qobject_type(obj) == QTYPE_QSTRING);
         g_assert_cmpstr(qstring_get_str(qobject_to_qstring(obj)), ==,
                         EnumOne_lookup[i]);
-        qobject_decref(obj);
+        visitor_reset(data);
     }
 }
 
 static void test_visitor_out_enum_errors(TestOutputVisitorData *data,
                                          const void *unused)
 {
-    EnumOne i, bad_values[] = { ENUM_ONE_MAX, -1 };
+    EnumOne i, bad_values[] = { ENUM_ONE__MAX, -1 };
     Error *err;
 
     for (i = 0; i < ARRAY_SIZE(bad_values) ; i++) {
         err = NULL;
-        visit_type_EnumOne(data->ov, &bad_values[i], "unused", &err);
+        visit_type_EnumOne(data->ov, "unused", &bad_values[i], &err);
         g_assert(err);
         error_free(err);
+        visitor_reset(data);
     }
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, "TestStruct", name, sizeof(TestStruct),
-                       &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static void test_visitor_out_struct(TestOutputVisitorData *data,
                                     const void *unused)
@@ -209,32 +160,26 @@
                                .boolean = false,
                                .string = (char *) "foo"};
     TestStruct *p = &test_struct;
-    Error *err = NULL;
     QObject *obj;
     QDict *qdict;
 
-    visit_type_TestStruct(data->ov, &p, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStruct(data->ov, NULL, &p, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QDICT);
 
     qdict = qobject_to_qdict(obj);
     g_assert_cmpint(qdict_size(qdict), ==, 3);
     g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 42);
-    g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, 0);
+    g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, false);
     g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, "foo");
-
-    QDECREF(qdict);
 }
 
 static void test_visitor_out_struct_nested(TestOutputVisitorData *data,
                                            const void *unused)
 {
     int64_t value = 42;
-    Error *err = NULL;
-    UserDefNested *ud2;
+    UserDefTwo *ud2;
     QObject *obj;
     QDict *qdict, *dict1, *dict2, *dict3, *userdef;
     const char *string = "user def string";
@@ -244,25 +189,25 @@
     ud2 = g_malloc0(sizeof(*ud2));
     ud2->string0 = g_strdup(strings[0]);
 
-    ud2->dict1.string1 = g_strdup(strings[1]);
-    ud2->dict1.dict2.userdef1 = g_malloc0(sizeof(UserDefOne));
-    ud2->dict1.dict2.userdef1->string = g_strdup(string);
-    ud2->dict1.dict2.userdef1->base = g_new0(UserDefZero, 1);
-    ud2->dict1.dict2.userdef1->base->integer = value;
-    ud2->dict1.dict2.string2 = g_strdup(strings[2]);
+    ud2->dict1 = g_malloc0(sizeof(*ud2->dict1));
+    ud2->dict1->string1 = g_strdup(strings[1]);
 
-    ud2->dict1.has_dict3 = true;
-    ud2->dict1.dict3.userdef2 = g_malloc0(sizeof(UserDefOne));
-    ud2->dict1.dict3.userdef2->string = g_strdup(string);
-    ud2->dict1.dict3.userdef2->base = g_new0(UserDefZero, 1);
-    ud2->dict1.dict3.userdef2->base->integer = value;
-    ud2->dict1.dict3.string3 = g_strdup(strings[3]);
+    ud2->dict1->dict2 = g_malloc0(sizeof(*ud2->dict1->dict2));
+    ud2->dict1->dict2->userdef = g_new0(UserDefOne, 1);
+    ud2->dict1->dict2->userdef->string = g_strdup(string);
+    ud2->dict1->dict2->userdef->integer = value;
+    ud2->dict1->dict2->string = g_strdup(strings[2]);
 
-    visit_type_UserDefNested(data->ov, &ud2, "unused", &err);
-    g_assert(!err);
+    ud2->dict1->dict3 = g_malloc0(sizeof(*ud2->dict1->dict3));
+    ud2->dict1->has_dict3 = true;
+    ud2->dict1->dict3->userdef = g_new0(UserDefOne, 1);
+    ud2->dict1->dict3->userdef->string = g_strdup(string);
+    ud2->dict1->dict3->userdef->integer = value;
+    ud2->dict1->dict3->string = g_strdup(strings[3]);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    visit_type_UserDefTwo(data->ov, "unused", &ud2, &error_abort);
+
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QDICT);
 
     qdict = qobject_to_qdict(obj);
@@ -275,30 +220,29 @@
 
     dict2 = qdict_get_qdict(dict1, "dict2");
     g_assert_cmpint(qdict_size(dict2), ==, 2);
-    g_assert_cmpstr(qdict_get_str(dict2, "string2"), ==, strings[2]);
-    userdef = qdict_get_qdict(dict2, "userdef1");
+    g_assert_cmpstr(qdict_get_str(dict2, "string"), ==, strings[2]);
+    userdef = qdict_get_qdict(dict2, "userdef");
     g_assert_cmpint(qdict_size(userdef), ==, 2);
     g_assert_cmpint(qdict_get_int(userdef, "integer"), ==, value);
     g_assert_cmpstr(qdict_get_str(userdef, "string"), ==, string);
 
     dict3 = qdict_get_qdict(dict1, "dict3");
     g_assert_cmpint(qdict_size(dict3), ==, 2);
-    g_assert_cmpstr(qdict_get_str(dict3, "string3"), ==, strings[3]);
-    userdef = qdict_get_qdict(dict3, "userdef2");
+    g_assert_cmpstr(qdict_get_str(dict3, "string"), ==, strings[3]);
+    userdef = qdict_get_qdict(dict3, "userdef");
     g_assert_cmpint(qdict_size(userdef), ==, 2);
     g_assert_cmpint(qdict_get_int(userdef, "integer"), ==, value);
     g_assert_cmpstr(qdict_get_str(userdef, "string"), ==, string);
 
-    QDECREF(qdict);
-    qapi_free_UserDefNested(ud2);
+    qapi_free_UserDefTwo(ud2);
 }
 
 static void test_visitor_out_struct_errors(TestOutputVisitorData *data,
                                            const void *unused)
 {
-    EnumOne bad_values[] = { ENUM_ONE_MAX, -1 };
-    UserDefZero b;
-    UserDefOne u = { .base = &b }, *pu = &u;
+    EnumOne bad_values[] = { ENUM_ONE__MAX, -1 };
+    UserDefOne u = {0};
+    UserDefOne *pu = &u;
     Error *err;
     int i;
 
@@ -306,71 +250,48 @@
         err = NULL;
         u.has_enum1 = true;
         u.enum1 = bad_values[i];
-        visit_type_UserDefOne(data->ov, &pu, "unused", &err);
+        visit_type_UserDefOne(data->ov, "unused", &pu, &err);
         g_assert(err);
         error_free(err);
+        visitor_reset(data);
     }
 }
 
-typedef struct TestStructList
-{
-    union {
-        TestStruct *value;
-        uint64_t padding;
-    };
-    struct TestStructList *next;
-} TestStructList;
-
-static void visit_type_TestStructList(Visitor *v, TestStructList **obj,
-                                      const char *name, Error **errp)
-{
-    GenericList *i, **head = (GenericList **)obj;
-
-    visit_start_list(v, name, errp);
-
-    for (*head = i = visit_next_list(v, head, errp); i; i = visit_next_list(v, &i, errp)) {
-        TestStructList *native_i = (TestStructList *)i;
-        visit_type_TestStruct(v, &native_i->value, NULL, errp);
-    }
-
-    visit_end_list(v, errp);
-}
 
 static void test_visitor_out_list(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    char *value_str = (char *) "list value";
+    const char *value_str = "list value";
     TestStructList *p, *head = NULL;
     const int max_items = 10;
     bool value_bool = true;
     int value_int = 10;
-    Error *err = NULL;
     QListEntry *entry;
     QObject *obj;
     QList *qlist;
     int i;
 
+    /* Build the list in reverse order... */
     for (i = 0; i < max_items; i++) {
         p = g_malloc0(sizeof(*p));
         p->value = g_malloc0(sizeof(*p->value));
-        p->value->integer = value_int;
+        p->value->integer = value_int + (max_items - i - 1);
         p->value->boolean = value_bool;
-        p->value->string = value_str;
+        p->value->string = g_strdup(value_str);
 
         p->next = head;
         head = p;
     }
 
-    visit_type_TestStructList(data->ov, &head, NULL, &err);
-    g_assert(!err);
+    visit_type_TestStructList(data->ov, NULL, &head, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    g_assert(obj != NULL);
+    obj = visitor_get(data);
     g_assert(qobject_type(obj) == QTYPE_QLIST);
 
     qlist = qobject_to_qlist(obj);
     g_assert(!qlist_empty(qlist));
 
+    /* ...and ensure that the visitor sees it in order */
     i = 0;
     QLIST_FOREACH_ENTRY(qlist, entry) {
         QDict *qdict;
@@ -378,27 +299,20 @@
         g_assert(qobject_type(entry->value) == QTYPE_QDICT);
         qdict = qobject_to_qdict(entry->value);
         g_assert_cmpint(qdict_size(qdict), ==, 3);
-        g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, value_int);
+        g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, value_int + i);
         g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, value_bool);
         g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, value_str);
         i++;
     }
     g_assert_cmpint(i, ==, max_items);
 
-    QDECREF(qlist);
-
-    for (p = head; p;) {
-        TestStructList *tmp = p->next;
-        g_free(p->value);
-        g_free(p);
-        p = tmp;
-    }
+    qapi_free_TestStructList(head);
 }
 
 static void test_visitor_out_list_qapi_free(TestOutputVisitorData *data,
                                             const void *unused)
 {
-    UserDefNestedList *p, *head = NULL;
+    UserDefTwoList *p, *head = NULL;
     const char string[] = "foo bar";
     int i, max_count = 1024;
 
@@ -407,53 +321,65 @@
         p->value = g_malloc0(sizeof(*p->value));
 
         p->value->string0 = g_strdup(string);
-        p->value->dict1.string1 = g_strdup(string);
-        p->value->dict1.dict2.userdef1 = g_malloc0(sizeof(UserDefOne));
-        p->value->dict1.dict2.userdef1->string = g_strdup(string);
-        p->value->dict1.dict2.userdef1->base = g_new0(UserDefZero, 1);
-        p->value->dict1.dict2.userdef1->base->integer = 42;
-        p->value->dict1.dict2.string2 = g_strdup(string);
-        p->value->dict1.has_dict3 = false;
+        p->value->dict1 = g_new0(UserDefTwoDict, 1);
+        p->value->dict1->string1 = g_strdup(string);
+        p->value->dict1->dict2 = g_new0(UserDefTwoDictDict, 1);
+        p->value->dict1->dict2->userdef = g_new0(UserDefOne, 1);
+        p->value->dict1->dict2->userdef->string = g_strdup(string);
+        p->value->dict1->dict2->userdef->integer = 42;
+        p->value->dict1->dict2->string = g_strdup(string);
+        p->value->dict1->has_dict3 = false;
 
         p->next = head;
         head = p;
     }
 
-    qapi_free_UserDefNestedList(head);
+    qapi_free_UserDefTwoList(head);
 }
 
-static void test_visitor_out_union(TestOutputVisitorData *data,
-                                   const void *unused)
+static void test_visitor_out_any(TestOutputVisitorData *data,
+                                 const void *unused)
 {
-    QObject *arg, *qvalue;
-    QDict *qdict, *value;
+    QObject *qobj;
+    QInt *qint;
+    QBool *qbool;
+    QString *qstring;
+    QDict *qdict;
+    QObject *obj;
 
-    Error *err = NULL;
+    qobj = QOBJECT(qint_from_int(-42));
+    visit_type_any(data->ov, NULL, &qobj, &error_abort);
+    obj = visitor_get(data);
+    g_assert(qobject_type(obj) == QTYPE_QINT);
+    g_assert_cmpint(qint_get_int(qobject_to_qint(obj)), ==, -42);
+    qobject_decref(qobj);
 
-    UserDefUnion *tmp = g_malloc0(sizeof(UserDefUnion));
-    tmp->kind = USER_DEF_UNION_KIND_A;
-    tmp->integer = 41;
-    tmp->a = g_malloc0(sizeof(UserDefA));
-    tmp->a->boolean = true;
-
-    visit_type_UserDefUnion(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    arg = qmp_output_get_qobject(data->qov);
-
-    g_assert(qobject_type(arg) == QTYPE_QDICT);
-    qdict = qobject_to_qdict(arg);
-
-    g_assert_cmpstr(qdict_get_str(qdict, "type"), ==, "a");
-    g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41);
-
-    qvalue = qdict_get(qdict, "data");
-    g_assert(data != NULL);
-    g_assert(qobject_type(qvalue) == QTYPE_QDICT);
-    value = qobject_to_qdict(qvalue);
-    g_assert_cmpint(qdict_get_bool(value, "boolean"), ==, true);
-
-    qapi_free_UserDefUnion(tmp);
-    QDECREF(qdict);
+    visitor_reset(data);
+    qdict = qdict_new();
+    qdict_put(qdict, "integer", qint_from_int(-42));
+    qdict_put(qdict, "boolean", qbool_from_bool(true));
+    qdict_put(qdict, "string", qstring_from_str("foo"));
+    qobj = QOBJECT(qdict);
+    visit_type_any(data->ov, NULL, &qobj, &error_abort);
+    qobject_decref(qobj);
+    obj = visitor_get(data);
+    qdict = qobject_to_qdict(obj);
+    g_assert(qdict);
+    qobj = qdict_get(qdict, "integer");
+    g_assert(qobj);
+    qint = qobject_to_qint(qobj);
+    g_assert(qint);
+    g_assert_cmpint(qint_get_int(qint), ==, -42);
+    qobj = qdict_get(qdict, "boolean");
+    g_assert(qobj);
+    qbool = qobject_to_qbool(qobj);
+    g_assert(qbool);
+    g_assert(qbool_get_bool(qbool) == true);
+    qobj = qdict_get(qdict, "string");
+    g_assert(qobj);
+    qstring = qobject_to_qstring(qobj);
+    g_assert(qstring);
+    g_assert_cmpstr(qstring_get_str(qstring), ==, "foo");
 }
 
 static void test_visitor_out_union_flat(TestOutputVisitorData *data,
@@ -462,66 +388,106 @@
     QObject *arg;
     QDict *qdict;
 
-    Error *err = NULL;
-
     UserDefFlatUnion *tmp = g_malloc0(sizeof(UserDefFlatUnion));
-    tmp->kind = ENUM_ONE_VALUE1;
+    tmp->enum1 = ENUM_ONE_VALUE1;
     tmp->string = g_strdup("str");
-    tmp->value1 = g_malloc0(sizeof(UserDefA));
-    /* TODO when generator bug is fixed: tmp->integer = 41; */
-    tmp->value1->boolean = true;
+    tmp->integer = 41;
+    tmp->u.value1.boolean = true;
 
-    visit_type_UserDefFlatUnion(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    arg = qmp_output_get_qobject(data->qov);
+    visit_type_UserDefFlatUnion(data->ov, NULL, &tmp, &error_abort);
+    arg = visitor_get(data);
 
     g_assert(qobject_type(arg) == QTYPE_QDICT);
     qdict = qobject_to_qdict(arg);
 
     g_assert_cmpstr(qdict_get_str(qdict, "enum1"), ==, "value1");
     g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, "str");
-    /* TODO g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41); */
+    g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 41);
     g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, true);
 
     qapi_free_UserDefFlatUnion(tmp);
-    QDECREF(qdict);
 }
 
-static void test_visitor_out_union_anon(TestOutputVisitorData *data,
-                                        const void *unused)
+static void test_visitor_out_alternate(TestOutputVisitorData *data,
+                                       const void *unused)
 {
     QObject *arg;
-    Error *err = NULL;
+    UserDefAlternate *tmp;
+    QDict *qdict;
 
-    UserDefAnonUnion *tmp = g_malloc0(sizeof(UserDefAnonUnion));
-    tmp->kind = USER_DEF_ANON_UNION_KIND_I;
-    tmp->i = 42;
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = QTYPE_QINT;
+    tmp->u.i = 42;
 
-    visit_type_UserDefAnonUnion(data->ov, &tmp, NULL, &err);
-    g_assert(err == NULL);
-    arg = qmp_output_get_qobject(data->qov);
+    visit_type_UserDefAlternate(data->ov, NULL, &tmp, &error_abort);
+    arg = visitor_get(data);
 
     g_assert(qobject_type(arg) == QTYPE_QINT);
     g_assert_cmpint(qint_get_int(qobject_to_qint(arg)), ==, 42);
 
-    qapi_free_UserDefAnonUnion(tmp);
+    qapi_free_UserDefAlternate(tmp);
+
+    visitor_reset(data);
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = QTYPE_QSTRING;
+    tmp->u.s = g_strdup("hello");
+
+    visit_type_UserDefAlternate(data->ov, NULL, &tmp, &error_abort);
+    arg = visitor_get(data);
+
+    g_assert(qobject_type(arg) == QTYPE_QSTRING);
+    g_assert_cmpstr(qstring_get_str(qobject_to_qstring(arg)), ==, "hello");
+
+    qapi_free_UserDefAlternate(tmp);
+
+    visitor_reset(data);
+    tmp = g_new0(UserDefAlternate, 1);
+    tmp->type = QTYPE_QDICT;
+    tmp->u.udfu.integer = 1;
+    tmp->u.udfu.string = g_strdup("str");
+    tmp->u.udfu.enum1 = ENUM_ONE_VALUE1;
+    tmp->u.udfu.u.value1.boolean = true;
+
+    visit_type_UserDefAlternate(data->ov, NULL, &tmp, &error_abort);
+    arg = visitor_get(data);
+
+    g_assert_cmpint(qobject_type(arg), ==, QTYPE_QDICT);
+    qdict = qobject_to_qdict(arg);
+    g_assert_cmpint(qdict_size(qdict), ==, 4);
+    g_assert_cmpint(qdict_get_int(qdict, "integer"), ==, 1);
+    g_assert_cmpstr(qdict_get_str(qdict, "string"), ==, "str");
+    g_assert_cmpstr(qdict_get_str(qdict, "enum1"), ==, "value1");
+    g_assert_cmpint(qdict_get_bool(qdict, "boolean"), ==, true);
+
+    qapi_free_UserDefAlternate(tmp);
 }
 
-static void test_visitor_out_empty(TestOutputVisitorData *data,
-                                   const void *unused)
+static void test_visitor_out_null(TestOutputVisitorData *data,
+                                  const void *unused)
 {
     QObject *arg;
+    QDict *qdict;
+    QObject *nil;
 
-    arg = qmp_output_get_qobject(data->qov);
-    g_assert(!arg);
+    visit_start_struct(data->ov, NULL, NULL, 0, &error_abort);
+    visit_type_null(data->ov, "a", &error_abort);
+    visit_check_struct(data->ov, &error_abort);
+    visit_end_struct(data->ov, NULL);
+    arg = visitor_get(data);
+    g_assert(qobject_type(arg) == QTYPE_QDICT);
+    qdict = qobject_to_qdict(arg);
+    g_assert_cmpint(qdict_size(qdict), ==, 1);
+    nil = qdict_get(qdict, "a");
+    g_assert(nil);
+    g_assert(qobject_type(nil) == QTYPE_QNULL);
 }
 
 static void init_native_list(UserDefNativeListUnion *cvalue)
 {
     int i;
-    switch (cvalue->kind) {
+    switch (cvalue->type) {
     case USER_DEF_NATIVE_LIST_UNION_KIND_INTEGER: {
-        intList **list = &cvalue->integer;
+        intList **list = &cvalue->u.integer.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(intList, 1);
             (*list)->value = i;
@@ -531,7 +497,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S8: {
-        int8List **list = &cvalue->s8;
+        int8List **list = &cvalue->u.s8.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int8List, 1);
             (*list)->value = i;
@@ -541,7 +507,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S16: {
-        int16List **list = &cvalue->s16;
+        int16List **list = &cvalue->u.s16.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int16List, 1);
             (*list)->value = i;
@@ -551,7 +517,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S32: {
-        int32List **list = &cvalue->s32;
+        int32List **list = &cvalue->u.s32.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int32List, 1);
             (*list)->value = i;
@@ -561,7 +527,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_S64: {
-        int64List **list = &cvalue->s64;
+        int64List **list = &cvalue->u.s64.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(int64List, 1);
             (*list)->value = i;
@@ -571,7 +537,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U8: {
-        uint8List **list = &cvalue->u8;
+        uint8List **list = &cvalue->u.u8.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint8List, 1);
             (*list)->value = i;
@@ -581,7 +547,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U16: {
-        uint16List **list = &cvalue->u16;
+        uint16List **list = &cvalue->u.u16.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint16List, 1);
             (*list)->value = i;
@@ -591,7 +557,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U32: {
-        uint32List **list = &cvalue->u32;
+        uint32List **list = &cvalue->u.u32.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint32List, 1);
             (*list)->value = i;
@@ -601,7 +567,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_U64: {
-        uint64List **list = &cvalue->u64;
+        uint64List **list = &cvalue->u.u64.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(uint64List, 1);
             (*list)->value = i;
@@ -611,7 +577,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_BOOLEAN: {
-        boolList **list = &cvalue->boolean;
+        boolList **list = &cvalue->u.boolean.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(boolList, 1);
             (*list)->value = (i % 3 == 0);
@@ -621,7 +587,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_STRING: {
-        strList **list = &cvalue->string;
+        strList **list = &cvalue->u.string.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(strList, 1);
             (*list)->value = g_strdup_printf("%d", i);
@@ -631,7 +597,7 @@
         break;
     }
     case USER_DEF_NATIVE_LIST_UNION_KIND_NUMBER: {
-        numberList **list = &cvalue->number;
+        numberList **list = &cvalue->u.number.data;
         for (i = 0; i < 32; i++) {
             *list = g_new0(numberList, 1);
             (*list)->value = (double)i / 3;
@@ -690,7 +656,7 @@
             tmp = qlist_peek(qlist);
             g_assert(tmp);
             qvalue = qobject_to_qbool(tmp);
-            g_assert_cmpint(qbool_get_int(qvalue), ==, (i % 3 == 0) ? 1 : 0);
+            g_assert_cmpint(qbool_get_bool(qvalue), ==, i % 3 == 0);
             qobject_decref(qlist_pop(qlist));
         }
         break;
@@ -737,19 +703,16 @@
                              UserDefNativeListUnionKind kind)
 {
     UserDefNativeListUnion *cvalue = g_new0(UserDefNativeListUnion, 1);
-    Error *err = NULL;
     QObject *obj;
 
-    cvalue->kind = kind;
+    cvalue->type = kind;
     init_native_list(cvalue);
 
-    visit_type_UserDefNativeListUnion(data->ov, &cvalue, NULL, &err);
-    g_assert(err == NULL);
+    visit_type_UserDefNativeListUnion(data->ov, NULL, &cvalue, &error_abort);
 
-    obj = qmp_output_get_qobject(data->qov);
-    check_native_list(obj, cvalue->kind);
+    obj = visitor_get(data);
+    check_native_list(obj, cvalue->type);
     qapi_free_UserDefNativeListUnion(cvalue);
-    qobject_decref(obj);
 }
 
 static void test_visitor_out_native_list_int(TestOutputVisitorData *data,
@@ -860,40 +823,52 @@
                             &out_visitor_data, test_visitor_out_struct_errors);
     output_visitor_test_add("/visitor/output/list",
                             &out_visitor_data, test_visitor_out_list);
+    output_visitor_test_add("/visitor/output/any",
+                            &out_visitor_data, test_visitor_out_any);
     output_visitor_test_add("/visitor/output/list-qapi-free",
                             &out_visitor_data, test_visitor_out_list_qapi_free);
-    output_visitor_test_add("/visitor/output/union",
-                            &out_visitor_data, test_visitor_out_union);
     output_visitor_test_add("/visitor/output/union-flat",
                             &out_visitor_data, test_visitor_out_union_flat);
-    output_visitor_test_add("/visitor/output/union-anon",
-                            &out_visitor_data, test_visitor_out_union_anon);
-    output_visitor_test_add("/visitor/output/empty",
-                            &out_visitor_data, test_visitor_out_empty);
+    output_visitor_test_add("/visitor/output/alternate",
+                            &out_visitor_data, test_visitor_out_alternate);
+    output_visitor_test_add("/visitor/output/null",
+                            &out_visitor_data, test_visitor_out_null);
     output_visitor_test_add("/visitor/output/native_list/int",
-                            &out_visitor_data, test_visitor_out_native_list_int);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_int);
     output_visitor_test_add("/visitor/output/native_list/int8",
-                            &out_visitor_data, test_visitor_out_native_list_int8);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_int8);
     output_visitor_test_add("/visitor/output/native_list/int16",
-                            &out_visitor_data, test_visitor_out_native_list_int16);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_int16);
     output_visitor_test_add("/visitor/output/native_list/int32",
-                            &out_visitor_data, test_visitor_out_native_list_int32);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_int32);
     output_visitor_test_add("/visitor/output/native_list/int64",
-                            &out_visitor_data, test_visitor_out_native_list_int64);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_int64);
     output_visitor_test_add("/visitor/output/native_list/uint8",
-                            &out_visitor_data, test_visitor_out_native_list_uint8);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_uint8);
     output_visitor_test_add("/visitor/output/native_list/uint16",
-                            &out_visitor_data, test_visitor_out_native_list_uint16);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_uint16);
     output_visitor_test_add("/visitor/output/native_list/uint32",
-                            &out_visitor_data, test_visitor_out_native_list_uint32);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_uint32);
     output_visitor_test_add("/visitor/output/native_list/uint64",
-                            &out_visitor_data, test_visitor_out_native_list_uint64);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_uint64);
     output_visitor_test_add("/visitor/output/native_list/bool",
-                            &out_visitor_data, test_visitor_out_native_list_bool);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_bool);
     output_visitor_test_add("/visitor/output/native_list/string",
-                            &out_visitor_data, test_visitor_out_native_list_str);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_str);
     output_visitor_test_add("/visitor/output/native_list/number",
-                            &out_visitor_data, test_visitor_out_native_list_number);
+                            &out_visitor_data,
+                            test_visitor_out_native_list_number);
 
     g_test_run();
 
diff --git a/tests/test-rcu-list.c b/tests/test-rcu-list.c
new file mode 100644
index 0000000..1514d7e
--- /dev/null
+++ b/tests/test-rcu-list.c
@@ -0,0 +1,311 @@
+/*
+ * rcuq_test.c
+ *
+ * usage: rcuq_test <readers> <duration>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (c) 2013 Mike D. Day, IBM Corporation.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/atomic.h"
+#include "qemu/rcu.h"
+#include "qemu/thread.h"
+#include "qemu/rcu_queue.h"
+
+/*
+ * Test variables.
+ */
+
+static QemuMutex counts_mutex;
+static long long n_reads = 0LL;
+static long long n_updates = 0LL;
+static long long n_reclaims = 0LL;
+static long long n_nodes_removed = 0LL;
+static long long n_nodes = 0LL;
+static int g_test_in_charge = 0;
+
+static int nthreadsrunning;
+
+#define GOFLAG_INIT 0
+#define GOFLAG_RUN  1
+#define GOFLAG_STOP 2
+
+static volatile int goflag = GOFLAG_INIT;
+
+#define RCU_READ_RUN 1000
+#define RCU_UPDATE_RUN 10
+#define NR_THREADS 100
+#define RCU_Q_LEN 100
+
+static QemuThread threads[NR_THREADS];
+static struct rcu_reader_data *data[NR_THREADS];
+static int n_threads;
+
+static int select_random_el(int max)
+{
+    return (rand() % max);
+}
+
+
+static void create_thread(void *(*func)(void *))
+{
+    if (n_threads >= NR_THREADS) {
+        fprintf(stderr, "Thread limit of %d exceeded!\n", NR_THREADS);
+        exit(-1);
+    }
+    qemu_thread_create(&threads[n_threads], "test", func, &data[n_threads],
+                       QEMU_THREAD_JOINABLE);
+    n_threads++;
+}
+
+static void wait_all_threads(void)
+{
+    int i;
+
+    for (i = 0; i < n_threads; i++) {
+        qemu_thread_join(&threads[i]);
+    }
+    n_threads = 0;
+}
+
+
+struct list_element {
+    QLIST_ENTRY(list_element) entry;
+    struct rcu_head rcu;
+};
+
+static void reclaim_list_el(struct rcu_head *prcu)
+{
+    struct list_element *el = container_of(prcu, struct list_element, rcu);
+    g_free(el);
+    /* Accessed only from call_rcu thread.  */
+    n_reclaims++;
+}
+
+static QLIST_HEAD(q_list_head, list_element) Q_list_head;
+
+static void *rcu_q_reader(void *arg)
+{
+    long long n_reads_local = 0;
+    struct list_element *el;
+
+    rcu_register_thread();
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    atomic_inc(&nthreadsrunning);
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+
+    while (goflag == GOFLAG_RUN) {
+        rcu_read_lock();
+        QLIST_FOREACH_RCU(el, &Q_list_head, entry) {
+            n_reads_local++;
+            if (goflag == GOFLAG_STOP) {
+                break;
+            }
+        }
+        rcu_read_unlock();
+
+        g_usleep(100);
+    }
+    qemu_mutex_lock(&counts_mutex);
+    n_reads += n_reads_local;
+    qemu_mutex_unlock(&counts_mutex);
+
+    rcu_unregister_thread();
+    return NULL;
+}
+
+
+static void *rcu_q_updater(void *arg)
+{
+    int j, target_el;
+    long long n_nodes_local = 0;
+    long long n_updates_local = 0;
+    long long n_removed_local = 0;
+    struct list_element *el, *prev_el;
+
+    *(struct rcu_reader_data **)arg = &rcu_reader;
+    atomic_inc(&nthreadsrunning);
+    while (goflag == GOFLAG_INIT) {
+        g_usleep(1000);
+    }
+
+    while (goflag == GOFLAG_RUN) {
+        target_el = select_random_el(RCU_Q_LEN);
+        j = 0;
+        /* FOREACH_RCU could work here but let's use both macros */
+        QLIST_FOREACH_SAFE_RCU(prev_el, &Q_list_head, entry, el) {
+            j++;
+            if (target_el == j) {
+                QLIST_REMOVE_RCU(prev_el, entry);
+                /* may be more than one updater in the future */
+                call_rcu1(&prev_el->rcu, reclaim_list_el);
+                n_removed_local++;
+                break;
+            }
+        }
+        if (goflag == GOFLAG_STOP) {
+            break;
+        }
+        target_el = select_random_el(RCU_Q_LEN);
+        j = 0;
+        QLIST_FOREACH_RCU(el, &Q_list_head, entry) {
+            j++;
+            if (target_el == j) {
+                prev_el = g_new(struct list_element, 1);
+                n_nodes += n_nodes_local;
+                QLIST_INSERT_BEFORE_RCU(el, prev_el, entry);
+                break;
+            }
+        }
+
+        n_updates_local += 2;
+        synchronize_rcu();
+    }
+    synchronize_rcu();
+    qemu_mutex_lock(&counts_mutex);
+    n_nodes += n_nodes_local;
+    n_updates += n_updates_local;
+    n_nodes_removed += n_removed_local;
+    qemu_mutex_unlock(&counts_mutex);
+    return NULL;
+}
+
+static void rcu_qtest_init(void)
+{
+    struct list_element *new_el;
+    int i;
+    nthreadsrunning = 0;
+    srand(time(0));
+    for (i = 0; i < RCU_Q_LEN; i++) {
+        new_el = g_new(struct list_element, 1);
+        QLIST_INSERT_HEAD_RCU(&Q_list_head, new_el, entry);
+    }
+    qemu_mutex_lock(&counts_mutex);
+    n_nodes += RCU_Q_LEN;
+    qemu_mutex_unlock(&counts_mutex);
+}
+
+static void rcu_qtest_run(int duration, int nreaders)
+{
+    int nthreads = nreaders + 1;
+    while (atomic_read(&nthreadsrunning) < nthreads) {
+        g_usleep(1000);
+    }
+
+    goflag = GOFLAG_RUN;
+    sleep(duration);
+    goflag = GOFLAG_STOP;
+    wait_all_threads();
+}
+
+
+static void rcu_qtest(const char *test, int duration, int nreaders)
+{
+    int i;
+    long long n_removed_local = 0;
+
+    struct list_element *el, *prev_el;
+
+    rcu_qtest_init();
+    for (i = 0; i < nreaders; i++) {
+        create_thread(rcu_q_reader);
+    }
+    create_thread(rcu_q_updater);
+    rcu_qtest_run(duration, nreaders);
+
+    QLIST_FOREACH_SAFE_RCU(prev_el, &Q_list_head, entry, el) {
+        QLIST_REMOVE_RCU(prev_el, entry);
+        call_rcu1(&prev_el->rcu, reclaim_list_el);
+        n_removed_local++;
+    }
+    qemu_mutex_lock(&counts_mutex);
+    n_nodes_removed += n_removed_local;
+    qemu_mutex_unlock(&counts_mutex);
+    synchronize_rcu();
+    while (n_nodes_removed > n_reclaims) {
+        g_usleep(100);
+        synchronize_rcu();
+    }
+    if (g_test_in_charge) {
+        g_assert_cmpint(n_nodes_removed, ==, n_reclaims);
+    } else {
+        printf("%s: %d readers; 1 updater; nodes read: "  \
+               "%lld, nodes removed: %lld; nodes reclaimed: %lld\n",
+               test, nthreadsrunning - 1, n_reads, n_nodes_removed, n_reclaims);
+        exit(0);
+    }
+}
+
+static void usage(int argc, char *argv[])
+{
+    fprintf(stderr, "Usage: %s duration nreaders\n", argv[0]);
+    exit(-1);
+}
+
+static int gtest_seconds;
+
+static void gtest_rcuq_one(void)
+{
+    rcu_qtest("rcuqtest", gtest_seconds / 4, 1);
+}
+
+static void gtest_rcuq_few(void)
+{
+    rcu_qtest("rcuqtest", gtest_seconds / 4, 5);
+}
+
+static void gtest_rcuq_many(void)
+{
+    rcu_qtest("rcuqtest", gtest_seconds / 2, 20);
+}
+
+
+int main(int argc, char *argv[])
+{
+    int duration = 0, readers = 0;
+
+    qemu_mutex_init(&counts_mutex);
+    if (argc >= 2) {
+        if (argv[1][0] == '-') {
+            g_test_init(&argc, &argv, NULL);
+            if (g_test_quick()) {
+                gtest_seconds = 4;
+            } else {
+                gtest_seconds = 20;
+            }
+            g_test_add_func("/rcu/qlist/single-threaded", gtest_rcuq_one);
+            g_test_add_func("/rcu/qlist/short-few", gtest_rcuq_few);
+            g_test_add_func("/rcu/qlist/long-many", gtest_rcuq_many);
+            g_test_in_charge = 1;
+            return g_test_run();
+        }
+        duration = strtoul(argv[1], NULL, 0);
+    }
+    if (argc >= 3) {
+        readers = strtoul(argv[2], NULL, 0);
+    }
+    if (duration && readers) {
+        rcu_qtest(argv[0], duration, readers);
+        return 0;
+    }
+
+    usage(argc, argv);
+    return -1;
+}
diff --git a/tests/test-rfifolock.c b/tests/test-rfifolock.c
index 0572ebb..471a811 100644
--- a/tests/test-rfifolock.c
+++ b/tests/test-rfifolock.c
@@ -10,7 +10,7 @@
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/rfifolock.h"
 
diff --git a/tests/test-string-input-visitor.c b/tests/test-string-input-visitor.c
index 8e3433e..d837ebe 100644
--- a/tests/test-string-input-visitor.c
+++ b/tests/test-string-input-visitor.c
@@ -10,25 +10,25 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/string-input-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
 
 typedef struct TestInputVisitorData {
-    StringInputVisitor *siv;
+    Visitor *v;
 } TestInputVisitorData;
 
 static void visitor_input_teardown(TestInputVisitorData *data,
                                    const void *unused)
 {
-    if (data->siv) {
-        string_input_visitor_cleanup(data->siv);
-        data->siv = NULL;
+    if (data->v) {
+        visit_free(data->v);
+        data->v = NULL;
     }
 }
 
@@ -39,15 +39,9 @@
 Visitor *visitor_input_test_init(TestInputVisitorData *data,
                                  const char *string)
 {
-    Visitor *v;
-
-    data->siv = string_input_visitor_new(string);
-    g_assert(data->siv != NULL);
-
-    v = string_input_get_visitor(data->siv);
-    g_assert(v != NULL);
-
-    return v;
+    data->v = string_input_visitor_new(string);
+    g_assert(data->v);
+    return data->v;
 }
 
 static void test_visitor_in_int(TestInputVisitorData *data,
@@ -59,9 +53,16 @@
 
     v = visitor_input_test_init(data, "-42");
 
-    visit_type_int(v, &res, NULL, &err);
+    visit_type_int(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, value);
+
+    visitor_input_teardown(data, unused);
+
+    v = visitor_input_test_init(data, "not an int");
+
+    visit_type_int(v, NULL, &res, &err);
+    error_free_or_abort(&err);
 }
 
 static void test_visitor_in_intList(TestInputVisitorData *data,
@@ -69,12 +70,13 @@
 {
     int64_t value[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 20};
     int16List *res = NULL, *tmp;
+    Error *err = NULL;
     Visitor *v;
     int i = 0;
 
     v = visitor_input_test_init(data, "1,2,0,2-4,20,5-9,1-8");
 
-    visit_type_int16List(v, &res, NULL, &error_abort);
+    visit_type_int16List(v, NULL, &res, &error_abort);
     tmp = res;
     while (i < sizeof(value) / sizeof(value[0])) {
         g_assert(tmp);
@@ -83,12 +85,15 @@
     }
     g_assert(!tmp);
 
-    tmp = res;
-    while (tmp) {
-        res = res->next;
-        g_free(tmp);
-        tmp = res;
-    }
+    qapi_free_int16List(res);
+
+    visitor_input_teardown(data, unused);
+
+    v = visitor_input_test_init(data, "not an int list");
+
+    visit_type_int16List(v, NULL, &res, &err);
+    error_free_or_abort(&err);
+    g_assert(!res);
 }
 
 static void test_visitor_in_bool(TestInputVisitorData *data,
@@ -100,42 +105,42 @@
 
     v = visitor_input_test_init(data, "true");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, true);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "yes");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, true);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "on");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, true);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "false");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, false);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "no");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, false);
     visitor_input_teardown(data, unused);
 
     v = visitor_input_test_init(data, "off");
 
-    visit_type_bool(v, &res, NULL, &err);
+    visit_type_bool(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpint(res, ==, false);
 }
@@ -149,7 +154,7 @@
 
     v = visitor_input_test_init(data, "3.14");
 
-    visit_type_number(v, &res, NULL, &err);
+    visit_type_number(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpfloat(res, ==, value);
 }
@@ -163,7 +168,7 @@
 
     v = visitor_input_test_init(data, value);
 
-    visit_type_str(v, &res, NULL, &err);
+    visit_type_str(v, NULL, &res, &err);
     g_assert(!err);
     g_assert_cmpstr(res, ==, value);
 
@@ -182,14 +187,12 @@
 
         v = visitor_input_test_init(data, EnumOne_lookup[i]);
 
-        visit_type_EnumOne(v, &res, NULL, &err);
+        visit_type_EnumOne(v, NULL, &res, &err);
         g_assert(!err);
         g_assert_cmpint(i, ==, res);
 
         visitor_input_teardown(data, NULL);
     }
-
-    data->siv = NULL;
 }
 
 /* Try to crash the visitors */
@@ -220,29 +223,29 @@
         }
 
         v = visitor_input_test_init(data, buf);
-        visit_type_int(v, &ires, NULL, NULL);
+        visit_type_int(v, NULL, &ires, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_intList(v, &ilres, NULL, NULL);
+        visit_type_intList(v, NULL, &ilres, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_bool(v, &bres, NULL, NULL);
+        visit_type_bool(v, NULL, &bres, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_number(v, &nres, NULL, NULL);
+        visit_type_number(v, NULL, &nres, NULL);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
         sres = NULL;
-        visit_type_str(v, &sres, NULL, NULL);
+        visit_type_str(v, NULL, &sres, NULL);
         g_free(sres);
         visitor_input_teardown(data, NULL);
 
         v = visitor_input_test_init(data, buf);
-        visit_type_EnumOne(v, &eres, NULL, NULL);
+        visit_type_EnumOne(v, NULL, &eres, NULL);
         visitor_input_teardown(data, NULL);
     }
 }
diff --git a/tests/test-string-output-visitor.c b/tests/test-string-output-visitor.c
index 101fb27..444844a 100644
--- a/tests/test-string-output-visitor.c
+++ b/tests/test-string-output-visitor.c
@@ -10,48 +10,63 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
+#include "qapi/error.h"
 #include "qapi/string-output-visitor.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
 #include "qapi/qmp/types.h"
 
 typedef struct TestOutputVisitorData {
-    StringOutputVisitor *sov;
     Visitor *ov;
+    char *str;
     bool human;
 } TestOutputVisitorData;
 
+static void visitor_output_setup_internal(TestOutputVisitorData *data,
+                                          bool human)
+{
+    data->human = human;
+    data->ov = string_output_visitor_new(human, &data->str);
+    g_assert(data->ov);
+}
+
 static void visitor_output_setup(TestOutputVisitorData *data,
                                  const void *unused)
 {
-    data->human = false;
-    data->sov = string_output_visitor_new(data->human);
-    g_assert(data->sov != NULL);
-
-    data->ov = string_output_get_visitor(data->sov);
-    g_assert(data->ov != NULL);
+    return visitor_output_setup_internal(data, false);
 }
 
 static void visitor_output_setup_human(TestOutputVisitorData *data,
                                        const void *unused)
 {
-    data->human = true;
-    data->sov = string_output_visitor_new(data->human);
-    g_assert(data->sov != NULL);
-
-    data->ov = string_output_get_visitor(data->sov);
-    g_assert(data->ov != NULL);
+    return visitor_output_setup_internal(data, true);
 }
 
 static void visitor_output_teardown(TestOutputVisitorData *data,
                                     const void *unused)
 {
-    string_output_visitor_cleanup(data->sov);
-    data->sov = NULL;
+    visit_free(data->ov);
     data->ov = NULL;
+    g_free(data->str);
+    data->str = NULL;
+}
+
+static char *visitor_get(TestOutputVisitorData *data)
+{
+    visit_complete(data->ov, &data->str);
+    g_assert(data->str);
+    return data->str;
+}
+
+static void visitor_reset(TestOutputVisitorData *data)
+{
+    bool human = data->human;
+
+    visitor_output_teardown(data, NULL);
+    visitor_output_setup_internal(data, human);
 }
 
 static void test_visitor_out_int(TestOutputVisitorData *data,
@@ -61,17 +76,15 @@
     Error *err = NULL;
     char *str;
 
-    visit_type_int(data->ov, &value, NULL, &err);
+    visit_type_int(data->ov, NULL, &value, &err);
     g_assert(!err);
 
-    str = string_output_get_string(data->sov);
-    g_assert(str != NULL);
+    str = visitor_get(data);
     if (data->human) {
         g_assert_cmpstr(str, ==, "42 (0x2a)");
     } else {
         g_assert_cmpstr(str, ==, "42");
     }
-    g_free(str);
 }
 
 static void test_visitor_out_intList(TestOutputVisitorData *data,
@@ -81,7 +94,7 @@
         3, 4, 5, 6, 11, 12, 13, 21, 22, INT64_MAX - 1, INT64_MAX};
     intList *list = NULL, **tmp = &list;
     int i;
-    Error *errp = NULL;
+    Error *err = NULL;
     char *str;
 
     for (i = 0; i < sizeof(value) / sizeof(value[0]); i++) {
@@ -90,11 +103,10 @@
         tmp = &(*tmp)->next;
     }
 
-    visit_type_intList(data->ov, &list, NULL, &errp);
-    g_assert(errp == NULL);
+    visit_type_intList(data->ov, NULL, &list, &err);
+    g_assert(err == NULL);
 
-    str = string_output_get_string(data->sov);
-    g_assert(str != NULL);
+    str = visitor_get(data);
     if (data->human) {
         g_assert_cmpstr(str, ==,
             "0-1,3-6,9-16,21-22,9223372036854775806-9223372036854775807 "
@@ -104,13 +116,7 @@
         g_assert_cmpstr(str, ==,
             "0-1,3-6,9-16,21-22,9223372036854775806-9223372036854775807");
     }
-    g_free(str);
-    while (list) {
-        intList *tmp2;
-        tmp2 = list->next;
-        g_free(list);
-        list = tmp2;
-    }
+    qapi_free_intList(list);
 }
 
 static void test_visitor_out_bool(TestOutputVisitorData *data,
@@ -120,13 +126,11 @@
     bool value = true;
     char *str;
 
-    visit_type_bool(data->ov, &value, NULL, &err);
+    visit_type_bool(data->ov, NULL, &value, &err);
     g_assert(!err);
 
-    str = string_output_get_string(data->sov);
-    g_assert(str != NULL);
+    str = visitor_get(data);
     g_assert_cmpstr(str, ==, "true");
-    g_free(str);
 }
 
 static void test_visitor_out_number(TestOutputVisitorData *data,
@@ -136,13 +140,11 @@
     Error *err = NULL;
     char *str;
 
-    visit_type_number(data->ov, &value, NULL, &err);
+    visit_type_number(data->ov, NULL, &value, &err);
     g_assert(!err);
 
-    str = string_output_get_string(data->sov);
-    g_assert(str != NULL);
+    str = visitor_get(data);
     g_assert_cmpstr(str, ==, "3.140000");
-    g_free(str);
 }
 
 static void test_visitor_out_string(TestOutputVisitorData *data,
@@ -153,78 +155,66 @@
     Error *err = NULL;
     char *str;
 
-    visit_type_str(data->ov, &string, NULL, &err);
+    visit_type_str(data->ov, NULL, &string, &err);
     g_assert(!err);
 
-    str = string_output_get_string(data->sov);
-    g_assert(str != NULL);
+    str = visitor_get(data);
     if (data->human) {
         g_assert_cmpstr(str, ==, string_human);
     } else {
         g_assert_cmpstr(str, ==, string);
     }
-    g_free(str);
 }
 
 static void test_visitor_out_no_string(TestOutputVisitorData *data,
                                        const void *unused)
 {
     char *string = NULL;
-    Error *err = NULL;
     char *str;
 
     /* A null string should return "" */
-    visit_type_str(data->ov, &string, NULL, &err);
-    g_assert(!err);
+    visit_type_str(data->ov, NULL, &string, &error_abort);
 
-    str = string_output_get_string(data->sov);
-    g_assert(str != NULL);
+    str = visitor_get(data);
     if (data->human) {
         g_assert_cmpstr(str, ==, "<null>");
     } else {
         g_assert_cmpstr(str, ==, "");
     }
-    g_free(str);
 }
 
 static void test_visitor_out_enum(TestOutputVisitorData *data,
                                   const void *unused)
 {
-    Error *err = NULL;
     char *str;
     EnumOne i;
 
-    for (i = 0; i < ENUM_ONE_MAX; i++) {
-        char *str_human;
+    for (i = 0; i < ENUM_ONE__MAX; i++) {
+        visit_type_EnumOne(data->ov, "unused", &i, &error_abort);
 
-        visit_type_EnumOne(data->ov, &i, "unused", &err);
-        g_assert(!err);
-
-        str_human = g_strdup_printf("\"%s\"", EnumOne_lookup[i]);
-
-        str = string_output_get_string(data->sov);
-        g_assert(str != NULL);
+        str = visitor_get(data);
         if (data->human) {
+            char *str_human = g_strdup_printf("\"%s\"", EnumOne_lookup[i]);
+
             g_assert_cmpstr(str, ==, str_human);
+            g_free(str_human);
         } else {
             g_assert_cmpstr(str, ==, EnumOne_lookup[i]);
         }
-        g_free(str_human);
-	g_free(str);
+        visitor_reset(data);
     }
 }
 
 static void test_visitor_out_enum_errors(TestOutputVisitorData *data,
                                          const void *unused)
 {
-    EnumOne i, bad_values[] = { ENUM_ONE_MAX, -1 };
+    EnumOne i, bad_values[] = { ENUM_ONE__MAX, -1 };
     Error *err;
 
     for (i = 0; i < ARRAY_SIZE(bad_values) ; i++) {
         err = NULL;
-        visit_type_EnumOne(data->ov, &bad_values[i], "unused", &err);
-        g_assert(err);
-        error_free(err);
+        visit_type_EnumOne(data->ov, "unused", &bad_values[i], &err);
+        error_free_or_abort(&err);
     }
 }
 
@@ -248,39 +238,39 @@
 
     output_visitor_test_add("/string-visitor/output/int",
                             &out_visitor_data, test_visitor_out_int, false);
-    output_visitor_test_add("/string-visitor/output/int",
+    output_visitor_test_add("/string-visitor/output/int-human",
                             &out_visitor_data, test_visitor_out_int, true);
     output_visitor_test_add("/string-visitor/output/bool",
                             &out_visitor_data, test_visitor_out_bool, false);
-    output_visitor_test_add("/string-visitor/output/bool",
+    output_visitor_test_add("/string-visitor/output/bool-human",
                             &out_visitor_data, test_visitor_out_bool, true);
     output_visitor_test_add("/string-visitor/output/number",
                             &out_visitor_data, test_visitor_out_number, false);
-    output_visitor_test_add("/string-visitor/output/number",
+    output_visitor_test_add("/string-visitor/output/number-human",
                             &out_visitor_data, test_visitor_out_number, true);
     output_visitor_test_add("/string-visitor/output/string",
                             &out_visitor_data, test_visitor_out_string, false);
-    output_visitor_test_add("/string-visitor/output/string",
+    output_visitor_test_add("/string-visitor/output/string-human",
                             &out_visitor_data, test_visitor_out_string, true);
     output_visitor_test_add("/string-visitor/output/no-string",
                             &out_visitor_data, test_visitor_out_no_string,
                             false);
-    output_visitor_test_add("/string-visitor/output/no-string",
+    output_visitor_test_add("/string-visitor/output/no-string-human",
                             &out_visitor_data, test_visitor_out_no_string,
                             true);
     output_visitor_test_add("/string-visitor/output/enum",
                             &out_visitor_data, test_visitor_out_enum, false);
-    output_visitor_test_add("/string-visitor/output/enum",
+    output_visitor_test_add("/string-visitor/output/enum-human",
                             &out_visitor_data, test_visitor_out_enum, true);
     output_visitor_test_add("/string-visitor/output/enum-errors",
                             &out_visitor_data, test_visitor_out_enum_errors,
                             false);
-    output_visitor_test_add("/string-visitor/output/enum-errors",
+    output_visitor_test_add("/string-visitor/output/enum-errors-human",
                             &out_visitor_data, test_visitor_out_enum_errors,
                             true);
     output_visitor_test_add("/string-visitor/output/intList",
                             &out_visitor_data, test_visitor_out_intList, false);
-    output_visitor_test_add("/string-visitor/output/intList",
+    output_visitor_test_add("/string-visitor/output/intList-human",
                             &out_visitor_data, test_visitor_out_intList, true);
 
     g_test_run();
diff --git a/tests/test-thread-pool.c b/tests/test-thread-pool.c
index 6a0b981..8dbf66a 100644
--- a/tests/test-thread-pool.c
+++ b/tests/test-thread-pool.c
@@ -1,8 +1,9 @@
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "block/aio.h"
 #include "block/thread-pool.h"
 #include "block/block.h"
+#include "qapi/error.h"
 #include "qemu/timer.h"
 #include "qemu/error-report.h"
 
@@ -90,9 +91,9 @@
 static void test_submit_co(void)
 {
     WorkerTestData data;
-    Coroutine *co = qemu_coroutine_create(co_test_cb);
+    Coroutine *co = qemu_coroutine_create(co_test_cb, &data);
 
-    qemu_coroutine_enter(co, &data);
+    qemu_coroutine_enter(co);
 
     /* Back here once the worker has started.  */
 
@@ -229,9 +230,7 @@
 
     ctx = aio_context_new(&local_error);
     if (!ctx) {
-        error_report("Failed to create AIO Context: '%s'",
-                     error_get_pretty(local_error));
-        error_free(local_error);
+        error_reportf_err(local_error, "Failed to create AIO Context: ");
         exit(1);
     }
     pool = aio_get_thread_pool(ctx);
diff --git a/tests/test-throttle.c b/tests/test-throttle.c
index d8ba415..363b59a 100644
--- a/tests/test-throttle.c
+++ b/tests/test-throttle.c
@@ -1,25 +1,31 @@
 /*
  * Throttle infrastructure tests
  *
- * Copyright Nodalink, SARL. 2013
+ * Copyright Nodalink, EURL. 2013-2014
+ * Copyright Igalia, S.L. 2015
  *
  * Authors:
- *  Benoît Canet     <benoit.canet@irqsave.net>
+ *  Benoît Canet     <benoit.canet@nodalink.com>
+ *  Alberto Garcia   <berto@igalia.com>
  *
  * This work is licensed under the terms of the GNU LGPL, version 2 or later.
  * See the COPYING.LIB file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 #include <math.h>
 #include "block/aio.h"
+#include "qapi/error.h"
 #include "qemu/throttle.h"
 #include "qemu/error-report.h"
+#include "block/throttle-groups.h"
+#include "sysemu/block-backend.h"
 
 static AioContext     *ctx;
 static LeakyBucket    bkt;
 static ThrottleConfig cfg;
 static ThrottleState  ts;
+static ThrottleTimers tt;
 
 /* useful function */
 static bool double_cmp(double x, double y)
@@ -30,6 +36,9 @@
 /* tests for single bucket operations */
 static void test_leak_bucket(void)
 {
+    throttle_config_init(&cfg);
+    bkt = cfg.buckets[THROTTLE_BPS_TOTAL];
+
     /* set initial value */
     bkt.avg = 150;
     bkt.max = 15;
@@ -52,13 +61,33 @@
     g_assert(bkt.avg == 150);
     g_assert(bkt.max == 15);
     g_assert(double_cmp(bkt.level, 0));
+
+    /* check that burst_level leaks correctly */
+    bkt.burst_level = 6;
+    bkt.max = 250;
+    bkt.burst_length = 2; /* otherwise burst_level will not leak */
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 3.5));
+
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 1));
+
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 0));
+
+    throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 100);
+    g_assert(double_cmp(bkt.burst_level, 0));
 }
 
 static void test_compute_wait(void)
 {
+    unsigned i;
     int64_t wait;
     int64_t result;
 
+    throttle_config_init(&cfg);
+    bkt = cfg.buckets[THROTTLE_BPS_TOTAL];
+
     /* no operation limit set */
     bkt.avg = 0;
     bkt.max = 15;
@@ -88,6 +117,27 @@
     /* time required to do half an operation */
     result = (int64_t)  NANOSECONDS_PER_SECOND / 150 / 2;
     g_assert(wait == result);
+
+    /* Perform I/O for 2.2 seconds at a rate of bkt.max */
+    bkt.burst_length = 2;
+    bkt.level = 0;
+    bkt.avg = 10;
+    bkt.max = 200;
+    for (i = 0; i < 22; i++) {
+        double units = bkt.max / 10;
+        bkt.level += units;
+        bkt.burst_level += units;
+        throttle_leak_bucket(&bkt, NANOSECONDS_PER_SECOND / 10);
+        wait = throttle_compute_wait(&bkt);
+        g_assert(double_cmp(bkt.burst_level, 0));
+        g_assert(double_cmp(bkt.level, (i + 1) * (bkt.max - bkt.avg) / 10));
+        /* We can do bursts for the 2 seconds we have configured in
+         * burst_length. We have 100 extra miliseconds of burst
+         * because bkt.level has been leaking during this time.
+         * After that, we have to wait. */
+        result = i < 21 ? 0 : 1.8 * NANOSECONDS_PER_SECOND;
+        g_assert(wait == result);
+    }
 }
 
 /* functions to test ThrottleState initialization/destroy methods */
@@ -103,17 +153,19 @@
 {
     int i;
 
-    /* fill the structure with crap */
+    /* fill the structures with crap */
     memset(&ts, 1, sizeof(ts));
+    memset(&tt, 1, sizeof(tt));
 
-    /* init the structure */
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    /* init structures */
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
 
     /* check initialized fields */
-    g_assert(ts.clock_type == QEMU_CLOCK_VIRTUAL);
-    g_assert(ts.timers[0]);
-    g_assert(ts.timers[1]);
+    g_assert(tt.clock_type == QEMU_CLOCK_VIRTUAL);
+    g_assert(tt.timers[0]);
+    g_assert(tt.timers[1]);
 
     /* check other fields where cleared */
     g_assert(!ts.previous_leak);
@@ -124,17 +176,18 @@
         g_assert(!ts.cfg.buckets[i].level);
     }
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 }
 
 static void test_destroy(void)
 {
     int i;
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
-    throttle_destroy(&ts);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
+    throttle_timers_destroy(&tt);
     for (i = 0; i < 2; i++) {
-        g_assert(!ts.timers[i]);
+        g_assert(!tt.timers[i]);
     }
 }
 
@@ -170,11 +223,12 @@
 
     orig_cfg.op_size = 1;
 
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
     /* structure reset by throttle_init previous_leak should be null */
     g_assert(!ts.previous_leak);
-    throttle_config(&ts, &orig_cfg);
+    throttle_config(&ts, &tt, &orig_cfg);
 
     /* has previous leak been initialized by throttle_config ? */
     g_assert(ts.previous_leak);
@@ -182,7 +236,7 @@
     /* get back the fixed configuration */
     throttle_get_config(&ts, &final_cfg);
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 
     g_assert(final_cfg.buckets[THROTTLE_BPS_TOTAL].avg == 153);
     g_assert(final_cfg.buckets[THROTTLE_BPS_READ].avg  == 56);
@@ -213,6 +267,8 @@
 {
     if (is_max) {
         cfg.buckets[index].max = value;
+        /* If max is set, avg should never be 0 */
+        cfg.buckets[index].avg = MAX(cfg.buckets[index].avg, 1);
     } else {
         cfg.buckets[index].avg = value;
     }
@@ -222,17 +278,17 @@
 {
     int i;
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     g_assert(!throttle_enabled(&cfg));
 
     for (i = 0; i < BUCKETS_COUNT; i++) {
-        memset(&cfg, 0, sizeof(cfg));
+        throttle_config_init(&cfg);
         set_cfg_value(false, i, 150);
         g_assert(throttle_enabled(&cfg));
     }
 
     for (i = 0; i < BUCKETS_COUNT; i++) {
-        memset(&cfg, 0, sizeof(cfg));
+        throttle_config_init(&cfg);
         set_cfg_value(false, i, -150);
         g_assert(!throttle_enabled(&cfg));
     }
@@ -245,32 +301,32 @@
                                        int read,
                                        int write)
 {
-    memset(&cfg, 0, sizeof(cfg));
-    g_assert(!throttle_conflicting(&cfg));
+    throttle_config_init(&cfg);
+    g_assert(throttle_is_valid(&cfg, NULL));
 
     set_cfg_value(is_max, total, 1);
     set_cfg_value(is_max, read,  1);
-    g_assert(throttle_conflicting(&cfg));
+    g_assert(!throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, total, 1);
     set_cfg_value(is_max, write, 1);
-    g_assert(throttle_conflicting(&cfg));
+    g_assert(!throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, total, 1);
     set_cfg_value(is_max, read,  1);
     set_cfg_value(is_max, write, 1);
-    g_assert(throttle_conflicting(&cfg));
+    g_assert(!throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, total, 1);
-    g_assert(!throttle_conflicting(&cfg));
+    g_assert(throttle_is_valid(&cfg, NULL));
 
-    memset(&cfg, 0, sizeof(cfg));
+    throttle_config_init(&cfg);
     set_cfg_value(is_max, read,  1);
     set_cfg_value(is_max, write, 1);
-    g_assert(!throttle_conflicting(&cfg));
+    g_assert(throttle_is_valid(&cfg, NULL));
 }
 
 static void test_conflicting_config(void)
@@ -304,9 +360,9 @@
     int is_max, index;
     for (is_max = 0; is_max < 2; is_max++) {
         for (index = 0; index < BUCKETS_COUNT; index++) {
-            memset(&cfg, 0, sizeof(cfg));
+            throttle_config_init(&cfg);
             set_cfg_value(is_max, index, value);
-            g_assert(throttle_is_valid(&cfg) == should_be_valid);
+            g_assert(throttle_is_valid(&cfg, NULL) == should_be_valid);
         }
     }
 }
@@ -321,45 +377,85 @@
     test_is_valid_for_value(1, true);
 }
 
+static void test_max_is_missing_limit(void)
+{
+    int i;
+
+    for (i = 0; i < BUCKETS_COUNT; i++) {
+        throttle_config_init(&cfg);
+        cfg.buckets[i].max = 100;
+        cfg.buckets[i].avg = 0;
+        g_assert(!throttle_is_valid(&cfg, NULL));
+
+        cfg.buckets[i].max = 0;
+        cfg.buckets[i].avg = 0;
+        g_assert(throttle_is_valid(&cfg, NULL));
+
+        cfg.buckets[i].max = 0;
+        cfg.buckets[i].avg = 100;
+        g_assert(throttle_is_valid(&cfg, NULL));
+
+        cfg.buckets[i].max = 30;
+        cfg.buckets[i].avg = 100;
+        g_assert(!throttle_is_valid(&cfg, NULL));
+
+        cfg.buckets[i].max = 100;
+        cfg.buckets[i].avg = 100;
+        g_assert(throttle_is_valid(&cfg, NULL));
+    }
+}
+
+static void test_iops_size_is_missing_limit(void)
+{
+    /* A total/read/write iops limit is required */
+    throttle_config_init(&cfg);
+    cfg.op_size = 4096;
+    g_assert(!throttle_is_valid(&cfg, NULL));
+}
+
 static void test_have_timer(void)
 {
-    /* zero the structure */
+    /* zero structures */
     memset(&ts, 0, sizeof(ts));
+    memset(&tt, 0, sizeof(tt));
 
     /* no timer set should return false */
-    g_assert(!throttle_have_timer(&ts));
+    g_assert(!throttle_timers_are_initialized(&tt));
 
-    /* init the structure */
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    /* init structures */
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
 
     /* timer set by init should return true */
-    g_assert(throttle_have_timer(&ts));
+    g_assert(throttle_timers_are_initialized(&tt));
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 }
 
 static void test_detach_attach(void)
 {
-    /* zero the structure */
+    /* zero structures */
     memset(&ts, 0, sizeof(ts));
+    memset(&tt, 0, sizeof(tt));
 
     /* init the structure */
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
 
     /* timer set by init should return true */
-    g_assert(throttle_have_timer(&ts));
+    g_assert(throttle_timers_are_initialized(&tt));
 
     /* timer should no longer exist after detaching */
-    throttle_detach_aio_context(&ts);
-    g_assert(!throttle_have_timer(&ts));
+    throttle_timers_detach_aio_context(&tt);
+    g_assert(!throttle_timers_are_initialized(&tt));
 
     /* timer should exist again after attaching */
-    throttle_attach_aio_context(&ts, ctx);
-    g_assert(throttle_have_timer(&ts));
+    throttle_timers_attach_aio_context(&tt, ctx);
+    g_assert(throttle_timers_are_initialized(&tt));
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 }
 
 static bool do_test_accounting(bool is_ops, /* are we testing bps or ops */
@@ -387,9 +483,10 @@
 
     cfg.op_size = op_size;
 
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
-    throttle_config(&ts, &cfg);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
+    throttle_config(&ts, &tt, &cfg);
 
     /* account a read */
     throttle_account(&ts, false, size);
@@ -414,7 +511,7 @@
         return false;
     }
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 
     return true;
 }
@@ -490,23 +587,72 @@
                                 (64.0 / 13)));
 }
 
+static void test_groups(void)
+{
+    ThrottleConfig cfg1, cfg2;
+    BlockBackend *blk1, *blk2, *blk3;
+    BlockBackendPublic *blkp1, *blkp2, *blkp3;
+
+    blk1 = blk_new();
+    blk2 = blk_new();
+    blk3 = blk_new();
+
+    blkp1 = blk_get_public(blk1);
+    blkp2 = blk_get_public(blk2);
+    blkp3 = blk_get_public(blk3);
+
+    g_assert(blkp1->throttle_state == NULL);
+    g_assert(blkp2->throttle_state == NULL);
+    g_assert(blkp3->throttle_state == NULL);
+
+    throttle_group_register_blk(blk1, "bar");
+    throttle_group_register_blk(blk2, "foo");
+    throttle_group_register_blk(blk3, "bar");
+
+    g_assert(blkp1->throttle_state != NULL);
+    g_assert(blkp2->throttle_state != NULL);
+    g_assert(blkp3->throttle_state != NULL);
+
+    g_assert(!strcmp(throttle_group_get_name(blk1), "bar"));
+    g_assert(!strcmp(throttle_group_get_name(blk2), "foo"));
+    g_assert(blkp1->throttle_state == blkp3->throttle_state);
+
+    /* Setting the config of a group member affects the whole group */
+    throttle_config_init(&cfg1);
+    cfg1.buckets[THROTTLE_BPS_READ].avg  = 500000;
+    cfg1.buckets[THROTTLE_BPS_WRITE].avg = 285000;
+    cfg1.buckets[THROTTLE_OPS_READ].avg  = 20000;
+    cfg1.buckets[THROTTLE_OPS_WRITE].avg = 12000;
+    throttle_group_config(blk1, &cfg1);
+
+    throttle_group_get_config(blk1, &cfg1);
+    throttle_group_get_config(blk3, &cfg2);
+    g_assert(!memcmp(&cfg1, &cfg2, sizeof(cfg1)));
+
+    cfg2.buckets[THROTTLE_BPS_READ].avg  = 4547;
+    cfg2.buckets[THROTTLE_BPS_WRITE].avg = 1349;
+    cfg2.buckets[THROTTLE_OPS_READ].avg  = 123;
+    cfg2.buckets[THROTTLE_OPS_WRITE].avg = 86;
+    throttle_group_config(blk3, &cfg1);
+
+    throttle_group_get_config(blk1, &cfg1);
+    throttle_group_get_config(blk3, &cfg2);
+    g_assert(!memcmp(&cfg1, &cfg2, sizeof(cfg1)));
+
+    throttle_group_unregister_blk(blk1);
+    throttle_group_unregister_blk(blk2);
+    throttle_group_unregister_blk(blk3);
+
+    g_assert(blkp1->throttle_state == NULL);
+    g_assert(blkp2->throttle_state == NULL);
+    g_assert(blkp3->throttle_state == NULL);
+}
+
 int main(int argc, char **argv)
 {
-    GSource *src;
-    Error *local_error = NULL;
-
-    init_clocks();
-
-    ctx = aio_context_new(&local_error);
-    if (!ctx) {
-        error_report("Failed to create AIO Context: '%s'",
-                     error_get_pretty(local_error));
-        error_free(local_error);
-        exit(1);
-    }
-    src = aio_get_g_source(ctx);
-    g_source_attach(src, NULL);
-    g_source_unref(src);
+    qemu_init_main_loop(&error_fatal);
+    ctx = qemu_get_aio_context();
+    bdrv_init();
 
     do {} while (g_main_context_iteration(NULL, false));
 
@@ -521,8 +667,12 @@
     g_test_add_func("/throttle/config/enabled",     test_enabled);
     g_test_add_func("/throttle/config/conflicting", test_conflicting_config);
     g_test_add_func("/throttle/config/is_valid",    test_is_valid);
+    g_test_add_func("/throttle/config/max",         test_max_is_missing_limit);
+    g_test_add_func("/throttle/config/iops_size",
+                    test_iops_size_is_missing_limit);
     g_test_add_func("/throttle/config_functions",   test_config_functions);
     g_test_add_func("/throttle/accounting",         test_accounting);
+    g_test_add_func("/throttle/groups",             test_groups);
     return g_test_run();
 }
 
diff --git a/tests/test-timed-average.c b/tests/test-timed-average.c
new file mode 100644
index 0000000..e2bcf5f
--- /dev/null
+++ b/tests/test-timed-average.c
@@ -0,0 +1,89 @@
+/*
+ * Timed average computation tests
+ *
+ * Copyright Nodalink, EURL. 2014
+ *
+ * Authors:
+ *  Benoît Canet     <benoit.canet@nodalink.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+
+#include "qemu/timed-average.h"
+
+/* This is the clock for QEMU_CLOCK_VIRTUAL */
+static int64_t my_clock_value;
+
+int64_t cpu_get_clock(void)
+{
+    return my_clock_value;
+}
+
+static void account(TimedAverage *ta)
+{
+    timed_average_account(ta, 1);
+    timed_average_account(ta, 5);
+    timed_average_account(ta, 2);
+    timed_average_account(ta, 4);
+    timed_average_account(ta, 3);
+}
+
+static void test_average(void)
+{
+    TimedAverage ta;
+    uint64_t result;
+    int i;
+
+    /* we will compute some average on a period of 1 second */
+    timed_average_init(&ta, QEMU_CLOCK_VIRTUAL, NANOSECONDS_PER_SECOND);
+
+    result = timed_average_min(&ta);
+    g_assert(result == 0);
+    result = timed_average_avg(&ta);
+    g_assert(result == 0);
+    result = timed_average_max(&ta);
+    g_assert(result == 0);
+
+    for (i = 0; i < 100; i++) {
+        account(&ta);
+        result = timed_average_min(&ta);
+        g_assert(result == 1);
+        result = timed_average_avg(&ta);
+        g_assert(result == 3);
+        result = timed_average_max(&ta);
+        g_assert(result == 5);
+        my_clock_value += NANOSECONDS_PER_SECOND / 10;
+    }
+
+    my_clock_value += NANOSECONDS_PER_SECOND * 100;
+
+    result = timed_average_min(&ta);
+    g_assert(result == 0);
+    result = timed_average_avg(&ta);
+    g_assert(result == 0);
+    result = timed_average_max(&ta);
+    g_assert(result == 0);
+
+    for (i = 0; i < 100; i++) {
+        account(&ta);
+        result = timed_average_min(&ta);
+        g_assert(result == 1);
+        result = timed_average_avg(&ta);
+        g_assert(result == 3);
+        result = timed_average_max(&ta);
+        g_assert(result == 5);
+        my_clock_value += NANOSECONDS_PER_SECOND / 10;
+    }
+}
+
+int main(int argc, char **argv)
+{
+    /* tests in the same order as the header function declarations */
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/timed-average/average", test_average);
+    return g_test_run();
+}
+
diff --git a/tests/test-visitor-serialization.c b/tests/test-visitor-serialization.c
index 7ad1886..dba4670 100644
--- a/tests/test-visitor-serialization.c
+++ b/tests/test-visitor-serialization.c
@@ -1,6 +1,7 @@
 /*
  * Unit-tests for visitor-based serialization
  *
+ * Copyright (C) 2014-2015 Red Hat, Inc.
  * Copyright IBM, Corp. 2012
  *
  * Authors:
@@ -10,15 +11,15 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <stdlib.h>
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include <float.h>
 
 #include "qemu-common.h"
 #include "test-qapi-types.h"
 #include "test-qapi-visit.h"
+#include "qapi/error.h"
 #include "qapi/qmp/types.h"
+#include "qapi/qmp/qjson.h"
 #include "qapi/qmp-input-visitor.h"
 #include "qapi/qmp-output-visitor.h"
 #include "qapi/string-input-visitor.h"
@@ -88,11 +89,11 @@
 
 static void dealloc_helper(void *native_in, VisitorFunc visit, Error **errp)
 {
-    QapiDeallocVisitor *qdv = qapi_dealloc_visitor_new();
+    Visitor *v = qapi_dealloc_visitor_new();
 
-    visit(qapi_dealloc_get_visitor(qdv), &native_in, errp);
+    visit(v, &native_in, errp);
 
-    qapi_dealloc_visitor_cleanup(qdv);
+    visit_free(v);
 }
 
 static void visit_primitive_type(Visitor *v, void **native, Error **errp)
@@ -100,40 +101,40 @@
     PrimitiveType *pt = *native;
     switch(pt->type) {
     case PTYPE_STRING:
-        visit_type_str(v, (char **)&pt->value.string, NULL, errp);
+        visit_type_str(v, NULL, (char **)&pt->value.string, errp);
         break;
     case PTYPE_BOOLEAN:
-        visit_type_bool(v, &pt->value.boolean, NULL, errp);
+        visit_type_bool(v, NULL, &pt->value.boolean, errp);
         break;
     case PTYPE_NUMBER:
-        visit_type_number(v, &pt->value.number, NULL, errp);
+        visit_type_number(v, NULL, &pt->value.number, errp);
         break;
     case PTYPE_INTEGER:
-        visit_type_int(v, &pt->value.integer, NULL, errp);
+        visit_type_int(v, NULL, &pt->value.integer, errp);
         break;
     case PTYPE_U8:
-        visit_type_uint8(v, &pt->value.u8, NULL, errp);
+        visit_type_uint8(v, NULL, &pt->value.u8, errp);
         break;
     case PTYPE_U16:
-        visit_type_uint16(v, &pt->value.u16, NULL, errp);
+        visit_type_uint16(v, NULL, &pt->value.u16, errp);
         break;
     case PTYPE_U32:
-        visit_type_uint32(v, &pt->value.u32, NULL, errp);
+        visit_type_uint32(v, NULL, &pt->value.u32, errp);
         break;
     case PTYPE_U64:
-        visit_type_uint64(v, &pt->value.u64, NULL, errp);
+        visit_type_uint64(v, NULL, &pt->value.u64, errp);
         break;
     case PTYPE_S8:
-        visit_type_int8(v, &pt->value.s8, NULL, errp);
+        visit_type_int8(v, NULL, &pt->value.s8, errp);
         break;
     case PTYPE_S16:
-        visit_type_int16(v, &pt->value.s16, NULL, errp);
+        visit_type_int16(v, NULL, &pt->value.s16, errp);
         break;
     case PTYPE_S32:
-        visit_type_int32(v, &pt->value.s32, NULL, errp);
+        visit_type_int32(v, NULL, &pt->value.s32, errp);
         break;
     case PTYPE_S64:
-        visit_type_int64(v, &pt->value.s64, NULL, errp);
+        visit_type_int64(v, NULL, &pt->value.s64, errp);
         break;
     case PTYPE_EOL:
         g_assert_not_reached();
@@ -145,80 +146,46 @@
     PrimitiveList *pl = *native;
     switch (pl->type) {
     case PTYPE_STRING:
-        visit_type_strList(v, &pl->value.strings, NULL, errp);
+        visit_type_strList(v, NULL, &pl->value.strings, errp);
         break;
     case PTYPE_BOOLEAN:
-        visit_type_boolList(v, &pl->value.booleans, NULL, errp);
+        visit_type_boolList(v, NULL, &pl->value.booleans, errp);
         break;
     case PTYPE_NUMBER:
-        visit_type_numberList(v, &pl->value.numbers, NULL, errp);
+        visit_type_numberList(v, NULL, &pl->value.numbers, errp);
         break;
     case PTYPE_INTEGER:
-        visit_type_intList(v, &pl->value.integers, NULL, errp);
+        visit_type_intList(v, NULL, &pl->value.integers, errp);
         break;
     case PTYPE_S8:
-        visit_type_int8List(v, &pl->value.s8_integers, NULL, errp);
+        visit_type_int8List(v, NULL, &pl->value.s8_integers, errp);
         break;
     case PTYPE_S16:
-        visit_type_int16List(v, &pl->value.s16_integers, NULL, errp);
+        visit_type_int16List(v, NULL, &pl->value.s16_integers, errp);
         break;
     case PTYPE_S32:
-        visit_type_int32List(v, &pl->value.s32_integers, NULL, errp);
+        visit_type_int32List(v, NULL, &pl->value.s32_integers, errp);
         break;
     case PTYPE_S64:
-        visit_type_int64List(v, &pl->value.s64_integers, NULL, errp);
+        visit_type_int64List(v, NULL, &pl->value.s64_integers, errp);
         break;
     case PTYPE_U8:
-        visit_type_uint8List(v, &pl->value.u8_integers, NULL, errp);
+        visit_type_uint8List(v, NULL, &pl->value.u8_integers, errp);
         break;
     case PTYPE_U16:
-        visit_type_uint16List(v, &pl->value.u16_integers, NULL, errp);
+        visit_type_uint16List(v, NULL, &pl->value.u16_integers, errp);
         break;
     case PTYPE_U32:
-        visit_type_uint32List(v, &pl->value.u32_integers, NULL, errp);
+        visit_type_uint32List(v, NULL, &pl->value.u32_integers, errp);
         break;
     case PTYPE_U64:
-        visit_type_uint64List(v, &pl->value.u64_integers, NULL, errp);
+        visit_type_uint64List(v, NULL, &pl->value.u64_integers, errp);
         break;
     default:
         g_assert_not_reached();
     }
 }
 
-typedef struct TestStruct
-{
-    int64_t integer;
-    bool boolean;
-    char *string;
-} TestStruct;
-
-static void visit_type_TestStruct(Visitor *v, TestStruct **obj,
-                                  const char *name, Error **errp)
-{
-    Error *err = NULL;
-
-    visit_start_struct(v, (void **)obj, NULL, name, sizeof(TestStruct), &err);
-    if (err) {
-        goto out;
-    }
-
-    visit_type_int(v, &(*obj)->integer, "integer", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_bool(v, &(*obj)->boolean, "boolean", &err);
-    if (err) {
-        goto out_end;
-    }
-    visit_type_str(v, &(*obj)->string, "string", &err);
-
-out_end:
-    error_propagate(errp, err);
-    err = NULL;
-    visit_end_struct(v, &err);
-out:
-    error_propagate(errp, err);
-}
 
 static TestStruct *struct_create(void)
 {
@@ -246,60 +213,63 @@
 
 static void visit_struct(Visitor *v, void **native, Error **errp)
 {
-    visit_type_TestStruct(v, (TestStruct **)native, NULL, errp);
+    visit_type_TestStruct(v, NULL, (TestStruct **)native, errp);
 }
 
-static UserDefNested *nested_struct_create(void)
+static UserDefTwo *nested_struct_create(void)
 {
-    UserDefNested *udnp = g_malloc0(sizeof(*udnp));
+    UserDefTwo *udnp = g_malloc0(sizeof(*udnp));
     udnp->string0 = strdup("test_string0");
-    udnp->dict1.string1 = strdup("test_string1");
-    udnp->dict1.dict2.userdef1 = g_malloc0(sizeof(UserDefOne));
-    udnp->dict1.dict2.userdef1->base = g_new0(UserDefZero, 1);
-    udnp->dict1.dict2.userdef1->base->integer = 42;
-    udnp->dict1.dict2.userdef1->string = strdup("test_string");
-    udnp->dict1.dict2.string2 = strdup("test_string2");
-    udnp->dict1.has_dict3 = true;
-    udnp->dict1.dict3.userdef2 = g_malloc0(sizeof(UserDefOne));
-    udnp->dict1.dict3.userdef2->base = g_new0(UserDefZero, 1);
-    udnp->dict1.dict3.userdef2->base->integer = 43;
-    udnp->dict1.dict3.userdef2->string = strdup("test_string");
-    udnp->dict1.dict3.string3 = strdup("test_string3");
+    udnp->dict1 = g_malloc0(sizeof(*udnp->dict1));
+    udnp->dict1->string1 = strdup("test_string1");
+    udnp->dict1->dict2 = g_malloc0(sizeof(*udnp->dict1->dict2));
+    udnp->dict1->dict2->userdef = g_new0(UserDefOne, 1);
+    udnp->dict1->dict2->userdef->integer = 42;
+    udnp->dict1->dict2->userdef->string = strdup("test_string");
+    udnp->dict1->dict2->string = strdup("test_string2");
+    udnp->dict1->dict3 = g_malloc0(sizeof(*udnp->dict1->dict3));
+    udnp->dict1->has_dict3 = true;
+    udnp->dict1->dict3->userdef = g_new0(UserDefOne, 1);
+    udnp->dict1->dict3->userdef->integer = 43;
+    udnp->dict1->dict3->userdef->string = strdup("test_string");
+    udnp->dict1->dict3->string = strdup("test_string3");
     return udnp;
 }
 
-static void nested_struct_compare(UserDefNested *udnp1, UserDefNested *udnp2)
+static void nested_struct_compare(UserDefTwo *udnp1, UserDefTwo *udnp2)
 {
     g_assert(udnp1);
     g_assert(udnp2);
     g_assert_cmpstr(udnp1->string0, ==, udnp2->string0);
-    g_assert_cmpstr(udnp1->dict1.string1, ==, udnp2->dict1.string1);
-    g_assert_cmpint(udnp1->dict1.dict2.userdef1->base->integer, ==,
-                    udnp2->dict1.dict2.userdef1->base->integer);
-    g_assert_cmpstr(udnp1->dict1.dict2.userdef1->string, ==,
-                    udnp2->dict1.dict2.userdef1->string);
-    g_assert_cmpstr(udnp1->dict1.dict2.string2, ==, udnp2->dict1.dict2.string2);
-    g_assert(udnp1->dict1.has_dict3 == udnp2->dict1.has_dict3);
-    g_assert_cmpint(udnp1->dict1.dict3.userdef2->base->integer, ==,
-                    udnp2->dict1.dict3.userdef2->base->integer);
-    g_assert_cmpstr(udnp1->dict1.dict3.userdef2->string, ==,
-                    udnp2->dict1.dict3.userdef2->string);
-    g_assert_cmpstr(udnp1->dict1.dict3.string3, ==, udnp2->dict1.dict3.string3);
+    g_assert_cmpstr(udnp1->dict1->string1, ==, udnp2->dict1->string1);
+    g_assert_cmpint(udnp1->dict1->dict2->userdef->integer, ==,
+                    udnp2->dict1->dict2->userdef->integer);
+    g_assert_cmpstr(udnp1->dict1->dict2->userdef->string, ==,
+                    udnp2->dict1->dict2->userdef->string);
+    g_assert_cmpstr(udnp1->dict1->dict2->string, ==,
+                    udnp2->dict1->dict2->string);
+    g_assert(udnp1->dict1->has_dict3 == udnp2->dict1->has_dict3);
+    g_assert_cmpint(udnp1->dict1->dict3->userdef->integer, ==,
+                    udnp2->dict1->dict3->userdef->integer);
+    g_assert_cmpstr(udnp1->dict1->dict3->userdef->string, ==,
+                    udnp2->dict1->dict3->userdef->string);
+    g_assert_cmpstr(udnp1->dict1->dict3->string, ==,
+                    udnp2->dict1->dict3->string);
 }
 
-static void nested_struct_cleanup(UserDefNested *udnp)
+static void nested_struct_cleanup(UserDefTwo *udnp)
 {
-    qapi_free_UserDefNested(udnp);
+    qapi_free_UserDefTwo(udnp);
 }
 
 static void visit_nested_struct(Visitor *v, void **native, Error **errp)
 {
-    visit_type_UserDefNested(v, (UserDefNested **)native, NULL, errp);
+    visit_type_UserDefTwo(v, NULL, (UserDefTwo **)native, errp);
 }
 
 static void visit_nested_struct_list(Visitor *v, void **native, Error **errp)
 {
-    visit_type_UserDefNestedList(v, (UserDefNestedList **)native, NULL, errp);
+    visit_type_UserDefTwoList(v, NULL, (UserDefTwoList **)native, errp);
 }
 
 /* test cases */
@@ -332,14 +302,13 @@
     const SerializeOps *ops = args->ops;
     PrimitiveType *pt = args->test_data;
     PrimitiveType *pt_copy = g_malloc0(sizeof(*pt_copy));
-    Error *err = NULL;
     void *serialize_data;
 
     pt_copy->type = pt->type;
-    ops->serialize(pt, &serialize_data, visit_primitive_type, &err);
-    ops->deserialize((void **)&pt_copy, serialize_data, visit_primitive_type, &err);
+    ops->serialize(pt, &serialize_data, visit_primitive_type, &error_abort);
+    ops->deserialize((void **)&pt_copy, serialize_data, visit_primitive_type,
+                     &error_abort);
 
-    g_assert(err == NULL);
     g_assert(pt_copy != NULL);
     if (pt->type == PTYPE_STRING) {
         g_assert_cmpstr(pt->value.string, ==, pt_copy->value.string);
@@ -375,7 +344,6 @@
     PrimitiveList pl = { .value = { NULL } };
     PrimitiveList pl_copy = { .value = { NULL } };
     PrimitiveList *pl_copy_ptr = &pl_copy;
-    Error *err = NULL;
     void *serialize_data;
     void *cur_head = NULL;
     int i;
@@ -522,10 +490,11 @@
         }
     }
 
-    ops->serialize((void **)&pl, &serialize_data, visit_primitive_list, &err);
-    ops->deserialize((void **)&pl_copy_ptr, serialize_data, visit_primitive_list, &err);
+    ops->serialize((void **)&pl, &serialize_data, visit_primitive_list,
+                   &error_abort);
+    ops->deserialize((void **)&pl_copy_ptr, serialize_data,
+                     visit_primitive_list, &error_abort);
 
-    g_assert(err == NULL);
     i = 0;
 
     /* compare our deserialized list of primitives to the original */
@@ -682,10 +651,8 @@
     g_assert_cmpint(i, ==, 33);
 
     ops->cleanup(serialize_data);
-    dealloc_helper(&pl, visit_primitive_list, &err);
-    g_assert(!err);
-    dealloc_helper(&pl_copy, visit_primitive_list, &err);
-    g_assert(!err);
+    dealloc_helper(&pl, visit_primitive_list, &error_abort);
+    dealloc_helper(&pl_copy, visit_primitive_list, &error_abort);
     g_free(args);
 }
 
@@ -695,13 +662,12 @@
     const SerializeOps *ops = args->ops;
     TestStruct *ts = struct_create();
     TestStruct *ts_copy = NULL;
-    Error *err = NULL;
     void *serialize_data;
 
-    ops->serialize(ts, &serialize_data, visit_struct, &err);
-    ops->deserialize((void **)&ts_copy, serialize_data, visit_struct, &err); 
+    ops->serialize(ts, &serialize_data, visit_struct, &error_abort);
+    ops->deserialize((void **)&ts_copy, serialize_data, visit_struct,
+                     &error_abort);
 
-    g_assert(err == NULL);
     struct_compare(ts, ts_copy);
 
     struct_cleanup(ts);
@@ -715,15 +681,14 @@
 {
     TestArgs *args = (TestArgs *) opaque;
     const SerializeOps *ops = args->ops;
-    UserDefNested *udnp = nested_struct_create();
-    UserDefNested *udnp_copy = NULL;
-    Error *err = NULL;
+    UserDefTwo *udnp = nested_struct_create();
+    UserDefTwo *udnp_copy = NULL;
     void *serialize_data;
-    
-    ops->serialize(udnp, &serialize_data, visit_nested_struct, &err);
-    ops->deserialize((void **)&udnp_copy, serialize_data, visit_nested_struct, &err); 
 
-    g_assert(err == NULL);
+    ops->serialize(udnp, &serialize_data, visit_nested_struct, &error_abort);
+    ops->deserialize((void **)&udnp_copy, serialize_data, visit_nested_struct,
+                     &error_abort);
+
     nested_struct_compare(udnp, udnp_copy);
 
     nested_struct_cleanup(udnp);
@@ -737,23 +702,21 @@
 {
     TestArgs *args = (TestArgs *) opaque;
     const SerializeOps *ops = args->ops;
-    UserDefNestedList *listp = NULL, *tmp, *tmp_copy, *listp_copy = NULL;
-    Error *err = NULL;
+    UserDefTwoList *listp = NULL, *tmp, *tmp_copy, *listp_copy = NULL;
     void *serialize_data;
     int i = 0;
 
     for (i = 0; i < 8; i++) {
-        tmp = g_malloc0(sizeof(UserDefNestedList));
+        tmp = g_new0(UserDefTwoList, 1);
         tmp->value = nested_struct_create();
         tmp->next = listp;
         listp = tmp;
     }
-    
-    ops->serialize(listp, &serialize_data, visit_nested_struct_list, &err);
-    ops->deserialize((void **)&listp_copy, serialize_data,
-                     visit_nested_struct_list, &err); 
 
-    g_assert(err == NULL);
+    ops->serialize(listp, &serialize_data, visit_nested_struct_list,
+                   &error_abort);
+    ops->deserialize((void **)&listp_copy, serialize_data,
+                     visit_nested_struct_list, &error_abort);
 
     tmp = listp;
     tmp_copy = listp_copy;
@@ -764,8 +727,8 @@
         listp_copy = listp_copy->next;
     }
 
-    qapi_free_UserDefNestedList(tmp);
-    qapi_free_UserDefNestedList(tmp_copy);
+    qapi_free_UserDefTwoList(tmp);
+    qapi_free_UserDefTwoList(tmp_copy);
 
     ops->cleanup(serialize_data);
     g_free(args);
@@ -1049,8 +1012,9 @@
 /* visitor-specific op implementations */
 
 typedef struct QmpSerializeData {
-    QmpOutputVisitor *qov;
-    QmpInputVisitor *qiv;
+    Visitor *qov;
+    QObject *obj;
+    Visitor *qiv;
 } QmpSerializeData;
 
 static void qmp_serialize(void *native_in, void **datap,
@@ -1058,8 +1022,8 @@
 {
     QmpSerializeData *d = g_malloc0(sizeof(*d));
 
-    d->qov = qmp_output_visitor_new();
-    visit(qmp_output_get_visitor(d->qov), &native_in, errp);
+    d->qov = qmp_output_visitor_new(&d->obj);
+    visit(d->qov, &native_in, errp);
     *datap = d;
 }
 
@@ -1070,30 +1034,31 @@
     QString *output_json;
     QObject *obj_orig, *obj;
 
-    obj_orig = qmp_output_get_qobject(d->qov);
+    visit_complete(d->qov, &d->obj);
+    obj_orig = d->obj;
     output_json = qobject_to_json(obj_orig);
     obj = qobject_from_json(qstring_get_str(output_json));
 
     QDECREF(output_json);
-    d->qiv = qmp_input_visitor_new(obj);
+    d->qiv = qmp_input_visitor_new(obj, true);
     qobject_decref(obj_orig);
     qobject_decref(obj);
-    visit(qmp_input_get_visitor(d->qiv), native_out, errp);
+    visit(d->qiv, native_out, errp);
 }
 
 static void qmp_cleanup(void *datap)
 {
     QmpSerializeData *d = datap;
-    qmp_output_visitor_cleanup(d->qov);
-    qmp_input_visitor_cleanup(d->qiv);
+    visit_free(d->qov);
+    visit_free(d->qiv);
 
     g_free(d);
 }
 
 typedef struct StringSerializeData {
     char *string;
-    StringOutputVisitor *sov;
-    StringInputVisitor *siv;
+    Visitor *sov;
+    Visitor *siv;
 } StringSerializeData;
 
 static void string_serialize(void *native_in, void **datap,
@@ -1101,8 +1066,8 @@
 {
     StringSerializeData *d = g_malloc0(sizeof(*d));
 
-    d->sov = string_output_visitor_new(false);
-    visit(string_output_get_visitor(d->sov), &native_in, errp);
+    d->sov = string_output_visitor_new(false, &d->string);
+    visit(d->sov, &native_in, errp);
     *datap = d;
 }
 
@@ -1111,17 +1076,17 @@
 {
     StringSerializeData *d = datap;
 
-    d->string = string_output_get_string(d->sov);
+    visit_complete(d->sov, &d->string);
     d->siv = string_input_visitor_new(d->string);
-    visit(string_input_get_visitor(d->siv), native_out, errp);
+    visit(d->siv, native_out, errp);
 }
 
 static void string_cleanup(void *datap)
 {
     StringSerializeData *d = datap;
 
-    string_output_visitor_cleanup(d->sov);
-    string_input_visitor_cleanup(d->siv);
+    visit_free(d->sov);
+    visit_free(d->siv);
     g_free(d->string);
     g_free(d);
 }
diff --git a/tests/test-vmstate.c b/tests/test-vmstate.c
index 5e0fd13..41fd841 100644
--- a/tests/test-vmstate.c
+++ b/tests/test-vmstate.c
@@ -22,12 +22,13 @@
  * THE SOFTWARE.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "qemu-common.h"
 #include "migration/migration.h"
 #include "migration/vmstate.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
+#include "io/channel-file.h"
 
 static char temp_file[] = "/tmp/vmst.test.XXXXXX";
 static int temp_fd;
@@ -43,45 +44,22 @@
     select(fd + 1, &fds, NULL, NULL, NULL);
 }
 
-/*
- * Some tests use 'open_test_file' to work on a real fd, some use
- * an in memory file (QEMUSizedBuffer+qemu_bufopen); we could pick one
- * but this way we test both.
- */
 
 /* Duplicate temp_fd and seek to the beginning of the file */
 static QEMUFile *open_test_file(bool write)
 {
     int fd = dup(temp_fd);
+    QIOChannel *ioc;
     lseek(fd, 0, SEEK_SET);
     if (write) {
         g_assert_cmpint(ftruncate(fd, 0), ==, 0);
     }
-    return qemu_fdopen(fd, write ? "wb" : "rb");
-}
-
-/* Open a read-only qemu-file from an existing memory block */
-static QEMUFile *open_mem_file_read(const void *data, size_t len)
-{
-    /* The qsb gets freed by qemu_fclose */
-    QEMUSizedBuffer *qsb = qsb_create(data, len);
-    g_assert(qsb);
-
-    return qemu_bufopen("r", qsb);
-}
-
-/*
- * Check that the contents of the memory-buffered file f match
- * the given size/data.
- */
-static void check_mem_file(QEMUFile *f, void *data, size_t size)
-{
-    uint8_t *result = g_malloc(size);
-    const QEMUSizedBuffer *qsb = qemu_buf_get(f);
-    g_assert_cmpint(qsb_get_length(qsb), ==, size);
-    g_assert_cmpint(qsb_get_buffer(qsb, 0, size, result), ==, size);
-    g_assert_cmpint(memcmp(result, data, size), ==, 0);
-    g_free(result);
+    ioc = QIO_CHANNEL(qio_channel_file_new_fd(fd));
+    if (write) {
+        return qemu_fopen_channel_output(ioc);
+    } else {
+        return qemu_fopen_channel_input(ioc);
+    }
 }
 
 #define SUCCESS(val) \
@@ -95,7 +73,7 @@
     QEMUFile *f = open_test_file(true);
 
     /* Save file with vmstate */
-    vmstate_save_state(f, desc, obj);
+    vmstate_save_state(f, desc, obj, NULL);
     qemu_put_byte(f, QEMU_VM_EOF);
     g_assert(!qemu_file_get_error(f));
     qemu_fclose(f);
@@ -401,10 +379,10 @@
 
 static void test_save_noskip(void)
 {
-    QEMUFile *fsave = qemu_bufopen("w", NULL);
+    QEMUFile *fsave = open_test_file(true);
     TestStruct obj = { .a = 1, .b = 2, .c = 3, .d = 4, .e = 5, .f = 6,
                        .skip_c_e = false };
-    vmstate_save_state(fsave, &vmstate_skipping, &obj);
+    vmstate_save_state(fsave, &vmstate_skipping, &obj, NULL);
     g_assert(!qemu_file_get_error(fsave));
 
     uint8_t expected[] = {
@@ -415,16 +393,17 @@
         0, 0, 0, 5,             /* e */
         0, 0, 0, 0, 0, 0, 0, 6, /* f */
     };
-    check_mem_file(fsave, expected, sizeof(expected));
+
     qemu_fclose(fsave);
+    compare_vmstate(expected, sizeof(expected));
 }
 
 static void test_save_skip(void)
 {
-    QEMUFile *fsave = qemu_bufopen("w", NULL);
+    QEMUFile *fsave = open_test_file(true);
     TestStruct obj = { .a = 1, .b = 2, .c = 3, .d = 4, .e = 5, .f = 6,
                        .skip_c_e = true };
-    vmstate_save_state(fsave, &vmstate_skipping, &obj);
+    vmstate_save_state(fsave, &vmstate_skipping, &obj, NULL);
     g_assert(!qemu_file_get_error(fsave));
 
     uint8_t expected[] = {
@@ -433,13 +412,14 @@
         0, 0, 0, 0, 0, 0, 0, 4, /* d */
         0, 0, 0, 0, 0, 0, 0, 6, /* f */
     };
-    check_mem_file(fsave, expected, sizeof(expected));
 
     qemu_fclose(fsave);
+    compare_vmstate(expected, sizeof(expected));
 }
 
 static void test_load_noskip(void)
 {
+    QEMUFile *fsave = open_test_file(true);
     uint8_t buf[] = {
         0, 0, 0, 10,             /* a */
         0, 0, 0, 20,             /* b */
@@ -449,8 +429,10 @@
         0, 0, 0, 0, 0, 0, 0, 60, /* f */
         QEMU_VM_EOF, /* just to ensure we won't get EOF reported prematurely */
     };
+    qemu_put_buffer(fsave, buf, sizeof(buf));
+    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_mem_file_read(buf, sizeof(buf));
+    QEMUFile *loading = open_test_file(false);
     TestStruct obj = { .skip_c_e = false };
     vmstate_load_state(loading, &vmstate_skipping, &obj, 2);
     g_assert(!qemu_file_get_error(loading));
@@ -465,6 +447,7 @@
 
 static void test_load_skip(void)
 {
+    QEMUFile *fsave = open_test_file(true);
     uint8_t buf[] = {
         0, 0, 0, 10,             /* a */
         0, 0, 0, 20,             /* b */
@@ -472,8 +455,10 @@
         0, 0, 0, 0, 0, 0, 0, 60, /* f */
         QEMU_VM_EOF, /* just to ensure we won't get EOF reported prematurely */
     };
+    qemu_put_buffer(fsave, buf, sizeof(buf));
+    qemu_fclose(fsave);
 
-    QEMUFile *loading = open_mem_file_read(buf, sizeof(buf));
+    QEMUFile *loading = open_test_file(false);
     TestStruct obj = { .skip_c_e = true, .c = 300, .e = 500 };
     vmstate_load_state(loading, &vmstate_skipping, &obj, 2);
     g_assert(!qemu_file_get_error(loading));
@@ -490,6 +475,8 @@
 {
     temp_fd = mkstemp(temp_file);
 
+    module_call_init(MODULE_INIT_QOM);
+
     g_test_init(&argc, &argv, NULL);
     g_test_add_func("/vmstate/simple/primitive", test_simple_primitive);
     g_test_add_func("/vmstate/versioned/load/v1", test_load_v1);
diff --git a/tests/test-write-threshold.c b/tests/test-write-threshold.c
new file mode 100644
index 0000000..97ca12f
--- /dev/null
+++ b/tests/test-write-threshold.c
@@ -0,0 +1,118 @@
+/*
+ * Test block device write threshold
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "block/block_int.h"
+#include "block/write-threshold.h"
+
+
+static void test_threshold_not_set_on_init(void)
+{
+    uint64_t res;
+    BlockDriverState bs;
+    memset(&bs, 0, sizeof(bs));
+
+    g_assert(!bdrv_write_threshold_is_set(&bs));
+
+    res = bdrv_write_threshold_get(&bs);
+    g_assert_cmpint(res, ==, 0);
+}
+
+static void test_threshold_set_get(void)
+{
+    uint64_t threshold = 4 * 1024 * 1024;
+    uint64_t res;
+    BlockDriverState bs;
+    memset(&bs, 0, sizeof(bs));
+
+    bdrv_write_threshold_set(&bs, threshold);
+
+    g_assert(bdrv_write_threshold_is_set(&bs));
+
+    res = bdrv_write_threshold_get(&bs);
+    g_assert_cmpint(res, ==, threshold);
+}
+
+static void test_threshold_multi_set_get(void)
+{
+    uint64_t threshold1 = 4 * 1024 * 1024;
+    uint64_t threshold2 = 15 * 1024 * 1024;
+    uint64_t res;
+    BlockDriverState bs;
+    memset(&bs, 0, sizeof(bs));
+
+    bdrv_write_threshold_set(&bs, threshold1);
+    bdrv_write_threshold_set(&bs, threshold2);
+    res = bdrv_write_threshold_get(&bs);
+    g_assert_cmpint(res, ==, threshold2);
+}
+
+static void test_threshold_not_trigger(void)
+{
+    uint64_t amount = 0;
+    uint64_t threshold = 4 * 1024 * 1024;
+    BlockDriverState bs;
+    BdrvTrackedRequest req;
+
+    memset(&bs, 0, sizeof(bs));
+    memset(&req, 0, sizeof(req));
+    req.offset = 1024;
+    req.bytes = 1024;
+
+    bdrv_write_threshold_set(&bs, threshold);
+    amount = bdrv_write_threshold_exceeded(&bs, &req);
+    g_assert_cmpuint(amount, ==, 0);
+}
+
+
+static void test_threshold_trigger(void)
+{
+    uint64_t amount = 0;
+    uint64_t threshold = 4 * 1024 * 1024;
+    BlockDriverState bs;
+    BdrvTrackedRequest req;
+
+    memset(&bs, 0, sizeof(bs));
+    memset(&req, 0, sizeof(req));
+    req.offset = (4 * 1024 * 1024) - 1024;
+    req.bytes = 2 * 1024;
+
+    bdrv_write_threshold_set(&bs, threshold);
+    amount = bdrv_write_threshold_exceeded(&bs, &req);
+    g_assert_cmpuint(amount, >=, 1024);
+}
+
+typedef struct TestStruct {
+    const char *name;
+    void (*func)(void);
+} TestStruct;
+
+
+int main(int argc, char **argv)
+{
+    size_t i;
+    TestStruct tests[] = {
+        { "/write-threshold/not-set-on-init",
+          test_threshold_not_set_on_init },
+        { "/write-threshold/set-get",
+          test_threshold_set_get },
+        { "/write-threshold/multi-set-get",
+          test_threshold_multi_set_get },
+        { "/write-threshold/not-trigger",
+          test_threshold_not_trigger },
+        { "/write-threshold/trigger",
+          test_threshold_trigger },
+        { NULL, NULL }
+    };
+
+    g_test_init(&argc, &argv, NULL);
+    for (i = 0; tests[i].name != NULL; i++) {
+        g_test_add_func(tests[i].name, tests[i].func);
+    }
+    return g_test_run();
+}
diff --git a/tests/test-x86-cpuid.c b/tests/test-x86-cpuid.c
index 8d9f96a..ff22500 100644
--- a/tests/test-x86-cpuid.c
+++ b/tests/test-x86-cpuid.c
@@ -22,9 +22,9 @@
  * THE SOFTWARE.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
-#include "topology.h"
+#include "hw/i386/topology.h"
 
 static void test_topo_bits(void)
 {
diff --git a/tests/test-xbzrle.c b/tests/test-xbzrle.c
index db93b0a..49f6419 100644
--- a/tests/test-xbzrle.c
+++ b/tests/test-xbzrle.c
@@ -10,14 +10,9 @@
  * See the COPYING file in the top-level directory.
  *
  */
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <strings.h>
-#include <string.h>
-#include <sys/time.h>
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "include/migration/migration.h"
 
 #define PAGE_SIZE 4096
diff --git a/tests/tmp105-test.c b/tests/tmp105-test.c
index 99db538..a7940a4 100644
--- a/tests/tmp105-test.c
+++ b/tests/tmp105-test.c
@@ -7,7 +7,7 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 #include "libqos/i2c.h"
diff --git a/tests/tpci200-test.c b/tests/tpci200-test.c
index 9ae0127..0321ec2 100644
--- a/tests/tpci200-test.c
+++ b/tests/tpci200-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/usb-hcd-ehci-test.c b/tests/usb-hcd-ehci-test.c
index 75073bf..eb247ad 100644
--- a/tests/usb-hcd-ehci-test.c
+++ b/tests/usb-hcd-ehci-test.c
@@ -7,12 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "libqos/pci-pc.h"
-#include "qemu/osdep.h"
 #include "hw/usb/uhci-regs.h"
 #include "hw/usb/ehci-regs.h"
 #include "libqos/usb.h"
diff --git a/tests/usb-hcd-ohci-test.c b/tests/usb-hcd-ohci-test.c
index 1160bde..4758813 100644
--- a/tests/usb-hcd-ohci-test.c
+++ b/tests/usb-hcd-ohci-test.c
@@ -1,16 +1,14 @@
 /*
  * QTest testcase for USB OHCI controller
  *
- * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO., LTD.
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 #include "libqos/usb.h"
 
 
diff --git a/tests/usb-hcd-uhci-test.c b/tests/usb-hcd-uhci-test.c
index 8cf2c5b..5cd59ad 100644
--- a/tests/usb-hcd-uhci-test.c
+++ b/tests/usb-hcd-uhci-test.c
@@ -1,16 +1,14 @@
 /*
  * QTest testcase for USB UHCI controller
  *
- * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO., LTD.
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 #include "libqos/usb.h"
 #include "hw/usb/uhci-regs.h"
 
@@ -87,7 +85,7 @@
     qtest_add_func("/uhci/pci/hotplug/usb-storage", test_usb_storage_hotplug);
 
     qtest_start("-device piix3-usb-uhci,id=uhci,addr=1d.0"
-                " -drive id=drive0,if=none,file=/dev/null"
+                " -drive id=drive0,if=none,file=/dev/null,format=raw"
                 " -device usb-tablet,bus=uhci.0,port=1");
     ret = g_test_run();
     qtest_end();
diff --git a/tests/usb-hcd-xhci-test.c b/tests/usb-hcd-xhci-test.c
index b1a7dec..22513e9 100644
--- a/tests/usb-hcd-xhci-test.c
+++ b/tests/usb-hcd-xhci-test.c
@@ -1,16 +1,14 @@
 /*
  * QTest testcase for USB xHCI controller
  *
- * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO.,LTD.
+ * Copyright (c) 2014 HUAWEI TECHNOLOGIES CO., LTD.
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 #include "libqos/usb.h"
 
 
@@ -91,7 +89,7 @@
     qtest_add_func("/xhci/pci/hotplug/usb-uas", test_usb_uas_hotplug);
 
     qtest_start("-device nec-usb-xhci,id=xhci"
-                " -drive id=drive0,if=none,file=/dev/null");
+                " -drive id=drive0,if=none,file=/dev/null,format=raw");
     ret = g_test_run();
     qtest_end();
 
diff --git a/tests/vhost-user-bridge.c b/tests/vhost-user-bridge.c
new file mode 100644
index 0000000..775e031
--- /dev/null
+++ b/tests/vhost-user-bridge.c
@@ -0,0 +1,1438 @@
+/*
+ * Vhost User Bridge
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Victor Kaplansky <victork@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+/*
+ * TODO:
+ *     - main should get parameters from the command line.
+ *     - implement all request handlers. Still not implemented:
+ *          vubr_get_queue_num_exec()
+ *          vubr_send_rarp_exec()
+ *     - test for broken requests and virtqueue.
+ *     - implement features defined by Virtio 1.0 spec.
+ *     - support mergeable buffers and indirect descriptors.
+ *     - implement clean shutdown.
+ *     - implement non-blocking writes to UDP backend.
+ *     - implement polling strategy.
+ *     - implement clean starting/stopping of vq processing
+ *     - implement clean starting/stopping of used and buffers
+ *       dirty page logging.
+ */
+
+#define _FILE_OFFSET_BITS 64
+
+#include "qemu/osdep.h"
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/unistd.h>
+#include <sys/eventfd.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <linux/vhost.h>
+
+#include "qemu/atomic.h"
+#include "standard-headers/linux/virtio_net.h"
+#include "standard-headers/linux/virtio_ring.h"
+
+#define VHOST_USER_BRIDGE_DEBUG 1
+
+#define DPRINT(...) \
+    do { \
+        if (VHOST_USER_BRIDGE_DEBUG) { \
+            printf(__VA_ARGS__); \
+        } \
+    } while (0)
+
+typedef void (*CallbackFunc)(int sock, void *ctx);
+
+typedef struct Event {
+    void *ctx;
+    CallbackFunc callback;
+} Event;
+
+typedef struct Dispatcher {
+    int max_sock;
+    fd_set fdset;
+    Event events[FD_SETSIZE];
+} Dispatcher;
+
+static void
+vubr_die(const char *s)
+{
+    perror(s);
+    exit(1);
+}
+
+static int
+dispatcher_init(Dispatcher *dispr)
+{
+    FD_ZERO(&dispr->fdset);
+    dispr->max_sock = -1;
+    return 0;
+}
+
+static int
+dispatcher_add(Dispatcher *dispr, int sock, void *ctx, CallbackFunc cb)
+{
+    if (sock >= FD_SETSIZE) {
+        fprintf(stderr,
+                "Error: Failed to add new event. sock %d should be less than %d\n",
+                sock, FD_SETSIZE);
+        return -1;
+    }
+
+    dispr->events[sock].ctx = ctx;
+    dispr->events[sock].callback = cb;
+
+    FD_SET(sock, &dispr->fdset);
+    if (sock > dispr->max_sock) {
+        dispr->max_sock = sock;
+    }
+    DPRINT("Added sock %d for watching. max_sock: %d\n",
+           sock, dispr->max_sock);
+    return 0;
+}
+
+/* dispatcher_remove() is not currently in use but may be useful
+ * in the future. */
+static int
+dispatcher_remove(Dispatcher *dispr, int sock)
+{
+    if (sock >= FD_SETSIZE) {
+        fprintf(stderr,
+                "Error: Failed to remove event. sock %d should be less than %d\n",
+                sock, FD_SETSIZE);
+        return -1;
+    }
+
+    FD_CLR(sock, &dispr->fdset);
+    DPRINT("Sock %d removed from dispatcher watch.\n", sock);
+    return 0;
+}
+
+/* timeout in us */
+static int
+dispatcher_wait(Dispatcher *dispr, uint32_t timeout)
+{
+    struct timeval tv;
+    tv.tv_sec = timeout / 1000000;
+    tv.tv_usec = timeout % 1000000;
+
+    fd_set fdset = dispr->fdset;
+
+    /* wait until some of sockets become readable. */
+    int rc = select(dispr->max_sock + 1, &fdset, 0, 0, &tv);
+
+    if (rc == -1) {
+        vubr_die("select");
+    }
+
+    /* Timeout */
+    if (rc == 0) {
+        return 0;
+    }
+
+    /* Now call callback for every ready socket. */
+
+    int sock;
+    for (sock = 0; sock < dispr->max_sock + 1; sock++) {
+        /* The callback on a socket can remove other sockets from the
+         * dispatcher, thus we have to check that the socket is
+         * still not removed from dispatcher's list
+         */
+        if (FD_ISSET(sock, &fdset) && FD_ISSET(sock, &dispr->fdset)) {
+            Event *e = &dispr->events[sock];
+            e->callback(sock, e->ctx);
+        }
+    }
+
+    return 0;
+}
+
+typedef struct VubrVirtq {
+    int call_fd;
+    int kick_fd;
+    uint32_t size;
+    uint16_t last_avail_index;
+    uint16_t last_used_index;
+    struct vring_desc *desc;
+    struct vring_avail *avail;
+    struct vring_used *used;
+    uint64_t log_guest_addr;
+    int enable;
+} VubrVirtq;
+
+/* Based on qemu/hw/virtio/vhost-user.c */
+
+#define VHOST_MEMORY_MAX_NREGIONS    8
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+/* v1.0 compliant. */
+#define VIRTIO_F_VERSION_1		32
+
+#define VHOST_LOG_PAGE 4096
+
+enum VhostUserProtocolFeature {
+    VHOST_USER_PROTOCOL_F_MQ = 0,
+    VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
+    VHOST_USER_PROTOCOL_F_RARP = 2,
+
+    VHOST_USER_PROTOCOL_F_MAX
+};
+
+#define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
+
+typedef enum VhostUserRequest {
+    VHOST_USER_NONE = 0,
+    VHOST_USER_GET_FEATURES = 1,
+    VHOST_USER_SET_FEATURES = 2,
+    VHOST_USER_SET_OWNER = 3,
+    VHOST_USER_RESET_OWNER = 4,
+    VHOST_USER_SET_MEM_TABLE = 5,
+    VHOST_USER_SET_LOG_BASE = 6,
+    VHOST_USER_SET_LOG_FD = 7,
+    VHOST_USER_SET_VRING_NUM = 8,
+    VHOST_USER_SET_VRING_ADDR = 9,
+    VHOST_USER_SET_VRING_BASE = 10,
+    VHOST_USER_GET_VRING_BASE = 11,
+    VHOST_USER_SET_VRING_KICK = 12,
+    VHOST_USER_SET_VRING_CALL = 13,
+    VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_GET_QUEUE_NUM = 17,
+    VHOST_USER_SET_VRING_ENABLE = 18,
+    VHOST_USER_SEND_RARP = 19,
+    VHOST_USER_MAX
+} VhostUserRequest;
+
+typedef struct VhostUserMemoryRegion {
+    uint64_t guest_phys_addr;
+    uint64_t memory_size;
+    uint64_t userspace_addr;
+    uint64_t mmap_offset;
+} VhostUserMemoryRegion;
+
+typedef struct VhostUserMemory {
+    uint32_t nregions;
+    uint32_t padding;
+    VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+} VhostUserMemory;
+
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
+typedef struct VhostUserMsg {
+    VhostUserRequest request;
+
+#define VHOST_USER_VERSION_MASK     (0x3)
+#define VHOST_USER_REPLY_MASK       (0x1<<2)
+    uint32_t flags;
+    uint32_t size; /* the following payload size */
+    union {
+#define VHOST_USER_VRING_IDX_MASK   (0xff)
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
+        uint64_t u64;
+        struct vhost_vring_state state;
+        struct vhost_vring_addr addr;
+        VhostUserMemory memory;
+        VhostUserLog log;
+    } payload;
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    int fd_num;
+} QEMU_PACKED VhostUserMsg;
+
+#define VHOST_USER_HDR_SIZE offsetof(VhostUserMsg, payload.u64)
+
+/* The version of the protocol we support */
+#define VHOST_USER_VERSION    (0x1)
+
+#define MAX_NR_VIRTQUEUE (8)
+
+typedef struct VubrDevRegion {
+    /* Guest Physical address. */
+    uint64_t gpa;
+    /* Memory region size. */
+    uint64_t size;
+    /* QEMU virtual address (userspace). */
+    uint64_t qva;
+    /* Starting offset in our mmaped space. */
+    uint64_t mmap_offset;
+    /* Start address of mmaped space. */
+    uint64_t mmap_addr;
+} VubrDevRegion;
+
+typedef struct VubrDev {
+    int sock;
+    Dispatcher dispatcher;
+    uint32_t nregions;
+    VubrDevRegion regions[VHOST_MEMORY_MAX_NREGIONS];
+    VubrVirtq vq[MAX_NR_VIRTQUEUE];
+    int log_call_fd;
+    uint64_t log_size;
+    uint8_t *log_table;
+    int backend_udp_sock;
+    struct sockaddr_in backend_udp_dest;
+    int ready;
+    uint64_t features;
+    int hdrlen;
+} VubrDev;
+
+static const char *vubr_request_str[] = {
+    [VHOST_USER_NONE]                   =  "VHOST_USER_NONE",
+    [VHOST_USER_GET_FEATURES]           =  "VHOST_USER_GET_FEATURES",
+    [VHOST_USER_SET_FEATURES]           =  "VHOST_USER_SET_FEATURES",
+    [VHOST_USER_SET_OWNER]              =  "VHOST_USER_SET_OWNER",
+    [VHOST_USER_RESET_OWNER]           =  "VHOST_USER_RESET_OWNER",
+    [VHOST_USER_SET_MEM_TABLE]          =  "VHOST_USER_SET_MEM_TABLE",
+    [VHOST_USER_SET_LOG_BASE]           =  "VHOST_USER_SET_LOG_BASE",
+    [VHOST_USER_SET_LOG_FD]             =  "VHOST_USER_SET_LOG_FD",
+    [VHOST_USER_SET_VRING_NUM]          =  "VHOST_USER_SET_VRING_NUM",
+    [VHOST_USER_SET_VRING_ADDR]         =  "VHOST_USER_SET_VRING_ADDR",
+    [VHOST_USER_SET_VRING_BASE]         =  "VHOST_USER_SET_VRING_BASE",
+    [VHOST_USER_GET_VRING_BASE]         =  "VHOST_USER_GET_VRING_BASE",
+    [VHOST_USER_SET_VRING_KICK]         =  "VHOST_USER_SET_VRING_KICK",
+    [VHOST_USER_SET_VRING_CALL]         =  "VHOST_USER_SET_VRING_CALL",
+    [VHOST_USER_SET_VRING_ERR]          =  "VHOST_USER_SET_VRING_ERR",
+    [VHOST_USER_GET_PROTOCOL_FEATURES]  =  "VHOST_USER_GET_PROTOCOL_FEATURES",
+    [VHOST_USER_SET_PROTOCOL_FEATURES]  =  "VHOST_USER_SET_PROTOCOL_FEATURES",
+    [VHOST_USER_GET_QUEUE_NUM]          =  "VHOST_USER_GET_QUEUE_NUM",
+    [VHOST_USER_SET_VRING_ENABLE]       =  "VHOST_USER_SET_VRING_ENABLE",
+    [VHOST_USER_SEND_RARP]              =  "VHOST_USER_SEND_RARP",
+    [VHOST_USER_MAX]                    =  "VHOST_USER_MAX",
+};
+
+static void
+print_buffer(uint8_t *buf, size_t len)
+{
+    int i;
+    printf("Raw buffer:\n");
+    for (i = 0; i < len; i++) {
+        if (i % 16 == 0) {
+            printf("\n");
+        }
+        if (i % 4 == 0) {
+            printf("   ");
+        }
+        printf("%02x ", buf[i]);
+    }
+    printf("\n............................................................\n");
+}
+
+/* Translate guest physical address to our virtual address.  */
+static uint64_t
+gpa_to_va(VubrDev *dev, uint64_t guest_addr)
+{
+    int i;
+
+    /* Find matching memory region.  */
+    for (i = 0; i < dev->nregions; i++) {
+        VubrDevRegion *r = &dev->regions[i];
+
+        if ((guest_addr >= r->gpa) && (guest_addr < (r->gpa + r->size))) {
+            return guest_addr - r->gpa + r->mmap_addr + r->mmap_offset;
+        }
+    }
+
+    assert(!"address not found in regions");
+    return 0;
+}
+
+/* Translate qemu virtual address to our virtual address.  */
+static uint64_t
+qva_to_va(VubrDev *dev, uint64_t qemu_addr)
+{
+    int i;
+
+    /* Find matching memory region.  */
+    for (i = 0; i < dev->nregions; i++) {
+        VubrDevRegion *r = &dev->regions[i];
+
+        if ((qemu_addr >= r->qva) && (qemu_addr < (r->qva + r->size))) {
+            return qemu_addr - r->qva + r->mmap_addr + r->mmap_offset;
+        }
+    }
+
+    assert(!"address not found in regions");
+    return 0;
+}
+
+static void
+vubr_message_read(int conn_fd, VhostUserMsg *vmsg)
+{
+    char control[CMSG_SPACE(VHOST_MEMORY_MAX_NREGIONS * sizeof(int))] = { };
+    struct iovec iov = {
+        .iov_base = (char *)vmsg,
+        .iov_len = VHOST_USER_HDR_SIZE,
+    };
+    struct msghdr msg = {
+        .msg_iov = &iov,
+        .msg_iovlen = 1,
+        .msg_control = control,
+        .msg_controllen = sizeof(control),
+    };
+    size_t fd_size;
+    struct cmsghdr *cmsg;
+    int rc;
+
+    rc = recvmsg(conn_fd, &msg, 0);
+
+    if (rc == 0) {
+        vubr_die("recvmsg");
+        fprintf(stderr, "Peer disconnected.\n");
+        exit(1);
+    }
+    if (rc < 0) {
+        vubr_die("recvmsg");
+    }
+
+    vmsg->fd_num = 0;
+    for (cmsg = CMSG_FIRSTHDR(&msg);
+         cmsg != NULL;
+         cmsg = CMSG_NXTHDR(&msg, cmsg))
+    {
+        if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) {
+            fd_size = cmsg->cmsg_len - CMSG_LEN(0);
+            vmsg->fd_num = fd_size / sizeof(int);
+            memcpy(vmsg->fds, CMSG_DATA(cmsg), fd_size);
+            break;
+        }
+    }
+
+    if (vmsg->size > sizeof(vmsg->payload)) {
+        fprintf(stderr,
+                "Error: too big message request: %d, size: vmsg->size: %u, "
+                "while sizeof(vmsg->payload) = %zu\n",
+                vmsg->request, vmsg->size, sizeof(vmsg->payload));
+        exit(1);
+    }
+
+    if (vmsg->size) {
+        rc = read(conn_fd, &vmsg->payload, vmsg->size);
+        if (rc == 0) {
+            vubr_die("recvmsg");
+            fprintf(stderr, "Peer disconnected.\n");
+            exit(1);
+        }
+        if (rc < 0) {
+            vubr_die("recvmsg");
+        }
+
+        assert(rc == vmsg->size);
+    }
+}
+
+static void
+vubr_message_write(int conn_fd, VhostUserMsg *vmsg)
+{
+    int rc;
+
+    do {
+        rc = write(conn_fd, vmsg, VHOST_USER_HDR_SIZE + vmsg->size);
+    } while (rc < 0 && errno == EINTR);
+
+    if (rc < 0) {
+        vubr_die("write");
+    }
+}
+
+static void
+vubr_backend_udp_sendbuf(VubrDev *dev, uint8_t *buf, size_t len)
+{
+    int slen = sizeof(struct sockaddr_in);
+
+    if (sendto(dev->backend_udp_sock, buf, len, 0,
+               (struct sockaddr *) &dev->backend_udp_dest, slen) == -1) {
+        vubr_die("sendto()");
+    }
+}
+
+static int
+vubr_backend_udp_recvbuf(VubrDev *dev, uint8_t *buf, size_t buflen)
+{
+    int slen = sizeof(struct sockaddr_in);
+    int rc;
+
+    rc = recvfrom(dev->backend_udp_sock, buf, buflen, 0,
+                  (struct sockaddr *) &dev->backend_udp_dest,
+                  (socklen_t *)&slen);
+    if (rc == -1) {
+        vubr_die("recvfrom()");
+    }
+
+    return rc;
+}
+
+static void
+vubr_consume_raw_packet(VubrDev *dev, uint8_t *buf, uint32_t len)
+{
+    int hdrlen = dev->hdrlen;
+    DPRINT("    hdrlen = %d\n", dev->hdrlen);
+
+    if (VHOST_USER_BRIDGE_DEBUG) {
+        print_buffer(buf, len);
+    }
+    vubr_backend_udp_sendbuf(dev, buf + hdrlen, len - hdrlen);
+}
+
+/* Kick the log_call_fd if required. */
+static void
+vubr_log_kick(VubrDev *dev)
+{
+    if (dev->log_call_fd != -1) {
+        DPRINT("Kicking the QEMU's log...\n");
+        eventfd_write(dev->log_call_fd, 1);
+    }
+}
+
+/* Kick the guest if necessary. */
+static void
+vubr_virtqueue_kick(VubrVirtq *vq)
+{
+    if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT)) {
+        DPRINT("Kicking the guest...\n");
+        eventfd_write(vq->call_fd, 1);
+    }
+}
+
+static void
+vubr_log_page(uint8_t *log_table, uint64_t page)
+{
+    DPRINT("Logged dirty guest page: %"PRId64"\n", page);
+    atomic_or(&log_table[page / 8], 1 << (page % 8));
+}
+
+static void
+vubr_log_write(VubrDev *dev, uint64_t address, uint64_t length)
+{
+    uint64_t page;
+
+    if (!(dev->features & (1ULL << VHOST_F_LOG_ALL)) ||
+        !dev->log_table || !length) {
+        return;
+    }
+
+    assert(dev->log_size > ((address + length - 1) / VHOST_LOG_PAGE / 8));
+
+    page = address / VHOST_LOG_PAGE;
+    while (page * VHOST_LOG_PAGE < address + length) {
+        vubr_log_page(dev->log_table, page);
+        page += VHOST_LOG_PAGE;
+    }
+    vubr_log_kick(dev);
+}
+
+static void
+vubr_post_buffer(VubrDev *dev, VubrVirtq *vq, uint8_t *buf, int32_t len)
+{
+    struct vring_desc *desc = vq->desc;
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+    int32_t remaining_len = len;
+
+    unsigned int size = vq->size;
+
+    uint16_t avail_index = atomic_mb_read(&avail->idx);
+
+    /* We check the available descriptors before posting the
+     * buffer, so here we assume that enough available
+     * descriptors. */
+    assert(vq->last_avail_index != avail_index);
+    uint16_t a_index = vq->last_avail_index % size;
+    uint16_t u_index = vq->last_used_index % size;
+    uint16_t d_index = avail->ring[a_index];
+
+    int i = d_index;
+    uint32_t written_len = 0;
+
+    do {
+        DPRINT("Post packet to guest on vq:\n");
+        DPRINT("    size             = %d\n", vq->size);
+        DPRINT("    last_avail_index = %d\n", vq->last_avail_index);
+        DPRINT("    last_used_index  = %d\n", vq->last_used_index);
+        DPRINT("    a_index = %d\n", a_index);
+        DPRINT("    u_index = %d\n", u_index);
+        DPRINT("    d_index = %d\n", d_index);
+        DPRINT("    desc[%d].addr    = 0x%016"PRIx64"\n", i, desc[i].addr);
+        DPRINT("    desc[%d].len     = %d\n", i, desc[i].len);
+        DPRINT("    desc[%d].flags   = %d\n", i, desc[i].flags);
+        DPRINT("    avail->idx = %d\n", avail_index);
+        DPRINT("    used->idx  = %d\n", used->idx);
+
+        if (!(desc[i].flags & VRING_DESC_F_WRITE)) {
+            /* FIXME: we should find writable descriptor. */
+            fprintf(stderr, "Error: descriptor is not writable. Exiting.\n");
+            exit(1);
+        }
+
+        void *chunk_start = (void *)(uintptr_t)gpa_to_va(dev, desc[i].addr);
+        uint32_t chunk_len = desc[i].len;
+        uint32_t chunk_write_len = MIN(remaining_len, chunk_len);
+
+        memcpy(chunk_start, buf + written_len, chunk_write_len);
+        vubr_log_write(dev, desc[i].addr, chunk_write_len);
+        remaining_len -= chunk_write_len;
+        written_len += chunk_write_len;
+
+        if ((remaining_len == 0) || !(desc[i].flags & VRING_DESC_F_NEXT)) {
+            break;
+        }
+
+        i = desc[i].next;
+    } while (1);
+
+    if (remaining_len > 0) {
+            fprintf(stderr,
+                    "Too long packet for RX, remaining_len = %d, Dropping...\n",
+                    remaining_len);
+            return;
+    }
+
+    /* Add descriptor to the used ring. */
+    used->ring[u_index].id = d_index;
+    used->ring[u_index].len = len;
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, ring[u_index]),
+                   sizeof(used->ring[u_index]));
+
+    vq->last_avail_index++;
+    vq->last_used_index++;
+
+    atomic_mb_set(&used->idx, vq->last_used_index);
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, idx),
+                   sizeof(used->idx));
+
+    /* Kick the guest if necessary. */
+    vubr_virtqueue_kick(vq);
+}
+
+static int
+vubr_process_desc(VubrDev *dev, VubrVirtq *vq)
+{
+    struct vring_desc *desc = vq->desc;
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    unsigned int size = vq->size;
+
+    uint16_t a_index = vq->last_avail_index % size;
+    uint16_t u_index = vq->last_used_index % size;
+    uint16_t d_index = avail->ring[a_index];
+
+    uint32_t i, len = 0;
+    size_t buf_size = 4096;
+    uint8_t buf[4096];
+
+    DPRINT("Chunks: ");
+    i = d_index;
+    do {
+        void *chunk_start = (void *)(uintptr_t)gpa_to_va(dev, desc[i].addr);
+        uint32_t chunk_len = desc[i].len;
+
+        assert(!(desc[i].flags & VRING_DESC_F_WRITE));
+
+        if (len + chunk_len < buf_size) {
+            memcpy(buf + len, chunk_start, chunk_len);
+            DPRINT("%d ", chunk_len);
+        } else {
+            fprintf(stderr, "Error: too long packet. Dropping...\n");
+            break;
+        }
+
+        len += chunk_len;
+
+        if (!(desc[i].flags & VRING_DESC_F_NEXT)) {
+            break;
+        }
+
+        i = desc[i].next;
+    } while (1);
+    DPRINT("\n");
+
+    if (!len) {
+        return -1;
+    }
+
+    /* Add descriptor to the used ring. */
+    used->ring[u_index].id = d_index;
+    used->ring[u_index].len = len;
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, ring[u_index]),
+                   sizeof(used->ring[u_index]));
+
+    vubr_consume_raw_packet(dev, buf, len);
+
+    return 0;
+}
+
+static void
+vubr_process_avail(VubrDev *dev, VubrVirtq *vq)
+{
+    struct vring_avail *avail = vq->avail;
+    struct vring_used *used = vq->used;
+    uint64_t log_guest_addr = vq->log_guest_addr;
+
+    while (vq->last_avail_index != atomic_mb_read(&avail->idx)) {
+        vubr_process_desc(dev, vq);
+        vq->last_avail_index++;
+        vq->last_used_index++;
+    }
+
+    atomic_mb_set(&used->idx, vq->last_used_index);
+    vubr_log_write(dev,
+                   log_guest_addr + offsetof(struct vring_used, idx),
+                   sizeof(used->idx));
+}
+
+static void
+vubr_backend_recv_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    VubrVirtq *rx_vq = &dev->vq[0];
+    uint8_t buf[4096];
+    struct virtio_net_hdr_v1 *hdr = (struct virtio_net_hdr_v1 *)buf;
+    int hdrlen = dev->hdrlen;
+    int buflen = sizeof(buf);
+    int len;
+
+    if (!dev->ready) {
+        return;
+    }
+
+    DPRINT("\n\n   ***   IN UDP RECEIVE CALLBACK    ***\n\n");
+    DPRINT("    hdrlen = %d\n", hdrlen);
+
+    uint16_t avail_index = atomic_mb_read(&rx_vq->avail->idx);
+
+    /* If there is no available descriptors, just do nothing.
+     * The buffer will be handled by next arrived UDP packet,
+     * or next kick on receive virtq. */
+    if (rx_vq->last_avail_index == avail_index) {
+        DPRINT("Got UDP packet, but no available descriptors on RX virtq.\n");
+        return;
+    }
+
+    memset(buf, 0, hdrlen);
+    /* TODO: support mergeable buffers. */
+    if (hdrlen == 12)
+        hdr->num_buffers = 1;
+    len = vubr_backend_udp_recvbuf(dev, buf + hdrlen, buflen - hdrlen);
+
+    vubr_post_buffer(dev, rx_vq, buf, len + hdrlen);
+}
+
+static void
+vubr_kick_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    eventfd_t kick_data;
+    ssize_t rc;
+
+    rc = eventfd_read(sock, &kick_data);
+    if (rc == -1) {
+        vubr_die("eventfd_read()");
+    } else {
+        DPRINT("Got kick_data: %016"PRIx64"\n", kick_data);
+        vubr_process_avail(dev, &dev->vq[1]);
+    }
+}
+
+static int
+vubr_none_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_get_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vmsg->payload.u64 =
+            ((1ULL << VIRTIO_NET_F_MRG_RXBUF) |
+             (1ULL << VHOST_F_LOG_ALL) |
+             (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) |
+             (1ULL << VHOST_USER_F_PROTOCOL_FEATURES));
+
+    vmsg->size = sizeof(vmsg->payload.u64);
+
+    DPRINT("Sending back to guest u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    dev->features = vmsg->payload.u64;
+    if ((dev->features & (1ULL << VIRTIO_F_VERSION_1)) ||
+        (dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF))) {
+        dev->hdrlen = 12;
+    } else {
+        dev->hdrlen = 10;
+    }
+
+    return 0;
+}
+
+static int
+vubr_set_owner_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    return 0;
+}
+
+static void
+vubr_close_log(VubrDev *dev)
+{
+    if (dev->log_table) {
+        if (munmap(dev->log_table, dev->log_size) != 0) {
+            vubr_die("munmap()");
+        }
+
+        dev->log_table = 0;
+    }
+    if (dev->log_call_fd != -1) {
+        close(dev->log_call_fd);
+        dev->log_call_fd = -1;
+    }
+}
+
+static int
+vubr_reset_device_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vubr_close_log(dev);
+    dev->ready = 0;
+    dev->features = 0;
+    return 0;
+}
+
+static int
+vubr_set_mem_table_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    int i;
+    VhostUserMemory *memory = &vmsg->payload.memory;
+    dev->nregions = memory->nregions;
+
+    DPRINT("Nregions: %d\n", memory->nregions);
+    for (i = 0; i < dev->nregions; i++) {
+        void *mmap_addr;
+        VhostUserMemoryRegion *msg_region = &memory->regions[i];
+        VubrDevRegion *dev_region = &dev->regions[i];
+
+        DPRINT("Region %d\n", i);
+        DPRINT("    guest_phys_addr: 0x%016"PRIx64"\n",
+               msg_region->guest_phys_addr);
+        DPRINT("    memory_size:     0x%016"PRIx64"\n",
+               msg_region->memory_size);
+        DPRINT("    userspace_addr   0x%016"PRIx64"\n",
+               msg_region->userspace_addr);
+        DPRINT("    mmap_offset      0x%016"PRIx64"\n",
+               msg_region->mmap_offset);
+
+        dev_region->gpa = msg_region->guest_phys_addr;
+        dev_region->size = msg_region->memory_size;
+        dev_region->qva = msg_region->userspace_addr;
+        dev_region->mmap_offset = msg_region->mmap_offset;
+
+        /* We don't use offset argument of mmap() since the
+         * mapped address has to be page aligned, and we use huge
+         * pages.  */
+        mmap_addr = mmap(0, dev_region->size + dev_region->mmap_offset,
+                         PROT_READ | PROT_WRITE, MAP_SHARED,
+                         vmsg->fds[i], 0);
+
+        if (mmap_addr == MAP_FAILED) {
+            vubr_die("mmap");
+        }
+        dev_region->mmap_addr = (uint64_t)(uintptr_t)mmap_addr;
+        DPRINT("    mmap_addr:       0x%016"PRIx64"\n", dev_region->mmap_addr);
+
+        close(vmsg->fds[i]);
+    }
+
+    return 0;
+}
+
+static int
+vubr_set_log_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    int fd;
+    uint64_t log_mmap_size, log_mmap_offset;
+    void *rc;
+
+    assert(vmsg->fd_num == 1);
+    fd = vmsg->fds[0];
+
+    assert(vmsg->size == sizeof(vmsg->payload.log));
+    log_mmap_offset = vmsg->payload.log.mmap_offset;
+    log_mmap_size = vmsg->payload.log.mmap_size;
+    DPRINT("Log mmap_offset: %"PRId64"\n", log_mmap_offset);
+    DPRINT("Log mmap_size:   %"PRId64"\n", log_mmap_size);
+
+    rc = mmap(0, log_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
+              log_mmap_offset);
+    if (rc == MAP_FAILED) {
+        vubr_die("mmap");
+    }
+    dev->log_table = rc;
+    dev->log_size = log_mmap_size;
+
+    vmsg->size = sizeof(vmsg->payload.u64);
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_log_fd_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    assert(vmsg->fd_num == 1);
+    dev->log_call_fd = vmsg->fds[0];
+    DPRINT("Got log_call_fd: %d\n", vmsg->fds[0]);
+    return 0;
+}
+
+static int
+vubr_set_vring_num_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int num = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.num:   %d\n", num);
+    dev->vq[index].size = num;
+    return 0;
+}
+
+static int
+vubr_set_vring_addr_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    struct vhost_vring_addr *vra = &vmsg->payload.addr;
+    unsigned int index = vra->index;
+    VubrVirtq *vq = &dev->vq[index];
+
+    DPRINT("vhost_vring_addr:\n");
+    DPRINT("    index:  %d\n", vra->index);
+    DPRINT("    flags:  %d\n", vra->flags);
+    DPRINT("    desc_user_addr:   0x%016llx\n", vra->desc_user_addr);
+    DPRINT("    used_user_addr:   0x%016llx\n", vra->used_user_addr);
+    DPRINT("    avail_user_addr:  0x%016llx\n", vra->avail_user_addr);
+    DPRINT("    log_guest_addr:   0x%016llx\n", vra->log_guest_addr);
+
+    vq->desc = (struct vring_desc *)(uintptr_t)qva_to_va(dev, vra->desc_user_addr);
+    vq->used = (struct vring_used *)(uintptr_t)qva_to_va(dev, vra->used_user_addr);
+    vq->avail = (struct vring_avail *)(uintptr_t)qva_to_va(dev, vra->avail_user_addr);
+    vq->log_guest_addr = vra->log_guest_addr;
+
+    DPRINT("Setting virtq addresses:\n");
+    DPRINT("    vring_desc  at %p\n", vq->desc);
+    DPRINT("    vring_used  at %p\n", vq->used);
+    DPRINT("    vring_avail at %p\n", vq->avail);
+
+    vq->last_used_index = vq->used->idx;
+
+    if (vq->last_avail_index != vq->used->idx) {
+        DPRINT("Last avail index != used index: %d != %d, resuming",
+               vq->last_avail_index, vq->used->idx);
+        vq->last_avail_index = vq->used->idx;
+    }
+
+    return 0;
+}
+
+static int
+vubr_set_vring_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int num = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.num:   %d\n", num);
+    dev->vq[index].last_avail_index = num;
+
+    return 0;
+}
+
+static int
+vubr_get_vring_base_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+
+    DPRINT("State.index: %d\n", index);
+    vmsg->payload.state.num = dev->vq[index].last_avail_index;
+    vmsg->size = sizeof(vmsg->payload.state);
+    /* FIXME: this is a work-around for a bug in QEMU enabling
+     * too early vrings. When protocol features are enabled,
+     * we have to respect * VHOST_USER_SET_VRING_ENABLE request. */
+    dev->ready = 0;
+
+    if (dev->vq[index].call_fd != -1) {
+        close(dev->vq[index].call_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].call_fd);
+        dev->vq[index].call_fd = -1;
+    }
+    if (dev->vq[index].kick_fd != -1) {
+        close(dev->vq[index].kick_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].kick_fd);
+        dev->vq[index].kick_fd = -1;
+    }
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_vring_kick_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    uint64_t u64_arg = vmsg->payload.u64;
+    int index = u64_arg & VHOST_USER_VRING_IDX_MASK;
+
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+
+    assert((u64_arg & VHOST_USER_VRING_NOFD_MASK) == 0);
+    assert(vmsg->fd_num == 1);
+
+    if (dev->vq[index].kick_fd != -1) {
+        close(dev->vq[index].kick_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].kick_fd);
+    }
+    dev->vq[index].kick_fd = vmsg->fds[0];
+    DPRINT("Got kick_fd: %d for vq: %d\n", vmsg->fds[0], index);
+
+    if (index % 2 == 1) {
+        /* TX queue. */
+        dispatcher_add(&dev->dispatcher, dev->vq[index].kick_fd,
+                       dev, vubr_kick_cb);
+
+        DPRINT("Waiting for kicks on fd: %d for vq: %d\n",
+               dev->vq[index].kick_fd, index);
+    }
+    /* We temporarily use this hack to determine that both TX and RX
+     * queues are set up and ready for processing.
+     * FIXME: we need to rely in VHOST_USER_SET_VRING_ENABLE and
+     * actual kicks. */
+    if (dev->vq[0].kick_fd != -1 &&
+        dev->vq[1].kick_fd != -1) {
+        dev->ready = 1;
+        DPRINT("vhost-user-bridge is ready for processing queues.\n");
+    }
+    return 0;
+
+}
+
+static int
+vubr_set_vring_call_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    uint64_t u64_arg = vmsg->payload.u64;
+    int index = u64_arg & VHOST_USER_VRING_IDX_MASK;
+
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    assert((u64_arg & VHOST_USER_VRING_NOFD_MASK) == 0);
+    assert(vmsg->fd_num == 1);
+
+    if (dev->vq[index].call_fd != -1) {
+        close(dev->vq[index].call_fd);
+        dispatcher_remove(&dev->dispatcher, dev->vq[index].call_fd);
+    }
+    dev->vq[index].call_fd = vmsg->fds[0];
+    DPRINT("Got call_fd: %d for vq: %d\n", vmsg->fds[0], index);
+
+    return 0;
+}
+
+static int
+vubr_set_vring_err_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    return 0;
+}
+
+static int
+vubr_get_protocol_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    vmsg->payload.u64 = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    vmsg->size = sizeof(vmsg->payload.u64);
+
+    /* Reply */
+    return 1;
+}
+
+static int
+vubr_set_protocol_features_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    /* FIXME: unimplented */
+    DPRINT("u64: 0x%016"PRIx64"\n", vmsg->payload.u64);
+    return 0;
+}
+
+static int
+vubr_get_queue_num_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_set_vring_enable_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    unsigned int index = vmsg->payload.state.index;
+    unsigned int enable = vmsg->payload.state.num;
+
+    DPRINT("State.index: %d\n", index);
+    DPRINT("State.enable:   %d\n", enable);
+    dev->vq[index].enable = enable;
+    return 0;
+}
+
+static int
+vubr_send_rarp_exec(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    DPRINT("Function %s() not implemented yet.\n", __func__);
+    return 0;
+}
+
+static int
+vubr_execute_request(VubrDev *dev, VhostUserMsg *vmsg)
+{
+    /* Print out generic part of the request. */
+    DPRINT(
+           "==================   Vhost user message from QEMU   ==================\n");
+    DPRINT("Request: %s (%d)\n", vubr_request_str[vmsg->request],
+           vmsg->request);
+    DPRINT("Flags:   0x%x\n", vmsg->flags);
+    DPRINT("Size:    %d\n", vmsg->size);
+
+    if (vmsg->fd_num) {
+        int i;
+        DPRINT("Fds:");
+        for (i = 0; i < vmsg->fd_num; i++) {
+            DPRINT(" %d", vmsg->fds[i]);
+        }
+        DPRINT("\n");
+    }
+
+    switch (vmsg->request) {
+    case VHOST_USER_NONE:
+        return vubr_none_exec(dev, vmsg);
+    case VHOST_USER_GET_FEATURES:
+        return vubr_get_features_exec(dev, vmsg);
+    case VHOST_USER_SET_FEATURES:
+        return vubr_set_features_exec(dev, vmsg);
+    case VHOST_USER_SET_OWNER:
+        return vubr_set_owner_exec(dev, vmsg);
+    case VHOST_USER_RESET_OWNER:
+        return vubr_reset_device_exec(dev, vmsg);
+    case VHOST_USER_SET_MEM_TABLE:
+        return vubr_set_mem_table_exec(dev, vmsg);
+    case VHOST_USER_SET_LOG_BASE:
+        return vubr_set_log_base_exec(dev, vmsg);
+    case VHOST_USER_SET_LOG_FD:
+        return vubr_set_log_fd_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_NUM:
+        return vubr_set_vring_num_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ADDR:
+        return vubr_set_vring_addr_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_BASE:
+        return vubr_set_vring_base_exec(dev, vmsg);
+    case VHOST_USER_GET_VRING_BASE:
+        return vubr_get_vring_base_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_KICK:
+        return vubr_set_vring_kick_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_CALL:
+        return vubr_set_vring_call_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ERR:
+        return vubr_set_vring_err_exec(dev, vmsg);
+    case VHOST_USER_GET_PROTOCOL_FEATURES:
+        return vubr_get_protocol_features_exec(dev, vmsg);
+    case VHOST_USER_SET_PROTOCOL_FEATURES:
+        return vubr_set_protocol_features_exec(dev, vmsg);
+    case VHOST_USER_GET_QUEUE_NUM:
+        return vubr_get_queue_num_exec(dev, vmsg);
+    case VHOST_USER_SET_VRING_ENABLE:
+        return vubr_set_vring_enable_exec(dev, vmsg);
+    case VHOST_USER_SEND_RARP:
+        return vubr_send_rarp_exec(dev, vmsg);
+
+    case VHOST_USER_MAX:
+        assert(vmsg->request != VHOST_USER_MAX);
+    }
+    return 0;
+}
+
+static void
+vubr_receive_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *) ctx;
+    VhostUserMsg vmsg;
+    int reply_requested;
+
+    vubr_message_read(sock, &vmsg);
+    reply_requested = vubr_execute_request(dev, &vmsg);
+    if (reply_requested) {
+        /* Set the version in the flags when sending the reply */
+        vmsg.flags &= ~VHOST_USER_VERSION_MASK;
+        vmsg.flags |= VHOST_USER_VERSION;
+        vmsg.flags |= VHOST_USER_REPLY_MASK;
+        vubr_message_write(sock, &vmsg);
+    }
+}
+
+static void
+vubr_accept_cb(int sock, void *ctx)
+{
+    VubrDev *dev = (VubrDev *)ctx;
+    int conn_fd;
+    struct sockaddr_un un;
+    socklen_t len = sizeof(un);
+
+    conn_fd = accept(sock, (struct sockaddr *) &un, &len);
+    if (conn_fd == -1) {
+        vubr_die("accept()");
+    }
+    DPRINT("Got connection from remote peer on sock %d\n", conn_fd);
+    dispatcher_add(&dev->dispatcher, conn_fd, ctx, vubr_receive_cb);
+}
+
+static VubrDev *
+vubr_new(const char *path, bool client)
+{
+    VubrDev *dev = (VubrDev *) calloc(1, sizeof(VubrDev));
+    dev->nregions = 0;
+    int i;
+    struct sockaddr_un un;
+    CallbackFunc cb;
+    size_t len;
+
+    for (i = 0; i < MAX_NR_VIRTQUEUE; i++) {
+        dev->vq[i] = (VubrVirtq) {
+            .call_fd = -1, .kick_fd = -1,
+            .size = 0,
+            .last_avail_index = 0, .last_used_index = 0,
+            .desc = 0, .avail = 0, .used = 0,
+            .enable = 0,
+        };
+    }
+
+    /* Init log */
+    dev->log_call_fd = -1;
+    dev->log_size = 0;
+    dev->log_table = 0;
+    dev->ready = 0;
+    dev->features = 0;
+
+    /* Get a UNIX socket. */
+    dev->sock = socket(AF_UNIX, SOCK_STREAM, 0);
+    if (dev->sock == -1) {
+        vubr_die("socket");
+    }
+
+    un.sun_family = AF_UNIX;
+    strcpy(un.sun_path, path);
+    len = sizeof(un.sun_family) + strlen(path);
+
+    if (!client) {
+        unlink(path);
+
+        if (bind(dev->sock, (struct sockaddr *) &un, len) == -1) {
+            vubr_die("bind");
+        }
+
+        if (listen(dev->sock, 1) == -1) {
+            vubr_die("listen");
+        }
+        cb = vubr_accept_cb;
+
+        DPRINT("Waiting for connections on UNIX socket %s ...\n", path);
+    } else {
+        if (connect(dev->sock, (struct sockaddr *)&un, len) == -1) {
+            vubr_die("connect");
+        }
+        cb = vubr_receive_cb;
+    }
+
+    dispatcher_init(&dev->dispatcher);
+    dispatcher_add(&dev->dispatcher, dev->sock, (void *)dev, cb);
+
+    return dev;
+}
+
+static void
+vubr_set_host(struct sockaddr_in *saddr, const char *host)
+{
+    if (isdigit(host[0])) {
+        if (!inet_aton(host, &saddr->sin_addr)) {
+            fprintf(stderr, "inet_aton() failed.\n");
+            exit(1);
+        }
+    } else {
+        struct hostent *he = gethostbyname(host);
+
+        if (!he) {
+            fprintf(stderr, "gethostbyname() failed.\n");
+            exit(1);
+        }
+        saddr->sin_addr = *(struct in_addr *)he->h_addr;
+    }
+}
+
+static void
+vubr_backend_udp_setup(VubrDev *dev,
+                       const char *local_host,
+                       const char *local_port,
+                       const char *remote_host,
+                       const char *remote_port)
+{
+    int sock;
+    const char *r;
+
+    int lport, rport;
+
+    lport = strtol(local_port, (char **)&r, 0);
+    if (r == local_port) {
+        fprintf(stderr, "lport parsing failed.\n");
+        exit(1);
+    }
+
+    rport = strtol(remote_port, (char **)&r, 0);
+    if (r == remote_port) {
+        fprintf(stderr, "rport parsing failed.\n");
+        exit(1);
+    }
+
+    struct sockaddr_in si_local = {
+        .sin_family = AF_INET,
+        .sin_port = htons(lport),
+    };
+
+    vubr_set_host(&si_local, local_host);
+
+    /* setup destination for sends */
+    dev->backend_udp_dest = (struct sockaddr_in) {
+        .sin_family = AF_INET,
+        .sin_port = htons(rport),
+    };
+    vubr_set_host(&dev->backend_udp_dest, remote_host);
+
+    sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
+    if (sock == -1) {
+        vubr_die("socket");
+    }
+
+    if (bind(sock, (struct sockaddr *)&si_local, sizeof(si_local)) == -1) {
+        vubr_die("bind");
+    }
+
+    dev->backend_udp_sock = sock;
+    dispatcher_add(&dev->dispatcher, sock, dev, vubr_backend_recv_cb);
+    DPRINT("Waiting for data from udp backend on %s:%d...\n",
+           local_host, lport);
+}
+
+static void
+vubr_run(VubrDev *dev)
+{
+    while (1) {
+        /* timeout 200ms */
+        dispatcher_wait(&dev->dispatcher, 200000);
+        /* Here one can try polling strategy. */
+    }
+}
+
+static int
+vubr_parse_host_port(const char **host, const char **port, const char *buf)
+{
+    char *p = strchr(buf, ':');
+
+    if (!p) {
+        return -1;
+    }
+    *p = '\0';
+    *host = strdup(buf);
+    *port = strdup(p + 1);
+    return 0;
+}
+
+#define DEFAULT_UD_SOCKET "/tmp/vubr.sock"
+#define DEFAULT_LHOST "127.0.0.1"
+#define DEFAULT_LPORT "4444"
+#define DEFAULT_RHOST "127.0.0.1"
+#define DEFAULT_RPORT "5555"
+
+static const char *ud_socket_path = DEFAULT_UD_SOCKET;
+static const char *lhost = DEFAULT_LHOST;
+static const char *lport = DEFAULT_LPORT;
+static const char *rhost = DEFAULT_RHOST;
+static const char *rport = DEFAULT_RPORT;
+
+int
+main(int argc, char *argv[])
+{
+    VubrDev *dev;
+    int opt;
+    bool client = false;
+
+    while ((opt = getopt(argc, argv, "l:r:u:c")) != -1) {
+
+        switch (opt) {
+        case 'l':
+            if (vubr_parse_host_port(&lhost, &lport, optarg) < 0) {
+                goto out;
+            }
+            break;
+        case 'r':
+            if (vubr_parse_host_port(&rhost, &rport, optarg) < 0) {
+                goto out;
+            }
+            break;
+        case 'u':
+            ud_socket_path = strdup(optarg);
+            break;
+        case 'c':
+            client = true;
+            break;
+        default:
+            goto out;
+        }
+    }
+
+    DPRINT("ud socket: %s (%s)\n", ud_socket_path,
+           client ? "client" : "server");
+    DPRINT("local:     %s:%s\n", lhost, lport);
+    DPRINT("remote:    %s:%s\n", rhost, rport);
+
+    dev = vubr_new(ud_socket_path, client);
+    if (!dev) {
+        return 1;
+    }
+
+    vubr_backend_udp_setup(dev, lhost, lport, rhost, rport);
+    vubr_run(dev);
+    return 0;
+
+out:
+    fprintf(stderr, "Usage: %s ", argv[0]);
+    fprintf(stderr, "[-c] [-u ud_socket_path] [-l lhost:lport] [-r rhost:rport]\n");
+    fprintf(stderr, "\t-u path to unix doman socket. default: %s\n",
+            DEFAULT_UD_SOCKET);
+    fprintf(stderr, "\t-l local host and port. default: %s:%s\n",
+            DEFAULT_LHOST, DEFAULT_LPORT);
+    fprintf(stderr, "\t-r remote host and port. default: %s:%s\n",
+            DEFAULT_RHOST, DEFAULT_RPORT);
+    fprintf(stderr, "\t-c client mode\n");
+
+    return 1;
+}
diff --git a/tests/vhost-user-test.c b/tests/vhost-user-test.c
index 75fedf0..27b10c1 100644
--- a/tests/vhost-user-test.c
+++ b/tests/vhost-user-test.c
@@ -8,18 +8,17 @@
  *
  */
 
-#define QEMU_GLIB_COMPAT_H
-#include <glib.h>
+#include "qemu/osdep.h"
 
 #include "libqtest.h"
 #include "qemu/option.h"
+#include "qemu/range.h"
+#include "qemu/sockets.h"
 #include "sysemu/char.h"
 #include "sysemu/sysemu.h"
 
 #include <linux/vhost.h>
-#include <sys/mman.h>
 #include <sys/vfs.h>
-#include <qemu/sockets.h>
 
 /* GLIB version compatibility flags */
 #if !GLIB_CHECK_VERSION(2, 26, 0)
@@ -30,22 +29,15 @@
 #define HAVE_MONOTONIC_TIME
 #endif
 
-#if GLIB_CHECK_VERSION(2, 32, 0)
-#define HAVE_MUTEX_INIT
-#define HAVE_COND_INIT
-#define HAVE_THREAD_NEW
-#endif
-
 #define QEMU_CMD_ACCEL  " -machine accel=tcg"
-#define QEMU_CMD_MEM    " -m 512 -object memory-backend-file,id=mem,size=512M,"\
+#define QEMU_CMD_MEM    " -m %d -object memory-backend-file,id=mem,size=%dM,"\
                         "mem-path=%s,share=on -numa node,memdev=mem"
-#define QEMU_CMD_CHR    " -chardev socket,id=chr0,path=%s"
-#define QEMU_CMD_NETDEV " -netdev vhost-user,id=net0,chardev=chr0,vhostforce"
-#define QEMU_CMD_NET    " -device virtio-net-pci,netdev=net0 "
-#define QEMU_CMD_ROM    " -option-rom ../pc-bios/pxe-virtio.rom"
+#define QEMU_CMD_CHR    " -chardev socket,id=%s,path=%s%s"
+#define QEMU_CMD_NETDEV " -netdev vhost-user,id=net0,chardev=%s,vhostforce"
+#define QEMU_CMD_NET    " -device virtio-net-pci,netdev=net0,romfile=./pc-bios/pxe-virtio.rom"
 
 #define QEMU_CMD        QEMU_CMD_ACCEL QEMU_CMD_MEM QEMU_CMD_CHR \
-                        QEMU_CMD_NETDEV QEMU_CMD_NET QEMU_CMD_ROM
+                        QEMU_CMD_NETDEV QEMU_CMD_NET
 
 #define HUGETLBFS_MAGIC       0x958458f6
 
@@ -53,6 +45,11 @@
 
 #define VHOST_MEMORY_MAX_NREGIONS    8
 
+#define VHOST_USER_F_PROTOCOL_FEATURES 30
+#define VHOST_USER_PROTOCOL_F_LOG_SHMFD 1
+
+#define VHOST_LOG_PAGE 0x1000
+
 typedef enum VhostUserRequest {
     VHOST_USER_NONE = 0,
     VHOST_USER_GET_FEATURES = 1,
@@ -69,6 +66,9 @@
     VHOST_USER_SET_VRING_KICK = 12,
     VHOST_USER_SET_VRING_CALL = 13,
     VHOST_USER_SET_VRING_ERR = 14,
+    VHOST_USER_GET_PROTOCOL_FEATURES = 15,
+    VHOST_USER_SET_PROTOCOL_FEATURES = 16,
+    VHOST_USER_SET_VRING_ENABLE = 18,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -85,6 +85,11 @@
     VhostUserMemoryRegion regions[VHOST_MEMORY_MAX_NREGIONS];
 } VhostUserMemory;
 
+typedef struct VhostUserLog {
+    uint64_t mmap_size;
+    uint64_t mmap_offset;
+} VhostUserLog;
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
@@ -93,11 +98,14 @@
     uint32_t flags;
     uint32_t size; /* the following payload size */
     union {
+#define VHOST_USER_VRING_IDX_MASK   (0xff)
+#define VHOST_USER_VRING_NOFD_MASK  (0x1<<8)
         uint64_t u64;
         struct vhost_vring_state state;
         struct vhost_vring_addr addr;
         VhostUserMemory memory;
-    };
+        VhostUserLog log;
+    } payload;
 } QEMU_PACKED VhostUserMsg;
 
 static VhostUserMsg m __attribute__ ((unused));
@@ -111,154 +119,91 @@
 #define VHOST_USER_VERSION    (0x1)
 /*****************************************************************************/
 
-int fds_num = 0, fds[VHOST_MEMORY_MAX_NREGIONS];
-static VhostUserMemory memory;
-static GMutex *data_mutex;
-static GCond *data_cond;
+typedef struct TestServer {
+    gchar *socket_path;
+    gchar *mig_path;
+    gchar *chr_name;
+    CharDriverState *chr;
+    int fds_num;
+    int fds[VHOST_MEMORY_MAX_NREGIONS];
+    VhostUserMemory memory;
+    CompatGMutex data_mutex;
+    CompatGCond data_cond;
+    int log_fd;
+    uint64_t rings;
+} TestServer;
 
-static gint64 _get_time(void)
+static const char *tmpfs;
+static const char *root;
+
+static void wait_for_fds(TestServer *s)
 {
-#ifdef HAVE_MONOTONIC_TIME
-    return g_get_monotonic_time();
-#else
-    GTimeVal time;
-    g_get_current_time(&time);
-
-    return time.tv_sec * G_TIME_SPAN_SECOND + time.tv_usec;
-#endif
-}
-
-static GMutex *_mutex_new(void)
-{
-    GMutex *mutex;
-
-#ifdef HAVE_MUTEX_INIT
-    mutex = g_new(GMutex, 1);
-    g_mutex_init(mutex);
-#else
-    mutex = g_mutex_new();
-#endif
-
-    return mutex;
-}
-
-static void _mutex_free(GMutex *mutex)
-{
-#ifdef HAVE_MUTEX_INIT
-    g_mutex_clear(mutex);
-    g_free(mutex);
-#else
-    g_mutex_free(mutex);
-#endif
-}
-
-static GCond *_cond_new(void)
-{
-    GCond *cond;
-
-#ifdef HAVE_COND_INIT
-    cond = g_new(GCond, 1);
-    g_cond_init(cond);
-#else
-    cond = g_cond_new();
-#endif
-
-    return cond;
-}
-
-static gboolean _cond_wait_until(GCond *cond, GMutex *mutex, gint64 end_time)
-{
-    gboolean ret = FALSE;
-#ifdef HAVE_COND_INIT
-    ret = g_cond_wait_until(cond, mutex, end_time);
-#else
-    GTimeVal time = { end_time / G_TIME_SPAN_SECOND,
-                      end_time % G_TIME_SPAN_SECOND };
-    ret = g_cond_timed_wait(cond, mutex, &time);
-#endif
-    return ret;
-}
-
-static void _cond_free(GCond *cond)
-{
-#ifdef HAVE_COND_INIT
-    g_cond_clear(cond);
-    g_free(cond);
-#else
-    g_cond_free(cond);
-#endif
-}
-
-static GThread *_thread_new(const gchar *name, GThreadFunc func, gpointer data)
-{
-    GThread *thread = NULL;
-    GError *error = NULL;
-#ifdef HAVE_THREAD_NEW
-    thread = g_thread_try_new(name, func, data, &error);
-#else
-    thread = g_thread_create(func, data, TRUE, &error);
-#endif
-    return thread;
-}
-
-static void read_guest_mem(void)
-{
-    uint32_t *guest_mem;
     gint64 end_time;
-    int i, j;
-    size_t size;
 
-    g_mutex_lock(data_mutex);
+    g_mutex_lock(&s->data_mutex);
 
-    end_time = _get_time() + 5 * G_TIME_SPAN_SECOND;
-    while (!fds_num) {
-        if (!_cond_wait_until(data_cond, data_mutex, end_time)) {
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (!s->fds_num) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
             /* timeout has passed */
-            g_assert(fds_num);
+            g_assert(s->fds_num);
             break;
         }
     }
 
     /* check for sanity */
-    g_assert_cmpint(fds_num, >, 0);
-    g_assert_cmpint(fds_num, ==, memory.nregions);
+    g_assert_cmpint(s->fds_num, >, 0);
+    g_assert_cmpint(s->fds_num, ==, s->memory.nregions);
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static void read_guest_mem(const void *data)
+{
+    TestServer *s = (void *)data;
+    uint32_t *guest_mem;
+    int i, j;
+    size_t size;
+
+    wait_for_fds(s);
+
+    g_mutex_lock(&s->data_mutex);
 
     /* iterate all regions */
-    for (i = 0; i < fds_num; i++) {
+    for (i = 0; i < s->fds_num; i++) {
 
         /* We'll check only the region statring at 0x0*/
-        if (memory.regions[i].guest_phys_addr != 0x0) {
+        if (s->memory.regions[i].guest_phys_addr != 0x0) {
             continue;
         }
 
-        g_assert_cmpint(memory.regions[i].memory_size, >, 1024);
+        g_assert_cmpint(s->memory.regions[i].memory_size, >, 1024);
 
-        size =  memory.regions[i].memory_size + memory.regions[i].mmap_offset;
+        size = s->memory.regions[i].memory_size +
+            s->memory.regions[i].mmap_offset;
 
         guest_mem = mmap(0, size, PROT_READ | PROT_WRITE,
-                         MAP_SHARED, fds[i], 0);
+                         MAP_SHARED, s->fds[i], 0);
 
         g_assert(guest_mem != MAP_FAILED);
-        guest_mem += (memory.regions[i].mmap_offset / sizeof(*guest_mem));
+        guest_mem += (s->memory.regions[i].mmap_offset / sizeof(*guest_mem));
 
         for (j = 0; j < 256; j++) {
-            uint32_t a = readl(memory.regions[i].guest_phys_addr + j*4);
+            uint32_t a = readl(s->memory.regions[i].guest_phys_addr + j*4);
             uint32_t b = guest_mem[j];
 
             g_assert_cmpint(a, ==, b);
         }
 
-        munmap(guest_mem, memory.regions[i].memory_size);
+        munmap(guest_mem, s->memory.regions[i].memory_size);
     }
 
-    g_assert_cmpint(1, ==, 1);
-    g_mutex_unlock(data_mutex);
+    g_mutex_unlock(&s->data_mutex);
 }
 
 static void *thread_function(void *data)
 {
-    GMainLoop *loop;
-    loop = g_main_loop_new(NULL, FALSE);
+    GMainLoop *loop = data;
     g_main_loop_run(loop);
     return NULL;
 }
@@ -270,7 +215,8 @@
 
 static void chr_read(void *opaque, const uint8_t *buf, int size)
 {
-    CharDriverState *chr = opaque;
+    TestServer *s = opaque;
+    CharDriverState *chr = s->chr;
     VhostUserMsg msg;
     uint8_t *p = (uint8_t *) &msg;
     int fd;
@@ -280,20 +226,40 @@
         return;
     }
 
-    g_mutex_lock(data_mutex);
+    g_mutex_lock(&s->data_mutex);
     memcpy(p, buf, VHOST_USER_HDR_SIZE);
 
     if (msg.size) {
         p += VHOST_USER_HDR_SIZE;
-        qemu_chr_fe_read_all(chr, p, msg.size);
+        size = qemu_chr_fe_read_all(chr, p, msg.size);
+        if (size != msg.size) {
+            g_test_message("Wrong message size received %d != %d\n",
+                           size, msg.size);
+            return;
+        }
     }
 
     switch (msg.request) {
     case VHOST_USER_GET_FEATURES:
         /* send back features to qemu */
         msg.flags |= VHOST_USER_REPLY_MASK;
-        msg.size = sizeof(m.u64);
-        msg.u64 = 0;
+        msg.size = sizeof(m.payload.u64);
+        msg.payload.u64 = 0x1ULL << VHOST_F_LOG_ALL |
+            0x1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
+        p = (uint8_t *) &msg;
+        qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
+        break;
+
+    case VHOST_USER_SET_FEATURES:
+	g_assert_cmpint(msg.payload.u64 & (0x1ULL << VHOST_USER_F_PROTOCOL_FEATURES),
+			!=, 0ULL);
+        break;
+
+    case VHOST_USER_GET_PROTOCOL_FEATURES:
+        /* send back features to qemu */
+        msg.flags |= VHOST_USER_REPLY_MASK;
+        msg.size = sizeof(m.payload.u64);
+        msg.payload.u64 = 1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD;
         p = (uint8_t *) &msg;
         qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
         break;
@@ -301,19 +267,22 @@
     case VHOST_USER_GET_VRING_BASE:
         /* send back vring base to qemu */
         msg.flags |= VHOST_USER_REPLY_MASK;
-        msg.size = sizeof(m.state);
-        msg.state.num = 0;
+        msg.size = sizeof(m.payload.state);
+        msg.payload.state.num = 0;
         p = (uint8_t *) &msg;
         qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE + msg.size);
+
+        assert(msg.payload.state.index < 2);
+        s->rings &= ~(0x1ULL << msg.payload.state.index);
         break;
 
     case VHOST_USER_SET_MEM_TABLE:
         /* received the mem table */
-        memcpy(&memory, &msg.memory, sizeof(msg.memory));
-        fds_num = qemu_chr_fe_get_msgfds(chr, fds, sizeof(fds) / sizeof(int));
+        memcpy(&s->memory, &msg.payload.memory, sizeof(msg.payload.memory));
+        s->fds_num = qemu_chr_fe_get_msgfds(chr, s->fds, G_N_ELEMENTS(s->fds));
 
         /* signal the test that it can continue */
-        g_cond_signal(data_cond);
+        g_cond_signal(&s->data_cond);
         break;
 
     case VHOST_USER_SET_VRING_KICK:
@@ -327,23 +296,38 @@
          */
         qemu_set_nonblock(fd);
         break;
+
+    case VHOST_USER_SET_LOG_BASE:
+        if (s->log_fd != -1) {
+            close(s->log_fd);
+            s->log_fd = -1;
+        }
+        qemu_chr_fe_get_msgfds(chr, &s->log_fd, 1);
+        msg.flags |= VHOST_USER_REPLY_MASK;
+        msg.size = 0;
+        p = (uint8_t *) &msg;
+        qemu_chr_fe_write_all(chr, p, VHOST_USER_HDR_SIZE);
+
+        g_cond_signal(&s->data_cond);
+        break;
+
+    case VHOST_USER_SET_VRING_BASE:
+        assert(msg.payload.state.index < 2);
+        s->rings |= 0x1ULL << msg.payload.state.index;
+        break;
+
     default:
         break;
     }
-    g_mutex_unlock(data_mutex);
+
+    g_mutex_unlock(&s->data_mutex);
 }
 
-static const char *init_hugepagefs(void)
+static const char *init_hugepagefs(const char *path)
 {
-    const char *path;
     struct statfs fs;
     int ret;
 
-    path = getenv("QTEST_HUGETLBFS_PATH");
-    if (!path) {
-        path = "/hugetlbfs";
-    }
-
     if (access(path, R_OK | W_OK | X_OK)) {
         g_test_message("access on path (%s): %s\n", path, strerror(errno));
         return NULL;
@@ -366,44 +350,392 @@
     return path;
 }
 
+static TestServer *test_server_new(const gchar *name)
+{
+    TestServer *server = g_new0(TestServer, 1);
+
+    server->socket_path = g_strdup_printf("%s/%s.sock", tmpfs, name);
+    server->mig_path = g_strdup_printf("%s/%s.mig", tmpfs, name);
+    server->chr_name = g_strdup_printf("chr-%s", name);
+
+    g_mutex_init(&server->data_mutex);
+    g_cond_init(&server->data_cond);
+
+    server->log_fd = -1;
+
+    return server;
+}
+
+static void test_server_create_chr(TestServer *server, const gchar *opt)
+{
+    gchar *chr_path;
+
+    chr_path = g_strdup_printf("unix:%s%s", server->socket_path, opt);
+    server->chr = qemu_chr_new(server->chr_name, chr_path, NULL);
+    g_free(chr_path);
+
+    qemu_chr_add_handlers(server->chr, chr_can_read, chr_read, NULL, server);
+}
+
+static void test_server_listen(TestServer *server)
+{
+    test_server_create_chr(server, ",server,nowait");
+}
+
+static inline void test_server_connect(TestServer *server)
+{
+    test_server_create_chr(server, ",reconnect=1");
+}
+
+#define GET_QEMU_CMD(s)                                         \
+    g_strdup_printf(QEMU_CMD, 512, 512, (root), (s)->chr_name,  \
+                    (s)->socket_path, "", (s)->chr_name)
+
+#define GET_QEMU_CMDE(s, mem, chr_opts, extra, ...)                     \
+    g_strdup_printf(QEMU_CMD extra, (mem), (mem), (root), (s)->chr_name, \
+                    (s)->socket_path, (chr_opts), (s)->chr_name, ##__VA_ARGS__)
+
+static gboolean _test_server_free(TestServer *server)
+{
+    int i;
+
+    qemu_chr_delete(server->chr);
+
+    for (i = 0; i < server->fds_num; i++) {
+        close(server->fds[i]);
+    }
+
+    if (server->log_fd != -1) {
+        close(server->log_fd);
+    }
+
+    unlink(server->socket_path);
+    g_free(server->socket_path);
+
+    unlink(server->mig_path);
+    g_free(server->mig_path);
+
+    g_free(server->chr_name);
+    g_free(server);
+
+    return FALSE;
+}
+
+static void test_server_free(TestServer *server)
+{
+    g_idle_add((GSourceFunc)_test_server_free, server);
+}
+
+static void wait_for_log_fd(TestServer *s)
+{
+    gint64 end_time;
+
+    g_mutex_lock(&s->data_mutex);
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (s->log_fd == -1) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
+            /* timeout has passed */
+            g_assert(s->log_fd != -1);
+            break;
+        }
+    }
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static void write_guest_mem(TestServer *s, uint32_t seed)
+{
+    uint32_t *guest_mem;
+    int i, j;
+    size_t size;
+
+    wait_for_fds(s);
+
+    /* iterate all regions */
+    for (i = 0; i < s->fds_num; i++) {
+
+        /* We'll write only the region statring at 0x0 */
+        if (s->memory.regions[i].guest_phys_addr != 0x0) {
+            continue;
+        }
+
+        g_assert_cmpint(s->memory.regions[i].memory_size, >, 1024);
+
+        size = s->memory.regions[i].memory_size +
+            s->memory.regions[i].mmap_offset;
+
+        guest_mem = mmap(0, size, PROT_READ | PROT_WRITE,
+                         MAP_SHARED, s->fds[i], 0);
+
+        g_assert(guest_mem != MAP_FAILED);
+        guest_mem += (s->memory.regions[i].mmap_offset / sizeof(*guest_mem));
+
+        for (j = 0; j < 256; j++) {
+            guest_mem[j] = seed + j;
+        }
+
+        munmap(guest_mem, s->memory.regions[i].memory_size);
+        break;
+    }
+}
+
+static guint64 get_log_size(TestServer *s)
+{
+    guint64 log_size = 0;
+    int i;
+
+    for (i = 0; i < s->memory.nregions; ++i) {
+        VhostUserMemoryRegion *reg = &s->memory.regions[i];
+        guint64 last = range_get_last(reg->guest_phys_addr,
+                                       reg->memory_size);
+        log_size = MAX(log_size, last / (8 * VHOST_LOG_PAGE) + 1);
+    }
+
+    return log_size;
+}
+
+typedef struct TestMigrateSource {
+    GSource source;
+    TestServer *src;
+    TestServer *dest;
+} TestMigrateSource;
+
+static gboolean
+test_migrate_source_check(GSource *source)
+{
+    TestMigrateSource *t = (TestMigrateSource *)source;
+    gboolean overlap = t->src->rings && t->dest->rings;
+
+    g_assert(!overlap);
+
+    return FALSE;
+}
+
+#if !GLIB_CHECK_VERSION(2,36,0)
+/* this callback is unnecessary with glib >2.36, the default
+ * prepare for the source does the same */
+static gboolean
+test_migrate_source_prepare(GSource *source, gint *timeout)
+{
+    *timeout = -1;
+    return FALSE;
+}
+#endif
+
+GSourceFuncs test_migrate_source_funcs = {
+#if !GLIB_CHECK_VERSION(2,36,0)
+    .prepare = test_migrate_source_prepare,
+#endif
+    .check = test_migrate_source_check,
+};
+
+static void test_migrate(void)
+{
+    TestServer *s = test_server_new("src");
+    TestServer *dest = test_server_new("dest");
+    char *uri = g_strdup_printf("%s%s", "unix:", dest->mig_path);
+    QTestState *global = global_qtest, *from, *to;
+    GSource *source;
+    gchar *cmd;
+    QDict *rsp;
+    guint8 *log;
+    guint64 size;
+
+    test_server_listen(s);
+    test_server_listen(dest);
+
+    cmd = GET_QEMU_CMDE(s, 2, "", "");
+    from = qtest_start(cmd);
+    g_free(cmd);
+
+    wait_for_fds(s);
+    size = get_log_size(s);
+    g_assert_cmpint(size, ==, (2 * 1024 * 1024) / (VHOST_LOG_PAGE * 8));
+
+    cmd = GET_QEMU_CMDE(dest, 2, "", " -incoming %s", uri);
+    to = qtest_init(cmd);
+    g_free(cmd);
+
+    source = g_source_new(&test_migrate_source_funcs,
+                          sizeof(TestMigrateSource));
+    ((TestMigrateSource *)source)->src = s;
+    ((TestMigrateSource *)source)->dest = dest;
+    g_source_attach(source, NULL);
+
+    /* slow down migration to have time to fiddle with log */
+    /* TODO: qtest could learn to break on some places */
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 10 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    cmd = g_strdup_printf("{ 'execute': 'migrate',"
+                          "'arguments': { 'uri': '%s' } }",
+                          uri);
+    rsp = qmp(cmd);
+    g_free(cmd);
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    wait_for_log_fd(s);
+
+    log = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, s->log_fd, 0);
+    g_assert(log != MAP_FAILED);
+
+    /* modify first page */
+    write_guest_mem(s, 0x42);
+    log[0] = 1;
+    munmap(log, size);
+
+    /* speed things up */
+    rsp = qmp("{ 'execute': 'migrate_set_speed',"
+              "'arguments': { 'value': 0 } }");
+    g_assert(qdict_haskey(rsp, "return"));
+    QDECREF(rsp);
+
+    qmp_eventwait("STOP");
+
+    global_qtest = to;
+    qmp_eventwait("RESUME");
+
+    read_guest_mem(dest);
+
+    g_source_destroy(source);
+    g_source_unref(source);
+
+    qtest_quit(to);
+    test_server_free(dest);
+    qtest_quit(from);
+    test_server_free(s);
+    g_free(uri);
+
+    global_qtest = global;
+}
+
+#ifdef CONFIG_HAS_GLIB_SUBPROCESS_TESTS
+static void wait_for_rings_started(TestServer *s, size_t count)
+{
+    gint64 end_time;
+
+    g_mutex_lock(&s->data_mutex);
+    end_time = g_get_monotonic_time() + 5 * G_TIME_SPAN_SECOND;
+    while (ctpop64(s->rings) != count) {
+        if (!g_cond_wait_until(&s->data_cond, &s->data_mutex, end_time)) {
+            /* timeout has passed */
+            g_assert_cmpint(ctpop64(s->rings), ==, count);
+            break;
+        }
+    }
+
+    g_mutex_unlock(&s->data_mutex);
+}
+
+static gboolean
+reconnect_cb(gpointer user_data)
+{
+    TestServer *s = user_data;
+
+    qemu_chr_disconnect(s->chr);
+
+    return FALSE;
+}
+
+static gpointer
+connect_thread(gpointer data)
+{
+    TestServer *s = data;
+
+    /* wait for qemu to start before first try, to avoid extra warnings */
+    g_usleep(G_USEC_PER_SEC);
+    test_server_connect(s);
+
+    return NULL;
+}
+
+static void test_reconnect_subprocess(void)
+{
+    TestServer *s = test_server_new("reconnect");
+    char *cmd;
+
+    g_thread_new("connect", connect_thread, s);
+    cmd = GET_QEMU_CMDE(s, 2, ",server", "");
+    qtest_start(cmd);
+    g_free(cmd);
+
+    wait_for_fds(s);
+    wait_for_rings_started(s, 2);
+
+    /* reconnect */
+    s->fds_num = 0;
+    s->rings = 0;
+    g_idle_add(reconnect_cb, s);
+    wait_for_fds(s);
+    wait_for_rings_started(s, 2);
+
+    qtest_end();
+    test_server_free(s);
+    return;
+}
+
+static void test_reconnect(void)
+{
+    gchar *path = g_strdup_printf("/%s/vhost-user/reconnect/subprocess",
+                                  qtest_get_arch());
+    g_test_trap_subprocess(path, 0, 0);
+    g_test_trap_assert_passed();
+    g_free(path);
+}
+#endif
+
 int main(int argc, char **argv)
 {
     QTestState *s = NULL;
-    CharDriverState *chr = NULL;
-    const char *hugefs = 0;
-    char *socket_path = 0;
-    char *qemu_cmd = 0;
-    char *chr_path = 0;
+    TestServer *server = NULL;
+    const char *hugefs;
+    char *qemu_cmd = NULL;
     int ret;
+    char template[] = "/tmp/vhost-test-XXXXXX";
+    GMainLoop *loop;
+    GThread *thread;
 
     g_test_init(&argc, &argv, NULL);
 
     module_call_init(MODULE_INIT_QOM);
+    qemu_add_opts(&qemu_chardev_opts);
 
-    hugefs = init_hugepagefs();
-    if (!hugefs) {
-        return 0;
+    tmpfs = mkdtemp(template);
+    if (!tmpfs) {
+        g_test_message("mkdtemp on path (%s): %s\n", template, strerror(errno));
+    }
+    g_assert(tmpfs);
+
+    hugefs = getenv("QTEST_HUGETLBFS_PATH");
+    if (hugefs) {
+        root = init_hugepagefs(hugefs);
+        g_assert(root);
+    } else {
+        root = tmpfs;
     }
 
-    socket_path = g_strdup_printf("/tmp/vhost-%d.sock", getpid());
+    server = test_server_new("test");
+    test_server_listen(server);
 
-    /* create char dev and add read handlers */
-    qemu_add_opts(&qemu_chardev_opts);
-    chr_path = g_strdup_printf("unix:%s,server,nowait", socket_path);
-    chr = qemu_chr_new("chr0", chr_path, NULL);
-    g_free(chr_path);
-    qemu_chr_add_handlers(chr, chr_can_read, chr_read, NULL, chr);
-
+    loop = g_main_loop_new(NULL, FALSE);
     /* run the main loop thread so the chardev may operate */
-    data_mutex = _mutex_new();
-    data_cond = _cond_new();
-    _thread_new(NULL, thread_function, NULL);
+    thread = g_thread_new(NULL, thread_function, loop);
 
-    qemu_cmd = g_strdup_printf(QEMU_CMD, hugefs, socket_path);
+    qemu_cmd = GET_QEMU_CMD(server);
+
     s = qtest_start(qemu_cmd);
     g_free(qemu_cmd);
 
-    qtest_add_func("/vhost-user/read-guest-mem", read_guest_mem);
+    qtest_add_data_func("/vhost-user/read-guest-mem", server, read_guest_mem);
+    qtest_add_func("/vhost-user/migrate", test_migrate);
+#ifdef CONFIG_HAS_GLIB_SUBPROCESS_TESTS
+    qtest_add_func("/vhost-user/reconnect/subprocess",
+                   test_reconnect_subprocess);
+    qtest_add_func("/vhost-user/reconnect", test_reconnect);
+#endif
 
     ret = g_test_run();
 
@@ -412,10 +744,22 @@
     }
 
     /* cleanup */
-    unlink(socket_path);
-    g_free(socket_path);
-    _cond_free(data_cond);
-    _mutex_free(data_mutex);
+    test_server_free(server);
+
+    /* finish the helper thread and dispatch pending sources */
+    g_main_loop_quit(loop);
+    g_thread_join(thread);
+    while (g_main_context_pending(NULL)) {
+        g_main_context_iteration (NULL, TRUE);
+    }
+    g_main_loop_unref(loop);
+
+    ret = rmdir(tmpfs);
+    if (ret != 0) {
+        g_test_message("unable to rmdir: path (%s): %s\n",
+                       tmpfs, strerror(errno));
+    }
+    g_assert_cmpint(ret, ==, 0);
 
     return ret;
 }
diff --git a/tests/virtio-9p-test.c b/tests/virtio-9p-test.c
index 1fae477..1e39335 100644
--- a/tests/virtio-9p-test.c
+++ b/tests/virtio-9p-test.c
@@ -7,12 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdlib.h>
-#include <string.h>
-#include <glib.h>
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "qemu-common.h"
-#include "qemu/osdep.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/tests/virtio-balloon-test.c b/tests/virtio-balloon-test.c
index becebb5..0d0046b 100644
--- a/tests/virtio-balloon-test.c
+++ b/tests/virtio-balloon-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/tests/virtio-blk-test.c b/tests/virtio-blk-test.c
index ead3911..811cf75 100644
--- a/tests/virtio-blk-test.c
+++ b/tests/virtio-blk-test.c
@@ -8,44 +8,32 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "libqtest.h"
 #include "libqos/virtio.h"
 #include "libqos/virtio-pci.h"
+#include "libqos/virtio-mmio.h"
 #include "libqos/pci-pc.h"
 #include "libqos/malloc.h"
 #include "libqos/malloc-pc.h"
+#include "libqos/malloc-generic.h"
 #include "qemu/bswap.h"
-
-#define QVIRTIO_BLK_F_BARRIER       0x00000001
-#define QVIRTIO_BLK_F_SIZE_MAX      0x00000002
-#define QVIRTIO_BLK_F_SEG_MAX       0x00000004
-#define QVIRTIO_BLK_F_GEOMETRY      0x00000010
-#define QVIRTIO_BLK_F_RO            0x00000020
-#define QVIRTIO_BLK_F_BLK_SIZE      0x00000040
-#define QVIRTIO_BLK_F_SCSI          0x00000080
-#define QVIRTIO_BLK_F_WCE           0x00000200
-#define QVIRTIO_BLK_F_TOPOLOGY      0x00000400
-#define QVIRTIO_BLK_F_CONFIG_WCE    0x00000800
-
-#define QVIRTIO_BLK_T_IN            0
-#define QVIRTIO_BLK_T_OUT           1
-#define QVIRTIO_BLK_T_SCSI_CMD      2
-#define QVIRTIO_BLK_T_SCSI_CMD_OUT  3
-#define QVIRTIO_BLK_T_FLUSH         4
-#define QVIRTIO_BLK_T_FLUSH_OUT     5
-#define QVIRTIO_BLK_T_GET_ID        8
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_config.h"
+#include "standard-headers/linux/virtio_ring.h"
+#include "standard-headers/linux/virtio_blk.h"
+#include "standard-headers/linux/virtio_pci.h"
 
 #define TEST_IMAGE_SIZE         (64 * 1024 * 1024)
 #define QVIRTIO_BLK_TIMEOUT_US  (30 * 1000 * 1000)
+#define PCI_SLOT_HP             0x06
 #define PCI_SLOT                0x04
 #define PCI_FN                  0x00
 
-#define PCI_SLOT_HP             0x06
+#define MMIO_PAGE_SIZE          4096
+#define MMIO_DEV_BASE_ADDR      0x0A003E00
+#define MMIO_RAM_ADDR           0x40000000
+#define MMIO_RAM_SIZE           0x20000000
 
 typedef struct QVirtioBlkReq {
     uint32_t type;
@@ -55,11 +43,10 @@
     uint8_t status;
 } QVirtioBlkReq;
 
-static QPCIBus *test_start(void)
+static char *drive_create(void)
 {
-    char *cmdline;
-    char tmp_path[] = "/tmp/qtest.XXXXXX";
     int fd, ret;
+    char *tmp_path = g_strdup("/tmp/qtest.XXXXXX");
 
     /* Create a temporary raw image */
     fd = mkstemp(tmp_path);
@@ -68,30 +55,58 @@
     g_assert_cmpint(ret, ==, 0);
     close(fd);
 
-    cmdline = g_strdup_printf("-drive if=none,id=drive0,file=%s "
-                              "-drive if=none,id=drive1,file=/dev/null "
-                              "-device virtio-blk-pci,id=drv0,drive=drive0,"
-                              "addr=%x.%x",
-                              tmp_path, PCI_SLOT, PCI_FN);
+    return tmp_path;
+}
+
+static QPCIBus *pci_test_start(void)
+{
+    char *cmdline;
+    char *tmp_path;
+
+    tmp_path = drive_create();
+
+    cmdline = g_strdup_printf("-drive if=none,id=drive0,file=%s,format=raw "
+                        "-drive if=none,id=drive1,file=/dev/null,format=raw "
+                        "-device virtio-blk-pci,id=drv0,drive=drive0,"
+                        "addr=%x.%x",
+                        tmp_path, PCI_SLOT, PCI_FN);
     qtest_start(cmdline);
     unlink(tmp_path);
+    g_free(tmp_path);
     g_free(cmdline);
 
     return qpci_init_pc();
 }
 
+static void arm_test_start(void)
+{
+    char *cmdline;
+    char *tmp_path;
+
+    tmp_path = drive_create();
+
+    cmdline = g_strdup_printf("-machine virt "
+                                "-drive if=none,id=drive0,file=%s,format=raw "
+                                "-device virtio-blk-device,drive=drive0",
+                                tmp_path);
+    qtest_start(cmdline);
+    unlink(tmp_path);
+    g_free(tmp_path);
+    g_free(cmdline);
+}
+
 static void test_end(void)
 {
     qtest_end();
 }
 
-static QVirtioPCIDevice *virtio_blk_init(QPCIBus *bus, int slot)
+static QVirtioPCIDevice *virtio_blk_pci_init(QPCIBus *bus, int slot)
 {
     QVirtioPCIDevice *dev;
 
-    dev = qvirtio_pci_device_find(bus, QVIRTIO_BLK_DEVICE_ID);
+    dev = qvirtio_pci_device_find(bus, VIRTIO_ID_BLOCK);
     g_assert(dev != NULL);
-    g_assert_cmphex(dev->vdev.device_type, ==, QVIRTIO_BLK_DEVICE_ID);
+    g_assert_cmphex(dev->vdev.device_type, ==, VIRTIO_ID_BLOCK);
     g_assert_cmphex(dev->pdev->devfn, ==, ((slot << 3) | PCI_FN));
 
     qvirtio_pci_device_enable(dev);
@@ -135,14 +150,10 @@
     return addr;
 }
 
-static void pci_basic(void)
+static void test_basic(const QVirtioBus *bus, QVirtioDevice *dev,
+            QGuestAllocator *alloc, QVirtQueue *vq, uint64_t device_specific)
 {
-    QVirtioPCIDevice *dev;
-    QPCIBus *bus;
-    QVirtQueuePCI *vqpci;
-    QGuestAllocator *alloc;
     QVirtioBlkReq req;
-    void *addr;
     uint64_t req_addr;
     uint64_t capacity;
     uint32_t features;
@@ -150,133 +161,148 @@
     uint8_t status;
     char *data;
 
-    bus = test_start();
+    capacity = qvirtio_config_readq(bus, dev, device_specific);
 
-    dev = virtio_blk_init(bus, PCI_SLOT);
-
-    /* MSI-X is not enabled */
-    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_NO_MSIX;
-
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
     g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
 
-    features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
+    features = qvirtio_get_features(bus, dev);
     features = features & ~(QVIRTIO_F_BAD_FEATURE |
-                    QVIRTIO_F_RING_INDIRECT_DESC | QVIRTIO_F_RING_EVENT_IDX |
-                            QVIRTIO_BLK_F_SCSI);
-    qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
+                    (1u << VIRTIO_RING_F_INDIRECT_DESC) |
+                    (1u << VIRTIO_RING_F_EVENT_IDX) |
+                    (1u << VIRTIO_BLK_F_SCSI));
+    qvirtio_set_features(bus, dev, features);
+
+    qvirtio_set_driver_ok(bus, dev);
+
+    /* Write and read with 3 descriptor layout */
+    /* Write request */
+    req.type = VIRTIO_BLK_T_OUT;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+    strcpy(req.data, "TEST");
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(vq, req_addr, 16, false, true);
+    qvirtqueue_add(vq, req_addr + 16, 512, false, true);
+    qvirtqueue_add(vq, req_addr + 528, 1, true, false);
+
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    guest_free(alloc, req_addr);
+
+    /* Read request */
+    req.type = VIRTIO_BLK_T_IN;
+    req.ioprio = 1;
+    req.sector = 0;
+    req.data = g_malloc0(512);
+
+    req_addr = virtio_blk_request(alloc, &req, 512);
+
+    g_free(req.data);
+
+    free_head = qvirtqueue_add(vq, req_addr, 16, false, true);
+    qvirtqueue_add(vq, req_addr + 16, 512, true, true);
+    qvirtqueue_add(vq, req_addr + 528, 1, true, false);
+
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_BLK_TIMEOUT_US);
+    status = readb(req_addr + 528);
+    g_assert_cmpint(status, ==, 0);
+
+    data = g_malloc0(512);
+    memread(req_addr + 16, data, 512);
+    g_assert_cmpstr(data, ==, "TEST");
+    g_free(data);
+
+    guest_free(alloc, req_addr);
+
+    if (features & (1u << VIRTIO_F_ANY_LAYOUT)) {
+        /* Write and read with 2 descriptor layout */
+        /* Write request */
+        req.type = VIRTIO_BLK_T_OUT;
+        req.ioprio = 1;
+        req.sector = 1;
+        req.data = g_malloc0(512);
+        strcpy(req.data, "TEST");
+
+        req_addr = virtio_blk_request(alloc, &req, 512);
+
+        g_free(req.data);
+
+        free_head = qvirtqueue_add(vq, req_addr, 528, false, true);
+        qvirtqueue_add(vq, req_addr + 528, 1, true, false);
+        qvirtqueue_kick(bus, dev, vq, free_head);
+
+        qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_BLK_TIMEOUT_US);
+        status = readb(req_addr + 528);
+        g_assert_cmpint(status, ==, 0);
+
+        guest_free(alloc, req_addr);
+
+        /* Read request */
+        req.type = VIRTIO_BLK_T_IN;
+        req.ioprio = 1;
+        req.sector = 1;
+        req.data = g_malloc0(512);
+
+        req_addr = virtio_blk_request(alloc, &req, 512);
+
+        g_free(req.data);
+
+        free_head = qvirtqueue_add(vq, req_addr, 16, false, true);
+        qvirtqueue_add(vq, req_addr + 16, 513, true, false);
+
+        qvirtqueue_kick(bus, dev, vq, free_head);
+
+        qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_BLK_TIMEOUT_US);
+        status = readb(req_addr + 528);
+        g_assert_cmpint(status, ==, 0);
+
+        data = g_malloc0(512);
+        memread(req_addr + 16, data, 512);
+        g_assert_cmpstr(data, ==, "TEST");
+        g_free(data);
+
+        guest_free(alloc, req_addr);
+    }
+}
+
+static void pci_basic(void)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *vqpci;
+    QGuestAllocator *alloc;
+    void *addr;
+
+    bus = pci_test_start();
+    dev = virtio_blk_pci_init(bus, PCI_SLOT);
 
     alloc = pc_alloc_init();
     vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
                                                                     alloc, 0);
 
-    qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
+    /* MSI-X is not enabled */
+    addr = dev->addr + VIRTIO_PCI_CONFIG_OFF(false);
 
-    /* Write and read with 2 descriptor layout */
-    /* Write request */
-    req.type = QVIRTIO_BLK_T_OUT;
-    req.ioprio = 1;
-    req.sector = 0;
-    req.data = g_malloc0(512);
-    strcpy(req.data, "TEST");
-
-    req_addr = virtio_blk_request(alloc, &req, 512);
-
-    g_free(req.data);
-
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
-    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
-
-    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
-                           QVIRTIO_BLK_TIMEOUT_US);
-    status = readb(req_addr + 528);
-    g_assert_cmpint(status, ==, 0);
-
-    guest_free(alloc, req_addr);
-
-    /* Read request */
-    req.type = QVIRTIO_BLK_T_IN;
-    req.ioprio = 1;
-    req.sector = 0;
-    req.data = g_malloc0(512);
-
-    req_addr = virtio_blk_request(alloc, &req, 512);
-
-    g_free(req.data);
-
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 16, 513, true, false);
-
-    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
-
-    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
-                           QVIRTIO_BLK_TIMEOUT_US);
-    status = readb(req_addr + 528);
-    g_assert_cmpint(status, ==, 0);
-
-    data = g_malloc0(512);
-    memread(req_addr + 16, data, 512);
-    g_assert_cmpstr(data, ==, "TEST");
-    g_free(data);
-
-    guest_free(alloc, req_addr);
-
-    /* Write and read with 3 descriptor layout */
-    /* Write request */
-    req.type = QVIRTIO_BLK_T_OUT;
-    req.ioprio = 1;
-    req.sector = 1;
-    req.data = g_malloc0(512);
-    strcpy(req.data, "TEST");
-
-    req_addr = virtio_blk_request(alloc, &req, 512);
-
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
-
-    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
-
-    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
-                           QVIRTIO_BLK_TIMEOUT_US);
-    status = readb(req_addr + 528);
-    g_assert_cmpint(status, ==, 0);
-
-    guest_free(alloc, req_addr);
-
-    /* Read request */
-    req.type = QVIRTIO_BLK_T_IN;
-    req.ioprio = 1;
-    req.sector = 1;
-    req.data = g_malloc0(512);
-
-    req_addr = virtio_blk_request(alloc, &req, 512);
-
-    g_free(req.data);
-
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, true, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
-
-    qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
-
-    qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
-                           QVIRTIO_BLK_TIMEOUT_US);
-    status = readb(req_addr + 528);
-    g_assert_cmpint(status, ==, 0);
-
-    data = g_malloc0(512);
-    memread(req_addr + 16, data, 512);
-    g_assert_cmpstr(data, ==, "TEST");
-    g_free(data);
-
-    guest_free(alloc, req_addr);
+    test_basic(&qvirtio_pci, &dev->vdev, alloc, &vqpci->vq,
+                                                    (uint64_t)(uintptr_t)addr);
 
     /* End test */
-    guest_free(alloc, vqpci->vq.desc);
+    qvirtqueue_cleanup(&qvirtio_pci, &vqpci->vq, alloc);
+    pc_alloc_uninit(alloc);
     qvirtio_pci_device_disable(dev);
     g_free(dev);
+    qpci_free_pc(bus);
     test_end();
 }
 
@@ -296,20 +322,22 @@
     uint8_t status;
     char *data;
 
-    bus = test_start();
+    bus = pci_test_start();
 
-    dev = virtio_blk_init(bus, PCI_SLOT);
+    dev = virtio_blk_pci_init(bus, PCI_SLOT);
 
     /* MSI-X is not enabled */
-    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_NO_MSIX;
+    addr = dev->addr + VIRTIO_PCI_CONFIG_OFF(false);
 
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev,
+                                                    (uint64_t)(uintptr_t)addr);
     g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
 
     features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
-    g_assert_cmphex(features & QVIRTIO_F_RING_INDIRECT_DESC, !=, 0);
-    features = features & ~(QVIRTIO_F_BAD_FEATURE | QVIRTIO_F_RING_EVENT_IDX |
-                                                            QVIRTIO_BLK_F_SCSI);
+    g_assert_cmphex(features & (1u << VIRTIO_RING_F_INDIRECT_DESC), !=, 0);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                            (1u << VIRTIO_RING_F_EVENT_IDX) |
+                            (1u << VIRTIO_BLK_F_SCSI));
     qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
 
     alloc = pc_alloc_init();
@@ -318,7 +346,7 @@
     qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
 
     /* Write request */
-    req.type = QVIRTIO_BLK_T_OUT;
+    req.type = VIRTIO_BLK_T_OUT;
     req.ioprio = 1;
     req.sector = 0;
     req.data = g_malloc0(512);
@@ -343,7 +371,7 @@
     guest_free(alloc, req_addr);
 
     /* Read request */
-    req.type = QVIRTIO_BLK_T_IN;
+    req.type = VIRTIO_BLK_T_IN;
     req.ioprio = 1;
     req.sector = 0;
     req.data = g_malloc0(512);
@@ -373,9 +401,11 @@
     guest_free(alloc, req_addr);
 
     /* End test */
-    guest_free(alloc, vqpci->vq.desc);
+    qvirtqueue_cleanup(&qvirtio_pci, &vqpci->vq, alloc);
+    pc_alloc_uninit(alloc);
     qvirtio_pci_device_disable(dev);
     g_free(dev);
+    qpci_free_pc(bus);
     test_end();
 }
 
@@ -387,14 +417,15 @@
     void *addr;
     uint64_t capacity;
 
-    bus = test_start();
+    bus = pci_test_start();
 
-    dev = virtio_blk_init(bus, PCI_SLOT);
+    dev = virtio_blk_pci_init(bus, PCI_SLOT);
 
     /* MSI-X is not enabled */
-    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_NO_MSIX;
+    addr = dev->addr + VIRTIO_PCI_CONFIG_OFF(false);
 
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev,
+                                                    (uint64_t)(uintptr_t)addr);
     g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
 
     qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
@@ -403,11 +434,13 @@
                                                     " 'size': %d } }", n_size);
     qvirtio_wait_config_isr(&qvirtio_pci, &dev->vdev, QVIRTIO_BLK_TIMEOUT_US);
 
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev,
+                                                    (uint64_t)(uintptr_t)addr);
     g_assert_cmpint(capacity, ==, n_size / 512);
 
     qvirtio_pci_device_disable(dev);
     g_free(dev);
+    qpci_free_pc(bus);
     test_end();
 }
 
@@ -427,24 +460,26 @@
     uint8_t status;
     char *data;
 
-    bus = test_start();
+    bus = pci_test_start();
     alloc = pc_alloc_init();
 
-    dev = virtio_blk_init(bus, PCI_SLOT);
+    dev = virtio_blk_pci_init(bus, PCI_SLOT);
     qpci_msix_enable(dev->pdev);
 
     qvirtio_pci_set_msix_configuration_vector(dev, alloc, 0);
 
     /* MSI-X is enabled */
-    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_MSIX;
+    addr = dev->addr + VIRTIO_PCI_CONFIG_OFF(true);
 
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev,
+                                                    (uint64_t)(uintptr_t)addr);
     g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
 
     features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
     features = features & ~(QVIRTIO_F_BAD_FEATURE |
-                            QVIRTIO_F_RING_INDIRECT_DESC |
-                            QVIRTIO_F_RING_EVENT_IDX | QVIRTIO_BLK_F_SCSI);
+                            (1u << VIRTIO_RING_F_INDIRECT_DESC) |
+                            (1u << VIRTIO_RING_F_EVENT_IDX) |
+                            (1u << VIRTIO_BLK_F_SCSI));
     qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
 
     vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
@@ -458,11 +493,12 @@
 
     qvirtio_wait_config_isr(&qvirtio_pci, &dev->vdev, QVIRTIO_BLK_TIMEOUT_US);
 
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev,
+                                                    (uint64_t)(uintptr_t)addr);
     g_assert_cmpint(capacity, ==, n_size / 512);
 
     /* Write request */
-    req.type = QVIRTIO_BLK_T_OUT;
+    req.type = VIRTIO_BLK_T_OUT;
     req.ioprio = 1;
     req.sector = 0;
     req.data = g_malloc0(512);
@@ -472,7 +508,8 @@
 
     g_free(req.data);
 
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, false, true);
     qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
     qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
 
@@ -485,7 +522,7 @@
     guest_free(alloc, req_addr);
 
     /* Read request */
-    req.type = QVIRTIO_BLK_T_IN;
+    req.type = VIRTIO_BLK_T_IN;
     req.ioprio = 1;
     req.sector = 0;
     req.data = g_malloc0(512);
@@ -495,7 +532,8 @@
     g_free(req.data);
 
     free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 16, 513, true, false);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, true, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
 
     qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
 
@@ -514,10 +552,12 @@
     guest_free(alloc, req_addr);
 
     /* End test */
-    guest_free(alloc, (uint64_t)vqpci->vq.desc);
+    qvirtqueue_cleanup(&qvirtio_pci, &vqpci->vq, alloc);
+    pc_alloc_uninit(alloc);
     qpci_msix_disable(dev->pdev);
     qvirtio_pci_device_disable(dev);
     g_free(dev);
+    qpci_free_pc(bus);
     test_end();
 }
 
@@ -536,24 +576,26 @@
     uint8_t status;
     char *data;
 
-    bus = test_start();
+    bus = pci_test_start();
     alloc = pc_alloc_init();
 
-    dev = virtio_blk_init(bus, PCI_SLOT);
+    dev = virtio_blk_pci_init(bus, PCI_SLOT);
     qpci_msix_enable(dev->pdev);
 
     qvirtio_pci_set_msix_configuration_vector(dev, alloc, 0);
 
     /* MSI-X is enabled */
-    addr = dev->addr + QVIRTIO_DEVICE_SPECIFIC_MSIX;
+    addr = dev->addr + VIRTIO_PCI_CONFIG_OFF(true);
 
-    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev, addr);
+    capacity = qvirtio_config_readq(&qvirtio_pci, &dev->vdev,
+                                                    (uint64_t)(uintptr_t)addr);
     g_assert_cmpint(capacity, ==, TEST_IMAGE_SIZE / 512);
 
     features = qvirtio_get_features(&qvirtio_pci, &dev->vdev);
     features = features & ~(QVIRTIO_F_BAD_FEATURE |
-                            QVIRTIO_F_RING_INDIRECT_DESC |
-                            QVIRTIO_F_NOTIFY_ON_EMPTY | QVIRTIO_BLK_F_SCSI);
+                            (1u << VIRTIO_RING_F_INDIRECT_DESC) |
+                            (1u << VIRTIO_F_NOTIFY_ON_EMPTY) |
+                            (1u << VIRTIO_BLK_F_SCSI));
     qvirtio_set_features(&qvirtio_pci, &dev->vdev, features);
 
     vqpci = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
@@ -563,7 +605,7 @@
     qvirtio_set_driver_ok(&qvirtio_pci, &dev->vdev);
 
     /* Write request */
-    req.type = QVIRTIO_BLK_T_OUT;
+    req.type = VIRTIO_BLK_T_OUT;
     req.ioprio = 1;
     req.sector = 0;
     req.data = g_malloc0(512);
@@ -573,7 +615,8 @@
 
     g_free(req.data);
 
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, false, true);
     qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
     qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
 
@@ -581,7 +624,7 @@
                            QVIRTIO_BLK_TIMEOUT_US);
 
     /* Write request */
-    req.type = QVIRTIO_BLK_T_OUT;
+    req.type = VIRTIO_BLK_T_OUT;
     req.ioprio = 1;
     req.sector = 1;
     req.data = g_malloc0(512);
@@ -593,7 +636,8 @@
 
     /* Notify after processing the third request */
     qvirtqueue_set_used_event(&vqpci->vq, 2);
-    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 528, false, true);
+    free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, false, true);
     qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
     qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
 
@@ -606,7 +650,7 @@
     guest_free(alloc, req_addr);
 
     /* Read request */
-    req.type = QVIRTIO_BLK_T_IN;
+    req.type = VIRTIO_BLK_T_IN;
     req.ioprio = 1;
     req.sector = 1;
     req.data = g_malloc0(512);
@@ -616,11 +660,11 @@
     g_free(req.data);
 
     free_head = qvirtqueue_add(&vqpci->vq, req_addr, 16, false, true);
-    qvirtqueue_add(&vqpci->vq, req_addr + 16, 513, true, false);
+    qvirtqueue_add(&vqpci->vq, req_addr + 16, 512, true, true);
+    qvirtqueue_add(&vqpci->vq, req_addr + 528, 1, true, false);
 
     qvirtqueue_kick(&qvirtio_pci, &dev->vdev, &vqpci->vq, free_head);
 
-
     qvirtio_wait_queue_isr(&qvirtio_pci, &dev->vdev, &vqpci->vq,
                            QVIRTIO_BLK_TIMEOUT_US);
 
@@ -635,48 +679,94 @@
     guest_free(alloc, req_addr);
 
     /* End test */
-    guest_free(alloc, vqpci->vq.desc);
+    qvirtqueue_cleanup(&qvirtio_pci, &vqpci->vq, alloc);
+    pc_alloc_uninit(alloc);
     qpci_msix_disable(dev->pdev);
     qvirtio_pci_device_disable(dev);
     g_free(dev);
+    qpci_free_pc(bus);
     test_end();
 }
 
-static void hotplug(void)
+static void pci_hotplug(void)
 {
     QPCIBus *bus;
     QVirtioPCIDevice *dev;
 
-    bus = test_start();
+    bus = pci_test_start();
 
     /* plug secondary disk */
     qpci_plug_device_test("virtio-blk-pci", "drv1", PCI_SLOT_HP,
                           "'drive': 'drive1'");
 
-    dev = virtio_blk_init(bus, PCI_SLOT_HP);
+    dev = virtio_blk_pci_init(bus, PCI_SLOT_HP);
     g_assert(dev);
     qvirtio_pci_device_disable(dev);
     g_free(dev);
 
     /* unplug secondary disk */
     qpci_unplug_acpi_device_test("drv1", PCI_SLOT_HP);
+    qpci_free_pc(bus);
+    test_end();
+}
+
+static void mmio_basic(void)
+{
+    QVirtioMMIODevice *dev;
+    QVirtQueue *vq;
+    QGuestAllocator *alloc;
+    int n_size = TEST_IMAGE_SIZE / 2;
+    uint64_t capacity;
+
+    arm_test_start();
+
+    dev = qvirtio_mmio_init_device(MMIO_DEV_BASE_ADDR, MMIO_PAGE_SIZE);
+    g_assert(dev != NULL);
+    g_assert_cmphex(dev->vdev.device_type, ==, VIRTIO_ID_BLOCK);
+
+    qvirtio_reset(&qvirtio_mmio, &dev->vdev);
+    qvirtio_set_acknowledge(&qvirtio_mmio, &dev->vdev);
+    qvirtio_set_driver(&qvirtio_mmio, &dev->vdev);
+
+    alloc = generic_alloc_init(MMIO_RAM_ADDR, MMIO_RAM_SIZE, MMIO_PAGE_SIZE);
+    vq = qvirtqueue_setup(&qvirtio_mmio, &dev->vdev, alloc, 0);
+
+    test_basic(&qvirtio_mmio, &dev->vdev, alloc, vq,
+                            QVIRTIO_MMIO_DEVICE_SPECIFIC);
+
+    qmp("{ 'execute': 'block_resize', 'arguments': { 'device': 'drive0', "
+                                                    " 'size': %d } }", n_size);
+
+    qvirtio_wait_queue_isr(&qvirtio_mmio, &dev->vdev, vq,
+                           QVIRTIO_BLK_TIMEOUT_US);
+
+    capacity = qvirtio_config_readq(&qvirtio_mmio, &dev->vdev,
+                                                QVIRTIO_MMIO_DEVICE_SPECIFIC);
+    g_assert_cmpint(capacity, ==, n_size / 512);
+
+    /* End test */
+    qvirtqueue_cleanup(&qvirtio_mmio, vq, alloc);
+    generic_alloc_uninit(alloc);
+    g_free(dev);
     test_end();
 }
 
 int main(int argc, char **argv)
 {
-    int ret;
+    const char *arch = qtest_get_arch();
 
     g_test_init(&argc, &argv, NULL);
 
-    g_test_add_func("/virtio/blk/pci/basic", pci_basic);
-    g_test_add_func("/virtio/blk/pci/indirect", pci_indirect);
-    g_test_add_func("/virtio/blk/pci/config", pci_config);
-    g_test_add_func("/virtio/blk/pci/msix", pci_msix);
-    g_test_add_func("/virtio/blk/pci/idx", pci_idx);
-    g_test_add_func("/virtio/blk/pci/hotplug", hotplug);
+    if (strcmp(arch, "i386") == 0 || strcmp(arch, "x86_64") == 0) {
+        qtest_add_func("/virtio/blk/pci/basic", pci_basic);
+        qtest_add_func("/virtio/blk/pci/indirect", pci_indirect);
+        qtest_add_func("/virtio/blk/pci/config", pci_config);
+        qtest_add_func("/virtio/blk/pci/msix", pci_msix);
+        qtest_add_func("/virtio/blk/pci/idx", pci_idx);
+        qtest_add_func("/virtio/blk/pci/hotplug", pci_hotplug);
+    } else if (strcmp(arch, "arm") == 0) {
+        qtest_add_func("/virtio/blk/mmio/basic", mmio_basic);
+    }
 
-    ret = g_test_run();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/virtio-console-test.c b/tests/virtio-console-test.c
index 6be96e8..1c3de07 100644
--- a/tests/virtio-console-test.c
+++ b/tests/virtio-console-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void console_pci_nop(void)
@@ -29,13 +27,9 @@
 
 int main(int argc, char **argv)
 {
-    int ret;
-
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/virtio/console/pci/nop", console_pci_nop);
     qtest_add_func("/virtio/serialport/pci/nop", serialport_pci_nop);
 
-    ret = g_test_run();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/virtio-net-test.c b/tests/virtio-net-test.c
index ea7478c..361506f 100644
--- a/tests/virtio-net-test.c
+++ b/tests/virtio-net-test.c
@@ -7,37 +7,262 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
-#include "libqos/pci.h"
+#include "libqtest.h"
+#include "qemu-common.h"
+#include "qemu/sockets.h"
+#include "qemu/iov.h"
+#include "libqos/pci-pc.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-pci.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+#include "libqos/malloc-generic.h"
+#include "qemu/bswap.h"
+#include "hw/virtio/virtio-net.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_ring.h"
 
 #define PCI_SLOT_HP             0x06
+#define PCI_SLOT                0x04
+#define PCI_FN                  0x00
 
-/* Tests only initialization so far. TODO: Replace with functional tests */
-static void pci_nop(void)
+#define QVIRTIO_NET_TIMEOUT_US (30 * 1000 * 1000)
+#define VNET_HDR_SIZE sizeof(struct virtio_net_hdr_mrg_rxbuf)
+
+static void test_end(void)
 {
+    qtest_end();
 }
 
+#ifndef _WIN32
+
+static QVirtioPCIDevice *virtio_net_pci_init(QPCIBus *bus, int slot)
+{
+    QVirtioPCIDevice *dev;
+
+    dev = qvirtio_pci_device_find(bus, VIRTIO_ID_NET);
+    g_assert(dev != NULL);
+    g_assert_cmphex(dev->vdev.device_type, ==, VIRTIO_ID_NET);
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_acknowledge(&qvirtio_pci, &dev->vdev);
+    qvirtio_set_driver(&qvirtio_pci, &dev->vdev);
+
+    return dev;
+}
+
+static QPCIBus *pci_test_start(int socket)
+{
+    char *cmdline;
+
+    cmdline = g_strdup_printf("-netdev socket,fd=%d,id=hs0 -device "
+                              "virtio-net-pci,netdev=hs0", socket);
+    qtest_start(cmdline);
+    g_free(cmdline);
+
+    return qpci_init_pc();
+}
+
+static void driver_init(const QVirtioBus *bus, QVirtioDevice *dev)
+{
+    uint32_t features;
+
+    features = qvirtio_get_features(bus, dev);
+    features = features & ~(QVIRTIO_F_BAD_FEATURE |
+                            (1u << VIRTIO_RING_F_INDIRECT_DESC) |
+                            (1u << VIRTIO_RING_F_EVENT_IDX));
+    qvirtio_set_features(bus, dev, features);
+
+    qvirtio_set_driver_ok(bus, dev);
+}
+
+static void rx_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                    QGuestAllocator *alloc, QVirtQueue *vq,
+                    int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    char test[] = "TEST";
+    char buffer[64];
+    int len = htonl(sizeof(test));
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        }, {
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, true, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
+    g_assert_cmpstr(buffer, ==, "TEST");
+
+    guest_free(alloc, req_addr);
+}
+
+static void tx_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                    QGuestAllocator *alloc, QVirtQueue *vq,
+                    int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    uint32_t len;
+    char buffer[64];
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+    memwrite(req_addr + VNET_HDR_SIZE, "TEST", 4);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, false, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    guest_free(alloc, req_addr);
+
+    ret = qemu_recv(socket, &len, sizeof(len), 0);
+    g_assert_cmpint(ret, ==, sizeof(len));
+    len = ntohl(len);
+
+    ret = qemu_recv(socket, buffer, len, 0);
+    g_assert_cmpstr(buffer, ==, "TEST");
+}
+
+static void rx_stop_cont_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                              QGuestAllocator *alloc, QVirtQueue *vq,
+                              int socket)
+{
+    uint64_t req_addr;
+    uint32_t free_head;
+    char test[] = "TEST";
+    char buffer[64];
+    int len = htonl(sizeof(test));
+    QDict *rsp;
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len = sizeof(len),
+        }, {
+            .iov_base = test,
+            .iov_len = sizeof(test),
+        },
+    };
+    int ret;
+
+    req_addr = guest_alloc(alloc, 64);
+
+    free_head = qvirtqueue_add(vq, req_addr, 64, true, false);
+    qvirtqueue_kick(bus, dev, vq, free_head);
+
+    rsp = qmp("{ 'execute' : 'stop'}");
+    QDECREF(rsp);
+
+    ret = iov_send(socket, iov, 2, 0, sizeof(len) + sizeof(test));
+    g_assert_cmpint(ret, ==, sizeof(test) + sizeof(len));
+
+    /* We could check the status, but this command is more importantly to
+     * ensure the packet data gets queued in QEMU, before we do 'cont'.
+     */
+    rsp = qmp("{ 'execute' : 'query-status'}");
+    QDECREF(rsp);
+    rsp = qmp("{ 'execute' : 'cont'}");
+    QDECREF(rsp);
+
+    qvirtio_wait_queue_isr(bus, dev, vq, QVIRTIO_NET_TIMEOUT_US);
+    memread(req_addr + VNET_HDR_SIZE, buffer, sizeof(test));
+    g_assert_cmpstr(buffer, ==, "TEST");
+
+    guest_free(alloc, req_addr);
+}
+
+static void send_recv_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                           QGuestAllocator *alloc, QVirtQueue *rvq,
+                           QVirtQueue *tvq, int socket)
+{
+    rx_test(bus, dev, alloc, rvq, socket);
+    tx_test(bus, dev, alloc, tvq, socket);
+}
+
+static void stop_cont_test(const QVirtioBus *bus, QVirtioDevice *dev,
+                           QGuestAllocator *alloc, QVirtQueue *rvq,
+                           QVirtQueue *tvq, int socket)
+{
+    rx_stop_cont_test(bus, dev, alloc, rvq, socket);
+}
+
+static void pci_basic(gconstpointer data)
+{
+    QVirtioPCIDevice *dev;
+    QPCIBus *bus;
+    QVirtQueuePCI *tx, *rx;
+    QGuestAllocator *alloc;
+    void (*func) (const QVirtioBus *bus,
+                  QVirtioDevice *dev,
+                  QGuestAllocator *alloc,
+                  QVirtQueue *rvq,
+                  QVirtQueue *tvq,
+                  int socket) = data;
+    int sv[2], ret;
+
+    ret = socketpair(PF_UNIX, SOCK_STREAM, 0, sv);
+    g_assert_cmpint(ret, !=, -1);
+
+    bus = pci_test_start(sv[1]);
+    dev = virtio_net_pci_init(bus, PCI_SLOT);
+
+    alloc = pc_alloc_init();
+    rx = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                           alloc, 0);
+    tx = (QVirtQueuePCI *)qvirtqueue_setup(&qvirtio_pci, &dev->vdev,
+                                           alloc, 1);
+
+    driver_init(&qvirtio_pci, &dev->vdev);
+    func(&qvirtio_pci, &dev->vdev, alloc, &rx->vq, &tx->vq, sv[0]);
+
+    /* End test */
+    close(sv[0]);
+    qvirtqueue_cleanup(&qvirtio_pci, &tx->vq, alloc);
+    qvirtqueue_cleanup(&qvirtio_pci, &rx->vq, alloc);
+    pc_alloc_uninit(alloc);
+    qvirtio_pci_device_disable(dev);
+    g_free(dev->pdev);
+    g_free(dev);
+    qpci_free_pc(bus);
+    test_end();
+}
+#endif
+
 static void hotplug(void)
 {
+    qtest_start("-device virtio-net-pci");
+
     qpci_plug_device_test("virtio-net-pci", "net1", PCI_SLOT_HP, NULL);
     qpci_unplug_acpi_device_test("net1", PCI_SLOT_HP);
+
+    test_end();
 }
 
 int main(int argc, char **argv)
 {
-    int ret;
-
     g_test_init(&argc, &argv, NULL);
-    qtest_add_func("/virtio/net/pci/nop", pci_nop);
+#ifndef _WIN32
+    qtest_add_data_func("/virtio/net/pci/basic", send_recv_test, pci_basic);
+    qtest_add_data_func("/virtio/net/pci/rx_stop_cont",
+                        stop_cont_test, pci_basic);
+#endif
     qtest_add_func("/virtio/net/pci/hotplug", hotplug);
 
-    qtest_start("-device virtio-net-pci");
-    ret = g_test_run();
-
-    qtest_end();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/virtio-rng-test.c b/tests/virtio-rng-test.c
index 41c1cdb..e1b2640 100644
--- a/tests/virtio-rng-test.c
+++ b/tests/virtio-rng-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 #include "libqos/pci.h"
 
 #define PCI_SLOT_HP             0x06
diff --git a/tests/virtio-scsi-test.c b/tests/virtio-scsi-test.c
index 41f9602..f1489e6 100644
--- a/tests/virtio-scsi-test.c
+++ b/tests/virtio-scsi-test.c
@@ -2,25 +2,197 @@
  * QTest testcase for VirtIO SCSI
  *
  * Copyright (c) 2014 SUSE LINUX Products GmbH
+ * Copyright (c) 2015 Red Hat Inc.
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
+#include "block/scsi.h"
+#include "libqos/virtio.h"
+#include "libqos/virtio-pci.h"
+#include "libqos/pci-pc.h"
+#include "libqos/malloc.h"
+#include "libqos/malloc-pc.h"
+#include "libqos/malloc-generic.h"
+#include "standard-headers/linux/virtio_ids.h"
+#include "standard-headers/linux/virtio_pci.h"
+#include "standard-headers/linux/virtio_scsi.h"
+
+#define PCI_SLOT                0x02
+#define PCI_FN                  0x00
+#define QVIRTIO_SCSI_TIMEOUT_US (1 * 1000 * 1000)
+
+#define MAX_NUM_QUEUES 64
+
+typedef struct {
+    QVirtioDevice *dev;
+    QGuestAllocator *alloc;
+    QPCIBus *bus;
+    int num_queues;
+    QVirtQueue *vq[MAX_NUM_QUEUES + 2];
+} QVirtIOSCSI;
+
+static void qvirtio_scsi_start(const char *extra_opts)
+{
+    char *cmdline;
+
+    cmdline = g_strdup_printf(
+                "-drive id=drv0,if=none,file=/dev/null,format=raw "
+                "-device virtio-scsi-pci,id=vs0 "
+                "-device scsi-hd,bus=vs0.0,drive=drv0 %s",
+                extra_opts ? : "");
+    qtest_start(cmdline);
+    g_free(cmdline);
+}
+
+static void qvirtio_scsi_stop(void)
+{
+    qtest_end();
+}
+
+static void qvirtio_scsi_pci_free(QVirtIOSCSI *vs)
+{
+    int i;
+
+    for (i = 0; i < vs->num_queues + 2; i++) {
+        qvirtqueue_cleanup(&qvirtio_pci, vs->vq[i], vs->alloc);
+    }
+    pc_alloc_uninit(vs->alloc);
+    qvirtio_pci_device_disable(container_of(vs->dev, QVirtioPCIDevice, vdev));
+    g_free(vs->dev);
+    qpci_free_pc(vs->bus);
+}
+
+static uint64_t qvirtio_scsi_alloc(QVirtIOSCSI *vs, size_t alloc_size,
+                                   const void *data)
+{
+    uint64_t addr;
+
+    addr = guest_alloc(vs->alloc, alloc_size);
+    if (data) {
+        memwrite(addr, data, alloc_size);
+    }
+
+    return addr;
+}
+
+static uint8_t virtio_scsi_do_command(QVirtIOSCSI *vs, const uint8_t *cdb,
+                                      const uint8_t *data_in,
+                                      size_t data_in_len,
+                                      uint8_t *data_out, size_t data_out_len,
+                                      struct virtio_scsi_cmd_resp *resp_out)
+{
+    QVirtQueue *vq;
+    struct virtio_scsi_cmd_req req = { { 0 } };
+    struct virtio_scsi_cmd_resp resp = { .response = 0xff, .status = 0xff };
+    uint64_t req_addr, resp_addr, data_in_addr = 0, data_out_addr = 0;
+    uint8_t response;
+    uint32_t free_head;
+
+    vq = vs->vq[2];
+
+    req.lun[0] = 1; /* Select LUN */
+    req.lun[1] = 1; /* Select target 1 */
+    memcpy(req.cdb, cdb, VIRTIO_SCSI_CDB_SIZE);
+
+    /* XXX: Fix endian if any multi-byte field in req/resp is used */
+
+    /* Add request header */
+    req_addr = qvirtio_scsi_alloc(vs, sizeof(req), &req);
+    free_head = qvirtqueue_add(vq, req_addr, sizeof(req), false, true);
+
+    if (data_out_len) {
+        data_out_addr = qvirtio_scsi_alloc(vs, data_out_len, data_out);
+        qvirtqueue_add(vq, data_out_addr, data_out_len, false, true);
+    }
+
+    /* Add response header */
+    resp_addr = qvirtio_scsi_alloc(vs, sizeof(resp), &resp);
+    qvirtqueue_add(vq, resp_addr, sizeof(resp), true, !!data_in_len);
+
+    if (data_in_len) {
+        data_in_addr = qvirtio_scsi_alloc(vs, data_in_len, data_in);
+        qvirtqueue_add(vq, data_in_addr, data_in_len, true, false);
+    }
+
+    qvirtqueue_kick(&qvirtio_pci, vs->dev, vq, free_head);
+    qvirtio_wait_queue_isr(&qvirtio_pci, vs->dev, vq, QVIRTIO_SCSI_TIMEOUT_US);
+
+    response = readb(resp_addr +
+                     offsetof(struct virtio_scsi_cmd_resp, response));
+
+    if (resp_out) {
+        memread(resp_addr, resp_out, sizeof(*resp_out));
+    }
+
+    guest_free(vs->alloc, req_addr);
+    guest_free(vs->alloc, resp_addr);
+    guest_free(vs->alloc, data_in_addr);
+    guest_free(vs->alloc, data_out_addr);
+    return response;
+}
+
+static QVirtIOSCSI *qvirtio_scsi_pci_init(int slot)
+{
+    const uint8_t test_unit_ready_cdb[VIRTIO_SCSI_CDB_SIZE] = {};
+    QVirtIOSCSI *vs;
+    QVirtioPCIDevice *dev;
+    struct virtio_scsi_cmd_resp resp;
+    void *addr;
+    int i;
+
+    vs = g_new0(QVirtIOSCSI, 1);
+    vs->alloc = pc_alloc_init();
+    vs->bus = qpci_init_pc();
+
+    dev = qvirtio_pci_device_find(vs->bus, VIRTIO_ID_SCSI);
+    vs->dev = (QVirtioDevice *)dev;
+    g_assert(dev != NULL);
+    g_assert_cmphex(vs->dev->device_type, ==, VIRTIO_ID_SCSI);
+
+    qvirtio_pci_device_enable(dev);
+    qvirtio_reset(&qvirtio_pci, vs->dev);
+    qvirtio_set_acknowledge(&qvirtio_pci, vs->dev);
+    qvirtio_set_driver(&qvirtio_pci, vs->dev);
+
+    addr = dev->addr + VIRTIO_PCI_CONFIG_OFF(false);
+    vs->num_queues = qvirtio_config_readl(&qvirtio_pci, vs->dev,
+                                          (uint64_t)(uintptr_t)addr);
+
+    g_assert_cmpint(vs->num_queues, <, MAX_NUM_QUEUES);
+
+    for (i = 0; i < vs->num_queues + 2; i++) {
+        vs->vq[i] = qvirtqueue_setup(&qvirtio_pci, vs->dev, vs->alloc, i);
+    }
+
+    /* Clear the POWER ON OCCURRED unit attention */
+    g_assert_cmpint(virtio_scsi_do_command(vs, test_unit_ready_cdb,
+                                           NULL, 0, NULL, 0, &resp),
+                    ==, 0);
+    g_assert_cmpint(resp.status, ==, CHECK_CONDITION);
+    g_assert_cmpint(resp.sense[0], ==, 0x70); /* Fixed format sense buffer */
+    g_assert_cmpint(resp.sense[2], ==, UNIT_ATTENTION);
+    g_assert_cmpint(resp.sense[12], ==, 0x29); /* POWER ON */
+    g_assert_cmpint(resp.sense[13], ==, 0x00);
+
+    return vs;
+}
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
 {
+    qvirtio_scsi_start(NULL);
+    qvirtio_scsi_stop();
 }
 
 static void hotplug(void)
 {
     QDict *response;
 
+    qvirtio_scsi_start("-drive id=drv1,if=none,file=/dev/null,format=raw");
     response = qmp("{\"execute\": \"device_add\","
                    " \"arguments\": {"
                    "   \"driver\": \"scsi-hd\","
@@ -42,23 +214,44 @@
     g_assert(qdict_haskey(response, "event"));
     g_assert(!strcmp(qdict_get_str(response, "event"), "DEVICE_DELETED"));
     QDECREF(response);
+    qvirtio_scsi_stop();
+}
+
+/* Test WRITE SAME with the lba not aligned */
+static void test_unaligned_write_same(void)
+{
+    QVirtIOSCSI *vs;
+    uint8_t buf1[512] = { 0 };
+    uint8_t buf2[512] = { 1 };
+    const uint8_t write_same_cdb_1[VIRTIO_SCSI_CDB_SIZE] = {
+        0x41, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0x00
+    };
+    const uint8_t write_same_cdb_2[VIRTIO_SCSI_CDB_SIZE] = {
+        0x41, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x33, 0x00, 0x00
+    };
+
+    qvirtio_scsi_start("-drive file=blkdebug::null-co://,if=none,id=dr1"
+                       ",format=raw,file.align=4k "
+                       "-device scsi-disk,drive=dr1,lun=0,scsi-id=1");
+    vs = qvirtio_scsi_pci_init(PCI_SLOT);
+
+    g_assert_cmphex(0, ==,
+        virtio_scsi_do_command(vs, write_same_cdb_1, NULL, 0, buf1, 512, NULL));
+
+    g_assert_cmphex(0, ==,
+        virtio_scsi_do_command(vs, write_same_cdb_2, NULL, 0, buf2, 512, NULL));
+
+    qvirtio_scsi_pci_free(vs);
+    qvirtio_scsi_stop();
 }
 
 int main(int argc, char **argv)
 {
-    int ret;
-
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/virtio/scsi/pci/nop", pci_nop);
     qtest_add_func("/virtio/scsi/pci/hotplug", hotplug);
+    qtest_add_func("/virtio/scsi/pci/scsi-disk/unaligned-write-same",
+                   test_unaligned_write_same);
 
-    qtest_start("-drive id=drv0,if=none,file=/dev/null "
-                "-drive id=drv1,if=none,file=/dev/null "
-                "-device virtio-scsi-pci,id=vscsi0 "
-                "-device scsi-hd,bus=vscsi0.0,drive=drv0");
-    ret = g_test_run();
-
-    qtest_end();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/tests/virtio-serial-test.c b/tests/virtio-serial-test.c
index bf030a6..b14d943 100644
--- a/tests/virtio-serial-test.c
+++ b/tests/virtio-serial-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void pci_nop(void)
diff --git a/tests/vmxnet3-test.c b/tests/vmxnet3-test.c
index a2ebed3..159c0ad 100644
--- a/tests/vmxnet3-test.c
+++ b/tests/vmxnet3-test.c
@@ -7,10 +7,8 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
+#include "libqtest.h"
 
 /* Tests only initialization so far. TODO: Replace with functional tests */
 static void nop(void)
diff --git a/tests/wdt_ib700-test.c b/tests/wdt_ib700-test.c
index 513a533..49f4f0c 100644
--- a/tests/wdt_ib700-test.c
+++ b/tests/wdt_ib700-test.c
@@ -7,12 +7,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <glib.h>
-#include <string.h>
-#include "libqtest.h"
 #include "qemu/osdep.h"
-
-#define NS_PER_SEC 1000000000ULL
+#include "libqtest.h"
+#include "qemu/timer.h"
 
 static void qmp_check_no_event(void)
 {
@@ -41,29 +38,29 @@
 
 static QDict *ib700_program_and_wait(QTestState *s)
 {
-    clock_step(NS_PER_SEC * 40);
+    clock_step(NANOSECONDS_PER_SECOND * 40);
     qmp_check_no_event();
 
     /* 2 second limit */
     outb(0x443, 14);
 
     /* Ping */
-    clock_step(NS_PER_SEC);
+    clock_step(NANOSECONDS_PER_SECOND);
     qmp_check_no_event();
     outb(0x443, 14);
 
     /* Disable */
-    clock_step(NS_PER_SEC);
+    clock_step(NANOSECONDS_PER_SECOND);
     qmp_check_no_event();
     outb(0x441, 1);
-    clock_step(3 * NS_PER_SEC);
+    clock_step(3 * NANOSECONDS_PER_SECOND);
     qmp_check_no_event();
 
     /* Enable and let it fire */
     outb(0x443, 13);
-    clock_step(3 * NS_PER_SEC);
+    clock_step(3 * NANOSECONDS_PER_SECOND);
     qmp_check_no_event();
-    clock_step(2 * NS_PER_SEC);
+    clock_step(2 * NANOSECONDS_PER_SECOND);
     return qmp_get_event("WATCHDOG");
 }
 
@@ -120,15 +117,11 @@
 
 int main(int argc, char **argv)
 {
-    int ret;
-
     g_test_init(&argc, &argv, NULL);
     qtest_add_func("/wdt_ib700/pause", ib700_pause);
     qtest_add_func("/wdt_ib700/reset", ib700_reset);
     qtest_add_func("/wdt_ib700/shutdown", ib700_shutdown);
     qtest_add_func("/wdt_ib700/none", ib700_none);
 
-    ret = g_test_run();
-
-    return ret;
+    return g_test_run();
 }
diff --git a/thread-pool.c b/thread-pool.c
index e2cac8e..6fba913 100644
--- a/thread-pool.c
+++ b/thread-pool.c
@@ -14,11 +14,11 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "qemu/thread.h"
-#include "qemu/osdep.h"
-#include "block/coroutine.h"
+#include "qemu/coroutine.h"
 #include "trace.h"
 #include "block/thread-pool.h"
 #include "qemu/main-loop.h"
@@ -170,12 +170,12 @@
         if (elem->state != THREAD_DONE) {
             continue;
         }
-        if (elem->state == THREAD_DONE) {
-            trace_thread_pool_complete(pool, elem, elem->common.opaque,
-                                       elem->ret);
-        }
-        if (elem->state == THREAD_DONE && elem->common.cb) {
-            QLIST_REMOVE(elem, all);
+
+        trace_thread_pool_complete(pool, elem, elem->common.opaque,
+                                   elem->ret);
+        QLIST_REMOVE(elem, all);
+
+        if (elem->common.cb) {
             /* Read state before ret.  */
             smp_rmb();
 
@@ -188,8 +188,6 @@
             qemu_aio_unref(elem);
             goto restart;
         } else {
-            /* remove the request */
-            QLIST_REMOVE(elem, all);
             qemu_aio_unref(elem);
         }
     }
@@ -269,7 +267,7 @@
     ThreadPoolCo *co = opaque;
 
     co->ret = ret;
-    qemu_coroutine_enter(co->co, NULL);
+    qemu_coroutine_enter(co->co);
 }
 
 int coroutine_fn thread_pool_submit_co(ThreadPool *pool, ThreadPoolFunc *func,
diff --git a/thunk.c b/thunk.c
index 3cca047..2dac366 100644
--- a/thunk.c
+++ b/thunk.c
@@ -16,19 +16,15 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
+#include "qemu/osdep.h"
 
 #include "qemu.h"
 #include "exec/user/thunk.h"
 
 //#define DEBUG
 
-#define MAX_STRUCTS 128
-
-/* XXX: make it dynamic */
-StructEntry struct_entries[MAX_STRUCTS];
+static unsigned int max_struct_entries;
+StructEntry *struct_entries;
 
 static const argtype *thunk_type_next_ptr(const argtype *type_ptr);
 
@@ -70,6 +66,7 @@
     StructEntry *se;
     int nb_fields, offset, max_align, align, size, i, j;
 
+    assert(id < max_struct_entries);
     se = struct_entries + id;
 
     /* first we count the number of fields */
@@ -117,6 +114,8 @@
                                   const StructEntry *se1)
 {
     StructEntry *se;
+
+    assert(id < max_struct_entries);
     se = struct_entries + id;
     *se = *se1;
     se->name = name;
@@ -244,6 +243,7 @@
             const argtype *field_types;
             const int *dst_offsets, *src_offsets;
 
+            assert(*type_ptr < max_struct_entries);
             se = struct_entries + *type_ptr++;
             if (se->convert[0] != NULL) {
                 /* specific conversion is needed */
@@ -273,37 +273,36 @@
 /* from em86 */
 
 /* Utility function: Table-driven functions to translate bitmasks
- * between X86 and Alpha formats...
+ * between host and target formats
  */
-unsigned int target_to_host_bitmask(unsigned int x86_mask,
+unsigned int target_to_host_bitmask(unsigned int target_mask,
                                     const bitmask_transtbl * trans_tbl)
 {
     const bitmask_transtbl *btp;
-    unsigned int	alpha_mask = 0;
+    unsigned int host_mask = 0;
 
-    for(btp = trans_tbl; btp->x86_mask && btp->alpha_mask; btp++) {
-	if((x86_mask & btp->x86_mask) == btp->x86_bits) {
-	    alpha_mask |= btp->alpha_bits;
-	}
+    for (btp = trans_tbl; btp->target_mask && btp->host_mask; btp++) {
+        if ((target_mask & btp->target_mask) == btp->target_bits) {
+            host_mask |= btp->host_bits;
+        }
     }
-    return(alpha_mask);
+    return host_mask;
 }
 
-unsigned int host_to_target_bitmask(unsigned int alpha_mask,
+unsigned int host_to_target_bitmask(unsigned int host_mask,
                                     const bitmask_transtbl * trans_tbl)
 {
     const bitmask_transtbl *btp;
-    unsigned int	x86_mask = 0;
+    unsigned int target_mask = 0;
 
-    for(btp = trans_tbl; btp->x86_mask && btp->alpha_mask; btp++) {
-	if((alpha_mask & btp->alpha_mask) == btp->alpha_bits) {
-	    x86_mask |= btp->x86_bits;
-	}
+    for (btp = trans_tbl; btp->target_mask && btp->host_mask; btp++) {
+        if ((host_mask & btp->host_mask) == btp->host_bits) {
+            target_mask |= btp->target_bits;
+        }
     }
-    return(x86_mask);
+    return target_mask;
 }
 
-#ifndef NO_THUNK_TYPE_SIZE
 int thunk_type_size_array(const argtype *type_ptr, int is_host)
 {
     return thunk_type_size(type_ptr, is_host);
@@ -313,4 +312,9 @@
 {
     return thunk_type_align(type_ptr, is_host);
 }
-#endif /* ndef NO_THUNK_TYPE_SIZE */
+
+void thunk_init(unsigned int max_structs)
+{
+    max_struct_entries = max_structs;
+    struct_entries = g_new0(StructEntry, max_structs);
+}
diff --git a/tpm.c b/tpm.c
index c371023..9a7c711 100644
--- a/tpm.c
+++ b/tpm.c
@@ -11,13 +11,13 @@
  *
  * Based on net.c
  */
-#include "config-host.h"
+#include "qemu/osdep.h"
 
-#include "monitor/monitor.h"
 #include "qapi/qmp/qerror.h"
 #include "sysemu/tpm_backend.h"
 #include "sysemu/tpm.h"
 #include "qemu/config-file.h"
+#include "qemu/error-report.h"
 #include "qmp-commands.h"
 
 static QLIST_HEAD(, TPMBackend) tpm_backends =
@@ -32,7 +32,7 @@
 };
 
 static enum TpmModel tpm_models[TPM_MAX_MODELS] = {
-    TPM_MODEL_MAX,
+    TPM_MODEL__MAX,
 };
 
 int tpm_register_model(enum TpmModel model)
@@ -40,7 +40,7 @@
     int i;
 
     for (i = 0; i < TPM_MAX_MODELS; i++) {
-        if (tpm_models[i] == TPM_MODEL_MAX) {
+        if (tpm_models[i] == TPM_MODEL__MAX) {
             tpm_models[i] = model;
             return 0;
         }
@@ -134,27 +134,27 @@
     Error *local_err = NULL;
 
     if (!QLIST_EMPTY(&tpm_backends)) {
-        error_report("Only one TPM is allowed.\n");
+        error_report("Only one TPM is allowed.");
         return 1;
     }
 
     id = qemu_opts_id(opts);
     if (id == NULL) {
-        qerror_report(QERR_MISSING_PARAMETER, "id");
+        error_report(QERR_MISSING_PARAMETER, "id");
         return 1;
     }
 
     value = qemu_opt_get(opts, "type");
     if (!value) {
-        qerror_report(QERR_MISSING_PARAMETER, "type");
+        error_report(QERR_MISSING_PARAMETER, "type");
         tpm_display_backend_drivers();
         return 1;
     }
 
     be = tpm_get_backend_driver(value);
     if (be == NULL) {
-        qerror_report(QERR_INVALID_PARAMETER_VALUE, "type",
-                      "a TPM backend type");
+        error_report(QERR_INVALID_PARAMETER_VALUE,
+                     "type", "a TPM backend type");
         tpm_display_backend_drivers();
         return 1;
     }
@@ -162,8 +162,7 @@
     /* validate backend specific opts */
     qemu_opts_validate(opts, be->opts, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return 1;
     }
 
@@ -174,8 +173,7 @@
 
     tpm_backend_open(drv, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return 1;
     }
 
@@ -184,7 +182,7 @@
     return 0;
 }
 
-static int tpm_init_tpmdev(QemuOpts *opts, void *dummy)
+static int tpm_init_tpmdev(void *dummy, QemuOpts *opts, Error **errp)
 {
     return configure_tpm(opts);
 }
@@ -210,12 +208,11 @@
 int tpm_init(void)
 {
     if (qemu_opts_foreach(qemu_find_opts("tpmdev"),
-                          tpm_init_tpmdev, NULL, 1) != 0) {
+                          tpm_init_tpmdev, NULL, NULL)) {
         return -1;
     }
 
     atexit(tpm_cleanup);
-
     return 0;
 }
 
@@ -231,7 +228,7 @@
         tpm_display_backend_drivers();
         return -1;
     }
-    opts = qemu_opts_parse(opts_list, optarg, 1);
+    opts = qemu_opts_parse_noisily(opts_list, optarg, true);
     if (!opts) {
         return -1;
     }
@@ -263,9 +260,9 @@
 
     switch (drv->ops->type) {
     case TPM_TYPE_PASSTHROUGH:
-        res->options->kind = TPM_TYPE_OPTIONS_KIND_PASSTHROUGH;
+        res->options->type = TPM_TYPE_OPTIONS_KIND_PASSTHROUGH;
         tpo = g_new0(TPMPassthroughOptions, 1);
-        res->options->passthrough = tpo;
+        res->options->u.passthrough.data = tpo;
         if (drv->path) {
             tpo->path = g_strdup(drv->path);
             tpo->has_path = true;
@@ -275,7 +272,7 @@
             tpo->has_cancel_path = true;
         }
         break;
-    case TPM_TYPE_MAX:
+    case TPM_TYPE__MAX:
         break;
     }
 
@@ -314,7 +311,7 @@
     unsigned int i = 0;
     TpmTypeList *head = NULL, *prev = NULL, *cur_item;
 
-    for (i = 0; i < TPM_TYPE_MAX; i++) {
+    for (i = 0; i < TPM_TYPE__MAX; i++) {
         if (!tpm_driver_find_by_type(i)) {
             continue;
         }
@@ -338,7 +335,7 @@
     unsigned int i = 0;
     TpmModelList *head = NULL, *prev = NULL, *cur_item;
 
-    for (i = 0; i < TPM_MODEL_MAX; i++) {
+    for (i = 0; i < TPM_MODEL__MAX; i++) {
         if (!tpm_model_is_registered(i)) {
             continue;
         }
diff --git a/trace-events b/trace-events
index 98d55da..616cc52 100644
--- a/trace-events
+++ b/trace-events
@@ -25,641 +25,31 @@
 #
 # The <format-string> should be a sprintf()-compatible format string.
 
-# util/oslib-win32.c
-# util/oslib-posix.c
-qemu_memalign(size_t alignment, size_t size, void *ptr) "alignment %zu size %zu ptr %p"
-qemu_anon_ram_alloc(size_t size, void *ptr) "size %zu ptr %p"
-qemu_vfree(void *ptr) "ptr %p"
-qemu_anon_ram_free(void *ptr, size_t size) "ptr %p size %zu"
-
-# hw/virtio/virtio.c
-virtqueue_fill(void *vq, const void *elem, unsigned int len, unsigned int idx) "vq %p elem %p len %u idx %u"
-virtqueue_flush(void *vq, unsigned int count) "vq %p count %u"
-virtqueue_pop(void *vq, void *elem, unsigned int in_num, unsigned int out_num) "vq %p elem %p in_num %u out_num %u"
-virtio_queue_notify(void *vdev, int n, void *vq) "vdev %p n %d vq %p"
-virtio_irq(void *vq) "vq %p"
-virtio_notify(void *vdev, void *vq) "vdev %p vq %p"
-virtio_set_status(void *vdev, uint8_t val) "vdev %p val %u"
-
-# hw/virtio/virtio-rng.c
-virtio_rng_guest_not_ready(void *rng) "rng %p: guest not ready"
-virtio_rng_pushed(void *rng, size_t len) "rng %p: %zd bytes pushed"
-virtio_rng_request(void *rng, size_t size, unsigned quota) "rng %p: %zd bytes requested, %u bytes quota left"
-
-# hw/char/virtio-serial-bus.c
-virtio_serial_send_control_event(unsigned int port, uint16_t event, uint16_t value) "port %u, event %u, value %u"
-virtio_serial_throttle_port(unsigned int port, bool throttle) "port %u, throttle %d"
-virtio_serial_handle_control_message(uint16_t event, uint16_t value) "event %u, value %u"
-virtio_serial_handle_control_message_port(unsigned int port) "port %u"
-
-# hw/char/virtio-console.c
-virtio_console_flush_buf(unsigned int port, size_t len, ssize_t ret) "port %u, in_len %zu, out_len %zd"
-virtio_console_chr_read(unsigned int port, int size) "port %u, size %d"
-virtio_console_chr_event(unsigned int port, int event) "port %u, event %d"
-
-# block.c
-bdrv_open_common(void *bs, const char *filename, int flags, const char *format_name) "bs %p filename \"%s\" flags %#x format_name \"%s\""
-multiwrite_cb(void *mcb, int ret) "mcb %p ret %d"
-bdrv_aio_multiwrite(void *mcb, int num_callbacks, int num_reqs) "mcb %p num_callbacks %d num_reqs %d"
-bdrv_aio_discard(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
-bdrv_aio_flush(void *bs, void *opaque) "bs %p opaque %p"
-bdrv_aio_readv(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
-bdrv_aio_writev(void *bs, int64_t sector_num, int nb_sectors, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d opaque %p"
-bdrv_aio_write_zeroes(void *bs, int64_t sector_num, int nb_sectors, int flags, void *opaque) "bs %p sector_num %"PRId64" nb_sectors %d flags %#x opaque %p"
-bdrv_lock_medium(void *bs, bool locked) "bs %p locked %d"
-bdrv_co_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
-bdrv_co_copy_on_readv(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
-bdrv_co_writev(void *bs, int64_t sector_num, int nb_sector) "bs %p sector_num %"PRId64" nb_sectors %d"
-bdrv_co_write_zeroes(void *bs, int64_t sector_num, int nb_sector, int flags) "bs %p sector_num %"PRId64" nb_sectors %d flags %#x"
-bdrv_co_io_em(void *bs, int64_t sector_num, int nb_sectors, int is_write, void *acb) "bs %p sector_num %"PRId64" nb_sectors %d is_write %d acb %p"
-bdrv_co_do_copy_on_readv(void *bs, int64_t sector_num, int nb_sectors, int64_t cluster_sector_num, int cluster_nb_sectors) "bs %p sector_num %"PRId64" nb_sectors %d cluster_sector_num %"PRId64" cluster_nb_sectors %d"
-
-# block/stream.c
-stream_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
-stream_start(void *bs, void *base, void *s, void *co, void *opaque) "bs %p base %p s %p co %p opaque %p"
-
-# block/commit.c
-commit_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
-commit_start(void *bs, void *base, void *top, void *s, void *co, void *opaque) "bs %p base %p top %p s %p co %p opaque %p"
-
-# block/mirror.c
-mirror_start(void *bs, void *s, void *co, void *opaque) "bs %p s %p co %p opaque %p"
-mirror_restart_iter(void *s, int64_t cnt) "s %p dirty count %"PRId64
-mirror_before_flush(void *s) "s %p"
-mirror_before_drain(void *s, int64_t cnt) "s %p dirty count %"PRId64
-mirror_before_sleep(void *s, int64_t cnt, int synced, uint64_t delay_ns) "s %p dirty count %"PRId64" synced %d delay %"PRIu64"ns"
-mirror_one_iteration(void *s, int64_t sector_num, int nb_sectors) "s %p sector_num %"PRId64" nb_sectors %d"
-mirror_iteration_done(void *s, int64_t sector_num, int nb_sectors, int ret) "s %p sector_num %"PRId64" nb_sectors %d ret %d"
-mirror_yield(void *s, int64_t cnt, int buf_free_count, int in_flight) "s %p dirty count %"PRId64" free buffers %d in_flight %d"
-mirror_yield_in_flight(void *s, int64_t sector_num, int in_flight) "s %p sector_num %"PRId64" in_flight %d"
-mirror_yield_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
-mirror_break_buf_busy(void *s, int nb_chunks, int in_flight) "s %p requested chunks %d in_flight %d"
-
-# block/backup.c
-backup_do_cow_enter(void *job, int64_t start, int64_t sector_num, int nb_sectors) "job %p start %"PRId64" sector_num %"PRId64" nb_sectors %d"
-backup_do_cow_return(void *job, int64_t sector_num, int nb_sectors, int ret) "job %p sector_num %"PRId64" nb_sectors %d ret %d"
-backup_do_cow_skip(void *job, int64_t start) "job %p start %"PRId64
-backup_do_cow_process(void *job, int64_t start) "job %p start %"PRId64
-backup_do_cow_read_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d"
-backup_do_cow_write_fail(void *job, int64_t start, int ret) "job %p start %"PRId64" ret %d"
-
-# blockdev.c
-qmp_block_job_cancel(void *job) "job %p"
-qmp_block_job_pause(void *job) "job %p"
-qmp_block_job_resume(void *job) "job %p"
-qmp_block_job_complete(void *job) "job %p"
-block_job_cb(void *bs, void *job, int ret) "bs %p job %p ret %d"
-qmp_block_stream(void *bs, void *job) "bs %p job %p"
-
-# hw/block/virtio-blk.c
-virtio_blk_req_complete(void *req, int status) "req %p status %d"
-virtio_blk_rw_complete(void *req, int ret) "req %p ret %d"
-virtio_blk_handle_write(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
-virtio_blk_handle_read(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
-
-# hw/block/dataplane/virtio-blk.c
-virtio_blk_data_plane_start(void *s) "dataplane %p"
-virtio_blk_data_plane_stop(void *s) "dataplane %p"
-virtio_blk_data_plane_process_request(void *s, unsigned int out_num, unsigned int in_num, unsigned int head) "dataplane %p out_num %u in_num %u head %u"
-
-# hw/virtio/dataplane/vring.c
-vring_setup(uint64_t physical, void *desc, void *avail, void *used) "vring physical %#"PRIx64" desc %p avail %p used %p"
-
 # thread-pool.c
 thread_pool_submit(void *pool, void *req, void *opaque) "pool %p req %p opaque %p"
 thread_pool_complete(void *pool, void *req, void *opaque, int ret) "pool %p req %p opaque %p ret %d"
 thread_pool_cancel(void *req, void *opaque) "req %p opaque %p"
 
-# block/raw-win32.c
-# block/raw-posix.c
-paio_submit_co(int64_t sector_num, int nb_sectors, int type) "sector_num %"PRId64" nb_sectors %d type %d"
-paio_submit(void *acb, void *opaque, int64_t sector_num, int nb_sectors, int type) "acb %p opaque %p sector_num %"PRId64" nb_sectors %d type %d"
-
 # ioport.c
-cpu_in(unsigned int addr, unsigned int val) "addr %#x value %u"
-cpu_out(unsigned int addr, unsigned int val) "addr %#x value %u"
+cpu_in(unsigned int addr, char size, unsigned int val) "addr %#x(%c) value %u"
+cpu_out(unsigned int addr, char size, unsigned int val) "addr %#x(%c) value %u"
 
 # balloon.c
 # Since requests are raised via monitor, not many tracepoints are needed.
 balloon_event(void *opaque, unsigned long addr) "opaque %p addr %lu"
-
-# hw/intc/apic_common.c
-cpu_set_apic_base(uint64_t val) "%016"PRIx64
-cpu_get_apic_base(uint64_t val) "%016"PRIx64
-# coalescing
-apic_report_irq_delivered(int apic_irq_delivered) "coalescing %d"
-apic_reset_irq_delivered(int apic_irq_delivered) "old coalescing %d"
-apic_get_irq_delivered(int apic_irq_delivered) "returning coalescing %d"
-
-# hw/intc/apic.c
-apic_local_deliver(int vector, uint32_t lvt) "vector %d delivery mode %d"
-apic_deliver_irq(uint8_t dest, uint8_t dest_mode, uint8_t delivery_mode, uint8_t vector_num, uint8_t trigger_mode) "dest %d dest_mode %d delivery_mode %d vector %d trigger_mode %d"
-apic_mem_readl(uint64_t addr, uint32_t val)  "%"PRIx64" = %08x"
-apic_mem_writel(uint64_t addr, uint32_t val) "%"PRIx64" = %08x"
-
-# hw/audio/cs4231.c
-cs4231_mem_readl_dreg(uint32_t reg, uint32_t ret) "read dreg %d: 0x%02x"
-cs4231_mem_readl_reg(uint32_t reg, uint32_t ret) "read reg %d: 0x%08x"
-cs4231_mem_writel_reg(uint32_t reg, uint32_t old, uint32_t val) "write reg %d: 0x%08x -> 0x%08x"
-cs4231_mem_writel_dreg(uint32_t reg, uint32_t old, uint32_t val) "write dreg %d: 0x%02x -> 0x%02x"
-
-# hw/nvram/ds1225y.c
-nvram_read(uint32_t addr, uint32_t ret) "read addr %d: 0x%02x"
-nvram_write(uint32_t addr, uint32_t old, uint32_t val) "write addr %d: 0x%02x -> 0x%02x"
-
-# hw/misc/eccmemctl.c
-ecc_mem_writel_mer(uint32_t val) "Write memory enable %08x"
-ecc_mem_writel_mdr(uint32_t val) "Write memory delay %08x"
-ecc_mem_writel_mfsr(uint32_t val) "Write memory fault status %08x"
-ecc_mem_writel_vcr(uint32_t val) "Write slot configuration %08x"
-ecc_mem_writel_dr(uint32_t val) "Write diagnostic %08x"
-ecc_mem_writel_ecr0(uint32_t val) "Write event count 1 %08x"
-ecc_mem_writel_ecr1(uint32_t val) "Write event count 2 %08x"
-ecc_mem_readl_mer(uint32_t ret) "Read memory enable %08x"
-ecc_mem_readl_mdr(uint32_t ret) "Read memory delay %08x"
-ecc_mem_readl_mfsr(uint32_t ret) "Read memory fault status %08x"
-ecc_mem_readl_vcr(uint32_t ret) "Read slot configuration %08x"
-ecc_mem_readl_mfar0(uint32_t ret) "Read memory fault address 0 %08x"
-ecc_mem_readl_mfar1(uint32_t ret) "Read memory fault address 1 %08x"
-ecc_mem_readl_dr(uint32_t ret) "Read diagnostic %08x"
-ecc_mem_readl_ecr0(uint32_t ret) "Read event count 1 %08x"
-ecc_mem_readl_ecr1(uint32_t ret) "Read event count 2 %08x"
-ecc_diag_mem_writeb(uint64_t addr, uint32_t val) "Write diagnostic %"PRId64" = %02x"
-ecc_diag_mem_readb(uint64_t addr, uint32_t ret) "Read diagnostic %"PRId64"= %02x"
-
-# hw/nvram/fw_cfg.c
-fw_cfg_write(void *s, uint8_t value) "%p %d"
-fw_cfg_select(void *s, uint16_t key, int ret) "%p key %d = %d"
-fw_cfg_read(void *s, uint8_t ret) "%p = %d"
-fw_cfg_add_file_dupe(void *s, char *name) "%p %s"
-fw_cfg_add_file(void *s, int index, char *name, size_t len) "%p #%d: %s (%zd bytes)"
-
-# hw/block/hd-geometry.c
-hd_geometry_lchs_guess(void *blk, int cyls, int heads, int secs) "blk %p LCHS %d %d %d"
-hd_geometry_guess(void *blk, uint32_t cyls, uint32_t heads, uint32_t secs, int trans) "blk %p CHS %u %u %u trans %d"
-
-# hw/display/jazz_led.c
-jazz_led_read(uint64_t addr, uint8_t val) "read addr=0x%"PRIx64": 0x%x"
-jazz_led_write(uint64_t addr, uint8_t new) "write addr=0x%"PRIx64": 0x%x"
-
-# hw/net/lance.c
-lance_mem_readw(uint64_t addr, uint32_t ret) "addr=%"PRIx64"val=0x%04x"
-lance_mem_writew(uint64_t addr, uint32_t val) "addr=%"PRIx64"val=0x%04x"
-
-# hw/intc/slavio_intctl.c
-slavio_intctl_mem_readl(uint32_t cpu, uint64_t addr, uint32_t ret) "read cpu %d reg 0x%"PRIx64" = %x"
-slavio_intctl_mem_writel(uint32_t cpu, uint64_t addr, uint32_t val) "write cpu %d reg 0x%"PRIx64" = %x"
-slavio_intctl_mem_writel_clear(uint32_t cpu, uint32_t val, uint32_t intreg_pending) "Cleared cpu %d irq mask %x, curmask %x"
-slavio_intctl_mem_writel_set(uint32_t cpu, uint32_t val, uint32_t intreg_pending) "Set cpu %d irq mask %x, curmask %x"
-slavio_intctlm_mem_readl(uint64_t addr, uint32_t ret) "read system reg 0x%"PRIx64" = %x"
-slavio_intctlm_mem_writel(uint64_t addr, uint32_t val) "write system reg 0x%"PRIx64" = %x"
-slavio_intctlm_mem_writel_enable(uint32_t val, uint32_t intregm_disabled) "Enabled master irq mask %x, curmask %x"
-slavio_intctlm_mem_writel_disable(uint32_t val, uint32_t intregm_disabled) "Disabled master irq mask %x, curmask %x"
-slavio_intctlm_mem_writel_target(uint32_t cpu) "Set master irq cpu %d"
-slavio_check_interrupts(uint32_t pending, uint32_t intregm_disabled) "pending %x disabled %x"
-slavio_set_irq(uint32_t target_cpu, int irq, uint32_t pil, int level) "Set cpu %d irq %d -> pil %d level %d"
-slavio_set_timer_irq_cpu(int cpu, int level) "Set cpu %d local timer level %d"
-
-# hw/misc/slavio_misc.c
-slavio_misc_update_irq_raise(void) "Raise IRQ"
-slavio_misc_update_irq_lower(void) "Lower IRQ"
-slavio_set_power_fail(int power_failing, uint8_t config) "Power fail: %d, config: %d"
-slavio_cfg_mem_writeb(uint32_t val) "Write config %02x"
-slavio_cfg_mem_readb(uint32_t ret) "Read config %02x"
-slavio_diag_mem_writeb(uint32_t val) "Write diag %02x"
-slavio_diag_mem_readb(uint32_t ret) "Read diag %02x"
-slavio_mdm_mem_writeb(uint32_t val) "Write modem control %02x"
-slavio_mdm_mem_readb(uint32_t ret) "Read modem control %02x"
-slavio_aux1_mem_writeb(uint32_t val) "Write aux1 %02x"
-slavio_aux1_mem_readb(uint32_t ret) "Read aux1 %02x"
-slavio_aux2_mem_writeb(uint32_t val) "Write aux2 %02x"
-slavio_aux2_mem_readb(uint32_t ret) "Read aux2 %02x"
-apc_mem_writeb(uint32_t val) "Write power management %02x"
-apc_mem_readb(uint32_t ret) "Read power management %02x"
-slavio_sysctrl_mem_writel(uint32_t val) "Write system control %08x"
-slavio_sysctrl_mem_readl(uint32_t ret) "Read system control %08x"
-slavio_led_mem_writew(uint32_t val) "Write diagnostic LED %04x"
-slavio_led_mem_readw(uint32_t ret) "Read diagnostic LED %04x"
-
-# hw/timer/slavio_timer.c
-slavio_timer_get_out(uint64_t limit, uint32_t counthigh, uint32_t count) "limit %"PRIx64" count %x%08x"
-slavio_timer_irq(uint32_t counthigh, uint32_t count) "callback: count %x%08x"
-slavio_timer_mem_readl_invalid(uint64_t addr) "invalid read address %"PRIx64
-slavio_timer_mem_readl(uint64_t addr, uint32_t ret) "read %"PRIx64" = %08x"
-slavio_timer_mem_writel(uint64_t addr, uint32_t val) "write %"PRIx64" = %08x"
-slavio_timer_mem_writel_limit(unsigned int timer_index, uint64_t count) "processor %d user timer set to %016"PRIx64
-slavio_timer_mem_writel_counter_invalid(void) "not user timer"
-slavio_timer_mem_writel_status_start(unsigned int timer_index) "processor %d user timer started"
-slavio_timer_mem_writel_status_stop(unsigned int timer_index) "processor %d user timer stopped"
-slavio_timer_mem_writel_mode_user(unsigned int timer_index) "processor %d changed from counter to user timer"
-slavio_timer_mem_writel_mode_counter(unsigned int timer_index) "processor %d changed from user timer to counter"
-slavio_timer_mem_writel_mode_invalid(void) "not system timer"
-slavio_timer_mem_writel_invalid(uint64_t addr) "invalid write address %"PRIx64
-
-# hw/dma/sparc32_dma.c
-ledma_memory_read(uint64_t addr) "DMA read addr 0x%"PRIx64
-ledma_memory_write(uint64_t addr) "DMA write addr 0x%"PRIx64
-sparc32_dma_set_irq_raise(void) "Raise IRQ"
-sparc32_dma_set_irq_lower(void) "Lower IRQ"
-espdma_memory_read(uint32_t addr) "DMA read addr 0x%08x"
-espdma_memory_write(uint32_t addr) "DMA write addr 0x%08x"
-sparc32_dma_mem_readl(uint64_t addr, uint32_t ret) "read dmareg %"PRIx64": 0x%08x"
-sparc32_dma_mem_writel(uint64_t addr, uint32_t old, uint32_t val) "write dmareg %"PRIx64": 0x%08x -> 0x%08x"
-sparc32_dma_enable_raise(void) "Raise DMA enable"
-sparc32_dma_enable_lower(void) "Lower DMA enable"
-
-# hw/sparc/sun4m.c
-sun4m_cpu_interrupt(unsigned int level) "Set CPU IRQ %d"
-sun4m_cpu_reset_interrupt(unsigned int level) "Reset CPU IRQ %d"
-sun4m_cpu_set_irq_raise(int level) "Raise CPU IRQ %d"
-sun4m_cpu_set_irq_lower(int level) "Lower CPU IRQ %d"
-
-# hw/dma/sun4m_iommu.c
-sun4m_iommu_mem_readl(uint64_t addr, uint32_t ret) "read reg[%"PRIx64"] = %x"
-sun4m_iommu_mem_writel(uint64_t addr, uint32_t val) "write reg[%"PRIx64"] = %x"
-sun4m_iommu_mem_writel_ctrl(uint64_t iostart) "iostart = %"PRIx64
-sun4m_iommu_mem_writel_tlbflush(uint32_t val) "tlb flush %x"
-sun4m_iommu_mem_writel_pgflush(uint32_t val) "page flush %x"
-sun4m_iommu_page_get_flags(uint64_t pa, uint64_t iopte, uint32_t ret) "get flags addr %"PRIx64" => pte %"PRIx64", *pte = %x"
-sun4m_iommu_translate_pa(uint64_t addr, uint64_t pa, uint32_t iopte) "xlate dva %"PRIx64" => pa %"PRIx64" iopte = %x"
-sun4m_iommu_bad_addr(uint64_t addr) "bad addr %"PRIx64
-
-# hw/usb/core.c
-usb_packet_state_change(int bus, const char *port, int ep, void *p, const char *o, const char *n) "bus %d, port %s, ep %d, packet %p, state %s -> %s"
-usb_packet_state_fault(int bus, const char *port, int ep, void *p, const char *o, const char *n) "bus %d, port %s, ep %d, packet %p, state %s, expected %s"
-
-# hw/usb/bus.c
-usb_port_claim(int bus, const char *port) "bus %d, port %s"
-usb_port_attach(int bus, const char *port, const char *devspeed, const char *portspeed) "bus %d, port %s, devspeed %s, portspeed %s"
-usb_port_detach(int bus, const char *port) "bus %d, port %s"
-usb_port_release(int bus, const char *port) "bus %d, port %s"
-
-# hw/usb/hcd-ohci.c
-usb_ohci_iso_td_read_failed(uint32_t addr) "ISO_TD read error at %x"
-usb_ohci_iso_td_head(uint32_t head, uint32_t tail, uint32_t flags, uint32_t bp, uint32_t next, uint32_t be, uint32_t framenum, uint32_t startframe, uint32_t framecount, int rel_frame_num) "ISO_TD ED head 0x%.8x tailp 0x%.8x\n0x%.8x 0x%.8x 0x%.8x 0x%.8x\nframe_number 0x%.8x starting_frame 0x%.8x\nframe_count  0x%.8x relative %d"
-usb_ohci_iso_td_head_offset(uint32_t o0, uint32_t o1, uint32_t o2, uint32_t o3, uint32_t o4, uint32_t o5, uint32_t o6, uint32_t o7) "0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x 0x%.8x"
-usb_ohci_iso_td_relative_frame_number_neg(int rel) "ISO_TD R=%d < 0"
-usb_ohci_iso_td_relative_frame_number_big(int rel, int count) "ISO_TD R=%d > FC=%d"
-usb_ohci_iso_td_bad_direction(int dir) "Bad direction %d"
-usb_ohci_iso_td_bad_bp_be(uint32_t bp, uint32_t be) "ISO_TD bp 0x%.8x be 0x%.8x"
-usb_ohci_iso_td_bad_cc_not_accessed(uint32_t start, uint32_t next) "ISO_TD cc != not accessed 0x%.8x 0x%.8x"
-usb_ohci_iso_td_bad_cc_overrun(uint32_t start, uint32_t next) "ISO_TD start_offset=0x%.8x > next_offset=0x%.8x"
-usb_ohci_iso_td_so(uint32_t so, uint32_t eo, uint32_t s, uint32_t e, const char *str, ssize_t len, int ret) "0x%.8x eo 0x%.8x\nsa 0x%.8x ea 0x%.8x\ndir %s len %zu ret %d"
-usb_ohci_iso_td_data_overrun(int ret, ssize_t len) "DataOverrun %d > %zu"
-usb_ohci_iso_td_data_underrun(int ret) "DataUnderrun %d"
-usb_ohci_iso_td_nak(int ret) "got NAK/STALL %d"
-usb_ohci_iso_td_bad_response(int ret) "Bad device response %d"
-usb_ohci_port_attach(int index) "port #%d"
-usb_ohci_port_detach(int index) "port #%d"
-usb_ohci_port_wakeup(int index) "port #%d"
-usb_ohci_port_suspend(int index) "port #%d"
-usb_ohci_port_reset(int index) "port #%d"
-usb_ohci_remote_wakeup(const char *s) "%s: SUSPEND->RESUME"
-usb_ohci_reset(const char *s) "%s"
-usb_ohci_start(const char *s) "%s: USB Operational"
-usb_ohci_resume(const char *s) "%s: USB Resume"
-usb_ohci_stop(const char *s) "%s: USB Suspended"
-usb_ohci_exit(const char *s) "%s"
-usb_ohci_set_ctl(const char *s, uint32_t new_state) "%s: new state 0x%x"
-usb_ohci_td_underrun(void) ""
-usb_ohci_td_dev_error(void) ""
-usb_ohci_td_nak(void) ""
-usb_ohci_td_stall(void) ""
-usb_ohci_td_babble(void) ""
-usb_ohci_td_bad_device_response(int rc) "%d"
-usb_ohci_td_read_error(uint32_t addr) "TD read error at %x"
-usb_ohci_td_bad_direction(int dir) "Bad direction %d"
-usb_ohci_td_skip_async(void) ""
-usb_ohci_td_pkt_hdr(uint32_t addr, int64_t pktlen, int64_t len, const char *s, int flag_r, uint32_t cbp, uint32_t be) " TD @ 0x%.8x %" PRId64 " of %" PRId64 " bytes %s r=%d cbp=0x%.8x be=0x%.8x"
-usb_ohci_td_pkt_short(const char *dir, const char *buf) "%s data: %s"
-usb_ohci_td_pkt_full(const char *dir, const char *buf) "%s data: %s"
-usb_ohci_td_too_many_pending(void) ""
-usb_ohci_td_packet_status(int status) "status=%d"
-usb_ohci_ed_read_error(uint32_t addr) "ED read error at %x"
-usb_ohci_ed_pkt(uint32_t cur, int h, int c, uint32_t head, uint32_t tail, uint32_t next) "ED @ 0x%.8x h=%u c=%u\n  head=0x%.8x tailp=0x%.8x next=0x%.8x"
-usb_ohci_ed_pkt_flags(uint32_t fa, uint32_t en, uint32_t d, int s, int k, int f, uint32_t mps) "fa=%u en=%u d=%u s=%u k=%u f=%u mps=%u"
-usb_ohci_hcca_read_error(uint32_t addr) "HCCA read error at %x"
-usb_ohci_mem_read_unaligned(uint32_t addr) "at %x"
-usb_ohci_mem_read_bad_offset(uint32_t addr) "%x"
-usb_ohci_mem_write_unaligned(uint32_t addr) "at %x"
-usb_ohci_mem_write_bad_offset(uint32_t addr) "%x"
-usb_ohci_process_lists(uint32_t head, uint32_t cur) "head %x, cur %x"
-usb_ohci_bus_eof_timer_failed(const char *name) "%s: timer_new_ns failed"
-usb_ohci_set_frame_interval(const char *name, uint16_t fi_x, uint16_t fi_u) "%s: FrameInterval = 0x%x (%u)"
-usb_ohci_hub_power_up(void) "powered up all ports"
-usb_ohci_hub_power_down(void) "powered down all ports"
-usb_ohci_init_time(int64_t frametime, int64_t bittime) "usb_bit_time=%" PRId64 " usb_frame_time=%" PRId64
-usb_ohci_die(void) ""
-usb_ohci_async_complete(void) ""
-
-# hw/usb/hcd-ehci.c
-usb_ehci_reset(void) "=== RESET ==="
-usb_ehci_unrealize(void) "=== UNREALIZE ==="
-usb_ehci_opreg_read(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
-usb_ehci_opreg_write(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
-usb_ehci_opreg_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
-usb_ehci_portsc_read(uint32_t addr, uint32_t port, uint32_t val) "rd mmio %04x [port %d] = %x"
-usb_ehci_portsc_write(uint32_t addr, uint32_t port, uint32_t val) "wr mmio %04x [port %d] = %x"
-usb_ehci_portsc_change(uint32_t addr, uint32_t port, uint32_t new, uint32_t old) "ch mmio %04x [port %d] = %x (old: %x)"
-usb_ehci_usbsts(const char *sts, int state) "usbsts %s %d"
-usb_ehci_state(const char *schedule, const char *state) "%s schedule %s"
-usb_ehci_qh_ptrs(void *q, uint32_t addr, uint32_t nxt, uint32_t c_qtd, uint32_t n_qtd, uint32_t a_qtd) "q %p - QH @ %08x: next %08x qtds %08x,%08x,%08x"
-usb_ehci_qh_fields(uint32_t addr, int rl, int mplen, int eps, int ep, int devaddr) "QH @ %08x - rl %d, mplen %d, eps %d, ep %d, dev %d"
-usb_ehci_qh_bits(uint32_t addr, int c, int h, int dtc, int i) "QH @ %08x - c %d, h %d, dtc %d, i %d"
-usb_ehci_qtd_ptrs(void *q, uint32_t addr, uint32_t nxt, uint32_t altnext) "q %p - QTD @ %08x: next %08x altnext %08x"
-usb_ehci_qtd_fields(uint32_t addr, int tbytes, int cpage, int cerr, int pid) "QTD @ %08x - tbytes %d, cpage %d, cerr %d, pid %d"
-usb_ehci_qtd_bits(uint32_t addr, int ioc, int active, int halt, int babble, int xacterr) "QTD @ %08x - ioc %d, active %d, halt %d, babble %d, xacterr %d"
-usb_ehci_itd(uint32_t addr, uint32_t nxt, uint32_t mplen, uint32_t mult, uint32_t ep, uint32_t devaddr) "ITD @ %08x: next %08x - mplen %d, mult %d, ep %d, dev %d"
-usb_ehci_sitd(uint32_t addr, uint32_t nxt, uint32_t active) "ITD @ %08x: next %08x - active %d"
-usb_ehci_port_attach(uint32_t port, const char *owner, const char *device) "attach port #%d, owner %s, device %s"
-usb_ehci_port_detach(uint32_t port, const char *owner) "detach port #%d, owner %s"
-usb_ehci_port_reset(uint32_t port, int enable) "reset port #%d - %d"
-usb_ehci_port_suspend(uint32_t port) "port #%d"
-usb_ehci_port_wakeup(uint32_t port) "port #%d"
-usb_ehci_port_resume(uint32_t port) "port #%d"
-usb_ehci_queue_action(void *q, const char *action) "q %p: %s"
-usb_ehci_packet_action(void *q, void *p, const char *action) "q %p p %p: %s"
-usb_ehci_irq(uint32_t level, uint32_t frindex, uint32_t sts, uint32_t mask) "level %d, frindex 0x%04x, sts 0x%x, mask 0x%x"
-usb_ehci_guest_bug(const char *reason) "%s"
-usb_ehci_doorbell_ring(void) ""
-usb_ehci_doorbell_ack(void) ""
-usb_ehci_dma_error(void) ""
-
-# hw/usb/hcd-uhci.c
-usb_uhci_reset(void) "=== RESET ==="
-usb_uhci_exit(void) "=== EXIT ==="
-usb_uhci_schedule_start(void) ""
-usb_uhci_schedule_stop(void) ""
-usb_uhci_frame_start(uint32_t num) "nr %d"
-usb_uhci_frame_stop_bandwidth(void) ""
-usb_uhci_frame_loop_stop_idle(void) ""
-usb_uhci_frame_loop_continue(void) ""
-usb_uhci_mmio_readw(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%04x"
-usb_uhci_mmio_writew(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%04x"
-usb_uhci_queue_add(uint32_t token) "token 0x%x"
-usb_uhci_queue_del(uint32_t token, const char *reason) "token 0x%x: %s"
-usb_uhci_packet_add(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_link_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_unlink_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_cancel(uint32_t token, uint32_t addr, int done) "token 0x%x, td 0x%x, done %d"
-usb_uhci_packet_complete_success(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_complete_shortxfer(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_complete_stall(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_complete_babble(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_complete_error(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_packet_del(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
-usb_uhci_qh_load(uint32_t qh) "qh 0x%x"
-usb_uhci_td_load(uint32_t qh, uint32_t td, uint32_t ctrl, uint32_t token) "qh 0x%x, td 0x%x, ctrl 0x%x, token 0x%x"
-usb_uhci_td_queue(uint32_t td, uint32_t ctrl, uint32_t token) "td 0x%x, ctrl 0x%x, token 0x%x"
-usb_uhci_td_nextqh(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
-usb_uhci_td_async(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
-usb_uhci_td_complete(uint32_t qh, uint32_t td) "qh 0x%x, td 0x%x"
-
-# hw/usb/hcd-xhci.c
-usb_xhci_reset(void) "=== RESET ==="
-usb_xhci_exit(void) "=== EXIT ==="
-usb_xhci_run(void) ""
-usb_xhci_stop(void) ""
-usb_xhci_cap_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
-usb_xhci_oper_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
-usb_xhci_port_read(uint32_t port, uint32_t off, uint32_t val) "port %d, off 0x%04x, ret 0x%08x"
-usb_xhci_runtime_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
-usb_xhci_doorbell_read(uint32_t off, uint32_t val) "off 0x%04x, ret 0x%08x"
-usb_xhci_oper_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
-usb_xhci_port_write(uint32_t port, uint32_t off, uint32_t val) "port %d, off 0x%04x, val 0x%08x"
-usb_xhci_runtime_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
-usb_xhci_doorbell_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
-usb_xhci_irq_intx(uint32_t level) "level %d"
-usb_xhci_irq_msi(uint32_t nr) "nr %d"
-usb_xhci_irq_msix(uint32_t nr) "nr %d"
-usb_xhci_irq_msix_use(uint32_t nr) "nr %d"
-usb_xhci_irq_msix_unuse(uint32_t nr) "nr %d"
-usb_xhci_queue_event(uint32_t vector, uint32_t idx, const char *trb, const char *evt, uint64_t param, uint32_t status, uint32_t control) "v %d, idx %d, %s, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
-usb_xhci_fetch_trb(uint64_t addr, const char *name, uint64_t param, uint32_t status, uint32_t control) "addr %016" PRIx64 ", %s, p %016" PRIx64 ", s %08x, c 0x%08x"
-usb_xhci_port_reset(uint32_t port, bool warm) "port %d, warm %d"
-usb_xhci_port_link(uint32_t port, uint32_t pls) "port %d, pls %d"
-usb_xhci_port_notify(uint32_t port, uint32_t pls) "port %d, bits %x"
-usb_xhci_slot_enable(uint32_t slotid) "slotid %d"
-usb_xhci_slot_disable(uint32_t slotid) "slotid %d"
-usb_xhci_slot_address(uint32_t slotid, const char *port) "slotid %d, port %s"
-usb_xhci_slot_configure(uint32_t slotid) "slotid %d"
-usb_xhci_slot_evaluate(uint32_t slotid) "slotid %d"
-usb_xhci_slot_reset(uint32_t slotid) "slotid %d"
-usb_xhci_ep_enable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
-usb_xhci_ep_disable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
-usb_xhci_ep_set_dequeue(uint32_t slotid, uint32_t epid, uint32_t streamid, uint64_t param) "slotid %d, epid %d, streamid %d, ptr %016" PRIx64
-usb_xhci_ep_kick(uint32_t slotid, uint32_t epid, uint32_t streamid) "slotid %d, epid %d, streamid %d"
-usb_xhci_ep_stop(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
-usb_xhci_ep_reset(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
-usb_xhci_ep_state(uint32_t slotid, uint32_t epid, const char *os, const char *ns) "slotid %d, epid %d, %s -> %s"
-usb_xhci_xfer_start(void *xfer, uint32_t slotid, uint32_t epid, uint32_t streamid) "%p: slotid %d, epid %d, streamid %d"
-usb_xhci_xfer_async(void *xfer) "%p"
-usb_xhci_xfer_nak(void *xfer) "%p"
-usb_xhci_xfer_retry(void *xfer) "%p"
-usb_xhci_xfer_success(void *xfer, uint32_t bytes) "%p: len %d"
-usb_xhci_xfer_error(void *xfer, uint32_t ret) "%p: ret %d"
-usb_xhci_unimplemented(const char *item, int nr) "%s (0x%x)"
-
-# hw/usb/desc.c
-usb_desc_device(int addr, int len, int ret) "dev %d query device, len %d, ret %d"
-usb_desc_device_qualifier(int addr, int len, int ret) "dev %d query device qualifier, len %d, ret %d"
-usb_desc_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
-usb_desc_other_speed_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
-usb_desc_string(int addr, int index, int len, int ret) "dev %d query string %d, len %d, ret %d"
-usb_desc_bos(int addr, int len, int ret) "dev %d bos, len %d, ret %d"
-usb_desc_msos(int addr, int index, int len, int ret) "dev %d msos, index 0x%x, len %d, ret %d"
-usb_set_addr(int addr) "dev %d"
-usb_set_config(int addr, int config, int ret) "dev %d, config %d, ret %d"
-usb_set_interface(int addr, int iface, int alt, int ret) "dev %d, interface %d, altsetting %d, ret %d"
-usb_clear_device_feature(int addr, int feature, int ret) "dev %d, feature %d, ret %d"
-usb_set_device_feature(int addr, int feature, int ret) "dev %d, feature %d, ret %d"
-
-# hw/usb/dev-hub.c
-usb_hub_reset(int addr) "dev %d"
-usb_hub_control(int addr, int request, int value, int index, int length) "dev %d, req 0x%x, value %d, index %d, langth %d"
-usb_hub_get_port_status(int addr, int nr, int status, int changed) "dev %d, port %d, status 0x%x, changed 0x%x"
-usb_hub_set_port_feature(int addr, int nr, const char *f) "dev %d, port %d, feature %s"
-usb_hub_clear_port_feature(int addr, int nr, const char *f) "dev %d, port %d, feature %s"
-usb_hub_attach(int addr, int nr) "dev %d, port %d"
-usb_hub_detach(int addr, int nr) "dev %d, port %d"
-usb_hub_status_report(int addr, int status) "dev %d, status 0x%x"
-
-# hw/usb/dev-uas.c
-usb_uas_reset(int addr) "dev %d"
-usb_uas_command(int addr, uint16_t tag, int lun, uint32_t lun64_1, uint32_t lun64_2) "dev %d, tag 0x%x, lun %d, lun64 %08x-%08x"
-usb_uas_response(int addr, uint16_t tag, uint8_t code) "dev %d, tag 0x%x, code 0x%x"
-usb_uas_sense(int addr, uint16_t tag, uint8_t status) "dev %d, tag 0x%x, status 0x%x"
-usb_uas_read_ready(int addr, uint16_t tag) "dev %d, tag 0x%x"
-usb_uas_write_ready(int addr, uint16_t tag) "dev %d, tag 0x%x"
-usb_uas_xfer_data(int addr, uint16_t tag, uint32_t copy, uint32_t uoff, uint32_t usize, uint32_t soff, uint32_t ssize) "dev %d, tag 0x%x, copy %d, usb-pkt %d/%d, scsi-buf %d/%d"
-usb_uas_scsi_data(int addr, uint16_t tag, uint32_t bytes) "dev %d, tag 0x%x, bytes %d"
-usb_uas_scsi_complete(int addr, uint16_t tag, uint32_t status, uint32_t resid) "dev %d, tag 0x%x, status 0x%x, residue %d"
-usb_uas_tmf_abort_task(int addr, uint16_t tag, uint16_t task_tag) "dev %d, tag 0x%x, task-tag 0x%x"
-usb_uas_tmf_logical_unit_reset(int addr, uint16_t tag, int lun) "dev %d, tag 0x%x, lun %d"
-usb_uas_tmf_unsupported(int addr, uint16_t tag, uint32_t function) "dev %d, tag 0x%x, function 0x%x"
-
-# hw/usb/dev-mtp.c
-usb_mtp_reset(int addr) "dev %d"
-usb_mtp_command(int dev, uint16_t code, uint32_t trans, uint32_t arg0, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint32_t arg4) "dev %d, code 0x%x, trans 0x%x, args 0x%x, 0x%x, 0x%x, 0x%x, 0x%x"
-usb_mtp_success(int dev, uint32_t trans, uint32_t arg0, uint32_t arg1) "dev %d, trans 0x%x, args 0x%x, 0x%x"
-usb_mtp_error(int dev, uint16_t code, uint32_t trans, uint32_t arg0, uint32_t arg1) "dev %d, code 0x%x, trans 0x%x, args 0x%x, 0x%x"
-usb_mtp_data_in(int dev, uint32_t trans, uint32_t len) "dev %d, trans 0x%x, len %d"
-usb_mtp_xfer(int dev, uint32_t ep, uint32_t dlen, uint32_t plen) "dev %d, ep %d, %d/%d"
-usb_mtp_nak(int dev, uint32_t ep) "dev %d, ep %d"
-usb_mtp_stall(int dev, const char *reason) "dev %d, reason: %s"
-usb_mtp_op_get_device_info(int dev) "dev %d"
-usb_mtp_op_open_session(int dev) "dev %d"
-usb_mtp_op_close_session(int dev) "dev %d"
-usb_mtp_op_get_storage_ids(int dev) "dev %d"
-usb_mtp_op_get_storage_info(int dev) "dev %d"
-usb_mtp_op_get_num_objects(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
-usb_mtp_op_get_object_handles(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
-usb_mtp_op_get_object_info(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
-usb_mtp_op_get_object(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
-usb_mtp_op_get_partial_object(int dev, uint32_t handle, const char *path, uint32_t offset, uint32_t length) "dev %d, handle 0x%x, path %s, off %d, len %d"
-usb_mtp_op_unknown(int dev, uint32_t code) "dev %d, command code 0x%x"
-usb_mtp_object_alloc(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
-usb_mtp_object_free(int dev, uint32_t handle, const char *path) "dev %d, handle 0x%x, path %s"
-
-# hw/usb/host-libusb.c
-usb_host_open_started(int bus, int addr) "dev %d:%d"
-usb_host_open_success(int bus, int addr) "dev %d:%d"
-usb_host_open_failure(int bus, int addr) "dev %d:%d"
-usb_host_close(int bus, int addr) "dev %d:%d"
-usb_host_attach_kernel(int bus, int addr, int interfaze) "dev %d:%d, if %d"
-usb_host_detach_kernel(int bus, int addr, int interfaze) "dev %d:%d, if %d"
-usb_host_set_address(int bus, int addr, int config) "dev %d:%d, address %d"
-usb_host_set_config(int bus, int addr, int config) "dev %d:%d, config %d"
-usb_host_set_interface(int bus, int addr, int interfaze, int alt) "dev %d:%d, interface %d, alt %d"
-usb_host_claim_interface(int bus, int addr, int config, int interfaze) "dev %d:%d, config %d, if %d"
-usb_host_release_interface(int bus, int addr, int interfaze) "dev %d:%d, if %d"
-usb_host_req_control(int bus, int addr, void *p, int req, int value, int index) "dev %d:%d, packet %p, req 0x%x, value %d, index %d"
-usb_host_req_data(int bus, int addr, void *p, int in, int ep, int size) "dev %d:%d, packet %p, in %d, ep %d, size %d"
-usb_host_req_complete(int bus, int addr, void *p, int status, int length) "dev %d:%d, packet %p, status %d, length %d"
-usb_host_req_emulated(int bus, int addr, void *p, int status) "dev %d:%d, packet %p, status %d"
-usb_host_req_canceled(int bus, int addr, void *p) "dev %d:%d, packet %p"
-usb_host_iso_start(int bus, int addr, int ep) "dev %d:%d, ep %d"
-usb_host_iso_stop(int bus, int addr, int ep) "dev %d:%d, ep %d"
-usb_host_iso_out_of_bufs(int bus, int addr, int ep) "dev %d:%d, ep %d"
-usb_host_reset(int bus, int addr) "dev %d:%d"
-usb_host_auto_scan_enabled(void)
-usb_host_auto_scan_disabled(void)
-usb_host_parse_config(int bus, int addr, int value, int active) "dev %d:%d, value %d, active %d"
-usb_host_parse_interface(int bus, int addr, int num, int alt, int active) "dev %d:%d, num %d, alt %d, active %d"
-usb_host_parse_endpoint(int bus, int addr, int ep, const char *dir, const char *type, int active) "dev %d:%d, ep %d, %s, %s, active %d"
-usb_host_parse_error(int bus, int addr, const char *errmsg) "dev %d:%d, msg %s"
-
-# hw/scsi/scsi-bus.c
-scsi_req_alloc(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_req_cancel(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_req_data(int target, int lun, int tag, int len) "target %d lun %d tag %d len %d"
-scsi_req_data_canceled(int target, int lun, int tag, int len) "target %d lun %d tag %d len %d"
-scsi_req_dequeue(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_req_continue(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_req_continue_canceled(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_req_parsed(int target, int lun, int tag, int cmd, int mode, int xfer) "target %d lun %d tag %d command %d dir %d length %d"
-scsi_req_parsed_lba(int target, int lun, int tag, int cmd, uint64_t lba) "target %d lun %d tag %d command %d lba %"PRIu64
-scsi_req_parse_bad(int target, int lun, int tag, int cmd) "target %d lun %d tag %d command %d"
-scsi_req_build_sense(int target, int lun, int tag, int key, int asc, int ascq) "target %d lun %d tag %d key %#02x asc %#02x ascq %#02x"
-scsi_device_set_ua(int target, int lun, int key, int asc, int ascq) "target %d lun %d key %#02x asc %#02x ascq %#02x"
-scsi_report_luns(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_inquiry(int target, int lun, int tag, int cdb1, int cdb2) "target %d lun %d tag %d page %#02x/%#02x"
-scsi_test_unit_ready(int target, int lun, int tag) "target %d lun %d tag %d"
-scsi_request_sense(int target, int lun, int tag) "target %d lun %d tag %d"
+virtio_balloon_handle_output(const char *name, uint64_t gpa) "section name: %s gpa: %"PRIx64
+virtio_balloon_get_config(uint32_t num_pages, uint32_t actual) "num_pages: %d actual: %d"
+virtio_balloon_set_config(uint32_t actual, uint32_t oldactual) "actual: %d oldactual: %d"
+virtio_balloon_to_target(uint64_t target, uint32_t num_pages) "balloon target: %"PRIx64" num_pages: %d"
 
 # vl.c
 vm_state_notify(int running, int reason) "running %d reason %d"
 load_file(const char *name, const char *path) "name %s location %s"
 runstate_set(int new_state) "new state %d"
-g_malloc(size_t size, void *ptr) "size %zu ptr %p"
-g_realloc(void *ptr, size_t size, void *newptr) "ptr %p size %zu newptr %p"
-g_free(void *ptr) "ptr %p"
 system_wakeup_request(int reason) "reason=%d"
 qemu_system_shutdown_request(void) ""
 qemu_system_powerdown_request(void) ""
 
-# block/qcow2.c
-qcow2_writev_start_req(void *co, int64_t sector, int nb_sectors) "co %p sector %" PRIx64 " nb_sectors %d"
-qcow2_writev_done_req(void *co, int ret) "co %p ret %d"
-qcow2_writev_start_part(void *co) "co %p"
-qcow2_writev_done_part(void *co, int cur_nr_sectors) "co %p cur_nr_sectors %d"
-qcow2_writev_data(void *co, uint64_t offset) "co %p offset %" PRIx64
-
-# block/qcow2-cluster.c
-qcow2_alloc_clusters_offset(void *co, uint64_t offset, int num) "co %p offset %" PRIx64 " num %d"
-qcow2_handle_copied(void *co, uint64_t guest_offset, uint64_t host_offset, uint64_t bytes) "co %p guest_offset %" PRIx64 " host_offset %" PRIx64 " bytes %" PRIx64
-qcow2_handle_alloc(void *co, uint64_t guest_offset, uint64_t host_offset, uint64_t bytes) "co %p guest_offset %" PRIx64 " host_offset %" PRIx64 " bytes %" PRIx64
-qcow2_do_alloc_clusters_offset(void *co, uint64_t guest_offset, uint64_t host_offset, int nb_clusters) "co %p guest_offset %" PRIx64 " host_offset %" PRIx64 " nb_clusters %d"
-qcow2_cluster_alloc_phys(void *co) "co %p"
-qcow2_cluster_link_l2(void *co, int nb_clusters) "co %p nb_clusters %d"
-
-qcow2_l2_allocate(void *bs, int l1_index) "bs %p l1_index %d"
-qcow2_l2_allocate_get_empty(void *bs, int l1_index) "bs %p l1_index %d"
-qcow2_l2_allocate_write_l2(void *bs, int l1_index) "bs %p l1_index %d"
-qcow2_l2_allocate_write_l1(void *bs, int l1_index) "bs %p l1_index %d"
-qcow2_l2_allocate_done(void *bs, int l1_index, int ret) "bs %p l1_index %d ret %d"
-
-# block/qcow2-cache.c
-qcow2_cache_get(void *co, int c, uint64_t offset, bool read_from_disk) "co %p is_l2_cache %d offset %" PRIx64 " read_from_disk %d"
-qcow2_cache_get_replace_entry(void *co, int c, int i) "co %p is_l2_cache %d index %d"
-qcow2_cache_get_read(void *co, int c, int i) "co %p is_l2_cache %d index %d"
-qcow2_cache_get_done(void *co, int c, int i) "co %p is_l2_cache %d index %d"
-qcow2_cache_flush(void *co, int c) "co %p is_l2_cache %d"
-qcow2_cache_entry_flush(void *co, int c, int i) "co %p is_l2_cache %d index %d"
-
-# block/qed-l2-cache.c
-qed_alloc_l2_cache_entry(void *l2_cache, void *entry) "l2_cache %p entry %p"
-qed_unref_l2_cache_entry(void *entry, int ref) "entry %p ref %d"
-qed_find_l2_cache_entry(void *l2_cache, void *entry, uint64_t offset, int ref) "l2_cache %p entry %p offset %"PRIu64" ref %d"
-
-# block/qed-table.c
-qed_read_table(void *s, uint64_t offset, void *table) "s %p offset %"PRIu64" table %p"
-qed_read_table_cb(void *s, void *table, int ret) "s %p table %p ret %d"
-qed_write_table(void *s, uint64_t offset, void *table, unsigned int index, unsigned int n) "s %p offset %"PRIu64" table %p index %u n %u"
-qed_write_table_cb(void *s, void *table, int flush, int ret) "s %p table %p flush %d ret %d"
-
-# block/qed.c
-qed_need_check_timer_cb(void *s) "s %p"
-qed_start_need_check_timer(void *s) "s %p"
-qed_cancel_need_check_timer(void *s) "s %p"
-qed_aio_complete(void *s, void *acb, int ret) "s %p acb %p ret %d"
-qed_aio_setup(void *s, void *acb, int64_t sector_num, int nb_sectors, void *opaque, int flags) "s %p acb %p sector_num %"PRId64" nb_sectors %d opaque %p flags %#x"
-qed_aio_next_io(void *s, void *acb, int ret, uint64_t cur_pos) "s %p acb %p ret %d cur_pos %"PRIu64
-qed_aio_read_data(void *s, void *acb, int ret, uint64_t offset, size_t len) "s %p acb %p ret %d offset %"PRIu64" len %zu"
-qed_aio_write_data(void *s, void *acb, int ret, uint64_t offset, size_t len) "s %p acb %p ret %d offset %"PRIu64" len %zu"
-qed_aio_write_prefill(void *s, void *acb, uint64_t start, size_t len, uint64_t offset) "s %p acb %p start %"PRIu64" len %zu offset %"PRIu64
-qed_aio_write_postfill(void *s, void *acb, uint64_t start, size_t len, uint64_t offset) "s %p acb %p start %"PRIu64" len %zu offset %"PRIu64
-qed_aio_write_main(void *s, void *acb, int ret, uint64_t offset, size_t len) "s %p acb %p ret %d offset %"PRIu64" len %zu"
-
-# hw/display/g364fb.c
-g364fb_read(uint64_t addr, uint32_t val) "read addr=0x%"PRIx64": 0x%x"
-g364fb_write(uint64_t addr, uint32_t new) "write addr=0x%"PRIx64": 0x%x"
-
-# hw/timer/grlib_gptimer.c
-grlib_gptimer_enable(int id, uint32_t count) "timer:%d set count 0x%x and run"
-grlib_gptimer_disabled(int id, uint32_t config) "timer:%d Timer disable config 0x%x"
-grlib_gptimer_restart(int id, uint32_t reload) "timer:%d reload val: 0x%x"
-grlib_gptimer_set_scaler(uint32_t scaler, uint32_t freq) "scaler:0x%x freq: 0x%x"
-grlib_gptimer_hit(int id) "timer:%d HIT"
-grlib_gptimer_readl(int id, uint64_t addr, uint32_t val) "timer:%d addr 0x%"PRIx64" 0x%x"
-grlib_gptimer_writel(int id, uint64_t addr, uint32_t val) "timer:%d addr 0x%"PRIx64" 0x%x"
-
-# hw/intc/grlib_irqmp.c
-grlib_irqmp_check_irqs(uint32_t pend, uint32_t force, uint32_t mask, uint32_t lvl1, uint32_t lvl2) "pend:0x%04x force:0x%04x mask:0x%04x lvl1:0x%04x lvl0:0x%04x"
-grlib_irqmp_ack(int intno) "interrupt:%d"
-grlib_irqmp_set_irq(int irq) "Raise CPU IRQ %d"
-grlib_irqmp_readl_unknown(uint64_t addr) "addr 0x%"PRIx64
-grlib_irqmp_writel_unknown(uint64_t addr, uint32_t value) "addr 0x%"PRIx64" value 0x%x"
-
-# hw/char/grlib_apbuart.c
-grlib_apbuart_event(int event) "event:%d"
-grlib_apbuart_writel_unknown(uint64_t addr, uint32_t value) "addr 0x%"PRIx64" value 0x%x"
-grlib_apbuart_readl_unknown(uint64_t addr) "addr 0x%"PRIx64""
-
-# hw/sparc/leon3.c
-leon3_set_irq(int intno) "Set CPU IRQ %d"
-leon3_reset_irq(int intno) "Reset CPU IRQ %d"
-
 # spice-qemu-char.c
 spice_vmc_write(ssize_t out, int len) "spice wrottn %zd of requested %d"
 spice_vmc_read(int bytes, int len) "spice read %d of requested %d"
@@ -667,245 +57,32 @@
 spice_vmc_unregister_interface(void *scd) "spice vmc unregistered interface %p"
 spice_vmc_event(int event) "spice vmc event %d"
 
-# hw/intc/lm32_pic.c
-lm32_pic_raise_irq(void) "Raise CPU interrupt"
-lm32_pic_lower_irq(void) "Lower CPU interrupt"
-lm32_pic_interrupt(int irq, int level) "Set IRQ%d %d"
-lm32_pic_set_im(uint32_t im) "im 0x%08x"
-lm32_pic_set_ip(uint32_t ip) "ip 0x%08x"
-lm32_pic_get_im(uint32_t im) "im 0x%08x"
-lm32_pic_get_ip(uint32_t ip) "ip 0x%08x"
-
-# hw/char/lm32_juart.c
-lm32_juart_get_jtx(uint32_t value) "jtx 0x%08x"
-lm32_juart_set_jtx(uint32_t value) "jtx 0x%08x"
-lm32_juart_get_jrx(uint32_t value) "jrx 0x%08x"
-lm32_juart_set_jrx(uint32_t value) "jrx 0x%08x"
-
-# hw/timer/lm32_timer.c
-lm32_timer_memory_write(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
-lm32_timer_memory_read(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
-lm32_timer_hit(void) "timer hit"
-lm32_timer_irq_state(int level) "irq state %d"
-
-# hw/char/lm32_uart.c
-lm32_uart_memory_write(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
-lm32_uart_memory_read(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
-lm32_uart_irq_state(int level) "irq state %d"
-
-# hw/scsi/megasas.c
-megasas_init_firmware(uint64_t pa) "pa %" PRIx64 " "
-megasas_init_queue(uint64_t queue_pa, int queue_len, uint64_t head, uint64_t tail, uint32_t flags) "queue at %" PRIx64 " len %d head %" PRIx64 " tail %" PRIx64 " flags %x"
-megasas_initq_map_failed(int frame) "scmd %d: failed to map queue"
-megasas_initq_mapped(uint64_t pa) "queue already mapped at %" PRIx64 ""
-megasas_initq_mismatch(int queue_len, int fw_cmds) "queue size %d max fw cmds %d"
-megasas_qf_mapped(unsigned int index) "skip mapped frame %x"
-megasas_qf_new(unsigned int index, uint64_t frame) "frame %x addr %" PRIx64 ""
-megasas_qf_busy(unsigned long pa) "all frames busy for frame %lx"
-megasas_qf_enqueue(unsigned int index, unsigned int count, uint64_t context, unsigned int head, unsigned int tail, int busy) "frame %x count %d context %" PRIx64 " head %x tail %x busy %d"
-megasas_qf_update(unsigned int head, unsigned int tail, unsigned int busy) "head %x tail %x busy %d"
-megasas_qf_map_failed(int cmd, unsigned long frame) "scmd %d: frame %lu"
-megasas_qf_complete_noirq(uint64_t context) "context %" PRIx64 " "
-megasas_qf_complete(uint64_t context, unsigned int head, unsigned int tail, int busy) "context %" PRIx64 " head %x tail %x busy %d"
-megasas_frame_busy(uint64_t addr) "frame %" PRIx64 " busy"
-megasas_unhandled_frame_cmd(int cmd, uint8_t frame_cmd) "scmd %d: MFI cmd %x"
-megasas_handle_scsi(const char *frame, int bus, int dev, int lun, void *sdev, unsigned long size) "%s dev %x/%x/%x sdev %p xfer %lu"
-megasas_scsi_target_not_present(const char *frame, int bus, int dev, int lun) "%s dev %x/%x/%x"
-megasas_scsi_invalid_cdb_len(const char *frame, int bus, int dev, int lun, int len) "%s dev %x/%x/%x invalid cdb len %d"
-megasas_iov_read_overflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
-megasas_iov_write_overflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
-megasas_iov_read_underflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
-megasas_iov_write_underflow(int cmd, int bytes, int len) "scmd %d: %d/%d bytes"
-megasas_scsi_req_alloc_failed(const char *frame, int dev, int lun) "%s dev %x/%x"
-megasas_scsi_read_start(int cmd, int len) "scmd %d: transfer %d bytes of data"
-megasas_scsi_write_start(int cmd, int len) "scmd %d: transfer %d bytes of data"
-megasas_scsi_nodata(int cmd) "scmd %d: no data to be transferred"
-megasas_scsi_complete(int cmd, uint32_t status, int len, int xfer) "scmd %d: status %x, len %u/%u"
-megasas_command_complete(int cmd, uint32_t status, uint32_t resid) "scmd %d: status %x, residual %d"
-megasas_handle_io(int cmd, const char *frame, int dev, int lun, unsigned long lba, unsigned long count) "scmd %d: %s dev %x/%x lba %lx count %lu"
-megasas_io_target_not_present(int cmd, const char *frame, int dev, int lun) "scmd %d: %s dev 1/%x/%x LUN not present"
-megasas_io_read_start(int cmd, unsigned long lba, unsigned long count, unsigned long len) "scmd %d: start LBA %lx %lu blocks (%lu bytes)"
-megasas_io_write_start(int cmd, unsigned long lba, unsigned long count, unsigned long len) "scmd %d: start LBA %lx %lu blocks (%lu bytes)"
-megasas_io_complete(int cmd, uint32_t len) "scmd %d: %d bytes"
-megasas_iovec_sgl_overflow(int cmd, int index, int limit) "scmd %d: iovec count %d limit %d"
-megasas_iovec_sgl_underflow(int cmd, int index) "scmd %d: iovec count %d"
-megasas_iovec_sgl_invalid(int cmd, int index, uint64_t pa, uint32_t len) "scmd %d: element %d pa %" PRIx64 " len %u"
-megasas_iovec_overflow(int cmd, int len, int limit) "scmd %d: len %d limit %d"
-megasas_iovec_underflow(int cmd, int len, int limit) "scmd %d: len %d limit %d"
-megasas_handle_dcmd(int cmd, int opcode) "scmd %d: MFI DCMD opcode %x"
-megasas_finish_dcmd(int cmd, int size) "scmd %d: MFI DCMD wrote %d bytes"
-megasas_dcmd_req_alloc_failed(int cmd, const char *desc) "scmd %d: %s"
-megasas_dcmd_internal_submit(int cmd, const char *desc, int dev) "scmd %d: %s to dev %d"
-megasas_dcmd_internal_finish(int cmd, int opcode, int lun) "scmd %d: cmd %x lun %d"
-megasas_dcmd_internal_invalid(int cmd, int opcode) "scmd %d: DCMD %x"
-megasas_dcmd_unhandled(int cmd, int opcode, int len) "scmd %d: opcode %x, len %d"
-megasas_dcmd_zero_sge(int cmd) "scmd %d: zero DCMD sge count"
-megasas_dcmd_invalid_sge(int cmd, int count) "scmd %d: DCMD sge count %d"
-megasas_dcmd_invalid_xfer_len(int cmd, unsigned long size, unsigned long max) "scmd %d: xfer len %ld, max %ld"
-megasas_dcmd_enter(int cmd, const char *dcmd, int len) "scmd %d: DCMD %s len %d"
-megasas_dcmd_dummy(int cmd, unsigned long size) "scmd %d: xfer len %ld"
-megasas_dcmd_set_fw_time(int cmd, unsigned long time) "scmd %d: Set FW time %lx"
-megasas_dcmd_pd_get_list(int cmd, int num, int max, int offset) "scmd %d: DCMD PD get list: %d / %d PDs, size %d"
-megasas_dcmd_ld_get_list(int cmd, int num, int max) "scmd %d: DCMD LD get list: found %d / %d LDs"
-megasas_dcmd_ld_get_info(int cmd, int ld_id) "scmd %d: dev %d"
-megasas_dcmd_ld_list_query(int cmd, int flags) "scmd %d: query flags %x"
-megasas_dcmd_pd_get_info(int cmd, int pd_id) "scmd %d: dev %d"
-megasas_dcmd_pd_list_query(int cmd, int flags) "scmd %d: query flags %x"
-megasas_dcmd_reset_ld(int cmd, int target_id) "scmd %d: dev %d"
-megasas_dcmd_unsupported(int cmd, unsigned long size) "scmd %d: set properties len %ld"
-megasas_abort_frame(int cmd, int abort_cmd) "scmd %d: frame %x"
-megasas_abort_no_cmd(int cmd, uint64_t context) "scmd %d: no active command for frame context %" PRIx64 ""
-megasas_abort_invalid_context(int cmd, uint64_t context, int abort_cmd) "scmd %d: invalid frame context %" PRIx64 " for abort frame %x"
-megasas_reset(int fw_state) "firmware state %x"
-megasas_init(int sges, int cmds, const char *mode) "Using %d sges, %d cmds, %s mode"
-megasas_msix_raise(int vector) "vector %d"
-megasas_msi_raise(int vector) "vector %d"
-megasas_irq_lower(void) "INTx"
-megasas_irq_raise(void) "INTx"
-megasas_intr_enabled(void) "Interrupts enabled"
-megasas_intr_disabled(void) "Interrupts disabled"
-megasas_msix_enabled(int vector) "vector %d"
-megasas_msi_enabled(int vector) "vector %d"
-megasas_mmio_readl(const char *reg, uint32_t val) "reg %s: 0x%x"
-megasas_mmio_invalid_readl(unsigned long addr) "addr 0x%lx"
-megasas_mmio_writel(const char *reg, uint32_t val) "reg %s: 0x%x"
-megasas_mmio_invalid_writel(uint32_t addr, uint32_t val) "addr 0x%x: 0x%x"
-
-# hw/audio/milkymist-ac97.c
-milkymist_ac97_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_ac97_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_ac97_pulse_irq_crrequest(void) "Pulse IRQ CR request"
-milkymist_ac97_pulse_irq_crreply(void) "Pulse IRQ CR reply"
-milkymist_ac97_pulse_irq_dmaw(void) "Pulse IRQ DMA write"
-milkymist_ac97_pulse_irq_dmar(void) "Pulse IRQ DMA read"
-milkymist_ac97_in_cb(int avail, uint32_t remaining) "avail %d remaining %u"
-milkymist_ac97_in_cb_transferred(int transferred) "transferred %d"
-milkymist_ac97_out_cb(int free, uint32_t remaining) "free %d remaining %u"
-milkymist_ac97_out_cb_transferred(int transferred) "transferred %d"
-
-# hw/misc/milkymist-hpdmc.c
-milkymist_hpdmc_memory_read(uint32_t addr, uint32_t value) "addr=%08x value=%08x"
-milkymist_hpdmc_memory_write(uint32_t addr, uint32_t value) "addr=%08x value=%08x"
-
-# hw/sd/milkymist-memcard.c
-milkymist_memcard_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_memcard_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-
-# hw/net/milkymist-minimac2.c
-milkymist_minimac2_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_minimac2_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_minimac2_mdio_write(uint8_t phy_addr, uint8_t addr, uint16_t value) "phy_addr %02x addr %02x value %04x"
-milkymist_minimac2_mdio_read(uint8_t phy_addr, uint8_t addr, uint16_t value) "phy_addr %02x addr %02x value %04x"
-milkymist_minimac2_tx_frame(uint32_t length) "length %u"
-milkymist_minimac2_rx_frame(const void *buf, uint32_t length) "buf %p length %u"
-milkymist_minimac2_drop_rx_frame(const void *buf) "buf %p"
-milkymist_minimac2_rx_transfer(const void *buf, uint32_t length) "buf %p length %d"
-milkymist_minimac2_raise_irq_rx(void) "Raise IRQ RX"
-milkymist_minimac2_lower_irq_rx(void) "Lower IRQ RX"
-milkymist_minimac2_pulse_irq_tx(void) "Pulse IRQ TX"
-
-# hw/misc/milkymist-pfpu.c
-milkymist_pfpu_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_pfpu_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_pfpu_vectout(uint32_t a, uint32_t b, uint32_t dma_ptr) "a %08x b %08x dma_ptr %08x"
-milkymist_pfpu_pulse_irq(void) "Pulse IRQ"
-
-# hw/input/milkymist-softusb.c
-milkymist_softusb_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_softusb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_softusb_mevt(uint8_t m) "m %d"
-milkymist_softusb_kevt(uint8_t m) "m %d"
-milkymist_softusb_pulse_irq(void) "Pulse IRQ"
-
-# hw/timer/milkymist-sysctl.c
-milkymist_sysctl_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_sysctl_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_sysctl_icap_write(uint32_t value) "value %08x"
-milkymist_sysctl_start_timer0(void) "Start timer0"
-milkymist_sysctl_stop_timer0(void) "Stop timer0"
-milkymist_sysctl_start_timer1(void) "Start timer1"
-milkymist_sysctl_stop_timer1(void) "Stop timer1"
-milkymist_sysctl_pulse_irq_timer0(void) "Pulse IRQ Timer0"
-milkymist_sysctl_pulse_irq_timer1(void) "Pulse IRQ Timer1"
-
-# hw/display/milkymist-tmu2.c
-milkymist_tmu2_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_tmu2_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_tmu2_start(void) "Start TMU"
-milkymist_tmu2_pulse_irq(void) "Pulse IRQ"
-
-# hw/char/milkymist-uart.c
-milkymist_uart_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_uart_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_uart_raise_irq(void) "Raise IRQ"
-milkymist_uart_lower_irq(void) "Lower IRQ"
-
-# hw/display/milkymist-vgafb.c
-milkymist_vgafb_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-milkymist_vgafb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-
-# hw/net/mipsnet.c
-mipsnet_send(uint32_t size) "sending len=%u"
-mipsnet_receive(uint32_t size) "receiving len=%u"
-mipsnet_read(uint64_t addr, uint32_t val) "read addr=0x%" PRIx64 " val=0x%x"
-mipsnet_write(uint64_t addr, uint64_t val) "write addr=0x%" PRIx64 " val=0x%" PRIx64 ""
-mipsnet_irq(uint32_t isr, uint32_t intctl) "set irq to %d (%02x)"
-
-# hw/isa/pc87312.c
-pc87312_io_read(uint32_t addr, uint32_t val) "read addr=%x val=%x"
-pc87312_io_write(uint32_t addr, uint32_t val) "write addr=%x val=%x"
-pc87312_info_floppy(uint32_t base) "base 0x%x"
-pc87312_info_ide(uint32_t base) "base 0x%x"
-pc87312_info_parallel(uint32_t base, uint32_t irq) "base 0x%x, irq %u"
-pc87312_info_serial(int n, uint32_t base, uint32_t irq) "id=%d, base 0x%x, irq %u"
-
-# hw/scsi/vmw_pvscsi.c
-pvscsi_ring_init_data(uint32_t txr_len_log2, uint32_t rxr_len_log2) "TX/RX rings logarithms set to %d/%d"
-pvscsi_ring_init_msg(uint32_t len_log2) "MSG ring logarithm set to %d"
-pvscsi_ring_flush_cmp(uint64_t filled_cmp_ptr) "new production counter of completion ring is 0x%"PRIx64""
-pvscsi_ring_flush_msg(uint64_t filled_cmp_ptr) "new production counter of message ring is 0x%"PRIx64""
-pvscsi_update_irq_level(bool raise, uint64_t mask, uint64_t status) "interrupt level set to %d (MASK: 0x%"PRIx64", STATUS: 0x%"PRIx64")"
-pvscsi_update_irq_msi(void) "sending MSI notification"
-pvscsi_cmp_ring_put(unsigned long addr) "got completion descriptor 0x%lx"
-pvscsi_msg_ring_put(unsigned long addr) "got message descriptor 0x%lx"
-pvscsi_complete_request(uint64_t context, uint64_t len, uint8_t sense_key) "completion: ctx: 0x%"PRIx64", len: 0x%"PRIx64", sense key: %u"
-pvscsi_get_sg_list(int nsg, size_t size) "get SG list: depth: %u, size: %zu"
-pvscsi_get_next_sg_elem(uint32_t flags) "unknown flags in SG element (val: 0x%x)"
-pvscsi_command_complete_not_found(uint32_t tag) "can't find request for tag 0x%x"
-pvscsi_command_complete_data_run(void) "not all data required for command transferred"
-pvscsi_command_complete_sense_len(int len) "sense information length is %d bytes"
-pvscsi_convert_sglist(uint64_t context, unsigned long addr, uint32_t resid) "element: ctx: 0x%"PRIx64" addr: 0x%lx, len: %ul"
-pvscsi_process_req_descr(uint8_t cmd, uint64_t ctx) "SCSI cmd 0x%x, ctx: 0x%"PRIx64""
-pvscsi_process_req_descr_unknown_device(void) "command directed to unknown device rejected"
-pvscsi_process_req_descr_invalid_dir(void) "command with invalid transfer direction rejected"
-pvscsi_process_io(unsigned long addr) "got descriptor 0x%lx"
-pvscsi_on_cmd_noimpl(const char* cmd) "unimplemented command %s ignored"
-pvscsi_on_cmd_reset_dev(uint32_t tgt, int lun, void* dev) "PVSCSI_CMD_RESET_DEVICE[target %u lun %d (dev 0x%p)]"
-pvscsi_on_cmd_arrived(const char* cmd) "command %s arrived"
-pvscsi_on_cmd_abort(uint64_t ctx, uint32_t tgt) "command PVSCSI_CMD_ABORT_CMD for ctx 0x%"PRIx64", target %u"
-pvscsi_on_cmd_unknown(uint64_t cmd_id) "unknown command %"PRIx64""
-pvscsi_on_cmd_unknown_data(uint32_t data) "data for unknown command 0x:%x"
-pvscsi_io_write(const char* cmd, uint64_t val) "%s write: %"PRIx64""
-pvscsi_io_write_unknown(unsigned long addr, unsigned sz, uint64_t val) "unknown write address: 0x%lx size: %u bytes value: 0x%"PRIx64""
-pvscsi_io_read(const char* cmd, uint64_t status) "%s read: 0x%"PRIx64""
-pvscsi_io_read_unknown(unsigned long addr, unsigned sz) "unknown read address: 0x%lx size: %u bytes"
-pvscsi_init_msi_fail(int res) "failed to initialize MSI, error %d"
-pvscsi_state(const char* state) "starting %s ..."
-pvscsi_tx_rings_ppn(const char* label, uint64_t ppn) "%s page: %"PRIx64""
-pvscsi_tx_rings_num_pages(const char* label, uint32_t num) "Number of %s pages: %u"
-
 # xen-hvm.c
 xen_ram_alloc(unsigned long ram_addr, unsigned long size) "requested: %#lx, size %#lx"
 xen_client_set_memory(uint64_t start_addr, unsigned long size, bool log_dirty) "%#"PRIx64" size %#lx, log_dirty %i"
+xen_default_ioreq_server(void) ""
+xen_ioreq_server_create(uint32_t id) "id: %u"
+xen_ioreq_server_destroy(uint32_t id) "id: %u"
+xen_ioreq_server_state(uint32_t id, bool enable) "id: %u: enable: %i"
+xen_map_mmio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %u start: %#"PRIx64" end: %#"PRIx64
+xen_unmap_mmio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %u start: %#"PRIx64" end: %#"PRIx64
+xen_map_portio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %u start: %#"PRIx64" end: %#"PRIx64
+xen_unmap_portio_range(uint32_t id, uint64_t start_addr, uint64_t end_addr) "id: %u start: %#"PRIx64" end: %#"PRIx64
+xen_map_pcidev(uint32_t id, uint8_t bus, uint8_t dev, uint8_t func) "id: %u bdf: %02x.%02x.%02x"
+xen_unmap_pcidev(uint32_t id, uint8_t bus, uint8_t dev, uint8_t func) "id: %u bdf: %02x.%02x.%02x"
+handle_ioreq(void *req, uint32_t type, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p type=%d dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+handle_ioreq_read(void *req, uint32_t type, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p read type=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+handle_ioreq_write(void *req, uint32_t type, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p write type=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+cpu_ioreq_pio(void *req, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p pio dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
+cpu_ioreq_pio_read_reg(void *req, uint64_t data, uint64_t addr, uint32_t size) "I/O=%p pio read reg data=%#"PRIx64" port=%#"PRIx64" size=%d"
+cpu_ioreq_pio_write_reg(void *req, uint64_t data, uint64_t addr, uint32_t size) "I/O=%p pio write reg data=%#"PRIx64" port=%#"PRIx64" size=%d"
+cpu_ioreq_move(void *req, uint32_t dir, uint32_t df, uint32_t data_is_ptr, uint64_t addr, uint64_t data, uint32_t count, uint32_t size) "I/O=%p copy dir=%d df=%d ptr=%d port=%#"PRIx64" data=%#"PRIx64" count=%d size=%d"
 
 # xen-mapcache.c
 xen_map_cache(uint64_t phys_addr) "want %#"PRIx64
 xen_remap_bucket(uint64_t index) "index %#"PRIx64
 xen_map_cache_return(void* ptr) "%p"
 
-# hw/i386/xen/xen_platform.c
-xen_platform_log(char *s) "xen platform: %s"
-
 # qemu-coroutine.c
 qemu_coroutine_enter(void *from, void *to, void *opaque) "from %p to %p opaque %p"
 qemu_coroutine_yield(void *from, void *to) "from %p to %p"
@@ -919,400 +96,21 @@
 qemu_co_mutex_unlock_entry(void *mutex, void *self) "mutex %p self %p"
 qemu_co_mutex_unlock_return(void *mutex, void *self) "mutex %p self %p"
 
-# hw/char/escc.c
-escc_put_queue(char channel, int b) "channel %c put: 0x%02x"
-escc_get_queue(char channel, int val) "channel %c get 0x%02x"
-escc_update_irq(int irq) "IRQ = %d"
-escc_update_parameters(char channel, int speed, int parity, int data_bits, int stop_bits) "channel %c: speed=%d parity=%c data=%d stop=%d"
-escc_mem_writeb_ctrl(char channel, uint32_t reg, uint32_t val) "Write channel %c, reg[%d] = %2.2x"
-escc_mem_writeb_data(char channel, uint32_t val) "Write channel %c, ch %d"
-escc_mem_readb_ctrl(char channel, uint32_t reg, uint8_t val) "Read channel %c, reg[%d] = %2.2x"
-escc_mem_readb_data(char channel, uint32_t ret) "Read channel %c, ch %d"
-escc_serial_receive_byte(char channel, int ch) "channel %c put ch %d"
-escc_sunkbd_event_in(int ch, const char *name, int down) "QKeyCode 0x%2.2x [%s], down %d"
-escc_sunkbd_event_out(int ch) "Translated keycode 0x%2.2x"
-escc_kbd_command(int val) "Command %d"
-escc_sunmouse_event(int dx, int dy, int buttons_state) "dx=%d dy=%d buttons=%01x"
-
-# hw/scsi/esp.c
-esp_error_fifo_overrun(void) "FIFO overrun"
-esp_error_unhandled_command(uint32_t val) "unhandled command (%2.2x)"
-esp_error_invalid_write(uint32_t val, uint32_t addr) "invalid write of 0x%02x at [0x%x]"
-esp_raise_irq(void) "Raise IRQ"
-esp_lower_irq(void) "Lower IRQ"
-esp_dma_enable(void) "Raise enable"
-esp_dma_disable(void) "Lower enable"
-esp_get_cmd(uint32_t dmalen, int target) "len %d target %d"
-esp_do_busid_cmd(uint8_t busid) "busid 0x%x"
-esp_handle_satn_stop(uint32_t cmdlen) "cmdlen %d"
-esp_write_response(uint32_t status) "Transfer status (status=%d)"
-esp_do_dma(uint32_t cmdlen, uint32_t len) "command len %d + %d"
-esp_command_complete(void) "SCSI Command complete"
-esp_command_complete_unexpected(void) "SCSI command completed unexpectedly"
-esp_command_complete_fail(void) "Command failed"
-esp_transfer_data(uint32_t dma_left, int32_t ti_size) "transfer %d/%d"
-esp_handle_ti(uint32_t minlen) "Transfer Information len %d"
-esp_handle_ti_cmd(uint32_t cmdlen) "command len %d"
-esp_mem_readb(uint32_t saddr, uint8_t reg) "reg[%d]: 0x%2.2x"
-esp_mem_writeb(uint32_t saddr, uint8_t reg, uint32_t val) "reg[%d]: 0x%2.2x -> 0x%2.2x"
-esp_mem_writeb_cmd_nop(uint32_t val) "NOP (%2.2x)"
-esp_mem_writeb_cmd_flush(uint32_t val) "Flush FIFO (%2.2x)"
-esp_mem_writeb_cmd_reset(uint32_t val) "Chip reset (%2.2x)"
-esp_mem_writeb_cmd_bus_reset(uint32_t val) "Bus reset (%2.2x)"
-esp_mem_writeb_cmd_iccs(uint32_t val) "Initiator Command Complete Sequence (%2.2x)"
-esp_mem_writeb_cmd_msgacc(uint32_t val) "Message Accepted (%2.2x)"
-esp_mem_writeb_cmd_pad(uint32_t val) "Transfer padding (%2.2x)"
-esp_mem_writeb_cmd_satn(uint32_t val) "Set ATN (%2.2x)"
-esp_mem_writeb_cmd_rstatn(uint32_t val) "Reset ATN (%2.2x)"
-esp_mem_writeb_cmd_sel(uint32_t val) "Select without ATN (%2.2x)"
-esp_mem_writeb_cmd_selatn(uint32_t val) "Select with ATN (%2.2x)"
-esp_mem_writeb_cmd_selatns(uint32_t val) "Select with ATN & stop (%2.2x)"
-esp_mem_writeb_cmd_ensel(uint32_t val) "Enable selection (%2.2x)"
-esp_mem_writeb_cmd_dissel(uint32_t val) "Disable selection (%2.2x)"
-
-# hw/scsi/esp-pci.c
-esp_pci_error_invalid_dma_direction(void) "invalid DMA transfer direction"
-esp_pci_error_invalid_read(uint32_t reg) "read access outside bounds (reg 0x%x)"
-esp_pci_error_invalid_write(uint32_t reg) "write access outside bounds (reg 0x%x)"
-esp_pci_error_invalid_write_dma(uint32_t val, uint32_t addr) "invalid write of 0x%02x at [0x%x]"
-esp_pci_dma_read(uint32_t saddr, uint32_t reg) "reg[%d]: 0x%8.8x"
-esp_pci_dma_write(uint32_t saddr, uint32_t reg, uint32_t val) "reg[%d]: 0x%8.8x -> 0x%8.8x"
-esp_pci_dma_idle(uint32_t val) "IDLE (%.8x)"
-esp_pci_dma_blast(uint32_t val) "BLAST (%.8x)"
-esp_pci_dma_abort(uint32_t val) "ABORT (%.8x)"
-esp_pci_dma_start(uint32_t val) "START (%.8x)"
-esp_pci_sbac_read(uint32_t reg) "sbac: 0x%8.8x"
-esp_pci_sbac_write(uint32_t reg, uint32_t val) "sbac: 0x%8.8x -> 0x%8.8x"
-
 # monitor.c
 handle_qmp_command(void *mon, const char *cmd_name) "mon %p cmd_name \"%s\""
 monitor_protocol_emitter(void *mon) "mon %p"
-monitor_protocol_event_handler(uint32_t event, void *data, uint64_t last, uint64_t now) "event=%d data=%p last=%" PRId64 " now=%" PRId64
+monitor_protocol_event_handler(uint32_t event, void *qdict) "event=%d data=%p"
 monitor_protocol_event_emit(uint32_t event, void *data) "event=%d data=%p"
-monitor_protocol_event_queue(uint32_t event, void *data, uint64_t rate, uint64_t last, uint64_t now) "event=%d data=%p rate=%" PRId64 " last=%" PRId64 " now=%" PRId64
+monitor_protocol_event_queue(uint32_t event, void *qdict, uint64_t rate) "event=%d data=%p rate=%" PRId64
 monitor_protocol_event_throttle(uint32_t event, uint64_t rate) "event=%d rate=%" PRId64
 
-# hw/net/opencores_eth.c
-open_eth_mii_write(unsigned idx, uint16_t v) "MII[%02x] <- %04x"
-open_eth_mii_read(unsigned idx, uint16_t v) "MII[%02x] -> %04x"
-open_eth_update_irq(uint32_t v) "IRQ <- %x"
-open_eth_receive(unsigned len) "RX: len: %u"
-open_eth_receive_mcast(unsigned idx, uint32_t h0, uint32_t h1) "MCAST: idx = %u, hash: %08x:%08x"
-open_eth_receive_reject(void) "RX: rejected"
-open_eth_receive_desc(uint32_t addr, uint32_t len_flags) "RX: %08x, len_flags: %08x"
-open_eth_start_xmit(uint32_t addr, unsigned len, unsigned tx_len) "TX: %08x, len: %u, tx_len: %u"
-open_eth_reg_read(uint32_t addr, uint32_t v) "MAC[%02x] -> %08x"
-open_eth_reg_write(uint32_t addr, uint32_t v) "MAC[%02x] <- %08x"
-open_eth_desc_read(uint32_t addr, uint32_t v) "DESC[%04x] -> %08x"
-open_eth_desc_write(uint32_t addr, uint32_t v) "DESC[%04x] <- %08x"
-
-# hw/9pfs/virtio-9p.c
-v9fs_rerror(uint16_t tag, uint8_t id, int err) "tag %d id %d err %d"
-v9fs_version(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
-v9fs_version_return(uint16_t tag, uint8_t id, int32_t msize, char* version) "tag %d id %d msize %d version %s"
-v9fs_attach(uint16_t tag, uint8_t id, int32_t fid, int32_t afid, char* uname, char* aname) "tag %u id %u fid %d afid %d uname %s aname %s"
-v9fs_attach_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d type %d version %d path %"PRId64""
-v9fs_stat(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
-v9fs_stat_return(uint16_t tag, uint8_t id, int32_t mode, int32_t atime, int32_t mtime, int64_t length) "tag %d id %d stat={mode %d atime %d mtime %d length %"PRId64"}"
-v9fs_getattr(uint16_t tag, uint8_t id, int32_t fid, uint64_t request_mask) "tag %d id %d fid %d request_mask %"PRIu64""
-v9fs_getattr_return(uint16_t tag, uint8_t id, uint64_t result_mask, uint32_t mode, uint32_t uid, uint32_t gid) "tag %d id %d getattr={result_mask %"PRId64" mode %u uid %u gid %u}"
-v9fs_walk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, uint16_t nwnames) "tag %d id %d fid %d newfid %d nwnames %d"
-v9fs_walk_return(uint16_t tag, uint8_t id, uint16_t nwnames, void* qids) "tag %d id %d nwnames %d qids %p"
-v9fs_open(uint16_t tag, uint8_t id, int32_t fid, int32_t mode) "tag %d id %d fid %d mode %d"
-v9fs_open_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int iounit) "tag %d id %d qid={type %d version %d path %"PRId64"} iounit %d"
-v9fs_lcreate(uint16_t tag, uint8_t id, int32_t dfid, int32_t flags, int32_t mode, uint32_t gid) "tag %d id %d dfid %d flags %d mode %d gid %u"
-v9fs_lcreate_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int32_t iounit) "tag %d id %d qid={type %d version %d path %"PRId64"} iounit %d"
-v9fs_fsync(uint16_t tag, uint8_t id, int32_t fid, int datasync) "tag %d id %d fid %d datasync %d"
-v9fs_clunk(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
-v9fs_read(uint16_t tag, uint8_t id, int32_t fid, uint64_t off, uint32_t max_count) "tag %d id %d fid %d off %"PRIu64" max_count %u"
-v9fs_read_return(uint16_t tag, uint8_t id, int32_t count, ssize_t err) "tag %d id %d count %d err %zd"
-v9fs_readdir(uint16_t tag, uint8_t id, int32_t fid, uint64_t offset, uint32_t max_count) "tag %d id %d fid %d offset %"PRIu64" max_count %u"
-v9fs_readdir_return(uint16_t tag, uint8_t id, uint32_t count, ssize_t retval) "tag %d id %d count %u retval %zd"
-v9fs_write(uint16_t tag, uint8_t id, int32_t fid, uint64_t off, uint32_t count, int cnt) "tag %d id %d fid %d off %"PRIu64" count %u cnt %d"
-v9fs_write_return(uint16_t tag, uint8_t id, int32_t total, ssize_t err) "tag %d id %d total %d err %zd"
-v9fs_create(uint16_t tag, uint8_t id, int32_t fid, char* name, int32_t perm, int8_t mode) "tag %d id %d fid %d name %s perm %d mode %d"
-v9fs_create_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int iounit) "tag %d id %d qid={type %d version %d path %"PRId64"} iounit %d"
-v9fs_symlink(uint16_t tag, uint8_t id, int32_t fid,  char* name, char* symname, uint32_t gid) "tag %d id %d fid %d name %s symname %s gid %u"
-v9fs_symlink_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d qid={type %d version %d path %"PRId64"}"
-v9fs_flush(uint16_t tag, uint8_t id, int16_t flush_tag) "tag %d id %d flush_tag %d"
-v9fs_link(uint16_t tag, uint8_t id, int32_t dfid, int32_t oldfid, char* name) "tag %d id %d dfid %d oldfid %d name %s"
-v9fs_remove(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
-v9fs_wstat(uint16_t tag, uint8_t id, int32_t fid, int32_t mode, int32_t atime, int32_t mtime) "tag %u id %u fid %d stat={mode %d atime %d mtime %d}"
-v9fs_mknod(uint16_t tag, uint8_t id, int32_t fid, int mode, int major, int minor) "tag %d id %d fid %d mode %d major %d minor %d"
-v9fs_mknod_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path) "tag %d id %d qid={type %d version %d path %"PRId64"}"
-v9fs_lock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length) "tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64""
-v9fs_lock_return(uint16_t tag, uint8_t id, int8_t status) "tag %d id %d status %d"
-v9fs_getlock(uint16_t tag, uint8_t id, int32_t fid, uint8_t type, uint64_t start, uint64_t length)"tag %d id %d fid %d type %d start %"PRIu64" length %"PRIu64""
-v9fs_getlock_return(uint16_t tag, uint8_t id, uint8_t type, uint64_t start, uint64_t length, uint32_t proc_id) "tag %d id %d type %d start %"PRIu64" length %"PRIu64" proc_id %u"
-v9fs_mkdir(uint16_t tag, uint8_t id, int32_t fid, char* name, int mode, uint32_t gid) "tag %u id %u fid %d name %s mode %d gid %u"
-v9fs_mkdir_return(uint16_t tag, uint8_t id, int8_t type, int32_t version, int64_t path, int err) "tag %u id %u qid={type %d version %d path %"PRId64"} err %d"
-v9fs_xattrwalk(uint16_t tag, uint8_t id, int32_t fid, int32_t newfid, char* name) "tag %d id %d fid %d newfid %d name %s"
-v9fs_xattrwalk_return(uint16_t tag, uint8_t id, int64_t size) "tag %d id %d size %"PRId64""
-v9fs_xattrcreate(uint16_t tag, uint8_t id, int32_t fid, char* name, int64_t size, int flags) "tag %d id %d fid %d name %s size %"PRId64" flags %d"
-v9fs_readlink(uint16_t tag, uint8_t id, int32_t fid) "tag %d id %d fid %d"
-v9fs_readlink_return(uint16_t tag, uint8_t id, char* target) "tag %d id %d name %s"
-
-# target-sparc/mmu_helper.c
-mmu_helper_dfault(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl) "DFAULT at %"PRIx64" context %"PRIx64" mmu_idx=%d tl=%d"
-mmu_helper_dprot(uint64_t address, uint64_t context, int mmu_idx, uint32_t tl) "DPROT at %"PRIx64" context %"PRIx64" mmu_idx=%d tl=%d"
-mmu_helper_dmiss(uint64_t address, uint64_t context) "DMISS at %"PRIx64" context %"PRIx64""
-mmu_helper_tfault(uint64_t address, uint64_t context) "TFAULT at %"PRIx64" context %"PRIx64""
-mmu_helper_tmiss(uint64_t address, uint64_t context) "TMISS at %"PRIx64" context %"PRIx64""
-mmu_helper_get_phys_addr_code(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64""
-mmu_helper_get_phys_addr_data(uint32_t tl, int mmu_idx, uint64_t prim_context, uint64_t sec_context, uint64_t address) "tl=%d mmu_idx=%d primary context=%"PRIx64" secondary context=%"PRIx64" address=%"PRIx64""
-mmu_helper_mmu_fault(uint64_t address, uint64_t paddr, int mmu_idx, uint32_t tl, uint64_t prim_context, uint64_t sec_context) "Translate at %"PRIx64" -> %"PRIx64", mmu_idx=%d tl=%d primary context=%"PRIx64" secondary context=%"PRIx64""
-
-# target-sparc/int64_helper.c
-int_helper_set_softint(uint32_t softint) "new %08x"
-int_helper_clear_softint(uint32_t softint) "new %08x"
-int_helper_write_softint(uint32_t softint) "new %08x"
-
-# target-sparc/int32_helper.c
-int_helper_icache_freeze(void) "Instruction cache: freeze"
-int_helper_dcache_freeze(void) "Data cache: freeze"
-
-# target-sparc/win_helper.c
-win_helper_gregset_error(uint32_t pstate) "ERROR in get_gregset: active pstate bits=%x"
-win_helper_switch_pstate(uint32_t pstate_regs, uint32_t new_pstate_regs) "change_pstate: switching regs old=%x new=%x"
-win_helper_no_switch_pstate(uint32_t new_pstate_regs) "change_pstate: regs new=%x (unchanged)"
-win_helper_wrpil(uint32_t psrpil, uint32_t new_pil) "old=%x new=%x"
-win_helper_done(uint32_t tl) "tl=%d"
-win_helper_retry(uint32_t tl) "tl=%d"
-
 # dma-helpers.c
-dma_blk_io(void *dbs, void *bs, int64_t sector_num, bool to_dev) "dbs=%p bs=%p sector_num=%" PRId64 " to_dev=%d"
+dma_blk_io(void *dbs, void *bs, int64_t offset, bool to_dev) "dbs=%p bs=%p offset=%" PRId64 " to_dev=%d"
 dma_aio_cancel(void *dbs) "dbs=%p"
 dma_complete(void *dbs, int ret, void *cb) "dbs=%p ret=%d cb=%p"
 dma_blk_cb(void *dbs, int ret) "dbs=%p ret=%d"
 dma_map_wait(void *dbs) "dbs=%p"
 
-# ui/console.c
-console_gfx_new(void) ""
-console_putchar_csi(int esc_param0, int esc_param1, int ch, int nb_esc_params) "escape sequence CSI%d;%d%c, %d parameters"
-console_putchar_unhandled(int ch) "unhandled escape character '%c'"
-console_txt_new(int w, int h) "%dx%d"
-console_select(int nr) "%d"
-console_refresh(int interval) "interval %d ms"
-displaysurface_create(void *display_surface, int w, int h) "surface=%p, %dx%d"
-displaysurface_create_from(void *display_surface, int w, int h, uint32_t format) "surface=%p, %dx%d, format 0x%x"
-displaysurface_free(void *display_surface) "surface=%p"
-displaychangelistener_register(void *dcl, const char *name) "%p [ %s ]"
-displaychangelistener_unregister(void *dcl, const char *name) "%p [ %s ]"
-ppm_save(const char *filename, void *display_surface) "%s surface=%p"
-
-# ui/gtk.c
-gd_switch(const char *tab, int width, int height) "tab=%s, width=%d, height=%d"
-gd_update(const char *tab, int x, int y, int w, int h) "tab=%s, x=%d, y=%d, w=%d, h=%d"
-gd_key_event(const char *tab, int gdk_keycode, int qemu_keycode, const char *action) "tab=%s, translated GDK keycode %d to QEMU keycode %d (%s)"
-gd_grab(const char *tab, const char *device, bool on) "tab=%s, %s %d"
-
-# ui/vnc.c
-vnc_key_guest_leds(bool caps, bool num, bool scroll) "caps %d, num %d, scroll %d"
-vnc_key_map_init(const char *layout) "%s"
-vnc_key_event_ext(bool down, int sym, int keycode, const char *name) "down %d, sym 0x%x, keycode 0x%x [%s]"
-vnc_key_event_map(bool down, int sym, int keycode, const char *name) "down %d, sym 0x%x -> keycode 0x%x [%s]"
-vnc_key_sync_numlock(bool on) "%d"
-vnc_key_sync_capslock(bool on) "%d"
-
-# ui/input.c
-input_event_key_number(int conidx, int number, const char *qcode, bool down) "con %d, key number 0x%x [%s], down %d"
-input_event_key_qcode(int conidx, const char *qcode, bool down) "con %d, key qcode %s, down %d"
-input_event_btn(int conidx, const char *btn, bool down) "con %d, button %s, down %d"
-input_event_rel(int conidx, const char *axis, int value) "con %d, axis %s, value %d"
-input_event_abs(int conidx, const char *axis, int value) "con %d, axis %s, value 0x%x"
-input_event_sync(void) ""
-input_mouse_mode(int absolute) "absolute %d"
-
-# hw/display/vmware_vga.c
-vmware_value_read(uint32_t index, uint32_t value) "index %d, value 0x%x"
-vmware_value_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
-vmware_palette_read(uint32_t index, uint32_t value) "index %d, value 0x%x"
-vmware_palette_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
-vmware_scratch_read(uint32_t index, uint32_t value) "index %d, value 0x%x"
-vmware_scratch_write(uint32_t index, uint32_t value) "index %d, value 0x%x"
-vmware_setmode(uint32_t w, uint32_t h, uint32_t bpp) "%dx%d @ %d bpp"
-
-# savevm.c
-savevm_section_start(const char *id, unsigned int section_id) "%s, section_id %u"
-savevm_section_end(const char *id, unsigned int section_id) "%s, section_id %u"
-savevm_state_begin(void) ""
-savevm_state_iterate(void) ""
-savevm_state_complete(void) ""
-savevm_state_cancel(void) ""
-vmstate_save(const char *idstr, const char *vmsd_name) "%s, %s"
-vmstate_load(const char *idstr, const char *vmsd_name) "%s, %s"
-qemu_announce_self_iter(const char *mac) "%s"
-
-# vmstate.c
-vmstate_load_field_error(const char *field, int ret) "field \"%s\" load failed, ret = %d"
-
-# qemu-file.c
-qemu_file_fclose(void) ""
-
-# arch_init.c
-migration_bitmap_sync_start(void) ""
-migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64""
-migration_throttle(void) ""
-
-# hw/display/qxl.c
-disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
-disable qxl_io_write_vga(int qid, const char *mode, uint32_t addr, uint32_t val) "%d %s addr=%u val=%u"
-qxl_create_guest_primary(int qid, uint32_t width, uint32_t height, uint64_t mem, uint32_t format, uint32_t position) "%d %ux%u mem=%" PRIx64 " %u,%u"
-qxl_create_guest_primary_rest(int qid, int32_t stride, uint32_t type, uint32_t flags) "%d %d,%d,%d"
-qxl_destroy_primary(int qid) "%d"
-qxl_enter_vga_mode(int qid) "%d"
-qxl_exit_vga_mode(int qid) "%d"
-qxl_hard_reset(int qid, int64_t loadvm) "%d loadvm=%"PRId64""
-qxl_interface_async_complete_io(int qid, uint32_t current_async, void *cookie) "%d current=%d cookie=%p"
-qxl_interface_attach_worker(int qid) "%d"
-qxl_interface_get_init_info(int qid) "%d"
-qxl_interface_set_compression_level(int qid, int64_t level) "%d %"PRId64
-qxl_interface_update_area_complete(int qid, uint32_t surface_id, uint32_t dirty_left, uint32_t dirty_right, uint32_t dirty_top, uint32_t dirty_bottom) "%d surface=%d [%d,%d,%d,%d]"
-qxl_interface_update_area_complete_rest(int qid, uint32_t num_updated_rects) "%d #=%d"
-qxl_interface_update_area_complete_overflow(int qid, int max) "%d max=%d"
-qxl_interface_update_area_complete_schedule_bh(int qid, uint32_t num_dirty) "%d #dirty=%d"
-qxl_io_destroy_primary_ignored(int qid, const char *mode) "%d %s"
-qxl_io_log(int qid, const uint8_t *log_buf) "%d %s"
-qxl_io_read_unexpected(int qid) "%d"
-qxl_io_unexpected_vga_mode(int qid, uint64_t addr, uint64_t val, const char *desc) "%d 0x%"PRIx64"=%"PRIu64" (%s)"
-qxl_io_write(int qid, const char *mode, uint64_t addr, const char *aname, uint64_t val, unsigned size, int async) "%d %s addr=%"PRIu64 " (%s) val=%"PRIu64" size=%u async=%d"
-qxl_memslot_add_guest(int qid, uint32_t slot_id, uint64_t guest_start, uint64_t guest_end) "%d %u: guest phys 0x%"PRIx64 " - 0x%" PRIx64
-qxl_post_load(int qid, const char *mode) "%d %s"
-qxl_pre_load(int qid) "%d"
-qxl_pre_save(int qid) "%d"
-qxl_reset_surfaces(int qid) "%d"
-qxl_ring_command_check(int qid, const char *mode) "%d %s"
-qxl_ring_command_get(int qid, const char *mode) "%d %s"
-qxl_ring_command_req_notification(int qid) "%d"
-qxl_ring_cursor_check(int qid, const char *mode) "%d %s"
-qxl_ring_cursor_get(int qid, const char *mode) "%d %s"
-qxl_ring_cursor_req_notification(int qid) "%d"
-qxl_ring_res_push(int qid, const char *mode, uint32_t surface_count, uint32_t free_res, void *last_release, const char *notify) "%d %s s#=%d res#=%d last=%p notify=%s"
-qxl_ring_res_push_rest(int qid, uint32_t ring_has, uint32_t ring_size, uint32_t prod, uint32_t cons) "%d ring %d/%d [%d,%d]"
-qxl_ring_res_put(int qid, uint32_t free_res) "%d #res=%d"
-qxl_set_mode(int qid, int modenr, uint32_t x_res, uint32_t y_res, uint32_t bits, uint64_t devmem) "%d mode=%d [ x=%d y=%d @ bpp=%d devmem=0x%" PRIx64 " ]"
-qxl_soft_reset(int qid) "%d"
-qxl_spice_destroy_surfaces_complete(int qid) "%d"
-qxl_spice_destroy_surfaces(int qid, int async) "%d async=%d"
-qxl_spice_destroy_surface_wait_complete(int qid, uint32_t id) "%d sid=%d"
-qxl_spice_destroy_surface_wait(int qid, uint32_t id, int async) "%d sid=%d async=%d"
-qxl_spice_flush_surfaces_async(int qid, uint32_t surface_count, uint32_t num_free_res) "%d s#=%d, res#=%d"
-qxl_spice_monitors_config(int qid) "%d"
-qxl_spice_loadvm_commands(int qid, void *ext, uint32_t count) "%d ext=%p count=%d"
-qxl_spice_oom(int qid) "%d"
-qxl_spice_reset_cursor(int qid) "%d"
-qxl_spice_reset_image_cache(int qid) "%d"
-qxl_spice_reset_memslots(int qid) "%d"
-qxl_spice_update_area(int qid, uint32_t surface_id, uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "%d sid=%d [%d,%d,%d,%d]"
-qxl_spice_update_area_rest(int qid, uint32_t num_dirty_rects, uint32_t clear_dirty_region) "%d #d=%d clear=%d"
-qxl_surfaces_dirty(int qid, int surface, int offset, int size) "%d surface=%d offset=%d size=%d"
-qxl_send_events(int qid, uint32_t events) "%d %d"
-qxl_send_events_vm_stopped(int qid, uint32_t events) "%d %d"
-qxl_set_guest_bug(int qid) "%d"
-qxl_interrupt_client_monitors_config(int qid, int num_heads, void *heads) "%d %d %p"
-qxl_client_monitors_config_unsupported_by_guest(int qid, uint32_t int_mask, void *client_monitors_config) "%d %X %p"
-qxl_client_monitors_config_unsupported_by_device(int qid, int revision) "%d revision=%d"
-qxl_client_monitors_config_capped(int qid, int requested, int limit) "%d %d %d"
-qxl_client_monitors_config_crc(int qid, unsigned size, uint32_t crc32) "%d %u %u"
-qxl_set_client_capabilities_unsupported_by_revision(int qid, int revision) "%d revision=%d"
-
-# ui/spice-display.c
-qemu_spice_add_memslot(int qid, uint32_t slot_id, unsigned long virt_start, unsigned long virt_end, int async) "%d %u: host virt 0x%lx - 0x%lx async=%d"
-qemu_spice_del_memslot(int qid, uint32_t gid, uint32_t slot_id) "%d gid=%u sid=%u"
-qemu_spice_create_primary_surface(int qid, uint32_t sid, void *surface, int async) "%d sid=%u surface=%p async=%d"
-qemu_spice_destroy_primary_surface(int qid, uint32_t sid, int async) "%d sid=%u async=%d"
-qemu_spice_wakeup(uint32_t qid) "%d"
-qemu_spice_create_update(uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "lr %d -> %d,  tb -> %d -> %d"
-
-# hw/display/qxl-render.c
-qxl_render_blit(int32_t stride, int32_t left, int32_t right, int32_t top, int32_t bottom) "stride=%d [%d, %d, %d, %d]"
-qxl_render_guest_primary_resized(int32_t width, int32_t height, int32_t stride, int32_t bytes_pp, int32_t bits_pp) "%dx%d, stride %d, bpp %d, depth %d"
-qxl_render_update_area_done(void *cookie) "%p"
-
-# hw/ppc/spapr_pci.c
-spapr_pci_msi(const char *msg, uint32_t ca) "%s (cfg=%x)"
-spapr_pci_msi_setup(const char *name, unsigned vector, uint64_t addr) "dev\"%s\" vector %u, addr=%"PRIx64
-spapr_pci_rtas_ibm_change_msi(unsigned cfg, unsigned func, unsigned req, unsigned first) "cfgaddr %x func %u, requested %u, first irq %u"
-spapr_pci_rtas_ibm_query_interrupt_source_number(unsigned ioa, unsigned intr) "queries for #%u, IRQ%u"
-spapr_pci_msi_write(uint64_t addr, uint64_t data, uint32_t dt_irq) "@%"PRIx64"<=%"PRIx64" IRQ %u"
-spapr_pci_lsi_set(const char *busname, int pin, uint32_t irq) "%s PIN%d IRQ %u"
-spapr_pci_msi_retry(unsigned config_addr, unsigned req_num, unsigned max_irqs) "Guest device at %x asked %u, have only %u"
-
-# hw/intc/xics.c
-xics_icp_check_ipi(int server, uint8_t mfrr) "CPU %d can take IPI mfrr=%#x"
-xics_icp_accept(uint32_t old_xirr, uint32_t new_xirr) "icp_accept: XIRR %#"PRIx32"->%#"PRIx32
-xics_icp_eoi(int server, uint32_t xirr, uint32_t new_xirr) "icp_eoi: server %d given XIRR %#"PRIx32" new XIRR %#"PRIx32
-xics_icp_irq(int server, int nr, uint8_t priority) "cpu %d trying to deliver irq %#"PRIx32" priority %#x"
-xics_icp_raise(uint32_t xirr, uint8_t pending_priority) "raising IRQ new XIRR=%#x new pending priority=%#x"
-xics_set_irq_msi(int srcno, int nr) "set_irq_msi: srcno %d [irq %#x]"
-xics_masked_pending(void) "set_irq_msi: masked pending"
-xics_set_irq_lsi(int srcno, int nr) "set_irq_lsi: srcno %d [irq %#x]"
-xics_ics_write_xive(int nr, int srcno, int server, uint8_t priority) "ics_write_xive: irq %#x [src %d] server %#x prio %#x"
-xics_ics_reject(int nr, int srcno) "reject irq %#x [src %d]"
-xics_ics_eoi(int nr) "ics_eoi: irq %#x"
-xics_alloc(int src, int irq) "source#%d, irq %d"
-xics_alloc_failed_hint(int src, int irq) "source#%d, irq %d is already in use"
-xics_alloc_failed_no_left(int src) "source#%d, no irq left"
-xics_alloc_block(int src, int first, int num, bool lsi, int align) "source#%d, first irq %d, %d irqs, lsi=%d, alignnum %d"
-xics_ics_free(int src, int irq, int num) "Source#%d, first irq %d, %d irqs"
-xics_ics_free_warn(int src, int irq) "Source#%d, irq %d is already free"
-
-# hw/ppc/spapr.c
-spapr_cas_failed(unsigned long n) "DT diff buffer is too small: %ld bytes"
-spapr_cas_continue(unsigned long n) "Copy changes to the guest: %ld bytes"
-
-# hw/ppc/spapr_hcall.c
-spapr_cas_pvr_try(uint32_t pvr) "%x"
-spapr_cas_pvr(uint32_t cur_pvr, bool cpu_match, uint32_t new_pvr, uint64_t pcr) "current=%x, cpu_match=%u, new=%x, compat flags=%"PRIx64
-
-# hw/ppc/spapr_iommu.c
-spapr_iommu_put(uint64_t liobn, uint64_t ioba, uint64_t tce, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tce=0x%"PRIx64" ret=%"PRId64
-spapr_iommu_get(uint64_t liobn, uint64_t ioba, uint64_t ret, uint64_t tce) "liobn=%"PRIx64" ioba=0x%"PRIx64" ret=%"PRId64" tce=0x%"PRIx64
-spapr_iommu_indirect(uint64_t liobn, uint64_t ioba, uint64_t tce, uint64_t iobaN, uint64_t tceN, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tcelist=0x%"PRIx64" iobaN=0x%"PRIx64" tceN=0x%"PRIx64" ret=%"PRId64
-spapr_iommu_stuff(uint64_t liobn, uint64_t ioba, uint64_t tce_value, uint64_t npages, uint64_t ret) "liobn=%"PRIx64" ioba=0x%"PRIx64" tcevalue=0x%"PRIx64" npages=%"PRId64" ret=%"PRId64
-spapr_iommu_xlate(uint64_t liobn, uint64_t ioba, uint64_t tce, unsigned perm, unsigned pgsize) "liobn=%"PRIx64" 0x%"PRIx64" -> 0x%"PRIx64" perm=%u mask=%x"
-spapr_iommu_new_table(uint64_t liobn, void *tcet, void *table, int fd) "liobn=%"PRIx64" tcet=%p table=%p fd=%d"
-
-# hw/ppc/ppc.c
-ppc_tb_adjust(uint64_t offs1, uint64_t offs2, int64_t diff, int64_t seconds) "adjusted from 0x%"PRIx64" to 0x%"PRIx64", diff %"PRId64" (%"PRId64"s)"
-
-# util/hbitmap.c
-hbitmap_iter_skip_words(const void *hb, void *hbi, uint64_t pos, unsigned long cur) "hb %p hbi %p pos %"PRId64" cur 0x%lx"
-hbitmap_reset(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
-hbitmap_set(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
-
-# target-s390x/ioinst.c
-ioinst(const char *insn) "IOINST: %s"
-ioinst_sch_id(const char *insn, int cssid, int ssid, int schid) "IOINST: %s (%x.%x.%04x)"
-ioinst_chp_id(const char *insn, int cssid, int chpid) "IOINST: %s (%x.%02x)"
-ioinst_chsc_cmd(uint16_t cmd, uint16_t len) "IOINST: chsc command %04x, len %04x"
-
-# hw/s390x/css.c
-css_enable_facility(const char *facility) "CSS: enable %s"
-css_crw(uint8_t rsc, uint8_t erc, uint16_t rsid, const char *chained) "CSS: queueing crw: rsc=%x, erc=%x, rsid=%x %s"
-css_chpid_add(uint8_t cssid, uint8_t chpid, uint8_t type) "CSS: add chpid %x.%02x (type %02x)"
-css_new_image(uint8_t cssid, const char *default_cssid) "CSS: add css image %02x %s"
-css_assign_subch(const char *do_assign, uint8_t cssid, uint8_t ssid, uint16_t schid, uint16_t devno) "CSS: %s %x.%x.%04x (devno %04x)"
-css_io_interrupt(int cssid, int ssid, int schid, uint32_t intparm, uint8_t isc, const char *conditional) "CSS: I/O interrupt on sch %x.%x.%04x (intparm %08x, isc %x) %s"
-css_adapter_interrupt(uint8_t isc) "CSS: adapter I/O interrupt (isc %x)"
-
-# hw/s390x/virtio-ccw.c
-virtio_ccw_interpret_ccw(int cssid, int ssid, int schid, int cmd_code) "VIRTIO-CCW: %x.%x.%04x: interpret command %x"
-virtio_ccw_new_device(int cssid, int ssid, int schid, int devno, const char *devno_mode) "VIRTIO-CCW: add subchannel %x.%x.%04x, devno %04x (%s)"
-
-# hw/intc/s390_flic_kvm.c
-flic_create_device(int err) "flic: create device failed %d"
-flic_no_device_api(int err) "flic: no Device Contral API support %d"
-flic_reset_failed(int err) "flic: reset failed %d"
-
-# migration.c
-migrate_set_state(int new_state) "new state %d"
-migrate_fd_cleanup(void) ""
-migrate_fd_error(void) ""
-migrate_fd_cancel(void) ""
-migrate_pending(uint64_t size, uint64_t max) "pending size %" PRIu64 " max %" PRIu64
-migrate_transferred(uint64_t tranferred, uint64_t time_spent, double bandwidth, uint64_t size) "transferred %" PRIu64 " time_spent %" PRIu64 " bandwidth %g max_size %" PRId64
-
 # kvm-all.c
 kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"
 kvm_vm_ioctl(int type, void *arg) "type 0x%x, arg %p"
@@ -1321,75 +119,60 @@
 kvm_device_ioctl(int fd, int type, void *arg) "dev fd %d, type 0x%x, arg %p"
 kvm_failed_reg_get(uint64_t id, const char *msg) "Warning: Unable to retrieve ONEREG %" PRIu64 " from KVM: %s"
 kvm_failed_reg_set(uint64_t id, const char *msg) "Warning: Unable to set ONEREG %" PRIu64 " to KVM: %s"
-
-# target-ppc/kvm.c
-kvm_failed_spr_set(int str, const char *msg) "Warning: Unable to set SPR %d to KVM: %s"
-kvm_failed_spr_get(int str, const char *msg) "Warning: Unable to retrieve SPR %d from KVM: %s"
+kvm_irqchip_commit_routes(void) ""
+kvm_irqchip_add_msi_route(int virq) "Adding MSI route virq=%d"
+kvm_irqchip_update_msi_route(int virq) "Updating MSI route virq=%d"
 
 # TCG related tracing (mostly disabled by default)
 # cpu-exec.c
 disable exec_tb(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR
 disable exec_tb_nocache(void *tb, uintptr_t pc) "tb:%p pc=0x%"PRIxPTR
-disable exec_tb_exit(void *next_tb, unsigned int flags) "tb:%p flags=%x"
+disable exec_tb_exit(void *last_tb, unsigned int flags) "tb:%p flags=%x"
 
 # translate-all.c
 translate_block(void *tb, uintptr_t pc, uint8_t *tb_code) "tb:%p, pc:0x%"PRIxPTR", tb_code:%p"
 
 # memory.c
-memory_region_ops_read(void *mr, uint64_t addr, uint64_t value, unsigned size) "mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
-memory_region_ops_write(void *mr, uint64_t addr, uint64_t value, unsigned size) "mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_ops_read(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_ops_write(int cpu_index, void *mr, uint64_t addr, uint64_t value, unsigned size) "cpu %d mr %p addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_subpage_read(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_subpage_write(int cpu_index, void *mr, uint64_t offset, uint64_t value, unsigned size) "cpu %d mr %p offset %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_tb_read(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr %#"PRIx64" value %#"PRIx64" size %u"
+memory_region_tb_write(int cpu_index, uint64_t addr, uint64_t value, unsigned size) "cpu %d addr %#"PRIx64" value %#"PRIx64" size %u"
 
-# qom/object.c
-object_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
-object_class_dynamic_cast_assert(const char *type, const char *target, const char *file, int line, const char *func) "%s->%s (%s:%d:%s)"
+### Guest events, keep at bottom
 
-# hw/i386/xen/xen_pvdevice.c
-xen_pv_mmio_read(uint64_t addr) "WARNING: read from Xen PV Device MMIO space (address %"PRIx64")"
-xen_pv_mmio_write(uint64_t addr) "WARNING: write to Xen PV Device MMIO space (address %"PRIx64")"
+# @vaddr: Access' virtual address.
+# @info : Access' information (see below).
+#
+# Start virtual memory access (before any potential access violation).
+#
+# Does not include memory accesses performed by devices.
+#
+# Access information can be parsed as:
+#
+# struct mem_info {
+#     uint8_t size_shift : 2; /* interpreted as "1 << size_shift" bytes */
+#     bool    sign_extend: 1; /* sign-extended */
+#     uint8_t endianness : 1; /* 0: little, 1: big */
+#     bool    store      : 1; /* wheter it's a store operation */
+# };
+#
+# Targets: TCG(all)
+disable vcpu tcg guest_mem_before(TCGv vaddr, uint8_t info) "info=%d", "vaddr=0x%016"PRIx64" info=%d"
 
-# hw/pci/pci_host.c
-pci_cfg_read(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x -> 0x%x"
-pci_cfg_write(const char *dev, unsigned devid, unsigned fnid, unsigned offs, unsigned val) "%s %02u:%u @0x%x <- 0x%x"
+# @num: System call number.
+# @arg*: System call argument value.
+#
+# Start executing a guest system call in syscall emulation mode.
+#
+# Targets: TCG(all)
+disable vcpu guest_user_syscall(uint64_t num, uint64_t arg1, uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5, uint64_t arg6, uint64_t arg7, uint64_t arg8) "num=0x%016"PRIx64" arg1=0x%016"PRIx64" arg2=0x%016"PRIx64" arg3=0x%016"PRIx64" arg4=0x%016"PRIx64" arg5=0x%016"PRIx64" arg6=0x%016"PRIx64" arg7=0x%016"PRIx64" arg8=0x%016"PRIx64
 
-# hw/acpi/memory_hotplug.c
-mhp_acpi_invalid_slot_selected(uint32_t slot) "0x%"PRIx32
-mhp_acpi_read_addr_lo(uint32_t slot, uint32_t addr) "slot[0x%"PRIx32"] addr lo: 0x%"PRIx32
-mhp_acpi_read_addr_hi(uint32_t slot, uint32_t addr) "slot[0x%"PRIx32"] addr hi: 0x%"PRIx32
-mhp_acpi_read_size_lo(uint32_t slot, uint32_t size) "slot[0x%"PRIx32"] size lo: 0x%"PRIx32
-mhp_acpi_read_size_hi(uint32_t slot, uint32_t size) "slot[0x%"PRIx32"] size hi: 0x%"PRIx32
-mhp_acpi_read_pxm(uint32_t slot, uint32_t pxm) "slot[0x%"PRIx32"] proximity: 0x%"PRIx32
-mhp_acpi_read_flags(uint32_t slot, uint32_t flags) "slot[0x%"PRIx32"] flags: 0x%"PRIx32
-mhp_acpi_write_slot(uint32_t slot) "set active slot: 0x%"PRIx32
-mhp_acpi_write_ost_ev(uint32_t slot, uint32_t ev) "slot[0x%"PRIx32"] OST EVENT: 0x%"PRIx32
-mhp_acpi_write_ost_status(uint32_t slot, uint32_t st) "slot[0x%"PRIx32"] OST STATUS: 0x%"PRIx32
-mhp_acpi_clear_insert_evt(uint32_t slot) "slot[0x%"PRIx32"] clear insert event"
-
-# hw/i386/pc.c
-mhp_pc_dimm_assigned_slot(int slot) "0x%d"
-mhp_pc_dimm_assigned_address(uint64_t addr) "0x%"PRIx64
-
-# target-s390x/kvm.c
-kvm_enable_cmma(int rc) "CMMA: enabling with result code %d"
-kvm_clear_cmma(int rc) "CMMA: clearing with result code %d"
-kvm_failed_cpu_state_set(int cpu_index, uint8_t state, const char *msg) "Warning: Unable to set cpu %d state %" PRIu8 " to KVM: %s"
-
-# hw/dma/i8257.c
-i8257_unregistered_dma(int nchan, int dma_pos, int dma_len) "unregistered DMA channel used nchan=%d dma_pos=%d dma_len=%d"
-
-# target-s390x/cpu.c
-cpu_set_state(int cpu_index, uint8_t state) "setting cpu %d state to %" PRIu8
-cpu_halt(int cpu_index) "halting cpu %d"
-cpu_unhalt(int cpu_index) "unhalting cpu %d"
-
-# hw/display/goldfish_fb.c
-goldfish_fb_memory_read(uint32_t addr, uint32_t value) "addr %08x value %08x"
-goldfish_fb_memory_write(uint32_t addr, uint32_t value) "addr %08x value %08x"
-goldfish_fb_update_display(int y, int h, int x, int w) "y:%d,h:%d,x=%d,w=%d"
-goldfish_fb_update_stats(float peak, float total) "peak %.2f %%  total %.2f %%"
-
-# hw/audio/goldfish_audio.c
-goldfish_audio_memory_read(const char *regname, uint32_t value) "%s returns %d"
-goldfish_audio_memory_write(const char *regname, uint32_t value) "%s %08x"
-goldfish_audio_buff_recv(int size, int read) "AUD_read (%d) returned %d"
-goldfish_audio_buff_send(int size, int buffer) "sent %5d bytes to audio output (buffer %d)"
-goldfish_audio_buff_full(int available) "AUDIO_INT_READ_BUFFER_FULL available=%d"
+# @num: System call number.
+# @ret: System call result value.
+#
+# Finish executing a guest system call in syscall emulation mode.
+#
+# Targets: TCG(all)
+disable vcpu guest_user_syscall_ret(uint64_t num, uint64_t ret) "num=0x%016"PRIx64" ret=0x%016"PRIx64
diff --git a/trace/Makefile.objs b/trace/Makefile.objs
index 32f7a32..4d91b3b 100644
--- a/trace/Makefile.objs
+++ b/trace/Makefile.objs
@@ -1,47 +1,60 @@
 # -*- mode: makefile -*-
 
 ######################################################################
+# tracetool source files
+# Every rule that invokes tracetool must depend on this so code is regenerated
+# if tracetool itself changes.
+
+tracetool-y = $(SRC_PATH)/scripts/tracetool.py
+tracetool-y += $(shell find $(SRC_PATH)/scripts/tracetool -name "*.py")
+
+$(BUILD_DIR)/trace-events-all: $(trace-events-y:%=$(SRC_PATH)/%)
+	$(call quiet-command,cat $^ > $@)
+
+######################################################################
 # Auto-generated event descriptions for LTTng ust code
 
 ifeq ($(findstring ust,$(TRACE_BACKENDS)),ust)
+
 $(obj)/generated-ust-provider.h: $(obj)/generated-ust-provider.h-timestamp
-$(obj)/generated-ust-provider.h-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-ust-provider.h-timestamp: $(BUILD_DIR)/trace-events-all $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=ust-events-h \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-ust.c: $(obj)/generated-ust.c-timestamp $(BUILD_DIR)/config-host.mak
-$(obj)/generated-ust.c-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-ust.c-timestamp: $(BUILD_DIR)/trace-events-all $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=ust-events-c \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-events.h: $(obj)/generated-ust-provider.h
 $(obj)/generated-events.c: $(obj)/generated-ust.c
+
 endif
 
 ######################################################################
 # Auto-generated event descriptions
 
 $(obj)/generated-events.h: $(obj)/generated-events.h-timestamp
-$(obj)/generated-events.h-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-events.h-timestamp: $(BUILD_DIR)/trace-events-all $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=events-h \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-events.c: $(obj)/generated-events.c-timestamp $(BUILD_DIR)/config-host.mak
-$(obj)/generated-events.c-timestamp: $(SRC_PATH)/trace-events
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-events.c-timestamp: $(BUILD_DIR)/trace-events-all $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=events-c \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 util-obj-y += generated-events.o
 
@@ -54,7 +67,7 @@
 
 $(obj)/generated-tracers.h: $(obj)/generated-tracers.h-timestamp
 	@cmp -s $< $@ || cp $< $@
-$(obj)/generated-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+$(obj)/generated-tracers.h-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=h \
 		--backends=$(TRACE_BACKENDS) \
@@ -65,7 +78,7 @@
 
 $(obj)/generated-tracers.c: $(obj)/generated-tracers.c-timestamp
 	@cmp -s $< $@ || cp $< $@
-$(obj)/generated-tracers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+$(obj)/generated-tracers.c-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=c \
 		--backends=$(TRACE_BACKENDS) \
@@ -80,13 +93,14 @@
 # but that gets picked up by QEMU's Makefile as an external dependency
 # rule file. So we use '.dtrace' instead
 ifeq ($(findstring dtrace,$(TRACE_BACKENDS)),dtrace)
+
 $(obj)/generated-tracers-dtrace.dtrace: $(obj)/generated-tracers-dtrace.dtrace-timestamp
-$(obj)/generated-tracers-dtrace.dtrace-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-tracers-dtrace.dtrace-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=d \
 		--backends=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-tracers-dtrace.h: $(obj)/generated-tracers-dtrace.dtrace
 	$(call quiet-command,dtrace -o $@ -h -s $<, "  GEN   $@")
@@ -100,28 +114,28 @@
 # Translation level
 
 $(obj)/generated-helpers-wrappers.h: $(obj)/generated-helpers-wrappers.h-timestamp
-$(obj)/generated-helpers-wrappers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-helpers-wrappers.h-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-helper-wrapper-h \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-helpers.h: $(obj)/generated-helpers.h-timestamp
-$(obj)/generated-helpers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-helpers.h-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-helper-h \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-helpers.c: $(obj)/generated-helpers.c-timestamp
-$(obj)/generated-helpers.c-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-helpers.c-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-helper-c \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 $(obj)/generated-helpers.o: $(obj)/generated-helpers.c
 
@@ -129,12 +143,12 @@
 
 
 $(obj)/generated-tcg-tracers.h: $(obj)/generated-tcg-tracers.h-timestamp
-$(obj)/generated-tcg-tracers.h-timestamp: $(SRC_PATH)/trace-events $(BUILD_DIR)/config-host.mak
+	@cmp $< $@ >/dev/null 2>&1 || cp $< $@
+$(obj)/generated-tcg-tracers.h-timestamp: $(BUILD_DIR)/trace-events-all $(BUILD_DIR)/config-host.mak $(tracetool-y)
 	$(call quiet-command,$(TRACETOOL) \
 		--format=tcg-h \
 		--backend=$(TRACE_BACKENDS) \
 		< $< > $@,"  GEN   $(patsubst %-timestamp,%,$@)")
-	@cmp -s $@ $(patsubst %-timestamp,%,$@) || cp $@ $(patsubst %-timestamp,%,$@)
 
 
 ######################################################################
@@ -144,4 +158,5 @@
 util-obj-$(CONFIG_TRACE_FTRACE) += ftrace.o
 util-obj-$(CONFIG_TRACE_UST) += generated-ust.o
 util-obj-y += control.o
+target-obj-y += control-target.o
 util-obj-y += qmp.o
diff --git a/trace/control-internal.h b/trace/control-internal.h
index 5a8df28..a4e5f4a 100644
--- a/trace/control-internal.h
+++ b/trace/control-internal.h
@@ -1,7 +1,7 @@
 /*
  * Interface for configuring and controlling the state of tracing events.
  *
- * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
+ * Copyright (C) 2011-2016 Lluís Vilanova <vilanova@ac.upc.edu>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -10,10 +10,14 @@
 #ifndef TRACE__CONTROL_INTERNAL_H
 #define TRACE__CONTROL_INTERNAL_H
 
-#include <string.h>
+#include <stddef.h>                     /* size_t */
+
+#include "qom/cpu.h"
 
 
 extern TraceEvent trace_events[];
+extern uint16_t trace_events_dstate[];
+extern int trace_events_enabled_count;
 
 
 static inline TraceEventID trace_event_count(void)
@@ -39,6 +43,16 @@
     return ev->id;
 }
 
+static inline TraceEventVCPUID trace_event_get_vcpu_id(TraceEvent *ev)
+{
+    return ev->vcpu_id;
+}
+
+static inline bool trace_event_is_vcpu(TraceEvent *ev)
+{
+    return ev->vcpu_id != TRACE_VCPU_EVENT_COUNT;
+}
+
 static inline const char * trace_event_get_name(TraceEvent *ev)
 {
     assert(ev != NULL);
@@ -51,17 +65,38 @@
     return ev->sstate;
 }
 
+static inline bool trace_event_get_state_dynamic_by_id(TraceEventID id)
+{
+    /* it's on fast path, avoid consistency checks (asserts) */
+    return unlikely(trace_events_enabled_count) && trace_events_dstate[id];
+}
+
 static inline bool trace_event_get_state_dynamic(TraceEvent *ev)
 {
-    assert(ev != NULL);
-    return ev->dstate;
-}
-
-static inline void trace_event_set_state_dynamic(TraceEvent *ev, bool state)
-{
-    assert(ev != NULL);
+    TraceEventID id;
     assert(trace_event_get_state_static(ev));
-    ev->dstate = state;
+    id = trace_event_get_id(ev);
+    return trace_event_get_state_dynamic_by_id(id);
 }
 
-#endif  /* TRACE__CONTROL_INTERNAL_H */
+static inline bool trace_event_get_vcpu_state_dynamic_by_vcpu_id(CPUState *vcpu,
+                                                                 TraceEventVCPUID id)
+{
+    /* it's on fast path, avoid consistency checks (asserts) */
+    if (unlikely(trace_events_enabled_count)) {
+        return test_bit(id, vcpu->trace_dstate);
+    } else {
+        return false;
+    }
+}
+
+static inline bool trace_event_get_vcpu_state_dynamic(CPUState *vcpu,
+                                                      TraceEvent *ev)
+{
+    TraceEventVCPUID id;
+    assert(trace_event_is_vcpu(ev));
+    id = trace_event_get_vcpu_id(ev);
+    return trace_event_get_vcpu_state_dynamic_by_vcpu_id(vcpu, id);
+}
+
+#endif /* TRACE__CONTROL_INTERNAL_H */
diff --git a/trace/control-target.c b/trace/control-target.c
new file mode 100644
index 0000000..74c029a
--- /dev/null
+++ b/trace/control-target.c
@@ -0,0 +1,53 @@
+/*
+ * Interface for configuring and controlling the state of tracing events.
+ *
+ * Copyright (C) 2014-2016 Lluís Vilanova <vilanova@ac.upc.edu>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "trace/control.h"
+#include "translate-all.h"
+
+
+void trace_event_set_state_dynamic(TraceEvent *ev, bool state)
+{
+    CPUState *vcpu;
+    assert(trace_event_get_state_static(ev));
+    if (trace_event_is_vcpu(ev)) {
+        CPU_FOREACH(vcpu) {
+            trace_event_set_vcpu_state_dynamic(vcpu, ev, state);
+        }
+    } else {
+        TraceEventID id = trace_event_get_id(ev);
+        trace_events_enabled_count += state - trace_events_dstate[id];
+        trace_events_dstate[id] = state;
+    }
+}
+
+void trace_event_set_vcpu_state_dynamic(CPUState *vcpu,
+                                        TraceEvent *ev, bool state)
+{
+    TraceEventID id;
+    TraceEventVCPUID vcpu_id;
+    bool state_pre;
+    assert(trace_event_get_state_static(ev));
+    assert(trace_event_is_vcpu(ev));
+    id = trace_event_get_id(ev);
+    vcpu_id = trace_event_get_vcpu_id(ev);
+    state_pre = test_bit(vcpu_id, vcpu->trace_dstate);
+    if (state_pre != state) {
+        if (state) {
+            trace_events_enabled_count++;
+            set_bit(vcpu_id, vcpu->trace_dstate);
+            trace_events_dstate[id]++;
+        } else {
+            trace_events_enabled_count--;
+            clear_bit(vcpu_id, vcpu->trace_dstate);
+            trace_events_dstate[id]--;
+        }
+    }
+}
diff --git a/trace/control.c b/trace/control.c
index 0d30801..d173c09 100644
--- a/trace/control.c
+++ b/trace/control.c
@@ -1,20 +1,59 @@
 /*
  * Interface for configuring and controlling the state of tracing events.
  *
- * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
+ * Copyright (C) 2011-2016 Lluís Vilanova <vilanova@ac.upc.edu>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "trace/control.h"
+#include "qemu/help_option.h"
 #ifdef CONFIG_TRACE_SIMPLE
 #include "trace/simple.h"
 #endif
 #ifdef CONFIG_TRACE_FTRACE
 #include "trace/ftrace.h"
 #endif
+#ifdef CONFIG_TRACE_LOG
+#include "qemu/log.h"
+#endif
+#include "qapi/error.h"
 #include "qemu/error-report.h"
+#include "qemu/config-file.h"
+#include "monitor/monitor.h"
+
+int trace_events_enabled_count;
+/*
+ * Interpretation depends on wether the event has the 'vcpu' property:
+ * - false: Boolean value indicating whether the event is active.
+ * - true : Integral counting the number of vCPUs that have this event enabled.
+ */
+uint16_t trace_events_dstate[TRACE_EVENT_COUNT];
+/* Marks events for late vCPU state init */
+static bool trace_events_dstate_init[TRACE_EVENT_COUNT];
+
+QemuOptsList qemu_trace_opts = {
+    .name = "trace",
+    .implied_opt_name = "enable",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
+    .desc = {
+        {
+            .name = "enable",
+            .type = QEMU_OPT_STRING,
+        },
+        {
+            .name = "events",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "file",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
 
 TraceEvent *trace_event_name(const char *name)
 {
@@ -85,6 +124,59 @@
     return NULL;
 }
 
+void trace_list_events(void)
+{
+    int i;
+    for (i = 0; i < trace_event_count(); i++) {
+        TraceEvent *res = trace_event_id(i);
+        fprintf(stderr, "%s\n", trace_event_get_name(res));
+    }
+}
+
+static void do_trace_enable_events(const char *line_buf)
+{
+    const bool enable = ('-' != line_buf[0]);
+    const char *line_ptr = enable ? line_buf : line_buf + 1;
+
+    if (trace_event_is_pattern(line_ptr)) {
+        TraceEvent *ev = NULL;
+        while ((ev = trace_event_pattern(line_ptr, ev)) != NULL) {
+            if (trace_event_get_state_static(ev)) {
+                /* start tracing */
+                trace_event_set_state_dynamic(ev, enable);
+                /* mark for late vCPU init */
+                trace_events_dstate_init[ev->id] = true;
+            }
+        }
+    } else {
+        TraceEvent *ev = trace_event_name(line_ptr);
+        if (ev == NULL) {
+            error_report("WARNING: trace event '%s' does not exist",
+                         line_ptr);
+        } else if (!trace_event_get_state_static(ev)) {
+            error_report("WARNING: trace event '%s' is not traceable",
+                         line_ptr);
+        } else {
+            /* start tracing */
+            trace_event_set_state_dynamic(ev, enable);
+            /* mark for late vCPU init */
+            trace_events_dstate_init[ev->id] = true;
+        }
+    }
+}
+
+void trace_enable_events(const char *line_buf)
+{
+    if (is_help_option(line_buf)) {
+        trace_list_events();
+        if (cur_mon == NULL) {
+            exit(0);
+        }
+    } else {
+        do_trace_enable_events(line_buf);
+    }
+}
+
 static void trace_init_events(const char *fname)
 {
     Location loc;
@@ -111,27 +203,7 @@
             if ('#' == line_buf[0]) { /* skip commented lines */
                 continue;
             }
-            const bool enable = ('-' != line_buf[0]);
-            char *line_ptr = enable ? line_buf : line_buf + 1;
-            if (trace_event_is_pattern(line_ptr)) {
-                TraceEvent *ev = NULL;
-                while ((ev = trace_event_pattern(line_ptr, ev)) != NULL) {
-                    if (trace_event_get_state_static(ev)) {
-                        trace_event_set_state_dynamic(ev, enable);
-                    }
-                }
-            } else {
-                TraceEvent *ev = trace_event_name(line_ptr);
-                if (ev == NULL) {
-                    error_report("WARNING: trace event '%s' does not exist",
-                                 line_ptr);
-                } else if (!trace_event_get_state_static(ev)) {
-                    error_report("WARNING: trace event '%s' is not traceable\n",
-                                 line_ptr);
-                } else {
-                    trace_event_set_state_dynamic(ev, enable);
-                }
-            }
+            trace_enable_events(line_buf);
         }
     }
     if (fclose(fp) != 0) {
@@ -142,17 +214,31 @@
     loc_pop(&loc);
 }
 
-bool trace_init_backends(const char *events, const char *file)
+void trace_init_file(const char *file)
 {
 #ifdef CONFIG_TRACE_SIMPLE
-    if (!st_init(file)) {
-        fprintf(stderr, "failed to initialize simple tracing backend.\n");
-        return false;
+    st_set_trace_file(file);
+#elif defined CONFIG_TRACE_LOG
+    /* If both the simple and the log backends are enabled, "-trace file"
+     * only applies to the simple backend; use "-D" for the log backend.
+     */
+    if (file) {
+        qemu_set_log_filename(file, &error_fatal);
     }
 #else
     if (file) {
         fprintf(stderr, "error: -trace file=...: "
                 "option not supported by the selected tracing backends\n");
+        exit(1);
+    }
+#endif
+}
+
+bool trace_init_backends(void)
+{
+#ifdef CONFIG_TRACE_SIMPLE
+    if (!st_init()) {
+        fprintf(stderr, "failed to initialize simple tracing backend.\n");
         return false;
     }
 #endif
@@ -164,6 +250,35 @@
     }
 #endif
 
-    trace_init_events(events);
     return true;
 }
+
+char *trace_opt_parse(const char *optarg)
+{
+    char *trace_file;
+    QemuOpts *opts = qemu_opts_parse_noisily(qemu_find_opts("trace"),
+                                             optarg, true);
+    if (!opts) {
+        exit(1);
+    }
+    if (qemu_opt_get(opts, "enable")) {
+        trace_enable_events(qemu_opt_get(opts, "enable"));
+    }
+    trace_init_events(qemu_opt_get(opts, "events"));
+    trace_file = g_strdup(qemu_opt_get(opts, "file"));
+    qemu_opts_del(opts);
+
+    return trace_file;
+}
+
+void trace_init_vcpu_events(void)
+{
+    TraceEvent *ev = NULL;
+    while ((ev = trace_event_pattern("*", ev)) != NULL) {
+        if (trace_event_is_vcpu(ev) &&
+            trace_event_get_state_static(ev) &&
+            trace_events_dstate_init[ev->id]) {
+            trace_event_set_state_dynamic(ev, true);
+        }
+    }
+}
diff --git a/trace/control.h b/trace/control.h
index da9bb6b..0413b28 100644
--- a/trace/control.h
+++ b/trace/control.h
@@ -1,7 +1,7 @@
 /*
  * Interface for configuring and controlling the state of tracing events.
  *
- * Copyright (C) 2011-2014 Lluís Vilanova <vilanova@ac.upc.edu>
+ * Copyright (C) 2011-2016 Lluís Vilanova <vilanova@ac.upc.edu>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -86,6 +86,23 @@
 static TraceEventID trace_event_get_id(TraceEvent *ev);
 
 /**
+ * trace_event_get_vcpu_id:
+ *
+ * Get the per-vCPU identifier of an event.
+ *
+ * Special value #TRACE_VCPU_EVENT_COUNT means the event is not vCPU-specific
+ * (does not have the "vcpu" property).
+ */
+static TraceEventVCPUID trace_event_get_vcpu_id(TraceEvent *ev);
+
+/**
+ * trace_event_is_vcpu:
+ *
+ * Whether this is a per-vCPU event.
+ */
+static bool trace_event_is_vcpu(TraceEvent *ev);
+
+/**
  * trace_event_get_name:
  *
  * Get the name of an event.
@@ -104,7 +121,24 @@
  * As a down side, you must always use an immediate #TraceEventID value.
  */
 #define trace_event_get_state(id)                       \
-    ((id ##_ENABLED) && trace_event_get_state_dynamic(trace_event_id(id)))
+    ((id ##_ENABLED) && trace_event_get_state_dynamic_by_id(id))
+
+/**
+ * trace_event_get_vcpu_state:
+ * @vcpu: Target vCPU.
+ * @id: Event identifier (TraceEventID).
+ * @vcpu_id: Per-vCPU event identifier (TraceEventVCPUID).
+ *
+ * Get the tracing state of an event (both static and dynamic) for the given
+ * vCPU.
+ *
+ * If the event has the disabled property, the check will have no performance
+ * impact.
+ *
+ * As a down side, you must always use an immediate #TraceEventID value.
+ */
+#define trace_event_get_vcpu_state(vcpu, id, vcpu_id)                   \
+    ((id ##_ENABLED) && trace_event_get_vcpu_state_dynamic_by_vcpu_id(vcpu, vcpu_id))
 
 /**
  * trace_event_get_state_static:
@@ -121,10 +155,19 @@
  * trace_event_get_state_dynamic:
  *
  * Get the dynamic tracing state of an event.
+ *
+ * If the event has the 'vcpu' property, gets the OR'ed state of all vCPUs.
  */
 static bool trace_event_get_state_dynamic(TraceEvent *ev);
 
 /**
+ * trace_event_get_vcpu_state_dynamic:
+ *
+ * Get the dynamic tracing state of an event for the given vCPU.
+ */
+static bool trace_event_get_vcpu_state_dynamic(CPUState *vcpu, TraceEvent *ev);
+
+/**
  * trace_event_set_state:
  *
  * Set the tracing state of an event (only if possible).
@@ -138,20 +181,43 @@
     } while (0)
 
 /**
+ * trace_event_set_vcpu_state:
+ *
+ * Set the tracing state of an event for the given vCPU (only if not disabled).
+ */
+#define trace_event_set_vcpu_state(vcpu, id, state)                     \
+    do {                                                                \
+        if ((id ##_ENABLED)) {                                          \
+            TraceEvent *_e = trace_event_id(id);                        \
+            trace_event_set_vcpu_state_dynamic(vcpu, _e, state);        \
+        }                                                               \
+    } while (0)
+
+/**
  * trace_event_set_state_dynamic:
  *
  * Set the dynamic tracing state of an event.
  *
+ * If the event has the 'vcpu' property, sets the state on all vCPUs.
+ *
  * Pre-condition: trace_event_get_state_static(ev) == true
  */
-static void trace_event_set_state_dynamic(TraceEvent *ev, bool state);
+void trace_event_set_state_dynamic(TraceEvent *ev, bool state);
+
+/**
+ * trace_event_set_vcpu_state_dynamic:
+ *
+ * Set the dynamic tracing state of an event for the given vCPU.
+ *
+ * Pre-condition: trace_event_get_vcpu_state_static(ev) == true
+ */
+void trace_event_set_vcpu_state_dynamic(CPUState *vcpu,
+                                        TraceEvent *ev, bool state);
 
 
 
 /**
  * trace_init_backends:
- * @events: Name of file with events to be enabled at startup; may be NULL.
- *          Corresponds to commandline option "-trace events=...".
  * @file:   Name of trace output file; may be NULL.
  *          Corresponds to commandline option "-trace file=...".
  *
@@ -159,9 +225,59 @@
  *
  * Returns: Whether the backends could be successfully initialized.
  */
-bool trace_init_backends(const char *events, const char *file);
+bool trace_init_backends(void);
+
+/**
+ * trace_init_file:
+ * @file:   Name of trace output file; may be NULL.
+ *          Corresponds to commandline option "-trace file=...".
+ *
+ * Record the name of the output file for the tracing backend.
+ * Exits if no selected backend does not support specifying the
+ * output file, and a non-NULL file was passed.
+ */
+void trace_init_file(const char *file);
+
+/**
+ * trace_list_events:
+ *
+ * List all available events.
+ */
+void trace_list_events(void);
+
+/**
+ * trace_enable_events:
+ * @line_buf: A string with a glob pattern of events to be enabled or,
+ *            if the string starts with '-', disabled.
+ *
+ * Enable or disable matching events.
+ */
+void trace_enable_events(const char *line_buf);
+
+/**
+ * Definition of QEMU options describing trace subsystem configuration
+ */
+extern QemuOptsList qemu_trace_opts;
+
+/**
+ * trace_opt_parse:
+ * @optarg: A string argument of --trace command line argument
+ *
+ * Initialize tracing subsystem.
+ *
+ * Returns the filename to save trace to.  It must be freed with g_free().
+ */
+char *trace_opt_parse(const char *optarg);
+
+/**
+ * trace_init_vcpu_events:
+ *
+ * Re-synchronize initial event state with vCPUs (which can be created after
+ * trace_init_events()).
+ */
+void trace_init_vcpu_events(void);
 
 
 #include "trace/control-internal.h"
 
-#endif  /* TRACE__CONTROL_H */
+#endif /* TRACE__CONTROL_H */
diff --git a/trace/event-internal.h b/trace/event-internal.h
index b2310d9..5d8fa97 100644
--- a/trace/event-internal.h
+++ b/trace/event-internal.h
@@ -1,7 +1,7 @@
 /*
  * Interface for configuring and controlling the state of tracing events.
  *
- * Copyright (C) 2012 Lluís Vilanova <vilanova@ac.upc.edu>
+ * Copyright (C) 2012-2016 Lluís Vilanova <vilanova@ac.upc.edu>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
@@ -16,18 +16,18 @@
 /**
  * TraceEvent:
  * @id: Unique event identifier.
+ * @vcpu_id: Unique per-vCPU event identifier.
  * @name: Event name.
  * @sstate: Static tracing state.
- * @dstate: Dynamic tracing state.
  *
  * Opaque generic description of a tracing event.
  */
 typedef struct TraceEvent {
     TraceEventID id;
+    TraceEventVCPUID vcpu_id;
     const char * name;
     const bool sstate;
-    bool dstate;
 } TraceEvent;
 
 
-#endif  /* TRACE__EVENT_INTERNAL_H */
+#endif /* TRACE__EVENT_INTERNAL_H */
diff --git a/trace/ftrace.c b/trace/ftrace.c
index a7ae371..e953922 100644
--- a/trace/ftrace.c
+++ b/trace/ftrace.c
@@ -9,10 +9,7 @@
  *
  */
 
-#include <stdio.h>
-#include <string.h>
-#include <fcntl.h>
-#include <limits.h>
+#include "qemu/osdep.h"
 #include "trace.h"
 #include "trace/control.h"
 
diff --git a/trace/ftrace.h b/trace/ftrace.h
index 863e052..cb5e35d 100644
--- a/trace/ftrace.h
+++ b/trace/ftrace.h
@@ -1,9 +1,6 @@
 #ifndef TRACE_FTRACE_H
 #define TRACE_FTRACE_H
 
-#include <stdbool.h>
-
-
 #define MAX_TRACE_STRLEN 512
 #define _STR(x) #x
 #define STR(x) _STR(x)
@@ -12,4 +9,4 @@
 
 bool ftrace_init(void);
 
-#endif /* ! TRACE_FTRACE_H */
+#endif /* TRACE_FTRACE_H */
diff --git a/trace/mem-internal.h b/trace/mem-internal.h
new file mode 100644
index 0000000..ddda934
--- /dev/null
+++ b/trace/mem-internal.h
@@ -0,0 +1,46 @@
+/*
+ * Helper functions for guest memory tracing
+ *
+ * Copyright (C) 2016 Lluís Vilanova <vilanova@ac.upc.edu>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef TRACE__MEM_INTERNAL_H
+#define TRACE__MEM_INTERNAL_H
+
+static inline uint8_t trace_mem_get_info(TCGMemOp op, bool store)
+{
+    uint8_t res = op;
+    bool be = (op & MO_BSWAP) == MO_BE;
+
+    /* remove untraced fields */
+    res &= (1ULL << 4) - 1;
+    /* make endianness absolute */
+    res &= ~MO_BSWAP;
+    if (be) {
+        res |= 1ULL << 3;
+    }
+    /* add fields */
+    if (store) {
+        res |= 1ULL << 4;
+    }
+
+    return res;
+}
+
+static inline uint8_t trace_mem_build_info(
+    TCGMemOp size, bool sign_extend, TCGMemOp endianness, bool store)
+{
+    uint8_t res = 0;
+    res |= size;
+    res |= (sign_extend << 2);
+    if (endianness == MO_BE) {
+        res |= (1ULL << 3);
+    }
+    res |= (store << 4);
+    return res;
+}
+
+#endif /* TRACE__MEM_INTERNAL_H */
diff --git a/trace/mem.h b/trace/mem.h
new file mode 100644
index 0000000..9c88bcb
--- /dev/null
+++ b/trace/mem.h
@@ -0,0 +1,34 @@
+/*
+ * Helper functions for guest memory tracing
+ *
+ * Copyright (C) 2016 Lluís Vilanova <vilanova@ac.upc.edu>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef TRACE__MEM_H
+#define TRACE__MEM_H
+
+#include "tcg/tcg.h"
+
+
+/**
+ * trace_mem_get_info:
+ *
+ * Return a value for the 'info' argument in guest memory access traces.
+ */
+static uint8_t trace_mem_get_info(TCGMemOp op, bool store);
+
+/**
+ * trace_mem_build_info:
+ *
+ * Return a value for the 'info' argument in guest memory access traces.
+ */
+static uint8_t trace_mem_build_info(TCGMemOp size, bool sign_extend,
+                                    TCGMemOp endianness, bool store);
+
+
+#include "trace/mem-internal.h"
+
+#endif /* TRACE__MEM_H */
diff --git a/trace/qmp.c b/trace/qmp.c
index 0b19489..11d2564 100644
--- a/trace/qmp.c
+++ b/trace/qmp.c
@@ -1,74 +1,164 @@
 /*
  * QMP commands for tracing events.
  *
- * Copyright (C) 2014 Lluís Vilanova <vilanova@ac.upc.edu>
+ * Copyright (C) 2014-2016 Lluís Vilanova <vilanova@ac.upc.edu>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  * See the COPYING file in the top-level directory.
  */
 
-#include "qemu/typedefs.h"
+#include "qemu/osdep.h"
 #include "qmp-commands.h"
 #include "trace/control.h"
 
 
-TraceEventInfoList *qmp_trace_event_get_state(const char *name, Error **errp)
+static CPUState *get_cpu(bool has_vcpu, int vcpu, Error **errp)
 {
-    TraceEventInfoList *events = NULL;
-    bool found = false;
-    TraceEvent *ev;
+    if (has_vcpu) {
+        CPUState *cpu = qemu_get_cpu(vcpu);
+        if (cpu == NULL) {
+            error_setg(errp, "invalid vCPU index %u", vcpu);
+        }
+        return cpu;
+    } else {
+        return NULL;
+    }
+}
 
+static bool check_events(bool has_vcpu, bool ignore_unavailable, bool is_pattern,
+                         const char *name, Error **errp)
+{
+    if (!is_pattern) {
+        TraceEvent *ev = trace_event_name(name);
+
+        /* error for non-existing event */
+        if (ev == NULL) {
+            error_setg(errp, "unknown event \"%s\"", name);
+            return false;
+        }
+
+        /* error for non-vcpu event */
+        if (has_vcpu && !trace_event_is_vcpu(ev)) {
+            error_setg(errp, "event \"%s\" is not vCPU-specific", name);
+            return false;
+        }
+
+        /* error for unavailable event */
+        if (!ignore_unavailable && !trace_event_get_state_static(ev)) {
+            error_setg(errp, "event \"%s\" is disabled", name);
+            return false;
+        }
+
+        return true;
+    } else {
+        /* error for unavailable events */
+        TraceEvent *ev = NULL;
+        while ((ev = trace_event_pattern(name, ev)) != NULL) {
+            if (!ignore_unavailable && !trace_event_get_state_static(ev)) {
+                error_setg(errp, "event \"%s\" is disabled", trace_event_get_name(ev));
+                return false;
+            }
+        }
+        return true;
+    }
+}
+
+TraceEventInfoList *qmp_trace_event_get_state(const char *name,
+                                              bool has_vcpu, int64_t vcpu,
+                                              Error **errp)
+{
+    Error *err = NULL;
+    TraceEventInfoList *events = NULL;
+    TraceEvent *ev;
+    bool is_pattern = trace_event_is_pattern(name);
+    CPUState *cpu;
+
+    /* Check provided vcpu */
+    cpu = get_cpu(has_vcpu, vcpu, &err);
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
+    /* Check events */
+    if (!check_events(has_vcpu, true, is_pattern, name, errp)) {
+        return NULL;
+    }
+
+    /* Get states (all errors checked above) */
     ev = NULL;
     while ((ev = trace_event_pattern(name, ev)) != NULL) {
-        TraceEventInfoList *elem = g_new(TraceEventInfoList, 1);
+        TraceEventInfoList *elem;
+        bool is_vcpu = trace_event_is_vcpu(ev);
+        if (has_vcpu && !is_vcpu) {
+            continue;
+        }
+
+        elem = g_new(TraceEventInfoList, 1);
         elem->value = g_new(TraceEventInfo, 1);
+        elem->value->vcpu = is_vcpu;
         elem->value->name = g_strdup(trace_event_get_name(ev));
+
         if (!trace_event_get_state_static(ev)) {
             elem->value->state = TRACE_EVENT_STATE_UNAVAILABLE;
-        } else if (!trace_event_get_state_dynamic(ev)) {
-            elem->value->state = TRACE_EVENT_STATE_DISABLED;
         } else {
-            elem->value->state = TRACE_EVENT_STATE_ENABLED;
+            if (has_vcpu) {
+                if (is_vcpu) {
+                    if (trace_event_get_vcpu_state_dynamic(cpu, ev)) {
+                        elem->value->state = TRACE_EVENT_STATE_ENABLED;
+                    } else {
+                        elem->value->state = TRACE_EVENT_STATE_DISABLED;
+                    }
+                }
+                /* else: already skipped above */
+            } else {
+                if (trace_event_get_state_dynamic(ev)) {
+                    elem->value->state = TRACE_EVENT_STATE_ENABLED;
+                } else {
+                    elem->value->state = TRACE_EVENT_STATE_DISABLED;
+                }
+            }
         }
         elem->next = events;
         events = elem;
-        found = true;
-    }
-
-    if (!found && !trace_event_is_pattern(name)) {
-        error_setg(errp, "unknown event \"%s\"", name);
     }
 
     return events;
 }
 
 void qmp_trace_event_set_state(const char *name, bool enable,
-                               bool has_ignore_unavailable,
-                               bool ignore_unavailable, Error **errp)
+                               bool has_ignore_unavailable, bool ignore_unavailable,
+                               bool has_vcpu, int64_t vcpu,
+                               Error **errp)
 {
-    bool found = false;
+    Error *err = NULL;
     TraceEvent *ev;
+    bool is_pattern = trace_event_is_pattern(name);
+    CPUState *cpu;
 
-    /* Check all selected events are dynamic */
-    ev = NULL;
-    while ((ev = trace_event_pattern(name, ev)) != NULL) {
-        found = true;
-        if (!(has_ignore_unavailable && ignore_unavailable) &&
-            !trace_event_get_state_static(ev)) {
-            error_setg(errp, "cannot set dynamic tracing state for \"%s\"",
-                       trace_event_get_name(ev));
-            return;
-        }
-    }
-    if (!found && !trace_event_is_pattern(name)) {
-        error_setg(errp, "unknown event \"%s\"", name);
+    /* Check provided vcpu */
+    cpu = get_cpu(has_vcpu, vcpu, &err);
+    if (err) {
+        error_propagate(errp, err);
         return;
     }
 
-    /* Apply changes */
+    /* Check events */
+    if (!check_events(has_vcpu, has_ignore_unavailable && ignore_unavailable,
+                      is_pattern, name, errp)) {
+        return;
+    }
+
+    /* Apply changes (all errors checked above) */
     ev = NULL;
     while ((ev = trace_event_pattern(name, ev)) != NULL) {
-        if (trace_event_get_state_static(ev)) {
+        if (!trace_event_get_state_static(ev) ||
+            (has_vcpu && !trace_event_is_vcpu(ev))) {
+            continue;
+        }
+        if (has_vcpu) {
+            trace_event_set_vcpu_state_dynamic(cpu, ev, enable);
+        } else {
             trace_event_set_state_dynamic(ev, enable);
         }
     }
diff --git a/trace/simple.c b/trace/simple.c
index 11ad030..2f09daf 100644
--- a/trace/simple.c
+++ b/trace/simple.c
@@ -8,12 +8,8 @@
  *
  */
 
-#include <stdlib.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <time.h>
+#include "qemu/osdep.h"
 #ifndef _WIN32
-#include <signal.h>
 #include <pthread.h>
 #endif
 #include "qemu/timer.h"
@@ -112,7 +108,7 @@
     smp_rmb(); /* read memory barrier before accessing record */
     /* read the record header to know record length */
     read_from_buffer(idx, &record, sizeof(TraceRecord));
-    *recordptr = malloc(record.length); /* dont use g_malloc, can deadlock when traced */
+    *recordptr = malloc(record.length); /* don't use g_malloc, can deadlock when traced */
     /* make a copy of record to avoid being overwritten */
     read_from_buffer(idx, *recordptr, record.length);
     smp_rmb(); /* memory barrier before clearing valid flag */
@@ -184,7 +180,7 @@
         while (get_trace_record(idx, &recordptr)) {
             unused = fwrite(recordptr, recordptr->length, 1, trace_fp);
             writeout_idx += recordptr->length;
-            free(recordptr); /* dont use g_free, can deadlock when traced */
+            free(recordptr); /* don't use g_free, can deadlock when traced */
             idx = writeout_idx % TRACE_BUF_LEN;
         }
 
@@ -322,20 +318,20 @@
  * @file        The trace file name or NULL for the default name-<pid> set at
  *              config time
  */
-bool st_set_trace_file(const char *file)
+void st_set_trace_file(const char *file)
 {
     st_set_trace_file_enabled(false);
 
     g_free(trace_file_name);
 
     if (!file) {
-        trace_file_name = g_strdup_printf(CONFIG_TRACE_FILE, getpid());
+        /* Type cast needed for Windows where getpid() returns an int. */
+        trace_file_name = g_strdup_printf(CONFIG_TRACE_FILE, (pid_t)getpid());
     } else {
         trace_file_name = g_strdup_printf("%s", file);
     }
 
     st_set_trace_file_enabled(true);
-    return true;
 }
 
 void st_print_trace_file_status(FILE *stream, int (*stream_printf)(FILE *stream, const char *fmt, ...))
@@ -373,7 +369,7 @@
     return thread;
 }
 
-bool st_init(const char *file)
+bool st_init(void)
 {
     GThread *thread;
 
@@ -386,6 +382,5 @@
     }
 
     atexit(st_flush_trace_buffer);
-    st_set_trace_file(file);
     return true;
 }
diff --git a/trace/simple.h b/trace/simple.h
index 6997996..1e7de45 100644
--- a/trace/simple.h
+++ b/trace/simple.h
@@ -11,17 +11,14 @@
 #ifndef TRACE_SIMPLE_H
 #define TRACE_SIMPLE_H
 
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdio.h>
 
 #include "trace/generated-events.h"
 
 
 void st_print_trace_file_status(FILE *stream, fprintf_function stream_printf);
 void st_set_trace_file_enabled(bool enable);
-bool st_set_trace_file(const char *file);
-bool st_init(const char *file);
+void st_set_trace_file(const char *file);
+bool st_init(void);
 void st_flush_trace_buffer(void);
 
 typedef struct {
diff --git a/translate-all.c b/translate-all.c
index 47866f8..264c3b2 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -18,23 +18,16 @@
  */
 #ifdef _WIN32
 #include <windows.h>
-#else
-#include <sys/types.h>
-#include <sys/mman.h>
 #endif
-#include <stdarg.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
+#include "qemu/osdep.h"
 
-#include "config.h"
 
 #include "qemu-common.h"
 #define NO_CPU_IO_DEFS
 #include "cpu.h"
 #include "trace.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
 #if defined(CONFIG_USER_ONLY)
 #include "qemu.h"
@@ -43,7 +36,6 @@
 #if __FreeBSD_version >= 700104
 #define HAVE_KINFO_GETVMMAP
 #define sigqueue sigqueue_freebsd  /* avoid redefinition */
-#include <sys/time.h>
 #include <sys/proc.h>
 #include <machine/profile.h>
 #define _KERNEL
@@ -58,8 +50,11 @@
 #endif
 
 #include "exec/cputlb.h"
+#include "exec/tb-hash.h"
 #include "translate-all.h"
+#include "qemu/bitmap.h"
 #include "qemu/timer.h"
+#include "exec/log.h"
 
 //#define DEBUG_TB_INVALIDATE
 //#define DEBUG_FLUSH
@@ -76,11 +71,12 @@
 typedef struct PageDesc {
     /* list of TBs intersecting this ram page */
     TranslationBlock *first_tb;
+#ifdef CONFIG_SOFTMMU
     /* in order to optimize self modifying code, we count the number
        of lookups we do to a given page to use a bitmap */
     unsigned int code_write_count;
-    uint8_t *code_bitmap;
-#if defined(CONFIG_USER_ONLY)
+    unsigned long *code_bitmap;
+#else
     unsigned long flags;
 #endif
 } PageDesc;
@@ -115,12 +111,10 @@
 
 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
 
-uintptr_t qemu_real_host_page_size;
 uintptr_t qemu_host_page_size;
-uintptr_t qemu_host_page_mask;
+intptr_t qemu_host_page_mask;
 
-/* This is a multi-level map on the virtual address space.
-   The bottom level has pointers to PageDesc.  */
+/* The bottom level has pointers to PageDesc */
 static void *l1_map[V_L1_SIZE];
 
 /* code generation context */
@@ -128,8 +122,39 @@
 
 bool g_tcg_enabled = false;
 
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
-                         tb_page_addr_t phys_page2);
+/* translation block context */
+#ifdef CONFIG_USER_ONLY
+__thread int have_tb_lock;
+#endif
+
+void tb_lock(void)
+{
+#ifdef CONFIG_USER_ONLY
+    assert(!have_tb_lock);
+    qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
+    have_tb_lock++;
+#endif
+}
+
+void tb_unlock(void)
+{
+#ifdef CONFIG_USER_ONLY
+    assert(have_tb_lock);
+    have_tb_lock--;
+    qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+#endif
+}
+
+void tb_lock_reset(void)
+{
+#ifdef CONFIG_USER_ONLY
+    if (have_tb_lock) {
+        qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
+        have_tb_lock = 0;
+    }
+#endif
+}
+
 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
 
 void cpu_gen_init(void)
@@ -137,124 +162,137 @@
     tcg_context_init(&tcg_ctx); 
 }
 
-/* return non zero if the very first instruction is invalid so that
-   the virtual CPU can trigger an exception.
-
-   '*gen_code_size_ptr' contains the size of the generated code (host
-   code).
-*/
-int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
+/* Encode VAL as a signed leb128 sequence at P.
+   Return P incremented past the encoded value.  */
+static uint8_t *encode_sleb128(uint8_t *p, target_long val)
 {
-    TCGContext *s = &tcg_ctx;
-    tcg_insn_unit *gen_code_buf;
-    int gen_code_size;
-#ifdef CONFIG_PROFILER
-    int64_t ti;
-#endif
+    int more, byte;
 
-#ifdef CONFIG_PROFILER
-    s->tb_count1++; /* includes aborted translations because of
-                       exceptions */
-    ti = profile_getclock();
-#endif
-    tcg_func_start(s);
+    do {
+        byte = val & 0x7f;
+        val >>= 7;
+        more = !((val == 0 && (byte & 0x40) == 0)
+                 || (val == -1 && (byte & 0x40) != 0));
+        if (more) {
+            byte |= 0x80;
+        }
+        *p++ = byte;
+    } while (more);
 
-    gen_intermediate_code(env, tb);
-
-    trace_translate_block(tb, tb->pc, tb->tc_ptr);
-
-    /* generate machine code */
-    gen_code_buf = tb->tc_ptr;
-    tb->tb_next_offset[0] = 0xffff;
-    tb->tb_next_offset[1] = 0xffff;
-    s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
-    s->tb_jmp_offset = tb->tb_jmp_offset;
-    s->tb_next = NULL;
-#else
-    s->tb_jmp_offset = NULL;
-    s->tb_next = tb->tb_next;
-#endif
-
-#ifdef CONFIG_PROFILER
-    s->tb_count++;
-    s->interm_time += profile_getclock() - ti;
-    s->code_time -= profile_getclock();
-#endif
-    gen_code_size = tcg_gen_code(s, gen_code_buf);
-    *gen_code_size_ptr = gen_code_size;
-#ifdef CONFIG_PROFILER
-    s->code_time += profile_getclock();
-    s->code_in_len += tb->size;
-    s->code_out_len += gen_code_size;
-#endif
-
-#ifdef DEBUG_DISAS
-    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
-        qemu_log("OUT: [size=%d]\n", gen_code_size);
-        log_disas(tb->tc_ptr, gen_code_size);
-        qemu_log("\n");
-        qemu_log_flush();
-    }
-#endif
-    return 0;
+    return p;
 }
 
-/* The cpu state corresponding to 'searched_pc' is restored.
- */
+/* Decode a signed leb128 sequence at *PP; increment *PP past the
+   decoded value.  Return the decoded value.  */
+static target_long decode_sleb128(uint8_t **pp)
+{
+    uint8_t *p = *pp;
+    target_long val = 0;
+    int byte, shift = 0;
+
+    do {
+        byte = *p++;
+        val |= (target_ulong)(byte & 0x7f) << shift;
+        shift += 7;
+    } while (byte & 0x80);
+    if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
+        val |= -(target_ulong)1 << shift;
+    }
+
+    *pp = p;
+    return val;
+}
+
+/* Encode the data collected about the instructions while compiling TB.
+   Place the data at BLOCK, and return the number of bytes consumed.
+
+   The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
+   which come from the target's insn_start data, followed by a uintptr_t
+   which comes from the host pc of the end of the code implementing the insn.
+
+   Each line of the table is encoded as sleb128 deltas from the previous
+   line.  The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
+   That is, the first column is seeded with the guest pc, the last column
+   with the host pc, and the middle columns with zeros.  */
+
+static int encode_search(TranslationBlock *tb, uint8_t *block)
+{
+    uint8_t *highwater = tcg_ctx.code_gen_highwater;
+    uint8_t *p = block;
+    int i, j, n;
+
+    tb->tc_search = block;
+
+    for (i = 0, n = tb->icount; i < n; ++i) {
+        target_ulong prev;
+
+        for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
+            if (i == 0) {
+                prev = (j == 0 ? tb->pc : 0);
+            } else {
+                prev = tcg_ctx.gen_insn_data[i - 1][j];
+            }
+            p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
+        }
+        prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
+        p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
+
+        /* Test for (pending) buffer overflow.  The assumption is that any
+           one row beginning below the high water mark cannot overrun
+           the buffer completely.  Thus we can test for overflow after
+           encoding a row without having to check during encoding.  */
+        if (unlikely(p > highwater)) {
+            return -1;
+        }
+    }
+
+    return p - block;
+}
+
+/* The cpu state corresponding to 'searched_pc' is restored.  */
 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
                                      uintptr_t searched_pc)
 {
+    target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
+    uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
     CPUArchState *env = cpu->env_ptr;
-    TCGContext *s = &tcg_ctx;
-    int j;
-    uintptr_t tc_ptr;
+    uint8_t *p = tb->tc_search;
+    int i, j, num_insns = tb->icount;
 #ifdef CONFIG_PROFILER
-    int64_t ti;
+    int64_t ti = profile_getclock();
 #endif
 
-#ifdef CONFIG_PROFILER
-    ti = profile_getclock();
-#endif
-    tcg_func_start(s);
+    if (searched_pc < host_pc) {
+        return -1;
+    }
 
-    gen_intermediate_code_pc(env, tb);
+    /* Reconstruct the stored insn data while looking for the point at
+       which the end of the insn exceeds the searched_pc.  */
+    for (i = 0; i < num_insns; ++i) {
+        for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
+            data[j] += decode_sleb128(&p);
+        }
+        host_pc += decode_sleb128(&p);
+        if (host_pc > searched_pc) {
+            goto found;
+        }
+    }
+    return -1;
 
-    if (use_icount) {
+ found:
+    if (tb->cflags & CF_USE_ICOUNT) {
+        assert(use_icount);
         /* Reset the cycle counter to the start of the block.  */
-        cpu->icount_decr.u16.low += tb->icount;
+        cpu->icount_decr.u16.low += num_insns;
         /* Clear the IO flag.  */
         cpu->can_do_io = 0;
     }
-
-    /* find opc index corresponding to search_pc */
-    tc_ptr = (uintptr_t)tb->tc_ptr;
-    if (searched_pc < tc_ptr)
-        return -1;
-
-    s->tb_next_offset = tb->tb_next_offset;
-#ifdef USE_DIRECT_JUMP
-    s->tb_jmp_offset = tb->tb_jmp_offset;
-    s->tb_next = NULL;
-#else
-    s->tb_jmp_offset = NULL;
-    s->tb_next = tb->tb_next;
-#endif
-    j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
-                               searched_pc - tc_ptr);
-    if (j < 0)
-        return -1;
-    /* now find start of instruction before */
-    while (s->gen_opc_instr_start[j] == 0) {
-        j--;
-    }
-    cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
-
-    restore_state_to_opc(env, tb, j);
+    cpu->icount_decr.u16.low -= i;
+    restore_state_to_opc(env, tb, data);
 
 #ifdef CONFIG_PROFILER
-    s->restore_time += profile_getclock() - ti;
-    s->restore_count++;
+    tcg_ctx.restore_time += profile_getclock() - ti;
+    tcg_ctx.restore_count++;
 #endif
     return 0;
 }
@@ -266,48 +304,29 @@
     tb = tb_find_pc(retaddr);
     if (tb) {
         cpu_restore_state_from_tb(cpu, tb, retaddr);
+        if (tb->cflags & CF_NOCACHE) {
+            /* one-shot translation, invalidate it immediately */
+            tb_phys_invalidate(tb, -1);
+            tb_free(tb);
+        }
         return true;
     }
     return false;
 }
 
-#ifdef _WIN32
-static inline void map_exec(void *addr, long size)
-{
-    DWORD old_protect;
-    VirtualProtect(addr, size,
-                   PAGE_EXECUTE_READWRITE, &old_protect);
-}
-#else
-static inline void map_exec(void *addr, long size)
-{
-    unsigned long start, end, page_size;
-
-    page_size = getpagesize();
-    start = (unsigned long)addr;
-    start &= ~(page_size - 1);
-
-    end = (unsigned long)addr + size;
-    end += page_size - 1;
-    end &= ~(page_size - 1);
-
-    mprotect((void *)start, end - start,
-             PROT_READ | PROT_WRITE | PROT_EXEC);
-}
-#endif
-
 void page_size_init(void)
 {
     /* NOTE: we can always suppose that qemu_host_page_size >=
        TARGET_PAGE_SIZE */
     qemu_real_host_page_size = getpagesize();
+    qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
     if (qemu_host_page_size == 0) {
         qemu_host_page_size = qemu_real_host_page_size;
     }
     if (qemu_host_page_size < TARGET_PAGE_SIZE) {
         qemu_host_page_size = TARGET_PAGE_SIZE;
     }
-    qemu_host_page_mask = ~(qemu_host_page_size - 1);
+    qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
 }
 
 static void page_init(void)
@@ -379,53 +398,42 @@
 #endif
 }
 
+/* If alloc=1:
+ * Called with mmap_lock held for user-mode emulation.
+ */
 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
 {
     PageDesc *pd;
     void **lp;
     int i;
 
-#if defined(CONFIG_USER_ONLY)
-    /* We can't use g_malloc because it may recurse into a locked mutex. */
-# define ALLOC(P, SIZE)                                 \
-    do {                                                \
-        P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE,    \
-                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);   \
-    } while (0)
-#else
-# define ALLOC(P, SIZE) \
-    do { P = g_malloc0(SIZE); } while (0)
-#endif
-
     /* Level 1.  Always allocated.  */
     lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
 
     /* Level 2..N-1.  */
     for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
-        void **p = *lp;
+        void **p = atomic_rcu_read(lp);
 
         if (p == NULL) {
             if (!alloc) {
                 return NULL;
             }
-            ALLOC(p, sizeof(void *) * V_L2_SIZE);
-            *lp = p;
+            p = g_new0(void *, V_L2_SIZE);
+            atomic_rcu_set(lp, p);
         }
 
         lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
     }
 
-    pd = *lp;
+    pd = atomic_rcu_read(lp);
     if (pd == NULL) {
         if (!alloc) {
             return NULL;
         }
-        ALLOC(pd, sizeof(PageDesc) * V_L2_SIZE);
-        *lp = pd;
+        pd = g_new0(PageDesc, V_L2_SIZE);
+        atomic_rcu_set(lp, pd);
     }
 
-#undef ALLOC
-
     return pd + (index & (V_L2_SIZE - 1));
 }
 
@@ -434,11 +442,6 @@
     return page_find_alloc(index, 0);
 }
 
-#if !defined(CONFIG_USER_ONLY)
-#define mmap_lock() do { } while (0)
-#define mmap_unlock() do { } while (0)
-#endif
-
 #if defined(CONFIG_USER_ONLY)
 /* Currently it is not recommended to allocate big chunks of data in
    user mode. It will change when a dedicated libc will be used.  */
@@ -447,14 +450,6 @@
 #define USE_STATIC_CODE_GEN_BUFFER
 #endif
 
-/* ??? Should configure for this, not list operating systems here.  */
-#if (defined(__linux__) \
-    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
-    || defined(__DragonFly__) || defined(__OpenBSD__) \
-    || defined(__NetBSD__))
-# define USE_MMAP
-#endif
-
 /* Minimum size of the code gen buffer.  This number is randomly chosen,
    but not so small that we can't have a fair number of TB's live.  */
 #define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
@@ -466,6 +461,10 @@
 # define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
 #elif defined(__sparc__)
 # define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__powerpc64__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__powerpc__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (32u * 1024 * 1024)
 #elif defined(__aarch64__)
 # define MAX_CODE_GEN_BUFFER_SIZE  (128ul * 1024 * 1024)
 #elif defined(__arm__)
@@ -507,7 +506,6 @@
     if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
         tb_size = MAX_CODE_GEN_BUFFER_SIZE;
     }
-    tcg_ctx.code_gen_buffer_size = tb_size;
     return tb_size;
 }
 
@@ -516,7 +514,7 @@
    that the buffer not cross a 256MB boundary.  */
 static inline bool cross_256mb(void *addr, size_t size)
 {
-    return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
+    return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
 }
 
 /* We weren't able to allocate a buffer without crossing that boundary,
@@ -524,7 +522,7 @@
    Returns the new base of the buffer, and adjusts code_gen_buffer_size.  */
 static inline void *split_cross_256mb(void *buf1, size_t size1)
 {
-    void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
+    void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
     size_t size2 = buf1 + size1 - buf2;
 
     size1 = buf2 - buf1;
@@ -542,22 +540,102 @@
 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
     __attribute__((aligned(CODE_GEN_ALIGN)));
 
+# ifdef _WIN32
+static inline void do_protect(void *addr, long size, int prot)
+{
+    DWORD old_protect;
+    VirtualProtect(addr, size, prot, &old_protect);
+}
+
+static inline void map_exec(void *addr, long size)
+{
+    do_protect(addr, size, PAGE_EXECUTE_READWRITE);
+}
+
+static inline void map_none(void *addr, long size)
+{
+    do_protect(addr, size, PAGE_NOACCESS);
+}
+# else
+static inline void do_protect(void *addr, long size, int prot)
+{
+    uintptr_t start, end;
+
+    start = (uintptr_t)addr;
+    start &= qemu_real_host_page_mask;
+
+    end = (uintptr_t)addr + size;
+    end = ROUND_UP(end, qemu_real_host_page_size);
+
+    mprotect((void *)start, end - start, prot);
+}
+
+static inline void map_exec(void *addr, long size)
+{
+    do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
+}
+
+static inline void map_none(void *addr, long size)
+{
+    do_protect(addr, size, PROT_NONE);
+}
+# endif /* WIN32 */
+
 static inline void *alloc_code_gen_buffer(void)
 {
     void *buf = static_code_gen_buffer;
+    size_t full_size, size;
+
+    /* The size of the buffer, rounded down to end on a page boundary.  */
+    full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
+                 & qemu_real_host_page_mask) - (uintptr_t)buf;
+
+    /* Reserve a guard page.  */
+    size = full_size - qemu_real_host_page_size;
+
+    /* Honor a command-line option limiting the size of the buffer.  */
+    if (size > tcg_ctx.code_gen_buffer_size) {
+        size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
+                & qemu_real_host_page_mask) - (uintptr_t)buf;
+    }
+    tcg_ctx.code_gen_buffer_size = size;
+
 #ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
-        buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
+    if (cross_256mb(buf, size)) {
+        buf = split_cross_256mb(buf, size);
+        size = tcg_ctx.code_gen_buffer_size;
     }
 #endif
-    map_exec(buf, tcg_ctx.code_gen_buffer_size);
+
+    map_exec(buf, size);
+    map_none(buf + size, qemu_real_host_page_size);
+    qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
+
     return buf;
 }
-#elif defined(USE_MMAP)
+#elif defined(_WIN32)
+static inline void *alloc_code_gen_buffer(void)
+{
+    size_t size = tcg_ctx.code_gen_buffer_size;
+    void *buf1, *buf2;
+
+    /* Perform the allocation in two steps, so that the guard page
+       is reserved but uncommitted.  */
+    buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
+                        MEM_RESERVE, PAGE_NOACCESS);
+    if (buf1 != NULL) {
+        buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+        assert(buf1 == buf2);
+    }
+
+    return buf1;
+}
+#else
 static inline void *alloc_code_gen_buffer(void)
 {
     int flags = MAP_PRIVATE | MAP_ANONYMOUS;
     uintptr_t start = 0;
+    size_t size = tcg_ctx.code_gen_buffer_size;
     void *buf;
 
     /* Constrain the position of the buffer based on the host cpu.
@@ -573,86 +651,70 @@
        Leave the choice of exact location with the kernel.  */
     flags |= MAP_32BIT;
     /* Cannot expect to map more than 800MB in low memory.  */
-    if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
-        tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
+    if (size > 800u * 1024 * 1024) {
+        tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
     }
 # elif defined(__sparc__)
     start = 0x40000000ul;
 # elif defined(__s390x__)
     start = 0x90000000ul;
 # elif defined(__mips__)
-    /* ??? We ought to more explicitly manage layout for softmmu too.  */
-#  ifdef CONFIG_USER_ONLY
-    start = 0x68000000ul;
-#  elif _MIPS_SIM == _ABI64
+#  if _MIPS_SIM == _ABI64
     start = 0x128000000ul;
 #  else
     start = 0x08000000ul;
 #  endif
 # endif
 
-    buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
-               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
+    buf = mmap((void *)start, size + qemu_real_host_page_size,
+               PROT_NONE, flags, -1, 0);
     if (buf == MAP_FAILED) {
         return NULL;
     }
 
 #ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
+    if (cross_256mb(buf, size)) {
         /* Try again, with the original still mapped, to avoid re-acquiring
            that 256mb crossing.  This time don't specify an address.  */
-        size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
-        void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
-                          flags, -1, 0);
-        if (buf2 != MAP_FAILED) {
-            if (!cross_256mb(buf2, size1)) {
+        size_t size2;
+        void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
+                          PROT_NONE, flags, -1, 0);
+        switch (buf2 != MAP_FAILED) {
+        case 1:
+            if (!cross_256mb(buf2, size)) {
                 /* Success!  Use the new buffer.  */
-                munmap(buf, size1);
-                return buf2;
+                munmap(buf, size + qemu_real_host_page_size);
+                break;
             }
             /* Failure.  Work with what we had.  */
-            munmap(buf2, size1);
+            munmap(buf2, size + qemu_real_host_page_size);
+            /* fallthru */
+        default:
+            /* Split the original buffer.  Free the smaller half.  */
+            buf2 = split_cross_256mb(buf, size);
+            size2 = tcg_ctx.code_gen_buffer_size;
+            if (buf == buf2) {
+                munmap(buf + size2 + qemu_real_host_page_size, size - size2);
+            } else {
+                munmap(buf, size - size2);
+            }
+            size = size2;
+            break;
         }
-
-        /* Split the original buffer.  Free the smaller half.  */
-        buf2 = split_cross_256mb(buf, size1);
-        size2 = tcg_ctx.code_gen_buffer_size;
-        munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
-        return buf2;
+        buf = buf2;
     }
 #endif
 
+    /* Make the final buffer accessible.  The guard page at the end
+       will remain inaccessible with PROT_NONE.  */
+    mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
+
+    /* Request large pages for the buffer.  */
+    qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
+
     return buf;
 }
-#else
-static inline void *alloc_code_gen_buffer(void)
-{
-    void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
-
-    if (buf == NULL) {
-        return NULL;
-    }
-
-#ifdef __mips__
-    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
-        void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
-        if (buf2 != NULL && !cross_256mb(buf2, size1)) {
-            /* Success!  Use the new buffer.  */
-            free(buf);
-            buf = buf2;
-        } else {
-            /* Failure.  Work with what we had.  Since this is malloc
-               and not mmap, we can't free the other half.  */
-            free(buf2);
-            buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
-        }
-    }
-#endif
-
-    map_exec(buf, tcg_ctx.code_gen_buffer_size);
-    return buf;
-}
-#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
+#endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
 
 static inline void code_gen_alloc(size_t tb_size)
 {
@@ -663,26 +725,23 @@
         exit(1);
     }
 
+    /* Estimate a good size for the number of TBs we can support.  We
+       still haven't deducted the prologue from the buffer size here,
+       but that's minimal and won't affect the estimate much.  */
+    tcg_ctx.code_gen_max_blocks
+        = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
+    tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
+
     g_tcg_enabled = true;
 
-    qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
-            QEMU_MADV_HUGEPAGE);
+    qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
+}
 
-    /* Steal room for the prologue at the end of the buffer.  This ensures
-       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
-       from TB's to the prologue are going to be in range.  It also means
-       that we don't need to mark (additional) portions of the data segment
-       as executable.  */
-    tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
-            tcg_ctx.code_gen_buffer_size - 1024;
-    tcg_ctx.code_gen_buffer_size -= 1024;
+static void tb_htable_init(void)
+{
+    unsigned int mode = QHT_MODE_AUTO_RESIZE;
 
-    tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
-        (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
-    tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
-            CODE_GEN_AVG_BLOCK_SIZE;
-    tcg_ctx.tb_ctx.tbs =
-            g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
+    qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode);
 }
 
 /* Must be called before using the QEMU cpus. 'tb_size' is the size
@@ -691,11 +750,10 @@
 void tcg_exec_init(unsigned long tb_size)
 {
     cpu_gen_init();
-    code_gen_alloc(tb_size);
-    tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
-    tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
     page_init();
-#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
+    tb_htable_init();
+    code_gen_alloc(tb_size);
+#if defined(CONFIG_SOFTMMU)
     /* There's no guest base to take into account, so go ahead and
        initialize the prologue now.  */
     tcg_prologue_init(&tcg_ctx);
@@ -708,9 +766,7 @@
 {
     TranslationBlock *tb;
 
-    if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
-        (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
-         tcg_ctx.code_gen_buffer_max_size) {
+    if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
         return NULL;
     }
     tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
@@ -733,11 +789,11 @@
 
 static inline void invalidate_page_bitmap(PageDesc *p)
 {
-    if (p->code_bitmap) {
-        g_free(p->code_bitmap);
-        p->code_bitmap = NULL;
-    }
+#ifdef CONFIG_SOFTMMU
+    g_free(p->code_bitmap);
+    p->code_bitmap = NULL;
     p->code_write_count = 0;
+#endif
 }
 
 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
@@ -775,10 +831,11 @@
 
 /* flush all the translation blocks */
 /* XXX: tb_flush is currently not thread safe */
-void tb_flush(CPUArchState *env1)
+void tb_flush(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env1);
-
+    if (!tcg_enabled()) {
+        return;
+    }
 #if defined(DEBUG_FLUSH)
     printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
            (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
@@ -794,9 +851,10 @@
 
     CPU_FOREACH(cpu) {
         memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
+        cpu->tb_flushed = true;
     }
 
-    memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
+    qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
     page_flush_tb();
 
     tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
@@ -807,59 +865,46 @@
 
 #ifdef DEBUG_TB_CHECK
 
+static void
+do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp)
+{
+    TranslationBlock *tb = p;
+    target_ulong addr = *(target_ulong *)userp;
+
+    if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
+        printf("ERROR invalidate: address=" TARGET_FMT_lx
+               " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
+    }
+}
+
 static void tb_invalidate_check(target_ulong address)
 {
-    TranslationBlock *tb;
-    int i;
-
     address &= TARGET_PAGE_MASK;
-    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
-            if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
-                  address >= tb->pc + tb->size)) {
-                printf("ERROR invalidate: address=" TARGET_FMT_lx
-                       " PC=%08lx size=%04x\n",
-                       address, (long)tb->pc, tb->size);
-            }
-        }
+    qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address);
+}
+
+static void
+do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp)
+{
+    TranslationBlock *tb = p;
+    int flags1, flags2;
+
+    flags1 = page_get_flags(tb->pc);
+    flags2 = page_get_flags(tb->pc + tb->size - 1);
+    if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
+        printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
+               (long)tb->pc, tb->size, flags1, flags2);
     }
 }
 
 /* verify that all the pages have correct rights for code */
 static void tb_page_check(void)
 {
-    TranslationBlock *tb;
-    int i, flags1, flags2;
-
-    for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
-        for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
-                tb = tb->phys_hash_next) {
-            flags1 = page_get_flags(tb->pc);
-            flags2 = page_get_flags(tb->pc + tb->size - 1);
-            if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
-                printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
-                       (long)tb->pc, tb->size, flags1, flags2);
-            }
-        }
-    }
+    qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL);
 }
 
 #endif
 
-static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
-{
-    TranslationBlock *tb1;
-
-    for (;;) {
-        tb1 = *ptb;
-        if (tb1 == tb) {
-            *ptb = tb1->phys_hash_next;
-            break;
-        }
-        ptb = &tb1->phys_hash_next;
-    }
-}
-
 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
 {
     TranslationBlock *tb1;
@@ -877,32 +922,33 @@
     }
 }
 
-static inline void tb_jmp_remove(TranslationBlock *tb, int n)
+/* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
+static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
 {
-    TranslationBlock *tb1, **ptb;
+    TranslationBlock *tb1;
+    uintptr_t *ptb, ntb;
     unsigned int n1;
 
-    ptb = &tb->jmp_next[n];
-    tb1 = *ptb;
-    if (tb1) {
+    ptb = &tb->jmp_list_next[n];
+    if (*ptb) {
         /* find tb(n) in circular list */
         for (;;) {
-            tb1 = *ptb;
-            n1 = (uintptr_t)tb1 & 3;
-            tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
+            ntb = *ptb;
+            n1 = ntb & 3;
+            tb1 = (TranslationBlock *)(ntb & ~3);
             if (n1 == n && tb1 == tb) {
                 break;
             }
             if (n1 == 2) {
-                ptb = &tb1->jmp_first;
+                ptb = &tb1->jmp_list_first;
             } else {
-                ptb = &tb1->jmp_next[n1];
+                ptb = &tb1->jmp_list_next[n1];
             }
         }
         /* now we can suppress tb(n) from the list */
-        *ptb = tb->jmp_next[n];
+        *ptb = tb->jmp_list_next[n];
 
-        tb->jmp_next[n] = NULL;
+        tb->jmp_list_next[n] = (uintptr_t)NULL;
     }
 }
 
@@ -910,7 +956,29 @@
    another TB */
 static inline void tb_reset_jump(TranslationBlock *tb, int n)
 {
-    tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
+    uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
+    tb_set_jmp_target(tb, n, addr);
+}
+
+/* remove any jumps to the TB */
+static inline void tb_jmp_unlink(TranslationBlock *tb)
+{
+    TranslationBlock *tb1;
+    uintptr_t *ptb, ntb;
+    unsigned int n1;
+
+    ptb = &tb->jmp_list_first;
+    for (;;) {
+        ntb = *ptb;
+        n1 = ntb & 3;
+        tb1 = (TranslationBlock *)(ntb & ~3);
+        if (n1 == 2) {
+            break;
+        }
+        tb_reset_jump(tb1, n1);
+        *ptb = tb1->jmp_list_next[n1];
+        tb1->jmp_list_next[n1] = (uintptr_t)NULL;
+    }
 }
 
 /* invalidate one TB */
@@ -918,14 +986,13 @@
 {
     CPUState *cpu;
     PageDesc *p;
-    unsigned int h, n1;
+    uint32_t h;
     tb_page_addr_t phys_pc;
-    TranslationBlock *tb1, *tb2;
 
     /* remove the TB from the hash list */
     phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
-    h = tb_phys_hash_func(phys_pc);
-    tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
+    h = tb_hash_func(phys_pc, tb->pc, tb->flags);
+    qht_remove(&tcg_ctx.tb_ctx.htable, tb, h);
 
     /* remove the TB from the page list */
     if (tb->page_addr[0] != page_addr) {
@@ -939,8 +1006,6 @@
         invalidate_page_bitmap(p);
     }
 
-    tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
-
     /* remove the TB from the hash list */
     h = tb_jmp_cache_hash_func(tb->pc);
     CPU_FOREACH(cpu) {
@@ -950,60 +1015,22 @@
     }
 
     /* suppress this TB from the two jump lists */
-    tb_jmp_remove(tb, 0);
-    tb_jmp_remove(tb, 1);
+    tb_remove_from_jmp_list(tb, 0);
+    tb_remove_from_jmp_list(tb, 1);
 
     /* suppress any remaining jumps to this TB */
-    tb1 = tb->jmp_first;
-    for (;;) {
-        n1 = (uintptr_t)tb1 & 3;
-        if (n1 == 2) {
-            break;
-        }
-        tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
-        tb2 = tb1->jmp_next[n1];
-        tb_reset_jump(tb1, n1);
-        tb1->jmp_next[n1] = NULL;
-        tb1 = tb2;
-    }
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
+    tb_jmp_unlink(tb);
 
     tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
 }
 
-static inline void set_bits(uint8_t *tab, int start, int len)
-{
-    int end, mask, end1;
-
-    end = start + len;
-    tab += start >> 3;
-    mask = 0xff << (start & 7);
-    if ((start & ~7) == (end & ~7)) {
-        if (start < end) {
-            mask &= ~(0xff << (end & 7));
-            *tab |= mask;
-        }
-    } else {
-        *tab++ |= mask;
-        start = (start + 8) & ~7;
-        end1 = end & ~7;
-        while (start < end1) {
-            *tab++ = 0xff;
-            start += 8;
-        }
-        if (start < end) {
-            mask = ~(0xff << (end & 7));
-            *tab |= mask;
-        }
-    }
-}
-
+#ifdef CONFIG_SOFTMMU
 static void build_page_bitmap(PageDesc *p)
 {
     int n, tb_start, tb_end;
     TranslationBlock *tb;
 
-    p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
+    p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
 
     tb = p->first_tb;
     while (tb != NULL) {
@@ -1022,38 +1049,213 @@
             tb_start = 0;
             tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
         }
-        set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
+        bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
         tb = tb->page_next[n];
     }
 }
+#endif
 
+/* add the tb in the target page and protect it if necessary
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+static inline void tb_alloc_page(TranslationBlock *tb,
+                                 unsigned int n, tb_page_addr_t page_addr)
+{
+    PageDesc *p;
+#ifndef CONFIG_USER_ONLY
+    bool page_already_protected;
+#endif
+
+    tb->page_addr[n] = page_addr;
+    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
+    tb->page_next[n] = p->first_tb;
+#ifndef CONFIG_USER_ONLY
+    page_already_protected = p->first_tb != NULL;
+#endif
+    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
+    invalidate_page_bitmap(p);
+
+#if defined(CONFIG_USER_ONLY)
+    if (p->flags & PAGE_WRITE) {
+        target_ulong addr;
+        PageDesc *p2;
+        int prot;
+
+        /* force the host page as non writable (writes will have a
+           page fault + mprotect overhead) */
+        page_addr &= qemu_host_page_mask;
+        prot = 0;
+        for (addr = page_addr; addr < page_addr + qemu_host_page_size;
+            addr += TARGET_PAGE_SIZE) {
+
+            p2 = page_find(addr >> TARGET_PAGE_BITS);
+            if (!p2) {
+                continue;
+            }
+            prot |= p2->flags;
+            p2->flags &= ~PAGE_WRITE;
+          }
+        mprotect(g2h(page_addr), qemu_host_page_size,
+                 (prot & PAGE_BITS) & ~PAGE_WRITE);
+#ifdef DEBUG_TB_INVALIDATE
+        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
+               page_addr);
+#endif
+    }
+#else
+    /* if some code is already present, then the pages are already
+       protected. So we handle the case where only the first TB is
+       allocated in a physical page */
+    if (!page_already_protected) {
+        tlb_protect_code(page_addr);
+    }
+#endif
+}
+
+/* add a new TB and link it to the physical page tables. phys_page2 is
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2)
+{
+    uint32_t h;
+
+    /* add in the hash table */
+    h = tb_hash_func(phys_pc, tb->pc, tb->flags);
+    qht_insert(&tcg_ctx.tb_ctx.htable, tb, h);
+
+    /* add in the page list */
+    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
+    if (phys_page2 != -1) {
+        tb_alloc_page(tb, 1, phys_page2);
+    } else {
+        tb->page_addr[1] = -1;
+    }
+
+#ifdef DEBUG_TB_CHECK
+    tb_page_check();
+#endif
+}
+
+/* Called with mmap_lock held for user mode emulation.  */
 TranslationBlock *tb_gen_code(CPUState *cpu,
                               target_ulong pc, target_ulong cs_base,
-                              int flags, int cflags)
+                              uint32_t flags, int cflags)
 {
     CPUArchState *env = cpu->env_ptr;
     TranslationBlock *tb;
     tb_page_addr_t phys_pc, phys_page2;
     target_ulong virt_page2;
-    int code_gen_size;
+    tcg_insn_unit *gen_code_buf;
+    int gen_code_size, search_size;
+#ifdef CONFIG_PROFILER
+    int64_t ti;
+#endif
 
     phys_pc = get_page_addr_code(env, pc);
+    if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
+        cflags |= CF_USE_ICOUNT;
+    }
+
     tb = tb_alloc(pc);
-    if (!tb) {
+    if (unlikely(!tb)) {
+ buffer_overflow:
         /* flush must be done */
-        tb_flush(env);
+        tb_flush(cpu);
         /* cannot fail at this point */
         tb = tb_alloc(pc);
-        /* Don't forget to invalidate previous TB info.  */
-        tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
+        assert(tb != NULL);
     }
-    tb->tc_ptr = tcg_ctx.code_gen_ptr;
+
+    gen_code_buf = tcg_ctx.code_gen_ptr;
+    tb->tc_ptr = gen_code_buf;
     tb->cs_base = cs_base;
     tb->flags = flags;
     tb->cflags = cflags;
-    cpu_gen_code(env, tb, &code_gen_size);
-    tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
-            code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.tb_count1++; /* includes aborted translations because of
+                       exceptions */
+    ti = profile_getclock();
+#endif
+
+    tcg_func_start(&tcg_ctx);
+
+    tcg_ctx.cpu = ENV_GET_CPU(env);
+    gen_intermediate_code(env, tb);
+    tcg_ctx.cpu = NULL;
+
+    trace_translate_block(tb, tb->pc, tb->tc_ptr);
+
+    /* generate machine code */
+    tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
+    tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
+    tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
+#ifdef USE_DIRECT_JUMP
+    tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
+    tcg_ctx.tb_jmp_target_addr = NULL;
+#else
+    tcg_ctx.tb_jmp_insn_offset = NULL;
+    tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
+#endif
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.tb_count++;
+    tcg_ctx.interm_time += profile_getclock() - ti;
+    tcg_ctx.code_time -= profile_getclock();
+#endif
+
+    /* ??? Overflow could be handled better here.  In particular, we
+       don't need to re-do gen_intermediate_code, nor should we re-do
+       the tcg optimization currently hidden inside tcg_gen_code.  All
+       that should be required is to flush the TBs, allocate a new TB,
+       re-initialize it per above, and re-do the actual code generation.  */
+    gen_code_size = tcg_gen_code(&tcg_ctx, tb);
+    if (unlikely(gen_code_size < 0)) {
+        goto buffer_overflow;
+    }
+    search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
+    if (unlikely(search_size < 0)) {
+        goto buffer_overflow;
+    }
+
+#ifdef CONFIG_PROFILER
+    tcg_ctx.code_time += profile_getclock();
+    tcg_ctx.code_in_len += tb->size;
+    tcg_ctx.code_out_len += gen_code_size;
+    tcg_ctx.search_out_len += search_size;
+#endif
+
+#ifdef DEBUG_DISAS
+    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
+        qemu_log_in_addr_range(tb->pc)) {
+        qemu_log("OUT: [size=%d]\n", gen_code_size);
+        log_disas(tb->tc_ptr, gen_code_size);
+        qemu_log("\n");
+        qemu_log_flush();
+    }
+#endif
+
+    tcg_ctx.code_gen_ptr = (void *)
+        ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
+                 CODE_GEN_ALIGN);
+
+    /* init jump list */
+    assert(((uintptr_t)tb & 3) == 0);
+    tb->jmp_list_first = (uintptr_t)tb | 2;
+    tb->jmp_list_next[0] = (uintptr_t)NULL;
+    tb->jmp_list_next[1] = (uintptr_t)NULL;
+
+    /* init original jump addresses wich has been set during tcg_gen_code() */
+    if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
+        tb_reset_jump(tb, 0);
+    }
+    if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
+        tb_reset_jump(tb, 1);
+    }
 
     /* check next page if needed */
     virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
@@ -1061,6 +1263,11 @@
     if ((pc & TARGET_PAGE_MASK) != virt_page2) {
         phys_page2 = get_page_addr_code(env, virt_page2);
     }
+    /* As long as consistency of the TB stuff is provided by tb_lock in user
+     * mode and is implicit in single-threaded softmmu emulation, no explicit
+     * memory barrier is required before tb_link_page() makes the TB visible
+     * through the physical hash table and physical page list.
+     */
     tb_link_page(tb, phys_pc, phys_page2);
     return tb;
 }
@@ -1071,12 +1278,13 @@
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
  */
-void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
-                              int is_cpu_write_access)
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
 {
     while (start < end) {
-        tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
+        tb_invalidate_phys_page_range(start, end, 0);
         start &= TARGET_PAGE_MASK;
         start += TARGET_PAGE_SIZE;
     }
@@ -1088,13 +1296,15 @@
  * 'is_cpu_write_access' should be true if called from a real cpu write
  * access: the virtual CPU will exit the current TB if code is modified inside
  * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation
  */
 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
                                    int is_cpu_write_access)
 {
-    TranslationBlock *tb, *tb_next, *saved_tb;
-    CPUState *cpu = current_cpu;
+    TranslationBlock *tb, *tb_next;
 #if defined(TARGET_HAS_PRECISE_SMC)
+    CPUState *cpu = current_cpu;
     CPUArchState *env = NULL;
 #endif
     tb_page_addr_t tb_start, tb_end;
@@ -1106,19 +1316,13 @@
     int current_tb_modified = 0;
     target_ulong current_pc = 0;
     target_ulong current_cs_base = 0;
-    int current_flags = 0;
+    uint32_t current_flags = 0;
 #endif /* TARGET_HAS_PRECISE_SMC */
 
     p = page_find(start >> TARGET_PAGE_BITS);
     if (!p) {
         return;
     }
-    if (!p->code_bitmap &&
-        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
-        is_cpu_write_access) {
-        /* build code bitmap */
-        build_page_bitmap(p);
-    }
 #if defined(TARGET_HAS_PRECISE_SMC)
     if (cpu != NULL) {
         env = cpu->env_ptr;
@@ -1167,20 +1371,7 @@
                                      &current_flags);
             }
 #endif /* TARGET_HAS_PRECISE_SMC */
-            /* we need to do that to handle the case where a signal
-               occurs while doing tb_phys_invalidate() */
-            saved_tb = NULL;
-            if (cpu != NULL) {
-                saved_tb = cpu->current_tb;
-                cpu->current_tb = NULL;
-            }
             tb_phys_invalidate(tb, -1);
-            if (cpu != NULL) {
-                cpu->current_tb = saved_tb;
-                if (cpu->interrupt_request && cpu->current_tb) {
-                    cpu_interrupt(cpu, cpu->interrupt_request);
-                }
-            }
         }
         tb = tb_next;
     }
@@ -1188,9 +1379,7 @@
     /* if no code remaining, no need to continue to use slow writes */
     if (!p->first_tb) {
         invalidate_page_bitmap(p);
-        if (is_cpu_write_access) {
-            tlb_unprotect_code_phys(cpu, start, cpu->mem_io_vaddr);
-        }
+        tlb_unprotect_code(start);
     }
 #endif
 #ifdef TARGET_HAS_PRECISE_SMC
@@ -1198,18 +1387,17 @@
         /* we generate a block containing just the instruction
            modifying the memory. It will ensure that it cannot modify
            itself */
-        cpu->current_tb = NULL;
         tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
-        cpu_resume_from_signal(cpu, NULL);
+        cpu_loop_exit_noexc(cpu);
     }
 #endif
 }
 
+#ifdef CONFIG_SOFTMMU
 /* len must be <= 8 and start must be a multiple of len */
 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
 {
     PageDesc *p;
-    int offset, b;
 
 #if 0
     if (1) {
@@ -1224,9 +1412,17 @@
     if (!p) {
         return;
     }
+    if (!p->code_bitmap &&
+        ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
+        /* build code bitmap */
+        build_page_bitmap(p);
+    }
     if (p->code_bitmap) {
-        offset = start & ~TARGET_PAGE_MASK;
-        b = p->code_bitmap[offset >> 3] >> (offset & 7);
+        unsigned int nr;
+        unsigned long b;
+
+        nr = start & ~TARGET_PAGE_MASK;
+        b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
         if (b & ((1 << len) - 1)) {
             goto do_invalidate;
         }
@@ -1235,11 +1431,14 @@
         tb_invalidate_phys_page_range(start, start + len, 1);
     }
 }
-
-#if !defined(CONFIG_SOFTMMU)
-static void tb_invalidate_phys_page(tb_page_addr_t addr,
-                                    uintptr_t pc, void *puc,
-                                    bool locked)
+#else
+/* Called with mmap_lock held. If pc is not 0 then it indicates the
+ * host PC of the faulting store instruction that caused this invalidate.
+ * Returns true if the caller needs to abort execution of the current
+ * TB (because it was modified by this store and the guest CPU has
+ * precise-SMC semantics).
+ */
+static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
 {
     TranslationBlock *tb;
     PageDesc *p;
@@ -1251,13 +1450,13 @@
     int current_tb_modified = 0;
     target_ulong current_pc = 0;
     target_ulong current_cs_base = 0;
-    int current_flags = 0;
+    uint32_t current_flags = 0;
 #endif
 
     addr &= TARGET_PAGE_MASK;
     p = page_find(addr >> TARGET_PAGE_BITS);
     if (!p) {
-        return;
+        return false;
     }
     tb = p->first_tb;
 #ifdef TARGET_HAS_PRECISE_SMC
@@ -1295,119 +1494,14 @@
         /* we generate a block containing just the instruction
            modifying the memory. It will ensure that it cannot modify
            itself */
-        cpu->current_tb = NULL;
         tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
-        if (locked) {
-            mmap_unlock();
-        }
-        cpu_resume_from_signal(cpu, puc);
+        return true;
     }
 #endif
+    return false;
 }
 #endif
 
-/* add the tb in the target page and protect it if necessary */
-static inline void tb_alloc_page(TranslationBlock *tb,
-                                 unsigned int n, tb_page_addr_t page_addr)
-{
-    PageDesc *p;
-#ifndef CONFIG_USER_ONLY
-    bool page_already_protected;
-#endif
-
-    tb->page_addr[n] = page_addr;
-    p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
-    tb->page_next[n] = p->first_tb;
-#ifndef CONFIG_USER_ONLY
-    page_already_protected = p->first_tb != NULL;
-#endif
-    p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
-    invalidate_page_bitmap(p);
-
-#if defined(TARGET_HAS_SMC) || 1
-
-#if defined(CONFIG_USER_ONLY)
-    if (p->flags & PAGE_WRITE) {
-        target_ulong addr;
-        PageDesc *p2;
-        int prot;
-
-        /* force the host page as non writable (writes will have a
-           page fault + mprotect overhead) */
-        page_addr &= qemu_host_page_mask;
-        prot = 0;
-        for (addr = page_addr; addr < page_addr + qemu_host_page_size;
-            addr += TARGET_PAGE_SIZE) {
-
-            p2 = page_find(addr >> TARGET_PAGE_BITS);
-            if (!p2) {
-                continue;
-            }
-            prot |= p2->flags;
-            p2->flags &= ~PAGE_WRITE;
-          }
-        mprotect(g2h(page_addr), qemu_host_page_size,
-                 (prot & PAGE_BITS) & ~PAGE_WRITE);
-#ifdef DEBUG_TB_INVALIDATE
-        printf("protecting code page: 0x" TARGET_FMT_lx "\n",
-               page_addr);
-#endif
-    }
-#else
-    /* if some code is already present, then the pages are already
-       protected. So we handle the case where only the first TB is
-       allocated in a physical page */
-    if (!page_already_protected) {
-        tlb_protect_code(page_addr);
-    }
-#endif
-
-#endif /* TARGET_HAS_SMC */
-}
-
-/* add a new TB and link it to the physical page tables. phys_page2 is
-   (-1) to indicate that only one page contains the TB. */
-static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
-                         tb_page_addr_t phys_page2)
-{
-    unsigned int h;
-    TranslationBlock **ptb;
-
-    /* Grab the mmap lock to stop another thread invalidating this TB
-       before we are done.  */
-    mmap_lock();
-    /* add in the physical hash table */
-    h = tb_phys_hash_func(phys_pc);
-    ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
-    tb->phys_hash_next = *ptb;
-    *ptb = tb;
-
-    /* add in the page list */
-    tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
-    if (phys_page2 != -1) {
-        tb_alloc_page(tb, 1, phys_page2);
-    } else {
-        tb->page_addr[1] = -1;
-    }
-
-    tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
-    tb->jmp_next[0] = NULL;
-    tb->jmp_next[1] = NULL;
-
-    /* init original jump addresses */
-    if (tb->tb_next_offset[0] != 0xffff) {
-        tb_reset_jump(tb, 0);
-    }
-    if (tb->tb_next_offset[1] != 0xffff) {
-        tb_reset_jump(tb, 1);
-    }
-
-#ifdef DEBUG_TB_CHECK
-    tb_page_check();
-#endif
-    mmap_unlock();
-}
-
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
    tb[1].tc_ptr. Return NULL if not found */
 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
@@ -1441,68 +1535,50 @@
     return &tcg_ctx.tb_ctx.tbs[m_max];
 }
 
-#if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
+#if !defined(CONFIG_USER_ONLY)
 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
 {
     ram_addr_t ram_addr;
     MemoryRegion *mr;
     hwaddr l = 1;
 
+    rcu_read_lock();
     mr = address_space_translate(as, addr, &addr, &l, false);
     if (!(memory_region_is_ram(mr)
           || memory_region_is_romd(mr))) {
+        rcu_read_unlock();
         return;
     }
-    ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
-        + addr;
+    ram_addr = memory_region_get_ram_addr(mr) + addr;
     tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
+    rcu_read_unlock();
 }
-#endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
+#endif /* !defined(CONFIG_USER_ONLY) */
 
 void tb_check_watchpoint(CPUState *cpu)
 {
     TranslationBlock *tb;
 
     tb = tb_find_pc(cpu->mem_io_pc);
-    if (!tb) {
-        cpu_abort(cpu, "check_watchpoint: could not find TB for pc=%p",
-                  (void *)cpu->mem_io_pc);
+    if (tb) {
+        /* We can use retranslation to find the PC.  */
+        cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
+        tb_phys_invalidate(tb, -1);
+    } else {
+        /* The exception probably happened in a helper.  The CPU state should
+           have been saved before calling it. Fetch the PC from there.  */
+        CPUArchState *env = cpu->env_ptr;
+        target_ulong pc, cs_base;
+        tb_page_addr_t addr;
+        uint32_t flags;
+
+        cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
+        addr = get_page_addr_code(env, pc);
+        tb_invalidate_phys_range(addr, addr + 1);
     }
-    cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
-    tb_phys_invalidate(tb, -1);
 }
 
 #ifndef CONFIG_USER_ONLY
-/* mask must never be zero, except for A20 change call */
-static void tcg_handle_interrupt(CPUState *cpu, int mask)
-{
-    int old_mask;
-
-    old_mask = cpu->interrupt_request;
-    cpu->interrupt_request |= mask;
-
-    /*
-     * If called from iothread context, wake the target cpu in
-     * case its halted.
-     */
-    if (!qemu_cpu_is_self(cpu)) {
-        qemu_cpu_kick(cpu);
-        return;
-    }
-
-    if (use_icount) {
-        cpu->icount_decr.u16.high = 0xffff;
-        if (!cpu_can_do_io(cpu)
-            && (mask & ~old_mask) != 0) {
-            cpu_abort(cpu, "Raised interrupt while not in I/O function");
-        }
-    } else {
-        cpu->tcg_exit_req = 1;
-    }
-}
-
-CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
-
 /* in deterministic execution mode, instructions doing device I/Os
    must be at the end of the TB */
 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
@@ -1513,7 +1589,7 @@
     TranslationBlock *tb;
     uint32_t n, cflags;
     target_ulong pc, cs_base;
-    uint64_t flags;
+    uint32_t flags;
 
     tb = tb_find_pc(retaddr);
     if (!tb) {
@@ -1533,7 +1609,7 @@
        branch.  */
 #if defined(TARGET_MIPS)
     if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
-        env->active_tc.PC -= 4;
+        env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
         cpu->icount_decr.u16.low++;
         env->hflags &= ~MIPS_HFLAG_BMASK;
     }
@@ -1555,6 +1631,14 @@
     cs_base = tb->cs_base;
     flags = tb->flags;
     tb_phys_invalidate(tb, -1);
+    if (tb->cflags & CF_NOCACHE) {
+        if (tb->orig_tb) {
+            /* Invalidate original TB if this TB was generated in
+             * cpu_exec_nocache() */
+            tb_phys_invalidate(tb->orig_tb, -1);
+        }
+        tb_free(tb);
+    }
     /* FIXME: In theory this could raise an exception.  In practice
        we have already translated the block once so it's probably ok.  */
     tb_gen_code(cpu, pc, cs_base, flags, cflags);
@@ -1563,7 +1647,7 @@
        repeating the fault, which is horribly inefficient.
        Better would be to execute just this insn uncached, or generate a
        second new TB.  */
-    cpu_resume_from_signal(cpu, NULL);
+    cpu_loop_exit_noexc(cpu);
 }
 
 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
@@ -1581,11 +1665,50 @@
            TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
 }
 
+static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf,
+                                 struct qht_stats hst)
+{
+    uint32_t hgram_opts;
+    size_t hgram_bins;
+    char *hgram;
+
+    if (!hst.head_buckets) {
+        return;
+    }
+    cpu_fprintf(f, "TB hash buckets     %zu/%zu (%0.2f%% head buckets used)\n",
+                hst.used_head_buckets, hst.head_buckets,
+                (double)hst.used_head_buckets / hst.head_buckets * 100);
+
+    hgram_opts =  QDIST_PR_BORDER | QDIST_PR_LABELS;
+    hgram_opts |= QDIST_PR_100X   | QDIST_PR_PERCENT;
+    if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
+        hgram_opts |= QDIST_PR_NODECIMAL;
+    }
+    hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
+    cpu_fprintf(f, "TB hash occupancy   %0.2f%% avg chain occ. Histogram: %s\n",
+                qdist_avg(&hst.occupancy) * 100, hgram);
+    g_free(hgram);
+
+    hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
+    hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
+    if (hgram_bins > 10) {
+        hgram_bins = 10;
+    } else {
+        hgram_bins = 0;
+        hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
+    }
+    hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
+    cpu_fprintf(f, "TB hash avg chain   %0.3f buckets. Histogram: %s\n",
+                qdist_avg(&hst.chain), hgram);
+    g_free(hgram);
+}
+
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
 {
     int i, target_code_size, max_target_code_size;
     int direct_jmp_count, direct_jmp2_count, cross_page;
     TranslationBlock *tb;
+    struct qht_stats hst;
 
     target_code_size = 0;
     max_target_code_size = 0;
@@ -1601,9 +1724,9 @@
         if (tb->page_addr[1] != -1) {
             cross_page++;
         }
-        if (tb->tb_next_offset[0] != 0xffff) {
+        if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
             direct_jmp_count++;
-            if (tb->tb_next_offset[1] != 0xffff) {
+            if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
                 direct_jmp2_count++;
             }
         }
@@ -1612,7 +1735,7 @@
     cpu_fprintf(f, "Translation buffer state:\n");
     cpu_fprintf(f, "gen code size       %td/%zd\n",
                 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
-                tcg_ctx.code_gen_buffer_max_size);
+                tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
     cpu_fprintf(f, "TB count            %d/%d\n",
             tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
     cpu_fprintf(f, "TB avg target size  %d max=%d bytes\n",
@@ -1636,6 +1759,11 @@
                 direct_jmp2_count,
                 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
                         tcg_ctx.tb_ctx.nb_tbs : 0);
+
+    qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst);
+    print_qht_statistics(f, cpu_fprintf, hst);
+    qht_statistics_destroy(&hst);
+
     cpu_fprintf(f, "\nStatistics:\n");
     cpu_fprintf(f, "TB flush count      %d\n", tcg_ctx.tb_ctx.tb_flush_count);
     cpu_fprintf(f, "TB invalidate count %d\n",
@@ -1644,6 +1772,11 @@
     tcg_dump_info(f, cpu_fprintf);
 }
 
+void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
+{
+    tcg_dump_op_count(f, cpu_fprintf);
+}
+
 #else /* CONFIG_USER_ONLY */
 
 void cpu_interrupt(CPUState *cpu, int mask)
@@ -1807,7 +1940,7 @@
         if (!(p->flags & PAGE_WRITE) &&
             (flags & PAGE_WRITE) &&
             p->first_tb) {
-            tb_invalidate_phys_page(addr, 0, NULL, false);
+            tb_invalidate_phys_page(addr, 0);
         }
         p->flags = flags;
     }
@@ -1859,7 +1992,7 @@
             /* unprotect the page if it was put read-only because it
                contains translated code */
             if (!(p->flags & PAGE_WRITE)) {
-                if (!page_unprotect(addr, 0, NULL)) {
+                if (!page_unprotect(addr, 0)) {
                     return -1;
                 }
             }
@@ -1869,10 +2002,15 @@
 }
 
 /* called from signal handler: invalidate the code and unprotect the
-   page. Return TRUE if the fault was successfully handled. */
-int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
+ * page. Return 0 if the fault was not handled, 1 if it was handled,
+ * and 2 if it was handled but the caller must cause the TB to be
+ * immediately exited. (We can only return 2 if the 'pc' argument is
+ * non-zero.)
+ */
+int page_unprotect(target_ulong address, uintptr_t pc)
 {
     unsigned int prot;
+    bool current_tb_invalidated;
     PageDesc *p;
     target_ulong host_start, host_end, addr;
 
@@ -1894,6 +2032,7 @@
         host_end = host_start + qemu_host_page_size;
 
         prot = 0;
+        current_tb_invalidated = false;
         for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
             p = page_find(addr >> TARGET_PAGE_BITS);
             p->flags |= PAGE_WRITE;
@@ -1901,7 +2040,7 @@
 
             /* and since the content will be modified, we must invalidate
                the corresponding translated code. */
-            tb_invalidate_phys_page(addr, pc, puc, true);
+            current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
 #ifdef DEBUG_TB_CHECK
             tb_invalidate_check(addr);
 #endif
@@ -1910,7 +2049,8 @@
                  prot & PAGE_BITS);
 
         mmap_unlock();
-        return 1;
+        /* If current TB was invalidated return to main loop */
+        return current_tb_invalidated ? 2 : 1;
     }
     mmap_unlock();
     return 0;
diff --git a/translate-all.h b/translate-all.h
index 02832b2..ba8e4d6 100644
--- a/translate-all.h
+++ b/translate-all.h
@@ -19,9 +19,18 @@
 #ifndef TRANSLATE_ALL_H
 #define TRANSLATE_ALL_H
 
+#include "exec/exec-all.h"
+
+
 /* translate-all.c */
 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len);
-void cpu_unlink_tb(CPUState *cpu);
+void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
+                                   int is_cpu_write_access);
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end);
 void tb_check_watchpoint(CPUState *cpu);
 
+#ifdef CONFIG_USER_ONLY
+int page_unprotect(target_ulong address, uintptr_t pc);
+#endif
+
 #endif /* TRANSLATE_ALL_H */
diff --git a/translate-common.c b/translate-common.c
new file mode 100644
index 0000000..5e989cd
--- /dev/null
+++ b/translate-common.c
@@ -0,0 +1,58 @@
+/*
+ *  Host code generation common components
+ *
+ *  Copyright (c) 2015 Peter Crosthwaite <crosthwaite.peter@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qom/cpu.h"
+#include "sysemu/cpus.h"
+
+uintptr_t qemu_real_host_page_size;
+intptr_t qemu_real_host_page_mask;
+
+#ifndef CONFIG_USER_ONLY
+/* mask must never be zero, except for A20 change call */
+static void tcg_handle_interrupt(CPUState *cpu, int mask)
+{
+    int old_mask;
+
+    old_mask = cpu->interrupt_request;
+    cpu->interrupt_request |= mask;
+
+    /*
+     * If called from iothread context, wake the target cpu in
+     * case its halted.
+     */
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
+        return;
+    }
+
+    if (use_icount) {
+        cpu->icount_decr.u16.high = 0xffff;
+        if (!cpu->can_do_io
+            && (mask & ~old_mask) != 0) {
+            cpu_abort(cpu, "Raised interrupt while not in I/O function");
+        }
+    } else {
+        cpu->tcg_exit_req = 1;
+    }
+}
+
+CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
+#endif
diff --git a/ui/Makefile.objs b/ui/Makefile.objs
index 8ba3d05..dc936f1 100644
--- a/ui/Makefile.objs
+++ b/ui/Makefile.objs
@@ -1,14 +1,15 @@
-vnc-obj-y += vnc.o d3des.o
+vnc-obj-y += vnc.o
 vnc-obj-y += vnc-enc-zlib.o vnc-enc-hextile.o
 vnc-obj-y += vnc-enc-tight.o vnc-palette.o
 vnc-obj-y += vnc-enc-zrle.o
-vnc-obj-$(CONFIG_VNC_TLS) += vnc-tls.o vnc-auth-vencrypt.o
+vnc-obj-y += vnc-auth-vencrypt.o
 vnc-obj-$(CONFIG_VNC_SASL) += vnc-auth-sasl.o
-vnc-obj-$(CONFIG_VNC_WS) += vnc-ws.o
+vnc-obj-y += vnc-ws.o
 vnc-obj-y += vnc-jobs.o
 
 common-obj-y += keymaps.o console.o cursor.o qemu-pixman.o
 common-obj-y += input.o input-keymap.o input-legacy.o
+common-obj-$(CONFIG_LINUX) += input-linux.o
 common-obj-$(CONFIG_SPICE) += spice-core.o spice-input.o spice-display.o
 common-obj-$(CONFIG_SDL) += sdl.mo x_keymap.o
 common-obj-$(CONFIG_COCOA) += cocoa.o
@@ -16,10 +17,37 @@
 common-obj-$(CONFIG_VNC) += $(vnc-obj-y)
 common-obj-$(CONFIG_GTK) += gtk.o x_keymap.o
 
-common-obj-$(CONFIG_ANDROID_PIPE) += android-opengles.o
-common-obj-$(CONFIG_ANDROID_PIPE) += android-gpu-frame-bridge.o
-
-sdl.mo-objs := sdl.o sdl_zoom.o sdl2.o
+ifeq ($(CONFIG_SDLABI),1.2)
+sdl.mo-objs := sdl.o sdl_zoom.o
+endif
+ifeq ($(CONFIG_SDLABI),2.0)
+sdl.mo-objs := sdl2.o sdl2-input.o sdl2-2d.o
+ifeq ($(CONFIG_OPENGL),y)
+sdl.mo-objs += sdl2-gl.o
+endif
+endif
 sdl.mo-cflags := $(SDL_CFLAGS)
 
+ifeq ($(CONFIG_OPENGL),y)
+common-obj-y += shader.o
+common-obj-y += console-gl.o
+common-obj-y += egl-helpers.o
+common-obj-y += egl-context.o
+ifeq ($(CONFIG_GTK_GL),y)
+common-obj-$(CONFIG_GTK) += gtk-gl-area.o
+else
+common-obj-$(CONFIG_GTK) += gtk-egl.o
+endif
+endif
+
 gtk.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS)
+gtk-egl.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS) $(OPENGL_CFLAGS)
+gtk-gl-area.o-cflags := $(GTK_CFLAGS) $(VTE_CFLAGS) $(OPENGL_CFLAGS)
+shader.o-cflags += $(OPENGL_CFLAGS)
+console-gl.o-cflags += $(OPENGL_CFLAGS)
+egl-helpers.o-cflags += $(OPENGL_CFLAGS)
+
+gtk-egl.o-libs += $(OPENGL_LIBS)
+shader.o-libs += $(OPENGL_LIBS)
+console-gl.o-libs += $(OPENGL_LIBS)
+egl-helpers.o-libs += $(OPENGL_LIBS)
diff --git a/ui/cocoa.m b/ui/cocoa.m
index d37c29b..36c6bf0 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+
 #import <Cocoa/Cocoa.h>
 #include <crt_externs.h>
 
@@ -29,16 +31,19 @@
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/sysemu.h"
+#include "qmp-commands.h"
+#include "sysemu/blockdev.h"
+#include <Carbon/Carbon.h>
 
-#ifndef MAC_OS_X_VERSION_10_4
-#define MAC_OS_X_VERSION_10_4 1040
-#endif
 #ifndef MAC_OS_X_VERSION_10_5
 #define MAC_OS_X_VERSION_10_5 1050
 #endif
 #ifndef MAC_OS_X_VERSION_10_6
 #define MAC_OS_X_VERSION_10_6 1060
 #endif
+#ifndef MAC_OS_X_VERSION_10_10
+#define MAC_OS_X_VERSION_10_10 101000
+#endif
 
 
 //#define DEBUG
@@ -64,182 +69,163 @@
 
 int gArgc;
 char **gArgv;
+bool stretch_video;
+NSTextField *pauseLabel;
+NSArray * supportedImageFileTypes;
 
-// keymap conversion
-int keymap[] =
-{
-//  SdlI    macI    macH    SdlH    104xtH  104xtC  sdl
-    30, //  0       0x00    0x1e            A       QZ_a
-    31, //  1       0x01    0x1f            S       QZ_s
-    32, //  2       0x02    0x20            D       QZ_d
-    33, //  3       0x03    0x21            F       QZ_f
-    35, //  4       0x04    0x23            H       QZ_h
-    34, //  5       0x05    0x22            G       QZ_g
-    44, //  6       0x06    0x2c            Z       QZ_z
-    45, //  7       0x07    0x2d            X       QZ_x
-    46, //  8       0x08    0x2e            C       QZ_c
-    47, //  9       0x09    0x2f            V       QZ_v
-    0,  //  10      0x0A    Undefined
-    48, //  11      0x0B    0x30            B       QZ_b
-    16, //  12      0x0C    0x10            Q       QZ_q
-    17, //  13      0x0D    0x11            W       QZ_w
-    18, //  14      0x0E    0x12            E       QZ_e
-    19, //  15      0x0F    0x13            R       QZ_r
-    21, //  16      0x10    0x15            Y       QZ_y
-    20, //  17      0x11    0x14            T       QZ_t
-    2,  //  18      0x12    0x02            1       QZ_1
-    3,  //  19      0x13    0x03            2       QZ_2
-    4,  //  20      0x14    0x04            3       QZ_3
-    5,  //  21      0x15    0x05            4       QZ_4
-    7,  //  22      0x16    0x07            6       QZ_6
-    6,  //  23      0x17    0x06            5       QZ_5
-    13, //  24      0x18    0x0d            =       QZ_EQUALS
-    10, //  25      0x19    0x0a            9       QZ_9
-    8,  //  26      0x1A    0x08            7       QZ_7
-    12, //  27      0x1B    0x0c            -       QZ_MINUS
-    9,  //  28      0x1C    0x09            8       QZ_8
-    11, //  29      0x1D    0x0b            0       QZ_0
-    27, //  30      0x1E    0x1b            ]       QZ_RIGHTBRACKET
-    24, //  31      0x1F    0x18            O       QZ_o
-    22, //  32      0x20    0x16            U       QZ_u
-    26, //  33      0x21    0x1a            [       QZ_LEFTBRACKET
-    23, //  34      0x22    0x17            I       QZ_i
-    25, //  35      0x23    0x19            P       QZ_p
-    28, //  36      0x24    0x1c            ENTER   QZ_RETURN
-    38, //  37      0x25    0x26            L       QZ_l
-    36, //  38      0x26    0x24            J       QZ_j
-    40, //  39      0x27    0x28            '       QZ_QUOTE
-    37, //  40      0x28    0x25            K       QZ_k
-    39, //  41      0x29    0x27            ;       QZ_SEMICOLON
-    43, //  42      0x2A    0x2b            \       QZ_BACKSLASH
-    51, //  43      0x2B    0x33            ,       QZ_COMMA
-    53, //  44      0x2C    0x35            /       QZ_SLASH
-    49, //  45      0x2D    0x31            N       QZ_n
-    50, //  46      0x2E    0x32            M       QZ_m
-    52, //  47      0x2F    0x34            .       QZ_PERIOD
-    15, //  48      0x30    0x0f            TAB     QZ_TAB
-    57, //  49      0x31    0x39            SPACE   QZ_SPACE
-    41, //  50      0x32    0x29            `       QZ_BACKQUOTE
-    14, //  51      0x33    0x0e            BKSP    QZ_BACKSPACE
-    0,  //  52      0x34    Undefined
-    1,  //  53      0x35    0x01            ESC     QZ_ESCAPE
-    220, // 54      0x36    0xdc    E0,5C   R GUI   QZ_RMETA
-    219, // 55      0x37    0xdb    E0,5B   L GUI   QZ_LMETA
-    42, //  56      0x38    0x2a            L SHFT  QZ_LSHIFT
-    58, //  57      0x39    0x3a            CAPS    QZ_CAPSLOCK
-    56, //  58      0x3A    0x38            L ALT   QZ_LALT
-    29, //  59      0x3B    0x1d            L CTRL  QZ_LCTRL
-    54, //  60      0x3C    0x36            R SHFT  QZ_RSHIFT
-    184,//  61      0x3D    0xb8    E0,38   R ALT   QZ_RALT
-    157,//  62      0x3E    0x9d    E0,1D   R CTRL  QZ_RCTRL
-    0,  //  63      0x3F    Undefined
-    0,  //  64      0x40    Undefined
-    0,  //  65      0x41    Undefined
-    0,  //  66      0x42    Undefined
-    55, //  67      0x43    0x37            KP *    QZ_KP_MULTIPLY
-    0,  //  68      0x44    Undefined
-    78, //  69      0x45    0x4e            KP +    QZ_KP_PLUS
-    0,  //  70      0x46    Undefined
-    69, //  71      0x47    0x45            NUM     QZ_NUMLOCK
-    0,  //  72      0x48    Undefined
-    0,  //  73      0x49    Undefined
-    0,  //  74      0x4A    Undefined
-    181,//  75      0x4B    0xb5    E0,35   KP /    QZ_KP_DIVIDE
-    152,//  76      0x4C    0x9c    E0,1C   KP EN   QZ_KP_ENTER
-    0,  //  77      0x4D    undefined
-    74, //  78      0x4E    0x4a            KP -    QZ_KP_MINUS
-    0,  //  79      0x4F    Undefined
-    0,  //  80      0x50    Undefined
-    0,  //  81      0x51                            QZ_KP_EQUALS
-    82, //  82      0x52    0x52            KP 0    QZ_KP0
-    79, //  83      0x53    0x4f            KP 1    QZ_KP1
-    80, //  84      0x54    0x50            KP 2    QZ_KP2
-    81, //  85      0x55    0x51            KP 3    QZ_KP3
-    75, //  86      0x56    0x4b            KP 4    QZ_KP4
-    76, //  87      0x57    0x4c            KP 5    QZ_KP5
-    77, //  88      0x58    0x4d            KP 6    QZ_KP6
-    71, //  89      0x59    0x47            KP 7    QZ_KP7
-    0,  //  90      0x5A    Undefined
-    72, //  91      0x5B    0x48            KP 8    QZ_KP8
-    73, //  92      0x5C    0x49            KP 9    QZ_KP9
-    0,  //  93      0x5D    Undefined
-    0,  //  94      0x5E    Undefined
-    0,  //  95      0x5F    Undefined
-    63, //  96      0x60    0x3f            F5      QZ_F5
-    64, //  97      0x61    0x40            F6      QZ_F6
-    65, //  98      0x62    0x41            F7      QZ_F7
-    61, //  99      0x63    0x3d            F3      QZ_F3
-    66, //  100     0x64    0x42            F8      QZ_F8
-    67, //  101     0x65    0x43            F9      QZ_F9
-    0,  //  102     0x66    Undefined
-    87, //  103     0x67    0x57            F11     QZ_F11
-    0,  //  104     0x68    Undefined
-    183,//  105     0x69    0xb7                    QZ_PRINT
-    0,  //  106     0x6A    Undefined
-    70, //  107     0x6B    0x46            SCROLL  QZ_SCROLLOCK
-    0,  //  108     0x6C    Undefined
-    68, //  109     0x6D    0x44            F10     QZ_F10
-    0,  //  110     0x6E    Undefined
-    88, //  111     0x6F    0x58            F12     QZ_F12
-    0,  //  112     0x70    Undefined
-    110,//  113     0x71    0x0                     QZ_PAUSE
-    210,//  114     0x72    0xd2    E0,52   INSERT  QZ_INSERT
-    199,//  115     0x73    0xc7    E0,47   HOME    QZ_HOME
-    201,//  116     0x74    0xc9    E0,49   PG UP   QZ_PAGEUP
-    211,//  117     0x75    0xd3    E0,53   DELETE  QZ_DELETE
-    62, //  118     0x76    0x3e            F4      QZ_F4
-    207,//  119     0x77    0xcf    E0,4f   END     QZ_END
-    60, //  120     0x78    0x3c            F2      QZ_F2
-    209,//  121     0x79    0xd1    E0,51   PG DN   QZ_PAGEDOWN
-    59, //  122     0x7A    0x3b            F1      QZ_F1
-    203,//  123     0x7B    0xcb    e0,4B   L ARROW QZ_LEFT
-    205,//  124     0x7C    0xcd    e0,4D   R ARROW QZ_RIGHT
-    208,//  125     0x7D    0xd0    E0,50   D ARROW QZ_DOWN
-    200,//  126     0x7E    0xc8    E0,48   U ARROW QZ_UP
-/* completed according to http://www.libsdl.org/cgi/cvsweb.cgi/SDL12/src/video/quartz/SDL_QuartzKeys.h?rev=1.6&content-type=text/x-cvsweb-markup */
+// Mac to QKeyCode conversion
+const int mac_to_qkeycode_map[] = {
+    [kVK_ANSI_A] = Q_KEY_CODE_A,
+    [kVK_ANSI_B] = Q_KEY_CODE_B,
+    [kVK_ANSI_C] = Q_KEY_CODE_C,
+    [kVK_ANSI_D] = Q_KEY_CODE_D,
+    [kVK_ANSI_E] = Q_KEY_CODE_E,
+    [kVK_ANSI_F] = Q_KEY_CODE_F,
+    [kVK_ANSI_G] = Q_KEY_CODE_G,
+    [kVK_ANSI_H] = Q_KEY_CODE_H,
+    [kVK_ANSI_I] = Q_KEY_CODE_I,
+    [kVK_ANSI_J] = Q_KEY_CODE_J,
+    [kVK_ANSI_K] = Q_KEY_CODE_K,
+    [kVK_ANSI_L] = Q_KEY_CODE_L,
+    [kVK_ANSI_M] = Q_KEY_CODE_M,
+    [kVK_ANSI_N] = Q_KEY_CODE_N,
+    [kVK_ANSI_O] = Q_KEY_CODE_O,
+    [kVK_ANSI_P] = Q_KEY_CODE_P,
+    [kVK_ANSI_Q] = Q_KEY_CODE_Q,
+    [kVK_ANSI_R] = Q_KEY_CODE_R,
+    [kVK_ANSI_S] = Q_KEY_CODE_S,
+    [kVK_ANSI_T] = Q_KEY_CODE_T,
+    [kVK_ANSI_U] = Q_KEY_CODE_U,
+    [kVK_ANSI_V] = Q_KEY_CODE_V,
+    [kVK_ANSI_W] = Q_KEY_CODE_W,
+    [kVK_ANSI_X] = Q_KEY_CODE_X,
+    [kVK_ANSI_Y] = Q_KEY_CODE_Y,
+    [kVK_ANSI_Z] = Q_KEY_CODE_Z,
 
-/* Additional 104 Key XP-Keyboard Scancodes from http://www.computer-engineering.org/ps2keyboard/scancodes1.html */
-/*
-    221 //          0xdd            e0,5d   APPS
-        //              E0,2A,E0,37         PRNT SCRN
-        //              E1,1D,45,E1,9D,C5   PAUSE
-    83  //          0x53    0x53            KP .
-// ACPI Scan Codes
-    222 //          0xde            E0, 5E  Power
-    223 //          0xdf            E0, 5F  Sleep
-    227 //          0xe3            E0, 63  Wake
-// Windows Multimedia Scan Codes
-    153 //          0x99            E0, 19  Next Track
-    144 //          0x90            E0, 10  Previous Track
-    164 //          0xa4            E0, 24  Stop
-    162 //          0xa2            E0, 22  Play/Pause
-    160 //          0xa0            E0, 20  Mute
-    176 //          0xb0            E0, 30  Volume Up
-    174 //          0xae            E0, 2E  Volume Down
-    237 //          0xed            E0, 6D  Media Select
-    236 //          0xec            E0, 6C  E-Mail
-    161 //          0xa1            E0, 21  Calculator
-    235 //          0xeb            E0, 6B  My Computer
-    229 //          0xe5            E0, 65  WWW Search
-    178 //          0xb2            E0, 32  WWW Home
-    234 //          0xea            E0, 6A  WWW Back
-    233 //          0xe9            E0, 69  WWW Forward
-    232 //          0xe8            E0, 68  WWW Stop
-    231 //          0xe7            E0, 67  WWW Refresh
-    230 //          0xe6            E0, 66  WWW Favorites
-*/
+    [kVK_ANSI_0] = Q_KEY_CODE_0,
+    [kVK_ANSI_1] = Q_KEY_CODE_1,
+    [kVK_ANSI_2] = Q_KEY_CODE_2,
+    [kVK_ANSI_3] = Q_KEY_CODE_3,
+    [kVK_ANSI_4] = Q_KEY_CODE_4,
+    [kVK_ANSI_5] = Q_KEY_CODE_5,
+    [kVK_ANSI_6] = Q_KEY_CODE_6,
+    [kVK_ANSI_7] = Q_KEY_CODE_7,
+    [kVK_ANSI_8] = Q_KEY_CODE_8,
+    [kVK_ANSI_9] = Q_KEY_CODE_9,
+
+    [kVK_ANSI_Grave] = Q_KEY_CODE_GRAVE_ACCENT,
+    [kVK_ANSI_Minus] = Q_KEY_CODE_MINUS,
+    [kVK_ANSI_Equal] = Q_KEY_CODE_EQUAL,
+    [kVK_Delete] = Q_KEY_CODE_BACKSPACE,
+    [kVK_CapsLock] = Q_KEY_CODE_CAPS_LOCK,
+    [kVK_Tab] = Q_KEY_CODE_TAB,
+    [kVK_Return] = Q_KEY_CODE_RET,
+    [kVK_ANSI_LeftBracket] = Q_KEY_CODE_BRACKET_LEFT,
+    [kVK_ANSI_RightBracket] = Q_KEY_CODE_BRACKET_RIGHT,
+    [kVK_ANSI_Backslash] = Q_KEY_CODE_BACKSLASH,
+    [kVK_ANSI_Semicolon] = Q_KEY_CODE_SEMICOLON,
+    [kVK_ANSI_Quote] = Q_KEY_CODE_APOSTROPHE,
+    [kVK_ANSI_Comma] = Q_KEY_CODE_COMMA,
+    [kVK_ANSI_Period] = Q_KEY_CODE_DOT,
+    [kVK_ANSI_Slash] = Q_KEY_CODE_SLASH,
+    [kVK_Shift] = Q_KEY_CODE_SHIFT,
+    [kVK_RightShift] = Q_KEY_CODE_SHIFT_R,
+    [kVK_Control] = Q_KEY_CODE_CTRL,
+    [kVK_RightControl] = Q_KEY_CODE_CTRL_R,
+    [kVK_Option] = Q_KEY_CODE_ALT,
+    [kVK_RightOption] = Q_KEY_CODE_ALT_R,
+    [kVK_Command] = Q_KEY_CODE_META_L,
+    [0x36] = Q_KEY_CODE_META_R, /* There is no kVK_RightCommand */
+    [kVK_Space] = Q_KEY_CODE_SPC,
+
+    [kVK_ANSI_Keypad0] = Q_KEY_CODE_KP_0,
+    [kVK_ANSI_Keypad1] = Q_KEY_CODE_KP_1,
+    [kVK_ANSI_Keypad2] = Q_KEY_CODE_KP_2,
+    [kVK_ANSI_Keypad3] = Q_KEY_CODE_KP_3,
+    [kVK_ANSI_Keypad4] = Q_KEY_CODE_KP_4,
+    [kVK_ANSI_Keypad5] = Q_KEY_CODE_KP_5,
+    [kVK_ANSI_Keypad6] = Q_KEY_CODE_KP_6,
+    [kVK_ANSI_Keypad7] = Q_KEY_CODE_KP_7,
+    [kVK_ANSI_Keypad8] = Q_KEY_CODE_KP_8,
+    [kVK_ANSI_Keypad9] = Q_KEY_CODE_KP_9,
+    [kVK_ANSI_KeypadDecimal] = Q_KEY_CODE_KP_DECIMAL,
+    [kVK_ANSI_KeypadEnter] = Q_KEY_CODE_KP_ENTER,
+    [kVK_ANSI_KeypadPlus] = Q_KEY_CODE_KP_ADD,
+    [kVK_ANSI_KeypadMinus] = Q_KEY_CODE_KP_SUBTRACT,
+    [kVK_ANSI_KeypadMultiply] = Q_KEY_CODE_KP_MULTIPLY,
+    [kVK_ANSI_KeypadDivide] = Q_KEY_CODE_KP_DIVIDE,
+    [kVK_ANSI_KeypadEquals] = Q_KEY_CODE_KP_EQUALS,
+    [kVK_ANSI_KeypadClear] = Q_KEY_CODE_NUM_LOCK,
+
+    [kVK_UpArrow] = Q_KEY_CODE_UP,
+    [kVK_DownArrow] = Q_KEY_CODE_DOWN,
+    [kVK_LeftArrow] = Q_KEY_CODE_LEFT,
+    [kVK_RightArrow] = Q_KEY_CODE_RIGHT,
+
+    [kVK_Help] = Q_KEY_CODE_INSERT,
+    [kVK_Home] = Q_KEY_CODE_HOME,
+    [kVK_PageUp] = Q_KEY_CODE_PGUP,
+    [kVK_PageDown] = Q_KEY_CODE_PGDN,
+    [kVK_End] = Q_KEY_CODE_END,
+    [kVK_ForwardDelete] = Q_KEY_CODE_DELETE,
+
+    [kVK_Escape] = Q_KEY_CODE_ESC,
+
+    /* The Power key can't be used directly because the operating system uses
+     * it. This key can be emulated by using it in place of another key such as
+     * F1. Don't forget to disable the real key binding.
+     */
+    /* [kVK_F1] = Q_KEY_CODE_POWER, */
+
+    [kVK_F1] = Q_KEY_CODE_F1,
+    [kVK_F2] = Q_KEY_CODE_F2,
+    [kVK_F3] = Q_KEY_CODE_F3,
+    [kVK_F4] = Q_KEY_CODE_F4,
+    [kVK_F5] = Q_KEY_CODE_F5,
+    [kVK_F6] = Q_KEY_CODE_F6,
+    [kVK_F7] = Q_KEY_CODE_F7,
+    [kVK_F8] = Q_KEY_CODE_F8,
+    [kVK_F9] = Q_KEY_CODE_F9,
+    [kVK_F10] = Q_KEY_CODE_F10,
+    [kVK_F11] = Q_KEY_CODE_F11,
+    [kVK_F12] = Q_KEY_CODE_F12,
+    [kVK_F13] = Q_KEY_CODE_PRINT,
+    [kVK_F14] = Q_KEY_CODE_SCROLL_LOCK,
+    [kVK_F15] = Q_KEY_CODE_PAUSE,
+
+    /*
+     * The eject and volume keys can't be used here because they are handled at
+     * a lower level than what an Application can see.
+     */
 };
 
 static int cocoa_keycode_to_qemu(int keycode)
 {
-    if (ARRAY_SIZE(keymap) <= keycode) {
+    if (ARRAY_SIZE(mac_to_qkeycode_map) <= keycode) {
         fprintf(stderr, "(cocoa) warning unknown keycode 0x%x\n", keycode);
         return 0;
     }
-    return keymap[keycode];
+    return mac_to_qkeycode_map[keycode];
 }
 
+/* Displays an alert dialog box with the specified message */
+static void QEMU_Alert(NSString *message)
+{
+    NSAlert *alert;
+    alert = [NSAlert new];
+    [alert setMessageText: message];
+    [alert runModal];
+}
 
+/* Handles any errors that happen with a device transaction */
+static void handleAnyDeviceErrors(Error * err)
+{
+    if (err) {
+        QEMU_Alert([NSString stringWithCString: error_get_pretty(err)
+                                      encoding: NSASCIIStringEncoding]);
+        error_free(err);
+    }
+}
 
 /*
  ------------------------------------------------------
@@ -282,6 +268,7 @@
 - (float) cdx;
 - (float) cdy;
 - (QEMUScreen) gscreen;
+- (void) raiseAllKeys;
 @end
 
 QemuCocoaView *cocoaView;
@@ -373,40 +360,26 @@
             0, //interpolate
             kCGRenderingIntentDefault //intent
         );
-// test if host supports "CGImageCreateWithImageInRect" at compile time
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4)
-        if (CGImageCreateWithImageInRect == NULL) { // test if "CGImageCreateWithImageInRect" is supported on host at runtime
-#endif
-            // compatibility drawing code (draws everything) (OS X < 10.4)
-            CGContextDrawImage (viewContextRef, CGRectMake(0, 0, [self bounds].size.width, [self bounds].size.height), imageRef);
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4)
-        } else {
-            // selective drawing code (draws only dirty rectangles) (OS X >= 10.4)
-            const NSRect *rectList;
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5)
-            NSInteger rectCount;
-#else
-            int rectCount;
-#endif
-            int i;
-            CGImageRef clipImageRef;
-            CGRect clipRect;
+        // selective drawing code (draws only dirty rectangles) (OS X >= 10.4)
+        const NSRect *rectList;
+        NSInteger rectCount;
+        int i;
+        CGImageRef clipImageRef;
+        CGRect clipRect;
 
-            [self getRectsBeingDrawn:&rectList count:&rectCount];
-            for (i = 0; i < rectCount; i++) {
-                clipRect.origin.x = rectList[i].origin.x / cdx;
-                clipRect.origin.y = (float)screen.height - (rectList[i].origin.y + rectList[i].size.height) / cdy;
-                clipRect.size.width = rectList[i].size.width / cdx;
-                clipRect.size.height = rectList[i].size.height / cdy;
-                clipImageRef = CGImageCreateWithImageInRect(
-                    imageRef,
-                    clipRect
-                );
-                CGContextDrawImage (viewContextRef, cgrect(rectList[i]), clipImageRef);
-                CGImageRelease (clipImageRef);
-            }
+        [self getRectsBeingDrawn:&rectList count:&rectCount];
+        for (i = 0; i < rectCount; i++) {
+            clipRect.origin.x = rectList[i].origin.x / cdx;
+            clipRect.origin.y = (float)screen.height - (rectList[i].origin.y + rectList[i].size.height) / cdy;
+            clipRect.size.width = rectList[i].size.width / cdx;
+            clipRect.size.height = rectList[i].size.height / cdy;
+            clipImageRef = CGImageCreateWithImageInRect(
+                                                        imageRef,
+                                                        clipRect
+                                                        );
+            CGContextDrawImage (viewContextRef, cgrect(rectList[i]), clipImageRef);
+            CGImageRelease (clipImageRef);
         }
-#endif
         CGImageRelease (imageRef);
     }
 }
@@ -418,6 +391,18 @@
     if (isFullscreen) {
         cdx = [[NSScreen mainScreen] frame].size.width / (float)screen.width;
         cdy = [[NSScreen mainScreen] frame].size.height / (float)screen.height;
+
+        /* stretches video, but keeps same aspect ratio */
+        if (stretch_video == true) {
+            /* use smallest stretch value - prevents clipping on sides */
+            if (MIN(cdx, cdy) == cdx) {
+                cdy = cdx;
+            } else {
+                cdx = cdy;
+            }
+        } else {  /* No stretching */
+            cdx = cdy = 1;
+        }
         cw = screen.width * cdx;
         ch = screen.height * cdy;
         cx = ([[NSScreen mainScreen] frame].size.width - cw) / 2.0;
@@ -487,43 +472,37 @@
         isFullscreen = FALSE;
         [self ungrabMouse];
         [self setContentDimensions];
-// test if host supports "exitFullScreenModeWithOptions" at compile time
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5)
         if ([NSView respondsToSelector:@selector(exitFullScreenModeWithOptions:)]) { // test if "exitFullScreenModeWithOptions" is supported on host at runtime
             [self exitFullScreenModeWithOptions:nil];
         } else {
-#endif
             [fullScreenWindow close];
             [normalWindow setContentView: self];
             [normalWindow makeKeyAndOrderFront: self];
             [NSMenu setMenuBarVisible:YES];
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5)
         }
-#endif
     } else { // switch from desktop to fullscreen
         isFullscreen = TRUE;
+        [normalWindow orderOut: nil]; /* Hide the window */
         [self grabMouse];
         [self setContentDimensions];
-// test if host supports "enterFullScreenMode:withOptions" at compile time
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5)
         if ([NSView respondsToSelector:@selector(enterFullScreenMode:withOptions:)]) { // test if "enterFullScreenMode:withOptions" is supported on host at runtime
             [self enterFullScreenMode:[NSScreen mainScreen] withOptions:[NSDictionary dictionaryWithObjectsAndKeys:
                 [NSNumber numberWithBool:NO], NSFullScreenModeAllScreens,
                 [NSDictionary dictionaryWithObjectsAndKeys:[NSNumber numberWithBool:NO], kCGDisplayModeIsStretched, nil], NSFullScreenModeSetting,
                  nil]];
         } else {
-#endif
             [NSMenu setMenuBarVisible:NO];
             fullScreenWindow = [[NSWindow alloc] initWithContentRect:[[NSScreen mainScreen] frame]
                 styleMask:NSBorderlessWindowMask
                 backing:NSBackingStoreBuffered
                 defer:NO];
+            [fullScreenWindow setAcceptsMouseMovedEvents: YES];
             [fullScreenWindow setHasShadow:NO];
-            [fullScreenWindow setContentView:self];
+            [fullScreenWindow setBackgroundColor: [NSColor blackColor]];
+            [self setFrame:NSMakeRect(cx, cy, cw, ch)];
+            [[fullScreenWindow contentView] addSubview: self];
             [fullScreenWindow makeKeyAndOrderFront:self];
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_5)
         }
-#endif
     }
 }
 
@@ -540,28 +519,31 @@
         case NSFlagsChanged:
             keycode = cocoa_keycode_to_qemu([event keyCode]);
 
-            if ((keycode == 219 || keycode == 220) && !isMouseGrabbed) {
+            if ((keycode == Q_KEY_CODE_META_L || keycode == Q_KEY_CODE_META_R)
+               && !isMouseGrabbed) {
               /* Don't pass command key changes to guest unless mouse is grabbed */
               keycode = 0;
             }
 
             if (keycode) {
-                if (keycode == 58 || keycode == 69) { // emulate caps lock and num lock keydown and keyup
-                    qemu_input_event_send_key_number(dcl->con, keycode, true);
-                    qemu_input_event_send_key_number(dcl->con, keycode, false);
+                // emulate caps lock and num lock keydown and keyup
+                if (keycode == Q_KEY_CODE_CAPS_LOCK ||
+                    keycode == Q_KEY_CODE_NUM_LOCK) {
+                    qemu_input_event_send_key_qcode(dcl->con, keycode, true);
+                    qemu_input_event_send_key_qcode(dcl->con, keycode, false);
                 } else if (qemu_console_is_graphic(NULL)) {
                     if (modifiers_state[keycode] == 0) { // keydown
-                        qemu_input_event_send_key_number(dcl->con, keycode, true);
+                        qemu_input_event_send_key_qcode(dcl->con, keycode, true);
                         modifiers_state[keycode] = 1;
                     } else { // keyup
-                        qemu_input_event_send_key_number(dcl->con, keycode, false);
+                        qemu_input_event_send_key_qcode(dcl->con, keycode, false);
                         modifiers_state[keycode] = 0;
                     }
                 }
             }
 
             // release Mouse grab when pressing ctrl+alt
-            if (!isFullscreen && ([event modifierFlags] & NSControlKeyMask) && ([event modifierFlags] & NSAlternateKeyMask)) {
+            if (([event modifierFlags] & NSControlKeyMask) && ([event modifierFlags] & NSAlternateKeyMask)) {
                 [self ungrabMouse];
             }
             break;
@@ -581,14 +563,14 @@
                 switch (keycode) {
 
                     // enable graphic console
-                    case 0x02 ... 0x0a: // '1' to '9' keys
-                        console_select(keycode - 0x02);
+                    case Q_KEY_CODE_1 ... Q_KEY_CODE_9: // '1' to '9' keys
+                        console_select(keycode - 11);
                         break;
                 }
 
             // handle keys for graphic console
             } else if (qemu_console_is_graphic(NULL)) {
-                qemu_input_event_send_key_number(dcl->con, keycode, true);
+                qemu_input_event_send_key_qcode(dcl->con, keycode, true);
 
             // handlekeys for Monitor
             } else {
@@ -636,7 +618,7 @@
             }
 
             if (qemu_console_is_graphic(NULL)) {
-                qemu_input_event_send_key_number(dcl->con, keycode, false);
+                qemu_input_event_send_key_qcode(dcl->con, keycode, false);
             }
             break;
         case NSMouseMoved:
@@ -709,8 +691,16 @@
     }
 
     if (mouse_event) {
-        if (last_buttons != buttons) {
-            static uint32_t bmap[INPUT_BUTTON_MAX] = {
+        /* Don't send button events to the guest unless we've got a
+         * mouse grab or window focus. If we have neither then this event
+         * is the user clicking on the background window to activate and
+         * bring us to the front, which will be done by the sendEvent
+         * call below. We definitely don't want to pass that click through
+         * to the guest.
+         */
+        if ((isMouseGrabbed || [[self window] isKeyWindow]) &&
+            (last_buttons != buttons)) {
+            static uint32_t bmap[INPUT_BUTTON__MAX] = {
                 [INPUT_BUTTON_LEFT]       = MOUSE_EVENT_LBUTTON,
                 [INPUT_BUTTON_MIDDLE]     = MOUSE_EVENT_MBUTTON,
                 [INPUT_BUTTON_RIGHT]      = MOUSE_EVENT_RBUTTON,
@@ -784,6 +774,24 @@
 - (float) cdx {return cdx;}
 - (float) cdy {return cdy;}
 - (QEMUScreen) gscreen {return screen;}
+
+/*
+ * Makes the target think all down keys are being released.
+ * This prevents a stuck key problem, since we will not see
+ * key up events for those keys after we have lost focus.
+ */
+- (void) raiseAllKeys
+{
+    int index;
+    const int max_index = ARRAY_SIZE(modifiers_state);
+
+   for (index = 0; index < max_index; index++) {
+       if (modifiers_state[index]) {
+           modifiers_state[index] = 0;
+           qemu_input_event_send_key_qcode(dcl->con, index, false);
+       }
+   }
+}
 @end
 
 
@@ -794,13 +802,28 @@
  ------------------------------------------------------
 */
 @interface QemuCocoaAppController : NSObject
+#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
+                                       <NSWindowDelegate, NSApplicationDelegate>
+#endif
 {
 }
 - (void)startEmulationWithArgc:(int)argc argv:(char**)argv;
-- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(int)returnCode contextInfo:(void *)contextInfo;
+- (void)doToggleFullScreen:(id)sender;
 - (void)toggleFullScreen:(id)sender;
 - (void)showQEMUDoc:(id)sender;
 - (void)showQEMUTec:(id)sender;
+- (void)zoomToFit:(id) sender;
+- (void)displayConsole:(id)sender;
+- (void)pauseQEMU:(id)sender;
+- (void)resumeQEMU:(id)sender;
+- (void)displayPause;
+- (void)removePause;
+- (void)restartQEMU:(id)sender;
+- (void)powerDownQEMU:(id)sender;
+- (void)ejectDeviceMedia:(id)sender;
+- (void)changeDeviceMedia:(id)sender;
+- (BOOL)verifyQuit;
+- (void)openDocumentation:(NSString *)filename;
 @end
 
 @implementation QemuCocoaAppController
@@ -827,12 +850,32 @@
             exit(1);
         }
         [normalWindow setAcceptsMouseMovedEvents:YES];
-        [normalWindow setTitle:[NSString stringWithFormat:@"QEMU"]];
+        [normalWindow setTitle:@"QEMU"];
         [normalWindow setContentView:cocoaView];
+#if (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_10)
         [normalWindow useOptimizedDrawing:YES];
+#endif
         [normalWindow makeKeyAndOrderFront:self];
         [normalWindow center];
+        [normalWindow setDelegate: self];
+        stretch_video = false;
 
+        /* Used for displaying pause on the screen */
+        pauseLabel = [NSTextField new];
+        [pauseLabel setBezeled:YES];
+        [pauseLabel setDrawsBackground:YES];
+        [pauseLabel setBackgroundColor: [NSColor whiteColor]];
+        [pauseLabel setEditable:NO];
+        [pauseLabel setSelectable:NO];
+        [pauseLabel setStringValue: @"Paused"];
+        [pauseLabel setFont: [NSFont fontWithName: @"Helvetica" size: 90]];
+        [pauseLabel setTextColor: [NSColor blackColor]];
+        [pauseLabel sizeToFit];
+
+        // set the supported image file types that can be opened
+        supportedImageFileTypes = [NSArray arrayWithObjects: @"img", @"iso", @"dmg",
+                                 @"qcow", @"qcow2", @"cloop", @"vmdk", @"cdr",
+                                  nil];
     }
     return self;
 }
@@ -849,31 +892,8 @@
 - (void)applicationDidFinishLaunching: (NSNotification *) note
 {
     COCOA_DEBUG("QemuCocoaAppController: applicationDidFinishLaunching\n");
-
-    // Display an open dialog box if no arguments were passed or
-    // if qemu was launched from the finder ( the Finder passes "-psn" )
-    if( gArgc <= 1 || strncmp ((char *)gArgv[1], "-psn", 4) == 0) {
-        NSOpenPanel *op = [[NSOpenPanel alloc] init];
-        [op setPrompt:@"Boot image"];
-        [op setMessage:@"Select the disk image you want to boot.\n\nHit the \"Cancel\" button to quit"];
-        NSArray *filetypes = [NSArray arrayWithObjects:@"img", @"iso", @"dmg",
-                                 @"qcow", @"qcow2", @"cloop", @"vmdk", nil];
-#if (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_6)
-        [op setAllowedFileTypes:filetypes];
-        [op beginSheetModalForWindow:normalWindow
-            completionHandler:^(NSInteger returnCode)
-            { [self openPanelDidEnd:op
-                  returnCode:returnCode contextInfo:NULL ]; } ];
-#else
-        // Compatibility code for pre-10.6, using deprecated method
-        [op beginSheetForDirectory:nil file:nil types:filetypes
-              modalForWindow:normalWindow modalDelegate:self
-              didEndSelector:@selector(openPanelDidEnd:returnCode:contextInfo:) contextInfo:NULL];
-#endif
-    } else {
-        // or launch QEMU, with the global args
-        [self startEmulationWithArgc:gArgc argv:(char **)gArgv];
-    }
+    // launch QEMU, with the global args
+    [self startEmulationWithArgc:gArgc argv:(char **)gArgv];
 }
 
 - (void)applicationWillTerminate:(NSNotification *)aNotification
@@ -889,6 +909,33 @@
     return YES;
 }
 
+- (NSApplicationTerminateReply)applicationShouldTerminate:
+                                                         (NSApplication *)sender
+{
+    COCOA_DEBUG("QemuCocoaAppController: applicationShouldTerminate\n");
+    return [self verifyQuit];
+}
+
+/* Called when the user clicks on a window's close button */
+- (BOOL)windowShouldClose:(id)sender
+{
+    COCOA_DEBUG("QemuCocoaAppController: windowShouldClose\n");
+    [NSApp terminate: sender];
+    /* If the user allows the application to quit then the call to
+     * NSApp terminate will never return. If we get here then the user
+     * cancelled the quit, so we should return NO to not permit the
+     * closing of this window.
+     */
+    return NO;
+}
+
+/* Called when QEMU goes into the background */
+- (void) applicationWillResignActive: (NSNotification *)aNotification
+{
+    COCOA_DEBUG("QemuCocoaAppController: applicationWillResignActive\n");
+    [cocoaView raiseAllKeys];
+}
+
 - (void)startEmulationWithArgc:(int)argc argv:(char**)argv
 {
     COCOA_DEBUG("QemuCocoaAppController: startEmulationWithArgc\n");
@@ -898,29 +945,15 @@
     exit(status);
 }
 
-- (void)openPanelDidEnd:(NSOpenPanel *)sheet returnCode:(int)returnCode contextInfo:(void *)contextInfo
+/* We abstract the method called by the Enter Fullscreen menu item
+ * because Mac OS 10.7 and higher disables it. This is because of the
+ * menu item's old selector's name toggleFullScreen:
+ */
+- (void) doToggleFullScreen:(id)sender
 {
-    COCOA_DEBUG("QemuCocoaAppController: openPanelDidEnd\n");
-
-    if(returnCode == NSCancelButton) {
-        exit(0);
-    } else if(returnCode == NSOKButton) {
-        char *img = (char*)[ [ [ sheet URL ] path ] cStringUsingEncoding:NSASCIIStringEncoding];
-
-        char **argv = g_new(char *, 4);
-
-        [sheet close];
-
-        argv[0] = g_strdup(gArgv[0]);
-        argv[1] = g_strdup("-hda");
-        argv[2] = g_strdup(img);
-        argv[3] = NULL;
-
-        // printf("Using argc %d argv %s -hda %s\n", 3, gArgv[0], img);
-
-        [self startEmulationWithArgc:3 argv:(char**)argv];
-    }
+    [self toggleFullScreen:(id)sender];
 }
+
 - (void)toggleFullScreen:(id)sender
 {
     COCOA_DEBUG("QemuCocoaAppController: toggleFullScreen\n");
@@ -928,23 +961,184 @@
     [cocoaView toggleFullScreen:sender];
 }
 
+/* Tries to find then open the specified filename */
+- (void) openDocumentation: (NSString *) filename
+{
+    /* Where to look for local files */
+    NSString *path_array[] = {@"../share/doc/qemu/", @"../doc/qemu/", @"../"};
+    NSString *full_file_path;
+
+    /* iterate thru the possible paths until the file is found */
+    int index;
+    for (index = 0; index < ARRAY_SIZE(path_array); index++) {
+        full_file_path = [[NSBundle mainBundle] executablePath];
+        full_file_path = [full_file_path stringByDeletingLastPathComponent];
+        full_file_path = [NSString stringWithFormat: @"%@/%@%@", full_file_path,
+                          path_array[index], filename];
+        if ([[NSWorkspace sharedWorkspace] openFile: full_file_path] == YES) {
+            return;
+        }
+    }
+
+    /* If none of the paths opened a file */
+    NSBeep();
+    QEMU_Alert(@"Failed to open file");
+}
+
 - (void)showQEMUDoc:(id)sender
 {
     COCOA_DEBUG("QemuCocoaAppController: showQEMUDoc\n");
 
-    [[NSWorkspace sharedWorkspace] openFile:[NSString stringWithFormat:@"%@/../doc/qemu/qemu-doc.html",
-        [[NSBundle mainBundle] resourcePath]] withApplication:@"Help Viewer"];
+    [self openDocumentation: @"qemu-doc.html"];
 }
 
 - (void)showQEMUTec:(id)sender
 {
     COCOA_DEBUG("QemuCocoaAppController: showQEMUTec\n");
 
-    [[NSWorkspace sharedWorkspace] openFile:[NSString stringWithFormat:@"%@/../doc/qemu/qemu-tech.html",
-        [[NSBundle mainBundle] resourcePath]] withApplication:@"Help Viewer"];
+    [self openDocumentation: @"qemu-tech.html"];
 }
-@end
 
+/* Stretches video to fit host monitor size */
+- (void)zoomToFit:(id) sender
+{
+    stretch_video = !stretch_video;
+    if (stretch_video == true) {
+        [sender setState: NSOnState];
+    } else {
+        [sender setState: NSOffState];
+    }
+}
+
+/* Displays the console on the screen */
+- (void)displayConsole:(id)sender
+{
+    console_select([sender tag]);
+}
+
+/* Pause the guest */
+- (void)pauseQEMU:(id)sender
+{
+    qmp_stop(NULL);
+    [sender setEnabled: NO];
+    [[[sender menu] itemWithTitle: @"Resume"] setEnabled: YES];
+    [self displayPause];
+}
+
+/* Resume running the guest operating system */
+- (void)resumeQEMU:(id) sender
+{
+    qmp_cont(NULL);
+    [sender setEnabled: NO];
+    [[[sender menu] itemWithTitle: @"Pause"] setEnabled: YES];
+    [self removePause];
+}
+
+/* Displays the word pause on the screen */
+- (void)displayPause
+{
+    /* Coordinates have to be calculated each time because the window can change its size */
+    int xCoord, yCoord, width, height;
+    xCoord = ([normalWindow frame].size.width - [pauseLabel frame].size.width)/2;
+    yCoord = [normalWindow frame].size.height - [pauseLabel frame].size.height - ([pauseLabel frame].size.height * .5);
+    width = [pauseLabel frame].size.width;
+    height = [pauseLabel frame].size.height;
+    [pauseLabel setFrame: NSMakeRect(xCoord, yCoord, width, height)];
+    [cocoaView addSubview: pauseLabel];
+}
+
+/* Removes the word pause from the screen */
+- (void)removePause
+{
+    [pauseLabel removeFromSuperview];
+}
+
+/* Restarts QEMU */
+- (void)restartQEMU:(id)sender
+{
+    qmp_system_reset(NULL);
+}
+
+/* Powers down QEMU */
+- (void)powerDownQEMU:(id)sender
+{
+    qmp_system_powerdown(NULL);
+}
+
+/* Ejects the media.
+ * Uses sender's tag to figure out the device to eject.
+ */
+- (void)ejectDeviceMedia:(id)sender
+{
+    NSString * drive;
+    drive = [sender representedObject];
+    if(drive == nil) {
+        NSBeep();
+        QEMU_Alert(@"Failed to find drive to eject!");
+        return;
+    }
+
+    Error *err = NULL;
+    qmp_eject([drive cStringUsingEncoding: NSASCIIStringEncoding], false, false, &err);
+    handleAnyDeviceErrors(err);
+}
+
+/* Displays a dialog box asking the user to select an image file to load.
+ * Uses sender's represented object value to figure out which drive to use.
+ */
+- (void)changeDeviceMedia:(id)sender
+{
+    /* Find the drive name */
+    NSString * drive;
+    drive = [sender representedObject];
+    if(drive == nil) {
+        NSBeep();
+        QEMU_Alert(@"Could not find drive!");
+        return;
+    }
+
+    /* Display the file open dialog */
+    NSOpenPanel * openPanel;
+    openPanel = [NSOpenPanel openPanel];
+    [openPanel setCanChooseFiles: YES];
+    [openPanel setAllowsMultipleSelection: NO];
+    [openPanel setAllowedFileTypes: supportedImageFileTypes];
+    if([openPanel runModal] == NSFileHandlingPanelOKButton) {
+        NSString * file = [[[openPanel URLs] objectAtIndex: 0] path];
+        if(file == nil) {
+            NSBeep();
+            QEMU_Alert(@"Failed to convert URL to file path!");
+            return;
+        }
+
+        Error *err = NULL;
+        qmp_blockdev_change_medium([drive cStringUsingEncoding:
+                                          NSASCIIStringEncoding],
+                                   [file cStringUsingEncoding:
+                                         NSASCIIStringEncoding],
+                                   true, "raw",
+                                   false, 0,
+                                   &err);
+        handleAnyDeviceErrors(err);
+    }
+}
+
+/* Verifies if the user really wants to quit */
+- (BOOL)verifyQuit
+{
+    NSAlert *alert = [NSAlert new];
+    [alert autorelease];
+    [alert setMessageText: @"Are you sure you want to quit QEMU?"];
+    [alert addButtonWithTitle: @"Cancel"];
+    [alert addButtonWithTitle: @"Quit"];
+    if([alert runModal] == NSAlertSecondButtonReturn) {
+        return YES;
+    } else {
+        return NO;
+    }
+}
+
+@end
 
 
 int main (int argc, const char * argv[]) {
@@ -967,6 +1161,7 @@
                 !strcmp(opt, "-nographic") ||
                 !strcmp(opt, "-version") ||
                 !strcmp(opt, "-curses") ||
+                !strcmp(opt, "-display") ||
                 !strcmp(opt, "-qtest")) {
                 return qemu_main(gArgc, gArgv, *_NSGetEnviron());
             }
@@ -1003,9 +1198,24 @@
     [[NSApp mainMenu] addItem:menuItem];
     [NSApp performSelector:@selector(setAppleMenu:) withObject:menu]; // Workaround (this method is private since 10.4+)
 
+    // Machine menu
+    menu = [[NSMenu alloc] initWithTitle: @"Machine"];
+    [menu setAutoenablesItems: NO];
+    [menu addItem: [[[NSMenuItem alloc] initWithTitle: @"Pause" action: @selector(pauseQEMU:) keyEquivalent: @""] autorelease]];
+    menuItem = [[[NSMenuItem alloc] initWithTitle: @"Resume" action: @selector(resumeQEMU:) keyEquivalent: @""] autorelease];
+    [menu addItem: menuItem];
+    [menuItem setEnabled: NO];
+    [menu addItem: [NSMenuItem separatorItem]];
+    [menu addItem: [[[NSMenuItem alloc] initWithTitle: @"Reset" action: @selector(restartQEMU:) keyEquivalent: @""] autorelease]];
+    [menu addItem: [[[NSMenuItem alloc] initWithTitle: @"Power Down" action: @selector(powerDownQEMU:) keyEquivalent: @""] autorelease]];
+    menuItem = [[[NSMenuItem alloc] initWithTitle: @"Machine" action:nil keyEquivalent:@""] autorelease];
+    [menuItem setSubmenu:menu];
+    [[NSApp mainMenu] addItem:menuItem];
+
     // View menu
     menu = [[NSMenu alloc] initWithTitle:@"View"];
-    [menu addItem: [[[NSMenuItem alloc] initWithTitle:@"Enter Fullscreen" action:@selector(toggleFullScreen:) keyEquivalent:@"f"] autorelease]]; // Fullscreen
+    [menu addItem: [[[NSMenuItem alloc] initWithTitle:@"Enter Fullscreen" action:@selector(doToggleFullScreen:) keyEquivalent:@"f"] autorelease]]; // Fullscreen
+    [menu addItem: [[[NSMenuItem alloc] initWithTitle:@"Zoom To Fit" action:@selector(zoomToFit:) keyEquivalent:@""] autorelease]];
     menuItem = [[[NSMenuItem alloc] initWithTitle:@"View" action:nil keyEquivalent:@""] autorelease];
     [menuItem setSubmenu:menu];
     [[NSApp mainMenu] addItem:menuItem];
@@ -1079,6 +1289,7 @@
     NSAutoreleasePool * pool = [[NSAutoreleasePool alloc] init];
 
     COCOA_DEBUG("qemu_cocoa: cocoa_refresh\n");
+    graphic_hw_update(NULL);
 
     if (qemu_input_is_absolute()) {
         if (![cocoaView isAbsoluteEnabled]) {
@@ -1099,7 +1310,6 @@
             [cocoaView handleEvent:event];
         }
     } while(event != nil);
-    graphic_hw_update(NULL);
     [pool release];
 }
 
@@ -1116,10 +1326,108 @@
     .dpy_refresh = cocoa_refresh,
 };
 
+/* Returns a name for a given console */
+static NSString * getConsoleName(QemuConsole * console)
+{
+    return [NSString stringWithFormat: @"%s", qemu_console_get_label(console)];
+}
+
+/* Add an entry to the View menu for each console */
+static void add_console_menu_entries(void)
+{
+    NSMenu *menu;
+    NSMenuItem *menuItem;
+    int index = 0;
+
+    menu = [[[NSApp mainMenu] itemWithTitle:@"View"] submenu];
+
+    [menu addItem:[NSMenuItem separatorItem]];
+
+    while (qemu_console_lookup_by_index(index) != NULL) {
+        menuItem = [[[NSMenuItem alloc] initWithTitle: getConsoleName(qemu_console_lookup_by_index(index))
+                                               action: @selector(displayConsole:) keyEquivalent: @""] autorelease];
+        [menuItem setTag: index];
+        [menu addItem: menuItem];
+        index++;
+    }
+}
+
+/* Make menu items for all removable devices.
+ * Each device is given an 'Eject' and 'Change' menu item.
+ */
+static void addRemovableDevicesMenuItems(void)
+{
+    NSMenu *menu;
+    NSMenuItem *menuItem;
+    BlockInfoList *currentDevice, *pointerToFree;
+    NSString *deviceName;
+
+    currentDevice = qmp_query_block(NULL);
+    pointerToFree = currentDevice;
+    if(currentDevice == NULL) {
+        NSBeep();
+        QEMU_Alert(@"Failed to query for block devices!");
+        return;
+    }
+
+    menu = [[[NSApp mainMenu] itemWithTitle:@"Machine"] submenu];
+
+    // Add a separator between related groups of menu items
+    [menu addItem:[NSMenuItem separatorItem]];
+
+    // Set the attributes to the "Removable Media" menu item
+    NSString *titleString = @"Removable Media";
+    NSMutableAttributedString *attString=[[NSMutableAttributedString alloc] initWithString:titleString];
+    NSColor *newColor = [NSColor blackColor];
+    NSFontManager *fontManager = [NSFontManager sharedFontManager];
+    NSFont *font = [fontManager fontWithFamily:@"Helvetica"
+                                          traits:NSBoldFontMask|NSItalicFontMask
+                                          weight:0
+                                            size:14];
+    [attString addAttribute:NSFontAttributeName value:font range:NSMakeRange(0, [titleString length])];
+    [attString addAttribute:NSForegroundColorAttributeName value:newColor range:NSMakeRange(0, [titleString length])];
+    [attString addAttribute:NSUnderlineStyleAttributeName value:[NSNumber numberWithInt: 1] range:NSMakeRange(0, [titleString length])];
+
+    // Add the "Removable Media" menu item
+    menuItem = [NSMenuItem new];
+    [menuItem setAttributedTitle: attString];
+    [menuItem setEnabled: NO];
+    [menu addItem: menuItem];
+
+    /* Loop through all the block devices in the emulator */
+    while (currentDevice) {
+        deviceName = [[NSString stringWithFormat: @"%s", currentDevice->value->device] retain];
+
+        if(currentDevice->value->removable) {
+            menuItem = [[NSMenuItem alloc] initWithTitle: [NSString stringWithFormat: @"Change %s...", currentDevice->value->device]
+                                                  action: @selector(changeDeviceMedia:)
+                                           keyEquivalent: @""];
+            [menu addItem: menuItem];
+            [menuItem setRepresentedObject: deviceName];
+            [menuItem autorelease];
+
+            menuItem = [[NSMenuItem alloc] initWithTitle: [NSString stringWithFormat: @"Eject %s", currentDevice->value->device]
+                                                  action: @selector(ejectDeviceMedia:)
+                                           keyEquivalent: @""];
+            [menu addItem: menuItem];
+            [menuItem setRepresentedObject: deviceName];
+            [menuItem autorelease];
+        }
+        currentDevice = currentDevice->next;
+    }
+    qapi_free_BlockInfoList(pointerToFree);
+}
+
 void cocoa_display_init(DisplayState *ds, int full_screen)
 {
     COCOA_DEBUG("qemu_cocoa: cocoa_display_init\n");
 
+    /* if fullscreen mode is to be used */
+    if (full_screen == true) {
+        [NSApp activateIgnoringOtherApps: YES];
+        [(QemuCocoaAppController *)[[NSApplication sharedApplication] delegate] toggleFullScreen: nil];
+    }
+
     dcl = g_malloc0(sizeof(DisplayChangeListener));
 
     // register vga output callbacks
@@ -1128,4 +1436,15 @@
 
     // register cleanup function
     atexit(cocoa_cleanup);
+
+    /* At this point QEMU has created all the consoles, so we can add View
+     * menu entries for them.
+     */
+    add_console_menu_entries();
+
+    /* Give all removable devices a menu item.
+     * Has to be called after QEMU has started to
+     * find out what removable devices it has.
+     */
+    addRemovableDevicesMenuItems();
 }
diff --git a/ui/console-gl.c b/ui/console-gl.c
new file mode 100644
index 0000000..5165e21
--- /dev/null
+++ b/ui/console-gl.c
@@ -0,0 +1,179 @@
+/*
+ * QEMU graphical console -- opengl helper bits
+ *
+ * Copyright (c) 2014 Red Hat
+ *
+ * Authors:
+ *    Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/shader.h"
+
+#include "shader/texture-blit-vert.h"
+#include "shader/texture-blit-frag.h"
+
+struct ConsoleGLState {
+    GLint texture_blit_prog;
+    GLint texture_blit_vao;
+};
+
+/* ---------------------------------------------------------------------- */
+
+ConsoleGLState *console_gl_init_context(void)
+{
+    ConsoleGLState *gls = g_new0(ConsoleGLState, 1);
+
+    gls->texture_blit_prog = qemu_gl_create_compile_link_program
+        (texture_blit_vert_src, texture_blit_frag_src);
+    if (!gls->texture_blit_prog) {
+        exit(1);
+    }
+
+    gls->texture_blit_vao =
+        qemu_gl_init_texture_blit(gls->texture_blit_prog);
+
+    return gls;
+}
+
+void console_gl_fini_context(ConsoleGLState *gls)
+{
+    if (!gls) {
+        return;
+    }
+    g_free(gls);
+}
+
+bool console_gl_check_format(DisplayChangeListener *dcl,
+                             pixman_format_code_t format)
+{
+    switch (format) {
+    case PIXMAN_BE_b8g8r8x8:
+    case PIXMAN_BE_b8g8r8a8:
+    case PIXMAN_r5g6b5:
+        return true;
+    default:
+        return false;
+    }
+}
+
+void surface_gl_create_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface)
+{
+    assert(gls);
+    assert(surface_stride(surface) % surface_bytes_per_pixel(surface) == 0);
+
+    switch (surface->format) {
+    case PIXMAN_BE_b8g8r8x8:
+    case PIXMAN_BE_b8g8r8a8:
+        surface->glformat = GL_BGRA_EXT;
+        surface->gltype = GL_UNSIGNED_BYTE;
+        break;
+    case PIXMAN_BE_x8r8g8b8:
+    case PIXMAN_BE_a8r8g8b8:
+        surface->glformat = GL_RGBA;
+        surface->gltype = GL_UNSIGNED_BYTE;
+        break;
+    case PIXMAN_r5g6b5:
+        surface->glformat = GL_RGB;
+        surface->gltype = GL_UNSIGNED_SHORT_5_6_5;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    glGenTextures(1, &surface->texture);
+    glEnable(GL_TEXTURE_2D);
+    glBindTexture(GL_TEXTURE_2D, surface->texture);
+    glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT,
+                  surface_stride(surface) / surface_bytes_per_pixel(surface));
+    glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB,
+                 surface_width(surface),
+                 surface_height(surface),
+                 0, surface->glformat, surface->gltype,
+                 surface_data(surface));
+
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+}
+
+void surface_gl_update_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface,
+                               int x, int y, int w, int h)
+{
+    uint8_t *data = (void *)surface_data(surface);
+
+    assert(gls);
+
+    glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT,
+                  surface_stride(surface) / surface_bytes_per_pixel(surface));
+    glTexSubImage2D(GL_TEXTURE_2D, 0,
+                    x, y, w, h,
+                    surface->glformat, surface->gltype,
+                    data + surface_stride(surface) * y
+                    + surface_bytes_per_pixel(surface) * x);
+}
+
+void surface_gl_render_texture(ConsoleGLState *gls,
+                               DisplaySurface *surface)
+{
+    assert(gls);
+
+    glClearColor(0.1f, 0.1f, 0.1f, 0.0f);
+    glClear(GL_COLOR_BUFFER_BIT);
+
+    qemu_gl_run_texture_blit(gls->texture_blit_prog,
+                             gls->texture_blit_vao);
+}
+
+void surface_gl_destroy_texture(ConsoleGLState *gls,
+                                DisplaySurface *surface)
+{
+    if (!surface || !surface->texture) {
+        return;
+    }
+    glDeleteTextures(1, &surface->texture);
+    surface->texture = 0;
+}
+
+void surface_gl_setup_viewport(ConsoleGLState *gls,
+                               DisplaySurface *surface,
+                               int ww, int wh)
+{
+    int gw, gh, stripe;
+    float sw, sh;
+
+    assert(gls);
+
+    gw = surface_width(surface);
+    gh = surface_height(surface);
+
+    sw = (float)ww/gw;
+    sh = (float)wh/gh;
+    if (sw < sh) {
+        stripe = wh - wh*sw/sh;
+        glViewport(0, stripe / 2, ww, wh - stripe);
+    } else {
+        stripe = ww - ww*sh/sw;
+        glViewport(stripe / 2, 0, ww - stripe, wh);
+    }
+}
diff --git a/ui/console.c b/ui/console.c
index 1174c2b..b5758ae 100644
--- a/ui/console.c
+++ b/ui/console.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/input.h"
@@ -28,6 +29,7 @@
 #include "qemu/timer.h"
 #include "qmp-commands.h"
 #include "sysemu/char.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 #include "exec/memory.h"
 
@@ -122,11 +124,13 @@
     DisplayState *ds;
     DisplaySurface *surface;
     int dcls;
+    DisplayChangeListener *gl;
 
     /* Graphic console state.  */
     Object *device;
     uint32_t head;
     QemuUIInfo ui_info;
+    QEMUTimer *ui_timer;
     const GraphicHwOps *hw_ops;
     void *hw;
 
@@ -170,7 +174,6 @@
     bool have_text;
 
     QLIST_HEAD(, DisplayChangeListener) listeners;
-    DisplayUpdateListener* update_listener;
 };
 
 static DisplayState *display_state;
@@ -236,9 +239,6 @@
             have_text = true;
         }
     }
-    if (ds->update_listener && ds->update_listener->dpy_gfx_update) {
-        have_gfx = true;
-    }
 
     if (need_timer && ds->gui_timer == NULL) {
         ds->gui_timer = timer_new_ms(QEMU_CLOCK_REALTIME, gui_update, ds);
@@ -264,6 +264,16 @@
     }
 }
 
+void graphic_hw_gl_block(QemuConsole *con, bool block)
+{
+    if (!con) {
+        con = active_console;
+    }
+    if (con && con->hw_ops->gl_block) {
+        con->hw_ops->gl_block(con->hw, block);
+    }
+}
+
 void graphic_hw_invalidate(QemuConsole *con)
 {
     if (!con) {
@@ -274,7 +284,7 @@
     }
 }
 
-static void ppm_save(const char *filename, struct DisplaySurface *ds,
+static void ppm_save(const char *filename, DisplaySurface *ds,
                      Error **errp)
 {
     int width = pixman_image_get_width(ds->image);
@@ -379,42 +389,29 @@
 
 #include "vgafont.h"
 
-#ifndef CONFIG_CURSES
-enum color_names {
-    COLOR_BLACK   = 0,
-    COLOR_RED     = 1,
-    COLOR_GREEN   = 2,
-    COLOR_YELLOW  = 3,
-    COLOR_BLUE    = 4,
-    COLOR_MAGENTA = 5,
-    COLOR_CYAN    = 6,
-    COLOR_WHITE   = 7
-};
-#endif
-
 #define QEMU_RGB(r, g, b)                                               \
     { .red = r << 8, .green = g << 8, .blue = b << 8, .alpha = 0xffff }
 
 static const pixman_color_t color_table_rgb[2][8] = {
     {   /* dark */
-        QEMU_RGB(0x00, 0x00, 0x00),  /* black */
-        QEMU_RGB(0xaa, 0x00, 0x00),  /* red */
-        QEMU_RGB(0x00, 0xaa, 0x00),  /* green */
-        QEMU_RGB(0xaa, 0xaa, 0x00),  /* yellow */
-        QEMU_RGB(0x00, 0x00, 0xaa),  /* blue */
-        QEMU_RGB(0xaa, 0x00, 0xaa),  /* magenta */
-        QEMU_RGB(0x00, 0xaa, 0xaa),  /* cyan */
-        QEMU_RGB(0xaa, 0xaa, 0xaa),  /* white */
+        [QEMU_COLOR_BLACK]   = QEMU_RGB(0x00, 0x00, 0x00),  /* black */
+        [QEMU_COLOR_BLUE]    = QEMU_RGB(0x00, 0x00, 0xaa),  /* blue */
+        [QEMU_COLOR_GREEN]   = QEMU_RGB(0x00, 0xaa, 0x00),  /* green */
+        [QEMU_COLOR_CYAN]    = QEMU_RGB(0x00, 0xaa, 0xaa),  /* cyan */
+        [QEMU_COLOR_RED]     = QEMU_RGB(0xaa, 0x00, 0x00),  /* red */
+        [QEMU_COLOR_MAGENTA] = QEMU_RGB(0xaa, 0x00, 0xaa),  /* magenta */
+        [QEMU_COLOR_YELLOW]  = QEMU_RGB(0xaa, 0xaa, 0x00),  /* yellow */
+        [QEMU_COLOR_WHITE]   = QEMU_RGB(0xaa, 0xaa, 0xaa),  /* white */
     },
     {   /* bright */
-        QEMU_RGB(0x00, 0x00, 0x00),  /* black */
-        QEMU_RGB(0xff, 0x00, 0x00),  /* red */
-        QEMU_RGB(0x00, 0xff, 0x00),  /* green */
-        QEMU_RGB(0xff, 0xff, 0x00),  /* yellow */
-        QEMU_RGB(0x00, 0x00, 0xff),  /* blue */
-        QEMU_RGB(0xff, 0x00, 0xff),  /* magenta */
-        QEMU_RGB(0x00, 0xff, 0xff),  /* cyan */
-        QEMU_RGB(0xff, 0xff, 0xff),  /* white */
+        [QEMU_COLOR_BLACK]   = QEMU_RGB(0x00, 0x00, 0x00),  /* black */
+        [QEMU_COLOR_BLUE]    = QEMU_RGB(0x00, 0x00, 0xff),  /* blue */
+        [QEMU_COLOR_GREEN]   = QEMU_RGB(0x00, 0xff, 0x00),  /* green */
+        [QEMU_COLOR_CYAN]    = QEMU_RGB(0x00, 0xff, 0xff),  /* cyan */
+        [QEMU_COLOR_RED]     = QEMU_RGB(0xff, 0x00, 0x00),  /* red */
+        [QEMU_COLOR_MAGENTA] = QEMU_RGB(0xff, 0x00, 0xff),  /* magenta */
+        [QEMU_COLOR_YELLOW]  = QEMU_RGB(0xff, 0xff, 0x00),  /* yellow */
+        [QEMU_COLOR_WHITE]   = QEMU_RGB(0xff, 0xff, 0xff),  /* white */
     }
 };
 
@@ -453,7 +450,7 @@
     if (s->width < w1)
         w1 = s->width;
 
-    cells = g_malloc(s->width * s->total_height * sizeof(TextCell));
+    cells = g_new(TextCell, s->width * s->total_height);
     for(y = 0; y < s->total_height; y++) {
         c = &cells[y * s->width];
         if (w1 > 0) {
@@ -563,7 +560,7 @@
     }
 
     vga_fill_rect(s, 0, 0, surface_width(surface), surface_height(surface),
-                  color_table_rgb[0][COLOR_BLACK]);
+                  color_table_rgb[0][QEMU_COLOR_BLACK]);
     y1 = s->y_displayed;
     for (y = 0; y < s->height; y++) {
         c = s->cells + y1 * s->width;
@@ -701,53 +698,53 @@
                 break;
             /* set foreground color */
             case 30:
-                s->t_attrib.fgcol=COLOR_BLACK;
+                s->t_attrib.fgcol = QEMU_COLOR_BLACK;
                 break;
             case 31:
-                s->t_attrib.fgcol=COLOR_RED;
+                s->t_attrib.fgcol = QEMU_COLOR_RED;
                 break;
             case 32:
-                s->t_attrib.fgcol=COLOR_GREEN;
+                s->t_attrib.fgcol = QEMU_COLOR_GREEN;
                 break;
             case 33:
-                s->t_attrib.fgcol=COLOR_YELLOW;
+                s->t_attrib.fgcol = QEMU_COLOR_YELLOW;
                 break;
             case 34:
-                s->t_attrib.fgcol=COLOR_BLUE;
+                s->t_attrib.fgcol = QEMU_COLOR_BLUE;
                 break;
             case 35:
-                s->t_attrib.fgcol=COLOR_MAGENTA;
+                s->t_attrib.fgcol = QEMU_COLOR_MAGENTA;
                 break;
             case 36:
-                s->t_attrib.fgcol=COLOR_CYAN;
+                s->t_attrib.fgcol = QEMU_COLOR_CYAN;
                 break;
             case 37:
-                s->t_attrib.fgcol=COLOR_WHITE;
+                s->t_attrib.fgcol = QEMU_COLOR_WHITE;
                 break;
             /* set background color */
             case 40:
-                s->t_attrib.bgcol=COLOR_BLACK;
+                s->t_attrib.bgcol = QEMU_COLOR_BLACK;
                 break;
             case 41:
-                s->t_attrib.bgcol=COLOR_RED;
+                s->t_attrib.bgcol = QEMU_COLOR_RED;
                 break;
             case 42:
-                s->t_attrib.bgcol=COLOR_GREEN;
+                s->t_attrib.bgcol = QEMU_COLOR_GREEN;
                 break;
             case 43:
-                s->t_attrib.bgcol=COLOR_YELLOW;
+                s->t_attrib.bgcol = QEMU_COLOR_YELLOW;
                 break;
             case 44:
-                s->t_attrib.bgcol=COLOR_BLUE;
+                s->t_attrib.bgcol = QEMU_COLOR_BLUE;
                 break;
             case 45:
-                s->t_attrib.bgcol=COLOR_MAGENTA;
+                s->t_attrib.bgcol = QEMU_COLOR_MAGENTA;
                 break;
             case 46:
-                s->t_attrib.bgcol=COLOR_CYAN;
+                s->t_attrib.bgcol = QEMU_COLOR_CYAN;
                 break;
             case 47:
-                s->t_attrib.bgcol=COLOR_WHITE;
+                s->t_attrib.bgcol = QEMU_COLOR_WHITE;
                 break;
         }
     }
@@ -762,6 +759,31 @@
     update_xy(s, x, y);
 }
 
+static void console_put_one(QemuConsole *s, int ch)
+{
+    TextCell *c;
+    int y1;
+    if (s->x >= s->width) {
+        /* line wrap */
+        s->x = 0;
+        console_put_lf(s);
+    }
+    y1 = (s->y_base + s->y) % s->total_height;
+    c = &s->cells[y1 * s->width + s->x];
+    c->ch = ch;
+    c->t_attrib = s->t_attrib;
+    update_xy(s, s->x, s->y);
+    s->x++;
+}
+
+static void console_respond_str(QemuConsole *s, const char *buf)
+{
+    while (*buf) {
+        console_put_one(s, *buf);
+        buf++;
+    }
+}
+
 /* set cursor, checking bounds */
 static void set_cursor(QemuConsole *s, int x, int y)
 {
@@ -784,9 +806,9 @@
 
 static void console_putchar(QemuConsole *s, int ch)
 {
-    TextCell *c;
-    int y1, i;
+    int i;
     int x, y;
+    char response[40];
 
     switch(s->state) {
     case TTY_STATE_NORM:
@@ -822,17 +844,7 @@
             s->state = TTY_STATE_ESC;
             break;
         default:
-            if (s->x >= s->width) {
-                /* line wrap */
-                s->x = 0;
-                console_put_lf(s);
-            }
-            y1 = (s->y_base + s->y) % s->total_height;
-            c = &s->cells[y1 * s->width + s->x];
-            c->ch = ch;
-            c->t_attrib = s->t_attrib;
-            update_xy(s, s->x, s->y);
-            s->x++;
+            console_put_one(s, ch);
             break;
         }
         break;
@@ -961,8 +973,19 @@
                 console_handle_escape(s);
                 break;
             case 'n':
-                /* report cursor position */
-                /* TODO: send ESC[row;colR */
+                switch (s->esc_params[0]) {
+                case 5:
+                    /* report console status (always succeed)*/
+                    console_respond_str(s, "\033[0n");
+                    break;
+                case 6:
+                    /* report cursor position */
+                    sprintf(response, "\033[%d;%dR",
+                           (s->y_base + s->y) % s->total_height + 1,
+                            s->x + 1);
+                    console_respond_str(s, response);
+                    break;
+                }
                 break;
             case 's':
                 /* save cursor position */
@@ -1111,7 +1134,7 @@
     }
 }
 
-static const int qcode_to_keysym[Q_KEY_CODE_MAX] = {
+static const int qcode_to_keysym[Q_KEY_CODE__MAX] = {
     [Q_KEY_CODE_UP]     = QEMU_KEY_UP,
     [Q_KEY_CODE_DOWN]   = QEMU_KEY_DOWN,
     [Q_KEY_CODE_RIGHT]  = QEMU_KEY_RIGHT,
@@ -1174,11 +1197,13 @@
         src = (s->y_base + s->text_y[0]) * s->width;
         chardata += s->text_y[0] * s->width;
         for (i = s->text_y[0]; i <= s->text_y[1]; i ++)
-            for (j = 0; j < s->width; j ++, src ++)
-                console_write_ch(chardata ++, s->cells[src].ch |
-                                (s->cells[src].t_attrib.fgcol << 12) |
-                                (s->cells[src].t_attrib.bgcol << 8) |
-                                (s->cells[src].t_attrib.bold << 21));
+            for (j = 0; j < s->width; j++, src++) {
+                console_write_ch(chardata ++,
+                                 ATTR2CHTYPE(s->cells[src].ch,
+                                             s->cells[src].t_attrib.fgcol,
+                                             s->cells[src].t_attrib.bgcol,
+                                             s->cells[src].t_attrib.bold));
+            }
         dpy_text_update(s, s->text_x[0], s->text_y[0],
                         s->text_x[1] - s->text_x[0], i - s->text_y[0]);
         s->text_x[0] = s->width;
@@ -1275,6 +1300,17 @@
     return surface;
 }
 
+DisplaySurface *qemu_create_displaysurface_pixman(pixman_image_t *image)
+{
+    DisplaySurface *surface = g_new0(DisplaySurface, 1);
+
+    trace_displaysurface_create_pixman(surface);
+    surface->format = pixman_image_get_format(image);
+    surface->image = pixman_image_ref(image);
+
+    return surface;
+}
+
 static void qemu_unmap_displaysurface_guestmem(pixman_image_t *image,
                                                void *unused)
 {
@@ -1296,9 +1332,9 @@
         linesize = width * PIXMAN_FORMAT_BPP(format) / 8;
     }
 
-    size = linesize * height;
+    size = (hwaddr)linesize * height;
     data = cpu_physical_memory_map(addr, &size, 0);
-    if (size != linesize * height) {
+    if (size != (hwaddr)linesize * height) {
         cpu_physical_memory_unmap(data, size, 0, 0);
         return NULL;
     }
@@ -1315,8 +1351,8 @@
                                                    const char *msg)
 {
     DisplaySurface *surface = qemu_create_displaysurface(w, h);
-    pixman_color_t bg = color_table_rgb[0][COLOR_BLACK];
-    pixman_color_t fg = color_table_rgb[0][COLOR_WHITE];
+    pixman_color_t bg = color_table_rgb[0][QEMU_COLOR_BLACK];
+    pixman_color_t fg = color_table_rgb[0][QEMU_COLOR_WHITE];
     pixman_image_t *glyph;
     int len, x, y, i;
 
@@ -1342,6 +1378,11 @@
     g_free(surface);
 }
 
+bool console_has_gl(QemuConsole *con)
+{
+    return con->gl != NULL;
+}
+
 void register_displaychangelistener(DisplayChangeListener *dcl)
 {
     static const char nodev[] =
@@ -1349,6 +1390,17 @@
     static DisplaySurface *dummy;
     QemuConsole *con;
 
+    if (dcl->ops->dpy_gl_ctx_create) {
+        /* display has opengl support */
+        assert(dcl->con);
+        if (dcl->con->gl) {
+            fprintf(stderr, "can't register two opengl displays (%s, %s)\n",
+                    dcl->ops->dpy_name, dcl->con->gl->ops->dpy_name);
+            exit(1);
+        }
+        dcl->con->gl = dcl;
+    }
+
     trace_displaychangelistener_register(dcl, dcl->ops->dpy_name);
     dcl->ds = get_alloc_displaystate();
     QLIST_INSERT_HEAD(&dcl->ds->listeners, dcl, next);
@@ -1372,12 +1424,6 @@
     text_console_update_cursor(NULL);
 }
 
-void register_displayupdatelistener(DisplayUpdateListener *dul) {
-    DisplayState* ds = get_alloc_displaystate();
-    ds->update_listener = dul;
-    gui_setup_refresh(ds);
-}
-
 void update_displaychangelistener(DisplayChangeListener *dcl,
                                   uint64_t interval)
 {
@@ -1400,23 +1446,51 @@
     gui_setup_refresh(ds);
 }
 
+static void dpy_set_ui_info_timer(void *opaque)
+{
+    QemuConsole *con = opaque;
+
+    con->hw_ops->ui_info(con->hw, con->head, &con->ui_info);
+}
+
+bool dpy_ui_info_supported(QemuConsole *con)
+{
+    return con->hw_ops->ui_info != NULL;
+}
+
 int dpy_set_ui_info(QemuConsole *con, QemuUIInfo *info)
 {
     assert(con != NULL);
-    con->ui_info = *info;
-    if (con->hw_ops->ui_info) {
-        return con->hw_ops->ui_info(con->hw, con->head, info);
+
+    if (!dpy_ui_info_supported(con)) {
+        return -1;
     }
-    return -1;
+    if (memcmp(&con->ui_info, info, sizeof(con->ui_info)) == 0) {
+        /* nothing changed -- ignore */
+        return 0;
+    }
+
+    /*
+     * Typically we get a flood of these as the user resizes the window.
+     * Wait until the dust has settled (one second without updates), then
+     * go notify the guest.
+     */
+    con->ui_info = *info;
+    timer_mod(con->ui_timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + 1000);
+    return 0;
 }
 
 void dpy_gfx_update(QemuConsole *con, int x, int y, int w, int h)
 {
     DisplayState *s = con->ds;
     DisplayChangeListener *dcl;
-    int width = surface_width(con->surface);
-    int height = surface_height(con->surface);
+    int width = w;
+    int height = h;
 
+    if (con->surface) {
+        width = surface_width(con->surface);
+        height = surface_height(con->surface);
+    }
     x = MAX(x, 0);
     y = MAX(y, 0);
     x = MIN(x, width);
@@ -1435,13 +1509,6 @@
             dcl->ops->dpy_gfx_update(dcl, x, y, w, h);
         }
     }
-
-    if (s->update_listener && s->update_listener->dpy_gfx_update) {
-        if (con == (s->update_listener->con
-                        ? s->update_listener->con : active_console)) {
-            s->update_listener->dpy_gfx_update(s->update_listener, x, y, w, h);
-        }
-    }
 }
 
 void dpy_gfx_replace_surface(QemuConsole *con,
@@ -1463,6 +1530,31 @@
     qemu_free_displaysurface(old_surface);
 }
 
+bool dpy_gfx_check_format(QemuConsole *con,
+                          pixman_format_code_t format)
+{
+    DisplayChangeListener *dcl;
+    DisplayState *s = con->ds;
+
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->con && dcl->con != con) {
+            /* dcl bound to another console -> skip */
+            continue;
+        }
+        if (dcl->ops->dpy_gfx_check_format) {
+            if (!dcl->ops->dpy_gfx_check_format(dcl, format)) {
+                return false;
+            }
+        } else {
+            /* default is to whitelist native 32 bpp only */
+            if (format != qemu_default_pixman_format(32, true)) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
 static void dpy_refresh(DisplayState *s)
 {
     DisplayChangeListener *dcl;
@@ -1534,7 +1626,7 @@
 void dpy_text_resize(QemuConsole *con, int w, int h)
 {
     DisplayState *s = con->ds;
-    struct DisplayChangeListener *dcl;
+    DisplayChangeListener *dcl;
 
     if (!qemu_console_is_visible(con)) {
         return;
@@ -1598,65 +1690,48 @@
     return false;
 }
 
-/*
- * Call dpy_gfx_update for all dirity scanlines.  Works for
- * DisplaySurfaces backed by guest memory (i.e. the ones created
- * using qemu_create_displaysurface_guestmem).
- */
-void dpy_gfx_update_dirty(QemuConsole *con,
-                          MemoryRegion *address_space,
-                          hwaddr base,
-                          bool invalidate)
+QEMUGLContext dpy_gl_ctx_create(QemuConsole *con,
+                                struct QEMUGLParams *qparams)
 {
-    DisplaySurface *ds = qemu_console_surface(con);
-    int width = surface_stride(ds);
-    int height = surface_height(ds);
-    hwaddr size = width * height;
-    MemoryRegionSection mem_section;
-    MemoryRegion *mem;
-    ram_addr_t addr;
-    int first, last, i;
-    bool dirty;
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_create(con->gl, qparams);
+}
 
-    mem_section = memory_region_find(address_space, base, size);
-    mem = mem_section.mr;
-    if (int128_get64(mem_section.size) != size ||
-        !memory_region_is_ram(mem_section.mr)) {
-        goto out;
-    }
-    assert(mem);
+void dpy_gl_ctx_destroy(QemuConsole *con, QEMUGLContext ctx)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_ctx_destroy(con->gl, ctx);
+}
 
-    memory_region_sync_dirty_bitmap(mem);
-    addr = mem_section.offset_within_region;
+int dpy_gl_ctx_make_current(QemuConsole *con, QEMUGLContext ctx)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_make_current(con->gl, ctx);
+}
 
-    first = -1;
-    last = -1;
-    for (i = 0; i < height; i++, addr += width) {
-        dirty = invalidate ||
-            memory_region_get_dirty(mem, addr, width, DIRTY_MEMORY_VGA);
-        if (dirty) {
-            if (first == -1) {
-                first = i;
-            }
-            last = i;
-        }
-        if (first != -1 && !dirty) {
-            assert(last != -1 && last >= first);
-            dpy_gfx_update(con, 0, first, surface_width(ds),
-                           last - first + 1);
-            first = -1;
-        }
-    }
-    if (first != -1) {
-        assert(last != -1 && last >= first);
-        dpy_gfx_update(con, 0, first, surface_width(ds),
-                       last - first + 1);
-    }
+QEMUGLContext dpy_gl_ctx_get_current(QemuConsole *con)
+{
+    assert(con->gl);
+    return con->gl->ops->dpy_gl_ctx_get_current(con->gl);
+}
 
-    memory_region_reset_dirty(mem, mem_section.offset_within_region, size,
-                              DIRTY_MEMORY_VGA);
-out:
-    memory_region_unref(mem);
+void dpy_gl_scanout(QemuConsole *con,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t backing_width, uint32_t backing_height,
+                    uint32_t x, uint32_t y, uint32_t width, uint32_t height)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_scanout(con->gl, backing_id,
+                                 backing_y_0_top,
+                                 backing_width, backing_height,
+                                 x, y, width, height);
+}
+
+void dpy_gl_update(QemuConsole *con,
+                   uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    assert(con->gl);
+    con->gl->ops->dpy_gl_update(con->gl, x, y, w, h);
 }
 
 void dpy_run_update(QemuConsole *con) {
@@ -1720,30 +1795,21 @@
     con->hw = opaque;
 }
 
-#if defined(CONFIG_ANDROID)
-extern int android_display_width;
-extern int android_display_height;
-#endif
-
 QemuConsole *graphic_console_init(DeviceState *dev, uint32_t head,
                                   const GraphicHwOps *hw_ops,
                                   void *opaque)
 {
     static const char noinit[] =
         "Guest has not initialized the display (yet).";
-#if defined(CONFIG_ANDROID)
-    int width = android_display_width;
-    int height = android_display_height;
-#else
-    int width = 640;
-    int height = 480;
-#endif
+    int width = graphic_width;
+    int height = graphic_height;
     QemuConsole *s;
     DisplayState *ds;
 
     ds = get_alloc_displaystate();
     trace_console_gfx_new();
     s = new_console(ds, GRAPHIC_CONSOLE, head);
+    s->ui_timer = timer_new_ms(QEMU_CLOCK_REALTIME, dpy_set_ui_info_timer, s);
     graphic_console_set_hwops(s, hw_ops, opaque);
     if (dev) {
         object_property_set_link(OBJECT(s), OBJECT(dev), "device",
@@ -1787,6 +1853,29 @@
     return NULL;
 }
 
+QemuConsole *qemu_console_lookup_by_device_name(const char *device_id,
+                                                uint32_t head, Error **errp)
+{
+    DeviceState *dev;
+    QemuConsole *con;
+
+    dev = qdev_find_recursive(sysbus_get_default(), device_id);
+    if (dev == NULL) {
+        error_set(errp, ERROR_CLASS_DEVICE_NOT_FOUND,
+                  "Device '%s' not found", device_id);
+        return NULL;
+    }
+
+    con = qemu_console_lookup_by_device(dev, head);
+    if (con == NULL) {
+        error_setg(errp, "Device %s (head %d) is not bound to a QemuConsole",
+                   device_id, head);
+        return NULL;
+    }
+
+    return con;
+}
+
 bool qemu_console_is_visible(QemuConsole *con)
 {
     return (con == active_console) || (con->dcls > 0);
@@ -1808,6 +1897,21 @@
     return con && (con->console_type != TEXT_CONSOLE);
 }
 
+char *qemu_console_get_label(QemuConsole *con)
+{
+    if (con->console_type == GRAPHIC_CONSOLE) {
+        if (con->device) {
+            return g_strdup(object_get_typename(con->device));
+        }
+        return g_strdup("VGA");
+    } else {
+        if (con->chr && con->chr->label) {
+            return g_strdup(con->chr->label);
+        }
+        return g_strdup_printf("vc%d", con->index);
+    }
+}
+
 int qemu_console_get_index(QemuConsole *con)
 {
     if (con == NULL) {
@@ -1923,8 +2027,8 @@
     s->t_attrib_default.blink = 0;
     s->t_attrib_default.invers = 0;
     s->t_attrib_default.unvisible = 0;
-    s->t_attrib_default.fgcol = COLOR_WHITE;
-    s->t_attrib_default.bgcol = COLOR_BLACK;
+    s->t_attrib_default.fgcol = QEMU_COLOR_WHITE;
+    s->t_attrib_default.bgcol = QEMU_COLOR_BLACK;
     /* set current text attributes to default */
     s->t_attrib = s->t_attrib_default;
     text_console_resize(s);
@@ -1933,7 +2037,7 @@
         char msg[128];
         int len;
 
-        s->t_attrib.bgcol = COLOR_BLUE;
+        s->t_attrib.bgcol = QEMU_COLOR_BLUE;
         len = snprintf(msg, sizeof(msg), "%s console\r\n", chr->label);
         console_puts(chr, (uint8_t*)msg, len);
         s->t_attrib = s->t_attrib_default;
@@ -1944,14 +2048,18 @@
         chr->init(chr);
 }
 
-static CharDriverState *text_console_init(ChardevVC *vc)
+static CharDriverState *text_console_init(ChardevVC *vc, Error **errp)
 {
+    ChardevCommon *common = qapi_ChardevVC_base(vc);
     CharDriverState *chr;
     QemuConsole *s;
     unsigned width = 0;
     unsigned height = 0;
 
-    chr = qemu_chr_alloc();
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
 
     if (vc->has_width) {
         width = vc->width;
@@ -1975,6 +2083,7 @@
 
     if (!s) {
         g_free(chr);
+        error_setg(errp, "cannot create text console");
         return NULL;
     }
 
@@ -1994,9 +2103,10 @@
 
 static VcHandler *vc_handler = text_console_init;
 
-CharDriverState *vc_init(ChardevVC *vc)
+static CharDriverState *vc_init(const char *id, ChardevBackend *backend,
+                                ChardevReturn *ret, Error **errp)
 {
-    return vc_handler(vc);
+    return vc_handler(backend->u.vc.data, errp);
 }
 
 void register_vc_handler(VcHandler *handler)
@@ -2025,18 +2135,6 @@
     return console->surface;
 }
 
-DisplayState *qemu_console_displaystate(QemuConsole *console)
-{
-    return console->ds;
-}
-
-PixelFormat qemu_different_endianness_pixelformat(int bpp)
-{
-    pixman_format_code_t fmt = qemu_default_pixman_format(bpp, false);
-    PixelFormat pf = qemu_pixelformat_from_pixman(fmt);
-    return pf;
-}
-
 PixelFormat qemu_default_pixelformat(int bpp)
 {
     pixman_format_code_t fmt = qemu_default_pixman_format(bpp, true);
@@ -2048,79 +2146,63 @@
                               Error **errp)
 {
     int val;
+    ChardevVC *vc;
 
-    backend->vc = g_new0(ChardevVC, 1);
+    vc = backend->u.vc.data = g_new0(ChardevVC, 1);
+    qemu_chr_parse_common(opts, qapi_ChardevVC_base(vc));
 
     val = qemu_opt_get_number(opts, "width", 0);
     if (val != 0) {
-        backend->vc->has_width = true;
-        backend->vc->width = val;
+        vc->has_width = true;
+        vc->width = val;
     }
 
     val = qemu_opt_get_number(opts, "height", 0);
     if (val != 0) {
-        backend->vc->has_height = true;
-        backend->vc->height = val;
+        vc->has_height = true;
+        vc->height = val;
     }
 
     val = qemu_opt_get_number(opts, "cols", 0);
     if (val != 0) {
-        backend->vc->has_cols = true;
-        backend->vc->cols = val;
+        vc->has_cols = true;
+        vc->cols = val;
     }
 
     val = qemu_opt_get_number(opts, "rows", 0);
     if (val != 0) {
-        backend->vc->has_rows = true;
-        backend->vc->rows = val;
+        vc->has_rows = true;
+        vc->rows = val;
     }
 }
 
-static const TypeInfo qemu_console_info = {
-    .name = TYPE_QEMU_CONSOLE,
-    .parent = TYPE_OBJECT,
-    .instance_size = sizeof(QemuConsole),
-    .class_size = sizeof(QemuConsoleClass),
-};
-
-
-static void register_types(void)
-{
-    type_register_static(&qemu_console_info);
-    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc);
-}
-
-#if defined(CONFIG_ANDROID)
-
-extern int graphic_rotate;
-extern int graphic_width;
-
-void kbd_mouse_event(int dx, int dy, int dz, int buttonsState) {
+void kbd_mouse_event(int dx, int dy, int dz, int button_state) {
 
     assert(active_console && qemu_console_is_graphic(active_console));
 
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE]     = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]      = MOUSE_EVENT_RBUTTON,
     };
     static uint32_t prev_state;
 
-    if (prev_state != buttonsState) {
-        qemu_input_update_buttons(active_console, bmap, prev_state, buttonsState);
-        prev_state = (uint32_t)buttonsState;
+    if (prev_state != button_state) {
+        qemu_input_update_buttons(active_console, bmap, prev_state,
+                                  button_state);
+        prev_state = (uint32_t)button_state;
     }
 
-    const bool isAbsolute = qemu_input_is_absolute();
+    const bool is_absolute = qemu_input_is_absolute();
     int y;
     if (graphic_rotate) {
-        const int width = isAbsolute ? INPUT_EVENT_ABS_SIZE : graphic_width;
+        const int width = is_absolute ? INPUT_EVENT_ABS_SIZE : graphic_width;
         y = width - 1 - dy;
     } else {
         y = dy;
     }
 
-    if (isAbsolute  &&  dz == 0) {
+    if (is_absolute  &&  dz == 0) {
         int w = surface_width(qemu_console_surface(active_console));
         int h = surface_height(qemu_console_surface(active_console));
 
@@ -2134,6 +2216,19 @@
     qemu_input_event_sync();
 }
 
-#endif // CONFIG_ANDROID
+static const TypeInfo qemu_console_info = {
+    .name = TYPE_QEMU_CONSOLE,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(QemuConsole),
+    .class_size = sizeof(QemuConsoleClass),
+};
+
+
+static void register_types(void)
+{
+    type_register_static(&qemu_console_info);
+    register_char_driver("vc", CHARDEV_BACKEND_KIND_VC, qemu_chr_parse_vc,
+                         vc_init);
+}
 
 type_init(register_types);
diff --git a/ui/curses.c b/ui/curses.c
index 8edb038..b475589 100644
--- a/ui/curses.c
+++ b/ui/curses.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include <curses.h>
 
 #ifndef _WIN32
@@ -42,6 +43,8 @@
 static int width, height, gwidth, gheight, invalidate;
 static int px, py, sminx, sminy, smaxx, smaxy;
 
+chtype vga_to_curses[256];
+
 static void curses_update(DisplayChangeListener *dcl,
                           int x, int y, int w, int h)
 {
@@ -331,8 +334,14 @@
 static void curses_setup(void)
 {
     int i, colour_default[8] = {
-        COLOR_BLACK, COLOR_BLUE, COLOR_GREEN, COLOR_CYAN,
-        COLOR_RED, COLOR_MAGENTA, COLOR_YELLOW, COLOR_WHITE,
+        [QEMU_COLOR_BLACK]   = COLOR_BLACK,
+        [QEMU_COLOR_BLUE]    = COLOR_BLUE,
+        [QEMU_COLOR_GREEN]   = COLOR_GREEN,
+        [QEMU_COLOR_CYAN]    = COLOR_CYAN,
+        [QEMU_COLOR_RED]     = COLOR_RED,
+        [QEMU_COLOR_MAGENTA] = COLOR_MAGENTA,
+        [QEMU_COLOR_YELLOW]  = COLOR_YELLOW,
+        [QEMU_COLOR_WHITE]   = COLOR_WHITE,
     };
 
     /* input as raw as possible, let everything be interpreted
@@ -341,8 +350,56 @@
     nodelay(stdscr, TRUE); nonl(); keypad(stdscr, TRUE);
     start_color(); raw(); scrollok(stdscr, FALSE);
 
-    for (i = 0; i < 64; i ++)
+    /* Make color pair to match color format (3bits bg:3bits fg) */
+    for (i = 0; i < 64; i++) {
         init_pair(i, colour_default[i & 7], colour_default[i >> 3]);
+    }
+    /* Set default color for more than 64 for safety. */
+    for (i = 64; i < COLOR_PAIRS; i++) {
+        init_pair(i, COLOR_WHITE, COLOR_BLACK);
+    }
+
+    /*
+     * Setup mapping for vga to curses line graphics.
+     * FIXME: for better font, have to use ncursesw and setlocale()
+     */
+#if 0
+    /* FIXME: map from where? */
+    ACS_S1;
+    ACS_S3;
+    ACS_S7;
+    ACS_S9;
+#endif
+    /* ACS_* is not constant. So, we can't initialize statically. */
+    vga_to_curses['\0'] = ' ';
+    vga_to_curses[0x04] = ACS_DIAMOND;
+    vga_to_curses[0x0a] = ACS_RARROW;
+    vga_to_curses[0x0b] = ACS_LARROW;
+    vga_to_curses[0x18] = ACS_UARROW;
+    vga_to_curses[0x19] = ACS_DARROW;
+    vga_to_curses[0x9c] = ACS_STERLING;
+    vga_to_curses[0xb0] = ACS_BOARD;
+    vga_to_curses[0xb1] = ACS_CKBOARD;
+    vga_to_curses[0xb3] = ACS_VLINE;
+    vga_to_curses[0xb4] = ACS_RTEE;
+    vga_to_curses[0xbf] = ACS_URCORNER;
+    vga_to_curses[0xc0] = ACS_LLCORNER;
+    vga_to_curses[0xc1] = ACS_BTEE;
+    vga_to_curses[0xc2] = ACS_TTEE;
+    vga_to_curses[0xc3] = ACS_LTEE;
+    vga_to_curses[0xc4] = ACS_HLINE;
+    vga_to_curses[0xc5] = ACS_PLUS;
+    vga_to_curses[0xce] = ACS_LANTERN;
+    vga_to_curses[0xd8] = ACS_NEQUAL;
+    vga_to_curses[0xd9] = ACS_LRCORNER;
+    vga_to_curses[0xda] = ACS_ULCORNER;
+    vga_to_curses[0xdb] = ACS_BLOCK;
+    vga_to_curses[0xe3] = ACS_PI;
+    vga_to_curses[0xf1] = ACS_PLMINUS;
+    vga_to_curses[0xf2] = ACS_GEQUAL;
+    vga_to_curses[0xf3] = ACS_LEQUAL;
+    vga_to_curses[0xf8] = ACS_DEGREE;
+    vga_to_curses[0xfe] = ACS_BULLET;
 }
 
 static void curses_keyboard_setup(void)
@@ -382,7 +439,7 @@
 
     curses_winch_init();
 
-    dcl = (DisplayChangeListener *) g_malloc0(sizeof(DisplayChangeListener));
+    dcl = g_new0(DisplayChangeListener, 1);
     dcl->ops = &dcl_ops;
     register_displaychangelistener(dcl);
 
diff --git a/ui/curses_keys.h b/ui/curses_keys.h
index 18ce6dc..e39ef9e 100644
--- a/ui/curses_keys.h
+++ b/ui/curses_keys.h
@@ -23,14 +23,13 @@
  */
 
 #ifndef QEMU_CURSES_KEYS_H
-#define QEMU_CURSES_KEYS_H 1
+#define QEMU_CURSES_KEYS_H
 
 #include <curses.h>
 #include "keymaps.h"
 
 
-#define KEY_RELEASE         0x80
-#define KEY_MASK            0x7f
+#define KEY_MASK            SCANCODE_KEYMASK
 #define GREY_CODE           0xe0
 #define GREY                SCANCODE_GREY
 #define SHIFT_CODE          0x2a
@@ -60,6 +59,8 @@
     ['\n'] = KEY_ENTER,
     [27] = 27,
     [KEY_BTAB] = '\t' | KEYSYM_SHIFT,
+    [KEY_SPREVIOUS] = KEY_PPAGE | KEYSYM_SHIFT,
+    [KEY_SNEXT] = KEY_NPAGE | KEYSYM_SHIFT,
 };
 
 static const int curses2keycode[CURSES_KEYS] = {
@@ -149,6 +150,9 @@
     [KEY_IC] = 82 | GREY, /* Insert */
     [KEY_DC] = 83 | GREY, /* Delete */
 
+    [KEY_SPREVIOUS] = 73 | GREY | SHIFT, /* Shift + Page Up */
+    [KEY_SNEXT] = 81 | GREY | SHIFT, /* Shift + Page Down */
+
     ['!'] = 2 | SHIFT,
     ['@'] = 3 | SHIFT,
     ['#'] = 4 | SHIFT,
diff --git a/ui/cursor.c b/ui/cursor.c
index 2b8dd3f..5155b39 100644
--- a/ui/cursor.c
+++ b/ui/cursor.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 
@@ -80,18 +81,12 @@
 
 QEMUCursor *cursor_builtin_hidden(void)
 {
-    QEMUCursor *c;
-
-    c = cursor_parse_xpm(cursor_hidden_xpm);
-    return c;
+    return cursor_parse_xpm(cursor_hidden_xpm);
 }
 
 QEMUCursor *cursor_builtin_left_ptr(void)
 {
-    QEMUCursor *c;
-
-    c = cursor_parse_xpm(cursor_left_ptr_xpm);
-    return c;
+    return cursor_parse_xpm(cursor_left_ptr_xpm);
 }
 
 QEMUCursor *cursor_alloc(int width, int height)
diff --git a/ui/d3des.c b/ui/d3des.c
deleted file mode 100644
index 60c840e..0000000
--- a/ui/d3des.c
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * This is D3DES (V5.09) by Richard Outerbridge with the double and
- * triple-length support removed for use in VNC.  Also the bytebit[] array
- * has been reversed so that the most significant bit in each byte of the
- * key is ignored, not the least significant.
- *
- * These changes are:
- *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- */
-
-/* D3DES (V5.09) -
- *
- * A portable, public domain, version of the Data Encryption Standard.
- *
- * Written with Symantec's THINK (Lightspeed) C by Richard Outerbridge.
- * Thanks to: Dan Hoey for his excellent Initial and Inverse permutation
- * code;  Jim Gillogly & Phil Karn for the DES key schedule code; Dennis
- * Ferguson, Eric Young and Dana How for comparing notes; and Ray Lau,
- * for humouring me on.
- *
- * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge.
- * (GEnie : OUTER; CIS : [71755,204]) Graven Imagery, 1992.
- */
-
-#include "d3des.h"
-
-static void scrunch(unsigned char *, unsigned long *);
-static void unscrun(unsigned long *, unsigned char *);
-static void desfunc(unsigned long *, unsigned long *);
-static void cookey(unsigned long *);
-
-static unsigned long KnL[32] = { 0L };
-
-static const unsigned short bytebit[8]	= {
-	01, 02, 04, 010, 020, 040, 0100, 0200 };
-
-static const unsigned long bigbyte[24] = {
-	0x800000L,	0x400000L,	0x200000L,	0x100000L,
-	0x80000L,	0x40000L,	0x20000L,	0x10000L,
-	0x8000L,	0x4000L,	0x2000L,	0x1000L,
-	0x800L, 	0x400L, 	0x200L, 	0x100L,
-	0x80L,		0x40L,		0x20L,		0x10L,
-	0x8L,		0x4L,		0x2L,		0x1L	};
-
-/* Use the key schedule specified in the Standard (ANSI X3.92-1981). */
-
-static const unsigned char pc1[56] = {
-	56, 48, 40, 32, 24, 16,  8,	 0, 57, 49, 41, 33, 25, 17,
-	 9,  1, 58, 50, 42, 34, 26,	18, 10,  2, 59, 51, 43, 35,
-	62, 54, 46, 38, 30, 22, 14,	 6, 61, 53, 45, 37, 29, 21,
-	13,  5, 60, 52, 44, 36, 28,	20, 12,  4, 27, 19, 11,  3 };
-
-static const unsigned char totrot[16] = {
-	1,2,4,6,8,10,12,14,15,17,19,21,23,25,27,28 };
-
-static const unsigned char pc2[48] = {
-	13, 16, 10, 23,  0,  4,  2, 27, 14,  5, 20,  9,
-	22, 18, 11,  3, 25,  7, 15,  6, 26, 19, 12,  1,
-	40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47,
-	43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31 };
-
-/* Thanks to James Gillogly & Phil Karn! */
-void deskey(unsigned char *key, int edf)
-{
-	register int i, j, l, m, n;
-	unsigned char pc1m[56], pcr[56];
-	unsigned long kn[32];
-
-	for ( j = 0; j < 56; j++ ) {
-		l = pc1[j];
-		m = l & 07;
-		pc1m[j] = (key[l >> 3] & bytebit[m]) ? 1 : 0;
-		}
-	for( i = 0; i < 16; i++ ) {
-		if( edf == DE1 ) m = (15 - i) << 1;
-		else m = i << 1;
-		n = m + 1;
-		kn[m] = kn[n] = 0L;
-		for( j = 0; j < 28; j++ ) {
-			l = j + totrot[i];
-			if( l < 28 ) pcr[j] = pc1m[l];
-			else pcr[j] = pc1m[l - 28];
-			}
-		for( j = 28; j < 56; j++ ) {
-		    l = j + totrot[i];
-		    if( l < 56 ) pcr[j] = pc1m[l];
-		    else pcr[j] = pc1m[l - 28];
-		    }
-		for( j = 0; j < 24; j++ ) {
-			if( pcr[pc2[j]] ) kn[m] |= bigbyte[j];
-			if( pcr[pc2[j+24]] ) kn[n] |= bigbyte[j];
-			}
-		}
-	cookey(kn);
-	return;
-	}
-
-static void cookey(register unsigned long *raw1)
-{
-	register unsigned long *cook, *raw0;
-	unsigned long dough[32];
-	register int i;
-
-	cook = dough;
-	for( i = 0; i < 16; i++, raw1++ ) {
-		raw0 = raw1++;
-		*cook	 = (*raw0 & 0x00fc0000L) << 6;
-		*cook	|= (*raw0 & 0x00000fc0L) << 10;
-		*cook	|= (*raw1 & 0x00fc0000L) >> 10;
-		*cook++ |= (*raw1 & 0x00000fc0L) >> 6;
-		*cook	 = (*raw0 & 0x0003f000L) << 12;
-		*cook	|= (*raw0 & 0x0000003fL) << 16;
-		*cook	|= (*raw1 & 0x0003f000L) >> 4;
-		*cook++ |= (*raw1 & 0x0000003fL);
-		}
-	usekey(dough);
-	return;
-	}
-
-void cpkey(register unsigned long *into)
-{
-	register unsigned long *from, *endp;
-
-	from = KnL, endp = &KnL[32];
-	while( from < endp ) *into++ = *from++;
-	return;
-	}
-
-void usekey(register unsigned long *from)
-{
-	register unsigned long *to, *endp;
-
-	to = KnL, endp = &KnL[32];
-	while( to < endp ) *to++ = *from++;
-	return;
-	}
-
-void des(unsigned char *inblock, unsigned char *outblock)
-{
-	unsigned long work[2];
-
-	scrunch(inblock, work);
-	desfunc(work, KnL);
-	unscrun(work, outblock);
-	return;
-	}
-
-static void scrunch(register unsigned char *outof, register unsigned long *into)
-{
-	*into	 = (*outof++ & 0xffL) << 24;
-	*into	|= (*outof++ & 0xffL) << 16;
-	*into	|= (*outof++ & 0xffL) << 8;
-	*into++ |= (*outof++ & 0xffL);
-	*into	 = (*outof++ & 0xffL) << 24;
-	*into	|= (*outof++ & 0xffL) << 16;
-	*into	|= (*outof++ & 0xffL) << 8;
-	*into	|= (*outof   & 0xffL);
-	return;
-	}
-
-static void unscrun(register unsigned long *outof, register unsigned char *into)
-{
-	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
-	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
-	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
-	*into++ = (unsigned char)(*outof++	 & 0xffL);
-	*into++ = (unsigned char)((*outof >> 24) & 0xffL);
-	*into++ = (unsigned char)((*outof >> 16) & 0xffL);
-	*into++ = (unsigned char)((*outof >>  8) & 0xffL);
-	*into	=  (unsigned char)(*outof	 & 0xffL);
-	return;
-	}
-
-static const unsigned long SP1[64] = {
-	0x01010400L, 0x00000000L, 0x00010000L, 0x01010404L,
-	0x01010004L, 0x00010404L, 0x00000004L, 0x00010000L,
-	0x00000400L, 0x01010400L, 0x01010404L, 0x00000400L,
-	0x01000404L, 0x01010004L, 0x01000000L, 0x00000004L,
-	0x00000404L, 0x01000400L, 0x01000400L, 0x00010400L,
-	0x00010400L, 0x01010000L, 0x01010000L, 0x01000404L,
-	0x00010004L, 0x01000004L, 0x01000004L, 0x00010004L,
-	0x00000000L, 0x00000404L, 0x00010404L, 0x01000000L,
-	0x00010000L, 0x01010404L, 0x00000004L, 0x01010000L,
-	0x01010400L, 0x01000000L, 0x01000000L, 0x00000400L,
-	0x01010004L, 0x00010000L, 0x00010400L, 0x01000004L,
-	0x00000400L, 0x00000004L, 0x01000404L, 0x00010404L,
-	0x01010404L, 0x00010004L, 0x01010000L, 0x01000404L,
-	0x01000004L, 0x00000404L, 0x00010404L, 0x01010400L,
-	0x00000404L, 0x01000400L, 0x01000400L, 0x00000000L,
-	0x00010004L, 0x00010400L, 0x00000000L, 0x01010004L };
-
-static const unsigned long SP2[64] = {
-	0x80108020L, 0x80008000L, 0x00008000L, 0x00108020L,
-	0x00100000L, 0x00000020L, 0x80100020L, 0x80008020L,
-	0x80000020L, 0x80108020L, 0x80108000L, 0x80000000L,
-	0x80008000L, 0x00100000L, 0x00000020L, 0x80100020L,
-	0x00108000L, 0x00100020L, 0x80008020L, 0x00000000L,
-	0x80000000L, 0x00008000L, 0x00108020L, 0x80100000L,
-	0x00100020L, 0x80000020L, 0x00000000L, 0x00108000L,
-	0x00008020L, 0x80108000L, 0x80100000L, 0x00008020L,
-	0x00000000L, 0x00108020L, 0x80100020L, 0x00100000L,
-	0x80008020L, 0x80100000L, 0x80108000L, 0x00008000L,
-	0x80100000L, 0x80008000L, 0x00000020L, 0x80108020L,
-	0x00108020L, 0x00000020L, 0x00008000L, 0x80000000L,
-	0x00008020L, 0x80108000L, 0x00100000L, 0x80000020L,
-	0x00100020L, 0x80008020L, 0x80000020L, 0x00100020L,
-	0x00108000L, 0x00000000L, 0x80008000L, 0x00008020L,
-	0x80000000L, 0x80100020L, 0x80108020L, 0x00108000L };
-
-static const unsigned long SP3[64] = {
-	0x00000208L, 0x08020200L, 0x00000000L, 0x08020008L,
-	0x08000200L, 0x00000000L, 0x00020208L, 0x08000200L,
-	0x00020008L, 0x08000008L, 0x08000008L, 0x00020000L,
-	0x08020208L, 0x00020008L, 0x08020000L, 0x00000208L,
-	0x08000000L, 0x00000008L, 0x08020200L, 0x00000200L,
-	0x00020200L, 0x08020000L, 0x08020008L, 0x00020208L,
-	0x08000208L, 0x00020200L, 0x00020000L, 0x08000208L,
-	0x00000008L, 0x08020208L, 0x00000200L, 0x08000000L,
-	0x08020200L, 0x08000000L, 0x00020008L, 0x00000208L,
-	0x00020000L, 0x08020200L, 0x08000200L, 0x00000000L,
-	0x00000200L, 0x00020008L, 0x08020208L, 0x08000200L,
-	0x08000008L, 0x00000200L, 0x00000000L, 0x08020008L,
-	0x08000208L, 0x00020000L, 0x08000000L, 0x08020208L,
-	0x00000008L, 0x00020208L, 0x00020200L, 0x08000008L,
-	0x08020000L, 0x08000208L, 0x00000208L, 0x08020000L,
-	0x00020208L, 0x00000008L, 0x08020008L, 0x00020200L };
-
-static const unsigned long SP4[64] = {
-	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
-	0x00802080L, 0x00800081L, 0x00800001L, 0x00002001L,
-	0x00000000L, 0x00802000L, 0x00802000L, 0x00802081L,
-	0x00000081L, 0x00000000L, 0x00800080L, 0x00800001L,
-	0x00000001L, 0x00002000L, 0x00800000L, 0x00802001L,
-	0x00000080L, 0x00800000L, 0x00002001L, 0x00002080L,
-	0x00800081L, 0x00000001L, 0x00002080L, 0x00800080L,
-	0x00002000L, 0x00802080L, 0x00802081L, 0x00000081L,
-	0x00800080L, 0x00800001L, 0x00802000L, 0x00802081L,
-	0x00000081L, 0x00000000L, 0x00000000L, 0x00802000L,
-	0x00002080L, 0x00800080L, 0x00800081L, 0x00000001L,
-	0x00802001L, 0x00002081L, 0x00002081L, 0x00000080L,
-	0x00802081L, 0x00000081L, 0x00000001L, 0x00002000L,
-	0x00800001L, 0x00002001L, 0x00802080L, 0x00800081L,
-	0x00002001L, 0x00002080L, 0x00800000L, 0x00802001L,
-	0x00000080L, 0x00800000L, 0x00002000L, 0x00802080L };
-
-static const unsigned long SP5[64] = {
-	0x00000100L, 0x02080100L, 0x02080000L, 0x42000100L,
-	0x00080000L, 0x00000100L, 0x40000000L, 0x02080000L,
-	0x40080100L, 0x00080000L, 0x02000100L, 0x40080100L,
-	0x42000100L, 0x42080000L, 0x00080100L, 0x40000000L,
-	0x02000000L, 0x40080000L, 0x40080000L, 0x00000000L,
-	0x40000100L, 0x42080100L, 0x42080100L, 0x02000100L,
-	0x42080000L, 0x40000100L, 0x00000000L, 0x42000000L,
-	0x02080100L, 0x02000000L, 0x42000000L, 0x00080100L,
-	0x00080000L, 0x42000100L, 0x00000100L, 0x02000000L,
-	0x40000000L, 0x02080000L, 0x42000100L, 0x40080100L,
-	0x02000100L, 0x40000000L, 0x42080000L, 0x02080100L,
-	0x40080100L, 0x00000100L, 0x02000000L, 0x42080000L,
-	0x42080100L, 0x00080100L, 0x42000000L, 0x42080100L,
-	0x02080000L, 0x00000000L, 0x40080000L, 0x42000000L,
-	0x00080100L, 0x02000100L, 0x40000100L, 0x00080000L,
-	0x00000000L, 0x40080000L, 0x02080100L, 0x40000100L };
-
-static const unsigned long SP6[64] = {
-	0x20000010L, 0x20400000L, 0x00004000L, 0x20404010L,
-	0x20400000L, 0x00000010L, 0x20404010L, 0x00400000L,
-	0x20004000L, 0x00404010L, 0x00400000L, 0x20000010L,
-	0x00400010L, 0x20004000L, 0x20000000L, 0x00004010L,
-	0x00000000L, 0x00400010L, 0x20004010L, 0x00004000L,
-	0x00404000L, 0x20004010L, 0x00000010L, 0x20400010L,
-	0x20400010L, 0x00000000L, 0x00404010L, 0x20404000L,
-	0x00004010L, 0x00404000L, 0x20404000L, 0x20000000L,
-	0x20004000L, 0x00000010L, 0x20400010L, 0x00404000L,
-	0x20404010L, 0x00400000L, 0x00004010L, 0x20000010L,
-	0x00400000L, 0x20004000L, 0x20000000L, 0x00004010L,
-	0x20000010L, 0x20404010L, 0x00404000L, 0x20400000L,
-	0x00404010L, 0x20404000L, 0x00000000L, 0x20400010L,
-	0x00000010L, 0x00004000L, 0x20400000L, 0x00404010L,
-	0x00004000L, 0x00400010L, 0x20004010L, 0x00000000L,
-	0x20404000L, 0x20000000L, 0x00400010L, 0x20004010L };
-
-static const unsigned long SP7[64] = {
-	0x00200000L, 0x04200002L, 0x04000802L, 0x00000000L,
-	0x00000800L, 0x04000802L, 0x00200802L, 0x04200800L,
-	0x04200802L, 0x00200000L, 0x00000000L, 0x04000002L,
-	0x00000002L, 0x04000000L, 0x04200002L, 0x00000802L,
-	0x04000800L, 0x00200802L, 0x00200002L, 0x04000800L,
-	0x04000002L, 0x04200000L, 0x04200800L, 0x00200002L,
-	0x04200000L, 0x00000800L, 0x00000802L, 0x04200802L,
-	0x00200800L, 0x00000002L, 0x04000000L, 0x00200800L,
-	0x04000000L, 0x00200800L, 0x00200000L, 0x04000802L,
-	0x04000802L, 0x04200002L, 0x04200002L, 0x00000002L,
-	0x00200002L, 0x04000000L, 0x04000800L, 0x00200000L,
-	0x04200800L, 0x00000802L, 0x00200802L, 0x04200800L,
-	0x00000802L, 0x04000002L, 0x04200802L, 0x04200000L,
-	0x00200800L, 0x00000000L, 0x00000002L, 0x04200802L,
-	0x00000000L, 0x00200802L, 0x04200000L, 0x00000800L,
-	0x04000002L, 0x04000800L, 0x00000800L, 0x00200002L };
-
-static const unsigned long SP8[64] = {
-	0x10001040L, 0x00001000L, 0x00040000L, 0x10041040L,
-	0x10000000L, 0x10001040L, 0x00000040L, 0x10000000L,
-	0x00040040L, 0x10040000L, 0x10041040L, 0x00041000L,
-	0x10041000L, 0x00041040L, 0x00001000L, 0x00000040L,
-	0x10040000L, 0x10000040L, 0x10001000L, 0x00001040L,
-	0x00041000L, 0x00040040L, 0x10040040L, 0x10041000L,
-	0x00001040L, 0x00000000L, 0x00000000L, 0x10040040L,
-	0x10000040L, 0x10001000L, 0x00041040L, 0x00040000L,
-	0x00041040L, 0x00040000L, 0x10041000L, 0x00001000L,
-	0x00000040L, 0x10040040L, 0x00001000L, 0x00041040L,
-	0x10001000L, 0x00000040L, 0x10000040L, 0x10040000L,
-	0x10040040L, 0x10000000L, 0x00040000L, 0x10001040L,
-	0x00000000L, 0x10041040L, 0x00040040L, 0x10000040L,
-	0x10040000L, 0x10001000L, 0x10001040L, 0x00000000L,
-	0x10041040L, 0x00041000L, 0x00041000L, 0x00001040L,
-	0x00001040L, 0x00040040L, 0x10000000L, 0x10041000L };
-
-static void desfunc(register unsigned long *block, register unsigned long *keys)
-{
-	register unsigned long fval, work, right, leftt;
-	register int round;
-
-	leftt = block[0];
-	right = block[1];
-	work = ((leftt >> 4) ^ right) & 0x0f0f0f0fL;
-	right ^= work;
-	leftt ^= (work << 4);
-	work = ((leftt >> 16) ^ right) & 0x0000ffffL;
-	right ^= work;
-	leftt ^= (work << 16);
-	work = ((right >> 2) ^ leftt) & 0x33333333L;
-	leftt ^= work;
-	right ^= (work << 2);
-	work = ((right >> 8) ^ leftt) & 0x00ff00ffL;
-	leftt ^= work;
-	right ^= (work << 8);
-	right = ((right << 1) | ((right >> 31) & 1L)) & 0xffffffffL;
-	work = (leftt ^ right) & 0xaaaaaaaaL;
-	leftt ^= work;
-	right ^= work;
-	leftt = ((leftt << 1) | ((leftt >> 31) & 1L)) & 0xffffffffL;
-
-	for( round = 0; round < 8; round++ ) {
-		work  = (right << 28) | (right >> 4);
-		work ^= *keys++;
-		fval  = SP7[ work		 & 0x3fL];
-		fval |= SP5[(work >>  8) & 0x3fL];
-		fval |= SP3[(work >> 16) & 0x3fL];
-		fval |= SP1[(work >> 24) & 0x3fL];
-		work  = right ^ *keys++;
-		fval |= SP8[ work		 & 0x3fL];
-		fval |= SP6[(work >>  8) & 0x3fL];
-		fval |= SP4[(work >> 16) & 0x3fL];
-		fval |= SP2[(work >> 24) & 0x3fL];
-		leftt ^= fval;
-		work  = (leftt << 28) | (leftt >> 4);
-		work ^= *keys++;
-		fval  = SP7[ work		 & 0x3fL];
-		fval |= SP5[(work >>  8) & 0x3fL];
-		fval |= SP3[(work >> 16) & 0x3fL];
-		fval |= SP1[(work >> 24) & 0x3fL];
-		work  = leftt ^ *keys++;
-		fval |= SP8[ work		 & 0x3fL];
-		fval |= SP6[(work >>  8) & 0x3fL];
-		fval |= SP4[(work >> 16) & 0x3fL];
-		fval |= SP2[(work >> 24) & 0x3fL];
-		right ^= fval;
-		}
-
-	right = (right << 31) | (right >> 1);
-	work = (leftt ^ right) & 0xaaaaaaaaL;
-	leftt ^= work;
-	right ^= work;
-	leftt = (leftt << 31) | (leftt >> 1);
-	work = ((leftt >> 8) ^ right) & 0x00ff00ffL;
-	right ^= work;
-	leftt ^= (work << 8);
-	work = ((leftt >> 2) ^ right) & 0x33333333L;
-	right ^= work;
-	leftt ^= (work << 2);
-	work = ((right >> 16) ^ leftt) & 0x0000ffffL;
-	leftt ^= work;
-	right ^= (work << 16);
-	work = ((right >> 4) ^ leftt) & 0x0f0f0f0fL;
-	leftt ^= work;
-	right ^= (work << 4);
-	*block++ = right;
-	*block = leftt;
-	return;
-	}
-
-/* Validation sets:
- *
- * Single-length key, single-length plaintext -
- * Key	  : 0123 4567 89ab cdef
- * Plain  : 0123 4567 89ab cde7
- * Cipher : c957 4425 6a5e d31d
- *
- * Double-length key, single-length plaintext -
- * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
- * Plain  : 0123 4567 89ab cde7
- * Cipher : 7f1d 0a77 826b 8aff
- *
- * Double-length key, double-length plaintext -
- * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210
- * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
- * Cipher : 27a0 8440 406a df60 278f 47cf 42d6 15d7
- *
- * Triple-length key, single-length plaintext -
- * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
- * Plain  : 0123 4567 89ab cde7
- * Cipher : de0b 7c06 ae5e 0ed5
- *
- * Triple-length key, double-length plaintext -
- * Key	  : 0123 4567 89ab cdef fedc ba98 7654 3210 89ab cdef 0123 4567
- * Plain  : 0123 4567 89ab cdef 0123 4567 89ab cdff
- * Cipher : ad0d 1b30 ac17 cf07 0ed1 1c63 81e4 4de5
- *
- * d3des V5.0a rwo 9208.07 18:44 Graven Imagery
- **********************************************************************/
diff --git a/ui/d3des.h b/ui/d3des.h
deleted file mode 100644
index 70cb6b5..0000000
--- a/ui/d3des.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * This is D3DES (V5.09) by Richard Outerbridge with the double and
- * triple-length support removed for use in VNC.
- *
- * These changes are:
- *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
- *
- * This software is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
- */
-#ifndef D3DES_H
-#define D3DES_H 1
-
-/* d3des.h -
- *
- *	Headers and defines for d3des.c
- *	Graven Imagery, 1992.
- *
- * Copyright (c) 1988,1989,1990,1991,1992 by Richard Outerbridge
- *	(GEnie : OUTER; CIS : [71755,204])
- */
-
-#define EN0	0	/* MODE == encrypt */
-#define DE1	1	/* MODE == decrypt */
-
-void deskey(unsigned char *, int);
-/*		      hexkey[8]     MODE
- * Sets the internal key register according to the hexadecimal
- * key contained in the 8 bytes of hexkey, according to the DES,
- * for encryption or decryption according to MODE.
- */
-
-void usekey(unsigned long *);
-/*		    cookedkey[32]
- * Loads the internal key register with the data in cookedkey.
- */
-
-void cpkey(unsigned long *);
-/*		   cookedkey[32]
- * Copies the contents of the internal key register into the storage
- * located at &cookedkey[0].
- */
-
-void des(unsigned char *, unsigned char *);
-/*		    from[8]	      to[8]
- * Encrypts/Decrypts (according to the key currently loaded in the
- * internal key register) one block of eight bytes at address 'from'
- * into the block at address 'to'.  They can be the same.
- */
-
-/* d3des.h V5.09 rwo 9208.04 15:06 Graven Imagery
- ********************************************************************/
-
-#endif
diff --git a/ui/egl-context.c b/ui/egl-context.c
new file mode 100644
index 0000000..3a02b68
--- /dev/null
+++ b/ui/egl-context.c
@@ -0,0 +1,35 @@
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/egl-context.h"
+
+QEMUGLContext qemu_egl_create_context(DisplayChangeListener *dcl,
+                                      QEMUGLParams *params)
+{
+   EGLContext ctx;
+   EGLint ctx_att[] = {
+      EGL_CONTEXT_CLIENT_VERSION, params->major_ver,
+      EGL_CONTEXT_MINOR_VERSION_KHR, params->minor_ver,
+      EGL_NONE
+   };
+
+   ctx = eglCreateContext(qemu_egl_display, qemu_egl_config,
+                          eglGetCurrentContext(), ctx_att);
+   return ctx;
+}
+
+void qemu_egl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    eglDestroyContext(qemu_egl_display, ctx);
+}
+
+int qemu_egl_make_context_current(DisplayChangeListener *dcl,
+                                  QEMUGLContext ctx)
+{
+   return eglMakeCurrent(qemu_egl_display,
+                         EGL_NO_SURFACE, EGL_NO_SURFACE, ctx);
+}
+
+QEMUGLContext qemu_egl_get_current_context(DisplayChangeListener *dcl)
+{
+    return eglGetCurrentContext();
+}
diff --git a/ui/egl-helpers.c b/ui/egl-helpers.c
new file mode 100644
index 0000000..79cee05
--- /dev/null
+++ b/ui/egl-helpers.c
@@ -0,0 +1,267 @@
+#include "qemu/osdep.h"
+#include <glob.h>
+#include <dirent.h>
+
+#include "qemu/error-report.h"
+#include "ui/egl-helpers.h"
+
+EGLDisplay *qemu_egl_display;
+EGLConfig qemu_egl_config;
+
+/* ---------------------------------------------------------------------- */
+
+static bool egl_gles;
+static int egl_debug;
+
+#define egl_dbg(_x ...)                          \
+    do {                                         \
+        if (egl_debug) {                         \
+            fprintf(stderr, "egl: " _x);         \
+        }                                        \
+    } while (0);
+
+/* ---------------------------------------------------------------------- */
+
+#ifdef CONFIG_OPENGL_DMABUF
+
+int qemu_egl_rn_fd;
+struct gbm_device *qemu_egl_rn_gbm_dev;
+EGLContext qemu_egl_rn_ctx;
+
+int qemu_egl_rendernode_open(void)
+{
+    DIR *dir;
+    struct dirent *e;
+    int r, fd;
+    char *p;
+
+    dir = opendir("/dev/dri");
+    if (!dir) {
+        return -1;
+    }
+
+    fd = -1;
+    while ((e = readdir(dir))) {
+        if (e->d_type != DT_CHR) {
+            continue;
+        }
+
+        if (strncmp(e->d_name, "renderD", 7)) {
+            continue;
+        }
+
+        p = g_strdup_printf("/dev/dri/%s", e->d_name);
+
+        r = open(p, O_RDWR | O_CLOEXEC | O_NOCTTY | O_NONBLOCK);
+        if (r < 0) {
+            g_free(p);
+            continue;
+        }
+        fd = r;
+        g_free(p);
+        break;
+    }
+
+    closedir(dir);
+    if (fd < 0) {
+        return -1;
+    }
+    return fd;
+}
+
+int egl_rendernode_init(void)
+{
+    qemu_egl_rn_fd = -1;
+
+    qemu_egl_rn_fd = qemu_egl_rendernode_open();
+    if (qemu_egl_rn_fd == -1) {
+        error_report("egl: no drm render node available");
+        goto err;
+    }
+
+    qemu_egl_rn_gbm_dev = gbm_create_device(qemu_egl_rn_fd);
+    if (!qemu_egl_rn_gbm_dev) {
+        error_report("egl: gbm_create_device failed");
+        goto err;
+    }
+
+    qemu_egl_init_dpy((EGLNativeDisplayType)qemu_egl_rn_gbm_dev, false, false);
+
+    if (!epoxy_has_egl_extension(qemu_egl_display,
+                                 "EGL_KHR_surfaceless_context")) {
+        error_report("egl: EGL_KHR_surfaceless_context not supported");
+        goto err;
+    }
+    if (!epoxy_has_egl_extension(qemu_egl_display,
+                                 "EGL_MESA_image_dma_buf_export")) {
+        error_report("egl: EGL_MESA_image_dma_buf_export not supported");
+        goto err;
+    }
+
+    qemu_egl_rn_ctx = qemu_egl_init_ctx();
+    if (!qemu_egl_rn_ctx) {
+        error_report("egl: egl_init_ctx failed");
+        goto err;
+    }
+
+    return 0;
+
+err:
+    if (qemu_egl_rn_gbm_dev) {
+        gbm_device_destroy(qemu_egl_rn_gbm_dev);
+    }
+    if (qemu_egl_rn_fd != -1) {
+        close(qemu_egl_rn_fd);
+    }
+
+    return -1;
+}
+
+int egl_get_fd_for_texture(uint32_t tex_id, EGLint *stride, EGLint *fourcc)
+{
+    EGLImageKHR image;
+    EGLint num_planes, fd;
+
+    image = eglCreateImageKHR(qemu_egl_display, eglGetCurrentContext(),
+                              EGL_GL_TEXTURE_2D_KHR,
+                              (EGLClientBuffer)(unsigned long)tex_id,
+                              NULL);
+    if (!image) {
+        return -1;
+    }
+
+    eglExportDMABUFImageQueryMESA(qemu_egl_display, image, fourcc,
+                                  &num_planes, NULL);
+    if (num_planes != 1) {
+        eglDestroyImageKHR(qemu_egl_display, image);
+        return -1;
+    }
+    eglExportDMABUFImageMESA(qemu_egl_display, image, &fd, stride, NULL);
+    eglDestroyImageKHR(qemu_egl_display, image);
+
+    return fd;
+}
+
+#endif /* CONFIG_OPENGL_DMABUF */
+
+/* ---------------------------------------------------------------------- */
+
+EGLSurface qemu_egl_init_surface_x11(EGLContext ectx, Window win)
+{
+    EGLSurface esurface;
+    EGLBoolean b;
+
+    egl_dbg("eglCreateWindowSurface (x11 win id 0x%lx) ...\n",
+            (unsigned long) win);
+    esurface = eglCreateWindowSurface(qemu_egl_display,
+                                      qemu_egl_config,
+                                      (EGLNativeWindowType)win, NULL);
+    if (esurface == EGL_NO_SURFACE) {
+        error_report("egl: eglCreateWindowSurface failed");
+        return NULL;
+    }
+
+    b = eglMakeCurrent(qemu_egl_display, esurface, esurface, ectx);
+    if (b == EGL_FALSE) {
+        error_report("egl: eglMakeCurrent failed");
+        return NULL;
+    }
+
+    return esurface;
+}
+
+/* ---------------------------------------------------------------------- */
+
+int qemu_egl_init_dpy(EGLNativeDisplayType dpy, bool gles, bool debug)
+{
+    static const EGLint conf_att_gl[] = {
+        EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
+        EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT,
+        EGL_RED_SIZE,   5,
+        EGL_GREEN_SIZE, 5,
+        EGL_BLUE_SIZE,  5,
+        EGL_ALPHA_SIZE, 0,
+        EGL_NONE,
+    };
+    static const EGLint conf_att_gles[] = {
+        EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
+        EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
+        EGL_RED_SIZE,   5,
+        EGL_GREEN_SIZE, 5,
+        EGL_BLUE_SIZE,  5,
+        EGL_ALPHA_SIZE, 0,
+        EGL_NONE,
+    };
+    EGLint major, minor;
+    EGLBoolean b;
+    EGLint n;
+
+    if (debug) {
+        egl_debug = 1;
+        setenv("EGL_LOG_LEVEL", "debug", true);
+        setenv("LIBGL_DEBUG", "verbose", true);
+    }
+
+    egl_dbg("eglGetDisplay (dpy %p) ...\n", dpy);
+    qemu_egl_display = eglGetDisplay(dpy);
+    if (qemu_egl_display == EGL_NO_DISPLAY) {
+        error_report("egl: eglGetDisplay failed");
+        return -1;
+    }
+
+    egl_dbg("eglInitialize ...\n");
+    b = eglInitialize(qemu_egl_display, &major, &minor);
+    if (b == EGL_FALSE) {
+        error_report("egl: eglInitialize failed");
+        return -1;
+    }
+
+    egl_dbg("eglBindAPI ...\n");
+    b = eglBindAPI(gles ? EGL_OPENGL_ES_API : EGL_OPENGL_API);
+    if (b == EGL_FALSE) {
+        error_report("egl: eglBindAPI failed");
+        return -1;
+    }
+
+    egl_dbg("eglChooseConfig ...\n");
+    b = eglChooseConfig(qemu_egl_display,
+                        gles ? conf_att_gles : conf_att_gl,
+                        &qemu_egl_config, 1, &n);
+    if (b == EGL_FALSE || n != 1) {
+        error_report("egl: eglChooseConfig failed");
+        return -1;
+    }
+
+    egl_gles = gles;
+    return 0;
+}
+
+EGLContext qemu_egl_init_ctx(void)
+{
+    static const EGLint ctx_att_gl[] = {
+        EGL_NONE
+    };
+    static const EGLint ctx_att_gles[] = {
+        EGL_CONTEXT_CLIENT_VERSION, 2,
+        EGL_NONE
+    };
+
+    EGLContext ectx;
+    EGLBoolean b;
+
+    egl_dbg("eglCreateContext ...\n");
+    ectx = eglCreateContext(qemu_egl_display, qemu_egl_config, EGL_NO_CONTEXT,
+                            egl_gles ? ctx_att_gles : ctx_att_gl);
+    if (ectx == EGL_NO_CONTEXT) {
+        error_report("egl: eglCreateContext failed");
+        return NULL;
+    }
+
+    b = eglMakeCurrent(qemu_egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE, ectx);
+    if (b == EGL_FALSE) {
+        error_report("egl: eglMakeCurrent failed");
+        return NULL;
+    }
+
+    return ectx;
+}
diff --git a/ui/gtk-egl.c b/ui/gtk-egl.c
new file mode 100644
index 0000000..3f5d328
--- /dev/null
+++ b/ui/gtk-egl.c
@@ -0,0 +1,258 @@
+/*
+ * GTK UI -- egl opengl code.
+ *
+ * Note that gtk 3.16+ (released 2015-03-23) has a GtkGLArea widget,
+ * which is GtkDrawingArea like widget with opengl rendering support.
+ *
+ * This code handles opengl support on older gtk versions, using egl
+ * to get a opengl context for the X11 window.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "trace.h"
+
+#include "ui/console.h"
+#include "ui/gtk.h"
+#include "ui/egl-helpers.h"
+
+#include "sysemu/sysemu.h"
+
+static void gtk_egl_set_scanout_mode(VirtualConsole *vc, bool scanout)
+{
+    if (vc->gfx.scanout_mode == scanout) {
+        return;
+    }
+
+    vc->gfx.scanout_mode = scanout;
+    if (!vc->gfx.scanout_mode) {
+        if (vc->gfx.fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            glDeleteFramebuffers(1, &vc->gfx.fbo_id);
+            vc->gfx.fbo_id = 0;
+        }
+        if (vc->gfx.surface) {
+            surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+}
+
+/** DisplayState Callbacks (opengl version) **/
+
+void gd_egl_init(VirtualConsole *vc)
+{
+    GdkWindow *gdk_window = gtk_widget_get_window(vc->gfx.drawing_area);
+    if (!gdk_window) {
+        return;
+    }
+
+#if GTK_CHECK_VERSION(3, 0, 0)
+    Window x11_window = gdk_x11_window_get_xid(gdk_window);
+#else
+    Window x11_window = gdk_x11_drawable_get_xid(gdk_window);
+#endif
+    if (!x11_window) {
+        return;
+    }
+
+    vc->gfx.ectx = qemu_egl_init_ctx();
+    vc->gfx.esurface = qemu_egl_init_surface_x11(vc->gfx.ectx, x11_window);
+
+    assert(vc->gfx.esurface);
+}
+
+void gd_egl_draw(VirtualConsole *vc)
+{
+    GdkWindow *window;
+    int ww, wh;
+
+    if (!vc->gfx.gls) {
+        return;
+    }
+
+    if (vc->gfx.scanout_mode) {
+        gd_egl_scanout_flush(&vc->gfx.dcl, 0, 0, vc->gfx.w, vc->gfx.h);
+    } else {
+        if (!vc->gfx.ds) {
+            return;
+        }
+        eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                       vc->gfx.esurface, vc->gfx.ectx);
+
+        window = gtk_widget_get_window(vc->gfx.drawing_area);
+        gdk_drawable_get_size(window, &ww, &wh);
+        surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
+        surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
+
+        eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+    }
+}
+
+void gd_egl_update(DisplayChangeListener *dcl,
+                   int x, int y, int w, int h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls || !vc->gfx.ds) {
+        return;
+    }
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+    surface_gl_update_texture(vc->gfx.gls, vc->gfx.ds, x, y, w, h);
+    vc->gfx.glupdates++;
+}
+
+void gd_egl_refresh(DisplayChangeListener *dcl)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.esurface) {
+        gd_egl_init(vc);
+        if (!vc->gfx.esurface) {
+            return;
+        }
+        vc->gfx.gls = console_gl_init_context();
+        if (vc->gfx.ds) {
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+
+    graphic_hw_update(dcl->con);
+
+    if (vc->gfx.glupdates) {
+        vc->gfx.glupdates = 0;
+        gtk_egl_set_scanout_mode(vc, false);
+        gd_egl_draw(vc);
+    }
+}
+
+void gd_egl_switch(DisplayChangeListener *dcl,
+                   DisplaySurface *surface)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    bool resized = true;
+
+    trace_gd_switch(vc->label, surface_width(surface), surface_height(surface));
+
+    if (vc->gfx.ds &&
+        surface_width(vc->gfx.ds) == surface_width(surface) &&
+        surface_height(vc->gfx.ds) == surface_height(surface)) {
+        resized = false;
+    }
+
+    surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+    vc->gfx.ds = surface;
+    if (vc->gfx.gls) {
+        surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+    }
+
+    if (resized) {
+        gd_update_windowsize(vc);
+    }
+}
+
+QEMUGLContext gd_egl_create_context(DisplayChangeListener *dcl,
+                                    QEMUGLParams *params)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+    return qemu_egl_create_context(dcl, params);
+}
+
+void gd_egl_scanout(DisplayChangeListener *dcl,
+                    uint32_t backing_id, bool backing_y_0_top,
+                    uint32_t backing_width, uint32_t backing_height,
+                    uint32_t x, uint32_t y,
+                    uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    vc->gfx.x = x;
+    vc->gfx.y = y;
+    vc->gfx.w = w;
+    vc->gfx.h = h;
+    vc->gfx.tex_id = backing_id;
+    vc->gfx.y0_top = backing_y_0_top;
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+
+    if (vc->gfx.tex_id == 0 || vc->gfx.w == 0 || vc->gfx.h == 0) {
+        gtk_egl_set_scanout_mode(vc, false);
+        return;
+    }
+
+    gtk_egl_set_scanout_mode(vc, true);
+    if (!vc->gfx.fbo_id) {
+        glGenFramebuffers(1, &vc->gfx.fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, vc->gfx.tex_id, 0);
+}
+
+void gd_egl_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *window;
+    int ww, wh, y1, y2;
+
+    if (!vc->gfx.scanout_mode) {
+        return;
+    }
+    if (!vc->gfx.fbo_id) {
+        return;
+    }
+
+    eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                   vc->gfx.esurface, vc->gfx.ectx);
+
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, vc->gfx.fbo_id);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+
+    window = gtk_widget_get_window(vc->gfx.drawing_area);
+    gdk_drawable_get_size(window, &ww, &wh);
+    glViewport(0, 0, ww, wh);
+    y1 = vc->gfx.y0_top ? 0 : vc->gfx.h;
+    y2 = vc->gfx.y0_top ? vc->gfx.h : 0;
+    glBlitFramebuffer(0, y1, vc->gfx.w, y2,
+                      0, 0, ww, wh,
+                      GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+
+    eglSwapBuffers(qemu_egl_display, vc->gfx.esurface);
+}
+
+void gtk_egl_init(void)
+{
+    GdkDisplay *gdk_display = gdk_display_get_default();
+    Display *x11_display = gdk_x11_display_get_xdisplay(gdk_display);
+
+    if (qemu_egl_init_dpy(x11_display, false, false) < 0) {
+        return;
+    }
+
+    display_opengl = 1;
+}
+
+int gd_egl_make_current(DisplayChangeListener *dcl,
+                        QEMUGLContext ctx)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    return eglMakeCurrent(qemu_egl_display, vc->gfx.esurface,
+                          vc->gfx.esurface, ctx);
+}
diff --git a/ui/gtk-gl-area.c b/ui/gtk-gl-area.c
new file mode 100644
index 0000000..0df5a36
--- /dev/null
+++ b/ui/gtk-gl-area.c
@@ -0,0 +1,225 @@
+/*
+ * GTK UI -- glarea opengl code.
+ *
+ * Requires 3.16+ (GtkGLArea widget).
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "trace.h"
+
+#include "ui/console.h"
+#include "ui/gtk.h"
+#include "ui/egl-helpers.h"
+
+#include "sysemu/sysemu.h"
+
+static void gtk_gl_area_set_scanout_mode(VirtualConsole *vc, bool scanout)
+{
+    if (vc->gfx.scanout_mode == scanout) {
+        return;
+    }
+
+    vc->gfx.scanout_mode = scanout;
+    if (!vc->gfx.scanout_mode) {
+        if (vc->gfx.fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            glDeleteFramebuffers(1, &vc->gfx.fbo_id);
+            vc->gfx.fbo_id = 0;
+        }
+        if (vc->gfx.surface) {
+            surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+}
+
+/** DisplayState Callbacks (opengl version) **/
+
+void gd_gl_area_draw(VirtualConsole *vc)
+{
+    int ww, wh, y1, y2;
+
+    if (!vc->gfx.gls) {
+        return;
+    }
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    ww = gtk_widget_get_allocated_width(vc->gfx.drawing_area);
+    wh = gtk_widget_get_allocated_height(vc->gfx.drawing_area);
+
+    if (vc->gfx.scanout_mode) {
+        if (!vc->gfx.fbo_id) {
+            return;
+        }
+
+        glBindFramebuffer(GL_READ_FRAMEBUFFER, vc->gfx.fbo_id);
+        /* GtkGLArea sets GL_DRAW_FRAMEBUFFER for us */
+
+        glViewport(0, 0, ww, wh);
+        y1 = vc->gfx.y0_top ? 0 : vc->gfx.h;
+        y2 = vc->gfx.y0_top ? vc->gfx.h : 0;
+        glBlitFramebuffer(0, y1, vc->gfx.w, y2,
+                          0, 0, ww, wh,
+                          GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    } else {
+        if (!vc->gfx.ds) {
+            return;
+        }
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+
+        surface_gl_setup_viewport(vc->gfx.gls, vc->gfx.ds, ww, wh);
+        surface_gl_render_texture(vc->gfx.gls, vc->gfx.ds);
+    }
+}
+
+void gd_gl_area_update(DisplayChangeListener *dcl,
+                   int x, int y, int w, int h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls || !vc->gfx.ds) {
+        return;
+    }
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    surface_gl_update_texture(vc->gfx.gls, vc->gfx.ds, x, y, w, h);
+    vc->gfx.glupdates++;
+}
+
+void gd_gl_area_refresh(DisplayChangeListener *dcl)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    if (!vc->gfx.gls) {
+        if (!gtk_widget_get_realized(vc->gfx.drawing_area)) {
+            return;
+        }
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+        vc->gfx.gls = console_gl_init_context();
+        if (vc->gfx.ds) {
+            surface_gl_create_texture(vc->gfx.gls, vc->gfx.ds);
+        }
+    }
+
+    graphic_hw_update(dcl->con);
+
+    if (vc->gfx.glupdates) {
+        vc->gfx.glupdates = 0;
+        gtk_gl_area_set_scanout_mode(vc, false);
+        gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+    }
+}
+
+void gd_gl_area_switch(DisplayChangeListener *dcl,
+                       DisplaySurface *surface)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    bool resized = true;
+
+    trace_gd_switch(vc->label, surface_width(surface), surface_height(surface));
+
+    if (vc->gfx.ds &&
+        surface_width(vc->gfx.ds) == surface_width(surface) &&
+        surface_height(vc->gfx.ds) == surface_height(surface)) {
+        resized = false;
+    }
+
+    if (vc->gfx.gls) {
+        gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+        surface_gl_destroy_texture(vc->gfx.gls, vc->gfx.ds);
+        surface_gl_create_texture(vc->gfx.gls, surface);
+    }
+    vc->gfx.ds = surface;
+
+    if (resized) {
+        gd_update_windowsize(vc);
+    }
+}
+
+QEMUGLContext gd_gl_area_create_context(DisplayChangeListener *dcl,
+                                        QEMUGLParams *params)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *window;
+    GdkGLContext *ctx;
+    GError *err = NULL;
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+    window = gtk_widget_get_window(vc->gfx.drawing_area);
+    ctx = gdk_window_create_gl_context(window, &err);
+    gdk_gl_context_set_required_version(ctx,
+                                        params->major_ver,
+                                        params->minor_ver);
+    gdk_gl_context_realize(ctx, &err);
+    return ctx;
+}
+
+void gd_gl_area_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    /* FIXME */
+}
+
+void gd_gl_area_scanout(DisplayChangeListener *dcl,
+                        uint32_t backing_id, bool backing_y_0_top,
+                        uint32_t backing_width, uint32_t backing_height,
+                        uint32_t x, uint32_t y,
+                        uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    vc->gfx.x = x;
+    vc->gfx.y = y;
+    vc->gfx.w = w;
+    vc->gfx.h = h;
+    vc->gfx.tex_id = backing_id;
+    vc->gfx.y0_top = backing_y_0_top;
+
+    gtk_gl_area_make_current(GTK_GL_AREA(vc->gfx.drawing_area));
+
+    if (vc->gfx.tex_id == 0 || vc->gfx.w == 0 || vc->gfx.h == 0) {
+        gtk_gl_area_set_scanout_mode(vc, false);
+        return;
+    }
+
+    gtk_gl_area_set_scanout_mode(vc, true);
+    if (!vc->gfx.fbo_id) {
+        glGenFramebuffers(1, &vc->gfx.fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, vc->gfx.fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, vc->gfx.tex_id, 0);
+}
+
+void gd_gl_area_scanout_flush(DisplayChangeListener *dcl,
+                          uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+
+    gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+}
+
+void gtk_gl_area_init(void)
+{
+    display_opengl = 1;
+}
+
+QEMUGLContext gd_gl_area_get_current_context(DisplayChangeListener *dcl)
+{
+    return gdk_gl_context_get_current();
+}
+
+int gd_gl_area_make_current(DisplayChangeListener *dcl,
+                            QEMUGLContext ctx)
+{
+    gdk_gl_context_make_current(ctx);
+    return 0;
+}
diff --git a/ui/gtk.c b/ui/gtk.c
index 0385757..58d20ee 100644
--- a/ui/gtk.c
+++ b/ui/gtk.c
@@ -34,24 +34,13 @@
 #define GETTEXT_PACKAGE "qemu"
 #define LOCALEDIR "po"
 
-#ifdef _WIN32
-# define _WIN32_WINNT 0x0601 /* needed to get definition of MAPVK_VK_TO_VSC */
-#endif
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 
-#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
-/* Work around an -Wstrict-prototypes warning in GTK headers */
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wstrict-prototypes"
-#endif
-#include <gtk/gtk.h>
-#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
-#pragma GCC diagnostic pop
-#endif
+#include "ui/console.h"
+#include "ui/gtk.h"
 
-
-#include <gdk/gdkkeysyms.h>
 #include <glib/gi18n.h>
 #include <locale.h>
 #if defined(CONFIG_VTE)
@@ -60,7 +49,6 @@
 #include <math.h>
 
 #include "trace.h"
-#include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/sysemu.h"
 #include "qmp-commands.h"
@@ -68,10 +56,6 @@
 #include "keymaps.h"
 #include "sysemu/char.h"
 #include "qom/object.h"
-#ifdef GDK_WINDOWING_X11
-#include <gdk/gdkx.h>
-#include <X11/XKBlib.h>
-#endif
 
 #define MAX_VCS 10
 #define VC_WINDOW_X_MIN  320
@@ -99,15 +83,6 @@
 # define VTE_RESIZE_HACK 1
 #endif
 
-/* Compatibility define to let us build on both Gtk2 and Gtk3 */
-#if GTK_CHECK_VERSION(3, 0, 0)
-static inline void gdk_drawable_get_size(GdkWindow *w, gint *ww, gint *wh)
-{
-    *ww = gdk_window_get_width(w);
-    *wh = gdk_window_get_height(w);
-}
-#endif
-
 #if !GTK_CHECK_VERSION(2, 20, 0)
 #define gtk_widget_get_realized(widget) GTK_WIDGET_REALIZED(widget)
 #endif
@@ -131,6 +106,15 @@
 #define GDK_KEY_Pause GDK_Pause
 #endif
 
+/* Some older mingw versions lack this constant or have
+ * it conditionally defined */
+#ifdef _WIN32
+# ifndef MAPVK_VK_TO_VSC
+#  define MAPVK_VK_TO_VSC 0
+# endif
+#endif
+
+
 #define HOTKEY_MODIFIERS        (GDK_CONTROL_MASK | GDK_MOD1_MASK)
 
 static const int modifier_keycode[] = {
@@ -138,47 +122,6 @@
     0x2a, 0x36, 0x1d, 0x9d, 0x38, 0xb8, 0xdb, 0xdd,
 };
 
-typedef struct GtkDisplayState GtkDisplayState;
-
-typedef struct VirtualGfxConsole {
-    GtkWidget *drawing_area;
-    DisplayChangeListener dcl;
-    DisplaySurface *ds;
-    pixman_image_t *convert;
-    cairo_surface_t *surface;
-    double scale_x;
-    double scale_y;
-} VirtualGfxConsole;
-
-#if defined(CONFIG_VTE)
-typedef struct VirtualVteConsole {
-    GtkWidget *box;
-    GtkWidget *scrollbar;
-    GtkWidget *terminal;
-    CharDriverState *chr;
-} VirtualVteConsole;
-#endif
-
-typedef enum VirtualConsoleType {
-    GD_VC_GFX,
-    GD_VC_VTE,
-} VirtualConsoleType;
-
-typedef struct VirtualConsole {
-    GtkDisplayState *s;
-    char *label;
-    GtkWidget *window;
-    GtkWidget *menu_item;
-    GtkWidget *tab_item;
-    VirtualConsoleType type;
-    union {
-        VirtualGfxConsole gfx;
-#if defined(CONFIG_VTE)
-        VirtualVteConsole vte;
-#endif
-    };
-} VirtualConsole;
-
 struct GtkDisplayState {
     GtkWidget *window;
 
@@ -196,6 +139,7 @@
     GtkWidget *view_menu_item;
     GtkWidget *view_menu;
     GtkWidget *full_screen_item;
+    GtkWidget *copy_item;
     GtkWidget *zoom_in_item;
     GtkWidget *zoom_out_item;
     GtkWidget *zoom_fixed_item;
@@ -230,10 +174,13 @@
 
     bool modifier_pressed[ARRAY_SIZE(modifier_keycode)];
     bool has_evdev;
+    bool ignore_keys;
 };
 
-static void gd_grab_pointer(VirtualConsole *vc);
+static void gd_grab_pointer(VirtualConsole *vc, const char *reason);
 static void gd_ungrab_pointer(GtkDisplayState *s);
+static void gd_grab_keyboard(VirtualConsole *vc, const char *reason);
+static void gd_ungrab_keyboard(GtkDisplayState *s);
 
 /** Utility Functions **/
 
@@ -290,7 +237,12 @@
     GtkDisplayState *s = vc->s;
     GdkWindow *window;
 
-    if (vc->type != GD_VC_GFX) {
+    if (vc->type != GD_VC_GFX ||
+        !qemu_console_is_graphic(vc->gfx.dcl.con)) {
+        return;
+    }
+
+    if (!gtk_widget_get_realized(vc->gfx.drawing_area)) {
         return;
     }
 
@@ -359,6 +311,9 @@
     GtkWindow *geo_window;
 
     if (vc->type == GD_VC_GFX) {
+        if (!vc->gfx.ds) {
+            return;
+        }
         if (s->free_scale) {
             geo.min_width  = surface_width(vc->gfx.ds) * VC_SCALE_MIN;
             geo.min_height = surface_height(vc->gfx.ds) * VC_SCALE_MIN;
@@ -374,7 +329,23 @@
 #if defined(CONFIG_VTE)
     } else if (vc->type == GD_VC_VTE) {
         VteTerminal *term = VTE_TERMINAL(vc->vte.terminal);
-        GtkBorder *ib;
+        GtkBorder padding = { 0 };
+
+#if VTE_CHECK_VERSION(0, 37, 0)
+        gtk_style_context_get_padding(
+                gtk_widget_get_style_context(vc->vte.terminal),
+                gtk_widget_get_state_flags(vc->vte.terminal),
+                &padding);
+#else
+        {
+            GtkBorder *ib = NULL;
+            gtk_widget_style_get(vc->vte.terminal, "inner-border", &ib, NULL);
+            if (ib) {
+                padding = *ib;
+                gtk_border_free(ib);
+            }
+        }
+#endif
 
         geo.width_inc  = vte_terminal_get_char_width(term);
         geo.height_inc = vte_terminal_get_char_height(term);
@@ -385,11 +356,11 @@
         geo.min_width  = geo.width_inc * VC_TERM_X_MIN;
         geo.min_height = geo.height_inc * VC_TERM_Y_MIN;
         mask |= GDK_HINT_MIN_SIZE;
-        gtk_widget_style_get(vc->vte.terminal, "inner-border", &ib, NULL);
-        geo.base_width  += ib->left + ib->right;
-        geo.base_height += ib->top + ib->bottom;
-        geo.min_width   += ib->left + ib->right;
-        geo.min_height  += ib->top + ib->bottom;
+
+        geo.base_width  += padding.left + padding.right;
+        geo.base_height += padding.top + padding.bottom;
+        geo.min_width   += padding.left + padding.right;
+        geo.min_height  += padding.top + padding.bottom;
         geo_widget = vc->vte.terminal;
 #endif
     }
@@ -398,7 +369,7 @@
     gtk_window_set_geometry_hints(geo_window, geo_widget, &geo, mask);
 }
 
-static void gd_update_windowsize(VirtualConsole *vc)
+void gd_update_windowsize(VirtualConsole *vc)
 {
     GtkDisplayState *s = vc->s;
 
@@ -415,6 +386,12 @@
     GtkWidget *area = vc->gfx.drawing_area;
     int ww, wh;
     gdk_drawable_get_size(gtk_widget_get_window(area), &ww, &wh);
+#if defined(CONFIG_GTK_GL)
+    if (vc->gfx.gls) {
+        gtk_gl_area_queue_render(GTK_GL_AREA(vc->gfx.drawing_area));
+        return;
+    }
+#endif
     gtk_widget_queue_draw_area(area, 0, 0, ww, wh);
 }
 
@@ -423,7 +400,8 @@
     VirtualConsole *vc = gd_vc_find_current(s);
     int i, keycode;
 
-    if (vc->type != GD_VC_GFX) {
+    if (vc->type != GD_VC_GFX ||
+        !qemu_console_is_graphic(vc->gfx.dcl.con)) {
         return;
     }
     for (i = 0; i < ARRAY_SIZE(modifier_keycode); i++) {
@@ -451,6 +429,7 @@
                       int x, int y, int w, int h)
 {
     VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
+    GdkWindow *win;
     int x1, x2, y1, y2;
     int mx, my;
     int fbw, fbh;
@@ -458,6 +437,10 @@
 
     trace_gd_update(vc->label, x, y, w, h);
 
+    if (!gtk_widget_get_realized(vc->gfx.drawing_area)) {
+        return;
+    }
+
     if (vc->gfx.convert) {
         pixman_image_composite(PIXMAN_OP_SRC, vc->gfx.ds->image,
                                NULL, vc->gfx.convert,
@@ -473,8 +456,11 @@
     fbw = surface_width(vc->gfx.ds) * vc->gfx.scale_x;
     fbh = surface_height(vc->gfx.ds) * vc->gfx.scale_y;
 
-    gdk_drawable_get_size(gtk_widget_get_window(vc->gfx.drawing_area),
-                          &ww, &wh);
+    win = gtk_widget_get_window(vc->gfx.drawing_area);
+    if (!win) {
+        return;
+    }
+    gdk_drawable_get_size(win, &ww, &wh);
 
     mx = my = 0;
     if (ww > fbw) {
@@ -494,12 +480,21 @@
 }
 
 #if GTK_CHECK_VERSION(3, 0, 0)
+static GdkDevice *gd_get_pointer(GdkDisplay *dpy)
+{
+#if GTK_CHECK_VERSION(3, 20, 0)
+    return gdk_seat_get_pointer(gdk_display_get_default_seat(dpy));
+#else
+    return gdk_device_manager_get_client_pointer(
+        gdk_display_get_device_manager(dpy));
+#endif
+}
+
 static void gd_mouse_set(DisplayChangeListener *dcl,
                          int x, int y, int visible)
 {
     VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     GdkDisplay *dpy;
-    GdkDeviceManager *mgr;
     gint x_root, y_root;
 
     if (qemu_input_is_absolute()) {
@@ -507,12 +502,13 @@
     }
 
     dpy = gtk_widget_get_display(vc->gfx.drawing_area);
-    mgr = gdk_display_get_device_manager(dpy);
     gdk_window_get_root_coords(gtk_widget_get_window(vc->gfx.drawing_area),
                                x, y, &x_root, &y_root);
-    gdk_device_warp(gdk_device_manager_get_client_pointer(mgr),
+    gdk_device_warp(gd_get_pointer(dpy),
                     gtk_widget_get_screen(vc->gfx.drawing_area),
                     x_root, y_root);
+    vc->s->last_x = x;
+    vc->s->last_y = y;
 }
 #else
 static void gd_mouse_set(DisplayChangeListener *dcl,
@@ -540,6 +536,10 @@
     GdkPixbuf *pixbuf;
     GdkCursor *cursor;
 
+    if (!gtk_widget_get_realized(vc->gfx.drawing_area)) {
+        return;
+    }
+
     pixbuf = gdk_pixbuf_new_from_data((guchar *)(c->data),
                                       GDK_COLORSPACE_RGB, true, 8,
                                       c->width, c->height, c->width * 4,
@@ -562,22 +562,28 @@
     VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     bool resized = true;
 
-    trace_gd_switch(vc->label, surface_width(surface), surface_height(surface));
+    trace_gd_switch(vc->label,
+                    surface ? surface_width(surface)  : 0,
+                    surface ? surface_height(surface) : 0);
 
     if (vc->gfx.surface) {
         cairo_surface_destroy(vc->gfx.surface);
+        vc->gfx.surface = NULL;
+    }
+    if (vc->gfx.convert) {
+        pixman_image_unref(vc->gfx.convert);
+        vc->gfx.convert = NULL;
     }
 
-    if (vc->gfx.ds &&
+    if (vc->gfx.ds && surface &&
         surface_width(vc->gfx.ds) == surface_width(surface) &&
         surface_height(vc->gfx.ds) == surface_height(surface)) {
         resized = false;
     }
     vc->gfx.ds = surface;
 
-    if (vc->gfx.convert) {
-        pixman_image_unref(vc->gfx.convert);
-        vc->gfx.convert = NULL;
+    if (!surface) {
+        return;
     }
 
     if (surface->format == PIXMAN_x8r8g8b8) {
@@ -619,6 +625,62 @@
     }
 }
 
+static const DisplayChangeListenerOps dcl_ops = {
+    .dpy_name             = "gtk",
+    .dpy_gfx_update       = gd_update,
+    .dpy_gfx_switch       = gd_switch,
+    .dpy_gfx_check_format = qemu_pixman_check_format,
+    .dpy_refresh          = gd_refresh,
+    .dpy_mouse_set        = gd_mouse_set,
+    .dpy_cursor_define    = gd_cursor_define,
+};
+
+
+#if defined(CONFIG_OPENGL)
+
+/** DisplayState Callbacks (opengl version) **/
+
+#if defined(CONFIG_GTK_GL)
+
+static const DisplayChangeListenerOps dcl_gl_area_ops = {
+    .dpy_name             = "gtk-egl",
+    .dpy_gfx_update       = gd_gl_area_update,
+    .dpy_gfx_switch       = gd_gl_area_switch,
+    .dpy_gfx_check_format = console_gl_check_format,
+    .dpy_refresh          = gd_gl_area_refresh,
+    .dpy_mouse_set        = gd_mouse_set,
+    .dpy_cursor_define    = gd_cursor_define,
+
+    .dpy_gl_ctx_create       = gd_gl_area_create_context,
+    .dpy_gl_ctx_destroy      = gd_gl_area_destroy_context,
+    .dpy_gl_ctx_make_current = gd_gl_area_make_current,
+    .dpy_gl_ctx_get_current  = gd_gl_area_get_current_context,
+    .dpy_gl_scanout          = gd_gl_area_scanout,
+    .dpy_gl_update           = gd_gl_area_scanout_flush,
+};
+
+#else
+
+static const DisplayChangeListenerOps dcl_egl_ops = {
+    .dpy_name             = "gtk-egl",
+    .dpy_gfx_update       = gd_egl_update,
+    .dpy_gfx_switch       = gd_egl_switch,
+    .dpy_gfx_check_format = console_gl_check_format,
+    .dpy_refresh          = gd_egl_refresh,
+    .dpy_mouse_set        = gd_mouse_set,
+    .dpy_cursor_define    = gd_cursor_define,
+
+    .dpy_gl_ctx_create       = gd_egl_create_context,
+    .dpy_gl_ctx_destroy      = qemu_egl_destroy_context,
+    .dpy_gl_ctx_make_current = gd_egl_make_current,
+    .dpy_gl_ctx_get_current  = qemu_egl_get_current_context,
+    .dpy_gl_scanout          = gd_egl_scanout,
+    .dpy_gl_update           = gd_egl_scanout_flush,
+};
+
+#endif /* CONFIG_GTK_GL */
+#endif /* CONFIG_OPENGL */
+
 /** QEMU Events **/
 
 static void gd_change_runstate(void *opaque, int running, RunState state)
@@ -667,6 +729,39 @@
     return TRUE;
 }
 
+static void gd_set_ui_info(VirtualConsole *vc, gint width, gint height)
+{
+    QemuUIInfo info;
+
+    memset(&info, 0, sizeof(info));
+    info.width = width;
+    info.height = height;
+    dpy_set_ui_info(vc->gfx.dcl.con, &info);
+}
+
+#if defined(CONFIG_GTK_GL)
+
+static gboolean gd_render_event(GtkGLArea *area, GdkGLContext *context,
+                                void *opaque)
+{
+    VirtualConsole *vc = opaque;
+
+    if (vc->gfx.gls) {
+        gd_gl_area_draw(vc);
+    }
+    return TRUE;
+}
+
+static void gd_resize_event(GtkGLArea *area,
+                            gint width, gint height, gpointer *opaque)
+{
+    VirtualConsole *vc = (void *)opaque;
+
+    gd_set_ui_info(vc, width, height);
+}
+
+#endif
+
 static gboolean gd_draw_event(GtkWidget *widget, cairo_t *cr, void *opaque)
 {
     VirtualConsole *vc = opaque;
@@ -675,9 +770,24 @@
     int ww, wh;
     int fbw, fbh;
 
+#if defined(CONFIG_OPENGL)
+    if (vc->gfx.gls) {
+#if defined(CONFIG_GTK_GL)
+        /* invoke render callback please */
+        return FALSE;
+#else
+        gd_egl_draw(vc);
+        return TRUE;
+#endif
+    }
+#endif
+
     if (!gtk_widget_get_realized(widget)) {
         return FALSE;
     }
+    if (!vc->gfx.ds) {
+        return FALSE;
+    }
 
     fbw = surface_width(vc->gfx.ds);
     fbh = surface_height(vc->gfx.ds);
@@ -759,6 +869,10 @@
     int fbh, fbw;
     int ww, wh;
 
+    if (!vc->gfx.ds) {
+        return TRUE;
+    }
+
     fbw = surface_width(vc->gfx.ds) * vc->gfx.scale_x;
     fbh = surface_height(vc->gfx.ds) * vc->gfx.scale_y;
 
@@ -846,13 +960,11 @@
     /* implicitly grab the input at the first click in the relative mode */
     if (button->button == 1 && button->type == GDK_BUTTON_PRESS &&
         !qemu_input_is_absolute() && s->ptr_owner != vc) {
-        gd_ungrab_pointer(s);
         if (!vc->window) {
             gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                            TRUE);
         } else {
-            gd_grab_pointer(vc);
-            gd_update_caption(s);
+            gd_grab_pointer(vc, "relative-mode-click");
         }
         return TRUE;
     }
@@ -933,6 +1045,23 @@
     return qemu_keycode;
 }
 
+static gboolean gd_text_key_down(GtkWidget *widget,
+                                 GdkEventKey *key, void *opaque)
+{
+    VirtualConsole *vc = opaque;
+    QemuConsole *con = vc->gfx.dcl.con;
+
+    if (key->length) {
+        kbd_put_string_console(con, key->string, key->length);
+    } else {
+        int num = gd_map_keycode(vc->s, gtk_widget_get_display(widget),
+                                 key->hardware_keycode);
+        int qcode = qemu_input_key_number_to_qcode(num);
+        kbd_put_qcode_console(con, qcode);
+    }
+    return TRUE;
+}
+
 static gboolean gd_key_event(GtkWidget *widget, GdkEventKey *key, void *opaque)
 {
     VirtualConsole *vc = opaque;
@@ -941,6 +1070,11 @@
     int qemu_keycode;
     int i;
 
+    if (s->ignore_keys) {
+        s->ignore_keys = (key->type == GDK_KEY_PRESS);
+        return TRUE;
+    }
+
     if (key->keyval == GDK_KEY_Pause) {
         qemu_input_event_send_key_qcode(vc->gfx.dcl.con, Q_KEY_CODE_PAUSE,
                                         key->type == GDK_KEY_PRESS);
@@ -1009,22 +1143,26 @@
     GtkDisplayState *s = opaque;
     VirtualConsole *vc = gd_vc_find_by_menu(s);
     GtkNotebook *nb = GTK_NOTEBOOK(s->notebook);
-    GtkWidget *child;
     gint page;
 
     gtk_release_modifiers(s);
     if (vc) {
         page = gtk_notebook_page_num(nb, vc->tab_item);
         gtk_notebook_set_current_page(nb, page);
-        child = gtk_notebook_get_nth_page(nb, page);
-        gtk_widget_grab_focus(child);
+        gtk_widget_grab_focus(vc->focus);
     }
+    s->ignore_keys = false;
 }
 
 static void gd_accel_switch_vc(void *opaque)
 {
     VirtualConsole *vc = opaque;
+
     gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(vc->menu_item), TRUE);
+#if !GTK_CHECK_VERSION(3, 0, 0)
+    /* GTK2 sends the accel key to the target console - ignore this until */
+    vc->s->ignore_keys = true;
+#endif
 }
 
 static void gd_menu_show_tabs(GtkMenuItem *item, void *opaque)
@@ -1063,9 +1201,8 @@
     if (vc->s->ptr_owner) {
         gd_ungrab_pointer(vc->s);
     } else {
-        gd_grab_pointer(vc);
+        gd_grab_pointer(vc, "user-request-detached-tab");
     }
-    gd_update_caption(vc->s);
     return TRUE;
 }
 
@@ -1074,7 +1211,8 @@
     GtkDisplayState *s = opaque;
     VirtualConsole *vc = gd_vc_find_current(s);
 
-    if (vc->type == GD_VC_GFX) {
+    if (vc->type == GD_VC_GFX &&
+        qemu_console_is_graphic(vc->gfx.dcl.con)) {
         gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                        FALSE);
     }
@@ -1087,11 +1225,14 @@
                          G_CALLBACK(gd_tab_window_close), vc);
         gtk_widget_show_all(vc->window);
 
-        GtkAccelGroup *ag = gtk_accel_group_new();
-        gtk_window_add_accel_group(GTK_WINDOW(vc->window), ag);
+        if (qemu_console_is_graphic(vc->gfx.dcl.con)) {
+            GtkAccelGroup *ag = gtk_accel_group_new();
+            gtk_window_add_accel_group(GTK_WINDOW(vc->window), ag);
 
-        GClosure *cb = g_cclosure_new_swap(G_CALLBACK(gd_win_grab), vc, NULL);
-        gtk_accel_group_connect(ag, GDK_KEY_g, HOTKEY_MODIFIERS, 0, cb);
+            GClosure *cb = g_cclosure_new_swap(G_CALLBACK(gd_win_grab),
+                                               vc, NULL);
+            gtk_accel_group_connect(ag, GDK_KEY_g, HOTKEY_MODIFIERS, 0, cb);
+        }
 
         gd_update_geometry_hints(vc);
         gd_update_caption(s);
@@ -1108,8 +1249,6 @@
         gtk_widget_hide(s->menu_bar);
         if (vc->type == GD_VC_GFX) {
             gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
-            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
-                                           TRUE);
         }
         gtk_window_fullscreen(GTK_WINDOW(s->window));
         s->full_screen = TRUE;
@@ -1122,8 +1261,6 @@
             vc->gfx.scale_x = 1.0;
             vc->gfx.scale_y = 1.0;
             gd_update_windowsize(vc);
-            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
-                                           FALSE);
         }
     }
 
@@ -1195,7 +1332,31 @@
     gd_update_full_redraw(vc);
 }
 
-#if GTK_CHECK_VERSION(3, 0, 0)
+#if GTK_CHECK_VERSION(3, 20, 0)
+static void gd_grab_update(VirtualConsole *vc, bool kbd, bool ptr)
+{
+    GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
+    GdkSeat *seat = gdk_display_get_default_seat(display);
+    GdkWindow *window = gtk_widget_get_window(vc->gfx.drawing_area);
+    GdkSeatCapabilities caps = 0;
+    GdkCursor *cursor = NULL;
+
+    if (kbd) {
+        caps |= GDK_SEAT_CAPABILITY_KEYBOARD;
+    }
+    if (ptr) {
+        caps |= GDK_SEAT_CAPABILITY_ALL_POINTING;
+        cursor = vc->s->null_cursor;
+    }
+
+    if (caps) {
+        gdk_seat_grab(seat, window, caps, false, cursor,
+                      NULL, NULL, NULL);
+    } else {
+        gdk_seat_ungrab(seat);
+    }
+}
+#elif GTK_CHECK_VERSION(3, 0, 0)
 static void gd_grab_devices(VirtualConsole *vc, bool grab,
                             GdkInputSource source, GdkEventMask mask,
                             GdkCursor *cursor)
@@ -1222,9 +1383,19 @@
 }
 #endif
 
-static void gd_grab_keyboard(VirtualConsole *vc)
+static void gd_grab_keyboard(VirtualConsole *vc, const char *reason)
 {
-#if GTK_CHECK_VERSION(3, 0, 0)
+    if (vc->s->kbd_owner) {
+        if (vc->s->kbd_owner == vc) {
+            return;
+        } else {
+            gd_ungrab_keyboard(vc->s);
+        }
+    }
+
+#if GTK_CHECK_VERSION(3, 20, 0)
+    gd_grab_update(vc, true, vc->s->ptr_owner == vc);
+#elif GTK_CHECK_VERSION(3, 0, 0)
     gd_grab_devices(vc, true, GDK_SOURCE_KEYBOARD,
                    GDK_KEY_PRESS_MASK | GDK_KEY_RELEASE_MASK,
                    NULL);
@@ -1234,7 +1405,8 @@
                       GDK_CURRENT_TIME);
 #endif
     vc->s->kbd_owner = vc;
-    trace_gd_grab(vc->label, "kbd", true);
+    gd_update_caption(vc->s);
+    trace_gd_grab(vc->label, "kbd", reason);
 }
 
 static void gd_ungrab_keyboard(GtkDisplayState *s)
@@ -1246,19 +1418,34 @@
     }
     s->kbd_owner = NULL;
 
-#if GTK_CHECK_VERSION(3, 0, 0)
+#if GTK_CHECK_VERSION(3, 20, 0)
+    gd_grab_update(vc, false, vc->s->ptr_owner == vc);
+#elif GTK_CHECK_VERSION(3, 0, 0)
     gd_grab_devices(vc, false, GDK_SOURCE_KEYBOARD, 0, NULL);
 #else
     gdk_keyboard_ungrab(GDK_CURRENT_TIME);
 #endif
-    trace_gd_grab(vc->label, "kbd", false);
+    gd_update_caption(s);
+    trace_gd_ungrab(vc->label, "kbd");
 }
 
-static void gd_grab_pointer(VirtualConsole *vc)
+static void gd_grab_pointer(VirtualConsole *vc, const char *reason)
 {
     GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
-#if GTK_CHECK_VERSION(3, 0, 0)
-    GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
+
+    if (vc->s->ptr_owner) {
+        if (vc->s->ptr_owner == vc) {
+            return;
+        } else {
+            gd_ungrab_pointer(vc->s);
+        }
+    }
+
+#if GTK_CHECK_VERSION(3, 20, 0)
+    gd_grab_update(vc, vc->s->kbd_owner == vc, true);
+    gdk_device_get_position(gd_get_pointer(display),
+                            NULL, &vc->s->grab_x_root, &vc->s->grab_y_root);
+#elif GTK_CHECK_VERSION(3, 0, 0)
     gd_grab_devices(vc, true, GDK_SOURCE_MOUSE,
                     GDK_POINTER_MOTION_MASK |
                     GDK_BUTTON_PRESS_MASK |
@@ -1266,7 +1453,7 @@
                     GDK_BUTTON_MOTION_MASK |
                     GDK_SCROLL_MASK,
                     vc->s->null_cursor);
-    gdk_device_get_position(gdk_device_manager_get_client_pointer(mgr),
+    gdk_device_get_position(gd_get_pointer(display),
                             NULL, &vc->s->grab_x_root, &vc->s->grab_y_root);
 #else
     gdk_pointer_grab(gtk_widget_get_window(vc->gfx.drawing_area),
@@ -1283,23 +1470,29 @@
                             &vc->s->grab_x_root, &vc->s->grab_y_root, NULL);
 #endif
     vc->s->ptr_owner = vc;
-    trace_gd_grab(vc->label, "ptr", true);
+    gd_update_caption(vc->s);
+    trace_gd_grab(vc->label, "ptr", reason);
 }
 
 static void gd_ungrab_pointer(GtkDisplayState *s)
 {
     VirtualConsole *vc = s->ptr_owner;
+    GdkDisplay *display;
 
     if (vc == NULL) {
         return;
     }
     s->ptr_owner = NULL;
 
-    GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
-#if GTK_CHECK_VERSION(3, 0, 0)
-    GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
+    display = gtk_widget_get_display(vc->gfx.drawing_area);
+#if GTK_CHECK_VERSION(3, 20, 0)
+    gd_grab_update(vc, vc->s->kbd_owner == vc, false);
+    gdk_device_warp(gd_get_pointer(display),
+                    gtk_widget_get_screen(vc->gfx.drawing_area),
+                    vc->s->grab_x_root, vc->s->grab_y_root);
+#elif GTK_CHECK_VERSION(3, 0, 0)
     gd_grab_devices(vc, false, GDK_SOURCE_MOUSE, 0, NULL);
-    gdk_device_warp(gdk_device_manager_get_client_pointer(mgr),
+    gdk_device_warp(gd_get_pointer(display),
                     gtk_widget_get_screen(vc->gfx.drawing_area),
                     vc->s->grab_x_root, vc->s->grab_y_root);
 #else
@@ -1308,7 +1501,8 @@
                              gtk_widget_get_screen(vc->gfx.drawing_area),
                              vc->s->grab_x_root, vc->s->grab_y_root);
 #endif
-    trace_gd_grab(vc->label, "ptr", false);
+    gd_update_caption(s);
+    trace_gd_ungrab(vc->label, "ptr");
 }
 
 static void gd_menu_grab_input(GtkMenuItem *item, void *opaque)
@@ -1317,16 +1511,13 @@
     VirtualConsole *vc = gd_vc_find_current(s);
 
     if (gd_is_grab_active(s)) {
-        if (!gd_grab_on_hover(s)) {
-            gd_grab_keyboard(vc);
-        }
-        gd_grab_pointer(vc);
+        gd_grab_keyboard(vc, "user-request-main-window");
+        gd_grab_pointer(vc, "user-request-main-window");
     } else {
         gd_ungrab_keyboard(s);
         gd_ungrab_pointer(s);
     }
 
-    gd_update_caption(s);
     gd_update_cursor(vc);
 }
 
@@ -1358,7 +1549,8 @@
 #endif
     gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(vc->menu_item),
                                    TRUE);
-    on_vga = (vc->type == GD_VC_GFX);
+    on_vga = (vc->type == GD_VC_GFX &&
+              qemu_console_is_graphic(vc->gfx.dcl.con));
     if (!on_vga) {
         gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                        FALSE);
@@ -1379,9 +1571,7 @@
     GtkDisplayState *s = vc->s;
 
     if (gd_grab_on_hover(s)) {
-        gd_ungrab_keyboard(s);
-        gd_grab_keyboard(vc);
-        gd_update_caption(s);
+        gd_grab_keyboard(vc, "grab-on-hover");
     }
     return TRUE;
 }
@@ -1394,7 +1584,6 @@
 
     if (gd_grab_on_hover(s)) {
         gd_ungrab_keyboard(s);
-        gd_update_caption(s);
     }
     return TRUE;
 }
@@ -1409,6 +1598,15 @@
     return TRUE;
 }
 
+static gboolean gd_configure(GtkWidget *widget,
+                             GdkEventConfigure *cfg, gpointer opaque)
+{
+    VirtualConsole *vc = opaque;
+
+    gd_set_ui_info(vc, cfg->width, cfg->height);
+    return FALSE;
+}
+
 /** Virtual Console Callbacks **/
 
 static GSList *gd_vc_menu_init(GtkDisplayState *s, VirtualConsole *vc,
@@ -1433,6 +1631,14 @@
 }
 
 #if defined(CONFIG_VTE)
+static void gd_menu_copy(GtkMenuItem *item, void *opaque)
+{
+    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
+
+    vte_terminal_copy_clipboard(VTE_TERMINAL(vc->vte.terminal));
+}
+
 static void gd_vc_adjustment_changed(GtkAdjustment *adjustment, void *opaque)
 {
     VirtualConsole *vc = opaque;
@@ -1453,15 +1659,32 @@
     return len;
 }
 
+static void gd_vc_chr_set_echo(CharDriverState *chr, bool echo)
+{
+    VirtualConsole *vc = chr->opaque;
+
+    vc->vte.echo = echo;
+}
+
 static int nb_vcs;
 static CharDriverState *vcs[MAX_VCS];
 
-static CharDriverState *gd_vc_handler(ChardevVC *unused)
+static CharDriverState *gd_vc_handler(ChardevVC *vc, Error **errp)
 {
+    ChardevCommon *common = qapi_ChardevVC_base(vc);
     CharDriverState *chr;
 
-    chr = g_malloc0(sizeof(*chr));
+    chr = qemu_chr_alloc(common, errp);
+    if (!chr) {
+        return NULL;
+    }
+
     chr->chr_write = gd_vc_chr_write;
+    chr->chr_set_echo = gd_vc_chr_set_echo;
+
+    /* Temporary, until gd_vc_vte_init runs.  */
+    chr->opaque = g_new0(VirtualConsole, 1);
+
     /* defer OPENED events until our vc is fully initialized */
     chr->explicit_be_open = true;
 
@@ -1475,6 +1698,24 @@
 {
     VirtualConsole *vc = user_data;
 
+    if (vc->vte.echo) {
+        VteTerminal *term = VTE_TERMINAL(vc->vte.terminal);
+        int i;
+        for (i = 0; i < size; i++) {
+            uint8_t c = text[i];
+            if (c >= 128 || isprint(c)) {
+                /* 8-bit characters are considered printable.  */
+                vte_terminal_feed(term, &text[i], 1);
+            } else if (c == '\r' || c == '\n') {
+                vte_terminal_feed(term, "\r\n", 2);
+            } else {
+                char ctrl[2] = { '^', 0};
+                ctrl[1] = text[i] ^ 64;
+                vte_terminal_feed(term, ctrl, 2);
+            }
+        }
+    }
+
     qemu_chr_be_write(vc->vte.chr, (uint8_t  *)text, (unsigned int)size);
     return TRUE;
 }
@@ -1487,9 +1728,14 @@
     GtkWidget *box;
     GtkWidget *scrollbar;
     GtkAdjustment *vadjustment;
+    VirtualConsole *tmp_vc = chr->opaque;
 
     vc->s = s;
+    vc->vte.echo = tmp_vc->vte.echo;
+
     vc->vte.chr = chr;
+    chr->opaque = vc;
+    g_free(tmp_vc);
 
     snprintf(buffer, sizeof(buffer), "vc%d", idx);
     vc->label = g_strdup_printf("%s", vc->vte.chr->label
@@ -1499,6 +1745,15 @@
     vc->vte.terminal = vte_terminal_new();
     g_signal_connect(vc->vte.terminal, "commit", G_CALLBACK(gd_vc_in), vc);
 
+    /* The documentation says that the default is UTF-8, but actually it is
+     * 7-bit ASCII at least in VTE 0.38.
+     */
+#if VTE_CHECK_VERSION(0, 38, 0)
+    vte_terminal_set_encoding(VTE_TERMINAL(vc->vte.terminal), "UTF-8", NULL);
+#else
+    vte_terminal_set_encoding(VTE_TERMINAL(vc->vte.terminal), "UTF-8");
+#endif
+
     vte_terminal_set_scrollback_lines(VTE_TERMINAL(vc->vte.terminal), -1);
     vte_terminal_set_size(VTE_TERMINAL(vc->vte.terminal),
                           VC_TERM_X_MIN, VC_TERM_Y_MIN);
@@ -1521,7 +1776,6 @@
     gtk_box_pack_start(GTK_BOX(box), vc->vte.terminal, TRUE, TRUE, 0);
     gtk_box_pack_start(GTK_BOX(box), scrollbar, FALSE, FALSE, 0);
 
-    vc->vte.chr->opaque = vc;
     vc->vte.box = box;
     vc->vte.scrollbar = scrollbar;
 
@@ -1530,6 +1784,7 @@
 
     vc->type = GD_VC_VTE;
     vc->tab_item = box;
+    vc->focus = vc->vte.terminal;
     gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook), vc->tab_item,
                              gtk_label_new(vc->label));
 
@@ -1561,29 +1816,45 @@
 #if GTK_CHECK_VERSION(3, 0, 0)
     g_signal_connect(vc->gfx.drawing_area, "draw",
                      G_CALLBACK(gd_draw_event), vc);
+#if defined(CONFIG_GTK_GL)
+    if (display_opengl) {
+        /* wire up GtkGlArea events */
+        g_signal_connect(vc->gfx.drawing_area, "render",
+                         G_CALLBACK(gd_render_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "resize",
+                         G_CALLBACK(gd_resize_event), vc);
+    }
+#endif
 #else
     g_signal_connect(vc->gfx.drawing_area, "expose-event",
                      G_CALLBACK(gd_expose_event), vc);
 #endif
-    g_signal_connect(vc->gfx.drawing_area, "event",
-                     G_CALLBACK(gd_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "button-press-event",
-                     G_CALLBACK(gd_button_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "button-release-event",
-                     G_CALLBACK(gd_button_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "scroll-event",
-                     G_CALLBACK(gd_scroll_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "key-press-event",
-                     G_CALLBACK(gd_key_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "key-release-event",
-                     G_CALLBACK(gd_key_event), vc);
+    if (qemu_console_is_graphic(vc->gfx.dcl.con)) {
+        g_signal_connect(vc->gfx.drawing_area, "event",
+                         G_CALLBACK(gd_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "button-press-event",
+                         G_CALLBACK(gd_button_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "button-release-event",
+                         G_CALLBACK(gd_button_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "scroll-event",
+                         G_CALLBACK(gd_scroll_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "key-press-event",
+                         G_CALLBACK(gd_key_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "key-release-event",
+                         G_CALLBACK(gd_key_event), vc);
 
-    g_signal_connect(vc->gfx.drawing_area, "enter-notify-event",
-                     G_CALLBACK(gd_enter_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "leave-notify-event",
-                     G_CALLBACK(gd_leave_event), vc);
-    g_signal_connect(vc->gfx.drawing_area, "focus-out-event",
-                     G_CALLBACK(gd_focus_out_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "enter-notify-event",
+                         G_CALLBACK(gd_enter_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "leave-notify-event",
+                         G_CALLBACK(gd_leave_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "focus-out-event",
+                         G_CALLBACK(gd_focus_out_event), vc);
+        g_signal_connect(vc->gfx.drawing_area, "configure-event",
+                         G_CALLBACK(gd_configure), vc);
+    } else {
+        g_signal_connect(vc->gfx.drawing_area, "key-press-event",
+                         G_CALLBACK(gd_text_key_down), vc);
+    }
 }
 
 static void gd_connect_signals(GtkDisplayState *s)
@@ -1604,6 +1875,10 @@
                      G_CALLBACK(gd_menu_powerdown), s);
     g_signal_connect(s->quit_item, "activate",
                      G_CALLBACK(gd_menu_quit), s);
+#if defined(CONFIG_VTE)
+    g_signal_connect(s->copy_item, "activate",
+                     G_CALLBACK(gd_menu_copy), s);
+#endif
     g_signal_connect(s->full_screen_item, "activate",
                      G_CALLBACK(gd_menu_full_screen), s);
     g_signal_connect(s->zoom_in_item, "activate",
@@ -1653,33 +1928,46 @@
     return machine_menu;
 }
 
-static const DisplayChangeListenerOps dcl_ops = {
-    .dpy_name          = "gtk",
-    .dpy_gfx_update    = gd_update,
-    .dpy_gfx_switch    = gd_switch,
-    .dpy_refresh       = gd_refresh,
-    .dpy_mouse_set     = gd_mouse_set,
-    .dpy_cursor_define = gd_cursor_define,
-};
-
 static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
                               QemuConsole *con, int idx,
                               GSList *group, GtkWidget *view_menu)
 {
-    Object *obj;
-
-    obj = object_property_get_link(OBJECT(con), "device", NULL);
-    if (obj) {
-        vc->label = g_strdup_printf("%s", object_get_typename(obj));
-    } else {
-        vc->label = g_strdup_printf("VGA");
-    }
-
+    vc->label = qemu_console_get_label(con);
     vc->s = s;
     vc->gfx.scale_x = 1.0;
     vc->gfx.scale_y = 1.0;
 
-    vc->gfx.drawing_area = gtk_drawing_area_new();
+#if defined(CONFIG_OPENGL)
+    if (display_opengl) {
+#if defined(CONFIG_GTK_GL)
+        vc->gfx.drawing_area = gtk_gl_area_new();
+        vc->gfx.dcl.ops = &dcl_gl_area_ops;
+#else
+        vc->gfx.drawing_area = gtk_drawing_area_new();
+        /*
+         * gtk_widget_set_double_buffered() was deprecated in 3.14.
+         * It is required for opengl rendering on X11 though.  A
+         * proper replacement (native opengl support) is only
+         * available in 3.16+.  Silence the warning if possible.
+         */
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#endif
+        gtk_widget_set_double_buffered(vc->gfx.drawing_area, FALSE);
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic pop
+#endif
+        vc->gfx.dcl.ops = &dcl_egl_ops;
+#endif /* CONFIG_GTK_GL */
+    } else
+#endif
+    {
+        vc->gfx.drawing_area = gtk_drawing_area_new();
+        vc->gfx.dcl.ops = &dcl_ops;
+    }
+
+
     gtk_widget_add_events(vc->gfx.drawing_area,
                           GDK_POINTER_MOTION_MASK |
                           GDK_BUTTON_PRESS_MASK |
@@ -1693,16 +1981,21 @@
 
     vc->type = GD_VC_GFX;
     vc->tab_item = vc->gfx.drawing_area;
+    vc->focus = vc->gfx.drawing_area;
     gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook),
                              vc->tab_item, gtk_label_new(vc->label));
-    gd_connect_vc_gfx_signals(vc);
 
-    group = gd_vc_menu_init(s, vc, idx, group, view_menu);
-
-    vc->gfx.dcl.ops = &dcl_ops;
     vc->gfx.dcl.con = con;
     register_displaychangelistener(&vc->gfx.dcl);
 
+    gd_connect_vc_gfx_signals(vc);
+    group = gd_vc_menu_init(s, vc, idx, group, view_menu);
+
+    if (dpy_ui_info_supported(vc->gfx.dcl.con)) {
+        gtk_menu_item_activate(GTK_MENU_ITEM(s->zoom_fit_item));
+        s->free_scale = true;
+    }
+
     return group;
 }
 
@@ -1719,6 +2012,11 @@
 
     s->full_screen_item = gtk_menu_item_new_with_mnemonic(_("_Fullscreen"));
 
+#if defined(CONFIG_VTE)
+    s->copy_item = gtk_menu_item_new_with_mnemonic(_("_Copy"));
+    gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), s->copy_item);
+#endif
+
     gtk_accel_group_connect(s->accel_group, GDK_KEY_f, HOTKEY_MODIFIERS, 0,
             g_cclosure_new_swap(G_CALLBACK(gd_accel_full_screen), s, NULL));
 #if GTK_CHECK_VERSION(3, 8, 0)
@@ -1774,7 +2072,7 @@
     /* gfx */
     for (vc = 0;; vc++) {
         con = qemu_console_lookup_by_index(vc);
-        if (!con || !qemu_console_is_graphic(con)) {
+        if (!con) {
             break;
         }
         group = gd_vc_gfx_init(s, &s->vc[vc], con,
@@ -1850,12 +2148,18 @@
 #endif
 }
 
+static gboolean gtkinit;
+
 void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
 {
     GtkDisplayState *s = g_malloc0(sizeof(*s));
     char *filename;
+    GdkDisplay *window_display;
 
-    gtk_init(NULL, NULL);
+    if (!gtkinit) {
+        fprintf(stderr, "gtk initialization failed\n");
+        exit(1);
+    }
 
     s->window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
 #if GTK_CHECK_VERSION(3, 2, 0)
@@ -1868,11 +2172,14 @@
 
     s->free_scale = FALSE;
 
-    setlocale(LC_ALL, "");
+    /* LC_MESSAGES only. See early_gtk_display_init() for details */
+    setlocale(LC_MESSAGES, "");
     bindtextdomain("qemu", CONFIG_QEMU_LOCALEDIR);
     textdomain("qemu");
 
-    s->null_cursor = gdk_cursor_new(GDK_BLANK_CURSOR);
+    window_display = gtk_widget_get_display(s->window);
+    s->null_cursor = gdk_cursor_new_for_display(window_display,
+                                                GDK_BLANK_CURSOR);
 
     s->mouse_mode_notifier.notify = gd_mouse_mode_change;
     qemu_add_mouse_mode_change_notifier(&s->mouse_mode_notifier);
@@ -1933,8 +2240,50 @@
     gd_set_keycode_type(s);
 }
 
-void early_gtk_display_init(void)
+void early_gtk_display_init(int opengl)
 {
+    /* The QEMU code relies on the assumption that it's always run in
+     * the C locale. Therefore it is not prepared to deal with
+     * operations that produce different results depending on the
+     * locale, such as printf's formatting of decimal numbers, and
+     * possibly others.
+     *
+     * Since GTK+ calls setlocale() by default -importing the locale
+     * settings from the environment- we must prevent it from doing so
+     * using gtk_disable_setlocale().
+     *
+     * QEMU's GTK+ UI, however, _does_ have translations for some of
+     * the menu items. As a trade-off between a functionally correct
+     * QEMU and a fully internationalized UI we support importing
+     * LC_MESSAGES from the environment (see the setlocale() call
+     * earlier in this file). This allows us to display translated
+     * messages leaving everything else untouched.
+     */
+    gtk_disable_setlocale();
+    gtkinit = gtk_init_check(NULL, NULL);
+    if (!gtkinit) {
+        /* don't exit yet, that'll break -help */
+        return;
+    }
+
+    switch (opengl) {
+    case -1: /* default */
+    case 0:  /* off */
+        break;
+    case 1: /* on */
+#if defined(CONFIG_OPENGL)
+#if defined(CONFIG_GTK_GL)
+        gtk_gl_area_init();
+#else
+        gtk_egl_init();
+#endif
+#endif
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+
 #if defined(CONFIG_VTE)
     register_vc_handler(gd_vc_handler);
 #endif
diff --git a/ui/input-keymap.c b/ui/input-keymap.c
index 5d29935..f1e700d 100644
--- a/ui/input-keymap.c
+++ b/ui/input-keymap.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
 #include "ui/keymaps.h"
 #include "ui/input.h"
@@ -128,18 +129,22 @@
 
     [Q_KEY_CODE_INSERT] = 0xd2,
     [Q_KEY_CODE_DELETE] = 0xd3,
-    [Q_KEY_CODE_MAX] = 0,
+
+    [Q_KEY_CODE_RO] = 0x73,
+    [Q_KEY_CODE_KP_COMMA] = 0x7e,
+
+    [Q_KEY_CODE__MAX] = 0,
 };
 
 static int number_to_qcode[0x100];
 
 int qemu_input_key_value_to_number(const KeyValue *value)
 {
-    if (value->kind == KEY_VALUE_KIND_QCODE) {
-        return qcode_to_number[value->qcode];
+    if (value->type == KEY_VALUE_KIND_QCODE) {
+        return qcode_to_number[value->u.qcode.data];
     } else {
-        assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return value->number;
+        assert(value->type == KEY_VALUE_KIND_NUMBER);
+        return value->u.number.data;
     }
 }
 
@@ -150,7 +155,7 @@
     if (first) {
         int qcode, number;
         first = false;
-        for (qcode = 0; qcode < Q_KEY_CODE_MAX; qcode++) {
+        for (qcode = 0; qcode < Q_KEY_CODE__MAX; qcode++) {
             number = qcode_to_number[qcode];
             assert(number < ARRAY_SIZE(number_to_qcode));
             number_to_qcode[number] = qcode;
@@ -162,11 +167,11 @@
 
 int qemu_input_key_value_to_qcode(const KeyValue *value)
 {
-    if (value->kind == KEY_VALUE_KIND_QCODE) {
-        return value->qcode;
+    if (value->type == KEY_VALUE_KIND_QCODE) {
+        return value->u.qcode.data;
     } else {
-        assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return qemu_input_key_number_to_qcode(value->number);
+        assert(value->type == KEY_VALUE_KIND_NUMBER);
+        return qemu_input_key_number_to_qcode(value->u.number.data);
     }
 }
 
@@ -176,8 +181,8 @@
     int keycode = qemu_input_key_value_to_number(value);
     int count = 0;
 
-    if (value->kind == KEY_VALUE_KIND_QCODE &&
-        value->qcode == Q_KEY_CODE_PAUSE) {
+    if (value->type == KEY_VALUE_KIND_QCODE &&
+        value->u.qcode.data == Q_KEY_CODE_PAUSE) {
         /* specific case */
         int v = down ? 0 : 0x80;
         codes[count++] = 0xe1;
diff --git a/ui/input-legacy.c b/ui/input-legacy.c
index 59caf3f..951e81e 100644
--- a/ui/input-legacy.c
+++ b/ui/input-legacy.c
@@ -22,15 +22,16 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "sysemu/sysemu.h"
-#include "monitor/monitor.h"
 #include "ui/console.h"
-#include "qapi/error.h"
 #include "qmp-commands.h"
 #include "qapi-types.h"
 #include "ui/keymaps.h"
 #include "ui/input.h"
 
+#include <stdbool.h>
+
 struct QEMUPutMouseEntry {
     QEMUPutMouseEvent *qemu_put_mouse_event;
     void *qemu_put_mouse_event_opaque;
@@ -39,9 +40,9 @@
     /* new input core */
     QemuInputHandler h;
     QemuInputHandlerState *s;
-    int axis[INPUT_AXIS_MAX];
+    int axis[INPUT_AXIS__MAX];
     int buttons;
-    bool isTrackball;
+    bool is_trackball;
 };
 
 struct QEMUPutKbdEntry {
@@ -58,20 +59,19 @@
 
 static QTAILQ_HEAD(, QEMUPutLEDEntry) led_handlers =
     QTAILQ_HEAD_INITIALIZER(led_handlers);
-static QTAILQ_HEAD(, QEMUPutMouseEntry) mouse_handlers =
-    QTAILQ_HEAD_INITIALIZER(mouse_handlers);
 
-int index_from_key(const char *key)
+int index_from_key(const char *key, size_t key_length)
 {
     int i;
 
     for (i = 0; QKeyCode_lookup[i] != NULL; i++) {
-        if (!strcmp(key, QKeyCode_lookup[i])) {
+        if (!strncmp(key, QKeyCode_lookup[i], key_length) &&
+            !QKeyCode_lookup[i][key_length]) {
             break;
         }
     }
 
-    /* Return Q_KEY_CODE_MAX if the key is invalid */
+    /* Return Q_KEY_CODE__MAX if the key is invalid */
     return i;
 }
 
@@ -113,12 +113,13 @@
 {
     QEMUPutKbdEntry *entry = (QEMUPutKbdEntry *)dev;
     int scancodes[3], i, count;
+    InputKeyEvent *key = evt->u.key.data;
 
     if (!entry || !entry->put_kbd) {
         return;
     }
-    count = qemu_input_key_value_to_scancode(evt->key->key,
-                                             evt->key->down,
+    count = qemu_input_key_value_to_scancode(key->key,
+                                             key->down,
                                              scancodes);
     for (i = 0; i < count; i++) {
         entry->put_kbd(entry->opaque, scancodes[i]);
@@ -144,39 +145,36 @@
     return entry;
 }
 
-void qemu_remove_kbd_event_handler(QEMUPutKbdEntry *entry)
-{
-    qemu_input_handler_unregister(entry->s);
-    g_free(entry);
-}
-
 static void legacy_mouse_event(DeviceState *dev, QemuConsole *src,
                                InputEvent *evt)
 {
-    static const int bmap[INPUT_BUTTON_MAX] = {
+    static const int bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]   = MOUSE_EVENT_LBUTTON,
         [INPUT_BUTTON_MIDDLE] = MOUSE_EVENT_MBUTTON,
         [INPUT_BUTTON_RIGHT]  = MOUSE_EVENT_RBUTTON,
     };
     QEMUPutMouseEntry *s = (QEMUPutMouseEntry *)dev;
+    InputBtnEvent *btn;
+    InputMoveEvent *move;
 
-    s->isTrackball = false;
+    s->is_trackball = false;
 
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_BTN:
-        if (evt->btn->down) {
-            s->buttons |= bmap[evt->btn->button];
+        btn = evt->u.btn.data;
+        if (btn->down) {
+            s->buttons |= bmap[btn->button];
         } else {
-            s->buttons &= ~bmap[evt->btn->button];
+            s->buttons &= ~bmap[btn->button];
         }
-        if (evt->btn->down && evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+        if (btn->down && btn->button == INPUT_BUTTON_WHEEL_UP) {
             s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                                     s->axis[INPUT_AXIS_X],
                                     s->axis[INPUT_AXIS_Y],
                                     -1,
                                     s->buttons);
         }
-        if (evt->btn->down && evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+        if (btn->down && btn->button == INPUT_BUTTON_WHEEL_DOWN) {
             s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                                     s->axis[INPUT_AXIS_X],
                                     s->axis[INPUT_AXIS_Y],
@@ -185,11 +183,13 @@
         }
         break;
     case INPUT_EVENT_KIND_ABS:
-        s->axis[evt->abs->axis] = evt->abs->value;
+        move = evt->u.abs.data;
+        s->axis[move->axis] = move->value;
         break;
     case INPUT_EVENT_KIND_REL:
-        s->axis[evt->rel->axis] += evt->rel->value;
-        s->isTrackball = true;
+        move = evt->u.rel.data;
+        s->axis[move->axis] += move->value;
+        s->is_trackball = true;
         break;
     default:
         break;
@@ -203,7 +203,7 @@
     s->qemu_put_mouse_event(s->qemu_put_mouse_event_opaque,
                             s->axis[INPUT_AXIS_X],
                             s->axis[INPUT_AXIS_Y],
-                            s->isTrackball ? 1 : 0, // "dz"
+                            s->is_trackball ? 1 : 0, // "dz"
                             s->buttons);
 
     if (!s->qemu_put_mouse_event_absolute) {
@@ -218,7 +218,7 @@
 {
     QEMUPutMouseEntry *s;
 
-    s = g_malloc0(sizeof(QEMUPutMouseEntry));
+    s = g_new0(QEMUPutMouseEntry, 1);
 
     s->qemu_put_mouse_event = func;
     s->qemu_put_mouse_event_opaque = opaque;
@@ -252,7 +252,7 @@
 {
     QEMUPutLEDEntry *s;
 
-    s = g_malloc0(sizeof(QEMUPutLEDEntry));
+    s = g_new0(QEMUPutLEDEntry, 1);
 
     s->put_led = func;
     s->opaque = opaque;
diff --git a/ui/input-linux.c b/ui/input-linux.c
new file mode 100644
index 0000000..0e230ce
--- /dev/null
+++ b/ui/input-linux.c
@@ -0,0 +1,518 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu-common.h"
+#include "qemu/config-file.h"
+#include "qemu/sockets.h"
+#include "sysemu/sysemu.h"
+#include "ui/input.h"
+#include "qom/object_interfaces.h"
+
+#include <sys/ioctl.h>
+#include "standard-headers/linux/input.h"
+
+static int linux_to_qcode[KEY_CNT] = {
+    [KEY_ESC]            = Q_KEY_CODE_ESC,
+    [KEY_1]              = Q_KEY_CODE_1,
+    [KEY_2]              = Q_KEY_CODE_2,
+    [KEY_3]              = Q_KEY_CODE_3,
+    [KEY_4]              = Q_KEY_CODE_4,
+    [KEY_5]              = Q_KEY_CODE_5,
+    [KEY_6]              = Q_KEY_CODE_6,
+    [KEY_7]              = Q_KEY_CODE_7,
+    [KEY_8]              = Q_KEY_CODE_8,
+    [KEY_9]              = Q_KEY_CODE_9,
+    [KEY_0]              = Q_KEY_CODE_0,
+    [KEY_MINUS]          = Q_KEY_CODE_MINUS,
+    [KEY_EQUAL]          = Q_KEY_CODE_EQUAL,
+    [KEY_BACKSPACE]      = Q_KEY_CODE_BACKSPACE,
+    [KEY_TAB]            = Q_KEY_CODE_TAB,
+    [KEY_Q]              = Q_KEY_CODE_Q,
+    [KEY_W]              = Q_KEY_CODE_W,
+    [KEY_E]              = Q_KEY_CODE_E,
+    [KEY_R]              = Q_KEY_CODE_R,
+    [KEY_T]              = Q_KEY_CODE_T,
+    [KEY_Y]              = Q_KEY_CODE_Y,
+    [KEY_U]              = Q_KEY_CODE_U,
+    [KEY_I]              = Q_KEY_CODE_I,
+    [KEY_O]              = Q_KEY_CODE_O,
+    [KEY_P]              = Q_KEY_CODE_P,
+    [KEY_LEFTBRACE]      = Q_KEY_CODE_BRACKET_LEFT,
+    [KEY_RIGHTBRACE]     = Q_KEY_CODE_BRACKET_RIGHT,
+    [KEY_ENTER]          = Q_KEY_CODE_RET,
+    [KEY_LEFTCTRL]       = Q_KEY_CODE_CTRL,
+    [KEY_A]              = Q_KEY_CODE_A,
+    [KEY_S]              = Q_KEY_CODE_S,
+    [KEY_D]              = Q_KEY_CODE_D,
+    [KEY_F]              = Q_KEY_CODE_F,
+    [KEY_G]              = Q_KEY_CODE_G,
+    [KEY_H]              = Q_KEY_CODE_H,
+    [KEY_J]              = Q_KEY_CODE_J,
+    [KEY_K]              = Q_KEY_CODE_K,
+    [KEY_L]              = Q_KEY_CODE_L,
+    [KEY_SEMICOLON]      = Q_KEY_CODE_SEMICOLON,
+    [KEY_APOSTROPHE]     = Q_KEY_CODE_APOSTROPHE,
+    [KEY_GRAVE]          = Q_KEY_CODE_GRAVE_ACCENT,
+    [KEY_LEFTSHIFT]      = Q_KEY_CODE_SHIFT,
+    [KEY_BACKSLASH]      = Q_KEY_CODE_BACKSLASH,
+    [KEY_102ND]          = Q_KEY_CODE_LESS,
+    [KEY_Z]              = Q_KEY_CODE_Z,
+    [KEY_X]              = Q_KEY_CODE_X,
+    [KEY_C]              = Q_KEY_CODE_C,
+    [KEY_V]              = Q_KEY_CODE_V,
+    [KEY_B]              = Q_KEY_CODE_B,
+    [KEY_N]              = Q_KEY_CODE_N,
+    [KEY_M]              = Q_KEY_CODE_M,
+    [KEY_COMMA]          = Q_KEY_CODE_COMMA,
+    [KEY_DOT]            = Q_KEY_CODE_DOT,
+    [KEY_SLASH]          = Q_KEY_CODE_SLASH,
+    [KEY_RIGHTSHIFT]     = Q_KEY_CODE_SHIFT_R,
+    [KEY_LEFTALT]        = Q_KEY_CODE_ALT,
+    [KEY_SPACE]          = Q_KEY_CODE_SPC,
+    [KEY_CAPSLOCK]       = Q_KEY_CODE_CAPS_LOCK,
+    [KEY_F1]             = Q_KEY_CODE_F1,
+    [KEY_F2]             = Q_KEY_CODE_F2,
+    [KEY_F3]             = Q_KEY_CODE_F3,
+    [KEY_F4]             = Q_KEY_CODE_F4,
+    [KEY_F5]             = Q_KEY_CODE_F5,
+    [KEY_F6]             = Q_KEY_CODE_F6,
+    [KEY_F7]             = Q_KEY_CODE_F7,
+    [KEY_F8]             = Q_KEY_CODE_F8,
+    [KEY_F9]             = Q_KEY_CODE_F9,
+    [KEY_F10]            = Q_KEY_CODE_F10,
+    [KEY_NUMLOCK]        = Q_KEY_CODE_NUM_LOCK,
+    [KEY_SCROLLLOCK]     = Q_KEY_CODE_SCROLL_LOCK,
+    [KEY_KP0]            = Q_KEY_CODE_KP_0,
+    [KEY_KP1]            = Q_KEY_CODE_KP_1,
+    [KEY_KP2]            = Q_KEY_CODE_KP_2,
+    [KEY_KP3]            = Q_KEY_CODE_KP_3,
+    [KEY_KP4]            = Q_KEY_CODE_KP_4,
+    [KEY_KP5]            = Q_KEY_CODE_KP_5,
+    [KEY_KP6]            = Q_KEY_CODE_KP_6,
+    [KEY_KP7]            = Q_KEY_CODE_KP_7,
+    [KEY_KP8]            = Q_KEY_CODE_KP_8,
+    [KEY_KP9]            = Q_KEY_CODE_KP_9,
+    [KEY_KPMINUS]        = Q_KEY_CODE_KP_SUBTRACT,
+    [KEY_KPPLUS]         = Q_KEY_CODE_KP_ADD,
+    [KEY_KPDOT]          = Q_KEY_CODE_KP_DECIMAL,
+    [KEY_KPENTER]        = Q_KEY_CODE_KP_ENTER,
+    [KEY_KPSLASH]        = Q_KEY_CODE_KP_DIVIDE,
+    [KEY_KPASTERISK]     = Q_KEY_CODE_KP_MULTIPLY,
+    [KEY_F11]            = Q_KEY_CODE_F11,
+    [KEY_F12]            = Q_KEY_CODE_F12,
+    [KEY_RIGHTCTRL]      = Q_KEY_CODE_CTRL_R,
+    [KEY_SYSRQ]          = Q_KEY_CODE_SYSRQ,
+    [KEY_RIGHTALT]       = Q_KEY_CODE_ALT_R,
+    [KEY_HOME]           = Q_KEY_CODE_HOME,
+    [KEY_UP]             = Q_KEY_CODE_UP,
+    [KEY_PAGEUP]         = Q_KEY_CODE_PGUP,
+    [KEY_LEFT]           = Q_KEY_CODE_LEFT,
+    [KEY_RIGHT]          = Q_KEY_CODE_RIGHT,
+    [KEY_END]            = Q_KEY_CODE_END,
+    [KEY_DOWN]           = Q_KEY_CODE_DOWN,
+    [KEY_PAGEDOWN]       = Q_KEY_CODE_PGDN,
+    [KEY_INSERT]         = Q_KEY_CODE_INSERT,
+    [KEY_DELETE]         = Q_KEY_CODE_DELETE,
+    [KEY_LEFTMETA]       = Q_KEY_CODE_META_L,
+    [KEY_RIGHTMETA]      = Q_KEY_CODE_META_R,
+    [KEY_MENU]           = Q_KEY_CODE_MENU,
+};
+
+static int qemu_input_linux_to_qcode(unsigned int lnx)
+{
+    assert(lnx < KEY_CNT);
+    return linux_to_qcode[lnx];
+}
+
+static bool linux_is_button(unsigned int lnx)
+{
+    if (lnx < 0x100) {
+        return false;
+    }
+    if (lnx >= 0x160 && lnx < 0x2c0) {
+        return false;
+    }
+    return true;
+}
+
+#define TYPE_INPUT_LINUX "input-linux"
+#define INPUT_LINUX(obj) \
+    OBJECT_CHECK(InputLinux, (obj), TYPE_INPUT_LINUX)
+#define INPUT_LINUX_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(InputLinuxClass, (obj), TYPE_INPUT_LINUX)
+#define INPUT_LINUX_CLASS(klass) \
+    OBJECT_CLASS_CHECK(InputLinuxClass, (klass), TYPE_INPUT_LINUX)
+
+typedef struct InputLinux InputLinux;
+typedef struct InputLinuxClass InputLinuxClass;
+
+struct InputLinux {
+    Object parent;
+
+    char        *evdev;
+    int         fd;
+    bool        repeat;
+    bool        grab_request;
+    bool        grab_active;
+    bool        grab_all;
+    bool        keydown[KEY_CNT];
+    int         keycount;
+    int         wheel;
+    bool        initialized;
+
+    bool        has_rel_x;
+    bool        has_abs_x;
+    int         num_keys;
+    int         num_btns;
+
+    QTAILQ_ENTRY(InputLinux) next;
+};
+
+struct InputLinuxClass {
+    ObjectClass parent_class;
+};
+
+static QTAILQ_HEAD(, InputLinux) inputs = QTAILQ_HEAD_INITIALIZER(inputs);
+
+static void input_linux_toggle_grab(InputLinux *il)
+{
+    intptr_t request = !il->grab_active;
+    InputLinux *item;
+    int rc;
+
+    rc = ioctl(il->fd, EVIOCGRAB, request);
+    if (rc < 0) {
+        return;
+    }
+    il->grab_active = !il->grab_active;
+
+    if (!il->grab_all) {
+        return;
+    }
+    QTAILQ_FOREACH(item, &inputs, next) {
+        if (item == il || item->grab_all) {
+            /* avoid endless loops */
+            continue;
+        }
+        if (item->grab_active != il->grab_active) {
+            input_linux_toggle_grab(item);
+        }
+    }
+}
+
+static void input_linux_handle_keyboard(InputLinux *il,
+                                        struct input_event *event)
+{
+    if (event->type == EV_KEY) {
+        if (event->value > 2 || (event->value > 1 && !il->repeat)) {
+            /*
+             * ignore autorepeat + unknown key events
+             * 0 == up, 1 == down, 2 == autorepeat, other == undefined
+             */
+            return;
+        }
+        if (event->code >= KEY_CNT) {
+            /*
+             * Should not happen.  But better safe than sorry,
+             * and we make Coverity happy too.
+             */
+            return;
+        }
+
+        /* keep track of key state */
+        if (!il->keydown[event->code] && event->value) {
+            il->keydown[event->code] = true;
+            il->keycount++;
+        }
+        if (il->keydown[event->code] && !event->value) {
+            il->keydown[event->code] = false;
+            il->keycount--;
+        }
+
+        /* send event to guest when grab is active */
+        if (il->grab_active) {
+            int qcode = qemu_input_linux_to_qcode(event->code);
+            qemu_input_event_send_key_qcode(NULL, qcode, event->value);
+        }
+
+        /* hotkey -> record switch request ... */
+        if (il->keydown[KEY_LEFTCTRL] &&
+            il->keydown[KEY_RIGHTCTRL]) {
+            il->grab_request = true;
+        }
+
+        /*
+         * ... and do the switch when all keys are lifted, so we
+         * confuse neither guest nor host with keys which seem to
+         * be stuck due to missing key-up events.
+         */
+        if (il->grab_request && !il->keycount) {
+            il->grab_request = false;
+            input_linux_toggle_grab(il);
+        }
+    }
+}
+
+static void input_linux_event_mouse_button(int button)
+{
+    qemu_input_queue_btn(NULL, button, true);
+    qemu_input_event_sync();
+    qemu_input_queue_btn(NULL, button, false);
+    qemu_input_event_sync();
+}
+
+static void input_linux_handle_mouse(InputLinux *il, struct input_event *event)
+{
+    if (!il->grab_active) {
+        return;
+    }
+
+    switch (event->type) {
+    case EV_KEY:
+        switch (event->code) {
+        case BTN_LEFT:
+            qemu_input_queue_btn(NULL, INPUT_BUTTON_LEFT, event->value);
+            break;
+        case BTN_RIGHT:
+            qemu_input_queue_btn(NULL, INPUT_BUTTON_RIGHT, event->value);
+            break;
+        case BTN_MIDDLE:
+            qemu_input_queue_btn(NULL, INPUT_BUTTON_MIDDLE, event->value);
+            break;
+        case BTN_GEAR_UP:
+            qemu_input_queue_btn(NULL, INPUT_BUTTON_WHEEL_UP, event->value);
+            break;
+        case BTN_GEAR_DOWN:
+            qemu_input_queue_btn(NULL, INPUT_BUTTON_WHEEL_DOWN,
+                                 event->value);
+            break;
+        };
+        break;
+    case EV_REL:
+        switch (event->code) {
+        case REL_X:
+            qemu_input_queue_rel(NULL, INPUT_AXIS_X, event->value);
+            break;
+        case REL_Y:
+            qemu_input_queue_rel(NULL, INPUT_AXIS_Y, event->value);
+            break;
+        case REL_WHEEL:
+            il->wheel = event->value;
+            break;
+        }
+        break;
+    case EV_SYN:
+        qemu_input_event_sync();
+        if (il->wheel != 0) {
+            input_linux_event_mouse_button((il->wheel > 0)
+                                           ? INPUT_BUTTON_WHEEL_UP
+                                           : INPUT_BUTTON_WHEEL_DOWN);
+            il->wheel = 0;
+        }
+        break;
+    }
+}
+
+static void input_linux_event(void *opaque)
+{
+    InputLinux *il = opaque;
+    struct input_event event;
+    int rc;
+
+    for (;;) {
+        rc = read(il->fd, &event, sizeof(event));
+        if (rc != sizeof(event)) {
+            if (rc < 0 && errno != EAGAIN) {
+                fprintf(stderr, "%s: read: %s\n", __func__, strerror(errno));
+                qemu_set_fd_handler(il->fd, NULL, NULL, NULL);
+                close(il->fd);
+            }
+            break;
+        }
+
+        if (il->num_keys) {
+            input_linux_handle_keyboard(il, &event);
+        }
+        if (il->has_rel_x && il->num_btns) {
+            input_linux_handle_mouse(il, &event);
+        }
+    }
+}
+
+static void input_linux_complete(UserCreatable *uc, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(uc);
+    uint8_t evtmap, relmap, absmap, keymap[KEY_CNT / 8];
+    unsigned int i;
+    int rc, ver;
+
+    if (!il->evdev) {
+        error_setg(errp, "no input device specified");
+        return;
+    }
+
+    il->fd = open(il->evdev, O_RDWR);
+    if (il->fd < 0)  {
+        error_setg_file_open(errp, errno, il->evdev);
+        return;
+    }
+    qemu_set_nonblock(il->fd);
+
+    rc = ioctl(il->fd, EVIOCGVERSION, &ver);
+    if (rc < 0) {
+        error_setg(errp, "%s: is not an evdev device", il->evdev);
+        goto err_close;
+    }
+
+    rc = ioctl(il->fd, EVIOCGBIT(0, sizeof(evtmap)), &evtmap);
+    if (rc < 0) {
+        error_setg(errp, "%s: failed to read event bits", il->evdev);
+        goto err_close;
+    }
+
+    if (evtmap & (1 << EV_REL)) {
+        relmap = 0;
+        rc = ioctl(il->fd, EVIOCGBIT(EV_REL, sizeof(relmap)), &relmap);
+        if (relmap & (1 << REL_X)) {
+            il->has_rel_x = true;
+        }
+    }
+
+    if (evtmap & (1 << EV_ABS)) {
+        absmap = 0;
+        rc = ioctl(il->fd, EVIOCGBIT(EV_ABS, sizeof(absmap)), &absmap);
+        if (absmap & (1 << ABS_X)) {
+            il->has_abs_x = true;
+        }
+    }
+
+    if (evtmap & (1 << EV_KEY)) {
+        memset(keymap, 0, sizeof(keymap));
+        rc = ioctl(il->fd, EVIOCGBIT(EV_KEY, sizeof(keymap)), keymap);
+        for (i = 0; i < KEY_CNT; i++) {
+            if (keymap[i / 8] & (1 << (i % 8))) {
+                if (linux_is_button(i)) {
+                    il->num_btns++;
+                } else {
+                    il->num_keys++;
+                }
+            }
+        }
+    }
+
+    qemu_set_fd_handler(il->fd, input_linux_event, NULL, il);
+    input_linux_toggle_grab(il);
+    QTAILQ_INSERT_TAIL(&inputs, il, next);
+    il->initialized = true;
+    return;
+
+err_close:
+    close(il->fd);
+    return;
+}
+
+static void input_linux_instance_finalize(Object *obj)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    if (il->initialized) {
+        QTAILQ_REMOVE(&inputs, il, next);
+        close(il->fd);
+    }
+    g_free(il->evdev);
+}
+
+static char *input_linux_get_evdev(Object *obj, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    return g_strdup(il->evdev);
+}
+
+static void input_linux_set_evdev(Object *obj, const char *value,
+                                  Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    if (il->evdev) {
+        error_setg(errp, "evdev property already set");
+        return;
+    }
+    il->evdev = g_strdup(value);
+}
+
+static bool input_linux_get_grab_all(Object *obj, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    return il->grab_all;
+}
+
+static void input_linux_set_grab_all(Object *obj, bool value,
+                                   Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    il->grab_all = value;
+}
+
+static bool input_linux_get_repeat(Object *obj, Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    return il->repeat;
+}
+
+static void input_linux_set_repeat(Object *obj, bool value,
+                                   Error **errp)
+{
+    InputLinux *il = INPUT_LINUX(obj);
+
+    il->repeat = value;
+}
+
+static void input_linux_instance_init(Object *obj)
+{
+    object_property_add_str(obj, "evdev",
+                            input_linux_get_evdev,
+                            input_linux_set_evdev, NULL);
+    object_property_add_bool(obj, "grab_all",
+                             input_linux_get_grab_all,
+                             input_linux_set_grab_all, NULL);
+    object_property_add_bool(obj, "repeat",
+                             input_linux_get_repeat,
+                             input_linux_set_repeat, NULL);
+}
+
+static void input_linux_class_init(ObjectClass *oc, void *data)
+{
+    UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+
+    ucc->complete = input_linux_complete;
+}
+
+static const TypeInfo input_linux_info = {
+    .name = TYPE_INPUT_LINUX,
+    .parent = TYPE_OBJECT,
+    .class_size = sizeof(InputLinuxClass),
+    .class_init = input_linux_class_init,
+    .instance_size = sizeof(InputLinux),
+    .instance_init = input_linux_instance_init,
+    .instance_finalize = input_linux_instance_finalize,
+    .interfaces = (InterfaceInfo[]) {
+        { TYPE_USER_CREATABLE },
+        { }
+    }
+};
+
+static void register_types(void)
+{
+    type_register_static(&input_linux_info);
+}
+
+type_init(register_types);
diff --git a/ui/input.c b/ui/input.c
index 7ba99e5..ed88cda 100644
--- a/ui/input.c
+++ b/ui/input.c
@@ -1,10 +1,13 @@
+#include "qemu/osdep.h"
 #include "hw/qdev.h"
 #include "sysemu/sysemu.h"
 #include "qapi-types.h"
+#include "qemu/error-report.h"
 #include "qmp-commands.h"
 #include "trace.h"
 #include "ui/input.h"
 #include "ui/console.h"
+#include "sysemu/replay.h"
 
 struct QemuInputHandlerState {
     DeviceState       *dev;
@@ -79,18 +82,12 @@
                              const char *device_id, int head,
                              Error **errp)
 {
-    DeviceState *dev;
     QemuConsole *con;
+    Error *err = NULL;
 
-    dev = qdev_find_recursive(sysbus_get_default(), device_id);
-    if (dev == NULL) {
-        error_set(errp, QERR_DEVICE_NOT_FOUND, device_id);
-        return;
-    }
-
-    con = qemu_console_lookup_by_device(dev, head);
-    if (con == NULL) {
-        error_setg(errp, "Device %s is not bound to a QemuConsole", device_id);
+    con = qemu_console_lookup_by_device_name(device_id, head, &err);
+    if (err) {
+        error_propagate(errp, err);
         return;
     }
 
@@ -122,17 +119,22 @@
     return NULL;
 }
 
-void qmp_x_input_send_event(bool has_console, int64_t console,
-                            InputEventList *events, Error **errp)
+void qmp_input_send_event(bool has_device, const char *device,
+                          bool has_head, int64_t head,
+                          InputEventList *events, Error **errp)
 {
     InputEventList *e;
     QemuConsole *con;
+    Error *err = NULL;
 
     con = NULL;
-    if (has_console) {
-        con = qemu_console_lookup_by_index(console);
-        if (!con) {
-            error_setg(errp, "console %" PRId64 " not found", console);
+    if (has_device) {
+        if (!has_head) {
+            head = 0;
+        }
+        con = qemu_console_lookup_by_device_name(device, head, &err);
+        if (err) {
+            error_propagate(errp, err);
             return;
         }
     }
@@ -145,10 +147,10 @@
     for (e = events; e != NULL; e = e->next) {
         InputEvent *event = e->value;
 
-        if (!qemu_input_find_handler(1 << event->kind, con)) {
+        if (!qemu_input_find_handler(1 << event->type, con)) {
             error_setg(errp, "Input handler not found for "
                              "event type %s",
-                            InputEventKind_lookup[event->kind]);
+                            InputEventKind_lookup[event->type]);
             return;
         }
     }
@@ -164,24 +166,25 @@
 
 static void qemu_input_transform_abs_rotate(InputEvent *evt)
 {
+    InputMoveEvent *move = evt->u.abs.data;
     switch (graphic_rotate) {
     case 90:
-        if (evt->abs->axis == INPUT_AXIS_X) {
-            evt->abs->axis = INPUT_AXIS_Y;
-        } else if (evt->abs->axis == INPUT_AXIS_Y) {
-            evt->abs->axis = INPUT_AXIS_X;
-            evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
+        if (move->axis == INPUT_AXIS_X) {
+            move->axis = INPUT_AXIS_Y;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            move->axis = INPUT_AXIS_X;
+            move->value = INPUT_EVENT_ABS_SIZE - 1 - move->value;
         }
         break;
     case 180:
-        evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
+        move->value = INPUT_EVENT_ABS_SIZE - 1 - move->value;
         break;
     case 270:
-        if (evt->abs->axis == INPUT_AXIS_X) {
-            evt->abs->axis = INPUT_AXIS_Y;
-            evt->abs->value = INPUT_EVENT_ABS_SIZE - 1 - evt->abs->value;
-        } else if (evt->abs->axis == INPUT_AXIS_Y) {
-            evt->abs->axis = INPUT_AXIS_X;
+        if (move->axis == INPUT_AXIS_X) {
+            move->axis = INPUT_AXIS_Y;
+            move->value = INPUT_EVENT_ABS_SIZE - 1 - move->value;
+        } else if (move->axis == INPUT_AXIS_Y) {
+            move->axis = INPUT_AXIS_X;
         }
         break;
     }
@@ -191,41 +194,48 @@
 {
     const char *name;
     int qcode, idx = -1;
+    InputKeyEvent *key;
+    InputBtnEvent *btn;
+    InputMoveEvent *move;
 
     if (src) {
         idx = qemu_console_get_index(src);
     }
-    switch (evt->kind) {
+    switch (evt->type) {
     case INPUT_EVENT_KIND_KEY:
-        switch (evt->key->key->kind) {
+        key = evt->u.key.data;
+        switch (key->key->type) {
         case KEY_VALUE_KIND_NUMBER:
-            qcode = qemu_input_key_number_to_qcode(evt->key->key->number);
+            qcode = qemu_input_key_number_to_qcode(key->key->u.number.data);
             name = QKeyCode_lookup[qcode];
-            trace_input_event_key_number(idx, evt->key->key->number,
-                                         name, evt->key->down);
+            trace_input_event_key_number(idx, key->key->u.number.data,
+                                         name, key->down);
             break;
         case KEY_VALUE_KIND_QCODE:
-            name = QKeyCode_lookup[evt->key->key->qcode];
-            trace_input_event_key_qcode(idx, name, evt->key->down);
+            name = QKeyCode_lookup[key->key->u.qcode.data];
+            trace_input_event_key_qcode(idx, name, key->down);
             break;
-        case KEY_VALUE_KIND_MAX:
+        case KEY_VALUE_KIND__MAX:
             /* keep gcc happy */
             break;
         }
         break;
     case INPUT_EVENT_KIND_BTN:
-        name = InputButton_lookup[evt->btn->button];
-        trace_input_event_btn(idx, name, evt->btn->down);
+        btn = evt->u.btn.data;
+        name = InputButton_lookup[btn->button];
+        trace_input_event_btn(idx, name, btn->down);
         break;
     case INPUT_EVENT_KIND_REL:
-        name = InputAxis_lookup[evt->rel->axis];
-        trace_input_event_rel(idx, name, evt->rel->value);
+        move = evt->u.rel.data;
+        name = InputAxis_lookup[move->axis];
+        trace_input_event_rel(idx, name, move->value);
         break;
     case INPUT_EVENT_KIND_ABS:
-        name = InputAxis_lookup[evt->abs->axis];
-        trace_input_event_abs(idx, name, evt->abs->value);
+        move = evt->u.abs.data;
+        name = InputAxis_lookup[move->axis];
+        trace_input_event_abs(idx, name, move->value);
         break;
-    case INPUT_EVENT_KIND_MAX:
+    case INPUT_EVENT_KIND__MAX:
         /* keep gcc happy */
         break;
     }
@@ -298,23 +308,19 @@
     QTAILQ_INSERT_TAIL(queue, item, node);
 }
 
-void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
+void qemu_input_event_send_impl(QemuConsole *src, InputEvent *evt)
 {
     QemuInputHandlerState *s;
 
-    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
-        return;
-    }
-
     qemu_input_event_trace(src, evt);
 
     /* pre processing */
-    if (graphic_rotate && (evt->kind == INPUT_EVENT_KIND_ABS)) {
+    if (graphic_rotate && (evt->type == INPUT_EVENT_KIND_ABS)) {
             qemu_input_transform_abs_rotate(evt);
     }
 
     /* send event */
-    s = qemu_input_find_handler(1 << evt->kind, src);
+    s = qemu_input_find_handler(1 << evt->type, src);
     if (!s) {
         return;
     }
@@ -322,14 +328,19 @@
     s->events++;
 }
 
-void qemu_input_event_sync(void)
+void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
 {
-    QemuInputHandlerState *s;
-
     if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
         return;
     }
 
+    replay_input_event(src, evt);
+}
+
+void qemu_input_event_sync_impl(void)
+{
+    QemuInputHandlerState *s;
+
     trace_input_event_sync();
 
     QTAILQ_FOREACH(s, &handlers, node) {
@@ -343,13 +354,22 @@
     }
 }
 
+void qemu_input_event_sync(void)
+{
+    if (!runstate_is_running() && !runstate_check(RUN_STATE_SUSPENDED)) {
+        return;
+    }
+
+    replay_input_sync_event();
+}
+
 InputEvent *qemu_input_event_new_key(KeyValue *key, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
-    evt->key = g_new0(InputKeyEvent, 1);
-    evt->kind = INPUT_EVENT_KIND_KEY;
-    evt->key->key = key;
-    evt->key->down = down;
+    evt->u.key.data = g_new0(InputKeyEvent, 1);
+    evt->type = INPUT_EVENT_KIND_KEY;
+    evt->u.key.data->key = key;
+    evt->u.key.data->down = down;
     return evt;
 }
 
@@ -370,16 +390,16 @@
 void qemu_input_event_send_key_number(QemuConsole *src, int num, bool down)
 {
     KeyValue *key = g_new0(KeyValue, 1);
-    key->kind = KEY_VALUE_KIND_NUMBER;
-    key->number = num;
+    key->type = KEY_VALUE_KIND_NUMBER;
+    key->u.number.data = num;
     qemu_input_event_send_key(src, key, down);
 }
 
 void qemu_input_event_send_key_qcode(QemuConsole *src, QKeyCode q, bool down)
 {
     KeyValue *key = g_new0(KeyValue, 1);
-    key->kind = KEY_VALUE_KIND_QCODE;
-    key->qcode = q;
+    key->type = KEY_VALUE_KIND_QCODE;
+    key->u.qcode.data = q;
     qemu_input_event_send_key(src, key, down);
 }
 
@@ -396,10 +416,10 @@
 InputEvent *qemu_input_event_new_btn(InputButton btn, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
-    evt->btn = g_new0(InputBtnEvent, 1);
-    evt->kind = INPUT_EVENT_KIND_BTN;
-    evt->btn->button = btn;
-    evt->btn->down = down;
+    evt->u.btn.data = g_new0(InputBtnEvent, 1);
+    evt->type = INPUT_EVENT_KIND_BTN;
+    evt->u.btn.data->button = btn;
+    evt->u.btn.data->down = down;
     return evt;
 }
 
@@ -417,7 +437,7 @@
     InputButton btn;
     uint32_t mask;
 
-    for (btn = 0; btn < INPUT_BUTTON_MAX; btn++) {
+    for (btn = 0; btn < INPUT_BUTTON__MAX; btn++) {
         mask = button_map[btn];
         if ((button_old & mask) == (button_new & mask)) {
             continue;
@@ -449,8 +469,8 @@
     InputEvent *evt = g_new0(InputEvent, 1);
     InputMoveEvent *move = g_new0(InputMoveEvent, 1);
 
-    evt->kind = kind;
-    evt->data = move;
+    evt->type = kind;
+    evt->u.rel.data = move; /* evt->u.rel is the same as evt->u.abs */
     move->axis = axis;
     move->value = value;
     return evt;
@@ -526,7 +546,7 @@
     return mice_list;
 }
 
-void do_mouse_set(Monitor *mon, const QDict *qdict)
+void hmp_mouse_set(Monitor *mon, const QDict *qdict)
 {
     QemuInputHandlerState *s;
     int index = qdict_get_int(qdict, "index");
diff --git a/ui/keymaps.c b/ui/keymaps.c
index 80d658d..8899a0b 100644
--- a/ui/keymaps.c
+++ b/ui/keymaps.c
@@ -22,22 +22,25 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "keymaps.h"
 #include "sysemu/sysemu.h"
 
 static int get_keysym(const name2keysym_t *table,
-		      const char *name)
+                      const char *name)
 {
     const name2keysym_t *p;
     for(p = table; p->name != NULL; p++) {
-        if (!strcmp(p->name, name))
+        if (!strcmp(p->name, name)) {
             return p->keysym;
+        }
     }
     if (name[0] == 'U' && strlen(name) == 5) { /* try unicode Uxxxx */
         char *end;
         int ret = (int)strtoul(name + 1, &end, 16);
-        if (*end == '\0' && ret > 0)
-          return ret;
+        if (*end == '\0' && ret > 0) {
+            return ret;
+        }
     }
     return 0;
 }
@@ -46,19 +49,20 @@
 static void add_to_key_range(struct key_range **krp, int code) {
     struct key_range *kr;
     for (kr = *krp; kr; kr = kr->next) {
-	if (code >= kr->start && code <= kr->end)
-	    break;
-	if (code == kr->start - 1) {
-	    kr->start--;
-	    break;
-	}
-	if (code == kr->end + 1) {
-	    kr->end++;
-	    break;
-	}
+        if (code >= kr->start && code <= kr->end) {
+            break;
+        }
+        if (code == kr->start - 1) {
+            kr->start--;
+            break;
+        }
+        if (code == kr->end + 1) {
+            kr->end++;
+            break;
+        }
     }
     if (kr == NULL) {
-	kr = g_malloc0(sizeof(*kr));
+        kr = g_malloc0(sizeof(*kr));
         kr->start = kr->end = code;
         kr->next = *krp;
         *krp = kr;
@@ -67,30 +71,30 @@
 
 static void add_keysym(char *line, int keysym, int keycode, kbd_layout_t *k) {
     if (keysym < MAX_NORMAL_KEYCODE) {
-	//fprintf(stderr,"Setting keysym %s (%d) to %d\n",line,keysym,keycode);
-	k->keysym2keycode[keysym] = keycode;
+        /* fprintf(stderr,"Setting keysym %s (%d) to %d\n",
+                   line, keysym, keycode); */
+        k->keysym2keycode[keysym] = keycode;
     } else {
-	if (k->extra_count >= MAX_EXTRA_COUNT) {
-	    fprintf(stderr,
-		    "Warning: Could not assign keysym %s (0x%x) because of memory constraints.\n",
-		    line, keysym);
-	} else {
+        if (k->extra_count >= MAX_EXTRA_COUNT) {
+            fprintf(stderr, "Warning: Could not assign keysym %s (0x%x)"
+                    " because of memory constraints.\n", line, keysym);
+        } else {
 #if 0
-	    fprintf(stderr, "Setting %d: %d,%d\n",
-		    k->extra_count, keysym, keycode);
+            fprintf(stderr, "Setting %d: %d,%d\n",
+                    k->extra_count, keysym, keycode);
 #endif
-	    k->keysym2keycode_extra[k->extra_count].
-		keysym = keysym;
-	    k->keysym2keycode_extra[k->extra_count].
-		keycode = keycode;
-	    k->extra_count++;
-	}
+            k->keysym2keycode_extra[k->extra_count].
+            keysym = keysym;
+            k->keysym2keycode_extra[k->extra_count].
+            keycode = keycode;
+            k->extra_count++;
+        }
     }
 }
 
 static kbd_layout_t *parse_keyboard_layout(const name2keysym_t *table,
-					   const char *language,
-					   kbd_layout_t * k)
+                                           const char *language,
+                                           kbd_layout_t *k)
 {
     FILE *f;
     char * filename;
@@ -101,69 +105,78 @@
     f = filename ? fopen(filename, "r") : NULL;
     g_free(filename);
     if (!f) {
-	fprintf(stderr,
-		"Could not read keymap file: '%s'\n", language);
-	return NULL;
+        fprintf(stderr, "Could not read keymap file: '%s'\n", language);
+        return NULL;
     }
 
-    if (!k)
-	k = g_malloc0(sizeof(kbd_layout_t));
+    if (!k) {
+        k = g_new0(kbd_layout_t, 1);
+    }
 
     for(;;) {
-	if (fgets(line, 1024, f) == NULL)
+        if (fgets(line, 1024, f) == NULL) {
             break;
+        }
         len = strlen(line);
-        if (len > 0 && line[len - 1] == '\n')
+        if (len > 0 && line[len - 1] == '\n') {
             line[len - 1] = '\0';
-        if (line[0] == '#')
-	    continue;
-	if (!strncmp(line, "map ", 4))
-	    continue;
-	if (!strncmp(line, "include ", 8)) {
-	    parse_keyboard_layout(table, line + 8, k);
+        }
+        if (line[0] == '#') {
+            continue;
+        }
+        if (!strncmp(line, "map ", 4)) {
+            continue;
+        }
+        if (!strncmp(line, "include ", 8)) {
+            parse_keyboard_layout(table, line + 8, k);
         } else {
-	    char *end_of_keysym = line;
-	    while (*end_of_keysym != 0 && *end_of_keysym != ' ')
-		end_of_keysym++;
-	    if (*end_of_keysym) {
-		int keysym;
-		*end_of_keysym = 0;
-		keysym = get_keysym(table, line);
-		if (keysym == 0) {
-                    //		    fprintf(stderr, "Warning: unknown keysym %s\n", line);
-		} else {
-		    const char *rest = end_of_keysym + 1;
+            char *end_of_keysym = line;
+            while (*end_of_keysym != 0 && *end_of_keysym != ' ') {
+                end_of_keysym++;
+            }
+            if (*end_of_keysym) {
+                int keysym;
+                *end_of_keysym = 0;
+                keysym = get_keysym(table, line);
+                if (keysym == 0) {
+                    /* fprintf(stderr, "Warning: unknown keysym %s\n", line);*/
+                } else {
+                    const char *rest = end_of_keysym + 1;
                     int keycode = strtol(rest, NULL, 0);
 
                     if (strstr(rest, "numlock")) {
-			add_to_key_range(&k->keypad_range, keycode);
-			add_to_key_range(&k->numlock_range, keysym);
-			//fprintf(stderr, "keypad keysym %04x keycode %d\n", keysym, keycode);
-		    }
+                        add_to_key_range(&k->keypad_range, keycode);
+                        add_to_key_range(&k->numlock_range, keysym);
+                        /* fprintf(stderr, "keypad keysym %04x keycode %d\n",
+                                   keysym, keycode); */
+                    }
 
                     if (strstr(rest, "shift")) {
-			keycode |= SCANCODE_SHIFT;
+                        keycode |= SCANCODE_SHIFT;
                     }
                     if (strstr(rest, "altgr")) {
-			keycode |= SCANCODE_ALTGR;
+                        keycode |= SCANCODE_ALTGR;
                     }
                     if (strstr(rest, "ctrl")) {
-			keycode |= SCANCODE_CTRL;
+                        keycode |= SCANCODE_CTRL;
                     }
 
-		    add_keysym(line, keysym, keycode, k);
+                    add_keysym(line, keysym, keycode, k);
 
                     if (strstr(rest, "addupper")) {
-			char *c;
-			for (c = line; *c; c++)
-			    *c = qemu_toupper(*c);
-			keysym = get_keysym(table, line);
-			if (keysym)
-			    add_keysym(line, keysym, keycode | SCANCODE_SHIFT, k);
-		    }
-		}
-	    }
-	}
+                        char *c;
+                        for (c = line; *c; c++) {
+                            *c = qemu_toupper(*c);
+                        }
+                        keysym = get_keysym(table, line);
+                        if (keysym) {
+                            add_keysym(line, keysym,
+                                       keycode | SCANCODE_SHIFT, k);
+                        }
+                    }
+                }
+            }
+        }
     }
     fclose(f);
     return k;
@@ -180,19 +193,23 @@
 {
     kbd_layout_t *k = kbd_layout;
     if (keysym < MAX_NORMAL_KEYCODE) {
-	if (k->keysym2keycode[keysym] == 0)
-	    fprintf(stderr, "Warning: no scancode found for keysym %d\n",
-		    keysym);
-	return k->keysym2keycode[keysym];
+        if (k->keysym2keycode[keysym] == 0) {
+            fprintf(stderr, "Warning: no scancode found for keysym %d\n",
+                    keysym);
+        }
+        return k->keysym2keycode[keysym];
     } else {
-	int i;
+        int i;
 #ifdef XK_ISO_Left_Tab
-	if (keysym == XK_ISO_Left_Tab)
-	    keysym = XK_Tab;
+        if (keysym == XK_ISO_Left_Tab) {
+            keysym = XK_Tab;
+        }
 #endif
-	for (i = 0; i < k->extra_count; i++)
-	    if (k->keysym2keycode_extra[i].keysym == keysym)
-		return k->keysym2keycode_extra[i].keycode;
+        for (i = 0; i < k->extra_count; i++) {
+            if (k->keysym2keycode_extra[i].keysym == keysym) {
+                return k->keysym2keycode_extra[i].keycode;
+            }
+        }
     }
     return 0;
 }
@@ -202,9 +219,11 @@
     kbd_layout_t *k = kbd_layout;
     struct key_range *kr;
 
-    for (kr = k->keypad_range; kr; kr = kr->next)
-        if (keycode >= kr->start && keycode <= kr->end)
+    for (kr = k->keypad_range; kr; kr = kr->next) {
+        if (keycode >= kr->start && keycode <= kr->end) {
             return 1;
+        }
+    }
     return 0;
 }
 
@@ -213,8 +232,10 @@
     kbd_layout_t *k = kbd_layout;
     struct key_range *kr;
 
-    for (kr = k->numlock_range; kr; kr = kr->next)
-        if (keysym >= kr->start && keysym <= kr->end)
+    for (kr = k->numlock_range; kr; kr = kr->next) {
+        if (keysym >= kr->start && keysym <= kr->end) {
             return 1;
+        }
+    }
     return 0;
 }
diff --git a/ui/keymaps.h b/ui/keymaps.h
index a7600d5..47d0613 100644
--- a/ui/keymaps.h
+++ b/ui/keymaps.h
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef __QEMU_KEYMAPS_H__
-#define __QEMU_KEYMAPS_H__
+#ifndef QEMU_KEYMAPS_H
+#define QEMU_KEYMAPS_H
 
 #include "qemu-common.h"
 
@@ -74,4 +74,4 @@
 int keycode_is_keypad(void *kbd_layout, int keycode);
 int keysym_is_numlock(void *kbd_layout, int keysym);
 
-#endif /* __QEMU_KEYMAPS_H__ */
+#endif /* QEMU_KEYMAPS_H */
diff --git a/ui/qemu-pixman.c b/ui/qemu-pixman.c
index 1f6fea5..6e8b83a 100644
--- a/ui/qemu-pixman.c
+++ b/ui/qemu-pixman.c
@@ -3,6 +3,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 
@@ -84,7 +85,7 @@
         break;
         }
     }
-    g_assert_not_reached();
+    return 0;
 }
 
 int qemu_pixman_get_type(int rshift, int gshift, int bshift)
@@ -125,6 +126,33 @@
     return format;
 }
 
+/*
+ * Return true for known-good pixman conversions.
+ *
+ * UIs using pixman for format conversion can hook this into
+ * DisplayChangeListenerOps->dpy_gfx_check_format
+ */
+bool qemu_pixman_check_format(DisplayChangeListener *dcl,
+                              pixman_format_code_t format)
+{
+    switch (format) {
+    /* 32 bpp */
+    case PIXMAN_x8r8g8b8:
+    case PIXMAN_a8r8g8b8:
+    case PIXMAN_b8g8r8x8:
+    case PIXMAN_b8g8r8a8:
+    /* 24 bpp */
+    case PIXMAN_r8g8b8:
+    case PIXMAN_b8g8r8:
+    /* 16 bpp */
+    case PIXMAN_x1r5g5b5:
+    case PIXMAN_r5g6b5:
+        return true;
+    default:
+        return false;
+    }
+}
+
 pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
                                            int width)
 {
@@ -152,14 +180,11 @@
 pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
                                           pixman_image_t *image)
 {
-    pixman_image_t *mirror;
-
-    mirror = pixman_image_create_bits(format,
-                                      pixman_image_get_width(image),
-                                      pixman_image_get_height(image),
-                                      NULL,
-                                      pixman_image_get_stride(image));
-    return mirror;
+    return pixman_image_create_bits(format,
+                                    pixman_image_get_width(image),
+                                    pixman_image_get_height(image),
+                                    NULL,
+                                    pixman_image_get_stride(image));
 }
 
 void qemu_pixman_image_unref(pixman_image_t *image)
diff --git a/ui/sdl.c b/ui/sdl.c
index 26614ff..0ac8e53 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -25,12 +25,12 @@
 /* Avoid compiler warning because macro is redefined in SDL_syswm.h. */
 #undef WIN32_LEAN_AND_MEAN
 
+#include "qemu/osdep.h"
 #include <SDL.h>
-
-#if SDL_MAJOR_VERSION == 1
 #include <SDL_syswm.h>
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "ui/console.h"
 #include "ui/input.h"
 #include "sysemu/sysemu.h"
@@ -62,17 +62,30 @@
 static SDL_PixelFormat host_format;
 static int scaling_active = 0;
 static Notifier mouse_mode_notifier;
+static int idle_counter;
+
+#define SDL_REFRESH_INTERVAL_BUSY 10
+#define SDL_MAX_IDLE_COUNT (2 * GUI_REFRESH_INTERVAL_DEFAULT \
+                            / SDL_REFRESH_INTERVAL_BUSY + 1)
+
+#if 0
+#define DEBUG_SDL
+#endif
 
 static void sdl_update(DisplayChangeListener *dcl,
                        int x, int y, int w, int h)
 {
-    //    printf("updating x=%d y=%d w=%d h=%d\n", x, y, w, h);
     SDL_Rect rec;
     rec.x = x;
     rec.y = y;
     rec.w = w;
     rec.h = h;
 
+#ifdef DEBUG_SDL
+    printf("SDL: Updating x=%d y=%d w=%d h=%d (scaling: %d)\n",
+           x, y, w, h, scaling_active);
+#endif
+
     if (guest_screen) {
         if (!scaling_active) {
             SDL_BlitSurface(guest_screen, &rec, real_screen, &rec);
@@ -91,7 +104,9 @@
     int flags;
     SDL_Surface *tmp_screen;
 
-    //    printf("resizing to %d %d\n", w, h);
+#ifdef DEBUG_SDL
+    printf("SDL: Resizing to %dx%d bpp %d\n", width, height, bpp);
+#endif
 
     flags = SDL_HWSURFACE | SDL_ASYNCBLIT | SDL_HWACCEL;
     if (gui_fullscreen) {
@@ -127,12 +142,13 @@
 static void sdl_switch(DisplayChangeListener *dcl,
                        DisplaySurface *new_surface)
 {
-    PixelFormat pf = qemu_pixelformat_from_pixman(new_surface->format);
+    PixelFormat pf;
 
     /* temporary hack: allows to call sdl_switch to handle scaling changes */
     if (new_surface) {
         surface = new_surface;
     }
+    pf = qemu_pixelformat_from_pixman(surface->format);
 
     if (!scaling_active) {
         do_sdl_resize(surface_width(surface), surface_height(surface), 0);
@@ -145,6 +161,12 @@
     if (guest_screen != NULL) {
         SDL_FreeSurface(guest_screen);
     }
+
+#ifdef DEBUG_SDL
+    printf("SDL: Creating surface with masks: %08x %08x %08x %08x\n",
+           pf.rmask, pf.gmask, pf.bmask, pf.amask);
+#endif
+
     guest_screen = SDL_CreateRGBSurfaceFrom
         (surface_data(surface),
          surface_width(surface), surface_height(surface),
@@ -153,6 +175,19 @@
          pf.bmask, pf.amask);
 }
 
+static bool sdl_check_format(DisplayChangeListener *dcl,
+                             pixman_format_code_t format)
+{
+    /*
+     * We let SDL convert for us a few more formats than,
+     * the native ones. Thes are the ones I have tested.
+     */
+    return (format == PIXMAN_x8r8g8b8 ||
+            format == PIXMAN_b8g8r8x8 ||
+            format == PIXMAN_x1r5g5b5 ||
+            format == PIXMAN_r5g6b5);
+}
+
 /* generic keyboard conversion */
 
 #include "sdl_keysym.h"
@@ -437,7 +472,7 @@
 
 static void sdl_send_mouse_event(int dx, int dy, int x, int y, int state)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = SDL_BUTTON(SDL_BUTTON_LEFT),
         [INPUT_BUTTON_MIDDLE]     = SDL_BUTTON(SDL_BUTTON_MIDDLE),
         [INPUT_BUTTON_RIGHT]      = SDL_BUTTON(SDL_BUTTON_RIGHT),
@@ -475,6 +510,10 @@
 {
     int bpp = real_screen->format->BitsPerPixel;
 
+#ifdef DEBUG_SDL
+    printf("SDL: Scaling to %dx%d bpp %d\n", width, height, bpp);
+#endif
+
     if (bpp != 16 && bpp != 32) {
         bpp = 32;
     }
@@ -770,6 +809,7 @@
 static void sdl_refresh(DisplayChangeListener *dcl)
 {
     SDL_Event ev1, *ev = &ev1;
+    int idle = 1;
 
     if (last_vm_running != runstate_is_running()) {
         last_vm_running = runstate_is_running();
@@ -785,9 +825,11 @@
             sdl_update(dcl, 0, 0, real_screen->w, real_screen->h);
             break;
         case SDL_KEYDOWN:
+            idle = 0;
             handle_keydown(ev);
             break;
         case SDL_KEYUP:
+            idle = 0;
             handle_keyup(ev);
             break;
         case SDL_QUIT:
@@ -797,10 +839,12 @@
             }
             break;
         case SDL_MOUSEMOTION:
+            idle = 0;
             handle_mousemotion(ev);
             break;
         case SDL_MOUSEBUTTONDOWN:
         case SDL_MOUSEBUTTONUP:
+            idle = 0;
             handle_mousebutton(ev);
             break;
         case SDL_ACTIVEEVENT:
@@ -815,6 +859,18 @@
             break;
         }
     }
+
+    if (idle) {
+        if (idle_counter < SDL_MAX_IDLE_COUNT) {
+            idle_counter++;
+            if (idle_counter >= SDL_MAX_IDLE_COUNT) {
+                dcl->update_interval = GUI_REFRESH_INTERVAL_DEFAULT;
+            }
+        }
+    } else {
+        idle_counter = 0;
+        dcl->update_interval = SDL_REFRESH_INTERVAL_BUSY;
+    }
 }
 
 static void sdl_mouse_warp(DisplayChangeListener *dcl,
@@ -867,14 +923,25 @@
 }
 
 static const DisplayChangeListenerOps dcl_ops = {
-    .dpy_name          = "sdl",
-    .dpy_gfx_update    = sdl_update,
-    .dpy_gfx_switch    = sdl_switch,
-    .dpy_refresh       = sdl_refresh,
-    .dpy_mouse_set     = sdl_mouse_warp,
-    .dpy_cursor_define = sdl_mouse_define,
+    .dpy_name             = "sdl",
+    .dpy_gfx_update       = sdl_update,
+    .dpy_gfx_switch       = sdl_switch,
+    .dpy_gfx_check_format = sdl_check_format,
+    .dpy_refresh          = sdl_refresh,
+    .dpy_mouse_set        = sdl_mouse_warp,
+    .dpy_cursor_define    = sdl_mouse_define,
 };
 
+void sdl_display_early_init(int opengl)
+{
+    if (opengl == 1 /* on */) {
+        fprintf(stderr,
+                "SDL1 display code has no opengl support.\n"
+                "Please recompile qemu with SDL2, using\n"
+                "./configure --enable-sdl --with-sdlabi=2.0\n");
+    }
+}
+
 bool sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
 {
     int flags;
@@ -890,7 +957,7 @@
     if(keyboard_layout) {
         kbd_layout = init_keyboard_layout(name2keysym, keyboard_layout);
         if (!kbd_layout)
-            exit(1);
+            return false;
     }
 
     if (no_frame)
@@ -920,7 +987,7 @@
     if (SDL_Init (flags)) {
         fprintf(stderr, "Could not initialize SDL(%s) - exiting\n",
                 SDL_GetError());
-        exit(1);
+        return false;
     }
     vi = SDL_GetVideoInfo();
     host_format = *(vi->vfmt);
@@ -942,7 +1009,7 @@
         sdl_grab_start();
     }
 
-    dcl = g_malloc0(sizeof(DisplayChangeListener));
+    dcl = g_new0(DisplayChangeListener, 1);
     dcl->ops = &dcl_ops;
     register_displaychangelistener(dcl);
 
@@ -957,7 +1024,5 @@
     sdl_cursor_normal = SDL_GetCursor();
 
     atexit(sdl_cleanup);
-
     return true;
 }
-#endif
diff --git a/ui/sdl2-2d.c b/ui/sdl2-2d.c
new file mode 100644
index 0000000..8ab68d6
--- /dev/null
+++ b/ui/sdl2-2d.c
@@ -0,0 +1,164 @@
+/*
+ * QEMU SDL display driver
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+/* Ported SDL 1.2 code to 2.0 by Dave Airlie. */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/input.h"
+#include "ui/sdl2.h"
+#include "sysemu/sysemu.h"
+
+void sdl2_2d_update(DisplayChangeListener *dcl,
+                    int x, int y, int w, int h)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    DisplaySurface *surf = qemu_console_surface(dcl->con);
+    SDL_Rect rect;
+
+    assert(!scon->opengl);
+
+    if (!surf) {
+        return;
+    }
+    if (!scon->texture) {
+        return;
+    }
+
+    /*
+     * SDL2 seems to do some double-buffering, and trying to only
+     * update the changed areas results in only one of the two buffers
+     * being updated.  Which flickers alot.  So lets not try to be
+     * clever do a full update every time ...
+     */
+#if 0
+    rect.x = x;
+    rect.y = y;
+    rect.w = w;
+    rect.h = h;
+#else
+    rect.x = 0;
+    rect.y = 0;
+    rect.w = surface_width(surf);
+    rect.h = surface_height(surf);
+#endif
+
+    SDL_UpdateTexture(scon->texture, NULL, surface_data(surf),
+                      surface_stride(surf));
+    SDL_RenderCopy(scon->real_renderer, scon->texture, &rect, &rect);
+    SDL_RenderPresent(scon->real_renderer);
+}
+
+void sdl2_2d_switch(DisplayChangeListener *dcl,
+                    DisplaySurface *new_surface)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    DisplaySurface *old_surface = scon->surface;
+    int format = 0;
+
+    assert(!scon->opengl);
+
+    scon->surface = new_surface;
+
+    if (scon->texture) {
+        SDL_DestroyTexture(scon->texture);
+        scon->texture = NULL;
+    }
+
+    if (!new_surface) {
+        sdl2_window_destroy(scon);
+        return;
+    }
+
+    if (!scon->real_window) {
+        sdl2_window_create(scon);
+    } else if (old_surface &&
+               ((surface_width(old_surface)  != surface_width(new_surface)) ||
+                (surface_height(old_surface) != surface_height(new_surface)))) {
+        sdl2_window_resize(scon);
+    }
+
+    SDL_RenderSetLogicalSize(scon->real_renderer,
+                             surface_width(new_surface),
+                             surface_height(new_surface));
+
+    switch (surface_format(scon->surface)) {
+    case PIXMAN_x1r5g5b5:
+        format = SDL_PIXELFORMAT_ARGB1555;
+        break;
+    case PIXMAN_r5g6b5:
+        format = SDL_PIXELFORMAT_RGB565;
+        break;
+    case PIXMAN_x8r8g8b8:
+        format = SDL_PIXELFORMAT_ARGB8888;
+        break;
+    case PIXMAN_r8g8b8x8:
+        format = SDL_PIXELFORMAT_RGBA8888;
+        break;
+    case PIXMAN_b8g8r8x8:
+        format = SDL_PIXELFORMAT_BGRX8888;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+    scon->texture = SDL_CreateTexture(scon->real_renderer, format,
+                                      SDL_TEXTUREACCESS_STREAMING,
+                                      surface_width(new_surface),
+                                      surface_height(new_surface));
+    sdl2_2d_redraw(scon);
+}
+
+void sdl2_2d_refresh(DisplayChangeListener *dcl)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+
+    assert(!scon->opengl);
+    graphic_hw_update(dcl->con);
+    sdl2_poll_events(scon);
+}
+
+void sdl2_2d_redraw(struct sdl2_console *scon)
+{
+    assert(!scon->opengl);
+
+    if (!scon->surface) {
+        return;
+    }
+    sdl2_2d_update(&scon->dcl, 0, 0,
+                   surface_width(scon->surface),
+                   surface_height(scon->surface));
+}
+
+bool sdl2_2d_check_format(DisplayChangeListener *dcl,
+                          pixman_format_code_t format)
+{
+    /*
+     * We let SDL convert for us a few more formats than,
+     * the native ones. Thes are the ones I have tested.
+     */
+    return (format == PIXMAN_x8r8g8b8 ||
+            format == PIXMAN_b8g8r8x8 ||
+            format == PIXMAN_x1r5g5b5 ||
+            format == PIXMAN_r5g6b5);
+}
diff --git a/ui/sdl2-gl.c b/ui/sdl2-gl.c
new file mode 100644
index 0000000..039645d
--- /dev/null
+++ b/ui/sdl2-gl.c
@@ -0,0 +1,249 @@
+/*
+ * QEMU SDL display driver -- opengl support
+ *
+ * Copyright (c) 2014 Red Hat
+ *
+ * Authors:
+ *     Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/input.h"
+#include "ui/sdl2.h"
+#include "sysemu/sysemu.h"
+
+#include <epoxy/gl.h>
+
+static void sdl2_set_scanout_mode(struct sdl2_console *scon, bool scanout)
+{
+    if (scon->scanout_mode == scanout) {
+        return;
+    }
+
+    scon->scanout_mode = scanout;
+    if (!scon->scanout_mode) {
+        if (scon->fbo_id) {
+            glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT,
+                                      GL_COLOR_ATTACHMENT0_EXT,
+                                      GL_TEXTURE_2D, 0, 0);
+            glDeleteFramebuffers(1, &scon->fbo_id);
+            glBindFramebuffer(GL_FRAMEBUFFER_EXT, 0);
+            scon->fbo_id = 0;
+        }
+        if (scon->surface) {
+            surface_gl_destroy_texture(scon->gls, scon->surface);
+            surface_gl_create_texture(scon->gls, scon->surface);
+        }
+    }
+}
+
+static void sdl2_gl_render_surface(struct sdl2_console *scon)
+{
+    int ww, wh;
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+    sdl2_set_scanout_mode(scon, false);
+
+    SDL_GetWindowSize(scon->real_window, &ww, &wh);
+    surface_gl_setup_viewport(scon->gls, scon->surface, ww, wh);
+
+    surface_gl_render_texture(scon->gls, scon->surface);
+    SDL_GL_SwapWindow(scon->real_window);
+}
+
+void sdl2_gl_update(DisplayChangeListener *dcl,
+                    int x, int y, int w, int h)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+
+    assert(scon->opengl);
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+    surface_gl_update_texture(scon->gls, scon->surface, x, y, w, h);
+    scon->updates++;
+}
+
+void sdl2_gl_switch(DisplayChangeListener *dcl,
+                    DisplaySurface *new_surface)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    DisplaySurface *old_surface = scon->surface;
+
+    assert(scon->opengl);
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+    surface_gl_destroy_texture(scon->gls, scon->surface);
+
+    scon->surface = new_surface;
+
+    if (!new_surface) {
+        console_gl_fini_context(scon->gls);
+        scon->gls = NULL;
+        sdl2_window_destroy(scon);
+        return;
+    }
+
+    if (!scon->real_window) {
+        sdl2_window_create(scon);
+        scon->gls = console_gl_init_context();
+    } else if (old_surface &&
+               ((surface_width(old_surface)  != surface_width(new_surface)) ||
+                (surface_height(old_surface) != surface_height(new_surface)))) {
+        sdl2_window_resize(scon);
+    }
+
+    surface_gl_create_texture(scon->gls, scon->surface);
+}
+
+void sdl2_gl_refresh(DisplayChangeListener *dcl)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+
+    assert(scon->opengl);
+
+    graphic_hw_update(dcl->con);
+    if (scon->updates && scon->surface) {
+        scon->updates = 0;
+        sdl2_gl_render_surface(scon);
+    }
+    sdl2_poll_events(scon);
+}
+
+void sdl2_gl_redraw(struct sdl2_console *scon)
+{
+    assert(scon->opengl);
+
+    if (scon->surface) {
+        sdl2_gl_render_surface(scon);
+    }
+}
+
+QEMUGLContext sdl2_gl_create_context(DisplayChangeListener *dcl,
+                                     QEMUGLParams *params)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    SDL_GLContext ctx;
+
+    assert(scon->opengl);
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+
+    SDL_GL_SetAttribute(SDL_GL_SHARE_WITH_CURRENT_CONTEXT, 1);
+    SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK,
+                        SDL_GL_CONTEXT_PROFILE_CORE);
+    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, params->major_ver);
+    SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, params->minor_ver);
+
+    ctx = SDL_GL_CreateContext(scon->real_window);
+    return (QEMUGLContext)ctx;
+}
+
+void sdl2_gl_destroy_context(DisplayChangeListener *dcl, QEMUGLContext ctx)
+{
+    SDL_GLContext sdlctx = (SDL_GLContext)ctx;
+
+    SDL_GL_DeleteContext(sdlctx);
+}
+
+int sdl2_gl_make_context_current(DisplayChangeListener *dcl,
+                                 QEMUGLContext ctx)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    SDL_GLContext sdlctx = (SDL_GLContext)ctx;
+
+    assert(scon->opengl);
+
+    return SDL_GL_MakeCurrent(scon->real_window, sdlctx);
+}
+
+QEMUGLContext sdl2_gl_get_current_context(DisplayChangeListener *dcl)
+{
+    SDL_GLContext sdlctx;
+
+    sdlctx = SDL_GL_GetCurrentContext();
+    return (QEMUGLContext)sdlctx;
+}
+
+void sdl2_gl_scanout(DisplayChangeListener *dcl,
+                     uint32_t backing_id, bool backing_y_0_top,
+                     uint32_t backing_width, uint32_t backing_height,
+                     uint32_t x, uint32_t y,
+                     uint32_t w, uint32_t h)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+
+    assert(scon->opengl);
+    scon->x = x;
+    scon->y = y;
+    scon->w = w;
+    scon->h = h;
+    scon->tex_id = backing_id;
+    scon->y0_top = backing_y_0_top;
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+
+    if (scon->tex_id == 0 || scon->w == 0 || scon->h == 0) {
+        sdl2_set_scanout_mode(scon, false);
+        return;
+    }
+
+    sdl2_set_scanout_mode(scon, true);
+    if (!scon->fbo_id) {
+        glGenFramebuffers(1, &scon->fbo_id);
+    }
+
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, scon->fbo_id);
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT,
+                              GL_TEXTURE_2D, scon->tex_id, 0);
+}
+
+void sdl2_gl_scanout_flush(DisplayChangeListener *dcl,
+                           uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
+    int ww, wh, y1, y2;
+
+    assert(scon->opengl);
+    if (!scon->scanout_mode) {
+        return;
+    }
+    if (!scon->fbo_id) {
+        return;
+    }
+
+    SDL_GL_MakeCurrent(scon->real_window, scon->winctx);
+
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, scon->fbo_id);
+    glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
+
+    SDL_GetWindowSize(scon->real_window, &ww, &wh);
+    glViewport(0, 0, ww, wh);
+    y1 = scon->y0_top ? 0 : scon->h;
+    y2 = scon->y0_top ? scon->h : 0;
+    glBlitFramebuffer(0, y1, scon->w, y2,
+                      0, 0, ww, wh,
+                      GL_COLOR_BUFFER_BIT, GL_NEAREST);
+    glBindFramebuffer(GL_FRAMEBUFFER_EXT, scon->fbo_id);
+
+    SDL_GL_SwapWindow(scon->real_window);
+}
diff --git a/ui/sdl2-input.c b/ui/sdl2-input.c
new file mode 100644
index 0000000..6e315ae
--- /dev/null
+++ b/ui/sdl2-input.c
@@ -0,0 +1,101 @@
+/*
+ * QEMU SDL display driver
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+/* Ported SDL 1.2 code to 2.0 by Dave Airlie. */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/console.h"
+#include "ui/input.h"
+#include "ui/sdl2.h"
+#include "sysemu/sysemu.h"
+
+#include "sdl2-keymap.h"
+
+static uint8_t modifiers_state[SDL_NUM_SCANCODES];
+
+void sdl2_reset_keys(struct sdl2_console *scon)
+{
+    QemuConsole *con = scon ? scon->dcl.con : NULL;
+    int i;
+
+    for (i = 0; i < SDL_NUM_SCANCODES; i++) {
+        if (modifiers_state[i]) {
+            int qcode = sdl2_scancode_to_qcode[i];
+            qemu_input_event_send_key_qcode(con, qcode, false);
+            modifiers_state[i] = 0;
+        }
+    }
+}
+
+void sdl2_process_key(struct sdl2_console *scon,
+                      SDL_KeyboardEvent *ev)
+{
+    int qcode = sdl2_scancode_to_qcode[ev->keysym.scancode];
+    QemuConsole *con = scon ? scon->dcl.con : NULL;
+
+    if (!qemu_console_is_graphic(con)) {
+        if (ev->type == SDL_KEYDOWN) {
+            switch (ev->keysym.scancode) {
+            case SDL_SCANCODE_RETURN:
+                kbd_put_keysym_console(con, '\n');
+                break;
+            case SDL_SCANCODE_BACKSPACE:
+                kbd_put_keysym_console(con, QEMU_KEY_BACKSPACE);
+                break;
+            default:
+                kbd_put_qcode_console(con, qcode);
+                break;
+            }
+        }
+        return;
+    }
+
+    switch (ev->keysym.scancode) {
+#if 0
+    case SDL_SCANCODE_NUMLOCKCLEAR:
+    case SDL_SCANCODE_CAPSLOCK:
+        /* SDL does not send the key up event, so we generate it */
+        qemu_input_event_send_key_qcode(con, qcode, true);
+        qemu_input_event_send_key_qcode(con, qcode, false);
+        return;
+#endif
+    case SDL_SCANCODE_LCTRL:
+    case SDL_SCANCODE_LSHIFT:
+    case SDL_SCANCODE_LALT:
+    case SDL_SCANCODE_LGUI:
+    case SDL_SCANCODE_RCTRL:
+    case SDL_SCANCODE_RSHIFT:
+    case SDL_SCANCODE_RALT:
+    case SDL_SCANCODE_RGUI:
+        if (ev->type == SDL_KEYUP) {
+            modifiers_state[ev->keysym.scancode] = 0;
+        } else {
+            modifiers_state[ev->keysym.scancode] = 1;
+        }
+        /* fall though */
+    default:
+        qemu_input_event_send_key_qcode(con, qcode,
+                                        ev->type == SDL_KEYDOWN);
+    }
+}
diff --git a/ui/sdl2.c b/ui/sdl2.c
index 88b0327..4e3ab6d 100644
--- a/ui/sdl2.c
+++ b/ui/sdl2.c
@@ -23,67 +23,40 @@
  */
 /* Ported SDL 1.2 code to 2.0 by Dave Airlie. */
 
-/* Avoid compiler warning because macro is redefined in SDL_syswm.h. */
-#undef WIN32_LEAN_AND_MEAN
-
-#include <SDL.h>
-
-#if SDL_MAJOR_VERSION == 2
-#include <SDL_syswm.h>
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/console.h"
 #include "ui/input.h"
+#include "ui/sdl2.h"
 #include "sysemu/sysemu.h"
 
-#include "sdl2-keymap.h"
-
-#define DEBUG 0
-
-#if DEBUG
-#define D(...)  printf(__VA_ARGS__), fflush(stdout)
-#else
-#define D(...)  ((void)0)
-#endif
-
 static int sdl2_num_outputs;
-static struct sdl2_state {
-    DisplayChangeListener dcl;
-    DisplaySurface *surface;
-    SDL_Texture *texture;
-    SDL_Window *real_window;
-    SDL_Renderer *real_renderer;
-    int idx;
-    int last_vm_running; /* per console for caption reasons */
-    int x, y;
-    int hidden;
-} *sdl2_console;
+static struct sdl2_console *sdl2_console;
 
 static SDL_Surface *guest_sprite_surface;
 static int gui_grab; /* if true, all keyboard/mouse events are grabbed */
 
-static bool gui_saved_scaling;
-static int gui_saved_width;
-static int gui_saved_height;
 static int gui_saved_grab;
 static int gui_fullscreen;
 static int gui_noframe;
 static int gui_key_modifier_pressed;
 static int gui_keysym;
 static int gui_grab_code = KMOD_LALT | KMOD_LCTRL;
-static uint8_t modifiers_state[SDL_NUM_SCANCODES];
 static SDL_Cursor *sdl_cursor_normal;
 static SDL_Cursor *sdl_cursor_hidden;
 static int absolute_enabled;
 static int guest_cursor;
 static int guest_x, guest_y;
 static SDL_Cursor *guest_sprite;
-static int scaling_active;
 static Notifier mouse_mode_notifier;
 
-static void sdl_update_caption(struct sdl2_state *scon);
+#define SDL2_REFRESH_INTERVAL_BUSY 10
+#define SDL2_MAX_IDLE_COUNT (2 * GUI_REFRESH_INTERVAL_DEFAULT \
+                             / SDL2_REFRESH_INTERVAL_BUSY + 1)
 
-static struct sdl2_state *get_scon_from_window(uint32_t window_id)
+static void sdl_update_caption(struct sdl2_console *scon);
+
+static struct sdl2_console *get_scon_from_window(uint32_t window_id)
 {
     int i;
     for (i = 0; i < sdl2_num_outputs; i++) {
@@ -94,187 +67,71 @@
     return NULL;
 }
 
-static void sdl_update(DisplayChangeListener *dcl,
-                       int x, int y, int w, int h)
+void sdl2_window_create(struct sdl2_console *scon)
 {
-    struct sdl2_state *scon = container_of(dcl, struct sdl2_state, dcl);
-    SDL_Rect rect;
-    DisplaySurface *surf = qemu_console_surface(dcl->con);
+    int flags = 0;
 
-    D("%s: scon=%p surface=%p texture=%p\n", __FUNCTION__, scon, surf, scon->texture);
-
-    if (!surf) {
+    if (!scon->surface) {
         return;
     }
-    if (!scon->texture) {
-        return;
-    }
+    assert(!scon->real_window);
 
-    rect.x = x;
-    rect.y = y;
-    rect.w = w;
-    rect.h = h;
-
-    SDL_UpdateTexture(scon->texture, NULL, surface_data(surf),
-                      surface_stride(surf));
-    SDL_RenderCopy(scon->real_renderer, scon->texture, &rect, &rect);
-    SDL_RenderPresent(scon->real_renderer);
-}
-
-static void do_sdl_resize(struct sdl2_state *scon, int width, int height,
-                          int bpp)
-{
-    int flags;
-
-    if (scon->real_window && scon->real_renderer) {
-        if (width && height) {
-            SDL_RenderSetLogicalSize(scon->real_renderer, width, height);
-            SDL_SetWindowSize(scon->real_window, width, height);
-        } else {
-            SDL_DestroyRenderer(scon->real_renderer);
-            SDL_DestroyWindow(scon->real_window);
-            scon->real_renderer = NULL;
-            scon->real_window = NULL;
-        }
+    if (gui_fullscreen) {
+        flags |= SDL_WINDOW_FULLSCREEN_DESKTOP;
     } else {
-        if (!width || !height) {
-            return;
-        }
-        flags = 0;
-        if (gui_fullscreen) {
-            flags |= SDL_WINDOW_FULLSCREEN;
-        } else {
-            flags |= SDL_WINDOW_RESIZABLE;
-        }
-        if (scon->hidden) {
-            flags |= SDL_WINDOW_HIDDEN;
-        }
-
-        scon->real_window = SDL_CreateWindow("", SDL_WINDOWPOS_UNDEFINED,
-                                             SDL_WINDOWPOS_UNDEFINED,
-                                             width, height, flags);
-        scon->real_renderer = SDL_CreateRenderer(scon->real_window, -1, 0);
-
-        sdl_update_caption(scon);
+        flags |= SDL_WINDOW_RESIZABLE;
     }
+    if (scon->hidden) {
+        flags |= SDL_WINDOW_HIDDEN;
+    }
+
+    scon->real_window = SDL_CreateWindow("", SDL_WINDOWPOS_UNDEFINED,
+                                         SDL_WINDOWPOS_UNDEFINED,
+                                         surface_width(scon->surface),
+                                         surface_height(scon->surface),
+                                         flags);
+    scon->real_renderer = SDL_CreateRenderer(scon->real_window, -1, 0);
+    if (scon->opengl) {
+        scon->winctx = SDL_GL_GetCurrentContext();
+    }
+    sdl_update_caption(scon);
 }
 
-static void sdl_switch(DisplayChangeListener *dcl,
-                       DisplaySurface *new_surface)
+void sdl2_window_destroy(struct sdl2_console *scon)
 {
-    struct sdl2_state *scon = container_of(dcl, struct sdl2_state, dcl);
-    int format = 0;
-    int idx = scon->idx;
-    DisplaySurface *old_surface = scon->surface;
-
-    D("%s: scon=%p old_surface=%p new_surface=%p is_graphics=%s\n",
-      __FUNCTION__, scon, scon->surface, new_surface,
-        qemu_console_is_graphic(dcl->con) ? "Y" : "N");
-
-    /* temporary hack: allows to call sdl_switch to handle scaling changes */
-    if (new_surface) {
-        scon->surface = new_surface;
-    }
-
-    if (!new_surface && idx > 0) {
-        scon->surface = NULL;
-    }
-
-    if (new_surface == NULL) {
-        do_sdl_resize(scon, 0, 0, 0);
-    } else {
-        do_sdl_resize(scon, surface_width(scon->surface),
-                      surface_height(scon->surface), 0);
-    }
-
-    if (old_surface && scon->texture) {
-        SDL_DestroyTexture(scon->texture);
-        scon->texture = NULL;
-    }
-
-    if (new_surface) {
-        if (!scon->texture) {
-            if (surface_bits_per_pixel(scon->surface) == 16) {
-                format = SDL_PIXELFORMAT_RGB565;
-            } else if (surface_bits_per_pixel(scon->surface) == 32) {
-                format = SDL_PIXELFORMAT_ARGB8888;
-            }
-
-            scon->texture = SDL_CreateTexture(scon->real_renderer, format,
-                                              SDL_TEXTUREACCESS_STREAMING,
-                                              surface_width(new_surface),
-                                              surface_height(new_surface));
-        }
-    }
-}
-
-static void reset_keys(struct sdl2_state *scon)
-{
-    QemuConsole *con = scon ? scon->dcl.con : NULL;
-    int i;
-
-    for (i = 0; i < 256; i++) {
-        if (modifiers_state[i]) {
-            int qcode = sdl2_scancode_to_qcode[i];
-            qemu_input_event_send_key_qcode(con, qcode, false);
-            modifiers_state[i] = 0;
-        }
-    }
-}
-
-static void sdl_process_key(struct sdl2_state *scon,
-                            SDL_KeyboardEvent *ev)
-{
-    int qcode = sdl2_scancode_to_qcode[ev->keysym.scancode];
-    QemuConsole *con = scon ? scon->dcl.con : NULL;
-
-    if (!qemu_console_is_graphic(con)) {
-        if (ev->type == SDL_KEYDOWN) {
-            switch (ev->keysym.scancode) {
-            case SDL_SCANCODE_RETURN:
-                kbd_put_keysym_console(con, '\n');
-                break;
-            case SDL_SCANCODE_BACKSPACE:
-                kbd_put_keysym_console(con, QEMU_KEY_BACKSPACE);
-                break;
-            default:
-                kbd_put_qcode_console(con, qcode);
-                break;
-            }
-        }
+    if (!scon->real_window) {
         return;
     }
 
-    switch (ev->keysym.scancode) {
-#if 0
-    case SDL_SCANCODE_NUMLOCKCLEAR:
-    case SDL_SCANCODE_CAPSLOCK:
-        /* SDL does not send the key up event, so we generate it */
-        qemu_input_event_send_key_qcode(con, qcode, true);
-        qemu_input_event_send_key_qcode(con, qcode, false);
+    SDL_DestroyRenderer(scon->real_renderer);
+    scon->real_renderer = NULL;
+    SDL_DestroyWindow(scon->real_window);
+    scon->real_window = NULL;
+}
+
+void sdl2_window_resize(struct sdl2_console *scon)
+{
+    if (!scon->real_window) {
         return;
+    }
+
+    SDL_SetWindowSize(scon->real_window,
+                      surface_width(scon->surface),
+                      surface_height(scon->surface));
+}
+
+static void sdl2_redraw(struct sdl2_console *scon)
+{
+    if (scon->opengl) {
+#ifdef CONFIG_OPENGL
+        sdl2_gl_redraw(scon);
 #endif
-    case SDL_SCANCODE_LCTRL:
-    case SDL_SCANCODE_LSHIFT:
-    case SDL_SCANCODE_LALT:
-    case SDL_SCANCODE_LGUI:
-    case SDL_SCANCODE_RCTRL:
-    case SDL_SCANCODE_RSHIFT:
-    case SDL_SCANCODE_RALT:
-    case SDL_SCANCODE_RGUI:
-        if (ev->type == SDL_KEYUP) {
-            modifiers_state[ev->keysym.scancode] = 0;
-        } else {
-            modifiers_state[ev->keysym.scancode] = 1;
-        }
-        /* fall though */
-    default:
-        qemu_input_event_send_key_qcode(con, qcode,
-                                        ev->type == SDL_KEYDOWN);
+    } else {
+        sdl2_2d_redraw(scon);
     }
 }
 
-static void sdl_update_caption(struct sdl2_state *scon)
+static void sdl_update_caption(struct sdl2_console *scon)
 {
     char win_title[1024];
     char icon_title[1024];
@@ -284,11 +141,11 @@
         status = " [Stopped]";
     } else if (gui_grab) {
         if (alt_grab) {
-            status = " - Press Ctrl-Alt-Shift to exit mouse grab";
+            status = " - Press Ctrl-Alt-Shift to exit grab";
         } else if (ctrl_grab) {
-            status = " - Press Right-Ctrl to exit mouse grab";
+            status = " - Press Right-Ctrl to exit grab";
         } else {
-            status = " - Press Ctrl-Alt to exit mouse grab";
+            status = " - Press Ctrl-Alt to exit grab";
         }
     }
 
@@ -338,7 +195,7 @@
     }
 }
 
-static void sdl_grab_start(struct sdl2_state *scon)
+static void sdl_grab_start(struct sdl2_console *scon)
 {
     QemuConsole *con = scon ? scon->dcl.con : NULL;
 
@@ -366,7 +223,7 @@
     sdl_update_caption(scon);
 }
 
-static void sdl_grab_end(struct sdl2_state *scon)
+static void sdl_grab_end(struct sdl2_console *scon)
 {
     SDL_SetWindowGrab(scon->real_window, SDL_FALSE);
     gui_grab = 0;
@@ -374,7 +231,7 @@
     sdl_update_caption(scon);
 }
 
-static void absolute_mouse_grab(struct sdl2_state *scon)
+static void absolute_mouse_grab(struct sdl2_console *scon)
 {
     int mouse_x, mouse_y;
     int scr_w, scr_h;
@@ -401,10 +258,10 @@
     }
 }
 
-static void sdl_send_mouse_event(struct sdl2_state *scon, int dx, int dy,
+static void sdl_send_mouse_event(struct sdl2_console *scon, int dx, int dy,
                                  int x, int y, int state)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = SDL_BUTTON(SDL_BUTTON_LEFT),
         [INPUT_BUTTON_MIDDLE]     = SDL_BUTTON(SDL_BUTTON_MIDDLE),
         [INPUT_BUTTON_RIGHT]      = SDL_BUTTON(SDL_BUTTON_RIGHT),
@@ -424,7 +281,7 @@
         int i;
 
         for (i = 0; i < sdl2_num_outputs; i++) {
-            struct sdl2_state *thiscon = &sdl2_console[i];
+            struct sdl2_console *thiscon = &sdl2_console[i];
             if (thiscon->real_window && thiscon->surface) {
                 SDL_GetWindowSize(thiscon->real_window, &scr_w, &scr_h);
                 cur_off_x = thiscon->x;
@@ -458,48 +315,27 @@
     qemu_input_event_sync();
 }
 
-static void sdl_scale(struct sdl2_state *scon, int width, int height)
+static void toggle_full_screen(struct sdl2_console *scon)
 {
-    int bpp = 0;
-    do_sdl_resize(scon, width, height, bpp);
-    scaling_active = 1;
-}
-
-static void toggle_full_screen(struct sdl2_state *scon)
-{
-    int width = surface_width(scon->surface);
-    int height = surface_height(scon->surface);
-    int bpp = surface_bits_per_pixel(scon->surface);
-
     gui_fullscreen = !gui_fullscreen;
     if (gui_fullscreen) {
-        SDL_GetWindowSize(scon->real_window,
-                          &gui_saved_width, &gui_saved_height);
-        gui_saved_scaling = scaling_active;
-
-        do_sdl_resize(scon, width, height, bpp);
-        scaling_active = 0;
-
+        SDL_SetWindowFullscreen(scon->real_window,
+                                SDL_WINDOW_FULLSCREEN_DESKTOP);
         gui_saved_grab = gui_grab;
         sdl_grab_start(scon);
     } else {
-        if (gui_saved_scaling) {
-            sdl_scale(scon, gui_saved_width, gui_saved_height);
-        } else {
-            do_sdl_resize(scon, width, height, 0);
-        }
         if (!gui_saved_grab) {
             sdl_grab_end(scon);
         }
+        SDL_SetWindowFullscreen(scon->real_window, 0);
     }
-    graphic_hw_invalidate(scon->dcl.con);
-    graphic_hw_update(scon->dcl.con);
+    sdl2_redraw(scon);
 }
 
 static void handle_keydown(SDL_Event *ev)
 {
     int mod_state, win;
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
+    struct sdl2_console *scon = get_scon_from_window(ev->key.windowID);
 
     if (alt_grab) {
         mod_state = (SDL_GetModState() & (gui_grab_code | KMOD_LSHIFT)) ==
@@ -521,6 +357,10 @@
         case SDL_SCANCODE_7:
         case SDL_SCANCODE_8:
         case SDL_SCANCODE_9:
+            if (gui_grab) {
+                sdl_grab_end(scon);
+            }
+
             win = ev->key.keysym.scancode - SDL_SCANCODE_1;
             if (win < sdl2_num_outputs) {
                 sdl2_console[win].hidden = !sdl2_console[win].hidden;
@@ -539,14 +379,15 @@
             gui_keysym = 1;
             break;
         case SDL_SCANCODE_U:
-            if (scaling_active) {
-                scaling_active = 0;
-                sdl_switch(&scon->dcl, NULL);
-                graphic_hw_invalidate(scon->dcl.con);
-                graphic_hw_update(scon->dcl.con);
+            sdl2_window_destroy(scon);
+            sdl2_window_create(scon);
+            if (!scon->opengl) {
+                /* re-create scon->texture */
+                sdl2_2d_switch(&scon->dcl, scon->surface);
             }
             gui_keysym = 1;
             break;
+#if 0
         case SDL_SCANCODE_KP_PLUS:
         case SDL_SCANCODE_KP_MINUS:
             if (!gui_fullscreen) {
@@ -559,25 +400,26 @@
                             160);
                 height = (surface_height(scon->surface) * width) /
                     surface_width(scon->surface);
-
+                fprintf(stderr, "%s: scale to %dx%d\n",
+                        __func__, width, height);
                 sdl_scale(scon, width, height);
-                graphic_hw_invalidate(NULL);
-                graphic_hw_update(NULL);
+                sdl2_redraw(scon);
                 gui_keysym = 1;
             }
+#endif
         default:
             break;
         }
     }
     if (!gui_keysym) {
-        sdl_process_key(scon, &ev->key);
+        sdl2_process_key(scon, &ev->key);
     }
 }
 
 static void handle_keyup(SDL_Event *ev)
 {
     int mod_state;
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
+    struct sdl2_console *scon = get_scon_from_window(ev->key.windowID);
 
     if (!alt_grab) {
         mod_state = (ev->key.keysym.mod & gui_grab_code);
@@ -595,19 +437,19 @@
             }
             /* SDL does not send back all the modifiers key, so we must
              * correct it. */
-            reset_keys(scon);
+            sdl2_reset_keys(scon);
             return;
         }
         gui_keysym = 0;
     }
     if (!gui_keysym) {
-        sdl_process_key(scon, &ev->key);
+        sdl2_process_key(scon, &ev->key);
     }
 }
 
 static void handle_textinput(SDL_Event *ev)
 {
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
+    struct sdl2_console *scon = get_scon_from_window(ev->key.windowID);
     QemuConsole *con = scon ? scon->dcl.con : NULL;
 
     if (qemu_console_is_graphic(con)) {
@@ -619,11 +461,7 @@
 static void handle_mousemotion(SDL_Event *ev)
 {
     int max_x, max_y;
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
-
-    if (!scon) {
-        return;
-    }
+    struct sdl2_console *scon = get_scon_from_window(ev->key.windowID);
 
     if (qemu_input_is_absolute() || absolute_enabled) {
         int scr_w, scr_h;
@@ -650,11 +488,7 @@
 {
     int buttonstate = SDL_GetMouseState(NULL, NULL);
     SDL_MouseButtonEvent *bev;
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
-
-    if (!scon) {
-        return;
-    }
+    struct sdl2_console *scon = get_scon_from_window(ev->key.windowID);
 
     bev = &ev->button;
     if (!gui_grab && !qemu_input_is_absolute()) {
@@ -674,14 +508,10 @@
 
 static void handle_mousewheel(SDL_Event *ev)
 {
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
+    struct sdl2_console *scon = get_scon_from_window(ev->key.windowID);
     SDL_MouseWheelEvent *wev = &ev->wheel;
     InputButton btn;
 
-    if (!scon) {
-        return;
-    }
-
     if (wev->y > 0) {
         btn = INPUT_BUTTON_WHEEL_UP;
     } else if (wev->y < 0) {
@@ -696,10 +526,9 @@
     qemu_input_event_sync();
 }
 
-static void handle_windowevent(DisplayChangeListener *dcl, SDL_Event *ev)
+static void handle_windowevent(SDL_Event *ev)
 {
-    int w, h;
-    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
+    struct sdl2_console *scon = get_scon_from_window(ev->window.windowID);
 
     if (!scon) {
         return;
@@ -707,7 +536,6 @@
 
     switch (ev->window.event) {
     case SDL_WINDOWEVENT_RESIZED:
-        sdl_scale(scon, ev->window.data1, ev->window.data2);
         {
             QemuUIInfo info;
             memset(&info, 0, sizeof(info));
@@ -715,12 +543,10 @@
             info.height = ev->window.data2;
             dpy_set_ui_info(scon->dcl.con, &info);
         }
-        graphic_hw_invalidate(scon->dcl.con);
-        graphic_hw_update(scon->dcl.con);
+        sdl2_redraw(scon);
         break;
     case SDL_WINDOWEVENT_EXPOSED:
-        SDL_GetWindowSize(SDL_GetWindowFromID(ev->window.windowID), &w, &h);
-        sdl_update(dcl, 0, 0, w, h);
+        sdl2_redraw(scon);
         break;
     case SDL_WINDOWEVENT_FOCUS_GAINED:
     case SDL_WINDOWEVENT_ENTER:
@@ -734,10 +560,10 @@
         }
         break;
     case SDL_WINDOWEVENT_RESTORED:
-        update_displaychangelistener(dcl, GUI_REFRESH_INTERVAL_DEFAULT);
+        update_displaychangelistener(&scon->dcl, GUI_REFRESH_INTERVAL_DEFAULT);
         break;
     case SDL_WINDOWEVENT_MINIMIZED:
-        update_displaychangelistener(dcl, 500);
+        update_displaychangelistener(&scon->dcl, 500);
         break;
     case SDL_WINDOWEVENT_CLOSE:
         if (!no_quit) {
@@ -758,27 +584,28 @@
     }
 }
 
-static void sdl_refresh(DisplayChangeListener *dcl)
+void sdl2_poll_events(struct sdl2_console *scon)
 {
-    struct sdl2_state *scon = container_of(dcl, struct sdl2_state, dcl);
     SDL_Event ev1, *ev = &ev1;
+    int idle = 1;
 
     if (scon->last_vm_running != runstate_is_running()) {
         scon->last_vm_running = runstate_is_running();
         sdl_update_caption(scon);
     }
 
-    graphic_hw_update(dcl->con);
-
     while (SDL_PollEvent(ev)) {
         switch (ev->type) {
         case SDL_KEYDOWN:
+            idle = 0;
             handle_keydown(ev);
             break;
         case SDL_KEYUP:
+            idle = 0;
             handle_keyup(ev);
             break;
         case SDL_TEXTINPUT:
+            idle = 0;
             handle_textinput(ev);
             break;
         case SDL_QUIT:
@@ -788,28 +615,43 @@
             }
             break;
         case SDL_MOUSEMOTION:
+            idle = 0;
             handle_mousemotion(ev);
             break;
         case SDL_MOUSEBUTTONDOWN:
         case SDL_MOUSEBUTTONUP:
+            idle = 0;
             handle_mousebutton(ev);
             break;
         case SDL_MOUSEWHEEL:
+            idle = 0;
             handle_mousewheel(ev);
             break;
         case SDL_WINDOWEVENT:
-            handle_windowevent(dcl, ev);
+            handle_windowevent(ev);
             break;
         default:
             break;
         }
     }
+
+    if (idle) {
+        if (scon->idle_counter < SDL2_MAX_IDLE_COUNT) {
+            scon->idle_counter++;
+            if (scon->idle_counter >= SDL2_MAX_IDLE_COUNT) {
+                scon->dcl.update_interval = GUI_REFRESH_INTERVAL_DEFAULT;
+            }
+        }
+    } else {
+        scon->idle_counter = 0;
+        scon->dcl.update_interval = SDL2_REFRESH_INTERVAL_BUSY;
+    }
 }
 
 static void sdl_mouse_warp(DisplayChangeListener *dcl,
                            int x, int y, int on)
 {
-    struct sdl2_state *scon = container_of(dcl, struct sdl2_state, dcl);
+    struct sdl2_console *scon = container_of(dcl, struct sdl2_console, dcl);
     if (on) {
         if (!guest_cursor) {
             sdl_show_cursor();
@@ -867,15 +709,52 @@
     SDL_QuitSubSystem(SDL_INIT_VIDEO);
 }
 
-static const DisplayChangeListenerOps dcl_ops = {
-    .dpy_name          = "sdl",
-    .dpy_gfx_update    = sdl_update,
-    .dpy_gfx_switch    = sdl_switch,
-    .dpy_refresh       = sdl_refresh,
-    .dpy_mouse_set     = sdl_mouse_warp,
-    .dpy_cursor_define = sdl_mouse_define,
+static const DisplayChangeListenerOps dcl_2d_ops = {
+    .dpy_name             = "sdl2-2d",
+    .dpy_gfx_update       = sdl2_2d_update,
+    .dpy_gfx_switch       = sdl2_2d_switch,
+    .dpy_gfx_check_format = sdl2_2d_check_format,
+    .dpy_refresh          = sdl2_2d_refresh,
+    .dpy_mouse_set        = sdl_mouse_warp,
+    .dpy_cursor_define    = sdl_mouse_define,
 };
 
+#ifdef CONFIG_OPENGL
+static const DisplayChangeListenerOps dcl_gl_ops = {
+    .dpy_name                = "sdl2-gl",
+    .dpy_gfx_update          = sdl2_gl_update,
+    .dpy_gfx_switch          = sdl2_gl_switch,
+    .dpy_gfx_check_format    = console_gl_check_format,
+    .dpy_refresh             = sdl2_gl_refresh,
+    .dpy_mouse_set           = sdl_mouse_warp,
+    .dpy_cursor_define       = sdl_mouse_define,
+
+    .dpy_gl_ctx_create       = sdl2_gl_create_context,
+    .dpy_gl_ctx_destroy      = sdl2_gl_destroy_context,
+    .dpy_gl_ctx_make_current = sdl2_gl_make_context_current,
+    .dpy_gl_ctx_get_current  = sdl2_gl_get_current_context,
+    .dpy_gl_scanout          = sdl2_gl_scanout,
+    .dpy_gl_update           = sdl2_gl_scanout_flush,
+};
+#endif
+
+void sdl_display_early_init(int opengl)
+{
+    switch (opengl) {
+    case -1: /* default */
+    case 0:  /* off */
+        break;
+    case 1: /* on */
+#ifdef CONFIG_OPENGL
+        display_opengl = 1;
+#endif
+        break;
+    default:
+        g_assert_not_reached();
+        break;
+    }
+}
+
 bool sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
 {
     int flags;
@@ -904,8 +783,9 @@
     if (SDL_Init(flags)) {
         fprintf(stderr, "Could not initialize SDL(%s) - exiting\n",
                 SDL_GetError());
-        exit(1);
+        return false;
     }
+    SDL_SetHint(SDL_HINT_GRAB_KEYBOARD, "1");
 
     for (i = 0;; i++) {
         QemuConsole *con = qemu_console_lookup_by_index(i);
@@ -913,18 +793,26 @@
             break;
         }
     }
-
     sdl2_num_outputs = i;
-    sdl2_console = g_new0(struct sdl2_state, sdl2_num_outputs);
+    if (sdl2_num_outputs == 0) {
+        return true;
+    }
+    sdl2_console = g_new0(struct sdl2_console, sdl2_num_outputs);
     for (i = 0; i < sdl2_num_outputs; i++) {
         QemuConsole *con = qemu_console_lookup_by_index(i);
         if (!qemu_console_is_graphic(con)) {
             sdl2_console[i].hidden = true;
         }
-        sdl2_console[i].dcl.ops = &dcl_ops;
+        sdl2_console[i].idx = i;
+#ifdef CONFIG_OPENGL
+        sdl2_console[i].opengl = display_opengl;
+        sdl2_console[i].dcl.ops = display_opengl ? &dcl_gl_ops : &dcl_2d_ops;
+#else
+        sdl2_console[i].opengl = 0;
+        sdl2_console[i].dcl.ops = &dcl_2d_ops;
+#endif
         sdl2_console[i].dcl.con = con;
         register_displaychangelistener(&sdl2_console[i].dcl);
-        sdl2_console[i].idx = i;
     }
 
     /* Load a 32x32x4 image. White pixels are transparent. */
@@ -953,7 +841,5 @@
     sdl_cursor_normal = SDL_GetCursor();
 
     atexit(sdl_cleanup);
-
     return true;
 }
-#endif
diff --git a/ui/sdl_zoom.c b/ui/sdl_zoom.c
index 2625c45..b96196b 100644
--- a/ui/sdl_zoom.c
+++ b/ui/sdl_zoom.c
@@ -11,11 +11,8 @@
  *
  */
 
-#include "sdl_zoom.h"
 #include "qemu/osdep.h"
-#include <glib.h>
-#include <stdint.h>
-#include <stdio.h>
+#include "sdl_zoom.h"
 
 static void sdl_zoom_rgb16(SDL_Surface *src, SDL_Surface *dst, int smooth,
                            SDL_Rect *dst_rect);
diff --git a/ui/sdl_zoom.h b/ui/sdl_zoom.h
index 74955bc..39696dd 100644
--- a/ui/sdl_zoom.h
+++ b/ui/sdl_zoom.h
@@ -11,8 +11,8 @@
  *
  */
 
-#ifndef SDL_zoom_h
-#define SDL_zoom_h
+#ifndef SDL_ZOOM_H
+#define SDL_ZOOM_H
 
 #include <SDL.h>
 
@@ -22,4 +22,4 @@
 int sdl_zoom_blit(SDL_Surface *src_sfc, SDL_Surface *dst_sfc,
                   int smooth, SDL_Rect *src_rect);
 
-#endif /* SDL_zoom_h */
+#endif /* SDL_ZOOM_H */
diff --git a/ui/shader.c b/ui/shader.c
new file mode 100644
index 0000000..1ffddbe
--- /dev/null
+++ b/ui/shader.c
@@ -0,0 +1,136 @@
+/*
+ * QEMU opengl shader helper functions
+ *
+ * Copyright (c) 2014 Red Hat
+ *
+ * Authors:
+ *    Gerd Hoffmann <kraxel@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "ui/shader.h"
+
+/* ---------------------------------------------------------------------- */
+
+GLuint qemu_gl_init_texture_blit(GLint texture_blit_prog)
+{
+    static const GLfloat in_position[] = {
+        -1, -1,
+        1,  -1,
+        -1,  1,
+        1,   1,
+    };
+    GLint l_position;
+    GLuint vao, buffer;
+
+    glGenVertexArrays(1, &vao);
+    glBindVertexArray(vao);
+
+    /* this is the VBO that holds the vertex data */
+    glGenBuffers(1, &buffer);
+    glBindBuffer(GL_ARRAY_BUFFER, buffer);
+    glBufferData(GL_ARRAY_BUFFER, sizeof(in_position), in_position,
+                 GL_STATIC_DRAW);
+
+    l_position = glGetAttribLocation(texture_blit_prog, "in_position");
+    glVertexAttribPointer(l_position, 2, GL_FLOAT, GL_FALSE, 0, 0);
+    glEnableVertexAttribArray(l_position);
+
+    glBindBuffer(GL_ARRAY_BUFFER, 0);
+    glBindVertexArray(0);
+
+    return vao;
+}
+
+void qemu_gl_run_texture_blit(GLint texture_blit_prog,
+                              GLint texture_blit_vao)
+{
+    glUseProgram(texture_blit_prog);
+    glBindVertexArray(texture_blit_vao);
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+}
+
+/* ---------------------------------------------------------------------- */
+
+GLuint qemu_gl_create_compile_shader(GLenum type, const GLchar *src)
+{
+    GLuint shader;
+    GLint status, length;
+    char *errmsg;
+
+    shader = glCreateShader(type);
+    glShaderSource(shader, 1, &src, 0);
+    glCompileShader(shader);
+
+    glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
+    if (!status) {
+        glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &length);
+        errmsg = g_malloc(length);
+        glGetShaderInfoLog(shader, length, &length, errmsg);
+        fprintf(stderr, "%s: compile %s error\n%s\n", __func__,
+                (type == GL_VERTEX_SHADER) ? "vertex" : "fragment",
+                errmsg);
+        g_free(errmsg);
+        return 0;
+    }
+    return shader;
+}
+
+GLuint qemu_gl_create_link_program(GLuint vert, GLuint frag)
+{
+    GLuint program;
+    GLint status, length;
+    char *errmsg;
+
+    program = glCreateProgram();
+    glAttachShader(program, vert);
+    glAttachShader(program, frag);
+    glLinkProgram(program);
+
+    glGetProgramiv(program, GL_LINK_STATUS, &status);
+    if (!status) {
+        glGetProgramiv(program, GL_INFO_LOG_LENGTH, &length);
+        errmsg = g_malloc(length);
+        glGetProgramInfoLog(program, length, &length, errmsg);
+        fprintf(stderr, "%s: link program: %s\n", __func__, errmsg);
+        g_free(errmsg);
+        return 0;
+    }
+    return program;
+}
+
+GLuint qemu_gl_create_compile_link_program(const GLchar *vert_src,
+                                           const GLchar *frag_src)
+{
+    GLuint vert_shader, frag_shader, program;
+
+    vert_shader = qemu_gl_create_compile_shader(GL_VERTEX_SHADER, vert_src);
+    frag_shader = qemu_gl_create_compile_shader(GL_FRAGMENT_SHADER, frag_src);
+    if (!vert_shader || !frag_shader) {
+        return 0;
+    }
+
+    program = qemu_gl_create_link_program(vert_shader, frag_shader);
+    glDeleteShader(vert_shader);
+    glDeleteShader(frag_shader);
+
+    return program;
+}
diff --git a/ui/shader/texture-blit.frag b/ui/shader/texture-blit.frag
new file mode 100644
index 0000000..bfa202c
--- /dev/null
+++ b/ui/shader/texture-blit.frag
@@ -0,0 +1,10 @@
+
+#version 300 es
+
+uniform sampler2D image;
+in  mediump vec2 ex_tex_coord;
+out mediump vec4 out_frag_color;
+
+void main(void) {
+     out_frag_color = texture(image, ex_tex_coord);
+}
diff --git a/ui/shader/texture-blit.vert b/ui/shader/texture-blit.vert
new file mode 100644
index 0000000..6fe2744
--- /dev/null
+++ b/ui/shader/texture-blit.vert
@@ -0,0 +1,10 @@
+
+#version 300 es
+
+in vec2  in_position;
+out vec2 ex_tex_coord;
+
+void main(void) {
+    gl_Position = vec4(in_position, 0.0, 1.0);
+    ex_tex_coord = vec2(1.0 + in_position.x, 1.0 - in_position.y) * 0.5;
+}
diff --git a/ui/spice-core.c b/ui/spice-core.c
index 6467fa4..da05054 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -15,14 +15,15 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include <spice.h>
-#include <spice-experimental.h>
 
 #include <netdb.h>
 #include "sysemu/sysemu.h"
 
 #include "qemu-common.h"
 #include "ui/qemu-spice.h"
+#include "qemu/error-report.h"
 #include "qemu/thread.h"
 #include "qemu/timer.h"
 #include "qemu/queue.h"
@@ -200,8 +201,6 @@
 {
     SpiceServerInfo *server = g_malloc0(sizeof(*server));
     SpiceChannel *client = g_malloc0(sizeof(*client));
-    server->base = g_malloc0(sizeof(*server->base));
-    client->base = g_malloc0(sizeof(*client->base));
 
     /*
      * Spice server might have called us from spice worker thread
@@ -218,9 +217,11 @@
     }
 
     if (info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
-        add_addr_info(client->base, (struct sockaddr *)&info->paddr_ext,
+        add_addr_info(qapi_SpiceChannel_base(client),
+                      (struct sockaddr *)&info->paddr_ext,
                       info->plen_ext);
-        add_addr_info(server->base, (struct sockaddr *)&info->laddr_ext,
+        add_addr_info(qapi_SpiceServerInfo_base(server),
+                      (struct sockaddr *)&info->laddr_ext,
                       info->llen_ext);
     } else {
         error_report("spice: %s, extended address is expected",
@@ -229,7 +230,9 @@
 
     switch (event) {
     case SPICE_CHANNEL_EVENT_CONNECTED:
-        qapi_event_send_spice_connected(server->base, client->base, &error_abort);
+        qapi_event_send_spice_connected(qapi_SpiceServerInfo_base(server),
+                                        qapi_SpiceChannel_base(client),
+                                        &error_abort);
         break;
     case SPICE_CHANNEL_EVENT_INITIALIZED:
         if (auth) {
@@ -242,7 +245,9 @@
         break;
     case SPICE_CHANNEL_EVENT_DISCONNECTED:
         channel_list_del(info);
-        qapi_event_send_spice_disconnected(server->base, client->base, &error_abort);
+        qapi_event_send_spice_disconnected(qapi_SpiceServerInfo_base(server),
+                                           qapi_SpiceChannel_base(client),
+                                           &error_abort);
         break;
     default:
         break;
@@ -274,14 +279,6 @@
     .channel_event      = channel_event,
 };
 
-typedef struct SpiceMigration {
-    SpiceMigrateInstance sin;
-    struct {
-        MonitorCompletion *cb;
-        void *opaque;
-    } connect_complete;
-} SpiceMigration;
-
 static void migrate_connect_complete_cb(SpiceMigrateInstance *sin);
 static void migrate_end_complete_cb(SpiceMigrateInstance *sin);
 
@@ -294,15 +291,11 @@
     .migrate_end_complete = migrate_end_complete_cb,
 };
 
-static SpiceMigration spice_migrate;
+static SpiceMigrateInstance spice_migrate;
 
 static void migrate_connect_complete_cb(SpiceMigrateInstance *sin)
 {
-    SpiceMigration *sm = container_of(sin, SpiceMigration, sin);
-    if (sm->connect_complete.cb) {
-        sm->connect_complete.cb(sm->connect_complete.opaque, NULL);
-    }
-    sm->connect_complete.cb = NULL;
+    /* nothing, but libspice-server expects this cb being present. */
 }
 
 static void migrate_end_complete_cb(SpiceMigrateInstance *sin)
@@ -386,23 +379,19 @@
         struct sockaddr *paddr;
         socklen_t plen;
 
-        if (!(item->info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT)) {
-            error_report("invalid channel event");
-            return NULL;
-        }
+        assert(item->info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT);
 
         chan = g_malloc0(sizeof(*chan));
         chan->value = g_malloc0(sizeof(*chan->value));
-        chan->value->base = g_malloc0(sizeof(*chan->value->base));
 
         paddr = (struct sockaddr *)&item->info->paddr_ext;
         plen = item->info->plen_ext;
         getnameinfo(paddr, plen,
                     host, sizeof(host), port, sizeof(port),
                     NI_NUMERICHOST | NI_NUMERICSERV);
-        chan->value->base->host = g_strdup(host);
-        chan->value->base->port = g_strdup(port);
-        chan->value->base->family = inet_netfamily(paddr->sa_family);
+        chan->value->host = g_strdup(host);
+        chan->value->port = g_strdup(port);
+        chan->value->family = inet_netfamily(paddr->sa_family);
 
         chan->value->connection_id = item->info->connection_id;
         chan->value->channel_type = item->info->type;
@@ -440,6 +429,11 @@
         },{
             .name = "ipv6",
             .type = QEMU_OPT_BOOL,
+#ifdef SPICE_ADDR_FLAG_UNIX_ONLY
+        },{
+            .name = "unix",
+            .type = QEMU_OPT_BOOL,
+#endif
         },{
             .name = "password",
             .type = QEMU_OPT_STRING,
@@ -500,9 +494,14 @@
         },{
             .name = "playback-compression",
             .type = QEMU_OPT_BOOL,
-        }, {
+        },{
             .name = "seamless-migration",
             .type = QEMU_OPT_BOOL,
+#ifdef HAVE_SPICE_GL
+        },{
+            .name = "gl",
+            .type = QEMU_OPT_BOOL,
+#endif
         },
         { /* end of list */ }
     },
@@ -574,7 +573,8 @@
 
     if (migration_in_setup(s)) {
         spice_server_migrate_start(spice_server);
-    } else if (migration_has_finished(s)) {
+    } else if (migration_has_finished(s) ||
+               migration_in_postcopy_after_devices(s)) {
         spice_server_migrate_end(spice_server, true);
         spice_have_target_host = false;
     } else if (migration_has_failed(s)) {
@@ -584,20 +584,18 @@
 }
 
 int qemu_spice_migrate_info(const char *hostname, int port, int tls_port,
-                            const char *subject,
-                            MonitorCompletion *cb, void *opaque)
+                            const char *subject)
 {
     int ret;
 
-    spice_migrate.connect_complete.cb = cb;
-    spice_migrate.connect_complete.opaque = opaque;
     ret = spice_server_migrate_connect(spice_server, hostname,
                                        port, tls_port, subject);
     spice_have_target_host = true;
     return ret;
 }
 
-static int add_channel(const char *name, const char *value, void *opaque)
+static int add_channel(void *opaque, const char *name, const char *value,
+                       Error **errp)
 {
     int security = 0;
     int rc;
@@ -661,10 +659,6 @@
     }
     port = qemu_opt_get_number(opts, "port", 0);
     tls_port = qemu_opt_get_number(opts, "tls-port", 0);
-    if (!port && !tls_port) {
-        error_report("neither port nor tls-port specified for spice");
-        exit(1);
-    }
     if (port < 0 || port > 65535) {
         error_report("spice port is out of range");
         exit(1);
@@ -716,6 +710,10 @@
         addr_flags |= SPICE_ADDR_FLAG_IPV4_ONLY;
     } else if (qemu_opt_get_bool(opts, "ipv6", 0)) {
         addr_flags |= SPICE_ADDR_FLAG_IPV6_ONLY;
+#ifdef SPICE_ADDR_FLAG_UNIX_ONLY
+    } else if (qemu_opt_get_bool(opts, "unix", 0)) {
+        addr_flags |= SPICE_ADDR_FLAG_UNIX_ONLY;
+#endif
     }
 
     spice_server = spice_server_new();
@@ -736,8 +734,7 @@
         qemu_spice_set_passwd(password, false, false);
     }
     if (qemu_opt_get_bool(opts, "sasl", 0)) {
-        if (spice_server_set_sasl_appname(spice_server, "qemu") == -1 ||
-            spice_server_set_sasl(spice_server, 1) == -1) {
+        if (spice_server_set_sasl(spice_server, 1) == -1) {
             error_report("spice: failed to enable sasl");
             exit(1);
         }
@@ -796,13 +793,14 @@
     spice_server_set_playback_compression
         (spice_server, qemu_opt_get_bool(opts, "playback-compression", 1));
 
-    qemu_opt_foreach(opts, add_channel, &tls_port, 0);
+    qemu_opt_foreach(opts, add_channel, &tls_port, NULL);
 
     spice_server_set_name(spice_server, qemu_name);
     spice_server_set_uuid(spice_server, qemu_uuid);
 
     seamless_migration = qemu_opt_get_bool(opts, "seamless-migration", 0);
     spice_server_set_seamless_migration(spice_server, seamless_migration);
+    spice_server_set_sasl_appname(spice_server, "qemu");
     if (spice_server_init(spice_server, &core_interface) != 0) {
         error_report("failed to initialize spice server");
         exit(1);
@@ -811,14 +809,14 @@
 
     migration_state.notify = migration_state_notifier;
     add_migration_state_change_notifier(&migration_state);
-    spice_migrate.sin.base.sif = &migrate_interface.base;
-    spice_migrate.connect_complete.cb = NULL;
-    qemu_spice_add_interface(&spice_migrate.sin.base);
+    spice_migrate.base.sif = &migrate_interface.base;
+    qemu_spice_add_interface(&spice_migrate.base);
 
     qemu_spice_input_init();
     qemu_spice_audio_init();
 
     qemu_add_vm_change_state_handler(vm_change_state_handler, NULL);
+    qemu_spice_display_stop();
 
     g_free(x509_key_file);
     g_free(x509_cert_file);
@@ -827,6 +825,21 @@
 #if SPICE_SERVER_VERSION >= 0x000c02
     qemu_spice_register_ports();
 #endif
+
+#ifdef HAVE_SPICE_GL
+    if (qemu_opt_get_bool(opts, "gl", 0)) {
+        if ((port != 0) || (tls_port != 0)) {
+            error_report("SPICE GL support is local-only for now and "
+                         "incompatible with -spice port/tls-port");
+            exit(1);
+        }
+        if (egl_rendernode_init() != 0) {
+            error_report("Failed to initialize EGL render node for SPICE GL");
+            exit(1);
+        }
+        display_opengl = 1;
+    }
+#endif
 }
 
 int qemu_spice_add_interface(SpiceBaseInstance *sin)
@@ -939,4 +952,4 @@
 {
     qemu_add_opts(&qemu_spice_opts);
 }
-machine_init(spice_register_config);
+opts_init(spice_register_config);
diff --git a/ui/spice-display.c b/ui/spice-display.c
index def7b52..99132b6 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -15,11 +15,11 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "ui/qemu-spice.h"
 #include "qemu/timer.h"
 #include "qemu/queue.h"
-#include "monitor/monitor.h"
 #include "ui/console.h"
 #include "sysemu/sysemu.h"
 #include "trace.h"
@@ -178,7 +178,7 @@
     image->bitmap.palette = 0;
     image->bitmap.format = SPICE_BITMAP_FMT_32BIT;
 
-    dest = pixman_image_create_bits(PIXMAN_x8r8g8b8, bw, bh,
+    dest = pixman_image_create_bits(PIXMAN_LE_x8r8g8b8, bw, bh,
                                     (void *)update->bitmap, bw * 4);
     pixman_image_composite(PIXMAN_OP_SRC, ssd->surface, NULL, ssd->mirror,
                            rect->left, rect->top, 0, 0,
@@ -197,9 +197,9 @@
 static void qemu_spice_create_update(SimpleSpiceDisplay *ssd)
 {
     static const int blksize = 32;
-    int blocks = (surface_width(ssd->ds) + blksize - 1) / blksize;
+    int blocks = DIV_ROUND_UP(surface_width(ssd->ds), blksize);
     int dirty_top[blocks];
-    int y, yoff, x, xoff, blk, bw;
+    int y, yoff1, yoff2, x, xoff, blk, bw;
     int bpp = surface_bytes_per_pixel(ssd->ds);
     uint8_t *guest, *mirror;
 
@@ -207,12 +207,6 @@
         return;
     };
 
-    if (ssd->surface == NULL) {
-        ssd->surface = pixman_image_ref(ssd->ds->image);
-        ssd->mirror  = qemu_pixman_mirror_create(ssd->ds->format,
-                                                 ssd->ds->image);
-    }
-
     for (blk = 0; blk < blocks; blk++) {
         dirty_top[blk] = -1;
     }
@@ -220,13 +214,14 @@
     guest = surface_data(ssd->ds);
     mirror = (void *)pixman_image_get_data(ssd->mirror);
     for (y = ssd->dirty.top; y < ssd->dirty.bottom; y++) {
-        yoff = y * surface_stride(ssd->ds);
+        yoff1 = y * surface_stride(ssd->ds);
+        yoff2 = y * pixman_image_get_stride(ssd->mirror);
         for (x = ssd->dirty.left; x < ssd->dirty.right; x += blksize) {
             xoff = x * bpp;
             blk = x / blksize;
             bw = MIN(blksize, ssd->dirty.right - x);
-            if (memcmp(guest + yoff + xoff,
-                       mirror + yoff + xoff,
+            if (memcmp(guest + yoff1 + xoff,
+                       mirror + yoff2 + xoff,
                        bw * bpp) == 0) {
                 if (dirty_top[blk] != -1) {
                     QXLRect update = {
@@ -266,7 +261,8 @@
 
 static SimpleSpiceCursor*
 qemu_spice_create_cursor_update(SimpleSpiceDisplay *ssd,
-                                QEMUCursor *c)
+                                QEMUCursor *c,
+                                int on)
 {
     size_t size = c ? c->width * c->height * 4 : 0;
     SimpleSpiceCursor *update;
@@ -281,8 +277,8 @@
 
     if (c) {
         ccmd->type = QXL_CURSOR_SET;
-        ccmd->u.set.position.x = ssd->ptr_x;
-        ccmd->u.set.position.y = ssd->ptr_y;
+        ccmd->u.set.position.x = ssd->ptr_x + ssd->hot_x;
+        ccmd->u.set.position.y = ssd->ptr_y + ssd->hot_y;
         ccmd->u.set.visible    = true;
         ccmd->u.set.shape      = (uintptr_t)cursor;
         cursor->header.unique     = ssd->unique++;
@@ -294,10 +290,12 @@
         cursor->data_size         = size;
         cursor->chunk.data_size   = size;
         memcpy(cursor->chunk.data, c->data, size);
+    } else if (!on) {
+        ccmd->type = QXL_CURSOR_HIDE;
     } else {
         ccmd->type = QXL_CURSOR_MOVE;
-        ccmd->u.position.x = ssd->ptr_x;
-        ccmd->u.position.y = ssd->ptr_y;
+        ccmd->u.position.x = ssd->ptr_x + ssd->hot_x;
+        ccmd->u.position.y = ssd->ptr_y + ssd->hot_y;
     }
     ccmd->release_info.id = (uintptr_t)(&update->ext);
 
@@ -409,7 +407,30 @@
     SimpleSpiceUpdate *update;
     bool need_destroy;
 
-    dprint(1, "%s/%d:\n", __func__, ssd->qxl.id);
+    if (surface && ssd->surface &&
+        surface_width(surface) == pixman_image_get_width(ssd->surface) &&
+        surface_height(surface) == pixman_image_get_height(ssd->surface) &&
+        surface_format(surface) == pixman_image_get_format(ssd->surface)) {
+        /* no-resize fast path: just swap backing store */
+        dprint(1, "%s/%d: fast (%dx%d)\n", __func__, ssd->qxl.id,
+               surface_width(surface), surface_height(surface));
+        qemu_mutex_lock(&ssd->lock);
+        ssd->ds = surface;
+        pixman_image_unref(ssd->surface);
+        ssd->surface = pixman_image_ref(ssd->ds->image);
+        qemu_mutex_unlock(&ssd->lock);
+        qemu_spice_display_update(ssd, 0, 0,
+                                  surface_width(surface),
+                                  surface_height(surface));
+        return;
+    }
+
+    /* full mode switch */
+    dprint(1, "%s/%d: full (%dx%d -> %dx%d)\n", __func__, ssd->qxl.id,
+           ssd->surface ? pixman_image_get_width(ssd->surface)  : 0,
+           ssd->surface ? pixman_image_get_height(ssd->surface) : 0,
+           surface ? surface_width(surface)  : 0,
+           surface ? surface_height(surface) : 0);
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
     if (ssd->surface) {
@@ -431,20 +452,28 @@
         qemu_spice_destroy_host_primary(ssd);
     }
     if (ssd->ds) {
+        ssd->surface = pixman_image_ref(ssd->ds->image);
+        ssd->mirror  = qemu_pixman_mirror_create(ssd->ds->format,
+                                                 ssd->ds->image);
         qemu_spice_create_host_primary(ssd);
     }
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
     ssd->notify++;
+
+    qemu_mutex_lock(&ssd->lock);
+    if (ssd->cursor) {
+        g_free(ssd->ptr_define);
+        ssd->ptr_define = qemu_spice_create_cursor_update(ssd, ssd->cursor, 0);
+    }
+    qemu_mutex_unlock(&ssd->lock);
 }
 
-void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
+static void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
 {
     if (ssd->cursor) {
         assert(ssd->dcl.con);
         dpy_cursor_define(ssd->dcl.con, ssd->cursor);
-        cursor_put(ssd->cursor);
-        ssd->cursor = NULL;
     }
     if (ssd->mouse_x != -1 && ssd->mouse_y != -1) {
         assert(ssd->dcl.con);
@@ -454,6 +483,15 @@
     }
 }
 
+void qemu_spice_cursor_refresh_bh(void *opaque)
+{
+    SimpleSpiceDisplay *ssd = opaque;
+
+    qemu_mutex_lock(&ssd->lock);
+    qemu_spice_cursor_refresh_unlocked(ssd);
+    qemu_mutex_unlock(&ssd->lock);
+}
+
 void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd)
 {
     dprint(3, "%s/%d:\n", __func__, ssd->qxl.id);
@@ -464,7 +502,6 @@
         qemu_spice_create_update(ssd);
         ssd->notify++;
     }
-    qemu_spice_cursor_refresh_unlocked(ssd);
     qemu_mutex_unlock(&ssd->lock);
 
     if (ssd->notify) {
@@ -490,11 +527,13 @@
     /* nothing to do */
 }
 
+#if SPICE_NEEDS_SET_MM_TIME
 static void interface_set_mm_time(QXLInstance *sin, uint32_t mm_time)
 {
     dprint(3, "%s/%d:\n", __func__, sin->id);
     /* nothing to do */
 }
+#endif
 
 static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
 {
@@ -509,7 +548,7 @@
     info->n_surfaces = ssd->num_surfaces;
 }
 
-static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
+static int interface_get_command(QXLInstance *sin, QXLCommandExt *ext)
 {
     SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
     SimpleSpiceUpdate *update;
@@ -531,12 +570,12 @@
 
 static int interface_req_cmd_notification(QXLInstance *sin)
 {
-    dprint(1, "%s/%d:\n", __func__, sin->id);
+    dprint(2, "%s/%d:\n", __func__, sin->id);
     return 1;
 }
 
 static void interface_release_resource(QXLInstance *sin,
-                                       struct QXLReleaseInfoExt rext)
+                                       QXLReleaseInfoExt rext)
 {
     SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
     SimpleSpiceUpdate *update;
@@ -559,7 +598,7 @@
     }
 }
 
-static int interface_get_cursor_command(QXLInstance *sin, struct QXLCommandExt *ext)
+static int interface_get_cursor_command(QXLInstance *sin, QXLCommandExt *ext)
 {
     SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
     int ret;
@@ -584,7 +623,7 @@
 
 static int interface_req_cursor_notification(QXLInstance *sin)
 {
-    dprint(1, "%s:\n", __FUNCTION__);
+    dprint(2, "%s:\n", __func__);
     return 1;
 }
 
@@ -613,9 +652,28 @@
 /* called from spice server thread context only */
 static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
 {
-    /* should never be called, used in qxl native mode only */
-    fprintf(stderr, "%s: abort()\n", __func__);
-    abort();
+    QXLCookie *cookie = (QXLCookie *)(uintptr_t)cookie_token;
+
+    switch (cookie->type) {
+#ifdef HAVE_SPICE_GL
+    case QXL_COOKIE_TYPE_GL_DRAW_DONE:
+    {
+        SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
+        qemu_bh_schedule(ssd->gl_unblock_bh);
+        break;
+    }
+    case QXL_COOKIE_TYPE_IO:
+        if (cookie->io == QXL_IO_MONITORS_CONFIG_ASYNC) {
+            g_free(cookie->u.data);
+        }
+        break;
+#endif
+    default:
+        /* should never be called, used in qxl native mode only */
+        fprintf(stderr, "%s: abort()\n", __func__);
+        abort();
+    }
+    g_free(cookie);
 }
 
 static void interface_set_client_capabilities(QXLInstance *sin,
@@ -630,29 +688,27 @@
 {
     SimpleSpiceDisplay *ssd = container_of(sin, SimpleSpiceDisplay, qxl);
     QemuUIInfo info;
-    int rc;
+    int head;
+
+    if (!dpy_ui_info_supported(ssd->dcl.con)) {
+        return 0; /* == not supported by guest */
+    }
 
     if (!mc) {
         return 1;
     }
 
-    /*
-     * FIXME: multihead is tricky due to the way
-     * spice has multihead implemented.
-     */
     memset(&info, 0, sizeof(info));
-    if (mc->num_of_monitors > 0) {
-        info.width  = mc->monitors[0].width;
-        info.height = mc->monitors[0].height;
+
+    head = qemu_console_get_head(ssd->dcl.con);
+    if (mc->num_of_monitors > head) {
+        info.width  = mc->monitors[head].width;
+        info.height = mc->monitors[head].height;
     }
-    rc = dpy_set_ui_info(ssd->dcl.con, &info);
-    dprint(1, "%s/%d: size %dx%d, rc %d   <---   ==========================\n",
-           __func__, ssd->qxl.id, info.width, info.height, rc);
-    if (rc != 0) {
-        return 0; /* == not supported by guest */
-    } else {
-        return 1;
-    }
+    dpy_set_ui_info(ssd->dcl.con, &info);
+    dprint(1, "%s/%d: size %dx%d\n", __func__, ssd->qxl.id,
+           info.width, info.height);
+    return 1;
 }
 
 static const QXLInterface dpy_interface = {
@@ -663,7 +719,9 @@
 
     .attache_worker          = interface_attach_worker,
     .set_compression_level   = interface_set_compression_level,
+#if SPICE_NEEDS_SET_MM_TIME
     .set_mm_time             = interface_set_mm_time,
+#endif
     .get_init_info           = interface_get_init_info,
 
     /* the callbacks below are called from spice server thread context */
@@ -688,7 +746,7 @@
 }
 
 static void display_switch(DisplayChangeListener *dcl,
-                           struct DisplaySurface *surface)
+                           DisplaySurface *surface)
 {
     SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
     qemu_spice_display_switch(ssd, surface);
@@ -707,11 +765,9 @@
 
     qemu_mutex_lock(&ssd->lock);
     ssd->ptr_x = x;
-    ssd->ptr_y = x;
-    if (ssd->ptr_move) {
-        g_free(ssd->ptr_move);
-    }
-    ssd->ptr_move = qemu_spice_create_cursor_update(ssd, NULL);
+    ssd->ptr_y = y;
+    g_free(ssd->ptr_move);
+    ssd->ptr_move = qemu_spice_create_cursor_update(ssd, NULL, on);
     qemu_mutex_unlock(&ssd->lock);
 }
 
@@ -721,40 +777,175 @@
     SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->ptr_move) {
-        g_free(ssd->ptr_move);
-        ssd->ptr_move = NULL;
-    }
-    if (ssd->ptr_define) {
-        g_free(ssd->ptr_define);
-    }
-    ssd->ptr_define = qemu_spice_create_cursor_update(ssd, c);
+    cursor_get(c);
+    cursor_put(ssd->cursor);
+    ssd->cursor = c;
+    ssd->hot_x = c->hot_x;
+    ssd->hot_y = c->hot_y;
+    g_free(ssd->ptr_move);
+    ssd->ptr_move = NULL;
+    g_free(ssd->ptr_define);
+    ssd->ptr_define = qemu_spice_create_cursor_update(ssd, c, 0);
     qemu_mutex_unlock(&ssd->lock);
 }
 
 static const DisplayChangeListenerOps display_listener_ops = {
-    .dpy_name          = "spice",
-    .dpy_gfx_update    = display_update,
-    .dpy_gfx_switch    = display_switch,
-    .dpy_refresh       = display_refresh,
-    .dpy_mouse_set     = display_mouse_set,
-    .dpy_cursor_define = display_mouse_define,
+    .dpy_name             = "spice",
+    .dpy_gfx_update       = display_update,
+    .dpy_gfx_switch       = display_switch,
+    .dpy_gfx_check_format = qemu_pixman_check_format,
+    .dpy_refresh          = display_refresh,
+    .dpy_mouse_set        = display_mouse_set,
+    .dpy_cursor_define    = display_mouse_define,
 };
 
+#ifdef HAVE_SPICE_GL
+
+static void qemu_spice_gl_monitor_config(SimpleSpiceDisplay *ssd,
+                                         int x, int y, int w, int h)
+{
+    QXLMonitorsConfig *config;
+    QXLCookie *cookie;
+
+    config = g_malloc0(sizeof(QXLMonitorsConfig) + sizeof(QXLHead));
+    config->count = 1;
+    config->max_allowed = 1;
+    config->heads[0].x = x;
+    config->heads[0].y = y;
+    config->heads[0].width = w;
+    config->heads[0].height = h;
+    cookie = qxl_cookie_new(QXL_COOKIE_TYPE_IO,
+                            QXL_IO_MONITORS_CONFIG_ASYNC);
+    cookie->u.data = config;
+
+    spice_qxl_monitors_config_async(&ssd->qxl,
+                                    (uintptr_t)config,
+                                    MEMSLOT_GROUP_HOST,
+                                    (uintptr_t)cookie);
+}
+
+static void qemu_spice_gl_block(SimpleSpiceDisplay *ssd, bool block)
+{
+    uint64_t timeout;
+
+    if (block) {
+        timeout = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+        timeout += 1000; /* one sec */
+        timer_mod(ssd->gl_unblock_timer, timeout);
+    } else {
+        timer_del(ssd->gl_unblock_timer);
+    }
+    graphic_hw_gl_block(ssd->dcl.con, block);
+}
+
+static void qemu_spice_gl_unblock_bh(void *opaque)
+{
+    SimpleSpiceDisplay *ssd = opaque;
+
+    qemu_spice_gl_block(ssd, false);
+}
+
+static void qemu_spice_gl_block_timer(void *opaque)
+{
+    fprintf(stderr, "WARNING: spice: no gl-draw-done within one second\n");
+}
+
+static QEMUGLContext qemu_spice_gl_create_context(DisplayChangeListener *dcl,
+                                                  QEMUGLParams *params)
+{
+    eglMakeCurrent(qemu_egl_display, EGL_NO_SURFACE, EGL_NO_SURFACE,
+                   qemu_egl_rn_ctx);
+    return qemu_egl_create_context(dcl, params);
+}
+
+static void qemu_spice_gl_scanout(DisplayChangeListener *dcl,
+                                  uint32_t tex_id,
+                                  bool y_0_top,
+                                  uint32_t backing_width,
+                                  uint32_t backing_height,
+                                  uint32_t x, uint32_t y,
+                                  uint32_t w, uint32_t h)
+{
+    SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
+    EGLint stride = 0, fourcc = 0;
+    int fd = -1;
+
+    if (tex_id) {
+        fd = egl_get_fd_for_texture(tex_id, &stride, &fourcc);
+        if (fd < 0) {
+            fprintf(stderr, "%s: failed to get fd for texture\n", __func__);
+            return;
+        }
+        dprint(1, "%s: %dx%d (stride %d, fourcc 0x%x)\n", __func__,
+               w, h, stride, fourcc);
+    } else {
+        dprint(1, "%s: no texture (no framebuffer)\n", __func__);
+    }
+
+    assert(!tex_id || fd >= 0);
+
+    /* note: spice server will close the fd */
+    spice_qxl_gl_scanout(&ssd->qxl, fd, backing_width, backing_height,
+                         stride, fourcc, y_0_top);
+
+    qemu_spice_gl_monitor_config(ssd, x, y, w, h);
+}
+
+static void qemu_spice_gl_update(DisplayChangeListener *dcl,
+                                 uint32_t x, uint32_t y, uint32_t w, uint32_t h)
+{
+    SimpleSpiceDisplay *ssd = container_of(dcl, SimpleSpiceDisplay, dcl);
+    uint64_t cookie;
+
+    dprint(2, "%s: %dx%d+%d+%d\n", __func__, w, h, x, y);
+    qemu_spice_gl_block(ssd, true);
+    cookie = (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_GL_DRAW_DONE, 0);
+    spice_qxl_gl_draw_async(&ssd->qxl, x, y, w, h, cookie);
+}
+
+static const DisplayChangeListenerOps display_listener_gl_ops = {
+    .dpy_name             = "spice-egl",
+    .dpy_gfx_update       = display_update,
+    .dpy_gfx_switch       = display_switch,
+    .dpy_gfx_check_format = qemu_pixman_check_format,
+    .dpy_refresh          = display_refresh,
+    .dpy_mouse_set        = display_mouse_set,
+    .dpy_cursor_define    = display_mouse_define,
+
+    .dpy_gl_ctx_create       = qemu_spice_gl_create_context,
+    .dpy_gl_ctx_destroy      = qemu_egl_destroy_context,
+    .dpy_gl_ctx_make_current = qemu_egl_make_context_current,
+    .dpy_gl_ctx_get_current  = qemu_egl_get_current_context,
+
+    .dpy_gl_scanout          = qemu_spice_gl_scanout,
+    .dpy_gl_update           = qemu_spice_gl_update,
+};
+
+#endif /* HAVE_SPICE_GL */
+
 static void qemu_spice_display_init_one(QemuConsole *con)
 {
     SimpleSpiceDisplay *ssd = g_new0(SimpleSpiceDisplay, 1);
 
     qemu_spice_display_init_common(ssd);
 
+    ssd->dcl.ops = &display_listener_ops;
+#ifdef HAVE_SPICE_GL
+    if (display_opengl) {
+        ssd->dcl.ops = &display_listener_gl_ops;
+        ssd->dmabuf_fd = -1;
+        ssd->gl_unblock_bh = qemu_bh_new(qemu_spice_gl_unblock_bh, ssd);
+        ssd->gl_unblock_timer = timer_new_ms(QEMU_CLOCK_REALTIME,
+                                             qemu_spice_gl_block_timer, ssd);
+    }
+#endif
+    ssd->dcl.con = con;
+
     ssd->qxl.base.sif = &dpy_interface.base;
     qemu_spice_add_display_interface(&ssd->qxl, con);
     assert(ssd->worker);
-
     qemu_spice_create_host_memslot(ssd);
 
-    ssd->dcl.ops = &display_listener_ops;
-    ssd->dcl.con = con;
     register_displaychangelistener(&ssd->dcl);
 }
 
diff --git a/ui/spice-input.c b/ui/spice-input.c
index c342e0d..8eeebdb 100644
--- a/ui/spice-input.c
+++ b/ui/spice-input.c
@@ -15,10 +15,7 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdbool.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
 #include <spice.h>
 #include <spice/enums.h>
@@ -107,7 +104,7 @@
 static void spice_update_buttons(QemuSpicePointer *pointer,
                                  int wheel, uint32_t button_mask)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]        = 0x01,
         [INPUT_BUTTON_MIDDLE]      = 0x04,
         [INPUT_BUTTON_RIGHT]       = 0x02,
diff --git a/ui/trace-events b/ui/trace-events
new file mode 100644
index 0000000..93fe548
--- /dev/null
+++ b/ui/trace-events
@@ -0,0 +1,48 @@
+# See docs/tracing.txt for syntax documentation.
+
+# ui/console.c
+console_gfx_new(void) ""
+console_putchar_csi(int esc_param0, int esc_param1, int ch, int nb_esc_params) "escape sequence CSI%d;%d%c, %d parameters"
+console_putchar_unhandled(int ch) "unhandled escape character '%c'"
+console_txt_new(int w, int h) "%dx%d"
+console_select(int nr) "%d"
+console_refresh(int interval) "interval %d ms"
+displaysurface_create(void *display_surface, int w, int h) "surface=%p, %dx%d"
+displaysurface_create_from(void *display_surface, int w, int h, uint32_t format) "surface=%p, %dx%d, format 0x%x"
+displaysurface_create_pixman(void *display_surface) "surface=%p"
+displaysurface_free(void *display_surface) "surface=%p"
+displaychangelistener_register(void *dcl, const char *name) "%p [ %s ]"
+displaychangelistener_unregister(void *dcl, const char *name) "%p [ %s ]"
+ppm_save(const char *filename, void *display_surface) "%s surface=%p"
+
+# ui/gtk.c
+gd_switch(const char *tab, int width, int height) "tab=%s, width=%d, height=%d"
+gd_update(const char *tab, int x, int y, int w, int h) "tab=%s, x=%d, y=%d, w=%d, h=%d"
+gd_key_event(const char *tab, int gdk_keycode, int qemu_keycode, const char *action) "tab=%s, translated GDK keycode %d to QEMU keycode %d (%s)"
+gd_grab(const char *tab, const char *device, const char *reason) "tab=%s, dev=%s, reason=%s"
+gd_ungrab(const char *tab, const char *device) "tab=%s, dev=%s"
+
+# ui/vnc.c
+vnc_key_guest_leds(bool caps, bool num, bool scroll) "caps %d, num %d, scroll %d"
+vnc_key_map_init(const char *layout) "%s"
+vnc_key_event_ext(bool down, int sym, int keycode, const char *name) "down %d, sym 0x%x, keycode 0x%x [%s]"
+vnc_key_event_map(bool down, int sym, int keycode, const char *name) "down %d, sym 0x%x -> keycode 0x%x [%s]"
+vnc_key_sync_numlock(bool on) "%d"
+vnc_key_sync_capslock(bool on) "%d"
+
+# ui/input.c
+input_event_key_number(int conidx, int number, const char *qcode, bool down) "con %d, key number 0x%x [%s], down %d"
+input_event_key_qcode(int conidx, const char *qcode, bool down) "con %d, key qcode %s, down %d"
+input_event_btn(int conidx, const char *btn, bool down) "con %d, button %s, down %d"
+input_event_rel(int conidx, const char *axis, int value) "con %d, axis %s, value %d"
+input_event_abs(int conidx, const char *axis, int value) "con %d, axis %s, value 0x%x"
+input_event_sync(void) ""
+input_mouse_mode(int absolute) "absolute %d"
+
+# ui/spice-display.c
+qemu_spice_add_memslot(int qid, uint32_t slot_id, unsigned long virt_start, unsigned long virt_end, int async) "%d %u: host virt 0x%lx - 0x%lx async=%d"
+qemu_spice_del_memslot(int qid, uint32_t gid, uint32_t slot_id) "%d gid=%u sid=%u"
+qemu_spice_create_primary_surface(int qid, uint32_t sid, void *surface, int async) "%d sid=%u surface=%p async=%d"
+qemu_spice_destroy_primary_surface(int qid, uint32_t sid, int async) "%d sid=%u async=%d"
+qemu_spice_wakeup(uint32_t qid) "%d"
+qemu_spice_create_update(uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "lr %d -> %d,  tb -> %d -> %d"
diff --git a/ui/vnc-auth-sasl.c b/ui/vnc-auth-sasl.c
index f3ad75d..5ae29c1 100644
--- a/ui/vnc-auth-sasl.c
+++ b/ui/vnc-auth-sasl.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "vnc.h"
 
 /* Max amount of data we send/recv for SASL steps to prevent DOS */
@@ -62,7 +64,7 @@
                           (const char **)&vs->sasl.encoded,
                           &vs->sasl.encodedLength);
         if (err != SASL_OK)
-            return vnc_client_io_error(vs, -1, EIO);
+            return vnc_client_io_error(vs, -1, NULL);
 
         vs->sasl.encodedOffset = 0;
     }
@@ -86,7 +88,11 @@
      * SASL encoded output
      */
     if (vs->output.offset == 0) {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 
     return ret;
@@ -110,7 +116,7 @@
                       &decoded, &decodedLen);
 
     if (err != SASL_OK)
-        return vnc_client_io_error(vs, -1, -EIO);
+        return vnc_client_io_error(vs, -1, NULL);
     VNC_DEBUG("Read SASL Encoded %p size %ld Decoded %p size %d\n",
               encoded, ret, decoded, decodedLen);
     buffer_reserve(&vs->input, decodedLen);
@@ -255,17 +261,17 @@
         vnc_read_when(vs, protocol_client_auth_sasl_step_len, 4);
     } else {
         if (!vnc_auth_sasl_check_ssf(vs)) {
-            VNC_DEBUG("Authentication rejected for weak SSF %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for weak SSF %p\n", vs->ioc);
             goto authreject;
         }
 
         /* Check username whitelist ACL */
         if (vnc_auth_sasl_check_access(vs) < 0) {
-            VNC_DEBUG("Authentication rejected for ACL %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for ACL %p\n", vs->ioc);
             goto authreject;
         }
 
-        VNC_DEBUG("Authentication successful %d\n", vs->csock);
+        VNC_DEBUG("Authentication successful %p\n", vs->ioc);
         vnc_write_u32(vs, 0); /* Accept auth */
         /*
          * Delay writing in SSF encoded mode until pending output
@@ -383,17 +389,17 @@
         vnc_read_when(vs, protocol_client_auth_sasl_step_len, 4);
     } else {
         if (!vnc_auth_sasl_check_ssf(vs)) {
-            VNC_DEBUG("Authentication rejected for weak SSF %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for weak SSF %p\n", vs->ioc);
             goto authreject;
         }
 
         /* Check username whitelist ACL */
         if (vnc_auth_sasl_check_access(vs) < 0) {
-            VNC_DEBUG("Authentication rejected for ACL %d\n", vs->csock);
+            VNC_DEBUG("Authentication rejected for ACL %p\n", vs->ioc);
             goto authreject;
         }
 
-        VNC_DEBUG("Authentication successful %d\n", vs->csock);
+        VNC_DEBUG("Authentication successful %p\n", vs->ioc);
         vnc_write_u32(vs, 0); /* Accept auth */
         start_client_init(vs);
     }
@@ -487,6 +493,33 @@
     return 0;
 }
 
+static char *
+vnc_socket_ip_addr_string(QIOChannelSocket *ioc,
+                          bool local,
+                          Error **errp)
+{
+    SocketAddress *addr;
+    char *ret;
+
+    if (local) {
+        addr = qio_channel_socket_get_local_address(ioc, errp);
+    } else {
+        addr = qio_channel_socket_get_remote_address(ioc, errp);
+    }
+    if (!addr) {
+        return NULL;
+    }
+
+    if (addr->type != SOCKET_ADDRESS_KIND_INET) {
+        error_setg(errp, "Not an inet socket type");
+        return NULL;
+    }
+    ret = g_strdup_printf("%s;%s", addr->u.inet.data->host,
+                          addr->u.inet.data->port);
+    qapi_free_SocketAddress(addr);
+    return ret;
+}
+
 void start_auth_sasl(VncState *vs)
 {
     const char *mechlist = NULL;
@@ -495,13 +528,16 @@
     char *localAddr, *remoteAddr;
     int mechlistlen;
 
-    VNC_DEBUG("Initialize SASL auth %d\n", vs->csock);
+    VNC_DEBUG("Initialize SASL auth %p\n", vs->ioc);
 
     /* Get local & remote client addresses in form  IPADDR;PORT */
-    if (!(localAddr = vnc_socket_local_addr("%s;%s", vs->csock)))
+    localAddr = vnc_socket_ip_addr_string(vs->sioc, true, NULL);
+    if (!localAddr) {
         goto authabort;
+    }
 
-    if (!(remoteAddr = vnc_socket_remote_addr("%s;%s", vs->csock))) {
+    remoteAddr = vnc_socket_ip_addr_string(vs->sioc, false, NULL);
+    if (!remoteAddr) {
         g_free(localAddr);
         goto authabort;
     }
@@ -525,21 +561,24 @@
         goto authabort;
     }
 
-#ifdef CONFIG_VNC_TLS
     /* Inform SASL that we've got an external SSF layer from TLS/x509 */
     if (vs->auth == VNC_AUTH_VENCRYPT &&
         vs->subauth == VNC_AUTH_VENCRYPT_X509SASL) {
-        gnutls_cipher_algorithm_t cipher;
+        Error *local_err = NULL;
+        int keysize;
         sasl_ssf_t ssf;
 
-        cipher = gnutls_cipher_get(vs->tls.session);
-        if (!(ssf = (sasl_ssf_t)gnutls_cipher_get_key_size(cipher))) {
-            VNC_DEBUG("%s", "cannot TLS get cipher size\n");
+        keysize = qcrypto_tls_session_get_key_size(vs->tls,
+                                                   &local_err);
+        if (keysize < 0) {
+            VNC_DEBUG("cannot TLS get cipher size: %s\n",
+                      error_get_pretty(local_err));
+            error_free(local_err);
             sasl_dispose(&vs->sasl.conn);
             vs->sasl.conn = NULL;
             goto authabort;
         }
-        ssf *= 8; /* tls key size is bytes, sasl wants bits */
+        ssf = keysize * CHAR_BIT; /* tls key size is bytes, sasl wants bits */
 
         err = sasl_setprop(vs->sasl.conn, SASL_SSF_EXTERNAL, &ssf);
         if (err != SASL_OK) {
@@ -549,20 +588,19 @@
             vs->sasl.conn = NULL;
             goto authabort;
         }
-    } else
-#endif /* CONFIG_VNC_TLS */
+    } else {
         vs->sasl.wantSSF = 1;
+    }
 
     memset (&secprops, 0, sizeof secprops);
-    /* Inform SASL that we've got an external SSF layer from TLS */
-    if (strncmp(vs->vd->display, "unix:", 5) == 0
-#ifdef CONFIG_VNC_TLS
-        /* Disable SSF, if using TLS+x509+SASL only. TLS without x509
-           is not sufficiently strong */
-        || (vs->auth == VNC_AUTH_VENCRYPT &&
-            vs->subauth == VNC_AUTH_VENCRYPT_X509SASL)
-#endif /* CONFIG_VNC_TLS */
-        ) {
+    /* Inform SASL that we've got an external SSF layer from TLS.
+     *
+     * Disable SSF, if using TLS+x509+SASL only. TLS without x509
+     * is not sufficiently strong
+     */
+    if (vs->vd->is_unix ||
+        (vs->auth == VNC_AUTH_VENCRYPT &&
+         vs->subauth == VNC_AUTH_VENCRYPT_X509SASL)) {
         /* If we've got TLS or UNIX domain sock, we don't care about SSF */
         secprops.min_ssf = 0;
         secprops.max_ssf = 0;
diff --git a/ui/vnc-auth-sasl.h b/ui/vnc-auth-sasl.h
index 3f59da6..cb42745 100644
--- a/ui/vnc-auth-sasl.h
+++ b/ui/vnc-auth-sasl.h
@@ -22,10 +22,8 @@
  * THE SOFTWARE.
  */
 
-
-#ifndef __QEMU_VNC_AUTH_SASL_H__
-#define __QEMU_VNC_AUTH_SASL_H__
-
+#ifndef QEMU_VNC_AUTH_SASL_H
+#define QEMU_VNC_AUTH_SASL_H
 
 #include <sasl/sasl.h>
 
@@ -71,5 +69,4 @@
 
 void start_auth_sasl(VncState *vs);
 
-#endif /* __QEMU_VNC_AUTH_SASL_H__ */
-
+#endif /* QEMU_VNC_AUTH_SASL_H */
diff --git a/ui/vnc-auth-vencrypt.c b/ui/vnc-auth-vencrypt.c
index bc7032e..11c8c9a 100644
--- a/ui/vnc-auth-vencrypt.c
+++ b/ui/vnc-auth-vencrypt.c
@@ -24,7 +24,9 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
+#include "qapi/error.h"
 #include "qemu/main-loop.h"
 
 static void start_auth_vencrypt_subauth(VncState *vs)
@@ -63,59 +65,23 @@
     }
 }
 
-static void vnc_tls_handshake_io(void *opaque);
+static void vnc_tls_handshake_done(Object *source,
+                                   Error *err,
+                                   gpointer user_data)
+{
+    VncState *vs = user_data;
 
-static int vnc_start_vencrypt_handshake(struct VncState *vs) {
-    int ret;
-
-    if ((ret = gnutls_handshake(vs->tls.session)) < 0) {
-       if (!gnutls_error_is_fatal(ret)) {
-           VNC_DEBUG("Handshake interrupted (blocking)\n");
-           if (!gnutls_record_get_direction(vs->tls.session))
-               qemu_set_fd_handler(vs->csock, vnc_tls_handshake_io, NULL, vs);
-           else
-               qemu_set_fd_handler(vs->csock, NULL, vnc_tls_handshake_io, vs);
-           return 0;
-       }
-       VNC_DEBUG("Handshake failed %s\n", gnutls_strerror(ret));
-       vnc_client_error(vs);
-       return -1;
+    if (err) {
+        VNC_DEBUG("Handshake failed %s\n",
+                  error_get_pretty(err));
+        vnc_client_error(vs);
+    } else {
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN | G_IO_OUT, vnc_client_io, vs, NULL);
+        start_auth_vencrypt_subauth(vs);
     }
-
-    if (vs->vd->tls.x509verify) {
-        if (vnc_tls_validate_certificate(vs) < 0) {
-            VNC_DEBUG("Client verification failed\n");
-            vnc_client_error(vs);
-            return -1;
-        } else {
-            VNC_DEBUG("Client verification passed\n");
-        }
-    }
-
-    VNC_DEBUG("Handshake done, switching to TLS data mode\n");
-    vs->tls.wiremode = VNC_WIREMODE_TLS;
-    qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, vnc_client_write, vs);
-
-    start_auth_vencrypt_subauth(vs);
-
-    return 0;
 }
 
-static void vnc_tls_handshake_io(void *opaque) {
-    struct VncState *vs = (struct VncState *)opaque;
-
-    VNC_DEBUG("Handshake IO continue\n");
-    vnc_start_vencrypt_handshake(vs);
-}
-
-
-
-#define NEED_X509_AUTH(vs)                              \
-    ((vs)->subauth == VNC_AUTH_VENCRYPT_X509NONE ||   \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509VNC ||    \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509PLAIN ||  \
-     (vs)->subauth == VNC_AUTH_VENCRYPT_X509SASL)
-
 
 static int protocol_client_vencrypt_auth(VncState *vs, uint8_t *data, size_t len)
 {
@@ -127,20 +93,38 @@
         vnc_flush(vs);
         vnc_client_error(vs);
     } else {
+        Error *err = NULL;
+        QIOChannelTLS *tls;
         VNC_DEBUG("Accepting auth %d, setting up TLS for handshake\n", auth);
         vnc_write_u8(vs, 1); /* Accept auth */
         vnc_flush(vs);
 
-        if (vnc_tls_client_setup(vs, NEED_X509_AUTH(vs)) < 0) {
-            VNC_DEBUG("Failed to setup TLS\n");
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+            vs->ioc_tag = 0;
+        }
+
+        tls = qio_channel_tls_new_server(
+            vs->ioc,
+            vs->vd->tlscreds,
+            vs->vd->tlsaclname,
+            &err);
+        if (!tls) {
+            VNC_DEBUG("Failed to setup TLS %s\n", error_get_pretty(err));
+            error_free(err);
+            vnc_client_error(vs);
             return 0;
         }
 
         VNC_DEBUG("Start TLS VeNCrypt handshake process\n");
-        if (vnc_start_vencrypt_handshake(vs) < 0) {
-            VNC_DEBUG("Failed to start TLS handshake\n");
-            return 0;
-        }
+        object_unref(OBJECT(vs->ioc));
+        vs->ioc = QIO_CHANNEL(tls);
+        vs->tls = qio_channel_tls_get_session(tls);
+
+        qio_channel_tls_handshake(tls,
+                                  vnc_tls_handshake_done,
+                                  vs,
+                                  NULL);
     }
     return 0;
 }
diff --git a/ui/vnc-auth-vencrypt.h b/ui/vnc-auth-vencrypt.h
index 9f674c5..1e35406 100644
--- a/ui/vnc-auth-vencrypt.h
+++ b/ui/vnc-auth-vencrypt.h
@@ -24,10 +24,9 @@
  * THE SOFTWARE.
  */
 
-
-#ifndef __QEMU_VNC_AUTH_VENCRYPT_H__
-#define __QEMU_VNC_AUTH_VENCRYPT_H__
+#ifndef QEMU_VNC_AUTH_VENCRYPT_H
+#define QEMU_VNC_AUTH_VENCRYPT_H
 
 void start_auth_vencrypt(VncState *vs);
 
-#endif /* __QEMU_VNC_AUTH_VENCRYPT_H__ */
+#endif /* QEMU_VNC_AUTH_VENCRYPT_H */
diff --git a/ui/vnc-enc-hextile.c b/ui/vnc-enc-hextile.c
index 2e768fd..4215bd7 100644
--- a/ui/vnc-enc-hextile.c
+++ b/ui/vnc-enc-hextile.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 
 static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index 3d1b5cd..49df85e 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -26,7 +26,7 @@
  * THE SOFTWARE.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 
 /* This needs to be before jpeglib.h line because of conflict with
    INT32 definitions between jmorecfg.h (included by jpeglib.h) and
@@ -40,7 +40,6 @@
 #include <png.h>
 #endif
 #ifdef CONFIG_VNC_JPEG
-#include <stdio.h>
 #include <jpeglib.h>
 #endif
 
@@ -350,7 +349,7 @@
     tight_fill_palette##bpp(VncState *vs, int x, int y,                 \
                             int max, size_t count,                      \
                             uint32_t *bg, uint32_t *fg,                 \
-                            VncPalette **palette) {                     \
+                            VncPalette *palette) {                      \
         uint##bpp##_t *data;                                            \
         uint##bpp##_t c0, c1, ci;                                       \
         int i, n0, n1;                                                  \
@@ -397,23 +396,23 @@
             return 0;                                                   \
         }                                                               \
                                                                         \
-        *palette = palette_new(max, bpp);                               \
-        palette_put(*palette, c0);                                      \
-        palette_put(*palette, c1);                                      \
-        palette_put(*palette, ci);                                      \
+        palette_init(palette, max, bpp);                                \
+        palette_put(palette, c0);                                       \
+        palette_put(palette, c1);                                       \
+        palette_put(palette, ci);                                       \
                                                                         \
         for (i++; i < count; i++) {                                     \
             if (data[i] == ci) {                                        \
                 continue;                                               \
             } else {                                                    \
                 ci = data[i];                                           \
-                if (!palette_put(*palette, (uint32_t)ci)) {             \
+                if (!palette_put(palette, (uint32_t)ci)) {              \
                     return 0;                                           \
                 }                                                       \
             }                                                           \
         }                                                               \
                                                                         \
-        return palette_size(*palette);                                  \
+        return palette_size(palette);                                   \
     }
 
 DEFINE_FILL_PALETTE_FUNCTION(8)
@@ -422,7 +421,7 @@
 
 static int tight_fill_palette(VncState *vs, int x, int y,
                               size_t count, uint32_t *bg, uint32_t *fg,
-                              VncPalette **palette)
+                              VncPalette *palette)
 {
     int max;
 
@@ -462,9 +461,10 @@
                                                                         \
         src = (uint##bpp##_t *) buf;                                    \
                                                                         \
-        for (i = 0; i < count; i++) {                                   \
+        for (i = 0; i < count; ) {                                      \
                                                                         \
             rgb = *src++;                                               \
+            i++;                                                        \
             rep = 0;                                                    \
             while (i < count && *src == rgb) {                          \
                 rep++, src++, i++;                                      \
@@ -1458,9 +1458,17 @@
 }
 #endif
 
+static __thread VncPalette *color_count_palette;
+static __thread Notifier vnc_tight_cleanup_notifier;
+
+static void vnc_tight_cleanup(Notifier *n, void *value)
+{
+    g_free(color_count_palette);
+    color_count_palette = NULL;
+}
+
 static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
 {
-    VncPalette *palette = NULL;
     uint32_t bg = 0, fg = 0;
     int colors;
     int ret = 0;
@@ -1469,6 +1477,12 @@
     bool allow_jpeg = true;
 #endif
 
+    if (!color_count_palette) {
+        color_count_palette = g_malloc(sizeof(VncPalette));
+        vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup;
+        qemu_thread_atexit_add(&vnc_tight_cleanup_notifier);
+    }
+
     vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
 
     vnc_tight_start(vs);
@@ -1489,20 +1503,21 @@
     }
 #endif
 
-    colors = tight_fill_palette(vs, x, y, w * h, &fg, &bg, &palette);
+    colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette);
 
 #ifdef CONFIG_VNC_JPEG
     if (allow_jpeg && vs->tight.quality != (uint8_t)-1) {
-        ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette,
-                                 force_jpeg);
+        ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors,
+                                 color_count_palette, force_jpeg);
     } else {
-        ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
+        ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
+                                   color_count_palette);
     }
 #else
-    ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette);
+    ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
+                               color_count_palette);
 #endif
 
-    palette_destroy(palette);
     return ret;
 }
 
diff --git a/ui/vnc-enc-tight.h b/ui/vnc-enc-tight.h
index a3add78..95c3dac 100644
--- a/ui/vnc-enc-tight.h
+++ b/ui/vnc-enc-tight.h
@@ -27,8 +27,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef VNC_ENCODING_TIGHT_H
-#define VNC_ENCODING_TIGHT_H
+#ifndef VNC_ENC_TIGHT_H
+#define VNC_ENC_TIGHT_H
 
 /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  * Tight Encoding.
@@ -180,4 +180,4 @@
 #define VNC_TIGHT_DETECT_MIN_WIDTH           8
 #define VNC_TIGHT_DETECT_MIN_HEIGHT          8
 
-#endif /* VNC_ENCODING_TIGHT_H */
+#endif /* VNC_ENC_TIGHT_H */
diff --git a/ui/vnc-enc-zlib.c b/ui/vnc-enc-zlib.c
index d1b97f2..33e9df2 100644
--- a/ui/vnc-enc-zlib.c
+++ b/ui/vnc-enc-zlib.c
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 
 #define ZALLOC_ALIGNMENT 16
diff --git a/ui/vnc-enc-zrle-template.c b/ui/vnc-enc-zrle-template.c
index 70ae624..abf6b86 100644
--- a/ui/vnc-enc-zrle-template.c
+++ b/ui/vnc-enc-zrle-template.c
@@ -22,7 +22,7 @@
  */
 
 
-#include <assert.h>
+#include "qemu/osdep.h"
 
 #undef ZRLE_ENDIAN_SUFFIX
 
diff --git a/ui/vnc-enc-zrle.c b/ui/vnc-enc-zrle.c
index ed3b484..5489870 100644
--- a/ui/vnc-enc-zrle.c
+++ b/ui/vnc-enc-zrle.c
@@ -26,6 +26,7 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 #include "vnc-enc-zrle.h"
 
diff --git a/ui/vnc-enc-zrle.h b/ui/vnc-enc-zrle.h
index 6b18213..6535cb2 100644
--- a/ui/vnc-enc-zrle.h
+++ b/ui/vnc-enc-zrle.h
@@ -26,8 +26,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef VNC_ENCODING_ZRLE_H
-#define VNC_ENCODING_ZRLE_H
+#ifndef VNC_ENC_ZRLE_H
+#define VNC_ENC_ZRLE_H
 
 /*- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  * ZRLE - encoding combining Zlib compression, tiling, palettisation and
diff --git a/ui/vnc-enc-zywrle-template.c b/ui/vnc-enc-zywrle-template.c
index 561f7bf..b446380 100644
--- a/ui/vnc-enc-zywrle-template.c
+++ b/ui/vnc-enc-zywrle-template.c
@@ -100,6 +100,7 @@
 #endif
 
 #define ZYWRLE_QUANTIZE
+#include "qemu/osdep.h"
 #include "vnc-enc-zywrle.h"
 
 #ifndef ZRLE_COMPACT_PIXEL
diff --git a/ui/vnc-enc-zywrle.h b/ui/vnc-enc-zywrle.h
index d436d58..610bd79 100644
--- a/ui/vnc-enc-zywrle.h
+++ b/ui/vnc-enc-zywrle.h
@@ -41,8 +41,8 @@
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  ********************************************************************/
 
-#ifndef VNC_ENCODING_ZYWRLE_H
-#define VNC_ENCODING_ZYWRLE_H
+#ifndef VNC_ENC_ZYWRLE_H
+#define VNC_ENC_ZYWRLE_H
 
 /* Tables for Coefficients filtering. */
 #ifndef ZYWRLE_QUANTIZE
diff --git a/ui/vnc-jobs.c b/ui/vnc-jobs.c
index 68f3d77..98ca978 100644
--- a/ui/vnc-jobs.c
+++ b/ui/vnc-jobs.c
@@ -26,9 +26,12 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 #include "vnc-jobs.h"
 #include "qemu/sockets.h"
+#include "qemu/main-loop.h"
+#include "block/aio.h"
 
 /*
  * Locking:
@@ -53,7 +56,6 @@
     QemuCond cond;
     QemuMutex mutex;
     QemuThread thread;
-    Buffer buffer;
     bool exit;
     QTAILQ_HEAD(, VncJob) jobs;
 };
@@ -78,7 +80,7 @@
 
 VncJob *vnc_job_new(VncState *vs)
 {
-    VncJob *job = g_malloc0(sizeof(VncJob));
+    VncJob *job = g_new0(VncJob, 1);
 
     job->vs = vs;
     vnc_lock_queue(queue);
@@ -89,7 +91,7 @@
 
 int vnc_job_add_rect(VncJob *job, int x, int y, int w, int h)
 {
-    VncRectEntry *entry = g_malloc0(sizeof(VncRectEntry));
+    VncRectEntry *entry = g_new0(VncRectEntry, 1);
 
     entry->rect.x = x;
     entry->rect.y = y;
@@ -165,10 +167,16 @@
 
     vnc_lock_output(vs);
     if (vs->jobs_buffer.offset) {
-        vnc_write(vs, vs->jobs_buffer.buffer, vs->jobs_buffer.offset);
-        buffer_reset(&vs->jobs_buffer);
+        if (vs->ioc != NULL && buffer_empty(&vs->output)) {
+            if (vs->ioc_tag) {
+                g_source_remove(vs->ioc_tag);
+            }
+            vs->ioc_tag = qio_channel_add_watch(
+                vs->ioc, G_IO_IN | G_IO_OUT, vnc_client_io, vs, NULL);
+        }
+        buffer_move(&vs->output, &vs->jobs_buffer);
     }
-    flush = vs->csock != -1 && vs->abort != true;
+    flush = vs->ioc != NULL && vs->abort != true;
     vnc_unlock_output(vs);
 
     if (flush) {
@@ -181,6 +189,10 @@
  */
 static void vnc_async_encoding_start(VncState *orig, VncState *local)
 {
+    buffer_init(&local->output, "vnc-worker-output");
+    local->sioc = NULL; /* Don't do any network work on this thread */
+    local->ioc = NULL; /* Don't do any network work on this thread */
+
     local->vnc_encoding = orig->vnc_encoding;
     local->features = orig->features;
     local->vd = orig->vd;
@@ -192,10 +204,6 @@
     local->zlib = orig->zlib;
     local->hextile = orig->hextile;
     local->zrle = orig->zrle;
-    local->output =  queue->buffer;
-    local->csock = -1; /* Don't do any network work on this thread */
-
-    buffer_reset(&local->output);
 }
 
 static void vnc_async_encoding_end(VncState *orig, VncState *local)
@@ -205,15 +213,13 @@
     orig->hextile = local->hextile;
     orig->zrle = local->zrle;
     orig->lossy_rect = local->lossy_rect;
-
-    queue->buffer = local->output;
 }
 
 static int vnc_worker_thread_loop(VncJobQueue *queue)
 {
     VncJob *job;
     VncRectEntry *entry, *tmp;
-    VncState vs;
+    VncState vs = {};
     int n_rectangles;
     int saved_offset;
 
@@ -230,10 +236,18 @@
     }
 
     vnc_lock_output(job->vs);
-    if (job->vs->csock == -1 || job->vs->abort == true) {
+    if (job->vs->ioc == NULL || job->vs->abort == true) {
         vnc_unlock_output(job->vs);
         goto disconnected;
     }
+    if (buffer_empty(&job->vs->output)) {
+        /*
+         * Looks like a NOP as it obviously moves no data.  But it
+         * moves the empty buffer, so we don't have to malloc a new
+         * one for vs.output
+         */
+        buffer_move_empty(&vs.output, &job->vs->output);
+    }
     vnc_unlock_output(job->vs);
 
     /* Make a local copy of vs and switch output buffers */
@@ -250,7 +264,7 @@
     QLIST_FOREACH_SAFE(entry, &job->rectangles, next, tmp) {
         int n;
 
-        if (job->vs->csock == -1) {
+        if (job->vs->ioc == NULL) {
             vnc_unlock_display(job->vs->vd);
             /* Copy persistent encoding data */
             vnc_async_encoding_end(job->vs, &vs);
@@ -272,15 +286,14 @@
     vs.output.buffer[saved_offset + 1] = n_rectangles & 0xFF;
 
     vnc_lock_output(job->vs);
-    if (job->vs->csock != -1) {
-        buffer_reserve(&job->vs->jobs_buffer, vs.output.offset);
-        buffer_append(&job->vs->jobs_buffer, vs.output.buffer,
-                      vs.output.offset);
+    if (job->vs->ioc != NULL) {
+        buffer_move(&job->vs->jobs_buffer, &vs.output);
         /* Copy persistent encoding data */
         vnc_async_encoding_end(job->vs, &vs);
 
 	qemu_bh_schedule(job->vs->bh);
     }  else {
+        buffer_reset(&vs.output);
         /* Copy persistent encoding data */
         vnc_async_encoding_end(job->vs, &vs);
     }
@@ -297,7 +310,7 @@
 
 static VncJobQueue *vnc_queue_init(void)
 {
-    VncJobQueue *queue = g_malloc0(sizeof(VncJobQueue));
+    VncJobQueue *queue = g_new0(VncJobQueue, 1);
 
     qemu_cond_init(&queue->cond);
     qemu_mutex_init(&queue->mutex);
@@ -309,7 +322,6 @@
 {
     qemu_cond_destroy(&queue->cond);
     qemu_mutex_destroy(&queue->mutex);
-    buffer_free(&queue->buffer);
     g_free(q);
     queue = NULL; /* Unset global queue */
 }
@@ -342,16 +354,3 @@
                        QEMU_THREAD_DETACHED);
     queue = q; /* Set global queue */
 }
-
-void vnc_stop_worker_thread(void)
-{
-    if (!vnc_worker_thread_running())
-        return ;
-
-    /* Remove all jobs and wake up the thread */
-    vnc_lock_queue(queue);
-    queue->exit = true;
-    vnc_unlock_queue(queue);
-    vnc_jobs_clear(NULL);
-    qemu_cond_broadcast(&queue->cond);
-}
diff --git a/ui/vnc-jobs.h b/ui/vnc-jobs.h
index 31da103..044bf9f 100644
--- a/ui/vnc-jobs.h
+++ b/ui/vnc-jobs.h
@@ -40,7 +40,6 @@
 
 void vnc_jobs_consume_buffer(VncState *vs);
 void vnc_start_worker_thread(void);
-void vnc_stop_worker_thread(void);
 
 /* Locks */
 static inline int vnc_trylock_display(VncDisplay *vd)
diff --git a/ui/vnc-palette.c b/ui/vnc-palette.c
index c130dee..dc7c0ba 100644
--- a/ui/vnc-palette.c
+++ b/ui/vnc-palette.c
@@ -26,9 +26,8 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc-palette.h"
-#include <glib.h>
-#include <string.h>
 
 static VncPaletteEntry *palette_find(const VncPalette *palette,
                                      uint32_t color, unsigned int hash)
diff --git a/ui/vnc-palette.h b/ui/vnc-palette.h
index d02f023..1bd4318 100644
--- a/ui/vnc-palette.h
+++ b/ui/vnc-palette.h
@@ -31,8 +31,6 @@
 
 #include "qapi/qmp/qlist.h"
 #include "qemu/queue.h"
-#include <stdint.h>
-#include <stdbool.h>
 
 #define VNC_PALETTE_HASH_SIZE 256
 #define VNC_PALETTE_MAX_SIZE  256
diff --git a/ui/vnc-tls.c b/ui/vnc-tls.c
deleted file mode 100644
index 0f59f9b..0000000
--- a/ui/vnc-tls.c
+++ /dev/null
@@ -1,488 +0,0 @@
-/*
- * QEMU VNC display driver: TLS helpers
- *
- * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2006 Fabrice Bellard
- * Copyright (C) 2009 Red Hat, Inc
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-#include "qemu-x509.h"
-#include "vnc.h"
-#include "qemu/sockets.h"
-
-#if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
-/* Very verbose, so only enabled for _VNC_DEBUG >= 2 */
-static void vnc_debug_gnutls_log(int level, const char* str) {
-    VNC_DEBUG("%d %s", level, str);
-}
-#endif /* defined(_VNC_DEBUG) && _VNC_DEBUG >= 2 */
-
-
-#define DH_BITS 1024
-static gnutls_dh_params_t dh_params;
-
-static int vnc_tls_initialize(void)
-{
-    static int tlsinitialized = 0;
-
-    if (tlsinitialized)
-        return 1;
-
-    if (gnutls_global_init () < 0)
-        return 0;
-
-    /* XXX ought to re-generate diffie-hellman params periodically */
-    if (gnutls_dh_params_init (&dh_params) < 0)
-        return 0;
-    if (gnutls_dh_params_generate2 (dh_params, DH_BITS) < 0)
-        return 0;
-
-#if defined(_VNC_DEBUG) && _VNC_DEBUG >= 2
-    gnutls_global_set_log_level(10);
-    gnutls_global_set_log_function(vnc_debug_gnutls_log);
-#endif
-
-    tlsinitialized = 1;
-
-    return 1;
-}
-
-static ssize_t vnc_tls_push(gnutls_transport_ptr_t transport,
-                            const void *data,
-                            size_t len) {
-    struct VncState *vs = (struct VncState *)transport;
-    int ret;
-
- retry:
-    ret = send(vs->csock, data, len, 0);
-    if (ret < 0) {
-        if (errno == EINTR)
-            goto retry;
-        return -1;
-    }
-    return ret;
-}
-
-
-static ssize_t vnc_tls_pull(gnutls_transport_ptr_t transport,
-                            void *data,
-                            size_t len) {
-    struct VncState *vs = (struct VncState *)transport;
-    int ret;
-
- retry:
-    ret = qemu_recv(vs->csock, data, len, 0);
-    if (ret < 0) {
-        if (errno == EINTR)
-            goto retry;
-        return -1;
-    }
-    return ret;
-}
-
-
-static gnutls_anon_server_credentials_t vnc_tls_initialize_anon_cred(void)
-{
-    gnutls_anon_server_credentials_t anon_cred;
-    int ret;
-
-    if ((ret = gnutls_anon_allocate_server_credentials(&anon_cred)) < 0) {
-        VNC_DEBUG("Cannot allocate credentials %s\n", gnutls_strerror(ret));
-        return NULL;
-    }
-
-    gnutls_anon_set_server_dh_params(anon_cred, dh_params);
-
-    return anon_cred;
-}
-
-
-static gnutls_certificate_credentials_t vnc_tls_initialize_x509_cred(VncDisplay *vd)
-{
-    gnutls_certificate_credentials_t x509_cred;
-    int ret;
-
-    if (!vd->tls.x509cacert) {
-        VNC_DEBUG("No CA x509 certificate specified\n");
-        return NULL;
-    }
-    if (!vd->tls.x509cert) {
-        VNC_DEBUG("No server x509 certificate specified\n");
-        return NULL;
-    }
-    if (!vd->tls.x509key) {
-        VNC_DEBUG("No server private key specified\n");
-        return NULL;
-    }
-
-    if ((ret = gnutls_certificate_allocate_credentials(&x509_cred)) < 0) {
-        VNC_DEBUG("Cannot allocate credentials %s\n", gnutls_strerror(ret));
-        return NULL;
-    }
-    if ((ret = gnutls_certificate_set_x509_trust_file(x509_cred,
-                                                      vd->tls.x509cacert,
-                                                      GNUTLS_X509_FMT_PEM)) < 0) {
-        VNC_DEBUG("Cannot load CA certificate %s\n", gnutls_strerror(ret));
-        gnutls_certificate_free_credentials(x509_cred);
-        return NULL;
-    }
-
-    if ((ret = gnutls_certificate_set_x509_key_file (x509_cred,
-                                                     vd->tls.x509cert,
-                                                     vd->tls.x509key,
-                                                     GNUTLS_X509_FMT_PEM)) < 0) {
-        VNC_DEBUG("Cannot load certificate & key %s\n", gnutls_strerror(ret));
-        gnutls_certificate_free_credentials(x509_cred);
-        return NULL;
-    }
-
-    if (vd->tls.x509cacrl) {
-        if ((ret = gnutls_certificate_set_x509_crl_file(x509_cred,
-                                                        vd->tls.x509cacrl,
-                                                        GNUTLS_X509_FMT_PEM)) < 0) {
-            VNC_DEBUG("Cannot load CRL %s\n", gnutls_strerror(ret));
-            gnutls_certificate_free_credentials(x509_cred);
-            return NULL;
-        }
-    }
-
-    gnutls_certificate_set_dh_params (x509_cred, dh_params);
-
-    return x509_cred;
-}
-
-
-int vnc_tls_validate_certificate(struct VncState *vs)
-{
-    int ret;
-    unsigned int status;
-    const gnutls_datum_t *certs;
-    unsigned int nCerts, i;
-    time_t now;
-
-    VNC_DEBUG("Validating client certificate\n");
-    if ((ret = gnutls_certificate_verify_peers2 (vs->tls.session, &status)) < 0) {
-        VNC_DEBUG("Verify failed %s\n", gnutls_strerror(ret));
-        return -1;
-    }
-
-    if ((now = time(NULL)) == ((time_t)-1)) {
-        return -1;
-    }
-
-    if (status != 0) {
-        if (status & GNUTLS_CERT_INVALID)
-            VNC_DEBUG("The certificate is not trusted.\n");
-
-        if (status & GNUTLS_CERT_SIGNER_NOT_FOUND)
-            VNC_DEBUG("The certificate hasn't got a known issuer.\n");
-
-        if (status & GNUTLS_CERT_REVOKED)
-            VNC_DEBUG("The certificate has been revoked.\n");
-
-        if (status & GNUTLS_CERT_INSECURE_ALGORITHM)
-            VNC_DEBUG("The certificate uses an insecure algorithm\n");
-
-        return -1;
-    } else {
-        VNC_DEBUG("Certificate is valid!\n");
-    }
-
-    /* Only support x509 for now */
-    if (gnutls_certificate_type_get(vs->tls.session) != GNUTLS_CRT_X509)
-        return -1;
-
-    if (!(certs = gnutls_certificate_get_peers(vs->tls.session, &nCerts)))
-        return -1;
-
-    for (i = 0 ; i < nCerts ; i++) {
-        gnutls_x509_crt_t cert;
-        VNC_DEBUG ("Checking certificate chain %d\n", i);
-        if (gnutls_x509_crt_init (&cert) < 0)
-            return -1;
-
-        if (gnutls_x509_crt_import(cert, &certs[i], GNUTLS_X509_FMT_DER) < 0) {
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_expiration_time (cert) < now) {
-            VNC_DEBUG("The certificate has expired\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_activation_time (cert) > now) {
-            VNC_DEBUG("The certificate is not yet activated\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (gnutls_x509_crt_get_activation_time (cert) > now) {
-            VNC_DEBUG("The certificate is not yet activated\n");
-            gnutls_x509_crt_deinit (cert);
-            return -1;
-        }
-
-        if (i == 0) {
-            size_t dnameSize = 1024;
-            vs->tls.dname = g_malloc(dnameSize);
-        requery:
-            if ((ret = gnutls_x509_crt_get_dn (cert, vs->tls.dname, &dnameSize)) != 0) {
-                if (ret == GNUTLS_E_SHORT_MEMORY_BUFFER) {
-                    vs->tls.dname = g_realloc(vs->tls.dname, dnameSize);
-                    goto requery;
-                }
-                gnutls_x509_crt_deinit (cert);
-                VNC_DEBUG("Cannot get client distinguished name: %s",
-                          gnutls_strerror (ret));
-                return -1;
-            }
-
-            if (vs->vd->tls.x509verify) {
-                int allow;
-                if (!vs->vd->tls.acl) {
-                    VNC_DEBUG("no ACL activated, allowing access");
-                    gnutls_x509_crt_deinit (cert);
-                    continue;
-                }
-
-                allow = qemu_acl_party_is_allowed(vs->vd->tls.acl,
-                                                  vs->tls.dname);
-
-                VNC_DEBUG("TLS x509 ACL check for %s is %s\n",
-                          vs->tls.dname, allow ? "allowed" : "denied");
-                if (!allow) {
-                    gnutls_x509_crt_deinit (cert);
-                    return -1;
-                }
-            }
-        }
-
-        gnutls_x509_crt_deinit (cert);
-    }
-
-    return 0;
-}
-
-#if defined(GNUTLS_VERSION_NUMBER) && \
-    GNUTLS_VERSION_NUMBER >= 0x020200 /* 2.2.0 */
-
-static int vnc_set_gnutls_priority(gnutls_session_t s, int x509)
-{
-    const char *priority = x509 ? "NORMAL" : "NORMAL:+ANON-DH";
-    int rc;
-
-    rc = gnutls_priority_set_direct(s, priority, NULL);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-    return 0;
-}
-
-#else
-
-static int vnc_set_gnutls_priority(gnutls_session_t s, int x509)
-{
-    static const int cert_types[] = { GNUTLS_CRT_X509, 0 };
-    static const int protocols[] = {
-        GNUTLS_TLS1_1, GNUTLS_TLS1_0, GNUTLS_SSL3, 0
-    };
-    static const int kx_anon[] = { GNUTLS_KX_ANON_DH, 0 };
-    static const int kx_x509[] = {
-        GNUTLS_KX_DHE_DSS, GNUTLS_KX_RSA,
-        GNUTLS_KX_DHE_RSA, GNUTLS_KX_SRP, 0
-    };
-    int rc;
-
-    rc = gnutls_kx_set_priority(s, x509 ? kx_x509 : kx_anon);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-
-    rc = gnutls_certificate_type_set_priority(s, cert_types);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-
-    rc = gnutls_protocol_set_priority(s, protocols);
-    if (rc != GNUTLS_E_SUCCESS) {
-        return -1;
-    }
-    return 0;
-}
-
-#endif
-
-int vnc_tls_client_setup(struct VncState *vs,
-                         int needX509Creds) {
-    VncStateTLS *tls;
-
-    VNC_DEBUG("Do TLS setup\n");
-#ifdef CONFIG_VNC_WS
-    if (vs->websocket) {
-        tls = &vs->ws_tls;
-    } else
-#endif /* CONFIG_VNC_WS */
-    {
-        tls = &vs->tls;
-    }
-    if (vnc_tls_initialize() < 0) {
-        VNC_DEBUG("Failed to init TLS\n");
-        vnc_client_error(vs);
-        return -1;
-    }
-    if (tls->session == NULL) {
-        if (gnutls_init(&tls->session, GNUTLS_SERVER) < 0) {
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (gnutls_set_default_priority(tls->session) < 0) {
-            gnutls_deinit(tls->session);
-            tls->session = NULL;
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (vnc_set_gnutls_priority(tls->session, needX509Creds) < 0) {
-            gnutls_deinit(tls->session);
-            tls->session = NULL;
-            vnc_client_error(vs);
-            return -1;
-        }
-
-        if (needX509Creds) {
-            gnutls_certificate_server_credentials x509_cred = vnc_tls_initialize_x509_cred(vs->vd);
-            if (!x509_cred) {
-                gnutls_deinit(tls->session);
-                tls->session = NULL;
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (gnutls_credentials_set(tls->session, GNUTLS_CRD_CERTIFICATE, x509_cred) < 0) {
-                gnutls_deinit(tls->session);
-                tls->session = NULL;
-                gnutls_certificate_free_credentials(x509_cred);
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (vs->vd->tls.x509verify) {
-                VNC_DEBUG("Requesting a client certificate\n");
-                gnutls_certificate_server_set_request (tls->session, GNUTLS_CERT_REQUEST);
-            }
-
-        } else {
-            gnutls_anon_server_credentials_t anon_cred = vnc_tls_initialize_anon_cred();
-            if (!anon_cred) {
-                gnutls_deinit(tls->session);
-                tls->session = NULL;
-                vnc_client_error(vs);
-                return -1;
-            }
-            if (gnutls_credentials_set(tls->session, GNUTLS_CRD_ANON, anon_cred) < 0) {
-                gnutls_deinit(tls->session);
-                tls->session = NULL;
-                gnutls_anon_free_server_credentials(anon_cred);
-                vnc_client_error(vs);
-                return -1;
-            }
-        }
-
-        gnutls_transport_set_ptr(tls->session, (gnutls_transport_ptr_t)vs);
-        gnutls_transport_set_push_function(tls->session, vnc_tls_push);
-        gnutls_transport_set_pull_function(tls->session, vnc_tls_pull);
-    }
-    return 0;
-}
-
-
-void vnc_tls_client_cleanup(struct VncState *vs)
-{
-    if (vs->tls.session) {
-        gnutls_deinit(vs->tls.session);
-        vs->tls.session = NULL;
-    }
-    vs->tls.wiremode = VNC_WIREMODE_CLEAR;
-    g_free(vs->tls.dname);
-#ifdef CONFIG_VNC_WS
-    if (vs->ws_tls.session) {
-        gnutls_deinit(vs->ws_tls.session);
-        vs->ws_tls.session = NULL;
-    }
-    vs->ws_tls.wiremode = VNC_WIREMODE_CLEAR;
-    g_free(vs->ws_tls.dname);
-#endif /* CONFIG_VNC_WS */
-}
-
-
-
-static int vnc_set_x509_credential(VncDisplay *vd,
-                                   const char *certdir,
-                                   const char *filename,
-                                   char **cred,
-                                   int ignoreMissing)
-{
-    struct stat sb;
-
-    g_free(*cred);
-    *cred = g_malloc(strlen(certdir) + strlen(filename) + 2);
-
-    strcpy(*cred, certdir);
-    strcat(*cred, "/");
-    strcat(*cred, filename);
-
-    VNC_DEBUG("Check %s\n", *cred);
-    if (stat(*cred, &sb) < 0) {
-        g_free(*cred);
-        *cred = NULL;
-        if (ignoreMissing && errno == ENOENT)
-            return 0;
-        return -1;
-    }
-
-    return 0;
-}
-
-
-int vnc_tls_set_x509_creds_dir(VncDisplay *vd,
-                               const char *certdir)
-{
-    if (vnc_set_x509_credential(vd, certdir, X509_CA_CERT_FILE, &vd->tls.x509cacert, 0) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_CA_CRL_FILE, &vd->tls.x509cacrl, 1) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_SERVER_CERT_FILE, &vd->tls.x509cert, 0) < 0)
-        goto cleanup;
-    if (vnc_set_x509_credential(vd, certdir, X509_SERVER_KEY_FILE, &vd->tls.x509key, 0) < 0)
-        goto cleanup;
-
-    return 0;
-
- cleanup:
-    g_free(vd->tls.x509cacert);
-    g_free(vd->tls.x509cacrl);
-    g_free(vd->tls.x509cert);
-    g_free(vd->tls.x509key);
-    vd->tls.x509cacert = vd->tls.x509cacrl = vd->tls.x509cert = vd->tls.x509key = NULL;
-    return -1;
-}
-
diff --git a/ui/vnc-tls.h b/ui/vnc-tls.h
deleted file mode 100644
index 36a2227..0000000
--- a/ui/vnc-tls.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * QEMU VNC display driver. TLS helpers
- *
- * Copyright (C) 2006 Anthony Liguori <anthony@codemonkey.ws>
- * Copyright (C) 2006 Fabrice Bellard
- * Copyright (C) 2009 Red Hat, Inc
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-
-
-#ifndef __QEMU_VNC_TLS_H__
-#define __QEMU_VNC_TLS_H__
-
-#include <gnutls/gnutls.h>
-#include <gnutls/x509.h>
-
-#include "qemu/acl.h"
-
-enum {
-    VNC_WIREMODE_CLEAR,
-    VNC_WIREMODE_TLS,
-};
-
-typedef struct VncDisplayTLS VncDisplayTLS;
-typedef struct VncStateTLS VncStateTLS;
-
-/* Server state */
-struct VncDisplayTLS {
-    int x509verify; /* Non-zero if server requests & validates client cert */
-    qemu_acl *acl;
-
-    /* Paths to x509 certs/keys */
-    char *x509cacert;
-    char *x509cacrl;
-    char *x509cert;
-    char *x509key;
-};
-
-/* Per client state */
-struct VncStateTLS {
-    /* Whether data is being TLS encrypted yet */
-    int wiremode;
-    gnutls_session_t session;
-
-    /* Client's Distinguished Name from the x509 cert */
-    char *dname;
-};
-
-int vnc_tls_client_setup(VncState *vs, int x509Creds);
-void vnc_tls_client_cleanup(VncState *vs);
-
-int vnc_tls_validate_certificate(VncState *vs);
-
-int vnc_tls_set_x509_creds_dir(VncDisplay *vd,
-			       const char *path);
-
-
-#endif /* __QEMU_VNC_TLS_H__ */
-
diff --git a/ui/vnc-ws.c b/ui/vnc-ws.c
index e304baf..3bac46e 100644
--- a/ui/vnc-ws.c
+++ b/ui/vnc-ws.c
@@ -18,332 +18,109 @@
  * along with this software; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "vnc.h"
-#include "qemu/main-loop.h"
+#include "io/channel-websock.h"
+#include "qemu/bswap.h"
 
-#ifdef CONFIG_VNC_TLS
-#include "qemu/sockets.h"
-
-static void vncws_tls_handshake_io(void *opaque);
-
-static int vncws_start_tls_handshake(struct VncState *vs)
+static void vncws_tls_handshake_done(Object *source,
+                                     Error *err,
+                                     gpointer user_data)
 {
-    int ret = gnutls_handshake(vs->ws_tls.session);
+    VncState *vs = user_data;
 
-    if (ret < 0) {
-        if (!gnutls_error_is_fatal(ret)) {
-            VNC_DEBUG("Handshake interrupted (blocking)\n");
-            if (!gnutls_record_get_direction(vs->ws_tls.session)) {
-                qemu_set_fd_handler(vs->csock, vncws_tls_handshake_io,
-                                    NULL, vs);
-            } else {
-                qemu_set_fd_handler(vs->csock, NULL, vncws_tls_handshake_io,
-                                    vs);
-            }
-            return 0;
-        }
-        VNC_DEBUG("Handshake failed %s\n", gnutls_strerror(ret));
+    if (err) {
+        VNC_DEBUG("Handshake failed %s\n", error_get_pretty(err));
         vnc_client_error(vs);
-        return -1;
+    } else {
+        VNC_DEBUG("TLS handshake complete, starting websocket handshake\n");
+        vs->ioc_tag = qio_channel_add_watch(
+            QIO_CHANNEL(vs->ioc), G_IO_IN, vncws_handshake_io, vs, NULL);
     }
-
-    VNC_DEBUG("Handshake done, switching to TLS data mode\n");
-    vs->ws_tls.wiremode = VNC_WIREMODE_TLS;
-    qemu_set_fd_handler2(vs->csock, NULL, vncws_handshake_read, NULL, vs);
-
-    return 0;
 }
 
-static void vncws_tls_handshake_io(void *opaque)
-{
-    struct VncState *vs = (struct VncState *)opaque;
 
-    VNC_DEBUG("Handshake IO continue\n");
-    vncws_start_tls_handshake(vs);
-}
-
-void vncws_tls_handshake_peek(void *opaque)
+gboolean vncws_tls_handshake_io(QIOChannel *ioc G_GNUC_UNUSED,
+                                GIOCondition condition G_GNUC_UNUSED,
+                                void *opaque)
 {
     VncState *vs = opaque;
-    long ret;
+    QIOChannelTLS *tls;
+    Error *err = NULL;
 
-    if (!vs->ws_tls.session) {
-        char peek[4];
-        ret = qemu_recv(vs->csock, peek, sizeof(peek), MSG_PEEK);
-        if (ret && (strncmp(peek, "\x16", 1) == 0
-                    || strncmp(peek, "\x80", 1) == 0)) {
-            VNC_DEBUG("TLS Websocket connection recognized");
-            vnc_tls_client_setup(vs, 1);
-            vncws_start_tls_handshake(vs);
-        } else {
-            vncws_handshake_read(vs);
-        }
+    VNC_DEBUG("TLS Websocket connection required\n");
+    if (vs->ioc_tag) {
+        g_source_remove(vs->ioc_tag);
+        vs->ioc_tag = 0;
+    }
+
+    tls = qio_channel_tls_new_server(
+        vs->ioc,
+        vs->vd->tlscreds,
+        vs->vd->tlsaclname,
+        &err);
+    if (!tls) {
+        VNC_DEBUG("Failed to setup TLS %s\n", error_get_pretty(err));
+        error_free(err);
+        vnc_client_error(vs);
+        return TRUE;
+    }
+
+    VNC_DEBUG("Start TLS WS handshake process\n");
+    object_unref(OBJECT(vs->ioc));
+    vs->ioc = QIO_CHANNEL(tls);
+    vs->tls = qio_channel_tls_get_session(tls);
+
+    qio_channel_tls_handshake(tls,
+                              vncws_tls_handshake_done,
+                              vs,
+                              NULL);
+
+    return TRUE;
+}
+
+
+static void vncws_handshake_done(Object *source,
+                                 Error *err,
+                                 gpointer user_data)
+{
+    VncState *vs = user_data;
+
+    if (err) {
+        VNC_DEBUG("Websock handshake failed %s\n", error_get_pretty(err));
+        vnc_client_error(vs);
     } else {
-        qemu_set_fd_handler2(vs->csock, NULL, vncws_handshake_read, NULL, vs);
+        VNC_DEBUG("Websock handshake complete, starting VNC protocol\n");
+        vnc_init_state(vs);
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 }
-#endif /* CONFIG_VNC_TLS */
 
-void vncws_handshake_read(void *opaque)
+
+gboolean vncws_handshake_io(QIOChannel *ioc G_GNUC_UNUSED,
+                            GIOCondition condition G_GNUC_UNUSED,
+                            void *opaque)
 {
     VncState *vs = opaque;
-    uint8_t *handshake_end;
-    long ret;
-    buffer_reserve(&vs->ws_input, 4096);
-    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
+    QIOChannelWebsock *wioc;
 
-    if (!ret) {
-        if (vs->csock == -1) {
-            vnc_disconnect_finish(vs);
-        }
-        return;
-    }
-    vs->ws_input.offset += ret;
-
-    handshake_end = (uint8_t *)g_strstr_len((char *)vs->ws_input.buffer,
-            vs->ws_input.offset, WS_HANDSHAKE_END);
-    if (handshake_end) {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
-        vncws_process_handshake(vs, vs->ws_input.buffer, vs->ws_input.offset);
-        buffer_advance(&vs->ws_input, handshake_end - vs->ws_input.buffer +
-                strlen(WS_HANDSHAKE_END));
-    }
-}
-
-
-long vnc_client_read_ws(VncState *vs)
-{
-    int ret, err;
-    uint8_t *payload;
-    size_t payload_size, frame_size;
-    VNC_DEBUG("Read websocket %p size %zd offset %zd\n", vs->ws_input.buffer,
-            vs->ws_input.capacity, vs->ws_input.offset);
-    buffer_reserve(&vs->ws_input, 4096);
-    ret = vnc_client_read_buf(vs, buffer_end(&vs->ws_input), 4096);
-    if (!ret) {
-        return 0;
-    }
-    vs->ws_input.offset += ret;
-
-    /* make sure that nothing is left in the ws_input buffer */
-    do {
-        err = vncws_decode_frame(&vs->ws_input, &payload,
-                              &payload_size, &frame_size);
-        if (err <= 0) {
-            return err;
-        }
-
-        buffer_reserve(&vs->input, payload_size);
-        buffer_append(&vs->input, payload, payload_size);
-
-        buffer_advance(&vs->ws_input, frame_size);
-    } while (vs->ws_input.offset > 0);
-
-    return ret;
-}
-
-long vnc_client_write_ws(VncState *vs)
-{
-    long ret;
-    VNC_DEBUG("Write WS: Pending output %p size %zd offset %zd\n",
-              vs->output.buffer, vs->output.capacity, vs->output.offset);
-    vncws_encode_frame(&vs->ws_output, vs->output.buffer, vs->output.offset);
-    buffer_reset(&vs->output);
-    ret = vnc_client_write_buf(vs, vs->ws_output.buffer, vs->ws_output.offset);
-    if (!ret) {
-        return 0;
+    VNC_DEBUG("Websocket negotiate starting\n");
+    if (vs->ioc_tag) {
+        g_source_remove(vs->ioc_tag);
+        vs->ioc_tag = 0;
     }
 
-    buffer_advance(&vs->ws_output, ret);
+    wioc = qio_channel_websock_new_server(vs->ioc);
 
-    if (vs->ws_output.offset == 0) {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
-    }
+    object_unref(OBJECT(vs->ioc));
+    vs->ioc = QIO_CHANNEL(wioc);
 
-    return ret;
-}
+    qio_channel_websock_handshake(wioc,
+                                  vncws_handshake_done,
+                                  vs,
+                                  NULL);
 
-static char *vncws_extract_handshake_entry(const char *handshake,
-        size_t handshake_len, const char *name)
-{
-    char *begin, *end, *ret = NULL;
-    char *line = g_strdup_printf("%s%s: ", WS_HANDSHAKE_DELIM, name);
-    begin = g_strstr_len(handshake, handshake_len, line);
-    if (begin != NULL) {
-        begin += strlen(line);
-        end = g_strstr_len(begin, handshake_len - (begin - handshake),
-                WS_HANDSHAKE_DELIM);
-        if (end != NULL) {
-            ret = g_strndup(begin, end - begin);
-        }
-    }
-    g_free(line);
-    return ret;
-}
-
-static void vncws_send_handshake_response(VncState *vs, const char* key)
-{
-    char combined_key[WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1];
-    unsigned char hash[SHA1_DIGEST_LEN];
-    size_t hash_size = sizeof(hash);
-    char *accept = NULL, *response = NULL;
-    gnutls_datum_t in;
-    int ret;
-
-    g_strlcpy(combined_key, key, WS_CLIENT_KEY_LEN + 1);
-    g_strlcat(combined_key, WS_GUID, WS_CLIENT_KEY_LEN + WS_GUID_LEN + 1);
-
-    /* hash and encode it */
-    in.data = (void *)combined_key;
-    in.size = WS_CLIENT_KEY_LEN + WS_GUID_LEN;
-    ret = gnutls_fingerprint(GNUTLS_DIG_SHA1, &in, hash, &hash_size);
-    if (ret == GNUTLS_E_SUCCESS && hash_size <= SHA1_DIGEST_LEN) {
-        accept = g_base64_encode(hash, hash_size);
-    }
-    if (accept == NULL) {
-        VNC_DEBUG("Hashing Websocket combined key failed\n");
-        vnc_client_error(vs);
-        return;
-    }
-
-    response = g_strdup_printf(WS_HANDSHAKE, accept);
-    vnc_write(vs, response, strlen(response));
-    vnc_flush(vs);
-
-    g_free(accept);
-    g_free(response);
-
-    vs->encode_ws = 1;
-    vnc_init_state(vs);
-}
-
-void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size)
-{
-    char *protocols = vncws_extract_handshake_entry((const char *)line, size,
-            "Sec-WebSocket-Protocol");
-    char *version = vncws_extract_handshake_entry((const char *)line, size,
-            "Sec-WebSocket-Version");
-    char *key = vncws_extract_handshake_entry((const char *)line, size,
-            "Sec-WebSocket-Key");
-
-    if (protocols && version && key
-            && g_strrstr(protocols, "binary")
-            && !strcmp(version, WS_SUPPORTED_VERSION)
-            && strlen(key) == WS_CLIENT_KEY_LEN) {
-        vncws_send_handshake_response(vs, key);
-    } else {
-        VNC_DEBUG("Defective Websockets header or unsupported protocol\n");
-        vnc_client_error(vs);
-    }
-
-    g_free(protocols);
-    g_free(version);
-    g_free(key);
-}
-
-void vncws_encode_frame(Buffer *output, const void *payload,
-        const size_t payload_size)
-{
-    size_t header_size = 0;
-    unsigned char opcode = WS_OPCODE_BINARY_FRAME;
-    union {
-        char buf[WS_HEAD_MAX_LEN];
-        WsHeader ws;
-    } header;
-
-    if (!payload_size) {
-        return;
-    }
-
-    header.ws.b0 = 0x80 | (opcode & 0x0f);
-    if (payload_size <= 125) {
-        header.ws.b1 = (uint8_t)payload_size;
-        header_size = 2;
-    } else if (payload_size < 65536) {
-        header.ws.b1 = 0x7e;
-        header.ws.u.s16.l16 = cpu_to_be16((uint16_t)payload_size);
-        header_size = 4;
-    } else {
-        header.ws.b1 = 0x7f;
-        header.ws.u.s64.l64 = cpu_to_be64(payload_size);
-        header_size = 10;
-    }
-
-    buffer_reserve(output, header_size + payload_size);
-    buffer_append(output, header.buf, header_size);
-    buffer_append(output, payload, payload_size);
-}
-
-int vncws_decode_frame(Buffer *input, uint8_t **payload,
-                           size_t *payload_size, size_t *frame_size)
-{
-    unsigned char opcode = 0, fin = 0, has_mask = 0;
-    size_t header_size = 0;
-    uint32_t *payload32;
-    WsHeader *header = (WsHeader *)input->buffer;
-    WsMask mask;
-    int i;
-
-    if (input->offset < WS_HEAD_MIN_LEN + 4) {
-        /* header not complete */
-        return 0;
-    }
-
-    fin = (header->b0 & 0x80) >> 7;
-    opcode = header->b0 & 0x0f;
-    has_mask = (header->b1 & 0x80) >> 7;
-    *payload_size = header->b1 & 0x7f;
-
-    if (opcode == WS_OPCODE_CLOSE) {
-        /* disconnect */
-        return -1;
-    }
-
-    /* Websocket frame sanity check:
-     * * Websocket fragmentation is not supported.
-     * * All  websockets frames sent by a client have to be masked.
-     * * Only binary encoding is supported.
-     */
-    if (!fin || !has_mask || opcode != WS_OPCODE_BINARY_FRAME) {
-        VNC_DEBUG("Received faulty/unsupported Websocket frame\n");
-        return -2;
-    }
-
-    if (*payload_size < 126) {
-        header_size = 6;
-        mask = header->u.m;
-    } else if (*payload_size == 126 && input->offset >= 8) {
-        *payload_size = be16_to_cpu(header->u.s16.l16);
-        header_size = 8;
-        mask = header->u.s16.m16;
-    } else if (*payload_size == 127 && input->offset >= 14) {
-        *payload_size = be64_to_cpu(header->u.s64.l64);
-        header_size = 14;
-        mask = header->u.s64.m64;
-    } else {
-        /* header not complete */
-        return 0;
-    }
-
-    *frame_size = header_size + *payload_size;
-
-    if (input->offset < *frame_size) {
-        /* frame not complete */
-        return 0;
-    }
-
-    *payload = input->buffer + header_size;
-
-    /* unmask frame */
-    /* process 1 frame (32 bit op) */
-    payload32 = (uint32_t *)(*payload);
-    for (i = 0; i < *payload_size / 4; i++) {
-        payload32[i] ^= mask.u;
-    }
-    /* process the remaining bytes (if any) */
-    for (i *= 4; i < *payload_size; i++) {
-        (*payload)[i] ^= mask.c[i % 4];
-    }
-
-    return 1;
+    return TRUE;
 }
diff --git a/ui/vnc-ws.h b/ui/vnc-ws.h
index 95c1b0a..396cacf 100644
--- a/ui/vnc-ws.h
+++ b/ui/vnc-ws.h
@@ -18,72 +18,14 @@
  * along with this software; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef __QEMU_UI_VNC_WS_H
-#define __QEMU_UI_VNC_WS_H
+#ifndef QEMU_UI_VNC_WS_H
+#define QEMU_UI_VNC_WS_H
 
-#include <gnutls/gnutls.h>
+gboolean vncws_tls_handshake_io(QIOChannel *ioc,
+                                GIOCondition condition,
+                                void *opaque);
+gboolean vncws_handshake_io(QIOChannel *ioc,
+                            GIOCondition condition,
+                            void *opaque);
 
-#define B64LEN(__x) (((__x + 2) / 3) * 12 / 3)
-#define SHA1_DIGEST_LEN 20
-
-#define WS_ACCEPT_LEN (B64LEN(SHA1_DIGEST_LEN) + 1)
-#define WS_CLIENT_KEY_LEN 24
-#define WS_GUID "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
-#define WS_GUID_LEN strlen(WS_GUID)
-
-#define WS_HANDSHAKE "HTTP/1.1 101 Switching Protocols\r\n\
-Upgrade: websocket\r\n\
-Connection: Upgrade\r\n\
-Sec-WebSocket-Accept: %s\r\n\
-Sec-WebSocket-Protocol: binary\r\n\
-\r\n"
-#define WS_HANDSHAKE_DELIM "\r\n"
-#define WS_HANDSHAKE_END "\r\n\r\n"
-#define WS_SUPPORTED_VERSION "13"
-
-#define WS_HEAD_MIN_LEN sizeof(uint16_t)
-#define WS_HEAD_MAX_LEN (WS_HEAD_MIN_LEN + sizeof(uint64_t) + sizeof(uint32_t))
-
-typedef union WsMask {
-    char c[4];
-    uint32_t u;
-} WsMask;
-
-typedef struct QEMU_PACKED WsHeader {
-    unsigned char b0;
-    unsigned char b1;
-    union {
-        struct QEMU_PACKED {
-            uint16_t l16;
-            WsMask m16;
-        } s16;
-        struct QEMU_PACKED {
-            uint64_t l64;
-            WsMask m64;
-        } s64;
-        WsMask m;
-    } u;
-} WsHeader;
-
-enum {
-    WS_OPCODE_CONTINUATION = 0x0,
-    WS_OPCODE_TEXT_FRAME = 0x1,
-    WS_OPCODE_BINARY_FRAME = 0x2,
-    WS_OPCODE_CLOSE = 0x8,
-    WS_OPCODE_PING = 0x9,
-    WS_OPCODE_PONG = 0xA
-};
-
-#ifdef CONFIG_VNC_TLS
-void vncws_tls_handshake_peek(void *opaque);
-#endif /* CONFIG_VNC_TLS */
-void vncws_handshake_read(void *opaque);
-long vnc_client_write_ws(VncState *vs);
-long vnc_client_read_ws(VncState *vs);
-void vncws_process_handshake(VncState *vs, uint8_t *line, size_t size);
-void vncws_encode_frame(Buffer *output, const void *payload,
-            const size_t payload_size);
-int vncws_decode_frame(Buffer *input, uint8_t **payload,
-                               size_t *payload_size, size_t *frame_size);
-
-#endif /* __QEMU_UI_VNC_WS_H */
+#endif /* QEMU_UI_VNC_WS_H */
diff --git a/ui/vnc.c b/ui/vnc.c
index 5707015..d1087c9 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -24,18 +24,27 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "vnc.h"
 #include "vnc-jobs.h"
 #include "trace.h"
+#include "hw/qdev.h"
 #include "sysemu/sysemu.h"
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "qemu/timer.h"
 #include "qemu/acl.h"
+#include "qemu/config-file.h"
+#include "qapi/qmp/qerror.h"
 #include "qapi/qmp/types.h"
 #include "qmp-commands.h"
-#include "qemu/osdep.h"
 #include "ui/input.h"
 #include "qapi-event.h"
+#include "crypto/hash.h"
+#include "crypto/tlscredsanon.h"
+#include "crypto/tlscredsx509.h"
+#include "qom/object_interfaces.h"
+#include "qemu/cutils.h"
 
 #define VNC_REFRESH_INTERVAL_BASE GUI_REFRESH_INTERVAL_DEFAULT
 #define VNC_REFRESH_INTERVAL_INC  50
@@ -44,9 +53,10 @@
 static const struct timeval VNC_REFRESH_LOSSY = { 2, 0 };
 
 #include "vnc_keysym.h"
-#include "d3des.h"
+#include "crypto/cipher.h"
 
-static VncDisplay *vnc_display; /* needed for info vnc */
+static QTAILQ_HEAD(, VncDisplay) vnc_displays =
+    QTAILQ_HEAD_INITIALIZER(vnc_displays);
 
 static int vnc_cursor_define(VncState *vs);
 static void vnc_release_modifiers(VncState *vs);
@@ -61,118 +71,105 @@
         [VNC_SHARE_MODE_EXCLUSIVE]    = "exclusive",
         [VNC_SHARE_MODE_DISCONNECTED] = "disconnected",
     };
-    fprintf(stderr, "%s/%d: %s -> %s\n", __func__,
-            vs->csock, mn[vs->share_mode], mn[mode]);
+    fprintf(stderr, "%s/%p: %s -> %s\n", __func__,
+            vs->ioc, mn[vs->share_mode], mn[mode]);
 #endif
 
-    if (vs->share_mode == VNC_SHARE_MODE_EXCLUSIVE) {
+    switch (vs->share_mode) {
+    case VNC_SHARE_MODE_CONNECTING:
+        vs->vd->num_connecting--;
+        break;
+    case VNC_SHARE_MODE_SHARED:
+        vs->vd->num_shared--;
+        break;
+    case VNC_SHARE_MODE_EXCLUSIVE:
         vs->vd->num_exclusive--;
+        break;
+    default:
+        break;
     }
+
     vs->share_mode = mode;
-    if (vs->share_mode == VNC_SHARE_MODE_EXCLUSIVE) {
+
+    switch (vs->share_mode) {
+    case VNC_SHARE_MODE_CONNECTING:
+        vs->vd->num_connecting++;
+        break;
+    case VNC_SHARE_MODE_SHARED:
+        vs->vd->num_shared++;
+        break;
+    case VNC_SHARE_MODE_EXCLUSIVE:
         vs->vd->num_exclusive++;
+        break;
+    default:
+        break;
     }
 }
 
-static char *addr_to_string(const char *format,
-                            struct sockaddr_storage *sa,
-                            socklen_t salen) {
-    char *addr;
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
-    int err;
-    size_t addrlen;
 
-    if ((err = getnameinfo((struct sockaddr *)sa, salen,
-                           host, sizeof(host),
-                           serv, sizeof(serv),
-                           NI_NUMERICHOST | NI_NUMERICSERV)) != 0) {
-        VNC_DEBUG("Cannot resolve address %d: %s\n",
-                  err, gai_strerror(err));
-        return NULL;
-    }
-
-    /* Enough for the existing format + the 2 vars we're
-     * substituting in. */
-    addrlen = strlen(format) + strlen(host) + strlen(serv);
-    addr = g_malloc(addrlen + 1);
-    snprintf(addr, addrlen, format, host, serv);
-    addr[addrlen] = '\0';
-
-    return addr;
-}
-
-
-char *vnc_socket_local_addr(const char *format, int fd) {
-    struct sockaddr_storage sa;
-    socklen_t salen;
-
-    salen = sizeof(sa);
-    if (getsockname(fd, (struct sockaddr*)&sa, &salen) < 0)
-        return NULL;
-
-    return addr_to_string(format, &sa, salen);
-}
-
-char *vnc_socket_remote_addr(const char *format, int fd) {
-    struct sockaddr_storage sa;
-    socklen_t salen;
-
-    salen = sizeof(sa);
-    if (getpeername(fd, (struct sockaddr*)&sa, &salen) < 0)
-        return NULL;
-
-    return addr_to_string(format, &sa, salen);
-}
-
-static VncBasicInfo *vnc_basic_info_get(struct sockaddr_storage *sa,
-                                        socklen_t salen)
+static void vnc_init_basic_info(SocketAddress *addr,
+                                VncBasicInfo *info,
+                                Error **errp)
 {
-    VncBasicInfo *info;
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
-    int err;
+    switch (addr->type) {
+    case SOCKET_ADDRESS_KIND_INET:
+        info->host = g_strdup(addr->u.inet.data->host);
+        info->service = g_strdup(addr->u.inet.data->port);
+        if (addr->u.inet.data->ipv6) {
+            info->family = NETWORK_ADDRESS_FAMILY_IPV6;
+        } else {
+            info->family = NETWORK_ADDRESS_FAMILY_IPV4;
+        }
+        break;
 
-    if ((err = getnameinfo((struct sockaddr *)sa, salen,
-                           host, sizeof(host),
-                           serv, sizeof(serv),
-                           NI_NUMERICHOST | NI_NUMERICSERV)) != 0) {
-        VNC_DEBUG("Cannot resolve address %d: %s\n",
-                  err, gai_strerror(err));
-        return NULL;
+    case SOCKET_ADDRESS_KIND_UNIX:
+        info->host = g_strdup("");
+        info->service = g_strdup(addr->u.q_unix.data->path);
+        info->family = NETWORK_ADDRESS_FAMILY_UNIX;
+        break;
+
+    default:
+        error_setg(errp, "Unsupported socket kind %d",
+                   addr->type);
+        break;
     }
 
-    info = g_malloc0(sizeof(VncBasicInfo));
-    info->host = g_strdup(host);
-    info->service = g_strdup(serv);
-    info->family = inet_netfamily(sa->ss_family);
-    return info;
+    return;
 }
 
-static VncBasicInfo *vnc_basic_info_get_from_server_addr(int fd)
+static void vnc_init_basic_info_from_server_addr(QIOChannelSocket *ioc,
+                                                 VncBasicInfo *info,
+                                                 Error **errp)
 {
-    struct sockaddr_storage sa;
-    socklen_t salen;
+    SocketAddress *addr = NULL;
 
-    salen = sizeof(sa);
-    if (getsockname(fd, (struct sockaddr*)&sa, &salen) < 0) {
-        return NULL;
+    if (!ioc) {
+        error_setg(errp, "No listener socket available");
+        return;
     }
 
-    return vnc_basic_info_get(&sa, salen);
+    addr = qio_channel_socket_get_local_address(ioc, errp);
+    if (!addr) {
+        return;
+    }
+
+    vnc_init_basic_info(addr, info, errp);
+    qapi_free_SocketAddress(addr);
 }
 
-static VncBasicInfo *vnc_basic_info_get_from_remote_addr(int fd)
+static void vnc_init_basic_info_from_remote_addr(QIOChannelSocket *ioc,
+                                                 VncBasicInfo *info,
+                                                 Error **errp)
 {
-    struct sockaddr_storage sa;
-    socklen_t salen;
+    SocketAddress *addr = NULL;
 
-    salen = sizeof(sa);
-    if (getpeername(fd, (struct sockaddr*)&sa, &salen) < 0) {
-        return NULL;
+    addr = qio_channel_socket_get_remote_address(ioc, errp);
+    if (!addr) {
+        return;
     }
 
-    return vnc_basic_info_get(&sa, salen);
+    vnc_init_basic_info(addr, info, errp);
+    qapi_free_SocketAddress(addr);
 }
 
 static const char *vnc_auth_name(VncDisplay *vd) {
@@ -194,7 +191,6 @@
     case VNC_AUTH_TLS:
         return "tls";
     case VNC_AUTH_VENCRYPT:
-#ifdef CONFIG_VNC_TLS
         switch (vd->subauth) {
         case VNC_AUTH_VENCRYPT_PLAIN:
             return "vencrypt+plain";
@@ -217,27 +213,27 @@
         default:
             return "vencrypt";
         }
-#else
-        return "vencrypt";
-#endif
     case VNC_AUTH_SASL:
         return "sasl";
     }
     return "unknown";
 }
 
-static VncServerInfo *vnc_server_info_get(void)
+static VncServerInfo *vnc_server_info_get(VncDisplay *vd)
 {
     VncServerInfo *info;
-    VncBasicInfo *bi = vnc_basic_info_get_from_server_addr(vnc_display->lsock);
-    if (!bi) {
-        return NULL;
-    }
+    Error *err = NULL;
 
-    info = g_malloc(sizeof(*info));
-    info->base = bi;
+    info = g_malloc0(sizeof(*info));
+    vnc_init_basic_info_from_server_addr(vd->lsock,
+                                         qapi_VncServerInfo_base(info), &err);
     info->has_auth = true;
-    info->auth = g_strdup(vnc_auth_name(vnc_display));
+    info->auth = g_strdup(vnc_auth_name(vd));
+    if (err) {
+        qapi_free_VncServerInfo(info);
+        info = NULL;
+        error_free(err);
+    }
     return info;
 }
 
@@ -247,13 +243,12 @@
         return;
     }
 
-#ifdef CONFIG_VNC_TLS
-    if (client->tls.session &&
-        client->tls.dname) {
-        client->info->has_x509_dname = true;
-        client->info->x509_dname = g_strdup(client->tls.dname);
+    if (client->tls) {
+        client->info->x509_dname =
+            qcrypto_tls_session_get_peer_name(client->tls);
+        client->info->has_x509_dname =
+            client->info->x509_dname != NULL;
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (client->sasl.conn &&
         client->sasl.username) {
@@ -265,11 +260,16 @@
 
 static void vnc_client_cache_addr(VncState *client)
 {
-    VncBasicInfo *bi = vnc_basic_info_get_from_remote_addr(client->csock);
+    Error *err = NULL;
 
-    if (bi) {
-        client->info = g_malloc0(sizeof(*client->info));
-        client->info->base = bi;
+    client->info = g_malloc0(sizeof(*client->info));
+    vnc_init_basic_info_from_remote_addr(client->sioc,
+                                         qapi_VncClientInfo_base(client->info),
+                                         &err);
+    if (err) {
+        qapi_free_VncClientInfo(client->info);
+        client->info = NULL;
+        error_free(err);
     }
 }
 
@@ -280,16 +280,16 @@
     if (!vs->info) {
         return;
     }
-    g_assert(vs->info->base);
 
-    si = vnc_server_info_get();
+    si = vnc_server_info_get(vs->vd);
     if (!si) {
         return;
     }
 
     switch (event) {
     case QAPI_EVENT_VNC_CONNECTED:
-        qapi_event_send_vnc_connected(si, vs->info->base, &error_abort);
+        qapi_event_send_vnc_connected(si, qapi_VncClientInfo_base(vs->info),
+                                      &error_abort);
         break;
     case QAPI_EVENT_VNC_INITIALIZED:
         qapi_event_send_vnc_initialized(si, vs->info, &error_abort);
@@ -306,35 +306,26 @@
 
 static VncClientInfo *qmp_query_vnc_client(const VncState *client)
 {
-    struct sockaddr_storage sa;
-    socklen_t salen = sizeof(sa);
-    char host[NI_MAXHOST];
-    char serv[NI_MAXSERV];
     VncClientInfo *info;
-
-    if (getpeername(client->csock, (struct sockaddr *)&sa, &salen) < 0) {
-        return NULL;
-    }
-
-    if (getnameinfo((struct sockaddr *)&sa, salen,
-                    host, sizeof(host),
-                    serv, sizeof(serv),
-                    NI_NUMERICHOST | NI_NUMERICSERV) < 0) {
-        return NULL;
-    }
+    Error *err = NULL;
 
     info = g_malloc0(sizeof(*info));
-    info->base = g_malloc0(sizeof(*info->base));
-    info->base->host = g_strdup(host);
-    info->base->service = g_strdup(serv);
-    info->base->family = inet_netfamily(sa.ss_family);
 
-#ifdef CONFIG_VNC_TLS
-    if (client->tls.session && client->tls.dname) {
-        info->has_x509_dname = true;
-        info->x509_dname = g_strdup(client->tls.dname);
+    vnc_init_basic_info_from_remote_addr(client->sioc,
+                                         qapi_VncClientInfo_base(info),
+                                         &err);
+    if (err) {
+        error_free(err);
+        qapi_free_VncClientInfo(info);
+        return NULL;
     }
-#endif
+
+    info->websocket = client->websocket;
+
+    if (client->tls) {
+        info->x509_dname = qcrypto_tls_session_get_peer_name(client->tls);
+        info->has_x509_dname = info->x509_dname != NULL;
+    }
 #ifdef CONFIG_VNC_SASL
     if (client->sasl.conn && client->sasl.username) {
         info->has_sasl_username = true;
@@ -345,76 +336,232 @@
     return info;
 }
 
+static VncDisplay *vnc_display_find(const char *id)
+{
+    VncDisplay *vd;
+
+    if (id == NULL) {
+        return QTAILQ_FIRST(&vnc_displays);
+    }
+    QTAILQ_FOREACH(vd, &vnc_displays, next) {
+        if (strcmp(id, vd->id) == 0) {
+            return vd;
+        }
+    }
+    return NULL;
+}
+
+static VncClientInfoList *qmp_query_client_list(VncDisplay *vd)
+{
+    VncClientInfoList *cinfo, *prev = NULL;
+    VncState *client;
+
+    QTAILQ_FOREACH(client, &vd->clients, next) {
+        cinfo = g_new0(VncClientInfoList, 1);
+        cinfo->value = qmp_query_vnc_client(client);
+        cinfo->next = prev;
+        prev = cinfo;
+    }
+    return prev;
+}
+
 VncInfo *qmp_query_vnc(Error **errp)
 {
     VncInfo *info = g_malloc0(sizeof(*info));
+    VncDisplay *vd = vnc_display_find(NULL);
+    SocketAddress *addr = NULL;
 
-    if (vnc_display == NULL || vnc_display->display == NULL) {
+    if (vd == NULL || !vd->enabled) {
         info->enabled = false;
     } else {
-        VncClientInfoList *cur_item = NULL;
-        struct sockaddr_storage sa;
-        socklen_t salen = sizeof(sa);
-        char host[NI_MAXHOST];
-        char serv[NI_MAXSERV];
-        VncState *client;
-
         info->enabled = true;
 
         /* for compatibility with the original command */
         info->has_clients = true;
+        info->clients = qmp_query_client_list(vd);
 
-        QTAILQ_FOREACH(client, &vnc_display->clients, next) {
-            VncClientInfoList *cinfo = g_malloc0(sizeof(*info));
-            cinfo->value = qmp_query_vnc_client(client);
-
-            /* XXX: waiting for the qapi to support GSList */
-            if (!cur_item) {
-                info->clients = cur_item = cinfo;
-            } else {
-                cur_item->next = cinfo;
-                cur_item = cinfo;
-            }
-        }
-
-        if (vnc_display->lsock == -1) {
+        if (vd->lsock == NULL) {
             return info;
         }
 
-        if (getsockname(vnc_display->lsock, (struct sockaddr *)&sa,
-                        &salen) == -1) {
-            error_set(errp, QERR_UNDEFINED_ERROR);
+        addr = qio_channel_socket_get_local_address(vd->lsock, errp);
+        if (!addr) {
             goto out_error;
         }
 
-        if (getnameinfo((struct sockaddr *)&sa, salen,
-                        host, sizeof(host),
-                        serv, sizeof(serv),
-                        NI_NUMERICHOST | NI_NUMERICSERV) < 0) {
-            error_set(errp, QERR_UNDEFINED_ERROR);
+        switch (addr->type) {
+        case SOCKET_ADDRESS_KIND_INET:
+            info->host = g_strdup(addr->u.inet.data->host);
+            info->service = g_strdup(addr->u.inet.data->port);
+            if (addr->u.inet.data->ipv6) {
+                info->family = NETWORK_ADDRESS_FAMILY_IPV6;
+            } else {
+                info->family = NETWORK_ADDRESS_FAMILY_IPV4;
+            }
+            break;
+
+        case SOCKET_ADDRESS_KIND_UNIX:
+            info->host = g_strdup("");
+            info->service = g_strdup(addr->u.q_unix.data->path);
+            info->family = NETWORK_ADDRESS_FAMILY_UNIX;
+            break;
+
+        default:
+            error_setg(errp, "Unsupported socket kind %d",
+                       addr->type);
             goto out_error;
         }
 
         info->has_host = true;
-        info->host = g_strdup(host);
-
         info->has_service = true;
-        info->service = g_strdup(serv);
-
         info->has_family = true;
-        info->family = inet_netfamily(sa.ss_family);
 
         info->has_auth = true;
-        info->auth = g_strdup(vnc_auth_name(vnc_display));
+        info->auth = g_strdup(vnc_auth_name(vd));
     }
 
+    qapi_free_SocketAddress(addr);
     return info;
 
 out_error:
+    qapi_free_SocketAddress(addr);
     qapi_free_VncInfo(info);
     return NULL;
 }
 
+static VncBasicInfoList *qmp_query_server_entry(QIOChannelSocket *ioc,
+                                                bool websocket,
+                                                VncBasicInfoList *prev)
+{
+    VncBasicInfoList *list;
+    VncBasicInfo *info;
+    Error *err = NULL;
+    SocketAddress *addr;
+
+    addr = qio_channel_socket_get_local_address(ioc, &err);
+    if (!addr) {
+        error_free(err);
+        return prev;
+    }
+
+    info = g_new0(VncBasicInfo, 1);
+    vnc_init_basic_info(addr, info, &err);
+    qapi_free_SocketAddress(addr);
+    if (err) {
+        qapi_free_VncBasicInfo(info);
+        error_free(err);
+        return prev;
+    }
+    info->websocket = websocket;
+
+    list = g_new0(VncBasicInfoList, 1);
+    list->value = info;
+    list->next = prev;
+    return list;
+}
+
+static void qmp_query_auth(VncDisplay *vd, VncInfo2 *info)
+{
+    switch (vd->auth) {
+    case VNC_AUTH_VNC:
+        info->auth = VNC_PRIMARY_AUTH_VNC;
+        break;
+    case VNC_AUTH_RA2:
+        info->auth = VNC_PRIMARY_AUTH_RA2;
+        break;
+    case VNC_AUTH_RA2NE:
+        info->auth = VNC_PRIMARY_AUTH_RA2NE;
+        break;
+    case VNC_AUTH_TIGHT:
+        info->auth = VNC_PRIMARY_AUTH_TIGHT;
+        break;
+    case VNC_AUTH_ULTRA:
+        info->auth = VNC_PRIMARY_AUTH_ULTRA;
+        break;
+    case VNC_AUTH_TLS:
+        info->auth = VNC_PRIMARY_AUTH_TLS;
+        break;
+    case VNC_AUTH_VENCRYPT:
+        info->auth = VNC_PRIMARY_AUTH_VENCRYPT;
+        info->has_vencrypt = true;
+        switch (vd->subauth) {
+        case VNC_AUTH_VENCRYPT_PLAIN:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_PLAIN;
+            break;
+        case VNC_AUTH_VENCRYPT_TLSNONE:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_TLS_NONE;
+            break;
+        case VNC_AUTH_VENCRYPT_TLSVNC:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_TLS_VNC;
+            break;
+        case VNC_AUTH_VENCRYPT_TLSPLAIN:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_TLS_PLAIN;
+            break;
+        case VNC_AUTH_VENCRYPT_X509NONE:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_X509_NONE;
+            break;
+        case VNC_AUTH_VENCRYPT_X509VNC:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_X509_VNC;
+            break;
+        case VNC_AUTH_VENCRYPT_X509PLAIN:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_X509_PLAIN;
+            break;
+        case VNC_AUTH_VENCRYPT_TLSSASL:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_TLS_SASL;
+            break;
+        case VNC_AUTH_VENCRYPT_X509SASL:
+            info->vencrypt = VNC_VENCRYPT_SUB_AUTH_X509_SASL;
+            break;
+        default:
+            info->has_vencrypt = false;
+            break;
+        }
+        break;
+    case VNC_AUTH_SASL:
+        info->auth = VNC_PRIMARY_AUTH_SASL;
+        break;
+    case VNC_AUTH_NONE:
+    default:
+        info->auth = VNC_PRIMARY_AUTH_NONE;
+        break;
+    }
+}
+
+VncInfo2List *qmp_query_vnc_servers(Error **errp)
+{
+    VncInfo2List *item, *prev = NULL;
+    VncInfo2 *info;
+    VncDisplay *vd;
+    DeviceState *dev;
+
+    QTAILQ_FOREACH(vd, &vnc_displays, next) {
+        info = g_new0(VncInfo2, 1);
+        info->id = g_strdup(vd->id);
+        info->clients = qmp_query_client_list(vd);
+        qmp_query_auth(vd, info);
+        if (vd->dcl.con) {
+            dev = DEVICE(object_property_get_link(OBJECT(vd->dcl.con),
+                                                  "device", NULL));
+            info->has_display = true;
+            info->display = g_strdup(dev->id);
+        }
+        if (vd->lsock != NULL) {
+            info->server = qmp_query_server_entry(
+                vd->lsock, false, info->server);
+        }
+        if (vd->lwebsock != NULL) {
+            info->server = qmp_query_server_entry(
+                vd->lwebsock, true, info->server);
+        }
+
+        item = g_new0(VncInfo2List, 1);
+        item->value = info;
+        item->next = prev;
+        prev = item;
+    }
+    return prev;
+}
+
 /* TODO
    1) Get the queue working for IO.
    2) there is some weirdness when using the -S option (the screen is grey
@@ -432,10 +579,25 @@
 static void vnc_refresh(DisplayChangeListener *dcl);
 static int vnc_refresh_server_surface(VncDisplay *vd);
 
+static int vnc_width(VncDisplay *vd)
+{
+    return MIN(VNC_MAX_WIDTH, ROUND_UP(surface_width(vd->ds),
+                                       VNC_DIRTY_PIXELS_PER_BIT));
+}
+
+static int vnc_height(VncDisplay *vd)
+{
+    return MIN(VNC_MAX_HEIGHT, surface_height(vd->ds));
+}
+
 static void vnc_set_area_dirty(DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT],
                                VNC_MAX_WIDTH / VNC_DIRTY_PIXELS_PER_BIT),
-                               int width, int height,
-                               int x, int y, int w, int h) {
+                               VncDisplay *vd,
+                               int x, int y, int w, int h)
+{
+    int width = vnc_width(vd);
+    int height = vnc_height(vd);
+
     /* this is needed this to ensure we updated all affected
      * blocks if x % VNC_DIRTY_PIXELS_PER_BIT != 0 */
     w += (x % VNC_DIRTY_PIXELS_PER_BIT);
@@ -457,10 +619,8 @@
 {
     VncDisplay *vd = container_of(dcl, VncDisplay, dcl);
     struct VncSurface *s = &vd->guest;
-    int width = pixman_image_get_width(vd->server);
-    int height = pixman_image_get_height(vd->server);
 
-    vnc_set_area_dirty(s->dirty, width, height, x, y, w, h);
+    vnc_set_area_dirty(s->dirty, vd, x, y, w, h);
 }
 
 void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
@@ -474,57 +634,10 @@
     vnc_write_s32(vs, encoding);
 }
 
-void buffer_reserve(Buffer *buffer, size_t len)
-{
-    if ((buffer->capacity - buffer->offset) < len) {
-        buffer->capacity += (len + 1024);
-        buffer->buffer = g_realloc(buffer->buffer, buffer->capacity);
-        if (buffer->buffer == NULL) {
-            fprintf(stderr, "vnc: out of memory\n");
-            exit(1);
-        }
-    }
-}
-
-static int buffer_empty(Buffer *buffer)
-{
-    return buffer->offset == 0;
-}
-
-uint8_t *buffer_end(Buffer *buffer)
-{
-    return buffer->buffer + buffer->offset;
-}
-
-void buffer_reset(Buffer *buffer)
-{
-        buffer->offset = 0;
-}
-
-void buffer_free(Buffer *buffer)
-{
-    g_free(buffer->buffer);
-    buffer->offset = 0;
-    buffer->capacity = 0;
-    buffer->buffer = NULL;
-}
-
-void buffer_append(Buffer *buffer, const void *data, size_t len)
-{
-    memcpy(buffer->buffer + buffer->offset, data, len);
-    buffer->offset += len;
-}
-
-void buffer_advance(Buffer *buf, size_t len)
-{
-    memmove(buf->buffer, buf->buffer + len,
-            (buf->offset - len));
-    buf->offset -= len;
-}
 
 static void vnc_desktop_resize(VncState *vs)
 {
-    if (vs->csock == -1 || !vnc_has_feature(vs, VNC_FEATURE_RESIZE)) {
+    if (vs->ioc == NULL || !vnc_has_feature(vs, VNC_FEATURE_RESIZE)) {
         return;
     }
     if (vs->client_width == pixman_image_get_width(vs->vd->server) &&
@@ -577,35 +690,44 @@
     return ptr;
 }
 
+static void vnc_update_server_surface(VncDisplay *vd)
+{
+    int width, height;
+
+    qemu_pixman_image_unref(vd->server);
+    vd->server = NULL;
+
+    if (QTAILQ_EMPTY(&vd->clients)) {
+        return;
+    }
+
+    width = vnc_width(vd);
+    height = vnc_height(vd);
+    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
+                                          width, height,
+                                          NULL, 0);
+
+    memset(vd->guest.dirty, 0x00, sizeof(vd->guest.dirty));
+    vnc_set_area_dirty(vd->guest.dirty, vd, 0, 0,
+                       width, height);
+}
+
 static void vnc_dpy_switch(DisplayChangeListener *dcl,
                            DisplaySurface *surface)
 {
     VncDisplay *vd = container_of(dcl, VncDisplay, dcl);
     VncState *vs;
-    int width, height;
 
     vnc_abort_display_jobs(vd);
+    vd->ds = surface;
 
     /* server surface */
-    qemu_pixman_image_unref(vd->server);
-    vd->ds = surface;
-    width = MIN(VNC_MAX_WIDTH, ROUND_UP(surface_width(vd->ds),
-                                        VNC_DIRTY_PIXELS_PER_BIT));
-    height = MIN(VNC_MAX_HEIGHT, surface_height(vd->ds));
-    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
-                                          width, height, NULL, 0);
+    vnc_update_server_surface(vd);
 
     /* guest surface */
-#if 0 /* FIXME */
-    if (ds_get_bytes_per_pixel(ds) != vd->guest.ds->pf.bytes_per_pixel)
-        console_color_init(ds);
-#endif
     qemu_pixman_image_unref(vd->guest.fb);
     vd->guest.fb = pixman_image_ref(surface->image);
     vd->guest.format = surface->format;
-    memset(vd->guest.dirty, 0x00, sizeof(vd->guest.dirty));
-    vnc_set_area_dirty(vd->guest.dirty, width, height, 0, 0,
-                       width, height);
 
     QTAILQ_FOREACH(vs, &vd->clients, next) {
         vnc_colordepth(vs);
@@ -614,8 +736,9 @@
             vnc_cursor_define(vs);
         }
         memset(vs->dirty, 0x00, sizeof(vs->dirty));
-        vnc_set_area_dirty(vs->dirty, width, height, 0, 0,
-                           width, height);
+        vnc_set_area_dirty(vs->dirty, vd, 0, 0,
+                           vnc_width(vd),
+                           vnc_height(vd));
     }
 }
 
@@ -704,6 +827,8 @@
 int vnc_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
 {
     int n = 0;
+    bool encode_raw = false;
+    size_t saved_offs = vs->output.offset;
 
     switch(vs->vnc_encoding) {
         case VNC_ENCODING_ZLIB:
@@ -726,10 +851,24 @@
             n = vnc_zywrle_send_framebuffer_update(vs, x, y, w, h);
             break;
         default:
-            vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
-            n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+            encode_raw = true;
             break;
     }
+
+    /* If the client has the same pixel format as our internal buffer and
+     * a RAW encoding would need less space fall back to RAW encoding to
+     * save bandwidth and processing power in the client. */
+    if (!encode_raw && vs->write_pixels == vnc_write_pixels_copy &&
+        12 + h * w * VNC_SERVER_FB_BYTES <= (vs->output.offset - saved_offs)) {
+        vs->output.offset = saved_offs;
+        encode_raw = true;
+    }
+
+    if (encode_raw) {
+        vnc_framebuffer_update(vs, x, y, w, h, VNC_ENCODING_RAW);
+        n = vnc_raw_send_framebuffer_update(vs, x, y, w, h);
+    }
+
     return n;
 }
 
@@ -758,6 +897,11 @@
     int i, x, y, pitch, inc, w_lim, s;
     int cmp_bytes;
 
+    if (!vd->server) {
+        /* no client connected */
+        return;
+    }
+
     vnc_refresh_server_surface(vd);
     QTAILQ_FOREACH_SAFE(vs, &vd->clients, next, vn) {
         if (vnc_has_feature(vs, VNC_FEATURE_COPYRECT)) {
@@ -853,7 +997,7 @@
 static void vnc_dpy_cursor_define(DisplayChangeListener *dcl,
                                   QEMUCursor *c)
 {
-    VncDisplay *vd = vnc_display;
+    VncDisplay *vd = container_of(dcl, VncDisplay, dcl);
     VncState *vs;
 
     cursor_put(vd->cursor);
@@ -870,7 +1014,7 @@
     }
 }
 
-static int find_and_clear_dirty_height(struct VncState *vs,
+static int find_and_clear_dirty_height(VncState *vs,
                                        int y, int last_x, int x, int height)
 {
     int h;
@@ -887,8 +1031,13 @@
 
 static int vnc_update_client(VncState *vs, int has_dirty, bool sync)
 {
+    if (vs->disconnecting) {
+        vnc_disconnect_finish(vs);
+        return 0;
+    }
+
     vs->has_dirty += has_dirty;
-    if (vs->need_update && vs->csock != -1) {
+    if (vs->need_update && !vs->disconnecting) {
         VncDisplay *vd = vs->vd;
         VncJob *job;
         int y;
@@ -935,6 +1084,12 @@
                 n += vnc_job_add_rect(job, x * VNC_DIRTY_PIXELS_PER_BIT, y,
                                       (x2 - x) * VNC_DIRTY_PIXELS_PER_BIT, h);
             }
+            if (!x && x2 == width / VNC_DIRTY_PIXELS_PER_BIT) {
+                y += h;
+                if (y == height) {
+                    break;
+                }
+            }
         }
 
         vnc_job_push(job);
@@ -946,7 +1101,7 @@
         return n;
     }
 
-    if (vs->csock == -1) {
+    if (vs->disconnecting) {
         vnc_disconnect_finish(vs);
     } else if (sync) {
         vnc_jobs_join(vs);
@@ -1028,12 +1183,15 @@
 
 static void vnc_disconnect_start(VncState *vs)
 {
-    if (vs->csock == -1)
+    if (vs->disconnecting) {
         return;
+    }
     vnc_set_share_mode(vs, VNC_SHARE_MODE_DISCONNECTED);
-    qemu_set_fd_handler2(vs->csock, NULL, NULL, NULL, NULL);
-    closesocket(vs->csock);
-    vs->csock = -1;
+    if (vs->ioc_tag) {
+        g_source_remove(vs->ioc_tag);
+    }
+    qio_channel_close(vs->ioc, NULL);
+    vs->disconnecting = TRUE;
 }
 
 void vnc_disconnect_finish(VncState *vs)
@@ -1047,10 +1205,6 @@
 
     buffer_free(&vs->input);
     buffer_free(&vs->output);
-#ifdef CONFIG_VNC_WS
-    buffer_free(&vs->ws_input);
-    buffer_free(&vs->ws_output);
-#endif /* CONFIG_VNC_WS */
 
     qapi_free_VncClientInfo(vs->info);
 
@@ -1058,9 +1212,6 @@
     vnc_tight_clear(vs);
     vnc_zrle_clear(vs);
 
-#ifdef CONFIG_VNC_TLS
-    vnc_tls_client_cleanup(vs);
-#endif /* CONFIG_VNC_TLS */
 #ifdef CONFIG_VNC_SASL
     vnc_sasl_client_cleanup(vs);
 #endif /* CONFIG_VNC_SASL */
@@ -1070,6 +1221,10 @@
     if (vs->initialized) {
         QTAILQ_REMOVE(&vs->vd->clients, vs, next);
         qemu_remove_mouse_mode_change_notifier(&vs->mouse_mode_notifier);
+        if (QTAILQ_EMPTY(&vs->vd->clients)) {
+            /* last client gone */
+            vnc_update_server_surface(vs->vd);
+        }
     }
 
     if (vs->vd->lock_key_sync)
@@ -1086,29 +1241,29 @@
         g_free(vs->lossy_rect[i]);
     }
     g_free(vs->lossy_rect);
+
+    object_unref(OBJECT(vs->ioc));
+    vs->ioc = NULL;
+    object_unref(OBJECT(vs->sioc));
+    vs->sioc = NULL;
     g_free(vs);
 }
 
-int vnc_client_io_error(VncState *vs, int ret, int last_errno)
+ssize_t vnc_client_io_error(VncState *vs, ssize_t ret, Error **errp)
 {
-    if (ret == 0 || ret == -1) {
-        if (ret == -1) {
-            switch (last_errno) {
-                case EINTR:
-                case EAGAIN:
-#ifdef _WIN32
-                case WSAEWOULDBLOCK:
-#endif
-                    return 0;
-                default:
-                    break;
-            }
+    if (ret <= 0) {
+        if (ret == 0) {
+            VNC_DEBUG("Closing down client sock: EOF\n");
+        } else if (ret != QIO_CHANNEL_ERR_BLOCK) {
+            VNC_DEBUG("Closing down client sock: ret %d (%s)\n",
+                      ret, errp ? error_get_pretty(*errp) : "Unknown");
         }
 
-        VNC_DEBUG("Closing down client sock: ret %d, errno %d\n",
-                  ret, ret < 0 ? last_errno : 0);
         vnc_disconnect_start(vs);
-
+        if (errp) {
+            error_free(*errp);
+            *errp = NULL;
+        }
         return 0;
     }
     return ret;
@@ -1121,23 +1276,6 @@
     vnc_disconnect_start(vs);
 }
 
-#ifdef CONFIG_VNC_TLS
-static long vnc_client_write_tls(gnutls_session_t *session,
-                                 const uint8_t *data,
-                                 size_t datalen)
-{
-    long ret = gnutls_write(*session, data, datalen);
-    if (ret < 0) {
-        if (ret == GNUTLS_E_AGAIN) {
-            errno = EAGAIN;
-        } else {
-            errno = EIO;
-        }
-        ret = -1;
-    }
-    return ret;
-}
-#endif /* CONFIG_VNC_TLS */
 
 /*
  * Called to write a chunk of data to the client socket. The data may
@@ -1154,27 +1292,14 @@
  * the requested 'datalen' if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
+ssize_t vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen)
 {
-    long ret;
-#ifdef CONFIG_VNC_TLS
-    if (vs->tls.session) {
-        ret = vnc_client_write_tls(&vs->tls.session, data, datalen);
-    } else {
-#ifdef CONFIG_VNC_WS
-        if (vs->ws_tls.session) {
-            ret = vnc_client_write_tls(&vs->ws_tls.session, data, datalen);
-        } else
-#endif /* CONFIG_VNC_WS */
-#endif /* CONFIG_VNC_TLS */
-        {
-            ret = send(vs->csock, (const void *)data, datalen, 0);
-        }
-#ifdef CONFIG_VNC_TLS
-    }
-#endif /* CONFIG_VNC_TLS */
+    Error *err = NULL;
+    ssize_t ret;
+    ret = qio_channel_write(
+        vs->ioc, (const char *)data, datalen, &err);
     VNC_DEBUG("Wrote wire %p %zd -> %ld\n", data, datalen, ret);
-    return vnc_client_io_error(vs, ret, socket_error());
+    return vnc_client_io_error(vs, ret, &err);
 }
 
 
@@ -1188,9 +1313,9 @@
  * the buffered output data if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-static long vnc_client_write_plain(VncState *vs)
+static ssize_t vnc_client_write_plain(VncState *vs)
 {
-    long ret;
+    ssize_t ret;
 
 #ifdef CONFIG_VNC_SASL
     VNC_DEBUG("Write Plain: Pending output %p size %zd offset %zd. Wait SSF %d\n",
@@ -1212,7 +1337,11 @@
     buffer_advance(&vs->output, ret);
 
     if (vs->output.offset == 0) {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 
     return ret;
@@ -1224,10 +1353,8 @@
  * the client socket. Will delegate actual work according to whether
  * SASL SSF layers are enabled (thus requiring encryption calls)
  */
-static void vnc_client_write_locked(void *opaque)
+static void vnc_client_write_locked(VncState *vs)
 {
-    VncState *vs = opaque;
-
 #ifdef CONFIG_VNC_SASL
     if (vs->sasl.conn &&
         vs->sasl.runSSF &&
@@ -1236,30 +1363,22 @@
     } else
 #endif /* CONFIG_VNC_SASL */
     {
-#ifdef CONFIG_VNC_WS
-        if (vs->encode_ws) {
-            vnc_client_write_ws(vs);
-        } else
-#endif /* CONFIG_VNC_WS */
-        {
-            vnc_client_write_plain(vs);
-        }
+        vnc_client_write_plain(vs);
     }
 }
 
-void vnc_client_write(void *opaque)
+static void vnc_client_write(VncState *vs)
 {
-    VncState *vs = opaque;
 
     vnc_lock_output(vs);
-    if (vs->output.offset
-#ifdef CONFIG_VNC_WS
-            || vs->ws_output.offset
-#endif
-            ) {
-        vnc_client_write_locked(opaque);
-    } else if (vs->csock != -1) {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+    if (vs->output.offset) {
+        vnc_client_write_locked(vs);
+    } else if (vs->ioc != NULL) {
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
     vnc_unlock_output(vs);
 }
@@ -1270,22 +1389,6 @@
     vs->read_handler_expect = expecting;
 }
 
-#ifdef CONFIG_VNC_TLS
-static long vnc_client_read_tls(gnutls_session_t *session, uint8_t *data,
-                                size_t datalen)
-{
-    long ret = gnutls_read(*session, data, datalen);
-    if (ret < 0) {
-        if (ret == GNUTLS_E_AGAIN) {
-            errno = EAGAIN;
-        } else {
-            errno = EIO;
-        }
-        ret = -1;
-    }
-    return ret;
-}
-#endif /* CONFIG_VNC_TLS */
 
 /*
  * Called to read a chunk of data from the client socket. The data may
@@ -1302,27 +1405,14 @@
  * the requested 'datalen' if the socket would block. Returns
  * -1 on error, and disconnects the client socket.
  */
-long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
+ssize_t vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen)
 {
-    long ret;
-#ifdef CONFIG_VNC_TLS
-    if (vs->tls.session) {
-        ret = vnc_client_read_tls(&vs->tls.session, data, datalen);
-    } else {
-#ifdef CONFIG_VNC_WS
-        if (vs->ws_tls.session) {
-            ret = vnc_client_read_tls(&vs->ws_tls.session, data, datalen);
-        } else
-#endif /* CONFIG_VNC_WS */
-#endif /* CONFIG_VNC_TLS */
-        {
-            ret = qemu_recv(vs->csock, data, datalen, 0);
-        }
-#ifdef CONFIG_VNC_TLS
-    }
-#endif /* CONFIG_VNC_TLS */
+    ssize_t ret;
+    Error *err = NULL;
+    ret = qio_channel_read(
+        vs->ioc, (char *)data, datalen, &err);
     VNC_DEBUG("Read wire %p %zd -> %ld\n", data, datalen, ret);
-    return vnc_client_io_error(vs, ret, socket_error());
+    return vnc_client_io_error(vs, ret, &err);
 }
 
 
@@ -1334,9 +1424,9 @@
  * Returns the number of bytes read. Returns -1 on error, and
  * disconnects the client socket.
  */
-static long vnc_client_read_plain(VncState *vs)
+static ssize_t vnc_client_read_plain(VncState *vs)
 {
-    int ret;
+    ssize_t ret;
     VNC_DEBUG("Read plain %p size %zd offset %zd\n",
               vs->input.buffer, vs->input.capacity, vs->input.offset);
     buffer_reserve(&vs->input, 4096);
@@ -1358,36 +1448,24 @@
  * First function called whenever there is more data to be read from
  * the client socket. Will delegate actual work according to whether
  * SASL SSF layers are enabled (thus requiring decryption calls)
+ * Returns 0 on success, -1 if client disconnected
  */
-void vnc_client_read(void *opaque)
+static int vnc_client_read(VncState *vs)
 {
-    VncState *vs = opaque;
-    long ret;
+    ssize_t ret;
 
 #ifdef CONFIG_VNC_SASL
     if (vs->sasl.conn && vs->sasl.runSSF)
         ret = vnc_client_read_sasl(vs);
     else
 #endif /* CONFIG_VNC_SASL */
-#ifdef CONFIG_VNC_WS
-        if (vs->encode_ws) {
-            ret = vnc_client_read_ws(vs);
-            if (ret == -1) {
-                vnc_disconnect_start(vs);
-                return;
-            } else if (ret == -2) {
-                vnc_client_error(vs);
-                return;
-            }
-        } else
-#endif /* CONFIG_VNC_WS */
-        {
         ret = vnc_client_read_plain(vs);
-        }
     if (!ret) {
-        if (vs->csock == -1)
+        if (vs->disconnecting) {
             vnc_disconnect_finish(vs);
-        return;
+            return -1;
+        }
+        return 0;
     }
 
     while (vs->read_handler && vs->input.offset >= vs->read_handler_expect) {
@@ -1395,9 +1473,9 @@
         int ret;
 
         ret = vs->read_handler(vs, vs->input.buffer, len);
-        if (vs->csock == -1) {
+        if (vs->disconnecting) {
             vnc_disconnect_finish(vs);
-            return;
+            return -1;
         }
 
         if (!ret) {
@@ -1406,14 +1484,35 @@
             vs->read_handler_expect = ret;
         }
     }
+    return 0;
 }
 
+gboolean vnc_client_io(QIOChannel *ioc G_GNUC_UNUSED,
+                       GIOCondition condition, void *opaque)
+{
+    VncState *vs = opaque;
+    if (condition & G_IO_IN) {
+        if (vnc_client_read(vs) < 0) {
+            return TRUE;
+        }
+    }
+    if (condition & G_IO_OUT) {
+        vnc_client_write(vs);
+    }
+    return TRUE;
+}
+
+
 void vnc_write(VncState *vs, const void *data, size_t len)
 {
     buffer_reserve(&vs->output, len);
 
-    if (vs->csock != -1 && buffer_empty(&vs->output)) {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, vnc_client_write, vs);
+    if (vs->ioc != NULL && buffer_empty(&vs->output)) {
+        if (vs->ioc_tag) {
+            g_source_remove(vs->ioc_tag);
+        }
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN | G_IO_OUT, vnc_client_io, vs, NULL);
     }
 
     buffer_append(&vs->output, data, len);
@@ -1454,11 +1553,7 @@
 void vnc_flush(VncState *vs)
 {
     vnc_lock_output(vs);
-    if (vs->csock != -1 && (vs->output.offset
-#ifdef CONFIG_VNC_WS
-                || vs->ws_output.offset
-#endif
-                )) {
+    if (vs->ioc != NULL && vs->output.offset) {
         vnc_client_write_locked(vs);
     }
     vnc_unlock_output(vs);
@@ -1512,7 +1607,7 @@
 
 static void pointer_event(VncState *vs, int button_mask, int x, int y)
 {
-    static uint32_t bmap[INPUT_BUTTON_MAX] = {
+    static uint32_t bmap[INPUT_BUTTON__MAX] = {
         [INPUT_BUTTON_LEFT]       = 0x01,
         [INPUT_BUTTON_MIDDLE]     = 0x02,
         [INPUT_BUTTON_RIGHT]      = 0x04,
@@ -1551,6 +1646,7 @@
     for(i = 0; i < 256; i++) {
         if (vs->modifiers_state[i]) {
             qemu_input_event_send_key_number(vs->vd->dcl.con, i, false);
+            qemu_input_event_send_key_delay(vs->vd->key_delay_ms);
             vs->modifiers_state[i] = 0;
         }
     }
@@ -1560,9 +1656,9 @@
 {
     int keycode = keysym2scancode(vs->vd->kbd_layout, keysym) & SCANCODE_KEYMASK;
     qemu_input_event_send_key_number(vs->vd->dcl.con, keycode, true);
-    qemu_input_event_send_key_delay(0);
+    qemu_input_event_send_key_delay(vs->vd->key_delay_ms);
     qemu_input_event_send_key_number(vs->vd->dcl.con, keycode, false);
-    qemu_input_event_send_key_delay(0);
+    qemu_input_event_send_key_delay(vs->vd->key_delay_ms);
 }
 
 static int current_led_state(VncState *vs)
@@ -1647,7 +1743,8 @@
             vs->modifiers_state[keycode] = 0;
         break;
     case 0x02 ... 0x0a: /* '1' to '9' keys */
-        if (down && vs->modifiers_state[0x1d] && vs->modifiers_state[0x38]) {
+        if (vs->vd->dcl.con == NULL &&
+            down && vs->modifiers_state[0x1d] && vs->modifiers_state[0x38]) {
             /* Reset the modifiers sent to the current console */
             reset_keys(vs);
             console_select(keycode - 0x02);
@@ -1713,6 +1810,7 @@
 
     if (qemu_console_is_graphic(NULL)) {
         qemu_input_event_send_key_number(vs->vd->dcl.con, keycode, down);
+        qemu_input_event_send_key_delay(vs->vd->key_delay_ms);
     } else {
         bool numlock = vs->modifiers_state[0x45];
         bool control = (vs->modifiers_state[0x1d] ||
@@ -1834,6 +1932,7 @@
             continue;
         }
         qemu_input_event_send_key_number(vs->vd->dcl.con, keycode, false);
+        qemu_input_event_send_key_delay(vs->vd->key_delay_ms);
     }
 }
 
@@ -1871,9 +1970,6 @@
 static void framebuffer_update_request(VncState *vs, int incremental,
                                        int x, int y, int w, int h)
 {
-    int width = pixman_image_get_width(vs->vd->server);
-    int height = pixman_image_get_height(vs->vd->server);
-
     vs->need_update = 1;
 
     if (incremental) {
@@ -1881,7 +1977,7 @@
     }
 
     vs->force_update = 1;
-    vnc_set_area_dirty(vs->dirty, width, height, x, y, w, h);
+    vnc_set_area_dirty(vs->dirty, vs->vd, x, y, w, h);
 }
 
 static void send_ext_key_event_ack(VncState *vs)
@@ -1971,6 +2067,9 @@
             break;
         case VNC_ENCODING_RICH_CURSOR:
             vs->features |= VNC_FEATURE_RICH_CURSOR_MASK;
+            if (vs->vd->cursor) {
+                vnc_cursor_define(vs);
+            }
             break;
         case VNC_ENCODING_EXT_KEY_EVENT:
             send_ext_key_event_ack(vs);
@@ -2015,15 +2114,38 @@
     }
 }
 
-static void set_pixel_format(VncState *vs,
-                             int bits_per_pixel, int depth,
+static void send_color_map(VncState *vs)
+{
+    int i;
+
+    vnc_write_u8(vs, VNC_MSG_SERVER_SET_COLOUR_MAP_ENTRIES);
+    vnc_write_u8(vs,  0);    /* padding     */
+    vnc_write_u16(vs, 0);    /* first color */
+    vnc_write_u16(vs, 256);  /* # of colors */
+
+    for (i = 0; i < 256; i++) {
+        PixelFormat *pf = &vs->client_pf;
+
+        vnc_write_u16(vs, (((i >> pf->rshift) & pf->rmax) << (16 - pf->rbits)));
+        vnc_write_u16(vs, (((i >> pf->gshift) & pf->gmax) << (16 - pf->gbits)));
+        vnc_write_u16(vs, (((i >> pf->bshift) & pf->bmax) << (16 - pf->bbits)));
+    }
+}
+
+static void set_pixel_format(VncState *vs, int bits_per_pixel,
                              int big_endian_flag, int true_color_flag,
                              int red_max, int green_max, int blue_max,
                              int red_shift, int green_shift, int blue_shift)
 {
     if (!true_color_flag) {
-        vnc_client_error(vs);
-        return;
+        /* Expose a reasonable default 256 color map */
+        bits_per_pixel = 8;
+        red_max = 7;
+        green_max = 7;
+        blue_max = 3;
+        red_shift = 0;
+        green_shift = 3;
+        blue_shift = 6;
     }
 
     switch (bits_per_pixel) {
@@ -2036,15 +2158,15 @@
         return;
     }
 
-    vs->client_pf.rmax = red_max;
+    vs->client_pf.rmax = red_max ? red_max : 0xFF;
     vs->client_pf.rbits = hweight_long(red_max);
     vs->client_pf.rshift = red_shift;
     vs->client_pf.rmask = red_max << red_shift;
-    vs->client_pf.gmax = green_max;
+    vs->client_pf.gmax = green_max ? green_max : 0xFF;
     vs->client_pf.gbits = hweight_long(green_max);
     vs->client_pf.gshift = green_shift;
     vs->client_pf.gmask = green_max << green_shift;
-    vs->client_pf.bmax = blue_max;
+    vs->client_pf.bmax = blue_max ? blue_max : 0xFF;
     vs->client_pf.bbits = hweight_long(blue_max);
     vs->client_pf.bshift = blue_shift;
     vs->client_pf.bmask = blue_max << blue_shift;
@@ -2053,10 +2175,14 @@
     vs->client_pf.depth = bits_per_pixel == 32 ? 24 : bits_per_pixel;
     vs->client_be = big_endian_flag;
 
+    if (!true_color_flag) {
+        send_color_map(vs);
+    }
+
     set_pixel_conversion(vs);
 
-    graphic_hw_invalidate(NULL);
-    graphic_hw_update(NULL);
+    graphic_hw_invalidate(vs->vd->dcl.con);
+    graphic_hw_update(vs->vd->dcl.con);
 }
 
 static void pixel_format_message (VncState *vs) {
@@ -2120,7 +2246,7 @@
         if (len == 1)
             return 20;
 
-        set_pixel_format(vs, read_u8(data, 4), read_u8(data, 5),
+        set_pixel_format(vs, read_u8(data, 4),
                          read_u8(data, 6), read_u8(data, 7),
                          read_u16(data, 8), read_u16(data, 10),
                          read_u16(data, 12), read_u8(data, 14),
@@ -2217,34 +2343,34 @@
                 case 4: vs->as.fmt = AUD_FMT_U32; break;
                 case 5: vs->as.fmt = AUD_FMT_S32; break;
                 default:
-                    printf("Invalid audio format %d\n", read_u8(data, 4));
+                    VNC_DEBUG("Invalid audio format %d\n", read_u8(data, 4));
                     vnc_client_error(vs);
                     break;
                 }
                 vs->as.nchannels = read_u8(data, 5);
                 if (vs->as.nchannels != 1 && vs->as.nchannels != 2) {
-                    printf("Invalid audio channel coount %d\n",
-                           read_u8(data, 5));
+                    VNC_DEBUG("Invalid audio channel coount %d\n",
+                              read_u8(data, 5));
                     vnc_client_error(vs);
                     break;
                 }
                 vs->as.freq = read_u32(data, 6);
                 break;
             default:
-                printf ("Invalid audio message %d\n", read_u8(data, 4));
+                VNC_DEBUG("Invalid audio message %d\n", read_u8(data, 4));
                 vnc_client_error(vs);
                 break;
             }
             break;
 
         default:
-            printf("Msg: %d\n", read_u16(data, 0));
+            VNC_DEBUG("Msg: %d\n", read_u16(data, 0));
             vnc_client_error(vs);
             break;
         }
         break;
     default:
-        printf("Msg: %d\n", data[0]);
+        VNC_DEBUG("Msg: %d\n", data[0]);
         vnc_client_error(vs);
         break;
     }
@@ -2317,6 +2443,11 @@
     }
     vnc_set_share_mode(vs, mode);
 
+    if (vs->vd->num_shared > vs->vd->connections_limit) {
+        vnc_disconnect_start(vs);
+        return 0;
+    }
+
     vs->client_width = pixman_image_get_width(vs->vd->server);
     vs->client_height = pixman_image_get_height(vs->vd->server);
     vnc_write_u16(vs, vs->client_width);
@@ -2359,9 +2490,11 @@
 static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len)
 {
     unsigned char response[VNC_AUTH_CHALLENGE_SIZE];
-    int i, j, pwlen;
+    size_t i, pwlen;
     unsigned char key[8];
     time_t now = time(NULL);
+    QCryptoCipher *cipher = NULL;
+    Error *err = NULL;
 
     if (!vs->vd->password) {
         VNC_DEBUG("No password configured on server");
@@ -2378,9 +2511,29 @@
     pwlen = strlen(vs->vd->password);
     for (i=0; i<sizeof(key); i++)
         key[i] = i<pwlen ? vs->vd->password[i] : 0;
-    deskey(key, EN0);
-    for (j = 0; j < VNC_AUTH_CHALLENGE_SIZE; j += 8)
-        des(response+j, response+j);
+
+    cipher = qcrypto_cipher_new(
+        QCRYPTO_CIPHER_ALG_DES_RFB,
+        QCRYPTO_CIPHER_MODE_ECB,
+        key, G_N_ELEMENTS(key),
+        &err);
+    if (!cipher) {
+        VNC_DEBUG("Cannot initialize cipher %s",
+                  error_get_pretty(err));
+        error_free(err);
+        goto reject;
+    }
+
+    if (qcrypto_cipher_encrypt(cipher,
+                               vs->challenge,
+                               response,
+                               VNC_AUTH_CHALLENGE_SIZE,
+                               &err) < 0) {
+        VNC_DEBUG("Cannot encrypt challenge %s",
+                  error_get_pretty(err));
+        error_free(err);
+        goto reject;
+    }
 
     /* Compare expected vs actual challenge response */
     if (memcmp(response, data, VNC_AUTH_CHALLENGE_SIZE) != 0) {
@@ -2393,6 +2546,8 @@
 
         start_client_init(vs);
     }
+
+    qcrypto_cipher_free(cipher);
     return 0;
 
 reject:
@@ -2404,6 +2559,7 @@
     }
     vnc_flush(vs);
     vnc_client_error(vs);
+    qcrypto_cipher_free(cipher);
     return 0;
 }
 
@@ -2448,12 +2604,10 @@
            start_auth_vnc(vs);
            break;
 
-#ifdef CONFIG_VNC_TLS
        case VNC_AUTH_VENCRYPT:
            VNC_DEBUG("Accept VeNCrypt auth\n");
            start_auth_vencrypt(vs);
            break;
-#endif /* CONFIG_VNC_TLS */
 
 #ifdef CONFIG_VNC_SASL
        case VNC_AUTH_SASL:
@@ -2689,7 +2843,7 @@
                     pixman_image_get_width(vd->server));
     int height = MIN(pixman_image_get_height(vd->guest.fb),
                      pixman_image_get_height(vd->server));
-    int cmp_bytes, server_stride, min_stride, guest_stride, y = 0;
+    int cmp_bytes, server_stride, line_bytes, guest_ll, guest_stride, y = 0;
     uint8_t *guest_row0 = NULL, *server_row0;
     VncState *vs;
     int has_dirty = 0;
@@ -2708,17 +2862,21 @@
      * Update server dirty map.
      */
     server_row0 = (uint8_t *)pixman_image_get_data(vd->server);
-    server_stride = guest_stride = pixman_image_get_stride(vd->server);
+    server_stride = guest_stride = guest_ll =
+        pixman_image_get_stride(vd->server);
     cmp_bytes = MIN(VNC_DIRTY_PIXELS_PER_BIT * VNC_SERVER_FB_BYTES,
                     server_stride);
     if (vd->guest.format != VNC_SERVER_FB_FORMAT) {
         int width = pixman_image_get_width(vd->server);
         tmpbuf = qemu_pixman_linebuf_create(VNC_SERVER_FB_FORMAT, width);
     } else {
+        int guest_bpp =
+            PIXMAN_FORMAT_BPP(pixman_image_get_format(vd->guest.fb));
         guest_row0 = (uint8_t *)pixman_image_get_data(vd->guest.fb);
         guest_stride = pixman_image_get_stride(vd->guest.fb);
+        guest_ll = pixman_image_get_width(vd->guest.fb) * ((guest_bpp + 7) / 8);
     }
-    min_stride = MIN(server_stride, guest_stride);
+    line_bytes = MIN(server_stride, guest_ll);
 
     for (;;) {
         int x;
@@ -2749,9 +2907,10 @@
             if (!test_and_clear_bit(x, vd->guest.dirty[y])) {
                 continue;
             }
-            if ((x + 1) * cmp_bytes > min_stride) {
-                _cmp_bytes = min_stride - x * cmp_bytes;
+            if ((x + 1) * cmp_bytes > line_bytes) {
+                _cmp_bytes = line_bytes - x * cmp_bytes;
             }
+            assert(_cmp_bytes >= 0);
             if (memcmp(server_ptr, guest_ptr, _cmp_bytes) == 0) {
                 continue;
             }
@@ -2783,7 +2942,7 @@
         return;
     }
 
-    graphic_hw_update(NULL);
+    graphic_hw_update(vd->dcl.con);
 
     if (vnc_trylock_display(vd)) {
         update_displaychangelistener(&vd->dcl, VNC_REFRESH_INTERVAL_BASE);
@@ -2811,71 +2970,96 @@
     }
 }
 
-static void vnc_connect(VncDisplay *vd, int csock,
+static void vnc_connect(VncDisplay *vd, QIOChannelSocket *sioc,
                         bool skipauth, bool websocket)
 {
-    VncState *vs = g_malloc0(sizeof(VncState));
+    VncState *vs = g_new0(VncState, 1);
     int i;
 
-    vs->csock = csock;
+    vs->sioc = sioc;
+    object_ref(OBJECT(vs->sioc));
+    vs->ioc = QIO_CHANNEL(sioc);
+    object_ref(OBJECT(vs->ioc));
+    vs->vd = vd;
+
+    buffer_init(&vs->input,          "vnc-input/%p", sioc);
+    buffer_init(&vs->output,         "vnc-output/%p", sioc);
+    buffer_init(&vs->jobs_buffer,    "vnc-jobs_buffer/%p", sioc);
+
+    buffer_init(&vs->tight.tight,    "vnc-tight/%p", sioc);
+    buffer_init(&vs->tight.zlib,     "vnc-tight-zlib/%p", sioc);
+    buffer_init(&vs->tight.gradient, "vnc-tight-gradient/%p", sioc);
+#ifdef CONFIG_VNC_JPEG
+    buffer_init(&vs->tight.jpeg,     "vnc-tight-jpeg/%p", sioc);
+#endif
+#ifdef CONFIG_VNC_PNG
+    buffer_init(&vs->tight.png,      "vnc-tight-png/%p", sioc);
+#endif
+    buffer_init(&vs->zlib.zlib,      "vnc-zlib/%p", sioc);
+    buffer_init(&vs->zrle.zrle,      "vnc-zrle/%p", sioc);
+    buffer_init(&vs->zrle.fb,        "vnc-zrle-fb/%p", sioc);
+    buffer_init(&vs->zrle.zlib,      "vnc-zrle-zlib/%p", sioc);
 
     if (skipauth) {
 	vs->auth = VNC_AUTH_NONE;
-#ifdef CONFIG_VNC_TLS
 	vs->subauth = VNC_AUTH_INVALID;
-#endif
     } else {
-	vs->auth = vd->auth;
-#ifdef CONFIG_VNC_TLS
-	vs->subauth = vd->subauth;
-#endif
+        if (websocket) {
+            vs->auth = vd->ws_auth;
+            vs->subauth = VNC_AUTH_INVALID;
+        } else {
+            vs->auth = vd->auth;
+            vs->subauth = vd->subauth;
+        }
     }
+    VNC_DEBUG("Client sioc=%p ws=%d auth=%d subauth=%d\n",
+              sioc, websocket, vs->auth, vs->subauth);
 
     vs->lossy_rect = g_malloc0(VNC_STAT_ROWS * sizeof (*vs->lossy_rect));
     for (i = 0; i < VNC_STAT_ROWS; ++i) {
-        vs->lossy_rect[i] = g_malloc0(VNC_STAT_COLS * sizeof (uint8_t));
+        vs->lossy_rect[i] = g_new0(uint8_t, VNC_STAT_COLS);
     }
 
-    VNC_DEBUG("New client on socket %d\n", csock);
+    VNC_DEBUG("New client on socket %p\n", vs->sioc);
     update_displaychangelistener(&vd->dcl, VNC_REFRESH_INTERVAL_BASE);
-    qemu_set_nonblock(vs->csock);
-#ifdef CONFIG_VNC_WS
+    qio_channel_set_blocking(vs->ioc, false, NULL);
     if (websocket) {
         vs->websocket = 1;
-#ifdef CONFIG_VNC_TLS
-        if (vd->tls.x509cert) {
-            qemu_set_fd_handler2(vs->csock, NULL, vncws_tls_handshake_peek,
-                                 NULL, vs);
-        } else
-#endif /* CONFIG_VNC_TLS */
-        {
-            qemu_set_fd_handler2(vs->csock, NULL, vncws_handshake_read,
-                                 NULL, vs);
+        if (vd->ws_tls) {
+            vs->ioc_tag = qio_channel_add_watch(
+                vs->ioc, G_IO_IN, vncws_tls_handshake_io, vs, NULL);
+        } else {
+            vs->ioc_tag = qio_channel_add_watch(
+                vs->ioc, G_IO_IN, vncws_handshake_io, vs, NULL);
         }
-    } else
-#endif /* CONFIG_VNC_WS */
-    {
-        qemu_set_fd_handler2(vs->csock, NULL, vnc_client_read, NULL, vs);
+    } else {
+        vs->ioc_tag = qio_channel_add_watch(
+            vs->ioc, G_IO_IN, vnc_client_io, vs, NULL);
     }
 
     vnc_client_cache_addr(vs);
     vnc_qmp_event(vs, QAPI_EVENT_VNC_CONNECTED);
     vnc_set_share_mode(vs, VNC_SHARE_MODE_CONNECTING);
 
-    vs->vd = vd;
-
-#ifdef CONFIG_VNC_WS
-    if (!vs->websocket)
-#endif
-    {
+    if (!vs->websocket) {
         vnc_init_state(vs);
     }
+
+    if (vd->num_connecting > vd->connections_limit) {
+        QTAILQ_FOREACH(vs, &vd->clients, next) {
+            if (vs->share_mode == VNC_SHARE_MODE_CONNECTING) {
+                vnc_disconnect_start(vs);
+                return;
+            }
+        }
+    }
 }
 
 void vnc_init_state(VncState *vs)
 {
     vs->initialized = true;
     VncDisplay *vd = vs->vd;
+    bool first_client = QTAILQ_EMPTY(&vd->clients);
 
     vs->last_x = -1;
     vs->last_y = -1;
@@ -2888,9 +3072,12 @@
     qemu_mutex_init(&vs->output_mutex);
     vs->bh = qemu_bh_new(vnc_jobs_bh, vs);
 
-    QTAILQ_INSERT_HEAD(&vd->clients, vs, next);
+    QTAILQ_INSERT_TAIL(&vd->clients, vs, next);
+    if (first_client) {
+        vnc_update_server_surface(vd);
+    }
 
-    graphic_hw_update(NULL);
+    graphic_hw_update(vd->dcl.con);
 
     vnc_write(vs, "RFB 003.008\n", 12);
     vnc_flush(vs);
@@ -2905,62 +3092,52 @@
     /* vs might be free()ed here */
 }
 
-static void vnc_listen_read(void *opaque, bool websocket)
+static gboolean vnc_listen_io(QIOChannel *ioc,
+                              GIOCondition condition,
+                              void *opaque)
 {
     VncDisplay *vs = opaque;
-    struct sockaddr_in addr;
-    socklen_t addrlen = sizeof(addr);
-    int csock;
+    QIOChannelSocket *sioc = NULL;
+    Error *err = NULL;
 
     /* Catch-up */
-    graphic_hw_update(NULL);
-#ifdef CONFIG_VNC_WS
-    if (websocket) {
-        csock = qemu_accept(vs->lwebsock, (struct sockaddr *)&addr, &addrlen);
-    } else
-#endif /* CONFIG_VNC_WS */
-    {
-        csock = qemu_accept(vs->lsock, (struct sockaddr *)&addr, &addrlen);
+    graphic_hw_update(vs->dcl.con);
+    sioc = qio_channel_socket_accept(QIO_CHANNEL_SOCKET(ioc), &err);
+    if (sioc != NULL) {
+        qio_channel_set_delay(QIO_CHANNEL(sioc), false);
+        vnc_connect(vs, sioc, false,
+                    ioc != QIO_CHANNEL(vs->lsock));
+        object_unref(OBJECT(sioc));
+    } else {
+        /* client probably closed connection before we got there */
+        error_free(err);
     }
 
-    if (csock != -1) {
-        socket_set_nodelay(csock);
-        vnc_connect(vs, csock, false, websocket);
-    }
+    return TRUE;
 }
 
-static void vnc_listen_regular_read(void *opaque)
-{
-    vnc_listen_read(opaque, false);
-}
-
-#ifdef CONFIG_VNC_WS
-static void vnc_listen_websocket_read(void *opaque)
-{
-    vnc_listen_read(opaque, true);
-}
-#endif /* CONFIG_VNC_WS */
-
 static const DisplayChangeListenerOps dcl_ops = {
-    .dpy_name          = "vnc",
-    .dpy_refresh       = vnc_refresh,
-    .dpy_gfx_copy      = vnc_dpy_copy,
-    .dpy_gfx_update    = vnc_dpy_update,
-    .dpy_gfx_switch    = vnc_dpy_switch,
-    .dpy_mouse_set     = vnc_mouse_set,
-    .dpy_cursor_define = vnc_dpy_cursor_define,
+    .dpy_name             = "vnc",
+    .dpy_refresh          = vnc_refresh,
+    .dpy_gfx_copy         = vnc_dpy_copy,
+    .dpy_gfx_update       = vnc_dpy_update,
+    .dpy_gfx_switch       = vnc_dpy_switch,
+    .dpy_gfx_check_format = qemu_pixman_check_format,
+    .dpy_mouse_set        = vnc_mouse_set,
+    .dpy_cursor_define    = vnc_dpy_cursor_define,
 };
 
-void vnc_display_init(DisplayState *ds)
+void vnc_display_init(const char *id)
 {
-    VncDisplay *vs = g_malloc0(sizeof(*vs));
+    VncDisplay *vs;
 
-    vnc_display = vs;
+    if (vnc_display_find(id) != NULL) {
+        return;
+    }
+    vs = g_malloc0(sizeof(*vs));
 
-    vs->lsock = -1;
-#ifdef CONFIG_VNC_WS
-    vs->lwebsock = -1;
-#endif
+    vs->id = strdup(id);
+    QTAILQ_INSERT_TAIL(&vnc_displays, vs, next);
 
     QTAILQ_INIT(&vs->clients);
     vs->expires = TIME_MAX;
@@ -2975,6 +3152,9 @@
     if (!vs->kbd_layout)
         exit(1);
 
+    vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
+    vs->connections_limit = 32;
+
     qemu_mutex_init(&vs->mutex);
     vnc_start_worker_thread();
 
@@ -2983,45 +3163,47 @@
 }
 
 
-static void vnc_display_close(DisplayState *ds)
+static void vnc_display_close(VncDisplay *vs)
 {
-    VncDisplay *vs = vnc_display;
-
     if (!vs)
         return;
-    g_free(vs->display);
-    vs->display = NULL;
-    if (vs->lsock != -1) {
-        qemu_set_fd_handler2(vs->lsock, NULL, NULL, NULL, NULL);
-        close(vs->lsock);
-        vs->lsock = -1;
+    vs->enabled = false;
+    vs->is_unix = false;
+    if (vs->lsock != NULL) {
+        if (vs->lsock_tag) {
+            g_source_remove(vs->lsock_tag);
+        }
+        object_unref(OBJECT(vs->lsock));
+        vs->lsock = NULL;
     }
-#ifdef CONFIG_VNC_WS
-    g_free(vs->ws_display);
-    vs->ws_display = NULL;
-    if (vs->lwebsock != -1) {
-        qemu_set_fd_handler2(vs->lwebsock, NULL, NULL, NULL, NULL);
-        close(vs->lwebsock);
-        vs->lwebsock = -1;
+    vs->ws_enabled = false;
+    if (vs->lwebsock != NULL) {
+        if (vs->lwebsock_tag) {
+            g_source_remove(vs->lwebsock_tag);
+        }
+        object_unref(OBJECT(vs->lwebsock));
+        vs->lwebsock = NULL;
     }
-#endif /* CONFIG_VNC_WS */
     vs->auth = VNC_AUTH_INVALID;
-#ifdef CONFIG_VNC_TLS
     vs->subauth = VNC_AUTH_INVALID;
-    vs->tls.x509verify = 0;
-#endif
+    if (vs->tlscreds) {
+        object_unparent(OBJECT(vs->tlscreds));
+        vs->tlscreds = NULL;
+    }
+    g_free(vs->tlsaclname);
+    vs->tlsaclname = NULL;
 }
 
-int vnc_display_password(DisplayState *ds, const char *password)
+int vnc_display_password(const char *id, const char *password)
 {
-    VncDisplay *vs = vnc_display;
+    VncDisplay *vs = vnc_display_find(id);
 
     if (!vs) {
         return -EINVAL;
     }
     if (vs->auth == VNC_AUTH_NONE) {
         error_printf_unless_qmp("If you want use passwords please enable "
-                                "password auth using '-vnc ${dpy},password'.");
+                                "password auth using '-vnc ${dpy},password'.\n");
         return -EINVAL;
     }
 
@@ -3031,9 +3213,9 @@
     return 0;
 }
 
-int vnc_display_pw_expire(DisplayState *ds, time_t expires)
+int vnc_display_pw_expire(const char *id, time_t expires)
 {
-    VncDisplay *vs = vnc_display;
+    VncDisplay *vs = vnc_display_find(id);
 
     if (!vs) {
         return -EINVAL;
@@ -3043,145 +3225,527 @@
     return 0;
 }
 
-char *vnc_display_local_addr(DisplayState *ds)
+static void vnc_display_print_local_addr(VncDisplay *vs)
 {
-    VncDisplay *vs = vnc_display;
-    
-    return vnc_socket_local_addr("%s:%s", vs->lsock);
+    SocketAddress *addr;
+    Error *err = NULL;
+
+    addr = qio_channel_socket_get_local_address(vs->lsock, &err);
+    if (!addr) {
+        return;
+    }
+
+    if (addr->type != SOCKET_ADDRESS_KIND_INET) {
+        qapi_free_SocketAddress(addr);
+        return;
+    }
+    error_printf_unless_qmp("VNC server running on %s:%s\n",
+                            addr->u.inet.data->host,
+                            addr->u.inet.data->port);
+    qapi_free_SocketAddress(addr);
 }
 
-void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
+static QemuOptsList qemu_vnc_opts = {
+    .name = "vnc",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_vnc_opts.head),
+    .implied_opt_name = "vnc",
+    .desc = {
+        {
+            .name = "vnc",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "websocket",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "tls-creds",
+            .type = QEMU_OPT_STRING,
+        },{
+            /* Deprecated in favour of tls-creds */
+            .name = "x509",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "share",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "display",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "head",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "connections",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "to",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "ipv4",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "ipv6",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "password",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "reverse",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "lock-key-sync",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "key-delay-ms",
+            .type = QEMU_OPT_NUMBER,
+        },{
+            .name = "sasl",
+            .type = QEMU_OPT_BOOL,
+        },{
+            /* Deprecated in favour of tls-creds */
+            .name = "tls",
+            .type = QEMU_OPT_BOOL,
+        },{
+            /* Deprecated in favour of tls-creds */
+            .name = "x509verify",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "acl",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "lossy",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "non-adaptive",
+            .type = QEMU_OPT_BOOL,
+        },
+        { /* end of list */ }
+    },
+};
+
+
+static int
+vnc_display_setup_auth(VncDisplay *vs,
+                       bool password,
+                       bool sasl,
+                       bool websocket,
+                       Error **errp)
 {
-    VncDisplay *vs = vnc_display;
-    const char *options;
-    int password = 0;
-    int reverse = 0;
-#ifdef CONFIG_VNC_TLS
-    int tls = 0, x509 = 0;
-#endif
+    /*
+     * We have a choice of 3 authentication options
+     *
+     *   1. none
+     *   2. vnc
+     *   3. sasl
+     *
+     * The channel can be run in 2 modes
+     *
+     *   1. clear
+     *   2. tls
+     *
+     * And TLS can use 2 types of credentials
+     *
+     *   1. anon
+     *   2. x509
+     *
+     * We thus have 9 possible logical combinations
+     *
+     *   1. clear + none
+     *   2. clear + vnc
+     *   3. clear + sasl
+     *   4. tls + anon + none
+     *   5. tls + anon + vnc
+     *   6. tls + anon + sasl
+     *   7. tls + x509 + none
+     *   8. tls + x509 + vnc
+     *   9. tls + x509 + sasl
+     *
+     * These need to be mapped into the VNC auth schemes
+     * in an appropriate manner. In regular VNC, all the
+     * TLS options get mapped into VNC_AUTH_VENCRYPT
+     * sub-auth types.
+     *
+     * In websockets, the https:// protocol already provides
+     * TLS support, so there is no need to make use of the
+     * VeNCrypt extension. Furthermore, websockets browser
+     * clients could not use VeNCrypt even if they wanted to,
+     * as they cannot control when the TLS handshake takes
+     * place. Thus there is no option but to rely on https://,
+     * meaning combinations 4->6 and 7->9 will be mapped to
+     * VNC auth schemes in the same way as combos 1->3.
+     *
+     * Regardless of fact that we have a different mapping to
+     * VNC auth mechs for plain VNC vs websockets VNC, the end
+     * result has the same security characteristics.
+     */
+    if (password) {
+        if (vs->tlscreds) {
+            vs->auth = VNC_AUTH_VENCRYPT;
+            if (websocket) {
+                vs->ws_tls = true;
+            }
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
+                VNC_DEBUG("Initializing VNC server with x509 password auth\n");
+                vs->subauth = VNC_AUTH_VENCRYPT_X509VNC;
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+                VNC_DEBUG("Initializing VNC server with TLS password auth\n");
+                vs->subauth = VNC_AUTH_VENCRYPT_TLSVNC;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
+            }
+        } else {
+            VNC_DEBUG("Initializing VNC server with password auth\n");
+            vs->auth = VNC_AUTH_VNC;
+            vs->subauth = VNC_AUTH_INVALID;
+        }
+        if (websocket) {
+            vs->ws_auth = VNC_AUTH_VNC;
+        } else {
+            vs->ws_auth = VNC_AUTH_INVALID;
+        }
+    } else if (sasl) {
+        if (vs->tlscreds) {
+            vs->auth = VNC_AUTH_VENCRYPT;
+            if (websocket) {
+                vs->ws_tls = true;
+            }
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
+                VNC_DEBUG("Initializing VNC server with x509 SASL auth\n");
+                vs->subauth = VNC_AUTH_VENCRYPT_X509SASL;
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+                VNC_DEBUG("Initializing VNC server with TLS SASL auth\n");
+                vs->subauth = VNC_AUTH_VENCRYPT_TLSSASL;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
+            }
+        } else {
+            VNC_DEBUG("Initializing VNC server with SASL auth\n");
+            vs->auth = VNC_AUTH_SASL;
+            vs->subauth = VNC_AUTH_INVALID;
+        }
+        if (websocket) {
+            vs->ws_auth = VNC_AUTH_SASL;
+        } else {
+            vs->ws_auth = VNC_AUTH_INVALID;
+        }
+    } else {
+        if (vs->tlscreds) {
+            vs->auth = VNC_AUTH_VENCRYPT;
+            if (websocket) {
+                vs->ws_tls = true;
+            }
+            if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                    TYPE_QCRYPTO_TLS_CREDS_X509)) {
+                VNC_DEBUG("Initializing VNC server with x509 no auth\n");
+                vs->subauth = VNC_AUTH_VENCRYPT_X509NONE;
+            } else if (object_dynamic_cast(OBJECT(vs->tlscreds),
+                                           TYPE_QCRYPTO_TLS_CREDS_ANON)) {
+                VNC_DEBUG("Initializing VNC server with TLS no auth\n");
+                vs->subauth = VNC_AUTH_VENCRYPT_TLSNONE;
+            } else {
+                error_setg(errp,
+                           "Unsupported TLS cred type %s",
+                           object_get_typename(OBJECT(vs->tlscreds)));
+                return -1;
+            }
+        } else {
+            VNC_DEBUG("Initializing VNC server with no auth\n");
+            vs->auth = VNC_AUTH_NONE;
+            vs->subauth = VNC_AUTH_INVALID;
+        }
+        if (websocket) {
+            vs->ws_auth = VNC_AUTH_NONE;
+        } else {
+            vs->ws_auth = VNC_AUTH_INVALID;
+        }
+    }
+    return 0;
+}
+
+
+/*
+ * Handle back compat with old CLI syntax by creating some
+ * suitable QCryptoTLSCreds objects
+ */
+static QCryptoTLSCreds *
+vnc_display_create_creds(bool x509,
+                         bool x509verify,
+                         const char *dir,
+                         const char *id,
+                         Error **errp)
+{
+    gchar *credsid = g_strdup_printf("tlsvnc%s", id);
+    Object *parent = object_get_objects_root();
+    Object *creds;
+    Error *err = NULL;
+
+    if (x509) {
+        creds = object_new_with_props(TYPE_QCRYPTO_TLS_CREDS_X509,
+                                      parent,
+                                      credsid,
+                                      &err,
+                                      "endpoint", "server",
+                                      "dir", dir,
+                                      "verify-peer", x509verify ? "yes" : "no",
+                                      NULL);
+    } else {
+        creds = object_new_with_props(TYPE_QCRYPTO_TLS_CREDS_ANON,
+                                      parent,
+                                      credsid,
+                                      &err,
+                                      "endpoint", "server",
+                                      NULL);
+    }
+
+    g_free(credsid);
+
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
+    return QCRYPTO_TLS_CREDS(creds);
+}
+
+
+void vnc_display_open(const char *id, Error **errp)
+{
+    VncDisplay *vs = vnc_display_find(id);
+    QemuOpts *opts = qemu_opts_find(&qemu_vnc_opts, id);
+    SocketAddress *saddr = NULL, *wsaddr = NULL;
+    const char *share, *device_id;
+    QemuConsole *con;
+    bool password = false;
+    bool reverse = false;
+    const char *vnc;
+    char *h;
+    const char *credid;
+    int show_vnc_port = 0;
+    bool sasl = false;
 #ifdef CONFIG_VNC_SASL
-    int sasl = 0;
     int saslErr;
 #endif
-#if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
     int acl = 0;
-#endif
     int lock_key_sync = 1;
+    int key_delay_ms;
 
-    if (!vnc_display) {
+    if (!vs) {
         error_setg(errp, "VNC display not active");
         return;
     }
-    vnc_display_close(ds);
-    if (strcmp(display, "none") == 0)
+    vnc_display_close(vs);
+
+    if (!opts) {
         return;
+    }
+    vnc = qemu_opt_get(opts, "vnc");
+    if (!vnc || strcmp(vnc, "none") == 0) {
+        return;
+    }
 
-    vs->display = g_strdup(display);
-    vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
+    h = strrchr(vnc, ':');
+    if (h) {
+        size_t hlen = h - vnc;
 
-    options = display;
-    while ((options = strchr(options, ','))) {
-        options++;
-        if (strncmp(options, "password", 8) == 0) {
-            if (fips_get_state()) {
+        const char *websocket = qemu_opt_get(opts, "websocket");
+        int to = qemu_opt_get_number(opts, "to", 0);
+        bool has_ipv4 = qemu_opt_get(opts, "ipv4");
+        bool has_ipv6 = qemu_opt_get(opts, "ipv6");
+        bool ipv4 = qemu_opt_get_bool(opts, "ipv4", false);
+        bool ipv6 = qemu_opt_get_bool(opts, "ipv6", false);
+
+        saddr = g_new0(SocketAddress, 1);
+        if (websocket) {
+            if (!qcrypto_hash_supports(QCRYPTO_HASH_ALG_SHA1)) {
                 error_setg(errp,
-                           "VNC password auth disabled due to FIPS mode, "
-                           "consider using the VeNCrypt or SASL authentication "
-                           "methods as an alternative");
+                           "SHA1 hash support is required for websockets");
                 goto fail;
             }
-            password = 1; /* Require password auth */
-        } else if (strncmp(options, "reverse", 7) == 0) {
-            reverse = 1;
-        } else if (strncmp(options, "no-lock-key-sync", 16) == 0) {
-            lock_key_sync = 0;
-#ifdef CONFIG_VNC_SASL
-        } else if (strncmp(options, "sasl", 4) == 0) {
-            sasl = 1; /* Require SASL auth */
-#endif
-#ifdef CONFIG_VNC_WS
-        } else if (strncmp(options, "websocket", 9) == 0) {
-            char *start, *end;
-            vs->websocket = 1;
 
-            /* Check for 'websocket=<port>' */
-            start = strchr(options, '=');
-            end = strchr(options, ',');
-            if (start && (!end || (start < end))) {
-                int len = end ? end-(start+1) : strlen(start+1);
-                if (len < 6) {
-                    /* extract the host specification from display */
-                    char  *host = NULL, *port = NULL, *host_end = NULL;
-                    port = g_strndup(start + 1, len);
+            wsaddr = g_new0(SocketAddress, 1);
+            vs->ws_enabled = true;
+        }
 
-                    /* ipv6 hosts have colons */
-                    end = strchr(display, ',');
-                    host_end = g_strrstr_len(display, end - display, ":");
+        if (strncmp(vnc, "unix:", 5) == 0) {
+            saddr->type = SOCKET_ADDRESS_KIND_UNIX;
+            saddr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+            saddr->u.q_unix.data->path = g_strdup(vnc + 5);
 
-                    if (host_end) {
-                        host = g_strndup(display, host_end - display + 1);
-                    } else {
-                        host = g_strndup(":", 1);
-                    }
-                    vs->ws_display = g_strconcat(host, port, NULL);
-                    g_free(host);
-                    g_free(port);
-                }
-            }
-#endif /* CONFIG_VNC_WS */
-#ifdef CONFIG_VNC_TLS
-        } else if (strncmp(options, "tls", 3) == 0) {
-            tls = 1; /* Require TLS */
-        } else if (strncmp(options, "x509", 4) == 0) {
-            char *start, *end;
-            x509 = 1; /* Require x509 certificates */
-            if (strncmp(options, "x509verify", 10) == 0)
-                vs->tls.x509verify = 1; /* ...and verify client certs */
-
-            /* Now check for 'x509=/some/path' postfix
-             * and use that to setup x509 certificate/key paths */
-            start = strchr(options, '=');
-            end = strchr(options, ',');
-            if (start && (!end || (start < end))) {
-                int len = end ? end-(start+1) : strlen(start+1);
-                char *path = g_strndup(start + 1, len);
-
-                VNC_DEBUG("Trying certificate path '%s'\n", path);
-                if (vnc_tls_set_x509_creds_dir(vs, path) < 0) {
-                    error_setg(errp, "Failed to find x509 certificates/keys in %s", path);
-                    g_free(path);
-                    goto fail;
-                }
-                g_free(path);
-            } else {
-                error_setg(errp, "No certificate path provided");
+            if (vs->ws_enabled) {
+                error_setg(errp, "UNIX sockets not supported with websock");
                 goto fail;
             }
-#endif
-#if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
-        } else if (strncmp(options, "acl", 3) == 0) {
-            acl = 1;
-#endif
-        } else if (strncmp(options, "lossy", 5) == 0) {
-#ifdef CONFIG_VNC_JPEG
-            vs->lossy = true;
-#endif
-        } else if (strncmp(options, "non-adaptive", 12) == 0) {
-            vs->non_adaptive = true;
-        } else if (strncmp(options, "share=", 6) == 0) {
-            if (strncmp(options+6, "ignore", 6) == 0) {
-                vs->share_policy = VNC_SHARE_POLICY_IGNORE;
-            } else if (strncmp(options+6, "allow-exclusive", 15) == 0) {
-                vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
-            } else if (strncmp(options+6, "force-shared", 12) == 0) {
-                vs->share_policy = VNC_SHARE_POLICY_FORCE_SHARED;
+        } else {
+            unsigned long long baseport;
+            InetSocketAddress *inet;
+            saddr->type = SOCKET_ADDRESS_KIND_INET;
+            inet = saddr->u.inet.data = g_new0(InetSocketAddress, 1);
+            if (vnc[0] == '[' && vnc[hlen - 1] == ']') {
+                inet->host = g_strndup(vnc + 1, hlen - 2);
             } else {
-                error_setg(errp, "unknown vnc share= option");
+                inet->host = g_strndup(vnc, hlen);
+            }
+            if (parse_uint_full(h + 1, &baseport, 10) < 0) {
+                error_setg(errp, "can't convert to a number: %s", h + 1);
+                goto fail;
+            }
+            if (baseport > 65535 ||
+                baseport + 5900 > 65535) {
+                error_setg(errp, "port %s out of range", h + 1);
+                goto fail;
+            }
+            inet->port = g_strdup_printf(
+                "%d", (int)baseport + 5900);
+
+            if (to) {
+                inet->has_to = true;
+                inet->to = to + 5900;
+                show_vnc_port = 1;
+            }
+            inet->ipv4 = ipv4;
+            inet->has_ipv4 = has_ipv4;
+            inet->ipv6 = ipv6;
+            inet->has_ipv6 = has_ipv6;
+
+            if (vs->ws_enabled) {
+                wsaddr->type = SOCKET_ADDRESS_KIND_INET;
+                inet = wsaddr->u.inet.data = g_new0(InetSocketAddress, 1);
+                inet->host = g_strdup(saddr->u.inet.data->host);
+                inet->port = g_strdup(websocket);
+
+                if (to) {
+                    inet->has_to = true;
+                    inet->to = to;
+                }
+                inet->ipv4 = ipv4;
+                inet->has_ipv4 = has_ipv4;
+                inet->ipv6 = ipv6;
+                inet->has_ipv6 = has_ipv6;
+            }
+        }
+    } else {
+        error_setg(errp, "no vnc port specified");
+        goto fail;
+    }
+
+    password = qemu_opt_get_bool(opts, "password", false);
+    if (password) {
+        if (fips_get_state()) {
+            error_setg(errp,
+                       "VNC password auth disabled due to FIPS mode, "
+                       "consider using the VeNCrypt or SASL authentication "
+                       "methods as an alternative");
+            goto fail;
+        }
+        if (!qcrypto_cipher_supports(
+                QCRYPTO_CIPHER_ALG_DES_RFB)) {
+            error_setg(errp,
+                       "Cipher backend does not support DES RFB algorithm");
+            goto fail;
+        }
+    }
+
+    reverse = qemu_opt_get_bool(opts, "reverse", false);
+    lock_key_sync = qemu_opt_get_bool(opts, "lock-key-sync", true);
+    key_delay_ms = qemu_opt_get_number(opts, "key-delay-ms", 1);
+    sasl = qemu_opt_get_bool(opts, "sasl", false);
+#ifndef CONFIG_VNC_SASL
+    if (sasl) {
+        error_setg(errp, "VNC SASL auth requires cyrus-sasl support");
+        goto fail;
+    }
+#endif /* CONFIG_VNC_SASL */
+    credid = qemu_opt_get(opts, "tls-creds");
+    if (credid) {
+        Object *creds;
+        if (qemu_opt_get(opts, "tls") ||
+            qemu_opt_get(opts, "x509") ||
+            qemu_opt_get(opts, "x509verify")) {
+            error_setg(errp,
+                       "'tls-creds' parameter is mutually exclusive with "
+                       "'tls', 'x509' and 'x509verify' parameters");
+            goto fail;
+        }
+
+        creds = object_resolve_path_component(
+            object_get_objects_root(), credid);
+        if (!creds) {
+            error_setg(errp, "No TLS credentials with id '%s'",
+                       credid);
+            goto fail;
+        }
+        vs->tlscreds = (QCryptoTLSCreds *)
+            object_dynamic_cast(creds,
+                                TYPE_QCRYPTO_TLS_CREDS);
+        if (!vs->tlscreds) {
+            error_setg(errp, "Object with id '%s' is not TLS credentials",
+                       credid);
+            goto fail;
+        }
+        object_ref(OBJECT(vs->tlscreds));
+
+        if (vs->tlscreds->endpoint != QCRYPTO_TLS_CREDS_ENDPOINT_SERVER) {
+            error_setg(errp,
+                       "Expecting TLS credentials with a server endpoint");
+            goto fail;
+        }
+    } else {
+        const char *path;
+        bool tls = false, x509 = false, x509verify = false;
+        tls  = qemu_opt_get_bool(opts, "tls", false);
+        if (tls) {
+            path = qemu_opt_get(opts, "x509");
+
+            if (path) {
+                x509 = true;
+            } else {
+                path = qemu_opt_get(opts, "x509verify");
+                if (path) {
+                    x509 = true;
+                    x509verify = true;
+                }
+            }
+            vs->tlscreds = vnc_display_create_creds(x509,
+                                                    x509verify,
+                                                    path,
+                                                    vs->id,
+                                                    errp);
+            if (!vs->tlscreds) {
                 goto fail;
             }
         }
     }
+    acl = qemu_opt_get_bool(opts, "acl", false);
 
+    share = qemu_opt_get(opts, "share");
+    if (share) {
+        if (strcmp(share, "ignore") == 0) {
+            vs->share_policy = VNC_SHARE_POLICY_IGNORE;
+        } else if (strcmp(share, "allow-exclusive") == 0) {
+            vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
+        } else if (strcmp(share, "force-shared") == 0) {
+            vs->share_policy = VNC_SHARE_POLICY_FORCE_SHARED;
+        } else {
+            error_setg(errp, "unknown vnc share= option");
+            goto fail;
+        }
+    } else {
+        vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
+    }
+    vs->connections_limit = qemu_opt_get_number(opts, "connections", 32);
+
+#ifdef CONFIG_VNC_JPEG
+    vs->lossy = qemu_opt_get_bool(opts, "lossy", false);
+#endif
+    vs->non_adaptive = qemu_opt_get_bool(opts, "non-adaptive", false);
     /* adaptive updates are only used with tight encoding and
      * if lossy updates are enabled so we can disable all the
      * calculations otherwise */
@@ -3189,98 +3753,30 @@
         vs->non_adaptive = true;
     }
 
-#ifdef CONFIG_VNC_TLS
-    if (acl && x509 && vs->tls.x509verify) {
-        if (!(vs->tls.acl = qemu_acl_init("vnc.x509dname"))) {
-            fprintf(stderr, "Failed to create x509 dname ACL\n");
-            exit(1);
+    if (acl) {
+        if (strcmp(vs->id, "default") == 0) {
+            vs->tlsaclname = g_strdup("vnc.x509dname");
+        } else {
+            vs->tlsaclname = g_strdup_printf("vnc.%s.x509dname", vs->id);
         }
+        qemu_acl_init(vs->tlsaclname);
     }
-#endif
 #ifdef CONFIG_VNC_SASL
     if (acl && sasl) {
-        if (!(vs->sasl.acl = qemu_acl_init("vnc.username"))) {
-            fprintf(stderr, "Failed to create username ACL\n");
-            exit(1);
+        char *aclname;
+
+        if (strcmp(vs->id, "default") == 0) {
+            aclname = g_strdup("vnc.username");
+        } else {
+            aclname = g_strdup_printf("vnc.%s.username", vs->id);
         }
+        vs->sasl.acl = qemu_acl_init(aclname);
+        g_free(aclname);
     }
 #endif
 
-    /*
-     * Combinations we support here:
-     *
-     *  - no-auth                (clear text, no auth)
-     *  - password               (clear text, weak auth)
-     *  - sasl                   (encrypt, good auth *IF* using Kerberos via GSSAPI)
-     *  - tls                    (encrypt, weak anonymous creds, no auth)
-     *  - tls + password         (encrypt, weak anonymous creds, weak auth)
-     *  - tls + sasl             (encrypt, weak anonymous creds, good auth)
-     *  - tls + x509             (encrypt, good x509 creds, no auth)
-     *  - tls + x509 + password  (encrypt, good x509 creds, weak auth)
-     *  - tls + x509 + sasl      (encrypt, good x509 creds, good auth)
-     *
-     * NB1. TLS is a stackable auth scheme.
-     * NB2. the x509 schemes have option to validate a client cert dname
-     */
-    if (password) {
-#ifdef CONFIG_VNC_TLS
-        if (tls) {
-            vs->auth = VNC_AUTH_VENCRYPT;
-            if (x509) {
-                VNC_DEBUG("Initializing VNC server with x509 password auth\n");
-                vs->subauth = VNC_AUTH_VENCRYPT_X509VNC;
-            } else {
-                VNC_DEBUG("Initializing VNC server with TLS password auth\n");
-                vs->subauth = VNC_AUTH_VENCRYPT_TLSVNC;
-            }
-        } else {
-#endif /* CONFIG_VNC_TLS */
-            VNC_DEBUG("Initializing VNC server with password auth\n");
-            vs->auth = VNC_AUTH_VNC;
-#ifdef CONFIG_VNC_TLS
-            vs->subauth = VNC_AUTH_INVALID;
-        }
-#endif /* CONFIG_VNC_TLS */
-#ifdef CONFIG_VNC_SASL
-    } else if (sasl) {
-#ifdef CONFIG_VNC_TLS
-        if (tls) {
-            vs->auth = VNC_AUTH_VENCRYPT;
-            if (x509) {
-                VNC_DEBUG("Initializing VNC server with x509 SASL auth\n");
-                vs->subauth = VNC_AUTH_VENCRYPT_X509SASL;
-            } else {
-                VNC_DEBUG("Initializing VNC server with TLS SASL auth\n");
-                vs->subauth = VNC_AUTH_VENCRYPT_TLSSASL;
-            }
-        } else {
-#endif /* CONFIG_VNC_TLS */
-            VNC_DEBUG("Initializing VNC server with SASL auth\n");
-            vs->auth = VNC_AUTH_SASL;
-#ifdef CONFIG_VNC_TLS
-            vs->subauth = VNC_AUTH_INVALID;
-        }
-#endif /* CONFIG_VNC_TLS */
-#endif /* CONFIG_VNC_SASL */
-    } else {
-#ifdef CONFIG_VNC_TLS
-        if (tls) {
-            vs->auth = VNC_AUTH_VENCRYPT;
-            if (x509) {
-                VNC_DEBUG("Initializing VNC server with x509 no auth\n");
-                vs->subauth = VNC_AUTH_VENCRYPT_X509NONE;
-            } else {
-                VNC_DEBUG("Initializing VNC server with TLS no auth\n");
-                vs->subauth = VNC_AUTH_VENCRYPT_TLSNONE;
-            }
-        } else {
-#endif
-            VNC_DEBUG("Initializing VNC server with no auth\n");
-            vs->auth = VNC_AUTH_NONE;
-#ifdef CONFIG_VNC_TLS
-            vs->subauth = VNC_AUTH_INVALID;
-        }
-#endif
+    if (vnc_display_setup_auth(vs, password, sasl, vs->ws_enabled, errp) < 0) {
+        goto fail;
     }
 
 #ifdef CONFIG_VNC_SASL
@@ -3291,83 +3787,151 @@
     }
 #endif
     vs->lock_key_sync = lock_key_sync;
+    vs->key_delay_ms = key_delay_ms;
+
+    device_id = qemu_opt_get(opts, "display");
+    if (device_id) {
+        int head = qemu_opt_get_number(opts, "head", 0);
+        Error *err = NULL;
+
+        con = qemu_console_lookup_by_device_name(device_id, head, &err);
+        if (err) {
+            error_propagate(errp, err);
+            goto fail;
+        }
+    } else {
+        con = NULL;
+    }
+
+    if (con != vs->dcl.con) {
+        unregister_displaychangelistener(&vs->dcl);
+        vs->dcl.con = con;
+        register_displaychangelistener(&vs->dcl);
+    }
 
     if (reverse) {
         /* connect to viewer */
-        int csock;
-        vs->lsock = -1;
-#ifdef CONFIG_VNC_WS
-        vs->lwebsock = -1;
-#endif
-        if (strncmp(display, "unix:", 5) == 0) {
-            csock = unix_connect(display+5, errp);
-        } else {
-            csock = inet_connect(display, errp);
-        }
-        if (csock < 0) {
+        QIOChannelSocket *sioc = NULL;
+        vs->lsock = NULL;
+        vs->lwebsock = NULL;
+        if (vs->ws_enabled) {
+            error_setg(errp, "Cannot use websockets in reverse mode");
             goto fail;
         }
-        vnc_connect(vs, csock, false, false);
+        vs->is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
+        sioc = qio_channel_socket_new();
+        if (qio_channel_socket_connect_sync(sioc, saddr, errp) < 0) {
+            goto fail;
+        }
+        vnc_connect(vs, sioc, false, false);
+        object_unref(OBJECT(sioc));
     } else {
-        /* listen for connects */
-        char *dpy;
-        dpy = g_malloc(256);
-        if (strncmp(display, "unix:", 5) == 0) {
-            pstrcpy(dpy, 256, "unix:");
-            vs->lsock = unix_listen(display+5, dpy+5, 256-5, errp);
-        } else {
-            vs->lsock = inet_listen(display, dpy, 256,
-                                    SOCK_STREAM, 5900, errp);
-            if (vs->lsock < 0) {
-                g_free(dpy);
+        vs->lsock = qio_channel_socket_new();
+        if (qio_channel_socket_listen_sync(vs->lsock, saddr, errp) < 0) {
+            goto fail;
+        }
+        vs->is_unix = saddr->type == SOCKET_ADDRESS_KIND_UNIX;
+        vs->enabled = true;
+
+        if (vs->ws_enabled) {
+            vs->lwebsock = qio_channel_socket_new();
+            if (qio_channel_socket_listen_sync(vs->lwebsock,
+                                               wsaddr, errp) < 0) {
+                object_unref(OBJECT(vs->lsock));
+                vs->lsock = NULL;
                 goto fail;
             }
-#ifdef CONFIG_VNC_WS
-            if (vs->websocket) {
-                if (vs->ws_display) {
-                    vs->lwebsock = inet_listen(vs->ws_display, NULL, 256,
-                        SOCK_STREAM, 0, errp);
-                } else {
-                    vs->lwebsock = inet_listen(vs->display, NULL, 256,
-                        SOCK_STREAM, 5700, errp);
-                }
+        }
 
-                if (vs->lwebsock < 0) {
-                    if (vs->lsock) {
-                        close(vs->lsock);
-                        vs->lsock = -1;
-                    }
-                    g_free(dpy);
-                    goto fail;
-                }
-            }
-#endif /* CONFIG_VNC_WS */
+        vs->lsock_tag = qio_channel_add_watch(
+            QIO_CHANNEL(vs->lsock),
+            G_IO_IN, vnc_listen_io, vs, NULL);
+        if (vs->ws_enabled) {
+            vs->lwebsock_tag = qio_channel_add_watch(
+                QIO_CHANNEL(vs->lwebsock),
+                G_IO_IN, vnc_listen_io, vs, NULL);
         }
-        g_free(vs->display);
-        vs->display = dpy;
-        qemu_set_fd_handler2(vs->lsock, NULL,
-                vnc_listen_regular_read, NULL, vs);
-#ifdef CONFIG_VNC_WS
-        if (vs->websocket) {
-            qemu_set_fd_handler2(vs->lwebsock, NULL,
-                    vnc_listen_websocket_read, NULL, vs);
-        }
-#endif /* CONFIG_VNC_WS */
     }
+
+    if (show_vnc_port) {
+        vnc_display_print_local_addr(vs);
+    }
+
+    qapi_free_SocketAddress(saddr);
+    qapi_free_SocketAddress(wsaddr);
     return;
 
 fail:
-    g_free(vs->display);
-    vs->display = NULL;
-#ifdef CONFIG_VNC_WS
-    g_free(vs->ws_display);
-    vs->ws_display = NULL;
-#endif /* CONFIG_VNC_WS */
+    qapi_free_SocketAddress(saddr);
+    qapi_free_SocketAddress(wsaddr);
+    vs->enabled = false;
+    vs->ws_enabled = false;
 }
 
-void vnc_display_add_client(DisplayState *ds, int csock, bool skipauth)
+void vnc_display_add_client(const char *id, int csock, bool skipauth)
 {
-    VncDisplay *vs = vnc_display;
+    VncDisplay *vs = vnc_display_find(id);
+    QIOChannelSocket *sioc;
 
-    vnc_connect(vs, csock, skipauth, false);
+    if (!vs) {
+        return;
+    }
+
+    sioc = qio_channel_socket_new_fd(csock, NULL);
+    if (sioc) {
+        vnc_connect(vs, sioc, skipauth, false);
+        object_unref(OBJECT(sioc));
+    }
 }
+
+static void vnc_auto_assign_id(QemuOptsList *olist, QemuOpts *opts)
+{
+    int i = 2;
+    char *id;
+
+    id = g_strdup("default");
+    while (qemu_opts_find(olist, id)) {
+        g_free(id);
+        id = g_strdup_printf("vnc%d", i++);
+    }
+    qemu_opts_set_id(opts, id);
+}
+
+QemuOpts *vnc_parse(const char *str, Error **errp)
+{
+    QemuOptsList *olist = qemu_find_opts("vnc");
+    QemuOpts *opts = qemu_opts_parse(olist, str, true, errp);
+    const char *id;
+
+    if (!opts) {
+        return NULL;
+    }
+
+    id = qemu_opts_id(opts);
+    if (!id) {
+        /* auto-assign id if not present */
+        vnc_auto_assign_id(olist, opts);
+    }
+    return opts;
+}
+
+int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp)
+{
+    Error *local_err = NULL;
+    char *id = (char *)qemu_opts_id(opts);
+
+    assert(id);
+    vnc_display_init(id);
+    vnc_display_open(id, &local_err);
+    if (local_err != NULL) {
+        error_reportf_err(local_err, "Failed to start VNC server: ");
+        exit(1);
+    }
+    return 0;
+}
+
+static void vnc_register_config(void)
+{
+    qemu_add_opts(&qemu_vnc_opts);
+}
+opts_init(vnc_register_config);
diff --git a/ui/vnc.h b/ui/vnc.h
index 334de9d..ab5f244 100644
--- a/ui/vnc.h
+++ b/ui/vnc.h
@@ -24,8 +24,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef __QEMU_VNC_H
-#define __QEMU_VNC_H
+#ifndef QEMU_VNC_H
+#define QEMU_VNC_H
 
 #include "qemu-common.h"
 #include "qemu/queue.h"
@@ -33,8 +33,11 @@
 #include "ui/console.h"
 #include "audio/audio.h"
 #include "qemu/bitmap.h"
+#include "crypto/tlssession.h"
+#include "qemu/buffer.h"
+#include "io/channel-socket.h"
+#include "io/channel-tls.h"
 #include <zlib.h>
-#include <stdbool.h>
 
 #include "keymaps.h"
 #include "vnc-palette.h"
@@ -55,13 +58,6 @@
  *
  *****************************************************************************/
 
-typedef struct Buffer
-{
-    size_t capacity;
-    size_t offset;
-    uint8_t *buffer;
-} Buffer;
-
 typedef struct VncState VncState;
 typedef struct VncJob VncJob;
 typedef struct VncRect VncRect;
@@ -101,16 +97,11 @@
 
 typedef struct VncDisplay VncDisplay;
 
-#ifdef CONFIG_VNC_TLS
-#include "vnc-tls.h"
 #include "vnc-auth-vencrypt.h"
-#endif
 #ifdef CONFIG_VNC_SASL
 #include "vnc-auth-sasl.h"
 #endif
-#ifdef CONFIG_VNC_WS
 #include "vnc-ws.h"
-#endif
 
 struct VncRectStat
 {
@@ -150,18 +141,21 @@
 struct VncDisplay
 {
     QTAILQ_HEAD(, VncState) clients;
+    int num_connecting;
+    int num_shared;
     int num_exclusive;
+    int connections_limit;
     VncSharePolicy share_policy;
-    int lsock;
-#ifdef CONFIG_VNC_WS
-    int lwebsock;
-    bool websocket;
-    char *ws_display;
-#endif
+    QIOChannelSocket *lsock;
+    guint lsock_tag;
+    QIOChannelSocket *lwebsock;
+    guint lwebsock_tag;
+    bool ws_enabled;
     DisplaySurface *ds;
     DisplayChangeListener dcl;
     kbd_layout_t *kbd_layout;
     int lock_key_sync;
+    int key_delay_ms;
     QemuMutex mutex;
 
     QEMUCursor *cursor;
@@ -171,16 +165,20 @@
     struct VncSurface guest;   /* guest visible surface (aka ds->surface) */
     pixman_image_t *server;    /* vnc server surface */
 
-    char *display;
+    const char *id;
+    QTAILQ_ENTRY(VncDisplay) next;
+    bool enabled;
+    bool is_unix;
     char *password;
     time_t expires;
     int auth;
+    int subauth; /* Used by VeNCrypt */
+    int ws_auth; /* Used by websockets */
+    bool ws_tls; /* Used by websockets */
     bool lossy;
     bool non_adaptive;
-#ifdef CONFIG_VNC_TLS
-    int subauth; /* Used by VeNCrypt */
-    VncDisplayTLS tls;
-#endif
+    QCryptoTLSCreds *tlscreds;
+    char *tlsaclname;
 #ifdef CONFIG_VNC_SASL
     VncDisplaySASL sasl;
 #endif
@@ -254,7 +252,10 @@
 
 struct VncState
 {
-    int csock;
+    QIOChannelSocket *sioc; /* The underlying socket */
+    QIOChannel *ioc; /* The channel currently used for I/O */
+    guint ioc_tag;
+    gboolean disconnecting;
 
     DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT], VNC_DIRTY_BITS);
     uint8_t **lossy_rect; /* Not an Array to avoid costly memcpy in
@@ -279,30 +280,19 @@
     int minor;
 
     int auth;
-    char challenge[VNC_AUTH_CHALLENGE_SIZE];
-#ifdef CONFIG_VNC_TLS
     int subauth; /* Used by VeNCrypt */
-    VncStateTLS tls;
-#endif
+    char challenge[VNC_AUTH_CHALLENGE_SIZE];
+    QCryptoTLSSession *tls; /* Borrowed pointer from channel, don't free */
 #ifdef CONFIG_VNC_SASL
     VncStateSASL sasl;
 #endif
-#ifdef CONFIG_VNC_WS
-#ifdef CONFIG_VNC_TLS
-    VncStateTLS ws_tls;
-#endif /* CONFIG_VNC_TLS */
     bool encode_ws;
     bool websocket;
-#endif /* CONFIG_VNC_WS */
 
     VncClientInfo *info;
 
     Buffer output;
     Buffer input;
-#ifdef CONFIG_VNC_WS
-    Buffer ws_input;
-    Buffer ws_output;
-#endif
     /* current output mode information */
     VncWritePixels *write_pixels;
     PixelFormat client_pf;
@@ -512,11 +502,12 @@
  *****************************************************************************/
 
 /* Event loop functions */
-void vnc_client_read(void *opaque);
-void vnc_client_write(void *opaque);
+gboolean vnc_client_io(QIOChannel *ioc,
+                       GIOCondition condition,
+                       void *opaque);
 
-long vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen);
-long vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen);
+ssize_t vnc_client_read_buf(VncState *vs, uint8_t *data, size_t datalen);
+ssize_t vnc_client_write_buf(VncState *vs, const uint8_t *data, size_t datalen);
 
 /* Protocol I/O functions */
 void vnc_write(VncState *vs, const void *data, size_t len);
@@ -535,25 +526,14 @@
 
 /* Protocol stage functions */
 void vnc_client_error(VncState *vs);
-int vnc_client_io_error(VncState *vs, int ret, int last_errno);
+ssize_t vnc_client_io_error(VncState *vs, ssize_t ret, Error **errp);
 
 void start_client_init(VncState *vs);
 void start_auth_vnc(VncState *vs);
 
-/* Buffer management */
-void buffer_reserve(Buffer *buffer, size_t len);
-void buffer_reset(Buffer *buffer);
-void buffer_free(Buffer *buffer);
-void buffer_append(Buffer *buffer, const void *data, size_t len);
-void buffer_advance(Buffer *buf, size_t len);
-uint8_t *buffer_end(Buffer *buffer);
-
 
 /* Misc helpers */
 
-char *vnc_socket_local_addr(const char *format, int fd);
-char *vnc_socket_remote_addr(const char *format, int fd);
-
 static inline uint32_t vnc_has_feature(VncState *vs, int feature) {
     return (vs->features & (1 << feature));
 }
@@ -597,4 +577,4 @@
 int vnc_zywrle_send_framebuffer_update(VncState *vs, int x, int y, int w, int h);
 void vnc_zrle_clear(VncState *vs);
 
-#endif /* __QEMU_VNC_H */
+#endif /* QEMU_VNC_H */
diff --git a/ui/vnc_keysym.h b/ui/vnc_keysym.h
index 1dc039f..7fa2bc1 100644
--- a/ui/vnc_keysym.h
+++ b/ui/vnc_keysym.h
@@ -404,6 +404,7 @@
 {"breve",                         0x01a2},  /* U+02D8 BREVE */
 {"caron",                         0x01b7},  /* U+02C7 CARON */
 {"Ccaron",                        0x01c8},  /* U+010C LATIN CAPITAL LETTER C WITH CARON */
+{"numerosign",                    0x06b0},  /* U+2116 NUMERO SIGN */
 {"Cyrillic_a",                    0x06c1},  /* U+0430 CYRILLIC SMALL LETTER A */
 {"Cyrillic_A",                    0x06e1},  /* U+0410 CYRILLIC CAPITAL LETTER A */
 {"Cyrillic_be",                   0x06c2},  /* U+0431 CYRILLIC SMALL LETTER BE */
diff --git a/ui/x_keymap.c b/ui/x_keymap.c
index b9b0944..2788485 100644
--- a/ui/x_keymap.c
+++ b/ui/x_keymap.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "x_keymap.h"
 
@@ -94,7 +95,7 @@
  */
 
 static const uint8_t evdev_keycode_to_pc_keycode[61] = {
-    0,         /*  97 EVDEV - RO   ("Internet" Keyboards) */
+    0x73,      /*  97 EVDEV - RO   ("Internet" Keyboards) */
     0,         /*  98 EVDEV - KATA (Katakana) */
     0,         /*  99 EVDEV - HIRA (Hiragana) */
     0x79,      /* 100 EVDEV - HENK (Henkan) */
@@ -126,7 +127,7 @@
     0,         /* 126 EVDEV - I126 ("Internet" Keyboards) */
     0,         /* 127 EVDEV - PAUS */
     0,         /* 128 EVDEV - ???? */
-    0,         /* 129 EVDEV - I129 ("Internet" Keyboards) */
+    0x7e,      /* 129 EVDEV - KP_COMMA (brazilian) */
     0xf1,      /* 130 EVDEV - HNGL (Korean Hangul Latin toggle) */
     0xf2,      /* 131 EVDEV - HJCV (Korean Hangul Hanja toggle) */
     0x7d,      /* 132 AE13 (Yen)*/
diff --git a/user-exec.c b/user-exec.c
index 1ff8673..95f9f97 100644
--- a/user-exec.c
+++ b/user-exec.c
@@ -16,12 +16,14 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
-#include "config.h"
+#include "qemu/osdep.h"
 #include "cpu.h"
 #include "disas/disas.h"
+#include "exec/exec-all.h"
 #include "tcg.h"
 #include "qemu/bitops.h"
 #include "exec/cpu_ldst.h"
+#include "translate-all.h"
 
 #undef EAX
 #undef ECX
@@ -32,50 +34,20 @@
 #undef ESI
 #undef EDI
 #undef EIP
-#include <signal.h>
 #ifdef __linux__
 #include <sys/ucontext.h>
 #endif
 
 //#define DEBUG_SIGNAL
 
-static void exception_action(CPUState *cpu)
-{
-#if defined(TARGET_I386)
-    X86CPU *x86_cpu = X86_CPU(cpu);
-    CPUX86State *env1 = &x86_cpu->env;
-
-    raise_exception_err(env1, cpu->exception_index, env1->error_code);
-#else
-    cpu_loop_exit(cpu);
-#endif
-}
-
 /* exit the current TB from a signal handler. The host registers are
    restored in a state compatible with the CPU emulator
  */
-void cpu_resume_from_signal(CPUState *cpu, void *puc)
+static void cpu_exit_tb_from_sighandler(CPUState *cpu, sigset_t *old_set)
 {
-#ifdef __linux__
-    struct ucontext *uc = puc;
-#elif defined(__OpenBSD__)
-    struct sigcontext *uc = puc;
-#endif
-
-    if (puc) {
-        /* XXX: use siglongjmp ? */
-#ifdef __linux__
-#ifdef __ia64
-        sigprocmask(SIG_SETMASK, (sigset_t *)&uc->uc_sigmask, NULL);
-#else
-        sigprocmask(SIG_SETMASK, &uc->uc_sigmask, NULL);
-#endif
-#elif defined(__OpenBSD__)
-        sigprocmask(SIG_SETMASK, &uc->sc_mask, NULL);
-#endif
-    }
-    cpu->exception_index = -1;
-    siglongjmp(cpu->jmp_env, 1);
+    /* XXX: use siglongjmp ? */
+    sigprocmask(SIG_SETMASK, old_set, NULL);
+    cpu_loop_exit_noexc(cpu);
 }
 
 /* 'pc' is the host PC at which the exception was raised. 'address' is
@@ -83,21 +55,39 @@
    write caused the exception and otherwise 0'. 'old_set' is the
    signal set which should be restored */
 static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
-                                    int is_write, sigset_t *old_set,
-                                    void *puc)
+                                    int is_write, sigset_t *old_set)
 {
     CPUState *cpu;
     CPUClass *cc;
     int ret;
 
 #if defined(DEBUG_SIGNAL)
-    qemu_printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
-                pc, address, is_write, *(unsigned long *)old_set);
+    printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
+           pc, address, is_write, *(unsigned long *)old_set);
 #endif
     /* XXX: locking issue */
-    if (is_write && h2g_valid(address)
-        && page_unprotect(h2g(address), pc, puc)) {
-        return 1;
+    if (is_write && h2g_valid(address)) {
+        switch (page_unprotect(h2g(address), pc)) {
+        case 0:
+            /* Fault not caused by a page marked unwritable to protect
+             * cached translations, must be the guest binary's problem
+             */
+            break;
+        case 1:
+            /* Fault caused by protection of cached translation; TBs
+             * invalidated, so resume execution
+             */
+            return 1;
+        case 2:
+            /* Fault caused by protection of cached translation, and the
+             * currently executing TB was modified and must be exited
+             * immediately.
+             */
+            cpu_exit_tb_from_sighandler(current_cpu, old_set);
+            g_assert_not_reached();
+        default:
+            g_assert_not_reached();
+        }
     }
 
     /* Convert forcefully to guest address space, invalid addresses
@@ -118,10 +108,8 @@
     /* now we have a real cpu fault */
     cpu_restore_state(cpu, pc);
 
-    /* we restore the process signal mask as the sigreturn should
-       do it (XXX: use sigsetjmp) */
     sigprocmask(SIG_SETMASK, old_set, NULL);
-    exception_action(cpu);
+    cpu_loop_exit(cpu);
 
     /* never comes here */
     return 1;
@@ -129,14 +117,7 @@
 
 #if defined(__i386__)
 
-#if defined(__APPLE__)
-#include <sys/ucontext.h>
-
-#define EIP_sig(context)  (*((unsigned long *)&(context)->uc_mcontext->ss.eip))
-#define TRAP_sig(context)    ((context)->uc_mcontext->es.trapno)
-#define ERROR_sig(context)   ((context)->uc_mcontext->es.err)
-#define MASK_sig(context)    ((context)->uc_sigmask)
-#elif defined(__NetBSD__)
+#if defined(__NetBSD__)
 #include <ucontext.h>
 
 #define EIP_sig(context)     ((context)->uc_mcontext.__gregs[_REG_EIP])
@@ -187,7 +168,7 @@
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
                              trapno == 0xe ?
                              (ERROR_sig(uc) >> 1) & 1 : 0,
-                             &MASK_sig(uc), puc);
+                             &MASK_sig(uc));
 }
 
 #elif defined(__x86_64__)
@@ -233,7 +214,7 @@
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
                              TRAP_sig(uc) == 0xe ?
                              (ERROR_sig(uc) >> 1) & 1 : 0,
-                             &MASK_sig(uc), puc);
+                             &MASK_sig(uc));
 }
 
 #elif defined(_ARCH_PPC)
@@ -286,44 +267,6 @@
 #define TRAP_sig(context)              ((context)->uc_mcontext.mc_exc)
 #endif /* __FreeBSD__|| __FreeBSD_kernel__ */
 
-#ifdef __APPLE__
-#include <sys/ucontext.h>
-typedef struct ucontext SIGCONTEXT;
-/* All Registers access - only for local access */
-#define REG_sig(reg_name, context)              \
-    ((context)->uc_mcontext->ss.reg_name)
-#define FLOATREG_sig(reg_name, context)         \
-    ((context)->uc_mcontext->fs.reg_name)
-#define EXCEPREG_sig(reg_name, context)         \
-    ((context)->uc_mcontext->es.reg_name)
-#define VECREG_sig(reg_name, context)           \
-    ((context)->uc_mcontext->vs.reg_name)
-/* Gpr Registers access */
-#define GPR_sig(reg_num, context)              REG_sig(r##reg_num, context)
-/* Program counter */
-#define IAR_sig(context)                       REG_sig(srr0, context)
-/* Machine State Register (Supervisor) */
-#define MSR_sig(context)                       REG_sig(srr1, context)
-#define CTR_sig(context)                       REG_sig(ctr, context)
-/* Link register */
-#define XER_sig(context)                       REG_sig(xer, context)
-/* User's integer exception register */
-#define LR_sig(context)                        REG_sig(lr, context)
-/* Condition register */
-#define CR_sig(context)                        REG_sig(cr, context)
-/* Float Registers access */
-#define FLOAT_sig(reg_num, context)             \
-    FLOATREG_sig(fpregs[reg_num], context)
-#define FPSCR_sig(context)                      \
-    ((double)FLOATREG_sig(fpscr, context))
-/* Exception Registers access */
-/* Fault registers for coredump */
-#define DAR_sig(context)                       EXCEPREG_sig(dar, context)
-#define DSISR_sig(context)                     EXCEPREG_sig(dsisr, context)
-/* number of powerpc exception taken */
-#define TRAP_sig(context)                      EXCEPREG_sig(exception, context)
-#endif /* __APPLE__ */
-
 int cpu_signal_handler(int host_signum, void *pinfo,
                        void *puc)
 {
@@ -349,7 +292,7 @@
     }
 #endif
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write, &uc->uc_sigmask, puc);
+                             is_write, &uc->uc_sigmask);
 }
 
 #elif defined(__alpha__)
@@ -380,7 +323,7 @@
     }
 
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write, &uc->uc_sigmask, puc);
+                             is_write, &uc->uc_sigmask);
 }
 #elif defined(__sparc__)
 
@@ -404,6 +347,10 @@
     struct sigcontext *uc = puc;
     unsigned long pc = uc->sc_pc;
     void *sigmask = (void *)(long)uc->sc_mask;
+#elif defined(__NetBSD__)
+    ucontext_t *uc = puc;
+    unsigned long pc = _UC_MACHINE_PC(uc);
+    void *sigmask = (void *)&uc->uc_sigmask;
 #endif
 #endif
 
@@ -436,20 +383,30 @@
         }
     }
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write, sigmask, NULL);
+                             is_write, sigmask);
 }
 
 #elif defined(__arm__)
 
+#if defined(__NetBSD__)
+#include <ucontext.h>
+#endif
+
 int cpu_signal_handler(int host_signum, void *pinfo,
                        void *puc)
 {
     siginfo_t *info = pinfo;
+#if defined(__NetBSD__)
+    ucontext_t *uc = puc;
+#else
     struct ucontext *uc = puc;
+#endif
     unsigned long pc;
     int is_write;
 
-#if defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
+#if defined(__NetBSD__)
+    pc = uc->uc_mcontext.__gregs[_REG_R15];
+#elif defined(__GLIBC__) && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 3))
     pc = uc->uc_mcontext.gregs[R15];
 #else
     pc = uc->uc_mcontext.arm_pc;
@@ -461,7 +418,7 @@
     is_write = extract32(uc->uc_mcontext.error_code, 11, 1);
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
                              is_write,
-                             &uc->uc_sigmask, puc);
+                             &uc->uc_sigmask);
 }
 
 #elif defined(__aarch64__)
@@ -489,25 +446,7 @@
                 || (insn & 0x3a400000) == 0x28000000); /* C3.3.7,14-16 */
 
     return handle_cpu_signal(pc, (uintptr_t)info->si_addr,
-                             is_write, &uc->uc_sigmask, puc);
-}
-
-#elif defined(__mc68000)
-
-int cpu_signal_handler(int host_signum, void *pinfo,
-                       void *puc)
-{
-    siginfo_t *info = pinfo;
-    struct ucontext *uc = puc;
-    unsigned long pc;
-    int is_write;
-
-    pc = uc->uc_mcontext.gregs[16];
-    /* XXX: compute is_write */
-    is_write = 0;
-    return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write,
-                             &uc->uc_sigmask, puc);
+                             is_write, &uc->uc_sigmask);
 }
 
 #elif defined(__ia64)
@@ -542,7 +481,7 @@
     }
     return handle_cpu_signal(ip, (unsigned long)info->si_addr,
                              is_write,
-                             (sigset_t *)&uc->uc_sigmask, puc);
+                             (sigset_t *)&uc->uc_sigmask);
 }
 
 #elif defined(__s390__)
@@ -595,7 +534,7 @@
         break;
     }
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write, &uc->uc_sigmask, puc);
+                             is_write, &uc->uc_sigmask);
 }
 
 #elif defined(__mips__)
@@ -611,49 +550,7 @@
     /* XXX: compute is_write */
     is_write = 0;
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write, &uc->uc_sigmask, puc);
-}
-
-#elif defined(__hppa__)
-
-int cpu_signal_handler(int host_signum, void *pinfo,
-                       void *puc)
-{
-    siginfo_t *info = pinfo;
-    struct ucontext *uc = puc;
-    unsigned long pc = uc->uc_mcontext.sc_iaoq[0];
-    uint32_t insn = *(uint32_t *)pc;
-    int is_write = 0;
-
-    /* XXX: need kernel patch to get write flag faster.  */
-    switch (insn >> 26) {
-    case 0x1a: /* STW */
-    case 0x19: /* STH */
-    case 0x18: /* STB */
-    case 0x1b: /* STWM */
-        is_write = 1;
-        break;
-
-    case 0x09: /* CSTWX, FSTWX, FSTWS */
-    case 0x0b: /* CSTDX, FSTDX, FSTDS */
-        /* Distinguish from coprocessor load ... */
-        is_write = (insn >> 9) & 1;
-        break;
-
-    case 0x03:
-        switch ((insn >> 6) & 15) {
-        case 0xa: /* STWS */
-        case 0x9: /* STHS */
-        case 0x8: /* STBS */
-        case 0xe: /* STWAS */
-        case 0xc: /* STBYS */
-            is_write = 1;
-        }
-        break;
-    }
-
-    return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             is_write, &uc->uc_sigmask, puc);
+                             is_write, &uc->uc_sigmask);
 }
 
 #else
diff --git a/util/Makefile.objs b/util/Makefile.objs
index 4633835..6180a78 100644
--- a/util/Makefile.objs
+++ b/util/Makefile.objs
@@ -1,15 +1,23 @@
 util-obj-y = osdep.o cutils.o unicode.o qemu-timer-common.o
-util-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o event_notifier-win32.o
-util-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o event_notifier-posix.o qemu-openpty.o
+util-obj-$(CONFIG_POSIX) += compatfd.o
+util-obj-$(CONFIG_POSIX) += event_notifier-posix.o
+util-obj-$(CONFIG_POSIX) += mmap-alloc.o
+util-obj-$(CONFIG_POSIX) += oslib-posix.o
+util-obj-$(CONFIG_POSIX) += qemu-openpty.o
+util-obj-$(CONFIG_POSIX) += qemu-thread-posix.o
+util-obj-$(CONFIG_WIN32) += event_notifier-win32.o
+util-obj-$(CONFIG_POSIX) += memfd.o
+util-obj-$(CONFIG_WIN32) += oslib-win32.o
+util-obj-$(CONFIG_WIN32) += qemu-thread-win32.o
+util-obj-y += abort.o
 util-obj-y += envlist.o path.o module.o
 util-obj-$(call lnot,$(CONFIG_INT128)) += host-utils.o
 util-obj-y += bitmap.o bitops.o hbitmap.o
 util-obj-y += fifo8.o
 util-obj-y += acl.o
 util-obj-y += error.o qemu-error.o
-util-obj-$(CONFIG_POSIX) += compatfd.o
 util-obj-y += id.o
-util-obj-y += iov.o aes.o qemu-config.o qemu-sockets.o uri.o notify.o
+util-obj-y += iov.o qemu-config.o qemu-sockets.o uri.o notify.o
 util-obj-y += qemu-option.o qemu-progress.o
 util-obj-y += hexdump.o
 util-obj-y += crc32c.o
@@ -17,5 +25,14 @@
 util-obj-y += getauxval.o
 util-obj-y += readline.o
 util-obj-y += rfifolock.o
-util-obj-$(CONFIG_POSIX) += shared-library-posix.o
-util-obj-$(CONFIG_WIN32) += shared-library-win32.o
+util-obj-y += rcu.o
+util-obj-y += qemu-coroutine.o qemu-coroutine-lock.o qemu-coroutine-io.o
+util-obj-y += qemu-coroutine-sleep.o
+util-obj-y += coroutine-$(CONFIG_COROUTINE_BACKEND).o
+util-obj-y += buffer.o
+util-obj-y += timed-average.o
+util-obj-y += base64.o
+util-obj-y += log.o
+util-obj-y += qdist.o
+util-obj-y += qht.o
+util-obj-y += range.o
diff --git a/util/abort.c b/util/abort.c
new file mode 100644
index 0000000..74eefcb
--- /dev/null
+++ b/util/abort.c
@@ -0,0 +1,49 @@
+/*
+ * QEMU abort handler
+ *
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/abort.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+static QemuAbortHandler qemu_abort_handler = NULL;
+
+void QEMU_NORETURN qemu_abort(const char* format, ...) {
+    va_list args;
+    va_start(args, format);
+    if (qemu_abort_handler) {
+        (*qemu_abort_handler)(format, args);
+    } else {
+        vfprintf(stderr, format, args);
+    }
+    va_end(args);
+    abort();
+}
+
+void qemu_abort_set_handler(QemuAbortHandler abort_handler) {
+    qemu_abort_handler = abort_handler;
+}
+
+bool qemu_abort_has_custom_handler(void) {
+    return qemu_abort_handler != NULL;
+}
diff --git a/util/acl.c b/util/acl.c
index 571d686..c105add 100644
--- a/util/acl.c
+++ b/util/acl.c
@@ -23,6 +23,7 @@
  */
 
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/acl.h"
 
@@ -176,12 +177,3 @@
     }
     return -1;
 }
-
-
-/*
- * Local variables:
- *  c-indent-level: 4
- *  c-basic-offset: 4
- *  tab-width: 8
- * End:
- */
diff --git a/util/aes.c b/util/aes.c
deleted file mode 100644
index 6058f19..0000000
--- a/util/aes.c
+++ /dev/null
@@ -1,1652 +0,0 @@
-/**
- *
- * aes.c - integrated in QEMU by Fabrice Bellard from the OpenSSL project.
- */
-/*
- * rijndael-alg-fst.c
- *
- * @version 3.0 (December 2000)
- *
- * Optimised ANSI C code for the Rijndael cipher (now AES)
- *
- * @author Vincent Rijmen <vincent.rijmen@esat.kuleuven.ac.be>
- * @author Antoon Bosselaers <antoon.bosselaers@esat.kuleuven.ac.be>
- * @author Paulo Barreto <paulo.barreto@terra.com.br>
- *
- * This code is hereby placed in the public domain.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS
- * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
- * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include "qemu-common.h"
-#include "qemu/aes.h"
-
-typedef uint32_t u32;
-typedef uint8_t u8;
-
-/* This controls loop-unrolling in aes_core.c */
-#undef FULL_UNROLL
-# define GETU32(pt) (((u32)(pt)[0] << 24) ^ ((u32)(pt)[1] << 16) ^ ((u32)(pt)[2] <<  8) ^ ((u32)(pt)[3]))
-# define PUTU32(ct, st) { (ct)[0] = (u8)((st) >> 24); (ct)[1] = (u8)((st) >> 16); (ct)[2] = (u8)((st) >>  8); (ct)[3] = (u8)(st); }
-
-const uint8_t AES_sbox[256] = {
-    0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5,
-    0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
-    0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0,
-    0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
-    0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC,
-    0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
-    0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A,
-    0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
-    0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0,
-    0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
-    0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B,
-    0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
-    0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85,
-    0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
-    0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5,
-    0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
-    0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17,
-    0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
-    0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88,
-    0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
-    0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C,
-    0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
-    0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9,
-    0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
-    0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6,
-    0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
-    0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E,
-    0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
-    0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94,
-    0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
-    0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68,
-    0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16,
-};
-
-const uint8_t AES_isbox[256] = {
-    0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38,
-    0xBF, 0x40, 0xA3, 0x9E, 0x81, 0xF3, 0xD7, 0xFB,
-    0x7C, 0xE3, 0x39, 0x82, 0x9B, 0x2F, 0xFF, 0x87,
-    0x34, 0x8E, 0x43, 0x44, 0xC4, 0xDE, 0xE9, 0xCB,
-    0x54, 0x7B, 0x94, 0x32, 0xA6, 0xC2, 0x23, 0x3D,
-    0xEE, 0x4C, 0x95, 0x0B, 0x42, 0xFA, 0xC3, 0x4E,
-    0x08, 0x2E, 0xA1, 0x66, 0x28, 0xD9, 0x24, 0xB2,
-    0x76, 0x5B, 0xA2, 0x49, 0x6D, 0x8B, 0xD1, 0x25,
-    0x72, 0xF8, 0xF6, 0x64, 0x86, 0x68, 0x98, 0x16,
-    0xD4, 0xA4, 0x5C, 0xCC, 0x5D, 0x65, 0xB6, 0x92,
-    0x6C, 0x70, 0x48, 0x50, 0xFD, 0xED, 0xB9, 0xDA,
-    0x5E, 0x15, 0x46, 0x57, 0xA7, 0x8D, 0x9D, 0x84,
-    0x90, 0xD8, 0xAB, 0x00, 0x8C, 0xBC, 0xD3, 0x0A,
-    0xF7, 0xE4, 0x58, 0x05, 0xB8, 0xB3, 0x45, 0x06,
-    0xD0, 0x2C, 0x1E, 0x8F, 0xCA, 0x3F, 0x0F, 0x02,
-    0xC1, 0xAF, 0xBD, 0x03, 0x01, 0x13, 0x8A, 0x6B,
-    0x3A, 0x91, 0x11, 0x41, 0x4F, 0x67, 0xDC, 0xEA,
-    0x97, 0xF2, 0xCF, 0xCE, 0xF0, 0xB4, 0xE6, 0x73,
-    0x96, 0xAC, 0x74, 0x22, 0xE7, 0xAD, 0x35, 0x85,
-    0xE2, 0xF9, 0x37, 0xE8, 0x1C, 0x75, 0xDF, 0x6E,
-    0x47, 0xF1, 0x1A, 0x71, 0x1D, 0x29, 0xC5, 0x89,
-    0x6F, 0xB7, 0x62, 0x0E, 0xAA, 0x18, 0xBE, 0x1B,
-    0xFC, 0x56, 0x3E, 0x4B, 0xC6, 0xD2, 0x79, 0x20,
-    0x9A, 0xDB, 0xC0, 0xFE, 0x78, 0xCD, 0x5A, 0xF4,
-    0x1F, 0xDD, 0xA8, 0x33, 0x88, 0x07, 0xC7, 0x31,
-    0xB1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xEC, 0x5F,
-    0x60, 0x51, 0x7F, 0xA9, 0x19, 0xB5, 0x4A, 0x0D,
-    0x2D, 0xE5, 0x7A, 0x9F, 0x93, 0xC9, 0x9C, 0xEF,
-    0xA0, 0xE0, 0x3B, 0x4D, 0xAE, 0x2A, 0xF5, 0xB0,
-    0xC8, 0xEB, 0xBB, 0x3C, 0x83, 0x53, 0x99, 0x61,
-    0x17, 0x2B, 0x04, 0x7E, 0xBA, 0x77, 0xD6, 0x26,
-    0xE1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0C, 0x7D,
-};
-
-const uint8_t AES_shifts[16] = {
-    0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, 1, 6, 11
-};
-
-const uint8_t AES_ishifts[16] = {
-    0, 13, 10, 7, 4, 1, 14, 11, 8, 5, 2, 15, 12, 9, 6, 3
-};
-
-/* AES_imc[x][0] = [x].[0e, 09, 0d, 0b]; */
-/* AES_imc[x][1] = [x].[0b, 0e, 09, 0d]; */
-/* AES_imc[x][2] = [x].[0d, 0b, 0e, 09]; */
-/* AES_imc[x][3] = [x].[09, 0d, 0b, 0e]; */
-const uint32_t AES_imc[256][4] = {
-    { 0x00000000, 0x00000000, 0x00000000, 0x00000000, }, /* x=00 */
-    { 0x0E090D0B, 0x0B0E090D, 0x0D0B0E09, 0x090D0B0E, }, /* x=01 */
-    { 0x1C121A16, 0x161C121A, 0x1A161C12, 0x121A161C, }, /* x=02 */
-    { 0x121B171D, 0x1D121B17, 0x171D121B, 0x1B171D12, }, /* x=03 */
-    { 0x3824342C, 0x2C382434, 0x342C3824, 0x24342C38, }, /* x=04 */
-    { 0x362D3927, 0x27362D39, 0x3927362D, 0x2D392736, }, /* x=05 */
-    { 0x24362E3A, 0x3A24362E, 0x2E3A2436, 0x362E3A24, }, /* x=06 */
-    { 0x2A3F2331, 0x312A3F23, 0x23312A3F, 0x3F23312A, }, /* x=07 */
-    { 0x70486858, 0x58704868, 0x68587048, 0x48685870, }, /* x=08 */
-    { 0x7E416553, 0x537E4165, 0x65537E41, 0x4165537E, }, /* x=09 */
-    { 0x6C5A724E, 0x4E6C5A72, 0x724E6C5A, 0x5A724E6C, }, /* x=0A */
-    { 0x62537F45, 0x4562537F, 0x7F456253, 0x537F4562, }, /* x=0B */
-    { 0x486C5C74, 0x74486C5C, 0x5C74486C, 0x6C5C7448, }, /* x=0C */
-    { 0x4665517F, 0x7F466551, 0x517F4665, 0x65517F46, }, /* x=0D */
-    { 0x547E4662, 0x62547E46, 0x4662547E, 0x7E466254, }, /* x=0E */
-    { 0x5A774B69, 0x695A774B, 0x4B695A77, 0x774B695A, }, /* x=0F */
-    { 0xE090D0B0, 0xB0E090D0, 0xD0B0E090, 0x90D0B0E0, }, /* x=10 */
-    { 0xEE99DDBB, 0xBBEE99DD, 0xDDBBEE99, 0x99DDBBEE, }, /* x=11 */
-    { 0xFC82CAA6, 0xA6FC82CA, 0xCAA6FC82, 0x82CAA6FC, }, /* x=12 */
-    { 0xF28BC7AD, 0xADF28BC7, 0xC7ADF28B, 0x8BC7ADF2, }, /* x=13 */
-    { 0xD8B4E49C, 0x9CD8B4E4, 0xE49CD8B4, 0xB4E49CD8, }, /* x=14 */
-    { 0xD6BDE997, 0x97D6BDE9, 0xE997D6BD, 0xBDE997D6, }, /* x=15 */
-    { 0xC4A6FE8A, 0x8AC4A6FE, 0xFE8AC4A6, 0xA6FE8AC4, }, /* x=16 */
-    { 0xCAAFF381, 0x81CAAFF3, 0xF381CAAF, 0xAFF381CA, }, /* x=17 */
-    { 0x90D8B8E8, 0xE890D8B8, 0xB8E890D8, 0xD8B8E890, }, /* x=18 */
-    { 0x9ED1B5E3, 0xE39ED1B5, 0xB5E39ED1, 0xD1B5E39E, }, /* x=19 */
-    { 0x8CCAA2FE, 0xFE8CCAA2, 0xA2FE8CCA, 0xCAA2FE8C, }, /* x=1A */
-    { 0x82C3AFF5, 0xF582C3AF, 0xAFF582C3, 0xC3AFF582, }, /* x=1B */
-    { 0xA8FC8CC4, 0xC4A8FC8C, 0x8CC4A8FC, 0xFC8CC4A8, }, /* x=1C */
-    { 0xA6F581CF, 0xCFA6F581, 0x81CFA6F5, 0xF581CFA6, }, /* x=1D */
-    { 0xB4EE96D2, 0xD2B4EE96, 0x96D2B4EE, 0xEE96D2B4, }, /* x=1E */
-    { 0xBAE79BD9, 0xD9BAE79B, 0x9BD9BAE7, 0xE79BD9BA, }, /* x=1F */
-    { 0xDB3BBB7B, 0x7BDB3BBB, 0xBB7BDB3B, 0x3BBB7BDB, }, /* x=20 */
-    { 0xD532B670, 0x70D532B6, 0xB670D532, 0x32B670D5, }, /* x=21 */
-    { 0xC729A16D, 0x6DC729A1, 0xA16DC729, 0x29A16DC7, }, /* x=22 */
-    { 0xC920AC66, 0x66C920AC, 0xAC66C920, 0x20AC66C9, }, /* x=23 */
-    { 0xE31F8F57, 0x57E31F8F, 0x8F57E31F, 0x1F8F57E3, }, /* x=24 */
-    { 0xED16825C, 0x5CED1682, 0x825CED16, 0x16825CED, }, /* x=25 */
-    { 0xFF0D9541, 0x41FF0D95, 0x9541FF0D, 0x0D9541FF, }, /* x=26 */
-    { 0xF104984A, 0x4AF10498, 0x984AF104, 0x04984AF1, }, /* x=27 */
-    { 0xAB73D323, 0x23AB73D3, 0xD323AB73, 0x73D323AB, }, /* x=28 */
-    { 0xA57ADE28, 0x28A57ADE, 0xDE28A57A, 0x7ADE28A5, }, /* x=29 */
-    { 0xB761C935, 0x35B761C9, 0xC935B761, 0x61C935B7, }, /* x=2A */
-    { 0xB968C43E, 0x3EB968C4, 0xC43EB968, 0x68C43EB9, }, /* x=2B */
-    { 0x9357E70F, 0x0F9357E7, 0xE70F9357, 0x57E70F93, }, /* x=2C */
-    { 0x9D5EEA04, 0x049D5EEA, 0xEA049D5E, 0x5EEA049D, }, /* x=2D */
-    { 0x8F45FD19, 0x198F45FD, 0xFD198F45, 0x45FD198F, }, /* x=2E */
-    { 0x814CF012, 0x12814CF0, 0xF012814C, 0x4CF01281, }, /* x=2F */
-    { 0x3BAB6BCB, 0xCB3BAB6B, 0x6BCB3BAB, 0xAB6BCB3B, }, /* x=30 */
-    { 0x35A266C0, 0xC035A266, 0x66C035A2, 0xA266C035, }, /* x=31 */
-    { 0x27B971DD, 0xDD27B971, 0x71DD27B9, 0xB971DD27, }, /* x=32 */
-    { 0x29B07CD6, 0xD629B07C, 0x7CD629B0, 0xB07CD629, }, /* x=33 */
-    { 0x038F5FE7, 0xE7038F5F, 0x5FE7038F, 0x8F5FE703, }, /* x=34 */
-    { 0x0D8652EC, 0xEC0D8652, 0x52EC0D86, 0x8652EC0D, }, /* x=35 */
-    { 0x1F9D45F1, 0xF11F9D45, 0x45F11F9D, 0x9D45F11F, }, /* x=36 */
-    { 0x119448FA, 0xFA119448, 0x48FA1194, 0x9448FA11, }, /* x=37 */
-    { 0x4BE30393, 0x934BE303, 0x03934BE3, 0xE303934B, }, /* x=38 */
-    { 0x45EA0E98, 0x9845EA0E, 0x0E9845EA, 0xEA0E9845, }, /* x=39 */
-    { 0x57F11985, 0x8557F119, 0x198557F1, 0xF1198557, }, /* x=3A */
-    { 0x59F8148E, 0x8E59F814, 0x148E59F8, 0xF8148E59, }, /* x=3B */
-    { 0x73C737BF, 0xBF73C737, 0x37BF73C7, 0xC737BF73, }, /* x=3C */
-    { 0x7DCE3AB4, 0xB47DCE3A, 0x3AB47DCE, 0xCE3AB47D, }, /* x=3D */
-    { 0x6FD52DA9, 0xA96FD52D, 0x2DA96FD5, 0xD52DA96F, }, /* x=3E */
-    { 0x61DC20A2, 0xA261DC20, 0x20A261DC, 0xDC20A261, }, /* x=3F */
-    { 0xAD766DF6, 0xF6AD766D, 0x6DF6AD76, 0x766DF6AD, }, /* x=40 */
-    { 0xA37F60FD, 0xFDA37F60, 0x60FDA37F, 0x7F60FDA3, }, /* x=41 */
-    { 0xB16477E0, 0xE0B16477, 0x77E0B164, 0x6477E0B1, }, /* x=42 */
-    { 0xBF6D7AEB, 0xEBBF6D7A, 0x7AEBBF6D, 0x6D7AEBBF, }, /* x=43 */
-    { 0x955259DA, 0xDA955259, 0x59DA9552, 0x5259DA95, }, /* x=44 */
-    { 0x9B5B54D1, 0xD19B5B54, 0x54D19B5B, 0x5B54D19B, }, /* x=45 */
-    { 0x894043CC, 0xCC894043, 0x43CC8940, 0x4043CC89, }, /* x=46 */
-    { 0x87494EC7, 0xC787494E, 0x4EC78749, 0x494EC787, }, /* x=47 */
-    { 0xDD3E05AE, 0xAEDD3E05, 0x05AEDD3E, 0x3E05AEDD, }, /* x=48 */
-    { 0xD33708A5, 0xA5D33708, 0x08A5D337, 0x3708A5D3, }, /* x=49 */
-    { 0xC12C1FB8, 0xB8C12C1F, 0x1FB8C12C, 0x2C1FB8C1, }, /* x=4A */
-    { 0xCF2512B3, 0xB3CF2512, 0x12B3CF25, 0x2512B3CF, }, /* x=4B */
-    { 0xE51A3182, 0x82E51A31, 0x3182E51A, 0x1A3182E5, }, /* x=4C */
-    { 0xEB133C89, 0x89EB133C, 0x3C89EB13, 0x133C89EB, }, /* x=4D */
-    { 0xF9082B94, 0x94F9082B, 0x2B94F908, 0x082B94F9, }, /* x=4E */
-    { 0xF701269F, 0x9FF70126, 0x269FF701, 0x01269FF7, }, /* x=4F */
-    { 0x4DE6BD46, 0x464DE6BD, 0xBD464DE6, 0xE6BD464D, }, /* x=50 */
-    { 0x43EFB04D, 0x4D43EFB0, 0xB04D43EF, 0xEFB04D43, }, /* x=51 */
-    { 0x51F4A750, 0x5051F4A7, 0xA75051F4, 0xF4A75051, }, /* x=52 */
-    { 0x5FFDAA5B, 0x5B5FFDAA, 0xAA5B5FFD, 0xFDAA5B5F, }, /* x=53 */
-    { 0x75C2896A, 0x6A75C289, 0x896A75C2, 0xC2896A75, }, /* x=54 */
-    { 0x7BCB8461, 0x617BCB84, 0x84617BCB, 0xCB84617B, }, /* x=55 */
-    { 0x69D0937C, 0x7C69D093, 0x937C69D0, 0xD0937C69, }, /* x=56 */
-    { 0x67D99E77, 0x7767D99E, 0x9E7767D9, 0xD99E7767, }, /* x=57 */
-    { 0x3DAED51E, 0x1E3DAED5, 0xD51E3DAE, 0xAED51E3D, }, /* x=58 */
-    { 0x33A7D815, 0x1533A7D8, 0xD81533A7, 0xA7D81533, }, /* x=59 */
-    { 0x21BCCF08, 0x0821BCCF, 0xCF0821BC, 0xBCCF0821, }, /* x=5A */
-    { 0x2FB5C203, 0x032FB5C2, 0xC2032FB5, 0xB5C2032F, }, /* x=5B */
-    { 0x058AE132, 0x32058AE1, 0xE132058A, 0x8AE13205, }, /* x=5C */
-    { 0x0B83EC39, 0x390B83EC, 0xEC390B83, 0x83EC390B, }, /* x=5D */
-    { 0x1998FB24, 0x241998FB, 0xFB241998, 0x98FB2419, }, /* x=5E */
-    { 0x1791F62F, 0x2F1791F6, 0xF62F1791, 0x91F62F17, }, /* x=5F */
-    { 0x764DD68D, 0x8D764DD6, 0xD68D764D, 0x4DD68D76, }, /* x=60 */
-    { 0x7844DB86, 0x867844DB, 0xDB867844, 0x44DB8678, }, /* x=61 */
-    { 0x6A5FCC9B, 0x9B6A5FCC, 0xCC9B6A5F, 0x5FCC9B6A, }, /* x=62 */
-    { 0x6456C190, 0x906456C1, 0xC1906456, 0x56C19064, }, /* x=63 */
-    { 0x4E69E2A1, 0xA14E69E2, 0xE2A14E69, 0x69E2A14E, }, /* x=64 */
-    { 0x4060EFAA, 0xAA4060EF, 0xEFAA4060, 0x60EFAA40, }, /* x=65 */
-    { 0x527BF8B7, 0xB7527BF8, 0xF8B7527B, 0x7BF8B752, }, /* x=66 */
-    { 0x5C72F5BC, 0xBC5C72F5, 0xF5BC5C72, 0x72F5BC5C, }, /* x=67 */
-    { 0x0605BED5, 0xD50605BE, 0xBED50605, 0x05BED506, }, /* x=68 */
-    { 0x080CB3DE, 0xDE080CB3, 0xB3DE080C, 0x0CB3DE08, }, /* x=69 */
-    { 0x1A17A4C3, 0xC31A17A4, 0xA4C31A17, 0x17A4C31A, }, /* x=6A */
-    { 0x141EA9C8, 0xC8141EA9, 0xA9C8141E, 0x1EA9C814, }, /* x=6B */
-    { 0x3E218AF9, 0xF93E218A, 0x8AF93E21, 0x218AF93E, }, /* x=6C */
-    { 0x302887F2, 0xF2302887, 0x87F23028, 0x2887F230, }, /* x=6D */
-    { 0x223390EF, 0xEF223390, 0x90EF2233, 0x3390EF22, }, /* x=6E */
-    { 0x2C3A9DE4, 0xE42C3A9D, 0x9DE42C3A, 0x3A9DE42C, }, /* x=6F */
-    { 0x96DD063D, 0x3D96DD06, 0x063D96DD, 0xDD063D96, }, /* x=70 */
-    { 0x98D40B36, 0x3698D40B, 0x0B3698D4, 0xD40B3698, }, /* x=71 */
-    { 0x8ACF1C2B, 0x2B8ACF1C, 0x1C2B8ACF, 0xCF1C2B8A, }, /* x=72 */
-    { 0x84C61120, 0x2084C611, 0x112084C6, 0xC6112084, }, /* x=73 */
-    { 0xAEF93211, 0x11AEF932, 0x3211AEF9, 0xF93211AE, }, /* x=74 */
-    { 0xA0F03F1A, 0x1AA0F03F, 0x3F1AA0F0, 0xF03F1AA0, }, /* x=75 */
-    { 0xB2EB2807, 0x07B2EB28, 0x2807B2EB, 0xEB2807B2, }, /* x=76 */
-    { 0xBCE2250C, 0x0CBCE225, 0x250CBCE2, 0xE2250CBC, }, /* x=77 */
-    { 0xE6956E65, 0x65E6956E, 0x6E65E695, 0x956E65E6, }, /* x=78 */
-    { 0xE89C636E, 0x6EE89C63, 0x636EE89C, 0x9C636EE8, }, /* x=79 */
-    { 0xFA877473, 0x73FA8774, 0x7473FA87, 0x877473FA, }, /* x=7A */
-    { 0xF48E7978, 0x78F48E79, 0x7978F48E, 0x8E7978F4, }, /* x=7B */
-    { 0xDEB15A49, 0x49DEB15A, 0x5A49DEB1, 0xB15A49DE, }, /* x=7C */
-    { 0xD0B85742, 0x42D0B857, 0x5742D0B8, 0xB85742D0, }, /* x=7D */
-    { 0xC2A3405F, 0x5FC2A340, 0x405FC2A3, 0xA3405FC2, }, /* x=7E */
-    { 0xCCAA4D54, 0x54CCAA4D, 0x4D54CCAA, 0xAA4D54CC, }, /* x=7F */
-    { 0x41ECDAF7, 0xF741ECDA, 0xDAF741EC, 0xECDAF741, }, /* x=80 */
-    { 0x4FE5D7FC, 0xFC4FE5D7, 0xD7FC4FE5, 0xE5D7FC4F, }, /* x=81 */
-    { 0x5DFEC0E1, 0xE15DFEC0, 0xC0E15DFE, 0xFEC0E15D, }, /* x=82 */
-    { 0x53F7CDEA, 0xEA53F7CD, 0xCDEA53F7, 0xF7CDEA53, }, /* x=83 */
-    { 0x79C8EEDB, 0xDB79C8EE, 0xEEDB79C8, 0xC8EEDB79, }, /* x=84 */
-    { 0x77C1E3D0, 0xD077C1E3, 0xE3D077C1, 0xC1E3D077, }, /* x=85 */
-    { 0x65DAF4CD, 0xCD65DAF4, 0xF4CD65DA, 0xDAF4CD65, }, /* x=86 */
-    { 0x6BD3F9C6, 0xC66BD3F9, 0xF9C66BD3, 0xD3F9C66B, }, /* x=87 */
-    { 0x31A4B2AF, 0xAF31A4B2, 0xB2AF31A4, 0xA4B2AF31, }, /* x=88 */
-    { 0x3FADBFA4, 0xA43FADBF, 0xBFA43FAD, 0xADBFA43F, }, /* x=89 */
-    { 0x2DB6A8B9, 0xB92DB6A8, 0xA8B92DB6, 0xB6A8B92D, }, /* x=8A */
-    { 0x23BFA5B2, 0xB223BFA5, 0xA5B223BF, 0xBFA5B223, }, /* x=8B */
-    { 0x09808683, 0x83098086, 0x86830980, 0x80868309, }, /* x=8C */
-    { 0x07898B88, 0x8807898B, 0x8B880789, 0x898B8807, }, /* x=8D */
-    { 0x15929C95, 0x9515929C, 0x9C951592, 0x929C9515, }, /* x=8E */
-    { 0x1B9B919E, 0x9E1B9B91, 0x919E1B9B, 0x9B919E1B, }, /* x=8F */
-    { 0xA17C0A47, 0x47A17C0A, 0x0A47A17C, 0x7C0A47A1, }, /* x=90 */
-    { 0xAF75074C, 0x4CAF7507, 0x074CAF75, 0x75074CAF, }, /* x=91 */
-    { 0xBD6E1051, 0x51BD6E10, 0x1051BD6E, 0x6E1051BD, }, /* x=92 */
-    { 0xB3671D5A, 0x5AB3671D, 0x1D5AB367, 0x671D5AB3, }, /* x=93 */
-    { 0x99583E6B, 0x6B99583E, 0x3E6B9958, 0x583E6B99, }, /* x=94 */
-    { 0x97513360, 0x60975133, 0x33609751, 0x51336097, }, /* x=95 */
-    { 0x854A247D, 0x7D854A24, 0x247D854A, 0x4A247D85, }, /* x=96 */
-    { 0x8B432976, 0x768B4329, 0x29768B43, 0x4329768B, }, /* x=97 */
-    { 0xD134621F, 0x1FD13462, 0x621FD134, 0x34621FD1, }, /* x=98 */
-    { 0xDF3D6F14, 0x14DF3D6F, 0x6F14DF3D, 0x3D6F14DF, }, /* x=99 */
-    { 0xCD267809, 0x09CD2678, 0x7809CD26, 0x267809CD, }, /* x=9A */
-    { 0xC32F7502, 0x02C32F75, 0x7502C32F, 0x2F7502C3, }, /* x=9B */
-    { 0xE9105633, 0x33E91056, 0x5633E910, 0x105633E9, }, /* x=9C */
-    { 0xE7195B38, 0x38E7195B, 0x5B38E719, 0x195B38E7, }, /* x=9D */
-    { 0xF5024C25, 0x25F5024C, 0x4C25F502, 0x024C25F5, }, /* x=9E */
-    { 0xFB0B412E, 0x2EFB0B41, 0x412EFB0B, 0x0B412EFB, }, /* x=9F */
-    { 0x9AD7618C, 0x8C9AD761, 0x618C9AD7, 0xD7618C9A, }, /* x=A0 */
-    { 0x94DE6C87, 0x8794DE6C, 0x6C8794DE, 0xDE6C8794, }, /* x=A1 */
-    { 0x86C57B9A, 0x9A86C57B, 0x7B9A86C5, 0xC57B9A86, }, /* x=A2 */
-    { 0x88CC7691, 0x9188CC76, 0x769188CC, 0xCC769188, }, /* x=A3 */
-    { 0xA2F355A0, 0xA0A2F355, 0x55A0A2F3, 0xF355A0A2, }, /* x=A4 */
-    { 0xACFA58AB, 0xABACFA58, 0x58ABACFA, 0xFA58ABAC, }, /* x=A5 */
-    { 0xBEE14FB6, 0xB6BEE14F, 0x4FB6BEE1, 0xE14FB6BE, }, /* x=A6 */
-    { 0xB0E842BD, 0xBDB0E842, 0x42BDB0E8, 0xE842BDB0, }, /* x=A7 */
-    { 0xEA9F09D4, 0xD4EA9F09, 0x09D4EA9F, 0x9F09D4EA, }, /* x=A8 */
-    { 0xE49604DF, 0xDFE49604, 0x04DFE496, 0x9604DFE4, }, /* x=A9 */
-    { 0xF68D13C2, 0xC2F68D13, 0x13C2F68D, 0x8D13C2F6, }, /* x=AA */
-    { 0xF8841EC9, 0xC9F8841E, 0x1EC9F884, 0x841EC9F8, }, /* x=AB */
-    { 0xD2BB3DF8, 0xF8D2BB3D, 0x3DF8D2BB, 0xBB3DF8D2, }, /* x=AC */
-    { 0xDCB230F3, 0xF3DCB230, 0x30F3DCB2, 0xB230F3DC, }, /* x=AD */
-    { 0xCEA927EE, 0xEECEA927, 0x27EECEA9, 0xA927EECE, }, /* x=AE */
-    { 0xC0A02AE5, 0xE5C0A02A, 0x2AE5C0A0, 0xA02AE5C0, }, /* x=AF */
-    { 0x7A47B13C, 0x3C7A47B1, 0xB13C7A47, 0x47B13C7A, }, /* x=B0 */
-    { 0x744EBC37, 0x37744EBC, 0xBC37744E, 0x4EBC3774, }, /* x=B1 */
-    { 0x6655AB2A, 0x2A6655AB, 0xAB2A6655, 0x55AB2A66, }, /* x=B2 */
-    { 0x685CA621, 0x21685CA6, 0xA621685C, 0x5CA62168, }, /* x=B3 */
-    { 0x42638510, 0x10426385, 0x85104263, 0x63851042, }, /* x=B4 */
-    { 0x4C6A881B, 0x1B4C6A88, 0x881B4C6A, 0x6A881B4C, }, /* x=B5 */
-    { 0x5E719F06, 0x065E719F, 0x9F065E71, 0x719F065E, }, /* x=B6 */
-    { 0x5078920D, 0x0D507892, 0x920D5078, 0x78920D50, }, /* x=B7 */
-    { 0x0A0FD964, 0x640A0FD9, 0xD9640A0F, 0x0FD9640A, }, /* x=B8 */
-    { 0x0406D46F, 0x6F0406D4, 0xD46F0406, 0x06D46F04, }, /* x=B9 */
-    { 0x161DC372, 0x72161DC3, 0xC372161D, 0x1DC37216, }, /* x=BA */
-    { 0x1814CE79, 0x791814CE, 0xCE791814, 0x14CE7918, }, /* x=BB */
-    { 0x322BED48, 0x48322BED, 0xED48322B, 0x2BED4832, }, /* x=BC */
-    { 0x3C22E043, 0x433C22E0, 0xE0433C22, 0x22E0433C, }, /* x=BD */
-    { 0x2E39F75E, 0x5E2E39F7, 0xF75E2E39, 0x39F75E2E, }, /* x=BE */
-    { 0x2030FA55, 0x552030FA, 0xFA552030, 0x30FA5520, }, /* x=BF */
-    { 0xEC9AB701, 0x01EC9AB7, 0xB701EC9A, 0x9AB701EC, }, /* x=C0 */
-    { 0xE293BA0A, 0x0AE293BA, 0xBA0AE293, 0x93BA0AE2, }, /* x=C1 */
-    { 0xF088AD17, 0x17F088AD, 0xAD17F088, 0x88AD17F0, }, /* x=C2 */
-    { 0xFE81A01C, 0x1CFE81A0, 0xA01CFE81, 0x81A01CFE, }, /* x=C3 */
-    { 0xD4BE832D, 0x2DD4BE83, 0x832DD4BE, 0xBE832DD4, }, /* x=C4 */
-    { 0xDAB78E26, 0x26DAB78E, 0x8E26DAB7, 0xB78E26DA, }, /* x=C5 */
-    { 0xC8AC993B, 0x3BC8AC99, 0x993BC8AC, 0xAC993BC8, }, /* x=C6 */
-    { 0xC6A59430, 0x30C6A594, 0x9430C6A5, 0xA59430C6, }, /* x=C7 */
-    { 0x9CD2DF59, 0x599CD2DF, 0xDF599CD2, 0xD2DF599C, }, /* x=C8 */
-    { 0x92DBD252, 0x5292DBD2, 0xD25292DB, 0xDBD25292, }, /* x=C9 */
-    { 0x80C0C54F, 0x4F80C0C5, 0xC54F80C0, 0xC0C54F80, }, /* x=CA */
-    { 0x8EC9C844, 0x448EC9C8, 0xC8448EC9, 0xC9C8448E, }, /* x=CB */
-    { 0xA4F6EB75, 0x75A4F6EB, 0xEB75A4F6, 0xF6EB75A4, }, /* x=CC */
-    { 0xAAFFE67E, 0x7EAAFFE6, 0xE67EAAFF, 0xFFE67EAA, }, /* x=CD */
-    { 0xB8E4F163, 0x63B8E4F1, 0xF163B8E4, 0xE4F163B8, }, /* x=CE */
-    { 0xB6EDFC68, 0x68B6EDFC, 0xFC68B6ED, 0xEDFC68B6, }, /* x=CF */
-    { 0x0C0A67B1, 0xB10C0A67, 0x67B10C0A, 0x0A67B10C, }, /* x=D0 */
-    { 0x02036ABA, 0xBA02036A, 0x6ABA0203, 0x036ABA02, }, /* x=D1 */
-    { 0x10187DA7, 0xA710187D, 0x7DA71018, 0x187DA710, }, /* x=D2 */
-    { 0x1E1170AC, 0xAC1E1170, 0x70AC1E11, 0x1170AC1E, }, /* x=D3 */
-    { 0x342E539D, 0x9D342E53, 0x539D342E, 0x2E539D34, }, /* x=D4 */
-    { 0x3A275E96, 0x963A275E, 0x5E963A27, 0x275E963A, }, /* x=D5 */
-    { 0x283C498B, 0x8B283C49, 0x498B283C, 0x3C498B28, }, /* x=D6 */
-    { 0x26354480, 0x80263544, 0x44802635, 0x35448026, }, /* x=D7 */
-    { 0x7C420FE9, 0xE97C420F, 0x0FE97C42, 0x420FE97C, }, /* x=D8 */
-    { 0x724B02E2, 0xE2724B02, 0x02E2724B, 0x4B02E272, }, /* x=D9 */
-    { 0x605015FF, 0xFF605015, 0x15FF6050, 0x5015FF60, }, /* x=DA */
-    { 0x6E5918F4, 0xF46E5918, 0x18F46E59, 0x5918F46E, }, /* x=DB */
-    { 0x44663BC5, 0xC544663B, 0x3BC54466, 0x663BC544, }, /* x=DC */
-    { 0x4A6F36CE, 0xCE4A6F36, 0x36CE4A6F, 0x6F36CE4A, }, /* x=DD */
-    { 0x587421D3, 0xD3587421, 0x21D35874, 0x7421D358, }, /* x=DE */
-    { 0x567D2CD8, 0xD8567D2C, 0x2CD8567D, 0x7D2CD856, }, /* x=DF */
-    { 0x37A10C7A, 0x7A37A10C, 0x0C7A37A1, 0xA10C7A37, }, /* x=E0 */
-    { 0x39A80171, 0x7139A801, 0x017139A8, 0xA8017139, }, /* x=E1 */
-    { 0x2BB3166C, 0x6C2BB316, 0x166C2BB3, 0xB3166C2B, }, /* x=E2 */
-    { 0x25BA1B67, 0x6725BA1B, 0x1B6725BA, 0xBA1B6725, }, /* x=E3 */
-    { 0x0F853856, 0x560F8538, 0x38560F85, 0x8538560F, }, /* x=E4 */
-    { 0x018C355D, 0x5D018C35, 0x355D018C, 0x8C355D01, }, /* x=E5 */
-    { 0x13972240, 0x40139722, 0x22401397, 0x97224013, }, /* x=E6 */
-    { 0x1D9E2F4B, 0x4B1D9E2F, 0x2F4B1D9E, 0x9E2F4B1D, }, /* x=E7 */
-    { 0x47E96422, 0x2247E964, 0x642247E9, 0xE9642247, }, /* x=E8 */
-    { 0x49E06929, 0x2949E069, 0x692949E0, 0xE0692949, }, /* x=E9 */
-    { 0x5BFB7E34, 0x345BFB7E, 0x7E345BFB, 0xFB7E345B, }, /* x=EA */
-    { 0x55F2733F, 0x3F55F273, 0x733F55F2, 0xF2733F55, }, /* x=EB */
-    { 0x7FCD500E, 0x0E7FCD50, 0x500E7FCD, 0xCD500E7F, }, /* x=EC */
-    { 0x71C45D05, 0x0571C45D, 0x5D0571C4, 0xC45D0571, }, /* x=ED */
-    { 0x63DF4A18, 0x1863DF4A, 0x4A1863DF, 0xDF4A1863, }, /* x=EE */
-    { 0x6DD64713, 0x136DD647, 0x47136DD6, 0xD647136D, }, /* x=EF */
-    { 0xD731DCCA, 0xCAD731DC, 0xDCCAD731, 0x31DCCAD7, }, /* x=F0 */
-    { 0xD938D1C1, 0xC1D938D1, 0xD1C1D938, 0x38D1C1D9, }, /* x=F1 */
-    { 0xCB23C6DC, 0xDCCB23C6, 0xC6DCCB23, 0x23C6DCCB, }, /* x=F2 */
-    { 0xC52ACBD7, 0xD7C52ACB, 0xCBD7C52A, 0x2ACBD7C5, }, /* x=F3 */
-    { 0xEF15E8E6, 0xE6EF15E8, 0xE8E6EF15, 0x15E8E6EF, }, /* x=F4 */
-    { 0xE11CE5ED, 0xEDE11CE5, 0xE5EDE11C, 0x1CE5EDE1, }, /* x=F5 */
-    { 0xF307F2F0, 0xF0F307F2, 0xF2F0F307, 0x07F2F0F3, }, /* x=F6 */
-    { 0xFD0EFFFB, 0xFBFD0EFF, 0xFFFBFD0E, 0x0EFFFBFD, }, /* x=F7 */
-    { 0xA779B492, 0x92A779B4, 0xB492A779, 0x79B492A7, }, /* x=F8 */
-    { 0xA970B999, 0x99A970B9, 0xB999A970, 0x70B999A9, }, /* x=F9 */
-    { 0xBB6BAE84, 0x84BB6BAE, 0xAE84BB6B, 0x6BAE84BB, }, /* x=FA */
-    { 0xB562A38F, 0x8FB562A3, 0xA38FB562, 0x62A38FB5, }, /* x=FB */
-    { 0x9F5D80BE, 0xBE9F5D80, 0x80BE9F5D, 0x5D80BE9F, }, /* x=FC */
-    { 0x91548DB5, 0xB591548D, 0x8DB59154, 0x548DB591, }, /* x=FD */
-    { 0x834F9AA8, 0xA8834F9A, 0x9AA8834F, 0x4F9AA883, }, /* x=FE */
-    { 0x8D4697A3, 0xA38D4697, 0x97A38D46, 0x4697A38D, }, /* x=FF */
-};
-
-
-
-/*
-AES_Te0[x] = S [x].[02, 01, 01, 03];
-AES_Te1[x] = S [x].[03, 02, 01, 01];
-AES_Te2[x] = S [x].[01, 03, 02, 01];
-AES_Te3[x] = S [x].[01, 01, 03, 02];
-AES_Te4[x] = S [x].[01, 01, 01, 01];
-
-AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
-AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
-AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
-AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
-AES_Td4[x] = Si[x].[01, 01, 01, 01];
-*/
-
-const uint32_t AES_Te0[256] = {
-    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
-    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
-    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
-    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
-    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
-    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
-    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
-    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
-    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
-    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
-    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
-    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
-    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
-    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
-    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
-    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
-    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
-    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
-    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
-    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
-    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
-    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
-    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
-    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
-    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
-    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
-    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
-    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
-    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
-    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
-    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
-    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
-    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
-    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
-    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
-    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
-    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
-    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
-    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
-    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
-    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
-    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
-    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
-    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
-    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
-    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
-    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
-    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
-    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
-    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
-    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
-    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
-    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
-    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
-    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
-    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
-    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
-    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
-    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
-    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
-    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
-    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
-    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
-    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
-};
-const uint32_t AES_Te1[256] = {
-    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
-    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
-    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
-    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
-    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
-    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
-    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
-    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
-    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
-    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
-    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
-    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
-    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
-    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
-    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
-    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
-    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
-    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
-    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
-    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
-    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
-    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
-    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
-    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
-    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
-    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
-    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
-    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
-    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
-    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
-    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
-    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
-    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
-    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
-    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
-    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
-    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
-    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
-    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
-    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
-    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
-    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
-    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
-    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
-    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
-    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
-    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
-    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
-    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
-    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
-    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
-    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
-    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
-    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
-    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
-    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
-    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
-    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
-    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
-    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
-    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
-    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
-    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
-    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
-};
-const uint32_t AES_Te2[256] = {
-    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
-    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
-    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
-    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
-    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
-    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
-    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
-    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
-    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
-    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
-    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
-    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
-    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
-    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
-    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
-    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
-    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
-    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
-    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
-    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
-    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
-    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
-    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
-    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
-    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
-    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
-    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
-    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
-    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
-    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
-    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
-    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
-    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
-    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
-    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
-    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
-    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
-    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
-    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
-    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
-    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
-    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
-    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
-    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
-    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
-    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
-    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
-    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
-    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
-    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
-    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
-    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
-    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
-    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
-    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
-    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
-    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
-    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
-    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
-    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
-    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
-    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
-    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
-    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
-};
-const uint32_t AES_Te3[256] = {
-
-    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
-    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
-    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
-    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
-    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
-    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
-    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
-    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
-    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
-    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
-    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
-    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
-    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
-    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
-    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
-    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
-    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
-    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
-    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
-    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
-    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
-    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
-    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
-    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
-    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
-    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
-    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
-    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
-    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
-    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
-    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
-    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
-    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
-    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
-    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
-    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
-    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
-    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
-    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
-    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
-    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
-    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
-    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
-    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
-    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
-    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
-    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
-    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
-    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
-    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
-    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
-    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
-    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
-    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
-    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
-    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
-    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
-    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
-    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
-    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
-    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
-    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
-    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
-    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
-};
-const uint32_t AES_Te4[256] = {
-    0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU,
-    0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U,
-    0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU,
-    0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U,
-    0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU,
-    0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U,
-    0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU,
-    0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U,
-    0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U,
-    0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU,
-    0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U,
-    0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U,
-    0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U,
-    0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU,
-    0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U,
-    0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U,
-    0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU,
-    0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U,
-    0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U,
-    0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U,
-    0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU,
-    0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU,
-    0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U,
-    0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU,
-    0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU,
-    0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U,
-    0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU,
-    0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U,
-    0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU,
-    0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U,
-    0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U,
-    0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U,
-    0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU,
-    0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U,
-    0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU,
-    0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U,
-    0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU,
-    0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U,
-    0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U,
-    0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU,
-    0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU,
-    0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU,
-    0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U,
-    0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U,
-    0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU,
-    0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U,
-    0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU,
-    0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U,
-    0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU,
-    0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U,
-    0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU,
-    0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU,
-    0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U,
-    0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU,
-    0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U,
-    0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU,
-    0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U,
-    0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U,
-    0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U,
-    0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU,
-    0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU,
-    0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U,
-    0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU,
-    0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U,
-};
-const uint32_t AES_Td0[256] = {
-    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
-    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
-    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
-    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
-    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
-    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
-    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
-    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
-    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
-    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
-    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
-    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
-    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
-    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
-    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
-    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
-    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
-    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
-    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
-    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
-    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
-    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
-    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
-    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
-    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
-    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
-    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
-    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
-    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
-    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
-    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
-    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
-    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
-    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
-    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
-    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
-    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
-    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
-    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
-    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
-    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
-    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
-    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
-    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
-    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
-    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
-    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
-    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
-    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
-    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
-    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
-    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
-    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
-    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
-    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
-    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
-    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
-    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
-    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
-    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
-    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
-    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
-    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
-    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
-};
-const uint32_t AES_Td1[256] = {
-    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
-    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
-    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
-    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
-    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
-    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
-    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
-    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
-    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
-    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
-    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
-    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
-    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
-    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
-    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
-    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
-    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
-    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
-    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
-    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
-    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
-    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
-    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
-    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
-    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
-    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
-    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
-    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
-    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
-    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
-    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
-    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
-    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
-    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
-    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
-    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
-    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
-    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
-    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
-    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
-    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
-    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
-    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
-    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
-    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
-    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
-    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
-    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
-    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
-    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
-    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
-    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
-    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
-    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
-    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
-    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
-    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
-    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
-    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
-    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
-    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
-    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
-    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
-    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
-};
-const uint32_t AES_Td2[256] = {
-    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
-    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
-    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
-    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
-    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
-    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
-    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
-    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
-    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
-    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
-    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
-    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
-    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
-    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
-    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
-    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
-    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
-    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
-    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
-    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
-
-    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
-    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
-    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
-    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
-    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
-    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
-    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
-    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
-    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
-    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
-    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
-    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
-    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
-    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
-    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
-    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
-    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
-    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
-    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
-    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
-    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
-    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
-    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
-    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
-    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
-    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
-    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
-    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
-    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
-    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
-    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
-    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
-    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
-    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
-    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
-    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
-    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
-    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
-    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
-    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
-    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
-    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
-    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
-    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
-};
-const uint32_t AES_Td3[256] = {
-    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
-    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
-    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
-    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
-    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
-    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
-    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
-    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
-    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
-    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
-    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
-    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
-    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
-    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
-    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
-    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
-    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
-    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
-    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
-    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
-    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
-    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
-    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
-    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
-    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
-    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
-    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
-    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
-    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
-    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
-    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
-    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
-    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
-    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
-    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
-    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
-    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
-    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
-    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
-    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
-    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
-    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
-    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
-    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
-    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
-    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
-    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
-    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
-    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
-    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
-    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
-    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
-    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
-    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
-    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
-    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
-    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
-    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
-    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
-    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
-    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
-    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
-    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
-    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
-};
-const uint32_t AES_Td4[256] = {
-    0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U,
-    0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U,
-    0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU,
-    0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU,
-    0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U,
-    0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U,
-    0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U,
-    0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU,
-    0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U,
-    0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU,
-    0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU,
-    0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU,
-    0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U,
-    0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U,
-    0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U,
-    0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U,
-    0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U,
-    0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U,
-    0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU,
-    0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U,
-    0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U,
-    0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU,
-    0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U,
-    0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U,
-    0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U,
-    0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU,
-    0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U,
-    0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U,
-    0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU,
-    0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U,
-    0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U,
-    0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU,
-    0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U,
-    0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU,
-    0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU,
-    0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U,
-    0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U,
-    0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U,
-    0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U,
-    0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU,
-    0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U,
-    0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U,
-    0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU,
-    0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU,
-    0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU,
-    0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U,
-    0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU,
-    0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U,
-    0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U,
-    0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U,
-    0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U,
-    0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU,
-    0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U,
-    0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU,
-    0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU,
-    0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU,
-    0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU,
-    0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U,
-    0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU,
-    0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U,
-    0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU,
-    0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U,
-    0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U,
-    0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU,
-};
-static const u32 rcon[] = {
-	0x01000000, 0x02000000, 0x04000000, 0x08000000,
-	0x10000000, 0x20000000, 0x40000000, 0x80000000,
-	0x1B000000, 0x36000000, /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
-};
-
-/**
- * Expand the cipher key into the encryption key schedule.
- */
-int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
-			AES_KEY *key) {
-
-	u32 *rk;
-   	int i = 0;
-	u32 temp;
-
-	if (!userKey || !key)
-		return -1;
-	if (bits != 128 && bits != 192 && bits != 256)
-		return -2;
-
-	rk = key->rd_key;
-
-	if (bits==128)
-		key->rounds = 10;
-	else if (bits==192)
-		key->rounds = 12;
-	else
-		key->rounds = 14;
-
-	rk[0] = GETU32(userKey     );
-	rk[1] = GETU32(userKey +  4);
-	rk[2] = GETU32(userKey +  8);
-	rk[3] = GETU32(userKey + 12);
-	if (bits == 128) {
-		while (1) {
-			temp  = rk[3];
-			rk[4] = rk[0] ^
-                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
-                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
-                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
-                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
-				rcon[i];
-			rk[5] = rk[1] ^ rk[4];
-			rk[6] = rk[2] ^ rk[5];
-			rk[7] = rk[3] ^ rk[6];
-			if (++i == 10) {
-				return 0;
-			}
-			rk += 4;
-		}
-	}
-	rk[4] = GETU32(userKey + 16);
-	rk[5] = GETU32(userKey + 20);
-	if (bits == 192) {
-		while (1) {
-			temp = rk[ 5];
-			rk[ 6] = rk[ 0] ^
-                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
-                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
-                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
-                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
-				rcon[i];
-			rk[ 7] = rk[ 1] ^ rk[ 6];
-			rk[ 8] = rk[ 2] ^ rk[ 7];
-			rk[ 9] = rk[ 3] ^ rk[ 8];
-			if (++i == 8) {
-				return 0;
-			}
-			rk[10] = rk[ 4] ^ rk[ 9];
-			rk[11] = rk[ 5] ^ rk[10];
-			rk += 6;
-		}
-	}
-	rk[6] = GETU32(userKey + 24);
-	rk[7] = GETU32(userKey + 28);
-	if (bits == 256) {
-		while (1) {
-			temp = rk[ 7];
-			rk[ 8] = rk[ 0] ^
-                                (AES_Te4[(temp >> 16) & 0xff] & 0xff000000) ^
-                                (AES_Te4[(temp >>  8) & 0xff] & 0x00ff0000) ^
-                                (AES_Te4[(temp      ) & 0xff] & 0x0000ff00) ^
-                                (AES_Te4[(temp >> 24)       ] & 0x000000ff) ^
-				rcon[i];
-			rk[ 9] = rk[ 1] ^ rk[ 8];
-			rk[10] = rk[ 2] ^ rk[ 9];
-			rk[11] = rk[ 3] ^ rk[10];
-			if (++i == 7) {
-				return 0;
-			}
-			temp = rk[11];
-			rk[12] = rk[ 4] ^
-                                (AES_Te4[(temp >> 24)       ] & 0xff000000) ^
-                                (AES_Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^
-                                (AES_Te4[(temp >>  8) & 0xff] & 0x0000ff00) ^
-                                (AES_Te4[(temp      ) & 0xff] & 0x000000ff);
-			rk[13] = rk[ 5] ^ rk[12];
-			rk[14] = rk[ 6] ^ rk[13];
-			rk[15] = rk[ 7] ^ rk[14];
-
-			rk += 8;
-        	}
-	}
-	return 0;
-}
-
-/**
- * Expand the cipher key into the decryption key schedule.
- */
-int AES_set_decrypt_key(const unsigned char *userKey, const int bits,
-			 AES_KEY *key) {
-
-        u32 *rk;
-	int i, j, status;
-	u32 temp;
-
-	/* first, start with an encryption schedule */
-	status = AES_set_encrypt_key(userKey, bits, key);
-	if (status < 0)
-		return status;
-
-	rk = key->rd_key;
-
-	/* invert the order of the round keys: */
-	for (i = 0, j = 4*(key->rounds); i < j; i += 4, j -= 4) {
-		temp = rk[i    ]; rk[i    ] = rk[j    ]; rk[j    ] = temp;
-		temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
-		temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
-		temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
-	}
-	/* apply the inverse MixColumn transform to all round keys but the first and the last: */
-	for (i = 1; i < (key->rounds); i++) {
-		rk += 4;
-		rk[0] =
-                        AES_Td0[AES_Te4[(rk[0] >> 24)       ] & 0xff] ^
-                        AES_Td1[AES_Te4[(rk[0] >> 16) & 0xff] & 0xff] ^
-                        AES_Td2[AES_Te4[(rk[0] >>  8) & 0xff] & 0xff] ^
-                        AES_Td3[AES_Te4[(rk[0]      ) & 0xff] & 0xff];
-		rk[1] =
-                        AES_Td0[AES_Te4[(rk[1] >> 24)       ] & 0xff] ^
-                        AES_Td1[AES_Te4[(rk[1] >> 16) & 0xff] & 0xff] ^
-                        AES_Td2[AES_Te4[(rk[1] >>  8) & 0xff] & 0xff] ^
-                        AES_Td3[AES_Te4[(rk[1]      ) & 0xff] & 0xff];
-		rk[2] =
-                        AES_Td0[AES_Te4[(rk[2] >> 24)       ] & 0xff] ^
-                        AES_Td1[AES_Te4[(rk[2] >> 16) & 0xff] & 0xff] ^
-                        AES_Td2[AES_Te4[(rk[2] >>  8) & 0xff] & 0xff] ^
-                        AES_Td3[AES_Te4[(rk[2]      ) & 0xff] & 0xff];
-		rk[3] =
-                        AES_Td0[AES_Te4[(rk[3] >> 24)       ] & 0xff] ^
-                        AES_Td1[AES_Te4[(rk[3] >> 16) & 0xff] & 0xff] ^
-                        AES_Td2[AES_Te4[(rk[3] >>  8) & 0xff] & 0xff] ^
-                        AES_Td3[AES_Te4[(rk[3]      ) & 0xff] & 0xff];
-	}
-	return 0;
-}
-
-#ifndef AES_ASM
-/*
- * Encrypt a single block
- * in and out can overlap
- */
-void AES_encrypt(const unsigned char *in, unsigned char *out,
-		 const AES_KEY *key) {
-
-	const u32 *rk;
-	u32 s0, s1, s2, s3, t0, t1, t2, t3;
-#ifndef FULL_UNROLL
-	int r;
-#endif /* ?FULL_UNROLL */
-
-	assert(in && out && key);
-	rk = key->rd_key;
-
-	/*
-	 * map byte array block to cipher state
-	 * and add initial round key:
-	 */
-	s0 = GETU32(in     ) ^ rk[0];
-	s1 = GETU32(in +  4) ^ rk[1];
-	s2 = GETU32(in +  8) ^ rk[2];
-	s3 = GETU32(in + 12) ^ rk[3];
-#ifdef FULL_UNROLL
-	/* round 1: */
-        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[ 4];
-        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[ 5];
-        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[ 6];
-        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[ 7];
-   	/* round 2: */
-        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[ 8];
-        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[ 9];
-        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[10];
-        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[11];
-	/* round 3: */
-        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[12];
-        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[13];
-        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[14];
-        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[15];
-   	/* round 4: */
-        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[16];
-        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[17];
-        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[18];
-        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[19];
-	/* round 5: */
-        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[20];
-        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[21];
-        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[22];
-        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[23];
-   	/* round 6: */
-        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[24];
-        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[25];
-        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[26];
-        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[27];
-	/* round 7: */
-        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[28];
-        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[29];
-        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[30];
-        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[31];
-   	/* round 8: */
-        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[32];
-        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[33];
-        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[34];
-        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[35];
-	/* round 9: */
-        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[36];
-        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[37];
-        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[38];
-        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[39];
-    if (key->rounds > 10) {
-        /* round 10: */
-        s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[40];
-        s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[41];
-        s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[42];
-        s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[43];
-        /* round 11: */
-        t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[44];
-        t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[45];
-        t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[46];
-        t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[47];
-        if (key->rounds > 12) {
-            /* round 12: */
-            s0 = AES_Te0[t0 >> 24] ^ AES_Te1[(t1 >> 16) & 0xff] ^ AES_Te2[(t2 >>  8) & 0xff] ^ AES_Te3[t3 & 0xff] ^ rk[48];
-            s1 = AES_Te0[t1 >> 24] ^ AES_Te1[(t2 >> 16) & 0xff] ^ AES_Te2[(t3 >>  8) & 0xff] ^ AES_Te3[t0 & 0xff] ^ rk[49];
-            s2 = AES_Te0[t2 >> 24] ^ AES_Te1[(t3 >> 16) & 0xff] ^ AES_Te2[(t0 >>  8) & 0xff] ^ AES_Te3[t1 & 0xff] ^ rk[50];
-            s3 = AES_Te0[t3 >> 24] ^ AES_Te1[(t0 >> 16) & 0xff] ^ AES_Te2[(t1 >>  8) & 0xff] ^ AES_Te3[t2 & 0xff] ^ rk[51];
-            /* round 13: */
-            t0 = AES_Te0[s0 >> 24] ^ AES_Te1[(s1 >> 16) & 0xff] ^ AES_Te2[(s2 >>  8) & 0xff] ^ AES_Te3[s3 & 0xff] ^ rk[52];
-            t1 = AES_Te0[s1 >> 24] ^ AES_Te1[(s2 >> 16) & 0xff] ^ AES_Te2[(s3 >>  8) & 0xff] ^ AES_Te3[s0 & 0xff] ^ rk[53];
-            t2 = AES_Te0[s2 >> 24] ^ AES_Te1[(s3 >> 16) & 0xff] ^ AES_Te2[(s0 >>  8) & 0xff] ^ AES_Te3[s1 & 0xff] ^ rk[54];
-            t3 = AES_Te0[s3 >> 24] ^ AES_Te1[(s0 >> 16) & 0xff] ^ AES_Te2[(s1 >>  8) & 0xff] ^ AES_Te3[s2 & 0xff] ^ rk[55];
-        }
-    }
-    rk += key->rounds << 2;
-#else  /* !FULL_UNROLL */
-    /*
-     * Nr - 1 full rounds:
-     */
-    r = key->rounds >> 1;
-    for (;;) {
-        t0 =
-            AES_Te0[(s0 >> 24)       ] ^
-            AES_Te1[(s1 >> 16) & 0xff] ^
-            AES_Te2[(s2 >>  8) & 0xff] ^
-            AES_Te3[(s3      ) & 0xff] ^
-            rk[4];
-        t1 =
-            AES_Te0[(s1 >> 24)       ] ^
-            AES_Te1[(s2 >> 16) & 0xff] ^
-            AES_Te2[(s3 >>  8) & 0xff] ^
-            AES_Te3[(s0      ) & 0xff] ^
-            rk[5];
-        t2 =
-            AES_Te0[(s2 >> 24)       ] ^
-            AES_Te1[(s3 >> 16) & 0xff] ^
-            AES_Te2[(s0 >>  8) & 0xff] ^
-            AES_Te3[(s1      ) & 0xff] ^
-            rk[6];
-        t3 =
-            AES_Te0[(s3 >> 24)       ] ^
-            AES_Te1[(s0 >> 16) & 0xff] ^
-            AES_Te2[(s1 >>  8) & 0xff] ^
-            AES_Te3[(s2      ) & 0xff] ^
-            rk[7];
-
-        rk += 8;
-        if (--r == 0) {
-            break;
-        }
-
-        s0 =
-            AES_Te0[(t0 >> 24)       ] ^
-            AES_Te1[(t1 >> 16) & 0xff] ^
-            AES_Te2[(t2 >>  8) & 0xff] ^
-            AES_Te3[(t3      ) & 0xff] ^
-            rk[0];
-        s1 =
-            AES_Te0[(t1 >> 24)       ] ^
-            AES_Te1[(t2 >> 16) & 0xff] ^
-            AES_Te2[(t3 >>  8) & 0xff] ^
-            AES_Te3[(t0      ) & 0xff] ^
-            rk[1];
-        s2 =
-            AES_Te0[(t2 >> 24)       ] ^
-            AES_Te1[(t3 >> 16) & 0xff] ^
-            AES_Te2[(t0 >>  8) & 0xff] ^
-            AES_Te3[(t1      ) & 0xff] ^
-            rk[2];
-        s3 =
-            AES_Te0[(t3 >> 24)       ] ^
-            AES_Te1[(t0 >> 16) & 0xff] ^
-            AES_Te2[(t1 >>  8) & 0xff] ^
-            AES_Te3[(t2      ) & 0xff] ^
-            rk[3];
-    }
-#endif /* ?FULL_UNROLL */
-    /*
-	 * apply last round and
-	 * map cipher state to byte array block:
-	 */
-	s0 =
-                (AES_Te4[(t0 >> 24)       ] & 0xff000000) ^
-                (AES_Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Te4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Te4[(t3      ) & 0xff] & 0x000000ff) ^
-		rk[0];
-	PUTU32(out     , s0);
-	s1 =
-                (AES_Te4[(t1 >> 24)       ] & 0xff000000) ^
-                (AES_Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Te4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Te4[(t0      ) & 0xff] & 0x000000ff) ^
-		rk[1];
-	PUTU32(out +  4, s1);
-	s2 =
-                (AES_Te4[(t2 >> 24)       ] & 0xff000000) ^
-                (AES_Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Te4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Te4[(t1      ) & 0xff] & 0x000000ff) ^
-		rk[2];
-	PUTU32(out +  8, s2);
-	s3 =
-                (AES_Te4[(t3 >> 24)       ] & 0xff000000) ^
-                (AES_Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Te4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Te4[(t2      ) & 0xff] & 0x000000ff) ^
-		rk[3];
-	PUTU32(out + 12, s3);
-}
-
-/*
- * Decrypt a single block
- * in and out can overlap
- */
-void AES_decrypt(const unsigned char *in, unsigned char *out,
-		 const AES_KEY *key) {
-
-	const u32 *rk;
-	u32 s0, s1, s2, s3, t0, t1, t2, t3;
-#ifndef FULL_UNROLL
-	int r;
-#endif /* ?FULL_UNROLL */
-
-	assert(in && out && key);
-	rk = key->rd_key;
-
-	/*
-	 * map byte array block to cipher state
-	 * and add initial round key:
-	 */
-    s0 = GETU32(in     ) ^ rk[0];
-    s1 = GETU32(in +  4) ^ rk[1];
-    s2 = GETU32(in +  8) ^ rk[2];
-    s3 = GETU32(in + 12) ^ rk[3];
-#ifdef FULL_UNROLL
-    /* round 1: */
-    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[ 4];
-    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[ 5];
-    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[ 6];
-    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[ 7];
-    /* round 2: */
-    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[ 8];
-    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[ 9];
-    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[10];
-    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[11];
-    /* round 3: */
-    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[12];
-    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[13];
-    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[14];
-    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[15];
-    /* round 4: */
-    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[16];
-    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[17];
-    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[18];
-    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[19];
-    /* round 5: */
-    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[20];
-    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[21];
-    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[22];
-    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[23];
-    /* round 6: */
-    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[24];
-    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[25];
-    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[26];
-    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[27];
-    /* round 7: */
-    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[28];
-    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[29];
-    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[30];
-    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[31];
-    /* round 8: */
-    s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[32];
-    s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[33];
-    s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[34];
-    s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[35];
-    /* round 9: */
-    t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[36];
-    t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[37];
-    t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[38];
-    t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[39];
-    if (key->rounds > 10) {
-        /* round 10: */
-        s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[40];
-        s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[41];
-        s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[42];
-        s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[43];
-        /* round 11: */
-        t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[44];
-        t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[45];
-        t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[46];
-        t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[47];
-        if (key->rounds > 12) {
-            /* round 12: */
-            s0 = AES_Td0[t0 >> 24] ^ AES_Td1[(t3 >> 16) & 0xff] ^ AES_Td2[(t2 >>  8) & 0xff] ^ AES_Td3[t1 & 0xff] ^ rk[48];
-            s1 = AES_Td0[t1 >> 24] ^ AES_Td1[(t0 >> 16) & 0xff] ^ AES_Td2[(t3 >>  8) & 0xff] ^ AES_Td3[t2 & 0xff] ^ rk[49];
-            s2 = AES_Td0[t2 >> 24] ^ AES_Td1[(t1 >> 16) & 0xff] ^ AES_Td2[(t0 >>  8) & 0xff] ^ AES_Td3[t3 & 0xff] ^ rk[50];
-            s3 = AES_Td0[t3 >> 24] ^ AES_Td1[(t2 >> 16) & 0xff] ^ AES_Td2[(t1 >>  8) & 0xff] ^ AES_Td3[t0 & 0xff] ^ rk[51];
-            /* round 13: */
-            t0 = AES_Td0[s0 >> 24] ^ AES_Td1[(s3 >> 16) & 0xff] ^ AES_Td2[(s2 >>  8) & 0xff] ^ AES_Td3[s1 & 0xff] ^ rk[52];
-            t1 = AES_Td0[s1 >> 24] ^ AES_Td1[(s0 >> 16) & 0xff] ^ AES_Td2[(s3 >>  8) & 0xff] ^ AES_Td3[s2 & 0xff] ^ rk[53];
-            t2 = AES_Td0[s2 >> 24] ^ AES_Td1[(s1 >> 16) & 0xff] ^ AES_Td2[(s0 >>  8) & 0xff] ^ AES_Td3[s3 & 0xff] ^ rk[54];
-            t3 = AES_Td0[s3 >> 24] ^ AES_Td1[(s2 >> 16) & 0xff] ^ AES_Td2[(s1 >>  8) & 0xff] ^ AES_Td3[s0 & 0xff] ^ rk[55];
-        }
-    }
-	rk += key->rounds << 2;
-#else  /* !FULL_UNROLL */
-    /*
-     * Nr - 1 full rounds:
-     */
-    r = key->rounds >> 1;
-    for (;;) {
-        t0 =
-            AES_Td0[(s0 >> 24)       ] ^
-            AES_Td1[(s3 >> 16) & 0xff] ^
-            AES_Td2[(s2 >>  8) & 0xff] ^
-            AES_Td3[(s1      ) & 0xff] ^
-            rk[4];
-        t1 =
-            AES_Td0[(s1 >> 24)       ] ^
-            AES_Td1[(s0 >> 16) & 0xff] ^
-            AES_Td2[(s3 >>  8) & 0xff] ^
-            AES_Td3[(s2      ) & 0xff] ^
-            rk[5];
-        t2 =
-            AES_Td0[(s2 >> 24)       ] ^
-            AES_Td1[(s1 >> 16) & 0xff] ^
-            AES_Td2[(s0 >>  8) & 0xff] ^
-            AES_Td3[(s3      ) & 0xff] ^
-            rk[6];
-        t3 =
-            AES_Td0[(s3 >> 24)       ] ^
-            AES_Td1[(s2 >> 16) & 0xff] ^
-            AES_Td2[(s1 >>  8) & 0xff] ^
-            AES_Td3[(s0      ) & 0xff] ^
-            rk[7];
-
-        rk += 8;
-        if (--r == 0) {
-            break;
-        }
-
-        s0 =
-            AES_Td0[(t0 >> 24)       ] ^
-            AES_Td1[(t3 >> 16) & 0xff] ^
-            AES_Td2[(t2 >>  8) & 0xff] ^
-            AES_Td3[(t1      ) & 0xff] ^
-            rk[0];
-        s1 =
-            AES_Td0[(t1 >> 24)       ] ^
-            AES_Td1[(t0 >> 16) & 0xff] ^
-            AES_Td2[(t3 >>  8) & 0xff] ^
-            AES_Td3[(t2      ) & 0xff] ^
-            rk[1];
-        s2 =
-            AES_Td0[(t2 >> 24)       ] ^
-            AES_Td1[(t1 >> 16) & 0xff] ^
-            AES_Td2[(t0 >>  8) & 0xff] ^
-            AES_Td3[(t3      ) & 0xff] ^
-            rk[2];
-        s3 =
-            AES_Td0[(t3 >> 24)       ] ^
-            AES_Td1[(t2 >> 16) & 0xff] ^
-            AES_Td2[(t1 >>  8) & 0xff] ^
-            AES_Td3[(t0      ) & 0xff] ^
-            rk[3];
-    }
-#endif /* ?FULL_UNROLL */
-    /*
-	 * apply last round and
-	 * map cipher state to byte array block:
-	 */
-   	s0 =
-                (AES_Td4[(t0 >> 24)       ] & 0xff000000) ^
-                (AES_Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Td4[(t2 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Td4[(t1      ) & 0xff] & 0x000000ff) ^
-   		rk[0];
-	PUTU32(out     , s0);
-   	s1 =
-                (AES_Td4[(t1 >> 24)       ] & 0xff000000) ^
-                (AES_Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Td4[(t3 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Td4[(t2      ) & 0xff] & 0x000000ff) ^
-   		rk[1];
-	PUTU32(out +  4, s1);
-   	s2 =
-                (AES_Td4[(t2 >> 24)       ] & 0xff000000) ^
-                (AES_Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Td4[(t0 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Td4[(t3      ) & 0xff] & 0x000000ff) ^
-   		rk[2];
-	PUTU32(out +  8, s2);
-   	s3 =
-                (AES_Td4[(t3 >> 24)       ] & 0xff000000) ^
-                (AES_Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^
-                (AES_Td4[(t1 >>  8) & 0xff] & 0x0000ff00) ^
-                (AES_Td4[(t0      ) & 0xff] & 0x000000ff) ^
-   		rk[3];
-	PUTU32(out + 12, s3);
-}
-
-#endif /* AES_ASM */
-
-void AES_cbc_encrypt(const unsigned char *in, unsigned char *out,
-		     const unsigned long length, const AES_KEY *key,
-		     unsigned char *ivec, const int enc)
-{
-
-	unsigned long n;
-	unsigned long len = length;
-	unsigned char tmp[AES_BLOCK_SIZE];
-
-	assert(in && out && key && ivec);
-
-	if (enc) {
-		while (len >= AES_BLOCK_SIZE) {
-			for(n=0; n < AES_BLOCK_SIZE; ++n)
-				tmp[n] = in[n] ^ ivec[n];
-			AES_encrypt(tmp, out, key);
-			memcpy(ivec, out, AES_BLOCK_SIZE);
-			len -= AES_BLOCK_SIZE;
-			in += AES_BLOCK_SIZE;
-			out += AES_BLOCK_SIZE;
-		}
-		if (len) {
-			for(n=0; n < len; ++n)
-				tmp[n] = in[n] ^ ivec[n];
-			for(n=len; n < AES_BLOCK_SIZE; ++n)
-				tmp[n] = ivec[n];
-			AES_encrypt(tmp, tmp, key);
-			memcpy(out, tmp, AES_BLOCK_SIZE);
-			memcpy(ivec, tmp, AES_BLOCK_SIZE);
-		}
-	} else {
-		while (len >= AES_BLOCK_SIZE) {
-			memcpy(tmp, in, AES_BLOCK_SIZE);
-			AES_decrypt(in, out, key);
-			for(n=0; n < AES_BLOCK_SIZE; ++n)
-				out[n] ^= ivec[n];
-			memcpy(ivec, tmp, AES_BLOCK_SIZE);
-			len -= AES_BLOCK_SIZE;
-			in += AES_BLOCK_SIZE;
-			out += AES_BLOCK_SIZE;
-		}
-		if (len) {
-			memcpy(tmp, in, AES_BLOCK_SIZE);
-			AES_decrypt(tmp, tmp, key);
-			for(n=0; n < len; ++n)
-				out[n] = tmp[n] ^ ivec[n];
-			memcpy(ivec, tmp, AES_BLOCK_SIZE);
-		}
-	}
-}
diff --git a/util/base64.c b/util/base64.c
new file mode 100644
index 0000000..9d3c46c
--- /dev/null
+++ b/util/base64.c
@@ -0,0 +1,60 @@
+/*
+ * QEMU base64 helpers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qapi/error.h"
+#include "qemu/base64.h"
+
+static const char *base64_valid_chars =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=\n";
+
+uint8_t *qbase64_decode(const char *input,
+                        size_t in_len,
+                        size_t *out_len,
+                        Error **errp)
+{
+    *out_len = 0;
+
+    if (in_len != -1) {
+        /* Lack of NUL terminator is an error */
+        if (input[in_len] != '\0') {
+            error_setg(errp, "Base64 data is not NUL terminated");
+            return NULL;
+        }
+        /* Check there's no NULs embedded since we expect
+         * this to be valid base64 data */
+        if (memchr(input, '\0', in_len) != NULL) {
+            error_setg(errp, "Base64 data contains embedded NUL characters");
+            return NULL;
+        }
+
+        /* Now we know its a valid nul terminated string
+         * strspn is safe to use... */
+    } else {
+        in_len = strlen(input);
+    }
+
+    if (strspn(input, base64_valid_chars) != in_len) {
+        error_setg(errp, "Base64 data contains invalid characters");
+        return NULL;
+    }
+
+    return g_base64_decode(input, out_len);
+}
diff --git a/util/bitmap.c b/util/bitmap.c
index 9c6bb52..40aadfb 100644
--- a/util/bitmap.c
+++ b/util/bitmap.c
@@ -9,11 +9,13 @@
  * Version 2.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 #include "qemu/bitmap.h"
+#include "qemu/atomic.h"
 
 /*
- * bitmaps provide an array of bits, implemented using an an
+ * bitmaps provide an array of bits, implemented using an
  * array of unsigned longs.  The number of valid bits in a
  * given bitmap does _not_ need to be an exact multiple of
  * BITS_PER_LONG.
@@ -177,6 +179,43 @@
     }
 }
 
+void bitmap_set_atomic(unsigned long *map, long start, long nr)
+{
+    unsigned long *p = map + BIT_WORD(start);
+    const long size = start + nr;
+    int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
+    unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
+
+    /* First word */
+    if (nr - bits_to_set > 0) {
+        atomic_or(p, mask_to_set);
+        nr -= bits_to_set;
+        bits_to_set = BITS_PER_LONG;
+        mask_to_set = ~0UL;
+        p++;
+    }
+
+    /* Full words */
+    if (bits_to_set == BITS_PER_LONG) {
+        while (nr >= BITS_PER_LONG) {
+            *p = ~0UL;
+            nr -= BITS_PER_LONG;
+            p++;
+        }
+    }
+
+    /* Last word */
+    if (nr) {
+        mask_to_set &= BITMAP_LAST_WORD_MASK(size);
+        atomic_or(p, mask_to_set);
+    } else {
+        /* If we avoided the full barrier in atomic_or(), issue a
+         * barrier to account for the assignments in the while loop.
+         */
+        smp_mb();
+    }
+}
+
 void bitmap_clear(unsigned long *map, long start, long nr)
 {
     unsigned long *p = map + BIT_WORD(start);
@@ -197,6 +236,51 @@
     }
 }
 
+bool bitmap_test_and_clear_atomic(unsigned long *map, long start, long nr)
+{
+    unsigned long *p = map + BIT_WORD(start);
+    const long size = start + nr;
+    int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
+    unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
+    unsigned long dirty = 0;
+    unsigned long old_bits;
+
+    /* First word */
+    if (nr - bits_to_clear > 0) {
+        old_bits = atomic_fetch_and(p, ~mask_to_clear);
+        dirty |= old_bits & mask_to_clear;
+        nr -= bits_to_clear;
+        bits_to_clear = BITS_PER_LONG;
+        mask_to_clear = ~0UL;
+        p++;
+    }
+
+    /* Full words */
+    if (bits_to_clear == BITS_PER_LONG) {
+        while (nr >= BITS_PER_LONG) {
+            if (*p) {
+                old_bits = atomic_xchg(p, 0);
+                dirty |= old_bits;
+            }
+            nr -= BITS_PER_LONG;
+            p++;
+        }
+    }
+
+    /* Last word */
+    if (nr) {
+        mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
+        old_bits = atomic_fetch_and(p, ~mask_to_clear);
+        dirty |= old_bits & mask_to_clear;
+    } else {
+        if (!dirty) {
+            smp_mb();
+        }
+    }
+
+    return dirty != 0;
+}
+
 #define ALIGN_MASK(x,mask)      (((x)+(mask))&~(mask))
 
 /**
diff --git a/util/bitops.c b/util/bitops.c
index 227c38b..b0c35dd 100644
--- a/util/bitops.c
+++ b/util/bitops.c
@@ -11,6 +11,7 @@
  * 2 of the License, or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu/bitops.h"
 
 #define BITOP_WORD(nr)		((nr) / BITS_PER_LONG)
diff --git a/util/buffer.c b/util/buffer.c
new file mode 100644
index 0000000..d8bb874
--- /dev/null
+++ b/util/buffer.c
@@ -0,0 +1,173 @@
+/*
+ * QEMU generic buffers
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/host-utils.h"
+#include "qemu/buffer.h"
+#include "trace.h"
+
+#define BUFFER_MIN_INIT_SIZE     4096
+#define BUFFER_MIN_SHRINK_SIZE  65536
+
+/* define the factor alpha for the exponential smoothing
+ * that is used in the average size calculation. a shift
+ * of 7 results in an alpha of 1/2^7. */
+#define BUFFER_AVG_SIZE_SHIFT       7
+
+static size_t buffer_req_size(Buffer *buffer, size_t len)
+{
+    return MAX(BUFFER_MIN_INIT_SIZE,
+               pow2ceil(buffer->offset + len));
+}
+
+static void buffer_adj_size(Buffer *buffer, size_t len)
+{
+    size_t old = buffer->capacity;
+    buffer->capacity = buffer_req_size(buffer, len);
+    buffer->buffer = g_realloc(buffer->buffer, buffer->capacity);
+    trace_buffer_resize(buffer->name ?: "unnamed",
+                        old, buffer->capacity);
+
+    /* make it even harder for the buffer to shrink, reset average size
+     * to current capacity if it is larger than the average. */
+    buffer->avg_size = MAX(buffer->avg_size,
+                           buffer->capacity << BUFFER_AVG_SIZE_SHIFT);
+}
+
+void buffer_init(Buffer *buffer, const char *name, ...)
+{
+    va_list ap;
+
+    va_start(ap, name);
+    buffer->name = g_strdup_vprintf(name, ap);
+    va_end(ap);
+}
+
+static uint64_t buffer_get_avg_size(Buffer *buffer)
+{
+    return buffer->avg_size >> BUFFER_AVG_SIZE_SHIFT;
+}
+
+void buffer_shrink(Buffer *buffer)
+{
+    size_t new;
+
+    /* Calculate the average size of the buffer as
+     * avg_size = avg_size * ( 1 - a ) + required_size * a
+     * where a is 1 / 2 ^ BUFFER_AVG_SIZE_SHIFT. */
+    buffer->avg_size *= (1 << BUFFER_AVG_SIZE_SHIFT) - 1;
+    buffer->avg_size >>= BUFFER_AVG_SIZE_SHIFT;
+    buffer->avg_size += buffer_req_size(buffer, 0);
+
+    /* And then only shrink if the average size of the buffer is much
+     * too big, to avoid bumping up & down the buffers all the time.
+     * realloc() isn't exactly cheap ...  */
+    new = buffer_req_size(buffer, buffer_get_avg_size(buffer));
+    if (new < buffer->capacity >> 3 &&
+        new >= BUFFER_MIN_SHRINK_SIZE) {
+        buffer_adj_size(buffer, buffer_get_avg_size(buffer));
+    }
+
+    buffer_adj_size(buffer, 0);
+}
+
+void buffer_reserve(Buffer *buffer, size_t len)
+{
+    if ((buffer->capacity - buffer->offset) < len) {
+        buffer_adj_size(buffer, len);
+    }
+}
+
+gboolean buffer_empty(Buffer *buffer)
+{
+    return buffer->offset == 0;
+}
+
+uint8_t *buffer_end(Buffer *buffer)
+{
+    return buffer->buffer + buffer->offset;
+}
+
+void buffer_reset(Buffer *buffer)
+{
+    buffer->offset = 0;
+    buffer_shrink(buffer);
+}
+
+void buffer_free(Buffer *buffer)
+{
+    trace_buffer_free(buffer->name ?: "unnamed", buffer->capacity);
+    g_free(buffer->buffer);
+    g_free(buffer->name);
+    buffer->offset = 0;
+    buffer->capacity = 0;
+    buffer->buffer = NULL;
+    buffer->name = NULL;
+}
+
+void buffer_append(Buffer *buffer, const void *data, size_t len)
+{
+    memcpy(buffer->buffer + buffer->offset, data, len);
+    buffer->offset += len;
+}
+
+void buffer_advance(Buffer *buffer, size_t len)
+{
+    memmove(buffer->buffer, buffer->buffer + len,
+            (buffer->offset - len));
+    buffer->offset -= len;
+    buffer_shrink(buffer);
+}
+
+void buffer_move_empty(Buffer *to, Buffer *from)
+{
+    trace_buffer_move_empty(to->name ?: "unnamed",
+                            from->offset,
+                            from->name ?: "unnamed");
+    assert(to->offset == 0);
+
+    g_free(to->buffer);
+    to->offset = from->offset;
+    to->capacity = from->capacity;
+    to->buffer = from->buffer;
+
+    from->offset = 0;
+    from->capacity = 0;
+    from->buffer = NULL;
+}
+
+void buffer_move(Buffer *to, Buffer *from)
+{
+    if (to->offset == 0) {
+        buffer_move_empty(to, from);
+        return;
+    }
+
+    trace_buffer_move(to->name ?: "unnamed",
+                      from->offset,
+                      from->name ?: "unnamed");
+    buffer_reserve(to, from->offset);
+    buffer_append(to, from->buffer, from->offset);
+
+    g_free(from->buffer);
+    from->offset = 0;
+    from->capacity = 0;
+    from->buffer = NULL;
+}
diff --git a/util/compatfd.c b/util/compatfd.c
index 341ada6..9a43042 100644
--- a/util/compatfd.c
+++ b/util/compatfd.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/compatfd.h"
 #include "qemu/thread.h"
@@ -108,22 +109,3 @@
 
     return qemu_signalfd_compat(mask);
 }
-
-bool qemu_signalfd_available(void)
-{
-#ifdef CONFIG_SIGNALFD
-    sigset_t mask;
-    int fd;
-    bool ok;
-    sigemptyset(&mask);
-    errno = 0;
-    fd = syscall(SYS_signalfd, -1, &mask, _NSIG / 8);
-    ok = (errno != ENOSYS);
-    if (fd >= 0) {
-        close(fd);
-    }
-    return ok;
-#else
-    return false;
-#endif
-}
diff --git a/util/coroutine-gthread.c b/util/coroutine-gthread.c
new file mode 100644
index 0000000..62bfb40
--- /dev/null
+++ b/util/coroutine-gthread.c
@@ -0,0 +1,198 @@
+/*
+ * GThread coroutine initialization code
+ *
+ * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
+ * Copyright (C) 2011  Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.0 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/coroutine_int.h"
+
+typedef struct {
+    Coroutine base;
+    GThread *thread;
+    bool runnable;
+    bool free_on_thread_exit;
+    CoroutineAction action;
+} CoroutineGThread;
+
+static CompatGMutex coroutine_lock;
+static CompatGCond coroutine_cond;
+
+/* GLib 2.31 and beyond deprecated various parts of the thread API,
+ * but the new interfaces are not available in older GLib versions
+ * so we have to cope with both.
+ */
+#if GLIB_CHECK_VERSION(2, 31, 0)
+/* Awkwardly, the GPrivate API doesn't provide a way to update the
+ * GDestroyNotify handler for the coroutine key dynamically. So instead
+ * we track whether or not the CoroutineGThread should be freed on
+ * thread exit / coroutine key update using the free_on_thread_exit
+ * field.
+ */
+static void coroutine_destroy_notify(gpointer data)
+{
+    CoroutineGThread *co = data;
+    if (co && co->free_on_thread_exit) {
+        g_free(co);
+    }
+}
+
+static GPrivate coroutine_key = G_PRIVATE_INIT(coroutine_destroy_notify);
+
+static inline CoroutineGThread *get_coroutine_key(void)
+{
+    return g_private_get(&coroutine_key);
+}
+
+static inline void set_coroutine_key(CoroutineGThread *co,
+                                     bool free_on_thread_exit)
+{
+    /* Unlike g_static_private_set() this does not call the GDestroyNotify
+     * if the previous value of the key was NULL. Fortunately we only need
+     * the GDestroyNotify in the non-NULL key case.
+     */
+    co->free_on_thread_exit = free_on_thread_exit;
+    g_private_replace(&coroutine_key, co);
+}
+
+static inline GThread *create_thread(GThreadFunc func, gpointer data)
+{
+    return g_thread_new("coroutine", func, data);
+}
+
+#else
+
+/* Handle older GLib versions */
+
+static GStaticPrivate coroutine_key = G_STATIC_PRIVATE_INIT;
+
+static inline CoroutineGThread *get_coroutine_key(void)
+{
+    return g_static_private_get(&coroutine_key);
+}
+
+static inline void set_coroutine_key(CoroutineGThread *co,
+                                     bool free_on_thread_exit)
+{
+    g_static_private_set(&coroutine_key, co,
+                         free_on_thread_exit ? (GDestroyNotify)g_free : NULL);
+}
+
+static inline GThread *create_thread(GThreadFunc func, gpointer data)
+{
+    return g_thread_create_full(func, data, 0, TRUE, TRUE,
+                                G_THREAD_PRIORITY_NORMAL, NULL);
+}
+
+#endif
+
+
+static void __attribute__((constructor)) coroutine_init(void)
+{
+#if !GLIB_CHECK_VERSION(2, 31, 0)
+    if (!g_thread_supported()) {
+        g_thread_init(NULL);
+    }
+#endif
+}
+
+static void coroutine_wait_runnable_locked(CoroutineGThread *co)
+{
+    while (!co->runnable) {
+        g_cond_wait(&coroutine_cond, &coroutine_lock);
+    }
+}
+
+static void coroutine_wait_runnable(CoroutineGThread *co)
+{
+    g_mutex_lock(&coroutine_lock);
+    coroutine_wait_runnable_locked(co);
+    g_mutex_unlock(&coroutine_lock);
+}
+
+static gpointer coroutine_thread(gpointer opaque)
+{
+    CoroutineGThread *co = opaque;
+
+    set_coroutine_key(co, false);
+    coroutine_wait_runnable(co);
+    co->base.entry(co->base.entry_arg);
+    qemu_coroutine_switch(&co->base, co->base.caller, COROUTINE_TERMINATE);
+    return NULL;
+}
+
+Coroutine *qemu_coroutine_new(void)
+{
+    CoroutineGThread *co;
+
+    co = g_malloc0(sizeof(*co));
+    co->thread = create_thread(coroutine_thread, co);
+    if (!co->thread) {
+        g_free(co);
+        return NULL;
+    }
+    return &co->base;
+}
+
+void qemu_coroutine_delete(Coroutine *co_)
+{
+    CoroutineGThread *co = DO_UPCAST(CoroutineGThread, base, co_);
+
+    g_thread_join(co->thread);
+    g_free(co);
+}
+
+CoroutineAction qemu_coroutine_switch(Coroutine *from_,
+                                      Coroutine *to_,
+                                      CoroutineAction action)
+{
+    CoroutineGThread *from = DO_UPCAST(CoroutineGThread, base, from_);
+    CoroutineGThread *to = DO_UPCAST(CoroutineGThread, base, to_);
+
+    g_mutex_lock(&coroutine_lock);
+    from->runnable = false;
+    from->action = action;
+    to->runnable = true;
+    to->action = action;
+    g_cond_broadcast(&coroutine_cond);
+
+    if (action != COROUTINE_TERMINATE) {
+        coroutine_wait_runnable_locked(from);
+    }
+    g_mutex_unlock(&coroutine_lock);
+    return from->action;
+}
+
+Coroutine *qemu_coroutine_self(void)
+{
+    CoroutineGThread *co = get_coroutine_key();
+    if (!co) {
+        co = g_malloc0(sizeof(*co));
+        co->runnable = true;
+        set_coroutine_key(co, true);
+    }
+
+    return &co->base;
+}
+
+bool qemu_in_coroutine(void)
+{
+    CoroutineGThread *co = get_coroutine_key();
+
+    return co && co->base.caller;
+}
diff --git a/util/coroutine-sigaltstack.c b/util/coroutine-sigaltstack.c
new file mode 100644
index 0000000..a7c3366
--- /dev/null
+++ b/util/coroutine-sigaltstack.c
@@ -0,0 +1,290 @@
+/*
+ * sigaltstack coroutine initialization code
+ *
+ * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
+ * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
+ * Copyright (C) 2012  Alex Barcelo <abarcelo@ac.upc.edu>
+** This file is partly based on pth_mctx.c, from the GNU Portable Threads
+**  Copyright (c) 1999-2006 Ralf S. Engelschall <rse@engelschall.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
+#ifdef _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+#include "qemu/osdep.h"
+#include <pthread.h>
+#include "qemu-common.h"
+#include "qemu/coroutine_int.h"
+
+typedef struct {
+    Coroutine base;
+    void *stack;
+    sigjmp_buf env;
+} CoroutineUContext;
+
+/**
+ * Per-thread coroutine bookkeeping
+ */
+typedef struct {
+    /** Currently executing coroutine */
+    Coroutine *current;
+
+    /** The default coroutine */
+    CoroutineUContext leader;
+
+    /** Information for the signal handler (trampoline) */
+    sigjmp_buf tr_reenter;
+    volatile sig_atomic_t tr_called;
+    void *tr_handler;
+} CoroutineThreadState;
+
+static pthread_key_t thread_state_key;
+
+static CoroutineThreadState *coroutine_get_thread_state(void)
+{
+    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
+
+    if (!s) {
+        s = g_malloc0(sizeof(*s));
+        s->current = &s->leader.base;
+        pthread_setspecific(thread_state_key, s);
+    }
+    return s;
+}
+
+static void qemu_coroutine_thread_cleanup(void *opaque)
+{
+    CoroutineThreadState *s = opaque;
+
+    g_free(s);
+}
+
+static void __attribute__((constructor)) coroutine_init(void)
+{
+    int ret;
+
+    ret = pthread_key_create(&thread_state_key, qemu_coroutine_thread_cleanup);
+    if (ret != 0) {
+        fprintf(stderr, "unable to create leader key: %s\n", strerror(errno));
+        abort();
+    }
+}
+
+/* "boot" function
+ * This is what starts the coroutine, is called from the trampoline
+ * (from the signal handler when it is not signal handling, read ahead
+ * for more information).
+ */
+static void coroutine_bootstrap(CoroutineUContext *self, Coroutine *co)
+{
+    /* Initialize longjmp environment and switch back the caller */
+    if (!sigsetjmp(self->env, 0)) {
+        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
+    }
+
+    while (true) {
+        co->entry(co->entry_arg);
+        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
+    }
+}
+
+/*
+ * This is used as the signal handler. This is called with the brand new stack
+ * (thanks to sigaltstack). We have to return, given that this is a signal
+ * handler and the sigmask and some other things are changed.
+ */
+static void coroutine_trampoline(int signal)
+{
+    CoroutineUContext *self;
+    Coroutine *co;
+    CoroutineThreadState *coTS;
+
+    /* Get the thread specific information */
+    coTS = coroutine_get_thread_state();
+    self = coTS->tr_handler;
+    coTS->tr_called = 1;
+    co = &self->base;
+
+    /*
+     * Here we have to do a bit of a ping pong between the caller, given that
+     * this is a signal handler and we have to do a return "soon". Then the
+     * caller can reestablish everything and do a siglongjmp here again.
+     */
+    if (!sigsetjmp(coTS->tr_reenter, 0)) {
+        return;
+    }
+
+    /*
+     * Ok, the caller has siglongjmp'ed back to us, so now prepare
+     * us for the real machine state switching. We have to jump
+     * into another function here to get a new stack context for
+     * the auto variables (which have to be auto-variables
+     * because the start of the thread happens later). Else with
+     * PIC (i.e. Position Independent Code which is used when PTH
+     * is built as a shared library) most platforms would
+     * horrible core dump as experience showed.
+     */
+    coroutine_bootstrap(self, co);
+}
+
+Coroutine *qemu_coroutine_new(void)
+{
+    const size_t stack_size = 1 << 20;
+    CoroutineUContext *co;
+    CoroutineThreadState *coTS;
+    struct sigaction sa;
+    struct sigaction osa;
+    stack_t ss;
+    stack_t oss;
+    sigset_t sigs;
+    sigset_t osigs;
+    sigjmp_buf old_env;
+
+    /* The way to manipulate stack is with the sigaltstack function. We
+     * prepare a stack, with it delivering a signal to ourselves and then
+     * put sigsetjmp/siglongjmp where needed.
+     * This has been done keeping coroutine-ucontext as a model and with the
+     * pth ideas (GNU Portable Threads). See coroutine-ucontext for the basics
+     * of the coroutines and see pth_mctx.c (from the pth project) for the
+     * sigaltstack way of manipulating stacks.
+     */
+
+    co = g_malloc0(sizeof(*co));
+    co->stack = g_malloc(stack_size);
+    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
+
+    coTS = coroutine_get_thread_state();
+    coTS->tr_handler = co;
+
+    /*
+     * Preserve the SIGUSR2 signal state, block SIGUSR2,
+     * and establish our signal handler. The signal will
+     * later transfer control onto the signal stack.
+     */
+    sigemptyset(&sigs);
+    sigaddset(&sigs, SIGUSR2);
+    pthread_sigmask(SIG_BLOCK, &sigs, &osigs);
+    sa.sa_handler = coroutine_trampoline;
+    sigfillset(&sa.sa_mask);
+    sa.sa_flags = SA_ONSTACK;
+    if (sigaction(SIGUSR2, &sa, &osa) != 0) {
+        abort();
+    }
+
+    /*
+     * Set the new stack.
+     */
+    ss.ss_sp = co->stack;
+    ss.ss_size = stack_size;
+    ss.ss_flags = 0;
+    if (sigaltstack(&ss, &oss) < 0) {
+        abort();
+    }
+
+    /*
+     * Now transfer control onto the signal stack and set it up.
+     * It will return immediately via "return" after the sigsetjmp()
+     * was performed. Be careful here with race conditions.  The
+     * signal can be delivered the first time sigsuspend() is
+     * called.
+     */
+    coTS->tr_called = 0;
+    pthread_kill(pthread_self(), SIGUSR2);
+    sigfillset(&sigs);
+    sigdelset(&sigs, SIGUSR2);
+    while (!coTS->tr_called) {
+        sigsuspend(&sigs);
+    }
+
+    /*
+     * Inform the system that we are back off the signal stack by
+     * removing the alternative signal stack. Be careful here: It
+     * first has to be disabled, before it can be removed.
+     */
+    sigaltstack(NULL, &ss);
+    ss.ss_flags = SS_DISABLE;
+    if (sigaltstack(&ss, NULL) < 0) {
+        abort();
+    }
+    sigaltstack(NULL, &ss);
+    if (!(oss.ss_flags & SS_DISABLE)) {
+        sigaltstack(&oss, NULL);
+    }
+
+    /*
+     * Restore the old SIGUSR2 signal handler and mask
+     */
+    sigaction(SIGUSR2, &osa, NULL);
+    pthread_sigmask(SIG_SETMASK, &osigs, NULL);
+
+    /*
+     * Now enter the trampoline again, but this time not as a signal
+     * handler. Instead we jump into it directly. The functionally
+     * redundant ping-pong pointer arithmetic is necessary to avoid
+     * type-conversion warnings related to the `volatile' qualifier and
+     * the fact that `jmp_buf' usually is an array type.
+     */
+    if (!sigsetjmp(old_env, 0)) {
+        siglongjmp(coTS->tr_reenter, 1);
+    }
+
+    /*
+     * Ok, we returned again, so now we're finished
+     */
+
+    return &co->base;
+}
+
+void qemu_coroutine_delete(Coroutine *co_)
+{
+    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
+
+    g_free(co->stack);
+    g_free(co);
+}
+
+CoroutineAction qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
+                                      CoroutineAction action)
+{
+    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
+    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
+    CoroutineThreadState *s = coroutine_get_thread_state();
+    int ret;
+
+    s->current = to_;
+
+    ret = sigsetjmp(from->env, 0);
+    if (ret == 0) {
+        siglongjmp(to->env, action);
+    }
+    return ret;
+}
+
+Coroutine *qemu_coroutine_self(void)
+{
+    CoroutineThreadState *s = coroutine_get_thread_state();
+
+    return s->current;
+}
+
+bool qemu_in_coroutine(void)
+{
+    CoroutineThreadState *s = pthread_getspecific(thread_state_key);
+
+    return s && s->current->caller;
+}
+
diff --git a/util/coroutine-ucontext.c b/util/coroutine-ucontext.c
new file mode 100644
index 0000000..2bb7e10
--- /dev/null
+++ b/util/coroutine-ucontext.c
@@ -0,0 +1,192 @@
+/*
+ * ucontext coroutine initialization code
+ *
+ * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
+ * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.0 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
+#ifdef _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+#include "qemu/osdep.h"
+#include <ucontext.h>
+#include "qemu-common.h"
+#include "qemu/coroutine_int.h"
+
+#ifdef CONFIG_VALGRIND_H
+#include <valgrind/valgrind.h>
+#endif
+
+typedef struct {
+    Coroutine base;
+    void *stack;
+    sigjmp_buf env;
+
+#ifdef CONFIG_VALGRIND_H
+    unsigned int valgrind_stack_id;
+#endif
+
+} CoroutineUContext;
+
+/**
+ * Per-thread coroutine bookkeeping
+ */
+static __thread CoroutineUContext leader;
+static __thread Coroutine *current;
+
+/*
+ * va_args to makecontext() must be type 'int', so passing
+ * the pointer we need may require several int args. This
+ * union is a quick hack to let us do that
+ */
+union cc_arg {
+    void *p;
+    int i[2];
+};
+
+static void coroutine_trampoline(int i0, int i1)
+{
+    union cc_arg arg;
+    CoroutineUContext *self;
+    Coroutine *co;
+
+    arg.i[0] = i0;
+    arg.i[1] = i1;
+    self = arg.p;
+    co = &self->base;
+
+    /* Initialize longjmp environment and switch back the caller */
+    if (!sigsetjmp(self->env, 0)) {
+        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
+    }
+
+    while (true) {
+        co->entry(co->entry_arg);
+        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
+    }
+}
+
+Coroutine *qemu_coroutine_new(void)
+{
+    const size_t stack_size = 1 << 20;
+    CoroutineUContext *co;
+    ucontext_t old_uc, uc;
+    sigjmp_buf old_env;
+    union cc_arg arg = {0};
+
+    /* The ucontext functions preserve signal masks which incurs a
+     * system call overhead.  sigsetjmp(buf, 0)/siglongjmp() does not
+     * preserve signal masks but only works on the current stack.
+     * Since we need a way to create and switch to a new stack, use
+     * the ucontext functions for that but sigsetjmp()/siglongjmp() for
+     * everything else.
+     */
+
+    if (getcontext(&uc) == -1) {
+        abort();
+    }
+
+    co = g_malloc0(sizeof(*co));
+    co->stack = g_malloc(stack_size);
+    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
+
+    uc.uc_link = &old_uc;
+    uc.uc_stack.ss_sp = co->stack;
+    uc.uc_stack.ss_size = stack_size;
+    uc.uc_stack.ss_flags = 0;
+
+#ifdef CONFIG_VALGRIND_H
+    co->valgrind_stack_id =
+        VALGRIND_STACK_REGISTER(co->stack, co->stack + stack_size);
+#endif
+
+    arg.p = co;
+
+    makecontext(&uc, (void (*)(void))coroutine_trampoline,
+                2, arg.i[0], arg.i[1]);
+
+    /* swapcontext() in, siglongjmp() back out */
+    if (!sigsetjmp(old_env, 0)) {
+        swapcontext(&old_uc, &uc);
+    }
+    return &co->base;
+}
+
+#ifdef CONFIG_VALGRIND_H
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+/* Work around an unused variable in the valgrind.h macro... */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
+#endif
+static inline void valgrind_stack_deregister(CoroutineUContext *co)
+{
+    VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
+}
+#ifdef CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE
+#pragma GCC diagnostic pop
+#endif
+#endif
+
+void qemu_coroutine_delete(Coroutine *co_)
+{
+    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
+
+#ifdef CONFIG_VALGRIND_H
+    valgrind_stack_deregister(co);
+#endif
+
+    g_free(co->stack);
+    g_free(co);
+}
+
+/* This function is marked noinline to prevent GCC from inlining it
+ * into coroutine_trampoline(). If we allow it to do that then it
+ * hoists the code to get the address of the TLS variable "current"
+ * out of the while() loop. This is an invalid transformation because
+ * the sigsetjmp() call may be called when running thread A but
+ * return in thread B, and so we might be in a different thread
+ * context each time round the loop.
+ */
+CoroutineAction __attribute__((noinline))
+qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
+                      CoroutineAction action)
+{
+    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
+    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
+    int ret;
+
+    current = to_;
+
+    ret = sigsetjmp(from->env, 0);
+    if (ret == 0) {
+        siglongjmp(to->env, action);
+    }
+    return ret;
+}
+
+Coroutine *qemu_coroutine_self(void)
+{
+    if (!current) {
+        current = &leader.base;
+    }
+    return current;
+}
+
+bool qemu_in_coroutine(void)
+{
+    return current && current->caller;
+}
diff --git a/util/coroutine-win32.c b/util/coroutine-win32.c
new file mode 100644
index 0000000..02e28e8
--- /dev/null
+++ b/util/coroutine-win32.c
@@ -0,0 +1,102 @@
+/*
+ * Win32 coroutine initialization code
+ *
+ * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/coroutine_int.h"
+
+typedef struct
+{
+    Coroutine base;
+
+    LPVOID fiber;
+    CoroutineAction action;
+} CoroutineWin32;
+
+static __thread CoroutineWin32 leader;
+static __thread Coroutine *current;
+
+/* This function is marked noinline to prevent GCC from inlining it
+ * into coroutine_trampoline(). If we allow it to do that then it
+ * hoists the code to get the address of the TLS variable "current"
+ * out of the while() loop. This is an invalid transformation because
+ * the SwitchToFiber() call may be called when running thread A but
+ * return in thread B, and so we might be in a different thread
+ * context each time round the loop.
+ */
+CoroutineAction __attribute__((noinline))
+qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
+                      CoroutineAction action)
+{
+    CoroutineWin32 *from = DO_UPCAST(CoroutineWin32, base, from_);
+    CoroutineWin32 *to = DO_UPCAST(CoroutineWin32, base, to_);
+
+    current = to_;
+
+    to->action = action;
+    SwitchToFiber(to->fiber);
+    return from->action;
+}
+
+static void CALLBACK coroutine_trampoline(void *co_)
+{
+    Coroutine *co = co_;
+
+    while (true) {
+        co->entry(co->entry_arg);
+        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
+    }
+}
+
+Coroutine *qemu_coroutine_new(void)
+{
+    const size_t stack_size = 1 << 20;
+    CoroutineWin32 *co;
+
+    co = g_malloc0(sizeof(*co));
+    co->fiber = CreateFiber(stack_size, coroutine_trampoline, &co->base);
+    return &co->base;
+}
+
+void qemu_coroutine_delete(Coroutine *co_)
+{
+    CoroutineWin32 *co = DO_UPCAST(CoroutineWin32, base, co_);
+
+    DeleteFiber(co->fiber);
+    g_free(co);
+}
+
+Coroutine *qemu_coroutine_self(void)
+{
+    if (!current) {
+        current = &leader.base;
+        leader.fiber = ConvertThreadToFiber(NULL);
+    }
+    return current;
+}
+
+bool qemu_in_coroutine(void)
+{
+    return current && current->caller;
+}
diff --git a/util/crc32c.c b/util/crc32c.c
index 8866327..7e99555 100644
--- a/util/crc32c.c
+++ b/util/crc32c.c
@@ -25,6 +25,7 @@
  *
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/crc32c.h"
 
diff --git a/util/cutils.c b/util/cutils.c
index dbe7412..7505fda 100644
--- a/util/cutils.c
+++ b/util/cutils.c
@@ -21,15 +21,15 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/host-utils.h"
 #include <math.h>
-#include <limits.h>
-#include <errno.h>
 
 #include "qemu/sockets.h"
 #include "qemu/iov.h"
 #include "net/net.h"
+#include "qemu/cutils.h"
 
 void strpadcpy(char *buf, int buf_size, const char *str, char pad)
 {
@@ -145,11 +145,6 @@
     return t;
 }
 
-int qemu_fls(int i)
-{
-    return 32 - clz32(i);
-}
-
 /*
  * Make sure data goes on disk, but if possible do not bother to
  * write out the inode just for timestamp updates.
@@ -166,6 +161,53 @@
 #endif
 }
 
+/* vector definitions */
+#ifdef __ALTIVEC__
+#include <altivec.h>
+/* The altivec.h header says we're allowed to undef these for
+ * C++ compatibility.  Here we don't care about C++, but we
+ * undef them anyway to avoid namespace pollution.
+ */
+#undef vector
+#undef pixel
+#undef bool
+#define VECTYPE        __vector unsigned char
+#define SPLAT(p)       vec_splat(vec_ld(0, p), 0)
+#define ALL_EQ(v1, v2) vec_all_eq(v1, v2)
+#define VEC_OR(v1, v2) ((v1) | (v2))
+/* altivec.h may redefine the bool macro as vector type.
+ * Reset it to POSIX semantics. */
+#define bool _Bool
+#elif defined __SSE2__
+#include <emmintrin.h>
+#define VECTYPE        __m128i
+#define SPLAT(p)       _mm_set1_epi8(*(p))
+#define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF)
+#define VEC_OR(v1, v2) (_mm_or_si128(v1, v2))
+#elif defined(__aarch64__)
+#include "arm_neon.h"
+#define VECTYPE        uint64x2_t
+#define ALL_EQ(v1, v2) \
+        ((vgetq_lane_u64(v1, 0) == vgetq_lane_u64(v2, 0)) && \
+         (vgetq_lane_u64(v1, 1) == vgetq_lane_u64(v2, 1)))
+#define VEC_OR(v1, v2) ((v1) | (v2))
+#else
+#define VECTYPE        unsigned long
+#define SPLAT(p)       (*(p) * (~0UL / 255))
+#define ALL_EQ(v1, v2) ((v1) == (v2))
+#define VEC_OR(v1, v2) ((v1) | (v2))
+#endif
+
+#define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8
+
+static bool
+can_use_buffer_find_nonzero_offset_inner(const void *buf, size_t len)
+{
+    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
+                   * sizeof(VECTYPE)) == 0
+            && ((uintptr_t) buf) % sizeof(VECTYPE) == 0);
+}
+
 /*
  * Searches for an area with non-zero content in a buffer
  *
@@ -174,8 +216,8 @@
  * and addr must be a multiple of sizeof(VECTYPE) due to
  * restriction of optimizations in this function.
  *
- * can_use_buffer_find_nonzero_offset() can be used to check
- * these requirements.
+ * can_use_buffer_find_nonzero_offset_inner() can be used to
+ * check these requirements.
  *
  * The return value is the offset of the non-zero area rounded
  * down to a multiple of sizeof(VECTYPE) for the first
@@ -186,13 +228,13 @@
  * If the buffer is all zero the return value is equal to len.
  */
 
-size_t buffer_find_nonzero_offset(const void *buf, size_t len)
+static size_t buffer_find_nonzero_offset_inner(const void *buf, size_t len)
 {
     const VECTYPE *p = buf;
     const VECTYPE zero = (VECTYPE){0};
     size_t i;
 
-    assert(can_use_buffer_find_nonzero_offset(buf, len));
+    assert(can_use_buffer_find_nonzero_offset_inner(buf, len));
 
     if (!len) {
         return 0;
@@ -207,13 +249,13 @@
     for (i = BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR;
          i < len / sizeof(VECTYPE);
          i += BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR) {
-        VECTYPE tmp0 = p[i + 0] | p[i + 1];
-        VECTYPE tmp1 = p[i + 2] | p[i + 3];
-        VECTYPE tmp2 = p[i + 4] | p[i + 5];
-        VECTYPE tmp3 = p[i + 6] | p[i + 7];
-        VECTYPE tmp01 = tmp0 | tmp1;
-        VECTYPE tmp23 = tmp2 | tmp3;
-        if (!ALL_EQ(tmp01 | tmp23, zero)) {
+        VECTYPE tmp0 = VEC_OR(p[i + 0], p[i + 1]);
+        VECTYPE tmp1 = VEC_OR(p[i + 2], p[i + 3]);
+        VECTYPE tmp2 = VEC_OR(p[i + 4], p[i + 5]);
+        VECTYPE tmp3 = VEC_OR(p[i + 6], p[i + 7]);
+        VECTYPE tmp01 = VEC_OR(tmp0, tmp1);
+        VECTYPE tmp23 = VEC_OR(tmp2, tmp3);
+        if (!ALL_EQ(VEC_OR(tmp01, tmp23), zero)) {
             break;
         }
     }
@@ -221,6 +263,108 @@
     return i * sizeof(VECTYPE);
 }
 
+#if defined CONFIG_AVX2_OPT
+#pragma GCC push_options
+#pragma GCC target("avx2")
+#include <cpuid.h>
+#include <immintrin.h>
+
+#define AVX2_VECTYPE        __m256i
+#define AVX2_SPLAT(p)       _mm256_set1_epi8(*(p))
+#define AVX2_ALL_EQ(v1, v2) \
+    (_mm256_movemask_epi8(_mm256_cmpeq_epi8(v1, v2)) == 0xFFFFFFFF)
+#define AVX2_VEC_OR(v1, v2) (_mm256_or_si256(v1, v2))
+
+static bool
+can_use_buffer_find_nonzero_offset_avx2(const void *buf, size_t len)
+{
+    return (len % (BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR
+                   * sizeof(AVX2_VECTYPE)) == 0
+            && ((uintptr_t) buf) % sizeof(AVX2_VECTYPE) == 0);
+}
+
+static size_t buffer_find_nonzero_offset_avx2(const void *buf, size_t len)
+{
+    const AVX2_VECTYPE *p = buf;
+    const AVX2_VECTYPE zero = (AVX2_VECTYPE){0};
+    size_t i;
+
+    assert(can_use_buffer_find_nonzero_offset_avx2(buf, len));
+
+    if (!len) {
+        return 0;
+    }
+
+    for (i = 0; i < BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR; i++) {
+        if (!AVX2_ALL_EQ(p[i], zero)) {
+            return i * sizeof(AVX2_VECTYPE);
+        }
+    }
+
+    for (i = BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR;
+         i < len / sizeof(AVX2_VECTYPE);
+         i += BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR) {
+        AVX2_VECTYPE tmp0 = AVX2_VEC_OR(p[i + 0], p[i + 1]);
+        AVX2_VECTYPE tmp1 = AVX2_VEC_OR(p[i + 2], p[i + 3]);
+        AVX2_VECTYPE tmp2 = AVX2_VEC_OR(p[i + 4], p[i + 5]);
+        AVX2_VECTYPE tmp3 = AVX2_VEC_OR(p[i + 6], p[i + 7]);
+        AVX2_VECTYPE tmp01 = AVX2_VEC_OR(tmp0, tmp1);
+        AVX2_VECTYPE tmp23 = AVX2_VEC_OR(tmp2, tmp3);
+        if (!AVX2_ALL_EQ(AVX2_VEC_OR(tmp01, tmp23), zero)) {
+            break;
+        }
+    }
+
+    return i * sizeof(AVX2_VECTYPE);
+}
+
+static bool avx2_support(void)
+{
+    int a, b, c, d;
+
+    if (__get_cpuid_max(0, NULL) < 7) {
+        return false;
+    }
+
+    __cpuid_count(7, 0, a, b, c, d);
+
+    return b & bit_AVX2;
+}
+
+bool can_use_buffer_find_nonzero_offset(const void *buf, size_t len) \
+         __attribute__ ((ifunc("can_use_buffer_find_nonzero_offset_ifunc")));
+size_t buffer_find_nonzero_offset(const void *buf, size_t len) \
+         __attribute__ ((ifunc("buffer_find_nonzero_offset_ifunc")));
+
+static void *buffer_find_nonzero_offset_ifunc(void)
+{
+    typeof(buffer_find_nonzero_offset) *func = (avx2_support()) ?
+        buffer_find_nonzero_offset_avx2 : buffer_find_nonzero_offset_inner;
+
+    return func;
+}
+
+static void *can_use_buffer_find_nonzero_offset_ifunc(void)
+{
+    typeof(can_use_buffer_find_nonzero_offset) *func = (avx2_support()) ?
+        can_use_buffer_find_nonzero_offset_avx2 :
+        can_use_buffer_find_nonzero_offset_inner;
+
+    return func;
+}
+#pragma GCC pop_options
+#else
+bool can_use_buffer_find_nonzero_offset(const void *buf, size_t len)
+{
+    return can_use_buffer_find_nonzero_offset_inner(buf, len);
+}
+
+size_t buffer_find_nonzero_offset(const void *buf, size_t len)
+{
+    return buffer_find_nonzero_offset_inner(buf, len);
+}
+#endif
+
 /*
  * Checks if a buffer is all zeroes
  *
@@ -281,19 +425,19 @@
 static int64_t suffix_mul(char suffix, int64_t unit)
 {
     switch (qemu_toupper(suffix)) {
-    case STRTOSZ_DEFSUFFIX_B:
+    case QEMU_STRTOSZ_DEFSUFFIX_B:
         return 1;
-    case STRTOSZ_DEFSUFFIX_KB:
+    case QEMU_STRTOSZ_DEFSUFFIX_KB:
         return unit;
-    case STRTOSZ_DEFSUFFIX_MB:
+    case QEMU_STRTOSZ_DEFSUFFIX_MB:
         return unit * unit;
-    case STRTOSZ_DEFSUFFIX_GB:
+    case QEMU_STRTOSZ_DEFSUFFIX_GB:
         return unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_TB:
+    case QEMU_STRTOSZ_DEFSUFFIX_TB:
         return unit * unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_PB:
+    case QEMU_STRTOSZ_DEFSUFFIX_PB:
         return unit * unit * unit * unit * unit;
-    case STRTOSZ_DEFSUFFIX_EB:
+    case QEMU_STRTOSZ_DEFSUFFIX_EB:
         return unit * unit * unit * unit * unit * unit;
     }
     return -1;
@@ -305,7 +449,7 @@
  * in *end, if not NULL. Return -ERANGE on overflow, Return -EINVAL on
  * other error.
  */
-int64_t strtosz_suffix_unit(const char *nptr, char **end,
+int64_t qemu_strtosz_suffix_unit(const char *nptr, char **end,
                             const char default_suffix, int64_t unit)
 {
     int64_t retval = -EINVAL;
@@ -348,14 +492,165 @@
     return retval;
 }
 
-int64_t strtosz_suffix(const char *nptr, char **end, const char default_suffix)
+int64_t qemu_strtosz_suffix(const char *nptr, char **end,
+                            const char default_suffix)
 {
-    return strtosz_suffix_unit(nptr, end, default_suffix, 1024);
+    return qemu_strtosz_suffix_unit(nptr, end, default_suffix, 1024);
 }
 
-int64_t strtosz(const char *nptr, char **end)
+int64_t qemu_strtosz(const char *nptr, char **end)
 {
-    return strtosz_suffix(nptr, end, STRTOSZ_DEFSUFFIX_MB);
+    return qemu_strtosz_suffix(nptr, end, QEMU_STRTOSZ_DEFSUFFIX_MB);
+}
+
+/**
+ * Helper function for qemu_strto*l() functions.
+ */
+static int check_strtox_error(const char *p, char *endptr, const char **next,
+                              int err)
+{
+    /* If no conversion was performed, prefer BSD behavior over glibc
+     * behavior.
+     */
+    if (err == 0 && endptr == p) {
+        err = EINVAL;
+    }
+    if (!next && *endptr) {
+        return -EINVAL;
+    }
+    if (next) {
+        *next = endptr;
+    }
+    return -err;
+}
+
+/**
+ * QEMU wrappers for strtol(), strtoll(), strtoul(), strotull() C functions.
+ *
+ * Convert ASCII string @nptr to a long integer value
+ * from the given @base. Parameters @nptr, @endptr, @base
+ * follows same semantics as strtol() C function.
+ *
+ * Unlike from strtol() function, if @endptr is not NULL, this
+ * function will return -EINVAL whenever it cannot fully convert
+ * the string in @nptr with given @base to a long. This function returns
+ * the result of the conversion only through the @result parameter.
+ *
+ * If NULL is passed in @endptr, then the whole string in @ntpr
+ * is a number otherwise it returns -EINVAL.
+ *
+ * RETURN VALUE
+ * Unlike from strtol() function, this wrapper returns either
+ * -EINVAL or the errno set by strtol() function (e.g -ERANGE).
+ * If the conversion overflows, -ERANGE is returned, and @result
+ * is set to the max value of the desired type
+ * (e.g. LONG_MAX, LLONG_MAX, ULONG_MAX, ULLONG_MAX). If the case
+ * of underflow, -ERANGE is returned, and @result is set to the min
+ * value of the desired type. For strtol(), strtoll(), @result is set to
+ * LONG_MIN, LLONG_MIN, respectively, and for strtoul(), strtoull() it
+ * is set to 0.
+ */
+int qemu_strtol(const char *nptr, const char **endptr, int base,
+                long *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtol(nptr, &p, base);
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to an unsigned long integer.
+ *
+ * If string contains a negative number, value will be converted to
+ * the unsigned representation of the signed value, unless the original
+ * (nonnegated) value would overflow, in this case, it will set @result
+ * to ULONG_MAX, and return ERANGE.
+ *
+ * The same behavior holds, for qemu_strtoull() but sets @result to
+ * ULLONG_MAX instead of ULONG_MAX.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoul(const char *nptr, const char **endptr, int base,
+                 unsigned long *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoul(nptr, &p, base);
+        /* Windows returns 1 for negative out-of-range values.  */
+        if (errno == ERANGE) {
+            *result = -1;
+        }
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to a long long integer.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoll(const char *nptr, const char **endptr, int base,
+                 int64_t *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoll(nptr, &p, base);
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
+}
+
+/**
+ * Converts ASCII string to an unsigned long long integer.
+ *
+ * See qemu_strtol() documentation for more info.
+ */
+int qemu_strtoull(const char *nptr, const char **endptr, int base,
+                  uint64_t *result)
+{
+    char *p;
+    int err = 0;
+    if (!nptr) {
+        if (endptr) {
+            *endptr = nptr;
+        }
+        err = -EINVAL;
+    } else {
+        errno = 0;
+        *result = strtoull(nptr, &p, base);
+        /* Windows returns 1 for negative out-of-range values.  */
+        if (errno == ERANGE) {
+            *result = -1;
+        }
+        err = check_strtox_error(nptr, p, endptr, errno);
+    }
+    return err;
 }
 
 /**
@@ -474,15 +769,6 @@
     return fd;
 }
 
-/* round down to the nearest power of 2*/
-int64_t pow2floor(int64_t value)
-{
-    if (!is_power_of_2(value)) {
-        value = 0x8000000000000000ULL >> clz64(value);
-    }
-    return value;
-}
-
 /*
  * Implementation of  ULEB128 (http://en.wikipedia.org/wiki/LEB128)
  * Input is limited to 14-bit numbers
@@ -523,16 +809,17 @@
 {
     char *debug_env = getenv(name);
     char *inv = NULL;
-    int debug;
+    long debug;
 
     if (!debug_env) {
         return initial;
     }
+    errno = 0;
     debug = strtol(debug_env, &inv, 10);
     if (inv == debug_env) {
         return initial;
     }
-    if (debug < 0 || debug > max) {
+    if (debug < 0 || debug > max || errno != 0) {
         fprintf(stderr, "warning: %s not in [0, %d]", name, max);
         return initial;
     }
diff --git a/util/envlist.c b/util/envlist.c
index ebc06cf..e86857e 100644
--- a/util/envlist.c
+++ b/util/envlist.c
@@ -1,3 +1,4 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/queue.h"
 #include "qemu/envlist.h"
@@ -94,30 +95,30 @@
 {
 	char *tmpenv, *envvar;
 	char *envsave = NULL;
-
-	assert(callback != NULL);
+    int ret = 0;
+    assert(callback != NULL);
 
 	if ((envlist == NULL) || (env == NULL))
 		return (EINVAL);
 
-	/*
-	 * We need to make temporary copy of the env string
-	 * as strtok_r(3) modifies it while it tokenizes.
-	 */
 	if ((tmpenv = strdup(env)) == NULL)
 		return (errno);
+    envsave = tmpenv;
 
-	envvar = strtok_r(tmpenv, ",", &envsave);
-	while (envvar != NULL) {
-		if ((*callback)(envlist, envvar) != 0) {
-			free(tmpenv);
-			return (errno);
+    do {
+        envvar = strchr(tmpenv, ',');
+        if (envvar != NULL) {
+            *envvar = '\0';
+        }
+        if ((*callback)(envlist, tmpenv) != 0) {
+            ret = errno;
+            break;
 		}
-		envvar = strtok_r(NULL, ",", &envsave);
-	}
+        tmpenv = envvar + 1;
+    } while (envvar != NULL);
 
-	free(tmpenv);
-	return (0);
+    free(envsave);
+    return ret;
 }
 
 /*
diff --git a/util/error.c b/util/error.c
index 9111b22..61fba43 100644
--- a/util/error.c
+++ b/util/error.c
@@ -2,36 +2,63 @@
  * QEMU Error Objects
  *
  * Copyright IBM, Corp. 2011
+ * Copyright (C) 2011-2015 Red Hat, Inc.
  *
  * Authors:
  *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Markus Armbruster <armbru@redhat.com>,
  *
  * This work is licensed under the terms of the GNU LGPL, version 2.  See
  * the COPYING.LIB file in the top-level directory.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/abort.h"
 #include "qapi/error.h"
+#include "qemu-common.h"
 #include "qemu/error-report.h"
-#ifdef CONFIG_ANDROID
-#include "android/error-messages.h"
-#endif  // CONFIG_ANDROID
 
 struct Error
 {
     char *msg;
     ErrorClass err_class;
+    const char *src, *func;
+    int line;
+    GString *hint;
 };
 
 Error *error_abort;
-#ifdef CONFIG_ANDROID
-Error *android_init_error_with_message;
-#endif  // CONFIG_ANDROID
+Error *error_fatal;
+Error *error_during_init;
 
-void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
+static void error_handle_fatal(Error **errp, Error *err)
+{
+    if (errp == &error_abort) {
+        if (qemu_abort_has_custom_handler()) {
+            qemu_abort("Unexpected error in %s() at %s:%d: %s\n",
+                    err->func, err->src, err->line,
+                    error_get_pretty(err));
+        }
+        fprintf(stderr, "Unexpected error in %s() at %s:%d:\n",
+                err->func, err->src, err->line);
+        error_report_err(err);
+        abort();
+    }
+    if (errp == &error_fatal) {
+        if (qemu_abort_has_custom_handler()) {
+            qemu_abort("Fatal error: %s\n", error_get_pretty(err));
+        }
+        error_report_err(err);
+        exit(1);
+    }
+}
+
+static void error_setv(Error **errp,
+                       const char *src, int line, const char *func,
+                       ErrorClass err_class, const char *fmt, va_list ap,
+                       const char *suffix)
 {
     Error *err;
-    va_list ap;
     int saved_errno = errno;
 
     if (errp == NULL) {
@@ -40,99 +67,140 @@
     assert(*errp == NULL);
 
     err = g_malloc0(sizeof(*err));
-
-    va_start(ap, fmt);
     err->msg = g_strdup_vprintf(fmt, ap);
-    va_end(ap);
-    err->err_class = err_class;
-
-    if (errp == &error_abort) {
-        error_report("%s", error_get_pretty(err));
-        abort();
+    if (suffix) {
+        char *msg = err->msg;
+        err->msg = g_strdup_printf("%s: %s", msg, suffix);
+        g_free(msg);
     }
+    err->err_class = err_class;
+    err->src = src;
+    err->line = line;
+    err->func = func;
 
+    error_handle_fatal(errp, err);
     *errp = err;
 
     errno = saved_errno;
 }
 
-void error_set_errno(Error **errp, int os_errno, ErrorClass err_class,
-                     const char *fmt, ...)
+void error_set_internal(Error **errp,
+                        const char *src, int line, const char *func,
+                        ErrorClass err_class, const char *fmt, ...)
 {
-    Error *err;
-    char *msg1;
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, err_class, fmt, ap, NULL);
+    va_end(ap);
+}
+
+void error_setg_internal(Error **errp,
+                         const char *src, int line, const char *func,
+                         const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap, NULL);
+    va_end(ap);
+}
+
+void error_setg_errno_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int os_errno, const char *fmt, ...)
+{
     va_list ap;
     int saved_errno = errno;
 
     if (errp == NULL) {
         return;
     }
-    assert(*errp == NULL);
-
-    err = g_malloc0(sizeof(*err));
 
     va_start(ap, fmt);
-    msg1 = g_strdup_vprintf(fmt, ap);
-    if (os_errno != 0) {
-        err->msg = g_strdup_printf("%s: %s", msg1, strerror(os_errno));
-        g_free(msg1);
-    } else {
-        err->msg = msg1;
-    }
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR, fmt, ap,
+               os_errno != 0 ? strerror(os_errno) : NULL);
     va_end(ap);
-    err->err_class = err_class;
-
-    if (errp == &error_abort) {
-        error_report("%s", error_get_pretty(err));
-        abort();
-    }
-
-    *errp = err;
 
     errno = saved_errno;
 }
 
-void error_setg_file_open(Error **errp, int os_errno, const char *filename)
+void error_setg_file_open_internal(Error **errp,
+                                   const char *src, int line, const char *func,
+                                   int os_errno, const char *filename)
 {
-    error_setg_errno(errp, os_errno, "Could not open '%s'", filename);
+    error_setg_errno_internal(errp, src, line, func, os_errno,
+                              "Could not open '%s'", filename);
+}
+
+void error_vprepend(Error **errp, const char *fmt, va_list ap)
+{
+    GString *newmsg;
+
+    if (!errp) {
+        return;
+    }
+
+    newmsg = g_string_new(NULL);
+    g_string_vprintf(newmsg, fmt, ap);
+    g_string_append(newmsg, (*errp)->msg);
+    (*errp)->msg = g_string_free(newmsg, 0);
+}
+
+void error_prepend(Error **errp, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_vprepend(errp, fmt, ap);
+    va_end(ap);
+}
+
+void error_append_hint(Error **errp, const char *fmt, ...)
+{
+    va_list ap;
+    int saved_errno = errno;
+    Error *err;
+
+    if (!errp) {
+        return;
+    }
+    err = *errp;
+    assert(err && errp != &error_abort && errp != &error_fatal);
+
+    if (!err->hint) {
+        err->hint = g_string_new(NULL);
+    }
+    va_start(ap, fmt);
+    g_string_append_vprintf(err->hint, fmt, ap);
+    va_end(ap);
+
+    errno = saved_errno;
 }
 
 #ifdef _WIN32
 
-void error_set_win32(Error **errp, int win32_err, ErrorClass err_class,
-                     const char *fmt, ...)
+void error_setg_win32_internal(Error **errp,
+                               const char *src, int line, const char *func,
+                               int win32_err, const char *fmt, ...)
 {
-    Error *err;
-    char *msg1;
     va_list ap;
+    char *suffix = NULL;
 
     if (errp == NULL) {
         return;
     }
-    assert(*errp == NULL);
 
-    err = g_malloc0(sizeof(*err));
+    if (win32_err != 0) {
+        suffix = g_win32_error_message(win32_err);
+    }
 
     va_start(ap, fmt);
-    msg1 = g_strdup_vprintf(fmt, ap);
-    if (win32_err != 0) {
-        char *msg2 = g_win32_error_message(win32_err);
-        err->msg = g_strdup_printf("%s: %s (error: %x)", msg1, msg2,
-                                   (unsigned)win32_err);
-        g_free(msg2);
-        g_free(msg1);
-    } else {
-        err->msg = msg1;
-    }
+    error_setv(errp, src, line, func, ERROR_CLASS_GENERIC_ERROR,
+               fmt, ap, suffix);
     va_end(ap);
-    err->err_class = err_class;
 
-    if (errp == &error_abort) {
-        error_report("%s", error_get_pretty(err));
-        abort();
-    }
-
-    *errp = err;
+    g_free(suffix);
 }
 
 #endif
@@ -144,6 +212,12 @@
     err_new = g_malloc0(sizeof(*err));
     err_new->msg = g_strdup(err->msg);
     err_new->err_class = err->err_class;
+    err_new->src = err->src;
+    err_new->line = err->line;
+    err_new->func = err->func;
+    if (err->hint) {
+        err_new->hint = g_string_new(err->hint->str);
+    }
 
     return err_new;
 }
@@ -153,35 +227,57 @@
     return err->err_class;
 }
 
-const char *error_get_pretty(Error *err)
+const char *error_get_pretty(const Error *err)
 {
     return err->msg;
 }
 
+void error_report_err(Error *err)
+{
+    error_report("%s", error_get_pretty(err));
+    if (err->hint) {
+        error_printf_unless_qmp("%s", err->hint->str);
+    }
+    error_free(err);
+}
+
+void error_reportf_err(Error *err, const char *fmt, ...)
+{
+    va_list ap;
+
+    va_start(ap, fmt);
+    error_vprepend(&err, fmt, ap);
+    va_end(ap);
+    error_report_err(err);
+}
+
 void error_free(Error *err)
 {
     if (err) {
         g_free(err->msg);
+        if (err->hint) {
+            g_string_free(err->hint, true);
+        }
         g_free(err);
     }
 }
 
+void error_free_or_abort(Error **errp)
+{
+    assert(errp && *errp);
+    error_free(*errp);
+    *errp = NULL;
+}
+
 void error_propagate(Error **dst_errp, Error *local_err)
 {
-#ifdef CONFIG_ANDROID
-    if (local_err && dst_errp == &android_init_error_with_message) {
-        android_init_error_set(local_err->err_class,
-                               error_get_pretty(local_err));
-        error_free(local_err);
+    if (!local_err) {
         return;
     }
-#endif  // CONFIG_ANDROID
-    if (local_err && dst_errp == &error_abort) {
-        error_report("%s", error_get_pretty(local_err));
-        abort();
-    } else if (dst_errp && !*dst_errp) {
+    error_handle_fatal(dst_errp, local_err);
+    if (dst_errp && !*dst_errp) {
         *dst_errp = local_err;
-    } else if (local_err) {
+    } else {
         error_free(local_err);
     }
 }
diff --git a/util/event_notifier-posix.c b/util/event_notifier-posix.c
index 8442c6e..c1f0d79 100644
--- a/util/event_notifier-posix.c
+++ b/util/event_notifier-posix.c
@@ -10,7 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/event_notifier.h"
 #include "sysemu/char.h"
 #include "qemu/main-loop.h"
@@ -19,11 +21,17 @@
 #include <sys/eventfd.h>
 #endif
 
+#ifdef CONFIG_EVENTFD
+/*
+ * Initialize @e with existing file descriptor @fd.
+ * @fd must be a genuine eventfd object, emulation with pipe won't do.
+ */
 void event_notifier_init_fd(EventNotifier *e, int fd)
 {
     e->rfd = fd;
     e->wfd = fd;
 }
+#endif
 
 int event_notifier_init(EventNotifier *e, int active)
 {
@@ -77,15 +85,18 @@
     close(e->wfd);
 }
 
-int event_notifier_get_fd(EventNotifier *e)
+int event_notifier_get_fd(const EventNotifier *e)
 {
     return e->rfd;
 }
 
 int event_notifier_set_handler(EventNotifier *e,
+                               bool is_external,
                                EventNotifierHandler *handler)
 {
-    return qemu_set_fd_handler(e->rfd, (IOHandler *)handler, NULL, e);
+    aio_set_fd_handler(iohandler_get_aio_context(), e->rfd, is_external,
+                       (IOHandler *)handler, NULL, e);
+    return 0;
 }
 
 int event_notifier_set(EventNotifier *e)
diff --git a/util/event_notifier-win32.c b/util/event_notifier-win32.c
index 6dbb530..de87df0 100644
--- a/util/event_notifier-win32.c
+++ b/util/event_notifier-win32.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/event_notifier.h"
 #include "qemu/main-loop.h"
@@ -32,6 +33,7 @@
 }
 
 int event_notifier_set_handler(EventNotifier *e,
+                               bool is_external,
                                EventNotifierHandler *handler)
 {
     if (handler) {
diff --git a/util/fifo8.c b/util/fifo8.c
index 0ea5ad9..5c64101 100644
--- a/util/fifo8.c
+++ b/util/fifo8.c
@@ -12,6 +12,7 @@
  * with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/fifo8.h"
 
diff --git a/util/getauxval.c b/util/getauxval.c
index 1732ace..0b3bae2 100644
--- a/util/getauxval.c
+++ b/util/getauxval.c
@@ -22,8 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
 #include "qemu/osdep.h"
+#include "qemu-common.h"
 
 #ifdef CONFIG_GETAUXVAL
 /* Don't inline this in qemu/osdep.h, because pulling in <sys/auxv.h> for
diff --git a/util/hbitmap.c b/util/hbitmap.c
index b3060e6..99fd2ba 100644
--- a/util/hbitmap.c
+++ b/util/hbitmap.c
@@ -9,9 +9,6 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include <string.h>
-#include <glib.h>
-#include <assert.h>
 #include "qemu/osdep.h"
 #include "qemu/hbitmap.h"
 #include "qemu/host-utils.h"
@@ -90,6 +87,9 @@
      * bitmap will still allocate HBITMAP_LEVELS arrays.
      */
     unsigned long *levels[HBITMAP_LEVELS];
+
+    /* The length of each levels[] array. */
+    uint64_t sizes[HBITMAP_LEVELS];
 };
 
 /* Advance hbi to the next nonzero word and return it.  hbi->pos
@@ -269,6 +269,7 @@
     start >>= hb->granularity;
     last >>= hb->granularity;
     count = last - start + 1;
+    assert(last < hb->size);
 
     hb->count += count - hb_count_between(hb, start, last);
     hb_set_between(hb, HBITMAP_LEVELS - 1, start, last);
@@ -348,16 +349,31 @@
 
     start >>= hb->granularity;
     last >>= hb->granularity;
+    assert(last < hb->size);
 
     hb->count -= hb_count_between(hb, start, last);
     hb_reset_between(hb, HBITMAP_LEVELS - 1, start, last);
 }
 
+void hbitmap_reset_all(HBitmap *hb)
+{
+    unsigned int i;
+
+    /* Same as hbitmap_alloc() except for memset() instead of malloc() */
+    for (i = HBITMAP_LEVELS; --i >= 1; ) {
+        memset(hb->levels[i], 0, hb->sizes[i] * sizeof(unsigned long));
+    }
+
+    hb->levels[0][0] = 1UL << (BITS_PER_LONG - 1);
+    hb->count = 0;
+}
+
 bool hbitmap_get(const HBitmap *hb, uint64_t item)
 {
     /* Compute position and bit in the last layer.  */
     uint64_t pos = item >> hb->granularity;
     unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1));
+    assert(pos < hb->size);
 
     return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0;
 }
@@ -373,7 +389,7 @@
 
 HBitmap *hbitmap_alloc(uint64_t size, int granularity)
 {
-    HBitmap *hb = g_malloc0(sizeof (struct HBitmap));
+    HBitmap *hb = g_new0(struct HBitmap, 1);
     unsigned i;
 
     assert(granularity >= 0 && granularity < 64);
@@ -384,7 +400,8 @@
     hb->granularity = granularity;
     for (i = HBITMAP_LEVELS; i-- > 0; ) {
         size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
-        hb->levels[i] = g_malloc0(size * sizeof(unsigned long));
+        hb->sizes[i] = size;
+        hb->levels[i] = g_new0(unsigned long, size);
     }
 
     /* We necessarily have free bits in level 0 due to the definition
@@ -395,3 +412,84 @@
     hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
     return hb;
 }
+
+void hbitmap_truncate(HBitmap *hb, uint64_t size)
+{
+    bool shrink;
+    unsigned i;
+    uint64_t num_elements = size;
+    uint64_t old;
+
+    /* Size comes in as logical elements, adjust for granularity. */
+    size = (size + (1ULL << hb->granularity) - 1) >> hb->granularity;
+    assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
+    shrink = size < hb->size;
+
+    /* bit sizes are identical; nothing to do. */
+    if (size == hb->size) {
+        return;
+    }
+
+    /* If we're losing bits, let's clear those bits before we invalidate all of
+     * our invariants. This helps keep the bitcount consistent, and will prevent
+     * us from carrying around garbage bits beyond the end of the map.
+     */
+    if (shrink) {
+        /* Don't clear partial granularity groups;
+         * start at the first full one. */
+        uint64_t start = QEMU_ALIGN_UP(num_elements, 1 << hb->granularity);
+        uint64_t fix_count = (hb->size << hb->granularity) - start;
+
+        assert(fix_count);
+        hbitmap_reset(hb, start, fix_count);
+    }
+
+    hb->size = size;
+    for (i = HBITMAP_LEVELS; i-- > 0; ) {
+        size = MAX(BITS_TO_LONGS(size), 1);
+        if (hb->sizes[i] == size) {
+            break;
+        }
+        old = hb->sizes[i];
+        hb->sizes[i] = size;
+        hb->levels[i] = g_realloc(hb->levels[i], size * sizeof(unsigned long));
+        if (!shrink) {
+            memset(&hb->levels[i][old], 0x00,
+                   (size - old) * sizeof(*hb->levels[i]));
+        }
+    }
+}
+
+
+/**
+ * Given HBitmaps A and B, let A := A (BITOR) B.
+ * Bitmap B will not be modified.
+ *
+ * @return true if the merge was successful,
+ *         false if it was not attempted.
+ */
+bool hbitmap_merge(HBitmap *a, const HBitmap *b)
+{
+    int i;
+    uint64_t j;
+
+    if ((a->size != b->size) || (a->granularity != b->granularity)) {
+        return false;
+    }
+
+    if (hbitmap_count(b) == 0) {
+        return true;
+    }
+
+    /* This merge is O(size), as BITS_PER_LONG and HBITMAP_LEVELS are constant.
+     * It may be possible to improve running times for sparsely populated maps
+     * by using hbitmap_iter_next, but this is suboptimal for dense maps.
+     */
+    for (i = HBITMAP_LEVELS - 1; i >= 0; i--) {
+        for (j = 0; j < a->sizes[i]; j++) {
+            a->levels[i][j] |= b->levels[i][j];
+        }
+    }
+
+    return true;
+}
diff --git a/util/hexdump.c b/util/hexdump.c
index 969b340..f879ff0 100644
--- a/util/hexdump.c
+++ b/util/hexdump.c
@@ -13,25 +13,37 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size)
 {
-    unsigned int b;
+    unsigned int b, len, i, c;
 
-    for (b = 0; b < size; b++) {
-        if ((b % 16) == 0) {
-            fprintf(fp, "%s: %04x:", prefix, b);
+    for (b = 0; b < size; b += 16) {
+        len = size - b;
+        if (len > 16) {
+            len = 16;
         }
-        if ((b % 4) == 0) {
-            fprintf(fp, " ");
+        fprintf(fp, "%s: %04x:", prefix, b);
+        for (i = 0; i < 16; i++) {
+            if ((i % 4) == 0) {
+                fprintf(fp, " ");
+            }
+            if (i < len) {
+                fprintf(fp, " %02x", (unsigned char)buf[b + i]);
+            } else {
+                fprintf(fp, "   ");
+            }
         }
-        fprintf(fp, " %02x", (unsigned char)buf[b]);
-        if ((b % 16) == 15) {
-            fprintf(fp, "\n");
+        fprintf(fp, " ");
+        for (i = 0; i < len; i++) {
+            c = buf[b + i];
+            if (c < ' ' || c > '~') {
+                c = '.';
+            }
+            fprintf(fp, "%c", c);
         }
-    }
-    if ((b % 16) != 0) {
         fprintf(fp, "\n");
     }
 }
diff --git a/util/host-utils.c b/util/host-utils.c
index 102e5bf..b166e57 100644
--- a/util/host-utils.c
+++ b/util/host-utils.c
@@ -23,8 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include <stdlib.h>
-#include <stdint.h>
+#include "qemu/osdep.h"
 #include "qemu/host-utils.h"
 
 /* Long integer helpers */
diff --git a/util/id.c b/util/id.c
index 09b22fb..6141352 100644
--- a/util/id.c
+++ b/util/id.c
@@ -10,7 +10,9 @@
  * or later.  See the COPYING.LIB file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/id.h"
 
 bool id_wellformed(const char *id)
 {
@@ -26,3 +28,40 @@
     }
     return true;
 }
+
+#define ID_SPECIAL_CHAR '#'
+
+static const char *const id_subsys_str[ID_MAX] = {
+    [ID_QDEV]  = "qdev",
+    [ID_BLOCK] = "block",
+};
+
+/*
+ *  Generates an ID of the form PREFIX SUBSYSTEM NUMBER
+ *  where:
+ *
+ *  - PREFIX is the reserved character '#'
+ *  - SUBSYSTEM identifies the subsystem creating the ID
+ *  - NUMBER is a decimal number unique within SUBSYSTEM.
+ *
+ *    Example: "#block146"
+ *
+ * Note that these IDs do not satisfy id_wellformed().
+ *
+ * The caller is responsible for freeing the returned string with g_free()
+ */
+char *id_generate(IdSubSystems id)
+{
+    static uint64_t id_counters[ID_MAX];
+    uint32_t rnd;
+
+    assert(id < ARRAY_SIZE(id_subsys_str));
+    assert(id_subsys_str[id]);
+
+    rnd = g_random_int_range(0, 100);
+
+    return g_strdup_printf("%c%s%" PRIu64 "%02" PRId32, ID_SPECIAL_CHAR,
+                                                        id_subsys_str[id],
+                                                        id_counters[id]++,
+                                                        rnd);
+}
diff --git a/util/iov.c b/util/iov.c
index 1dc2ef2..dc12bab 100644
--- a/util/iov.c
+++ b/util/iov.c
@@ -16,11 +16,14 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
+#include "qemu-common.h"
 #include "qemu/iov.h"
 #include "qemu/sockets.h"
+#include "qemu/cutils.h"
 
-size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
-                    size_t offset, const void *buf, size_t bytes)
+size_t iov_from_buf_full(const struct iovec *iov, unsigned int iov_cnt,
+                         size_t offset, const void *buf, size_t bytes)
 {
     size_t done;
     unsigned int i;
@@ -38,8 +41,8 @@
     return done;
 }
 
-size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
-                  size_t offset, void *buf, size_t bytes)
+size_t iov_to_buf_full(const struct iovec *iov, const unsigned int iov_cnt,
+                       size_t offset, void *buf, size_t bytes)
 {
     size_t done;
     unsigned int i;
@@ -133,7 +136,7 @@
 #endif
 }
 
-ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
+ssize_t iov_send_recv(int sockfd, const struct iovec *_iov, unsigned iov_cnt,
                       size_t offset, size_t bytes,
                       bool do_send)
 {
@@ -141,6 +144,16 @@
     ssize_t ret;
     size_t orig_len, tail;
     unsigned niov;
+    struct iovec *local_iov, *iov;
+
+    if (bytes <= 0) {
+        return 0;
+    }
+
+    local_iov = g_new0(struct iovec, iov_cnt);
+    iov_copy(local_iov, iov_cnt, _iov, iov_cnt, offset, bytes);
+    offset = 0;
+    iov = local_iov;
 
     while (bytes > 0) {
         /* Find the start position, skipping `offset' bytes:
@@ -187,6 +200,7 @@
 
         if (ret < 0) {
             assert(errno != EINTR);
+            g_free(local_iov);
             if (errno == EAGAIN && total > 0) {
                 return total;
             }
@@ -205,6 +219,7 @@
         bytes -= ret;
     }
 
+    g_free(local_iov);
     return total;
 }
 
@@ -232,7 +247,8 @@
 {
     size_t len;
     unsigned int i, j;
-    for (i = 0, j = 0; i < iov_cnt && j < dst_iov_cnt && bytes; i++) {
+    for (i = 0, j = 0;
+         i < iov_cnt && j < dst_iov_cnt && (offset || bytes); i++) {
         if (offset >= iov[i].iov_len) {
             offset -= iov[i].iov_len;
             continue;
@@ -253,7 +269,7 @@
 
 void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
 {
-    qiov->iov = g_malloc(alloc_hint * sizeof(struct iovec));
+    qiov->iov = g_new(struct iovec, alloc_hint);
     qiov->niov = 0;
     qiov->nalloc = alloc_hint;
     qiov->size = 0;
@@ -277,7 +293,7 @@
 
     if (qiov->niov == qiov->nalloc) {
         qiov->nalloc = 2 * qiov->nalloc + 1;
-        qiov->iov = g_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
+        qiov->iov = g_renew(struct iovec, qiov->iov, qiov->nalloc);
     }
     qiov->iov[qiov->niov].iov_base = base;
     qiov->iov[qiov->niov].iov_len = len;
diff --git a/util/log.c b/util/log.c
new file mode 100644
index 0000000..54b54e8
--- /dev/null
+++ b/util/log.c
@@ -0,0 +1,338 @@
+/*
+ * Logging support
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/log.h"
+#include "qemu/range.h"
+#include "qemu/error-report.h"
+#include "qapi/error.h"
+#include "qemu/cutils.h"
+#include "trace/control.h"
+
+static char *logfilename;
+FILE *qemu_logfile;
+int qemu_loglevel;
+static int log_append = 0;
+static GArray *debug_regions;
+
+/* Return the number of characters emitted.  */
+int qemu_log(const char *fmt, ...)
+{
+    int ret = 0;
+    if (qemu_logfile) {
+        va_list ap;
+        va_start(ap, fmt);
+        ret = vfprintf(qemu_logfile, fmt, ap);
+        va_end(ap);
+
+        /* Don't pass back error results.  */
+        if (ret < 0) {
+            ret = 0;
+        }
+    }
+    return ret;
+}
+
+static bool log_uses_own_buffers;
+
+/* enable or disable low levels log */
+void qemu_set_log(int log_flags)
+{
+    qemu_loglevel = log_flags;
+#ifdef CONFIG_TRACE_LOG
+    qemu_loglevel |= LOG_TRACE;
+#endif
+    if (!qemu_logfile &&
+        (is_daemonized() ? logfilename != NULL : qemu_loglevel)) {
+        if (logfilename) {
+            qemu_logfile = fopen(logfilename, log_append ? "a" : "w");
+            if (!qemu_logfile) {
+                perror(logfilename);
+                _exit(1);
+            }
+            /* In case we are a daemon redirect stderr to logfile */
+            if (is_daemonized()) {
+                dup2(fileno(qemu_logfile), STDERR_FILENO);
+                fclose(qemu_logfile);
+                /* This will skip closing logfile in qemu_log_close() */
+                qemu_logfile = stderr;
+            }
+        } else {
+            /* Default to stderr if no log file specified */
+            assert(!is_daemonized());
+            qemu_logfile = stderr;
+        }
+        /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
+        if (log_uses_own_buffers) {
+            static char logfile_buf[4096];
+
+            setvbuf(qemu_logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
+        } else {
+#if defined(_WIN32)
+            /* Win32 doesn't support line-buffering, so use unbuffered output. */
+            setvbuf(qemu_logfile, NULL, _IONBF, 0);
+#else
+            setvbuf(qemu_logfile, NULL, _IOLBF, 0);
+#endif
+            log_append = 1;
+        }
+    }
+    if (qemu_logfile &&
+        (is_daemonized() ? logfilename == NULL : !qemu_loglevel)) {
+        qemu_log_close();
+    }
+}
+
+void qemu_log_needs_buffers(void)
+{
+    log_uses_own_buffers = true;
+}
+
+/*
+ * Allow the user to include %d in their logfile which will be
+ * substituted with the current PID. This is useful for debugging many
+ * nested linux-user tasks but will result in lots of logs.
+ */
+void qemu_set_log_filename(const char *filename, Error **errp)
+{
+    char *pidstr;
+    g_free(logfilename);
+
+    pidstr = strstr(filename, "%");
+    if (pidstr) {
+        /* We only accept one %d, no other format strings */
+        if (pidstr[1] != 'd' || strchr(pidstr + 2, '%')) {
+            error_setg(errp, "Bad logfile format: %s", filename);
+            return;
+        } else {
+            logfilename = g_strdup_printf(filename, getpid());
+        }
+    } else {
+        logfilename = g_strdup(filename);
+    }
+    qemu_log_close();
+    qemu_set_log(qemu_loglevel);
+}
+
+/* Returns true if addr is in our debug filter or no filter defined
+ */
+bool qemu_log_in_addr_range(uint64_t addr)
+{
+    if (debug_regions) {
+        int i = 0;
+        for (i = 0; i < debug_regions->len; i++) {
+            Range *range = &g_array_index(debug_regions, Range, i);
+            if (range_contains(range, addr)) {
+                return true;
+            }
+        }
+        return false;
+    } else {
+        return true;
+    }
+}
+
+
+void qemu_set_dfilter_ranges(const char *filter_spec, Error **errp)
+{
+    gchar **ranges = g_strsplit(filter_spec, ",", 0);
+    int i;
+
+    if (debug_regions) {
+        g_array_unref(debug_regions);
+        debug_regions = NULL;
+    }
+
+    debug_regions = g_array_sized_new(FALSE, FALSE,
+                                      sizeof(Range), g_strv_length(ranges));
+    for (i = 0; ranges[i]; i++) {
+        const char *r = ranges[i];
+        const char *range_op, *r2, *e;
+        uint64_t r1val, r2val, lob, upb;
+        struct Range range;
+
+        range_op = strstr(r, "-");
+        r2 = range_op ? range_op + 1 : NULL;
+        if (!range_op) {
+            range_op = strstr(r, "+");
+            r2 = range_op ? range_op + 1 : NULL;
+        }
+        if (!range_op) {
+            range_op = strstr(r, "..");
+            r2 = range_op ? range_op + 2 : NULL;
+        }
+        if (!range_op) {
+            error_setg(errp, "Bad range specifier");
+            goto out;
+        }
+
+        if (qemu_strtoull(r, &e, 0, &r1val)
+            || e != range_op) {
+            error_setg(errp, "Invalid number to the left of %.*s",
+                       (int)(r2 - range_op), range_op);
+            goto out;
+        }
+        if (qemu_strtoull(r2, NULL, 0, &r2val)) {
+            error_setg(errp, "Invalid number to the right of %.*s",
+                       (int)(r2 - range_op), range_op);
+            goto out;
+        }
+
+        switch (*range_op) {
+        case '+':
+            lob = r1val;
+            upb = r1val + r2val - 1;
+            break;
+        case '-':
+            upb = r1val;
+            lob = r1val - (r2val - 1);
+            break;
+        case '.':
+            lob = r1val;
+            upb = r2val;
+            break;
+        default:
+            g_assert_not_reached();
+        }
+        if (lob > upb) {
+            error_setg(errp, "Invalid range");
+            goto out;
+        }
+        range_set_bounds(&range, lob, upb);
+        g_array_append_val(debug_regions, range);
+    }
+out:
+    g_strfreev(ranges);
+}
+
+/* fflush() the log file */
+void qemu_log_flush(void)
+{
+    fflush(qemu_logfile);
+}
+
+/* Close the log file */
+void qemu_log_close(void)
+{
+    if (qemu_logfile) {
+        if (qemu_logfile != stderr) {
+            fclose(qemu_logfile);
+        }
+        qemu_logfile = NULL;
+    }
+}
+
+const QEMULogItem qemu_log_items[] = {
+    { CPU_LOG_TB_OUT_ASM, "out_asm",
+      "show generated host assembly code for each compiled TB" },
+    { CPU_LOG_TB_IN_ASM, "in_asm",
+      "show target assembly code for each compiled TB" },
+    { CPU_LOG_TB_OP, "op",
+      "show micro ops for each compiled TB" },
+    { CPU_LOG_TB_OP_OPT, "op_opt",
+      "show micro ops after optimization" },
+    { CPU_LOG_TB_OP_IND, "op_ind",
+      "show micro ops before indirect lowering" },
+    { CPU_LOG_INT, "int",
+      "show interrupts/exceptions in short format" },
+    { CPU_LOG_EXEC, "exec",
+      "show trace before each executed TB (lots of logs)" },
+    { CPU_LOG_TB_CPU, "cpu",
+      "show CPU registers before entering a TB (lots of logs)" },
+    { CPU_LOG_MMU, "mmu",
+      "log MMU-related activities" },
+    { CPU_LOG_PCALL, "pcall",
+      "x86 only: show protected mode far calls/returns/exceptions" },
+    { CPU_LOG_RESET, "cpu_reset",
+      "show CPU state before CPU resets" },
+    { LOG_UNIMP, "unimp",
+      "log unimplemented functionality" },
+    { LOG_GUEST_ERROR, "guest_errors",
+      "log when the guest OS does something invalid (eg accessing a\n"
+      "non-existent register)" },
+    { CPU_LOG_PAGE, "page",
+      "dump pages at beginning of user mode emulation" },
+    { CPU_LOG_TB_NOCHAIN, "nochain",
+      "do not chain compiled TBs so that \"exec\" and \"cpu\" show\n"
+      "complete traces" },
+    { 0, NULL, NULL },
+};
+
+static int cmp1(const char *s1, int n, const char *s2)
+{
+    if (strlen(s2) != n) {
+        return 0;
+    }
+    return memcmp(s1, s2, n) == 0;
+}
+
+/* takes a comma separated list of log masks. Return 0 if error. */
+int qemu_str_to_log_mask(const char *str)
+{
+    const QEMULogItem *item;
+    int mask;
+    const char *p, *p1;
+
+    p = str;
+    mask = 0;
+    for (;;) {
+        p1 = strchr(p, ',');
+        if (!p1) {
+            p1 = p + strlen(p);
+        }
+        if (cmp1(p,p1-p,"all")) {
+            for (item = qemu_log_items; item->mask != 0; item++) {
+                mask |= item->mask;
+            }
+#ifdef CONFIG_TRACE_LOG
+        } else if (strncmp(p, "trace:", 6) == 0 && p + 6 != p1) {
+            trace_enable_events(p + 6);
+            mask |= LOG_TRACE;
+#endif
+        } else {
+            for (item = qemu_log_items; item->mask != 0; item++) {
+                if (cmp1(p, p1 - p, item->name)) {
+                    goto found;
+                }
+            }
+            return 0;
+        found:
+            mask |= item->mask;
+        }
+        if (*p1 != ',') {
+            break;
+        }
+        p = p1 + 1;
+    }
+    return mask;
+}
+
+void qemu_print_log_usage(FILE *f)
+{
+    const QEMULogItem *item;
+    fprintf(f, "Log items (comma separated):\n");
+    for (item = qemu_log_items; item->mask != 0; item++) {
+        fprintf(f, "%-15s %s\n", item->name, item->help);
+    }
+#ifdef CONFIG_TRACE_LOG
+    fprintf(f, "trace:PATTERN   enable trace events\n");
+    fprintf(f, "\nUse \"-d trace:help\" to get a list of trace events.\n\n");
+#endif
+}
diff --git a/util/memfd.c b/util/memfd.c
new file mode 100644
index 0000000..4571d1a
--- /dev/null
+++ b/util/memfd.c
@@ -0,0 +1,159 @@
+/*
+ * memfd.c
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * QEMU library functions on POSIX which are shared between QEMU and
+ * the QEMU tools.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+
+#include <glib/gprintf.h>
+
+#include "qemu/memfd.h"
+
+#ifdef CONFIG_MEMFD
+#include <sys/memfd.h>
+#elif defined CONFIG_LINUX
+#include <sys/syscall.h>
+#include <asm/unistd.h>
+
+static int memfd_create(const char *name, unsigned int flags)
+{
+#ifdef __NR_memfd_create
+    return syscall(__NR_memfd_create, name, flags);
+#else
+    return -1;
+#endif
+}
+#endif
+
+#ifndef MFD_CLOEXEC
+#define MFD_CLOEXEC 0x0001U
+#endif
+
+#ifndef MFD_ALLOW_SEALING
+#define MFD_ALLOW_SEALING 0x0002U
+#endif
+
+/*
+ * This is a best-effort helper for shared memory allocation, with
+ * optional sealing. The helper will do his best to allocate using
+ * memfd with sealing, but may fallback on other methods without
+ * sealing.
+ */
+void *qemu_memfd_alloc(const char *name, size_t size, unsigned int seals,
+                       int *fd)
+{
+    void *ptr;
+    int mfd = -1;
+
+    *fd = -1;
+
+#ifdef CONFIG_LINUX
+    if (seals) {
+        mfd = memfd_create(name, MFD_ALLOW_SEALING | MFD_CLOEXEC);
+    }
+
+    if (mfd == -1) {
+        /* some systems have memfd without sealing */
+        mfd = memfd_create(name, MFD_CLOEXEC);
+        seals = 0;
+    }
+#endif
+
+    if (mfd != -1) {
+        if (ftruncate(mfd, size) == -1) {
+            perror("ftruncate");
+            close(mfd);
+            return NULL;
+        }
+
+        if (seals && fcntl(mfd, F_ADD_SEALS, seals) == -1) {
+            perror("fcntl");
+            close(mfd);
+            return NULL;
+        }
+    } else {
+        const char *tmpdir = g_get_tmp_dir();
+        gchar *fname;
+
+        fname = g_strdup_printf("%s/memfd-XXXXXX", tmpdir);
+        mfd = mkstemp(fname);
+        unlink(fname);
+        g_free(fname);
+
+        if (mfd == -1) {
+            perror("mkstemp");
+            return NULL;
+        }
+
+        if (ftruncate(mfd, size) == -1) {
+            perror("ftruncate");
+            close(mfd);
+            return NULL;
+        }
+    }
+
+    ptr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, mfd, 0);
+    if (ptr == MAP_FAILED) {
+        perror("mmap");
+        close(mfd);
+        return NULL;
+    }
+
+    *fd = mfd;
+    return ptr;
+}
+
+void qemu_memfd_free(void *ptr, size_t size, int fd)
+{
+    if (ptr) {
+        munmap(ptr, size);
+    }
+
+    if (fd != -1) {
+        close(fd);
+    }
+}
+
+enum {
+    MEMFD_KO,
+    MEMFD_OK,
+    MEMFD_TODO
+};
+
+bool qemu_memfd_check(void)
+{
+    static int memfd_check = MEMFD_TODO;
+
+    if (memfd_check == MEMFD_TODO) {
+        int fd;
+        void *ptr;
+
+        ptr = qemu_memfd_alloc("test", 4096, 0, &fd);
+        memfd_check = ptr ? MEMFD_OK : MEMFD_KO;
+        qemu_memfd_free(ptr, 4096, fd);
+    }
+
+    return memfd_check == MEMFD_OK;
+}
diff --git a/util/mmap-alloc.c b/util/mmap-alloc.c
new file mode 100644
index 0000000..5a85aa3
--- /dev/null
+++ b/util/mmap-alloc.c
@@ -0,0 +1,110 @@
+/*
+ * Support for RAM backed by mmaped host memory.
+ *
+ * Copyright (c) 2015 Red Hat, Inc.
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/mmap-alloc.h"
+
+#define HUGETLBFS_MAGIC       0x958458f6
+
+#ifdef CONFIG_LINUX
+#include <sys/vfs.h>
+#endif
+
+size_t qemu_fd_getpagesize(int fd)
+{
+#ifdef CONFIG_LINUX
+    struct statfs fs;
+    int ret;
+
+    if (fd != -1) {
+        do {
+            ret = fstatfs(fd, &fs);
+        } while (ret != 0 && errno == EINTR);
+
+        if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
+            return fs.f_bsize;
+        }
+    }
+#endif
+
+    return getpagesize();
+}
+
+void *qemu_ram_mmap(int fd, size_t size, size_t align, bool shared)
+{
+    /*
+     * Note: this always allocates at least one extra page of virtual address
+     * space, even if size is already aligned.
+     */
+    size_t total = size + align;
+#if defined(__powerpc64__) && defined(__linux__)
+    /* On ppc64 mappings in the same segment (aka slice) must share the same
+     * page size. Since we will be re-allocating part of this segment
+     * from the supplied fd, we should make sure to use the same page size, to
+     * this end we mmap the supplied fd.  In this case, set MAP_NORESERVE to
+     * avoid allocating backing store memory.
+     * We do this unless we are using the system page size, in which case
+     * anonymous memory is OK.
+     */
+    int anonfd = fd == -1 || qemu_fd_getpagesize(fd) == getpagesize() ? -1 : fd;
+    int flags = anonfd == -1 ? MAP_ANONYMOUS : MAP_NORESERVE;
+    void *ptr = mmap(0, total, PROT_NONE, flags | MAP_PRIVATE, anonfd, 0);
+#else
+    void *ptr = mmap(0, total, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+#endif
+    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+    void *ptr1;
+
+    if (ptr == MAP_FAILED) {
+        return MAP_FAILED;
+    }
+
+    /* Make sure align is a power of 2 */
+    assert(!(align & (align - 1)));
+    /* Always align to host page size */
+    assert(align >= getpagesize());
+
+    ptr1 = mmap(ptr + offset, size, PROT_READ | PROT_WRITE,
+                MAP_FIXED |
+                (fd == -1 ? MAP_ANONYMOUS : 0) |
+                (shared ? MAP_SHARED : MAP_PRIVATE),
+                fd, 0);
+    if (ptr1 == MAP_FAILED) {
+        munmap(ptr, total);
+        return MAP_FAILED;
+    }
+
+    ptr += offset;
+    total -= offset;
+
+    if (offset > 0) {
+        munmap(ptr - offset, offset);
+    }
+
+    /*
+     * Leave a single PROT_NONE page allocated after the RAM block, to serve as
+     * a guard page guarding against potential buffer overflows.
+     */
+    if (total > size + getpagesize()) {
+        munmap(ptr + size + getpagesize(), total - size - getpagesize());
+    }
+
+    return ptr;
+}
+
+void qemu_ram_munmap(void *ptr, size_t size)
+{
+    if (ptr) {
+        /* Unmap both the RAM block and the guard page */
+        munmap(ptr, size + getpagesize());
+    }
+}
diff --git a/util/module.c b/util/module.c
index 4bd4a94..86e3f7a 100644
--- a/util/module.c
+++ b/util/module.c
@@ -13,7 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <stdlib.h>
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #ifdef CONFIG_MODULES
 #include <gmodule.h>
@@ -55,13 +55,9 @@
 
 static ModuleTypeList *find_type(module_init_type type)
 {
-    ModuleTypeList *l;
-
     init_lists();
 
-    l = &init_type_list[type];
-
-    return l;
+    return &init_type_list[type];
 }
 
 void register_module_init(void (*fn)(void), module_init_type type)
diff --git a/util/notify.c b/util/notify.c
index f215dfc..06de63a 100644
--- a/util/notify.c
+++ b/util/notify.c
@@ -13,6 +13,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/notify.h"
 
diff --git a/util/osdep.c b/util/osdep.c
index bf9c246..06fb1cf 100644
--- a/util/osdep.c
+++ b/util/osdep.c
@@ -21,24 +21,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <string.h>
-#include <errno.h>
-#include <unistd.h>
-#include <fcntl.h>
+#include "qemu/osdep.h"
 
 /* Needed early for CONFIG_BSD etc. */
-#include "config-host.h"
-
-#if defined(CONFIG_MADVISE) || defined(CONFIG_POSIX_MADVISE)
-#include <sys/mman.h>
-#endif
 
 #ifdef CONFIG_SOLARIS
-#include <sys/types.h>
 #include <sys/statvfs.h>
 /* See MySQL bug #7156 (http://bugs.mysql.com/bug.php?id=7156) for
    discussion about Solaris header problems */
@@ -46,14 +33,15 @@
 #endif
 
 #include "qemu-common.h"
+#include "qemu/cutils.h"
 #include "qemu/sockets.h"
+#include "qemu/error-report.h"
 #include "monitor/monitor.h"
 
 static bool fips_enabled = false;
 
-static const char *qemu_version = QEMU_VERSION;
+static const char *hw_version = QEMU_HW_VERSION;
 
-#if !defined(CONFIG_ANDROID)
 int socket_set_cork(int fd, int v)
 {
 #if defined(SOL_TCP) && defined(TCP_CORK)
@@ -68,7 +56,6 @@
     int v = 1;
     return qemu_setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &v, sizeof(v));
 }
-#endif
 
 int qemu_madvise(void *addr, size_t len, int advice)
 {
@@ -96,14 +83,7 @@
     int serrno;
     int dup_flags;
 
-#ifdef F_DUPFD_CLOEXEC
-    ret = fcntl(fd, F_DUPFD_CLOEXEC, 0);
-#else
-    ret = dup(fd);
-    if (ret != -1) {
-        qemu_set_cloexec(ret);
-    }
-#endif
+    ret = qemu_dup(fd);
     if (ret == -1) {
         goto fail;
     }
@@ -142,6 +122,20 @@
     return -1;
 }
 
+int qemu_dup(int fd)
+{
+    int ret;
+#ifdef F_DUPFD_CLOEXEC
+    ret = fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    ret = dup(fd);
+    if (ret != -1) {
+        qemu_set_cloexec(ret);
+    }
+#endif
+    return ret;
+}
+
 static int qemu_parse_fdset(const char *param)
 {
     return qemu_parse_fd(param);
@@ -312,80 +306,14 @@
     return ret;
 }
 
-/*
- * A variant of send(2) which handles partial write.
- *
- * Return the number of bytes transferred, which is only
- * smaller than `count' if there is an error.
- *
- * This function won't work with non-blocking fd's.
- * Any of the possibilities with non-bloking fd's is bad:
- *   - return a short write (then name is wrong)
- *   - busy wait adding (errno == EAGAIN) to the loop
- */
-ssize_t qemu_send_full(int fd, const void *buf, size_t count, int flags)
+void qemu_set_hw_version(const char *version)
 {
-    ssize_t ret = 0;
-    ssize_t total = 0;
-
-    while (count) {
-        ret = send(fd, buf, count, flags);
-        if (ret < 0) {
-            if (errno == EINTR) {
-                continue;
-            }
-            break;
-        }
-
-        count -= ret;
-        buf += ret;
-        total += ret;
-    }
-
-    return total;
+    hw_version = version;
 }
 
-/*
- * A variant of recv(2) which handles partial write.
- *
- * Return the number of bytes transferred, which is only
- * smaller than `count' if there is an error.
- *
- * This function won't work with non-blocking fd's.
- * Any of the possibilities with non-bloking fd's is bad:
- *   - return a short write (then name is wrong)
- *   - busy wait adding (errno == EAGAIN) to the loop
- */
-ssize_t qemu_recv_full(int fd, void *buf, size_t count, int flags)
+const char *qemu_hw_version(void)
 {
-    ssize_t ret = 0;
-    ssize_t total = 0;
-
-    while (count) {
-        ret = qemu_recv(fd, buf, count, flags);
-        if (ret <= 0) {
-            if (ret < 0 && errno == EINTR) {
-                continue;
-            }
-            break;
-        }
-
-        count -= ret;
-        buf += ret;
-        total += ret;
-    }
-
-    return total;
-}
-
-void qemu_set_version(const char *version)
-{
-    qemu_version = version;
-}
-
-const char *qemu_get_version(void)
-{
-    return qemu_version;
+    return hw_version;
 }
 
 void fips_set_state(bool requested)
diff --git a/util/oslib-posix.c b/util/oslib-posix.c
index 16fcec2..79efcee 100644
--- a/util/oslib-posix.c
+++ b/util/oslib-posix.c
@@ -26,51 +26,30 @@
  * THE SOFTWARE.
  */
 
-/* The following block of code temporarily renames the daemon() function so the
-   compiler does not see the warning associated with it in stdlib.h on OSX */
-#ifdef __APPLE__
-#define daemon qemu_fake_daemon_function
-#include <stdlib.h>
-#undef daemon
-extern int daemon(int, int);
-#endif
-
-#if defined(__linux__) && (defined(__x86_64__) || defined(__arm__))
-   /* Use 2 MiB alignment so transparent hugepages can be used by KVM.
-      Valgrind does not support alignments larger than 1 MiB,
-      therefore we need special code which handles running on Valgrind. */
-#  define QEMU_VMALLOC_ALIGN (512 * 4096)
-#elif defined(__linux__) && defined(__s390x__)
-   /* Use 1 MiB (segment size) alignment so gmap can be used by KVM. */
-#  define QEMU_VMALLOC_ALIGN (256 * 4096)
-#else
-#  define QEMU_VMALLOC_ALIGN getpagesize()
-#endif
-#define HUGETLBFS_MAGIC       0x958458f6
-
+#include "qemu/osdep.h"
 #include <termios.h>
-#include <unistd.h>
+#include <termios.h>
 
 #include <glib/gprintf.h>
 
-#include "config-host.h"
 #include "sysemu/sysemu.h"
 #include "trace.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h"
-#include <sys/mman.h>
 #include <libgen.h>
-#include <setjmp.h>
 #include <sys/signal.h>
+#include "qemu/cutils.h"
 
 #ifdef CONFIG_LINUX
 #include <sys/syscall.h>
-#include <sys/vfs.h>
 #endif
 
 #ifdef __FreeBSD__
 #include <sys/sysctl.h>
 #endif
 
+#include "qemu/mmap-alloc.h"
+
 int qemu_get_thread_id(void)
 {
 #if defined(__linux__)
@@ -127,10 +106,7 @@
 void *qemu_anon_ram_alloc(size_t size, uint64_t *alignment)
 {
     size_t align = QEMU_VMALLOC_ALIGN;
-    size_t total = size + align - getpagesize();
-    void *ptr = mmap(0, total, PROT_READ | PROT_WRITE,
-                     MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
-    size_t offset = QEMU_ALIGN_UP((uintptr_t)ptr, align) - (uintptr_t)ptr;
+    void *ptr = qemu_ram_mmap(-1, size, align, false);
 
     if (ptr == MAP_FAILED) {
         return NULL;
@@ -139,15 +115,6 @@
     if (alignment) {
         *alignment = align;
     }
-    ptr += offset;
-    total -= offset;
-
-    if (offset > 0) {
-        munmap(ptr - offset, offset);
-    }
-    if (total > size) {
-        munmap(ptr + size, total - size);
-    }
 
     trace_qemu_anon_ram_alloc(size, ptr);
     return ptr;
@@ -162,9 +129,7 @@
 void qemu_anon_ram_free(void *ptr, size_t size)
 {
     trace_qemu_anon_ram_free(ptr, size);
-    if (ptr) {
-        munmap(ptr, size);
-    }
+    qemu_ram_munmap(ptr, size);
 }
 
 void qemu_set_block(int fd)
@@ -250,7 +215,7 @@
         gettimeofday(&tv_now, NULL);
     }
     if (times[0].tv_nsec == UTIME_OMIT || times[1].tv_nsec == UTIME_OMIT) {
-        stat(path, &st);
+        qemu_stat(path, &st);
     }
 
     for (i = 0; i < 2; i++) {
@@ -334,9 +299,11 @@
             return;
         }
     }
-    dir = dirname(p);
+    dir = g_path_get_dirname(p);
 
     pstrcpy(exec_dir, sizeof(exec_dir), dir);
+
+    g_free(dir);
 }
 
 char *qemu_get_exec_dir(void)
@@ -351,27 +318,7 @@
     siglongjmp(sigjump, 1);
 }
 
-static size_t fd_getpagesize(int fd)
-{
-#ifdef CONFIG_LINUX
-    struct statfs fs;
-    int ret;
-
-    if (fd != -1) {
-        do {
-            ret = fstatfs(fd, &fs);
-        } while (ret != 0 && errno == EINTR);
-
-        if (ret == 0 && fs.f_type == HUGETLBFS_MAGIC) {
-            return fs.f_bsize;
-        }
-    }
-#endif
-
-    return getpagesize();
-}
-
-void os_mem_prealloc(int fd, char *area, size_t memory)
+void os_mem_prealloc(int fd, char *area, size_t memory, Error **errp)
 {
     int ret;
     struct sigaction act, oldact;
@@ -383,8 +330,9 @@
 
     ret = sigaction(SIGBUS, &act, &oldact);
     if (ret) {
-        perror("os_mem_prealloc: failed to install signal handler");
-        exit(1);
+        error_setg_errno(errp, errno,
+            "os_mem_prealloc: failed to install signal handler");
+        return;
     }
 
     /* unblock SIGBUS */
@@ -393,25 +341,161 @@
     pthread_sigmask(SIG_UNBLOCK, &set, &oldset);
 
     if (sigsetjmp(sigjump, 1)) {
-        fprintf(stderr, "os_mem_prealloc: Insufficient free host memory "
-                        "pages available to allocate guest RAM\n");
-        exit(1);
+        error_setg(errp, "os_mem_prealloc: Insufficient free host memory "
+            "pages available to allocate guest RAM\n");
     } else {
         int i;
-        size_t hpagesize = fd_getpagesize(fd);
+        size_t hpagesize = qemu_fd_getpagesize(fd);
+        size_t numpages = DIV_ROUND_UP(memory, hpagesize);
 
         /* MAP_POPULATE silently ignores failures */
-        memory = (memory + hpagesize - 1) & -hpagesize;
-        for (i = 0; i < (memory / hpagesize); i++) {
+        for (i = 0; i < numpages; i++) {
             memset(area + (hpagesize * i), 0, 1);
         }
+    }
 
-        ret = sigaction(SIGBUS, &oldact, NULL);
-        if (ret) {
-            perror("os_mem_prealloc: failed to reinstall signal handler");
-            exit(1);
+    ret = sigaction(SIGBUS, &oldact, NULL);
+    if (ret) {
+        /* Terminate QEMU since it can't recover from error */
+        perror("os_mem_prealloc: failed to reinstall signal handler");
+        exit(1);
+    }
+    pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+}
+
+
+static struct termios oldtty;
+
+static void term_exit(void)
+{
+    tcsetattr(0, TCSANOW, &oldtty);
+}
+
+static void term_init(void)
+{
+    struct termios tty;
+
+    tcgetattr(0, &tty);
+    oldtty = tty;
+
+    tty.c_iflag &= ~(IGNBRK|BRKINT|PARMRK|ISTRIP
+                          |INLCR|IGNCR|ICRNL|IXON);
+    tty.c_oflag |= OPOST;
+    tty.c_lflag &= ~(ECHO|ECHONL|ICANON|IEXTEN);
+    tty.c_cflag &= ~(CSIZE|PARENB);
+    tty.c_cflag |= CS8;
+    tty.c_cc[VMIN] = 1;
+    tty.c_cc[VTIME] = 0;
+
+    tcsetattr(0, TCSANOW, &tty);
+
+    atexit(term_exit);
+}
+
+int qemu_read_password(char *buf, int buf_size)
+{
+    uint8_t ch;
+    int i, ret;
+
+    printf("password: ");
+    fflush(stdout);
+    term_init();
+    i = 0;
+    for (;;) {
+        ret = read(0, &ch, 1);
+        if (ret == -1) {
+            if (errno == EAGAIN || errno == EINTR) {
+                continue;
+            } else {
+                break;
+            }
+        } else if (ret == 0) {
+            ret = -1;
+            break;
+        } else {
+            if (ch == '\r' ||
+                ch == '\n') {
+                ret = 0;
+                break;
+            }
+            if (i < (buf_size - 1)) {
+                buf[i++] = ch;
+            }
+        }
+    }
+    term_exit();
+    buf[i] = '\0';
+    printf("\n");
+    return ret;
+}
+
+
+pid_t qemu_fork(Error **errp)
+{
+    sigset_t oldmask, newmask;
+    struct sigaction sig_action;
+    int saved_errno;
+    pid_t pid;
+
+    /*
+     * Need to block signals now, so that child process can safely
+     * kill off caller's signal handlers without a race.
+     */
+    sigfillset(&newmask);
+    if (pthread_sigmask(SIG_SETMASK, &newmask, &oldmask) != 0) {
+        error_setg_errno(errp, errno,
+                         "cannot block signals");
+        return -1;
+    }
+
+    pid = fork();
+    saved_errno = errno;
+
+    if (pid < 0) {
+        /* attempt to restore signal mask, but ignore failure, to
+         * avoid obscuring the fork failure */
+        (void)pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+        error_setg_errno(errp, saved_errno,
+                         "cannot fork child process");
+        errno = saved_errno;
+        return -1;
+    } else if (pid) {
+        /* parent process */
+
+        /* Restore our original signal mask now that the child is
+         * safely running. Only documented failures are EFAULT (not
+         * possible, since we are using just-grabbed mask) or EINVAL
+         * (not possible, since we are using correct arguments).  */
+        (void)pthread_sigmask(SIG_SETMASK, &oldmask, NULL);
+    } else {
+        /* child process */
+        size_t i;
+
+        /* Clear out all signal handlers from parent so nothing
+         * unexpected can happen in our child once we unblock
+         * signals */
+        sig_action.sa_handler = SIG_DFL;
+        sig_action.sa_flags = 0;
+        sigemptyset(&sig_action.sa_mask);
+
+        for (i = 1; i < NSIG; i++) {
+            /* Only possible errors are EFAULT or EINVAL The former
+             * won't happen, the latter we expect, so no need to check
+             * return value */
+            (void)sigaction(i, &sig_action, NULL);
         }
 
-        pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+        /* Unmask all signals in child, since we've no idea what the
+         * caller's done with their signal mask and don't want to
+         * propagate that to children */
+        sigemptyset(&newmask);
+        if (pthread_sigmask(SIG_SETMASK, &newmask, NULL) != 0) {
+            Error *local_err = NULL;
+            error_setg_errno(&local_err, errno,
+                             "cannot unblock signals");
+            error_report_err(local_err);
+            _exit(1);
+        }
     }
+    return pid;
 }
diff --git a/util/oslib-win32.c b/util/oslib-win32.c
index 28a623e..6c104c4 100644
--- a/util/oslib-win32.c
+++ b/util/oslib-win32.c
@@ -2,7 +2,7 @@
  * os-win32.c
  *
  * Copyright (c) 2003-2008 Fabrice Bellard
- * Copyright (c) 2010 Red Hat, Inc.
+ * Copyright (c) 2010-2016 Red Hat, Inc.
  *
  * QEMU library functions for win32 which are shared between QEMU and
  * the QEMU tools.
@@ -29,18 +29,14 @@
  * this file are based on code from GNOME glib-2 and use a different license,
  * see the license comment there.
  */
+#include "qemu/osdep.h"
 #include <windows.h>
-#include <glib.h>
-#include <stdlib.h>
-#include "config-host.h"
+#include "qapi/error.h"
 #include "sysemu/sysemu.h"
 #include "qemu/main-loop.h"
 #include "trace.h"
 #include "qemu/sockets.h"
-
-#ifdef CONFIG_ANDROID
-#include "android/utils/win32_unicode.h"
-#endif  // CONFIG_ANDROID
+#include "qemu/cutils.h"
 
 /* this must come after including "trace.h" */
 #include <shlobj.h>
@@ -99,6 +95,7 @@
     }
 }
 
+#ifndef CONFIG_LOCALTIME_R
 /* FIXME: add proper locking */
 struct tm *gmtime_r(const time_t *timep, struct tm *result)
 {
@@ -122,6 +119,7 @@
     }
     return p;
 }
+#endif /* CONFIG_LOCALTIME_R */
 
 void qemu_set_block(int fd)
 {
@@ -147,6 +145,83 @@
     return 0;
 }
 
+
+static int socket_error(void)
+{
+    switch (WSAGetLastError()) {
+    case 0:
+        return 0;
+    case WSAEINTR:
+        return EINTR;
+    case WSAEINVAL:
+        return EINVAL;
+    case WSA_INVALID_HANDLE:
+        return EBADF;
+    case WSA_NOT_ENOUGH_MEMORY:
+        return ENOMEM;
+    case WSA_INVALID_PARAMETER:
+        return EINVAL;
+    case WSAENAMETOOLONG:
+        return ENAMETOOLONG;
+    case WSAENOTEMPTY:
+        return ENOTEMPTY;
+    case WSAEWOULDBLOCK:
+         /* not using EWOULDBLOCK as we don't want code to have
+          * to check both EWOULDBLOCK and EAGAIN */
+        return EAGAIN;
+    case WSAEINPROGRESS:
+        return EINPROGRESS;
+    case WSAEALREADY:
+        return EALREADY;
+    case WSAENOTSOCK:
+        return ENOTSOCK;
+    case WSAEDESTADDRREQ:
+        return EDESTADDRREQ;
+    case WSAEMSGSIZE:
+        return EMSGSIZE;
+    case WSAEPROTOTYPE:
+        return EPROTOTYPE;
+    case WSAENOPROTOOPT:
+        return ENOPROTOOPT;
+    case WSAEPROTONOSUPPORT:
+        return EPROTONOSUPPORT;
+    case WSAEOPNOTSUPP:
+        return EOPNOTSUPP;
+    case WSAEAFNOSUPPORT:
+        return EAFNOSUPPORT;
+    case WSAEADDRINUSE:
+        return EADDRINUSE;
+    case WSAEADDRNOTAVAIL:
+        return EADDRNOTAVAIL;
+    case WSAENETDOWN:
+        return ENETDOWN;
+    case WSAENETUNREACH:
+        return ENETUNREACH;
+    case WSAENETRESET:
+        return ENETRESET;
+    case WSAECONNABORTED:
+        return ECONNABORTED;
+    case WSAECONNRESET:
+        return ECONNRESET;
+    case WSAENOBUFS:
+        return ENOBUFS;
+    case WSAEISCONN:
+        return EISCONN;
+    case WSAENOTCONN:
+        return ENOTCONN;
+    case WSAETIMEDOUT:
+        return ETIMEDOUT;
+    case WSAECONNREFUSED:
+        return ECONNREFUSED;
+    case WSAELOOP:
+        return ELOOP;
+    case WSAEHOSTUNREACH:
+        return EHOSTUNREACH;
+    default:
+        return EIO;
+    }
+}
+
 int inet_aton(const char *cp, struct in_addr *ia)
 {
     uint32_t addr = inet_addr(cp);
@@ -231,23 +306,10 @@
     char buf[MAX_PATH];
     DWORD len;
 
-#ifdef CONFIG_ANDROID
-    wchar_t wideBuf[MAX_PATH];
-    len = GetModuleFileNameW(NULL, wideBuf, MAX_PATH);
-    if (len == 0 || len >= MAX_PATH) {
-        return;
-    }
-    int written = win32_utf16_to_utf8_buf(wideBuf, buf, sizeof(buf));
-    if (written < 0) {
-        return;
-    }
-    len = written;
-#else
-    len = GetModuleFileName(NULL, buf, sizeof(buf) - 1);
+    len = win32GetModuleFileName(NULL, buf, sizeof(buf) - 1);
     if (len == 0) {
         return;
     }
-#endif
 
     buf[len] = 0;
     p = buf + len - 1;
@@ -469,7 +531,7 @@
     return retval;
 }
 
-size_t getpagesize(void)
+int getpagesize(void)
 {
     SYSTEM_INFO system_info;
 
@@ -477,7 +539,7 @@
     return system_info.dwPageSize;
 }
 
-void os_mem_prealloc(int fd, char *area, size_t memory)
+void os_mem_prealloc(int fd, char *area, size_t memory, Error **errp)
 {
     int i;
     size_t pagesize = getpagesize();
@@ -488,29 +550,246 @@
     }
 }
 
-#ifdef CONFIG_ANDROID
-// Required by the debug Android build.
-int ffs(int x) {
-    int result = 0;
-    unsigned ux = (unsigned)x;
-    while (ux) {
-        if (ux & 1)
-            return result + 1;
-        ux >>= 1;
-        ++result;
+
+/* XXX: put correct support for win32 */
+int qemu_read_password(char *buf, int buf_size)
+{
+    int c, i;
+
+    printf("Password: ");
+    fflush(stdout);
+    i = 0;
+    for (;;) {
+        c = getchar();
+        if (c < 0) {
+            buf[i] = '\0';
+            return -1;
+        } else if (c == '\n') {
+            break;
+        } else if (i < (buf_size - 1)) {
+            buf[i++] = c;
+        }
     }
+    buf[i] = '\0';
     return 0;
 }
 
-int ffsl(long x) {
-    int result = 0;
-    unsigned long ux = (unsigned long)x;
-    while (ux) {
-        if (ux & 1)
-            return result + 1;
-        ux >>= 1;
-        ++result;
-    }
-    return 0;
+
+pid_t qemu_fork(Error **errp)
+{
+    errno = ENOSYS;
+    error_setg_errno(errp, errno,
+                     "cannot fork child process");
+    return -1;
 }
-#endif
+
+
+#undef connect
+int qemu_connect_wrap(int sockfd, const struct sockaddr *addr,
+                      socklen_t addrlen)
+{
+    int ret;
+    ret = connect(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef listen
+int qemu_listen_wrap(int sockfd, int backlog)
+{
+    int ret;
+    ret = listen(sockfd, backlog);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef bind
+int qemu_bind_wrap(int sockfd, const struct sockaddr *addr,
+                   socklen_t addrlen)
+{
+    int ret;
+    ret = bind(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef socket
+int qemu_socket_wrap(int domain, int type, int protocol)
+{
+    int ret;
+    ret = socket(domain, type, protocol);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef accept
+int qemu_accept_wrap(int sockfd, struct sockaddr *addr,
+                     socklen_t *addrlen)
+{
+    int ret;
+    ret = accept(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef shutdown
+int qemu_shutdown_wrap(int sockfd, int how)
+{
+    int ret;
+    ret = shutdown(sockfd, how);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef ioctlsocket
+int qemu_ioctlsocket_wrap(int fd, int req, void *val)
+{
+    int ret;
+    ret = ioctlsocket(fd, req, val);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef closesocket
+int qemu_closesocket_wrap(int fd)
+{
+    int ret;
+    ret = closesocket(fd);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef getsockopt
+int qemu_getsockopt_wrap(int sockfd, int level, int optname,
+                         void *optval, socklen_t *optlen)
+{
+    int ret;
+    ret = getsockopt(sockfd, level, optname, optval, optlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef setsockopt
+int qemu_setsockopt_wrap(int sockfd, int level, int optname,
+                         const void *optval, socklen_t optlen)
+{
+    int ret;
+    ret = setsockopt(sockfd, level, optname, optval, optlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef getpeername
+int qemu_getpeername_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen)
+{
+    int ret;
+    ret = getpeername(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef getsockname
+int qemu_getsockname_wrap(int sockfd, struct sockaddr *addr,
+                          socklen_t *addrlen)
+{
+    int ret;
+    ret = getsockname(sockfd, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef send
+ssize_t qemu_send_wrap(int sockfd, const void *buf, size_t len, int flags)
+{
+    int ret;
+    ret = send(sockfd, buf, len, flags);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef sendto
+ssize_t qemu_sendto_wrap(int sockfd, const void *buf, size_t len, int flags,
+                         const struct sockaddr *addr, socklen_t addrlen)
+{
+    int ret;
+    ret = sendto(sockfd, buf, len, flags, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef recv
+ssize_t qemu_recv_wrap(int sockfd, void *buf, size_t len, int flags)
+{
+    int ret;
+    ret = recv(sockfd, buf, len, flags);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+
+#undef recvfrom
+ssize_t qemu_recvfrom_wrap(int sockfd, void *buf, size_t len, int flags,
+                           struct sockaddr *addr, socklen_t *addrlen)
+{
+    int ret;
+    ret = recvfrom(sockfd, buf, len, flags, addr, addrlen);
+    if (ret < 0) {
+        errno = socket_error();
+    }
+    return ret;
+}
+
+int qemu_stat(const char* path, struct stat* st)
+{
+    return stat(path, st);
+}
+
+int qemu_lstat(const char* path, struct stat* st)
+{
+    return qemu_stat(path, st);
+}
diff --git a/util/path.c b/util/path.c
index 4e4877e..5479f76 100644
--- a/util/path.c
+++ b/util/path.c
@@ -3,15 +3,12 @@
 
    The assumption is that this area does not change.
 */
-#include <sys/types.h>
+#include "qemu/osdep.h"
 #include <sys/param.h>
 #include <dirent.h>
-#include <unistd.h>
-#include <stdlib.h>
-#include <string.h>
-#include <errno.h>
-#include <stdio.h>
 #include "qemu-common.h"
+#include "qemu/cutils.h"
+#include "qemu/path.h"
 
 struct pathelem
 {
diff --git a/util/qdist.c b/util/qdist.c
new file mode 100644
index 0000000..5f75e24
--- /dev/null
+++ b/util/qdist.c
@@ -0,0 +1,397 @@
+/*
+ * qdist.c - QEMU helpers for handling frequency distributions of data.
+ *
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ */
+#include "qemu/osdep.h"
+#include "qemu/qdist.h"
+
+#include <math.h>
+#ifndef NAN
+#define NAN (0.0 / 0.0)
+#endif
+
+#define QDIST_EMPTY_STR "(empty)"
+
+void qdist_init(struct qdist *dist)
+{
+    dist->entries = g_new(struct qdist_entry, 1);
+    dist->size = 1;
+    dist->n = 0;
+}
+
+void qdist_destroy(struct qdist *dist)
+{
+    g_free(dist->entries);
+}
+
+static inline int qdist_cmp_double(double a, double b)
+{
+    if (a > b) {
+        return 1;
+    } else if (a < b) {
+        return -1;
+    }
+    return 0;
+}
+
+static int qdist_cmp(const void *ap, const void *bp)
+{
+    const struct qdist_entry *a = ap;
+    const struct qdist_entry *b = bp;
+
+    return qdist_cmp_double(a->x, b->x);
+}
+
+void qdist_add(struct qdist *dist, double x, long count)
+{
+    struct qdist_entry *entry = NULL;
+
+    if (dist->n) {
+        struct qdist_entry e;
+
+        e.x = x;
+        entry = bsearch(&e, dist->entries, dist->n, sizeof(e), qdist_cmp);
+    }
+
+    if (entry) {
+        entry->count += count;
+        return;
+    }
+
+    if (unlikely(dist->n == dist->size)) {
+        dist->size *= 2;
+        dist->entries = g_renew(struct qdist_entry, dist->entries, dist->size);
+    }
+    dist->n++;
+    entry = &dist->entries[dist->n - 1];
+    entry->x = x;
+    entry->count = count;
+    qsort(dist->entries, dist->n, sizeof(*entry), qdist_cmp);
+}
+
+void qdist_inc(struct qdist *dist, double x)
+{
+    qdist_add(dist, x, 1);
+}
+
+/*
+ * Unicode for block elements. See:
+ *   https://en.wikipedia.org/wiki/Block_Elements
+ */
+static const gunichar qdist_blocks[] = {
+    0x2581,
+    0x2582,
+    0x2583,
+    0x2584,
+    0x2585,
+    0x2586,
+    0x2587,
+    0x2588
+};
+
+#define QDIST_NR_BLOCK_CODES ARRAY_SIZE(qdist_blocks)
+
+/*
+ * Print a distribution into a string.
+ *
+ * This function assumes that appropriate binning has been done on the input;
+ * see qdist_bin__internal() and qdist_pr_plain().
+ *
+ * Callers must free the returned string with g_free().
+ */
+static char *qdist_pr_internal(const struct qdist *dist)
+{
+    double min, max;
+    GString *s = g_string_new("");
+    size_t i;
+
+    /* if only one entry, its printout will be either full or empty */
+    if (dist->n == 1) {
+        if (dist->entries[0].count) {
+            g_string_append_unichar(s, qdist_blocks[QDIST_NR_BLOCK_CODES - 1]);
+        } else {
+            g_string_append_c(s, ' ');
+        }
+        goto out;
+    }
+
+    /* get min and max counts */
+    min = dist->entries[0].count;
+    max = min;
+    for (i = 0; i < dist->n; i++) {
+        struct qdist_entry *e = &dist->entries[i];
+
+        if (e->count < min) {
+            min = e->count;
+        }
+        if (e->count > max) {
+            max = e->count;
+        }
+    }
+
+    for (i = 0; i < dist->n; i++) {
+        struct qdist_entry *e = &dist->entries[i];
+        int index;
+
+        /* make an exception with 0; instead of using block[0], print a space */
+        if (e->count) {
+            /* divide first to avoid loss of precision when e->count == max */
+            index = (e->count - min) / (max - min) * (QDIST_NR_BLOCK_CODES - 1);
+            g_string_append_unichar(s, qdist_blocks[index]);
+        } else {
+            g_string_append_c(s, ' ');
+        }
+    }
+ out:
+    return g_string_free(s, FALSE);
+}
+
+/*
+ * Bin the distribution in @from into @n bins of consecutive, non-overlapping
+ * intervals, copying the result to @to.
+ *
+ * This function is internal to qdist: only this file and test code should
+ * ever call it.
+ *
+ * Note: calling this function on an already-binned qdist is a bug.
+ *
+ * If @n == 0 or @from->n == 1, use @from->n.
+ */
+void qdist_bin__internal(struct qdist *to, const struct qdist *from, size_t n)
+{
+    double xmin, xmax;
+    double step;
+    size_t i, j;
+
+    qdist_init(to);
+
+    if (from->n == 0) {
+        return;
+    }
+    if (n == 0 || from->n == 1) {
+        n = from->n;
+    }
+
+    /* set equally-sized bins between @from's left and right */
+    xmin = qdist_xmin(from);
+    xmax = qdist_xmax(from);
+    step = (xmax - xmin) / n;
+
+    if (n == from->n) {
+        /* if @from's entries are equally spaced, no need to re-bin */
+        for (i = 0; i < from->n; i++) {
+            if (from->entries[i].x != xmin + i * step) {
+                goto rebin;
+            }
+        }
+        /* they're equally spaced, so copy the dist and bail out */
+        to->entries = g_renew(struct qdist_entry, to->entries, n);
+        to->n = from->n;
+        memcpy(to->entries, from->entries, sizeof(*to->entries) * to->n);
+        return;
+    }
+
+ rebin:
+    j = 0;
+    for (i = 0; i < n; i++) {
+        double x;
+        double left, right;
+
+        left = xmin + i * step;
+        right = xmin + (i + 1) * step;
+
+        /* Add x, even if it might not get any counts later */
+        x = left;
+        qdist_add(to, x, 0);
+
+        /*
+         * To avoid double-counting we capture [left, right) ranges, except for
+         * the righmost bin, which captures a [left, right] range.
+         */
+        while (j < from->n && (from->entries[j].x < right || i == n - 1)) {
+            struct qdist_entry *o = &from->entries[j];
+
+            qdist_add(to, x, o->count);
+            j++;
+        }
+    }
+}
+
+/*
+ * Print @dist into a string, after re-binning it into @n bins of consecutive,
+ * non-overlapping intervals.
+ *
+ * If @n == 0, use @orig->n.
+ *
+ * Callers must free the returned string with g_free().
+ */
+char *qdist_pr_plain(const struct qdist *dist, size_t n)
+{
+    struct qdist binned;
+    char *ret;
+
+    if (dist->n == 0) {
+        return g_strdup(QDIST_EMPTY_STR);
+    }
+    qdist_bin__internal(&binned, dist, n);
+    ret = qdist_pr_internal(&binned);
+    qdist_destroy(&binned);
+    return ret;
+}
+
+static char *qdist_pr_label(const struct qdist *dist, size_t n_bins,
+                            uint32_t opt, bool is_left)
+{
+    const char *percent;
+    const char *lparen;
+    const char *rparen;
+    GString *s;
+    double x1, x2, step;
+    double x;
+    double n;
+    int dec;
+
+    s = g_string_new("");
+    if (!(opt & QDIST_PR_LABELS)) {
+        goto out;
+    }
+
+    dec = opt & QDIST_PR_NODECIMAL ? 0 : 1;
+    percent = opt & QDIST_PR_PERCENT ? "%" : "";
+
+    n = n_bins ? n_bins : dist->n;
+    x = is_left ? qdist_xmin(dist) : qdist_xmax(dist);
+    step = (qdist_xmax(dist) - qdist_xmin(dist)) / n;
+
+    if (opt & QDIST_PR_100X) {
+        x *= 100.0;
+        step *= 100.0;
+    }
+    if (opt & QDIST_PR_NOBINRANGE) {
+        lparen = rparen = "";
+        x1 = x;
+        x2 = x; /* unnecessary, but a dumb compiler might not figure it out */
+    } else {
+        lparen = "[";
+        rparen = is_left ? ")" : "]";
+        if (is_left) {
+            x1 = x;
+            x2 = x + step;
+        } else {
+            x1 = x - step;
+            x2 = x;
+        }
+    }
+    g_string_append_printf(s, "%s%.*f", lparen, dec, x1);
+    if (!(opt & QDIST_PR_NOBINRANGE)) {
+        g_string_append_printf(s, ",%.*f%s", dec, x2, rparen);
+    }
+    g_string_append(s, percent);
+ out:
+    return g_string_free(s, FALSE);
+}
+
+/*
+ * Print the distribution's histogram into a string.
+ *
+ * See also: qdist_pr_plain().
+ *
+ * Callers must free the returned string with g_free().
+ */
+char *qdist_pr(const struct qdist *dist, size_t n_bins, uint32_t opt)
+{
+    const char *border = opt & QDIST_PR_BORDER ? "|" : "";
+    char *llabel, *rlabel;
+    char *hgram;
+    GString *s;
+
+    if (dist->n == 0) {
+        return g_strdup(QDIST_EMPTY_STR);
+    }
+
+    s = g_string_new("");
+
+    llabel = qdist_pr_label(dist, n_bins, opt, true);
+    rlabel = qdist_pr_label(dist, n_bins, opt, false);
+    hgram = qdist_pr_plain(dist, n_bins);
+    g_string_append_printf(s, "%s%s%s%s%s",
+                           llabel, border, hgram, border, rlabel);
+    g_free(llabel);
+    g_free(rlabel);
+    g_free(hgram);
+
+    return g_string_free(s, FALSE);
+}
+
+static inline double qdist_x(const struct qdist *dist, int index)
+{
+    if (dist->n == 0) {
+        return NAN;
+    }
+    return dist->entries[index].x;
+}
+
+double qdist_xmin(const struct qdist *dist)
+{
+    return qdist_x(dist, 0);
+}
+
+double qdist_xmax(const struct qdist *dist)
+{
+    return qdist_x(dist, dist->n - 1);
+}
+
+size_t qdist_unique_entries(const struct qdist *dist)
+{
+    return dist->n;
+}
+
+unsigned long qdist_sample_count(const struct qdist *dist)
+{
+    unsigned long count = 0;
+    size_t i;
+
+    for (i = 0; i < dist->n; i++) {
+        struct qdist_entry *e = &dist->entries[i];
+
+        count += e->count;
+    }
+    return count;
+}
+
+static double qdist_pairwise_avg(const struct qdist *dist, size_t index,
+                                 size_t n, unsigned long count)
+{
+    /* amortize the recursion by using a base case > 2 */
+    if (n <= 8) {
+        size_t i;
+        double ret = 0;
+
+        for (i = 0; i < n; i++) {
+            struct qdist_entry *e = &dist->entries[index + i];
+
+            ret += e->x * e->count / count;
+        }
+        return ret;
+    } else {
+        size_t n2 = n / 2;
+
+        return qdist_pairwise_avg(dist, index, n2, count) +
+               qdist_pairwise_avg(dist, index + n2, n - n2, count);
+    }
+}
+
+double qdist_avg(const struct qdist *dist)
+{
+    unsigned long count;
+
+    count = qdist_sample_count(dist);
+    if (!count) {
+        return NAN;
+    }
+    return qdist_pairwise_avg(dist, 0, dist->n, count);
+}
diff --git a/util/qemu-config.c b/util/qemu-config.c
index ba375c0..fb97307 100644
--- a/util/qemu-config.c
+++ b/util/qemu-config.c
@@ -1,13 +1,11 @@
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
-#include "qapi/qmp/qerror.h"
-#include "hw/qdev.h"
-#include "qapi/error.h"
 #include "qmp-commands.h"
 
-static QemuOptsList *vm_config_groups[32];
+static QemuOptsList *vm_config_groups[48];
 static QemuOptsList *drive_config_groups[4];
 
 static QemuOptsList *find_list(QemuOptsList **lists, const char *group,
@@ -32,8 +30,7 @@
 
     ret = find_list(vm_config_groups, group, &local_err);
     if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
     }
 
     return ret;
@@ -149,6 +146,92 @@
     return head;
 }
 
+/* restore machine options that are now machine's properties */
+static QemuOptsList machine_opts = {
+    .merge_lists = true,
+    .head = QTAILQ_HEAD_INITIALIZER(machine_opts.head),
+    .desc = {
+        {
+            .name = "type",
+            .type = QEMU_OPT_STRING,
+            .help = "emulated machine"
+        },{
+            .name = "accel",
+            .type = QEMU_OPT_STRING,
+            .help = "accelerator list",
+        },{
+            .name = "kernel_irqchip",
+            .type = QEMU_OPT_BOOL,
+            .help = "use KVM in-kernel irqchip",
+        },{
+            .name = "kvm_shadow_mem",
+            .type = QEMU_OPT_SIZE,
+            .help = "KVM shadow MMU size",
+        },{
+            .name = "kernel",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel image file",
+        },{
+            .name = "initrd",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux initial ramdisk file",
+        },{
+            .name = "append",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel command line",
+        },{
+            .name = "dtb",
+            .type = QEMU_OPT_STRING,
+            .help = "Linux kernel device tree file",
+        },{
+            .name = "dumpdtb",
+            .type = QEMU_OPT_STRING,
+            .help = "Dump current dtb to a file and quit",
+        },{
+            .name = "phandle_start",
+            .type = QEMU_OPT_NUMBER,
+            .help = "The first phandle ID we may generate dynamically",
+        },{
+            .name = "dt_compatible",
+            .type = QEMU_OPT_STRING,
+            .help = "Overrides the \"compatible\" property of the dt root node",
+        },{
+            .name = "dump-guest-core",
+            .type = QEMU_OPT_BOOL,
+            .help = "Include guest memory in  a core dump",
+        },{
+            .name = "mem-merge",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable memory merge support",
+        },{
+            .name = "usb",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on/off to enable/disable usb",
+        },{
+            .name = "firmware",
+            .type = QEMU_OPT_STRING,
+            .help = "firmware image",
+        },{
+            .name = "iommu",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on/off to enable/disable Intel IOMMU (VT-d)",
+        },{
+            .name = "suppress-vmdesc",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on to disable self-describing migration",
+        },{
+            .name = "aes-key-wrap",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable AES key wrapping using the CPACF wrapping key",
+        },{
+            .name = "dea-key-wrap",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable DEA key wrapping using the CPACF wrapping key",
+        },
+        { /* End of list */ }
+    }
+};
+
 CommandLineOptionInfoList *qmp_query_command_line_options(bool has_option,
                                                           const char *option,
                                                           Error **errp)
@@ -163,6 +246,8 @@
             info->option = g_strdup(vm_config_groups[i]->name);
             if (!strcmp("drive", vm_config_groups[i]->name)) {
                 info->parameters = get_drive_infolist();
+            } else if (!strcmp("machine", vm_config_groups[i]->name)) {
+                info->parameters = query_option_descs(machine_opts.desc);
             } else {
                 info->parameters =
                     query_option_descs(vm_config_groups[i]->desc);
@@ -220,6 +305,7 @@
 
 int qemu_set_option(const char *str)
 {
+    Error *local_err = NULL;
     char group[64], id[64], arg[64];
     QemuOptsList *list;
     QemuOpts *opts;
@@ -243,7 +329,9 @@
         return -1;
     }
 
-    if (qemu_opt_set(opts, arg, str+offset+1) == -1) {
+    qemu_opt_set(opts, arg, str + offset + 1, &local_err);
+    if (local_err) {
+        error_report_err(local_err);
         return -1;
     }
     return 0;
@@ -254,7 +342,8 @@
     FILE *fp;
 };
 
-static int config_write_opt(const char *name, const char *value, void *opaque)
+static int config_write_opt(void *opaque, const char *name, const char *value,
+                            Error **errp)
 {
     struct ConfigWriteData *data = opaque;
 
@@ -262,7 +351,7 @@
     return 0;
 }
 
-static int config_write_opts(QemuOpts *opts, void *opaque)
+static int config_write_opts(void *opaque, QemuOpts *opts, Error **errp)
 {
     struct ConfigWriteData *data = opaque;
     const char *id = qemu_opts_id(opts);
@@ -272,7 +361,7 @@
     } else {
         fprintf(data->fp, "[%s]\n", data->list->name);
     }
-    qemu_opt_foreach(opts, config_write_opt, data, 0);
+    qemu_opt_foreach(opts, config_write_opt, data, NULL);
     fprintf(data->fp, "\n");
     return 0;
 }
@@ -286,7 +375,7 @@
     fprintf(fp, "# qemu config file\n\n");
     for (i = 0; lists[i] != NULL; i++) {
         data.list = lists[i];
-        qemu_opts_foreach(data.list, config_write_opts, &data, 0);
+        qemu_opts_foreach(data.list, config_write_opts, &data, NULL);
     }
 }
 
@@ -314,8 +403,7 @@
             /* group with id */
             list = find_list(lists, group, &local_err);
             if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                error_free(local_err);
+                error_report_err(local_err);
                 goto out;
             }
             opts = qemu_opts_create(list, id, 1, NULL);
@@ -325,20 +413,23 @@
             /* group without id */
             list = find_list(lists, group, &local_err);
             if (local_err) {
-                error_report("%s", error_get_pretty(local_err));
-                error_free(local_err);
+                error_report_err(local_err);
                 goto out;
             }
             opts = qemu_opts_create(list, NULL, 0, &error_abort);
             continue;
         }
-        if (sscanf(line, " %63s = \"%1023[^\"]\"", arg, value) == 2) {
+        value[0] = '\0';
+        if (sscanf(line, " %63s = \"%1023[^\"]\"", arg, value) == 2 ||
+            sscanf(line, " %63s = \"\"", arg) == 1) {
             /* arg = value */
             if (opts == NULL) {
                 error_report("no group defined");
                 goto out;
             }
-            if (qemu_opt_set(opts, arg, value) != 0) {
+            qemu_opt_set(opts, arg, value, &local_err);
+            if (local_err) {
+                error_report_err(local_err);
                 goto out;
             }
             continue;
diff --git a/util/qemu-coroutine-io.c b/util/qemu-coroutine-io.c
new file mode 100644
index 0000000..44a8969
--- /dev/null
+++ b/util/qemu-coroutine-io.c
@@ -0,0 +1,90 @@
+/*
+ * Coroutine-aware I/O functions
+ *
+ * Copyright (C) 2009-2010 Nippon Telegraph and Telephone Corporation.
+ * Copyright (c) 2011, Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/sockets.h"
+#include "qemu/coroutine.h"
+#include "qemu/iov.h"
+#include "qemu/main-loop.h"
+
+ssize_t coroutine_fn
+qemu_co_sendv_recvv(int sockfd, struct iovec *iov, unsigned iov_cnt,
+                    size_t offset, size_t bytes, bool do_send)
+{
+    size_t done = 0;
+    ssize_t ret;
+    while (done < bytes) {
+        ret = iov_send_recv(sockfd, iov, iov_cnt,
+                            offset + done, bytes - done, do_send);
+        if (ret > 0) {
+            done += ret;
+        } else if (ret < 0) {
+            if (errno == EAGAIN || errno == EWOULDBLOCK) {
+                qemu_coroutine_yield();
+            } else if (done == 0) {
+                return -errno;
+            } else {
+                break;
+            }
+        } else if (ret == 0 && !do_send) {
+            /* write (send) should never return 0.
+             * read (recv) returns 0 for end-of-file (-data).
+             * In both cases there's little point retrying,
+             * but we do for write anyway, just in case */
+            break;
+        }
+    }
+    return done;
+}
+
+ssize_t coroutine_fn
+qemu_co_send_recv(int sockfd, void *buf, size_t bytes, bool do_send)
+{
+    struct iovec iov = { .iov_base = buf, .iov_len = bytes };
+    return qemu_co_sendv_recvv(sockfd, &iov, 1, 0, bytes, do_send);
+}
+
+typedef struct {
+    Coroutine *co;
+    int fd;
+} FDYieldUntilData;
+
+static void fd_coroutine_enter(void *opaque)
+{
+    FDYieldUntilData *data = opaque;
+    qemu_set_fd_handler(data->fd, NULL, NULL, NULL);
+    qemu_coroutine_enter(data->co);
+}
+
+void coroutine_fn yield_until_fd_readable(int fd)
+{
+    FDYieldUntilData data;
+
+    assert(qemu_in_coroutine());
+    data.co = qemu_coroutine_self();
+    data.fd = fd;
+    qemu_set_fd_handler(fd, fd_coroutine_enter, NULL, &data);
+    qemu_coroutine_yield();
+}
diff --git a/util/qemu-coroutine-lock.c b/util/qemu-coroutine-lock.c
new file mode 100644
index 0000000..22aa9ab
--- /dev/null
+++ b/util/qemu-coroutine-lock.c
@@ -0,0 +1,187 @@
+/*
+ * coroutine queues and locks
+ *
+ * Copyright (c) 2011 Kevin Wolf <kwolf@redhat.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
+#include "qemu/queue.h"
+#include "trace.h"
+
+void qemu_co_queue_init(CoQueue *queue)
+{
+    QSIMPLEQ_INIT(&queue->entries);
+}
+
+void coroutine_fn qemu_co_queue_wait(CoQueue *queue)
+{
+    Coroutine *self = qemu_coroutine_self();
+    QSIMPLEQ_INSERT_TAIL(&queue->entries, self, co_queue_next);
+    qemu_coroutine_yield();
+    assert(qemu_in_coroutine());
+}
+
+/**
+ * qemu_co_queue_run_restart:
+ *
+ * Enter each coroutine that was previously marked for restart by
+ * qemu_co_queue_next() or qemu_co_queue_restart_all().  This function is
+ * invoked by the core coroutine code when the current coroutine yields or
+ * terminates.
+ */
+void qemu_co_queue_run_restart(Coroutine *co)
+{
+    Coroutine *next;
+
+    trace_qemu_co_queue_run_restart(co);
+    while ((next = QSIMPLEQ_FIRST(&co->co_queue_wakeup))) {
+        QSIMPLEQ_REMOVE_HEAD(&co->co_queue_wakeup, co_queue_next);
+        qemu_coroutine_enter(next);
+    }
+}
+
+static bool qemu_co_queue_do_restart(CoQueue *queue, bool single)
+{
+    Coroutine *self = qemu_coroutine_self();
+    Coroutine *next;
+
+    if (QSIMPLEQ_EMPTY(&queue->entries)) {
+        return false;
+    }
+
+    while ((next = QSIMPLEQ_FIRST(&queue->entries)) != NULL) {
+        QSIMPLEQ_REMOVE_HEAD(&queue->entries, co_queue_next);
+        QSIMPLEQ_INSERT_TAIL(&self->co_queue_wakeup, next, co_queue_next);
+        trace_qemu_co_queue_next(next);
+        if (single) {
+            break;
+        }
+    }
+    return true;
+}
+
+bool coroutine_fn qemu_co_queue_next(CoQueue *queue)
+{
+    assert(qemu_in_coroutine());
+    return qemu_co_queue_do_restart(queue, true);
+}
+
+void coroutine_fn qemu_co_queue_restart_all(CoQueue *queue)
+{
+    assert(qemu_in_coroutine());
+    qemu_co_queue_do_restart(queue, false);
+}
+
+bool qemu_co_enter_next(CoQueue *queue)
+{
+    Coroutine *next;
+
+    next = QSIMPLEQ_FIRST(&queue->entries);
+    if (!next) {
+        return false;
+    }
+
+    QSIMPLEQ_REMOVE_HEAD(&queue->entries, co_queue_next);
+    qemu_coroutine_enter(next);
+    return true;
+}
+
+bool qemu_co_queue_empty(CoQueue *queue)
+{
+    return QSIMPLEQ_FIRST(&queue->entries) == NULL;
+}
+
+void qemu_co_mutex_init(CoMutex *mutex)
+{
+    memset(mutex, 0, sizeof(*mutex));
+    qemu_co_queue_init(&mutex->queue);
+}
+
+void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex)
+{
+    Coroutine *self = qemu_coroutine_self();
+
+    trace_qemu_co_mutex_lock_entry(mutex, self);
+
+    while (mutex->locked) {
+        qemu_co_queue_wait(&mutex->queue);
+    }
+
+    mutex->locked = true;
+
+    trace_qemu_co_mutex_lock_return(mutex, self);
+}
+
+void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex)
+{
+    Coroutine *self = qemu_coroutine_self();
+
+    trace_qemu_co_mutex_unlock_entry(mutex, self);
+
+    assert(mutex->locked == true);
+    assert(qemu_in_coroutine());
+
+    mutex->locked = false;
+    qemu_co_queue_next(&mutex->queue);
+
+    trace_qemu_co_mutex_unlock_return(mutex, self);
+}
+
+void qemu_co_rwlock_init(CoRwlock *lock)
+{
+    memset(lock, 0, sizeof(*lock));
+    qemu_co_queue_init(&lock->queue);
+}
+
+void qemu_co_rwlock_rdlock(CoRwlock *lock)
+{
+    while (lock->writer) {
+        qemu_co_queue_wait(&lock->queue);
+    }
+    lock->reader++;
+}
+
+void qemu_co_rwlock_unlock(CoRwlock *lock)
+{
+    assert(qemu_in_coroutine());
+    if (lock->writer) {
+        lock->writer = false;
+        qemu_co_queue_restart_all(&lock->queue);
+    } else {
+        lock->reader--;
+        assert(lock->reader >= 0);
+        /* Wakeup only one waiting writer */
+        if (!lock->reader) {
+            qemu_co_queue_next(&lock->queue);
+        }
+    }
+}
+
+void qemu_co_rwlock_wrlock(CoRwlock *lock)
+{
+    while (lock->writer || lock->reader) {
+        qemu_co_queue_wait(&lock->queue);
+    }
+    lock->writer = true;
+}
diff --git a/util/qemu-coroutine-sleep.c b/util/qemu-coroutine-sleep.c
new file mode 100644
index 0000000..25de3ed
--- /dev/null
+++ b/util/qemu-coroutine-sleep.c
@@ -0,0 +1,42 @@
+/*
+ * QEMU coroutine sleep
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/coroutine.h"
+#include "qemu/timer.h"
+#include "block/aio.h"
+
+typedef struct CoSleepCB {
+    QEMUTimer *ts;
+    Coroutine *co;
+} CoSleepCB;
+
+static void co_sleep_cb(void *opaque)
+{
+    CoSleepCB *sleep_cb = opaque;
+
+    qemu_coroutine_enter(sleep_cb->co);
+}
+
+void coroutine_fn co_aio_sleep_ns(AioContext *ctx, QEMUClockType type,
+                                  int64_t ns)
+{
+    CoSleepCB sleep_cb = {
+        .co = qemu_coroutine_self(),
+    };
+    sleep_cb.ts = aio_timer_new(ctx, type, SCALE_NS, co_sleep_cb, &sleep_cb);
+    timer_mod(sleep_cb.ts, qemu_clock_get_ns(type) + ns);
+    qemu_coroutine_yield();
+    timer_del(sleep_cb.ts);
+    timer_free(sleep_cb.ts);
+}
diff --git a/util/qemu-coroutine.c b/util/qemu-coroutine.c
new file mode 100644
index 0000000..89f21a9
--- /dev/null
+++ b/util/qemu-coroutine.c
@@ -0,0 +1,147 @@
+/*
+ * QEMU coroutines
+ *
+ * Copyright IBM, Corp. 2011
+ *
+ * Authors:
+ *  Stefan Hajnoczi    <stefanha@linux.vnet.ibm.com>
+ *  Kevin Wolf         <kwolf@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "trace.h"
+#include "qemu-common.h"
+#include "qemu/thread.h"
+#include "qemu/atomic.h"
+#include "qemu/coroutine.h"
+#include "qemu/coroutine_int.h"
+
+enum {
+    POOL_BATCH_SIZE = 64,
+};
+
+/** Free list to speed up creation */
+static QSLIST_HEAD(, Coroutine) release_pool = QSLIST_HEAD_INITIALIZER(pool);
+static unsigned int release_pool_size;
+static __thread QSLIST_HEAD(, Coroutine) alloc_pool = QSLIST_HEAD_INITIALIZER(pool);
+static __thread unsigned int alloc_pool_size;
+static __thread Notifier coroutine_pool_cleanup_notifier;
+
+static void coroutine_pool_cleanup(Notifier *n, void *value)
+{
+    Coroutine *co;
+    Coroutine *tmp;
+
+    QSLIST_FOREACH_SAFE(co, &alloc_pool, pool_next, tmp) {
+        QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
+        qemu_coroutine_delete(co);
+    }
+}
+
+Coroutine *qemu_coroutine_create(CoroutineEntry *entry, void *opaque)
+{
+    Coroutine *co = NULL;
+
+    if (CONFIG_COROUTINE_POOL) {
+        co = QSLIST_FIRST(&alloc_pool);
+        if (!co) {
+            if (release_pool_size > POOL_BATCH_SIZE) {
+                /* Slow path; a good place to register the destructor, too.  */
+                if (!coroutine_pool_cleanup_notifier.notify) {
+                    coroutine_pool_cleanup_notifier.notify = coroutine_pool_cleanup;
+                    qemu_thread_atexit_add(&coroutine_pool_cleanup_notifier);
+                }
+
+                /* This is not exact; there could be a little skew between
+                 * release_pool_size and the actual size of release_pool.  But
+                 * it is just a heuristic, it does not need to be perfect.
+                 */
+                alloc_pool_size = atomic_xchg(&release_pool_size, 0);
+                QSLIST_MOVE_ATOMIC(&alloc_pool, &release_pool);
+                co = QSLIST_FIRST(&alloc_pool);
+            }
+        }
+        if (co) {
+            QSLIST_REMOVE_HEAD(&alloc_pool, pool_next);
+            alloc_pool_size--;
+        }
+    }
+
+    if (!co) {
+        co = qemu_coroutine_new();
+    }
+
+    co->entry = entry;
+    co->entry_arg = opaque;
+    QSIMPLEQ_INIT(&co->co_queue_wakeup);
+    return co;
+}
+
+static void coroutine_delete(Coroutine *co)
+{
+    co->caller = NULL;
+
+    if (CONFIG_COROUTINE_POOL) {
+        if (release_pool_size < POOL_BATCH_SIZE * 2) {
+            QSLIST_INSERT_HEAD_ATOMIC(&release_pool, co, pool_next);
+            atomic_inc(&release_pool_size);
+            return;
+        }
+        if (alloc_pool_size < POOL_BATCH_SIZE) {
+            QSLIST_INSERT_HEAD(&alloc_pool, co, pool_next);
+            alloc_pool_size++;
+            return;
+        }
+    }
+
+    qemu_coroutine_delete(co);
+}
+
+void qemu_coroutine_enter(Coroutine *co)
+{
+    Coroutine *self = qemu_coroutine_self();
+    CoroutineAction ret;
+
+    trace_qemu_coroutine_enter(self, co, co->entry_arg);
+
+    if (co->caller) {
+        fprintf(stderr, "Co-routine re-entered recursively\n");
+        abort();
+    }
+
+    co->caller = self;
+    ret = qemu_coroutine_switch(self, co, COROUTINE_ENTER);
+
+    qemu_co_queue_run_restart(co);
+
+    switch (ret) {
+    case COROUTINE_YIELD:
+        return;
+    case COROUTINE_TERMINATE:
+        trace_qemu_coroutine_terminate(co);
+        coroutine_delete(co);
+        return;
+    default:
+        abort();
+    }
+}
+
+void coroutine_fn qemu_coroutine_yield(void)
+{
+    Coroutine *self = qemu_coroutine_self();
+    Coroutine *to = self->caller;
+
+    trace_qemu_coroutine_yield(self, to);
+
+    if (!to) {
+        fprintf(stderr, "Co-routine is yielding to no one\n");
+        abort();
+    }
+
+    self->caller = NULL;
+    qemu_coroutine_switch(self, to, COROUTINE_YIELD);
+}
diff --git a/util/qemu-error.c b/util/qemu-error.c
index ed18182..af9538d 100644
--- a/util/qemu-error.c
+++ b/util/qemu-error.c
@@ -10,8 +10,9 @@
  * See the COPYING file in the top-level directory.
  */
 
-#include <stdio.h>
+#include "qemu/osdep.h"
 #include "monitor/monitor.h"
+#include "qemu/error-report.h"
 
 /*
  * Print to current monitor if we have one, else to stderr.
@@ -199,17 +200,17 @@
 bool enable_timestamp_msg;
 /*
  * Print an error message to current monitor if we have one, else to stderr.
- * Format arguments like vsprintf().  The result should not contain
- * newlines.
+ * Format arguments like vsprintf().  The resulting message should be
+ * a single phrase, with no newline or trailing punctuation.
  * Prepend the current location and append a newline.
- * It's wrong to call this in a QMP monitor.  Use qerror_report() there.
+ * It's wrong to call this in a QMP monitor.  Use error_setg() there.
  */
 void error_vreport(const char *fmt, va_list ap)
 {
     GTimeVal tv;
     gchar *timestr;
 
-    if (enable_timestamp_msg) {
+    if (enable_timestamp_msg && !cur_mon) {
         g_get_current_time(&tv);
         timestr = g_time_val_to_iso8601(&tv);
         error_printf("%s ", timestr);
@@ -223,10 +224,10 @@
 
 /*
  * Print an error message to current monitor if we have one, else to stderr.
- * Format arguments like sprintf().  The result should not contain
- * newlines.
+ * Format arguments like sprintf().  The resulting message should be a
+ * single phrase, with no newline or trailing punctuation.
  * Prepend the current location and append a newline.
- * It's wrong to call this in a QMP monitor.  Use qerror_report() there.
+ * It's wrong to call this in a QMP monitor.  Use error_setg() there.
  */
 void error_report(const char *fmt, ...)
 {
diff --git a/util/qemu-openpty.c b/util/qemu-openpty.c
index 4c53211..2e8b43b 100644
--- a/util/qemu-openpty.c
+++ b/util/qemu-openpty.c
@@ -32,7 +32,7 @@
  * linked with -lutil.
  */
 
-#include "config-host.h"
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 
 #if defined(__GLIBC__)
diff --git a/util/qemu-option.c b/util/qemu-option.c
index 83e9424..135242a 100644
--- a/util/qemu-option.c
+++ b/util/qemu-option.c
@@ -23,15 +23,17 @@
  * THE SOFTWARE.
  */
 
-#include <stdio.h>
-#include <string.h>
+#include "qemu/osdep.h"
 
+#include "qapi/error.h"
 #include "qemu-common.h"
 #include "qemu/error-report.h"
 #include "qapi/qmp/types.h"
-#include "qapi/error.h"
 #include "qapi/qmp/qerror.h"
 #include "qemu/option_int.h"
+#include "qemu/cutils.h"
+#include "qemu/id.h"
+#include "qemu/help_option.h"
 
 /*
  * Extracts the name of an option from the parameter string (p points at the
@@ -132,7 +134,8 @@
         } else if (!strcmp(value, "off")) {
             *ret = 0;
         } else {
-            error_set(errp,QERR_INVALID_PARAMETER_VALUE, name, "'on' or 'off'");
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE,
+                       name, "'on' or 'off'");
         }
     } else {
         *ret = 1;
@@ -148,12 +151,12 @@
     if (value != NULL) {
         number = strtoull(value, &postfix, 0);
         if (*postfix != '\0') {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
             return;
         }
         *ret = number;
     } else {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a number");
     }
 }
 
@@ -179,6 +182,11 @@
 
     if (value != NULL) {
         sizef = strtod(value, &postfix);
+        if (sizef < 0 || sizef > UINT64_MAX) {
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name,
+                             "a non-negative number below 2^64");
+            return;
+        }
         switch (*postfix) {
         case 'T':
             sizef *= 1024;
@@ -198,22 +206,20 @@
             *ret = (uint64_t) sizef;
             break;
         default:
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            error_printf_unless_qmp("You may use k, M, G or T suffixes for "
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
+            error_append_hint(errp, "You may use k, M, G or T suffixes for "
                     "kilobytes, megabytes, gigabytes and terabytes.\n");
-#endif
             return;
         }
     } else {
-        error_set(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
+        error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, "a size");
     }
 }
 
 bool has_help_option(const char *param)
 {
     size_t buflen = strlen(param) + 1;
-    char *buf = g_malloc0(buflen);
+    char *buf = g_malloc(buflen);
     const char *p = param;
     bool result = false;
 
@@ -230,14 +236,14 @@
     }
 
 out:
-    free(buf);
+    g_free(buf);
     return result;
 }
 
 bool is_valid_option_list(const char *param)
 {
     size_t buflen = strlen(param) + 1;
-    char *buf = g_malloc0(buflen);
+    char *buf = g_malloc(buflen);
     const char *p = param;
     bool result = true;
 
@@ -255,7 +261,7 @@
     }
 
 out:
-    free(buf);
+    g_free(buf);
     return result;
 }
 
@@ -527,7 +533,7 @@
 
     desc = find_desc_by_name(opts->list->desc, name);
     if (!desc && !opts_accepts_any(opts)) {
-        error_set(errp, QERR_INVALID_PARAMETER, name);
+        error_setg(errp, QERR_INVALID_PARAMETER, name);
         return;
     }
 
@@ -548,27 +554,14 @@
     }
 }
 
-int qemu_opt_set(QemuOpts *opts, const char *name, const char *value)
-{
-    Error *local_err = NULL;
-
-    opt_set(opts, name, value, false, &local_err);
-    if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
-    }
-
-    return 0;
-}
-
-void qemu_opt_set_err(QemuOpts *opts, const char *name, const char *value,
-                      Error **errp)
+void qemu_opt_set(QemuOpts *opts, const char *name, const char *value,
+                  Error **errp)
 {
     opt_set(opts, name, value, false, errp);
 }
 
-int qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val)
+void qemu_opt_set_bool(QemuOpts *opts, const char *name, bool val,
+                       Error **errp)
 {
     QemuOpt *opt;
     const QemuOptDesc *desc = opts->list->desc;
@@ -576,9 +569,9 @@
     opt = g_malloc0(sizeof(*opt));
     opt->desc = find_desc_by_name(desc, name);
     if (!opt->desc && !opts_accepts_any(opts)) {
-        qerror_report(QERR_INVALID_PARAMETER, name);
+        error_setg(errp, QERR_INVALID_PARAMETER, name);
         g_free(opt);
-        return -1;
+        return;
     }
 
     opt->name = g_strdup(name);
@@ -586,11 +579,10 @@
     opt->value.boolean = !!val;
     opt->str = g_strdup(val ? "on" : "off");
     QTAILQ_INSERT_TAIL(&opts->head, opt, next);
-
-    return 0;
 }
 
-int qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val)
+void qemu_opt_set_number(QemuOpts *opts, const char *name, int64_t val,
+                         Error **errp)
 {
     QemuOpt *opt;
     const QemuOptDesc *desc = opts->list->desc;
@@ -598,9 +590,9 @@
     opt = g_malloc0(sizeof(*opt));
     opt->desc = find_desc_by_name(desc, name);
     if (!opt->desc && !opts_accepts_any(opts)) {
-        qerror_report(QERR_INVALID_PARAMETER, name);
+        error_setg(errp, QERR_INVALID_PARAMETER, name);
         g_free(opt);
-        return -1;
+        return;
     }
 
     opt->name = g_strdup(name);
@@ -608,22 +600,28 @@
     opt->value.uint = val;
     opt->str = g_strdup_printf("%" PRId64, val);
     QTAILQ_INSERT_TAIL(&opts->head, opt, next);
-
-    return 0;
 }
 
+/**
+ * For each member of @opts, call @func(@opaque, name, value, @errp).
+ * @func() may store an Error through @errp, but must return non-zero then.
+ * When @func() returns non-zero, break the loop and return that value.
+ * Return zero when the loop completes.
+ */
 int qemu_opt_foreach(QemuOpts *opts, qemu_opt_loopfunc func, void *opaque,
-                     int abort_on_failure)
+                     Error **errp)
 {
     QemuOpt *opt;
-    int rc = 0;
+    int rc;
 
     QTAILQ_FOREACH(opt, &opts->head, next) {
-        rc = func(opt->name, opt->str, opaque);
-        if (abort_on_failure  &&  rc != 0)
-            break;
+        rc = func(opaque, opt->name, opt->str, errp);
+        if (rc) {
+            return rc;
+        }
+        assert(!errp || !*errp);
     }
-    return rc;
+    return 0;
 }
 
 QemuOpts *qemu_opts_find(QemuOptsList *list, const char *id)
@@ -648,10 +646,10 @@
 
     if (id) {
         if (!id_wellformed(id)) {
-            error_set(errp,QERR_INVALID_PARAMETER_VALUE, "id", "an identifier");
-#if 0 /* conversion from qerror_report() to error_set() broke this: */
-            error_printf_unless_qmp("Identifiers consist of letters, digits, '-', '.', '_', starting with a letter.\n");
-#endif
+            error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "id",
+                       "an identifier");
+            error_append_hint(errp, "Identifiers consist of letters, digits, "
+                              "'-', '.', '_', starting with a letter.\n");
             return NULL;
         }
         opts = qemu_opts_find(list, id);
@@ -692,19 +690,18 @@
     loc_restore(&opts->loc);
 }
 
-int qemu_opts_set(QemuOptsList *list, const char *id,
-                  const char *name, const char *value)
+void qemu_opts_set(QemuOptsList *list, const char *id,
+                   const char *name, const char *value, Error **errp)
 {
     QemuOpts *opts;
     Error *local_err = NULL;
 
     opts = qemu_opts_create(list, id, 1, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        return -1;
+        error_propagate(errp, local_err);
+        return;
     }
-    return qemu_opt_set(opts, name, value);
+    qemu_opt_set(opts, name, value, errp);
 }
 
 const char *qemu_opts_id(QemuOpts *opts)
@@ -737,14 +734,35 @@
     g_free(opts);
 }
 
-void qemu_opts_print(QemuOpts *opts)
+/* print value, escaping any commas in value */
+static void escaped_print(const char *value)
+{
+    const char *ptr;
+
+    for (ptr = value; *ptr; ++ptr) {
+        if (*ptr == ',') {
+            putchar(',');
+        }
+        putchar(*ptr);
+    }
+}
+
+void qemu_opts_print(QemuOpts *opts, const char *separator)
 {
     QemuOpt *opt;
     QemuOptDesc *desc = opts->list->desc;
+    const char *sep = "";
+
+    if (opts->id) {
+        printf("id=%s", opts->id); /* passed id_wellformed -> no commas */
+        sep = separator;
+    }
 
     if (desc[0].name == NULL) {
         QTAILQ_FOREACH(opt, &opts->head, next) {
-            printf("%s=\"%s\" ", opt->name, opt->str);
+            printf("%s%s=", sep, opt->name);
+            escaped_print(opt->str);
+            sep = separator;
         }
         return;
     }
@@ -757,18 +775,20 @@
             continue;
         }
         if (desc->type == QEMU_OPT_STRING) {
-            printf("%s='%s' ", desc->name, value);
+            printf("%s%s=", sep, desc->name);
+            escaped_print(value);
         } else if ((desc->type == QEMU_OPT_SIZE ||
                     desc->type == QEMU_OPT_NUMBER) && opt) {
-            printf("%s=%" PRId64 " ", desc->name, opt->value.uint);
+            printf("%s%s=%" PRId64, sep, desc->name, opt->value.uint);
         } else {
-            printf("%s=%s ", desc->name, value);
+            printf("%s%s=%s", sep, desc->name, value);
         }
+        sep = separator;
     }
 }
 
-static int opts_do_parse(QemuOpts *opts, const char *params,
-                         const char *firstname, bool prepend)
+static void opts_do_parse(QemuOpts *opts, const char *params,
+                          const char *firstname, bool prepend, Error **errp)
 {
     char option[128], value[1024];
     const char *p,*pe,*pc;
@@ -806,25 +826,30 @@
             /* store and parse */
             opt_set(opts, option, value, prepend, &local_err);
             if (local_err) {
-                qerror_report_err(local_err);
-                error_free(local_err);
-                return -1;
+                error_propagate(errp, local_err);
+                return;
             }
         }
         if (*p != ',') {
             break;
         }
     }
-    return 0;
 }
 
-int qemu_opts_do_parse(QemuOpts *opts, const char *params, const char *firstname)
+/**
+ * Store options parsed from @params into @opts.
+ * If @firstname is non-null, the first key=value in @params may omit
+ * key=, and is treated as if key was @firstname.
+ * On error, store an error object through @errp if non-null.
+ */
+void qemu_opts_do_parse(QemuOpts *opts, const char *params,
+                       const char *firstname, Error **errp)
 {
-    return opts_do_parse(opts, params, firstname, false);
+    opts_do_parse(opts, params, firstname, false, errp);
 }
 
 static QemuOpts *opts_parse(QemuOptsList *list, const char *params,
-                            int permit_abbrev, bool defaults)
+                            bool permit_abbrev, bool defaults, Error **errp)
 {
     const char *firstname;
     char value[1024], *id = NULL;
@@ -853,14 +878,13 @@
     assert(!defaults || list->merge_lists);
     opts = qemu_opts_create(list, id, !defaults, &local_err);
     if (opts == NULL) {
-        if (local_err) {
-            qerror_report_err(local_err);
-            error_free(local_err);
-        }
+        error_propagate(errp, local_err);
         return NULL;
     }
 
-    if (opts_do_parse(opts, params, firstname, defaults) != 0) {
+    opts_do_parse(opts, params, firstname, defaults, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
         qemu_opts_del(opts);
         return NULL;
     }
@@ -868,10 +892,38 @@
     return opts;
 }
 
+/**
+ * Create a QemuOpts in @list and with options parsed from @params.
+ * If @permit_abbrev, the first key=value in @params may omit key=,
+ * and is treated as if key was @list->implied_opt_name.
+ * On error, store an error object through @errp if non-null.
+ * Return the new QemuOpts on success, null pointer on error.
+ */
 QemuOpts *qemu_opts_parse(QemuOptsList *list, const char *params,
-                          int permit_abbrev)
+                          bool permit_abbrev, Error **errp)
 {
-    return opts_parse(list, params, permit_abbrev, false);
+    return opts_parse(list, params, permit_abbrev, false, errp);
+}
+
+/**
+ * Create a QemuOpts in @list and with options parsed from @params.
+ * If @permit_abbrev, the first key=value in @params may omit key=,
+ * and is treated as if key was @list->implied_opt_name.
+ * Report errors with error_report_err().  This is inappropriate in
+ * QMP context.  Do not use this function there!
+ * Return the new QemuOpts on success, null pointer on error.
+ */
+QemuOpts *qemu_opts_parse_noisily(QemuOptsList *list, const char *params,
+                                  bool permit_abbrev)
+{
+    Error *err = NULL;
+    QemuOpts *opts;
+
+    opts = opts_parse(list, params, permit_abbrev, false, &err);
+    if (err) {
+        error_report_err(err);
+    }
+    return opts;
 }
 
 void qemu_opts_set_defaults(QemuOptsList *list, const char *params,
@@ -879,7 +931,7 @@
 {
     QemuOpts *opts;
 
-    opts = opts_parse(list, params, permit_abbrev, true);
+    opts = opts_parse(list, params, permit_abbrev, true, NULL);
     assert(opts);
     (void)opts;
 }
@@ -920,14 +972,14 @@
         break;
     case QTYPE_QBOOL:
         pstrcpy(buf, sizeof(buf),
-                qbool_get_int(qobject_to_qbool(obj)) ? "on" : "off");
+                qbool_get_bool(qobject_to_qbool(obj)) ? "on" : "off");
         value = buf;
         break;
     default:
         return;
     }
 
-    qemu_opt_set_err(state->opts, key, value, state->errp);
+    qemu_opt_set(state->opts, key, value, state->errp);
 }
 
 /*
@@ -1035,7 +1087,7 @@
     QTAILQ_FOREACH(opt, &opts->head, next) {
         opt->desc = find_desc_by_name(desc, opt->name);
         if (!opt->desc) {
-            error_set(errp, QERR_INVALID_PARAMETER, opt->name);
+            error_setg(errp, QERR_INVALID_PARAMETER, opt->name);
             return;
         }
 
@@ -1047,8 +1099,15 @@
     }
 }
 
-int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func, void *opaque,
-                      int abort_on_failure)
+/**
+ * For each member of @list, call @func(@opaque, member, @errp).
+ * Call it with the current location temporarily set to the member's.
+ * @func() may store an Error through @errp, but must return non-zero then.
+ * When @func() returns non-zero, break the loop and return that value.
+ * Return zero when the loop completes.
+ */
+int qemu_opts_foreach(QemuOptsList *list, qemu_opts_loopfunc func,
+                      void *opaque, Error **errp)
 {
     Location loc;
     QemuOpts *opts;
@@ -1057,9 +1116,11 @@
     loc_push_none(&loc);
     QTAILQ_FOREACH(opts, &list->head, next) {
         loc_restore(&opts->loc);
-        rc |= func(opts, opaque);
-        if (abort_on_failure  &&  rc != 0)
+        rc = func(opaque, opts, errp);
+        if (rc) {
             break;
+        }
+        assert(!errp || !*errp);
     }
     loc_pop(&loc);
     return rc;
diff --git a/util/qemu-progress.c b/util/qemu-progress.c
index 4ee5cd0..f745233 100644
--- a/util/qemu-progress.c
+++ b/util/qemu-progress.c
@@ -22,9 +22,8 @@
  * THE SOFTWARE.
  */
 
-#include "qemu-common.h"
 #include "qemu/osdep.h"
-#include <stdio.h>
+#include "qemu-common.h"
 
 struct progress_state {
     float current;
@@ -152,7 +151,8 @@
     state.current = current;
 
     if (current > (state.last_print + state.min_skip) ||
-        (current == 100) || (current == 0)) {
+        current < (state.last_print - state.min_skip) ||
+        current == 100 || current == 0) {
         state.last_print = state.current;
         state.print();
     }
diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c
index 980358c..2aed799 100644
--- a/util/qemu-sockets.c
+++ b/util/qemu-sockets.c
@@ -15,48 +15,25 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <ctype.h>
-#include <errno.h>
-#include <unistd.h>
+#include "qemu/osdep.h"
 
 #include "monitor/monitor.h"
+#include "qapi/error.h"
 #include "qemu/sockets.h"
 #include "qemu/main-loop.h"
+#include "qapi/qmp-input-visitor.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qapi-visit.h"
+#include "qemu/cutils.h"
 
 #ifndef AI_ADDRCONFIG
 # define AI_ADDRCONFIG 0
 #endif
 
-/* used temporarily until all users are converted to QemuOpts */
-QemuOptsList socket_optslist = {
-    .name = "socket",
-    .head = QTAILQ_HEAD_INITIALIZER(socket_optslist.head),
-    .desc = {
-        {
-            .name = "path",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "host",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "port",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "to",
-            .type = QEMU_OPT_NUMBER,
-        },{
-            .name = "ipv4",
-            .type = QEMU_OPT_BOOL,
-        },{
-            .name = "ipv6",
-            .type = QEMU_OPT_BOOL,
-        },
-        { /* end if list */ }
-    },
-};
+#ifndef AI_V4MAPPED
+# define AI_V4MAPPED 0
+#endif
+
 
 static int inet_getport(struct addrinfo *e)
 {
@@ -102,37 +79,86 @@
     return NETWORK_ADDRESS_FAMILY_UNKNOWN;
 }
 
-int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
+/*
+ * Matrix we're trying to apply
+ *
+ *  ipv4  ipv6   family
+ *   -     -       PF_UNSPEC
+ *   -     f       PF_INET
+ *   -     t       PF_INET6
+ *   f     -       PF_INET6
+ *   f     f       <error>
+ *   f     t       PF_INET6
+ *   t     -       PF_INET
+ *   t     f       PF_INET
+ *   t     t       PF_INET6
+ *
+ * NB, this matrix is only about getting the necessary results
+ * from getaddrinfo(). Some of the cases require further work
+ * after reading results from getaddrinfo in order to fully
+ * apply the logic the end user wants. eg with the last case
+ * ipv4=t + ipv6=t + PF_INET6, getaddrinfo alone can only
+ * guarantee the ipv6=t part of the request - we need more
+ * checks to provide ipv4=t part of the guarantee. This is
+ * outside scope of this method and not currently handled by
+ * callers at all.
+ */
+static int inet_ai_family_from_address(InetSocketAddress *addr,
+                                       Error **errp)
+{
+    if (addr->has_ipv6 && addr->has_ipv4 &&
+        !addr->ipv6 && !addr->ipv4) {
+        error_setg(errp, "Cannot disable IPv4 and IPv6 at same time");
+        return PF_UNSPEC;
+    }
+    if ((addr->has_ipv6 && addr->ipv6) || (addr->has_ipv4 && !addr->ipv4)) {
+        return PF_INET6;
+    }
+    if ((addr->has_ipv4 && addr->ipv4) || (addr->has_ipv6 && !addr->ipv6)) {
+        return PF_INET;
+    }
+    return PF_UNSPEC;
+}
+
+static int inet_listen_saddr(InetSocketAddress *saddr,
+                             int port_offset,
+                             bool update_addr,
+                             Error **errp)
 {
     struct addrinfo ai,*res,*e;
-    const char *addr;
     char port[33];
     char uaddr[INET6_ADDRSTRLEN+1];
     char uport[33];
-    int slisten, rc, to, port_min, port_max, p;
+    int slisten, rc, port_min, port_max, p;
+    Error *err = NULL;
 
     memset(&ai,0, sizeof(ai));
-    ai.ai_flags = AI_PASSIVE | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
+    ai.ai_flags = AI_PASSIVE;
+    ai.ai_family = inet_ai_family_from_address(saddr, &err);
     ai.ai_socktype = SOCK_STREAM;
 
-    if ((qemu_opt_get(opts, "host") == NULL) ||
-        (qemu_opt_get(opts, "port") == NULL)) {
-        error_setg(errp, "host and/or port not specified");
+    if (err) {
+        error_propagate(errp, err);
         return -1;
     }
-    pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
-    addr = qemu_opt_get(opts, "host");
 
-    to = qemu_opt_get_number(opts, "to", 0);
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
+    if (saddr->host == NULL) {
+        error_setg(errp, "host not specified");
+        return -1;
+    }
+    if (saddr->port != NULL) {
+        pstrcpy(port, sizeof(port), saddr->port);
+    } else {
+        port[0] = '\0';
+    }
 
     /* lookup */
     if (port_offset) {
         unsigned long long baseport;
+        if (strlen(port) == 0) {
+            error_setg(errp, "port not specified");
+            return -1;
+        }
         if (parse_uint_full(port, &baseport, 10) < 0) {
             error_setg(errp, "can't convert to a number: %s", port);
             return -1;
@@ -144,10 +170,11 @@
         }
         snprintf(port, sizeof(port), "%d", (int)baseport + port_offset);
     }
-    rc = getaddrinfo(strlen(addr) ? addr : NULL, port, &ai, &res);
+    rc = getaddrinfo(strlen(saddr->host) ? saddr->host : NULL,
+                     strlen(port) ? port : NULL, &ai, &res);
     if (rc != 0) {
-        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
-                   gai_strerror(rc));
+        error_setg(errp, "address resolution failed for %s:%s: %s",
+                   saddr->host, port, gai_strerror(rc));
         return -1;
     }
 
@@ -175,7 +202,7 @@
 #endif
 
         port_min = inet_getport(e);
-        port_max = to ? to + port_offset : port_min;
+        port_max = saddr->has_to ? saddr->to + port_offset : port_min;
         for (p = port_min; p <= port_max; p++) {
             inet_setport(e, p);
             if (bind(slisten, e->ai_addr, e->ai_addrlen) == 0) {
@@ -199,25 +226,25 @@
         freeaddrinfo(res);
         return -1;
     }
-    snprintf(uport, sizeof(uport), "%d", inet_getport(e) - port_offset);
-    qemu_opt_set(opts, "host", uaddr);
-    qemu_opt_set(opts, "port", uport);
-    qemu_opt_set(opts, "ipv6", (e->ai_family == PF_INET6) ? "on" : "off");
-    qemu_opt_set(opts, "ipv4", (e->ai_family != PF_INET6) ? "on" : "off");
+    if (update_addr) {
+        g_free(saddr->host);
+        saddr->host = g_strdup(uaddr);
+        g_free(saddr->port);
+        saddr->port = g_strdup_printf("%d",
+                                      inet_getport(e) - port_offset);
+        saddr->has_ipv6 = saddr->ipv6 = e->ai_family == PF_INET6;
+        saddr->has_ipv4 = saddr->ipv4 = e->ai_family != PF_INET6;
+    }
     freeaddrinfo(res);
     return slisten;
 }
 
 #ifdef _WIN32
 #define QEMU_SOCKET_RC_INPROGRESS(rc) \
-    ((rc) == -WSAEINPROGRESS || (rc) ==  -WSAEWOULDBLOCK || (rc) == -WSAEALREADY)
-#define QEMU_SOCKET_RC_INTR(rc) \
-    ((rc) == -WSAEINTR)
+    ((rc) == -EINPROGRESS || (rc) == -EWOULDBLOCK || (rc) == -WSAEALREADY)
 #else
 #define QEMU_SOCKET_RC_INPROGRESS(rc) \
     ((rc) == -EINPROGRESS)
-#define QEMU_SOCKET_RC_INTR(rc) \
-    ((rc) == -EINTR)
 #endif
 
 /* Struct to store connect state for non blocking connect */
@@ -240,11 +267,11 @@
     bool in_progress;
     Error *err = NULL;
 
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
+    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
 
     do {
         rc = qemu_getsockopt(s->fd, SOL_SOCKET, SO_ERROR, &val, &valsize);
-    } while (rc == -1 && socket_error() == EINTR);
+    } while (rc == -1 && errno == EINTR);
 
     /* update rc to contain error */
     if (!rc && val) {
@@ -306,14 +333,13 @@
     do {
         rc = 0;
         if (connect(sock, addr->ai_addr, addr->ai_addrlen) < 0) {
-            rc = -socket_error();
+            rc = -errno;
         }
-    } while (QEMU_SOCKET_RC_INTR(rc));
+    } while (rc == -EINTR);
 
     if (connect_state != NULL && QEMU_SOCKET_RC_INPROGRESS(rc)) {
         connect_state->fd = sock;
-        qemu_set_fd_handler2(sock, NULL, NULL, wait_for_connect,
-                             connect_state);
+        qemu_set_fd_handler(sock, NULL, wait_for_connect, connect_state);
         *in_progress = true;
     } else if (rc < 0) {
         error_setg_errno(errp, errno, "Failed to connect socket");
@@ -323,38 +349,50 @@
     return sock;
 }
 
-static struct addrinfo *inet_parse_connect_opts(QemuOpts *opts, Error **errp)
+static struct addrinfo *inet_parse_connect_saddr(InetSocketAddress *saddr,
+                                                 Error **errp)
 {
     struct addrinfo ai, *res;
     int rc;
-    const char *addr;
-    const char *port;
+    Error *err = NULL;
+    static int useV4Mapped = 1;
 
     memset(&ai, 0, sizeof(ai));
 
     ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
+    if (atomic_read(&useV4Mapped)) {
+        ai.ai_flags |= AI_V4MAPPED;
+    }
+    ai.ai_family = inet_ai_family_from_address(saddr, &err);
     ai.ai_socktype = SOCK_STREAM;
 
-    addr = qemu_opt_get(opts, "host");
-    port = qemu_opt_get(opts, "port");
-    if (addr == NULL || port == NULL) {
+    if (err) {
+        error_propagate(errp, err);
+        return NULL;
+    }
+
+    if (saddr->host == NULL || saddr->port == NULL) {
         error_setg(errp, "host and/or port not specified");
         return NULL;
     }
 
-    if (qemu_opt_get_bool(opts, "ipv4", 0)) {
-        ai.ai_family = PF_INET;
-    }
-    if (qemu_opt_get_bool(opts, "ipv6", 0)) {
-        ai.ai_family = PF_INET6;
-    }
-
     /* lookup */
-    rc = getaddrinfo(addr, port, &ai, &res);
+    rc = getaddrinfo(saddr->host, saddr->port, &ai, &res);
+
+    /* At least FreeBSD and OS-X 10.6 declare AI_V4MAPPED but
+     * then don't implement it in their getaddrinfo(). Detect
+     * this and retry without the flag since that's preferrable
+     * to a fatal error
+     */
+    if (rc == EAI_BADFLAGS &&
+        (ai.ai_flags & AI_V4MAPPED)) {
+        atomic_set(&useV4Mapped, 0);
+        ai.ai_flags &= ~AI_V4MAPPED;
+        rc = getaddrinfo(saddr->host, saddr->port, &ai, &res);
+    }
     if (rc != 0) {
-        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
-                   gai_strerror(rc));
+        error_setg(errp, "address resolution failed for %s:%s: %s",
+                   saddr->host, saddr->port, gai_strerror(rc));
         return NULL;
     }
     return res;
@@ -363,8 +401,7 @@
 /**
  * Create a socket and connect it to an address.
  *
- * @opts: QEMU options, recognized parameters strings "host" and "port",
- *        bools "ipv4" and "ipv6".
+ * @saddr: Inet socket address specification
  * @errp: set on error
  * @callback: callback function for non-blocking connect
  * @opaque: opaque for callback function
@@ -375,8 +412,8 @@
  * function succeeds, callback will be called when the connection
  * completes, with the file descriptor on success, or -1 on error.
  */
-int inet_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque)
+static int inet_connect_saddr(InetSocketAddress *saddr, Error **errp,
+                              NonBlockingConnectHandler *callback, void *opaque)
 {
     Error *local_err = NULL;
     struct addrinfo *res, *e;
@@ -384,7 +421,7 @@
     bool in_progress;
     ConnectState *connect_state = NULL;
 
-    res = inet_parse_connect_opts(opts, errp);
+    res = inet_parse_connect_saddr(saddr, errp);
     if (!res) {
         return -1;
     }
@@ -423,38 +460,41 @@
     return sock;
 }
 
-int inet_dgram_opts(QemuOpts *opts, Error **errp)
+static int inet_dgram_saddr(InetSocketAddress *sraddr,
+                            InetSocketAddress *sladdr,
+                            Error **errp)
 {
     struct addrinfo ai, *peer = NULL, *local = NULL;
     const char *addr;
     const char *port;
     int sock = -1, rc;
+    Error *err = NULL;
 
     /* lookup peer addr */
     memset(&ai,0, sizeof(ai));
-    ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
-    ai.ai_family = PF_UNSPEC;
+    ai.ai_flags = AI_CANONNAME | AI_V4MAPPED | AI_ADDRCONFIG;
+    ai.ai_family = inet_ai_family_from_address(sraddr, &err);
     ai.ai_socktype = SOCK_DGRAM;
 
-    addr = qemu_opt_get(opts, "host");
-    port = qemu_opt_get(opts, "port");
+    if (err) {
+        error_propagate(errp, err);
+        goto err;
+    }
+
+    addr = sraddr->host;
+    port = sraddr->port;
     if (addr == NULL || strlen(addr) == 0) {
         addr = "localhost";
     }
     if (port == NULL || strlen(port) == 0) {
         error_setg(errp, "remote port not specified");
-        return -1;
+        goto err;
     }
 
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
-        ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
-        ai.ai_family = PF_INET6;
-
     if (0 != (rc = getaddrinfo(addr, port, &ai, &peer))) {
         error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
                    gai_strerror(rc));
-	return -1;
+	goto err;
     }
 
     /* lookup local addr */
@@ -463,13 +503,19 @@
     ai.ai_family = peer->ai_family;
     ai.ai_socktype = SOCK_DGRAM;
 
-    addr = qemu_opt_get(opts, "localaddr");
-    port = qemu_opt_get(opts, "localport");
-    if (addr == NULL || strlen(addr) == 0) {
+    if (sladdr) {
+        addr = sladdr->host;
+        port = sladdr->port;
+        if (addr == NULL || strlen(addr) == 0) {
+            addr = NULL;
+        }
+        if (!port || strlen(port) == 0) {
+            port = "0";
+        }
+    } else {
         addr = NULL;
-    }
-    if (!port || strlen(port) == 0)
         port = "0";
+    }
 
     if (0 != (rc = getaddrinfo(addr, port, &ai, &local))) {
         error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
@@ -578,56 +624,6 @@
     return NULL;
 }
 
-static void inet_addr_to_opts(QemuOpts *opts, const InetSocketAddress *addr)
-{
-    bool ipv4 = addr->ipv4 || !addr->has_ipv4;
-    bool ipv6 = addr->ipv6 || !addr->has_ipv6;
-
-    if (!ipv4 || !ipv6) {
-        qemu_opt_set_bool(opts, "ipv4", ipv4);
-        qemu_opt_set_bool(opts, "ipv6", ipv6);
-    }
-    if (addr->has_to) {
-        char to[20];
-        snprintf(to, sizeof(to), "%d", addr->to);
-        qemu_opt_set(opts, "to", to);
-    }
-    qemu_opt_set(opts, "host", addr->host);
-    qemu_opt_set(opts, "port", addr->port);
-}
-
-int inet_listen(const char *str, char *ostr, int olen,
-                int socktype, int port_offset, Error **errp)
-{
-    QemuOpts *opts;
-    char *optstr;
-    int sock = -1;
-    InetSocketAddress *addr;
-
-    addr = inet_parse(str, errp);
-    if (addr != NULL) {
-        opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-        inet_addr_to_opts(opts, addr);
-        qapi_free_InetSocketAddress(addr);
-        sock = inet_listen_opts(opts, port_offset, errp);
-        if (sock != -1 && ostr) {
-            optstr = strchr(str, ',');
-            if (qemu_opt_get_bool(opts, "ipv6", 0)) {
-                snprintf(ostr, olen, "[%s]:%s%s",
-                         qemu_opt_get(opts, "host"),
-                         qemu_opt_get(opts, "port"),
-                         optstr ? optstr : "");
-            } else {
-                snprintf(ostr, olen, "%s:%s%s",
-                         qemu_opt_get(opts, "host"),
-                         qemu_opt_get(opts, "port"),
-                         optstr ? optstr : "");
-            }
-        }
-        qemu_opts_del(opts);
-    }
-    return sock;
-}
 
 /**
  * Create a blocking socket and connect it to an address.
@@ -639,77 +635,46 @@
  **/
 int inet_connect(const char *str, Error **errp)
 {
-    QemuOpts *opts;
     int sock = -1;
     InetSocketAddress *addr;
 
     addr = inet_parse(str, errp);
     if (addr != NULL) {
-        opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-        inet_addr_to_opts(opts, addr);
+        sock = inet_connect_saddr(addr, errp, NULL, NULL);
         qapi_free_InetSocketAddress(addr);
-        sock = inet_connect_opts(opts, errp, NULL, NULL);
-        qemu_opts_del(opts);
-    }
-    return sock;
-}
-
-/**
- * Create a non-blocking socket and connect it to an address.
- * Calls the callback function with fd in case of success or -1 in case of
- * error.
- *
- * @str: address string
- * @callback: callback function that is called when connect completes,
- *            cannot be NULL.
- * @opaque: opaque for callback function
- * @errp: set in case of an error
- *
- * Returns: -1 on immediate error, file descriptor on success.
- **/
-int inet_nonblocking_connect(const char *str,
-                             NonBlockingConnectHandler *callback,
-                             void *opaque, Error **errp)
-{
-    QemuOpts *opts;
-    int sock = -1;
-    InetSocketAddress *addr;
-
-    g_assert(callback != NULL);
-
-    addr = inet_parse(str, errp);
-    if (addr != NULL) {
-        opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-        inet_addr_to_opts(opts, addr);
-        qapi_free_InetSocketAddress(addr);
-        sock = inet_connect_opts(opts, errp, callback, opaque);
-        qemu_opts_del(opts);
     }
     return sock;
 }
 
 #ifndef _WIN32
 
-int unix_listen_opts(QemuOpts *opts, Error **errp)
+static int unix_listen_saddr(UnixSocketAddress *saddr,
+                             bool update_addr,
+                             Error **errp)
 {
     struct sockaddr_un un;
-    const char *path = qemu_opt_get(opts, "path");
     int sock, fd;
 
     sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
     if (sock < 0) {
-        error_setg_errno(errp, errno, "Failed to create socket");
+        error_setg_errno(errp, errno, "Failed to create Unix socket");
         return -1;
     }
 
     memset(&un, 0, sizeof(un));
     un.sun_family = AF_UNIX;
-    if (path && strlen(path)) {
-        snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
+    if (saddr->path && strlen(saddr->path)) {
+        snprintf(un.sun_path, sizeof(un.sun_path), "%s", saddr->path);
     } else {
-        char *tmpdir = getenv("TMPDIR");
-        snprintf(un.sun_path, sizeof(un.sun_path), "%s/qemu-socket-XXXXXX",
-                 tmpdir ? tmpdir : "/tmp");
+        const char *tmpdir = getenv("TMPDIR");
+        tmpdir = tmpdir ? tmpdir : "/tmp";
+        if (snprintf(un.sun_path, sizeof(un.sun_path), "%s/qemu-socket-XXXXXX",
+                     tmpdir) >= sizeof(un.sun_path)) {
+            error_setg_errno(errp, errno,
+                             "TMPDIR environment variable (%s) too large", tmpdir);
+            goto err;
+        }
+
         /*
          * This dummy fd usage silences the mktemp() unsecure warning.
          * Using mkstemp() doesn't make things more secure here
@@ -717,13 +682,26 @@
          * to unlink first and thus re-open the race window.  The
          * worst case possible is bind() failing, i.e. a DoS attack.
          */
-        fd = mkstemp(un.sun_path); close(fd);
-        qemu_opt_set(opts, "path", un.sun_path);
+        fd = mkstemp(un.sun_path);
+        if (fd < 0) {
+            error_setg_errno(errp, errno,
+                             "Failed to make a temporary socket name in %s", tmpdir);
+            goto err;
+        }
+        close(fd);
+        if (update_addr) {
+            g_free(saddr->path);
+            saddr->path = g_strdup(un.sun_path);
+        }
     }
 
-    unlink(un.sun_path);
+    if (unlink(un.sun_path) < 0 && errno != ENOENT) {
+        error_setg_errno(errp, errno,
+                         "Failed to unlink socket %s", un.sun_path);
+        goto err;
+    }
     if (bind(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
-        error_setg_errno(errp, errno, "Failed to bind socket");
+        error_setg_errno(errp, errno, "Failed to bind socket to %s", un.sun_path);
         goto err;
     }
     if (listen(sock, 1) < 0) {
@@ -738,15 +716,14 @@
     return -1;
 }
 
-int unix_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque)
+static int unix_connect_saddr(UnixSocketAddress *saddr, Error **errp,
+                              NonBlockingConnectHandler *callback, void *opaque)
 {
     struct sockaddr_un un;
-    const char *path = qemu_opt_get(opts, "path");
     ConnectState *connect_state = NULL;
     int sock, rc;
 
-    if (path == NULL) {
+    if (saddr->path == NULL) {
         error_setg(errp, "unix connect: no path specified");
         return -1;
     }
@@ -765,20 +742,19 @@
 
     memset(&un, 0, sizeof(un));
     un.sun_family = AF_UNIX;
-    snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
+    snprintf(un.sun_path, sizeof(un.sun_path), "%s", saddr->path);
 
     /* connect to peer */
     do {
         rc = 0;
         if (connect(sock, (struct sockaddr *) &un, sizeof(un)) < 0) {
-            rc = -socket_error();
+            rc = -errno;
         }
     } while (rc == -EINTR);
 
     if (connect_state != NULL && QEMU_SOCKET_RC_INPROGRESS(rc)) {
         connect_state->fd = sock;
-        qemu_set_fd_handler2(sock, NULL, NULL, wait_for_connect,
-                             connect_state);
+        qemu_set_fd_handler(sock, NULL, wait_for_connect, connect_state);
         return sock;
     } else if (rc >= 0) {
         /* non blocking socket immediate success, call callback */
@@ -799,15 +775,17 @@
 
 #else
 
-int unix_listen_opts(QemuOpts *opts, Error **errp)
+static int unix_listen_saddr(UnixSocketAddress *saddr,
+                             bool update_addr,
+                             Error **errp)
 {
     error_setg(errp, "unix sockets are not available on windows");
     errno = ENOTSUP;
     return -1;
 }
 
-int unix_connect_opts(QemuOpts *opts, Error **errp,
-                      NonBlockingConnectHandler *callback, void *opaque)
+static int unix_connect_saddr(UnixSocketAddress *saddr, Error **errp,
+                              NonBlockingConnectHandler *callback, void *opaque)
 {
     error_setg(errp, "unix sockets are not available on windows");
     errno = ENOTSUP;
@@ -818,11 +796,11 @@
 /* compatibility wrapper */
 int unix_listen(const char *str, char *ostr, int olen, Error **errp)
 {
-    QemuOpts *opts;
     char *path, *optstr;
     int sock, len;
+    UnixSocketAddress *saddr;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
+    saddr = g_new0(UnixSocketAddress, 1);
 
     optstr = strchr(str, ',');
     if (optstr) {
@@ -830,50 +808,33 @@
         if (len) {
             path = g_malloc(len+1);
             snprintf(path, len+1, "%.*s", len, str);
-            qemu_opt_set(opts, "path", path);
-            g_free(path);
+            saddr->path = path;
         }
     } else {
-        qemu_opt_set(opts, "path", str);
+        saddr->path = g_strdup(str);
     }
 
-    sock = unix_listen_opts(opts, errp);
+    sock = unix_listen_saddr(saddr, true, errp);
 
     if (sock != -1 && ostr)
-        snprintf(ostr, olen, "%s%s", qemu_opt_get(opts, "path"), optstr ? optstr : "");
-    qemu_opts_del(opts);
+        snprintf(ostr, olen, "%s%s", saddr->path, optstr ? optstr : "");
+    qapi_free_UnixSocketAddress(saddr);
     return sock;
 }
 
 int unix_connect(const char *path, Error **errp)
 {
-    QemuOpts *opts;
+    UnixSocketAddress *saddr;
     int sock;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    qemu_opt_set(opts, "path", path);
-    sock = unix_connect_opts(opts, errp, NULL, NULL);
-    qemu_opts_del(opts);
+    saddr = g_new0(UnixSocketAddress, 1);
+    saddr->path = g_strdup(path);
+    sock = unix_connect_saddr(saddr, errp, NULL, NULL);
+    qapi_free_UnixSocketAddress(saddr);
     return sock;
 }
 
 
-int unix_nonblocking_connect(const char *path,
-                             NonBlockingConnectHandler *callback,
-                             void *opaque, Error **errp)
-{
-    QemuOpts *opts;
-    int sock = -1;
-
-    g_assert(callback != NULL);
-
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    qemu_opt_set(opts, "path", path);
-    sock = unix_connect_opts(opts, errp, callback, opaque);
-    qemu_opts_del(opts);
-    return sock;
-}
-
 SocketAddress *socket_parse(const char *str, Error **errp)
 {
     SocketAddress *addr;
@@ -884,23 +845,23 @@
             error_setg(errp, "invalid Unix socket address");
             goto fail;
         } else {
-            addr->kind = SOCKET_ADDRESS_KIND_UNIX;
-            addr->q_unix = g_new(UnixSocketAddress, 1);
-            addr->q_unix->path = g_strdup(str + 5);
+            addr->type = SOCKET_ADDRESS_KIND_UNIX;
+            addr->u.q_unix.data = g_new(UnixSocketAddress, 1);
+            addr->u.q_unix.data->path = g_strdup(str + 5);
         }
     } else if (strstart(str, "fd:", NULL)) {
         if (str[3] == '\0') {
             error_setg(errp, "invalid file descriptor address");
             goto fail;
         } else {
-            addr->kind = SOCKET_ADDRESS_KIND_FD;
-            addr->fd = g_new(String, 1);
-            addr->fd->str = g_strdup(str + 3);
+            addr->type = SOCKET_ADDRESS_KIND_FD;
+            addr->u.fd.data = g_new(String, 1);
+            addr->u.fd.data->str = g_strdup(str + 3);
         }
     } else {
-        addr->kind = SOCKET_ADDRESS_KIND_INET;
-        addr->inet = inet_parse(str, errp);
-        if (addr->inet == NULL) {
+        addr->type = SOCKET_ADDRESS_KIND_INET;
+        addr->u.inet.data = inet_parse(str, errp);
+        if (addr->u.inet.data == NULL) {
             goto fail;
         }
     }
@@ -911,26 +872,22 @@
     return NULL;
 }
 
-int socket_connect_addr(SocketAddress *addr, Error **errp,
-                        NonBlockingConnectHandler *callback, void *opaque)
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque)
 {
-    QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, addr->inet);
-        fd = inet_connect_opts(opts, errp, callback, opaque);
+        fd = inet_connect_saddr(addr->u.inet.data, errp, callback, opaque);
         break;
 
     case SOCKET_ADDRESS_KIND_UNIX:
-        qemu_opt_set(opts, "path", addr->q_unix->path);
-        fd = unix_connect_opts(opts, errp, callback, opaque);
+        fd = unix_connect_saddr(addr->u.q_unix.data, errp, callback, opaque);
         break;
 
     case SOCKET_ADDRESS_KIND_FD:
-        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        fd = monitor_get_fd(cur_mon, addr->u.fd.data->str, errp);
         if (fd >= 0 && callback) {
             qemu_set_nonblock(fd);
             callback(fd, NULL, opaque);
@@ -940,58 +897,210 @@
     default:
         abort();
     }
-    qemu_opts_del(opts);
     return fd;
 }
 
-int socket_listen_addr(SocketAddress *addr, Error **errp)
+int socket_listen(SocketAddress *addr, Error **errp)
 {
-    QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (addr->kind) {
+    switch (addr->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, addr->inet);
-        fd = inet_listen_opts(opts, 0, errp);
+        fd = inet_listen_saddr(addr->u.inet.data, 0, false, errp);
         break;
 
     case SOCKET_ADDRESS_KIND_UNIX:
-        qemu_opt_set(opts, "path", addr->q_unix->path);
-        fd = unix_listen_opts(opts, errp);
+        fd = unix_listen_saddr(addr->u.q_unix.data, false, errp);
         break;
 
     case SOCKET_ADDRESS_KIND_FD:
-        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        fd = monitor_get_fd(cur_mon, addr->u.fd.data->str, errp);
         break;
 
     default:
         abort();
     }
-    qemu_opts_del(opts);
     return fd;
 }
 
+void socket_listen_cleanup(int fd, Error **errp)
+{
+    SocketAddress *addr;
+
+    addr = socket_local_address(fd, errp);
+
+    if (addr->type == SOCKET_ADDRESS_KIND_UNIX
+        && addr->u.q_unix.data->path) {
+        if (unlink(addr->u.q_unix.data->path) < 0 && errno != ENOENT) {
+            error_setg_errno(errp, errno,
+                             "Failed to unlink socket %s",
+                             addr->u.q_unix.data->path);
+        }
+    }
+
+    qapi_free_SocketAddress(addr);
+}
+
 int socket_dgram(SocketAddress *remote, SocketAddress *local, Error **errp)
 {
-    QemuOpts *opts;
     int fd;
 
-    opts = qemu_opts_create(&socket_optslist, NULL, 0, &error_abort);
-    switch (remote->kind) {
+    switch (remote->type) {
     case SOCKET_ADDRESS_KIND_INET:
-        inet_addr_to_opts(opts, remote->inet);
-        if (local) {
-            qemu_opt_set(opts, "localaddr", local->inet->host);
-            qemu_opt_set(opts, "localport", local->inet->port);
-        }
-        fd = inet_dgram_opts(opts, errp);
+        fd = inet_dgram_saddr(remote->u.inet.data,
+                              local ? local->u.inet.data : NULL, errp);
         break;
 
     default:
         error_setg(errp, "socket type unsupported for datagram");
         fd = -1;
     }
-    qemu_opts_del(opts);
     return fd;
 }
+
+
+static SocketAddress *
+socket_sockaddr_to_address_inet(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                Error **errp)
+{
+    char host[NI_MAXHOST];
+    char serv[NI_MAXSERV];
+    SocketAddress *addr;
+    InetSocketAddress *inet;
+    int ret;
+
+    ret = getnameinfo((struct sockaddr *)sa, salen,
+                      host, sizeof(host),
+                      serv, sizeof(serv),
+                      NI_NUMERICHOST | NI_NUMERICSERV);
+    if (ret != 0) {
+        error_setg(errp, "Cannot format numeric socket address: %s",
+                   gai_strerror(ret));
+        return NULL;
+    }
+
+    addr = g_new0(SocketAddress, 1);
+    addr->type = SOCKET_ADDRESS_KIND_INET;
+    inet = addr->u.inet.data = g_new0(InetSocketAddress, 1);
+    inet->host = g_strdup(host);
+    inet->port = g_strdup(serv);
+    if (sa->ss_family == AF_INET) {
+        inet->has_ipv4 = inet->ipv4 = true;
+    } else {
+        inet->has_ipv6 = inet->ipv6 = true;
+    }
+
+    return addr;
+}
+
+
+#ifndef WIN32
+static SocketAddress *
+socket_sockaddr_to_address_unix(struct sockaddr_storage *sa,
+                                socklen_t salen,
+                                Error **errp)
+{
+    SocketAddress *addr;
+    struct sockaddr_un *su = (struct sockaddr_un *)sa;
+
+    addr = g_new0(SocketAddress, 1);
+    addr->type = SOCKET_ADDRESS_KIND_UNIX;
+    addr->u.q_unix.data = g_new0(UnixSocketAddress, 1);
+    if (su->sun_path[0]) {
+        addr->u.q_unix.data->path = g_strndup(su->sun_path,
+                                              sizeof(su->sun_path));
+    }
+
+    return addr;
+}
+#endif /* WIN32 */
+
+SocketAddress *
+socket_sockaddr_to_address(struct sockaddr_storage *sa,
+                           socklen_t salen,
+                           Error **errp)
+{
+    switch (sa->ss_family) {
+    case AF_INET:
+    case AF_INET6:
+        return socket_sockaddr_to_address_inet(sa, salen, errp);
+
+#ifndef WIN32
+    case AF_UNIX:
+        return socket_sockaddr_to_address_unix(sa, salen, errp);
+#endif /* WIN32 */
+
+    default:
+        error_setg(errp, "socket family %d unsupported",
+                   sa->ss_family);
+        return NULL;
+    }
+    return 0;
+}
+
+
+SocketAddress *socket_local_address(int fd, Error **errp)
+{
+    struct sockaddr_storage ss;
+    socklen_t sslen = sizeof(ss);
+
+    if (getsockname(fd, (struct sockaddr *)&ss, &sslen) < 0) {
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to query local socket address");
+        return NULL;
+    }
+
+    return socket_sockaddr_to_address(&ss, sslen, errp);
+}
+
+
+SocketAddress *socket_remote_address(int fd, Error **errp)
+{
+    struct sockaddr_storage ss;
+    socklen_t sslen = sizeof(ss);
+
+    if (getpeername(fd, (struct sockaddr *)&ss, &sslen) < 0) {
+        error_setg_errno(errp, errno, "%s",
+                         "Unable to query remote socket address");
+        return NULL;
+    }
+
+    return socket_sockaddr_to_address(&ss, sslen, errp);
+}
+
+char *socket_address_to_string(struct SocketAddress *addr, Error **errp)
+{
+    char *buf;
+    InetSocketAddress *inet;
+    char host_port[INET6_ADDRSTRLEN + 5 + 4];
+
+    switch (addr->type) {
+    case SOCKET_ADDRESS_KIND_INET:
+        inet = addr->u.inet.data;
+        if (strchr(inet->host, ':') == NULL) {
+            snprintf(host_port, sizeof(host_port), "%s:%s", inet->host,
+                    inet->port);
+            buf = g_strdup(host_port);
+        } else {
+            snprintf(host_port, sizeof(host_port), "[%s]:%s", inet->host,
+                    inet->port);
+            buf = g_strdup(host_port);
+        }
+        break;
+
+    case SOCKET_ADDRESS_KIND_UNIX:
+        buf = g_strdup(addr->u.q_unix.data->path);
+        break;
+
+    case SOCKET_ADDRESS_KIND_FD:
+        buf = g_strdup(addr->u.fd.data->str);
+        break;
+
+    default:
+        error_setg(errp, "socket family %d unsupported",
+                   addr->type);
+        return NULL;
+    }
+    return buf;
+}
diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c
index 9ccd862..1417ef3 100644
--- a/util/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -10,30 +10,25 @@
  * See the COPYING file in the top-level directory.
  *
  */
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <time.h>
-#include <signal.h>
-#include <stdint.h>
-#include <string.h>
-#include <limits.h>
-#include <unistd.h>
-#include <sys/time.h>
+#include "qemu/osdep.h"
 #ifdef __linux__
 #include <sys/syscall.h>
 #include <linux/futex.h>
 #endif
+#include "qemu/abort.h"
 #include "qemu/thread.h"
 #include "qemu/atomic.h"
-
-#if defined(CONFIG_ANDROID)
-#include "android-qemu2-glue/looper-qemu.h"
-#include "android/crashreport/crash-handler.h"
-#endif
+#include "qemu/notify.h"
 
 static bool name_threads;
 
+static QemuThreadSetupFunc thread_setup_func;
+
+void qemu_thread_register_setup_callback(QemuThreadSetupFunc setup_func)
+{
+    thread_setup_func = setup_func;
+}
+
 void qemu_thread_naming(bool enable)
 {
     name_threads = enable;
@@ -48,25 +43,14 @@
 
 static void error_exit(int err, const char *msg)
 {
-    fprintf(stderr, "qemu: %s: %s\n", msg, strerror(err));
-#ifdef CONFIG_ANDROID
-    crashhandler_die_format(
-                "Internal error in %s: %s (%d)",
-                msg, strerror(err), err);
-#else
-    abort();
-#endif
+    qemu_abort("qemu: %s: %s\n", msg, strerror(err));
 }
 
 void qemu_mutex_init(QemuMutex *mutex)
 {
     int err;
-    pthread_mutexattr_t mutexattr;
 
-    pthread_mutexattr_init(&mutexattr);
-    pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_ERRORCHECK);
-    err = pthread_mutex_init(&mutex->lock, &mutexattr);
-    pthread_mutexattr_destroy(&mutexattr);
+    err = pthread_mutex_init(&mutex->lock, NULL);
     if (err)
         error_exit(err, __func__);
 }
@@ -312,16 +296,27 @@
 
 static inline void futex_wait(QemuEvent *ev, unsigned val)
 {
-    futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0);
+    while (futex(ev, FUTEX_WAIT, (int) val, NULL, NULL, 0)) {
+        switch (errno) {
+        case EWOULDBLOCK:
+            return;
+        case EINTR:
+            break; /* get out of switch and retry */
+        default:
+            abort();
+        }
+    }
 }
 #else
 static inline void futex_wake(QemuEvent *ev, int n)
 {
+    pthread_mutex_lock(&ev->lock);
     if (n == 1) {
         pthread_cond_signal(&ev->cond);
     } else {
         pthread_cond_broadcast(&ev->cond);
     }
+    pthread_mutex_unlock(&ev->lock);
 }
 
 static inline void futex_wait(QemuEvent *ev, unsigned val)
@@ -401,7 +396,7 @@
             /*
              * Leave the event reset and tell qemu_event_set that there
              * are waiters.  No need to retry, because there cannot be
-             * a concurent busy->free transition.  After the CAS, the
+             * a concurrent busy->free transition.  After the CAS, the
              * event will be either set or busy.
              */
             if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
@@ -412,6 +407,42 @@
     }
 }
 
+static pthread_key_t exit_key;
+
+union NotifierThreadData {
+    void *ptr;
+    NotifierList list;
+};
+QEMU_BUILD_BUG_ON(sizeof(union NotifierThreadData) != sizeof(void *));
+
+void qemu_thread_atexit_add(Notifier *notifier)
+{
+    union NotifierThreadData ntd;
+    ntd.ptr = pthread_getspecific(exit_key);
+    notifier_list_add(&ntd.list, notifier);
+    pthread_setspecific(exit_key, ntd.ptr);
+}
+
+void qemu_thread_atexit_remove(Notifier *notifier)
+{
+    union NotifierThreadData ntd;
+    ntd.ptr = pthread_getspecific(exit_key);
+    notifier_remove(notifier);
+    pthread_setspecific(exit_key, ntd.ptr);
+}
+
+static void qemu_thread_atexit_run(void *arg)
+{
+    union NotifierThreadData ntd = { .ptr = arg };
+    notifier_list_notify(&ntd.list, NULL);
+}
+
+static void __attribute__((constructor)) qemu_thread_atexit_init(void)
+{
+    pthread_key_create(&exit_key, qemu_thread_atexit_run);
+}
+
+
 /* Attempt to set the threads name; note that this is for debug, so
  * we're not going to fail if we can't set it.
  */
@@ -422,7 +453,6 @@
 #endif
 }
 
-#if defined(CONFIG_ANDROID)
 /* A small data structure to group the thread startup parameters.
  * This is passed to the trampoline function below which will first
  * setup AndroidEmu to use the QEMU-based looper implementation before
@@ -445,13 +475,12 @@
     ThreadStartData data = *(ThreadStartData*)data_;
     free(data_);
 
-    /* Ensure AndroidEmu timers / looper work on this thread. */
-    qemu_looper_setForThread();
+    if (thread_setup_func)
+        (*thread_setup_func)();
 
     /* Start the thread. */
     return (*data.start_routine)(data.arg);
 }
-#endif  // CONFIG_ANDROID
 
 void qemu_thread_create(QemuThread *thread, const char *name,
                        void *(*start_routine)(void*),
@@ -475,7 +504,7 @@
     /* Leave signal handling to the iothread.  */
     sigfillset(&set);
     pthread_sigmask(SIG_SETMASK, &set, &oldset);
-#if defined(CONFIG_ANDROID)
+
     /* Create heap-allocated ThreadStartData object and pass its ownership
      * to the trampoline. */
     ThreadStartData* data = malloc(sizeof(*data));
@@ -487,11 +516,6 @@
         free(data);
         error_exit(err, __func__);
     }
-#else  /* !CONFIG_ANDROID */
-    err = pthread_create(&thread->thread, &attr, start_routine, arg);
-    if (err)
-        error_exit(err, __func__);
-#endif  /* !CONFIG_ANDROID */
 
     if (name_threads) {
         qemu_thread_set_name(thread, name);
diff --git a/util/qemu-thread-win32.c b/util/qemu-thread-win32.c
index 2c6d87e..fe852ef 100644
--- a/util/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -10,19 +10,22 @@
  * See the COPYING file in the top-level directory.
  *
  */
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "qemu/abort.h"
 #include "qemu/thread.h"
+#include "qemu/notify.h"
 #include <process.h>
-#include <assert.h>
-#include <limits.h>
-
-#if defined(CONFIG_ANDROID)
-#include "android-qemu2-glue/looper-qemu.h"
-#include "android/crashreport/crash-handler.h"
-#endif
 
 static bool name_threads;
 
+static QemuThreadSetupFunc thread_setup_func;
+
+void qemu_thread_register_setup_callback(QemuThreadSetupFunc setup_func)
+{
+    thread_setup_func = setup_func;
+}
+
 void qemu_thread_naming(bool enable)
 {
     /* But note we don't actually name them on Windows yet */
@@ -37,15 +40,7 @@
 
     FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER,
                   NULL, err, 0, (LPTSTR)&pstr, 2, NULL);
-    fprintf(stderr, "qemu: %s: %s\n", msg, pstr);
-#ifdef CONFIG_ANDROID
-    crashhandler_die_format(
-                "Internal error in %s: %s (%d)",
-                msg, pstr, err);
-#else
-    LocalFree(pstr);
-    abort();
-#endif
+    qemu_abort("qemu: %s: %s\n", msg, pstr);
 }
 
 void qemu_mutex_init(QemuMutex *mutex)
@@ -248,10 +243,34 @@
     }
 }
 
+/* Wrap a Win32 manual-reset event with a fast userspace path.  The idea
+ * is to reset the Win32 event lazily, as part of a test-reset-test-wait
+ * sequence.  Such a sequence is, indeed, how QemuEvents are used by
+ * RCU and other subsystems!
+ *
+ * Valid transitions:
+ * - free->set, when setting the event
+ * - busy->set, when setting the event, followed by futex_wake
+ * - set->free, when resetting the event
+ * - free->busy, when waiting
+ *
+ * set->busy does not happen (it can be observed from the outside but
+ * it really is set->free->busy).
+ *
+ * busy->free provably cannot happen; to enforce it, the set->free transition
+ * is done with an OR, which becomes a no-op if the event has concurrently
+ * transitioned to free or busy (and is faster than cmpxchg).
+ */
+
+#define EV_SET         0
+#define EV_FREE        1
+#define EV_BUSY       -1
+
 void qemu_event_init(QemuEvent *ev, bool init)
 {
     /* Manual reset.  */
-    ev->event = CreateEvent(NULL, TRUE, init, NULL);
+    ev->event = CreateEvent(NULL, TRUE, TRUE, NULL);
+    ev->value = (init ? EV_SET : EV_FREE);
 }
 
 void qemu_event_destroy(QemuEvent *ev)
@@ -261,17 +280,51 @@
 
 void qemu_event_set(QemuEvent *ev)
 {
-    SetEvent(ev->event);
+    if (atomic_mb_read(&ev->value) != EV_SET) {
+        if (atomic_xchg(&ev->value, EV_SET) == EV_BUSY) {
+            /* There were waiters, wake them up.  */
+            SetEvent(ev->event);
+        }
+    }
 }
 
 void qemu_event_reset(QemuEvent *ev)
 {
-    ResetEvent(ev->event);
+    if (atomic_mb_read(&ev->value) == EV_SET) {
+        /* If there was a concurrent reset (or even reset+wait),
+         * do nothing.  Otherwise change EV_SET->EV_FREE.
+         */
+        atomic_or(&ev->value, EV_FREE);
+    }
 }
 
 void qemu_event_wait(QemuEvent *ev)
 {
-    WaitForSingleObject(ev->event, INFINITE);
+    unsigned value;
+
+    value = atomic_mb_read(&ev->value);
+    if (value != EV_SET) {
+        if (value == EV_FREE) {
+            /* qemu_event_set is not yet going to call SetEvent, but we are
+             * going to do another check for EV_SET below when setting EV_BUSY.
+             * At that point it is safe to call WaitForSingleObject.
+             */
+            ResetEvent(ev->event);
+
+            /* Tell qemu_event_set that there are waiters.  No need to retry
+             * because there cannot be a concurent busy->free transition.
+             * After the CAS, the event will be either set or busy.
+             */
+            if (atomic_cmpxchg(&ev->value, EV_FREE, EV_BUSY) == EV_SET) {
+                value = EV_SET;
+            } else {
+                value = EV_BUSY;
+            }
+        }
+        if (value == EV_BUSY) {
+            WaitForSingleObject(ev->event, INFINITE);
+        }
+    }
 }
 
 struct QemuThreadData {
@@ -279,6 +332,7 @@
     void             *(*start_routine)(void *);
     void             *arg;
     short             mode;
+    NotifierList      exit;
 
     /* Only used for joinable threads. */
     bool              exited;
@@ -286,23 +340,45 @@
     CRITICAL_SECTION  cs;
 };
 
+static bool atexit_registered;
+static NotifierList main_thread_exit;
+
 static __thread QemuThreadData *qemu_thread_data;
 
+static void run_main_thread_exit(void)
+{
+    notifier_list_notify(&main_thread_exit, NULL);
+}
+
+void qemu_thread_atexit_add(Notifier *notifier)
+{
+    if (!qemu_thread_data) {
+        if (!atexit_registered) {
+            atexit_registered = true;
+            atexit(run_main_thread_exit);
+        }
+        notifier_list_add(&main_thread_exit, notifier);
+    } else {
+        notifier_list_add(&qemu_thread_data->exit, notifier);
+    }
+}
+
+void qemu_thread_atexit_remove(Notifier *notifier)
+{
+    notifier_remove(notifier);
+}
+
 static unsigned __stdcall win32_start_routine(void *arg)
 {
     QemuThreadData *data = (QemuThreadData *) arg;
     void *(*start_routine)(void *) = data->start_routine;
     void *thread_arg = data->arg;
 
-    if (data->mode == QEMU_THREAD_DETACHED) {
-        g_free(data);
-        data = NULL;
-    }
     qemu_thread_data = data;
-#if defined(CONFIG_ANDROID)
-    /* Ensure AndroidEmu timers and looper work correctly on this thread. */
-    qemu_looper_setForThread();
-#endif
+
+    if (thread_setup_func)
+        (*thread_setup_func)();
+
     qemu_thread_exit(start_routine(thread_arg));
     abort();
 }
@@ -311,12 +387,14 @@
 {
     QemuThreadData *data = qemu_thread_data;
 
-    if (data) {
-        assert(data->mode != QEMU_THREAD_DETACHED);
+    notifier_list_notify(&data->exit, NULL);
+    if (data->mode == QEMU_THREAD_JOINABLE) {
         data->ret = arg;
         EnterCriticalSection(&data->cs);
         data->exited = true;
         LeaveCriticalSection(&data->cs);
+    } else {
+        g_free(data);
     }
     _endthreadex(0);
 }
@@ -328,9 +406,10 @@
     HANDLE handle;
 
     data = thread->data;
-    if (!data) {
+    if (data->mode == QEMU_THREAD_DETACHED) {
         return NULL;
     }
+
     /*
      * Because multiple copies of the QemuThread can exist via
      * qemu_thread_get_self, we need to store a value that cannot
@@ -344,7 +423,6 @@
         CloseHandle(handle);
     }
     ret = data->ret;
-    assert(data->mode != QEMU_THREAD_DETACHED);
     DeleteCriticalSection(&data->cs);
     g_free(data);
     return ret;
@@ -362,6 +440,7 @@
     data->arg = arg;
     data->mode = mode;
     data->exited = false;
+    notifier_list_init(&data->exit);
 
     if (data->mode != QEMU_THREAD_DETACHED) {
         InitializeCriticalSection(&data->cs);
@@ -373,7 +452,7 @@
         error_exit(GetLastError(), __func__);
     }
     CloseHandle(hThread);
-    thread->data = (mode == QEMU_THREAD_DETACHED) ? NULL : data;
+    thread->data = data;
 }
 
 void qemu_thread_get_self(QemuThread *thread)
@@ -388,11 +467,10 @@
     HANDLE handle;
 
     data = thread->data;
-    if (!data) {
+    if (data->mode == QEMU_THREAD_DETACHED) {
         return NULL;
     }
 
-    assert(data->mode != QEMU_THREAD_DETACHED);
     EnterCriticalSection(&data->cs);
     if (!data->exited) {
         handle = OpenThread(SYNCHRONIZE | THREAD_SUSPEND_RESUME, FALSE,
diff --git a/util/qemu-timer-common.c b/util/qemu-timer-common.c
index 95e0847..06d084d 100644
--- a/util/qemu-timer-common.c
+++ b/util/qemu-timer-common.c
@@ -21,6 +21,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qemu/osdep.h"
 #include "qemu/timer.h"
 
 /***********************************************************/
diff --git a/util/qht.c b/util/qht.c
new file mode 100644
index 0000000..16a8d79
--- /dev/null
+++ b/util/qht.c
@@ -0,0 +1,844 @@
+/*
+ * qht.c - QEMU Hash Table, designed to scale for read-mostly workloads.
+ *
+ * Copyright (C) 2016, Emilio G. Cota <cota@braap.org>
+ *
+ * License: GNU GPL, version 2 or later.
+ *   See the COPYING file in the top-level directory.
+ *
+ * Assumptions:
+ * - NULL cannot be inserted/removed as a pointer value.
+ * - Trying to insert an already-existing hash-pointer pair is OK. However,
+ *   it is not OK to insert into the same hash table different hash-pointer
+ *   pairs that have the same pointer value, but not the hashes.
+ * - Lookups are performed under an RCU read-critical section; removals
+ *   must wait for a grace period to elapse before freeing removed objects.
+ *
+ * Features:
+ * - Reads (i.e. lookups and iterators) can be concurrent with other reads.
+ *   Lookups that are concurrent with writes to the same bucket will retry
+ *   via a seqlock; iterators acquire all bucket locks and therefore can be
+ *   concurrent with lookups and are serialized wrt writers.
+ * - Writes (i.e. insertions/removals) can be concurrent with writes to
+ *   different buckets; writes to the same bucket are serialized through a lock.
+ * - Optional auto-resizing: the hash table resizes up if the load surpasses
+ *   a certain threshold. Resizing is done concurrently with readers; writes
+ *   are serialized with the resize operation.
+ *
+ * The key structure is the bucket, which is cacheline-sized. Buckets
+ * contain a few hash values and pointers; the u32 hash values are stored in
+ * full so that resizing is fast. Having this structure instead of directly
+ * chaining items has two advantages:
+ * - Failed lookups fail fast, and touch a minimum number of cache lines.
+ * - Resizing the hash table with concurrent lookups is easy.
+ *
+ * There are two types of buckets:
+ * 1. "head" buckets are the ones allocated in the array of buckets in qht_map.
+ * 2. all "non-head" buckets (i.e. all others) are members of a chain that
+ *    starts from a head bucket.
+ * Note that the seqlock and spinlock of a head bucket applies to all buckets
+ * chained to it; these two fields are unused in non-head buckets.
+ *
+ * On removals, we move the last valid item in the chain to the position of the
+ * just-removed entry. This makes lookups slightly faster, since the moment an
+ * invalid entry is found, the (failed) lookup is over.
+ *
+ * Resizing is done by taking all bucket spinlocks (so that no other writers can
+ * race with us) and then copying all entries into a new hash map. Then, the
+ * ht->map pointer is set, and the old map is freed once no RCU readers can see
+ * it anymore.
+ *
+ * Writers check for concurrent resizes by comparing ht->map before and after
+ * acquiring their bucket lock. If they don't match, a resize has occured
+ * while the bucket spinlock was being acquired.
+ *
+ * Related Work:
+ * - Idea of cacheline-sized buckets with full hashes taken from:
+ *   David, Guerraoui & Trigonakis, "Asynchronized Concurrency:
+ *   The Secret to Scaling Concurrent Search Data Structures", ASPLOS'15.
+ * - Why not RCU-based hash tables? They would allow us to get rid of the
+ *   seqlock, but resizing would take forever since RCU read critical
+ *   sections in QEMU take quite a long time.
+ *   More info on relativistic hash tables:
+ *   + Triplett, McKenney & Walpole, "Resizable, Scalable, Concurrent Hash
+ *     Tables via Relativistic Programming", USENIX ATC'11.
+ *   + Corbet, "Relativistic hash tables, part 1: Algorithms", @ lwn.net, 2014.
+ *     https://lwn.net/Articles/612021/
+ */
+#include "qemu/osdep.h"
+#include "qemu/qht.h"
+#include "qemu/atomic.h"
+#include "qemu/rcu.h"
+
+//#define QHT_DEBUG
+
+/*
+ * We want to avoid false sharing of cache lines. Most systems have 64-byte
+ * cache lines so we go with it for simplicity.
+ *
+ * Note that systems with smaller cache lines will be fine (the struct is
+ * almost 64-bytes); systems with larger cache lines might suffer from
+ * some false sharing.
+ */
+#define QHT_BUCKET_ALIGN 64
+
+/* define these to keep sizeof(qht_bucket) within QHT_BUCKET_ALIGN */
+#if HOST_LONG_BITS == 32
+#define QHT_BUCKET_ENTRIES 6
+#else /* 64-bit */
+#define QHT_BUCKET_ENTRIES 4
+#endif
+
+/*
+ * Note: reading partially-updated pointers in @pointers could lead to
+ * segfaults. We thus access them with atomic_read/set; this guarantees
+ * that the compiler makes all those accesses atomic. We also need the
+ * volatile-like behavior in atomic_read, since otherwise the compiler
+ * might refetch the pointer.
+ * atomic_read's are of course not necessary when the bucket lock is held.
+ *
+ * If both ht->lock and b->lock are grabbed, ht->lock should always
+ * be grabbed first.
+ */
+struct qht_bucket {
+    QemuSpin lock;
+    QemuSeqLock sequence;
+    uint32_t hashes[QHT_BUCKET_ENTRIES];
+    void *pointers[QHT_BUCKET_ENTRIES];
+    struct qht_bucket *next;
+} QEMU_ALIGNED(QHT_BUCKET_ALIGN);
+
+QEMU_BUILD_BUG_ON(sizeof(struct qht_bucket) > QHT_BUCKET_ALIGN);
+
+/**
+ * struct qht_map - structure to track an array of buckets
+ * @rcu: used by RCU. Keep it as the top field in the struct to help valgrind
+ *       find the whole struct.
+ * @buckets: array of head buckets. It is constant once the map is created.
+ * @n_buckets: number of head buckets. It is constant once the map is created.
+ * @n_added_buckets: number of added (i.e. "non-head") buckets
+ * @n_added_buckets_threshold: threshold to trigger an upward resize once the
+ *                             number of added buckets surpasses it.
+ *
+ * Buckets are tracked in what we call a "map", i.e. this structure.
+ */
+struct qht_map {
+    struct rcu_head rcu;
+    struct qht_bucket *buckets;
+    size_t n_buckets;
+    size_t n_added_buckets;
+    size_t n_added_buckets_threshold;
+};
+
+/* trigger a resize when n_added_buckets > n_buckets / div */
+#define QHT_NR_ADDED_BUCKETS_THRESHOLD_DIV 8
+
+static void qht_do_resize(struct qht *ht, struct qht_map *new);
+static void qht_grow_maybe(struct qht *ht);
+
+#ifdef QHT_DEBUG
+
+#define qht_debug_assert(X) do { assert(X); } while (0)
+
+static void qht_bucket_debug__locked(struct qht_bucket *b)
+{
+    bool seen_empty = false;
+    bool corrupt = false;
+    int i;
+
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            if (b->pointers[i] == NULL) {
+                seen_empty = true;
+                continue;
+            }
+            if (seen_empty) {
+                fprintf(stderr, "%s: b: %p, pos: %i, hash: 0x%x, p: %p\n",
+                        __func__, b, i, b->hashes[i], b->pointers[i]);
+                corrupt = true;
+            }
+        }
+        b = b->next;
+    } while (b);
+    qht_debug_assert(!corrupt);
+}
+
+static void qht_map_debug__all_locked(struct qht_map *map)
+{
+    int i;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        qht_bucket_debug__locked(&map->buckets[i]);
+    }
+}
+#else
+
+#define qht_debug_assert(X) do { (void)(X); } while (0)
+
+static inline void qht_bucket_debug__locked(struct qht_bucket *b)
+{ }
+
+static inline void qht_map_debug__all_locked(struct qht_map *map)
+{ }
+#endif /* QHT_DEBUG */
+
+static inline size_t qht_elems_to_buckets(size_t n_elems)
+{
+    return pow2ceil(n_elems / QHT_BUCKET_ENTRIES);
+}
+
+static inline void qht_head_init(struct qht_bucket *b)
+{
+    memset(b, 0, sizeof(*b));
+    qemu_spin_init(&b->lock);
+    seqlock_init(&b->sequence);
+}
+
+static inline
+struct qht_bucket *qht_map_to_bucket(struct qht_map *map, uint32_t hash)
+{
+    return &map->buckets[hash & (map->n_buckets - 1)];
+}
+
+/* acquire all bucket locks from a map */
+static void qht_map_lock_buckets(struct qht_map *map)
+{
+    size_t i;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        struct qht_bucket *b = &map->buckets[i];
+
+        qemu_spin_lock(&b->lock);
+    }
+}
+
+static void qht_map_unlock_buckets(struct qht_map *map)
+{
+    size_t i;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        struct qht_bucket *b = &map->buckets[i];
+
+        qemu_spin_unlock(&b->lock);
+    }
+}
+
+/*
+ * Call with at least a bucket lock held.
+ * @map should be the value read before acquiring the lock (or locks).
+ */
+static inline bool qht_map_is_stale__locked(struct qht *ht, struct qht_map *map)
+{
+    return map != ht->map;
+}
+
+/*
+ * Grab all bucket locks, and set @pmap after making sure the map isn't stale.
+ *
+ * Pairs with qht_map_unlock_buckets(), hence the pass-by-reference.
+ *
+ * Note: callers cannot have ht->lock held.
+ */
+static inline
+void qht_map_lock_buckets__no_stale(struct qht *ht, struct qht_map **pmap)
+{
+    struct qht_map *map;
+
+    map = atomic_rcu_read(&ht->map);
+    qht_map_lock_buckets(map);
+    if (likely(!qht_map_is_stale__locked(ht, map))) {
+        *pmap = map;
+        return;
+    }
+    qht_map_unlock_buckets(map);
+
+    /* we raced with a resize; acquire ht->lock to see the updated ht->map */
+    qemu_mutex_lock(&ht->lock);
+    map = ht->map;
+    qht_map_lock_buckets(map);
+    qemu_mutex_unlock(&ht->lock);
+    *pmap = map;
+    return;
+}
+
+/*
+ * Get a head bucket and lock it, making sure its parent map is not stale.
+ * @pmap is filled with a pointer to the bucket's parent map.
+ *
+ * Unlock with qemu_spin_unlock(&b->lock).
+ *
+ * Note: callers cannot have ht->lock held.
+ */
+static inline
+struct qht_bucket *qht_bucket_lock__no_stale(struct qht *ht, uint32_t hash,
+                                             struct qht_map **pmap)
+{
+    struct qht_bucket *b;
+    struct qht_map *map;
+
+    map = atomic_rcu_read(&ht->map);
+    b = qht_map_to_bucket(map, hash);
+
+    qemu_spin_lock(&b->lock);
+    if (likely(!qht_map_is_stale__locked(ht, map))) {
+        *pmap = map;
+        return b;
+    }
+    qemu_spin_unlock(&b->lock);
+
+    /* we raced with a resize; acquire ht->lock to see the updated ht->map */
+    qemu_mutex_lock(&ht->lock);
+    map = ht->map;
+    b = qht_map_to_bucket(map, hash);
+    qemu_spin_lock(&b->lock);
+    qemu_mutex_unlock(&ht->lock);
+    *pmap = map;
+    return b;
+}
+
+static inline bool qht_map_needs_resize(struct qht_map *map)
+{
+    return atomic_read(&map->n_added_buckets) > map->n_added_buckets_threshold;
+}
+
+static inline void qht_chain_destroy(struct qht_bucket *head)
+{
+    struct qht_bucket *curr = head->next;
+    struct qht_bucket *prev;
+
+    while (curr) {
+        prev = curr;
+        curr = curr->next;
+        qemu_vfree(prev);
+    }
+}
+
+/* pass only an orphan map */
+static void qht_map_destroy(struct qht_map *map)
+{
+    size_t i;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        qht_chain_destroy(&map->buckets[i]);
+    }
+    qemu_vfree(map->buckets);
+    g_free(map);
+}
+
+static struct qht_map *qht_map_create(size_t n_buckets)
+{
+    struct qht_map *map;
+    size_t i;
+
+    map = g_malloc(sizeof(*map));
+    map->n_buckets = n_buckets;
+
+    map->n_added_buckets = 0;
+    map->n_added_buckets_threshold = n_buckets /
+        QHT_NR_ADDED_BUCKETS_THRESHOLD_DIV;
+
+    /* let tiny hash tables to at least add one non-head bucket */
+    if (unlikely(map->n_added_buckets_threshold == 0)) {
+        map->n_added_buckets_threshold = 1;
+    }
+
+    map->buckets = qemu_memalign(QHT_BUCKET_ALIGN,
+                                 sizeof(*map->buckets) * n_buckets);
+    for (i = 0; i < n_buckets; i++) {
+        qht_head_init(&map->buckets[i]);
+    }
+    return map;
+}
+
+void qht_init(struct qht *ht, size_t n_elems, unsigned int mode)
+{
+    struct qht_map *map;
+    size_t n_buckets = qht_elems_to_buckets(n_elems);
+
+    ht->mode = mode;
+    qemu_mutex_init(&ht->lock);
+    map = qht_map_create(n_buckets);
+    atomic_rcu_set(&ht->map, map);
+}
+
+/* call only when there are no readers/writers left */
+void qht_destroy(struct qht *ht)
+{
+    qht_map_destroy(ht->map);
+    memset(ht, 0, sizeof(*ht));
+}
+
+static void qht_bucket_reset__locked(struct qht_bucket *head)
+{
+    struct qht_bucket *b = head;
+    int i;
+
+    seqlock_write_begin(&head->sequence);
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            if (b->pointers[i] == NULL) {
+                goto done;
+            }
+            b->hashes[i] = 0;
+            atomic_set(&b->pointers[i], NULL);
+        }
+        b = b->next;
+    } while (b);
+ done:
+    seqlock_write_end(&head->sequence);
+}
+
+/* call with all bucket locks held */
+static void qht_map_reset__all_locked(struct qht_map *map)
+{
+    size_t i;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        qht_bucket_reset__locked(&map->buckets[i]);
+    }
+    qht_map_debug__all_locked(map);
+}
+
+void qht_reset(struct qht *ht)
+{
+    struct qht_map *map;
+
+    qht_map_lock_buckets__no_stale(ht, &map);
+    qht_map_reset__all_locked(map);
+    qht_map_unlock_buckets(map);
+}
+
+bool qht_reset_size(struct qht *ht, size_t n_elems)
+{
+    struct qht_map *new;
+    struct qht_map *map;
+    size_t n_buckets;
+    bool resize = false;
+
+    n_buckets = qht_elems_to_buckets(n_elems);
+
+    qemu_mutex_lock(&ht->lock);
+    map = ht->map;
+    if (n_buckets != map->n_buckets) {
+        new = qht_map_create(n_buckets);
+        resize = true;
+    }
+
+    qht_map_lock_buckets(map);
+    qht_map_reset__all_locked(map);
+    if (resize) {
+        qht_do_resize(ht, new);
+    }
+    qht_map_unlock_buckets(map);
+    qemu_mutex_unlock(&ht->lock);
+
+    return resize;
+}
+
+static inline
+void *qht_do_lookup(struct qht_bucket *head, qht_lookup_func_t func,
+                    const void *userp, uint32_t hash)
+{
+    struct qht_bucket *b = head;
+    int i;
+
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            if (b->hashes[i] == hash) {
+                /* The pointer is dereferenced before seqlock_read_retry,
+                 * so (unlike qht_insert__locked) we need to use
+                 * atomic_rcu_read here.
+                 */
+                void *p = atomic_rcu_read(&b->pointers[i]);
+
+                if (likely(p) && likely(func(p, userp))) {
+                    return p;
+                }
+            }
+        }
+        b = atomic_rcu_read(&b->next);
+    } while (b);
+
+    return NULL;
+}
+
+static __attribute__((noinline))
+void *qht_lookup__slowpath(struct qht_bucket *b, qht_lookup_func_t func,
+                           const void *userp, uint32_t hash)
+{
+    unsigned int version;
+    void *ret;
+
+    do {
+        version = seqlock_read_begin(&b->sequence);
+        ret = qht_do_lookup(b, func, userp, hash);
+    } while (seqlock_read_retry(&b->sequence, version));
+    return ret;
+}
+
+void *qht_lookup(struct qht *ht, qht_lookup_func_t func, const void *userp,
+                 uint32_t hash)
+{
+    struct qht_bucket *b;
+    struct qht_map *map;
+    unsigned int version;
+    void *ret;
+
+    map = atomic_rcu_read(&ht->map);
+    b = qht_map_to_bucket(map, hash);
+
+    version = seqlock_read_begin(&b->sequence);
+    ret = qht_do_lookup(b, func, userp, hash);
+    if (likely(!seqlock_read_retry(&b->sequence, version))) {
+        return ret;
+    }
+    /*
+     * Removing the do/while from the fastpath gives a 4% perf. increase when
+     * running a 100%-lookup microbenchmark.
+     */
+    return qht_lookup__slowpath(b, func, userp, hash);
+}
+
+/* call with head->lock held */
+static bool qht_insert__locked(struct qht *ht, struct qht_map *map,
+                               struct qht_bucket *head, void *p, uint32_t hash,
+                               bool *needs_resize)
+{
+    struct qht_bucket *b = head;
+    struct qht_bucket *prev = NULL;
+    struct qht_bucket *new = NULL;
+    int i;
+
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            if (b->pointers[i]) {
+                if (unlikely(b->pointers[i] == p)) {
+                    return false;
+                }
+            } else {
+                goto found;
+            }
+        }
+        prev = b;
+        b = b->next;
+    } while (b);
+
+    b = qemu_memalign(QHT_BUCKET_ALIGN, sizeof(*b));
+    memset(b, 0, sizeof(*b));
+    new = b;
+    i = 0;
+    atomic_inc(&map->n_added_buckets);
+    if (unlikely(qht_map_needs_resize(map)) && needs_resize) {
+        *needs_resize = true;
+    }
+
+ found:
+    /* found an empty key: acquire the seqlock and write */
+    seqlock_write_begin(&head->sequence);
+    if (new) {
+        atomic_rcu_set(&prev->next, b);
+    }
+    b->hashes[i] = hash;
+    /* smp_wmb() implicit in seqlock_write_begin.  */
+    atomic_set(&b->pointers[i], p);
+    seqlock_write_end(&head->sequence);
+    return true;
+}
+
+static __attribute__((noinline)) void qht_grow_maybe(struct qht *ht)
+{
+    struct qht_map *map;
+
+    /*
+     * If the lock is taken it probably means there's an ongoing resize,
+     * so bail out.
+     */
+    if (qemu_mutex_trylock(&ht->lock)) {
+        return;
+    }
+    map = ht->map;
+    /* another thread might have just performed the resize we were after */
+    if (qht_map_needs_resize(map)) {
+        struct qht_map *new = qht_map_create(map->n_buckets * 2);
+
+        qht_map_lock_buckets(map);
+        qht_do_resize(ht, new);
+        qht_map_unlock_buckets(map);
+    }
+    qemu_mutex_unlock(&ht->lock);
+}
+
+bool qht_insert(struct qht *ht, void *p, uint32_t hash)
+{
+    struct qht_bucket *b;
+    struct qht_map *map;
+    bool needs_resize = false;
+    bool ret;
+
+    /* NULL pointers are not supported */
+    qht_debug_assert(p);
+
+    b = qht_bucket_lock__no_stale(ht, hash, &map);
+    ret = qht_insert__locked(ht, map, b, p, hash, &needs_resize);
+    qht_bucket_debug__locked(b);
+    qemu_spin_unlock(&b->lock);
+
+    if (unlikely(needs_resize) && ht->mode & QHT_MODE_AUTO_RESIZE) {
+        qht_grow_maybe(ht);
+    }
+    return ret;
+}
+
+static inline bool qht_entry_is_last(struct qht_bucket *b, int pos)
+{
+    if (pos == QHT_BUCKET_ENTRIES - 1) {
+        if (b->next == NULL) {
+            return true;
+        }
+        return b->next->pointers[0] == NULL;
+    }
+    return b->pointers[pos + 1] == NULL;
+}
+
+static void
+qht_entry_move(struct qht_bucket *to, int i, struct qht_bucket *from, int j)
+{
+    qht_debug_assert(!(to == from && i == j));
+    qht_debug_assert(to->pointers[i]);
+    qht_debug_assert(from->pointers[j]);
+
+    to->hashes[i] = from->hashes[j];
+    atomic_set(&to->pointers[i], from->pointers[j]);
+
+    from->hashes[j] = 0;
+    atomic_set(&from->pointers[j], NULL);
+}
+
+/*
+ * Find the last valid entry in @head, and swap it with @orig[pos], which has
+ * just been invalidated.
+ */
+static inline void qht_bucket_remove_entry(struct qht_bucket *orig, int pos)
+{
+    struct qht_bucket *b = orig;
+    struct qht_bucket *prev = NULL;
+    int i;
+
+    if (qht_entry_is_last(orig, pos)) {
+        orig->hashes[pos] = 0;
+        atomic_set(&orig->pointers[pos], NULL);
+        return;
+    }
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            if (b->pointers[i]) {
+                continue;
+            }
+            if (i > 0) {
+                return qht_entry_move(orig, pos, b, i - 1);
+            }
+            qht_debug_assert(prev);
+            return qht_entry_move(orig, pos, prev, QHT_BUCKET_ENTRIES - 1);
+        }
+        prev = b;
+        b = b->next;
+    } while (b);
+    /* no free entries other than orig[pos], so swap it with the last one */
+    qht_entry_move(orig, pos, prev, QHT_BUCKET_ENTRIES - 1);
+}
+
+/* call with b->lock held */
+static inline
+bool qht_remove__locked(struct qht_map *map, struct qht_bucket *head,
+                        const void *p, uint32_t hash)
+{
+    struct qht_bucket *b = head;
+    int i;
+
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            void *q = b->pointers[i];
+
+            if (unlikely(q == NULL)) {
+                return false;
+            }
+            if (q == p) {
+                qht_debug_assert(b->hashes[i] == hash);
+                seqlock_write_begin(&head->sequence);
+                qht_bucket_remove_entry(b, i);
+                seqlock_write_end(&head->sequence);
+                return true;
+            }
+        }
+        b = b->next;
+    } while (b);
+    return false;
+}
+
+bool qht_remove(struct qht *ht, const void *p, uint32_t hash)
+{
+    struct qht_bucket *b;
+    struct qht_map *map;
+    bool ret;
+
+    /* NULL pointers are not supported */
+    qht_debug_assert(p);
+
+    b = qht_bucket_lock__no_stale(ht, hash, &map);
+    ret = qht_remove__locked(map, b, p, hash);
+    qht_bucket_debug__locked(b);
+    qemu_spin_unlock(&b->lock);
+    return ret;
+}
+
+static inline void qht_bucket_iter(struct qht *ht, struct qht_bucket *b,
+                                   qht_iter_func_t func, void *userp)
+{
+    int i;
+
+    do {
+        for (i = 0; i < QHT_BUCKET_ENTRIES; i++) {
+            if (b->pointers[i] == NULL) {
+                return;
+            }
+            func(ht, b->pointers[i], b->hashes[i], userp);
+        }
+        b = b->next;
+    } while (b);
+}
+
+/* call with all of the map's locks held */
+static inline void qht_map_iter__all_locked(struct qht *ht, struct qht_map *map,
+                                            qht_iter_func_t func, void *userp)
+{
+    size_t i;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        qht_bucket_iter(ht, &map->buckets[i], func, userp);
+    }
+}
+
+void qht_iter(struct qht *ht, qht_iter_func_t func, void *userp)
+{
+    struct qht_map *map;
+
+    map = atomic_rcu_read(&ht->map);
+    qht_map_lock_buckets(map);
+    /* Note: ht here is merely for carrying ht->mode; ht->map won't be read */
+    qht_map_iter__all_locked(ht, map, func, userp);
+    qht_map_unlock_buckets(map);
+}
+
+static void qht_map_copy(struct qht *ht, void *p, uint32_t hash, void *userp)
+{
+    struct qht_map *new = userp;
+    struct qht_bucket *b = qht_map_to_bucket(new, hash);
+
+    /* no need to acquire b->lock because no thread has seen this map yet */
+    qht_insert__locked(ht, new, b, p, hash, NULL);
+}
+
+/*
+ * Call with ht->lock and all bucket locks held.
+ *
+ * Creating the @new map here would add unnecessary delay while all the locks
+ * are held--holding up the bucket locks is particularly bad, since no writes
+ * can occur while these are held. Thus, we let callers create the new map,
+ * hopefully without the bucket locks held.
+ */
+static void qht_do_resize(struct qht *ht, struct qht_map *new)
+{
+    struct qht_map *old;
+
+    old = ht->map;
+    g_assert_cmpuint(new->n_buckets, !=, old->n_buckets);
+
+    qht_map_iter__all_locked(ht, old, qht_map_copy, new);
+    qht_map_debug__all_locked(new);
+
+    atomic_rcu_set(&ht->map, new);
+    call_rcu(old, qht_map_destroy, rcu);
+}
+
+bool qht_resize(struct qht *ht, size_t n_elems)
+{
+    size_t n_buckets = qht_elems_to_buckets(n_elems);
+    size_t ret = false;
+
+    qemu_mutex_lock(&ht->lock);
+    if (n_buckets != ht->map->n_buckets) {
+        struct qht_map *new;
+        struct qht_map *old = ht->map;
+
+        new = qht_map_create(n_buckets);
+        qht_map_lock_buckets(old);
+        qht_do_resize(ht, new);
+        qht_map_unlock_buckets(old);
+        ret = true;
+    }
+    qemu_mutex_unlock(&ht->lock);
+
+    return ret;
+}
+
+/* pass @stats to qht_statistics_destroy() when done */
+void qht_statistics_init(struct qht *ht, struct qht_stats *stats)
+{
+    struct qht_map *map;
+    int i;
+
+    map = atomic_rcu_read(&ht->map);
+
+    stats->used_head_buckets = 0;
+    stats->entries = 0;
+    qdist_init(&stats->chain);
+    qdist_init(&stats->occupancy);
+    /* bail out if the qht has not yet been initialized */
+    if (unlikely(map == NULL)) {
+        stats->head_buckets = 0;
+        return;
+    }
+    stats->head_buckets = map->n_buckets;
+
+    for (i = 0; i < map->n_buckets; i++) {
+        struct qht_bucket *head = &map->buckets[i];
+        struct qht_bucket *b;
+        unsigned int version;
+        size_t buckets;
+        size_t entries;
+        int j;
+
+        do {
+            version = seqlock_read_begin(&head->sequence);
+            buckets = 0;
+            entries = 0;
+            b = head;
+            do {
+                for (j = 0; j < QHT_BUCKET_ENTRIES; j++) {
+                    if (atomic_read(&b->pointers[j]) == NULL) {
+                        break;
+                    }
+                    entries++;
+                }
+                buckets++;
+                b = atomic_rcu_read(&b->next);
+            } while (b);
+        } while (seqlock_read_retry(&head->sequence, version));
+
+        if (entries) {
+            qdist_inc(&stats->chain, buckets);
+            qdist_inc(&stats->occupancy,
+                      (double)entries / QHT_BUCKET_ENTRIES / buckets);
+            stats->used_head_buckets++;
+            stats->entries += entries;
+        } else {
+            qdist_inc(&stats->occupancy, 0);
+        }
+    }
+}
+
+void qht_statistics_destroy(struct qht_stats *stats)
+{
+    qdist_destroy(&stats->occupancy);
+    qdist_destroy(&stats->chain);
+}
diff --git a/util/range.c b/util/range.c
new file mode 100644
index 0000000..416df7c
--- /dev/null
+++ b/util/range.c
@@ -0,0 +1,73 @@
+/*
+ * QEMU 64-bit address ranges
+ *
+ * Copyright (c) 2015-2016 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu/range.h"
+
+/*
+ * Return -1 if @a < @b, 1 @a > @b, and 0 if they touch or overlap.
+ * Both @a and @b must not be empty.
+ */
+static inline int range_compare(Range *a, Range *b)
+{
+    assert(!range_is_empty(a) && !range_is_empty(b));
+
+    /* Careful, avoid wraparound */
+    if (b->lob && b->lob - 1 > a->upb) {
+        return -1;
+    }
+    if (a->lob && a->lob - 1 > b->upb) {
+        return 1;
+    }
+    return 0;
+}
+
+/* Insert @data into @list of ranges; caller no longer owns @data */
+GList *range_list_insert(GList *list, Range *data)
+{
+    GList *l;
+
+    assert(!range_is_empty(data));
+
+    /* Skip all list elements strictly less than data */
+    for (l = list; l && range_compare(l->data, data) < 0; l = l->next) {
+    }
+
+    if (!l || range_compare(l->data, data) > 0) {
+        /* Rest of the list (if any) is strictly greater than @data */
+        return g_list_insert_before(list, l, data);
+    }
+
+    /* Current list element overlaps @data, merge the two */
+    range_extend(l->data, data);
+    g_free(data);
+
+    /* Merge any subsequent list elements that now also overlap */
+    while (l->next && range_compare(l->data, l->next->data) == 0) {
+        GList *new_l;
+
+        range_extend(l->data, l->next->data);
+        g_free(l->next->data);
+        new_l = g_list_delete_link(list, l->next);
+        assert(new_l == list);
+    }
+
+    return list;
+}
diff --git a/util/rcu.c b/util/rcu.c
new file mode 100644
index 0000000..bceb3e4
--- /dev/null
+++ b/util/rcu.c
@@ -0,0 +1,348 @@
+/*
+ * urcu-mb.c
+ *
+ * Userspace RCU library with explicit memory barriers
+ *
+ * Copyright (c) 2009 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * Copyright (c) 2009 Paul E. McKenney, IBM Corporation.
+ * Copyright 2015 Red Hat, Inc.
+ *
+ * Ported to QEMU by Paolo Bonzini  <pbonzini@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * IBM's contributions to this file may be relicensed under LGPLv2 or later.
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+#include "qemu/rcu.h"
+#include "qemu/atomic.h"
+#include "qemu/thread.h"
+#include "qemu/main-loop.h"
+
+/*
+ * Global grace period counter.  Bit 0 is always one in rcu_gp_ctr.
+ * Bits 1 and above are defined in synchronize_rcu.
+ */
+#define RCU_GP_LOCKED           (1UL << 0)
+#define RCU_GP_CTR              (1UL << 1)
+
+unsigned long rcu_gp_ctr = RCU_GP_LOCKED;
+
+QemuEvent rcu_gp_event;
+static QemuMutex rcu_registry_lock;
+static QemuMutex rcu_sync_lock;
+
+/*
+ * Check whether a quiescent state was crossed between the beginning of
+ * update_counter_and_wait and now.
+ */
+static inline int rcu_gp_ongoing(unsigned long *ctr)
+{
+    unsigned long v;
+
+    v = atomic_read(ctr);
+    return v && (v != rcu_gp_ctr);
+}
+
+/* Written to only by each individual reader. Read by both the reader and the
+ * writers.
+ */
+__thread struct rcu_reader_data rcu_reader;
+
+/* Protected by rcu_registry_lock.  */
+typedef QLIST_HEAD(, rcu_reader_data) ThreadList;
+static ThreadList registry = QLIST_HEAD_INITIALIZER(registry);
+
+/* Wait for previous parity/grace period to be empty of readers.  */
+static void wait_for_readers(void)
+{
+    ThreadList qsreaders = QLIST_HEAD_INITIALIZER(qsreaders);
+    struct rcu_reader_data *index, *tmp;
+
+    for (;;) {
+        /* We want to be notified of changes made to rcu_gp_ongoing
+         * while we walk the list.
+         */
+        qemu_event_reset(&rcu_gp_event);
+
+        /* Instead of using atomic_mb_set for index->waiting, and
+         * atomic_mb_read for index->ctr, memory barriers are placed
+         * manually since writes to different threads are independent.
+         * atomic_mb_set has a smp_wmb before...
+         */
+        smp_wmb();
+        QLIST_FOREACH(index, &registry, node) {
+            atomic_set(&index->waiting, true);
+        }
+
+        /* ... and a smp_mb after.  */
+        smp_mb();
+
+        QLIST_FOREACH_SAFE(index, &registry, node, tmp) {
+            if (!rcu_gp_ongoing(&index->ctr)) {
+                QLIST_REMOVE(index, node);
+                QLIST_INSERT_HEAD(&qsreaders, index, node);
+
+                /* No need for mb_set here, worst of all we
+                 * get some extra futex wakeups.
+                 */
+                atomic_set(&index->waiting, false);
+            }
+        }
+
+        /* atomic_mb_read has smp_rmb after.  */
+        smp_rmb();
+
+        if (QLIST_EMPTY(&registry)) {
+            break;
+        }
+
+        /* Wait for one thread to report a quiescent state and try again.
+         * Release rcu_registry_lock, so rcu_(un)register_thread() doesn't
+         * wait too much time.
+         *
+         * rcu_register_thread() may add nodes to &registry; it will not
+         * wake up synchronize_rcu, but that is okay because at least another
+         * thread must exit its RCU read-side critical section before
+         * synchronize_rcu is done.  The next iteration of the loop will
+         * move the new thread's rcu_reader from &registry to &qsreaders,
+         * because rcu_gp_ongoing() will return false.
+         *
+         * rcu_unregister_thread() may remove nodes from &qsreaders instead
+         * of &registry if it runs during qemu_event_wait.  That's okay;
+         * the node then will not be added back to &registry by QLIST_SWAP
+         * below.  The invariant is that the node is part of one list when
+         * rcu_registry_lock is released.
+         */
+        qemu_mutex_unlock(&rcu_registry_lock);
+        qemu_event_wait(&rcu_gp_event);
+        qemu_mutex_lock(&rcu_registry_lock);
+    }
+
+    /* put back the reader list in the registry */
+    QLIST_SWAP(&registry, &qsreaders, node);
+}
+
+void synchronize_rcu(void)
+{
+    qemu_mutex_lock(&rcu_sync_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
+
+    if (!QLIST_EMPTY(&registry)) {
+        /* In either case, the atomic_mb_set below blocks stores that free
+         * old RCU-protected pointers.
+         */
+        if (sizeof(rcu_gp_ctr) < 8) {
+            /* For architectures with 32-bit longs, a two-subphases algorithm
+             * ensures we do not encounter overflow bugs.
+             *
+             * Switch parity: 0 -> 1, 1 -> 0.
+             */
+            atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
+            wait_for_readers();
+            atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
+        } else {
+            /* Increment current grace period.  */
+            atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
+        }
+
+        wait_for_readers();
+    }
+
+    qemu_mutex_unlock(&rcu_registry_lock);
+    qemu_mutex_unlock(&rcu_sync_lock);
+}
+
+
+#define RCU_CALL_MIN_SIZE        30
+
+/* Multi-producer, single-consumer queue based on urcu/static/wfqueue.h
+ * from liburcu.  Note that head is only used by the consumer.
+ */
+static struct rcu_head dummy;
+static struct rcu_head *head = &dummy, **tail = &dummy.next;
+static int rcu_call_count;
+static QemuEvent rcu_call_ready_event;
+
+static void enqueue(struct rcu_head *node)
+{
+    struct rcu_head **old_tail;
+
+    node->next = NULL;
+    old_tail = atomic_xchg(&tail, &node->next);
+    atomic_mb_set(old_tail, node);
+}
+
+static struct rcu_head *try_dequeue(void)
+{
+    struct rcu_head *node, *next;
+
+retry:
+    /* Test for an empty list, which we do not expect.  Note that for
+     * the consumer head and tail are always consistent.  The head
+     * is consistent because only the consumer reads/writes it.
+     * The tail, because it is the first step in the enqueuing.
+     * It is only the next pointers that might be inconsistent.
+     */
+    if (head == &dummy && atomic_mb_read(&tail) == &dummy.next) {
+        abort();
+    }
+
+    /* If the head node has NULL in its next pointer, the value is
+     * wrong and we need to wait until its enqueuer finishes the update.
+     */
+    node = head;
+    next = atomic_mb_read(&head->next);
+    if (!next) {
+        return NULL;
+    }
+
+    /* Since we are the sole consumer, and we excluded the empty case
+     * above, the queue will always have at least two nodes: the
+     * dummy node, and the one being removed.  So we do not need to update
+     * the tail pointer.
+     */
+    head = next;
+
+    /* If we dequeued the dummy node, add it back at the end and retry.  */
+    if (node == &dummy) {
+        enqueue(node);
+        goto retry;
+    }
+
+    return node;
+}
+
+static void *call_rcu_thread(void *opaque)
+{
+    struct rcu_head *node;
+
+    rcu_register_thread();
+
+    for (;;) {
+        int tries = 0;
+        int n = atomic_read(&rcu_call_count);
+
+        /* Heuristically wait for a decent number of callbacks to pile up.
+         * Fetch rcu_call_count now, we only must process elements that were
+         * added before synchronize_rcu() starts.
+         */
+        while (n == 0 || (n < RCU_CALL_MIN_SIZE && ++tries <= 5)) {
+            g_usleep(10000);
+            if (n == 0) {
+                qemu_event_reset(&rcu_call_ready_event);
+                n = atomic_read(&rcu_call_count);
+                if (n == 0) {
+                    qemu_event_wait(&rcu_call_ready_event);
+                }
+            }
+            n = atomic_read(&rcu_call_count);
+        }
+
+        atomic_sub(&rcu_call_count, n);
+        synchronize_rcu();
+        qemu_mutex_lock_iothread();
+        while (n > 0) {
+            node = try_dequeue();
+            while (!node) {
+                qemu_mutex_unlock_iothread();
+                qemu_event_reset(&rcu_call_ready_event);
+                node = try_dequeue();
+                if (!node) {
+                    qemu_event_wait(&rcu_call_ready_event);
+                    node = try_dequeue();
+                }
+                qemu_mutex_lock_iothread();
+            }
+
+            n--;
+            node->func(node);
+        }
+        qemu_mutex_unlock_iothread();
+    }
+    abort();
+}
+
+void call_rcu1(struct rcu_head *node, void (*func)(struct rcu_head *node))
+{
+    node->func = func;
+    enqueue(node);
+    atomic_inc(&rcu_call_count);
+    qemu_event_set(&rcu_call_ready_event);
+}
+
+void rcu_register_thread(void)
+{
+    assert(rcu_reader.ctr == 0);
+    qemu_mutex_lock(&rcu_registry_lock);
+    QLIST_INSERT_HEAD(&registry, &rcu_reader, node);
+    qemu_mutex_unlock(&rcu_registry_lock);
+}
+
+void rcu_unregister_thread(void)
+{
+    qemu_mutex_lock(&rcu_registry_lock);
+    QLIST_REMOVE(&rcu_reader, node);
+    qemu_mutex_unlock(&rcu_registry_lock);
+}
+
+static void rcu_init_complete(void)
+{
+    QemuThread thread;
+
+    qemu_mutex_init(&rcu_registry_lock);
+    qemu_mutex_init(&rcu_sync_lock);
+    qemu_event_init(&rcu_gp_event, true);
+
+    qemu_event_init(&rcu_call_ready_event, false);
+
+    /* The caller is assumed to have iothread lock, so the call_rcu thread
+     * must have been quiescent even after forking, just recreate it.
+     */
+    qemu_thread_create(&thread, "call_rcu", call_rcu_thread,
+                       NULL, QEMU_THREAD_DETACHED);
+
+    rcu_register_thread();
+}
+
+#ifdef CONFIG_POSIX
+static void rcu_init_lock(void)
+{
+    qemu_mutex_lock(&rcu_sync_lock);
+    qemu_mutex_lock(&rcu_registry_lock);
+}
+
+static void rcu_init_unlock(void)
+{
+    qemu_mutex_unlock(&rcu_registry_lock);
+    qemu_mutex_unlock(&rcu_sync_lock);
+}
+#endif
+
+void rcu_after_fork(void)
+{
+    memset(&registry, 0, sizeof(registry));
+    rcu_init_complete();
+}
+
+static void __attribute__((__constructor__)) rcu_init(void)
+{
+#ifdef CONFIG_POSIX
+    pthread_atfork(rcu_init_lock, rcu_init_unlock, rcu_init_unlock);
+#endif
+    rcu_init_complete();
+}
diff --git a/util/readline.c b/util/readline.c
index 63f4467..507cfdc 100644
--- a/util/readline.c
+++ b/util/readline.c
@@ -22,8 +22,10 @@
  * THE SOFTWARE.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "qemu/readline.h"
+#include "qemu/cutils.h"
 
 #define IS_NORM 0
 #define IS_ESC  1
diff --git a/util/rfifolock.c b/util/rfifolock.c
index afbf748..084c2f0 100644
--- a/util/rfifolock.c
+++ b/util/rfifolock.c
@@ -11,7 +11,7 @@
  *
  */
 
-#include <assert.h>
+#include "qemu/osdep.h"
 #include "qemu/rfifolock.h"
 
 void rfifolock_init(RFifoLock *r, void (*cb)(void *), void *opaque)
@@ -58,9 +58,9 @@
             }
             qemu_cond_wait(&r->cond, &r->lock);
         }
+        qemu_thread_get_self(&r->owner_thread);
     }
 
-    qemu_thread_get_self(&r->owner_thread);
     r->nesting++;
     qemu_mutex_unlock(&r->lock);
 }
diff --git a/util/shared-library-posix.c b/util/shared-library-posix.c
deleted file mode 100644
index cd59465..0000000
--- a/util/shared-library-posix.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/* Copyright (C) 2015 The Android Open Source Project
-**
-** This software is licensed under the terms of the GNU General Public
-** License version 2, as published by the Free Software Foundation, and
-** may be copied, distributed, and modified under those terms.
-**
-** This program is distributed in the hope that it will be useful,
-** but WITHOUT ANY WARRANTY; without even the implied warranty of
-** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-** GNU General Public License for more details.
-*/
-#include "qemu/shared-library.h"
-
-#include <glib.h>
-
-#include <dlfcn.h>
-#include <string.h>
-
-#ifdef __APPLE__
-#  define SO_EXTENSION ".dylib"
-#else
-#  define SO_EXTENSION ".so"
-#endif
-
-SharedLibrary* shared_library_open(const char *name, Error **error) {
-    GString* libpath = g_string_new(name);
-
-    /* Append a library file extension to the name if it doesn't have one */
-    const char* basename = strrchr(name, '/');
-    if (!basename) {
-        basename = name;
-    }
-    if (!strchr(basename, '.')) {
-        g_string_append(libpath, SO_EXTENSION);
-    }
-
-    dlerror();
-    SharedLibrary* result = dlopen(libpath->str, RTLD_LAZY);
-    if (!result) {
-        error_setg(error, "Could not open library %s [%s]", name, dlerror());
-    }
-    g_string_free(libpath, TRUE);
-    return result;
-}
-
-void* shared_library_find(SharedLibrary* library, const char* symbol) {
-    if (!library) {
-        return NULL;
-    }
-    return dlsym(library, symbol);
-}
-
-void shared_library_close(SharedLibrary* library) {
-    if (library) {
-        dlclose(library);
-    }
-}
diff --git a/util/shared-library-win32.c b/util/shared-library-win32.c
deleted file mode 100644
index 0548234..0000000
--- a/util/shared-library-win32.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/* Copyright (C) 2015 The Android Open Source Project
-**
-** This software is licensed under the terms of the GNU General Public
-** License version 2, as published by the Free Software Foundation, and
-** may be copied, distributed, and modified under those terms.
-**
-** This program is distributed in the hope that it will be useful,
-** but WITHOUT ANY WARRANTY; without even the implied warranty of
-** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-** GNU General Public License for more details.
-*/
-#include "qemu/shared-library.h"
-
-#include <glib.h>
-
-#include <string.h>
-#include <windows.h>
-
-SharedLibrary* shared_library_open(const char *name, Error **error) {
-    GString* libpath = g_string_new(name);
-
-    /* Append a .dll suffix to the library name if there isn't one */
-    const char* basename = strrchr(name, '\\');
-    const char* basename2 = strrchr(name, '/');
-    if (!basename || (basename2 && basename2 > basename)) {
-        basename = basename2;
-    }
-    if (!basename) {
-        basename = name;
-    }
-    if (!strchr(basename, '.')) {
-        g_string_append(libpath, ".dll");
-    }
-
-    HMODULE lib = LoadLibrary(libpath->str);
-    if (!lib) {
-        char *pstr;
-        FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
-                      FORMAT_MESSAGE_ALLOCATE_BUFFER,
-                      NULL,
-                      GetLastError(),
-                      0,
-                      (LPTSTR)&pstr,
-                      2,
-                      NULL);
-        error_setg(error, "Could not open library %s [%s]",
-                   libpath->str, pstr);
-        LocalFree(pstr);
-    }
-    g_string_free(libpath, TRUE);
-
-    return (SharedLibrary*)lib;
-}
-
-void* shared_library_find(SharedLibrary* library, const char* symbol) {
-    if (!library) {
-        return NULL;
-    }
-    void* result = GetProcAddress((HMODULE)library, symbol);
-    return result;
-}
-
-void shared_library_close(SharedLibrary* library) {
-    if (library) {
-        FreeLibrary((HMODULE)library);
-    }
-}
diff --git a/util/throttle.c b/util/throttle.c
index f976ac7..3817d9b 100644
--- a/util/throttle.c
+++ b/util/throttle.c
@@ -1,10 +1,12 @@
 /*
  * QEMU throttling infrastructure
  *
- * Copyright (C) Nodalink, SARL. 2013
+ * Copyright (C) Nodalink, EURL. 2013-2014
+ * Copyright (C) Igalia, S.L. 2015
  *
- * Author:
- *   Benoît Canet <benoit.canet@irqsave.net>
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -20,6 +22,8 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "qemu/osdep.h"
+#include "qapi/error.h"
 #include "qemu/throttle.h"
 #include "qemu/timer.h"
 #include "block/aio.h"
@@ -38,6 +42,14 @@
 
     /* make the bucket leak */
     bkt->level = MAX(bkt->level - leak, 0);
+
+    /* if we allow bursts for more than one second we also need to
+     * keep track of bkt->burst_level so the bkt->max goal per second
+     * is attained */
+    if (bkt->burst_length > 1) {
+        leak = (bkt->max * (double) delta_ns) / NANOSECONDS_PER_SECOND;
+        bkt->burst_level = MAX(bkt->burst_level - leak, 0);
+    }
 }
 
 /* Calculate the time delta since last leak and make proportionals leaks
@@ -88,13 +100,24 @@
         return 0;
     }
 
-    extra = bkt->level - bkt->max;
-
-    if (extra <= 0) {
-        return 0;
+    /* If the bucket is full then we have to wait */
+    extra = bkt->level - bkt->max * bkt->burst_length;
+    if (extra > 0) {
+        return throttle_do_compute_wait(bkt->avg, extra);
     }
 
-    return throttle_do_compute_wait(bkt->avg, extra);
+    /* If the bucket is not full yet we have to make sure that we
+     * fulfill the goal of bkt->max units per second. */
+    if (bkt->burst_length > 1) {
+        /* We use 1/10 of the max value to smooth the throttling.
+         * See throttle_fix_bucket() for more details. */
+        extra = bkt->burst_level - bkt->max / 10;
+        if (extra > 0) {
+            return throttle_do_compute_wait(bkt->max, extra);
+        }
+    }
+
+    return 0;
 }
 
 /* This function compute the time that must be waited while this IO
@@ -134,10 +157,10 @@
  * @next_timestamp: the resulting timer
  * @ret:        true if a timer must be set
  */
-bool throttle_compute_timer(ThrottleState *ts,
-                            bool is_write,
-                            int64_t now,
-                            int64_t *next_timestamp)
+static bool throttle_compute_timer(ThrottleState *ts,
+                                   bool is_write,
+                                   int64_t now,
+                                   int64_t *next_timestamp)
 {
     int64_t wait;
 
@@ -159,29 +182,50 @@
 }
 
 /* Add timers to event loop */
-void throttle_attach_aio_context(ThrottleState *ts, AioContext *new_context)
+void throttle_timers_attach_aio_context(ThrottleTimers *tt,
+                                        AioContext *new_context)
 {
-    ts->timers[0] = aio_timer_new(new_context, ts->clock_type, SCALE_NS,
-                                  ts->read_timer_cb, ts->timer_opaque);
-    ts->timers[1] = aio_timer_new(new_context, ts->clock_type, SCALE_NS,
-                                  ts->write_timer_cb, ts->timer_opaque);
+    tt->timers[0] = aio_timer_new(new_context, tt->clock_type, SCALE_NS,
+                                  tt->read_timer_cb, tt->timer_opaque);
+    tt->timers[1] = aio_timer_new(new_context, tt->clock_type, SCALE_NS,
+                                  tt->write_timer_cb, tt->timer_opaque);
+}
+
+/*
+ * Initialize the ThrottleConfig structure to a valid state
+ * @cfg: the config to initialize
+ */
+void throttle_config_init(ThrottleConfig *cfg)
+{
+    unsigned i;
+    memset(cfg, 0, sizeof(*cfg));
+    for (i = 0; i < BUCKETS_COUNT; i++) {
+        cfg->buckets[i].burst_length = 1;
+    }
 }
 
 /* To be called first on the ThrottleState */
-void throttle_init(ThrottleState *ts,
-                   AioContext *aio_context,
-                   QEMUClockType clock_type,
-                   QEMUTimerCB *read_timer_cb,
-                   QEMUTimerCB *write_timer_cb,
-                   void *timer_opaque)
+void throttle_init(ThrottleState *ts)
 {
     memset(ts, 0, sizeof(ThrottleState));
+    throttle_config_init(&ts->cfg);
+}
 
-    ts->clock_type = clock_type;
-    ts->read_timer_cb = read_timer_cb;
-    ts->write_timer_cb = write_timer_cb;
-    ts->timer_opaque = timer_opaque;
-    throttle_attach_aio_context(ts, aio_context);
+/* To be called first on the ThrottleTimers */
+void throttle_timers_init(ThrottleTimers *tt,
+                          AioContext *aio_context,
+                          QEMUClockType clock_type,
+                          QEMUTimerCB *read_timer_cb,
+                          QEMUTimerCB *write_timer_cb,
+                          void *timer_opaque)
+{
+    memset(tt, 0, sizeof(ThrottleTimers));
+
+    tt->clock_type = clock_type;
+    tt->read_timer_cb = read_timer_cb;
+    tt->write_timer_cb = write_timer_cb;
+    tt->timer_opaque = timer_opaque;
+    throttle_timers_attach_aio_context(tt, aio_context);
 }
 
 /* destroy a timer */
@@ -195,25 +239,25 @@
 }
 
 /* Remove timers from event loop */
-void throttle_detach_aio_context(ThrottleState *ts)
+void throttle_timers_detach_aio_context(ThrottleTimers *tt)
 {
     int i;
 
     for (i = 0; i < 2; i++) {
-        throttle_timer_destroy(&ts->timers[i]);
+        throttle_timer_destroy(&tt->timers[i]);
     }
 }
 
-/* To be called last on the ThrottleState */
-void throttle_destroy(ThrottleState *ts)
+/* To be called last on the ThrottleTimers */
+void throttle_timers_destroy(ThrottleTimers *tt)
 {
-    throttle_detach_aio_context(ts);
+    throttle_timers_detach_aio_context(tt);
 }
 
 /* is any throttling timer configured */
-bool throttle_have_timer(ThrottleState *ts)
+bool throttle_timers_are_initialized(ThrottleTimers *tt)
 {
-    if (ts->timers[0]) {
+    if (tt->timers[0]) {
         return true;
     }
 
@@ -238,13 +282,14 @@
     return false;
 }
 
-/* return true if any two throttling parameters conflicts
- *
+/* check if a throttling configuration is valid
  * @cfg: the throttling configuration to inspect
- * @ret: true if any conflict detected else false
+ * @ret: true if valid else false
+ * @errp: error object
  */
-bool throttle_conflicting(ThrottleConfig *cfg)
+bool throttle_is_valid(ThrottleConfig *cfg, Error **errp)
 {
+    int i;
     bool bps_flag, ops_flag;
     bool bps_max_flag, ops_max_flag;
 
@@ -264,31 +309,53 @@
                    (cfg->buckets[THROTTLE_OPS_READ].max ||
                    cfg->buckets[THROTTLE_OPS_WRITE].max);
 
-    return bps_flag || ops_flag || bps_max_flag || ops_max_flag;
-}
+    if (bps_flag || ops_flag || bps_max_flag || ops_max_flag) {
+        error_setg(errp, "bps/iops/max total values and read/write values"
+                   " cannot be used at the same time");
+        return false;
+    }
 
-/* check if a throttling configuration is valid
- * @cfg: the throttling configuration to inspect
- * @ret: true if valid else false
- */
-bool throttle_is_valid(ThrottleConfig *cfg)
-{
-    bool invalid = false;
-    int i;
-
-    for (i = 0; i < BUCKETS_COUNT; i++) {
-        if (cfg->buckets[i].avg < 0) {
-            invalid = true;
-        }
+    if (cfg->op_size &&
+        !cfg->buckets[THROTTLE_OPS_TOTAL].avg &&
+        !cfg->buckets[THROTTLE_OPS_READ].avg &&
+        !cfg->buckets[THROTTLE_OPS_WRITE].avg) {
+        error_setg(errp, "iops size requires an iops value to be set");
+        return false;
     }
 
     for (i = 0; i < BUCKETS_COUNT; i++) {
-        if (cfg->buckets[i].max < 0) {
-            invalid = true;
+        if (cfg->buckets[i].avg < 0 ||
+            cfg->buckets[i].max < 0 ||
+            cfg->buckets[i].avg > THROTTLE_VALUE_MAX ||
+            cfg->buckets[i].max > THROTTLE_VALUE_MAX) {
+            error_setg(errp, "bps/iops/max values must be within [0, %lld]",
+                       THROTTLE_VALUE_MAX);
+            return false;
+        }
+
+        if (!cfg->buckets[i].burst_length) {
+            error_setg(errp, "the burst length cannot be 0");
+            return false;
+        }
+
+        if (cfg->buckets[i].burst_length > 1 && !cfg->buckets[i].max) {
+            error_setg(errp, "burst length set without burst rate");
+            return false;
+        }
+
+        if (cfg->buckets[i].max && !cfg->buckets[i].avg) {
+            error_setg(errp, "bps_max/iops_max require corresponding"
+                       " bps/iops values");
+            return false;
+        }
+
+        if (cfg->buckets[i].max && cfg->buckets[i].max < cfg->buckets[i].avg) {
+            error_setg(errp, "bps_max/iops_max cannot be lower than bps/iops");
+            return false;
         }
     }
 
-    return !invalid;
+    return true;
 }
 
 /* fix bucket parameters */
@@ -297,7 +364,7 @@
     double min;
 
     /* zero bucket level */
-    bkt->level = 0;
+    bkt->level = bkt->burst_level = 0;
 
     /* The following is done to cope with the Linux CFQ block scheduler
      * which regroup reads and writes by block of 100ms in the guest.
@@ -324,9 +391,12 @@
 /* Used to configure the throttle
  *
  * @ts: the throttle state we are working on
+ * @tt: the throttle timers we use in this aio context
  * @cfg: the config to set
  */
-void throttle_config(ThrottleState *ts, ThrottleConfig *cfg)
+void throttle_config(ThrottleState *ts,
+                     ThrottleTimers *tt,
+                     ThrottleConfig *cfg)
 {
     int i;
 
@@ -336,10 +406,10 @@
         throttle_fix_bucket(&ts->cfg.buckets[i]);
     }
 
-    ts->previous_leak = qemu_clock_get_ns(ts->clock_type);
+    ts->previous_leak = qemu_clock_get_ns(tt->clock_type);
 
     for (i = 0; i < 2; i++) {
-        throttle_cancel_timer(ts->timers[i]);
+        throttle_cancel_timer(tt->timers[i]);
     }
 }
 
@@ -358,12 +428,15 @@
  *
  * NOTE: this function is not unit tested due to it's usage of timer_mod
  *
+ * @tt:       the timers structure
  * @is_write: the type of operation (read/write)
  * @ret:      true if the timer has been scheduled else false
  */
-bool throttle_schedule_timer(ThrottleState *ts, bool is_write)
+bool throttle_schedule_timer(ThrottleState *ts,
+                             ThrottleTimers *tt,
+                             bool is_write)
 {
-    int64_t now = qemu_clock_get_ns(ts->clock_type);
+    int64_t now = qemu_clock_get_ns(tt->clock_type);
     int64_t next_timestamp;
     bool must_wait;
 
@@ -378,12 +451,12 @@
     }
 
     /* request throttled and timer pending -> do nothing */
-    if (timer_pending(ts->timers[is_write])) {
+    if (timer_pending(tt->timers[is_write])) {
         return true;
     }
 
     /* request throttled and timer not pending -> arm timer */
-    timer_mod(ts->timers[is_write], next_timestamp);
+    timer_mod(tt->timers[is_write], next_timestamp);
     return true;
 }
 
@@ -394,22 +467,36 @@
  */
 void throttle_account(ThrottleState *ts, bool is_write, uint64_t size)
 {
+    const BucketType bucket_types_size[2][2] = {
+        { THROTTLE_BPS_TOTAL, THROTTLE_BPS_READ },
+        { THROTTLE_BPS_TOTAL, THROTTLE_BPS_WRITE }
+    };
+    const BucketType bucket_types_units[2][2] = {
+        { THROTTLE_OPS_TOTAL, THROTTLE_OPS_READ },
+        { THROTTLE_OPS_TOTAL, THROTTLE_OPS_WRITE }
+    };
     double units = 1.0;
+    unsigned i;
 
     /* if cfg.op_size is defined and smaller than size we compute unit count */
     if (ts->cfg.op_size && size > ts->cfg.op_size) {
         units = (double) size / ts->cfg.op_size;
     }
 
-    ts->cfg.buckets[THROTTLE_BPS_TOTAL].level += size;
-    ts->cfg.buckets[THROTTLE_OPS_TOTAL].level += units;
+    for (i = 0; i < 2; i++) {
+        LeakyBucket *bkt;
 
-    if (is_write) {
-        ts->cfg.buckets[THROTTLE_BPS_WRITE].level += size;
-        ts->cfg.buckets[THROTTLE_OPS_WRITE].level += units;
-    } else {
-        ts->cfg.buckets[THROTTLE_BPS_READ].level += size;
-        ts->cfg.buckets[THROTTLE_OPS_READ].level += units;
+        bkt = &ts->cfg.buckets[bucket_types_size[is_write][i]];
+        bkt->level += size;
+        if (bkt->burst_length > 1) {
+            bkt->burst_level += size;
+        }
+
+        bkt = &ts->cfg.buckets[bucket_types_units[is_write][i]];
+        bkt->level += units;
+        if (bkt->burst_length > 1) {
+            bkt->burst_level += units;
+        }
     }
 }
 
diff --git a/util/timed-average.c b/util/timed-average.c
new file mode 100644
index 0000000..2b49d53
--- /dev/null
+++ b/util/timed-average.c
@@ -0,0 +1,231 @@
+/*
+ * QEMU timed average computation
+ *
+ * Copyright (C) Nodalink, EURL. 2014
+ * Copyright (C) Igalia, S.L. 2015
+ *
+ * Authors:
+ *   Benoît Canet <benoit.canet@nodalink.com>
+ *   Alberto Garcia <berto@igalia.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) version 3 or any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+
+#include "qemu/timed-average.h"
+
+/* This module computes an average of a set of values within a time
+ * window.
+ *
+ * Algorithm:
+ *
+ * - Create two windows with a certain expiration period, and
+ *   offsetted by period / 2.
+ * - Each time you want to account a new value, do it in both windows.
+ * - The minimum / maximum / average values are always returned from
+ *   the oldest window.
+ *
+ * Example:
+ *
+ *        t=0          |t=0.5           |t=1          |t=1.5            |t=2
+ *        wnd0: [0,0.5)|wnd0: [0.5,1.5) |             |wnd0: [1.5,2.5)  |
+ *        wnd1: [0,1)  |                |wnd1: [1,2)  |                 |
+ *
+ * Values are returned from:
+ *
+ *        wnd0---------|wnd1------------|wnd0---------|wnd1-------------|
+ */
+
+/* Update the expiration of a time window
+ *
+ * @w:      the window used
+ * @now:    the current time in nanoseconds
+ * @period: the expiration period in nanoseconds
+ */
+static void update_expiration(TimedAverageWindow *w, int64_t now,
+                              int64_t period)
+{
+    /* time elapsed since the last theoretical expiration */
+    int64_t elapsed = (now - w->expiration) % period;
+    /* time remaininging until the next expiration */
+    int64_t remaining = period - elapsed;
+    /* compute expiration */
+    w->expiration = now + remaining;
+}
+
+/* Reset a window
+ *
+ * @w: the window to reset
+ */
+static void window_reset(TimedAverageWindow *w)
+{
+    w->min = UINT64_MAX;
+    w->max = 0;
+    w->sum = 0;
+    w->count = 0;
+}
+
+/* Get the current window (that is, the one with the earliest
+ * expiration time).
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: a pointer to the current window
+ */
+static TimedAverageWindow *current_window(TimedAverage *ta)
+{
+     return &ta->windows[ta->current];
+}
+
+/* Initialize a TimedAverage structure
+ *
+ * @ta:         the TimedAverage structure
+ * @clock_type: the type of clock to use
+ * @period:     the time window period in nanoseconds
+ */
+void timed_average_init(TimedAverage *ta, QEMUClockType clock_type,
+                        uint64_t period)
+{
+    int64_t now = qemu_clock_get_ns(clock_type);
+
+    /* Returned values are from the oldest window, so they belong to
+     * the interval [ta->period/2,ta->period). By adjusting the
+     * requested period by 4/3, we guarantee that they're in the
+     * interval [2/3 period,4/3 period), closer to the requested
+     * period on average */
+    ta->period = (uint64_t) period * 4 / 3;
+    ta->clock_type = clock_type;
+    ta->current = 0;
+
+    window_reset(&ta->windows[0]);
+    window_reset(&ta->windows[1]);
+
+    /* Both windows are offsetted by half a period */
+    ta->windows[0].expiration = now + ta->period / 2;
+    ta->windows[1].expiration = now + ta->period;
+}
+
+/* Check if the time windows have expired, updating their counters and
+ * expiration time if that's the case.
+ *
+ * @ta: the TimedAverage structure
+ * @elapsed: if non-NULL, the elapsed time (in ns) within the current
+ *           window will be stored here
+ */
+static void check_expirations(TimedAverage *ta, uint64_t *elapsed)
+{
+    int64_t now = qemu_clock_get_ns(ta->clock_type);
+    int i;
+
+    assert(ta->period != 0);
+
+    /* Check if the windows have expired */
+    for (i = 0; i < 2; i++) {
+        TimedAverageWindow *w = &ta->windows[i];
+        if (w->expiration <= now) {
+            window_reset(w);
+            update_expiration(w, now, ta->period);
+        }
+    }
+
+    /* Make ta->current point to the oldest window */
+    if (ta->windows[0].expiration < ta->windows[1].expiration) {
+        ta->current = 0;
+    } else {
+        ta->current = 1;
+    }
+
+    /* Calculate the elapsed time within the current window */
+    if (elapsed) {
+        int64_t remaining = ta->windows[ta->current].expiration - now;
+        *elapsed = ta->period - remaining;
+    }
+}
+
+/* Account a value
+ *
+ * @ta:    the TimedAverage structure
+ * @value: the value to account
+ */
+void timed_average_account(TimedAverage *ta, uint64_t value)
+{
+    int i;
+    check_expirations(ta, NULL);
+
+    /* Do the accounting in both windows at the same time */
+    for (i = 0; i < 2; i++) {
+        TimedAverageWindow *w = &ta->windows[i];
+
+        w->sum += value;
+        w->count++;
+
+        if (value < w->min) {
+            w->min = value;
+        }
+
+        if (value > w->max) {
+            w->max = value;
+        }
+    }
+}
+
+/* Get the minimum value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the minimum value
+ */
+uint64_t timed_average_min(TimedAverage *ta)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, NULL);
+    w = current_window(ta);
+    return w->min < UINT64_MAX ? w->min : 0;
+}
+
+/* Get the average value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the average value
+ */
+uint64_t timed_average_avg(TimedAverage *ta)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, NULL);
+    w = current_window(ta);
+    return w->count > 0 ? w->sum / w->count : 0;
+}
+
+/* Get the maximum value
+ *
+ * @ta:  the TimedAverage structure
+ * @ret: the maximum value
+ */
+uint64_t timed_average_max(TimedAverage *ta)
+{
+    check_expirations(ta, NULL);
+    return current_window(ta)->max;
+}
+
+/* Get the sum of all accounted values
+ * @ta:      the TimedAverage structure
+ * @elapsed: if non-NULL, the elapsed time (in ns) will be stored here
+ * @ret:     the sum of all accounted values
+ */
+uint64_t timed_average_sum(TimedAverage *ta, uint64_t *elapsed)
+{
+    TimedAverageWindow *w;
+    check_expirations(ta, elapsed);
+    w = current_window(ta);
+    return w->sum;
+}
diff --git a/util/trace-events b/util/trace-events
new file mode 100644
index 0000000..747e6ba
--- /dev/null
+++ b/util/trace-events
@@ -0,0 +1,13 @@
+# See docs/tracing.txt for syntax documentation.
+
+# util/oslib-win32.c
+# util/oslib-posix.c
+qemu_memalign(size_t alignment, size_t size, void *ptr) "alignment %zu size %zu ptr %p"
+qemu_anon_ram_alloc(size_t size, void *ptr) "size %zu ptr %p"
+qemu_vfree(void *ptr) "ptr %p"
+qemu_anon_ram_free(void *ptr, size_t size) "ptr %p size %zu"
+
+# util/hbitmap.c
+hbitmap_iter_skip_words(const void *hb, void *hbi, uint64_t pos, unsigned long cur) "hb %p hbi %p pos %"PRId64" cur 0x%lx"
+hbitmap_reset(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
+hbitmap_set(void *hb, uint64_t start, uint64_t count, uint64_t sbit, uint64_t ebit) "hb %p items %"PRIu64",%"PRIu64" bits %"PRIu64"..%"PRIu64
diff --git a/util/unicode.c b/util/unicode.c
index d1c8658..a812a35 100644
--- a/util/unicode.c
+++ b/util/unicode.c
@@ -10,7 +10,8 @@
  * later.  See the COPYING file in the top-level directory.
  */
 
-#include "qemu-common.h"
+#include "qemu/osdep.h"
+#include "qemu/unicode.h"
 
 /**
  * mod_utf8_codepoint:
diff --git a/util/uri.c b/util/uri.c
index e348c17..70a9cbc 100644
--- a/util/uri.c
+++ b/util/uri.c
@@ -51,9 +51,7 @@
  *
  */
 
-#include <glib.h>
-#include <string.h>
-#include <stdio.h>
+#include "qemu/osdep.h"
 
 #include "qemu/uri.h"
 
@@ -225,7 +223,7 @@
     while (ISA_ALPHA(cur) || ISA_DIGIT(cur) ||
            (*cur == '+') || (*cur == '-') || (*cur == '.')) cur++;
     if (uri != NULL) {
-	if (uri->scheme != NULL) g_free(uri->scheme);
+        g_free(uri->scheme);
 	uri->scheme = g_strndup(*str, cur - *str);
     }
     *str = cur;
@@ -262,8 +260,7 @@
            ((uri != NULL) && (uri->cleanup & 1) && (IS_UNWISE(cur))))
         NEXT(cur);
     if (uri != NULL) {
-        if (uri->fragment != NULL)
-            g_free(uri->fragment);
+        g_free(uri->fragment);
 	if (uri->cleanup & 2)
 	    uri->fragment = g_strndup(*str, cur - *str);
 	else
@@ -298,8 +295,7 @@
            ((uri != NULL) && (uri->cleanup & 1) && (IS_UNWISE(cur))))
         NEXT(cur);
     if (uri != NULL) {
-	if (uri->query != NULL)
-	    g_free (uri->query);
+        g_free(uri->query);
 	uri->query = g_strndup (*str, cur - *str);
     }
     *str = cur;
@@ -322,19 +318,23 @@
 rfc3986_parse_port(URI *uri, const char **str)
 {
     const char *cur = *str;
+    int port = 0;
 
     if (ISA_DIGIT(cur)) {
-	if (uri != NULL)
-	    uri->port = 0;
-	while (ISA_DIGIT(cur)) {
-	    if (uri != NULL)
-		uri->port = uri->port * 10 + (*cur - '0');
-	    cur++;
-	}
-	*str = cur;
-	return(0);
+        while (ISA_DIGIT(cur)) {
+            port = port * 10 + (*cur - '0');
+            if (port > 65535) {
+                return 1;
+            }
+            cur++;
+        }
+        if (uri) {
+            uri->port = port;
+        }
+        *str = cur;
+        return 0;
     }
-    return(1);
+    return 1;
 }
 
 /**
@@ -360,7 +360,7 @@
 	NEXT(cur);
     if (*cur == '@') {
 	if (uri != NULL) {
-	    if (uri->user != NULL) g_free(uri->user);
+            g_free(uri->user);
 	    if (uri->cleanup & 2)
 		uri->user = g_strndup(*str, cur - *str);
 	    else
@@ -473,9 +473,9 @@
         NEXT(cur);
 found:
     if (uri != NULL) {
-	if (uri->authority != NULL) g_free(uri->authority);
+        g_free(uri->authority);
 	uri->authority = NULL;
-	if (uri->server != NULL) g_free(uri->server);
+        g_free(uri->server);
 	if (cur != host) {
 	    if (uri->cleanup & 2)
 		uri->server = g_strndup(host, cur - host);
@@ -585,7 +585,7 @@
 	if (ret != 0) return(ret);
     }
     if (uri != NULL) {
-	if (uri->path != NULL) g_free(uri->path);
+        g_free(uri->path);
         if (*str != cur) {
             if (uri->cleanup & 2)
                 uri->path = g_strndup(*str, cur - *str);
@@ -631,7 +631,7 @@
 	}
     }
     if (uri != NULL) {
-	if (uri->path != NULL) g_free(uri->path);
+        g_free(uri->path);
         if (cur != *str) {
             if (uri->cleanup & 2)
                 uri->path = g_strndup(*str, cur - *str);
@@ -673,7 +673,7 @@
 	if (ret != 0) return(ret);
     }
     if (uri != NULL) {
-	if (uri->path != NULL) g_free(uri->path);
+        g_free(uri->path);
         if (cur != *str) {
             if (uri->cleanup & 2)
                 uri->path = g_strndup(*str, cur - *str);
@@ -715,7 +715,7 @@
 	if (ret != 0) return(ret);
     }
     if (uri != NULL) {
-	if (uri->path != NULL) g_free(uri->path);
+        g_free(uri->path);
         if (cur != *str) {
             if (uri->cleanup & 2)
                 uri->path = g_strndup(*str, cur - *str);
@@ -769,7 +769,7 @@
     } else {
 	/* path-empty is effectively empty */
 	if (uri != NULL) {
-	    if (uri->path != NULL) g_free(uri->path);
+            g_free(uri->path);
 	    uri->path = NULL;
 	}
     }
@@ -812,7 +812,7 @@
     } else {
 	/* path-empty is effectively empty */
 	if (uri != NULL) {
-	    if (uri->path != NULL) g_free(uri->path);
+            g_free(uri->path);
 	    uri->path = NULL;
 	}
     }
@@ -930,12 +930,10 @@
     if (str == NULL)
 	return(NULL);
     uri = uri_new();
-    if (uri != NULL) {
-	ret = rfc3986_parse_uri_reference(uri, str);
-        if (ret) {
-	    uri_free(uri);
-	    return(NULL);
-	}
+    ret = rfc3986_parse_uri_reference(uri, str);
+    if (ret) {
+        uri_free(uri);
+        return(NULL);
     }
     return(uri);
 }
@@ -976,15 +974,13 @@
     if (str == NULL)
 	return(NULL);
     uri = uri_new();
-    if (uri != NULL) {
-        if (raw) {
-	    uri->cleanup |= 2;
-	}
-	ret = uri_parse_into(uri, str);
-        if (ret) {
-	    uri_free(uri);
-	    return(NULL);
-	}
+    if (raw) {
+        uri->cleanup |= 2;
+    }
+    ret = uri_parse_into(uri, str);
+    if (ret) {
+        uri_free(uri);
+        return(NULL);
     }
     return(uri);
 }
@@ -1006,8 +1002,7 @@
 uri_new(void) {
     URI *ret;
 
-    ret = (URI *) g_malloc(sizeof(URI));
-    memset(ret, 0, sizeof(URI));
+    ret = g_new0(URI, 1);
     return(ret);
 }
 
@@ -1056,14 +1051,12 @@
 	while (*p != 0) {
 	    if (len >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
 		ret = temp;
 	    }
 	    ret[len++] = *p++;
 	}
 	if (len >= max) {
             temp = realloc2n(ret, &max);
-            if (temp == NULL) goto mem_error;
             ret = temp;
 	}
 	ret[len++] = ':';
@@ -1073,7 +1066,6 @@
 	while (*p != 0) {
 	    if (len + 3 >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
                 ret = temp;
 	    }
 	    if (IS_RESERVED(*(p)) || IS_UNRESERVED(*(p)))
@@ -1090,7 +1082,6 @@
 	if (uri->server != NULL) {
 	    if (len + 3 >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
                 ret = temp;
 	    }
 	    ret[len++] = '/';
@@ -1100,7 +1091,6 @@
 		while (*p != 0) {
 		    if (len + 3 >= max) {
                         temp = realloc2n(ret, &max);
-                        if (temp == NULL) goto mem_error;
                         ret = temp;
 		    }
 		    if ((IS_UNRESERVED(*(p))) ||
@@ -1119,7 +1109,6 @@
 		}
 		if (len + 3 >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		ret[len++] = '@';
@@ -1128,7 +1117,6 @@
 	    while (*p != 0) {
 		if (len >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		ret[len++] = *p++;
@@ -1136,7 +1124,6 @@
 	    if (uri->port > 0) {
 		if (len + 10 >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		len += snprintf(&ret[len], max - len, ":%d", uri->port);
@@ -1144,7 +1131,6 @@
 	} else if (uri->authority != NULL) {
 	    if (len + 3 >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
                 ret = temp;
 	    }
 	    ret[len++] = '/';
@@ -1153,7 +1139,6 @@
 	    while (*p != 0) {
 		if (len + 3 >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		if ((IS_UNRESERVED(*(p))) ||
@@ -1172,7 +1157,6 @@
 	} else if (uri->scheme != NULL) {
 	    if (len + 3 >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
                 ret = temp;
 	    }
 	    ret[len++] = '/';
@@ -1192,7 +1176,6 @@
 	        (!strcmp(uri->scheme, "file"))) {
 		if (len + 3 >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		ret[len++] = *p++;
@@ -1202,7 +1185,6 @@
 	    while (*p != 0) {
 		if (len + 3 >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		if ((IS_UNRESERVED(*(p))) || ((*(p) == '/')) ||
@@ -1222,7 +1204,6 @@
 	if (uri->query != NULL) {
 	    if (len + 1 >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
                 ret = temp;
 	    }
 	    ret[len++] = '?';
@@ -1230,7 +1211,6 @@
 	    while (*p != 0) {
 		if (len + 1 >= max) {
                     temp = realloc2n(ret, &max);
-                    if (temp == NULL) goto mem_error;
                     ret = temp;
 		}
 		ret[len++] = *p++;
@@ -1240,7 +1220,6 @@
     if (uri->fragment != NULL) {
 	if (len + 3 >= max) {
             temp = realloc2n(ret, &max);
-            if (temp == NULL) goto mem_error;
             ret = temp;
 	}
 	ret[len++] = '#';
@@ -1248,7 +1227,6 @@
 	while (*p != 0) {
 	    if (len + 3 >= max) {
                 temp = realloc2n(ret, &max);
-                if (temp == NULL) goto mem_error;
                 ret = temp;
 	    }
 	    if ((IS_UNRESERVED(*(p))) || (IS_RESERVED(*(p))))
@@ -1264,15 +1242,10 @@
     }
     if (len >= max) {
         temp = realloc2n(ret, &max);
-        if (temp == NULL) goto mem_error;
         ret = temp;
     }
     ret[len] = 0;
     return(ret);
-
-mem_error:
-    g_free(ret);
-    return(NULL);
 }
 
 /**
@@ -1285,21 +1258,21 @@
 uri_clean(URI *uri) {
     if (uri == NULL) return;
 
-    if (uri->scheme != NULL) g_free(uri->scheme);
+    g_free(uri->scheme);
     uri->scheme = NULL;
-    if (uri->server != NULL) g_free(uri->server);
+    g_free(uri->server);
     uri->server = NULL;
-    if (uri->user != NULL) g_free(uri->user);
+    g_free(uri->user);
     uri->user = NULL;
-    if (uri->path != NULL) g_free(uri->path);
+    g_free(uri->path);
     uri->path = NULL;
-    if (uri->fragment != NULL) g_free(uri->fragment);
+    g_free(uri->fragment);
     uri->fragment = NULL;
-    if (uri->opaque != NULL) g_free(uri->opaque);
+    g_free(uri->opaque);
     uri->opaque = NULL;
-    if (uri->authority != NULL) g_free(uri->authority);
+    g_free(uri->authority);
     uri->authority = NULL;
-    if (uri->query != NULL) g_free(uri->query);
+    g_free(uri->query);
     uri->query = NULL;
 }
 
@@ -1678,8 +1651,6 @@
     else {
 	if (*uri) {
 	    ref = uri_new();
-	    if (ref == NULL)
-		goto done;
 	    ret = uri_parse_into(ref, uri);
 	}
 	else
@@ -1698,8 +1669,6 @@
 	ret = -1;
     else {
 	bas = uri_new();
-	if (bas == NULL)
-	    goto done;
 	ret = uri_parse_into(bas, base);
     }
     if (ret != 0) {
@@ -1711,10 +1680,8 @@
 	/*
 	 * the base fragment must be ignored
 	 */
-	if (bas->fragment != NULL) {
-	    g_free(bas->fragment);
-	    bas->fragment = NULL;
-	}
+        g_free(bas->fragment);
+        bas->fragment = NULL;
 	val = uri_to_string(bas);
 	goto done;
     }
@@ -1732,28 +1699,23 @@
      *    document.
      */
     res = uri_new();
-    if (res == NULL)
-	goto done;
     if ((ref->scheme == NULL) && (ref->path == NULL) &&
 	((ref->authority == NULL) && (ref->server == NULL))) {
-	if (bas->scheme != NULL)
-	    res->scheme = g_strdup(bas->scheme);
+        res->scheme = g_strdup(bas->scheme);
 	if (bas->authority != NULL)
 	    res->authority = g_strdup(bas->authority);
 	else if (bas->server != NULL) {
-	    res->server = g_strdup(bas->server);
-	    if (bas->user != NULL)
-		res->user = g_strdup(bas->user);
-	    res->port = bas->port;
+            res->server = g_strdup(bas->server);
+            res->user = g_strdup(bas->user);
+            res->port = bas->port;
 	}
-	if (bas->path != NULL)
-	    res->path = g_strdup(bas->path);
-	if (ref->query != NULL)
+        res->path = g_strdup(bas->path);
+        if (ref->query != NULL) {
 	    res->query = g_strdup (ref->query);
-	else if (bas->query != NULL)
-	    res->query = g_strdup(bas->query);
-	if (ref->fragment != NULL)
-	    res->fragment = g_strdup(ref->fragment);
+        } else {
+            res->query = g_strdup(bas->query);
+        }
+        res->fragment = g_strdup(ref->fragment);
 	goto step_7;
     }
 
@@ -1767,13 +1729,10 @@
 	val = uri_to_string(ref);
 	goto done;
     }
-    if (bas->scheme != NULL)
-	res->scheme = g_strdup(bas->scheme);
+    res->scheme = g_strdup(bas->scheme);
 
-    if (ref->query != NULL)
-	res->query = g_strdup(ref->query);
-    if (ref->fragment != NULL)
-	res->fragment = g_strdup(ref->fragment);
+    res->query = g_strdup(ref->query);
+    res->fragment = g_strdup(ref->fragment);
 
     /*
      * 4) If the authority component is defined, then the reference is a
@@ -1787,20 +1746,17 @@
 	    res->authority = g_strdup(ref->authority);
 	else {
 	    res->server = g_strdup(ref->server);
-	    if (ref->user != NULL)
-		res->user = g_strdup(ref->user);
+            res->user = g_strdup(ref->user);
             res->port = ref->port;
 	}
-	if (ref->path != NULL)
-	    res->path = g_strdup(ref->path);
+        res->path = g_strdup(ref->path);
 	goto step_7;
     }
     if (bas->authority != NULL)
 	res->authority = g_strdup(bas->authority);
     else if (bas->server != NULL) {
-	res->server = g_strdup(bas->server);
-	if (bas->user != NULL)
-	    res->user = g_strdup(bas->user);
+        res->server = g_strdup(bas->server);
+        res->user = g_strdup(bas->user);
 	res->port = bas->port;
     }
 
@@ -1947,8 +1903,6 @@
      * First parse URI into a standard form
      */
     ref = uri_new ();
-    if (ref == NULL)
-	return NULL;
     /* If URI not already in "relative" form */
     if (uri[0] != '.') {
 	ret = uri_parse_into (ref, uri);
@@ -1965,8 +1919,6 @@
 	goto done;
     }
     bas = uri_new ();
-    if (bas == NULL)
-	goto done;
     if (base[0] != '.') {
 	ret = uri_parse_into (bas, base);
 	if (ret != 0)
@@ -1985,7 +1937,8 @@
 	val = g_strdup (uri);
 	goto done;
     }
-    if (!strcmp(bas->path, ref->path)) {
+    if (bas->path == ref->path ||
+        (bas->path && ref->path && !strcmp(bas->path, ref->path))) {
 	val = g_strdup("");
 	goto done;
     }
diff --git a/vl.c b/vl.c
index ad240dd..653f2fc 100755
--- a/vl.c
+++ b/vl.c
@@ -21,14 +21,10 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include <unistd.h>
-#include <fcntl.h>
-#include <signal.h>
-#include <time.h>
-#include <errno.h>
-#include <sys/time.h>
-
-#include "config-host.h"
+#include "qemu/osdep.h"
+#include "qemu-version.h"
+#include "qemu/cutils.h"
+#include "qemu/help_option.h"
 
 #ifdef CONFIG_SECCOMP
 #include "sysemu/seccomp.h"
@@ -38,8 +34,25 @@
 #include <libvdeplug.h>
 #endif
 
-#include <glib.h>
+#if defined(CONFIG_SDL) && !defined(CONFIG_ANDROID)
+#if defined(__APPLE__) || defined(main)
+#include <SDL.h>
+int qemu_main(int argc, char **argv, char **envp);
+int main(int argc, char **argv)
+{
+    return qemu_main(argc, argv, NULL);
+}
+#undef main
+#define main qemu_main
+#endif
+#endif /* CONFIG_SDL */
 
+#ifdef CONFIG_COCOA
+#undef main
+#define main qemu_main
+#endif /* CONFIG_COCOA */
+
+#include "qemu/error-report.h"
 #include "qemu/sockets.h"
 #include "hw/hw.h"
 #include "hw/boards.h"
@@ -49,7 +62,7 @@
 #include "hw/isa/isa.h"
 #include "hw/bt.h"
 #include "sysemu/watchdog.h"
-#include "hw/i386/smbios.h"
+#include "hw/smbios/smbios.h"
 #include "hw/xen/xen.h"
 #include "hw/qdev.h"
 #include "hw/loader.h"
@@ -60,11 +73,14 @@
 #include "net/slirp.h"
 #include "monitor/monitor.h"
 #include "ui/console.h"
+#include "ui/input.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/numa.h"
 #include "exec/gdbstub.h"
 #include "qemu/timer.h"
 #include "sysemu/char.h"
 #include "qemu/bitmap.h"
+#include "qemu/log.h"
 #include "sysemu/blockdev.h"
 #include "hw/block/block.h"
 #include "migration/block.h"
@@ -72,6 +88,7 @@
 #include "sysemu/dma.h"
 #include "audio/audio.h"
 #include "migration/migration.h"
+#include "sysemu/cpus.h"
 #include "sysemu/kvm.h"
 #include "sysemu/hax.h"
 #include "qapi/qmp/qjson.h"
@@ -95,59 +112,63 @@
 #include "qemu/queue.h"
 #include "sysemu/cpus.h"
 #include "sysemu/arch_init.h"
-#include "qemu/osdep.h"
 
 #include "ui/qemu-spice.h"
 #include "qapi/string-input-visitor.h"
 #include "qapi/opts-visitor.h"
 #include "qom/object_interfaces.h"
 #include "qapi-event.h"
+#include "exec/semihost.h"
+#include "crypto/init.h"
+#include "sysemu/replay.h"
+#include "qapi/qmp/qerror.h"
 
 #ifdef CONFIG_ANDROID
 
+#undef socket_connect
+#undef socket_listen
 
-#include "config.h"
-
+#include "android/android.h"
 #include "android/boot-properties.h"
-#include "android/error-messages.h"
 #include "android/crashreport/crash-handler.h"
 #include "android/emulation/bufprint_config_dirs.h"
-#include "android/metrics/metrics.h"
-#include "android/update-check/update_check.h"
-#include "android/utils/async.h"
-#include "android/utils/debug.h"
-#include "android/utils/path.h"
-#include "android/utils/property_file.h"
-#include "android/utils/lineinput.h"
-#include "android/utils/bufprint.h"
-#include "android/utils/filelock.h"
-#include "android/utils/ini.h"
-#include "android/utils/tempfile.h"
-#include "android/utils/timezone.h"
-#include "android/skin/winsys.h"
+#include "android/error-messages.h"
+#include "android/globals.h"
+#include "android/gps.h"
+#include "android/help.h"
+#include "android/hw-control.h"
+#include "android/hw-qemud.h"
 #include "android/main-common.h"
+#include "android/metrics/metrics.h"
 #include "android/multitouch-port.h"
 #include "android/network/control.h"
 #include "android/network/globals.h"
 #include "android/opengl/emugl_config.h"
-#include "android/ui-emu-agent.h"
-#include "android/globals.h"
-#include "android/help.h"
-#include "android-qemu2-glue/looper-qemu.h"
-#include "android/gps.h"
-#include "android/telephony/modem_driver.h"
-#include "android/hw-control.h"
-#include "android/hw-qemud.h"
-#include "android/utils/socket_drainer.h"
-#include "android/wear-agent/android_wear_agent.h"
-#include "android-qemu2-glue/android_qemud.h"
-#include "android-qemu2-glue/qemu-control-impl.h"
-#include "android-qemu2-glue/qemu-setup.h"
+#include "android/skin/winsys.h"
 #include "android/snapshot.h"
 #include "android/snaphost-android.h"
-#include "android/android.h"
-#include "android/camera/camera-service.h"
+#include "android/telephony/modem_driver.h"
+#include "android/update-check/update_check.h"
+#include "android/ui-emu-agent.h"
+#include "android/utils/async.h"
+#include "android/utils/bufprint.h"
+#include "android/utils/debug.h"
+#include "android/utils/filelock.h"
+#include "android/utils/ini.h"
+#include "android/utils/lineinput.h"
+#include "android/utils/path.h"
+#include "android/utils/property_file.h"
+#include "android/utils/socket_drainer.h"
+#include "android/utils/tempfile.h"
+#include "android/utils/timezone.h"
 #include "android/version.h"
+#include "android/wear-agent/android_wear_agent.h"
+#include "android-qemu2-glue/android_qemud.h"
+#include "android-qemu2-glue/looper-qemu.h"
+#include "android-qemu2-glue/qemu-control-impl.h"
+#include "android-qemu2-glue/qemu-setup.h"
+#include "android/camera/camera-service.h"
+
 #include "hw/input/goldfish_events.h"
 
 // this path has to be relative as AndroidEmu include paths go after the qemu2
@@ -156,9 +177,6 @@
 
 #define QEMU_CORE_VERSION "qemu2 " QEMU_VERSION
 
-int android_display_width  = 640;
-int android_display_height = 480;
-
 /////////////////////////////////////////////////////////////
 
 #define  LCD_DENSITY_LDPI      120
@@ -178,8 +196,6 @@
 
 #endif // CONFIG_ANDROID
 
-#define DEFAULT_RAM_SIZE 128
-
 #define MAX_VIRTIO_CONSOLES 1
 #define MAX_SCLP_CONSOLES 1
 #define QCOW2_SUFFIX "qcow2"
@@ -188,8 +204,8 @@
 static int data_dir_idx;
 const char *bios_name = NULL;
 enum vga_retrace_method vga_retrace_method = VGA_RETRACE_DUMB;
-DisplayType display_type = DT_DEFAULT;
-static int display_remote;
+int request_opengl = -1;
+int display_opengl;
 const char* keyboard_layout = NULL;
 ram_addr_t ram_size;
 const char *mem_path = NULL;
@@ -205,9 +221,7 @@
 static int full_screen = 0;
 static int no_frame = 0;
 int no_quit = 0;
-#ifdef CONFIG_GTK
 static bool grab_on_hover;
-#endif
 CharDriverState *serial_hds[MAX_SERIAL_PORTS];
 CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 CharDriverState *virtcon_hds[MAX_VIRTIO_CONSOLES];
@@ -215,12 +229,9 @@
 int win2k_install_hack = 0;
 int singlestep = 0;
 int smp_cpus = 1;
-int max_cpus = 0;
+int max_cpus = 1;
 int smp_cores = 1;
 int smp_threads = 1;
-#ifdef CONFIG_VNC
-const char *vnc_display;
-#endif
 int acpi_enabled = 1;
 int no_hpet = 0;
 int fd_bootchk = 1;
@@ -229,17 +240,16 @@
 int cursor_hide = 1;
 int graphic_rotate = 0;
 #ifdef CONFIG_ANDROID
-char* android_op_dns_server = NULL;
 int lcd_density = LCD_DENSITY_MDPI;
 extern char* op_http_proxy;
 extern char* android_op_ports;
 extern int android_op_ports_numbers[2];
 extern char* android_op_report_console;
+static const char* android_hw_file = NULL;
 #endif  // CONFIG_ANDROID
 const char *watchdog;
 QEMUOptionRom option_rom[MAX_OPTION_ROMS];
 int nb_option_roms;
-int semihosting_enabled = 0;
 int old_param = 0;
 const char *qemu_name;
 int alt_grab = 0;
@@ -254,19 +264,12 @@
 
 int icount_align_option;
 
-int nb_numa_nodes;
-int max_numa_nodeid;
-NodeInfo numa_info[MAX_NODES];
-
 /* The bytes in qemu_uuid[] are in the order specified by RFC4122, _not_ in the
  * little-endian "wire format" described in the SMBIOS 2.6 specification.
  */
 uint8_t qemu_uuid[16];
 bool qemu_uuid_set;
 
-static QEMUBootSetHandler *boot_set_handler;
-static void *boot_set_opaque;
-
 static NotifierList exit_notifiers =
     NOTIFIER_LIST_INITIALIZER(exit_notifiers);
 
@@ -287,6 +290,7 @@
 static int default_cdrom = 1;
 static int default_sdcard = 1;
 static int default_vga = 1;
+static int default_net = 1;
 
 static struct {
     const char *driver;
@@ -300,7 +304,6 @@
     { .driver = "ide-drive",            .flag = &default_cdrom     },
     { .driver = "scsi-cd",              .flag = &default_cdrom     },
     { .driver = "virtio-serial-pci",    .flag = &default_virtcon   },
-    { .driver = "virtio-serial-s390",   .flag = &default_virtcon   },
     { .driver = "virtio-serial",        .flag = &default_virtcon   },
     { .driver = "VGA",                  .flag = &default_vga       },
     { .driver = "isa-vga",              .flag = &default_vga       },
@@ -308,6 +311,7 @@
     { .driver = "isa-cirrus-vga",       .flag = &default_vga       },
     { .driver = "vmware-svga",          .flag = &default_vga       },
     { .driver = "qxl-vga",              .flag = &default_vga       },
+    { .driver = "virtio-vga",           .flag = &default_vga       },
 };
 
 static QemuOptsList qemu_rtc_opts = {
@@ -341,22 +345,6 @@
     },
 };
 
-static QemuOptsList qemu_trace_opts = {
-    .name = "trace",
-    .implied_opt_name = "trace",
-    .head = QTAILQ_HEAD_INITIALIZER(qemu_trace_opts.head),
-    .desc = {
-        {
-            .name = "events",
-            .type = QEMU_OPT_STRING,
-        },{
-            .name = "file",
-            .type = QEMU_OPT_STRING,
-        },
-        { /* end of list */ }
-    },
-};
-
 static QemuOptsList qemu_option_rom_opts = {
     .name = "option-rom",
     .implied_opt_name = "romfile",
@@ -379,84 +367,12 @@
     .merge_lists = true,
     .head = QTAILQ_HEAD_INITIALIZER(qemu_machine_opts.head),
     .desc = {
-        {
-            .name = "type",
-            .type = QEMU_OPT_STRING,
-            .help = "emulated machine"
-        }, {
-            .name = "accel",
-            .type = QEMU_OPT_STRING,
-            .help = "accelerator list",
-        }, {
-            .name = "kernel_irqchip",
-            .type = QEMU_OPT_BOOL,
-            .help = "use KVM in-kernel irqchip",
-        }, {
-            .name = "kvm_shadow_mem",
-            .type = QEMU_OPT_SIZE,
-            .help = "KVM shadow MMU size",
-        }, {
-            .name = "kernel",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel image file",
-        }, {
-            .name = "initrd",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux initial ramdisk file",
-        }, {
-            .name = "append",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel command line",
-        }, {
-            .name = "dtb",
-            .type = QEMU_OPT_STRING,
-            .help = "Linux kernel device tree file",
-        }, {
-            .name = "dumpdtb",
-            .type = QEMU_OPT_STRING,
-            .help = "Dump current dtb to a file and quit",
-        }, {
-            .name = "phandle_start",
-            .type = QEMU_OPT_NUMBER,
-            .help = "The first phandle ID we may generate dynamically",
-        }, {
-            .name = "dt_compatible",
-            .type = QEMU_OPT_STRING,
-            .help = "Overrides the \"compatible\" property of the dt root node",
-        }, {
-            .name = "dump-guest-core",
-            .type = QEMU_OPT_BOOL,
-            .help = "Include guest memory in  a core dump",
-        }, {
-            .name = "mem-merge",
-            .type = QEMU_OPT_BOOL,
-            .help = "enable/disable memory merge support",
-        },{
-            .name = "usb",
-            .type = QEMU_OPT_BOOL,
-            .help = "Set on/off to enable/disable usb",
-        },{
-            .name = "firmware",
-            .type = QEMU_OPT_STRING,
-            .help = "firmware image",
-        },{
-            .name = "kvm-type",
-            .type = QEMU_OPT_STRING,
-            .help = "Specifies the KVM virtualization mode (HV, PR)",
-        },{
-            .name = PC_MACHINE_MAX_RAM_BELOW_4G,
-            .type = QEMU_OPT_SIZE,
-            .help = "maximum ram below the 4G boundary (32bit boundary)",
-        }, {
-            .name = PC_MACHINE_VMPORT,
-            .type = QEMU_OPT_STRING,
-            .help = "Enable vmport (pc & q35)",
-        },{
-            .name = "iommu",
-            .type = QEMU_OPT_BOOL,
-            .help = "Set on/off to enable/disable Intel IOMMU (VT-d)",
-        },
-        { /* End of list */ }
+        /*
+         * no elements => accept any
+         * sanity checking will happen later
+         * when setting machine properties
+         */
+        { }
     },
 };
 
@@ -617,6 +533,57 @@
         }, {
             .name = "align",
             .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "sleep",
+            .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "rr",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "rrfile",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_semihosting_config_opts = {
+    .name = "semihosting-config",
+    .implied_opt_name = "enable",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_semihosting_config_opts.head),
+    .desc = {
+        {
+            .name = "enable",
+            .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "target",
+            .type = QEMU_OPT_STRING,
+        }, {
+            .name = "arg",
+            .type = QEMU_OPT_STRING,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_fw_cfg_opts = {
+    .name = "fw_cfg",
+    .implied_opt_name = "name",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_fw_cfg_opts.head),
+    .desc = {
+        {
+            .name = "name",
+            .type = QEMU_OPT_STRING,
+            .help = "Sets the fw_cfg name of the blob to be inserted",
+        }, {
+            .name = "file",
+            .type = QEMU_OPT_STRING,
+            .help = "Sets the name of the file from which\n"
+                    "the fw_cfg blob will be loaded",
+        }, {
+            .name = "string",
+            .type = QEMU_OPT_STRING,
+            .help = "Sets content of the blob to be inserted from a string",
         },
         { /* end of list */ }
     },
@@ -671,13 +638,11 @@
 
 static void res_free(void)
 {
-    if (boot_splash_filedata != NULL) {
-        g_free(boot_splash_filedata);
-        boot_splash_filedata = NULL;
-    }
+    g_free(boot_splash_filedata);
+    boot_splash_filedata = NULL;
 }
 
-static int default_driver_check(QemuOpts *opts, void *opaque)
+static int default_driver_check(void *opaque, QemuOpts *opts, Error **errp)
 {
     const char *driver = qemu_opt_get(opts, "driver");
     int i;
@@ -697,8 +662,8 @@
 
 static RunState current_run_state = RUN_STATE_PRELAUNCH;
 
-/* We use RUN_STATE_MAX but any invalid value will do */
-static RunState vmstop_requested = RUN_STATE_MAX;
+/* We use RUN_STATE__MAX but any invalid value will do */
+static RunState vmstop_requested = RUN_STATE__MAX;
 static QemuMutex vmstop_lock;
 
 typedef struct {
@@ -710,21 +675,35 @@
     /*     from      ->     to      */
     { RUN_STATE_DEBUG, RUN_STATE_RUNNING },
     { RUN_STATE_DEBUG, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_DEBUG, RUN_STATE_PRELAUNCH },
 
-    { RUN_STATE_INMIGRATE, RUN_STATE_RUNNING },
+    { RUN_STATE_INMIGRATE, RUN_STATE_INTERNAL_ERROR },
+    { RUN_STATE_INMIGRATE, RUN_STATE_IO_ERROR },
     { RUN_STATE_INMIGRATE, RUN_STATE_PAUSED },
+    { RUN_STATE_INMIGRATE, RUN_STATE_RUNNING },
+    { RUN_STATE_INMIGRATE, RUN_STATE_SHUTDOWN },
+    { RUN_STATE_INMIGRATE, RUN_STATE_SUSPENDED },
+    { RUN_STATE_INMIGRATE, RUN_STATE_WATCHDOG },
+    { RUN_STATE_INMIGRATE, RUN_STATE_GUEST_PANICKED },
+    { RUN_STATE_INMIGRATE, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_INMIGRATE, RUN_STATE_PRELAUNCH },
+    { RUN_STATE_INMIGRATE, RUN_STATE_POSTMIGRATE },
 
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PAUSED },
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_IO_ERROR, RUN_STATE_RUNNING },
     { RUN_STATE_IO_ERROR, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_IO_ERROR, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_PAUSED, RUN_STATE_RUNNING },
     { RUN_STATE_PAUSED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_PAUSED, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_POSTMIGRATE, RUN_STATE_RUNNING },
     { RUN_STATE_POSTMIGRATE, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_POSTMIGRATE, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_PRELAUNCH, RUN_STATE_RUNNING },
     { RUN_STATE_PRELAUNCH, RUN_STATE_FINISH_MIGRATE },
@@ -732,8 +711,10 @@
 
     { RUN_STATE_FINISH_MIGRATE, RUN_STATE_RUNNING },
     { RUN_STATE_FINISH_MIGRATE, RUN_STATE_POSTMIGRATE },
+    { RUN_STATE_FINISH_MIGRATE, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_RESTORE_VM, RUN_STATE_RUNNING },
+    { RUN_STATE_RESTORE_VM, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_RUNNING, RUN_STATE_DEBUG },
     { RUN_STATE_RUNNING, RUN_STATE_INTERNAL_ERROR },
@@ -750,34 +731,50 @@
 
     { RUN_STATE_SHUTDOWN, RUN_STATE_PAUSED },
     { RUN_STATE_SHUTDOWN, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_SHUTDOWN, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_DEBUG, RUN_STATE_SUSPENDED },
     { RUN_STATE_RUNNING, RUN_STATE_SUSPENDED },
     { RUN_STATE_SUSPENDED, RUN_STATE_RUNNING },
     { RUN_STATE_SUSPENDED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_SUSPENDED, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_WATCHDOG, RUN_STATE_RUNNING },
     { RUN_STATE_WATCHDOG, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_WATCHDOG, RUN_STATE_PRELAUNCH },
 
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_RUNNING },
     { RUN_STATE_GUEST_PANICKED, RUN_STATE_FINISH_MIGRATE },
+    { RUN_STATE_GUEST_PANICKED, RUN_STATE_PRELAUNCH },
 
-    { RUN_STATE_MAX, RUN_STATE_MAX },
+    { RUN_STATE__MAX, RUN_STATE__MAX },
 };
 
-static bool runstate_valid_transitions[RUN_STATE_MAX][RUN_STATE_MAX];
+static bool runstate_valid_transitions[RUN_STATE__MAX][RUN_STATE__MAX];
 
 bool runstate_check(RunState state)
 {
     return current_run_state == state;
 }
 
+bool runstate_store(char *str, size_t size)
+{
+    const char *state = RunState_lookup[current_run_state];
+    size_t len = strlen(state) + 1;
+
+    if (len > size) {
+        return false;
+    }
+    memcpy(str, state, len);
+    return true;
+}
+
 static void runstate_init(void)
 {
     const RunStateTransition *p;
 
     memset(&runstate_valid_transitions, 0, sizeof(runstate_valid_transitions));
-    for (p = &runstate_transitions_def[0]; p->from != RUN_STATE_MAX; p++) {
+    for (p = &runstate_transitions_def[0]; p->from != RUN_STATE__MAX; p++) {
         runstate_valid_transitions[p->from][p->to] = true;
     }
 
@@ -787,12 +784,16 @@
 /* This function will abort() on invalid state transitions */
 void runstate_set(RunState new_state)
 {
-    assert(new_state < RUN_STATE_MAX);
+    assert(new_state < RUN_STATE__MAX);
+
+    if (current_run_state == new_state) {
+        return;
+    }
 
     if (!runstate_valid_transitions[current_run_state][new_state]) {
-        fprintf(stderr, "ERROR: invalid runstate transition: '%s' -> '%s'\n",
-                RunState_lookup[current_run_state],
-                RunState_lookup[new_state]);
+        error_report("invalid runstate transition: '%s' -> '%s'",
+                     RunState_lookup[current_run_state],
+                     RunState_lookup[new_state]);
         abort();
     }
     trace_runstate_set(new_state);
@@ -825,9 +826,9 @@
 {
     qemu_mutex_lock(&vmstop_lock);
     *r = vmstop_requested;
-    vmstop_requested = RUN_STATE_MAX;
+    vmstop_requested = RUN_STATE__MAX;
     qemu_mutex_unlock(&vmstop_lock);
-    return *r < RUN_STATE_MAX;
+    return *r < RUN_STATE__MAX;
 }
 
 void qemu_system_vmstop_request_prepare(void)
@@ -843,7 +844,7 @@
 }
 
 #ifdef CONFIG_ANDROID
-static int create_qcow2_images() {
+static int create_qcow2_images(void) {
     /* First, determine if any of the backing images have been altered.
      * QCoW2 images won't work in that case, and need to be recreated (this
      * will obliterate previous snapshots).
@@ -929,7 +930,7 @@
                 "raw",
                 NULL,
                 -1,
-                BDRV_O_CACHE_WB,
+                0,
                 &img_creation_error,
                 false);
         }
@@ -961,7 +962,7 @@
     RunState requested;
 
     qemu_vmstop_requested(&requested);
-    if (runstate_is_running() && requested == RUN_STATE_MAX) {
+    if (runstate_is_running() && requested == RUN_STATE__MAX) {
         return;
     }
 
@@ -986,13 +987,17 @@
 /***********************************************************/
 /* real time host monotonic timer */
 
+static time_t qemu_time(void)
+{
+    return qemu_clock_get_ms(QEMU_CLOCK_HOST) / 1000;
+}
+
 /***********************************************************/
 /* host time/date access */
 void qemu_get_timedate(struct tm *tm, int offset)
 {
-    time_t ti;
+    time_t ti = qemu_time();
 
-    time(&ti);
     ti += offset;
     if (rtc_date_offset == -1) {
         if (rtc_utc)
@@ -1020,7 +1025,7 @@
     else
         seconds = mktimegm(tm) + rtc_date_offset;
 
-    return seconds - time(NULL);
+    return seconds - qemu_time();
 }
 
 static bool configure_rtc_date_offset(const char *startdate, int legacy)
@@ -1054,13 +1059,13 @@
         rtc_start_date = mktimegm(&tm);
         if (rtc_start_date == -1) {
         date_fail:
-            fprintf(stderr, "Invalid date format. Valid formats are:\n"
-                            "'2006-06-17T16:01:21' or '2006-06-17'\n");
+            error_report("invalid date format");
+            error_printf("valid formats: "
+                         "'2006-06-17T16:01:21' or '2006-06-17'\n");
             return false;
         }
-        rtc_date_offset = time(NULL) - rtc_start_date;
+        rtc_date_offset = qemu_time() - rtc_start_date;
     }
-
     return true;
 }
 
@@ -1073,7 +1078,11 @@
         if (!strcmp(value, "utc")) {
             rtc_utc = 1;
         } else if (!strcmp(value, "localtime")) {
+            Error *blocker = NULL;
             rtc_utc = 0;
+            error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED,
+                      "-rtc base=localtime");
+            replay_add_blocker(blocker);
         } else {
             if (!configure_rtc_date_offset(value, 0)) {
                 return false;
@@ -1089,31 +1098,27 @@
         } else if (!strcmp(value, "vm")) {
             rtc_clock = QEMU_CLOCK_VIRTUAL;
         } else {
-            fprintf(stderr, "qemu: invalid option value '%s'\n", value);
+            error_report("invalid option value '%s'", value);
             return false;
         }
     }
     value = qemu_opt_get(opts, "driftfix");
     if (value) {
         if (!strcmp(value, "slew")) {
-            static GlobalProperty slew_lost_ticks[] = {
-                {
-                    .driver   = "mc146818rtc",
-                    .property = "lost_tick_policy",
-                    .value    = "slew",
-                },
-                { /* end of list */ }
+            static GlobalProperty slew_lost_ticks = {
+                .driver   = "mc146818rtc",
+                .property = "lost_tick_policy",
+                .value    = "slew",
             };
 
-            qdev_prop_register_global_list(slew_lost_ticks);
+            qdev_prop_register_global(&slew_lost_ticks);
         } else if (!strcmp(value, "none")) {
             /* discard is default */
         } else {
-            fprintf(stderr, "qemu: invalid option value '%s'\n", value);
+            error_report("invalid option value '%s'", value);
             return false;
         }
     }
-
     return true;
 }
 
@@ -1137,7 +1142,7 @@
     bdaddr_t bdaddr;
 
     if (nb_hcis >= MAX_NICS) {
-        fprintf(stderr, "qemu: Too many bluetooth HCIs (max %i).\n", MAX_NICS);
+        error_report("too many bluetooth HCIs (max %i)", MAX_NICS);
         return -1;
     }
 
@@ -1163,8 +1168,8 @@
     struct bt_scatternet_s *vlan = qemu_find_bt_vlan(vlan_id);
 
     if (!vlan->slave)
-        fprintf(stderr, "qemu: warning: adding a VHCI to "
-                        "an empty scatternet %i\n", vlan_id);
+        error_report("warning: adding a VHCI to an empty scatternet %i",
+                     vlan_id);
 
     bt_vhci_init(bt_new_hci(vlan));
 }
@@ -1182,7 +1187,7 @@
     if (endp) {
         vlan_id = strtol(endp + 6, &endp, 0);
         if (*endp) {
-            fprintf(stderr, "qemu: unrecognised bluetooth vlan Id\n");
+            error_report("unrecognised bluetooth vlan Id");
             return 0;
         }
     }
@@ -1190,13 +1195,13 @@
     vlan = qemu_find_bt_vlan(vlan_id);
 
     if (!vlan->slave)
-        fprintf(stderr, "qemu: warning: adding a slave device to "
-                        "an empty scatternet %i\n", vlan_id);
+        error_report("warning: adding a slave device to an empty scatternet %i",
+                     vlan_id);
 
     if (!strcmp(devname, "keyboard"))
         return bt_keyboard_init(vlan);
 
-    fprintf(stderr, "qemu: unsupported bluetooth device `%s'\n", devname);
+    error_report("unsupported bluetooth device '%s'", devname);
     return 0;
 }
 
@@ -1219,11 +1224,11 @@
                 if (strstart(endp, ",vlan=", &p)) {
                     vlan = strtol(p, (char **) &endp, 0);
                     if (*endp) {
-                        fprintf(stderr, "qemu: bad scatternet '%s'\n", p);
+                        error_report("bad scatternet '%s'", p);
                         return 1;
                     }
                 } else {
-                    fprintf(stderr, "qemu: bad parameter '%s'\n", endp + 1);
+                    error_report("bad parameter '%s'", endp + 1);
                     return 1;
                 }
             } else
@@ -1235,23 +1240,22 @@
     } else if (strstart(opt, "device:", &endp))
         return !bt_device_add(endp);
 
-    fprintf(stderr, "qemu: bad bluetooth parameter '%s'\n", opt);
+    error_report("bad bluetooth parameter '%s'", opt);
     return 1;
 }
 
-static int parse_sandbox(QemuOpts *opts, void *opaque)
+static int parse_sandbox(void *opaque, QemuOpts *opts, Error **errp)
 {
     /* FIXME: change this to true for 1.3 */
     if (qemu_opt_get_bool(opts, "enable", false)) {
 #ifdef CONFIG_SECCOMP
         if (seccomp_start() < 0) {
-            qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                          "failed to install seccomp syscall filter in the kernel");
+            error_report("failed to install seccomp syscall filter "
+                         "in the kernel");
             return -1;
         }
 #else
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "sandboxing request but seccomp is not compiled into this build");
+        error_report("seccomp support is disabled");
         return -1;
 #endif
     }
@@ -1259,7 +1263,7 @@
     return 0;
 }
 
-static int parse_name(QemuOpts *opts, void *opaque)
+static int parse_name(void *opaque, QemuOpts *opts, Error **errp)
 {
     const char *proc_name;
 
@@ -1276,32 +1280,30 @@
     return 0;
 }
 
-bool usb_enabled(bool default_usb)
+bool defaults_enabled(void)
 {
-    return qemu_opt_get_bool(qemu_get_machine_opts(), "usb",
-                             has_defaults && default_usb);
+    return has_defaults;
 }
 
 #ifndef _WIN32
-static int parse_add_fd(QemuOpts *opts, void *opaque)
+static int parse_add_fd(void *opaque, QemuOpts *opts, Error **errp)
 {
     int fd, dupfd, flags;
     int64_t fdset_id;
     const char *fd_opaque = NULL;
+    AddfdInfo *fdinfo;
 
     fd = qemu_opt_get_number(opts, "fd", -1);
     fdset_id = qemu_opt_get_number(opts, "set", -1);
     fd_opaque = qemu_opt_get(opts, "opaque");
 
     if (fd < 0) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "fd option is required and must be non-negative");
+        error_report("fd option is required and must be non-negative");
         return -1;
     }
 
     if (fd <= STDERR_FILENO) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "fd cannot be a standard I/O stream");
+        error_report("fd cannot be a standard I/O stream");
         return -1;
     }
 
@@ -1311,14 +1313,12 @@
      */
     flags = fcntl(fd, F_GETFD);
     if (flags == -1 || (flags & FD_CLOEXEC)) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "fd is not valid or already in use");
+        error_report("fd is not valid or already in use");
         return -1;
     }
 
     if (fdset_id < 0) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "set option is required and must be non-negative");
+        error_report("set option is required and must be non-negative");
         return -1;
     }
 
@@ -1331,19 +1331,19 @@
     }
 #endif
     if (dupfd == -1) {
-        qerror_report(ERROR_CLASS_GENERIC_ERROR,
-                      "Error duplicating fd: %s", strerror(errno));
+        error_report("error duplicating fd: %s", strerror(errno));
         return -1;
     }
 
     /* add the duplicate fd, and optionally the opaque string, to the fd set */
-    monitor_fdset_add_fd(dupfd, true, fdset_id, fd_opaque ? true : false,
-                         fd_opaque, NULL);
+    fdinfo = monitor_fdset_add_fd(dupfd, true, fdset_id, !!fd_opaque, fd_opaque,
+                                  &error_abort);
+    g_free(fdinfo);
 
     return 0;
 }
 
-static int cleanup_add_fd(QemuOpts *opts, void *opaque)
+static int cleanup_add_fd(void *opaque, QemuOpts *opts, Error **errp)
 {
     int fd;
 
@@ -1364,17 +1364,17 @@
 #define MTD_OPTS ""
 #define SD_OPTS ""
 
-static int drive_init_func(QemuOpts *opts, void *opaque)
+static int drive_init_func(void *opaque, QemuOpts *opts, Error **errp)
 {
     BlockInterfaceType *block_default_type = opaque;
 
     return drive_new(opts, *block_default_type) == NULL;
 }
 
-static int drive_enable_snapshot(QemuOpts *opts, void *opaque)
+static int drive_enable_snapshot(void *opaque, QemuOpts *opts, Error **errp)
 {
     if (qemu_opt_get(opts, "snapshot") == NULL) {
-        qemu_opt_set(opts, "snapshot", "on");
+        qemu_opt_set(opts, "snapshot", "on", &error_abort);
     }
     return 0;
 }
@@ -1391,7 +1391,7 @@
 
     opts = drive_add(type, index, NULL, optstr);
     if (snapshot) {
-        drive_enable_snapshot(opts, NULL);
+        drive_enable_snapshot(NULL, opts, NULL);
     }
 
     dinfo = drive_new(opts, type);
@@ -1402,67 +1402,6 @@
     return true;
 }
 
-void qemu_register_boot_set(QEMUBootSetHandler *func, void *opaque)
-{
-    boot_set_handler = func;
-    boot_set_opaque = opaque;
-}
-
-int qemu_boot_set(const char *boot_order)
-{
-    if (!boot_set_handler) {
-        return -EINVAL;
-    }
-    return boot_set_handler(boot_set_opaque, boot_order);
-}
-
-static bool validate_bootdevices(const char *devices)
-{
-    /* We just do some generic consistency checks */
-    const char *p;
-    int bitmap = 0;
-
-    for (p = devices; *p != '\0'; p++) {
-        /* Allowed boot devices are:
-         * a-b: floppy disk drives
-         * c-f: IDE disk drives
-         * g-m: machine implementation dependent drives
-         * n-p: network devices
-         * It's up to each machine implementation to check if the given boot
-         * devices match the actual hardware implementation and firmware
-         * features.
-         */
-        if (*p < 'a' || *p > 'p') {
-            fprintf(stderr, "Invalid boot device '%c'\n", *p);
-            return false;
-        }
-        if (bitmap & (1 << (*p - 'a'))) {
-            fprintf(stderr, "Boot device '%c' was given twice\n", *p);
-            return false;
-        }
-        bitmap |= 1 << (*p - 'a');
-    }
-
-    return true;
-}
-
-static void restore_boot_order(void *opaque)
-{
-    char *normal_boot_order = opaque;
-    static int first = 1;
-
-    /* Restore boot order and remove ourselves after the first boot */
-    if (first) {
-        first = 0;
-        return;
-    }
-
-    qemu_boot_set(normal_boot_order);
-
-    qemu_unregister_reset(restore_boot_order, normal_boot_order);
-    g_free(normal_boot_order);
-}
-
 static QemuOptsList qemu_smp_opts = {
     .name = "smp-opts",
     .implied_opt_name = "cpus",
@@ -1492,7 +1431,6 @@
 static bool smp_parse(QemuOpts *opts)
 {
     if (opts) {
-
         unsigned cpus    = qemu_opt_get_number(opts, "cpus", 0);
         unsigned sockets = qemu_opt_get_number(opts, "sockets", 0);
         unsigned cores   = qemu_opt_get_number(opts, "cores", 0);
@@ -1506,36 +1444,51 @@
             if (cpus == 0) {
                 cpus = cores * threads * sockets;
             }
-        } else {
-            if (cores == 0) {
-                threads = threads > 0 ? threads : 1;
-                cores = cpus / (sockets * threads);
-            } else {
-                threads = cpus / (cores * sockets);
-            }
+        } else if (cores == 0) {
+            threads = threads > 0 ? threads : 1;
+            cores = cpus / (sockets * threads);
+            cores = cores > 0 ? cores : 1;
+        } else if (threads == 0) {
+            threads = cpus / (cores * sockets);
+            threads = threads > 0 ? threads : 1;
+        } else if (sockets * cores * threads < cpus) {
+            error_report("cpu topology: "
+                         "sockets (%u) * cores (%u) * threads (%u) < "
+                         "smp_cpus (%u)",
+                         sockets, cores, threads, cpus);
+            return false;
         }
 
-        max_cpus = qemu_opt_get_number(opts, "maxcpus", 0);
+        max_cpus = qemu_opt_get_number(opts, "maxcpus", cpus);
+
+        if (max_cpus > MAX_CPUMASK_BITS) {
+            error_report("unsupported number of maxcpus");
+            return false;
+        }
+
+        if (max_cpus < cpus) {
+            error_report("maxcpus must be equal to or greater than smp");
+            return false;
+        }
+
+        if (sockets * cores * threads > max_cpus) {
+            error_report("cpu topology: "
+                         "sockets (%u) * cores (%u) * threads (%u) > "
+                         "maxcpus (%u)",
+                         sockets, cores, threads, max_cpus);
+            return false;
+        }
 
         smp_cpus = cpus;
-        smp_cores = cores > 0 ? cores : 1;
-        smp_threads = threads > 0 ? threads : 1;
-
+        smp_cores = cores;
+        smp_threads = threads;
     }
 
-    if (max_cpus == 0) {
-        max_cpus = smp_cpus;
+    if (smp_cpus > 1) {
+        Error *blocker = NULL;
+        error_setg(&blocker, QERR_REPLAY_NOT_SUPPORTED, "smp");
+        replay_add_blocker(blocker);
     }
-
-    if (max_cpus > MAX_CPUMASK_BITS) {
-        fprintf(stderr, "Unsupported number of maxcpus\n");
-        return false;
-    }
-    if (max_cpus < smp_cpus) {
-        fprintf(stderr, "maxcpus must be equal to or greater than smp\n");
-        return false;
-    }
-
     return true;
 }
 
@@ -1543,11 +1496,10 @@
 {
     if (enable_mlock) {
         if (os_mlock() < 0) {
-            fprintf(stderr, "qemu: locking memory failed\n");
+            error_report("locking memory failed");
             return false;
         }
     }
-
     return true;
 }
 
@@ -1558,6 +1510,88 @@
 }
 
 /***********************************************************/
+/* Semihosting */
+
+typedef struct SemihostingConfig {
+    bool enabled;
+    SemihostingTarget target;
+    const char **argv;
+    int argc;
+    const char *cmdline; /* concatenated argv */
+} SemihostingConfig;
+
+static SemihostingConfig semihosting;
+
+bool semihosting_enabled(void)
+{
+    return semihosting.enabled;
+}
+
+SemihostingTarget semihosting_get_target(void)
+{
+    return semihosting.target;
+}
+
+const char *semihosting_get_arg(int i)
+{
+    if (i >= semihosting.argc) {
+        return NULL;
+    }
+    return semihosting.argv[i];
+}
+
+int semihosting_get_argc(void)
+{
+    return semihosting.argc;
+}
+
+const char *semihosting_get_cmdline(void)
+{
+    if (semihosting.cmdline == NULL && semihosting.argc > 0) {
+        semihosting.cmdline = g_strjoinv(" ", (gchar **)semihosting.argv);
+    }
+    return semihosting.cmdline;
+}
+
+static int add_semihosting_arg(void *opaque,
+                               const char *name, const char *val,
+                               Error **errp)
+{
+    SemihostingConfig *s = opaque;
+    if (strcmp(name, "arg") == 0) {
+        s->argc++;
+        /* one extra element as g_strjoinv() expects NULL-terminated array */
+        s->argv = g_realloc(s->argv, (s->argc + 1) * sizeof(void *));
+        s->argv[s->argc - 1] = val;
+        s->argv[s->argc] = NULL;
+    }
+    return 0;
+}
+
+/* Use strings passed via -kernel/-append to initialize semihosting.argv[] */
+static inline void semihosting_arg_fallback(const char *file, const char *cmd)
+{
+    char *cmd_token;
+
+    /* argv[0] */
+    add_semihosting_arg(&semihosting, "arg", file, NULL);
+
+    /* split -append and initialize argv[1..n] */
+    cmd_token = strtok(g_strdup(cmd), " ");
+    while (cmd_token) {
+        add_semihosting_arg(&semihosting, "arg", cmd_token, NULL);
+        cmd_token = strtok(NULL, " ");
+    }
+}
+
+/* Now we still need this for compatibility with XEN. */
+bool has_igd_gfx_passthru;
+static void igd_gfx_passthru(void)
+{
+    has_igd_gfx_passthru = current_machine->igd_gfx_passthru;
+}
+
+/***********************************************************/
 /* USB devices */
 
 static int usb_device_add(const char *devname)
@@ -1567,7 +1601,7 @@
     const char *p;
 #endif
 
-    if (!usb_enabled(false)) {
+    if (!machine_usb(current_machine)) {
         return -1;
     }
 
@@ -1599,7 +1633,7 @@
         return -1;
     }
 
-    if (!usb_enabled(false)) {
+    if (!machine_usb(current_machine)) {
         return -1;
     }
 
@@ -1617,12 +1651,12 @@
     int r;
     r = usb_device_add(cmdline);
     if (r < 0) {
-        fprintf(stderr, "qemu: could not add USB device '%s'\n", cmdline);
+        error_report("could not add USB device '%s'", cmdline);
     }
     return r;
 }
 
-void do_usb_add(Monitor *mon, const QDict *qdict)
+void hmp_usb_add(Monitor *mon, const QDict *qdict)
 {
     const char *devname = qdict_get_str(qdict, "devname");
     if (usb_device_add(devname) < 0) {
@@ -1630,7 +1664,7 @@
     }
 }
 
-void do_usb_del(Monitor *mon, const QDict *qdict)
+void hmp_usb_del(Monitor *mon, const QDict *qdict)
 {
     const char *devname = qdict_get_str(qdict, "devname");
     if (usb_device_del(devname) < 0) {
@@ -1643,54 +1677,6 @@
 
 MachineState *current_machine;
 
-static void machine_class_init(ObjectClass *oc, void *data)
-{
-    MachineClass *mc = MACHINE_CLASS(oc);
-    QEMUMachine *qm = data;
-
-    mc->family = qm->family;
-    mc->name = qm->name;
-    mc->alias = qm->alias;
-    mc->desc = qm->desc;
-    mc->init = qm->init;
-    mc->reset = qm->reset;
-    mc->hot_add_cpu = qm->hot_add_cpu;
-    mc->kvm_type = qm->kvm_type;
-    mc->block_default_type = qm->block_default_type;
-    mc->units_per_default_bus = qm->units_per_default_bus;
-    mc->max_cpus = qm->max_cpus;
-    mc->no_serial = qm->no_serial;
-    mc->no_parallel = qm->no_parallel;
-    mc->use_virtcon = qm->use_virtcon;
-    mc->use_sclp = qm->use_sclp;
-    mc->no_floppy = qm->no_floppy;
-    mc->no_cdrom = qm->no_cdrom;
-    mc->no_sdcard = qm->no_sdcard;
-    mc->has_dynamic_sysbus = qm->has_dynamic_sysbus;
-    mc->is_default = qm->is_default;
-    mc->default_machine_opts = qm->default_machine_opts;
-    mc->default_boot_order = qm->default_boot_order;
-    mc->default_display = qm->default_display;
-    mc->compat_props = qm->compat_props;
-    mc->hw_version = qm->hw_version;
-}
-
-int qemu_register_machine(QEMUMachine *m)
-{
-    char *name = g_strconcat(m->name, TYPE_MACHINE_SUFFIX, NULL);
-    TypeInfo ti = {
-        .name       = name,
-        .parent     = TYPE_MACHINE,
-        .class_init = machine_class_init,
-        .class_data = (void *)m,
-    };
-
-    type_register(&ti);
-    g_free(name);
-
-    return 0;
-}
-
 static MachineClass *find_machine(const char *name)
 {
     GSList *el, *machines = object_class_get_list(TYPE_MACHINE, false);
@@ -1755,6 +1741,7 @@
 
         info->name = g_strdup(mc->name);
         info->cpu_max = !mc->max_cpus ? 1 : mc->max_cpus;
+        info->hotpluggable_cpus = !!mc->query_hotpluggable_cpus;
 
         entry = g_malloc0(sizeof(*entry));
         entry->value = info;
@@ -1766,6 +1753,33 @@
     return mach_list;
 }
 
+static int machine_help_func(QemuOpts *opts, MachineState *machine)
+{
+    ObjectProperty *prop;
+    ObjectPropertyIterator iter;
+
+    if (!qemu_opt_has_help_opt(opts)) {
+        return 0;
+    }
+
+    object_property_iter_init(&iter, OBJECT(machine));
+    while ((prop = object_property_iter_next(&iter))) {
+        if (!prop->set) {
+            continue;
+        }
+
+        error_printf("%s.%s=%s", MACHINE_GET_CLASS(machine)->name,
+                     prop->name, prop->type);
+        if (prop->description) {
+            error_printf(" (%s)\n", prop->description);
+        } else {
+            error_printf("\n");
+        }
+    }
+
+    return 1;
+}
+
 /***********************************************************/
 /* main execution loop */
 
@@ -1850,14 +1864,14 @@
 static void qemu_kill_report(void)
 {
     if (!qtest_driver() && shutdown_signal != -1) {
-        fprintf(stderr, "qemu: terminating on signal %d", shutdown_signal);
         if (shutdown_pid == 0) {
             /* This happens for eg ^C at the terminal, so it's worth
              * avoiding printing an odd message in that case.
              */
-            fputc('\n', stderr);
+            error_report("terminating on signal %d", shutdown_signal);
         } else {
-            fprintf(stderr, " from pid " FMT_pid "\n", shutdown_pid);
+            error_report("terminating on signal %d from pid " FMT_pid,
+                         shutdown_signal, shutdown_pid);
         }
         shutdown_signal = -1;
     }
@@ -1866,15 +1880,21 @@
 static int qemu_reset_requested(void)
 {
     int r = reset_requested;
-    reset_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_RESET_REQUESTED)) {
+        reset_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static int qemu_suspend_requested(void)
 {
     int r = suspend_requested;
-    suspend_requested = 0;
-    return r;
+    if (r && replay_checkpoint(CHECKPOINT_SUSPEND_REQUESTED)) {
+        suspend_requested = 0;
+        return r;
+    }
+    return false;
 }
 
 static WakeupReason qemu_wakeup_requested(void)
@@ -1934,6 +1954,8 @@
 
     mc = current_machine ? MACHINE_GET_CLASS(current_machine) : NULL;
 
+    cpu_synchronize_all_states();
+
     if (mc && mc->reset) {
         mc->reset();
     } else {
@@ -1945,6 +1967,15 @@
     cpu_synchronize_all_post_reset();
 }
 
+void qemu_system_guest_panicked(void)
+{
+    if (current_cpu) {
+        current_cpu->crash_occurred = true;
+    }
+    qapi_event_send_guest_panicked(GUEST_PANIC_ACTION_PAUSE, &error_abort);
+    vm_stop(RUN_STATE_GUEST_PANICKED);
+}
+
 void qemu_system_reset_request(void)
 {
     if (no_reboot) {
@@ -2013,12 +2044,18 @@
     shutdown_signal = signal;
     shutdown_pid = pid;
     no_shutdown = 0;
-    qemu_system_shutdown_request();
+
+    /* Cannot call qemu_system_shutdown_request directly because
+     * we are in a signal handler.
+     */
+    shutdown_requested = 1;
+    qemu_notify_event();
 }
 
 void qemu_system_shutdown_request(void)
 {
     trace_qemu_system_shutdown_request();
+    replay_shutdown_request();
     shutdown_requested = 1;
     qemu_notify_event();
 }
@@ -2067,16 +2104,15 @@
     }
     if (qemu_reset_requested()) {
         pause_all_vcpus();
-        cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_REPORT);
         resume_all_vcpus();
-        if (runstate_needs_reset()) {
-            runstate_set(RUN_STATE_PAUSED);
+        if (!runstate_check(RUN_STATE_RUNNING) &&
+                !runstate_check(RUN_STATE_INMIGRATE)) {
+            runstate_set(RUN_STATE_PRELAUNCH);
         }
     }
     if (qemu_wakeup_requested()) {
         pause_all_vcpus();
-        cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         notifier_list_notify(&wakeup_notifiers, &wakeup_reason);
         wakeup_reason = QEMU_WAKEUP_REASON_NONE;
@@ -2100,13 +2136,19 @@
     int64_t ti;
 #endif
 
+#ifdef CONFIG_HAX
     if (hax_sync_vcpus() < 0) {
         fprintf(stderr, "Internal error: hax sync failed\n");
         return;
     }
+#endif
 
     do {
+#ifdef CONFIG_HAX
         nonblocking = !kvm_enabled() && !xen_enabled() && !hax_enabled() && last_io > 0;
+#else
+        nonblocking = !kvm_enabled() && !xen_enabled() && last_io > 0;
+#endif
 #ifdef CONFIG_PROFILER
         ti = profile_getclock();
 #endif
@@ -2119,7 +2161,8 @@
 
 static void version(void)
 {
-    printf("QEMU emulator version " QEMU_VERSION " " QEMU_PKGVERSION ", Copyright (c) 2003-2008 Fabrice Bellard\n");
+    printf("QEMU emulator version " QEMU_VERSION " " QEMU_PKGVERSION ", "
+           QEMU_COPYRIGHT "\n");
 }
 
 static void help(void)
@@ -2156,89 +2199,88 @@
     { NULL },
 };
 
-static bool vga_available(void)
+typedef struct VGAInterfaceInfo {
+    const char *opt_name;    /* option name */
+    const char *name;        /* human-readable name */
+    /* Class names indicating that support is available.
+     * If no class is specified, the interface is always available */
+    const char *class_names[2];
+} VGAInterfaceInfo;
+
+static VGAInterfaceInfo vga_interfaces[VGA_TYPE_MAX] = {
+    [VGA_NONE] = {
+        .opt_name = "none",
+    },
+    [VGA_STD] = {
+        .opt_name = "std",
+        .name = "standard VGA",
+        .class_names = { "VGA", "isa-vga" },
+    },
+    [VGA_CIRRUS] = {
+        .opt_name = "cirrus",
+        .name = "Cirrus VGA",
+        .class_names = { "cirrus-vga", "isa-cirrus-vga" },
+    },
+    [VGA_VMWARE] = {
+        .opt_name = "vmware",
+        .name = "VMWare SVGA",
+        .class_names = { "vmware-svga" },
+    },
+    [VGA_VIRTIO] = {
+        .opt_name = "virtio",
+        .name = "Virtio VGA",
+        .class_names = { "virtio-vga" },
+    },
+    [VGA_QXL] = {
+        .opt_name = "qxl",
+        .name = "QXL VGA",
+        .class_names = { "qxl-vga" },
+    },
+    [VGA_TCX] = {
+        .opt_name = "tcx",
+        .name = "TCX framebuffer",
+        .class_names = { "SUNW,tcx" },
+    },
+    [VGA_CG3] = {
+        .opt_name = "cg3",
+        .name = "CG3 framebuffer",
+        .class_names = { "cgthree" },
+    },
+    [VGA_XENFB] = {
+        .opt_name = "xenfb",
+    },
+};
+
+static bool vga_interface_available(VGAInterfaceType t)
 {
-    return object_class_by_name("VGA") || object_class_by_name("isa-vga");
+    VGAInterfaceInfo *ti = &vga_interfaces[t];
+
+    assert(t < VGA_TYPE_MAX);
+    return !ti->class_names[0] ||
+           object_class_by_name(ti->class_names[0]) ||
+           object_class_by_name(ti->class_names[1]);
 }
 
-static bool cirrus_vga_available(void)
-{
-    return object_class_by_name("cirrus-vga")
-           || object_class_by_name("isa-cirrus-vga");
-}
-
-static bool vmware_vga_available(void)
-{
-    return object_class_by_name("vmware-svga");
-}
-
-static bool qxl_vga_available(void)
-{
-    return object_class_by_name("qxl-vga");
-}
-
-static bool tcx_vga_available(void)
-{
-    return object_class_by_name("SUNW,tcx");
-}
-
-static bool cg3_vga_available(void)
-{
-    return object_class_by_name("cgthree");
-}
-
-static bool select_vgahw (const char *p)
+static bool select_vgahw(const char *p)
 {
     const char *opts;
+    int t;
 
     assert(vga_interface_type == VGA_NONE);
-    if (strstart(p, "std", &opts)) {
-        if (vga_available()) {
-            vga_interface_type = VGA_STD;
-        } else {
-            fprintf(stderr, "Error: standard VGA not available\n");
-            return false;
+    for (t = 0; t < VGA_TYPE_MAX; t++) {
+        VGAInterfaceInfo *ti = &vga_interfaces[t];
+        if (ti->opt_name && strstart(p, ti->opt_name, &opts)) {
+            if (!vga_interface_available(t)) {
+                error_report("%s not available", ti->name);
+                return false;
+            }
+            vga_interface_type = t;
+            break;
         }
-    } else if (strstart(p, "cirrus", &opts)) {
-        if (cirrus_vga_available()) {
-            vga_interface_type = VGA_CIRRUS;
-        } else {
-            fprintf(stderr, "Error: Cirrus VGA not available\n");
-            return false;
-        }
-    } else if (strstart(p, "vmware", &opts)) {
-        if (vmware_vga_available()) {
-            vga_interface_type = VGA_VMWARE;
-        } else {
-            fprintf(stderr, "Error: VMWare SVGA not available\n");
-            return false;
-        }
-    } else if (strstart(p, "xenfb", &opts)) {
-        vga_interface_type = VGA_XENFB;
-    } else if (strstart(p, "qxl", &opts)) {
-        if (qxl_vga_available()) {
-            vga_interface_type = VGA_QXL;
-        } else {
-            fprintf(stderr, "Error: QXL VGA not available\n");
-            return false;
-        }
-    } else if (strstart(p, "tcx", &opts)) {
-        if (tcx_vga_available()) {
-            vga_interface_type = VGA_TCX;
-        } else {
-            fprintf(stderr, "Error: TCX framebuffer not available\n");
-            return false;
-        }
-    } else if (strstart(p, "cg3", &opts)) {
-        if (cg3_vga_available()) {
-            vga_interface_type = VGA_CG3;
-        } else {
-            fprintf(stderr, "Error: CG3 framebuffer not available\n");
-            return false;
-        }
-    } else if (!strstart(p, "none", &opts)) {
+    }
+    if (t == VGA_TYPE_MAX) {
     invalid_vga:
-        fprintf(stderr, "Unknown vga type: %s\n", p);
+        error_report("unknown vga type: %s", p);
         return false;
     }
     while (*opts) {
@@ -2254,10 +2296,19 @@
         } else goto invalid_vga;
         opts = nextopt;
     }
-
     return true;
 }
 
+typedef enum DisplayType {
+    DT_DEFAULT,
+    DT_CURSES,
+    DT_SDL,
+    DT_COCOA,
+    DT_GTK,
+    DT_NONE,
+    DT_ERROR = -1,
+} DisplayType;
+
 static DisplayType select_display(const char *p)
 {
     const char *opts;
@@ -2305,42 +2356,39 @@
                 } else {
                     goto invalid_sdl_args;
                 }
+            } else if (strstart(opts, ",gl=", &nextopt)) {
+                opts = nextopt;
+                if (strstart(opts, "on", &nextopt)) {
+                    request_opengl = 1;
+                } else if (strstart(opts, "off", &nextopt)) {
+                    request_opengl = 0;
+                } else {
+                    goto invalid_sdl_args;
+                }
             } else {
             invalid_sdl_args:
-                fprintf(stderr, "Invalid SDL option string: %s\n", p);
-                return -1;
+                error_report("invalid SDL option string");
+                return DT_ERROR;
             }
             opts = nextopt;
         }
 #else
-        fprintf(stderr, "SDL support is disabled\n");
-        return -1;
+        error_report("SDL support is disabled");
+        return DT_ERROR;
 #endif
     } else if (strstart(p, "vnc", &opts)) {
-#ifdef CONFIG_VNC
-        display_remote++;
-
-        if (*opts) {
-            const char *nextopt;
-
-            if (strstart(opts, "=", &nextopt)) {
-                vnc_display = nextopt;
-            }
+        if (*opts == '=') {
+            vnc_parse(opts + 1, &error_fatal);
+        } else {
+            error_report("VNC requires a display argument vnc=<display>");
+            return DT_ERROR;
         }
-        if (!vnc_display) {
-            fprintf(stderr, "VNC requires a display argument vnc=<display>\n");
-            return -1;
-        }
-#else
-        fprintf(stderr, "VNC support is disabled\n");
-        return -1;
-#endif
     } else if (strstart(p, "curses", &opts)) {
 #ifdef CONFIG_CURSES
         display = DT_CURSES;
 #else
-        fprintf(stderr, "Curses support is disabled\n");
-        return -1;
+        error_report("curses support is disabled");
+        return DT_ERROR;
 #endif
     } else if (strstart(p, "gtk", &opts)) {
 #ifdef CONFIG_GTK
@@ -2357,22 +2405,31 @@
                 } else {
                     goto invalid_gtk_args;
                 }
+            } else if (strstart(opts, ",gl=", &nextopt)) {
+                opts = nextopt;
+                if (strstart(opts, "on", &nextopt)) {
+                    request_opengl = 1;
+                } else if (strstart(opts, "off", &nextopt)) {
+                    request_opengl = 0;
+                } else {
+                    goto invalid_gtk_args;
+                }
             } else {
             invalid_gtk_args:
-                fprintf(stderr, "Invalid GTK option string: %s\n", p);
-                return -1;
+                error_report("invalid GTK option string");
+                return DT_ERROR;
             }
             opts = nextopt;
         }
 #else
-        fprintf(stderr, "GTK support is disabled\n");
-        return -1;
+        error_report("GTK support is disabled");
+        return DT_ERROR;
 #endif
     } else if (strstart(p, "none", &opts)) {
         display = DT_NONE;
     } else {
-        fprintf(stderr, "Unknown display type: %s\n", p);
-        return -1;
+        error_report("unknown display type");
+        return DT_ERROR;
     }
 
     return display;
@@ -2389,7 +2446,8 @@
     if (!strncmp(arg, "virtio", 6)) {
         if (arg[6] == ',') {
             /* have params -> parse them */
-            opts = qemu_opts_parse(qemu_find_opts("device"), arg+7, 0);
+            opts = qemu_opts_parse_noisily(qemu_find_opts("device"), arg + 7,
+                                           false);
             if (!opts)
                 return  -1;
         } else {
@@ -2397,7 +2455,7 @@
             opts = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
                                     &error_abort);
         }
-        qemu_opt_set(opts, "driver", "virtio-balloon");
+        qemu_opt_set(opts, "driver", "virtio-balloon", &error_abort);
         return 0;
     }
 
@@ -2438,46 +2496,98 @@
     return NULL;
 }
 
-static int device_help_func(QemuOpts *opts, void *opaque)
+static inline bool nonempty_str(const char *str)
+{
+    return str && *str;
+}
+
+static int parse_fw_cfg(void *opaque, QemuOpts *opts, Error **errp)
+{
+    gchar *buf;
+    size_t size;
+    const char *name, *file, *str;
+    FWCfgState *fw_cfg = (FWCfgState *) opaque;
+
+    if (fw_cfg == NULL) {
+        error_report("fw_cfg device not available");
+        return -1;
+    }
+    name = qemu_opt_get(opts, "name");
+    file = qemu_opt_get(opts, "file");
+    str = qemu_opt_get(opts, "string");
+
+    /* we need name and either a file or the content string */
+    if (!(nonempty_str(name) && (nonempty_str(file) || nonempty_str(str)))) {
+        error_report("invalid argument(s)");
+        return -1;
+    }
+    if (nonempty_str(file) && nonempty_str(str)) {
+        error_report("file and string are mutually exclusive");
+        return -1;
+    }
+    if (strlen(name) > FW_CFG_MAX_FILE_PATH - 1) {
+        error_report("name too long (max. %d char)", FW_CFG_MAX_FILE_PATH - 1);
+        return -1;
+    }
+    if (strncmp(name, "opt/", 4) != 0) {
+        error_report("warning: externally provided fw_cfg item names "
+                     "should be prefixed with \"opt/\"");
+    }
+    if (nonempty_str(str)) {
+        size = strlen(str); /* NUL terminator NOT included in fw_cfg blob */
+        buf = g_memdup(str, size);
+    } else {
+        if (!g_file_get_contents(file, &buf, &size, NULL)) {
+            error_report("can't load %s", file);
+            return -1;
+        }
+    }
+    /* For legacy, keep user files in a specific global order. */
+    fw_cfg_set_order_override(fw_cfg, FW_CFG_ORDER_OVERRIDE_USER);
+    fw_cfg_add_file(fw_cfg, name, buf, size);
+    fw_cfg_reset_order_override(fw_cfg);
+    return 0;
+}
+
+static int device_help_func(void *opaque, QemuOpts *opts, Error **errp)
 {
     return qdev_device_help(opts);
 }
 
-static int device_init_func(QemuOpts *opts, void *opaque)
+static int device_init_func(void *opaque, QemuOpts *opts, Error **errp)
 {
+    Error *err = NULL;
     DeviceState *dev;
 
-    dev = qdev_device_add(opts);
-    if (!dev)
+    dev = qdev_device_add(opts, &err);
+    if (!dev) {
+        error_report_err(err);
         return -1;
+    }
     object_unref(OBJECT(dev));
     return 0;
 }
 
-static int chardev_init_func(QemuOpts *opts, void *opaque)
+static int chardev_init_func(void *opaque, QemuOpts *opts, Error **errp)
 {
     Error *local_err = NULL;
 
     qemu_chr_new_from_opts(opts, NULL, &local_err);
     if (local_err) {
-        error_report("%s", error_get_pretty(local_err));
-        error_free(local_err);
+        error_report_err(local_err);
         return -1;
     }
     return 0;
 }
 
 #ifdef CONFIG_VIRTFS
-static int fsdev_init_func(QemuOpts *opts, void *opaque)
+static int fsdev_init_func(void *opaque, QemuOpts *opts, Error **errp)
 {
-    int ret;
-    ret = qemu_fsdev_add(opts);
-
-    return ret;
+    return qemu_fsdev_add(opts);
 }
 #endif
 
-static int mon_init_func(QemuOpts *opts, void *opaque)
+static int mon_init_func(void *opaque, QemuOpts *opts, Error **errp)
 {
     CharDriverState *chr;
     const char *chardev;
@@ -2493,8 +2603,8 @@
     } else if (strcmp(mode, "control") == 0) {
         flags = MONITOR_USE_CONTROL;
     } else {
-        fprintf(stderr, "unknown monitor mode \"%s\"\n", mode);
-        return 1;
+        error_report("unknown monitor mode \"%s\"", mode);
+        return -1;
     }
 
     if (qemu_opt_get_bool(opts, "pretty", 0))
@@ -2506,8 +2616,8 @@
     chardev = qemu_opt_get(opts, "chardev");
     chr = qemu_chr_find(chardev);
     if (chr == NULL) {
-        fprintf(stderr, "chardev \"%s\" not found\n", chardev);
-        return 1;
+        error_report("chardev \"%s\" not found", chardev);
+        return -1;
     }
 
     qemu_chr_fe_claim_no_fail(chr);
@@ -2515,7 +2625,7 @@
     return 0;
 }
 
-static bool monitor_parse(const char *optarg, const char *mode)
+static bool monitor_parse(const char *optarg, const char *mode, bool pretty)
 {
     static int monitor_device_index = 0;
     QemuOpts *opts;
@@ -2533,22 +2643,18 @@
         }
         opts = qemu_chr_parse_compat(label, optarg);
         if (!opts) {
-            fprintf(stderr, "parse error: %s\n", optarg);
+            error_report("parse error: %s", optarg);
             return false;
         }
     }
 
-    opts = qemu_opts_create(qemu_find_opts("mon"), label, 1, NULL);
-    if (!opts) {
-        fprintf(stderr, "duplicate chardev: %s\n", label);
-        return false;
-    }
-    qemu_opt_set(opts, "mode", mode);
-    qemu_opt_set(opts, "chardev", label);
+    opts = qemu_opts_create(qemu_find_opts("mon"), label, 1, &error_fatal);
+    qemu_opt_set(opts, "mode", mode, &error_abort);
+    qemu_opt_set(opts, "chardev", label, &error_abort);
+    qemu_opt_set_bool(opts, "pretty", pretty, &error_abort);
     if (def)
-        qemu_opt_set(opts, "default", "on");
+        qemu_opt_set(opts, "default", "on", &error_abort);
     monitor_device_index++;
-
     return true;
 }
 
@@ -2608,14 +2714,14 @@
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_SERIAL_PORTS) {
-        fprintf(stderr, "qemu: too many serial ports\n");
+        error_report("too many serial ports");
         return -1;
     }
     snprintf(label, sizeof(label), "serial%d", index);
     serial_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!serial_hds[index]) {
-        fprintf(stderr, "qemu: could not connect serial device"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect serial device"
+                     " to character backend '%s'", devname);
         return -1;
     }
     index++;
@@ -2630,14 +2736,14 @@
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_PARALLEL_PORTS) {
-        fprintf(stderr, "qemu: too many parallel ports\n");
+        error_report("too many parallel ports");
         return -1;
     }
     snprintf(label, sizeof(label), "parallel%d", index);
     parallel_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!parallel_hds[index]) {
-        fprintf(stderr, "qemu: could not connect parallel device"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect parallel device"
+                     " to character backend '%s'", devname);
         return -1;
     }
     index++;
@@ -2654,28 +2760,24 @@
     if (strcmp(devname, "none") == 0)
         return 0;
     if (index == MAX_VIRTIO_CONSOLES) {
-        fprintf(stderr, "qemu: too many virtio consoles\n");
+        error_report("too many virtio consoles");
         return -1;
     }
 
     bus_opts = qemu_opts_create(device, NULL, 0, &error_abort);
-    if (arch_type == QEMU_ARCH_S390X) {
-        qemu_opt_set(bus_opts, "driver", "virtio-serial-s390");
-    } else {
-        qemu_opt_set(bus_opts, "driver", "virtio-serial-pci");
-    }
+    qemu_opt_set(bus_opts, "driver", "virtio-serial", &error_abort);
 
     dev_opts = qemu_opts_create(device, NULL, 0, &error_abort);
-    qemu_opt_set(dev_opts, "driver", "virtconsole");
+    qemu_opt_set(dev_opts, "driver", "virtconsole", &error_abort);
 
     snprintf(label, sizeof(label), "virtcon%d", index);
     virtcon_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!virtcon_hds[index]) {
-        fprintf(stderr, "qemu: could not connect virtio console"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect virtio console"
+                     " to character backend '%s'", devname);
         return -1;
     }
-    qemu_opt_set(dev_opts, "chardev", label);
+    qemu_opt_set(dev_opts, "chardev", label, &error_abort);
 
     index++;
     return 0;
@@ -2692,23 +2794,23 @@
         return 0;
     }
     if (index == MAX_SCLP_CONSOLES) {
-        fprintf(stderr, "qemu: too many sclp consoles\n");
+        error_report("too many sclp consoles");
         return -1;
     }
 
     assert(arch_type == QEMU_ARCH_S390X);
 
     dev_opts = qemu_opts_create(device, NULL, 0, NULL);
-    qemu_opt_set(dev_opts, "driver", "sclpconsole");
+    qemu_opt_set(dev_opts, "driver", "sclpconsole", &error_abort);
 
     snprintf(label, sizeof(label), "sclpcon%d", index);
     sclp_hds[index] = qemu_chr_new(label, devname, NULL);
     if (!sclp_hds[index]) {
-        fprintf(stderr, "qemu: could not connect sclp console"
-                " to character backend '%s'\n", devname);
+        error_report("could not connect sclp console"
+                     " to character backend '%s'", devname);
         return -1;
     }
-    qemu_opt_set(dev_opts, "chardev", label);
+    qemu_opt_set(dev_opts, "chardev", label, &error_abort);
 
     index++;
     return 0;
@@ -2723,11 +2825,11 @@
     }
     opts = qemu_opts_create(qemu_find_opts("device"), "debugcon", 1, NULL);
     if (!opts) {
-        fprintf(stderr, "qemu: already have a debugcon device\n");
+        error_report("already have a debugcon device");
         return -1;
     }
-    qemu_opt_set(opts, "driver", "isa-debugcon");
-    qemu_opt_set(opts, "chardev", "debugcon");
+    qemu_opt_set(opts, "driver", "isa-debugcon", &error_abort);
+    qemu_opt_set(opts, "chardev", "debugcon", &error_abort);
     return 0;
 }
 
@@ -2774,11 +2876,12 @@
         mc = find_machine(name);
     }
     if (mc) {
+        g_slist_free(machines);
         return mc;
     }
     if (name && !is_help_option(name)) {
-        error_report("Unsupported machine type");
-        error_printf("Use -machine help to list supported machines!\n");
+        error_report("unsupported machine type");
+        error_printf("Use -machine help to list supported machines\n");
     } else {
         printf("Supported machines are:\n");
         machines = g_slist_sort(machines, machine_class_cmp);
@@ -2811,14 +2914,25 @@
     notifier_list_notify(&exit_notifiers, NULL);
 }
 
+static bool machine_init_done;
+
 void qemu_add_machine_init_done_notifier(Notifier *notify)
 {
     notifier_list_add(&machine_init_done_notifiers, notify);
+    if (machine_init_done) {
+        notify->notify(notify, NULL);
+    }
+}
+
+void qemu_remove_machine_init_done_notifier(Notifier *notify)
+{
+    notifier_remove(notify);
 }
 
 static void qemu_run_machine_init_done_notifiers(void)
 {
     notifier_list_notify(&machine_init_done_notifiers, NULL);
+    machine_init_done = true;
 }
 
 static const QEMUOption *lookup_opt(int argc, const char **argv,
@@ -2829,7 +2943,7 @@
     const char *r = argv[optind];
     const char *optarg;
 
-    loc_set_cmdline(argv, optind, 1);
+    loc_set_cmdline((const char**)argv, optind, 1);
     optind++;
     /* Treat --foo the same as -foo.  */
     if (r[1] == '-')
@@ -2850,7 +2964,7 @@
             return NULL;
         }
         optarg = argv[optind++];
-        loc_set_cmdline(argv, optind - 2, 2);
+        loc_set_cmdline((const char**)argv, optind - 2, 2);
     } else {
         optarg = NULL;
     }
@@ -2861,31 +2975,38 @@
     return popt;
 }
 
-static gpointer malloc_and_trace(gsize n_bytes)
+static MachineClass *select_machine(void)
 {
-    void *ptr = malloc(n_bytes);
-    trace_g_malloc(n_bytes, ptr);
-    return ptr;
+    MachineClass *machine_class = find_default_machine();
+    const char *optarg;
+    QemuOpts *opts;
+    Location loc;
+
+    loc_push_none(&loc);
+
+    opts = qemu_get_machine_opts();
+    qemu_opts_loc_restore(opts);
+
+    optarg = qemu_opt_get(opts, "type");
+    if (optarg) {
+        machine_class = machine_parse(optarg);
+    }
+
+    if (!machine_class) {
+        error_report("No machine specified, and there is no default");
+        error_printf("Use -machine help to list supported machines\n");
+        return NULL;
+    }
+
+    loc_pop(&loc);
+    return machine_class;
 }
 
-static gpointer realloc_and_trace(gpointer mem, gsize n_bytes)
-{
-    void *ptr = realloc(mem, n_bytes);
-    trace_g_realloc(mem, n_bytes, ptr);
-    return ptr;
-}
-
-static void free_and_trace(gpointer mem)
-{
-    trace_g_free(mem);
-    free(mem);
-}
-
-static int machine_set_property(const char *name, const char *value,
-                                void *opaque)
+static int machine_set_property(void *opaque,
+                                const char *name, const char *value,
+                                Error **errp)
 {
     Object *obj = OBJECT(opaque);
-    StringInputVisitor *siv;
     Error *local_err = NULL;
     char *c, *qom_name;
 
@@ -2901,83 +3022,164 @@
         }
     }
 
-    siv = string_input_visitor_new(value);
-    object_property_set(obj, string_input_get_visitor(siv), qom_name, &local_err);
-    string_input_visitor_cleanup(siv);
+    object_property_parse(obj, value, qom_name, &local_err);
     g_free(qom_name);
 
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_report_err(local_err);
         return -1;
     }
 
     return 0;
 }
 
-static int object_create(QemuOpts *opts, void *opaque)
+
+/*
+ * Initial object creation happens before all other
+ * QEMU data types are created. The majority of objects
+ * can be created at this point. The rng-egd object
+ * cannot be created here, as it depends on the chardev
+ * already existing.
+ */
+static bool object_create_initial(const char *type)
 {
-    Error *err = NULL;
-    char *type = NULL;
-    char *id = NULL;
-    void *dummy = NULL;
-    OptsVisitor *ov;
-    QDict *pdict;
-
-    ov = opts_visitor_new(opts);
-    pdict = qemu_opts_to_qdict(opts, NULL);
-
-    visit_start_struct(opts_get_visitor(ov), &dummy, NULL, NULL, 0, &err);
-    if (err) {
-        goto out;
+    if (g_str_equal(type, "rng-egd")) {
+        return false;
     }
 
-    qdict_del(pdict, "qom-type");
-    visit_type_str(opts_get_visitor(ov), &type, "qom-type", &err);
-    if (err) {
-        goto out;
+    /*
+     * return false for concrete netfilters since
+     * they depend on netdevs already existing
+     */
+    if (g_str_equal(type, "filter-buffer") ||
+        g_str_equal(type, "filter-dump") ||
+        g_str_equal(type, "filter-mirror") ||
+        g_str_equal(type, "filter-redirector")) {
+        return false;
     }
 
-    qdict_del(pdict, "id");
-    visit_type_str(opts_get_visitor(ov), &id, "id", &err);
-    if (err) {
-        goto out;
+    return true;
+}
+
+
+/*
+ * The remainder of object creation happens after the
+ * creation of chardev, fsdev, net clients and device data types.
+ */
+static bool object_create_delayed(const char *type)
+{
+    return !object_create_initial(type);
+}
+
+
+static bool set_memory_options(uint64_t *ram_slots, ram_addr_t *maxram_size,
+                               MachineClass *mc)
+{
+    uint64_t sz;
+    const char *mem_str;
+    const char *maxmem_str, *slots_str;
+    const ram_addr_t default_ram_size = mc->default_ram_size;
+    QemuOpts *opts = qemu_find_opts_singleton("memory");
+    Location loc;
+
+    loc_push_none(&loc);
+    qemu_opts_loc_restore(opts);
+
+    sz = 0;
+    mem_str = qemu_opt_get(opts, "size");
+    if (mem_str) {
+        if (!*mem_str) {
+            error_report("missing 'size' option value");
+            return false;
+        }
+
+        sz = qemu_opt_get_size(opts, "size", ram_size);
+
+        /* Fix up legacy suffix-less format */
+        if (g_ascii_isdigit(mem_str[strlen(mem_str) - 1])) {
+            uint64_t overflow_check = sz;
+
+            sz <<= 20;
+            if ((sz >> 20) != overflow_check) {
+                error_report("too large 'size' option value");
+                return false;
+            }
+        }
     }
 
-    object_add(type, id, pdict, opts_get_visitor(ov), &err);
-    if (err) {
-        goto out;
-    }
-    visit_end_struct(opts_get_visitor(ov), &err);
-    if (err) {
-        qmp_object_del(id, NULL);
+    /* backward compatibility behaviour for case "-m 0" */
+    if (sz == 0) {
+        sz = default_ram_size;
     }
 
-out:
-    opts_visitor_cleanup(ov);
-
-    QDECREF(pdict);
-    g_free(id);
-    g_free(type);
-    g_free(dummy);
-    if (err) {
-        qerror_report_err(err);
-        error_free(err);
-        return -1;
+    sz = QEMU_ALIGN_UP(sz, 8192);
+    ram_size = sz;
+    if (ram_size != sz) {
+        error_report("ram size too large");
+        return false;
     }
+#ifdef CONFIG_HAX
+    hax_pre_init(ram_size);
+#endif
+    /* store value for the future use */
+    qemu_opt_set_number(opts, "size", ram_size, &error_abort);
+    *maxram_size = ram_size;
+
+    maxmem_str = qemu_opt_get(opts, "maxmem");
+    slots_str = qemu_opt_get(opts, "slots");
+    if (maxmem_str && slots_str) {
+        uint64_t slots;
+
+        sz = qemu_opt_get_size(opts, "maxmem", 0);
+        slots = qemu_opt_get_number(opts, "slots", 0);
+        if (sz < ram_size) {
+            error_report("invalid value of -m option maxmem: "
+                         "maximum memory size (0x%" PRIx64 ") must be at least "
+                         "the initial memory size (0x" RAM_ADDR_FMT ")",
+                         sz, ram_size);
+            return false;
+        } else if (sz > ram_size) {
+            if (!slots) {
+                error_report("invalid value of -m option: maxmem was "
+                             "specified, but no hotplug slots were specified");
+                return false;
+            }
+        } else if (slots) {
+            error_report("invalid value of -m option maxmem: "
+                         "memory slots were specified but maximum memory size "
+                         "(0x%" PRIx64 ") is equal to the initial memory size "
+                         "(0x" RAM_ADDR_FMT ")", sz, ram_size);
+            return false;
+        }
+
+        *maxram_size = sz;
+        *ram_slots = slots;
+    } else if ((!maxmem_str && slots_str) ||
+            (maxmem_str && !slots_str)) {
+        error_report("invalid -m option value: missing "
+                "'%s' option", slots_str ? "maxmem" : "slots");
+        return false;
+    }
+
+    loc_pop(&loc);
+    return true;
+}
+
+static int global_init_func(void *opaque, QemuOpts *opts, Error **errp)
+{
+    GlobalProperty *g;
+
+    g = g_malloc0(sizeof(*g));
+    g->driver   = qemu_opt_get(opts, "driver");
+    g->property = qemu_opt_get(opts, "property");
+    g->value    = qemu_opt_get(opts, "value");
+    g->user_provided = true;
+    g->errp = &error_fatal;
+    qdev_prop_register_global(g);
     return 0;
 }
 
-#if !defined(CONFIG_ANDROID)
-// We don't use the AndroidEmu library in the original qemu2 build,
-// so let's return their main function back
-#define run_qemu_main main
-
-// To avoid compiler warning about missing prototype when main is a macro
-// for SDL_main.
-extern int run_qemu_main(int, const char**);
-
-#else  /* CONFIG_ANDROID */
+#if defined(CONFIG_ANDROID)
 
 static int is_opengl_alive = 1;
 
@@ -2995,6 +3197,11 @@
     android_metrics_report_common_info(is_opengl_alive);
 }
 
+static void android_teardown_metrics()
+{
+    android_metrics_stop();
+}
+
 static bool android_reporting_setup(void)
 {
     android_init_metrics();
@@ -3010,18 +3217,20 @@
 
 static void android_reporting_teardown(void)
 {
-    android_metrics_stop();
+    android_teardown_metrics();
 }
 
-#endif  /* CONFIG_ANDROID */
-
 int run_qemu_main(int argc, const char **argv)
+#else
+int main(int argc, char** argv, char** envp)
+#endif
 {
     int i;
     int snapshot, linux_boot;
     const char *initrd_filename;
     const char *kernel_filename, *kernel_cmdline;
-    const char *boot_order;
+    const char *boot_order = NULL;
+    const char *boot_once = NULL;
     DisplayState *ds;
     int cyls, heads, secs, translation;
     QemuOpts *hda_opts = NULL, *opts, *machine_opts, *icount_opts = NULL;
@@ -3036,38 +3245,33 @@
     const char *qtest_log = NULL;
     const char *pid_file = NULL;
     const char *incoming = NULL;
-#ifdef CONFIG_VNC
-    int show_vnc_port = 0;
-#endif
     bool defconfig = true;
     bool userconfig = true;
+    bool nographic = false;
+    DisplayType display_type = DT_DEFAULT;
+    int display_remote = 0;
     const char *log_mask = NULL;
     const char *log_file = NULL;
-    GMemVTable mem_trace = {
-        .malloc = malloc_and_trace,
-        .realloc = realloc_and_trace,
-        .free = free_and_trace,
-    };
-    const char *trace_events = NULL;
-    const char *trace_file = NULL;
-    const ram_addr_t default_ram_size = (ram_addr_t)DEFAULT_RAM_SIZE *
-                                        1024 * 1024;
-    ram_addr_t maxram_size = default_ram_size;
+    char *trace_file = NULL;
+    ram_addr_t maxram_size;
     uint64_t ram_slots = 0;
     FILE *vmstate_dump_file = NULL;
     Error *main_loop_err = NULL;
+    Error *err = NULL;
+    bool list_data_dirs = false;
+
 #ifdef CONFIG_ANDROID
-    char* additional_kernel_params = NULL;
-    const char* android_hw_file = NULL;
-#endif
+    char* android_op_dns_server = NULL;
     int adb_auth = 1;
+#endif
+
+    qemu_init_cpu_loop();
+    qemu_mutex_lock_iothread();
 
     atexit(qemu_run_exit_notifiers);
     error_set_progname(argv[0]);
     qemu_init_exec_dir(argv[0]);
 
-    g_mem_set_vtable(&mem_trace);
-
     module_call_init(MODULE_INIT_QOM);
 
     qemu_add_opts(&qemu_drive_opts);
@@ -3096,30 +3300,26 @@
     qemu_add_opts(&qemu_name_opts);
     qemu_add_opts(&qemu_numa_opts);
     qemu_add_opts(&qemu_icount_opts);
+    qemu_add_opts(&qemu_semihosting_config_opts);
+    qemu_add_opts(&qemu_fw_cfg_opts);
+    module_call_init(MODULE_INIT_OPTS);
 
     runstate_init();
 
+    if (qcrypto_init(&err) < 0) {
+        error_reportf_err(err, "cannot initialize crypto: ");
+        return 1;
+    }
     rtc_clock = QEMU_CLOCK_HOST;
 
     QLIST_INIT (&vm_change_state_head);
     os_setup_early_signal_handling();
 
-    module_call_init(MODULE_INIT_MACHINE);
-    machine_class = find_default_machine();
     cpu_model = NULL;
-    ram_size = default_ram_size;
     snapshot = 0;
     cyls = heads = secs = 0;
     translation = BIOS_ATA_TRANSLATION_AUTO;
 
-    for (i = 0; i < MAX_NODES; i++) {
-        numa_info[i].node_mem = 0;
-        numa_info[i].present = false;
-        bitmap_zero(numa_info[i].node_cpu, MAX_CPUMASK_BITS);
-    }
-
-    nb_numa_nodes = 0;
-    max_numa_nodeid = 0;
     nb_nics = 0;
 
     bdrv_init_with_whitelist();
@@ -3135,7 +3335,7 @@
         } else {
             const QEMUOption *popt;
 
-            popt = lookup_opt(argc, argv, &optarg, &optind);
+            popt = lookup_opt(argc, (const char**)argv, &optarg, &optind);
             if (!popt) {
                 return 1;
             }
@@ -3164,28 +3364,22 @@
         if (optind >= argc)
             break;
         if (argv[optind][0] != '-') {
-	    hda_opts = drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);
+            hda_opts = drive_add(IF_DEFAULT, 0, argv[optind++], HD_OPTS);
         } else {
             const QEMUOption *popt;
 
-            popt = lookup_opt(argc, argv, &optarg, &optind);
+            popt = lookup_opt(argc, (const char**)argv, &optarg, &optind);
             if (!popt) {
                 return 1;
             }
             if (!(popt->arch_mask & arch_type)) {
-                printf("Option %s not supported for this target\n", popt->name);
+                error_report("Option not supported for this target");
                 return 1;
             }
             switch(popt->index) {
-            case QEMU_OPTION_M:
-                machine_class = machine_parse(optarg);
-                if (!machine_class) {
-                    return 1;
-                }
-                break;
             case QEMU_OPTION_no_kvm_irqchip: {
                 olist = qemu_find_opts("machine");
-                qemu_opts_parse(olist, "kernel_irqchip=off", 0);
+                qemu_opts_parse_noisily(olist, "kernel_irqchip=off", false);
                 break;
             }
             case QEMU_OPTION_cpu:
@@ -3218,15 +3412,15 @@
                 if (drive_def(optarg) == NULL) {
                     return 1;
                 }
-	        break;
+                break;
             case QEMU_OPTION_set:
                 if (qemu_set_option(optarg) != 0)
                     return 1;
-	        break;
+                break;
             case QEMU_OPTION_global:
                 if (qemu_global_option(optarg) != 0)
                     return 1;
-	        break;
+                break;
             case QEMU_OPTION_mtdblock:
                 drive_add(IF_MTD, -1, optarg, MTD_OPTS);
                 break;
@@ -3275,49 +3469,56 @@
                         }
                     } else if (*p != '\0') {
                     chs_fail:
-                        fprintf(stderr, "qemu: invalid physical CHS format\n");
+                        error_report("invalid physical CHS format");
                         return 1;
                     }
-		    if (hda_opts != NULL) {
-                        char num[16];
-                        snprintf(num, sizeof(num), "%d", cyls);
-                        qemu_opt_set(hda_opts, "cyls", num);
-                        snprintf(num, sizeof(num), "%d", heads);
-                        qemu_opt_set(hda_opts, "heads", num);
-                        snprintf(num, sizeof(num), "%d", secs);
-                        qemu_opt_set(hda_opts, "secs", num);
+                    if (hda_opts != NULL) {
+                        qemu_opt_set_number(hda_opts, "cyls", cyls,
+                                            &error_abort);
+                        qemu_opt_set_number(hda_opts, "heads", heads,
+                                            &error_abort);
+                        qemu_opt_set_number(hda_opts, "secs", secs,
+                                            &error_abort);
                         if (translation == BIOS_ATA_TRANSLATION_LARGE) {
-                            qemu_opt_set(hda_opts, "trans", "large");
+                            qemu_opt_set(hda_opts, "trans", "large",
+                                         &error_abort);
                         } else if (translation == BIOS_ATA_TRANSLATION_RECHS) {
-                            qemu_opt_set(hda_opts, "trans", "rechs");
+                            qemu_opt_set(hda_opts, "trans", "rechs",
+                                         &error_abort);
                         } else if (translation == BIOS_ATA_TRANSLATION_LBA) {
-                            qemu_opt_set(hda_opts, "trans", "lba");
+                            qemu_opt_set(hda_opts, "trans", "lba",
+                                         &error_abort);
                         } else if (translation == BIOS_ATA_TRANSLATION_NONE) {
-                            qemu_opt_set(hda_opts, "trans", "none");
+                            qemu_opt_set(hda_opts, "trans", "none",
+                                         &error_abort);
                         }
                     }
                 }
                 break;
             case QEMU_OPTION_numa:
-                opts = qemu_opts_parse(qemu_find_opts("numa"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("numa"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_display:
                 display_type = select_display(optarg);
-                if (display_type < 0) {
+                if (display_type == DT_ERROR) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_nographic:
-                display_type = DT_NOGRAPHIC;
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse_noisily(olist, "graphics=off", false);
+                nographic = true;
+                display_type = DT_NONE;
                 break;
             case QEMU_OPTION_curses:
 #ifdef CONFIG_CURSES
                 display_type = DT_CURSES;
 #else
-                fprintf(stderr, "Curses support is disabled\n");
+                error_report("curses support is disabled");
                 return 1;
 #endif
                 break;
@@ -3328,28 +3529,32 @@
                 graphic_rotate = strtol(optarg, (char **) &optarg, 10);
                 if (graphic_rotate != 0 && graphic_rotate != 90 &&
                     graphic_rotate != 180 && graphic_rotate != 270) {
-                    fprintf(stderr,
-                        "qemu: only 90, 180, 270 deg rotation is available\n");
+                    error_report("only 90, 180, 270 deg rotation is available");
                     return 1;
                 }
                 break;
             case QEMU_OPTION_kernel:
-                qemu_opts_set(qemu_find_opts("machine"), 0, "kernel", optarg);
+                qemu_opts_set(qemu_find_opts("machine"), 0, "kernel", optarg,
+                              &error_abort);
                 break;
             case QEMU_OPTION_initrd:
-                qemu_opts_set(qemu_find_opts("machine"), 0, "initrd", optarg);
+                qemu_opts_set(qemu_find_opts("machine"), 0, "initrd", optarg,
+                              &error_abort);
                 break;
             case QEMU_OPTION_append:
-                qemu_opts_set(qemu_find_opts("machine"), 0, "append", optarg);
+                qemu_opts_set(qemu_find_opts("machine"), 0, "append", optarg,
+                              &error_abort);
                 break;
             case QEMU_OPTION_dtb:
-                qemu_opts_set(qemu_find_opts("machine"), 0, "dtb", optarg);
+                qemu_opts_set(qemu_find_opts("machine"), 0, "dtb", optarg,
+                              &error_abort);
                 break;
             case QEMU_OPTION_cdrom:
                 drive_add(IF_DEFAULT, 2, optarg, CDROM_OPTS);
                 break;
             case QEMU_OPTION_boot:
-                opts = qemu_opts_parse(qemu_find_opts("boot-opts"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("boot-opts"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
@@ -3363,18 +3568,21 @@
                 fd_bootchk = 0;
                 break;
             case QEMU_OPTION_netdev:
+                default_net = 0;
                 if (net_client_parse(qemu_find_opts("netdev"), optarg) == -1) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_net:
+                default_net = 0;
                 if (net_client_parse(qemu_find_opts("net"), optarg) == -1) {
                     return 1;
                 }
                 break;
 #ifdef CONFIG_LIBISCSI
             case QEMU_OPTION_iscsi:
-                opts = qemu_opts_parse(qemu_find_opts("iscsi"), optarg, 0);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("iscsi"),
+                                               optarg, false);
                 if (!opts) {
                     return 1;
                 }
@@ -3382,12 +3590,18 @@
 #endif
 #ifdef CONFIG_SLIRP
             case QEMU_OPTION_tftp:
+                error_report("The -tftp option is deprecated. "
+                             "Please use '-netdev user,tftp=...' instead.");
                 legacy_tftp_prefix = optarg;
                 break;
             case QEMU_OPTION_bootp:
+                error_report("The -bootp option is deprecated. "
+                             "Please use '-netdev user,bootfile=...' instead.");
                 legacy_bootp_filename = optarg;
                 break;
             case QEMU_OPTION_redir:
+                error_report("The -redir option is deprecated. "
+                             "Please use '-netdev user,hostfwd=...' instead.");
                 if (net_slirp_redir(optarg) < 0)
                     return 1;
                 break;
@@ -3397,10 +3611,7 @@
                 break;
             case QEMU_OPTION_audio_help:
                 AUD_help ();
-                return (0);
-                break;
-            case QEMU_OPTION_audio_none:
-                setenv("QEMU_AUDIO_DRV", "none", 1);
+                return 0;
                 break;
             case QEMU_OPTION_soundhw:
                 select_soundhw (optarg);
@@ -3413,92 +3624,13 @@
                 version();
                 return 0;
                 break;
-            case QEMU_OPTION_m: {
-                uint64_t sz;
-                const char *mem_str;
-                const char *maxmem_str, *slots_str;
-
-                opts = qemu_opts_parse(qemu_find_opts("memory"),
-                                       optarg, 1);
+            case QEMU_OPTION_m:
+                opts = qemu_opts_parse_noisily(qemu_find_opts("memory"),
+                                               optarg, true);
                 if (!opts) {
-                    return EXIT_FAILURE;
-                }
-
-                mem_str = qemu_opt_get(opts, "size");
-                if (!mem_str) {
-                    error_report("invalid -m option, missing 'size' option");
-                    return EXIT_FAILURE;
-                }
-                if (!*mem_str) {
-                    error_report("missing 'size' option value");
-                    return EXIT_FAILURE;
-                }
-
-                sz = qemu_opt_get_size(opts, "size", ram_size);
-
-                /* Fix up legacy suffix-less format */
-                if (g_ascii_isdigit(mem_str[strlen(mem_str) - 1])) {
-                    uint64_t overflow_check = sz;
-
-                    sz <<= 20;
-                    if ((sz >> 20) != overflow_check) {
-                        error_report("too large 'size' option value");
-                        return EXIT_FAILURE;
-                    }
-                }
-
-                /* backward compatibility behaviour for case "-m 0" */
-                if (sz == 0) {
-                    sz = default_ram_size;
-                }
-
-                sz = QEMU_ALIGN_UP(sz, 8192);
-                ram_size = sz;
-                if (ram_size != sz) {
-                    error_report("ram size too large");
-                    return EXIT_FAILURE;
-                }
-                maxram_size = ram_size;
-
-                maxmem_str = qemu_opt_get(opts, "maxmem");
-                slots_str = qemu_opt_get(opts, "slots");
-                if (maxmem_str && slots_str) {
-                    uint64_t slots;
-
-                    sz = qemu_opt_get_size(opts, "maxmem", 0);
-                    if (sz < ram_size) {
-                        error_report("invalid -m option value: maxmem "
-                                "(0x%" PRIx64 ") <= initial memory (0x"
-                                RAM_ADDR_FMT ")", sz, ram_size);
-                        return EXIT_FAILURE;
-                    }
-
-                    slots = qemu_opt_get_number(opts, "slots", 0);
-                    if ((sz > ram_size) && !slots) {
-                        error_report("invalid -m option value: maxmem "
-                                "(0x%" PRIx64 ") more than initial memory (0x"
-                                RAM_ADDR_FMT ") but no hotplug slots where "
-                                "specified", sz, ram_size);
-                        return EXIT_FAILURE;
-                    }
-
-                    if ((sz <= ram_size) && slots) {
-                        error_report("invalid -m option value:  %"
-                                PRIu64 " hotplug slots where specified but "
-                                "maxmem (0x%" PRIx64 ") <= initial memory (0x"
-                                RAM_ADDR_FMT ")", slots, sz, ram_size);
-                        return EXIT_FAILURE;
-                    }
-                    maxram_size = sz;
-                    ram_slots = slots;
-                } else if ((!maxmem_str && slots_str) ||
-                           (maxmem_str && !slots_str)) {
-                    error_report("invalid -m option value: missing "
-                            "'%s' option", slots_str ? "maxmem" : "slots");
-                    return EXIT_FAILURE;
+                    return 1;
                 }
                 break;
-            }
 #ifdef CONFIG_TPM
             case QEMU_OPTION_tpmdev:
                 if (tpm_config_parse(qemu_find_opts("tpmdev"), optarg) < 0) {
@@ -3518,6 +3650,9 @@
             case QEMU_OPTION_D:
                 log_file = optarg;
                 break;
+            case QEMU_OPTION_DFILTER:
+                qemu_set_dfilter_ranges(optarg, &error_fatal);
+                break;
             case QEMU_OPTION_s:
                 add_device_config(DEV_GDB, "tcp::" DEFAULT_GDBSTUB_PORT);
                 break;
@@ -3525,12 +3660,15 @@
                 add_device_config(DEV_GDB, optarg);
                 break;
             case QEMU_OPTION_L:
-                if (data_dir_idx < ARRAY_SIZE(data_dir)) {
+                if (is_help_option(optarg)) {
+                    list_data_dirs = true;
+                } else if (data_dir_idx < ARRAY_SIZE(data_dir)) {
                     data_dir[data_dir_idx++] = optarg;
                 }
                 break;
             case QEMU_OPTION_bios:
-                qemu_opts_set(qemu_find_opts("machine"), 0, "firmware", optarg);
+                qemu_opts_set(qemu_find_opts("machine"), 0, "firmware", optarg,
+                              &error_abort);
                 break;
             case QEMU_OPTION_singlestep:
                 singlestep = 1;
@@ -3538,9 +3676,9 @@
             case QEMU_OPTION_S:
                 autostart = 0;
                 break;
-	    case QEMU_OPTION_k:
-		keyboard_layout = optarg;
-		break;
+            case QEMU_OPTION_k:
+                keyboard_layout = optarg;
+                break;
             case QEMU_OPTION_localtime:
                 rtc_utc = 0;
                 break;
@@ -3556,7 +3694,7 @@
                     w = strtol(p, (char **)&p, 10);
                     if (w <= 0) {
                     graphic_error:
-                        fprintf(stderr, "qemu: invalid resolution or depth\n");
+                        error_report("invalid resolution or depth");
                         return 1;
                     }
                     if (*p != 'x')
@@ -3593,26 +3731,34 @@
             case QEMU_OPTION_monitor:
                 default_monitor = 0;
                 if (strncmp(optarg, "none", 4)) {
-                    if (!monitor_parse(optarg, "readline")) {
+                    if (!monitor_parse(optarg, "readline", false)) {
                         return 1;
                     }
                 }
                 break;
             case QEMU_OPTION_qmp:
-                if (!monitor_parse(optarg, "control")) {
+                if (!monitor_parse(optarg, "control", false)) {
+                    return 1;
+                }
+                default_monitor = 0;
+                break;
+            case QEMU_OPTION_qmp_pretty:
+                if (!monitor_parse(optarg, "control", true)) {
                     return 1;
                 }
                 default_monitor = 0;
                 break;
             case QEMU_OPTION_mon:
-                opts = qemu_opts_parse(qemu_find_opts("mon"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("mon"), optarg,
+                                               true);
                 if (!opts) {
                     return 1;
                 }
                 default_monitor = 0;
                 break;
             case QEMU_OPTION_chardev:
-                opts = qemu_opts_parse(qemu_find_opts("chardev"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("chardev"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
@@ -3620,10 +3766,10 @@
             case QEMU_OPTION_fsdev:
                 olist = qemu_find_opts("fsdev");
                 if (!olist) {
-                    fprintf(stderr, "fsdev is not supported by this qemu build.\n");
+                    error_report("fsdev support is disabled");
                     return 1;
                 }
-                opts = qemu_opts_parse(olist, optarg, 1);
+                opts = qemu_opts_parse_noisily(olist, optarg, true);
                 if (!opts) {
                     return 1;
                 }
@@ -3635,60 +3781,64 @@
 
                 olist = qemu_find_opts("virtfs");
                 if (!olist) {
-                    fprintf(stderr, "virtfs is not supported by this qemu build.\n");
+                    error_report("virtfs support is disabled");
                     return 1;
                 }
-                opts = qemu_opts_parse(olist, optarg, 1);
+                opts = qemu_opts_parse_noisily(olist, optarg, true);
                 if (!opts) {
                     return 1;
                 }
 
                 if (qemu_opt_get(opts, "fsdriver") == NULL ||
                     qemu_opt_get(opts, "mount_tag") == NULL) {
-                    fprintf(stderr, "Usage: -virtfs fsdriver,mount_tag=tag.\n");
+                    error_report("Usage: -virtfs fsdriver,mount_tag=tag");
                     return 1;
                 }
                 fsdev = qemu_opts_create(qemu_find_opts("fsdev"),
                                          qemu_opt_get(opts, "mount_tag"),
                                          1, NULL);
                 if (!fsdev) {
-                    fprintf(stderr, "duplicate fsdev id: %s\n",
-                            qemu_opt_get(opts, "mount_tag"));
+                    error_report("duplicate fsdev id: %s",
+                                 qemu_opt_get(opts, "mount_tag"));
                     return 1;
                 }
 
                 writeout = qemu_opt_get(opts, "writeout");
                 if (writeout) {
 #ifdef CONFIG_SYNC_FILE_RANGE
-                    qemu_opt_set(fsdev, "writeout", writeout);
+                    qemu_opt_set(fsdev, "writeout", writeout, &error_abort);
 #else
-                    fprintf(stderr, "writeout=immediate not supported on "
-                            "this platform\n");
+                    error_report("writeout=immediate not supported "
+                                 "on this platform");
                     return 1;
 #endif
                 }
-                qemu_opt_set(fsdev, "fsdriver", qemu_opt_get(opts, "fsdriver"));
-                qemu_opt_set(fsdev, "path", qemu_opt_get(opts, "path"));
+                qemu_opt_set(fsdev, "fsdriver",
+                             qemu_opt_get(opts, "fsdriver"), &error_abort);
+                qemu_opt_set(fsdev, "path", qemu_opt_get(opts, "path"),
+                             &error_abort);
                 qemu_opt_set(fsdev, "security_model",
-                             qemu_opt_get(opts, "security_model"));
+                             qemu_opt_get(opts, "security_model"),
+                             &error_abort);
                 socket = qemu_opt_get(opts, "socket");
                 if (socket) {
-                    qemu_opt_set(fsdev, "socket", socket);
+                    qemu_opt_set(fsdev, "socket", socket, &error_abort);
                 }
                 sock_fd = qemu_opt_get(opts, "sock_fd");
                 if (sock_fd) {
-                    qemu_opt_set(fsdev, "sock_fd", sock_fd);
+                    qemu_opt_set(fsdev, "sock_fd", sock_fd, &error_abort);
                 }
 
                 qemu_opt_set_bool(fsdev, "readonly",
-                                qemu_opt_get_bool(opts, "readonly", 0));
+                                  qemu_opt_get_bool(opts, "readonly", 0),
+                                  &error_abort);
                 device = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
                                           &error_abort);
-                qemu_opt_set(device, "driver", "virtio-9p-pci");
+                qemu_opt_set(device, "driver", "virtio-9p-pci", &error_abort);
                 qemu_opt_set(device, "fsdev",
-                             qemu_opt_get(opts, "mount_tag"));
+                             qemu_opt_get(opts, "mount_tag"), &error_abort);
                 qemu_opt_set(device, "mount_tag",
-                             qemu_opt_get(opts, "mount_tag"));
+                             qemu_opt_get(opts, "mount_tag"), &error_abort);
                 break;
             }
             case QEMU_OPTION_virtfs_synth: {
@@ -3698,16 +3848,16 @@
                 fsdev = qemu_opts_create(qemu_find_opts("fsdev"), "v_synth",
                                          1, NULL);
                 if (!fsdev) {
-                    fprintf(stderr, "duplicate option: %s\n", "virtfs_synth");
+                    error_report("duplicate option: %s", "virtfs_synth");
                     return 1;
                 }
-                qemu_opt_set(fsdev, "fsdriver", "synth");
+                qemu_opt_set(fsdev, "fsdriver", "synth", &error_abort);
 
                 device = qemu_opts_create(qemu_find_opts("device"), NULL, 0,
                                           &error_abort);
-                qemu_opt_set(device, "driver", "virtio-9p-pci");
-                qemu_opt_set(device, "fsdev", "v_synth");
-                qemu_opt_set(device, "mount_tag", "v_synth");
+                qemu_opt_set(device, "driver", "virtio-9p-pci", &error_abort);
+                qemu_opt_set(device, "fsdev", "v_synth", &error_abort);
+                qemu_opt_set(device, "mount_tag", "v_synth", &error_abort);
                 break;
             }
             case QEMU_OPTION_serial:
@@ -3719,15 +3869,14 @@
                 break;
             case QEMU_OPTION_watchdog:
                 if (watchdog) {
-                    fprintf(stderr,
-                            "qemu: only one watchdog option may be given\n");
+                    error_report("only one watchdog option may be given");
                     return 1;
                 }
                 watchdog = optarg;
                 break;
             case QEMU_OPTION_watchdog_action:
                 if (select_watchdog_action(optarg) == -1) {
-                    fprintf(stderr, "Unknown -watchdog-action parameter\n");
+                    error_report("unknown -watchdog-action parameter");
                     return 1;
                 }
                 break;
@@ -3748,9 +3897,9 @@
             case QEMU_OPTION_debugcon:
                 add_device_config(DEV_DEBUGCON, optarg);
                 break;
-	    case QEMU_OPTION_loadvm:
-		loadvm = optarg;
-		break;
+            case QEMU_OPTION_loadvm:
+                loadvm = optarg;
+                break;
             case QEMU_OPTION_full_screen:
                 full_screen = 1;
                 break;
@@ -3767,13 +3916,13 @@
                 no_quit = 1;
                 break;
             case QEMU_OPTION_sdl:
-#if defined(CONFIG_SDL) || defined(CONFIG_ANDROID)
+#ifdef CONFIG_SDL
                 display_type = DT_SDL;
+                break;
 #else
-                fprintf(stderr, "SDL support is disabled\n");
+                error_report("SDL support is disabled");
                 return 1;
 #endif
-                break;
             case QEMU_OPTION_pidfile:
                 pid_file = optarg;
                 break;
@@ -3781,106 +3930,100 @@
                 win2k_install_hack = 1;
                 break;
             case QEMU_OPTION_rtc_td_hack: {
-                static GlobalProperty slew_lost_ticks[] = {
-                    {
-                        .driver   = "mc146818rtc",
-                        .property = "lost_tick_policy",
-                        .value    = "slew",
-                    },
-                    { /* end of list */ }
+                static GlobalProperty slew_lost_ticks = {
+                    .driver   = "mc146818rtc",
+                    .property = "lost_tick_policy",
+                    .value    = "slew",
                 };
 
-                qdev_prop_register_global_list(slew_lost_ticks);
+                qdev_prop_register_global(&slew_lost_ticks);
                 break;
             }
             case QEMU_OPTION_acpitable:
-                opts = qemu_opts_parse(qemu_find_opts("acpi"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("acpi"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
                 do_acpitable_option(opts);
                 break;
             case QEMU_OPTION_smbios:
-                opts = qemu_opts_parse(qemu_find_opts("smbios"), optarg, 0);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("smbios"),
+                                               optarg, false);
                 if (!opts) {
                     return 1;
                 }
                 do_smbios_option(opts);
                 break;
-            case QEMU_OPTION_enable_kvm:
-                olist = qemu_find_opts("machine");
-                qemu_opts_parse(olist, "accel=kvm", 0);
-                break;
-            case QEMU_OPTION_enable_hax:
-                olist = qemu_find_opts("machine");
-                qemu_opts_parse(olist, "accel=hax", 0);
-                hax_disable(0);
-                break;
-            case QEMU_OPTION_machine:
-                olist = qemu_find_opts("machine");
-                opts = qemu_opts_parse(olist, optarg, 1);
-                if (!opts) {
+            case QEMU_OPTION_fwcfg:
+                opts = qemu_opts_parse_noisily(qemu_find_opts("fw_cfg"),
+                                               optarg, true);
+                if (opts == NULL) {
                     return 1;
                 }
-                optarg = qemu_opt_get(opts, "type");
-                if (optarg) {
-                    machine_class = machine_parse(optarg);
-                    if (!machine_class) {
-                        return 1;
-                    }
+                break;
+            case QEMU_OPTION_enable_kvm:
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse_noisily(olist, "accel=kvm", false);
+                break;
+#ifdef CONFIG_HAX
+            case QEMU_OPTION_enable_hax:
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse_noisily(olist, "accel=hax", false);
+                hax_disable(0);
+                break;
+#endif /* CONFIG_HAX */
+            case QEMU_OPTION_M:
+            case QEMU_OPTION_machine:
+                olist = qemu_find_opts("machine");
+                opts = qemu_opts_parse_noisily(olist, optarg, true);
+                if (!opts) {
+                    return 1;
                 }
                 break;
              case QEMU_OPTION_no_kvm:
                 olist = qemu_find_opts("machine");
-                qemu_opts_parse(olist, "accel=tcg", 0);
+                qemu_opts_parse_noisily(olist, "accel=tcg", false);
                 break;
             case QEMU_OPTION_no_kvm_pit: {
-                fprintf(stderr, "Warning: KVM PIT can no longer be disabled "
-                                "separately.\n");
+                error_report("warning: ignoring deprecated option");
                 break;
             }
             case QEMU_OPTION_no_kvm_pit_reinjection: {
-                static GlobalProperty kvm_pit_lost_tick_policy[] = {
-                    {
-                        .driver   = "kvm-pit",
-                        .property = "lost_tick_policy",
-                        .value    = "discard",
-                    },
-                    { /* end of list */ }
+                static GlobalProperty kvm_pit_lost_tick_policy = {
+                    .driver   = "kvm-pit",
+                    .property = "lost_tick_policy",
+                    .value    = "discard",
                 };
 
-                fprintf(stderr, "Warning: option deprecated, use "
-                        "lost_tick_policy property of kvm-pit instead.\n");
-                qdev_prop_register_global_list(kvm_pit_lost_tick_policy);
+                error_report("warning: deprecated, replaced by "
+                             "-global kvm-pit.lost_tick_policy=discard");
+                qdev_prop_register_global(&kvm_pit_lost_tick_policy);
                 break;
             }
             case QEMU_OPTION_usb:
                 olist = qemu_find_opts("machine");
-                qemu_opts_parse(olist, "usb=on", 0);
+                qemu_opts_parse_noisily(olist, "usb=on", false);
                 break;
             case QEMU_OPTION_usbdevice:
                 olist = qemu_find_opts("machine");
-                qemu_opts_parse(olist, "usb=on", 0);
+                qemu_opts_parse_noisily(olist, "usb=on", false);
                 add_device_config(DEV_USB, optarg);
                 break;
             case QEMU_OPTION_device:
-                if (!qemu_opts_parse(qemu_find_opts("device"), optarg, 1)) {
+                if (!qemu_opts_parse_noisily(qemu_find_opts("device"),
+                                             optarg, true)) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_smp:
-                if (!qemu_opts_parse(qemu_find_opts("smp-opts"), optarg, 1)) {
+                if (!qemu_opts_parse_noisily(qemu_find_opts("smp-opts"),
+                                             optarg, true)) {
                     return 1;
                 }
                 break;
-	    case QEMU_OPTION_vnc:
-#ifdef CONFIG_VNC
-                display_remote++;
-                vnc_display = optarg;
-#else
-                fprintf(stderr, "VNC support is disabled\n");
-                return 1;
-#endif
+            case QEMU_OPTION_vnc:
+                vnc_parse(optarg, &error_fatal);
                 break;
             case QEMU_OPTION_no_acpi:
                 acpi_enabled = 0;
@@ -3890,7 +4033,7 @@
                 break;
             case QEMU_OPTION_balloon:
                 if (balloon_parse(optarg) < 0) {
-                    fprintf(stderr, "Unknown -balloon argument %s\n", optarg);
+                    error_report("unknown -balloon argument %s", optarg);
                     return 1;
                 }
                 break;
@@ -3905,18 +4048,18 @@
                 break;
             case QEMU_OPTION_uuid:
                 if(qemu_uuid_parse(optarg, qemu_uuid) < 0) {
-                    fprintf(stderr, "Fail to parse UUID string."
-                            " Wrong format.\n");
+                    error_report("failed to parse UUID string: wrong format");
                     return 1;
                 }
                 qemu_uuid_set = true;
                 break;
-	    case QEMU_OPTION_option_rom:
-		if (nb_option_roms >= MAX_OPTION_ROMS) {
-		    fprintf(stderr, "Too many option ROMs\n");
-            return 1;
-		}
-                opts = qemu_opts_parse(qemu_find_opts("option-rom"), optarg, 1);
+            case QEMU_OPTION_option_rom:
+                if (nb_option_roms >= MAX_OPTION_ROMS) {
+                    error_report("too many option ROMs");
+                    return 1;
+                }
+                opts = qemu_opts_parse_noisily(qemu_find_opts("option-rom"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
@@ -3924,27 +4067,59 @@
                 option_rom[nb_option_roms].bootindex =
                     qemu_opt_get_number(opts, "bootindex", -1);
                 if (!option_rom[nb_option_roms].name) {
-                    fprintf(stderr, "Option ROM file is not specified\n");
+                    error_report("Option ROM file is not specified");
                     return 1;
                 }
-		nb_option_roms++;
-		break;
+                nb_option_roms++;
+                break;
             case QEMU_OPTION_semihosting:
-                semihosting_enabled = 1;
+                semihosting.enabled = true;
+                semihosting.target = SEMIHOSTING_TARGET_AUTO;
+                break;
+            case QEMU_OPTION_semihosting_config:
+                semihosting.enabled = true;
+                opts = qemu_opts_parse_noisily(qemu_find_opts("semihosting-config"),
+                                               optarg, false);
+                if (opts != NULL) {
+                    semihosting.enabled = qemu_opt_get_bool(opts, "enable",
+                                                            true);
+                    const char *target = qemu_opt_get(opts, "target");
+                    if (target != NULL) {
+                        if (strcmp("native", target) == 0) {
+                            semihosting.target = SEMIHOSTING_TARGET_NATIVE;
+                        } else if (strcmp("gdb", target) == 0) {
+                            semihosting.target = SEMIHOSTING_TARGET_GDB;
+                        } else  if (strcmp("auto", target) == 0) {
+                            semihosting.target = SEMIHOSTING_TARGET_AUTO;
+                        } else {
+                            error_report("unsupported semihosting-config %s",
+                                         optarg);
+                            return 1;
+                        }
+                    } else {
+                        semihosting.target = SEMIHOSTING_TARGET_AUTO;
+                    }
+                    /* Set semihosting argument count and vector */
+                    qemu_opt_foreach(opts, add_semihosting_arg,
+                                     &semihosting, NULL);
+                } else {
+                    error_report("unsupported semihosting-config %s", optarg);
+                    return 1;
+                }
                 break;
             case QEMU_OPTION_tdf:
-                fprintf(stderr, "Warning: user space PIT time drift fix "
-                                "is no longer supported.\n");
+                error_report("warning: ignoring deprecated option");
                 break;
             case QEMU_OPTION_name:
-                opts = qemu_opts_parse(qemu_find_opts("name"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("name"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_prom_env:
                 if (nb_prom_envs >= MAX_PROM_ENVS) {
-                    fprintf(stderr, "Too many prom variables\n");
+                    error_report("too many prom variables");
                     return 1;
                 }
                 prom_envs[nb_prom_envs] = optarg;
@@ -3964,7 +4139,8 @@
                 }
                 break;
             case QEMU_OPTION_rtc:
-                opts = qemu_opts_parse(qemu_find_opts("rtc"), optarg, 0);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("rtc"), optarg,
+                                               false);
                 if (!opts) {
                     return 1;
                 }
@@ -3979,56 +4155,52 @@
                 }
                 break;
             case QEMU_OPTION_icount:
-                icount_opts = qemu_opts_parse(qemu_find_opts("icount"),
-                                              optarg, 1);
+                icount_opts = qemu_opts_parse_noisily(qemu_find_opts("icount"),
+                                                      optarg, true);
                 if (!icount_opts) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_incoming:
+                if (!incoming) {
+                    runstate_set(RUN_STATE_INMIGRATE);
+                }
                 incoming = optarg;
-                runstate_set(RUN_STATE_INMIGRATE);
                 break;
             case QEMU_OPTION_nodefaults:
                 has_defaults = 0;
                 break;
             case QEMU_OPTION_xen_domid:
                 if (!(xen_available())) {
-                    printf("Option %s not supported for this target\n", popt->name);
+                    error_report("Option not supported for this target");
                     return 1;
                 }
                 xen_domid = atoi(optarg);
                 break;
             case QEMU_OPTION_xen_create:
                 if (!(xen_available())) {
-                    printf("Option %s not supported for this target\n", popt->name);
+                    error_report("Option not supported for this target");
                     return 1;
                 }
                 xen_mode = XEN_CREATE;
                 break;
             case QEMU_OPTION_xen_attach:
                 if (!(xen_available())) {
-                    printf("Option %s not supported for this target\n", popt->name);
+                    error_report("Option not supported for this target");
                     return 1;
                 }
                 xen_mode = XEN_ATTACH;
                 break;
             case QEMU_OPTION_trace:
-            {
-                opts = qemu_opts_parse(qemu_find_opts("trace"), optarg, 0);
-                if (!opts) {
-                    return 1;
-                }
-                trace_events = qemu_opt_get(opts, "events");
-                trace_file = qemu_opt_get(opts, "file");
+                g_free(trace_file);
+                trace_file = trace_opt_parse(optarg);
                 break;
-            }
             case QEMU_OPTION_readconfig:
                 {
                     int ret = qemu_read_config_file(optarg);
                     if (ret < 0) {
-                        fprintf(stderr, "read config %s: %s\n", optarg,
-                            strerror(-ret));
+                        error_report("read config %s: %s", optarg,
+                                     strerror(-ret));
                         return 1;
                     }
                     break;
@@ -4036,10 +4208,10 @@
             case QEMU_OPTION_spice:
                 olist = qemu_find_opts("spice");
                 if (!olist) {
-                    fprintf(stderr, "spice is not supported by this qemu build.\n");
+                    error_report("spice support is disabled");
                     return 1;
                 }
-                opts = qemu_opts_parse(olist, optarg, 0);
+                opts = qemu_opts_parse_noisily(olist, optarg, false);
                 if (!opts) {
                     return 1;
                 }
@@ -4053,7 +4225,8 @@
                     } else {
                         fp = fopen(optarg, "w");
                         if (fp == NULL) {
-                            fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
+                            error_report("open %s: %s", optarg,
+                                         strerror(errno));
                             return 1;
                         }
                     }
@@ -4070,14 +4243,16 @@
                 qtest_log = optarg;
                 break;
             case QEMU_OPTION_sandbox:
-                opts = qemu_opts_parse(qemu_find_opts("sandbox"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("sandbox"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_add_fd:
 #ifndef _WIN32
-                opts = qemu_opts_parse(qemu_find_opts("add-fd"), optarg, 0);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("add-fd"),
+                                               optarg, false);
                 if (!opts) {
                     return 1;
                 }
@@ -4088,20 +4263,23 @@
 #endif
                 break;
             case QEMU_OPTION_object:
-                opts = qemu_opts_parse(qemu_find_opts("object"), optarg, 1);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("object"),
+                                               optarg, true);
                 if (!opts) {
                     return 1;
                 }
                 break;
             case QEMU_OPTION_realtime:
-                opts = qemu_opts_parse(qemu_find_opts("realtime"), optarg, 0);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("realtime"),
+                                               optarg, false);
                 if (!opts) {
                     return 1;
                 }
                 enable_mlock = qemu_opt_get_bool(opts, "mlock", true);
                 break;
             case QEMU_OPTION_msg:
-                opts = qemu_opts_parse(qemu_find_opts("msg"), optarg, 0);
+                opts = qemu_opts_parse_noisily(qemu_find_opts("msg"), optarg,
+                                               false);
                 if (!opts) {
                     return 1;
                 }
@@ -4109,13 +4287,13 @@
                 break;
             case QEMU_OPTION_dump_vmstate:
                 if (vmstate_dump_file) {
-                    fprintf(stderr, "qemu: only one '-dump-vmstate' "
-                            "option may be given\n");
+                    error_report("only one '-dump-vmstate' "
+                                 "option may be given");
                     return 1;
                 }
                 vmstate_dump_file = fopen(optarg, "w");
                 if (vmstate_dump_file == NULL) {
-                    fprintf(stderr, "open %s: %s\n", optarg, strerror(errno));
+                    error_report("open %s: %s", optarg, strerror(errno));
                     return 1;
                 }
                 break;
@@ -4167,6 +4345,7 @@
             case QEMU_OPTION_android_report_console:
                 android_op_report_console = (char*)optarg;
                 break;
+
             case QEMU_OPTION_android_skip_adb_auth:
                 adb_auth = 0;
                 break;
@@ -4176,24 +4355,43 @@
                             (char*)optarg);
                 }
                 break;
+
 #endif  // CONFIG_ANDROID
             default:
                 os_parse_cmd_args(popt->index, optarg);
             }
         }
     }
-
+    /*
+     * Clear error location left behind by the loop.
+     * Best done right after the loop.  Do not insert code here!
+     */
     loc_set_none();
 
+    replay_configure(icount_opts);
+
+    machine_class = select_machine();
+    if (!machine_class) {
+        return 1;
+    }
+
+    if (!set_memory_options(&ram_slots, &maxram_size, machine_class)) {
+        return 1;
+    }
+
     os_daemonize();
 
     if (qemu_init_main_loop(&main_loop_err)) {
-        error_report("%s", error_get_pretty(main_loop_err));
+        error_report_err(main_loop_err);
         return 1;
     }
 
 #ifdef CONFIG_ANDROID
 
+    if (!qemu_android_emulation_early_setup()) {
+        return 1;
+    }
+
     /* At this point, both block drivers and main loop have been initialized
      * which means we can create the QCoW2 images to boot from.
      * The QCoW2 images are backed by the "raw" images specified in the AVD
@@ -4203,13 +4401,6 @@
         return 1;
     }
 
-    /* Ensure Looper implementation for this thread is based on the QEMU
-     * main loop. */
-    qemu_looper_setForThread();
-
-    /* make sure qemud is initialized before any calls to it */
-    android_qemu2_qemud_init();
-
     boot_property_init_service();
     android_hw_control_init();
 
@@ -4253,8 +4444,8 @@
                   width, height, depth);
             return 1;
         }
-        android_display_width  = width;
-        android_display_height = height;
+        graphic_width  = width;
+        graphic_height = height;
         goldfish_fb_set_display_depth(depth);
     }
 
@@ -4311,8 +4502,9 @@
     /* Initialize presence of hardware nav button */
     boot_property_add("qemu.hw.mainkeys", android_hw->hw_mainKeys ? "1" : "0");
 
-    /* Bypass adb security or not. */
-    boot_property_add("qemu.adb.secure", adb_auth ? "1": "0");
+    /* Enable ADB authenticaiton, or not. */
+    boot_property_add("qemu.adb.secure", adb_auth ? "1" : "0");
+
     if (android_hw->hw_gsmModem) {
         if (android_qemud_get_channel(ANDROID_QEMUD_GSM,
                                       &android_modem_serial_line) < 0) {
@@ -4335,108 +4527,69 @@
         boot_property_add("qemu.sf.lcd_density", temp);
     }
 
-    int dns_count = 0;
-    if (android_op_dns_server) {
-        dns_count = slirp_parse_dns_servers(android_op_dns_server);
-        if (dns_count == -2) {
-            // Special case for better user feedback on this error message
-            fprintf(stderr,
-                    "too many servers specified in -dns-server-parameter "
-                    "argument '%s'. A maximum of %d is supported.\n",
-                    android_op_dns_server,
-                    slirp_get_max_dns_servers());
-            return 1;
-        } else if (dns_count < 0) {
-            fprintf(stderr, "invalid -dns-server parameter '%s'\n",
-                    android_op_dns_server);
-            return 1;
-        }
-        if (dns_count == 0) {
-            printf("### WARNING: will use system default DNS server\n");
-        }
-    }
-    if (dns_count == 0) {
-        dns_count = slirp_get_system_dns_servers();
-        if (dns_count < 0) {
-            printf("### WARNING: unable to configure any DNS servers, "
-                   "name resolution will not work\n");
-        }
-    }
-    if (dns_count > 1) {
-        additional_kernel_params = g_strdup_printf("ndns=%d", dns_count);
-    }
-
 #endif // CONFIG_ANDROID
 
-    if (qemu_opts_foreach(qemu_find_opts("sandbox"), parse_sandbox, NULL, 0)) {
+    if (qemu_opts_foreach(qemu_find_opts("sandbox"),
+                          parse_sandbox, NULL, NULL)) {
         return 1;
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("name"), parse_name, NULL, 1)) {
+    if (qemu_opts_foreach(qemu_find_opts("name"),
+                          parse_name, NULL, NULL)) {
         return 1;
     }
 
 #ifndef _WIN32
-    if (qemu_opts_foreach(qemu_find_opts("add-fd"), parse_add_fd, NULL, 1)) {
+    if (qemu_opts_foreach(qemu_find_opts("add-fd"),
+                          parse_add_fd, NULL, NULL)) {
         return 1;
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("add-fd"), cleanup_add_fd, NULL, 1)) {
+    if (qemu_opts_foreach(qemu_find_opts("add-fd"),
+                          cleanup_add_fd, NULL, NULL)) {
         return 1;
     }
 #endif
 
-    if (machine_class == NULL) {
-        fprintf(stderr, "No machine specified, and there is no default.\n"
-                "Use -machine help to list supported machines!\n");
-        return 1;
-    }
-
     current_machine = MACHINE(object_new(object_class_get_name(
                           OBJECT_CLASS(machine_class))));
+    if (machine_help_func(qemu_get_machine_opts(), current_machine)) {
+        return 0;
+    }
     object_property_add_child(object_get_root(), "machine",
                               OBJECT(current_machine), &error_abort);
     cpu_exec_init_all();
 
     if (machine_class->hw_version) {
-        qemu_set_version(machine_class->hw_version);
+        qemu_set_hw_version(machine_class->hw_version);
     }
 
-    /* Init CPU def lists, based on config
-     * - Must be called after all the qemu_read_config_file() calls
-     * - Must be called before list_cpus()
-     * - Must be called before machine->init()
-     */
-    cpudef_init();
-
     if (cpu_model && is_help_option(cpu_model)) {
         list_cpus(stdout, &fprintf, cpu_model);
         return 0;
     }
 
-    /* Open the logfile at this point, if necessary. We can't open the logfile
-     * when encountering either of the logging options (-d or -D) because the
-     * other one may be encountered later on the command line, changing the
-     * location or level of logging.
+    if (!trace_init_backends()) {
+        return 1;
+    }
+    trace_init_file(trace_file);
+
+    /* Open the logfile at this point and set the log mask if necessary.
      */
+    if (log_file) {
+        qemu_set_log_filename(log_file, &error_fatal);
+    }
+
     if (log_mask) {
         int mask;
-        if (log_file) {
-            qemu_set_log_filename(log_file);
-        }
-
         mask = qemu_str_to_log_mask(log_mask);
         if (!mask) {
             qemu_print_log_usage(stdout);
             return 1;
         }
         qemu_set_log(mask);
-    }
-
-    if (!is_daemonized()) {
-        if (!trace_init_backends(trace_events, trace_file)) {
-            return 1;
-        }
+    } else {
+        qemu_set_log(0);
     }
 
     /* If no data_dir is specified then try to find it relative to the
@@ -4452,16 +4605,24 @@
         data_dir[data_dir_idx++] = CONFIG_QEMU_DATADIR;
     }
 
+    /* -L help lists the data directories and exits. */
+    if (list_data_dirs) {
+        for (i = 0; i < data_dir_idx; i++) {
+            printf("%s\n", data_dir[i]);
+        }
+        return 0;
+    }
+
     if (!smp_parse(qemu_opts_find(qemu_find_opts("smp-opts"), NULL))) {
-        return false;
+        return 1;
     }
 
     machine_class->max_cpus = machine_class->max_cpus ?: 1; /* Default to UP */
-    if (smp_cpus > machine_class->max_cpus) {
-        fprintf(stderr, "Number of SMP cpus requested (%d), exceeds max cpus "
-                "supported by machine `%s' (%d)\n", smp_cpus,
-                machine_class->name, machine_class->max_cpus);
-        return 1;
+    if (max_cpus > machine_class->max_cpus) {
+        error_report("Number of SMP CPUs requested (%d) exceeds max CPUs "
+                     "supported by machine '%s' (%d)", max_cpus,
+                     machine_class->name, machine_class->max_cpus);
+        return 1;;
     }
 
     /*
@@ -4473,8 +4634,10 @@
                                machine_class->default_machine_opts, 0);
     }
 
-    qemu_opts_foreach(qemu_find_opts("device"), default_driver_check, NULL, 0);
-    qemu_opts_foreach(qemu_find_opts("global"), default_driver_check, NULL, 0);
+    qemu_opts_foreach(qemu_find_opts("device"),
+                      default_driver_check, NULL, NULL);
+    qemu_opts_foreach(qemu_find_opts("global"),
+                      default_driver_check, NULL, NULL);
 
     if (!vga_model && !default_vga) {
         vga_interface_type = VGA_DEVICE;
@@ -4516,21 +4679,21 @@
          * -nographic _and_ redirects all ports explicitly - this is valid
          * usage, -nographic is just a no-op in this case.
          */
-        if (display_type == DT_NOGRAPHIC
+        if (nographic
             && (default_parallel || default_serial
                 || default_monitor || default_virtcon)) {
-            fprintf(stderr, "-nographic can not be used with -daemonize\n");
+            error_report("-nographic cannot be used with -daemonize");
             return 1;
         }
 #ifdef CONFIG_CURSES
         if (display_type == DT_CURSES) {
-            fprintf(stderr, "curses display can not be used with -daemonize\n");
+            error_report("curses display cannot be used with -daemonize");
             return 1;
         }
 #endif
     }
 
-    if (display_type == DT_NOGRAPHIC) {
+    if (nographic) {
         if (default_parallel)
             add_device_config(DEV_PARALLEL, "null");
         if (default_serial && default_monitor) {
@@ -4547,11 +4710,10 @@
             if (default_sclp) {
                 add_device_config(DEV_SCLP, "stdio");
             }
-            if (default_monitor) {
-                if (!monitor_parse("stdio", "readline")) {
+            if (default_monitor)
+                if (!monitor_parse("stdio", "readline", false)) {
                     return 1;
                 }
-            }
         }
     } else {
         if (default_serial)
@@ -4559,7 +4721,7 @@
         if (default_parallel)
             add_device_config(DEV_PARALLEL, "vc:80Cx24C");
         if (default_monitor) {
-            if (!monitor_parse("vc:80Cx24C", "readline")) {
+            if (!monitor_parse("vc:80Cx24C", "readline", false)) {
                 return 1;
             }
         }
@@ -4570,33 +4732,52 @@
         }
     }
 
+#if defined(CONFIG_VNC)
+    if (!QTAILQ_EMPTY(&(qemu_find_opts("vnc")->head))) {
+        display_remote++;
+    }
+#endif
     if (display_type == DT_DEFAULT && !display_remote) {
 #if defined(CONFIG_GTK)
         display_type = DT_GTK;
-#elif defined(CONFIG_SDL) || defined(CONFIG_COCOA) || defined(CONFIG_ANDROID)
+#elif defined(CONFIG_SDL) || defined(CONFIG_ANDROID)
         display_type = DT_SDL;
+#elif defined(CONFIG_COCOA)
+        display_type = DT_COCOA;
 #elif defined(CONFIG_VNC)
-        vnc_display = "localhost:0,to=99";
-        show_vnc_port = 1;
+        vnc_parse("localhost:0,to=99,id=default", &error_abort);
 #else
         display_type = DT_NONE;
 #endif
     }
 
     if ((no_frame || alt_grab || ctrl_grab) && display_type != DT_SDL) {
-        fprintf(stderr, "-no-frame, -alt-grab and -ctrl-grab are only valid "
-                        "for SDL, ignoring option\n");
+        error_report("-no-frame, -alt-grab and -ctrl-grab are only valid "
+                     "for SDL, ignoring option");
     }
     if (no_quit && (display_type != DT_GTK && display_type != DT_SDL)) {
-        fprintf(stderr, "-no-quit is only valid for GTK and SDL, "
-                        "ignoring option\n");
+        error_report("-no-quit is only valid for GTK and SDL, "
+                     "ignoring option");
     }
 
-#if defined(CONFIG_GTK)
     if (display_type == DT_GTK) {
-        early_gtk_display_init();
+        early_gtk_display_init(request_opengl);
     }
+
+    if (display_type == DT_SDL) {
+        sdl_display_early_init(request_opengl);
+    }
+
+    if (request_opengl == 1 && display_opengl == 0) {
+#if defined(CONFIG_OPENGL)
+        error_report("OpenGL is not supported by the display");
+#else
+        error_report("OpenGL support is disabled");
 #endif
+        return 1;
+    }
+
+    page_size_init();
 
 #ifndef CONFIG_ANDROID
     // When using AndroidEmu, this "main" is no longer the entry point on the
@@ -4605,31 +4786,31 @@
     socket_init();
 #endif
 
-    if (qemu_opts_foreach(qemu_find_opts("chardev"), chardev_init_func, NULL, 1) != 0)
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          user_creatable_add_opts_foreach,
+                          object_create_initial, NULL)) {
         return 1;
+    }
+
+    if (qemu_opts_foreach(qemu_find_opts("chardev"),
+                          chardev_init_func, NULL, NULL)) {
+        return 1;
+    }
+
 #ifdef CONFIG_VIRTFS
-    if (qemu_opts_foreach(qemu_find_opts("fsdev"), fsdev_init_func, NULL, 1) != 0) {
+    if (qemu_opts_foreach(qemu_find_opts("fsdev"),
+                          fsdev_init_func, NULL, NULL)) {
         return 1;
     }
 #endif
 
     if (pid_file && qemu_create_pidfile(pid_file) != 0) {
-        fprintf(stderr, "Could not acquire pid file: %s\n", strerror(errno));
+        error_report("could not acquire pid file: %s", strerror(errno));
         return 1;
     }
-#if defined(CONFIG_HAX)
+#ifdef CONFIG_HAX
     uint64_t hax_max_ram = 0;
     if (hax_get_max_ram(&hax_max_ram) == 0 && hax_max_ram > 0) {
-#ifdef CONFIG_ANDROID
-        char str[32] = {0};
-        snprintf(str, sizeof(str) - 1, "%"PRIu64, hax_max_ram);
-        crashhandler_add_string("hax_max_ram.txt", str);
-#endif
-        // current haxm cannot handle more than 3G memory
-        // b.android.com/214422
-        if (hax_max_ram  > 3072LL * 1024 * 1024) {
-            hax_max_ram = 3072LL * 1024 * 1024;
-        }
         if (ram_size > hax_max_ram) {
             const int requested_meg = ram_size / (1024 * 1024);
             const int actual_meg = hax_max_ram / (1024 * 1024);
@@ -4638,42 +4819,26 @@
             ram_size = hax_max_ram;
         }
     }
-#endif  /* CONFIG_HAX */
+#endif /* CONFIG_HAX */
 
-    /* store value for the future use */
-    qemu_opt_set_number(qemu_find_opts_singleton("memory"), "size", ram_size);
-    hax_pre_init(ram_size);
-
-    if (qemu_opts_foreach(qemu_find_opts("device"), device_help_func, NULL, 0)
-        != 0) {
+    if (qemu_opts_foreach(qemu_find_opts("device"),
+                          device_help_func, NULL, NULL)) {
         return 0;
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("object"),
-                          object_create, NULL, 0) != 0) {
-        return 1;
-    }
-
     machine_opts = qemu_get_machine_opts();
     if (qemu_opt_foreach(machine_opts, machine_set_property, current_machine,
-                         1) < 0) {
+                         NULL)) {
         object_unref(OBJECT(current_machine));
         return 1;
     }
 
-    if (configure_accelerator(current_machine)) {
+    if (configure_accelerator(current_machine) < 0) {
         return 1;
     }
 
-
     if (qtest_chrdev) {
-        Error *local_err = NULL;
-        qtest_init(qtest_chrdev, qtest_log, &local_err);
-        if (local_err) {
-            error_report("%s", error_get_pretty(local_err));
-            error_free(local_err);
-            return 1;
-        }
+        qtest_init(qtest_chrdev, qtest_log, &error_fatal);
     }
 
     machine_opts = qemu_get_machine_opts();
@@ -4682,94 +4847,117 @@
     kernel_cmdline = qemu_opt_get(machine_opts, "append");
     bios_name = qemu_opt_get(machine_opts, "firmware");
 
-    boot_order = machine_class->default_boot_order;
     opts = qemu_opts_find(qemu_find_opts("boot-opts"), NULL);
     if (opts) {
-        char *normal_boot_order;
-        const char *order, *once;
-
-        order = qemu_opt_get(opts, "order");
-        if (order) {
-            if (!validate_bootdevices(order)) {
-                return 1;
-            }
-            boot_order = order;
+        boot_order = qemu_opt_get(opts, "order");
+        if (boot_order) {
+            validate_bootdevices(boot_order, &error_fatal);
         }
 
-        once = qemu_opt_get(opts, "once");
-        if (once) {
-            if (!validate_bootdevices(once)) {
-                return 1;
-            }
-            normal_boot_order = g_strdup(boot_order);
-            boot_order = once;
-            qemu_register_reset(restore_boot_order, normal_boot_order);
+        boot_once = qemu_opt_get(opts, "once");
+        if (boot_once) {
+            validate_bootdevices(boot_once, &error_fatal);
         }
 
         boot_menu = qemu_opt_get_bool(opts, "menu", boot_menu);
         boot_strict = qemu_opt_get_bool(opts, "strict", false);
     }
 
+    if (!boot_order) {
+        boot_order = machine_class->default_boot_order;
+    }
+
     current_machine->kernel_cmdline =
             g_strdup(kernel_cmdline ? kernel_cmdline : "");
 
-#ifdef CONFIG_ANDROID
-    if (additional_kernel_params) {
-        char* combined = g_strdup_printf("%s %s",
-                                         current_machine->kernel_cmdline,
-                                         additional_kernel_params);
-        g_free(additional_kernel_params);
-        additional_kernel_params = NULL;
-
-        g_free(current_machine->kernel_cmdline);
-        current_machine->kernel_cmdline = combined;
-    }
-#endif  // CONFIG_ANDROID
-
     linux_boot = (kernel_filename != NULL);
 
     if (!linux_boot && *current_machine->kernel_cmdline != '\0') {
-        fprintf(stderr, "-append only allowed with -kernel option\n");
+        error_report("-append only allowed with -kernel option");
         return 1;
     }
 
     if (!linux_boot && initrd_filename != NULL) {
-        fprintf(stderr, "-initrd only allowed with -kernel option\n");
+        error_report("-initrd only allowed with -kernel option");
         return 1;
     }
 
     if (!linux_boot && qemu_opt_get(machine_opts, "dtb")) {
-        fprintf(stderr, "-dtb only allowed with -kernel option\n");
+        error_report("-dtb only allowed with -kernel option");
         return 1;
     }
 
+    if (semihosting_enabled() && !semihosting_get_argc() && kernel_filename) {
+        /* fall back to the -kernel/-append */
+        semihosting_arg_fallback(kernel_filename,
+                                 current_machine->kernel_cmdline);
+    }
+
     os_set_line_buffering();
 
-    qemu_init_cpu_loop();
-    qemu_mutex_lock_iothread();
-
-#ifdef CONFIG_SPICE
     /* spice needs the timers to be initialized by this point */
     qemu_spice_init();
-#endif
 
     cpu_ticks_init();
     if (icount_opts) {
+#ifdef CONFIG_HAX
         if (kvm_enabled() || xen_enabled() || hax_enabled()) {
-            fprintf(stderr, "-icount is not allowed with kvm or xen or hax\n");
+            error_report("-icount is not allowed with kvm, hax or xen");
             return 1;
         }
+#else
+        if (kvm_enabled() || xen_enabled()) {
+            error_report("-icount is not allowed with kvm or xen");
+            return 1;
+        }
+#endif
         configure_icount(icount_opts, &error_abort);
         qemu_opts_del(icount_opts);
     }
 
-    /* clean up network at qemu process termination */
-    atexit(&net_cleanup);
+    if (default_net) {
+        QemuOptsList *net = qemu_find_opts("net");
+        qemu_opts_set(net, NULL, "type", "nic", &error_abort);
+#ifdef CONFIG_SLIRP
+        qemu_opts_set(net, NULL, "type", "user", &error_abort);
+#endif
+    }
 
     if (net_init_clients() < 0) {
         return 1;
     }
 
+#ifdef CONFIG_ANDROID
+    char *additional_kernel_params = NULL;
+    int dns_count = 0;
+    int dns6_count = 0;
+    if (android_op_dns_server) {
+        if (!qemu_android_emulation_setup_dns_servers(
+                android_op_dns_server, &dns_count, &dns6_count)) {
+            fprintf(stderr, "invalid -dns-server parameter '%s'\n",
+                    android_op_dns_server);
+            return 1;
+        }
+        // TODO: Find a way to pass the number of IPv6 DNS servers to
+        // the guest system.
+        if (dns_count > 1) {
+            char* combined = g_strdup_printf("%s ndns=%d",
+                                             current_machine->kernel_cmdline,
+                                             dns_count);
+            g_free(current_machine->kernel_cmdline);
+            current_machine->kernel_cmdline = combined;
+        }
+    }
+
+    qemu_android_emulation_init_slirp();
+#endif
+
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          user_creatable_add_opts_foreach,
+                          object_create_delayed, NULL)) {
+        return 1;
+    }
+
 #ifdef CONFIG_TPM
     if (tpm_init() < 0) {
         return 1;
@@ -4783,7 +4971,7 @@
     if (!xen_enabled()) {
         /* On 32-bit hosts, QEMU is limited by virtual address space */
         if (ram_size > (2047 << 20) && HOST_LONG_BITS == 32) {
-            fprintf(stderr, "qemu: at most 2047 MB RAM can be simulated\n");
+            error_report("at most 2047 MB RAM can be simulated");
             return 1;
         }
     }
@@ -4799,10 +4987,12 @@
     }
 
     /* open the virtual block devices */
-    if (snapshot)
-        qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot, NULL, 0);
+    if (snapshot || replay_mode != REPLAY_MODE_NONE) {
+        qemu_opts_foreach(qemu_find_opts("drive"), drive_enable_snapshot,
+                          NULL, NULL);
+    }
     if (qemu_opts_foreach(qemu_find_opts("drive"), drive_init_func,
-                          &machine_class->block_default_type, 1) != 0) {
+                          &machine_class->block_default_type, NULL)) {
         return 1;
     }
 
@@ -4817,13 +5007,13 @@
         return 1;
     }
 
-    if (qemu_opts_foreach(qemu_find_opts("numa"), numa_init_func,
-                          NULL, 1) != 0) {
+    parse_numa_opts(machine_class);
+
+    if (qemu_opts_foreach(qemu_find_opts("mon"),
+                          mon_init_func, NULL, NULL)) {
         return 1;
     }
 
-    set_numa_nodes();
-
     if (foreach_device_config(DEV_SERIAL, serial_parse) < 0)
         return 1;
     if (foreach_device_config(DEV_PARALLEL, parallel_parse) < 0)
@@ -4840,14 +5030,16 @@
     if (default_vga) {
         if (machine_class->default_display) {
             vga_model = machine_class->default_display;
-        } else if (cirrus_vga_available()) {
+        } else if (vga_interface_available(VGA_CIRRUS)) {
             vga_model = "cirrus";
-        } else if (vga_available()) {
+        } else if (vga_interface_available(VGA_STD)) {
             vga_model = "std";
         }
     }
     if (vga_model) {
-        select_vgahw(vga_model);
+        if (!select_vgahw(vga_model)) {
+            return 1;
+        }
     }
 
     if (watchdog) {
@@ -4856,9 +5048,10 @@
             return (i == 1 ? 1 : 0);
     }
 
-    if (machine_class->compat_props) {
-        qdev_prop_register_global_list(machine_class->compat_props);
-    }
+    machine_register_compat_props(current_machine);
+
+    qemu_opts_foreach(qemu_find_opts("global"),
+                      global_init_func, NULL, NULL);
 
 #if defined(CONFIG_ANDROID)
     /* Configure goldfish events device */
@@ -4925,10 +5118,17 @@
                     &qemu2_goldfish_event_multitouch_funcs);
         }
     }
+
+    // Parse the System boot parameters from the command line last,
+    // so they take precedence
+    process_cmd_properties();
+
 #endif  // CONFIG_ANDROID
 
-    qemu_add_globals();
-
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_INIT);
     qdev_machine_init();
 
     current_machine->ram_size = ram_size;
@@ -4938,12 +5138,18 @@
     current_machine->cpu_model = cpu_model;
 
     machine_class->init(current_machine);
-#ifdef CONFIG_ANDROID
-    if (android_init_error_occurred()) {
-        // Something went wrong when initializing the virtual machine
+    if (error_during_init != NULL) {
         return 1;
     }
-#endif
+
+#ifdef CONFIG_ANDROID
+    if (!qemu_android_emulation_setup()) {
+        return 1;
+    }
+
+    extern void android_emulator_set_base_port(int);
+    android_emulator_set_base_port(android_base_port);
+#endif  // CONFIG_ANDROID
 
     if (!realtime_init()) {
         return 1;
@@ -4955,68 +5161,76 @@
 
     cpu_synchronize_all_post_init();
 
-    set_numa_modes();
+    numa_post_machine_init();
 
+#ifdef CONFIG_HAX
     if (hax_enabled()) {
         if (hax_sync_vcpus() < 0) {
-            fprintf(stderr, "Internal error: initial hax sync failed\n");
+            fprintf(stderr, "Internal error: Initial hax sync failed\n");
             return 1;
         }
     }
+#endif  /* CONFIG_HAX */
+
+    if (qemu_opts_foreach(qemu_find_opts("fw_cfg"),
+                          parse_fw_cfg, fw_cfg_find(), NULL) != 0) {
+        return 1;
+    }
 
     /* init USB devices */
-    if (usb_enabled(false)) {
+    if (machine_usb(current_machine)) {
         if (foreach_device_config(DEV_USB, usb_parse) < 0)
             return 1;
     }
 
+    /* Check if IGD GFX passthrough. */
+    igd_gfx_passthru();
+
     /* init generic devices */
-    if (qemu_opts_foreach(qemu_find_opts("device"), device_init_func, NULL, 1) != 0)
+    rom_set_order_override(FW_CFG_ORDER_OVERRIDE_DEVICE);
+    if (qemu_opts_foreach(qemu_find_opts("device"),
+                          device_init_func, NULL, NULL)) {
         return 1;
+    }
+    rom_reset_order_override();
 
     /* Did we create any drives that we failed to create a device for? */
     drive_check_orphaned();
 
-    net_check_clients();
-
-#if defined(CONFIG_ANDROID)
-    /* call android-specific setup function */
-    if (!qemu_android_emulation_setup()) {
-        return 1;
+    /* Don't warn about the default network setup that you get if
+     * no command line -net or -netdev options are specified. There
+     * are two cases that we would otherwise complain about:
+     * (1) board doesn't support a NIC but the implicit "-net nic"
+     * requested one
+     * (2) CONFIG_SLIRP not set, in which case the implicit "-net nic"
+     * sets up a nic that isn't connected to anything.
+     */
+    if (!default_net) {
+        net_check_clients();
     }
 
-    extern void android_emulator_set_base_port(int);
-    android_emulator_set_base_port(android_base_port);
-#endif  // CONFIG_ANDROID
+
+    if (boot_once) {
+        qemu_boot_set(boot_once, &error_fatal);
+        qemu_register_reset(restore_boot_order, g_strdup(boot_order));
+    }
 
     ds = init_displaystate();
 
     /* init local displays */
     switch (display_type) {
-    case DT_NOGRAPHIC:
-        (void)ds;	/* avoid warning if no display is configured */
-        break;
-#if defined(CONFIG_CURSES)
     case DT_CURSES:
         curses_display_init(ds, full_screen);
         break;
-#endif
-#if defined(CONFIG_SDL) || defined(CONFIG_ANDROID)
     case DT_SDL:
-        if (!sdl_display_init(ds, full_screen, no_frame)) {
-            return 1;
-        }
+        sdl_display_init(ds, full_screen, no_frame);
         break;
-#elif defined(CONFIG_COCOA)
-    case DT_SDL:
+    case DT_COCOA:
         cocoa_display_init(ds, full_screen);
         break;
-#endif
-#if defined(CONFIG_GTK)
     case DT_GTK:
         gtk_display_init(ds, full_screen, grab_on_hover);
         break;
-#endif
     default:
         break;
     }
@@ -5024,29 +5238,15 @@
     /* must be after terminal init, SDL library changes signal handlers */
     os_setup_signal_handling();
 
-#ifdef CONFIG_VNC
     /* init remote displays */
-    if (vnc_display) {
-        Error *local_err = NULL;
-        vnc_display_init(ds);
-        vnc_display_open(ds, vnc_display, &local_err);
-        if (local_err != NULL) {
-            error_report("Failed to start VNC server on `%s': %s",
-                         vnc_display, error_get_pretty(local_err));
-            error_free(local_err);
-            return 1;
-        }
-
-        if (show_vnc_port) {
-            printf("VNC server running on `%s'\n", vnc_display_local_addr(ds));
-        }
-    }
+#ifdef CONFIG_VNC
+    qemu_opts_foreach(qemu_find_opts("vnc"),
+                      vnc_init_func, NULL, NULL);
 #endif
-#ifdef CONFIG_SPICE
+
     if (using_spice) {
         qemu_spice_display_init();
     }
-#endif
 
     if (foreach_device_config(DEV_GDB, gdbserver_start) < 0) {
         return 1;
@@ -5054,30 +5254,24 @@
 
     qdev_machine_creation_done();
 
-    if (rom_load_all() != 0) {
-        fprintf(stderr, "rom loading failed\n");
-        return 1;
-    }
-
-    if (qemu_opts_foreach(qemu_find_opts("mon"), mon_init_func, NULL, 1) != 0) {
-        return 1;
-    }
-
-#ifdef CONFIG_ANDROID
-    // Parse the System boot parameters from the command line last,
-    // so they take precedence
-    process_cmd_properties();
-#endif  // CONFIG_ANDROID
-
     /* TODO: once all bus devices are qdevified, this should be done
      * when bus is created by qdev.c */
     qemu_register_reset(qbus_reset_all_fn, sysbus_get_default());
     qemu_run_machine_init_done_notifiers();
 
-    /* Done notifiers can load ROMs */
-    rom_load_done();
+    if (rom_check_and_register_reset() != 0) {
+        error_report("rom check and register reset failed");
+        return 1;
+    }
 
+    replay_start();
+
+    /* This checkpoint is required by replay to separate prior clock
+       reading from the other reads, because timer polling functions query
+       clock values from the log. */
+    replay_checkpoint(CHECKPOINT_RESET);
     qemu_system_reset(VMRESET_SILENT);
+    register_global_state();
     if (loadvm) {
         if (load_vmstate(loadvm) < 0) {
             autostart = 0;
@@ -5095,9 +5289,7 @@
         Error *local_err = NULL;
         qemu_start_incoming_migration(incoming, &local_err);
         if (local_err) {
-            error_report("-incoming %s: %s", incoming,
-                         error_get_pretty(local_err));
-            error_free(local_err);
+            error_reportf_err(local_err, "-incoming %s: ", incoming);
             return 1;
         }
     } else if (autostart) {
@@ -5106,12 +5298,6 @@
 
     os_setup_post();
 
-    if (is_daemonized()) {
-        if (!trace_init_backends(trace_events, trace_file)) {
-            return 1;
-        }
-    }
-
 #ifdef CONFIG_ANDROID
     // Initialize reporting right before entering main loop.
     // We want to track performance of a running emulator, ignoring any early
@@ -5119,15 +5305,18 @@
     if (!android_reporting_setup()) {
         return 1;
     }
-#endif
+#endif  // CONFIG_ANDROID
 
+    trace_init_vcpu_events();
     main_loop();
+    replay_disable_events();
 
 #ifdef CONFIG_ANDROID
     crashhandler_exitmode("after main_loop");
     android_wear_agent_stop();
     socket_drainer_stop();
 #endif
+
     bdrv_close_all();
     pause_all_vcpus();
     res_free();
@@ -5136,8 +5325,15 @@
 #endif
 
 #ifdef CONFIG_ANDROID
+    qemu_android_emulation_teardown();
     android_reporting_teardown();
 #endif
 
+    /* vhost-user must be cleaned up before chardevs.  */
+    net_cleanup();
+    audio_cleanup();
+    monitor_cleanup();
+    qemu_chr_cleanup();
+
     return 0;
 }
diff --git a/vmstate.c b/vmstate.c
deleted file mode 100644
index 3dde574..0000000
--- a/vmstate.c
+++ /dev/null
@@ -1,687 +0,0 @@
-#include "qemu-common.h"
-#include "migration/migration.h"
-#include "migration/qemu-file.h"
-#include "migration/vmstate.h"
-#include "qemu/bitops.h"
-#include "trace.h"
-
-static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
-                                    void *opaque);
-static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd,
-                                   void *opaque);
-
-static int vmstate_n_elems(void *opaque, VMStateField *field)
-{
-    int n_elems = 1;
-
-    if (field->flags & VMS_ARRAY) {
-        n_elems = field->num;
-    } else if (field->flags & VMS_VARRAY_INT32) {
-        n_elems = *(int32_t *)(opaque+field->num_offset);
-    } else if (field->flags & VMS_VARRAY_UINT32) {
-        n_elems = *(uint32_t *)(opaque+field->num_offset);
-    } else if (field->flags & VMS_VARRAY_UINT16) {
-        n_elems = *(uint16_t *)(opaque+field->num_offset);
-    } else if (field->flags & VMS_VARRAY_UINT8) {
-        n_elems = *(uint8_t *)(opaque+field->num_offset);
-    }
-
-    return n_elems;
-}
-
-static int vmstate_size(void *opaque, VMStateField *field)
-{
-    int size = field->size;
-
-    if (field->flags & VMS_VBUFFER) {
-        size = *(int32_t *)(opaque+field->size_offset);
-        if (field->flags & VMS_MULTIPLY) {
-            size *= field->size;
-        }
-    }
-
-    return size;
-}
-
-static void *vmstate_base_addr(void *opaque, VMStateField *field, bool alloc)
-{
-    void *base_addr = opaque + field->offset;
-
-    if (field->flags & VMS_POINTER) {
-        if (alloc && (field->flags & VMS_ALLOC)) {
-            gsize size = 0;
-            if (field->flags & VMS_VBUFFER) {
-                size = vmstate_size(opaque, field);
-            } else {
-                int n_elems = vmstate_n_elems(opaque, field);
-                if (n_elems) {
-                    size = n_elems * field->size;
-                }
-            }
-            if (size) {
-                *((void **)base_addr + field->start) = g_malloc(size);
-            }
-        }
-        base_addr = *(void **)base_addr + field->start;
-    }
-
-    return base_addr;
-}
-
-int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
-                       void *opaque, int version_id)
-{
-    VMStateField *field = vmsd->fields;
-    int ret;
-
-    if (version_id > vmsd->version_id) {
-        return -EINVAL;
-    }
-    if  (version_id < vmsd->minimum_version_id) {
-        if (vmsd->load_state_old &&
-            version_id >= vmsd->minimum_version_id_old) {
-            return vmsd->load_state_old(f, opaque, version_id);
-        }
-        return -EINVAL;
-    }
-    if (vmsd->pre_load) {
-        int ret = vmsd->pre_load(opaque);
-        if (ret) {
-            return ret;
-        }
-    }
-    while (field->name) {
-        if ((field->field_exists &&
-             field->field_exists(opaque, version_id)) ||
-            (!field->field_exists &&
-             field->version_id <= version_id)) {
-            void *base_addr = vmstate_base_addr(opaque, field, true);
-            int i, n_elems = vmstate_n_elems(opaque, field);
-            int size = vmstate_size(opaque, field);
-
-            for (i = 0; i < n_elems; i++) {
-                void *addr = base_addr + size * i;
-
-                if (field->flags & VMS_ARRAY_OF_POINTER) {
-                    addr = *(void **)addr;
-                }
-                if (field->flags & VMS_STRUCT) {
-                    ret = vmstate_load_state(f, field->vmsd, addr,
-                                             field->vmsd->version_id);
-                } else {
-                    ret = field->info->get(f, addr, size);
-
-                }
-                if (ret >= 0) {
-                    ret = qemu_file_get_error(f);
-                }
-                if (ret < 0) {
-                    qemu_file_set_error(f, ret);
-                    trace_vmstate_load_field_error(field->name, ret);
-                    return ret;
-                }
-            }
-        } else if (field->flags & VMS_MUST_EXIST) {
-            fprintf(stderr, "Input validation failed: %s/%s\n",
-                    vmsd->name, field->name);
-            return -1;
-        }
-        field++;
-    }
-    ret = vmstate_subsection_load(f, vmsd, opaque);
-    if (ret != 0) {
-        return ret;
-    }
-    if (vmsd->post_load) {
-        return vmsd->post_load(opaque, version_id);
-    }
-    return 0;
-}
-
-void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
-                        void *opaque)
-{
-    VMStateField *field = vmsd->fields;
-
-    if (vmsd->pre_save) {
-        vmsd->pre_save(opaque);
-    }
-    while (field->name) {
-        if (!field->field_exists ||
-            field->field_exists(opaque, vmsd->version_id)) {
-            void *base_addr = vmstate_base_addr(opaque, field, false);
-            int i, n_elems = vmstate_n_elems(opaque, field);
-            int size = vmstate_size(opaque, field);
-
-            for (i = 0; i < n_elems; i++) {
-                void *addr = base_addr + size * i;
-
-                if (field->flags & VMS_ARRAY_OF_POINTER) {
-                    addr = *(void **)addr;
-                }
-                if (field->flags & VMS_STRUCT) {
-                    vmstate_save_state(f, field->vmsd, addr);
-                } else {
-                    field->info->put(f, addr, size);
-                }
-            }
-        } else {
-            if (field->flags & VMS_MUST_EXIST) {
-                fprintf(stderr, "Output state validation failed: %s/%s\n",
-                        vmsd->name, field->name);
-                assert(!(field->flags & VMS_MUST_EXIST));
-            }
-        }
-        field++;
-    }
-    vmstate_subsection_save(f, vmsd, opaque);
-}
-
-static const VMStateDescription *
-    vmstate_get_subsection(const VMStateSubsection *sub, char *idstr)
-{
-    while (sub && sub->needed) {
-        if (strcmp(idstr, sub->vmsd->name) == 0) {
-            return sub->vmsd;
-        }
-        sub++;
-    }
-    return NULL;
-}
-
-static int vmstate_subsection_load(QEMUFile *f, const VMStateDescription *vmsd,
-                                   void *opaque)
-{
-    while (qemu_peek_byte(f, 0) == QEMU_VM_SUBSECTION) {
-        char idstr[256];
-        int ret;
-        uint8_t version_id, len, size;
-        const VMStateDescription *sub_vmsd;
-
-        len = qemu_peek_byte(f, 1);
-        if (len < strlen(vmsd->name) + 1) {
-            /* subsection name has be be "section_name/a" */
-            return 0;
-        }
-        size = qemu_peek_buffer(f, (uint8_t *)idstr, len, 2);
-        if (size != len) {
-            return 0;
-        }
-        idstr[size] = 0;
-
-        if (strncmp(vmsd->name, idstr, strlen(vmsd->name)) != 0) {
-            /* it don't have a valid subsection name */
-            return 0;
-        }
-        sub_vmsd = vmstate_get_subsection(vmsd->subsections, idstr);
-        if (sub_vmsd == NULL) {
-            return -ENOENT;
-        }
-        qemu_file_skip(f, 1); /* subsection */
-        qemu_file_skip(f, 1); /* len */
-        qemu_file_skip(f, len); /* idstr */
-        version_id = qemu_get_be32(f);
-
-        ret = vmstate_load_state(f, sub_vmsd, opaque, version_id);
-        if (ret) {
-            return ret;
-        }
-    }
-    return 0;
-}
-
-static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
-                                    void *opaque)
-{
-    const VMStateSubsection *sub = vmsd->subsections;
-
-    while (sub && sub->needed) {
-        if (sub->needed(opaque)) {
-            const VMStateDescription *vmsd = sub->vmsd;
-            uint8_t len;
-
-            qemu_put_byte(f, QEMU_VM_SUBSECTION);
-            len = strlen(vmsd->name);
-            qemu_put_byte(f, len);
-            qemu_put_buffer(f, (uint8_t *)vmsd->name, len);
-            qemu_put_be32(f, vmsd->version_id);
-            vmstate_save_state(f, vmsd, opaque);
-        }
-        sub++;
-    }
-}
-
-/* bool */
-
-static int get_bool(QEMUFile *f, void *pv, size_t size)
-{
-    bool *v = pv;
-    *v = qemu_get_byte(f);
-    return 0;
-}
-
-static void put_bool(QEMUFile *f, void *pv, size_t size)
-{
-    bool *v = pv;
-    qemu_put_byte(f, *v);
-}
-
-const VMStateInfo vmstate_info_bool = {
-    .name = "bool",
-    .get  = get_bool,
-    .put  = put_bool,
-};
-
-/* 8 bit int */
-
-static int get_int8(QEMUFile *f, void *pv, size_t size)
-{
-    int8_t *v = pv;
-    qemu_get_s8s(f, v);
-    return 0;
-}
-
-static void put_int8(QEMUFile *f, void *pv, size_t size)
-{
-    int8_t *v = pv;
-    qemu_put_s8s(f, v);
-}
-
-const VMStateInfo vmstate_info_int8 = {
-    .name = "int8",
-    .get  = get_int8,
-    .put  = put_int8,
-};
-
-/* 16 bit int */
-
-static int get_int16(QEMUFile *f, void *pv, size_t size)
-{
-    int16_t *v = pv;
-    qemu_get_sbe16s(f, v);
-    return 0;
-}
-
-static void put_int16(QEMUFile *f, void *pv, size_t size)
-{
-    int16_t *v = pv;
-    qemu_put_sbe16s(f, v);
-}
-
-const VMStateInfo vmstate_info_int16 = {
-    .name = "int16",
-    .get  = get_int16,
-    .put  = put_int16,
-};
-
-/* 32 bit int */
-
-static int get_int32(QEMUFile *f, void *pv, size_t size)
-{
-    int32_t *v = pv;
-    qemu_get_sbe32s(f, v);
-    return 0;
-}
-
-static void put_int32(QEMUFile *f, void *pv, size_t size)
-{
-    int32_t *v = pv;
-    qemu_put_sbe32s(f, v);
-}
-
-const VMStateInfo vmstate_info_int32 = {
-    .name = "int32",
-    .get  = get_int32,
-    .put  = put_int32,
-};
-
-/* 32 bit int. See that the received value is the same than the one
-   in the field */
-
-static int get_int32_equal(QEMUFile *f, void *pv, size_t size)
-{
-    int32_t *v = pv;
-    int32_t v2;
-    qemu_get_sbe32s(f, &v2);
-
-    if (*v == v2) {
-        return 0;
-    }
-    return -EINVAL;
-}
-
-const VMStateInfo vmstate_info_int32_equal = {
-    .name = "int32 equal",
-    .get  = get_int32_equal,
-    .put  = put_int32,
-};
-
-/* 32 bit int. Check that the received value is non-negative
- * and less than or equal to the one in the field.
- */
-
-static int get_int32_le(QEMUFile *f, void *pv, size_t size)
-{
-    int32_t *cur = pv;
-    int32_t loaded;
-    qemu_get_sbe32s(f, &loaded);
-
-    if (loaded >= 0 && loaded <= *cur) {
-        *cur = loaded;
-        return 0;
-    }
-    return -EINVAL;
-}
-
-const VMStateInfo vmstate_info_int32_le = {
-    .name = "int32 le",
-    .get  = get_int32_le,
-    .put  = put_int32,
-};
-
-/* 64 bit int */
-
-static int get_int64(QEMUFile *f, void *pv, size_t size)
-{
-    int64_t *v = pv;
-    qemu_get_sbe64s(f, v);
-    return 0;
-}
-
-static void put_int64(QEMUFile *f, void *pv, size_t size)
-{
-    int64_t *v = pv;
-    qemu_put_sbe64s(f, v);
-}
-
-const VMStateInfo vmstate_info_int64 = {
-    .name = "int64",
-    .get  = get_int64,
-    .put  = put_int64,
-};
-
-/* 8 bit unsigned int */
-
-static int get_uint8(QEMUFile *f, void *pv, size_t size)
-{
-    uint8_t *v = pv;
-    qemu_get_8s(f, v);
-    return 0;
-}
-
-static void put_uint8(QEMUFile *f, void *pv, size_t size)
-{
-    uint8_t *v = pv;
-    qemu_put_8s(f, v);
-}
-
-const VMStateInfo vmstate_info_uint8 = {
-    .name = "uint8",
-    .get  = get_uint8,
-    .put  = put_uint8,
-};
-
-/* 16 bit unsigned int */
-
-static int get_uint16(QEMUFile *f, void *pv, size_t size)
-{
-    uint16_t *v = pv;
-    qemu_get_be16s(f, v);
-    return 0;
-}
-
-static void put_uint16(QEMUFile *f, void *pv, size_t size)
-{
-    uint16_t *v = pv;
-    qemu_put_be16s(f, v);
-}
-
-const VMStateInfo vmstate_info_uint16 = {
-    .name = "uint16",
-    .get  = get_uint16,
-    .put  = put_uint16,
-};
-
-/* 32 bit unsigned int */
-
-static int get_uint32(QEMUFile *f, void *pv, size_t size)
-{
-    uint32_t *v = pv;
-    qemu_get_be32s(f, v);
-    return 0;
-}
-
-static void put_uint32(QEMUFile *f, void *pv, size_t size)
-{
-    uint32_t *v = pv;
-    qemu_put_be32s(f, v);
-}
-
-const VMStateInfo vmstate_info_uint32 = {
-    .name = "uint32",
-    .get  = get_uint32,
-    .put  = put_uint32,
-};
-
-/* 32 bit uint. See that the received value is the same than the one
-   in the field */
-
-static int get_uint32_equal(QEMUFile *f, void *pv, size_t size)
-{
-    uint32_t *v = pv;
-    uint32_t v2;
-    qemu_get_be32s(f, &v2);
-
-    if (*v == v2) {
-        return 0;
-    }
-    return -EINVAL;
-}
-
-const VMStateInfo vmstate_info_uint32_equal = {
-    .name = "uint32 equal",
-    .get  = get_uint32_equal,
-    .put  = put_uint32,
-};
-
-/* 64 bit unsigned int */
-
-static int get_uint64(QEMUFile *f, void *pv, size_t size)
-{
-    uint64_t *v = pv;
-    qemu_get_be64s(f, v);
-    return 0;
-}
-
-static void put_uint64(QEMUFile *f, void *pv, size_t size)
-{
-    uint64_t *v = pv;
-    qemu_put_be64s(f, v);
-}
-
-const VMStateInfo vmstate_info_uint64 = {
-    .name = "uint64",
-    .get  = get_uint64,
-    .put  = put_uint64,
-};
-
-/* 64 bit unsigned int. See that the received value is the same than the one
-   in the field */
-
-static int get_uint64_equal(QEMUFile *f, void *pv, size_t size)
-{
-    uint64_t *v = pv;
-    uint64_t v2;
-    qemu_get_be64s(f, &v2);
-
-    if (*v == v2) {
-        return 0;
-    }
-    return -EINVAL;
-}
-
-const VMStateInfo vmstate_info_uint64_equal = {
-    .name = "int64 equal",
-    .get  = get_uint64_equal,
-    .put  = put_uint64,
-};
-
-/* 8 bit int. See that the received value is the same than the one
-   in the field */
-
-static int get_uint8_equal(QEMUFile *f, void *pv, size_t size)
-{
-    uint8_t *v = pv;
-    uint8_t v2;
-    qemu_get_8s(f, &v2);
-
-    if (*v == v2) {
-        return 0;
-    }
-    return -EINVAL;
-}
-
-const VMStateInfo vmstate_info_uint8_equal = {
-    .name = "uint8 equal",
-    .get  = get_uint8_equal,
-    .put  = put_uint8,
-};
-
-/* 16 bit unsigned int int. See that the received value is the same than the one
-   in the field */
-
-static int get_uint16_equal(QEMUFile *f, void *pv, size_t size)
-{
-    uint16_t *v = pv;
-    uint16_t v2;
-    qemu_get_be16s(f, &v2);
-
-    if (*v == v2) {
-        return 0;
-    }
-    return -EINVAL;
-}
-
-const VMStateInfo vmstate_info_uint16_equal = {
-    .name = "uint16 equal",
-    .get  = get_uint16_equal,
-    .put  = put_uint16,
-};
-
-/* floating point */
-
-static int get_float64(QEMUFile *f, void *pv, size_t size)
-{
-    float64 *v = pv;
-
-    *v = make_float64(qemu_get_be64(f));
-    return 0;
-}
-
-static void put_float64(QEMUFile *f, void *pv, size_t size)
-{
-    uint64_t *v = pv;
-
-    qemu_put_be64(f, float64_val(*v));
-}
-
-const VMStateInfo vmstate_info_float64 = {
-    .name = "float64",
-    .get  = get_float64,
-    .put  = put_float64,
-};
-
-/* uint8_t buffers */
-
-static int get_buffer(QEMUFile *f, void *pv, size_t size)
-{
-    uint8_t *v = pv;
-    qemu_get_buffer(f, v, size);
-    return 0;
-}
-
-static void put_buffer(QEMUFile *f, void *pv, size_t size)
-{
-    uint8_t *v = pv;
-    qemu_put_buffer(f, v, size);
-}
-
-const VMStateInfo vmstate_info_buffer = {
-    .name = "buffer",
-    .get  = get_buffer,
-    .put  = put_buffer,
-};
-
-/* unused buffers: space that was used for some fields that are
-   not useful anymore */
-
-static int get_unused_buffer(QEMUFile *f, void *pv, size_t size)
-{
-    uint8_t buf[1024];
-    int block_len;
-
-    while (size > 0) {
-        block_len = MIN(sizeof(buf), size);
-        size -= block_len;
-        qemu_get_buffer(f, buf, block_len);
-    }
-   return 0;
-}
-
-static void put_unused_buffer(QEMUFile *f, void *pv, size_t size)
-{
-    static const uint8_t buf[1024];
-    int block_len;
-
-    while (size > 0) {
-        block_len = MIN(sizeof(buf), size);
-        size -= block_len;
-        qemu_put_buffer(f, buf, block_len);
-    }
-}
-
-const VMStateInfo vmstate_info_unused_buffer = {
-    .name = "unused_buffer",
-    .get  = get_unused_buffer,
-    .put  = put_unused_buffer,
-};
-
-/* bitmaps (as defined by bitmap.h). Note that size here is the size
- * of the bitmap in bits. The on-the-wire format of a bitmap is 64
- * bit words with the bits in big endian order. The in-memory format
- * is an array of 'unsigned long', which may be either 32 or 64 bits.
- */
-/* This is the number of 64 bit words sent over the wire */
-#define BITS_TO_U64S(nr) DIV_ROUND_UP(nr, 64)
-static int get_bitmap(QEMUFile *f, void *pv, size_t size)
-{
-    unsigned long *bmp = pv;
-    int i, idx = 0;
-    for (i = 0; i < BITS_TO_U64S(size); i++) {
-        uint64_t w = qemu_get_be64(f);
-        bmp[idx++] = w;
-        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
-            bmp[idx++] = w >> 32;
-        }
-    }
-    return 0;
-}
-
-static void put_bitmap(QEMUFile *f, void *pv, size_t size)
-{
-    unsigned long *bmp = pv;
-    int i, idx = 0;
-    for (i = 0; i < BITS_TO_U64S(size); i++) {
-        uint64_t w = bmp[idx++];
-        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
-            w |= ((uint64_t)bmp[idx++]) << 32;
-        }
-        qemu_put_be64(f, w);
-    }
-}
-
-const VMStateInfo vmstate_info_bitmap = {
-    .name = "bitmap",
-    .get = get_bitmap,
-    .put = put_bitmap,
-};
diff --git a/xbzrle.c b/xbzrle.c
deleted file mode 100644
index 8e220bf..0000000
--- a/xbzrle.c
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Xor Based Zero Run Length Encoding
- *
- * Copyright 2013 Red Hat, Inc. and/or its affiliates
- *
- * Authors:
- *  Orit Wasserman  <owasserm@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- *
- */
-#include "qemu-common.h"
-#include "include/migration/migration.h"
-
-/*
-  page = zrun nzrun
-       | zrun nzrun page
-
-  zrun = length
-
-  nzrun = length byte...
-
-  length = uleb128 encoded integer
- */
-int xbzrle_encode_buffer(uint8_t *old_buf, uint8_t *new_buf, int slen,
-                         uint8_t *dst, int dlen)
-{
-    uint32_t zrun_len = 0, nzrun_len = 0;
-    int d = 0, i = 0;
-    long res;
-    uint8_t *nzrun_start = NULL;
-
-    g_assert(!(((uintptr_t)old_buf | (uintptr_t)new_buf | slen) %
-               sizeof(long)));
-
-    while (i < slen) {
-        /* overflow */
-        if (d + 2 > dlen) {
-            return -1;
-        }
-
-        /* not aligned to sizeof(long) */
-        res = (slen - i) % sizeof(long);
-        while (res && old_buf[i] == new_buf[i]) {
-            zrun_len++;
-            i++;
-            res--;
-        }
-
-        /* word at a time for speed */
-        if (!res) {
-            while (i < slen &&
-                   (*(long *)(old_buf + i)) == (*(long *)(new_buf + i))) {
-                i += sizeof(long);
-                zrun_len += sizeof(long);
-            }
-
-            /* go over the rest */
-            while (i < slen && old_buf[i] == new_buf[i]) {
-                zrun_len++;
-                i++;
-            }
-        }
-
-        /* buffer unchanged */
-        if (zrun_len == slen) {
-            return 0;
-        }
-
-        /* skip last zero run */
-        if (i == slen) {
-            return d;
-        }
-
-        d += uleb128_encode_small(dst + d, zrun_len);
-
-        zrun_len = 0;
-        nzrun_start = new_buf + i;
-
-        /* overflow */
-        if (d + 2 > dlen) {
-            return -1;
-        }
-        /* not aligned to sizeof(long) */
-        res = (slen - i) % sizeof(long);
-        while (res && old_buf[i] != new_buf[i]) {
-            i++;
-            nzrun_len++;
-            res--;
-        }
-
-        /* word at a time for speed, use of 32-bit long okay */
-        if (!res) {
-            /* truncation to 32-bit long okay */
-            unsigned long mask = (unsigned long)0x0101010101010101ULL;
-            while (i < slen) {
-                unsigned long xor;
-                xor = *(unsigned long *)(old_buf + i)
-                    ^ *(unsigned long *)(new_buf + i);
-                if ((xor - mask) & ~xor & (mask << 7)) {
-                    /* found the end of an nzrun within the current long */
-                    while (old_buf[i] != new_buf[i]) {
-                        nzrun_len++;
-                        i++;
-                    }
-                    break;
-                } else {
-                    i += sizeof(long);
-                    nzrun_len += sizeof(long);
-                }
-            }
-        }
-
-        d += uleb128_encode_small(dst + d, nzrun_len);
-        /* overflow */
-        if (d + nzrun_len > dlen) {
-            return -1;
-        }
-        memcpy(dst + d, nzrun_start, nzrun_len);
-        d += nzrun_len;
-        nzrun_len = 0;
-    }
-
-    return d;
-}
-
-int xbzrle_decode_buffer(uint8_t *src, int slen, uint8_t *dst, int dlen)
-{
-    int i = 0, d = 0;
-    int ret;
-    uint32_t count = 0;
-
-    while (i < slen) {
-
-        /* zrun */
-        if ((slen - i) < 2) {
-            return -1;
-        }
-
-        ret = uleb128_decode_small(src + i, &count);
-        if (ret < 0 || (i && !count)) {
-            return -1;
-        }
-        i += ret;
-        d += count;
-
-        /* overflow */
-        if (d > dlen) {
-            return -1;
-        }
-
-        /* nzrun */
-        if ((slen - i) < 2) {
-            return -1;
-        }
-
-        ret = uleb128_decode_small(src + i, &count);
-        if (ret < 0 || !count) {
-            return -1;
-        }
-        i += ret;
-
-        /* overflow */
-        if (d + count > dlen || i + count > slen) {
-            return -1;
-        }
-
-        memcpy(dst + d, src + i, count);
-        d += count;
-        i += count;
-    }
-
-    return d;
-}
diff --git a/xen-common-stub.c b/xen-common-stub.c
index 906f991..699c3f1 100644
--- a/xen-common-stub.c
+++ b/xen-common-stub.c
@@ -5,6 +5,7 @@
  * See the COPYING file in the top-level directory.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/xen/xen.h"
 
diff --git a/xen-common.c b/xen-common.c
index 56359ca..e641ad1 100644
--- a/xen-common.c
+++ b/xen-common.c
@@ -8,10 +8,12 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "hw/xen/xen_backend.h"
 #include "qmp-commands.h"
 #include "sysemu/char.h"
 #include "sysemu/accel.h"
+#include "migration/migration.h"
 
 //#define DEBUG_XEN
 
@@ -112,13 +114,23 @@
 
 static int xen_init(MachineState *ms)
 {
-    xen_xc = xen_xc_interface_open(0, 0, 0);
-    if (xen_xc == XC_HANDLER_INITIAL_VALUE) {
+    xen_xc = xc_interface_open(0, 0, 0);
+    if (xen_xc == NULL) {
         xen_be_printf(NULL, 0, "can't open xen interface\n");
         return -1;
     }
+    xen_fmem = xenforeignmemory_open(0, 0);
+    if (xen_fmem == NULL) {
+        xen_be_printf(NULL, 0, "can't open xen fmem interface\n");
+        xc_interface_close(xen_xc);
+        return -1;
+    }
     qemu_add_vm_change_state_handler(xen_change_state_handler, NULL);
 
+    global_state_set_optional();
+    savevm_skip_configuration();
+    savevm_skip_section_footers();
+
     return 0;
 }
 
diff --git a/xen-hvm-stub.c b/xen-hvm-stub.c
index 2d98696..c500325 100644
--- a/xen-hvm-stub.c
+++ b/xen-hvm-stub.c
@@ -8,6 +8,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
+#include "qemu/osdep.h"
 #include "qemu-common.h"
 #include "hw/xen/xen.h"
 #include "exec/memory.h"
@@ -30,11 +31,13 @@
 {
 }
 
-void xen_cmos_set_s3_resume(void *opaque, int irq, int level)
+int xen_is_pirq_msi(uint32_t msi_data)
 {
+    return 0;
 }
 
-void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
+void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
+                   Error **errp)
 {
 }
 
@@ -51,10 +54,8 @@
 {
 }
 
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory)
+void xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory)
 {
-    return 0;
 }
 
 void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
diff --git a/xen-hvm.c b/xen-hvm.c
index 7548794..2f348ed 100644
--- a/xen-hvm.c
+++ b/xen-hvm.c
@@ -8,15 +8,18 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include <sys/mman.h>
+#include "qemu/osdep.h"
 
+#include "cpu.h"
 #include "hw/pci/pci.h"
 #include "hw/i386/pc.h"
+#include "hw/i386/apic-msidef.h"
 #include "hw/xen/xen_common.h"
 #include "hw/xen/xen_backend.h"
 #include "qmp-commands.h"
 
 #include "sysemu/char.h"
+#include "qemu/error-report.h"
 #include "qemu/range.h"
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
@@ -64,17 +67,6 @@
 typedef struct shared_vmport_iopage shared_vmport_iopage_t;
 #endif
 
-#if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a
-static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
-{
-    return shared_page->vcpu_iodata[i].vp_eport;
-}
-static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
-{
-    return &shared_page->vcpu_iodata[vcpu].vp_ioreq;
-}
-#  define FMT_ioreq_size PRIx64
-#else
 static inline uint32_t xen_vcpu_eport(shared_iopage_t *shared_page, int i)
 {
     return shared_page->vcpu_ioreq[i].vp_eport;
@@ -83,11 +75,6 @@
 {
     return &shared_page->vcpu_ioreq[vcpu];
 }
-#  define FMT_ioreq_size "u"
-#endif
-#ifndef HVM_PARAM_BUFIOREQ_EVTCHN
-#define HVM_PARAM_BUFIOREQ_EVTCHN 26
-#endif
 
 #define BUFFER_IO_MAX_DELAY  100
 
@@ -101,6 +88,7 @@
 } XenPhysmap;
 
 typedef struct XenIOState {
+    ioservid_t ioservid;
     shared_iopage_t *shared_page;
     shared_vmport_iopage_t *shared_vmport_page;
     buffered_iopage_t *buffered_io_page;
@@ -111,12 +99,14 @@
     /* evtchn local port for buffered io */
     evtchn_port_t bufioreq_local_port;
     /* the evtchn fd for polling */
-    XenEvtchn xce_handle;
+    xenevtchn_handle *xce_handle;
     /* which vcpu we are serving */
     int send_vcpu;
 
     struct xs_handle *xenstore;
     MemoryListener memory_listener;
+    MemoryListener io_listener;
+    DeviceListener device_listener;
     QLIST_HEAD(, XenPhysmap) physmap;
     hwaddr free_phys_offset;
     const XenPhysmap *log_for_dirtybit;
@@ -156,9 +146,17 @@
     }
 }
 
+int xen_is_pirq_msi(uint32_t msi_data)
+{
+    /* If vector is 0, the msi is remapped into a pirq, passed as
+     * dest_id.
+     */
+    return ((msi_data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT) == 0;
+}
+
 void xen_hvm_inject_msi(uint64_t addr, uint32_t data)
 {
-    xen_xc_hvm_inject_msi(xen_xc, xen_domid, addr, data);
+    xc_hvm_inject_msi(xen_xc, xen_domid, addr, data);
 }
 
 static void xen_suspend_notifier(Notifier *notifier, void *data)
@@ -180,8 +178,7 @@
 
 /* Memory Ops */
 
-static void xen_ram_init(ram_addr_t *below_4g_mem_size,
-                         ram_addr_t *above_4g_mem_size,
+static void xen_ram_init(PCMachineState *pcms,
                          ram_addr_t ram_size, MemoryRegion **ram_memory_p)
 {
     MemoryRegion *sysmem = get_system_memory();
@@ -193,28 +190,31 @@
     /* Handle the machine opt max-ram-below-4g.  It is basically doing
      * min(xen limit, user limit).
      */
+    if (!user_lowmem) {
+        user_lowmem = HVM_BELOW_4G_RAM_END; /* default */
+    }
     if (HVM_BELOW_4G_RAM_END <= user_lowmem) {
         user_lowmem = HVM_BELOW_4G_RAM_END;
     }
 
     if (ram_size >= user_lowmem) {
-        *above_4g_mem_size = ram_size - user_lowmem;
-        *below_4g_mem_size = user_lowmem;
+        pcms->above_4g_mem_size = ram_size - user_lowmem;
+        pcms->below_4g_mem_size = user_lowmem;
     } else {
-        *above_4g_mem_size = 0;
-        *below_4g_mem_size = ram_size;
+        pcms->above_4g_mem_size = 0;
+        pcms->below_4g_mem_size = ram_size;
     }
-    if (!*above_4g_mem_size) {
+    if (!pcms->above_4g_mem_size) {
         block_len = ram_size;
     } else {
         /*
          * Xen does not allocate the memory continuously, it keeps a
          * hole of the size computed above or passed in.
          */
-        block_len = (1ULL << 32) + *above_4g_mem_size;
+        block_len = (1ULL << 32) + pcms->above_4g_mem_size;
     }
     memory_region_init_ram(&ram_memory, NULL, "xen.ram", block_len,
-                           &error_abort);
+                           &error_fatal);
     *ram_memory_p = &ram_memory;
     vmstate_register_ram_global(&ram_memory);
 
@@ -229,17 +229,18 @@
      */
     memory_region_init_alias(&ram_lo, NULL, "xen.ram.lo",
                              &ram_memory, 0xc0000,
-                             *below_4g_mem_size - 0xc0000);
+                             pcms->below_4g_mem_size - 0xc0000);
     memory_region_add_subregion(sysmem, 0xc0000, &ram_lo);
-    if (*above_4g_mem_size > 0) {
+    if (pcms->above_4g_mem_size > 0) {
         memory_region_init_alias(&ram_hi, NULL, "xen.ram.hi",
                                  &ram_memory, 0x100000000ULL,
-                                 *above_4g_mem_size);
+                                 pcms->above_4g_mem_size);
         memory_region_add_subregion(sysmem, 0x100000000ULL, &ram_hi);
     }
 }
 
-void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
+void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr,
+                   Error **errp)
 {
     unsigned long nr_pfn;
     xen_pfn_t *pfn_list;
@@ -267,7 +268,8 @@
     }
 
     if (xc_domain_populate_physmap_exact(xen_xc, xen_domid, nr_pfn, 0, 0, pfn_list)) {
-        hw_error("xen: failed to populate ram at " RAM_ADDR_FMT, ram_addr);
+        error_setg(errp, "xen: failed to populate ram at " RAM_ADDR_FMT,
+                   ram_addr);
     }
 
     g_free(pfn_list);
@@ -304,7 +306,6 @@
     return start_addr;
 }
 
-#if CONFIG_XEN_CTRL_INTERFACE_VERSION >= 340
 static int xen_add_to_physmap(XenIOState *state,
                               hwaddr start_addr,
                               ram_addr_t size,
@@ -345,10 +346,10 @@
         unsigned long idx = pfn + i;
         xen_pfn_t gpfn = start_gpfn + i;
 
-        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
+        rc = xen_xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
         if (rc) {
             DPRINTF("add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
-                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
+                    PRI_xen_pfn" failed: %d (errno: %d)\n", idx, gpfn, rc, errno);
             return -rc;
         }
     }
@@ -422,10 +423,10 @@
         xen_pfn_t idx = start_addr + i;
         xen_pfn_t gpfn = phys_offset + i;
 
-        rc = xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
+        rc = xen_xc_domain_add_to_physmap(xen_xc, xen_domid, XENMAPSPACE_gmfn, idx, gpfn);
         if (rc) {
             fprintf(stderr, "add_to_physmap MFN %"PRI_xen_pfn" to PFN %"
-                    PRI_xen_pfn" failed: %d\n", idx, gpfn, rc);
+                    PRI_xen_pfn" failed: %d (errno: %d)\n", idx, gpfn, rc, errno);
             return -rc;
         }
     }
@@ -439,24 +440,6 @@
     return 0;
 }
 
-#else
-static int xen_add_to_physmap(XenIOState *state,
-                              hwaddr start_addr,
-                              ram_addr_t size,
-                              MemoryRegion *mr,
-                              hwaddr offset_within_region)
-{
-    return -ENOSYS;
-}
-
-static int xen_remove_from_physmap(XenIOState *state,
-                                   hwaddr start_addr,
-                                   ram_addr_t size)
-{
-    return -ENOSYS;
-}
-#endif
-
 static void xen_set_memory(struct MemoryListener *listener,
                            MemoryRegionSection *section,
                            bool add)
@@ -464,15 +447,26 @@
     XenIOState *state = container_of(listener, XenIOState, memory_listener);
     hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = int128_get64(section->size);
-    bool log_dirty = memory_region_is_logging(section->mr);
+    bool log_dirty = memory_region_is_logging(section->mr, DIRTY_MEMORY_VGA);
     hvmmem_type_t mem_type;
 
+    if (section->mr == &ram_memory) {
+        return;
+    } else {
+        if (add) {
+            xen_map_memory_section(xen_xc, xen_domid, state->ioservid,
+                                   section);
+        } else {
+            xen_unmap_memory_section(xen_xc, xen_domid, state->ioservid,
+                                     section);
+        }
+    }
+
     if (!memory_region_is_ram(section->mr)) {
         return;
     }
 
-    if (!(section->mr != &ram_memory
-          && ( (log_dirty && add) || (!log_dirty && !add)))) {
+    if (log_dirty != add) {
         return;
     }
 
@@ -515,13 +509,67 @@
     memory_region_unref(section->mr);
 }
 
+static void xen_io_add(MemoryListener *listener,
+                       MemoryRegionSection *section)
+{
+    XenIOState *state = container_of(listener, XenIOState, io_listener);
+    MemoryRegion *mr = section->mr;
+
+    if (mr->ops == &unassigned_io_ops) {
+        return;
+    }
+
+    memory_region_ref(mr);
+
+    xen_map_io_section(xen_xc, xen_domid, state->ioservid, section);
+}
+
+static void xen_io_del(MemoryListener *listener,
+                       MemoryRegionSection *section)
+{
+    XenIOState *state = container_of(listener, XenIOState, io_listener);
+    MemoryRegion *mr = section->mr;
+
+    if (mr->ops == &unassigned_io_ops) {
+        return;
+    }
+
+    xen_unmap_io_section(xen_xc, xen_domid, state->ioservid, section);
+
+    memory_region_unref(mr);
+}
+
+static void xen_device_realize(DeviceListener *listener,
+			       DeviceState *dev)
+{
+    XenIOState *state = container_of(listener, XenIOState, device_listener);
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+        PCIDevice *pci_dev = PCI_DEVICE(dev);
+
+        xen_map_pcidev(xen_xc, xen_domid, state->ioservid, pci_dev);
+    }
+}
+
+static void xen_device_unrealize(DeviceListener *listener,
+				 DeviceState *dev)
+{
+    XenIOState *state = container_of(listener, XenIOState, device_listener);
+
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+        PCIDevice *pci_dev = PCI_DEVICE(dev);
+
+        xen_unmap_pcidev(xen_xc, xen_domid, state->ioservid, pci_dev);
+    }
+}
+
 static void xen_sync_dirty_bitmap(XenIOState *state,
                                   hwaddr start_addr,
                                   ram_addr_t size)
 {
     hwaddr npages = size >> TARGET_PAGE_BITS;
     const int width = sizeof(unsigned long) * 8;
-    unsigned long bitmap[(npages + width - 1) / width];
+    unsigned long bitmap[DIV_ROUND_UP(npages, width)];
     int rc, i, j;
     const XenPhysmap *physmap = NULL;
 
@@ -567,21 +615,27 @@
 }
 
 static void xen_log_start(MemoryListener *listener,
-                          MemoryRegionSection *section)
+                          MemoryRegionSection *section,
+                          int old, int new)
 {
     XenIOState *state = container_of(listener, XenIOState, memory_listener);
 
-    xen_sync_dirty_bitmap(state, section->offset_within_address_space,
-                          int128_get64(section->size));
+    if (new & ~old & (1 << DIRTY_MEMORY_VGA)) {
+        xen_sync_dirty_bitmap(state, section->offset_within_address_space,
+                              int128_get64(section->size));
+    }
 }
 
-static void xen_log_stop(MemoryListener *listener, MemoryRegionSection *section)
+static void xen_log_stop(MemoryListener *listener, MemoryRegionSection *section,
+                         int old, int new)
 {
     XenIOState *state = container_of(listener, XenIOState, memory_listener);
 
-    state->log_for_dirtybit = NULL;
-    /* Disable dirty bit tracking */
-    xc_hvm_track_dirty_vram(xen_xc, xen_domid, 0, 0, NULL);
+    if (old & ~new & (1 << DIRTY_MEMORY_VGA)) {
+        state->log_for_dirtybit = NULL;
+        /* Disable dirty bit tracking */
+        xc_hvm_track_dirty_vram(xen_xc, xen_domid, 0, 0, NULL);
+    }
 }
 
 static void xen_log_sync(MemoryListener *listener, MemoryRegionSection *section)
@@ -615,6 +669,17 @@
     .priority = 10,
 };
 
+static MemoryListener xen_io_listener = {
+    .region_add = xen_io_add,
+    .region_del = xen_io_del,
+    .priority = 10,
+};
+
+static DeviceListener xen_device_listener = {
+    .realize = xen_device_realize,
+    .unrealize = xen_device_unrealize,
+};
+
 /* get the ioreq packets from share mem */
 static ioreq_t *cpu_get_ioreq_from_shared_memory(XenIOState *state, int vcpu)
 {
@@ -623,7 +688,7 @@
     if (req->state != STATE_IOREQ_READY) {
         DPRINTF("I/O request not ready: "
                 "%x, ptr: %x, port: %"PRIx64", "
-                "data: %"PRIx64", count: %" FMT_ioreq_size ", size: %" FMT_ioreq_size "\n",
+                "data: %"PRIx64", count: %u, size: %u\n",
                 req->state, req->data_is_ptr, req->addr,
                 req->data, req->count, req->size);
         return NULL;
@@ -643,7 +708,7 @@
     int i;
     evtchn_port_t port;
 
-    port = xc_evtchn_pending(state->xce_handle);
+    port = xenevtchn_pending(state->xce_handle);
     if (port == state->bufioreq_local_port) {
         timer_mod(state->buffered_io_timer,
                 BUFFER_IO_MAX_DELAY + qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
@@ -662,7 +727,7 @@
         }
 
         /* unmask the wanted port again */
-        xc_evtchn_unmask(state->xce_handle, port);
+        xenevtchn_unmask(state->xce_handle, port);
 
         /* get the io packet from shared memory */
         state->send_vcpu = i;
@@ -673,7 +738,7 @@
     return NULL;
 }
 
-static uint32_t do_inp(pio_addr_t addr, unsigned long size)
+static uint32_t do_inp(uint32_t addr, unsigned long size)
 {
     switch (size) {
         case 1:
@@ -683,11 +748,11 @@
         case 4:
             return cpu_inl(addr);
         default:
-            hw_error("inp: bad size: %04"FMT_pioaddr" %lx", addr, size);
+            hw_error("inp: bad size: %04x %lx", addr, size);
     }
 }
 
-static void do_outp(pio_addr_t addr,
+static void do_outp(uint32_t addr,
         unsigned long size, uint32_t val)
 {
     switch (size) {
@@ -698,7 +763,7 @@
         case 4:
             return cpu_outl(addr, val);
         default:
-            hw_error("outp: bad size: %04"FMT_pioaddr" %lx", addr, size);
+            hw_error("outp: bad size: %04x %lx", addr, size);
     }
 }
 
@@ -742,9 +807,14 @@
 {
     uint32_t i;
 
+    trace_cpu_ioreq_pio(req, req->dir, req->df, req->data_is_ptr, req->addr,
+                         req->data, req->count, req->size);
+
     if (req->dir == IOREQ_READ) {
         if (!req->data_is_ptr) {
             req->data = do_inp(req->addr, req->size);
+            trace_cpu_ioreq_pio_read_reg(req, req->data, req->addr,
+                                         req->size);
         } else {
             uint32_t tmp;
 
@@ -755,6 +825,8 @@
         }
     } else if (req->dir == IOREQ_WRITE) {
         if (!req->data_is_ptr) {
+            trace_cpu_ioreq_pio_write_reg(req, req->data, req->addr,
+                                          req->size);
             do_outp(req->addr, req->size, req->data);
         } else {
             for (i = 0; i < req->count; i++) {
@@ -771,6 +843,9 @@
 {
     uint32_t i;
 
+    trace_cpu_ioreq_move(req, req->dir, req->df, req->data_is_ptr, req->addr,
+                         req->data, req->count, req->size);
+
     if (!req->data_is_ptr) {
         if (req->dir == IOREQ_READ) {
             for (i = 0; i < req->count; i++) {
@@ -843,11 +918,18 @@
 
 static void handle_ioreq(XenIOState *state, ioreq_t *req)
 {
+    trace_handle_ioreq(req, req->type, req->dir, req->df, req->data_is_ptr,
+                       req->addr, req->data, req->count, req->size);
+
     if (!req->data_is_ptr && (req->dir == IOREQ_WRITE) &&
             (req->size < sizeof (target_ulong))) {
         req->data &= ((target_ulong) 1 << (8 * req->size)) - 1;
     }
 
+    if (req->dir == IOREQ_WRITE)
+        trace_handle_ioreq_write(req, req->type, req->df, req->data_is_ptr,
+                                 req->addr, req->data, req->count, req->size);
+
     switch (req->type) {
         case IOREQ_TYPE_PIO:
             cpu_ioreq_pio(req);
@@ -863,26 +945,62 @@
         case IOREQ_TYPE_INVALIDATE:
             xen_invalidate_map_cache();
             break;
+        case IOREQ_TYPE_PCI_CONFIG: {
+            uint32_t sbdf = req->addr >> 32;
+            uint32_t val;
+
+            /* Fake a write to port 0xCF8 so that
+             * the config space access will target the
+             * correct device model.
+             */
+            val = (1u << 31) |
+                  ((req->addr & 0x0f00) << 16) |
+                  ((sbdf & 0xffff) << 8) |
+                  (req->addr & 0xfc);
+            do_outp(0xcf8, 4, val);
+
+            /* Now issue the config space access via
+             * port 0xCFC
+             */
+            req->addr = 0xcfc | (req->addr & 0x03);
+            cpu_ioreq_pio(req);
+            break;
+        }
         default:
             hw_error("Invalid ioreq type 0x%x\n", req->type);
     }
+    if (req->dir == IOREQ_READ) {
+        trace_handle_ioreq_read(req, req->type, req->df, req->data_is_ptr,
+                                req->addr, req->data, req->count, req->size);
+    }
 }
 
 static int handle_buffered_iopage(XenIOState *state)
 {
+    buffered_iopage_t *buf_page = state->buffered_io_page;
     buf_ioreq_t *buf_req = NULL;
     ioreq_t req;
     int qw;
 
-    if (!state->buffered_io_page) {
+    if (!buf_page) {
         return 0;
     }
 
     memset(&req, 0x00, sizeof(req));
 
-    while (state->buffered_io_page->read_pointer != state->buffered_io_page->write_pointer) {
-        buf_req = &state->buffered_io_page->buf_ioreq[
-            state->buffered_io_page->read_pointer % IOREQ_BUFFER_SLOT_NUM];
+    for (;;) {
+        uint32_t rdptr = buf_page->read_pointer, wrptr;
+
+        xen_rmb();
+        wrptr = buf_page->write_pointer;
+        xen_rmb();
+        if (rdptr != buf_page->read_pointer) {
+            continue;
+        }
+        if (rdptr == wrptr) {
+            break;
+        }
+        buf_req = &buf_page->buf_ioreq[rdptr % IOREQ_BUFFER_SLOT_NUM];
         req.size = 1UL << buf_req->size;
         req.count = 1;
         req.addr = buf_req->addr;
@@ -894,15 +1012,14 @@
         req.data_is_ptr = 0;
         qw = (req.size == 8);
         if (qw) {
-            buf_req = &state->buffered_io_page->buf_ioreq[
-                (state->buffered_io_page->read_pointer + 1) % IOREQ_BUFFER_SLOT_NUM];
+            buf_req = &buf_page->buf_ioreq[(rdptr + 1) %
+                                           IOREQ_BUFFER_SLOT_NUM];
             req.data |= ((uint64_t)buf_req->data) << 32;
         }
 
         handle_ioreq(state, &req);
 
-        xen_mb();
-        state->buffered_io_page->read_pointer += qw ? 2 : 1;
+        atomic_add(&buf_page->read_pointer, qw + 1);
     }
 
     return req.count;
@@ -917,7 +1034,7 @@
                 BUFFER_IO_MAX_DELAY + qemu_clock_get_ms(QEMU_CLOCK_REALTIME));
     } else {
         timer_del(state->buffered_io_timer);
-        xc_evtchn_unmask(state->xce_handle, state->bufioreq_local_port);
+        xenevtchn_unmask(state->xce_handle, state->bufioreq_local_port);
     }
 }
 
@@ -933,9 +1050,7 @@
         if (req->state != STATE_IOREQ_INPROCESS) {
             fprintf(stderr, "Badness in I/O request ... not in service?!: "
                     "%x, ptr: %x, port: %"PRIx64", "
-                    "data: %"PRIx64", count: %" FMT_ioreq_size
-                    ", size: %" FMT_ioreq_size
-                    ", type: %"FMT_ioreq_size"\n",
+                    "data: %"PRIx64", count: %u, size: %u, type: %u\n",
                     req->state, req->data_is_ptr, req->addr,
                     req->data, req->count, req->size, req->type);
             destroy_hvm_domain(false);
@@ -961,7 +1076,8 @@
         }
 
         req->state = STATE_IORESP_READY;
-        xc_evtchn_notify(state->xce_handle, state->ioreq_local_port[state->send_vcpu]);
+        xenevtchn_notify(state->xce_handle,
+                         state->ioreq_local_port[state->send_vcpu]);
     }
 }
 
@@ -969,8 +1085,8 @@
 {
     int evtchn_fd = -1;
 
-    if (state->xce_handle != XC_HANDLER_INITIAL_VALUE) {
-        evtchn_fd = xc_evtchn_fd(state->xce_handle);
+    if (state->xce_handle != NULL) {
+        evtchn_fd = xenevtchn_fd(state->xce_handle);
     }
 
     state->buffered_io_timer = timer_new_ms(QEMU_CLOCK_REALTIME, handle_buffered_io,
@@ -993,16 +1109,22 @@
 static void xen_hvm_change_state_handler(void *opaque, int running,
                                          RunState rstate)
 {
+    XenIOState *state = opaque;
+
     if (running) {
-        xen_main_loop_prepare((XenIOState *)opaque);
+        xen_main_loop_prepare(state);
     }
+
+    xen_set_ioreq_server_state(xen_xc, xen_domid,
+                               state->ioservid,
+                               (rstate == RUN_STATE_RUNNING));
 }
 
 static void xen_exit_notifier(Notifier *n, void *data)
 {
     XenIOState *state = container_of(n, XenIOState, exit);
 
-    xc_evtchn_close(state->xce_handle);
+    xenevtchn_close(state->xce_handle);
     xs_daemon_close(state->xenstore);
 }
 
@@ -1059,29 +1181,30 @@
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
 }
 
-/* return 0 means OK, or -1 means critical issue -- will exit(1) */
-int xen_hvm_init(ram_addr_t *below_4g_mem_size, ram_addr_t *above_4g_mem_size,
-                 MemoryRegion **ram_memory)
+void xen_hvm_init(PCMachineState *pcms, MemoryRegion **ram_memory)
 {
     int i, rc;
-    unsigned long ioreq_pfn;
-    unsigned long bufioreq_evtchn;
+    xen_pfn_t ioreq_pfn;
+    xen_pfn_t bufioreq_pfn;
+    evtchn_port_t bufioreq_evtchn;
     XenIOState *state;
 
     state = g_malloc0(sizeof (XenIOState));
 
-    state->xce_handle = xen_xc_evtchn_open(NULL, 0);
-    if (state->xce_handle == XC_HANDLER_INITIAL_VALUE) {
+    state->xce_handle = xenevtchn_open(NULL, 0);
+    if (state->xce_handle == NULL) {
         perror("xen: event channel open");
-        return -1;
+        goto err;
     }
 
     state->xenstore = xs_daemon_open();
     if (state->xenstore == NULL) {
         perror("xen: xenstore open");
-        return -1;
+        goto err;
     }
 
+    xen_create_ioreq_server(xen_xc, xen_domid, &state->ioservid);
+
     state->exit.notify = xen_exit_notifier;
     qemu_add_exit_notifier(&state->exit);
 
@@ -1091,70 +1214,87 @@
     state->wakeup.notify = xen_wakeup_notifier;
     qemu_register_wakeup_notifier(&state->wakeup);
 
-    xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
+    rc = xen_get_ioreq_server_info(xen_xc, xen_domid, state->ioservid,
+                                   &ioreq_pfn, &bufioreq_pfn,
+                                   &bufioreq_evtchn);
+    if (rc < 0) {
+        error_report("failed to get ioreq server info: error %d handle=%p",
+                     errno, xen_xc);
+        goto err;
+    }
+
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
-    state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-                                              PROT_READ|PROT_WRITE, ioreq_pfn);
+    DPRINTF("buffered io page at pfn %lx\n", bufioreq_pfn);
+    DPRINTF("buffered io evtchn is %x\n", bufioreq_evtchn);
+
+    state->shared_page = xenforeignmemory_map(xen_fmem, xen_domid,
+                                              PROT_READ|PROT_WRITE,
+                                              1, &ioreq_pfn, NULL);
     if (state->shared_page == NULL) {
-        hw_error("map shared IO page returned error %d handle=" XC_INTERFACE_FMT,
-                 errno, xen_xc);
+        error_report("map shared IO page returned error %d handle=%p",
+                     errno, xen_xc);
+        goto err;
     }
 
     rc = xen_get_vmport_regs_pfn(xen_xc, xen_domid, &ioreq_pfn);
     if (!rc) {
         DPRINTF("shared vmport page at pfn %lx\n", ioreq_pfn);
         state->shared_vmport_page =
-            xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-                                 PROT_READ|PROT_WRITE, ioreq_pfn);
+            xenforeignmemory_map(xen_fmem, xen_domid, PROT_READ|PROT_WRITE,
+                                 1, &ioreq_pfn, NULL);
         if (state->shared_vmport_page == NULL) {
-            hw_error("map shared vmport IO page returned error %d handle="
-                     XC_INTERFACE_FMT, errno, xen_xc);
+            error_report("map shared vmport IO page returned error %d handle=%p",
+                         errno, xen_xc);
+            goto err;
         }
     } else if (rc != -ENOSYS) {
-        hw_error("get vmport regs pfn returned error %d, rc=%d", errno, rc);
+        error_report("get vmport regs pfn returned error %d, rc=%d",
+                     errno, rc);
+        goto err;
     }
 
-    xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_BUFIOREQ_PFN, &ioreq_pfn);
-    DPRINTF("buffered io page at pfn %lx\n", ioreq_pfn);
-    state->buffered_io_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-                                                   PROT_READ|PROT_WRITE, ioreq_pfn);
+    state->buffered_io_page = xenforeignmemory_map(xen_fmem, xen_domid,
+                                                   PROT_READ|PROT_WRITE,
+                                                   1, &bufioreq_pfn, NULL);
     if (state->buffered_io_page == NULL) {
-        hw_error("map buffered IO page returned error %d", errno);
+        error_report("map buffered IO page returned error %d", errno);
+        goto err;
     }
 
     /* Note: cpus is empty at this point in init */
     state->cpu_by_vcpu_id = g_malloc0(max_cpus * sizeof(CPUState *));
 
+    rc = xen_set_ioreq_server_state(xen_xc, xen_domid, state->ioservid, true);
+    if (rc < 0) {
+        error_report("failed to enable ioreq server info: error %d handle=%p",
+                     errno, xen_xc);
+        goto err;
+    }
+
     state->ioreq_local_port = g_malloc0(max_cpus * sizeof (evtchn_port_t));
 
     /* FIXME: how about if we overflow the page here? */
     for (i = 0; i < max_cpus; i++) {
-        rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
+        rc = xenevtchn_bind_interdomain(state->xce_handle, xen_domid,
                                         xen_vcpu_eport(state->shared_page, i));
         if (rc == -1) {
-            fprintf(stderr, "bind interdomain ioctl error %d\n", errno);
-            return -1;
+            error_report("shared evtchn %d bind error %d", i, errno);
+            goto err;
         }
         state->ioreq_local_port[i] = rc;
     }
 
-    rc = xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_BUFIOREQ_EVTCHN,
-            &bufioreq_evtchn);
-    if (rc < 0) {
-        fprintf(stderr, "failed to get HVM_PARAM_BUFIOREQ_EVTCHN\n");
-        return -1;
-    }
-    rc = xc_evtchn_bind_interdomain(state->xce_handle, xen_domid,
-            (uint32_t)bufioreq_evtchn);
+    rc = xenevtchn_bind_interdomain(state->xce_handle, xen_domid,
+                                    bufioreq_evtchn);
     if (rc == -1) {
-        fprintf(stderr, "bind interdomain ioctl error %d\n", errno);
-        return -1;
+        error_report("buffered evtchn bind error %d", errno);
+        goto err;
     }
     state->bufioreq_local_port = rc;
 
     /* Init RAM management */
     xen_map_cache_init(xen_phys_offset_to_gaddr, state);
-    xen_ram_init(below_4g_mem_size, above_4g_mem_size, ram_size, ram_memory);
+    xen_ram_init(pcms, ram_size, ram_memory);
 
     qemu_add_vm_change_state_handler(xen_hvm_change_state_handler, state);
 
@@ -1163,26 +1303,33 @@
     memory_listener_register(&state->memory_listener, &address_space_memory);
     state->log_for_dirtybit = NULL;
 
+    state->io_listener = xen_io_listener;
+    memory_listener_register(&state->io_listener, &address_space_io);
+
+    state->device_listener = xen_device_listener;
+    device_listener_register(&state->device_listener);
+
     /* Initialize backend core & drivers */
     if (xen_be_init() != 0) {
-        fprintf(stderr, "%s: xen backend core setup failed\n", __FUNCTION__);
-        return -1;
+        error_report("xen backend core setup failed");
+        goto err;
     }
-    xen_be_register("console", &xen_console_ops);
-    xen_be_register("vkbd", &xen_kbdmouse_ops);
-    xen_be_register("qdisk", &xen_blkdev_ops);
+    xen_be_register_common();
     xen_read_physmap(state);
+    return;
 
-    return 0;
+err:
+    error_report("xen hardware virtual machine initialisation failed");
+    exit(1);
 }
 
 void destroy_hvm_domain(bool reboot)
 {
-    XenXC xc_handle;
+    xc_interface *xc_handle;
     int sts;
 
-    xc_handle = xen_xc_interface_open(0, 0, 0);
-    if (xc_handle == XC_HANDLER_INITIAL_VALUE) {
+    xc_handle = xc_interface_open(0, 0, 0);
+    if (xc_handle == NULL) {
         fprintf(stderr, "Cannot acquire xenctrl handle\n");
     } else {
         sts = xc_domain_shutdown(xc_handle, xen_domid,
diff --git a/xen-mapcache.c b/xen-mapcache.c
index 66da1a6..8f3a592 100644
--- a/xen-mapcache.c
+++ b/xen-mapcache.c
@@ -8,7 +8,7 @@
  * GNU GPL, version 2 or (at your option) any later version.
  */
 
-#include "config.h"
+#include "qemu/osdep.h"
 
 #include <sys/resource.h>
 
@@ -17,7 +17,6 @@
 #include "qemu/bitmap.h"
 
 #include <xen/hvm/params.h>
-#include <sys/mman.h>
 
 #include "sysemu/xen-mapcache.h"
 #include "trace.h"
@@ -49,9 +48,6 @@
  */
 #define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
 
-#define mapcache_lock()   ((void)0)
-#define mapcache_unlock() ((void)0)
-
 typedef struct MapCacheEntry {
     hwaddr paddr_index;
     uint8_t *vaddr_base;
@@ -79,11 +75,22 @@
     unsigned int mcache_bucket_shift;
 
     phys_offset_to_gaddr_t phys_offset_to_gaddr;
+    QemuMutex lock;
     void *opaque;
 } MapCache;
 
 static MapCache *mapcache;
 
+static inline void mapcache_lock(void)
+{
+    qemu_mutex_lock(&mapcache->lock);
+}
+
+static inline void mapcache_unlock(void)
+{
+    qemu_mutex_unlock(&mapcache->lock);
+}
+
 static inline int test_bits(int nr, int size, const unsigned long *addr)
 {
     unsigned long res = find_next_zero_bit(addr, size + nr, nr);
@@ -102,6 +109,7 @@
 
     mapcache->phys_offset_to_gaddr = f;
     mapcache->opaque = opaque;
+    qemu_mutex_init(&mapcache->lock);
 
     QTAILQ_INIT(&mapcache->locked_entries);
 
@@ -160,19 +168,17 @@
             exit(-1);
         }
     }
-    if (entry->valid_mapping != NULL) {
-        g_free(entry->valid_mapping);
-        entry->valid_mapping = NULL;
-    }
+    g_free(entry->valid_mapping);
+    entry->valid_mapping = NULL;
 
     for (i = 0; i < nb_pfn; i++) {
         pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
     }
 
-    vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
-                                     pfns, err, nb_pfn);
+    vaddr_base = xenforeignmemory_map(xen_fmem, xen_domid, PROT_READ|PROT_WRITE,
+                                      nb_pfn, pfns, err);
     if (vaddr_base == NULL) {
-        perror("xc_map_foreign_bulk");
+        perror("xenforeignmemory_map");
         exit(-1);
     }
 
@@ -193,14 +199,14 @@
     g_free(err);
 }
 
-uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
-                       uint8_t lock)
+static uint8_t *xen_map_cache_unlocked(hwaddr phys_addr, hwaddr size,
+                                       uint8_t lock)
 {
     MapCacheEntry *entry, *pentry = NULL;
     hwaddr address_index;
     hwaddr address_offset;
-    hwaddr __size = size;
-    hwaddr __test_bit_size = size;
+    hwaddr cache_size = size;
+    hwaddr test_bit_size;
     bool translated = false;
 
 tryagain:
@@ -209,22 +215,22 @@
 
     trace_xen_map_cache(phys_addr);
 
-    /* __test_bit_size is always a multiple of XC_PAGE_SIZE */
+    /* test_bit_size is always a multiple of XC_PAGE_SIZE */
     if (size) {
-        __test_bit_size = size + (phys_addr & (XC_PAGE_SIZE - 1));
+        test_bit_size = size + (phys_addr & (XC_PAGE_SIZE - 1));
 
-        if (__test_bit_size % XC_PAGE_SIZE) {
-            __test_bit_size += XC_PAGE_SIZE - (__test_bit_size % XC_PAGE_SIZE);
+        if (test_bit_size % XC_PAGE_SIZE) {
+            test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE);
         }
     } else {
-        __test_bit_size = XC_PAGE_SIZE;
+        test_bit_size = XC_PAGE_SIZE;
     }
 
     if (mapcache->last_entry != NULL &&
         mapcache->last_entry->paddr_index == address_index &&
-        !lock && !__size &&
+        !lock && !size &&
         test_bits(address_offset >> XC_PAGE_SHIFT,
-                  __test_bit_size >> XC_PAGE_SHIFT,
+                  test_bit_size >> XC_PAGE_SHIFT,
                   mapcache->last_entry->valid_mapping)) {
         trace_xen_map_cache_return(mapcache->last_entry->vaddr_base + address_offset);
         return mapcache->last_entry->vaddr_base + address_offset;
@@ -232,20 +238,20 @@
 
     /* size is always a multiple of MCACHE_BUCKET_SIZE */
     if (size) {
-        __size = size + address_offset;
-        if (__size % MCACHE_BUCKET_SIZE) {
-            __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
+        cache_size = size + address_offset;
+        if (cache_size % MCACHE_BUCKET_SIZE) {
+            cache_size += MCACHE_BUCKET_SIZE - (cache_size % MCACHE_BUCKET_SIZE);
         }
     } else {
-        __size = MCACHE_BUCKET_SIZE;
+        cache_size = MCACHE_BUCKET_SIZE;
     }
 
     entry = &mapcache->entry[address_index % mapcache->nr_buckets];
 
     while (entry && entry->lock && entry->vaddr_base &&
-            (entry->paddr_index != address_index || entry->size != __size ||
+            (entry->paddr_index != address_index || entry->size != cache_size ||
              !test_bits(address_offset >> XC_PAGE_SHIFT,
-                 __test_bit_size >> XC_PAGE_SHIFT,
+                 test_bit_size >> XC_PAGE_SHIFT,
                  entry->valid_mapping))) {
         pentry = entry;
         entry = entry->next;
@@ -253,19 +259,19 @@
     if (!entry) {
         entry = g_malloc0(sizeof (MapCacheEntry));
         pentry->next = entry;
-        xen_remap_bucket(entry, __size, address_index);
+        xen_remap_bucket(entry, cache_size, address_index);
     } else if (!entry->lock) {
         if (!entry->vaddr_base || entry->paddr_index != address_index ||
-                entry->size != __size ||
+                entry->size != cache_size ||
                 !test_bits(address_offset >> XC_PAGE_SHIFT,
-                    __test_bit_size >> XC_PAGE_SHIFT,
+                    test_bit_size >> XC_PAGE_SHIFT,
                     entry->valid_mapping)) {
-            xen_remap_bucket(entry, __size, address_index);
+            xen_remap_bucket(entry, cache_size, address_index);
         }
     }
 
     if(!test_bits(address_offset >> XC_PAGE_SHIFT,
-                __test_bit_size >> XC_PAGE_SHIFT,
+                test_bit_size >> XC_PAGE_SHIFT,
                 entry->valid_mapping)) {
         mapcache->last_entry = NULL;
         if (!translated && mapcache->phys_offset_to_gaddr) {
@@ -291,14 +297,27 @@
     return mapcache->last_entry->vaddr_base + address_offset;
 }
 
+uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
+                       uint8_t lock)
+{
+    uint8_t *p;
+
+    mapcache_lock();
+    p = xen_map_cache_unlocked(phys_addr, size, lock);
+    mapcache_unlock();
+    return p;
+}
+
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
 {
     MapCacheEntry *entry = NULL;
     MapCacheRev *reventry;
     hwaddr paddr_index;
     hwaddr size;
+    ram_addr_t raddr;
     int found = 0;
 
+    mapcache_lock();
     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
         if (reventry->vaddr_req == ptr) {
             paddr_index = reventry->paddr_index;
@@ -323,13 +342,16 @@
     }
     if (!entry) {
         DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
-        return 0;
+        raddr = 0;
+    } else {
+        raddr = (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
+             ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
     }
-    return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
-        ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
+    mapcache_unlock();
+    return raddr;
 }
 
-void xen_invalidate_map_cache_entry(uint8_t *buffer)
+static void xen_invalidate_map_cache_entry_unlocked(uint8_t *buffer)
 {
     MapCacheEntry *entry = NULL, *pentry = NULL;
     MapCacheRev *reventry;
@@ -383,6 +405,13 @@
     g_free(entry);
 }
 
+void xen_invalidate_map_cache_entry(uint8_t *buffer)
+{
+    mapcache_lock();
+    xen_invalidate_map_cache_entry_unlocked(buffer);
+    mapcache_unlock();
+}
+
 void xen_invalidate_map_cache(void)
 {
     unsigned long i;
@@ -391,14 +420,14 @@
     /* Flush pending AIO before destroying the mapcache */
     bdrv_drain_all();
 
+    mapcache_lock();
+
     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
         DPRINTF("There should be no locked mappings at this time, "
                 "but "TARGET_FMT_plx" -> %p is present\n",
                 reventry->paddr_index, reventry->vaddr_req);
     }
 
-    mapcache_lock();
-
     for (i = 0; i < mapcache->nr_buckets; i++) {
         MapCacheEntry *entry = &mapcache->entry[i];